3157 files changed, 178948 insertions, 72076 deletions

diff --git a/.gitignore b/.gitignore
index a20ac26aa2f5..d263ca9a276c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -58,6 +58,7 @@ modules.builtin
 /vmlinuz
 /System.map
 /Module.markers
+/modules.builtin.modinfo
 
 #
 # RPM spec file (make rpm-pkg)
@@ -90,10 +91,10 @@ modules.builtin
 #
 # Generated include files
 #
-include/config
-include/generated
-include/ksym
-arch/*/include/generated
+/include/config/
+/include/generated/
+/include/ksym/
+/arch/*/include/generated/
 
 # stgit generated dirs
 patches-*
@@ -129,7 +130,12 @@ signing_key.x509
 x509.genkey
 
 # Kconfig presets
-all.config
+/all.config
+/alldef.config
+/allmod.config
+/allno.config
+/allrandom.config
+/allyes.config
 
 # Kdevelop4
 *.kdev4
diff --git a/.mailmap b/.mailmap
index 0049c0b49056..07a777f9d687 100644
--- a/.mailmap
+++ b/.mailmap
@@ -16,6 +16,8 @@ Alan Cox <alan@lxorguk.ukuu.org.uk>
 Alan Cox <root@hraefn.swansea.linux.org.uk>
 Aleksey Gorelov <aleksey_gorelov@phoenix.com>
 Aleksandar Markovic <aleksandar.markovic@mips.com> <aleksandar.markovic@imgtec.com>
+Alex Shi <alex.shi@linux.alibaba.com> <alex.shi@intel.com>
+Alex Shi <alex.shi@linux.alibaba.com> <alex.shi@linaro.org>
 Alexei Starovoitov <ast@kernel.org> <ast@plumgrid.com>
 Alexei Starovoitov <ast@kernel.org> <alexei.starovoitov@gmail.com>
 Alexei Starovoitov <ast@kernel.org> <ast@fb.com>
@@ -126,6 +128,8 @@ Leonid I Ananiev <leonid.i.ananiev@intel.com>
 Linas Vepstas <linas@austin.ibm.com>
 Linus Lüssing <linus.luessing@c0d3.blue> <linus.luessing@web.de>
 Linus Lüssing <linus.luessing@c0d3.blue> <linus.luessing@ascom.ch>
+Li Yang <leoyang.li@nxp.com> <leo@zh-kernel.org>
+Li Yang <leoyang.li@nxp.com> <leoli@freescale.com>
 Maciej W. Rozycki <macro@mips.com> <macro@imgtec.com>
 Marcin Nowakowski <marcin.nowakowski@mips.com> <marcin.nowakowski@imgtec.com>
 Mark Brown <broonie@sirena.org.uk>
@@ -217,6 +221,8 @@ Tejun Heo <htejun@gmail.com>
 Thomas Graf <tgraf@suug.ch>
 Thomas Pedersen <twp@codeaurora.org>
 Tony Luck <tony.luck@intel.com>
+TripleX Chung <xxx.phy@gmail.com> <zhongyu@18mail.cn>
+TripleX Chung <xxx.phy@gmail.com> <triplex@zh-kernel.org>
 Tsuneo Yoshioka <Tsuneo.Yoshioka@f-secure.com>
 Uwe Kleine-König <ukleinek@informatik.uni-freiburg.de>
 Uwe Kleine-König <ukl@pengutronix.de>
diff --git a/Documentation/ABI/testing/sysfs-bus-i2c-devices-pca954x b/Documentation/ABI/testing/sysfs-bus-i2c-devices-pca954x
new file mode 100644
index 000000000000..0b0de8cd0d13
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-bus-i2c-devices-pca954x
@@ -0,0 +1,20 @@
+What:		/sys/bus/i2c/.../idle_state
+Date:		January 2019
+KernelVersion:	5.2
+Contact:	Robert Shearman <robert.shearman@att.com>
+Description:
+		Value that exists only for mux devices that can be
+		written to control the behaviour of the multiplexer on
+		idle. Possible values:
+		-2 - disconnect on idle, i.e. deselect the last used
+		     channel, which is useful when there is a device
+		     with an address that conflicts with another
+		     device on another mux on the same parent bus.
+		-1 - leave the mux as-is, which is the most optimal
+		     setting in terms of I2C operations and is the
+		     default mode.
+		0..<nchans> - set the mux to a predetermined channel,
+		     which is useful if there is one channel that is
+		     used almost always, and you want to reduce the
+		     latency for normal operations after rare
+		     transactions on other channels
diff --git a/Documentation/ABI/testing/sysfs-devices-platform-ipmi b/Documentation/ABI/testing/sysfs-devices-platform-ipmi
index 2a781e7513b7..afb5db856e1c 100644
--- a/Documentation/ABI/testing/sysfs-devices-platform-ipmi
+++ b/Documentation/ABI/testing/sysfs-devices-platform-ipmi
@@ -212,7 +212,7 @@ Description:
 					Messages may be broken into parts if
 					they are long.
 
-		receieved_messages:	(RO) Number of message responses
+		received_messages:	(RO) Number of message responses
 					received.
 
 		received_message_parts: (RO) Number of message fragments
diff --git a/Documentation/ABI/testing/sysfs-driver-ucsi-ccg b/Documentation/ABI/testing/sysfs-driver-ucsi-ccg
new file mode 100644
index 000000000000..45cf62ad89e9
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-driver-ucsi-ccg
@@ -0,0 +1,6 @@
+What:		/sys/bus/i2c/drivers/ucsi_ccg/.../do_flash
+Date:		May 2019
+Contact:	Ajay Gupta <ajayg@nvidia.com>
+Description:
+		Tell the driver for Cypress CCGx Type-C controller to attempt
+		firmware upgrade by writing [Yy1] to the file.
diff --git a/Documentation/ABI/testing/sysfs-kernel-livepatch b/Documentation/ABI/testing/sysfs-kernel-livepatch
index 85db352f68f9..bea7bd5a1d5f 100644
--- a/Documentation/ABI/testing/sysfs-kernel-livepatch
+++ b/Documentation/ABI/testing/sysfs-kernel-livepatch
@@ -45,7 +45,7 @@ Description:
 		use this feature without a clearance from a patch
 		distributor. Removal (rmmod) of patch modules is permanently
 		disabled when the feature is used. See
-		Documentation/livepatch/livepatch.txt for more information.
+		Documentation/livepatch/livepatch.rst for more information.
 
 What:		/sys/kernel/livepatch/<patch>/<object>
 Date:		Nov 2014
diff --git a/Documentation/ABI/testing/usb-uevent b/Documentation/ABI/testing/usb-uevent
new file mode 100644
index 000000000000..d35c3cad892c
--- /dev/null
+++ b/Documentation/ABI/testing/usb-uevent
@@ -0,0 +1,27 @@
+What:		Raise a uevent when a USB Host Controller has died
+Date:		2019-04-17
+KernelVersion:	5.2
+Contact:	linux-usb@vger.kernel.org
+Description:	When the USB Host Controller has entered a state where it is no
+		longer functional a uevent will be raised. The uevent will
+		contain ACTION=offline and ERROR=DEAD.
+
+		Here is an example taken using udevadm monitor -p:
+
+		KERNEL[130.428945] offline  /devices/pci0000:00/0000:00:10.0/usb2 (usb)
+		ACTION=offline
+		BUSNUM=002
+		DEVNAME=/dev/bus/usb/002/001
+		DEVNUM=001
+		DEVPATH=/devices/pci0000:00/0000:00:10.0/usb2
+		DEVTYPE=usb_device
+		DRIVER=usb
+		ERROR=DEAD
+		MAJOR=189
+		MINOR=128
+		PRODUCT=1d6b/2/414
+		SEQNUM=2168
+		SUBSYSTEM=usb
+		TYPE=9/0/1
+
+Users:		chromium-os-dev@chromium.org
diff --git a/Documentation/DMA-API-HOWTO.txt b/Documentation/DMA-API-HOWTO.txt
index 1a721d0f35c8..cb712a02f59f 100644
--- a/Documentation/DMA-API-HOWTO.txt
+++ b/Documentation/DMA-API-HOWTO.txt
@@ -147,7 +147,7 @@ networking subsystems make sure that the buffers they use are valid
 for you to DMA from/to.
 
 DMA addressing capabilities
-==========================
+===========================
 
 By default, the kernel assumes that your device can address 32-bits of DMA
 addressing.  For a 64-bit capable device, this needs to be increased, and for
@@ -365,13 +365,12 @@ __get_free_pages() (but takes size instead of a page order).  If your
 driver needs regions sized smaller than a page, you may prefer using
 the dma_pool interface, described below.
 
-The consistent DMA mapping interfaces, for non-NULL dev, will by
-default return a DMA address which is 32-bit addressable.  Even if the
-device indicates (via DMA mask) that it may address the upper 32-bits,
-consistent allocation will only return > 32-bit addresses for DMA if
-the consistent DMA mask has been explicitly changed via
-dma_set_coherent_mask().  This is true of the dma_pool interface as
-well.
+The consistent DMA mapping interfaces, will by default return a DMA address
+which is 32-bit addressable.  Even if the device indicates (via the DMA mask)
+that it may address the upper 32-bits, consistent allocation will only
+return > 32-bit addresses for DMA if the consistent DMA mask has been
+explicitly changed via dma_set_coherent_mask().  This is true of the
+dma_pool interface as well.
 
 dma_alloc_coherent() returns two values: the virtual address which you
 can use to access it from the CPU and dma_handle which you pass to the
diff --git a/Documentation/Makefile b/Documentation/Makefile
index 9786957c6a35..e889e7cb8511 100644
--- a/Documentation/Makefile
+++ b/Documentation/Makefile
@@ -28,8 +28,13 @@ ifeq ($(HAVE_SPHINX),0)
 
 else # HAVE_SPHINX
 
-# User-friendly check for pdflatex
+# User-friendly check for pdflatex and latexmk
 HAVE_PDFLATEX := $(shell if which $(PDFLATEX) >/dev/null 2>&1; then echo 1; else echo 0; fi)
+HAVE_LATEXMK := $(shell if which latexmk >/dev/null 2>&1; then echo 1; else echo 0; fi)
+
+ifeq ($(HAVE_LATEXMK),1)
+	PDFLATEX := latexmk -$(PDFLATEX)
+endif #HAVE_LATEXMK
 
 # Internal variables.
 PAPEROPT_a4     = -D latex_paper_size=a4
@@ -82,7 +87,7 @@ pdfdocs:
 else # HAVE_PDFLATEX
 
 pdfdocs: latexdocs
-	$(foreach var,$(SPHINXDIRS), $(MAKE) PDFLATEX=$(PDFLATEX) LATEXOPTS="$(LATEXOPTS)" -C $(BUILDDIR)/$(var)/latex || exit;)
+	$(foreach var,$(SPHINXDIRS), $(MAKE) PDFLATEX="$(PDFLATEX)" LATEXOPTS="$(LATEXOPTS)" -C $(BUILDDIR)/$(var)/latex || exit;)
 
 endif # HAVE_PDFLATEX
 
diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst
index 20f92c16ffbf..88e746074252 100644
--- a/Documentation/admin-guide/cgroup-v2.rst
+++ b/Documentation/admin-guide/cgroup-v2.rst
@@ -864,6 +864,8 @@ All cgroup core files are prefixed with "cgroup."
 	  populated
 		1 if the cgroup or its descendants contains any live
 		processes; otherwise, 0.
+	  frozen
+		1 if the cgroup is frozen; otherwise, 0.
 
   cgroup.max.descendants
 	A read-write single value files.  The default is "max".
@@ -897,6 +899,31 @@ All cgroup core files are prefixed with "cgroup."
 		A dying cgroup can consume system resources not exceeding
 		limits, which were active at the moment of cgroup deletion.
 
+  cgroup.freeze
+	A read-write single value file which exists on non-root cgroups.
+	Allowed values are "0" and "1". The default is "0".
+
+	Writing "1" to the file causes freezing of the cgroup and all
+	descendant cgroups. This means that all belonging processes will
+	be stopped and will not run until the cgroup will be explicitly
+	unfrozen. Freezing of the cgroup may take some time; when this action
+	is completed, the "frozen" value in the cgroup.events control file
+	will be updated to "1" and the corresponding notification will be
+	issued.
+
+	A cgroup can be frozen either by its own settings, or by settings
+	of any ancestor cgroups. If any of ancestor cgroups is frozen, the
+	cgroup will remain frozen.
+
+	Processes in the frozen cgroup can be killed by a fatal signal.
+	They also can enter and leave a frozen cgroup: either by an explicit
+	move by a user, or if freezing of the cgroup races with fork().
+	If a process is moved to a frozen cgroup, it stops. If a process is
+	moved out of a frozen cgroup, it becomes running.
+
+	Frozen status of a cgroup doesn't affect any cgroup tree operations:
+	it's possible to delete a frozen (and empty) cgroup, as well as
+	create new sub-cgroups.
 
 Controllers
 ===========
diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index fd03e2b629bb..a1fe7e8c4f15 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -1588,7 +1588,7 @@
 			Format: { "off" | "enforce" | "fix" | "log" }
 			default: "enforce"
 
-	ima_appraise_tcb [IMA]
+	ima_appraise_tcb [IMA] Deprecated.  Use ima_policy= instead.
 			The builtin appraise policy appraises all files
 			owned by uid=0.
 
@@ -1615,8 +1615,7 @@
 			uid=0.
 
 			The "appraise_tcb" policy appraises the integrity of
-			all files owned by root. (This is the equivalent
-			of ima_appraise_tcb.)
+			all files owned by root.
 
 			The "secure_boot" policy appraises the integrity
 			of files (eg. kexec kernel image, kernel modules,
diff --git a/Documentation/atomic_bitops.txt b/Documentation/atomic_bitops.txt
index be70b32c95d9..093cdaefdb37 100644
--- a/Documentation/atomic_bitops.txt
+++ b/Documentation/atomic_bitops.txt
@@ -1,6 +1,6 @@
-
-On atomic bitops.
-
+=============
+Atomic bitops
+=============
 
 While our bitmap_{}() functions are non-atomic, we have a number of operations
 operating on single bits in a bitmap that are atomic.
diff --git a/Documentation/bpf/btf.rst b/Documentation/bpf/btf.rst
index 29396e6943b0..8820360d00da 100644
--- a/Documentation/bpf/btf.rst
+++ b/Documentation/bpf/btf.rst
@@ -578,6 +578,7 @@ For line_info, the line number and column number are defined as below:
     #define BPF_LINE_INFO_LINE_COL(line_col)        ((line_col) & 0x3ff)
 
 3.4 BPF_{PROG,MAP}_GET_NEXT_ID
+==============================
 
 In kernel, every loaded program, map or btf has a unique id. The id won't
 change during the lifetime of a program, map, or btf.
@@ -587,6 +588,7 @@ each command, to user space, for bpf program or maps, respectively, so an
 inspection tool can inspect all programs and maps.
 
 3.5 BPF_{PROG,MAP}_GET_FD_BY_ID
+===============================
 
 An introspection tool cannot use id to get details about program or maps.
 A file descriptor needs to be obtained first for reference-counting purpose.
diff --git a/Documentation/clearing-warn-once.txt b/Documentation/clearing-warn-once.txt
index c68598b31428..211fd926cf00 100644
--- a/Documentation/clearing-warn-once.txt
+++ b/Documentation/clearing-warn-once.txt
@@ -1,3 +1,5 @@
+Clearing WARN_ONCE
+------------------
 
 WARN_ONCE / WARN_ON_ONCE / printk_once only emit a message once.
 
diff --git a/Documentation/core-api/index.rst b/Documentation/core-api/index.rst
index 6870baffef82..ee1bb8983a88 100644
--- a/Documentation/core-api/index.rst
+++ b/Documentation/core-api/index.rst
@@ -22,7 +22,6 @@ Core utilities
    workqueue
    genericirq
    xarray
-   flexible-arrays
    librs
    genalloc
    errseq
diff --git a/Documentation/dev-tools/kselftest.rst b/Documentation/dev-tools/kselftest.rst
index c8c03388b9de..25604904fa6e 100644
--- a/Documentation/dev-tools/kselftest.rst
+++ b/Documentation/dev-tools/kselftest.rst
@@ -7,6 +7,11 @@ directory. These are intended to be small tests to exercise individual code
 paths in the kernel. Tests are intended to be run after building, installing
 and booting a kernel.
 
+You can find additional information on Kselftest framework, how to
+write new tests using the framework on Kselftest wiki:
+
+https://kselftest.wiki.kernel.org/
+
 On some systems, hot-plug tests could hang forever waiting for cpu and
 memory to be ready to be offlined. A special hot-plug target is created
 to run the full range of hot-plug tests. In default mode, hot-plug tests run
@@ -35,17 +40,32 @@ To build and run the tests with a single command, use::
 
 Note that some tests will require root privileges.
 
-Build and run from user specific object directory (make O=dir)::
+Kselftest supports saving output files in a separate directory and then
+running tests. To locate output files in a separate directory two syntaxes
+are supported. In both cases the working directory must be the root of the
+kernel src. This is applicable to "Running a subset of selftests" section
+below.
+
+To build, save output files in a separate directory with O= ::
 
   $ make O=/tmp/kselftest kselftest
 
-Build and run KBUILD_OUTPUT directory (make KBUILD_OUTPUT=)::
+To build, save output files in a separate directory with KBUILD_OUTPUT ::
+
+  $ export KBUILD_OUTPUT=/tmp/kselftest; make kselftest
 
-  $ make KBUILD_OUTPUT=/tmp/kselftest kselftest
+The O= assignment takes precedence over the KBUILD_OUTPUT environment
+variable.
 
-The above commands run the tests and print pass/fail summary to make it
-easier to understand the test results. Please find the detailed individual
-test results for each test in /tmp/testname file(s).
+The above commands by default run the tests and print full pass/fail report.
+Kselftest supports "summary" option to make it easier to understand the test
+results. Please find the detailed individual test results for each test in
+/tmp/testname file(s) when summary option is specified. This is applicable
+to "Running a subset of selftests" section below.
+
+To run kselftest with summary option enabled ::
+
+  $ make summary=1 kselftest
 
 Running a subset of selftests
 =============================
@@ -61,17 +81,13 @@ You can specify multiple tests to build and run::
 
   $  make TARGETS="size timers" kselftest
 
-Build and run from user specific object directory (make O=dir)::
+To build, save output files in a separate directory with O= ::
 
   $ make O=/tmp/kselftest TARGETS="size timers" kselftest
 
-Build and run KBUILD_OUTPUT directory (make KBUILD_OUTPUT=)::
-
-  $ make KBUILD_OUTPUT=/tmp/kselftest TARGETS="size timers" kselftest
+To build, save output files in a separate directory with KBUILD_OUTPUT ::
 
-The above commands run the tests and print pass/fail summary to make it
-easier to understand the test results. Please find the detailed individual
-test results for each test in /tmp/testname file(s).
+  $ export KBUILD_OUTPUT=/tmp/kselftest; make TARGETS="size timers" kselftest
 
 See the top-level tools/testing/selftests/Makefile for the list of all
 possible targets.
diff --git a/Documentation/devicetree/bindings/arm/mediatek/mediatek,apmixedsys.txt b/Documentation/devicetree/bindings/arm/mediatek/mediatek,apmixedsys.txt
index de4075413d91..161e63a6c254 100644
--- a/Documentation/devicetree/bindings/arm/mediatek/mediatek,apmixedsys.txt
+++ b/Documentation/devicetree/bindings/arm/mediatek/mediatek,apmixedsys.txt
@@ -14,6 +14,8 @@ Required Properties:
 	- "mediatek,mt7629-apmixedsys"
 	- "mediatek,mt8135-apmixedsys"
 	- "mediatek,mt8173-apmixedsys"
+	- "mediatek,mt8183-apmixedsys", "syscon"
+	- "mediatek,mt8516-apmixedsys"
 - #clock-cells: Must be 1
 
 The apmixedsys controller uses the common clk binding from
diff --git a/Documentation/devicetree/bindings/arm/mediatek/mediatek,audsys.txt b/Documentation/devicetree/bindings/arm/mediatek/mediatek,audsys.txt
index d1606b2c3e63..f3cef1a6d95c 100644
--- a/Documentation/devicetree/bindings/arm/mediatek/mediatek,audsys.txt
+++ b/Documentation/devicetree/bindings/arm/mediatek/mediatek,audsys.txt
@@ -9,6 +9,7 @@ Required Properties:
 	- "mediatek,mt2701-audsys", "syscon"
 	- "mediatek,mt7622-audsys", "syscon"
 	- "mediatek,mt7623-audsys", "mediatek,mt2701-audsys", "syscon"
+	- "mediatek,mt8183-audiosys", "syscon"
 - #clock-cells: Must be 1
 
 The AUDSYS controller uses the common clk binding from
diff --git a/Documentation/devicetree/bindings/arm/mediatek/mediatek,camsys.txt b/Documentation/devicetree/bindings/arm/mediatek/mediatek,camsys.txt
new file mode 100644
index 000000000000..d8930f64aa98
--- /dev/null
+++ b/Documentation/devicetree/bindings/arm/mediatek/mediatek,camsys.txt
@@ -0,0 +1,22 @@
+MediaTek CAMSYS controller
+============================
+
+The MediaTek camsys controller provides various clocks to the system.
+
+Required Properties:
+
+- compatible: Should be one of:
+	- "mediatek,mt8183-camsys", "syscon"
+- #clock-cells: Must be 1
+
+The camsys controller uses the common clk binding from
+Documentation/devicetree/bindings/clock/clock-bindings.txt
+The available clocks are defined in dt-bindings/clock/mt*-clk.h.
+
+Example:
+
+camsys: camsys@1a000000  {
+	compatible = "mediatek,mt8183-camsys", "syscon";
+	reg = <0 0x1a000000  0 0x1000>;
+	#clock-cells = <1>;
+};
diff --git a/Documentation/devicetree/bindings/arm/mediatek/mediatek,imgsys.txt b/Documentation/devicetree/bindings/arm/mediatek/mediatek,imgsys.txt
index 3f99672163e3..e3bc4a1e7a6e 100644
--- a/Documentation/devicetree/bindings/arm/mediatek/mediatek,imgsys.txt
+++ b/Documentation/devicetree/bindings/arm/mediatek/mediatek,imgsys.txt
@@ -11,6 +11,7 @@ Required Properties:
 	- "mediatek,mt6797-imgsys", "syscon"
 	- "mediatek,mt7623-imgsys", "mediatek,mt2701-imgsys", "syscon"
 	- "mediatek,mt8173-imgsys", "syscon"
+	- "mediatek,mt8183-imgsys", "syscon"
 - #clock-cells: Must be 1
 
 The imgsys controller uses the common clk binding from
diff --git a/Documentation/devicetree/bindings/arm/mediatek/mediatek,infracfg.txt b/Documentation/devicetree/bindings/arm/mediatek/mediatek,infracfg.txt
index 417bd83d1378..a90913988d7e 100644
--- a/Documentation/devicetree/bindings/arm/mediatek/mediatek,infracfg.txt
+++ b/Documentation/devicetree/bindings/arm/mediatek/mediatek,infracfg.txt
@@ -15,6 +15,8 @@ Required Properties:
 	- "mediatek,mt7629-infracfg", "syscon"
 	- "mediatek,mt8135-infracfg", "syscon"
 	- "mediatek,mt8173-infracfg", "syscon"
+	- "mediatek,mt8183-infracfg", "syscon"
+	- "mediatek,mt8516-infracfg", "syscon"
 - #clock-cells: Must be 1
 - #reset-cells: Must be 1
 
diff --git a/Documentation/devicetree/bindings/arm/mediatek/mediatek,ipu.txt b/Documentation/devicetree/bindings/arm/mediatek/mediatek,ipu.txt
new file mode 100644
index 000000000000..aabc8c5c8ed2
--- /dev/null
+++ b/Documentation/devicetree/bindings/arm/mediatek/mediatek,ipu.txt
@@ -0,0 +1,43 @@
+Mediatek IPU controller
+============================
+
+The Mediatek ipu controller provides various clocks to the system.
+
+Required Properties:
+
+- compatible: Should be one of:
+	- "mediatek,mt8183-ipu_conn", "syscon"
+	- "mediatek,mt8183-ipu_adl", "syscon"
+	- "mediatek,mt8183-ipu_core0", "syscon"
+	- "mediatek,mt8183-ipu_core1", "syscon"
+- #clock-cells: Must be 1
+
+The ipu controller uses the common clk binding from
+Documentation/devicetree/bindings/clock/clock-bindings.txt
+The available clocks are defined in dt-bindings/clock/mt*-clk.h.
+
+Example:
+
+ipu_conn: syscon@19000000 {
+	compatible = "mediatek,mt8183-ipu_conn", "syscon";
+	reg = <0 0x19000000 0 0x1000>;
+	#clock-cells = <1>;
+};
+
+ipu_adl: syscon@19010000 {
+	compatible = "mediatek,mt8183-ipu_adl", "syscon";
+	reg = <0 0x19010000 0 0x1000>;
+	#clock-cells = <1>;
+};
+
+ipu_core0: syscon@19180000 {
+	compatible = "mediatek,mt8183-ipu_core0", "syscon";
+	reg = <0 0x19180000 0 0x1000>;
+	#clock-cells = <1>;
+};
+
+ipu_core1: syscon@19280000 {
+	compatible = "mediatek,mt8183-ipu_core1", "syscon";
+	reg = <0 0x19280000 0 0x1000>;
+	#clock-cells = <1>;
+};
diff --git a/Documentation/devicetree/bindings/arm/mediatek/mediatek,mcucfg.txt b/Documentation/devicetree/bindings/arm/mediatek/mediatek,mcucfg.txt
index b8fb03f3613e..2b882b7ca72e 100644
--- a/Documentation/devicetree/bindings/arm/mediatek/mediatek,mcucfg.txt
+++ b/Documentation/devicetree/bindings/arm/mediatek/mediatek,mcucfg.txt
@@ -7,6 +7,7 @@ Required Properties:
 
 - compatible: Should be one of:
 	- "mediatek,mt2712-mcucfg", "syscon"
+	- "mediatek,mt8183-mcucfg", "syscon"
 - #clock-cells: Must be 1
 
 The mcucfg controller uses the common clk binding from
diff --git a/Documentation/devicetree/bindings/arm/mediatek/mediatek,mfgcfg.txt b/Documentation/devicetree/bindings/arm/mediatek/mediatek,mfgcfg.txt
index 859e67b416d5..72787e7dd227 100644
--- a/Documentation/devicetree/bindings/arm/mediatek/mediatek,mfgcfg.txt
+++ b/Documentation/devicetree/bindings/arm/mediatek/mediatek,mfgcfg.txt
@@ -7,6 +7,7 @@ Required Properties:
 
 - compatible: Should be one of:
 	- "mediatek,mt2712-mfgcfg", "syscon"
+	- "mediatek,mt8183-mfgcfg", "syscon"
 - #clock-cells: Must be 1
 
 The mfgcfg controller uses the common clk binding from
diff --git a/Documentation/devicetree/bindings/arm/mediatek/mediatek,mmsys.txt b/Documentation/devicetree/bindings/arm/mediatek/mediatek,mmsys.txt
index 15d977afad31..545eab717c96 100644
--- a/Documentation/devicetree/bindings/arm/mediatek/mediatek,mmsys.txt
+++ b/Documentation/devicetree/bindings/arm/mediatek/mediatek,mmsys.txt
@@ -11,6 +11,7 @@ Required Properties:
 	- "mediatek,mt6797-mmsys", "syscon"
 	- "mediatek,mt7623-mmsys", "mediatek,mt2701-mmsys", "syscon"
 	- "mediatek,mt8173-mmsys", "syscon"
+	- "mediatek,mt8183-mmsys", "syscon"
 - #clock-cells: Must be 1
 
 The mmsys controller uses the common clk binding from
diff --git a/Documentation/devicetree/bindings/arm/mediatek/mediatek,topckgen.txt b/Documentation/devicetree/bindings/arm/mediatek/mediatek,topckgen.txt
index d160c2b4b6fe..a023b8338960 100644
--- a/Documentation/devicetree/bindings/arm/mediatek/mediatek,topckgen.txt
+++ b/Documentation/devicetree/bindings/arm/mediatek/mediatek,topckgen.txt
@@ -14,6 +14,8 @@ Required Properties:
 	- "mediatek,mt7629-topckgen"
 	- "mediatek,mt8135-topckgen"
 	- "mediatek,mt8173-topckgen"
+	- "mediatek,mt8183-topckgen", "syscon"
+	- "mediatek,mt8516-topckgen"
 - #clock-cells: Must be 1
 
 The topckgen controller uses the common clk binding from
diff --git a/Documentation/devicetree/bindings/arm/mediatek/mediatek,vdecsys.txt b/Documentation/devicetree/bindings/arm/mediatek/mediatek,vdecsys.txt
index 3212afc753c8..57176bb8dbb5 100644
--- a/Documentation/devicetree/bindings/arm/mediatek/mediatek,vdecsys.txt
+++ b/Documentation/devicetree/bindings/arm/mediatek/mediatek,vdecsys.txt
@@ -11,6 +11,7 @@ Required Properties:
 	- "mediatek,mt6797-vdecsys", "syscon"
 	- "mediatek,mt7623-vdecsys", "mediatek,mt2701-vdecsys", "syscon"
 	- "mediatek,mt8173-vdecsys", "syscon"
+	- "mediatek,mt8183-vdecsys", "syscon"
 - #clock-cells: Must be 1
 
 The vdecsys controller uses the common clk binding from
diff --git a/Documentation/devicetree/bindings/arm/mediatek/mediatek,vencsys.txt b/Documentation/devicetree/bindings/arm/mediatek/mediatek,vencsys.txt
index 851545357e94..c9faa6269087 100644
--- a/Documentation/devicetree/bindings/arm/mediatek/mediatek,vencsys.txt
+++ b/Documentation/devicetree/bindings/arm/mediatek/mediatek,vencsys.txt
@@ -9,6 +9,7 @@ Required Properties:
 	- "mediatek,mt2712-vencsys", "syscon"
 	- "mediatek,mt6797-vencsys", "syscon"
 	- "mediatek,mt8173-vencsys", "syscon"
+	- "mediatek,mt8183-vencsys", "syscon"
 - #clock-cells: Must be 1
 
 The vencsys controller uses the common clk binding from
diff --git a/Documentation/devicetree/bindings/clock/amlogic,axg-audio-clkc.txt b/Documentation/devicetree/bindings/clock/amlogic,axg-audio-clkc.txt
index 61777ad24f61..0f777749f4f1 100644
--- a/Documentation/devicetree/bindings/clock/amlogic,axg-audio-clkc.txt
+++ b/Documentation/devicetree/bindings/clock/amlogic,axg-audio-clkc.txt
@@ -6,7 +6,8 @@ devices.
 
 Required Properties:
 
-- compatible	: should be "amlogic,axg-audio-clkc" for the A113X and A113D
+- compatible	: should be "amlogic,axg-audio-clkc" for the A113X and A113D,
+		  "amlogic,g12a-audio-clkc" for G12A.
 - reg		: physical base address of the clock controller and length of
 		  memory mapped region.
 - clocks	: a list of phandle + clock-specifier pairs for the clocks listed
diff --git a/Documentation/devicetree/bindings/clock/at91-clock.txt b/Documentation/devicetree/bindings/clock/at91-clock.txt
index e9f70fcdfe80..b520280e33ff 100644
--- a/Documentation/devicetree/bindings/clock/at91-clock.txt
+++ b/Documentation/devicetree/bindings/clock/at91-clock.txt
@@ -8,35 +8,30 @@ Slow Clock controller:
 
 Required properties:
 - compatible : shall be one of the following:
-	"atmel,at91sam9x5-sckc" or
+	"atmel,at91sam9x5-sckc",
+	"atmel,sama5d3-sckc" or
 	"atmel,sama5d4-sckc":
 		at91 SCKC (Slow Clock Controller)
-		This node contains the slow clock definitions.
-
-	"atmel,at91sam9x5-clk-slow-osc":
-		at91 slow oscillator
-
-	"atmel,at91sam9x5-clk-slow-rc-osc":
-		at91 internal slow RC oscillator
-- reg : defines the IO memory reserved for the SCKC.
-- #size-cells : shall be 0 (reg is used to encode clk id).
-- #address-cells : shall be 1 (reg is used to encode clk id).
+- #clock-cells : shall be 0.
+- clocks : shall be the input parent clock phandle for the clock.
 
+Optional properties:
+- atmel,osc-bypass : boolean property. Set this when a clock signal is directly
+  provided on XIN.
 
 For example:
-	sckc: sckc@fffffe50 {
-		compatible = "atmel,sama5d3-pmc";
-		reg = <0xfffffe50 0x4>
-		#size-cells = <0>;
-		#address-cells = <1>;
-
-		/* put at91 slow clocks here */
+	sckc@fffffe50 {
+		compatible = "atmel,at91sam9x5-sckc";
+		reg = <0xfffffe50 0x4>;
+		clocks = <&slow_xtal>;
+		#clock-cells = <0>;
 	};
 
 Power Management Controller (PMC):
 
 Required properties:
-- compatible : shall be "atmel,<chip>-pmc", "syscon":
+- compatible : shall be "atmel,<chip>-pmc", "syscon" or
+	"microchip,sam9x60-pmc"
 	<chip> can be: at91rm9200, at91sam9260, at91sam9261,
 	at91sam9263, at91sam9g45, at91sam9n12, at91sam9rl, at91sam9g15,
 	at91sam9g25, at91sam9g35, at91sam9x25, at91sam9x35, at91sam9x5,
diff --git a/Documentation/devicetree/bindings/clock/cirrus,lochnagar.txt b/Documentation/devicetree/bindings/clock/cirrus,lochnagar.txt
new file mode 100644
index 000000000000..b8d8ef3bdc5f
--- /dev/null
+++ b/Documentation/devicetree/bindings/clock/cirrus,lochnagar.txt
@@ -0,0 +1,93 @@
+Cirrus Logic Lochnagar Audio Development Board
+
+Lochnagar is an evaluation and development board for Cirrus Logic
+Smart CODEC and Amp devices. It allows the connection of most Cirrus
+Logic devices on mini-cards, as well as allowing connection of
+various application processor systems to provide a full evaluation
+platform.  Audio system topology, clocking and power can all be
+controlled through the Lochnagar, allowing the device under test
+to be used in a variety of possible use cases.
+
+This binding document describes the binding for the clock portion of
+the driver.
+
+Also see these documents for generic binding information:
+  [1] Clock : ../clock/clock-bindings.txt
+
+And these for relevant defines:
+  [2] include/dt-bindings/clock/lochnagar.h
+
+This binding must be part of the Lochnagar MFD binding:
+  [3] ../mfd/cirrus,lochnagar.txt
+
+Required properties:
+
+  - compatible : One of the following strings:
+                 "cirrus,lochnagar1-clk"
+                 "cirrus,lochnagar2-clk"
+
+  - #clock-cells : Must be 1. The first cell indicates the clock
+    number, see [2] for available clocks and [1].
+
+Optional properties:
+
+  - clocks : Must contain an entry for each clock in clock-names.
+  - clock-names : May contain entries for each of the following
+    clocks:
+     - ln-cdc-clkout : Output clock from CODEC card.
+     - ln-dsp-clkout : Output clock from DSP card.
+     - ln-gf-mclk1,ln-gf-mclk2,ln-gf-mclk3,ln-gf-mclk4 : Optional
+       input audio clocks from host system.
+     - ln-psia1-mclk, ln-psia2-mclk : Optional input audio clocks from
+       external connector.
+     - ln-spdif-clkout : Optional input audio clock from SPDIF.
+     - ln-adat-mclk : Optional input audio clock from ADAT.
+     - ln-pmic-32k : On board fixed clock.
+     - ln-clk-12m : On board fixed clock.
+     - ln-clk-11m : On board fixed clock.
+     - ln-clk-24m : On board fixed clock.
+     - ln-clk-22m : On board fixed clock.
+     - ln-clk-8m : On board fixed clock.
+     - ln-usb-clk-24m : On board fixed clock.
+     - ln-usb-clk-12m : On board fixed clock.
+
+  - assigned-clocks : A list of Lochnagar clocks to be reparented, see
+    [2] for available clocks.
+  - assigned-clock-parents : Parents to be assigned to the clocks
+    listed in "assigned-clocks".
+
+Optional nodes:
+
+  - fixed-clock nodes may be registered for the following on board clocks:
+     - ln-pmic-32k : 32768 Hz
+     - ln-clk-12m : 12288000 Hz
+     - ln-clk-11m : 11298600 Hz
+     - ln-clk-24m : 24576000 Hz
+     - ln-clk-22m : 22579200 Hz
+     - ln-clk-8m : 8192000 Hz
+     - ln-usb-clk-24m : 24576000 Hz
+     - ln-usb-clk-12m : 12288000 Hz
+
+Example:
+
+lochnagar {
+	lochnagar-clk {
+		compatible = "cirrus,lochnagar2-clk";
+
+		#clock-cells = <1>;
+
+		clocks = <&clk-audio>, <&clk_pmic>;
+		clock-names = "ln-gf-mclk2", "ln-pmic-32k";
+
+		assigned-clocks = <&lochnagar-clk LOCHNAGAR_CDC_MCLK1>,
+				  <&lochnagar-clk LOCHNAGAR_CDC_MCLK2>;
+		assigned-clock-parents = <&clk-audio>,
+					 <&clk-pmic>;
+	};
+
+	clk-pmic: clk-pmic {
+		compatible = "fixed-clock";
+		clock-cells = <0>;
+		clock-frequency = <32768>;
+	};
+};
diff --git a/Documentation/devicetree/bindings/clock/milbeaut-clock.yaml b/Documentation/devicetree/bindings/clock/milbeaut-clock.yaml
new file mode 100644
index 000000000000..5cf0b811821e
--- /dev/null
+++ b/Documentation/devicetree/bindings/clock/milbeaut-clock.yaml
@@ -0,0 +1,73 @@
+# SPDX-License-Identifier: GPL-2.0
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/bindings/clock/milbeaut-clock.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Milbeaut SoCs Clock Controller Binding
+
+maintainers:
+  - Taichi Sugaya <sugaya.taichi@socionext.com>
+
+description: |
+  Milbeaut SoCs Clock controller is an integrated clock controller, which
+  generates and supplies to all modules.
+
+  This binding uses common clock bindings
+  [1] Documentation/devicetree/bindings/clock/clock-bindings.txt
+
+properties:
+  compatible:
+    oneOf:
+      - items:
+          - enum:
+              - socionext,milbeaut-m10v-ccu
+  clocks:
+    maxItems: 1
+    description: external clock
+
+  '#clock-cells':
+    const: 1
+
+required:
+  - compatible
+  - reg
+  - clocks
+  - '#clock-cells'
+
+examples:
+  # Clock controller node:
+  - |
+    m10v-clk-ctrl@1d021000 {
+        compatible = "socionext,milbeaut-m10v-clk-ccu";
+        reg = <0x1d021000 0x4000>;
+        #clock-cells = <1>;
+        clocks = <&clki40mhz>;
+    };
+
+  # Required an external clock for Clock controller node:
+  - |
+    clocks {
+        clki40mhz: clki40mhz {
+            compatible = "fixed-clock";
+            #clock-cells = <0>;
+            clock-frequency = <40000000>;
+        };
+        /* other clocks */
+    };
+
+  # The clock consumer shall specify the desired clock-output of the clock
+  # controller as below by specifying output-id in its "clk" phandle cell.
+  # 2: uart
+  # 4: 32-bit timer
+  # 7: UHS-I/II
+  - |
+    serial@1e700010 {
+        compatible = "socionext,milbeaut-usio-uart";
+        reg = <0x1e700010 0x10>;
+        interrupts = <0 141 0x4>, <0 149 0x4>;
+        interrupt-names = "rx", "tx";
+        clocks = <&clk 2>;
+    };
+
+...
diff --git a/Documentation/devicetree/bindings/clock/qcom,turingcc.txt b/Documentation/devicetree/bindings/clock/qcom,turingcc.txt
new file mode 100644
index 000000000000..126517de5f9a
--- /dev/null
+++ b/Documentation/devicetree/bindings/clock/qcom,turingcc.txt
@@ -0,0 +1,19 @@
+Qualcomm Turing Clock & Reset Controller Binding
+------------------------------------------------
+
+Required properties :
+- compatible: shall contain "qcom,qcs404-turingcc".
+- reg: shall contain base register location and length.
+- clocks: ahb clock for the TuringCC
+- #clock-cells: from common clock binding, shall contain 1.
+- #reset-cells: from common reset binding, shall contain 1.
+
+Example:
+	turingcc: clock-controller@800000 {
+		compatible = "qcom,qcs404-turingcc";
+		reg = <0x00800000 0x30000>;
+		clocks = <&gcc GCC_CDSP_CFG_AHB_CLK>;
+
+		#clock-cells = <1>;
+		#reset-cells = <1>;
+	};
diff --git a/Documentation/devicetree/bindings/clock/qoriq-clock.txt b/Documentation/devicetree/bindings/clock/qoriq-clock.txt
index c655f28d5918..f7d48f23da44 100644
--- a/Documentation/devicetree/bindings/clock/qoriq-clock.txt
+++ b/Documentation/devicetree/bindings/clock/qoriq-clock.txt
@@ -39,6 +39,7 @@ Required properties:
 	* "fsl,b4860-clockgen"
 	* "fsl,ls1012a-clockgen"
 	* "fsl,ls1021a-clockgen"
+	* "fsl,ls1028a-clockgen"
 	* "fsl,ls1043a-clockgen"
 	* "fsl,ls1046a-clockgen"
 	* "fsl,ls1088a-clockgen"
@@ -83,8 +84,8 @@ second cell is the clock index for the specified type.
 	1	cmux		index (n in CLKCnCSR)
 	2	hwaccel		index (n in CLKCGnHWACSR)
 	3	fman		0 for fm1, 1 for fm2
-	4	platform pll	0=pll, 1=pll/2, 2=pll/3, 3=pll/4
-				4=pll/5, 5=pll/6, 6=pll/7, 7=pll/8
+	4	platform pll	n=pll/(n+1). For example, when n=1,
+				that means output_freq=PLL_freq/2.
 	5	coreclk		must be 0
 
 3. Example
diff --git a/Documentation/devicetree/bindings/clock/sifive/fu540-prci.txt b/Documentation/devicetree/bindings/clock/sifive/fu540-prci.txt
new file mode 100644
index 000000000000..349808f4fb8c
--- /dev/null
+++ b/Documentation/devicetree/bindings/clock/sifive/fu540-prci.txt
@@ -0,0 +1,46 @@
+SiFive FU540 PRCI bindings
+
+On the FU540 family of SoCs, most system-wide clock and reset integration
+is via the PRCI IP block.
+
+Required properties:
+- compatible: Should be "sifive,<chip>-prci".  Only one value is
+	supported: "sifive,fu540-c000-prci"
+- reg: Should describe the PRCI's register target physical address region
+- clocks: Should point to the hfclk device tree node and the rtcclk
+          device tree node.  The RTC clock here is not a time-of-day clock,
+	  but is instead a high-stability clock source for system timers
+	  and cycle counters.
+- #clock-cells: Should be <1>
+
+The clock consumer should specify the desired clock via the clock ID
+macros defined in include/dt-bindings/clock/sifive-fu540-prci.h.
+These macros begin with PRCI_CLK_.
+
+The hfclk and rtcclk nodes are required, and represent physical
+crystals or resonators located on the PCB.  These nodes should be present
+underneath /, rather than /soc.
+
+Examples:
+
+/* under /, in PCB-specific DT data */
+hfclk: hfclk {
+	#clock-cells = <0>;
+	compatible = "fixed-clock";
+	clock-frequency = <33333333>;
+	clock-output-names = "hfclk";
+};
+rtcclk: rtcclk {
+	#clock-cells = <0>;
+	compatible = "fixed-clock";
+	clock-frequency = <1000000>;
+	clock-output-names = "rtcclk";
+};
+
+/* under /soc, in SoC-specific DT data */
+prci: clock-controller@10000000 {
+	compatible = "sifive,fu540-c000-prci";
+	reg = <0x0 0x10000000 0x0 0x1000>;
+	clocks = <&hfclk>, <&rtcclk>;
+	#clock-cells = <1>;
+};
diff --git a/Documentation/devicetree/bindings/clock/st,stm32-rcc.txt b/Documentation/devicetree/bindings/clock/st,stm32-rcc.txt
index b240121d2ac9..cfa04b614d8a 100644
--- a/Documentation/devicetree/bindings/clock/st,stm32-rcc.txt
+++ b/Documentation/devicetree/bindings/clock/st,stm32-rcc.txt
@@ -11,6 +11,8 @@ Required properties:
   "st,stm32f42xx-rcc"
   "st,stm32f469-rcc"
   "st,stm32f746-rcc"
+  "st,stm32f769-rcc"
+
 - reg: should be register base and length as documented in the
   datasheet
 - #reset-cells: 1, see below
@@ -102,6 +104,10 @@ The secondary index is bound with the following magic numbers:
 	28	CLK_I2C3
 	29	CLK_I2C4
 	30	CLK_LPTIMER	(LPTimer1 clock)
+	31	CLK_PLL_SRC
+	32	CLK_DFSDM1
+	33	CLK_ADFSDM1
+	34	CLK_F769_DSI
 )
 
 Example:
diff --git a/Documentation/devicetree/bindings/display/amlogic,meson-dw-hdmi.txt b/Documentation/devicetree/bindings/display/amlogic,meson-dw-hdmi.txt
index bf4a18047309..3a50a7862cf3 100644
--- a/Documentation/devicetree/bindings/display/amlogic,meson-dw-hdmi.txt
+++ b/Documentation/devicetree/bindings/display/amlogic,meson-dw-hdmi.txt
@@ -37,6 +37,7 @@ Required properties:
 	- GXL (S905X, S905D) : "amlogic,meson-gxl-dw-hdmi"
 	- GXM (S912) : "amlogic,meson-gxm-dw-hdmi"
 	followed by the common "amlogic,meson-gx-dw-hdmi"
+	- G12A (S905X2, S905Y2, S905D2) : "amlogic,meson-g12a-dw-hdmi"
 - reg: Physical base address and length of the controller's registers.
 - interrupts: The HDMI interrupt number
 - clocks, clock-names : must have the phandles to the HDMI iahb and isfr clocks,
@@ -66,6 +67,9 @@ corresponding to each HDMI output and input.
  S905X (GXL)	VENC Input	TMDS Output
  S905D (GXL)	VENC Input	TMDS Output
  S912 (GXM)	VENC Input	TMDS Output
+ S905X2 (G12A)	VENC Input	TMDS Output
+ S905Y2 (G12A)	VENC Input	TMDS Output
+ S905D2 (G12A)	VENC Input	TMDS Output
 
 Example:
 
diff --git a/Documentation/devicetree/bindings/display/amlogic,meson-vpu.txt b/Documentation/devicetree/bindings/display/amlogic,meson-vpu.txt
index c65fd7a7467c..be40a780501c 100644
--- a/Documentation/devicetree/bindings/display/amlogic,meson-vpu.txt
+++ b/Documentation/devicetree/bindings/display/amlogic,meson-vpu.txt
@@ -57,18 +57,18 @@ Required properties:
 	- GXL (S905X, S905D) : "amlogic,meson-gxl-vpu"
 	- GXM (S912) : "amlogic,meson-gxm-vpu"
 	followed by the common "amlogic,meson-gx-vpu"
+	- G12A (S905X2, S905Y2, S905D2) : "amlogic,meson-g12a-vpu"
 - reg: base address and size of he following memory-mapped regions :
 	- vpu
 	- hhi
-	- dmc
 - reg-names: should contain the names of the previous memory regions
 - interrupts: should contain the VENC Vsync interrupt number
+- amlogic,canvas: phandle to canvas provider node as described in the file
+	../soc/amlogic/amlogic,canvas.txt
 
 Optional properties:
 - power-domains: Optional phandle to associated power domain as described in
 	the file ../power/power_domain.txt
-- amlogic,canvas: phandle to canvas provider node as described in the file
-	../soc/amlogic/amlogic,canvas.txt
 
 Required nodes:
 
@@ -84,6 +84,9 @@ corresponding to each VPU output.
  S905X (GXL)	CVBS VDAC	HDMI-TX
  S905D (GXL)	CVBS VDAC	HDMI-TX
  S912 (GXM)	CVBS VDAC	HDMI-TX
+ S905X2 (G12A)	CVBS VDAC	HDMI-TX
+ S905Y2 (G12A)	CVBS VDAC	HDMI-TX
+ S905D2 (G12A)	CVBS VDAC	HDMI-TX
 
 Example:
 
diff --git a/Documentation/devicetree/bindings/display/bridge/ti,tfp410.txt b/Documentation/devicetree/bindings/display/bridge/ti,tfp410.txt
index 54d7e31525ec..5ff4f64ef8e8 100644
--- a/Documentation/devicetree/bindings/display/bridge/ti,tfp410.txt
+++ b/Documentation/devicetree/bindings/display/bridge/ti,tfp410.txt
@@ -6,15 +6,32 @@ Required properties:
 
 Optional properties:
 - powerdown-gpios: power-down gpio
-- reg: I2C address. If and only if present the device node
-       should be placed into the i2c controller node where the
-       tfp410 i2c is connected to.
+- reg: I2C address. If and only if present the device node should be placed
+  into the I2C controller node where the TFP410 I2C is connected to.
+- ti,deskew: data de-skew in 350ps increments, from -4 to +3, as configured
+  through th DK[3:1] pins. This property shall be present only if the TFP410
+  is not connected through I2C.
 
 Required nodes:
-- Video port 0 for DPI input [1].
-- Video port 1 for DVI output [1].
 
-[1]: Documentation/devicetree/bindings/media/video-interfaces.txt
+This device has two video ports. Their connections are modeled using the OF
+graph bindings specified in [1]. Each port node shall have a single endpoint.
+
+- Port 0 is the DPI input port. Its endpoint subnode shall contain a
+  pclk-sample and bus-width property and a remote-endpoint property as specified
+  in [1].
+  - If pclk-sample is not defined, pclk-sample = 0 should be assumed for
+    backward compatibility.
+  - If bus-width is not defined then bus-width = 24 should be assumed for
+    backward compatibility.
+    bus-width = 24: 24 data lines are connected and single-edge mode
+    bus-width = 12: 12 data lines are connected and dual-edge mode
+
+- Port 1 is the DVI output port. Its endpoint subnode shall contain a
+  remote-endpoint property is specified in [1].
+
+[1] Documentation/devicetree/bindings/media/video-interfaces.txt
+
 
 Example
 -------
@@ -22,6 +39,7 @@ Example
 tfp410: encoder@0 {
 	compatible = "ti,tfp410";
 	powerdown-gpios = <&twl_gpio 2 GPIO_ACTIVE_LOW>;
+	ti,deskew = <4>;
 
 	ports {
 		#address-cells = <1>;
@@ -31,6 +49,8 @@ tfp410: encoder@0 {
 			reg = <0>;
 
 			tfp410_in: endpoint@0 {
+				pclk-sample = <1>;
+				bus-width = <24>;
 				remote-endpoint = <&dpi_out>;
 			};
 		};
diff --git a/Documentation/devicetree/bindings/display/msm/gmu.txt b/Documentation/devicetree/bindings/display/msm/gmu.txt
index 3439b38e60f2..90af5b0a56a9 100644
--- a/Documentation/devicetree/bindings/display/msm/gmu.txt
+++ b/Documentation/devicetree/bindings/display/msm/gmu.txt
@@ -24,7 +24,10 @@ Required properties:
    * "cxo"
    * "axi"
    * "mnoc"
-- power-domains: should be <&clock_gpucc GPU_CX_GDSC>
+- power-domains: should be:
+	<&clock_gpucc GPU_CX_GDSC>
+	<&clock_gpucc GPU_GX_GDSC>
+- power-domain-names: Matching names for the power domains
 - iommus: phandle to the adreno iommu
 - operating-points-v2: phandle to the OPP operating points
 
@@ -51,7 +54,10 @@ Example:
 			<&gcc GCC_GPU_MEMNOC_GFX_CLK>;
 		clock-names = "gmu", "cxo", "axi", "memnoc";
 
-		power-domains = <&gpucc GPU_CX_GDSC>;
+		power-domains = <&gpucc GPU_CX_GDSC>,
+				<&gpucc GPU_GX_GDSC>;
+		power-domain-names = "cx", "gx";
+
 		iommus = <&adreno_smmu 5>;
 
 		operating-points-v2 = <&gmu_opp_table>;
diff --git a/Documentation/devicetree/bindings/display/msm/gpu.txt b/Documentation/devicetree/bindings/display/msm/gpu.txt
index aad1aef682f7..2b8fd26c43b0 100644
--- a/Documentation/devicetree/bindings/display/msm/gpu.txt
+++ b/Documentation/devicetree/bindings/display/msm/gpu.txt
@@ -22,9 +22,14 @@ Required properties:
    - qcom,adreno-630.2
 - iommus: optional phandle to an adreno iommu instance
 - operating-points-v2: optional phandle to the OPP operating points
+- interconnects: optional phandle to an interconnect provider.  See
+  ../interconnect/interconnect.txt for details.
 - qcom,gmu: For GMU attached devices a phandle to the GMU device that will
   control the power for the GPU. Applicable targets:
     - qcom,adreno-630.2
+- zap-shader: For a5xx and a6xx devices this node contains a memory-region that
+  points to reserved memory to store the zap shader that can be used to help
+  bring the GPU out of secure mode.
 
 Example 3xx/4xx/a5xx:
 
@@ -70,6 +75,12 @@ Example a6xx (with GMU):
 
 		operating-points-v2 = <&gpu_opp_table>;
 
+		interconnects = <&rsc_hlos MASTER_GFX3D &rsc_hlos SLAVE_EBI1>;
+
 		qcom,gmu = <&gmu>;
+
+		zap-shader {
+			memory-region = <&zap_shader_region>;
+		};
 	};
 };
diff --git a/Documentation/devicetree/bindings/display/panel/feiyang,fy07024di26a30d.txt b/Documentation/devicetree/bindings/display/panel/feiyang,fy07024di26a30d.txt
new file mode 100644
index 000000000000..82caa7b65ae8
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/panel/feiyang,fy07024di26a30d.txt
@@ -0,0 +1,20 @@
+Feiyang FY07024DI26A30-D 7" MIPI-DSI LCD Panel
+
+Required properties:
+- compatible: must be "feiyang,fy07024di26a30d"
+- reg: DSI virtual channel used by that screen
+- avdd-supply: analog regulator dc1 switch
+- dvdd-supply: 3v3 digital regulator
+- reset-gpios: a GPIO phandle for the reset pin
+
+Optional properties:
+- backlight: phandle for the backlight control.
+
+panel@0 {
+	compatible = "feiyang,fy07024di26a30d";
+	reg = <0>;
+	avdd-supply = <&reg_dc1sw>;
+	dvdd-supply = <&reg_dldo2>;
+	reset-gpios = <&pio 3 24 GPIO_ACTIVE_HIGH>; /* LCD-RST: PD24 */
+	backlight = <&backlight>;
+};
diff --git a/Documentation/devicetree/bindings/display/panel/innolux,p079zca.txt b/Documentation/devicetree/bindings/display/panel/innolux,p079zca.txt
index d0f55161579a..3ab8c7412cf6 100644
--- a/Documentation/devicetree/bindings/display/panel/innolux,p079zca.txt
+++ b/Documentation/devicetree/bindings/display/panel/innolux,p079zca.txt
@@ -12,7 +12,7 @@ Optional properties:
 Example:
 
 	&mipi_dsi {
-		panel {
+		panel@0 {
 			compatible = "innolux,p079zca";
 			reg = <0>;
 			power-supply = <...>;
diff --git a/Documentation/devicetree/bindings/display/panel/innolux,p097pfg.txt b/Documentation/devicetree/bindings/display/panel/innolux,p097pfg.txt
index 595d9dfeffd3..d1cab3a8f0fb 100644
--- a/Documentation/devicetree/bindings/display/panel/innolux,p097pfg.txt
+++ b/Documentation/devicetree/bindings/display/panel/innolux,p097pfg.txt
@@ -13,7 +13,7 @@ Optional properties:
 Example:
 
 	&mipi_dsi {
-		panel {
+		panel@0 {
 			compatible = "innolux,p079zca";
 			reg = <0>;
 			avdd-supply = <...>;
diff --git a/Documentation/devicetree/bindings/display/panel/kingdisplay,kd097d04.txt b/Documentation/devicetree/bindings/display/panel/kingdisplay,kd097d04.txt
index 164a5fa236da..cfefff688614 100644
--- a/Documentation/devicetree/bindings/display/panel/kingdisplay,kd097d04.txt
+++ b/Documentation/devicetree/bindings/display/panel/kingdisplay,kd097d04.txt
@@ -12,7 +12,7 @@ Optional properties:
 Example:
 
 	&mipi_dsi {
-		panel {
+		panel@0 {
 			compatible = "kingdisplay,kd097d04";
 			reg = <0>;
 			power-supply = <...>;
diff --git a/Documentation/devicetree/bindings/display/panel/lg,acx467akm-7.txt b/Documentation/devicetree/bindings/display/panel/lg,acx467akm-7.txt
new file mode 100644
index 000000000000..fc1e1b325e49
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/panel/lg,acx467akm-7.txt
@@ -0,0 +1,7 @@
+LG ACX467AKM-7 4.95" 1080×1920 LCD Panel
+
+Required properties:
+- compatible: must be "lg,acx467akm-7"
+
+This binding is compatible with the simple-panel binding, which is specified
+in simple-panel.txt in this directory.
diff --git a/Documentation/devicetree/bindings/display/panel/osddisplays,osd070t1718-19ts.txt b/Documentation/devicetree/bindings/display/panel/osddisplays,osd070t1718-19ts.txt
new file mode 100644
index 000000000000..e57883ccdf2f
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/panel/osddisplays,osd070t1718-19ts.txt
@@ -0,0 +1,12 @@
+OSD Displays OSD070T1718-19TS 7" WVGA TFT LCD panel
+
+Required properties:
+- compatible: shall be "osddisplays,osd070t1718-19ts"
+- power-supply: see simple-panel.txt
+
+Optional properties:
+- backlight: see simple-panel.txt
+
+This binding is compatible with the simple-panel binding, which is specified
+in simple-panel.txt in this directory. No other simple-panel properties than
+the ones specified herein are valid.
diff --git a/Documentation/devicetree/bindings/display/panel/rocktech,jh057n00900.txt b/Documentation/devicetree/bindings/display/panel/rocktech,jh057n00900.txt
new file mode 100644
index 000000000000..1b5763200cf6
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/panel/rocktech,jh057n00900.txt
@@ -0,0 +1,18 @@
+Rocktech jh057n00900 5.5" 720x1440 TFT LCD panel
+
+Required properties:
+- compatible: should be "rocktech,jh057n00900"
+- reg: DSI virtual channel of the peripheral
+- reset-gpios: panel reset gpio
+- backlight: phandle of the backlight device attached to the panel
+
+Example:
+
+	&mipi_dsi {
+		panel@0 {
+			compatible = "rocktech,jh057n00900";
+			reg = <0>;
+			backlight = <&backlight>;
+			reset-gpios = <&gpio3 13 GPIO_ACTIVE_LOW>;
+		};
+	};
diff --git a/Documentation/devicetree/bindings/display/panel/ronbo,rb070d30.yaml b/Documentation/devicetree/bindings/display/panel/ronbo,rb070d30.yaml
new file mode 100644
index 000000000000..0e7987f1cdb7
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/panel/ronbo,rb070d30.yaml
@@ -0,0 +1,51 @@
+# SPDX-License-Identifier: (GPL-2.0+ OR X11)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/display/panel/ronbo,rb070d30.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Ronbo RB070D30 DSI Display Panel
+
+maintainers:
+  - Maxime Ripard <maxime.ripard@bootlin.com>
+
+properties:
+  compatible:
+    const: ronbo,rb070d30
+
+  reg:
+    description: MIPI-DSI virtual channel
+
+  power-gpios:
+    description: GPIO used for the power pin
+    maxItems: 1
+
+  reset-gpios:
+    description: GPIO used for the reset pin
+    maxItems: 1
+
+  shlr-gpios:
+    description: GPIO used for the shlr pin (horizontal flip)
+    maxItems: 1
+
+  updn-gpios:
+    description: GPIO used for the updn pin (vertical flip)
+    maxItems: 1
+
+  vcc-lcd-supply:
+    description: Power regulator
+
+  backlight:
+    description: Backlight used by the panel
+    $ref: "/schemas/types.yaml#/definitions/phandle"
+
+required:
+  - compatible
+  - power-gpios
+  - reg
+  - reset-gpios
+  - shlr-gpios
+  - updn-gpios
+  - vcc-lcd-supply
+
+additionalProperties: false
diff --git a/Documentation/devicetree/bindings/display/panel/tpo,td028ttec1.txt b/Documentation/devicetree/bindings/display/panel/tpo,td028ttec1.txt
index ed34253d9fb1..898e06ecf4ef 100644
--- a/Documentation/devicetree/bindings/display/panel/tpo,td028ttec1.txt
+++ b/Documentation/devicetree/bindings/display/panel/tpo,td028ttec1.txt
@@ -6,6 +6,7 @@ Required properties:
 
 Optional properties:
 - label: a symbolic name for the panel
+- backlight: phandle of the backlight device
 
 Required nodes:
 - Video port for DPI input
@@ -21,6 +22,7 @@ lcd-panel: td028ttec1@0 {
 	spi-cpha;
 
 	label = "lcd";
+	backlight = <&backlight>;
 	port {
 		lcd_in: endpoint {
 			remote-endpoint = <&dpi_out>;
diff --git a/Documentation/devicetree/bindings/display/rockchip/rockchip,rk3066-hdmi.txt b/Documentation/devicetree/bindings/display/rockchip/rockchip,rk3066-hdmi.txt
new file mode 100644
index 000000000000..d1ad31bca8d9
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/rockchip/rockchip,rk3066-hdmi.txt
@@ -0,0 +1,72 @@
+Rockchip specific extensions for rk3066 HDMI
+============================================
+
+Required properties:
+- compatible:
+	"rockchip,rk3066-hdmi";
+- reg:
+	Physical base address and length of the controller's registers.
+- clocks, clock-names:
+	Phandle to HDMI controller clock, name should be "hclk".
+- interrupts:
+	HDMI interrupt number.
+- power-domains:
+	Phandle to the RK3066_PD_VIO power domain.
+- rockchip,grf:
+	This soc uses GRF regs to switch the HDMI TX input between vop0 and vop1.
+- ports:
+	Contains one port node with two endpoints, numbered 0 and 1,
+	connected respectively to vop0 and vop1.
+	Contains one port node with one endpoint
+	connected to a hdmi-connector node.
+- pinctrl-0, pinctrl-name:
+	Switch the iomux for the HPD/I2C pins to HDMI function.
+
+Example:
+	hdmi: hdmi@10116000 {
+		compatible = "rockchip,rk3066-hdmi";
+		reg = <0x10116000 0x2000>;
+		interrupts = <GIC_SPI 64 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&cru HCLK_HDMI>;
+		clock-names = "hclk";
+		power-domains = <&power RK3066_PD_VIO>;
+		rockchip,grf = <&grf>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&hdmii2c_xfer>, <&hdmi_hpd>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			hdmi_in: port@0 {
+				reg = <0>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+				hdmi_in_vop0: endpoint@0 {
+					reg = <0>;
+					remote-endpoint = <&vop0_out_hdmi>;
+				};
+				hdmi_in_vop1: endpoint@1 {
+					reg = <1>;
+					remote-endpoint = <&vop1_out_hdmi>;
+				};
+			};
+			hdmi_out: port@1 {
+				reg = <1>;
+				hdmi_out_con: endpoint {
+					remote-endpoint = <&hdmi_con_in>;
+				};
+			};
+		};
+	};
+
+&pinctrl {
+		hdmi {
+			hdmi_hpd: hdmi-hpd {
+				rockchip,pins = <0 RK_PA0 1 &pcfg_pull_default>;
+			};
+			hdmii2c_xfer: hdmii2c-xfer {
+				rockchip,pins = <0 RK_PA1 1 &pcfg_pull_none>,
+						<0 RK_PA2 1 &pcfg_pull_none>;
+			};
+		};
+};
diff --git a/Documentation/devicetree/bindings/display/ste,mcde.txt b/Documentation/devicetree/bindings/display/ste,mcde.txt
new file mode 100644
index 000000000000..4c33c692bd5f
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/ste,mcde.txt
@@ -0,0 +1,104 @@
+ST-Ericsson Multi Channel Display Engine MCDE
+
+The ST-Ericsson MCDE is a display controller with support for compositing
+and displaying several channels memory resident graphics data on DSI or
+LCD displays or bridges. It is used in the ST-Ericsson U8500 platform.
+
+Required properties:
+
+- compatible: must be:
+  "ste,mcde"
+- reg: register base for the main MCDE control registers, should be
+  0x1000 in size
+- interrupts: the interrupt line for the MCDE
+- epod-supply: a phandle to the EPOD regulator
+- vana-supply: a phandle to the analog voltage regulator
+- clocks: an array of the MCDE clocks in this strict order:
+  MCDECLK (main MCDE clock), LCDCLK (LCD clock), PLLDSI
+  (HDMI clock), DSI0ESCLK (DSI0 energy save clock),
+  DSI1ESCLK (DSI1 energy save clock), DSI2ESCLK (DSI2 energy
+  save clock)
+- clock-names: must be the following array:
+  "mcde", "lcd", "hdmi"
+  to match the required clock inputs above.
+- #address-cells: should be <1> (for the DSI hosts that will be children)
+- #size-cells: should be <1> (for the DSI hosts that will be children)
+- ranges: this should always be stated
+
+Required subnodes:
+
+The devicetree must specify subnodes for the DSI host adapters.
+These must have the following characteristics:
+
+- compatible: must be:
+  "ste,mcde-dsi"
+- reg: must specify the register range for the DSI host
+- vana-supply: phandle to the VANA voltage regulator
+- clocks: phandles to the high speed and low power (energy save) clocks
+  the high speed clock is not present on the third (dsi2) block, so it
+  should only have the "lp" clock
+- clock-names: "hs" for the high speed clock and "lp" for the low power
+  (energy save) clock
+- #address-cells: should be <1>
+- #size-cells: should be <0>
+
+Display panels and bridges will appear as children on the DSI hosts, and
+the displays are connected to the DSI hosts using the common binding
+for video transmitter interfaces; see
+Documentation/devicetree/bindings/media/video-interfaces.txt
+
+If a DSI host is unused (not connected) it will have no children defined.
+
+Example:
+
+mcde@a0350000 {
+	compatible = "ste,mcde";
+	reg = <0xa0350000 0x1000>;
+	interrupts = <GIC_SPI 48 IRQ_TYPE_LEVEL_HIGH>;
+	epod-supply = <&db8500_b2r2_mcde_reg>;
+	vana-supply = <&ab8500_ldo_ana_reg>;
+	clocks = <&prcmu_clk PRCMU_MCDECLK>, /* Main MCDE clock */
+		 <&prcmu_clk PRCMU_LCDCLK>, /* LCD clock */
+		 <&prcmu_clk PRCMU_PLLDSI>; /* HDMI clock */
+	clock-names = "mcde", "lcd", "hdmi";
+	#address-cells = <1>;
+	#size-cells = <1>;
+	ranges;
+
+	dsi0: dsi@a0351000 {
+		compatible = "ste,mcde-dsi";
+		reg = <0xa0351000 0x1000>;
+		vana-supply = <&ab8500_ldo_ana_reg>;
+		clocks = <&prcmu_clk PRCMU_DSI0CLK>, <&prcmu_clk PRCMU_DSI0ESCCLK>;
+		clock-names = "hs", "lp";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		panel {
+			compatible = "samsung,s6d16d0";
+			reg = <0>;
+			vdd1-supply = <&ab8500_ldo_aux1_reg>;
+			reset-gpios = <&gpio2 1 GPIO_ACTIVE_LOW>;
+		};
+
+	};
+	dsi1: dsi@a0352000 {
+		compatible = "ste,mcde-dsi";
+		reg = <0xa0352000 0x1000>;
+		vana-supply = <&ab8500_ldo_ana_reg>;
+		clocks = <&prcmu_clk PRCMU_DSI1CLK>, <&prcmu_clk PRCMU_DSI1ESCCLK>;
+		clock-names = "hs", "lp";
+		#address-cells = <1>;
+		#size-cells = <0>;
+	};
+	dsi2: dsi@a0353000 {
+		compatible = "ste,mcde-dsi";
+		reg = <0xa0353000 0x1000>;
+		vana-supply = <&ab8500_ldo_ana_reg>;
+		/* This DSI port only has the Low Power / Energy Save clock */
+		clocks = <&prcmu_clk PRCMU_DSI2ESCCLK>;
+		clock-names = "lp";
+		#address-cells = <1>;
+		#size-cells = <0>;
+	};
+};
diff --git a/Documentation/devicetree/bindings/dma/adi,axi-dmac.txt b/Documentation/devicetree/bindings/dma/adi,axi-dmac.txt
index 47cb1d14b690..b38ee732efa9 100644
--- a/Documentation/devicetree/bindings/dma/adi,axi-dmac.txt
+++ b/Documentation/devicetree/bindings/dma/adi,axi-dmac.txt
@@ -18,7 +18,6 @@ Required properties for adi,channels sub-node:
 
 Required channel sub-node properties:
  - reg: Which channel this node refers to.
- - adi,length-width: Width of the DMA transfer length register.
  - adi,source-bus-width,
    adi,destination-bus-width: Width of the source or destination bus in bits.
  - adi,source-bus-type,
@@ -28,7 +27,8 @@ Required channel sub-node properties:
 	1 (AXI_DMAC_TYPE_AXI_STREAM): Streaming AXI interface
 	2 (AXI_DMAC_TYPE_AXI_FIFO): FIFO interface
 
-Optional channel properties:
+Deprecated optional channel properties:
+ - adi,length-width: Width of the DMA transfer length register.
  - adi,cyclic: Must be set if the channel supports hardware cyclic DMA
    transfers.
  - adi,2d: Must be set if the channel supports hardware 2D DMA transfers.
diff --git a/Documentation/devicetree/bindings/dma/nvidia,tegra210-adma.txt b/Documentation/devicetree/bindings/dma/nvidia,tegra210-adma.txt
index 2f35b047f772..245d3063715c 100644
--- a/Documentation/devicetree/bindings/dma/nvidia,tegra210-adma.txt
+++ b/Documentation/devicetree/bindings/dma/nvidia,tegra210-adma.txt
@@ -4,7 +4,9 @@ The Tegra Audio DMA controller that is used for transferring data
 between system memory and the Audio Processing Engine (APE).
 
 Required properties:
-- compatible: Must be "nvidia,tegra210-adma".
+- compatible: Should contain one of the following:
+  - "nvidia,tegra210-adma": for Tegra210
+  - "nvidia,tegra186-adma": for Tegra186 and Tegra194
 - reg: Should contain DMA registers location and length. This should be
   a single entry that includes all of the per-channel registers in one
   contiguous bank.
diff --git a/Documentation/devicetree/bindings/eeprom/at24.txt b/Documentation/devicetree/bindings/eeprom/at24.txt
index 0e456bbc1213..22aead844d0f 100644
--- a/Documentation/devicetree/bindings/eeprom/at24.txt
+++ b/Documentation/devicetree/bindings/eeprom/at24.txt
@@ -50,6 +50,7 @@ Required properties:
 
                 "nxp,se97b" - the fallback is "atmel,24c02",
                 "renesas,r1ex24002" - the fallback is "atmel,24c02"
+                "renesas,r1ex24016" - the fallback is "atmel,24c16"
                 "renesas,r1ex24128" - the fallback is "atmel,24c128"
                 "rohm,br24t01" - the fallback is "atmel,24c01"
 
diff --git a/Documentation/devicetree/bindings/gpu/arm,mali-bifrost.txt b/Documentation/devicetree/bindings/gpu/arm,mali-bifrost.txt
new file mode 100644
index 000000000000..b8be9dbc68b4
--- /dev/null
+++ b/Documentation/devicetree/bindings/gpu/arm,mali-bifrost.txt
@@ -0,0 +1,92 @@
+ARM Mali Bifrost GPU
+====================
+
+Required properties:
+
+- compatible :
+  * Since Mali Bifrost GPU model/revision is fully discoverable by reading
+    some determined registers, must contain the following:
+    + "arm,mali-bifrost"
+  * which must be preceded by one of the following vendor specifics:
+    + "amlogic,meson-g12a-mali"
+
+- reg : Physical base address of the device and length of the register area.
+
+- interrupts : Contains the three IRQ lines required by Mali Bifrost devices,
+  in the following defined order.
+
+- interrupt-names : Contains the names of IRQ resources in this exact defined
+  order: "job", "mmu", "gpu".
+
+Optional properties:
+
+- clocks : Phandle to clock for the Mali Bifrost device.
+
+- mali-supply : Phandle to regulator for the Mali device. Refer to
+  Documentation/devicetree/bindings/regulator/regulator.txt for details.
+
+- operating-points-v2 : Refer to Documentation/devicetree/bindings/opp/opp.txt
+  for details.
+
+- resets : Phandle of the GPU reset line.
+
+Vendor-specific bindings
+------------------------
+
+The Mali GPU is integrated very differently from one SoC to
+another. In order to accommodate those differences, you have the option
+to specify one more vendor-specific compatible, among:
+
+- "amlogic,meson-g12a-mali"
+  Required properties:
+  - resets : Should contain phandles of :
+    + GPU reset line
+    + GPU APB glue reset line
+
+Example for a Mali-G31:
+
+gpu@ffa30000 {
+	compatible = "amlogic,meson-g12a-mali", "arm,mali-bifrost";
+	reg = <0xffe40000 0x10000>;
+	interrupts = <GIC_SPI 160 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 161 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 162 IRQ_TYPE_LEVEL_HIGH>;
+	interrupt-names = "job", "mmu", "gpu";
+	clocks = <&clk CLKID_MALI>;
+	mali-supply = <&vdd_gpu>;
+	operating-points-v2 = <&gpu_opp_table>;
+	resets = <&reset RESET_DVALIN_CAPB3>, <&reset RESET_DVALIN>;
+};
+
+gpu_opp_table: opp_table0 {
+	compatible = "operating-points-v2";
+
+	opp@533000000 {
+		opp-hz = /bits/ 64 <533000000>;
+		opp-microvolt = <1250000>;
+	};
+	opp@450000000 {
+		opp-hz = /bits/ 64 <450000000>;
+		opp-microvolt = <1150000>;
+	};
+	opp@400000000 {
+		opp-hz = /bits/ 64 <400000000>;
+		opp-microvolt = <1125000>;
+	};
+	opp@350000000 {
+		opp-hz = /bits/ 64 <350000000>;
+		opp-microvolt = <1075000>;
+	};
+	opp@266000000 {
+		opp-hz = /bits/ 64 <266000000>;
+		opp-microvolt = <1025000>;
+	};
+	opp@160000000 {
+		opp-hz = /bits/ 64 <160000000>;
+		opp-microvolt = <925000>;
+	};
+	opp@100000000 {
+		opp-hz = /bits/ 64 <100000000>;
+		opp-microvolt = <912500>;
+	};
+};
diff --git a/Documentation/devicetree/bindings/gpu/aspeed-gfx.txt b/Documentation/devicetree/bindings/gpu/aspeed-gfx.txt
new file mode 100644
index 000000000000..958bdf962339
--- /dev/null
+++ b/Documentation/devicetree/bindings/gpu/aspeed-gfx.txt
@@ -0,0 +1,41 @@
+Device tree configuration for the GFX display device on the ASPEED SoCs
+
+Required properties:
+  - compatible
+    * Must be one of the following:
+      + aspeed,ast2500-gfx
+      + aspeed,ast2400-gfx
+    * In addition, the ASPEED pinctrl bindings require the 'syscon' property to
+      be present
+
+  - reg: Physical base address and length of the GFX registers
+
+  - interrupts: interrupt number for the GFX device
+
+  - clocks: clock number used to generate the pixel clock
+
+  - resets: reset line that must be released to use the GFX device
+
+  - memory-region:
+    Phandle to a memory region to allocate from, as defined in
+    Documentation/devicetree/bindings/reserved-memory/reserved-memory.txt
+
+
+Example:
+
+gfx: display@1e6e6000 {
+	compatible = "aspeed,ast2500-gfx", "syscon";
+	reg = <0x1e6e6000 0x1000>;
+	reg-io-width = <4>;
+	clocks = <&syscon ASPEED_CLK_GATE_D1CLK>;
+	resets = <&syscon ASPEED_RESET_CRT1>;
+	interrupts = <0x19>;
+	memory-region = <&gfx_memory>;
+};
+
+gfx_memory: framebuffer {
+	size = <0x01000000>;
+	alignment = <0x01000000>;
+	compatible = "shared-dma-pool";
+	reusable;
+};
diff --git a/Documentation/devicetree/bindings/gpu/brcm,bcm-v3d.txt b/Documentation/devicetree/bindings/gpu/brcm,bcm-v3d.txt
index c907aa8dd755..b2df82b44625 100644
--- a/Documentation/devicetree/bindings/gpu/brcm,bcm-v3d.txt
+++ b/Documentation/devicetree/bindings/gpu/brcm,bcm-v3d.txt
@@ -6,15 +6,20 @@ For V3D 2.x, see brcm,bcm-vc4.txt.
 Required properties:
 - compatible:	Should be "brcm,7268-v3d" or "brcm,7278-v3d"
 - reg:		Physical base addresses and lengths of the register areas
-- reg-names:	Names for the register areas.  The "hub", "bridge", and "core0"
+- reg-names:	Names for the register areas.  The "hub" and "core0"
 		  register areas are always required.  The "gca" register area
-		  is required if the GCA cache controller is present.
+		  is required if the GCA cache controller is present.  The
+		  "bridge" register area is required if an external reset
+		  controller is not present.
 - interrupts:	The interrupt numbers.  The first interrupt is for the hub,
-		  while the following interrupts are for the cores.
+		  while the following interrupts are separate interrupt lines
+		  for the cores (if they don't share the hub's interrupt).
 		  See bindings/interrupt-controller/interrupts.txt
 
 Optional properties:
 - clocks:	The core clock the unit runs on
+- resets:	The reset line for v3d, if not using a mapping of the bridge
+		  See bindings/reset/reset.txt
 
 v3d {
 	compatible = "brcm,7268-v3d";
diff --git a/Documentation/devicetree/bindings/i2c/brcm,iproc-i2c.txt b/Documentation/devicetree/bindings/i2c/brcm,iproc-i2c.txt
index 81f982ccca31..d12cc33cca6c 100644
--- a/Documentation/devicetree/bindings/i2c/brcm,iproc-i2c.txt
+++ b/Documentation/devicetree/bindings/i2c/brcm,iproc-i2c.txt
@@ -3,15 +3,12 @@ Broadcom iProc I2C controller
 Required properties:
 
 - compatible:
-    Must be "brcm,iproc-i2c"
+    Must be "brcm,iproc-i2c" or "brcm,iproc-nic-i2c"
 
 - reg:
     Define the base and range of the I/O address space that contain the iProc
     I2C controller registers
 
-- interrupts:
-    Should contain the I2C interrupt
-
 - clock-frequency:
     This is the I2C bus clock. Need to be either 100000 or 400000
 
@@ -21,6 +18,18 @@ Required properties:
 - #size-cells:
     Always 0
 
+Optional properties:
+
+- interrupts:
+    Should contain the I2C interrupt. For certain revisions of the I2C
+    controller, I2C interrupt is unwired to the interrupt controller. In such
+    case, this property should be left unspecified, and driver will fall back
+    to polling mode
+
+- brcm,ape-hsls-addr-mask:
+    Required for "brcm,iproc-nic-i2c". Host view of address mask into the
+    'APE' co-processor. Value must be unsigned, 32-bit
+
 Example:
 	i2c0: i2c@18008000 {
 		compatible = "brcm,iproc-i2c";
diff --git a/Documentation/devicetree/bindings/i2c/i2c-designware.txt b/Documentation/devicetree/bindings/i2c/i2c-designware.txt
index 3e4bcc2fb6f7..08be4d3846e5 100644
--- a/Documentation/devicetree/bindings/i2c/i2c-designware.txt
+++ b/Documentation/devicetree/bindings/i2c/i2c-designware.txt
@@ -6,12 +6,21 @@ Required properties :
                 or "mscc,ocelot-i2c" with "snps,designware-i2c" for fallback
  - reg : Offset and length of the register set for the device
  - interrupts : <IRQ> where IRQ is the interrupt number.
+ - clocks : phandles for the clocks, see the description of clock-names below.
+   The phandle for the "ic_clk" clock is required. The phandle for the "pclk"
+   clock is optional. If a single clock is specified but no clock-name, it is
+   the "ic_clk" clock. If both clocks are listed, the "ic_clk" must be first.
 
 Recommended properties :
 
  - clock-frequency : desired I2C bus clock frequency in Hz.
 
 Optional properties :
+
+ - clock-names : Contains the names of the clocks:
+    "ic_clk", for the core clock used to generate the external I2C clock.
+    "pclk", the interface clock, required for register access.
+
  - reg : for "mscc,ocelot-i2c", a second register set to configure the SDA hold
    time, named ICPU_CFG:TWI_DELAY in the datasheet.
 
diff --git a/Documentation/devicetree/bindings/i2c/i2c-mt65xx.txt b/Documentation/devicetree/bindings/i2c/i2c-mt65xx.txt
index ee4c32454198..68f6d73a8b73 100644
--- a/Documentation/devicetree/bindings/i2c/i2c-mt65xx.txt
+++ b/Documentation/devicetree/bindings/i2c/i2c-mt65xx.txt
@@ -12,13 +12,16 @@ Required properties:
       "mediatek,mt7623-i2c", "mediatek,mt6577-i2c": for MediaTek MT7623
       "mediatek,mt7629-i2c", "mediatek,mt2712-i2c": for MediaTek MT7629
       "mediatek,mt8173-i2c": for MediaTek MT8173
+      "mediatek,mt8183-i2c": for MediaTek MT8183
+      "mediatek,mt8516-i2c", "mediatek,mt2712-i2c": for MediaTek MT8516
   - reg: physical base address of the controller and dma base, length of memory
     mapped region.
   - interrupts: interrupt number to the cpu.
   - clock-div: the fixed value for frequency divider of clock source in i2c
     module. Each IC may be different.
   - clocks: clock name from clock manager
-  - clock-names: Must include "main" and "dma", if enable have-pmic need include
+  - clock-names: Must include "main" and "dma", "arb" is for multi-master that
+    one bus has more than two i2c controllers, if enable have-pmic need include
     "pmic" extra.
 
 Optional properties:
diff --git a/Documentation/devicetree/bindings/i2c/i2c-riic.txt b/Documentation/devicetree/bindings/i2c/i2c-riic.txt
index 0bcc4716c319..e26fe3ad86a9 100644
--- a/Documentation/devicetree/bindings/i2c/i2c-riic.txt
+++ b/Documentation/devicetree/bindings/i2c/i2c-riic.txt
@@ -1,7 +1,10 @@
 Device tree configuration for Renesas RIIC driver
 
 Required properties:
-- compatible      : "renesas,riic-<soctype>". "renesas,riic-rz" as fallback
+- compatible      :
+	"renesas,riic-r7s72100" if the device is a part of a R7S72100 SoC.
+	"renesas,riic-r7s9210" if the device is a part of a R7S9210 SoC.
+	"renesas,riic-rz" for a generic RZ/A compatible device.
 - reg             : address start and address range size of device
 - interrupts      : 8 interrupts (TEI, RI, TI, SPI, STI, NAKI, ALI, TMOI)
 - clock-frequency : frequency of bus clock in Hz
diff --git a/Documentation/devicetree/bindings/i2c/i2c-stm32.txt b/Documentation/devicetree/bindings/i2c/i2c-stm32.txt
index 69240e189b01..f334738f7a35 100644
--- a/Documentation/devicetree/bindings/i2c/i2c-stm32.txt
+++ b/Documentation/devicetree/bindings/i2c/i2c-stm32.txt
@@ -1,11 +1,11 @@
 * I2C controller embedded in STMicroelectronics STM32 I2C platform
 
-Required properties :
-- compatible : Must be one of the following
+Required properties:
+- compatible: Must be one of the following
   - "st,stm32f4-i2c"
   - "st,stm32f7-i2c"
-- reg : Offset and length of the register set for the device
-- interrupts : Must contain the interrupt id for I2C event and then the
+- reg: Offset and length of the register set for the device
+- interrupts: Must contain the interrupt id for I2C event and then the
   interrupt id for I2C error.
 - resets: Must contain the phandle to the reset controller.
 - clocks: Must contain the input clock of the I2C instance.
@@ -14,25 +14,26 @@ Required properties :
 - #address-cells = <1>;
 - #size-cells = <0>;
 
-Optional properties :
-- clock-frequency : Desired I2C bus clock frequency in Hz. If not specified,
+Optional properties:
+- clock-frequency: Desired I2C bus clock frequency in Hz. If not specified,
   the default 100 kHz frequency will be used.
   For STM32F4 SoC Standard-mode and Fast-mode are supported, possible values are
   100000 and 400000.
-  For STM32F7 SoC, Standard-mode, Fast-mode and Fast-mode Plus are supported,
-  possible values are 100000, 400000 and 1000000.
-- i2c-scl-rising-time-ns : Only for STM32F7, I2C SCL Rising time for the board
-  (default: 25)
-- i2c-scl-falling-time-ns : Only for STM32F7, I2C SCL Falling time for the board
-  (default: 10)
+  For STM32F7, STM32H7 and STM32MP1 SoCs, Standard-mode, Fast-mode and Fast-mode
+  Plus are supported, possible values are 100000, 400000 and 1000000.
+- i2c-scl-rising-time-ns: I2C SCL Rising time for the board (default: 25)
+  For STM32F7, STM32H7 and STM32MP1 only.
+- i2c-scl-falling-time-ns: I2C SCL Falling time for the board (default: 10)
+  For STM32F7, STM32H7 and STM32MP1 only.
   I2C Timings are derived from these 2 values
-- st,syscfg-fmp:  Only for STM32F7, use to set Fast Mode Plus bit within SYSCFG
-  whether Fast Mode Plus speed is selected by slave.
-	1st cell : phandle to syscfg
-	2nd cell : register offset within SYSCFG
-	3rd cell : register bitmask for FMP bit
+- st,syscfg-fmp: Use to set Fast Mode Plus bit within SYSCFG when Fast Mode
+  Plus speed is selected by slave.
+	1st cell: phandle to syscfg
+	2nd cell: register offset within SYSCFG
+	3rd cell: register bitmask for FMP bit
+  For STM32F7, STM32H7 and STM32MP1 only.
 
-Example :
+Example:
 
 	i2c@40005400 {
 		compatible = "st,stm32f4-i2c";
diff --git a/Documentation/devicetree/bindings/media/aspeed-video.txt b/Documentation/devicetree/bindings/media/aspeed-video.txt
index 78b464ae2672..ce2894506e1f 100644
--- a/Documentation/devicetree/bindings/media/aspeed-video.txt
+++ b/Documentation/devicetree/bindings/media/aspeed-video.txt
@@ -14,6 +14,11 @@ Required properties:
 			the VE
  - interrupts:		the interrupt associated with the VE on this platform
 
+Optional properties:
+ - memory-region:
+	phandle to a memory region to allocate from, as defined in
+	Documentation/devicetree/bindings/reserved-memory/reserved-memory.txt
+
 Example:
 
 video-engine@1e700000 {
@@ -23,4 +28,5 @@ video-engine@1e700000 {
     clock-names = "vclk", "eclk";
     resets = <&syscon ASPEED_RESET_VIDEO>;
     interrupts = <7>;
+    memory-region = <&video_engine_memory>;
 };
diff --git a/Documentation/devicetree/bindings/media/cedrus.txt b/Documentation/devicetree/bindings/media/cedrus.txt
index bce0705df953..20c82fb0c343 100644
--- a/Documentation/devicetree/bindings/media/cedrus.txt
+++ b/Documentation/devicetree/bindings/media/cedrus.txt
@@ -13,6 +13,7 @@ Required properties:
 			- "allwinner,sun8i-h3-video-engine"
 			- "allwinner,sun50i-a64-video-engine"
 			- "allwinner,sun50i-h5-video-engine"
+			- "allwinner,sun50i-h6-video-engine"
 - reg			: register base and length of VE;
 - clocks		: list of clock specifiers, corresponding to entries in
 			  the clock-names property;
diff --git a/Documentation/devicetree/bindings/media/i2c/st,st-mipid02.txt b/Documentation/devicetree/bindings/media/i2c/st,st-mipid02.txt
new file mode 100644
index 000000000000..7976e6c40a80
--- /dev/null
+++ b/Documentation/devicetree/bindings/media/i2c/st,st-mipid02.txt
@@ -0,0 +1,82 @@
+STMicroelectronics MIPID02 CSI-2 to PARALLEL bridge
+
+MIPID02 has two CSI-2 input ports, only one of those ports can be active at a
+time. Active port input stream will be de-serialized and its content outputted
+through PARALLEL output port.
+CSI-2 first input port is a dual lane 800Mbps per lane whereas CSI-2 second
+input port is a single lane 800Mbps. Both ports support clock and data lane
+polarity swap. First port also supports data lane swap.
+PARALLEL output port has a maximum width of 12 bits.
+Supported formats are RAW6, RAW7, RAW8, RAW10, RAW12, RGB565, RGB888, RGB444,
+YUV420 8-bit, YUV422 8-bit and YUV420 10-bit.
+
+Required Properties:
+- compatible: shall be "st,st-mipid02"
+- clocks: reference to the xclk input clock.
+- clock-names: shall be "xclk".
+- VDDE-supply: sensor digital IO supply. Must be 1.8 volts.
+- VDDIN-supply: sensor internal regulator supply. Must be 1.8 volts.
+
+Optional Properties:
+- reset-gpios: reference to the GPIO connected to the xsdn pin, if any.
+	       This is an active low signal to the mipid02.
+
+Required subnodes:
+  - ports: A ports node with one port child node per device input and output
+	   port, in accordance with the video interface bindings defined in
+	   Documentation/devicetree/bindings/media/video-interfaces.txt. The
+	   port nodes are numbered as follows:
+
+	   Port Description
+	   -----------------------------
+	   0    CSI-2 first input port
+	   1    CSI-2 second input port
+	   2    PARALLEL output
+
+Endpoint node required property for CSI-2 connection is:
+- data-lanes: shall be <1> for Port 1. for Port 0 dual-lane operation shall be
+<1 2> or <2 1>. For Port 0 single-lane operation shall be <1> or <2>.
+Endpoint node optional property for CSI-2 connection is:
+- lane-polarities: any lane can be inverted or not.
+
+Endpoint node required property for PARALLEL connection is:
+- bus-width: shall be set to <6>, <7>, <8>, <10> or <12>.
+Endpoint node optional properties for PARALLEL connection are:
+- hsync-active: active state of the HSYNC signal, 0/1 for LOW/HIGH respectively.
+LOW being the default.
+- vsync-active: active state of the VSYNC signal, 0/1 for LOW/HIGH respectively.
+LOW being the default.
+
+Example:
+
+mipid02: csi2rx@14 {
+	compatible = "st,st-mipid02";
+	reg = <0x14>;
+	status = "okay";
+	clocks = <&clk_ext_camera_12>;
+	clock-names = "xclk";
+	VDDE-supply = <&vdd>;
+	VDDIN-supply = <&vdd>;
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		port@0 {
+			reg = <0>;
+
+			ep0: endpoint {
+				data-lanes = <1 2>;
+				remote-endpoint = <&mipi_csi2_in>;
+			};
+		};
+		port@2 {
+			reg = <2>;
+
+			ep2: endpoint {
+				bus-width = <8>;
+				hsync-active = <0>;
+				vsync-active = <0>;
+				remote-endpoint = <&parallel_out>;
+			};
+		};
+	};
+};
diff --git a/Documentation/devicetree/bindings/media/meson-ao-cec.txt b/Documentation/devicetree/bindings/media/meson-ao-cec.txt
index 8671bdb08080..c67fc41d4aa2 100644
--- a/Documentation/devicetree/bindings/media/meson-ao-cec.txt
+++ b/Documentation/devicetree/bindings/media/meson-ao-cec.txt
@@ -4,16 +4,23 @@ The Amlogic Meson AO-CEC module is present is Amlogic SoCs and its purpose is
 to handle communication between HDMI connected devices over the CEC bus.
 
 Required properties:
-  - compatible : value should be following
+  - compatible : value should be following depending on the SoC :
+	For GXBB, GXL, GXM and G12A (AO_CEC_A module) :
 	"amlogic,meson-gx-ao-cec"
+	For G12A (AO_CEC_B module) :
+	"amlogic,meson-g12a-ao-cec"
 
   - reg : Physical base address of the IP registers and length of memory
 	  mapped region.
 
   - interrupts : AO-CEC interrupt number to the CPU.
   - clocks : from common clock binding: handle to AO-CEC clock.
-  - clock-names : from common clock binding: must contain "core",
-		  corresponding to entry in the clocks property.
+  - clock-names : from common clock binding, must contain :
+		For GXBB, GXL, GXM and G12A (AO_CEC_A module) :
+		- "core"
+		For G12A (AO_CEC_B module) :
+		- "oscin"
+		corresponding to entry in the clocks property.
   - hdmi-phandle: phandle to the HDMI controller
 
 Example:
diff --git a/Documentation/devicetree/bindings/media/rcar_imr.txt b/Documentation/devicetree/bindings/media/rcar_imr.txt
new file mode 100644
index 000000000000..b0614153ed36
--- /dev/null
+++ b/Documentation/devicetree/bindings/media/rcar_imr.txt
@@ -0,0 +1,31 @@
+Renesas R-Car Image Renderer (Distortion Correction Engine)
+-----------------------------------------------------------
+
+The image renderer, or the distortion correction engine, is a drawing processor
+with a simple instruction system capable of referencing video capture data or
+data in an external memory as 2D texture data and performing texture mapping
+and drawing with respect to any shape that is split into triangular objects.
+
+Required properties:
+
+- compatible: "renesas,<soctype>-imr-lx4", "renesas,imr-lx4" as a fallback for
+  the image renderer light extended 4 (IMR-LX4) found in the R-Car gen3 SoCs,
+  where the examples with <soctype> are:
+  - "renesas,r8a7795-imr-lx4" for R-Car H3,
+  - "renesas,r8a7796-imr-lx4" for R-Car M3-W.
+- reg: offset and length of the register block;
+- interrupts: single interrupt specifier;
+- clocks: single clock phandle/specifier pair;
+- power-domains: power domain phandle/specifier pair;
+- resets: reset phandle/specifier pair.
+
+Example:
+
+	imr-lx4@fe860000 {
+		compatible = "renesas,r8a7795-imr-lx4", "renesas,imr-lx4";
+		reg = <0 0xfe860000 0 0x2000>;
+		interrupts = <GIC_SPI 192 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&cpg CPG_MOD 823>;
+		power-domains = <&sysc R8A7795_PD_A3VC>;
+		resets = <&cpg 823>;
+	};
diff --git a/Documentation/devicetree/bindings/media/rcar_vin.txt b/Documentation/devicetree/bindings/media/rcar_vin.txt
index 224a4615b418..aa217b096279 100644
--- a/Documentation/devicetree/bindings/media/rcar_vin.txt
+++ b/Documentation/devicetree/bindings/media/rcar_vin.txt
@@ -13,6 +13,7 @@ on Gen3 and RZ/G2 platforms to a CSI-2 receiver.
    - "renesas,vin-r8a7743" for the R8A7743 device
    - "renesas,vin-r8a7744" for the R8A7744 device
    - "renesas,vin-r8a7745" for the R8A7745 device
+   - "renesas,vin-r8a774a1" for the R8A774A1 device
    - "renesas,vin-r8a774c0" for the R8A774C0 device
    - "renesas,vin-r8a7778" for the R8A7778 device
    - "renesas,vin-r8a7779" for the R8A7779 device
diff --git a/Documentation/devicetree/bindings/media/renesas,rcar-csi2.txt b/Documentation/devicetree/bindings/media/renesas,rcar-csi2.txt
index d63275e17afd..331409259752 100644
--- a/Documentation/devicetree/bindings/media/renesas,rcar-csi2.txt
+++ b/Documentation/devicetree/bindings/media/renesas,rcar-csi2.txt
@@ -8,6 +8,7 @@ R-Car VIN module, which provides the video capture capabilities.
 Mandatory properties
 --------------------
  - compatible: Must be one or more of the following
+   - "renesas,r8a774a1-csi2" for the R8A774A1 device.
    - "renesas,r8a774c0-csi2" for the R8A774C0 device.
    - "renesas,r8a7795-csi2" for the R8A7795 device.
    - "renesas,r8a7796-csi2" for the R8A7796 device.
@@ -18,7 +19,8 @@ Mandatory properties
 
  - reg: the register base and size for the device registers
  - interrupts: the interrupt for the device
- - clocks: reference to the parent clock
+ - clocks: A phandle + clock specifier for the module clock
+ - resets: A phandle + reset specifier for the module reset
 
 The device node shall contain two 'port' child nodes according to the
 bindings defined in Documentation/devicetree/bindings/media/
diff --git a/Documentation/devicetree/bindings/phy/brcm,stingray-usb-phy.txt b/Documentation/devicetree/bindings/phy/brcm,stingray-usb-phy.txt
new file mode 100644
index 000000000000..4ba298966af9
--- /dev/null
+++ b/Documentation/devicetree/bindings/phy/brcm,stingray-usb-phy.txt
@@ -0,0 +1,32 @@
+Broadcom Stingray USB PHY
+
+Required properties:
+ - compatible : should be one of the listed compatibles
+	- "brcm,sr-usb-combo-phy" is combo PHY has two PHYs, one SS and one HS.
+	- "brcm,sr-usb-hs-phy" is a single HS PHY.
+ - reg: offset and length of the PHY blocks registers
+ - #phy-cells:
+   - Must be 1 for brcm,sr-usb-combo-phy as it expects one argument to indicate
+     the PHY number of two PHYs. 0 for HS PHY and 1 for SS PHY.
+   - Must be 0 for brcm,sr-usb-hs-phy.
+
+Refer to phy/phy-bindings.txt for the generic PHY binding properties
+
+Example:
+	usbphy0: usb-phy@0 {
+		compatible = "brcm,sr-usb-combo-phy";
+		reg = <0x00000000 0x100>;
+		#phy-cells = <1>;
+	};
+
+	usbphy1: usb-phy@10000 {
+		compatible = "brcm,sr-usb-combo-phy";
+		reg = <0x00010000 0x100>,
+		#phy-cells = <1>;
+	};
+
+	usbphy2: usb-phy@20000 {
+		compatible = "brcm,sr-usb-hs-phy";
+		reg = <0x00020000 0x100>,
+		#phy-cells = <0>;
+	};
diff --git a/Documentation/devicetree/bindings/phy/fsl,imx8mq-usb-phy.txt b/Documentation/devicetree/bindings/phy/fsl,imx8mq-usb-phy.txt
index a22e853d710c..ed47e5cd067e 100644
--- a/Documentation/devicetree/bindings/phy/fsl,imx8mq-usb-phy.txt
+++ b/Documentation/devicetree/bindings/phy/fsl,imx8mq-usb-phy.txt
@@ -7,6 +7,9 @@ Required properties:
 - clocks:	phandles to the clocks for each clock listed in clock-names
 - clock-names:	must contain "phy"
 
+Optional properties:
+- vbus-supply: A phandle to the regulator for USB VBUS.
+
 Example:
 	usb3_phy0: phy@381f0040 {
 		compatible = "fsl,imx8mq-usb-phy";
diff --git a/Documentation/devicetree/bindings/phy/meson-g12a-usb2-phy.txt b/Documentation/devicetree/bindings/phy/meson-g12a-usb2-phy.txt
new file mode 100644
index 000000000000..a6ebc3dea159
--- /dev/null
+++ b/Documentation/devicetree/bindings/phy/meson-g12a-usb2-phy.txt
@@ -0,0 +1,22 @@
+* Amlogic G12A USB2 PHY binding
+
+Required properties:
+- compatible:	Should be "amlogic,meson-g12a-usb2-phy"
+- reg:		The base address and length of the registers
+- #phys-cells:	must be 0 (see phy-bindings.txt in this directory)
+- clocks:	a phandle to the clock of this PHY
+- clock-names:	must be "xtal"
+- resets:	a phandle to the reset line of this PHY
+- reset-names:	must be "phy"
+- phy-supply:	see phy-bindings.txt in this directory
+
+Example:
+	usb2_phy0: phy@36000 {
+		compatible = "amlogic,g12a-usb2-phy";
+		reg = <0x0 0x36000 0x0 0x2000>;
+		clocks = <&xtal>;
+		clock-names = "xtal";
+		resets = <&reset RESET_USB_PHY21>;
+		reset-names = "phy";
+		#phy-cells = <0>;
+	};
diff --git a/Documentation/devicetree/bindings/phy/meson-g12a-usb3-pcie-phy.txt b/Documentation/devicetree/bindings/phy/meson-g12a-usb3-pcie-phy.txt
new file mode 100644
index 000000000000..7cfc17e2df31
--- /dev/null
+++ b/Documentation/devicetree/bindings/phy/meson-g12a-usb3-pcie-phy.txt
@@ -0,0 +1,22 @@
+* Amlogic G12A USB3 + PCIE Combo PHY binding
+
+Required properties:
+- compatible:	Should be "amlogic,meson-g12a-usb3-pcie-phy"
+- #phys-cells:	must be 1. The cell number is used to select the phy mode
+  as defined in <dt-bindings/phy/phy.h> between PHY_TYPE_USB3 and PHY_TYPE_PCIE
+- reg:		The base address and length of the registers
+- clocks:	a phandle to the 100MHz reference clock of this PHY
+- clock-names:	must be "ref_clk"
+- resets:	phandle to the reset lines for the PHY control
+- reset-names:	must be "phy"
+
+Example:
+	usb3_pcie_phy: phy@46000 {
+		compatible = "amlogic,g12a-usb3-pcie-phy";
+		reg = <0x0 0x46000 0x0 0x2000>;
+		clocks = <&clkc CLKID_PCIE_PLL>;
+		clock-names = "ref_clk";
+		resets = <&reset RESET_PCIE_PHY>;
+		reset-names = "phy";
+		#phy-cells = <1>;
+	};
diff --git a/Documentation/devicetree/bindings/phy/nvidia,tegra124-xusb-padctl.txt b/Documentation/devicetree/bindings/phy/nvidia,tegra124-xusb-padctl.txt
index 3742c152c467..daedb15f322e 100644
--- a/Documentation/devicetree/bindings/phy/nvidia,tegra124-xusb-padctl.txt
+++ b/Documentation/devicetree/bindings/phy/nvidia,tegra124-xusb-padctl.txt
@@ -36,11 +36,20 @@ Required properties:
   - Tegra124: "nvidia,tegra124-xusb-padctl"
   - Tegra132: "nvidia,tegra132-xusb-padctl", "nvidia,tegra124-xusb-padctl"
   - Tegra210: "nvidia,tegra210-xusb-padctl"
+  - Tegra186: "nvidia,tegra186-xusb-padctl"
 - reg: Physical base address and length of the controller's registers.
 - resets: Must contain an entry for each entry in reset-names.
 - reset-names: Must include the following entries:
   - "padctl"
 
+For Tegra186:
+- avdd-pll-erefeut-supply: UPHY brick and reference clock as well as UTMI PHY
+  power supply. Must supply 1.8 V.
+- avdd-usb-supply: USB I/Os, VBUS, ID, REXT, D+/D- power supply. Must supply
+  3.3 V.
+- vclamp-usb-supply: Bias rail for USB pad. Must supply 1.8 V.
+- vddio-hsic-supply: HSIC PHY power supply. Must supply 1.2 V.
+
 
 Pad nodes:
 ==========
diff --git a/Documentation/devicetree/bindings/phy/phy-hi3660-usb3.txt b/Documentation/devicetree/bindings/phy/phy-hi3660-usb3.txt
new file mode 100644
index 000000000000..e88ba7d92dcb
--- /dev/null
+++ b/Documentation/devicetree/bindings/phy/phy-hi3660-usb3.txt
@@ -0,0 +1,26 @@
+Hisilicon hi3660 USB PHY
+-----------------------
+
+Required properties:
+- compatible: should be "hisilicon,hi3660-usb-phy"
+- #phy-cells: must be 0
+- hisilicon,pericrg-syscon: phandle of syscon used to control phy.
+- hisilicon,pctrl-syscon: phandle of syscon used to control phy.
+- hisilicon,eye-diagram-param: parameter set for phy
+Refer to phy/phy-bindings.txt for the generic PHY binding properties
+
+This is a subnode of usb3_otg_bc register node.
+
+Example:
+	usb3_otg_bc: usb3_otg_bc@ff200000 {
+		compatible = "syscon", "simple-mfd";
+		reg = <0x0 0xff200000 0x0 0x1000>;
+
+		usb-phy {
+			compatible = "hisilicon,hi3660-usb-phy";
+			#phy-cells = <0>;
+			hisilicon,pericrg-syscon = <&crg_ctrl>;
+			hisilicon,pctrl-syscon = <&pctrl>;
+			hisilicon,eye-diagram-param = <0x22466e4>;
+		};
+	};
diff --git a/Documentation/devicetree/bindings/phy/phy-mtk-ufs.txt b/Documentation/devicetree/bindings/phy/phy-mtk-ufs.txt
new file mode 100644
index 000000000000..5789029a1d42
--- /dev/null
+++ b/Documentation/devicetree/bindings/phy/phy-mtk-ufs.txt
@@ -0,0 +1,38 @@
+MediaTek Universal Flash Storage (UFS) M-PHY binding
+--------------------------------------------------------
+
+UFS M-PHY nodes are defined to describe on-chip UFS M-PHY hardware macro.
+Each UFS M-PHY node should have its own node.
+
+To bind UFS M-PHY with UFS host controller, the controller node should
+contain a phandle reference to UFS M-PHY node.
+
+Required properties for UFS M-PHY nodes:
+- compatible         : Compatible list, contains the following controller:
+                       "mediatek,mt8183-ufsphy" for ufs phy
+                       persent on MT81xx chipsets.
+- reg                : Address and length of the UFS M-PHY register set.
+- #phy-cells         : This property shall be set to 0.
+- clocks             : List of phandle and clock specifier pairs.
+- clock-names        : List of clock input name strings sorted in the same
+                       order as the clocks property. Following clocks are
+                       mandatory.
+                       "unipro": Unipro core control clock.
+                       "mp": M-PHY core control clock.
+
+Example:
+
+	ufsphy: phy@11fa0000 {
+		compatible = "mediatek,mt8183-ufsphy";
+		reg = <0 0x11fa0000 0 0xc000>;
+		#phy-cells = <0>;
+
+		clocks = <&infracfg_ao INFRACFG_AO_UNIPRO_SCK_CG>,
+			 <&infracfg_ao INFRACFG_AO_UFS_MP_SAP_BCLK_CG>;
+		clock-names = "unipro", "mp";
+	};
+
+	ufshci@11270000 {
+		...
+		phys = <&ufsphy>;
+	};
diff --git a/Documentation/devicetree/bindings/phy/qcom-qmp-phy.txt b/Documentation/devicetree/bindings/phy/qcom-qmp-phy.txt
index 5d181fc3cc18..085fbd676cfc 100644
--- a/Documentation/devicetree/bindings/phy/qcom-qmp-phy.txt
+++ b/Documentation/devicetree/bindings/phy/qcom-qmp-phy.txt
@@ -11,6 +11,7 @@ Required properties:
 	       "qcom,msm8996-qmp-usb3-phy" for 14nm USB3 phy on msm8996,
 	       "qcom,msm8998-qmp-usb3-phy" for USB3 QMP V3 phy on msm8998,
 	       "qcom,msm8998-qmp-ufs-phy" for UFS QMP phy on msm8998,
+	       "qcom,msm8998-qmp-pcie-phy" for PCIe QMP phy on msm8998,
 	       "qcom,sdm845-qmp-usb3-phy" for USB3 QMP V3 phy on sdm845,
 	       "qcom,sdm845-qmp-usb3-uni-phy" for USB3 QMP V3 UNI phy on sdm845,
 	       "qcom,sdm845-qmp-ufs-phy" for UFS QMP phy on sdm845.
@@ -48,6 +49,8 @@ Required properties:
 			"aux", "cfg_ahb", "ref".
 		For "qcom,msm8998-qmp-ufs-phy" must contain:
 			"ref", "ref_aux".
+		For "qcom,msm8998-qmp-pcie-phy" must contain:
+			"aux", "cfg_ahb", "ref".
 		For "qcom,sdm845-qmp-usb3-phy" must contain:
 			"aux", "cfg_ahb", "ref", "com_aux".
 		For "qcom,sdm845-qmp-usb3-uni-phy" must contain:
@@ -59,7 +62,8 @@ Required properties:
 	   one for each entry in reset-names.
  - reset-names: "phy" for reset of phy block,
 		"common" for phy common block reset,
-		"cfg" for phy's ahb cfg block reset.
+		"cfg" for phy's ahb cfg block reset,
+		"ufsphy" for the PHY reset in the UFS controller.
 
 		For "qcom,ipq8074-qmp-pcie-phy" must contain:
 			"phy", "common".
@@ -69,12 +73,16 @@ Required properties:
 			"phy", "common".
 		For "qcom,msm8998-qmp-usb3-phy" must contain
 			"phy", "common".
-		For "qcom,msm8998-qmp-ufs-phy": no resets are listed.
+		For "qcom,msm8998-qmp-ufs-phy": must contain:
+			"ufsphy".
+		For "qcom,msm8998-qmp-pcie-phy" must contain:
+			"phy", "common".
 		For "qcom,sdm845-qmp-usb3-phy" must contain:
 			"phy", "common".
 		For "qcom,sdm845-qmp-usb3-uni-phy" must contain:
 			"phy", "common".
-		For "qcom,sdm845-qmp-ufs-phy": no resets are listed.
+		For "qcom,sdm845-qmp-ufs-phy": must contain:
+			"ufsphy".
 
  - vdda-phy-supply: Phandle to a regulator supply to PHY core block.
  - vdda-pll-supply: Phandle to 1.8V regulator supply to PHY refclk pll block.
diff --git a/Documentation/devicetree/bindings/phy/rcar-gen2-phy.txt b/Documentation/devicetree/bindings/phy/rcar-gen2-phy.txt
index 4f0879a0ca12..ac96d6481bb8 100644
--- a/Documentation/devicetree/bindings/phy/rcar-gen2-phy.txt
+++ b/Documentation/devicetree/bindings/phy/rcar-gen2-phy.txt
@@ -7,6 +7,7 @@ Required properties:
 - compatible: "renesas,usb-phy-r8a7743" if the device is a part of R8A7743 SoC.
 	      "renesas,usb-phy-r8a7744" if the device is a part of R8A7744 SoC.
 	      "renesas,usb-phy-r8a7745" if the device is a part of R8A7745 SoC.
+	      "renesas,usb-phy-r8a77470" if the device is a part of R8A77470 SoC.
 	      "renesas,usb-phy-r8a7790" if the device is a part of R8A7790 SoC.
 	      "renesas,usb-phy-r8a7791" if the device is a part of R8A7791 SoC.
 	      "renesas,usb-phy-r8a7794" if the device is a part of R8A7794 SoC.
@@ -30,7 +31,7 @@ channels. These subnodes must contain the following properties:
 - #phy-cells: see phy-bindings.txt in the same directory, must be <1>.
 
 The phandle's argument in the PHY specifier is the USB controller selector for
-the USB channel; see the selector meanings below:
+the USB channel other than r8a77470 SoC; see the selector meanings below:
 
 +-----------+---------------+---------------+
 |\ Selector |               |               |
@@ -41,6 +42,16 @@ the USB channel; see the selector meanings below:
 | 2         | PCI EHCI/OHCI | xHCI          |
 +-----------+---------------+---------------+
 
+For r8a77470 SoC;see the selector meaning below:
+
++-----------+---------------+---------------+
+|\ Selector |               |               |
++ --------- +       0       |       1       |
+| Channel  \|               |               |
++-----------+---------------+---------------+
+| 0         | EHCI/OHCI     | HS-USB        |
++-----------+---------------+---------------+
+
 Example (Lager board):
 
 	usb-phy@e6590100 {
@@ -48,15 +59,53 @@ Example (Lager board):
 		reg = <0 0xe6590100 0 0x100>;
 		#address-cells = <1>;
 		#size-cells = <0>;
-		clocks = <&mstp7_clks R8A7790_CLK_HSUSB>;
+		clocks = <&cpg CPG_MOD 704>;
 		clock-names = "usbhs";
+		power-domains = <&sysc R8A7790_PD_ALWAYS_ON>;
+		resets = <&cpg 704>;
 
-		usb-channel@0 {
+		usb0: usb-channel@0 {
 			reg = <0>;
 			#phy-cells = <1>;
 		};
-		usb-channel@2 {
+		usb2: usb-channel@2 {
 			reg = <2>;
 			#phy-cells = <1>;
 		};
 	};
+
+Example (iWave RZ/G1C sbc):
+
+	usbphy0: usb-phy0@e6590100 {
+		compatible = "renesas,usb-phy-r8a77470",
+			     "renesas,rcar-gen2-usb-phy";
+		reg = <0 0xe6590100 0 0x100>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clocks = <&cpg CPG_MOD 704>;
+		clock-names = "usbhs";
+		power-domains = <&sysc R8A77470_PD_ALWAYS_ON>;
+		resets = <&cpg 704>;
+
+		usb0: usb-channel@0 {
+			reg = <0>;
+			#phy-cells = <1>;
+		};
+	};
+
+	usbphy1: usb-phy@e6598100 {
+		compatible = "renesas,usb-phy-r8a77470",
+			     "renesas,rcar-gen2-usb-phy";
+		reg = <0 0xe6598100 0 0x100>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clocks = <&cpg CPG_MOD 706>;
+		clock-names = "usbhs";
+		power-domains = <&sysc R8A77470_PD_ALWAYS_ON>;
+		resets = <&cpg 706>;
+
+		usb1: usb-channel@0 {
+			reg = <0>;
+			#phy-cells = <1>;
+		};
+	};
diff --git a/Documentation/devicetree/bindings/phy/rcar-gen3-phy-usb2.txt b/Documentation/devicetree/bindings/phy/rcar-gen3-phy-usb2.txt
index ad9c290d8f15..d46188f450bf 100644
--- a/Documentation/devicetree/bindings/phy/rcar-gen3-phy-usb2.txt
+++ b/Documentation/devicetree/bindings/phy/rcar-gen3-phy-usb2.txt
@@ -1,10 +1,12 @@
 * Renesas R-Car generation 3 USB 2.0 PHY
 
 This file provides information on what the device node for the R-Car generation
-3 and RZ/G2 USB 2.0 PHY contain.
+3, RZ/G1C and RZ/G2 USB 2.0 PHY contain.
 
 Required properties:
-- compatible: "renesas,usb2-phy-r8a774a1" if the device is a part of an R8A774A1
+- compatible: "renesas,usb2-phy-r8a77470" if the device is a part of an R8A77470
+	      SoC.
+	      "renesas,usb2-phy-r8a774a1" if the device is a part of an R8A774A1
 	      SoC.
 	      "renesas,usb2-phy-r8a774c0" if the device is a part of an R8A774C0
 	      SoC.
@@ -27,7 +29,13 @@ Required properties:
 
 - reg: offset and length of the partial USB 2.0 Host register block.
 - clocks: clock phandle and specifier pair(s).
-- #phy-cells: see phy-bindings.txt in the same directory, must be <0>.
+- #phy-cells: see phy-bindings.txt in the same directory, must be <1> (and
+	      using <0> is deprecated).
+
+The phandle's argument in the PHY specifier is the INT_STATUS bit of controller:
+- 1 = USBH_INTA (OHCI)
+- 2 = USBH_INTB (EHCI)
+- 3 = UCOM_INT (OTG and BC)
 
 Optional properties:
 To use a USB channel where USB 2.0 Host and HSUSB (USB 2.0 Peripheral) are
diff --git a/Documentation/devicetree/bindings/phy/rockchip-emmc-phy.txt b/Documentation/devicetree/bindings/phy/rockchip-emmc-phy.txt
index e3ea55763b0a..e728786f21e0 100644
--- a/Documentation/devicetree/bindings/phy/rockchip-emmc-phy.txt
+++ b/Documentation/devicetree/bindings/phy/rockchip-emmc-phy.txt
@@ -7,12 +7,15 @@ Required properties:
  - reg: PHY register address offset and length in "general
    register files"
 
-Optional clocks using the clock bindings (see ../clock/clock-bindings.txt),
-specified by name:
+Optional properties:
  - clock-names: Should contain "emmcclk".  Although this is listed as optional
 		(because most boards can get basic functionality without having
 		access to it), it is strongly suggested.
+		See ../clock/clock-bindings.txt for details.
  - clocks: Should have a phandle to the card clock exported by the SDHCI driver.
+ - drive-impedance-ohm: Specifies the drive impedance in Ohm.
+                        Possible values are 33, 40, 50, 66 and 100.
+                        If not set, the default value of 50 will be applied.
 
 Example:
 
@@ -29,6 +32,7 @@ grf: syscon@ff770000 {
 		reg = <0xf780 0x20>;
 		clocks = <&sdhci>;
 		clock-names = "emmcclk";
+		drive-impedance-ohm = <50>;
 		#phy-cells = <0>;
 	};
 };
diff --git a/Documentation/devicetree/bindings/phy/ti,phy-am654-serdes.txt b/Documentation/devicetree/bindings/phy/ti,phy-am654-serdes.txt
new file mode 100644
index 000000000000..64b286d2d398
--- /dev/null
+++ b/Documentation/devicetree/bindings/phy/ti,phy-am654-serdes.txt
@@ -0,0 +1,82 @@
+TI AM654 SERDES
+
+Required properties:
+ - compatible: Should be "ti,phy-am654-serdes"
+ - reg : Address and length of the register set for the device.
+ - #phy-cells: determine the number of cells that should be given in the
+	phandle while referencing this phy. Should be "2". The 1st cell
+	corresponds to the phy type (should be one of the types specified in
+	include/dt-bindings/phy/phy.h) and the 2nd cell should be the serdes
+	lane function.
+	If SERDES0 is referenced 2nd cell should be:
+		0 - USB3
+		1 - PCIe0 Lane0
+		2 - ICSS2 SGMII Lane0
+	If SERDES1 is referenced 2nd cell should be:
+		0 - PCIe1 Lane0
+		1 - PCIe0 Lane1
+		2 - ICSS2 SGMII Lane1
+ - power-domains: As documented by the generic PM domain bindings in
+	Documentation/devicetree/bindings/power/power_domain.txt.
+ - clocks: List of clock-specifiers representing the input to the SERDES.
+	Should have 3 items representing the left input clock, external
+	reference clock and right input clock in that order.
+ - clock-output-names: List of clock names for each of the clock outputs of
+	SERDES. Should have 3 items for CMU reference clock,
+	left output clock and right output clock in that order.
+ - assigned-clocks: As defined in
+	Documentation/devicetree/bindings/clock/clock-bindings.txt
+ - assigned-clock-parents: As defined in
+	Documentation/devicetree/bindings/clock/clock-bindings.txt
+ - #clock-cells: Should be <1> to choose between the 3 output clocks.
+	Defined in Documentation/devicetree/bindings/clock/clock-bindings.txt
+
+   The following macros are defined in dt-bindings/phy/phy-am654-serdes.h
+   for selecting the correct reference clock. This can be used while
+   specifying the clocks created by SERDES.
+	=> AM654_SERDES_CMU_REFCLK
+	=> AM654_SERDES_LO_REFCLK
+	=> AM654_SERDES_RO_REFCLK
+
+ - mux-controls: Phandle to the multiplexer that is used to select the lane
+	function. See #phy-cells above to see the multiplex values.
+
+Example:
+
+Example for SERDES0 is given below. It has 3 clock inputs;
+left input reference clock as indicated by <&k3_clks 153 4>, external
+reference clock as indicated by <&k3_clks 153 1> and right input
+reference clock as indicated by <&serdes1 AM654_SERDES_LO_REFCLK>. (The
+right input of SERDES0 is connected to the left output of SERDES1).
+
+SERDES0 registers 3 clock outputs as indicated in clock-output-names. The
+first refers to the CMU reference clock, second refers to the left output
+reference clock and the third refers to the right output reference clock.
+
+The assigned-clocks and assigned-clock-parents is used here to set the
+parent of left input reference clock to MAINHSDIV_CLKOUT4 and parent of
+CMU reference clock to left input reference clock.
+
+serdes0: serdes@900000 {
+	compatible = "ti,phy-am654-serdes";
+	reg = <0x0 0x900000 0x0 0x2000>;
+	reg-names = "serdes";
+	#phy-cells = <2>;
+	power-domains = <&k3_pds 153>;
+	clocks = <&k3_clks 153 4>, <&k3_clks 153 1>,
+			<&serdes1 AM654_SERDES_LO_REFCLK>;
+	clock-output-names = "serdes0_cmu_refclk", "serdes0_lo_refclk",
+				"serdes0_ro_refclk";
+	assigned-clocks = <&k3_clks 153 4>, <&serdes0 AM654_SERDES_CMU_REFCLK>;
+	assigned-clock-parents = <&k3_clks 153 8>, <&k3_clks 153 4>;
+	ti,serdes-clk = <&serdes0_clk>;
+	mux-controls = <&serdes_mux 0>;
+	#clock-cells = <1>;
+};
+
+Example for PCIe consumer node using the SERDES PHY specifier is given below.
+&pcie0_rc {
+        num-lanes = <2>;
+        phys = <&serdes0 PHY_TYPE_PCIE 1>, <&serdes1 PHY_TYPE_PCIE 1>;
+        phy-names = "pcie-phy0", "pcie-phy1";
+};
diff --git a/Documentation/devicetree/bindings/pinctrl/bitmain,bm1880-pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/bitmain,bm1880-pinctrl.txt
new file mode 100644
index 000000000000..ed34bb1ee81c
--- /dev/null
+++ b/Documentation/devicetree/bindings/pinctrl/bitmain,bm1880-pinctrl.txt
@@ -0,0 +1,98 @@
+Bitmain BM1880 Pin Controller
+
+This binding describes the pin controller found in the BM1880 SoC.
+
+Required Properties:
+
+- compatible:   Should be "bitmain,bm1880-pinctrl"
+- reg:          Offset and length of pinctrl space in SCTRL.
+
+Please refer to pinctrl-bindings.txt in this directory for details of the
+common pinctrl bindings used by client devices, including the meaning of the
+phrase "pin configuration node".
+
+The pin configuration nodes act as a container for an arbitrary number of
+subnodes. Each of these subnodes represents some desired configuration for a
+pin, a group, or a list of pins or groups. This configuration for BM1880 SoC
+includes only pinmux as there is no pinconf support available in SoC.
+
+Each configuration node can consist of multiple nodes describing the pinmux
+options. The name of each subnode is not important; all subnodes should be
+enumerated and processed purely based on their content.
+
+The following generic properties as defined in pinctrl-bindings.txt are valid
+to specify in a pinmux subnode:
+
+Required Properties:
+
+- pins:           An array of strings, each string containing the name of a pin.
+                  Valid values for pins are:
+
+                  MIO0 - MIO111
+
+- groups:         An array of strings, each string containing the name of a pin
+                  group. Valid values for groups are:
+
+                  nand_grp, spi_grp, emmc_grp, sdio_grp, eth0_grp, pwm0_grp,
+                  pwm1_grp, pwm2_grp, pwm3_grp, pwm4_grp, pwm5_grp, pwm6_grp,
+                  pwm7_grp, pwm8_grp, pwm9_grp, pwm10_grp, pwm11_grp, pwm12_grp,
+                  pwm13_grp, pwm14_grp, pwm15_grp, pwm16_grp, pwm17_grp,
+                  pwm18_grp, pwm19_grp, pwm20_grp, pwm21_grp, pwm22_grp,
+                  pwm23_grp, pwm24_grp, pwm25_grp, pwm26_grp, pwm27_grp,
+                  pwm28_grp, pwm29_grp, pwm30_grp, pwm31_grp, pwm32_grp,
+                  pwm33_grp, pwm34_grp, pwm35_grp, pwm36_grp, i2c0_grp,
+                  i2c1_grp, i2c2_grp, i2c3_grp, i2c4_grp, uart0_grp, uart1_grp,
+                  uart2_grp, uart3_grp, uart4_grp, uart5_grp, uart6_grp,
+                  uart7_grp, uart8_grp, uart9_grp, uart10_grp, uart11_grp,
+                  uart12_grp, uart13_grp, uart14_grp, uart15_grp, gpio0_grp,
+                  gpio1_grp, gpio2_grp, gpio3_grp, gpio4_grp, gpio5_grp,
+                  gpio6_grp, gpio7_grp, gpio8_grp, gpio9_grp, gpio10_grp,
+                  gpio11_grp, gpio12_grp, gpio13_grp, gpio14_grp, gpio15_grp,
+                  gpio16_grp, gpio17_grp, gpio18_grp, gpio19_grp, gpio20_grp,
+                  gpio21_grp, gpio22_grp, gpio23_grp, gpio24_grp, gpio25_grp,
+                  gpio26_grp, gpio27_grp, gpio28_grp, gpio29_grp, gpio30_grp,
+                  gpio31_grp, gpio32_grp, gpio33_grp, gpio34_grp, gpio35_grp,
+                  gpio36_grp, gpio37_grp, gpio38_grp, gpio39_grp, gpio40_grp,
+                  gpio41_grp, gpio42_grp, gpio43_grp, gpio44_grp, gpio45_grp,
+                  gpio46_grp, gpio47_grp, gpio48_grp, gpio49_grp, gpio50_grp,
+                  gpio51_grp, gpio52_grp, gpio53_grp, gpio54_grp, gpio55_grp,
+                  gpio56_grp, gpio57_grp, gpio58_grp, gpio59_grp, gpio60_grp,
+                  gpio61_grp, gpio62_grp, gpio63_grp, gpio64_grp, gpio65_grp,
+                  gpio66_grp, gpio67_grp, eth1_grp, i2s0_grp, i2s0_mclkin_grp,
+                  i2s1_grp, i2s1_mclkin_grp, spi0_grp
+
+- function:       An array of strings, each string containing the name of the
+                  pinmux functions. The following are the list of pinmux
+                  functions available:
+
+                  nand, spi, emmc, sdio, eth0, pwm0, pwm1, pwm2, pwm3, pwm4,
+                  pwm5, pwm6, pwm7, pwm8, pwm9, pwm10, pwm11, pwm12, pwm13,
+                  pwm14, pwm15, pwm16, pwm17, pwm18, pwm19, pwm20, pwm21, pwm22,
+                  pwm23, pwm24, pwm25, pwm26, pwm27, pwm28, pwm29, pwm30, pwm31,
+                  pwm32, pwm33, pwm34, pwm35, pwm36, i2c0, i2c1, i2c2, i2c3,
+                  i2c4, uart0, uart1, uart2, uart3, uart4, uart5, uart6, uart7,
+                  uart8, uart9, uart10, uart11, uart12, uart13, uart14, uart15,
+                  gpio0, gpio1, gpio2, gpio3, gpio4, gpio5, gpio6, gpio7, gpio8,
+                  gpio9, gpio10, gpio11, gpio12, gpio13, gpio14, gpio15, gpio16,
+                  gpio17, gpio18, gpio19, gpio20, gpio21, gpio22, gpio23,
+                  gpio24, gpio25, gpio26, gpio27, gpio28, gpio29, gpio30,
+                  gpio31, gpio32, gpio33, gpio34, gpio35, gpio36, gpio37,
+                  gpio38, gpio39, gpio40, gpio41, gpio42, gpio43, gpio44,
+                  gpio45, gpio46, gpio47, gpio48, gpio49, gpio50, gpio51,
+                  gpio52, gpio53, gpio54, gpio55, gpio56, gpio57, gpio58,
+                  gpio59, gpio60, gpio61, gpio62, gpio63, gpio64, gpio65,
+                  gpio66, gpio67, eth1, i2s0, i2s0_mclkin, i2s1, i2s1_mclkin,
+                  spi0
+
+Example:
+        pinctrl: pinctrl@50 {
+                compatible = "bitmain,bm1880-pinctrl";
+                reg = <0x50 0x4B0>;
+
+                pinctrl_uart0_default: uart0-default {
+                        pinmux {
+                                groups = "uart0_grp";
+                                function = "uart0";
+                        };
+                };
+        };
diff --git a/Documentation/devicetree/bindings/pinctrl/cirrus,lochnagar.txt b/Documentation/devicetree/bindings/pinctrl/cirrus,lochnagar.txt
new file mode 100644
index 000000000000..a87447180e83
--- /dev/null
+++ b/Documentation/devicetree/bindings/pinctrl/cirrus,lochnagar.txt
@@ -0,0 +1,141 @@
+Cirrus Logic Lochnagar Audio Development Board
+
+Lochnagar is an evaluation and development board for Cirrus Logic
+Smart CODEC and Amp devices. It allows the connection of most Cirrus
+Logic devices on mini-cards, as well as allowing connection of
+various application processor systems to provide a full evaluation
+platform.  Audio system topology, clocking and power can all be
+controlled through the Lochnagar, allowing the device under test
+to be used in a variety of possible use cases.
+
+This binding document describes the binding for the pinctrl portion
+of the driver.
+
+Also see these documents for generic binding information:
+  [1] GPIO : ../gpio/gpio.txt
+  [2] Pinctrl: ../pinctrl/pinctrl-bindings.txt
+
+And these for relevant defines:
+  [3] include/dt-bindings/pinctrl/lochnagar.h
+
+This binding must be part of the Lochnagar MFD binding:
+  [4] ../mfd/cirrus,lochnagar.txt
+
+Required properties:
+
+  - compatible : One of the following strings:
+                 "cirrus,lochnagar-pinctrl"
+
+  - gpio-controller : Indicates this device is a GPIO controller.
+  - #gpio-cells : Must be 2. The first cell is the pin number, see
+    [3] for available pins and the second cell is used to specify
+    optional parameters, see [1].
+  - gpio-ranges : Range of pins managed by the GPIO controller, see
+    [1]. Both the GPIO and Pinctrl base should be set to zero and the
+    count to the appropriate of the LOCHNAGARx_PIN_NUM_GPIOS define,
+    see [3].
+
+  - pinctrl-names : A pinctrl state named "default" must be defined.
+  - pinctrl-0 : A phandle to the default pinctrl state.
+
+Required sub-nodes:
+
+The pin configurations are defined as a child of the pinctrl states
+node, see [2]. Each sub-node can have the following properties:
+  - groups : A list of groups to select (either this or "pins" must be
+    specified), available groups:
+      codec-aif1, codec-aif2, codec-aif3, dsp-aif1, dsp-aif2, psia1,
+      psia2, gf-aif1, gf-aif2, gf-aif3, gf-aif4, spdif-aif, usb-aif1,
+      usb-aif2, adat-aif, soundcard-aif
+  - pins : A list of pin names to select (either this or "groups" must
+    be specified), available pins:
+      fpga-gpio1, fpga-gpio2, fpga-gpio3, fpga-gpio4, fpga-gpio5,
+      fpga-gpio6, codec-gpio1, codec-gpio2, codec-gpio3, codec-gpio4,
+      codec-gpio5, codec-gpio6, codec-gpio7, codec-gpio8, dsp-gpio1,
+      dsp-gpio2, dsp-gpio3, dsp-gpio4, dsp-gpio5, dsp-gpio6, gf-gpio2,
+      gf-gpio3, gf-gpio7, codec-aif1-bclk, codec-aif1-rxdat,
+      codec-aif1-lrclk, codec-aif1-txdat, codec-aif2-bclk,
+      codec-aif2-rxdat, codec-aif2-lrclk, codec-aif2-txdat,
+      codec-aif3-bclk, codec-aif3-rxdat, codec-aif3-lrclk,
+      codec-aif3-txdat, dsp-aif1-bclk, dsp-aif1-rxdat, dsp-aif1-lrclk,
+      dsp-aif1-txdat, dsp-aif2-bclk, dsp-aif2-rxdat,
+      dsp-aif2-lrclk, dsp-aif2-txdat, psia1-bclk, psia1-rxdat,
+      psia1-lrclk, psia1-txdat, psia2-bclk, psia2-rxdat, psia2-lrclk,
+      psia2-txdat, gf-aif3-bclk, gf-aif3-rxdat, gf-aif3-lrclk,
+      gf-aif3-txdat, gf-aif4-bclk, gf-aif4-rxdat, gf-aif4-lrclk,
+      gf-aif4-txdat, gf-aif1-bclk, gf-aif1-rxdat, gf-aif1-lrclk,
+      gf-aif1-txdat, gf-aif2-bclk, gf-aif2-rxdat, gf-aif2-lrclk,
+      gf-aif2-txdat, dsp-uart1-rx, dsp-uart1-tx, dsp-uart2-rx,
+      dsp-uart2-tx, gf-uart2-rx, gf-uart2-tx, usb-uart-rx,
+      codec-pdmclk1, codec-pdmdat1, codec-pdmclk2, codec-pdmdat2,
+      codec-dmicclk1, codec-dmicdat1, codec-dmicclk2, codec-dmicdat2,
+      codec-dmicclk3, codec-dmicdat3, codec-dmicclk4, codec-dmicdat4,
+      dsp-dmicclk1, dsp-dmicdat1, dsp-dmicclk2, dsp-dmicdat2, i2c2-scl,
+      i2c2-sda, i2c3-scl, i2c3-sda, i2c4-scl, i2c4-sda, dsp-standby,
+      codec-mclk1, codec-mclk2, dsp-clkin, psia1-mclk, psia2-mclk,
+      gf-gpio1, gf-gpio5, dsp-gpio20, led1, led2
+  - function : The mux function to select, available functions:
+      aif, fpga-gpio1, fpga-gpio2, fpga-gpio3, fpga-gpio4, fpga-gpio5,
+      fpga-gpio6, codec-gpio1, codec-gpio2, codec-gpio3, codec-gpio4,
+      codec-gpio5, codec-gpio6, codec-gpio7, codec-gpio8, dsp-gpio1,
+      dsp-gpio2, dsp-gpio3, dsp-gpio4, dsp-gpio5, dsp-gpio6, gf-gpio2,
+      gf-gpio3, gf-gpio7, gf-gpio1, gf-gpio5, dsp-gpio20, codec-clkout,
+      dsp-clkout, pmic-32k, spdif-clkout, clk-12m288, clk-11m2986,
+      clk-24m576, clk-22m5792, xmos-mclk, gf-clkout1, gf-mclk1,
+      gf-mclk3, gf-mclk2, gf-clkout2, codec-mclk1, codec-mclk2,
+      dsp-clkin, psia1-mclk, psia2-mclk, spdif-mclk, codec-irq,
+      codec-reset, dsp-reset, dsp-irq, dsp-standby, codec-pdmclk1,
+      codec-pdmdat1, codec-pdmclk2, codec-pdmdat2, codec-dmicclk1,
+      codec-dmicdat1, codec-dmicclk2, codec-dmicdat2, codec-dmicclk3,
+      codec-dmicdat3, codec-dmicclk4, codec-dmicdat4, dsp-dmicclk1,
+      dsp-dmicdat1, dsp-dmicclk2, dsp-dmicdat2, dsp-uart1-rx,
+      dsp-uart1-tx, dsp-uart2-rx, dsp-uart2-tx, gf-uart2-rx,
+      gf-uart2-tx, usb-uart-rx, usb-uart-tx, i2c2-scl, i2c2-sda,
+      i2c3-scl, i2c3-sda, i2c4-scl, i2c4-sda, spdif-aif, psia1,
+      psia1-bclk, psia1-lrclk, psia1-rxdat, psia1-txdat, psia2,
+      psia2-bclk, psia2-lrclk, psia2-rxdat, psia2-txdat, codec-aif1,
+      codec-aif1-bclk, codec-aif1-lrclk, codec-aif1-rxdat,
+      codec-aif1-txdat, codec-aif2, codec-aif2-bclk, codec-aif2-lrclk,
+      codec-aif2-rxdat, codec-aif2-txdat, codec-aif3, codec-aif3-bclk,
+      codec-aif3-lrclk, codec-aif3-rxdat, codec-aif3-txdat, dsp-aif1,
+      dsp-aif1-bclk, dsp-aif1-lrclk, dsp-aif1-rxdat, dsp-aif1-txdat,
+      dsp-aif2, dsp-aif2-bclk, dsp-aif2-lrclk, dsp-aif2-rxdat,
+      dsp-aif2-txdat, gf-aif3, gf-aif3-bclk, gf-aif3-lrclk,
+      gf-aif3-rxdat, gf-aif3-txdat, gf-aif4, gf-aif4-bclk,
+      gf-aif4-lrclk, gf-aif4-rxdat, gf-aif4-txdat, gf-aif1,
+      gf-aif1-bclk, gf-aif1-lrclk, gf-aif1-rxdat, gf-aif1-txdat,
+      gf-aif2, gf-aif2-bclk, gf-aif2-lrclk, gf-aif2-rxdat,
+      gf-aif2-txdat, usb-aif1, usb-aif2, adat-aif, soundcard-aif,
+
+  - output-enable : Specifies that an AIF group will be used as a master
+    interface (either this or input-enable is required if a group is
+    being muxed to an AIF)
+  - input-enable : Specifies that an AIF group will be used as a slave
+    interface (either this or output-enable is required if a group is
+    being muxed to an AIF)
+
+Example:
+
+lochnagar-pinctrl {
+	compatible = "cirrus,lochnagar-pinctrl";
+
+	gpio-controller;
+	#gpio-cells = <2>;
+	gpio-ranges = <&lochnagar 0 0 LOCHNAGAR2_PIN_NUM_GPIOS>;
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&pin-settings>;
+
+	pin-settings: pin-settings {
+		ap-aif {
+			input-enable;
+			groups = "gf-aif1";
+			function = "codec-aif3";
+		};
+		codec-aif {
+			output-enable;
+			groups = "codec-aif3";
+			function = "gf-aif1";
+		};
+	};
+};
diff --git a/Documentation/devicetree/bindings/pinctrl/fsl,imx7d-pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/fsl,imx7d-pinctrl.txt
index 6666277c3acb..8ac1d0851a0f 100644
--- a/Documentation/devicetree/bindings/pinctrl/fsl,imx7d-pinctrl.txt
+++ b/Documentation/devicetree/bindings/pinctrl/fsl,imx7d-pinctrl.txt
@@ -48,9 +48,9 @@ PAD_CTL_HYS                     (1 << 3)
 PAD_CTL_SRE_SLOW                (1 << 2)
 PAD_CTL_SRE_FAST                (0 << 2)
 PAD_CTL_DSE_X1                  (0 << 0)
-PAD_CTL_DSE_X2                  (1 << 0)
-PAD_CTL_DSE_X3                  (2 << 0)
-PAD_CTL_DSE_X4                  (3 << 0)
+PAD_CTL_DSE_X4                  (1 << 0)
+PAD_CTL_DSE_X2                  (2 << 0)
+PAD_CTL_DSE_X6                  (3 << 0)
 
 Examples:
 While iomuxc-lpsr is intended to be used by dedicated peripherals to take
diff --git a/Documentation/devicetree/bindings/pinctrl/pinctrl-mt65xx.txt b/Documentation/devicetree/bindings/pinctrl/pinctrl-mt65xx.txt
index e7d6f81c227f..205be98ae078 100644
--- a/Documentation/devicetree/bindings/pinctrl/pinctrl-mt65xx.txt
+++ b/Documentation/devicetree/bindings/pinctrl/pinctrl-mt65xx.txt
@@ -11,6 +11,7 @@ Required properties:
 	"mediatek,mt8127-pinctrl", compatible with mt8127 pinctrl.
 	"mediatek,mt8135-pinctrl", compatible with mt8135 pinctrl.
 	"mediatek,mt8173-pinctrl", compatible with mt8173 pinctrl.
+	"mediatek,mt8516-pinctrl", compatible with mt8516 pinctrl.
 - pins-are-numbered: Specify the subnodes are using numbered pinmux to
   specify pins.
 - gpio-controller : Marks the device node as a gpio controller.
diff --git a/Documentation/devicetree/bindings/pinctrl/pinctrl-mt8183.txt b/Documentation/devicetree/bindings/pinctrl/pinctrl-mt8183.txt
new file mode 100644
index 000000000000..eccbe3f55d3f
--- /dev/null
+++ b/Documentation/devicetree/bindings/pinctrl/pinctrl-mt8183.txt
@@ -0,0 +1,132 @@
+* Mediatek MT8183 Pin Controller
+
+The Mediatek's Pin controller is used to control SoC pins.
+
+Required properties:
+- compatible: value should be one of the following.
+	"mediatek,mt8183-pinctrl", compatible with mt8183 pinctrl.
+- gpio-controller : Marks the device node as a gpio controller.
+- #gpio-cells: number of cells in GPIO specifier. Since the generic GPIO
+  binding is used, the amount of cells must be specified as 2. See the below
+  mentioned gpio binding representation for description of particular cells.
+- gpio-ranges : gpio valid number range.
+- reg: physical address base for gpio base registers. There are 10 GPIO
+  physical address base in mt8183.
+
+Optional properties:
+- reg-names: gpio base register names. There are 10 gpio base register
+  names in mt8183. They are "iocfg0", "iocfg1", "iocfg2", "iocfg3", "iocfg4",
+  "iocfg5", "iocfg6", "iocfg7", "iocfg8", "eint".
+- interrupt-controller: Marks the device node as an interrupt controller
+- #interrupt-cells: Should be two.
+- interrupts : The interrupt outputs to sysirq.
+
+Please refer to pinctrl-bindings.txt in this directory for details of the
+common pinctrl bindings used by client devices.
+
+Subnode format
+A pinctrl node should contain at least one subnodes representing the
+pinctrl groups available on the machine. Each subnode will list the
+pins it needs, and how they should be configured, with regard to muxer
+configuration, pullups, drive strength, input enable/disable and input schmitt.
+
+    node {
+	pinmux = <PIN_NUMBER_PINMUX>;
+	GENERIC_PINCONFIG;
+    };
+
+Required properties:
+- pinmux: integer array, represents gpio pin number and mux setting.
+    Supported pin number and mux varies for different SoCs, and are defined
+    as macros in boot/dts/<soc>-pinfunc.h directly.
+
+Optional properties:
+- GENERIC_PINCONFIG: is the generic pinconfig options to use, bias-disable,
+    bias-pull-down, bias-pull-up, input-enable, input-disable, output-low,
+    output-high, input-schmitt-enable, input-schmitt-disable
+    and drive-strength are valid.
+
+    Some special pins have extra pull up strength, there are R0 and R1 pull-up
+    resistors available, but for user, it's only need to set R1R0 as 00, 01,
+    10 or 11. So It needs config "mediatek,pull-up-adv" or
+    "mediatek,pull-down-adv" to support arguments for those special pins.
+    Valid arguments are from 0 to 3.
+
+    mediatek,tdsel: An integer describing the steps for output level shifter
+      duty cycle when asserted (high pulse width adjustment). Valid arguments
+      are from 0 to 15.
+    mediatek,rdsel: An integer describing the steps for input level shifter
+      duty cycle when asserted (high pulse width adjustment). Valid arguments
+      are from 0 to 63.
+
+    When config drive-strength, it can support some arguments, such as
+    MTK_DRIVE_4mA, MTK_DRIVE_6mA, etc. See dt-bindings/pinctrl/mt65xx.h.
+    It can only support 2/4/6/8/10/12/14/16mA in mt8183.
+    For I2C pins, there are existing generic driving setup and the specific
+    driving setup. I2C pins can only support 2/4/6/8/10/12/14/16mA driving
+    adjustment in generic driving setup. But in specific driving setup,
+    they can support 0.125/0.25/0.5/1mA adjustment. If we enable specific
+    driving setup for I2C pins, the existing generic driving setup will be
+    disabled. For some special features, we need the I2C pins specific
+    driving setup. The specific driving setup is controlled by E1E0EN.
+    So we need add extra vendor driving preperty instead of
+    the generic driving property.
+    We can add "mediatek,drive-strength-adv = <XXX>;" to describe the specific
+    driving setup property. "XXX" means the value of E1E0EN. EN is 0 or 1.
+    It is used to enable or disable the specific driving setup.
+    E1E0 is used to describe the detail strength specification of the I2C pin.
+    When E1=0/E0=0, the strength is 0.125mA.
+    When E1=0/E0=1, the strength is 0.25mA.
+    When E1=1/E0=0, the strength is 0.5mA.
+    When E1=1/E0=1, the strength is 1mA.
+    So the valid arguments of "mediatek,drive-strength-adv" are from 0 to 7.
+
+Examples:
+
+#include "mt8183-pinfunc.h"
+
+...
+{
+	pio: pinctrl@10005000 {
+		compatible = "mediatek,mt8183-pinctrl";
+		reg = <0 0x10005000 0 0x1000>,
+		      <0 0x11f20000 0 0x1000>,
+		      <0 0x11e80000 0 0x1000>,
+		      <0 0x11e70000 0 0x1000>,
+		      <0 0x11e90000 0 0x1000>,
+		      <0 0x11d30000 0 0x1000>,
+		      <0 0x11d20000 0 0x1000>,
+		      <0 0x11c50000 0 0x1000>,
+		      <0 0x11f30000 0 0x1000>,
+		      <0 0x1000b000 0 0x1000>;
+		reg-names = "iocfg0", "iocfg1", "iocfg2",
+			    "iocfg3", "iocfg4", "iocfg5",
+			    "iocfg6", "iocfg7", "iocfg8",
+			    "eint";
+		gpio-controller;
+		#gpio-cells = <2>;
+		gpio-ranges = <&pio 0 0 192>;
+		interrupt-controller;
+		interrupts = <GIC_SPI 177 IRQ_TYPE_LEVEL_HIGH>;
+		#interrupt-cells = <2>;
+
+		i2c0_pins_a: i2c0 {
+			pins1 {
+				pinmux = <PINMUX_GPIO48__FUNC_SCL5>,
+					 <PINMUX_GPIO49__FUNC_SDA5>;
+				mediatek,pull-up-adv = <3>;
+				mediatek,drive-strength-adv = <7>;
+			};
+		};
+
+		i2c1_pins_a: i2c1 {
+			pins {
+				pinmux = <PINMUX_GPIO50__FUNC_SCL3>,
+					 <PINMUX_GPIO51__FUNC_SDA3>;
+				mediatek,pull-down-adv = <2>;
+				mediatek,drive-strength-adv = <4>;
+			};
+		};
+		...
+	};
+};
diff --git a/Documentation/devicetree/bindings/pinctrl/st,stm32-pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/st,stm32-pinctrl.txt
index 48df30a36b01..00169255e48c 100644
--- a/Documentation/devicetree/bindings/pinctrl/st,stm32-pinctrl.txt
+++ b/Documentation/devicetree/bindings/pinctrl/st,stm32-pinctrl.txt
@@ -57,6 +57,8 @@ Optional properties:
  - st,bank-ioport: should correspond to the EXTI IOport selection (EXTI line
    used to select GPIOs as interrupts).
  - hwlocks: reference to a phandle of a hardware spinlock provider node.
+ - st,package: Indicates the SOC package used.
+   More details in include/dt-bindings/pinctrl/stm32-pinfunc.h
 
 Example 1:
 #include <dt-bindings/pinctrl/stm32f429-pinfunc.h>
diff --git a/Documentation/devicetree/bindings/rtc/nxp,pcf85063.txt b/Documentation/devicetree/bindings/rtc/nxp,pcf85063.txt
index d3e380ad712d..627bb533eff7 100644
--- a/Documentation/devicetree/bindings/rtc/nxp,pcf85063.txt
+++ b/Documentation/devicetree/bindings/rtc/nxp,pcf85063.txt
@@ -1,7 +1,11 @@
 * NXP PCF85063 Real Time Clock
 
 Required properties:
-- compatible: Should contain "nxp,pcf85063".
+- compatible: Should one of contain:
+	"nxp,pcf85063",
+	"nxp,pcf85063a",
+	"nxp,pcf85063tp",
+	"microcrystal,rv8263"
 - reg: I2C address for chip.
 
 Optional property:
diff --git a/Documentation/devicetree/bindings/rtc/rtc-aspeed.txt b/Documentation/devicetree/bindings/rtc/rtc-aspeed.txt
new file mode 100644
index 000000000000..2e956b3dc276
--- /dev/null
+++ b/Documentation/devicetree/bindings/rtc/rtc-aspeed.txt
@@ -0,0 +1,22 @@
+ASPEED BMC RTC
+==============
+
+Required properties:
+ - compatible: should be one of the following
+   * aspeed,ast2400-rtc for the ast2400
+   * aspeed,ast2500-rtc for the ast2500
+   * aspeed,ast2600-rtc for the ast2600
+
+ - reg: physical base address of the controller and length of memory mapped
+   region
+
+ - interrupts: The interrupt number
+
+Example:
+
+   rtc@1e781000 {
+           compatible = "aspeed,ast2400-rtc";
+           reg = <0x1e781000 0x18>;
+           interrupts = <22>;
+           status = "disabled";
+   };
diff --git a/Documentation/devicetree/bindings/rtc/rtc.txt b/Documentation/devicetree/bindings/rtc/rtc.txt
index f4687c68c08c..a97fc6a9a75e 100644
--- a/Documentation/devicetree/bindings/rtc/rtc.txt
+++ b/Documentation/devicetree/bindings/rtc/rtc.txt
@@ -69,3 +69,4 @@ ricoh,rv5c386		I2C bus SERIAL INTERFACE REAL-TIME CLOCK IC
 ricoh,rv5c387a		I2C bus SERIAL INTERFACE REAL-TIME CLOCK IC
 sii,s35390a		2-wire CMOS real-time clock
 whwave,sd3078		I2C bus SERIAL INTERFACE REAL-TIME CLOCK IC
+xircom,x1205		Xircom X1205 I2C RTC
diff --git a/Documentation/devicetree/bindings/serial/cdns,uart.txt b/Documentation/devicetree/bindings/serial/cdns,uart.txt
index 227bb770b027..4efc560f90ab 100644
--- a/Documentation/devicetree/bindings/serial/cdns,uart.txt
+++ b/Documentation/devicetree/bindings/serial/cdns,uart.txt
@@ -12,6 +12,11 @@ Required properties:
   See ../clocks/clock-bindings.txt for details.
 
 
+Optional properties:
+- cts-override : Override the CTS modem status signal. This signal will
+  always be reported as active instead of being obtained from the modem status
+  register. Define this if your serial port does not use this pin
+
 Example:
 	uart@e0000000 {
 		compatible = "cdns,uart-r1p8";
diff --git a/Documentation/devicetree/bindings/serial/nxp,sc16is7xx.txt b/Documentation/devicetree/bindings/serial/nxp,sc16is7xx.txt
index e7921a8e276b..c1091a923a89 100644
--- a/Documentation/devicetree/bindings/serial/nxp,sc16is7xx.txt
+++ b/Documentation/devicetree/bindings/serial/nxp,sc16is7xx.txt
@@ -12,6 +12,8 @@ Required properties:
 - reg: I2C address of the SC16IS7xx device.
 - interrupts: Should contain the UART interrupt
 - clocks: Reference to the IC source clock.
+	OR (when there is no clock provider visible to the platform)
+- clock-frequency: The source clock frequency for the IC.
 
 Optional properties:
 - gpio-controller: Marks the device node as a GPIO controller.
diff --git a/Documentation/devicetree/bindings/serial/sifive-serial.txt b/Documentation/devicetree/bindings/serial/sifive-serial.txt
new file mode 100644
index 000000000000..c86b1e524159
--- /dev/null
+++ b/Documentation/devicetree/bindings/serial/sifive-serial.txt
@@ -0,0 +1,33 @@
+SiFive asynchronous serial interface (UART)
+
+Required properties:
+
+- compatible: should be something similar to
+	      "sifive,<chip>-uart" for the UART as integrated
+	      on a particular chip, and "sifive,uart<version>" for the
+	      general UART IP block programming model.	Supported
+	      compatible strings as of the date of this writing are:
+	      "sifive,fu540-c000-uart" for the SiFive UART v0 as
+	      integrated onto the SiFive FU540 chip, or "sifive,uart0"
+	      for the SiFive UART v0 IP block with no chip integration
+	      tweaks (if any)
+- reg: address and length of the register space
+- interrupts: Should contain the UART interrupt identifier
+- clocks: Should contain a clock identifier for the UART's parent clock
+
+
+UART HDL that corresponds to the IP block version numbers can be found
+here:
+
+https://github.com/sifive/sifive-blocks/tree/master/src/main/scala/devices/uart
+
+
+Example:
+
+uart0: serial@10010000 {
+	compatible = "sifive,fu540-c000-uart", "sifive,uart0";
+	interrupt-parent = <&plic0>;
+	interrupts = <80>;
+	reg = <0x0 0x10010000 0x0 0x1000>;
+	clocks = <&prci PRCI_CLK_TLCLK>;
+};
diff --git a/Documentation/devicetree/bindings/serial/sprd-uart.txt b/Documentation/devicetree/bindings/serial/sprd-uart.txt
index cab40f0f6f49..9607dc616205 100644
--- a/Documentation/devicetree/bindings/serial/sprd-uart.txt
+++ b/Documentation/devicetree/bindings/serial/sprd-uart.txt
@@ -7,7 +7,17 @@ Required properties:
 
 - reg: offset and length of the register set for the device
 - interrupts: exactly one interrupt specifier
-- clocks: phandles to input clocks.
+- clock-names: Should contain following entries:
+  "enable" for UART module enable clock,
+  "uart" for UART clock,
+  "source" for UART source (parent) clock.
+- clocks: Should contain a clock specifier for each entry in clock-names.
+  UART clock and source clock are optional properties, but enable clock
+  is required.
+
+Optional properties:
+- dma-names: Should contain "rx" for receive and "tx" for transmit channels.
+- dmas: A list of dma specifiers, one for each entry in dma-names.
 
 Example:
 	uart0: serial@0 {
@@ -15,5 +25,8 @@ Example:
 			     "sprd,sc9836-uart";
 		reg = <0x0 0x100>;
 		interrupts = <GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>;
-		clocks = <&ext_26m>;
+		dma-names = "rx", "tx";
+		dmas = <&ap_dma 19>, <&ap_dma 20>;
+		clock-names = "enable", "uart", "source";
+		clocks = <&clk_ap_apb_gates 9>, <&clk_uart0>, <&ext_26m>;
 	};
diff --git a/Documentation/devicetree/bindings/sound/adi,axi-i2s.txt b/Documentation/devicetree/bindings/sound/adi,axi-i2s.txt
index 4248b662deff..229ad1392cdc 100644
--- a/Documentation/devicetree/bindings/sound/adi,axi-i2s.txt
+++ b/Documentation/devicetree/bindings/sound/adi,axi-i2s.txt
@@ -1,5 +1,8 @@
 ADI AXI-I2S controller
 
+The core can be generated with transmit (playback), only receive
+(capture) or both directions enabled.
+
 Required properties:
  - compatible : Must be "adi,axi-i2s-1.00.a"
  - reg : Must contain I2S core's registers location and length
@@ -9,8 +12,8 @@ Required properties:
  - clock-names : "axi" for the clock to the AXI interface, "ref" for the sample
    rate reference clock.
  - dmas: Pairs of phandle and specifier for the DMA channels that are used by
-   the core. The core expects two dma channels, one for transmit and one for
-   receive.
+   the core. The core expects two dma channels if both transmit and receive are
+   enabled, one channel otherwise.
  - dma-names : "tx" for the transmit channel, "rx" for the receive channel.
 
 For more details on the 'dma', 'dma-names', 'clock' and 'clock-names' properties
diff --git a/Documentation/devicetree/bindings/sound/amlogic,axg-fifo.txt b/Documentation/devicetree/bindings/sound/amlogic,axg-fifo.txt
index 3dfc2515e5c6..4330fc9dca6d 100644
--- a/Documentation/devicetree/bindings/sound/amlogic,axg-fifo.txt
+++ b/Documentation/devicetree/bindings/sound/amlogic,axg-fifo.txt
@@ -2,7 +2,9 @@
 
 Required properties:
 - compatible: 'amlogic,axg-toddr' or
-	      'amlogic,axg-frddr'
+	      'amlogic,axg-toddr' or
+	      'amlogic,g12a-frddr' or
+	      'amlogic,g12a-toddr'
 - reg: physical base address of the controller and length of memory
        mapped region.
 - interrupts: interrupt specifier for the fifo.
diff --git a/Documentation/devicetree/bindings/sound/amlogic,axg-pdm.txt b/Documentation/devicetree/bindings/sound/amlogic,axg-pdm.txt
index 5672d0bc5b16..73f473a9365f 100644
--- a/Documentation/devicetree/bindings/sound/amlogic,axg-pdm.txt
+++ b/Documentation/devicetree/bindings/sound/amlogic,axg-pdm.txt
@@ -1,7 +1,8 @@
 * Amlogic Audio PDM input
 
 Required properties:
-- compatible: 'amlogic,axg-pdm'
+- compatible: 'amlogic,axg-pdm' or
+	      'amlogic,g12a-pdm'
 - reg: physical base address of the controller and length of memory
        mapped region.
 - clocks: list of clock phandle, one for each entry clock-names.
diff --git a/Documentation/devicetree/bindings/sound/amlogic,axg-spdifin.txt b/Documentation/devicetree/bindings/sound/amlogic,axg-spdifin.txt
index 2e6cb7d9b202..0b82504fa419 100644
--- a/Documentation/devicetree/bindings/sound/amlogic,axg-spdifin.txt
+++ b/Documentation/devicetree/bindings/sound/amlogic,axg-spdifin.txt
@@ -1,7 +1,8 @@
 * Amlogic Audio SPDIF Input
 
 Required properties:
-- compatible: 'amlogic,axg-spdifin'
+- compatible: 'amlogic,axg-spdifin' or
+	      'amlogic,g12a-spdifin'
 - interrupts: interrupt specifier for the spdif input.
 - clocks: list of clock phandle, one for each entry clock-names.
 - clock-names: should contain the following:
diff --git a/Documentation/devicetree/bindings/sound/amlogic,axg-spdifout.txt b/Documentation/devicetree/bindings/sound/amlogic,axg-spdifout.txt
index 521c38ad89e7..826152730508 100644
--- a/Documentation/devicetree/bindings/sound/amlogic,axg-spdifout.txt
+++ b/Documentation/devicetree/bindings/sound/amlogic,axg-spdifout.txt
@@ -1,7 +1,8 @@
 * Amlogic Audio SPDIF Output
 
 Required properties:
-- compatible: 'amlogic,axg-spdifout'
+- compatible: 'amlogic,axg-spdifout' or
+	      'amlogic,g12a-spdifout'
 - clocks: list of clock phandle, one for each entry clock-names.
 - clock-names: should contain the following:
   * "pclk" : peripheral clock.
diff --git a/Documentation/devicetree/bindings/sound/amlogic,axg-tdm-formatters.txt b/Documentation/devicetree/bindings/sound/amlogic,axg-tdm-formatters.txt
index 1c1b7490554e..3b94a715a0b9 100644
--- a/Documentation/devicetree/bindings/sound/amlogic,axg-tdm-formatters.txt
+++ b/Documentation/devicetree/bindings/sound/amlogic,axg-tdm-formatters.txt
@@ -2,7 +2,9 @@
 
 Required properties:
 - compatible: 'amlogic,axg-tdmin' or
-	      'amlogic,axg-tdmout'
+	      'amlogic,axg-tdmout' or
+	      'amlogic,g12a-tdmin' or
+	      'amlogic,g12a-tdmout'
 - reg: physical base address of the controller and length of memory
        mapped region.
 - clocks: list of clock phandle, one for each entry clock-names.
diff --git a/Documentation/devicetree/bindings/sound/cirrus,lochnagar.txt b/Documentation/devicetree/bindings/sound/cirrus,lochnagar.txt
new file mode 100644
index 000000000000..41ae2699f07a
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/cirrus,lochnagar.txt
@@ -0,0 +1,39 @@
+Cirrus Logic Lochnagar Audio Development Board
+
+Lochnagar is an evaluation and development board for Cirrus Logic
+Smart CODEC and Amp devices. It allows the connection of most Cirrus
+Logic devices on mini-cards, as well as allowing connection of
+various application processor systems to provide a full evaluation
+platform.  Audio system topology, clocking and power can all be
+controlled through the Lochnagar, allowing the device under test
+to be used in a variety of possible use cases.
+
+This binding document describes the binding for the audio portion
+of the driver.
+
+This binding must be part of the Lochnagar MFD binding:
+  [4] ../mfd/cirrus,lochnagar.txt
+
+Required properties:
+
+  - compatible : One of the following strings:
+                 "cirrus,lochnagar2-soundcard"
+
+  - #sound-dai-cells : Must be set to 1.
+
+  - clocks : Contains an entry for each entry in clock-names.
+  - clock-names : Must include the following clocks:
+      "mclk" Master clock source for the sound card, should normally
+      be set to LOCHNAGAR_SOUNDCARD_MCLK provided by the Lochnagar
+      clock driver.
+
+Example:
+
+lochnagar-sc {
+	compatible = "cirrus,lochnagar2-soundcard";
+
+	#sound-dai-cells = <1>;
+
+	clocks = <&lochnagar_clk LOCHNAGAR_SOUNDCARD_MCLK>;
+	clock-names = "mclk";
+};
diff --git a/Documentation/devicetree/bindings/sound/cs42l51.txt b/Documentation/devicetree/bindings/sound/cs42l51.txt
index 4b5de33ce377..acbd68ddd2cb 100644
--- a/Documentation/devicetree/bindings/sound/cs42l51.txt
+++ b/Documentation/devicetree/bindings/sound/cs42l51.txt
@@ -1,6 +1,17 @@
 CS42L51 audio CODEC
 
+Required properties:
+
+  - compatible : "cirrus,cs42l51"
+
+  - reg : the I2C address of the device for I2C.
+
 Optional properties:
+  - VL-supply, VD-supply, VA-supply, VAHP-supply: power supplies for the device,
+    as covered in Documentation/devicetree/bindings/regulator/regulator.txt.
+
+  - reset-gpios : GPIO specification for the reset pin. If specified, it will be
+    deasserted before starting the communication with the codec.
 
   - clocks : a list of phandles + clock-specifiers, one for each entry in
     clock-names
@@ -14,4 +25,9 @@ cs42l51: cs42l51@4a {
 	reg = <0x4a>;
 	clocks = <&mclk_prov>;
 	clock-names = "MCLK";
+	VL-supply = <&reg_audio>;
+	VD-supply = <&reg_audio>;
+	VA-supply = <&reg_audio>;
+	VAHP-supply = <&reg_audio>;
+	reset-gpios = <&gpiog 9 GPIO_ACTIVE_LOW>;
 };
diff --git a/Documentation/devicetree/bindings/sound/da7219.txt b/Documentation/devicetree/bindings/sound/da7219.txt
index e9d0baeb94e2..add1caf26ac2 100644
--- a/Documentation/devicetree/bindings/sound/da7219.txt
+++ b/Documentation/devicetree/bindings/sound/da7219.txt
@@ -23,8 +23,8 @@ Optional properties:
   interrupt is to be used to wake system, otherwise "irq" should be used.
 - wakeup-source: Flag to indicate this device can wake system (suspend/resume).
 
-- #clock-cells :  Should be set to '<0>', only one clock source provided;
-- clock-output-names : Name given for DAI clocks output;
+- #clock-cells :  Should be set to '<1>', two clock sources provided;
+- clock-output-names : Names given for DAI clock outputs (WCLK & BCLK);
 
 - clocks : phandle and clock specifier for codec MCLK.
 - clock-names : Clock name string for 'clocks' attribute, should be "mclk".
@@ -84,8 +84,8 @@ Example:
 		VDDMIC-supply = <&reg_audio>;
 		VDDIO-supply = <&reg_audio>;
 
-		#clock-cells = <0>;
-		clock-output-names = "dai-clks";
+		#clock-cells = <1>;
+		clock-output-names = "dai-wclk", "dai-bclk";
 
 		clocks = <&clks 201>;
 		clock-names = "mclk";
diff --git a/Documentation/devicetree/bindings/sound/fsl,audmix.txt b/Documentation/devicetree/bindings/sound/fsl,audmix.txt
new file mode 100644
index 000000000000..840b7e0d6a63
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/fsl,audmix.txt
@@ -0,0 +1,50 @@
+NXP Audio Mixer (AUDMIX).
+
+The Audio Mixer is a on-chip functional module that allows mixing of two
+audio streams into a single audio stream. Audio Mixer has two input serial
+audio interfaces. These are driven by two Synchronous Audio interface
+modules (SAI). Each input serial interface carries 8 audio channels in its
+frame in TDM manner. Mixer mixes audio samples of corresponding channels
+from two interfaces into a single sample. Before mixing, audio samples of
+two inputs can be attenuated based on configuration. The output of the
+Audio Mixer is also a serial audio interface. Like input interfaces it has
+the same TDM frame format. This output is used to drive the serial DAC TDM
+interface of audio codec and also sent to the external pins along with the
+receive path of normal audio SAI module for readback by the CPU.
+
+The output of Audio Mixer can be selected from any of the three streams
+ - serial audio input 1
+ - serial audio input 2
+ - mixed audio
+
+Mixing operation is independent of audio sample rate but the two audio
+input streams must have same audio sample rate with same number of channels
+in TDM frame to be eligible for mixing.
+
+Device driver required properties:
+=================================
+  - compatible		: Compatible list, contains "fsl,imx8qm-audmix"
+
+  - reg			: Offset and length of the register set for the device.
+
+  - clocks		: Must contain an entry for each entry in clock-names.
+
+  - clock-names		: Must include the "ipg" for register access.
+
+  - power-domains	: Must contain the phandle to AUDMIX power domain node
+
+  - dais		: Must contain a list of phandles to AUDMIX connected
+			  DAIs. The current implementation requires two phandles
+			  to SAI interfaces to be provided, the first SAI in the
+			  list being used to route the AUDMIX output.
+
+Device driver configuration example:
+======================================
+  audmix: audmix@59840000 {
+    compatible = "fsl,imx8qm-audmix";
+    reg = <0x0 0x59840000 0x0 0x10000>;
+    clocks = <&clk IMX8QXP_AUD_AUDMIX_IPG>;
+    clock-names = "ipg";
+    power-domains = <&pd_audmix>;
+    dais = <&sai4>, <&sai5>;
+  };
diff --git a/Documentation/devicetree/bindings/sound/mchp-i2s-mcc.txt b/Documentation/devicetree/bindings/sound/mchp-i2s-mcc.txt
new file mode 100644
index 000000000000..91ec83a6faed
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/mchp-i2s-mcc.txt
@@ -0,0 +1,43 @@
+* Microchip I2S Multi-Channel Controller
+
+Required properties:
+- compatible:     Should be "microchip,sam9x60-i2smcc".
+- reg:            Should be the physical base address of the controller and the
+                  length of memory mapped region.
+- interrupts:     Should contain the interrupt for the controller.
+- dmas:           Should be one per channel name listed in the dma-names property,
+                  as described in atmel-dma.txt and dma.txt files.
+- dma-names:      Identifier string for each DMA request line in the dmas property.
+		  Two dmas have to be defined, "tx" and "rx".
+- clocks:         Must contain an entry for each entry in clock-names.
+                  Please refer to clock-bindings.txt.
+- clock-names:    Should be one of each entry matching the clocks phandles list:
+                  - "pclk" (peripheral clock) Required.
+                  - "gclk" (generated clock) Optional (1).
+
+Optional properties:
+- pinctrl-0:      Should specify pin control groups used for this controller.
+- princtrl-names: Should contain only one value - "default".
+
+
+(1) : Only the peripheral clock is required. The generated clock is optional
+      and should be set mostly when Master Mode is required.
+
+Example:
+
+	i2s@f001c000 {
+		compatible = "microchip,sam9x60-i2smcc";
+		reg = <0xf001c000 0x100>;
+		interrupts = <34 IRQ_TYPE_LEVEL_HIGH 7>;
+		dmas = <&dma0
+			(AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) |
+			 AT91_XDMAC_DT_PERID(36))>,
+		       <&dma0
+			(AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) |
+			 AT91_XDMAC_DT_PERID(37))>;
+		dma-names = "tx", "rx";
+		clocks = <&i2s_clk>, <&i2s_gclk>;
+		clock-names = "pclk", "gclk";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_i2s_default>;
+	};
diff --git a/Documentation/devicetree/bindings/sound/mt8183-da7219-max98357.txt b/Documentation/devicetree/bindings/sound/mt8183-da7219-max98357.txt
new file mode 100644
index 000000000000..92ac86f83822
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/mt8183-da7219-max98357.txt
@@ -0,0 +1,15 @@
+MT8183 with MT6358, DA7219 and MAX98357 CODECS
+
+Required properties:
+- compatible : "mediatek,mt8183_da7219_max98357"
+- mediatek,headset-codec: the phandles of da7219 codecs
+- mediatek,platform: the phandle of MT8183 ASoC platform
+
+Example:
+
+	sound {
+		compatible = "mediatek,mt8183_da7219_max98357";
+		mediatek,headset-codec = <&da7219>;
+		mediatek,platform = <&afe>;
+	};
+
diff --git a/Documentation/devicetree/bindings/sound/mt8183-mt6358-ts3a227-max98357.txt b/Documentation/devicetree/bindings/sound/mt8183-mt6358-ts3a227-max98357.txt
new file mode 100644
index 000000000000..d6d5207fa996
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/mt8183-mt6358-ts3a227-max98357.txt
@@ -0,0 +1,15 @@
+MT8183 with MT6358, TS3A227 and MAX98357 CODECS
+
+Required properties:
+- compatible : "mediatek,mt8183_mt6358_ts3a227_max98357"
+- mediatek,headset-codec: the phandles of ts3a227 codecs
+- mediatek,platform: the phandle of MT8183 ASoC platform
+
+Example:
+
+	sound {
+		compatible = "mediatek,mt8183_mt6358_ts3a227_max98357";
+		mediatek,headset-codec = <&ts3a227>;
+		mediatek,platform = <&afe>;
+	};
+
diff --git a/Documentation/devicetree/bindings/sound/renesas,rsnd.txt b/Documentation/devicetree/bindings/sound/renesas,rsnd.txt
index 648d43e1b1e9..5c52182f7dcf 100644
--- a/Documentation/devicetree/bindings/sound/renesas,rsnd.txt
+++ b/Documentation/devicetree/bindings/sound/renesas,rsnd.txt
@@ -266,6 +266,7 @@ Required properties:
 				    - "renesas,rcar_sound-r8a7743" (RZ/G1M)
 				    - "renesas,rcar_sound-r8a7744" (RZ/G1N)
 				    - "renesas,rcar_sound-r8a7745" (RZ/G1E)
+				    - "renesas,rcar_sound-r8a77470" (RZ/G1C)
 				    - "renesas,rcar_sound-r8a774a1" (RZ/G2M)
 				    - "renesas,rcar_sound-r8a774c0" (RZ/G2E)
 				    - "renesas,rcar_sound-r8a7778" (R-Car M1A)
@@ -282,7 +283,12 @@ Required properties:
 - reg				: Should contain the register physical address.
 				  required register is
 				   SRU/ADG/SSI      if generation1
-				   SRU/ADG/SSIU/SSI if generation2
+				   SRU/ADG/SSIU/SSI/AUDIO-DMAC-periperi if generation2/generation3
+				   Select extended AUDIO-DMAC-periperi address if SoC has it,
+				   otherwise select normal AUDIO-DMAC-periperi address.
+- reg-names			: Should contain the register names.
+				   scu/adg/ssi	if generation1
+				   scu/adg/ssiu/ssi/audmapp if generation2/generation3
 - rcar_sound,ssi		: Should contain SSI feature.
 				  The number of SSI subnode should be same as HW.
 				  see below for detail.
diff --git a/Documentation/devicetree/bindings/sound/rockchip,pdm.txt b/Documentation/devicetree/bindings/sound/rockchip,pdm.txt
index 47f164fbd1d7..98572a25122f 100644
--- a/Documentation/devicetree/bindings/sound/rockchip,pdm.txt
+++ b/Documentation/devicetree/bindings/sound/rockchip,pdm.txt
@@ -3,6 +3,9 @@
 Required properties:
 
 - compatible: "rockchip,pdm"
+  - "rockchip,px30-pdm"
+  - "rockchip,rk1808-pdm"
+  - "rockchip,rk3308-pdm"
 - reg: physical base address of the controller and length of memory mapped
   region.
 - dmas: DMA specifiers for rx dma. See the DMA client binding,
@@ -12,6 +15,8 @@ Required properties:
 - clock-names: should contain following:
    - "pdm_hclk": clock for PDM BUS
    - "pdm_clk" : clock for PDM controller
+- resets: a list of phandle + reset-specifer paris, one for each entry in reset-names.
+- reset-names: reset names, should include "pdm-m".
 - pinctrl-names: Must contain a "default" entry.
 - pinctrl-N: One property must exist for each entry in
 	     pinctrl-names. See ../pinctrl/pinctrl-bindings.txt
diff --git a/Documentation/devicetree/bindings/sound/rt5651.txt b/Documentation/devicetree/bindings/sound/rt5651.txt
index a41199a5cd79..56e736a1cba9 100644
--- a/Documentation/devicetree/bindings/sound/rt5651.txt
+++ b/Documentation/devicetree/bindings/sound/rt5651.txt
@@ -22,6 +22,11 @@ Optional properties:
   2: Use JD1_2 pin for jack-detect
   3: Use JD2 pin for jack-detect
 
+- realtek,jack-detect-not-inverted
+  bool. Normal jack-detect switches give an inverted (active-low) signal,
+  set this bool in the rare case you've a jack-detect switch which is not
+  inverted.
+
 - realtek,over-current-threshold-microamp
   u32, micbias over-current detection threshold in µA, valid values are
   600, 1500 and 2000µA.
diff --git a/Documentation/devicetree/bindings/sound/simple-amplifier.txt b/Documentation/devicetree/bindings/sound/simple-amplifier.txt
index 7182ac4f1e65..b1b097cc9b68 100644
--- a/Documentation/devicetree/bindings/sound/simple-amplifier.txt
+++ b/Documentation/devicetree/bindings/sound/simple-amplifier.txt
@@ -2,9 +2,9 @@ Simple Amplifier Audio Driver
 
 Required properties:
 - compatible : "dioo,dio2125" or "simple-audio-amplifier"
-- enable-gpios : the gpio connected to the enable pin of the simple amplifier
 
 Optional properties:
+- enable-gpios : the gpio connected to the enable pin of the simple amplifier
 - VCC-supply   : power supply for the device, as covered
                  in Documentation/devicetree/bindings/regulator/regulator.txt
 
diff --git a/Documentation/devicetree/bindings/sound/simple-card.txt b/Documentation/devicetree/bindings/sound/simple-card.txt
index 4629c8f8a6b6..79954cd6e37b 100644
--- a/Documentation/devicetree/bindings/sound/simple-card.txt
+++ b/Documentation/devicetree/bindings/sound/simple-card.txt
@@ -24,6 +24,8 @@ Optional properties:
 					  a microphone is attached.
 - simple-audio-card,aux-devs		: List of phandles pointing to auxiliary devices, such
 					  as amplifiers, to be added to the sound card.
+- simple-audio-card,pin-switches	: List of strings containing the widget names for
+					  which pin switches must be created.
 
 Optional subnodes:
 
diff --git a/Documentation/devicetree/bindings/sound/sprd-mcdt.txt b/Documentation/devicetree/bindings/sound/sprd-mcdt.txt
new file mode 100644
index 000000000000..274ba0acbfd6
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/sprd-mcdt.txt
@@ -0,0 +1,19 @@
+Spreadtrum Multi-Channel Data Transfer Binding
+
+The Multi-channel data transfer controller is used for sound stream
+transmission between audio subsystem and other AP/CP subsystem. It
+supports 10 DAC channel and 10 ADC channel, and each channel can be
+configured with DMA mode or interrupt mode.
+
+Required properties:
+- compatible: Should be "sprd,sc9860-mcdt".
+- reg: Should contain registers address and length.
+- interrupts: Should contain one interrupt shared by all channel.
+
+Example:
+
+mcdt@41490000 {
+	compatible = "sprd,sc9860-mcdt";
+	reg = <0 0x41490000 0 0x170>;
+	interrupts = <GIC_SPI 48 IRQ_TYPE_LEVEL_HIGH>;
+};
diff --git a/Documentation/devicetree/bindings/ufs/cdns,ufshc.txt b/Documentation/devicetree/bindings/ufs/cdns,ufshc.txt
index a04a4989ec7f..02347b017abd 100644
--- a/Documentation/devicetree/bindings/ufs/cdns,ufshc.txt
+++ b/Documentation/devicetree/bindings/ufs/cdns,ufshc.txt
@@ -5,8 +5,9 @@ Each UFS controller instance should have its own node.
 Please see the ufshcd-pltfrm.txt for a list of all available properties.
 
 Required properties:
-- compatible	: Compatible list, contains the following controller:
-			"cdns,ufshc"
+- compatible	: Compatible list, contains one of the following controllers:
+			"cdns,ufshc" - Generic CDNS HCI,
+			"cdns,ufshc-m31-16nm" - CDNS UFS HC + M31 16nm PHY
 		  complemented with the JEDEC version:
 			"jedec,ufs-2.0"
 
diff --git a/Documentation/devicetree/bindings/ufs/ufs-mediatek.txt b/Documentation/devicetree/bindings/ufs/ufs-mediatek.txt
new file mode 100644
index 000000000000..72aab8547308
--- /dev/null
+++ b/Documentation/devicetree/bindings/ufs/ufs-mediatek.txt
@@ -0,0 +1,43 @@
+* Mediatek Universal Flash Storage (UFS) Host Controller
+
+UFS nodes are defined to describe on-chip UFS hardware macro.
+Each UFS Host Controller should have its own node.
+
+To bind UFS PHY with UFS host controller, the controller node should
+contain a phandle reference to UFS M-PHY node.
+
+Required properties for UFS nodes:
+- compatible         : Compatible list, contains the following controller:
+                       "mediatek,mt8183-ufshci" for MediaTek UFS host controller
+                       present on MT81xx chipsets.
+- reg                : Address and length of the UFS register set.
+- phys               : phandle to m-phy.
+- clocks             : List of phandle and clock specifier pairs.
+- clock-names        : List of clock input name strings sorted in the same
+                       order as the clocks property. "ufs" is mandatory.
+                       "ufs": ufshci core control clock.
+- freq-table-hz      : Array of <min max> operating frequencies stored in the same
+                       order as the clocks property. If this property is not
+                       defined or a value in the array is "0" then it is assumed
+                       that the frequency is set by the parent clock or a
+                       fixed rate clock source.
+- vcc-supply         : phandle to VCC supply regulator node.
+
+Example:
+
+	ufsphy: phy@11fa0000 {
+		...
+	};
+
+	ufshci@11270000 {
+		compatible = "mediatek,mt8183-ufshci";
+		reg = <0 0x11270000 0 0x2300>;
+		interrupts = <GIC_SPI 104 IRQ_TYPE_LEVEL_LOW>;
+		phys = <&ufsphy>;
+
+		clocks = <&infracfg_ao INFRACFG_AO_UFS_CG>;
+		clock-names = "ufs";
+		freq-table-hz = <0 0>;
+
+		vcc-supply = <&mt_pmic_vemc_ldo_reg>;
+	};
diff --git a/Documentation/devicetree/bindings/ufs/ufs-qcom.txt b/Documentation/devicetree/bindings/ufs/ufs-qcom.txt
index 21d9a93db2e9..fd59f93e9556 100644
--- a/Documentation/devicetree/bindings/ufs/ufs-qcom.txt
+++ b/Documentation/devicetree/bindings/ufs/ufs-qcom.txt
@@ -29,6 +29,7 @@ Optional properties:
 - vdda-pll-max-microamp : specifies max. load that can be drawn from pll supply
 - vddp-ref-clk-supply   : phandle to UFS device ref_clk pad power supply
 - vddp-ref-clk-max-microamp : specifies max. load that can be drawn from this supply
+- resets : specifies the PHY reset in the UFS controller
 
 Example:
 
@@ -51,9 +52,11 @@ Example:
 			<&clock_gcc clk_ufs_phy_ldo>,
 			<&clock_gcc clk_gcc_ufs_tx_cfg_clk>,
 			<&clock_gcc clk_gcc_ufs_rx_cfg_clk>;
+		resets = <&ufshc 0>;
 	};
 
-	ufshc@fc598000 {
+	ufshc: ufshc@fc598000 {
+		#reset-cells = <1>;
 		...
 		phys = <&ufsphy1>;
 		phy-names = "ufsphy";
diff --git a/Documentation/devicetree/bindings/ufs/ufshcd-pltfrm.txt b/Documentation/devicetree/bindings/ufs/ufshcd-pltfrm.txt
index 5111e9130bc3..56bccde9953a 100644
--- a/Documentation/devicetree/bindings/ufs/ufshcd-pltfrm.txt
+++ b/Documentation/devicetree/bindings/ufs/ufshcd-pltfrm.txt
@@ -31,7 +31,6 @@ Optional properties:
 - vcc-max-microamp      : specifies max. load that can be drawn from vcc supply
 - vccq-max-microamp     : specifies max. load that can be drawn from vccq supply
 - vccq2-max-microamp    : specifies max. load that can be drawn from vccq2 supply
-- <name>-fixed-regulator : boolean property specifying that <name>-supply is a fixed regulator
 
 - clocks                : List of phandle and clock specifier pairs
 - clock-names           : List of clock input name strings sorted in the same
@@ -50,6 +49,8 @@ Optional properties:
 -lanes-per-direction	: number of lanes available per direction - either 1 or 2.
 			  Note that it is assume same number of lanes is used both
 			  directions at once. If not specified, default is 2 lanes per direction.
+- #reset-cells		: Must be <1> for Qualcomm UFS controllers that expose
+			  PHY reset from the UFS controller.
 - resets            : reset node register
 - reset-names       : describe reset node register, the "rst" corresponds to reset the whole UFS IP.
 
@@ -63,7 +64,6 @@ Example:
 		interrupts = <0 28 0>;
 
 		vdd-hba-supply = <&xxx_reg0>;
-		vdd-hba-fixed-regulator;
 		vcc-supply = <&xxx_reg1>;
 		vcc-supply-1p8;
 		vccq-supply = <&xxx_reg2>;
@@ -79,4 +79,5 @@ Example:
 		reset-names = "rst";
 		phys = <&ufsphy1>;
 		phy-names = "ufsphy";
+		#reset-cells = <1>;
 	};
diff --git a/Documentation/devicetree/bindings/usb/amlogic,dwc3.txt b/Documentation/devicetree/bindings/usb/amlogic,dwc3.txt
index 9a8b631904fd..b9f04e617eb7 100644
--- a/Documentation/devicetree/bindings/usb/amlogic,dwc3.txt
+++ b/Documentation/devicetree/bindings/usb/amlogic,dwc3.txt
@@ -40,3 +40,91 @@ Example device nodes:
 				phy-names = "usb2-phy", "usb3-phy";
 			};
 		};
+
+Amlogic Meson G12A DWC3 USB SoC Controller Glue
+
+The Amlogic G12A embeds a DWC3 USB IP Core configured for USB2 and USB3
+in host-only mode, and a DWC2 IP Core configured for USB2 peripheral mode
+only.
+
+A glue connects the DWC3 core to USB2 PHYs and optionnaly to an USB3 PHY.
+
+One of the USB2 PHY can be re-routed in peripheral mode to a DWC2 USB IP.
+
+The DWC3 Glue controls the PHY routing and power, an interrupt line is
+connected to the Glue to serve as OTG ID change detection.
+
+Required properties:
+- compatible:	Should be "amlogic,meson-g12a-usb-ctrl"
+- clocks:	a handle for the "USB" clock
+- resets:	a handle for the shared "USB" reset line
+- reg:		The base address and length of the registers
+- interrupts:	the interrupt specifier for the OTG detection
+- phys: 	handle to used PHYs on the system
+	- a <0> phandle can be used if a PHY is not used
+- phy-names:	names of the used PHYs on the system :
+	- "usb2-phy0" for USB2 PHY0 if USBHOST_A port is used
+	- "usb2-phy1" for USB2 PHY1 if USBOTG_B port is used
+	- "usb3-phy0" for USB3 PHY if USB3_0 is used
+- dr_mode:	should be "host", "peripheral", or "otg" depending on
+	the usage and configuration of the OTG Capable port.
+	- "host" and "peripheral" means a fixed Host or Device only connection
+	- "otg" means the port can be used as both Host or Device and
+	  be switched automatically using the OTG ID pin.
+
+Optional properties:
+- vbus-supply:	should be a phandle to the regulator controlling the VBUS
+		power supply when used in OTG switchable mode
+
+Required child nodes:
+
+A child node must exist to represent the core DWC3 IP block. The name of
+the node is not important. The content of the node is defined in dwc3.txt.
+
+A child node must exist to represent the core DWC2 IP block. The name of
+the node is not important. The content of the node is defined in dwc2.txt.
+
+PHY documentation is provided in the following places:
+- Documentation/devicetree/bindings/phy/meson-g12a-usb2-phy.txt
+- Documentation/devicetree/bindings/phy/meson-g12a-usb3-pcie-phy.txt
+
+Example device nodes:
+	usb: usb@ffe09000 {
+			compatible = "amlogic,meson-g12a-usb-ctrl";
+			reg = <0x0 0xffe09000 0x0 0xa0>;
+			interrupts = <GIC_SPI 16 IRQ_TYPE_LEVEL_HIGH>;
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges;
+
+			clocks = <&clkc CLKID_USB>;
+			resets = <&reset RESET_USB>;
+
+			dr_mode = "otg";
+
+			phys = <&usb2_phy0>, <&usb2_phy1>,
+			       <&usb3_pcie_phy PHY_TYPE_USB3>;
+			phy-names = "usb2-phy0", "usb2-phy1", "usb3-phy0";
+
+			dwc2: usb@ff400000 {
+				compatible = "amlogic,meson-g12a-usb", "snps,dwc2";
+				reg = <0x0 0xff400000 0x0 0x40000>;
+				interrupts = <GIC_SPI 31 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&clkc CLKID_USB1_DDR_BRIDGE>;
+				clock-names = "ddr";
+				phys = <&usb2_phy1>;
+				dr_mode = "peripheral";
+				g-rx-fifo-size = <192>;
+				g-np-tx-fifo-size = <128>;
+				g-tx-fifo-size = <128 128 16 16 16>;
+			};
+
+			dwc3: usb@ff500000 {
+				compatible = "snps,dwc3";
+				reg = <0x0 0xff500000 0x0 0x100000>;
+				interrupts = <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>;
+				dr_mode = "host";
+				snps,dis_u2_susphy_quirk;
+				snps,quirk-frame-length-adjustment;
+			};
+	};
diff --git a/Documentation/devicetree/bindings/usb/dwc2.txt b/Documentation/devicetree/bindings/usb/dwc2.txt
index 6dc3c4a34483..49eac0dc86b0 100644
--- a/Documentation/devicetree/bindings/usb/dwc2.txt
+++ b/Documentation/devicetree/bindings/usb/dwc2.txt
@@ -14,6 +14,7 @@ Required properties:
   - "amlogic,meson8-usb": The DWC2 USB controller instance in Amlogic Meson8 SoCs;
   - "amlogic,meson8b-usb": The DWC2 USB controller instance in Amlogic Meson8b SoCs;
   - "amlogic,meson-gxbb-usb": The DWC2 USB controller instance in Amlogic S905 SoCs;
+  - "amlogic,meson-g12a-usb": The DWC2 USB controller instance in Amlogic G12A SoCs;
   - "amcc,dwc-otg": The DWC2 USB controller instance in AMCC Canyonlands 460EX SoCs;
   - snps,dwc2: A generic DWC2 USB controller with default parameters.
   - "st,stm32f4x9-fsotg": The DWC2 USB FS/HS controller instance in STM32F4x9 SoCs
@@ -31,12 +32,18 @@ Refer to clk/clock-bindings.txt for generic clock consumer properties
 Optional properties:
 - phys: phy provider specifier
 - phy-names: shall be "usb2-phy"
+- vbus-supply: reference to the VBUS regulator. Depending on the current mode
+  this is enabled (in "host" mode") or disabled (in "peripheral" mode). The
+  regulator is updated if the controller is configured in "otg" mode and the
+  status changes between "host" and "peripheral".
 Refer to phy/phy-bindings.txt for generic phy consumer properties
 - dr_mode: shall be one of "host", "peripheral" and "otg"
   Refer to usb/generic.txt
 - g-rx-fifo-size: size of rx fifo size in gadget mode.
 - g-np-tx-fifo-size: size of non-periodic tx fifo size in gadget mode.
 - g-tx-fifo-size: size of periodic tx fifo per endpoint (except ep0) in gadget mode.
+- snps,reset-phy-on-wake: If present indicates that we need to reset the PHY when
+                          we detect a wakeup.  This is due to a hardware errata.
 
 Deprecated properties:
 - g-use-dma: gadget DMA mode is automatically detected
diff --git a/Documentation/devicetree/bindings/usb/generic-ehci.yaml b/Documentation/devicetree/bindings/usb/generic-ehci.yaml
new file mode 100644
index 000000000000..d3b4f6415920
--- /dev/null
+++ b/Documentation/devicetree/bindings/usb/generic-ehci.yaml
@@ -0,0 +1,95 @@
+# SPDX-License-Identifier: GPL-2.0
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/usb/generic-ehci.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: USB EHCI Controller Device Tree Bindings
+
+allOf:
+  - $ref: "usb-hcd.yaml"
+
+maintainers:
+  - Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+
+properties:
+  compatible:
+    contains:
+      const: generic-ehci
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  resets:
+    minItems: 1
+    maxItems: 4
+
+  clocks:
+    minItems: 1
+    maxItems: 4
+    description: |
+      In case the Renesas R-Car Gen3 SoCs:
+        - if a host only channel: first clock should be host.
+        - if a USB DRD channel: first clock should be host and second
+          one should be peripheral
+
+  big-endian:
+    $ref: /schemas/types.yaml#/definitions/flag
+    description:
+      Set this flag for HCDs with big endian descriptors and big
+      endian registers.
+
+  big-endian-desc:
+    $ref: /schemas/types.yaml#/definitions/flag
+    description:
+      Set this flag for HCDs with big endian descriptors.
+
+  big-endian-regs:
+    $ref: /schemas/types.yaml#/definitions/flag
+    description:
+      Set this flag for HCDs with big endian registers.
+
+  has-transaction-translator:
+    $ref: /schemas/types.yaml#/definitions/flag
+    description:
+      Set this flag if EHCI has a Transaction Translator built into
+      the root hub.
+
+  needs-reset-on-resume:
+    $ref: /schemas/types.yaml#/definitions/flag
+    description:
+      Set this flag to force EHCI reset after resume.
+
+  phys: true
+
+required:
+  - compatible
+  - reg
+  - interrupts
+
+additionalProperties: false
+
+examples:
+  - |
+    ehci@e0000300 {
+        compatible = "ibm,usb-ehci-440epx", "generic-ehci";
+        interrupt-parent = <&UIC0>;
+        interrupts = <0x1a 4>;
+        reg = <0 0xe0000300 90 0 0xe0000390 70>;
+        big-endian;
+    };
+
+  - |
+    ehci0: usb@1c14000 {
+        compatible = "allwinner,sun4i-a10-ehci", "generic-ehci";
+        reg = <0x01c14000 0x100>;
+        interrupts = <39>;
+        clocks = <&ahb_gates 1>;
+        phys = <&usbphy 1>;
+        phy-names = "usb";
+    };
+
+...
diff --git a/Documentation/devicetree/bindings/usb/generic-ohci.yaml b/Documentation/devicetree/bindings/usb/generic-ohci.yaml
new file mode 100644
index 000000000000..da5a14becbe5
--- /dev/null
+++ b/Documentation/devicetree/bindings/usb/generic-ohci.yaml
@@ -0,0 +1,89 @@
+# SPDX-License-Identifier: GPL-2.0
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/usb/generic-ohci.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: USB OHCI Controller Device Tree Bindings
+
+allOf:
+  - $ref: "usb-hcd.yaml"
+
+maintainers:
+  - Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+
+properties:
+  compatible:
+    contains:
+      const: generic-ohci
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  resets:
+    minItems: 1
+    maxItems: 2
+
+  clocks:
+    minItems: 1
+    maxItems: 3
+    description: |
+      In case the Renesas R-Car Gen3 SoCs:
+        - if a host only channel: first clock should be host.
+        - if a USB DRD channel: first clock should be host and second
+          one should be peripheral
+
+  big-endian:
+    $ref: /schemas/types.yaml#/definitions/flag
+    description:
+      Set this flag for HCDs with big endian descriptors and big
+      endian registers.
+
+  big-endian-desc:
+    $ref: /schemas/types.yaml#/definitions/flag
+    description:
+      Set this flag for HCDs with big endian descriptors.
+
+  big-endian-regs:
+    $ref: /schemas/types.yaml#/definitions/flag
+    description:
+      Set this flag for HCDs with big endian registers.
+
+  remote-wakeup-connected:
+    $ref: /schemas/types.yaml#/definitions/flag
+    description:
+      Remote wakeup is wired on the platform.
+
+  no-big-frame-no:
+    $ref: /schemas/types.yaml#/definitions/flag
+    description:
+      Set if frame_no lives in bits [15:0] of HCCA
+
+  num-ports:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description:
+      Overrides the detected port count
+
+  phys: true
+
+required:
+  - compatible
+  - reg
+  - interrupts
+
+additionalProperties: false
+
+examples:
+  - |
+      ohci0: usb@1c14400 {
+          compatible = "allwinner,sun4i-a10-ohci", "generic-ohci";
+          reg = <0x01c14400 0x100>;
+          interrupts = <64>;
+          clocks = <&usb_clk 6>, <&ahb_gates 2>;
+          phys = <&usbphy 1>;
+      };
+
+...
diff --git a/Documentation/devicetree/bindings/usb/ingenic,jz4740-musb.txt b/Documentation/devicetree/bindings/usb/ingenic,jz4740-musb.txt
index 620355cee63f..16808721f3ff 100644
--- a/Documentation/devicetree/bindings/usb/ingenic,jz4740-musb.txt
+++ b/Documentation/devicetree/bindings/usb/ingenic,jz4740-musb.txt
@@ -8,9 +8,15 @@ Required properties:
 - interrupt-names: must be "mc"
 - clocks: phandle to the "udc" clock
 - clock-names: must be "udc"
+- phys: phandle to the USB PHY
 
 Example:
 
+usb_phy: usb-phy@0 {
+	compatible = "usb-nop-xceiv";
+	#phy-cells = <0>;
+};
+
 udc: usb@13040000 {
 	compatible = "ingenic,jz4740-musb";
 	reg = <0x13040000 0x10000>;
@@ -21,4 +27,6 @@ udc: usb@13040000 {
 
 	clocks = <&cgu JZ4740_CLK_UDC>;
 	clock-names = "udc";
+
+	phys = <&usb_phy>;
 };
diff --git a/Documentation/devicetree/bindings/usb/nvidia,tegra124-xusb.txt b/Documentation/devicetree/bindings/usb/nvidia,tegra124-xusb.txt
index 4156c3e181c5..5bfcc0b4d6b9 100644
--- a/Documentation/devicetree/bindings/usb/nvidia,tegra124-xusb.txt
+++ b/Documentation/devicetree/bindings/usb/nvidia,tegra124-xusb.txt
@@ -10,6 +10,7 @@ Required properties:
   - Tegra124: "nvidia,tegra124-xusb"
   - Tegra132: "nvidia,tegra132-xusb", "nvidia,tegra124-xusb"
   - Tegra210: "nvidia,tegra210-xusb"
+  - Tegra186: "nvidia,tegra186-xusb"
 - reg: Must contain the base and length of the xHCI host registers, XUSB FPCI
   registers and XUSB IPFS registers.
 - reg-names: Must contain the following entries:
@@ -59,6 +60,8 @@ For Tegra210:
 - avdd-pll-uerefe-supply: PLLE reference PLL power supply. Must supply 1.05 V.
 - dvdd-pex-pll-supply: PCIe/USB3 PLL power supply. Must supply 1.05 V.
 - hvdd-pex-pll-e-supply: High-voltage PLLE power supply. Must supply 1.8 V.
+
+For Tegra210 and Tegra186:
 - power-domains: A list of PM domain specifiers that reference each power-domain
   used by the xHCI controller. This list must comprise of a specifier for the
   XUSBA and XUSBC power-domains. See ../power/power_domain.txt and
@@ -78,6 +81,7 @@ Optional properties:
   - Tegra132: usb2-0, usb2-1, usb2-2, hsic-0, hsic-1, usb3-0, usb3-1
   - Tegra210: usb2-0, usb2-1, usb2-2, usb2-3, hsic-0, usb3-0, usb3-1, usb3-2,
               usb3-3
+  - Tegra186: usb2-0, usb2-1, usb2-2, hsic-0, usb3-0, usb3-1, usb3-2
 
 Example:
 --------
diff --git a/Documentation/devicetree/bindings/usb/renesas_usbhs.txt b/Documentation/devicetree/bindings/usb/renesas_usbhs.txt
index d93b6a1504f2..b8acc2a994a8 100644
--- a/Documentation/devicetree/bindings/usb/renesas_usbhs.txt
+++ b/Documentation/devicetree/bindings/usb/renesas_usbhs.txt
@@ -6,6 +6,7 @@ Required properties:
 	- "renesas,usbhs-r8a7743" for r8a7743 (RZ/G1M) compatible device
 	- "renesas,usbhs-r8a7744" for r8a7744 (RZ/G1N) compatible device
 	- "renesas,usbhs-r8a7745" for r8a7745 (RZ/G1E) compatible device
+	- "renesas,usbhs-r8a77470" for r8a77470 (RZ/G1C) compatible device
 	- "renesas,usbhs-r8a774a1" for r8a774a1 (RZ/G2M) compatible device
 	- "renesas,usbhs-r8a774c0" for r8a774c0 (RZ/G2E) compatible device
 	- "renesas,usbhs-r8a7790" for r8a7790 (R-Car H2) compatible device
diff --git a/Documentation/devicetree/bindings/usb/usb-ehci.txt b/Documentation/devicetree/bindings/usb/usb-ehci.txt
deleted file mode 100644
index 406252d14c6b..000000000000
--- a/Documentation/devicetree/bindings/usb/usb-ehci.txt
+++ /dev/null
@@ -1,46 +0,0 @@
-USB EHCI controllers
-
-Required properties:
-  - compatible : should be "generic-ehci".
-  - reg : should contain at least address and length of the standard EHCI
-    register set for the device. Optional platform-dependent registers
-    (debug-port or other) can be also specified here, but only after
-    definition of standard EHCI registers.
-  - interrupts : one EHCI interrupt should be described here.
-
-Optional properties:
- - big-endian-regs : boolean, set this for hcds with big-endian registers
- - big-endian-desc : boolean, set this for hcds with big-endian descriptors
- - big-endian : boolean, for hcds with big-endian-regs + big-endian-desc
- - needs-reset-on-resume : boolean, set this to force EHCI reset after resume
- - has-transaction-translator : boolean, set this if EHCI have a Transaction
-				Translator built into the root hub.
- - clocks : a list of phandle + clock specifier pairs. In case of Renesas
-	    R-Car Gen3 SoCs:
-	    - if a host only channel: first clock should be host.
-	    - if a USB DRD channel: first clock should be host and second one
-				    should be peripheral.
- - phys : see usb-hcd.txt in the current directory
- - resets : phandle + reset specifier pair
-
-additionally the properties from usb-hcd.txt (in the current directory) are
-supported.
-
-Example (Sequoia 440EPx):
-    ehci@e0000300 {
-	   compatible = "ibm,usb-ehci-440epx", "usb-ehci";
-	   interrupt-parent = <&UIC0>;
-	   interrupts = <1a 4>;
-	   reg = <0 e0000300 90 0 e0000390 70>;
-	   big-endian;
-   };
-
-Example (Allwinner sun4i A10 SoC):
-   ehci0: usb@1c14000 {
-	   compatible = "allwinner,sun4i-a10-ehci", "generic-ehci";
-	   reg = <0x01c14000 0x100>;
-	   interrupts = <39>;
-	   clocks = <&ahb_gates 1>;
-	   phys = <&usbphy 1>;
-	   phy-names = "usb";
-   };
diff --git a/Documentation/devicetree/bindings/usb/usb-hcd.txt b/Documentation/devicetree/bindings/usb/usb-hcd.txt
deleted file mode 100644
index 50529b838c9c..000000000000
--- a/Documentation/devicetree/bindings/usb/usb-hcd.txt
+++ /dev/null
@@ -1,9 +0,0 @@
-Generic USB HCD (Host Controller Device) Properties
-
-Optional properties:
-- phys: a list of all USB PHYs on this HCD
-
-Example:
-	&usb1 {
-		phys = <&usb2_phy1>, <&usb3_phy1>;
-	};
diff --git a/Documentation/devicetree/bindings/usb/usb-hcd.yaml b/Documentation/devicetree/bindings/usb/usb-hcd.yaml
new file mode 100644
index 000000000000..9c8c56d3a792
--- /dev/null
+++ b/Documentation/devicetree/bindings/usb/usb-hcd.yaml
@@ -0,0 +1,25 @@
+# SPDX-License-Identifier: GPL-2.0
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/usb/usb-hcd.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Generic USB Host Controller Device Tree Bindings
+
+maintainers:
+  - Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+
+properties:
+  $nodename:
+    pattern: "^usb(@.*)?"
+
+  phys:
+    $ref: /schemas/types.yaml#/definitions/phandle-array
+    description:
+      List of all the USB PHYs on this HCD
+
+examples:
+  - |
+    usb {
+        phys = <&usb2_phy1>, <&usb3_phy1>;
+    };
diff --git a/Documentation/devicetree/bindings/usb/usb-ohci.txt b/Documentation/devicetree/bindings/usb/usb-ohci.txt
deleted file mode 100644
index aaaa5255c972..000000000000
--- a/Documentation/devicetree/bindings/usb/usb-ohci.txt
+++ /dev/null
@@ -1,35 +0,0 @@
-USB OHCI controllers
-
-Required properties:
-- compatible : "generic-ohci"
-- reg : ohci controller register range (address and length)
-- interrupts : ohci controller interrupt
-
-Optional properties:
-- big-endian-regs : boolean, set this for hcds with big-endian registers
-- big-endian-desc : boolean, set this for hcds with big-endian descriptors
-- big-endian : boolean, for hcds with big-endian-regs + big-endian-desc
-- no-big-frame-no : boolean, set if frame_no lives in bits [15:0] of HCCA
-- remote-wakeup-connected: remote wakeup is wired on the platform
-- num-ports : u32, to override the detected port count
-- clocks : a list of phandle + clock specifier pairs. In case of Renesas
-	   R-Car Gen3 SoCs:
-	   - if a host only channel: first clock should be host.
-	   - if a USB DRD channel: first clock should be host and second one
-				   should be peripheral.
-- phys : see usb-hcd.txt in the current directory
-- resets : a list of phandle + reset specifier pairs
-
-additionally the properties from usb-hcd.txt (in the current directory) are
-supported.
-
-Example:
-
-	ohci0: usb@1c14400 {
-		compatible = "allwinner,sun4i-a10-ohci", "generic-ohci";
-		reg = <0x01c14400 0x100>;
-		interrupts = <64>;
-		clocks = <&usb_clk 6>, <&ahb_gates 2>;
-		phys = <&usbphy 1>;
-		phy-names = "usb";
-	};
diff --git a/Documentation/devicetree/bindings/usb/usb-xhci.txt b/Documentation/devicetree/bindings/usb/usb-xhci.txt
index fea8b1545751..97400e8f8605 100644
--- a/Documentation/devicetree/bindings/usb/usb-xhci.txt
+++ b/Documentation/devicetree/bindings/usb/usb-xhci.txt
@@ -10,6 +10,7 @@ Required properties:
     - "renesas,xhci-r8a7743" for r8a7743 SoC
     - "renesas,xhci-r8a7744" for r8a7744 SoC
     - "renesas,xhci-r8a774a1" for r8a774a1 SoC
+    - "renesas,xhci-r8a774c0" for r8a774c0 SoC
     - "renesas,xhci-r8a7790" for r8a7790 SoC
     - "renesas,xhci-r8a7791" for r8a7791 SoC
     - "renesas,xhci-r8a7793" for r8a7793 SoC
diff --git a/Documentation/devicetree/bindings/usb/usb251xb.txt b/Documentation/devicetree/bindings/usb/usb251xb.txt
index 17915f64b8ee..bc7945e9dbfe 100644
--- a/Documentation/devicetree/bindings/usb/usb251xb.txt
+++ b/Documentation/devicetree/bindings/usb/usb251xb.txt
@@ -64,8 +64,10 @@ Optional properties :
  - power-on-time-ms : Specifies the time it takes from the time the host
 	initiates the power-on sequence to a port until the port has adequate
 	power. The value is given in ms in a 0 - 510 range (default is 100ms).
- - swap-dx-lanes : Specifies the ports which will swap the differential-pair
-	(D+/D-), default is not-swapped.
+ - swap-dx-lanes : Specifies the downstream ports which will swap the
+	differential-pair (D+/D-), default is not-swapped.
+ - swap-us-lanes : Selects the upstream port differential-pair (D+/D-)
+	swapping (boolean, default is not-swapped)
 
 Examples:
 	usb2512b@2c {
diff --git a/Documentation/devicetree/bindings/vendor-prefixes.txt b/Documentation/devicetree/bindings/vendor-prefixes.txt
index 686771d056c7..9ed399977297 100644
--- a/Documentation/devicetree/bindings/vendor-prefixes.txt
+++ b/Documentation/devicetree/bindings/vendor-prefixes.txt
@@ -305,6 +305,7 @@ oranth	Shenzhen Oranth Technology Co., Ltd.
 ORCL	Oracle Corporation
 orisetech	Orise Technology
 ortustech	Ortus Technology Co., Ltd.
+osddisplays	OSD Displays
 ovti	OmniVision Technologies
 oxsemi	Oxford Semiconductor, Ltd.
 panasonic	Panasonic Corporation
@@ -347,7 +348,9 @@ ricoh	Ricoh Co. Ltd.
 rikomagic	Rikomagic Tech Corp. Ltd
 riscv	RISC-V Foundation
 rockchip	Fuzhou Rockchip Electronics Co., Ltd
+rocktech	ROCKTECH DISPLAYS LIMITED
 rohm	ROHM Semiconductor Co., Ltd
+ronbo   Ronbo Electronics
 roofull	Shenzhen Roofull Technology Co, Ltd
 samsung	Samsung Semiconductor
 samtec	Samtec/Softing company
diff --git a/Documentation/doc-guide/index.rst b/Documentation/doc-guide/index.rst
index a7f95d7d3a63..603f3ff55d5a 100644
--- a/Documentation/doc-guide/index.rst
+++ b/Documentation/doc-guide/index.rst
@@ -7,9 +7,9 @@ How to write kernel documentation
 .. toctree::
    :maxdepth: 1
 
-   sphinx.rst
-   kernel-doc.rst
-   parse-headers.rst
+   sphinx
+   kernel-doc
+   parse-headers
 
 .. only::  subproject and html
 
diff --git a/Documentation/dontdiff b/Documentation/dontdiff
index 93693775639f..5eba889ea84d 100644
--- a/Documentation/dontdiff
+++ b/Documentation/dontdiff
@@ -127,7 +127,7 @@ flask.h
 fore200e_mkfirm
 fore200e_pca_fw.c*
 gconf
-gconf.glade.h
+gconf-cfg
 gen-devlist
 gen_crc32table
 gen_init_cpio
@@ -148,24 +148,22 @@ int32.c
 int4.c
 int8.c
 kallsyms
-kconfig
 keywords.c
 ksym.c*
 ksym.h*
-kxgettext
 *lex.c
 *lex.*.c
 linux
 logo_*.c
 logo_*_clut224.c
 logo_*_mono.c
-lxdialog
 mach-types
 mach-types.h
 machtypes.h
 map
 map_hugetlb
 mconf
+mconf-cfg
 miboot*
 mk_elfconfig
 mkboot
@@ -179,9 +177,11 @@ mktree
 mkutf8data
 modpost
 modules.builtin
+modules.builtin.modinfo
 modules.order
 modversions.h*
 nconf
+nconf-cfg
 ncscope.*
 offset.h
 oui.c*
@@ -201,6 +201,7 @@ pnmtologo
 ppc_defs.h*
 pss_boot.h
 qconf
+qconf-cfg
 r100_reg_safe.h
 r200_reg_safe.h
 r300_reg_safe.h
diff --git a/Documentation/driver-api/component.rst b/Documentation/driver-api/component.rst
index 2da4a8f20607..57e37590733f 100644
--- a/Documentation/driver-api/component.rst
+++ b/Documentation/driver-api/component.rst
@@ -1,3 +1,5 @@
+.. _component:
+
 ======================================
 Component Helper for Aggregate Drivers
 ======================================
diff --git a/Documentation/driver-api/soundwire/stream.rst b/Documentation/driver-api/soundwire/stream.rst
index 26a6064503fd..5351bd2f34a8 100644
--- a/Documentation/driver-api/soundwire/stream.rst
+++ b/Documentation/driver-api/soundwire/stream.rst
@@ -201,7 +201,7 @@ Bus implements below API for allocate a stream which needs to be called once
 per stream. From ASoC DPCM framework, this stream state maybe linked to
 .startup() operation.
 
-  .. code-block:: c
+.. code-block:: c
 
   int sdw_alloc_stream(char * stream_name);
 
@@ -228,7 +228,7 @@ the respective Master(s) and Slave(s) associated with stream. These APIs can
 only be invoked once by respective Master(s) and Slave(s). From ASoC DPCM
 framework, this stream state is linked to .hw_params() operation.
 
-  .. code-block:: c
+.. code-block:: c
 
   int sdw_stream_add_master(struct sdw_bus * bus,
 		struct sdw_stream_config * stream_config,
@@ -274,7 +274,7 @@ Bus implements below API for PREPARE state which needs to be called once per
 stream. From ASoC DPCM framework, this stream state is linked to
 .prepare() operation.
 
-  .. code-block:: c
+.. code-block:: c
 
   int sdw_prepare_stream(struct sdw_stream_runtime * stream);
 
@@ -304,7 +304,7 @@ Bus implements below API for ENABLE state which needs to be called once per
 stream. From ASoC DPCM framework, this stream state is linked to
 .trigger() start operation.
 
-  .. code-block:: c
+.. code-block:: c
 
   int sdw_enable_stream(struct sdw_stream_runtime * stream);
 
@@ -332,7 +332,7 @@ Bus implements below API for DISABLED state which needs to be called once
 per stream. From ASoC DPCM framework, this stream state is linked to
 .trigger() stop operation.
 
-  .. code-block:: c
+.. code-block:: c
 
   int sdw_disable_stream(struct sdw_stream_runtime * stream);
 
@@ -357,7 +357,7 @@ Bus implements below API for DEPREPARED state which needs to be called once
 per stream. From ASoC DPCM framework, this stream state is linked to
 .trigger() stop operation.
 
-  .. code-block:: c
+.. code-block:: c
 
   int sdw_deprepare_stream(struct sdw_stream_runtime * stream);
 
@@ -382,7 +382,7 @@ Bus implements below APIs for RELEASE state which needs to be called by
 all the Master(s) and Slave(s) associated with stream. From ASoC DPCM
 framework, this stream state is linked to .hw_free() operation.
 
-  .. code-block:: c
+.. code-block:: c
 
   int sdw_stream_remove_master(struct sdw_bus * bus,
 		struct sdw_stream_runtime * stream);
@@ -395,7 +395,7 @@ stream assigned as part of ALLOCATED state.
 
 In .shutdown() the data structure maintaining stream state are freed up.
 
-  .. code-block:: c
+.. code-block:: c
 
   void sdw_release_stream(struct sdw_stream_runtime * stream);
 
diff --git a/Documentation/driver-model/devres.txt b/Documentation/driver-model/devres.txt
index d7d6f01e81ff..99994a461359 100644
--- a/Documentation/driver-model/devres.txt
+++ b/Documentation/driver-model/devres.txt
@@ -256,6 +256,9 @@ DMA
   dmam_pool_create()
   dmam_pool_destroy()
 
+DRM
+  devm_drm_dev_init()
+
 GPIO
   devm_gpiod_get()
   devm_gpiod_get_index()
diff --git a/Documentation/gpu/drm-internals.rst b/Documentation/gpu/drm-internals.rst
index 3ae23a5454ac..966bd2d9f0cc 100644
--- a/Documentation/gpu/drm-internals.rst
+++ b/Documentation/gpu/drm-internals.rst
@@ -93,6 +93,11 @@ Device Instance and Driver Handling
 Driver Load
 -----------
 
+Component Helper Usage
+~~~~~~~~~~~~~~~~~~~~~~
+
+.. kernel-doc:: drivers/gpu/drm/drm_drv.c
+   :doc: component helper usage recommendations
 
 IRQ Helper Library
 ~~~~~~~~~~~~~~~~~~
diff --git a/Documentation/gpu/drm-kms-helpers.rst b/Documentation/gpu/drm-kms-helpers.rst
index 17ca7f8bf3d3..14102ae035dc 100644
--- a/Documentation/gpu/drm-kms-helpers.rst
+++ b/Documentation/gpu/drm-kms-helpers.rst
@@ -107,6 +107,12 @@ fbdev Helper Functions Reference
 .. kernel-doc:: drivers/gpu/drm/drm_fb_helper.c
    :export:
 
+format Helper Functions Reference
+=================================
+
+.. kernel-doc:: drivers/gpu/drm/drm_format_helper.c
+   :export:
+
 Framebuffer CMA Helper Functions Reference
 ==========================================
 
@@ -369,3 +375,15 @@ Legacy CRTC/Modeset Helper Functions Reference
 
 .. kernel-doc:: drivers/gpu/drm/drm_crtc_helper.c
    :export:
+
+SHMEM GEM Helper Reference
+==========================
+
+.. kernel-doc:: drivers/gpu/drm/drm_gem_shmem_helper.c
+   :doc: overview
+
+.. kernel-doc:: include/drm/drm_gem_shmem_helper.h
+   :internal:
+
+.. kernel-doc:: drivers/gpu/drm/drm_gem_shmem_helper.c
+   :export:
diff --git a/Documentation/gpu/kms-properties.csv b/Documentation/gpu/kms-properties.csv
index bfde04eddd14..07ed22ea3bd6 100644
--- a/Documentation/gpu/kms-properties.csv
+++ b/Documentation/gpu/kms-properties.csv
@@ -17,7 +17,6 @@ Owner Module/Drivers,Group,Property Name,Type,Property Values,Object attached,De
 ,Virtual GPU,“suggested X�,RANGE,"Min=0, Max=0xffffffff",Connector,property to suggest an X offset for a connector
 ,,“suggested Y�,RANGE,"Min=0, Max=0xffffffff",Connector,property to suggest an Y offset for a connector
 ,Optional,"""aspect ratio""",ENUM,"{ ""None"", ""4:3"", ""16:9"" }",Connector,TDB
-,Optional,"""content type""",ENUM,"{ ""No Data"", ""Graphics"", ""Photo"", ""Cinema"", ""Game"" }",Connector,TBD
 i915,Generic,"""Broadcast RGB""",ENUM,"{ ""Automatic"", ""Full"", ""Limited 16:235"" }",Connector,"When this property is set to Limited 16:235 and CTM is set, the hardware will be programmed with the result of the multiplication of CTM by the limited range matrix to ensure the pixels normaly in the range 0..1.0 are remapped to the range 16/255..235/255."
 ,,“audio�,ENUM,"{ ""force-dvi"", ""off"", ""auto"", ""on"" }",Connector,TBD
 ,SDVO-TV,“mode�,ENUM,"{ ""NTSC_M"", ""NTSC_J"", ""NTSC_443"", ""PAL_B"" } etc.",Connector,TBD
diff --git a/Documentation/gpu/meson.rst b/Documentation/gpu/meson.rst
index 479f6f51a13b..b9e2f9aa3bd8 100644
--- a/Documentation/gpu/meson.rst
+++ b/Documentation/gpu/meson.rst
@@ -42,12 +42,6 @@ Video Encoder
 .. kernel-doc:: drivers/gpu/drm/meson/meson_venc.c
    :doc: Video Encoder
 
-Video Canvas Management
-=======================
-
-.. kernel-doc:: drivers/gpu/drm/meson/meson_canvas.c
-   :doc: Canvas
-
 Video Clocks
 ============
 
diff --git a/Documentation/gpu/tinydrm.rst b/Documentation/gpu/tinydrm.rst
index a913644bfc19..33a41544f659 100644
--- a/Documentation/gpu/tinydrm.rst
+++ b/Documentation/gpu/tinydrm.rst
@@ -1,34 +1,22 @@
-==========================
-drm/tinydrm Driver library
-==========================
+============================
+drm/tinydrm Tiny DRM drivers
+============================
 
-.. kernel-doc:: drivers/gpu/drm/tinydrm/core/tinydrm-core.c
-   :doc: overview
-
-Core functionality
-==================
+tinydrm is a collection of DRM drivers that are so small they can fit in a
+single source file.
 
-.. kernel-doc:: drivers/gpu/drm/tinydrm/core/tinydrm-core.c
-   :doc: core
+Helpers
+=======
 
-.. kernel-doc:: include/drm/tinydrm/tinydrm.h
+.. kernel-doc:: include/drm/tinydrm/tinydrm-helpers.h
    :internal:
 
-.. kernel-doc:: drivers/gpu/drm/tinydrm/core/tinydrm-core.c
+.. kernel-doc:: drivers/gpu/drm/tinydrm/core/tinydrm-helpers.c
    :export:
 
 .. kernel-doc:: drivers/gpu/drm/tinydrm/core/tinydrm-pipe.c
    :export:
 
-Additional helpers
-==================
-
-.. kernel-doc:: include/drm/tinydrm/tinydrm-helpers.h
-   :internal:
-
-.. kernel-doc:: drivers/gpu/drm/tinydrm/core/tinydrm-helpers.c
-   :export:
-
 MIPI DBI Compatible Controllers
 ===============================
 
diff --git a/Documentation/gpu/todo.rst b/Documentation/gpu/todo.rst
index 159a4aba49e6..1528ad2d598b 100644
--- a/Documentation/gpu/todo.rst
+++ b/Documentation/gpu/todo.rst
@@ -215,12 +215,12 @@ Might be good to also have some igt testcases for this.
 
 Contact: Daniel Vetter, Noralf Tronnes
 
-Put a reservation_object into drm_gem_object
+Remove the ->gem_prime_res_obj callback
 --------------------------------------------
 
-This would remove the need for the ->gem_prime_res_obj callback. It would also
-allow us to implement generic helpers for waiting for a bo, allowing for quite a
-bit of refactoring in the various wait ioctl implementations.
+The ->gem_prime_res_obj callback can be removed from drivers by using the
+reservation_object in the drm_gem_object. It may also be possible to use the
+generic drm_gem_reservation_object_wait helper for waiting for a bo.
 
 Contact: Daniel Vetter
 
@@ -469,10 +469,6 @@ those drivers as simple as possible, so lots of room for refactoring:
   one of the ideas for having a shared dsi/dbi helper, abstracting away the
   transport details more.
 
-- Quick aside: The unregister devm stuff is kinda getting the lifetimes of
-  a drm_device wrong. Doesn't matter, since everyone else gets it wrong
-  too :-)
-
 Contact: Noralf Trønnes, Daniel Vetter
 
 AMD DC Display Driver
diff --git a/Documentation/i2c/busses/i2c-amd-mp2 b/Documentation/i2c/busses/i2c-amd-mp2
new file mode 100644
index 000000000000..6571487171f4
--- /dev/null
+++ b/Documentation/i2c/busses/i2c-amd-mp2
@@ -0,0 +1,23 @@
+Kernel driver i2c-amd-mp2
+
+Supported adapters:
+  * AMD MP2 PCIe interface
+
+Datasheet: not publicly available.
+
+Authors:
+	Shyam Sundar S K <Shyam-sundar.S-k@amd.com>
+	Nehal Shah <nehal-bakulchandra.shah@amd.com>
+	Elie Morisse <syniurge@gmail.com>
+
+Description
+-----------
+
+The MP2 is an ARM processor programmed as an I2C controller and communicating
+with the x86 host through PCI.
+
+If you see something like this:
+
+03:00.7 MP2 I2C controller: Advanced Micro Devices, Inc. [AMD] Device 15e6
+
+in your 'lspci -v', then this driver is for your device.
diff --git a/Documentation/i2c/busses/i2c-piix4 b/Documentation/i2c/busses/i2c-piix4
index aa959fd22450..2703bc3acad0 100644
--- a/Documentation/i2c/busses/i2c-piix4
+++ b/Documentation/i2c/busses/i2c-piix4
@@ -15,6 +15,8 @@ Supported adapters:
     http://support.amd.com/us/Embedded_TechDocs/44413.pdf
   * AMD Hudson-2, ML, CZ
     Datasheet: Not publicly available
+  * Hygon CZ
+    Datasheet: Not publicly available
   * Standard Microsystems (SMSC) SLC90E66 (Victory66) southbridge
     Datasheet: Publicly available at the SMSC website http://www.smsc.com
 
diff --git a/Documentation/kbuild/kbuild.txt b/Documentation/kbuild/kbuild.txt
index 8a3830b39c7d..9c230ea71963 100644
--- a/Documentation/kbuild/kbuild.txt
+++ b/Documentation/kbuild/kbuild.txt
@@ -11,6 +11,11 @@ modules.builtin
 This file lists all modules that are built into the kernel. This is used
 by modprobe to not fail when trying to load something builtin.
 
+modules.builtin.modinfo
+--------------------------------------------------
+This file contains modinfo from all modules that are built into the kernel.
+Unlike modinfo of a separate module, all fields are prefixed with module name.
+
 
 Environment variables
 
diff --git a/Documentation/livepatch/callbacks.txt b/Documentation/livepatch/callbacks.rst
index 182e31d4abce..470944aa8658 100644
--- a/Documentation/livepatch/callbacks.txt
+++ b/Documentation/livepatch/callbacks.rst
@@ -4,7 +4,7 @@
 
 Livepatch (un)patch-callbacks provide a mechanism for livepatch modules
 to execute callback functions when a kernel object is (un)patched.  They
-can be considered a "power feature" that extends livepatching abilities
+can be considered a **power feature** that **extends livepatching abilities**
 to include:
 
   - Safe updates to global data
@@ -17,6 +17,9 @@ In most cases, (un)patch callbacks will need to be used in conjunction
 with memory barriers and kernel synchronization primitives, like
 mutexes/spinlocks, or even stop_machine(), to avoid concurrency issues.
 
+1. Motivation
+=============
+
 Callbacks differ from existing kernel facilities:
 
   - Module init/exit code doesn't run when disabling and re-enabling a
@@ -28,21 +31,31 @@ Callbacks are part of the klp_object structure and their implementation
 is specific to that klp_object.  Other livepatch objects may or may not
 be patched, irrespective of the target klp_object's current state.
 
+2. Callback types
+=================
+
 Callbacks can be registered for the following livepatch actions:
 
-  * Pre-patch    - before a klp_object is patched
+  * Pre-patch
+                 - before a klp_object is patched
 
-  * Post-patch   - after a klp_object has been patched and is active
+  * Post-patch
+                 - after a klp_object has been patched and is active
                    across all tasks
 
-  * Pre-unpatch  - before a klp_object is unpatched (ie, patched code is
+  * Pre-unpatch
+                 - before a klp_object is unpatched (ie, patched code is
                    active), used to clean up post-patch callback
                    resources
 
-  * Post-unpatch - after a klp_object has been patched, all code has
+  * Post-unpatch
+                 - after a klp_object has been patched, all code has
                    been restored and no tasks are running patched code,
                    used to cleanup pre-patch callback resources
 
+3. How it works
+===============
+
 Each callback is optional, omitting one does not preclude specifying any
 other.  However, the livepatching core executes the handlers in
 symmetry: pre-patch callbacks have a post-unpatch counterpart and
@@ -86,11 +99,14 @@ If the object did successfully patch, but the patch transition never
 started for some reason (e.g., if another object failed to patch),
 only the post-unpatch callback will be called.
 
+4. Use cases
+============
 
-Example Use-cases
-=================
+Sample livepatch modules demonstrating the callback API can be found in
+samples/livepatch/ directory.  These samples were modified for use in
+kselftests and can be found in the lib/livepatch directory.
 
-Update global data
+Global data update
 ------------------
 
 A pre-patch callback can be useful to update a global variable.  For
@@ -103,24 +119,15 @@ patch the data *after* patching is complete with a post-patch callback,
 so that tcp_send_challenge_ack() could first be changed to read
 sysctl_tcp_challenge_ack_limit with READ_ONCE.
 
-
-Support __init and probe function patches
+__init and probe function patches support
 -----------------------------------------
 
 Although __init and probe functions are not directly livepatch-able, it
 may be possible to implement similar updates via pre/post-patch
 callbacks.
 
-48900cb6af42 ("virtio-net: drop NETIF_F_FRAGLIST") change the way that
+The commit ``48900cb6af42 ("virtio-net: drop NETIF_F_FRAGLIST")`` change the way that
 virtnet_probe() initialized its driver's net_device features.  A
 pre/post-patch callback could iterate over all such devices, making a
 similar change to their hw_features value.  (Client functions of the
 value may need to be updated accordingly.)
-
-
-Other Examples
-==============
-
-Sample livepatch modules demonstrating the callback API can be found in
-samples/livepatch/ directory.  These samples were modified for use in
-kselftests and can be found in the lib/livepatch directory.
diff --git a/Documentation/livepatch/cumulative-patches.txt b/Documentation/livepatch/cumulative-patches.rst
index 0012808e8d44..1931f318976a 100644
--- a/Documentation/livepatch/cumulative-patches.txt
+++ b/Documentation/livepatch/cumulative-patches.rst
@@ -18,7 +18,7 @@ Usage
 -----
 
 The atomic replace can be enabled by setting "replace" flag in struct klp_patch,
-for example:
+for example::
 
 	static struct klp_patch patch = {
 		.mod = THIS_MODULE,
@@ -49,19 +49,19 @@ Features
 
 The atomic replace allows:
 
-  + Atomically revert some functions in a previous patch while
+  - Atomically revert some functions in a previous patch while
     upgrading other functions.
 
-  + Remove eventual performance impact caused by core redirection
+  - Remove eventual performance impact caused by core redirection
     for functions that are no longer patched.
 
-  + Decrease user confusion about dependencies between livepatches.
+  - Decrease user confusion about dependencies between livepatches.
 
 
 Limitations:
 ------------
 
-  + Once the operation finishes, there is no straightforward way
+  - Once the operation finishes, there is no straightforward way
     to reverse it and restore the replaced patches atomically.
 
     A good practice is to set .replace flag in any released livepatch.
@@ -74,7 +74,7 @@ Limitations:
     only when the transition was not forced.
 
 
-  + Only the (un)patching callbacks from the _new_ cumulative livepatch are
+  - Only the (un)patching callbacks from the _new_ cumulative livepatch are
     executed. Any callbacks from the replaced patches are ignored.
 
     In other words, the cumulative patch is responsible for doing any actions
@@ -93,7 +93,7 @@ Limitations:
     enabled patches were called.
 
 
-  + There is no special handling of shadow variables. Livepatch authors
+  - There is no special handling of shadow variables. Livepatch authors
     must create their own rules how to pass them from one cumulative
     patch to the other. Especially that they should not blindly remove
     them in module_exit() functions.
diff --git a/Documentation/livepatch/index.rst b/Documentation/livepatch/index.rst
new file mode 100644
index 000000000000..edd291d51847
--- /dev/null
+++ b/Documentation/livepatch/index.rst
@@ -0,0 +1,21 @@
+:orphan:
+
+===================
+Kernel Livepatching
+===================
+
+.. toctree::
+    :maxdepth: 1
+
+    livepatch
+    callbacks
+    cumulative-patches
+    module-elf-format
+    shadow-vars
+
+.. only::  subproject and html
+
+   Indices
+   =======
+
+   * :ref:`genindex`
diff --git a/Documentation/livepatch/livepatch.txt b/Documentation/livepatch/livepatch.rst
index 4627b41ff02e..c2c598c4ead8 100644
--- a/Documentation/livepatch/livepatch.txt
+++ b/Documentation/livepatch/livepatch.rst
@@ -4,22 +4,22 @@ Livepatch
 
 This document outlines basic information about kernel livepatching.
 
-Table of Contents:
-
-1. Motivation
-2. Kprobes, Ftrace, Livepatching
-3. Consistency model
-4. Livepatch module
-   4.1. New functions
-   4.2. Metadata
-5. Livepatch life-cycle
-   5.1. Loading
-   5.2. Enabling
-   5.3. Replacing
-   5.4. Disabling
-   5.5. Removing
-6. Sysfs
-7. Limitations
+.. Table of Contents:
+
+    1. Motivation
+    2. Kprobes, Ftrace, Livepatching
+    3. Consistency model
+    4. Livepatch module
+       4.1. New functions
+       4.2. Metadata
+    5. Livepatch life-cycle
+       5.1. Loading
+       5.2. Enabling
+       5.3. Replacing
+       5.4. Disabling
+       5.5. Removing
+    6. Sysfs
+    7. Limitations
 
 
 1. Motivation
@@ -40,14 +40,14 @@ There are multiple mechanisms in the Linux kernel that are directly related
 to redirection of code execution; namely: kernel probes, function tracing,
 and livepatching:
 
-  + The kernel probes are the most generic. The code can be redirected by
+  - The kernel probes are the most generic. The code can be redirected by
     putting a breakpoint instruction instead of any instruction.
 
-  + The function tracer calls the code from a predefined location that is
+  - The function tracer calls the code from a predefined location that is
     close to the function entry point. This location is generated by the
     compiler using the '-pg' gcc option.
 
-  + Livepatching typically needs to redirect the code at the very beginning
+  - Livepatching typically needs to redirect the code at the very beginning
     of the function entry before the function parameters or the stack
     are in any way modified.
 
@@ -249,7 +249,7 @@ The patch contains only functions that are really modified. But they
 might want to access functions or data from the original source file
 that may only be locally accessible. This can be solved by a special
 relocation section in the generated livepatch module, see
-Documentation/livepatch/module-elf-format.txt for more details.
+Documentation/livepatch/module-elf-format.rst for more details.
 
 
 4.2. Metadata
@@ -258,7 +258,7 @@ Documentation/livepatch/module-elf-format.txt for more details.
 The patch is described by several structures that split the information
 into three levels:
 
-  + struct klp_func is defined for each patched function. It describes
+  - struct klp_func is defined for each patched function. It describes
     the relation between the original and the new implementation of a
     particular function.
 
@@ -275,7 +275,7 @@ into three levels:
     only for a particular object ( vmlinux or a kernel module ). Note that
     kallsyms allows for searching symbols according to the object name.
 
-  + struct klp_object defines an array of patched functions (struct
+  - struct klp_object defines an array of patched functions (struct
     klp_func) in the same object. Where the object is either vmlinux
     (NULL) or a module name.
 
@@ -285,7 +285,7 @@ into three levels:
     only when they are available.
 
 
-  + struct klp_patch defines an array of patched objects (struct
+  - struct klp_patch defines an array of patched objects (struct
     klp_object).
 
     This structure handles all patched functions consistently and eventually,
@@ -337,14 +337,16 @@ operation fails.
 Second, livepatch enters into a transition state where tasks are converging
 to the patched state. If an original function is patched for the first
 time, a function specific struct klp_ops is created and an universal
-ftrace handler is registered[*]. This stage is indicated by a value of '1'
+ftrace handler is registered\ [#]_. This stage is indicated by a value of '1'
 in /sys/kernel/livepatch/<name>/transition. For more information about
 this process, see the "Consistency model" section.
 
 Finally, once all tasks have been patched, the 'transition' value changes
 to '0'.
 
-[*] Note that functions might be patched multiple times. The ftrace handler
+.. [#]
+
+    Note that functions might be patched multiple times. The ftrace handler
     is registered only once for a given function. Further patches just add
     an entry to the list (see field `func_stack`) of the struct klp_ops.
     The right implementation is selected by the ftrace handler, see
@@ -368,7 +370,7 @@ the ftrace handler is unregistered and the struct klp_ops is
 freed when the related function is not modified by the new patch
 and func_stack list becomes empty.
 
-See Documentation/livepatch/cumulative-patches.txt for more details.
+See Documentation/livepatch/cumulative-patches.rst for more details.
 
 
 5.4. Disabling
@@ -421,7 +423,7 @@ See Documentation/ABI/testing/sysfs-kernel-livepatch for more details.
 
 The current Livepatch implementation has several limitations:
 
-  + Only functions that can be traced could be patched.
+  - Only functions that can be traced could be patched.
 
     Livepatch is based on the dynamic ftrace. In particular, functions
     implementing ftrace or the livepatch ftrace handler could not be
@@ -431,7 +433,7 @@ The current Livepatch implementation has several limitations:
 
 
 
-  + Livepatch works reliably only when the dynamic ftrace is located at
+  - Livepatch works reliably only when the dynamic ftrace is located at
     the very beginning of the function.
 
     The function need to be redirected before the stack or the function
@@ -445,7 +447,7 @@ The current Livepatch implementation has several limitations:
     this is handled on the ftrace level.
 
 
-  + Kretprobes using the ftrace framework conflict with the patched
+  - Kretprobes using the ftrace framework conflict with the patched
     functions.
 
     Both kretprobes and livepatches use a ftrace handler that modifies
@@ -453,7 +455,7 @@ The current Livepatch implementation has several limitations:
     is rejected when the handler is already in use by the other.
 
 
-  + Kprobes in the original function are ignored when the code is
+  - Kprobes in the original function are ignored when the code is
     redirected to the new implementation.
 
     There is a work in progress to add warnings about this situation.
diff --git a/Documentation/livepatch/module-elf-format.txt b/Documentation/livepatch/module-elf-format.rst
index f21a5289a09c..2a591e6f8e6c 100644
--- a/Documentation/livepatch/module-elf-format.txt
+++ b/Documentation/livepatch/module-elf-format.rst
@@ -4,33 +4,21 @@ Livepatch module Elf format
 
 This document outlines the Elf format requirements that livepatch modules must follow.
 
------------------
-Table of Contents
------------------
-0. Background and motivation
-1. Livepatch modinfo field
-2. Livepatch relocation sections
-   2.1 What are livepatch relocation sections?
-   2.2 Livepatch relocation section format
-       2.2.1 Required flags
-       2.2.2 Required name format
-       2.2.3 Example livepatch relocation section names
-       2.2.4 Example `readelf --sections` output
-       2.2.5 Example `readelf --relocs` output
-3. Livepatch symbols
-   3.1 What are livepatch symbols?
-   3.2 A livepatch module's symbol table
-   3.3 Livepatch symbol format
-       3.3.1 Required flags
-       3.3.2 Required name format
-       3.3.3 Example livepatch symbol names
-       3.3.4 Example `readelf --symbols` output
-4. Architecture-specific sections
-5. Symbol table and Elf section access
-
-----------------------------
-0. Background and motivation
-----------------------------
+
+.. Table of Contents
+
+   1. Background and motivation
+   2. Livepatch modinfo field
+   3. Livepatch relocation sections
+      3.1 Livepatch relocation section format
+   4. Livepatch symbols
+      4.1 A livepatch module's symbol table
+      4.2 Livepatch symbol format
+   5. Architecture-specific sections
+   6. Symbol table and Elf section access
+
+1. Background and motivation
+============================
 
 Formerly, livepatch required separate architecture-specific code to write
 relocations. However, arch-specific code to write relocations already
@@ -52,8 +40,8 @@ relocation sections and symbols, which are described in this document. The
 Elf constants used to mark livepatch symbols and relocation sections were
 selected from OS-specific ranges according to the definitions from glibc.
 
-0.1 Why does livepatch need to write its own relocations?
----------------------------------------------------------
+Why does livepatch need to write its own relocations?
+-----------------------------------------------------
 A typical livepatch module contains patched versions of functions that can
 reference non-exported global symbols and non-included local symbols.
 Relocations referencing these types of symbols cannot be left in as-is
@@ -72,13 +60,8 @@ relas reference are special livepatch symbols (see section 2 and 3). The
 arch-specific livepatch relocation code is replaced by a call to
 apply_relocate_add().
 
-================================
-PATCH MODULE FORMAT REQUIREMENTS
-================================
-
---------------------------
-1. Livepatch modinfo field
---------------------------
+2. Livepatch modinfo field
+==========================
 
 Livepatch modules are required to have the "livepatch" modinfo attribute.
 See the sample livepatch module in samples/livepatch/ for how this is done.
@@ -87,22 +70,23 @@ Livepatch modules can be identified by users by using the 'modinfo' command
 and looking for the presence of the "livepatch" field. This field is also
 used by the kernel module loader to identify livepatch modules.
 
-Example modinfo output:
------------------------
-% modinfo livepatch-meminfo.ko
-filename:		livepatch-meminfo.ko
-livepatch:		Y
-license:		GPL
-depends:
-vermagic:		4.3.0+ SMP mod_unload
-
---------------------------------
-2. Livepatch relocation sections
---------------------------------
-
--------------------------------------------
-2.1 What are livepatch relocation sections?
--------------------------------------------
+Example:
+--------
+
+**Modinfo output:**
+
+::
+
+	% modinfo livepatch-meminfo.ko
+	filename:		livepatch-meminfo.ko
+	livepatch:		Y
+	license:		GPL
+	depends:
+	vermagic:		4.3.0+ SMP mod_unload
+
+3. Livepatch relocation sections
+================================
+
 A livepatch module manages its own Elf relocation sections to apply
 relocations to modules as well as to the kernel (vmlinux) at the
 appropriate time. For example, if a patch module patches a driver that is
@@ -127,12 +111,9 @@ Every symbol referenced by a rela in a livepatch relocation section is a
 livepatch symbol. These must be resolved before livepatch can call
 apply_relocate_add(). See Section 3 for more information.
 
----------------------------------------
-2.2 Livepatch relocation section format
----------------------------------------
+3.1 Livepatch relocation section format
+=======================================
 
-2.2.1 Required flags
---------------------
 Livepatch relocation sections must be marked with the SHF_RELA_LIVEPATCH
 section flag. See include/uapi/linux/elf.h for the definition. The module
 loader recognizes this flag and will avoid applying those relocation sections
@@ -140,28 +121,39 @@ at patch module load time. These sections must also be marked with SHF_ALLOC,
 so that the module loader doesn't discard them on module load (i.e. they will
 be copied into memory along with the other SHF_ALLOC sections).
 
-2.2.2 Required name format
---------------------------
-The name of a livepatch relocation section must conform to the following format:
+The name of a livepatch relocation section must conform to the following
+format::
 
-.klp.rela.objname.section_name
-^        ^^     ^ ^          ^
-|________||_____| |__________|
-   [A]      [B]        [C]
+  .klp.rela.objname.section_name
+  ^        ^^     ^ ^          ^
+  |________||_____| |__________|
+     [A]      [B]        [C]
 
-[A] The relocation section name is prefixed with the string ".klp.rela."
-[B] The name of the object (i.e. "vmlinux" or name of module) to
-    which the relocation section belongs follows immediately after the prefix.
-[C] The actual name of the section to which this relocation section applies.
+[A]
+  The relocation section name is prefixed with the string ".klp.rela."
 
-2.2.3 Example livepatch relocation section names:
--------------------------------------------------
-.klp.rela.ext4.text.ext4_attr_store
-.klp.rela.vmlinux.text.cmdline_proc_show
+[B]
+  The name of the object (i.e. "vmlinux" or name of module) to
+  which the relocation section belongs follows immediately after the prefix.
+
+[C]
+  The actual name of the section to which this relocation section applies.
+
+Examples:
+---------
+
+**Livepatch relocation section names:**
+
+::
+
+  .klp.rela.ext4.text.ext4_attr_store
+  .klp.rela.vmlinux.text.cmdline_proc_show
+
+**`readelf --sections` output for a patch
+module that patches vmlinux and modules 9p, btrfs, ext4:**
+
+::
 
-2.2.4 Example `readelf --sections` output for a patch
-module that patches vmlinux and modules 9p, btrfs, ext4:
---------------------------------------------------------
   Section Headers:
   [Nr] Name                          Type                    Address          Off    Size   ES Flg Lk Inf Al
   [ snip ]
@@ -175,31 +167,33 @@ module that patches vmlinux and modules 9p, btrfs, ext4:
   [ snip ]                                       ^                                             ^
                                                  |                                             |
                                                 [*]                                           [*]
-[*] Livepatch relocation sections are SHT_RELA sections but with a few special
-characteristics. Notice that they are marked SHF_ALLOC ("A") so that they will
-not be discarded when the module is loaded into memory, as well as with the
-SHF_RELA_LIVEPATCH flag ("o" - for OS-specific).
-
-2.2.5 Example `readelf --relocs` output for a patch module:
------------------------------------------------------------
-Relocation section '.klp.rela.btrfs.text.btrfs_feature_attr_show' at offset 0x2ba0 contains 4 entries:
-    Offset             Info             Type               Symbol's Value  Symbol's Name + Addend
-000000000000001f  0000005e00000002 R_X86_64_PC32          0000000000000000 .klp.sym.vmlinux.printk,0 - 4
-0000000000000028  0000003d0000000b R_X86_64_32S           0000000000000000 .klp.sym.btrfs.btrfs_ktype,0 + 0
-0000000000000036  0000003b00000002 R_X86_64_PC32          0000000000000000 .klp.sym.btrfs.can_modify_feature.isra.3,0 - 4
-000000000000004c  0000004900000002 R_X86_64_PC32          0000000000000000 .klp.sym.vmlinux.snprintf,0 - 4
-[ snip ]                                                                   ^
-                                                                           |
-                                                                          [*]
-[*] Every symbol referenced by a relocation is a livepatch symbol.
-
---------------------
-3. Livepatch symbols
---------------------
-
--------------------------------
-3.1 What are livepatch symbols?
--------------------------------
+
+[*]
+  Livepatch relocation sections are SHT_RELA sections but with a few special
+  characteristics. Notice that they are marked SHF_ALLOC ("A") so that they will
+  not be discarded when the module is loaded into memory, as well as with the
+  SHF_RELA_LIVEPATCH flag ("o" - for OS-specific).
+
+**`readelf --relocs` output for a patch module:**
+
+::
+
+  Relocation section '.klp.rela.btrfs.text.btrfs_feature_attr_show' at offset 0x2ba0 contains 4 entries:
+      Offset             Info             Type               Symbol's Value  Symbol's Name + Addend
+  000000000000001f  0000005e00000002 R_X86_64_PC32          0000000000000000 .klp.sym.vmlinux.printk,0 - 4
+  0000000000000028  0000003d0000000b R_X86_64_32S           0000000000000000 .klp.sym.btrfs.btrfs_ktype,0 + 0
+  0000000000000036  0000003b00000002 R_X86_64_PC32          0000000000000000 .klp.sym.btrfs.can_modify_feature.isra.3,0 - 4
+  000000000000004c  0000004900000002 R_X86_64_PC32          0000000000000000 .klp.sym.vmlinux.snprintf,0 - 4
+  [ snip ]                                                                   ^
+                                                                             |
+                                                                            [*]
+
+[*]
+  Every symbol referenced by a relocation is a livepatch symbol.
+
+4. Livepatch symbols
+====================
+
 Livepatch symbols are symbols referred to by livepatch relocation sections.
 These are symbols accessed from new versions of functions for patched
 objects, whose addresses cannot be resolved by the module loader (because
@@ -219,9 +213,8 @@ loader can identify and ignore them. Livepatch modules keep these symbols
 in their symbol tables, and the symbol table is made accessible through
 module->symtab.
 
--------------------------------------
-3.2 A livepatch module's symbol table
--------------------------------------
+4.1 A livepatch module's symbol table
+=====================================
 Normally, a stripped down copy of a module's symbol table (containing only
 "core" symbols) is made available through module->symtab (See layout_symtab()
 in kernel/module.c). For livepatch modules, the symbol table copied into memory
@@ -231,71 +224,82 @@ relocation section refer to their respective symbols with their symbol indices,
 and the original symbol indices (and thus the symtab ordering) must be
 preserved in order for apply_relocate_add() to find the right symbol.
 
-For example, take this particular rela from a livepatch module:
-Relocation section '.klp.rela.btrfs.text.btrfs_feature_attr_show' at offset 0x2ba0 contains 4 entries:
-    Offset             Info             Type               Symbol's Value  Symbol's Name + Addend
-000000000000001f  0000005e00000002 R_X86_64_PC32          0000000000000000 .klp.sym.vmlinux.printk,0 - 4
-
-This rela refers to the symbol '.klp.sym.vmlinux.printk,0', and the symbol index is encoded
-in 'Info'. Here its symbol index is 0x5e, which is 94 in decimal, which refers to the
-symbol index 94.
-And in this patch module's corresponding symbol table, symbol index 94 refers to that very symbol:
-[ snip ]
-94: 0000000000000000     0 NOTYPE  GLOBAL DEFAULT OS [0xff20] .klp.sym.vmlinux.printk,0
-[ snip ]
-
----------------------------
-3.3 Livepatch symbol format
----------------------------
-
-3.3.1 Required flags
---------------------
+For example, take this particular rela from a livepatch module:::
+
+  Relocation section '.klp.rela.btrfs.text.btrfs_feature_attr_show' at offset 0x2ba0 contains 4 entries:
+      Offset             Info             Type               Symbol's Value  Symbol's Name + Addend
+  000000000000001f  0000005e00000002 R_X86_64_PC32          0000000000000000 .klp.sym.vmlinux.printk,0 - 4
+
+  This rela refers to the symbol '.klp.sym.vmlinux.printk,0', and the symbol index is encoded
+  in 'Info'. Here its symbol index is 0x5e, which is 94 in decimal, which refers to the
+  symbol index 94.
+  And in this patch module's corresponding symbol table, symbol index 94 refers to that very symbol:
+  [ snip ]
+  94: 0000000000000000     0 NOTYPE  GLOBAL DEFAULT OS [0xff20] .klp.sym.vmlinux.printk,0
+  [ snip ]
+
+4.2 Livepatch symbol format
+===========================
+
 Livepatch symbols must have their section index marked as SHN_LIVEPATCH, so
 that the module loader can identify them and not attempt to resolve them.
 See include/uapi/linux/elf.h for the actual definitions.
 
-3.3.2 Required name format
---------------------------
-Livepatch symbol names must conform to the following format:
-
-.klp.sym.objname.symbol_name,sympos
-^       ^^     ^ ^         ^ ^
-|_______||_____| |_________| |
-   [A]     [B]       [C]    [D]
-
-[A] The symbol name is prefixed with the string ".klp.sym."
-[B] The name of the object (i.e. "vmlinux" or name of module) to
-    which the symbol belongs follows immediately after the prefix.
-[C] The actual name of the symbol.
-[D] The position of the symbol in the object (as according to kallsyms)
-    This is used to differentiate duplicate symbols within the same
-    object. The symbol position is expressed numerically (0, 1, 2...).
-    The symbol position of a unique symbol is 0.
-
-3.3.3 Example livepatch symbol names:
--------------------------------------
-.klp.sym.vmlinux.snprintf,0
-.klp.sym.vmlinux.printk,0
-.klp.sym.btrfs.btrfs_ktype,0
-
-3.3.4 Example `readelf --symbols` output for a patch module:
-------------------------------------------------------------
-Symbol table '.symtab' contains 127 entries:
-   Num:    Value          Size Type    Bind   Vis     Ndx         Name
-   [ snip ]
-    73: 0000000000000000     0 NOTYPE  GLOBAL DEFAULT OS [0xff20] .klp.sym.vmlinux.snprintf,0
-    74: 0000000000000000     0 NOTYPE  GLOBAL DEFAULT OS [0xff20] .klp.sym.vmlinux.capable,0
-    75: 0000000000000000     0 NOTYPE  GLOBAL DEFAULT OS [0xff20] .klp.sym.vmlinux.find_next_bit,0
-    76: 0000000000000000     0 NOTYPE  GLOBAL DEFAULT OS [0xff20] .klp.sym.vmlinux.si_swapinfo,0
-  [ snip ]                                               ^
-                                                         |
-                                                        [*]
-[*] Note that the 'Ndx' (Section index) for these symbols is SHN_LIVEPATCH (0xff20).
-    "OS" means OS-specific.
-
----------------------------------
-4. Architecture-specific sections
----------------------------------
+Livepatch symbol names must conform to the following format::
+
+  .klp.sym.objname.symbol_name,sympos
+  ^       ^^     ^ ^         ^ ^
+  |_______||_____| |_________| |
+     [A]     [B]       [C]    [D]
+
+[A]
+  The symbol name is prefixed with the string ".klp.sym."
+
+[B]
+  The name of the object (i.e. "vmlinux" or name of module) to
+  which the symbol belongs follows immediately after the prefix.
+
+[C]
+  The actual name of the symbol.
+
+[D]
+  The position of the symbol in the object (as according to kallsyms)
+  This is used to differentiate duplicate symbols within the same
+  object. The symbol position is expressed numerically (0, 1, 2...).
+  The symbol position of a unique symbol is 0.
+
+Examples:
+---------
+
+**Livepatch symbol names:**
+
+::
+
+	.klp.sym.vmlinux.snprintf,0
+	.klp.sym.vmlinux.printk,0
+	.klp.sym.btrfs.btrfs_ktype,0
+
+**`readelf --symbols` output for a patch module:**
+
+::
+
+  Symbol table '.symtab' contains 127 entries:
+     Num:    Value          Size Type    Bind   Vis     Ndx         Name
+     [ snip ]
+      73: 0000000000000000     0 NOTYPE  GLOBAL DEFAULT OS [0xff20] .klp.sym.vmlinux.snprintf,0
+      74: 0000000000000000     0 NOTYPE  GLOBAL DEFAULT OS [0xff20] .klp.sym.vmlinux.capable,0
+      75: 0000000000000000     0 NOTYPE  GLOBAL DEFAULT OS [0xff20] .klp.sym.vmlinux.find_next_bit,0
+      76: 0000000000000000     0 NOTYPE  GLOBAL DEFAULT OS [0xff20] .klp.sym.vmlinux.si_swapinfo,0
+    [ snip ]                                               ^
+                                                           |
+                                                          [*]
+
+[*]
+  Note that the 'Ndx' (Section index) for these symbols is SHN_LIVEPATCH (0xff20).
+  "OS" means OS-specific.
+
+5. Architecture-specific sections
+=================================
 Architectures may override arch_klp_init_object_loaded() to perform
 additional arch-specific tasks when a target module loads, such as applying
 arch-specific sections. On x86 for example, we must apply per-object
@@ -304,20 +308,19 @@ These sections must be prefixed with ".klp.arch.$objname." so that they can
 be easily identified when iterating through a patch module's Elf sections
 (See arch/x86/kernel/livepatch.c for a complete example).
 
---------------------------------------
-5. Symbol table and Elf section access
---------------------------------------
+6. Symbol table and Elf section access
+======================================
 A livepatch module's symbol table is accessible through module->symtab.
 
 Since apply_relocate_add() requires access to a module's section headers,
 symbol table, and relocation section indices, Elf information is preserved for
 livepatch modules and is made accessible by the module loader through
 module->klp_info, which is a klp_modinfo struct. When a livepatch module loads,
-this struct is filled in by the module loader. Its fields are documented below:
-
-struct klp_modinfo {
-	Elf_Ehdr hdr; /* Elf header */
-	Elf_Shdr *sechdrs; /* Section header table */
-	char *secstrings; /* String table for the section headers */
-	unsigned int symndx; /* The symbol table section index */
-};
+this struct is filled in by the module loader. Its fields are documented below::
+
+	struct klp_modinfo {
+		Elf_Ehdr hdr; /* Elf header */
+		Elf_Shdr *sechdrs; /* Section header table */
+		char *secstrings; /* String table for the section headers */
+		unsigned int symndx; /* The symbol table section index */
+	};
diff --git a/Documentation/livepatch/shadow-vars.txt b/Documentation/livepatch/shadow-vars.rst
index ecc09a7be5dd..c05715aeafa4 100644
--- a/Documentation/livepatch/shadow-vars.txt
+++ b/Documentation/livepatch/shadow-vars.rst
@@ -27,10 +27,13 @@ A hashtable references all shadow variables.  These references are
 stored and retrieved through a <obj, id> pair.
 
 * The klp_shadow variable data structure encapsulates both tracking
-meta-data and shadow-data:
+  meta-data and shadow-data:
+
   - meta-data
+
     - obj - pointer to parent object
     - id - data identifier
+
   - data[] - storage for shadow data
 
 It is important to note that the klp_shadow_alloc() and
@@ -47,31 +50,43 @@ to do actions that can be done only once when a new variable is allocated.
 
 * klp_shadow_alloc() - allocate and add a new shadow variable
   - search hashtable for <obj, id> pair
+
   - if exists
+
     - WARN and return NULL
+
   - if <obj, id> doesn't already exist
+
     - allocate a new shadow variable
     - initialize the variable using a custom constructor and data when provided
     - add <obj, id> to the global hashtable
 
 * klp_shadow_get_or_alloc() - get existing or alloc a new shadow variable
   - search hashtable for <obj, id> pair
+
   - if exists
+
     - return existing shadow variable
+
   - if <obj, id> doesn't already exist
+
     - allocate a new shadow variable
     - initialize the variable using a custom constructor and data when provided
     - add <obj, id> pair to the global hashtable
 
 * klp_shadow_free() - detach and free a <obj, id> shadow variable
   - find and remove a <obj, id> reference from global hashtable
+
     - if found
+
       - call destructor function if defined
       - free shadow variable
 
 * klp_shadow_free_all() - detach and free all <*, id> shadow variables
   - find and remove any <*, id> references from global hashtable
+
     - if found
+
       - call destructor function if defined
       - free shadow variable
 
@@ -102,12 +117,12 @@ parent "goes live" (ie, any shadow variable get-API requests are made
 for this <obj, id> pair.)
 
 For commit 1d147bfa6429, when a parent sta_info structure is allocated,
-allocate a shadow copy of the ps_lock pointer, then initialize it:
+allocate a shadow copy of the ps_lock pointer, then initialize it::
 
-#define PS_LOCK 1
-struct sta_info *sta_info_alloc(struct ieee80211_sub_if_data *sdata,
-				const u8 *addr, gfp_t gfp)
-{
+  #define PS_LOCK 1
+  struct sta_info *sta_info_alloc(struct ieee80211_sub_if_data *sdata,
+				  const u8 *addr, gfp_t gfp)
+  {
 	struct sta_info *sta;
 	spinlock_t *ps_lock;
 
@@ -123,10 +138,10 @@ struct sta_info *sta_info_alloc(struct ieee80211_sub_if_data *sdata,
 	...
 
 When requiring a ps_lock, query the shadow variable API to retrieve one
-for a specific struct sta_info:
+for a specific struct sta_info:::
 
-void ieee80211_sta_ps_deliver_wakeup(struct sta_info *sta)
-{
+  void ieee80211_sta_ps_deliver_wakeup(struct sta_info *sta)
+  {
 	spinlock_t *ps_lock;
 
 	/* sync with ieee80211_tx_h_unicast_ps_buf */
@@ -136,10 +151,10 @@ void ieee80211_sta_ps_deliver_wakeup(struct sta_info *sta)
 	...
 
 When the parent sta_info structure is freed, first free the shadow
-variable:
+variable::
 
-void sta_info_free(struct ieee80211_local *local, struct sta_info *sta)
-{
+  void sta_info_free(struct ieee80211_local *local, struct sta_info *sta)
+  {
 	klp_shadow_free(sta, PS_LOCK, NULL);
 	kfree(sta);
 	...
@@ -155,19 +170,19 @@ these cases, the klp_shadow_get_or_alloc() call can be used to attach
 shadow variables to parents already in-flight.
 
 For commit 1d147bfa6429, a good spot to allocate a shadow spinlock is
-inside ieee80211_sta_ps_deliver_wakeup():
+inside ieee80211_sta_ps_deliver_wakeup()::
 
-int ps_lock_shadow_ctor(void *obj, void *shadow_data, void *ctor_data)
-{
+  int ps_lock_shadow_ctor(void *obj, void *shadow_data, void *ctor_data)
+  {
 	spinlock_t *lock = shadow_data;
 
 	spin_lock_init(lock);
 	return 0;
-}
+  }
 
-#define PS_LOCK 1
-void ieee80211_sta_ps_deliver_wakeup(struct sta_info *sta)
-{
+  #define PS_LOCK 1
+  void ieee80211_sta_ps_deliver_wakeup(struct sta_info *sta)
+  {
 	spinlock_t *ps_lock;
 
 	/* sync with ieee80211_tx_h_unicast_ps_buf */
@@ -200,10 +215,12 @@ suggests how to handle the parent object.
 =============
 
 * https://github.com/dynup/kpatch
-The livepatch implementation is based on the kpatch version of shadow
-variables.
+
+  The livepatch implementation is based on the kpatch version of shadow
+  variables.
 
 * http://files.mkgnu.net/files/dynamos/doc/papers/dynamos_eurosys_07.pdf
-Dynamic and Adaptive Updates of Non-Quiescent Subsystems in Commodity
-Operating System Kernels (Kritis Makris, Kyung Dong Ryu 2007) presented
-a datatype update technique called "shadow data structures".
+
+  Dynamic and Adaptive Updates of Non-Quiescent Subsystems in Commodity
+  Operating System Kernels (Kritis Makris, Kyung Dong Ryu 2007) presented
+  a datatype update technique called "shadow data structures".
diff --git a/Documentation/media/index.rst b/Documentation/media/index.rst
index 0a222fc1d7ca..0301c25ff887 100644
--- a/Documentation/media/index.rst
+++ b/Documentation/media/index.rst
@@ -18,7 +18,7 @@ Linux Media Subsystem Documentation
    v4l-drivers/index
    cec-drivers/index
 
-.. only::  subproject
+.. only:: html and subproject
 
    Indices
    =======
diff --git a/Documentation/media/kapi/mc-core.rst b/Documentation/media/kapi/mc-core.rst
index f930725e0d6b..05bba0b61748 100644
--- a/Documentation/media/kapi/mc-core.rst
+++ b/Documentation/media/kapi/mc-core.rst
@@ -259,6 +259,45 @@ Subsystems should facilitate link validation by providing subsystem specific
 helper functions to provide easy access for commonly needed information, and
 in the end provide a way to use driver-specific callbacks.
 
+Media Controller Device Allocator API
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+When the media device belongs to more than one driver, the shared media
+device is allocated with the shared struct device as the key for look ups.
+
+The shared media device should stay in registered state until the last
+driver unregisters it. In addition, the media device should be released when
+all the references are released. Each driver gets a reference to the media
+device during probe, when it allocates the media device. If media device is
+already allocated, the allocate API bumps up the refcount and returns the
+existing media device. The driver puts the reference back in its disconnect
+routine when it calls :c:func:`media_device_delete()`.
+
+The media device is unregistered and cleaned up from the kref put handler to
+ensure that the media device stays in registered state until the last driver
+unregisters the media device.
+
+**Driver Usage**
+
+Drivers should use the appropriate media-core routines to manage the shared
+media device life-time handling the two states:
+1. allocate -> register -> delete
+2. get reference to already registered device -> delete
+
+call :c:func:`media_device_delete()` routine to make sure the shared media
+device delete is handled correctly.
+
+**driver probe:**
+Call :c:func:`media_device_usb_allocate()` to allocate or get a reference
+Call :c:func:`media_device_register()`, if media devnode isn't registered
+
+**driver disconnect:**
+Call :c:func:`media_device_delete()` to free the media_device. Freeing is
+handled by the kref put handler.
+
+API Definitions
+^^^^^^^^^^^^^^^
+
 .. kernel-doc:: include/media/media-device.h
 
 .. kernel-doc:: include/media/media-devnode.h
@@ -266,3 +305,5 @@ in the end provide a way to use driver-specific callbacks.
 .. kernel-doc:: include/media/media-entity.h
 
 .. kernel-doc:: include/media/media-request.h
+
+.. kernel-doc:: include/media/media-dev-allocator.h
diff --git a/Documentation/media/lirc.h.rst.exceptions b/Documentation/media/lirc.h.rst.exceptions
index 7a8b8ff4f076..ac768d769113 100644
--- a/Documentation/media/lirc.h.rst.exceptions
+++ b/Documentation/media/lirc.h.rst.exceptions
@@ -63,6 +63,7 @@ ignore symbol RC_PROTO_IMON
 ignore symbol RC_PROTO_RCMM12
 ignore symbol RC_PROTO_RCMM24
 ignore symbol RC_PROTO_RCMM32
+ignore symbol RC_PROTO_XBOX_DVD
 
 # Undocumented macros
 
diff --git a/Documentation/media/uapi/mediactl/request-api.rst b/Documentation/media/uapi/mediactl/request-api.rst
index 1ad631e549fe..a74c82d95609 100644
--- a/Documentation/media/uapi/mediactl/request-api.rst
+++ b/Documentation/media/uapi/mediactl/request-api.rst
@@ -93,7 +93,7 @@ A queued request cannot be modified anymore.
 .. caution::
    For :ref:`memory-to-memory devices <mem2mem>` you can use requests only for
    output buffers, not for capture buffers. Attempting to add a capture buffer
-   to a request will result in an ``EACCES`` error.
+   to a request will result in an ``EBADR`` error.
 
 If the request contains configurations for multiple entities, individual drivers
 may synchronize so the requested pipeline's topology is applied before the
diff --git a/Documentation/media/uapi/v4l/buffer.rst b/Documentation/media/uapi/v4l/buffer.rst
index 81ffdcb89057..1cbd9cde57f3 100644
--- a/Documentation/media/uapi/v4l/buffer.rst
+++ b/Documentation/media/uapi/v4l/buffer.rst
@@ -165,7 +165,7 @@ of appropriately sized buffers for each use case).
 struct v4l2_buffer
 ==================
 
-.. tabularcolumns:: |p{2.8cm}|p{2.5cm}|p{1.3cm}|p{10.5cm}|
+.. tabularcolumns:: |p{2.8cm}|p{2.5cm}|p{1.6cm}|p{10.2cm}|
 
 .. cssclass:: longtable
 
@@ -326,7 +326,7 @@ struct v4l2_buffer
 	Applications should not set ``V4L2_BUF_FLAG_REQUEST_FD`` for any ioctls
 	other than :ref:`VIDIOC_QBUF <VIDIOC_QBUF>`.
 
-	If the device does not support requests, then ``EACCES`` will be returned.
+	If the device does not support requests, then ``EBADR`` will be returned.
 	If requests are supported but an invalid request file descriptor is
 	given, then ``EINVAL`` will be returned.
 
@@ -420,7 +420,7 @@ enum v4l2_buf_type
 
 .. cssclass:: longtable
 
-.. tabularcolumns:: |p{7.2cm}|p{0.6cm}|p{9.7cm}|
+.. tabularcolumns:: |p{7.8cm}|p{0.6cm}|p{9.1cm}|
 
 .. flat-table::
     :header-rows:  0
@@ -482,7 +482,11 @@ enum v4l2_buf_type
 Buffer Flags
 ============
 
-.. tabularcolumns:: |p{7.0cm}|p{2.2cm}|p{8.3cm}|
+.. raw:: latex
+
+    \small
+
+.. tabularcolumns:: |p{7.0cm}|p{2.1cm}|p{8.4cm}|
 
 .. cssclass:: longtable
 
@@ -681,6 +685,9 @@ Buffer Flags
 	exposure of the frame has begun. This is only valid for the
 	``V4L2_BUF_TYPE_VIDEO_CAPTURE`` buffer type.
 
+.. raw:: latex
+
+    \normalsize
 
 
 .. c:type:: v4l2_memory
@@ -688,7 +695,7 @@ Buffer Flags
 enum v4l2_memory
 ================
 
-.. tabularcolumns:: |p{6.6cm}|p{2.2cm}|p{8.7cm}|
+.. tabularcolumns:: |p{5.0cm}|p{0.8cm}|p{11.7cm}|
 
 .. flat-table::
     :header-rows:  0
@@ -724,7 +731,7 @@ The :c:type:`v4l2_buffer_timecode` structure is designed to hold a
 struct v4l2_timecode
 --------------------
 
-.. tabularcolumns:: |p{4.4cm}|p{4.4cm}|p{8.7cm}|
+.. tabularcolumns:: |p{1.4cm}|p{2.8cm}|p{12.3cm}|
 
 .. flat-table::
     :header-rows:  0
@@ -761,7 +768,7 @@ struct v4l2_timecode
 Timecode Types
 --------------
 
-.. tabularcolumns:: |p{6.6cm}|p{2.2cm}|p{8.7cm}|
+.. tabularcolumns:: |p{5.6cm}|p{0.8cm}|p{11.1cm}|
 
 .. flat-table::
     :header-rows:  0
diff --git a/Documentation/media/uapi/v4l/colorspaces-defs.rst b/Documentation/media/uapi/v4l/colorspaces-defs.rst
index c4e8fc620379..e122bbe3d799 100644
--- a/Documentation/media/uapi/v4l/colorspaces-defs.rst
+++ b/Documentation/media/uapi/v4l/colorspaces-defs.rst
@@ -39,7 +39,7 @@ whole range, 0-255, dividing the angular value by 1.41. The enum
    colorspaces except for BT.2020 which uses limited range R'G'B'
    quantization.
 
-.. tabularcolumns:: |p{6.0cm}|p{11.5cm}|
+.. tabularcolumns:: |p{6.7cm}|p{10.8cm}|
 
 .. c:type:: v4l2_colorspace
 
@@ -112,7 +112,7 @@ whole range, 0-255, dividing the angular value by 1.41. The enum
 
 .. c:type:: v4l2_ycbcr_encoding
 
-.. tabularcolumns:: |p{6.5cm}|p{11.0cm}|
+.. tabularcolumns:: |p{7.2cm}|p{10.3cm}|
 
 .. flat-table:: V4L2 Y'CbCr Encodings
     :header-rows:  1
diff --git a/Documentation/media/uapi/v4l/colorspaces.rst b/Documentation/media/uapi/v4l/colorspaces.rst
index c5a560f0c13d..4f6c82fa057f 100644
--- a/Documentation/media/uapi/v4l/colorspaces.rst
+++ b/Documentation/media/uapi/v4l/colorspaces.rst
@@ -56,9 +56,9 @@ The Y value in the CIE XYZ colorspace corresponds to luminance. Often
 the CIE XYZ colorspace is transformed to the normalized CIE xyY
 colorspace:
 
-x = X / (X + Y + Z)
+	x = X / (X + Y + Z)
 
-y = Y / (X + Y + Z)
+	y = Y / (X + Y + Z)
 
 The x and y values are the chromaticity coordinates and can be used to
 define a color without the luminance component Y. It is very confusing
diff --git a/Documentation/media/uapi/v4l/dev-raw-vbi.rst b/Documentation/media/uapi/v4l/dev-raw-vbi.rst
index d6a707f0b24f..e06b03ca2ab2 100644
--- a/Documentation/media/uapi/v4l/dev-raw-vbi.rst
+++ b/Documentation/media/uapi/v4l/dev-raw-vbi.rst
@@ -106,7 +106,7 @@ VBI devices must implement both the :ref:`VIDIOC_G_FMT <VIDIOC_G_FMT>` and
 and always returns default parameters as :ref:`VIDIOC_G_FMT <VIDIOC_G_FMT>` does.
 :ref:`VIDIOC_TRY_FMT <VIDIOC_G_FMT>` is optional.
 
-.. tabularcolumns:: |p{2.4cm}|p{4.4cm}|p{10.7cm}|
+.. tabularcolumns:: |p{1.6cm}|p{4.2cm}|p{11.7cm}|
 
 .. c:type:: v4l2_vbi_format
 
@@ -190,7 +190,7 @@ and always returns default parameters as :ref:`VIDIOC_G_FMT <VIDIOC_G_FMT>` does
 	applications must set it to zero.
 
 
-.. tabularcolumns:: |p{4.0cm}|p{1.5cm}|p{12.0cm}|
+.. tabularcolumns:: |p{4.4cm}|p{1.5cm}|p{11.6cm}|
 
 .. _vbifmt-flags:
 
diff --git a/Documentation/media/uapi/v4l/dev-rds.rst b/Documentation/media/uapi/v4l/dev-rds.rst
index 624d6f95b842..64a724ef58f5 100644
--- a/Documentation/media/uapi/v4l/dev-rds.rst
+++ b/Documentation/media/uapi/v4l/dev-rds.rst
@@ -146,7 +146,7 @@ RDS datastructures
 
 .. _v4l2-rds-block-codes:
 
-.. tabularcolumns:: |p{5.6cm}|p{2.0cm}|p{1.5cm}|p{7.0cm}|
+.. tabularcolumns:: |p{6.4cm}|p{2.0cm}|p{1.2cm}|p{7.9cm}|
 
 .. flat-table:: Block defines
     :header-rows:  0
diff --git a/Documentation/media/uapi/v4l/dev-sliced-vbi.rst b/Documentation/media/uapi/v4l/dev-sliced-vbi.rst
index 0aa6cb8a272b..e86346f66017 100644
--- a/Documentation/media/uapi/v4l/dev-sliced-vbi.rst
+++ b/Documentation/media/uapi/v4l/dev-sliced-vbi.rst
@@ -118,7 +118,7 @@ struct v4l2_sliced_vbi_format
     \scriptsize
     \setlength{\tabcolsep}{2pt}
 
-.. tabularcolumns:: |p{.75cm}|p{3.3cm}|p{3.4cm}|p{3.4cm}|p{3.4cm}|
+.. tabularcolumns:: |p{.85cm}|p{3.3cm}|p{4.4cm}|p{4.4cm}|p{4.4cm}|
 
 .. cssclass:: longtable
 
@@ -223,7 +223,7 @@ Sliced VBI services
 
 .. raw:: latex
 
-    \footnotesize
+    \scriptsize
 
 .. tabularcolumns:: |p{4.1cm}|p{1.1cm}|p{2.4cm}|p{2.0cm}|p{7.3cm}|
 
@@ -541,7 +541,7 @@ Magic Constants for struct v4l2_mpeg_vbi_fmt_ivtv magic field
 structs v4l2_mpeg_vbi_itv0 and v4l2_mpeg_vbi_ITV0
 -------------------------------------------------
 
-.. tabularcolumns:: |p{4.9cm}|p{2.4cm}|p{10.2cm}|
+.. tabularcolumns:: |p{5.2cm}|p{2.4cm}|p{9.9cm}|
 
 .. flat-table::
     :header-rows:  0
@@ -561,12 +561,12 @@ structs v4l2_mpeg_vbi_itv0 and v4l2_mpeg_vbi_ITV0
 
 	::
 
-	    linemask[0] b0:     line  6     first field
-	    linemask[0] b17:    line 23     first field
-	    linemask[0] b18:    line  6     second field
-	    linemask[0] b31:    line 19     second field
-	    linemask[1] b0:     line 20     second field
-	    linemask[1] b3:     line 23     second field
+	    linemask[0] b0:     line  6  first field
+	    linemask[0] b17:    line 23  first field
+	    linemask[0] b18:    line  6  second field
+	    linemask[0] b31:    line 19  second field
+	    linemask[1] b0:     line 20  second field
+	    linemask[1] b3:     line 23  second field
 	    linemask[1] b4-b31: unused and set to 0
     * - struct
 	:c:type:`v4l2_mpeg_vbi_itv0_line`
@@ -590,7 +590,7 @@ structs v4l2_mpeg_vbi_itv0 and v4l2_mpeg_vbi_ITV0
 struct v4l2_mpeg_vbi_ITV0
 -------------------------
 
-.. tabularcolumns:: |p{4.9cm}|p{4.4cm}|p{8.2cm}|
+.. tabularcolumns:: |p{5.2cm}|p{2.4cm}|p{9.9cm}|
 
 .. flat-table::
     :header-rows:  0
@@ -635,7 +635,7 @@ struct v4l2_mpeg_vbi_itv0_line
 Line Identifiers for struct v4l2_mpeg_vbi_itv0_line id field
 ------------------------------------------------------------
 
-.. tabularcolumns:: |p{6.6cm}|p{2.2cm}|p{8.7cm}|
+.. tabularcolumns:: |p{7.0cm}|p{1.8cm}|p{8.7cm}|
 
 .. flat-table::
     :header-rows:  1
diff --git a/Documentation/media/uapi/v4l/dev-subdev.rst b/Documentation/media/uapi/v4l/dev-subdev.rst
index 2c2768c7343b..029bb2d9928a 100644
--- a/Documentation/media/uapi/v4l/dev-subdev.rst
+++ b/Documentation/media/uapi/v4l/dev-subdev.rst
@@ -211,7 +211,7 @@ list entity names and pad numbers).
 
 .. raw:: latex
 
-    \tiny
+    \scriptsize
 
 .. tabularcolumns:: |p{2.0cm}|p{2.3cm}|p{2.3cm}|p{2.3cm}|p{2.3cm}|p{2.3cm}|p{2.3cm}|
 
@@ -223,40 +223,80 @@ list entity names and pad numbers).
     :widths: 5 5 5 5 5 5 5
 
     * -
-      - Sensor/0 format
-      - Frontend/0 format
-      - Frontend/1 format
-      - Scaler/0 format
-      - Scaler/0 compose selection rectangle
-      - Scaler/1 format
+      - Sensor/0
+
+        format
+      - Frontend/0
+
+        format
+      - Frontend/1
+
+        format
+      - Scaler/0
+
+        format
+      - Scaler/0
+
+        compose selection rectangle
+      - Scaler/1
+
+        format
     * - Initial state
-      - 2048x1536/SGRBG8_1X8
+      - 2048x1536
+
+        SGRBG8_1X8
       - (default)
       - (default)
       - (default)
       - (default)
       - (default)
     * - Configure frontend sink format
-      - 2048x1536/SGRBG8_1X8
-      - *2048x1536/SGRBG8_1X8*
-      - *2046x1534/SGRBG8_1X8*
+      - 2048x1536
+
+        SGRBG8_1X8
+      - *2048x1536*
+
+        *SGRBG8_1X8*
+      - *2046x1534*
+
+        *SGRBG8_1X8*
       - (default)
       - (default)
       - (default)
     * - Configure scaler sink format
-      - 2048x1536/SGRBG8_1X8
-      - 2048x1536/SGRBG8_1X8
-      - 2046x1534/SGRBG8_1X8
-      - *2046x1534/SGRBG8_1X8*
+      - 2048x1536
+
+        SGRBG8_1X8
+      - 2048x1536
+
+        SGRBG8_1X8
+      - 2046x1534
+
+        SGRBG8_1X8
+      - *2046x1534*
+
+        *SGRBG8_1X8*
       - *0,0/2046x1534*
-      - *2046x1534/SGRBG8_1X8*
+      - *2046x1534*
+
+        *SGRBG8_1X8*
     * - Configure scaler sink compose selection
-      - 2048x1536/SGRBG8_1X8
-      - 2048x1536/SGRBG8_1X8
-      - 2046x1534/SGRBG8_1X8
-      - 2046x1534/SGRBG8_1X8
+      - 2048x1536
+
+        SGRBG8_1X8
+      - 2048x1536
+
+        SGRBG8_1X8
+      - 2046x1534
+
+        SGRBG8_1X8
+      - 2046x1534
+
+        SGRBG8_1X8
       - *0,0/1280x960*
-      - *1280x960/SGRBG8_1X8*
+      - *1280x960*
+
+        *SGRBG8_1X8*
 
 .. raw:: latex
 
diff --git a/Documentation/media/uapi/v4l/ext-ctrls-camera.rst b/Documentation/media/uapi/v4l/ext-ctrls-camera.rst
index d3a553cd86c9..51c1d5c9eb00 100644
--- a/Documentation/media/uapi/v4l/ext-ctrls-camera.rst
+++ b/Documentation/media/uapi/v4l/ext-ctrls-camera.rst
@@ -88,7 +88,7 @@ enum v4l2_exposure_metering -
     Determines how the camera measures the amount of light available for
     the frame exposure. Possible values are:
 
-.. tabularcolumns:: |p{8.5cm}|p{9.0cm}|
+.. tabularcolumns:: |p{8.7cm}|p{8.8cm}|
 
 .. flat-table::
     :header-rows:  0
@@ -180,7 +180,7 @@ enum v4l2_exposure_metering -
     control may stop updates of the ``V4L2_CID_AUTO_FOCUS_STATUS``
     control value.
 
-.. tabularcolumns:: |p{6.5cm}|p{11.0cm}|
+.. tabularcolumns:: |p{6.7cm}|p{10.8cm}|
 
 .. flat-table::
     :header-rows:  0
@@ -206,7 +206,7 @@ enum v4l2_exposure_metering -
 enum v4l2_auto_focus_range -
     Determines auto focus distance range for which lens may be adjusted.
 
-.. tabularcolumns:: |p{6.5cm}|p{11.0cm}|
+.. tabularcolumns:: |p{6.8cm}|p{10.7cm}|
 
 .. flat-table::
     :header-rows:  0
@@ -281,7 +281,7 @@ enum v4l2_auto_n_preset_white_balance -
     representation. The following white balance presets are listed in
     order of increasing color temperature.
 
-.. tabularcolumns:: |p{7.0 cm}|p{10.5cm}|
+.. tabularcolumns:: |p{7.2 cm}|p{10.3cm}|
 
 .. flat-table::
     :header-rows:  0
@@ -387,7 +387,11 @@ enum v4l2_scene_mode -
     to ``V4L2_SCENE_MODE_NONE`` to make sure the other possibly related
     controls are accessible. The following scene programs are defined:
 
-.. tabularcolumns:: |p{6.0cm}|p{11.5cm}|
+.. raw:: latex
+
+    \small
+
+.. tabularcolumns:: |p{5.9cm}|p{11.5cm}|
 
 .. flat-table::
     :header-rows:  0
@@ -459,6 +463,9 @@ enum v4l2_scene_mode -
 	may be switched to close-up mode and this setting may also involve
 	some lens-distortion correction.
 
+.. raw:: latex
+
+    \normalsize
 
 
 ``V4L2_CID_3A_LOCK (bitmask)``
diff --git a/Documentation/media/uapi/v4l/ext-ctrls-codec.rst b/Documentation/media/uapi/v4l/ext-ctrls-codec.rst
index c97fb7923be5..4a8446203085 100644
--- a/Documentation/media/uapi/v4l/ext-ctrls-codec.rst
+++ b/Documentation/media/uapi/v4l/ext-ctrls-codec.rst
@@ -105,7 +105,7 @@ enum v4l2_mpeg_stream_vbi_fmt -
 
 
 
-.. tabularcolumns:: |p{6 cm}|p{11.5cm}|
+.. tabularcolumns:: |p{6.6 cm}|p{10.9cm}|
 
 .. flat-table::
     :header-rows:  0
@@ -477,7 +477,7 @@ enum v4l2_mpeg_audio_dec_playback -
 
 
 
-.. tabularcolumns:: |p{9.0cm}|p{8.5cm}|
+.. tabularcolumns:: |p{9.8cm}|p{7.7cm}|
 
 .. flat-table::
     :header-rows:  0
@@ -888,7 +888,7 @@ enum v4l2_mpeg_video_multi_slice_mode -
 
 
 
-.. tabularcolumns:: |p{8.7cm}|p{8.8cm}|
+.. tabularcolumns:: |p{9.6cm}|p{7.9cm}|
 
 .. flat-table::
     :header-rows:  0
@@ -923,9 +923,11 @@ enum v4l2_mpeg_video_multi_slice_mode -
 enum v4l2_mpeg_video_h264_loop_filter_mode -
     Loop filter mode for H264 encoder. Possible values are:
 
+.. raw:: latex
 
+    \small
 
-.. tabularcolumns:: |p{14.0cm}|p{3.5cm}|
+.. tabularcolumns:: |p{13.6cm}|p{3.9cm}|
 
 .. flat-table::
     :header-rows:  0
@@ -938,6 +940,9 @@ enum v4l2_mpeg_video_h264_loop_filter_mode -
     * - ``V4L2_MPEG_VIDEO_H264_LOOP_FILTER_MODE_DISABLED_AT_SLICE_BOUNDARY``
       - Loop filter is disabled at the slice boundary.
 
+.. raw:: latex
+
+    \normalsize
 
 
 ``V4L2_CID_MPEG_VIDEO_H264_LOOP_FILTER_ALPHA (integer)``
@@ -964,6 +969,8 @@ enum v4l2_mpeg_video_h264_entropy_mode -
     encoder. Possible values are:
 
 
+.. tabularcolumns:: |p{9.0cm}|p{8.5cm}|
+
 
 .. flat-table::
     :header-rows:  0
@@ -1048,6 +1055,30 @@ enum v4l2_mpeg_video_h264_entropy_mode -
     Quantization parameter for an B frame for H264. Valid range: from 0
     to 51.
 
+``V4L2_CID_MPEG_VIDEO_H264_I_FRAME_MIN_QP (integer)``
+    Minimum quantization parameter for the H264 I frame to limit I frame
+    quality to a range. Valid range: from 0 to 51. If
+    V4L2_CID_MPEG_VIDEO_H264_MIN_QP is also set, the quantization parameter
+    should be chosen to meet both requirements.
+
+``V4L2_CID_MPEG_VIDEO_H264_I_FRAME_MAX_QP (integer)``
+    Maximum quantization parameter for the H264 I frame to limit I frame
+    quality to a range. Valid range: from 0 to 51. If
+    V4L2_CID_MPEG_VIDEO_H264_MAX_QP is also set, the quantization parameter
+    should be chosen to meet both requirements.
+
+``V4L2_CID_MPEG_VIDEO_H264_P_FRAME_MIN_QP (integer)``
+    Minimum quantization parameter for the H264 P frame to limit P frame
+    quality to a range. Valid range: from 0 to 51. If
+    V4L2_CID_MPEG_VIDEO_H264_MIN_QP is also set, the quantization parameter
+    should be chosen to meet both requirements.
+
+``V4L2_CID_MPEG_VIDEO_H264_P_FRAME_MAX_QP (integer)``
+    Maximum quantization parameter for the H264 P frame to limit P frame
+    quality to a range. Valid range: from 0 to 51. If
+    V4L2_CID_MPEG_VIDEO_H264_MAX_QP is also set, the quantization parameter
+    should be chosen to meet both requirements.
+
 ``V4L2_CID_MPEG_VIDEO_MPEG4_I_FRAME_QP (integer)``
     Quantization parameter for an I frame for MPEG4. Valid range: from 1
     to 31.
@@ -1129,7 +1160,9 @@ enum v4l2_mpeg_video_header_mode -
     it returned together with the first frame. Applicable to encoders.
     Possible values are:
 
+.. raw:: latex
 
+    \small
 
 .. tabularcolumns:: |p{10.3cm}|p{7.2cm}|
 
@@ -1143,6 +1176,9 @@ enum v4l2_mpeg_video_header_mode -
       - The stream header is returned together with the first encoded
 	frame.
 
+.. raw:: latex
+
+    \normalsize
 
 
 ``V4L2_CID_MPEG_VIDEO_REPEAT_SEQ_HEADER (boolean)``
@@ -1181,6 +1217,10 @@ enum v4l2_mpeg_video_h264_sei_fp_arrangement_type -
     Frame packing arrangement type for H264 SEI. Applicable to the H264
     encoder. Possible values are:
 
+.. raw:: latex
+
+    \small
+
 .. tabularcolumns:: |p{12cm}|p{5.5cm}|
 
 .. flat-table::
@@ -1200,6 +1240,10 @@ enum v4l2_mpeg_video_h264_sei_fp_arrangement_type -
     * - ``V4L2_MPEG_VIDEO_H264_SEI_FP_ARRANGEMENT_TYPE_TEMPORAL``
       - One view per frame.
 
+.. raw:: latex
+
+    \normalsize
+
 
 
 ``V4L2_CID_MPEG_VIDEO_H264_FMO (boolean)``
@@ -1217,6 +1261,10 @@ enum v4l2_mpeg_video_h264_fmo_map_type -
     patterns of macroblocks. Applicable to the H264 encoder. Possible
     values are:
 
+.. raw:: latex
+
+    \small
+
 .. tabularcolumns:: |p{12.5cm}|p{5.0cm}|
 
 .. flat-table::
@@ -1240,6 +1288,10 @@ enum v4l2_mpeg_video_h264_fmo_map_type -
     * - ``V4L2_MPEG_VIDEO_H264_FMO_MAP_TYPE_EXPLICIT``
       - User defined map type.
 
+.. raw:: latex
+
+    \normalsize
+
 
 
 ``V4L2_CID_MPEG_VIDEO_H264_FMO_SLICE_GROUP (integer)``
@@ -1361,6 +1413,8 @@ enum v4l2_mpeg_video_h264_hierarchical_coding_type -
 
 .. cssclass:: longtable
 
+.. tabularcolumns:: |p{5.8cm}|p{4.8cm}|p{6.6cm}|
+
 .. flat-table:: struct v4l2_ctrl_mpeg2_slice_params
     :header-rows:  0
     :stub-columns: 0
@@ -1402,6 +1456,8 @@ enum v4l2_mpeg_video_h264_hierarchical_coding_type -
 
 .. cssclass:: longtable
 
+.. tabularcolumns:: |p{1.5cm}|p{6.3cm}|p{9.4cm}|
+
 .. flat-table:: struct v4l2_mpeg2_sequence
     :header-rows:  0
     :stub-columns: 0
@@ -1433,6 +1489,8 @@ enum v4l2_mpeg_video_h264_hierarchical_coding_type -
 
 .. cssclass:: longtable
 
+.. tabularcolumns:: |p{1.5cm}|p{6.3cm}|p{9.4cm}|
+
 .. flat-table:: struct v4l2_mpeg2_picture
     :header-rows:  0
     :stub-columns: 0
@@ -1492,6 +1550,12 @@ enum v4l2_mpeg_video_h264_hierarchical_coding_type -
 
 .. cssclass:: longtable
 
+.. tabularcolumns:: |p{1.2cm}|p{8.0cm}|p{7.4cm}|
+
+.. raw:: latex
+
+    \small
+
 .. flat-table:: struct v4l2_ctrl_mpeg2_quantization
     :header-rows:  0
     :stub-columns: 0
@@ -1537,6 +1601,19 @@ enum v4l2_mpeg_video_h264_hierarchical_coding_type -
 	non-intra-coded frames, in zigzag scanning order. Only relevant for
 	non-4:2:0 YUV formats.
 
+``V4L2_CID_FWHT_I_FRAME_QP (integer)``
+    Quantization parameter for an I frame for FWHT. Valid range: from 1
+    to 31.
+
+``V4L2_CID_FWHT_P_FRAME_QP (integer)``
+    Quantization parameter for a P frame for FWHT. Valid range: from 1
+    to 31.
+
+.. raw:: latex
+
+    \normalsize
+
+
 MFC 5.1 MPEG Controls
 =====================
 
@@ -1644,7 +1721,11 @@ enum v4l2_mpeg_mfc51_video_frame_skip_mode -
     are:
 
 
-.. tabularcolumns:: |p{9.0cm}|p{8.5cm}|
+.. tabularcolumns:: |p{9.2cm}|p{8.3cm}|
+
+.. raw:: latex
+
+    \small
 
 .. flat-table::
     :header-rows:  0
@@ -1659,7 +1740,9 @@ enum v4l2_mpeg_mfc51_video_frame_skip_mode -
       - Frame skip mode enabled and buffer limit is set by the VBV
 	(MPEG1/2/4) or CPB (H264) buffer size control.
 
+.. raw:: latex
 
+    \normalsize
 
 ``V4L2_CID_MPEG_MFC51_VIDEO_RC_FIXED_TARGET_BIT (integer)``
     Enable rate-control with fixed target bit. If this setting is
@@ -1682,7 +1765,7 @@ enum v4l2_mpeg_mfc51_video_force_frame_type -
     Force a frame type for the next queued buffer. Applicable to
     encoders. Possible values are:
 
-
+.. tabularcolumns:: |p{9.5cm}|p{8.0cm}|
 
 .. flat-table::
     :header-rows:  0
@@ -1696,6 +1779,125 @@ enum v4l2_mpeg_mfc51_video_force_frame_type -
       - Force a non-coded frame.
 
 
+.. _v4l2-mpeg-fwht:
+
+``V4L2_CID_MPEG_VIDEO_FWHT_PARAMS (struct)``
+    Specifies the fwht parameters (as extracted from the bitstream) for the
+    associated FWHT data. This includes the necessary parameters for
+    configuring a stateless hardware decoding pipeline for FWHT.
+
+    .. note::
+
+       This compound control is not yet part of the public kernel API and
+       it is expected to change.
+
+.. c:type:: v4l2_ctrl_fwht_params
+
+.. cssclass:: longtable
+
+.. tabularcolumns:: |p{1.4cm}|p{4.3cm}|p{11.8cm}|
+
+.. flat-table:: struct v4l2_ctrl_fwht_params
+    :header-rows:  0
+    :stub-columns: 0
+    :widths:       1 1 2
+
+    * - __u64
+      - ``backward_ref_ts``
+      - Timestamp of the V4L2 capture buffer to use as backward reference, used
+        with P-coded frames. The timestamp refers to the
+	``timestamp`` field in struct :c:type:`v4l2_buffer`. Use the
+	:c:func:`v4l2_timeval_to_ns()` function to convert the struct
+	:c:type:`timeval` in struct :c:type:`v4l2_buffer` to a __u64.
+    * - __u32
+      - ``version``
+      - The version of the codec
+    * - __u32
+      - ``width``
+      - The width of the frame
+    * - __u32
+      - ``height``
+      - The height of the frame
+    * - __u32
+      - ``flags``
+      - The flags of the frame, see :ref:`fwht-flags`.
+    * - __u32
+      - ``colorspace``
+      - The colorspace of the frame, from enum :c:type:`v4l2_colorspace`.
+    * - __u32
+      - ``xfer_func``
+      - The transfer function, from enum :c:type:`v4l2_xfer_func`.
+    * - __u32
+      - ``ycbcr_enc``
+      - The Y'CbCr encoding, from enum :c:type:`v4l2_ycbcr_encoding`.
+    * - __u32
+      - ``quantization``
+      - The quantization range, from enum :c:type:`v4l2_quantization`.
+
+
+
+.. _fwht-flags:
+
+FWHT Flags
+============
+
+.. cssclass:: longtable
+
+.. tabularcolumns:: |p{6.8cm}|p{2.4cm}|p{8.3cm}|
+
+.. flat-table::
+    :header-rows:  0
+    :stub-columns: 0
+    :widths:       3 1 4
+
+    * - ``FWHT_FL_IS_INTERLACED``
+      - 0x00000001
+      - Set if this is an interlaced format
+    * - ``FWHT_FL_IS_BOTTOM_FIRST``
+      - 0x00000002
+      - Set if this is a bottom-first (NTSC) interlaced format
+    * - ``FWHT_FL_IS_ALTERNATE``
+      - 0x00000004
+      - Set if each 'frame' contains just one field
+    * - ``FWHT_FL_IS_BOTTOM_FIELD``
+      - 0x00000008
+      - If FWHT_FL_IS_ALTERNATE was set, then this is set if this 'frame' is the
+	bottom field, else it is the top field.
+    * - ``FWHT_FL_LUMA_IS_UNCOMPRESSED``
+      - 0x00000010
+      - Set if the luma plane is uncompressed
+    * - ``FWHT_FL_CB_IS_UNCOMPRESSED``
+      - 0x00000020
+      - Set if the cb plane is uncompressed
+    * - ``FWHT_FL_CR_IS_UNCOMPRESSED``
+      - 0x00000040
+      - Set if the cr plane is uncompressed
+    * - ``FWHT_FL_CHROMA_FULL_HEIGHT``
+      - 0x00000080
+      - Set if the chroma plane has the same height as the luma plane,
+	else the chroma plane is half the height of the luma plane
+    * - ``FWHT_FL_CHROMA_FULL_WIDTH``
+      - 0x00000100
+      - Set if the chroma plane has the same width as the luma plane,
+	else the chroma plane is half the width of the luma plane
+    * - ``FWHT_FL_ALPHA_IS_UNCOMPRESSED``
+      - 0x00000200
+      - Set if the alpha plane is uncompressed
+    * - ``FWHT_FL_I_FRAME``
+      - 0x00000400
+      - Set if this is an I-frame
+    * - ``FWHT_FL_COMPONENTS_NUM_MSK``
+      - 0x00070000
+      - A 4-values flag - the number of components - 1
+    * - ``FWHT_FL_PIXENC_YUV``
+      - 0x00080000
+      - Set if the pixel encoding is YUV
+    * - ``FWHT_FL_PIXENC_RGB``
+      - 0x00100000
+      - Set if the pixel encoding is RGB
+    * - ``FWHT_FL_PIXENC_HSV``
+      - 0x00180000
+      - Set if the pixel encoding is HSV
 
 
 CX2341x MPEG Controls
@@ -1745,9 +1947,11 @@ enum v4l2_mpeg_cx2341x_video_luma_spatial_filter_type -
     Select the algorithm to use for the Luma Spatial Filter (default
     ``1D_HOR``). Possible values:
 
+.. tabularcolumns:: |p{14.5cm}|p{3.0cm}|
 
+.. raw:: latex
 
-.. tabularcolumns:: |p{14.5cm}|p{3.0cm}|
+    \small
 
 .. flat-table::
     :header-rows:  0
@@ -1764,6 +1968,10 @@ enum v4l2_mpeg_cx2341x_video_luma_spatial_filter_type -
     * - ``V4L2_MPEG_CX2341X_VIDEO_LUMA_SPATIAL_FILTER_TYPE_2D_SYM_NON_SEPARABLE``
       - Two-dimensional symmetrical non-separable
 
+.. raw:: latex
+
+    \normalsize
+
 
 
 .. _chroma-spatial-filter-type:
@@ -1776,6 +1984,7 @@ enum v4l2_mpeg_cx2341x_video_chroma_spatial_filter_type -
     ``1D_HOR``). Possible values are:
 
 
+.. tabularcolumns:: |p{14.0cm}|p{3.5cm}|
 
 .. flat-table::
     :header-rows:  0
@@ -1918,6 +2127,10 @@ enum v4l2_vp8_num_ref_frames -
 
 .. tabularcolumns:: |p{7.9cm}|p{9.6cm}|
 
+.. raw:: latex
+
+    \small
+
 .. flat-table::
     :header-rows:  0
     :stub-columns: 0
@@ -1932,6 +2145,10 @@ enum v4l2_vp8_num_ref_frames -
       - The last encoded frame, the golden frame and the altref frame will
 	be searched.
 
+.. raw:: latex
+
+    \normalsize
+
 
 
 ``V4L2_CID_MPEG_VIDEO_VPX_FILTER_LEVEL (integer)``
@@ -1961,7 +2178,7 @@ enum v4l2_vp8_golden_frame_sel -
 
 .. raw:: latex
 
-    \footnotesize
+    \scriptsize
 
 .. tabularcolumns:: |p{9.0cm}|p{8.0cm}|
 
@@ -2283,7 +2500,7 @@ enum v4l2_mpeg_video_hevc_loop_filter_mode -
 
     \footnotesize
 
-.. tabularcolumns:: |p{10.7cm}|p{6.3cm}|
+.. tabularcolumns:: |p{12.1cm}|p{5.4cm}|
 
 .. flat-table::
     :header-rows:  0
diff --git a/Documentation/media/uapi/v4l/ext-ctrls-detect.rst b/Documentation/media/uapi/v4l/ext-ctrls-detect.rst
index 8a45ce642829..80981d0cff42 100644
--- a/Documentation/media/uapi/v4l/ext-ctrls-detect.rst
+++ b/Documentation/media/uapi/v4l/ext-ctrls-detect.rst
@@ -30,7 +30,7 @@ Detect Control IDs
 ``V4L2_CID_DETECT_MD_MODE (menu)``
     Sets the motion detection mode.
 
-.. tabularcolumns:: |p{7.5cm}|p{10.0cm}|
+.. tabularcolumns:: |p{7.7cm}|p{9.8cm}|
 
 .. flat-table::
     :header-rows:  0
diff --git a/Documentation/media/uapi/v4l/ext-ctrls-dv.rst b/Documentation/media/uapi/v4l/ext-ctrls-dv.rst
index 57edf211875c..5c70ac98f710 100644
--- a/Documentation/media/uapi/v4l/ext-ctrls-dv.rst
+++ b/Documentation/media/uapi/v4l/ext-ctrls-dv.rst
@@ -106,7 +106,7 @@ enum v4l2_dv_it_content_type -
     or an analog source. The enum v4l2_dv_it_content_type defines
     the possible content types:
 
-.. tabularcolumns:: |p{7.0cm}|p{10.5cm}|
+.. tabularcolumns:: |p{7.3cm}|p{10.4cm}|
 
 .. flat-table::
     :header-rows:  0
diff --git a/Documentation/media/uapi/v4l/ext-ctrls-flash.rst b/Documentation/media/uapi/v4l/ext-ctrls-flash.rst
index 5f30791c35b5..eff056b17167 100644
--- a/Documentation/media/uapi/v4l/ext-ctrls-flash.rst
+++ b/Documentation/media/uapi/v4l/ext-ctrls-flash.rst
@@ -87,7 +87,7 @@ Flash Control IDs
 ``V4L2_CID_FLASH_STROBE_SOURCE (menu)``
     Defines the source of the flash LED strobe.
 
-.. tabularcolumns:: |p{7.0cm}|p{10.5cm}|
+.. tabularcolumns:: |p{7.5cm}|p{10.0cm}|
 
 .. flat-table::
     :header-rows:  0
@@ -146,7 +146,7 @@ Flash Control IDs
     an effect is chip dependent. Reading the faults resets the control
     and returns the chip to a usable state if possible.
 
-.. tabularcolumns:: |p{8.0cm}|p{9.5cm}|
+.. tabularcolumns:: |p{8.4cm}|p{9.1cm}|
 
 .. flat-table::
     :header-rows:  0
diff --git a/Documentation/media/uapi/v4l/ext-ctrls-jpeg.rst b/Documentation/media/uapi/v4l/ext-ctrls-jpeg.rst
index cf9cd8a9f9b4..60ce3f949319 100644
--- a/Documentation/media/uapi/v4l/ext-ctrls-jpeg.rst
+++ b/Documentation/media/uapi/v4l/ext-ctrls-jpeg.rst
@@ -37,7 +37,7 @@ JPEG Control IDs
     how Cb and Cr components are downsampled after converting an input
     image from RGB to Y'CbCr color space.
 
-.. tabularcolumns:: |p{7.0cm}|p{10.5cm}|
+.. tabularcolumns:: |p{7.5cm}|p{10.0cm}|
 
 .. flat-table::
     :header-rows:  0
diff --git a/Documentation/media/uapi/v4l/field-order.rst b/Documentation/media/uapi/v4l/field-order.rst
index 8415268d439c..3fb473e3b8e2 100644
--- a/Documentation/media/uapi/v4l/field-order.rst
+++ b/Documentation/media/uapi/v4l/field-order.rst
@@ -64,7 +64,9 @@ enum v4l2_field
 
 .. c:type:: v4l2_field
 
-.. tabularcolumns:: |p{6.6cm}|p{2.2cm}|p{8.7cm}|
+.. tabularcolumns:: |p{5.8cm}|p{0.6cm}|p{11.1cm}|
+
+.. cssclass:: longtable
 
 .. flat-table::
     :header-rows:  0
diff --git a/Documentation/media/uapi/v4l/pixfmt-compressed.rst b/Documentation/media/uapi/v4l/pixfmt-compressed.rst
index 2675bef3eefe..6c961cfb74da 100644
--- a/Documentation/media/uapi/v4l/pixfmt-compressed.rst
+++ b/Documentation/media/uapi/v4l/pixfmt-compressed.rst
@@ -125,3 +125,9 @@ Compressed Formats
       - Video elementary stream using a codec based on the Fast Walsh Hadamard
         Transform. This codec is implemented by the vicodec ('Virtual Codec')
 	driver. See the codec-fwht.h header for more details.
+    * .. _V4L2-PIX-FMT-FWHT-STATELESS:
+
+      - ``V4L2_PIX_FMT_FWHT_STATELESS``
+      - 'SFWH'
+      - Same format as V4L2_PIX_FMT_FWHT but requires stateless codec implementation.
+	See the :ref:`associated Codec Control IDs <v4l2-mpeg-fwht>`.
diff --git a/Documentation/media/uapi/v4l/pixfmt-meta-d4xx.rst b/Documentation/media/uapi/v4l/pixfmt-meta-d4xx.rst
index 862e1f327150..87e8fd7d5d02 100644
--- a/Documentation/media/uapi/v4l/pixfmt-meta-d4xx.rst
+++ b/Documentation/media/uapi/v4l/pixfmt-meta-d4xx.rst
@@ -36,13 +36,16 @@ per frame, therefore their headers cannot be larger than 255 bytes.
 Below are proprietary Microsoft style metadata types, used by D4xx cameras,
 where all fields are in little endian order:
 
+.. tabularcolumns:: |p{5.0cm}|p{12.5cm}|
+
+
 .. flat-table:: D4xx metadata
-    :widths: 1 4
+    :widths: 1 2
     :header-rows:  1
     :stub-columns: 0
 
-    * - Field
-      - Description
+    * - **Field**
+      - **Description**
     * - :cspan:`1` *Depth Control*
     * - __u32 ID
       - 0x80000000
diff --git a/Documentation/media/uapi/v4l/pixfmt-meta-vsp1-hgt.rst b/Documentation/media/uapi/v4l/pixfmt-meta-vsp1-hgt.rst
index 2ebccdcca95d..d1a341af9c48 100644
--- a/Documentation/media/uapi/v4l/pixfmt-meta-vsp1-hgt.rst
+++ b/Documentation/media/uapi/v4l/pixfmt-meta-vsp1-hgt.rst
@@ -41,6 +41,10 @@ The Hue position **m** (0 - 5) of the bucket in the matrix depends on
 how the HGT Hue areas are configured. There are 6 user configurable Hue
 Areas which can be configured to cover overlapping Hue values:
 
+.. raw:: latex
+
+    \small
+
 ::
 
          Area 0       Area 1       Area 2       Area 3       Area 4       Area 5
@@ -53,6 +57,11 @@ Areas which can be configured to cover overlapping Hue values:
   5U   0L      0U   1L      1U   2L      2U   3L      3U   4L      4U   5L      5U   0L
         <0..............................Hue Value............................255>
 
+
+.. raw:: latex
+
+    \normalsize
+
 When two consecutive areas don't overlap (n+1L is equal to nU) the boundary
 value is considered as part of the lower area.
 
diff --git a/Documentation/media/uapi/v4l/pixfmt-packed-hsv.rst b/Documentation/media/uapi/v4l/pixfmt-packed-hsv.rst
index 38b1895a509f..dfc4a8367b3d 100644
--- a/Documentation/media/uapi/v4l/pixfmt-packed-hsv.rst
+++ b/Documentation/media/uapi/v4l/pixfmt-packed-hsv.rst
@@ -31,7 +31,7 @@ The values are packed in 24 or 32 bit formats.
     \tiny
     \setlength{\tabcolsep}{2pt}
 
-.. tabularcolumns:: |p{2.0cm}|p{0.54cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|
+.. tabularcolumns:: |p{2.6cm}|p{0.8cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|
 
 .. _packed-hsv-formats:
 
diff --git a/Documentation/media/uapi/v4l/pixfmt-packed-rgb.rst b/Documentation/media/uapi/v4l/pixfmt-packed-rgb.rst
index 6b3781c04dd5..738bb14c0ee2 100644
--- a/Documentation/media/uapi/v4l/pixfmt-packed-rgb.rst
+++ b/Documentation/media/uapi/v4l/pixfmt-packed-rgb.rst
@@ -27,7 +27,7 @@ next to each other in memory.
     \tiny
     \setlength{\tabcolsep}{2pt}
 
-.. tabularcolumns:: |p{2.3cm}|p{1.6cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|
+.. tabularcolumns:: |p{2.8cm}|p{2.0cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|
 
 
 .. _rgb-formats:
@@ -139,6 +139,144 @@ next to each other in memory.
       - r\ :sub:`1`
       - r\ :sub:`0`
       -
+    * .. _V4L2-PIX-FMT-RGBA444:
+
+      - ``V4L2_PIX_FMT_RGBA444``
+      - 'RA12'
+
+      - b\ :sub:`3`
+      - b\ :sub:`2`
+      - b\ :sub:`1`
+      - b\ :sub:`0`
+      - a\ :sub:`3`
+      - a\ :sub:`2`
+      - a\ :sub:`1`
+      - a\ :sub:`0`
+
+      - r\ :sub:`3`
+      - r\ :sub:`2`
+      - r\ :sub:`1`
+      - r\ :sub:`0`
+      - g\ :sub:`3`
+      - g\ :sub:`2`
+      - g\ :sub:`1`
+      - g\ :sub:`0`
+      -
+    * .. _V4L2-PIX-FMT-RGBX444:
+
+      - ``V4L2_PIX_FMT_RGBX444``
+      - 'RX12'
+
+      - b\ :sub:`3`
+      - b\ :sub:`2`
+      - b\ :sub:`1`
+      - b\ :sub:`0`
+      -
+      -
+      -
+      -
+
+      - r\ :sub:`3`
+      - r\ :sub:`2`
+      - r\ :sub:`1`
+      - r\ :sub:`0`
+      - g\ :sub:`3`
+      - g\ :sub:`2`
+      - g\ :sub:`1`
+      - g\ :sub:`0`
+      -
+    * .. _V4L2-PIX-FMT-ABGR444:
+
+      - ``V4L2_PIX_FMT_ABGR444``
+      - 'AB12'
+
+      - g\ :sub:`3`
+      - g\ :sub:`2`
+      - g\ :sub:`1`
+      - g\ :sub:`0`
+      - r\ :sub:`3`
+      - r\ :sub:`2`
+      - r\ :sub:`1`
+      - r\ :sub:`0`
+
+      - a\ :sub:`3`
+      - a\ :sub:`2`
+      - a\ :sub:`1`
+      - a\ :sub:`0`
+      - b\ :sub:`3`
+      - b\ :sub:`2`
+      - b\ :sub:`1`
+      - b\ :sub:`0`
+      -
+    * .. _V4L2-PIX-FMT-XBGR444:
+
+      - ``V4L2_PIX_FMT_XBGR444``
+      - 'XB12'
+
+      - g\ :sub:`3`
+      - g\ :sub:`2`
+      - g\ :sub:`1`
+      - g\ :sub:`0`
+      - r\ :sub:`3`
+      - r\ :sub:`2`
+      - r\ :sub:`1`
+      - r\ :sub:`0`
+
+      -
+      -
+      -
+      -
+      - b\ :sub:`3`
+      - b\ :sub:`2`
+      - b\ :sub:`1`
+      - b\ :sub:`0`
+      -
+    * .. _V4L2-PIX-FMT-BGRA444:
+
+      - ``V4L2_PIX_FMT_BGRA444``
+      - 'BA12'
+
+      - r\ :sub:`3`
+      - r\ :sub:`2`
+      - r\ :sub:`1`
+      - r\ :sub:`0`
+      - a\ :sub:`3`
+      - a\ :sub:`2`
+      - a\ :sub:`1`
+      - a\ :sub:`0`
+
+      - b\ :sub:`3`
+      - b\ :sub:`2`
+      - b\ :sub:`1`
+      - b\ :sub:`0`
+      - g\ :sub:`3`
+      - g\ :sub:`2`
+      - g\ :sub:`1`
+      - g\ :sub:`0`
+      -
+    * .. _V4L2-PIX-FMT-BGRX444:
+
+      - ``V4L2_PIX_FMT_BGRX444``
+      - 'BX12'
+
+      - r\ :sub:`3`
+      - r\ :sub:`2`
+      - r\ :sub:`1`
+      - r\ :sub:`0`
+      -
+      -
+      -
+      -
+
+      - b\ :sub:`3`
+      - b\ :sub:`2`
+      - b\ :sub:`1`
+      - b\ :sub:`0`
+      - g\ :sub:`3`
+      - g\ :sub:`2`
+      - g\ :sub:`1`
+      - g\ :sub:`0`
+      -
     * .. _V4L2-PIX-FMT-ARGB555:
 
       - ``V4L2_PIX_FMT_ARGB555``
@@ -185,6 +323,144 @@ next to each other in memory.
       - g\ :sub:`4`
       - g\ :sub:`3`
       -
+    * .. _V4L2-PIX-FMT-RGBA555:
+
+      - ``V4L2_PIX_FMT_RGBA555``
+      - 'RA15'
+
+      - g\ :sub:`1`
+      - g\ :sub:`0`
+      - b\ :sub:`4`
+      - b\ :sub:`3`
+      - b\ :sub:`2`
+      - b\ :sub:`1`
+      - b\ :sub:`0`
+      - a
+
+      - r\ :sub:`4`
+      - r\ :sub:`3`
+      - r\ :sub:`2`
+      - r\ :sub:`1`
+      - r\ :sub:`0`
+      - g\ :sub:`4`
+      - g\ :sub:`3`
+      - g\ :sub:`2`
+      -
+    * .. _V4L2-PIX-FMT-RGBX555:
+
+      - ``V4L2_PIX_FMT_RGBX555``
+      - 'RX15'
+
+      - g\ :sub:`1`
+      - g\ :sub:`0`
+      - b\ :sub:`4`
+      - b\ :sub:`3`
+      - b\ :sub:`2`
+      - b\ :sub:`1`
+      - b\ :sub:`0`
+      -
+
+      - r\ :sub:`4`
+      - r\ :sub:`3`
+      - r\ :sub:`2`
+      - r\ :sub:`1`
+      - r\ :sub:`0`
+      - g\ :sub:`4`
+      - g\ :sub:`3`
+      - g\ :sub:`2`
+      -
+    * .. _V4L2-PIX-FMT-ABGR555:
+
+      - ``V4L2_PIX_FMT_ABGR555``
+      - 'AB15'
+
+      - g\ :sub:`2`
+      - g\ :sub:`1`
+      - g\ :sub:`0`
+      - r\ :sub:`4`
+      - r\ :sub:`3`
+      - r\ :sub:`2`
+      - r\ :sub:`1`
+      - r\ :sub:`0`
+
+      - a
+      - b\ :sub:`4`
+      - b\ :sub:`3`
+      - b\ :sub:`2`
+      - b\ :sub:`1`
+      - b\ :sub:`0`
+      - g\ :sub:`4`
+      - g\ :sub:`3`
+      -
+    * .. _V4L2-PIX-FMT-XBGR555:
+
+      - ``V4L2_PIX_FMT_XBGR555``
+      - 'XB15'
+
+      - g\ :sub:`2`
+      - g\ :sub:`1`
+      - g\ :sub:`0`
+      - r\ :sub:`4`
+      - r\ :sub:`3`
+      - r\ :sub:`2`
+      - r\ :sub:`1`
+      - r\ :sub:`0`
+
+      -
+      - b\ :sub:`4`
+      - b\ :sub:`3`
+      - b\ :sub:`2`
+      - b\ :sub:`1`
+      - b\ :sub:`0`
+      - g\ :sub:`4`
+      - g\ :sub:`3`
+      -
+    * .. _V4L2-PIX-FMT-BGRA555:
+
+      - ``V4L2_PIX_FMT_BGRA555``
+      - 'BA15'
+
+      - g\ :sub:`1`
+      - g\ :sub:`0`
+      - r\ :sub:`4`
+      - r\ :sub:`3`
+      - r\ :sub:`2`
+      - r\ :sub:`1`
+      - r\ :sub:`0`
+      - a
+
+      - b\ :sub:`4`
+      - b\ :sub:`3`
+      - b\ :sub:`2`
+      - b\ :sub:`1`
+      - b\ :sub:`0`
+      - g\ :sub:`4`
+      - g\ :sub:`3`
+      - g\ :sub:`2`
+      -
+    * .. _V4L2-PIX-FMT-BGRX555:
+
+      - ``V4L2_PIX_FMT_BGRX555``
+      - 'BX15'
+
+      - g\ :sub:`1`
+      - g\ :sub:`0`
+      - r\ :sub:`4`
+      - r\ :sub:`3`
+      - r\ :sub:`2`
+      - r\ :sub:`1`
+      - r\ :sub:`0`
+      -
+
+      - b\ :sub:`4`
+      - b\ :sub:`3`
+      - b\ :sub:`2`
+      - b\ :sub:`1`
+      - b\ :sub:`0`
+      - g\ :sub:`4`
+      - g\ :sub:`3`
+      - g\ :sub:`2`
+      -
     * .. _V4L2-PIX-FMT-RGB565:
 
       - ``V4L2_PIX_FMT_RGB565``
@@ -461,6 +737,166 @@ next to each other in memory.
       -
       -
       -
+    * .. _V4L2-PIX-FMT-BGRA32:
+
+      - ``V4L2_PIX_FMT_BGRA32``
+      - 'RA24'
+
+      - a\ :sub:`7`
+      - a\ :sub:`6`
+      - a\ :sub:`5`
+      - a\ :sub:`4`
+      - a\ :sub:`3`
+      - a\ :sub:`2`
+      - a\ :sub:`1`
+      - a\ :sub:`0`
+
+      - b\ :sub:`7`
+      - b\ :sub:`6`
+      - b\ :sub:`5`
+      - b\ :sub:`4`
+      - b\ :sub:`3`
+      - b\ :sub:`2`
+      - b\ :sub:`1`
+      - b\ :sub:`0`
+
+      - g\ :sub:`7`
+      - g\ :sub:`6`
+      - g\ :sub:`5`
+      - g\ :sub:`4`
+      - g\ :sub:`3`
+      - g\ :sub:`2`
+      - g\ :sub:`1`
+      - g\ :sub:`0`
+
+      - r\ :sub:`7`
+      - r\ :sub:`6`
+      - r\ :sub:`5`
+      - r\ :sub:`4`
+      - r\ :sub:`3`
+      - r\ :sub:`2`
+      - r\ :sub:`1`
+      - r\ :sub:`0`
+    * .. _V4L2-PIX-FMT-BGRX32:
+
+      - ``V4L2_PIX_FMT_BGRX32``
+      - 'RX24'
+
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+
+      - b\ :sub:`7`
+      - b\ :sub:`6`
+      - b\ :sub:`5`
+      - b\ :sub:`4`
+      - b\ :sub:`3`
+      - b\ :sub:`2`
+      - b\ :sub:`1`
+      - b\ :sub:`0`
+
+      - g\ :sub:`7`
+      - g\ :sub:`6`
+      - g\ :sub:`5`
+      - g\ :sub:`4`
+      - g\ :sub:`3`
+      - g\ :sub:`2`
+      - g\ :sub:`1`
+      - g\ :sub:`0`
+
+      - r\ :sub:`7`
+      - r\ :sub:`6`
+      - r\ :sub:`5`
+      - r\ :sub:`4`
+      - r\ :sub:`3`
+      - r\ :sub:`2`
+      - r\ :sub:`1`
+      - r\ :sub:`0`
+    * .. _V4L2-PIX-FMT-RGBA32:
+
+      - ``V4L2_PIX_FMT_RGBA32``
+      - 'AB24'
+
+      - r\ :sub:`7`
+      - r\ :sub:`6`
+      - r\ :sub:`5`
+      - r\ :sub:`4`
+      - r\ :sub:`3`
+      - r\ :sub:`2`
+      - r\ :sub:`1`
+      - r\ :sub:`0`
+
+      - g\ :sub:`7`
+      - g\ :sub:`6`
+      - g\ :sub:`5`
+      - g\ :sub:`4`
+      - g\ :sub:`3`
+      - g\ :sub:`2`
+      - g\ :sub:`1`
+      - g\ :sub:`0`
+
+      - b\ :sub:`7`
+      - b\ :sub:`6`
+      - b\ :sub:`5`
+      - b\ :sub:`4`
+      - b\ :sub:`3`
+      - b\ :sub:`2`
+      - b\ :sub:`1`
+      - b\ :sub:`0`
+
+      - a\ :sub:`7`
+      - a\ :sub:`6`
+      - a\ :sub:`5`
+      - a\ :sub:`4`
+      - a\ :sub:`3`
+      - a\ :sub:`2`
+      - a\ :sub:`1`
+      - a\ :sub:`0`
+    * .. _V4L2-PIX-FMT-RGBX32:
+
+      - ``V4L2_PIX_FMT_RGBX32``
+      - 'XB24'
+
+      - r\ :sub:`7`
+      - r\ :sub:`6`
+      - r\ :sub:`5`
+      - r\ :sub:`4`
+      - r\ :sub:`3`
+      - r\ :sub:`2`
+      - r\ :sub:`1`
+      - r\ :sub:`0`
+
+      - g\ :sub:`7`
+      - g\ :sub:`6`
+      - g\ :sub:`5`
+      - g\ :sub:`4`
+      - g\ :sub:`3`
+      - g\ :sub:`2`
+      - g\ :sub:`1`
+      - g\ :sub:`0`
+
+      - b\ :sub:`7`
+      - b\ :sub:`6`
+      - b\ :sub:`5`
+      - b\ :sub:`4`
+      - b\ :sub:`3`
+      - b\ :sub:`2`
+      - b\ :sub:`1`
+      - b\ :sub:`0`
+
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
     * .. _V4L2-PIX-FMT-ARGB32:
 
       - ``V4L2_PIX_FMT_ARGB32``
@@ -656,7 +1092,7 @@ either the corresponding ARGB or XRGB format, depending on the driver.
     \tiny
     \setlength{\tabcolsep}{2pt}
 
-.. tabularcolumns:: |p{2.2cm}|p{0.60cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|
+.. tabularcolumns:: |p{2.6cm}|p{0.70cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|
 
 .. _rgb-formats-deprecated:
 
diff --git a/Documentation/media/uapi/v4l/pixfmt-packed-yuv.rst b/Documentation/media/uapi/v4l/pixfmt-packed-yuv.rst
index 7fcee1c11ac4..41b60fae703a 100644
--- a/Documentation/media/uapi/v4l/pixfmt-packed-yuv.rst
+++ b/Documentation/media/uapi/v4l/pixfmt-packed-yuv.rst
@@ -28,7 +28,7 @@ component of each pixel in one 16 or 32 bit word.
 
 .. _packed-yuv-formats:
 
-.. tabularcolumns:: |p{2.0cm}|p{0.67cm}|p{0.29cm}|p{0.29cm}|p{0.29cm}|p{0.29cm}|p{0.29cm}|p{0.29cm}|p{0.29cm}|p{0.29cm}|p{0.29cm}|p{0.29cm}|p{0.29cm}|p{0.29cm}|p{0.29cm}|p{0.29cm}|p{0.29cm}|p{0.29cm}|p{0.29cm}|p{0.29cm}|p{0.29cm}|p{0.29cm}|p{0.29cm}|p{0.29cm}|p{0.29cm}|p{0.29cm}|p{0.29cm}|p{0.29cm}|p{0.29cm}|p{0.29cm}|p{0.29cm}|p{0.29cm}|p{0.29cm}|p{0.29cm}|
+.. tabularcolumns:: |p{2.5cm}|p{0.69cm}|p{0.31cm}|p{0.31cm}|p{0.31cm}|p{0.31cm}|p{0.31cm}|p{0.31cm}|p{0.31cm}|p{0.31cm}|p{0.31cm}|p{0.31cm}|p{0.31cm}|p{0.31cm}|p{0.31cm}|p{0.31cm}|p{0.31cm}|p{0.31cm}|p{0.31cm}|p{0.31cm}|p{0.31cm}|p{0.31cm}|p{0.31cm}|p{0.31cm}|p{0.31cm}|p{0.31cm}|p{0.31cm}|p{0.31cm}|p{0.31cm}|p{0.31cm}|p{0.31cm}|p{0.31cm}|p{0.31cm}|p{0.31cm}|
 
 .. flat-table:: Packed YUV Image Formats
     :header-rows:  2
@@ -44,6 +44,7 @@ component of each pixel in one 16 or 32 bit word.
       - :cspan:`7` Byte 2
 
       - :cspan:`7` Byte 3
+
     * -
       -
       - 7
@@ -81,6 +82,7 @@ component of each pixel in one 16 or 32 bit word.
       - 2
       - 1
       - 0
+
     * .. _V4L2-PIX-FMT-YUV444:
 
       - ``V4L2_PIX_FMT_YUV444``
@@ -103,7 +105,9 @@ component of each pixel in one 16 or 32 bit word.
       - Y'\ :sub:`2`
       - Y'\ :sub:`1`
       - Y'\ :sub:`0`
-      -
+
+      -  :cspan:`15`
+
     * .. _V4L2-PIX-FMT-YUV555:
 
       - ``V4L2_PIX_FMT_YUV555``
@@ -126,7 +130,8 @@ component of each pixel in one 16 or 32 bit word.
       - Y'\ :sub:`0`
       - Cb\ :sub:`4`
       - Cb\ :sub:`3`
-      -
+
+      -  :cspan:`15`
     * .. _V4L2-PIX-FMT-YUV565:
 
       - ``V4L2_PIX_FMT_YUV565``
@@ -149,7 +154,9 @@ component of each pixel in one 16 or 32 bit word.
       - Cb\ :sub:`5`
       - Cb\ :sub:`4`
       - Cb\ :sub:`3`
-      -
+
+      -  :cspan:`15`
+
     * .. _V4L2-PIX-FMT-YUV32:
 
       - ``V4L2_PIX_FMT_YUV32``
@@ -190,7 +197,7 @@ component of each pixel in one 16 or 32 bit word.
       - Cr\ :sub:`2`
       - Cr\ :sub:`1`
       - Cr\ :sub:`0`
-      -
+
     * .. _V4L2-PIX-FMT-AYUV32:
 
       - ``V4L2_PIX_FMT_AYUV32``
@@ -231,7 +238,7 @@ component of each pixel in one 16 or 32 bit word.
       - Cr\ :sub:`2`
       - Cr\ :sub:`1`
       - Cr\ :sub:`0`
-      -
+
     * .. _V4L2-PIX-FMT-XYUV32:
 
       - ``V4L2_PIX_FMT_XYUV32``
@@ -272,7 +279,7 @@ component of each pixel in one 16 or 32 bit word.
       - Cr\ :sub:`2`
       - Cr\ :sub:`1`
       - Cr\ :sub:`0`
-      -
+
     * .. _V4L2-PIX-FMT-VUYA32:
 
       - ``V4L2_PIX_FMT_VUYA32``
@@ -313,7 +320,7 @@ component of each pixel in one 16 or 32 bit word.
       - a\ :sub:`2`
       - a\ :sub:`1`
       - a\ :sub:`0`
-      -
+
     * .. _V4L2-PIX-FMT-VUYX32:
 
       - ``V4L2_PIX_FMT_VUYX32``
diff --git a/Documentation/media/uapi/v4l/pixfmt-srggb10p.rst b/Documentation/media/uapi/v4l/pixfmt-srggb10p.rst
index cdb70ac26126..fd32660a3766 100644
--- a/Documentation/media/uapi/v4l/pixfmt-srggb10p.rst
+++ b/Documentation/media/uapi/v4l/pixfmt-srggb10p.rst
@@ -40,7 +40,7 @@ of a small V4L2_PIX_FMT_SBGGR10P image:
 **Byte Order.**
 Each cell is one byte.
 
-.. tabularcolumns:: |p{2.0cm}|p{1.0cm}|p{1.0cm}|p{1.0cm}|p{1.0cm}|p{5.4cm}|
+.. tabularcolumns:: |p{2.4cm}|p{1.4cm}|p{1.2cm}|p{1.2cm}|p{1.2cm}|p{6.4cm}|
 
 .. flat-table::
     :header-rows:  0
diff --git a/Documentation/media/uapi/v4l/pixfmt-srggb12p.rst b/Documentation/media/uapi/v4l/pixfmt-srggb12p.rst
index 01413be12916..960851275f23 100644
--- a/Documentation/media/uapi/v4l/pixfmt-srggb12p.rst
+++ b/Documentation/media/uapi/v4l/pixfmt-srggb12p.rst
@@ -18,6 +18,7 @@ V4L2_PIX_FMT_SRGGB12P ('pRAA'), V4L2_PIX_FMT_SGRBG12P ('pgAA'), V4L2_PIX_FMT_SGB
 
 
 12-bit packed Bayer formats
+---------------------------
 
 
 Description
@@ -37,7 +38,7 @@ Below is an example of a small V4L2_PIX_FMT_SBGGR12P image:
 **Byte Order.**
 Each cell is one byte.
 
-.. tabularcolumns:: |p{2.0cm}|p{1.0cm}|p{1.0cm}|p{2.7cm}|p{1.0cm}|p{1.0cm}|p{2.7cm}|
+.. tabularcolumns:: |p{2.2cm}|p{1.2cm}|p{1.2cm}|p{3.1cm}|p{1.2cm}|p{1.2cm}|p{3.1cm}|
 
 
 .. flat-table::
diff --git a/Documentation/media/uapi/v4l/pixfmt-srggb14p.rst b/Documentation/media/uapi/v4l/pixfmt-srggb14p.rst
index b583531c2853..1a988d7e7ff8 100644
--- a/Documentation/media/uapi/v4l/pixfmt-srggb14p.rst
+++ b/Documentation/media/uapi/v4l/pixfmt-srggb14p.rst
@@ -41,17 +41,21 @@ of one of these formats:
 **Byte Order.**
 Each cell is one byte.
 
+.. raw:: latex
 
+    \footnotesize
+
+.. tabularcolumns:: |p{1.8cm}|p{1.0cm}|p{1.0cm}|p{1.0cm}|p{1.1cm}|p{3.3cm}|p{3.3cm}|p{3.3cm}|
 
 .. flat-table::
     :header-rows:  0
     :stub-columns: 0
-    :widths:       2 1 1 1 1 1 1 1
+    :widths:       2 1 1 1 1 3 3 3
 
 
     -  .. row 1
 
-       -  start + 0:
+       -  start + 0
 
        -  B\ :sub:`00high`
 
@@ -62,17 +66,20 @@ Each cell is one byte.
        -  G\ :sub:`03high`
 
        -  G\ :sub:`01low bits 1--0`\ (bits 7--6)
+
 	  B\ :sub:`00low bits 5--0`\ (bits 5--0)
 
        -  R\ :sub:`02low bits 3--0`\ (bits 7--4)
+
 	  G\ :sub:`01low bits 5--2`\ (bits 3--0)
 
        -  G\ :sub:`03low bits 5--0`\ (bits 7--2)
+
 	  R\ :sub:`02low bits 5--4`\ (bits 1--0)
 
     -  .. row 2
 
-       -  start + 7:
+       -  start + 7
 
        -  G\ :sub:`00high`
 
@@ -83,12 +90,15 @@ Each cell is one byte.
        -  R\ :sub:`03high`
 
        -  R\ :sub:`01low bits 1--0`\ (bits 7--6)
+
 	  G\ :sub:`00low bits 5--0`\ (bits 5--0)
 
        -  G\ :sub:`02low bits 3--0`\ (bits 7--4)
+
 	  R\ :sub:`01low bits 5--2`\ (bits 3--0)
 
        -  R\ :sub:`03low bits 5--0`\ (bits 7--2)
+
 	  G\ :sub:`02low bits 5--4`\ (bits 1--0)
 
     -  .. row 3
@@ -104,12 +114,15 @@ Each cell is one byte.
        -  G\ :sub:`23high`
 
        -  G\ :sub:`21low bits 1--0`\ (bits 7--6)
+
 	  B\ :sub:`20low bits 5--0`\ (bits 5--0)
 
        -  R\ :sub:`22low bits 3--0`\ (bits 7--4)
+
 	  G\ :sub:`21low bits 5--2`\ (bits 3--0)
 
        -  G\ :sub:`23low bits 5--0`\ (bits 7--2)
+
 	  R\ :sub:`22low bits 5--4`\ (bits 1--0)
 
     -  .. row 4
@@ -132,3 +145,8 @@ Each cell is one byte.
 
        -  R\ :sub:`33low bits 5--0`\ (bits 7--2)
 	  G\ :sub:`32low bits 5--4`\ (bits 1--0)
+
+.. raw:: latex
+
+    \normalsize
+
diff --git a/Documentation/media/uapi/v4l/pixfmt-v4l2-mplane.rst b/Documentation/media/uapi/v4l/pixfmt-v4l2-mplane.rst
index 7f82dad9013a..5688c816e334 100644
--- a/Documentation/media/uapi/v4l/pixfmt-v4l2-mplane.rst
+++ b/Documentation/media/uapi/v4l/pixfmt-v4l2-mplane.rst
@@ -19,6 +19,7 @@ array of struct :c:type:`v4l2_plane_pix_format` structures,
 describing all planes of that format.
 
 
+
 .. tabularcolumns:: |p{4.4cm}|p{4.4cm}|p{8.7cm}|
 
 .. c:type:: v4l2_plane_pix_format
@@ -41,6 +42,10 @@ describing all planes of that format.
 	applications.
 
 
+.. raw:: latex
+
+    \small
+
 .. tabularcolumns:: |p{4.4cm}|p{5.6cm}|p{7.5cm}|
 
 .. c:type:: v4l2_pix_format_mplane
@@ -82,9 +87,7 @@ describing all planes of that format.
     * - __u8
       - ``flags``
       - Flags set by the application or driver, see :ref:`format-flags`.
-    * - union {
-      - (anonymous)
-      -
+    * - :cspan:`2` union { (anonymous)
     * - __u8
       - ``ycbcr_enc``
       - Y'CbCr encoding, from enum :c:type:`v4l2_ycbcr_encoding`.
@@ -97,9 +100,7 @@ describing all planes of that format.
         This information supplements the ``colorspace`` and must be set by
 	the driver for capture streams and by the application for output
 	streams, see :ref:`colorspaces`.
-    * - }
-      -
-      -
+    * - :cspan:`2` }
     * - __u8
       - ``quantization``
       - Quantization range, from enum :c:type:`v4l2_quantization`.
@@ -116,3 +117,7 @@ describing all planes of that format.
       - ``reserved[7]``
       - Reserved for future extensions. Should be zeroed by drivers and
 	applications.
+
+.. raw:: latex
+
+    \normalsize
diff --git a/Documentation/media/uapi/v4l/pixfmt-y10p.rst b/Documentation/media/uapi/v4l/pixfmt-y10p.rst
index 7893642faee3..39cd789dcb59 100644
--- a/Documentation/media/uapi/v4l/pixfmt-y10p.rst
+++ b/Documentation/media/uapi/v4l/pixfmt-y10p.rst
@@ -27,6 +27,12 @@ in the same order.
 
 **Bit-packed representation.**
 
+.. raw:: latex
+
+    \small
+
+.. tabularcolumns:: |p{1.2cm}||p{1.2cm}||p{1.2cm}||p{1.2cm}|p{3.2cm}|p{3.2cm}|
+
 .. flat-table::
     :header-rows:  0
     :stub-columns: 0
@@ -38,3 +44,7 @@ in the same order.
       - Y'\ :sub:`03[9:2]`
       - Y'\ :sub:`03[1:0]`\ (bits 7--6) Y'\ :sub:`02[1:0]`\ (bits 5--4)
 	Y'\ :sub:`01[1:0]`\ (bits 3--2) Y'\ :sub:`00[1:0]`\ (bits 1--0)
+
+.. raw:: latex
+
+    \normalsize
diff --git a/Documentation/media/uapi/v4l/subdev-formats.rst b/Documentation/media/uapi/v4l/subdev-formats.rst
index f5440d55d510..ab1a48a5ae80 100644
--- a/Documentation/media/uapi/v4l/subdev-formats.rst
+++ b/Documentation/media/uapi/v4l/subdev-formats.rst
@@ -980,6 +980,113 @@ The following tables list existing packed RGB formats.
       - r\ :sub:`2`
       - r\ :sub:`1`
       - r\ :sub:`0`
+    * .. _MEDIA-BUS-FMT-BGR888-3X8:
+
+      - MEDIA_BUS_FMT_BGR888_3X8
+      - 0x101b
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      - b\ :sub:`7`
+      - b\ :sub:`6`
+      - b\ :sub:`5`
+      - b\ :sub:`4`
+      - b\ :sub:`3`
+      - b\ :sub:`2`
+      - b\ :sub:`1`
+      - b\ :sub:`0`
+    * -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      - g\ :sub:`7`
+      - g\ :sub:`6`
+      - g\ :sub:`5`
+      - g\ :sub:`4`
+      - g\ :sub:`3`
+      - g\ :sub:`2`
+      - g\ :sub:`1`
+      - g\ :sub:`0`
+    * -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      -
+      - r\ :sub:`7`
+      - r\ :sub:`6`
+      - r\ :sub:`5`
+      - r\ :sub:`4`
+      - r\ :sub:`3`
+      - r\ :sub:`2`
+      - r\ :sub:`1`
+      - r\ :sub:`0`
     * .. _MEDIA-BUS-FMT-GBR888-1X24:
 
       - MEDIA_BUS_FMT_GBR888_1X24
@@ -7414,7 +7521,7 @@ The following table lists existing HSV/HSL formats.
     \tiny
     \setlength{\tabcolsep}{2pt}
 
-.. tabularcolumns:: |p{3.0cm}|p{0.60cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|
+.. tabularcolumns:: |p{3.9cm}|p{0.73cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|p{0.22cm}|
 
 .. _v4l2-mbus-pixelcode-hsv:
 
@@ -7524,7 +7631,7 @@ The following table lists existing JPEG compressed formats.
 
 .. _v4l2-mbus-pixelcode-jpeg:
 
-.. tabularcolumns:: |p{5.4cm}|p{1.4cm}|p{10.7cm}|
+.. tabularcolumns:: |p{6.0cm}|p{1.4cm}|p{10.1cm}|
 
 .. flat-table:: JPEG Formats
     :header-rows:  1
@@ -7557,7 +7664,7 @@ formats.
 
 .. _v4l2-mbus-pixelcode-vendor-specific:
 
-.. tabularcolumns:: |p{6.8cm}|p{1.4cm}|p{9.3cm}|
+.. tabularcolumns:: |p{8.0cm}|p{1.4cm}|p{7.7cm}|
 
 .. flat-table:: Vendor and device specific formats
     :header-rows:  1
diff --git a/Documentation/media/uapi/v4l/vidioc-qbuf.rst b/Documentation/media/uapi/v4l/vidioc-qbuf.rst
index c138d149faea..dbf7b445a27b 100644
--- a/Documentation/media/uapi/v4l/vidioc-qbuf.rst
+++ b/Documentation/media/uapi/v4l/vidioc-qbuf.rst
@@ -111,7 +111,7 @@ in use. Setting it means that the buffer will not be passed to the driver
 until the request itself is queued. Also, the driver will apply any
 settings associated with the request for this buffer. This field will
 be ignored unless the ``V4L2_BUF_FLAG_REQUEST_FD`` flag is set.
-If the device does not support requests, then ``EACCES`` will be returned.
+If the device does not support requests, then ``EBADR`` will be returned.
 If requests are supported but an invalid request file descriptor is given,
 then ``EINVAL`` will be returned.
 
@@ -125,7 +125,7 @@ then ``EINVAL`` will be returned.
 
    For :ref:`memory-to-memory devices <mem2mem>` you can specify the
    ``request_fd`` only for output buffers, not for capture buffers. Attempting
-   to specify this for a capture buffer will result in an ``EACCES`` error.
+   to specify this for a capture buffer will result in an ``EBADR`` error.
 
 Applications call the ``VIDIOC_DQBUF`` ioctl to dequeue a filled
 (capturing) or displayed (output) buffer from the driver's outgoing
@@ -185,9 +185,11 @@ EPIPE
     codecs if a buffer with the ``V4L2_BUF_FLAG_LAST`` was already
     dequeued and no new buffers are expected to become available.
 
-EACCES
+EBADR
     The ``V4L2_BUF_FLAG_REQUEST_FD`` flag was set but the device does not
-    support requests for the given buffer type.
+    support requests for the given buffer type, or
+    the ``V4L2_BUF_FLAG_REQUEST_FD`` flag was not set but the device requires
+    that the buffer is part of a request.
 
 EBUSY
     The first buffer was queued via a request, but the application now tries
diff --git a/Documentation/media/v4l-drivers/index.rst b/Documentation/media/v4l-drivers/index.rst
index dfd4b205937c..33a055907258 100644
--- a/Documentation/media/v4l-drivers/index.rst
+++ b/Documentation/media/v4l-drivers/index.rst
@@ -65,5 +65,4 @@ For more details see the file COPYING in the source distribution of Linux.
 	soc-camera
 	uvcvideo
 	vivid
-	zoran
 	zr364xx
diff --git a/Documentation/media/v4l-drivers/zoran.rst b/Documentation/media/v4l-drivers/zoran.rst
deleted file mode 100644
index d2724a863d1d..000000000000
--- a/Documentation/media/v4l-drivers/zoran.rst
+++ /dev/null
@@ -1,583 +0,0 @@
-.. SPDX-License-Identifier: GPL-2.0
-
-The Zoran driver
-================
-
-unified zoran driver (zr360x7, zoran, buz, dc10(+), dc30(+), lml33)
-
-website: http://mjpeg.sourceforge.net/driver-zoran/
-
-
-Frequently Asked Questions
---------------------------
-
-What cards are supported
-------------------------
-
-Iomega Buz, Linux Media Labs LML33/LML33R10, Pinnacle/Miro
-DC10/DC10+/DC30/DC30+ and related boards (available under various names).
-
-Iomega Buz
-~~~~~~~~~~
-
-* Zoran zr36067 PCI controller
-* Zoran zr36060 MJPEG codec
-* Philips saa7111 TV decoder
-* Philips saa7185 TV encoder
-
-Drivers to use: videodev, i2c-core, i2c-algo-bit,
-videocodec, saa7111, saa7185, zr36060, zr36067
-
-Inputs/outputs: Composite and S-video
-
-Norms: PAL, SECAM (720x576 @ 25 fps), NTSC (720x480 @ 29.97 fps)
-
-Card number: 7
-
-AverMedia 6 Eyes AVS6EYES
-~~~~~~~~~~~~~~~~~~~~~~~~~
-
-* Zoran zr36067 PCI controller
-* Zoran zr36060 MJPEG codec
-* Samsung ks0127 TV decoder
-* Conexant bt866  TV encoder
-
-Drivers to use: videodev, i2c-core, i2c-algo-bit,
-videocodec, ks0127, bt866, zr36060, zr36067
-
-Inputs/outputs:
-	Six physical inputs. 1-6 are composite,
-	1-2, 3-4, 5-6 doubles as S-video,
-	1-3 triples as component.
-	One composite output.
-
-Norms: PAL, SECAM (720x576 @ 25 fps), NTSC (720x480 @ 29.97 fps)
-
-Card number: 8
-
-.. note::
-
-   Not autodetected, card=8 is necessary.
-
-Linux Media Labs LML33
-~~~~~~~~~~~~~~~~~~~~~~
-
-* Zoran zr36067 PCI controller
-* Zoran zr36060 MJPEG codec
-* Brooktree bt819 TV decoder
-* Brooktree bt856 TV encoder
-
-Drivers to use: videodev, i2c-core, i2c-algo-bit,
-videocodec, bt819, bt856, zr36060, zr36067
-
-Inputs/outputs: Composite and S-video
-
-Norms: PAL (720x576 @ 25 fps), NTSC (720x480 @ 29.97 fps)
-
-Card number: 5
-
-Linux Media Labs LML33R10
-~~~~~~~~~~~~~~~~~~~~~~~~~
-
-* Zoran zr36067 PCI controller
-* Zoran zr36060 MJPEG codec
-* Philips saa7114 TV decoder
-* Analog Devices adv7170 TV encoder
-
-Drivers to use: videodev, i2c-core, i2c-algo-bit,
-videocodec, saa7114, adv7170, zr36060, zr36067
-
-Inputs/outputs: Composite and S-video
-
-Norms: PAL (720x576 @ 25 fps), NTSC (720x480 @ 29.97 fps)
-
-Card number: 6
-
-Pinnacle/Miro DC10(new)
-~~~~~~~~~~~~~~~~~~~~~~~
-
-* Zoran zr36057 PCI controller
-* Zoran zr36060 MJPEG codec
-* Philips saa7110a TV decoder
-* Analog Devices adv7176 TV encoder
-
-Drivers to use: videodev, i2c-core, i2c-algo-bit,
-videocodec, saa7110, adv7175, zr36060, zr36067
-
-Inputs/outputs: Composite, S-video and Internal
-
-Norms: PAL, SECAM (768x576 @ 25 fps), NTSC (640x480 @ 29.97 fps)
-
-Card number: 1
-
-Pinnacle/Miro DC10+
-~~~~~~~~~~~~~~~~~~~
-
-* Zoran zr36067 PCI controller
-* Zoran zr36060 MJPEG codec
-* Philips saa7110a TV decoder
-* Analog Devices adv7176 TV encoder
-
-Drivers to use: videodev, i2c-core, i2c-algo-bit,
-videocodec, sa7110, adv7175, zr36060, zr36067
-
-Inputs/outputs: Composite, S-video and Internal
-
-Norms: PAL, SECAM (768x576 @ 25 fps), NTSC (640x480 @ 29.97 fps)
-
-Card number: 2
-
-Pinnacle/Miro DC10(old)
-~~~~~~~~~~~~~~~~~~~~~~~
-
-* Zoran zr36057 PCI controller
-* Zoran zr36050 MJPEG codec
-* Zoran zr36016 Video Front End or Fuji md0211 Video Front End (clone?)
-* Micronas vpx3220a TV decoder
-* mse3000 TV encoder or Analog Devices adv7176 TV encoder
-
-Drivers to use: videodev, i2c-core, i2c-algo-bit,
-videocodec, vpx3220, mse3000/adv7175, zr36050, zr36016, zr36067
-
-Inputs/outputs: Composite, S-video and Internal
-
-Norms: PAL, SECAM (768x576 @ 25 fps), NTSC (640x480 @ 29.97 fps)
-
-Card number: 0
-
-Pinnacle/Miro DC30
-~~~~~~~~~~~~~~~~~~
-
-* Zoran zr36057 PCI controller
-* Zoran zr36050 MJPEG codec
-* Zoran zr36016 Video Front End
-* Micronas vpx3225d/vpx3220a/vpx3216b TV decoder
-* Analog Devices adv7176 TV encoder
-
-Drivers to use: videodev, i2c-core, i2c-algo-bit,
-videocodec, vpx3220/vpx3224, adv7175, zr36050, zr36016, zr36067
-
-Inputs/outputs: Composite, S-video and Internal
-
-Norms: PAL, SECAM (768x576 @ 25 fps), NTSC (640x480 @ 29.97 fps)
-
-Card number: 3
-
-Pinnacle/Miro DC30+
-~~~~~~~~~~~~~~~~~~~
-
-* Zoran zr36067 PCI controller
-* Zoran zr36050 MJPEG codec
-* Zoran zr36016 Video Front End
-* Micronas vpx3225d/vpx3220a/vpx3216b TV decoder
-* Analog Devices adv7176 TV encoder
-
-Drivers to use: videodev, i2c-core, i2c-algo-bit,
-videocodec, vpx3220/vpx3224, adv7175, zr36050, zr36015, zr36067
-
-Inputs/outputs: Composite, S-video and Internal
-
-Norms: PAL, SECAM (768x576 @ 25 fps), NTSC (640x480 @ 29.97 fps)
-
-Card number: 4
-
-.. note::
-
-   #) No module for the mse3000 is available yet
-   #) No module for the vpx3224 is available yet
-
-1.1 What the TV decoder can do an what not
-------------------------------------------
-
-The best know TV standards are NTSC/PAL/SECAM. but for decoding a frame that
-information is not enough. There are several formats of the TV standards.
-And not every TV decoder is able to handle every format. Also the every
-combination is supported by the driver. There are currently 11 different
-tv broadcast formats all aver the world.
-
-The CCIR defines parameters needed for broadcasting the signal.
-The CCIR has defined different standards: A,B,D,E,F,G,D,H,I,K,K1,L,M,N,...
-The CCIR says not much about the colorsystem used !!!
-And talking about a colorsystem says not to much about how it is broadcast.
-
-The CCIR standards A,E,F are not used any more.
-
-When you speak about NTSC, you usually mean the standard: CCIR - M using
-the NTSC colorsystem which is used in the USA, Japan, Mexico, Canada
-and a few others.
-
-When you talk about PAL, you usually mean: CCIR - B/G using the PAL
-colorsystem which is used in many Countries.
-
-When you talk about SECAM, you mean: CCIR - L using the SECAM Colorsystem
-which is used in France, and a few others.
-
-There the other version of SECAM, CCIR - D/K is used in Bulgaria, China,
-Slovakai, Hungary, Korea (Rep.), Poland, Rumania and a others.
-
-The CCIR - H uses the PAL colorsystem (sometimes SECAM) and is used in
-Egypt, Libya, Sri Lanka, Syrain Arab. Rep.
-
-The CCIR - I uses the PAL colorsystem, and is used in Great Britain, Hong Kong,
-Ireland, Nigeria, South Africa.
-
-The CCIR - N uses the PAL colorsystem and PAL frame size but the NTSC framerate,
-and is used in Argentinia, Uruguay, an a few others
-
-We do not talk about how the audio is broadcast !
-
-A rather good sites about the TV standards are:
-http://www.sony.jp/support/
-http://info.electronicwerkstatt.de/bereiche/fernsehtechnik/frequenzen_und_normen/Fernsehnormen/
-and http://www.cabl.com/restaurant/channel.html
-
-Other weird things around: NTSC 4.43 is a modificated NTSC, which is mainly
-used in PAL VCR's that are able to play back NTSC. PAL 60 seems to be the same
-as NTSC 4.43 . The Datasheets also talk about NTSC 44, It seems as if it would
-be the same as NTSC 4.43.
-NTSC Combs seems to be a decoder mode where the decoder uses a comb filter
-to split coma and luma instead of a Delay line.
-
-But I did not defiantly find out what NTSC Comb is.
-
-Philips saa7111 TV decoder
-~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-- was introduced in 1997, is used in the BUZ and
-- can handle: PAL B/G/H/I, PAL N, PAL M, NTSC M, NTSC N, NTSC 4.43 and SECAM
-
-Philips saa7110a TV decoder
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-- was introduced in 1995, is used in the Pinnacle/Miro DC10(new), DC10+ and
-- can handle: PAL B/G, NTSC M and SECAM
-
-Philips saa7114 TV decoder
-~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-- was introduced in 2000, is used in the LML33R10 and
-- can handle: PAL B/G/D/H/I/N, PAL N, PAL M, NTSC M, NTSC 4.43 and SECAM
-
-Brooktree bt819 TV decoder
-~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-- was introduced in 1996, and is used in the LML33 and
-- can handle: PAL B/D/G/H/I, NTSC M
-
-Micronas vpx3220a TV decoder
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-- was introduced in 1996, is used in the DC30 and DC30+ and
-- can handle: PAL B/G/H/I, PAL N, PAL M, NTSC M, NTSC 44, PAL 60, SECAM,NTSC Comb
-
-Samsung ks0127 TV decoder
-~~~~~~~~~~~~~~~~~~~~~~~~~
-
-- is used in the AVS6EYES card and
-- can handle: NTSC-M/N/44, PAL-M/N/B/G/H/I/D/K/L and SECAM
-
-
-What the TV encoder can do an what not
---------------------------------------
-
-The TV encoder are doing the "same" as the decoder, but in the oder direction.
-You feed them digital data and the generate a Composite or SVHS signal.
-For information about the colorsystems and TV norm take a look in the
-TV decoder section.
-
-Philips saa7185 TV Encoder
-~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-- was introduced in 1996, is used in the BUZ
-- can generate: PAL B/G, NTSC M
-
-Brooktree bt856 TV Encoder
-~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-- was introduced in 1994, is used in the LML33
-- can generate: PAL B/D/G/H/I/N, PAL M, NTSC M, PAL-N (Argentina)
-
-Analog Devices adv7170 TV Encoder
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-- was introduced in 2000, is used in the LML300R10
-- can generate: PAL B/D/G/H/I/N, PAL M, NTSC M, PAL 60
-
-Analog Devices adv7175 TV Encoder
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-- was introduced in 1996, is used in the DC10, DC10+, DC10 old, DC30, DC30+
-- can generate: PAL B/D/G/H/I/N, PAL M, NTSC M
-
-ITT mse3000 TV encoder
-~~~~~~~~~~~~~~~~~~~~~~
-
-- was introduced in 1991, is used in the DC10 old
-- can generate: PAL , NTSC , SECAM
-
-Conexant bt866 TV encoder
-~~~~~~~~~~~~~~~~~~~~~~~~~
-
-- is used in AVS6EYES, and
-- can generate: NTSC/PAL, PAL­M, PAL­N
-
-The adv717x, should be able to produce PAL N. But you find nothing PAL N
-specific in the registers. Seem that you have to reuse a other standard
-to generate PAL N, maybe it would work if you use the PAL M settings.
-
-How do I get this damn thing to work
-------------------------------------
-
-Load zr36067.o. If it can't autodetect your card, use the card=X insmod
-option with X being the card number as given in the previous section.
-To have more than one card, use card=X1[,X2[,X3,[X4[..]]]]
-
-To automate this, add the following to your /etc/modprobe.d/zoran.conf:
-
-options zr36067 card=X1[,X2[,X3[,X4[..]]]]
-alias char-major-81-0 zr36067
-
-One thing to keep in mind is that this doesn't load zr36067.o itself yet. It
-just automates loading. If you start using xawtv, the device won't load on
-some systems, since you're trying to load modules as a user, which is not
-allowed ("permission denied"). A quick workaround is to add 'Load "v4l"' to
-XF86Config-4 when you use X by default, or to run 'v4l-conf -c <device>' in
-one of your startup scripts (normally rc.local) if you don't use X. Both
-make sure that the modules are loaded on startup, under the root account.
-
-What mainboard should I use (or why doesn't my card work)
----------------------------------------------------------
-
-
-<insert lousy disclaimer here>. In short: good=SiS/Intel, bad=VIA.
-
-Experience tells us that people with a Buz, on average, have more problems
-than users with a DC10+/LML33. Also, it tells us that people owning a VIA-
-based mainboard (ktXXX, MVP3) have more problems than users with a mainboard
-based on a different chipset. Here's some notes from Andrew Stevens:
-
-Here's my experience of using LML33 and Buz on various motherboards:
-
-- VIA MVP3
-	- Forget it. Pointless. Doesn't work.
-- Intel 430FX (Pentium 200)
-	- LML33 perfect, Buz tolerable (3 or 4 frames dropped per movie)
-- Intel 440BX (early stepping)
-	- LML33 tolerable. Buz starting to get annoying (6-10 frames/hour)
-- Intel 440BX (late stepping)
-	- Buz tolerable, LML3 almost perfect (occasional single frame drops)
-- SiS735
-	- LML33 perfect, Buz tolerable.
-- VIA KT133(*)
-	- LML33 starting to get annoying, Buz poor enough that I have up.
-
-- Both 440BX boards were dual CPU versions.
-
-Bernhard Praschinger later added:
-
-- AMD 751
-	- Buz perfect-tolerable
-- AMD 760
-	- Buz perfect-tolerable
-
-In general, people on the user mailinglist won't give you much of a chance
-if you have a VIA-based motherboard. They may be cheap, but sometimes, you'd
-rather want to spend some more money on better boards. In general, VIA
-mainboard's IDE/PCI performance will also suck badly compared to others.
-You'll noticed the DC10+/DC30+ aren't mentioned anywhere in the overview.
-Basically, you can assume that if the Buz works, the LML33 will work too. If
-the LML33 works, the DC10+/DC30+ will work too. They're most tolerant to
-different mainboard chipsets from all of the supported cards.
-
-If you experience timeouts during capture, buy a better mainboard or lower
-the quality/buffersize during capture (see 'Concerning buffer sizes, quality,
-output size etc.'). If it hangs, there's little we can do as of now. Check
-your IRQs and make sure the card has its own interrupts.
-
-Programming interface
----------------------
-
-This driver conforms to video4linux2. Support for V4L1 and for the custom
-zoran ioctls has been removed in kernel 2.6.38.
-
-For programming example, please, look at lavrec.c and lavplay.c code in
-the MJPEG-tools (http://mjpeg.sf.net/).
-
-Additional notes for software developers:
-
-   The driver returns maxwidth and maxheight parameters according to
-   the current TV standard (norm). Therefore, the software which
-   communicates with the driver and "asks" for these parameters should
-   first set the correct norm. Well, it seems logically correct: TV
-   standard is "more constant" for current country than geometry
-   settings of a variety of TV capture cards which may work in ITU or
-   square pixel format.
-
-Applications
-------------
-
-Applications known to work with this driver:
-
-TV viewing:
-
-* xawtv
-* kwintv
-* probably any TV application that supports video4linux or video4linux2.
-
-MJPEG capture/playback:
-
-* mjpegtools/lavtools (or Linux Video Studio)
-* gstreamer
-* mplayer
-
-General raw capture:
-
-* xawtv
-* gstreamer
-* probably any application that supports video4linux or video4linux2
-
-Video editing:
-
-* Cinelerra
-* MainActor
-* mjpegtools (or Linux Video Studio)
-
-
-Concerning buffer sizes, quality, output size etc.
---------------------------------------------------
-
-
-The zr36060 can do 1:2 JPEG compression. This is really the theoretical
-maximum that the chipset can reach. The driver can, however, limit compression
-to a maximum (size) of 1:4. The reason for this is that some cards (e.g. Buz)
-can't handle 1:2 compression without stopping capture after only a few minutes.
-With 1:4, it'll mostly work. If you have a Buz, use 'low_bitrate=1' to go into
-1:4 max. compression mode.
-
-100% JPEG quality is thus 1:2 compression in practice. So for a full PAL frame
-(size 720x576). The JPEG fields are stored in YUY2 format, so the size of the
-fields are 720x288x16/2 bits/field (2 fields/frame) = 207360 bytes/field x 2 =
-414720 bytes/frame (add some more bytes for headers and DHT (huffman)/DQT
-(quantization) tables, and you'll get to something like 512kB per frame for
-1:2 compression. For 1:4 compression, you'd have frames of half this size.
-
-Some additional explanation by Martin Samuelsson, which also explains the
-importance of buffer sizes:
---
-> Hmm, I do not think it is really that way. With the current (downloaded
-> at 18:00 Monday) driver I get that output sizes for 10 sec:
-> -q 50 -b 128 : 24.283.332 Bytes
-> -q 50 -b 256 : 48.442.368
-> -q 25 -b 128 : 24.655.992
-> -q 25 -b 256 : 25.859.820
-
-I woke up, and can't go to sleep again. I'll kill some time explaining why
-this doesn't look strange to me.
-
-Let's do some math using a width of 704 pixels. I'm not sure whether the Buz
-actually use that number or not, but that's not too important right now.
-
-704x288 pixels, one field, is 202752 pixels. Divided by 64 pixels per block;
-3168 blocks per field. Each pixel consist of two bytes; 128 bytes per block;
-1024 bits per block. 100% in the new driver mean 1:2 compression; the maximum
-output becomes 512 bits per block. Actually 510, but 512 is simpler to use
-for calculations.
-
-Let's say that we specify d1q50. We thus want 256 bits per block; times 3168
-becomes 811008 bits; 101376 bytes per field. We're talking raw bits and bytes
-here, so we don't need to do any fancy corrections for bits-per-pixel or such
-things. 101376 bytes per field.
-
-d1 video contains two fields per frame. Those sum up to 202752 bytes per
-frame, and one of those frames goes into each buffer.
-
-But wait a second! -b128 gives 128kB buffers! It's not possible to cram
-202752 bytes of JPEG data into 128kB!
-
-This is what the driver notice and automatically compensate for in your
-examples. Let's do some math using this information:
-
-128kB is 131072 bytes. In this buffer, we want to store two fields, which
-leaves 65536 bytes for each field. Using 3168 blocks per field, we get
-20.68686868... available bytes per block; 165 bits. We can't allow the
-request for 256 bits per block when there's only 165 bits available! The -q50
-option is silently overridden, and the -b128 option takes precedence, leaving
-us with the equivalence of -q32.
-
-This gives us a data rate of 165 bits per block, which, times 3168, sums up
-to 65340 bytes per field, out of the allowed 65536. The current driver has
-another level of rate limiting; it won't accept -q values that fill more than
-6/8 of the specified buffers. (I'm not sure why. "Playing it safe" seem to be
-a safe bet. Personally, I think I would have lowered requested-bits-per-block
-by one, or something like that.) We can't use 165 bits per block, but have to
-lower it again, to 6/8 of the available buffer space: We end up with 124 bits
-per block, the equivalence of -q24. With 128kB buffers, you can't use greater
-than -q24 at -d1. (And PAL, and 704 pixels width...)
-
-The third example is limited to -q24 through the same process. The second
-example, using very similar calculations, is limited to -q48. The only
-example that actually grab at the specified -q value is the last one, which
-is clearly visible, looking at the file size.
---
-
-Conclusion: the quality of the resulting movie depends on buffer size, quality,
-whether or not you use 'low_bitrate=1' as insmod option for the zr36060.c
-module to do 1:4 instead of 1:2 compression, etc.
-
-If you experience timeouts, lowering the quality/buffersize or using
-'low_bitrate=1 as insmod option for zr36060.o might actually help, as is
-proven by the Buz.
-
-It hangs/crashes/fails/whatevers! Help!
----------------------------------------
-
-Make sure that the card has its own interrupts (see /proc/interrupts), check
-the output of dmesg at high verbosity (load zr36067.o with debug=2,
-load all other modules with debug=1). Check that your mainboard is favorable
-(see question 2) and if not, test the card in another computer. Also see the
-notes given in question 3 and try lowering quality/buffersize/capturesize
-if recording fails after a period of time.
-
-If all this doesn't help, give a clear description of the problem including
-detailed hardware information (memory+brand, mainboard+chipset+brand, which
-MJPEG card, processor, other PCI cards that might be of interest), give the
-system PnP information (/proc/interrupts, /proc/dma, /proc/devices), and give
-the kernel version, driver version, glibc version, gcc version and any other
-information that might possibly be of interest. Also provide the dmesg output
-at high verbosity. See 'Contacting' on how to contact the developers.
-
-Maintainers/Contacting
-----------------------
-
-The driver is currently maintained by Laurent Pinchart and Ronald Bultje
-(<laurent.pinchart@skynet.be> and <rbultje@ronald.bitfreak.net>). For bug
-reports or questions, please contact the mailinglist instead of the developers
-individually. For user questions (i.e. bug reports or how-to questions), send
-an email to <mjpeg-users@lists.sf.net>, for developers (i.e. if you want to
-help programming), send an email to <mjpeg-developer@lists.sf.net>. See
-http://www.sf.net/projects/mjpeg/ for subscription information.
-
-For bug reports, be sure to include all the information as described in
-the section 'It hangs/crashes/fails/whatevers! Help!'. Please make sure
-you're using the latest version (http://mjpeg.sf.net/driver-zoran/).
-
-Previous maintainers/developers of this driver include Serguei Miridonov
-<mirsev@cicese.mx>, Wolfgang Scherr <scherr@net4you.net>, Dave Perks
-<dperks@ibm.net> and Rainer Johanni <Rainer@Johanni.de>.
-
-Driver's License
-----------------
-
-    This driver is distributed under the terms of the General Public License.
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-See http://www.gnu.org/ for more information.
diff --git a/Documentation/ntb.txt b/Documentation/ntb.txt
index a043854d28df..074a423c853c 100644
--- a/Documentation/ntb.txt
+++ b/Documentation/ntb.txt
@@ -41,9 +41,10 @@ mainly used to perform the proper memory window initialization. Typically
 there are two types of memory window interfaces supported by the NTB API:
 inbound translation configured on the local ntb port and outbound translation
 configured by the peer, on the peer ntb port. The first type is
-depicted on the next figure
+depicted on the next figure::
+
+ Inbound translation:
 
-Inbound translation:
  Memory:              Local NTB Port:      Peer NTB Port:      Peer MMIO:
   ____________
  | dma-mapped |-ntb_mw_set_trans(addr)  |
@@ -58,9 +59,10 @@ maps corresponding outbound memory window so to have access to the shared
 memory region.
 
 The second type of interface, that implies the shared windows being
-initialized by a peer device, is depicted on the figure:
+initialized by a peer device, is depicted on the figure::
+
+ Outbound translation:
 
-Outbound translation:
  Memory:        Local NTB Port:    Peer NTB Port:      Peer MMIO:
   ____________                      ______________
  | dma-mapped |                |   | MW base addr |<== memory-mapped IO
@@ -75,11 +77,13 @@ outbound memory window so to have access to the shared memory region.
 
 As one can see the described scenarios can be combined in one portable
 algorithm.
+
  Local device:
   1) Allocate memory for a shared window
   2) Initialize memory window by translated address of the allocated region
      (it may fail if local memory window initialization is unsupported)
   3) Send the translated address and memory window index to a peer device
+
  Peer device:
   1) Initialize memory window with retrieved address of the allocated
      by another device memory region (it may fail if peer memory window
@@ -88,6 +92,7 @@ algorithm.
 
 In accordance with this scenario, the NTB Memory Window API can be used as
 follows:
+
  Local device:
   1) ntb_mw_count(pidx) - retrieve number of memory ranges, which can
      be allocated for memory windows between local device and peer device
@@ -103,6 +108,7 @@ follows:
   5) Send translated base address (usually together with memory window
      number) to the peer device using, for instance, scratchpad or message
      registers.
+
  Peer device:
   1) ntb_peer_mw_set_trans(pidx, midx) - try to set received from other
      device (related to pidx) translated address for specified memory
diff --git a/Documentation/process/5.Posting.rst b/Documentation/process/5.Posting.rst
index 4213e580f273..855a70b80269 100644
--- a/Documentation/process/5.Posting.rst
+++ b/Documentation/process/5.Posting.rst
@@ -216,10 +216,12 @@ The tags in common use are:
    which can be found in :ref:`Documentation/process/submitting-patches.rst <submittingpatches>`
    Code without a proper signoff cannot be merged into the mainline.
 
- - Co-developed-by: states that the patch was also created by another developer
-   along with the original author.  This is useful at times when multiple
-   people work on a single patch.  Note, this person also needs to have a
-   Signed-off-by: line in the patch as well.
+ - Co-developed-by: states that the patch was co-created by several developers;
+   it is a used to give attribution to co-authors (in addition to the author
+   attributed by the From: tag) when multiple people work on a single patch.
+   Every Co-developed-by: must be immediately followed by a Signed-off-by: of
+   the associated co-author.  Details and examples can be found in
+   :ref:`Documentation/process/submitting-patches.rst <submittingpatches>`.
 
  - Acked-by: indicates an agreement by another developer (often a
    maintainer of the relevant code) that the patch is appropriate for
diff --git a/Documentation/process/coding-style.rst b/Documentation/process/coding-style.rst
index 8ea913e99fa1..fa864a51e6ea 100644
--- a/Documentation/process/coding-style.rst
+++ b/Documentation/process/coding-style.rst
@@ -843,7 +843,8 @@ used.
 The kernel provides the following general purpose memory allocators:
 kmalloc(), kzalloc(), kmalloc_array(), kcalloc(), vmalloc(), and
 vzalloc().  Please refer to the API documentation for further information
-about them.
+about them.  :ref:`Documentation/core-api/memory-allocation.rst
+<memory_allocation>`
 
 The preferred form for passing a size of a struct is the following:
 
@@ -874,6 +875,9 @@ The preferred form for allocating a zeroed array is the following:
 Both forms check for overflow on the allocation size n * sizeof(...),
 and return NULL if that occurred.
 
+These generic allocation functions all emit a stack dump on failure when used
+without __GFP_NOWARN so there is no use in emitting an additional failure
+message when NULL is returned.
 
 15) The inline disease
 ----------------------
diff --git a/Documentation/process/deprecated.rst b/Documentation/process/deprecated.rst
index 0ef5a63c06ba..49e0f64a3427 100644
--- a/Documentation/process/deprecated.rst
+++ b/Documentation/process/deprecated.rst
@@ -1,5 +1,7 @@
 .. SPDX-License-Identifier: GPL-2.0
 
+.. _deprecated:
+
 =====================================================================
 Deprecated Interfaces, Language Features, Attributes, and Conventions
 =====================================================================
diff --git a/Documentation/process/howto.rst b/Documentation/process/howto.rst
index ad2b6c852b95..6ab75c11d2c3 100644
--- a/Documentation/process/howto.rst
+++ b/Documentation/process/howto.rst
@@ -62,7 +62,7 @@ Legal Issues
 The Linux kernel source code is released under the GPL.  Please see the file
 COPYING in the main directory of the source tree. The Linux kernel licensing
 rules and how to use `SPDX <https://spdx.org/>`_ identifiers in source code are
-descibed in :ref:`Documentation/process/license-rules.rst <kernel_licensing>`.
+described in :ref:`Documentation/process/license-rules.rst <kernel_licensing>`.
 If you have further questions about the license, please contact a lawyer, and do
 not ask on the Linux kernel mailing list.  The people on the mailing lists are
 not lawyers, and you should not rely on their statements on legal matters.
diff --git a/Documentation/process/kernel-docs.rst b/Documentation/process/kernel-docs.rst
index ab12dddc773e..7a45a8e36ea7 100644
--- a/Documentation/process/kernel-docs.rst
+++ b/Documentation/process/kernel-docs.rst
@@ -95,18 +95,6 @@ On-line docs
         [...]. This paper examines some common problems for
         submitting larger changes and some strategies to avoid problems.
 
-    * Title: **Overview of the Virtual File System**
-
-      :Author: Richard Gooch.
-      :URL: http://www.mjmwired.net/kernel/Documentation/filesystems/vfs.txt
-      :Date: 2007
-      :Keywords: VFS, File System, mounting filesystems, opening files,
-        dentries, dcache.
-      :Description: Brief introduction to the Linux Virtual File System.
-        What is it, how it works, operations taken when opening a file or
-        mounting a file system and description of important data
-        structures explaining the purpose of each of their entries.
-
     * Title: **Linux Device Drivers, Third Edition**
 
       :Author: Jonathan Corbet, Alessandro Rubini, Greg Kroah-Hartman
diff --git a/Documentation/process/license-rules.rst b/Documentation/process/license-rules.rst
index 6b09033a8e9e..2ef44ada3f11 100644
--- a/Documentation/process/license-rules.rst
+++ b/Documentation/process/license-rules.rst
@@ -234,13 +234,13 @@ kernel, can be broken down into:
 
 |
 
-2. Not recommended licenses:
+2. Deprecated licenses:
 
    These licenses should only be used for existing code or for importing
    code from a different project.  These licenses are available from the
    directory::
 
-      LICENSES/other/
+      LICENSES/deprecated/
 
    in the kernel source tree.
 
@@ -250,14 +250,14 @@ kernel, can be broken down into:
 
    Examples::
 
-      LICENSES/other/ISC
+      LICENSES/deprecated/ISC
 
    Contains the Internet Systems Consortium license text and the required
    metatags::
 
-      LICENSES/other/ZLib
+      LICENSES/deprecated/GPL-1.0
 
-   Contains the ZLIB license text and the required metatags.
+   Contains the GPL version 1 license text and the required metatags.
 
    Metatags:
 
@@ -281,7 +281,56 @@ kernel, can be broken down into:
 
 |
 
-3. _`Exceptions`:
+3. Dual Licensing Only
+
+   These licenses should only be used to dual license code with another
+   license in addition to a preferred license.  These licenses are available
+   from the directory::
+
+      LICENSES/dual/
+
+   in the kernel source tree.
+
+   The files in this directory contain the full license text and
+   `Metatags`_.  The file names are identical to the SPDX license
+   identifier which shall be used for the license in source files.
+
+   Examples::
+
+      LICENSES/dual/MPL-1.1
+
+   Contains the Mozilla Public License version 1.1 license text and the
+   required metatags::
+
+      LICENSES/dual/Apache-2.0
+
+   Contains the Apache License version 2.0 license text and the required
+   metatags.
+
+   Metatags:
+
+   The metatag requirements for 'other' licenses are identical to the
+   requirements of the `Preferred licenses`_.
+
+   File format example::
+
+      Valid-License-Identifier: MPL-1.1
+      SPDX-URL: https://spdx.org/licenses/MPL-1.1.html
+      Usage-Guide:
+        Do NOT use. The MPL-1.1 is not GPL2 compatible. It may only be used for
+        dual-licensed files where the other license is GPL2 compatible.
+        If you end up using this it MUST be used together with a GPL2 compatible
+        license using "OR".
+        To use the Mozilla Public License version 1.1 put the following SPDX
+        tag/value pair into a comment according to the placement guidelines in
+        the licensing rules documentation:
+      SPDX-License-Identifier: MPL-1.1
+      License-Text:
+        Full license text
+
+|
+
+4. _`Exceptions`:
 
    Some licenses can be amended with exceptions which grant certain rights
    which the original license does not.  These exceptions are available
diff --git a/Documentation/process/maintainer-pgp-guide.rst b/Documentation/process/maintainer-pgp-guide.rst
index aff9b1a4d77b..4bab7464ff8c 100644
--- a/Documentation/process/maintainer-pgp-guide.rst
+++ b/Documentation/process/maintainer-pgp-guide.rst
@@ -943,7 +943,7 @@ have on your keyring::
 
 Next, open the `PGP pathfinder`_. In the "From" field, paste the key
 fingerprint of Linus Torvalds from the output above. In the "To" field,
-paste they key-id you found via ``gpg --search`` of the unknown key, and
+paste the key-id you found via ``gpg --search`` of the unknown key, and
 check the results:
 
 - `Finding paths to Linus`_
diff --git a/Documentation/process/submitting-patches.rst b/Documentation/process/submitting-patches.rst
index be7d1829c3af..9c4299293c72 100644
--- a/Documentation/process/submitting-patches.rst
+++ b/Documentation/process/submitting-patches.rst
@@ -60,8 +60,8 @@ not in any lower subdirectory.
 
 To create a patch for a single file, it is often sufficient to do::
 
-	SRCTREE= linux
-	MYFILE=  drivers/net/mydriver.c
+	SRCTREE=linux
+	MYFILE=drivers/net/mydriver.c
 
 	cd $SRCTREE
 	cp $MYFILE $MYFILE.orig
@@ -73,7 +73,7 @@ To create a patch for multiple files, you should unpack a "vanilla",
 or unmodified kernel source tree, and generate a ``diff`` against your
 own source tree.  For example::
 
-	MYSRC= /devel/linux
+	MYSRC=/devel/linux
 
 	tar xvfz linux-3.19.tar.gz
 	mv linux-3.19 linux-3.19-vanilla
@@ -545,10 +545,40 @@ person it names - but it should indicate that this person was copied on the
 patch.  This tag documents that potentially interested parties
 have been included in the discussion.
 
-A Co-developed-by: states that the patch was also created by another developer
-along with the original author.  This is useful at times when multiple people
-work on a single patch.  Note, this person also needs to have a Signed-off-by:
-line in the patch as well.
+Co-developed-by: states that the patch was co-created by multiple developers;
+it is a used to give attribution to co-authors (in addition to the author
+attributed by the From: tag) when several people work on a single patch.  Since
+Co-developed-by: denotes authorship, every Co-developed-by: must be immediately
+followed by a Signed-off-by: of the associated co-author.  Standard sign-off
+procedure applies, i.e. the ordering of Signed-off-by: tags should reflect the
+chronological history of the patch insofar as possible, regardless of whether
+the author is attributed via From: or Co-developed-by:.  Notably, the last
+Signed-off-by: must always be that of the developer submitting the patch.
+
+Note, the From: tag is optional when the From: author is also the person (and
+email) listed in the From: line of the email header.
+
+Example of a patch submitted by the From: author::
+
+	<changelog>
+
+	Co-developed-by: First Co-Author <first@coauthor.example.org>
+	Signed-off-by: First Co-Author <first@coauthor.example.org>
+	Co-developed-by: Second Co-Author <second@coauthor.example.org>
+	Signed-off-by: Second Co-Author <second@coauthor.example.org>
+	Signed-off-by: From Author <from@author.example.org>
+
+Example of a patch submitted by a Co-developed-by: author::
+
+	From: From Author <from@author.example.org>
+
+	<changelog>
+
+	Co-developed-by: Random Co-Author <random@coauthor.example.org>
+	Signed-off-by: Random Co-Author <random@coauthor.example.org>
+	Signed-off-by: From Author <from@author.example.org>
+	Co-developed-by: Submitting Co-Author <sub@coauthor.example.org>
+	Signed-off-by: Submitting Co-Author <sub@coauthor.example.org>
 
 
 13) Using Reported-by:, Tested-by:, Reviewed-by:, Suggested-by: and Fixes:
@@ -696,7 +726,7 @@ A couple of example Subjects::
 The ``from`` line must be the very first line in the message body,
 and has the form:
 
-        From: Original Author <author@example.com>
+        From: Patch Author <author@example.com>
 
 The ``from`` line specifies who will be credited as the author of the
 patch in the permanent changelog.  If the ``from`` line is missing,
diff --git a/Documentation/rtc.txt b/Documentation/rtc.txt
index a129acf38537..688c95b11919 100644
--- a/Documentation/rtc.txt
+++ b/Documentation/rtc.txt
@@ -136,5 +136,5 @@ a high functionality RTC is integrated into the SOC.  That system might read
 the system clock from the discrete RTC, but use the integrated one for all
 other tasks, because of its greater functionality.
 
-Check out tools/testing/selftests/timers/rtctest.c for an example usage of the
+Check out tools/testing/selftests/rtc/rtctest.c for an example usage of the
 ioctl interface.
diff --git a/Documentation/serial/README.cycladesZ b/Documentation/serial/cyclades_z.rst
index 024a69443cc2..532ff67e2f1c 100644
--- a/Documentation/serial/README.cycladesZ
+++ b/Documentation/serial/cyclades_z.rst
@@ -1,8 +1,11 @@
+================
+Cyclades-Z notes
+================
 
 The Cyclades-Z must have firmware loaded onto the card before it will
 operate.  This operation should be performed during system startup,
 
 The firmware, loader program and the latest device driver code are
 available from Cyclades at
-    ftp://ftp.cyclades.com/pub/cyclades/cyclades-z/linux/
 
+    ftp://ftp.cyclades.com/pub/cyclades/cyclades-z/linux/
diff --git a/Documentation/serial/driver b/Documentation/serial/driver.rst
index 86e47c19a924..4537119bf624 100644
--- a/Documentation/serial/driver
+++ b/Documentation/serial/driver.rst
@@ -1,6 +1,6 @@
-
-			Low Level Serial API
-			--------------------
+====================
+Low Level Serial API
+====================
 
 
 This document is meant as a brief overview of some aspects of the new serial
@@ -44,7 +44,7 @@ are described in the uart_ops listing below.)
 There are two locks.  A per-port spinlock, and an overall semaphore.
 
 From the core driver perspective, the port->lock locks the following
-data:
+data::
 
 	port->mctrl
 	port->icount
@@ -75,41 +75,51 @@ hardware.
 	return TIOCSER_TEMT.
 
 	Locking: none.
+
 	Interrupts: caller dependent.
+
 	This call must not sleep
 
   set_mctrl(port, mctrl)
 	This function sets the modem control lines for port described
 	by 'port' to the state described by mctrl.  The relevant bits
 	of mctrl are:
+
 		- TIOCM_RTS	RTS signal.
 		- TIOCM_DTR	DTR signal.
 		- TIOCM_OUT1	OUT1 signal.
 		- TIOCM_OUT2	OUT2 signal.
 		- TIOCM_LOOP	Set the port into loopback mode.
+
 	If the appropriate bit is set, the signal should be driven
 	active.  If the bit is clear, the signal should be driven
 	inactive.
 
 	Locking: port->lock taken.
+
 	Interrupts: locally disabled.
+
 	This call must not sleep
 
   get_mctrl(port)
 	Returns the current state of modem control inputs.  The state
 	of the outputs should not be returned, since the core keeps
 	track of their state.  The state information should include:
+
 		- TIOCM_CAR	state of DCD signal
 		- TIOCM_CTS	state of CTS signal
 		- TIOCM_DSR	state of DSR signal
 		- TIOCM_RI	state of RI signal
+
 	The bit is set if the signal is currently driven active.  If
 	the port does not support CTS, DCD or DSR, the driver should
 	indicate that the signal is permanently active.  If RI is
 	not available, the signal should not be indicated as active.
 
 	Locking: port->lock taken.
+
 	Interrupts: locally disabled.
+
 	This call must not sleep
 
   stop_tx(port)
@@ -121,14 +131,18 @@ hardware.
 	possible.
 
 	Locking: port->lock taken.
+
 	Interrupts: locally disabled.
+
 	This call must not sleep
 
   start_tx(port)
 	Start transmitting characters.
 
 	Locking: port->lock taken.
+
 	Interrupts: locally disabled.
+
 	This call must not sleep
 
   throttle(port)
@@ -138,16 +152,17 @@ hardware.
 	This will be called only if hardware assisted flow control is enabled.
 
 	Locking: serialized with .unthrottle() and termios modification by the
-		 tty layer.
+	tty layer.
 
   unthrottle(port)
 	Notify the serial driver that characters can now be sent to the serial
 	port without fear of overrunning the input buffers of the line
 	disciplines.
+
 	This will be called only if hardware assisted flow control is enabled.
 
 	Locking: serialized with .throttle() and termios modification by the
-		 tty layer.
+	tty layer.
 
   send_xchar(port,ch)
 	Transmit a high priority character, even if the port is stopped.
@@ -159,6 +174,7 @@ hardware.
 	Do not transmit if ch == '\0' (__DISABLED_CHAR).
 
 	Locking: none.
+
 	Interrupts: caller dependent.
 
   stop_rx(port)
@@ -166,7 +182,9 @@ hardware.
 	being closed.
 
 	Locking: port->lock taken.
+
 	Interrupts: locally disabled.
+
 	This call must not sleep
 
   enable_ms(port)
@@ -177,7 +195,9 @@ hardware.
 	called.
 
 	Locking: port->lock taken.
+
 	Interrupts: locally disabled.
+
 	This call must not sleep
 
   break_ctl(port,ctl)
@@ -196,6 +216,7 @@ hardware.
 	This method will only be called when the port is initially opened.
 
 	Locking: port_sem taken.
+
 	Interrupts: globally disabled.
 
   shutdown(port)
@@ -210,6 +231,7 @@ hardware.
 	this port.
 
 	Locking: port_sem taken.
+
 	Interrupts: caller dependent.
 
   flush_buffer(port)
@@ -220,7 +242,9 @@ hardware.
 	buffer is cleared.
 
 	Locking: port->lock taken.
+
 	Interrupts: locally disabled.
+
 	This call must not sleep
 
   set_termios(port,termios,oldtermios)
@@ -228,29 +252,46 @@ hardware.
 	bits.  Update read_status_mask and ignore_status_mask to indicate
 	the types of events we are interested in receiving.  Relevant
 	termios->c_cflag bits are:
-		CSIZE	- word size
-		CSTOPB	- 2 stop bits
-		PARENB	- parity enable
-		PARODD	- odd parity (when PARENB is in force)
-		CREAD	- enable reception of characters (if not set,
+
+		CSIZE
+			- word size
+		CSTOPB
+			- 2 stop bits
+		PARENB
+			- parity enable
+		PARODD
+			- odd parity (when PARENB is in force)
+		CREAD
+			- enable reception of characters (if not set,
 			  still receive characters from the port, but
 			  throw them away.
-		CRTSCTS	- if set, enable CTS status change reporting
-		CLOCAL	- if not set, enable modem status change
+		CRTSCTS
+			- if set, enable CTS status change reporting
+		CLOCAL
+			- if not set, enable modem status change
 			  reporting.
+
 	Relevant termios->c_iflag bits are:
-		INPCK	- enable frame and parity error events to be
+
+		INPCK
+			- enable frame and parity error events to be
 			  passed to the TTY layer.
-		BRKINT
-		PARMRK	- both of these enable break events to be
+		BRKINT / PARMRK
+			- both of these enable break events to be
 			  passed to the TTY layer.
 
-		IGNPAR	- ignore parity and framing errors
-		IGNBRK	- ignore break errors,  If IGNPAR is also
+		IGNPAR
+			- ignore parity and framing errors
+		IGNBRK
+			- ignore break errors,  If IGNPAR is also
 			  set, ignore overrun errors as well.
+
 	The interaction of the iflag bits is as follows (parity error
 	given as an example):
+
+	=============== ======= ======  =============================
 	Parity error	INPCK	IGNPAR
+	=============== ======= ======  =============================
 	n/a		0	n/a	character received, marked as
 					TTY_NORMAL
 	None		1	n/a	character received, marked as
@@ -258,16 +299,19 @@ hardware.
 	Yes		1	0	character received, marked as
 					TTY_PARITY
 	Yes		1	1	character discarded
+	=============== ======= ======  =============================
 
 	Other flags may be used (eg, xon/xoff characters) if your
 	hardware supports hardware "soft" flow control.
 
 	Locking: caller holds tty_port->mutex
+
 	Interrupts: caller dependent.
+
 	This call must not sleep
 
   set_ldisc(port,termios)
-	Notifier for discipline change. See Documentation/serial/tty.txt.
+	Notifier for discipline change. See Documentation/serial/tty.rst.
 
 	Locking: caller holds tty_port->mutex
 
@@ -283,6 +327,7 @@ hardware.
 	will occur even if CONFIG_PM is not set.
 
 	Locking: none.
+
 	Interrupts: caller dependent.
 
   type(port)
@@ -291,6 +336,7 @@ hardware.
 	substituted.
 
 	Locking: none.
+
 	Interrupts: caller dependent.
 
   release_port(port)
@@ -298,6 +344,7 @@ hardware.
 	the port.
 
 	Locking: none.
+
 	Interrupts: caller dependent.
 
   request_port(port)
@@ -306,6 +353,7 @@ hardware.
 	returns, and it should return -EBUSY on failure.
 
 	Locking: none.
+
 	Interrupts: caller dependent.
 
   config_port(port,type)
@@ -321,6 +369,7 @@ hardware.
 	internally hard wired (eg, system on a chip implementations).
 
 	Locking: none.
+
 	Interrupts: caller dependent.
 
   verify_port(port,serinfo)
@@ -328,6 +377,7 @@ hardware.
 	suitable for this port type.
 
 	Locking: none.
+
 	Interrupts: caller dependent.
 
   ioctl(port,cmd,arg)
@@ -335,6 +385,7 @@ hardware.
 	using the standard numbering system found in <asm/ioctl.h>
 
 	Locking: none.
+
 	Interrupts: caller dependent.
 
   poll_init(port)
@@ -343,6 +394,7 @@ hardware.
 	this should not request interrupts.
 
 	Locking: tty_mutex and tty_port->mutex taken.
+
 	Interrupts: n/a.
 
   poll_put_char(port,ch)
@@ -350,7 +402,9 @@ hardware.
 	port.  It can and should block until there is space in the TX FIFO.
 
 	Locking: none.
+
 	Interrupts: caller dependent.
+
 	This call must not sleep
 
   poll_get_char(port)
@@ -359,7 +413,9 @@ hardware.
 	the function should return NO_POLL_CHAR immediately.
 
 	Locking: none.
+
 	Interrupts: caller dependent.
+
 	This call must not sleep
 
 Other functions
@@ -370,6 +426,7 @@ uart_update_timeout(port,cflag,baud)
 	number of bits, parity, stop bits and baud rate.
 
 	Locking: caller is expected to take port->lock
+
 	Interrupts: n/a
 
 uart_get_baud_rate(port,termios,old,min,max)
@@ -385,6 +442,7 @@ uart_get_baud_rate(port,termios,old,min,max)
 	Note: min..max must always allow 9600 baud to be selected.
 
 	Locking: caller dependent.
+
 	Interrupts: n/a
 
 uart_get_divisor(port,baud)
@@ -395,6 +453,7 @@ uart_get_divisor(port,baud)
 	custom divisor instead.
 
 	Locking: caller dependent.
+
 	Interrupts: n/a
 
 uart_match_port(port1,port2)
@@ -402,6 +461,7 @@ uart_match_port(port1,port2)
 	uart_port structures describe the same port.
 
 	Locking: n/a
+
 	Interrupts: n/a
 
 uart_write_wakeup(port)
@@ -409,6 +469,7 @@ uart_write_wakeup(port)
 	characters in the transmit buffer have dropped below a threshold.
 
 	Locking: port->lock should be held.
+
 	Interrupts: n/a
 
 uart_register_driver(drv)
@@ -419,6 +480,7 @@ uart_register_driver(drv)
 	registered using uart_add_one_port after this call has succeeded.
 
 	Locking: none
+
 	Interrupts: enabled
 
 uart_unregister_driver()
@@ -427,15 +489,16 @@ uart_unregister_driver()
 	uart_remove_one_port() if it registered them with uart_add_one_port().
 
 	Locking: none
+
 	Interrupts: enabled
 
-uart_suspend_port()
+**uart_suspend_port()**
 
-uart_resume_port()
+**uart_resume_port()**
 
-uart_add_one_port()
+**uart_add_one_port()**
 
-uart_remove_one_port()
+**uart_remove_one_port()**
 
 Other notes
 -----------
@@ -444,7 +507,7 @@ It is intended some day to drop the 'unused' entries from uart_port, and
 allow low level drivers to register their own individual uart_port's with
 the core.  This will allow drivers to use uart_port as a pointer to a
 structure containing both the uart_port entry with their own extensions,
-thus:
+thus::
 
 	struct my_port {
 		struct uart_port	port;
@@ -459,14 +522,14 @@ Some helpers are provided in order to set/get modem control lines via GPIO.
 mctrl_gpio_init(port, idx):
 	This will get the {cts,rts,...}-gpios from device tree if they are
 	present and request them, set direction etc, and return an
-	allocated structure. devm_* functions are used, so there's no need
+	allocated structure. `devm_*` functions are used, so there's no need
 	to call mctrl_gpio_free().
 	As this sets up the irq handling make sure to not handle changes to the
 	gpio input lines in your driver, too.
 
 mctrl_gpio_free(dev, gpios):
 	This will free the requested gpios in mctrl_gpio_init().
-	As devm_* functions are used, there's generally no need to call
+	As `devm_*` functions are used, there's generally no need to call
 	this function.
 
 mctrl_gpio_to_gpiod(gpios, gidx)
diff --git a/Documentation/serial/index.rst b/Documentation/serial/index.rst
new file mode 100644
index 000000000000..d0ba22ea23bf
--- /dev/null
+++ b/Documentation/serial/index.rst
@@ -0,0 +1,32 @@
+:orphan:
+
+==========================
+Support for Serial devices
+==========================
+
+.. toctree::
+    :maxdepth: 1
+
+
+    driver
+    tty
+
+Serial drivers
+==============
+
+.. toctree::
+    :maxdepth: 1
+
+    cyclades_z
+    moxa-smartio
+    n_gsm
+    rocket
+    serial-iso7816
+    serial-rs485
+
+.. only::  subproject and html
+
+   Indices
+   =======
+
+   * :ref:`genindex`
diff --git a/Documentation/serial/moxa-smartio b/Documentation/serial/moxa-smartio
deleted file mode 100644
index 5d2a33be0bd8..000000000000
--- a/Documentation/serial/moxa-smartio
+++ /dev/null
@@ -1,523 +0,0 @@
-=============================================================================
-          MOXA Smartio/Industio Family Device Driver Installation Guide
-		    for Linux Kernel 2.4.x, 2.6.x
-	       Copyright (C) 2008, Moxa Inc.
-=============================================================================
-Date: 01/21/2008
-
-Content
-
-1. Introduction
-2. System Requirement
-3. Installation
-   3.1 Hardware installation
-   3.2 Driver files
-   3.3 Device naming convention
-   3.4 Module driver configuration
-   3.5 Static driver configuration for Linux kernel 2.4.x and 2.6.x.
-   3.6 Custom configuration
-   3.7 Verify driver installation
-4. Utilities
-5. Setserial
-6. Troubleshooting
-
------------------------------------------------------------------------------
-1. Introduction
-
-   The Smartio/Industio/UPCI family Linux driver supports following multiport
-   boards.
-
-    - 2 ports multiport board
-	CP-102U, CP-102UL, CP-102UF
-	CP-132U-I, CP-132UL,
-	CP-132, CP-132I, CP132S, CP-132IS,
-	CI-132, CI-132I, CI-132IS,
-	(C102H, C102HI, C102HIS, C102P, CP-102, CP-102S)
-
-    - 4 ports multiport board
-	CP-104EL,
-	CP-104UL, CP-104JU,
-	CP-134U, CP-134U-I,
-	C104H/PCI, C104HS/PCI,
-	CP-114, CP-114I, CP-114S, CP-114IS, CP-114UL,
-	C104H, C104HS,
-	CI-104J, CI-104JS,
-	CI-134, CI-134I, CI-134IS,
-	(C114HI, CT-114I, C104P)
-	POS-104UL,
-	CB-114,
-	CB-134I
-
-    - 8 ports multiport board
-	CP-118EL, CP-168EL,
-	CP-118U, CP-168U,
-	C168H/PCI,
-	C168H, C168HS,
-	(C168P),
-	CB-108
-
-   This driver and installation procedure have been developed upon Linux Kernel
-   2.4.x and 2.6.x. This driver supports Intel x86 hardware platform. In order
-   to maintain compatibility, this version has also been properly tested with
-   RedHat, Mandrake, Fedora and S.u.S.E Linux. However, if compatibility problem
-   occurs, please contact Moxa at support@moxa.com.tw.
-
-   In addition to device driver, useful utilities are also provided in this
-   version. They are
-    - msdiag     Diagnostic program for displaying installed Moxa
-                 Smartio/Industio boards.
-    - msmon      Monitor program to observe data count and line status signals.
-    - msterm     A simple terminal program which is useful in testing serial
-	         ports.
-    - io-irq.exe Configuration program to setup ISA boards. Please note that
-                 this program can only be executed under DOS.
-
-   All the drivers and utilities are published in form of source code under
-   GNU General Public License in this version. Please refer to GNU General
-   Public License announcement in each source code file for more detail.
-
-   In Moxa's Web sites, you may always find latest driver at http://www.moxa.com/.
-
-   This version of driver can be installed as Loadable Module (Module driver)
-   or built-in into kernel (Static driver). You may refer to following
-   installation procedure for suitable one. Before you install the driver,
-   please refer to hardware installation procedure in the User's Manual.
-
-   We assume the user should be familiar with following documents.
-   - Serial-HOWTO
-   - Kernel-HOWTO
-
------------------------------------------------------------------------------
-2. System Requirement
-   - Hardware platform: Intel x86 machine
-   - Kernel version: 2.4.x or 2.6.x
-   - gcc version 2.72 or later
-   - Maximum 4 boards can be installed in combination
-
------------------------------------------------------------------------------
-3. Installation
-
-   3.1 Hardware installation
-   3.2 Driver files
-   3.3 Device naming convention
-   3.4 Module driver configuration
-   3.5 Static driver configuration for Linux kernel 2.4.x, 2.6.x.
-   3.6 Custom configuration
-   3.7 Verify driver installation
-
-
-   3.1 Hardware installation
-
-       There are two types of buses, ISA and PCI, for Smartio/Industio
-       family multiport board.
-
-       ISA board
-       ---------
-       You'll have to configure CAP address, I/O address, Interrupt Vector
-       as well as IRQ before installing this driver. Please refer to hardware
-       installation procedure in User's Manual before proceed any further.
-       Please make sure the JP1 is open after the ISA board is set properly.
-
-       PCI/UPCI board
-       --------------
-       You may need to adjust IRQ usage in BIOS to avoid from IRQ conflict
-       with other ISA devices. Please refer to hardware installation
-       procedure in User's Manual in advance.
-
-       PCI IRQ Sharing
-       -----------
-       Each port within the same multiport board shares the same IRQ. Up to
-       4 Moxa Smartio/Industio PCI Family multiport boards can be installed
-       together on one system and they can share the same IRQ.
-
-
-   3.2 Driver files
-
-       The driver file may be obtained from ftp, CD-ROM or floppy disk. The
-       first step, anyway, is to copy driver file "mxser.tgz" into specified
-       directory. e.g. /moxa. The execute commands as below.
-
-       # cd /
-       # mkdir moxa
-       # cd /moxa
-       # tar xvf /dev/fd0
-
-       or
-
-       # cd /
-       # mkdir moxa
-       # cd /moxa
-       # cp /mnt/cdrom/<driver directory>/mxser.tgz .
-       # tar xvfz mxser.tgz
-
-
-   3.3 Device naming convention
-
-       You may find all the driver and utilities files in /moxa/mxser.
-       Following installation procedure depends on the model you'd like to
-       run the driver. If you prefer module driver, please refer to 3.4.
-       If static driver is required, please refer to 3.5.
-
-       Dialin and callout port
-       -----------------------
-       This driver remains traditional serial device properties. There are
-       two special file name for each serial port. One is dial-in port
-       which is named "ttyMxx". For callout port, the naming convention
-       is "cumxx".
-
-       Device naming when more than 2 boards installed
-       -----------------------------------------------
-       Naming convention for each Smartio/Industio multiport board is
-       pre-defined as below.
-
-       Board Num.	 Dial-in Port	      Callout port
-       1st board	ttyM0  - ttyM7	      cum0  - cum7
-       2nd board	ttyM8  - ttyM15       cum8  - cum15
-       3rd board	ttyM16 - ttyM23       cum16 - cum23
-       4th board	ttyM24 - ttym31       cum24 - cum31
-
-
-       !!!!!!!!!!!!!!!!!!!! NOTE !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-       Under Kernel 2.6 the cum Device is Obsolete. So use ttyM*
-       device instead.
-       !!!!!!!!!!!!!!!!!!!! NOTE !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-
-       Board sequence
-       --------------
-       This driver will activate ISA boards according to the parameter set
-       in the driver. After all specified ISA board activated, PCI board
-       will be installed in the system automatically driven.
-       Therefore the board number is sorted by the CAP address of ISA boards.
-       For PCI boards, their sequence will be after ISA boards and C168H/PCI
-       has higher priority than C104H/PCI boards.
-
-   3.4 Module driver configuration
-       Module driver is easiest way to install. If you prefer static driver
-       installation, please skip this paragraph.
-
-
-       ------------- Prepare to use the MOXA driver--------------------
-       3.4.1 Create tty device with correct major number
-          Before using MOXA driver, your system must have the tty devices
-          which are created with driver's major number. We offer one shell
-          script "msmknod" to simplify the procedure.
-          This step is only needed to be executed once. But you still
-          need to do this procedure when:
-          a. You change the driver's major number. Please refer the "3.7"
-             section.
-          b. Your total installed MOXA boards number is changed. Maybe you
-             add/delete one MOXA board.
-          c. You want to change the tty name. This needs to modify the
-             shell script "msmknod"
-
-          The procedure is:
-	  # cd /moxa/mxser/driver
-	  # ./msmknod
-
-          This shell script will require the major number for dial-in
-          device and callout device to create tty device. You also need
-          to specify the total installed MOXA board number. Default major
-          numbers for dial-in device and callout device are 30, 35. If
-          you need to change to other number, please refer section "3.7"
-          for more detailed procedure.
-          Msmknod will delete any special files occupying the same device
-          naming.
-
-       3.4.2 Build the MOXA driver and utilities
-          Before using the MOXA driver and utilities, you need compile the
-          all the source code. This step is only need to be executed once.
-          But you still re-compile the source code if you modify the source
-          code. For example, if you change the driver's major number (see
-          "3.7" section), then you need to do this step again.
-
-          Find "Makefile" in /moxa/mxser, then run
-
-	  # make clean; make install
-
-          !!!!!!!!!! NOTE !!!!!!!!!!!!!!!!!
-	  For Red Hat 9, Red Hat Enterprise Linux AS3/ES3/WS3 & Fedora Core1:
-	  # make clean; make installsp1
-
-	  For Red Hat Enterprise Linux AS4/ES4/WS4:
-	  # make clean; make installsp2
-          !!!!!!!!!! NOTE !!!!!!!!!!!!!!!!!
-
-	  The driver files "mxser.o" and utilities will be properly compiled
-	  and copied to system directories respectively.
-
-       ------------- Load MOXA driver--------------------
-       3.4.3 Load the MOXA driver
-
-	  # modprobe mxser <argument>
-
-	  will activate the module driver. You may run "lsmod" to check
-	  if "mxser" is activated. If the MOXA board is ISA board, the
-          <argument> is needed. Please refer to section "3.4.5" for more
-          information.
-
-
-       ------------- Load MOXA driver on boot --------------------
-       3.4.4 For the above description, you may manually execute
-          "modprobe mxser" to activate this driver and run
-	  "rmmod mxser" to remove it.
-          However, it's better to have a boot time configuration to
-          eliminate manual operation. Boot time configuration can be
-          achieved by rc file. We offer one "rc.mxser" file to simplify
-          the procedure under "moxa/mxser/driver".
-
-          But if you use ISA board, please modify the "modprobe ..." command
-          to add the argument (see "3.4.5" section). After modifying the
-          rc.mxser, please try to execute "/moxa/mxser/driver/rc.mxser"
-          manually to make sure the modification is ok. If any error
-          encountered, please try to modify again. If the modification is
-          completed, follow the below step.
-
-	  Run following command for setting rc files.
-
-	  # cd /moxa/mxser/driver
-	  # cp ./rc.mxser /etc/rc.d
-	  # cd /etc/rc.d
-
-	  Check "rc.serial" is existed or not. If "rc.serial" doesn't exist,
-	  create it by vi, run "chmod 755 rc.serial" to change the permission.
-	  Add "/etc/rc.d/rc.mxser" in last line,
-
-          Reboot and check if moxa.o activated by "lsmod" command.
-
-       3.4.5. If you'd like to drive Smartio/Industio ISA boards in the system,
-          you'll have to add parameter to specify CAP address of given
-	  board while activating "mxser.o". The format for parameters are
-	  as follows.
-
-	  modprobe mxser ioaddr=0x???,0x???,0x???,0x???
-				|      |     |	  |
-				|      |     |	  +- 4th ISA board
-				|      |     +------ 3rd ISA board
-				|      +------------ 2nd ISA board
-				+------------------- 1st ISA board
-
-   3.5 Static driver configuration for Linux kernel 2.4.x and 2.6.x
-
-       Note: To use static driver, you must install the linux kernel
-             source package.
-
-       3.5.1 Backup the built-in driver in the kernel.
-          # cd /usr/src/linux/drivers/char
-          # mv mxser.c mxser.c.old
-
-          For Red Hat 7.x user, you need to create link:
-          # cd /usr/src
-          # ln -s linux-2.4 linux
-
-       3.5.2 Create link
-	  # cd /usr/src/linux/drivers/char
-	  # ln -s /moxa/mxser/driver/mxser.c mxser.c
-
-       3.5.3 Add CAP address list for ISA boards. For PCI boards user,
-          please skip this step.
-
-	  In module mode, the CAP address for ISA board is given by
-	  parameter. In static driver configuration, you'll have to
-	  assign it within driver's source code. If you will not
-	  install any ISA boards, you may skip to next portion.
-	  The instructions to modify driver source code are as
-	  below.
-	  a. # cd /moxa/mxser/driver
-	     # vi mxser.c
-	  b. Find the array mxserBoardCAP[] as below.
-
-	     static int mxserBoardCAP[]
-	     = {0x00, 0x00, 0x00, 0x00};
-
-	  c. Change the address within this array using vi. For
-	     example, to driver 2 ISA boards with CAP address
-	     0x280 and 0x180 as 1st and 2nd board. Just to change
-	     the source code as follows.
-
-	     static int mxserBoardCAP[]
-	     = {0x280, 0x180, 0x00, 0x00};
-
-       3.5.4 Setup kernel configuration
-
-          Configure the kernel:
-
-            # cd /usr/src/linux
-            # make menuconfig
-
-          You will go into a menu-driven system. Please select [Character
-          devices][Non-standard serial port support], enable the [Moxa
-          SmartIO support] driver with "[*]" for built-in (not "[M]"), then
-          select [Exit] to exit this program.
-
-       3.5.5 Rebuild kernel
-	  The following are for Linux kernel rebuilding, for your
-          reference only.
-	  For appropriate details, please refer to the Linux document.
-
-	   a. cd /usr/src/linux
-	   b. make clean	     /* take a few minutes */
-	   c. make dep		     /* take a few minutes */
-	   d. make bzImage	     /* take probably 10-20 minutes */
-	   e. make install	     /* copy boot image to correct position */
-	   f. Please make sure the boot kernel (vmlinuz) is in the
-	      correct position.
-	   g. If you use 'lilo' utility, you should check /etc/lilo.conf
-	      'image' item specified the path which is the 'vmlinuz' path,
-	      or you will load wrong (or old) boot kernel image (vmlinuz).
-	      After checking /etc/lilo.conf, please run "lilo".
-
-	  Note that if the result of "make bzImage" is ERROR, then you have to
-	  go back to Linux configuration Setup. Type "make menuconfig" in
-          directory /usr/src/linux.
-
-
-       3.5.6 Make tty device and special file
-          # cd /moxa/mxser/driver
-          # ./msmknod
-
-       3.5.7 Make utility
-	  # cd /moxa/mxser/utility
-	  # make clean; make install
-
-       3.5.8 Reboot
-
-
-
-   3.6 Custom configuration
-       Although this driver already provides you default configuration, you
-       still can change the device name and major number. The instruction to
-       change these parameters are shown as below.
-
-       Change Device name
-       ------------------
-       If you'd like to use other device names instead of default naming
-       convention, all you have to do is to modify the internal code
-       within the shell script "msmknod". First, you have to open "msmknod"
-       by vi. Locate each line contains "ttyM" and "cum" and change them
-       to the device name you desired. "msmknod" creates the device names
-       you need next time executed.
-
-       Change Major number
-       -------------------
-       If major number 30 and 35 had been occupied, you may have to select
-       2 free major numbers for this driver. There are 3 steps to change
-       major numbers.
-
-       3.6.1 Find free major numbers
-	  In /proc/devices, you may find all the major numbers occupied
-	  in the system. Please select 2 major numbers that are available.
-	  e.g. 40, 45.
-       3.6.2 Create special files
-	  Run /moxa/mxser/driver/msmknod to create special files with
-	  specified major numbers.
-       3.6.3 Modify driver with new major number
-	  Run vi to open /moxa/mxser/driver/mxser.c. Locate the line
-	  contains "MXSERMAJOR". Change the content as below.
-	  #define	  MXSERMAJOR		  40
-	  #define	  MXSERCUMAJOR		  45
-       3.6.4 Run "make clean; make install" in /moxa/mxser/driver.
-
-   3.7 Verify driver installation
-       You may refer to /var/log/messages to check the latest status
-       log reported by this driver whenever it's activated.
-
------------------------------------------------------------------------------
-4. Utilities
-   There are 3 utilities contained in this driver. They are msdiag, msmon and
-   msterm. These 3 utilities are released in form of source code. They should
-   be compiled into executable file and copied into /usr/bin.
-
-   Before using these utilities, please load driver (refer 3.4 & 3.5) and
-   make sure you had run the "msmknod" utility.
-
-   msdiag - Diagnostic
-   --------------------
-   This utility provides the function to display what Moxa Smartio/Industio
-   board found by driver in the system.
-
-   msmon - Port Monitoring
-   -----------------------
-   This utility gives the user a quick view about all the MOXA ports'
-   activities. One can easily learn each port's total received/transmitted
-   (Rx/Tx) character count since the time when the monitoring is started.
-   Rx/Tx throughputs per second are also reported in interval basis (e.g.
-   the last 5 seconds) and in average basis (since the time the monitoring
-   is started). You can reset all ports' count by <HOME> key. <+> <->
-   (plus/minus) keys to change the displaying time interval. Press <ENTER>
-   on the port, that cursor stay, to view the port's communication
-   parameters, signal status, and input/output queue.
-
-   msterm - Terminal Emulation
-   ---------------------------
-   This utility provides data sending and receiving ability of all tty ports,
-   especially for MOXA ports. It is quite useful for testing simple
-   application, for example, sending AT command to a modem connected to the
-   port or used as a terminal for login purpose. Note that this is only a
-   dumb terminal emulation without handling full screen operation.
-
------------------------------------------------------------------------------
-5. Setserial
-
-   Supported Setserial parameters are listed as below.
-
-   uart		  set UART type(16450-->disable FIFO, 16550A-->enable FIFO)
-   close_delay	  set the amount of time(in 1/100 of a second) that DTR
-		  should be kept low while being closed.
-   closing_wait   set the amount of time(in 1/100 of a second) that the
-		  serial port should wait for data to be drained while
-		  being closed, before the receiver is disable.
-   spd_hi	  Use  57.6kb  when  the application requests 38.4kb.
-   spd_vhi	  Use  115.2kb	when  the application requests 38.4kb.
-   spd_shi	  Use  230.4kb	when  the application requests 38.4kb.
-   spd_warp	  Use  460.8kb	when  the application requests 38.4kb.
-   spd_normal	  Use  38.4kb  when  the application requests 38.4kb.
-   spd_cust	  Use  the custom divisor to set the speed when  the
-		  application requests 38.4kb.
-   divisor	  This option set the custom division.
-   baud_base	  This option set the base baud rate.
-
------------------------------------------------------------------------------
-6. Troubleshooting
-
-   The boot time error messages and solutions are stated as clearly as
-   possible. If all the possible solutions fail, please contact our technical
-   support team to get more help.
-
-
-   Error msg: More than 4 Moxa Smartio/Industio family boards found. Fifth board
-              and after are ignored.
-   Solution:
-   To avoid this problem, please unplug fifth and after board, because Moxa
-   driver supports up to 4 boards.
-
-   Error msg: Request_irq fail, IRQ(?) may be conflict with another device.
-   Solution:
-   Other PCI or ISA devices occupy the assigned IRQ. If you are not sure
-   which device causes the situation, please check /proc/interrupts to find
-   free IRQ and simply change another free IRQ for Moxa board.
-
-   Error msg: Board #: C1xx Series(CAP=xxx) interrupt number invalid.
-   Solution:
-   Each port within the same multiport board shares the same IRQ. Please set
-   one IRQ (IRQ doesn't equal to zero) for one Moxa board.
-
-   Error msg: No interrupt vector be set for Moxa ISA board(CAP=xxx).
-   Solution:
-   Moxa ISA board needs an interrupt vector.Please refer to user's manual
-   "Hardware Installation" chapter to set interrupt vector.
-
-   Error msg: Couldn't install MOXA Smartio/Industio family driver!
-   Solution:
-   Load Moxa driver fail, the major number may conflict with other devices.
-   Please refer to previous section 3.7 to change a free major number for
-   Moxa driver.
-
-   Error msg: Couldn't install MOXA Smartio/Industio family callout driver!
-   Solution:
-   Load Moxa callout driver fail, the callout device major number may
-   conflict with other devices. Please refer to previous section 3.7 to
-   change a free callout device major number for Moxa driver.
-
-
------------------------------------------------------------------------------
-
diff --git a/Documentation/serial/moxa-smartio.rst b/Documentation/serial/moxa-smartio.rst
new file mode 100644
index 000000000000..156100f17c3f
--- /dev/null
+++ b/Documentation/serial/moxa-smartio.rst
@@ -0,0 +1,615 @@
+=============================================================
+MOXA Smartio/Industio Family Device Driver Installation Guide
+=============================================================
+
+.. note::
+
+   This file is outdated. It needs some care in order to make it
+   updated to Kernel 5.0 and upper
+
+Copyright (C) 2008, Moxa Inc.
+
+Date: 01/21/2008
+
+.. Content
+
+   1. Introduction
+   2. System Requirement
+   3. Installation
+      3.1 Hardware installation
+      3.2 Driver files
+      3.3 Device naming convention
+      3.4 Module driver configuration
+      3.5 Static driver configuration for Linux kernel 2.4.x and 2.6.x.
+      3.6 Custom configuration
+      3.7 Verify driver installation
+   4. Utilities
+   5. Setserial
+   6. Troubleshooting
+
+1. Introduction
+^^^^^^^^^^^^^^^
+
+   The Smartio/Industio/UPCI family Linux driver supports following multiport
+   boards.
+
+    - 2 ports multiport board
+	CP-102U, CP-102UL, CP-102UF
+	CP-132U-I, CP-132UL,
+	CP-132, CP-132I, CP132S, CP-132IS,
+	CI-132, CI-132I, CI-132IS,
+	(C102H, C102HI, C102HIS, C102P, CP-102, CP-102S)
+
+    - 4 ports multiport board
+	CP-104EL,
+	CP-104UL, CP-104JU,
+	CP-134U, CP-134U-I,
+	C104H/PCI, C104HS/PCI,
+	CP-114, CP-114I, CP-114S, CP-114IS, CP-114UL,
+	C104H, C104HS,
+	CI-104J, CI-104JS,
+	CI-134, CI-134I, CI-134IS,
+	(C114HI, CT-114I, C104P),
+	POS-104UL,
+	CB-114,
+	CB-134I
+
+    - 8 ports multiport board
+	CP-118EL, CP-168EL,
+	CP-118U, CP-168U,
+	C168H/PCI,
+	C168H, C168HS,
+	(C168P),
+	CB-108
+
+   This driver and installation procedure have been developed upon Linux Kernel
+   2.4.x and 2.6.x. This driver supports Intel x86 hardware platform. In order
+   to maintain compatibility, this version has also been properly tested with
+   RedHat, Mandrake, Fedora and S.u.S.E Linux. However, if compatibility problem
+   occurs, please contact Moxa at support@moxa.com.tw.
+
+   In addition to device driver, useful utilities are also provided in this
+   version. They are:
+
+    - msdiag
+		 Diagnostic program for displaying installed Moxa
+                 Smartio/Industio boards.
+    - msmon
+		 Monitor program to observe data count and line status signals.
+    - msterm     A simple terminal program which is useful in testing serial
+	         ports.
+    - io-irq.exe
+		 Configuration program to setup ISA boards. Please note that
+                 this program can only be executed under DOS.
+
+   All the drivers and utilities are published in form of source code under
+   GNU General Public License in this version. Please refer to GNU General
+   Public License announcement in each source code file for more detail.
+
+   In Moxa's Web sites, you may always find latest driver at http://www.moxa.com/.
+
+   This version of driver can be installed as Loadable Module (Module driver)
+   or built-in into kernel (Static driver). You may refer to following
+   installation procedure for suitable one. Before you install the driver,
+   please refer to hardware installation procedure in the User's Manual.
+
+   We assume the user should be familiar with following documents.
+
+   - Serial-HOWTO
+   - Kernel-HOWTO
+
+2. System Requirement
+^^^^^^^^^^^^^^^^^^^^^
+
+   - Hardware platform: Intel x86 machine
+   - Kernel version: 2.4.x or 2.6.x
+   - gcc version 2.72 or later
+   - Maximum 4 boards can be installed in combination
+
+3. Installation
+^^^^^^^^^^^^^^^
+
+3.1 Hardware installation
+=========================
+
+   There are two types of buses, ISA and PCI, for Smartio/Industio
+   family multiport board.
+
+ISA board
+---------
+
+   You'll have to configure CAP address, I/O address, Interrupt Vector
+   as well as IRQ before installing this driver. Please refer to hardware
+   installation procedure in User's Manual before proceed any further.
+   Please make sure the JP1 is open after the ISA board is set properly.
+
+PCI/UPCI board
+--------------
+
+   You may need to adjust IRQ usage in BIOS to avoid from IRQ conflict
+   with other ISA devices. Please refer to hardware installation
+   procedure in User's Manual in advance.
+
+PCI IRQ Sharing
+---------------
+
+   Each port within the same multiport board shares the same IRQ. Up to
+   4 Moxa Smartio/Industio PCI Family multiport boards can be installed
+   together on one system and they can share the same IRQ.
+
+
+3.2 Driver files
+================
+
+   The driver file may be obtained from ftp, CD-ROM or floppy disk. The
+   first step, anyway, is to copy driver file "mxser.tgz" into specified
+   directory. e.g. /moxa. The execute commands as below::
+
+       # cd /
+       # mkdir moxa
+       # cd /moxa
+       # tar xvf /dev/fd0
+
+or::
+
+       # cd /
+       # mkdir moxa
+       # cd /moxa
+       # cp /mnt/cdrom/<driver directory>/mxser.tgz .
+       # tar xvfz mxser.tgz
+
+
+3.3 Device naming convention
+============================
+
+   You may find all the driver and utilities files in /moxa/mxser.
+   Following installation procedure depends on the model you'd like to
+   run the driver. If you prefer module driver, please refer to 3.4.
+   If static driver is required, please refer to 3.5.
+
+Dialin and callout port
+-----------------------
+
+   This driver remains traditional serial device properties. There are
+   two special file name for each serial port. One is dial-in port
+   which is named "ttyMxx". For callout port, the naming convention
+   is "cumxx".
+
+Device naming when more than 2 boards installed
+-----------------------------------------------
+
+   Naming convention for each Smartio/Industio multiport board is
+   pre-defined as below.
+
+   ============ ===============       ==============
+   Board Num.	 Dial-in Port	      Callout port
+   1st board	ttyM0  - ttyM7	      cum0  - cum7
+   2nd board	ttyM8  - ttyM15       cum8  - cum15
+   3rd board	ttyM16 - ttyM23       cum16 - cum23
+   4th board	ttyM24 - ttym31       cum24 - cum31
+   ============ ===============       ==============
+
+.. note::
+
+   Under Kernel 2.6 and upper, the cum Device is Obsolete. So use ttyM*
+   device instead.
+
+Board sequence
+--------------
+
+   This driver will activate ISA boards according to the parameter set
+   in the driver. After all specified ISA board activated, PCI board
+   will be installed in the system automatically driven.
+   Therefore the board number is sorted by the CAP address of ISA boards.
+   For PCI boards, their sequence will be after ISA boards and C168H/PCI
+   has higher priority than C104H/PCI boards.
+
+3.4 Module driver configuration
+===============================
+
+   Module driver is easiest way to install. If you prefer static driver
+   installation, please skip this paragraph.
+
+
+   ------------- Prepare to use the MOXA driver --------------------
+
+3.4.1 Create tty device with correct major number
+-------------------------------------------------
+
+   Before using MOXA driver, your system must have the tty devices
+   which are created with driver's major number. We offer one shell
+   script "msmknod" to simplify the procedure.
+   This step is only needed to be executed once. But you still
+   need to do this procedure when:
+
+   a. You change the driver's major number. Please refer the "3.7"
+      section.
+   b. Your total installed MOXA boards number is changed. Maybe you
+      add/delete one MOXA board.
+   c. You want to change the tty name. This needs to modify the
+      shell script "msmknod"
+
+   The procedure is::
+
+	 # cd /moxa/mxser/driver
+	 # ./msmknod
+
+   This shell script will require the major number for dial-in
+   device and callout device to create tty device. You also need
+   to specify the total installed MOXA board number. Default major
+   numbers for dial-in device and callout device are 30, 35. If
+   you need to change to other number, please refer section "3.7"
+   for more detailed procedure.
+   Msmknod will delete any special files occupying the same device
+   naming.
+
+3.4.2 Build the MOXA driver and utilities
+-----------------------------------------
+
+   Before using the MOXA driver and utilities, you need compile the
+   all the source code. This step is only need to be executed once.
+   But you still re-compile the source code if you modify the source
+   code. For example, if you change the driver's major number (see
+   "3.7" section), then you need to do this step again.
+
+   Find "Makefile" in /moxa/mxser, then run
+
+	 # make clean; make install
+
+   ..note::
+
+	 For Red Hat 9, Red Hat Enterprise Linux AS3/ES3/WS3 & Fedora Core1:
+	 # make clean; make installsp1
+
+	 For Red Hat Enterprise Linux AS4/ES4/WS4:
+	 # make clean; make installsp2
+
+   The driver files "mxser.o" and utilities will be properly compiled
+   and copied to system directories respectively.
+
+------------- Load MOXA driver--------------------
+
+3.4.3 Load the MOXA driver
+--------------------------
+
+   ::
+
+	 # modprobe mxser <argument>
+
+   will activate the module driver. You may run "lsmod" to check
+   if "mxser" is activated. If the MOXA board is ISA board, the
+   <argument> is needed. Please refer to section "3.4.5" for more
+   information.
+
+------------- Load MOXA driver on boot --------------------
+
+3.4.4 Load the mxser driver
+---------------------------
+
+
+   For the above description, you may manually execute
+   "modprobe mxser" to activate this driver and run
+   "rmmod mxser" to remove it.
+
+   However, it's better to have a boot time configuration to
+   eliminate manual operation. Boot time configuration can be
+   achieved by rc file. We offer one "rc.mxser" file to simplify
+   the procedure under "moxa/mxser/driver".
+
+   But if you use ISA board, please modify the "modprobe ..." command
+   to add the argument (see "3.4.5" section). After modifying the
+   rc.mxser, please try to execute "/moxa/mxser/driver/rc.mxser"
+   manually to make sure the modification is ok. If any error
+   encountered, please try to modify again. If the modification is
+   completed, follow the below step.
+
+   Run following command for setting rc files::
+
+	 # cd /moxa/mxser/driver
+	 # cp ./rc.mxser /etc/rc.d
+	 # cd /etc/rc.d
+
+   Check "rc.serial" is existed or not. If "rc.serial" doesn't exist,
+   create it by vi, run "chmod 755 rc.serial" to change the permission.
+
+   Add "/etc/rc.d/rc.mxser" in last line.
+
+   Reboot and check if moxa.o activated by "lsmod" command.
+
+3.4.5. specify CAP address
+--------------------------
+
+   If you'd like to drive Smartio/Industio ISA boards in the system,
+   you'll have to add parameter to specify CAP address of given
+   board while activating "mxser.o". The format for parameters are
+   as follows.::
+
+	   modprobe mxser ioaddr=0x???,0x???,0x???,0x???
+				  |  |  |    |
+				  |  |  |    +- 4th ISA board
+				  |  |  +------ 3rd ISA board
+				  |  +------------ 2nd ISA board
+				  +-------------------1st ISA board
+
+3.5 Static driver configuration for Linux kernel 2.4.x and 2.6.x
+================================================================
+
+    Note:
+          To use static driver, you must install the linux kernel
+          source package.
+
+3.5.1 Backup the built-in driver in the kernel
+----------------------------------------------
+
+    ::
+
+       # cd /usr/src/linux/drivers/char
+       # mv mxser.c mxser.c.old
+
+       For Red Hat 7.x user, you need to create link:
+       # cd /usr/src
+       # ln -s linux-2.4 linux
+
+3.5.2 Create link
+-----------------
+    ::
+
+	  # cd /usr/src/linux/drivers/char
+	  # ln -s /moxa/mxser/driver/mxser.c mxser.c
+
+3.5.3 Add CAP address list for ISA boards.
+------------------------------------------
+
+    For PCI boards user, please skip this step.
+
+    In module mode, the CAP address for ISA board is given by
+    parameter. In static driver configuration, you'll have to
+    assign it within driver's source code. If you will not
+    install any ISA boards, you may skip to next portion.
+    The instructions to modify driver source code are as
+    below.
+
+    a. run::
+
+	# cd /moxa/mxser/driver
+	# vi mxser.c
+
+    b. Find the array mxserBoardCAP[] as below::
+
+	  static int mxserBoardCAP[] = {0x00, 0x00, 0x00, 0x00};
+
+    c. Change the address within this array using vi. For
+       example, to driver 2 ISA boards with CAP address
+       0x280 and 0x180 as 1st and 2nd board. Just to change
+       the source code as follows::
+
+	  static int mxserBoardCAP[] = {0x280, 0x180, 0x00, 0x00};
+
+3.5.4 Setup kernel configuration
+--------------------------------
+
+    Configure the kernel::
+
+      # cd /usr/src/linux
+      # make menuconfig
+
+    You will go into a menu-driven system. Please select [Character
+    devices][Non-standard serial port support], enable the [Moxa
+    SmartIO support] driver with "[*]" for built-in (not "[M]"), then
+    select [Exit] to exit this program.
+
+3.5.5 Rebuild kernel
+--------------------
+
+    The following are for Linux kernel rebuilding, for your
+    reference only.
+
+    For appropriate details, please refer to the Linux document:
+
+        a. Run the following commands::
+
+	     cd /usr/src/linux
+	     make clean		     # take a few minutes
+	     make dep		     # take a few minutes
+	     make bzImage	     # take probably 10-20 minutes
+	     make install	     # copy boot image to correct position
+
+	f. Please make sure the boot kernel (vmlinuz) is in the
+	   correct position.
+	g. If you use 'lilo' utility, you should check /etc/lilo.conf
+	   'image' item specified the path which is the 'vmlinuz' path,
+	   or you will load wrong (or old) boot kernel image (vmlinuz).
+	   After checking /etc/lilo.conf, please run "lilo".
+
+	  Note that if the result of "make bzImage" is ERROR, then you have to
+	  go back to Linux configuration Setup. Type "make menuconfig" in
+          directory /usr/src/linux.
+
+
+3.5.6 Make tty device and special file
+--------------------------------------
+
+    ::
+       # cd /moxa/mxser/driver
+       # ./msmknod
+
+3.5.7 Make utility
+------------------
+
+    ::
+
+	  # cd /moxa/mxser/utility
+	  # make clean; make install
+
+3.5.8 Reboot
+------------
+
+
+
+3.6 Custom configuration
+========================
+
+    Although this driver already provides you default configuration, you
+    still can change the device name and major number. The instruction to
+    change these parameters are shown as below.
+
+a. Change Device name
+
+    If you'd like to use other device names instead of default naming
+    convention, all you have to do is to modify the internal code
+    within the shell script "msmknod". First, you have to open "msmknod"
+    by vi. Locate each line contains "ttyM" and "cum" and change them
+    to the device name you desired. "msmknod" creates the device names
+    you need next time executed.
+
+b. Change Major number
+
+    If major number 30 and 35 had been occupied, you may have to select
+    2 free major numbers for this driver. There are 3 steps to change
+    major numbers.
+
+3.6.1 Find free major numbers
+-----------------------------
+
+    In /proc/devices, you may find all the major numbers occupied
+    in the system. Please select 2 major numbers that are available.
+    e.g. 40, 45.
+
+3.6.2 Create special files
+--------------------------
+
+   Run /moxa/mxser/driver/msmknod to create special files with
+   specified major numbers.
+
+3.6.3 Modify driver with new major number
+-----------------------------------------
+
+   Run vi to open /moxa/mxser/driver/mxser.c. Locate the line
+   contains "MXSERMAJOR". Change the content as below::
+
+	  #define	  MXSERMAJOR		  40
+	  #define	  MXSERCUMAJOR		  45
+
+    3.6.4 Run "make clean; make install" in /moxa/mxser/driver.
+
+3.7 Verify driver installation
+==============================
+
+    You may refer to /var/log/messages to check the latest status
+    log reported by this driver whenever it's activated.
+
+4. Utilities
+^^^^^^^^^^^^
+
+   There are 3 utilities contained in this driver. They are msdiag, msmon and
+   msterm. These 3 utilities are released in form of source code. They should
+   be compiled into executable file and copied into /usr/bin.
+
+   Before using these utilities, please load driver (refer 3.4 & 3.5) and
+   make sure you had run the "msmknod" utility.
+
+msdiag - Diagnostic
+===================
+
+   This utility provides the function to display what Moxa Smartio/Industio
+   board found by driver in the system.
+
+msmon - Port Monitoring
+=======================
+
+   This utility gives the user a quick view about all the MOXA ports'
+   activities. One can easily learn each port's total received/transmitted
+   (Rx/Tx) character count since the time when the monitoring is started.
+
+   Rx/Tx throughputs per second are also reported in interval basis (e.g.
+   the last 5 seconds) and in average basis (since the time the monitoring
+   is started). You can reset all ports' count by <HOME> key. <+> <->
+   (plus/minus) keys to change the displaying time interval. Press <ENTER>
+   on the port, that cursor stay, to view the port's communication
+   parameters, signal status, and input/output queue.
+
+msterm - Terminal Emulation
+===========================
+
+   This utility provides data sending and receiving ability of all tty ports,
+   especially for MOXA ports. It is quite useful for testing simple
+   application, for example, sending AT command to a modem connected to the
+   port or used as a terminal for login purpose. Note that this is only a
+   dumb terminal emulation without handling full screen operation.
+
+5. Setserial
+^^^^^^^^^^^^
+
+   Supported Setserial parameters are listed as below.
+
+   ============== =========================================================
+   uart		  set UART type(16450-->disable FIFO, 16550A-->enable FIFO)
+   close_delay	  set the amount of time(in 1/100 of a second) that DTR
+		  should be kept low while being closed.
+   closing_wait   set the amount of time(in 1/100 of a second) that the
+		  serial port should wait for data to be drained while
+		  being closed, before the receiver is disable.
+   spd_hi	  Use  57.6kb  when  the application requests 38.4kb.
+   spd_vhi	  Use  115.2kb	when  the application requests 38.4kb.
+   spd_shi	  Use  230.4kb	when  the application requests 38.4kb.
+   spd_warp	  Use  460.8kb	when  the application requests 38.4kb.
+   spd_normal	  Use  38.4kb  when  the application requests 38.4kb.
+   spd_cust	  Use  the custom divisor to set the speed when  the
+		  application requests 38.4kb.
+   divisor	  This option set the custom division.
+   baud_base	  This option set the base baud rate.
+   ============== =========================================================
+
+6. Troubleshooting
+^^^^^^^^^^^^^^^^^^
+
+   The boot time error messages and solutions are stated as clearly as
+   possible. If all the possible solutions fail, please contact our technical
+   support team to get more help.
+
+
+   Error msg:
+	      More than 4 Moxa Smartio/Industio family boards found. Fifth board
+              and after are ignored.
+
+   Solution:
+   To avoid this problem, please unplug fifth and after board, because Moxa
+   driver supports up to 4 boards.
+
+   Error msg:
+	      Request_irq fail, IRQ(?) may be conflict with another device.
+
+   Solution:
+   Other PCI or ISA devices occupy the assigned IRQ. If you are not sure
+   which device causes the situation, please check /proc/interrupts to find
+   free IRQ and simply change another free IRQ for Moxa board.
+
+   Error msg:
+	      Board #: C1xx Series(CAP=xxx) interrupt number invalid.
+
+   Solution:
+   Each port within the same multiport board shares the same IRQ. Please set
+   one IRQ (IRQ doesn't equal to zero) for one Moxa board.
+
+   Error msg:
+	      No interrupt vector be set for Moxa ISA board(CAP=xxx).
+
+   Solution:
+   Moxa ISA board needs an interrupt vector.Please refer to user's manual
+   "Hardware Installation" chapter to set interrupt vector.
+
+   Error msg:
+              Couldn't install MOXA Smartio/Industio family driver!
+
+   Solution:
+   Load Moxa driver fail, the major number may conflict with other devices.
+   Please refer to previous section 3.7 to change a free major number for
+   Moxa driver.
+
+   Error msg:
+              Couldn't install MOXA Smartio/Industio family callout driver!
+
+   Solution:
+   Load Moxa callout driver fail, the callout device major number may
+   conflict with other devices. Please refer to previous section 3.7 to
+   change a free callout device major number for Moxa driver.
diff --git a/Documentation/serial/n_gsm.rst b/Documentation/serial/n_gsm.rst
new file mode 100644
index 000000000000..f3ad9fd26408
--- /dev/null
+++ b/Documentation/serial/n_gsm.rst
@@ -0,0 +1,103 @@
+==============================
+GSM 0710 tty multiplexor HOWTO
+==============================
+
+This line discipline implements the GSM 07.10 multiplexing protocol
+detailed in the following 3GPP document:
+
+	http://www.3gpp.org/ftp/Specs/archive/07_series/07.10/0710-720.zip
+
+This document give some hints on how to use this driver with GPRS and 3G
+modems connected to a physical serial port.
+
+How to use it
+-------------
+1. initialize the modem in 0710 mux mode (usually AT+CMUX= command) through
+   its serial port. Depending on the modem used, you can pass more or less
+   parameters to this command,
+2. switch the serial line to using the n_gsm line discipline by using
+   TIOCSETD ioctl,
+3. configure the mux using GSMIOC_GETCONF / GSMIOC_SETCONF ioctl,
+
+Major parts of the initialization program :
+(a good starting point is util-linux-ng/sys-utils/ldattach.c)::
+
+  #include <linux/gsmmux.h>
+  #define N_GSM0710	21	/* GSM 0710 Mux */
+  #define DEFAULT_SPEED	B115200
+  #define SERIAL_PORT	/dev/ttyS0
+
+	int ldisc = N_GSM0710;
+	struct gsm_config c;
+	struct termios configuration;
+
+	/* open the serial port connected to the modem */
+	fd = open(SERIAL_PORT, O_RDWR | O_NOCTTY | O_NDELAY);
+
+	/* configure the serial port : speed, flow control ... */
+
+	/* send the AT commands to switch the modem to CMUX mode
+	   and check that it's successful (should return OK) */
+	write(fd, "AT+CMUX=0\r", 10);
+
+	/* experience showed that some modems need some time before
+	   being able to answer to the first MUX packet so a delay
+	   may be needed here in some case */
+	sleep(3);
+
+	/* use n_gsm line discipline */
+	ioctl(fd, TIOCSETD, &ldisc);
+
+	/* get n_gsm configuration */
+	ioctl(fd, GSMIOC_GETCONF, &c);
+	/* we are initiator and need encoding 0 (basic) */
+	c.initiator = 1;
+	c.encapsulation = 0;
+	/* our modem defaults to a maximum size of 127 bytes */
+	c.mru = 127;
+	c.mtu = 127;
+	/* set the new configuration */
+	ioctl(fd, GSMIOC_SETCONF, &c);
+
+	/* and wait for ever to keep the line discipline enabled */
+	daemon(0,0);
+	pause();
+
+4. create the devices corresponding to the "virtual" serial ports (take care,
+   each modem has its configuration and some DLC have dedicated functions,
+   for example GPS), starting with minor 1 (DLC0 is reserved for the management
+   of the mux)::
+
+     MAJOR=`cat /proc/devices |grep gsmtty | awk '{print $1}`
+     for i in `seq 1 4`; do
+	mknod /dev/ttygsm$i c $MAJOR $i
+     done
+
+5. use these devices as plain serial ports.
+
+   for example, it's possible:
+
+   - and to use gnokii to send / receive SMS on ttygsm1
+   - to use ppp to establish a datalink on ttygsm2
+
+6. first close all virtual ports before closing the physical port.
+
+   Note that after closing the physical port the modem is still in multiplexing
+   mode. This may prevent a successful re-opening of the port later. To avoid
+   this situation either reset the modem if your hardware allows that or send
+   a disconnect command frame manually before initializing the multiplexing mode
+   for the second time. The byte sequence for the disconnect command frame is::
+
+      0xf9, 0x03, 0xef, 0x03, 0xc3, 0x16, 0xf9.
+
+Additional Documentation
+------------------------
+More practical details on the protocol and how it's supported by industrial
+modems can be found in the following documents :
+
+- http://www.telit.com/module/infopool/download.php?id=616
+- http://www.u-blox.com/images/downloads/Product_Docs/LEON-G100-G200-MuxImplementation_ApplicationNote_%28GSM%20G1-CS-10002%29.pdf
+- http://www.sierrawireless.com/Support/Downloads/AirPrime/WMP_Series/~/media/Support_Downloads/AirPrime/Application_notes/CMUX_Feature_Application_Note-Rev004.ashx
+- http://wm.sim.com/sim/News/photo/2010721161442.pdf
+
+11-03-08 - Eric Bénard - <eric@eukrea.com>
diff --git a/Documentation/serial/n_gsm.txt b/Documentation/serial/n_gsm.txt
deleted file mode 100644
index 875361bb7cb4..000000000000
--- a/Documentation/serial/n_gsm.txt
+++ /dev/null
@@ -1,96 +0,0 @@
-n_gsm.c GSM 0710 tty multiplexor HOWTO
-===================================================
-
-This line discipline implements the GSM 07.10 multiplexing protocol
-detailed in the following 3GPP document :
-http://www.3gpp.org/ftp/Specs/archive/07_series/07.10/0710-720.zip
-
-This document give some hints on how to use this driver with GPRS and 3G
-modems connected to a physical serial port.
-
-How to use it
--------------
-1- initialize the modem in 0710 mux mode (usually AT+CMUX= command) through
-its serial port. Depending on the modem used, you can pass more or less
-parameters to this command,
-2- switch the serial line to using the n_gsm line discipline by using
-TIOCSETD ioctl,
-3- configure the mux using GSMIOC_GETCONF / GSMIOC_SETCONF ioctl,
-
-Major parts of the initialization program :
-(a good starting point is util-linux-ng/sys-utils/ldattach.c)
-#include <linux/gsmmux.h>
-#define N_GSM0710	21	/* GSM 0710 Mux */
-#define DEFAULT_SPEED	B115200
-#define SERIAL_PORT	/dev/ttyS0
-
-	int ldisc = N_GSM0710;
-	struct gsm_config c;
-	struct termios configuration;
-
-	/* open the serial port connected to the modem */
-	fd = open(SERIAL_PORT, O_RDWR | O_NOCTTY | O_NDELAY);
-
-	/* configure the serial port : speed, flow control ... */
-
-	/* send the AT commands to switch the modem to CMUX mode
-	   and check that it's successful (should return OK) */
-	write(fd, "AT+CMUX=0\r", 10);
-
-	/* experience showed that some modems need some time before
-	   being able to answer to the first MUX packet so a delay
-	   may be needed here in some case */
-	sleep(3);
-
-	/* use n_gsm line discipline */
-	ioctl(fd, TIOCSETD, &ldisc);
-
-	/* get n_gsm configuration */
-	ioctl(fd, GSMIOC_GETCONF, &c);
-	/* we are initiator and need encoding 0 (basic) */
-	c.initiator = 1;
-	c.encapsulation = 0;
-	/* our modem defaults to a maximum size of 127 bytes */
-	c.mru = 127;
-	c.mtu = 127;
-	/* set the new configuration */
-	ioctl(fd, GSMIOC_SETCONF, &c);
-
-	/* and wait for ever to keep the line discipline enabled */
-	daemon(0,0);
-	pause();
-
-4- create the devices corresponding to the "virtual" serial ports (take care,
-each modem has its configuration and some DLC have dedicated functions,
-for example GPS), starting with minor 1 (DLC0 is reserved for the management
-of the mux)
-
-MAJOR=`cat /proc/devices |grep gsmtty | awk '{print $1}`
-for i in `seq 1 4`; do
-	mknod /dev/ttygsm$i c $MAJOR $i
-done
-
-5- use these devices as plain serial ports.
-for example, it's possible :
-- and to use gnokii to send / receive SMS on ttygsm1
-- to use ppp to establish a datalink on ttygsm2
-
-6- first close all virtual ports before closing the physical port.
-
-Note that after closing the physical port the modem is still in multiplexing
-mode. This may prevent a successful re-opening of the port later. To avoid
-this situation either reset the modem if your hardware allows that or send
-a disconnect command frame manually before initializing the multiplexing mode
-for the second time. The byte sequence for the disconnect command frame is:
-0xf9, 0x03, 0xef, 0x03, 0xc3, 0x16, 0xf9.
-
-Additional Documentation
-------------------------
-More practical details on the protocol and how it's supported by industrial
-modems can be found in the following documents :
-http://www.telit.com/module/infopool/download.php?id=616
-http://www.u-blox.com/images/downloads/Product_Docs/LEON-G100-G200-MuxImplementation_ApplicationNote_%28GSM%20G1-CS-10002%29.pdf
-http://www.sierrawireless.com/Support/Downloads/AirPrime/WMP_Series/~/media/Support_Downloads/AirPrime/Application_notes/CMUX_Feature_Application_Note-Rev004.ashx
-http://wm.sim.com/sim/News/photo/2010721161442.pdf
-
-11-03-08 - Eric Bénard - <eric@eukrea.com>
diff --git a/Documentation/serial/rocket.txt b/Documentation/serial/rocket.rst
index 60b039891057..23761eae4282 100644
--- a/Documentation/serial/rocket.txt
+++ b/Documentation/serial/rocket.rst
@@ -1,20 +1,22 @@
-Comtrol(tm) RocketPort(R)/RocketModem(TM) Series 
-Device Driver for the Linux Operating System
+================================================
+Comtrol(tm) RocketPort(R)/RocketModem(TM) Series
+================================================
 
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+Device Driver for the Linux Operating System
+============================================
 
-PRODUCT OVERVIEW
+Product overview
 ----------------
 
 This driver provides a loadable kernel driver for the Comtrol RocketPort
-and RocketModem PCI boards. These boards provide, 2, 4, 8, 16, or 32 
+and RocketModem PCI boards. These boards provide, 2, 4, 8, 16, or 32
 high-speed serial ports or modems.  This driver supports up to a combination
 of four RocketPort or RocketModems boards in one machine simultaneously.
 This file assumes that you are using the RocketPort driver which is
-integrated into the kernel sources.  
+integrated into the kernel sources.
 
-The driver can also be installed as an external module using the usual 
-"make;make install" routine.  This external module driver, obtainable 
+The driver can also be installed as an external module using the usual
+"make;make install" routine.  This external module driver, obtainable
 from the Comtrol website listed below, is useful for updating the driver
 or installing it into kernels which do not have the driver configured
 into them.  Installations instructions for the external module
@@ -29,57 +31,59 @@ information on how to set the DIP switches.
 
 You pass the I/O port to the driver using the following module parameters:
 
-board1 :	I/O port for the first ISA board
-board2 :	I/O port for the second ISA board
-board3 :	I/O port for the third ISA board
-board4 :	I/O port for the fourth ISA board
+board1:
+	I/O port for the first ISA board
+board2:
+	I/O port for the second ISA board
+board3:
+	I/O port for the third ISA board
+board4:
+	I/O port for the fourth ISA board
 
 There is a set of utilities and scripts provided with the external driver
-( downloadable from http://www.comtrol.com ) that ease the configuration and
+(downloadable from http://www.comtrol.com) that ease the configuration and
 setup of the ISA cards.
 
 The RocketModem II PCI boards require firmware to be loaded into the card
 before it will function.  The driver has only been tested as a module for this
 board.
 
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
-
-INSTALLATION PROCEDURES
+Installation Procedures
 -----------------------
 
-RocketPort/RocketModem PCI cards require no driver configuration, they are 
+RocketPort/RocketModem PCI cards require no driver configuration, they are
 automatically detected and configured.
 
-The RocketPort driver can be installed as a module (recommended) or built 
+The RocketPort driver can be installed as a module (recommended) or built
 into the kernel. This is selected, as for other drivers, through the `make config`
-command from the root of the Linux source tree during the kernel build process. 
+command from the root of the Linux source tree during the kernel build process.
 
 The RocketPort/RocketModem serial ports installed by this driver are assigned
-device major number 46, and will be named /dev/ttyRx, where x is the port number 
+device major number 46, and will be named /dev/ttyRx, where x is the port number
 starting at zero (ex. /dev/ttyR0, /devttyR1, ...).  If you have multiple cards
 installed in the system, the mapping of port names to serial ports is displayed
 in the system log at /var/log/messages.
 
 If installed as a module, the module must be loaded.  This can be done
 manually by entering "modprobe rocket".  To have the module loaded automatically
-upon system boot, edit a /etc/modprobe.d/*.conf file and add the line
+upon system boot, edit a `/etc/modprobe.d/*.conf` file and add the line
 "alias char-major-46 rocket".
 
 In order to use the ports, their device names (nodes) must be created with mknod.
-This is only required once, the system will retain the names once created.  To 
-create the RocketPort/RocketModem device names, use the command 
-"mknod /dev/ttyRx c 46 x" where x is the port number starting at zero.  For example:
+This is only required once, the system will retain the names once created.  To
+create the RocketPort/RocketModem device names, use the command
+"mknod /dev/ttyRx c 46 x" where x is the port number starting at zero.
 
->mknod /dev/ttyR0 c 46 0
->mknod /dev/ttyR1 c 46 1
->mknod /dev/ttyR2 c 46 2  
+For example::
 
-The Linux script MAKEDEV will create the first 16 ttyRx device names (nodes)
-for you:
+	> mknod /dev/ttyR0 c 46 0
+	> mknod /dev/ttyR1 c 46 1
+	> mknod /dev/ttyR2 c 46 2
 
->/dev/MAKEDEV ttyR
+The Linux script MAKEDEV will create the first 16 ttyRx device names (nodes)
+for you::
 
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
+	>/dev/MAKEDEV ttyR
 
 ISA Rocketport Boards
 ---------------------
@@ -89,7 +93,7 @@ card before installing and using it.  This is done by setting a set of DIP
 switches on the Rocketport board.
 
 
-SETTING THE I/O ADDRESS
+Setting the I/O address
 -----------------------
 
 Before installing RocketPort(R) or RocketPort RA boards, you must find
@@ -130,40 +134,36 @@ the first 4 bytes of that range are used by the first board.  You would
 need to set the second, third, or fourth board to one of the next available
 blocks such as 0x180.
 
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
-
-RocketPort and RocketPort RA SW1 Settings:
-
-          +-------------------------------+
-          | 8 | 7 | 6 | 5 | 4 | 3 | 2 | 1 |
-          +-------+-------+---------------+
-          | Unused| Card  | I/O Port Block|
-          +-------------------------------+
-
-DIP Switches                             DIP Switches
-7    8                                   6    5
-===================                      ===================
-On   On   UNUSED, MUST BE ON.            On   On   First Card    <==== Default
-                                         On   Off  Second Card
-                                         Off  On   Third Card
-                                         Off  Off  Fourth Card
-
-DIP Switches         I/O Address Range
-4    3    2    1     Used by the First Card
-=====================================
-On   Off  On   Off   100-143
-On   Off  Off  On    140-183
-On   Off  Off  Off   180-1C3       <==== Default
-Off  On   On   Off   200-243
-Off  On   Off  On    240-283
-Off  On   Off  Off   280-2C3
-Off  Off  On   Off   300-343
-Off  Off  Off  On    340-383
-Off  Off  Off  Off   380-3C3
-
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
-
-REPORTING BUGS
+RocketPort and RocketPort RA SW1 Settings::
+
+            +-------------------------------+
+            | 8 | 7 | 6 | 5 | 4 | 3 | 2 | 1 |
+            +-------+-------+---------------+
+            | Unused| Card  | I/O Port Block|
+            +-------------------------------+
+
+  DIP Switches                             DIP Switches
+  7    8                                   6    5
+  ===================                      ===================
+  On   On   UNUSED, MUST BE ON.            On   On   First Card    <==== Default
+                                           On   Off  Second Card
+                                           Off  On   Third Card
+                                           Off  Off  Fourth Card
+
+  DIP Switches         I/O Address Range
+  4    3    2    1     Used by the First Card
+  =====================================
+  On   Off  On   Off   100-143
+  On   Off  Off  On    140-183
+  On   Off  Off  Off   180-1C3       <==== Default
+  Off  On   On   Off   200-243
+  Off  On   Off  On    240-283
+  Off  On   Off  Off   280-2C3
+  Off  Off  On   Off   300-343
+  Off  Off  Off  On    340-383
+  Off  Off  Off  Off   380-3C3
+
+Reporting Bugs
 --------------
 
 For technical support, please provide the following
@@ -171,19 +171,15 @@ information: Driver version, kernel release, distribution of
 kernel, and type of board you are using. Error messages and log
 printouts port configuration details are especially helpful.
 
-USA
-    Phone: (612) 494-4100
-      FAX: (612) 494-4199
-    email: support@comtrol.com
+USA:
+    :Phone: (612) 494-4100
+    :FAX: (612) 494-4199
+    :email: support@comtrol.com
 
-Comtrol Europe
-    Phone: +44 (0) 1 869 323-220
-      FAX: +44 (0) 1 869 323-211
-    email: support@comtrol.co.uk
+Comtrol Europe:
+    :Phone: +44 (0) 1 869 323-220
+    :FAX: +44 (0) 1 869 323-211
+    :email: support@comtrol.co.uk
 
 Web:	http://www.comtrol.com
 FTP:	ftp.comtrol.com
-
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
-
-
diff --git a/Documentation/serial/serial-iso7816.txt b/Documentation/serial/serial-iso7816.rst
index 3193d24a2b0f..d990143de0c6 100644
--- a/Documentation/serial/serial-iso7816.txt
+++ b/Documentation/serial/serial-iso7816.rst
@@ -1,11 +1,15 @@
-                        ISO7816 SERIAL COMMUNICATIONS
+=============================
+ISO7816 Serial Communications
+=============================
 
-1. INTRODUCTION
+1. Introduction
+===============
 
   ISO/IEC7816 is a series of standards specifying integrated circuit cards (ICC)
   also known as smart cards.
 
-2. HARDWARE-RELATED CONSIDERATIONS
+2. Hardware-related considerations
+==================================
 
   Some CPUs/UARTs (e.g., Microchip AT91) contain a built-in mode capable of
   handling communication with a smart card.
@@ -15,7 +19,8 @@
   available at user-level to allow switching from one mode to the other, and
   vice versa.
 
-3. DATA STRUCTURES ALREADY AVAILABLE IN THE KERNEL
+3. Data Structures Already Available in the Kernel
+==================================================
 
   The Linux kernel provides the serial_iso7816 structure (see [1]) to handle
   ISO7816 communications. This data structure is used to set and configure
@@ -27,10 +32,11 @@
   to TIOCGISO7816 and TIOCSISO7816 ioctls (see below). The iso7816_config
   callback receives a pointer to struct serial_iso7816.
 
-4. USAGE FROM USER-LEVEL
+4. Usage from user-level
+========================
 
   From user-level, ISO7816 configuration can be get/set using the previous
-  ioctls. For instance, to set ISO7816 you can use the following code:
+  ioctls. For instance, to set ISO7816 you can use the following code::
 
 	#include <linux/serial.h>
 
@@ -78,6 +84,7 @@
 		/* Error handling. See errno. */
 	}
 
-5. REFERENCES
+5. References
+=============
 
  [1]    include/uapi/linux/serial.h
diff --git a/Documentation/serial/serial-rs485.txt b/Documentation/serial/serial-rs485.rst
index ce0c1a9b8aab..6bc824f948f9 100644
--- a/Documentation/serial/serial-rs485.txt
+++ b/Documentation/serial/serial-rs485.rst
@@ -1,6 +1,9 @@
-                        RS485 SERIAL COMMUNICATIONS
+===========================
+RS485 Serial Communications
+===========================
 
-1. INTRODUCTION
+1. Introduction
+===============
 
    EIA-485, also known as TIA/EIA-485 or RS-485, is a standard defining the
    electrical characteristics of drivers and receivers for use in balanced
@@ -9,7 +12,8 @@
    because it can be used effectively over long distances and in electrically
    noisy environments.
 
-2. HARDWARE-RELATED CONSIDERATIONS
+2. Hardware-related Considerations
+==================================
 
    Some CPUs/UARTs (e.g., Atmel AT91 or 16C950 UART) contain a built-in
    half-duplex mode capable of automatically controlling line direction by
@@ -22,7 +26,8 @@
    available at user-level to allow switching from one mode to the other, and
    vice versa.
 
-3. DATA STRUCTURES ALREADY AVAILABLE IN THE KERNEL
+3. Data Structures Already Available in the Kernel
+==================================================
 
    The Linux kernel provides the serial_rs485 structure (see [1]) to handle
    RS485 communications. This data structure is used to set and configure RS485
@@ -38,10 +43,11 @@
    to TIOCSRS485 and TIOCGRS485 ioctls (see below). The rs485_config callback
    receives a pointer to struct serial_rs485.
 
-4. USAGE FROM USER-LEVEL
+4. Usage from user-level
+========================
 
    From user-level, RS485 configuration can be get/set using the previous
-   ioctls. For instance, to set RS485 you can use the following code:
+   ioctls. For instance, to set RS485 you can use the following code::
 
 	#include <linux/serial.h>
 
@@ -89,7 +95,9 @@
 		/* Error handling. See errno. */
 	}
 
-5. REFERENCES
+5. References
+=============
 
  [1]	include/uapi/linux/serial.h
+
  [2]	Documentation/devicetree/bindings/serial/rs485.txt
diff --git a/Documentation/serial/tty.txt b/Documentation/serial/tty.rst
index b48780977a68..dd972caacf3e 100644
--- a/Documentation/serial/tty.txt
+++ b/Documentation/serial/tty.rst
@@ -1,5 +1,6 @@
-
-			The Lockronomicon
+=================
+The Lockronomicon
+=================
 
 Your guide to the ancient and twisted locking policies of the tty layer and
 the warped logic behind them. Beware all ye who read on.
@@ -9,12 +10,12 @@ Line Discipline
 ---------------
 
 Line disciplines are registered with tty_register_ldisc() passing the
-discipline number and the ldisc structure. At the point of registration the 
+discipline number and the ldisc structure. At the point of registration the
 discipline must be ready to use and it is possible it will get used before
 the call returns success. If the call returns an error then it won't get
 called. Do not re-use ldisc numbers as they are part of the userspace ABI
 and writing over an existing ldisc will cause demons to eat your computer.
-After the return the ldisc data has been copied so you may free your own 
+After the return the ldisc data has been copied so you may free your own
 copy of the structure. You must not re-register over the top of the line
 discipline even with the same data or your computer again will be eaten by
 demons.
@@ -26,7 +27,7 @@ code manages the module counts this should not usually be a concern.
 
 Heed this warning: the reference count field of the registered copies of the
 tty_ldisc structure in the ldisc table counts the number of lines using this
-discipline. The reference count of the tty_ldisc structure within a tty 
+discipline. The reference count of the tty_ldisc structure within a tty
 counts the number of active users of the ldisc at this instant. In effect it
 counts the number of threads of execution within an ldisc method (plus those
 about to enter and exit although this detail matters not).
@@ -34,9 +35,11 @@ about to enter and exit although this detail matters not).
 Line Discipline Methods
 -----------------------
 
-TTY side interfaces:
+TTY side interfaces
+^^^^^^^^^^^^^^^^^^^
 
-open()		-	Called when the line discipline is attached to
+======================= =======================================================
+open()			Called when the line discipline is attached to
 			the terminal. No other call into the line
 			discipline for this tty will occur until it
 			completes successfully. Should initialize any
@@ -47,66 +50,69 @@ open()		-	Called when the line discipline is attached to
 			Returning an error will prevent the ldisc from
 			being attached. Can sleep.
 
-close()		-	This is called on a terminal when the line
+close()			This is called on a terminal when the line
 			discipline is being unplugged. At the point of
 			execution no further users will enter the
 			ldisc code for this tty. Can sleep.
 
-hangup()	-	Called when the tty line is hung up.
+hangup()		Called when the tty line is hung up.
 			The line discipline should cease I/O to the tty.
 			No further calls into the ldisc code will occur.
 			The return value is ignored. Can sleep.
 
-read()		-	(optional) A process requests reading data from
+read()			(optional) A process requests reading data from
 			the line. Multiple read calls may occur in parallel
 			and the ldisc must deal with serialization issues.
 			If not defined, the process will receive an EIO
 			error. May sleep.
 
-write()		-	(optional) A process requests writing data to the
+write()			(optional) A process requests writing data to the
 			line. Multiple write calls are serialized by the
 			tty layer for the ldisc. If not defined, the
 			process will receive an EIO error. May sleep.
 
-flush_buffer()	-	(optional) May be called at any point between
+flush_buffer()		(optional) May be called at any point between
 			open and close, and instructs the line discipline
 			to empty its input buffer.
 
-set_termios()	-	(optional) Called on termios structure changes.
+set_termios()		(optional) Called on termios structure changes.
 			The caller passes the old termios data and the
 			current data is in the tty. Called under the
 			termios semaphore so allowed to sleep. Serialized
 			against itself only.
 
-poll()		-	(optional) Check the status for the poll/select
+poll()			(optional) Check the status for the poll/select
 			calls. Multiple poll calls may occur in parallel.
 			May sleep.
 
-ioctl()		-	(optional) Called when an ioctl is handed to the
+ioctl()			(optional) Called when an ioctl is handed to the
 			tty layer that might be for the ldisc. Multiple
 			ioctl calls may occur in parallel. May sleep.
 
-compat_ioctl()	-	(optional) Called when a 32 bit ioctl is handed
+compat_ioctl()		(optional) Called when a 32 bit ioctl is handed
 			to the tty layer that might be for the ldisc.
 			Multiple ioctl calls may occur in parallel.
 			May sleep.
+======================= =======================================================
 
-Driver Side Interfaces:
+Driver Side Interfaces
+^^^^^^^^^^^^^^^^^^^^^^
 
-receive_buf()	-	(optional) Called by the low-level driver to hand
+======================= =======================================================
+receive_buf()		(optional) Called by the low-level driver to hand
 			a buffer of received bytes to the ldisc for
 			processing. The number of bytes is guaranteed not
 			to exceed the current value of tty->receive_room.
 			All bytes must be processed.
 
-receive_buf2()	-	(optional) Called by the low-level driver to hand
+receive_buf2()		(optional) Called by the low-level driver to hand
 			a buffer of received bytes to the ldisc for
 			processing. Returns the number of bytes processed.
 
 			If both receive_buf() and receive_buf2() are
 			defined, receive_buf2() should be preferred.
 
-write_wakeup()	-	May be called at any point between open and close.
+write_wakeup()		May be called at any point between open and close.
 			The TTY_DO_WRITE_WAKEUP flag indicates if a call
 			is needed but always races versus calls. Thus the
 			ldisc must be careful about setting order and to
@@ -117,17 +123,20 @@ write_wakeup()	-	May be called at any point between open and close.
 			is permitted to call the driver write method from
 			this function. In such a situation defer it.
 
-dcd_change()	-	Report to the tty line the current DCD pin status
+dcd_change()		Report to the tty line the current DCD pin status
 			changes and the relative timestamp. The timestamp
 			cannot be NULL.
+======================= =======================================================
 
 
 Driver Access
+^^^^^^^^^^^^^
 
 Line discipline methods can call the following methods of the underlying
 hardware driver through the function pointers within the tty->driver
 structure:
 
+======================= =======================================================
 write()			Write a block of characters to the tty device.
 			Returns the number of characters accepted. The
 			character buffer passed to this method is already
@@ -189,13 +198,16 @@ wait_until_sent()	Waits until the device has written out all of the
 			characters in its transmitter FIFO.
 
 send_xchar()		Send a high-priority XON/XOFF character to the device.
+======================= =======================================================
 
 
 Flags
+^^^^^
 
 Line discipline methods have access to tty->flags field containing the
 following interesting flags:
 
+======================= =======================================================
 TTY_THROTTLED		Driver input is throttled. The ldisc should call
 			tty->driver->unthrottle() in order to resume
 			reception when it is ready to process more data.
@@ -212,102 +224,105 @@ TTY_OTHER_CLOSED	Device is a pty and the other side has closed.
 
 TTY_NO_WRITE_SPLIT	Prevent driver from splitting up writes into
 			smaller chunks.
+======================= =======================================================
 
 
 Locking
+^^^^^^^
 
 Callers to the line discipline functions from the tty layer are required to
 take line discipline locks. The same is true of calls from the driver side
 but not yet enforced.
 
-Three calls are now provided
+Three calls are now provided::
 
 	ldisc = tty_ldisc_ref(tty);
 
 takes a handle to the line discipline in the tty and returns it. If no ldisc
 is currently attached or the ldisc is being closed and re-opened at this
 point then NULL is returned. While this handle is held the ldisc will not
-change or go away.
+change or go away::
 
 	tty_ldisc_deref(ldisc)
 
 Returns the ldisc reference and allows the ldisc to be closed. Returning the
 reference takes away your right to call the ldisc functions until you take
-a new reference.
+a new reference::
 
 	ldisc = tty_ldisc_ref_wait(tty);
 
 Performs the same function as tty_ldisc_ref except that it will wait for an
-ldisc change to complete and then return a reference to the new ldisc. 
+ldisc change to complete and then return a reference to the new ldisc.
 
 While these functions are slightly slower than the old code they should have
 minimal impact as most receive logic uses the flip buffers and they only
 need to take a reference when they push bits up through the driver.
 
-A caution: The ldisc->open(), ldisc->close() and driver->set_ldisc 
+A caution: The ldisc->open(), ldisc->close() and driver->set_ldisc
 functions are called with the ldisc unavailable. Thus tty_ldisc_ref will
 fail in this situation if used within these functions. Ldisc and driver
-code calling its own functions must be careful in this case. 
+code calling its own functions must be careful in this case.
 
 
 Driver Interface
 ----------------
 
-open()		-	Called when a device is opened. May sleep
+======================= =======================================================
+open()			Called when a device is opened. May sleep
 
-close()		-	Called when a device is closed. At the point of
-			return from this call the driver must make no 
+close()			Called when a device is closed. At the point of
+			return from this call the driver must make no
 			further ldisc calls of any kind. May sleep
 
-write()		-	Called to write bytes to the device. May not
-			sleep. May occur in parallel in special cases. 
+write()			Called to write bytes to the device. May not
+			sleep. May occur in parallel in special cases.
 			Because this includes panic paths drivers generally
 			shouldn't try and do clever locking here.
 
-put_char()	-	Stuff a single character onto the queue. The
+put_char()		Stuff a single character onto the queue. The
 			driver is guaranteed following up calls to
 			flush_chars.
 
-flush_chars()	-	Ask the kernel to write put_char queue
+flush_chars()		Ask the kernel to write put_char queue
 
-write_room()	-	Return the number of characters that can be stuffed
+write_room()		Return the number of characters that can be stuffed
 			into the port buffers without overflow (or less).
 			The ldisc is responsible for being intelligent
- 			about multi-threading of write_room/write calls
+			about multi-threading of write_room/write calls
 
-ioctl()		-	Called when an ioctl may be for the driver
+ioctl()			Called when an ioctl may be for the driver
 
-set_termios()	-	Called on termios change, serialized against
+set_termios()		Called on termios change, serialized against
 			itself by a semaphore. May sleep.
 
-set_ldisc()	-	Notifier for discipline change. At the point this 
+set_ldisc()		Notifier for discipline change. At the point this
 			is done the discipline is not yet usable. Can now
 			sleep (I think)
 
-throttle()	-	Called by the ldisc to ask the driver to do flow
+throttle()		Called by the ldisc to ask the driver to do flow
 			control.  Serialization including with unthrottle
 			is the job of the ldisc layer.
 
-unthrottle()	-	Called by the ldisc to ask the driver to stop flow
+unthrottle()		Called by the ldisc to ask the driver to stop flow
 			control.
 
-stop()		-	Ldisc notifier to the driver to stop output. As with
+stop()			Ldisc notifier to the driver to stop output. As with
 			throttle the serializations with start() are down
 			to the ldisc layer.
 
-start()		-	Ldisc notifier to the driver to start output.
+start()			Ldisc notifier to the driver to start output.
 
-hangup()	-	Ask the tty driver to cause a hangup initiated
+hangup()		Ask the tty driver to cause a hangup initiated
 			from the host side. [Can sleep ??]
 
-break_ctl()	-	Send RS232 break. Can sleep. Can get called in
+break_ctl()		Send RS232 break. Can sleep. Can get called in
 			parallel, driver must serialize (for now), and
 			with write calls.
 
-wait_until_sent() -	Wait for characters to exit the hardware queue
+wait_until_sent()	Wait for characters to exit the hardware queue
 			of the driver. Can sleep
 
-send_xchar()	  -	Send XON/XOFF and if possible jump the queue with
+send_xchar()	  	Send XON/XOFF and if possible jump the queue with
 			it in order to get fast flow control responses.
 			Cannot sleep ??
-
+======================= =======================================================
diff --git a/Documentation/sound/kernel-api/writing-an-alsa-driver.rst b/Documentation/sound/kernel-api/writing-an-alsa-driver.rst
index 6b154dbb02cc..132f5eb9b530 100644
--- a/Documentation/sound/kernel-api/writing-an-alsa-driver.rst
+++ b/Documentation/sound/kernel-api/writing-an-alsa-driver.rst
@@ -324,7 +324,7 @@ to details explained in the following section.
               strcpy(card->driver, "My Chip");
               strcpy(card->shortname, "My Own Chip 123");
               sprintf(card->longname, "%s at 0x%lx irq %i",
-                      card->shortname, chip->ioport, chip->irq);
+                      card->shortname, chip->port, chip->irq);
 
               /* (5) */
               .... /* implemented later */
@@ -437,7 +437,7 @@ Since each component can be properly freed, the single
   strcpy(card->driver, "My Chip");
   strcpy(card->shortname, "My Own Chip 123");
   sprintf(card->longname, "%s at 0x%lx irq %i",
-          card->shortname, chip->ioport, chip->irq);
+          card->shortname, chip->port, chip->irq);
 
 The driver field holds the minimal ID string of the chip. This is used
 by alsa-lib's configurator, so keep it simple but unique. Even the
diff --git a/Documentation/sparc/adi.txt b/Documentation/sparc/adi.rst
index e1aed155fb89..857ad30f9569 100644
--- a/Documentation/sparc/adi.txt
+++ b/Documentation/sparc/adi.rst
@@ -1,3 +1,4 @@
+================================
 Application Data Integrity (ADI)
 ================================
 
@@ -44,12 +45,15 @@ provided by the hypervisor to the kernel.  Kernel returns the value of
 ADI block size to userspace using auxiliary vector along with other ADI
 info. Following auxiliary vectors are provided by the kernel:
 
+	============	===========================================
 	AT_ADI_BLKSZ	ADI block size. This is the granularity and
 			alignment, in bytes, of ADI versioning.
 	AT_ADI_NBITS	Number of ADI version bits in the VA
+	============	===========================================
 
 
-IMPORTANT NOTES:
+IMPORTANT NOTES
+===============
 
 - Version tag values of 0x0 and 0xf are reserved. These values match any
   tag in virtual address and never generate a mismatch exception.
@@ -86,11 +90,12 @@ IMPORTANT NOTES:
 
 
 ADI related traps
------------------
+=================
 
 With ADI enabled, following new traps may occur:
 
 Disrupting memory corruption
+----------------------------
 
 	When a store accesses a memory localtion that has TTE.mcd=1,
 	the task is running with ADI enabled (PSTATE.mcde=1), and the ADI
@@ -100,7 +105,7 @@ Disrupting memory corruption
 	first. Hypervisor creates a sun4v error report and sends a
 	resumable error (TT=0x7e) trap to the kernel. The kernel sends
 	a SIGSEGV to the task that resulted in this trap with the following
-	info:
+	info::
 
 		siginfo.si_signo = SIGSEGV;
 		siginfo.errno = 0;
@@ -110,6 +115,7 @@ Disrupting memory corruption
 
 
 Precise memory corruption
+-------------------------
 
 	When a store accesses a memory location that has TTE.mcd=1,
 	the task is running with ADI enabled (PSTATE.mcde=1), and the ADI
@@ -118,7 +124,7 @@ Precise memory corruption
 	MCD precise exception is enabled (MCDPERR=1), a precise
 	exception is sent to the kernel with TT=0x1a. The kernel sends
 	a SIGSEGV to the task that resulted in this trap with the following
-	info:
+	info::
 
 		siginfo.si_signo = SIGSEGV;
 		siginfo.errno = 0;
@@ -126,17 +132,19 @@ Precise memory corruption
 		siginfo.si_addr = addr;	/* address that caused trap */
 		siginfo.si_trapno = 0;
 
-	NOTE: ADI tag mismatch on a load always results in precise trap.
+	NOTE:
+		ADI tag mismatch on a load always results in precise trap.
 
 
 MCD disabled
+------------
 
 	When a task has not enabled ADI and attempts to set ADI version
 	on a memory address, processor sends an MCD disabled trap. This
 	trap is handled by hypervisor first and the hypervisor vectors this
 	trap through to the kernel as Data Access Exception trap with
 	fault type set to 0xa (invalid ASI). When this occurs, the kernel
-	sends the task SIGSEGV signal with following info:
+	sends the task SIGSEGV signal with following info::
 
 		siginfo.si_signo = SIGSEGV;
 		siginfo.errno = 0;
@@ -149,35 +157,35 @@ Sample program to use ADI
 -------------------------
 
 Following sample program is meant to illustrate how to use the ADI
-functionality.
-
-#include <unistd.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <elf.h>
-#include <sys/ipc.h>
-#include <sys/shm.h>
-#include <sys/mman.h>
-#include <asm/asi.h>
-
-#ifndef AT_ADI_BLKSZ
-#define AT_ADI_BLKSZ	48
-#endif
-#ifndef AT_ADI_NBITS
-#define AT_ADI_NBITS	49
-#endif
-
-#ifndef PROT_ADI
-#define PROT_ADI	0x10
-#endif
-
-#define BUFFER_SIZE     32*1024*1024UL
-
-main(int argc, char* argv[], char* envp[])
-{
-        unsigned long i, mcde, adi_blksz, adi_nbits;
-        char *shmaddr, *tmp_addr, *end, *veraddr, *clraddr;
-        int shmid, version;
+functionality::
+
+  #include <unistd.h>
+  #include <stdio.h>
+  #include <stdlib.h>
+  #include <elf.h>
+  #include <sys/ipc.h>
+  #include <sys/shm.h>
+  #include <sys/mman.h>
+  #include <asm/asi.h>
+
+  #ifndef AT_ADI_BLKSZ
+  #define AT_ADI_BLKSZ	48
+  #endif
+  #ifndef AT_ADI_NBITS
+  #define AT_ADI_NBITS	49
+  #endif
+
+  #ifndef PROT_ADI
+  #define PROT_ADI	0x10
+  #endif
+
+  #define BUFFER_SIZE     32*1024*1024UL
+
+  main(int argc, char* argv[], char* envp[])
+  {
+          unsigned long i, mcde, adi_blksz, adi_nbits;
+          char *shmaddr, *tmp_addr, *end, *veraddr, *clraddr;
+          int shmid, version;
 	Elf64_auxv_t *auxv;
 
 	adi_blksz = 0;
@@ -202,77 +210,77 @@ main(int argc, char* argv[], char* envp[])
 	printf("\tBlock size = %ld\n", adi_blksz);
 	printf("\tNumber of bits = %ld\n", adi_nbits);
 
-        if ((shmid = shmget(2, BUFFER_SIZE,
-                                IPC_CREAT | SHM_R | SHM_W)) < 0) {
-                perror("shmget failed");
-                exit(1);
-        }
+          if ((shmid = shmget(2, BUFFER_SIZE,
+                                  IPC_CREAT | SHM_R | SHM_W)) < 0) {
+                  perror("shmget failed");
+                  exit(1);
+          }
 
-        shmaddr = shmat(shmid, NULL, 0);
-        if (shmaddr == (char *)-1) {
-                perror("shm attach failed");
-                shmctl(shmid, IPC_RMID, NULL);
-                exit(1);
-        }
+          shmaddr = shmat(shmid, NULL, 0);
+          if (shmaddr == (char *)-1) {
+                  perror("shm attach failed");
+                  shmctl(shmid, IPC_RMID, NULL);
+                  exit(1);
+          }
 
 	if (mprotect(shmaddr, BUFFER_SIZE, PROT_READ|PROT_WRITE|PROT_ADI)) {
 		perror("mprotect failed");
 		goto err_out;
 	}
 
-        /* Set the ADI version tag on the shm segment
-         */
-        version = 10;
-        tmp_addr = shmaddr;
-        end = shmaddr + BUFFER_SIZE;
-        while (tmp_addr < end) {
-                asm volatile(
-                        "stxa %1, [%0]0x90\n\t"
-                        :
-                        : "r" (tmp_addr), "r" (version));
-                tmp_addr += adi_blksz;
-        }
+          /* Set the ADI version tag on the shm segment
+           */
+          version = 10;
+          tmp_addr = shmaddr;
+          end = shmaddr + BUFFER_SIZE;
+          while (tmp_addr < end) {
+                  asm volatile(
+                          "stxa %1, [%0]0x90\n\t"
+                          :
+                          : "r" (tmp_addr), "r" (version));
+                  tmp_addr += adi_blksz;
+          }
 	asm volatile("membar #Sync\n\t");
 
-        /* Create a versioned address from the normal address by placing
+          /* Create a versioned address from the normal address by placing
 	 * version tag in the upper adi_nbits bits
-         */
-        tmp_addr = (void *) ((unsigned long)shmaddr << adi_nbits);
-        tmp_addr = (void *) ((unsigned long)tmp_addr >> adi_nbits);
-        veraddr = (void *) (((unsigned long)version << (64-adi_nbits))
-                        | (unsigned long)tmp_addr);
-
-        printf("Starting the writes:\n");
-        for (i = 0; i < BUFFER_SIZE; i++) {
-                veraddr[i] = (char)(i);
-                if (!(i % (1024 * 1024)))
-                        printf(".");
-        }
-        printf("\n");
-
-        printf("Verifying data...");
+           */
+          tmp_addr = (void *) ((unsigned long)shmaddr << adi_nbits);
+          tmp_addr = (void *) ((unsigned long)tmp_addr >> adi_nbits);
+          veraddr = (void *) (((unsigned long)version << (64-adi_nbits))
+                          | (unsigned long)tmp_addr);
+
+          printf("Starting the writes:\n");
+          for (i = 0; i < BUFFER_SIZE; i++) {
+                  veraddr[i] = (char)(i);
+                  if (!(i % (1024 * 1024)))
+                          printf(".");
+          }
+          printf("\n");
+
+          printf("Verifying data...");
 	fflush(stdout);
-        for (i = 0; i < BUFFER_SIZE; i++)
-                if (veraddr[i] != (char)i)
-                        printf("\nIndex %lu mismatched\n", i);
-        printf("Done.\n");
+          for (i = 0; i < BUFFER_SIZE; i++)
+                  if (veraddr[i] != (char)i)
+                          printf("\nIndex %lu mismatched\n", i);
+          printf("Done.\n");
 
-        /* Disable ADI and clean up
-         */
+          /* Disable ADI and clean up
+           */
 	if (mprotect(shmaddr, BUFFER_SIZE, PROT_READ|PROT_WRITE)) {
 		perror("mprotect failed");
 		goto err_out;
 	}
 
-        if (shmdt((const void *)shmaddr) != 0)
-                perror("Detach failure");
-        shmctl(shmid, IPC_RMID, NULL);
+          if (shmdt((const void *)shmaddr) != 0)
+                  perror("Detach failure");
+          shmctl(shmid, IPC_RMID, NULL);
 
-        exit(0);
+          exit(0);
 
-err_out:
-        if (shmdt((const void *)shmaddr) != 0)
-                perror("Detach failure");
-        shmctl(shmid, IPC_RMID, NULL);
-        exit(1);
-}
+  err_out:
+          if (shmdt((const void *)shmaddr) != 0)
+                  perror("Detach failure");
+          shmctl(shmid, IPC_RMID, NULL);
+          exit(1);
+  }
diff --git a/Documentation/sparc/console.txt b/Documentation/sparc/console.rst
index 5aa735a44e02..73132db83ece 100644
--- a/Documentation/sparc/console.txt
+++ b/Documentation/sparc/console.rst
@@ -1,5 +1,5 @@
-Steps for sending 'break' on sunhv console:
-===========================================
+Steps for sending 'break' on sunhv console
+==========================================
 
 On Baremetal:
    1. press   Esc + 'B'
diff --git a/Documentation/sparc/index.rst b/Documentation/sparc/index.rst
new file mode 100644
index 000000000000..91f7d6643dd5
--- /dev/null
+++ b/Documentation/sparc/index.rst
@@ -0,0 +1,13 @@
+:orphan:
+
+==================
+Sparc Architecture
+==================
+
+.. toctree::
+   :maxdepth: 1
+
+   console
+   adi
+
+   oradax/oracle-dax
diff --git a/Documentation/sparc/oradax/oracle-dax.txt b/Documentation/sparc/oradax/oracle-dax.rst
index 9d53ac93286f..d1e14d572918 100644
--- a/Documentation/sparc/oradax/oracle-dax.txt
+++ b/Documentation/sparc/oradax/oracle-dax.rst
@@ -1,5 +1,6 @@
+=======================================
 Oracle Data Analytics Accelerator (DAX)
----------------------------------------
+=======================================
 
 DAX is a coprocessor which resides on the SPARC M7 (DAX1) and M8
 (DAX2) processor chips, and has direct access to the CPU's L3 caches
@@ -17,6 +18,7 @@ code sufficient to write user or kernel applications that use DAX
 functionality.
 
 The user library is open source and available at:
+
     https://oss.oracle.com/git/gitweb.cgi?p=libdax.git
 
 The Hypervisor interface to the coprocessor is described in detail in
@@ -26,7 +28,7 @@ Specification" version 3.0.20+15, dated 2017-09-25.
 
 
 High Level Overview
--------------------
+===================
 
 A coprocessor request is described by a Command Control Block
 (CCB). The CCB contains an opcode and various parameters. The opcode
@@ -52,7 +54,7 @@ thread.
 
 
 Addressing Memory
------------------
+=================
 
 The kernel does not have access to physical memory in the Sun4v
 architecture, as there is an additional level of memory virtualization
@@ -77,7 +79,7 @@ the request.
 
 
 The Driver API
---------------
+==============
 
 An application makes requests to the driver via the write() system
 call, and gets results (if any) via read(). The completion areas are
@@ -108,6 +110,7 @@ equal to the number of bytes given in the call. Otherwise -1 is
 returned and errno is set.
 
 CCB_DEQUEUE
+-----------
 
 Tells the driver to clean up resources associated with past
 requests. Since no interrupt is generated upon the completion of a
@@ -116,12 +119,14 @@ further status information is returned, so the user should not
 subsequently call read().
 
 CCB_KILL
+--------
 
 Kills a CCB during execution. The CCB is guaranteed to not continue
 executing once this call returns successfully. On success, read() must
 be called to retrieve the result of the action.
 
 CCB_INFO
+--------
 
 Retrieves information about a currently executing CCB. Note that some
 Hypervisors might return 'notfound' when the CCB is in 'inprogress'
@@ -130,6 +135,7 @@ CCB_KILL must be invoked on that CCB. Upon success, read() must be
 called to retrieve the details of the action.
 
 Submission of an array of CCBs for execution
+---------------------------------------------
 
 A write() whose length is a multiple of the CCB size is treated as a
 submit operation. The file offset is treated as the index of the
@@ -146,6 +152,7 @@ status will reflect the error caused by the first CCB that was not
 accepted, and status_data will provide additional data in some cases.
 
 MMAP
+----
 
 The mmap() function provides access to the completion area allocated
 in the driver.  Note that the completion area is not writeable by the
@@ -153,7 +160,7 @@ user process, and the mmap call must not specify PROT_WRITE.
 
 
 Completion of a Request
------------------------
+=======================
 
 The first byte in each completion area is the command status which is
 updated by the coprocessor hardware. Software may take advantage of
@@ -172,7 +179,7 @@ and resumption of execution may be just a few nanoseconds.
 
 
 Application Life Cycle of a DAX Submission
-------------------------------------------
+==========================================
 
  - open dax device
  - call mmap() to get the completion area address
@@ -187,7 +194,7 @@ Application Life Cycle of a DAX Submission
 
 
 Memory Constraints
-------------------
+==================
 
 The DAX hardware operates only on physical addresses. Therefore, it is
 not aware of virtual memory mappings and the discontiguities that may
@@ -226,7 +233,7 @@ CCB Structure
 -------------
 A CCB is an array of 8 64-bit words. Several of these words provide
 command opcodes, parameters, flags, etc., and the rest are addresses
-for the completion area, output buffer, and various inputs:
+for the completion area, output buffer, and various inputs::
 
    struct ccb {
        u64   control;
@@ -252,7 +259,7 @@ The first word (control) is examined by the driver for the following:
 
 
 Example Code
-------------
+============
 
 The DAX is accessible to both user and kernel code.  The kernel code
 can make hypercalls directly while the user code must use wrappers
@@ -265,7 +272,7 @@ arch/sparc/include/uapi/asm/oradax.h must be included.
 
 First, the proper device must be opened. For M7 it will be
 /dev/oradax1 and for M8 it will be /dev/oradax2. The simplest
-procedure is to attempt to open both, as only one will succeed:
+procedure is to attempt to open both, as only one will succeed::
 
 	fd = open("/dev/oradax1", O_RDWR);
 	if (fd < 0)
@@ -273,7 +280,7 @@ procedure is to attempt to open both, as only one will succeed:
 	if (fd < 0)
 	       /* No DAX found */
 
-Next, the completion area must be mapped:
+Next, the completion area must be mapped::
 
       completion_area = mmap(NULL, DAX_MMAP_LEN, PROT_READ, MAP_SHARED, fd, 0);
 
@@ -295,7 +302,7 @@ is the input bitmap inverted.
 
 For details of all the parameters and bits used in this CCB, please
 refer to section 36.2.1.3 of the DAX Hypervisor API document, which
-describes the Scan command in detail.
+describes the Scan command in detail::
 
 	ccb->control =       /* Table 36.1, CCB Header Format */
 		  (2L << 48)     /* command = Scan Value */
@@ -326,7 +333,7 @@ describes the Scan command in detail.
 
 The CCB submission is a write() or pwrite() system call to the
 driver. If the call fails, then a read() must be used to retrieve the
-status:
+status::
 
 	if (pwrite(fd, ccb, 64, 0) != 64) {
 		struct ccb_exec_result status;
@@ -337,7 +344,7 @@ status:
 After a successful submission of the CCB, the completion area may be
 polled to determine when the DAX is finished. Detailed information on
 the contents of the completion area can be found in section 36.2.2 of
-the DAX HV API document.
+the DAX HV API document::
 
 	while (1) {
 		/* Monitored Load */
@@ -355,7 +362,7 @@ the DAX HV API document.
 A completion area status of 1 indicates successful completion of the
 CCB and validity of the output bitmap, which may be used immediately.
 All other non-zero values indicate error conditions which are
-described in section 36.2.2.
+described in section 36.2.2::
 
 	if (completion_area[0] != 1) {	/* section 36.2.2, 1 = command ran and succeeded */
 		/* completion_area[0] contains the completion status */
@@ -364,7 +371,7 @@ described in section 36.2.2.
 
 After the completion area has been processed, the driver must be
 notified that it can release any resources associated with the
-request. This is done via the dequeue operation:
+request. This is done via the dequeue operation::
 
 	struct dax_command cmd;
 	cmd.command = CCB_DEQUEUE;
@@ -375,13 +382,14 @@ request. This is done via the dequeue operation:
 Finally, normal program cleanup should be done, i.e., unmapping
 completion area, closing the dax device, freeing memory etc.
 
-[Kernel example]
+Kernel example
+--------------
 
 The only difference in using the DAX in kernel code is the treatment
 of the completion area. Unlike user applications which mmap the
 completion area allocated by the driver, kernel code must allocate its
 own memory to use for the completion area, and this address and its
-type must be given in the CCB:
+type must be given in the CCB::
 
 	ccb->control |=      /* Table 36.1, CCB Header Format */
 	        (3L << 32);     /* completion area address type = primary virtual */
@@ -389,9 +397,11 @@ type must be given in the CCB:
 	ccb->completion = (unsigned long) completion_area;   /* Completion area address */
 
 The dax submit hypercall is made directly. The flags used in the
-ccb_submit call are documented in the DAX HV API in section 36.3.1.
+ccb_submit call are documented in the DAX HV API in section 36.3.1/
 
-#include <asm/hypervisor.h>
+::
+
+  #include <asm/hypervisor.h>
 
 	hv_rv = sun4v_ccb_submit((unsigned long)ccb, 64,
 				 HV_CCB_QUERY_CMD |
@@ -405,7 +415,7 @@ ccb_submit call are documented in the DAX HV API in section 36.3.1.
 	}
 
 After the submission, the completion area polling code is identical to
-that in user land:
+that in user land::
 
 	while (1) {
 		/* Monitored Load */
@@ -427,3 +437,9 @@ that in user land:
 
 The output bitmap is ready for consumption immediately after the
 completion status indicates success.
+
+Excer[t from UltraSPARC Virtual Machine Specification
+=====================================================
+
+ .. include:: dax-hv-api.txt
+    :literal:
diff --git a/Documentation/speculation.txt b/Documentation/speculation.txt
index e9e6cbae2841..50d7ea857cff 100644
--- a/Documentation/speculation.txt
+++ b/Documentation/speculation.txt
@@ -17,7 +17,7 @@ observed to extract secret information.
 
 For example, in the presence of branch prediction, it is possible for bounds
 checks to be ignored by code which is speculatively executed. Consider the
-following code:
+following code::
 
 	int load_array(int *array, unsigned int index)
 	{
@@ -27,7 +27,7 @@ following code:
 			return array[index];
 	}
 
-Which, on arm64, may be compiled to an assembly sequence such as:
+Which, on arm64, may be compiled to an assembly sequence such as::
 
 	CMP	<index>, #MAX_ARRAY_ELEMS
 	B.LT	less
@@ -44,7 +44,7 @@ microarchitectural state which can be subsequently measured.
 
 More complex sequences involving multiple dependent memory accesses may
 result in sensitive information being leaked. Consider the following
-code, building on the prior example:
+code, building on the prior example::
 
 	int load_dependent_arrays(int *arr1, int *arr2, int index)
 	{
@@ -77,7 +77,7 @@ A call to array_index_nospec(index, size) returns a sanitized index
 value that is bounded to [0, size) even under cpu speculation
 conditions.
 
-This can be used to protect the earlier load_array() example:
+This can be used to protect the earlier load_array() example::
 
 	int load_array(int *array, unsigned int index)
 	{
diff --git a/Documentation/sysctl/kernel.txt b/Documentation/sysctl/kernel.txt
index aa058aa7bf28..f0c86fbb3b48 100644
--- a/Documentation/sysctl/kernel.txt
+++ b/Documentation/sysctl/kernel.txt
@@ -196,7 +196,7 @@ CAP_LAST_CAP from the kernel.
 core_pattern:
 
 core_pattern is used to specify a core dumpfile pattern name.
-. max length 128 characters; default value is "core"
+. max length 127 characters; default value is "core"
 . core_pattern is used as a pattern template for the output filename;
   certain string patterns (beginning with '%') are substituted with
   their actual values.
diff --git a/Documentation/trace/ftrace.rst b/Documentation/trace/ftrace.rst
index 7c5e6d6ab5d1..c3b9bd2fd512 100644
--- a/Documentation/trace/ftrace.rst
+++ b/Documentation/trace/ftrace.rst
@@ -1404,6 +1404,7 @@ trace has provided some very helpful debugging information.
 
 If we prefer function graph output instead of function, we can set
 display-graph option::
+
  with echo 1 > options/display-graph
 
   # tracer: irqsoff
diff --git a/Documentation/trace/histogram.rst b/Documentation/trace/histogram.rst
index 0ea59d45aef1..f95d94d19c22 100644
--- a/Documentation/trace/histogram.rst
+++ b/Documentation/trace/histogram.rst
@@ -2133,33 +2133,33 @@ The following commonly-used handler.action pairs are available:
     the end the event that triggered the snapshot (in this case you
     can verify the timestamps between the sched_waking and
     sched_switch events, which should match the time displayed in the
-    global maximum):
-
-    # cat /sys/kernel/debug/tracing/snapshot
-
-     <...>-2103  [005] d..3   309.873125: sched_switch: prev_comm=cyclictest prev_pid=2103 prev_prio=19 prev_state=D ==> next_comm=swapper/5 next_pid=0 next_prio=120
-     <idle>-0     [005] d.h3   309.873611: sched_waking: comm=cyclictest pid=2102 prio=19 target_cpu=005
-     <idle>-0     [005] dNh4   309.873613: sched_wakeup: comm=cyclictest pid=2102 prio=19 target_cpu=005
-     <idle>-0     [005] d..3   309.873616: sched_switch: prev_comm=swapper/5 prev_pid=0 prev_prio=120 prev_state=S ==> next_comm=cyclictest next_pid=2102 next_prio=19
-     <...>-2102  [005] d..3   309.873625: sched_switch: prev_comm=cyclictest prev_pid=2102 prev_prio=19 prev_state=D ==> next_comm=swapper/5 next_pid=0 next_prio=120
-     <idle>-0     [005] d.h3   309.874624: sched_waking: comm=cyclictest pid=2102 prio=19 target_cpu=005
-     <idle>-0     [005] dNh4   309.874626: sched_wakeup: comm=cyclictest pid=2102 prio=19 target_cpu=005
-     <idle>-0     [005] dNh3   309.874628: sched_waking: comm=cyclictest pid=2103 prio=19 target_cpu=005
-     <idle>-0     [005] dNh4   309.874630: sched_wakeup: comm=cyclictest pid=2103 prio=19 target_cpu=005
-     <idle>-0     [005] d..3   309.874633: sched_switch: prev_comm=swapper/5 prev_pid=0 prev_prio=120 prev_state=S ==> next_comm=cyclictest next_pid=2102 next_prio=19
-     <idle>-0     [004] d.h3   309.874757: sched_waking: comm=gnome-terminal- pid=1699 prio=120 target_cpu=004
-     <idle>-0     [004] dNh4   309.874762: sched_wakeup: comm=gnome-terminal- pid=1699 prio=120 target_cpu=004
-     <idle>-0     [004] d..3   309.874766: sched_switch: prev_comm=swapper/4 prev_pid=0 prev_prio=120 prev_state=S ==> next_comm=gnome-terminal- next_pid=1699 next_prio=120
- gnome-terminal--1699  [004] d.h2   309.874941: sched_stat_runtime: comm=gnome-terminal- pid=1699 runtime=180706 [ns] vruntime=1126870572 [ns]
-     <idle>-0     [003] d.s4   309.874956: sched_waking: comm=rcu_sched pid=9 prio=120 target_cpu=007
-     <idle>-0     [003] d.s5   309.874960: sched_wake_idle_without_ipi: cpu=7
-     <idle>-0     [003] d.s5   309.874961: sched_wakeup: comm=rcu_sched pid=9 prio=120 target_cpu=007
-     <idle>-0     [007] d..3   309.874963: sched_switch: prev_comm=swapper/7 prev_pid=0 prev_prio=120 prev_state=S ==> next_comm=rcu_sched next_pid=9 next_prio=120
-  rcu_sched-9     [007] d..3   309.874973: sched_stat_runtime: comm=rcu_sched pid=9 runtime=13646 [ns] vruntime=22531430286 [ns]
-  rcu_sched-9     [007] d..3   309.874978: sched_switch: prev_comm=rcu_sched prev_pid=9 prev_prio=120 prev_state=R+ ==> next_comm=swapper/7 next_pid=0 next_prio=120
-      <...>-2102  [005] d..4   309.874994: sched_migrate_task: comm=cyclictest pid=2103 prio=19 orig_cpu=5 dest_cpu=1
-      <...>-2102  [005] d..4   309.875185: sched_wake_idle_without_ipi: cpu=1
-     <idle>-0     [001] d..3   309.875200: sched_switch: prev_comm=swapper/1 prev_pid=0 prev_prio=120 prev_state=S ==> next_comm=cyclictest next_pid=2103 next_prio=19
+    global maximum)::
+
+     # cat /sys/kernel/debug/tracing/snapshot
+
+         <...>-2103  [005] d..3   309.873125: sched_switch: prev_comm=cyclictest prev_pid=2103 prev_prio=19 prev_state=D ==> next_comm=swapper/5 next_pid=0 next_prio=120
+         <idle>-0     [005] d.h3   309.873611: sched_waking: comm=cyclictest pid=2102 prio=19 target_cpu=005
+         <idle>-0     [005] dNh4   309.873613: sched_wakeup: comm=cyclictest pid=2102 prio=19 target_cpu=005
+         <idle>-0     [005] d..3   309.873616: sched_switch: prev_comm=swapper/5 prev_pid=0 prev_prio=120 prev_state=S ==> next_comm=cyclictest next_pid=2102 next_prio=19
+         <...>-2102  [005] d..3   309.873625: sched_switch: prev_comm=cyclictest prev_pid=2102 prev_prio=19 prev_state=D ==> next_comm=swapper/5 next_pid=0 next_prio=120
+         <idle>-0     [005] d.h3   309.874624: sched_waking: comm=cyclictest pid=2102 prio=19 target_cpu=005
+         <idle>-0     [005] dNh4   309.874626: sched_wakeup: comm=cyclictest pid=2102 prio=19 target_cpu=005
+         <idle>-0     [005] dNh3   309.874628: sched_waking: comm=cyclictest pid=2103 prio=19 target_cpu=005
+         <idle>-0     [005] dNh4   309.874630: sched_wakeup: comm=cyclictest pid=2103 prio=19 target_cpu=005
+         <idle>-0     [005] d..3   309.874633: sched_switch: prev_comm=swapper/5 prev_pid=0 prev_prio=120 prev_state=S ==> next_comm=cyclictest next_pid=2102 next_prio=19
+         <idle>-0     [004] d.h3   309.874757: sched_waking: comm=gnome-terminal- pid=1699 prio=120 target_cpu=004
+         <idle>-0     [004] dNh4   309.874762: sched_wakeup: comm=gnome-terminal- pid=1699 prio=120 target_cpu=004
+         <idle>-0     [004] d..3   309.874766: sched_switch: prev_comm=swapper/4 prev_pid=0 prev_prio=120 prev_state=S ==> next_comm=gnome-terminal- next_pid=1699 next_prio=120
+     gnome-terminal--1699  [004] d.h2   309.874941: sched_stat_runtime: comm=gnome-terminal- pid=1699 runtime=180706 [ns] vruntime=1126870572 [ns]
+         <idle>-0     [003] d.s4   309.874956: sched_waking: comm=rcu_sched pid=9 prio=120 target_cpu=007
+         <idle>-0     [003] d.s5   309.874960: sched_wake_idle_without_ipi: cpu=7
+         <idle>-0     [003] d.s5   309.874961: sched_wakeup: comm=rcu_sched pid=9 prio=120 target_cpu=007
+         <idle>-0     [007] d..3   309.874963: sched_switch: prev_comm=swapper/7 prev_pid=0 prev_prio=120 prev_state=S ==> next_comm=rcu_sched next_pid=9 next_prio=120
+      rcu_sched-9     [007] d..3   309.874973: sched_stat_runtime: comm=rcu_sched pid=9 runtime=13646 [ns] vruntime=22531430286 [ns]
+      rcu_sched-9     [007] d..3   309.874978: sched_switch: prev_comm=rcu_sched prev_pid=9 prev_prio=120 prev_state=R+ ==> next_comm=swapper/7 next_pid=0 next_prio=120
+          <...>-2102  [005] d..4   309.874994: sched_migrate_task: comm=cyclictest pid=2103 prio=19 orig_cpu=5 dest_cpu=1
+          <...>-2102  [005] d..4   309.875185: sched_wake_idle_without_ipi: cpu=1
+         <idle>-0     [001] d..3   309.875200: sched_switch: prev_comm=swapper/1 prev_pid=0 prev_prio=120 prev_state=S ==> next_comm=cyclictest next_pid=2103 next_prio=19
 
   - onchange(var).save(field,..	.)
 
@@ -2213,9 +2213,10 @@ The following commonly-used handler.action pairs are available:
     following the rest of the fields.
 
     If a snaphot was taken, there is also a message indicating that,
-    along with the value and event that triggered the snapshot:
+    along with the value and event that triggered the snapshot::
+
+      # cat /sys/kernel/debug/tracing/events/tcp/tcp_probe/hist
 
-    # cat /sys/kernel/debug/tracing/events/tcp/tcp_probe/hist
       { dport:       1521 } hitcount:          8
 	changed:         10  snd_wnd:      35456  srtt:     154262  rcv_wnd:      42112
 
@@ -2228,14 +2229,15 @@ The following commonly-used handler.action pairs are available:
       { dport:        443 } hitcount:        211
 	changed:         10  snd_wnd:      26960  srtt:      17379  rcv_wnd:      28800
 
-    Snapshot taken (see tracing/snapshot).  Details:
+    Snapshot taken (see tracing/snapshot).  Details::
+
         triggering value { onchange($cwnd) }:         10
         triggered by event with key: { dport:         80 }
 
-    Totals:
-        Hits: 414
-        Entries: 4
-        Dropped: 0
+      Totals:
+          Hits: 414
+          Entries: 4
+          Dropped: 0
 
     In the above case, the event that triggered the snapshot has the
     key with dport == 80.  If you look at the bucket that has 80 as
@@ -2245,18 +2247,18 @@ The following commonly-used handler.action pairs are available:
     the global snapshot).
 
     And finally, looking at the snapshot data should show at or near
-    the end the event that triggered the snapshot:
-
-    # cat /sys/kernel/debug/tracing/snapshot
-
-       gnome-shell-1261  [006] dN.3    49.823113: sched_stat_runtime: comm=gnome-shell pid=1261 runtime=49347 [ns] vruntime=1835730389 [ns]
-     kworker/u16:4-773   [003] d..3    49.823114: sched_switch: prev_comm=kworker/u16:4 prev_pid=773 prev_prio=120 prev_state=R+ ==> next_comm=kworker/3:2 next_pid=135 next_prio=120
-       gnome-shell-1261  [006] d..3    49.823114: sched_switch: prev_comm=gnome-shell prev_pid=1261 prev_prio=120 prev_state=R+ ==> next_comm=kworker/6:2 next_pid=387 next_prio=120
-       kworker/3:2-135   [003] d..3    49.823118: sched_stat_runtime: comm=kworker/3:2 pid=135 runtime=5339 [ns] vruntime=17815800388 [ns]
-       kworker/6:2-387   [006] d..3    49.823120: sched_stat_runtime: comm=kworker/6:2 pid=387 runtime=9594 [ns] vruntime=14589605367 [ns]
-       kworker/6:2-387   [006] d..3    49.823122: sched_switch: prev_comm=kworker/6:2 prev_pid=387 prev_prio=120 prev_state=R+ ==> next_comm=gnome-shell next_pid=1261 next_prio=120
-       kworker/3:2-135   [003] d..3    49.823123: sched_switch: prev_comm=kworker/3:2 prev_pid=135 prev_prio=120 prev_state=T ==> next_comm=swapper/3 next_pid=0 next_prio=120
-            <idle>-0     [004] ..s7    49.823798: tcp_probe: src=10.0.0.10:54326 dest=23.215.104.193:80 mark=0x0 length=32 snd_nxt=0xe3ae2ff5 snd_una=0xe3ae2ecd snd_cwnd=10 ssthresh=2147483647 snd_wnd=28960 srtt=19604 rcv_wnd=29312
+    the end the event that triggered the snapshot::
+
+      # cat /sys/kernel/debug/tracing/snapshot
+
+         gnome-shell-1261  [006] dN.3    49.823113: sched_stat_runtime: comm=gnome-shell pid=1261 runtime=49347 [ns] vruntime=1835730389 [ns]
+       kworker/u16:4-773   [003] d..3    49.823114: sched_switch: prev_comm=kworker/u16:4 prev_pid=773 prev_prio=120 prev_state=R+ ==> next_comm=kworker/3:2 next_pid=135 next_prio=120
+         gnome-shell-1261  [006] d..3    49.823114: sched_switch: prev_comm=gnome-shell prev_pid=1261 prev_prio=120 prev_state=R+ ==> next_comm=kworker/6:2 next_pid=387 next_prio=120
+         kworker/3:2-135   [003] d..3    49.823118: sched_stat_runtime: comm=kworker/3:2 pid=135 runtime=5339 [ns] vruntime=17815800388 [ns]
+         kworker/6:2-387   [006] d..3    49.823120: sched_stat_runtime: comm=kworker/6:2 pid=387 runtime=9594 [ns] vruntime=14589605367 [ns]
+         kworker/6:2-387   [006] d..3    49.823122: sched_switch: prev_comm=kworker/6:2 prev_pid=387 prev_prio=120 prev_state=R+ ==> next_comm=gnome-shell next_pid=1261 next_prio=120
+         kworker/3:2-135   [003] d..3    49.823123: sched_switch: prev_comm=kworker/3:2 prev_pid=135 prev_prio=120 prev_state=T ==> next_comm=swapper/3 next_pid=0 next_prio=120
+              <idle>-0     [004] ..s7    49.823798: tcp_probe: src=10.0.0.10:54326 dest=23.215.104.193:80 mark=0x0 length=32 snd_nxt=0xe3ae2ff5 snd_una=0xe3ae2ecd snd_cwnd=10 ssthresh=2147483647 snd_wnd=28960 srtt=19604 rcv_wnd=29312
 
 3. User space creating a trigger
 --------------------------------
diff --git a/Documentation/translations/index.rst b/Documentation/translations/index.rst
index 7f77c52d33aa..e446e5ed00a6 100644
--- a/Documentation/translations/index.rst
+++ b/Documentation/translations/index.rst
@@ -11,3 +11,43 @@ Translations
    it_IT/index
    ko_KR/index
    ja_JP/index
+
+
+.. _translations_disclaimer:
+
+Disclaimer
+----------
+
+Translation's purpose is to ease reading and understanding in languages other
+than English. Its aim is to help people who do not understand English or have
+doubts about its interpretation. Additionally, some people prefer to read
+documentation in their native language, but please bear in mind that the
+*only* official documentation is the English one: :ref:`linux_doc`.
+
+It is very unlikely that an update to :ref:`linux_doc` will be propagated
+immediately to all translations.  Translations' maintainers - and
+contributors - follow the evolution of the official documentation and they
+maintain translations aligned as much as they can.  For this reason there is
+no guarantee that a translation is up to date.  If what you read in a
+translation does not sound right compared to what you read in the code, please
+inform the translation maintainer and - if you can - check also the English
+documentation.
+
+A translation is not a fork of the official documentation, therefore
+translations' users should not find information that differs from the official
+English documentation.  Any content addition, removal or update, must be
+applied to the English documents first.  Afterwards and when possible, the
+same change should be applied to translations.  Translations' maintainers
+accept only contributions that are merely translation related (e.g. new
+translations, updates, fixes).
+
+Translations try to be as accurate as possible but it is not possible to map
+one language directly to all other languages. Each language has its own
+grammar and culture, so the translation of an English statement may need to be
+adapted to fit a different language.  For this reason, when viewing
+translations, you may find slight differences that carry the same message but
+in a different form.
+
+If you need to communicate with the Linux community but you do not feel
+comfortable writing in English, you can ask the translation's maintainers
+for help.
diff --git a/Documentation/translations/it_IT/core-api/memory-allocation.rst b/Documentation/translations/it_IT/core-api/memory-allocation.rst
new file mode 100644
index 000000000000..11d5148f8d6b
--- /dev/null
+++ b/Documentation/translations/it_IT/core-api/memory-allocation.rst
@@ -0,0 +1,13 @@
+.. include:: ../disclaimer-ita.rst
+
+:Original: :ref:`Documentation/core-api/memory-allocation.rst <memory_allocation>`
+
+.. _it_memory_allocation:
+
+================================
+Guida all'allocazione di memoria
+================================
+
+.. warning::
+
+    TODO ancora da tradurre
diff --git a/Documentation/translations/it_IT/disclaimer-ita.rst b/Documentation/translations/it_IT/disclaimer-ita.rst
index d68e52de6a5d..bfe8a384baed 100644
--- a/Documentation/translations/it_IT/disclaimer-ita.rst
+++ b/Documentation/translations/it_IT/disclaimer-ita.rst
@@ -1,13 +1,6 @@
 :orphan:
 
-.. note::
-   This document is maintained by Federico Vaga <federico.vaga@vaga.pv.it>.
-   If you find any difference between this document and the original file or a
-   problem with the translation, please contact the maintainer of this file.
-   Following people helped to translate or review:
-   Alessia Mantegazza <amantegazza@vaga.pv.it>
-
 .. warning::
-   The purpose of this file is to be easier to read and understand for Italian
-   speakers and is not intended as a fork. So, if you have any comments or
-   updates for this file please try to update the original English file first.
+   In caso di dubbi sulla correttezza del contenuto di questa traduzione,
+   l'unico riferimento valido è la documentazione ufficiale in inglese.
+   Per maggiori informazioni consultate le :ref:`avvertenze <it_disclaimer>`.
diff --git a/Documentation/translations/it_IT/doc-guide/index.rst b/Documentation/translations/it_IT/doc-guide/index.rst
index 7a6562b547ee..9fffff626711 100644
--- a/Documentation/translations/it_IT/doc-guide/index.rst
+++ b/Documentation/translations/it_IT/doc-guide/index.rst
@@ -12,9 +12,9 @@ Come scrivere la documentazione del kernel
 .. toctree::
    :maxdepth: 1
 
-   sphinx.rst
-   kernel-doc.rst
-   parse-headers.rst
+   sphinx
+   kernel-doc
+   parse-headers
 
 .. only::  subproject and html
 
diff --git a/Documentation/translations/it_IT/index.rst b/Documentation/translations/it_IT/index.rst
index ea9b2916b3e4..409eaac03e9f 100644
--- a/Documentation/translations/it_IT/index.rst
+++ b/Documentation/translations/it_IT/index.rst
@@ -4,26 +4,49 @@
 Traduzione italiana
 ===================
 
-L'obiettivo di questa traduzione è di rendere più facile la lettura e
-la comprensione per chi preferisce leggere in lingua italiana.
-Tenete presente che la documentazione di riferimento rimane comunque
-quella in lingua inglese: :ref:`linux_doc`
-
-Questa traduzione cerca di essere il più fedele possibile all'originale ma
-è ovvio che alcune frasi vadano trasformate: non aspettatevi una traduzione
-letterale. Quando possibile, si eviteranno gli inglesismi ed al loro posto
-verranno utilizzate le corrispettive parole italiane.
+:manutentore: Federico Vaga <federico.vaga@vaga.pv.it>
 
-Se notate che la traduzione non è più aggiornata potete contattare
-direttamente il manutentore della traduzione italiana.
+.. _it_disclaimer:
 
-Se notate che la documentazione contiene errori o dimenticanze, allora
-verificate la documentazione di riferimento in lingua inglese. Se il problema
-è presente anche nella documentazione di riferimento, contattate il suo
-manutentore. Se avete problemi a scrivere in inglese, potete comunque
-riportare il problema al manutentore della traduzione italiana.
+Avvertenze
+==========
 
-Manutentore della traduzione italiana: Federico Vaga <federico.vaga@vaga.pv.it>
+L'obiettivo di questa traduzione è di rendere più facile la lettura e
+la comprensione per chi non capisce l'inglese o ha dubbi sulla sua
+interpretazione, oppure semplicemente per chi preferisce leggere in lingua
+italiana. Tuttavia, tenete ben presente che l'*unica* documentazione
+ufficiale è quella in lingua inglese: :ref:`linux_doc`
+
+La propagazione simultanea a tutte le traduzioni di una modifica in
+:ref:`linux_doc` è altamente improbabile. I manutentori delle traduzioni -
+e i contributori - seguono l'evolversi della documentazione ufficiale e
+cercano di mantenere le rispettive traduzioni allineate nel limite del
+possibile.  Per questo motivo non c'è garanzia che una traduzione sia
+aggiornata all'ultima modifica.  Se quello che leggete in una traduzione
+non corrisponde a quello che leggete nel codice, informate il manutentore
+della traduzione e - se potete - verificate anche la documentazione in
+inglese.
+
+Una traduzione non è un *fork* della documentazione ufficiale, perciò gli
+utenti non vi troveranno alcuna informazione diversa rispetto alla versione
+ufficiale.  Ogni aggiunta, rimozione o modifica dei contenuti deve essere
+fatta prima nei documenti in inglese. In seguito, e quando è possibile, la
+stessa modifica dovrebbe essere applicata anche alle traduzioni.  I manutentori
+delle traduzioni accettano contributi che interessano prettamente l'attività
+di traduzione (per esempio, nuove traduzioni, aggiornamenti, correzioni).
+
+Le traduzioni cercano di essere il più possibile accurate ma non è possibile
+mappare direttamente una lingua in un'altra. Ogni lingua ha la sua grammatica
+e una sua cultura alle spalle, quindi la traduzione di una frase in inglese
+potrebbe essere modificata per adattarla all'italiano. Per questo motivo,
+quando leggete questa traduzione, potreste trovare alcune differenze di forma
+ma che trasmettono comunque il messaggio originale.  Nonostante la grande
+diffusione di inglesismi nella lingua parlata, quando possibile, questi
+verranno sostituiti dalle corrispettive parole italiane
+
+Se avete bisogno d'aiuto per comunicare con la comunità Linux ma non vi sentite
+a vostro agio nello scrivere in inglese, potete chiedere aiuto al manutentore
+della traduzione.
 
 La documentazione del kernel Linux
 ==================================
@@ -47,9 +70,7 @@ I seguenti documenti descrivono la licenza usata nei sorgenti del kernel Linux
 (GPLv2), come licenziare i singoli file; inoltre troverete i riferimenti al
 testo integrale della licenza.
 
-.. warning::
-
-    TODO ancora da tradurre
+* :ref:`it_kernel_licensing`
 
 Documentazione per gli utenti
 -----------------------------
@@ -90,10 +111,6 @@ vostre modifiche molto più semplice
    doc-guide/index
    kernel-hacking/index
 
-.. warning::
-
-    TODO ancora da tradurre
-
 Documentazione della API del kernel
 -----------------------------------
 
diff --git a/Documentation/translations/it_IT/networking/netdev-FAQ.rst b/Documentation/translations/it_IT/networking/netdev-FAQ.rst
new file mode 100644
index 000000000000..8489ead7cff1
--- /dev/null
+++ b/Documentation/translations/it_IT/networking/netdev-FAQ.rst
@@ -0,0 +1,13 @@
+.. include:: ../disclaimer-ita.rst
+
+:Original: :ref:`Documentation/process/stable-kernel-rules.rst <stable_kernel_rules>`
+
+.. _it_netdev-FAQ:
+
+==========
+netdev FAQ
+==========
+
+.. warning::
+
+    TODO ancora da tradurre
diff --git a/Documentation/translations/it_IT/process/5.Posting.rst b/Documentation/translations/it_IT/process/5.Posting.rst
index b979266aa884..1476d51eb5e5 100644
--- a/Documentation/translations/it_IT/process/5.Posting.rst
+++ b/Documentation/translations/it_IT/process/5.Posting.rst
@@ -233,10 +233,12 @@ Le etichette in uso più comuni sono:
    :ref:`Documentation/translations/it_IT/process/submitting-patches.rst <it_submittingpatches>`.
    Codice che non presenta una firma appropriata non potrà essere integrato.
 
- - Co-developed-by: indica che la patch è stata sviluppata anche da un altro
-   sviluppatore assieme all'autore originale.  Questo è utile quando più
-   persone lavorano sulla stessa patch.  Da notare che questa persona deve
-   avere anche una riga "Signed-off-by:" nella patch.
+ - Co-developed-by: indica che la patch è stata cosviluppata da diversi
+   sviluppatori; viene usato per assegnare più autori (in aggiunta a quello
+   associato all'etichetta From:) quando più persone lavorano ad una patch.
+   Ogni Co-developed-by: dev'essere seguito immediatamente da un Signed-off-by:
+   del corrispondente coautore.  Maggiori dettagli ed esempi sono disponibili
+   in :ref:`Documentation/translations/it_IT/process/submitting-patches.rst <it_submittingpatches>`.
 
  - Acked-by: indica il consenso di un altro sviluppatore (spesso il manutentore
    del codice in oggetto) all'integrazione della patch nel kernel.
diff --git a/Documentation/translations/it_IT/process/coding-style.rst b/Documentation/translations/it_IT/process/coding-style.rst
index 2fd0e7f79d55..5ef534c95e69 100644
--- a/Documentation/translations/it_IT/process/coding-style.rst
+++ b/Documentation/translations/it_IT/process/coding-style.rst
@@ -859,7 +859,8 @@ racchiusa in #ifdef, potete usare printk(KERN_DEBUG ...).
 
 Il kernel fornisce i seguenti assegnatori ad uso generico:
 kmalloc(), kzalloc(), kmalloc_array(), kcalloc(), vmalloc(), e vzalloc().
-Per maggiori informazioni, consultate la documentazione dell'API.
+Per maggiori informazioni, consultate la documentazione dell'API:
+:ref:`Documentation/translations/it_IT/core-api/memory-allocation.rst <it_memory_allocation>`
 
 Il modo preferito per passare la dimensione di una struttura è il seguente:
 
@@ -890,6 +891,11 @@ Il modo preferito per assegnare un vettore a zero è il seguente:
 Entrambe verificano la condizione di overflow per la dimensione
 d'assegnamento n * sizeof(...), se accade ritorneranno NULL.
 
+Questi allocatori generici producono uno *stack dump* in caso di fallimento
+a meno che non venga esplicitamente specificato __GFP_NOWARN. Quindi, nella
+maggior parte dei casi, è inutile stampare messaggi aggiuntivi quando uno di
+questi allocatori ritornano un puntatore NULL.
+
 15) Il morbo inline
 -------------------
 
diff --git a/Documentation/translations/it_IT/process/deprecated.rst b/Documentation/translations/it_IT/process/deprecated.rst
new file mode 100644
index 000000000000..776f26732a94
--- /dev/null
+++ b/Documentation/translations/it_IT/process/deprecated.rst
@@ -0,0 +1,129 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+.. include:: ../disclaimer-ita.rst
+
+:Original: :ref:`Documentation/process/deprecated.rst <deprecated>`
+:Translator: Federico Vaga <federico.vaga@vaga.pv.it>
+
+.. _it_deprecated:
+
+==============================================================================
+Interfacce deprecate, caratteristiche del linguaggio, attributi, e convenzioni
+==============================================================================
+
+In un mondo perfetto, sarebbe possibile prendere tutti gli usi di
+un'interfaccia deprecata e convertirli in quella nuova, e così sarebbe
+possibile rimuovere la vecchia interfaccia in un singolo ciclo di sviluppo.
+Tuttavia, per via delle dimensioni del kernel, la gerarchia dei manutentori e
+le tempistiche, non è sempre possibile fare questo tipo di conversione tutta
+in una volta. Questo significa che nuove istanze di una vecchia interfaccia
+potrebbero aggiungersi al kernel proprio quando si sta cercando di rimuoverle,
+aumentando così il carico di lavoro. Al fine di istruire gli sviluppatori su
+cosa è considerato deprecato (e perché), è stata create la seguente lista a cui
+fare riferimento quando qualcuno propone modifiche che usano cose deprecate.
+
+__deprecated
+------------
+Nonostante questo attributo marchi visibilmente un interfaccia come deprecata,
+`non produce più alcun avviso durante la compilazione
+<https://git.kernel.org/linus/771c035372a036f83353eef46dbb829780330234>`_
+perché uno degli obiettivi del kernel è quello di compilare senza avvisi;
+inoltre, nessuno stava agendo per rimuovere queste interfacce. Nonostante l'uso
+di `__deprecated` in un file d'intestazione sia opportuno per segnare una
+interfaccia come 'vecchia', questa non è una soluzione completa. L'interfaccia
+deve essere rimossa dal kernel, o aggiunta a questo documento per scoraggiarne
+l'uso.
+
+Calcoli codificati negli argomenti di un allocatore
+----------------------------------------------------
+Il calcolo dinamico delle dimensioni (specialmente le moltiplicazioni) non
+dovrebbero essere fatto negli argomenti di funzioni di allocazione di memoria
+(o simili) per via del rischio di overflow. Questo può portare a valori più
+piccoli di quelli che il chiamante si aspettava. L'uso di questo modo di
+allocare può portare ad un overflow della memoria di heap e altri
+malfunzionamenti. (Si fa eccezione per valori numerici per i quali il
+compilatore può generare avvisi circa un potenziale overflow. Tuttavia usare
+i valori numerici come suggerito di seguito è innocuo).
+
+Per esempio, non usate ``count * size`` come argomento::
+
+	foo = kmalloc(count * size, GFP_KERNEL);
+
+Al suo posto, si dovrebbe usare l'allocatore a due argomenti::
+
+	foo = kmalloc_array(count, size, GFP_KERNEL);
+
+Se questo tipo di allocatore non è disponibile, allora dovrebbero essere usate
+le funzioni del tipo *saturate-on-overflow*::
+
+	bar = vmalloc(array_size(count, size));
+
+Un altro tipico caso da evitare è quello di calcolare la dimensione di una
+struttura seguita da un vettore di altre strutture, come nel seguente caso::
+
+	header = kzalloc(sizeof(*header) + count * sizeof(*header->item),
+			 GFP_KERNEL);
+
+Invece, usate la seguente funzione::
+
+	header = kzalloc(struct_size(header, item, count), GFP_KERNEL);
+
+Per maggiori dettagli fate riferimento a :c:func:`array_size`,
+:c:func:`array3_size`, e :c:func:`struct_size`, così come la famiglia di
+funzioni :c:func:`check_add_overflow` e :c:func:`check_mul_overflow`.
+
+simple_strtol(), simple_strtoll(), simple_strtoul(), simple_strtoull()
+----------------------------------------------------------------------
+Le funzioni :c:func:`simple_strtol`, :c:func:`simple_strtoll`,
+:c:func:`simple_strtoul`, e :c:func:`simple_strtoull` ignorano volutamente
+i possibili overflow, e questo può portare il chiamante a generare risultati
+inaspettati. Le rispettive funzioni :c:func:`kstrtol`, :c:func:`kstrtoll`,
+:c:func:`kstrtoul`, e :c:func:`kstrtoull` sono da considerarsi le corrette
+sostitute; tuttavia va notato che queste richiedono che la stringa sia
+terminata con il carattere NUL o quello di nuova riga.
+
+strcpy()
+--------
+La funzione :c:func:`strcpy` non fa controlli agli estremi del buffer
+di destinazione. Questo può portare ad un overflow oltre i limiti del
+buffer e generare svariati tipi di malfunzionamenti. Nonostante l'opzione
+`CONFIG_FORTIFY_SOURCE=y` e svariate opzioni del compilatore aiutano
+a ridurne il rischio, non c'è alcuna buona ragione per continuare ad usare
+questa funzione. La versione sicura da usare è :c:func:`strscpy`.
+
+strncpy() su stringe terminate con NUL
+--------------------------------------
+L'utilizzo di :c:func:`strncpy` non fornisce alcuna garanzia sul fatto che
+il buffer di destinazione verrà terminato con il carattere NUL. Questo
+potrebbe portare a diversi overflow di lettura o altri malfunzionamenti
+causati, appunto, dalla mancanza del terminatore. Questa estende la
+terminazione nel buffer di destinazione quando la stringa d'origine è più
+corta; questo potrebbe portare ad una penalizzazione delle prestazioni per
+chi usa solo stringe terminate. La versione sicura da usare è
+:c:func:`strscpy`. (chi usa :c:func:`strscpy` e necessita di estendere la
+terminazione con NUL deve aggiungere una chiamata a :c:func:`memset`)
+
+Se il chiamate no usa stringhe terminate con NUL, allore :c:func:`strncpy()`
+può continuare ad essere usata, ma i buffer di destinazione devono essere
+marchiati con l'attributo `__nonstring <https://gcc.gnu.org/onlinedocs/gcc/Common-Variable-Attributes.html>`_
+per evitare avvisi durante la compilazione.
+
+strlcpy()
+---------
+La funzione :c:func:`strlcpy`, per prima cosa, legge interamente il buffer di
+origine, magari leggendo più di quanto verrà effettivamente copiato. Questo
+è inefficiente e può portare a overflow di lettura quando la stringa non è
+terminata con NUL. La versione sicura da usare è :c:func:`strscpy`.
+
+Vettori a dimensione variabile (VLA)
+------------------------------------
+
+Usare VLA sullo stack produce codice molto peggiore rispetto a quando si usano
+vettori a dimensione fissa. Questi `problemi di prestazioni <https://git.kernel.org/linus/02361bc77888>`_,
+tutt'altro che banali, sono già un motivo valido per eliminare i VLA; in
+aggiunta sono anche un problema per la sicurezza. La crescita dinamica di un
+vettore nello stack potrebbe eccedere la memoria rimanente in tale segmento.
+Questo può portare a dei malfunzionamenti, potrebbe sovrascrivere
+dati importanti alla fine dello stack (quando il kernel è compilato senza
+`CONFIG_THREAD_INFO_IN_TASK=y`), o sovrascrivere un pezzo di memoria adiacente
+allo stack (quando il kernel è compilato senza `CONFIG_VMAP_STACK=y`).
diff --git a/Documentation/translations/it_IT/process/kernel-enforcement-statement.rst b/Documentation/translations/it_IT/process/kernel-enforcement-statement.rst
index 4ddf5a35b270..1f62da622e26 100644
--- a/Documentation/translations/it_IT/process/kernel-enforcement-statement.rst
+++ b/Documentation/translations/it_IT/process/kernel-enforcement-statement.rst
@@ -1,13 +1,175 @@
 .. include:: ../disclaimer-ita.rst
 
 :Original: :ref:`Documentation/process/kernel-enforcement-statement.rst <process_statement_kernel>`
-
+:Translator: Federico Vaga <federico.vaga@vaga.pv.it>
 
 .. _it_process_statement_kernel:
 
 Applicazione della licenza sul kernel Linux
 ===========================================
 
-.. warning::
+Come sviluppatori del kernel Linux, abbiamo un certo interessa su come il
+nostro software viene usato e su come la sua licenza viene fatta rispettare.
+Il rispetto reciproco degli obblighi di condivisione della GPL-2.0 è
+fondamentale per la sostenibilità di lungo periodo del nostro software e
+della nostra comunità.
+
+Benché ognuno abbia il diritto a far rispettare il diritto d'autore per i
+propri contributi alla nostra comunità, condividiamo l'interesse a far si che
+ogni azione individuale nel far rispettare i propri diritti sia condotta in
+modo da portare beneficio alla comunità e che non abbia, involontariamente,
+impatti negativi sulla salute e la crescita del nostro ecosistema software.
+Al fine di scoraggiare l'esecuzione di azioni inutili, concordiamo che è nel
+migliore interesse della nostra comunità di sviluppo di impegnarci nel
+rispettare i seguenti obblighi nei confronti degli utenti del kernel Linux
+per conto nostro e di qualsiasi successore ai nostri interessi sul diritto
+d'autore:
+
+    Malgrado le clausole di risoluzione della licenza GPL-2.0, abbiamo
+    concordato che è nel migliore interesse della nostra comunità di sviluppo
+    adottare le seguenti disposizioni della GPL-3.0 come permessi aggiuntivi
+    alla nostra licenza nei confronti di qualsiasi affermazione non difensiva
+    di diritti sulla licenza.
+
+	In ogni caso, se cessano tutte le violazioni di questa Licenza, allora
+	la tua licenza da parte di un dato detentore del copyright viene
+	ripristinata (a) in via cautelativa, a meno che e fino a quando il
+	detentore del copyright non cessa esplicitamente e definitivamente
+	la tua licenza, e (b) in via permanente se il detentore del copyright
+	non ti notifica in alcun modo la violazione entro 60 giorni dalla
+	cessazione della licenza.
+
+	Inoltre, la tua licenza da parte di un dato detentore del copyright
+	viene ripristinata in maniera permanente se il detentore del copyright
+	ti notifica la violazione in maniera adeguata, se questa è la prima
+	volta che ricevi una notifica di violazione di questa Licenza (per
+	qualunque Programma) dallo stesso detentore di copyright, e se rimedi
+	alla violazione entro 30 giorni dalla data di ricezione della notifica
+	di violazione.
+
+Fornendo queste garanzie, abbiamo l'intenzione di incoraggiare l'uso del
+software.  Vogliamo che le aziende e le persone usino, modifichino e
+distribuiscano a questo software.  Vogliamo lavorare con gli utenti in modo
+aperto e trasparente per eliminare ogni incertezza circa le nostre aspettative
+sul rispetto o l'ottemperanza alla licenza che possa limitare l'uso del nostro
+software.  Vediamo l'azione legale come ultima spiaggia, da avviare solo quando
+gli altri sforzi della comunità hanno fallito nel risolvere il problema.
+
+Per finire, una volta che un problema di non rispetto della licenza viene
+risolto, speriamo che gli utenti si sentano i benvenuti ad aggregarsi a noi
+nello sviluppo di questo progetto.  Lavorando assieme, saremo più forti.
+
+Ad eccezione deve specificato, parliamo per noi stessi, e non per una qualsiasi
+azienda per la quale lavoriamo oggi, o per cui abbiamo lavorato in passato, o
+lavoreremo in futuro.
+
 
-    TODO ancora da tradurre
+  - Laura Abbott
+  - Bjorn Andersson (Linaro)
+  - Andrea Arcangeli
+  - Neil Armstrong
+  - Jens Axboe
+  - Pablo Neira Ayuso
+  - Khalid Aziz
+  - Ralf Baechle
+  - Felipe Balbi
+  - Arnd Bergmann
+  - Ard Biesheuvel
+  - Tim Bird
+  - Paolo Bonzini
+  - Christian Borntraeger
+  - Mark Brown (Linaro)
+  - Paul Burton
+  - Javier Martinez Canillas
+  - Rob Clark
+  - Kees Cook (Google)
+  - Jonathan Corbet
+  - Dennis Dalessandro
+  - Vivien Didelot (Savoir-faire Linux)
+  - Hans de Goede
+  - Mel Gorman (SUSE)
+  - Sven Eckelmann
+  - Alex Elder (Linaro)
+  - Fabio Estevam
+  - Larry Finger
+  - Bhumika Goyal
+  - Andy Gross
+  - Juergen Gross
+  - Shawn Guo
+  - Ulf Hansson
+  - Stephen Hemminger (Microsoft)
+  - Tejun Heo
+  - Rob Herring
+  - Masami Hiramatsu
+  - Michal Hocko
+  - Simon Horman
+  - Johan Hovold (Hovold Consulting AB)
+  - Christophe JAILLET
+  - Olof Johansson
+  - Lee Jones (Linaro)
+  - Heiner Kallweit
+  - Srinivas Kandagatla
+  - Jan Kara
+  - Shuah Khan (Samsung)
+  - David Kershner
+  - Jaegeuk Kim
+  - Namhyung Kim
+  - Colin Ian King
+  - Jeff Kirsher
+  - Greg Kroah-Hartman (Linux Foundation)
+  - Christian König
+  - Vinod Koul
+  - Krzysztof Kozlowski
+  - Viresh Kumar
+  - Aneesh Kumar K.V
+  - Julia Lawall
+  - Doug Ledford
+  - Chuck Lever (Oracle)
+  - Daniel Lezcano
+  - Shaohua Li
+  - Xin Long
+  - Tony Luck
+  - Catalin Marinas (Arm Ltd)
+  - Mike Marshall
+  - Chris Mason
+  - Paul E. McKenney
+  - Arnaldo Carvalho de Melo
+  - David S. Miller
+  - Ingo Molnar
+  - Kuninori Morimoto
+  - Trond Myklebust
+  - Martin K. Petersen (Oracle)
+  - Borislav Petkov
+  - Jiri Pirko
+  - Josh Poimboeuf
+  - Sebastian Reichel (Collabora)
+  - Guenter Roeck
+  - Joerg Roedel
+  - Leon Romanovsky
+  - Steven Rostedt (VMware)
+  - Frank Rowand
+  - Ivan Safonov
+  - Anna Schumaker
+  - Jes Sorensen
+  - K.Y. Srinivasan
+  - David Sterba (SUSE)
+  - Heiko Stuebner
+  - Jiri Kosina (SUSE)
+  - Willy Tarreau
+  - Dmitry Torokhov
+  - Linus Torvalds
+  - Thierry Reding
+  - Rik van Riel
+  - Luis R. Rodriguez
+  - Geert Uytterhoeven (Glider bvba)
+  - Eduardo Valentin (Amazon.com)
+  - Daniel Vetter
+  - Linus Walleij
+  - Richard Weinberger
+  - Dan Williams
+  - Rafael J. Wysocki
+  - Arvind Yadav
+  - Masahiro Yamada
+  - Wei Yongjun
+  - Lv Zheng
+  - Marc Zyngier (Arm Ltd)
diff --git a/Documentation/translations/it_IT/process/license-rules.rst b/Documentation/translations/it_IT/process/license-rules.rst
new file mode 100644
index 000000000000..91a8794ffd79
--- /dev/null
+++ b/Documentation/translations/it_IT/process/license-rules.rst
@@ -0,0 +1,452 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+.. include:: ../disclaimer-ita.rst
+
+:Original: :ref:`Documentation/process/license-rules.rst <kernel_licensing>`
+:Translator: Federico Vaga <federico.vaga@vaga.pv.it>
+
+.. _it_kernel_licensing:
+
+Regole per licenziare il kernel Linux
+=====================================
+
+Il kernel Linux viene rilasciato sotto i termini definiti dalla seconda
+versione della licenza *GNU General Public License* (GPL-2.0), di cui una
+copia è disponibile nel file LICENSES/preferred/GPL-2.0; a questo si
+aggiunge eccezione per le chiamate di sistema come descritto in
+LICENSES/exceptions/Linux-syscall-note; tutto ciò è descritto nel file COPYING.
+
+Questo documento fornisce una descrizione su come ogni singolo file sorgente
+debba essere licenziato per far si che sia chiaro e non ambiguo. Questo non
+sostituisce la licenza del kernel.
+
+La licenza descritta nel file COPYING si applica ai sorgenti del kernel nella
+loro interezza, quindi i singoli file sorgenti possono avere diverse licenze ma
+devono essere compatibili con la GPL-2.0::
+
+    GPL-1.0+  :  GNU General Public License v1.0 o successiva
+    GPL-2.0+  :  GNU General Public License v2.0 o successiva
+    LGPL-2.0  :  GNU Library General Public License v2
+    LGPL-2.0+ :  GNU Library General Public License v2 o successiva
+    LGPL-2.1  :  GNU Lesser General Public License v2.1
+    LGPL-2.1+ :  GNU Lesser General Public License v2.1 o successiva
+
+A parte questo, i singolo file possono essere forniti con una doppia licenza,
+per esempio con una delle varianti compatibili della GPL e alternativamente con
+una licenza permissiva come BSD, MIT eccetera.
+
+I file d'intestazione per l'API verso lo spazio utente (UAPI) descrivono
+le interfacce usate dai programmi, e per questo sono un caso speciale.
+Secondo le note nel file COPYING, le chiamate di sistema sono un chiaro
+confine oltre il quale non si estendono i requisiti della GPL per quei
+programmi che le usano per comunicare con il kernel.  Dato che i file
+d'intestazione UAPI devono poter essere inclusi nei sorgenti di un
+qualsiasi programma eseguibile sul kernel Linux, questi meritano
+un'eccezione documentata da una clausola speciale.
+
+Il modo più comune per indicare la licenza dei file sorgenti è quello di
+aggiungere il corrispondente blocco di testo come commento in testa a detto
+file.  Per via della formattazione, dei refusi, eccetera, questi blocchi di
+testo sono difficili da identificare dagli strumenti usati per verificare il
+rispetto delle licenze.
+
+Un'alternativa ai blocchi di testo è data dall'uso degli identificatori
+*Software Package Data Exchange* (SPDX) in ogni file sorgente.  Gli
+identificatori di licenza SPDX sono analizzabili dalle macchine e sono precisi
+simboli stenografici che identificano la licenza sotto la quale viene
+licenziato il file che lo include.  Gli identificatori di licenza SPDX sono
+gestiti del gruppo di lavoro SPDX presso la Linux Foundation e sono stati
+concordati fra i soci nell'industria, gli sviluppatori di strumenti, e i
+rispettivi gruppi legali. Per maggiori informazioni, consultate
+https://spdx.org/
+
+Il kernel Linux richiede un preciso identificatore SPDX in tutti i file
+sorgenti.  Gli identificatori validi verranno spiegati nella sezione
+`Identificatori di licenza`_ e sono stati copiati dalla lista ufficiale di
+licenze SPDX assieme al rispettivo testo come mostrato in
+https://spdx.org/licenses/.
+
+Sintassi degli identificatori di licenza
+----------------------------------------
+
+1. Posizionamento:
+
+   L'identificativo di licenza SPDX dev'essere posizionato come prima riga
+   possibile di un file che possa contenere commenti.  Per la maggior parte
+   dei file questa è la prima riga, fanno eccezione gli script che richiedono
+   come prima riga '#!PATH_TO_INTERPRETER'.  Per questi script l'identificativo
+   SPDX finisce nella seconda riga.
+
+|
+
+2. Stile:
+
+   L'identificativo di licenza SPDX viene aggiunto sotto forma di commento.
+   Lo stile del commento dipende dal tipo di file::
+
+      sorgenti C:	// SPDX-License-Identifier: <SPDX License Expression>
+      intestazioni C:	/* SPDX-License-Identifier: <SPDX License Expression> */
+      ASM:	/* SPDX-License-Identifier: <SPDX License Expression> */
+      scripts:	# SPDX-License-Identifier: <SPDX License Expression>
+      .rst:	.. SPDX-License-Identifier: <SPDX License Expression>
+      .dts{i}:	// SPDX-License-Identifier: <SPDX License Expression>
+
+   Se un particolare programma non dovesse riuscire a gestire lo stile
+   principale per i commenti, allora dev'essere usato il meccanismo accettato
+   dal programma.  Questo è il motivo per cui si ha "/\* \*/" nei file
+   d'intestazione C.  Notammo che 'ld' falliva nell'analizzare i commenti del
+   C++ nei file .lds che venivano prodotti.  Oggi questo è stato corretto,
+   ma ci sono in giro ancora vecchi programmi che non sono in grado di
+   gestire lo stile dei commenti del C++.
+
+|
+
+3. Sintassi:
+
+   Una <espressione di licenza SPDX> può essere scritta usando l'identificatore
+   SPDX della licenza come indicato nella lista di licenze SPDX, oppure la
+   combinazione di due identificatori SPDX separati da "WITH" per i casi
+   eccezionali. Quando si usano più licenze l'espressione viene formata da
+   sottoespressioni separate dalle parole chiave "AND", "OR" e racchiuse fra
+   parentesi tonde "(", ")".
+
+   Gli identificativi di licenza per licenze come la [L]GPL che si avvalgono
+   dell'opzione 'o successive' si formano aggiungendo alla fine il simbolo "+"
+   per indicare l'opzione 'o successive'.::
+
+      // SPDX-License-Identifier: GPL-2.0+
+      // SPDX-License-Identifier: LGPL-2.1+
+
+   WITH dovrebbe essere usato quando sono necessarie delle modifiche alla
+   licenza.  Per esempio, la UAPI del kernel linux usa l'espressione::
+
+      // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
+      // SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note
+
+   Altri esempi di usi di WITH all'interno del kernel sono::
+
+      // SPDX-License-Identifier: GPL-2.0 WITH mif-exception
+      // SPDX-License-Identifier: GPL-2.0+ WITH GCC-exception-2.0
+
+   Le eccezioni si possono usare solo in combinazione con identificatori di
+   licenza. Gli identificatori di licenza riconosciuti sono elencati nei
+   corrispondenti file d'eccezione. Per maggiori dettagli consultate
+   `Eccezioni`_ nel capitolo `Identificatori di licenza`_
+
+   La parola chiave OR dovrebbe essere usata solo quando si usa una doppia
+   licenza e solo una dev'essere scelta.  Per esempio, alcuni file dtsi sono
+   disponibili con doppia licenza::
+
+      // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+
+   Esempi dal kernel di espressioni per file licenziati con doppia licenza
+   sono::
+
+      // SPDX-License-Identifier: GPL-2.0 OR MIT
+      // SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
+      // SPDX-License-Identifier: GPL-2.0 OR Apache-2.0
+      // SPDX-License-Identifier: GPL-2.0 OR MPL-1.1
+      // SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) OR MIT
+      // SPDX-License-Identifier: GPL-1.0+ OR BSD-3-Clause OR OpenSSL
+
+   La parola chiave AND dovrebbe essere usata quando i termini di più licenze
+   si applicano ad un file. Per esempio, quando il codice viene preso da
+   un altro progetto il quale da i permessi per aggiungerlo nel kernel ma
+   richiede che i termini originali della licenza rimangano intatti::
+
+      // SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) AND MIT
+
+   Di seguito, un altro esempio dove entrambe i termini di licenza devono
+   essere rispettati::
+
+      // SPDX-License-Identifier: GPL-1.0+ AND LGPL-2.1+
+
+Identificatori di licenza
+-------------------------
+
+Le licenze attualmente in uso, così come le licenze aggiunte al kernel, possono
+essere categorizzate in:
+
+1. _`Licenze raccomandate`:
+
+   Ovunque possibile le licenze qui indicate dovrebbero essere usate perché
+   pienamente compatibili e molto usate.  Queste licenze sono disponibile nei
+   sorgenti del kernel, nella cartella::
+
+     LICENSES/preferred/
+
+   I file in questa cartella contengono il testo completo della licenza e i
+   `Metatag`_.  Il nome di questi file è lo stesso usato come identificatore
+   di licenza SPDX e che deve essere usato nei file sorgenti.
+
+   Esempi::
+
+     LICENSES/preferred/GPL-2.0
+
+   Contiene il testo della seconda versione della licenza GPL e i metatag
+   necessari::
+
+     LICENSES/preferred/MIT
+
+   Contiene il testo della licenza MIT e i metatag necessari.
+
+   _`Metatag`:
+
+   I seguenti metatag devono essere presenti in un file di licenza:
+
+   - Valid-License-Identifier:
+
+     Una o più righe che dichiarano quali identificatori di licenza sono validi
+     all'interno del progetto per far riferimento alla licenza in questione.
+     Solitamente, questo è un unico identificatore valido, ma per esempio le
+     licenze che permettono l'opzione 'o successive' hanno due identificatori
+     validi.
+
+   - SPDX-URL:
+
+     L'URL della pagina SPDX che contiene informazioni aggiuntive riguardanti
+     la licenza.
+
+   - Usage-Guidance:
+
+     Testo in formato libero per dare suggerimenti agli utenti. Il testo deve
+     includere degli esempi su come usare gli identificatori di licenza SPDX
+     in un file sorgente in conformità con le linea guida in
+     `Sintassi degli identificatori di licenza`_.
+
+   - License-Text:
+
+     Tutto il testo che compare dopo questa etichetta viene trattato
+     come se fosse parte del testo originale della licenza.
+
+   Esempi::
+
+      Valid-License-Identifier: GPL-2.0
+      Valid-License-Identifier: GPL-2.0+
+      SPDX-URL: https://spdx.org/licenses/GPL-2.0.html
+      Usage-Guide:
+        To use this license in source code, put one of the following SPDX
+	tag/value pairs into a comment according to the placement
+	guidelines in the licensing rules documentation.
+	For 'GNU General Public License (GPL) version 2 only' use:
+	  SPDX-License-Identifier: GPL-2.0
+	For 'GNU General Public License (GPL) version 2 or any later version' use:
+	  SPDX-License-Identifier: GPL-2.0+
+      License-Text:
+        Full license text
+
+   ::
+
+      SPDX-License-Identifier: MIT
+      SPDX-URL: https://spdx.org/licenses/MIT.html
+      Usage-Guide:
+	To use this license in source code, put the following SPDX
+	tag/value pair into a comment according to the placement
+	guidelines in the licensing rules documentation.
+	  SPDX-License-Identifier: MIT
+      License-Text:
+        Full license text
+
+|
+
+2. Licenze non raccomandate:
+
+   Questo tipo di licenze dovrebbero essere usate solo per codice già esistente
+   o quando si prende codice da altri progetti.  Le licenze sono disponibili
+   nei sorgenti del kernel nella cartella::
+
+     LICENSES/other/
+
+   I file in questa cartella contengono il testo completo della licenza e i
+   `Metatag`_.  Il nome di questi file è lo stesso usato come identificatore
+   di licenza SPDX e che deve essere usato nei file sorgenti.
+
+   Esempi::
+
+     LICENSES/other/ISC
+
+   Contiene il testo della licenza Internet System Consortium e i suoi
+   metatag::
+
+     LICENSES/other/ZLib
+
+   Contiene il testo della licenza ZLIB e i suoi metatag.
+
+   Metatag:
+
+   I metatag necessari per le 'altre' ('other') licenze sono gli stessi
+   di usati per le `Licenze raccomandate`_.
+
+   Esempio del formato del file::
+
+      Valid-License-Identifier: ISC
+      SPDX-URL: https://spdx.org/licenses/ISC.html
+      Usage-Guide:
+        Usage of this license in the kernel for new code is discouraged
+        and it should solely be used for importing code from an already
+        existing project.
+        To use this license in source code, put the following SPDX
+        tag/value pair into a comment according to the placement
+        guidelines in the licensing rules documentation.
+          SPDX-License-Identifier: ISC
+      License-Text:
+        Full license text
+
+|
+
+3. _`Eccezioni`:
+
+   Alcune licenze possono essere corrette con delle eccezioni che forniscono
+   diritti aggiuntivi.  Queste eccezioni sono disponibili nei sorgenti del
+   kernel nella cartella::
+
+     LICENSES/exceptions/
+
+   I file in questa cartella contengono il testo completo dell'eccezione e i
+   `Metatag per le eccezioni`_.
+
+   Esempi::
+
+      LICENSES/exceptions/Linux-syscall-note
+
+   Contiene la descrizione dell'eccezione per le chiamate di sistema Linux
+   così come documentato nel file COPYING del kernel Linux; questo viene usato
+   per i file d'intestazione per la UAPI.  Per esempio
+   /\* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note \*/::
+
+      LICENSES/exceptions/GCC-exception-2.0
+
+   Contiene la 'eccezione di linking' che permette di collegare qualsiasi
+   binario, indipendentemente dalla sua licenza, con un compilato il cui file
+   sorgente è marchiato con questa eccezione. Questo è necessario per creare
+   eseguibili dai sorgenti che non sono compatibili con la GPL.
+
+   _`Metatag per le eccezioni`:
+
+   Un file contenente un'eccezione deve avere i seguenti metatag:
+
+   - SPDX-Exception-Identifier:
+
+     Un identificatore d'eccezione che possa essere usato in combinazione con
+     un identificatore di licenza SPDX.
+
+   - SPDX-URL:
+
+     L'URL della pagina SPDX che contiene informazioni aggiuntive riguardanti
+     l'eccezione.
+
+   - SPDX-Licenses:
+
+     Una lista di licenze SPDX separate da virgola, che possono essere usate
+     con l'eccezione.
+
+   - Usage-Guidance:
+
+     Testo in formato libero per dare suggerimenti agli utenti. Il testo deve
+     includere degli esempi su come usare gli identificatori di licenza SPDX
+     in un file sorgente in conformità con le linea guida in
+     `Sintassi degli identificatori di licenza`_.
+
+   - Exception-Text:
+
+     Tutto il testo che compare dopo questa etichetta viene trattato
+     come se fosse parte del testo originale della licenza.
+
+   Esempi::
+
+      SPDX-Exception-Identifier: Linux-syscall-note
+      SPDX-URL: https://spdx.org/licenses/Linux-syscall-note.html
+      SPDX-Licenses: GPL-2.0, GPL-2.0+, GPL-1.0+, LGPL-2.0, LGPL-2.0+, LGPL-2.1, LGPL-2.1+
+      Usage-Guidance:
+        This exception is used together with one of the above SPDX-Licenses
+	to mark user-space API (uapi) header files so they can be included
+	into non GPL compliant user-space application code.
+        To use this exception add it with the keyword WITH to one of the
+	identifiers in the SPDX-Licenses tag:
+	  SPDX-License-Identifier: <SPDX-License> WITH Linux-syscall-note
+      Exception-Text:
+        Full exception text
+
+   ::
+
+      SPDX-Exception-Identifier: GCC-exception-2.0
+      SPDX-URL: https://spdx.org/licenses/GCC-exception-2.0.html
+      SPDX-Licenses: GPL-2.0, GPL-2.0+
+      Usage-Guidance:
+        The "GCC Runtime Library exception 2.0" is used together with one
+	of the above SPDX-Licenses for code imported from the GCC runtime
+	library.
+        To use this exception add it with the keyword WITH to one of the
+	identifiers in the SPDX-Licenses tag:
+	  SPDX-License-Identifier: <SPDX-License> WITH GCC-exception-2.0
+      Exception-Text:
+        Full exception text
+
+Per ogni identificatore di licenza SPDX e per le eccezioni dev'esserci un file
+nella sotto-cartella LICENSES.  Questo è necessario per permettere agli
+strumenti di effettuare verifiche (come checkpatch.pl), per avere le licenze
+disponibili per la lettura e per estrarre i diritti dai sorgenti, così come
+raccomandato da diverse organizzazioni FOSS, per esempio l'`iniziativa FSFE
+REUSE <https://reuse.software/>`_.
+
+_`MODULE_LICENSE`
+-----------------
+
+   I moduli del kernel necessitano di un'etichetta MODULE_LICENSE(). Questa
+   etichetta non sostituisce le informazioni sulla licenza del codice sorgente
+   (SPDX-License-Identifier) né fornisce informazioni che esprimono o
+   determinano l'esatta licenza sotto la quale viene rilasciato.
+
+   Il solo scopo di questa etichetta è quello di fornire sufficienti
+   informazioni al caricatore di moduli del kernel, o agli strumenti in spazio
+   utente, per capire se il modulo è libero o proprietario.
+
+   Le stringe di licenza valide per MODULE_LICENSE() sono:
+
+    ============================= =============================================
+    "GPL"			  Il modulo è licenziato con la GPL versione 2.
+				  Questo non fa distinzione fra GPL'2.0-only o
+				  GPL-2.0-or-later. L'esatta licenza può essere
+				  determinata solo leggendo i corrispondenti
+				  file sorgenti.
+
+    "GPL v2"			  Stesso significato di "GPL". Esiste per
+				  motivi storici.
+
+    "GPL and additional rights"   Questa è una variante che esiste per motivi
+				  storici che indica che i sorgenti di un
+				  modulo sono rilasciati sotto una variante
+				  della licenza GPL v2 e quella MIT. Per favore
+				  non utilizzatela per codice nuovo.
+
+    "Dual MIT/GPL"		  Questo è il modo corretto per esprimere il
+				  il fatto che il modulo è rilasciato con
+				  doppia licenza a scelta fra: una variante
+				  della GPL v2 o la licenza MIT.
+
+    "Dual BSD/GPL"		  Questo modulo è rilasciato con doppia licenza
+				  a scelta fra: una variante della GPL v2 o la
+				  licenza BSD. La variante esatta della licenza
+				  BSD può essere determinata solo attraverso i
+				  corrispondenti file sorgenti.
+
+    "Dual MPL/GPL"		  Questo modulo è rilasciato con doppia licenza
+				  a scelta fra: una variante della GPL v2 o la
+				  Mozilla Public License (MPL). La variante
+				  esatta della licenza MPL può essere
+				  determinata solo attraverso i corrispondenti
+				  file sorgenti.
+
+    "Proprietary"		  Questo modulo è rilasciato con licenza
+				  proprietaria. Questa stringa è solo per i
+				  moduli proprietari di terze parti e non può
+				  essere usata per quelli che risiedono nei
+				  sorgenti del kernel. I moduli etichettati in
+				  questo modo stanno contaminando il kernel e
+				  gli viene assegnato un flag 'P'; quando
+				  vengono caricati, il caricatore di moduli del
+				  kernel si rifiuterà di collegare questi
+				  moduli ai simboli che sono stati esportati
+				  con EXPORT_SYMBOL_GPL().
+
+    ============================= =============================================
diff --git a/Documentation/translations/it_IT/process/maintainer-pgp-guide.rst b/Documentation/translations/it_IT/process/maintainer-pgp-guide.rst
index 24a133f0a51d..276db0e37f43 100644
--- a/Documentation/translations/it_IT/process/maintainer-pgp-guide.rst
+++ b/Documentation/translations/it_IT/process/maintainer-pgp-guide.rst
@@ -1,13 +1,946 @@
 .. include:: ../disclaimer-ita.rst
 
 :Original: :ref:`Documentation/process/maintainer-pgp-guide.rst <pgpguide>`
+:Translator: Alessia Mantegazza <amantegazza@vaga.pv.it>
 
 .. _it_pgpguide:
 
+=========================================
+La guida a PGP per manutentori del kernel
+=========================================
+
+:Author: Konstantin Ryabitsev <konstantin@linuxfoundation.org>
+
+Questo documento è destinato agli sviluppatori del kernel Linux, in particolar
+modo ai manutentori. Contiene degli approfondimenti riguardo informazioni che
+sono state affrontate in maniera più generale nella sezione
+"`Protecting Code Integrity`_" pubblicata dalla Linux Foundation.
+Per approfondire alcuni argomenti trattati in questo documento è consigliato
+leggere il documento sopraindicato
+
+.. _`Protecting Code Integrity`: https://github.com/lfit/itpol/blob/master/protecting-code-integrity.md
+
+Il ruolo di PGP nello sviluppo del kernel Linux
+===============================================
+
+PGP aiuta ad assicurare l'integrità del codice prodotto dalla comunità
+di sviluppo del kernel e, in secondo luogo, stabilisce canali di comunicazione
+affidabili tra sviluppatori attraverso lo scambio di email firmate con PGP.
+
+Il codice sorgente del kernel Linux è disponibile principalmente in due
+formati:
+
+- repositori distribuiti di sorgenti (git)
+- rilasci periodici di istantanee (archivi tar)
+
+Sia i repositori git che gli archivi tar portano le firme PGP degli
+sviluppatori che hanno creato i rilasci ufficiali del kernel. Queste firme
+offrono una garanzia crittografica che le versioni scaricabili rese disponibili
+via kernel.org, o altri portali, siano identiche a quelle che gli sviluppatori
+hanno sul loro posto di lavoro. A tal scopo:
+
+- i repositori git forniscono firme PGP per ogni tag
+- gli archivi tar hanno firme separate per ogni archivio
+
+.. _it_devs_not_infra:
+
+Fidatevi degli sviluppatori e non dell'infrastruttura
+-----------------------------------------------------
+
+Fin dal 2011, quando i sistemi di kernel.org furono compromessi, il principio
+generale del progetto Kernel Archives è stato quello di assumere che qualsiasi
+parte dell'infrastruttura possa essere compromessa in ogni momento. Per questa
+ragione, gli amministratori hanno intrapreso deliberatemene dei passi per
+enfatizzare che la fiducia debba risiedere sempre negli sviluppatori e mai nel
+codice che gestisce l'infrastruttura, indipendentemente da quali che siano le
+pratiche di sicurezza messe in atto.
+
+Il principio sopra indicato è la ragione per la quale è necessaria questa
+guida. Vogliamo essere sicuri che il riporre la fiducia negli sviluppatori
+non sia fatto semplicemente per incolpare qualcun'altro per future falle di
+sicurezza. L'obiettivo è quello di fornire una serie di linee guida che gli
+sviluppatori possano seguire per creare un ambiente di lavoro sicuro e
+salvaguardare le chiavi PGP usate nello stabilire l'integrità del kernel Linux
+stesso.
+
+.. _it_pgp_tools:
+
+Strumenti PGP
+=============
+
+Usare GnuPG v2
+--------------
+
+La vostra distribuzione potrebbe avere già installato GnuPG, dovete solo
+verificare che stia utilizzando la versione 2.x e non la serie 1.4 --
+molte distribuzioni forniscono entrambe, di base il comando ''gpg''
+invoca GnuPG v.1. Per controllate usate::
+
+    $ gpg --version | head -n1
+
+Se visualizzate ``gpg (GnuPG) 1.4.x``, allora state usando GnuPG v.1.
+Provate il comando ``gpg2`` (se non lo avete, potreste aver bisogno
+di installare il pacchetto gnupg2)::
+
+    $ gpg2 --version | head -n1
+
+Se visualizzate  ``gpg (GnuPG) 2.x.x``, allora siete pronti a partire.
+Questa guida assume che abbiate la versione 2.2.(o successiva) di GnuPG.
+Se state usando la versione 2.0, alcuni dei comandi indicati qui non
+funzioneranno, in questo caso considerate un aggiornamento all'ultima versione,
+la 2.2. Versioni di gnupg-2.1.11 e successive dovrebbero essere compatibili
+per gli obiettivi di questa guida.
+
+Se avete entrambi i comandi: ``gpg`` e ``gpg2``, assicuratevi di utilizzare
+sempre la versione V2, e non quella vecchia. Per evitare errori potreste creare
+un alias::
+
+    $ alias gpg=gpg2
+
+Potete mettere questa opzione nel vostro  ``.bashrc`` in modo da essere sicuri.
+
+Configurare le opzioni di gpg-agent
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+L'agente GnuPG è uno strumento di aiuto che partirà automaticamente ogni volta
+che userete il comando ``gpg`` e funzionerà in background con l'obiettivo di
+individuare la passphrase. Ci sono due opzioni che dovreste conoscere
+per personalizzare la scadenza della passphrase nella cache:
+
+- ``default-cache-ttl`` (secondi): Se usate ancora la stessa chiave prima
+  che il time-to-live termini, il conto alla rovescia si resetterà per un
+  altro periodo. Di base è di 600 (10 minuti).
+
+- ``max-cache-ttl`` (secondi): indipendentemente da quanto sia recente l'ultimo
+  uso della chiave da quando avete inserito la passphrase, se il massimo
+  time-to-live è scaduto, dovrete reinserire nuovamente la passphrase.
+  Di base è di 30 minuti.
+
+Se ritenete entrambe questi valori di base troppo corti (o troppo lunghi),
+potete creare il vostro file ``~/.gnupg/gpg-agent.conf`` ed impostare i vostri
+valori::
+
+    # set to 30 minutes for regular ttl, and 2 hours for max ttl
+    default-cache-ttl 1800
+    max-cache-ttl 7200
+
+.. note::
+
+    Non è più necessario far partire l'agente gpg manualmente all'inizio della
+    vostra sessione. Dovreste controllare i file rc per rimuovere tutto ciò che
+    riguarda vecchie le versioni di GnuPG, poiché potrebbero non svolgere più
+    bene il loro compito.
+
+Impostare un *refresh* con cronjob
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Potreste aver bisogno di rinfrescare regolarmente il vostro portachiavi in
+modo aggiornare le chiavi pubbliche di altre persone, lavoro che è svolto
+al meglio con un cronjob giornaliero::
+
+    @daily /usr/bin/gpg2 --refresh >/dev/null 2>&1
+
+Controllate il percorso assoluto del vostro comando ``gpg`` o ``gpg2`` e usate
+il comando ``gpg2`` se per voi ``gpg`` corrisponde alla versione GnuPG v.1.
+
+.. _it_master_key:
+
+Proteggere la vostra chiave PGP primaria
 ========================================
-Guida a PGP per i manutentori del kernel
-========================================
+
+Questa guida parte dal presupposto che abbiate già una chiave PGP che usate
+per lo sviluppo del kernel Linux. Se non ne avete ancora una, date uno sguardo
+al documento "`Protecting Code Integrity`_" che abbiamo menzionato prima.
+
+Dovreste inoltre creare una nuova chiave se quella attuale è inferiore a 2048
+bit (RSA).
+
+Chiave principale o sottochiavi
+-------------------------------
+
+Le sottochiavi sono chiavi PGP totalmente indipendenti, e sono collegate alla
+chiave principale attraverso firme certificate. È quindi importante
+comprendere i seguenti punti:
+
+1. Non ci sono differenze tecniche tra la chiave principale e la sottochiave.
+2. In fesa di creazione, assegniamo limitazioni funzionali ad ogni chiave
+   assegnando capacità specifiche.
+3. Una chiave PGP può avere 4 capacità:
+
+   - **[S]** può essere usata per firmare
+   - **[E]** può essere usata per criptare
+   - **[A]** può essere usata per autenticare
+   - **[C]** può essere usata per certificare altre chiavi
+
+4. Una singola chiave può avere più capacità
+5. Una sottochiave è completamente indipendente dalla chiave principale.
+   Un messaggio criptato con la sottochiave non può essere decrittato con
+   quella principale. Se perdete la vostra sottochiave privata, non può
+   essere rigenerata in nessun modo da quella principale.
+
+La chiave con capacità **[C]** (certify) è identificata come la chiave
+principale perché è l'unica che può essere usata per indicare la relazione
+con altre chiavi. Solo la chiave **[C]** può essere usata per:
+
+- Aggiungere o revocare altre chiavi (sottochiavi) che hanno capacità S/E/A
+- Aggiungere, modificare o eliminare le identità (unids) associate alla chiave
+- Aggiungere o modificare la data di termine di sé stessa o di ogni sottochiave
+- Firmare le chiavi di altre persone a scopo di creare una rete di fiducia
+
+Di base, alla creazione di nuove chiavi, GnuPG genera quanto segue:
+
+- Una chiave madre che porta sia la capacità di certificazione che quella
+  di firma (**[SC]**)
+- Una sottochiave separata con capacità di criptaggio (**[E]**)
+
+Se avete usato i parametri di base per generare la vostra chiave, quello
+sarà il risultato. Potete verificarlo utilizzando ``gpg --list-secret-keys``,
+per esempio::
+
+    sec   rsa2048 2018-01-23 [SC] [expires: 2020-01-23]
+          000000000000000000000000AAAABBBBCCCCDDDD
+    uid           [ultimate] Alice Dev <adev@kernel.org>
+    ssb   rsa2048 2018-01-23 [E] [expires: 2020-01-23]
+
+Qualsiasi chiave che abbia la capacità **[C]** è la vostra chiave madre,
+indipendentemente da quali altre capacità potreste averle assegnato.
+
+La lunga riga sotto la voce ``sec`` è la vostra impronta digitale --
+negli esempi che seguono, quando vedere ``[fpr]`` ci si riferisce a questa
+stringa di 40 caratteri.
+
+Assicuratevi che la vostra passphrase sia forte
+-----------------------------------------------
+
+GnuPG utilizza le passphrases per criptare la vostra chiave privata prima
+di salvarla sul disco. In questo modo, anche se il contenuto della vostra
+cartella ``.gnupg`` venisse letto o trafugato nella sia interezza, gli
+attaccanti non potrebbero comunque utilizzare le vostre chiavi private senza
+aver prima ottenuto la passphrase per decriptarle.
+
+È assolutamente essenziale che le vostre chiavi private siano protette da
+una passphrase forte. Per impostarla o cambiarla, usate::
+
+    $ gpg --change-passphrase [fpr]
+
+Create una sottochiave di firma separata
+----------------------------------------
+
+Il nostro obiettivo è di proteggere la chiave primaria spostandola su un
+dispositivo sconnesso dalla rete, dunque se avete solo una chiave combinata
+**[SC]** allora dovreste creare una sottochiave di firma separata::
+
+    $ gpg --quick-add-key [fpr] ed25519 sign
+
+Ricordate di informare il keyserver del vostro cambiamento, cosicché altri
+possano ricevere la vostra nuova sottochiave::
+
+    $ gpg --send-key [fpr]
+
+.. note:: Supporto ECC in GnuPG
+    GnuPG 2.1 e successivi supportano pienamente *Elliptic Curve Cryptography*,
+    con la possibilità di combinare sottochiavi ECC con le tradizionali chiavi
+    primarie RSA. Il principale vantaggio della crittografia ECC è che è molto
+    più veloce da calcolare e crea firme più piccole se confrontate byte per
+    byte con le chiavi RSA a più di 2048 bit. A meno che non pensiate di
+    utilizzare un dispositivo smartcard che non supporta le operazioni ECC, vi
+    raccomandiamo ti creare sottochiavi di firma ECC per il vostro lavoro col
+    kernel.
+
+    Se per qualche ragione preferite rimanere con sottochiavi RSA, nel comando
+    precedente, sostituite "ed25519" con "rsa2048".
+
+Copia di riserva della chiave primaria per gestire il recupero da disastro
+--------------------------------------------------------------------------
+
+Maggiori sono le firme di altri sviluppatori che vengono applicate alla vostra,
+maggiori saranno i motivi per avere una copia di riserva che non sia digitale,
+al fine di effettuare un recupero da disastro.
+
+Il modo migliore per creare una copia fisica della vostra chiave privata è
+l'uso del programma ``paperkey``. Consultate ``man paperkey`` per maggiori
+dettagli sul formato dell'output ed i suoi punti di forza rispetto ad altre
+soluzioni. Paperkey dovrebbe essere già pacchettizzato per la maggior parte
+delle distribuzioni.
+
+Eseguite il seguente comando per creare una copia fisica di riserva della
+vostra chiave privata::
+
+    $ gpg --export-secret-key [fpr] | paperkey -o /tmp/key-backup.txt
+
+Stampate il file (o fate un pipe direttamente verso lpr), poi prendete
+una penna e scrivete la passphare sul margine del foglio.  **Questo è
+caldamente consigliato** perché la copia cartacea è comunque criptata con
+la passphrase, e se mai doveste cambiarla non vi ricorderete qual'era al
+momento della creazione di quella copia -- *garantito*.
+
+Mettete la copia cartacea e la passphrase scritta a mano in una busta e
+mettetela in un posto sicuro e ben protetto, preferibilmente fuori casa,
+magari in una cassetta di sicurezza in banca.
+
+.. note::
+
+    Probabilmente la vostra stampante non è più quello stupido dispositivo
+    connesso alla porta parallela, ma dato che il suo output è comunque
+    criptato con la passphrase, eseguire la stampa in un sistema "cloud"
+    moderno dovrebbe essere comunque relativamente sicuro. Un'opzione potrebbe
+    essere quella di cambiare la passphrase della vostra chiave primaria
+    subito dopo aver finito con paperkey.
+
+Copia di riserva di tutta la cartella GnuPG
+-------------------------------------------
 
 .. warning::
 
-    TODO ancora da tradurre
+    **!!!Non saltate questo passo!!!**
+
+Quando avete bisogno di recuperare le vostre chiavi PGP è importante avere
+una copia di riserva pronta all'uso. Questo sta su un diverso piano di
+prontezza rispetto al recupero da disastro che abbiamo risolto con
+``paperkey``. Vi affiderete a queste copie esterne quando dovreste usare la
+vostra chiave Certify -- ovvero quando fate modifiche alle vostre chiavi o
+firmate le chiavi di altre persone ad una conferenza o ad un gruppo d'incontro.
+
+Incominciate con una piccola chiavetta di memoria USB (preferibilmente due)
+che userete per le copie di riserva. Dovrete criptarle usando LUKS -- fate
+riferimento alla documentazione della vostra distribuzione per capire come
+fare.
+
+Per la passphrase di criptazione, potete usare la stessa della vostra chiave
+primaria.
+
+Una volta che il processo di criptazione è finito, reinserite il disco USB ed
+assicurativi che venga montato correttamente. Copiate interamente la cartella
+``.gnugp`` nel disco criptato::
+
+    $ cp -a ~/.gnupg /media/disk/foo/gnupg-backup
+
+Ora dovreste verificare che tutto continui a funzionare::
+
+    $ gpg --homedir=/media/disk/foo/gnupg-backup --list-key [fpr]
+
+Se non vedete errori, allora dovreste avere fatto tutto con successo.
+Smontate il disco USB, etichettatelo per bene di modo da evitare di
+distruggerne il contenuto non appena vi serve una chiavetta USB a caso, ed
+infine mettetelo in un posto sicuro -- ma non troppo lontano, perché vi servirà
+di tanto in tanto per modificare le identità, aggiungere o revocare
+sottochiavi, o firmare le chiavi di altre persone.
+
+Togliete la chiave primaria dalla vostra home
+---------------------------------------------
+
+I file che si trovano nella vostra cartella home non sono poi così ben protetti
+come potreste pensare. Potrebbero essere letti o trafugati in diversi modi:
+
+- accidentalmente quando fate una rapida copia della cartella home per
+  configurare una nuova postazione
+- da un amministratore di sistema negligente o malintenzionato
+- attraverso copie di riserva insicure
+- attraverso malware installato in alcune applicazioni (browser, lettori PDF,
+  eccetera)
+- attraverso coercizione quando attraversate confini internazionali
+
+Proteggere la vostra chiave con una buona passphare aiuta notevolmente a
+ridurre i rischi elencati qui sopra, ma le passphrase possono essere scoperte
+attraverso i keylogger, il shoulder-surfing, o altri modi. Per questi motivi,
+nella configurazione si raccomanda di rimuove la chiave primaria dalla vostra
+cartella home e la si archivia su un dispositivo disconnesso.
+
+.. warning::
+
+    Per favore, fate riferimento alla sezione precedente e assicuratevi
+    di aver fatto una copia di riserva totale della cartella GnuPG. Quello
+    che stiamo per fare renderà la vostra chiave inutile se non avete delle
+    copie di riserva utilizzabili!
+
+Per prima cosa, identificate il keygrip della vostra chiave primaria::
+
+    $ gpg --with-keygrip --list-key [fpr]
+
+L'output assomiglierà a questo::
+
+    pub   rsa2048 2018-01-24 [SC] [expires: 2020-01-24]
+          000000000000000000000000AAAABBBBCCCCDDDD
+          Keygrip = 1111000000000000000000000000000000000000
+    uid           [ultimate] Alice Dev <adev@kernel.org>
+    sub   rsa2048 2018-01-24 [E] [expires: 2020-01-24]
+          Keygrip = 2222000000000000000000000000000000000000
+    sub   ed25519 2018-01-24 [S]
+          Keygrip = 3333000000000000000000000000000000000000
+
+Trovate la voce keygrid che si trova sotto alla riga ``pub`` (appena sotto
+all'impronta digitale della chiave primaria). Questo corrisponderà direttamente
+ad un file nella cartella ``~/.gnupg``::
+
+    $ cd ~/.gnupg/private-keys-v1.d
+    $ ls
+    1111000000000000000000000000000000000000.key
+    2222000000000000000000000000000000000000.key
+    3333000000000000000000000000000000000000.key
+
+Quello che dovrete fare è rimuovere il file .key che corrisponde al keygrip
+della chiave primaria::
+
+    $ cd ~/.gnupg/private-keys-v1.d
+    $ rm 1111000000000000000000000000000000000000.key
+
+Ora, se eseguite il comando ``--list-secret-keys``, vedrete che la chiave
+primaria non compare più (il simbolo ``#`` indica che non è disponibile)::
+
+    $ gpg --list-secret-keys
+    sec#  rsa2048 2018-01-24 [SC] [expires: 2020-01-24]
+          000000000000000000000000AAAABBBBCCCCDDDD
+    uid           [ultimate] Alice Dev <adev@kernel.org>
+    ssb   rsa2048 2018-01-24 [E] [expires: 2020-01-24]
+    ssb   ed25519 2018-01-24 [S]
+
+Dovreste rimuovere anche i file ``secring.gpg`` che si trovano nella cartella
+``~/.gnupg``, in quanto rimasugli delle versioni precedenti di GnuPG.
+
+Se non avete la cartella "private-keys-v1.d"
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Se non avete la cartella ``~/.gnupg/private-keys-v1.d``, allora le vostre
+chiavi segrete sono ancora salvate nel vecchio file ``secring.gpg`` usato
+da GnuPG v1. Effettuare una qualsiasi modifica alla vostra chiave, come
+cambiare la passphare o aggiungere una sottochiave, dovrebbe convertire
+automaticamente il vecchio formato ``secring.gpg``nel nuovo
+``private-keys-v1.d``.
+
+Una volta che l'avete fatto, assicuratevi di rimuovere il file ``secring.gpg``,
+che continua a contenere la vostra chiave privata.
+
+.. _it_smartcards:
+
+Spostare le sottochiavi in un apposito dispositivo criptato
+===========================================================
+
+Nonostante la chiave primaria sia ora al riparo da occhi e mani indiscrete,
+le sottochiavi si trovano ancora nella vostra cartella home. Chiunque riesca
+a mettere le sue mani su quelle chiavi riuscirà a decriptare le vostre
+comunicazioni o a falsificare le vostre firme (se conoscono la passphrase).
+Inoltre, ogni volta che viene fatta un'operazione con GnuPG, le chiavi vengono
+caricate nella memoria di sistema e potrebbero essere rubate con l'uso di
+malware sofisticati (pensate a Meltdown e a Spectre).
+
+Il miglior modo per proteggere le proprie chiave è di spostarle su un
+dispositivo specializzato in grado di effettuare operazioni smartcard.
+
+I benefici di una smartcard
+---------------------------
+
+Una smartcard contiene un chip crittografico che è capace di immagazzinare
+le chiavi private ed effettuare operazioni crittografiche direttamente sulla
+carta stessa. Dato che la chiave non lascia mai la smartcard, il sistema
+operativo usato sul computer non sarà in grado di accedere alle chiavi.
+Questo è molto diverso dai dischi USB criptati che abbiamo usato allo scopo di
+avere una copia di riserva sicura -- quando il dispositivo USB è connesso e
+montato, il sistema operativo potrà accedere al contenuto delle chiavi private.
+
+L'uso di un disco USB criptato non può sostituire le funzioni di un dispositivo
+capace di operazioni di tipo smartcard.
+
+Dispositivi smartcard disponibili
+---------------------------------
+
+A meno che tutti i vostri computer dispongano di lettori smartcard, il modo
+più semplice è equipaggiarsi di un dispositivo USB specializzato che
+implementi le funzionalità delle smartcard.  Sul mercato ci sono diverse
+soluzioni disponibili:
+
+- `Nitrokey Start`_: è Open hardware e Free Software, è basata sul progetto
+  `GnuK`_ della FSIJ. Ha il supporto per chiavi ECC, ma meno funzionalità di
+  sicurezza (come la resistenza alla manomissione o alcuni attacchi ad un
+  canale laterale).
+- `Nitrokey Pro`_: è simile alla Nitrokey Start, ma è più resistente alla
+  manomissione e offre più funzionalità di sicurezza, ma l'ECC.
+- `Yubikey 4`_: l'hardware e il software sono proprietari, ma è più economica
+  della  Nitrokey Pro ed è venduta anche con porta USB-C il che è utile con i
+  computer portatili più recenti. In aggiunta, offre altre funzionalità di
+  sicurezza come FIDO, U2F, ma non l'ECC
+
+`Su LWN c'è una buona recensione`_ dei modelli elencati qui sopra e altri.
+Se volete usare chiavi ECC, la vostra migliore scelta sul mercato è la
+Nitrokey Start.
+
+.. _`Nitrokey Start`: https://shop.nitrokey.com/shop/product/nitrokey-start-6
+.. _`Nitrokey Pro`: https://shop.nitrokey.com/shop/product/nitrokey-pro-3
+.. _`Yubikey 4`: https://www.yubico.com/product/yubikey-4-series/
+.. _Gnuk: http://www.fsij.org/doc-gnuk/
+.. _`Su LWN c'è una buona recensione`: https://lwn.net/Articles/736231/
+
+Configurare il vostro dispositivo smartcard
+-------------------------------------------
+
+Il vostro dispositivo smartcard dovrebbe iniziare a funzionare non appena
+lo collegate ad un qualsiasi computer Linux moderno. Potete verificarlo
+eseguendo::
+
+    $ gpg --card-status
+
+Se vedete tutti i dettagli della smartcard, allora ci siamo. Sfortunatamente,
+affrontare tutti i possibili motivi per cui le cose potrebbero non funzionare
+non è lo scopo di questa guida. Se avete problemi nel far funzionare la carta
+con GnuPG, cercate aiuto attraverso i soliti canali di supporto.
+
+Per configurare la vostra smartcard, dato che non c'è una via facile dalla
+riga di comando, dovrete usate il menu di GnuPG::
+
+    $ gpg --card-edit
+    [...omitted...]
+    gpg/card> admin
+    Admin commands are allowed
+    gpg/card> passwd
+
+Dovreste impostare il PIN dell'utente (1), quello dell'amministratore (3) e il
+codice di reset (4). Assicuratevi di annotare e salvare questi codici in un
+posto sicuro -- specialmente il PIN dell'amministratore e il codice di reset
+(che vi permetterà di azzerare completamente la smartcard).  Il PIN
+dell'amministratore viene usato così raramente che è inevitabile dimenticarselo
+se non lo si annota.
+
+Tornando al nostro menu, potete impostare anche altri valori (come il nome,
+il sesso, informazioni d'accesso, eccetera), ma non sono necessari e aggiunge
+altre informazioni sulla carta che potrebbero trapelare in caso di smarrimento.
+
+.. note::
+
+    A dispetto del nome "PIN", né il PIN utente né quello dell'amministratore
+    devono essere esclusivamente numerici.
+
+Spostare le sottochiavi sulla smartcard
+---------------------------------------
+
+Uscite dal menu (usando "q") e salverete tutte le modifiche. Poi, spostiamo
+tutte le sottochiavi sulla smartcard. Per la maggior parte delle operazioni
+vi serviranno sia la passphrase della chiave PGP che il PIN
+dell'amministratore::
+
+    $ gpg --edit-key [fpr]
+
+    Secret subkeys are available.
+
+    pub  rsa2048/AAAABBBBCCCCDDDD
+         created: 2018-01-23  expires: 2020-01-23  usage: SC
+         trust: ultimate      validity: ultimate
+    ssb  rsa2048/1111222233334444
+         created: 2018-01-23  expires: never       usage: E
+    ssb  ed25519/5555666677778888
+         created: 2017-12-07  expires: never       usage: S
+    [ultimate] (1). Alice Dev <adev@kernel.org>
+
+    gpg>
+
+Usando ``--edit-key`` si tornerà alla modalità menu e noterete che
+la lista delle chiavi è leggermente diversa. Da questo momento in poi,
+tutti i comandi saranno eseguiti nella modalità menu, come indicato
+da ``gpg>``.
+
+Per prima cosa, selezioniamo la chiave che verrà messa sulla carta --
+potete farlo digitando ``key 1`` (è la prima della lista, la sottochiave
+**[E]**)::
+
+    gpg> key 1
+
+Nel'output dovreste vedere ``ssb*`` associato alla chiave **[E]**. Il simbolo
+``*`` indica che la chiave è stata "selezionata". Funziona come un
+interruttore, ovvero se scrivete nuovamente ``key 1``, il simbolo ``*`` sparirà
+e la chiave non sarà più selezionata.
+
+Ora, spostiamo la chiave sulla smartcard::
+
+    gpg> keytocard
+    Please select where to store the key:
+       (2) Encryption key
+    Your selection? 2
+
+Dato che è la nostra chiave  **[E]**, ha senso metterla nella sezione criptata.
+Quando confermerete la selezione, vi verrà chiesta la passphrase della vostra
+chiave PGP, e poi il PIN dell'amministratore. Se il comando ritorna senza
+errori, allora la vostra chiave è stata spostata con successo.
+
+**Importante**: digitate nuovamente ``key 1`` per deselezionare la prima chiave
+e selezionate la seconda chiave **[S]** con ``key 2``::
+
+    gpg> key 1
+    gpg> key 2
+    gpg> keytocard
+    Please select where to store the key:
+       (1) Signature key
+       (3) Authentication key
+    Your selection? 1
+
+Potete usare la chiave **[S]** sia per firmare che per autenticare, ma vogliamo
+che sia nella sezione di firma, quindi scegliete (1). Ancora una volta, se il
+comando ritorna senza errori, allora l'operazione è avvenuta con successo::
+
+    gpg> q
+    Save changes? (y/N) y
+
+Salvando le modifiche cancellerete dalla vostra cartella home tutte le chiavi
+che avete spostato sulla carta (ma questo non è un problema, perché abbiamo
+fatto delle copie di sicurezza nel caso in cui dovessimo configurare una
+nuova smartcard).
+
+Verificare che le chiavi siano state spostate
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Ora, se doveste usare l'opzione ``--list-secret-keys``, vedrete una
+sottile differenza nell'output::
+
+    $ gpg --list-secret-keys
+    sec#  rsa2048 2018-01-24 [SC] [expires: 2020-01-24]
+          000000000000000000000000AAAABBBBCCCCDDDD
+    uid           [ultimate] Alice Dev <adev@kernel.org>
+    ssb>  rsa2048 2018-01-24 [E] [expires: 2020-01-24]
+    ssb>  ed25519 2018-01-24 [S]
+
+Il simbolo ``>`` in ``ssb>`` indica che la sottochiave è disponibile solo
+nella smartcard. Se tornate nella vostra cartella delle chiavi segrete e
+guardate al suo contenuto, noterete che i file ``.key`` sono stati sostituiti
+con degli stub::
+
+    $ cd ~/.gnupg/private-keys-v1.d
+    $ strings *.key | grep 'private-key'
+
+Per indicare che i file sono solo degli stub e che in realtà il contenuto è
+sulla smartcard, l'output dovrebbe mostrarvi ``shadowed-private-key``.
+
+Verificare che la smartcard funzioni
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Per verificare che la smartcard funzioni come dovuto, potete creare
+una firma::
+
+    $ echo "Hello world" | gpg --clearsign > /tmp/test.asc
+    $ gpg --verify /tmp/test.asc
+
+Col primo comando dovrebbe chiedervi il PIN della smartcard, e poi dovrebbe
+mostrare "Good signature" dopo l'esecuzione di ``gpg --verify``.
+
+Complimenti, siete riusciti a rendere estremamente difficile il furto della
+vostra identità digitale di sviluppatore.
+
+Altre operazioni possibili con GnuPG
+------------------------------------
+
+Segue un breve accenno ad alcune delle operazioni più comuni che dovrete
+fare con le vostre chiavi PGP.
+
+Montare il disco con la chiave primaria
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Vi servirà la vostra chiave principale per tutte le operazioni che seguiranno,
+per cui per prima cosa dovrete accedere ai vostri backup e dire a GnuPG di
+usarli::
+
+    $ export GNUPGHOME=/media/disk/foo/gnupg-backup
+    $ gpg --list-secret-keys
+
+Dovete assicurarvi di vedere ``sec`` e non ``sec#`` nell'output del programma
+(il simbolo ``#`` significa che la chiave non è disponibile e che state ancora
+utilizzando la vostra solita cartella di lavoro).
+
+Estendere la data di scadenza di una chiave
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+La chiave principale ha una data di scadenza di 2 anni dal momento della sua
+creazione. Questo per motivi di sicurezza e per rendere obsolete le chiavi
+che, eventualmente, dovessero sparire dai keyserver.
+
+Per estendere di un anno, dalla data odierna, la scadenza di una vostra chiave,
+eseguite::
+
+    $ gpg --quick-set-expire [fpr] 1y
+
+Se per voi è più facile da memorizzare, potete anche utilizzare una data
+specifica (per esempio, il vostro compleanno o capodanno)::
+
+    $ gpg --quick-set-expire [fpr] 2020-07-01
+
+Ricordatevi di inviare l'aggiornamento ai keyserver::
+
+    $ gpg --send-key [fpr]
+
+Aggiornare la vostra cartella di lavoro dopo ogni modifica
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Dopo aver fatto delle modifiche alle vostre chiavi usando uno spazio a parte,
+dovreste importarle nella vostra cartella di lavoro abituale::
+
+    $ gpg --export | gpg --homedir ~/.gnupg --import
+    $ unset GNUPGHOME
+
+
+Usare PGP con Git
+=================
+
+Una delle caratteristiche fondanti di Git è la sua natura decentralizzata --
+una volta che il repositorio è stato clonato sul vostro sistema, avete la
+storia completa del progetto, inclusi i suoi tag, i commit ed i rami. Tuttavia,
+con i centinaia di repositori clonati che ci sono in giro, come si fa a
+verificare che la loro copia di linux.git non è stata manomessa da qualcuno?
+
+Oppure, cosa succede se viene scoperta una backdoor nel codice e la riga
+"Autore" dice che sei stato tu, mentre tu sei abbastanza sicuro di
+`non averci niente a che fare`_?
+
+Per risolvere entrambi i problemi, Git ha introdotto l'integrazione con PGP.
+I tag firmati dimostrano che il repositorio è integro assicurando che il suo
+contenuto è lo stesso che si trova sulle macchine degli sviluppatori che hanno
+creato il tag; mentre i commit firmati rendono praticamente impossibile
+ad un malintenzionato di impersonarvi senza avere accesso alle vostre chiavi
+PGP.
+
+.. _`non averci niente a che fare`: https://github.com/jayphelps/git-blame-someone-else
+
+Configurare git per usare la vostra chiave PGP
+----------------------------------------------
+
+Se avete solo una chiave segreta nel vostro portachiavi, allora non avete nulla
+da fare in più dato che sarà la vostra chiave di base. Tuttavia, se doveste
+avere più chiavi segrete, potete dire a git quale dovrebbe usare (``[fpg]``
+è la vostra impronta digitale)::
+
+    $ git config --global user.signingKey [fpr]
+
+**IMPORTANTE**: se avete una comando dedicato per ``gpg2``, allora dovreste
+dire a git di usare sempre quello piuttosto che il vecchio comando ``gpg``::
+
+    $ git config --global gpg.program gpg2
+
+Come firmare i tag
+------------------
+
+Per creare un tag firmato, passate l'opzione ``-s`` al comando tag::
+
+    $ git tag -s [tagname]
+
+La nostra raccomandazione è quella di firmare sempre i tag git, perché
+questo permette agli altri sviluppatori di verificare che il repositorio
+git dal quale stanno prendendo il codice non è stato alterato intenzionalmente.
+
+Come verificare i tag firmati
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Per verificare un tag firmato, potete usare il comando ``verify-tag``::
+
+    $ git verify-tag [tagname]
+
+Se state prendendo un tag da un fork del repositorio del progetto, git
+dovrebbe verificare automaticamente la firma di quello che state prendendo
+e vi mostrerà il risultato durante l'operazione di merge::
+
+    $ git pull [url] tags/sometag
+
+Il merge conterrà qualcosa di simile::
+
+    Merge tag 'sometag' of [url]
+
+    [Tag message]
+
+    # gpg: Signature made [...]
+    # gpg: Good signature from [...]
+
+Se state verificando il tag di qualcun altro, allora dovrete importare
+la loro chiave PGP. Fate riferimento alla sezione ":ref:`it_verify_identities`"
+che troverete più avanti.
+
+Configurare git per firmare sempre i tag con annotazione
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Se state creando un tag con annotazione è molto probabile che vogliate
+firmarlo. Per imporre a git di firmare sempre un tag con annotazione,
+dovete impostare la seguente opzione globale::
+
+    $ git config --global tag.forceSignAnnotated true
+
+Come usare commit firmati
+-------------------------
+
+Creare dei commit firmati è facile, ma è molto più difficile utilizzarli
+nello sviluppo del kernel linux per via del fatto che ci si affida alle
+liste di discussione e questo modo di procedere non mantiene le firme PGP
+nei commit. In aggiunta, quando si usa *rebase* nel proprio repositorio
+locale per allinearsi al kernel anche le proprie firme PGP verranno scartate.
+Per questo motivo, la maggior parte degli sviluppatori del kernel non si
+preoccupano troppo di firmare i propri commit ed ignoreranno quelli firmati
+che si trovano in altri repositori usati per il proprio lavoro.
+
+Tuttavia, se avete il vostro repositorio di lavoro disponibile al pubblico
+su un qualche servizio di hosting git (kernel.org, infradead.org, ozlabs.org,
+o altri), allora la raccomandazione è di firmare tutti i vostri commit
+anche se gli sviluppatori non ne beneficeranno direttamente.
+
+Vi raccomandiamo di farlo per i seguenti motivi:
+
+1. Se dovesse mai esserci la necessità di fare delle analisi forensi o
+   tracciare la provenienza di un codice, anche sorgenti mantenuti
+   esternamente che hanno firme PGP sui commit avranno un certo valore a
+   questo scopo.
+2. Se dovesse mai capitarvi di clonare il vostro repositorio locale (per
+   esempio dopo un danneggiamento del disco), la firma vi permetterà di
+   verificare l'integrità del repositorio prima di riprendere il lavoro.
+3. Se qualcuno volesse usare *cherry-pick* sui vostri commit, allora la firma
+   permetterà di verificare l'integrità dei commit prima di applicarli.
+
+Creare commit firmati
+~~~~~~~~~~~~~~~~~~~~~
+
+Per creare un commit firmato, dovete solamente aggiungere l'opzione ``-S``
+al comando ``git commit`` (si usa la lettera maiuscola per evitare
+conflitti con un'altra opzione)::
+
+    $ git commit -S
+
+Configurare git per firmare sempre i commit
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Potete dire a git di firmare sempre i commit::
+
+    git config --global commit.gpgSign true
+
+.. note::
+
+    Assicuratevi di aver configurato ``gpg-agent`` prima di abilitare
+    questa opzione.
+
+.. _it_verify_identities:
+
+Come verificare l'identità degli sviluppatori del kernel
+========================================================
+
+Firmare i tag e i commit è facile, ma come si fa a verificare che la chiave
+usata per firmare qualcosa appartenga davvero allo sviluppatore e non ad un
+impostore?
+
+Configurare l'auto-key-retrieval usando WKD e DANE
+--------------------------------------------------
+
+Se non siete ancora in possesso di una vasta collezione di chiavi pubbliche
+di altri sviluppatori, allora potreste iniziare il vostro portachiavi
+affidandovi ai servizi di auto-scoperta e auto-recupero. GnuPG può affidarsi
+ad altre tecnologie di delega della fiducia, come DNSSEC e TLS, per sostenervi
+nel caso in cui iniziare una propria rete di fiducia da zero sia troppo
+scoraggiante.
+
+Aggiungete il seguente testo al vostro file ``~/.gnupg/gpg.conf``::
+
+    auto-key-locate wkd,dane,local
+    auto-key-retrieve
+
+La *DNS-Based Authentication of Named Entities* ("DANE") è un metodo
+per la pubblicazione di chiavi pubbliche su DNS e per renderle sicure usando
+zone firmate con DNSSEC. Il *Web Key Directory* ("WKD") è un metodo
+alternativo che usa https a scopo di ricerca. Quando si usano DANE o WKD
+per la ricerca di chiavi pubbliche, GnuPG validerà i certificati DNSSEC o TLS
+prima di aggiungere al vostro portachiavi locale le eventuali chiavi trovate.
+
+Kernel.org pubblica la WKD per tutti gli sviluppatori che hanno un account
+kernel.org. Una volta che avete applicato le modifiche al file ``gpg.conf``,
+potrete auto-recuperare le chiavi di Linus Torvalds e Greg Kroah-Hartman
+(se non le avete già)::
+
+    $ gpg --locate-keys torvalds@kernel.org gregkh@kernel.org
+
+Se avete un account kernel.org, al fine di rendere più utile l'uso di WKD
+da parte di altri sviluppatori del kernel, dovreste `aggiungere alla vostra
+chiave lo UID di kernel.org`_.
+
+.. _`aggiungere alla vostra chiave lo UID di kernel.org`: https://korg.wiki.kernel.org/userdoc/mail#adding_a_kernelorg_uid_to_your_pgp_key
+
+Web of Trust (WOT) o Trust on First Use (TOFU)
+----------------------------------------------
+
+PGP incorpora un meccanismo di delega della fiducia conosciuto come
+"Web of Trust". Di base, questo è un tentativo di sostituire la necessità
+di un'autorità certificativa centralizzata tipica del mondo HTTPS/TLS.
+Invece di avere svariati produttori software che decidono chi dovrebbero
+essere le entità di certificazione di cui dovreste fidarvi, PGP lascia
+la responsabilità ad ogni singolo utente.
+
+Sfortunatamente, solo poche persone capiscono come funziona la rete di fiducia.
+Nonostante sia un importante aspetto della specifica OpenPGP, recentemente
+le versioni di GnuPG (2.2 e successive) hanno implementato un meccanisco
+alternativo chiamato "Trust on First Use" (TOFU). Potete pensare a TOFU come
+"ad un approccio all fidicia simile ad SSH". In SSH, la prima volta che vi
+connettete ad un sistema remoto, l'impronta digitale della chiave viene
+registrata e ricordata. Se la chiave dovesse cambiare in futuro, il programma
+SSH vi avviserà e si rifiuterà di connettersi, obbligandovi a prendere una
+decisione circa la fiducia che riponete nella nuova chiave. In modo simile,
+la prima volta che importate la chiave PGP di qualcuno, si assume sia valida.
+Se ad un certo punto GnuPG trova un'altra chiave con la stessa identità,
+entrambe, la vecchia e la nuova, verranno segnate come invalide e dovrete
+verificare manualmente quale tenere.
+
+Vi raccomandiamo di usare il meccanisco TOFU+PGP (che è la nuova configurazione
+di base di GnuPG v2). Per farlo, aggiungete (o modificate) l'impostazione
+``trust-model`` in ``~/.gnupg/gpg.conf``::
+
+    trust-model tofu+pgp
+
+Come usare i keyserver in sicurezza
+-----------------------------------
+Se ottenete l'errore "No public key" quando cercate di validate il tag di
+qualcuno, allora dovreste cercare quella chiave usando un keyserver. È
+importante tenere bene a mente che non c'è alcuna garanzia che la chiave
+che avete recuperato da un keyserver PGP appartenga davvero alla persona
+reale -- è progettato così. Dovreste usare il Web of Trust per assicurarvi
+che la chiave sia valida.
+
+Come mantenere il Web of Trust va oltre gli scopi di questo documento,
+semplicemente perché farlo come si deve richiede sia sforzi che perseveranza
+che tendono ad andare oltre al livello di interesse della maggior parte degli
+esseri umani. Qui di seguito alcuni rapidi suggerimenti per aiutarvi a ridurre
+il rischio di importare chiavi maligne.
+
+Primo, diciamo che avete provato ad eseguire ``git verify-tag`` ma restituisce
+un errore dicendo che la chiave non è stata trovata::
+
+    $ git verify-tag sunxi-fixes-for-4.15-2
+    gpg: Signature made Sun 07 Jan 2018 10:51:55 PM EST
+    gpg:                using RSA key DA73759BF8619E484E5A3B47389A54219C0F2430
+    gpg:                issuer "wens@...org"
+    gpg: Can't check signature: No public key
+
+Cerchiamo nel keyserver per maggiori informazioni sull'impronta digitale
+della chiave (l'impronta digitale, probabilmente, appartiene ad una
+sottochiave, dunque non possiamo usarla direttamente senza trovare prima
+l'ID della chiave primaria associata ad essa)::
+
+    $ gpg --search DA73759BF8619E484E5A3B47389A54219C0F2430
+    gpg: data source: hkp://keys.gnupg.net
+    (1) Chen-Yu Tsai <wens@...org>
+          4096 bit RSA key C94035C21B4F2AEB, created: 2017-03-14, expires: 2019-03-15
+    Keys 1-1 of 1 for "DA73759BF8619E484E5A3B47389A54219C0F2430".  Enter number(s), N)ext, or Q)uit > q
+
+Localizzate l'ID della chiave primaria, nel nostro esempio
+``C94035C21B4F2AEB``. Ora visualizzate le chiavi di Linus Torvalds
+che avete nel vostro portachiavi::
+
+    $ gpg --list-key torvalds@kernel.org
+    pub   rsa2048 2011-09-20 [SC]
+          ABAF11C65A2970B130ABE3C479BE3E4300411886
+    uid           [ unknown] Linus Torvalds <torvalds@kernel.org>
+    sub   rsa2048 2011-09-20 [E]
+
+Poi, aprite il `PGP pathfinder`_. Nel campo "From", incollate l'impronta
+digitale della chiave di Linus Torvalds che si vede nell'output qui sopra.
+Nel campo "to", incollate il key-id della chiave sconosciuta che avete
+trovato con ``gpg --search``, e poi verificare il risultato:
+
+- `Finding paths to Linus`_
+
+Se trovate un paio di percorsi affidabili è un buon segno circa la validità
+della chiave. Ora, potete aggiungerla al vostro portachiavi dal keyserver::
+
+    $ gpg --recv-key C94035C21B4F2AEB
+
+Questa procedura non è perfetta, e ovviamente state riponendo la vostra
+fiducia nell'amministratore del servizio *PGP Pathfinder* sperando che non
+sia malintenzionato (infatti, questo va contro :ref:`it_devs_not_infra`).
+Tuttavia, se mantenete con cura la vostra rete di fiducia sarà un deciso
+miglioramento rispetto alla cieca fiducia nei keyserver.
+
+.. _`PGP pathfinder`: https://pgp.cs.uu.nl/
+.. _`Finding paths to Linus`: https://pgp.cs.uu.nl/paths/79BE3E4300411886/to/C94035C21B4F2AEB.html
diff --git a/Documentation/translations/it_IT/process/stable-kernel-rules.rst b/Documentation/translations/it_IT/process/stable-kernel-rules.rst
index 6fa5ce9c3572..48e88e5ad2c5 100644
--- a/Documentation/translations/it_IT/process/stable-kernel-rules.rst
+++ b/Documentation/translations/it_IT/process/stable-kernel-rules.rst
@@ -1,12 +1,202 @@
 .. include:: ../disclaimer-ita.rst
 
 :Original: :ref:`Documentation/process/stable-kernel-rules.rst <stable_kernel_rules>`
+:Translator: Federico Vaga <federico.vaga@vaga.pv.it>
 
 .. _it_stable_kernel_rules:
 
 Tutto quello che volevate sapere sui rilasci -stable di Linux
 ==============================================================
 
-.. warning::
+Regole sul tipo di patch che vengono o non vengono accettate nei sorgenti
+"-stable":
 
-    TODO ancora da tradurre
+ - Ovviamente dev'essere corretta e verificata.
+ - Non dev'essere più grande di 100 righe, incluso il contesto.
+ - Deve correggere una cosa sola.
+ - Deve correggere un baco vero che sta disturbando gli utenti (non cose del
+   tipo "Questo potrebbe essere un problema ...").
+ - Deve correggere un problema di compilazione (ma non per cose già segnate
+   con CONFIG_BROKEN), un kernel oops, un blocco, una corruzione di dati,
+   un vero problema di sicurezza, o problemi del tipo "oh, questo non va bene".
+   In pratica, qualcosa di critico.
+ - Problemi importanti riportati dagli utenti di una distribuzione potrebbero
+   essere considerati se correggono importanti problemi di prestazioni o di
+   interattività.  Dato che questi problemi non sono così ovvi e la loro
+   correzione ha un'alta probabilità d'introdurre una regressione, dovrebbero
+   essere sottomessi solo dal manutentore della distribuzione includendo un
+   link, se esiste, ad un rapporto su bugzilla, e informazioni aggiuntive
+   sull'impatto che ha sugli utenti.
+ - Non deve correggere problemi relativi a una "teorica sezione critica",
+   a meno che non venga fornita anche una spiegazione su come questa si
+   possa verificare.
+ - Non deve includere alcuna correzione "banale" (correzioni grammaticali,
+   pulizia dagli spazi bianchi, eccetera).
+ - Deve rispettare le regole scritte in
+   :ref:`Documentation/translation/it_IT/process/submitting-patches.rst <it_submittingpatches>`
+ - Questa patch o una equivalente deve esistere già nei sorgenti principali di
+   Linux
+
+
+Procedura per sottomettere patch per i sorgenti -stable
+-------------------------------------------------------
+
+ - Se la patch contiene modifiche a dei file nelle cartelle net/ o drivers/net,
+   allora seguite le linee guida descritte in
+   :ref:`Documentation/translation/it_IT/networking/netdev-FAQ.rst <it_netdev-FAQ>`;
+   ma solo dopo aver verificato al seguente indirizzo che la patch non sia
+   già in coda:
+   https://patchwork.ozlabs.org/bundle/davem/stable/?series=&submitter=&state=*&q=&archive=
+ - Una patch di sicurezza non dovrebbero essere gestite (solamente) dal processo
+   di revisione -stable, ma dovrebbe seguire le procedure descritte in
+   :ref:`Documentation/translations/it_IT/admin-guide/security-bugs.rst <it_securitybugs>`.
+
+
+Per tutte le altre sottomissioni, scegliere una delle seguenti procedure
+------------------------------------------------------------------------
+
+.. _it_option_1:
+
+Opzione 1
+*********
+
+Per far sì che una patch venga automaticamente inclusa nei sorgenti stabili,
+aggiungete l'etichetta
+
+.. code-block:: none
+
+     Cc: stable@vger.kernel.org
+
+nell'area dedicata alla firme. Una volta che la patch è stata inclusa, verrà
+applicata anche sui sorgenti stabili senza che l'autore o il manutentore
+del sottosistema debba fare qualcosa.
+
+.. _it_option_2:
+
+Opzione 2
+*********
+
+Dopo che la patch è stata inclusa nei sorgenti Linux, inviate una mail a
+stable@vger.kernel.org includendo: il titolo della patch, l'identificativo
+del commit, il perché pensate che debba essere applicata, e in quale versione
+del kernel la vorreste vedere.
+
+.. _it_option_3:
+
+Opzione 3
+*********
+
+Inviata la patch, dopo aver verificato che rispetta le regole descritte in
+precedenza, a stable@vger.kernel.org.  Dovete annotare nel changelog
+l'identificativo del commit nei sorgenti principali, così come la versione
+del kernel nel quale vorreste vedere la patch.
+
+L':ref:`it_option_1` è fortemente raccomandata; è il modo più facile e usato.
+L':ref:`it_option_2` e l':ref:`it_option_3` sono più utili quando, al momento
+dell'inclusione dei sorgenti principali, si ritiene che non debbano essere
+incluse anche in quelli stabili (per esempio, perché si crede che si dovrebbero
+fare più verifiche per eventuali regressioni). L':ref:`it_option_3` è
+particolarmente utile se la patch ha bisogno di qualche modifica per essere
+applicata ad un kernel più vecchio (per esempio, perché nel frattempo l'API è
+cambiata).
+
+Notate che per l':ref:`it_option_3`, se la patch è diversa da quella nei
+sorgenti principali (per esempio perché è stato necessario un lavoro di
+adattamento) allora dev'essere ben documentata e giustificata nella descrizione
+della patch.
+
+L'identificativo del commit nei sorgenti principali dev'essere indicato sopra
+al messaggio della patch, così:
+
+.. code-block:: none
+
+    commit <sha1> upstream.
+
+In aggiunta, alcune patch inviate attraverso l':ref:`it_option_1` potrebbero
+dipendere da altre che devo essere incluse. Questa situazione può essere
+indicata nel seguente modo nell'area dedicata alle firme:
+
+.. code-block:: none
+
+     Cc: <stable@vger.kernel.org> # 3.3.x: a1f84a3: sched: Check for idle
+     Cc: <stable@vger.kernel.org> # 3.3.x: 1b9508f: sched: Rate-limit newidle
+     Cc: <stable@vger.kernel.org> # 3.3.x: fd21073: sched: Fix affinity logic
+     Cc: <stable@vger.kernel.org> # 3.3.x
+     Signed-off-by: Ingo Molnar <mingo@elte.hu>
+
+La sequenza di etichette ha il seguente significato:
+
+.. code-block:: none
+
+     git cherry-pick a1f84a3
+     git cherry-pick 1b9508f
+     git cherry-pick fd21073
+     git cherry-pick <this commit>
+
+Inoltre, alcune patch potrebbero avere dei requisiti circa la versione del
+kernel. Questo può essere indicato usando il seguente formato nell'area
+dedicata alle firme:
+
+.. code-block:: none
+
+     Cc: <stable@vger.kernel.org> # 3.3.x
+
+L'etichetta ha il seguente significato:
+
+.. code-block:: none
+
+     git cherry-pick <this commit>
+
+per ogni sorgente "-stable" che inizia con la versione indicata.
+
+Dopo la sottomissione:
+
+ - Il mittente riceverà un ACK quando la patch è stata accettata e messa in
+   coda, oppure un NAK se la patch è stata rigettata.  A seconda degli impegni
+   degli sviluppatori, questa risposta potrebbe richiedere alcuni giorni.
+ - Se accettata, la patch verrà aggiunta alla coda -stable per essere
+   revisionata dal altri sviluppatori e dal principale manutentore del
+   sottosistema.
+
+
+Ciclo di una revisione
+----------------------
+
+ - Quando i manutentori -stable decidono di fare un ciclo di revisione, le
+   patch vengono mandate al comitato per la revisione, ai manutentori soggetti
+   alle modifiche delle patch (a meno che il mittente non sia anche il
+   manutentore di quell'area del kernel) e in CC: alla lista di discussione
+   linux-kernel.
+ - La commissione per la revisione ha 48 ore per dare il proprio ACK o NACK
+   alle patch.
+ - Se una patch viene rigettata da un membro della commissione, o un membro
+   della lista linux-kernel obietta la bontà della patch, sollevando problemi
+   che i manutentori ed i membri non avevano compreso, allora la patch verrà
+   rimossa dalla coda.
+ - Alla fine del ciclo di revisione tutte le patch che hanno ricevuto l'ACK
+   verranno aggiunte per il prossimo rilascio -stable, e successivamente
+   questo nuovo rilascio verrà fatto.
+ - Le patch di sicurezza verranno accettate nei sorgenti -stable direttamente
+   dalla squadra per la sicurezza del kernel, e non passerà per il normale
+   ciclo di revisione. Contattate la suddetta squadra per maggiori dettagli
+   su questa procedura.
+
+Sorgenti
+--------
+
+ - La coda delle patch, sia quelle già applicate che in fase di revisione,
+   possono essere trovate al seguente indirizzo:
+
+	https://git.kernel.org/pub/scm/linux/kernel/git/stable/stable-queue.git
+
+ - Il rilascio definitivo, e marchiato, di tutti i kernel stabili può essere
+   trovato in rami distinti per versione al seguente indirizzo:
+
+	https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git
+
+
+Comitato per la revisione
+-------------------------
+
+ - Questo comitato è fatto di sviluppatori del kernel che si sono offerti
+   volontari per questo lavoro, e pochi altri che non sono proprio volontari.
diff --git a/Documentation/translations/it_IT/process/submitting-patches.rst b/Documentation/translations/it_IT/process/submitting-patches.rst
index 2ab9c1401aa1..7d7ea92c5c5a 100644
--- a/Documentation/translations/it_IT/process/submitting-patches.rst
+++ b/Documentation/translations/it_IT/process/submitting-patches.rst
@@ -67,8 +67,8 @@ sulla radice dei sorgenti del kernel, e non sulle sue sottocartelle.
 
 Per creare una patch per un singolo file, spesso è sufficiente fare::
 
-	SRCTREE= linux
-	MYFILE=  drivers/net/mydriver.c
+	SRCTREE=linux
+	MYFILE=drivers/net/mydriver.c
 
 	cd $SRCTREE
 	cp $MYFILE $MYFILE.orig
@@ -80,7 +80,7 @@ Per creare una patch per molteplici file, dovreste spacchettare i sorgenti
 "vergini", o comunque non modificati, e fare un ``diff`` coi vostri.
 Per esempio::
 
-	MYSRC= /devel/linux
+	MYSRC=/devel/linux
 
 	tar xvfz linux-3.19.tar.gz
 	mv linux-3.19 linux-3.19-vanilla
@@ -567,11 +567,42 @@ alcunché - ma dovrebbe indicare che la persona ha ricevuto una copia della
 patch.  Questa etichetta documenta che terzi potenzialmente interessati sono
 stati inclusi nella discussione.
 
-L'etichetta Co-developed-by: indica che la patch è stata scritta dall'autore in
-collaborazione con un altro sviluppatore.  Qualche volta questo è utile quando
-più persone lavorano sulla stessa patch.  Notate, questa persona deve avere
-nella patch anche una riga Signed-off-by:.
+Co-developed-by: indica che la patch è stata cosviluppata da diversi
+sviluppatori; viene usato per assegnare più autori (in aggiunta a quello
+associato all'etichetta From:) quando più persone lavorano ad una patch.  Dato
+che Co-developed-by: implica la paternità della patch, ogni Co-developed-by:
+dev'essere seguito immediatamente dal Signed-off-by: del corrispondente
+coautore. Qui si applica la procedura di base per sign-off, in pratica
+l'ordine delle etichette Signed-off-by: dovrebbe riflettere il più possibile
+l'ordine cronologico della storia della patch, indipendentemente dal fatto che
+la paternità venga assegnata via From: o Co-developed-by:. Da notare che
+l'ultimo Signed-off-by: dev'essere quello di colui che ha sottomesso la patch.
 
+Notate anche che l'etichetta From: è opzionale quando l'autore in From: è
+anche la persona (e indirizzo email) indicato nel From: dell'intestazione
+dell'email.
+
+Esempio di una patch sottomessa dall'autore in From:::
+
+	<changelog>
+
+	Co-developed-by: First Co-Author <first@coauthor.example.org>
+	Signed-off-by: First Co-Author <first@coauthor.example.org>
+	Co-developed-by: Second Co-Author <second@coauthor.example.org>
+	Signed-off-by: Second Co-Author <second@coauthor.example.org>
+	Signed-off-by: From Author <from@author.example.org>
+
+Esempio di una patch sottomessa dall'autore Co-developed-by:::
+
+	From: From Author <from@author.example.org>
+
+	<changelog>
+
+	Co-developed-by: Random Co-Author <random@coauthor.example.org>
+	Signed-off-by: Random Co-Author <random@coauthor.example.org>
+	Signed-off-by: From Author <from@author.example.org>
+	Co-developed-by: Submitting Co-Author <sub@coauthor.example.org>
+	Signed-off-by: Submitting Co-Author <sub@coauthor.example.org>
 
 13) Utilizzare Reported-by:, Tested-by:, Reviewed-by:, Suggested-by: e Fixes:
 -----------------------------------------------------------------------------
@@ -719,7 +750,7 @@ Un paio di esempi di oggetti::
 La riga ``from`` dev'essere la prima nel corpo del messaggio ed è nel
 formato:
 
-        From: Original Author <author@example.com>
+        From: Patch Author <author@example.com>
 
 La riga ``from`` indica chi verrà accreditato nel changelog permanente come
 l'autore della patch.  Se la riga ``from`` è mancante, allora per determinare
diff --git a/Documentation/translations/ja_JP/SubmittingPatches b/Documentation/translations/ja_JP/SubmittingPatches
index 02139656463e..ad979c3c06a6 100644
--- a/Documentation/translations/ja_JP/SubmittingPatches
+++ b/Documentation/translations/ja_JP/SubmittingPatches
@@ -58,8 +58,8 @@ Linux カーãƒ�ルã�«å¯¾ã�™ã‚‹å…¨ã�¦ã�®å¤‰æ›´ã�¯ diff(1) コマンドã�«ã‚ˆã‚‹ãƒ
 1個�ファイル�����パッ�を作��る�������ん��場��
 以下�作業を行���分��。
 
-	SRCTREE= linux-2.6
-	MYFILE=  drivers/net/mydriver.c
+	SRCTREE=linux-2.6
+	MYFILE=drivers/net/mydriver.c
 
 	cd $SRCTREE
 	cp $MYFILE $MYFILE.orig
@@ -71,7 +71,7 @@ Linux カーãƒ�ルã�«å¯¾ã�™ã‚‹å…¨ã�¦ã�®å¤‰æ›´ã�¯ diff(1) コマンドã�«ã‚ˆã‚‹ãƒ
 ���変更を加���� Linux カー�ルを展開��自分� Linux カー�ル
 ソース��差分を生���������ん。例���
 
-	MYSRC= /devel/linux-2.6
+	MYSRC=/devel/linux-2.6
 
 	tar xvfz linux-2.6.12.tar.gz
 	mv linux-2.6.12 linux-2.6.12-vanilla
diff --git a/Documentation/translations/zh_CN/SubmittingPatches b/Documentation/translations/zh_CN/SubmittingPatches
deleted file mode 100644
index e9098da8f1a4..000000000000
--- a/Documentation/translations/zh_CN/SubmittingPatches
+++ /dev/null
@@ -1,412 +0,0 @@
-Chinese translated version of Documentation/process/submitting-patches.rst
-
-If you have any comment or update to the content, please contact the
-original document maintainer directly.  However, if you have a problem
-communicating in English you can also ask the Chinese maintainer for
-help.  Contact the Chinese maintainer if this translation is outdated
-or if there is a problem with the translation.
-
-Chinese maintainer: TripleX Chung <triplex@zh-kernel.org>
----------------------------------------------------------------------
-Documentation/process/submitting-patches.rst 的中文翻译
-
-如果想评论或更新本文的内容，请直接�系原文档的维护者。如果你使用英文
-交�有困难的�，也�以�中文版维护者求助。如果本翻译更新��时或者翻
-译存在问题，请�系中文版维护者。
-
-中文版维护者： 钟宇 TripleX Chung <triplex@zh-kernel.org>
-中文版翻译者： 钟宇 TripleX Chung <triplex@zh-kernel.org>
-中文版校译者： �阳 Li Yang <leo@zh-kernel.org>
-               王� Wang Cong <xiyou.wangcong@gmail.com>
-
-以下为正文
----------------------------------------------------------------------
-
-   如何让你的改动进入内核
-     或者
-  获得亲爱的 Linus Torvalds 的关注和处�
-----------------------------------
-
-对于想�将改动�交到 Linux 内核的个人或者公��说，如果�熟悉“规矩�，
-�交的�程会让人�惧。本文档收集了一系列建议，这些建议�以大大的�高你
-的改动被接�的机会。
-阅读 Documentation/process/submit-checklist.rst �获得在�交代��需�检查的项目的列
-表。如果你在�交一个驱动程�，那么�时阅读一下
-Documentation/process/submitting-drivers.rst 。
-
-
---------------------------
-第一节 - 创建并��你的改动
---------------------------
-
-1) "diff -up"
------------
-
-使用 "diff -up" 或者 "diff -uprN" �创建补�。
-
-所有内核的改动，都是以补�的形�呈现的，补�由 diff(1) 生�。创建补�的
-时候，�确认它是以 "unified diff" 格�创建的，这�格�由 diff(1) 的 '-u'
-�数生�。而且，请使用 '-p' �数，那样会显示�个改动所在的C函数，使得
-产生的补�容易读得多。补�应该基于内核�代�树的根目录，而�是里边的任
-何�目录。
-为一个�独的文件创建补�，一般�说这样�就够了：
-
-        SRCTREE= linux-2.6
-        MYFILE=  drivers/net/mydriver.c
-
-        cd $SRCTREE
-        cp $MYFILE $MYFILE.orig
-        vi $MYFILE      # make your change
-        cd ..
-        diff -up $SRCTREE/$MYFILE{.orig,} > /tmp/patch
-
-为多个文件创建补�，你�以解开一个没有修改过的内核�代�树，然�和你自
-己的代�树之间� diff 。例如：
-
-        MYSRC= /devel/linux-2.6
-
-        tar xvfz linux-2.6.12.tar.gz
-        mv linux-2.6.12 linux-2.6.12-vanilla
-        diff -uprN -X linux-2.6.12-vanilla/Documentation/dontdiff \
-                linux-2.6.12-vanilla $MYSRC > /tmp/patch
-
-"dontdiff" 是内核在编译的时候产生的文件的列表，列表中的文件在 diff(1)
-产生的补�里会被跳过。"dontdiff" 文件被包�在2.6.12和之�版本的内核�代
-�树中。对于更早的内核版本，你�以从
-<http://www.xenotime.net/linux/doc/dontdiff> 获�它。
-确定你的补�里没有包�任何�属于这次补��交的�外文件。记得在用diff(1)
-生�补�之�，审阅一次补�，以确�准确。
-如果你的改动很散乱，你应该研究一下如何将补�分割�独立的部分，将改动分
-割�一系列�乎逻辑的步骤。这样更容易让其他内核开�者审核，如果你想你的
-补�被接�，这是很��的。下�这些脚本能够帮助你�这件事情：
-Quilt:
-http://savannah.nongnu.org/projects/quilt
-
-2)�述你的改动。
-�述你的改动包�的技术细节。
-
-�多具体就写多具体。最糟糕的�述�能是�下�这些语�：“更新了�驱动程
-��，“修正了�驱动程�的bug�，或者“这个补�包�了��系统的修改，请
-使用。�
-
-如果你的�述开始�长，这表示你也许需�拆分你的补�了，请看第3�节，
-继续。
-
-3)拆分你的改动
-
-将改动拆分，逻辑类似的放到�一个补�文件里。
-
-例如，如果你的改动里�时有bug修正和性能优化，那么把这些改动拆分到两个或
-者更多的补�文件中。如果你的改动包�对API的修改，并且修改了驱动程��适
-应这些新的API，那么把这些修改分�两个补�。
-
-�一方�，如果你将一个�独的改动��多个补�文件，那么将它们�并�一个
-�独的补�文件。这样一个逻辑上�独的改动�被包�在一个补�文件里。
-
-如果有一个补��赖�外一个补��完�它的改动，那没问题。简�的在你的补
-��述里指出“这个补��赖�补��就好了。
-
-如果你�能将补�浓缩�更少的文件，那么�次大约��出15个，然�等待审查
-和整�。
-
-4)选择 e-mail 的收件人
-
-看一� MAINTAINERS 文件和�代�，看看你所的改动所在的内核�系统有没有指
-定的维护者。如果有，给他们�e-mail。
-
-如果没有找到维护者，或者维护者没有�馈，将你的补���到内核开�者主邮
-件列表 linux-kernel@vger.kernel.org。大部分的内核开�者都跟踪这个邮件列
-表，�以评价你的改动。
-
-�次����超过15个补�到 vger 邮件列表���
-
-Linus Torvalds 是决定改动能�进入 Linux 内核的最终�决者。他的 e-mail
-地�是 <torvalds@linux-foundation.org> 。他收到的 e-mail 很多，所以一般
-的说，最好别给他� e-mail。
-
-那些修正bug，“显而易��的修改或者是类似的�需�很少讨论的补��以直接
-��或者CC给Linus。那些需�讨论或者没有很清楚的好处的补�，一般先��到
-linux-kernel邮件列表。�有当补�被讨论得差�多了，��交给Linus。
-
-5)选择CC( e-mail 抄�)列表
-
-除�你有�由�这样�，�则CC linux-kernel@vger.kernel.org。
-
-除了 Linus 之外，其他内核开�者也需�注�到你的改动，这样他们�能评论你
-的改动并�供代�审查和建议。linux-kernel 是 Linux 内核开�者主邮件列表
-。其它的邮件列表为特定的�系统�供�务，比如 USB，framebuffer 设备，虚
-拟文件系统，SCSI �系统，等等。查看 MAINTAINERS 文件�获得和你的改动有
-关的邮件列表。
-
-Majordomo lists of VGER.KERNEL.ORG at:
-        <http://vger.kernel.org/vger-lists.html>
-
-如果改动影�了用户空间和内核之间的接�，请给 MAN-PAGES 的维护者（列在
-MAINTAINERS 文件里的）��一个手册页（man-pages）补�，或者至少通知一下改
-�，让一些信�有途径进入手册页。
-
-�使在第四步的时候，维护者没有作出回应，也�确认在修改他们的代�的时候
-，一直将维护者拷�到CC列表中。
-
-对于�的补�，你也许会CC到 Adrian Bunk 管�的�集�碎补�的邮件列表
-(Trivial Patch Monkey)trivial@kernel.org，那里专门收集�碎的补�。下�这样
-的补�会被看作“�碎的�补�：
-  文档的拼写修正。
-  修正会影�到 grep(1) 的拼写。
-  警告信�修正(频�的打�无用的警告是�好的。)
-  编译错误修正（代�逻辑的确是对的，�是编译有问题。）
-  �行时修正（��真的修正了错误。）
-  移除使用了被废弃的函数/�的代�(例如 check_region。)
-  �系方�和文档修正。
-  用�移�的代�替���移�的代�（�使在体系结构相关的代�中，既然有
-  人拷�，��它是�碎的）
-  任何文件的作者/维护者对该文件的改动（例如 patch monkey 在�传模�下）
-
-EMAIL: trivial@kernel.org
-
-(译注，关于“�碎补��的一些说明：因为原文的这一部分写得比较简�，所以�得�
-�例写一下译注。"trivial"这个英文��的本�是“�碎的，���的。�但是在这里
-有�微有一些�化，例如对一些明显的NULL指针的修正，属于�行时修正，会被归类
-到�碎补�里。虽然NULL指针的修正很��，但是这样的修正往往很�而且很容易得到
-检验，所以也被归入�碎补�。�碎补�更精确的归类应该是
-“simple, localized & easy to verify�，也就是说简�的，局部的和易于检验的。
-trivial@kernel.org邮件列表的目的是针对这样的补�，为�交者�供一个中心，�
-�低�交的门槛。)
-
-6)没有 MIME 编�，没有链接，没有压缩，没有附件，�有纯文本。
-
-Linus 和其他的内核开�者需�阅读和评论你�交的改动。对于内核开�者�说
-，�以“引用�你的改动很��，使用一般的 e-mail 工具，他们就�以在你的
-代�的任何�置添加评论。
-
-因为这个原因，所有的�交的补�都是 e-mail 中“内嵌�的。
-警告：如果你使用剪切-粘贴你的补�，�心你的编辑器的自动�行功能破�你的
-补�。
-
-��将补�作为 MIME 编�的附件，�管是�压缩。很多�行的 e-mail 软件�
-是任何时候都将 MIME 编�的附件当作纯文本��的，这会使得别人无法在你的
-代�中加评论。�外，MIME 编�的附件会让 Linus 多花一点时间�处�，这就
-�低了你的改动被接�的�能性。
-
-警告：一些邮件软件，比如 Mozilla 会将你的信�以如下格���：
----- 邮件头 ----
-Content-Type: text/plain; charset=us-ascii; format=flowed
----- 邮件头 ----
-问题在于 “format=flowed� 会让接收端的�些邮件软件将邮件中的制表符替�
-�空格以��一些类似的替�。这样，你��的时候看起�没问题的补�就被破
-�了。
-
-�修正这个问题，�需�将你的 mozilla 的 defaults/pref/mailnews.js 文件
-里的
-pref("mailnews.send_plaintext_flowed", false); // RFC 2646=======
-修改�
-pref("mailnews.display.disable_format_flowed_support", true);
-就�以了。
-
-7) e-mail 的大�
-
-给 Linus ��补�的时候，永远按照第6�节说的�。
-
-大的改动对邮件列表��适，对�些维护者也��适。如果你的补�，在�压缩
-的情况下，超过了40kB，那么你最好将补�放在一个能通过 internet 访问的�
-务器上，然�用指�你的补�的 URL 替代。
-
-8) 指出你的内核版本
-
-在标题和在补�的�述中，指出补�对应的内核的版本，是很��的。
-
-如果补��能干净的在最新版本的内核上打上，Linus 是�会接�它的。
-
-9) ��气�，继续�交。
-
-当你�交了改动以�，�心地等待。如果 Linus 喜欢你的改动并且��它，那么
-它将在下一个内核�布版本中出现。
-
-然而，如果你的改动没有出现在下一个版本的内核中，�能有若干原因。�少那
-些原因，修正错误，�新�交更新�的改动，是你自己的工作。
-
-Linus�给出任何评论就“丢弃�你的补�是常�的事情。在系统中这样的事情很
-平常。如果他没有接�你的补�，也许是由于以下原因：
-* 你的补��能在最新版本的内核上干净的打上。
-* 你的补�在 linux-kernel 邮件列表中没有得到充分的讨论。
-* 风格问题（�照第2�节）
-* 邮件格�问题（�读本节）
-* 你的改动有技术问题。
-* 他收到了��的 e-mail，而你的在混乱中丢失了。
-* 你让人为难。
-
-有疑问的时候，在 linux-kernel 邮件列表上请求评论。
-
-10) 在标题上加上 PATCH 的字样
-
-Linus 和 linux-kernel 邮件列表的 e-mail ��都很高，一个通常的约定是标
-题行以 [PATCH] 开头。这样�以让 Linus 和其他内核开�人员�以从 e-mail
-的讨论中很轻易的将补�分辨出�。
-
-11）为你的工作签�
-
-为了加强对��了何事的追踪，尤其是对那些�过好几层的维护者的补�，我们
-建议在��出去的补�上加一个 “sign-off� 的过程。
-
-"sign-off" 是在补�的注释的最�的简�的一行文字，认�你编写了它或者其他
-人有�力将它作为开放�代�的补�传递。规则很简�：如果你能认�如下信�
-：
-      开�者���书 1.1
-      对于本项目的贡献，我认�如下信�：
-      （a）这些贡献是完全或者部分的由我创建，我有�利以文件中指出
-       的开放�代�许���交它；或者
-      （b）这些贡献基于以�的工作，�我所知，这些以�的工作��当的开放
-       �代�许���护，而且，根�许��，我有��交修改�的贡献，
-       无论是完全还是部分由我创造，这些贡献都使用�一个开放�代�许��
-       （除�我被�许用其它的许��），正如文件中指出的；或者
-      （c）这些贡献由认�（a），（b）或者（c）的人直接�供给我，而
-       且我没有修改它。
-      （d）我�解并��这个项目和贡献是公开的，贡献的记录（包括我
-       一起�交的个人记录，包括 sign-off ）被永久维护并且�以和这个项目
-       或者开放�代�的许���步地��行。
-       那么加入这样一行：
-       Signed-off-by: Random J Developer <random@developer.example.org>
-
-使用你的真�（抱歉，�能使用��或者匿�。）
-
-有人在最�加上标签。现在这些东西会被忽略，但是你�以这样�，�标记公�
-内部的过程，或者�是指出关于 sign-off 的一些特殊细节。
-
-12）标准补�格�
-
-标准的补�，标题行是：
-    Subject: [PATCH 001/123] �系统:一��概述
-
-标准补�的信体存在如下部分：
-
-  - 一个 "from" 行指出补�作者。
-
-  - 一个空行
-
-  - 说明的主体，这些说明文字会被拷�到�述该补�的永久改动记录里。
-
-  - 一个由"---"构�的标记行
-
-  - ��适放到改动记录里的�外的注解。
-
-  - 补�本身（diff 输出）
-
-标题行的格�，使得对标题行按字��排��常的容易 - 很多 e-mail 客户端都
-�以支� - 因为�列�是用零填充的，所以按数字排�和按字�排�是一样的。
-
-e-mail 标题中的“�系统�标识哪个内核�系统将被打补�。
-
-e-mail 标题中的“一��概述�扼�的�述 e-mail 中的补�。“一��概述�
-�应该是一个文件�。对于一个补�系列（“补�系列�指一系列的多个相关补
-�），��对�个补�都使用�样的“一��概述�。
-
-记� e-mail 的“一��概述�会�为该补�的全局唯一标识。它会蔓延到 git
-的改动记录里。然�“一��概述�会被用在开�者的讨论里，用�指代这个补
-�。用户将希望通过 google ��索"一��概述"�找到那些讨论这个补�的文
-章。
-
-一些标题的例�：
-
-    Subject: [patch 2/5] ext2: improve scalability of bitmap searching
-    Subject: [PATCHv2 001/207] x86: fix eflags tracking
-
-"from" 行是信体里的最上�一行，具有如下格�：
-        From: Original Author <author@example.com>
-
-"from" 行指明在永久改动日志里，�会被确认为作者。如果没有 "from" 行，那
-么邮件头里的 "From: " 行会被用�决定改动日志中的作者。
-
-说明的主题将会被�交到永久的�代�改动日志里，因此对那些早已��记得和
-这个补�相关的讨论细节的有能力的读者�说，是有�义的。
-
-"---" 标记行对于补�处�工具�找到哪里是改动日志信�的结�，是��缺少
-的。
-
-对于 "---" 标记之�的�外注解，一个好的用途就是用�写 diffstat，用�显
-示修改了什么文件和�个文件都增加和删除了多少行。diffstat 对于比较大的补
-�特别有用。其余那些�是和时刻或者开�者相关的注解，��适放到永久的改
-动日志里的，也应该放这里。
-使用 diffstat的选项 "-p 1 -w 70" 这样文件�就会从内核�代�树的目录开始
-，�会�用太宽的空间（很容易适�80列的宽度，也许会有一些缩进。）
-
-在��的�考资料中能看到适当的补�格�的更多细节。
-
--------------------------------
-第二节 �示，建议和诀�
--------------------------------
-
-本节包�很多和�交到内核的代�有关的通常的"规则"。事情永远有例外...但是
-你必须真的有好的�由这样�。你�以把本节��Linus的计算机科学入门课。
-
-1) 读 Document/process/coding-style.rst
-
-Nuff 说过，如果你的代�和这个�离太多，那么它有�能会被拒�，没有更多的
-审查，没有更多的评价。
-
-2) #ifdef 是丑陋的
-混�了 ifdef 的代�难以阅读和维护。别这样�。作为替代，将你的 ifdef 放
-在头文件里，有�件地定义 "static inline" 函数，或者�，在代�里用这些东
-西。让编译器把那些"空�作"优化掉。
-
-一个简�的例�，�好的代�：
-
-    dev = alloc_etherdev (sizeof(struct funky_private));
-    if (!dev)
-        return -ENODEV;
-    #ifdef CONFIG_NET_FUNKINESS
-    init_funky_net(dev);
-    #endif
-
-清��的例�:
-
-(头文件里)
-    #ifndef CONFIG_NET_FUNKINESS
-    static inline void init_funky_net (struct net_device *d) {}
-    #endif
-
-(代�文件里)
-    dev = alloc_etherdev (sizeof(struct funky_private));
-    if (!dev)
-        return -ENODEV;
-    init_funky_net(dev);
-
-3) 'static inline' 比�好
-
-Static inline 函数相比��说，是好得多的选择。Static inline 函数�供了
-类型安全，没有长度�制，没有格��制，在 gcc 下开销和�一样�。
-
-��在 static inline 函数�是最优的时候[在 fast paths 里有很少的独立的
-案例]，或者��能用 static inline 函数的时候[例如字符串分�]。
-应该用 'static inline' 而�是 'static __inline__', 'extern inline' 和
-'extern __inline__' 。
-
-4) ��过度设计
-
-��试图预计模糊的未�事情，这些事情也许有用也许没有用："让事情尽�能的
-简�，而�是更简�"。
-
-----------------
-第三节 �考文献
-----------------
-
-Andrew Morton, "The perfect patch" (tpp).
-  <http://www.ozlabs.org/~akpm/stuff/tpp.txt>
-
-Jeff Garzik, "Linux kernel patch submission format".
-  <http://linux.yyz.us/patch-format.html>
-
-Greg Kroah-Hartman, "How to piss off a kernel subsystem maintainer".
-  <http://www.kroah.com/log/2005/03/31/>
-  <http://www.kroah.com/log/2005/07/08/>
-  <http://www.kroah.com/log/2005/10/19/>
-  <http://www.kroah.com/log/2006/01/11/>
-
-NO!!!! No more huge patch bombs to linux-kernel@vger.kernel.org people!
-  <https://lkml.org/lkml/2005/7/11/336>
-
-Kernel Documentation/process/coding-style.rst:
-  <http://sosdg.org/~coywolf/lxr/source/Documentation/process/coding-style.rst>
-
-Linus Torvalds's mail on the canonical patch format:
-  <http://lkml.org/lkml/2005/4/7/183>
---
diff --git a/Documentation/translations/zh_CN/disclaimer-zh_CN.rst b/Documentation/translations/zh_CN/disclaimer-zh_CN.rst
new file mode 100644
index 000000000000..dcf803ede85a
--- /dev/null
+++ b/Documentation/translations/zh_CN/disclaimer-zh_CN.rst
@@ -0,0 +1,9 @@
+:orphan:
+
+.. warning::
+     此文件的目的是为让中文读者更容易阅读和�解，而�是作为一个分支。 因此，
+     如果您对此文件有任何��或更新，请先�试更新原始英文文件。
+
+.. note::
+     如果您�现本文档与原始文件有任何��或者有翻译问题，请�系该文件的译者，
+     或者请求时奎亮的帮助：<alex.shi@linux.alibaba.com>。
diff --git a/Documentation/translations/zh_CN/index.rst b/Documentation/translations/zh_CN/index.rst
index 75956d669962..d3165535ec9e 100644
--- a/Documentation/translations/zh_CN/index.rst
+++ b/Documentation/translations/zh_CN/index.rst
@@ -3,10 +3,19 @@
 	\renewcommand\thesection*
 	\renewcommand\thesubsection*
 
-Chinese translations
-====================
+中文翻译
+========
+
+这些手册包�有关如何开�内核的整体信�。内核社区�常庞大，一年下�有数��开�
+人员�出贡献。 与任何大型社区一样，知�如何完�任务将使得更改�并的过程�得更
+加容易。
 
 .. toctree::
-   :maxdepth: 1
+   :maxdepth: 2
+
+   process/index
+
+目录和表格
+----------
 
-   coding-style
+* :ref:`genindex`
diff --git a/Documentation/translations/zh_CN/magic-number.txt b/Documentation/translations/zh_CN/magic-number.txt
deleted file mode 100644
index 7159cec04090..000000000000
--- a/Documentation/translations/zh_CN/magic-number.txt
+++ /dev/null
@@ -1,153 +0,0 @@
-Chinese translated version of Documentation/process/magic-number.rst
-
-If you have any comment or update to the content, please post to LKML directly.
-However, if you have problem communicating in English you can also ask the
-Chinese maintainer for help.  Contact the Chinese maintainer, if this
-translation is outdated or there is problem with translation.
-
-Chinese maintainer: Jia Wei Wei <harryxiyou@gmail.com>
----------------------------------------------------------------------
-Documentation/process/magic-number.rst的中文翻译
-
-如果想评论或更新本文的内容，请直接�信到LKML。如果你使用英文交�有困难的�，也�
-以�中文版维护者求助。如果本翻译更新��时或者翻译存在问题，请�系中文版维护者。
-
-中文版维护者： 贾�� Jia Wei Wei <harryxiyou@gmail.com>
-中文版翻译者： 贾�� Jia Wei Wei <harryxiyou@gmail.com>
-中文版校译者： 贾�� Jia Wei Wei <harryxiyou@gmail.com>
-
-以下为正文
----------------------------------------------------------------------
-这个文件是有关当�使用的魔术值注册表。当你给一个结构添加了一个魔术值，你也应该把这个魔术值添加到这个文件，因为我们最好把用于��结构的魔术值统一起�。
-
-使用魔术值��护内核数�结构是一个�常好的主�。这就�许你在�行期检查(a)一个结构是�已�被攻击，或者(b)你已�给一个例行程�通过了一个错误的结构。�一�情况特别地有用---特别是当你通过一个空指针指�结构体的时候。tty��，例如，�常通过特定驱动使用这�方法并且��地排列特定方�的结构。
-
-使用魔术值的方法是在结构的开始处声明的，如下：
-
-struct tty_ldisc {
-	int	magic;
-	...
-};
-
-当你以�给内核添加增强功能的时候，请�守这�规则�这样就会节�数�清的调试时间，特别是一些�怪的情况，例如，数组超出范围并且�新写了超出部分。�守这个规则，‪这些情况�以被快速地，安全地��。
-
-		Theodore Ts'o
-		  31 Mar 94
-
-给当�的Linux 2.1.55添加魔术表。
-
-		Michael Chastain
-		<mailto:mec@shout.net>
-		22 Sep 1997
-
-现在应该最新的Linux 2.1.112.因为在特性冻结期间，�能在2.2.x�改�任何东西。这些�目被数域所排�。
-
-		Krzysztof G.Baranowski
-	        <mailto: kgb@knm.org.pl>
-		29 Jul 1998
-
-更新魔术表到Linux 2.5.45。刚好越过特性冻结，但是有�能还会有一些新的魔术值在2.6.x之��入到内核中。
-
-		Petr Baudis
-		<pasky@ucw.cz>
-		03 Nov 2002
-
-更新魔术表到Linux 2.5.74。
-
-		Fabian Frederick
-                <ffrederick@users.sourceforge.net>
-		09 Jul 2003
-
-魔术�                 地�         结构               所在文件
-===========================================================================
-PG_MAGIC              'P'         pg_{read,write}_hdr include/linux/pg.h
-CMAGIC                0x0111      user              include/linux/a.out.h
-MKISS_DRIVER_MAGIC    0x04bf      mkiss_channel     drivers/net/mkiss.h
-HDLC_MAGIC            0x239e      n_hdlc            drivers/char/n_hdlc.c
-APM_BIOS_MAGIC        0x4101      apm_user          arch/x86/kernel/apm_32.c
-CYCLADES_MAGIC        0x4359      cyclades_port     include/linux/cyclades.h
-DB_MAGIC              0x4442      fc_info           drivers/net/iph5526_novram.c
-DL_MAGIC              0x444d      fc_info           drivers/net/iph5526_novram.c
-FASYNC_MAGIC          0x4601      fasync_struct     include/linux/fs.h
-FF_MAGIC              0x4646      fc_info           drivers/net/iph5526_novram.c
-ISICOM_MAGIC          0x4d54      isi_port          include/linux/isicom.h
-PTY_MAGIC             0x5001                        drivers/char/pty.c
-PPP_MAGIC             0x5002      ppp               include/linux/if_pppvar.h
-SERIAL_MAGIC          0x5301      async_struct      include/linux/serial.h
-SSTATE_MAGIC          0x5302      serial_state      include/linux/serial.h
-SLIP_MAGIC            0x5302      slip              drivers/net/slip.h
-STRIP_MAGIC           0x5303      strip             drivers/net/strip.c
-X25_ASY_MAGIC         0x5303      x25_asy           drivers/net/x25_asy.h
-SIXPACK_MAGIC         0x5304      sixpack           drivers/net/hamradio/6pack.h
-AX25_MAGIC            0x5316      ax_disp           drivers/net/mkiss.h
-TTY_MAGIC             0x5401      tty_struct        include/linux/tty.h
-MGSL_MAGIC            0x5401      mgsl_info         drivers/char/synclink.c
-TTY_DRIVER_MAGIC      0x5402      tty_driver        include/linux/tty_driver.h
-MGSLPC_MAGIC          0x5402      mgslpc_info       drivers/char/pcmcia/synclink_cs.c
-TTY_LDISC_MAGIC       0x5403      tty_ldisc         include/linux/tty_ldisc.h
-USB_SERIAL_MAGIC      0x6702      usb_serial        drivers/usb/serial/usb-serial.h
-FULL_DUPLEX_MAGIC     0x6969                        drivers/net/ethernet/dec/tulip/de2104x.c
-USB_BLUETOOTH_MAGIC   0x6d02      usb_bluetooth     drivers/usb/class/bluetty.c
-RFCOMM_TTY_MAGIC      0x6d02                        net/bluetooth/rfcomm/tty.c
-USB_SERIAL_PORT_MAGIC 0x7301      usb_serial_port   drivers/usb/serial/usb-serial.h
-CG_MAGIC              0x00090255  ufs_cylinder_group include/linux/ufs_fs.h
-RPORT_MAGIC           0x00525001  r_port            drivers/char/rocket_int.h
-LSEMAGIC              0x05091998  lse               drivers/fc4/fc.c
-GDTIOCTL_MAGIC        0x06030f07  gdth_iowr_str     drivers/scsi/gdth_ioctl.h
-RIEBL_MAGIC           0x09051990                    drivers/net/atarilance.c
-NBD_REQUEST_MAGIC     0x12560953  nbd_request       include/linux/nbd.h
-RED_MAGIC2            0x170fc2a5  (any)             mm/slab.c
-BAYCOM_MAGIC          0x19730510  baycom_state      drivers/net/baycom_epp.c
-ISDN_X25IFACE_MAGIC   0x1e75a2b9  isdn_x25iface_proto_data
-                                                    drivers/isdn/isdn_x25iface.h
-ECP_MAGIC             0x21504345  cdkecpsig         include/linux/cdk.h
-LSOMAGIC              0x27091997  lso               drivers/fc4/fc.c
-LSMAGIC               0x2a3b4d2a  ls                drivers/fc4/fc.c
-WANPIPE_MAGIC         0x414C4453  sdla_{dump,exec}  include/linux/wanpipe.h
-CS_CARD_MAGIC         0x43525553  cs_card           sound/oss/cs46xx.c
-LABELCL_MAGIC         0x4857434c  labelcl_info_s    include/asm/ia64/sn/labelcl.h
-ISDN_ASYNC_MAGIC      0x49344C01  modem_info        include/linux/isdn.h
-CTC_ASYNC_MAGIC       0x49344C01  ctc_tty_info      drivers/s390/net/ctctty.c
-ISDN_NET_MAGIC        0x49344C02  isdn_net_local_s  drivers/isdn/i4l/isdn_net_lib.h
-SAVEKMSG_MAGIC2       0x4B4D5347  savekmsg          arch/*/amiga/config.c
-CS_STATE_MAGIC        0x4c4f4749  cs_state          sound/oss/cs46xx.c
-SLAB_C_MAGIC          0x4f17a36d  kmem_cache        mm/slab.c
-COW_MAGIC             0x4f4f4f4d  cow_header_v1     arch/um/drivers/ubd_user.c
-I810_CARD_MAGIC       0x5072696E  i810_card         sound/oss/i810_audio.c
-TRIDENT_CARD_MAGIC    0x5072696E  trident_card      sound/oss/trident.c
-ROUTER_MAGIC          0x524d4157  wan_device        [in wanrouter.h pre 3.9]
-SAVEKMSG_MAGIC1       0x53415645  savekmsg          arch/*/amiga/config.c
-GDA_MAGIC             0x58464552  gda               arch/mips/include/asm/sn/gda.h
-RED_MAGIC1            0x5a2cf071  (any)             mm/slab.c
-EEPROM_MAGIC_VALUE    0x5ab478d2  lanai_dev         drivers/atm/lanai.c
-HDLCDRV_MAGIC         0x5ac6e778  hdlcdrv_state     include/linux/hdlcdrv.h
-PCXX_MAGIC            0x5c6df104  channel           drivers/char/pcxx.h
-KV_MAGIC              0x5f4b565f  kernel_vars_s     arch/mips/include/asm/sn/klkernvars.h
-I810_STATE_MAGIC      0x63657373  i810_state        sound/oss/i810_audio.c
-TRIDENT_STATE_MAGIC   0x63657373  trient_state      sound/oss/trident.c
-M3_CARD_MAGIC         0x646e6f50  m3_card           sound/oss/maestro3.c
-FW_HEADER_MAGIC       0x65726F66  fw_header         drivers/atm/fore200e.h
-SLOT_MAGIC            0x67267321  slot              drivers/hotplug/cpqphp.h
-SLOT_MAGIC            0x67267322  slot              drivers/hotplug/acpiphp.h
-LO_MAGIC              0x68797548  nbd_device        include/linux/nbd.h
-OPROFILE_MAGIC        0x6f70726f  super_block       drivers/oprofile/oprofilefs.h
-M3_STATE_MAGIC        0x734d724d  m3_state          sound/oss/maestro3.c
-VMALLOC_MAGIC         0x87654320  snd_alloc_track   sound/core/memory.c
-KMALLOC_MAGIC         0x87654321  snd_alloc_track   sound/core/memory.c
-PWC_MAGIC             0x89DC10AB  pwc_device        drivers/usb/media/pwc.h
-NBD_REPLY_MAGIC       0x96744668  nbd_reply         include/linux/nbd.h
-ENI155_MAGIC          0xa54b872d  midway_eprom	    drivers/atm/eni.h
-CODA_MAGIC            0xC0DAC0DA  coda_file_info    include/linux/coda_fs_i.h
-DPMEM_MAGIC           0xc0ffee11  gdt_pci_sram      drivers/scsi/gdth.h
-YAM_MAGIC             0xF10A7654  yam_port          drivers/net/hamradio/yam.c
-CCB_MAGIC             0xf2691ad2  ccb               drivers/scsi/ncr53c8xx.c
-QUEUE_MAGIC_FREE      0xf7e1c9a3  queue_entry       drivers/scsi/arm/queue.c
-QUEUE_MAGIC_USED      0xf7e1cc33  queue_entry       drivers/scsi/arm/queue.c
-HTB_CMAGIC            0xFEFAFEF1  htb_class         net/sched/sch_htb.c
-NMI_MAGIC             0x48414d4d455201 nmi_s        arch/mips/include/asm/sn/nmi.h
-
-请注�，在声音记忆管�中�然有一些特殊的为�个驱动定义的魔术值。查看include/sound/sndmagic.h�获�他们完整的列表信�。很多OSS声音驱动拥有自己从声�PCI ID构建的魔术值-他们也没有被列在这里。
-
-IrDA�系统也使用了大�的自己的魔术值，查看include/net/irda/irda.h�获�他们完整的信�。
-
-HFS是�外一个比较大的使用魔术值的文件系统-你�以在fs/hfs/hfs.h中找到他们。
diff --git a/Documentation/translations/zh_CN/oops-tracing.txt b/Documentation/translations/zh_CN/oops-tracing.txt
index a893f04dfd5d..93fa061cf9e4 100644
--- a/Documentation/translations/zh_CN/oops-tracing.txt
+++ b/Documentation/translations/zh_CN/oops-tracing.txt
@@ -16,7 +16,7 @@ Documentation/admin-guide/bug-hunting.rst 的中文翻译
 
 中文版维护者： �瑞 Dave Young <hidave.darkstar@gmail.com>
 中文版翻译者： �瑞 Dave Young <hidave.darkstar@gmail.com>
-中文版校译者： �阳 Li Yang <leo@zh-kernel.org>
+中文版校译者： �阳 Li Yang <leoyang.li@nxp.com>
                王� Wang Cong <xiyou.wangcong@gmail.com>
 
 以下为正文
diff --git a/Documentation/translations/zh_CN/process/1.Intro.rst b/Documentation/translations/zh_CN/process/1.Intro.rst
new file mode 100644
index 000000000000..10a15f3dc282
--- /dev/null
+++ b/Documentation/translations/zh_CN/process/1.Intro.rst
@@ -0,0 +1,186 @@
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: :ref:`Documentation/process/1.Intro.rst <development_process_intro>`
+:Translator: Alex Shi <alex.shi@linux.alibaba.com>
+
+.. _cn_development_process_intro:
+
+介�
+====
+
+执行摘�
+--------
+
+本节的其余部分涵盖了内核开�过程的范围，以�开�人员�其雇主在这方��能�
+到的��挫折。内核代�应该�并到正�的（“主线�）内核中有很多原因，包括对用
+户的自动�用性�多�形�的社区支�以�影�内核开�方�的能力。�供给Linux
+内核的代�必须在与GPL兼容的许��下�用。
+
+:ref:`cn_development_process` 介�了开�过程�内核�布周期和�并窗�的机制。
+涵盖了补�开��审查和�并周期中的�个阶段。有一些关于工具和邮件列表的讨论。
+鼓励希望开始内核开�的开�人员作为�始练习跟踪并修�bug。
+
+
+:ref:`cn_development_early_stage` 包括早期项目规划，�点是尽快让开�社区�与
+
+:ref:`cn_development_coding` 是关于编�过程的；讨论了其他开�人员�到的几个
+陷阱。对补�的一些�求已�涵盖，并且介�了一些工具，这些工具有助于确�内核
+补�是正确的。
+
+:ref:`cn_development_posting` 讨论�布补�以供评审的过程。为了让开�社区
+认真对待，补�必须正确格�化和�述，并且必须��到正确的地方。�循本节中的
+建议有助于确�为您的工作�供最好的接纳。
+
+:ref:`cn_development_followthrough` 介�了�布补�之��生的事情；该工作
+在这一点上还远远没有完�。与审阅者一起工作是开�过程中的一个��部分；本节
+�供了一些关于如何在这个��阶段��问题的�示。当补�被�并到主线中时，
+开�人员�注����定任务已�完�。
+
+:ref:`cn_development_advancedtopics` 介�了两个“高级�主题：
+使用Git管�补�和查看其他人�布的补�。
+
+:ref:`cn_development_conclusion` 总结了有关内核开�的更多信�，附带有带有
+指�资�的链接.
+
+这个文件是关于什么的
+--------------------
+
+Linux内核有超过800万行代�，�个版本的贡献者超过1000人，是现存最大�最活跃
+的�费软件项目之一。从1991年开始，这个内核已��展�为一个最好的�作系统
+组件，�行在袖�数字音�播放器���PC�现存最大的超级计算机以�所有类型的
+系统上。它是一�适用于几乎任何情况的�壮�高效和�扩展的解决方案。
+
+��Linux的�展，希望�与其开�的开�人员（和公�）的数�也在增加。硬件供应商
+希望确�Linux能够很好地支�他们的产�，使这些产�对Linux用户具有�引力。嵌入
+�系统供应商使用Linux作为集�产�的组件，希望Linux能够尽�能地胜任手头的任务。
+分销商和其他基于Linux的软件供应商对Linux内核的功能�性能和��性有�明确的
+兴趣。最终用户也常常希望修改Linux，使之更好地满足他们的需求。
+
+Linux最引人注目的特性之一是这些开�人员�以访问它；任何具备必�技能的人都�以
+改进Linux并影�其开�方�。专有产��能�供这�开放性，这是自由软件的一个特点。
+但是，如果有什么��的�，内核比大多数其他自由软件项目更开放。一个典型的三个月
+内核开�周期�以涉�1000多个开�人员，他们为100多个��的公�
+（或者根本没有公�）工作。
+
+与内核开�社区�作并�是特别困难。但是，尽管如此，许多潜在的贡献者在�试�
+内核工作时�到了困难。内核社区已��展了自己独特的�作方�，使其能够在�天
+都�更改数�行代�的环境中顺利�行（并生�高质�的产�）。因此，Linux内核开�
+过程与专有的开�方法有很大的��也就�足为奇了。
+
+对于新开�人员�说，内核的开�过程�能会让人感到奇怪和�惧，但这个背�有充分的
+�由和�实的�验。一个�了解内核社区的方�的开�人员（或者更糟的是，他们试图
+抛弃或规�内核社区的方�）会有一个令人沮丧的体验。开�社区, 在帮助那些试图学习
+的人的�时，没有时间帮助那些�愿�倾�或�关心开�过程的人。
+
+希望阅读本文的人能够��这�令人沮丧的�历。这里有很多�料，但阅读时所�的
+努力会在短时间内得到回报。开�社区总是需�能让内核�更好的开�人员；下�的
+文本应该帮助您或为您工作的人员加入我们的社区。
+
+致谢
+----
+
+本文件由Jonathan Corbet撰写，corbet@lwn.net。以下人员的建议使之更为完善：
+Johannes Berg, James Berry, Alex Chiang, Roland Dreier, Randy Dunlap,
+Jake Edge, Jiri Kosina, Matt Mackall, Arthur Marsh, Amanda McPherson,
+Andrew Morton, Andrew Price, Tsugikazu Shibata, 和 Jochen Voß.
+
+这项工作得到了Linux基金会的支�，特别感谢Amanda McPherson，他看到了这项工作
+的价值并把它��现实。
+
+代�进入主线的��性
+--------------------
+
+有些公�和开�人员�尔会想，为什么他们�费心学习如何与内核社区�作，并将代�
+放入主线内核（“主线�是由Linus Torvalds维护的内核，Linux�行商将其用作基础）。
+在短期内，贡献代�看起��是一��以��的开销；仅仅将代�分开并直接支�用户
+似乎更容易。事实上，��代�独立（“树外�）是在�济上是错误的。
+
+作为说明树外代��本的一�方法，下�是内核开�过程的一些相关方�；本文��将
+更详细地讨论其中的大部分内容。考虑：
+
+- 所有Linux用户都�以使用�并到主线内核中的代�。它将自动出现在所有�用它的
+  �行版上。�需�驱动程��盘�下载，也�需�为多个�行版的多个版本�供支�；
+  对于开�人员和用户�说，这一切都是�行的。并入主线解决了大�的分布和支�问题
+
+- 当内核开�人员努力维护一个稳定的用户空间接�时，内部内核API处于�断�化之中.
+  缺�一个稳定的内部接�是一个深�熟虑的设计决策；它�许在任何时候进行基本的改
+  进，并产生更高质�的代�。但该策略的一个结果是，如果�使用新的内核，任何树外
+  代�都需��续的维护。维护树外代�需�大�的工作�能使代���工作状�。
+
+  相�，�于主线中的代��需�这样�，因为一个简�的规则�求进行API更改的任何
+  开�人员也必须修�由于该更改而破�的任何代�。因此，�并到主线中的代�大大
+  �低了维护�本。
+
+- 除此之外，内核中的代�通常会被其他开�人员改进。令人惊讶的结果�能�自授�
+  您的用户社区和客户改进您的产�。
+
+- 内核代�在�并到主线之�和之�都��过审查。�管原始开�人员的技能有多强，
+  这个审查过程总是能找到改进代�的方法。审查�常�现严�的错误和安全问题。
+  这对于在�闭环境中开�的代�尤其如此；这�代�从外部开�人员的审查中获益
+  匪浅。树外代�是低质�代�。
+
+- �与开�过程是您影�内核开�方�的方�。�观者的抱怨会被�到，但是活跃的
+  开�人员有更强的声音——并且能够实现使内核更好地满足其需求的更改。
+
+- 当�独维护代�时，总是存在第三方为类似功能�供��实现的�能性。如果�生
+  这�情况，�并代�将�得更加困难——甚至到了��能的地步。然�，您将�临以下
+  令人�快的选择：（1）无�期地维护树外的�标准特性，或（2）放弃代�并将用户
+  �移到树内版本。
+
+- 代�的贡献是使整个过程工作的根本。通过贡献代�，您�以�内核添加新功能，并
+  �供其他内核开�人员使用的功能和示例。如果您已�为Linux开�了代�（或者
+  正在考虑这样�），那么您显然对这个平�的�续�功感兴趣；贡献代�是确��功
+  的最好方法之一。
+
+上述所有�由都适用于任何树外内核代�，包括以专有的�仅二进制形�分�的代�。
+然而，在考虑任何类型的纯二进制内核代�分布之�，还需�考虑其他因素。这些包括：
+
+- 围绕专有内核模�分�的法律问题充其�是模糊的；相当多的内核版�所有者认为，
+  大多数仅�二进制的模�是内核的派生产�，因此，它们的分���了GNU通用公共
+  许��（下�将详细介�）。您的作者�是律师，本文档中的任何内容都��能被
+  视为法律建议。�闭�代�模�的真实法律地��能由法院决定。但�管怎样，困扰
+  这些模�的�确定性�然存在。
+
+- 二进制模�大大增加了调试内核问题的难度，以至于大多数内核开�人员甚至都�会
+  �试。因此，�分�二进制模�将使您的用户更难从社区获得支�。
+
+- 对于�支�二进制的模�的�行者�说，支�也更加困难，他们必须为他们希望支�
+  的�个�行版和�个内核版本�供一个版本的模�。为了�供相当全�的覆盖范围，
+  �能需�一个模�的几�个构建，并且�次�级内核时，您的用户都必须�独�级
+  您的模�。
+
+- 上��到的关于代�评审的所有问题都更加存在于�闭�代�。由于该代�根本��
+  用，因此社区无法对其进行审查，毫无疑问，它将存在严�问题。
+
+尤其是嵌入�系统的制造商，�能会倾�于忽视本节中所说的大部分内容，因为他们
+相信自己正在商用一�使用冻结内核版本的独立产�，在�布��需��进行开�。
+这个论点忽略了广泛的代�审查的价值以��许用户�产�添加功能的价值。但这些
+产�也有有�的商业寿命，之�必须�布新版本的产�。在这一点上，代�在主线上
+并得到良好维护的供应商将能够更好地��，以使新产�快速上市。
+
+许�
+----
+
+代�是根�一些许���供给Linux内核的，但是所有代�都必须与GNU通用公共许�
+�（GPLV2）的版本2兼容，该版本是覆盖整个内核分�的许��。在实践中，这�味
+�所有代�贡献都由GPLv2（�选地，语言�许在更高版本的GPL下分�）或3��BSD
+许�（New BSD License, 译者注）覆盖。任何�包�在兼容许��中的贡献都�会
+被接�到内核中。
+
+贡献给内核的代��需�（或请求）版�分�。�并到主线内核中的所有代�都�留
+其原始所有�；因此，内核现在拥有数�个所有者。
+
+这�所有�结构的一个暗示是，任何改�内核许�的�试都注定会失败。很少有实际
+的场景�以获得所有版�所有者的��（或者从内核中删除他们的代�）。因此，特
+别是，在�预�的将�，��能�移到GPL的版本3。
+
+所有贡献给内核的代�都必须是�法的�费软件。因此，�接�匿�（或匿�）贡献
+者的代�。所有贡献者都需�在他们的代�上“sign off�，声明代��以在GPL下与内
+核一起分�。无法�供未被其所有者许�为�费软件的代�，或�能为内核造�版�
+相关问题的代�（例如，由缺�适当�护的��工程工作派生的代�）�能被接�。
+
+有关版�相关问题的问题在Linux开�邮件列表中很常�。这样的问题通常会得到�少
+答案，但�记�，回答这些问题的人�是律师，�能�供法律咨询。如果您有关于
+Linux�代�的法律问题，那么与了解该领域的律师交�是无法替代的。��从技术
+邮件列表中获得的答案是一件冒险的事情。
+
diff --git a/Documentation/translations/zh_CN/process/2.Process.rst b/Documentation/translations/zh_CN/process/2.Process.rst
new file mode 100644
index 000000000000..ceb733bb0294
--- /dev/null
+++ b/Documentation/translations/zh_CN/process/2.Process.rst
@@ -0,0 +1,360 @@
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: :ref:`Documentation/process/2.Process.rst <development_process>`
+:Translator: Alex Shi <alex.shi@linux.alibaba.com>
+
+.. _cn_development_process:
+
+开��程如何工作
+================
+
+90年代早期的Linux内核开�是一件相当�散的事情，涉�的用户和开�人员相对较
+少。由于拥有数以百万计的用户群，并且在一年的时间里有大约2000�开�人员�与
+进�，内核因此必须�展许多�程���开�的顺利进行。��为�程的有效组�
+部分，需�对�程的工作方�有一个扎实的�解。
+
+总览
+----
+
+内核开�人员使用一个�散的基于时间的�布过程，�两到三个月�布一次新的主�
+内核版本。最近的�布历�记录如下：
+
+	======  =================
+	4.11	四月 30, 2017
+	4.12	七月 2, 2017
+	4.13	�月 3, 2017
+	4.14	�一月 12, 2017
+	4.15	一月 28, 2018
+	4.16	四月 1, 2018
+	======  =================
+
+�4.x版本都是一个主�的内核版本，具有新特性�内部API更改等等。一个典型的4.x
+版本包�大约13000个�更集，�更了几�万行代�。因此，4.x是Linux内核开�的�
+沿；内核使用滚动开�模型，�断集��大�化。
+
+对于�个版本的补��并，�循一个相对简�的规则。在�个开�周期的开始，“�并
+窗��被打开。当时，被认为足够稳定（并且被开�社区接�）的代�被�并到主线内
+核中。在这段时间内，新开�周期的大部分�更（以�所有主��更）将以接近�天
+1000次�更（“补��或“�更集�）的速度�并。
+
+（顺便说一�，值得注�的是，�并窗�期间集�的更改并�是凭空产生的；它们是
+��收集�测试和分级的。��将详细�述该过程的工作方�）。
+
+�并窗��续大约两周。在这段时间结�时，LinusTorvalds将声明窗�已关闭，并
+释放第一个“rc�内核。例如，对于目标为4.14的内核，在�并窗�结�时�生的释放
+将被称为4.14-rc1。RC1版本是一个信�，表示�并新特性的时间已�过去，稳定下一
+个内核的时间已�开始。
+
+在接下�的6到10周内，�有修�问题的补��应该�交给主线。有时会�许更大的
+更改，但这�情况很少�生；试图在�并窗�外�并新功能的开�人员往往会�到�
+�好的接待。一般�说，如果您错过了给定特性的�并窗�，最好的�法是等待下一
+个开�周期。（对于以��支�的硬件，�尔会对驱动程�进行例外；如果它们�
+改�已有代�，则�会导致回归，并且应该�以�时安全地添加）。
+
+��修�程�进入主线，补�速度将��时间的推移而�慢。Linus大约�周�布一次
+新的-rc内核；一个正常的系列将在-rc6和-rc9之间，内核被认为足够稳定并最终�布。
+然�，整个过程��新开始了。
+
+例如，这里是4.16的开�周期进行情况（2018年的所有日期）:
+
+	==============  ==============================
+	一月 28	        4.15 稳定版�布
+	二月 11	        4.16-rc1, �并窗�关闭
+	二月 18	        4.16-rc2
+	二月 25	        4.16-rc3
+	三月 4		4.16-rc4
+	三月 11	        4.16-rc5
+	三月 18	        4.16-rc6
+	三月 25	        4.16-rc7
+	四月 1		4.16 稳定版�布
+	==============  ==============================
+
+开�人员如何决定何时结�开�周期并创建稳定的版本？使用的最��的指标是以�
+版本的回归列表。�欢迎出现任何错误，但是那些破�了以�能工作的系统的错误被
+认为是特别严�的。因此，导致回归的补�是��欢迎的，很�能在稳定期内删除。
+
+开�人员的目标是在稳定�布之�修�所有已知的回归。在现实世界中，这�完美是
+很难实现的；在这�规模的项目中，��太多了。有一点，延迟最终版本�会使问题
+�得更糟；等待下一个�并窗�的一堆更改将�大，从而在下次创建更多的回归错误。
+因此，大多数4.x内核都有一些已知的回归错误，�过，希望没有一个是严�的。
+
+一旦一个稳定的版本�布，它正在进行的维护工作就被移交给“稳定团队�，目�由
+Greg Kroah-Hartman组�。稳定团队将使用4.x.y编�方案�定期的�布稳定版本的更
+新。�加入更新版本，补�程�必须（1）修�一个��的bug，（2）已��并到
+下一个开�主线中。内核通常会在超过其�始版本的一个以上的开�周期内接收稳定
+的更新。例如，4.13内核的历�如下
+
+	==============  ===============================
+        �月 3 	        4.13 稳定版�布
+	�月 13	        4.13.1
+	�月 20	        4.13.2
+	�月 27	        4.13.3
+	�月 5	        4.13.4
+	�月 12         4.13.5
+	...	        ...
+	�一月 24       4.13.16
+	==============  ===============================
+
+4.13.16是4.13版本的最终稳定更新。
+
+有些内核被指定为“长期�内核；它们将得到更长时间的支�。在本文中，当�的长期
+内核�其维护者是：
+
+	======  ======================  ==============================
+	3.16	Ben Hutchings		(长期稳定内核)
+	4.1	Sasha Levin
+	4.4	Greg Kroah-Hartman	(长期稳定内核)
+	4.9	Greg Kroah-Hartman
+	4.14	Greg Kroah-Hartman
+	======  ======================  ==============================
+
+为长期支�选择内核纯粹是维护人员有必�和时间�维护该版本的问题。目�还没有
+为�将�布的任何特定版本�供长期支�的已知计划。
+
+补�的生命周期
+--------------
+
+补��会直接从开�人员的键盘进入主线内核。相�，有一个�微��（如果有些�
+正�）的过程，旨在确�对�个补�进行质�审查，并确��个补�实现了一个在主线
+中需�的更改。对于�的修�，这个过程�能会很快�生，或者，在大的和有争议的
+�更的情况下，会�续数年。许多开�人员的挫折�自于对这个过程缺��解或者
+试图绕过它。
+
+为了�少这�挫折感，本文将�述补�如何进入内核。下�是一个介�，它以��
+�想化的方��述了这个过程。更详细的过程将在��的章节中介�。
+
+补�程��历的阶段通常是：
+
+- 设计。这就是补�的真正需求——以�满足这些需求的方�——的所在。设计工作通常
+  是在�涉�社区的情况下完�的，但是如果�能的�，最好是在公开的情况下完�
+  这项工作；这样�以节�很多����新设计的时间。
+
+- 早期评审。补�被�布到相关的邮件列表中，列表中的开�人员会回�他们�能有
+  的任何评论。如果一切顺利的�，这个过程应该会�现补�的任何主�问题。
+
+- 更广泛的评审。当补�接近准备好纳入主线时，它应该被相关的�系统维护人员
+  接�——尽管这�接�并�能��补�会一直延伸到主线。补�将出现在维护人员的
+  �系统树中，并进入 -next 树（如下所述）。当�程工作时，此步骤将导致对补�
+  进行更广泛的审查，并�现由于将此补�与其他人所�的工作集�而导致的任何
+  问题。
+
+- 请注�，大多数维护人员也有日常工作，因此�并补��能�是他们的最高优先级。
+  如果您的补�程�得到了关于所需更改的�馈，那么您应该进行这些更改，或者为
+  �应该进行这些更改的原因辩护。如果您的补�没有评审��，但没有被其相应的
+  �系统或驱动程�维护者接�，那么您应该���懈地将补�更新到当�内核，使
+  其干净地应用，并�断地将其��以供审查和�并。
+
+- �并到主线。最终，一个�功的补�将被�并到由LinusTorvalds管�的主线存储库
+  中。此时�能会出现更多的评论和/或问题；开�人员应对这些问题并解决出现的
+  任何问题很��。
+
+- 稳定版�布。�能�补�影�的用户数�现在很大，因此�能�次出现新的问题。
+
+- 长期维护。虽然开�人员在�并代���能会忘记代�，但这�行为往往会给开�
+  社区留下�良�象。�并代�消除了一些维护负担，因为其他代�将修�由API
+  更改引起的问题。但是，如果代��长期��有用，原始开�人员应该继续为
+  代�负责。
+
+内核开�人员（或他们的雇主）犯的最大错误之一是试图将�程简化为一个
+“�并到主线�步骤。这�方法总是会让所有相关人员感到沮丧。
+
+补�如何进入内核
+----------------
+
+�有一个人�以将补��并到主线内核存储库中：LinusTorvalds。但是，在进入
+2.6.38内核的9500多个补�中，�有112个（大约1.3%）是由Linus自己直接选择的。
+内核项目已��展到一个规模，没有一个开�人员�以在没有支�的情况下检查和
+选择�个补�。内核开�人员处�这�增长的方�是通过使用围绕信任链构建的
+助�系统。
+
+内核代�库在逻辑上被分解为一组�系统：网络�特定的体系结构支��内存管��
+视频设备等。大多数�系统都有一个指定的维护人员，开�人员对该�系统中的代�
+负有全部责任。这些�系统维护者（�散地）是他们所管�的内核部分的守护者；
+他们（通常）会接�一个补�以包�到主线内核中。
+
+�系统维护人员�个人都使用git�代�管�工具管�自己版本的内核�代�树。Git
+等工具（以�Quilt或Mercurial等相关工具）�许维护人员跟踪补�列表，包括作者
+信�和其他元数�。在任何给定的时间，维护人员都�以确定他或她的存储库中的哪
+些补�在主线中找�到。
+
+当�并窗�打开时，顶级维护人员将�求Linus从其存储库中“拉出�他们为�并选择
+的补�。如果Linus��，补��将��他的存储库，�为主线内核的一部分。
+Linus对拉�作中接收到的特定补�的关注程度��相�。很明显，有时他看起�很
+关注。但是，作为一般规则，Linus相信�系统维护人员�会�上游���补�。
+
+�系统维护人员�过�也�以从其他维护人员那里获�补�。例如，网络树是由首先
+在专用于网络设备驱动程��无线网络等的树中积累的补�构建的。此存储链�以
+任�长，但很少超过两个或三个链接。由于链中的�个维护者都信任那些管�较低
+级别树的维护者，所以这个过程称为“信任链�。
+
+显然，在这样的系统中，获�内核补��决于找到正确的维护者。直接�Linus��
+补�通常�是正确的方法。
+
+Next æ ‘
+-------
+
+�系统树链引导补��到内核，但它也�出了一个有趣的问题：如果有人想查看为
+下一个�并窗�准备的所有补�怎么办？开�人员将感兴趣的是，还有什么其他的
+更改有待解决，以查看是�存在需�担心的冲�；例如，更改核心内核函数原型的
+修补程�将与使用该函数旧形�的任何其他修补程�冲�。审查人员和测试人员希望
+在所有这些�更到达主线内核之�，能够访问它们的集�形�中的�更。您�以从所有
+有趣的�系统树中��更改，但这将是一项大型且容易出错的工作。
+
+答案以-next树的形�出现，在这里�系统树被收集以供测试和审查。Andrew Morton
+维护的这些旧树被称为“-mm�（用于内存管�，这就是它的�动�字）。-mm 树集�了
+一长串�系统树中的补�；它还包�一些旨在帮助调试的补�。
+
+除此之外，-mm 还包�大�由Andrew直接选择的补�。这些补��能已��布在邮件
+列表上，或者它们�能应用于内核中没有指定�系统树的部分。结果，-mm 作为一�
+最�手段的�系统树�行；如果没有其他明显的路径�以让补�进入主线，那么它很
+�能以-mm 结�。累积在-mm 中的��补�最终将被转�到适当的�系统树，或者直接
+��到Linus。在典型的开�周期中，大约5-10%的补�通过-mm 进入主线。
+
+当�-mm 补��在“mmotm�（-mm of the moment）目录中找到，地�：
+
+        http://www.ozlabs.org/~akpm/mmotm/
+
+然而，使用mmotm树�能是一�令人沮丧的体验；它甚至�能无法编译。
+
+下一个周期补��并的主�树是linux-next，由Stephen Rothwell 维护。根�设计
+linux-next 是下一个�并窗�关闭�主线的快照。linux-next树在Linux-kernel 和
+Linux-next 邮件列表中�布，�从以下�置下载：
+
+        http://www.kernel.org/pub/linux/kernel/next/
+
+Linux-next 已��为内核开�过程中��或缺的一部分；在一个给定的�并窗�中�并
+的所有补�都应该在�并窗�打开之�的一段时间内找到进入Linux-next 的方法。
+
+Staging æ ‘
+----------
+
+内核�代�树包�drivers/staging/directory，其中有许多驱动程�或文件系统的
+�目录正在被添加到内核树中。它们然需�更多的工作的时候�以�留在
+driver/staging目录中；一旦完�，就�以将它们移到内核中。这是一�跟踪�符�
+Linux内核编�或质�标准的驱动程�的方法，但人们�能希望使用它们并跟踪开�。
+
+Greg Kroah Hartman 目�负责维护staging 树。�需�工作的驱动程�将��给他，
+�个驱动程�在drivers/staging/中都有自己的�目录。除了驱动程��文件之外，
+目录中还应该有一个TODO文件。todo文件列出了驱动程�需�接�的挂起的工作，
+以�驱动程�的任何补�都应该抄�的人员列表。当�的规则�求，staging的驱动
+程�必须至少正确编译。
+
+Staging 是一�相对容易的方法，�以让新的驱动程�进入主线，幸�的是，他们会
+引起其他开�人员的注�，并迅速改进。然而，进入staging并�是故事的结尾；
+staging中没有看到常规进展的代�最终将被删除。�销商也倾�于相对�愿�使用
+staging驱动程�。因此，在�为一��适的主线驱动的路上，staging 充其��是
+一个�留。
+
+工具
+----
+
+从上�的文本�以看出，内核开�过程在很大程度上�赖于在��方�上�集补�的
+能力。如果没有适当强大的工具，整个系统将无法在任何地方正常工作。关于如何使用
+这些工具的教程远远超出了本文档的范围，但是还是有一些指�的空间。
+
+到目�为止，内核社区使用的主��代�管�系统是git。Git是在自由软件社区中开�
+的许多分布�版本控制系统之一。它�常适�内核开�，因为它在处�大型存储库和
+大�补�时性能�常好。它还有一个难以学习和使用的�声，尽管��时间的推移它
+�得更好了。对于内核开�人员�说，对Git的��熟悉几乎是一��求；�使他们�
+将它用于自己的工作，他们也需�Git�跟上其他开�人员（以�主线）正在�的事情。
+
+现在几乎所有的Linux�行版都打包了Git。主页�于：
+
+        http://git-scm.com/
+
+那个页�有指�文档和教程的指针。
+
+在�使用git的内核开�人员中，最�行的选择几乎肯定是mercurial：
+
+        http://www.seleric.com/mercurial/
+
+Mercurial与Git共享许多特性，但它�供了一个界�，许多人觉得它更易于使用。
+
+�一个值得了解的工具是quilt:
+
+        http://savannah.nongnu.org/projects/quilt
+
+Quilt 是一个补�管�系统，而�是�代�管�系统。它�会��时间的推移跟踪历�；
+相�，它��根��断�展的代�库跟踪一组特定的更改。一些主�的�系统维护人员
+使用Quilt�管�打算�上游移动的补�。对于�些树的管�（例如-mm），quilt 是
+最好的工具。
+
+邮件列表
+--------
+
+大�的Linux内核开�工作是通过邮件列表完�的。如果�在�个地方加入至少一个列表，
+就很难�为社区中一个功能完备的�员。但是，Linux邮件列表对开�人员�说也是一个
+潜在的�险，他们�能会被一堆电�邮件淹没，��Linux列表上使用的约定，或者
+两者兼而有之。
+
+大多数内核邮件列表都在vger.kernel.org上�行；主列表�于：
+
+        http://vger.kernel.org/vger-lists.html
+
+�过，也有一些列表托管在别处；其中一些列表�于lists.redhat.com。
+
+当然，内核开�的核心邮件列表是linux-kernel。这个��是一个令人生�的地方；
+�天的信���以达到500�，噪音很高，谈�技术性很强，�与者并�总是表现出
+高度的礼貌。但是，没有其他地方�以让内核开�社区作为一个整体�集在一起；
+��使用此列表的开�人员将错过��信�。
+
+有一些�示�以帮助在linux-kernel生存：
+
+- 将邮件转移到�独的文件夹，而�是主邮箱。我们必须能够�续地忽略洪�。
+
+- ��试图跟踪�一次谈�-其他人都�会。��的是�对感兴趣的主题（尽管请
+  注�，长时间的对��以在�更改电�邮件主题行的情况下�离原始主题）和�与
+  的人进行筛选。
+
+- ��挑事。如果有人试图激起愤怒的�应，忽略他们。
+
+- 当�应Linux内核电�邮件（或其他列表上的电�邮件）时，请为所有相关人员�留
+  cc:header。如果没有强有力的�由（如明确的请求），则�应删除收件人。一定�
+  确�你�回�的人在cc:list中。这个惯例也使你�必在回�邮件时明确�求被抄�。
+
+- 在�出问题之�，�索列表档案（和整个网络）。有些开�人员�能会对那些显然
+  没有完�家庭作业的人感到��烦。
+
+- ��贴顶帖（把你的答案放在你�回�的引文上�的�法）。这会让你的回答更难
+  �解，�象也很差。
+
+- 询问正确的邮件列表。linux-kernel �能是通用的讨论点，但它�是从所有�系统
+  中寻找开�人员的最佳场所。
+
+最�一点——找到正确的邮件列表——是开�人员出错的常�地方。在Linux内核上�出与
+网络相关的问题的人几乎肯定会收到一个礼貌的建议，转而在netdev列表上�出，
+因为这是大多数网络开�人员�常出现的列表。还有其他列表�用于scsi�
+video4linux�ide�filesystem等�系统。查找邮件列表的最佳�置是与内核�代�
+一起打包的MAINTAINERS文件。
+
+开始内核开�
+------------
+
+关于如何开始内核开�过程的问题很常�——�自个人和公�。�样常�的是错误，这
+使得关系的开始比必须的更困难。
+
+公�通常希望�请知�的开�人员��动开�团队。实际上，这是一�有效的技术。
+但它也往往是昂贵的，而且没有增长�验丰富的内核开�人员储备。考虑到时间的
+投入，�以让内部开�人员加快Linux内核的开�速度。花这个时间�以让雇主拥有
+一批了解内核和公�的开�人员，他们也�以帮助培训其他人。从中期�看，这往往
+是更有利�图的方法。
+
+�以�解的是，�个开�人员往往对起步感到茫然。从一个大型项目开始�能会很
+�人；人们往往想先用一些较�的东西�测试水域。这是一些开�人员开始创建修补
+拼写错误或轻微编�风格问题的补�的地方。�幸的是，这样的补�会产生一定程度
+的噪音，这会分散整个开�社区的注�力，因此，越�越多的人看�起它们。希望�
+社区介�自己的新开�人员将无法通过这些方�获得他们想�的那�接待。
+
+Andrew Morton 为有抱负的内核开�人员�供了这个建议
+
+::
+
+        所有内核�学者的No.1项目肯定是“确�内核在所有的机器上，你�以触摸
+        到的，始终�行良好" 通常这样�的方法是与其他人一起解决问题（这
+        �能需����）但这很好——这是内核开�的一部分
+
+(http://lwn.net/articles/283982/)
+
+在没有明显问题需�解决的情况下，建议开�人员查看当�的回归和开放�错误列表.
+解决需�修�的问题没有任何缺点；通过解决这些问题，开�人员将获得处�过程的
+�验，�时与开�社区的其他人建立尊�。
diff --git a/Documentation/translations/zh_CN/process/3.Early-stage.rst b/Documentation/translations/zh_CN/process/3.Early-stage.rst
new file mode 100644
index 000000000000..b8676aec6005
--- /dev/null
+++ b/Documentation/translations/zh_CN/process/3.Early-stage.rst
@@ -0,0 +1,161 @@
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: :ref:`Documentation/process/3.Early-stage.rst <development_early_stage>`
+:Translator: Alex Shi <alex.shi@linux.alibaba.com>
+
+.. _cn_development_early_stage:
+
+早期规划
+========
+
+当考虑一个Linux内核开�项目时，很�能会直接跳进去开始编�。然而，与任何��
+的项目一样，�功的许多基础最好是在第一行代�编写之�就�好了。在早期计划和
+沟通中花费一些时间�以节�更多的时间。
+
+详述问题
+--------
+
+与任何工程项目一样，�功的内核增强从�解决的问题的清晰�述开始。在�些情况
+下，这个步骤很容易：例如，当�个特定硬件需�驱动程�时。�过，在其他方�，
+将实际问题与建议的解决方案混淆是很有诱惑力的，这�能会导致困难。
+
+举个例�：几年�，使用Linux音频的开�人员寻求一�方法��行应用程�，而�因
+系统延迟过大而导致退出或其他工件。他们得到的解决方案是一个内核模�，旨在连
+接到Linux安全模�（LSM）框架中；这个模��以�置为�许特定的应用程�访问
+实时调度程�。这个模�被实现并��到Linux内核邮件列表，在那里它立��到问题。
+
+对于音频开�人员�说，这个安全模�足以解决他们当�的问题。但是，对于更广泛的
+内核社区�说，这被视为对LSM框架的滥用（LSM框架并�打算授予他们原本�具备的
+进程特�），并对系统稳定性造�风险。他们首选的解决方案包括短期的通过rlimit
+机制进行实时调度访问，以�长期的�少延迟的工作。
+
+然而，音频社区看�到他们实施的特定解决方案的过去；他们�愿�接�替代方案。
+由此产生的分歧使这些开�人员对整个内核开�过程感到失望；其中一个开�人员返回
+到音频列表并�布了以下内容：
+
+        有很多�常好的Linux内核开�人员，但他们往往会被一群傲慢的傻瓜所压倒。
+        试图�这些人传达用户需求是浪费时间。他们太“�明�了，根本��到少数人
+        的�。
+
+（http://lwn.net/articles/131776/）
+
+实际情况��；与特定模�相比，内核开�人员更关心系统稳定性�长期维护以�找到
+正确的问题解决方案。这个故事的寓�是把�点放在问题上——而�是具体的解决方案
+上——并在投入创建代�之�与开�社区讨论这个问题。
+
+因此，在考虑一个内核开�项目时，我们应该得到一组简短问题的答案：
+
+ - 究竟需�解决的问题是什么？
+
+ - �此问题影�的用户是�？解决方案应该解决哪些用例？
+
+ - 内核现在为何没能解决这个问题？
+
+�有这样，�能开始考虑�能的解决方案。
+
+
+早期讨论
+--------
+
+在计划内核开�项目时，在开始实施之�与社区进行讨论是很有�义的。早期沟通�以
+通过多�方�节�时间和麻烦：
+
+ - 很�能问题是由内核以您��解的方�解决的。Linux内核很大，具有许多�明显
+   的特性和功能。并�是所有的内核功能都�人们所希望的那样有文档记录，而且很
+   容易��一些东西。你的作者�出了一个完整的驱动程�，�制了一个新作者�
+   知�的现有驱动程�。�新设计现有轮�的代��仅浪费，而且�会被接�到主线
+   内核中。
+
+ - 建议的解决方案中�能有一些元素�适用于主线�并。在编写代�之�，最好先
+   了解这样的问题。
+
+ - 其他开�人员完全有�能考虑过这个问题；他们�能有更好的解决方案的想法，并且
+   �能愿�帮助创建这个解决方案。
+
+在内核开�社区的多年�验给了我们一个明确的教训：闭门设计和开�的内核代�总是
+有一些问题，这些问题�有在代��布到社区中时�会被�现。有时这些问题很严�，
+需�数月或数年的努力�能使代�达到内核社区的标准。一些例�包括：
+
+ - 设计并实现了�处�器系统的DeviceScape网络栈。�有使其适�于多处�器系统，
+   �能将其�并到主线中。在代�中改装�等等是一项困难的任务；因此，这段代�
+   （现在称为mac80211）的�并被推迟了一年多。
+
+ - Reiser4文件系统包�许多功能，核心内核开�人员认为这些功能应该在虚拟文件
+   系统层中实现。它还包括一些特性，这些特性在�将系统暴露于用户引起的死�的
+   情况下是�容易实现的。这些问题的最新�现——以�对其中一些问题的拒�——已�
+   导致Reiser4远离了主线内核。
+
+ - Apparmor安全模�以被认为�安全和���的方�使用内部虚拟文件系统数�结构。
+   这�担心（包括其他）使Apparmor多年�在主线上。
+
+在�一�情况下，通过与内核开�人员的早期讨论，�以��大�的痛苦和�外的工作。
+
+找�交�
+--------
+
+当开�人员决定公开他们的计划时，下一个问题是：我们从哪里开始？答案是找到正确
+的邮件列表和正确的维护者。对于邮件列表，最好的方法是在维护者(MAINTAINERS)文件
+中查找��布的相关�置。如果有一个�适的�系统列表，那么�布它通常比在Linux
+内核上�布更��；您更有�能接触到在相关�系统中具有专业知识的开�人员，并且
+环境�能具支�性。
+
+找到维护人员�能会有点困难。�样，维护者文件是开始的地方。但是，该文件往往�总
+是最新的，并且并�所有�系统都在那里表示。实际上，维护者文件中列出的人员�能
+�是当�实际担任该角色的人员。因此，当对�系�有疑问时，一个有用的技巧是使用
+git（尤其是“git-log�）查看感兴趣的�系统中当�活动的用户。看看�在写补�，
+如果有人的�，�会在这些补�上加上用线签�的。这些人将是帮助新开�项目的最佳
+人选。
+
+找到�适的维护者的任务有时是�常具有挑战性的，以至于内核开�人员添加了一个
+脚本�简化过程：
+
+::
+
+	.../scripts/get_maintainer.pl
+
+当给定“-f�选项时，此脚本将返回给定文件或目录的当�维护者。如果在命令行上传递
+了一个补�，它将列出�能接收补�副本的维护人员。有许多选项�以调节
+get_maintainer.pl�索维护者的难易程度；请�心使用更具攻击性的选项，因为最终
+�能会包括对您正在修改的代�没有真正兴趣的开�人员。
+
+如果所有其他方法都失败了，那么与Andrew Morton交谈�以�为一�有效的方法�跟踪
+特定代�段的维护人员。
+
+何时邮寄？
+----------
+
+如果�能的�，在早期阶段�布你的计划�会有帮助。�述正在解决的问题以�已�
+制定的关于如何实施的任何计划。您�以�供的任何信�都�以帮助开�社区为项目
+�供有用的输入。
+
+在这个阶段�能�生的一件令人沮丧的事情�是敌对的�应，而是很少或根本没有
+�应。�悲的事实是：（1）内核开�人员往往很忙；（2）�缺少有�伟计划和很少
+代�（甚至代��景）支�他们的人；（3）没有人有义务审查或评论别人�表的
+想法。除此之外，高级设计常常��一些问题，这些问题�有在有人真正�试实现
+这些设计时�会被�现；因此，内核开�人员�愿看到代�。
+
+如果�表评论的请求在评论的方�上没有什么效果，���设这�味�对项目没有
+兴趣。�幸的是，你也�能�设你的想法没有问题。在这�情况下，最好的�法是
+继续进行，把你的进展�时通知社区。
+
+获得官方认�
+-----------------------
+
+如果您的工作是在公�环境中完�的，就�大多数Linux内核工作一样，显然，在您将
+公�的计划或代��布到公共邮件列表之�，必须获得适当授�的��的许�。�布
+�确定是�兼容GPL的代��能是有特别问题的；公�的管�层和法律人员越早能够就
+�布内核开�项目达�一致，对�与的�个人都越好。
+
+一些读者�能会认为他们的核心工作是为了支�还没有正�承认存在的产�。将雇主
+的计划公布在公共邮件列表上�能�是一个�行的选择。在这�情况下，有必�考虑
+�密是�真的是必�的；通常�需�把开�计划关在门内。
+
+也就是说，有些情况下，一家公�在开�过程的早期就�能�法地披露其计划。拥有
+�验丰富的内核开�人员的公��以选择以开环的方�进行，��是他们以�能够��
+严�的集�问题。对于没有这�内部专业知识的公�，最好的选择往往是�请外部
+开�商根��密�议审查计划。Linux基金会�行了一个NDA程�，旨在帮助解决这�
+情况；
+
+    http://www.linuxfoundation.org/en/NDA_program
+
+这�审查通常足以��以�出现严�问题，而无需公开披露项目。
diff --git a/Documentation/translations/zh_CN/process/4.Coding.rst b/Documentation/translations/zh_CN/process/4.Coding.rst
new file mode 100644
index 000000000000..5301e9d55255
--- /dev/null
+++ b/Documentation/translations/zh_CN/process/4.Coding.rst
@@ -0,0 +1,290 @@
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: :ref:`Documentation/process/4.Coding.rst <development_coding>`
+:Translator: Alex Shi <alex.shi@linux.alibaba.com>
+
+.. _cn_development_coding:
+
+使代�正确
+======================
+
+虽然对于一个�实的���社区的设计过程有很多��说，但是任何内核开�项目的
+�明都在生�的代�中。它是将由其他开�人员检查并�并（或��并）到主线树中
+的代�。所以这段代�的质�决定了项目的最终�功。
+
+本节将检查编�过程。我们将从内核开�人员出错的几�方�开始。然��点将转移
+到正确的事情和�以帮助这个任务的工具上。
+
+陷阱
+----
+
+编�风格
+********
+
+内核长期以�都有一�标准的编�风格，如
+:ref:`Documentation/translations/zh_CN/process/coding-style.rst <cn_codingstyle>`
+中所述。在大部分时间里，该文件中�述的政策被认为至多是建议性的。因此，内核
+中存在大��符�编�风格准则的代�。代�的存在会给内核开�人员带�两个独立
+的�害。
+
+首先，�相信内核编�标准并���，也�强制执行。事实上，如果没有按照标准对代
+�进行编�，那么�内核添加新代�是�常困难的；许多开�人员甚至会在审查代�之
+��求对代�进行�新格�化。一个与内核一样大的代�库需�一些统一的代�，以使
+开�人员能够快速�解其中的任何部分。所以已�没有空间�存放奇怪的格�化代�了。
+
+�尔，内核的编�风格会与雇主的强制风格�生冲�。在这�情况下，内核的风格必须
+在代��并之�获胜。将代�放入内核�味�以多�方�放弃一定程度的控制�——包括
+控制代�的格�化方�。
+
+�一个陷阱是�定已�在内核中的代�迫切需�编�样�的修�。开�人员�能会开始
+生��新格�化补�，作为熟悉过程的一�方�，或者作为将其�称写入内核�更日志
+的一�方�，或者两者兼而有之。但是纯编�风格的修�被开�社区视为噪音；它们往
+往�到冷�。因此，最好��使用这�类型的补�。由于其他原因，在处�一段代�的
+�时修�它的样�是很自然的，但是编�样�的更改�应该仅为了更改而进行。
+
+编�风格的文档也�应该被视为�对的法律，这是永远�会被��的。如果有一个很好
+的�由�对这�样�（例如，如果拆分为适�80列�制的行，那么它的�读性就会大大
+�低），那么就这样�。
+
+请注�，您还�以使用 ``clang-format`` 工具�帮助您处�这些规则，自动�新格�
+化部分代�，并查看完整的文件，以�现编�样�错误�拼写错误和�能的改进。它还
+�以方便地进行排�，包括对���/��回�文本和其他类似任务。有关详细信�，请
+�阅文件 :ref:`Documentation/process/clang-format.rst <clangformat>`
+
+抽象层
+******
+
+计算机科学教授教学生以�活性和信���的�义广泛使用抽象层。当然，内核广泛
+地使用了抽象；任何涉�数百万行代�的项目都�能�到这一点并存活下�。但�验
+表明，过度或过早的抽象�能和过早的优化一样有害。抽象应用于所需的级别，
+��过度。
+
+在一个简�的级别上，考虑一个函数的�数，该�数总是由所有调用方作为零传递。
+我们�以�留这个论点: 以防有人最终需�使用它�供的�外�活性。�过，到那时，
+实现这个�外�数的代�很有�能以��从未被注�到的微妙方�被破�——因为它从
+未被使用过。或者，当需��外的�活性时，它�会以符�程�员早期期望的方��
+这样�。内核开�人员通常会�交补��删除未使用的�数；一般�说，首先�应该
+添加这些�数。
+
+��硬件访问的抽象层——通常�许大�的驱动程�在多个�作系统中使用——尤其��
+欢迎。这样的层使代��得模糊，�能会造�性能�失；它们�属于Linux内核。
+
+�一方�，如果您�现自己从�一个内核�系统�制了大�的代�，那么现在是时候
+问一下，事实上，将这些代�中的一些��到�独的库中，或者在更高的层次上实现
+这些功能是�有�义。在整个内核中�制相�的代�没有价值。
+
+#ifdef 和预处�
+***************
+
+C预处�器似乎给一些C程�员带�了强大的诱惑，他们认为它是一�有效地将大��
+活性编�到�文件中的方法。但是预处�器�是C，大�使用它会导致代�对其他人�
+说更难读�，对编译器�说更难检查正确性。大�的预处�器几乎总是代�需�一些
+清�工作的标志。
+
+使用ifdef的�件编译实际上是一个强大的功能，它在内核中使用。但是很少有人希望
+看到代�被大�地撒上ifdef�。作为一般规则，ifdef的使用应尽�能�制在头文件
+中。有�件编译的代��以�制函数，如果代��存在，这些函数就会��空的。然�
+编译器将悄悄地优化对空函数的调用。结果是代�更加清晰，更容易�解。
+
+C预处�器�存在许多�险，包括�能对具有副作用且没有类型安全性的表达�进行多
+�评估。如果您试图定义�，请考虑创建一个内�函数。结果相�的代�，但是内�
+函数更容易读�，�会多次计算其�数，并且�许编译器对�数和返回值执行类型检查。
+
+内�函数
+********
+
+�过，内�函数本身也存在风险。程�员�以倾心于��函数调用和用内�函数填充�
+文件所固有的效率。然而，这些功能实际上会�低性能。因为它们的代�在�个调用站
+点都被�制，所以它们最终会增加编译内核的大�。�过�，这会对处�器的内存缓存
+造�压力，从而大大�低执行速度。通常，内�函数应该�常�，而且相对较少。毕竟，
+函数调用的�本并�高；大�内�函数的创建是过早优化的典型例�。
+
+一般�说，内核程�员会忽略缓存效果，这会带��险。在开始的数�结构课程中，�
+典的时间/空间�衡通常�适用于当代硬件。空间就是时间，因为一个大的程�比一个
+更紧凑的程��行得慢。
+
+最近的编译器在决定一个给定函数是�应该被内�方�扮演�越�越积�的角色。
+因此，“inline�关键字的自由放置�能�仅仅是过度的，它也�能是无关的。
+
+�
+**
+
+2006年5月，“deviceescape�网络堆栈在GPL下�布，并被纳入主线内核。这是一个�
+欢迎的消�；对Linux中无线网络的支�充其�被认为是��格的，而deviceescape
+堆栈�供了修�这�情况的承诺。然而，直到2007年6月（2.6.22），这段代��真
+正进入主线。�生了什么？
+
+这段代�显示了许多闭门造车的迹象。但一个特别大的问题是，它并�是设计用于多
+处�器系统。在�并这个网络堆栈（现在称为mac80211）之�，需�对其进行一个�
+方案的改造。
+
+曾�，Linux内核代��以在�考虑多处�器系统所带�的并�性问题的情况下进行
+开�。然而，现在，这个文件是写在�核笔记本电脑上的。�使在�处�器系统上，
+为�高�应能力所�的工作也会�高内核内的并�性水平。编写内核代�而�考虑�
+的日�已�过去很长了。
+
+�以由多个线程并�访问的任何资�（数�结构�硬件寄存器等）必须由��护。新
+的代�应该记�这一�求；事�改装�是一项相当困难的任务。内核开�人员应该花
+时间充分了解�用的�原语，以便为作业选择正确的工具。显示对并�性缺�关注的
+代�进入主线将很困难。
+
+回归
+****
+
+最�一个值得一�的�险是：它�能会引起改�（这�能会带�很大的改进），从而
+导致现有用户的�些东西中断。这��化被称为“回归�，回归已��为主线内核最�
+�欢迎的。除少数例外情况外，如果回归�能�时修正，会导致回归的�化将被�消。
+最好首先��回归。
+
+人们常常争论，如果回归让更多人�以工作，远超过产生问题，那么回归是��的。
+如果它破�的一个系统�为�个系统带�新的功能，为什么�进行更改呢？2007年7月，
+Linus对这个问题给出了最佳答案:
+
+::
+        所以我们�会通过引入新问题�修�错误。那样的谎言很疯狂，没有人知�
+        你是�真的有进展。是�进两步，�退一步，还是��一步，��两步？
+
+（http://lwn.net/articles/243460/）
+
+一�特别��欢迎的回归类型是用户空间ABI的任何�化。一旦接�被导出到用户空间，
+就必须无�期地支�它。这一事实使得用户空间接�的创建特别具有挑战性：因为它们
+�能以�兼容的方�进行更改，所以必须第一次正确地进行更改。因此，用户空间界�
+总是需�大�的�考�清晰的文档和广泛的审查。
+
+
+代�检查工具
+------------
+
+至少目�，编写无错误代��然是我们中很少人能达到的�想状�。�过，我们希望�
+的是，在代�进入主线内核之�，尽�能多地�获并修�这些错误。为此，内核开�人
+员已�组装了一系列令人�象深刻的工具，�以自动�获���样的模糊问题。计算机
+�现的任何问题都是一个以��会困扰用户的问题，因此，��有�能，就应该使用
+自动化工具。
+
+第一步�是注�编译器产生的警告。当代版本的GCC�以检测（并警告）大�潜在错误。
+通常，这些警告都指�真正的问题。�交以供审阅的代�通常�会产生任何编译器警告。
+在消除警告时，注�了解真正的原因，并尽���“修��，使警告消失而�解决其原因。
+
+请注�，并�所有编译器警告都默认�用。使用“make EXTRA_CFLAGS=-W�构建内核以
+获得完整集�。
+
+内核�供了几个�置选项，�以打开调试功能；大多数�置选项�于“kernel hacking�
+���中。对于任何用于开�或测试目的的内核，都应该�用其中几个选项。特别是，
+您应该打开：
+
+ - �用 ENABLE_MUST_CHECK and FRAME_WARN 以获得一组�外的警告，以解决使用�
+   推�使用的接�或忽略函数的��返回值等问题。这些警告生�的输出�能是冗长
+   的，但您�必担心�自内核其他部分的警告。
+
+ - DEBUG_OBJECTS 将添加代�，以跟踪内核创建的��对象的生存期，并在出现问题时
+   �出警告。如果�添加创建（和导出）自己的��对象的�系统，请考虑添加对对象
+   调试基础结构的支�。
+
+ - DEBUG_SLAB �以�现��内存分�和使用错误；它应该用于大多数开�内核。
+
+ - DEBUG_SPINLOCK, DEBUG_ATOMIC_SLEEP and DEBUG_MUTEXES 会�现许多常�的
+   �定错误.
+
+还有很多其他调试选项，其中一些将在下�讨论。其中一些具有显著的性能影�，�应
+一直使用。但是，在学习�用选项上花费的一些时间�能会在短期内得到多次回报。
+
+其中一个较�的调试工具是�定检查器或“lockdep�。该工具将跟踪系统中�个�
+（spinlock或mutex）的获�和释放�获��的相对顺��当�中断环境等等。然�，
+它�以确�总是以相�的顺�获��，相�的中断�设适用于所有情况，等等。���
+说，lockdep�以找到许多场景，在这些场景中，系统很少会死�。在部署的系统中，
+这�问题�能会很痛苦（对于开�人员和用户而言）；LockDep�许��以自动方�
+�现问题。具有任何类型的�普通�定的代�在�交包��应在�用lockdep的情况
+下�行。
+
+作为一个勤奋的内核程�员，毫无疑问，您将检查任何�能失败的�作（如内存分�）
+的返回状�。然而，事实上，最终的故障��路径�能完全没有�过测试。未测试的
+代�往往会被破�；如果所有这些错误处�路径都被执行了几次，那么您�能对代�
+更有信心。
+
+内核�供了一个�以�到这一点的错误注入框架，特别是在涉�内存分�的情况下。
+�用故障注入�，内存分�的��置百分比将失败；这些失败�以�制在特定的代�
+范围内。在�用了故障注入的情况下�行，程�员�以看到当情况�化时代�如何�
+应。有关如何使用此工具的详细信�，请�阅
+Documentation/fault-injection/fault-injection.txt。
+
+使用“sparse���分�工具�以�现其他类型的错误。对于sparse，�以警告程�员
+用户空间和内核空间地�之间的混淆�big endian和small endian数�的混��在需
+�一组�标志的地方传递整数值等等。sparse必须�独安装(如果您的分��务器没
+有将其打包，�以在 https://sparse.wiki.kernel.org/index.php/Main_page)找到,
+然��以通过在make命令中添加“C=1�在代�上�行它。
+
+“Coccinelle�工具 :ref:`http://coccinelle.lip6.fr/ <devtools_coccinelle>`
+能够�现��潜在的编�问题；它还�以为这些问题�出修�方案。在
+scripts/coccinelle目录下已�打包了相当多的内核“语义补��；�行
+“make coccicheck�将�行这些语义补�并报告�现的任何问题。有关详细信�，请�阅
+:ref:`Documentation/dev-tools/coccinelle.rst <devtools_coccinelle>`
+
+
+其他类型的�移�性错误最好通过为其他体系结构编译代���现。如果没有S/390系统
+或Blackfin开��，您�然�以执行编译步骤。�以在以下�置找到一组用于x86系统的
+大型交�编译器：
+
+        http://www.kernel.org/pub/tools/crosstool/
+
+花一些时间安装和使用这些编译器将有助于��以�的尴尬。
+
+文档
+----
+
+文档通常比内核开�规则更为例外。�便如此，足够的文档将有助于简化将新代��并
+到内核中的过程，使其他开�人员的生活更轻�，并对您的用户有所帮助。在许多情况
+下，文件的添加已基本上�为强制性的。
+
+任何补�的第一个文档是其关�的�更日志。日志�目应该�述正在解决的问题�解决
+方案的形��处�补�的人员�对性能的任何相关影�，以��解补��能需�的任何
+其他内容。确�changelog说明了为什么补�值得应用；大�开�人员未能�供这些信�。
+
+任何添加新用户空间界�的代�（包括新的sysfs或/proc文件）都应该包�该界�的
+文档，该文档使用户空间开�人员能够知�他们在使用什么。请�阅
+Documentation/abi/readme，了解如何格�化此文档以�需��供哪些信�。
+
+文件 :ref:`Documentation/admin-guide/kernel-parameters.rst <kernelparameters>`
+�述了内核的所有引导时间�数。任何添加新�数的补�都应该�该文件添加适当的
+�目。
+
+任何新的�置选项都必须附有帮助文本，帮助文本清楚地解释了这些选项以�用户�能
+希望何时选择它们。
+
+许多�系统的内部API信�通过专门格�化的注释进行记录；这些注释�以通过
+“kernel-doc�脚本以多�方���和格�化。如果您在具有kerneldoc注释的�系统中
+工作，则应该维护它们，并根�需�为外部�用的功能添加它们。�使在没有如此记录
+的领域中，为将�添加kerneldoc注释也没有�处；实际上，这对于刚开始开�内核的人
+�说是一个有用的活动。这些注释的格�以�如何创建kerneldoc模�的一些信��以在
+:ref:`Documentation/doc-guide/ <doc_guide>` 上找到。
+
+任何阅读大�现有内核代�的人都会注�到，注释的缺失往往是最值得注�的。�一次，
+对新代�的期望比过去更高；�并未注释的代�将更加困难。这就是说，人们几乎�希望
+用语言注释代�。代�本身应该是�读的，注释解释了更微妙的方�。
+
+�些事情应该总是被注释。使用内存�障时，应附上一行文字，解释为什么需�设置内存
+�障。数�结构的�定规则通常需�在�个地方解释。一般�说，主�数�结构需�全�
+的文档。应该指出�独代��之间�明显的�赖性。任何�能诱使代�看门人进行错误的
+“清��的事情都需�一个注释�说明为什么�这样�。等等。
+
+
+内部API更改
+-----------
+
+内核�供给用户空间的二进制接��能被破�，除�在最严�的情况下。相�，内核的
+内部编程接�是高度�动的，当需�时�以更改。如果你�现自己�得�处�一个内核
+API，或者仅仅因为它�满足你的需求而�使用特定的功能，这�能是API需�改�的一
+个标志。作为内核开�人员，您有�进行此类更改。
+
+当然, �以进行API更改，但它们必须是��的。因此，任何进行内部API更改的补�都
+应该附带一个关于更改内容和必�原因的�述。这��化也应该分解�一个�独的补�，
+而�是埋在一个更大的补�中。
+
+�一个�点是，更改内部API的开�人员通常�负责修�内核树中被更改破�的任何代�。
+对于一个广泛使用的函数，这个�责�以导致�百上�的�化，其中许多�化�能与其他
+开�人员正在�的工作相冲�。�用说，这�能是一项大工作，所以最好确��由是
+��的。请注�，coccinelle工具�以帮助进行广泛的API更改。
+
+在进行�兼容的API更改时，应尽�能确�编译器�获未更新的代�。这将帮助您确�找
+到该接�的树内用处。它还将警告开�人员树外代�存在他们需��应的更改。支�树外
+代��是内核开�人员需�担心的事情，但是我们也�必使树外开�人员的生活有�必�
+的困难。
diff --git a/Documentation/translations/zh_CN/process/5.Posting.rst b/Documentation/translations/zh_CN/process/5.Posting.rst
new file mode 100644
index 000000000000..41aba21ff050
--- /dev/null
+++ b/Documentation/translations/zh_CN/process/5.Posting.rst
@@ -0,0 +1,240 @@
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: :ref:`Documentation/process/5.Posting.rst <development_posting>`
+:Translator: Alex Shi <alex.shi@linux.alibaba.com>
+
+.. _cn_development_posting:
+
+��补�
+========
+
+迟早，当您的工作准备好�交给社区进行审查，并最终包�到主线内核中时。�出所料，
+内核开�社区已��展出一套用于�布补�的约定和过程；�循这些约定和过程将使
+�与其中的�个人的生活更加轻�。本文件将试图��详细地涵盖这些期望；更多信�
+也�在以下文件中找到
+:ref:`Documentation/translations/zh_CN/process/submitting-patches.rst <cn_submittingpatches>`,
+:ref:`Documentation/process/submitting-drivers.rst  <submittingdrivers>`
+和 :ref:`Documentation/translations/zh_CN/process/submit-checklist.rst <cn_submitchecklist>`.
+
+何时邮寄
+--------
+
+在补�完全“准备好�之�，有一个�断的诱惑����布补�。对于简�的补�，
+这�是问题。但是，如果正在完�的工作很��，那么在工作完�之�从社区获得
+�馈就�以获得很多好处。因此，您应该考虑�布正在进行的工作，甚至使Git树
+�用，以便感兴趣的开�人员�以�时赶上您的工作。
+
+当�布还没有准备好包�的代�时，最好在�布本身中这样说。还应��任何有待完�
+的主�工作和任何已知问题。很少有人会看到那些被认为是�生�熟的补�，但是那些
+人会想到他们�以帮助你把工作推�正确的方�。
+
+创建补�之�
+------------
+
+在考虑将补���到开�社区之�，有许多事情应该�。这些包括：
+
+ - 尽�能地测试代�。利用内核的调试工具，确�内核使用所有��的�置选项组�
+   进行构建，使用跨编译器为��的体系结构进行构建等。
+
+ - 确�您的代�符�内核编�风格指�。
+
+ - 您的更改是�具有性能影�？如果是这样，您应该�行基准测试�显示您的�更的
+   影�（或好处）；结果的摘�应该包�在补�中。
+
+ - 确�您有��布代�。如果这项工作是为雇主完�的，雇主对这项工作具有所有�，
+   并且必须��根�GPL对其进行放行。
+
+一般�说，在�布代�之�进行一些�外的�考，几乎总是能在短时间内得到回报。
+
+补�准备
+--------
+
+准备�布补��能是一个惊人的工作�，但�次�试节�时间在这里通常是�明智的，
+�使在短期内。
+
+必须针对内核的特定版本准备补�。作为一般规则，补�程�应该基于Linus的Git树中
+的当�主线。当以主线为基础时，从一个众所周知的�布点开始——一个稳定的或RC的
+�布——而�是在一个主线分支任�点。
+
+但是，�能需�针对-mm�linux-next或�系统树生�版本，以便于更广泛的测试和审查。
+根�补�的区域以�其他地方的情况，针对这些其他树建立补��能需�大�的工作�
+解决冲�和处�API更改。
+
+�有最简�的更改�应格�化为�个补�；其他所有更改都应作为一系列逻辑更改进行。
+分割补�是一门艺术；一些开�人员花了很长时间�弄清楚如何按照社区期望的方��
+�。然而，有一些�验法则�以大大帮助：
+
+ - 您�布的补�程�系列几乎肯定�会是工作系统中的一系列更改。相�，您所�的
+   更改需�在最终形�中加以考虑，然�以有�义的方�进行拆分。开�人员对离散的�
+   自包�的更改感兴趣，而�是您获�这些更改的路径。
+
+ - �个逻辑上独立的�更都应该格�化为�独的补�。这些更改�以是�的（“�此
+   结构添加字段�）或大的（例如，添加一个��的新驱动程�），但它们在概念上
+   应该是�的，并且�以接�一行�述。�个补�都应该�一个特定的更改，�以�独
+   检查并验�它所�的事情。
+
+ - 作为�申上述准则的一�方法：��在�一补�中混���类型的更改。如果一个
+   补�修�了一个关键的安全�洞，�新排列了一些结构，并�新格�化了代�，那么
+   很有�能它会被忽略，而��的修�将丢失。
+
+ - �个补�都应该产生一个内核，它�以正确地构建和�行；如果补�系列在中间被
+   中断，那么结果应该�然是一个工作的内核。补�系列的部分应用是使用
+   “git bisct�工具查找回归的一个常�场景；如果结果是一个��的内核，那么对于
+   那些从事追踪问题的高尚工作的开�人员和用户�说，将使他们的生活更加艰难。
+
+ - �过，��过分。一�开�人员曾�将一组编辑内容作为500个�独的补��布到一个
+   文件中，这并没有使他�为内核邮件列表中最�欢迎的人。一个补��以相当大，
+   ��它�然包�一个�一的逻辑�更。
+
+ - 用一系列补�添加一个全新的基础设施是很有诱惑力的，但是在系列中的最�一个
+   补��用整个补�之�，该基础设施是�使用的。如果�能的�，应该��这�
+   诱惑；如果这个系列增加了回归，那么二分法将指出最�一个补�是导致问题的
+   补�，�使真正的bug在其他地方。��有�能，添加新代�的补�程�应该立�
+   激活该代�。
+
+创建完美补�系列的工作�能是一个令人沮丧的过程，在完�“真正的工作�之�需�花费
+大�的时间和�考。但是，如果�得好，这是一段很好的时间。
+
+补�格�和更改日志
+------------------
+
+所以现在你有了一系列完美的补��以�布，但是这项工作还没有完�。�个补�都
+需�被格�化�一�消�，它�以快速而清晰地将其目的传达给世界其他地方。为此，
+�个补�将由以下部分组�：
+
+ - 命�补�作者的�选“from�行。�有当你通过电�邮件传递别人的补�时，这一行
+   �是必�的，但是如果有疑问，添加它�会有任何伤害。
+
+ - 一行�述补�的作用。对于没有其他上下文的读者�说，此消�应该足够了解补�
+   的范围；这是将在“短格���更日志中显示的行。此消�通常首先用相关的�系统
+   �称格�化，然�是补�的目的。例如：
+
+    ::
+
+	gpio: fix build on CONFIG_GPIO_SYSFS=n
+
+ - 一个空白行，��是补�内容的详细�述。这个�述�以是必需的；它应该说明补�
+   的作用以�为什么它应该应用于内核。
+
+ - 一个或多个标记行，至少有一个由补�作者的：signed-off-by 签�。签�将在下�
+   更详细地�述。
+
+上�的项目一起构�补�的�更日志。写一篇好的�更日志是一门至关��但常常被
+忽视的艺术；值得花一点时间�讨论这个问题。当你写一个�更日志时，你应该记�
+有很多��的人会读你的�。其中包括�系统维护人员和审查人员，他们需�决定是�
+应该包括补�，分销商和其他维护人员试图决定是�应该将补���移�到其他内核，
+bug�寻人员想知�补�是�负责他们正在追查的问题，想知�内核如何�化的用户。
+等等。一个好的�更日志以最直接和最简�的方��所有这些人传达所需的信�。
+
+为此，总结行应该�述�更的影�和动机，以�在一行约��件下�能�生的�化。
+然�，详细的�述�以详述这些主题，并�供任何需�的附加信�。如果补�修�了
+一个bug，请引用引入该bug的commit（如果�能，请在引用commits时�时�供commit id
+和标题）。如果�个问题与特定的日志或编译器输出相关�，请包�该输出以帮助其他
+人�索�一问题的解决方案。如果更改是为了支�以�补�中的其他更改，那么就这么
+说。如果更改了内部API，请详细说明这些更改以�其他开�人员应该如何�应。一般
+�说，你越能把自己放在�个阅读你的changelog的人的�置上，changelog（和内核
+作为一个整体）就越好。
+
+�用说，�更日志应该是将�更�交到修订控制系统时使用的文本。接下�是：
+
+ - 补�本身，采用统一的（“-u�）补�格�。将“-p�选项用于diff将使函数�与更改
+   相关�，从而使结果补�更容易被其他人读�。
+
+您应该��在补�中包括对�相关文件（例如，由构建过程生�的文件或编辑器
+备份文件）的更改。文档目录中的文件“dontdiff�在这方�有帮助；使用“-X�选项将
+其传递给diff。
+
+上��到的标签用于�述��开�人员如何与这个补�的开�相关�。
+:ref:`Documentation/translations/zh_CN/process/submitting-patches.rst <cn_submittingpatches>`
+文档中对它们进行了详细�述；下�是一个简短的总结。�一行的格�如下：
+
+::
+
+	tag: Full Name <email address>  optional-other-stuff
+
+常用的标签有：
+
+ - Signed-off-by: 这是一个开�人员的�明，他或她有��交补�以包�到内核中。
+   这是开���认��议，其全文�在
+   :ref:`Documentation/translations/zh_CN/process/submitting-patches.rst <cn_submittingpatches>`
+   中找到，如果没有适当的签字，则�能�并到主线中。
+
+ - Co-developed-by: 声明补�是由多个开�人员共�创建的；当几个人在一个补�上
+   工作时，它用于将属性赋予共�作者（除了 From: 所赋予的作者之外）。因为
+   Co-developed-by: 表示作者身份，所以�个共�开�人, 必须紧跟在相关�作作者
+   的签�之�。具体内容和示例�以在以下文件中找到
+   :ref:`Documentation/translations/zh_CN/process/submitting-patches.rst <cn_submittingpatches>`
+
+ - Acked-by: 表示�一个开�人员（通常是相关代�的维护人员）��补�适�包�
+   在内核中。
+
+ - Tested-by: 声明指定的人已�测试了补�并�现它�以工作。
+
+ - Reviewed-by: 指定的开�人员已�审查了补�的正确性；有关详细信�，请�阅
+   :ref:`Documentation/translations/zh_CN/process/submitting-patches.rst <cn_submittingpatches>`
+
+ - Reported-by: 指定报告此补�修�的问题的用户；此标记用于�供感谢。
+
+ - Cc：指定的人收到了补�的副本，并有机会对此�表评论。
+
+在补�中添加标签时��心：�有cc:�适�在没有指定人员明确许�的情况下添加。
+
+��补�
+--------
+
+在邮寄补�之�，您还需�注�以下几点：
+
+ - 您确定您的邮件��程��会��补��？有�费的空白更改或由邮件客户端
+   执行的行包装的补��会在�一端�原，并且通常�会进行任何详细检查。如果有
+   任何疑问，把补�寄给你自己，让你自己相信它是完好无�的。
+
+   :ref:`Documentation/translations/zh_CN/process/email-clients.rst <cn_email_clients>`
+   �供了一些有用的�示，�以让特定的邮件客户机工作以��补�。
+
+ - 你确定你的补�没有愚蠢的错误�？您应该始终通过scripts/checkpatch.pl�行
+   补�程�，并解决它�出的投诉。请记�，checkpatch.pl虽然是大��考内核
+   补�应该是什么样�的体现，但它并�比您�明。如果修�checkpatch.pl投诉会
+   使代��得更糟，请��这样�。
+
+补�应始终以纯文本形���。请��将它们作为附件��；这使得审阅者在答�中更难
+引用补�的部分。相�，�需将补�直接放到您的消�中。
+
+邮寄补�时，��的是将副本��给任何�能感兴趣的人。与其他一些项目��，内核
+鼓励人们错误地��过多的副本；���定相关人员会看到您在邮件列表中的�布。
+尤其是，副本应��至：
+
+ - �影��系统的维护人员。如�所述，维护人员文件是查找这些人员的第一个地方。
+
+ - 其他在�一领域工作的开�人员，尤其是那些现在�能在那里工作的开�人员。使用
+   git查看还有�修改了您正在处�的文件，这很有帮助。
+
+ - 如果您对错误报告或功能请求�出�应，也�以抄�原始��人。
+
+ - 将副本��到相关邮件列表，或者，如果没有其他应用，则��到Linux内核列表。
+
+ - 如果您正在修�一个bug，请考虑该修�是�应进入下一个稳定更新。如果是这样，
+   stable@vger.kernel.org 应该得到补�的副本。�外，在补�本身的标签中添加
+   一个“cc:stable@vger.kernel.org�；这将使稳定团队在修�进入主线时收到通知。
+
+当为一个补�选择接收者时，最好知�你认为�最终会接�这个补�并将其�并。虽然
+�以将补�直接��给LinusTorvalds并让他�并，但通常情况下�会这样�。Linus
+很忙，并且有�系统维护人员负责监视内核的特定部分。通常您会希望维护人员�并您
+的补�。如果没有明显的维护人员，Andrew Morton通常是最�的补�目标。
+
+补�需�好的主题行。补�程�行的规范格�如下：
+
+::
+
+	[PATCH nn/mm] subsys: one-line description of the patch
+
+其中“nn�是补�的��，“mm�是系列中补�的总数，“subsys�是�影��系统的�称。
+显然，一个�独的补��以�略nn/mm。
+
+如果您有一系列��的补�，那么通常将介�性�述作为零部分��。�过，这�约定
+并没有得到普��循；如果您使用它，请记�简介中的信��会使它进入内核�更日志。
+因此，请确�补�本身具有完整的�更日志信�。
+
+一般�说，多部分补�的第二部分和�续部分应作为对第一部分的回���，以便它们
+在接收端都连接在一起。�git和coilt这样的工具有命令，�以通过适当的线程��
+一组补�。但是，如果您有一个长系列，并且正在使用git，请远离–chain reply-to
+选项，以��创建异常深的嵌套。
diff --git a/Documentation/translations/zh_CN/process/6.Followthrough.rst b/Documentation/translations/zh_CN/process/6.Followthrough.rst
new file mode 100644
index 000000000000..f509e077e1cb
--- /dev/null
+++ b/Documentation/translations/zh_CN/process/6.Followthrough.rst
@@ -0,0 +1,145 @@
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: :ref:`Documentation/process/6.Followthrough.rst <development_followthrough>`
+:Translator: Alex Shi <alex.shi@linux.alibaba.com>
+
+.. _cn_development_followthrough:
+
+è·Ÿè¿›
+====
+
+在这一点上，您已��循了到目�为止给出的指导方针，并且，��您自己的工程技能
+的增加，已��布了一系列完美的补�。�使是�验丰富的内核开�人员也能犯的最大
+错误之一是，认为他们的工作现在已�完�了。事实上，�布补��味�进入�程的下
+一个阶段，�能还需��很多工作。
+
+一个补�在第一次�布时就�常出色，没有改进的余地,这是很罕�的。内核开��程
+认识到这一事实，因此，它�常注�对已�布代�的改进。作为代�的作者，您应该与
+内核社区�作，以确�您的代�符�内核的质�标准。如果��与这个过程，很�能会
+阻止将补�包�到主线中。
+
+与审阅者�作
+------------
+
+任何�义上的补�都会导致其他开�人员在审查代�时�表大�评论。对于许多开�
+人员�说，与审查人员�作�能是内核开�过程中最令人生�的部分。但是，如果你
+记�一些事情，生活会�得容易得多：
+
+ - 如果你已�很好地解释了你的补�，评论人员会�解它的价值，以�为什么你会
+   费尽心�去写它。但是这个并�能阻止他们�出一个基本的问题：五年或�年�
+   用这个代�维护一个内核会是什么感觉？你�能被�求�出的许多改�——从编�风格
+   的调整到大�的�写——都�自于对Linux的�解，�从现在起�年�，Linux�将在
+   开�中。
+
+ - 代�审查是一项艰苦的工作，这是一项相对�力�讨好的工作；人们记得�编写了
+   内核代�，但对于那些审查它的人�说，几乎没有什么�久的�声。因此，评论
+   人员�能会�得暴�，尤其是当他们看到�样的错误被一��一�地犯下时。如果
+   你得到了一个看起�愤怒�侮辱或完全冒犯你的评论，抵制以�样方�回应的冲动。
+   代�审查是关于代�的，而�是关于人的，代�审查人员�会亲自攻击您。
+
+ - �样，代�审查人员也�想以牺牲你雇主的利益为代价�宣传他们雇主的议程。
+   内核开�人员通常希望今�几年能在内核上工作，但他们明白他们的雇主�能会改
+   �。他们真的，几乎毫无例外地，致力于创造他们所能�到的最好的内核；他们并
+   没有试图给雇主的竞争对手造��适。
+
+所有这些归根结底都是，当审阅者�您��评论时，您需�注�他们正在进行的技术
+观察。��让他们的表达方�或你自己的骄傲阻止这�事情的�生。当你在一个补�
+上得到评论时，花点时间去�解评论人想说什么。如果�能的�，请修�审阅者�求
+您修�的内容。然�回�审稿人：谢谢他们，并�述你将如何回答他们的问题。
+
+请注�，您�必��审阅者建议的�个更改。如果您认为审阅者误解了您的代�，请
+解释到底�生了什么。如果您对建议的更改有技术上的异议，请�述它并�明您对该
+问题的解决方案是正确的。如果你的解释有��，审稿人会接�的。�过，如果你的
+解释�能�明是有说�力的，尤其是当其他人开始��审稿人的观点时，请花些时间
+�新考虑一下。你很容易对自己解决问题的方法视而��，以至于你没有�识到�个
+问题根本是错误的，或者你甚至没有解决正确的问题。
+
+Andrew Morton建议，�一��会导致代�更改的评论都应该导致�外的代�注释；
+这�以帮助未�的评论人员��出现第一次出现的问题。
+
+一个致命的错误是忽视评论，希望它们会消失。他们�会走的。如果您在没有对之�
+收到的注释�出�应的情况下�新�布代�，那么很�能会�现补�毫无用处。
+
+说到�新�布代�：请记�，审阅者�会记�您上次�布的代�的所有细节。因此，
+�醒审查人员以��出的问题以�您如何处�这些问题总是一个好主�；补��更
+日志是�供此类信�的好地方。审阅者�必�索列表档案�熟悉上次所说的内容；
+如果您帮助他们开始�行，当他们�新访问您的代�时，他们的心情会更好。
+
+如果你已�试��正确的事情，但事情�然没有进展呢？大多数技术上的分歧都�以
+通过讨论�解决，但有时人们�需��出决定。如果你真的认为这个决定对你�利，
+你�以试��更高的�力上诉。在这篇文章中，更高的�力倾�于Andrew Morton。
+Andrew在内核开�社区中�i很大的尊�；他�常为似乎被�望地阻塞事情清障。
+尽管如此，对Andrew的呼��应轻而易举，也�应在所有其他替代方案都被探索之�
+使用。当然，记�，他也�能���你的��。
+
+接下�会�生什么
+----------------
+
+如果一个补�被认为是添加到内核中的一件好事，并且一旦大多数审查问题得到解决，
+下一步通常是进入�系统维护人员的树中。工作方�因�系统而异；�个维护人员都
+有自己的工作方�。特别是，�能有�止一棵树——一棵树，也许，专门用于计划下一
+个�并窗�的补�，�一棵树用于长期工作。
+
+对于应用于没有明显�系统树（例如内存管�修补程�）的区域的修补程�，默认树
+通常以-mm结尾。影�多个�系统的补�也�以最终通过-mm树。
+
+包�在�系统树中�以�高补�的��性。现在，使用该树的其他开�人员将默认获
+得补�。�系统树通常也为Linux�供支�，使其内容对整个开�社区��。在这一点
+上，您很�能会从一组新的审阅者那里得到更多的评论；这些评论需��上一轮那样
+得到回答。
+
+在这一点上也会�生什么，这�决于你的补�的性质，是与其他人正在�的工作�生
+冲�。在最�的情况下，严�的补�冲��能会导致一些工作被�置，以便剩余的补�
+�以�形并�并。�一些时候，冲�解决将涉�到与其他开�人员�作，�能还会
+在树之间移动一些补�，以确�所有的应用都是干净的。这项工作�能是一件痛苦的
+事情，但�计算您的�祉：在Linux下一棵树出现之�，这些冲�通常�在�并窗�
+中出现，必须迅速解决。现在�以在�并窗�打开之�，在空闲时解决这些问题。
+
+有�一日，如果一切顺利，您将登录并看到您的补�已��并到主线内核中。�贺你�
+然而，一旦庆�活动完�（并且您已�将自己添加到维护人员文件中），就值得记�
+一个��的�事实：工作�然没有完�。并入主线带�了自身的挑战。
+
+首先，补�的��性�次�高。�能会有新一轮的开�者评论，他们以��知�这
+个补�。忽略它们�能很有诱惑力，因为您的代���存在任何被�并的问题。但是，
+�抵制这�诱惑，您�然需�对有问题或建议的开�人员作出�应。
+
+�过，更��的是：将代�包�在主线中会将代�交给更大的一组测试人员。�使您
+为尚未�供的硬件�供了驱动程�，您也会惊讶于有多少人会将您的代�构建到内核
+中。当然，如果有测试人员，也会有错误报告。
+
+最糟糕的错误报告是回归。如果你的补�导致回归，你会�现很多�舒�的眼�盯�
+你；回归需�尽快修�。如果您�愿�或无法修�回归（其他人都�会为您修�），
+那么在稳定期内，您的补�几乎肯定会被移除。除了�定您为使补�进入主线所�的
+所有工作之外，如果由于未能修�回归而�消补�，很�能会使将�的工作更难�并。
+
+在处�完任何回归之�，�能还有其他普通的bug需�处�。稳定期是修�这些错误并
+确�代�在主线内核版本中的首次�布尽�能��的最好机会。所以，请回答错误
+报告，并尽�能解决问题。这就是稳定期的目的；一旦解决了旧补�的任何问题，就
+�以开始创建酷的新补�。
+
+别忘了，还有其他里程碑也�能会创建bug报告：下一个主线稳定版本，当著�的�行
+商选择包�补�的内核版本时，等等。继续�应这些报告是您工作的基本骄傲。但是，
+如果这�是足够的动机，那么也值得考虑的是，开�社区会记�那些在�并�对代�
+失去兴趣的开�人员。下一次你�布补�时，他们会以你以��会在身边维护它为�
+设�评估它。
+
+其他�能�生的事情
+------------------
+
+有一天，你�以打开你的邮件客户端，看到有人给你寄了一个代�补�。毕竟，这是
+让您的代�公开存在的好处之一。如果您��这个补�，您�以将它转�给�系统
+维护人员（确�包�一个正确的From:行，这样属性是正确的，并添加一个您自己
+的签准），或者回�一个Acked-by，让原始��者�上��它。
+
+如果您���补�，请��一个礼貌的回�，解释原因。如果�能的�，告诉作者需�
+�哪些更改�能让您接�补�。对于代�的编写者和维护者所�对的�并补�，存在�
+一定的阻力，但仅此而已。如果你被认为�必�的阻�了好的工作，那么这些补�最
+终会�过你身边并进入主线。在Linux内核中，没有人对任何代�拥有�对的�决�。
+除了Linus。
+
+在�常罕�的情况下，您�能会看到完全��的东西：�一个开�人员�布了针对您
+的问题的��解决方案。在这一点上，两个补�中的一个�能�会�并，“我的在这里
+是第一个��被认为是一个令人信�的技术论�。如果有人的补��代了你的补�而进
+入了主线，那么�有一�方法�以回应你：高兴你的问题得到解决，继续你的工作。
+以这�方�把一个人的工作推到一边�能会伤害和气�，但是在他们忘记了�的补�
+真正被�并很久之�，社区会记�你的�应。
diff --git a/Documentation/translations/zh_CN/process/7.AdvancedTopics.rst b/Documentation/translations/zh_CN/process/7.AdvancedTopics.rst
new file mode 100644
index 000000000000..956815edbd18
--- /dev/null
+++ b/Documentation/translations/zh_CN/process/7.AdvancedTopics.rst
@@ -0,0 +1,124 @@
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: :ref:`Documentation/process/7.AdvancedTopics.rst <development_advancedtopics>`
+:Translator: Alex Shi <alex.shi@linux.alibaba.com>
+
+.. _cn_development_advancedtopics:
+
+高级主题
+========
+
+现在，希望您能够掌�开��程的工作方�。然而，还有更多的东西�学�本节将介�
+一些主题，这些主题对希望�为Linux内核开�过程常规部分的开�人员有帮助。
+
+使用Git管�补�
+---------------
+
+内核使用分布�版本控制始于2002年�，当时Linus首次开始使用专有的Bitkeeper应用
+程�。虽然bitkeeper存在争议，但它所体现的软件版本管�方法�肯定�是。分布�
+版本控制�以立�加速内核开�项目。在当�的时代，有几��费的比特��器替代�。
+无论好�，内核项目都将Git作为其选择的工具。
+
+使用Git管�补��以使开�人员的生活更加轻�，尤其是��补�数�的增加。Git
+也有其粗糙的边缘和一定的�险，它是一个年轻和强大的工具，�然在其开�人员完善
+中。本文档�会试图教会读者如何使用git；这会是个巨长的文档。相�，这里的�点
+将是Git如何特别适�内核开�过程。想�加快Git的开�人员�以在以下网站上找到
+更多信�：
+
+	http://git-scm.com/
+
+	http://www.kernel.org/pub/software/scm/git/docs/user-manual.html
+
+在�试使用它使补��供其他人使用之�，第一�务是阅读上述站点，对Git的工作
+方�有一个扎实的了解。使用Git的开�人员应该能够获得主线存储库的副本，探索
+修订历�，�交对树的更改，使用分支等。了解Git用于�写历�的工具（如Rebase）
+也很有用。Git有自己的术语和概念；Git的新用户应该了解refs�远程分支�索引�
+快进�并�推拉�分离头等。一开始�能有点�人，但这些概念�难通过一点学习�
+�解。
+
+使用git生�通过电�邮件�交的补�是�高速度的一个很好的练习。
+
+当您准备好开始安装Git树供其他人查看时，您当然需�一个�以从中��的�务器。
+如果您有一个�以访问Internet的系统，那么使用git守护进程设置这样的�务器相
+对简�。�则，�费的公共托管网站（例如github）开始出现在网络上。�熟的开�
+人员�以在kernel.org上获得一个�户，但这些�户并�容易找到；有关更多信�，
+请�阅 http://kernel.org/faq/
+
+正常的Git工作�程涉�到许多分支的使用。�一�开�线都�以分为�独的“主题
+分支�，并独立维护。Git的分支机构很便宜，没有�由��费使用它们。而且，在
+任何情况下，您都�应该在任何您打算让其他人从中�益的分支中进行开�。应该
+�心地创建公开�用的分支；当它们处于完整的形�并准备好�行时(而�是之�），
+�并开�分支的补�。
+
+Git�供了一些强大的工具，�以让您�写开�历�。一个�方便的补�（比如说，
+一个打破二分法的补�，或者有其他一些明显的缺陷）�以在适当的�置修�，或者
+完全从历�中消失。一个补�系列�以被�写，就好�它是在今天的主线之上写的
+一样，�使你已�花了几个月的时间在写它。�以�明地将更改从一个分支转移到�
+一个分支。等等。明智地使用git修改历�的能力�以帮助创建问题更少的干净补�集。
+
+然而，过度使用这�能力�能会导致其他问题，而�仅仅是对创建完美项目历�的
+简�痴迷。�写历�将�写该历�中包�的更改，将�过测试（希望）的内核树�
+为未�测试的内核树。但是，除此之外，如果开�人员没有对项目历�的共享视图，
+他们就无法轻�地�作；如果您�写了其他开�人员拉入他们存储库的历�，您将
+使这些开�人员的生活更加困难。因此，这里有一个简�的�验法则：被导出到其他
+人的历�在此�通常被认为是���的。
+
+因此，一旦将一组更改推�到公开�用的�务器上，就�应该�写这些更改。如果您
+�试强制进行�会导致快进�并（��共享�一历�记录的更改）的更改，Git将�
+试强制执行此规则。�以�写此检查，有时�能需��写导出的树。在树之间移动�
+更集以��Linux-next中的冲�就是一个例�。但这�行为应该是罕�的。这就是为
+什么开�应该在�有分支中进行（必�时�以�写）并且�有在公共分支处于��的
+高级状�时�转移到公共分支中的原因之一。
+
+当主线（或其他一组�更所基于的树）�进时，很容易与该树�并以��领先地�。
+对于一个�有的分支，rebasing �能是一个很容易跟上�一棵树的方法，但是一旦
+一棵树被导出到全世界，rebasing就�是一个选项。一旦�生这�情况，就必须进行
+完全�并（merge）。�并有时是很有�义的，但是过于频�的�并会�必�地扰乱
+历�。在这�情况下，建议的技术是��常�并，通常�在特定的�布点（如主线-rc
+�布）�并。如果您对特定的更改感到紧张，则�以始终在�有分支中执行测试�并。
+在这�情况下，git rerere 工具很有用；它记��并冲�是如何解决的，这样您就
+�必��相�的工作。
+
+关于Git这样的工具的一个最大的��抱怨是：补�从一个存储库到�一个存储库的
+大�移动使得很容易陷入错误建议的�更中，这些�更�开审查雷达进入主线。当内
+核开�人员看到这�情况�生时，他们往往会感到�高兴；在Git树上放置未查看或
+主题外的补��能会影�您将�获�树的能力。引用Linus:
+
+::
+
+        你�以给我�补�，但�我从你哪里�一个Git补�，我需�知�你知�
+        你在�什么，我需�能够相信事情而�去检查�个个人改�。
+
+（http://lwn.net/articles/224135/）。
+
+为了��这�情况，请确�给定分支中的所有补�都与相关主题紧密相关；“驱动程�
+修��分支�应更改核心内存管�代�。而且，最��的是，��使用Git树�绕过
+审查过程。�时的将树的摘��布到相关的列表中，当时间�适时，请求
+Linux-next 中包�该树。
+
+如果其他人开始��补�以包�到您的树中，��忘记查看它们。还�确�您维护正确
+的作者信�； ``git am`` 工具在这方��得最好，但是如果它通过第三方转�给您，
+您�能需�在补�中添加“From：�行。
+
+请求pull�作时，请务必�供所有相关信�：树的�置��拉的分支以�拉�作将导致
+的更改。在这方�，git request pull 命令�常有用；它将按照其他开�人员的预期
+格�化请求，并检查以确�您记�了将这些更改推�到公共�务器。
+
+审查补�
+--------
+
+一些读者当然会�对将本节与“高级主题�放在一起，因为�使是刚开始的内核开�人员
+也应该检查补�。当然，学习如何在内核环境中编程没有比查看其他人�布的代�更好
+的方法了。此外，审阅者永远供�应求；通过查看代�，您�以对整个�程�出�大贡献。
+
+审查代��能是一个令人生�的�景，特别是对于一个新的内核开�人员�说，他们
+�能会对公开询问代�感到紧张，而这些代�是由那些有更多�验的人�布的。�过，
+�使是最有�验的开�人员编写的代�也�以得到改进。也许对评审员（所有评审员）
+最好的建议是：把评审评论当�问题而�是批评。询问“在这�路径中如何释放�？�
+总是比说“这里的�是错误的�更好。
+
+��的开�人员将从��的角度审查代�。一些主�关注的是编�样�以�代�行是
+�有尾�空格。其他人将主�关注补�作为一个整体实现的�更是�对内核有好处。
+然而，其他人会检查是�存在�定问题�堆栈使用过度��能的安全问题�在其他
+地方�现的代����足够的文档�对性能的�利影��用户空间ABI更改等。所有
+类型的检查，如果它们导致更好的代�进入内核，都是�欢迎和值得的。
diff --git a/Documentation/translations/zh_CN/process/8.Conclusion.rst b/Documentation/translations/zh_CN/process/8.Conclusion.rst
new file mode 100644
index 000000000000..2bbd76161e10
--- /dev/null
+++ b/Documentation/translations/zh_CN/process/8.Conclusion.rst
@@ -0,0 +1,64 @@
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: :ref:`Documentation/process/8.Conclusion.rst <development_conclusion>`
+:Translator: Alex Shi <alex.shi@linux.alibaba.com>
+
+.. _cn_development_conclusion:
+
+更多信�
+========
+
+关于Linux内核开�和相关主题的信���很多。首先是在内核�代�分�中找到的
+文档目录。顶级 :ref:`Documentation/translations/zh_CN/process/howto.rst <cn_process_howto>`
+文件是一个��的起点
+:ref:`Documentation/translations/zh_CN/process/submitting-patches.rst <cn_submittingpatches>`
+和 :ref:`process/submitting-drivers.rst <submittingdrivers>`
+也是所有内核开�人员都应该阅读的内容。许多内部内核API都是使用kerneldoc机制
+记录的；“make htmldocs�或“make pdfdocs��用于以HTML或PDF格�生�这些文档（
+尽管�些�行版�供的tex版本会�到内部�制，无法正确处�文档）。
+
+��的网站在�个细节层次上讨论内核开�。您的作者想谦虚地建议用 http://lwn.net/
+作为��；有关许多特定内核主题的信��以通过以下网�的lwn内核索引找到：
+
+        http://lwn.net/kernel/index/
+
+除此之外，内核开�人员的一个�贵资�是：
+
+        http://kernelnewbies.org/
+
+当然，我们�应该忘记 http://kernel.org/ 这是内核�布信�的最终�置。
+
+关于内核开�有很多书：
+
+        Linux设备驱动程�，第三版（Jonathan Corbet�Alessandro Rubini和Greg Kroah Hartman）。
+        在线：http://lwn.net/kernel/ldd3/
+
+        Linux内核开�（Robert Love）。
+
+        了解Linux内核（Daniel Bovet和Marco Cesati）。
+
+然而，所有这些书都有一个共�的缺点：当它们上架时，它们往往有些过时，而且它们
+已�上架一段时间了。�过，在那里还�以找到相当多的好信�。
+
+有关git的文档，请访问：
+
+        http://www.kernel.org/pub/software/scm/git/docs/
+
+        http://www.kernel.org/pub/software/scm/git/docs/user-manual.html
+
+结论
+====
+
+�贺所有通过这篇冗长的文件的人。希望它能够帮助您�解Linux内核是如何开�的，
+以�您如何�与这个过程。
+
+最�，��的是�与。任何开�软件项目都�超过其贡献者投入其中的总和。Linux内核
+的�展速度和以�一样快，因为它得到了大�开�人员的帮助，他们都在努力使它�得
+更好。内核是一个主�的例�，说明当��上万的人为了一个共�的目标一起工作时，
+�以�些什么。
+
+�过，内核总是�以从更大的开�人员基础中获益。总有更多的工作��。但是，�样
+��的是，Linux生�系统中的大多数其他�与者�以通过为内核�出贡献而�益。使
+代�进入主线是�高代�质���低维护和分��本��高对内核开�方�的影�程度
+等的关键。这是一�人人都赢的局�。踢开你的编辑，�加入我们�，你会�常�
+欢迎的。
diff --git a/Documentation/translations/zh_CN/process/code-of-conduct-interpretation.rst b/Documentation/translations/zh_CN/process/code-of-conduct-interpretation.rst
new file mode 100644
index 000000000000..c323ce76e0cb
--- /dev/null
+++ b/Documentation/translations/zh_CN/process/code-of-conduct-interpretation.rst
@@ -0,0 +1,108 @@
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: :ref:`Documentation/process/code-of-conduct-interpretation.rst <code_of_conduct_interpretation>`
+:Translator: Alex Shi <alex.shi@linux.alibaba.com>
+
+.. _cn_code_of_conduct_interpretation:
+
+Linux内核贡献者契约行为准则解释
+===============================
+
+:ref:`cn_code_of_conduct` 准则是一个通用文档，旨在为几乎所有开�社区�供一套规则。
+�个开�社区都是独一无二的，Linux内核也�例外。因此，本文�述了Linux内核社区中
+如何解释它。我们也�希望这�解释��时间的推移是��的，并将根�需�进行调整。
+
+与开�软件的“传统�方法相比，Linux内核开�工作是一个�常个人化的过程。你的贡献
+和背�的想法将被仔细审查，往往导致批判和批评。审查将几乎总是需�改进，�料�
+能包括在内核中。�知�这是因为所有相关人员都希望看到Linux整体�功的最佳解决方
+案。这个开�过程已�被�明�以创建有�以�最�壮的�作系统内核，我们�想�任何
+事情�导致�交质�和最终结果的下�。
+
+维护者
+------
+
+行为准则多次使用“维护者�一�。在内核社区中，“维护者�是负责�系统�驱动程�或
+文件的任何人，并在内核�代�树的维护者文件中列出。
+
+责任
+----
+
+《行为准则》�到了维护人员的�利和责任，这需�进一步澄清。
+
+首先，最��的是，有一个��的期望是由维护人员通过实例�领导。
+
+也就是说，我们的社区是广阔的，对维护者没有新的�求，他们�方�处�其他人在
+他们活跃的社区的行为。这一责任由我们所有人承担，最终《行为准则》记录了最终的
+上诉路径，以防有关行为问题的问题悬而未决。
+
+维护人员应该愿�在出现问题时�供帮助，并在需�时与社区中的其他人�作。如果您
+�确定如何处�出现的情况，请��害怕�系技术咨询委员会（TAB）或其他维护人员。
+除�您愿�，�则�会将其视为�规报告。如果您�确定是�该�系TAB 或任何其他维
+护人员，请�系我们的冲�调解人 Mishi Choudhary <mishi@linux.com>。
+
+最�，“善待对方��是�个人的最终目标。我们知��个人都是人，有时我们都会失败，
+但我们所有人的首�目标应该是努力�好地解决问题。执行行为准则将是最�的选择。
+
+我们的目标是创建一个强大的�技术先进的�作系统，以�所涉�的技术��性，这自
+然需�专业知识和决策。
+
+所需的专业知识因贡献领域而异。它主�由上下文和技术��性决定，其次由贡献者和
+维护者的期望决定。
+
+专家的期望和决策都��过讨论，但在最�，为了�得进展，必须能够�出决策。这一
+特�掌�在维护人员和项目领导的手中，预计将善�使用。
+
+因此，设定专业知识期望�作出决定和拒��适当的贡献�被视为��行为准则。
+
+虽然维护人员一般都欢迎新�者，但他们帮助（新）贡献者克�障�的能力有�，因此
+他们必须确定优先事项。这也�应被视为��了行为准则。内核社区�识到这一点，并
+以��形��供入门级节目，如 kernelnewbies.org 。
+
+范围
+----
+
+Linux内核社区主�在一组公共电�邮件列表上进行交互，这些列表分布在由多个��
+公�或个人控制的多个���务器上。所有这些列表都在内核�代�树中的
+MAINTAINERS 文件中定义。��到这些邮件列表的任何电�邮件都被视为包�在行为
+准则中。
+
+使用 kernel.org bugzilla和其他�系统bugzilla 或bug跟踪工具的开�人员应该�循
+行为准则的指导原则。Linux内核社区没有“官方�项目电�邮件地�或“官方�社交媒体
+地�。使用kernel.org电�邮件�户执行的任何活动必须�循为kernel.org�布的行为
+准则，就�任何使用公�电�邮件�户的个人必须�循该公�的特定规则一样。
+
+行为准则并��止在邮件列表消��内核更改日志消�或代�注释中继续包��称�
+电�邮件地�和相关注释。
+
+其他论�中的互动包括在适用于上述论�的任何规则中，通常�包括在行为准则中。
+除了在�端情况下�考虑的例外情况。
+
+�交给内核的贡献应该使用适当的语言。在行为准则之�已�存在的内容现在�会被
+视为��。然而，�适当的语言�以被视为一个bug；如果任何相关方�交补�，
+这样的bug将被更快地修�。当�属于用户/内核API的一部分的表达�，或者�映已
+�布标准或规范中使用的术语的表达�，�被视为bug。
+
+执行
+----
+
+行为准则中列出的地�属于行为准则委员会。https://kernel.org/code-of-conduct.html
+列出了在任何给定时间接收这些电�邮件的确切�员。�员�能访问在加入委员会之�
+或离开委员会之�所�的报告。
+
+最�的行为准则委员会由TAB的志愿者以�作为中立第三方的专业调解人组�。委员会
+的首�任务是建立文件化的�程，并将其公开。
+
+如果报告人�希望将整个委员会纳入投诉或关切，�直接�系委员会的任何�员，包括
+调解人。
+
+行为准则委员会根��程审查案例（�上文），并根�需�和适当与TAB�商，例如请求
+和接收有关内核社区的信�。
+
+委员会�出的任何决定都将�交到表中，以便在必�时与相关维护人员一起执行。行为
+准则委员会的决定�以通过三分之二的投票推翻。
+
+�季度，行为准则委员会和标签将�供一份报告，概述行为准则委员会收到的匿�报告
+�其状�，以�任何�决决定的细节，包括完整和�识别的投票细节。
+
+我们希望在�动期之�为行为准则委员会人员�备建立一个��的�程。�生此情况时，
+将使用该信�更新此文档。
diff --git a/Documentation/translations/zh_CN/process/code-of-conduct.rst b/Documentation/translations/zh_CN/process/code-of-conduct.rst
new file mode 100644
index 000000000000..99024df058e9
--- /dev/null
+++ b/Documentation/translations/zh_CN/process/code-of-conduct.rst
@@ -0,0 +1,72 @@
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: :ref:`Documentation/process/code-of-conduct.rst <code_of_conduct>`
+:Translator: Alex Shi <alex.shi@linux.alibaba.com>
+
+.. _cn_code_of_conduct:
+
+贡献者契约行为准则
+++++++++++++++++++
+
+我们的誓言
+==========
+
+为了�造一个开放��好的环境，我们作为贡献者和维护人承诺，让我们的社区和�
+与者，拥有一个无骚扰的体验，无论年龄�体型�残疾����性别特��性别认�
+和表达��验水平�教育程度�社会状况，�济地��国��个人外貌����宗教
+或性身份和��。
+
+我们的标准
+==========
+
+有助于创造积�环境的行为包括：
+
+* 使用欢迎和包容的语言
+* 尊���的观点和�验
+* 优雅地接�建设性的批评
+* 关注什么对社区最有利
+* 对其他社区�员表示�情
+
+�与者的��接�行为包括：
+
+* 使用性�味的语言或�象以���欢迎的性注�或者更过分的行为
+* 煽动�侮辱/贬�评论以�个人或政治攻击
+* 公开或�下骚扰
+* 未�明确许�，�布他人的�人信�，如物�或电�地�。
+* 在专业场�被��认为�适当的其他行为
+
+我们的责任
+==========
+
+维护人员负责澄清�接�行为的标准，并应针对任何��接�行为采�适当和公平的
+纠正措施。
+
+维护人员有�和责任删除�编辑或拒�与本行为准则�一致的评论�承诺�代��
+wiki编辑�问题和其他贡献，或暂时或永久�止任何贡献者从事他们认为�适当�
+���冒犯或有害的其他行为。
+
+范围
+====
+
+当个人代表项目或其社区时，本行为准则既适用于项目空间，也适用于公共空间。
+代表一个项目或社区的例�包括使用一个正�的项目电�邮件地�，通过一个正�
+的社交媒体�户�布，或者在在线或离线事件中担任指定的代表。项目维护人员�以
+进一步定义和澄清项目的表示。
+
+执行
+====
+
+如有滥用�骚扰或其他��接�的行为，��系行为准则委员会<conduct@kernel.org>。
+所有投诉都将接�审查和调查，并将得到必�和适当的答�。行为准则委员会有义务
+对事件报告人�密。具体执行政策的进一步细节��独公布。
+
+归属
+====
+
+本行为准则改编自《贡献者契约》，版本1.4，�从
+https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 获�。
+
+解释
+====
+
+有关Linux内核社区如何解释此文档，请�阅 :ref:`cn_code_of_conduct_interpretation`
diff --git a/Documentation/translations/zh_CN/coding-style.rst b/Documentation/translations/zh_CN/process/coding-style.rst
index 3cb09803e084..5479c591c2f7 100644
--- a/Documentation/translations/zh_CN/coding-style.rst
+++ b/Documentation/translations/zh_CN/process/coding-style.rst
@@ -1,19 +1,10 @@
-Chinese translated version of Documentation/process/coding-style.rst
+.. include:: ../disclaimer-zh_CN.rst
 
-If you have any comment or update to the content, please post to LKML directly.
-However, if you have problem communicating in English you can also ask the
-Chinese maintainer for help.  Contact the Chinese maintainer, if this
-translation is outdated or there is problem with translation.
+:Original: :ref:`Documentation/process/coding-style.rst <codingstyle>`
 
-Chinese maintainer: Zhang Le <r0bertz@gentoo.org>
+.. _cn_codingstyle:
 
----------------------------------------------------------------------
-
-Documentation/process/coding-style.rst 的中文翻译
-
-如果想评论或更新本文的内容，请直接�信到LKML。如果你使用英文交�有困难的�，
-也�以�中文版维护者求助。如果本翻译更新��时或者翻译存在问题，请�系中文版
-维护者::
+译者::
 
   中文版维护者： 张� Zhang Le <r0bertz@gentoo.org>
   中文版翻译者： 张� Zhang Le <r0bertz@gentoo.org>
@@ -23,10 +14,6 @@ Documentation/process/coding-style.rst 的中文翻译
                  Li Zefan <lizf@cn.fujitsu.com>
                  Wang Chen <wangchen@cn.fujitsu.com>
 
-以下为正文
-
----------------------------------------------------------------------
-
 Linux 内核代�风格
 =========================
 
diff --git a/Documentation/translations/zh_CN/process/development-process.rst b/Documentation/translations/zh_CN/process/development-process.rst
new file mode 100644
index 000000000000..30cffe66c075
--- /dev/null
+++ b/Documentation/translations/zh_CN/process/development-process.rst
@@ -0,0 +1,26 @@
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: :ref:`Documentation/process/development-process.rst <development_process_main>`
+:Translator: Alex Shi <alex.shi@linux.alibaba.com>
+
+.. _cn_development_process_main:
+
+内核开�过程指�
+================
+
+内容:
+
+.. toctree::
+   :numbered:
+   :maxdepth: 2
+
+   1.Intro
+   2.Process
+   3.Early-stage
+   4.Coding
+   5.Posting
+   6.Followthrough
+   7.AdvancedTopics
+   8.Conclusion
+
+本文档的目的是帮助开�人员（�其��）以最�的挫折感与开�社区�作。它试图记录这个社区如何以一��熟悉Linux内核开�（或者实际上是自由软件开�）的人�以访问的方�工作。虽然这里有一些技术资料，但这是一个��过程的讨论，�需�深入了解内核编程就�以�解。
diff --git a/Documentation/translations/zh_CN/email-clients.txt b/Documentation/translations/zh_CN/process/email-clients.rst
index ec31d97e8d0e..102023651118 100644
--- a/Documentation/translations/zh_CN/email-clients.txt
+++ b/Documentation/translations/zh_CN/process/email-clients.rst
@@ -1,33 +1,34 @@
-Chinese translated version of Documentation/process/email-clients.rst
+.. _cn_email_clients:
 
-If you have any comment or update to the content, please contact the
-original document maintainer directly.  However, if you have a problem
-communicating in English you can also ask the Chinese maintainer for
-help.  Contact the Chinese maintainer if this translation is outdated
-or if there is a problem with the translation.
+.. include:: ../disclaimer-zh_CN.rst
 
-Chinese maintainer: Harry Wei <harryxiyou@gmail.com>
----------------------------------------------------------------------
-Documentation/process/email-clients.rst 的中文翻译
+:Original: :ref:`Documentation/process/email-clients.rst <email_clients>`
 
-如果想评论或更新本文的内容，请直接�系原文档的维护者。如果你使用英文
-交�有困难的�，也�以�中文版维护者求助。如果本翻译更新��时或者翻
-译存在问题，请�系中文版维护者。
+译者::
 
-中文版维护者： 贾��  Harry Wei <harryxiyou@gmail.com>
-中文版翻译者： 贾��  Harry Wei <harryxiyou@gmail.com>
-中文版校译者： Yinglin Luan <synmyth@gmail.com>
-		Xiaochen Wang <wangxiaochen0@gmail.com>
-		yaxinsn <yaxinsn@163.com>
-
-以下为正文
----------------------------------------------------------------------
+        中文版维护者： 贾��  Harry Wei <harryxiyou@gmail.com>
+        中文版翻译者： 贾��  Harry Wei <harryxiyou@gmail.com>
+                       时奎亮  Alex Shi <alex.shi@linux.alibaba.com>
+        中文版校译者： Yinglin Luan <synmyth@gmail.com>
+        	       Xiaochen Wang <wangxiaochen0@gmail.com>
+                       yaxinsn <yaxinsn@163.com>
 
 Linux邮件客户端�置信�
-======================================================================
+=======================
+
+Git
+---
+
+现在大多数开�人员使用 ``git send-email`` 而�是常规的电�邮件客户端。这方�
+的手册�常好。在接收端，维护人员使用 ``git am`` 加载补�。
+
+如果你是 ``git`` 新手，那么把你的第一个补���给你自己。将其�存为包�所有
+标题的原始文本。�行 ``git am raw_email.txt`` ，然�使用 ``git log`` 查看更
+改日志。如果工作正常，�将补���到相应的邮件列表。
+
 
 普通�置
-----------------------------------------------------------------------
+--------
 Linux内核补�是通过邮件被�交的，最好把补�作为邮件体的内嵌文本。有些维护者
 接收附件，但是附件的内容格�应该是"text/plain"。然而，附件一般是�赞�的，
 因为这会使补�的引用部分在评论过程中�的很困难。
@@ -56,7 +57,7 @@ Linux内核补�是通过邮件被�交的，最好把补�作为邮件体的
 
 
 一些邮件客户端�示
-----------------------------------------------------------------------
+------------------
 这里给出一些详细的MUA�置�示，�以用于给Linux内核��补�。这些并��味是
 所有的软件包�置总结。
 
@@ -64,8 +65,8 @@ Linux内核补�是通过邮件被�交的，最好把补�作为邮件体的
 TUI = 以文本为基础的用户接�
 GUI = 图形界�用户接�
 
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Alpine (TUI)
+~~~~~~~~~~~~
 
 �置选项：
 在"Sending Preferences"部分：
@@ -76,8 +77,8 @@ Alpine (TUI)
 当写邮件时，光标应该放在补�会出现的地方，然�按下CTRL-R组�键，使指定的
 补�文件嵌入到邮件中。
 
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Evolution (GUI)
+~~~~~~~~~~~~~~~
 
 一些开�者�功的使用它��补�
 
@@ -89,8 +90,8 @@ Evolution (GUI)
 
 你还�以"diff -Nru old.c new.c | xclip"，选择Preformat，然�使用中间键进行粘帖。
 
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Kmail (GUI)
+~~~~~~~~~~~
 
 一些开�者�功的使用它��补�。
 
@@ -118,13 +119,13 @@ display"，这样内嵌附件更容易让读者看到。
 并且希望这将会被处�。邮件是以�针对�个用户�读写的��被�存的，所以如果你想把邮件�制到其他地方，
 你�得�把他们的��改为组或者整体�读。
 
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Lotus Notes (GUI)
+~~~~~~~~~~~~~~~~~
 
 ��使用它。
 
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Mutt (TUI)
+~~~~~~~~~~
 
 很多Linux开�人员使用mutt客户端，所以�明它肯定工作的�常漂亮。
 
@@ -142,12 +143,49 @@ Muttä¸�自带编辑器，所以ä¸�管你使用什么编辑器都ä¸�应该带有è
 如果想�把补�作为内嵌文本。
 (a)ttach工作的很好，�带有"set paste"。
 
+你�以通过 ``git format-patch`` 生�补�，然�用 Mutt��它们::
+
+        $ mutt -H 0001-some-bug-fix.patch
+
 �置选项：
 它应该以默认设置的形�工作。
 然而，把"send_charset"设置为"us-ascii::utf-8"也是一个�错的主�。
 
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Mutt 是高度��置的。 这里是个使用mutt通过 Gmail ��的补�的最��置::
+
+  # .muttrc
+  # ================  IMAP ====================
+  set imap_user = 'yourusername@gmail.com'
+  set imap_pass = 'yourpassword'
+  set spoolfile = imaps://imap.gmail.com/INBOX
+  set folder = imaps://imap.gmail.com/
+  set record="imaps://imap.gmail.com/[Gmail]/Sent Mail"
+  set postponed="imaps://imap.gmail.com/[Gmail]/Drafts"
+  set mbox="imaps://imap.gmail.com/[Gmail]/All Mail"
+
+  # ================  SMTP  ====================
+  set smtp_url = "smtp://username@smtp.gmail.com:587/"
+  set smtp_pass = $imap_pass
+  set ssl_force_tls = yes # Require encrypted connection
+
+  # ================  Composition  ====================
+  set editor = `echo \$EDITOR`
+  set edit_headers = yes  # See the headers when editing
+  set charset = UTF-8     # value of $LANG; also fallback for send_charset
+  # Sender, email address, and sign-off line must match
+  unset use_domain        # because joe@localhost is just embarrassing
+  set realname = "YOUR NAME"
+  set from = "username@gmail.com"
+  set use_from = yes
+
+Mutt文档�有更多信�:
+
+    http://dev.mutt.org/trac/wiki/UseCases/Gmail
+
+    http://dev.mutt.org/doc/manual.html
+
 Pine (TUI)
+~~~~~~~~~~
 
 Pine过去有一些空格删�问题，但是这些现在应该都被修�了。
 
@@ -158,8 +196,8 @@ Pine过去有一些空格删å‡�问题，但是这些现在应该都被修å¤�了ã
 - "no-strip-whitespace-before-send"选项也是需�的。
 
 
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Sylpheed (GUI)
+~~~~~~~~~~~~~~
 
 - 内嵌文本�以很好的工作（或者使用附件）。
 - �许使用外部的编辑器。
@@ -168,8 +206,8 @@ Sylpheed (GUI)
 - 在组�窗�中有一个很有用的ruler bar。
 - 给地�本中添加地�就�会正确的了解显示�。
 
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Thunderbird (GUI)
+~~~~~~~~~~~~~~~~~
 
 默认情况下，thunderbird很容易��文本，但是还有一些方法�以强制它�得更好。
 
@@ -191,13 +229,13 @@ Thunderbird (GUI)
   $EDITOR�读�或者�并补�到文本中。�实现它，�以下载并且安装这个扩展，然�添加一个使用它的
   按键View->Toolbars->Customize...最�当你书写信�的时候仅仅点击它就�以了。
 
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 TkRat (GUI)
+~~~~~~~~~~~
 
 �以使用它。使用"Insert file..."或者外部的编辑器。
 
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Gmail (Web GUI)
+~~~~~~~~~~~~~~~
 
 ��使用它��补�。
 
diff --git a/Documentation/translations/zh_CN/HOWTO b/Documentation/translations/zh_CN/process/howto.rst
index 7a00a8a4eb15..5b671178b17b 100644
--- a/Documentation/translations/zh_CN/HOWTO
+++ b/Documentation/translations/zh_CN/process/howto.rst
@@ -1,32 +1,22 @@
-Chinese translated version of Documentation/process/howto.rst
+.. _cn_process_howto:
 
-If you have any comment or update to the content, please contact the
-original document maintainer directly.  However, if you have a problem
-communicating in English you can also ask the Chinese maintainer for
-help.  Contact the Chinese maintainer if this translation is outdated
-or if there is a problem with the translation.
+.. include:: ../disclaimer-zh_CN.rst
 
-Maintainer: Greg Kroah-Hartman <greg@kroah.com>
-Chinese maintainer: Li Yang <leoli@freescale.com>
----------------------------------------------------------------------
-Documentation/process/howto.rst 的中文翻译
+:Original: :ref:`Documentation/process/howto.rst <process_howto>`
 
-如果想评论或更新本文的内容，请直接�系原文档的维护者。如果你使用英文
-交�有困难的�，也�以�中文版维护者求助。如果本翻译更新��时或者翻
-译存在问题，请�系中文版维护者。
+译者::
 
-英文版维护者： Greg Kroah-Hartman <greg@kroah.com>
-中文版维护者： �阳  Li Yang <leoli@freescale.com>
-中文版翻译者： �阳  Li Yang <leoli@freescale.com>
-中文版校译者： 钟宇  TripleX Chung <xxx.phy@gmail.com>
-               陈�  Maggie Chen <chenqi@beyondsoft.com>
-               王�  Wang Cong <xiyou.wangcong@gmail.com>
-
-以下为正文
----------------------------------------------------------------------
+    英文版维护者： Greg Kroah-Hartman <greg@kroah.com>
+    中文版维护者： �阳  Li Yang <leoyang.li@nxp.com>
+    中文版翻译者： �阳  Li Yang <leoyang.li@nxp.com>
+                   时奎亮 Alex Shi <alex.shi@linux.alibaba.com>
+    中文版校译者:
+                   钟宇  TripleX Chung <xxx.phy@gmail.com>
+                   陈�  Maggie Chen <chenqi@beyondsoft.com>
+                   王�  Wang Cong <xiyou.wangcong@gmail.com>
 
 如何�与Linux内核开�
----------------------
+=====================
 
 这是一篇将如何�与Linux内核开�的相关问题一网打尽的终�秘笈。它将指导你
 �为一�Linux内核开�者，并且学会如何�Linux内核开�社区�作。它尽�能�
@@ -47,6 +37,7 @@ Linux内核大部分是由C语言写æˆ�的，一些体系结构相关的代ç �ç”
 �与内核开�，你必须精通C语言。除�你想为�个架构开�底层代�，�则你并
 �需�了解（任何体系结构的）汇编语言。下�列举的书�虽然�能替代扎实的C
 语言教育和多年的开��验，但如果需�的�，�为�考还是�错的：
+
  - "The C Programming Language" by Kernighan and Ritchie [Prentice Hall]
    《C程�设计语言（第2版·新版）》（��文 �志 译）[机械工业出版社]
  - "Practical C Programming" by Steve Oualline [O'Reilly]
@@ -71,9 +62,11 @@ Linux内核使用GNU C和GNU工具链开�。虽然它�循ISO C89标准，但
 --------
 
 Linux内核�代�都是在GPL（通用公共许��）的�护下�布的。�了解这�许�
-的细节请查看�代�主目录下的COPYING文件。如果你对它还有更深入问题请�系
-律师，而��在Linux内核邮件组上�问。因为邮件组里的人并�是律师，��期
-望他们的�有法律效力。
+的细节请查看�代�主目录下的COPYING文件。Linux内核许�准则和如何使用
+`SPDX <https://spdx.org/>` 标志符说明在这个文件中
+:ref:`Documentation/translations/zh_CN/process/license-rules.rst <cn_kernel_licensing>`
+如果你对它还有更深入问题请�系律师，而��在Linux内核邮件组上�问。因为
+邮件组里的人并�是律师，��期望他们的�有法律效力。
 
 对于GPL的常�问题和解答，请访问以下链接：
 	http://www.gnu.org/licenses/gpl-faq.html
@@ -89,65 +82,75 @@ Linux内核代�中包�有大�的文档。这些文档对于学习如何与
 的维护者解释这些�化。
 
 以下是内核代�中需�阅读的文档：
-  README
+  :ref:`Documentation/admin-guide/README.rst <readme>`
     文件简�介�了Linux内核的背景，并且�述了如何�置和编译内核。内核的
     新用户应该从这里开始。
 
-  Documentation/process/changes.rst
+
+  :ref:`Documentation/process/changes.rst <changes>`
     文件给出了用�编译和使用内核所需�的最�软件包列表。
 
-  Documentation/process/coding-style.rst
+  :ref:`Documentation/translations/zh_CN/process/coding-style.rst <cn_codingstyle>`
     �述Linux内核的代�风格和�由。所有新代�需��守这篇文档中定义的规
     范。大多数维护者�会接收符�规定的补�，很多人也�会帮忙检查符�风格
     的代�。
 
-  Documentation/process/submitting-patches.rst
-  Documentation/process/submitting-drivers.rst
+  :ref:`Documentation/translations/zh_CN/process/submitting-patches.rst <cn_submittingpatches>`
+  :ref:`Documentation/process/submitting-drivers.rst <submittingdrivers>`
+
     这两份文档明确�述如何创建和��补�，其中包括（但�仅�于)：
        - 邮件内容
        - 邮件格�
        - 选择收件人
+
     �守这些规定并�能���交�功（因为所有补�需�通过严格的内容和风格
     审查），但是忽视他们几乎就�味�失败。
 
     其他关于如何正确地生�补�的优秀文档包括：
     "The Perfect Patch"
+
         http://www.ozlabs.org/~akpm/stuff/tpp.txt
+
     "Linux kernel patch submission format"
+
         http://linux.yyz.us/patch-format.html
 
-  Documentation/process/stable-api-nonsense.rst
+  :ref:`Documentation/translations/zh_CN/process/stable-api-nonsense.rst <cn_stable_api_nonsense>`
     论�内核为什么特��包括稳定的内核内部API，也就是说�包括�这样的特
     性：
+
        - �系统中间层（为了兼容性？）
        - 在���作系统间易于移�的驱动程�
        - �缓（甚至阻止）内核代�的快速�化
+
     这篇文档对于�解Linux的开�哲学至关��。对于将开�平�从其他�作系
     统转移到Linux的人�说也很��。
 
-  Documentation/admin-guide/security-bugs.rst
+  :ref:`Documentation/admin-guide/security-bugs.rst <securitybugs>`
     如果你认为自己�现了Linux内核的安全性问题，请根�这篇文档中的步骤�
     �醒其他内核开�者并帮助解决这个问题。
 
-  Documentation/process/management-style.rst
+  :ref:`Documentation/translations/zh_CN/process/management-style.rst <cn_managementstyle>`
+
     �述内核维护者的工作方法�其共有特点。这对于刚刚接触内核开�（或者对
     它感到好奇）的人�说很��，因为它解释了很多对于内核维护者独特行为的
     普�误解与迷惑。
 
-  Documentation/process/stable-kernel-rules.rst
+  :ref:`Documentation/process/stable-kernel-rules.rst <stable_kernel_rules>`
     解释了稳定版内核�布的规则，以�如何将改动放入这些版本的步骤。
 
-  Documentation/process/kernel-docs.rst
+  :ref:`Documentation/process/kernel-docs.rst <kernel_docs>`
     有助于内核开�的外部文档列表。如果你在内核自带的文档中没有找到你想找
     的内容，�以查看这些文档。
 
-  Documentation/process/applying-patches.rst
+  :ref:`Documentation/process/applying-patches.rst <applying_patches>`
     关于补�是什么以�如何将它打在��内核开�分支上的好介�
 
 内核还拥有大�从代�自动生�的文档。它包�内核内部API的全�介�以�如何
 妥善处�加�的规则。生�的文档会放在 Documentation/DocBook/目录下。在内
 核��的主目录中使用以下��命令将会分别生�PDF�Postscript�HTML和手册
-页等��格�的文档：
+页等��格�的文档::
+
     make pdfdocs
     make htmldocs
 
@@ -155,7 +158,9 @@ Linux内核代�中包�有大�的文档。这些文档对于学习如何与
 如何�为内核开�者
 ------------------
 如果你对Linux内核开�一无所知，你应该访问“Linux内核新手�计划：
+
 	http://kernelnewbies.org
+
 它拥有一个�以问��最基本的内核开�问题的邮件列表（在�问之�一定�记得
 查找已往的邮件，确认是�有人已�回答过相�的问题）。它还拥有一个�以获得
 实时�馈的IRC�天频�，以�大�对于学习Linux内核开�相当有帮助的文档。
@@ -166,23 +171,21 @@ Linux内核代�中包�有大�的文档。这些文档对于学习如何与
 
 如果你想加入内核开�社区并�助完�一些任务，�找�到从哪里开始，�以访问
 “Linux内核房管员�计划：
+
 	http://kernelnewbies.org/KernelJanitors
+
 这是�佳的起点。它�供一个相对简�的任务列表，列出内核代�中需�被�新
 整�或者改正的地方。通过和负责这个计划的开�者们一�工作，你会学到将补�
 集�进内核的基本原�。如果还没有决定下一步��什么的�，你还�能会得到方
 �性的指点。
 
-如果你已�有一些现�的代�想�放到内核中，但是需�一些帮助�使它们拥有正
-确的格�。请访问“内核导师�计划。这个计划就是用�帮助你完�这个目标的。它
-是一个邮件列表，地�如下：
-	http://selenic.com/mailman/listinfo/kernel-mentors
-
 在真正动手修改内核代�之�，�解�修改的代�如何�作是必需的。�达到这个
 目的，没什么办法比直接读代�更有效了（大多数花招都会有相应的注释），而且
 一些特制的工具还�以�供帮助。例如，“Linux代�交�引用�项目就是一个值得
 特别推�的帮助工具，它将�代�显示在有编目和索引的网页上。其中一个更新�
 时的内核��库，�以通过以下地�访问：
-	http://sosdg.org/~coywolf/lxr/
+
+        https://elixir.bootlin.com/
 
 
 开��程
@@ -190,22 +193,23 @@ Linux内核代�中包�有大�的文档。这些文档对于学习如何与
 
 目�Linux内核开��程包括几个“主内核分支�和很多�系统相关的内核分支。这
 些分支包括：
-  - 2.6.x主内核��树
-  - 2.6.x.y -stable内核��树
-  - 2.6.x -mm内核补�集
-  - �系统相关的内核��树和补�集
 
+  - Linus 的内核��树
+  - 多个主�版本的稳定版内核树
+  - �系统相关的内核树
+  - linux-next 集�测试树
+
+
+主线树
+------
+主线树是由Linus Torvalds 维护的。你�以在https://kernel.org 网站或者代�
+库中下找到它。它的开��循以下步骤：
 
-2.6.x内核主��树
------------------
-2.6.x内核是由Linus Torvalds（Linux的创造者）亲自维护的。你�以在
-kernel.org网站的pub/linux/kernel/v2.6/目录下找到它。它的开��循以下步
-骤：
   - �当一个新版本的内核被�布，为期两周的集�窗�将被打开。在这段时间里
     维护者�以�Linus�交大段的修改，通常这些修改已�被放到-mm内核中几个
     星期了。�交大�修改的首选方�是使用git工具（内核的代�版本管�工具
-    ，更多的信��以在http://git-scm.com/获�），�过使用普通补�也是�以
-    的。
+    ，更多的信��以在 http://git-scm.com/ 获�），�过使用普通补�也是
+    �以的。
   - 两个星期以�-rc1版本内核�布。之��有�包��能影�整个内核稳定性的
     新功能的补���能被接�。请注�一个全新的驱动程�（或者文件系统）有
     �能在-rc1�被接�是因为这样的修改完全独立，�会影�其他的代�，所以
@@ -220,106 +224,61 @@ kernel.org网站的pub/linux/kernel/v2.6/目录下找到它。它的开��循
 	“没有人知�新内核何时会被�布，因为�布是根�已知bug的情况�决定
 	的，而�是根�一个事先制定好的时间表。�
 
+�系统特定树
+------------
 
-2.6.x.y -stable（稳定版）内核��树
------------------------------------
-由4个数字组�的内核版本�说明此内核是-stable版本。它们包�基于2.6.x版本
-内核的相对较�且至关��的修补，这些修补针对安全性问题或者严�的内核退步。
-
-这�版本的内核适用于那些期望获得最新的稳定版内核并且�想�与测试开�版或
-者实验版的用户。
-
-如果没有2.6.x.y版本内核存在，那么最新的2.6.x版本内核就相当于是当�的稳定
-版内核。
-
-2.6.x.y版本由“稳定版��组（邮件地�<stable@vger.kernel.org>）维护，一般隔周�
-布新版本。
-
-内核��中的Documentation/process/stable-kernel-rules.rst文件具体�述了�被稳定
-版内核接�的修改类型以��布的�程。
+��内核�系统的维护者——以�许多内核�系统开�人员——在�代�库中公开了他们
+当�的开�状�。这样，其他人就�以看到内核的��区域�生了什么。在开�速度
+很快的领域，�能会�求开�人员将�交的内容建立在这样的�系统内核树上，这样
+就��了�交与其他已�进行的工作之间的冲�。
 
+这些存储库中的大多数都是Git树，但是也有其他的scm在使用，或者补�队列被�布
+为Quilt系列。这些�系统存储库的地�列在MAINTAINERS文件中。其中许多�以在
+https://git.kernel.org/上�览。
 
-2.6.x -mm补�集
----------------
-这是由Andrew Morton维护的试验性内核补�集。Andrew将所有�系统的内核��
-和补�拼凑到一起，并且加入了大�从linux-kernel邮件列表中采集的补�。这个
-��树是新功能和补�的试炼场。当补�在-mm补�集里�明了其价值以�Andrew
-或者相应�系统的维护者会将补��给Linus以便集�进主内核��树。
+在将一个建议的补��交到这样的�系统树之�，需�对它进行审查，审查主��生
+在邮件列表上（请��下�相应的部分）。对于几个内核�系统，这个审查过程是通
+过工具补�跟踪的。Patchwork�供了一个Web界�，显示补��布�对补�的任何评
+论或修订，维护人员�以将补�标记为正在审查�接�或拒�。大多数补�网站都列
+在 https://patchwork.kernel.org/
 
-在将所有新补��给Linus以集�到主内核��树之�，我们�常鼓励先把这些补
-�放在-mm版内核��树中进行测试。
-
-这些内核版本�适�在需�稳定�行的系统上�行，因为�行它们比�行任何其他
-内核分支都更具有风险。
-
-如果你想为内核开�进程�供帮助，请�试并使用这些内核版本，并在
-linux-kernel邮件列表中�供�馈，告诉大家你�到了问题还是一切正常。
-
-通常-mm版补�集�光包括这些�外的试验性补�，还包括�布时-git版主��树
-中的改动。
-
--mm版内核没有固定的�布周期，但是通常在�两个-rc版内核�布之间都会有若干
-个-mm版内核�布（一般是1至3个）。
-
-
-�系统相关内核��树和补�集
-----------------------------
-相当一部分内核�系统开�者会公开他们自己的开���树，以便其他人能了解内
-核的��领域正在�生的事情。如上所述，这些��树会被集�到-mm版本内核中。
-
-下�是目��用的一些内核��树的列表：
-  通过git管�的��树：
-    - Kbuild开���树， Sam Ravnborg <sam@ravnborg.org>
-	git.kernel.org:/pub/scm/linux/kernel/git/sam/kbuild.git
-
-    - ACPI开���树, Len Brown <len.brown@intel.com>
-	git.kernel.org:/pub/scm/linux/kernel/git/lenb/linux-acpi-2.6.git
-
-    - �设备开���树, Jens Axboe <axboe@suse.de>
-	git.kernel.org:/pub/scm/linux/kernel/git/axboe/linux-2.6-block.git
-
-    - DRM开���树, Dave Airlie <airlied@linux.ie>
-	git.kernel.org:/pub/scm/linux/kernel/git/airlied/drm-2.6.git
-
-    - ia64开���树, Tony Luck <tony.luck@intel.com>
-	git.kernel.org:/pub/scm/linux/kernel/git/aegl/linux-2.6.git
+Linux-next 集�测试树
+---------------------
 
-    - ieee1394开���树, Jody McIntyre <scjody@modernduck.com>
-	git.kernel.org:/pub/scm/linux/kernel/git/scjody/ieee1394.git
+在将�系统树的更新�并到主线树之�，需�对它们进行集�测试。为此，存在一个
+特殊的测试存储库，其中几乎�天都会��所有�系统树：
 
-    - infiniband开���树, Roland Dreier <rolandd@cisco.com>
-	git.kernel.org:/pub/scm/linux/kernel/git/roland/infiniband.git
+        https://git.kernel.org/？p=linux/kernel/git/next/linux-next.git
 
-    - libata开���树, Jeff Garzik <jgarzik@pobox.com>
-	git.kernel.org:/pub/scm/linux/kernel/git/jgarzik/libata-dev.git
+通过这�方�，Linux-next 对下一个�并阶段将进入主线内核的内容给出了一个概�
+展望。�常欢冒险的测试者�行测试Linux-next。
 
-    - 网络驱动程�开���树, Jeff Garzik <jgarzik@pobox.com>
-	git.kernel.org:/pub/scm/linux/kernel/git/jgarzik/netdev-2.6.git
+多个主�版本的稳定版内核树
+-----------------------------------
+由3个数字组�的内核版本�说明此内核是-stable版本。它们包�内核的相对较�且
+至关��的修补，这些修补针对安全性问题或者严�的内核退步。
 
-    - pcmcia开���树, Dominik Brodowski <linux@dominikbrodowski.net>
-	git.kernel.org:/pub/scm/linux/kernel/git/brodo/pcmcia-2.6.git
+这�版本的内核适用于那些期望获得最新的稳定版内核并且�想�与测试开�版或
+者实验版的用户。
 
-    - SCSI开���树, James Bottomley <James.Bottomley@SteelEye.com>
-	git.kernel.org:/pub/scm/linux/kernel/git/jejb/scsi-misc-2.6.git
+稳定版内核树版本由“稳定版��组（邮件地�<stable@vger.kernel.org>）维护，一般
+隔周�布新版本。
 
-  使用quilt管�的补�集：
-    - USB, PCI, 驱动程�核心和I2C, Greg Kroah-Hartman <gregkh@linuxfoundation.org>
-	kernel.org/pub/linux/kernel/people/gregkh/gregkh-2.6/
-    - x86-64, 部分i386, Andi Kleen <ak@suse.de>
-	ftp.firstfloor.org:/pub/ak/x86_64/quilt/
+内核��中的 :ref:`Documentation/process/stable-kernel-rules.rst <stable_kernel_rules>`
+文件具体�述了�被稳定版内核接�的修改类型以��布的�程。
 
-  其他内核��树�以在http://git.kernel.org的列表中和MAINTAINERS文件里
-  找到。
 
 报告bug
 -------
 
 bugzilla.kernel.org是Linux内核开�者们用�跟踪内核Bug的网站。我们鼓励用
 户在这个工具中报告找到的所有bug。如何使用内核bugzilla的细节请访问：
+
 	http://test.kernel.org/bugzilla/faq.html
 
-内核��主目录中的admin-guide/reporting-bugs.rst文件里有一个很好的模�。它指导用户如何报
-告�能的内核bug以�需��供哪些信��帮助内核开�者们找到问题的根�。
+内核��主目录中的:ref:`admin-guide/reporting-bugs.rst <reportingbugs>`
+文件里有一个很好的模�。它指导用户如何报告�能的内核bug以�需��供哪些信�
+�帮助内核开�者们找到问题的根�。
 
 
 利用bug报告
@@ -330,12 +289,7 @@ bugzilla.kernel.org是Linux内核开å�‘者们用æ�¥è·Ÿè¸ªå†…æ ¸Bug的网站。æˆ
 者感�到你的存在。修改bug是赢得其他开�者赞誉的最好办法，因为并�是很多
 人都喜欢浪费时间去修改别人报告的bug。
 
-��试修改已知的bug，请访问http://bugzilla.kernel.org网�。如果你想获得
-最新bug的通知，�以订阅bugme-new邮件列表（�有新的bug报告会被寄到这里）
-或者订阅bugme-janitor邮件列表（所有bugzilla的�动都会被寄到这里）。
-
-	https://lists.linux-foundation.org/mailman/listinfo/bugme-new
-	https://lists.linux-foundation.org/mailman/listinfo/bugme-janitors
+��试修改已知的bug，请访问 http://bugzilla.kernel.org 网�。
 
 
 邮件列表
@@ -343,10 +297,14 @@ bugzilla.kernel.org是Linux内核开å�‘者们用æ�¥è·Ÿè¸ªå†…æ ¸Bug的网站。æˆ
 
 正如上�的文档所�述，大多数的骨干内核开�者都加入了Linux Kernel邮件列
 表。如何订阅和退订列表的细节�以在这里找到：
+
 	http://vger.kernel.org/vger-lists.html#linux-kernel
+
 网上很多地方都有这个邮件列表的存档(archive)。�以使用�索引擎�找到这些
 存档。比如：
+
 	http://dir.gmane.org/gmane.linux.kernel
+
 在�信之�，我们强烈建议你先在存档中�索你想�讨论的问题。很多已�被详细
 讨论过的问题�在邮件列表的存档中�以找到。
 
@@ -354,10 +312,12 @@ bugzilla.kernel.org是Linux内核开å�‘者们用æ�¥è·Ÿè¸ªå†…æ ¸Bug的网站。æˆ
 MAINTAINERS文件中�以找到���题对应的邮件列表。
 
 很多邮件列表架设在kernel.org�务器上。这些列表的信��以在这里找到：
+
 	http://vger.kernel.org/vger-lists.html
 
 在使用这些邮件列表时，请记���良好的行为习惯。下�的链接�供了与这些列
 表（或任何其它邮件列表）交�的一些简�规则，虽然内容有点滥竽充数。
+
 	http://www.albion.com/netiquette/
 
 当有很多人回�你的邮件时，邮件的抄�列表会�得很长。请��将任何人从抄�
@@ -369,11 +329,12 @@ MAINTAINERS文件中�以找到���题对应的邮件列表。
 这几行。将你的评论加在被引用的段�之间而��放在邮件的顶部。
 
 如果你在邮件中附带补�，请确认它们是�以直接阅读的纯文本（如
-Documentation/process/submitting-patches.rst文档中所述）。内核开�者们�希望�到附件
-或者被压缩了的补�。�有这样�能��他们�以直接评论你的�行代�。请确�
-你使用的邮件��程��会修改空格和制表符。一个防范性的测试方法是先将邮件
-��给自己，然�自己�试是��以顺利地打上收到的补�。如果测试��功，请
-调整或者更�你的邮件��程�直到它正确工作为止。
+:ref:`Documentation/translations/zh_CN/process/submitting-patches.rst <cn_submittingpatches>`
+文档中所述）。内核开�者们�希望�到附件或者被压缩了的补�。�有这样�能
+��他们�以直接评论你的�行代�。请确�你使用的邮件��程��会修改空格
+和制表符。一个防范性的测试方法是先将邮件��给自己，然�自己�试是��以
+顺利地打上收到的补�。如果测试��功，请调整或者更�你的邮件��程�直到
+它正确工作为止。
 
 总而言之，请尊�其他的邮件列表订阅者。
 
@@ -383,6 +344,7 @@ Documentation/process/submitting-patches.rst文档中所述）。内核开�者
 
 内核社区的目标就是�供尽善尽美的内核。所以当你�交补�期望被接�进内核的
 时候，它的技术价值以�其他方�都将被评审。那么你�能会得到什么呢？
+
   - 批评
   - 评论
   - �求修改
@@ -395,6 +357,7 @@ Documentation/process/submitting-patches.rst文档中所述）。内核开�者
 没在茫茫信海中。
 
 你�应该�的事情：
+
   - 期望自己的补���任何质疑就直接被接�
   - 翻脸
   - 忽略别人的评论
@@ -414,7 +377,8 @@ Documentation/process/submitting-patches.rst文档中所述）。内核开�者
 
 内核社区的工作模��大多数传统公�开�队�的工作模�并�相�。下�这些例
 �，�以帮助你���些�能�生问题：
-  用这些�介�你的修改�案会有好处：
+用这些�介�你的修改�案会有好处：
+
     - 它�时解决了多个问题
     - 它删除了2000行代�
     - 这是补�，它已�解释了我想�说明的
@@ -422,7 +386,8 @@ Documentation/process/submitting-patches.rst文档中所述）。内核开�者
     - 这是一系列�补�用�……
     - 这个修改�高了普通机器的性能……
 
-  应该��如下的说法：
+应该��如下的说法：
+
     - 我们在AIX/ptx/Solaris就是这么�的，所以这么�肯定是好的……
     - 我�这行已�20年了，所以……
     - 为了我们公�赚钱考虑必须这么�
@@ -495,6 +460,7 @@ Linux内核社区并�喜欢一下接收大段的代�。修改需�被�当
 当你��补�的时候，需�特别留�邮件正文的内容。因为这里的信�将会�为补
 �的修改记录(ChangeLog)，会被一直�留以备大家查阅。它需�完全地�述补�，
 包括：
+
   - 为什么需�这个修改
   - 补�的总体设计
   - 实现细节
@@ -510,7 +476,8 @@ Linux内核社区并�喜欢一下接收大段的代�。修改需�被�当
 很多人已��到了，而他们都曾�和现在的你站在�样的起点上。
 
 
----------------
+æ„Ÿè°¢
+----
 感谢Paolo Ciarrocchi�许“开��程�部分基于他所写的文章
 (http://www.kerneltravel.net/newbie/2.6-development_process)，感谢Randy
 Dunlap和Gerrit Huizenga完善了应该说和�该说的列表。感谢Pat Mochel, Hanna
diff --git a/Documentation/translations/zh_CN/process/index.rst b/Documentation/translations/zh_CN/process/index.rst
new file mode 100644
index 000000000000..be1e764a80d2
--- /dev/null
+++ b/Documentation/translations/zh_CN/process/index.rst
@@ -0,0 +1,60 @@
+.. raw:: latex
+
+	\renewcommand\thesection*
+	\renewcommand\thesubsection*
+
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: :ref:`Documentation/process/index.rst <process_index>`
+:Translator: Alex Shi <alex.shi@linux.alibaba.com>
+
+.. _cn_process_index:
+
+与Linux 内核社区一起工作
+========================
+
+那么你想�为Linux内核开�人员？ 欢迎� �但从技术�义上讲有很多关于内核的知识
+需�学，而且了解我们社区的工作方�也很��。 阅读这些文章�以让您以更轻�地,
+麻烦最少的方�将更改�并到内核。
+
+以下是��开�人员应阅读的基本指�。
+
+.. toctree::
+   :maxdepth: 1
+
+   howto
+   code-of-conduct
+   code-of-conduct-interpretation
+   submitting-patches
+   programming-language
+   coding-style
+   development-process
+   email-clients
+   license-rules
+
+其它大多数开�人员感兴趣的社区指�：
+
+
+.. toctree::
+   :maxdepth: 1
+
+   submitting-drivers
+   submit-checklist
+   stable-api-nonsense
+   stable-kernel-rules
+   management-style
+
+这些是一些总体技术指�，由于缺�更好的地方，现在已�放在这里
+
+.. toctree::
+   :maxdepth: 1
+
+   magic-number
+   volatile-considered-harmful
+
+.. only::  subproject and html
+
+   目录
+   ====
+
+   * :ref:`genindex`
diff --git a/Documentation/translations/zh_CN/process/license-rules.rst b/Documentation/translations/zh_CN/process/license-rules.rst
new file mode 100644
index 000000000000..30c272b2a292
--- /dev/null
+++ b/Documentation/translations/zh_CN/process/license-rules.rst
@@ -0,0 +1,370 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: :ref:`Documentation/process/license-rules.rst <kernel_licensing>`
+:Translator: Alex Shi <alex.shi@linux.alibaba.com>
+
+.. _cn_kernel_licensing:
+
+Linux内核许�规则
+=================
+
+Linux内核根�LICENSES/preferred/GPL-2.0中�供的GNU通用公共许��版本2
+（GPL-2.0）的�款�供，并在LICENSES/exceptions/Linux-syscall-note中显�
+�述了例外的系统调用，如COPYING文件中所述。
+
+此文档文件�供了如何对�个�文件进行注释以使其许��清晰明确的说明。
+它�会�代内核的许��。
+
+内核�代�作为一个整体适用于COPYING文件中�述的许��，但是�个�文件�以
+具有��的与GPL-20兼容的许��::
+
+    GPL-1.0+ : GNU通用公共许��v1.0或更高版本
+    GPL-2.0+ : GNU通用公共许��v2.0或更高版本
+    LGPL-2.0 : 仅�GNU库通用公共许��v2
+    LGPL-2.0+: GNU 库通用公共许��v2或更高版本
+    LGPL-2.1 : 仅�GNU宽通用公共许��v2.1
+    LGPL-2.1+: GNU宽通用公共许��v2.1或更高版本
+
+除此之外，个人文件�以在��许�下�供，例如一个兼容的GPL�体，或者BSD，
+MIT等许�。
+
+用户空间API（UAPI）头文件�述了用户空间程�与内核的接�，这是一�特殊情况。
+根�内核COPYING文件中的注释，syscall接�是一个明确的边界，它�会将GPL�求
+扩展到任何使用它与内核通信的软件。由于UAPI头文件必须包�在创建在Linux内核
+上�行的�执行文件的任何�文件中，因此此例外必须记录在特别的许��表述中。
+
+表达�文件许��的常用方法是将匹�的样�文本添加到文件的顶部注释中。由于
+格�，拼写错误等，这些“样��很难通过那些在上下文中使用的验�许���规性
+的工具。
+
+样�文本的替代方法是在�个�文件中使用软件包数�交�（SPDX）许��标识符。
+SPDX许��标识符是机器�解�的，并且是用于�供文件内容的许��的精确缩写。
+SPDX许��标识符由Linux 基金会的SPDX 工作组管�，并得到了整个行业，工具
+供应商和法律团队的�作伙伴的一致��。有关详细信�，请�阅
+https://spdx.org/
+
+Linux内核需�所有�文件中的精确SPDX标识符。内核中使用的有效标识符在
+`许�标识符`_ 一节中进行了解释，并且已�以在
+https://spdx.org/licenses/ 上的官方SPDX许��列表中检索，并附带许��
+文本。
+
+许�标识符语法
+--------------
+
+1.安置:
+
+   内核文件中的SPDX许��标识符应添加到�包�注释的文件中的第一行。对于大多
+   数文件，这是第一行，除了那些在第一行中需�'#!PATH_TO_INTERPRETER'的脚本。
+   对于这些脚本，SPDX标识符进入第二行。
+
+|
+
+2. 风格:
+
+   SPDX许��标识符以注释的形�添加。注释样��决于文件类型::
+
+      C source:	// SPDX-License-Identifier: <SPDX License Expression>
+      C header:	/* SPDX-License-Identifier: <SPDX License Expression> */
+      ASM:	/* SPDX-License-Identifier: <SPDX License Expression> */
+      scripts:	# SPDX-License-Identifier: <SPDX License Expression>
+      .rst:	.. SPDX-License-Identifier: <SPDX License Expression>
+      .dts{i}:	// SPDX-License-Identifier: <SPDX License Expression>
+
+   如果特定工具无法处�标准注释样�，则应使用工具接�的相应注释机制。这是在
+   C 头文件中使用“/\*\*/�样�注释的原因。过去在使用生�的.lds文件中观察到
+   构建被破�，其中'ld'无法解�C++注释。现在已�解决了这个问题，但�然有较
+   旧的汇编程�工具无法处�C++样�的注释。
+
+|
+
+3. �法:
+
+   <SPDX许��表达�>是SPDX许��列表中的SPDX短格�许��标识符，或者在许�
+   �例外适用时由“WITH�分隔的两个SPDX短格�许��标识符的组�。当应用多个许
+   ��时，表达�由分隔�表达�的关键字“AND�，“OR�组�，并由“（�，“）�包围。
+
+   带有“或更高�选项的[L]GPL等许��的许��标识符通过使用“+��表示“或更高�
+   选项�构建。::
+
+      // SPDX-License-Identifier: GPL-2.0+
+      // SPDX-License-Identifier: LGPL-2.1+
+
+   当需�修正的许��时，应使用WITH。 例如，linux内核UAPI文件使用表达�::
+
+      // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
+      // SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note
+
+   其它在内核中使用WITH例外的事例如下::
+
+      // SPDX-License-Identifier: GPL-2.0 WITH mif-exception
+      // SPDX-License-Identifier: GPL-2.0+ WITH GCC-exception-2.0
+
+   例外�能与特定的许��标识符一起使用。有效的许��标识符列在异常文本文件
+   的标记中。有关详细信�，请�阅 `许�标识符`_ 一章中的 `例外`_ 。
+
+   如果文件是��许�且�选择一个许��，则应使用OR。例如，一些dtsi文件在�
+   许�下�用::
+
+      // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+
+   内核中�许�文件中许�表达�的示例::
+
+      // SPDX-License-Identifier: GPL-2.0 OR MIT
+      // SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
+      // SPDX-License-Identifier: GPL-2.0 OR Apache-2.0
+      // SPDX-License-Identifier: GPL-2.0 OR MPL-1.1
+      // SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) OR MIT
+      // SPDX-License-Identifier: GPL-1.0+ OR BSD-3-Clause OR OpenSSL
+
+   如果文件具有多个许��，其�款全部适用于使用该文件，则应使用AND。例如，
+   如果代�是从�一个项目继承的，并且已�授予了将其放入内核的��，但原始
+   许��款需���有效::
+
+      // SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) AND MIT
+
+   �一个需��守两套许��款的例�是::
+
+      // SPDX-License-Identifier: GPL-1.0+ AND LGPL-2.1+
+
+许�标识符
+----------
+
+当�使用的许��以�添加到内核的代�许���以分解为：
+
+1. _`优先许�`:
+
+   应尽�能使用这些许��，因为它们已知完全兼容并广泛使用。这些许��在内核
+   目录::
+
+      LICENSES/preferred/
+
+   此目录中的文件包�完整的许��文本和 `元标记`_ 。文件�与SPDX许��标识
+   符相�，�者应用于�文件中的许��。
+
+   例如::
+
+      LICENSES/preferred/GPL-2.0
+
+   包�GPLv2许��文本和所需的元标签::
+
+      LICENSES/preferred/MIT
+
+   包�MIT许��文本和所需的元标记
+
+   _`元标记`:
+
+   许��文件中必须包�以下元标记：
+
+   - Valid-License-Identifier:
+
+     一行或多行, 声明那些许�标识符在项目内有效, 以引用此特定许�的文本。通
+     常这是一个有效的标识符，但是例如对于带有'或更高'选项的许��，两个标识
+     符都有效。
+
+   - SPDX-URL:
+
+     SPDX页�的URL，其中包�与许��相关的其他信�.
+
+   - Usage-Guidance:
+
+     使用建议的自由格�文本。该文本必须包�SPDX许��标识符的正确示例，因为
+     它们应根� `许�标识符语法`_ 指�放入�文件中。
+
+   - License-Text:
+
+     此标记之�的所有文本都被视为原始许�文本
+
+   文件格�示例::
+
+      Valid-License-Identifier: GPL-2.0
+      Valid-License-Identifier: GPL-2.0+
+      SPDX-URL: https://spdx.org/licenses/GPL-2.0.html
+      Usage-Guide:
+        To use this license in source code, put one of the following SPDX
+	tag/value pairs into a comment according to the placement
+	guidelines in the licensing rules documentation.
+	For 'GNU General Public License (GPL) version 2 only' use:
+	  SPDX-License-Identifier: GPL-2.0
+	For 'GNU General Public License (GPL) version 2 or any later version' use:
+	  SPDX-License-Identifier: GPL-2.0+
+      License-Text:
+        Full license text
+
+   ::
+
+      SPDX-License-Identifier: MIT
+      SPDX-URL: https://spdx.org/licenses/MIT.html
+      Usage-Guide:
+	To use this license in source code, put the following SPDX
+	tag/value pair into a comment according to the placement
+	guidelines in the licensing rules documentation.
+	  SPDX-License-Identifier: MIT
+      License-Text:
+        Full license text
+
+|
+
+2. �推�的许��:
+
+   这些许���应用于现有代�或从其他项目导入代�。这些许��在内核目录::
+
+      LICENSES/other/
+
+   此目录中的文件包�完整的许��文本和 `元标记`_ 。文件�与SPDX许��标识
+   符相�，�者应用于�文件中的许��。
+
+   例如::
+
+      LICENSES/other/ISC
+
+   包�国际系统��许�文本和所需的元标签::
+
+      LICENSES/other/ZLib
+
+   包�ZLIB许�文本和所需的元标签.
+
+   元标签:
+
+   “其他�许��的元标签�求与 `优先许�`_ 的�求相�。
+
+   文件格�示例::
+
+      Valid-License-Identifier: ISC
+      SPDX-URL: https://spdx.org/licenses/ISC.html
+      Usage-Guide:
+        Usage of this license in the kernel for new code is discouraged
+	and it should solely be used for importing code from an already
+	existing project.
+        To use this license in source code, put the following SPDX
+	tag/value pair into a comment according to the placement
+	guidelines in the licensing rules documentation.
+	  SPDX-License-Identifier: ISC
+      License-Text:
+        Full license text
+
+|
+
+3. _`例外`:
+
+   �些许���以修改，并�许原始许���具有的�些例外�利。这些例外在
+   内核目录::
+
+      LICENSES/exceptions/
+
+   此目录中的文件包�完整的例外文本和所需的 `例外元标记`_ 。
+
+   例如::
+
+      LICENSES/exceptions/Linux-syscall-note
+
+   包�Linux内核的COPYING文件中记录的Linux系统调用例外，该文件用于UAPI
+   头文件。例如::
+
+      LICENSES/exceptions/GCC-exception-2.0
+
+   包�GCC'链接例外'，它�许独立于其许��的任何二进制文件与标记有此例外的
+   文件的编译版本链接。这是从GPL�兼容�代�创建��行的�执行文件所必需的。
+
+   _`例外元标记`:
+
+   以下元标记必须在例外文件中�用：
+
+   - SPDX-Exception-Identifier:
+
+     一个�与SPDX许��标识符一起使用的例外标识符。
+
+   - SPDX-URL:
+
+     SPDX页�的URL，其中包�与例外相关的其他信�。
+
+   - SPDX-Licenses:
+
+     以逗�分隔的例外�用的SPDX许��标识符列表。
+
+   - Usage-Guidance:
+
+     使用建议的自由格�文本。必须在文本��加上SPDX许��标识符的正确示例，
+     因为它们应根� `许�标识符语法`_ 指�放入�文件中。
+
+   - Exception-Text:
+
+     此标记之�的所有文本都被视为原始异常文本
+
+   文件格�示例::
+
+      SPDX-Exception-Identifier: Linux-syscall-note
+      SPDX-URL: https://spdx.org/licenses/Linux-syscall-note.html
+      SPDX-Licenses: GPL-2.0, GPL-2.0+, GPL-1.0+, LGPL-2.0, LGPL-2.0+, LGPL-2.1, LGPL-2.1+
+      Usage-Guidance:
+        This exception is used together with one of the above SPDX-Licenses
+	to mark user-space API (uapi) header files so they can be included
+	into non GPL compliant user-space application code.
+        To use this exception add it with the keyword WITH to one of the
+	identifiers in the SPDX-Licenses tag:
+	  SPDX-License-Identifier: <SPDX-License> WITH Linux-syscall-note
+      Exception-Text:
+        Full exception text
+
+   ::
+
+      SPDX-Exception-Identifier: GCC-exception-2.0
+      SPDX-URL: https://spdx.org/licenses/GCC-exception-2.0.html
+      SPDX-Licenses: GPL-2.0, GPL-2.0+
+      Usage-Guidance:
+        The "GCC Runtime Library exception 2.0" is used together with one
+	of the above SPDX-Licenses for code imported from the GCC runtime
+	library.
+        To use this exception add it with the keyword WITH to one of the
+	identifiers in the SPDX-Licenses tag:
+	  SPDX-License-Identifier: <SPDX-License> WITH GCC-exception-2.0
+      Exception-Text:
+        Full exception text
+
+
+所有SPDX许��标识符和例外都必须在LICENSES�目录中具有相应的文件。这是�许
+工具验�（例如checkpatch.pl）以�准备好从�读�和��许��所必需的, 这是
+��FOSS组织推�的，例如 `FSFE REUSE initiative <https://reuse.software/>`_.
+
+_`模�许�`
+-----------------
+
+   �加载内核模�还需�MODULE_LICENSE（）标记。此标记既�替代正确的�代�
+   许��信�（SPDX-License-Identifier），也�以任何方�表示或确定�供模�
+   �代�的确切许��。
+
+   此标记的唯一目的是�供足够的信�，该模�是�是自由软件或者是内核模�加
+   载器和用户空间工具的专有模�。
+
+   MODULE_LICENSE（）的有效许��字符串是:
+
+    ============================= =============================================
+    "GPL"			  模�是根�GPL版本2许�的。这并�表示仅�于
+                                  GPL-2.0或GPL-2.0或更高版本之间的任何区别。
+                                  最正确许��信��能通过相应�文件中的许��
+                                  信��确定
+
+    "GPL v2"			  和"GPL"相�，它的存在是因为历�原因。
+
+    "GPL and additional rights"   表示模��在GPL v2�体和MIT许�下��许�的
+                                  历��体。请��在新代�中使用。
+
+    "Dual MIT/GPL"		  表达该模�在GPL v2�体或MIT许��选择下��
+                                  许�的正确方�。
+
+    "Dual BSD/GPL"		  该模�根�GPL v2�体或BSD许��选择进行��
+                                  许�。 BSD许��的确切�体�能通过相应�文件
+                                  中的许��信��确定。
+
+    "Dual MPL/GPL"		  该模�根�GPL v2�体或Mozilla Public License
+                                  （MPL）选项进行��许�。 MPL许��的确切�体
+                                  �能通过相应的�文件中的许��信��确定。
+
+    "Proprietary"		  该模�属于专有许�。此字符串仅用于专有的第三
+                                  方模�，�能用于在内核树中具有�代�的模�。
+                                  以这�方�标记的模�在加载时会使用'P'标记污
+                                  染内核，并且内核模�加载器拒�将这些模�链接
+                                  到使用EXPORT_SYMBOL_GPL（）导出的符�。
+    ============================= =============================================
+
diff --git a/Documentation/translations/zh_CN/process/magic-number.rst b/Documentation/translations/zh_CN/process/magic-number.rst
new file mode 100644
index 000000000000..15c592518194
--- /dev/null
+++ b/Documentation/translations/zh_CN/process/magic-number.rst
@@ -0,0 +1,151 @@
+.. _cn_magicnumbers:
+
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: :ref:`Documentation/process/magic-number.rst <magicnumbers>`
+
+如果想评论或更新本文的内容，请直接�信到LKML。如果你使用英文交�有困难的�，也�
+以�中文版维护者求助。如果本翻译更新��时或者翻译存在问题，请�系中文版维护者::
+
+        中文版维护者： 贾�� Jia Wei Wei <harryxiyou@gmail.com>
+        中文版翻译者： 贾�� Jia Wei Wei <harryxiyou@gmail.com>
+        中文版校译者： 贾�� Jia Wei Wei <harryxiyou@gmail.com>
+
+Linux 魔术数
+============
+
+这个文件是有关当�使用的魔术值注册表。当你给一个结构添加了一个魔术值，你也应该把这个魔术值添加到这个文件，因为我们最好把用于��结构的魔术值统一起�。
+
+使用魔术值��护内核数�结构是一个�常好的主�。这就�许你在�行期检查(a)一个结构是�已�被攻击，或者(b)你已�给一个例行程�通过了一个错误的结构。�一�情况特别地有用---特别是当你通过一个空指针指�结构体的时候。tty��，例如，�常通过特定驱动使用这�方法并且��地排列特定方�的结构。
+
+使用魔术值的方法是在结构的开始处声明的，如下::
+
+        struct tty_ldisc {
+	        int	magic;
+        	...
+        };
+
+当你以�给内核添加增强功能的时候，请�守这�规则�这样就会节�数�清的调试时间，特别是一些�怪的情况，例如，数组超出范围并且�新写了超出部分。�守这个规则，‪这些情况�以被快速地，安全地��。
+
+		Theodore Ts'o
+		  31 Mar 94
+
+给当�的Linux 2.1.55添加魔术表。
+
+		Michael Chastain
+		<mailto:mec@shout.net>
+		22 Sep 1997
+
+现在应该最新的Linux 2.1.112.因为在特性冻结期间，�能在2.2.x�改�任何东西。这些�目被数域所排�。
+
+		Krzysztof G.Baranowski
+	        <mailto: kgb@knm.org.pl>
+		29 Jul 1998
+
+更新魔术表到Linux 2.5.45。刚好越过特性冻结，但是有�能还会有一些新的魔术值在2.6.x之��入到内核中。
+
+		Petr Baudis
+		<pasky@ucw.cz>
+		03 Nov 2002
+
+更新魔术表到Linux 2.5.74。
+
+		Fabian Frederick
+                <ffrederick@users.sourceforge.net>
+		09 Jul 2003
+
+===================== ================ ======================== ==========================================
+魔术数�              数字             结构                     文件
+===================== ================ ======================== ==========================================
+PG_MAGIC              'P'              pg_{read,write}_hdr      ``include/linux/pg.h``
+CMAGIC                0x0111           user                     ``include/linux/a.out.h``
+MKISS_DRIVER_MAGIC    0x04bf           mkiss_channel            ``drivers/net/mkiss.h``
+HDLC_MAGIC            0x239e           n_hdlc                   ``drivers/char/n_hdlc.c``
+APM_BIOS_MAGIC        0x4101           apm_user                 ``arch/x86/kernel/apm_32.c``
+CYCLADES_MAGIC        0x4359           cyclades_port            ``include/linux/cyclades.h``
+DB_MAGIC              0x4442           fc_info                  ``drivers/net/iph5526_novram.c``
+DL_MAGIC              0x444d           fc_info                  ``drivers/net/iph5526_novram.c``
+FASYNC_MAGIC          0x4601           fasync_struct            ``include/linux/fs.h``
+FF_MAGIC              0x4646           fc_info                  ``drivers/net/iph5526_novram.c``
+ISICOM_MAGIC          0x4d54           isi_port                 ``include/linux/isicom.h``
+PTY_MAGIC             0x5001                                    ``drivers/char/pty.c``
+PPP_MAGIC             0x5002           ppp                      ``include/linux/if_pppvar.h``
+SERIAL_MAGIC          0x5301           async_struct             ``include/linux/serial.h``
+SSTATE_MAGIC          0x5302           serial_state             ``include/linux/serial.h``
+SLIP_MAGIC            0x5302           slip                     ``drivers/net/slip.h``
+STRIP_MAGIC           0x5303           strip                    ``drivers/net/strip.c``
+X25_ASY_MAGIC         0x5303           x25_asy                  ``drivers/net/x25_asy.h``
+SIXPACK_MAGIC         0x5304           sixpack                  ``drivers/net/hamradio/6pack.h``
+AX25_MAGIC            0x5316           ax_disp                  ``drivers/net/mkiss.h``
+TTY_MAGIC             0x5401           tty_struct               ``include/linux/tty.h``
+MGSL_MAGIC            0x5401           mgsl_info                ``drivers/char/synclink.c``
+TTY_DRIVER_MAGIC      0x5402           tty_driver               ``include/linux/tty_driver.h``
+MGSLPC_MAGIC          0x5402           mgslpc_info              ``drivers/char/pcmcia/synclink_cs.c``
+TTY_LDISC_MAGIC       0x5403           tty_ldisc                ``include/linux/tty_ldisc.h``
+USB_SERIAL_MAGIC      0x6702           usb_serial               ``drivers/usb/serial/usb-serial.h``
+FULL_DUPLEX_MAGIC     0x6969                                    ``drivers/net/ethernet/dec/tulip/de2104x.c``
+USB_BLUETOOTH_MAGIC   0x6d02           usb_bluetooth            ``drivers/usb/class/bluetty.c``
+RFCOMM_TTY_MAGIC      0x6d02                                    ``net/bluetooth/rfcomm/tty.c``
+USB_SERIAL_PORT_MAGIC 0x7301           usb_serial_port          ``drivers/usb/serial/usb-serial.h``
+CG_MAGIC              0x00090255       ufs_cylinder_group       ``include/linux/ufs_fs.h``
+RPORT_MAGIC           0x00525001       r_port                   ``drivers/char/rocket_int.h``
+LSEMAGIC              0x05091998       lse                      ``drivers/fc4/fc.c``
+GDTIOCTL_MAGIC        0x06030f07       gdth_iowr_str            ``drivers/scsi/gdth_ioctl.h``
+RIEBL_MAGIC           0x09051990                                ``drivers/net/atarilance.c``
+NBD_REQUEST_MAGIC     0x12560953       nbd_request              ``include/linux/nbd.h``
+RED_MAGIC2            0x170fc2a5       (any)                    ``mm/slab.c``
+BAYCOM_MAGIC          0x19730510       baycom_state             ``drivers/net/baycom_epp.c``
+ISDN_X25IFACE_MAGIC   0x1e75a2b9       isdn_x25iface_proto_data ``drivers/isdn/isdn_x25iface.h``
+ECP_MAGIC             0x21504345       cdkecpsig                ``include/linux/cdk.h``
+LSOMAGIC              0x27091997       lso                      ``drivers/fc4/fc.c``
+LSMAGIC               0x2a3b4d2a       ls                       ``drivers/fc4/fc.c``
+WANPIPE_MAGIC         0x414C4453       sdla_{dump,exec}         ``include/linux/wanpipe.h``
+CS_CARD_MAGIC         0x43525553       cs_card                  ``sound/oss/cs46xx.c``
+LABELCL_MAGIC         0x4857434c       labelcl_info_s           ``include/asm/ia64/sn/labelcl.h``
+ISDN_ASYNC_MAGIC      0x49344C01       modem_info               ``include/linux/isdn.h``
+CTC_ASYNC_MAGIC       0x49344C01       ctc_tty_info             ``drivers/s390/net/ctctty.c``
+ISDN_NET_MAGIC        0x49344C02       isdn_net_local_s         ``drivers/isdn/i4l/isdn_net_lib.h``
+SAVEKMSG_MAGIC2       0x4B4D5347       savekmsg                 ``arch/*/amiga/config.c``
+CS_STATE_MAGIC        0x4c4f4749       cs_state                 ``sound/oss/cs46xx.c``
+SLAB_C_MAGIC          0x4f17a36d       kmem_cache               ``mm/slab.c``
+COW_MAGIC             0x4f4f4f4d       cow_header_v1            ``arch/um/drivers/ubd_user.c``
+I810_CARD_MAGIC       0x5072696E       i810_card                ``sound/oss/i810_audio.c``
+TRIDENT_CARD_MAGIC    0x5072696E       trident_card             ``sound/oss/trident.c``
+ROUTER_MAGIC          0x524d4157       wan_device               [in ``wanrouter.h`` pre 3.9]
+SAVEKMSG_MAGIC1       0x53415645       savekmsg                 ``arch/*/amiga/config.c``
+GDA_MAGIC             0x58464552       gda                      ``arch/mips/include/asm/sn/gda.h``
+RED_MAGIC1            0x5a2cf071       (any)                    ``mm/slab.c``
+EEPROM_MAGIC_VALUE    0x5ab478d2       lanai_dev                ``drivers/atm/lanai.c``
+HDLCDRV_MAGIC         0x5ac6e778       hdlcdrv_state            ``include/linux/hdlcdrv.h``
+PCXX_MAGIC            0x5c6df104       channel                  ``drivers/char/pcxx.h``
+KV_MAGIC              0x5f4b565f       kernel_vars_s            ``arch/mips/include/asm/sn/klkernvars.h``
+I810_STATE_MAGIC      0x63657373       i810_state               ``sound/oss/i810_audio.c``
+TRIDENT_STATE_MAGIC   0x63657373       trient_state             ``sound/oss/trident.c``
+M3_CARD_MAGIC         0x646e6f50       m3_card                  ``sound/oss/maestro3.c``
+FW_HEADER_MAGIC       0x65726F66       fw_header                ``drivers/atm/fore200e.h``
+SLOT_MAGIC            0x67267321       slot                     ``drivers/hotplug/cpqphp.h``
+SLOT_MAGIC            0x67267322       slot                     ``drivers/hotplug/acpiphp.h``
+LO_MAGIC              0x68797548       nbd_device               ``include/linux/nbd.h``
+OPROFILE_MAGIC        0x6f70726f       super_block              ``drivers/oprofile/oprofilefs.h``
+M3_STATE_MAGIC        0x734d724d       m3_state                 ``sound/oss/maestro3.c``
+VMALLOC_MAGIC         0x87654320       snd_alloc_track          ``sound/core/memory.c``
+KMALLOC_MAGIC         0x87654321       snd_alloc_track          ``sound/core/memory.c``
+PWC_MAGIC             0x89DC10AB       pwc_device               ``drivers/usb/media/pwc.h``
+NBD_REPLY_MAGIC       0x96744668       nbd_reply                ``include/linux/nbd.h``
+ENI155_MAGIC          0xa54b872d       midway_eprom	        ``drivers/atm/eni.h``
+CODA_MAGIC            0xC0DAC0DA       coda_file_info           ``fs/coda/coda_fs_i.h``
+DPMEM_MAGIC           0xc0ffee11       gdt_pci_sram             ``drivers/scsi/gdth.h``
+YAM_MAGIC             0xF10A7654       yam_port                 ``drivers/net/hamradio/yam.c``
+CCB_MAGIC             0xf2691ad2       ccb                      ``drivers/scsi/ncr53c8xx.c``
+QUEUE_MAGIC_FREE      0xf7e1c9a3       queue_entry              ``drivers/scsi/arm/queue.c``
+QUEUE_MAGIC_USED      0xf7e1cc33       queue_entry              ``drivers/scsi/arm/queue.c``
+HTB_CMAGIC            0xFEFAFEF1       htb_class                ``net/sched/sch_htb.c``
+NMI_MAGIC             0x48414d4d455201 nmi_s                    ``arch/mips/include/asm/sn/nmi.h``
+===================== ================ ======================== ==========================================
+
+
+请注�，在声音记忆管�中�然有一些特殊的为�个驱动定义的魔术值。查看include/sound/sndmagic.h�获�他们完整的列表信�。很多OSS声音驱动拥有自己从声�PCI ID构建的魔术值-他们也没有被列在这里。
+
+IrDA�系统也使用了大�的自己的魔术值，查看include/net/irda/irda.h�获�他们完整的信�。
+
+HFS是�外一个比较大的使用魔术值的文件系统-你�以在fs/hfs/hfs.h中找到他们。
diff --git a/Documentation/translations/zh_CN/process/management-style.rst b/Documentation/translations/zh_CN/process/management-style.rst
new file mode 100644
index 000000000000..a181fa56d19e
--- /dev/null
+++ b/Documentation/translations/zh_CN/process/management-style.rst
@@ -0,0 +1,207 @@
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: :ref:`Documentation/process/management-style.rst <managementstyle>`
+:Translator: Alex Shi <alex.shi@linux.alibaba.com>
+
+.. _cn_managementstyle:
+
+Linux内核管�风格
+=================
+
+这是一个简短的文档，�述了Linux内核首选的（或胡编的，�决于您问�）管�风格。
+它的目的是在��程度上�照 :ref:`process/coding-style.rst <codingstyle>`
+主�是为了����回答 [#cnf1]_ 相�（或类似）的问题。
+
+管�风格是�常个人化的，比简�的编�风格规则更难以�化，因此本文档�能与实
+际情况有关，也�能与实际情况无关。起�它是一个玩笑，但这并��味�它�能�
+是真的。你得自己决定。
+
+顺便说一�，在谈到“核心管�者�时，主�是技术负责人，而�是在公�内部进行传
+统管�的人。如果你签署了采购订�或者对你的团队的预算有任何了解，你几乎肯定
+�是一个核心管�者。这些建议�能适用于您，也�能�适用于您。
+
+首先，我建议你购买“高效人的七个习惯�，而�是阅读它。烧了它，这是一个伟大的
+象�性姿�。
+
+.. [#cnf1] 本文件并�是通过回答问题，而是通过让�问者痛苦地明白，我们�知�
+   答案是什么。
+
+�管怎样，这里是：
+
+.. _decisions:
+
+1）决策
+-------
+
+�个人都认为管�者�决定，而且决策很��。决定越大越痛苦，管�者就必须越高级。
+这很明显，但事实并�如此。
+
+游�的�字是 **��** �出决定。尤其是，如果有人告诉你“选择（a）或（b），
+我们真的需�你��决定�，你就是陷入麻烦的管�者。你管�的人比你更了解细节，
+所以如果他们�找你�技术决策，你完蛋了。你显然没有能力为他们�这个决定。
+
+（推论：如果你管�的人�比你更了解细节，你也会被�砸，尽管原因完全��。
+也就是说，你的工作是错的，他们应该管�你的�智）
+
+所以游�的�字是 **��** �出决定，至少是那些大而痛苦的决定。�一些�的
+和�结果性的决定是很好的，并且使您看起�好�知�自己在�什么，所以内核管�者
+需��的是将那些大的和痛苦的决定��那些没有人真正关心的�事情。
+
+这有助于认识到一个大的决定和一个�的决定之间的关键区别是你是��以在事�修正
+你的决定。任何决定都�以通过始终确�如果你错了（而且你一定会错），你以�总是
+�以通过回溯�弥补�失。�然间，你就��两个无关紧�的决定，一个是错误的，�
+一个是正确的。
+
+人们甚至会认为这是真正的领导能力（咳，胡说，咳）。
+
+因此，���大决策的关键在于���那些无法挽回的事情。��被引导到一个你无法
+逃离的角�。走投无路的�鼠�能很�险——走投无路的管�者真�怜。
+
+事实�明，由于没有人会愚蠢到让内核管�者承担巨大的财政责任，所以通常很容易
+回溯。既然你��能浪费掉你无法�还的巨�资金，你唯一�以回溯的就是技术决策，
+而回溯很容易：��告诉大家你是个�称�的傻瓜，说对�起，然�撤销你去年让别
+人所�的毫无价值的工作。�然间，你一年��的决定�在是一个�大的决定，因为
+它很容易被推翻。
+
+事实�明，有些人对接�这�方法有困难，原因有两个：
+
+ - 承认你是个白痴比看起�更难。我们都喜欢��形象，在公共场�说你错了有时
+   确实很难。
+ - 如果有人告诉你，你去年所�的工作终究是�值得的，那么对那些�怜的低级工
+   程师�说也是很困难的，虽然实际的 **工作** 很容易删除，但你�能已���
+   挽回地失去了工程师的信任。记�：“��撤销�是我们一开始就试图��的，
+   而你的决定终究是一个�大的决定。
+
+令人欣慰的是，这两个原因都�以通过预先承认你没有任何线索，��告诉人们你的
+决定完全是�步的，而且�能是错误的事情�有效地缓解。你应该始终�留改�主�
+的�利，并让人们 **�识** 到这一点。当你 **还没有** �过真正愚蠢的事情的时
+候，承认自己是愚蠢的�容易得多。
+
+然�，当它真的被�明是愚蠢的时候，人们就转动他们的眼�说“哎呀，下次��了�。
+
+这�对�称�的先�制人的承认，也�能使真正�这项工作的人也会三�是�值得�。
+毕竟，如果他们�确定这是�是一个好主�，你肯定�应该通过�他们��他们所�
+的工作将会进入（内核）鼓励他们。在他们开始一项巨大的努力之�，至少让他们三
+�而�行。
+
+记�：他们最好比你更了解细节，而且他们通常认为他们对�件事都有答案。作为一
+个管�者，你能�的最好的事情�是�输自信，而是对他们所�的事情进行�康的批
+判性�考。
+
+顺便说一�，�一����出决定的方法是看起�很�怜的抱怨 “我们�能两者兼
+得�？� 相信我，它是有效的。如果�清楚哪�方法更好，他们最终会弄清楚的。
+最终的答案�能是两个团队都会因为这�情况而感到沮丧，以至于他们放弃了。
+
+这�起��是一个失败，但这通常是一个迹象，表明两个项目都有问题，而�与其中
+的人�能�决定的原因是他们都是错误的。你最终会闻到玫瑰的味�，你��了�一
+个你本�以�砸的决定。
+
+2）人
+-----
+
+大多数人都是白痴，�一�管�者�味�你必须处�好这件事，也许更��的是,
+**他们** 必须处�好你。
+
+事实�明，虽然很容易纠正技术错误，但�容易纠正人格障�。你�能和他们的和
+你的（人格障�）共处。
+
+但是，为了�好作为内核管�者的准备，最好记���烧掉任何桥�，��轰炸任何
+无辜的�民，也���远太多的内核开�人员。事实�明，�远人是相当容易的，而
+亲近一个�远的人是很难的。因此，“�远�立�属于“��逆�的范畴，并根�
+:ref:`decisions` �为���以�的事情。
+
+这里�有几个简�的规则：
+
+ (1) ���人笨蛋（至少��在公共场�）
+ (2) 学习如何在忘记规则(1)时�歉
+
+问题在于 #1 很容易去�，因为你�以用数百万���的方�说“你是一个笨蛋� [#cnf2]_
+有时甚至没有�识到，而且几乎总是带�一�白热化的信念，认为你是对的。
+
+你越确信自己是对的（让我们�对现实�，你�以把几乎所有人都称为�人，而且你
+�常是对的），事��歉就越难。
+
+�解决此问题，您实际上�有两个选项：
+
+ - �常擅长�歉
+ - 把“爱��匀地散开，没有人会真正感觉到自己被�公平地瞄准了。让它有足够的
+   创造性，他们甚至�能会觉得好笑。
+
+选择永远��礼貌是�存在的。没有人会相信一个如此明显地��了他们真实性格的人。
+
+.. [#cnf2] �罗·西蒙演唱了“离开爱人的50�方法�，因为�率地说，“告诉开�者
+   他们是D*CKHEAD" 的100万�方法都无法确认。但我确信他已�这么想了。
+
+3）人2 - 好人
+-------------
+
+虽然大多数人都是白痴，但�幸的是，�此推论你也是白痴，尽管我们都自我感觉良
+好，我们比普通人更好（让我们�对现实�，没有人相信他们是普通人或低于普通人），
+我们也应该承认我们�是最锋利的刀，而且会有其他人比你更��白痴。
+
+有些人对�明人�应�好。其他人利用它们。
+
+作为内核维护人员，确�您在第二组中。接�他们，因为他们会让你的工作更容易。
+特别是，他们能够为你�决定，这就是游�的全部内容。
+
+所以当你�现一个比你�明的人时，就顺其自然�。你的管��责在很大程度上��
+了“�起��是个好主�——去�试��，或者“�起��错，但是XXX呢？�“。第二个版
+本尤其是一个很好的方法，�么学习一些关于“XXX�的新东西，�么通过指出一些�明
+人没有想到的东西�显得更具管�性。无论哪�情况，你都会赢。
+
+�注�的一件事是认识到一个领域的伟大�一定会转化为其他领域。所以你�能会�
+特定的方�刺激人们，但让我们�对现实�，他们�能擅长他们所�的事情，而且对
+其他事情都很差劲。好消�是，人们往往会自然而然地�拾他们擅长的东西，所以当
+你��个方�刺激他们时，你并�是在���逆转的事情，�是��用力推。
+
+4）责备
+-------
+
+事情会出问题的，人们希望去责备人。贴标签，你就是�责备的人。
+
+事实上，接�责备并�难，尤其是当人们�识到这� **全是** 你的过错时。这让我
+们找到了承担责任的最佳方�：为别人承担这件事。你会感觉很好，他们会感觉很好，
+没有�到指责. 那�，失去了他们的全部36GB色情收�的人，因为你的无能将勉强承
+认，你至少没有试图逃�责任。
+
+然�让真正�砸了的开�人员（如果你能找到他们）�下知�他们�砸了。�仅是为
+了将��以��，而且为了让他们知�他们欠你一个人情。而且，也许更��的是，
+他们也�能是能够解决问题的人。因为，让我们�对现实�，肯定�是你。
+
+承担责任也是你首先�为管�者的原因。这是让人们信任你，让你获得潜在的�耀的
+一部分，因为你就是那个会说“我�砸了�的人。如果你已��循了以�的规则，你现
+在已�很擅长说了。
+
+5）应��的事情
+---------------
+
+有一件事人们甚至比被称为“笨蛋�更讨厌，那就是在一个神圣的声音中被称为“笨蛋�。
+第一个你�以�歉，第二个你�会真正得到机会。�使你�得很好，他们也�能��
+倾�。
+
+我们都认为自己比别人强，这�味�当别人装腔作势时，这会让我们很��。你也许
+在�德和智力上比你周围的�个人都优越，但��试图太明显，除�你真的打算激怒
+�人 [#cnf3]_
+
+�样，��对事情太客气或太微妙。礼貌很容易�得�花�水，把问题��起�，
+正如他们所说，“在互�网上，没人能�到你的�蓄。�用一个�器把这一点锤进去，
+因为你�能真的��别人�获得你的观点。
+
+一些幽默�以帮助缓和直率和�德化。过度到�谬的地步，�以�输一个观点，而�
+会让接�者感到痛苦，他们�是认为你是愚蠢的。因此，它�以帮助我们摆脱对批评
+的个人心�障�。
+
+.. [#cnf3] �示：与你的工作没有直接关系的网络新闻组是消除你对他人�满的好
+   方法。�尔写些侮辱性的帖�，打个喷�，让你的情绪得到净化。别把牢骚带回家
+
+6）为什么是我？
+---------------
+
+既然你的主�责任似乎是为别人的错误承担责任，并且让别人痛苦地明白你是�称�
+的，那么显而易�的问题之一就��了为什么首先�这样�。
+
+首先，虽然你�能会或�能�会�到�几�女孩（或男孩，让我们��在这里评判或
+性别歧视）敲你的更衣室门，你会得到一个巨大的个人�就感为“负责�。别介�你真
+的在领导别人，你�跟上别人，尽�能快地追赶他们。�个人都会认为你是负责人。
+
+如果你�以�到这个， 这是个伟大的工作�
diff --git a/Documentation/translations/zh_CN/process/programming-language.rst b/Documentation/translations/zh_CN/process/programming-language.rst
new file mode 100644
index 000000000000..51fd4ef48ea1
--- /dev/null
+++ b/Documentation/translations/zh_CN/process/programming-language.rst
@@ -0,0 +1,41 @@
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: :ref:`Documentation/process/programming-language.rst <programming_language>`
+:Translator: Alex Shi <alex.shi@linux.alibaba.com>
+
+.. _cn_programming_language:
+
+程�设计语言
+============
+
+内核是用C语言 [c-language]_ 编写的。更准确地说，内核通常是用 ``gcc`` [gcc]_
+在 ``-std=gnu89`` [gcc-c-dialect-options]_ 下编译的：ISO C90的 GNU 方言（
+包括一些C99特性）
+
+这�方言包�对语言 [gnu-extensions]_ 的许多扩展，当然，它们许多都在内核中使用。
+
+对于一些体系结构，有一些使用 ``clang`` [clang]_ 和 ``icc`` [icc]_ 编译内核
+的支�，尽管在编写此文档时还没有完�，�需�第三方补�。
+
+属性
+----
+
+在整个内核中使用的一个常�扩展是属性（attributes） [gcc-attribute-syntax]_
+属性�许将实现定义的语义引入语言实体（如���函数或类型），而无需对语言进行
+�大的语法更改（例如添加新关键字） [n2049]_
+
+在�些情况下，属性是�选的（��支�这些属性的编译器�然应该生�正确的代�，
+�使其速度较慢或执行的编译时检查/诊断次数�够）
+
+内核定义了伪关键字（例如， ``pure`` ），而�是直接使用GNU属性语法（例如,
+``__attribute__((__pure__))`` ），以检测�以使用哪些关键字和/或缩短代�, 具体
+请�阅 ``include/linux/compiler_attributes.h``
+
+.. [c-language] http://www.open-std.org/jtc1/sc22/wg14/www/standards
+.. [gcc] https://gcc.gnu.org
+.. [clang] https://clang.llvm.org
+.. [icc] https://software.intel.com/en-us/c-compilers
+.. [gcc-c-dialect-options] https://gcc.gnu.org/onlinedocs/gcc/C-Dialect-Options.html
+.. [gnu-extensions] https://gcc.gnu.org/onlinedocs/gcc/C-Extensions.html
+.. [gcc-attribute-syntax] https://gcc.gnu.org/onlinedocs/gcc/Attribute-Syntax.html
+.. [n2049] http://www.open-std.org/jtc1/sc22/wg14/www/docs/n2049.pdf
diff --git a/Documentation/translations/zh_CN/stable_api_nonsense.txt b/Documentation/translations/zh_CN/process/stable-api-nonsense.rst
index a2b27fab382c..b4ddb6e88d9d 100644
--- a/Documentation/translations/zh_CN/stable_api_nonsense.txt
+++ b/Documentation/translations/zh_CN/process/stable-api-nonsense.rst
@@ -1,26 +1,18 @@
-Chinese translated version of Documentation/process/stable-api-nonsense.rst
+.. _cn_stable_api_nonsense:
 
-If you have any comment or update to the content, please contact the
-original document maintainer directly.  However, if you have problem
-communicating in English you can also ask the Chinese maintainer for help.
-Contact the Chinese maintainer, if this translation is outdated or there
-is problem with translation.
+.. include:: ../disclaimer-zh_CN.rst
 
-Maintainer: Greg Kroah-Hartman <greg@kroah.com>
-Chinese maintainer: TripleX Chung <zhongyu@18mail.cn>
----------------------------------------------------------------------
-Documentation/process/stable-api-nonsense.rst 的中文翻译
+:Original: :ref:`Documentation/process/stable-api-nonsense.rst
+           <stable_api_nonsense>`
 
-如果想评论或更新本文的内容，请直接�系原文档的维护者。如果你使用英文
-交�有困难的�，也�以�中文版维护者求助。如果本翻译更新��时或者翻
-译存在问题，请�系中文版维护者。
+译者::
 
-英文版维护者： Greg Kroah-Hartman <greg@kroah.com>
-中文版维护者： 钟宇  TripleX Chung <zhongyu@18mail.cn>
-中文版翻译者： 钟宇  TripleX Chung <zhongyu@18mail.cn>
-中文版校译者： �阳  Li Yang <leoli@freescale.com>
-以下为正文
----------------------------------------------------------------------
+        中文版维护者： 钟宇  TripleX Chung <xxx.phy@gmail.com>
+        中文版翻译者： 钟宇  TripleX Chung <xxx.phy@gmail.com>
+        中文版校译者： �阳  Li Yang <leoyang.li@nxp.com>
+
+Linux 内核驱动接�
+==================
 
 写作本文档的目的，是为了解释为什么Linux既没有二进制内核接�，也没有稳定
 的内核接�。这里所说的内核接�，是指内核里的接�，而�是内核和用户空间
@@ -59,18 +51,22 @@ Linux能�为强壮，稳定，�熟的�作系统，这也是你最开始选
 --------------
 �如我们有一个稳定的内核�代�接�，那么自然而然的，我们就拥有了稳定的
 二进制接�，是这样的�？错。让我们看看关于Linux内核的几点事实：
+
     - �决于所用的C编译器的版本，��的内核数�结构里的结构体的对�方
-�会有差别，代�中��函数的表现形�也�一样（函数是�是被inline编译�
-决于编译器行为）。��的函数的表现形�并���，但是数�结构内部的对�
-方�很关键。
+      �会有差别，代�中��函数的表现形�也�一样（函数是�是被inline
+      编译�决于编译器行为）。��的函数的表现形�并���，但是数�
+      结构内部的对�方�很关键。
+
     - �决于内核的�置选项，��的选项会让内核的很多东西�生改�：
+
       - �一个结构体�能包���的�员��
       - 有的函数�能根本�会被实现（比如编译的时候没有选择SMP支�
-，一些�函数就会被定义�空函数）。
+        一些�函数就会被定义�空函数）。
       - 内核使用的内存会以��的方�对�，这�决于��的内核�置选
-项。
+        项。
+
     - Linux�以在很多的��体系结构的处�器上�行。在�个体系结构上编
-译好的二进制驱动程�，��能在�外一个体系结构上正确的�行。
+      译好的二进制驱动程�，��能在�外一个体系结构上正确的�行。
 
 对于一个特定的内核，满足这些�件并�难，使用�一个C编译器和�样的内核�
 置选项�编译驱动程�模�就�以了。这对于给一个特定Linux�布的特定版本�
@@ -90,7 +86,7 @@ Linux能�为强壮，稳定，�熟的�作系统，这也是你最开始选
 
 如果有人�将他的内核驱动程�，放入公版内核的�代�树，而�想让驱动程�
 一直��在最新的内核中�用，那么这个�题将会�得没完没了。
- 内核开�是�续而且快节�的，从�都�会慢下�。内核开�人员在当�接�中
+内核开�是�续而且快节�的，从�都�会慢下�。内核开�人员在当�接�中
 找到bug，或者找到更好的实现方�。一旦�现这些，他们就很快会去修改当�的
 接�。修改接��味�，函数��能会改�，结构体�能被扩充或者删�，函数
 的�数也�能�生改�。一旦接�被修改，内核中使用这些接�的地方需��时
@@ -98,21 +94,22 @@ Linux能�为强壮，稳定，�熟的�作系统，这也是你最开始选
 
 举一个例�，内核的USB驱动程�接�在USB�系统的整个生命周期中，至少�历
 了三次�写。这些�写解决以下问题：
+
     - 把数��从�步模�改���步模�，这个改动�少了一些驱动程�的
-��度，�高了所有USB驱动程�的��率，这样几乎所有的USB设备都能以最大
-速率工作了。
+      ��度，�高了所有USB驱动程�的��率，这样几乎所有的USB设备都
+      能以最大速率工作了。
     - 修改了USB核心代�中为USB驱动分�数�包内存的方�，所有的驱动都
-需��供更多的�数给USB核心，以修正了很多已�被记录在案的死�。
+      需��供更多的�数给USB核心，以修正了很多已�被记录在案的死�。
 
 这和一些�闭�代�的�作系统形�鲜明的对比，在那些�作系统上，�得��
 外的维护旧的USB接�。这导致了一个�能性，新的开�者�然会��心使用旧的
 接�，以��当的方�编写代�，进而影�到�作系统的稳定性。
- 在上�的例�中，所有的开�者都��这些��的改动，在这样的情况下修改代
+在上�的例�中，所有的开�者都��这些��的改动，在这样的情况下修改代
 价很低。如果Linux��一个稳定的内核�代�接�，那么就得创建一个新的接�
 ；旧的，有问题的接�必须一直维护，给Linux USB开�者带��外的工作。既然
 所有的Linux USB驱动的作者都是利用自己的时间工作，那么�求他们去�毫无�
 义的�费�外工作，是��能的。
- 安全问题对Linux�说�分��。一个安全问题被�现，就会在短时间内得到修
+安全问题对Linux�说�分��。一个安全问题被�现，就会在短时间内得到修
 正。在很多情况下，这将导致Linux内核中的一些接�被�写，以从根本上��安
 全问题。一旦接�被�写，所有使用这些接�的驱动程�，必须�时得到修正，
 以确定安全问题已�得到修�并且��能在未�还有�样的安全问题。如果内核
@@ -124,7 +121,7 @@ Linux能�为强壮，稳定，�熟的�作系统，这也是你最开始选
 
 
 ��什么
--------
+--------
 
 如果你写了一个Linux内核驱动，但是它还�在Linux�代�树里，作为一个开�
 者，你应该怎么�？为�个�布的�个版本�供一个二进制驱动，那简直是一个
@@ -137,20 +134,21 @@ Linux能�为强壮，稳定，�熟的�作系统，这也是你最开始选
 �什么事情。
 
 把驱动放到内核�代�树里会有很多的好处：
+
     - 驱动的质�会��，而维护�本（对原始作者�说）会下�。
     - 其他人会给驱动添加新特性。
     - 其他人会找到驱动中的bug并修�。
     - 其他人会在驱动中找到性能优化的机会。
     - 当外部的接�的改�需�修改驱动程�的时候，其他人会修改驱动程�
-。
     - �需��系任何�行商，这个驱动会自动的��所有的Linux�布一起�
-布。
+      布。
 
 和别的�作系统相比，Linux为更多��的设备�供现�的驱动，而且能在更多�
 �体系结构的处�器上支�这些设备。这个�过考验的开�模�，必然是错�了
 çš„ :)
 
--------------
+æ„Ÿè°¢
+----
 æ„Ÿè°¢ Randy Dunlap, Andrew Morton, David Brownell, Hanna Linder,
 Robert Love, and Nishanth Aravamudan 对于本文档早期版本的评审和建议。
 
diff --git a/Documentation/translations/zh_CN/stable_kernel_rules.txt b/Documentation/translations/zh_CN/process/stable-kernel-rules.rst
index db4ba5a0c39a..fba361f2ddfd 100644
--- a/Documentation/translations/zh_CN/stable_kernel_rules.txt
+++ b/Documentation/translations/zh_CN/process/stable-kernel-rules.rst
@@ -1,31 +1,26 @@
-Chinese translated version of Documentation/process/stable-kernel-rules.rst
+.. _cn_stable_kernel_rules:
 
-If you have any comment or update to the content, please contact the
-original document maintainer directly.  However, if you have a problem
-communicating in English you can also ask the Chinese maintainer for
-help.  Contact the Chinese maintainer if this translation is outdated
-or if there is a problem with the translation.
+.. include:: ../disclaimer-zh_CN.rst
 
-Chinese maintainer: TripleX Chung <triplex@zh-kernel.org>
----------------------------------------------------------------------
-Documentation/process/stable-kernel-rules.rst 的中文翻译
+:Original: :ref:`Documentation/process/stable-kernel-rules.rst <stable_kernel_rules>`
 
 如果想评论或更新本文的内容，请直接�系原文档的维护者。如果你使用英文
 交�有困难的�，也�以�中文版维护者求助。如果本翻译更新��时或者翻
-译存在问题，请�系中文版维护者。
+译存在问题，请�系中文版维护者::
 
+        中文版维护者： 钟宇  TripleX Chung <xxx.phy@gmail.com>
+        中文版翻译者： 钟宇  TripleX Chung <xxx.phy@gmail.com>
+        中文版校译者：
+            - �阳  Li Yang <leoyang.li@nxp.com>
+            - Kangkai Yin <e12051@motorola.com>
 
-中文版维护者： 钟宇  TripleX Chung <triplex@zh-kernel.org>
-中文版翻译者： 钟宇  TripleX Chung <triplex@zh-kernel.org>
-中文版校译者： �阳  Li Yang <leo@zh-kernel.org>
-               Kangkai Yin <e12051@motorola.com>
-
-以下为正文
----------------------------------------------------------------------
+所有你想知�的事情 - 关于linux稳定版�布
+========================================
 
 关于Linux 2.6稳定版�布，所有你想知�的事情。
 
 关于哪些类型的补��以被接收进入稳定版代�树，哪些��以的规则：
+----------------------------------------------------------------
 
   - 必须是显而易�的正确，并且�过测试的。
   - 连�上下文，�能大于100行。
@@ -38,9 +33,10 @@ Documentation/process/stable-kernel-rules.rst 的中文翻译
   - 没有“�论上的竞争�件�，除�能给出竞争�件如何被利用的解释。
   - �能存在任何的“�碎的�修正（拼写修正，去掉多余空格之类的）。
   - 必须被相关�系统的维护者接�。
-  - 必须�循Documentation/process/submitting-patches.rst里的规则。
+  - 必须�循Documentation/translations/zh_CN/process/submitting-patches.rst里的规则。
 
 �稳定版代�树�交补�的过程：
+------------------------------
 
   - 在确认了补�符�以上的规则�，将补���到stable@vger.kernel.org。
   - 如果补�被接�到队列里，��者会收到一个ACK回�，如果没有被接�，收
@@ -49,6 +45,7 @@ Documentation/process/stable-kernel-rules.rst 的中文翻译
   - 安全方�的补����到这个列表，应该��到security@kernel.org。
 
 审查周期：
+----------
 
   - 当稳定版的维护者决定开始一个审查周期，补�将被��到审查委员会，以
     �被补�影�的领域的维护者（除��交者就是该领域的维护者）并且抄�
@@ -63,4 +60,5 @@ Documentation/process/stable-kernel-rules.rst 的中文翻译
     通常的审查周期。请�系内核安全�组以获得关于这个过程的更多细节。
 
 审查委员会：
+------------
   - 由一些自愿承担这项任务的内核开�者，和几个�志愿的组�。
diff --git a/Documentation/translations/zh_CN/process/submit-checklist.rst b/Documentation/translations/zh_CN/process/submit-checklist.rst
new file mode 100644
index 000000000000..89061aa8fdbe
--- /dev/null
+++ b/Documentation/translations/zh_CN/process/submit-checklist.rst
@@ -0,0 +1,107 @@
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: :ref:`Documentation/process/submit-checklist.rst <submitchecklist>`
+:Translator: Alex Shi <alex.shi@linux.alibaba.com>
+
+.. _cn_submitchecklist:
+
+Linux内核补��交清�
+~~~~~~~~~~~~~~~~~~~~~
+
+如果开�人员希望看到他们的内核补��交更快地被接�，那么他们应该�一些基本
+的事情。
+
+这些都是在
+:ref:`Documentation/translations/zh_CN/process/submitting-patches.rst <cn_submittingpatches>`
+和其他有关�交Linux内核补�的文档中�供的。
+
+1) 如果使用工具，则包括定义/声明该工具的文件。���赖于其他头文件拉入您使用
+   的头文件。
+
+2) 干净的编译：
+
+   a) 使用适用或修改的 ``CONFIG`` 选项 ``=y``�``=m`` 和 ``=n`` 。没有GCC
+      警告/错误，没有链接器警告/错误。
+
+   b) 通过allnoconfig�allmodconfig
+
+   c) 使用 ``O=builddir`` 时�以�功编译
+
+3) 通过使用本地交�编译工具或其他一些构建场在多个CPU体系结构上构建。
+
+4) PPC64是一�很好的交�编译检查体系结构，因为它倾�于对64�的数使用无符�
+   长整型。
+
+5) 如下所述 :ref:`Documentation/translations/zh_CN/process/coding-style.rst <cn_codingstyle>`.
+   检查您的补�是�为常规样�。在�交（ ``scripts/check patch.pl`` ）之�，
+   使用补�样�检查器检查是�有轻微的冲�。您应该能够处�您的补�中存在的所有
+   �规行为。
+
+6) 任何新的或修改过的 ``CONFIG`` 选项都�会弄��置��，并默认为关闭，除�
+   它们符� ``Documentation/kbuild/kconfig-language.txt`` 中记录的异常�件,
+   ��属性：默认值.
+
+7) 所有新的 ``kconfig`` 选项都有帮助文本。
+
+8) 已仔细审查了相关的 ``Kconfig`` 组�。这很难用测试�纠正——脑力在这里是有
+   回报的。
+
+9) 用 sparse 检查干净。
+
+10) 使用 ``make checkstack`` 和 ``make namespacecheck`` 并修�他们�现的任何
+    问题。
+
+    .. note::
+
+        ``checkstack`` 并没有明确指出问题，但是任何一个在堆栈上使用超过512
+        字节的函数都�以进行更改。
+
+11) 包括 :ref:`kernel-doc <kernel_doc>` 内核文档以记录全局内核API。（��函数
+    �需�，但也�以。）使用 ``make htmldocs`` 或 ``make pdfdocs`` 检查
+    :ref:`kernel-doc <kernel_doc>` 并修�任何问题。
+
+12) 通过以下选项�时�用的测试 ``CONFIG_PREEMPT``, ``CONFIG_DEBUG_PREEMPT``,
+    ``CONFIG_DEBUG_SLAB``, ``CONFIG_DEBUG_PAGEALLOC``, ``CONFIG_DEBUG_MUTEXES``,
+    ``CONFIG_DEBUG_SPINLOCK``, ``CONFIG_DEBUG_ATOMIC_SLEEP``,
+    ``CONFIG_PROVE_RCU`` and ``CONFIG_DEBUG_OBJECTS_RCU_HEAD``
+
+13) 已�过构建和�行时测试，包括有或没有 ``CONFIG_SMP``, ``CONFIG_PREEMPT``.
+
+14) 如果补�程�影�IO/�盘等：使用或�使用 ``CONFIG_LBDAF`` 进行测试。
+
+15) 所有代�路径都已在�用所有lockdep功能的情况下�行。
+
+16) 所有新的/proc�目都记录在 ``Documentation/``
+
+17) 所有新的内核引导�数都记录在
+    Documentation/admin-guide/kernel-parameters.rst 中。
+
+18) 所有新的模��数都记录在 ``MODULE_PARM_DESC()``
+
+19) 所有新的用户空间接�都记录在 ``Documentation/ABI/`` 中。有关详细信�，
+    请�阅 ``Documentation/ABI/README`` 。更改用户空间接�的补�应该抄�
+    linux-api@vger.kernel.org。
+
+20) 检查是�全部通过 ``make headers_check`` 。
+
+21) 已通过至少注入slab和page分�失败进行检查。请�阅 ``Documentation/fault-injection/``
+    如果新代�是实质性的，那么添加�系统特定的故障注入�能是�适的。
+
+22) 新添加的代�已�用 ``gcc -W`` 编译（使用 ``make EXTRA-CFLAGS=-W`` ）。这
+    将产生大�噪声，但对于查找诸如“警告：有符�和无符�之间的比较�之类的错误
+    很有用。
+
+23) 在它被�并到-mm补�集中之�进行测试，以确�它�然与所有其他排队的补�以
+    �VM�VFS和其他�系统中的��更改一起工作。
+
+24) 所有内存�障例如 ``barrier()``, ``rmb()``, ``wmb()`` 都需��代�中的注
+    释�解释它们正在执行的�作�其原因的逻辑。
+
+25) 如果补�添加了任何ioctl，那么也�更新 ``Documentation/ioctl/ioctl-number.txt``
+
+26) 如果修改�的�代��赖或使用与以下 ``Kconfig`` 符�相关的任何内核API或
+    功能，则在�用相关 ``Kconfig`` 符�和/或 ``=m`` （如果该选项�用）的情况
+    下测试以下多个构建[并�所有这些都�时存在，�是它们的��/�机组�]：
+
+    ``CONFIG_SMP``, ``CONFIG_SYSFS``, ``CONFIG_PROC_FS``, ``CONFIG_INPUT``, ``CONFIG_PCI``, ``CONFIG_BLOCK``, ``CONFIG_PM``, ``CONFIG_MAGIC_SYSRQ``,
+    ``CONFIG_NET``, ``CONFIG_INET=n`` (但是�者伴� ``CONFIG_NET=y``).
diff --git a/Documentation/translations/zh_CN/SubmittingDrivers b/Documentation/translations/zh_CN/process/submitting-drivers.rst
index 15e73562f710..72c6cd935821 100644
--- a/Documentation/translations/zh_CN/SubmittingDrivers
+++ b/Documentation/translations/zh_CN/process/submitting-drivers.rst
@@ -1,36 +1,28 @@
-Chinese translated version of Documentation/process/submitting-drivers.rst
+.. _cn_submittingdrivers:
 
-If you have any comment or update to the content, please contact the
-original document maintainer directly.  However, if you have a problem
-communicating in English you can also ask the Chinese maintainer for
-help.  Contact the Chinese maintainer if this translation is outdated
-or if there is a problem with the translation.
+.. include:: ../disclaimer-zh_CN.rst
 
-Chinese maintainer: Li Yang <leo@zh-kernel.org>
----------------------------------------------------------------------
-Documentation/process/submitting-drivers.rst 的中文翻译
+:Original: :ref:`Documentation/process/submitting-drivers.rst
+           <submittingdrivers>`
 
 如果想评论或更新本文的内容，请直接�系原文档的维护者。如果你使用英文
 交�有困难的�，也�以�中文版维护者求助。如果本翻译更新��时或者翻
-译存在问题，请�系中文版维护者。
+译存在问题，请�系中文版维护者::
 
-中文版维护者： �阳  Li Yang <leo@zh-kernel.org>
-中文版翻译者： �阳  Li Yang <leo@zh-kernel.org>
-中文版校译者： 陈� Maggie Chen <chenqi@beyondsoft.com>
-               王� Wang Cong <xiyou.wangcong@gmail.com>
-               å¼ å·� Zhang Wei <Wei.Zhang@freescale.com>
-
-以下为正文
----------------------------------------------------------------------
+        中文版维护者： �阳  Li Yang <leoyang.li@nxp.com>
+        中文版翻译者： �阳  Li Yang <leoyang.li@nxp.com>
+        中文版校译者： 陈� Maggie Chen <chenqi@beyondsoft.com>
+                       王� Wang Cong <xiyou.wangcong@gmail.com>
+                       å¼ å·� Zhang Wei <wezhang@outlook.com>
 
 如何� Linux 内核�交驱动程�
------------------------------
+=============================
 
 这篇文档将会解释如何���的内核��树�交设备驱动程�。请注�，如果你感
 兴趣的是显�驱动程�，你也许应该访问 XFree86 项目(http://www.xfree86.org/)
 和�或 X.org 项目 (http://x.org)。
 
-�请�阅 Documentation/process/submitting-patches.rst 文档。
+�请�阅 Documentation/Documentation/translations/zh_CN/process/submitting-patches.rst 文档。
 
 
 分�设备�
@@ -145,9 +137,13 @@ Linux 设备驱动程�，第三版（探讨 2.6.10 版内核）：
 
 LWN.net:
 	�周内核开�活动摘� - http://lwn.net/
+
 	2.6 版中 API 的�更：
+
 		http://lwn.net/Articles/2.6-kernel-api/
+
 	将旧版内核的驱动程�移�到 2.6 版：
+
 		http://lwn.net/Articles/driver-porting/
 
 内核新手(KernelNewbies):
diff --git a/Documentation/translations/zh_CN/process/submitting-patches.rst b/Documentation/translations/zh_CN/process/submitting-patches.rst
new file mode 100644
index 000000000000..437c23b367bb
--- /dev/null
+++ b/Documentation/translations/zh_CN/process/submitting-patches.rst
@@ -0,0 +1,682 @@
+.. _cn_submittingpatches:
+
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: :ref:`Documentation/process/submitting-patches.rst <submittingpatches>`
+
+译者::
+
+        中文版维护者： 钟宇 TripleX Chung <xxx.phy@gmail.com>
+        中文版翻译者： 钟宇 TripleX Chung <xxx.phy@gmail.com>
+                       时奎亮 Alex Shi <alex.shi@linux.alibaba.com>
+        中文版校译者： �阳 Li Yang <leoyang.li@nxp.com>
+                       王� Wang Cong <xiyou.wangcong@gmail.com>
+
+
+如何让你的改动进入内核
+======================
+
+对于想�将改动�交到 Linux 内核的个人或者公��说，如果�熟悉“规矩�，
+�交的�程会让人�惧。本文档收集了一系列建议，这些建议�以大大的�高你
+的改动被接�的机会.
+
+以下文档�有大�简�的建议， 具体请�：
+:ref:`Documentation/process <development_process_main>`
+�样，:ref:`Documentation/translations/zh_CN/process/submit-checklist.rst <cn_submitchecklist>`
+给出在�交代��需�检查的项目的列表。如果你在�交一个驱动程�，那么
+�时阅读一下:
+:ref:`Documentation/process/submitting-drivers.rst <submittingdrivers>`
+
+其中许多步骤�述了Git版本控制系统的默认行为；如果您使用Git�准备补�，
+您将�现它为您完�的大部分机械工作，尽管您�然需�准备和记录一组��的
+补�。一般�说，使用git将使您作为内核开�人员的生活更轻�。
+
+
+0) 获�当���树
+-----------------
+
+如果您没有一个�以使用当�内核�代�的存储库，请使用git获�一个。您将�
+从主线存储库开始，它�以通过以下方�获�::
+
+        git clone git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
+
+但是，请注�，您�能�希望直接针对主线树进行开�。大多数�系统维护人员�
+行自己的树，并希望看到针对这些树准备的补�。请��MAINTAINERS文件中�系
+统的 **T:** 项以查找该树，或者简�地询问维护者该树是�未在其中列出。
+
+�然�以通过tarballs下载内核版本（如下一节所述），但这是进行内核开�的
+一�困难的方�。
+
+1) "diff -up"
+-------------
+
+使用 "diff -up" 或者 "diff -uprN" �创建补�。
+
+所有内核的改动，都是以补�的形�呈现的，补�由 diff(1) 生�。创建补�的
+时候，�确认它是以 "unified diff" 格�创建的，这�格�由 diff(1) 的 '-u'
+�数生�。而且，请使用 '-p' �数，那样会显示�个改动所在的C函数，使得
+产生的补�容易读得多。补�应该基于内核�代�树的根目录，而�是里边的任
+何�目录。
+
+为一个�独的文件创建补�，一般�说这样�就够了::
+
+        SRCTREE=linux
+        MYFILE=drivers/net/mydriver.c
+
+        cd $SRCTREE
+        cp $MYFILE $MYFILE.orig
+        vi $MYFILE      # make your change
+        cd ..
+        diff -up $SRCTREE/$MYFILE{.orig,} > /tmp/patch
+
+为多个文件创建补�，你�以解开一个没有修改过的内核�代�树，然�和你自
+己的代�树之间� diff 。例如::
+
+        MYSRC=/devel/linux
+
+        tar xvfz linux-3.19.tar.gz
+        mv linux-3.19 linux-3.19-vanilla
+        diff -uprN -X linux-3.19-vanilla/Documentation/dontdiff \
+                linux-3.19-vanilla $MYSRC > /tmp/patch
+
+"dontdiff" 是内核在编译的时候产生的文件的列表，列表中的文件在 diff(1)
+产生的补�里会被跳过。
+
+确定你的补�里没有包�任何�属于这次补��交的�外文件。记得在用diff(1)
+生�补�之�，审阅一次补�，以确�准确。
+
+如果你的改动很散乱，你应该研究一下如何将补�分割�独立的部分，将改动分
+割�一系列�乎逻辑的步骤。这样更容易让其他内核开�者审核，如果你想你的
+补�被接�，这是很��的。请�阅：
+:ref:`cn_split_changes`
+
+如果你用 ``git`` , ``git rebase -i`` �以帮助你这一点。如果你�用 ``git``,
+``quilt`` <http://savannah.nongnu.org/projects/quilt> �外一个�行的选择。
+
+.. _cn_describe_changes:
+
+2) �述你的改动
+---------------
+
+�述你的问题。无论您的补�是一行错误修�还是5000行新功能，都必须有一个潜在
+的问题激励您完�这项工作。让审稿人相信有一个问题值得解决，让他们读完第一段
+是有�义的。
+
+�述用户��的影�。直接崩溃和�定是相当有说�力的，但并�是所有的错误都那么
+明目张胆。�使在代�审查期间�现了这个问题，也��述一下您认为它�能对用户产
+生的影�。请记�，大多数Linux安装�行的内核�自二级稳定树或特定于供应商/产�
+的树，�从上游精选特定的补�，因此请包�任何�以帮助您将更改定�到下游的内容：
+触�的场景�DMESG的摘录�崩溃�述�性能回归�延迟尖峰��定等。
+
+�化优化和�衡。如果您声称在性能�内存消耗�堆栈�用空间或二进制大�方�有所
+改进，请包括支�它们的数字。但也��述�明显的�本。优化通常�是�费的，而是
+在CPU�内存和�读性之间进行�衡；或者，探索性的工作，在��的工作负载之间进
+行�衡。请�述优化的预期缺点，以便审阅者�以�衡�本和收益。
+
+一旦问题建立起�，就�详细地�述一下您实际在�什么。对于审阅者�说，用简�的
+英语�述代�的�化是很��的，以验�代�的行为是�符�您的�愿。
+
+如果您将补��述写在一个表�中，这个表��以很容易地作为“�交日志�放入Linux
+的�代�管�系统git中，那么维护人员将�常感谢您。� :ref:`cn_explicit_in_reply_to`.
+
+�个补��解决一个问题。如果你的�述开始�长，这就表明你�能需�拆分你的补�。
+请� :ref:`cn_split_changes`
+
+�交或�新�交修补程�或修补程�系列时，请包括完整的修补程�说明和�由。��
+�说这是补�（系列）的第几版。��期望�系统维护人员引用更早的补�版本或引用
+URL�查找补��述并将其放入补�中。也就是说，补�（系列）�其�述应该是独立的。
+这对维护人员和审查人员都有好处。一些评审者�能甚至没有收到补�的早期版本。
+
+�述你在命令语气中的�化，例如“make xyzzy do frotz�而�是“[这个补�]make
+xyzzy do frotz�或“[我]changed xyzzy to do frotz�，就好�你在命令代�库改�
+它的行为一样。
+
+如果修补程�修�了一个记录的bug�目，请按编�和URL引用该bug�目。如果补��
+自邮件列表讨论，请给出邮件列表存档的URL；使用带有 ``Message-ID`` 的
+https://lkml.kernel.org/ �定�，以确�链接�会过时。
+
+但是，在没有外部资�的情况下，尽�让你的解释��解。除了�供邮件列表存档或
+bug的URL之外，还�总结需��交补�的相关讨论�点。
+
+如果您想�引用一个特定的�交，���引用�交的 SHA-1 ID。还请包括�交的一行
+摘�，以便于审阅者了解它是关于什么的。例如::
+
+        Commit e21d2170f36602ae2708 ("video: remove unnecessary
+        platform_set_drvdata()") removed the unnecessary
+        platform_set_drvdata(), but left the variable "dev" unused,
+        delete it.
+
+您还应该确�至少使用�12� SHA-1 ID. 内核存储库包�*许多*对象，使与较短的ID
+�生冲�的�能性很大。记�，�使现在�会与您的六个字符ID�生冲�，这�情况
+�能五年�改�。
+
+如果修补程�修�了特定�交中的错误，例如，使用 ``git bisct`` ，请使用带有�
+12个字符SHA-1 ID 的"Fixes:"标记和�行摘�。为了简化��将标记拆分为多个，
+行�标记��分�脚本“75列�行�规则的�制。例如::
+
+        Fixes: 54a4f0239f2e ("KVM: MMU: make kvm_mmu_zap_page() return the number of pages it actually freed")
+
+下列 ``git config`` 设置�以添加让 ``git log``, ``git show`` 漂亮的显示格�::
+
+	[core]
+		abbrev = 12
+	[pretty]
+		fixes = Fixes: %h (\"%s\")
+
+.. _cn_split_changes:
+
+3) 拆分你的改动
+---------------
+
+将�个逻辑更改分隔�一个�独的补�。
+
+例如，如果你的改动里�时有bug修正和性能优化，那么把这些改动拆分到两个或
+者更多的补�文件中。如果你的改动包�对API的修改，并且修改了驱动程��适
+应这些新的API，那么把这些修改分�两个补�。
+
+�一方�，如果你将一个�独的改动��多个补�文件，那么将它们�并�一个
+�独的补�文件。这样一个逻辑上�独的改动�被包�在一个补�文件里。
+
+如果有一个补��赖�外一个补��完�它的改动，那没问题。简�的在你的补
+��述里指出“这个补��赖�补��就好了。
+
+在将您的更改划分为一系列补�时，�特别注�确�内核在系列中的�个补�之�
+都能正常构建和�行。使用 ``git bisect`` �追踪问题的开�者�能会在任何时
+候分割你的补�系列；如果你在中间引入错误，他们�会感谢你。
+
+如果你�能将补�浓缩�更少的文件，那么�次大约��出15个，然�等待审查
+和集�。
+
+4) 检查你的更改风格
+-------------------
+
+检查您的补�是�存在基本样�冲�，详细信��在
+:ref:`Documentation/translations/zh_CN/process/coding-style.rst <cn_codingstyle>`
+中找到。如果�这样�，�会浪费审稿人的时间，并且会导致你的补�被拒�，甚至
+�能没有被阅读。
+
+一个��的例外是在将代�从一个文件移动到�一个文件时——在这�情况下，您�应
+该在移动代�的�一个补�中修改移动的代�。这清楚地�述了移动代�和您的更改
+的行为。这大大有助于审查实际差异，并�许工具更好地跟踪代�本身的历�。
+
+在�交之�，使用补�样�检查程�检查补�（scripts/check patch.pl）。�过，
+请注�，样�检查程�应该被视为一个指�，而�是作为人类判断的替代�。如果您
+的代�看起�更好，但有�规行为，那么最好��使用它。
+
+检查者报告三个级别：
+
+ - ERROR：很�能出错的事情
+ - WARNING：需�仔细审查的事项
+ - CHECK：需��考的事情
+
+您应该能够判断您的补�中存在的所有�规行为。
+
+5) 选择补�收件人
+-----------------
+
+您应该总是在任何补�上�制相应的�系统维护人员，以获得他们维护的代�；查看
+维护人员文件和�代�修订历�记录，以了解这些维护人员是�。脚本
+scripts/get_Maintainer.pl在这个步骤中�常有用。如果您找�到正在工作的�系统
+的维护人员，那么Andrew Morton（akpm@linux-foundation.org）将充当最�的维护
+人员。
+
+您通常还应该选择至少一个邮件列表�接收补�集的。linux-kernel@vger.kernel.org
+作为最�一个解决办法的列表，但是这个列表上的体积已�引起了许多开�人员的拒�。
+在MAINTAINERS文件中查找�系统特定的列表；您的补��能会在那里得到更多的关注。
+�过，请����垃圾邮件到无关的列表。
+
+许多与内核相关的列表托管在vger.kernel.org上；您�以在
+http://vger.kernel.org/vger-lists.html 上找到它们的列表。�过，也有与内核相关
+的列表托管在其他地方。
+
+��一次��超过15个补�到vger邮件列表����
+
+Linus Torvalds 是决定改动能�进入 Linux 内核的最终�决者。他的 e-mail
+地�是 <torvalds@linux-foundation.org> 。他收到的 e-mail 很多，所以一般
+的说，最好别给他� e-mail。
+
+如果您有修��利用安全�洞的补�，请将该补���到 security@kernel.org。对于
+严�的bug，�以考虑短期暂�以�许分销商�用户�布补�；在这�情况下，显然�应
+将补���到任何公共列表。
+
+修�已�布内核中严�错误的补�程�应该指�稳定版维护人员，方法是放这样的一行::
+
+        Cc: stable@vger.kernel.org
+
+进入补�的签准区（注�，�是电�邮件收件人）。除了这个文件之外，您还应该阅读
+:ref:`Documentation/process/stable-kernel-rules.rst <stable_kernel_rules>`
+
+但是，请注�，一些�系统维护人员希望得出他们自己的结论，�哪些补�应该被放到
+稳定的树上。尤其是网络维护人员，�希望看到�个开�人员在补�中添加�上�这样
+的行。
+
+如果更改影�到用户和内核接�，请�手册页维护人员（如维护人员文件中所列）��
+手册页补�，或至少��更改通知，以便一些信�进入手册页。还应将用户空间API
+更改�制到 linux-api@vger.kernel.org。
+
+对于�的补�，你也许会CC到�集�碎补�的邮件列表(Trivial Patch Monkey)
+trivial@kernel.org，那里专门收集�碎的补�。下�这样的补�会被看作“�碎的�
+补�：
+
+ - 文档的拼写修正。
+ - 修正会影�到 grep(1) 的拼写。
+ - 警告信�修正(频�的打�无用的警告是�好的。)
+ - 编译错误修正（代�逻辑的确是对的，�是编译有问题。）
+ - �行时修正（��真的修正了错误。）
+ - 移除使用了被废弃的函数/�的代�(例如 check_region。)
+ - �系方�和文档修正。
+ - 用�移�的代�替���移�的代�（�使在体系结构相关的代�中，既然有
+ - 人拷�，��它是�碎的）
+ - 任何文件的作者/维护者对该文件的改动（例如 patch monkey 在�传模�下）
+
+(译注，关于“�碎补��的一些说明：因为原文的这一部分写得比较简�，所以�得�
+�例写一下译注。"trivial"这个英文��的本�是“�碎的，���的。�但是在这里
+有�微有一些�化，例如对一些明显的NULL指针的修正，属于�行时修正，会被归类
+到�碎补�里。虽然NULL指针的修正很��，但是这样的修正往往很�而且很容易得到
+检验，所以也被归入�碎补�。�碎补�更精确的归类应该是
+“simple, localized & easy to verify�，也就是说简�的，局部的和易于检验的。
+trivial@kernel.org邮件列表的目的是针对这样的补�，为�交者�供一个中心，�
+�低�交的门槛。)
+
+6) 没有 MIME 编�，没有链接，没有压缩，没有附件，�有纯文本
+-----------------------------------------------------------
+
+Linus 和其他的内核开�者需�阅读和评论你�交的改动。对于内核开�者�说
+，�以“引用�你的改动很��，使用一般的 e-mail 工具，他们就�以在你的
+代�的任何�置添加评论。
+
+因为这个原因，所有的�交的补�都是 e-mail 中“内嵌�的。
+
+.. warning::
+   如果你使用剪切-粘贴你的补�，�心你的编辑器的自动�行功能破�你的补�
+
+��将补�作为 MIME 编�的附件，�管是�压缩。很多�行的 e-mail 软件�
+是任何时候都将 MIME 编�的附件当作纯文本��的，这会使得别人无法在你的
+代�中加评论。�外，MIME 编�的附件会让 Linus 多花一点时间�处�，这就
+�低了你的改动被接�的�能性。
+
+例外：如果你的邮递员弄�了补�，那么有人�能会�求你使用mime�新��补�
+
+请�阅 :ref:`Documentation/translations/zh_CN/process/email-clients.rst <cn_email_clients>`
+以获�有关�置电�邮件客户端以使其��影�地��修补程�的�示。
+
+7) e-mail 的大�
+----------------
+
+大的改动对邮件列表��适，对�些维护者也��适。如果你的补�，在�压缩
+的情况下，超过了300kB，那么你最好将补�放在一个能通过 internet 访问的�
+务器上，然�用指�你的补�的 URL 替代。但是请注�，如果您的补�超过了
+300kb，那么它几乎肯定需�被破�。
+
+8）回�评审��
+---------------
+
+你的补�几乎肯定会得到评审者对补�改进方法的评论。您必须对这些评论作出
+回应；让补�被忽略的一个好办法就是忽略审阅者的��。�会导致代�更改的
+��或问题几乎肯定会带�注释或�更日志的改�，以便下一个评审者更好地了解
+正在�生的事情。
+
+一定�告诉审稿人你在�什么改�，并感谢他们的时间。代�审查是一个累人且
+耗时的过程，审查人员有时会�得暴�。�使在这�情况下，也�礼貌地回应并
+解决他们指出的问题。
+
+9）��泄气或��烦
+-------------------
+
+�交更改�，请�心等待。审阅者是忙碌的人，�能无法立�访问您的修补程�。
+
+曾几何时，补�曾在没有评论的情况下消失在空白中，但开�过程比现在更加顺利。
+您应该在一周左�的时间内收到评论；如果没有收到评论，请确�您已将补���
+到正确的�置。在�新�交或�系审阅者之�至少等待一周-在诸如�并窗�之类的
+�忙时间�能更长。
+
+10）主题中包� PATCH
+--------------------
+
+由于到linus和linux内核的电�邮件��很高，通常会在主题行��加上[PATCH]
+�缀. 这使Linus和其他内核开�人员更容易将补�与其他电�邮件讨论区分开。
+
+11）签署你的作�-开�者原始认�
+-------------------------------
+
+为了加强对��了何事的追踪，尤其是对那些�过好几层的维护者的补�，我们
+建议在��出去的补�上加一个 “sign-off� 的过程。
+
+"sign-off" 是在补�的注释的最�的简�的一行文字，认�你编写了它或者其他
+人有�力将它作为开放�代�的补�传递。规则很简�：如果你能认�如下信�:
+
+开�者���书 1.1
+^^^^^^^^^^^^^^^^^^
+
+对于本项目的贡献，我认�如下信�：
+
+      （a）这些贡献是完全或者部分的由我创建，我有�利以文件中指出
+           的开放�代�许���交它；或者
+      （b）这些贡献基于以�的工作，�我所知，这些以�的工作��当的开放
+           �代�许���护，而且，根�许��，我有��交修改�的贡献，
+           无论是完全还是部分由我创造，这些贡献都使用�一个开放�代�许��
+           （除�我被�许用其它的许��），正如文件中指出的；或者
+      （c）这些贡献由认�（a），（b）或者（c）的人直接�供给我，而
+           且我没有修改它。
+      （d）我�解并��这个项目和贡献是公开的，贡献的记录（包括我
+           一起�交的个人记录，包括 sign-off ）被永久维护并且�以和这个项目
+           或者开放�代�的许���步地��行。
+
+那么加入这样一行::
+
+       Signed-off-by: Random J Developer <random@developer.example.org>
+
+使用你的真�（抱歉，�能使用��或者匿�。）
+
+有人在最�加上标签。现在这些东西会被忽略，但是你�以这样�，�标记公�
+内部的过程，或者�是指出关于 sign-off 的一些特殊细节。
+
+如果您是�系统或分支维护人员，有时需��微修改收到的补�，以便�并它们，
+因为树和�交者中的代��完全相�。如果你严格�守规则（c），你应该�求�交者
+�新�布，但这完全是在浪费时间和精力。规则（b）�许您调整代�，但是更改一个
+�交者的代�并让他认�您的错误是�常�礼貌的。�解决此问题，建议在最�一个
+由签�行和您的行之间添加一行，指示更改的性质。虽然这并�是强制性的，但似乎
+在�述�加上您的邮件和/或姓�（全部用方括�括起�），这足以让人注�到您对最
+�一分钟的更改负有责任。例如::
+
+	Signed-off-by: Random J Developer <random@developer.example.org>
+	[lucky@maintainer.example.org: struct foo moved from foo.c to foo.h]
+	Signed-off-by: Lucky K Maintainer <lucky@maintainer.example.org>
+
+如果您维护一个稳定的分支机构，�时希望对作者进行致谢�跟踪更改��并修�并
+�护�交者��投诉，那么这��法尤其有用。请注�，在任何情况下都�能更改作者
+的ID（From 头），因为它是出现在更改日志中的标识。
+
+对回�（back-porters）的特别说明：在�交消�的顶部（主题行之�）�入一个补�
+的起�指示似乎是一�常�且有用的实践，以便于跟踪。例如，下�是我们在3.x稳定
+版本中看到的内容::
+
+  Date:   Tue Oct 7 07:26:38 2014 -0400
+
+    libata: Un-break ATA blacklist
+
+    commit 1c40279960bcd7d52dbdf1d466b20d24b99176c8 upstream.
+
+还有， 这里是一个旧版内核中的一个回�补�::
+
+    Date:   Tue May 13 22:12:27 2008 +0200
+
+        wireless, airo: waitbusy() won't delay
+
+        [backport of 2.6 commit b7acbdfbd1f277c1eb23f344f899cfa4cd0bf36a]
+
+12）何时使用Acked-by:，CC:，和Co-Developed by:
+----------------------------------------------
+
+Singed-off-by: 标记表示签�者�与了补�的开�，或者他/她在补�的传递路径中。
+
+如果一个人没有直接�与补�的准备或处�，但希望表示并记录他们对补�的批准，
+那么他们�以�求在补�的�更日志中添加一个 Acked-by:
+
+Acked-by：通常由�影�代�的维护者使用，当该维护者既没有贡献也没有转�补�时。
+
+Acked-by: ��签字人那样正�。这是一个记录，确认人至少审查了补�，并表示接�。
+因此，补��并有时会手动将Acker的“Yep，looks good to me�转�为 Acked-By：（但
+请注�，通常最好�求一个明确的Ack）。
+
+Acked-by：�一定表示对整个补�的确认。例如，如果一个补�影�多个�系统，并且
+有一个：�自一个�系统维护者，那么这通常表示�确认影�维护者代�的部分。这里
+应该仔细判断。如有疑问，应�考邮件列表档案中的原始讨论。
+
+如果�人有机会对补�进行评论，但没有�供此类评论，您�以选择在补�中添加 ``Cc:``
+这是唯一一个标签，它�以在没有被它命�的人显��作的情况下添加，但它应该表明
+这个人是在补�上抄�的。讨论中包�了潜在利益相关方。
+
+Co-developed-by: 声明补�是由多个开�人员共�创建的；当几个人在一个补�上工
+作时，它用于将属性赋予共�作者（除了 From: 所赋予的作者之外）。因为
+Co-developed-by: 表示作者身份，所以�个共�开�人：必须紧跟在相关�作作者的
+签�之�。标准的签核程��求：标记的签核顺�应尽�能�映补�的时间历�，而�
+管作者是通过 From ：还是由 Co-developed-by: 共�开�的。值得注�的是，最�一
+个签字人：必须始终是�交补�的开�人员。
+
+注�，当作者也是电�邮件标题“�件人：�行中列出的人时，“From: � 标记是�选的。
+
+作者�交的补�程�示例::
+
+	<changelog>
+
+	Co-developed-by: First Co-Author <first@coauthor.example.org>
+	Signed-off-by: First Co-Author <first@coauthor.example.org>
+	Co-developed-by: Second Co-Author <second@coauthor.example.org>
+	Signed-off-by: Second Co-Author <second@coauthor.example.org>
+	Signed-off-by: From Author <from@author.example.org>
+
+�作开�者�交的补�示例::
+
+	From: From Author <from@author.example.org>
+
+	<changelog>
+
+	Co-developed-by: Random Co-Author <random@coauthor.example.org>
+	Signed-off-by: Random Co-Author <random@coauthor.example.org>
+	Signed-off-by: From Author <from@author.example.org>
+	Co-developed-by: Submitting Co-Author <sub@coauthor.example.org>
+	Signed-off-by: Submitting Co-Author <sub@coauthor.example.org>
+
+
+13）使用报告人：�测试人：�审核人：�建议人：�修�人：
+--------------------------------------------------------
+
+Reported-by: 给那些�现错误并报告错误的人致谢，它希望激励他们在将��次帮助
+我们。请注�，如果bug是以�有方�报告的，那么在使用Reported-by标记之�，请
+先请求��。
+
+Tested-by: 标记表示补�已由指定的人（在�些环境中）�功测试。这个标签通知
+维护人员已�执行了一些测试，为将�的补��供了一�定�测试人员的方法，并确
+�测试人员的信誉。
+
+Reviewed-by：相�，根�审查人的声明，表明该补�已被审查并被认为是�接�的：
+
+
+审查人的监�声明
+^^^^^^^^^^^^^^^^
+
+通过�供我的 Reviewed-by，我声明：
+
+        (a) 我已�对这个补�进行了一次技术审查，以评估它是�适�被包�到
+            主线内核中。
+
+        (b) 与补�相关的任何问题�顾虑或问题都已�馈给�交者。我对�交者对
+            我的评论的回应感到满�。
+
+        (c) 虽然这一�交�能会改进一些东西，但我相信，此时，（1）对内核
+            进行了有价值的修改，（2）没有包�争论中涉�的已知问题。
+
+        (d) 虽然我已�审查了补�并认为它是�全的，但我�会（除��有明确
+            说明）作出任何��或��它将在任何给定情况下实现其规定的目的
+            或正常�行。
+
+Reviewed-by 是一�观点声明，�补�是对内核的适当修改，没有任何�留的严�技术
+问题。任何感兴趣的审阅者（完�工作的人）都�以为一个补��供一个 Review-by
+标签。此标签用于�审阅者�供致谢，并通知维护者已在修补程�上完�的审阅程度。
+Reviewed-by: 当由已知了解主题区域并执行彻底检查的审阅者�供时，通常会增加
+补�进入内核的�能性。
+
+Suggested-by: 表示补�的想法是由指定的人�出的，并确�将此想法归功于指定的
+人。请注�，未�许�，�得添加此标签，特别是如果该想法未在公共论�上�布。
+这就是说，如果我们勤快地致谢我们的创�者，他们很有希望在未�得到鼓舞，�次
+帮助我们。
+
+Fixes: 指示补�在以�的�交中修�了一个问题。它�以很容易地确定错误的��，
+这有助于检查错误修�。这个标记还帮助稳定内核团队确定应该接收修�的稳定内核
+版本。这是指示补�修�的错误的首选方法。请�阅 :ref:`cn_describe_changes`
+�述您的更改以了解更多详细信�。
+
+.. _cn_the_canonical_patch_format:
+
+12）标准补�格�
+----------------
+
+本节�述如何格�化补�本身。请注�，如果您的补�存储在 ``Git`` 存储库中，则
+�以使用 ``git format-patch`` 进行正确的补�格�设置。但是，这些工具无法创建
+必�的文本，因此请务必阅读下�的说明。
+
+标准的补�，标题行是::
+
+    Subject: [PATCH 001/123] �系统:一��概述
+
+标准补�的信体存在如下部分：
+
+  - 一个 "from" 行指出补�作者。�跟空行（仅当��修补程�的人�是作者时�需�）。
+
+  - 解释的正文，行以75列包装，这将被�制到永久�更日志��述这个补�。
+
+  - 一个空行
+
+  - 上��述的“Signed-off-by� 行，也将出现在更改日志中。
+
+  - �包� ``---`` 的标记线。
+
+  - 任何其他�适�放在�更日志的注释。
+
+  - 实际补�（ ``diff`` 输出）。
+
+标题行的格�，使得对标题行按字��排��常的容易 - 很多 e-mail 客户端都
+�以支� - 因为�列�是用零填充的，所以按数字排�和按字�排�是一样的。
+
+e-mail 标题中的“�系统�标识哪个内核�系统将被打补�。
+
+e-mail 标题中的“一��概述�扼�的�述 e-mail 中的补�。“一��概述�
+�应该是一个文件�。对于一个补�系列（“补�系列�指一系列的多个相关补
+�），��对�个补�都使用�样的“一��概述�。
+
+记� e-mail 的“一��概述�会�为该补�的全局唯一标识。它会蔓延到 git
+的改动记录里。然�“一��概述�会被用在开�者的讨论里，用�指代这个补
+�。用户将希望通过 google ��索"一��概述"�找到那些讨论这个补�的文
+章。当人们在两三个月�使用诸如 ``gitk`` 或 ``git log --oneline`` 之类
+的工具查看数�个补�时，也会很快看到它。
+
+出于这些原因，概述必须�超过70-75个字符，并且必须�述补�的更改以�为
+什么需�补�。既�简����述性很有挑战性，但写得好的概述应该这样�。
+
+概述的�缀�以用方括�括起�：“Subject: [PATCH <tag>...] <概述>�。标记
+�被视为概述的一部分，而是�述应该如何处�补�。如果补�的多个版本已�
+�出�以�应评审（�“v1，v2，v3�）或“rfc�，以指示评审请求，那么通用标记
+�能包括版本�述符。如果一个补�系列中有四个补�，那么�个补��以这样
+编�：1/4�2/4�3/4�4/4。这�以确�开�人员了解补�应用的顺�，并且他们
+已�查看或应用了补�系列中的所有补�。
+
+一些标题的例�::
+
+    Subject: [patch 2/5] ext2: improve scalability of bitmap searching
+    Subject: [PATCHv2 001/207] x86: fix eflags tracking
+
+"From" 行是信体里的最上�一行，具有如下格�：
+        From: Patch Author <author@example.com>
+
+"From" 行指明在永久改动日志里，�会被确认为作者。如果没有 "From" 行，那
+么邮件头里的 "From: " 行会被用�决定改动日志中的作者。
+
+说明的主题将会被�交到永久的�代�改动日志里，因此对那些早已��记得和
+这个补�相关的讨论细节的有能力的读者�说，是有�义的。包括补�程�定�
+错误的（内核日志消��OOPS消�等）症状，对于�索�交日志以寻找适用补�的人
+尤其有用。如果一个补�修�了一个编译失败，那么�能�需�包�所有编译失败；
+��足够让�索补�的人能够找到它就行了。与概述一样，既�简����述性。
+
+"---" 标记行对于补�处�工具�找到哪里是改动日志信�的结�，是��缺少
+的。
+
+对于 "---" 标记之�的�外注解，一个好的用途就是用�写 diffstat，用�显
+示修改了什么文件和�个文件都增加和删除了多少行。diffstat 对于比较大的补
+�特别有用。其余那些�是和时刻或者开�者相关的注解，��适放到永久的改
+动日志里的，也应该放这里。
+使用 diffstat的选项 "-p 1 -w 70" 这样文件�就会从内核�代�树的目录开始
+，�会�用太宽的空间（很容易适�80列的宽度，也许会有一些缩进。）
+
+在��的�考资料中能看到适当的补�格�的更多细节。
+
+.. _cn_explicit_in_reply_to:
+
+15) 明确回�邮件头(In-Reply-To)
+-------------------------------
+
+手动添加回�补�的的标题头(In-Reply_To:) 是有帮助的（例如，使用 ``git send-email`` ）
+将补�与以�的相关讨论关�起�，例如，将bug修�程�链接到电�邮件和bug报告。
+但是，对于多补�系列，最好��在回�时使用链接到该系列的旧版本。这样，
+补�的多个版本就�会�为电�邮件客户端中无法管�的引用�列。如果链接有用，
+�以使用 https://lkml.kernel.org/ �定�器（例如，在��电�邮件文本中）
+链接到补�系列的早期版本。
+
+16) ��git pull请求
+--------------------
+
+如果您有一系列补�，那么让维护人员通过git pull�作将它们直接拉入�系统存储
+库�能是最方便的。但是，请注�，从开�人员那里获�补�比从邮件列表中获�补
+�需�更高的信任度。因此，许多�系统维护人员�愿�接�请求，特别是�自新的
+未知开�人员的请求。如果有疑问，您�以在��邮件中使用pull 请求作为补�系列
+正常�布的一个选项，让维护人员�以选择使用其中之一。
+
+pull 请求的主题行中应该有[Git Pull]。请求本身应该在一行中包�存储库�称和
+感兴趣的分支；它应该看起��::
+
+  Please pull from
+
+      git://jdelvare.pck.nerim.net/jdelvare-2.6 i2c-for-linus
+
+  to get these changes:
+
+
+pull 请求还应该包�一�整体消�，说明请求中将包�什么，一个补�本身的 ``Git shortlog``
+以�一个显示补�系列整体效果的 ``diffstat`` 。当然，将所有这些信�收集在一起
+的最简�方法是让 ``git`` 使用 ``git request-pull`` 命令为您完�这些工作。
+
+一些维护人员（包括Linus）希望看到�自已签��交的请求；这增加了他们对你的
+请求信心。特别是，在没有签�标签的情况下，Linus �会从� Github 这样的公共
+托管站点拉请求。
+
+创建此类签�的第一步是生�一个 GNRPG 密钥，并由一个或多个核心内核开�人员对
+其进行签�。这一步对新开�人员�说�能很困难，但没有办法绕过它。�加会议是
+找到�以签署您的密钥的开�人员的好方法。
+
+一旦您在Git 中准备了一个您希望有人拉的补�系列，就用 ``git tag -s`` 创建一
+个签�标记。这将创建一个新标记，标识该系列中的最�一次�交，并包�用您的�
+钥创建的签�。您还�以将changelog样�的消�添加到标记中；这是一个�述拉请求
+整体效果的�想�置。
+
+如果维护人员将�从中��的树�是您正在使用的存储库，请��忘记将已签�的标记
+显�推�到公共树。
+
+生�拉请求时，请使用已签�的标记作为目标。这样的命令�以实现::
+
+  git request-pull master git://my.public.tree/linux.git my-signed-tag
+
+�考文献
+--------
+
+Andrew Morton, "The perfect patch" (tpp).
+  <http://www.ozlabs.org/~akpm/stuff/tpp.txt>
+
+Jeff Garzik, "Linux kernel patch submission format".
+  <http://linux.yyz.us/patch-format.html>
+
+Greg Kroah-Hartman, "How to piss off a kernel subsystem maintainer".
+  <http://www.kroah.com/log/linux/maintainer.html>
+
+  <http://www.kroah.com/log/linux/maintainer-02.html>
+
+  <http://www.kroah.com/log/linux/maintainer-03.html>
+
+  <http://www.kroah.com/log/linux/maintainer-04.html>
+
+  <http://www.kroah.com/log/linux/maintainer-05.html>
+
+  <http://www.kroah.com/log/linux/maintainer-06.html>
+
+NO!!!! No more huge patch bombs to linux-kernel@vger.kernel.org people!
+  <https://lkml.org/lkml/2005/7/11/336>
+
+Kernel Documentation/process/coding-style.rst:
+  :ref:`Documentation/translations/zh_CN/process/coding-style.rst <cn_codingstyle>`
+
+Linus Torvalds's mail on the canonical patch format:
+  <http://lkml.org/lkml/2005/4/7/183>
+
+Andi Kleen, "On submitting kernel patches"
+  Some strategies to get difficult or controversial changes in.
+
+  http://halobates.de/on-submitting-patches.pdf
diff --git a/Documentation/translations/zh_CN/volatile-considered-harmful.txt b/Documentation/translations/zh_CN/process/volatile-considered-harmful.rst
index 475125967197..48b32ce58ef1 100644
--- a/Documentation/translations/zh_CN/volatile-considered-harmful.txt
+++ b/Documentation/translations/zh_CN/process/volatile-considered-harmful.rst
@@ -1,30 +1,23 @@
-Chinese translated version of Documentation/process/volatile-considered-harmful.rst
+.. _cn_volatile_considered_harmful:
 
-If you have any comment or update to the content, please contact the
-original document maintainer directly.  However, if you have a problem
-communicating in English you can also ask the Chinese maintainer for
-help.  Contact the Chinese maintainer if this translation is outdated
-or if there is a problem with the translation.
+.. include:: ../disclaimer-zh_CN.rst
 
-Maintainer: Jonathan Corbet <corbet@lwn.net>
-Chinese maintainer: Bryan Wu <bryan.wu@analog.com>
----------------------------------------------------------------------
-Documentation/process/volatile-considered-harmful.rst 的中文翻译
+:Original: :ref:`Documentation/process/volatile-considered-harmful.rst
+           <volatile_considered_harmful>`
 
 如果想评论或更新本文的内容，请直接�系原文档的维护者。如果你使用英文
 交�有困难的�，也�以�中文版维护者求助。如果本翻译更新��时或者翻
-译存在问题，请�系中文版维护者。
+译存在问题，请�系中文版维护者::
 
-英文版维护者： Jonathan Corbet <corbet@lwn.net>
-中文版维护者： ��  Bryan Wu <bryan.wu@analog.com>
-中文版翻译者： ��  Bryan Wu <bryan.wu@analog.com>
-中文版校译者： 张汉辉  Eugene Teo <eugeneteo@kernel.sg>
-               �瑞  Dave Young <hidave.darkstar@gmail.com>
-以下为正文
----------------------------------------------------------------------
+        英文版维护者： Jonathan Corbet <corbet@lwn.net>
+        中文版维护者： ��  Bryan Wu <bryan.wu@analog.com>
+        中文版翻译者： ��  Bryan Wu <bryan.wu@analog.com>
+        中文版校译者： 张汉辉  Eugene Teo <eugeneteo@kernel.sg>
+                       �瑞  Dave Young <hidave.darkstar@gmail.com>
+                       时奎亮 Alex Shi <alex.shi@linux.alibaba.com>
 
 为什么�应该使用“volatile�类型
-------------------------------
+==============================
 
 C程�员通常认为volatile表示�个���以在当�执行的线程之外被改�；因此，在内核
 中用到共享数�结构时，常常会有C程�员喜欢使用volatile这类��。���说，他们�
@@ -41,7 +34,7 @@ C程åº�员通常认为volatile表示æŸ�个å�˜é‡�å�¯ä»¥åœ¨å½“å‰�执行的线程ä¹
 必��使用volatile。如果�然必须使用volatile，那么几乎�以肯定在代�的�处有一
 个bug。在正确设计的内核代�中，volatile能带�的仅仅是使事情�慢。
 
-�考一下这段典型的内核代�：
+�考一下这段典型的内核代�::
 
     spin_lock(&the_lock);
     do_something_on(&shared_data);
@@ -66,7 +59,7 @@ volatile的存储类型最�是为那些内存映射的I/O寄存器而定义。
 是必需的。
 
 �一�引起用户�能使用volatile的情况是当处�器正忙�等待一个��的值。正确执行一
-个忙等待的方法是：
+个忙等待的方法是::
 
     while (my_variable != what_i_want)
         cpu_relax();
diff --git a/Documentation/translations/zh_CN/sparse.txt b/Documentation/translations/zh_CN/sparse.txt
index 2f728962a8e2..47fc4a06ebe8 100644
--- a/Documentation/translations/zh_CN/sparse.txt
+++ b/Documentation/translations/zh_CN/sparse.txt
@@ -6,7 +6,7 @@ communicating in English you can also ask the Chinese maintainer for
 help.  Contact the Chinese maintainer if this translation is outdated
 or if there is a problem with the translation.
 
-Chinese maintainer: Li Yang <leo@zh-kernel.org>
+Chinese maintainer: Li Yang <leoyang.li@nxp.com>
 ---------------------------------------------------------------------
 Documentation/dev-tools/sparse.rst 的中文翻译
 
@@ -14,8 +14,8 @@ Documentation/dev-tools/sparse.rst 的中文翻译
 交�有困难的�，也�以�中文版维护者求助。如果本翻译更新��时或者翻
 译存在问题，请�系中文版维护者。
 
-中文版维护者： �阳  Li Yang <leo@zh-kernel.org>
-中文版翻译者： �阳  Li Yang <leo@zh-kernel.org>
+中文版维护者： �阳  Li Yang <leoyang.li@nxp.com>
+中文版翻译者： �阳  Li Yang <leoyang.li@nxp.com>
 
 
 以下为正文
diff --git a/Documentation/unaligned-memory-access.txt b/Documentation/unaligned-memory-access.txt
index 51b4ff031586..1ee82419d8aa 100644
--- a/Documentation/unaligned-memory-access.txt
+++ b/Documentation/unaligned-memory-access.txt
@@ -1,5 +1,5 @@
 =========================
-UNALIGNED MEMORY ACCESSES
+Unaligned Memory Accesses
 =========================
 
 :Author: Daniel Drake <dsd@gentoo.org>,
diff --git a/Documentation/usb/WUSB-Design-overview.txt b/Documentation/usb/WUSB-Design-overview.txt
index fdb47637720e..dc5e21609bb5 100644
--- a/Documentation/usb/WUSB-Design-overview.txt
+++ b/Documentation/usb/WUSB-Design-overview.txt
@@ -1,7 +1,9 @@
-
+================================
 Linux UWB + Wireless USB + WiNET
+================================
+
+   Copyright (C) 2005-2006 Intel Corporation
 
-   (C) 2005-2006 Intel Corporation
    Inaky Perez-Gonzalez <inaky.perez-gonzalez@intel.com>
 
    This program is free software; you can redistribute it and/or
@@ -29,6 +31,7 @@ drivers for the USB based UWB radio controllers defined in the
 Wireless USB 1.0 specification (including Wireless USB host controller
 and an Intel WiNET controller).
 
+.. Contents
    1. Introduction
          1. HWA: Host Wire adapters, your Wireless USB dongle
 
@@ -51,7 +54,8 @@ and an Intel WiNET controller).
    4. Glossary
 
 
-    Introduction
+Introduction
+============
 
 UWB is a wide-band communication protocol that is to serve also as the
 low-level protocol for others (much like TCP sits on IP). Currently
@@ -93,7 +97,8 @@ The different logical parts of this driver are:
       do the actual WUSB.
 
 
-      HWA: Host Wire adapters, your Wireless USB dongle
+HWA: Host Wire adapters, your Wireless USB dongle
+-------------------------------------------------
 
 WUSB also defines a device called a Host Wire Adaptor (HWA), which in
 mere terms is a USB dongle that enables your PC to have UWB and Wireless
@@ -125,7 +130,8 @@ The HWA itself is broken in two or three main interfaces:
       their type and kick into gear.
 
 
-      DWA: Device Wired Adaptor, a Wireless USB hub for wired devices
+DWA: Device Wired Adaptor, a Wireless USB hub for wired devices
+---------------------------------------------------------------
 
 These are the complement to HWAs. They are a USB host for connecting
 wired devices, but it is connected to your PC connected via Wireless
@@ -137,7 +143,8 @@ code with the HWA-RC driver; there is a bunch of factorization work that
 has been done to support that in upcoming releases.
 
 
-      WHCI: Wireless Host Controller Interface, the PCI WUSB host adapter
+WHCI: Wireless Host Controller Interface, the PCI WUSB host adapter
+-------------------------------------------------------------------
 
 This is your usual PCI device that implements WHCI. Similar in concept
 to EHCI, it allows your wireless USB devices (including DWAs) to connect
@@ -148,7 +155,8 @@ There is still no driver support for this, but will be in upcoming
 releases.
 
 
-    The UWB stack
+The UWB stack
+=============
 
 The main mission of the UWB stack is to keep a tally of which devices
 are in radio proximity to allow drivers to connect to them. As well, it
@@ -156,7 +164,8 @@ provides an API for controlling the local radio controllers (RCs from
 now on), such as to start/stop beaconing, scan, allocate bandwidth, etc.
 
 
-      Devices and hosts: the basic structure
+Devices and hosts: the basic structure
+--------------------------------------
 
 The main building block here is the UWB device (struct uwb_dev). For
 each device that pops up in radio presence (ie: the UWB host receives a
@@ -187,7 +196,8 @@ the USB connected HWA. Eventually, drivers/whci-rc.c will do the same
 for the PCI connected WHCI controller.
 
 
-      Host Controller life cycle
+Host Controller life cycle
+--------------------------
 
 So let's say we connect a dongle to the system: it is detected and
 firmware uploaded if needed [for Intel's i1480
@@ -209,7 +219,8 @@ When a dongle is disconnected, /drivers/uwb/hwa-rc.c:hwarc_disconnect()/
 takes time of tearing everything down safely (or not...).
 
 
-      On the air: beacons and enumerating the radio neighborhood
+On the air: beacons and enumerating the radio neighborhood
+----------------------------------------------------------
 
 So assuming we have devices and we have agreed for a channel to connect
 on (let's say 9), we put the new RC to beacon:
@@ -235,12 +246,14 @@ are received in some time, the device is considered gone and wiped out
 the beacon cache of dead devices].
 
 
-      Device lists
+Device lists
+------------
 
 All UWB devices are kept in the list of the struct bus_type uwb_bus_type.
 
 
-      Bandwidth allocation
+Bandwidth allocation
+--------------------
 
 The UWB stack maintains a local copy of DRP availability through
 processing of incoming *DRP Availability Change* notifications. This
@@ -260,7 +273,8 @@ completion. [Note: The bandwidth reservation work is in progress and
 subject to change.]
 
 
-    Wireless USB Host Controller drivers
+Wireless USB Host Controller drivers
+====================================
 
 *WARNING* This section needs a lot of work!
 
@@ -296,7 +310,8 @@ starts sending MMCs.
 
 Now it all depends on external stimuli.
 
-*New device connection*
+New device connection
+---------------------
 
 A new device pops up, it scans the radio looking for MMCs that give out
 the existence of Wireless USB channels. Once one (or more) are found,
@@ -322,7 +337,8 @@ has seen the port status changes, as we have been toggling them. It will
 start enumerating and doing transfers through usb_hcd->urb_enqueue() to
 read descriptors and move our data.
 
-*Device life cycle and keep alives*
+Device life cycle and keep alives
+---------------------------------
 
 Every time there is a successful transfer to/from a device, we update a
 per-device activity timestamp. If not, every now and then we check and
@@ -340,7 +356,8 @@ device list looking for whom needs refreshing.
 If the device wants to disconnect, it will either die (ugly) or send a
 /DN_Disconnect/ that will prompt a disconnection from the system.
 
-*Sending and receiving data*
+Sending and receiving data
+--------------------------
 
 Data is sent and received through /Remote Pipes/ (rpipes). An rpipe is
 /aimed/ at an endpoint in a WUSB device. This is the same for HWAs and
@@ -394,7 +411,8 @@ finalize the transfer.
 For IN xfers, we only issue URBs for the segments we want to read and
 then wait for the xfer result data.
 
-*URB mapping into xfers*
+URB mapping into xfers
+^^^^^^^^^^^^^^^^^^^^^^
 
 This is done by hwahc_op_urb_[en|de]queue(). In enqueue() we aim an
 rpipe to the endpoint where we have to transmit, create a transfer
@@ -407,7 +425,8 @@ and not yet done and when all that is done, the xfer callback will be
 called--this will call the URB callback.
 
 
-    Glossary
+Glossary
+========
 
 *DWA* -- Device Wire Adapter
 
@@ -436,4 +455,3 @@ the host.
 
 Design-overview.txt-1.8 (last edited 2006-11-04 12:22:24 by
 InakyPerezGonzalez)
-
diff --git a/Documentation/usb/acm.txt b/Documentation/usb/acm.txt
index 903abca10517..e8bda98e9b51 100644
--- a/Documentation/usb/acm.txt
+++ b/Documentation/usb/acm.txt
@@ -1,127 +1,131 @@
-			  Linux ACM driver v0.16
-		 (c) 1999 Vojtech Pavlik <vojtech@suse.cz>
-			     Sponsored by SuSE
-----------------------------------------------------------------------------
+======================
+Linux ACM driver v0.16
+======================
+
+Copyright (c) 1999 Vojtech Pavlik <vojtech@suse.cz>
+
+Sponsored by SuSE
 
 0. Disclaimer
 ~~~~~~~~~~~~~
-  This program is free software; you can redistribute it and/or modify it
+This program is free software; you can redistribute it and/or modify it
 under the terms of the GNU General Public License as published by the Free
 Software Foundation; either version 2 of the License, or (at your option)
 any later version.
 
-  This program is distributed in the hope that it will be useful, but
+This program is distributed in the hope that it will be useful, but
 WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 more details.
 
-  You should have received a copy of the GNU General Public License along
+You should have received a copy of the GNU General Public License along
 with this program; if not, write to the Free Software Foundation, Inc., 59
 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 
-  Should you need to contact me, the author, you can do so either by e-mail
-- mail your message to <vojtech@suse.cz>, or by paper mail: Vojtech Pavlik,
+Should you need to contact me, the author, you can do so either by e-mail -
+mail your message to <vojtech@suse.cz>, or by paper mail: Vojtech Pavlik,
 Ucitelska 1576, Prague 8, 182 00 Czech Republic
 
-  For your convenience, the GNU General Public License version 2 is included
+For your convenience, the GNU General Public License version 2 is included
 in the package: See the file COPYING.
 
 1. Usage
 ~~~~~~~~
-  The drivers/usb/class/cdc-acm.c drivers works with USB modems and USB ISDN terminal
+The drivers/usb/class/cdc-acm.c drivers works with USB modems and USB ISDN terminal
 adapters that conform to the Universal Serial Bus Communication Device Class
 Abstract Control Model (USB CDC ACM) specification.
 
-  Many modems do, here is a list of those I know of:
+Many modems do, here is a list of those I know of:
 
-	3Com OfficeConnect 56k
-	3Com Voice FaxModem Pro
-	3Com Sportster
-	MultiTech MultiModem 56k
-	Zoom 2986L FaxModem
-	Compaq 56k FaxModem
-	ELSA Microlink 56k
+	- 3Com OfficeConnect 56k
+	- 3Com Voice FaxModem Pro
+	- 3Com Sportster
+	- MultiTech MultiModem 56k
+	- Zoom 2986L FaxModem
+	- Compaq 56k FaxModem
+	- ELSA Microlink 56k
 
-  I know of one ISDN TA that does work with the acm driver:
+I know of one ISDN TA that does work with the acm driver:
 
-	3Com USR ISDN Pro TA
+	- 3Com USR ISDN Pro TA
 
-  Some cell phones also connect via USB. I know the following phones work:
+Some cell phones also connect via USB. I know the following phones work:
 
-	SonyEricsson K800i
+	- SonyEricsson K800i
 
-  Unfortunately many modems and most ISDN TAs use proprietary interfaces and
+Unfortunately many modems and most ISDN TAs use proprietary interfaces and
 thus won't work with this drivers. Check for ACM compliance before buying.
 
-  To use the modems you need these modules loaded:
+To use the modems you need these modules loaded::
 
 	usbcore.ko
 	uhci-hcd.ko ohci-hcd.ko or ehci-hcd.ko
 	cdc-acm.ko
 
-  After that, the modem[s] should be accessible. You should be able to use
+After that, the modem[s] should be accessible. You should be able to use
 minicom, ppp and mgetty with them.
 
 2. Verifying that it works
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
-  The first step would be to check /sys/kernel/debug/usb/devices, it should look
-like this:
-
-T:  Bus=01 Lev=00 Prnt=00 Port=00 Cnt=00 Dev#=  1 Spd=12  MxCh= 2
-B:  Alloc=  0/900 us ( 0%), #Int=  0, #Iso=  0
-D:  Ver= 1.00 Cls=09(hub  ) Sub=00 Prot=00 MxPS= 8 #Cfgs=  1
-P:  Vendor=0000 ProdID=0000 Rev= 0.00
-S:  Product=USB UHCI Root Hub
-S:  SerialNumber=6800
-C:* #Ifs= 1 Cfg#= 1 Atr=40 MxPwr=  0mA
-I:  If#= 0 Alt= 0 #EPs= 1 Cls=09(hub  ) Sub=00 Prot=00 Driver=hub
-E:  Ad=81(I) Atr=03(Int.) MxPS=   8 Ivl=255ms
-T:  Bus=01 Lev=01 Prnt=01 Port=01 Cnt=01 Dev#=  2 Spd=12  MxCh= 0
-D:  Ver= 1.00 Cls=02(comm.) Sub=00 Prot=00 MxPS= 8 #Cfgs=  2
-P:  Vendor=04c1 ProdID=008f Rev= 2.07
-S:  Manufacturer=3Com Inc.
-S:  Product=3Com U.S. Robotics Pro ISDN TA
-S:  SerialNumber=UFT53A49BVT7
-C:  #Ifs= 1 Cfg#= 1 Atr=60 MxPwr=  0mA
-I:  If#= 0 Alt= 0 #EPs= 3 Cls=ff(vend.) Sub=ff Prot=ff Driver=acm
-E:  Ad=85(I) Atr=02(Bulk) MxPS=  64 Ivl=  0ms
-E:  Ad=04(O) Atr=02(Bulk) MxPS=  64 Ivl=  0ms
-E:  Ad=81(I) Atr=03(Int.) MxPS=  16 Ivl=128ms
-C:* #Ifs= 2 Cfg#= 2 Atr=60 MxPwr=  0mA
-I:  If#= 0 Alt= 0 #EPs= 1 Cls=02(comm.) Sub=02 Prot=01 Driver=acm
-E:  Ad=81(I) Atr=03(Int.) MxPS=  16 Ivl=128ms
-I:  If#= 1 Alt= 0 #EPs= 2 Cls=0a(data ) Sub=00 Prot=00 Driver=acm
-E:  Ad=85(I) Atr=02(Bulk) MxPS=  64 Ivl=  0ms
-E:  Ad=04(O) Atr=02(Bulk) MxPS=  64 Ivl=  0ms
+
+The first step would be to check /sys/kernel/debug/usb/devices, it should look
+like this::
+
+  T:  Bus=01 Lev=00 Prnt=00 Port=00 Cnt=00 Dev#=  1 Spd=12  MxCh= 2
+  B:  Alloc=  0/900 us ( 0%), #Int=  0, #Iso=  0
+  D:  Ver= 1.00 Cls=09(hub  ) Sub=00 Prot=00 MxPS= 8 #Cfgs=  1
+  P:  Vendor=0000 ProdID=0000 Rev= 0.00
+  S:  Product=USB UHCI Root Hub
+  S:  SerialNumber=6800
+  C:* #Ifs= 1 Cfg#= 1 Atr=40 MxPwr=  0mA
+  I:  If#= 0 Alt= 0 #EPs= 1 Cls=09(hub  ) Sub=00 Prot=00 Driver=hub
+  E:  Ad=81(I) Atr=03(Int.) MxPS=   8 Ivl=255ms
+  T:  Bus=01 Lev=01 Prnt=01 Port=01 Cnt=01 Dev#=  2 Spd=12  MxCh= 0
+  D:  Ver= 1.00 Cls=02(comm.) Sub=00 Prot=00 MxPS= 8 #Cfgs=  2
+  P:  Vendor=04c1 ProdID=008f Rev= 2.07
+  S:  Manufacturer=3Com Inc.
+  S:  Product=3Com U.S. Robotics Pro ISDN TA
+  S:  SerialNumber=UFT53A49BVT7
+  C:  #Ifs= 1 Cfg#= 1 Atr=60 MxPwr=  0mA
+  I:  If#= 0 Alt= 0 #EPs= 3 Cls=ff(vend.) Sub=ff Prot=ff Driver=acm
+  E:  Ad=85(I) Atr=02(Bulk) MxPS=  64 Ivl=  0ms
+  E:  Ad=04(O) Atr=02(Bulk) MxPS=  64 Ivl=  0ms
+  E:  Ad=81(I) Atr=03(Int.) MxPS=  16 Ivl=128ms
+  C:* #Ifs= 2 Cfg#= 2 Atr=60 MxPwr=  0mA
+  I:  If#= 0 Alt= 0 #EPs= 1 Cls=02(comm.) Sub=02 Prot=01 Driver=acm
+  E:  Ad=81(I) Atr=03(Int.) MxPS=  16 Ivl=128ms
+  I:  If#= 1 Alt= 0 #EPs= 2 Cls=0a(data ) Sub=00 Prot=00 Driver=acm
+  E:  Ad=85(I) Atr=02(Bulk) MxPS=  64 Ivl=  0ms
+  E:  Ad=04(O) Atr=02(Bulk) MxPS=  64 Ivl=  0ms
 
 The presence of these three lines (and the Cls= 'comm' and 'data' classes)
 is important, it means it's an ACM device. The Driver=acm means the acm
 driver is used for the device. If you see only Cls=ff(vend.) then you're out
-of luck, you have a device with vendor specific-interface.
-
-D:  Ver= 1.00 Cls=02(comm.) Sub=00 Prot=00 MxPS= 8 #Cfgs=  2
-I:  If#= 0 Alt= 0 #EPs= 1 Cls=02(comm.) Sub=02 Prot=01 Driver=acm
-I:  If#= 1 Alt= 0 #EPs= 2 Cls=0a(data ) Sub=00 Prot=00 Driver=acm
-
-In the system log you should see:
-
-usb.c: USB new device connect, assigned device number 2
-usb.c: kmalloc IF c7691fa0, numif 1
-usb.c: kmalloc IF c7b5f3e0, numif 2
-usb.c: skipped 4 class/vendor specific interface descriptors
-usb.c: new device strings: Mfr=1, Product=2, SerialNumber=3
-usb.c: USB device number 2 default language ID 0x409
-Manufacturer: 3Com Inc.
-Product: 3Com U.S. Robotics Pro ISDN TA
-SerialNumber: UFT53A49BVT7
-acm.c: probing config 1
-acm.c: probing config 2
-ttyACM0: USB ACM device
-acm.c: acm_control_msg: rq: 0x22 val: 0x0 len: 0x0 result: 0
-acm.c: acm_control_msg: rq: 0x20 val: 0x0 len: 0x7 result: 7
-usb.c: acm driver claimed interface c7b5f3e0
-usb.c: acm driver claimed interface c7b5f3f8
-usb.c: acm driver claimed interface c7691fa0
+of luck, you have a device with vendor specific-interface::
+
+  D:  Ver= 1.00 Cls=02(comm.) Sub=00 Prot=00 MxPS= 8 #Cfgs=  2
+  I:  If#= 0 Alt= 0 #EPs= 1 Cls=02(comm.) Sub=02 Prot=01 Driver=acm
+  I:  If#= 1 Alt= 0 #EPs= 2 Cls=0a(data ) Sub=00 Prot=00 Driver=acm
+
+In the system log you should see::
+
+  usb.c: USB new device connect, assigned device number 2
+  usb.c: kmalloc IF c7691fa0, numif 1
+  usb.c: kmalloc IF c7b5f3e0, numif 2
+  usb.c: skipped 4 class/vendor specific interface descriptors
+  usb.c: new device strings: Mfr=1, Product=2, SerialNumber=3
+  usb.c: USB device number 2 default language ID 0x409
+  Manufacturer: 3Com Inc.
+  Product: 3Com U.S. Robotics Pro ISDN TA
+  SerialNumber: UFT53A49BVT7
+  acm.c: probing config 1
+  acm.c: probing config 2
+  ttyACM0: USB ACM device
+  acm.c: acm_control_msg: rq: 0x22 val: 0x0 len: 0x0 result: 0
+  acm.c: acm_control_msg: rq: 0x20 val: 0x0 len: 0x7 result: 7
+  usb.c: acm driver claimed interface c7b5f3e0
+  usb.c: acm driver claimed interface c7b5f3f8
+  usb.c: acm driver claimed interface c7691fa0
 
 If all this seems to be OK, fire up minicom and set it to talk to the ttyACM
 device and try typing 'at'. If it responds with 'OK', then everything is
diff --git a/Documentation/usb/authorization.txt b/Documentation/usb/authorization.txt
index 9dd1dc7b1009..9e53909d04c2 100644
--- a/Documentation/usb/authorization.txt
+++ b/Documentation/usb/authorization.txt
@@ -1,7 +1,8 @@
-
+==============================================================
 Authorizing (or not) your USB devices to connect to the system
+==============================================================
 
-(C) 2007 Inaky Perez-Gonzalez <inaky@linux.intel.com> Intel Corporation
+Copyright (C) 2007 Inaky Perez-Gonzalez <inaky@linux.intel.com> Intel Corporation
 
 This feature allows you to control if a USB device can be used (or
 not) in a system. This feature will allow you to implement a lock-down
@@ -12,24 +13,25 @@ its interfaces are immediately made available to the users.  With this
 modification, only if root authorizes the device to be configured will
 then it be possible to use it.
 
-Usage:
+Usage
+=====
 
-Authorize a device to connect:
+Authorize a device to connect::
 
-$ echo 1 > /sys/bus/usb/devices/DEVICE/authorized
+	$ echo 1 > /sys/bus/usb/devices/DEVICE/authorized
 
-Deauthorize a device:
+De-authorize a device::
 
-$ echo 0 > /sys/bus/usb/devices/DEVICE/authorized
+	$ echo 0 > /sys/bus/usb/devices/DEVICE/authorized
 
 Set new devices connected to hostX to be deauthorized by default (ie:
-lock down):
+lock down)::
 
-$ echo 0 > /sys/bus/usb/devices/usbX/authorized_default
+	$ echo 0 > /sys/bus/usb/devices/usbX/authorized_default
 
-Remove the lock down:
+Remove the lock down::
 
-$ echo 1 > /sys/bus/usb/devices/usbX/authorized_default
+	$ echo 1 > /sys/bus/usb/devices/usbX/authorized_default
 
 By default, Wired USB devices are authorized by default to
 connect. Wireless USB hosts deauthorize by default all new connected
@@ -40,21 +42,21 @@ USB ports.
 
 
 Example system lockdown (lame)
------------------------
+------------------------------
 
 Imagine you want to implement a lockdown so only devices of type XYZ
 can be connected (for example, it is a kiosk machine with a visible
-USB port):
+USB port)::
 
-boot up
-rc.local ->
+  boot up
+  rc.local ->
 
- for host in /sys/bus/usb/devices/usb*
- do
-    echo 0 > $host/authorized_default
- done
+   for host in /sys/bus/usb/devices/usb*
+   do
+      echo 0 > $host/authorized_default
+   done
 
-Hookup an script to udev, for new USB devices
+Hookup an script to udev, for new USB devices::
 
  if device_is_my_type $DEV
  then
@@ -67,10 +69,10 @@ checking if the class, type and protocol match something is the worse
 security verification you can make (or the best, for someone willing
 to break it). If you need something secure, use crypto and Certificate
 Authentication or stuff like that. Something simple for an storage key
-could be:
+could be::
 
-function device_is_my_type()
-{
+ function device_is_my_type()
+ {
    echo 1 > authorized		# temporarily authorize it
                                 # FIXME: make sure none can mount it
    mount DEVICENODE /mntpoint
@@ -83,7 +85,7 @@ function device_is_my_type()
    else
         echo 0 > authorized
    fi
-}
+ }
 
 
 Of course, this is lame, you'd want to do a real certificate
@@ -95,30 +97,35 @@ welcome.
 
 Interface authorization
 -----------------------
+
 There is a similar approach to allow or deny specific USB interfaces.
 That allows to block only a subset of an USB device.
 
-Authorize an interface:
-$ echo 1 > /sys/bus/usb/devices/INTERFACE/authorized
+Authorize an interface::
 
-Deauthorize an interface:
-$ echo 0 > /sys/bus/usb/devices/INTERFACE/authorized
+	$ echo 1 > /sys/bus/usb/devices/INTERFACE/authorized
+
+Deauthorize an interface::
+
+	$ echo 0 > /sys/bus/usb/devices/INTERFACE/authorized
 
 The default value for new interfaces
 on a particular USB bus can be changed, too.
 
-Allow interfaces per default:
-$ echo 1 > /sys/bus/usb/devices/usbX/interface_authorized_default
+Allow interfaces per default::
+
+	$ echo 1 > /sys/bus/usb/devices/usbX/interface_authorized_default
+
+Deny interfaces per default::
 
-Deny interfaces per default:
-$ echo 0 > /sys/bus/usb/devices/usbX/interface_authorized_default
+	$ echo 0 > /sys/bus/usb/devices/usbX/interface_authorized_default
 
 Per default the interface_authorized_default bit is 1.
 So all interfaces would authorized per default.
 
 Note:
-If a deauthorized interface will be authorized so the driver probing must
-be triggered manually by writing INTERFACE to /sys/bus/usb/drivers_probe
+  If a deauthorized interface will be authorized so the driver probing must
+  be triggered manually by writing INTERFACE to /sys/bus/usb/drivers_probe
 
 For drivers that need multiple interfaces all needed interfaces should be
 authorized first. After that the drivers should be probed.
diff --git a/Documentation/usb/chipidea.txt b/Documentation/usb/chipidea.txt
index d1eedc01b00a..68473abe2823 100644
--- a/Documentation/usb/chipidea.txt
+++ b/Documentation/usb/chipidea.txt
@@ -1,22 +1,37 @@
+==============================================
+ChipIdea Highspeed Dual Role Controller Driver
+==============================================
+
 1. How to test OTG FSM(HNP and SRP)
 -----------------------------------
+
 To show how to demo OTG HNP and SRP functions via sys input files
 with 2 Freescale i.MX6Q sabre SD boards.
 
 1.1 How to enable OTG FSM
----------------------------------------
+-------------------------
+
 1.1.1 Select CONFIG_USB_OTG_FSM in menuconfig, rebuild kernel
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
 Image and modules. If you want to check some internal
 variables for otg fsm, mount debugfs, there are 2 files
-which can show otg fsm variables and some controller registers value:
-cat /sys/kernel/debug/ci_hdrc.0/otg
-cat /sys/kernel/debug/ci_hdrc.0/registers
+which can show otg fsm variables and some controller registers value::
+
+	cat /sys/kernel/debug/ci_hdrc.0/otg
+	cat /sys/kernel/debug/ci_hdrc.0/registers
+
 1.1.2 Add below entries in your dts file for your controller node
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+::
+
 	otg-rev = <0x0200>;
 	adp-disable;
 
 1.2 Test operations
 -------------------
+
 1) Power up 2 Freescale i.MX6Q sabre SD boards with gadget class driver loaded
    (e.g. g_mass_storage).
 
@@ -26,19 +41,24 @@ cat /sys/kernel/debug/ci_hdrc.0/registers
    The A-device(with micro A plug inserted) should enumerate B-device.
 
 3) Role switch
-   On B-device:
-   echo 1 > /sys/bus/platform/devices/ci_hdrc.0/inputs/b_bus_req
+
+   On B-device::
+
+	echo 1 > /sys/bus/platform/devices/ci_hdrc.0/inputs/b_bus_req
 
    B-device should take host role and enumerate A-device.
 
 4) A-device switch back to host.
-   On B-device:
-   echo 0 > /sys/bus/platform/devices/ci_hdrc.0/inputs/b_bus_req
+
+   On B-device::
+
+	echo 0 > /sys/bus/platform/devices/ci_hdrc.0/inputs/b_bus_req
 
    or, by introducing HNP polling, B-Host can know when A-peripheral wish
    to be host role, so this role switch also can be trigged in A-peripheral
-   side by answering the polling from B-Host, this can be done on A-device:
-   echo 1 > /sys/bus/platform/devices/ci_hdrc.0/inputs/a_bus_req
+   side by answering the polling from B-Host, this can be done on A-device::
+
+	echo 1 > /sys/bus/platform/devices/ci_hdrc.0/inputs/a_bus_req
 
    A-device should switch back to host and enumerate B-device.
 
@@ -49,23 +69,31 @@ cat /sys/kernel/debug/ci_hdrc.0/registers
    A-device should NOT enumerate B-device.
 
    if A-device wants to use bus:
-   On A-device:
-   echo 0 > /sys/bus/platform/devices/ci_hdrc.0/inputs/a_bus_drop
-   echo 1 > /sys/bus/platform/devices/ci_hdrc.0/inputs/a_bus_req
+
+   On A-device::
+
+	echo 0 > /sys/bus/platform/devices/ci_hdrc.0/inputs/a_bus_drop
+	echo 1 > /sys/bus/platform/devices/ci_hdrc.0/inputs/a_bus_req
 
    if B-device wants to use bus:
-   On B-device:
-   echo 1 > /sys/bus/platform/devices/ci_hdrc.0/inputs/b_bus_req
+
+   On B-device::
+
+	echo 1 > /sys/bus/platform/devices/ci_hdrc.0/inputs/b_bus_req
 
 7) A-device power down the bus.
-   On A-device:
-   echo 1 > /sys/bus/platform/devices/ci_hdrc.0/inputs/a_bus_drop
+
+   On A-device::
+
+	echo 1 > /sys/bus/platform/devices/ci_hdrc.0/inputs/a_bus_drop
 
    A-device should disconnect with B-device and power down the bus.
 
 8) B-device does data pulse for SRP.
-   On B-device:
-   echo 1 > /sys/bus/platform/devices/ci_hdrc.0/inputs/b_bus_req
+
+   On B-device::
+
+	echo 1 > /sys/bus/platform/devices/ci_hdrc.0/inputs/b_bus_req
 
    A-device should resume usb bus and enumerate B-device.
 
@@ -75,22 +103,31 @@ cat /sys/kernel/debug/ci_hdrc.0/registers
 July 27, 2012 Revision 2.0 version 1.1a"
 
 2. How to enable USB as system wakeup source
------------------------------------
+--------------------------------------------
 Below is the example for how to enable USB as system wakeup source
 at imx6 platform.
 
-2.1 Enable core's wakeup
-echo enabled > /sys/bus/platform/devices/ci_hdrc.0/power/wakeup
-2.2 Enable glue layer's wakeup
-echo enabled > /sys/bus/platform/devices/2184000.usb/power/wakeup
-2.3 Enable PHY's wakeup (optional)
-echo enabled > /sys/bus/platform/devices/20c9000.usbphy/power/wakeup
-2.4 Enable roothub's wakeup
-echo enabled > /sys/bus/usb/devices/usb1/power/wakeup
-2.5 Enable related device's wakeup
-echo enabled > /sys/bus/usb/devices/1-1/power/wakeup
+2.1 Enable core's wakeup::
+
+	echo enabled > /sys/bus/platform/devices/ci_hdrc.0/power/wakeup
+
+2.2 Enable glue layer's wakeup::
+
+	echo enabled > /sys/bus/platform/devices/2184000.usb/power/wakeup
+
+2.3 Enable PHY's wakeup (optional)::
+
+	echo enabled > /sys/bus/platform/devices/20c9000.usbphy/power/wakeup
+
+2.4 Enable roothub's wakeup::
+
+	echo enabled > /sys/bus/usb/devices/usb1/power/wakeup
+
+2.5 Enable related device's wakeup::
+
+	echo enabled > /sys/bus/usb/devices/1-1/power/wakeup
 
 If the system has only one usb port, and you want usb wakeup at this port, you
-can use below script to enable usb wakeup.
-for i in $(find /sys -name wakeup | grep usb);do echo enabled > $i;done;
+can use below script to enable usb wakeup::
 
+	for i in $(find /sys -name wakeup | grep usb);do echo enabled > $i;done;
diff --git a/Documentation/usb/dwc3.txt b/Documentation/usb/dwc3.txt
index 1d02c01d1c7c..f94a7ba16573 100644
--- a/Documentation/usb/dwc3.txt
+++ b/Documentation/usb/dwc3.txt
@@ -1,6 +1,11 @@
+===========
+DWC3 driver
+===========
+
+
+TODO
+~~~~
 
- TODO
-~~~~~~
 Please pick something while reading :)
 
 - Convert interrupt handler to per-ep-thread-irq
@@ -9,6 +14,7 @@ Please pick something while reading :)
   until the command completes which is bad.
 
   Implementation idea:
+
   - dwc core implements a demultiplexing irq chip for interrupts per
     endpoint. The interrupt numbers are allocated during probe and belong
     to the device. If MSI provides per-endpoint interrupt this dummy
@@ -19,6 +25,7 @@ Please pick something while reading :)
   - dwc3_send_gadget_ep_cmd() will sleep in wait_for_completion_timeout()
     until the command completes.
   - the interrupt handler is split into the following pieces:
+
     - primary handler of the device
       goes through every event and calls generic_handle_irq() for event
       it. On return from generic_handle_irq() in acknowledges the event
@@ -40,6 +47,7 @@ Please pick something while reading :)
       for command completion.
 
   Latency:
+
    There should be no increase in latency since the interrupt-thread has a
    high priority and will be run before an average task in user land
    (except the user changed priorities).
diff --git a/Documentation/usb/ehci.txt b/Documentation/usb/ehci.txt
index 160bd6c3ab7b..31f650e7c1b4 100644
--- a/Documentation/usb/ehci.txt
+++ b/Documentation/usb/ehci.txt
@@ -1,3 +1,7 @@
+===========
+EHCI driver
+===========
+
 27-Dec-2002
 
 The EHCI driver is used to talk to high speed USB 2.0 devices using
@@ -40,7 +44,8 @@ APIs exposed to USB device drivers.
   <dbrownell@users.sourceforge.net>
 
 
-FUNCTIONALITY
+Functionality
+=============
 
 This driver is regularly tested on x86 hardware, and has also been
 used on PPC hardware so big/little endianness issues should be gone.
@@ -48,6 +53,7 @@ It's believed to do all the right PCI magic so that I/O works even on
 systems with interesting DMA mapping issues.
 
 Transfer Types
+--------------
 
 At this writing the driver should comfortably handle all control, bulk,
 and interrupt transfers, including requests to USB 1.1 devices through
@@ -63,6 +69,7 @@ since EHCI represents these with a different data structure.  So for now,
 most USB audio and video devices can't be connected to high speed buses.
 
 Driver Behavior
+---------------
 
 Transfers of all types can be queued.  This means that control transfers
 from a driver on one interface (or through usbfs) won't interfere with
@@ -83,14 +90,15 @@ limits on the number of periodic transactions that can be scheduled,
 and prevent use of polling intervals of less than one frame.
 
 
-USE BY
+Use by
+======
 
 Assuming you have an EHCI controller (on a PCI card or motherboard)
-and have compiled this driver as a module, load this like:
+and have compiled this driver as a module, load this like::
 
     # modprobe ehci-hcd
 
-and remove it by:
+and remove it by::
 
     # rmmod ehci-hcd
 
@@ -112,13 +120,16 @@ If you're using this driver on a 2.5 kernel, and you've enabled USB
 debugging support, you'll see three files in the "sysfs" directory for
 any EHCI controller:
 
-	"async" dumps the asynchronous schedule, used for control
+	"async"
+		dumps the asynchronous schedule, used for control
 		and bulk transfers.  Shows each active qh and the qtds
 		pending, usually one qtd per urb.  (Look at it with
 		usb-storage doing disk I/O; watch the request queues!)
-	"periodic" dumps the periodic schedule, used for interrupt
+	"periodic"
+		dumps the periodic schedule, used for interrupt
 		and isochronous transfers.  Doesn't show qtds.
-	"registers" show controller register state, and
+	"registers"
+		show controller register state, and
 
 The contents of those files can help identify driver problems.
 
@@ -136,7 +147,8 @@ transaction translators are in use; some drivers have been seen to behave
 badly when they see different faults than OHCI or UHCI report.
 
 
-PERFORMANCE
+Performance
+===========
 
 USB 2.0 throughput is gated by two main factors:  how fast the host
 controller can process requests, and how fast devices can respond to
@@ -156,6 +168,7 @@ hardware and device driver software allow it.  Periodic transfer modes
 approach the quoted 480 MBit/sec transfer rate.
 
 Hardware Performance
+--------------------
 
 At this writing, individual USB 2.0 devices tend to max out at around
 20 MByte/sec transfer rates.  This is of course subject to change;
@@ -183,6 +196,7 @@ you issue a control or bulk request you can often expect to learn that
 it completed in less than 250 usec (depending on transfer size).
 
 Software Performance
+--------------------
 
 To get even 20 MByte/sec transfer rates, Linux-USB device drivers will
 need to keep the EHCI queue full.  That means issuing large requests,
@@ -206,9 +220,11 @@ mapping (which might apply an IOMMU) and IRQ reduction, all of which will
 help make high speed transfers run as fast as they can.
 
 
-TBD:  Interrupt and ISO transfer performance issues.  Those periodic
-transfers are fully scheduled, so the main issue is likely to be how
-to trigger "high bandwidth" modes.
+TBD:
+   Interrupt and ISO transfer performance issues.  Those periodic
+   transfers are fully scheduled, so the main issue is likely to be how
+   to trigger "high bandwidth" modes.
 
-TBD:  More than standard 80% periodic bandwidth allocation is possible
-through sysfs uframe_periodic_max parameter. Describe that.
+TBD:
+   More than standard 80% periodic bandwidth allocation is possible
+   through sysfs uframe_periodic_max parameter. Describe that.
diff --git a/Documentation/usb/functionfs.txt b/Documentation/usb/functionfs.txt
index eaaaea019fc7..7fdc6d840ac5 100644
--- a/Documentation/usb/functionfs.txt
+++ b/Documentation/usb/functionfs.txt
@@ -1,4 +1,6 @@
-*How FunctionFS works*
+====================
+How FunctionFS works
+====================
 
 From kernel point of view it is just a composite function with some
 unique behaviour.  It may be added to an USB configuration only after
@@ -38,13 +40,13 @@ when mounting.
 
 One can imagine a gadget that has an Ethernet, MTP and HID interfaces
 where the last two are implemented via FunctionFS.  On user space
-level it would look like this:
+level it would look like this::
 
-$ insmod g_ffs.ko idVendor=<ID> iSerialNumber=<string> functions=mtp,hid
-$ mkdir /dev/ffs-mtp && mount -t functionfs mtp /dev/ffs-mtp
-$ ( cd /dev/ffs-mtp && mtp-daemon ) &
-$ mkdir /dev/ffs-hid && mount -t functionfs hid /dev/ffs-hid
-$ ( cd /dev/ffs-hid && hid-daemon ) &
+  $ insmod g_ffs.ko idVendor=<ID> iSerialNumber=<string> functions=mtp,hid
+  $ mkdir /dev/ffs-mtp && mount -t functionfs mtp /dev/ffs-mtp
+  $ ( cd /dev/ffs-mtp && mtp-daemon ) &
+  $ mkdir /dev/ffs-hid && mount -t functionfs hid /dev/ffs-hid
+  $ ( cd /dev/ffs-hid && hid-daemon ) &
 
 On kernel level the gadget checks ffs_data->dev_name to identify
 whether it's FunctionFS designed for MTP ("mtp") or HID ("hid").
@@ -64,4 +66,3 @@ have been written to their ep0's.
 
 Conversely, the gadget is unregistered after the first USB function
 closes its endpoints.
-
diff --git a/Documentation/usb/gadget-testing.txt b/Documentation/usb/gadget-testing.txt
index 5908a21fddb6..7d7f2340af42 100644
--- a/Documentation/usb/gadget-testing.txt
+++ b/Documentation/usb/gadget-testing.txt
@@ -1,26 +1,32 @@
+==============
+Gadget Testing
+==============
+
 This file summarizes information on basic testing of USB functions
 provided by gadgets.
 
-1. ACM function
-2. ECM function
-3. ECM subset function
-4. EEM function
-5. FFS function
-6. HID function
-7. LOOPBACK function
-8. MASS STORAGE function
-9. MIDI function
-10. NCM function
-11. OBEX function
-12. PHONET function
-13. RNDIS function
-14. SERIAL function
-15. SOURCESINK function
-16. UAC1 function (legacy implementation)
-17. UAC2 function
-18. UVC function
-19. PRINTER function
-20. UAC1 function (new API)
+.. contents
+
+   1. ACM function
+   2. ECM function
+   3. ECM subset function
+   4. EEM function
+   5. FFS function
+   6. HID function
+   7. LOOPBACK function
+   8. MASS STORAGE function
+   9. MIDI function
+   10. NCM function
+   11. OBEX function
+   12. PHONET function
+   13. RNDIS function
+   14. SERIAL function
+   15. SOURCESINK function
+   16. UAC1 function (legacy implementation)
+   17. UAC2 function
+   18. UVC function
+   19. PRINTER function
+   20. UAC1 function (new API)
 
 
 1. ACM function
@@ -44,13 +50,23 @@ There can be at most 4 ACM/generic serial/OBEX ports in the system.
 Testing the ACM function
 ------------------------
 
-On the host: cat > /dev/ttyACM<X>
-On the device : cat /dev/ttyGS<Y>
+On the host::
+
+	cat > /dev/ttyACM<X>
+
+On the device::
+
+	cat /dev/ttyGS<Y>
 
 then the other way round
 
-On the device: cat > /dev/ttyGS<Y>
-On the host: cat /dev/ttyACM<X>
+On the device::
+
+	cat > /dev/ttyGS<Y>
+
+On the host::
+
+	cat /dev/ttyACM<X>
 
 2. ECM function
 ===============
@@ -63,13 +79,15 @@ Function-specific configfs interface
 The function name to use when creating the function directory is "ecm".
 The ECM function provides these attributes in its function directory:
 
-	ifname		- network device interface name associated with this
+	=============== ==================================================
+	ifname		network device interface name associated with this
 			function instance
-	qmult		- queue length multiplier for high and super speed
-	host_addr	- MAC address of host's end of this
+	qmult		queue length multiplier for high and super speed
+	host_addr	MAC address of host's end of this
 			Ethernet over USB link
-	dev_addr	- MAC address of device's end of this
+	dev_addr	MAC address of device's end of this
 			Ethernet over USB link
+	=============== ==================================================
 
 and after creating the functions/ecm.<instance name> they contain default
 values: qmult is 5, dev_addr and host_addr are randomly selected.
@@ -82,8 +100,13 @@ Testing the ECM function
 
 Configure IP addresses of the device and the host. Then:
 
-On the device: ping <host's IP>
-On the host: ping <device's IP>
+On the device::
+
+	ping <host's IP>
+
+On the host::
+
+	ping <device's IP>
 
 3. ECM subset function
 ======================
@@ -96,13 +119,15 @@ Function-specific configfs interface
 The function name to use when creating the function directory is "geth".
 The ECM subset function provides these attributes in its function directory:
 
-	ifname		- network device interface name associated with this
+	=============== ==================================================
+	ifname		network device interface name associated with this
 			function instance
-	qmult		- queue length multiplier for high and super speed
-	host_addr	- MAC address of host's end of this
+	qmult		queue length multiplier for high and super speed
+	host_addr	MAC address of host's end of this
 			Ethernet over USB link
-	dev_addr	- MAC address of device's end of this
+	dev_addr	MAC address of device's end of this
 			Ethernet over USB link
+	=============== ==================================================
 
 and after creating the functions/ecm.<instance name> they contain default
 values: qmult is 5, dev_addr and host_addr are randomly selected.
@@ -115,8 +140,13 @@ Testing the ECM subset function
 
 Configure IP addresses of the device and the host. Then:
 
-On the device: ping <host's IP>
-On the host: ping <device's IP>
+On the device::
+
+	ping <host's IP>
+
+On the host::
+
+	ping <device's IP>
 
 4. EEM function
 ===============
@@ -129,13 +159,15 @@ Function-specific configfs interface
 The function name to use when creating the function directory is "eem".
 The EEM function provides these attributes in its function directory:
 
-	ifname		- network device interface name associated with this
+	=============== ==================================================
+	ifname		network device interface name associated with this
 			function instance
-	qmult		- queue length multiplier for high and super speed
-	host_addr	- MAC address of host's end of this
+	qmult		queue length multiplier for high and super speed
+	host_addr	MAC address of host's end of this
 			Ethernet over USB link
-	dev_addr	- MAC address of device's end of this
+	dev_addr	MAC address of device's end of this
 			Ethernet over USB link
+	=============== ==================================================
 
 and after creating the functions/eem.<instance name> they contain default
 values: qmult is 5, dev_addr and host_addr are randomly selected.
@@ -148,8 +180,13 @@ Testing the EEM function
 
 Configure IP addresses of the device and the host. Then:
 
-On the device: ping <host's IP>
-On the host: ping <device's IP>
+On the device::
+
+	ping <host's IP>
+
+On the host::
+
+	ping <device's IP>
 
 5. FFS function
 ===============
@@ -172,6 +209,7 @@ Testing the FFS function
 ------------------------
 
 On the device: start the function's userspace daemon, enable the gadget
+
 On the host: use the USB function provided by the device
 
 6. HID function
@@ -185,39 +223,43 @@ Function-specific configfs interface
 The function name to use when creating the function directory is "hid".
 The HID function provides these attributes in its function directory:
 
-	protocol	- HID protocol to use
-	report_desc	- data to be used in HID reports, except data
+	=============== ===========================================
+	protocol	HID protocol to use
+	report_desc	data to be used in HID reports, except data
 			passed with /dev/hidg<X>
-	report_length	- HID report length
-	subclass	- HID subclass to use
+	report_length	HID report length
+	subclass	HID subclass to use
+	=============== ===========================================
 
 For a keyboard the protocol and the subclass are 1, the report_length is 8,
-while the report_desc is:
+while the report_desc is::
 
-$ hd my_report_desc
-00000000  05 01 09 06 a1 01 05 07  19 e0 29 e7 15 00 25 01  |..........)...%.|
-00000010  75 01 95 08 81 02 95 01  75 08 81 03 95 05 75 01  |u.......u.....u.|
-00000020  05 08 19 01 29 05 91 02  95 01 75 03 91 03 95 06  |....).....u.....|
-00000030  75 08 15 00 25 65 05 07  19 00 29 65 81 00 c0     |u...%e....)e...|
-0000003f
+  $ hd my_report_desc
+  00000000  05 01 09 06 a1 01 05 07  19 e0 29 e7 15 00 25 01  |..........)...%.|
+  00000010  75 01 95 08 81 02 95 01  75 08 81 03 95 05 75 01  |u.......u.....u.|
+  00000020  05 08 19 01 29 05 91 02  95 01 75 03 91 03 95 06  |....).....u.....|
+  00000030  75 08 15 00 25 65 05 07  19 00 29 65 81 00 c0     |u...%e....)e...|
+  0000003f
 
-Such a sequence of bytes can be stored to the attribute with echo:
+Such a sequence of bytes can be stored to the attribute with echo::
 
-$ echo -ne \\x05\\x01\\x09\\x06\\xa1.....
+  $ echo -ne \\x05\\x01\\x09\\x06\\xa1.....
 
 Testing the HID function
 ------------------------
 
 Device:
+
 - create the gadget
 - connect the gadget to a host, preferably not the one used
-to control the gadget
+  to control the gadget
 - run a program which writes to /dev/hidg<N>, e.g.
-a userspace program found in Documentation/usb/gadget_hid.txt:
+  a userspace program found in Documentation/usb/gadget_hid.txt::
 
-$ ./hid_gadget_test /dev/hidg0 keyboard
+	$ ./hid_gadget_test /dev/hidg0 keyboard
 
 Host:
+
 - observe the keystrokes from the gadget
 
 7. LOOPBACK function
@@ -231,13 +273,16 @@ Function-specific configfs interface
 The function name to use when creating the function directory is "Loopback".
 The LOOPBACK function provides these attributes in its function directory:
 
-	qlen		- depth of loopback queue
-	bulk_buflen	- buffer length
+	=============== =======================
+	qlen		depth of loopback queue
+	bulk_buflen	buffer length
+	=============== =======================
 
 Testing the LOOPBACK function
 -----------------------------
 
 device: run the gadget
+
 host: test-usb (tools/usb/testusb.c)
 
 8. MASS STORAGE function
@@ -252,18 +297,20 @@ The function name to use when creating the function directory is "mass_storage".
 The MASS STORAGE function provides these attributes in its directory:
 files:
 
-	stall		- Set to permit function to halt bulk endpoints.
+	=============== ==============================================
+	stall		Set to permit function to halt bulk endpoints.
 			Disabled on some USB devices known not to work
 			correctly. You should set it to true.
-	num_buffers	- Number of pipeline buffers. Valid numbers
+	num_buffers	Number of pipeline buffers. Valid numbers
 			are 2..4. Available only if
 			CONFIG_USB_GADGET_DEBUG_FILES is set.
+	=============== ==============================================
 
 and a default lun.0 directory corresponding to SCSI LUN #0.
 
-A new lun can be added with mkdir:
+A new lun can be added with mkdir::
 
-$ mkdir functions/mass_storage.0/partition.5
+	$ mkdir functions/mass_storage.0/partition.5
 
 Lun numbering does not have to be continuous, except for lun #0 which is
 created by default. A maximum of 8 luns can be specified and they all must be
@@ -273,18 +320,20 @@ although it is not mandatory.
 
 In each lun directory there are the following attribute files:
 
-	file		- The path to the backing file for the LUN.
+	=============== ==============================================
+	file		The path to the backing file for the LUN.
 			Required if LUN is not marked as removable.
-	ro		- Flag specifying access to the LUN shall be
+	ro		Flag specifying access to the LUN shall be
 			read-only. This is implied if CD-ROM emulation
 			is enabled as well as when it was impossible
 			to open "filename" in R/W mode.
-	removable	- Flag specifying that LUN shall be indicated as
+	removable	Flag specifying that LUN shall be indicated as
 			being removable.
-	cdrom		- Flag specifying that LUN shall be reported as
+	cdrom		Flag specifying that LUN shall be reported as
 			being a CD-ROM.
-	nofua		- Flag specifying that FUA flag
+	nofua		Flag specifying that FUA flag
 			in SCSI WRITE(10,12)
+	=============== ==============================================
 
 Testing the MASS STORAGE function
 ---------------------------------
@@ -304,12 +353,14 @@ Function-specific configfs interface
 The function name to use when creating the function directory is "midi".
 The MIDI function provides these attributes in its function directory:
 
-	buflen		- MIDI buffer length
-	id		- ID string for the USB MIDI adapter
-	in_ports	- number of MIDI input ports
-	index		- index value for the USB MIDI adapter
-	out_ports	- number of MIDI output ports
-	qlen		- USB read request queue length
+	=============== ====================================
+	buflen		MIDI buffer length
+	id		ID string for the USB MIDI adapter
+	in_ports	number of MIDI input ports
+	index		index value for the USB MIDI adapter
+	out_ports	number of MIDI output ports
+	qlen		USB read request queue length
+	=============== ====================================
 
 Testing the MIDI function
 -------------------------
@@ -317,60 +368,63 @@ Testing the MIDI function
 There are two cases: playing a mid from the gadget to
 the host and playing a mid from the host to the gadget.
 
-1) Playing a mid from the gadget to the host
-host)
+1) Playing a mid from the gadget to the host:
+
+host::
 
-$ arecordmidi -l
- Port    Client name                      Port name
- 14:0    Midi Through                     Midi Through Port-0
- 24:0    MIDI Gadget                      MIDI Gadget MIDI 1
-$ arecordmidi -p 24:0 from_gadget.mid
+  $ arecordmidi -l
+   Port    Client name                      Port name
+   14:0    Midi Through                     Midi Through Port-0
+   24:0    MIDI Gadget                      MIDI Gadget MIDI 1
+  $ arecordmidi -p 24:0 from_gadget.mid
 
-gadget)
+gadget::
 
-$ aplaymidi -l
- Port    Client name                      Port name
- 20:0    f_midi                           f_midi
+  $ aplaymidi -l
+   Port    Client name                      Port name
+   20:0    f_midi                           f_midi
 
-$ aplaymidi -p 20:0 to_host.mid
+  $ aplaymidi -p 20:0 to_host.mid
 
 2) Playing a mid from the host to the gadget
-gadget)
 
-$ arecordmidi -l
- Port    Client name                      Port name
- 20:0    f_midi                           f_midi
+gadget::
+
+  $ arecordmidi -l
+   Port    Client name                      Port name
+   20:0    f_midi                           f_midi
 
-$ arecordmidi -p 20:0 from_host.mid
+  $ arecordmidi -p 20:0 from_host.mid
 
-host)
+host::
 
-$ aplaymidi -l
- Port    Client name                      Port name
- 14:0    Midi Through                     Midi Through Port-0
- 24:0    MIDI Gadget                      MIDI Gadget MIDI 1
+  $ aplaymidi -l
+   Port    Client name                      Port name
+   14:0    Midi Through                     Midi Through Port-0
+   24:0    MIDI Gadget                      MIDI Gadget MIDI 1
 
-$ aplaymidi -p24:0 to_gadget.mid
+  $ aplaymidi -p24:0 to_gadget.mid
 
 The from_gadget.mid should sound identical to the to_host.mid.
+
 The from_host.id should sound identical to the to_gadget.mid.
 
-MIDI files can be played to speakers/headphones with e.g. timidity installed
+MIDI files can be played to speakers/headphones with e.g. timidity installed::
 
-$ aplaymidi -l
- Port    Client name                      Port name
- 14:0    Midi Through                     Midi Through Port-0
- 24:0    MIDI Gadget                      MIDI Gadget MIDI 1
-128:0    TiMidity                         TiMidity port 0
-128:1    TiMidity                         TiMidity port 1
-128:2    TiMidity                         TiMidity port 2
-128:3    TiMidity                         TiMidity port 3
+  $ aplaymidi -l
+   Port    Client name                      Port name
+   14:0    Midi Through                     Midi Through Port-0
+   24:0    MIDI Gadget                      MIDI Gadget MIDI 1
+  128:0    TiMidity                         TiMidity port 0
+  128:1    TiMidity                         TiMidity port 1
+  128:2    TiMidity                         TiMidity port 2
+  128:3    TiMidity                         TiMidity port 3
 
-$ aplaymidi -p 128:0 file.mid
+  $ aplaymidi -p 128:0 file.mid
 
-MIDI ports can be logically connected using the aconnect utility, e.g.:
+MIDI ports can be logically connected using the aconnect utility, e.g.::
 
-$ aconnect 24:0 128:0 # try it on the host
+  $ aconnect 24:0 128:0 # try it on the host
 
 After the gadget's MIDI port is connected to timidity's MIDI port,
 whatever is played at the gadget side with aplaymidi -l is audible
@@ -387,13 +441,15 @@ Function-specific configfs interface
 The function name to use when creating the function directory is "ncm".
 The NCM function provides these attributes in its function directory:
 
-	ifname		- network device interface name associated with this
+	=============== ==================================================
+	ifname		network device interface name associated with this
 			function instance
-	qmult		- queue length multiplier for high and super speed
-	host_addr	- MAC address of host's end of this
+	qmult		queue length multiplier for high and super speed
+	host_addr	MAC address of host's end of this
 			Ethernet over USB link
-	dev_addr	- MAC address of device's end of this
+	dev_addr	MAC address of device's end of this
 			Ethernet over USB link
+	=============== ==================================================
 
 and after creating the functions/ncm.<instance name> they contain default
 values: qmult is 5, dev_addr and host_addr are randomly selected.
@@ -406,8 +462,13 @@ Testing the NCM function
 
 Configure IP addresses of the device and the host. Then:
 
-On the device: ping <host's IP>
-On the host: ping <device's IP>
+On the device::
+
+	ping <host's IP>
+
+On the host::
+
+	ping <device's IP>
 
 11. OBEX function
 =================
@@ -429,13 +490,18 @@ There can be at most 4 ACM/generic serial/OBEX ports in the system.
 Testing the OBEX function
 -------------------------
 
-On device: seriald -f /dev/ttyGS<Y> -s 1024
-On host: serialc -v <vendorID> -p <productID> -i<interface#> -a1 -s1024 \
-             -t<out endpoint addr> -r<in endpoint addr>
+On device::
+
+	seriald -f /dev/ttyGS<Y> -s 1024
+
+On host::
+
+	serialc -v <vendorID> -p <productID> -i<interface#> -a1 -s1024 \
+                -t<out endpoint addr> -r<in endpoint addr>
 
 where seriald and serialc are Felipe's utilities found here:
 
-https://github.com/felipebalbi/usb-tools.git master
+	https://github.com/felipebalbi/usb-tools.git master
 
 12. PHONET function
 ===================
@@ -448,8 +514,10 @@ Function-specific configfs interface
 The function name to use when creating the function directory is "phonet".
 The PHONET function provides just one attribute in its function directory:
 
-	ifname		- network device interface name associated with this
+	=============== ==================================================
+	ifname		network device interface name associated with this
 			function instance
+	=============== ==================================================
 
 Testing the PHONET function
 ---------------------------
@@ -464,41 +532,41 @@ These tools are required:
 
 git://git.gitorious.org/meego-cellular/phonet-utils.git
 
-On the host:
+On the host::
 
-$ ./phonet -a 0x10 -i usbpn0
-$ ./pnroute add 0x6c usbpn0
-$./pnroute add 0x10 usbpn0
-$ ifconfig usbpn0 up
+	$ ./phonet -a 0x10 -i usbpn0
+	$ ./pnroute add 0x6c usbpn0
+	$./pnroute add 0x10 usbpn0
+	$ ifconfig usbpn0 up
 
-On the device:
+On the device::
 
-$ ./phonet -a 0x6c -i upnlink0
-$ ./pnroute add 0x10 upnlink0
-$ ifconfig upnlink0 up
+	$ ./phonet -a 0x6c -i upnlink0
+	$ ./pnroute add 0x10 upnlink0
+	$ ifconfig upnlink0 up
 
-Then a test program can be used:
+Then a test program can be used::
 
-http://www.spinics.net/lists/linux-usb/msg85690.html
+	http://www.spinics.net/lists/linux-usb/msg85690.html
 
-On the device:
+On the device::
 
-$ ./pnxmit -a 0x6c -r
+	$ ./pnxmit -a 0x6c -r
 
-On the host:
+On the host::
 
-$ ./pnxmit -a 0x10 -s 0x6c
+	$ ./pnxmit -a 0x10 -s 0x6c
 
 As a result some data should be sent from host to device.
 Then the other way round:
 
-On the host:
+On the host::
 
-$ ./pnxmit -a 0x10 -r
+	$ ./pnxmit -a 0x10 -r
 
-On the device:
+On the device::
 
-$ ./pnxmit -a 0x6c -s 0x10
+	$ ./pnxmit -a 0x6c -s 0x10
 
 13. RNDIS function
 ==================
@@ -511,13 +579,15 @@ Function-specific configfs interface
 The function name to use when creating the function directory is "rndis".
 The RNDIS function provides these attributes in its function directory:
 
-	ifname		- network device interface name associated with this
+	=============== ==================================================
+	ifname		network device interface name associated with this
 			function instance
-	qmult		- queue length multiplier for high and super speed
-	host_addr	- MAC address of host's end of this
+	qmult		queue length multiplier for high and super speed
+	host_addr	MAC address of host's end of this
 			Ethernet over USB link
-	dev_addr	- MAC address of device's end of this
+	dev_addr	MAC address of device's end of this
 			Ethernet over USB link
+	=============== ==================================================
 
 and after creating the functions/rndis.<instance name> they contain default
 values: qmult is 5, dev_addr and host_addr are randomly selected.
@@ -530,8 +600,13 @@ Testing the RNDIS function
 
 Configure IP addresses of the device and the host. Then:
 
-On the device: ping <host's IP>
-On the host: ping <device's IP>
+On the device::
+
+	ping <host's IP>
+
+On the host::
+
+	ping <device's IP>
 
 14. SERIAL function
 ===================
@@ -553,15 +628,28 @@ There can be at most 4 ACM/generic serial/OBEX ports in the system.
 Testing the SERIAL function
 ---------------------------
 
-On host: insmod usbserial
-	 echo VID PID >/sys/bus/usb-serial/drivers/generic/new_id
-On host: cat > /dev/ttyUSB<X>
-On target: cat /dev/ttyGS<Y>
+On host::
+
+	insmod usbserial
+	echo VID PID >/sys/bus/usb-serial/drivers/generic/new_id
+
+On host::
+
+	cat > /dev/ttyUSB<X>
+
+On target::
+
+	cat /dev/ttyGS<Y>
 
 then the other way round
 
-On target: cat > /dev/ttyGS<Y>
-On host: cat /dev/ttyUSB<X>
+On target::
+
+	cat > /dev/ttyGS<Y>
+
+On host::
+
+	cat /dev/ttyUSB<X>
 
 15. SOURCESINK function
 =======================
@@ -574,24 +662,27 @@ Function-specific configfs interface
 The function name to use when creating the function directory is "SourceSink".
 The SOURCESINK function provides these attributes in its function directory:
 
-	pattern		- 0 (all zeros), 1 (mod63), 2 (none)
-	isoc_interval	- 1..16
-	isoc_maxpacket	- 0 - 1023 (fs), 0 - 1024 (hs/ss)
-	isoc_mult	- 0..2 (hs/ss only)
-	isoc_maxburst	- 0..15 (ss only)
-	bulk_buflen	- buffer length
-	bulk_qlen	- depth of queue for bulk
-	iso_qlen	- depth of queue for iso
+	=============== ==================================
+	pattern		0 (all zeros), 1 (mod63), 2 (none)
+	isoc_interval	1..16
+	isoc_maxpacket	0 - 1023 (fs), 0 - 1024 (hs/ss)
+	isoc_mult	0..2 (hs/ss only)
+	isoc_maxburst	0..15 (ss only)
+	bulk_buflen	buffer length
+	bulk_qlen	depth of queue for bulk
+	iso_qlen	depth of queue for iso
+	=============== ==================================
 
 Testing the SOURCESINK function
 -------------------------------
 
 device: run the gadget
+
 host: test-usb (tools/usb/testusb.c)
 
 
 16. UAC1 function (legacy implementation)
-=================
+=========================================
 
 The function is provided by usb_f_uac1_legacy.ko module.
 
@@ -602,12 +693,14 @@ The function name to use when creating the function directory
 is "uac1_legacy".
 The uac1 function provides these attributes in its function directory:
 
-	audio_buf_size - audio buffer size
-	fn_cap - capture pcm device file name
-	fn_cntl - control device file name
-	fn_play - playback pcm device file name
-	req_buf_size - ISO OUT endpoint request buffer size
-	req_count - ISO OUT endpoint request count
+	=============== ====================================
+	audio_buf_size	audio buffer size
+	fn_cap		capture pcm device file name
+	fn_cntl		control device file name
+	fn_play		playback pcm device file name
+	req_buf_size	ISO OUT endpoint request buffer size
+	req_count	ISO OUT endpoint request count
+	=============== ====================================
 
 The attributes have sane default values.
 
@@ -615,7 +708,10 @@ Testing the UAC1 function
 -------------------------
 
 device: run the gadget
-host: aplay -l # should list our USB Audio Gadget
+
+host::
+
+	aplay -l # should list our USB Audio Gadget
 
 17. UAC2 function
 =================
@@ -628,14 +724,16 @@ Function-specific configfs interface
 The function name to use when creating the function directory is "uac2".
 The uac2 function provides these attributes in its function directory:
 
-	c_chmask - capture channel mask
-	c_srate - capture sampling rate
-	c_ssize - capture sample size (bytes)
-	p_chmask - playback channel mask
-	p_srate - playback sampling rate
-	p_ssize - playback sample size (bytes)
-	req_number - the number of pre-allocated request for both capture
-		     and playback
+	=============== ====================================================
+	c_chmask	capture channel mask
+	c_srate		capture sampling rate
+	c_ssize		capture sample size (bytes)
+	p_chmask	playback channel mask
+	p_srate		playback sampling rate
+	p_ssize		playback sample size (bytes)
+	req_number	the number of pre-allocated request for both capture
+			and playback
+	=============== ====================================================
 
 The attributes have sane default values.
 
@@ -648,14 +746,14 @@ host: aplay -l # should list our USB Audio Gadget
 This function does not require real hardware support, it just
 sends a stream of audio data to/from the host. In order to
 actually hear something at the device side, a command similar
-to this must be used at the device side:
+to this must be used at the device side::
 
-$ arecord -f dat -t wav -D hw:2,0 | aplay -D hw:0,0 &
+	$ arecord -f dat -t wav -D hw:2,0 | aplay -D hw:0,0 &
 
-e.g.:
+e.g.::
 
-$ arecord -f dat -t wav -D hw:CARD=UAC2Gadget,DEV=0 | \
-aplay -D default:CARD=OdroidU3
+	$ arecord -f dat -t wav -D hw:CARD=UAC2Gadget,DEV=0 | \
+	  aplay -D default:CARD=OdroidU3
 
 18. UVC function
 ================
@@ -668,66 +766,73 @@ Function-specific configfs interface
 The function name to use when creating the function directory is "uvc".
 The uvc function provides these attributes in its function directory:
 
-	streaming_interval - interval for polling endpoint for data transfers
-	streaming_maxburst - bMaxBurst for super speed companion descriptor
-	streaming_maxpacket - maximum packet size this endpoint is capable of
-			      sending or receiving when this configuration is
-			      selected
+	=================== ================================================
+	streaming_interval  interval for polling endpoint for data transfers
+	streaming_maxburst  bMaxBurst for super speed companion descriptor
+	streaming_maxpacket maximum packet size this endpoint is capable of
+			    sending or receiving when this configuration is
+			    selected
+	=================== ================================================
 
 There are also "control" and "streaming" subdirectories, each of which contain
 a number of their subdirectories. There are some sane defaults provided, but
 the user must provide the following:
 
-	control header - create in control/header, link from control/class/fs
-			and/or control/class/ss
-	streaming header - create in streaming/header, link from
-			streaming/class/fs and/or streaming/class/hs and/or
-			streaming/class/ss
-	format description - create in streaming/mjpeg and/or
-			streaming/uncompressed
-	frame description - create in streaming/mjpeg/<format> and/or in
-			streaming/uncompressed/<format>
+	================== ====================================================
+	control header     create in control/header, link from control/class/fs
+			   and/or control/class/ss
+	streaming header   create in streaming/header, link from
+			   streaming/class/fs and/or streaming/class/hs and/or
+			   streaming/class/ss
+	format description create in streaming/mjpeg and/or
+			   streaming/uncompressed
+	frame description  create in streaming/mjpeg/<format> and/or in
+			   streaming/uncompressed/<format>
+	================== ====================================================
 
 Each frame description contains frame interval specification, and each
 such specification consists of a number of lines with an inverval value
-in each line. The rules stated above are best illustrated with an example:
-
-# mkdir functions/uvc.usb0/control/header/h
-# cd functions/uvc.usb0/control/
-# ln -s header/h class/fs
-# ln -s header/h class/ss
-# mkdir -p functions/uvc.usb0/streaming/uncompressed/u/360p
-# cat <<EOF > functions/uvc.usb0/streaming/uncompressed/u/360p/dwFrameInterval
-666666
-1000000
-5000000
-EOF
-# cd $GADGET_CONFIGFS_ROOT
-# mkdir functions/uvc.usb0/streaming/header/h
-# cd functions/uvc.usb0/streaming/header/h
-# ln -s ../../uncompressed/u
-# cd ../../class/fs
-# ln -s ../../header/h
-# cd ../../class/hs
-# ln -s ../../header/h
-# cd ../../class/ss
-# ln -s ../../header/h
+in each line. The rules stated above are best illustrated with an example::
+
+  # mkdir functions/uvc.usb0/control/header/h
+  # cd functions/uvc.usb0/control/
+  # ln -s header/h class/fs
+  # ln -s header/h class/ss
+  # mkdir -p functions/uvc.usb0/streaming/uncompressed/u/360p
+  # cat <<EOF > functions/uvc.usb0/streaming/uncompressed/u/360p/dwFrameInterval
+  666666
+  1000000
+  5000000
+  EOF
+  # cd $GADGET_CONFIGFS_ROOT
+  # mkdir functions/uvc.usb0/streaming/header/h
+  # cd functions/uvc.usb0/streaming/header/h
+  # ln -s ../../uncompressed/u
+  # cd ../../class/fs
+  # ln -s ../../header/h
+  # cd ../../class/hs
+  # ln -s ../../header/h
+  # cd ../../class/ss
+  # ln -s ../../header/h
 
 
 Testing the UVC function
 ------------------------
 
-device: run the gadget, modprobe vivid
+device: run the gadget, modprobe vivid::
 
-# uvc-gadget -u /dev/video<uvc video node #> -v /dev/video<vivid video node #>
+  # uvc-gadget -u /dev/video<uvc video node #> -v /dev/video<vivid video node #>
 
 where uvc-gadget is this program:
-http://git.ideasonboard.org/uvc-gadget.git
+	http://git.ideasonboard.org/uvc-gadget.git
 
 with these patches:
-http://www.spinics.net/lists/linux-usb/msg99220.html
 
-host: luvcview -f yuv
+	http://www.spinics.net/lists/linux-usb/msg99220.html
+
+host::
+
+	luvcview -f yuv
 
 19. PRINTER function
 ====================
@@ -740,16 +845,19 @@ Function-specific configfs interface
 The function name to use when creating the function directory is "printer".
 The printer function provides these attributes in its function directory:
 
-	pnp_string	- Data to be passed to the host in pnp string
-	q_len		- Number of requests per endpoint
+	==========	===========================================
+	pnp_string	Data to be passed to the host in pnp string
+	q_len		Number of requests per endpoint
+	==========	===========================================
 
 Testing the PRINTER function
 ----------------------------
 
 The most basic testing:
 
-device: run the gadget
-# ls -l /devices/virtual/usb_printer_gadget/
+device: run the gadget::
+
+	# ls -l /devices/virtual/usb_printer_gadget/
 
 should show g_printer<number>.
 
@@ -761,23 +869,28 @@ If udev is active, then e.g. /dev/usb/lp0 should appear.
 
 host->device transmission:
 
-device:
-# cat /dev/g_printer<number>
-host:
-# cat > /dev/usb/lp0
+device::
 
-device->host transmission:
+	# cat /dev/g_printer<number>
 
-# cat > /dev/g_printer<number>
-host:
-# cat /dev/usb/lp0
+host::
+
+	# cat > /dev/usb/lp0
+
+device->host transmission::
+
+	# cat > /dev/g_printer<number>
+
+host::
+
+	# cat /dev/usb/lp0
 
 More advanced testing can be done with the prn_example
 described in Documentation/usb/gadget_printer.txt.
 
 
 20. UAC1 function (virtual ALSA card, using u_audio API)
-=================
+========================================================
 
 The function is provided by usb_f_uac1.ko module.
 It will create a virtual ALSA card and the audio streams are simply
@@ -789,14 +902,16 @@ Function-specific configfs interface
 The function name to use when creating the function directory is "uac1".
 The uac1 function provides these attributes in its function directory:
 
-	c_chmask - capture channel mask
-	c_srate - capture sampling rate
-	c_ssize - capture sample size (bytes)
-	p_chmask - playback channel mask
-	p_srate - playback sampling rate
-	p_ssize - playback sample size (bytes)
-	req_number - the number of pre-allocated request for both capture
-		     and playback
+	========== ====================================================
+	c_chmask   capture channel mask
+	c_srate    capture sampling rate
+	c_ssize    capture sample size (bytes)
+	p_chmask   playback channel mask
+	p_srate    playback sampling rate
+	p_ssize    playback sample size (bytes)
+	req_number the number of pre-allocated request for both capture
+		   and playback
+	========== ====================================================
 
 The attributes have sane default values.
 
@@ -809,11 +924,11 @@ host: aplay -l # should list our USB Audio Gadget
 This function does not require real hardware support, it just
 sends a stream of audio data to/from the host. In order to
 actually hear something at the device side, a command similar
-to this must be used at the device side:
+to this must be used at the device side::
 
-$ arecord -f dat -t wav -D hw:2,0 | aplay -D hw:0,0 &
+	$ arecord -f dat -t wav -D hw:2,0 | aplay -D hw:0,0 &
 
-e.g.:
+e.g.::
 
-$ arecord -f dat -t wav -D hw:CARD=UAC1Gadget,DEV=0 | \
-aplay -D default:CARD=OdroidU3
+	$ arecord -f dat -t wav -D hw:CARD=UAC1Gadget,DEV=0 | \
+	  aplay -D default:CARD=OdroidU3
diff --git a/Documentation/usb/gadget_configfs.txt b/Documentation/usb/gadget_configfs.txt
index b8cb38a98c19..54fb08baae22 100644
--- a/Documentation/usb/gadget_configfs.txt
+++ b/Documentation/usb/gadget_configfs.txt
@@ -1,11 +1,9 @@
+============================================
+Linux USB gadget configured through configfs
+============================================
 
 
-
-
-		Linux USB gadget configured through configfs
-
-
-			     25th April 2013
+25th April 2013
 
 
 
@@ -26,7 +24,7 @@ Linux provides a number of functions for gadgets to use.
 Creating a gadget means deciding what configurations there will be
 and which functions each configuration will provide.
 
-Configfs (please see Documentation/filesystems/configfs/*) lends itself nicely
+Configfs (please see `Documentation/filesystems/configfs/*`) lends itself nicely
 for the purpose of telling the kernel about the above mentioned decision.
 This document is about how to do it.
 
@@ -51,44 +49,46 @@ Usage
 made available through configfs can be seen here:
 http://www.spinics.net/lists/linux-usb/msg76388.html)
 
-$ modprobe libcomposite
-$ mount none $CONFIGFS_HOME -t configfs
+::
+
+	$ modprobe libcomposite
+	$ mount none $CONFIGFS_HOME -t configfs
 
 where CONFIGFS_HOME is the mount point for configfs
 
 1. Creating the gadgets
 -----------------------
 
-For each gadget to be created its corresponding directory must be created:
+For each gadget to be created its corresponding directory must be created::
 
-$ mkdir $CONFIGFS_HOME/usb_gadget/<gadget name>
+	$ mkdir $CONFIGFS_HOME/usb_gadget/<gadget name>
 
-e.g.:
+e.g.::
 
-$ mkdir $CONFIGFS_HOME/usb_gadget/g1
+	$ mkdir $CONFIGFS_HOME/usb_gadget/g1
 
-...
-...
-...
+	...
+	...
+	...
 
-$ cd $CONFIGFS_HOME/usb_gadget/g1
+	$ cd $CONFIGFS_HOME/usb_gadget/g1
 
-Each gadget needs to have its vendor id <VID> and product id <PID> specified:
+Each gadget needs to have its vendor id <VID> and product id <PID> specified::
 
-$ echo <VID> > idVendor
-$ echo <PID> > idProduct
+	$ echo <VID> > idVendor
+	$ echo <PID> > idProduct
 
 A gadget also needs its serial number, manufacturer and product strings.
 In order to have a place to store them, a strings subdirectory must be created
-for each language, e.g.:
+for each language, e.g.::
 
-$ mkdir strings/0x409
+	$ mkdir strings/0x409
 
-Then the strings can be specified:
+Then the strings can be specified::
 
-$ echo <serial number> > strings/0x409/serialnumber
-$ echo <manufacturer> > strings/0x409/manufacturer
-$ echo <product> > strings/0x409/product
+	$ echo <serial number> > strings/0x409/serialnumber
+	$ echo <manufacturer> > strings/0x409/manufacturer
+	$ echo <product> > strings/0x409/product
 
 2. Creating the configurations
 ------------------------------
@@ -99,43 +99,43 @@ directories must be created:
 $ mkdir configs/<name>.<number>
 
 where <name> can be any string which is legal in a filesystem and the
-<number> is the configuration's number, e.g.:
+<number> is the configuration's number, e.g.::
 
-$ mkdir configs/c.1
+	$ mkdir configs/c.1
 
-...
-...
-...
+	...
+	...
+	...
 
 Each configuration also needs its strings, so a subdirectory must be created
-for each language, e.g.:
+for each language, e.g.::
 
-$ mkdir configs/c.1/strings/0x409
+	$ mkdir configs/c.1/strings/0x409
 
-Then the configuration string can be specified:
+Then the configuration string can be specified::
 
-$ echo <configuration> > configs/c.1/strings/0x409/configuration
+	$ echo <configuration> > configs/c.1/strings/0x409/configuration
 
-Some attributes can also be set for a configuration, e.g.:
+Some attributes can also be set for a configuration, e.g.::
 
-$ echo 120 > configs/c.1/MaxPower
+	$ echo 120 > configs/c.1/MaxPower
 
 3. Creating the functions
 -------------------------
 
 The gadget will provide some functions, for each function its corresponding
-directory must be created:
+directory must be created::
 
-$ mkdir functions/<name>.<instance name>
+	$ mkdir functions/<name>.<instance name>
 
 where <name> corresponds to one of allowed function names and instance name
-is an arbitrary string allowed in a filesystem, e.g.:
+is an arbitrary string allowed in a filesystem, e.g.::
 
-$ mkdir functions/ncm.usb0 # usb_f_ncm.ko gets loaded with request_module()
+  $ mkdir functions/ncm.usb0 # usb_f_ncm.ko gets loaded with request_module()
 
-...
-...
-...
+  ...
+  ...
+  ...
 
 Each function provides its specific set of attributes, with either read-only
 or read-write access. Where applicable they need to be written to as
@@ -149,17 +149,17 @@ At this moment a number of gadgets is created, each of which has a number of
 configurations specified and a number of functions available. What remains
 is specifying which function is available in which configuration (the same
 function can be used in multiple configurations). This is achieved with
-creating symbolic links:
+creating symbolic links::
 
-$ ln -s functions/<name>.<instance name> configs/<name>.<number>
+	$ ln -s functions/<name>.<instance name> configs/<name>.<number>
 
-e.g.:
+e.g.::
 
-$ ln -s functions/ncm.usb0 configs/c.1
+	$ ln -s functions/ncm.usb0 configs/c.1
 
-...
-...
-...
+	...
+	...
+	...
 
 5. Enabling the gadget
 ----------------------
@@ -167,123 +167,127 @@ $ ln -s functions/ncm.usb0 configs/c.1
 All the above steps serve the purpose of composing the gadget of
 configurations and functions.
 
-An example directory structure might look like this:
-
-.
-./strings
-./strings/0x409
-./strings/0x409/serialnumber
-./strings/0x409/product
-./strings/0x409/manufacturer
-./configs
-./configs/c.1
-./configs/c.1/ncm.usb0 -> ../../../../usb_gadget/g1/functions/ncm.usb0
-./configs/c.1/strings
-./configs/c.1/strings/0x409
-./configs/c.1/strings/0x409/configuration
-./configs/c.1/bmAttributes
-./configs/c.1/MaxPower
-./functions
-./functions/ncm.usb0
-./functions/ncm.usb0/ifname
-./functions/ncm.usb0/qmult
-./functions/ncm.usb0/host_addr
-./functions/ncm.usb0/dev_addr
-./UDC
-./bcdUSB
-./bcdDevice
-./idProduct
-./idVendor
-./bMaxPacketSize0
-./bDeviceProtocol
-./bDeviceSubClass
-./bDeviceClass
+An example directory structure might look like this::
+
+  .
+  ./strings
+  ./strings/0x409
+  ./strings/0x409/serialnumber
+  ./strings/0x409/product
+  ./strings/0x409/manufacturer
+  ./configs
+  ./configs/c.1
+  ./configs/c.1/ncm.usb0 -> ../../../../usb_gadget/g1/functions/ncm.usb0
+  ./configs/c.1/strings
+  ./configs/c.1/strings/0x409
+  ./configs/c.1/strings/0x409/configuration
+  ./configs/c.1/bmAttributes
+  ./configs/c.1/MaxPower
+  ./functions
+  ./functions/ncm.usb0
+  ./functions/ncm.usb0/ifname
+  ./functions/ncm.usb0/qmult
+  ./functions/ncm.usb0/host_addr
+  ./functions/ncm.usb0/dev_addr
+  ./UDC
+  ./bcdUSB
+  ./bcdDevice
+  ./idProduct
+  ./idVendor
+  ./bMaxPacketSize0
+  ./bDeviceProtocol
+  ./bDeviceSubClass
+  ./bDeviceClass
 
 
 Such a gadget must be finally enabled so that the USB host can enumerate it.
-In order to enable the gadget it must be bound to a UDC (USB Device Controller).
 
-$ echo <udc name> > UDC
+In order to enable the gadget it must be bound to a UDC (USB Device
+Controller)::
+
+	$ echo <udc name> > UDC
 
 where <udc name> is one of those found in /sys/class/udc/*
-e.g.:
+e.g.::
 
-$ echo s3c-hsotg > UDC
+	$ echo s3c-hsotg > UDC
 
 
 6. Disabling the gadget
 -----------------------
 
-$ echo "" > UDC
+::
+
+	$ echo "" > UDC
 
 7. Cleaning up
 --------------
 
-Remove functions from configurations:
+Remove functions from configurations::
 
-$ rm configs/<config name>.<number>/<function>
+	$ rm configs/<config name>.<number>/<function>
 
 where <config name>.<number> specify the configuration and <function> is
-a symlink to a function being removed from the configuration, e.g.:
+a symlink to a function being removed from the configuration, e.g.::
 
-$ rm configs/c.1/ncm.usb0
+	$ rm configs/c.1/ncm.usb0
 
-...
-...
-...
+	...
+	...
+	...
 
-Remove strings directories in configurations
+Remove strings directories in configurations:
 
-$ rmdir configs/<config name>.<number>/strings/<lang>
+	$ rmdir configs/<config name>.<number>/strings/<lang>
 
-e.g.:
+e.g.::
 
-$ rmdir configs/c.1/strings/0x409
+	$ rmdir configs/c.1/strings/0x409
 
-...
-...
-...
+	...
+	...
+	...
 
-and remove the configurations
+and remove the configurations::
 
-$ rmdir configs/<config name>.<number>
+	$ rmdir configs/<config name>.<number>
 
-e.g.:
+e.g.::
 
-rmdir configs/c.1
+	rmdir configs/c.1
 
-...
-...
-...
+	...
+	...
+	...
 
-Remove functions (function modules are not unloaded, though)
+Remove functions (function modules are not unloaded, though):
 
-$ rmdir functions/<name>.<instance name>
+	$ rmdir functions/<name>.<instance name>
 
-e.g.:
+e.g.::
 
-$ rmdir functions/ncm.usb0
+	$ rmdir functions/ncm.usb0
 
-...
-...
-...
+	...
+	...
+	...
 
-Remove strings directories in the gadget
+Remove strings directories in the gadget::
 
-$ rmdir strings/<lang>
+	$ rmdir strings/<lang>
 
-e.g.:
+e.g.::
 
-$ rmdir strings/0x409
+	$ rmdir strings/0x409
 
-and finally remove the gadget:
+and finally remove the gadget::
 
-$ cd ..
-$ rmdir <gadget name>
+	$ cd ..
+	$ rmdir <gadget name>
 
-e.g.:
+e.g.::
 
-$ rmdir g1
+	$ rmdir g1
 
 
 
@@ -305,16 +309,16 @@ configured elements. However, they are embedded in usage-specific
 larger structures. In the picture below there is a "cs" which contains
 a config_item and an "sa" which contains a configfs_attribute.
 
-The filesystem view would be like this:
+The filesystem view would be like this::
 
-./
-./cs        (directory)
-   |
-   +--sa    (file)
-   |
-   .
-   .
-   .
+  ./
+  ./cs        (directory)
+     |
+     +--sa    (file)
+     |
+     .
+     .
+     .
 
 Whenever a user reads/writes the "sa" file, a function is called
 which accepts a struct config_item and a struct configfs_attribute.
@@ -326,29 +330,31 @@ buffer), while the "store" is for modifying the file's contents (copy data
 from the buffer to the cs), but it is up to the implementer of the
 two functions to decide what they actually do.
 
-typedef struct configured_structure cs;
-typedef struct specific_attribute sa;
-
-                                       sa
-                       +----------------------------------+
-        cs             |  (*show)(cs *, buffer);          |
-+-----------------+    |  (*store)(cs *, buffer, length); |
-|                 |    |                                  |
-| +-------------+ |    |       +------------------+       |
-| | struct      |-|----|------>|struct            |       |
-| | config_item | |    |       |configfs_attribute|       |
-| +-------------+ |    |       +------------------+       |
-|                 |    +----------------------------------+
-| data to be set  |                .
-|                 |                .
-+-----------------+                .
+::
+
+  typedef struct configured_structure cs;
+  typedef struct specific_attribute sa;
+
+                                         sa
+                         +----------------------------------+
+          cs             |  (*show)(cs *, buffer);          |
+  +-----------------+    |  (*store)(cs *, buffer, length); |
+  |                 |    |                                  |
+  | +-------------+ |    |       +------------------+       |
+  | | struct      |-|----|------>|struct            |       |
+  | | config_item | |    |       |configfs_attribute|       |
+  | +-------------+ |    |       +------------------+       |
+  |                 |    +----------------------------------+
+  | data to be set  |                .
+  |                 |                .
+  +-----------------+                .
 
 The file names are decided by the config item/group designer, while
 the directories in general can be named at will. A group can have
 a number of its default sub-groups created automatically.
 
 For more information on configfs please see
-Documentation/filesystems/configfs/*.
+`Documentation/filesystems/configfs/*`.
 
 The concepts described above translate to USB gadgets like this:
 
diff --git a/Documentation/usb/gadget_hid.txt b/Documentation/usb/gadget_hid.txt
index 7a0fb8e16e27..098d563040cc 100644
--- a/Documentation/usb/gadget_hid.txt
+++ b/Documentation/usb/gadget_hid.txt
@@ -1,28 +1,31 @@
-
-		     Linux USB HID gadget driver
+===========================
+Linux USB HID gadget driver
+===========================
 
 Introduction
+============
 
-	The HID Gadget driver provides emulation of USB Human Interface
-	Devices (HID). The basic HID handling is done in the kernel,
-	and HID reports can be sent/received through I/O on the
-	/dev/hidgX character devices.
+The HID Gadget driver provides emulation of USB Human Interface
+Devices (HID). The basic HID handling is done in the kernel,
+and HID reports can be sent/received through I/O on the
+/dev/hidgX character devices.
 
-	For more details about HID, see the developer page on
-	http://www.usb.org/developers/hidpage/
+For more details about HID, see the developer page on
+http://www.usb.org/developers/hidpage/
 
 Configuration
+=============
 
-	g_hid is a platform driver, so to use it you need to add
-	struct platform_device(s) to your platform code defining the
-	HID function descriptors you want to use - E.G. something
-	like:
+g_hid is a platform driver, so to use it you need to add
+struct platform_device(s) to your platform code defining the
+HID function descriptors you want to use - E.G. something
+like::
 
-#include <linux/platform_device.h>
-#include <linux/usb/g_hid.h>
+  #include <linux/platform_device.h>
+  #include <linux/usb/g_hid.h>
 
-/* hid descriptor for a keyboard */
-static struct hidg_func_descriptor my_hid_data = {
+  /* hid descriptor for a keyboard */
+  static struct hidg_func_descriptor my_hid_data = {
 	.subclass		= 0, /* No subclass */
 	.protocol		= 1, /* Keyboard */
 	.report_length		= 8,
@@ -61,85 +64,87 @@ static struct hidg_func_descriptor my_hid_data = {
 		0x81, 0x00,	/*   INPUT (Data,Ary,Abs)                 */
 		0xc0		/* END_COLLECTION                         */
 	}
-};
+  };
 
-static struct platform_device my_hid = {
+  static struct platform_device my_hid = {
 	.name			= "hidg",
 	.id			= 0,
 	.num_resources		= 0,
 	.resource		= 0,
 	.dev.platform_data	= &my_hid_data,
-};
+  };
 
-	You can add as many HID functions as you want, only limited by
-	the amount of interrupt endpoints your gadget driver supports.
+You can add as many HID functions as you want, only limited by
+the amount of interrupt endpoints your gadget driver supports.
 
 Configuration with configfs
+===========================
 
-	Instead of adding fake platform devices and drivers in order to pass
-	some data to the kernel, if HID is a part of a gadget composed with
-	configfs the hidg_func_descriptor.report_desc is passed to the kernel
-	by writing the appropriate stream of bytes to a configfs attribute.
+Instead of adding fake platform devices and drivers in order to pass
+some data to the kernel, if HID is a part of a gadget composed with
+configfs the hidg_func_descriptor.report_desc is passed to the kernel
+by writing the appropriate stream of bytes to a configfs attribute.
 
 Send and receive HID reports
+============================
 
-	HID reports can be sent/received using read/write on the
-	/dev/hidgX character devices. See below for an example program
-	to do this.
+HID reports can be sent/received using read/write on the
+/dev/hidgX character devices. See below for an example program
+to do this.
 
-	hid_gadget_test is a small interactive program to test the HID
-	gadget driver. To use, point it at a hidg device and set the
-	device type (keyboard / mouse / joystick) - E.G.:
+hid_gadget_test is a small interactive program to test the HID
+gadget driver. To use, point it at a hidg device and set the
+device type (keyboard / mouse / joystick) - E.G.::
 
-		# hid_gadget_test /dev/hidg0 keyboard
+	# hid_gadget_test /dev/hidg0 keyboard
 
-	You are now in the prompt of hid_gadget_test. You can type any
-	combination of options and values. Available options and
-	values are listed at program start. In keyboard mode you can
-	send up to six values.
+You are now in the prompt of hid_gadget_test. You can type any
+combination of options and values. Available options and
+values are listed at program start. In keyboard mode you can
+send up to six values.
 
-	For example type: g i s t r --left-shift
+For example type: g i s t r --left-shift
 
-	Hit return and the corresponding report will be sent by the
-	HID gadget.
+Hit return and the corresponding report will be sent by the
+HID gadget.
 
-	Another interesting example is the caps lock test. Type
-	--caps-lock and hit return. A report is then sent by the
-	gadget and you should receive the host answer, corresponding
-	to the caps lock LED status.
+Another interesting example is the caps lock test. Type
+--caps-lock and hit return. A report is then sent by the
+gadget and you should receive the host answer, corresponding
+to the caps lock LED status::
 
-		--caps-lock
-		recv report:2
+	--caps-lock
+	recv report:2
 
-	With this command:
+With this command::
 
-		# hid_gadget_test /dev/hidg1 mouse
+	# hid_gadget_test /dev/hidg1 mouse
 
-	You can test the mouse emulation. Values are two signed numbers.
+You can test the mouse emulation. Values are two signed numbers.
 
 
-Sample code
+Sample code::
 
-/* hid_gadget_test */
+    /* hid_gadget_test */
 
-#include <pthread.h>
-#include <string.h>
-#include <stdio.h>
-#include <ctype.h>
-#include <fcntl.h>
-#include <errno.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
+    #include <pthread.h>
+    #include <string.h>
+    #include <stdio.h>
+    #include <ctype.h>
+    #include <fcntl.h>
+    #include <errno.h>
+    #include <stdio.h>
+    #include <stdlib.h>
+    #include <unistd.h>
 
-#define BUF_LEN 512
+    #define BUF_LEN 512
 
-struct options {
+    struct options {
 	const char    *opt;
 	unsigned char val;
-};
+  };
 
-static struct options kmod[] = {
+  static struct options kmod[] = {
 	{.opt = "--left-ctrl",		.val = 0x01},
 	{.opt = "--right-ctrl",		.val = 0x10},
 	{.opt = "--left-shift",		.val = 0x02},
@@ -149,9 +154,9 @@ static struct options kmod[] = {
 	{.opt = "--left-meta",		.val = 0x08},
 	{.opt = "--right-meta",		.val = 0x80},
 	{.opt = NULL}
-};
+  };
 
-static struct options kval[] = {
+  static struct options kval[] = {
 	{.opt = "--return",	.val = 0x28},
 	{.opt = "--esc",	.val = 0x29},
 	{.opt = "--bckspc",	.val = 0x2a},
@@ -183,10 +188,10 @@ static struct options kval[] = {
 	{.opt = "--up",		.val = 0x52},
 	{.opt = "--num-lock",	.val = 0x53},
 	{.opt = NULL}
-};
+  };
 
-int keyboard_fill_report(char report[8], char buf[BUF_LEN], int *hold)
-{
+  int keyboard_fill_report(char report[8], char buf[BUF_LEN], int *hold)
+  {
 	char *tok = strtok(buf, " ");
 	int key = 0;
 	int i = 0;
@@ -229,17 +234,17 @@ int keyboard_fill_report(char report[8], char buf[BUF_LEN], int *hold)
 			fprintf(stderr, "unknown option: %s\n", tok);
 	}
 	return 8;
-}
+  }
 
-static struct options mmod[] = {
+  static struct options mmod[] = {
 	{.opt = "--b1", .val = 0x01},
 	{.opt = "--b2", .val = 0x02},
 	{.opt = "--b3", .val = 0x04},
 	{.opt = NULL}
-};
+  };
 
-int mouse_fill_report(char report[8], char buf[BUF_LEN], int *hold)
-{
+  int mouse_fill_report(char report[8], char buf[BUF_LEN], int *hold)
+  {
 	char *tok = strtok(buf, " ");
 	int mvt = 0;
 	int i = 0;
@@ -274,9 +279,9 @@ int mouse_fill_report(char report[8], char buf[BUF_LEN], int *hold)
 		fprintf(stderr, "unknown option: %s\n", tok);
 	}
 	return 3;
-}
+  }
 
-static struct options jmod[] = {
+  static struct options jmod[] = {
 	{.opt = "--b1",		.val = 0x10},
 	{.opt = "--b2",		.val = 0x20},
 	{.opt = "--b3",		.val = 0x40},
@@ -287,10 +292,10 @@ static struct options jmod[] = {
 	{.opt = "--hat4",	.val = 0x03},
 	{.opt = "--hatneutral",	.val = 0x04},
 	{.opt = NULL}
-};
+  };
 
-int joystick_fill_report(char report[8], char buf[BUF_LEN], int *hold)
-{
+  int joystick_fill_report(char report[8], char buf[BUF_LEN], int *hold)
+  {
 	char *tok = strtok(buf, " ");
 	int mvt = 0;
 	int i = 0;
@@ -326,10 +331,10 @@ int joystick_fill_report(char report[8], char buf[BUF_LEN], int *hold)
 		fprintf(stderr, "unknown option: %s\n", tok);
 	}
 	return 4;
-}
+  }
 
-void print_options(char c)
-{
+  void print_options(char c)
+  {
 	int i = 0;
 
 	if (c == 'k') {
@@ -358,10 +363,10 @@ void print_options(char c)
 		       "		three signed numbers\n"
 		       "--quit to close\n");
 	}
-}
+  }
 
-int main(int argc, const char *argv[])
-{
+  int main(int argc, const char *argv[])
+  {
 	const char *filename = NULL;
 	int fd = 0;
 	char buf[BUF_LEN];
@@ -449,4 +454,4 @@ int main(int argc, const char *argv[])
 
 	close(fd);
 	return 0;
-}
+  }
diff --git a/Documentation/usb/gadget_multi.txt b/Documentation/usb/gadget_multi.txt
index b3146dd7aa43..9806b55af301 100644
--- a/Documentation/usb/gadget_multi.txt
+++ b/Documentation/usb/gadget_multi.txt
@@ -1,6 +1,9 @@
-                                                             -*- org -*-
+==============================
+Multifunction Composite Gadget
+==============================
 
-* Overview
+Overview
+========
 
 The Multifunction Composite Gadget (or g_multi) is a composite gadget
 that makes extensive use of the composite framework to provide
@@ -17,13 +20,15 @@ have two configurations -- one with RNDIS and another with CDC ECM[3].
 Please note that if you use non-standard configuration (that is enable
 CDC ECM) you may need to change vendor and/or product ID.
 
-* Host drivers
+Host drivers
+============
 
 To make use of the gadget one needs to make it work on host side --
 without that there's no hope of achieving anything with the gadget.
 As one might expect, things one need to do very from system to system.
 
-** Linux host drivers
+Linux host drivers
+------------------
 
 Since the gadget uses standard composite framework and appears as such
 to Linux host it does not need any additional drivers on Linux host
@@ -34,11 +39,13 @@ This is also true for two configuration set-up with RNDIS
 configuration being the first one.  Linux host will use the second
 configuration with CDC ECM which should work better under Linux.
 
-** Windows host drivers
+Windows host drivers
+--------------------
 
 For the gadget to work under Windows two conditions have to be met:
 
-*** Detecting as composite gadget
+Detecting as composite gadget
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 First of all, Windows need to detect the gadget as an USB composite
 gadget which on its own have some conditions[4].  If they are met,
@@ -53,7 +60,8 @@ The only thing to worry is that the gadget has to have a single
 configuration so a dual RNDIS and CDC ECM gadget won't work unless you
 create a proper INF -- and of course, if you do submit it!
 
-*** Installing drivers for each function
+Installing drivers for each function
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 The other, trickier thing is making Windows install drivers for each
 individual function.
@@ -63,7 +71,8 @@ implementing USB Mass Storage class and selects appropriate driver.
 
 Things are harder with RDNIS and CDC ACM.
 
-**** RNDIS
+RNDIS
+.....
 
 To make Windows select RNDIS drivers for the first function in the
 gadget, one needs to use the [[file:linux.inf]] file provided with this
@@ -75,11 +84,13 @@ RNDIS was not the first interface.  You do not need to worry abut it
 unless you are trying to develop your own gadget in which case watch
 out for this bug.
 
-**** CDC ACM
+CDC ACM
+.......
 
 Similarly, [[file:linux-cdc-acm.inf]] is provided for CDC ACM.
 
-**** Customising the gadget
+Customising the gadget
+......................
 
 If you intend to hack the g_multi gadget be advised that rearranging
 functions will obviously change interface numbers for each of the
@@ -97,14 +108,16 @@ things don't work as intended before realising Windows have cached
 some drivers information (changing USB port may sometimes help plus
 you might try using USBDeview[8] to remove the phantom device).
 
-**** INF testing
+INF testing
+...........
 
 Provided INF files have been tested on Windows XP SP3, Windows Vista
 and Windows 7, all 32-bit versions.  It should work on 64-bit versions
 as well.  It most likely won't work on Windows prior to Windows XP
 SP2.
 
-** Other systems
+Other systems
+-------------
 
 At this moment, drivers for any other systems have not been tested.
 Knowing how MacOS is based on BSD and BSD is an Open Source it is
@@ -115,7 +128,8 @@ For more exotic systems I have even less to say...
 
 Any testing and drivers *are* *welcome*!
 
-* Authors
+Authors
+=======
 
 This document has been written by Michal Nazarewicz
 ([[mailto:mina86@mina86.com]]).  INF files have been hacked with
@@ -124,7 +138,8 @@ Xiaofan Chen ([[mailto:xiaofanc@gmail.com]]) basing on the MS RNDIS
 template[9], Microchip's CDC ACM INF file and David Brownell's
 ([[mailto:dbrownell@users.sourceforge.net]]) original INF files.
 
-* Footnotes
+Footnotes
+=========
 
 [1] Remote Network Driver Interface Specification,
 [[http://msdn.microsoft.com/en-us/library/ee484414.aspx]].
diff --git a/Documentation/usb/gadget_printer.txt b/Documentation/usb/gadget_printer.txt
index ad995bf0db41..5e5516c69075 100644
--- a/Documentation/usb/gadget_printer.txt
+++ b/Documentation/usb/gadget_printer.txt
@@ -1,12 +1,14 @@
+===============================
+Linux USB Printer Gadget Driver
+===============================
 
-                       Linux USB Printer Gadget Driver
-                                 06/04/2007
+06/04/2007
 
-              Copyright (C) 2007 Craig W. Nadler <craig@nadler.us>
+Copyright (C) 2007 Craig W. Nadler <craig@nadler.us>
 
 
 
-GENERAL
+General
 =======
 
 This driver may be used if you are writing printer firmware using Linux as
@@ -29,52 +31,60 @@ user space firmware can read or write this status byte using a device file
 
 
 
-HOWTO USE THIS DRIVER
+Howto Use This Driver
 =====================
 
 To load the USB device controller driver and the printer gadget driver. The
-following example uses the Netchip 2280 USB device controller driver:
+following example uses the Netchip 2280 USB device controller driver::
 
-modprobe net2280
-modprobe g_printer
+	modprobe net2280
+	modprobe g_printer
 
 
 The follow command line parameter can be used when loading the printer gadget
 (ex: modprobe g_printer idVendor=0x0525 idProduct=0xa4a8 ):
 
-idVendor - This is the Vendor ID used in the device descriptor. The default is
+idVendor
+	This is the Vendor ID used in the device descriptor. The default is
 	the Netchip vendor id 0x0525. YOU MUST CHANGE TO YOUR OWN VENDOR ID
 	BEFORE RELEASING A PRODUCT. If you plan to release a product and don't
 	already have a Vendor ID please see www.usb.org for details on how to
 	get one.
 
-idProduct - This is the Product ID used in the device descriptor. The default
+idProduct
+	This is the Product ID used in the device descriptor. The default
 	is 0xa4a8, you should change this to an ID that's not used by any of
 	your other USB products if you have any. It would be a good idea to
 	start numbering your products starting with say 0x0001.
 
-bcdDevice - This is the version number of your product. It would be a good idea
+bcdDevice
+	This is the version number of your product. It would be a good idea
 	to put your firmware version here.
 
-iManufacturer - A string containing the name of the Vendor.
+iManufacturer
+	A string containing the name of the Vendor.
 
-iProduct - A string containing the Product Name.
+iProduct
+	A string containing the Product Name.
 
-iSerialNum - A string containing the Serial Number. This should be changed for
+iSerialNum
+	A string containing the Serial Number. This should be changed for
 	each unit of your product.
 
-iPNPstring -  The PNP ID string used for this printer. You will want to set
+iPNPstring
+	The PNP ID string used for this printer. You will want to set
 	either on the command line or hard code the PNP ID string used for
 	your printer product.
 
-qlen - The number of 8k buffers to use per endpoint. The default is 10, you
+qlen
+	The number of 8k buffers to use per endpoint. The default is 10, you
 	should tune this for your product. You may also want to tune the
 	size of each buffer for your product.
 
 
 
 
-USING THE EXAMPLE CODE
+Using The Example Code
 ======================
 
 This example code talks to stdout, instead of a print engine.
@@ -82,22 +92,23 @@ This example code talks to stdout, instead of a print engine.
 To compile the test code below:
 
 1) save it to a file called prn_example.c
-2) compile the code with the follow command:
+2) compile the code with the follow command::
+
 	 gcc prn_example.c -o prn_example
 
 
 
-To read printer data from the host to stdout:
+To read printer data from the host to stdout::
 
 	# prn_example -read_data
 
 
-To write printer data from a file (data_file) to the host:
+To write printer data from a file (data_file) to the host::
 
 	# cat data_file | prn_example -write_data
 
 
-To get the current printer status for the gadget driver:
+To get the current printer status for the gadget driver:::
 
 	# prn_example -get_status
 
@@ -107,60 +118,62 @@ To get the current printer status for the gadget driver:
 	     Printer OK
 
 
-To set printer to Selected/On-line:
+To set printer to Selected/On-line::
 
 	# prn_example -selected
 
 
-To set printer to Not Selected/Off-line:
+To set printer to Not Selected/Off-line::
 
 	# prn_example -not_selected
 
 
-To set paper status to paper out:
+To set paper status to paper out::
 
 	# prn_example -paper_out
 
 
-To set paper status to paper loaded:
+To set paper status to paper loaded::
 
 	# prn_example -paper_loaded
 
 
-To set error status to printer OK:
+To set error status to printer OK::
 
 	# prn_example -no_error
 
 
-To set error status to ERROR:
+To set error status to ERROR::
 
 	# prn_example -error
 
 
 
 
-EXAMPLE CODE
+Example Code
 ============
 
+::
+
 
-#include <stdio.h>
-#include <stdlib.h>
-#include <fcntl.h>
-#include <linux/poll.h>
-#include <sys/ioctl.h>
-#include <linux/usb/g_printer.h>
+  #include <stdio.h>
+  #include <stdlib.h>
+  #include <fcntl.h>
+  #include <linux/poll.h>
+  #include <sys/ioctl.h>
+  #include <linux/usb/g_printer.h>
 
-#define PRINTER_FILE			"/dev/g_printer"
-#define BUF_SIZE			512
+  #define PRINTER_FILE			"/dev/g_printer"
+  #define BUF_SIZE			512
 
 
-/*
- * 'usage()' - Show program usage.
- */
+  /*
+   * 'usage()' - Show program usage.
+   */
 
-static void
-usage(const char *option)		/* I - Option string or NULL */
-{
+  static void
+  usage(const char *option)		/* I - Option string or NULL */
+  {
 	if (option) {
 		fprintf(stderr,"prn_example: Unknown option \"%s\"!\n",
 				option);
@@ -186,12 +199,12 @@ usage(const char *option)		/* I - Option string or NULL */
 	fputs("\n\n", stderr);
 
 	exit(1);
-}
+  }
 
 
-static int
-read_printer_data()
-{
+  static int
+  read_printer_data()
+  {
 	struct pollfd	fd[1];
 
 	/* Open device file for printer gadget. */
@@ -236,12 +249,12 @@ read_printer_data()
 	close(fd[0].fd);
 
 	return 0;
-}
+  }
 
 
-static int
-write_printer_data()
-{
+  static int
+  write_printer_data()
+  {
 	struct pollfd	fd[1];
 
 	/* Open device file for printer gadget. */
@@ -295,12 +308,12 @@ write_printer_data()
 	close(fd[0].fd);
 
 	return 0;
-}
+  }
 
 
-static int
-read_NB_printer_data()
-{
+  static int
+  read_NB_printer_data()
+  {
 	int		fd;
 	static char	buf[BUF_SIZE];
 	int		bytes_read;
@@ -329,12 +342,12 @@ read_NB_printer_data()
 	close(fd);
 
 	return 0;
-}
+  }
 
 
-static int
-get_printer_status()
-{
+  static int
+  get_printer_status()
+  {
 	int	retval;
 	int	fd;
 
@@ -357,12 +370,12 @@ get_printer_status()
 	close(fd);
 
 	return(retval);
-}
+  }
 
 
-static int
-set_printer_status(unsigned char buf, int clear_printer_status_bit)
-{
+  static int
+  set_printer_status(unsigned char buf, int clear_printer_status_bit)
+  {
 	int	retval;
 	int	fd;
 
@@ -397,12 +410,12 @@ set_printer_status(unsigned char buf, int clear_printer_status_bit)
 	close(fd);
 
 	return 0;
-}
+  }
 
 
-static int
-display_printer_status()
-{
+  static int
+  display_printer_status()
+  {
 	char	printer_status;
 
 	printer_status = get_printer_status();
@@ -429,12 +442,12 @@ display_printer_status()
 	}
 
 	return(0);
-}
+  }
 
 
-int
-main(int  argc, char *argv[])
-{
+  int
+  main(int  argc, char *argv[])
+  {
 	int	i;		/* Looping var */
 	int	retval = 0;
 
@@ -507,4 +520,4 @@ main(int  argc, char *argv[])
 	}
 
 	exit(retval);
-}
+  }
diff --git a/Documentation/usb/gadget_serial.txt b/Documentation/usb/gadget_serial.txt
index d1def3186782..dce8bc1fb1f2 100644
--- a/Documentation/usb/gadget_serial.txt
+++ b/Documentation/usb/gadget_serial.txt
@@ -1,7 +1,10 @@
+===============================
+Linux Gadget Serial Driver v2.0
+===============================
 
-                 Linux Gadget Serial Driver v2.0
-                           11/20/2004
-                  (updated 8-May-2008 for v2.3)
+11/20/2004
+
+(updated 8-May-2008 for v2.3)
 
 
 License and Disclaimer
@@ -56,7 +59,7 @@ hardware; for example, a PDA, an embedded Linux system, or a PC
 with a USB development card.
 
 The gadget serial driver talks over USB to either a CDC ACM driver
-or a generic USB serial driver running on a host PC.
+or a generic USB serial driver running on a host PC::
 
    Host
    --------------------------------------
@@ -112,11 +115,11 @@ configuring the kernel.  Then rebuild and install the kernel or
 modules.
 
 Then you must load the gadget serial driver.  To load it as an
-ACM device (recommended for interoperability), do this:
+ACM device (recommended for interoperability), do this::
 
   modprobe g_serial
 
-To load it as a vendor specific bulk in/out device, do this:
+To load it as a vendor specific bulk in/out device, do this::
 
   modprobe g_serial use_acm=0
 
@@ -127,7 +130,7 @@ desired.
 
 Your system should use mdev (from busybox) or udev to make the
 device nodes.  After this gadget driver has been set up you should
-then see a /dev/ttyGS0 node:
+then see a /dev/ttyGS0 node::
 
   # ls -l /dev/ttyGS0 | cat
   crw-rw----    1 root     root     253,   0 May  8 14:10 /dev/ttyGS0
@@ -187,24 +190,24 @@ support".
 
 Once the gadget serial driver is loaded and the USB device connected
 to the Linux host with a USB cable, the host system should recognize
-the gadget serial device.  For example, the command
+the gadget serial device.  For example, the command::
 
   cat /sys/kernel/debug/usb/devices
 
-should show something like this:
-
-T:  Bus=01 Lev=01 Prnt=01 Port=01 Cnt=02 Dev#=  5 Spd=480 MxCh= 0
-D:  Ver= 2.00 Cls=02(comm.) Sub=00 Prot=00 MxPS=64 #Cfgs=  1
-P:  Vendor=0525 ProdID=a4a7 Rev= 2.01
-S:  Manufacturer=Linux 2.6.8.1 with net2280
-S:  Product=Gadget Serial
-S:  SerialNumber=0
-C:* #Ifs= 2 Cfg#= 2 Atr=c0 MxPwr=  2mA
-I:  If#= 0 Alt= 0 #EPs= 1 Cls=02(comm.) Sub=02 Prot=01 Driver=acm
-E:  Ad=83(I) Atr=03(Int.) MxPS=   8 Ivl=32ms
-I:  If#= 1 Alt= 0 #EPs= 2 Cls=0a(data ) Sub=00 Prot=00 Driver=acm
-E:  Ad=81(I) Atr=02(Bulk) MxPS= 512 Ivl=0ms
-E:  Ad=02(O) Atr=02(Bulk) MxPS= 512 Ivl=0ms
+should show something like this:::
+
+  T:  Bus=01 Lev=01 Prnt=01 Port=01 Cnt=02 Dev#=  5 Spd=480 MxCh= 0
+  D:  Ver= 2.00 Cls=02(comm.) Sub=00 Prot=00 MxPS=64 #Cfgs=  1
+  P:  Vendor=0525 ProdID=a4a7 Rev= 2.01
+  S:  Manufacturer=Linux 2.6.8.1 with net2280
+  S:  Product=Gadget Serial
+  S:  SerialNumber=0
+  C:* #Ifs= 2 Cfg#= 2 Atr=c0 MxPwr=  2mA
+  I:  If#= 0 Alt= 0 #EPs= 1 Cls=02(comm.) Sub=02 Prot=01 Driver=acm
+  E:  Ad=83(I) Atr=03(Int.) MxPS=   8 Ivl=32ms
+  I:  If#= 1 Alt= 0 #EPs= 2 Cls=0a(data ) Sub=00 Prot=00 Driver=acm
+  E:  Ad=81(I) Atr=02(Bulk) MxPS= 512 Ivl=0ms
+  E:  Ad=02(O) Atr=02(Bulk) MxPS= 512 Ivl=0ms
 
 If the host side Linux system is configured properly, the ACM driver
 should be loaded automatically.  The command "lsmod" should show the
@@ -219,29 +222,29 @@ Serial Converter support", and for the "USB Generic Serial Driver".
 
 Once the gadget serial driver is loaded and the USB device connected
 to the Linux host with a USB cable, the host system should recognize
-the gadget serial device.  For example, the command
+the gadget serial device.  For example, the command::
 
   cat /sys/kernel/debug/usb/devices
 
-should show something like this:
+should show something like this:::
 
-T:  Bus=01 Lev=01 Prnt=01 Port=01 Cnt=02 Dev#=  6 Spd=480 MxCh= 0
-D:  Ver= 2.00 Cls=ff(vend.) Sub=00 Prot=00 MxPS=64 #Cfgs=  1
-P:  Vendor=0525 ProdID=a4a6 Rev= 2.01
-S:  Manufacturer=Linux 2.6.8.1 with net2280
-S:  Product=Gadget Serial
-S:  SerialNumber=0
-C:* #Ifs= 1 Cfg#= 1 Atr=c0 MxPwr=  2mA
-I:  If#= 0 Alt= 0 #EPs= 2 Cls=0a(data ) Sub=00 Prot=00 Driver=serial
-E:  Ad=81(I) Atr=02(Bulk) MxPS= 512 Ivl=0ms
-E:  Ad=02(O) Atr=02(Bulk) MxPS= 512 Ivl=0ms
+  T:  Bus=01 Lev=01 Prnt=01 Port=01 Cnt=02 Dev#=  6 Spd=480 MxCh= 0
+  D:  Ver= 2.00 Cls=ff(vend.) Sub=00 Prot=00 MxPS=64 #Cfgs=  1
+  P:  Vendor=0525 ProdID=a4a6 Rev= 2.01
+  S:  Manufacturer=Linux 2.6.8.1 with net2280
+  S:  Product=Gadget Serial
+  S:  SerialNumber=0
+  C:* #Ifs= 1 Cfg#= 1 Atr=c0 MxPwr=  2mA
+  I:  If#= 0 Alt= 0 #EPs= 2 Cls=0a(data ) Sub=00 Prot=00 Driver=serial
+  E:  Ad=81(I) Atr=02(Bulk) MxPS= 512 Ivl=0ms
+  E:  Ad=02(O) Atr=02(Bulk) MxPS= 512 Ivl=0ms
 
 You must load the usbserial driver and explicitly set its parameters
-to configure it to recognize the gadget serial device, like this:
+to configure it to recognize the gadget serial device, like this::
 
   echo 0x0525 0xA4A6 >/sys/bus/usb-serial/drivers/generic/new_id
 
-The legacy way is to use module parameters:
+The legacy way is to use module parameters::
 
   modprobe usbserial vendor=0x0525 product=0xA4A6
 
diff --git a/Documentation/usb/iuu_phoenix.txt b/Documentation/usb/iuu_phoenix.txt
index e5f048067da4..b76268728450 100644
--- a/Documentation/usb/iuu_phoenix.txt
+++ b/Documentation/usb/iuu_phoenix.txt
@@ -1,5 +1,6 @@
+=============================
 Infinity Usb Unlimited Readme
------------------------------
+=============================
 
 Hi all,
 
@@ -19,7 +20,8 @@ have his own device file(/dev/ttyUSB0,/dev/ttyUSB1,...)
 
 
 
-How to tune the reader speed ?
+How to tune the reader speed?
+=============================
 
  A few parameters can be used at load time
  To use parameters, just unload the module if it is
@@ -27,26 +29,33 @@ How to tune the reader speed ?
  In case of prebuilt module, use the command
  insmod iuu_phoenix param=value.
 
- Example:
+ Example::
 
- modprobe iuu_phoenix clockmode=3
+	modprobe iuu_phoenix clockmode=3
 
  The parameters are:
 
- parm:           clockmode:1=3Mhz579,2=3Mhz680,3=6Mhz (int)
- parm:           boost:overclock boost percent 100 to 500 (int)
- parm:           cdmode:Card detect mode 0=none, 1=CD, 2=!CD, 3=DSR, 4=!DSR, 5=CTS, 6=!CTS, 7=RING, 8=!RING (int)
- parm:           xmas:xmas color enabled or not (bool)
- parm:           debug:Debug enabled or not (bool)
+clockmode:
+	1=3Mhz579,2=3Mhz680,3=6Mhz (int)
+boost:
+	overclock boost percent 100 to 500 (int)
+cdmode:
+	Card detect mode
+	0=none, 1=CD, 2=!CD, 3=DSR, 4=!DSR, 5=CTS, 6=!CTS, 7=RING, 8=!RING (int)
+xmas:
+	xmas color enabled or not (bool)
+debug:
+	Debug enabled or not (bool)
 
 -  clockmode will provide 3 different base settings commonly adopted by
    different software:
- 	1. 3Mhz579
+
+	1. 3Mhz579
 	2. 3Mhz680
 	3. 6Mhz
 
 -  boost provide a way to overclock the reader ( my favorite :-)  )
-   For example to have best performance than a simple clockmode=3, try this:
+   For example to have best performance than a simple clockmode=3, try this::
 
       modprobe boost=195
 
@@ -66,7 +75,8 @@ How to tune the reader speed ?
 -  debug will produce a lot of debugging messages...
 
 
- Last notes:
+Last notes
+==========
 
  Don't worry about the serial settings, the serial emulation
  is an abstraction, so use any speed or parity setting will
diff --git a/Documentation/usb/mass-storage.txt b/Documentation/usb/mass-storage.txt
index e89803a5a960..d181b47c3cb6 100644
--- a/Documentation/usb/mass-storage.txt
+++ b/Documentation/usb/mass-storage.txt
@@ -1,4 +1,9 @@
-* Overview
+=========================
+Mass Storage Gadget (MSG)
+=========================
+
+Overview
+========
 
   Mass Storage Gadget (or MSG) acts as a USB Mass Storage device,
   appearing to the host as a disk or a CD-ROM drive.  It supports
@@ -24,7 +29,8 @@
   (which is no longer included in Linux).  It will talk only briefly
   about how to use MSF within composite gadgets.
 
-* Module parameters
+Module parameters
+=================
 
   The mass storage gadget accepts the following mass storage specific
   module parameters:
@@ -146,7 +152,8 @@
   - iProduct      -- USB Product string (string)
   - iSerialNumber -- SerialNumber string (sting)
 
-* sysfs entries
+sysfs entries
+=============
 
   For each logical unit, the gadget creates a directory in the sysfs
   hierarchy.  Inside of it the following three files are created:
@@ -177,7 +184,8 @@
   Other then those, as usual, the values of module parameters can be
   read from /sys/module/g_mass_storage/parameters/* files.
 
-* Other gadgets using mass storage function
+Other gadgets using mass storage function
+=========================================
 
   The Mass Storage Gadget uses the Mass Storage Function to handle
   mass storage protocol.  As a composite function, MSF may be used by
@@ -193,7 +201,8 @@
   may take a look at mass_storage.c, acm_ms.c and multi.c (sorted by
   complexity).
 
-* Relation to file storage gadget
+Relation to file storage gadget
+===============================
 
   The Mass Storage Function and thus the Mass Storage Gadget has been
   based on the File Storage Gadget.  The difference between the two is
diff --git a/Documentation/usb/misc_usbsevseg.txt b/Documentation/usb/misc_usbsevseg.txt
index 0f6be4f9930b..6274aee083ed 100644
--- a/Documentation/usb/misc_usbsevseg.txt
+++ b/Documentation/usb/misc_usbsevseg.txt
@@ -1,4 +1,7 @@
+=============================
 USB 7-Segment Numeric Display
+=============================
+
 Manufactured by Delcom Engineering
 
 Device Information
@@ -13,9 +16,13 @@ Device Modes
 ------------
 By default, the driver assumes the display is only 6 characters
 The mode for 6 characters is:
+
 	MSB 0x06; LSB 0x3f
+
 For the 8 character display:
+
 	MSB 0x08; LSB 0xff
+
 The device can accept "text" either in raw, hex, or ascii textmode.
 raw controls each segment manually,
 hex expects a value between 0-15 per character,
@@ -42,5 +49,3 @@ Device Operation
 	To set multiple decimals points sum up each power.
 	For example, to set the 0th and 3rd decimal place
 	echo 1001 > /sys/bus/usb/.../decimals
-
-
diff --git a/Documentation/usb/mtouchusb.txt b/Documentation/usb/mtouchusb.txt
index a91adb26ea7b..d1111b74bf75 100644
--- a/Documentation/usb/mtouchusb.txt
+++ b/Documentation/usb/mtouchusb.txt
@@ -1,19 +1,27 @@
-CHANGES
+================
+mtouchusb driver
+================
+
+Changes
+=======
 
 - 0.3 - Created based off of scanner & INSTALL from the original touchscreen
   driver on freecode (http://freecode.com/projects/3mtouchscreendriver)
 - Amended for linux-2.4.18, then 2.4.19
 
 - 0.5 - Complete rewrite using Linux Input in 2.6.3
-   Unfortunately no calibration support at this time
+  Unfortunately no calibration support at this time
 
 - 1.4 - Multiple changes to support the EXII 5000UC and house cleaning
-   Changed reset from standard USB dev reset to vendor reset
-   Changed data sent to host from compensated to raw coordinates
-   Eliminated vendor/product module params
-   Performed multiple successful tests with an EXII-5010UC
+  Changed reset from standard USB dev reset to vendor reset
+  Changed data sent to host from compensated to raw coordinates
+  Eliminated vendor/product module params
+  Performed multiple successful tests with an EXII-5010UC
+
+Supported Hardware
+==================
 
-SUPPORTED HARDWARE:
+::
 
         All controllers have the Vendor: 0x0596 & Product: 0x0001
 
@@ -29,9 +37,10 @@ SUPPORTED HARDWARE:
         USB Capacitive - Black Case     EXII-5030UC
         USB Capacitive - No Case        EXII-5050UC
 
-DRIVER NOTES:
+Driver Notes
+============
 
-Installation is simple, you only need to add Linux Input, Linux USB, and the 
+Installation is simple, you only need to add Linux Input, Linux USB, and the
 driver to the kernel.  The driver can also be optionally built as a module.
 
 This driver appears to be one of possible 2 Linux USB Input Touchscreen
@@ -49,24 +58,27 @@ The controller screen resolution is now 0 to 16384 for both X and Y reporting
 the raw touch data.  This is the same for the old and new capacitive USB
 controllers.
 
-Perhaps at some point an abstract function will be placed into evdev so 
-generic functions like calibrations, resets, and vendor information can be 
+Perhaps at some point an abstract function will be placed into evdev so
+generic functions like calibrations, resets, and vendor information can be
 requested from the userspace (And the drivers would handle the vendor specific
 tasks).
 
-TODO:
+TODO
+====
 
 Implement a control urb again to handle requests to and from the device
 such as calibration, etc once/if it becomes available.
 
-DISCLAIMER:
+Disclaimer
+==========
 
-I am not a MicroTouch/3M employee, nor have I ever been.  3M does not support 
+I am not a MicroTouch/3M employee, nor have I ever been.  3M does not support
 this driver!  If you want touch drivers only supported within X, please go to:
 
 http://www.3m.com/3MTouchSystems/
 
-THANKS:
+Thanks
+======
 
 A huge thank you to 3M Touch Systems for the EXII-5010UC controllers for
 testing!
diff --git a/Documentation/usb/ohci.txt b/Documentation/usb/ohci.txt
index 99320d9fa523..bb3c49719e6b 100644
--- a/Documentation/usb/ohci.txt
+++ b/Documentation/usb/ohci.txt
@@ -1,3 +1,7 @@
+====
+OHCI
+====
+
 23-Aug-2002
 
 The "ohci-hcd" driver is a USB Host Controller Driver (HCD) that is derived
@@ -29,4 +33,3 @@ work on while the OS is getting around to the relevant IRQ processing.
 
 - David Brownell
   <dbrownell@users.sourceforge.net>
-
diff --git a/Documentation/usb/rio.txt b/Documentation/usb/rio.txt
index aee715af7db7..ca9adcf56355 100644
--- a/Documentation/usb/rio.txt
+++ b/Documentation/usb/rio.txt
@@ -1,72 +1,80 @@
+============
+Diamonds Rio
+============
+
 Copyright (C) 1999, 2000 Bruce Tenison
+
 Portions Copyright (C) 1999, 2000 David Nelson
+
 Thanks to David Nelson for guidance and the usage of the scanner.txt
 and scanner.c files to model our driver and this informative file.
 
 Mar. 2, 2000
 
-CHANGES
+Changes
+=======
 
 - Initial Revision
 
 
-OVERVIEW
+Overview
+========
 
 This README will address issues regarding how to configure the kernel
-to access a RIO 500 mp3 player.  
+to access a RIO 500 mp3 player.
 Before I explain how to use this to access the Rio500 please be warned:
 
-W A R N I N G:
---------------
+.. warning::
 
-Please note that this software is still under development.  The authors
-are in no way responsible for any damage that may occur, no matter how
-inconsequential.
+   Please note that this software is still under development.  The authors
+   are in no way responsible for any damage that may occur, no matter how
+   inconsequential.
 
 It seems that the Rio has a problem when sending .mp3 with low batteries.
 I suggest when the batteries are low and you want to transfer stuff that you
 replace it with a fresh one. In my case, what happened is I lost two 16kb
 blocks (they are no longer usable to store information to it). But I don't
-know if that's normal or not; it could simply be a problem with the flash 
+know if that's normal or not; it could simply be a problem with the flash
 memory.
 
-In an extreme case, I left my Rio playing overnight and the batteries wore 
-down to nothing and appear to have corrupted the flash memory. My RIO 
-needed to be replaced as a result.  Diamond tech support is aware of the 
-problem.  Do NOT allow your batteries to wear down to nothing before 
-changing them.  It appears RIO 500 firmware does not handle low battery 
-power well at all. 
+In an extreme case, I left my Rio playing overnight and the batteries wore
+down to nothing and appear to have corrupted the flash memory. My RIO
+needed to be replaced as a result.  Diamond tech support is aware of the
+problem.  Do NOT allow your batteries to wear down to nothing before
+changing them.  It appears RIO 500 firmware does not handle low battery
+power well at all.
 
-On systems with OHCI controllers, the kernel OHCI code appears to have 
-power on problems with some chipsets.  If you are having problems 
-connecting to your RIO 500, try turning it on first and then plugging it 
-into the USB cable.  
+On systems with OHCI controllers, the kernel OHCI code appears to have
+power on problems with some chipsets.  If you are having problems
+connecting to your RIO 500, try turning it on first and then plugging it
+into the USB cable.
 
-Contact information:
---------------------
+Contact Information
+-------------------
 
    The main page for the project is hosted at sourceforge.net in the following
    URL: <http://rio500.sourceforge.net>. You can also go to the project's
    sourceforge home page at: <http://sourceforge.net/projects/rio500/>.
    There is also a mailing list: rio500-users@lists.sourceforge.net
 
-Authors:
+Authors
 -------
 
-Most of the code was written by Cesar Miquel <miquel@df.uba.ar>. Keith 
+Most of the code was written by Cesar Miquel <miquel@df.uba.ar>. Keith
 Clayton <kclayton@jps.net> is incharge of the PPC port and making sure
 things work there. Bruce Tenison <btenison@dibbs.net> is adding support
 for .fon files and also does testing. The program will mostly sure be
 re-written and Pete Ikusz along with the rest will re-design it. I would
-also like to thank Tri Nguyen <tmn_3022000@hotmail.com> who provided use 
+also like to thank Tri Nguyen <tmn_3022000@hotmail.com> who provided use
 with some important information regarding the communication with the Rio.
 
-ADDITIONAL INFORMATION and Userspace tools
+Additional Information and userspace tools
 
-http://rio500.sourceforge.net/
+	http://rio500.sourceforge.net/
 
 
-REQUIREMENTS
+Requirements
+============
 
 A host with a USB port.  Ideally, either a UHCI (Intel) or OHCI
 (Compaq and others) hardware port should work.
@@ -80,11 +88,11 @@ A Linux kernel with RIO 500 support enabled.
 'lspci' which is only needed to determine the type of USB hardware
 available in your machine.
 
-CONFIGURATION
+Configuration
 
 Using `lspci -v`, determine the type of USB hardware available.
 
-  If you see something like:
+  If you see something like::
 
     USB Controller: ......
     Flags: .....
@@ -92,7 +100,7 @@ Using `lspci -v`, determine the type of USB hardware available.
 
   Then you have a UHCI based controller.
 
-  If you see something like:
+  If you see something like::
 
      USB Controller: .....
      Flags: ....
@@ -107,8 +115,9 @@ hardware (determined from the steps above), 'USB Diamond Rio500 support', and
 (you may need to execute `depmod -a` to update the module
 dependencies).
 
-Add a device for the USB rio500:
-  `mknod /dev/usb/rio500 c 180 64`
+Add a device for the USB rio500::
+
+  mknod /dev/usb/rio500 c 180 64
 
 Set appropriate permissions for /dev/usb/rio500 (don't forget about
 group and world permissions).  Both read and write permissions are
@@ -116,12 +125,14 @@ required for proper operation.
 
 Load the appropriate modules (if compiled as modules):
 
-  OHCI:
+  OHCI::
+
     modprobe usbcore
     modprobe usb-ohci
     modprobe rio500
 
-  UHCI:
+  UHCI::
+
     modprobe usbcore
     modprobe usb-uhci  (or uhci)
     modprobe rio500
@@ -129,10 +140,10 @@ Load the appropriate modules (if compiled as modules):
 That's it.  The Rio500 Utils at: http://rio500.sourceforge.net should
 be able to access the rio500.
 
-BUGS
+Bugs
+====
 
 If you encounter any problems feel free to drop me an email.
 
 Bruce Tenison
 btenison@dibbs.net
-
diff --git a/Documentation/usb/usb-help.txt b/Documentation/usb/usb-help.txt
index 4273ca2b86ba..dc23ecd4d802 100644
--- a/Documentation/usb/usb-help.txt
+++ b/Documentation/usb/usb-help.txt
@@ -1,16 +1,17 @@
-usb-help.txt
+==============
+USB references
+==============
+
 2008-Mar-7
 
 For USB help other than the readme files that are located in
-Documentation/usb/*, see the following:
+`Documentation/usb/*`, see the following:
 
-Linux-USB project:  http://www.linux-usb.org
-  mirrors at        http://usb.in.tum.de/linux-usb/
-         and        http://it.linux-usb.org
-Linux USB Guide:    http://linux-usb.sourceforge.net
-Linux-USB device overview (working devices and drivers):
-                    http://www.qbik.ch/usb/devices/
+- Linux-USB project:  http://www.linux-usb.org
+  mirrors at          http://usb.in.tum.de/linux-usb/
+  and                 http://it.linux-usb.org
+- Linux USB Guide:    http://linux-usb.sourceforge.net
+- Linux-USB device overview (working devices and drivers):
+  http://www.qbik.ch/usb/devices/
 
 The Linux-USB mailing list is at linux-usb@vger.kernel.org
-
-###
diff --git a/Documentation/usb/usb-serial.txt b/Documentation/usb/usb-serial.txt
index ab100d6ee436..8fa7dbd3da9a 100644
--- a/Documentation/usb/usb-serial.txt
+++ b/Documentation/usb/usb-serial.txt
@@ -1,4 +1,9 @@
-INTRODUCTION
+==========
+USB serial
+==========
+
+Introduction
+============
 
   The USB serial driver currently supports a number of different USB to
   serial converter products, as well as some devices that use a serial
@@ -8,13 +13,15 @@ INTRODUCTION
   the different devices.
 
 
-CONFIGURATION
+Configuration
+=============
 
   Currently the driver can handle up to 256 different serial interfaces at
-  one time. 
+  one time.
 
     The major number that the driver uses is 188 so to use the driver,
-    create the following nodes:
+    create the following nodes::
+
 	mknod /dev/ttyUSB0 c 188 0
 	mknod /dev/ttyUSB1 c 188 1
 	mknod /dev/ttyUSB2 c 188 2
@@ -28,12 +35,14 @@ CONFIGURATION
   When the device is connected and recognized by the driver, the driver
   will print to the system log, which node(s) the device has been bound
   to.
-  
 
-SPECIFIC DEVICES SUPPORTED
+
+Specific Devices Supported
+==========================
 
 
 ConnectTech WhiteHEAT 4 port converter
+--------------------------------------
 
   ConnectTech has been very forthcoming with information about their
   device, including providing a unit to test with.
@@ -46,6 +55,7 @@ ConnectTech WhiteHEAT 4 port converter
 
 
 HandSpring Visor, Palm USB, and Clié USB driver
+-----------------------------------------------
 
   This driver works with all HandSpring USB, Palm USB, and Sony Clié USB
   devices.
@@ -62,7 +72,7 @@ HandSpring Visor, Palm USB, and Clié USB driver
     This goes against the current documentation for pilot-xfer and other
     packages, but is the only way that it will work due to the hardware
     in the device.
-  
+
   When the device is connected, try talking to it on the second port
   (this is usually /dev/ttyUSB1 if you do not have any other usb-serial
   devices in the system.) The system log should tell you which port is
@@ -78,10 +88,10 @@ HandSpring Visor, Palm USB, and Clié USB driver
   try resetting the device, first a hot reset, and then a cold reset if
   necessary.  Some devices need this before they can talk to the USB port
   properly.
-  
+
   Devices that are not compiled into the kernel can be specified with module
   parameters.  e.g. modprobe visor vendor=0x54c product=0x66
-  
+
   There is a webpage and mailing lists for this portion of the driver at:
   http://sourceforge.net/projects/usbvisor/
 
@@ -90,6 +100,7 @@ HandSpring Visor, Palm USB, and Clié USB driver
 
 
 PocketPC PDA Driver
+-------------------
 
   This driver can be used to connect to Compaq iPAQ, HP Jornada, Casio EM500
   and other PDAs running Windows CE 3.0 or PocketPC 2002 using a USB
@@ -135,12 +146,13 @@ PocketPC PDA Driver
   be used to flash the ROM, as well as the microP code..  so much for needing
   Toshiba's $350 serial cable for flashing!! :D
   NOTE: This has NOT been tested. Use at your own risk.
- 
+
   For any questions or problems with the driver, please contact Ganesh
   Varadarajan <ganesh@veritas.com>
 
 
 Keyspan PDA Serial Adapter
+--------------------------
 
   Single port DB-9 serial adapter, pushed as a PDA adapter for iMacs (mostly
   sold in Macintosh catalogs, comes in a translucent white/green dongle).
@@ -148,32 +160,37 @@ Keyspan PDA Serial Adapter
   This driver also works for the Xircom/Entrega single port serial adapter.
 
   Current status:
+
    Things that work:
-     basic input/output (tested with 'cu')
-     blocking write when serial line can't keep up
-     changing baud rates (up to 115200)
-     getting/setting modem control pins (TIOCM{GET,SET,BIS,BIC})
-     sending break (although duration looks suspect)
+     - basic input/output (tested with 'cu')
+     - blocking write when serial line can't keep up
+     - changing baud rates (up to 115200)
+     - getting/setting modem control pins (TIOCM{GET,SET,BIS,BIC})
+     - sending break (although duration looks suspect)
+
    Things that don't:
-     device strings (as logged by kernel) have trailing binary garbage
-     device ID isn't right, might collide with other Keyspan products
-     changing baud rates ought to flush tx/rx to avoid mangled half characters
+     - device strings (as logged by kernel) have trailing binary garbage
+     - device ID isn't right, might collide with other Keyspan products
+     - changing baud rates ought to flush tx/rx to avoid mangled half characters
+
    Big Things on the todo list:
-     parity, 7 vs 8 bits per char, 1 or 2 stop bits
-     HW flow control
-     not all of the standard USB descriptors are handled: Get_Status, Set_Feature
-     O_NONBLOCK, select()
+     - parity, 7 vs 8 bits per char, 1 or 2 stop bits
+     - HW flow control
+     - not all of the standard USB descriptors are handled:
+       Get_Status, Set_Feature, O_NONBLOCK, select()
 
   For any questions or problems with this driver, please contact Brian
-  Warner at warner@lothar.com 
+  Warner at warner@lothar.com
 
 
 Keyspan USA-series Serial Adapters
+----------------------------------
 
-  Single, Dual and Quad port adapters - driver uses Keyspan supplied 
+  Single, Dual and Quad port adapters - driver uses Keyspan supplied
   firmware and is being developed with their support.
-  
+
   Current status:
+
     The USA-18X, USA-28X, USA-19, USA-19W and USA-49W are supported and
     have been pretty thoroughly tested at various baud rates with 8-N-1
     character settings.  Other character lengths and parity setups are
@@ -182,32 +199,37 @@ Keyspan USA-series Serial Adapters
     The USA-28 isn't yet supported though doing so should be pretty
     straightforward.  Contact the maintainer if you require this
     functionality.
-  
+
   More information is available at:
+
         http://www.carnationsoftware.com/carnation/Keyspan.html
-   
+
   For any questions or problems with this driver, please contact Hugh
   Blemings at hugh@misc.nu
 
 
 FTDI Single Port Serial Driver
+------------------------------
 
   This is a single port DB-25 serial adapter.
 
   Devices supported include:
-                -TripNav TN-200 USB GPS
-                -Navis Engineering Bureau CH-4711 USB GPS
+
+                - TripNav TN-200 USB GPS
+                - Navis Engineering Bureau CH-4711 USB GPS
 
   For any questions or problems with this driver, please contact Bill Ryder.
 
 
 ZyXEL omni.net lcd plus ISDN TA
+-------------------------------
 
   This is an ISDN TA. Please report both successes and troubles to
   azummo@towertech.it
 
 
 Cypress M8 CY4601 Family Serial Driver
+--------------------------------------
 
   This driver was in most part developed by Neil "koyama" Whelchel.  It
   has been improved since that previous form to support dynamic serial
@@ -215,18 +237,19 @@ Cypress M8 CY4601 Family Serial Driver
   part stable and has been tested on an smp machine. (dual p2)
 
     Chipsets supported under CY4601 family:
-	
+
 		CY7C63723, CY7C63742, CY7C63743, CY7C64013
 
     Devices supported:
 
-		-DeLorme's USB Earthmate GPS (SiRF Star II lp arch)
-		-Cypress HID->COM RS232 adapter
-	
-		Note: Cypress Semiconductor claims no affiliation with the
+		- DeLorme's USB Earthmate GPS (SiRF Star II lp arch)
+		- Cypress HID->COM RS232 adapter
+
+		Note:
+			Cypress Semiconductor claims no affiliation with the
 			hid->com device.
 
-	Most devices using chipsets under the CY4601 family should
+     Most devices using chipsets under the CY4601 family should
      work with the driver.  As long as they stay true to the CY4601
      usbserial specification.
 
@@ -236,8 +259,9 @@ Cypress M8 CY4601 Family Serial Driver
 	upon start init to this setting.  usbserial core provides the rest
 	of the termios settings, along with some custom termios so that the
 	output is in proper format and parsable.
-	
-	The device can be put into sirf mode by issuing NMEA command:
+
+	The device can be put into sirf mode by issuing NMEA command::
+
 		$PSRF100,<protocol>,<baud>,<databits>,<stopbits>,<parity>*CHECKSUM
 		$PSRF100,0,9600,8,1,0*0C
 
@@ -259,11 +283,14 @@ Cypress M8 CY4601 Family Serial Driver
 
 	If you have any questions, problems, patches, feature requests, etc. you can
 	contact me here via email:
+
 					dignome@gmail.com
+
 		(your problems/patches can alternately be submitted to usb-devel)
 
 
 Digi AccelePort Driver
+----------------------
 
   This driver supports the Digi AccelePort USB 2 and 4 devices, 2 port
   (plus a parallel port) and 4 port USB serial converters.  The driver
@@ -285,42 +312,49 @@ Digi AccelePort Driver
 
 
 Belkin USB Serial Adapter F5U103
+--------------------------------
 
   Single port DB-9/PS-2 serial adapter from Belkin with firmware by eTEK Labs.
   The Peracom single port serial adapter also works with this driver, as
   well as the GoHubs adapter.
 
   Current status:
+
     The following have been tested and work:
-      Baud rate    300-230400               
-      Data bits    5-8
-      Stop bits    1-2
-      Parity       N,E,O,M,S
-      Handshake    None, Software (XON/XOFF), Hardware (CTSRTS,CTSDTR)*
-      Break        Set and clear
-      Line control Input/Output query and control **
-
-      *  Hardware input flow control is only enabled for firmware
+
+      - Baud rate    300-230400
+      - Data bits    5-8
+      - Stop bits    1-2
+      - Parity       N,E,O,M,S
+      - Handshake    None, Software (XON/XOFF), Hardware (CTSRTS,CTSDTR) [1]_
+      - Break        Set and clear
+      - Line control Input/Output query and control [2]_
+
+  .. [1]
+         Hardware input flow control is only enabled for firmware
          levels above 2.06.  Read source code comments describing Belkin
          firmware errata.  Hardware output flow control is working for all
          firmware versions.
-      ** Queries of inputs (CTS,DSR,CD,RI) show the last
+
+  .. [2]
+         Queries of inputs (CTS,DSR,CD,RI) show the last
          reported state.  Queries of outputs (DTR,RTS) show the last
          requested state and may not reflect current state as set by
          automatic hardware flow control.
 
   TO DO List:
-    -- Add true modem control line query capability.  Currently tracks the
-       states reported by the interrupt and the states requested.
-    -- Add error reporting back to application for UART error conditions.
-    -- Add support for flush ioctls.
-    -- Add everything else that is missing :)
+    - Add true modem control line query capability.  Currently tracks the
+      states reported by the interrupt and the states requested.
+    - Add error reporting back to application for UART error conditions.
+    - Add support for flush ioctls.
+    - Add everything else that is missing :)
 
   For any questions or problems with this driver, please contact William
   Greathouse at wgreathouse@smva.com
 
 
 Empeg empeg-car Mark I/II Driver
+--------------------------------
 
   This is an experimental driver to provide connectivity support for the
   client synchronization tools for an Empeg empeg-car mp3 player.
@@ -335,6 +369,7 @@ Empeg empeg-car Mark I/II Driver
 
 
 MCT USB Single Port Serial Adapter U232
+---------------------------------------
 
   This driver is for the MCT USB-RS232 Converter (25 pin, Model No.
   U232-P25) from Magic Control Technology Corp. (there is also a 9 pin
@@ -355,35 +390,39 @@ MCT USB Single Port Serial Adapter U232
 
 
 Inside Out Networks Edgeport Driver
+-----------------------------------
 
   This driver supports all devices made by Inside Out Networks, specifically
   the following models:
-       Edgeport/4
-       Rapidport/4
-       Edgeport/4t
-       Edgeport/2
-       Edgeport/4i
-       Edgeport/2i
-       Edgeport/421
-       Edgeport/21
-       Edgeport/8
-       Edgeport/8 Dual
-       Edgeport/2D8
-       Edgeport/4D8
-       Edgeport/8i
-       Edgeport/2 DIN
-       Edgeport/4 DIN
-       Edgeport/16 Dual
+
+       - Edgeport/4
+       - Rapidport/4
+       - Edgeport/4t
+       - Edgeport/2
+       - Edgeport/4i
+       - Edgeport/2i
+       - Edgeport/421
+       - Edgeport/21
+       - Edgeport/8
+       - Edgeport/8 Dual
+       - Edgeport/2D8
+       - Edgeport/4D8
+       - Edgeport/8i
+       - Edgeport/2 DIN
+       - Edgeport/4 DIN
+       - Edgeport/16 Dual
 
   For any questions or problems with this driver, please contact Greg
   Kroah-Hartman at greg@kroah.com
 
 
 REINER SCT cyberJack pinpad/e-com USB chipcard reader
-   
+-----------------------------------------------------
+
   Interface to ISO 7816 compatible contactbased chipcards, e.g. GSM SIMs.
-  
+
   Current status:
+
     This is the kernel part of the driver for this USB card reader.
     There is also a user part for a CT-API driver available. A site
     for downloading is TBA. For now, you can request it from the
@@ -394,6 +433,7 @@ REINER SCT cyberJack pinpad/e-com USB chipcard reader
 
 
 Prolific PL2303 Driver
+----------------------
 
   This driver supports any device that has the PL2303 chip from Prolific
   in it.  This includes a number of single port USB to serial converters,
@@ -403,11 +443,13 @@ Prolific PL2303 Driver
 
   For any questions or problems with this driver, please contact Greg
   Kroah-Hartman at greg@kroah.com
-  
+
 
 KL5KUSB105 chipset / PalmConnect USB single-port adapter
-  
+--------------------------------------------------------
+
 Current status:
+
   The driver was put together by looking at the usb bus transactions
   done by Palm's driver under Windows, so a lot of functionality is
   still missing.  Notably, serial ioctls are sometimes faked or not yet
@@ -417,21 +459,25 @@ Current status:
   are supported, but handshaking (software or hardware) is not, which is
   why it is wise to cut down on the rate used is wise for large
   transfers until this is settled.
-  
+
   See http://www.uuhaus.de/linux/palmconnect.html for up-to-date
   information on this driver.
 
 Winchiphead CH341 Driver
+------------------------
 
   This driver is for the Winchiphead CH341 USB-RS232 Converter. This chip
   also implements an IEEE 1284 parallel port, I2C and SPI, but that is not
   supported by the driver. The protocol was analyzed from the behaviour
   of the Windows driver, no datasheet is available at present.
+
   The manufacturer's website: http://www.winchiphead.com/.
+
   For any questions or problems with this driver, please contact
   frank@kingswood-consulting.co.uk.
 
 Moschip MCS7720, MCS7715 driver
+-------------------------------
 
   These chips are present in devices sold by various manufacturers, such as Syba
   and Cables Unlimited.  There may be others.  The 7720 provides two serial
@@ -449,20 +495,24 @@ Moschip MCS7720, MCS7715 driver
       don't have one of these devices, so I can't say for sure.
 
 Generic Serial driver
+---------------------
 
   If your device is not one of the above listed devices, compatible with
   the above models, you can try out the "generic" interface. This
   interface does not provide any type of control messages sent to the
   device, and does not support any kind of device flow control. All that
   is required of your device is that it has at least one bulk in endpoint,
-  or one bulk out endpoint. 
+  or one bulk out endpoint.
+
+  To enable the generic driver to recognize your device, provide::
 
-  To enable the generic driver to recognize your device, provide
 	echo <vid> <pid> >/sys/bus/usb-serial/drivers/generic/new_id
+
   where the <vid> and <pid> is replaced with the hex representation of your
   device's vendor id and product id.
   If the driver is compiled as a module you can also provide one id when
-  loading the module
+  loading the module::
+
 	insmod usbserial vendor=0x#### product=0x####
 
   This driver has been successfully used to connect to the NetChip USB
@@ -473,7 +523,8 @@ Generic Serial driver
   Kroah-Hartman at greg@kroah.com
 
 
-CONTACT:
+Contact
+=======
 
   If anyone has any problems using these drivers, with any of the above
   specified products, please contact the specific driver's author listed
diff --git a/Documentation/usb/usbip_protocol.txt b/Documentation/usb/usbip_protocol.txt
index c7a0f4c7e7f1..988c832166cd 100644
--- a/Documentation/usb/usbip_protocol.txt
+++ b/Documentation/usb/usbip_protocol.txt
@@ -1,3 +1,7 @@
+===============
+USB/IP protocol
+===============
+
 PRELIMINARY DRAFT, MAY CONTAIN MISTAKES!
 28 Jun 2011
 
@@ -12,6 +16,8 @@ in one or more pieces at the low level transport layer). The server sends back
 the OP_REP_DEVLIST packet which lists the exported USB devices. Finally the
 TCP/IP connection is closed.
 
+::
+
  virtual host controller                                 usb host
       "client"                                           "server"
   (imports USB devices)                             (exports USB devices)
@@ -32,6 +38,8 @@ send two types of packets: the USBIP_CMD_SUBMIT to submit an URB, and
 USBIP_CMD_UNLINK to unlink a previously submitted URB. The answers of the
 server may be USBIP_RET_SUBMIT and USBIP_RET_UNLINK respectively.
 
+::
+
  virtual host controller                                 usb host
       "client"                                           "server"
   (imports USB devices)                             (exports USB devices)
@@ -88,270 +96,316 @@ The fields are in network (big endian) byte order meaning that the most signific
 byte (MSB) is stored at the lowest address.
 
 
-OP_REQ_DEVLIST: Retrieve the list of exported USB devices.
+OP_REQ_DEVLIST:
+	Retrieve the list of exported USB devices.
 
- Offset    | Length | Value      | Description
------------+--------+------------+---------------------------------------------------
- 0         | 2      | 0x0100     | Binary-coded decimal USBIP version number: v1.0.0
------------+--------+------------+---------------------------------------------------
- 2         | 2      | 0x8005     | Command code: Retrieve the list of exported USB
-           |        |            |   devices.
------------+--------+------------+---------------------------------------------------
- 4         | 4      | 0x00000000 | Status: unused, shall be set to 0
++-----------+--------+------------+---------------------------------------------------+
+| Offset    | Length | Value      | Description                                       |
++===========+========+============+===================================================+
+| 0         | 2      | 0x0100     | Binary-coded decimal USBIP version number: v1.0.0 |
++-----------+--------+------------+---------------------------------------------------+
+| 2         | 2      | 0x8005     | Command code: Retrieve the list of exported USB   |
+|           |        |            | devices.                                          |
++-----------+--------+------------+---------------------------------------------------+
+| 4         | 4      | 0x00000000 | Status: unused, shall be set to 0                 |
++-----------+--------+------------+---------------------------------------------------+
 
-OP_REP_DEVLIST: Reply with the list of exported USB devices.
+OP_REP_DEVLIST:
+	Reply with the list of exported USB devices.
 
- Offset    | Length | Value      | Description
------------+--------+------------+---------------------------------------------------
- 0         | 2      | 0x0100     | Binary-coded decimal USBIP version number: v1.0.0.
------------+--------+------------+---------------------------------------------------
- 2         | 2      | 0x0005     | Reply code: The list of exported USB devices.
------------+--------+------------+---------------------------------------------------
- 4         | 4      | 0x00000000 | Status: 0 for OK
------------+--------+------------+---------------------------------------------------
- 8         | 4      | n          | Number of exported devices: 0 means no exported
-           |        |            |   devices.
------------+--------+------------+---------------------------------------------------
- 0x0C      |        |            | From now on the exported n devices are described,
-           |        |            |   if any. If no devices are exported the message
-           |        |            |   ends with the previous "number of exported
-           |        |            |   devices" field.
------------+--------+------------+---------------------------------------------------
-           | 256    |            | path: Path of the device on the host exporting the
-           |        |            |   USB device, string closed with zero byte, e.g.
-           |        |            |   "/sys/devices/pci0000:00/0000:00:1d.1/usb3/3-2"
-           |        |            |   The unused bytes shall be filled with zero
-           |        |            |   bytes.
------------+--------+------------+---------------------------------------------------
- 0x10C     | 32     |            | busid: Bus ID of the exported device, string
-           |        |            |   closed with zero byte, e.g. "3-2". The unused
-           |        |            |   bytes shall be filled with zero bytes.
------------+--------+------------+---------------------------------------------------
- 0x12C     | 4      |            | busnum
------------+--------+------------+---------------------------------------------------
- 0x130     | 4      |            | devnum
------------+--------+------------+---------------------------------------------------
- 0x134     | 4      |            | speed
------------+--------+------------+---------------------------------------------------
- 0x138     | 2      |            | idVendor
------------+--------+------------+---------------------------------------------------
- 0x13A     | 2      |            | idProduct
------------+--------+------------+---------------------------------------------------
- 0x13C     | 2      |            | bcdDevice
------------+--------+------------+---------------------------------------------------
- 0x13E     | 1      |            | bDeviceClass
------------+--------+------------+---------------------------------------------------
- 0x13F     | 1      |            | bDeviceSubClass
------------+--------+------------+---------------------------------------------------
- 0x140     | 1      |            | bDeviceProtocol
------------+--------+------------+---------------------------------------------------
- 0x141     | 1      |            | bConfigurationValue
------------+--------+------------+---------------------------------------------------
- 0x142     | 1      |            | bNumConfigurations
------------+--------+------------+---------------------------------------------------
- 0x143     | 1      |            | bNumInterfaces
------------+--------+------------+---------------------------------------------------
- 0x144     |        | m_0        | From now on each interface is described, all
-           |        |            |   together bNumInterfaces times, with the
-           |        |            |   the following 4 fields:
------------+--------+------------+---------------------------------------------------
-           | 1      |            | bInterfaceClass
------------+--------+------------+---------------------------------------------------
- 0x145     | 1      |            | bInterfaceSubClass
------------+--------+------------+---------------------------------------------------
- 0x146     | 1      |            | bInterfaceProtocol
------------+--------+------------+---------------------------------------------------
- 0x147     | 1      |            | padding byte for alignment, shall be set to zero
------------+--------+------------+---------------------------------------------------
- 0xC +     |        |            | The second exported USB device starts at i=1
- i*0x138 + |        |            | with the busid field.
- m_(i-1)*4 |        |            |
++-----------+--------+------------+---------------------------------------------------+
+| Offset    | Length | Value      | Description                                       |
++===========+========+============+===================================================+
+| 0         | 2      | 0x0100     | Binary-coded decimal USBIP version number: v1.0.0.|
++-----------+--------+------------+---------------------------------------------------+
+| 2         | 2      | 0x0005     | Reply code: The list of exported USB devices.     |
++-----------+--------+------------+---------------------------------------------------+
+| 4         | 4      | 0x00000000 | Status: 0 for OK                                  |
++-----------+--------+------------+---------------------------------------------------+
+| 8         | 4      | n          | Number of exported devices: 0 means no exported   |
+|           |        |            | devices.                                          |
++-----------+--------+------------+---------------------------------------------------+
+| 0x0C      |        |            | From now on the exported n devices are described, |
+|           |        |            | if any. If no devices are exported the message    |
+|           |        |            | ends with the previous "number of exported        |
+|           |        |            | devices" field.                                   |
++-----------+--------+------------+---------------------------------------------------+
+|           | 256    |            | path: Path of the device on the host exporting the|
+|           |        |            | USB device, string closed with zero byte, e.g.    |
+|           |        |            | "/sys/devices/pci0000:00/0000:00:1d.1/usb3/3-2"   |
+|           |        |            | The unused bytes shall be filled with zero        |
+|           |        |            | bytes.                                            |
++-----------+--------+------------+---------------------------------------------------+
+| 0x10C     | 32     |            | busid: Bus ID of the exported device, string      |
+|           |        |            | closed with zero byte, e.g. "3-2". The unused     |
+|           |        |            | bytes shall be filled with zero bytes.            |
++-----------+--------+------------+---------------------------------------------------+
+| 0x12C     | 4      |            | busnum                                            |
++-----------+--------+------------+---------------------------------------------------+
+| 0x130     | 4      |            | devnum                                            |
++-----------+--------+------------+---------------------------------------------------+
+| 0x134     | 4      |            | speed                                             |
++-----------+--------+------------+---------------------------------------------------+
+| 0x138     | 2      |            | idVendor                                          |
++-----------+--------+------------+---------------------------------------------------+
+| 0x13A     | 2      |            | idProduct                                         |
++-----------+--------+------------+---------------------------------------------------+
+| 0x13C     | 2      |            | bcdDevice                                         |
++-----------+--------+------------+---------------------------------------------------+
+| 0x13E     | 1      |            | bDeviceClass                                      |
++-----------+--------+------------+---------------------------------------------------+
+| 0x13F     | 1      |            | bDeviceSubClass                                   |
++-----------+--------+------------+---------------------------------------------------+
+| 0x140     | 1      |            | bDeviceProtocol                                   |
++-----------+--------+------------+---------------------------------------------------+
+| 0x141     | 1      |            | bConfigurationValue                               |
++-----------+--------+------------+---------------------------------------------------+
+| 0x142     | 1      |            | bNumConfigurations                                |
++-----------+--------+------------+---------------------------------------------------+
+| 0x143     | 1      |            | bNumInterfaces                                    |
++-----------+--------+------------+---------------------------------------------------+
+| 0x144     |        | m_0        | From now on each interface is described, all      |
+|           |        |            | together bNumInterfaces times, with the           |
+|           |        |            | the following 4 fields:                           |
++-----------+--------+------------+---------------------------------------------------+
+|           | 1      |            | bInterfaceClass                                   |
++-----------+--------+------------+---------------------------------------------------+
+| 0x145     | 1      |            | bInterfaceSubClass                                |
++-----------+--------+------------+---------------------------------------------------+
+| 0x146     | 1      |            | bInterfaceProtocol                                |
++-----------+--------+------------+---------------------------------------------------+
+| 0x147     | 1      |            | padding byte for alignment, shall be set to zero  |
++-----------+--------+------------+---------------------------------------------------+
+| 0xC +     |        |            | The second exported USB device starts at i=1      |
+| i*0x138 + |        |            | with the busid field.                             |
+| m_(i-1)*4 |        |            |                                                   |
++-----------+--------+------------+---------------------------------------------------+
 
-OP_REQ_IMPORT: Request to import (attach) a remote USB device.
+OP_REQ_IMPORT:
+	Request to import (attach) a remote USB device.
 
- Offset    | Length | Value      | Description
------------+--------+------------+---------------------------------------------------
- 0         | 2      | 0x0100     | Binary-coded decimal USBIP version number: v1.0.0
------------+--------+------------+---------------------------------------------------
- 2         | 2      | 0x8003     | Command code: import a remote USB device.
------------+--------+------------+---------------------------------------------------
- 4         | 4      | 0x00000000 | Status: unused, shall be set to 0
------------+--------+------------+---------------------------------------------------
- 8         | 32     |            | busid: the busid of the exported device on the
-           |        |            |   remote host. The possible values are taken
-           |        |            |   from the message field OP_REP_DEVLIST.busid.
-           |        |            |   A string closed with zero, the unused bytes
-           |        |            |   shall be filled with zeros.
------------+--------+------------+---------------------------------------------------
++-----------+--------+------------+---------------------------------------------------+
+| Offset    | Length | Value      | Description                                       |
++===========+========+============+===================================================+
+| 0         | 2      | 0x0100     | Binary-coded decimal USBIP version number: v1.0.0 |
++-----------+--------+------------+---------------------------------------------------+
+| 2         | 2      | 0x8003     | Command code: import a remote USB device.         |
++-----------+--------+------------+---------------------------------------------------+
+| 4         | 4      | 0x00000000 | Status: unused, shall be set to 0                 |
++-----------+--------+------------+---------------------------------------------------+
+| 8         | 32     |            | busid: the busid of the exported device on the    |
+|           |        |            | remote host. The possible values are taken        |
+|           |        |            | from the message field OP_REP_DEVLIST.busid.      |
+|           |        |            | A string closed with zero, the unused bytes       |
+|           |        |            | shall be filled with zeros.                       |
++-----------+--------+------------+---------------------------------------------------+
 
-OP_REP_IMPORT: Reply to import (attach) a remote USB device.
+OP_REP_IMPORT:
+	Reply to import (attach) a remote USB device.
 
- Offset    | Length | Value      | Description
------------+--------+------------+---------------------------------------------------
- 0         | 2      | 0x0100     | Binary-coded decimal USBIP version number: v1.0.0
------------+--------+------------+---------------------------------------------------
- 2         | 2      | 0x0003     | Reply code: Reply to import.
------------+--------+------------+---------------------------------------------------
- 4         | 4      | 0x00000000 | Status: 0 for OK
-           |        |            |         1 for error
------------+--------+------------+---------------------------------------------------
- 8         |        |            | From now on comes the details of the imported
-           |        |            |   device, if the previous status field was OK (0),
-           |        |            |   otherwise the reply ends with the status field.
------------+--------+------------+---------------------------------------------------
-           | 256    |            | path: Path of the device on the host exporting the
-           |        |            |   USB device, string closed with zero byte, e.g.
-           |        |            |   "/sys/devices/pci0000:00/0000:00:1d.1/usb3/3-2"
-           |        |            |   The unused bytes shall be filled with zero
-           |        |            |   bytes.
------------+--------+------------+---------------------------------------------------
- 0x108     | 32     |            | busid: Bus ID of the exported device, string
-           |        |            |   closed with zero byte, e.g. "3-2". The unused
-           |        |            |   bytes shall be filled with zero bytes.
------------+--------+------------+---------------------------------------------------
- 0x128     | 4      |            | busnum
------------+--------+------------+---------------------------------------------------
- 0x12C     | 4      |            | devnum
------------+--------+------------+---------------------------------------------------
- 0x130     | 4      |            | speed
------------+--------+------------+---------------------------------------------------
- 0x134     | 2      |            | idVendor
------------+--------+------------+---------------------------------------------------
- 0x136     | 2      |            | idProduct
------------+--------+------------+---------------------------------------------------
- 0x138     | 2      |            | bcdDevice
------------+--------+------------+---------------------------------------------------
- 0x139     | 1      |            | bDeviceClass
------------+--------+------------+---------------------------------------------------
- 0x13A     | 1      |            | bDeviceSubClass
------------+--------+------------+---------------------------------------------------
- 0x13B     | 1      |            | bDeviceProtocol
------------+--------+------------+---------------------------------------------------
- 0x13C     | 1      |            | bConfigurationValue
------------+--------+------------+---------------------------------------------------
- 0x13D     | 1      |            | bNumConfigurations
------------+--------+------------+---------------------------------------------------
- 0x13E     | 1      |            | bNumInterfaces
++-----------+--------+------------+---------------------------------------------------+
+| Offset    | Length | Value      | Description                                       |
++===========+========+============+===================================================+
+| 0         | 2      | 0x0100     | Binary-coded decimal USBIP version number: v1.0.0 |
++-----------+--------+------------+---------------------------------------------------+
+| 2         | 2      | 0x0003     | Reply code: Reply to import.                      |
++-----------+--------+------------+---------------------------------------------------+
+| 4         | 4      | 0x00000000 | Status:                                           |
+|           |        |            |                                                   |
+|           |        |            |   - 0 for OK                                      |
+|           |        |            |   - 1 for error                                   |
++-----------+--------+------------+---------------------------------------------------+
+| 8         |        |            | From now on comes the details of the imported     |
+|           |        |            | device, if the previous status field was OK (0),  |
+|           |        |            | otherwise the reply ends with the status field.   |
++-----------+--------+------------+---------------------------------------------------+
+|           | 256    |            | path: Path of the device on the host exporting the|
+|           |        |            | USB device, string closed with zero byte, e.g.    |
+|           |        |            | "/sys/devices/pci0000:00/0000:00:1d.1/usb3/3-2"   |
+|           |        |            | The unused bytes shall be filled with zero        |
+|           |        |            | bytes.                                            |
++-----------+--------+------------+---------------------------------------------------+
+| 0x108     | 32     |            | busid: Bus ID of the exported device, string      |
+|           |        |            | closed with zero byte, e.g. "3-2". The unused     |
+|           |        |            | bytes shall be filled with zero bytes.            |
++-----------+--------+------------+---------------------------------------------------+
+| 0x128     | 4      |            | busnum                                            |
++-----------+--------+------------+---------------------------------------------------+
+| 0x12C     | 4      |            | devnum                                            |
++-----------+--------+------------+---------------------------------------------------+
+| 0x130     | 4      |            | speed                                             |
++-----------+--------+------------+---------------------------------------------------+
+| 0x134     | 2      |            | idVendor                                          |
++-----------+--------+------------+---------------------------------------------------+
+| 0x136     | 2      |            | idProduct                                         |
++-----------+--------+------------+---------------------------------------------------+
+| 0x138     | 2      |            | bcdDevice                                         |
++-----------+--------+------------+---------------------------------------------------+
+| 0x139     | 1      |            | bDeviceClass                                      |
++-----------+--------+------------+---------------------------------------------------+
+| 0x13A     | 1      |            | bDeviceSubClass                                   |
++-----------+--------+------------+---------------------------------------------------+
+| 0x13B     | 1      |            | bDeviceProtocol                                   |
++-----------+--------+------------+---------------------------------------------------+
+| 0x13C     | 1      |            | bConfigurationValue                               |
++-----------+--------+------------+---------------------------------------------------+
+| 0x13D     | 1      |            | bNumConfigurations                                |
++-----------+--------+------------+---------------------------------------------------+
+| 0x13E     | 1      |            | bNumInterfaces                                    |
++-----------+--------+------------+---------------------------------------------------+
 
-USBIP_CMD_SUBMIT: Submit an URB
+USBIP_CMD_SUBMIT:
+	Submit an URB
 
- Offset    | Length | Value      | Description
------------+--------+------------+---------------------------------------------------
- 0         | 4      | 0x00000001 | command: Submit an URB
------------+--------+------------+---------------------------------------------------
- 4         | 4      |            | seqnum: the sequence number of the URB to submit
------------+--------+------------+---------------------------------------------------
- 8         | 4      |            | devid
------------+--------+------------+---------------------------------------------------
- 0xC       | 4      |            | direction: 0: USBIP_DIR_OUT
-           |        |            |            1: USBIP_DIR_IN
------------+--------+------------+---------------------------------------------------
- 0x10      | 4      |            | ep: endpoint number, possible values are: 0...15
------------+--------+------------+---------------------------------------------------
- 0x14      | 4      |            | transfer_flags: possible values depend on the
-           |        |            |   URB transfer type, see below
------------+--------+------------+---------------------------------------------------
- 0x18      | 4      |            | transfer_buffer_length
------------+--------+------------+---------------------------------------------------
- 0x1C      | 4      |            | start_frame: specify the selected frame to
-           |        |            |   transmit an ISO frame, ignored if URB_ISO_ASAP
-           |        |            |   is specified at transfer_flags
------------+--------+------------+---------------------------------------------------
- 0x20      | 4      |            | number_of_packets: number of ISO packets
------------+--------+------------+---------------------------------------------------
- 0x24      | 4      |            | interval: maximum time for the request on the
-           |        |            |   server-side host controller
------------+--------+------------+---------------------------------------------------
- 0x28      | 8      |            | setup: data bytes for USB setup, filled with
-           |        |            |   zeros if not used
------------+--------+------------+---------------------------------------------------
- 0x30      |        |            | URB data. For ISO transfers the padding between
-           |        |            |   each ISO packets is not transmitted.
++-----------+--------+------------+---------------------------------------------------+
+| Offset    | Length | Value      | Description                                       |
++===========+========+============+===================================================+
+| 0         | 4      | 0x00000001 | command: Submit an URB                            |
++-----------+--------+------------+---------------------------------------------------+
+| 4         | 4      |            | seqnum: the sequence number of the URB to submit  |
++-----------+--------+------------+---------------------------------------------------+
+| 8         | 4      |            | devid                                             |
++-----------+--------+------------+---------------------------------------------------+
+| 0xC       | 4      |            | direction:                                        |
+|           |        |            |                                                   |
+|           |        |            |    - 0: USBIP_DIR_OUT                             |
+|           |        |            |    - 1: USBIP_DIR_IN                              |
++-----------+--------+------------+---------------------------------------------------+
+| 0x10      | 4      |            | ep: endpoint number, possible values are: 0...15  |
++-----------+--------+------------+---------------------------------------------------+
+| 0x14      | 4      |            | transfer_flags: possible values depend on the     |
+|           |        |            | URB transfer type, see below                      |
++-----------+--------+------------+---------------------------------------------------+
+| 0x18      | 4      |            | transfer_buffer_length                            |
++-----------+--------+------------+---------------------------------------------------+
+| 0x1C      | 4      |            | start_frame: specify the selected frame to        |
+|           |        |            | transmit an ISO frame, ignored if URB_ISO_ASAP    |
+|           |        |            | is specified at transfer_flags                    |
++-----------+--------+------------+---------------------------------------------------+
+| 0x20      | 4      |            | number_of_packets: number of ISO packets          |
++-----------+--------+------------+---------------------------------------------------+
+| 0x24      | 4      |            | interval: maximum time for the request on the     |
+|           |        |            | server-side host controller                       |
++-----------+--------+------------+---------------------------------------------------+
+| 0x28      | 8      |            | setup: data bytes for USB setup, filled with      |
+|           |        |            | zeros if not used                                 |
++-----------+--------+------------+---------------------------------------------------+
+| 0x30      |        |            | URB data. For ISO transfers the padding between   |
+|           |        |            | each ISO packets is not transmitted.              |
++-----------+--------+------------+---------------------------------------------------+
 
 
-  Allowed transfer_flags  | value      | control | interrupt | bulk     | isochronous
- -------------------------+------------+---------+-----------+----------+-------------
-  URB_SHORT_NOT_OK        | 0x00000001 | only in | only in   | only in  | no
-  URB_ISO_ASAP            | 0x00000002 | no      | no        | no       | yes
-  URB_NO_TRANSFER_DMA_MAP | 0x00000004 | yes     | yes       | yes      | yes
-  URB_ZERO_PACKET         | 0x00000040 | no      | no        | only out | no
-  URB_NO_INTERRUPT        | 0x00000080 | yes     | yes       | yes      | yes
-  URB_FREE_BUFFER         | 0x00000100 | yes     | yes       | yes      | yes
-  URB_DIR_MASK            | 0x00000200 | yes     | yes       | yes      | yes
+ +-------------------------+------------+---------+-----------+----------+-------------+
+ | Allowed transfer_flags  | value      | control | interrupt | bulk     | isochronous |
+ +=========================+============+=========+===========+==========+=============+
+ | URB_SHORT_NOT_OK        | 0x00000001 | only in | only in   | only in  | no          |
+ +-------------------------+------------+---------+-----------+----------+-------------+
+ | URB_ISO_ASAP            | 0x00000002 | no      | no        | no       | yes         |
+ +-------------------------+------------+---------+-----------+----------+-------------+
+ | URB_NO_TRANSFER_DMA_MAP | 0x00000004 | yes     | yes       | yes      | yes         |
+ +-------------------------+------------+---------+-----------+----------+-------------+
+ | URB_ZERO_PACKET         | 0x00000040 | no      | no        | only out | no          |
+ +-------------------------+------------+---------+-----------+----------+-------------+
+ | URB_NO_INTERRUPT        | 0x00000080 | yes     | yes       | yes      | yes         |
+ +-------------------------+------------+---------+-----------+----------+-------------+
+ | URB_FREE_BUFFER         | 0x00000100 | yes     | yes       | yes      | yes         |
+ +-------------------------+------------+---------+-----------+----------+-------------+
+ | URB_DIR_MASK            | 0x00000200 | yes     | yes       | yes      | yes         |
+ +-------------------------+------------+---------+-----------+----------+-------------+
 
 
-USBIP_RET_SUBMIT: Reply for submitting an URB
+USBIP_RET_SUBMIT:
+	Reply for submitting an URB
 
- Offset    | Length | Value      | Description
------------+--------+------------+---------------------------------------------------
- 0         | 4      | 0x00000003 | command
------------+--------+------------+---------------------------------------------------
- 4         | 4      |            | seqnum: URB sequence number
------------+--------+------------+---------------------------------------------------
- 8         | 4      |            | devid
------------+--------+------------+---------------------------------------------------
- 0xC       | 4      |            | direction: 0: USBIP_DIR_OUT
-           |        |            |            1: USBIP_DIR_IN
------------+--------+------------+---------------------------------------------------
- 0x10      | 4      |            | ep: endpoint number
------------+--------+------------+---------------------------------------------------
- 0x14      | 4      |            | status: zero for successful URB transaction,
-           |        |            |   otherwise some kind of error happened.
------------+--------+------------+---------------------------------------------------
- 0x18      | 4      | n          | actual_length: number of URB data bytes
------------+--------+------------+---------------------------------------------------
- 0x1C      | 4      |            | start_frame: for an ISO frame the actually
-           |        |            |   selected frame for transmit.
------------+--------+------------+---------------------------------------------------
- 0x20      | 4      |            | number_of_packets
------------+--------+------------+---------------------------------------------------
- 0x24      | 4      |            | error_count
------------+--------+------------+---------------------------------------------------
- 0x28      | 8      |            | setup: data bytes for USB setup, filled with
-           |        |            |   zeros if not used
------------+--------+------------+---------------------------------------------------
- 0x30      | n      |            | URB data bytes. For ISO transfers the padding
-           |        |            |   between each ISO packets is not transmitted.
++-----------+--------+------------+---------------------------------------------------+
+| Offset    | Length | Value      | Description                                       |
++===========+========+============+===================================================+
+| 0         | 4      | 0x00000003 | command                                           |
++-----------+--------+------------+---------------------------------------------------+
+| 4         | 4      |            | seqnum: URB sequence number                       |
++-----------+--------+------------+---------------------------------------------------+
+| 8         | 4      |            | devid                                             |
++-----------+--------+------------+---------------------------------------------------+
+| 0xC       | 4      |            | direction:                                        |
+|           |        |            |                                                   |
+|           |        |            |    - 0: USBIP_DIR_OUT                             |
+|           |        |            |    - 1: USBIP_DIR_IN                              |
++-----------+--------+------------+---------------------------------------------------+
+| 0x10      | 4      |            | ep: endpoint number                               |
++-----------+--------+------------+---------------------------------------------------+
+| 0x14      | 4      |            | status: zero for successful URB transaction,      |
+|           |        |            | otherwise some kind of error happened.            |
++-----------+--------+------------+---------------------------------------------------+
+| 0x18      | 4      | n          | actual_length: number of URB data bytes           |
++-----------+--------+------------+---------------------------------------------------+
+| 0x1C      | 4      |            | start_frame: for an ISO frame the actually        |
+|           |        |            | selected frame for transmit.                      |
++-----------+--------+------------+---------------------------------------------------+
+| 0x20      | 4      |            | number_of_packets                                 |
++-----------+--------+------------+---------------------------------------------------+
+| 0x24      | 4      |            | error_count                                       |
++-----------+--------+------------+---------------------------------------------------+
+| 0x28      | 8      |            | setup: data bytes for USB setup, filled with      |
+|           |        |            | zeros if not used                                 |
++-----------+--------+------------+---------------------------------------------------+
+| 0x30      | n      |            | URB data bytes. For ISO transfers the padding     |
+|           |        |            | between each ISO packets is not transmitted.      |
++-----------+--------+------------+---------------------------------------------------+
 
-USBIP_CMD_UNLINK: Unlink an URB
+USBIP_CMD_UNLINK:
+	Unlink an URB
 
- Offset    | Length | Value      | Description
------------+--------+------------+---------------------------------------------------
- 0         | 4      | 0x00000002 | command: URB unlink command
------------+--------+------------+---------------------------------------------------
- 4         | 4      |            | seqnum: URB sequence number to unlink: FIXME: is this so?
------------+--------+------------+---------------------------------------------------
- 8         | 4      |            | devid
------------+--------+------------+---------------------------------------------------
- 0xC       | 4      |            | direction: 0: USBIP_DIR_OUT
-           |        |            |            1: USBIP_DIR_IN
------------+--------+------------+---------------------------------------------------
- 0x10      | 4      |            | ep: endpoint number: zero
------------+--------+------------+---------------------------------------------------
- 0x14      | 4      |            | seqnum: the URB sequence number given previously
-           |        |            |   at USBIP_CMD_SUBMIT.seqnum field
------------+--------+------------+---------------------------------------------------
- 0x30      | n      |            | URB data bytes. For ISO transfers the padding
-           |        |            |   between each ISO packets is not transmitted.
++-----------+--------+------------+---------------------------------------------------+
+| Offset    | Length | Value      | Description                                       |
++===========+========+============+===================================================+
+| 0         | 4      | 0x00000002 | command: URB unlink command                       |
++-----------+--------+------------+---------------------------------------------------+
+| 4         | 4      |            | seqnum: URB sequence number to unlink:            |
+|           |        |            |                                                   |
+|           |        |            | FIXME:                                            |
+|           |        |            |    is this so?                                    |
++-----------+--------+------------+---------------------------------------------------+
+| 8         | 4      |            | devid                                             |
++-----------+--------+------------+---------------------------------------------------+
+| 0xC       | 4      |            | direction:                                        |
+|           |        |            |                                                   |
+|           |        |            |    - 0: USBIP_DIR_OUT                             |
+|           |        |            |    - 1: USBIP_DIR_IN                              |
++-----------+--------+------------+---------------------------------------------------+
+| 0x10      | 4      |            | ep: endpoint number: zero                         |
++-----------+--------+------------+---------------------------------------------------+
+| 0x14      | 4      |            | seqnum: the URB sequence number given previously  |
+|           |        |            | at USBIP_CMD_SUBMIT.seqnum field                  |
++-----------+--------+------------+---------------------------------------------------+
+| 0x30      | n      |            | URB data bytes. For ISO transfers the padding     |
+|           |        |            | between each ISO packets is not transmitted.      |
++-----------+--------+------------+---------------------------------------------------+
 
-USBIP_RET_UNLINK: Reply for URB unlink
+USBIP_RET_UNLINK:
+	Reply for URB unlink
 
- Offset    | Length | Value      | Description
------------+--------+------------+---------------------------------------------------
- 0         | 4      | 0x00000004 | command: reply for the URB unlink command
------------+--------+------------+---------------------------------------------------
- 4         | 4      |            | seqnum: the unlinked URB sequence number
------------+--------+------------+---------------------------------------------------
- 8         | 4      |            | devid
------------+--------+------------+---------------------------------------------------
- 0xC       | 4      |            | direction: 0: USBIP_DIR_OUT
-           |        |            |            1: USBIP_DIR_IN
------------+--------+------------+---------------------------------------------------
- 0x10      | 4      |            | ep: endpoint number
------------+--------+------------+---------------------------------------------------
- 0x14      | 4      |            | status: This is the value contained in the
-           |        |            |   urb->status in the URB completition handler.
-           |        |            |   FIXME: a better explanation needed.
------------+--------+------------+---------------------------------------------------
- 0x30      | n      |            | URB data bytes. For ISO transfers the padding
-           |        |            |   between each ISO packets is not transmitted.
++-----------+--------+------------+---------------------------------------------------+
+| Offset    | Length | Value      | Description                                       |
++===========+========+============+===================================================+
+| 0         | 4      | 0x00000004 | command: reply for the URB unlink command         |
++-----------+--------+------------+---------------------------------------------------+
+| 4         | 4      |            | seqnum: the unlinked URB sequence number          |
++-----------+--------+------------+---------------------------------------------------+
+| 8         | 4      |            | devid                                             |
++-----------+--------+------------+---------------------------------------------------+
+| 0xC       | 4      |            | direction:                                        |
+|           |        |            |                                                   |
+|           |        |            |    - 0: USBIP_DIR_OUT                             |
+|           |        |            |    - 1: USBIP_DIR_IN                              |
++-----------+--------+------------+---------------------------------------------------+
+| 0x10      | 4      |            | ep: endpoint number                               |
++-----------+--------+------------+---------------------------------------------------+
+| 0x14      | 4      |            | status: This is the value contained in the        |
+|           |        |            | urb->status in the URB completition handler.      |
+|           |        |            |                                                   |
+|           |        |            | FIXME:                                            |
+|           |        |            |      a better explanation needed.                 |
++-----------+--------+------------+---------------------------------------------------+
+| 0x30      | n      |            | URB data bytes. For ISO transfers the padding     |
+|           |        |            | between each ISO packets is not transmitted.      |
++-----------+--------+------------+---------------------------------------------------+
diff --git a/Documentation/usb/usbmon.txt b/Documentation/usb/usbmon.txt
index 28425f736756..b0bd51080799 100644
--- a/Documentation/usb/usbmon.txt
+++ b/Documentation/usb/usbmon.txt
@@ -1,4 +1,9 @@
-* Introduction
+======
+usbmon
+======
+
+Introduction
+============
 
 The name "usbmon" in lowercase refers to a facility in kernel which is
 used to collect traces of I/O on the USB bus. This function is analogous
@@ -16,7 +21,8 @@ Two APIs are currently implemented: "text" and "binary". The binary API
 is available through a character device in /dev namespace and is an ABI.
 The text API is deprecated since 2.6.35, but available for convenience.
 
-* How to use usbmon to collect raw text traces
+How to use usbmon to collect raw text traces
+============================================
 
 Unlike the packet socket, usbmon has an interface which provides traces
 in a text format. This is used for two purposes. First, it serves as a
@@ -26,38 +32,41 @@ are finalized. Second, humans can read it in case tools are not available.
 To collect a raw text trace, execute following steps.
 
 1. Prepare
+----------
 
 Mount debugfs (it has to be enabled in your kernel configuration), and
 load the usbmon module (if built as module). The second step is skipped
-if usbmon is built into the kernel.
+if usbmon is built into the kernel::
 
-# mount -t debugfs none_debugs /sys/kernel/debug
-# modprobe usbmon
-#
+	# mount -t debugfs none_debugs /sys/kernel/debug
+	# modprobe usbmon
+	#
 
-Verify that bus sockets are present.
+Verify that bus sockets are present:
 
-# ls /sys/kernel/debug/usb/usbmon
-0s  0u  1s  1t  1u  2s  2t  2u  3s  3t  3u  4s  4t  4u
-#
+	# ls /sys/kernel/debug/usb/usbmon
+	0s  0u  1s  1t  1u  2s  2t  2u  3s  3t  3u  4s  4t  4u
+	#
 
 Now you can choose to either use the socket '0u' (to capture packets on all
 buses), and skip to step #3, or find the bus used by your device with step #2.
 This allows to filter away annoying devices that talk continuously.
 
 2. Find which bus connects to the desired device
+------------------------------------------------
 
 Run "cat /sys/kernel/debug/usb/devices", and find the T-line which corresponds
 to the device. Usually you do it by looking for the vendor string. If you have
 many similar devices, unplug one and compare the two
 /sys/kernel/debug/usb/devices outputs. The T-line will have a bus number.
-Example:
 
-T:  Bus=03 Lev=01 Prnt=01 Port=00 Cnt=01 Dev#=  2 Spd=12  MxCh= 0
-D:  Ver= 1.10 Cls=00(>ifc ) Sub=00 Prot=00 MxPS= 8 #Cfgs=  1
-P:  Vendor=0557 ProdID=2004 Rev= 1.00
-S:  Manufacturer=ATEN
-S:  Product=UC100KM V2.00
+Example::
+
+  T:  Bus=03 Lev=01 Prnt=01 Port=00 Cnt=01 Dev#=  2 Spd=12  MxCh= 0
+  D:  Ver= 1.10 Cls=00(>ifc ) Sub=00 Prot=00 MxPS= 8 #Cfgs=  1
+  P:  Vendor=0557 ProdID=2004 Rev= 1.00
+  S:  Manufacturer=ATEN
+  S:  Product=UC100KM V2.00
 
 "Bus=03" means it's bus 3. Alternatively, you can look at the output from
 "lsusb" and get the bus number from the appropriate line. Example:
@@ -65,23 +74,28 @@ S:  Product=UC100KM V2.00
 Bus 003 Device 002: ID 0557:2004 ATEN UC100KM V2.00
 
 3. Start 'cat'
+--------------
+
+::
 
-# cat /sys/kernel/debug/usb/usbmon/3u > /tmp/1.mon.out
+	# cat /sys/kernel/debug/usb/usbmon/3u > /tmp/1.mon.out
 
-to listen on a single bus, otherwise, to listen on all buses, type:
+to listen on a single bus, otherwise, to listen on all buses, type::
 
-# cat /sys/kernel/debug/usb/usbmon/0u > /tmp/1.mon.out
+	# cat /sys/kernel/debug/usb/usbmon/0u > /tmp/1.mon.out
 
 This process will read until it is killed. Naturally, the output can be
 redirected to a desirable location. This is preferred, because it is going
 to be quite long.
 
 4. Perform the desired operation on the USB bus
+-----------------------------------------------
 
 This is where you do something that creates the traffic: plug in a flash key,
 copy files, control a webcam, etc.
 
 5. Kill cat
+-----------
 
 Usually it's done with a keyboard interrupt (Control-C).
 
@@ -89,7 +103,8 @@ At this point the output file (/tmp/1.mon.out in this example) can be saved,
 sent by e-mail, or inspected with a text editor. In the last case make sure
 that the file size is not excessive for your favourite editor.
 
-* Raw text data format
+Raw text data format
+====================
 
 Two formats are supported currently: the original, or '1t' format, and
 the '1u' format. The '1t' format is deprecated in kernel 2.6.21. The '1u'
@@ -122,10 +137,14 @@ Here is the list of words, from left to right:
 - "Address" word (formerly a "pipe"). It consists of four fields, separated by
   colons: URB type and direction, Bus number, Device address, Endpoint number.
   Type and direction are encoded with two bytes in the following manner:
+
+    == ==   =============================
     Ci Co   Control input and output
     Zi Zo   Isochronous input and output
     Ii Io   Interrupt input and output
     Bi Bo   Bulk input and output
+    == ==   =============================
+
   Bus number, Device address, and Endpoint are decimal numbers, but they may
   have leading zeros, for the sake of human readers.
 
@@ -178,24 +197,25 @@ Here is the list of words, from left to right:
 
 Examples:
 
-An input control transfer to get a port status.
+An input control transfer to get a port status::
 
-d5ea89a0 3575914555 S Ci:1:001:0 s a3 00 0000 0003 0004 4 <
-d5ea89a0 3575914560 C Ci:1:001:0 0 4 = 01050000
+  d5ea89a0 3575914555 S Ci:1:001:0 s a3 00 0000 0003 0004 4 <
+  d5ea89a0 3575914560 C Ci:1:001:0 0 4 = 01050000
 
 An output bulk transfer to send a SCSI command 0x28 (READ_10) in a 31-byte
-Bulk wrapper to a storage device at address 5:
+Bulk wrapper to a storage device at address 5::
 
-dd65f0e8 4128379752 S Bo:1:005:2 -115 31 = 55534243 ad000000 00800000 80010a28 20000000 20000040 00000000 000000
-dd65f0e8 4128379808 C Bo:1:005:2 0 31 >
+  dd65f0e8 4128379752 S Bo:1:005:2 -115 31 = 55534243 ad000000 00800000 80010a28 20000000 20000040 00000000 000000
+  dd65f0e8 4128379808 C Bo:1:005:2 0 31 >
 
-* Raw binary format and API
+Raw binary format and API
+=========================
 
 The overall architecture of the API is about the same as the one above,
 only the events are delivered in binary format. Each event is sent in
-the following structure (its name is made up, so that we can refer to it):
+the following structure (its name is made up, so that we can refer to it)::
 
-struct usbmon_packet {
+  struct usbmon_packet {
 	u64 id;			/*  0: URB ID - from submission to callback */
 	unsigned char type;	/*  8: Same as text; extensible. */
 	unsigned char xfer_type; /*    ISO (0), Intr, Control, Bulk (3) */
@@ -220,7 +240,7 @@ struct usbmon_packet {
 	int start_frame;	/* 52: For ISO */
 	unsigned int xfer_flags; /* 56: copy of URB's transfer_flags */
 	unsigned int ndesc;	/* 60: Actual number of ISO descriptors */
-};				/* 64 total length */
+  };				/* 64 total length */
 
 These events can be received from a character device by reading with read(2),
 with an ioctl(2), or by accessing the buffer with mmap. However, read(2)
@@ -244,12 +264,12 @@ no events are available.
 
  MON_IOCG_STATS, defined as _IOR(MON_IOC_MAGIC, 3, struct mon_bin_stats)
 
-The argument is a pointer to the following structure:
+The argument is a pointer to the following structure::
 
-struct mon_bin_stats {
+  struct mon_bin_stats {
 	u32 queued;
 	u32 dropped;
-};
+  };
 
 The member "queued" refers to the number of events currently queued in the
 buffer (and not to the number of events processed since the last reset).
@@ -273,13 +293,13 @@ This call returns the current size of the buffer in bytes.
 
 These calls wait for events to arrive if none were in the kernel buffer,
 then return the first event. The argument is a pointer to the following
-structure:
+structure::
 
-struct mon_get_arg {
+  struct mon_get_arg {
 	struct usbmon_packet *hdr;
 	void *data;
 	size_t alloc;		/* Length of data (can be zero) */
-};
+  };
 
 Before the call, hdr, data, and alloc should be filled. Upon return, the area
 pointed by hdr contains the next event structure, and the data buffer contains
@@ -290,13 +310,13 @@ The MON_IOCX_GET copies 48 bytes to hdr area, MON_IOCX_GETX copies 64 bytes.
  MON_IOCX_MFETCH, defined as _IOWR(MON_IOC_MAGIC, 7, struct mon_mfetch_arg)
 
 This ioctl is primarily used when the application accesses the buffer
-with mmap(2). Its argument is a pointer to the following structure:
+with mmap(2). Its argument is a pointer to the following structure::
 
-struct mon_mfetch_arg {
+  struct mon_mfetch_arg {
 	uint32_t *offvec;	/* Vector of events fetched */
 	uint32_t nfetch;	/* Number of events to fetch (out: fetched) */
 	uint32_t nflush;	/* Number of events to flush */
-};
+  };
 
 The ioctl operates in 3 stages.
 
@@ -329,7 +349,7 @@ be polled with select(2) and poll(2). But lseek(2) does not work.
 The basic idea is simple:
 
 To prepare, map the buffer by getting the current size, then using mmap(2).
-Then, execute a loop similar to the one written in pseudo-code below:
+Then, execute a loop similar to the one written in pseudo-code below::
 
    struct mon_mfetch_arg fetch;
    struct usbmon_packet *hdr;
diff --git a/Documentation/userspace-api/seccomp_filter.rst b/Documentation/userspace-api/seccomp_filter.rst
index b1b846d8a094..bd9165241b6c 100644
--- a/Documentation/userspace-api/seccomp_filter.rst
+++ b/Documentation/userspace-api/seccomp_filter.rst
@@ -123,9 +123,9 @@ In precedence order, they are:
 	to userland as the errno without executing the system call.
 
 ``SECCOMP_RET_USER_NOTIF``:
-    Results in a ``struct seccomp_notif`` message sent on the userspace
-    notification fd, if it is attached, or ``-ENOSYS`` if it is not. See below
-    on discussion of how to handle user notifications.
+	Results in a ``struct seccomp_notif`` message sent on the userspace
+	notification fd, if it is attached, or ``-ENOSYS`` if it is not. See
+	below on discussion of how to handle user notifications.
 
 ``SECCOMP_RET_TRACE``:
 	When returned, this value will cause the kernel to attempt to
@@ -133,7 +133,7 @@ In precedence order, they are:
 	call.  If there is no tracer present, ``-ENOSYS`` is returned to
 	userland and the system call is not executed.
 
-	A tracer will be notified if it requests ``PTRACE_O_TRACESECCOM``P
+	A tracer will be notified if it requests ``PTRACE_O_TRACESECCOMP``
 	using ``ptrace(PTRACE_SETOPTIONS)``.  The tracer will be notified
 	of a ``PTRACE_EVENT_SECCOMP`` and the ``SECCOMP_RET_DATA`` portion of
 	the BPF program return value will be available to the tracer
diff --git a/Documentation/video-output.txt b/Documentation/video-output.txt
index e517011be4f9..56d6fa2e2368 100644
--- a/Documentation/video-output.txt
+++ b/Documentation/video-output.txt
@@ -1,34 +1,34 @@
+Video Output Switcher Control
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-		Video Output Switcher Control
-		~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-		2006 luming.yu@intel.com
+2006 luming.yu@intel.com
 
 The output sysfs class driver provides an abstract video output layer that
 can be used to hook platform specific methods to enable/disable video output
 device through common sysfs interface. For example, on my IBM ThinkPad T42
 laptop, The ACPI video driver registered its output devices and read/write
-method for 'state' with output sysfs class. The user interface under sysfs is:
+method for 'state' with output sysfs class. The user interface under sysfs is::
 
-linux:/sys/class/video_output # tree .
-.
-|-- CRT0
-|   |-- device -> ../../../devices/pci0000:00/0000:00:01.0
-|   |-- state
-|   |-- subsystem -> ../../../class/video_output
-|   `-- uevent
-|-- DVI0
-|   |-- device -> ../../../devices/pci0000:00/0000:00:01.0
-|   |-- state
-|   |-- subsystem -> ../../../class/video_output
-|   `-- uevent
-|-- LCD0
-|   |-- device -> ../../../devices/pci0000:00/0000:00:01.0
-|   |-- state
-|   |-- subsystem -> ../../../class/video_output
-|   `-- uevent
-`-- TV0
-   |-- device -> ../../../devices/pci0000:00/0000:00:01.0
-   |-- state
-   |-- subsystem -> ../../../class/video_output
-   `-- uevent
+  linux:/sys/class/video_output # tree .
+  .
+  |-- CRT0
+  |   |-- device -> ../../../devices/pci0000:00/0000:00:01.0
+  |   |-- state
+  |   |-- subsystem -> ../../../class/video_output
+  |   `-- uevent
+  |-- DVI0
+  |   |-- device -> ../../../devices/pci0000:00/0000:00:01.0
+  |   |-- state
+  |   |-- subsystem -> ../../../class/video_output
+  |   `-- uevent
+  |-- LCD0
+  |   |-- device -> ../../../devices/pci0000:00/0000:00:01.0
+  |   |-- state
+  |   |-- subsystem -> ../../../class/video_output
+  |   `-- uevent
+  `-- TV0
+     |-- device -> ../../../devices/pci0000:00/0000:00:01.0
+     |-- state
+     |-- subsystem -> ../../../class/video_output
+     `-- uevent
 
diff --git a/Documentation/vm/hugetlbfs_reserv.rst b/Documentation/vm/hugetlbfs_reserv.rst
index 9d200762114f..f143954e0d05 100644
--- a/Documentation/vm/hugetlbfs_reserv.rst
+++ b/Documentation/vm/hugetlbfs_reserv.rst
@@ -85,10 +85,10 @@ Reservation Map Location (Private or Shared)
 A huge page mapping or segment is either private or shared.  If private,
 it is typically only available to a single address space (task).  If shared,
 it can be mapped into multiple address spaces (tasks).  The location and
-semantics of the reservation map is significantly different for two types
+semantics of the reservation map is significantly different for the two types
 of mappings.  Location differences are:
 
-- For private mappings, the reservation map hangs off the the VMA structure.
+- For private mappings, the reservation map hangs off the VMA structure.
   Specifically, vma->vm_private_data.  This reserve map is created at the
   time the mapping (mmap(MAP_PRIVATE)) is created.
 - For shared mappings, the reservation map hangs off the inode.  Specifically,
@@ -109,15 +109,15 @@ These operations result in a call to the routine hugetlb_reserve_pages()::
 				  struct vm_area_struct *vma,
 				  vm_flags_t vm_flags)
 
-The first thing hugetlb_reserve_pages() does is check for the NORESERVE
+The first thing hugetlb_reserve_pages() does is check if the NORESERVE
 flag was specified in either the shmget() or mmap() call.  If NORESERVE
-was specified, then this routine returns immediately as no reservation
+was specified, then this routine returns immediately as no reservations
 are desired.
 
 The arguments 'from' and 'to' are huge page indices into the mapping or
 underlying file.  For shmget(), 'from' is always 0 and 'to' corresponds to
 the length of the segment/mapping.  For mmap(), the offset argument could
-be used to specify the offset into the underlying file.  In such a case
+be used to specify the offset into the underlying file.  In such a case,
 the 'from' and 'to' arguments have been adjusted by this offset.
 
 One of the big differences between PRIVATE and SHARED mappings is the way
@@ -138,7 +138,8 @@ to indicate this VMA owns the reservations.
 
 The reservation map is consulted to determine how many huge page reservations
 are needed for the current mapping/segment.  For private mappings, this is
-always the value (to - from).  However, for shared mappings it is possible that some reservations may already exist within the range (to - from).  See the
+always the value (to - from).  However, for shared mappings it is possible that
+some reservations may already exist within the range (to - from).  See the
 section :ref:`Reservation Map Modifications <resv_map_modifications>`
 for details on how this is accomplished.
 
@@ -165,7 +166,7 @@ these counters.
 If there were enough free huge pages and the global count resv_huge_pages
 was adjusted, then the reservation map associated with the mapping is
 modified to reflect the reservations.  In the case of a shared mapping, a
-file_region will exist that includes the range 'from' 'to'.  For private
+file_region will exist that includes the range 'from' - 'to'.  For private
 mappings, no modifications are made to the reservation map as lack of an
 entry indicates a reservation exists.
 
@@ -239,7 +240,7 @@ subpool accounting when the page is freed.
 The routine vma_commit_reservation() is then called to adjust the reserve
 map based on the consumption of the reservation.  In general, this involves
 ensuring the page is represented within a file_region structure of the region
-map.  For shared mappings where the the reservation was present, an entry
+map.  For shared mappings where the reservation was present, an entry
 in the reserve map already existed so no change is made.  However, if there
 was no reservation in a shared mapping or this was a private mapping a new
 entry must be created.
diff --git a/Documentation/vm/index.rst b/Documentation/vm/index.rst
index b58cc3bfe777..e8d943b21cf9 100644
--- a/Documentation/vm/index.rst
+++ b/Documentation/vm/index.rst
@@ -37,6 +37,7 @@ descriptions of data structures and algorithms.
    hwpoison
    hugetlbfs_reserv
    ksm
+   memory-model
    mmu_notifier
    numa
    overcommit-accounting
diff --git a/Documentation/vm/memory-model.rst b/Documentation/vm/memory-model.rst
new file mode 100644
index 000000000000..382f72ace1fc
--- /dev/null
+++ b/Documentation/vm/memory-model.rst
@@ -0,0 +1,183 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+.. _physical_memory_model:
+
+=====================
+Physical Memory Model
+=====================
+
+Physical memory in a system may be addressed in different ways. The
+simplest case is when the physical memory starts at address 0 and
+spans a contiguous range up to the maximal address. It could be,
+however, that this range contains small holes that are not accessible
+for the CPU. Then there could be several contiguous ranges at
+completely distinct addresses. And, don't forget about NUMA, where
+different memory banks are attached to different CPUs.
+
+Linux abstracts this diversity using one of the three memory models:
+FLATMEM, DISCONTIGMEM and SPARSEMEM. Each architecture defines what
+memory models it supports, what the default memory model is and
+whether it is possible to manually override that default.
+
+.. note::
+   At time of this writing, DISCONTIGMEM is considered deprecated,
+   although it is still in use by several architectures.
+
+All the memory models track the status of physical page frames using
+:c:type:`struct page` arranged in one or more arrays.
+
+Regardless of the selected memory model, there exists one-to-one
+mapping between the physical page frame number (PFN) and the
+corresponding `struct page`.
+
+Each memory model defines :c:func:`pfn_to_page` and :c:func:`page_to_pfn`
+helpers that allow the conversion from PFN to `struct page` and vice
+versa.
+
+FLATMEM
+=======
+
+The simplest memory model is FLATMEM. This model is suitable for
+non-NUMA systems with contiguous, or mostly contiguous, physical
+memory.
+
+In the FLATMEM memory model, there is a global `mem_map` array that
+maps the entire physical memory. For most architectures, the holes
+have entries in the `mem_map` array. The `struct page` objects
+corresponding to the holes are never fully initialized.
+
+To allocate the `mem_map` array, architecture specific setup code
+should call :c:func:`free_area_init_node` function or its convenience
+wrapper :c:func:`free_area_init`. Yet, the mappings array is not
+usable until the call to :c:func:`memblock_free_all` that hands all
+the memory to the page allocator.
+
+If an architecture enables `CONFIG_ARCH_HAS_HOLES_MEMORYMODEL` option,
+it may free parts of the `mem_map` array that do not cover the
+actual physical pages. In such case, the architecture specific
+:c:func:`pfn_valid` implementation should take the holes in the
+`mem_map` into account.
+
+With FLATMEM, the conversion between a PFN and the `struct page` is
+straightforward: `PFN - ARCH_PFN_OFFSET` is an index to the
+`mem_map` array.
+
+The `ARCH_PFN_OFFSET` defines the first page frame number for
+systems with physical memory starting at address different from 0.
+
+DISCONTIGMEM
+============
+
+The DISCONTIGMEM model treats the physical memory as a collection of
+`nodes` similarly to how Linux NUMA support does. For each node Linux
+constructs an independent memory management subsystem represented by
+`struct pglist_data` (or `pg_data_t` for short). Among other
+things, `pg_data_t` holds the `node_mem_map` array that maps
+physical pages belonging to that node. The `node_start_pfn` field of
+`pg_data_t` is the number of the first page frame belonging to that
+node.
+
+The architecture setup code should call :c:func:`free_area_init_node` for
+each node in the system to initialize the `pg_data_t` object and its
+`node_mem_map`.
+
+Every `node_mem_map` behaves exactly as FLATMEM's `mem_map` -
+every physical page frame in a node has a `struct page` entry in the
+`node_mem_map` array. When DISCONTIGMEM is enabled, a portion of the
+`flags` field of the `struct page` encodes the node number of the
+node hosting that page.
+
+The conversion between a PFN and the `struct page` in the
+DISCONTIGMEM model became slightly more complex as it has to determine
+which node hosts the physical page and which `pg_data_t` object
+holds the `struct page`.
+
+Architectures that support DISCONTIGMEM provide :c:func:`pfn_to_nid`
+to convert PFN to the node number. The opposite conversion helper
+:c:func:`page_to_nid` is generic as it uses the node number encoded in
+page->flags.
+
+Once the node number is known, the PFN can be used to index
+appropriate `node_mem_map` array to access the `struct page` and
+the offset of the `struct page` from the `node_mem_map` plus
+`node_start_pfn` is the PFN of that page.
+
+SPARSEMEM
+=========
+
+SPARSEMEM is the most versatile memory model available in Linux and it
+is the only memory model that supports several advanced features such
+as hot-plug and hot-remove of the physical memory, alternative memory
+maps for non-volatile memory devices and deferred initialization of
+the memory map for larger systems.
+
+The SPARSEMEM model presents the physical memory as a collection of
+sections. A section is represented with :c:type:`struct mem_section`
+that contains `section_mem_map` that is, logically, a pointer to an
+array of struct pages. However, it is stored with some other magic
+that aids the sections management. The section size and maximal number
+of section is specified using `SECTION_SIZE_BITS` and
+`MAX_PHYSMEM_BITS` constants defined by each architecture that
+supports SPARSEMEM. While `MAX_PHYSMEM_BITS` is an actual width of a
+physical address that an architecture supports, the
+`SECTION_SIZE_BITS` is an arbitrary value.
+
+The maximal number of sections is denoted `NR_MEM_SECTIONS` and
+defined as
+
+.. math::
+
+   NR\_MEM\_SECTIONS = 2 ^ {(MAX\_PHYSMEM\_BITS - SECTION\_SIZE\_BITS)}
+
+The `mem_section` objects are arranged in a two-dimensional array
+called `mem_sections`. The size and placement of this array depend
+on `CONFIG_SPARSEMEM_EXTREME` and the maximal possible number of
+sections:
+
+* When `CONFIG_SPARSEMEM_EXTREME` is disabled, the `mem_sections`
+  array is static and has `NR_MEM_SECTIONS` rows. Each row holds a
+  single `mem_section` object.
+* When `CONFIG_SPARSEMEM_EXTREME` is enabled, the `mem_sections`
+  array is dynamically allocated. Each row contains PAGE_SIZE worth of
+  `mem_section` objects and the number of rows is calculated to fit
+  all the memory sections.
+
+The architecture setup code should call :c:func:`memory_present` for
+each active memory range or use :c:func:`memblocks_present` or
+:c:func:`sparse_memory_present_with_active_regions` wrappers to
+initialize the memory sections. Next, the actual memory maps should be
+set up using :c:func:`sparse_init`.
+
+With SPARSEMEM there are two possible ways to convert a PFN to the
+corresponding `struct page` - a "classic sparse" and "sparse
+vmemmap". The selection is made at build time and it is determined by
+the value of `CONFIG_SPARSEMEM_VMEMMAP`.
+
+The classic sparse encodes the section number of a page in page->flags
+and uses high bits of a PFN to access the section that maps that page
+frame. Inside a section, the PFN is the index to the array of pages.
+
+The sparse vmemmap uses a virtually mapped memory map to optimize
+pfn_to_page and page_to_pfn operations. There is a global `struct
+page *vmemmap` pointer that points to a virtually contiguous array of
+`struct page` objects. A PFN is an index to that array and the the
+offset of the `struct page` from `vmemmap` is the PFN of that
+page.
+
+To use vmemmap, an architecture has to reserve a range of virtual
+addresses that will map the physical pages containing the memory
+map and make sure that `vmemmap` points to that range. In addition,
+the architecture should implement :c:func:`vmemmap_populate` method
+that will allocate the physical memory and create page tables for the
+virtual memory map. If an architecture does not have any special
+requirements for the vmemmap mappings, it can use default
+:c:func:`vmemmap_populate_basepages` provided by the generic memory
+management.
+
+The virtually mapped memory map allows storing `struct page` objects
+for persistent memory devices in pre-allocated storage on those
+devices. This storage is represented with :c:type:`struct vmem_altmap`
+that is eventually passed to vmemmap_populate() through a long chain
+of function calls. The vmemmap_populate() implementation may use the
+`vmem_altmap` along with :c:func:`altmap_alloc_block_buf` helper to
+allocate memory map on the persistent memory device.
diff --git a/Documentation/vm/numa.rst b/Documentation/vm/numa.rst
index 185d8a568168..5cae13e9a08b 100644
--- a/Documentation/vm/numa.rst
+++ b/Documentation/vm/numa.rst
@@ -109,8 +109,8 @@ System administrators and application designers can restrict a task's migration
 to improve NUMA locality using various CPU affinity command line interfaces,
 such as taskset(1) and numactl(1), and program interfaces such as
 sched_setaffinity(2).  Further, one can modify the kernel's default local
-allocation behavior using Linux NUMA memory policy.
-[see Documentation/admin-guide/mm/numa_memory_policy.rst.]
+allocation behavior using Linux NUMA memory policy. [see
+:ref:`Documentation/admin-guide/mm/numa_memory_policy.rst <numa_memory_policy>`].
 
 System administrators can restrict the CPUs and nodes' memories that a non-
 privileged user can specify in the scheduling or NUMA commands and functions
diff --git a/Documentation/vm/transhuge.rst b/Documentation/vm/transhuge.rst
index a8cf6809e36e..37c57ca32629 100644
--- a/Documentation/vm/transhuge.rst
+++ b/Documentation/vm/transhuge.rst
@@ -4,8 +4,9 @@
 Transparent Hugepage Support
 ============================
 
-This document describes design principles Transparent Hugepage (THP)
-Support and its interaction with other parts of the memory management.
+This document describes design principles for Transparent Hugepage (THP)
+support and its interaction with other parts of the memory management
+system.
 
 Design principles
 =================
@@ -37,31 +38,25 @@ get_user_pages and follow_page
 
 get_user_pages and follow_page if run on a hugepage, will return the
 head or tail pages as usual (exactly as they would do on
-hugetlbfs). Most gup users will only care about the actual physical
+hugetlbfs). Most GUP users will only care about the actual physical
 address of the page and its temporary pinning to release after the I/O
 is complete, so they won't ever notice the fact the page is huge. But
 if any driver is going to mangle over the page structure of the tail
 page (like for checking page->mapping or other bits that are relevant
 for the head page and not the tail page), it should be updated to jump
-to check head page instead. Taking reference on any head/tail page would
-prevent page from being split by anyone.
+to check head page instead. Taking a reference on any head/tail page would
+prevent the page from being split by anyone.
 
 .. note::
    these aren't new constraints to the GUP API, and they match the
-   same constrains that applies to hugetlbfs too, so any driver capable
+   same constraints that apply to hugetlbfs too, so any driver capable
    of handling GUP on hugetlbfs will also work fine on transparent
    hugepage backed mappings.
 
 In case you can't handle compound pages if they're returned by
-follow_page, the FOLL_SPLIT bit can be specified as parameter to
+follow_page, the FOLL_SPLIT bit can be specified as a parameter to
 follow_page, so that it will split the hugepages before returning
-them. Migration for example passes FOLL_SPLIT as parameter to
-follow_page because it's not hugepage aware and in fact it can't work
-at all on hugetlbfs (but it instead works fine on transparent
-hugepages thanks to FOLL_SPLIT). migration simply can't deal with
-hugepages being returned (as it's not only checking the pfn of the
-page and pinning it during the copy but it pretends to migrate the
-memory in regular page sizes and with regular pte/pmd mappings).
+them.
 
 Graceful fallback
 =================
@@ -72,11 +67,11 @@ pmd_offset. It's trivial to make the code transparent hugepage aware
 by just grepping for "pmd_offset" and adding split_huge_pmd where
 missing after pmd_offset returns the pmd. Thanks to the graceful
 fallback design, with a one liner change, you can avoid to write
-hundred if not thousand of lines of complex code to make your code
+hundreds if not thousands of lines of complex code to make your code
 hugepage aware.
 
 If you're not walking pagetables but you run into a physical hugepage
-but you can't handle it natively in your code, you can split it by
+that you can't handle natively in your code, you can split it by
 calling split_huge_page(page). This is what the Linux VM does before
 it tries to swapout the hugepage for example. split_huge_page() can fail
 if the page is pinned and you must handle this correctly.
@@ -103,18 +98,18 @@ split_huge_page() or split_huge_pmd() has a cost.
 
 To make pagetable walks huge pmd aware, all you need to do is to call
 pmd_trans_huge() on the pmd returned by pmd_offset. You must hold the
-mmap_sem in read (or write) mode to be sure an huge pmd cannot be
+mmap_sem in read (or write) mode to be sure a huge pmd cannot be
 created from under you by khugepaged (khugepaged collapse_huge_page
 takes the mmap_sem in write mode in addition to the anon_vma lock). If
 pmd_trans_huge returns false, you just fallback in the old code
 paths. If instead pmd_trans_huge returns true, you have to take the
 page table lock (pmd_lock()) and re-run pmd_trans_huge. Taking the
-page table lock will prevent the huge pmd to be converted into a
+page table lock will prevent the huge pmd being converted into a
 regular pmd from under you (split_huge_pmd can run in parallel to the
 pagetable walk). If the second pmd_trans_huge returns false, you
 should just drop the page table lock and fallback to the old code as
-before. Otherwise you can proceed to process the huge pmd and the
-hugepage natively. Once finished you can drop the page table lock.
+before. Otherwise, you can proceed to process the huge pmd and the
+hugepage natively. Once finished, you can drop the page table lock.
 
 Refcounts and transparent huge pages
 ====================================
@@ -122,61 +117,61 @@ Refcounts and transparent huge pages
 Refcounting on THP is mostly consistent with refcounting on other compound
 pages:
 
-  - get_page()/put_page() and GUP operate in head page's ->_refcount.
+  - get_page()/put_page() and GUP operate on head page's ->_refcount.
 
   - ->_refcount in tail pages is always zero: get_page_unless_zero() never
-    succeed on tail pages.
+    succeeds on tail pages.
 
   - map/unmap of the pages with PTE entry increment/decrement ->_mapcount
     on relevant sub-page of the compound page.
 
-  - map/unmap of the whole compound page accounted in compound_mapcount
+  - map/unmap of the whole compound page is accounted for in compound_mapcount
     (stored in first tail page). For file huge pages, we also increment
     ->_mapcount of all sub-pages in order to have race-free detection of
     last unmap of subpages.
 
 PageDoubleMap() indicates that the page is *possibly* mapped with PTEs.
 
-For anonymous pages PageDoubleMap() also indicates ->_mapcount in all
+For anonymous pages, PageDoubleMap() also indicates ->_mapcount in all
 subpages is offset up by one. This additional reference is required to
 get race-free detection of unmap of subpages when we have them mapped with
 both PMDs and PTEs.
 
-This is optimization required to lower overhead of per-subpage mapcount
-tracking. The alternative is alter ->_mapcount in all subpages on each
+This optimization is required to lower the overhead of per-subpage mapcount
+tracking. The alternative is to alter ->_mapcount in all subpages on each
 map/unmap of the whole compound page.
 
-For anonymous pages, we set PG_double_map when a PMD of the page got split
-for the first time, but still have PMD mapping. The additional references
-go away with last compound_mapcount.
+For anonymous pages, we set PG_double_map when a PMD of the page is split
+for the first time, but still have a PMD mapping. The additional references
+go away with the last compound_mapcount.
 
-File pages get PG_double_map set on first map of the page with PTE and
-goes away when the page gets evicted from page cache.
+File pages get PG_double_map set on the first map of the page with PTE and
+goes away when the page gets evicted from the page cache.
 
 split_huge_page internally has to distribute the refcounts in the head
 page to the tail pages before clearing all PG_head/tail bits from the page
 structures. It can be done easily for refcounts taken by page table
-entries. But we don't have enough information on how to distribute any
+entries, but we don't have enough information on how to distribute any
 additional pins (i.e. from get_user_pages). split_huge_page() fails any
-requests to split pinned huge page: it expects page count to be equal to
-sum of mapcount of all sub-pages plus one (split_huge_page caller must
-have reference for head page).
+requests to split pinned huge pages: it expects page count to be equal to
+the sum of mapcount of all sub-pages plus one (split_huge_page caller must
+have a reference to the head page).
 
 split_huge_page uses migration entries to stabilize page->_refcount and
-page->_mapcount of anonymous pages. File pages just got unmapped.
+page->_mapcount of anonymous pages. File pages just get unmapped.
 
-We safe against physical memory scanners too: the only legitimate way
-scanner can get reference to a page is get_page_unless_zero().
+We are safe against physical memory scanners too: the only legitimate way
+a scanner can get a reference to a page is get_page_unless_zero().
 
 All tail pages have zero ->_refcount until atomic_add(). This prevents the
 scanner from getting a reference to the tail page up to that point. After the
-atomic_add() we don't care about the ->_refcount value. We already known how
+atomic_add() we don't care about the ->_refcount value. We already know how
 many references should be uncharged from the head page.
 
 For head page get_page_unless_zero() will succeed and we don't mind. It's
-clear where reference should go after split: it will stay on head page.
+clear where references should go after split: it will stay on the head page.
 
-Note that split_huge_pmd() doesn't have any limitation on refcounting:
+Note that split_huge_pmd() doesn't have any limitations on refcounting:
 pmd can be split at any point and never fails.
 
 Partial unmap and deferred_split_huge_page()
@@ -188,10 +183,10 @@ in page_remove_rmap() and queue the THP for splitting if memory pressure
 comes. Splitting will free up unused subpages.
 
 Splitting the page right away is not an option due to locking context in
-the place where we can detect partial unmap. It's also might be
+the place where we can detect partial unmap. It also might be
 counterproductive since in many cases partial unmap happens during exit(2) if
 a THP crosses a VMA boundary.
 
-Function deferred_split_huge_page() is used to queue page for splitting.
+The function deferred_split_huge_page() is used to queue a page for splitting.
 The splitting itself will happen when we get memory pressure via shrinker
 interface.
diff --git a/Documentation/x86/boot.txt b/Documentation/x86/boot.txt
index f4c2a97bfdbd..223e484a1304 100644
--- a/Documentation/x86/boot.txt
+++ b/Documentation/x86/boot.txt
@@ -61,6 +61,10 @@ Protocol 2.12:	(Kernel 3.8) Added the xloadflags field and extension fields
 		to struct boot_params for loading bzImage and ramdisk
 		above 4G in 64bit.
 
+Protocol 2.13:	(Kernel 3.14) Support 32- and 64-bit flags being set in
+		xloadflags to support booting a 64-bit kernel from 32-bit
+		EFI
+
 **** MEMORY LAYOUT
 
 The traditional memory map for the kernel loader, used for Image or
diff --git a/LICENSES/other/GPL-1.0 b/LICENSES/deprecated/GPL-1.0
index 3a4fa969e4c2..3a4fa969e4c2 100644
--- a/LICENSES/other/GPL-1.0
+++ b/LICENSES/deprecated/GPL-1.0
diff --git a/LICENSES/other/ISC b/LICENSES/deprecated/ISC
index 8953c3142079..8953c3142079 100644
--- a/LICENSES/other/ISC
+++ b/LICENSES/deprecated/ISC
diff --git a/LICENSES/other/Linux-OpenIB b/LICENSES/deprecated/Linux-OpenIB
index 1ad85f6b3a89..1ad85f6b3a89 100644
--- a/LICENSES/other/Linux-OpenIB
+++ b/LICENSES/deprecated/Linux-OpenIB
diff --git a/LICENSES/other/X11 b/LICENSES/deprecated/X11
index fe4353fd0000..fe4353fd0000 100644
--- a/LICENSES/other/X11
+++ b/LICENSES/deprecated/X11
diff --git a/LICENSES/other/Apache-2.0 b/LICENSES/dual/Apache-2.0
index 7cd903f573e5..6e89ddeab187 100644
--- a/LICENSES/other/Apache-2.0
+++ b/LICENSES/dual/Apache-2.0
@@ -1,6 +1,10 @@
 Valid-License-Identifier: Apache-2.0
 SPDX-URL: https://spdx.org/licenses/Apache-2.0.html
 Usage-Guide:
+  Do NOT use. The Apache-2.0 is not GPL2 compatible. It may only be used
+  for dual-licensed files where the other license is GPL2 compatible.
+  If you end up using this it MUST be used together with a GPL2 compatible
+  license using "OR".
   To use the Apache License version 2.0 put the following SPDX tag/value
   pair into a comment according to the placement guidelines in the
   licensing rules documentation:
diff --git a/LICENSES/other/CDDL-1.0 b/LICENSES/dual/CDDL-1.0
index 25f614276ddd..b0ca1016db79 100644
--- a/LICENSES/other/CDDL-1.0
+++ b/LICENSES/dual/CDDL-1.0
@@ -1,8 +1,8 @@
 Valid-License-Identifier: CDDL-1.0
 SPDX-URL: https://spdx.org/licenses/CDDL-1.0.html
 Usage-Guide:
-  Do NOT use. The CDDL-1.0 is not GPL compatible. It may only be used for
-  dual-licensed files where the other license is GPL compatible.
+  Do NOT use. The CDDL-1.0 is not GPL2 compatible. It may only be used for
+  dual-licensed files where the other license is GPL2 compatible.
   If you end up using this it MUST be used together with a GPL2 compatible
   license using "OR".
   To use the Common Development and Distribution License 1.0 put the
diff --git a/LICENSES/other/MPL-1.1 b/LICENSES/dual/MPL-1.1
index 568b6049efe6..61706859e1b2 100644
--- a/LICENSES/other/MPL-1.1
+++ b/LICENSES/dual/MPL-1.1
@@ -1,6 +1,10 @@
 Valid-License-Identifier: MPL-1.1
 SPDX-URL: https://spdx.org/licenses/MPL-1.1.html
 Usage-Guide:
+  Do NOT use. The MPL-1.1 is not GPL2 compatible. It may only be used for
+  dual-licensed files where the other license is GPL2 compatible.
+  If you end up using this it MUST be used together with a GPL2 compatible
+  license using "OR".
   To use the Mozilla Public License version 1.1 put the following SPDX
   tag/value pair into a comment according to the placement guidelines in
   the licensing rules documentation:
diff --git a/MAINTAINERS b/MAINTAINERS
index 920a0a1545b7..960070e36bd9 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -745,6 +745,15 @@ S:	Supported
 F:	Documentation/networking/device_drivers/amazon/ena.txt
 F:	drivers/net/ethernet/amazon/
 
+AMAZON RDMA EFA DRIVER
+M:	Gal Pressman <galpress@amazon.com>
+R:	Yossi Leybovich <sleybo@amazon.com>
+L:	linux-rdma@vger.kernel.org
+Q:	https://patchwork.kernel.org/project/linux-rdma/list/
+S:	Supported
+F:	drivers/infiniband/hw/efa/
+F:	include/uapi/rdma/efa-abi.h
+
 AMD CRYPTOGRAPHIC COPROCESSOR (CCP) DRIVER
 M:	Tom Lendacky <thomas.lendacky@amd.com>
 M:	Gary Hook <gary.hook@amd.com>
@@ -817,6 +826,14 @@ F:	drivers/gpu/drm/amd/include/vi_structs.h
 F:	drivers/gpu/drm/amd/include/v9_structs.h
 F:	include/uapi/linux/kfd_ioctl.h
 
+AMD MP2 I2C DRIVER
+M:	Elie Morisse <syniurge@gmail.com>
+M:	Nehal Shah <nehal-bakulchandra.shah@amd.com>
+M:	Shyam Sundar S K <shyam-sundar.s-k@amd.com>
+L:	linux-i2c@vger.kernel.org
+S:	Maintained
+F:	drivers/i2c/busses/i2c-amd-mp2*
+
 AMD POWERPLAY
 M:	Rex Zhu <rex.zhu@amd.com>
 M:	Evan Quan <evan.quan@amd.com>
@@ -962,6 +979,12 @@ F:	drivers/iio/adc/ltc2497*
 X:	drivers/iio/*/adjd*
 F:	drivers/staging/iio/*/ad*
 
+ANALOGBITS PLL LIBRARIES
+M:	Paul Walmsley <paul.walmsley@sifive.com>
+S:	Supported
+F:	drivers/clk/analogbits/*
+F:	include/linux/clk/analogbits*
+
 ANDES ARCHITECTURE
 M:	Greentime Hu <green.hu@gmail.com>
 M:	Vincent Chen <deanbo422@gmail.com>
@@ -1169,7 +1192,7 @@ S:	Supported
 T:	git git://linux-arm.org/linux-ld.git for-upstream/mali-dp
 F:	drivers/gpu/drm/arm/display/include/
 F:	drivers/gpu/drm/arm/display/komeda/
-F:	Documentation/devicetree/bindings/display/arm/arm,komeda.txt
+F:	Documentation/devicetree/bindings/display/arm,komeda.txt
 F:	Documentation/gpu/komeda-kms.rst
 
 ARM MALI-DP DRM DRIVER
@@ -1182,6 +1205,15 @@ F:	drivers/gpu/drm/arm/
 F:	Documentation/devicetree/bindings/display/arm,malidp.txt
 F:	Documentation/gpu/afbc.rst
 
+ARM MALI PANFROST DRM DRIVER
+M:	Rob Herring <robh@kernel.org>
+M:	Tomeu Vizoso <tomeu.vizoso@collabora.com>
+L:	dri-devel@lists.freedesktop.org
+S:	Supported
+T:	git git://anongit.freedesktop.org/drm/drm-misc
+F:	drivers/gpu/drm/panfrost/
+F:	include/uapi/drm/panfrost_drm.h
+
 ARM MFM AND FLOPPY DRIVERS
 M:	Ian Molton <spyro@f2s.com>
 S:	Maintained
@@ -1418,7 +1450,9 @@ M:	Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
 L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
 S:	Maintained
 F:	arch/arm64/boot/dts/bitmain/
+F:	drivers/pinctrl/pinctrl-bm1880.c
 F:	Documentation/devicetree/bindings/arm/bitmain.yaml
+F:	Documentation/devicetree/bindings/pinctrl/bitmain,bm1880-pinctrl.txt
 
 ARM/CALXEDA HIGHBANK ARCHITECTURE
 M:	Rob Herring <robh@kernel.org>
@@ -2562,7 +2596,7 @@ F:	include/linux/dmaengine.h
 F:	include/linux/async_tx.h
 
 AT24 EEPROM DRIVER
-M:	Bartosz Golaszewski <brgl@bgdev.pl>
+M:	Bartosz Golaszewski <bgolaszewski@baylibre.com>
 L:	linux-i2c@vger.kernel.org
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git
 S:	Maintained
@@ -3358,7 +3392,7 @@ F:	include/uapi/linux/bsg.h
 BT87X AUDIO DRIVER
 M:	Clemens Ladisch <clemens@ladisch.de>
 L:	alsa-devel@alsa-project.org (moderated for non-subscribers)
-T:	git git://git.alsa-project.org/alsa-kernel.git
+T:	git git://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound.git
 S:	Maintained
 F:	Documentation/sound/cards/bt87x.rst
 F:	sound/pci/bt87x.c
@@ -3411,7 +3445,7 @@ F:	drivers/scsi/FlashPoint.*
 C-MEDIA CMI8788 DRIVER
 M:	Clemens Ladisch <clemens@ladisch.de>
 L:	alsa-devel@alsa-project.org (moderated for non-subscribers)
-T:	git git://git.alsa-project.org/alsa-kernel.git
+T:	git git://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound.git
 S:	Maintained
 F:	sound/pci/oxygen/
 
@@ -3735,8 +3769,8 @@ F:	scripts/checkpatch.pl
 
 CHINESE DOCUMENTATION
 M:	Harry Wei <harryxiyou@gmail.com>
+M:	Alex Shi <alex.shi@linux.alibaba.com>
 L:	xiyoulinuxkernelgroup@googlegroups.com (subscribers-only)
-L:	linux-kernel@zh-kernel.org (moderated for non-subscribers)
 S:	Maintained
 F:	Documentation/translations/zh_CN/
 
@@ -3807,6 +3841,7 @@ F:	drivers/hwmon/lochnagar-hwmon.c
 F:	drivers/mfd/lochnagar-i2c.c
 F:	drivers/pinctrl/cirrus/pinctrl-lochnagar.c
 F:	drivers/regulator/lochnagar-regulator.c
+F:	sound/soc/codecs/lochnagar-sc.c
 F:	include/dt-bindings/clk/lochnagar.h
 F:	include/dt-bindings/pinctrl/lochnagar.h
 F:	include/linux/mfd/lochnagar*
@@ -3815,6 +3850,7 @@ F:	Documentation/devicetree/bindings/clock/cirrus,lochnagar.txt
 F:	Documentation/devicetree/bindings/hwmon/cirrus,lochnagar.txt
 F:	Documentation/devicetree/bindings/pinctrl/cirrus,lochnagar.txt
 F:	Documentation/devicetree/bindings/regulator/cirrus,lochnagar.txt
+F:	Documentation/devicetree/bindings/sound/cirrus,lochnagar.txt
 F:	Documentation/hwmon/lochnagar
 
 CISCO FCOE HBA DRIVER
@@ -4266,7 +4302,7 @@ S:	Supported
 F:	drivers/scsi/cxgbi/cxgb3i
 
 CXGB3 IWARP RNIC DRIVER (IW_CXGB3)
-M:	Steve Wise <swise@chelsio.com>
+M:	Potnuri Bharat Teja <bharat@chelsio.com>
 L:	linux-rdma@vger.kernel.org
 W:	http://www.openfabrics.org
 S:	Supported
@@ -4295,7 +4331,7 @@ S:	Supported
 F:	drivers/scsi/cxgbi/cxgb4i
 
 CXGB4 IWARP RNIC DRIVER (IW_CXGB4)
-M:	Steve Wise <swise@chelsio.com>
+M:	Potnuri Bharat Teja <bharat@chelsio.com>
 L:	linux-rdma@vger.kernel.org
 W:	http://www.openfabrics.org
 S:	Supported
@@ -4915,6 +4951,14 @@ M:	Dave Airlie <airlied@redhat.com>
 S:	Odd Fixes
 F:	drivers/gpu/drm/ast/
 
+DRM DRIVER FOR ASPEED BMC GFX
+M:	Joel Stanley <joel@jms.id.au>
+L:	linux-aspeed@lists.ozlabs.org
+T:	git git://anongit.freedesktop.org/drm/drm-misc
+S:	Supported
+F:	drivers/gpu/drm/aspeed/
+F:	Documentation/devicetree/bindings/gpu/aspeed-gfx.txt
+
 DRM DRIVER FOR BOCHS VIRTUAL GPU
 M:	Gerd Hoffmann <kraxel@redhat.com>
 L:	virtualization@lists.linux-foundation.org
@@ -4928,6 +4972,12 @@ T:	git git://anongit.freedesktop.org/drm/drm-misc
 S:	Maintained
 F:	drivers/gpu/drm/tve200/
 
+DRM DRIVER FOR FEIYANG FY07024DI26A30-D MIPI-DSI LCD PANELS
+M:	Jagan Teki <jagan@amarulasolutions.com>
+S:	Maintained
+F:	drivers/gpu/drm/panel/panel-feiyang-fy07024di26a30d.c
+F:	Documentation/devicetree/bindings/display/panel/feiyang,fy07024di26a30d.txt
+
 DRM DRIVER FOR ILITEK ILI9225 PANELS
 M:	David Lechner <david@lechnology.com>
 S:	Maintained
@@ -5019,6 +5069,12 @@ S:	Orphan / Obsolete
 F:	drivers/gpu/drm/r128/
 F:	include/uapi/drm/r128_drm.h
 
+DRM DRIVER FOR ROCKTECH JH057N00900 PANELS
+M:	Guido Günther <agx@sigxcpu.org>
+S:	Maintained
+F:	drivers/gpu/drm/panel/panel-rocktech-jh057n00900.c
+F:	Documentation/devicetree/bindings/display/panel/rocktech,jh057n00900.txt
+
 DRM DRIVER FOR SAVAGE VIDEO CARDS
 S:	Orphan / Obsolete
 F:	drivers/gpu/drm/savage/
@@ -5066,6 +5122,13 @@ S:	Odd Fixes
 F:	drivers/gpu/drm/udl/
 T:	git git://anongit.freedesktop.org/drm/drm-misc
 
+DRM DRIVER FOR VIRTUALBOX VIRTUAL GPU
+M:	Hans de Goede <hdegoede@redhat.com>
+L:	dri-devel@lists.freedesktop.org
+S:	Maintained
+F:	drivers/gpu/drm/vboxvideo/
+T:	git git://anongit.freedesktop.org/drm/drm-misc
+
 DRM DRIVER FOR VIRTUAL KERNEL MODESETTING (VKMS)
 M:	Rodrigo Siqueira <rodrigosiqueiramelo@gmail.com>
 R:	Haneen Mohammed <hamohammed.sa@gmail.com>
@@ -5200,6 +5263,15 @@ S:	Maintained
 F:	drivers/gpu/drm/hisilicon/
 F:	Documentation/devicetree/bindings/display/hisilicon/
 
+DRM DRIVERS FOR LIMA
+M:	Qiang Yu <yuq825@gmail.com>
+L:	dri-devel@lists.freedesktop.org
+L:	lima@lists.freedesktop.org (moderated for non-subscribers)
+S:	Maintained
+F:	drivers/gpu/drm/lima/
+F:	include/uapi/drm/lima_drm.h
+T:	git git://anongit.freedesktop.org/drm/drm-misc
+
 DRM DRIVERS FOR MEDIATEK
 M:	CK Hu <ck.hu@mediatek.com>
 M:	Philipp Zabel <p.zabel@pengutronix.de>
@@ -5723,7 +5795,7 @@ F:	drivers/edac/qcom_edac.c
 EDIROL UA-101/UA-1000 DRIVER
 M:	Clemens Ladisch <clemens@ladisch.de>
 L:	alsa-devel@alsa-project.org (moderated for non-subscribers)
-T:	git git://git.alsa-project.org/alsa-kernel.git
+T:	git git://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound.git
 S:	Maintained
 F:	sound/usb/misc/ua101.c
 
@@ -6063,7 +6135,7 @@ F:	include/linux/f75375s.h
 FIREWIRE AUDIO DRIVERS
 M:	Clemens Ladisch <clemens@ladisch.de>
 L:	alsa-devel@alsa-project.org (moderated for non-subscribers)
-T:	git git://git.alsa-project.org/alsa-kernel.git
+T:	git git://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound.git
 S:	Maintained
 F:	sound/firewire/
 
@@ -7678,6 +7750,10 @@ F:	drivers/infiniband/
 F:	include/uapi/linux/if_infiniband.h
 F:	include/uapi/rdma/
 F:	include/rdma/
+F:	include/trace/events/ib_mad.h
+F:	include/trace/events/ib_umad.h
+F:	samples/bpf/ibumad_kern.c
+F:	samples/bpf/ibumad_user.c
 
 INGENIC JZ4780 DMA Driver
 M:	Zubair Lutfullah Kakakhel <Zubair.Kakakhel@imgtec.com>
@@ -10088,6 +10164,7 @@ L:	linux-amlogic@lists.infradead.org
 W:	http://linux-meson.com/
 S:	Supported
 F:	drivers/media/platform/meson/ao-cec.c
+F:	drivers/media/platform/meson/ao-cec-g12a.c
 F:	Documentation/devicetree/bindings/media/meson-ao-cec.txt
 T:	git git://linuxtv.org/media_tree.git
 
@@ -10144,7 +10221,8 @@ MICROCHIP I2C DRIVER
 M:	Ludovic Desroches <ludovic.desroches@microchip.com>
 L:	linux-i2c@vger.kernel.org
 S:	Supported
-F:	drivers/i2c/busses/i2c-at91.c
+F:	drivers/i2c/busses/i2c-at91.h
+F:	drivers/i2c/busses/i2c-at91-*.c
 
 MICROCHIP ISC DRIVER
 M:	Eugen Hristev <eugen.hristev@microchip.com>
@@ -10466,7 +10544,7 @@ F:	include/uapi/linux/meye.h
 MOXA SMARTIO/INDUSTIO/INTELLIO SERIAL CARD
 M:	Jiri Slaby <jirislaby@gmail.com>
 S:	Maintained
-F:	Documentation/serial/moxa-smartio
+F:	Documentation/serial/moxa-smartio.rst
 F:	drivers/tty/mxser.*
 
 MR800 AVERMEDIA USB FM RADIO DRIVER
@@ -11652,7 +11730,7 @@ F:	Documentation/devicetree/bindings/opp/
 OPL4 DRIVER
 M:	Clemens Ladisch <clemens@ladisch.de>
 L:	alsa-devel@alsa-project.org (moderated for non-subscribers)
-T:	git git://git.alsa-project.org/alsa-kernel.git
+T:	git git://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound.git
 S:	Maintained
 F:	sound/drivers/opl4/
 
@@ -13376,7 +13454,7 @@ ROCKETPORT DRIVER
 P:	Comtrol Corp.
 W:	http://www.comtrol.com
 S:	Maintained
-F:	Documentation/serial/rocket.txt
+F:	Documentation/serial/rocket.rst
 F:	drivers/tty/rocket*
 
 ROCKETPORT EXPRESS/INFINITY DRIVER
@@ -14399,9 +14477,8 @@ SOC-CAMERA V4L2 SUBSYSTEM
 L:	linux-media@vger.kernel.org
 T:	git git://linuxtv.org/media_tree.git
 S:	Orphan
-F:	include/media/soc*
-F:	drivers/media/i2c/soc_camera/
-F:	drivers/media/platform/soc_camera/
+F:	include/media/soc_camera.h
+F:	drivers/staging/media/soc_camera/
 
 SOCIONEXT SYNQUACER I2C DRIVER
 M:	Ard Biesheuvel <ard.biesheuvel@linaro.org>
@@ -14563,7 +14640,6 @@ M:	Takashi Iwai <tiwai@suse.com>
 L:	alsa-devel@alsa-project.org (moderated for non-subscribers)
 W:	http://www.alsa-project.org/
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound.git
-T:	git git://git.alsa-project.org/alsa-kernel.git
 Q:	http://patchwork.kernel.org/project/alsa-devel/list/
 S:	Maintained
 F:	Documentation/sound/
@@ -14741,6 +14817,14 @@ S:	Maintained
 F:	drivers/iio/imu/st_lsm6dsx/
 F:	Documentation/devicetree/bindings/iio/imu/st_lsm6dsx.txt
 
+ST MIPID02 CSI-2 TO PARALLEL BRIDGE DRIVER
+M:	Mickael Guene <mickael.guene@st.com>
+L:	linux-media@vger.kernel.org
+T:	git git://linuxtv.org/media_tree.git
+S:	Maintained
+F:	drivers/media/i2c/st-mipid02.c
+F:	Documentation/devicetree/bindings/media/i2c/st,st-mipid02.txt
+
 ST STM32 I2C/SMBUS DRIVER
 M:	Pierre-Yves MORDRET <pierre-yves.mordret@st.com>
 L:	linux-i2c@vger.kernel.org
@@ -15377,6 +15461,11 @@ M:	Laxman Dewangan <ldewangan@nvidia.com>
 S:	Supported
 F:	drivers/spi/spi-tegra*
 
+TEGRA XUSB PADCTL DRIVER
+M:	JC Kuo <jckuo@nvidia.com>
+S:	Supported
+F:	drivers/phy/tegra/xusb*
+
 TEHUTI ETHERNET DRIVER
 M:	Andy Gospodarek <andy@greyhouse.net>
 L:	netdev@vger.kernel.org
@@ -16052,6 +16141,13 @@ L:	linux-scsi@vger.kernel.org
 S:	Supported
 F:	drivers/scsi/ufs/*dwc*
 
+UNIVERSAL FLASH STORAGE HOST CONTROLLER DRIVER MEDIATEK HOOKS
+M:	Stanley Chu <stanley.chu@mediatek.com>
+L:	linux-scsi@vger.kernel.org
+L:	linux-mediatek@lists.infradead.org (moderated for non-subscribers)
+S:	Maintained
+F:	drivers/scsi/ufs/ufs-mediatek*
+
 UNSORTED BLOCK IMAGES (UBI)
 M:	Artem Bityutskiy <dedekind1@gmail.com>
 M:	Richard Weinberger <richard@nod.at>
@@ -16155,6 +16251,14 @@ L:	linux-usb@vger.kernel.org
 S:	Maintained
 F:	drivers/usb/roles/intel-xhci-usb-role-switch.c
 
+USB IP DRIVER FOR HISILICON KIRIN
+M:	Yu Chen <chenyu56@huawei.com>
+M:	Binghui Wang <wangbinghui@hisilicon.com>
+L:	linux-usb@vger.kernel.org
+S:	Maintained
+F:	Documentation/devicetree/bindings/phy/phy-hi3660-usb3.txt
+F:	drivers/phy/hisilicon/phy-hi3660-usb3.c
+
 USB ISP116X DRIVER
 M:	Olav Kongas <ok@artecdesign.ee>
 L:	linux-usb@vger.kernel.org
@@ -16181,7 +16285,7 @@ F:	drivers/usb/storage/
 USB MIDI DRIVER
 M:	Clemens Ladisch <clemens@ladisch.de>
 L:	alsa-devel@alsa-project.org (moderated for non-subscribers)
-T:	git git://git.alsa-project.org/alsa-kernel.git
+T:	git git://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound.git
 S:	Maintained
 F:	sound/usb/midi.*
 
diff --git a/Makefile b/Makefile
index bcf2e7628fe5..a61a95b6b38f 100644
--- a/Makefile
+++ b/Makefile
@@ -96,56 +96,65 @@ endif
 
 export quiet Q KBUILD_VERBOSE
 
-# kbuild supports saving output files in a separate directory.
-# To locate output files in a separate directory two syntaxes are supported.
-# In both cases the working directory must be the root of the kernel src.
+# Kbuild will save output files in the current working directory.
+# This does not need to match to the root of the kernel source tree.
+#
+# For example, you can do this:
+#
+#  cd /dir/to/store/output/files; make -f /dir/to/kernel/source/Makefile
+#
+# If you want to save output files in a different location, there are
+# two syntaxes to specify it.
+#
 # 1) O=
 # Use "make O=dir/to/store/output/files/"
 #
 # 2) Set KBUILD_OUTPUT
-# Set the environment variable KBUILD_OUTPUT to point to the directory
-# where the output files shall be placed.
-# export KBUILD_OUTPUT=dir/to/store/output/files/
-# make
+# Set the environment variable KBUILD_OUTPUT to point to the output directory.
+# export KBUILD_OUTPUT=dir/to/store/output/files/; make
 #
 # The O= assignment takes precedence over the KBUILD_OUTPUT environment
 # variable.
 
-# KBUILD_SRC is not intended to be used by the regular user (for now),
-# it is set on invocation of make with KBUILD_OUTPUT or O= specified.
-
-# OK, Make called in directory where kernel src resides
-# Do we want to locate output files in a separate directory?
+# Do we want to change the working directory?
 ifeq ("$(origin O)", "command line")
   KBUILD_OUTPUT := $(O)
 endif
 
-ifneq ($(words $(subst :, ,$(CURDIR))), 1)
-  $(error main directory cannot contain spaces nor colons)
+ifneq ($(KBUILD_OUTPUT),)
+# Make's built-in functions such as $(abspath ...), $(realpath ...) cannot
+# expand a shell special character '~'. We use a somewhat tedious way here.
+abs_objtree := $(shell mkdir -p $(KBUILD_OUTPUT) && cd $(KBUILD_OUTPUT) && pwd)
+$(if $(abs_objtree),, \
+     $(error failed to create output directory "$(KBUILD_OUTPUT)"))
+
+# $(realpath ...) resolves symlinks
+abs_objtree := $(realpath $(abs_objtree))
+else
+abs_objtree := $(CURDIR)
+endif # ifneq ($(KBUILD_OUTPUT),)
+
+ifeq ($(abs_objtree),$(CURDIR))
+# Suppress "Entering directory ..." unless we are changing the work directory.
+MAKEFLAGS += --no-print-directory
+else
+need-sub-make := 1
 endif
 
-ifneq ($(KBUILD_OUTPUT),)
-# check that the output directory actually exists
-saved-output := $(KBUILD_OUTPUT)
-KBUILD_OUTPUT := $(shell mkdir -p $(KBUILD_OUTPUT) && cd $(KBUILD_OUTPUT) \
-								&& pwd)
-$(if $(KBUILD_OUTPUT),, \
-     $(error failed to create output directory "$(saved-output)"))
+abs_srctree := $(realpath $(dir $(lastword $(MAKEFILE_LIST))))
 
+ifneq ($(words $(subst :, ,$(abs_srctree))), 1)
+$(error source directory cannot contain spaces or colons)
+endif
+
+ifneq ($(abs_srctree),$(abs_objtree))
 # Look for make include files relative to root of kernel src
 #
 # This does not become effective immediately because MAKEFLAGS is re-parsed
-# once after the Makefile is read.  It is OK since we are going to invoke
-# 'sub-make' below.
-MAKEFLAGS += --include-dir=$(CURDIR)
-
+# once after the Makefile is read. We need to invoke sub-make.
+MAKEFLAGS += --include-dir=$(abs_srctree)
 need-sub-make := 1
-else
-
-# Do not print "Entering directory ..." at all for in-tree build.
-MAKEFLAGS += --no-print-directory
-
-endif # ifneq ($(KBUILD_OUTPUT),)
+endif
 
 ifneq ($(filter 3.%,$(MAKE_VERSION)),)
 # 'MAKEFLAGS += -rR' does not immediately become effective for GNU Make 3.x
@@ -155,20 +164,19 @@ need-sub-make := 1
 $(lastword $(MAKEFILE_LIST)): ;
 endif
 
+export abs_srctree abs_objtree
 export sub_make_done := 1
 
 ifeq ($(need-sub-make),1)
 
 PHONY += $(MAKECMDGOALS) sub-make
 
-$(filter-out _all sub-make $(CURDIR)/Makefile, $(MAKECMDGOALS)) _all: sub-make
+$(filter-out _all sub-make $(lastword $(MAKEFILE_LIST)), $(MAKECMDGOALS)) _all: sub-make
 	@:
 
 # Invoke a second make in the output directory, passing relevant variables
 sub-make:
-	$(Q)$(MAKE) \
-	$(if $(KBUILD_OUTPUT),-C $(KBUILD_OUTPUT) KBUILD_SRC=$(CURDIR)) \
-	-f $(CURDIR)/Makefile $(filter-out _all sub-make,$(MAKECMDGOALS))
+	$(Q)$(MAKE) -C $(abs_objtree) -f $(abs_srctree)/Makefile $(MAKECMDGOALS)
 
 endif # need-sub-make
 endif # sub_make_done
@@ -213,16 +221,21 @@ ifeq ("$(origin M)", "command line")
   KBUILD_EXTMOD := $(M)
 endif
 
-ifeq ($(KBUILD_SRC),)
+ifeq ($(abs_srctree),$(abs_objtree))
         # building in the source tree
         srctree := .
 else
-        ifeq ($(KBUILD_SRC)/,$(dir $(CURDIR)))
+        ifeq ($(abs_srctree)/,$(dir $(abs_objtree)))
                 # building in a subdirectory of the source tree
                 srctree := ..
         else
-                srctree := $(KBUILD_SRC)
+                srctree := $(abs_srctree)
         endif
+
+	# TODO:
+	# KBUILD_SRC is only used to distinguish in-tree/out-of-tree build.
+	# Replace it with $(srctree) or something.
+	KBUILD_SRC := $(abs_srctree)
 endif
 
 export KBUILD_CHECKSRC KBUILD_EXTMOD KBUILD_SRC
@@ -436,7 +449,7 @@ USERINCLUDE    := \
 LINUXINCLUDE    := \
 		-I$(srctree)/arch/$(SRCARCH)/include \
 		-I$(objtree)/arch/$(SRCARCH)/include/generated \
-		$(if $(KBUILD_SRC), -I$(srctree)/include) \
+		$(if $(filter .,$(srctree)),,-I$(srctree)/include) \
 		-I$(objtree)/include \
 		$(USERINCLUDE)
 
@@ -497,7 +510,7 @@ PHONY += outputmakefile
 # At the same time when output Makefile generated, generate .gitignore to
 # ignore whole output directory
 outputmakefile:
-ifneq ($(KBUILD_SRC),)
+ifneq ($(srctree),.)
 	$(Q)ln -fsn $(srctree) source
 	$(Q)$(CONFIG_SHELL) $(srctree)/scripts/mkmakefile $(srctree)
 	$(Q)test -e .gitignore || \
@@ -520,15 +533,6 @@ KBUILD_AFLAGS	+= $(CLANG_FLAGS)
 export CLANG_FLAGS
 endif
 
-RETPOLINE_CFLAGS_GCC := -mindirect-branch=thunk-extern -mindirect-branch-register
-RETPOLINE_VDSO_CFLAGS_GCC := -mindirect-branch=thunk-inline -mindirect-branch-register
-RETPOLINE_CFLAGS_CLANG := -mretpoline-external-thunk
-RETPOLINE_VDSO_CFLAGS_CLANG := -mretpoline
-RETPOLINE_CFLAGS := $(call cc-option,$(RETPOLINE_CFLAGS_GCC),$(call cc-option,$(RETPOLINE_CFLAGS_CLANG)))
-RETPOLINE_VDSO_CFLAGS := $(call cc-option,$(RETPOLINE_VDSO_CFLAGS_GCC),$(call cc-option,$(RETPOLINE_VDSO_CFLAGS_CLANG)))
-export RETPOLINE_CFLAGS
-export RETPOLINE_VDSO_CFLAGS
-
 # The expansion should be delayed until arch/$(SRCARCH)/Makefile is included.
 # Some architectures define CROSS_COMPILE in arch/$(SRCARCH)/Makefile.
 # CC_VERSION_TEXT is referenced from Kconfig (so it needs export),
@@ -595,20 +599,21 @@ endif
 
 export KBUILD_MODULES KBUILD_BUILTIN
 
+ifeq ($(dot-config),1)
+include include/config/auto.conf
+endif
+
 ifeq ($(KBUILD_EXTMOD),)
 # Objects we will link into vmlinux / subdirs we need to visit
 init-y		:= init/
 drivers-y	:= drivers/ sound/
+drivers-$(CONFIG_SAMPLES) += samples/
 net-y		:= net/
 libs-y		:= lib/
 core-y		:= usr/
 virt-y		:= virt/
 endif # KBUILD_EXTMOD
 
-ifeq ($(dot-config),1)
-include include/config/auto.conf
-endif
-
 # The all: target is the default when no target is given on the
 # command line.
 # This allow a user to issue only 'make' to build a kernel including modules
@@ -625,6 +630,15 @@ ifdef CONFIG_FUNCTION_TRACER
   CC_FLAGS_FTRACE := -pg
 endif
 
+RETPOLINE_CFLAGS_GCC := -mindirect-branch=thunk-extern -mindirect-branch-register
+RETPOLINE_VDSO_CFLAGS_GCC := -mindirect-branch=thunk-inline -mindirect-branch-register
+RETPOLINE_CFLAGS_CLANG := -mretpoline-external-thunk
+RETPOLINE_VDSO_CFLAGS_CLANG := -mretpoline
+RETPOLINE_CFLAGS := $(call cc-option,$(RETPOLINE_CFLAGS_GCC),$(call cc-option,$(RETPOLINE_CFLAGS_CLANG)))
+RETPOLINE_VDSO_CFLAGS := $(call cc-option,$(RETPOLINE_VDSO_CFLAGS_GCC),$(call cc-option,$(RETPOLINE_VDSO_CFLAGS_CLANG)))
+export RETPOLINE_CFLAGS
+export RETPOLINE_VDSO_CFLAGS
+
 # The arch Makefile can set ARCH_{CPP,A,C}FLAGS to override the default
 # values of the respective KBUILD_* variables
 ARCH_CPPFLAGS :=
@@ -985,8 +999,9 @@ vmlinux-dirs	:= $(patsubst %/,%,$(filter %/, $(init-y) $(init-m) \
 		     $(core-y) $(core-m) $(drivers-y) $(drivers-m) \
 		     $(net-y) $(net-m) $(libs-y) $(libs-m) $(virt-y)))
 
-vmlinux-alldirs	:= $(sort $(vmlinux-dirs) $(patsubst %/,%,$(filter %/, \
-		     $(init-) $(core-) $(drivers-) $(net-) $(libs-) $(virt-))))
+vmlinux-alldirs	:= $(sort $(vmlinux-dirs) Documentation \
+		     $(patsubst %/,%,$(filter %/, $(init-) $(core-) \
+			$(drivers-) $(net-) $(libs-) $(virt-))))
 
 init-y		:= $(patsubst %/, %/built-in.a, $(init-y))
 core-y		:= $(patsubst %/, %/built-in.a, $(core-y))
@@ -1003,7 +1018,7 @@ export KBUILD_VMLINUX_LIBS := $(libs-y1)
 export KBUILD_LDS          := arch/$(SRCARCH)/kernel/vmlinux.lds
 export LDFLAGS_vmlinux
 # used by scripts/package/Makefile
-export KBUILD_ALLDIRS := $(sort $(filter-out arch/%,$(vmlinux-alldirs)) arch Documentation include samples scripts tools)
+export KBUILD_ALLDIRS := $(sort $(filter-out arch/%,$(vmlinux-alldirs)) arch include scripts tools)
 
 vmlinux-deps := $(KBUILD_LDS) $(KBUILD_VMLINUX_OBJS) $(KBUILD_VMLINUX_LIBS)
 
@@ -1040,11 +1055,8 @@ vmlinux: scripts/link-vmlinux.sh autoksyms_recursive $(vmlinux-deps) FORCE
 
 targets := vmlinux
 
-# Build samples along the rest of the kernel. This needs headers_install.
-ifdef CONFIG_SAMPLES
-vmlinux-dirs += samples
+# Some samples need headers_install.
 samples: headers_install
-endif
 
 # The actual objects are generated when descending,
 # make sure no implicit rule kicks in
@@ -1064,7 +1076,7 @@ filechk_kernel.release = \
 	echo "$(KERNELVERSION)$$($(CONFIG_SHELL) $(srctree)/scripts/setlocalversion $(srctree))"
 
 # Store (new) KERNELRELEASE string in include/config/kernel.release
-include/config/kernel.release: $(srctree)/Makefile FORCE
+include/config/kernel.release: FORCE
 	$(call filechk,kernel.release)
 
 # Additional helpers built in scripts/
@@ -1086,9 +1098,11 @@ PHONY += prepare archprepare prepare1 prepare3
 # and if so do:
 # 1) Check that make has not been executed in the kernel src $(srctree)
 prepare3: include/config/kernel.release
-ifneq ($(KBUILD_SRC),)
+ifneq ($(srctree),.)
 	@$(kecho) '  Using $(srctree) as source for kernel'
-	$(Q)if [ -f $(srctree)/.config -o -d $(srctree)/include/config ]; then \
+	$(Q)if [ -f $(srctree)/.config -o \
+		 -d $(srctree)/include/config -o \
+		 -d $(srctree)/arch/$(SRCARCH)/include/generated ]; then \
 		echo >&2 "  $(srctree) is not clean, please run 'make mrproper'"; \
 		echo >&2 "  in the '$(srctree)' directory.";\
 		/bin/false; \
@@ -1304,6 +1318,7 @@ _modinst_:
 	fi
 	@cp -f $(objtree)/modules.order $(MODLIB)/
 	@cp -f $(objtree)/modules.builtin $(MODLIB)/
+	@cp -f $(objtree)/modules.builtin.modinfo $(MODLIB)/
 	$(Q)$(MAKE) -f $(srctree)/scripts/Makefile.modinst
 
 # This depmod is only for convenience to give the initial
@@ -1344,10 +1359,11 @@ endif # CONFIG_MODULES
 
 # Directories & files removed with 'make clean'
 CLEAN_DIRS  += $(MODVERDIR) include/ksym
+CLEAN_FILES += modules.builtin.modinfo
 
 # Directories & files removed with 'make mrproper'
 MRPROPER_DIRS  += include/config usr/include include/generated          \
-		  arch/*/include/generated .tmp_objdiff
+		  arch/$(SRCARCH)/include/generated .tmp_objdiff
 MRPROPER_FILES += .config .config.old .version \
 		  Module.symvers tags TAGS cscope* GPATH GTAGS GRTAGS GSYMS \
 		  signing_key.pem signing_key.priv signing_key.x509	\
@@ -1358,7 +1374,7 @@ MRPROPER_FILES += .config .config.old .version \
 #
 clean: rm-dirs  := $(CLEAN_DIRS)
 clean: rm-files := $(CLEAN_FILES)
-clean-dirs      := $(addprefix _clean_, . $(vmlinux-alldirs) Documentation samples)
+clean-dirs      := $(addprefix _clean_, . $(vmlinux-alldirs))
 
 PHONY += $(clean-dirs) clean archclean vmlinuxclean
 $(clean-dirs):
diff --git a/arch/arm/boot/dts/rk3288.dtsi b/arch/arm/boot/dts/rk3288.dtsi
index a024d1e7e74c..8ce3dd2264b1 100644
--- a/arch/arm/boot/dts/rk3288.dtsi
+++ b/arch/arm/boot/dts/rk3288.dtsi
@@ -616,6 +616,7 @@
 		dr_mode = "host";
 		phys = <&usbphy2>;
 		phy-names = "usb2-phy";
+		snps,reset-phy-on-wake;
 		status = "disabled";
 	};
 
@@ -904,6 +905,8 @@
 				clocks = <&cru SCLK_OTGPHY0>;
 				clock-names = "phyclk";
 				#clock-cells = <0>;
+				resets = <&cru SRST_USBOTG_PHY>;
+				reset-names = "phy-reset";
 			};
 
 			usbphy1: usb-phy@334 {
@@ -912,6 +915,8 @@
 				clocks = <&cru SCLK_OTGPHY1>;
 				clock-names = "phyclk";
 				#clock-cells = <0>;
+				resets = <&cru SRST_USBHOST0_PHY>;
+				reset-names = "phy-reset";
 			};
 
 			usbphy2: usb-phy@348 {
@@ -920,6 +925,8 @@
 				clocks = <&cru SCLK_OTGPHY2>;
 				clock-names = "phyclk";
 				#clock-cells = <0>;
+				resets = <&cru SRST_USBHOST1_PHY>;
+				reset-names = "phy-reset";
 			};
 		};
 	};
diff --git a/arch/arm/kernel/dma-isa.c b/arch/arm/kernel/dma-isa.c
index 84363fe7bad2..10c45cc6b957 100644
--- a/arch/arm/kernel/dma-isa.c
+++ b/arch/arm/kernel/dma-isa.c
@@ -55,6 +55,12 @@ static int isa_get_dma_residue(unsigned int chan, dma_t *dma)
 	return chan < 4 ? count : (count << 1);
 }
 
+static struct device isa_dma_dev = {
+	.init_name		= "fallback device",
+	.coherent_dma_mask	= ~(dma_addr_t)0,
+	.dma_mask		= &isa_dma_dev.coherent_dma_mask,
+};
+
 static void isa_enable_dma(unsigned int chan, dma_t *dma)
 {
 	if (dma->invalid) {
@@ -89,7 +95,7 @@ static void isa_enable_dma(unsigned int chan, dma_t *dma)
 			dma->sg = &dma->buf;
 			dma->sgcount = 1;
 			dma->buf.length = dma->count;
-			dma->buf.dma_address = dma_map_single(NULL,
+			dma->buf.dma_address = dma_map_single(&isa_dma_dev,
 				dma->addr, dma->count,
 				direction);
 		}
diff --git a/arch/arm/mach-omap2/clock.c b/arch/arm/mach-omap2/clock.c
index 42881f21cede..3e0f09cc0028 100644
--- a/arch/arm/mach-omap2/clock.c
+++ b/arch/arm/mach-omap2/clock.c
@@ -119,6 +119,9 @@ void __init ti_clk_init_features(void)
 	if (cpu_is_omap343x())
 		features.flags |= TI_CLK_DPLL_HAS_FREQSEL;
 
+	if (omap_type() == OMAP2_DEVICE_TYPE_GP)
+		features.flags |= TI_CLK_DEVICE_TYPE_GP;
+
 	/* Idlest value for interface clocks.
 	 * 24xx uses 0 to indicate not ready, and 1 to indicate ready.
 	 * 34xx reverses this, just to keep us on our toes
diff --git a/arch/arm/mach-omap2/omap_hwmod.c b/arch/arm/mach-omap2/omap_hwmod.c
index 3a04c73ac03c..baadddf9aad4 100644
--- a/arch/arm/mach-omap2/omap_hwmod.c
+++ b/arch/arm/mach-omap2/omap_hwmod.c
@@ -648,10 +648,10 @@ static struct clockdomain *_get_clkdm(struct omap_hwmod *oh)
 	if (oh->clkdm) {
 		return oh->clkdm;
 	} else if (oh->_clk) {
-		if (__clk_get_flags(oh->_clk) & CLK_IS_BASIC)
+		if (!omap2_clk_is_hw_omap(__clk_get_hw(oh->_clk)))
 			return NULL;
 		clk = to_clk_hw_omap(__clk_get_hw(oh->_clk));
-		return  clk->clkdm;
+		return clk->clkdm;
 	}
 	return NULL;
 }
diff --git a/arch/arm/mach-rpc/dma.c b/arch/arm/mach-rpc/dma.c
index fb48f3141fb4..f2703ca17954 100644
--- a/arch/arm/mach-rpc/dma.c
+++ b/arch/arm/mach-rpc/dma.c
@@ -151,6 +151,12 @@ static void iomd_free_dma(unsigned int chan, dma_t *dma)
 	free_irq(idma->irq, idma);
 }
 
+static struct device isa_dma_dev = {
+	.init_name		= "fallback device",
+	.coherent_dma_mask	= ~(dma_addr_t)0,
+	.dma_mask		= &isa_dma_dev.coherent_dma_mask,
+};
+
 static void iomd_enable_dma(unsigned int chan, dma_t *dma)
 {
 	struct iomd_dma *idma = container_of(dma, struct iomd_dma, dma);
@@ -168,7 +174,7 @@ static void iomd_enable_dma(unsigned int chan, dma_t *dma)
 			idma->dma.sg = &idma->dma.buf;
 			idma->dma.sgcount = 1;
 			idma->dma.buf.length = idma->dma.count;
-			idma->dma.buf.dma_address = dma_map_single(NULL,
+			idma->dma.buf.dma_address = dma_map_single(&isa_dma_dev,
 				idma->dma.addr, idma->dma.count,
 				idma->dma.dma_mode == DMA_MODE_READ ?
 				DMA_FROM_DEVICE : DMA_TO_DEVICE);
diff --git a/arch/arm/plat-omap/dma.c b/arch/arm/plat-omap/dma.c
index d4012d6c0dcb..5ca4c5fd627a 100644
--- a/arch/arm/plat-omap/dma.c
+++ b/arch/arm/plat-omap/dma.c
@@ -1449,7 +1449,6 @@ static void __exit omap_system_dma_exit(void)
 
 MODULE_DESCRIPTION("OMAP SYSTEM DMA DRIVER");
 MODULE_LICENSE("GPL");
-MODULE_ALIAS("platform:" DRIVER_NAME);
 MODULE_AUTHOR("Texas Instruments Inc");
 
 /*
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index df350f4e1e7a..3f957443f286 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -13,6 +13,7 @@ config ARM64
 	select ARCH_HAS_DEVMEM_IS_ALLOWED
 	select ARCH_HAS_DMA_COHERENT_TO_PFN
 	select ARCH_HAS_DMA_MMAP_PGPROT
+	select ARCH_HAS_DMA_PREP_COHERENT
 	select ARCH_HAS_ACPI_TABLE_UPGRADE if ACPI
 	select ARCH_HAS_ELF_RANDOMIZE
 	select ARCH_HAS_FAST_MULTIPLIER
diff --git a/arch/csky/Kconfig b/arch/csky/Kconfig
index 6555d1781132..ce0799077f3b 100644
--- a/arch/csky/Kconfig
+++ b/arch/csky/Kconfig
@@ -1,6 +1,7 @@
 config CSKY
 	def_bool y
 	select ARCH_32BIT_OFF_T
+	select ARCH_HAS_DMA_PREP_COHERENT
 	select ARCH_HAS_SYNC_DMA_FOR_CPU
 	select ARCH_HAS_SYNC_DMA_FOR_DEVICE
 	select ARCH_USE_BUILTIN_BSWAP
@@ -29,15 +30,20 @@ config CSKY
 	select GENERIC_SCHED_CLOCK
 	select GENERIC_SMP_IDLE_THREAD
 	select HAVE_ARCH_TRACEHOOK
+	select HAVE_ARCH_AUDITSYSCALL
+	select HAVE_DYNAMIC_FTRACE
 	select HAVE_FUNCTION_TRACER
 	select HAVE_FUNCTION_GRAPH_TRACER
+	select HAVE_FTRACE_MCOUNT_RECORD
 	select HAVE_KERNEL_GZIP
 	select HAVE_KERNEL_LZO
 	select HAVE_KERNEL_LZMA
 	select HAVE_PERF_EVENTS
-	select HAVE_C_RECORDMCOUNT
+	select HAVE_PERF_REGS
+	select HAVE_PERF_USER_STACK_DUMP
 	select HAVE_DMA_API_DEBUG
 	select HAVE_DMA_CONTIGUOUS
+	select HAVE_SYSCALL_TRACEPOINTS
 	select MAY_HAVE_SPARSE_IRQ
 	select MODULES_USE_ELF_RELA if MODULES
 	select OF
diff --git a/arch/csky/Makefile b/arch/csky/Makefile
index 3607a6e8f66c..6b87f6c22ad6 100644
--- a/arch/csky/Makefile
+++ b/arch/csky/Makefile
@@ -36,7 +36,7 @@ endif
 
 ifneq ($(CSKYABI),)
 MCPU_STR = $(CPUTYPE)$(FPUEXT)$(VDSPEXT)$(TEEEXT)
-KBUILD_CFLAGS += -mcpu=$(MCPU_STR)
+KBUILD_CFLAGS += -mcpu=$(CPUTYPE) -Wa,-mcpu=$(MCPU_STR)
 KBUILD_CFLAGS += -DCSKYCPU_DEF_NAME=\"$(MCPU_STR)\"
 KBUILD_CFLAGS += -msoft-float -mdiv
 KBUILD_CFLAGS += -fno-tree-vectorize
diff --git a/arch/csky/abiv1/inc/abi/ckmmu.h b/arch/csky/abiv1/inc/abi/ckmmu.h
index 3a002017bebe..81f37715c0d2 100644
--- a/arch/csky/abiv1/inc/abi/ckmmu.h
+++ b/arch/csky/abiv1/inc/abi/ckmmu.h
@@ -40,6 +40,26 @@ static inline void write_mmu_entryhi(int value)
 	cpwcr("cpcr4", value);
 }
 
+static inline unsigned long read_mmu_msa0(void)
+{
+	return cprcr("cpcr30");
+}
+
+static inline void write_mmu_msa0(unsigned long value)
+{
+	cpwcr("cpcr30", value);
+}
+
+static inline unsigned long read_mmu_msa1(void)
+{
+	return cprcr("cpcr31");
+}
+
+static inline void write_mmu_msa1(unsigned long value)
+{
+	cpwcr("cpcr31", value);
+}
+
 /*
  * TLB operations.
  */
@@ -65,11 +85,11 @@ static inline void tlb_invalid_indexed(void)
 
 static inline void setup_pgd(unsigned long pgd, bool kernel)
 {
-	cpwcr("cpcr29", pgd);
+	cpwcr("cpcr29", pgd | BIT(0));
 }
 
 static inline unsigned long get_pgd(void)
 {
-	return cprcr("cpcr29");
+	return cprcr("cpcr29") & ~BIT(0);
 }
 #endif /* __ASM_CSKY_CKMMUV1_H */
diff --git a/arch/csky/abiv1/inc/abi/entry.h b/arch/csky/abiv1/inc/abi/entry.h
index 3f3faab3d747..7ab78bd0f3b1 100644
--- a/arch/csky/abiv1/inc/abi/entry.h
+++ b/arch/csky/abiv1/inc/abi/entry.h
@@ -16,9 +16,6 @@
 #define LSAVE_A4	40
 #define LSAVE_A5	44
 
-#define EPC_INCREASE	2
-#define EPC_KEEP	0
-
 .macro USPTOKSP
 	mtcr	sp, ss1
 	mfcr	sp, ss0
@@ -29,10 +26,6 @@
 	mfcr	sp, ss1
 .endm
 
-.macro INCTRAP	rx
-	addi	\rx, EPC_INCREASE
-.endm
-
 .macro	SAVE_ALL epc_inc
 	mtcr    r13, ss2
 	mfcr    r13, epsr
@@ -150,11 +143,35 @@
 	cpwcr   \rx, cpcr8
 .endm
 
-.macro SETUP_MMU rx
-	lrw	\rx, PHYS_OFFSET | 0xe
-	cpwcr	\rx, cpcr30
-	lrw	\rx, (PHYS_OFFSET + 0x20000000) | 0xe
-	cpwcr	\rx, cpcr31
+.macro SETUP_MMU
+	/* Init psr and enable ee */
+	lrw	r6, DEFAULT_PSR_VALUE
+	mtcr    r6, psr
+	psrset  ee
+
+	/* Select MMU as co-processor */
+	cpseti	cp15
+
+	/*
+	 * cpcr30 format:
+	 * 31 - 29 | 28 - 4 | 3 | 2 | 1 | 0
+	 *   BA     Reserved  C   D   V
+	 */
+	cprcr	r6, cpcr30
+	lsri	r6, 28
+	lsli	r6, 28
+	addi	r6, 0xe
+	cpwcr	r6, cpcr30
+
+	lsri	r6, 28
+	addi	r6, 2
+	lsli	r6, 28
+	addi	r6, 0xe
+	cpwcr	r6, cpcr31
 .endm
 
+.macro ANDI_R3 rx, imm
+	lsri	\rx, 3
+	andi	\rx, (\imm >> 3)
+.endm
 #endif /* __ASM_CSKY_ENTRY_H */
diff --git a/arch/csky/abiv1/inc/abi/regdef.h b/arch/csky/abiv1/inc/abi/regdef.h
index 876689291b71..104707fbdcc1 100644
--- a/arch/csky/abiv1/inc/abi/regdef.h
+++ b/arch/csky/abiv1/inc/abi/regdef.h
@@ -5,9 +5,8 @@
 #define __ASM_CSKY_REGDEF_H
 
 #define syscallid	r1
-#define r11_sig		r11
-
 #define regs_syscallid(regs) regs->regs[9]
+#define regs_fp(regs) regs->regs[2]
 
 /*
  * PSR format:
@@ -23,4 +22,6 @@
 
 #define SYSTRACE_SAVENUM	2
 
+#define TRAP0_SIZE		2
+
 #endif /* __ASM_CSKY_REGDEF_H */
diff --git a/arch/csky/abiv2/cacheflush.c b/arch/csky/abiv2/cacheflush.c
index d22c95ffc74d..5bb887b275e1 100644
--- a/arch/csky/abiv2/cacheflush.c
+++ b/arch/csky/abiv2/cacheflush.c
@@ -34,10 +34,6 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address,
 {
 	unsigned long addr, pfn;
 	struct page *page;
-	void *va;
-
-	if (!(vma->vm_flags & VM_EXEC))
-		return;
 
 	pfn = pte_pfn(*pte);
 	if (unlikely(!pfn_valid(pfn)))
@@ -47,14 +43,9 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address,
 	if (page == ZERO_PAGE(0))
 		return;
 
-	va = page_address(page);
-	addr = (unsigned long) va;
-
-	if (va == NULL && PageHighMem(page))
-		addr = (unsigned long) kmap_atomic(page);
+	addr = (unsigned long) kmap_atomic(page);
 
 	cache_wbinv_range(addr, addr + PAGE_SIZE);
 
-	if (va == NULL && PageHighMem(page))
-		kunmap_atomic((void *) addr);
+	kunmap_atomic((void *) addr);
 }
diff --git a/arch/csky/abiv2/inc/abi/ckmmu.h b/arch/csky/abiv2/inc/abi/ckmmu.h
index 97230ad9427c..e4480e6bc3b3 100644
--- a/arch/csky/abiv2/inc/abi/ckmmu.h
+++ b/arch/csky/abiv2/inc/abi/ckmmu.h
@@ -42,6 +42,26 @@ static inline void write_mmu_entryhi(int value)
 	mtcr("cr<4, 15>", value);
 }
 
+static inline unsigned long read_mmu_msa0(void)
+{
+	return mfcr("cr<30, 15>");
+}
+
+static inline void write_mmu_msa0(unsigned long value)
+{
+	mtcr("cr<30, 15>", value);
+}
+
+static inline unsigned long read_mmu_msa1(void)
+{
+	return mfcr("cr<31, 15>");
+}
+
+static inline void write_mmu_msa1(unsigned long value)
+{
+	mtcr("cr<31, 15>", value);
+}
+
 /*
  * TLB operations.
  */
@@ -70,18 +90,16 @@ static inline void tlb_invalid_indexed(void)
 	mtcr("cr<8, 15>", 0x02000000);
 }
 
-/* setup hardrefil pgd */
-static inline unsigned long get_pgd(void)
-{
-	return mfcr("cr<29, 15>");
-}
-
 static inline void setup_pgd(unsigned long pgd, bool kernel)
 {
 	if (kernel)
-		mtcr("cr<28, 15>", pgd);
+		mtcr("cr<28, 15>", pgd | BIT(0));
 	else
-		mtcr("cr<29, 15>", pgd);
+		mtcr("cr<29, 15>", pgd | BIT(0));
 }
 
+static inline unsigned long get_pgd(void)
+{
+	return mfcr("cr<29, 15>") & ~BIT(0);
+}
 #endif /* __ASM_CSKY_CKMMUV2_H */
diff --git a/arch/csky/abiv2/inc/abi/entry.h b/arch/csky/abiv2/inc/abi/entry.h
index edc5cc04c4de..9897a16b45e5 100644
--- a/arch/csky/abiv2/inc/abi/entry.h
+++ b/arch/csky/abiv2/inc/abi/entry.h
@@ -14,18 +14,11 @@
 #define LSAVE_A2	32
 #define LSAVE_A3	36
 
-#define EPC_INCREASE	4
-#define EPC_KEEP	0
-
 #define KSPTOUSP
 #define USPTOKSP
 
 #define usp cr<14, 1>
 
-.macro INCTRAP	rx
-	addi	\rx, EPC_INCREASE
-.endm
-
 .macro SAVE_ALL epc_inc
 	subi    sp, 152
 	stw	tls, (sp, 0)
@@ -169,10 +162,80 @@
 	mtcr	\rx, cr<8, 15>
 .endm
 
-.macro SETUP_MMU rx
-	lrw	\rx, PHYS_OFFSET | 0xe
-	mtcr	\rx, cr<30, 15>
-	lrw	\rx, (PHYS_OFFSET + 0x20000000) | 0xe
-	mtcr	\rx, cr<31, 15>
+.macro SETUP_MMU
+	/* Init psr and enable ee */
+	lrw	r6, DEFAULT_PSR_VALUE
+	mtcr    r6, psr
+	psrset  ee
+
+	/* Invalid I/Dcache BTB BHT */
+	movi	r6, 7
+	lsli	r6, 16
+	addi	r6, (1<<4) | 3
+	mtcr	r6, cr17
+
+	/* Invalid all TLB */
+	bgeni   r6, 26
+	mtcr	r6, cr<8, 15> /* Set MCIR */
+
+	/* Check MMU on/off */
+	mfcr	r6, cr18
+	btsti	r6, 0
+	bt	1f
+
+	/* MMU off: setup mapping tlb entry */
+	movi	r6, 0
+	mtcr	r6, cr<6, 15> /* Set MPR with 4K page size */
+
+	grs	r6, 1f /* Get current pa by PC */
+	bmaski  r7, (PAGE_SHIFT + 1) /* r7 = 0x1fff */
+	andn    r6, r7
+	mtcr	r6, cr<4, 15> /* Set MEH */
+
+	mov	r8, r6
+	movi    r7, 0x00000006
+	or      r8, r7
+	mtcr	r8, cr<2, 15> /* Set MEL0 */
+	movi    r7, 0x00001006
+	or      r8, r7
+	mtcr	r8, cr<3, 15> /* Set MEL1 */
+
+	bgeni   r8, 28
+	mtcr	r8, cr<8, 15> /* Set MCIR to write TLB */
+
+	br	2f
+1:
+	/*
+	 * MMU on: use origin MSA value from bootloader
+	 *
+	 * cr<30/31, 15> MSA register format:
+	 * 31 - 29 | 28 - 9 | 8 | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0
+	 *   BA     Reserved  SH  WA  B   SO SEC  C   D   V
+	 */
+	mfcr	r6, cr<30, 15> /* Get MSA0 */
+2:
+	lsri	r6, 28
+	lsli	r6, 28
+	addi	r6, 0x1ce
+	mtcr	r6, cr<30, 15> /* Set MSA0 */
+
+	lsri	r6, 28
+	addi	r6, 2
+	lsli	r6, 28
+	addi	r6, 0x1ce
+	mtcr	r6, cr<31, 15> /* Set MSA1 */
+
+	/* enable MMU */
+	mfcr    r6, cr18
+	bseti	r6, 0
+	mtcr    r6, cr18
+
+	jmpi	3f /* jump to va */
+3:
+.endm
+
+.macro ANDI_R3 rx, imm
+	lsri	\rx, 3
+	andi	\rx, (\imm >> 3)
 .endm
 #endif /* __ASM_CSKY_ENTRY_H */
diff --git a/arch/csky/abiv2/inc/abi/regdef.h b/arch/csky/abiv2/inc/abi/regdef.h
index c72abb781bdc..d7328bbc1ce7 100644
--- a/arch/csky/abiv2/inc/abi/regdef.h
+++ b/arch/csky/abiv2/inc/abi/regdef.h
@@ -5,9 +5,8 @@
 #define __ASM_CSKY_REGDEF_H
 
 #define syscallid	r7
-#define r11_sig		r11
-
 #define regs_syscallid(regs) regs->regs[3]
+#define regs_fp(regs) regs->regs[4]
 
 /*
  * PSR format:
@@ -23,4 +22,6 @@
 
 #define SYSTRACE_SAVENUM	5
 
+#define TRAP0_SIZE		4
+
 #endif /* __ASM_CSKY_REGDEF_H */
diff --git a/arch/csky/abiv2/mcount.S b/arch/csky/abiv2/mcount.S
index c633379956f5..326402e65f9e 100644
--- a/arch/csky/abiv2/mcount.S
+++ b/arch/csky/abiv2/mcount.S
@@ -61,10 +61,17 @@
 	addi	sp, 16
 .endm
 
+.macro nop32_stub
+	nop32
+	nop32
+	nop32
+.endm
+
 ENTRY(ftrace_stub)
 	jmp	lr
 END(ftrace_stub)
 
+#ifndef CONFIG_DYNAMIC_FTRACE
 ENTRY(_mcount)
 	mcount_enter
 
@@ -76,7 +83,7 @@ ENTRY(_mcount)
 	bf	skip_ftrace
 
 	mov	a0, lr
-	subi	a0, MCOUNT_INSN_SIZE
+	subi	a0, 4
 	ldw	a1, (sp, 24)
 
 	jsr	r26
@@ -101,13 +108,41 @@ skip_ftrace:
 	mcount_exit
 #endif
 END(_mcount)
+#else /* CONFIG_DYNAMIC_FTRACE */
+ENTRY(_mcount)
+	mov	t1, lr
+	ldw	lr, (sp, 0)
+	addi	sp, 4
+	jmp	t1
+ENDPROC(_mcount)
+
+ENTRY(ftrace_caller)
+	mcount_enter
+
+	ldw	a0, (sp, 16)
+	subi	a0, 4
+	ldw	a1, (sp, 24)
+
+	nop
+GLOBAL(ftrace_call)
+	nop32_stub
+
+#ifdef CONFIG_FUNCTION_GRAPH_TRACER
+	nop
+GLOBAL(ftrace_graph_call)
+	nop32_stub
+#endif
+
+	mcount_exit
+ENDPROC(ftrace_caller)
+#endif /* CONFIG_DYNAMIC_FTRACE */
 
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 ENTRY(ftrace_graph_caller)
 	mov	a0, sp
 	addi	a0, 24
 	ldw	a1, (sp, 16)
-	subi	a1, MCOUNT_INSN_SIZE
+	subi	a1, 4
 	mov	a2, r8
 	lrw	r26, prepare_ftrace_return
 	jsr	r26
diff --git a/arch/csky/abiv2/memmove.S b/arch/csky/abiv2/memmove.S
index b0c42ecf1889..5721e73ad3d8 100644
--- a/arch/csky/abiv2/memmove.S
+++ b/arch/csky/abiv2/memmove.S
@@ -35,11 +35,7 @@ ENTRY(memmove)
 .L_len_larger_16bytes:
 	subi	r1, 16
 	subi	r0, 16
-#if defined(__CSKY_VDSPV2__)
-	vldx.8	vr0, (r1), r19
-	PRE_BNEZAD (r18)
-	vstx.8	vr0, (r0), r19
-#elif defined(__CK860__)
+#if defined(__CK860__)
 	ldw	r3, (r1, 12)
 	stw	r3, (r0, 12)
 	ldw	r3, (r1, 8)
diff --git a/arch/csky/include/asm/Kbuild b/arch/csky/include/asm/Kbuild
index 95f4e550db8a..a9b63efef416 100644
--- a/arch/csky/include/asm/Kbuild
+++ b/arch/csky/include/asm/Kbuild
@@ -12,7 +12,6 @@ generic-y += dma-mapping.h
 generic-y += emergency-restart.h
 generic-y += exec.h
 generic-y += fb.h
-generic-y += ftrace.h
 generic-y += futex.h
 generic-y += gpio.h
 generic-y += hardirq.h
diff --git a/arch/csky/include/asm/ftrace.h b/arch/csky/include/asm/ftrace.h
index 7547c45312a8..ba35d93ecda2 100644
--- a/arch/csky/include/asm/ftrace.h
+++ b/arch/csky/include/asm/ftrace.h
@@ -4,10 +4,26 @@
 #ifndef __ASM_CSKY_FTRACE_H
 #define __ASM_CSKY_FTRACE_H
 
-#define MCOUNT_INSN_SIZE 4
+#define MCOUNT_INSN_SIZE	14
 
 #define HAVE_FUNCTION_GRAPH_FP_TEST
 
 #define HAVE_FUNCTION_GRAPH_RET_ADDR_PTR
 
+#define MCOUNT_ADDR	((unsigned long)_mcount)
+
+#ifndef __ASSEMBLY__
+
+extern void _mcount(unsigned long);
+
+extern void ftrace_graph_call(void);
+
+static inline unsigned long ftrace_call_adjust(unsigned long addr)
+{
+	return addr;
+}
+
+struct dyn_arch_ftrace {
+};
+#endif /* !__ASSEMBLY__ */
 #endif /* __ASM_CSKY_FTRACE_H */
diff --git a/arch/csky/include/asm/mmu_context.h b/arch/csky/include/asm/mmu_context.h
index b2905c0485a7..734db3a122e1 100644
--- a/arch/csky/include/asm/mmu_context.h
+++ b/arch/csky/include/asm/mmu_context.h
@@ -14,23 +14,10 @@
 #include <linux/sched.h>
 #include <abi/ckmmu.h>
 
-static inline void tlbmiss_handler_setup_pgd(unsigned long pgd, bool kernel)
-{
-	pgd -= PAGE_OFFSET;
-	pgd += PHYS_OFFSET;
-	pgd |= 1;
-	setup_pgd(pgd, kernel);
-}
-
 #define TLBMISS_HANDLER_SETUP_PGD(pgd) \
-	tlbmiss_handler_setup_pgd((unsigned long)pgd, 0)
+	setup_pgd(__pa(pgd), false)
 #define TLBMISS_HANDLER_SETUP_PGD_KERNEL(pgd) \
-	tlbmiss_handler_setup_pgd((unsigned long)pgd, 1)
-
-static inline unsigned long tlb_get_pgd(void)
-{
-	return ((get_pgd() - PHYS_OFFSET) & ~1) + PAGE_OFFSET;
-}
+	setup_pgd(__pa(pgd), true)
 
 #define cpu_context(cpu, mm)	((mm)->context.asid[cpu])
 #define cpu_asid(cpu, mm)	(cpu_context((cpu), (mm)) & ASID_MASK)
diff --git a/arch/csky/include/asm/page.h b/arch/csky/include/asm/page.h
index 73cf2bd66a13..9738eacefdc7 100644
--- a/arch/csky/include/asm/page.h
+++ b/arch/csky/include/asm/page.h
@@ -8,7 +8,7 @@
 #include <linux/const.h>
 
 /*
- * PAGE_SHIFT determines the page size
+ * PAGE_SHIFT determines the page size: 4KB
  */
 #define PAGE_SHIFT	12
 #define PAGE_SIZE	(_AC(1, UL) << PAGE_SHIFT)
@@ -17,12 +17,18 @@
 #define THREAD_MASK	(~(THREAD_SIZE - 1))
 #define THREAD_SHIFT	(PAGE_SHIFT + 1)
 
+
 /*
- * NOTE: virtual isn't really correct, actually it should be the offset into the
- * memory node, but we have no highmem, so that works for now.
- * TODO: implement (fast) pfn<->pgdat_idx conversion functions, this makes lots
- * of the shifts unnecessary.
+ * For C-SKY "User-space:Kernel-space" is "2GB:2GB" fixed by hardware and there
+ * are two segment registers (MSA0 + MSA1) to mapping 512MB + 512MB physical
+ * address region. We use them mapping kernel 1GB direct-map address area and
+ * for more than 1GB of memory we use highmem.
  */
+#define PAGE_OFFSET	0x80000000
+#define SSEG_SIZE	0x20000000
+#define LOWMEM_LIMIT	(SSEG_SIZE * 2)
+
+#define PHYS_OFFSET_OFFSET (CONFIG_RAM_BASE & (SSEG_SIZE - 1))
 
 #ifndef __ASSEMBLY__
 
@@ -50,9 +56,6 @@ struct page;
 
 struct vm_area_struct;
 
-/*
- * These are used to make use of C type-checking..
- */
 typedef struct { unsigned long pte_low; } pte_t;
 #define pte_val(x)	((x).pte_low)
 
@@ -69,18 +72,13 @@ typedef struct page *pgtable_t;
 #define __pgd(x)	((pgd_t) { (x) })
 #define __pgprot(x)	((pgprot_t) { (x) })
 
-#endif /* !__ASSEMBLY__ */
+extern unsigned long va_pa_offset;
 
-#define PHYS_OFFSET		(CONFIG_RAM_BASE & ~(LOWMEM_LIMIT - 1))
-#define PHYS_OFFSET_OFFSET	(CONFIG_RAM_BASE & (LOWMEM_LIMIT - 1))
-#define ARCH_PFN_OFFSET		PFN_DOWN(CONFIG_RAM_BASE)
+#define ARCH_PFN_OFFSET	PFN_DOWN(va_pa_offset + PHYS_OFFSET_OFFSET)
 
-#define	PAGE_OFFSET	0x80000000
-#define LOWMEM_LIMIT	0x40000000
+#define __pa(x)		 ((unsigned long)(x) - PAGE_OFFSET + va_pa_offset)
+#define __va(x) ((void *)((unsigned long)(x) + PAGE_OFFSET - va_pa_offset))
 
-#define __pa(x)		((unsigned long)(x) - PAGE_OFFSET + PHYS_OFFSET)
-#define __va(x)		((void *)((unsigned long)(x) + PAGE_OFFSET - \
-				  PHYS_OFFSET))
 #define __pa_symbol(x)	__pa(RELOC_HIDE((unsigned long)(x), 0))
 
 #define MAP_NR(x)	PFN_DOWN((unsigned long)(x) - PAGE_OFFSET - \
@@ -90,15 +88,10 @@ typedef struct page *pgtable_t;
 #define VM_DATA_DEFAULT_FLAGS	(VM_READ | VM_WRITE | VM_EXEC | \
 				VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC)
 
-/*
- * main RAM and kernel working space are coincident at 0x80000000, but to make
- * life more interesting, there's also an uncached virtual shadow at 0xb0000000
- * - these mappings are fixed in the MMU
- */
-
 #define pfn_to_kaddr(x)	__va(PFN_PHYS(x))
 
 #include <asm-generic/memory_model.h>
 #include <asm-generic/getorder.h>
 
+#endif /* !__ASSEMBLY__ */
 #endif /* __ASM_CSKY_PAGE_H */
diff --git a/arch/csky/include/asm/perf_event.h b/arch/csky/include/asm/perf_event.h
index ea8193122294..572093e11001 100644
--- a/arch/csky/include/asm/perf_event.h
+++ b/arch/csky/include/asm/perf_event.h
@@ -4,4 +4,12 @@
 #ifndef __ASM_CSKY_PERF_EVENT_H
 #define __ASM_CSKY_PERF_EVENT_H
 
+#include <abi/regdef.h>
+
+#define perf_arch_fetch_caller_regs(regs, __ip) { \
+	(regs)->pc = (__ip); \
+	regs_fp(regs) = (unsigned long) __builtin_frame_address(0); \
+	asm volatile("mov %0, sp\n":"=r"((regs)->usp)); \
+}
+
 #endif /* __ASM_PERF_EVENT_ELF_H */
diff --git a/arch/csky/include/asm/ptrace.h b/arch/csky/include/asm/ptrace.h
new file mode 100644
index 000000000000..d0aba7b32417
--- /dev/null
+++ b/arch/csky/include/asm/ptrace.h
@@ -0,0 +1,41 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd.
+
+#ifndef __ASM_CSKY_PTRACE_H
+#define __ASM_CSKY_PTRACE_H
+
+#include <uapi/asm/ptrace.h>
+#include <asm/traps.h>
+#include <linux/types.h>
+
+#ifndef __ASSEMBLY__
+
+#define PS_S	0x80000000 /* Supervisor Mode */
+
+#define arch_has_single_step() (1)
+#define current_pt_regs() \
+({ (struct pt_regs *)((char *)current_thread_info() + THREAD_SIZE) - 1; })
+
+#define user_stack_pointer(regs) ((regs)->usp)
+
+#define user_mode(regs) (!((regs)->sr & PS_S))
+#define instruction_pointer(regs) ((regs)->pc)
+#define profile_pc(regs) instruction_pointer(regs)
+
+static inline bool in_syscall(struct pt_regs const *regs)
+{
+	return ((regs->sr >> 16) & 0xff) == VEC_TRAP0;
+}
+
+static inline void forget_syscall(struct pt_regs *regs)
+{
+	regs->sr &= ~(0xff << 16);
+}
+
+static inline unsigned long regs_return_value(struct pt_regs *regs)
+{
+	return regs->a0;
+}
+
+#endif /* __ASSEMBLY__ */
+#endif /* __ASM_CSKY_PTRACE_H */
diff --git a/arch/csky/include/asm/syscall.h b/arch/csky/include/asm/syscall.h
index bc0d8717d28b..f624fa3bbc22 100644
--- a/arch/csky/include/asm/syscall.h
+++ b/arch/csky/include/asm/syscall.h
@@ -8,6 +8,8 @@
 #include <abi/regdef.h>
 #include <uapi/linux/audit.h>
 
+extern void *sys_call_table[];
+
 static inline int
 syscall_get_nr(struct task_struct *task, struct pt_regs *regs)
 {
@@ -15,6 +17,13 @@ syscall_get_nr(struct task_struct *task, struct pt_regs *regs)
 }
 
 static inline void
+syscall_set_nr(struct task_struct *task, struct pt_regs *regs,
+	       int sysno)
+{
+	regs_syscallid(regs) = sysno;
+}
+
+static inline void
 syscall_rollback(struct task_struct *task, struct pt_regs *regs)
 {
 	regs->a0 = regs->orig_a0;
diff --git a/arch/csky/include/asm/thread_info.h b/arch/csky/include/asm/thread_info.h
index 0e9d035d712b..0b546a55a8bf 100644
--- a/arch/csky/include/asm/thread_info.h
+++ b/arch/csky/include/asm/thread_info.h
@@ -51,29 +51,26 @@ static inline struct thread_info *current_thread_info(void)
 
 #endif /* !__ASSEMBLY__ */
 
-/* entry.S relies on these definitions!
- * bits 0-5 are tested at every exception exit
- */
 #define TIF_SIGPENDING		0	/* signal pending */
 #define TIF_NOTIFY_RESUME	1       /* callback before returning to user */
 #define TIF_NEED_RESCHED	2	/* rescheduling necessary */
-#define TIF_SYSCALL_TRACE	5	/* syscall trace active */
-#define TIF_DELAYED_TRACE	14	/* single step a syscall */
+#define TIF_SYSCALL_TRACE	3	/* syscall trace active */
+#define TIF_SYSCALL_TRACEPOINT	4       /* syscall tracepoint instrumentation */
+#define TIF_SYSCALL_AUDIT	5	/* syscall auditing */
 #define TIF_POLLING_NRFLAG	16	/* poll_idle() is TIF_NEED_RESCHED */
 #define TIF_MEMDIE		18      /* is terminating due to OOM killer */
-#define TIF_FREEZE		19	/* thread is freezing for suspend */
 #define TIF_RESTORE_SIGMASK	20	/* restore signal mask in do_signal() */
 #define TIF_SECCOMP		21	/* secure computing */
 
-#define _TIF_SIGPENDING         (1 << TIF_SIGPENDING)
-#define _TIF_NOTIFY_RESUME      (1 << TIF_NOTIFY_RESUME)
-#define _TIF_NEED_RESCHED       (1 << TIF_NEED_RESCHED)
-#define _TIF_SYSCALL_TRACE      (1 << TIF_SYSCALL_TRACE)
-#define _TIF_DELAYED_TRACE	(1 << TIF_DELAYED_TRACE)
-#define _TIF_POLLING_NRFLAG     (1 << TIF_POLLING_NRFLAG)
+#define _TIF_SIGPENDING		(1 << TIF_SIGPENDING)
+#define _TIF_NOTIFY_RESUME	(1 << TIF_NOTIFY_RESUME)
+#define _TIF_NEED_RESCHED	(1 << TIF_NEED_RESCHED)
+#define _TIF_SYSCALL_TRACE	(1 << TIF_SYSCALL_TRACE)
+#define _TIF_SYSCALL_TRACEPOINT	(1 << TIF_SYSCALL_TRACEPOINT)
+#define _TIF_SYSCALL_AUDIT	(1 << TIF_SYSCALL_AUDIT)
+#define _TIF_POLLING_NRFLAG	(1 << TIF_POLLING_NRFLAG)
 #define _TIF_MEMDIE		(1 << TIF_MEMDIE)
-#define _TIF_FREEZE             (1 << TIF_FREEZE)
-#define _TIF_RESTORE_SIGMASK    (1 << TIF_RESTORE_SIGMASK)
-#define _TIF_SECCOMP            (1 << TIF_SECCOMP)
+#define _TIF_RESTORE_SIGMASK	(1 << TIF_RESTORE_SIGMASK)
+#define _TIF_SECCOMP		(1 << TIF_SECCOMP)
 
 #endif	/* _ASM_CSKY_THREAD_INFO_H */
diff --git a/arch/csky/include/asm/unistd.h b/arch/csky/include/asm/unistd.h
index 284487477a61..da7a18295615 100644
--- a/arch/csky/include/asm/unistd.h
+++ b/arch/csky/include/asm/unistd.h
@@ -2,3 +2,5 @@
 // Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd.
 
 #include <uapi/asm/unistd.h>
+
+#define NR_syscalls (__NR_syscalls)
diff --git a/arch/csky/include/uapi/asm/perf_regs.h b/arch/csky/include/uapi/asm/perf_regs.h
new file mode 100644
index 000000000000..ee323d818592
--- /dev/null
+++ b/arch/csky/include/uapi/asm/perf_regs.h
@@ -0,0 +1,51 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+// Copyright (C) 2019 Hangzhou C-SKY Microsystems co.,ltd.
+
+#ifndef _ASM_CSKY_PERF_REGS_H
+#define _ASM_CSKY_PERF_REGS_H
+
+/* Index of struct pt_regs */
+enum perf_event_csky_regs {
+	PERF_REG_CSKY_TLS,
+	PERF_REG_CSKY_LR,
+	PERF_REG_CSKY_PC,
+	PERF_REG_CSKY_SR,
+	PERF_REG_CSKY_SP,
+	PERF_REG_CSKY_ORIG_A0,
+	PERF_REG_CSKY_A0,
+	PERF_REG_CSKY_A1,
+	PERF_REG_CSKY_A2,
+	PERF_REG_CSKY_A3,
+	PERF_REG_CSKY_REGS0,
+	PERF_REG_CSKY_REGS1,
+	PERF_REG_CSKY_REGS2,
+	PERF_REG_CSKY_REGS3,
+	PERF_REG_CSKY_REGS4,
+	PERF_REG_CSKY_REGS5,
+	PERF_REG_CSKY_REGS6,
+	PERF_REG_CSKY_REGS7,
+	PERF_REG_CSKY_REGS8,
+	PERF_REG_CSKY_REGS9,
+#if defined(__CSKYABIV2__)
+	PERF_REG_CSKY_EXREGS0,
+	PERF_REG_CSKY_EXREGS1,
+	PERF_REG_CSKY_EXREGS2,
+	PERF_REG_CSKY_EXREGS3,
+	PERF_REG_CSKY_EXREGS4,
+	PERF_REG_CSKY_EXREGS5,
+	PERF_REG_CSKY_EXREGS6,
+	PERF_REG_CSKY_EXREGS7,
+	PERF_REG_CSKY_EXREGS8,
+	PERF_REG_CSKY_EXREGS9,
+	PERF_REG_CSKY_EXREGS10,
+	PERF_REG_CSKY_EXREGS11,
+	PERF_REG_CSKY_EXREGS12,
+	PERF_REG_CSKY_EXREGS13,
+	PERF_REG_CSKY_EXREGS14,
+	PERF_REG_CSKY_HI,
+	PERF_REG_CSKY_LO,
+	PERF_REG_CSKY_DCSR,
+#endif
+	PERF_REG_CSKY_MAX,
+};
+#endif /* _ASM_CSKY_PERF_REGS_H */
diff --git a/arch/csky/include/uapi/asm/ptrace.h b/arch/csky/include/uapi/asm/ptrace.h
index a4eaa8ddf0b1..4e248d5b86ef 100644
--- a/arch/csky/include/uapi/asm/ptrace.h
+++ b/arch/csky/include/uapi/asm/ptrace.h
@@ -48,20 +48,5 @@ struct user_fp {
 	unsigned long	reserved;
 };
 
-#ifdef __KERNEL__
-
-#define PS_S	0x80000000 /* Supervisor Mode */
-
-#define arch_has_single_step() (1)
-#define current_pt_regs() \
-({ (struct pt_regs *)((char *)current_thread_info() + THREAD_SIZE) - 1; })
-
-#define user_stack_pointer(regs) ((regs)->usp)
-
-#define user_mode(regs) (!((regs)->sr & PS_S))
-#define instruction_pointer(regs) ((regs)->pc)
-#define profile_pc(regs) instruction_pointer(regs)
-
-#endif /* __KERNEL__ */
 #endif /* __ASSEMBLY__ */
 #endif /* _CSKY_PTRACE_H */
diff --git a/arch/csky/kernel/Makefile b/arch/csky/kernel/Makefile
index 484e6d3a3647..1624b04bffb5 100644
--- a/arch/csky/kernel/Makefile
+++ b/arch/csky/kernel/Makefile
@@ -9,6 +9,8 @@ obj-$(CONFIG_SMP)			+= smp.o
 obj-$(CONFIG_FUNCTION_TRACER)		+= ftrace.o
 obj-$(CONFIG_STACKTRACE)		+= stacktrace.o
 obj-$(CONFIG_CSKY_PMU_V1)		+= perf_event.o
+obj-$(CONFIG_PERF_EVENTS)		+= perf_callchain.o
+obj-$(CONFIG_HAVE_PERF_REGS)            += perf_regs.o
 
 ifdef CONFIG_FUNCTION_TRACER
 CFLAGS_REMOVE_ftrace.o = $(CC_FLAGS_FTRACE)
diff --git a/arch/csky/kernel/atomic.S b/arch/csky/kernel/atomic.S
index d2357c8f85bd..5b84f11485ae 100644
--- a/arch/csky/kernel/atomic.S
+++ b/arch/csky/kernel/atomic.S
@@ -12,11 +12,10 @@
  * If *ptr != oldval && return 1,
  * else *ptr = newval return 0.
  */
-#ifdef CONFIG_CPU_HAS_LDSTEX
 ENTRY(csky_cmpxchg)
 	USPTOKSP
 	mfcr	a3, epc
-	INCTRAP	a3
+	addi	a3, TRAP0_SIZE
 
 	subi    sp, 8
 	stw     a3, (sp, 0)
@@ -24,6 +23,7 @@ ENTRY(csky_cmpxchg)
 	stw     a3, (sp, 4)
 
 	psrset	ee
+#ifdef CONFIG_CPU_HAS_LDSTEX
 1:
 	ldex	a3, (a2)
 	cmpne	a0, a3
@@ -33,27 +33,7 @@ ENTRY(csky_cmpxchg)
 	bez	a3, 1b
 2:
 	sync.is
-	mvc	a0
-	ldw	a3, (sp, 0)
-	mtcr	a3, epc
-	ldw     a3, (sp, 4)
-	mtcr	a3, epsr
-	addi	sp, 8
-	KSPTOUSP
-	rte
-END(csky_cmpxchg)
 #else
-ENTRY(csky_cmpxchg)
-	USPTOKSP
-	mfcr	a3, epc
-	INCTRAP	a3
-
-	subi    sp, 8
-	stw     a3, (sp, 0)
-	mfcr    a3, epsr
-	stw     a3, (sp, 4)
-
-	psrset	ee
 1:
 	ldw	a3, (a2)
 	cmpne	a0, a3
@@ -61,6 +41,7 @@ ENTRY(csky_cmpxchg)
 2:
 	stw	a1, (a2)
 3:
+#endif
 	mvc	a0
 	ldw	a3, (sp, 0)
 	mtcr	a3, epc
@@ -71,6 +52,7 @@ ENTRY(csky_cmpxchg)
 	rte
 END(csky_cmpxchg)
 
+#ifndef CONFIG_CPU_HAS_LDSTEX
 /*
  * Called from tlbmodified exception
  */
diff --git a/arch/csky/kernel/entry.S b/arch/csky/kernel/entry.S
index 5137ed9062bd..a7e84ccccbd8 100644
--- a/arch/csky/kernel/entry.S
+++ b/arch/csky/kernel/entry.S
@@ -40,7 +40,8 @@ ENTRY(csky_\name)
 	WR_MCIR	a2
 #endif
 	bclri   r6, 0
-	lrw	a2, PHYS_OFFSET
+	lrw	a2, va_pa_offset
+	ld.w	a2, (a2, 0)
 	subu	r6, a2
 	bseti	r6, 31
 
@@ -50,7 +51,8 @@ ENTRY(csky_\name)
 	addu    r6, a2
 	ldw     r6, (r6)
 
-	lrw	a2, PHYS_OFFSET
+	lrw	a2, va_pa_offset
+	ld.w	a2, (a2, 0)
 	subu	r6, a2
 	bseti	r6, 31
 
@@ -91,7 +93,7 @@ ENTRY(csky_\name)
 	mfcr    a3, ss2
 	mfcr    r6, ss3
 	mfcr    a2, ss4
-	SAVE_ALL EPC_KEEP
+	SAVE_ALL 0
 .endm
 .macro tlbop_end is_write
 	RD_MEH	a2
@@ -99,7 +101,6 @@ ENTRY(csky_\name)
 	mov     a0, sp
 	movi    a1, \is_write
 	jbsr    do_page_fault
-	movi    r11_sig, 0             /* r11 = 0, Not a syscall. */
 	jmpi    ret_from_exception
 .endm
 
@@ -118,7 +119,7 @@ jbsr csky_cmpxchg_fixup
 tlbop_end 1
 
 ENTRY(csky_systemcall)
-	SAVE_ALL EPC_INCREASE
+	SAVE_ALL TRAP0_SIZE
 
 	psrset  ee, ie
 
@@ -136,8 +137,9 @@ ENTRY(csky_systemcall)
 	bmaski  r10, THREAD_SHIFT
 	andn    r9, r10
 	ldw     r8, (r9, TINFO_FLAGS)
-	btsti   r8, TIF_SYSCALL_TRACE
-	bt      1f
+	ANDI_R3	r8, (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_TRACEPOINT | _TIF_SYSCALL_AUDIT)
+	cmpnei	r8, 0
+	bt      csky_syscall_trace
 #if defined(__CSKYABIV2__)
 	subi    sp, 8
 	stw  	r5, (sp, 0x4)
@@ -150,10 +152,9 @@ ENTRY(csky_systemcall)
 	stw     a0, (sp, LSAVE_A0)      /* Save return value */
 	jmpi    ret_from_exception
 
-1:
-	movi	a0, 0                   /* enter system call */
-	mov	a1, sp                  /* sp = pt_regs pointer */
-	jbsr	syscall_trace
+csky_syscall_trace:
+	mov	a0, sp                  /* sp = pt_regs pointer */
+	jbsr	syscall_trace_enter
 	/* Prepare args before do system call */
 	ldw	a0, (sp, LSAVE_A0)
 	ldw	a1, (sp, LSAVE_A1)
@@ -173,9 +174,8 @@ ENTRY(csky_systemcall)
 #endif
 	stw	a0, (sp, LSAVE_A0)	/* Save return value */
 
-	movi    a0, 1                   /* leave system call */
-	mov     a1, sp                  /* right now, sp --> pt_regs */
-	jbsr    syscall_trace
+	mov     a0, sp                  /* right now, sp --> pt_regs */
+	jbsr    syscall_trace_exit
 	br	ret_from_exception
 
 ENTRY(ret_from_kernel_thread)
@@ -190,14 +190,11 @@ ENTRY(ret_from_fork)
 	bmaski	r10, THREAD_SHIFT
 	andn	r9, r10
 	ldw	r8, (r9, TINFO_FLAGS)
-	movi	r11_sig, 1
-	btsti	r8, TIF_SYSCALL_TRACE
-	bf	3f
-	movi	a0, 1
-	mov	a1, sp			/* sp = pt_regs pointer */
-	jbsr	syscall_trace
-3:
-	jbsr	ret_from_exception
+	ANDI_R3	r8, (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_TRACEPOINT | _TIF_SYSCALL_AUDIT)
+	cmpnei	r8, 0
+	bf	ret_from_exception
+	mov	a0, sp			/* sp = pt_regs pointer */
+	jbsr	syscall_trace_exit
 
 ret_from_exception:
 	ld	syscallid, (sp, LSAVE_PSR)
@@ -212,41 +209,30 @@ ret_from_exception:
 	bmaski	r10, THREAD_SHIFT
 	andn	r9, r10
 
-resume_userspace:
 	ldw	r8, (r9, TINFO_FLAGS)
 	andi	r8, (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED)
 	cmpnei	r8, 0
 	bt	exit_work
-1:  RESTORE_ALL
+1:
+	RESTORE_ALL
 
 exit_work:
+	lrw	syscallid, ret_from_exception
+	mov	lr, syscallid
+
 	btsti	r8, TIF_NEED_RESCHED
 	bt	work_resched
-	/* If thread_info->flag is empty, RESTORE_ALL */
-	cmpnei	r8, 0
-	bf	1b
-	mov	a1, sp
-	mov	a0, r8
-	mov	a2, r11_sig		/* syscall? */
-	btsti	r8, TIF_SIGPENDING	/* delivering a signal? */
-	/* prevent further restarts(set r11 = 0) */
-	clrt	r11_sig
-	jbsr	do_notify_resume	/* do signals */
-	br	resume_userspace
+
+	mov	a0, sp
+	mov	a1, r8
+	jmpi	do_notify_resume
 
 work_resched:
-	lrw	syscallid, ret_from_exception
-	mov	r15, syscallid		/* Return address in link */
 	jmpi	schedule
 
-ENTRY(sys_rt_sigreturn)
-	movi	r11_sig, 0
-	jmpi	do_rt_sigreturn
-
 ENTRY(csky_trap)
-	SAVE_ALL EPC_KEEP
+	SAVE_ALL 0
 	psrset	ee
-	movi	r11_sig, 0             /* r11 = 0, Not a syscall. */
 	mov	a0, sp                 /* Push Stack pointer arg */
 	jbsr	trap_c                 /* Call C-level trap handler */
 	jmpi	ret_from_exception
@@ -261,7 +247,7 @@ ENTRY(csky_get_tls)
 
 	/* increase epc for continue */
 	mfcr	a0, epc
-	INCTRAP	a0
+	addi	a0, TRAP0_SIZE
 	mtcr	a0, epc
 
 	/* get current task thread_info with kernel 8K stack */
@@ -278,9 +264,8 @@ ENTRY(csky_get_tls)
 	rte
 
 ENTRY(csky_irq)
-	SAVE_ALL EPC_KEEP
+	SAVE_ALL 0
 	psrset	ee
-	movi	r11_sig, 0		/* r11 = 0, Not a syscall. */
 
 #ifdef CONFIG_PREEMPT
 	mov	r9, sp			/* Get current stack  pointer */
diff --git a/arch/csky/kernel/ftrace.c b/arch/csky/kernel/ftrace.c
index 274c431f1810..44f4880179b7 100644
--- a/arch/csky/kernel/ftrace.c
+++ b/arch/csky/kernel/ftrace.c
@@ -3,6 +3,137 @@
 
 #include <linux/ftrace.h>
 #include <linux/uaccess.h>
+#include <asm/cacheflush.h>
+
+#ifdef CONFIG_DYNAMIC_FTRACE
+
+#define NOP		0x4000
+#define NOP32_HI	0xc400
+#define NOP32_LO	0x4820
+#define PUSH_LR		0x14d0
+#define MOVIH_LINK	0xea3a
+#define ORI_LINK	0xef5a
+#define JSR_LINK	0xe8fa
+#define BSR_LINK	0xe000
+
+/*
+ * Gcc-csky with -pg will insert stub in function prologue:
+ *	push	lr
+ *	jbsr	_mcount
+ *	nop32
+ *	nop32
+ *
+ * If the (callee - current_pc) is less then 64MB, we'll use bsr:
+ *	push	lr
+ *	bsr	_mcount
+ *	nop32
+ *	nop32
+ * else we'll use (movih + ori + jsr):
+ *	push	lr
+ *	movih	r26, ...
+ *	ori	r26, ...
+ *	jsr	r26
+ *
+ * (r26 is our reserved link-reg)
+ *
+ */
+static inline void make_jbsr(unsigned long callee, unsigned long pc,
+			     uint16_t *call, bool nolr)
+{
+	long offset;
+
+	call[0]	= nolr ? NOP : PUSH_LR;
+
+	offset = (long) callee - (long) pc;
+
+	if (unlikely(offset < -67108864 || offset > 67108864)) {
+		call[1] = MOVIH_LINK;
+		call[2] = callee >> 16;
+		call[3] = ORI_LINK;
+		call[4] = callee & 0xffff;
+		call[5] = JSR_LINK;
+		call[6] = 0;
+	} else {
+		offset = offset >> 1;
+
+		call[1] = BSR_LINK |
+			 ((uint16_t)((unsigned long) offset >> 16) & 0x3ff);
+		call[2] = (uint16_t)((unsigned long) offset & 0xffff);
+		call[3] = call[5] = NOP32_HI;
+		call[4] = call[6] = NOP32_LO;
+	}
+}
+
+static uint16_t nops[7] = {NOP, NOP32_HI, NOP32_LO, NOP32_HI, NOP32_LO,
+				NOP32_HI, NOP32_LO};
+static int ftrace_check_current_nop(unsigned long hook)
+{
+	uint16_t olds[7];
+	unsigned long hook_pos = hook - 2;
+
+	if (probe_kernel_read((void *)olds, (void *)hook_pos, sizeof(nops)))
+		return -EFAULT;
+
+	if (memcmp((void *)nops, (void *)olds, sizeof(nops))) {
+		pr_err("%p: nop but get (%04x %04x %04x %04x %04x %04x %04x)\n",
+			(void *)hook_pos,
+			olds[0], olds[1], olds[2], olds[3], olds[4], olds[5],
+			olds[6]);
+
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int ftrace_modify_code(unsigned long hook, unsigned long target,
+			      bool enable, bool nolr)
+{
+	uint16_t call[7];
+
+	unsigned long hook_pos = hook - 2;
+	int ret = 0;
+
+	make_jbsr(target, hook, call, nolr);
+
+	ret = probe_kernel_write((void *)hook_pos, enable ? call : nops,
+				 sizeof(nops));
+	if (ret)
+		return -EPERM;
+
+	flush_icache_range(hook_pos, hook_pos + MCOUNT_INSN_SIZE);
+
+	return 0;
+}
+
+int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
+{
+	int ret = ftrace_check_current_nop(rec->ip);
+
+	if (ret)
+		return ret;
+
+	return ftrace_modify_code(rec->ip, addr, true, false);
+}
+
+int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec,
+		    unsigned long addr)
+{
+	return ftrace_modify_code(rec->ip, addr, false, false);
+}
+
+int ftrace_update_ftrace_func(ftrace_func_t func)
+{
+	int ret = ftrace_modify_code((unsigned long)&ftrace_call,
+				(unsigned long)func, true, true);
+	return ret;
+}
+
+int __init ftrace_dyn_arch_init(void)
+{
+	return 0;
+}
+#endif /* CONFIG_DYNAMIC_FTRACE */
 
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
@@ -43,8 +174,21 @@ void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
 			*(unsigned long *)frame_pointer = return_hooker;
 	}
 }
-#endif
+
+#ifdef CONFIG_DYNAMIC_FTRACE
+int ftrace_enable_ftrace_graph_caller(void)
+{
+	return ftrace_modify_code((unsigned long)&ftrace_graph_call,
+			(unsigned long)&ftrace_graph_caller, true, true);
+}
+
+int ftrace_disable_ftrace_graph_caller(void)
+{
+	return ftrace_modify_code((unsigned long)&ftrace_graph_call,
+			(unsigned long)&ftrace_graph_caller, false, true);
+}
+#endif /* CONFIG_DYNAMIC_FTRACE */
+#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
 
 /* _mcount is defined in abi's mcount.S */
-extern void _mcount(void);
 EXPORT_SYMBOL(_mcount);
diff --git a/arch/csky/kernel/head.S b/arch/csky/kernel/head.S
index 9c4ec473b76b..61989f9241c0 100644
--- a/arch/csky/kernel/head.S
+++ b/arch/csky/kernel/head.S
@@ -7,16 +7,11 @@
 
 __HEAD
 ENTRY(_start)
-	/* set super user mode */
-	lrw	a3, DEFAULT_PSR_VALUE
-	mtcr    a3, psr
-	psrset  ee
-
-	SETUP_MMU a3
+	SETUP_MMU
 
 	/* set stack point */
-	lrw     a3, init_thread_union + THREAD_SIZE
-	mov	sp, a3
+	lrw     r6, init_thread_union + THREAD_SIZE
+	mov	sp, r6
 
 	jmpi	csky_start
 END(_start)
@@ -24,53 +19,12 @@ END(_start)
 #ifdef CONFIG_SMP
 .align 10
 ENTRY(_start_smp_secondary)
-	/* Invalid I/Dcache BTB BHT */
-	movi	a3, 7
-	lsli	a3, 16
-	addi	a3, (1<<4) | 3
-	mtcr	a3, cr17
-
-	tlbi.alls
-
-	/* setup PAGEMASK */
-	movi	a3, 0
-	mtcr	a3, cr<6, 15>
-
-	/* setup MEL0/MEL1 */
-	grs	a0, _start_smp_pc
-_start_smp_pc:
-	bmaski  a1, 13
-	andn    a0, a1
-	movi    a1, 0x00000006
-	movi    a2, 0x00001006
-	or      a1, a0
-	or      a2, a0
-	mtcr	a1, cr<2, 15>
-	mtcr	a2, cr<3, 15>
-
-	/* setup MEH */
-	mtcr	a0, cr<4, 15>
-
-	/* write TLB */
-	bgeni   a3, 28
-	mtcr	a3, cr<8, 15>
-
-	SETUP_MMU a3
-
-	/* enable MMU */
-	movi	a3, 1
-	mtcr    a3, cr18
-
-	jmpi	_goto_mmu_on
-_goto_mmu_on:
-	lrw	a3, DEFAULT_PSR_VALUE
-	mtcr    a3, psr
-	psrset  ee
+	SETUP_MMU
 
 	/* set stack point */
-	lrw     a3, secondary_stack
-	ld.w	a3, (a3, 0)
-	mov	sp, a3
+	lrw     r6, secondary_stack
+	ld.w	r6, (r6, 0)
+	mov	sp, r6
 
 	jmpi	csky_start_secondary
 END(_start_smp_secondary)
diff --git a/arch/csky/kernel/perf_callchain.c b/arch/csky/kernel/perf_callchain.c
new file mode 100644
index 000000000000..e68ff375c8f8
--- /dev/null
+++ b/arch/csky/kernel/perf_callchain.c
@@ -0,0 +1,119 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2019 Hangzhou C-SKY Microsystems co.,ltd.
+
+#include <linux/perf_event.h>
+#include <linux/uaccess.h>
+
+/* Kernel callchain */
+struct stackframe {
+	unsigned long fp;
+	unsigned long lr;
+};
+
+static int unwind_frame_kernel(struct stackframe *frame)
+{
+	if (kstack_end((void *)frame->fp))
+		return -EPERM;
+	if (frame->fp & 0x3 || frame->fp < TASK_SIZE)
+		return -EPERM;
+
+	*frame = *(struct stackframe *)frame->fp;
+	if (__kernel_text_address(frame->lr)) {
+		int graph = 0;
+
+		frame->lr = ftrace_graph_ret_addr(NULL, &graph, frame->lr,
+				NULL);
+	}
+	return 0;
+}
+
+static void notrace walk_stackframe(struct stackframe *fr,
+			struct perf_callchain_entry_ctx *entry)
+{
+	do {
+		perf_callchain_store(entry, fr->lr);
+	} while (unwind_frame_kernel(fr) >= 0);
+}
+
+/*
+ * Get the return address for a single stackframe and return a pointer to the
+ * next frame tail.
+ */
+static unsigned long user_backtrace(struct perf_callchain_entry_ctx *entry,
+			unsigned long fp, unsigned long reg_lr)
+{
+	struct stackframe buftail;
+	unsigned long lr = 0;
+	unsigned long *user_frame_tail = (unsigned long *)fp;
+
+	/* Check accessibility of one struct frame_tail beyond */
+	if (!access_ok(user_frame_tail, sizeof(buftail)))
+		return 0;
+	if (__copy_from_user_inatomic(&buftail, user_frame_tail,
+				      sizeof(buftail)))
+		return 0;
+
+	if (reg_lr != 0)
+		lr = reg_lr;
+	else
+		lr = buftail.lr;
+
+	fp = buftail.fp;
+	perf_callchain_store(entry, lr);
+
+	return fp;
+}
+
+/*
+ * This will be called when the target is in user mode
+ * This function will only be called when we use
+ * "PERF_SAMPLE_CALLCHAIN" in
+ * kernel/events/core.c:perf_prepare_sample()
+ *
+ * How to trigger perf_callchain_[user/kernel] :
+ * $ perf record -e cpu-clock --call-graph fp ./program
+ * $ perf report --call-graph
+ *
+ * On C-SKY platform, the program being sampled and the C library
+ * need to be compiled with * -mbacktrace, otherwise the user
+ * stack will not contain function frame.
+ */
+void perf_callchain_user(struct perf_callchain_entry_ctx *entry,
+			 struct pt_regs *regs)
+{
+	unsigned long fp = 0;
+
+	/* C-SKY does not support virtualization. */
+	if (perf_guest_cbs && perf_guest_cbs->is_in_guest())
+		return;
+
+	fp = regs->regs[4];
+	perf_callchain_store(entry, regs->pc);
+
+	/*
+	 * While backtrace from leaf function, lr is normally
+	 * not saved inside frame on C-SKY, so get lr from pt_regs
+	 * at the sample point. However, lr value can be incorrect if
+	 * lr is used as temp register
+	 */
+	fp = user_backtrace(entry, fp, regs->lr);
+
+	while (fp && !(fp & 0x3) && entry->nr < entry->max_stack)
+		fp = user_backtrace(entry, fp, 0);
+}
+
+void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry,
+			   struct pt_regs *regs)
+{
+	struct stackframe fr;
+
+	/* C-SKY does not support virtualization. */
+	if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
+		pr_warn("C-SKY does not support perf in guest mode!");
+		return;
+	}
+
+	fr.fp = regs->regs[4];
+	fr.lr = regs->lr;
+	walk_stackframe(&fr, entry);
+}
diff --git a/arch/csky/kernel/perf_regs.c b/arch/csky/kernel/perf_regs.c
new file mode 100644
index 000000000000..eb32838b8210
--- /dev/null
+++ b/arch/csky/kernel/perf_regs.c
@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2019 Hangzhou C-SKY Microsystems co.,ltd.
+
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <linux/perf_event.h>
+#include <linux/bug.h>
+#include <asm/perf_regs.h>
+#include <asm/ptrace.h>
+
+u64 perf_reg_value(struct pt_regs *regs, int idx)
+{
+	if (WARN_ON_ONCE((u32)idx >= PERF_REG_CSKY_MAX))
+		return 0;
+
+	return (u64)*((u32 *)regs + idx);
+}
+
+#define REG_RESERVED (~((1ULL << PERF_REG_CSKY_MAX) - 1))
+
+int perf_reg_validate(u64 mask)
+{
+	if (!mask || mask & REG_RESERVED)
+		return -EINVAL;
+
+	return 0;
+}
+
+u64 perf_reg_abi(struct task_struct *task)
+{
+	return PERF_SAMPLE_REGS_ABI_32;
+}
+
+void perf_get_regs_user(struct perf_regs *regs_user,
+			struct pt_regs *regs,
+			struct pt_regs *regs_user_copy)
+{
+	regs_user->regs = task_pt_regs(current);
+	regs_user->abi = perf_reg_abi(current);
+}
diff --git a/arch/csky/kernel/ptrace.c b/arch/csky/kernel/ptrace.c
index f2f12fff36f7..313623a19ecb 100644
--- a/arch/csky/kernel/ptrace.c
+++ b/arch/csky/kernel/ptrace.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 // Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd.
 
+#include <linux/audit.h>
 #include <linux/elf.h>
 #include <linux/errno.h>
 #include <linux/kernel.h>
@@ -11,6 +12,7 @@
 #include <linux/sched/task_stack.h>
 #include <linux/signal.h>
 #include <linux/smp.h>
+#include <linux/tracehook.h>
 #include <linux/uaccess.h>
 #include <linux/user.h>
 
@@ -22,6 +24,9 @@
 
 #include <abi/regdef.h>
 
+#define CREATE_TRACE_POINTS
+#include <trace/events/syscalls.h>
+
 /* sets the trace bits. */
 #define TRACE_MODE_SI      (1 << 14)
 #define TRACE_MODE_RUN     0
@@ -207,35 +212,27 @@ long arch_ptrace(struct task_struct *child, long request,
 	return ret;
 }
 
-/*
- * If process's system calls is traces, do some corresponding handles in this
- * function before entering system call function and after exiting system call
- * function.
- */
-asmlinkage void syscall_trace(int why, struct pt_regs *regs)
+asmlinkage void syscall_trace_enter(struct pt_regs *regs)
 {
-	long saved_why;
-	/*
-	 * Save saved_why, why is used to denote syscall entry/exit;
-	 * why = 0:entry, why = 1: exit
-	 */
-	saved_why = regs->regs[SYSTRACE_SAVENUM];
-	regs->regs[SYSTRACE_SAVENUM] = why;
-
-	ptrace_notify(SIGTRAP | ((current->ptrace & PT_TRACESYSGOOD)
-					? 0x80 : 0));
-
-	/*
-	 * this isn't the same as continuing with a signal, but it will do
-	 * for normal use.  strace only continues with a signal if the
-	 * stopping signal is not SIGTRAP.  -brl
-	 */
-	if (current->exit_code) {
-		send_sig(current->exit_code, current, 1);
-		current->exit_code = 0;
-	}
+	if (test_thread_flag(TIF_SYSCALL_TRACE))
+		if (tracehook_report_syscall_entry(regs))
+			syscall_set_nr(current, regs, -1);
+
+	if (test_thread_flag(TIF_SYSCALL_TRACEPOINT))
+		trace_sys_enter(regs, syscall_get_nr(current, regs));
+
+	audit_syscall_entry(regs_syscallid(regs), regs->a0, regs->a1, regs->a2, regs->a3);
+}
+
+asmlinkage void syscall_trace_exit(struct pt_regs *regs)
+{
+	audit_syscall_exit(regs);
+
+	if (test_thread_flag(TIF_SYSCALL_TRACE))
+		tracehook_report_syscall_exit(regs, 0);
 
-	regs->regs[SYSTRACE_SAVENUM] = saved_why;
+	if (test_thread_flag(TIF_SYSCALL_TRACEPOINT))
+		trace_sys_exit(regs, syscall_get_return_value(current, regs));
 }
 
 extern void show_stack(struct task_struct *task, unsigned long *stack);
diff --git a/arch/csky/kernel/setup.c b/arch/csky/kernel/setup.c
index dff8b89444ec..23ee604aafdb 100644
--- a/arch/csky/kernel/setup.c
+++ b/arch/csky/kernel/setup.c
@@ -142,18 +142,24 @@ void __init setup_arch(char **cmdline_p)
 #endif
 }
 
-asmlinkage __visible void __init csky_start(unsigned int unused, void *param)
+unsigned long va_pa_offset;
+EXPORT_SYMBOL(va_pa_offset);
+
+asmlinkage __visible void __init csky_start(unsigned int unused,
+					    void *dtb_start)
 {
 	/* Clean up bss section */
 	memset(__bss_start, 0, __bss_stop - __bss_start);
 
+	va_pa_offset = read_mmu_msa0() & ~(SSEG_SIZE - 1);
+
 	pre_trap_init();
 	pre_mmu_init();
 
-	if (param == NULL)
+	if (dtb_start == NULL)
 		early_init_dt_scan(__dtb_start);
 	else
-		early_init_dt_scan(param);
+		early_init_dt_scan(dtb_start);
 
 	start_kernel();
 
diff --git a/arch/csky/kernel/signal.c b/arch/csky/kernel/signal.c
index 207a891479d2..04a43cfd4e09 100644
--- a/arch/csky/kernel/signal.c
+++ b/arch/csky/kernel/signal.c
@@ -1,26 +1,10 @@
 // SPDX-License-Identifier: GPL-2.0
-// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd.
 
-#include <linux/sched.h>
-#include <linux/mm.h>
-#include <linux/kernel.h>
 #include <linux/signal.h>
+#include <linux/uaccess.h>
 #include <linux/syscalls.h>
-#include <linux/errno.h>
-#include <linux/wait.h>
-#include <linux/ptrace.h>
-#include <linux/unistd.h>
-#include <linux/stddef.h>
-#include <linux/highuid.h>
-#include <linux/personality.h>
-#include <linux/tty.h>
-#include <linux/binfmts.h>
 #include <linux/tracehook.h>
-#include <linux/freezer.h>
-#include <linux/uaccess.h>
 
-#include <asm/setup.h>
-#include <asm/pgtable.h>
 #include <asm/traps.h>
 #include <asm/ucontext.h>
 #include <asm/vdso.h>
@@ -29,110 +13,117 @@
 
 #ifdef CONFIG_CPU_HAS_FPU
 #include <abi/fpu.h>
-
-static int restore_fpu_state(struct sigcontext *sc)
+static int restore_fpu_state(struct sigcontext __user *sc)
 {
 	int err = 0;
 	struct user_fp user_fp;
 
-	err = copy_from_user(&user_fp, &sc->sc_user_fp, sizeof(user_fp));
+	err = __copy_from_user(&user_fp, &sc->sc_user_fp, sizeof(user_fp));
 
 	restore_from_user_fp(&user_fp);
 
 	return err;
 }
 
-static int save_fpu_state(struct sigcontext *sc)
+static int save_fpu_state(struct sigcontext __user *sc)
 {
 	struct user_fp user_fp;
 
 	save_to_user_fp(&user_fp);
 
-	return copy_to_user(&sc->sc_user_fp, &user_fp, sizeof(user_fp));
+	return __copy_to_user(&sc->sc_user_fp, &user_fp, sizeof(user_fp));
 }
 #else
-static inline int restore_fpu_state(struct sigcontext *sc) { return 0; }
-static inline int save_fpu_state(struct sigcontext *sc) { return 0; }
+#define restore_fpu_state(sigcontext)	(0)
+#define save_fpu_state(sigcontext)	(0)
 #endif
 
 struct rt_sigframe {
-	int sig;
-	struct siginfo *pinfo;
-	void *puc;
 	struct siginfo info;
 	struct ucontext uc;
 };
 
-static int
-restore_sigframe(struct pt_regs *regs,
-		 struct sigcontext *sc, int *pr2)
+static long restore_sigcontext(struct pt_regs *regs,
+	struct sigcontext __user *sc)
 {
 	int err = 0;
 
-	/* Always make any pending restarted system calls return -EINTR */
-	current_thread_info()->task->restart_block.fn = do_no_restart_syscall;
-
-	err |= copy_from_user(regs, &sc->sc_pt_regs, sizeof(struct pt_regs));
+	/* sc_pt_regs is structured the same as the start of pt_regs */
+	err |= __copy_from_user(regs, &sc->sc_pt_regs, sizeof(struct pt_regs));
 
+	/* Restore the floating-point state. */
 	err |= restore_fpu_state(sc);
 
-	*pr2 = regs->a0;
 	return err;
 }
 
-asmlinkage int
-do_rt_sigreturn(void)
+SYSCALL_DEFINE0(rt_sigreturn)
 {
-	sigset_t set;
-	int a0;
 	struct pt_regs *regs = current_pt_regs();
-	struct rt_sigframe *frame = (struct rt_sigframe *)(regs->usp);
+	struct rt_sigframe __user *frame;
+	struct task_struct *task;
+	sigset_t set;
+
+	/* Always make any pending restarted system calls return -EINTR */
+	current->restart_block.fn = do_no_restart_syscall;
+
+	frame = (struct rt_sigframe __user *)regs->usp;
 
 	if (!access_ok(frame, sizeof(*frame)))
 		goto badframe;
+
 	if (__copy_from_user(&set, &frame->uc.uc_sigmask, sizeof(set)))
 		goto badframe;
 
-	sigdelsetmask(&set, (sigmask(SIGKILL) | sigmask(SIGSTOP)));
-	spin_lock_irq(&current->sighand->siglock);
-	current->blocked = set;
-	recalc_sigpending();
-	spin_unlock_irq(&current->sighand->siglock);
+	set_current_blocked(&set);
 
-	if (restore_sigframe(regs, &frame->uc.uc_mcontext, &a0))
+	if (restore_sigcontext(regs, &frame->uc.uc_mcontext))
 		goto badframe;
 
-	return a0;
+	if (restore_altstack(&frame->uc.uc_stack))
+		goto badframe;
+
+	return regs->a0;
 
 badframe:
-	force_sig(SIGSEGV, current);
+	task = current;
+	force_sig(SIGSEGV, task);
 	return 0;
 }
 
-static int setup_sigframe(struct sigcontext *sc, struct pt_regs *regs)
+static int setup_sigcontext(struct rt_sigframe __user *frame,
+	struct pt_regs *regs)
 {
+	struct sigcontext __user *sc = &frame->uc.uc_mcontext;
 	int err = 0;
 
-	err |= copy_to_user(&sc->sc_pt_regs, regs, sizeof(struct pt_regs));
+	err |= __copy_to_user(&sc->sc_pt_regs, regs, sizeof(struct pt_regs));
 	err |= save_fpu_state(sc);
 
 	return err;
 }
 
-static inline void *
-get_sigframe(struct k_sigaction *ka, struct pt_regs *regs, size_t frame_size)
+static inline void __user *get_sigframe(struct ksignal *ksig,
+	struct pt_regs *regs, size_t framesize)
 {
-	unsigned long usp;
+	unsigned long sp;
+	/* Default to using normal stack */
+	sp = regs->usp;
+
+	/*
+	 * If we are on the alternate signal stack and would overflow it, don't.
+	 * Return an always-bogus address instead so we will die with SIGSEGV.
+	 */
+	if (on_sig_stack(sp) && !likely(on_sig_stack(sp - framesize)))
+		return (void __user __force *)(-1UL);
 
-	/* Default to using normal stack.  */
-	usp = regs->usp;
+	/* This is the X/Open sanctioned signal stack switching. */
+	sp = sigsp(sp, ksig) - framesize;
 
-	/* This is the X/Open sanctioned signal stack switching.  */
-	if ((ka->sa.sa_flags & SA_ONSTACK) && !sas_ss_flags(usp)) {
-		if (!on_sig_stack(usp))
-			usp = current->sas_ss_sp + current->sas_ss_size;
-	}
-	return (void *)((usp - frame_size) & -8UL);
+	/* Align the stack frame. */
+	sp &= -8UL;
+
+	return (void __user *)sp;
 }
 
 static int
@@ -140,205 +131,128 @@ setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs)
 {
 	struct rt_sigframe *frame;
 	int err = 0;
-
 	struct csky_vdso *vdso = current->mm->context.vdso;
 
-	frame = get_sigframe(&ksig->ka, regs, sizeof(*frame));
-	if (!frame)
-		return 1;
+	frame = get_sigframe(ksig, regs, sizeof(*frame));
+	if (!access_ok(frame, sizeof(*frame)))
+		return -EFAULT;
 
-	err |= __put_user(ksig->sig, &frame->sig);
-	err |= __put_user(&frame->info, &frame->pinfo);
-	err |= __put_user(&frame->uc, &frame->puc);
 	err |= copy_siginfo_to_user(&frame->info, &ksig->info);
 
-	/* Create the ucontext.  */
+	/* Create the ucontext. */
 	err |= __put_user(0, &frame->uc.uc_flags);
-	err |= __put_user(0, &frame->uc.uc_link);
-	err |= __put_user((void *)current->sas_ss_sp,
-			&frame->uc.uc_stack.ss_sp);
-	err |= __put_user(sas_ss_flags(regs->usp),
-			&frame->uc.uc_stack.ss_flags);
-	err |= __put_user(current->sas_ss_size, &frame->uc.uc_stack.ss_size);
-	err |= setup_sigframe(&frame->uc.uc_mcontext, regs);
-	err |= copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set));
-
+	err |= __put_user(NULL, &frame->uc.uc_link);
+	err |= __save_altstack(&frame->uc.uc_stack, regs->usp);
+	err |= setup_sigcontext(frame, regs);
+	err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set));
 	if (err)
-		goto give_sigsegv;
+		return -EFAULT;
 
-	/* Set up registers for signal handler */
-	regs->usp = (unsigned long)frame;
-	regs->pc = (unsigned long)ksig->ka.sa.sa_handler;
-	regs->lr = (unsigned long)vdso->rt_signal_retcode;
+	/* Set up to return from userspace. */
+	regs->lr = (unsigned long)(vdso->rt_signal_retcode);
 
-adjust_stack:
-	regs->a0 = ksig->sig; /* first arg is signo */
-	regs->a1 = (unsigned long)(&(frame->info));
-	regs->a2 = (unsigned long)(&(frame->uc));
-	return err;
+	/*
+	 * Set up registers for signal handler.
+	 * Registers that we don't modify keep the value they had from
+	 * user-space at the time we took the signal.
+	 * We always pass siginfo and mcontext, regardless of SA_SIGINFO,
+	 * since some things rely on this (e.g. glibc's debug/segfault.c).
+	 */
+	regs->pc  = (unsigned long)ksig->ka.sa.sa_handler;
+	regs->usp = (unsigned long)frame;
+	regs->a0  = ksig->sig;				/* a0: signal number */
+	regs->a1  = (unsigned long)(&(frame->info));	/* a1: siginfo pointer */
+	regs->a2  = (unsigned long)(&(frame->uc));	/* a2: ucontext pointer */
 
-give_sigsegv:
-	if (ksig->sig == SIGSEGV)
-		ksig->ka.sa.sa_handler = SIG_DFL;
-	force_sig(SIGSEGV, current);
-	goto adjust_stack;
+	return 0;
 }
 
-/*
- * OK, we're invoking a handler
- */
-static int
-handle_signal(struct ksignal *ksig, struct pt_regs *regs)
+static void handle_signal(struct ksignal *ksig, struct pt_regs *regs)
 {
-	int ret;
 	sigset_t *oldset = sigmask_to_save();
+	int ret;
 
-	/*
-	 * set up the stack frame, regardless of SA_SIGINFO,
-	 * and pass info anyway.
-	 */
-	ret = setup_rt_frame(ksig, oldset, regs);
+	/* Are we from a system call? */
+	if (in_syscall(regs)) {
+		/* Avoid additional syscall restarting via ret_from_exception */
+		forget_syscall(regs);
+
+		/* If so, check system call restarting.. */
+		switch (regs->a0) {
+		case -ERESTART_RESTARTBLOCK:
+		case -ERESTARTNOHAND:
+			regs->a0 = -EINTR;
+			break;
 
-	if (ret != 0) {
-		force_sigsegv(ksig->sig, current);
-		return ret;
+		case -ERESTARTSYS:
+			if (!(ksig->ka.sa.sa_flags & SA_RESTART)) {
+				regs->a0 = -EINTR;
+				break;
+			}
+			/* fallthrough */
+		case -ERESTARTNOINTR:
+			regs->a0 = regs->orig_a0;
+			regs->pc -= TRAP0_SIZE;
+			break;
+		}
 	}
 
-	/* Block the signal if we were successful. */
-	spin_lock_irq(&current->sighand->siglock);
-	sigorsets(&current->blocked, &current->blocked, &ksig->ka.sa.sa_mask);
-	if (!(ksig->ka.sa.sa_flags & SA_NODEFER))
-		sigaddset(&current->blocked, ksig->sig);
-	recalc_sigpending();
-	spin_unlock_irq(&current->sighand->siglock);
+	/* Set up the stack frame */
+	ret = setup_rt_frame(ksig, oldset, regs);
 
-	return 0;
+	signal_setup_done(ret, ksig, 0);
 }
 
-/*
- * Note that 'init' is a special process: it doesn't get signals it doesn't
- * want to handle. Thus you cannot kill init even with a SIGKILL even by
- * mistake.
- *
- * Note that we go through the signals twice: once to check the signals
- * that the kernel can handle, and then we build all the user-level signal
- * handling stack-frames in one go after that.
- */
-static void do_signal(struct pt_regs *regs, int syscall)
+static void do_signal(struct pt_regs *regs)
 {
-	unsigned int retval = 0, continue_addr = 0, restart_addr = 0;
 	struct ksignal ksig;
 
-	/*
-	 * We want the common case to go fast, which
-	 * is why we may in certain cases get here from
-	 * kernel mode. Just return without doing anything
-	 * if so.
-	 */
-	if (!user_mode(regs))
+	if (get_signal(&ksig)) {
+		/* Actually deliver the signal */
+		handle_signal(&ksig, regs);
 		return;
+	}
 
-	/*
-	 * If we were from a system call, check for system call restarting...
-	 */
-	if (syscall) {
-		continue_addr = regs->pc;
-#if defined(__CSKYABIV2__)
-		restart_addr = continue_addr - 4;
-#else
-		restart_addr = continue_addr - 2;
-#endif
-		retval = regs->a0;
+	/* Did we come from a system call? */
+	if (in_syscall(regs)) {
+		/* Avoid additional syscall restarting via ret_from_exception */
+		forget_syscall(regs);
 
-		/*
-		 * Prepare for system call restart.  We do this here so that a
-		 * debugger will see the already changed.
-		 */
-		switch (retval) {
+		/* Restart the system call - no handlers present */
+		switch (regs->a0) {
 		case -ERESTARTNOHAND:
 		case -ERESTARTSYS:
 		case -ERESTARTNOINTR:
 			regs->a0 = regs->orig_a0;
-			regs->pc = restart_addr;
+			regs->pc -= TRAP0_SIZE;
 			break;
 		case -ERESTART_RESTARTBLOCK:
-			regs->a0 = -EINTR;
+			regs->a0 = regs->orig_a0;
+			regs_syscallid(regs) = __NR_restart_syscall;
+			regs->pc -= TRAP0_SIZE;
 			break;
 		}
 	}
 
-	if (try_to_freeze())
-		goto no_signal;
-
 	/*
-	 * Get the signal to deliver.  When running under ptrace, at this
-	 * point the debugger may change all our registers ...
+	 * If there is no signal to deliver, we just put the saved
+	 * sigmask back.
 	 */
-	if (get_signal(&ksig)) {
-		/*
-		 * Depending on the signal settings we may need to revert the
-		 * decision to restart the system call.  But skip this if a
-		 * debugger has chosen to restart at a different PC.
-		 */
-		if (regs->pc == restart_addr) {
-			if (retval == -ERESTARTNOHAND ||
-			    (retval == -ERESTARTSYS &&
-			     !(ksig.ka.sa.sa_flags & SA_RESTART))) {
-				regs->a0 = -EINTR;
-				regs->pc = continue_addr;
-			}
-		}
-
-		/* Whee!  Actually deliver the signal.  */
-		if (handle_signal(&ksig, regs) == 0) {
-			/*
-			 * A signal was successfully delivered; the saved
-			 * sigmask will have been stored in the signal frame,
-			 * and will be restored by sigreturn, so we can simply
-			 * clear the TIF_RESTORE_SIGMASK flag.
-			 */
-			if (test_thread_flag(TIF_RESTORE_SIGMASK))
-				clear_thread_flag(TIF_RESTORE_SIGMASK);
-		}
-		return;
-	}
-
-no_signal:
-	if (syscall) {
-		/*
-		 * Handle restarting a different system call.  As above,
-		 * if a debugger has chosen to restart at a different PC,
-		 * ignore the restart.
-		 */
-		if (retval == -ERESTART_RESTARTBLOCK
-				&& regs->pc == continue_addr) {
-#if defined(__CSKYABIV2__)
-			regs->regs[3] = __NR_restart_syscall;
-			regs->pc -= 4;
-#else
-			regs->regs[9] = __NR_restart_syscall;
-			regs->pc -= 2;
-#endif
-		}
-
-		/*
-		 * If there's no signal to deliver, we just put the saved
-		 * sigmask back.
-		 */
-		if (test_thread_flag(TIF_RESTORE_SIGMASK)) {
-			clear_thread_flag(TIF_RESTORE_SIGMASK);
-			sigprocmask(SIG_SETMASK, &current->saved_sigmask, NULL);
-		}
-	}
+	restore_saved_sigmask();
 }
 
-asmlinkage void
-do_notify_resume(unsigned int thread_flags, struct pt_regs *regs, int syscall)
+/*
+ * notification of userspace execution resumption
+ * - triggered by the _TIF_WORK_MASK flags
+ */
+asmlinkage void do_notify_resume(struct pt_regs *regs,
+	unsigned long thread_info_flags)
 {
-	if (thread_flags & _TIF_SIGPENDING)
-		do_signal(regs, syscall);
+	/* Handle pending signal delivery */
+	if (thread_info_flags & _TIF_SIGPENDING)
+		do_signal(regs);
 
-	if (thread_flags & _TIF_NOTIFY_RESUME) {
+	if (thread_info_flags & _TIF_NOTIFY_RESUME) {
 		clear_thread_flag(TIF_NOTIFY_RESUME);
 		tracehook_notify_resume(regs);
 	}
diff --git a/arch/csky/mm/fault.c b/arch/csky/mm/fault.c
index d6f4b66b93e2..18041f46ded1 100644
--- a/arch/csky/mm/fault.c
+++ b/arch/csky/mm/fault.c
@@ -15,9 +15,9 @@
 #include <linux/smp.h>
 #include <linux/version.h>
 #include <linux/vt_kern.h>
-#include <linux/kernel.h>
 #include <linux/extable.h>
 #include <linux/uaccess.h>
+#include <linux/perf_event.h>
 
 #include <asm/hardirq.h>
 #include <asm/mmu_context.h>
@@ -82,7 +82,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long write,
 
 		unsigned long pgd_base;
 
-		pgd_base = tlb_get_pgd();
+		pgd_base = (unsigned long)__va(get_pgd());
 		pgd = (pgd_t *)pgd_base + offset;
 		pgd_k = init_mm.pgd + offset;
 
@@ -107,6 +107,8 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long write,
 		return;
 	}
 #endif
+
+	perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
 	/*
 	 * If we're in an interrupt or have no user
 	 * context, we must not take the fault..
@@ -154,10 +156,15 @@ good_area:
 			goto bad_area;
 		BUG();
 	}
-	if (fault & VM_FAULT_MAJOR)
+	if (fault & VM_FAULT_MAJOR) {
 		tsk->maj_flt++;
-	else
+		perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs,
+			      address);
+	} else {
 		tsk->min_flt++;
+		perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs,
+			      address);
+	}
 
 	up_read(&mm->mmap_sem);
 	return;
diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig
index b9c48b27162d..ff8cff9fcf54 100644
--- a/arch/mips/Kconfig
+++ b/arch/mips/Kconfig
@@ -44,8 +44,7 @@ config MIPS
 	select HAVE_ARCH_SECCOMP_FILTER
 	select HAVE_ARCH_TRACEHOOK
 	select HAVE_ARCH_TRANSPARENT_HUGEPAGE if CPU_SUPPORTS_HUGEPAGES && 64BIT
-	select HAVE_CBPF_JIT if (!64BIT && !CPU_MICROMIPS)
-	select HAVE_EBPF_JIT if (64BIT && !CPU_MICROMIPS)
+	select HAVE_EBPF_JIT if (!CPU_MICROMIPS)
 	select HAVE_CONTEXT_TRACKING
 	select HAVE_COPY_THREAD_TLS
 	select HAVE_C_RECORDMCOUNT
@@ -276,7 +275,7 @@ config BCM47XX
 	select BCM47XX_SPROM
 	select BCM47XX_SSB if !BCM47XX_BCMA
 	help
-	 Support for BCM47XX based boards
+	  Support for BCM47XX based boards
 
 config BCM63XX
 	bool "Broadcom BCM63XX based boards"
@@ -295,7 +294,7 @@ config BCM63XX
 	select MIPS_L1_CACHE_SHIFT_4
 	select CLKDEV_LOOKUP
 	help
-	 Support for BCM63XX based boards
+	  Support for BCM63XX based boards
 
 config MIPS_COBALT
 	bool "Cobalt Server"
@@ -374,10 +373,10 @@ config MACH_JAZZ
 	select SYS_SUPPORTS_64BIT_KERNEL
 	select SYS_SUPPORTS_100HZ
 	help
-	 This a family of machines based on the MIPS R4030 chipset which was
-	 used by several vendors to build RISC/os and Windows NT workstations.
-	 Members include the Acer PICA, MIPS Magnum 4000, MIPS Millennium and
-	 Olivetti M700-10 workstations.
+	  This a family of machines based on the MIPS R4030 chipset which was
+	  used by several vendors to build RISC/os and Windows NT workstations.
+	  Members include the Acer PICA, MIPS Magnum 4000, MIPS Millennium and
+	  Olivetti M700-10 workstations.
 
 config MACH_INGENIC
 	bool "Ingenic SoC based machines"
@@ -573,14 +572,14 @@ config NXP_STB220
 	bool "NXP STB220 board"
 	select SOC_PNX833X
 	help
-	 Support for NXP Semiconductors STB220 Development Board.
+	  Support for NXP Semiconductors STB220 Development Board.
 
 config NXP_STB225
 	bool "NXP 225 board"
 	select SOC_PNX833X
 	select SOC_PNX8335
 	help
-	 Support for NXP Semiconductors STB225 Development Board.
+	  Support for NXP Semiconductors STB225 Development Board.
 
 config PMC_MSP
 	bool "PMC-Sierra MSP chipsets"
@@ -722,9 +721,9 @@ config SGI_IP28
 	select SYS_SUPPORTS_64BIT_KERNEL
 	select SYS_SUPPORTS_BIG_ENDIAN
 	select MIPS_L1_CACHE_SHIFT_7
-      help
-        This is the SGI Indigo2 with R10000 processor.  To compile a Linux
-        kernel that runs on these, say Y here.
+	help
+	  This is the SGI Indigo2 with R10000 processor.  To compile a Linux
+	  kernel that runs on these, say Y here.
 
 config SGI_IP32
 	bool "SGI IP32 (O2)"
@@ -1168,9 +1167,9 @@ config HOLES_IN_ZONE
 config SYS_SUPPORTS_RELOCATABLE
 	bool
 	help
-	 Selected if the platform supports relocating the kernel.
-	 The platform must provide plat_get_fdt() if it selects CONFIG_USE_OF
-	 to allow access to command line and entropy sources.
+	  Selected if the platform supports relocating the kernel.
+	  The platform must provide plat_get_fdt() if it selects CONFIG_USE_OF
+	  to allow access to command line and entropy sources.
 
 config MIPS_CBPF_JIT
 	def_bool y
@@ -2113,8 +2112,8 @@ config MIPS_PGD_C0_CONTEXT
 # Set to y for ptrace access to watch registers.
 #
 config HARDWARE_WATCHPOINTS
-       bool
-       default y if CPU_MIPSR1 || CPU_MIPSR2 || CPU_MIPSR6
+	bool
+	default y if CPU_MIPSR1 || CPU_MIPSR2 || CPU_MIPSR6
 
 menu "Kernel type"
 
@@ -2178,10 +2177,10 @@ config PAGE_SIZE_4KB
 	bool "4kB"
 	depends on !CPU_LOONGSON2 && !CPU_LOONGSON3
 	help
-	 This option select the standard 4kB Linux page size.  On some
-	 R3000-family processors this is the only available page size.  Using
-	 4kB page size will minimize memory consumption and is therefore
-	 recommended for low memory systems.
+	  This option select the standard 4kB Linux page size.  On some
+	  R3000-family processors this is the only available page size.  Using
+	  4kB page size will minimize memory consumption and is therefore
+	  recommended for low memory systems.
 
 config PAGE_SIZE_8KB
 	bool "8kB"
@@ -2474,7 +2473,6 @@ config SB1_PASS_2_1_WORKAROUNDS
 	depends on CPU_SB1 && CPU_SB1_PASS_2
 	default y
 
-
 choice
 	prompt "SmartMIPS or microMIPS ASE support"
 
@@ -2682,16 +2680,16 @@ config RANDOMIZE_BASE
 	bool "Randomize the address of the kernel image"
 	depends on RELOCATABLE
 	---help---
-	   Randomizes the physical and virtual address at which the
-	   kernel image is loaded, as a security feature that
-	   deters exploit attempts relying on knowledge of the location
-	   of kernel internals.
+	  Randomizes the physical and virtual address at which the
+	  kernel image is loaded, as a security feature that
+	  deters exploit attempts relying on knowledge of the location
+	  of kernel internals.
 
-	   Entropy is generated using any coprocessor 0 registers available.
+	  Entropy is generated using any coprocessor 0 registers available.
 
-	   The kernel will be offset by up to RANDOMIZE_BASE_MAX_OFFSET.
+	  The kernel will be offset by up to RANDOMIZE_BASE_MAX_OFFSET.
 
-	   If unsure, say N.
+	  If unsure, say N.
 
 config RANDOMIZE_BASE_MAX_OFFSET
 	hex "Maximum kASLR offset" if EXPERT
@@ -2821,7 +2819,7 @@ choice
 	prompt "Timer frequency"
 	default HZ_250
 	help
-	 Allows the configuration of the timer frequency.
+	  Allows the configuration of the timer frequency.
 
 	config HZ_24
 		bool "24 HZ" if SYS_SUPPORTS_24HZ || SYS_SUPPORTS_ARBIT_HZ
@@ -3121,10 +3119,10 @@ config ARCH_MMAP_RND_BITS_MAX
 	default 15
 
 config ARCH_MMAP_RND_COMPAT_BITS_MIN
-       default 8
+	default 8
 
 config ARCH_MMAP_RND_COMPAT_BITS_MAX
-       default 15
+	default 15
 
 config I8253
 	bool
diff --git a/arch/mips/alchemy/common/clock.c b/arch/mips/alchemy/common/clock.c
index d129475fd40d..a95a894aceaf 100644
--- a/arch/mips/alchemy/common/clock.c
+++ b/arch/mips/alchemy/common/clock.c
@@ -160,7 +160,7 @@ static struct clk __init *alchemy_clk_setup_cpu(const char *parent_name,
 	id.name = ALCHEMY_CPU_CLK;
 	id.parent_names = &parent_name;
 	id.num_parents = 1;
-	id.flags = CLK_IS_BASIC;
+	id.flags = 0;
 	id.ops = &alchemy_clkops_cpu;
 	h->init = &id;
 
diff --git a/arch/mips/bcm47xx/Kconfig b/arch/mips/bcm47xx/Kconfig
index 29471038d817..6889f74e06f5 100644
--- a/arch/mips/bcm47xx/Kconfig
+++ b/arch/mips/bcm47xx/Kconfig
@@ -15,9 +15,9 @@ config BCM47XX_SSB
 	select SSB_DRIVER_GPIO
 	default y
 	help
-	 Add support for old Broadcom BCM47xx boards with Sonics Silicon Backplane support.
+	  Add support for old Broadcom BCM47xx boards with Sonics Silicon Backplane support.
 
-	 This will generate an image with support for SSB and MIPS32 R1 instruction set.
+	  This will generate an image with support for SSB and MIPS32 R1 instruction set.
 
 config BCM47XX_BCMA
 	bool "BCMA Support for Broadcom BCM47XX"
@@ -31,8 +31,8 @@ config BCM47XX_BCMA
 	select BCMA_DRIVER_GPIO
 	default y
 	help
-	 Add support for new Broadcom BCM47xx boards with Broadcom specific Advanced Microcontroller Bus.
+	  Add support for new Broadcom BCM47xx boards with Broadcom specific Advanced Microcontroller Bus.
 
-	 This will generate an image with support for BCMA and MIPS32 R2 instruction set.
+	  This will generate an image with support for BCMA and MIPS32 R2 instruction set.
 
 endif
diff --git a/arch/mips/bcm63xx/boards/Kconfig b/arch/mips/bcm63xx/boards/Kconfig
index f60d96610ace..492c3bd005d5 100644
--- a/arch/mips/bcm63xx/boards/Kconfig
+++ b/arch/mips/bcm63xx/boards/Kconfig
@@ -5,7 +5,7 @@ choice
 	default BOARD_BCM963XX
 
 config BOARD_BCM963XX
-       bool "Generic Broadcom 963xx boards"
+	bool "Generic Broadcom 963xx boards"
 	select SSB
 
 endchoice
diff --git a/arch/mips/configs/generic_defconfig b/arch/mips/configs/generic_defconfig
index 5d80521e5d5a..714169e411cf 100644
--- a/arch/mips/configs/generic_defconfig
+++ b/arch/mips/configs/generic_defconfig
@@ -26,6 +26,7 @@ CONFIG_MIPS_CPS=y
 CONFIG_HIGHMEM=y
 CONFIG_NR_CPUS=16
 CONFIG_MIPS_O32_FP64_SUPPORT=y
+CONFIG_JUMP_LABEL=y
 CONFIG_MODULES=y
 CONFIG_MODULE_UNLOAD=y
 CONFIG_TRIM_UNUSED_KSYMS=y
diff --git a/arch/mips/include/asm/bootinfo.h b/arch/mips/include/asm/bootinfo.h
index a301a8f4bc66..235bc2f52113 100644
--- a/arch/mips/include/asm/bootinfo.h
+++ b/arch/mips/include/asm/bootinfo.h
@@ -92,6 +92,7 @@ extern unsigned long mips_machtype;
 #define BOOT_MEM_ROM_DATA	2
 #define BOOT_MEM_RESERVED	3
 #define BOOT_MEM_INIT_RAM	4
+#define BOOT_MEM_NOMAP		5
 
 /*
  * A memory map that's built upon what was determined
diff --git a/arch/mips/include/asm/jump_label.h b/arch/mips/include/asm/jump_label.h
index e4456e450f94..3185fd3220ec 100644
--- a/arch/mips/include/asm/jump_label.h
+++ b/arch/mips/include/asm/jump_label.h
@@ -11,6 +11,7 @@
 #ifndef __ASSEMBLY__
 
 #include <linux/types.h>
+#include <asm/isa-rev.h>
 
 #define JUMP_LABEL_NOP_SIZE 4
 
@@ -21,15 +22,20 @@
 #endif
 
 #ifdef CONFIG_CPU_MICROMIPS
-#define B_INSN "b32"
+# define B_INSN "b32"
+# define J_INSN "j32"
+#elif MIPS_ISA_REV >= 6
+# define B_INSN "bc"
+# define J_INSN "bc"
 #else
-#define B_INSN "b"
+# define B_INSN "b"
+# define J_INSN "j"
 #endif
 
 static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
 {
 	asm_volatile_goto("1:\t" B_INSN " 2f\n\t"
-		"2:\tnop\n\t"
+		"2:\t.insn\n\t"
 		".pushsection __jump_table,  \"aw\"\n\t"
 		WORD_INSN " 1b, %l[l_yes], %0\n\t"
 		".popsection\n\t"
@@ -42,8 +48,7 @@ l_yes:
 
 static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
 {
-	asm_volatile_goto("1:\tj %l[l_yes]\n\t"
-		"nop\n\t"
+	asm_volatile_goto("1:\t" J_INSN " %l[l_yes]\n\t"
 		".pushsection __jump_table,  \"aw\"\n\t"
 		WORD_INSN " 1b, %l[l_yes], %0\n\t"
 		".popsection\n\t"
diff --git a/arch/mips/include/asm/uasm.h b/arch/mips/include/asm/uasm.h
index b1990dd75f27..f7effca791a5 100644
--- a/arch/mips/include/asm/uasm.h
+++ b/arch/mips/include/asm/uasm.h
@@ -86,14 +86,18 @@ Ip_u2u1(_ctcmsa);
 Ip_u2u1s3(_daddiu);
 Ip_u3u1u2(_daddu);
 Ip_u1u2(_ddivu);
+Ip_u3u1u2(_ddivu_r6);
 Ip_u1(_di);
 Ip_u2u1msbu3(_dins);
 Ip_u2u1msbu3(_dinsm);
 Ip_u2u1msbu3(_dinsu);
 Ip_u1u2(_divu);
+Ip_u3u1u2(_divu_r6);
 Ip_u1u2u3(_dmfc0);
+Ip_u3u1u2(_dmodu);
 Ip_u1u2u3(_dmtc0);
 Ip_u1u2(_dmultu);
+Ip_u3u1u2(_dmulu);
 Ip_u2u1u3(_drotr);
 Ip_u2u1u3(_drotr32);
 Ip_u2u1(_dsbh);
@@ -131,6 +135,7 @@ Ip_u1u2u3(_mfc0);
 Ip_u1u2u3(_mfhc0);
 Ip_u1(_mfhi);
 Ip_u1(_mflo);
+Ip_u3u1u2(_modu);
 Ip_u3u1u2(_movn);
 Ip_u3u1u2(_movz);
 Ip_u1u2u3(_mtc0);
@@ -139,6 +144,7 @@ Ip_u1(_mthi);
 Ip_u1(_mtlo);
 Ip_u3u1u2(_mul);
 Ip_u1u2(_multu);
+Ip_u3u1u2(_mulu);
 Ip_u3u1u2(_nor);
 Ip_u3u1u2(_or);
 Ip_u2u1u3(_ori);
@@ -149,6 +155,8 @@ Ip_u2s3u1(_sb);
 Ip_u2s3u1(_sc);
 Ip_u2s3u1(_scd);
 Ip_u2s3u1(_sd);
+Ip_u3u1u2(_seleqz);
+Ip_u3u1u2(_selnez);
 Ip_u2s3u1(_sh);
 Ip_u2u1u3(_sll);
 Ip_u3u2u1(_sllv);
diff --git a/arch/mips/include/uapi/asm/inst.h b/arch/mips/include/uapi/asm/inst.h
index 40fbb5dd66df..eaa3a80affdf 100644
--- a/arch/mips/include/uapi/asm/inst.h
+++ b/arch/mips/include/uapi/asm/inst.h
@@ -55,9 +55,9 @@ enum spec_op {
 	spec3_unused_op, spec4_unused_op, slt_op, sltu_op,
 	dadd_op, daddu_op, dsub_op, dsubu_op,
 	tge_op, tgeu_op, tlt_op, tltu_op,
-	teq_op, spec5_unused_op, tne_op, spec6_unused_op,
-	dsll_op, spec7_unused_op, dsrl_op, dsra_op,
-	dsll32_op, spec8_unused_op, dsrl32_op, dsra32_op
+	teq_op, seleqz_op, tne_op, selnez_op,
+	dsll_op, spec5_unused_op, dsrl_op, dsra_op,
+	dsll32_op, spec6_unused_op, dsrl32_op, dsra32_op
 };
 
 /*
diff --git a/arch/mips/kernel/entry.S b/arch/mips/kernel/entry.S
index d7de8adcfcc8..5469d43b6966 100644
--- a/arch/mips/kernel/entry.S
+++ b/arch/mips/kernel/entry.S
@@ -58,15 +58,14 @@ resume_kernel:
 	local_irq_disable
 	lw	t0, TI_PRE_COUNT($28)
 	bnez	t0, restore_all
-need_resched:
 	LONG_L	t0, TI_FLAGS($28)
 	andi	t1, t0, _TIF_NEED_RESCHED
 	beqz	t1, restore_all
 	LONG_L	t0, PT_STATUS(sp)		# Interrupts off?
 	andi	t0, 1
 	beqz	t0, restore_all
-	jal	preempt_schedule_irq
-	b	need_resched
+	PTR_LA	ra, restore_all
+	j	preempt_schedule_irq
 #endif
 
 FEXPORT(ret_from_kernel_thread)
diff --git a/arch/mips/kernel/jump_label.c b/arch/mips/kernel/jump_label.c
index ab943927f97a..662c8db9f45b 100644
--- a/arch/mips/kernel/jump_label.c
+++ b/arch/mips/kernel/jump_label.c
@@ -40,18 +40,38 @@ void arch_jump_label_transform(struct jump_entry *e,
 {
 	union mips_instruction *insn_p;
 	union mips_instruction insn;
+	long offset;
 
 	insn_p = (union mips_instruction *)msk_isa16_mode(e->code);
 
-	/* Jump only works within an aligned region its delay slot is in. */
-	BUG_ON((e->target & ~J_RANGE_MASK) != ((e->code + 4) & ~J_RANGE_MASK));
-
 	/* Target must have the right alignment and ISA must be preserved. */
 	BUG_ON((e->target & J_ALIGN_MASK) != J_ISA_BIT);
 
 	if (type == JUMP_LABEL_JMP) {
-		insn.j_format.opcode = J_ISA_BIT ? mm_j32_op : j_op;
-		insn.j_format.target = e->target >> J_RANGE_SHIFT;
+		if (!IS_ENABLED(CONFIG_CPU_MICROMIPS) && MIPS_ISA_REV >= 6) {
+			offset = e->target - ((unsigned long)insn_p + 4);
+			offset >>= 2;
+
+			/*
+			 * The branch offset must fit in the instruction's 26
+			 * bit field.
+			 */
+			WARN_ON((offset >= BIT(25)) ||
+				(offset < -(long)BIT(25)));
+
+			insn.j_format.opcode = bc6_op;
+			insn.j_format.target = offset;
+		} else {
+			/*
+			 * Jump only works within an aligned region its delay
+			 * slot is in.
+			 */
+			WARN_ON((e->target & ~J_RANGE_MASK) !=
+				((e->code + 4) & ~J_RANGE_MASK));
+
+			insn.j_format.opcode = J_ISA_BIT ? mm_j32_op : j_op;
+			insn.j_format.target = e->target >> J_RANGE_SHIFT;
+		}
 	} else {
 		insn.word = 0; /* nop */
 	}
diff --git a/arch/mips/kernel/prom.c b/arch/mips/kernel/prom.c
index 93b8e0b4332f..28bf01961bb2 100644
--- a/arch/mips/kernel/prom.c
+++ b/arch/mips/kernel/prom.c
@@ -41,13 +41,27 @@ char *mips_get_machine_name(void)
 #ifdef CONFIG_USE_OF
 void __init early_init_dt_add_memory_arch(u64 base, u64 size)
 {
-	return add_memory_region(base, size, BOOT_MEM_RAM);
+	if (base >= PHYS_ADDR_MAX) {
+		pr_warn("Trying to add an invalid memory region, skipped\n");
+		return;
+	}
+
+	/* Truncate the passed memory region instead of type casting */
+	if (base + size - 1 >= PHYS_ADDR_MAX || base + size < base) {
+		pr_warn("Truncate memory region %llx @ %llx to size %llx\n",
+			size, base, PHYS_ADDR_MAX - base);
+		size = PHYS_ADDR_MAX - base;
+	}
+
+	add_memory_region(base, size, BOOT_MEM_RAM);
 }
 
 int __init early_init_dt_reserve_memory_arch(phys_addr_t base,
 					phys_addr_t size, bool nomap)
 {
-	add_memory_region(base, size, BOOT_MEM_RESERVED);
+	add_memory_region(base, size,
+			  nomap ? BOOT_MEM_NOMAP : BOOT_MEM_RESERVED);
+
 	return 0;
 }
 
diff --git a/arch/mips/kernel/setup.c b/arch/mips/kernel/setup.c
index 8d1dc6c71173..ab349d2381c3 100644
--- a/arch/mips/kernel/setup.c
+++ b/arch/mips/kernel/setup.c
@@ -27,6 +27,7 @@
 #include <linux/dma-contiguous.h>
 #include <linux/decompress/generic.h>
 #include <linux/of_fdt.h>
+#include <linux/of_reserved_mem.h>
 
 #include <asm/addrspace.h>
 #include <asm/bootinfo.h>
@@ -178,6 +179,7 @@ static bool __init __maybe_unused memory_region_available(phys_addr_t start,
 				in_ram = true;
 			break;
 		case BOOT_MEM_RESERVED:
+		case BOOT_MEM_NOMAP:
 			if ((start >= start_ && start < end_) ||
 			    (start < start_ && start + size >= start_))
 				free = false;
@@ -213,6 +215,9 @@ static void __init print_memory_map(void)
 		case BOOT_MEM_RESERVED:
 			printk(KERN_CONT "(reserved)\n");
 			break;
+		case BOOT_MEM_NOMAP:
+			printk(KERN_CONT "(nomap)\n");
+			break;
 		default:
 			printk(KERN_CONT "type %lu\n", boot_mem_map.map[i].type);
 			break;
@@ -371,7 +376,6 @@ static void __init bootmem_init(void)
 
 static void __init bootmem_init(void)
 {
-	unsigned long reserved_end;
 	phys_addr_t ramstart = PHYS_ADDR_MAX;
 	int i;
 
@@ -382,10 +386,10 @@ static void __init bootmem_init(void)
 	 * will reserve the area used for the initrd.
 	 */
 	init_initrd();
-	reserved_end = (unsigned long) PFN_UP(__pa_symbol(&_end));
 
-	memblock_reserve(PHYS_OFFSET,
-			 (reserved_end << PAGE_SHIFT) - PHYS_OFFSET);
+	/* Reserve memory occupied by kernel. */
+	memblock_reserve(__pa_symbol(&_text),
+			__pa_symbol(&_end) - __pa_symbol(&_text));
 
 	/*
 	 * max_low_pfn is not a number of pages. The number of pages
@@ -394,10 +398,7 @@ static void __init bootmem_init(void)
 	min_low_pfn = ~0UL;
 	max_low_pfn = 0;
 
-	/*
-	 * Find the highest page frame number we have available
-	 * and the lowest used RAM address
-	 */
+	/* Find the highest and lowest page frame numbers we have available. */
 	for (i = 0; i < boot_mem_map.nr_map; i++) {
 		unsigned long start, end;
 
@@ -427,13 +428,6 @@ static void __init bootmem_init(void)
 			max_low_pfn = end;
 		if (start < min_low_pfn)
 			min_low_pfn = start;
-		if (end <= reserved_end)
-			continue;
-#ifdef CONFIG_BLK_DEV_INITRD
-		/* Skip zones before initrd and initrd itself */
-		if (initrd_end && end <= (unsigned long)PFN_UP(__pa(initrd_end)))
-			continue;
-#endif
 	}
 
 	if (min_low_pfn >= max_low_pfn)
@@ -474,6 +468,7 @@ static void __init bootmem_init(void)
 		max_low_pfn = PFN_DOWN(HIGHMEM_START);
 	}
 
+	/* Install all valid RAM ranges to the memblock memory region */
 	for (i = 0; i < boot_mem_map.nr_map; i++) {
 		unsigned long start, end;
 
@@ -481,98 +476,38 @@ static void __init bootmem_init(void)
 		end = PFN_DOWN(boot_mem_map.map[i].addr
 				+ boot_mem_map.map[i].size);
 
-		if (start <= min_low_pfn)
+		if (start < min_low_pfn)
 			start = min_low_pfn;
-		if (start >= end)
-			continue;
-
 #ifndef CONFIG_HIGHMEM
+		/* Ignore highmem regions if highmem is unsupported */
 		if (end > max_low_pfn)
 			end = max_low_pfn;
-
-		/*
-		 * ... finally, is the area going away?
-		 */
+#endif
 		if (end <= start)
 			continue;
-#endif
 
 		memblock_add_node(PFN_PHYS(start), PFN_PHYS(end - start), 0);
-	}
-
-	/*
-	 * Register fully available low RAM pages with the bootmem allocator.
-	 */
-	for (i = 0; i < boot_mem_map.nr_map; i++) {
-		unsigned long start, end, size;
 
-		start = PFN_UP(boot_mem_map.map[i].addr);
-		end   = PFN_DOWN(boot_mem_map.map[i].addr
-				    + boot_mem_map.map[i].size);
-
-		/*
-		 * Reserve usable memory.
-		 */
+		/* Reserve any memory except the ordinary RAM ranges. */
 		switch (boot_mem_map.map[i].type) {
 		case BOOT_MEM_RAM:
 			break;
-		case BOOT_MEM_INIT_RAM:
-			memory_present(0, start, end);
-			continue;
-		default:
-			/* Not usable memory */
-			if (start > min_low_pfn && end < max_low_pfn)
-				memblock_reserve(boot_mem_map.map[i].addr,
-						boot_mem_map.map[i].size);
-
+		case BOOT_MEM_NOMAP: /* Discard the range from the system. */
+			memblock_remove(PFN_PHYS(start), PFN_PHYS(end - start));
 			continue;
+		default: /* Reserve the rest of the memory types at boot time */
+			memblock_reserve(PFN_PHYS(start), PFN_PHYS(end - start));
+			break;
 		}
 
 		/*
-		 * We are rounding up the start address of usable memory
-		 * and at the end of the usable range downwards.
+		 * In any case the added to the memblock memory regions
+		 * (highmem/lowmem, available/reserved, etc) are considered
+		 * as present, so inform sparsemem about them.
 		 */
-		if (start >= max_low_pfn)
-			continue;
-		if (start < reserved_end)
-			start = reserved_end;
-		if (end > max_low_pfn)
-			end = max_low_pfn;
-
-		/*
-		 * ... finally, is the area going away?
-		 */
-		if (end <= start)
-			continue;
-		size = end - start;
-
-		/* Register lowmem ranges */
 		memory_present(0, start, end);
 	}
 
-#ifdef CONFIG_RELOCATABLE
-	/*
-	 * The kernel reserves all memory below its _end symbol as bootmem,
-	 * but the kernel may now be at a much higher address. The memory
-	 * between the original and new locations may be returned to the system.
-	 */
-	if (__pa_symbol(_text) > __pa_symbol(VMLINUX_LOAD_ADDRESS)) {
-		unsigned long offset;
-		extern void show_kernel_relocation(const char *level);
-
-		offset = __pa_symbol(_text) - __pa_symbol(VMLINUX_LOAD_ADDRESS);
-		memblock_free(__pa_symbol(VMLINUX_LOAD_ADDRESS), offset);
-
-#if defined(CONFIG_DEBUG_KERNEL) && defined(CONFIG_DEBUG_INFO)
-		/*
-		 * This information is necessary when debugging the kernel
-		 * But is a security vulnerability otherwise!
-		 */
-		show_kernel_relocation(KERN_INFO);
-#endif
-	}
-#endif
-
 	/*
 	 * Reserve initrd memory if needed.
 	 */
@@ -781,7 +716,6 @@ static void __init request_crashkernel(struct resource *res)
  */
 static void __init arch_mem_init(char **cmdline_p)
 {
-	struct memblock_region *reg;
 	extern void plat_mem_setup(void);
 
 	/*
@@ -809,6 +743,9 @@ static void __init arch_mem_init(char **cmdline_p)
 	arch_mem_addpart(PFN_UP(__pa_symbol(&__init_begin)) << PAGE_SHIFT,
 			 PFN_DOWN(__pa_symbol(&__init_end)) << PAGE_SHIFT,
 			 BOOT_MEM_INIT_RAM);
+	arch_mem_addpart(PFN_DOWN(__pa_symbol(&__bss_start)) << PAGE_SHIFT,
+			 PFN_UP(__pa_symbol(&__bss_stop)) << PAGE_SHIFT,
+			 BOOT_MEM_RAM);
 
 	pr_info("Determined physical RAM map:\n");
 	print_memory_map();
@@ -884,13 +821,16 @@ static void __init arch_mem_init(char **cmdline_p)
 	plat_swiotlb_setup();
 
 	dma_contiguous_reserve(PFN_PHYS(max_low_pfn));
-	/* Tell bootmem about cma reserved memblock section */
-	for_each_memblock(reserved, reg)
-		if (reg->size != 0)
-			memblock_reserve(reg->base, reg->size);
 
-	reserve_bootmem_region(__pa_symbol(&__nosave_begin),
-			__pa_symbol(&__nosave_end)); /* Reserve for hibernation */
+	/* Reserve for hibernation. */
+	memblock_reserve(__pa_symbol(&__nosave_begin),
+		__pa_symbol(&__nosave_end) - __pa_symbol(&__nosave_begin));
+
+	fdt_init_reserved_mem();
+
+	memblock_dump_all();
+
+	early_memtest(PFN_PHYS(min_low_pfn), PFN_PHYS(max_low_pfn));
 }
 
 static void __init resource_init(void)
@@ -935,6 +875,7 @@ static void __init resource_init(void)
 			res->flags |= IORESOURCE_SYSRAM;
 			break;
 		case BOOT_MEM_RESERVED:
+		case BOOT_MEM_NOMAP:
 		default:
 			res->name = "reserved";
 		}
diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c
index 98ca55d62201..c52766a5b85f 100644
--- a/arch/mips/kernel/traps.c
+++ b/arch/mips/kernel/traps.c
@@ -2151,7 +2151,7 @@ static void configure_hwrena(void)
 
 static void configure_exception_vector(void)
 {
-	if (cpu_has_veic || cpu_has_vint) {
+	if (cpu_has_mips_r2_r6) {
 		unsigned long sr = set_c0_status(ST0_BEV);
 		/* If available, use WG to set top bits of EBASE */
 		if (cpu_has_ebase_wg) {
@@ -2163,6 +2163,8 @@ static void configure_exception_vector(void)
 		}
 		write_c0_ebase(ebase);
 		write_c0_status(sr);
+	}
+	if (cpu_has_veic || cpu_has_vint) {
 		/* Setting vector spacing enables EI/VI mode  */
 		change_c0_intctl(0x3e0, VECTORSPACING);
 	}
@@ -2193,22 +2195,6 @@ void per_cpu_trap_init(bool is_boot_cpu)
 	 *  o read IntCtl.IPFDC to determine the fast debug channel interrupt
 	 */
 	if (cpu_has_mips_r2_r6) {
-		/*
-		 * We shouldn't trust a secondary core has a sane EBASE register
-		 * so use the one calculated by the boot CPU.
-		 */
-		if (!is_boot_cpu) {
-			/* If available, use WG to set top bits of EBASE */
-			if (cpu_has_ebase_wg) {
-#ifdef CONFIG_64BIT
-				write_c0_ebase_64(ebase | MIPS_EBASE_WG);
-#else
-				write_c0_ebase(ebase | MIPS_EBASE_WG);
-#endif
-			}
-			write_c0_ebase(ebase);
-		}
-
 		cp0_compare_irq_shift = CAUSEB_TI - CAUSEB_IP;
 		cp0_compare_irq = (read_c0_intctl() >> INTCTLB_IPTI) & 7;
 		cp0_perfcount_irq = (read_c0_intctl() >> INTCTLB_IPPCI) & 7;
@@ -2284,19 +2270,27 @@ void __init trap_init(void)
 	extern char except_vec3_generic;
 	extern char except_vec4;
 	extern char except_vec3_r4000;
-	unsigned long i;
+	unsigned long i, vec_size;
+	phys_addr_t ebase_pa;
 
 	check_wait();
 
-	if (cpu_has_veic || cpu_has_vint) {
-		unsigned long size = 0x200 + VECTORSPACING*64;
-		phys_addr_t ebase_pa;
+	if (!cpu_has_mips_r2_r6) {
+		ebase = CAC_BASE;
+		ebase_pa = virt_to_phys((void *)ebase);
+		vec_size = 0x400;
 
-		ebase = (unsigned long)
-			memblock_alloc(size, 1 << fls(size));
-		if (!ebase)
+		memblock_reserve(ebase_pa, vec_size);
+	} else {
+		if (cpu_has_veic || cpu_has_vint)
+			vec_size = 0x200 + VECTORSPACING*64;
+		else
+			vec_size = PAGE_SIZE;
+
+		ebase_pa = memblock_phys_alloc(vec_size, 1 << fls(vec_size));
+		if (!ebase_pa)
 			panic("%s: Failed to allocate %lu bytes align=0x%x\n",
-			      __func__, size, 1 << fls(size));
+			      __func__, vec_size, 1 << fls(vec_size));
 
 		/*
 		 * Try to ensure ebase resides in KSeg0 if possible.
@@ -2309,23 +2303,10 @@ void __init trap_init(void)
 		 * EVA is special though as it allows segments to be rearranged
 		 * and to become uncached during cache error handling.
 		 */
-		ebase_pa = __pa(ebase);
 		if (!IS_ENABLED(CONFIG_EVA) && !WARN_ON(ebase_pa >= 0x20000000))
 			ebase = CKSEG0ADDR(ebase_pa);
-	} else {
-		ebase = CAC_BASE;
-
-		if (cpu_has_mips_r2_r6) {
-			if (cpu_has_ebase_wg) {
-#ifdef CONFIG_64BIT
-				ebase = (read_c0_ebase_64() & ~0xfff);
-#else
-				ebase = (read_c0_ebase() & ~0xfff);
-#endif
-			} else {
-				ebase += (read_c0_ebase() & 0x3ffff000);
-			}
-		}
+		else
+			ebase = (unsigned long)phys_to_virt(ebase_pa);
 	}
 
 	if (cpu_has_mmips) {
@@ -2459,7 +2440,7 @@ void __init trap_init(void)
 	else
 		set_handler(0x080, &except_vec3_generic, 0x80);
 
-	local_flush_icache_range(ebase, ebase + 0x400);
+	local_flush_icache_range(ebase, ebase + vec_size);
 
 	sort_extable(__start___dbe_table, __stop___dbe_table);
 
diff --git a/arch/mips/kvm/emulate.c b/arch/mips/kvm/emulate.c
index 0074427b04fb..e5de6bac8197 100644
--- a/arch/mips/kvm/emulate.c
+++ b/arch/mips/kvm/emulate.c
@@ -1141,9 +1141,7 @@ enum emulation_result kvm_mips_emul_tlbwr(struct kvm_vcpu *vcpu)
 	unsigned long pc = vcpu->arch.pc;
 	int index;
 
-	get_random_bytes(&index, sizeof(index));
-	index &= (KVM_MIPS_GUEST_TLB_SIZE - 1);
-
+	index = prandom_u32_max(KVM_MIPS_GUEST_TLB_SIZE);
 	tlb = &vcpu->arch.guest_tlb[index];
 
 	kvm_mips_invalidate_guest_tlb(vcpu, tlb);
diff --git a/arch/mips/mm/uasm-mips.c b/arch/mips/mm/uasm-mips.c
index 6abe40fc413d..7154a1d99aad 100644
--- a/arch/mips/mm/uasm-mips.c
+++ b/arch/mips/mm/uasm-mips.c
@@ -76,14 +76,22 @@ static const struct insn insn_table[insn_invalid] = {
 	[insn_daddiu]	= {M(daddiu_op, 0, 0, 0, 0, 0), RS | RT | SIMM},
 	[insn_daddu]	= {M(spec_op, 0, 0, 0, 0, daddu_op), RS | RT | RD},
 	[insn_ddivu]	= {M(spec_op, 0, 0, 0, 0, ddivu_op), RS | RT},
+	[insn_ddivu_r6]	= {M(spec_op, 0, 0, 0, ddivu_ddivu6_op, ddivu_op),
+				RS | RT | RD},
 	[insn_di]	= {M(cop0_op, mfmc0_op, 0, 12, 0, 0), RT},
 	[insn_dins]	= {M(spec3_op, 0, 0, 0, 0, dins_op), RS | RT | RD | RE},
 	[insn_dinsm]	= {M(spec3_op, 0, 0, 0, 0, dinsm_op), RS | RT | RD | RE},
 	[insn_dinsu]	= {M(spec3_op, 0, 0, 0, 0, dinsu_op), RS | RT | RD | RE},
 	[insn_divu]	= {M(spec_op, 0, 0, 0, 0, divu_op), RS | RT},
+	[insn_divu_r6]	= {M(spec_op, 0, 0, 0, divu_divu6_op, divu_op),
+				RS | RT | RD},
 	[insn_dmfc0]	= {M(cop0_op, dmfc_op, 0, 0, 0, 0), RT | RD | SET},
+	[insn_dmodu]	= {M(spec_op, 0, 0, 0, ddivu_dmodu_op, ddivu_op),
+				RS | RT | RD},
 	[insn_dmtc0]	= {M(cop0_op, dmtc_op, 0, 0, 0, 0), RT | RD | SET},
 	[insn_dmultu]	= {M(spec_op, 0, 0, 0, 0, dmultu_op), RS | RT},
+	[insn_dmulu]	= {M(spec_op, 0, 0, 0, dmult_dmul_op, dmultu_op),
+				RS | RT | RD},
 	[insn_drotr]	= {M(spec_op, 1, 0, 0, 0, dsrl_op), RT | RD | RE},
 	[insn_drotr32]	= {M(spec_op, 1, 0, 0, 0, dsrl32_op), RT | RD | RE},
 	[insn_dsbh]	= {M(spec3_op, 0, 0, 0, dsbh_op, dbshfl_op), RT | RD},
@@ -132,12 +140,16 @@ static const struct insn insn_table[insn_invalid] = {
 	[insn_mfhc0]	= {M(cop0_op, mfhc0_op, 0, 0, 0, 0),  RT | RD | SET},
 	[insn_mfhi]	= {M(spec_op, 0, 0, 0, 0, mfhi_op), RD},
 	[insn_mflo]	= {M(spec_op, 0, 0, 0, 0, mflo_op), RD},
+	[insn_modu]	= {M(spec_op, 0, 0, 0, divu_modu_op, divu_op),
+				RS | RT | RD},
 	[insn_movn]	= {M(spec_op, 0, 0, 0, 0, movn_op), RS | RT | RD},
 	[insn_movz]	= {M(spec_op, 0, 0, 0, 0, movz_op), RS | RT | RD},
 	[insn_mtc0]	= {M(cop0_op, mtc_op, 0, 0, 0, 0),  RT | RD | SET},
 	[insn_mthc0]	= {M(cop0_op, mthc0_op, 0, 0, 0, 0),  RT | RD | SET},
 	[insn_mthi]	= {M(spec_op, 0, 0, 0, 0, mthi_op), RS},
 	[insn_mtlo]	= {M(spec_op, 0, 0, 0, 0, mtlo_op), RS},
+	[insn_mulu]	= {M(spec_op, 0, 0, 0, multu_mulu_op, multu_op),
+				RS | RT | RD},
 #ifndef CONFIG_CPU_MIPSR6
 	[insn_mul]	= {M(spec2_op, 0, 0, 0, 0, mul_op), RS | RT | RD},
 #else
@@ -163,6 +175,8 @@ static const struct insn insn_table[insn_invalid] = {
 	[insn_scd]	= {M6(spec3_op, 0, 0, 0, scd6_op),  RS | RT | SIMM9},
 #endif
 	[insn_sd]	= {M(sd_op, 0, 0, 0, 0, 0),  RS | RT | SIMM},
+	[insn_seleqz]	= {M(spec_op, 0, 0, 0, 0, seleqz_op), RS | RT | RD},
+	[insn_selnez]	= {M(spec_op, 0, 0, 0, 0, selnez_op), RS | RT | RD},
 	[insn_sh]	= {M(sh_op, 0, 0, 0, 0, 0),  RS | RT | SIMM},
 	[insn_sll]	= {M(spec_op, 0, 0, 0, 0, sll_op),  RT | RD | RE},
 	[insn_sllv]	= {M(spec_op, 0, 0, 0, 0, sllv_op),  RS | RT | RD},
diff --git a/arch/mips/mm/uasm.c b/arch/mips/mm/uasm.c
index 45b6264ff308..c56f129c9a4b 100644
--- a/arch/mips/mm/uasm.c
+++ b/arch/mips/mm/uasm.c
@@ -50,21 +50,22 @@ enum opcode {
 	insn_beq, insn_beql, insn_bgez, insn_bgezl, insn_bgtz, insn_blez,
 	insn_bltz, insn_bltzl, insn_bne, insn_break, insn_cache, insn_cfc1,
 	insn_cfcmsa, insn_ctc1, insn_ctcmsa, insn_daddiu, insn_daddu, insn_ddivu,
-	insn_di, insn_dins, insn_dinsm, insn_dinsu, insn_divu, insn_dmfc0,
-	insn_dmtc0, insn_dmultu, insn_drotr, insn_drotr32, insn_dsbh, insn_dshd,
-	insn_dsll, insn_dsll32, insn_dsllv, insn_dsra, insn_dsra32, insn_dsrav,
-	insn_dsrl, insn_dsrl32, insn_dsrlv, insn_dsubu, insn_eret, insn_ext,
-	insn_ins, insn_j, insn_jal, insn_jalr, insn_jr, insn_lb, insn_lbu,
-	insn_ld, insn_lddir, insn_ldpte, insn_ldx, insn_lh, insn_lhu,
-	insn_ll, insn_lld, insn_lui, insn_lw, insn_lwu, insn_lwx, insn_mfc0,
-	insn_mfhc0, insn_mfhi, insn_mflo, insn_movn, insn_movz, insn_mtc0,
-	insn_mthc0, insn_mthi, insn_mtlo, insn_mul, insn_multu, insn_nor,
-	insn_or, insn_ori, insn_pref, insn_rfe, insn_rotr, insn_sb,
-	insn_sc, insn_scd, insn_sd, insn_sh, insn_sll, insn_sllv,
-	insn_slt, insn_slti, insn_sltiu, insn_sltu, insn_sra, insn_srav,
-	insn_srl, insn_srlv, insn_subu, insn_sw, insn_sync, insn_syscall,
-	insn_tlbp, insn_tlbr, insn_tlbwi, insn_tlbwr, insn_wait, insn_wsbh,
-	insn_xor, insn_xori, insn_yield,
+	insn_ddivu_r6, insn_di, insn_dins, insn_dinsm, insn_dinsu, insn_divu,
+	insn_divu_r6, insn_dmfc0, insn_dmodu, insn_dmtc0, insn_dmultu,
+	insn_dmulu, insn_drotr, insn_drotr32, insn_dsbh, insn_dshd, insn_dsll,
+	insn_dsll32, insn_dsllv, insn_dsra, insn_dsra32, insn_dsrav, insn_dsrl,
+	insn_dsrl32, insn_dsrlv, insn_dsubu, insn_eret, insn_ext, insn_ins,
+	insn_j, insn_jal, insn_jalr, insn_jr, insn_lb, insn_lbu, insn_ld,
+	insn_lddir, insn_ldpte, insn_ldx, insn_lh, insn_lhu, insn_ll, insn_lld,
+	insn_lui, insn_lw, insn_lwu, insn_lwx, insn_mfc0, insn_mfhc0, insn_mfhi,
+	insn_mflo, insn_modu, insn_movn, insn_movz, insn_mtc0, insn_mthc0,
+	insn_mthi, insn_mtlo, insn_mul, insn_multu, insn_mulu, insn_nor,
+	insn_or, insn_ori, insn_pref, insn_rfe, insn_rotr, insn_sb, insn_sc,
+	insn_scd, insn_seleqz, insn_selnez, insn_sd, insn_sh, insn_sll,
+	insn_sllv, insn_slt, insn_slti, insn_sltiu, insn_sltu, insn_sra,
+	insn_srav, insn_srl, insn_srlv, insn_subu, insn_sw, insn_sync,
+	insn_syscall, insn_tlbp, insn_tlbr, insn_tlbwi, insn_tlbwr, insn_wait,
+	insn_wsbh, insn_xor, insn_xori, insn_yield,
 	insn_invalid /* insn_invalid must be last */
 };
 
@@ -287,13 +288,17 @@ I_u2u1(_cfcmsa)
 I_u1u2(_ctc1)
 I_u2u1(_ctcmsa)
 I_u1u2(_ddivu)
+I_u3u1u2(_ddivu_r6)
 I_u1u2u3(_dmfc0)
+I_u3u1u2(_dmodu)
 I_u1u2u3(_dmtc0)
 I_u1u2(_dmultu)
+I_u3u1u2(_dmulu)
 I_u2u1s3(_daddiu)
 I_u3u1u2(_daddu)
 I_u1(_di);
 I_u1u2(_divu)
+I_u3u1u2(_divu_r6)
 I_u2u1(_dsbh);
 I_u2u1(_dshd);
 I_u2u1u3(_dsll)
@@ -327,6 +332,7 @@ I_u2s3u1(_lw)
 I_u2s3u1(_lwu)
 I_u1u2u3(_mfc0)
 I_u1u2u3(_mfhc0)
+I_u3u1u2(_modu)
 I_u3u1u2(_movn)
 I_u3u1u2(_movz)
 I_u1(_mfhi)
@@ -337,6 +343,7 @@ I_u1(_mthi)
 I_u1(_mtlo)
 I_u3u1u2(_mul)
 I_u1u2(_multu)
+I_u3u1u2(_mulu)
 I_u3u1u2(_nor)
 I_u3u1u2(_or)
 I_u2u1u3(_ori)
@@ -345,6 +352,8 @@ I_u2s3u1(_sb)
 I_u2s3u1(_sc)
 I_u2s3u1(_scd)
 I_u2s3u1(_sd)
+I_u3u1u2(_seleqz)
+I_u3u1u2(_selnez)
 I_u2s3u1(_sh)
 I_u2u1u3(_sll)
 I_u3u2u1(_sllv)
diff --git a/arch/mips/net/Makefile b/arch/mips/net/Makefile
index 47d678416715..72a78462f872 100644
--- a/arch/mips/net/Makefile
+++ b/arch/mips/net/Makefile
@@ -1,4 +1,3 @@
 # MIPS networking code
 
-obj-$(CONFIG_MIPS_CBPF_JIT) += bpf_jit.o bpf_jit_asm.o
 obj-$(CONFIG_MIPS_EBPF_JIT) += ebpf_jit.o
diff --git a/arch/mips/net/bpf_jit.c b/arch/mips/net/bpf_jit.c
deleted file mode 100644
index 3a0e34f4e615..000000000000
--- a/arch/mips/net/bpf_jit.c
+++ /dev/null
@@ -1,1270 +0,0 @@
-/*
- * Just-In-Time compiler for BPF filters on MIPS
- *
- * Copyright (c) 2014 Imagination Technologies Ltd.
- * Author: Markos Chandras <markos.chandras@imgtec.com>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation; version 2 of the License.
- */
-
-#include <linux/bitops.h>
-#include <linux/compiler.h>
-#include <linux/errno.h>
-#include <linux/filter.h>
-#include <linux/if_vlan.h>
-#include <linux/moduleloader.h>
-#include <linux/netdevice.h>
-#include <linux/string.h>
-#include <linux/slab.h>
-#include <linux/types.h>
-#include <asm/asm.h>
-#include <asm/bitops.h>
-#include <asm/cacheflush.h>
-#include <asm/cpu-features.h>
-#include <asm/uasm.h>
-
-#include "bpf_jit.h"
-
-/* ABI
- * r_skb_hl	SKB header length
- * r_data	SKB data pointer
- * r_off	Offset
- * r_A		BPF register A
- * r_X		BPF register X
- * r_skb	*skb
- * r_M		*scratch memory
- * r_skb_len	SKB length
- *
- * On entry (*bpf_func)(*skb, *filter)
- * a0 = MIPS_R_A0 = skb;
- * a1 = MIPS_R_A1 = filter;
- *
- * Stack
- * ...
- * M[15]
- * M[14]
- * M[13]
- * ...
- * M[0] <-- r_M
- * saved reg k-1
- * saved reg k-2
- * ...
- * saved reg 0 <-- r_sp
- * <no argument area>
- *
- *                     Packet layout
- *
- * <--------------------- len ------------------------>
- * <--skb-len(r_skb_hl)-->< ----- skb->data_len ------>
- * ----------------------------------------------------
- * |                  skb->data                       |
- * ----------------------------------------------------
- */
-
-#define ptr typeof(unsigned long)
-
-#define SCRATCH_OFF(k)		(4 * (k))
-
-/* JIT flags */
-#define SEEN_CALL		(1 << BPF_MEMWORDS)
-#define SEEN_SREG_SFT		(BPF_MEMWORDS + 1)
-#define SEEN_SREG_BASE		(1 << SEEN_SREG_SFT)
-#define SEEN_SREG(x)		(SEEN_SREG_BASE << (x))
-#define SEEN_OFF		SEEN_SREG(2)
-#define SEEN_A			SEEN_SREG(3)
-#define SEEN_X			SEEN_SREG(4)
-#define SEEN_SKB		SEEN_SREG(5)
-#define SEEN_MEM		SEEN_SREG(6)
-/* SEEN_SK_DATA also implies skb_hl an skb_len */
-#define SEEN_SKB_DATA		(SEEN_SREG(7) | SEEN_SREG(1) | SEEN_SREG(0))
-
-/* Arguments used by JIT */
-#define ARGS_USED_BY_JIT	2 /* only applicable to 64-bit */
-
-#define SBIT(x)			(1 << (x)) /* Signed version of BIT() */
-
-/**
- * struct jit_ctx - JIT context
- * @skf:		The sk_filter
- * @prologue_bytes:	Number of bytes for prologue
- * @idx:		Instruction index
- * @flags:		JIT flags
- * @offsets:		Instruction offsets
- * @target:		Memory location for the compiled filter
- */
-struct jit_ctx {
-	const struct bpf_prog *skf;
-	unsigned int prologue_bytes;
-	u32 idx;
-	u32 flags;
-	u32 *offsets;
-	u32 *target;
-};
-
-
-static inline int optimize_div(u32 *k)
-{
-	/* power of 2 divides can be implemented with right shift */
-	if (!(*k & (*k-1))) {
-		*k = ilog2(*k);
-		return 1;
-	}
-
-	return 0;
-}
-
-static inline void emit_jit_reg_move(ptr dst, ptr src, struct jit_ctx *ctx);
-
-/* Simply emit the instruction if the JIT memory space has been allocated */
-#define emit_instr(ctx, func, ...)			\
-do {							\
-	if ((ctx)->target != NULL) {			\
-		u32 *p = &(ctx)->target[ctx->idx];	\
-		uasm_i_##func(&p, ##__VA_ARGS__);	\
-	}						\
-	(ctx)->idx++;					\
-} while (0)
-
-/*
- * Similar to emit_instr but it must be used when we need to emit
- * 32-bit or 64-bit instructions
- */
-#define emit_long_instr(ctx, func, ...)			\
-do {							\
-	if ((ctx)->target != NULL) {			\
-		u32 *p = &(ctx)->target[ctx->idx];	\
-		UASM_i_##func(&p, ##__VA_ARGS__);	\
-	}						\
-	(ctx)->idx++;					\
-} while (0)
-
-/* Determine if immediate is within the 16-bit signed range */
-static inline bool is_range16(s32 imm)
-{
-	return !(imm >= SBIT(15) || imm < -SBIT(15));
-}
-
-static inline void emit_addu(unsigned int dst, unsigned int src1,
-			     unsigned int src2, struct jit_ctx *ctx)
-{
-	emit_instr(ctx, addu, dst, src1, src2);
-}
-
-static inline void emit_nop(struct jit_ctx *ctx)
-{
-	emit_instr(ctx, nop);
-}
-
-/* Load a u32 immediate to a register */
-static inline void emit_load_imm(unsigned int dst, u32 imm, struct jit_ctx *ctx)
-{
-	if (ctx->target != NULL) {
-		/* addiu can only handle s16 */
-		if (!is_range16(imm)) {
-			u32 *p = &ctx->target[ctx->idx];
-			uasm_i_lui(&p, r_tmp_imm, (s32)imm >> 16);
-			p = &ctx->target[ctx->idx + 1];
-			uasm_i_ori(&p, dst, r_tmp_imm, imm & 0xffff);
-		} else {
-			u32 *p = &ctx->target[ctx->idx];
-			uasm_i_addiu(&p, dst, r_zero, imm);
-		}
-	}
-	ctx->idx++;
-
-	if (!is_range16(imm))
-		ctx->idx++;
-}
-
-static inline void emit_or(unsigned int dst, unsigned int src1,
-			   unsigned int src2, struct jit_ctx *ctx)
-{
-	emit_instr(ctx, or, dst, src1, src2);
-}
-
-static inline void emit_ori(unsigned int dst, unsigned src, u32 imm,
-			    struct jit_ctx *ctx)
-{
-	if (imm >= BIT(16)) {
-		emit_load_imm(r_tmp, imm, ctx);
-		emit_or(dst, src, r_tmp, ctx);
-	} else {
-		emit_instr(ctx, ori, dst, src, imm);
-	}
-}
-
-static inline void emit_daddiu(unsigned int dst, unsigned int src,
-			       int imm, struct jit_ctx *ctx)
-{
-	/*
-	 * Only used for stack, so the imm is relatively small
-	 * and it fits in 15-bits
-	 */
-	emit_instr(ctx, daddiu, dst, src, imm);
-}
-
-static inline void emit_addiu(unsigned int dst, unsigned int src,
-			      u32 imm, struct jit_ctx *ctx)
-{
-	if (!is_range16(imm)) {
-		emit_load_imm(r_tmp, imm, ctx);
-		emit_addu(dst, r_tmp, src, ctx);
-	} else {
-		emit_instr(ctx, addiu, dst, src, imm);
-	}
-}
-
-static inline void emit_and(unsigned int dst, unsigned int src1,
-			    unsigned int src2, struct jit_ctx *ctx)
-{
-	emit_instr(ctx, and, dst, src1, src2);
-}
-
-static inline void emit_andi(unsigned int dst, unsigned int src,
-			     u32 imm, struct jit_ctx *ctx)
-{
-	/* If imm does not fit in u16 then load it to register */
-	if (imm >= BIT(16)) {
-		emit_load_imm(r_tmp, imm, ctx);
-		emit_and(dst, src, r_tmp, ctx);
-	} else {
-		emit_instr(ctx, andi, dst, src, imm);
-	}
-}
-
-static inline void emit_xor(unsigned int dst, unsigned int src1,
-			    unsigned int src2, struct jit_ctx *ctx)
-{
-	emit_instr(ctx, xor, dst, src1, src2);
-}
-
-static inline void emit_xori(ptr dst, ptr src, u32 imm, struct jit_ctx *ctx)
-{
-	/* If imm does not fit in u16 then load it to register */
-	if (imm >= BIT(16)) {
-		emit_load_imm(r_tmp, imm, ctx);
-		emit_xor(dst, src, r_tmp, ctx);
-	} else {
-		emit_instr(ctx, xori, dst, src, imm);
-	}
-}
-
-static inline void emit_stack_offset(int offset, struct jit_ctx *ctx)
-{
-	emit_long_instr(ctx, ADDIU, r_sp, r_sp, offset);
-}
-
-static inline void emit_subu(unsigned int dst, unsigned int src1,
-			     unsigned int src2, struct jit_ctx *ctx)
-{
-	emit_instr(ctx, subu, dst, src1, src2);
-}
-
-static inline void emit_neg(unsigned int reg, struct jit_ctx *ctx)
-{
-	emit_subu(reg, r_zero, reg, ctx);
-}
-
-static inline void emit_sllv(unsigned int dst, unsigned int src,
-			     unsigned int sa, struct jit_ctx *ctx)
-{
-	emit_instr(ctx, sllv, dst, src, sa);
-}
-
-static inline void emit_sll(unsigned int dst, unsigned int src,
-			    unsigned int sa, struct jit_ctx *ctx)
-{
-	/* sa is 5-bits long */
-	if (sa >= BIT(5))
-		/* Shifting >= 32 results in zero */
-		emit_jit_reg_move(dst, r_zero, ctx);
-	else
-		emit_instr(ctx, sll, dst, src, sa);
-}
-
-static inline void emit_srlv(unsigned int dst, unsigned int src,
-			     unsigned int sa, struct jit_ctx *ctx)
-{
-	emit_instr(ctx, srlv, dst, src, sa);
-}
-
-static inline void emit_srl(unsigned int dst, unsigned int src,
-			    unsigned int sa, struct jit_ctx *ctx)
-{
-	/* sa is 5-bits long */
-	if (sa >= BIT(5))
-		/* Shifting >= 32 results in zero */
-		emit_jit_reg_move(dst, r_zero, ctx);
-	else
-		emit_instr(ctx, srl, dst, src, sa);
-}
-
-static inline void emit_slt(unsigned int dst, unsigned int src1,
-			    unsigned int src2, struct jit_ctx *ctx)
-{
-	emit_instr(ctx, slt, dst, src1, src2);
-}
-
-static inline void emit_sltu(unsigned int dst, unsigned int src1,
-			     unsigned int src2, struct jit_ctx *ctx)
-{
-	emit_instr(ctx, sltu, dst, src1, src2);
-}
-
-static inline void emit_sltiu(unsigned dst, unsigned int src,
-			      unsigned int imm, struct jit_ctx *ctx)
-{
-	/* 16 bit immediate */
-	if (!is_range16((s32)imm)) {
-		emit_load_imm(r_tmp, imm, ctx);
-		emit_sltu(dst, src, r_tmp, ctx);
-	} else {
-		emit_instr(ctx, sltiu, dst, src, imm);
-	}
-
-}
-
-/* Store register on the stack */
-static inline void emit_store_stack_reg(ptr reg, ptr base,
-					unsigned int offset,
-					struct jit_ctx *ctx)
-{
-	emit_long_instr(ctx, SW, reg, offset, base);
-}
-
-static inline void emit_store(ptr reg, ptr base, unsigned int offset,
-			      struct jit_ctx *ctx)
-{
-	emit_instr(ctx, sw, reg, offset, base);
-}
-
-static inline void emit_load_stack_reg(ptr reg, ptr base,
-				       unsigned int offset,
-				       struct jit_ctx *ctx)
-{
-	emit_long_instr(ctx, LW, reg, offset, base);
-}
-
-static inline void emit_load(unsigned int reg, unsigned int base,
-			     unsigned int offset, struct jit_ctx *ctx)
-{
-	emit_instr(ctx, lw, reg, offset, base);
-}
-
-static inline void emit_load_byte(unsigned int reg, unsigned int base,
-				  unsigned int offset, struct jit_ctx *ctx)
-{
-	emit_instr(ctx, lb, reg, offset, base);
-}
-
-static inline void emit_half_load(unsigned int reg, unsigned int base,
-				  unsigned int offset, struct jit_ctx *ctx)
-{
-	emit_instr(ctx, lh, reg, offset, base);
-}
-
-static inline void emit_half_load_unsigned(unsigned int reg, unsigned int base,
-					   unsigned int offset, struct jit_ctx *ctx)
-{
-	emit_instr(ctx, lhu, reg, offset, base);
-}
-
-static inline void emit_mul(unsigned int dst, unsigned int src1,
-			    unsigned int src2, struct jit_ctx *ctx)
-{
-	emit_instr(ctx, mul, dst, src1, src2);
-}
-
-static inline void emit_div(unsigned int dst, unsigned int src,
-			    struct jit_ctx *ctx)
-{
-	if (ctx->target != NULL) {
-		u32 *p = &ctx->target[ctx->idx];
-		uasm_i_divu(&p, dst, src);
-		p = &ctx->target[ctx->idx + 1];
-		uasm_i_mflo(&p, dst);
-	}
-	ctx->idx += 2; /* 2 insts */
-}
-
-static inline void emit_mod(unsigned int dst, unsigned int src,
-			    struct jit_ctx *ctx)
-{
-	if (ctx->target != NULL) {
-		u32 *p = &ctx->target[ctx->idx];
-		uasm_i_divu(&p, dst, src);
-		p = &ctx->target[ctx->idx + 1];
-		uasm_i_mfhi(&p, dst);
-	}
-	ctx->idx += 2; /* 2 insts */
-}
-
-static inline void emit_dsll(unsigned int dst, unsigned int src,
-			     unsigned int sa, struct jit_ctx *ctx)
-{
-	emit_instr(ctx, dsll, dst, src, sa);
-}
-
-static inline void emit_dsrl32(unsigned int dst, unsigned int src,
-			       unsigned int sa, struct jit_ctx *ctx)
-{
-	emit_instr(ctx, dsrl32, dst, src, sa);
-}
-
-static inline void emit_wsbh(unsigned int dst, unsigned int src,
-			     struct jit_ctx *ctx)
-{
-	emit_instr(ctx, wsbh, dst, src);
-}
-
-/* load pointer to register */
-static inline void emit_load_ptr(unsigned int dst, unsigned int src,
-				     int imm, struct jit_ctx *ctx)
-{
-	/* src contains the base addr of the 32/64-pointer */
-	emit_long_instr(ctx, LW, dst, imm, src);
-}
-
-/* load a function pointer to register */
-static inline void emit_load_func(unsigned int reg, ptr imm,
-				  struct jit_ctx *ctx)
-{
-	if (IS_ENABLED(CONFIG_64BIT)) {
-		/* At this point imm is always 64-bit */
-		emit_load_imm(r_tmp, (u64)imm >> 32, ctx);
-		emit_dsll(r_tmp_imm, r_tmp, 16, ctx); /* left shift by 16 */
-		emit_ori(r_tmp, r_tmp_imm, (imm >> 16) & 0xffff, ctx);
-		emit_dsll(r_tmp_imm, r_tmp, 16, ctx); /* left shift by 16 */
-		emit_ori(reg, r_tmp_imm, imm & 0xffff, ctx);
-	} else {
-		emit_load_imm(reg, imm, ctx);
-	}
-}
-
-/* Move to real MIPS register */
-static inline void emit_reg_move(ptr dst, ptr src, struct jit_ctx *ctx)
-{
-	emit_long_instr(ctx, ADDU, dst, src, r_zero);
-}
-
-/* Move to JIT (32-bit) register */
-static inline void emit_jit_reg_move(ptr dst, ptr src, struct jit_ctx *ctx)
-{
-	emit_addu(dst, src, r_zero, ctx);
-}
-
-/* Compute the immediate value for PC-relative branches. */
-static inline u32 b_imm(unsigned int tgt, struct jit_ctx *ctx)
-{
-	if (ctx->target == NULL)
-		return 0;
-
-	/*
-	 * We want a pc-relative branch. We only do forward branches
-	 * so tgt is always after pc. tgt is the instruction offset
-	 * we want to jump to.
-
-	 * Branch on MIPS:
-	 * I: target_offset <- sign_extend(offset)
-	 * I+1: PC += target_offset (delay slot)
-	 *
-	 * ctx->idx currently points to the branch instruction
-	 * but the offset is added to the delay slot so we need
-	 * to subtract 4.
-	 */
-	return ctx->offsets[tgt] -
-		(ctx->idx * 4 - ctx->prologue_bytes) - 4;
-}
-
-static inline void emit_bcond(int cond, unsigned int reg1, unsigned int reg2,
-			     unsigned int imm, struct jit_ctx *ctx)
-{
-	if (ctx->target != NULL) {
-		u32 *p = &ctx->target[ctx->idx];
-
-		switch (cond) {
-		case MIPS_COND_EQ:
-			uasm_i_beq(&p, reg1, reg2, imm);
-			break;
-		case MIPS_COND_NE:
-			uasm_i_bne(&p, reg1, reg2, imm);
-			break;
-		case MIPS_COND_ALL:
-			uasm_i_b(&p, imm);
-			break;
-		default:
-			pr_warn("%s: Unhandled branch conditional: %d\n",
-				__func__, cond);
-		}
-	}
-	ctx->idx++;
-}
-
-static inline void emit_b(unsigned int imm, struct jit_ctx *ctx)
-{
-	emit_bcond(MIPS_COND_ALL, r_zero, r_zero, imm, ctx);
-}
-
-static inline void emit_jalr(unsigned int link, unsigned int reg,
-			     struct jit_ctx *ctx)
-{
-	emit_instr(ctx, jalr, link, reg);
-}
-
-static inline void emit_jr(unsigned int reg, struct jit_ctx *ctx)
-{
-	emit_instr(ctx, jr, reg);
-}
-
-static inline u16 align_sp(unsigned int num)
-{
-	/* Double word alignment for 32-bit, quadword for 64-bit */
-	unsigned int align = IS_ENABLED(CONFIG_64BIT) ? 16 : 8;
-	num = (num + (align - 1)) & -align;
-	return num;
-}
-
-static void save_bpf_jit_regs(struct jit_ctx *ctx, unsigned offset)
-{
-	int i = 0, real_off = 0;
-	u32 sflags, tmp_flags;
-
-	/* Adjust the stack pointer */
-	if (offset)
-		emit_stack_offset(-align_sp(offset), ctx);
-
-	tmp_flags = sflags = ctx->flags >> SEEN_SREG_SFT;
-	/* sflags is essentially a bitmap */
-	while (tmp_flags) {
-		if ((sflags >> i) & 0x1) {
-			emit_store_stack_reg(MIPS_R_S0 + i, r_sp, real_off,
-					     ctx);
-			real_off += SZREG;
-		}
-		i++;
-		tmp_flags >>= 1;
-	}
-
-	/* save return address */
-	if (ctx->flags & SEEN_CALL) {
-		emit_store_stack_reg(r_ra, r_sp, real_off, ctx);
-		real_off += SZREG;
-	}
-
-	/* Setup r_M leaving the alignment gap if necessary */
-	if (ctx->flags & SEEN_MEM) {
-		if (real_off % (SZREG * 2))
-			real_off += SZREG;
-		emit_long_instr(ctx, ADDIU, r_M, r_sp, real_off);
-	}
-}
-
-static void restore_bpf_jit_regs(struct jit_ctx *ctx,
-				 unsigned int offset)
-{
-	int i, real_off = 0;
-	u32 sflags, tmp_flags;
-
-	tmp_flags = sflags = ctx->flags >> SEEN_SREG_SFT;
-	/* sflags is a bitmap */
-	i = 0;
-	while (tmp_flags) {
-		if ((sflags >> i) & 0x1) {
-			emit_load_stack_reg(MIPS_R_S0 + i, r_sp, real_off,
-					    ctx);
-			real_off += SZREG;
-		}
-		i++;
-		tmp_flags >>= 1;
-	}
-
-	/* restore return address */
-	if (ctx->flags & SEEN_CALL)
-		emit_load_stack_reg(r_ra, r_sp, real_off, ctx);
-
-	/* Restore the sp and discard the scrach memory */
-	if (offset)
-		emit_stack_offset(align_sp(offset), ctx);
-}
-
-static unsigned int get_stack_depth(struct jit_ctx *ctx)
-{
-	int sp_off = 0;
-
-
-	/* How may s* regs do we need to preserved? */
-	sp_off += hweight32(ctx->flags >> SEEN_SREG_SFT) * SZREG;
-
-	if (ctx->flags & SEEN_MEM)
-		sp_off += 4 * BPF_MEMWORDS; /* BPF_MEMWORDS are 32-bit */
-
-	if (ctx->flags & SEEN_CALL)
-		sp_off += SZREG; /* Space for our ra register */
-
-	return sp_off;
-}
-
-static void build_prologue(struct jit_ctx *ctx)
-{
-	int sp_off;
-
-	/* Calculate the total offset for the stack pointer */
-	sp_off = get_stack_depth(ctx);
-	save_bpf_jit_regs(ctx, sp_off);
-
-	if (ctx->flags & SEEN_SKB)
-		emit_reg_move(r_skb, MIPS_R_A0, ctx);
-
-	if (ctx->flags & SEEN_SKB_DATA) {
-		/* Load packet length */
-		emit_load(r_skb_len, r_skb, offsetof(struct sk_buff, len),
-			  ctx);
-		emit_load(r_tmp, r_skb, offsetof(struct sk_buff, data_len),
-			  ctx);
-		/* Load the data pointer */
-		emit_load_ptr(r_skb_data, r_skb,
-			      offsetof(struct sk_buff, data), ctx);
-		/* Load the header length */
-		emit_subu(r_skb_hl, r_skb_len, r_tmp, ctx);
-	}
-
-	if (ctx->flags & SEEN_X)
-		emit_jit_reg_move(r_X, r_zero, ctx);
-
-	/*
-	 * Do not leak kernel data to userspace, we only need to clear
-	 * r_A if it is ever used.  In fact if it is never used, we
-	 * will not save/restore it, so clearing it in this case would
-	 * corrupt the state of the caller.
-	 */
-	if (bpf_needs_clear_a(&ctx->skf->insns[0]) &&
-	    (ctx->flags & SEEN_A))
-		emit_jit_reg_move(r_A, r_zero, ctx);
-}
-
-static void build_epilogue(struct jit_ctx *ctx)
-{
-	unsigned int sp_off;
-
-	/* Calculate the total offset for the stack pointer */
-
-	sp_off = get_stack_depth(ctx);
-	restore_bpf_jit_regs(ctx, sp_off);
-
-	/* Return */
-	emit_jr(r_ra, ctx);
-	emit_nop(ctx);
-}
-
-#define CHOOSE_LOAD_FUNC(K, func) \
-	((int)K < 0 ? ((int)K >= SKF_LL_OFF ? func##_negative : func) : \
-	 func##_positive)
-
-static int build_body(struct jit_ctx *ctx)
-{
-	const struct bpf_prog *prog = ctx->skf;
-	const struct sock_filter *inst;
-	unsigned int i, off, condt;
-	u32 k, b_off __maybe_unused;
-	u8 (*sk_load_func)(unsigned long *skb, int offset);
-
-	for (i = 0; i < prog->len; i++) {
-		u16 code;
-
-		inst = &(prog->insns[i]);
-		pr_debug("%s: code->0x%02x, jt->0x%x, jf->0x%x, k->0x%x\n",
-			 __func__, inst->code, inst->jt, inst->jf, inst->k);
-		k = inst->k;
-		code = bpf_anc_helper(inst);
-
-		if (ctx->target == NULL)
-			ctx->offsets[i] = ctx->idx * 4;
-
-		switch (code) {
-		case BPF_LD | BPF_IMM:
-			/* A <- k ==> li r_A, k */
-			ctx->flags |= SEEN_A;
-			emit_load_imm(r_A, k, ctx);
-			break;
-		case BPF_LD | BPF_W | BPF_LEN:
-			BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, len) != 4);
-			/* A <- len ==> lw r_A, offset(skb) */
-			ctx->flags |= SEEN_SKB | SEEN_A;
-			off = offsetof(struct sk_buff, len);
-			emit_load(r_A, r_skb, off, ctx);
-			break;
-		case BPF_LD | BPF_MEM:
-			/* A <- M[k] ==> lw r_A, offset(M) */
-			ctx->flags |= SEEN_MEM | SEEN_A;
-			emit_load(r_A, r_M, SCRATCH_OFF(k), ctx);
-			break;
-		case BPF_LD | BPF_W | BPF_ABS:
-			/* A <- P[k:4] */
-			sk_load_func = CHOOSE_LOAD_FUNC(k, sk_load_word);
-			goto load;
-		case BPF_LD | BPF_H | BPF_ABS:
-			/* A <- P[k:2] */
-			sk_load_func = CHOOSE_LOAD_FUNC(k, sk_load_half);
-			goto load;
-		case BPF_LD | BPF_B | BPF_ABS:
-			/* A <- P[k:1] */
-			sk_load_func = CHOOSE_LOAD_FUNC(k, sk_load_byte);
-load:
-			emit_load_imm(r_off, k, ctx);
-load_common:
-			ctx->flags |= SEEN_CALL | SEEN_OFF |
-				SEEN_SKB | SEEN_A | SEEN_SKB_DATA;
-
-			emit_load_func(r_s0, (ptr)sk_load_func, ctx);
-			emit_reg_move(MIPS_R_A0, r_skb, ctx);
-			emit_jalr(MIPS_R_RA, r_s0, ctx);
-			/* Load second argument to delay slot */
-			emit_reg_move(MIPS_R_A1, r_off, ctx);
-			/* Check the error value */
-			emit_bcond(MIPS_COND_EQ, r_ret, 0, b_imm(i + 1, ctx),
-				   ctx);
-			/* Load return register on DS for failures */
-			emit_reg_move(r_ret, r_zero, ctx);
-			/* Return with error */
-			emit_b(b_imm(prog->len, ctx), ctx);
-			emit_nop(ctx);
-			break;
-		case BPF_LD | BPF_W | BPF_IND:
-			/* A <- P[X + k:4] */
-			sk_load_func = sk_load_word;
-			goto load_ind;
-		case BPF_LD | BPF_H | BPF_IND:
-			/* A <- P[X + k:2] */
-			sk_load_func = sk_load_half;
-			goto load_ind;
-		case BPF_LD | BPF_B | BPF_IND:
-			/* A <- P[X + k:1] */
-			sk_load_func = sk_load_byte;
-load_ind:
-			ctx->flags |= SEEN_OFF | SEEN_X;
-			emit_addiu(r_off, r_X, k, ctx);
-			goto load_common;
-		case BPF_LDX | BPF_IMM:
-			/* X <- k */
-			ctx->flags |= SEEN_X;
-			emit_load_imm(r_X, k, ctx);
-			break;
-		case BPF_LDX | BPF_MEM:
-			/* X <- M[k] */
-			ctx->flags |= SEEN_X | SEEN_MEM;
-			emit_load(r_X, r_M, SCRATCH_OFF(k), ctx);
-			break;
-		case BPF_LDX | BPF_W | BPF_LEN:
-			/* X <- len */
-			ctx->flags |= SEEN_X | SEEN_SKB;
-			off = offsetof(struct sk_buff, len);
-			emit_load(r_X, r_skb, off, ctx);
-			break;
-		case BPF_LDX | BPF_B | BPF_MSH:
-			/* X <- 4 * (P[k:1] & 0xf) */
-			ctx->flags |= SEEN_X | SEEN_CALL | SEEN_SKB;
-			/* Load offset to a1 */
-			emit_load_func(r_s0, (ptr)sk_load_byte, ctx);
-			/*
-			 * This may emit two instructions so it may not fit
-			 * in the delay slot. So use a0 in the delay slot.
-			 */
-			emit_load_imm(MIPS_R_A1, k, ctx);
-			emit_jalr(MIPS_R_RA, r_s0, ctx);
-			emit_reg_move(MIPS_R_A0, r_skb, ctx); /* delay slot */
-			/* Check the error value */
-			emit_bcond(MIPS_COND_NE, r_ret, 0,
-				   b_imm(prog->len, ctx), ctx);
-			emit_reg_move(r_ret, r_zero, ctx);
-			/* We are good */
-			/* X <- P[1:K] & 0xf */
-			emit_andi(r_X, r_A, 0xf, ctx);
-			/* X << 2 */
-			emit_b(b_imm(i + 1, ctx), ctx);
-			emit_sll(r_X, r_X, 2, ctx); /* delay slot */
-			break;
-		case BPF_ST:
-			/* M[k] <- A */
-			ctx->flags |= SEEN_MEM | SEEN_A;
-			emit_store(r_A, r_M, SCRATCH_OFF(k), ctx);
-			break;
-		case BPF_STX:
-			/* M[k] <- X */
-			ctx->flags |= SEEN_MEM | SEEN_X;
-			emit_store(r_X, r_M, SCRATCH_OFF(k), ctx);
-			break;
-		case BPF_ALU | BPF_ADD | BPF_K:
-			/* A += K */
-			ctx->flags |= SEEN_A;
-			emit_addiu(r_A, r_A, k, ctx);
-			break;
-		case BPF_ALU | BPF_ADD | BPF_X:
-			/* A += X */
-			ctx->flags |= SEEN_A | SEEN_X;
-			emit_addu(r_A, r_A, r_X, ctx);
-			break;
-		case BPF_ALU | BPF_SUB | BPF_K:
-			/* A -= K */
-			ctx->flags |= SEEN_A;
-			emit_addiu(r_A, r_A, -k, ctx);
-			break;
-		case BPF_ALU | BPF_SUB | BPF_X:
-			/* A -= X */
-			ctx->flags |= SEEN_A | SEEN_X;
-			emit_subu(r_A, r_A, r_X, ctx);
-			break;
-		case BPF_ALU | BPF_MUL | BPF_K:
-			/* A *= K */
-			/* Load K to scratch register before MUL */
-			ctx->flags |= SEEN_A;
-			emit_load_imm(r_s0, k, ctx);
-			emit_mul(r_A, r_A, r_s0, ctx);
-			break;
-		case BPF_ALU | BPF_MUL | BPF_X:
-			/* A *= X */
-			ctx->flags |= SEEN_A | SEEN_X;
-			emit_mul(r_A, r_A, r_X, ctx);
-			break;
-		case BPF_ALU | BPF_DIV | BPF_K:
-			/* A /= k */
-			if (k == 1)
-				break;
-			if (optimize_div(&k)) {
-				ctx->flags |= SEEN_A;
-				emit_srl(r_A, r_A, k, ctx);
-				break;
-			}
-			ctx->flags |= SEEN_A;
-			emit_load_imm(r_s0, k, ctx);
-			emit_div(r_A, r_s0, ctx);
-			break;
-		case BPF_ALU | BPF_MOD | BPF_K:
-			/* A %= k */
-			if (k == 1) {
-				ctx->flags |= SEEN_A;
-				emit_jit_reg_move(r_A, r_zero, ctx);
-			} else {
-				ctx->flags |= SEEN_A;
-				emit_load_imm(r_s0, k, ctx);
-				emit_mod(r_A, r_s0, ctx);
-			}
-			break;
-		case BPF_ALU | BPF_DIV | BPF_X:
-			/* A /= X */
-			ctx->flags |= SEEN_X | SEEN_A;
-			/* Check if r_X is zero */
-			emit_bcond(MIPS_COND_EQ, r_X, r_zero,
-				   b_imm(prog->len, ctx), ctx);
-			emit_load_imm(r_ret, 0, ctx); /* delay slot */
-			emit_div(r_A, r_X, ctx);
-			break;
-		case BPF_ALU | BPF_MOD | BPF_X:
-			/* A %= X */
-			ctx->flags |= SEEN_X | SEEN_A;
-			/* Check if r_X is zero */
-			emit_bcond(MIPS_COND_EQ, r_X, r_zero,
-				   b_imm(prog->len, ctx), ctx);
-			emit_load_imm(r_ret, 0, ctx); /* delay slot */
-			emit_mod(r_A, r_X, ctx);
-			break;
-		case BPF_ALU | BPF_OR | BPF_K:
-			/* A |= K */
-			ctx->flags |= SEEN_A;
-			emit_ori(r_A, r_A, k, ctx);
-			break;
-		case BPF_ALU | BPF_OR | BPF_X:
-			/* A |= X */
-			ctx->flags |= SEEN_A;
-			emit_ori(r_A, r_A, r_X, ctx);
-			break;
-		case BPF_ALU | BPF_XOR | BPF_K:
-			/* A ^= k */
-			ctx->flags |= SEEN_A;
-			emit_xori(r_A, r_A, k, ctx);
-			break;
-		case BPF_ANC | SKF_AD_ALU_XOR_X:
-		case BPF_ALU | BPF_XOR | BPF_X:
-			/* A ^= X */
-			ctx->flags |= SEEN_A;
-			emit_xor(r_A, r_A, r_X, ctx);
-			break;
-		case BPF_ALU | BPF_AND | BPF_K:
-			/* A &= K */
-			ctx->flags |= SEEN_A;
-			emit_andi(r_A, r_A, k, ctx);
-			break;
-		case BPF_ALU | BPF_AND | BPF_X:
-			/* A &= X */
-			ctx->flags |= SEEN_A | SEEN_X;
-			emit_and(r_A, r_A, r_X, ctx);
-			break;
-		case BPF_ALU | BPF_LSH | BPF_K:
-			/* A <<= K */
-			ctx->flags |= SEEN_A;
-			emit_sll(r_A, r_A, k, ctx);
-			break;
-		case BPF_ALU | BPF_LSH | BPF_X:
-			/* A <<= X */
-			ctx->flags |= SEEN_A | SEEN_X;
-			emit_sllv(r_A, r_A, r_X, ctx);
-			break;
-		case BPF_ALU | BPF_RSH | BPF_K:
-			/* A >>= K */
-			ctx->flags |= SEEN_A;
-			emit_srl(r_A, r_A, k, ctx);
-			break;
-		case BPF_ALU | BPF_RSH | BPF_X:
-			ctx->flags |= SEEN_A | SEEN_X;
-			emit_srlv(r_A, r_A, r_X, ctx);
-			break;
-		case BPF_ALU | BPF_NEG:
-			/* A = -A */
-			ctx->flags |= SEEN_A;
-			emit_neg(r_A, ctx);
-			break;
-		case BPF_JMP | BPF_JA:
-			/* pc += K */
-			emit_b(b_imm(i + k + 1, ctx), ctx);
-			emit_nop(ctx);
-			break;
-		case BPF_JMP | BPF_JEQ | BPF_K:
-			/* pc += ( A == K ) ? pc->jt : pc->jf */
-			condt = MIPS_COND_EQ | MIPS_COND_K;
-			goto jmp_cmp;
-		case BPF_JMP | BPF_JEQ | BPF_X:
-			ctx->flags |= SEEN_X;
-			/* pc += ( A == X ) ? pc->jt : pc->jf */
-			condt = MIPS_COND_EQ | MIPS_COND_X;
-			goto jmp_cmp;
-		case BPF_JMP | BPF_JGE | BPF_K:
-			/* pc += ( A >= K ) ? pc->jt : pc->jf */
-			condt = MIPS_COND_GE | MIPS_COND_K;
-			goto jmp_cmp;
-		case BPF_JMP | BPF_JGE | BPF_X:
-			ctx->flags |= SEEN_X;
-			/* pc += ( A >= X ) ? pc->jt : pc->jf */
-			condt = MIPS_COND_GE | MIPS_COND_X;
-			goto jmp_cmp;
-		case BPF_JMP | BPF_JGT | BPF_K:
-			/* pc += ( A > K ) ? pc->jt : pc->jf */
-			condt = MIPS_COND_GT | MIPS_COND_K;
-			goto jmp_cmp;
-		case BPF_JMP | BPF_JGT | BPF_X:
-			ctx->flags |= SEEN_X;
-			/* pc += ( A > X ) ? pc->jt : pc->jf */
-			condt = MIPS_COND_GT | MIPS_COND_X;
-jmp_cmp:
-			/* Greater or Equal */
-			if ((condt & MIPS_COND_GE) ||
-			    (condt & MIPS_COND_GT)) {
-				if (condt & MIPS_COND_K) { /* K */
-					ctx->flags |= SEEN_A;
-					emit_sltiu(r_s0, r_A, k, ctx);
-				} else { /* X */
-					ctx->flags |= SEEN_A |
-						SEEN_X;
-					emit_sltu(r_s0, r_A, r_X, ctx);
-				}
-				/* A < (K|X) ? r_scrach = 1 */
-				b_off = b_imm(i + inst->jf + 1, ctx);
-				emit_bcond(MIPS_COND_NE, r_s0, r_zero, b_off,
-					   ctx);
-				emit_nop(ctx);
-				/* A > (K|X) ? scratch = 0 */
-				if (condt & MIPS_COND_GT) {
-					/* Checking for equality */
-					ctx->flags |= SEEN_A | SEEN_X;
-					if (condt & MIPS_COND_K)
-						emit_load_imm(r_s0, k, ctx);
-					else
-						emit_jit_reg_move(r_s0, r_X,
-								  ctx);
-					b_off = b_imm(i + inst->jf + 1, ctx);
-					emit_bcond(MIPS_COND_EQ, r_A, r_s0,
-						   b_off, ctx);
-					emit_nop(ctx);
-					/* Finally, A > K|X */
-					b_off = b_imm(i + inst->jt + 1, ctx);
-					emit_b(b_off, ctx);
-					emit_nop(ctx);
-				} else {
-					/* A >= (K|X) so jump */
-					b_off = b_imm(i + inst->jt + 1, ctx);
-					emit_b(b_off, ctx);
-					emit_nop(ctx);
-				}
-			} else {
-				/* A == K|X */
-				if (condt & MIPS_COND_K) { /* K */
-					ctx->flags |= SEEN_A;
-					emit_load_imm(r_s0, k, ctx);
-					/* jump true */
-					b_off = b_imm(i + inst->jt + 1, ctx);
-					emit_bcond(MIPS_COND_EQ, r_A, r_s0,
-						   b_off, ctx);
-					emit_nop(ctx);
-					/* jump false */
-					b_off = b_imm(i + inst->jf + 1,
-						      ctx);
-					emit_bcond(MIPS_COND_NE, r_A, r_s0,
-						   b_off, ctx);
-					emit_nop(ctx);
-				} else { /* X */
-					/* jump true */
-					ctx->flags |= SEEN_A | SEEN_X;
-					b_off = b_imm(i + inst->jt + 1,
-						      ctx);
-					emit_bcond(MIPS_COND_EQ, r_A, r_X,
-						   b_off, ctx);
-					emit_nop(ctx);
-					/* jump false */
-					b_off = b_imm(i + inst->jf + 1, ctx);
-					emit_bcond(MIPS_COND_NE, r_A, r_X,
-						   b_off, ctx);
-					emit_nop(ctx);
-				}
-			}
-			break;
-		case BPF_JMP | BPF_JSET | BPF_K:
-			ctx->flags |= SEEN_A;
-			/* pc += (A & K) ? pc -> jt : pc -> jf */
-			emit_load_imm(r_s1, k, ctx);
-			emit_and(r_s0, r_A, r_s1, ctx);
-			/* jump true */
-			b_off = b_imm(i + inst->jt + 1, ctx);
-			emit_bcond(MIPS_COND_NE, r_s0, r_zero, b_off, ctx);
-			emit_nop(ctx);
-			/* jump false */
-			b_off = b_imm(i + inst->jf + 1, ctx);
-			emit_b(b_off, ctx);
-			emit_nop(ctx);
-			break;
-		case BPF_JMP | BPF_JSET | BPF_X:
-			ctx->flags |= SEEN_X | SEEN_A;
-			/* pc += (A & X) ? pc -> jt : pc -> jf */
-			emit_and(r_s0, r_A, r_X, ctx);
-			/* jump true */
-			b_off = b_imm(i + inst->jt + 1, ctx);
-			emit_bcond(MIPS_COND_NE, r_s0, r_zero, b_off, ctx);
-			emit_nop(ctx);
-			/* jump false */
-			b_off = b_imm(i + inst->jf + 1, ctx);
-			emit_b(b_off, ctx);
-			emit_nop(ctx);
-			break;
-		case BPF_RET | BPF_A:
-			ctx->flags |= SEEN_A;
-			if (i != prog->len - 1)
-				/*
-				 * If this is not the last instruction
-				 * then jump to the epilogue
-				 */
-				emit_b(b_imm(prog->len, ctx), ctx);
-			emit_reg_move(r_ret, r_A, ctx); /* delay slot */
-			break;
-		case BPF_RET | BPF_K:
-			/*
-			 * It can emit two instructions so it does not fit on
-			 * the delay slot.
-			 */
-			emit_load_imm(r_ret, k, ctx);
-			if (i != prog->len - 1) {
-				/*
-				 * If this is not the last instruction
-				 * then jump to the epilogue
-				 */
-				emit_b(b_imm(prog->len, ctx), ctx);
-				emit_nop(ctx);
-			}
-			break;
-		case BPF_MISC | BPF_TAX:
-			/* X = A */
-			ctx->flags |= SEEN_X | SEEN_A;
-			emit_jit_reg_move(r_X, r_A, ctx);
-			break;
-		case BPF_MISC | BPF_TXA:
-			/* A = X */
-			ctx->flags |= SEEN_A | SEEN_X;
-			emit_jit_reg_move(r_A, r_X, ctx);
-			break;
-		/* AUX */
-		case BPF_ANC | SKF_AD_PROTOCOL:
-			/* A = ntohs(skb->protocol */
-			ctx->flags |= SEEN_SKB | SEEN_OFF | SEEN_A;
-			BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff,
-						  protocol) != 2);
-			off = offsetof(struct sk_buff, protocol);
-			emit_half_load(r_A, r_skb, off, ctx);
-#ifdef CONFIG_CPU_LITTLE_ENDIAN
-			/* This needs little endian fixup */
-			if (cpu_has_wsbh) {
-				/* R2 and later have the wsbh instruction */
-				emit_wsbh(r_A, r_A, ctx);
-			} else {
-				/* Get first byte */
-				emit_andi(r_tmp_imm, r_A, 0xff, ctx);
-				/* Shift it */
-				emit_sll(r_tmp, r_tmp_imm, 8, ctx);
-				/* Get second byte */
-				emit_srl(r_tmp_imm, r_A, 8, ctx);
-				emit_andi(r_tmp_imm, r_tmp_imm, 0xff, ctx);
-				/* Put everyting together in r_A */
-				emit_or(r_A, r_tmp, r_tmp_imm, ctx);
-			}
-#endif
-			break;
-		case BPF_ANC | SKF_AD_CPU:
-			ctx->flags |= SEEN_A | SEEN_OFF;
-			/* A = current_thread_info()->cpu */
-			BUILD_BUG_ON(FIELD_SIZEOF(struct thread_info,
-						  cpu) != 4);
-			off = offsetof(struct thread_info, cpu);
-			/* $28/gp points to the thread_info struct */
-			emit_load(r_A, 28, off, ctx);
-			break;
-		case BPF_ANC | SKF_AD_IFINDEX:
-			/* A = skb->dev->ifindex */
-		case BPF_ANC | SKF_AD_HATYPE:
-			/* A = skb->dev->type */
-			ctx->flags |= SEEN_SKB | SEEN_A;
-			off = offsetof(struct sk_buff, dev);
-			/* Load *dev pointer */
-			emit_load_ptr(r_s0, r_skb, off, ctx);
-			/* error (0) in the delay slot */
-			emit_bcond(MIPS_COND_EQ, r_s0, r_zero,
-				   b_imm(prog->len, ctx), ctx);
-			emit_reg_move(r_ret, r_zero, ctx);
-			if (code == (BPF_ANC | SKF_AD_IFINDEX)) {
-				BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, ifindex) != 4);
-				off = offsetof(struct net_device, ifindex);
-				emit_load(r_A, r_s0, off, ctx);
-			} else { /* (code == (BPF_ANC | SKF_AD_HATYPE) */
-				BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, type) != 2);
-				off = offsetof(struct net_device, type);
-				emit_half_load_unsigned(r_A, r_s0, off, ctx);
-			}
-			break;
-		case BPF_ANC | SKF_AD_MARK:
-			ctx->flags |= SEEN_SKB | SEEN_A;
-			BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4);
-			off = offsetof(struct sk_buff, mark);
-			emit_load(r_A, r_skb, off, ctx);
-			break;
-		case BPF_ANC | SKF_AD_RXHASH:
-			ctx->flags |= SEEN_SKB | SEEN_A;
-			BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, hash) != 4);
-			off = offsetof(struct sk_buff, hash);
-			emit_load(r_A, r_skb, off, ctx);
-			break;
-		case BPF_ANC | SKF_AD_VLAN_TAG:
-			ctx->flags |= SEEN_SKB | SEEN_A;
-			BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff,
-						  vlan_tci) != 2);
-			off = offsetof(struct sk_buff, vlan_tci);
-			emit_half_load_unsigned(r_A, r_skb, off, ctx);
-			break;
-		case BPF_ANC | SKF_AD_VLAN_TAG_PRESENT:
-			ctx->flags |= SEEN_SKB | SEEN_A;
-			emit_load_byte(r_A, r_skb, PKT_VLAN_PRESENT_OFFSET(), ctx);
-			if (PKT_VLAN_PRESENT_BIT)
-				emit_srl(r_A, r_A, PKT_VLAN_PRESENT_BIT, ctx);
-			if (PKT_VLAN_PRESENT_BIT < 7)
-				emit_andi(r_A, r_A, 1, ctx);
-			break;
-		case BPF_ANC | SKF_AD_PKTTYPE:
-			ctx->flags |= SEEN_SKB;
-
-			emit_load_byte(r_tmp, r_skb, PKT_TYPE_OFFSET(), ctx);
-			/* Keep only the last 3 bits */
-			emit_andi(r_A, r_tmp, PKT_TYPE_MAX, ctx);
-#ifdef __BIG_ENDIAN_BITFIELD
-			/* Get the actual packet type to the lower 3 bits */
-			emit_srl(r_A, r_A, 5, ctx);
-#endif
-			break;
-		case BPF_ANC | SKF_AD_QUEUE:
-			ctx->flags |= SEEN_SKB | SEEN_A;
-			BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff,
-						  queue_mapping) != 2);
-			BUILD_BUG_ON(offsetof(struct sk_buff,
-					      queue_mapping) > 0xff);
-			off = offsetof(struct sk_buff, queue_mapping);
-			emit_half_load_unsigned(r_A, r_skb, off, ctx);
-			break;
-		default:
-			pr_debug("%s: Unhandled opcode: 0x%02x\n", __FILE__,
-				 inst->code);
-			return -1;
-		}
-	}
-
-	/* compute offsets only during the first pass */
-	if (ctx->target == NULL)
-		ctx->offsets[i] = ctx->idx * 4;
-
-	return 0;
-}
-
-void bpf_jit_compile(struct bpf_prog *fp)
-{
-	struct jit_ctx ctx;
-	unsigned int alloc_size, tmp_idx;
-
-	if (!bpf_jit_enable)
-		return;
-
-	memset(&ctx, 0, sizeof(ctx));
-
-	ctx.offsets = kcalloc(fp->len + 1, sizeof(*ctx.offsets), GFP_KERNEL);
-	if (ctx.offsets == NULL)
-		return;
-
-	ctx.skf = fp;
-
-	if (build_body(&ctx))
-		goto out;
-
-	tmp_idx = ctx.idx;
-	build_prologue(&ctx);
-	ctx.prologue_bytes = (ctx.idx - tmp_idx) * 4;
-	/* just to complete the ctx.idx count */
-	build_epilogue(&ctx);
-
-	alloc_size = 4 * ctx.idx;
-	ctx.target = module_alloc(alloc_size);
-	if (ctx.target == NULL)
-		goto out;
-
-	/* Clean it */
-	memset(ctx.target, 0, alloc_size);
-
-	ctx.idx = 0;
-
-	/* Generate the actual JIT code */
-	build_prologue(&ctx);
-	build_body(&ctx);
-	build_epilogue(&ctx);
-
-	/* Update the icache */
-	flush_icache_range((ptr)ctx.target, (ptr)(ctx.target + ctx.idx));
-
-	if (bpf_jit_enable > 1)
-		/* Dump JIT code */
-		bpf_jit_dump(fp->len, alloc_size, 2, ctx.target);
-
-	fp->bpf_func = (void *)ctx.target;
-	fp->jited = 1;
-
-out:
-	kfree(ctx.offsets);
-}
-
-void bpf_jit_free(struct bpf_prog *fp)
-{
-	if (fp->jited)
-		module_memfree(fp->bpf_func);
-
-	bpf_prog_unlock_free(fp);
-}
diff --git a/arch/mips/net/bpf_jit_asm.S b/arch/mips/net/bpf_jit_asm.S
deleted file mode 100644
index 57154c5883b6..000000000000
--- a/arch/mips/net/bpf_jit_asm.S
+++ /dev/null
@@ -1,285 +0,0 @@
-/*
- * bpf_jib_asm.S: Packet/header access helper functions for MIPS/MIPS64 BPF
- * compiler.
- *
- * Copyright (C) 2015 Imagination Technologies Ltd.
- * Author: Markos Chandras <markos.chandras@imgtec.com>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation; version 2 of the License.
- */
-
-#include <asm/asm.h>
-#include <asm/isa-rev.h>
-#include <asm/regdef.h>
-#include "bpf_jit.h"
-
-/* ABI
- *
- * r_skb_hl	skb header length
- * r_skb_data	skb data
- * r_off(a1)	offset register
- * r_A		BPF register A
- * r_X		PF register X
- * r_skb(a0)	*skb
- * r_M		*scratch memory
- * r_skb_le	skb length
- * r_s0		Scratch register 0
- * r_s1		Scratch register 1
- *
- * On entry:
- * a0: *skb
- * a1: offset (imm or imm + X)
- *
- * All non-BPF-ABI registers are free for use. On return, we only
- * care about r_ret. The BPF-ABI registers are assumed to remain
- * unmodified during the entire filter operation.
- */
-
-#define skb	a0
-#define offset	a1
-#define SKF_LL_OFF  (-0x200000) /* Can't include linux/filter.h in assembly */
-
-	/* We know better :) so prevent assembler reordering etc */
-	.set 	noreorder
-
-#define is_offset_negative(TYPE)				\
-	/* If offset is negative we have more work to do */	\
-	slti	t0, offset, 0;					\
-	bgtz	t0, bpf_slow_path_##TYPE##_neg;			\
-	/* Be careful what follows in DS. */
-
-#define is_offset_in_header(SIZE, TYPE)				\
-	/* Reading from header? */				\
-	addiu	$r_s0, $r_skb_hl, -SIZE;			\
-	slt	t0, $r_s0, offset;				\
-	bgtz	t0, bpf_slow_path_##TYPE;			\
-
-LEAF(sk_load_word)
-	is_offset_negative(word)
-FEXPORT(sk_load_word_positive)
-	is_offset_in_header(4, word)
-	/* Offset within header boundaries */
-	PTR_ADDU t1, $r_skb_data, offset
-	.set	reorder
-	lw	$r_A, 0(t1)
-	.set	noreorder
-#ifdef CONFIG_CPU_LITTLE_ENDIAN
-# if MIPS_ISA_REV >= 2
-	wsbh	t0, $r_A
-	rotr	$r_A, t0, 16
-# else
-	sll	t0, $r_A, 24
-	srl	t1, $r_A, 24
-	srl	t2, $r_A, 8
-	or	t0, t0, t1
-	andi	t2, t2, 0xff00
-	andi	t1, $r_A, 0xff00
-	or	t0, t0, t2
-	sll	t1, t1, 8
-	or	$r_A, t0, t1
-# endif
-#endif
-	jr	$r_ra
-	 move	$r_ret, zero
-	END(sk_load_word)
-
-LEAF(sk_load_half)
-	is_offset_negative(half)
-FEXPORT(sk_load_half_positive)
-	is_offset_in_header(2, half)
-	/* Offset within header boundaries */
-	PTR_ADDU t1, $r_skb_data, offset
-	lhu	$r_A, 0(t1)
-#ifdef CONFIG_CPU_LITTLE_ENDIAN
-# if MIPS_ISA_REV >= 2
-	wsbh	$r_A, $r_A
-# else
-	sll	t0, $r_A, 8
-	srl	t1, $r_A, 8
-	andi	t0, t0, 0xff00
-	or	$r_A, t0, t1
-# endif
-#endif
-	jr	$r_ra
-	 move	$r_ret, zero
-	END(sk_load_half)
-
-LEAF(sk_load_byte)
-	is_offset_negative(byte)
-FEXPORT(sk_load_byte_positive)
-	is_offset_in_header(1, byte)
-	/* Offset within header boundaries */
-	PTR_ADDU t1, $r_skb_data, offset
-	lbu	$r_A, 0(t1)
-	jr	$r_ra
-	 move	$r_ret, zero
-	END(sk_load_byte)
-
-/*
- * call skb_copy_bits:
- * (prototype in linux/skbuff.h)
- *
- * int skb_copy_bits(sk_buff *skb, int offset, void *to, int len)
- *
- * o32 mandates we leave 4 spaces for argument registers in case
- * the callee needs to use them. Even though we don't care about
- * the argument registers ourselves, we need to allocate that space
- * to remain ABI compliant since the callee may want to use that space.
- * We also allocate 2 more spaces for $r_ra and our return register (*to).
- *
- * n64 is a bit different. The *caller* will allocate the space to preserve
- * the arguments. So in 64-bit kernels, we allocate the 4-arg space for no
- * good reason but it does not matter that much really.
- *
- * (void *to) is returned in r_s0
- *
- */
-#ifdef CONFIG_CPU_LITTLE_ENDIAN
-#define DS_OFFSET(SIZE) (4 * SZREG)
-#else
-#define DS_OFFSET(SIZE) ((4 * SZREG) + (4 - SIZE))
-#endif
-#define bpf_slow_path_common(SIZE)				\
-	/* Quick check. Are we within reasonable boundaries? */ \
-	LONG_ADDIU	$r_s1, $r_skb_len, -SIZE;		\
-	sltu		$r_s0, offset, $r_s1;			\
-	beqz		$r_s0, fault;				\
-	/* Load 4th argument in DS */				\
-	 LONG_ADDIU	a3, zero, SIZE;				\
-	PTR_ADDIU	$r_sp, $r_sp, -(6 * SZREG);		\
-	PTR_LA		t0, skb_copy_bits;			\
-	PTR_S		$r_ra, (5 * SZREG)($r_sp);		\
-	/* Assign low slot to a2 */				\
-	PTR_ADDIU	a2, $r_sp, DS_OFFSET(SIZE);		\
-	jalr		t0;					\
-	/* Reset our destination slot (DS but it's ok) */	\
-	 INT_S		zero, (4 * SZREG)($r_sp);		\
-	/*							\
-	 * skb_copy_bits returns 0 on success and -EFAULT	\
-	 * on error. Our data live in a2. Do not bother with	\
-	 * our data if an error has been returned.		\
-	 */							\
-	/* Restore our frame */					\
-	PTR_L		$r_ra, (5 * SZREG)($r_sp);		\
-	INT_L		$r_s0, (4 * SZREG)($r_sp);		\
-	bltz		v0, fault;				\
-	 PTR_ADDIU	$r_sp, $r_sp, 6 * SZREG;		\
-	move		$r_ret, zero;				\
-
-NESTED(bpf_slow_path_word, (6 * SZREG), $r_sp)
-	bpf_slow_path_common(4)
-#ifdef CONFIG_CPU_LITTLE_ENDIAN
-# if MIPS_ISA_REV >= 2
-	wsbh	t0, $r_s0
-	jr	$r_ra
-	 rotr	$r_A, t0, 16
-# else
-	sll	t0, $r_s0, 24
-	srl	t1, $r_s0, 24
-	srl	t2, $r_s0, 8
-	or	t0, t0, t1
-	andi	t2, t2, 0xff00
-	andi	t1, $r_s0, 0xff00
-	or	t0, t0, t2
-	sll	t1, t1, 8
-	jr	$r_ra
-	 or	$r_A, t0, t1
-# endif
-#else
-	jr	$r_ra
-	 move	$r_A, $r_s0
-#endif
-
-	END(bpf_slow_path_word)
-
-NESTED(bpf_slow_path_half, (6 * SZREG), $r_sp)
-	bpf_slow_path_common(2)
-#ifdef CONFIG_CPU_LITTLE_ENDIAN
-# if MIPS_ISA_REV >= 2
-	jr	$r_ra
-	 wsbh	$r_A, $r_s0
-# else
-	sll	t0, $r_s0, 8
-	andi	t1, $r_s0, 0xff00
-	andi	t0, t0, 0xff00
-	srl	t1, t1, 8
-	jr	$r_ra
-	 or	$r_A, t0, t1
-# endif
-#else
-	jr	$r_ra
-	 move	$r_A, $r_s0
-#endif
-
-	END(bpf_slow_path_half)
-
-NESTED(bpf_slow_path_byte, (6 * SZREG), $r_sp)
-	bpf_slow_path_common(1)
-	jr	$r_ra
-	 move	$r_A, $r_s0
-
-	END(bpf_slow_path_byte)
-
-/*
- * Negative entry points
- */
-	.macro bpf_is_end_of_data
-	li	t0, SKF_LL_OFF
-	/* Reading link layer data? */
-	slt	t1, offset, t0
-	bgtz	t1, fault
-	/* Be careful what follows in DS. */
-	.endm
-/*
- * call skb_copy_bits:
- * (prototype in linux/filter.h)
- *
- * void *bpf_internal_load_pointer_neg_helper(const struct sk_buff *skb,
- *                                            int k, unsigned int size)
- *
- * see above (bpf_slow_path_common) for ABI restrictions
- */
-#define bpf_negative_common(SIZE)					\
-	PTR_ADDIU	$r_sp, $r_sp, -(6 * SZREG);			\
-	PTR_LA		t0, bpf_internal_load_pointer_neg_helper;	\
-	PTR_S		$r_ra, (5 * SZREG)($r_sp);			\
-	jalr		t0;						\
-	 li		a2, SIZE;					\
-	PTR_L		$r_ra, (5 * SZREG)($r_sp);			\
-	/* Check return pointer */					\
-	beqz		v0, fault;					\
-	 PTR_ADDIU	$r_sp, $r_sp, 6 * SZREG;			\
-	/* Preserve our pointer */					\
-	move		$r_s0, v0;					\
-	/* Set return value */						\
-	move		$r_ret, zero;					\
-
-bpf_slow_path_word_neg:
-	bpf_is_end_of_data
-NESTED(sk_load_word_negative, (6 * SZREG), $r_sp)
-	bpf_negative_common(4)
-	jr	$r_ra
-	 lw	$r_A, 0($r_s0)
-	END(sk_load_word_negative)
-
-bpf_slow_path_half_neg:
-	bpf_is_end_of_data
-NESTED(sk_load_half_negative, (6 * SZREG), $r_sp)
-	bpf_negative_common(2)
-	jr	$r_ra
-	 lhu	$r_A, 0($r_s0)
-	END(sk_load_half_negative)
-
-bpf_slow_path_byte_neg:
-	bpf_is_end_of_data
-NESTED(sk_load_byte_negative, (6 * SZREG), $r_sp)
-	bpf_negative_common(1)
-	jr	$r_ra
-	 lbu	$r_A, 0($r_s0)
-	END(sk_load_byte_negative)
-
-fault:
-	jr	$r_ra
-	 addiu $r_ret, zero, 1
diff --git a/arch/mips/net/ebpf_jit.c b/arch/mips/net/ebpf_jit.c
index 98bf0c222b5f..dfd5a4b1b779 100644
--- a/arch/mips/net/ebpf_jit.c
+++ b/arch/mips/net/ebpf_jit.c
@@ -22,6 +22,7 @@
 #include <asm/byteorder.h>
 #include <asm/cacheflush.h>
 #include <asm/cpu-features.h>
+#include <asm/isa-rev.h>
 #include <asm/uasm.h>
 
 /* Registers used by JIT */
@@ -125,15 +126,21 @@ static enum reg_val_type get_reg_val_type(const struct jit_ctx *ctx,
 }
 
 /* Simply emit the instruction if the JIT memory space has been allocated */
-#define emit_instr(ctx, func, ...)			\
-do {							\
-	if ((ctx)->target != NULL) {			\
-		u32 *p = &(ctx)->target[ctx->idx];	\
-		uasm_i_##func(&p, ##__VA_ARGS__);	\
-	}						\
-	(ctx)->idx++;					\
+#define emit_instr_long(ctx, func64, func32, ...)		\
+do {								\
+	if ((ctx)->target != NULL) {				\
+		u32 *p = &(ctx)->target[ctx->idx];		\
+		if (IS_ENABLED(CONFIG_64BIT))			\
+			uasm_i_##func64(&p, ##__VA_ARGS__);	\
+		else						\
+			uasm_i_##func32(&p, ##__VA_ARGS__);	\
+	}							\
+	(ctx)->idx++;						\
 } while (0)
 
+#define emit_instr(ctx, func, ...)				\
+	emit_instr_long(ctx, func, func, ##__VA_ARGS__)
+
 static unsigned int j_target(struct jit_ctx *ctx, int target_idx)
 {
 	unsigned long target_va, base_va;
@@ -274,17 +281,17 @@ static int gen_int_prologue(struct jit_ctx *ctx)
 		 * If RA we are doing a function call and may need
 		 * extra 8-byte tmp area.
 		 */
-		stack_adjust += 16;
+		stack_adjust += 2 * sizeof(long);
 	if (ctx->flags & EBPF_SAVE_S0)
-		stack_adjust += 8;
+		stack_adjust += sizeof(long);
 	if (ctx->flags & EBPF_SAVE_S1)
-		stack_adjust += 8;
+		stack_adjust += sizeof(long);
 	if (ctx->flags & EBPF_SAVE_S2)
-		stack_adjust += 8;
+		stack_adjust += sizeof(long);
 	if (ctx->flags & EBPF_SAVE_S3)
-		stack_adjust += 8;
+		stack_adjust += sizeof(long);
 	if (ctx->flags & EBPF_SAVE_S4)
-		stack_adjust += 8;
+		stack_adjust += sizeof(long);
 
 	BUILD_BUG_ON(MAX_BPF_STACK & 7);
 	locals_size = (ctx->flags & EBPF_SEEN_FP) ? MAX_BPF_STACK : 0;
@@ -298,41 +305,49 @@ static int gen_int_prologue(struct jit_ctx *ctx)
 	 * On tail call we skip this instruction, and the TCC is
 	 * passed in $v1 from the caller.
 	 */
-	emit_instr(ctx, daddiu, MIPS_R_V1, MIPS_R_ZERO, MAX_TAIL_CALL_CNT);
+	emit_instr(ctx, addiu, MIPS_R_V1, MIPS_R_ZERO, MAX_TAIL_CALL_CNT);
 	if (stack_adjust)
-		emit_instr(ctx, daddiu, MIPS_R_SP, MIPS_R_SP, -stack_adjust);
+		emit_instr_long(ctx, daddiu, addiu,
+					MIPS_R_SP, MIPS_R_SP, -stack_adjust);
 	else
 		return 0;
 
-	store_offset = stack_adjust - 8;
+	store_offset = stack_adjust - sizeof(long);
 
 	if (ctx->flags & EBPF_SAVE_RA) {
-		emit_instr(ctx, sd, MIPS_R_RA, store_offset, MIPS_R_SP);
-		store_offset -= 8;
+		emit_instr_long(ctx, sd, sw,
+					MIPS_R_RA, store_offset, MIPS_R_SP);
+		store_offset -= sizeof(long);
 	}
 	if (ctx->flags & EBPF_SAVE_S0) {
-		emit_instr(ctx, sd, MIPS_R_S0, store_offset, MIPS_R_SP);
-		store_offset -= 8;
+		emit_instr_long(ctx, sd, sw,
+					MIPS_R_S0, store_offset, MIPS_R_SP);
+		store_offset -= sizeof(long);
 	}
 	if (ctx->flags & EBPF_SAVE_S1) {
-		emit_instr(ctx, sd, MIPS_R_S1, store_offset, MIPS_R_SP);
-		store_offset -= 8;
+		emit_instr_long(ctx, sd, sw,
+					MIPS_R_S1, store_offset, MIPS_R_SP);
+		store_offset -= sizeof(long);
 	}
 	if (ctx->flags & EBPF_SAVE_S2) {
-		emit_instr(ctx, sd, MIPS_R_S2, store_offset, MIPS_R_SP);
-		store_offset -= 8;
+		emit_instr_long(ctx, sd, sw,
+					MIPS_R_S2, store_offset, MIPS_R_SP);
+		store_offset -= sizeof(long);
 	}
 	if (ctx->flags & EBPF_SAVE_S3) {
-		emit_instr(ctx, sd, MIPS_R_S3, store_offset, MIPS_R_SP);
-		store_offset -= 8;
+		emit_instr_long(ctx, sd, sw,
+					MIPS_R_S3, store_offset, MIPS_R_SP);
+		store_offset -= sizeof(long);
 	}
 	if (ctx->flags & EBPF_SAVE_S4) {
-		emit_instr(ctx, sd, MIPS_R_S4, store_offset, MIPS_R_SP);
-		store_offset -= 8;
+		emit_instr_long(ctx, sd, sw,
+					MIPS_R_S4, store_offset, MIPS_R_SP);
+		store_offset -= sizeof(long);
 	}
 
 	if ((ctx->flags & EBPF_SEEN_TC) && !(ctx->flags & EBPF_TCC_IN_V1))
-		emit_instr(ctx, daddu, MIPS_R_S4, MIPS_R_V1, MIPS_R_ZERO);
+		emit_instr_long(ctx, daddu, addu,
+					MIPS_R_S4, MIPS_R_V1, MIPS_R_ZERO);
 
 	return 0;
 }
@@ -341,7 +356,7 @@ static int build_int_epilogue(struct jit_ctx *ctx, int dest_reg)
 {
 	const struct bpf_prog *prog = ctx->skf;
 	int stack_adjust = ctx->stack_size;
-	int store_offset = stack_adjust - 8;
+	int store_offset = stack_adjust - sizeof(long);
 	enum reg_val_type td;
 	int r0 = MIPS_R_V0;
 
@@ -353,33 +368,40 @@ static int build_int_epilogue(struct jit_ctx *ctx, int dest_reg)
 	}
 
 	if (ctx->flags & EBPF_SAVE_RA) {
-		emit_instr(ctx, ld, MIPS_R_RA, store_offset, MIPS_R_SP);
-		store_offset -= 8;
+		emit_instr_long(ctx, ld, lw,
+					MIPS_R_RA, store_offset, MIPS_R_SP);
+		store_offset -= sizeof(long);
 	}
 	if (ctx->flags & EBPF_SAVE_S0) {
-		emit_instr(ctx, ld, MIPS_R_S0, store_offset, MIPS_R_SP);
-		store_offset -= 8;
+		emit_instr_long(ctx, ld, lw,
+					MIPS_R_S0, store_offset, MIPS_R_SP);
+		store_offset -= sizeof(long);
 	}
 	if (ctx->flags & EBPF_SAVE_S1) {
-		emit_instr(ctx, ld, MIPS_R_S1, store_offset, MIPS_R_SP);
-		store_offset -= 8;
+		emit_instr_long(ctx, ld, lw,
+					MIPS_R_S1, store_offset, MIPS_R_SP);
+		store_offset -= sizeof(long);
 	}
 	if (ctx->flags & EBPF_SAVE_S2) {
-		emit_instr(ctx, ld, MIPS_R_S2, store_offset, MIPS_R_SP);
-		store_offset -= 8;
+		emit_instr_long(ctx, ld, lw,
+				MIPS_R_S2, store_offset, MIPS_R_SP);
+		store_offset -= sizeof(long);
 	}
 	if (ctx->flags & EBPF_SAVE_S3) {
-		emit_instr(ctx, ld, MIPS_R_S3, store_offset, MIPS_R_SP);
-		store_offset -= 8;
+		emit_instr_long(ctx, ld, lw,
+					MIPS_R_S3, store_offset, MIPS_R_SP);
+		store_offset -= sizeof(long);
 	}
 	if (ctx->flags & EBPF_SAVE_S4) {
-		emit_instr(ctx, ld, MIPS_R_S4, store_offset, MIPS_R_SP);
-		store_offset -= 8;
+		emit_instr_long(ctx, ld, lw,
+					MIPS_R_S4, store_offset, MIPS_R_SP);
+		store_offset -= sizeof(long);
 	}
 	emit_instr(ctx, jr, dest_reg);
 
 	if (stack_adjust)
-		emit_instr(ctx, daddiu, MIPS_R_SP, MIPS_R_SP, stack_adjust);
+		emit_instr_long(ctx, daddiu, addiu,
+					MIPS_R_SP, MIPS_R_SP, stack_adjust);
 	else
 		emit_instr(ctx, nop);
 
@@ -646,6 +668,10 @@ static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
 	s64 t64s;
 	int bpf_op = BPF_OP(insn->code);
 
+	if (IS_ENABLED(CONFIG_32BIT) && ((BPF_CLASS(insn->code) == BPF_ALU64)
+						|| (bpf_op == BPF_DW)))
+		return -EINVAL;
+
 	switch (insn->code) {
 	case BPF_ALU64 | BPF_ADD | BPF_K: /* ALU64_IMM */
 	case BPF_ALU64 | BPF_SUB | BPF_K: /* ALU64_IMM */
@@ -678,8 +704,12 @@ static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
 		if (insn->imm == 1) /* Mult by 1 is a nop */
 			break;
 		gen_imm_to_reg(insn, MIPS_R_AT, ctx);
-		emit_instr(ctx, dmultu, MIPS_R_AT, dst);
-		emit_instr(ctx, mflo, dst);
+		if (MIPS_ISA_REV >= 6) {
+			emit_instr(ctx, dmulu, dst, dst, MIPS_R_AT);
+		} else {
+			emit_instr(ctx, dmultu, MIPS_R_AT, dst);
+			emit_instr(ctx, mflo, dst);
+		}
 		break;
 	case BPF_ALU64 | BPF_NEG | BPF_K: /* ALU64_IMM */
 		dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
@@ -701,8 +731,12 @@ static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
 		if (insn->imm == 1) /* Mult by 1 is a nop */
 			break;
 		gen_imm_to_reg(insn, MIPS_R_AT, ctx);
-		emit_instr(ctx, multu, dst, MIPS_R_AT);
-		emit_instr(ctx, mflo, dst);
+		if (MIPS_ISA_REV >= 6) {
+			emit_instr(ctx, mulu, dst, dst, MIPS_R_AT);
+		} else {
+			emit_instr(ctx, multu, dst, MIPS_R_AT);
+			emit_instr(ctx, mflo, dst);
+		}
 		break;
 	case BPF_ALU | BPF_NEG | BPF_K: /* ALU_IMM */
 		dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
@@ -733,6 +767,13 @@ static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
 			break;
 		}
 		gen_imm_to_reg(insn, MIPS_R_AT, ctx);
+		if (MIPS_ISA_REV >= 6) {
+			if (bpf_op == BPF_DIV)
+				emit_instr(ctx, divu_r6, dst, dst, MIPS_R_AT);
+			else
+				emit_instr(ctx, modu, dst, dst, MIPS_R_AT);
+			break;
+		}
 		emit_instr(ctx, divu, dst, MIPS_R_AT);
 		if (bpf_op == BPF_DIV)
 			emit_instr(ctx, mflo, dst);
@@ -755,6 +796,13 @@ static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
 			break;
 		}
 		gen_imm_to_reg(insn, MIPS_R_AT, ctx);
+		if (MIPS_ISA_REV >= 6) {
+			if (bpf_op == BPF_DIV)
+				emit_instr(ctx, ddivu_r6, dst, dst, MIPS_R_AT);
+			else
+				emit_instr(ctx, modu, dst, dst, MIPS_R_AT);
+			break;
+		}
 		emit_instr(ctx, ddivu, dst, MIPS_R_AT);
 		if (bpf_op == BPF_DIV)
 			emit_instr(ctx, mflo, dst);
@@ -820,11 +868,23 @@ static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
 			emit_instr(ctx, and, dst, dst, src);
 			break;
 		case BPF_MUL:
-			emit_instr(ctx, dmultu, dst, src);
-			emit_instr(ctx, mflo, dst);
+			if (MIPS_ISA_REV >= 6) {
+				emit_instr(ctx, dmulu, dst, dst, src);
+			} else {
+				emit_instr(ctx, dmultu, dst, src);
+				emit_instr(ctx, mflo, dst);
+			}
 			break;
 		case BPF_DIV:
 		case BPF_MOD:
+			if (MIPS_ISA_REV >= 6) {
+				if (bpf_op == BPF_DIV)
+					emit_instr(ctx, ddivu_r6,
+							dst, dst, src);
+				else
+					emit_instr(ctx, modu, dst, dst, src);
+				break;
+			}
 			emit_instr(ctx, ddivu, dst, src);
 			if (bpf_op == BPF_DIV)
 				emit_instr(ctx, mflo, dst);
@@ -904,6 +964,13 @@ static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
 			break;
 		case BPF_DIV:
 		case BPF_MOD:
+			if (MIPS_ISA_REV >= 6) {
+				if (bpf_op == BPF_DIV)
+					emit_instr(ctx, divu_r6, dst, dst, src);
+				else
+					emit_instr(ctx, modu, dst, dst, src);
+				break;
+			}
 			emit_instr(ctx, divu, dst, src);
 			if (bpf_op == BPF_DIV)
 				emit_instr(ctx, mflo, dst);
@@ -1007,8 +1074,15 @@ static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
 			emit_instr(ctx, dsubu, MIPS_R_T8, dst, src);
 			emit_instr(ctx, sltu, MIPS_R_AT, dst, src);
 			/* SP known to be non-zero, movz becomes boolean not */
-			emit_instr(ctx, movz, MIPS_R_T9, MIPS_R_SP, MIPS_R_T8);
-			emit_instr(ctx, movn, MIPS_R_T9, MIPS_R_ZERO, MIPS_R_T8);
+			if (MIPS_ISA_REV >= 6) {
+				emit_instr(ctx, seleqz, MIPS_R_T9,
+						MIPS_R_SP, MIPS_R_T8);
+			} else {
+				emit_instr(ctx, movz, MIPS_R_T9,
+						MIPS_R_SP, MIPS_R_T8);
+				emit_instr(ctx, movn, MIPS_R_T9,
+						MIPS_R_ZERO, MIPS_R_T8);
+			}
 			emit_instr(ctx, or, MIPS_R_AT, MIPS_R_T9, MIPS_R_AT);
 			cmp_eq = bpf_op == BPF_JGT;
 			dst = MIPS_R_AT;
@@ -1235,7 +1309,7 @@ jeq_common:
 
 	case BPF_JMP | BPF_CALL:
 		ctx->flags |= EBPF_SAVE_RA;
-		t64s = (s64)insn->imm + (s64)__bpf_call_base;
+		t64s = (s64)insn->imm + (long)__bpf_call_base;
 		emit_const_to_reg(ctx, MIPS_R_T9, (u64)t64s);
 		emit_instr(ctx, jalr, MIPS_R_RA, MIPS_R_T9);
 		/* delay slot */
@@ -1367,6 +1441,17 @@ jeq_common:
 		if (src < 0)
 			return src;
 		if (BPF_MODE(insn->code) == BPF_XADD) {
+			/*
+			 * If mem_off does not fit within the 9 bit ll/sc
+			 * instruction immediate field, use a temp reg.
+			 */
+			if (MIPS_ISA_REV >= 6 &&
+			    (mem_off >= BIT(8) || mem_off < -BIT(8))) {
+				emit_instr(ctx, daddiu, MIPS_R_T6,
+						dst, mem_off);
+				mem_off = 0;
+				dst = MIPS_R_T6;
+			}
 			switch (BPF_SIZE(insn->code)) {
 			case BPF_W:
 				if (get_reg_val_type(ctx, this_idx, insn->src_reg) == REG_32BIT) {
@@ -1721,7 +1806,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 	unsigned int image_size;
 	u8 *image_ptr;
 
-	if (!prog->jit_requested || !cpu_has_mips64r2)
+	if (!prog->jit_requested || MIPS_ISA_REV < 2)
 		return prog;
 
 	tmp = bpf_jit_blind_constants(prog);
diff --git a/arch/mips/pic32/Kconfig b/arch/mips/pic32/Kconfig
index e284e89183cc..7acbb50c1dcd 100644
--- a/arch/mips/pic32/Kconfig
+++ b/arch/mips/pic32/Kconfig
@@ -39,12 +39,12 @@ choice
 	  Select the devicetree.
 
 config DTB_PIC32_NONE
-       bool "None"
+	bool "None"
 
 config DTB_PIC32_MZDA_SK
-       bool "PIC32MZDA Starter Kit"
-       depends on PIC32MZDA
-       select BUILTIN_DTB
+	bool "PIC32MZDA Starter Kit"
+	depends on PIC32MZDA
+	select BUILTIN_DTB
 
 endchoice
 
diff --git a/arch/mips/vdso/Makefile b/arch/mips/vdso/Makefile
index 0ede4deb8181..7221df24cb23 100644
--- a/arch/mips/vdso/Makefile
+++ b/arch/mips/vdso/Makefile
@@ -46,9 +46,7 @@ endif
 VDSO_LDFLAGS := \
 	-Wl,-Bsymbolic -Wl,--no-undefined -Wl,-soname=linux-vdso.so.1 \
 	$(addprefix -Wl$(comma),$(filter -E%,$(KBUILD_CFLAGS))) \
-	-nostdlib -shared \
-	$(call cc-ldoption, -Wl$(comma)--hash-style=sysv) \
-	$(call cc-ldoption, -Wl$(comma)--build-id)
+	-nostdlib -shared -Wl,--hash-style=sysv -Wl,--build-id
 
 GCOV_PROFILE := n
 UBSAN_SANITIZE := n
diff --git a/arch/powerpc/platforms/512x/clock-commonclk.c b/arch/powerpc/platforms/512x/clock-commonclk.c
index b3097fe6441b..af265ae40a61 100644
--- a/arch/powerpc/platforms/512x/clock-commonclk.c
+++ b/arch/powerpc/platforms/512x/clock-commonclk.c
@@ -239,6 +239,7 @@ static inline struct clk *mpc512x_clk_divider(
 	const char *name, const char *parent_name, u8 clkflags,
 	u32 __iomem *reg, u8 pos, u8 len, int divflags)
 {
+	divflags |= CLK_DIVIDER_BIG_ENDIAN;
 	return clk_register_divider(NULL, name, parent_name, clkflags,
 				    reg, pos, len, divflags, &clklock);
 }
@@ -250,7 +251,7 @@ static inline struct clk *mpc512x_clk_divtable(
 {
 	u8 divflags;
 
-	divflags = 0;
+	divflags = CLK_DIVIDER_BIG_ENDIAN;
 	return clk_register_divider_table(NULL, name, parent_name, 0,
 					  reg, pos, len, divflags,
 					  divtab, &clklock);
@@ -261,10 +262,12 @@ static inline struct clk *mpc512x_clk_gated(
 	u32 __iomem *reg, u8 pos)
 {
 	int clkflags;
+	u8 gateflags;
 
 	clkflags = CLK_SET_RATE_PARENT;
+	gateflags = CLK_GATE_BIG_ENDIAN;
 	return clk_register_gate(NULL, name, parent_name, clkflags,
-				 reg, pos, 0, &clklock);
+				 reg, pos, gateflags, &clklock);
 }
 
 static inline struct clk *mpc512x_clk_muxed(const char *name,
@@ -275,7 +278,7 @@ static inline struct clk *mpc512x_clk_muxed(const char *name,
 	u8 muxflags;
 
 	clkflags = CLK_SET_RATE_PARENT;
-	muxflags = 0;
+	muxflags = CLK_MUX_BIG_ENDIAN;
 	return clk_register_mux(NULL, name,
 				parent_names, parent_count, clkflags,
 				reg, pos, len, muxflags, &clklock);
diff --git a/arch/sh/include/cpu-sh4/cpu/sh7786.h b/arch/sh/include/cpu-sh4/cpu/sh7786.h
index 8f9bfbf3cdb1..d6cce65b4871 100644
--- a/arch/sh/include/cpu-sh4/cpu/sh7786.h
+++ b/arch/sh/include/cpu-sh4/cpu/sh7786.h
@@ -132,7 +132,7 @@ enum {
 
 static inline u32 sh7786_mm_sel(void)
 {
-	return __raw_readl(0xFC400020) & 0x7;
+	return __raw_readl((const volatile void __iomem *)0xFC400020) & 0x7;
 }
 
 #endif /* __CPU_SH7786_H__ */
diff --git a/arch/sparc/kernel/cpumap.c b/arch/sparc/kernel/cpumap.c
index d1d52822603d..1cb62bfeaa1f 100644
--- a/arch/sparc/kernel/cpumap.c
+++ b/arch/sparc/kernel/cpumap.c
@@ -194,8 +194,7 @@ static struct cpuinfo_tree *build_cpuinfo_tree(void)
 
 	n = enumerate_cpuinfo_nodes(tmp_level);
 
-	new_tree = kzalloc(sizeof(struct cpuinfo_tree) +
-	                   (sizeof(struct cpuinfo_node) * n), GFP_ATOMIC);
+	new_tree = kzalloc(struct_size(new_tree, nodes, n), GFP_ATOMIC);
 	if (!new_tree)
 		return NULL;
 
diff --git a/arch/sparc/kernel/uprobes.c b/arch/sparc/kernel/uprobes.c
index d852ae56ddc1..c44bf5b85de8 100644
--- a/arch/sparc/kernel/uprobes.c
+++ b/arch/sparc/kernel/uprobes.c
@@ -29,7 +29,6 @@
 #include <linux/kdebug.h>
 
 #include <asm/cacheflush.h>
-#include <linux/uaccess.h>
 
 /* Compute the address of the breakpoint instruction and return it.
  *
diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index f2d70ff7a284..bc2aaa47bc8a 100644
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -2269,19 +2269,6 @@ static unsigned long last_valid_pfn;
 static void sun4u_pgprot_init(void);
 static void sun4v_pgprot_init(void);
 
-static phys_addr_t __init available_memory(void)
-{
-	phys_addr_t available = 0ULL;
-	phys_addr_t pa_start, pa_end;
-	u64 i;
-
-	for_each_free_mem_range(i, NUMA_NO_NODE, MEMBLOCK_NONE, &pa_start,
-				&pa_end, NULL)
-		available = available + (pa_end  - pa_start);
-
-	return available;
-}
-
 #define _PAGE_CACHE_4U	(_PAGE_CP_4U | _PAGE_CV_4U)
 #define _PAGE_CACHE_4V	(_PAGE_CP_4V | _PAGE_CV_4V)
 #define __DIRTY_BITS_4U	 (_PAGE_MODIFIED_4U | _PAGE_WRITE_4U | _PAGE_W_4U)
@@ -2295,33 +2282,8 @@ static phys_addr_t __init available_memory(void)
  */
 static void __init reduce_memory(phys_addr_t limit_ram)
 {
-	phys_addr_t avail_ram = available_memory();
-	phys_addr_t pa_start, pa_end;
-	u64 i;
-
-	if (limit_ram >= avail_ram)
-		return;
-
-	for_each_free_mem_range(i, NUMA_NO_NODE, MEMBLOCK_NONE, &pa_start,
-				&pa_end, NULL) {
-		phys_addr_t region_size = pa_end - pa_start;
-		phys_addr_t clip_start = pa_start;
-
-		avail_ram = avail_ram - region_size;
-		/* Are we consuming too much? */
-		if (avail_ram < limit_ram) {
-			phys_addr_t give_back = limit_ram - avail_ram;
-
-			region_size = region_size - give_back;
-			clip_start = clip_start + give_back;
-		}
-
-		memblock_remove(clip_start, region_size);
-
-		if (avail_ram <= limit_ram)
-			break;
-		i = 0UL;
-	}
+	limit_ram += memblock_reserved_size();
+	memblock_enforce_memory_limit(limit_ram);
 }
 
 void __init paging_init(void)
diff --git a/arch/sparc/mm/iommu.c b/arch/sparc/mm/iommu.c
index e8d5d73ca40d..71ac353032b6 100644
--- a/arch/sparc/mm/iommu.c
+++ b/arch/sparc/mm/iommu.c
@@ -175,16 +175,37 @@ static void iommu_flush_iotlb(iopte_t *iopte, unsigned int niopte)
 	}
 }
 
-static u32 iommu_get_one(struct device *dev, struct page *page, int npages)
+static dma_addr_t __sbus_iommu_map_page(struct device *dev, struct page *page,
+		unsigned long offset, size_t len, bool per_page_flush)
 {
 	struct iommu_struct *iommu = dev->archdata.iommu;
-	int ioptex;
-	iopte_t *iopte, *iopte0;
+	phys_addr_t paddr = page_to_phys(page) + offset;
+	unsigned long off = paddr & ~PAGE_MASK;
+	unsigned long npages = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
+	unsigned long pfn = __phys_to_pfn(paddr);
 	unsigned int busa, busa0;
-	int i;
+	iopte_t *iopte, *iopte0;
+	int ioptex, i;
+
+	/* XXX So what is maxphys for us and how do drivers know it? */
+	if (!len || len > 256 * 1024)
+		return DMA_MAPPING_ERROR;
+
+	/*
+	 * We expect unmapped highmem pages to be not in the cache.
+	 * XXX Is this a good assumption?
+	 * XXX What if someone else unmaps it here and races us?
+	 */
+	if (per_page_flush && !PageHighMem(page)) {
+		unsigned long vaddr, p;
+
+		vaddr = (unsigned long)page_address(page) + offset;
+		for (p = vaddr & PAGE_MASK; p < vaddr + len; p += PAGE_SIZE)
+			flush_page_for_dma(p);
+	}
 
 	/* page color = pfn of page */
-	ioptex = bit_map_string_get(&iommu->usemap, npages, page_to_pfn(page));
+	ioptex = bit_map_string_get(&iommu->usemap, npages, pfn);
 	if (ioptex < 0)
 		panic("iommu out");
 	busa0 = iommu->start + (ioptex << PAGE_SHIFT);
@@ -193,29 +214,15 @@ static u32 iommu_get_one(struct device *dev, struct page *page, int npages)
 	busa = busa0;
 	iopte = iopte0;
 	for (i = 0; i < npages; i++) {
-		iopte_val(*iopte) = MKIOPTE(page_to_pfn(page), IOPERM);
+		iopte_val(*iopte) = MKIOPTE(pfn, IOPERM);
 		iommu_invalidate_page(iommu->regs, busa);
 		busa += PAGE_SIZE;
 		iopte++;
-		page++;
+		pfn++;
 	}
 
 	iommu_flush_iotlb(iopte0, npages);
-
-	return busa0;
-}
-
-static dma_addr_t __sbus_iommu_map_page(struct device *dev, struct page *page,
-		unsigned long offset, size_t len)
-{
-	void *vaddr = page_address(page) + offset;
-	unsigned long off = (unsigned long)vaddr & ~PAGE_MASK;
-	unsigned long npages = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
-	
-	/* XXX So what is maxphys for us and how do drivers know it? */
-	if (!len || len > 256 * 1024)
-		return DMA_MAPPING_ERROR;
-	return iommu_get_one(dev, virt_to_page(vaddr), npages) + off;
+	return busa0 + off;
 }
 
 static dma_addr_t sbus_iommu_map_page_gflush(struct device *dev,
@@ -223,81 +230,58 @@ static dma_addr_t sbus_iommu_map_page_gflush(struct device *dev,
 		enum dma_data_direction dir, unsigned long attrs)
 {
 	flush_page_for_dma(0);
-	return __sbus_iommu_map_page(dev, page, offset, len);
+	return __sbus_iommu_map_page(dev, page, offset, len, false);
 }
 
 static dma_addr_t sbus_iommu_map_page_pflush(struct device *dev,
 		struct page *page, unsigned long offset, size_t len,
 		enum dma_data_direction dir, unsigned long attrs)
 {
-	void *vaddr = page_address(page) + offset;
-	unsigned long p = ((unsigned long)vaddr) & PAGE_MASK;
-
-	while (p < (unsigned long)vaddr + len) {
-		flush_page_for_dma(p);
-		p += PAGE_SIZE;
-	}
-
-	return __sbus_iommu_map_page(dev, page, offset, len);
+	return __sbus_iommu_map_page(dev, page, offset, len, true);
 }
 
-static int sbus_iommu_map_sg_gflush(struct device *dev, struct scatterlist *sgl,
-		int nents, enum dma_data_direction dir, unsigned long attrs)
+static int __sbus_iommu_map_sg(struct device *dev, struct scatterlist *sgl,
+		int nents, enum dma_data_direction dir, unsigned long attrs,
+		bool per_page_flush)
 {
 	struct scatterlist *sg;
-	int i, n;
-
-	flush_page_for_dma(0);
+	int j;
 
-	for_each_sg(sgl, sg, nents, i) {
-		n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT;
-		sg->dma_address = iommu_get_one(dev, sg_page(sg), n) + sg->offset;
+	for_each_sg(sgl, sg, nents, j) {
+		sg->dma_address =__sbus_iommu_map_page(dev, sg_page(sg),
+				sg->offset, sg->length, per_page_flush);
+		if (sg->dma_address == DMA_MAPPING_ERROR)
+			return 0;
 		sg->dma_length = sg->length;
 	}
 
 	return nents;
 }
 
-static int sbus_iommu_map_sg_pflush(struct device *dev, struct scatterlist *sgl,
+static int sbus_iommu_map_sg_gflush(struct device *dev, struct scatterlist *sgl,
 		int nents, enum dma_data_direction dir, unsigned long attrs)
 {
-	unsigned long page, oldpage = 0;
-	struct scatterlist *sg;
-	int i, j, n;
-
-	for_each_sg(sgl, sg, nents, j) {
-		n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT;
-
-		/*
-		 * We expect unmapped highmem pages to be not in the cache.
-		 * XXX Is this a good assumption?
-		 * XXX What if someone else unmaps it here and races us?
-		 */
-		if ((page = (unsigned long) page_address(sg_page(sg))) != 0) {
-			for (i = 0; i < n; i++) {
-				if (page != oldpage) {	/* Already flushed? */
-					flush_page_for_dma(page);
-					oldpage = page;
-				}
-				page += PAGE_SIZE;
-			}
-		}
-
-		sg->dma_address = iommu_get_one(dev, sg_page(sg), n) + sg->offset;
-		sg->dma_length = sg->length;
-	}
+	flush_page_for_dma(0);
+	return __sbus_iommu_map_sg(dev, sgl, nents, dir, attrs, false);
+}
 
-	return nents;
+static int sbus_iommu_map_sg_pflush(struct device *dev, struct scatterlist *sgl,
+		int nents, enum dma_data_direction dir, unsigned long attrs)
+{
+	return __sbus_iommu_map_sg(dev, sgl, nents, dir, attrs, true);
 }
 
-static void iommu_release_one(struct device *dev, u32 busa, int npages)
+static void sbus_iommu_unmap_page(struct device *dev, dma_addr_t dma_addr,
+		size_t len, enum dma_data_direction dir, unsigned long attrs)
 {
 	struct iommu_struct *iommu = dev->archdata.iommu;
-	int ioptex;
-	int i;
+	unsigned int busa = dma_addr & PAGE_MASK;
+	unsigned long off = dma_addr & ~PAGE_MASK;
+	unsigned int npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT;
+	unsigned int ioptex = (busa - iommu->start) >> PAGE_SHIFT;
+	unsigned int i;
 
 	BUG_ON(busa < iommu->start);
-	ioptex = (busa - iommu->start) >> PAGE_SHIFT;
 	for (i = 0; i < npages; i++) {
 		iopte_val(iommu->page_table[ioptex + i]) = 0;
 		iommu_invalidate_page(iommu->regs, busa);
@@ -306,25 +290,15 @@ static void iommu_release_one(struct device *dev, u32 busa, int npages)
 	bit_map_clear(&iommu->usemap, ioptex, npages);
 }
 
-static void sbus_iommu_unmap_page(struct device *dev, dma_addr_t dma_addr,
-		size_t len, enum dma_data_direction dir, unsigned long attrs)
-{
-	unsigned long off = dma_addr & ~PAGE_MASK;
-	int npages;
-
-	npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT;
-	iommu_release_one(dev, dma_addr & PAGE_MASK, npages);
-}
-
 static void sbus_iommu_unmap_sg(struct device *dev, struct scatterlist *sgl,
 		int nents, enum dma_data_direction dir, unsigned long attrs)
 {
 	struct scatterlist *sg;
-	int i, n;
+	int i;
 
 	for_each_sg(sgl, sg, nents, i) {
-		n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT;
-		iommu_release_one(dev, sg->dma_address & PAGE_MASK, n);
+		sbus_iommu_unmap_page(dev, sg->dma_address, sg->length, dir,
+				attrs);
 		sg->dma_address = 0x21212121;
 	}
 }
diff --git a/arch/sparc/vdso/Makefile b/arch/sparc/vdso/Makefile
index 74e97f77e23b..83c4b463cb3d 100644
--- a/arch/sparc/vdso/Makefile
+++ b/arch/sparc/vdso/Makefile
@@ -68,7 +68,7 @@ CFLAGS_REMOVE_vdso-note.o = -pg
 CFLAGS_REMOVE_vclock_gettime.o = -pg
 
 $(obj)/%.so: OBJCOPYFLAGS := -S
-$(obj)/%.so: $(obj)/%.so.dbg
+$(obj)/%.so: $(obj)/%.so.dbg FORCE
 	$(call if_changed,objcopy)
 
 CPPFLAGS_vdso32.lds = $(CPPFLAGS_vdso.lds)
diff --git a/arch/x86/include/asm/dma-mapping.h b/arch/x86/include/asm/dma-mapping.h
index ce4d176b3d13..6b15a24930e0 100644
--- a/arch/x86/include/asm/dma-mapping.h
+++ b/arch/x86/include/asm/dma-mapping.h
@@ -13,14 +13,7 @@
 #include <asm/swiotlb.h>
 #include <linux/dma-contiguous.h>
 
-#ifdef CONFIG_ISA
-# define ISA_DMA_BIT_MASK DMA_BIT_MASK(24)
-#else
-# define ISA_DMA_BIT_MASK DMA_BIT_MASK(32)
-#endif
-
 extern int iommu_merge;
-extern struct device x86_dma_fallback_dev;
 extern int panic_on_overflow;
 
 extern const struct dma_map_ops *dma_ops;
@@ -30,7 +23,4 @@ static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
 	return dma_ops;
 }
 
-bool arch_dma_alloc_attrs(struct device **dev);
-#define arch_dma_alloc_attrs arch_dma_alloc_attrs
-
 #endif
diff --git a/arch/x86/kernel/amd_gart_64.c b/arch/x86/kernel/amd_gart_64.c
index 2c0aa34af69c..bf7f13ea3c64 100644
--- a/arch/x86/kernel/amd_gart_64.c
+++ b/arch/x86/kernel/amd_gart_64.c
@@ -233,9 +233,6 @@ static dma_addr_t gart_map_page(struct device *dev, struct page *page,
 	unsigned long bus;
 	phys_addr_t paddr = page_to_phys(page) + offset;
 
-	if (!dev)
-		dev = &x86_dma_fallback_dev;
-
 	if (!need_iommu(dev, paddr, size))
 		return paddr;
 
@@ -392,9 +389,6 @@ static int gart_map_sg(struct device *dev, struct scatterlist *sg, int nents,
 	if (nents == 0)
 		return 0;
 
-	if (!dev)
-		dev = &x86_dma_fallback_dev;
-
 	out		= 0;
 	start		= 0;
 	start_sg	= sg;
diff --git a/arch/x86/kernel/early-quirks.c b/arch/x86/kernel/early-quirks.c
index 50d5848bf22e..6c4f01540833 100644
--- a/arch/x86/kernel/early-quirks.c
+++ b/arch/x86/kernel/early-quirks.c
@@ -525,7 +525,8 @@ static const struct pci_device_id intel_early_ids[] __initconst = {
 	INTEL_I945G_IDS(&gen3_early_ops),
 	INTEL_I945GM_IDS(&gen3_early_ops),
 	INTEL_VLV_IDS(&gen6_early_ops),
-	INTEL_PINEVIEW_IDS(&gen3_early_ops),
+	INTEL_PINEVIEW_G_IDS(&gen3_early_ops),
+	INTEL_PINEVIEW_M_IDS(&gen3_early_ops),
 	INTEL_I965G_IDS(&gen3_early_ops),
 	INTEL_G33_IDS(&gen3_early_ops),
 	INTEL_I965GM_IDS(&gen3_early_ops),
@@ -547,6 +548,7 @@ static const struct pci_device_id intel_early_ids[] __initconst = {
 	INTEL_GLK_IDS(&gen9_early_ops),
 	INTEL_CNL_IDS(&gen9_early_ops),
 	INTEL_ICL_11_IDS(&gen11_early_ops),
+	INTEL_EHL_IDS(&gen11_early_ops),
 };
 
 struct resource intel_graphics_stolen_res __ro_after_init = DEFINE_RES_MEM(0, 0);
diff --git a/arch/x86/kernel/ima_arch.c b/arch/x86/kernel/ima_arch.c
index e47cd9390ab4..85de790583f9 100644
--- a/arch/x86/kernel/ima_arch.c
+++ b/arch/x86/kernel/ima_arch.c
@@ -3,6 +3,7 @@
  * Copyright (C) 2018 IBM Corporation
  */
 #include <linux/efi.h>
+#include <linux/module.h>
 #include <linux/ima.h>
 
 extern struct boot_params boot_params;
@@ -64,12 +65,19 @@ static const char * const sb_arch_rules[] = {
 	"appraise func=KEXEC_KERNEL_CHECK appraise_type=imasig",
 #endif /* CONFIG_KEXEC_VERIFY_SIG */
 	"measure func=KEXEC_KERNEL_CHECK",
+#if !IS_ENABLED(CONFIG_MODULE_SIG)
+	"appraise func=MODULE_CHECK appraise_type=imasig",
+#endif
+	"measure func=MODULE_CHECK",
 	NULL
 };
 
 const char * const *arch_get_ima_policy(void)
 {
-	if (IS_ENABLED(CONFIG_IMA_ARCH_POLICY) && arch_ima_get_secureboot())
+	if (IS_ENABLED(CONFIG_IMA_ARCH_POLICY) && arch_ima_get_secureboot()) {
+		if (IS_ENABLED(CONFIG_MODULE_SIG))
+			set_module_sig_enforced();
 		return sb_arch_rules;
+	}
 	return NULL;
 }
diff --git a/arch/x86/kernel/pci-dma.c b/arch/x86/kernel/pci-dma.c
index d460998ae828..dcd272dbd0a9 100644
--- a/arch/x86/kernel/pci-dma.c
+++ b/arch/x86/kernel/pci-dma.c
@@ -51,14 +51,6 @@ int iommu_pass_through __read_mostly;
 
 extern struct iommu_table_entry __iommu_table[], __iommu_table_end[];
 
-/* Dummy device used for NULL arguments (normally ISA). */
-struct device x86_dma_fallback_dev = {
-	.init_name = "fallback device",
-	.coherent_dma_mask = ISA_DMA_BIT_MASK,
-	.dma_mask = &x86_dma_fallback_dev.coherent_dma_mask,
-};
-EXPORT_SYMBOL(x86_dma_fallback_dev);
-
 void __init pci_iommu_alloc(void)
 {
 	struct iommu_table_entry *p;
@@ -77,18 +69,6 @@ void __init pci_iommu_alloc(void)
 	}
 }
 
-bool arch_dma_alloc_attrs(struct device **dev)
-{
-	if (!*dev)
-		*dev = &x86_dma_fallback_dev;
-
-	if (!is_device_dma_capable(*dev))
-		return false;
-	return true;
-
-}
-EXPORT_SYMBOL(arch_dma_alloc_attrs);
-
 /*
  * See <Documentation/x86/x86_64/boot-options.txt> for the iommu kernel
  * parameter documentation.
diff --git a/drivers/char/ipmi/ipmi_devintf.c b/drivers/char/ipmi/ipmi_devintf.c
index 99c9f581a1f3..f7b1c004a12b 100644
--- a/drivers/char/ipmi/ipmi_devintf.c
+++ b/drivers/char/ipmi/ipmi_devintf.c
@@ -29,7 +29,6 @@ struct ipmi_file_private
 	struct ipmi_user     *user;
 	spinlock_t           recv_msg_lock;
 	struct list_head     recv_msgs;
-	struct file          *file;
 	struct fasync_struct *fasync_queue;
 	wait_queue_head_t    wait;
 	struct mutex	     recv_mutex;
@@ -95,8 +94,6 @@ static int ipmi_open(struct inode *inode, struct file *file)
 	if (!priv)
 		return -ENOMEM;
 
-	priv->file = file;
-
 	rv = ipmi_create_user(if_num,
 			      &ipmi_hndlrs,
 			      priv,
diff --git a/drivers/char/ipmi/ipmi_dmi.c b/drivers/char/ipmi/ipmi_dmi.c
index f2411468f33f..f38e651dd1b5 100644
--- a/drivers/char/ipmi/ipmi_dmi.c
+++ b/drivers/char/ipmi/ipmi_dmi.c
@@ -47,9 +47,11 @@ static void __init dmi_add_platform_ipmi(unsigned long base_addr,
 	memset(&p, 0, sizeof(p));
 
 	name = "dmi-ipmi-si";
+	p.iftype = IPMI_PLAT_IF_SI;
 	switch (type) {
 	case IPMI_DMI_TYPE_SSIF:
 		name = "dmi-ipmi-ssif";
+		p.iftype = IPMI_PLAT_IF_SSIF;
 		p.type = SI_TYPE_INVALID;
 		break;
 	case IPMI_DMI_TYPE_BT:
diff --git a/drivers/char/ipmi/ipmi_msghandler.c b/drivers/char/ipmi/ipmi_msghandler.c
index 00bf4b17edbf..1dc10740fc0f 100644
--- a/drivers/char/ipmi/ipmi_msghandler.c
+++ b/drivers/char/ipmi/ipmi_msghandler.c
@@ -635,7 +635,7 @@ static DEFINE_MUTEX(ipmidriver_mutex);
 
 static LIST_HEAD(ipmi_interfaces);
 static DEFINE_MUTEX(ipmi_interfaces_mutex);
-struct srcu_struct ipmi_interfaces_srcu;
+static struct srcu_struct ipmi_interfaces_srcu;
 
 /*
  * List of watchers that want to know when smi's are added and deleted.
@@ -5179,7 +5179,7 @@ static void __exit cleanup_ipmi(void)
 		 * avoids problems with race conditions removing the timer
 		 * here.
 		 */
-		atomic_inc(&stop_operation);
+		atomic_set(&stop_operation, 1);
 		del_timer_sync(&ipmi_timer);
 
 		initialized = false;
diff --git a/drivers/char/ipmi/ipmi_plat_data.c b/drivers/char/ipmi/ipmi_plat_data.c
index 8f0ca2a848eb..28471ff2a3a3 100644
--- a/drivers/char/ipmi/ipmi_plat_data.c
+++ b/drivers/char/ipmi/ipmi_plat_data.c
@@ -12,7 +12,7 @@ struct platform_device *ipmi_platform_add(const char *name, unsigned int inst,
 					  struct ipmi_plat_data *p)
 {
 	struct platform_device *pdev;
-	unsigned int num_r = 1, size, pidx = 0;
+	unsigned int num_r = 1, size = 0, pidx = 0;
 	struct resource r[4];
 	struct property_entry pr[6];
 	u32 flags;
@@ -21,19 +21,22 @@ struct platform_device *ipmi_platform_add(const char *name, unsigned int inst,
 	memset(pr, 0, sizeof(pr));
 	memset(r, 0, sizeof(r));
 
-	if (p->type == SI_BT)
-		size = 3;
-	else if (p->type == SI_TYPE_INVALID)
-		size = 0;
-	else
-		size = 2;
+	if (p->iftype == IPMI_PLAT_IF_SI) {
+		if (p->type == SI_BT)
+			size = 3;
+		else if (p->type != SI_TYPE_INVALID)
+			size = 2;
+
+		if (p->regsize == 0)
+			p->regsize = DEFAULT_REGSIZE;
+		if (p->regspacing == 0)
+			p->regspacing = p->regsize;
 
-	if (p->regsize == 0)
-		p->regsize = DEFAULT_REGSIZE;
-	if (p->regspacing == 0)
-		p->regspacing = p->regsize;
+		pr[pidx++] = PROPERTY_ENTRY_U8("ipmi-type", p->type);
+	} else if (p->iftype == IPMI_PLAT_IF_SSIF) {
+		pr[pidx++] = PROPERTY_ENTRY_U16("i2c-addr", p->addr);
+	}
 
-	pr[pidx++] = PROPERTY_ENTRY_U8("ipmi-type", p->type);
 	if (p->slave_addr)
 		pr[pidx++] = PROPERTY_ENTRY_U8("slave-addr", p->slave_addr);
 	pr[pidx++] = PROPERTY_ENTRY_U8("addr-source", p->addr_source);
diff --git a/drivers/char/ipmi/ipmi_plat_data.h b/drivers/char/ipmi/ipmi_plat_data.h
index 567cfcec8ada..9ba744ea9571 100644
--- a/drivers/char/ipmi/ipmi_plat_data.h
+++ b/drivers/char/ipmi/ipmi_plat_data.h
@@ -6,7 +6,10 @@
 
 #include <linux/ipmi.h>
 
+enum ipmi_plat_interface_type { IPMI_PLAT_IF_SI, IPMI_PLAT_IF_SSIF };
+
 struct ipmi_plat_data {
+	enum ipmi_plat_interface_type iftype;
 	unsigned int type; /* si_type for si, SI_INVALID for others */
 	unsigned int space; /* addr_space for si, intf# for ssif. */
 	unsigned long addr;
diff --git a/drivers/char/ipmi/ipmi_si_hardcode.c b/drivers/char/ipmi/ipmi_si_hardcode.c
index 682221eebd66..f6ece7569504 100644
--- a/drivers/char/ipmi/ipmi_si_hardcode.c
+++ b/drivers/char/ipmi/ipmi_si_hardcode.c
@@ -83,6 +83,7 @@ static void __init ipmi_hardcode_init_one(const char *si_type_str,
 
 	memset(&p, 0, sizeof(p));
 
+	p.iftype = IPMI_PLAT_IF_SI;
 	if (!si_type_str || !*si_type_str || strcmp(si_type_str, "kcs") == 0) {
 		p.type = SI_KCS;
 	} else if (strcmp(si_type_str, "smic") == 0) {
diff --git a/drivers/char/ipmi/ipmi_si_hotmod.c b/drivers/char/ipmi/ipmi_si_hotmod.c
index 03140f6cdf6f..42a925f8cf69 100644
--- a/drivers/char/ipmi/ipmi_si_hotmod.c
+++ b/drivers/char/ipmi/ipmi_si_hotmod.c
@@ -108,6 +108,7 @@ static int parse_hotmod_str(const char *curr, enum hotmod_op *op,
 	int rv;
 	unsigned int ival;
 
+	h->iftype = IPMI_PLAT_IF_SI;
 	rv = parse_str(hotmod_ops, &ival, "operation", &curr);
 	if (rv)
 		return rv;
diff --git a/drivers/char/ipmi/ipmi_si_intf.c b/drivers/char/ipmi/ipmi_si_intf.c
index b1732882b97e..f124a2d2bb9f 100644
--- a/drivers/char/ipmi/ipmi_si_intf.c
+++ b/drivers/char/ipmi/ipmi_si_intf.c
@@ -1931,7 +1931,6 @@ static int try_smi_init(struct smi_info *new_smi)
 {
 	int rv = 0;
 	int i;
-	char *init_name = NULL;
 
 	pr_info("Trying %s-specified %s state machine at %s address 0x%lx, slave address 0x%x, irq %d\n",
 		ipmi_addr_src_to_str(new_smi->io.addr_source),
@@ -2073,7 +2072,6 @@ static int try_smi_init(struct smi_info *new_smi)
 		new_smi->io.io_cleanup = NULL;
 	}
 
-	kfree(init_name);
 	return rv;
 }
 
diff --git a/drivers/char/ipmi/ipmi_si_platform.c b/drivers/char/ipmi/ipmi_si_platform.c
index 54c7ded2a1ff..f2a91c4d8cab 100644
--- a/drivers/char/ipmi/ipmi_si_platform.c
+++ b/drivers/char/ipmi/ipmi_si_platform.c
@@ -188,12 +188,10 @@ static int platform_ipmi_probe(struct platform_device *pdev)
 		return -EINVAL;
 
 	rv = device_property_read_u8(&pdev->dev, "slave-addr", &slave_addr);
-	if (rv) {
-		dev_warn(&pdev->dev, "device has no slave-addr property\n");
+	if (rv)
 		io.slave_addr = 0x20;
-	} else {
+	else
 		io.slave_addr = slave_addr;
-	}
 
 	io.irq = platform_get_irq(pdev, 0);
 	if (io.irq > 0)
diff --git a/drivers/char/ipmi/ipmi_ssif.c b/drivers/char/ipmi/ipmi_ssif.c
index 8b5aec5430f1..cf8156d6bc07 100644
--- a/drivers/char/ipmi/ipmi_ssif.c
+++ b/drivers/char/ipmi/ipmi_ssif.c
@@ -727,12 +727,16 @@ static void msg_done_handler(struct ssif_info *ssif_info, int result,
 			/* End of read */
 			len = ssif_info->multi_len;
 			data = ssif_info->data;
-		} else if (blocknum != ssif_info->multi_pos) {
+		} else if (blocknum + 1 != ssif_info->multi_pos) {
 			/*
 			 * Out of sequence block, just abort.  Block
 			 * numbers start at zero for the second block,
 			 * but multi_pos starts at one, so the +1.
 			 */
+			if (ssif_info->ssif_debug & SSIF_DEBUG_MSG)
+				dev_dbg(&ssif_info->client->dev,
+					"Received message out of sequence, expected %u, got %u\n",
+					ssif_info->multi_pos - 1, blocknum);
 			result = -EIO;
 		} else {
 			ssif_inc_stat(ssif_info, received_message_parts);
@@ -1991,7 +1995,7 @@ static int dmi_ipmi_probe(struct platform_device *pdev)
 
 	rv = device_property_read_u8(&pdev->dev, "slave-addr", &slave_addr);
 	if (rv)
-		dev_warn(&pdev->dev, "device has no slave-addr property");
+		slave_addr = 0x20;
 
 	return new_ssif_client(i2c_addr, NULL, 0,
 			       slave_addr, SI_SMBIOS, &pdev->dev);
@@ -2107,7 +2111,8 @@ static void cleanup_ipmi_ssif(void)
 
 	kfree(ssif_i2c_driver.address_list);
 
-	platform_driver_unregister(&ipmi_driver);
+	if (ssif_trydmi)
+		platform_driver_unregister(&ipmi_driver);
 
 	free_ssif_clients();
 }
diff --git a/drivers/char/virtio_console.c b/drivers/char/virtio_console.c
index fbeb71953526..05dbfdb9f4af 100644
--- a/drivers/char/virtio_console.c
+++ b/drivers/char/virtio_console.c
@@ -75,7 +75,7 @@ struct ports_driver_data {
 	/* All the console devices handled by this driver */
 	struct list_head consoles;
 };
-static struct ports_driver_data pdrvdata;
+static struct ports_driver_data pdrvdata = { .next_vtermno = 1};
 
 static DEFINE_SPINLOCK(pdrvdata_lock);
 static DECLARE_COMPLETION(early_console_added);
@@ -1394,6 +1394,7 @@ static int add_port(struct ports_device *portdev, u32 id)
 	port->async_queue = NULL;
 
 	port->cons.ws.ws_row = port->cons.ws.ws_col = 0;
+	port->cons.vtermno = 0;
 
 	port->host_connected = port->guest_connected = false;
 	port->stats = (struct port_stats) { 0 };
diff --git a/drivers/clk/Kconfig b/drivers/clk/Kconfig
index e705aab9e38b..fc1e0cf44995 100644
--- a/drivers/clk/Kconfig
+++ b/drivers/clk/Kconfig
@@ -1,3 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
 
 config CLKDEV_LOOKUP
 	bool
@@ -219,6 +220,13 @@ config COMMON_CLK_XGENE
 	---help---
 	  Sypport for the APM X-Gene SoC reference, PLL, and device clocks.
 
+config COMMON_CLK_LOCHNAGAR
+	tristate "Cirrus Logic Lochnagar clock driver"
+	depends on MFD_LOCHNAGAR
+	help
+	  This driver supports the clocking features of the Cirrus Logic
+	  Lochnagar audio development board.
+
 config COMMON_CLK_NXP
 	def_bool COMMON_CLK && (ARCH_LPC18XX || ARCH_LPC32XX)
 	select REGMAP_MMIO if ARCH_LPC32XX
@@ -297,6 +305,7 @@ config COMMON_CLK_FIXED_MMIO
 	  Support for Memory Mapped IO Fixed clocks
 
 source "drivers/clk/actions/Kconfig"
+source "drivers/clk/analogbits/Kconfig"
 source "drivers/clk/bcm/Kconfig"
 source "drivers/clk/hisilicon/Kconfig"
 source "drivers/clk/imgtec/Kconfig"
@@ -309,7 +318,9 @@ source "drivers/clk/mvebu/Kconfig"
 source "drivers/clk/qcom/Kconfig"
 source "drivers/clk/renesas/Kconfig"
 source "drivers/clk/samsung/Kconfig"
+source "drivers/clk/sifive/Kconfig"
 source "drivers/clk/sprd/Kconfig"
+source "drivers/clk/sunxi/Kconfig"
 source "drivers/clk/sunxi-ng/Kconfig"
 source "drivers/clk/tegra/Kconfig"
 source "drivers/clk/ti/Kconfig"
diff --git a/drivers/clk/Makefile b/drivers/clk/Makefile
index 1db133652f0c..9ef4305d55e0 100644
--- a/drivers/clk/Makefile
+++ b/drivers/clk/Makefile
@@ -32,8 +32,10 @@ obj-$(CONFIG_COMMON_CLK_GEMINI)		+= clk-gemini.o
 obj-$(CONFIG_COMMON_CLK_ASPEED)		+= clk-aspeed.o
 obj-$(CONFIG_ARCH_HIGHBANK)		+= clk-highbank.o
 obj-$(CONFIG_CLK_HSDK)			+= clk-hsdk-pll.o
+obj-$(CONFIG_COMMON_CLK_LOCHNAGAR)	+= clk-lochnagar.o
 obj-$(CONFIG_COMMON_CLK_MAX77686)	+= clk-max77686.o
 obj-$(CONFIG_COMMON_CLK_MAX9485)	+= clk-max9485.o
+obj-$(CONFIG_ARCH_MILBEAUT_M10V)	+= clk-milbeaut.o
 obj-$(CONFIG_ARCH_MOXART)		+= clk-moxart.o
 obj-$(CONFIG_ARCH_NOMADIK)		+= clk-nomadik.o
 obj-$(CONFIG_ARCH_NPCM7XX)	    	+= clk-npcm7xx.o
@@ -64,6 +66,7 @@ obj-$(CONFIG_COMMON_CLK_XGENE)		+= clk-xgene.o
 
 # please keep this section sorted lexicographically by directory path name
 obj-y					+= actions/
+obj-y					+= analogbits/
 obj-$(CONFIG_COMMON_CLK_AT91)		+= at91/
 obj-$(CONFIG_ARCH_ARTPEC)		+= axis/
 obj-$(CONFIG_ARC_PLAT_AXS10X)		+= axs10x/
@@ -93,6 +96,7 @@ obj-$(CONFIG_COMMON_CLK_QCOM)		+= qcom/
 obj-y					+= renesas/
 obj-$(CONFIG_ARCH_ROCKCHIP)		+= rockchip/
 obj-$(CONFIG_COMMON_CLK_SAMSUNG)	+= samsung/
+obj-$(CONFIG_CLK_SIFIVE)		+= sifive/
 obj-$(CONFIG_ARCH_SIRF)			+= sirf/
 obj-$(CONFIG_ARCH_SOCFPGA)		+= socfpga/
 obj-$(CONFIG_PLAT_SPEAR)		+= spear/
diff --git a/drivers/clk/actions/owl-common.h b/drivers/clk/actions/owl-common.h
index 5a866a8b913d..c000a431471e 100644
--- a/drivers/clk/actions/owl-common.h
+++ b/drivers/clk/actions/owl-common.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 //
 // OWL common clock driver
 //
diff --git a/drivers/clk/actions/owl-composite.h b/drivers/clk/actions/owl-composite.h
index b410ed5bf308..bca38bf8f218 100644
--- a/drivers/clk/actions/owl-composite.h
+++ b/drivers/clk/actions/owl-composite.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 //
 // OWL composite clock driver
 //
diff --git a/drivers/clk/actions/owl-divider.h b/drivers/clk/actions/owl-divider.h
index 92d3e3d23967..083be6d80954 100644
--- a/drivers/clk/actions/owl-divider.h
+++ b/drivers/clk/actions/owl-divider.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 //
 // OWL divider clock driver
 //
diff --git a/drivers/clk/actions/owl-factor.h b/drivers/clk/actions/owl-factor.h
index f1a7ffe896e1..04b89cbfdccb 100644
--- a/drivers/clk/actions/owl-factor.h
+++ b/drivers/clk/actions/owl-factor.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 //
 // OWL factor clock driver
 //
diff --git a/drivers/clk/actions/owl-fixed-factor.h b/drivers/clk/actions/owl-fixed-factor.h
index cc9fe36c0964..3dfd7fd7d292 100644
--- a/drivers/clk/actions/owl-fixed-factor.h
+++ b/drivers/clk/actions/owl-fixed-factor.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 //
 // OWL fixed factor clock driver
 //
diff --git a/drivers/clk/actions/owl-gate.h b/drivers/clk/actions/owl-gate.h
index c2d61ceebce2..c2f161c93fda 100644
--- a/drivers/clk/actions/owl-gate.h
+++ b/drivers/clk/actions/owl-gate.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 //
 // OWL gate clock driver
 //
diff --git a/drivers/clk/actions/owl-mux.h b/drivers/clk/actions/owl-mux.h
index 834284c8c3ae..53b9ab665294 100644
--- a/drivers/clk/actions/owl-mux.h
+++ b/drivers/clk/actions/owl-mux.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 //
 // OWL mux clock driver
 //
diff --git a/drivers/clk/actions/owl-pll.h b/drivers/clk/actions/owl-pll.h
index 6fb0d45bb088..78e5fc360b03 100644
--- a/drivers/clk/actions/owl-pll.h
+++ b/drivers/clk/actions/owl-pll.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0+
+/* SPDX-License-Identifier: GPL-2.0+ */
 //
 // OWL pll clock driver
 //
diff --git a/drivers/clk/actions/owl-reset.h b/drivers/clk/actions/owl-reset.h
index 10f5774979a6..a947ffcb5a02 100644
--- a/drivers/clk/actions/owl-reset.h
+++ b/drivers/clk/actions/owl-reset.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
+/* SPDX-License-Identifier: GPL-2.0-or-later */
 //
 // Actions Semi Owl SoCs Reset Management Unit driver
 //
diff --git a/drivers/clk/analogbits/Kconfig b/drivers/clk/analogbits/Kconfig
new file mode 100644
index 000000000000..b5fd60c7f136
--- /dev/null
+++ b/drivers/clk/analogbits/Kconfig
@@ -0,0 +1,2 @@
+config CLK_ANALOGBITS_WRPLL_CLN28HPC
+	bool
diff --git a/drivers/clk/analogbits/Makefile b/drivers/clk/analogbits/Makefile
new file mode 100644
index 000000000000..bf017447451e
--- /dev/null
+++ b/drivers/clk/analogbits/Makefile
@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: GPL-2.0
+
+obj-$(CONFIG_CLK_ANALOGBITS_WRPLL_CLN28HPC)	+= wrpll-cln28hpc.o
diff --git a/drivers/clk/analogbits/wrpll-cln28hpc.c b/drivers/clk/analogbits/wrpll-cln28hpc.c
new file mode 100644
index 000000000000..776ead319ae9
--- /dev/null
+++ b/drivers/clk/analogbits/wrpll-cln28hpc.c
@@ -0,0 +1,364 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2018-2019 SiFive, Inc.
+ * Wesley Terpstra
+ * Paul Walmsley
+ *
+ * This library supports configuration parsing and reprogramming of
+ * the CLN28HPC variant of the Analog Bits Wide Range PLL.  The
+ * intention is for this library to be reusable for any device that
+ * integrates this PLL; thus the register structure and programming
+ * details are expected to be provided by a separate IP block driver.
+ *
+ * The bulk of this code is primarily useful for clock configurations
+ * that must operate at arbitrary rates, as opposed to clock configurations
+ * that are restricted by software or manufacturer guidance to a small,
+ * pre-determined set of performance points.
+ *
+ * References:
+ * - Analog Bits "Wide Range PLL Datasheet", version 2015.10.01
+ * - SiFive FU540-C000 Manual v1p0, Chapter 7 "Clocking and Reset"
+ *   https://static.dev.sifive.com/FU540-C000-v1.0.pdf
+ */
+
+#include <linux/bug.h>
+#include <linux/err.h>
+#include <linux/log2.h>
+#include <linux/math64.h>
+#include <linux/clk/analogbits-wrpll-cln28hpc.h>
+
+/* MIN_INPUT_FREQ: minimum input clock frequency, in Hz (Fref_min) */
+#define MIN_INPUT_FREQ			7000000
+
+/* MAX_INPUT_FREQ: maximum input clock frequency, in Hz (Fref_max) */
+#define MAX_INPUT_FREQ			600000000
+
+/* MIN_POST_DIVIDE_REF_FREQ: minimum post-divider reference frequency, in Hz */
+#define MIN_POST_DIVR_FREQ		7000000
+
+/* MAX_POST_DIVIDE_REF_FREQ: maximum post-divider reference frequency, in Hz */
+#define MAX_POST_DIVR_FREQ		200000000
+
+/* MIN_VCO_FREQ: minimum VCO frequency, in Hz (Fvco_min) */
+#define MIN_VCO_FREQ			2400000000UL
+
+/* MAX_VCO_FREQ: maximum VCO frequency, in Hz (Fvco_max) */
+#define MAX_VCO_FREQ			4800000000ULL
+
+/* MAX_DIVQ_DIVISOR: maximum output divisor.  Selected by DIVQ = 6 */
+#define MAX_DIVQ_DIVISOR		64
+
+/* MAX_DIVR_DIVISOR: maximum reference divisor.  Selected by DIVR = 63 */
+#define MAX_DIVR_DIVISOR		64
+
+/* MAX_LOCK_US: maximum PLL lock time, in microseconds (tLOCK_max) */
+#define MAX_LOCK_US			70
+
+/*
+ * ROUND_SHIFT: number of bits to shift to avoid precision loss in the rounding
+ *              algorithm
+ */
+#define ROUND_SHIFT			20
+
+/*
+ * Private functions
+ */
+
+/**
+ * __wrpll_calc_filter_range() - determine PLL loop filter bandwidth
+ * @post_divr_freq: input clock rate after the R divider
+ *
+ * Select the value to be presented to the PLL RANGE input signals, based
+ * on the input clock frequency after the post-R-divider @post_divr_freq.
+ * This code follows the recommendations in the PLL datasheet for filter
+ * range selection.
+ *
+ * Return: The RANGE value to be presented to the PLL configuration inputs,
+ *         or a negative return code upon error.
+ */
+static int __wrpll_calc_filter_range(unsigned long post_divr_freq)
+{
+	if (post_divr_freq < MIN_POST_DIVR_FREQ ||
+	    post_divr_freq > MAX_POST_DIVR_FREQ) {
+		WARN(1, "%s: post-divider reference freq out of range: %lu",
+		     __func__, post_divr_freq);
+		return -ERANGE;
+	}
+
+	switch (post_divr_freq) {
+	case 0 ... 10999999:
+		return 1;
+	case 11000000 ... 17999999:
+		return 2;
+	case 18000000 ... 29999999:
+		return 3;
+	case 30000000 ... 49999999:
+		return 4;
+	case 50000000 ... 79999999:
+		return 5;
+	case 80000000 ... 129999999:
+		return 6;
+	}
+
+	return 7;
+}
+
+/**
+ * __wrpll_calc_fbdiv() - return feedback fixed divide value
+ * @c: ptr to a struct wrpll_cfg record to read from
+ *
+ * The internal feedback path includes a fixed by-two divider; the
+ * external feedback path does not.  Return the appropriate divider
+ * value (2 or 1) depending on whether internal or external feedback
+ * is enabled.  This code doesn't test for invalid configurations
+ * (e.g. both or neither of WRPLL_FLAGS_*_FEEDBACK are set); it relies
+ * on the caller to do so.
+ *
+ * Context: Any context.  Caller must protect the memory pointed to by
+ *          @c from simultaneous modification.
+ *
+ * Return: 2 if internal feedback is enabled or 1 if external feedback
+ *         is enabled.
+ */
+static u8 __wrpll_calc_fbdiv(const struct wrpll_cfg *c)
+{
+	return (c->flags & WRPLL_FLAGS_INT_FEEDBACK_MASK) ? 2 : 1;
+}
+
+/**
+ * __wrpll_calc_divq() - determine DIVQ based on target PLL output clock rate
+ * @target_rate: target PLL output clock rate
+ * @vco_rate: pointer to a u64 to store the computed VCO rate into
+ *
+ * Determine a reasonable value for the PLL Q post-divider, based on the
+ * target output rate @target_rate for the PLL.  Along with returning the
+ * computed Q divider value as the return value, this function stores the
+ * desired target VCO rate into the variable pointed to by @vco_rate.
+ *
+ * Context: Any context.  Caller must protect the memory pointed to by
+ *          @vco_rate from simultaneous access or modification.
+ *
+ * Return: a positive integer DIVQ value to be programmed into the hardware
+ *         upon success, or 0 upon error (since 0 is an invalid DIVQ value)
+ */
+static u8 __wrpll_calc_divq(u32 target_rate, u64 *vco_rate)
+{
+	u64 s;
+	u8 divq = 0;
+
+	if (!vco_rate) {
+		WARN_ON(1);
+		goto wcd_out;
+	}
+
+	s = div_u64(MAX_VCO_FREQ, target_rate);
+	if (s <= 1) {
+		divq = 1;
+		*vco_rate = MAX_VCO_FREQ;
+	} else if (s > MAX_DIVQ_DIVISOR) {
+		divq = ilog2(MAX_DIVQ_DIVISOR);
+		*vco_rate = MIN_VCO_FREQ;
+	} else {
+		divq = ilog2(s);
+		*vco_rate = (u64)target_rate << divq;
+	}
+
+wcd_out:
+	return divq;
+}
+
+/**
+ * __wrpll_update_parent_rate() - update PLL data when parent rate changes
+ * @c: ptr to a struct wrpll_cfg record to write PLL data to
+ * @parent_rate: PLL input refclk rate (pre-R-divider)
+ *
+ * Pre-compute some data used by the PLL configuration algorithm when
+ * the PLL's reference clock rate changes.  The intention is to avoid
+ * computation when the parent rate remains constant - expected to be
+ * the common case.
+ *
+ * Returns: 0 upon success or -ERANGE if the reference clock rate is
+ * out of range.
+ */
+static int __wrpll_update_parent_rate(struct wrpll_cfg *c,
+				      unsigned long parent_rate)
+{
+	u8 max_r_for_parent;
+
+	if (parent_rate > MAX_INPUT_FREQ || parent_rate < MIN_POST_DIVR_FREQ)
+		return -ERANGE;
+
+	c->parent_rate = parent_rate;
+	max_r_for_parent = div_u64(parent_rate, MIN_POST_DIVR_FREQ);
+	c->max_r = min_t(u8, MAX_DIVR_DIVISOR, max_r_for_parent);
+
+	c->init_r = DIV_ROUND_UP_ULL(parent_rate, MAX_POST_DIVR_FREQ);
+
+	return 0;
+}
+
+/**
+ * wrpll_configure() - compute PLL configuration for a target rate
+ * @c: ptr to a struct wrpll_cfg record to write into
+ * @target_rate: target PLL output clock rate (post-Q-divider)
+ * @parent_rate: PLL input refclk rate (pre-R-divider)
+ *
+ * Compute the appropriate PLL signal configuration values and store
+ * in PLL context @c.  PLL reprogramming is not glitchless, so the
+ * caller should switch any downstream logic to a different clock
+ * source or clock-gate it before presenting these values to the PLL
+ * configuration signals.
+ *
+ * The caller must pass this function a pre-initialized struct
+ * wrpll_cfg record: either initialized to zero (with the
+ * exception of the .name and .flags fields) or read from the PLL.
+ *
+ * Context: Any context.  Caller must protect the memory pointed to by @c
+ *          from simultaneous access or modification.
+ *
+ * Return: 0 upon success; anything else upon failure.
+ */
+int wrpll_configure_for_rate(struct wrpll_cfg *c, u32 target_rate,
+			     unsigned long parent_rate)
+{
+	unsigned long ratio;
+	u64 target_vco_rate, delta, best_delta, f_pre_div, vco, vco_pre;
+	u32 best_f, f, post_divr_freq;
+	u8 fbdiv, divq, best_r, r;
+	int range;
+
+	if (c->flags == 0) {
+		WARN(1, "%s called with uninitialized PLL config", __func__);
+		return -EINVAL;
+	}
+
+	/* Initialize rounding data if it hasn't been initialized already */
+	if (parent_rate != c->parent_rate) {
+		if (__wrpll_update_parent_rate(c, parent_rate)) {
+			pr_err("%s: PLL input rate is out of range\n",
+			       __func__);
+			return -ERANGE;
+		}
+	}
+
+	c->flags &= ~WRPLL_FLAGS_RESET_MASK;
+
+	/* Put the PLL into bypass if the user requests the parent clock rate */
+	if (target_rate == parent_rate) {
+		c->flags |= WRPLL_FLAGS_BYPASS_MASK;
+		return 0;
+	}
+
+	c->flags &= ~WRPLL_FLAGS_BYPASS_MASK;
+
+	/* Calculate the Q shift and target VCO rate */
+	divq = __wrpll_calc_divq(target_rate, &target_vco_rate);
+	if (!divq)
+		return -1;
+	c->divq = divq;
+
+	/* Precalculate the pre-Q divider target ratio */
+	ratio = div64_u64((target_vco_rate << ROUND_SHIFT), parent_rate);
+
+	fbdiv = __wrpll_calc_fbdiv(c);
+	best_r = 0;
+	best_f = 0;
+	best_delta = MAX_VCO_FREQ;
+
+	/*
+	 * Consider all values for R which land within
+	 * [MIN_POST_DIVR_FREQ, MAX_POST_DIVR_FREQ]; prefer smaller R
+	 */
+	for (r = c->init_r; r <= c->max_r; ++r) {
+		f_pre_div = ratio * r;
+		f = (f_pre_div + (1 << ROUND_SHIFT)) >> ROUND_SHIFT;
+		f >>= (fbdiv - 1);
+
+		post_divr_freq = div_u64(parent_rate, r);
+		vco_pre = fbdiv * post_divr_freq;
+		vco = vco_pre * f;
+
+		/* Ensure rounding didn't take us out of range */
+		if (vco > target_vco_rate) {
+			--f;
+			vco = vco_pre * f;
+		} else if (vco < MIN_VCO_FREQ) {
+			++f;
+			vco = vco_pre * f;
+		}
+
+		delta = abs(target_rate - vco);
+		if (delta < best_delta) {
+			best_delta = delta;
+			best_r = r;
+			best_f = f;
+		}
+	}
+
+	c->divr = best_r - 1;
+	c->divf = best_f - 1;
+
+	post_divr_freq = div_u64(parent_rate, best_r);
+
+	/* Pick the best PLL jitter filter */
+	range = __wrpll_calc_filter_range(post_divr_freq);
+	if (range < 0)
+		return range;
+	c->range = range;
+
+	return 0;
+}
+
+/**
+ * wrpll_calc_output_rate() - calculate the PLL's target output rate
+ * @c: ptr to a struct wrpll_cfg record to read from
+ * @parent_rate: PLL refclk rate
+ *
+ * Given a pointer to the PLL's current input configuration @c and the
+ * PLL's input reference clock rate @parent_rate (before the R
+ * pre-divider), calculate the PLL's output clock rate (after the Q
+ * post-divider).
+ *
+ * Context: Any context.  Caller must protect the memory pointed to by @c
+ *          from simultaneous modification.
+ *
+ * Return: the PLL's output clock rate, in Hz.  The return value from
+ *         this function is intended to be convenient to pass directly
+ *         to the Linux clock framework; thus there is no explicit
+ *         error return value.
+ */
+unsigned long wrpll_calc_output_rate(const struct wrpll_cfg *c,
+				     unsigned long parent_rate)
+{
+	u8 fbdiv;
+	u64 n;
+
+	if (c->flags & WRPLL_FLAGS_EXT_FEEDBACK_MASK) {
+		WARN(1, "external feedback mode not yet supported");
+		return ULONG_MAX;
+	}
+
+	fbdiv = __wrpll_calc_fbdiv(c);
+	n = parent_rate * fbdiv * (c->divf + 1);
+	n = div_u64(n, c->divr + 1);
+	n >>= c->divq;
+
+	return n;
+}
+
+/**
+ * wrpll_calc_max_lock_us() - return the time for the PLL to lock
+ * @c: ptr to a struct wrpll_cfg record to read from
+ *
+ * Return the minimum amount of time (in microseconds) that the caller
+ * must wait after reprogramming the PLL to ensure that it is locked
+ * to the input frequency and stable.  This is likely to depend on the DIVR
+ * value; this is under discussion with the manufacturer.
+ *
+ * Return: the minimum amount of time the caller must wait for the PLL
+ *         to lock (in microseconds)
+ */
+unsigned int wrpll_calc_max_lock_us(const struct wrpll_cfg *c)
+{
+	return MAX_LOCK_US;
+}
diff --git a/drivers/clk/at91/Makefile b/drivers/clk/at91/Makefile
index c75df1cad60e..3732241352ce 100644
--- a/drivers/clk/at91/Makefile
+++ b/drivers/clk/at91/Makefile
@@ -14,6 +14,8 @@ obj-$(CONFIG_HAVE_AT91_SMD)		+= clk-smd.o
 obj-$(CONFIG_HAVE_AT91_H32MX)		+= clk-h32mx.o
 obj-$(CONFIG_HAVE_AT91_GENERATED_CLK)	+= clk-generated.o
 obj-$(CONFIG_HAVE_AT91_I2S_MUX_CLK)	+= clk-i2s-mux.o
+obj-$(CONFIG_HAVE_AT91_SAM9X60_PLL)	+= clk-sam9x60-pll.o
 obj-$(CONFIG_SOC_AT91SAM9) += at91sam9260.o at91sam9rl.o at91sam9x5.o
+obj-$(CONFIG_SOC_SAM9X60) += sam9x60.o
 obj-$(CONFIG_SOC_SAMA5D4) += sama5d4.o
 obj-$(CONFIG_SOC_SAMA5D2) += sama5d2.o
diff --git a/drivers/clk/at91/at91sam9260.c b/drivers/clk/at91/at91sam9260.c
index b1af5a395423..0aabe49aed09 100644
--- a/drivers/clk/at91/at91sam9260.c
+++ b/drivers/clk/at91/at91sam9260.c
@@ -41,7 +41,7 @@ static u8 sam9260_plla_out[] = { 0, 2 };
 
 static u16 sam9260_plla_icpll[] = { 1, 1 };
 
-static struct clk_range sam9260_plla_outputs[] = {
+static const struct clk_range sam9260_plla_outputs[] = {
 	{ .min = 80000000, .max = 160000000 },
 	{ .min = 150000000, .max = 240000000 },
 };
@@ -58,7 +58,7 @@ static u8 sam9260_pllb_out[] = { 1 };
 
 static u16 sam9260_pllb_icpll[] = { 1 };
 
-static struct clk_range sam9260_pllb_outputs[] = {
+static const struct clk_range sam9260_pllb_outputs[] = {
 	{ .min = 70000000, .max = 130000000 },
 };
 
@@ -128,7 +128,7 @@ static u8 sam9g20_plla_out[] = { 0, 1, 2, 3, 0, 1, 2, 3 };
 
 static u16 sam9g20_plla_icpll[] = { 0, 0, 0, 0, 1, 1, 1, 1 };
 
-static struct clk_range sam9g20_plla_outputs[] = {
+static const struct clk_range sam9g20_plla_outputs[] = {
 	{ .min = 745000000, .max = 800000000 },
 	{ .min = 695000000, .max = 750000000 },
 	{ .min = 645000000, .max = 700000000 },
@@ -151,7 +151,7 @@ static u8 sam9g20_pllb_out[] = { 0 };
 
 static u16 sam9g20_pllb_icpll[] = { 0 };
 
-static struct clk_range sam9g20_pllb_outputs[] = {
+static const struct clk_range sam9g20_pllb_outputs[] = {
 	{ .min = 30000000, .max = 100000000 },
 };
 
@@ -182,7 +182,7 @@ static const struct clk_master_characteristics sam9261_mck_characteristics = {
 	.divisors = { 1, 2, 4, 0 },
 };
 
-static struct clk_range sam9261_plla_outputs[] = {
+static const struct clk_range sam9261_plla_outputs[] = {
 	{ .min = 80000000, .max = 200000000 },
 	{ .min = 190000000, .max = 240000000 },
 };
@@ -199,7 +199,7 @@ static u8 sam9261_pllb_out[] = { 1 };
 
 static u16 sam9261_pllb_icpll[] = { 1 };
 
-static struct clk_range sam9261_pllb_outputs[] = {
+static const struct clk_range sam9261_pllb_outputs[] = {
 	{ .min = 70000000, .max = 130000000 },
 };
 
@@ -262,7 +262,7 @@ static const struct clk_master_characteristics sam9263_mck_characteristics = {
 	.divisors = { 1, 2, 4, 0 },
 };
 
-static struct clk_range sam9263_pll_outputs[] = {
+static const struct clk_range sam9263_pll_outputs[] = {
 	{ .min = 80000000, .max = 200000000 },
 	{ .min = 190000000, .max = 240000000 },
 };
diff --git a/drivers/clk/at91/at91sam9rl.c b/drivers/clk/at91/at91sam9rl.c
index 5aeef68b4bdd..0ac34cdaa106 100644
--- a/drivers/clk/at91/at91sam9rl.c
+++ b/drivers/clk/at91/at91sam9rl.c
@@ -14,7 +14,7 @@ static const struct clk_master_characteristics sam9rl_mck_characteristics = {
 
 static u8 sam9rl_plla_out[] = { 0, 2 };
 
-static struct clk_range sam9rl_plla_outputs[] = {
+static const struct clk_range sam9rl_plla_outputs[] = {
 	{ .min = 80000000, .max = 200000000 },
 	{ .min = 190000000, .max = 240000000 },
 };
diff --git a/drivers/clk/at91/at91sam9x5.c b/drivers/clk/at91/at91sam9x5.c
index 3487e03d4bc6..0855f3a80cc7 100644
--- a/drivers/clk/at91/at91sam9x5.c
+++ b/drivers/clk/at91/at91sam9x5.c
@@ -17,7 +17,7 @@ static u8 plla_out[] = { 0, 1, 2, 3, 0, 1, 2, 3 };
 
 static u16 plla_icpll[] = { 0, 0, 0, 0, 1, 1, 1, 1 };
 
-static struct clk_range plla_outputs[] = {
+static const struct clk_range plla_outputs[] = {
 	{ .min = 745000000, .max = 800000000 },
 	{ .min = 695000000, .max = 750000000 },
 	{ .min = 645000000, .max = 700000000 },
@@ -49,6 +49,13 @@ static const struct {
 	{ .n = "pck1",  .p = "prog1",    .id = 9 },
 };
 
+static const struct clk_pcr_layout at91sam9x5_pcr_layout = {
+	.offset = 0x10c,
+	.cmd = BIT(12),
+	.pid_mask = GENMASK(5, 0),
+	.div_mask = GENMASK(17, 16),
+};
+
 struct pck {
 	char *n;
 	u8 id;
@@ -242,6 +249,7 @@ static void __init at91sam9x5_pmc_setup(struct device_node *np,
 
 	for (i = 0; i < ARRAY_SIZE(at91sam9x5_periphck); i++) {
 		hw = at91_clk_register_sam9x5_peripheral(regmap, &pmc_pcr_lock,
+							 &at91sam9x5_pcr_layout,
 							 at91sam9x5_periphck[i].n,
 							 "masterck",
 							 at91sam9x5_periphck[i].id,
@@ -254,6 +262,7 @@ static void __init at91sam9x5_pmc_setup(struct device_node *np,
 
 	for (i = 0; extra_pcks[i].id; i++) {
 		hw = at91_clk_register_sam9x5_peripheral(regmap, &pmc_pcr_lock,
+							 &at91sam9x5_pcr_layout,
 							 extra_pcks[i].n,
 							 "masterck",
 							 extra_pcks[i].id,
diff --git a/drivers/clk/at91/clk-generated.c b/drivers/clk/at91/clk-generated.c
index 66e7f7baf958..5f18847965c1 100644
--- a/drivers/clk/at91/clk-generated.c
+++ b/drivers/clk/at91/clk-generated.c
@@ -11,6 +11,7 @@
  *
  */
 
+#include <linux/bitfield.h>
 #include <linux/clk-provider.h>
 #include <linux/clkdev.h>
 #include <linux/clk/at91_pmc.h>
@@ -31,6 +32,7 @@ struct clk_generated {
 	spinlock_t *lock;
 	u32 id;
 	u32 gckdiv;
+	const struct clk_pcr_layout *layout;
 	u8 parent_id;
 	bool audio_pll_allowed;
 };
@@ -47,14 +49,14 @@ static int clk_generated_enable(struct clk_hw *hw)
 		 __func__, gck->gckdiv, gck->parent_id);
 
 	spin_lock_irqsave(gck->lock, flags);
-	regmap_write(gck->regmap, AT91_PMC_PCR,
-		     (gck->id & AT91_PMC_PCR_PID_MASK));
-	regmap_update_bits(gck->regmap, AT91_PMC_PCR,
-			   AT91_PMC_PCR_GCKDIV_MASK | AT91_PMC_PCR_GCKCSS_MASK |
-			   AT91_PMC_PCR_CMD | AT91_PMC_PCR_GCKEN,
-			   AT91_PMC_PCR_GCKCSS(gck->parent_id) |
-			   AT91_PMC_PCR_CMD |
-			   AT91_PMC_PCR_GCKDIV(gck->gckdiv) |
+	regmap_write(gck->regmap, gck->layout->offset,
+		     (gck->id & gck->layout->pid_mask));
+	regmap_update_bits(gck->regmap, gck->layout->offset,
+			   AT91_PMC_PCR_GCKDIV_MASK | gck->layout->gckcss_mask |
+			   gck->layout->cmd | AT91_PMC_PCR_GCKEN,
+			   field_prep(gck->layout->gckcss_mask, gck->parent_id) |
+			   gck->layout->cmd |
+			   FIELD_PREP(AT91_PMC_PCR_GCKDIV_MASK, gck->gckdiv) |
 			   AT91_PMC_PCR_GCKEN);
 	spin_unlock_irqrestore(gck->lock, flags);
 	return 0;
@@ -66,11 +68,11 @@ static void clk_generated_disable(struct clk_hw *hw)
 	unsigned long flags;
 
 	spin_lock_irqsave(gck->lock, flags);
-	regmap_write(gck->regmap, AT91_PMC_PCR,
-		     (gck->id & AT91_PMC_PCR_PID_MASK));
-	regmap_update_bits(gck->regmap, AT91_PMC_PCR,
-			   AT91_PMC_PCR_CMD | AT91_PMC_PCR_GCKEN,
-			   AT91_PMC_PCR_CMD);
+	regmap_write(gck->regmap, gck->layout->offset,
+		     (gck->id & gck->layout->pid_mask));
+	regmap_update_bits(gck->regmap, gck->layout->offset,
+			   gck->layout->cmd | AT91_PMC_PCR_GCKEN,
+			   gck->layout->cmd);
 	spin_unlock_irqrestore(gck->lock, flags);
 }
 
@@ -81,9 +83,9 @@ static int clk_generated_is_enabled(struct clk_hw *hw)
 	unsigned int status;
 
 	spin_lock_irqsave(gck->lock, flags);
-	regmap_write(gck->regmap, AT91_PMC_PCR,
-		     (gck->id & AT91_PMC_PCR_PID_MASK));
-	regmap_read(gck->regmap, AT91_PMC_PCR, &status);
+	regmap_write(gck->regmap, gck->layout->offset,
+		     (gck->id & gck->layout->pid_mask));
+	regmap_read(gck->regmap, gck->layout->offset, &status);
 	spin_unlock_irqrestore(gck->lock, flags);
 
 	return status & AT91_PMC_PCR_GCKEN ? 1 : 0;
@@ -259,19 +261,18 @@ static void clk_generated_startup(struct clk_generated *gck)
 	unsigned long flags;
 
 	spin_lock_irqsave(gck->lock, flags);
-	regmap_write(gck->regmap, AT91_PMC_PCR,
-		     (gck->id & AT91_PMC_PCR_PID_MASK));
-	regmap_read(gck->regmap, AT91_PMC_PCR, &tmp);
+	regmap_write(gck->regmap, gck->layout->offset,
+		     (gck->id & gck->layout->pid_mask));
+	regmap_read(gck->regmap, gck->layout->offset, &tmp);
 	spin_unlock_irqrestore(gck->lock, flags);
 
-	gck->parent_id = (tmp & AT91_PMC_PCR_GCKCSS_MASK)
-					>> AT91_PMC_PCR_GCKCSS_OFFSET;
-	gck->gckdiv = (tmp & AT91_PMC_PCR_GCKDIV_MASK)
-					>> AT91_PMC_PCR_GCKDIV_OFFSET;
+	gck->parent_id = field_get(gck->layout->gckcss_mask, tmp);
+	gck->gckdiv = FIELD_GET(AT91_PMC_PCR_GCKDIV_MASK, tmp);
 }
 
 struct clk_hw * __init
 at91_clk_register_generated(struct regmap *regmap, spinlock_t *lock,
+			    const struct clk_pcr_layout *layout,
 			    const char *name, const char **parent_names,
 			    u8 num_parents, u8 id, bool pll_audio,
 			    const struct clk_range *range)
@@ -298,6 +299,7 @@ at91_clk_register_generated(struct regmap *regmap, spinlock_t *lock,
 	gck->lock = lock;
 	gck->range = *range;
 	gck->audio_pll_allowed = pll_audio;
+	gck->layout = layout;
 
 	clk_generated_startup(gck);
 	hw = &gck->hw;
diff --git a/drivers/clk/at91/clk-master.c b/drivers/clk/at91/clk-master.c
index eb53b4a8fab6..12b5bf4cc7bb 100644
--- a/drivers/clk/at91/clk-master.c
+++ b/drivers/clk/at91/clk-master.c
@@ -29,6 +29,7 @@ struct clk_master {
 	struct regmap *regmap;
 	const struct clk_master_layout *layout;
 	const struct clk_master_characteristics *characteristics;
+	u32 mckr;
 };
 
 static inline bool clk_master_ready(struct regmap *regmap)
@@ -69,7 +70,7 @@ static unsigned long clk_master_recalc_rate(struct clk_hw *hw,
 						master->characteristics;
 	unsigned int mckr;
 
-	regmap_read(master->regmap, AT91_PMC_MCKR, &mckr);
+	regmap_read(master->regmap, master->layout->offset, &mckr);
 	mckr &= layout->mask;
 
 	pres = (mckr >> layout->pres_shift) & MASTER_PRES_MASK;
@@ -95,7 +96,7 @@ static u8 clk_master_get_parent(struct clk_hw *hw)
 	struct clk_master *master = to_clk_master(hw);
 	unsigned int mckr;
 
-	regmap_read(master->regmap, AT91_PMC_MCKR, &mckr);
+	regmap_read(master->regmap, master->layout->offset, &mckr);
 
 	return mckr & AT91_PMC_CSS;
 }
@@ -147,13 +148,14 @@ at91_clk_register_master(struct regmap *regmap,
 	return hw;
 }
 
-
 const struct clk_master_layout at91rm9200_master_layout = {
 	.mask = 0x31F,
 	.pres_shift = 2,
+	.offset = AT91_PMC_MCKR,
 };
 
 const struct clk_master_layout at91sam9x5_master_layout = {
 	.mask = 0x373,
 	.pres_shift = 4,
+	.offset = AT91_PMC_MCKR,
 };
diff --git a/drivers/clk/at91/clk-peripheral.c b/drivers/clk/at91/clk-peripheral.c
index 65c1defa78e4..6b7748b9588a 100644
--- a/drivers/clk/at91/clk-peripheral.c
+++ b/drivers/clk/at91/clk-peripheral.c
@@ -8,6 +8,7 @@
  *
  */
 
+#include <linux/bitops.h>
 #include <linux/clk-provider.h>
 #include <linux/clkdev.h>
 #include <linux/clk/at91_pmc.h>
@@ -23,9 +24,6 @@ DEFINE_SPINLOCK(pmc_pcr_lock);
 #define PERIPHERAL_ID_MAX	31
 #define PERIPHERAL_MASK(id)	(1 << ((id) & PERIPHERAL_ID_MAX))
 
-#define PERIPHERAL_RSHIFT_MASK	0x3
-#define PERIPHERAL_RSHIFT(val)	(((val) >> 16) & PERIPHERAL_RSHIFT_MASK)
-
 #define PERIPHERAL_MAX_SHIFT	3
 
 struct clk_peripheral {
@@ -43,6 +41,7 @@ struct clk_sam9x5_peripheral {
 	spinlock_t *lock;
 	u32 id;
 	u32 div;
+	const struct clk_pcr_layout *layout;
 	bool auto_div;
 };
 
@@ -169,13 +168,13 @@ static int clk_sam9x5_peripheral_enable(struct clk_hw *hw)
 		return 0;
 
 	spin_lock_irqsave(periph->lock, flags);
-	regmap_write(periph->regmap, AT91_PMC_PCR,
-		     (periph->id & AT91_PMC_PCR_PID_MASK));
-	regmap_update_bits(periph->regmap, AT91_PMC_PCR,
-			   AT91_PMC_PCR_DIV_MASK | AT91_PMC_PCR_CMD |
+	regmap_write(periph->regmap, periph->layout->offset,
+		     (periph->id & periph->layout->pid_mask));
+	regmap_update_bits(periph->regmap, periph->layout->offset,
+			   periph->layout->div_mask | periph->layout->cmd |
 			   AT91_PMC_PCR_EN,
-			   AT91_PMC_PCR_DIV(periph->div) |
-			   AT91_PMC_PCR_CMD |
+			   field_prep(periph->layout->div_mask, periph->div) |
+			   periph->layout->cmd |
 			   AT91_PMC_PCR_EN);
 	spin_unlock_irqrestore(periph->lock, flags);
 
@@ -191,11 +190,11 @@ static void clk_sam9x5_peripheral_disable(struct clk_hw *hw)
 		return;
 
 	spin_lock_irqsave(periph->lock, flags);
-	regmap_write(periph->regmap, AT91_PMC_PCR,
-		     (periph->id & AT91_PMC_PCR_PID_MASK));
-	regmap_update_bits(periph->regmap, AT91_PMC_PCR,
-			   AT91_PMC_PCR_EN | AT91_PMC_PCR_CMD,
-			   AT91_PMC_PCR_CMD);
+	regmap_write(periph->regmap, periph->layout->offset,
+		     (periph->id & periph->layout->pid_mask));
+	regmap_update_bits(periph->regmap, periph->layout->offset,
+			   AT91_PMC_PCR_EN | periph->layout->cmd,
+			   periph->layout->cmd);
 	spin_unlock_irqrestore(periph->lock, flags);
 }
 
@@ -209,9 +208,9 @@ static int clk_sam9x5_peripheral_is_enabled(struct clk_hw *hw)
 		return 1;
 
 	spin_lock_irqsave(periph->lock, flags);
-	regmap_write(periph->regmap, AT91_PMC_PCR,
-		     (periph->id & AT91_PMC_PCR_PID_MASK));
-	regmap_read(periph->regmap, AT91_PMC_PCR, &status);
+	regmap_write(periph->regmap, periph->layout->offset,
+		     (periph->id & periph->layout->pid_mask));
+	regmap_read(periph->regmap, periph->layout->offset, &status);
 	spin_unlock_irqrestore(periph->lock, flags);
 
 	return status & AT91_PMC_PCR_EN ? 1 : 0;
@@ -229,13 +228,13 @@ clk_sam9x5_peripheral_recalc_rate(struct clk_hw *hw,
 		return parent_rate;
 
 	spin_lock_irqsave(periph->lock, flags);
-	regmap_write(periph->regmap, AT91_PMC_PCR,
-		     (periph->id & AT91_PMC_PCR_PID_MASK));
-	regmap_read(periph->regmap, AT91_PMC_PCR, &status);
+	regmap_write(periph->regmap, periph->layout->offset,
+		     (periph->id & periph->layout->pid_mask));
+	regmap_read(periph->regmap, periph->layout->offset, &status);
 	spin_unlock_irqrestore(periph->lock, flags);
 
 	if (status & AT91_PMC_PCR_EN) {
-		periph->div = PERIPHERAL_RSHIFT(status);
+		periph->div = field_get(periph->layout->div_mask, status);
 		periph->auto_div = false;
 	} else {
 		clk_sam9x5_peripheral_autodiv(periph);
@@ -328,6 +327,7 @@ static const struct clk_ops sam9x5_peripheral_ops = {
 
 struct clk_hw * __init
 at91_clk_register_sam9x5_peripheral(struct regmap *regmap, spinlock_t *lock,
+				    const struct clk_pcr_layout *layout,
 				    const char *name, const char *parent_name,
 				    u32 id, const struct clk_range *range)
 {
@@ -354,7 +354,9 @@ at91_clk_register_sam9x5_peripheral(struct regmap *regmap, spinlock_t *lock,
 	periph->div = 0;
 	periph->regmap = regmap;
 	periph->lock = lock;
-	periph->auto_div = true;
+	if (layout->div_mask)
+		periph->auto_div = true;
+	periph->layout = layout;
 	periph->range = *range;
 
 	hw = &periph->hw;
diff --git a/drivers/clk/at91/clk-sam9x60-pll.c b/drivers/clk/at91/clk-sam9x60-pll.c
new file mode 100644
index 000000000000..34b817825b22
--- /dev/null
+++ b/drivers/clk/at91/clk-sam9x60-pll.c
@@ -0,0 +1,330 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ *  Copyright (C) 2019 Microchip Technology Inc.
+ *
+ */
+
+#include <linux/bitfield.h>
+#include <linux/clk-provider.h>
+#include <linux/clkdev.h>
+#include <linux/clk/at91_pmc.h>
+#include <linux/of.h>
+#include <linux/mfd/syscon.h>
+#include <linux/regmap.h>
+
+#include "pmc.h"
+
+#define PMC_PLL_CTRL0	0xc
+#define		PMC_PLL_CTRL0_DIV_MSK		GENMASK(7, 0)
+#define		PMC_PLL_CTRL0_ENPLL		BIT(28)
+#define		PMC_PLL_CTRL0_ENPLLCK		BIT(29)
+#define		PMC_PLL_CTRL0_ENLOCK		BIT(31)
+
+#define PMC_PLL_CTRL1	0x10
+#define		PMC_PLL_CTRL1_FRACR_MSK		GENMASK(21, 0)
+#define		PMC_PLL_CTRL1_MUL_MSK		GENMASK(30, 24)
+
+#define PMC_PLL_ACR	0x18
+#define		PMC_PLL_ACR_DEFAULT		0x1b040010UL
+#define		PMC_PLL_ACR_UTMIVR		BIT(12)
+#define		PMC_PLL_ACR_UTMIBG		BIT(13)
+#define		PMC_PLL_ACR_LOOP_FILTER_MSK	GENMASK(31, 24)
+
+#define PMC_PLL_UPDT	0x1c
+#define		PMC_PLL_UPDT_UPDATE		BIT(8)
+
+#define PMC_PLL_ISR0	0xec
+
+#define PLL_DIV_MAX		(FIELD_GET(PMC_PLL_CTRL0_DIV_MSK, UINT_MAX) + 1)
+#define UPLL_DIV		2
+#define PLL_MUL_MAX		(FIELD_GET(PMC_PLL_CTRL1_MUL_MSK, UINT_MAX) + 1)
+
+#define PLL_MAX_ID		1
+
+struct sam9x60_pll {
+	struct clk_hw hw;
+	struct regmap *regmap;
+	spinlock_t *lock;
+	const struct clk_pll_characteristics *characteristics;
+	u32 frac;
+	u8 id;
+	u8 div;
+	u16 mul;
+};
+
+#define to_sam9x60_pll(hw) container_of(hw, struct sam9x60_pll, hw)
+
+static inline bool sam9x60_pll_ready(struct regmap *regmap, int id)
+{
+	unsigned int status;
+
+	regmap_read(regmap, PMC_PLL_ISR0, &status);
+
+	return !!(status & BIT(id));
+}
+
+static int sam9x60_pll_prepare(struct clk_hw *hw)
+{
+	struct sam9x60_pll *pll = to_sam9x60_pll(hw);
+	struct regmap *regmap = pll->regmap;
+	unsigned long flags;
+	u8 div;
+	u16 mul;
+	u32 val;
+
+	spin_lock_irqsave(pll->lock, flags);
+	regmap_write(regmap, PMC_PLL_UPDT, pll->id);
+
+	regmap_read(regmap, PMC_PLL_CTRL0, &val);
+	div = FIELD_GET(PMC_PLL_CTRL0_DIV_MSK, val);
+
+	regmap_read(regmap, PMC_PLL_CTRL1, &val);
+	mul = FIELD_GET(PMC_PLL_CTRL1_MUL_MSK, val);
+
+	if (sam9x60_pll_ready(regmap, pll->id) &&
+	    (div == pll->div && mul == pll->mul)) {
+		spin_unlock_irqrestore(pll->lock, flags);
+		return 0;
+	}
+
+	/* Recommended value for PMC_PLL_ACR */
+	val = PMC_PLL_ACR_DEFAULT;
+	regmap_write(regmap, PMC_PLL_ACR, val);
+
+	regmap_write(regmap, PMC_PLL_CTRL1,
+		     FIELD_PREP(PMC_PLL_CTRL1_MUL_MSK, pll->mul));
+
+	if (pll->characteristics->upll) {
+		/* Enable the UTMI internal bandgap */
+		val |= PMC_PLL_ACR_UTMIBG;
+		regmap_write(regmap, PMC_PLL_ACR, val);
+
+		udelay(10);
+
+		/* Enable the UTMI internal regulator */
+		val |= PMC_PLL_ACR_UTMIVR;
+		regmap_write(regmap, PMC_PLL_ACR, val);
+
+		udelay(10);
+	}
+
+	regmap_update_bits(regmap, PMC_PLL_UPDT,
+			   PMC_PLL_UPDT_UPDATE, PMC_PLL_UPDT_UPDATE);
+
+	regmap_write(regmap, PMC_PLL_CTRL0,
+		     PMC_PLL_CTRL0_ENLOCK | PMC_PLL_CTRL0_ENPLL |
+		     PMC_PLL_CTRL0_ENPLLCK | pll->div);
+
+	regmap_update_bits(regmap, PMC_PLL_UPDT,
+			   PMC_PLL_UPDT_UPDATE, PMC_PLL_UPDT_UPDATE);
+
+	while (!sam9x60_pll_ready(regmap, pll->id))
+		cpu_relax();
+
+	spin_unlock_irqrestore(pll->lock, flags);
+
+	return 0;
+}
+
+static int sam9x60_pll_is_prepared(struct clk_hw *hw)
+{
+	struct sam9x60_pll *pll = to_sam9x60_pll(hw);
+
+	return sam9x60_pll_ready(pll->regmap, pll->id);
+}
+
+static void sam9x60_pll_unprepare(struct clk_hw *hw)
+{
+	struct sam9x60_pll *pll = to_sam9x60_pll(hw);
+	unsigned long flags;
+
+	spin_lock_irqsave(pll->lock, flags);
+
+	regmap_write(pll->regmap, PMC_PLL_UPDT, pll->id);
+
+	regmap_update_bits(pll->regmap, PMC_PLL_CTRL0,
+			   PMC_PLL_CTRL0_ENPLLCK, 0);
+
+	regmap_update_bits(pll->regmap, PMC_PLL_UPDT,
+			   PMC_PLL_UPDT_UPDATE, PMC_PLL_UPDT_UPDATE);
+
+	regmap_update_bits(pll->regmap, PMC_PLL_CTRL0, PMC_PLL_CTRL0_ENPLL, 0);
+
+	if (pll->characteristics->upll)
+		regmap_update_bits(pll->regmap, PMC_PLL_ACR,
+				   PMC_PLL_ACR_UTMIBG | PMC_PLL_ACR_UTMIVR, 0);
+
+	regmap_update_bits(pll->regmap, PMC_PLL_UPDT,
+			   PMC_PLL_UPDT_UPDATE, PMC_PLL_UPDT_UPDATE);
+
+	spin_unlock_irqrestore(pll->lock, flags);
+}
+
+static unsigned long sam9x60_pll_recalc_rate(struct clk_hw *hw,
+					     unsigned long parent_rate)
+{
+	struct sam9x60_pll *pll = to_sam9x60_pll(hw);
+
+	return (parent_rate * (pll->mul + 1)) / (pll->div + 1);
+}
+
+static long sam9x60_pll_get_best_div_mul(struct sam9x60_pll *pll,
+					 unsigned long rate,
+					 unsigned long parent_rate,
+					 bool update)
+{
+	const struct clk_pll_characteristics *characteristics =
+							pll->characteristics;
+	unsigned long bestremainder = ULONG_MAX;
+	unsigned long maxdiv, mindiv, tmpdiv;
+	long bestrate = -ERANGE;
+	unsigned long bestdiv = 0;
+	unsigned long bestmul = 0;
+	unsigned long bestfrac = 0;
+
+	if (rate < characteristics->output[0].min ||
+	    rate > characteristics->output[0].max)
+		return -ERANGE;
+
+	if (!pll->characteristics->upll) {
+		mindiv = parent_rate / rate;
+		if (mindiv < 2)
+			mindiv = 2;
+
+		maxdiv = DIV_ROUND_UP(parent_rate * PLL_MUL_MAX, rate);
+		if (maxdiv > PLL_DIV_MAX)
+			maxdiv = PLL_DIV_MAX;
+	} else {
+		mindiv = maxdiv = UPLL_DIV;
+	}
+
+	for (tmpdiv = mindiv; tmpdiv <= maxdiv; tmpdiv++) {
+		unsigned long remainder;
+		unsigned long tmprate;
+		unsigned long tmpmul;
+		unsigned long tmpfrac = 0;
+
+		/*
+		 * Calculate the multiplier associated with the current
+		 * divider that provide the closest rate to the requested one.
+		 */
+		tmpmul = mult_frac(rate, tmpdiv, parent_rate);
+		tmprate = mult_frac(parent_rate, tmpmul, tmpdiv);
+		remainder = rate - tmprate;
+
+		if (remainder) {
+			tmpfrac = DIV_ROUND_CLOSEST_ULL((u64)remainder * tmpdiv * (1 << 22),
+							parent_rate);
+
+			tmprate += DIV_ROUND_CLOSEST_ULL((u64)tmpfrac * parent_rate,
+							 tmpdiv * (1 << 22));
+
+			if (tmprate > rate)
+				remainder = tmprate - rate;
+			else
+				remainder = rate - tmprate;
+		}
+
+		/*
+		 * Compare the remainder with the best remainder found until
+		 * now and elect a new best multiplier/divider pair if the
+		 * current remainder is smaller than the best one.
+		 */
+		if (remainder < bestremainder) {
+			bestremainder = remainder;
+			bestdiv = tmpdiv;
+			bestmul = tmpmul;
+			bestrate = tmprate;
+			bestfrac = tmpfrac;
+		}
+
+		/* We've found a perfect match!  */
+		if (!remainder)
+			break;
+	}
+
+	/* Check if bestrate is a valid output rate  */
+	if (bestrate < characteristics->output[0].min &&
+	    bestrate > characteristics->output[0].max)
+		return -ERANGE;
+
+	if (update) {
+		pll->div = bestdiv - 1;
+		pll->mul = bestmul - 1;
+		pll->frac = bestfrac;
+	}
+
+	return bestrate;
+}
+
+static long sam9x60_pll_round_rate(struct clk_hw *hw, unsigned long rate,
+				   unsigned long *parent_rate)
+{
+	struct sam9x60_pll *pll = to_sam9x60_pll(hw);
+
+	return sam9x60_pll_get_best_div_mul(pll, rate, *parent_rate, false);
+}
+
+static int sam9x60_pll_set_rate(struct clk_hw *hw, unsigned long rate,
+				unsigned long parent_rate)
+{
+	struct sam9x60_pll *pll = to_sam9x60_pll(hw);
+
+	return sam9x60_pll_get_best_div_mul(pll, rate, parent_rate, true);
+}
+
+static const struct clk_ops pll_ops = {
+	.prepare = sam9x60_pll_prepare,
+	.unprepare = sam9x60_pll_unprepare,
+	.is_prepared = sam9x60_pll_is_prepared,
+	.recalc_rate = sam9x60_pll_recalc_rate,
+	.round_rate = sam9x60_pll_round_rate,
+	.set_rate = sam9x60_pll_set_rate,
+};
+
+struct clk_hw * __init
+sam9x60_clk_register_pll(struct regmap *regmap, spinlock_t *lock,
+			 const char *name, const char *parent_name, u8 id,
+			 const struct clk_pll_characteristics *characteristics)
+{
+	struct sam9x60_pll *pll;
+	struct clk_hw *hw;
+	struct clk_init_data init;
+	unsigned int pllr;
+	int ret;
+
+	if (id > PLL_MAX_ID)
+		return ERR_PTR(-EINVAL);
+
+	pll = kzalloc(sizeof(*pll), GFP_KERNEL);
+	if (!pll)
+		return ERR_PTR(-ENOMEM);
+
+	init.name = name;
+	init.ops = &pll_ops;
+	init.parent_names = &parent_name;
+	init.num_parents = 1;
+	init.flags = CLK_SET_RATE_GATE;
+
+	pll->id = id;
+	pll->hw.init = &init;
+	pll->characteristics = characteristics;
+	pll->regmap = regmap;
+	pll->lock = lock;
+
+	regmap_write(regmap, PMC_PLL_UPDT, id);
+	regmap_read(regmap, PMC_PLL_CTRL0, &pllr);
+	pll->div = FIELD_GET(PMC_PLL_CTRL0_DIV_MSK, pllr);
+	regmap_read(regmap, PMC_PLL_CTRL1, &pllr);
+	pll->mul = FIELD_GET(PMC_PLL_CTRL1_MUL_MSK, pllr);
+
+	hw = &pll->hw;
+	ret = clk_hw_register(NULL, hw);
+	if (ret) {
+		kfree(pll);
+		hw = ERR_PTR(ret);
+	}
+
+	return hw;
+}
+
diff --git a/drivers/clk/at91/clk-usb.c b/drivers/clk/at91/clk-usb.c
index 79ee1c760f2a..ebc37ee33518 100644
--- a/drivers/clk/at91/clk-usb.c
+++ b/drivers/clk/at91/clk-usb.c
@@ -23,9 +23,13 @@
 #define RM9200_USB_DIV_SHIFT	28
 #define RM9200_USB_DIV_TAB_SIZE	4
 
+#define SAM9X5_USBS_MASK	GENMASK(0, 0)
+#define SAM9X60_USBS_MASK	GENMASK(1, 0)
+
 struct at91sam9x5_clk_usb {
 	struct clk_hw hw;
 	struct regmap *regmap;
+	u32 usbs_mask;
 };
 
 #define to_at91sam9x5_clk_usb(hw) \
@@ -111,8 +115,7 @@ static int at91sam9x5_clk_usb_set_parent(struct clk_hw *hw, u8 index)
 	if (index > 1)
 		return -EINVAL;
 
-	regmap_update_bits(usb->regmap, AT91_PMC_USB, AT91_PMC_USBS,
-			   index ? AT91_PMC_USBS : 0);
+	regmap_update_bits(usb->regmap, AT91_PMC_USB, usb->usbs_mask, index);
 
 	return 0;
 }
@@ -124,7 +127,7 @@ static u8 at91sam9x5_clk_usb_get_parent(struct clk_hw *hw)
 
 	regmap_read(usb->regmap, AT91_PMC_USB, &usbr);
 
-	return usbr & AT91_PMC_USBS;
+	return usbr & usb->usbs_mask;
 }
 
 static int at91sam9x5_clk_usb_set_rate(struct clk_hw *hw, unsigned long rate,
@@ -190,9 +193,10 @@ static const struct clk_ops at91sam9n12_usb_ops = {
 	.set_rate = at91sam9x5_clk_usb_set_rate,
 };
 
-struct clk_hw * __init
-at91sam9x5_clk_register_usb(struct regmap *regmap, const char *name,
-			    const char **parent_names, u8 num_parents)
+static struct clk_hw * __init
+_at91sam9x5_clk_register_usb(struct regmap *regmap, const char *name,
+			     const char **parent_names, u8 num_parents,
+			     u32 usbs_mask)
 {
 	struct at91sam9x5_clk_usb *usb;
 	struct clk_hw *hw;
@@ -212,6 +216,7 @@ at91sam9x5_clk_register_usb(struct regmap *regmap, const char *name,
 
 	usb->hw.init = &init;
 	usb->regmap = regmap;
+	usb->usbs_mask = SAM9X5_USBS_MASK;
 
 	hw = &usb->hw;
 	ret = clk_hw_register(NULL, &usb->hw);
@@ -224,6 +229,22 @@ at91sam9x5_clk_register_usb(struct regmap *regmap, const char *name,
 }
 
 struct clk_hw * __init
+at91sam9x5_clk_register_usb(struct regmap *regmap, const char *name,
+			    const char **parent_names, u8 num_parents)
+{
+	return _at91sam9x5_clk_register_usb(regmap, name, parent_names,
+					    num_parents, SAM9X5_USBS_MASK);
+}
+
+struct clk_hw * __init
+sam9x60_clk_register_usb(struct regmap *regmap, const char *name,
+			 const char **parent_names, u8 num_parents)
+{
+	return _at91sam9x5_clk_register_usb(regmap, name, parent_names,
+					    num_parents, SAM9X60_USBS_MASK);
+}
+
+struct clk_hw * __init
 at91sam9n12_clk_register_usb(struct regmap *regmap, const char *name,
 			     const char *parent_name)
 {
diff --git a/drivers/clk/at91/dt-compat.c b/drivers/clk/at91/dt-compat.c
index b95bb4e2a927..aa1754eac59f 100644
--- a/drivers/clk/at91/dt-compat.c
+++ b/drivers/clk/at91/dt-compat.c
@@ -93,6 +93,14 @@ CLK_OF_DECLARE(of_sama5d2_clk_audio_pll_pmc_setup,
 	       of_sama5d2_clk_audio_pll_pmc_setup);
 #endif /* CONFIG_HAVE_AT91_AUDIO_PLL */
 
+static const struct clk_pcr_layout dt_pcr_layout = {
+	.offset = 0x10c,
+	.cmd = BIT(12),
+	.pid_mask = GENMASK(5, 0),
+	.div_mask = GENMASK(17, 16),
+	.gckcss_mask = GENMASK(10, 8),
+};
+
 #ifdef CONFIG_HAVE_AT91_GENERATED_CLK
 #define GENERATED_SOURCE_MAX	6
 
@@ -146,7 +154,8 @@ static void __init of_sama5d2_clk_generated_setup(struct device_node *np)
 		     id == GCK_ID_CLASSD))
 			pll_audio = true;
 
-		hw = at91_clk_register_generated(regmap, &pmc_pcr_lock, name,
+		hw = at91_clk_register_generated(regmap, &pmc_pcr_lock,
+						 &dt_pcr_layout, name,
 						 parent_names, num_parents,
 						 id, pll_audio, &range);
 		if (IS_ERR(hw))
@@ -448,6 +457,7 @@ of_at91_clk_periph_setup(struct device_node *np, u8 type)
 
 			hw = at91_clk_register_sam9x5_peripheral(regmap,
 								 &pmc_pcr_lock,
+								 &dt_pcr_layout,
 								 name,
 								 parent_name,
 								 id, &range);
diff --git a/drivers/clk/at91/pmc.h b/drivers/clk/at91/pmc.h
index a0e5ce9c9b9e..2311204948be 100644
--- a/drivers/clk/at91/pmc.h
+++ b/drivers/clk/at91/pmc.h
@@ -38,6 +38,7 @@ struct clk_range {
 #define CLK_RANGE(MIN, MAX) {.min = MIN, .max = MAX,}
 
 struct clk_master_layout {
+	u32 offset;
 	u32 mask;
 	u8 pres_shift;
 };
@@ -65,9 +66,10 @@ extern const struct clk_pll_layout sama5d3_pll_layout;
 struct clk_pll_characteristics {
 	struct clk_range input;
 	int num_output;
-	struct clk_range *output;
+	const struct clk_range *output;
 	u16 *icpll;
 	u8 *out;
+	u8 upll : 1;
 };
 
 struct clk_programmable_layout {
@@ -82,6 +84,17 @@ extern const struct clk_programmable_layout at91rm9200_programmable_layout;
 extern const struct clk_programmable_layout at91sam9g45_programmable_layout;
 extern const struct clk_programmable_layout at91sam9x5_programmable_layout;
 
+struct clk_pcr_layout {
+	u32 offset;
+	u32 cmd;
+	u32 div_mask;
+	u32 gckcss_mask;
+	u32 pid_mask;
+};
+
+#define field_get(_mask, _reg) (((_reg) & (_mask)) >> (ffs(_mask) - 1))
+#define field_prep(_mask, _val) (((_val) << (ffs(_mask) - 1)) & (_mask))
+
 #define ndck(a, s) (a[s - 1].id + 1)
 #define nck(a) (a[ARRAY_SIZE(a) - 1].id + 1)
 struct pmc_data *pmc_data_allocate(unsigned int ncore, unsigned int nsystem,
@@ -107,6 +120,7 @@ at91_clk_register_audio_pll_pmc(struct regmap *regmap, const char *name,
 
 struct clk_hw * __init
 at91_clk_register_generated(struct regmap *regmap, spinlock_t *lock,
+			    const struct clk_pcr_layout *layout,
 			    const char *name, const char **parent_names,
 			    u8 num_parents, u8 id, bool pll_audio,
 			    const struct clk_range *range);
@@ -145,6 +159,7 @@ at91_clk_register_peripheral(struct regmap *regmap, const char *name,
 			     const char *parent_name, u32 id);
 struct clk_hw * __init
 at91_clk_register_sam9x5_peripheral(struct regmap *regmap, spinlock_t *lock,
+				    const struct clk_pcr_layout *layout,
 				    const char *name, const char *parent_name,
 				    u32 id, const struct clk_range *range);
 
@@ -158,6 +173,11 @@ at91_clk_register_plldiv(struct regmap *regmap, const char *name,
 			 const char *parent_name);
 
 struct clk_hw * __init
+sam9x60_clk_register_pll(struct regmap *regmap, spinlock_t *lock,
+			 const char *name, const char *parent_name, u8 id,
+			 const struct clk_pll_characteristics *characteristics);
+
+struct clk_hw * __init
 at91_clk_register_programmable(struct regmap *regmap, const char *name,
 			       const char **parent_names, u8 num_parents, u8 id,
 			       const struct clk_programmable_layout *layout);
@@ -183,6 +203,9 @@ struct clk_hw * __init
 at91sam9n12_clk_register_usb(struct regmap *regmap, const char *name,
 			     const char *parent_name);
 struct clk_hw * __init
+sam9x60_clk_register_usb(struct regmap *regmap, const char *name,
+			 const char **parent_names, u8 num_parents);
+struct clk_hw * __init
 at91rm9200_clk_register_usb(struct regmap *regmap, const char *name,
 			    const char *parent_name, const u32 *divisors);
 
diff --git a/drivers/clk/at91/sam9x60.c b/drivers/clk/at91/sam9x60.c
new file mode 100644
index 000000000000..9790ddfa5b3c
--- /dev/null
+++ b/drivers/clk/at91/sam9x60.c
@@ -0,0 +1,307 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/clk-provider.h>
+#include <linux/mfd/syscon.h>
+#include <linux/slab.h>
+
+#include <dt-bindings/clock/at91.h>
+
+#include "pmc.h"
+
+static DEFINE_SPINLOCK(pmc_pll_lock);
+
+static const struct clk_master_characteristics mck_characteristics = {
+	.output = { .min = 140000000, .max = 200000000 },
+	.divisors = { 1, 2, 4, 3 },
+	.have_div3_pres = 1,
+};
+
+static const struct clk_master_layout sam9x60_master_layout = {
+	.mask = 0x373,
+	.pres_shift = 4,
+	.offset = 0x28,
+};
+
+static const struct clk_range plla_outputs[] = {
+	{ .min = 300000000, .max = 600000000 },
+};
+
+static const struct clk_pll_characteristics plla_characteristics = {
+	.input = { .min = 12000000, .max = 48000000 },
+	.num_output = ARRAY_SIZE(plla_outputs),
+	.output = plla_outputs,
+};
+
+static const struct clk_range upll_outputs[] = {
+	{ .min = 300000000, .max = 500000000 },
+};
+
+static const struct clk_pll_characteristics upll_characteristics = {
+	.input = { .min = 12000000, .max = 48000000 },
+	.num_output = ARRAY_SIZE(upll_outputs),
+	.output = upll_outputs,
+	.upll = true,
+};
+
+static const struct clk_programmable_layout sam9x60_programmable_layout = {
+	.pres_shift = 8,
+	.css_mask = 0x1f,
+	.have_slck_mck = 0,
+};
+
+static const struct clk_pcr_layout sam9x60_pcr_layout = {
+	.offset = 0x88,
+	.cmd = BIT(31),
+	.gckcss_mask = GENMASK(12, 8),
+	.pid_mask = GENMASK(6, 0),
+};
+
+static const struct {
+	char *n;
+	char *p;
+	u8 id;
+} sam9x60_systemck[] = {
+	{ .n = "ddrck",  .p = "masterck", .id = 2 },
+	{ .n = "uhpck",  .p = "usbck",    .id = 6 },
+	{ .n = "pck0",   .p = "prog0",    .id = 8 },
+	{ .n = "pck1",   .p = "prog1",    .id = 9 },
+	{ .n = "qspick", .p = "masterck", .id = 19 },
+};
+
+static const struct {
+	char *n;
+	u8 id;
+} sam9x60_periphck[] = {
+	{ .n = "pioA_clk",   .id = 2, },
+	{ .n = "pioB_clk",   .id = 3, },
+	{ .n = "pioC_clk",   .id = 4, },
+	{ .n = "flex0_clk",  .id = 5, },
+	{ .n = "flex1_clk",  .id = 6, },
+	{ .n = "flex2_clk",  .id = 7, },
+	{ .n = "flex3_clk",  .id = 8, },
+	{ .n = "flex6_clk",  .id = 9, },
+	{ .n = "flex7_clk",  .id = 10, },
+	{ .n = "flex8_clk",  .id = 11, },
+	{ .n = "sdmmc0_clk", .id = 12, },
+	{ .n = "flex4_clk",  .id = 13, },
+	{ .n = "flex5_clk",  .id = 14, },
+	{ .n = "flex9_clk",  .id = 15, },
+	{ .n = "flex10_clk", .id = 16, },
+	{ .n = "tcb0_clk",   .id = 17, },
+	{ .n = "pwm_clk",    .id = 18, },
+	{ .n = "adc_clk",    .id = 19, },
+	{ .n = "dma0_clk",   .id = 20, },
+	{ .n = "matrix_clk", .id = 21, },
+	{ .n = "uhphs_clk",  .id = 22, },
+	{ .n = "udphs_clk",  .id = 23, },
+	{ .n = "macb0_clk",  .id = 24, },
+	{ .n = "lcd_clk",    .id = 25, },
+	{ .n = "sdmmc1_clk", .id = 26, },
+	{ .n = "macb1_clk",  .id = 27, },
+	{ .n = "ssc_clk",    .id = 28, },
+	{ .n = "can0_clk",   .id = 29, },
+	{ .n = "can1_clk",   .id = 30, },
+	{ .n = "flex11_clk", .id = 32, },
+	{ .n = "flex12_clk", .id = 33, },
+	{ .n = "i2s_clk",    .id = 34, },
+	{ .n = "qspi_clk",   .id = 35, },
+	{ .n = "gfx2d_clk",  .id = 36, },
+	{ .n = "pit64b_clk", .id = 37, },
+	{ .n = "trng_clk",   .id = 38, },
+	{ .n = "aes_clk",    .id = 39, },
+	{ .n = "tdes_clk",   .id = 40, },
+	{ .n = "sha_clk",    .id = 41, },
+	{ .n = "classd_clk", .id = 42, },
+	{ .n = "isi_clk",    .id = 43, },
+	{ .n = "pioD_clk",   .id = 44, },
+	{ .n = "tcb1_clk",   .id = 45, },
+	{ .n = "dbgu_clk",   .id = 47, },
+	{ .n = "mpddr_clk",  .id = 49, },
+};
+
+static const struct {
+	char *n;
+	u8 id;
+	struct clk_range r;
+	bool pll;
+} sam9x60_gck[] = {
+	{ .n = "flex0_gclk",  .id = 5, },
+	{ .n = "flex1_gclk",  .id = 6, },
+	{ .n = "flex2_gclk",  .id = 7, },
+	{ .n = "flex3_gclk",  .id = 8, },
+	{ .n = "flex6_gclk",  .id = 9, },
+	{ .n = "flex7_gclk",  .id = 10, },
+	{ .n = "flex8_gclk",  .id = 11, },
+	{ .n = "sdmmc0_gclk", .id = 12, .r = { .min = 0, .max = 105000000 }, },
+	{ .n = "flex4_gclk",  .id = 13, },
+	{ .n = "flex5_gclk",  .id = 14, },
+	{ .n = "flex9_gclk",  .id = 15, },
+	{ .n = "flex10_gclk", .id = 16, },
+	{ .n = "tcb0_gclk",   .id = 17, },
+	{ .n = "adc_gclk",    .id = 19, },
+	{ .n = "lcd_gclk",    .id = 25, .r = { .min = 0, .max = 140000000 }, },
+	{ .n = "sdmmc1_gclk", .id = 26, .r = { .min = 0, .max = 105000000 }, },
+	{ .n = "flex11_gclk", .id = 32, },
+	{ .n = "flex12_gclk", .id = 33, },
+	{ .n = "i2s_gclk",    .id = 34, .r = { .min = 0, .max = 105000000 },
+		.pll = true, },
+	{ .n = "pit64b_gclk", .id = 37, },
+	{ .n = "classd_gclk", .id = 42, .r = { .min = 0, .max = 100000000 },
+		.pll = true, },
+	{ .n = "tcb1_gclk",   .id = 45, },
+	{ .n = "dbgu_gclk",   .id = 47, },
+};
+
+static void __init sam9x60_pmc_setup(struct device_node *np)
+{
+	struct clk_range range = CLK_RANGE(0, 0);
+	const char *td_slck_name, *md_slck_name, *mainxtal_name;
+	struct pmc_data *sam9x60_pmc;
+	const char *parent_names[6];
+	struct regmap *regmap;
+	struct clk_hw *hw;
+	int i;
+	bool bypass;
+
+	i = of_property_match_string(np, "clock-names", "td_slck");
+	if (i < 0)
+		return;
+
+	td_slck_name = of_clk_get_parent_name(np, i);
+
+	i = of_property_match_string(np, "clock-names", "md_slck");
+	if (i < 0)
+		return;
+
+	md_slck_name = of_clk_get_parent_name(np, i);
+
+	i = of_property_match_string(np, "clock-names", "main_xtal");
+	if (i < 0)
+		return;
+	mainxtal_name = of_clk_get_parent_name(np, i);
+
+	regmap = syscon_node_to_regmap(np);
+	if (IS_ERR(regmap))
+		return;
+
+	sam9x60_pmc = pmc_data_allocate(PMC_MAIN + 1,
+					nck(sam9x60_systemck),
+					nck(sam9x60_periphck),
+					nck(sam9x60_gck));
+	if (!sam9x60_pmc)
+		return;
+
+	hw = at91_clk_register_main_rc_osc(regmap, "main_rc_osc", 24000000,
+					   50000000);
+	if (IS_ERR(hw))
+		goto err_free;
+
+	bypass = of_property_read_bool(np, "atmel,osc-bypass");
+
+	hw = at91_clk_register_main_osc(regmap, "main_osc", mainxtal_name,
+					bypass);
+	if (IS_ERR(hw))
+		goto err_free;
+
+	parent_names[0] = "main_rc_osc";
+	parent_names[1] = "main_osc";
+	hw = at91_clk_register_sam9x5_main(regmap, "mainck", parent_names, 2);
+	if (IS_ERR(hw))
+		goto err_free;
+
+	sam9x60_pmc->chws[PMC_MAIN] = hw;
+
+	hw = sam9x60_clk_register_pll(regmap, &pmc_pll_lock, "pllack",
+				      "mainck", 0, &plla_characteristics);
+	if (IS_ERR(hw))
+		goto err_free;
+
+	hw = sam9x60_clk_register_pll(regmap, &pmc_pll_lock, "upllck",
+				      "main_osc", 1, &upll_characteristics);
+	if (IS_ERR(hw))
+		goto err_free;
+
+	sam9x60_pmc->chws[PMC_UTMI] = hw;
+
+	parent_names[0] = md_slck_name;
+	parent_names[1] = "mainck";
+	parent_names[2] = "pllack";
+	hw = at91_clk_register_master(regmap, "masterck", 3, parent_names,
+				      &sam9x60_master_layout,
+				      &mck_characteristics);
+	if (IS_ERR(hw))
+		goto err_free;
+
+	sam9x60_pmc->chws[PMC_MCK] = hw;
+
+	parent_names[0] = "pllack";
+	parent_names[1] = "upllck";
+	parent_names[2] = "mainck";
+	parent_names[3] = "mainck";
+	hw = sam9x60_clk_register_usb(regmap, "usbck", parent_names, 4);
+	if (IS_ERR(hw))
+		goto err_free;
+
+	parent_names[0] = md_slck_name;
+	parent_names[1] = td_slck_name;
+	parent_names[2] = "mainck";
+	parent_names[3] = "masterck";
+	parent_names[4] = "pllack";
+	parent_names[5] = "upllck";
+	for (i = 0; i < 8; i++) {
+		char name[6];
+
+		snprintf(name, sizeof(name), "prog%d", i);
+
+		hw = at91_clk_register_programmable(regmap, name,
+						    parent_names, 6, i,
+						    &sam9x60_programmable_layout);
+		if (IS_ERR(hw))
+			goto err_free;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(sam9x60_systemck); i++) {
+		hw = at91_clk_register_system(regmap, sam9x60_systemck[i].n,
+					      sam9x60_systemck[i].p,
+					      sam9x60_systemck[i].id);
+		if (IS_ERR(hw))
+			goto err_free;
+
+		sam9x60_pmc->shws[sam9x60_systemck[i].id] = hw;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(sam9x60_periphck); i++) {
+		hw = at91_clk_register_sam9x5_peripheral(regmap, &pmc_pcr_lock,
+							 &sam9x60_pcr_layout,
+							 sam9x60_periphck[i].n,
+							 "masterck",
+							 sam9x60_periphck[i].id,
+							 &range);
+		if (IS_ERR(hw))
+			goto err_free;
+
+		sam9x60_pmc->phws[sam9x60_periphck[i].id] = hw;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(sam9x60_gck); i++) {
+		hw = at91_clk_register_generated(regmap, &pmc_pcr_lock,
+						 &sam9x60_pcr_layout,
+						 sam9x60_gck[i].n,
+						 parent_names, 6,
+						 sam9x60_gck[i].id,
+						 sam9x60_gck[i].pll,
+						 &sam9x60_gck[i].r);
+		if (IS_ERR(hw))
+			goto err_free;
+
+		sam9x60_pmc->ghws[sam9x60_gck[i].id] = hw;
+	}
+
+	of_clk_add_hw_provider(np, of_clk_hw_pmc_get, sam9x60_pmc);
+
+	return;
+
+err_free:
+	pmc_data_free(sam9x60_pmc);
+}
+/* Some clks are used for a clocksource */
+CLK_OF_DECLARE(sam9x60_pmc, "microchip,sam9x60-pmc", sam9x60_pmc_setup);
diff --git a/drivers/clk/at91/sama5d2.c b/drivers/clk/at91/sama5d2.c
index 81943fac4537..6509d0934804 100644
--- a/drivers/clk/at91/sama5d2.c
+++ b/drivers/clk/at91/sama5d2.c
@@ -16,7 +16,7 @@ static u8 plla_out[] = { 0 };
 
 static u16 plla_icpll[] = { 0 };
 
-static struct clk_range plla_outputs[] = {
+static const struct clk_range plla_outputs[] = {
 	{ .min = 600000000, .max = 1200000000 },
 };
 
@@ -28,6 +28,13 @@ static const struct clk_pll_characteristics plla_characteristics = {
 	.out = plla_out,
 };
 
+static const struct clk_pcr_layout sama5d2_pcr_layout = {
+	.offset = 0x10c,
+	.cmd = BIT(12),
+	.gckcss_mask = GENMASK(10, 8),
+	.pid_mask = GENMASK(6, 0),
+};
+
 static const struct {
 	char *n;
 	char *p;
@@ -274,6 +281,7 @@ static void __init sama5d2_pmc_setup(struct device_node *np)
 
 	for (i = 0; i < ARRAY_SIZE(sama5d2_periphck); i++) {
 		hw = at91_clk_register_sam9x5_peripheral(regmap, &pmc_pcr_lock,
+							 &sama5d2_pcr_layout,
 							 sama5d2_periphck[i].n,
 							 "masterck",
 							 sama5d2_periphck[i].id,
@@ -286,6 +294,7 @@ static void __init sama5d2_pmc_setup(struct device_node *np)
 
 	for (i = 0; i < ARRAY_SIZE(sama5d2_periph32ck); i++) {
 		hw = at91_clk_register_sam9x5_peripheral(regmap, &pmc_pcr_lock,
+							 &sama5d2_pcr_layout,
 							 sama5d2_periph32ck[i].n,
 							 "h32mxck",
 							 sama5d2_periph32ck[i].id,
@@ -304,6 +313,7 @@ static void __init sama5d2_pmc_setup(struct device_node *np)
 	parent_names[5] = "audiopll_pmcck";
 	for (i = 0; i < ARRAY_SIZE(sama5d2_gck); i++) {
 		hw = at91_clk_register_generated(regmap, &pmc_pcr_lock,
+						 &sama5d2_pcr_layout,
 						 sama5d2_gck[i].n,
 						 parent_names, 6,
 						 sama5d2_gck[i].id,
diff --git a/drivers/clk/at91/sama5d4.c b/drivers/clk/at91/sama5d4.c
index b645a9d59cdb..25b156d4e645 100644
--- a/drivers/clk/at91/sama5d4.c
+++ b/drivers/clk/at91/sama5d4.c
@@ -16,7 +16,7 @@ static u8 plla_out[] = { 0 };
 
 static u16 plla_icpll[] = { 0 };
 
-static struct clk_range plla_outputs[] = {
+static const struct clk_range plla_outputs[] = {
 	{ .min = 600000000, .max = 1200000000 },
 };
 
@@ -28,6 +28,12 @@ static const struct clk_pll_characteristics plla_characteristics = {
 	.out = plla_out,
 };
 
+static const struct clk_pcr_layout sama5d4_pcr_layout = {
+	.offset = 0x10c,
+	.cmd = BIT(12),
+	.pid_mask = GENMASK(6, 0),
+};
+
 static const struct {
 	char *n;
 	char *p;
@@ -232,6 +238,7 @@ static void __init sama5d4_pmc_setup(struct device_node *np)
 
 	for (i = 0; i < ARRAY_SIZE(sama5d4_periphck); i++) {
 		hw = at91_clk_register_sam9x5_peripheral(regmap, &pmc_pcr_lock,
+							 &sama5d4_pcr_layout,
 							 sama5d4_periphck[i].n,
 							 "masterck",
 							 sama5d4_periphck[i].id,
@@ -244,6 +251,7 @@ static void __init sama5d4_pmc_setup(struct device_node *np)
 
 	for (i = 0; i < ARRAY_SIZE(sama5d4_periph32ck); i++) {
 		hw = at91_clk_register_sam9x5_peripheral(regmap, &pmc_pcr_lock,
+							 &sama5d4_pcr_layout,
 							 sama5d4_periph32ck[i].n,
 							 "h32mxck",
 							 sama5d4_periph32ck[i].id,
diff --git a/drivers/clk/at91/sckc.c b/drivers/clk/at91/sckc.c
index ab6ecefc49ad..e76b1d64e905 100644
--- a/drivers/clk/at91/sckc.c
+++ b/drivers/clk/at91/sckc.c
@@ -152,28 +152,6 @@ at91_clk_register_slow_osc(void __iomem *sckcr,
 	return hw;
 }
 
-static void __init
-of_at91sam9x5_clk_slow_osc_setup(struct device_node *np, void __iomem *sckcr)
-{
-	struct clk_hw *hw;
-	const char *parent_name;
-	const char *name = np->name;
-	u32 startup;
-	bool bypass;
-
-	parent_name = of_clk_get_parent_name(np, 0);
-	of_property_read_string(np, "clock-output-names", &name);
-	of_property_read_u32(np, "atmel,startup-time-usec", &startup);
-	bypass = of_property_read_bool(np, "atmel,osc-bypass");
-
-	hw = at91_clk_register_slow_osc(sckcr, name, parent_name, startup,
-					 bypass);
-	if (IS_ERR(hw))
-		return;
-
-	of_clk_add_hw_provider(np, of_clk_hw_simple_get, hw);
-}
-
 static unsigned long clk_slow_rc_osc_recalc_rate(struct clk_hw *hw,
 						 unsigned long parent_rate)
 {
@@ -266,28 +244,6 @@ at91_clk_register_slow_rc_osc(void __iomem *sckcr,
 	return hw;
 }
 
-static void __init
-of_at91sam9x5_clk_slow_rc_osc_setup(struct device_node *np, void __iomem *sckcr)
-{
-	struct clk_hw *hw;
-	u32 frequency = 0;
-	u32 accuracy = 0;
-	u32 startup = 0;
-	const char *name = np->name;
-
-	of_property_read_string(np, "clock-output-names", &name);
-	of_property_read_u32(np, "clock-frequency", &frequency);
-	of_property_read_u32(np, "clock-accuracy", &accuracy);
-	of_property_read_u32(np, "atmel,startup-time-usec", &startup);
-
-	hw = at91_clk_register_slow_rc_osc(sckcr, name, frequency, accuracy,
-					    startup);
-	if (IS_ERR(hw))
-		return;
-
-	of_clk_add_hw_provider(np, of_clk_hw_simple_get, hw);
-}
-
 static int clk_sam9x5_slow_set_parent(struct clk_hw *hw, u8 index)
 {
 	struct clk_sam9x5_slow *slowck = to_clk_sam9x5_slow(hw);
@@ -365,68 +321,72 @@ at91_clk_register_sam9x5_slow(void __iomem *sckcr,
 	return hw;
 }
 
-static void __init
-of_at91sam9x5_clk_slow_setup(struct device_node *np, void __iomem *sckcr)
+static void __init at91sam9x5_sckc_register(struct device_node *np,
+					    unsigned int rc_osc_startup_us)
 {
+	const char *parent_names[2] = { "slow_rc_osc", "slow_osc" };
+	void __iomem *regbase = of_iomap(np, 0);
+	struct device_node *child = NULL;
+	const char *xtal_name;
 	struct clk_hw *hw;
-	const char *parent_names[2];
-	unsigned int num_parents;
-	const char *name = np->name;
+	bool bypass;
 
-	num_parents = of_clk_get_parent_count(np);
-	if (num_parents == 0 || num_parents > 2)
+	if (!regbase)
+		return;
+
+	hw = at91_clk_register_slow_rc_osc(regbase, parent_names[0], 32768,
+					   50000000, rc_osc_startup_us);
+	if (IS_ERR(hw))
 		return;
 
-	of_clk_parent_fill(np, parent_names, num_parents);
+	xtal_name = of_clk_get_parent_name(np, 0);
+	if (!xtal_name) {
+		/* DT backward compatibility */
+		child = of_get_compatible_child(np, "atmel,at91sam9x5-clk-slow-osc");
+		if (!child)
+			return;
+
+		xtal_name = of_clk_get_parent_name(child, 0);
+		bypass = of_property_read_bool(child, "atmel,osc-bypass");
+
+		child =  of_get_compatible_child(np, "atmel,at91sam9x5-clk-slow");
+	} else {
+		bypass = of_property_read_bool(np, "atmel,osc-bypass");
+	}
+
+	if (!xtal_name)
+		return;
 
-	of_property_read_string(np, "clock-output-names", &name);
+	hw = at91_clk_register_slow_osc(regbase, parent_names[1], xtal_name,
+					1200000, bypass);
+	if (IS_ERR(hw))
+		return;
 
-	hw = at91_clk_register_sam9x5_slow(sckcr, name, parent_names,
-					    num_parents);
+	hw = at91_clk_register_sam9x5_slow(regbase, "slowck", parent_names, 2);
 	if (IS_ERR(hw))
 		return;
 
 	of_clk_add_hw_provider(np, of_clk_hw_simple_get, hw);
-}
 
-static const struct of_device_id sckc_clk_ids[] __initconst = {
-	/* Slow clock */
-	{
-		.compatible = "atmel,at91sam9x5-clk-slow-osc",
-		.data = of_at91sam9x5_clk_slow_osc_setup,
-	},
-	{
-		.compatible = "atmel,at91sam9x5-clk-slow-rc-osc",
-		.data = of_at91sam9x5_clk_slow_rc_osc_setup,
-	},
-	{
-		.compatible = "atmel,at91sam9x5-clk-slow",
-		.data = of_at91sam9x5_clk_slow_setup,
-	},
-	{ /*sentinel*/ }
-};
+	/* DT backward compatibility */
+	if (child)
+		of_clk_add_hw_provider(child, of_clk_hw_simple_get, hw);
+}
 
 static void __init of_at91sam9x5_sckc_setup(struct device_node *np)
 {
-	struct device_node *childnp;
-	void (*clk_setup)(struct device_node *, void __iomem *);
-	const struct of_device_id *clk_id;
-	void __iomem *regbase = of_iomap(np, 0);
-
-	if (!regbase)
-		return;
-
-	for_each_child_of_node(np, childnp) {
-		clk_id = of_match_node(sckc_clk_ids, childnp);
-		if (!clk_id)
-			continue;
-		clk_setup = clk_id->data;
-		clk_setup(childnp, regbase);
-	}
+	at91sam9x5_sckc_register(np, 75);
 }
 CLK_OF_DECLARE(at91sam9x5_clk_sckc, "atmel,at91sam9x5-sckc",
 	       of_at91sam9x5_sckc_setup);
 
+static void __init of_sama5d3_sckc_setup(struct device_node *np)
+{
+	at91sam9x5_sckc_register(np, 500);
+}
+CLK_OF_DECLARE(sama5d3_clk_sckc, "atmel,sama5d3-sckc",
+	       of_sama5d3_sckc_setup);
+
 static int clk_sama5d4_slow_osc_prepare(struct clk_hw *hw)
 {
 	struct clk_sama5d4_slow_osc *osc = to_clk_sama5d4_slow_osc(hw);
diff --git a/drivers/clk/clk-aspeed.c b/drivers/clk/clk-aspeed.c
index 596136793fc4..42b4df6ba249 100644
--- a/drivers/clk/clk-aspeed.c
+++ b/drivers/clk/clk-aspeed.c
@@ -87,10 +87,10 @@ struct aspeed_clk_gate {
 /* TODO: ask Aspeed about the actual parent data */
 static const struct aspeed_gate_data aspeed_gates[] = {
 	/*				 clk rst   name			parent	flags */
-	[ASPEED_CLK_GATE_ECLK] =	{  0, -1, "eclk-gate",		"eclk",	0 }, /* Video Engine */
+	[ASPEED_CLK_GATE_ECLK] =	{  0,  6, "eclk-gate",		"eclk",	0 }, /* Video Engine */
 	[ASPEED_CLK_GATE_GCLK] =	{  1,  7, "gclk-gate",		NULL,	0 }, /* 2D engine */
 	[ASPEED_CLK_GATE_MCLK] =	{  2, -1, "mclk-gate",		"mpll",	CLK_IS_CRITICAL }, /* SDRAM */
-	[ASPEED_CLK_GATE_VCLK] =	{  3,  6, "vclk-gate",		NULL,	0 }, /* Video Capture */
+	[ASPEED_CLK_GATE_VCLK] =	{  3, -1, "vclk-gate",		NULL,	0 }, /* Video Capture */
 	[ASPEED_CLK_GATE_BCLK] =	{  4,  8, "bclk-gate",		"bclk",	CLK_IS_CRITICAL }, /* PCIe/PCI */
 	[ASPEED_CLK_GATE_DCLK] =	{  5, -1, "dclk-gate",		NULL,	CLK_IS_CRITICAL }, /* DAC */
 	[ASPEED_CLK_GATE_REFCLK] =	{  6, -1, "refclk-gate",	"clkin", CLK_IS_CRITICAL },
@@ -113,6 +113,24 @@ static const struct aspeed_gate_data aspeed_gates[] = {
 	[ASPEED_CLK_GATE_LHCCLK] =	{ 28, -1, "lhclk-gate",		"lhclk", 0 }, /* LPC master/LPC+ */
 };
 
+static const char * const eclk_parent_names[] = {
+	"mpll",
+	"hpll",
+	"dpll",
+};
+
+static const struct clk_div_table ast2500_eclk_div_table[] = {
+	{ 0x0, 2 },
+	{ 0x1, 2 },
+	{ 0x2, 3 },
+	{ 0x3, 4 },
+	{ 0x4, 5 },
+	{ 0x5, 6 },
+	{ 0x6, 7 },
+	{ 0x7, 8 },
+	{ 0 }
+};
+
 static const struct clk_div_table ast2500_mac_div_table[] = {
 	{ 0x0, 4 }, /* Yep, really. Aspeed confirmed this is correct */
 	{ 0x1, 4 },
@@ -192,18 +210,21 @@ static struct clk_hw *aspeed_ast2500_calc_pll(const char *name, u32 val)
 
 struct aspeed_clk_soc_data {
 	const struct clk_div_table *div_table;
+	const struct clk_div_table *eclk_div_table;
 	const struct clk_div_table *mac_div_table;
 	struct clk_hw *(*calc_pll)(const char *name, u32 val);
 };
 
 static const struct aspeed_clk_soc_data ast2500_data = {
 	.div_table = ast2500_div_table,
+	.eclk_div_table = ast2500_eclk_div_table,
 	.mac_div_table = ast2500_mac_div_table,
 	.calc_pll = aspeed_ast2500_calc_pll,
 };
 
 static const struct aspeed_clk_soc_data ast2400_data = {
 	.div_table = ast2400_div_table,
+	.eclk_div_table = ast2400_div_table,
 	.mac_div_table = ast2400_div_table,
 	.calc_pll = aspeed_ast2400_calc_pll,
 };
@@ -522,6 +543,22 @@ static int aspeed_clk_probe(struct platform_device *pdev)
 		return PTR_ERR(hw);
 	aspeed_clk_data->hws[ASPEED_CLK_24M] = hw;
 
+	hw = clk_hw_register_mux(dev, "eclk-mux", eclk_parent_names,
+				 ARRAY_SIZE(eclk_parent_names), 0,
+				 scu_base + ASPEED_CLK_SELECTION, 2, 0x3, 0,
+				 &aspeed_clk_lock);
+	if (IS_ERR(hw))
+		return PTR_ERR(hw);
+	aspeed_clk_data->hws[ASPEED_CLK_ECLK_MUX] = hw;
+
+	hw = clk_hw_register_divider_table(dev, "eclk", "eclk-mux", 0,
+					   scu_base + ASPEED_CLK_SELECTION, 28,
+					   3, 0, soc_data->eclk_div_table,
+					   &aspeed_clk_lock);
+	if (IS_ERR(hw))
+		return PTR_ERR(hw);
+	aspeed_clk_data->hws[ASPEED_CLK_ECLK] = hw;
+
 	/*
 	 * TODO: There are a number of clocks that not included in this driver
 	 * as more information is required:
@@ -531,7 +568,6 @@ static int aspeed_clk_probe(struct platform_device *pdev)
 	 *   RGMII
 	 *   RMII
 	 *   UART[1..5] clock source mux
-	 *   Video Engine (ECLK) mux and clock divider
 	 */
 
 	for (i = 0; i < ARRAY_SIZE(aspeed_gates); i++) {
diff --git a/drivers/clk/clk-composite.c b/drivers/clk/clk-composite.c
index 46604214bba0..b06038b8f658 100644
--- a/drivers/clk/clk-composite.c
+++ b/drivers/clk/clk-composite.c
@@ -218,7 +218,7 @@ struct clk_hw *clk_hw_register_composite(struct device *dev, const char *name,
 		return ERR_PTR(-ENOMEM);
 
 	init.name = name;
-	init.flags = flags | CLK_IS_BASIC;
+	init.flags = flags;
 	init.parent_names = parent_names;
 	init.num_parents = num_parents;
 	hw = &composite->hw;
diff --git a/drivers/clk/clk-divider.c b/drivers/clk/clk-divider.c
index e5a17265cfaf..3f9ff78c4a2a 100644
--- a/drivers/clk/clk-divider.c
+++ b/drivers/clk/clk-divider.c
@@ -25,6 +25,22 @@
  * parent - fixed parent.  No clk_set_parent support
  */
 
+static inline u32 clk_div_readl(struct clk_divider *divider)
+{
+	if (divider->flags & CLK_DIVIDER_BIG_ENDIAN)
+		return ioread32be(divider->reg);
+
+	return readl(divider->reg);
+}
+
+static inline void clk_div_writel(struct clk_divider *divider, u32 val)
+{
+	if (divider->flags & CLK_DIVIDER_BIG_ENDIAN)
+		iowrite32be(val, divider->reg);
+	else
+		writel(val, divider->reg);
+}
+
 static unsigned int _get_table_maxdiv(const struct clk_div_table *table,
 				      u8 width)
 {
@@ -135,7 +151,7 @@ static unsigned long clk_divider_recalc_rate(struct clk_hw *hw,
 	struct clk_divider *divider = to_clk_divider(hw);
 	unsigned int val;
 
-	val = clk_readl(divider->reg) >> divider->shift;
+	val = clk_div_readl(divider) >> divider->shift;
 	val &= clk_div_mask(divider->width);
 
 	return divider_recalc_rate(hw, parent_rate, val, divider->table,
@@ -370,7 +386,7 @@ static long clk_divider_round_rate(struct clk_hw *hw, unsigned long rate,
 	if (divider->flags & CLK_DIVIDER_READ_ONLY) {
 		u32 val;
 
-		val = clk_readl(divider->reg) >> divider->shift;
+		val = clk_div_readl(divider) >> divider->shift;
 		val &= clk_div_mask(divider->width);
 
 		return divider_ro_round_rate(hw, rate, prate, divider->table,
@@ -420,11 +436,11 @@ static int clk_divider_set_rate(struct clk_hw *hw, unsigned long rate,
 	if (divider->flags & CLK_DIVIDER_HIWORD_MASK) {
 		val = clk_div_mask(divider->width) << (divider->shift + 16);
 	} else {
-		val = clk_readl(divider->reg);
+		val = clk_div_readl(divider);
 		val &= ~(clk_div_mask(divider->width) << divider->shift);
 	}
 	val |= (u32)value << divider->shift;
-	clk_writel(val, divider->reg);
+	clk_div_writel(divider, val);
 
 	if (divider->lock)
 		spin_unlock_irqrestore(divider->lock, flags);
@@ -475,7 +491,7 @@ static struct clk_hw *_register_divider(struct device *dev, const char *name,
 		init.ops = &clk_divider_ro_ops;
 	else
 		init.ops = &clk_divider_ops;
-	init.flags = flags | CLK_IS_BASIC;
+	init.flags = flags;
 	init.parent_names = (parent_name ? &parent_name: NULL);
 	init.num_parents = (parent_name ? 1 : 0);
 
diff --git a/drivers/clk/clk-fixed-factor.c b/drivers/clk/clk-fixed-factor.c
index 241b3f8c61a9..8b343e59dc61 100644
--- a/drivers/clk/clk-fixed-factor.c
+++ b/drivers/clk/clk-fixed-factor.c
@@ -64,12 +64,14 @@ const struct clk_ops clk_fixed_factor_ops = {
 };
 EXPORT_SYMBOL_GPL(clk_fixed_factor_ops);
 
-struct clk_hw *clk_hw_register_fixed_factor(struct device *dev,
-		const char *name, const char *parent_name, unsigned long flags,
-		unsigned int mult, unsigned int div)
+static struct clk_hw *
+__clk_hw_register_fixed_factor(struct device *dev, struct device_node *np,
+		const char *name, const char *parent_name, int index,
+		unsigned long flags, unsigned int mult, unsigned int div)
 {
 	struct clk_fixed_factor *fix;
-	struct clk_init_data init;
+	struct clk_init_data init = { };
+	struct clk_parent_data pdata = { .index = index };
 	struct clk_hw *hw;
 	int ret;
 
@@ -84,12 +86,18 @@ struct clk_hw *clk_hw_register_fixed_factor(struct device *dev,
 
 	init.name = name;
 	init.ops = &clk_fixed_factor_ops;
-	init.flags = flags | CLK_IS_BASIC;
-	init.parent_names = &parent_name;
+	init.flags = flags;
+	if (parent_name)
+		init.parent_names = &parent_name;
+	else
+		init.parent_data = &pdata;
 	init.num_parents = 1;
 
 	hw = &fix->hw;
-	ret = clk_hw_register(dev, hw);
+	if (dev)
+		ret = clk_hw_register(dev, hw);
+	else
+		ret = of_clk_hw_register(np, hw);
 	if (ret) {
 		kfree(fix);
 		hw = ERR_PTR(ret);
@@ -97,6 +105,14 @@ struct clk_hw *clk_hw_register_fixed_factor(struct device *dev,
 
 	return hw;
 }
+
+struct clk_hw *clk_hw_register_fixed_factor(struct device *dev,
+		const char *name, const char *parent_name, unsigned long flags,
+		unsigned int mult, unsigned int div)
+{
+	return __clk_hw_register_fixed_factor(dev, NULL, name, parent_name, -1,
+					      flags, mult, div);
+}
 EXPORT_SYMBOL_GPL(clk_hw_register_fixed_factor);
 
 struct clk *clk_register_fixed_factor(struct device *dev, const char *name,
@@ -143,11 +159,10 @@ static const struct of_device_id set_rate_parent_matches[] = {
 	{ /* Sentinel */ },
 };
 
-static struct clk *_of_fixed_factor_clk_setup(struct device_node *node)
+static struct clk_hw *_of_fixed_factor_clk_setup(struct device_node *node)
 {
-	struct clk *clk;
+	struct clk_hw *hw;
 	const char *clk_name = node->name;
-	const char *parent_name;
 	unsigned long flags = 0;
 	u32 div, mult;
 	int ret;
@@ -165,30 +180,28 @@ static struct clk *_of_fixed_factor_clk_setup(struct device_node *node)
 	}
 
 	of_property_read_string(node, "clock-output-names", &clk_name);
-	parent_name = of_clk_get_parent_name(node, 0);
 
 	if (of_match_node(set_rate_parent_matches, node))
 		flags |= CLK_SET_RATE_PARENT;
 
-	clk = clk_register_fixed_factor(NULL, clk_name, parent_name, flags,
-					mult, div);
-	if (IS_ERR(clk)) {
+	hw = __clk_hw_register_fixed_factor(NULL, node, clk_name, NULL, 0,
+					    flags, mult, div);
+	if (IS_ERR(hw)) {
 		/*
-		 * If parent clock is not registered, registration would fail.
 		 * Clear OF_POPULATED flag so that clock registration can be
 		 * attempted again from probe function.
 		 */
 		of_node_clear_flag(node, OF_POPULATED);
-		return clk;
+		return ERR_CAST(hw);
 	}
 
-	ret = of_clk_add_provider(node, of_clk_src_simple_get, clk);
+	ret = of_clk_add_hw_provider(node, of_clk_hw_simple_get, hw);
 	if (ret) {
-		clk_unregister(clk);
+		clk_hw_unregister_fixed_factor(hw);
 		return ERR_PTR(ret);
 	}
 
-	return clk;
+	return hw;
 }
 
 /**
@@ -203,17 +216,17 @@ CLK_OF_DECLARE(fixed_factor_clk, "fixed-factor-clock",
 
 static int of_fixed_factor_clk_remove(struct platform_device *pdev)
 {
-	struct clk *clk = platform_get_drvdata(pdev);
+	struct clk_hw *clk = platform_get_drvdata(pdev);
 
 	of_clk_del_provider(pdev->dev.of_node);
-	clk_unregister_fixed_factor(clk);
+	clk_hw_unregister_fixed_factor(clk);
 
 	return 0;
 }
 
 static int of_fixed_factor_clk_probe(struct platform_device *pdev)
 {
-	struct clk *clk;
+	struct clk_hw *clk;
 
 	/*
 	 * This function is not executed when of_fixed_factor_clk_setup
diff --git a/drivers/clk/clk-fixed-rate.c b/drivers/clk/clk-fixed-rate.c
index 00ef4f5e53fe..a7e4aef7a376 100644
--- a/drivers/clk/clk-fixed-rate.c
+++ b/drivers/clk/clk-fixed-rate.c
@@ -68,7 +68,7 @@ struct clk_hw *clk_hw_register_fixed_rate_with_accuracy(struct device *dev,
 
 	init.name = name;
 	init.ops = &clk_fixed_rate_ops;
-	init.flags = flags | CLK_IS_BASIC;
+	init.flags = flags;
 	init.parent_names = (parent_name ? &parent_name: NULL);
 	init.num_parents = (parent_name ? 1 : 0);
 
diff --git a/drivers/clk/clk-fractional-divider.c b/drivers/clk/clk-fractional-divider.c
index fdfe2e423d15..d81f1d2e9129 100644
--- a/drivers/clk/clk-fractional-divider.c
+++ b/drivers/clk/clk-fractional-divider.c
@@ -13,6 +13,22 @@
 #include <linux/slab.h>
 #include <linux/rational.h>
 
+static inline u32 clk_fd_readl(struct clk_fractional_divider *fd)
+{
+	if (fd->flags & CLK_FRAC_DIVIDER_BIG_ENDIAN)
+		return ioread32be(fd->reg);
+
+	return readl(fd->reg);
+}
+
+static inline void clk_fd_writel(struct clk_fractional_divider *fd, u32 val)
+{
+	if (fd->flags & CLK_FRAC_DIVIDER_BIG_ENDIAN)
+		iowrite32be(val, fd->reg);
+	else
+		writel(val, fd->reg);
+}
+
 static unsigned long clk_fd_recalc_rate(struct clk_hw *hw,
 					unsigned long parent_rate)
 {
@@ -27,7 +43,7 @@ static unsigned long clk_fd_recalc_rate(struct clk_hw *hw,
 	else
 		__acquire(fd->lock);
 
-	val = clk_readl(fd->reg);
+	val = clk_fd_readl(fd);
 
 	if (fd->lock)
 		spin_unlock_irqrestore(fd->lock, flags);
@@ -115,10 +131,10 @@ static int clk_fd_set_rate(struct clk_hw *hw, unsigned long rate,
 	else
 		__acquire(fd->lock);
 
-	val = clk_readl(fd->reg);
+	val = clk_fd_readl(fd);
 	val &= ~(fd->mmask | fd->nmask);
 	val |= (m << fd->mshift) | (n << fd->nshift);
-	clk_writel(val, fd->reg);
+	clk_fd_writel(fd, val);
 
 	if (fd->lock)
 		spin_unlock_irqrestore(fd->lock, flags);
@@ -151,7 +167,7 @@ struct clk_hw *clk_hw_register_fractional_divider(struct device *dev,
 
 	init.name = name;
 	init.ops = &clk_fractional_divider_ops;
-	init.flags = flags | CLK_IS_BASIC;
+	init.flags = flags;
 	init.parent_names = parent_name ? &parent_name : NULL;
 	init.num_parents = parent_name ? 1 : 0;
 
diff --git a/drivers/clk/clk-gate.c b/drivers/clk/clk-gate.c
index f05823cd9b21..1b99fc962745 100644
--- a/drivers/clk/clk-gate.c
+++ b/drivers/clk/clk-gate.c
@@ -23,6 +23,22 @@
  * parent - fixed parent.  No clk_set_parent support
  */
 
+static inline u32 clk_gate_readl(struct clk_gate *gate)
+{
+	if (gate->flags & CLK_GATE_BIG_ENDIAN)
+		return ioread32be(gate->reg);
+
+	return readl(gate->reg);
+}
+
+static inline void clk_gate_writel(struct clk_gate *gate, u32 val)
+{
+	if (gate->flags & CLK_GATE_BIG_ENDIAN)
+		iowrite32be(val, gate->reg);
+	else
+		writel(val, gate->reg);
+}
+
 /*
  * It works on following logic:
  *
@@ -55,7 +71,7 @@ static void clk_gate_endisable(struct clk_hw *hw, int enable)
 		if (set)
 			reg |= BIT(gate->bit_idx);
 	} else {
-		reg = clk_readl(gate->reg);
+		reg = clk_gate_readl(gate);
 
 		if (set)
 			reg |= BIT(gate->bit_idx);
@@ -63,7 +79,7 @@ static void clk_gate_endisable(struct clk_hw *hw, int enable)
 			reg &= ~BIT(gate->bit_idx);
 	}
 
-	clk_writel(reg, gate->reg);
+	clk_gate_writel(gate, reg);
 
 	if (gate->lock)
 		spin_unlock_irqrestore(gate->lock, flags);
@@ -88,7 +104,7 @@ int clk_gate_is_enabled(struct clk_hw *hw)
 	u32 reg;
 	struct clk_gate *gate = to_clk_gate(hw);
 
-	reg = clk_readl(gate->reg);
+	reg = clk_gate_readl(gate);
 
 	/* if a set bit disables this clk, flip it before masking */
 	if (gate->flags & CLK_GATE_SET_TO_DISABLE)
@@ -142,7 +158,7 @@ struct clk_hw *clk_hw_register_gate(struct device *dev, const char *name,
 
 	init.name = name;
 	init.ops = &clk_gate_ops;
-	init.flags = flags | CLK_IS_BASIC;
+	init.flags = flags;
 	init.parent_names = parent_name ? &parent_name : NULL;
 	init.num_parents = parent_name ? 1 : 0;
 
diff --git a/drivers/clk/clk-gpio.c b/drivers/clk/clk-gpio.c
index c2f07f0d077c..9d930edd6516 100644
--- a/drivers/clk/clk-gpio.c
+++ b/drivers/clk/clk-gpio.c
@@ -137,7 +137,7 @@ static struct clk_hw *clk_register_gpio(struct device *dev, const char *name,
 
 	init.name = name;
 	init.ops = clk_gpio_ops;
-	init.flags = flags | CLK_IS_BASIC;
+	init.flags = flags;
 	init.parent_names = parent_names;
 	init.num_parents = num_parents;
 
diff --git a/drivers/clk/clk-highbank.c b/drivers/clk/clk-highbank.c
index 8e4581004695..bd328b0eb243 100644
--- a/drivers/clk/clk-highbank.c
+++ b/drivers/clk/clk-highbank.c
@@ -17,7 +17,6 @@
 #include <linux/kernel.h>
 #include <linux/slab.h>
 #include <linux/err.h>
-#include <linux/clk.h>
 #include <linux/clk-provider.h>
 #include <linux/io.h>
 #include <linux/of.h>
@@ -272,7 +271,7 @@ static const struct clk_ops periclk_ops = {
 	.set_rate = clk_periclk_set_rate,
 };
 
-static __init struct clk *hb_clk_init(struct device_node *node, const struct clk_ops *ops)
+static void __init hb_clk_init(struct device_node *node, const struct clk_ops *ops, unsigned long clkflags)
 {
 	u32 reg;
 	struct hb_clk *hb_clk;
@@ -284,11 +283,11 @@ static __init struct clk *hb_clk_init(struct device_node *node, const struct clk
 
 	rc = of_property_read_u32(node, "reg", &reg);
 	if (WARN_ON(rc))
-		return NULL;
+		return;
 
 	hb_clk = kzalloc(sizeof(*hb_clk), GFP_KERNEL);
 	if (WARN_ON(!hb_clk))
-		return NULL;
+		return;
 
 	/* Map system registers */
 	srnp = of_find_compatible_node(NULL, NULL, "calxeda,hb-sregs");
@@ -301,7 +300,7 @@ static __init struct clk *hb_clk_init(struct device_node *node, const struct clk
 
 	init.name = clk_name;
 	init.ops = ops;
-	init.flags = 0;
+	init.flags = clkflags;
 	parent_name = of_clk_get_parent_name(node, 0);
 	init.parent_names = &parent_name;
 	init.num_parents = 1;
@@ -311,33 +310,31 @@ static __init struct clk *hb_clk_init(struct device_node *node, const struct clk
 	rc = clk_hw_register(NULL, &hb_clk->hw);
 	if (WARN_ON(rc)) {
 		kfree(hb_clk);
-		return NULL;
+		return;
 	}
-	rc = of_clk_add_hw_provider(node, of_clk_hw_simple_get, &hb_clk->hw);
-	return hb_clk->hw.clk;
+	of_clk_add_hw_provider(node, of_clk_hw_simple_get, &hb_clk->hw);
 }
 
 static void __init hb_pll_init(struct device_node *node)
 {
-	hb_clk_init(node, &clk_pll_ops);
+	hb_clk_init(node, &clk_pll_ops, 0);
 }
 CLK_OF_DECLARE(hb_pll, "calxeda,hb-pll-clock", hb_pll_init);
 
 static void __init hb_a9periph_init(struct device_node *node)
 {
-	hb_clk_init(node, &a9periphclk_ops);
+	hb_clk_init(node, &a9periphclk_ops, 0);
 }
 CLK_OF_DECLARE(hb_a9periph, "calxeda,hb-a9periph-clock", hb_a9periph_init);
 
 static void __init hb_a9bus_init(struct device_node *node)
 {
-	struct clk *clk = hb_clk_init(node, &a9bclk_ops);
-	clk_prepare_enable(clk);
+	hb_clk_init(node, &a9bclk_ops, CLK_IS_CRITICAL);
 }
 CLK_OF_DECLARE(hb_a9bus, "calxeda,hb-a9bus-clock", hb_a9bus_init);
 
 static void __init hb_emmc_init(struct device_node *node)
 {
-	hb_clk_init(node, &periclk_ops);
+	hb_clk_init(node, &periclk_ops, 0);
 }
 CLK_OF_DECLARE(hb_emmc, "calxeda,hb-emmc-clock", hb_emmc_init);
diff --git a/drivers/clk/clk-lochnagar.c b/drivers/clk/clk-lochnagar.c
new file mode 100644
index 000000000000..a2f31e58ee48
--- /dev/null
+++ b/drivers/clk/clk-lochnagar.c
@@ -0,0 +1,336 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Lochnagar clock control
+ *
+ * Copyright (c) 2017-2018 Cirrus Logic, Inc. and
+ *                         Cirrus Logic International Semiconductor Ltd.
+ *
+ * Author: Charles Keepax <ckeepax@opensource.cirrus.com>
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/device.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#include <linux/mfd/lochnagar.h>
+#include <linux/mfd/lochnagar1_regs.h>
+#include <linux/mfd/lochnagar2_regs.h>
+
+#include <dt-bindings/clk/lochnagar.h>
+
+#define LOCHNAGAR_NUM_CLOCKS	(LOCHNAGAR_SPDIF_CLKOUT + 1)
+
+struct lochnagar_clk {
+	const char * const name;
+	struct clk_hw hw;
+
+	struct lochnagar_clk_priv *priv;
+
+	u16 cfg_reg;
+	u16 ena_mask;
+
+	u16 src_reg;
+	u16 src_mask;
+};
+
+struct lochnagar_clk_priv {
+	struct device *dev;
+	struct regmap *regmap;
+	enum lochnagar_type type;
+
+	const char **parents;
+	unsigned int nparents;
+
+	struct lochnagar_clk lclks[LOCHNAGAR_NUM_CLOCKS];
+};
+
+static const char * const lochnagar1_clk_parents[] = {
+	"ln-none",
+	"ln-spdif-mclk",
+	"ln-psia1-mclk",
+	"ln-psia2-mclk",
+	"ln-cdc-clkout",
+	"ln-dsp-clkout",
+	"ln-pmic-32k",
+	"ln-gf-mclk1",
+	"ln-gf-mclk3",
+	"ln-gf-mclk2",
+	"ln-gf-mclk4",
+};
+
+static const char * const lochnagar2_clk_parents[] = {
+	"ln-none",
+	"ln-cdc-clkout",
+	"ln-dsp-clkout",
+	"ln-pmic-32k",
+	"ln-spdif-mclk",
+	"ln-clk-12m",
+	"ln-clk-11m",
+	"ln-clk-24m",
+	"ln-clk-22m",
+	"ln-clk-8m",
+	"ln-usb-clk-24m",
+	"ln-gf-mclk1",
+	"ln-gf-mclk3",
+	"ln-gf-mclk2",
+	"ln-psia1-mclk",
+	"ln-psia2-mclk",
+	"ln-spdif-clkout",
+	"ln-adat-mclk",
+	"ln-usb-clk-12m",
+};
+
+#define LN1_CLK(ID, NAME, REG) \
+	[LOCHNAGAR_##ID] = { \
+		.name = NAME, \
+		.cfg_reg = LOCHNAGAR1_##REG, \
+		.ena_mask = LOCHNAGAR1_##ID##_ENA_MASK, \
+		.src_reg = LOCHNAGAR1_##ID##_SEL, \
+		.src_mask = LOCHNAGAR1_SRC_MASK, \
+	}
+
+#define LN2_CLK(ID, NAME) \
+	[LOCHNAGAR_##ID] = { \
+		.name = NAME, \
+		.cfg_reg = LOCHNAGAR2_##ID##_CTRL, \
+		.src_reg = LOCHNAGAR2_##ID##_CTRL, \
+		.ena_mask = LOCHNAGAR2_CLK_ENA_MASK, \
+		.src_mask = LOCHNAGAR2_CLK_SRC_MASK, \
+	}
+
+static const struct lochnagar_clk lochnagar1_clks[LOCHNAGAR_NUM_CLOCKS] = {
+	LN1_CLK(CDC_MCLK1,      "ln-cdc-mclk1",  CDC_AIF_CTRL2),
+	LN1_CLK(CDC_MCLK2,      "ln-cdc-mclk2",  CDC_AIF_CTRL2),
+	LN1_CLK(DSP_CLKIN,      "ln-dsp-clkin",  DSP_AIF),
+	LN1_CLK(GF_CLKOUT1,     "ln-gf-clkout1", GF_AIF1),
+};
+
+static const struct lochnagar_clk lochnagar2_clks[LOCHNAGAR_NUM_CLOCKS] = {
+	LN2_CLK(CDC_MCLK1,      "ln-cdc-mclk1"),
+	LN2_CLK(CDC_MCLK2,      "ln-cdc-mclk2"),
+	LN2_CLK(DSP_CLKIN,      "ln-dsp-clkin"),
+	LN2_CLK(GF_CLKOUT1,     "ln-gf-clkout1"),
+	LN2_CLK(GF_CLKOUT2,     "ln-gf-clkout2"),
+	LN2_CLK(PSIA1_MCLK,     "ln-psia1-mclk"),
+	LN2_CLK(PSIA2_MCLK,     "ln-psia2-mclk"),
+	LN2_CLK(SPDIF_MCLK,     "ln-spdif-mclk"),
+	LN2_CLK(ADAT_MCLK,      "ln-adat-mclk"),
+	LN2_CLK(SOUNDCARD_MCLK, "ln-soundcard-mclk"),
+};
+
+static inline struct lochnagar_clk *lochnagar_hw_to_lclk(struct clk_hw *hw)
+{
+	return container_of(hw, struct lochnagar_clk, hw);
+}
+
+static int lochnagar_clk_prepare(struct clk_hw *hw)
+{
+	struct lochnagar_clk *lclk = lochnagar_hw_to_lclk(hw);
+	struct lochnagar_clk_priv *priv = lclk->priv;
+	struct regmap *regmap = priv->regmap;
+	int ret;
+
+	ret = regmap_update_bits(regmap, lclk->cfg_reg,
+				 lclk->ena_mask, lclk->ena_mask);
+	if (ret < 0)
+		dev_dbg(priv->dev, "Failed to prepare %s: %d\n",
+			lclk->name, ret);
+
+	return ret;
+}
+
+static void lochnagar_clk_unprepare(struct clk_hw *hw)
+{
+	struct lochnagar_clk *lclk = lochnagar_hw_to_lclk(hw);
+	struct lochnagar_clk_priv *priv = lclk->priv;
+	struct regmap *regmap = priv->regmap;
+	int ret;
+
+	ret = regmap_update_bits(regmap, lclk->cfg_reg, lclk->ena_mask, 0);
+	if (ret < 0)
+		dev_dbg(priv->dev, "Failed to unprepare %s: %d\n",
+			lclk->name, ret);
+}
+
+static int lochnagar_clk_set_parent(struct clk_hw *hw, u8 index)
+{
+	struct lochnagar_clk *lclk = lochnagar_hw_to_lclk(hw);
+	struct lochnagar_clk_priv *priv = lclk->priv;
+	struct regmap *regmap = priv->regmap;
+	int ret;
+
+	ret = regmap_update_bits(regmap, lclk->src_reg, lclk->src_mask, index);
+	if (ret < 0)
+		dev_dbg(priv->dev, "Failed to reparent %s: %d\n",
+			lclk->name, ret);
+
+	return ret;
+}
+
+static u8 lochnagar_clk_get_parent(struct clk_hw *hw)
+{
+	struct lochnagar_clk *lclk = lochnagar_hw_to_lclk(hw);
+	struct lochnagar_clk_priv *priv = lclk->priv;
+	struct regmap *regmap = priv->regmap;
+	unsigned int val;
+	int ret;
+
+	ret = regmap_read(regmap, lclk->src_reg, &val);
+	if (ret < 0) {
+		dev_dbg(priv->dev, "Failed to read parent of %s: %d\n",
+			lclk->name, ret);
+		return priv->nparents;
+	}
+
+	val &= lclk->src_mask;
+
+	return val;
+}
+
+static const struct clk_ops lochnagar_clk_ops = {
+	.prepare = lochnagar_clk_prepare,
+	.unprepare = lochnagar_clk_unprepare,
+	.set_parent = lochnagar_clk_set_parent,
+	.get_parent = lochnagar_clk_get_parent,
+};
+
+static int lochnagar_init_parents(struct lochnagar_clk_priv *priv)
+{
+	struct device_node *np = priv->dev->of_node;
+	int i, j;
+
+	switch (priv->type) {
+	case LOCHNAGAR1:
+		memcpy(priv->lclks, lochnagar1_clks, sizeof(lochnagar1_clks));
+
+		priv->nparents = ARRAY_SIZE(lochnagar1_clk_parents);
+		priv->parents = devm_kmemdup(priv->dev, lochnagar1_clk_parents,
+					     sizeof(lochnagar1_clk_parents),
+					     GFP_KERNEL);
+		break;
+	case LOCHNAGAR2:
+		memcpy(priv->lclks, lochnagar2_clks, sizeof(lochnagar2_clks));
+
+		priv->nparents = ARRAY_SIZE(lochnagar2_clk_parents);
+		priv->parents = devm_kmemdup(priv->dev, lochnagar2_clk_parents,
+					     sizeof(lochnagar2_clk_parents),
+					     GFP_KERNEL);
+		break;
+	default:
+		dev_err(priv->dev, "Unknown Lochnagar type: %d\n", priv->type);
+		return -EINVAL;
+	}
+
+	if (!priv->parents)
+		return -ENOMEM;
+
+	for (i = 0; i < priv->nparents; i++) {
+		j = of_property_match_string(np, "clock-names",
+					     priv->parents[i]);
+		if (j >= 0)
+			priv->parents[i] = of_clk_get_parent_name(np, j);
+	}
+
+	return 0;
+}
+
+static struct clk_hw *
+lochnagar_of_clk_hw_get(struct of_phandle_args *clkspec, void *data)
+{
+	struct lochnagar_clk_priv *priv = data;
+	unsigned int idx = clkspec->args[0];
+
+	if (idx >= ARRAY_SIZE(priv->lclks)) {
+		dev_err(priv->dev, "Invalid index %u\n", idx);
+		return ERR_PTR(-EINVAL);
+	}
+
+	return &priv->lclks[idx].hw;
+}
+
+static int lochnagar_init_clks(struct lochnagar_clk_priv *priv)
+{
+	struct clk_init_data clk_init = {
+		.ops = &lochnagar_clk_ops,
+		.parent_names = priv->parents,
+		.num_parents = priv->nparents,
+	};
+	struct lochnagar_clk *lclk;
+	int ret, i;
+
+	for (i = 0; i < ARRAY_SIZE(priv->lclks); i++) {
+		lclk = &priv->lclks[i];
+
+		if (!lclk->name)
+			continue;
+
+		clk_init.name = lclk->name;
+
+		lclk->priv = priv;
+		lclk->hw.init = &clk_init;
+
+		ret = devm_clk_hw_register(priv->dev, &lclk->hw);
+		if (ret) {
+			dev_err(priv->dev, "Failed to register %s: %d\n",
+				lclk->name, ret);
+			return ret;
+		}
+	}
+
+	ret = devm_of_clk_add_hw_provider(priv->dev, lochnagar_of_clk_hw_get,
+					  priv);
+	if (ret < 0)
+		dev_err(priv->dev, "Failed to register provider: %d\n", ret);
+
+	return ret;
+}
+
+static const struct of_device_id lochnagar_of_match[] = {
+	{ .compatible = "cirrus,lochnagar1-clk", .data = (void *)LOCHNAGAR1 },
+	{ .compatible = "cirrus,lochnagar2-clk", .data = (void *)LOCHNAGAR2 },
+	{}
+};
+MODULE_DEVICE_TABLE(of, lochnagar_of_match);
+
+static int lochnagar_clk_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct lochnagar_clk_priv *priv;
+	const struct of_device_id *of_id;
+	int ret;
+
+	of_id = of_match_device(lochnagar_of_match, dev);
+	if (!of_id)
+		return -EINVAL;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->dev = dev;
+	priv->regmap = dev_get_regmap(dev->parent, NULL);
+	priv->type = (enum lochnagar_type)of_id->data;
+
+	ret = lochnagar_init_parents(priv);
+	if (ret)
+		return ret;
+
+	return lochnagar_init_clks(priv);
+}
+
+static struct platform_driver lochnagar_clk_driver = {
+	.driver = {
+		.name = "lochnagar-clk",
+		.of_match_table = lochnagar_of_match,
+	},
+	.probe = lochnagar_clk_probe,
+};
+module_platform_driver(lochnagar_clk_driver);
+
+MODULE_AUTHOR("Charles Keepax <ckeepax@opensource.cirrus.com>");
+MODULE_DESCRIPTION("Clock driver for Cirrus Logic Lochnagar Board");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/clk/clk-milbeaut.c b/drivers/clk/clk-milbeaut.c
new file mode 100644
index 000000000000..5fc78faf820c
--- /dev/null
+++ b/drivers/clk/clk-milbeaut.c
@@ -0,0 +1,663 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2018 Socionext Inc.
+ * Copyright (C) 2016 Linaro Ltd.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/of_address.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+
+#define M10V_CLKSEL1		0x0
+#define CLKSEL(n)	(((n) - 1) * 4 + M10V_CLKSEL1)
+
+#define M10V_PLL1		"pll1"
+#define M10V_PLL1DIV2		"pll1-2"
+#define M10V_PLL2		"pll2"
+#define M10V_PLL2DIV2		"pll2-2"
+#define M10V_PLL6		"pll6"
+#define M10V_PLL6DIV2		"pll6-2"
+#define M10V_PLL6DIV3		"pll6-3"
+#define M10V_PLL7		"pll7"
+#define M10V_PLL7DIV2		"pll7-2"
+#define M10V_PLL7DIV5		"pll7-5"
+#define M10V_PLL9		"pll9"
+#define M10V_PLL10		"pll10"
+#define M10V_PLL10DIV2		"pll10-2"
+#define M10V_PLL11		"pll11"
+
+#define M10V_SPI_PARENT0	"spi-parent0"
+#define M10V_SPI_PARENT1	"spi-parent1"
+#define M10V_SPI_PARENT2	"spi-parent2"
+#define M10V_UHS1CLK2_PARENT0	"uhs1clk2-parent0"
+#define M10V_UHS1CLK2_PARENT1	"uhs1clk2-parent1"
+#define M10V_UHS1CLK2_PARENT2	"uhs1clk2-parent2"
+#define M10V_UHS1CLK1_PARENT0	"uhs1clk1-parent0"
+#define M10V_UHS1CLK1_PARENT1	"uhs1clk1-parent1"
+#define M10V_NFCLK_PARENT0	"nfclk-parent0"
+#define M10V_NFCLK_PARENT1	"nfclk-parent1"
+#define M10V_NFCLK_PARENT2	"nfclk-parent2"
+#define M10V_NFCLK_PARENT3	"nfclk-parent3"
+#define M10V_NFCLK_PARENT4	"nfclk-parent4"
+#define M10V_NFCLK_PARENT5	"nfclk-parent5"
+
+#define M10V_DCHREQ		1
+#define M10V_UPOLL_RATE		1
+#define M10V_UTIMEOUT		250
+
+#define M10V_EMMCCLK_ID		0
+#define M10V_ACLK_ID		1
+#define M10V_HCLK_ID		2
+#define M10V_PCLK_ID		3
+#define M10V_RCLK_ID		4
+#define M10V_SPICLK_ID		5
+#define M10V_NFCLK_ID		6
+#define M10V_UHS1CLK2_ID	7
+#define M10V_NUM_CLKS		8
+
+#define to_m10v_div(_hw)        container_of(_hw, struct m10v_clk_divider, hw)
+
+static struct clk_hw_onecell_data *m10v_clk_data;
+
+static DEFINE_SPINLOCK(m10v_crglock);
+
+struct m10v_clk_div_factors {
+	const char			*name;
+	const char			*parent_name;
+	u32				offset;
+	u8				shift;
+	u8				width;
+	const struct clk_div_table	*table;
+	unsigned long			div_flags;
+	int				onecell_idx;
+};
+
+struct m10v_clk_div_fixed_data {
+	const char	*name;
+	const char	*parent_name;
+	u8		div;
+	u8		mult;
+	int		onecell_idx;
+};
+
+struct m10v_clk_mux_factors {
+	const char		*name;
+	const char * const	*parent_names;
+	u8			num_parents;
+	u32			offset;
+	u8			shift;
+	u8			mask;
+	u32			*table;
+	unsigned long		mux_flags;
+	int			onecell_idx;
+};
+
+static const struct clk_div_table emmcclk_table[] = {
+	{ .val = 0, .div = 8 },
+	{ .val = 1, .div = 9 },
+	{ .val = 2, .div = 10 },
+	{ .val = 3, .div = 15 },
+	{ .div = 0 },
+};
+
+static const struct clk_div_table mclk400_table[] = {
+	{ .val = 1, .div = 2 },
+	{ .val = 3, .div = 4 },
+	{ .div = 0 },
+};
+
+static const struct clk_div_table mclk200_table[] = {
+	{ .val = 3, .div = 4 },
+	{ .val = 7, .div = 8 },
+	{ .div = 0 },
+};
+
+static const struct clk_div_table aclk400_table[] = {
+	{ .val = 1, .div = 2 },
+	{ .val = 3, .div = 4 },
+	{ .div = 0 },
+};
+
+static const struct clk_div_table aclk300_table[] = {
+	{ .val = 0, .div = 2 },
+	{ .val = 1, .div = 3 },
+	{ .div = 0 },
+};
+
+static const struct clk_div_table aclk_table[] = {
+	{ .val = 3, .div = 4 },
+	{ .val = 7, .div = 8 },
+	{ .div = 0 },
+};
+
+static const struct clk_div_table aclkexs_table[] = {
+	{ .val = 3, .div = 4 },
+	{ .val = 4, .div = 5 },
+	{ .val = 5, .div = 6 },
+	{ .val = 7, .div = 8 },
+	{ .div = 0 },
+};
+
+static const struct clk_div_table hclk_table[] = {
+	{ .val = 7, .div = 8 },
+	{ .val = 15, .div = 16 },
+	{ .div = 0 },
+};
+
+static const struct clk_div_table hclkbmh_table[] = {
+	{ .val = 3, .div = 4 },
+	{ .val = 7, .div = 8 },
+	{ .div = 0 },
+};
+
+static const struct clk_div_table pclk_table[] = {
+	{ .val = 15, .div = 16 },
+	{ .val = 31, .div = 32 },
+	{ .div = 0 },
+};
+
+static const struct clk_div_table rclk_table[] = {
+	{ .val = 0, .div = 8 },
+	{ .val = 1, .div = 16 },
+	{ .val = 2, .div = 24 },
+	{ .val = 3, .div = 32 },
+	{ .div = 0 },
+};
+
+static const struct clk_div_table uhs1clk0_table[] = {
+	{ .val = 0, .div = 2 },
+	{ .val = 1, .div = 3 },
+	{ .val = 2, .div = 4 },
+	{ .val = 3, .div = 8 },
+	{ .val = 4, .div = 16 },
+	{ .div = 0 },
+};
+
+static const struct clk_div_table uhs2clk_table[] = {
+	{ .val = 0, .div = 9 },
+	{ .val = 1, .div = 10 },
+	{ .val = 2, .div = 11 },
+	{ .val = 3, .div = 12 },
+	{ .val = 4, .div = 13 },
+	{ .val = 5, .div = 14 },
+	{ .val = 6, .div = 16 },
+	{ .val = 7, .div = 18 },
+	{ .div = 0 },
+};
+
+static u32 spi_mux_table[] = {0, 1, 2};
+static const char * const spi_mux_names[] = {
+	M10V_SPI_PARENT0, M10V_SPI_PARENT1, M10V_SPI_PARENT2
+};
+
+static u32 uhs1clk2_mux_table[] = {2, 3, 4, 8};
+static const char * const uhs1clk2_mux_names[] = {
+	M10V_UHS1CLK2_PARENT0, M10V_UHS1CLK2_PARENT1,
+	M10V_UHS1CLK2_PARENT2, M10V_PLL6DIV2
+};
+
+static u32 uhs1clk1_mux_table[] = {3, 4, 8};
+static const char * const uhs1clk1_mux_names[] = {
+	M10V_UHS1CLK1_PARENT0, M10V_UHS1CLK1_PARENT1, M10V_PLL6DIV2
+};
+
+static u32 nfclk_mux_table[] = {0, 1, 2, 3, 4, 8};
+static const char * const nfclk_mux_names[] = {
+	M10V_NFCLK_PARENT0, M10V_NFCLK_PARENT1, M10V_NFCLK_PARENT2,
+	M10V_NFCLK_PARENT3, M10V_NFCLK_PARENT4, M10V_NFCLK_PARENT5
+};
+
+static const struct m10v_clk_div_fixed_data m10v_pll_fixed_data[] = {
+	{M10V_PLL1, NULL, 1, 40, -1},
+	{M10V_PLL2, NULL, 1, 30, -1},
+	{M10V_PLL6, NULL, 1, 35, -1},
+	{M10V_PLL7, NULL, 1, 40, -1},
+	{M10V_PLL9, NULL, 1, 33, -1},
+	{M10V_PLL10, NULL, 5, 108, -1},
+	{M10V_PLL10DIV2, M10V_PLL10, 2, 1, -1},
+	{M10V_PLL11, NULL, 2, 75, -1},
+};
+
+static const struct m10v_clk_div_fixed_data m10v_div_fixed_data[] = {
+	{"usb2", NULL, 2, 1, -1},
+	{"pcisuppclk", NULL, 20, 1, -1},
+	{M10V_PLL1DIV2, M10V_PLL1, 2, 1, -1},
+	{M10V_PLL2DIV2, M10V_PLL2, 2, 1, -1},
+	{M10V_PLL6DIV2, M10V_PLL6, 2, 1, -1},
+	{M10V_PLL6DIV3, M10V_PLL6, 3, 1, -1},
+	{M10V_PLL7DIV2, M10V_PLL7, 2, 1, -1},
+	{M10V_PLL7DIV5, M10V_PLL7, 5, 1, -1},
+	{"ca7wd", M10V_PLL2DIV2, 12, 1, -1},
+	{"pclkca7wd", M10V_PLL1DIV2, 16, 1, -1},
+	{M10V_SPI_PARENT0, M10V_PLL10DIV2, 2, 1, -1},
+	{M10V_SPI_PARENT1, M10V_PLL10DIV2, 4, 1, -1},
+	{M10V_SPI_PARENT2, M10V_PLL7DIV2, 8, 1, -1},
+	{M10V_UHS1CLK2_PARENT0, M10V_PLL7, 4, 1, -1},
+	{M10V_UHS1CLK2_PARENT1, M10V_PLL7, 8, 1, -1},
+	{M10V_UHS1CLK2_PARENT2, M10V_PLL7, 16, 1, -1},
+	{M10V_UHS1CLK1_PARENT0, M10V_PLL7, 8, 1, -1},
+	{M10V_UHS1CLK1_PARENT1, M10V_PLL7, 16, 1, -1},
+	{M10V_NFCLK_PARENT0, M10V_PLL7DIV2, 8, 1, -1},
+	{M10V_NFCLK_PARENT1, M10V_PLL7DIV2, 10, 1, -1},
+	{M10V_NFCLK_PARENT2, M10V_PLL7DIV2, 13, 1, -1},
+	{M10V_NFCLK_PARENT3, M10V_PLL7DIV2, 16, 1, -1},
+	{M10V_NFCLK_PARENT4, M10V_PLL7DIV2, 40, 1, -1},
+	{M10V_NFCLK_PARENT5, M10V_PLL7DIV5, 10, 1, -1},
+};
+
+static const struct m10v_clk_div_factors m10v_div_factor_data[] = {
+	{"emmc", M10V_PLL11, CLKSEL(1), 28, 3, emmcclk_table, 0,
+		M10V_EMMCCLK_ID},
+	{"mclk400", M10V_PLL1DIV2, CLKSEL(10), 7, 3, mclk400_table, 0, -1},
+	{"mclk200", M10V_PLL1DIV2, CLKSEL(10), 3, 4, mclk200_table, 0, -1},
+	{"aclk400", M10V_PLL1DIV2, CLKSEL(10), 0, 3, aclk400_table, 0, -1},
+	{"aclk300", M10V_PLL2DIV2, CLKSEL(12), 0, 2, aclk300_table, 0, -1},
+	{"aclk", M10V_PLL1DIV2, CLKSEL(9), 20, 4, aclk_table, 0, M10V_ACLK_ID},
+	{"aclkexs", M10V_PLL1DIV2, CLKSEL(9), 16, 4, aclkexs_table, 0, -1},
+	{"hclk", M10V_PLL1DIV2, CLKSEL(9), 7, 5, hclk_table, 0, M10V_HCLK_ID},
+	{"hclkbmh", M10V_PLL1DIV2, CLKSEL(9), 12, 4, hclkbmh_table, 0, -1},
+	{"pclk", M10V_PLL1DIV2, CLKSEL(9), 0, 7, pclk_table, 0, M10V_PCLK_ID},
+	{"uhs1clk0", M10V_PLL7, CLKSEL(1), 3, 5, uhs1clk0_table, 0, -1},
+	{"uhs2clk", M10V_PLL6DIV3, CLKSEL(1), 18, 4, uhs2clk_table, 0, -1},
+};
+
+static const struct m10v_clk_mux_factors m10v_mux_factor_data[] = {
+	{"spi", spi_mux_names, ARRAY_SIZE(spi_mux_names),
+		CLKSEL(8), 3, 7, spi_mux_table, 0, M10V_SPICLK_ID},
+	{"uhs1clk2", uhs1clk2_mux_names, ARRAY_SIZE(uhs1clk2_mux_names),
+		CLKSEL(1), 13, 31, uhs1clk2_mux_table, 0, M10V_UHS1CLK2_ID},
+	{"uhs1clk1", uhs1clk1_mux_names, ARRAY_SIZE(uhs1clk1_mux_names),
+		CLKSEL(1), 8, 31, uhs1clk1_mux_table, 0, -1},
+	{"nfclk", nfclk_mux_names, ARRAY_SIZE(nfclk_mux_names),
+		CLKSEL(1), 22, 127, nfclk_mux_table, 0, M10V_NFCLK_ID},
+};
+
+static u8 m10v_mux_get_parent(struct clk_hw *hw)
+{
+	struct clk_mux *mux = to_clk_mux(hw);
+	u32 val;
+
+	val = readl(mux->reg) >> mux->shift;
+	val &= mux->mask;
+
+	return clk_mux_val_to_index(hw, mux->table, mux->flags, val);
+}
+
+static int m10v_mux_set_parent(struct clk_hw *hw, u8 index)
+{
+	struct clk_mux *mux = to_clk_mux(hw);
+	u32 val = clk_mux_index_to_val(mux->table, mux->flags, index);
+	unsigned long flags = 0;
+	u32 reg;
+	u32 write_en = BIT(fls(mux->mask) - 1);
+
+	if (mux->lock)
+		spin_lock_irqsave(mux->lock, flags);
+	else
+		__acquire(mux->lock);
+
+	reg = readl(mux->reg);
+	reg &= ~(mux->mask << mux->shift);
+
+	val = (val | write_en) << mux->shift;
+	reg |= val;
+	writel(reg, mux->reg);
+
+	if (mux->lock)
+		spin_unlock_irqrestore(mux->lock, flags);
+	else
+		__release(mux->lock);
+
+	return 0;
+}
+
+static const struct clk_ops m10v_mux_ops = {
+	.get_parent = m10v_mux_get_parent,
+	.set_parent = m10v_mux_set_parent,
+	.determine_rate = __clk_mux_determine_rate,
+};
+
+static struct clk_hw *m10v_clk_hw_register_mux(struct device *dev,
+			const char *name, const char * const *parent_names,
+			u8 num_parents, unsigned long flags, void __iomem *reg,
+			u8 shift, u32 mask, u8 clk_mux_flags, u32 *table,
+			spinlock_t *lock)
+{
+	struct clk_mux *mux;
+	struct clk_hw *hw;
+	struct clk_init_data init;
+	int ret;
+
+	mux = kzalloc(sizeof(*mux), GFP_KERNEL);
+	if (!mux)
+		return ERR_PTR(-ENOMEM);
+
+	init.name = name;
+	init.ops = &m10v_mux_ops;
+	init.flags = flags;
+	init.parent_names = parent_names;
+	init.num_parents = num_parents;
+
+	mux->reg = reg;
+	mux->shift = shift;
+	mux->mask = mask;
+	mux->flags = clk_mux_flags;
+	mux->lock = lock;
+	mux->table = table;
+	mux->hw.init = &init;
+
+	hw = &mux->hw;
+	ret = clk_hw_register(dev, hw);
+	if (ret) {
+		kfree(mux);
+		hw = ERR_PTR(ret);
+	}
+
+	return hw;
+
+}
+
+struct m10v_clk_divider {
+	struct clk_hw	hw;
+	void __iomem	*reg;
+	u8		shift;
+	u8		width;
+	u8		flags;
+	const struct clk_div_table	*table;
+	spinlock_t	*lock;
+	void __iomem	*write_valid_reg;
+};
+
+static unsigned long m10v_clk_divider_recalc_rate(struct clk_hw *hw,
+		unsigned long parent_rate)
+{
+	struct m10v_clk_divider *divider = to_m10v_div(hw);
+	unsigned int val;
+
+	val = readl(divider->reg) >> divider->shift;
+	val &= clk_div_mask(divider->width);
+
+	return divider_recalc_rate(hw, parent_rate, val, divider->table,
+				   divider->flags, divider->width);
+}
+
+static long m10v_clk_divider_round_rate(struct clk_hw *hw, unsigned long rate,
+				unsigned long *prate)
+{
+	struct m10v_clk_divider *divider = to_m10v_div(hw);
+
+	/* if read only, just return current value */
+	if (divider->flags & CLK_DIVIDER_READ_ONLY) {
+		u32 val;
+
+		val = readl(divider->reg) >> divider->shift;
+		val &= clk_div_mask(divider->width);
+
+		return divider_ro_round_rate(hw, rate, prate, divider->table,
+					     divider->width, divider->flags,
+					     val);
+	}
+
+	return divider_round_rate(hw, rate, prate, divider->table,
+				  divider->width, divider->flags);
+}
+
+static int m10v_clk_divider_set_rate(struct clk_hw *hw, unsigned long rate,
+				unsigned long parent_rate)
+{
+	struct m10v_clk_divider *divider = to_m10v_div(hw);
+	int value;
+	unsigned long flags = 0;
+	u32 val;
+	u32 write_en = BIT(divider->width - 1);
+
+	value = divider_get_val(rate, parent_rate, divider->table,
+				divider->width, divider->flags);
+	if (value < 0)
+		return value;
+
+	if (divider->lock)
+		spin_lock_irqsave(divider->lock, flags);
+	else
+		__acquire(divider->lock);
+
+	val = readl(divider->reg);
+	val &= ~(clk_div_mask(divider->width) << divider->shift);
+
+	val |= ((u32)value | write_en) << divider->shift;
+	writel(val, divider->reg);
+
+	if (divider->write_valid_reg) {
+		writel(M10V_DCHREQ, divider->write_valid_reg);
+		if (readl_poll_timeout(divider->write_valid_reg, val,
+			!val, M10V_UPOLL_RATE, M10V_UTIMEOUT))
+			pr_err("%s:%s couldn't stabilize\n",
+				__func__, divider->hw.init->name);
+	}
+
+	if (divider->lock)
+		spin_unlock_irqrestore(divider->lock, flags);
+	else
+		__release(divider->lock);
+
+	return 0;
+}
+
+static const struct clk_ops m10v_clk_divider_ops = {
+	.recalc_rate = m10v_clk_divider_recalc_rate,
+	.round_rate = m10v_clk_divider_round_rate,
+	.set_rate = m10v_clk_divider_set_rate,
+};
+
+static struct clk_hw *m10v_clk_hw_register_divider(struct device *dev,
+		const char *name, const char *parent_name, unsigned long flags,
+		void __iomem *reg, u8 shift, u8 width,
+		u8 clk_divider_flags, const struct clk_div_table *table,
+		spinlock_t *lock, void __iomem *write_valid_reg)
+{
+	struct m10v_clk_divider *div;
+	struct clk_hw *hw;
+	struct clk_init_data init;
+	int ret;
+
+	div = kzalloc(sizeof(*div), GFP_KERNEL);
+	if (!div)
+		return ERR_PTR(-ENOMEM);
+
+	init.name = name;
+	init.ops = &m10v_clk_divider_ops;
+	init.flags = flags;
+	init.parent_names = &parent_name;
+	init.num_parents = 1;
+
+	div->reg = reg;
+	div->shift = shift;
+	div->width = width;
+	div->flags = clk_divider_flags;
+	div->lock = lock;
+	div->hw.init = &init;
+	div->table = table;
+	div->write_valid_reg = write_valid_reg;
+
+	/* register the clock */
+	hw = &div->hw;
+	ret = clk_hw_register(dev, hw);
+	if (ret) {
+		kfree(div);
+		hw = ERR_PTR(ret);
+	}
+
+	return hw;
+}
+
+static void m10v_reg_div_pre(const struct m10v_clk_div_factors *factors,
+			     struct clk_hw_onecell_data *clk_data,
+			     void __iomem *base)
+{
+	struct clk_hw *hw;
+	void __iomem *write_valid_reg;
+
+	/*
+	 * The registers on CLKSEL(9) or CLKSEL(10) need additional
+	 * writing to become valid.
+	 */
+	if ((factors->offset == CLKSEL(9)) || (factors->offset == CLKSEL(10)))
+		write_valid_reg = base + CLKSEL(11);
+	else
+		write_valid_reg = NULL;
+
+	hw = m10v_clk_hw_register_divider(NULL, factors->name,
+					  factors->parent_name,
+					  CLK_SET_RATE_PARENT,
+					  base + factors->offset,
+					  factors->shift,
+					  factors->width, factors->div_flags,
+					  factors->table,
+					  &m10v_crglock, write_valid_reg);
+
+	if (factors->onecell_idx >= 0)
+		clk_data->hws[factors->onecell_idx] = hw;
+}
+
+static void m10v_reg_fixed_pre(const struct m10v_clk_div_fixed_data *factors,
+			       struct clk_hw_onecell_data *clk_data,
+			       const char *parent_name)
+{
+	struct clk_hw *hw;
+	const char *pn = factors->parent_name ?
+				factors->parent_name : parent_name;
+
+	hw = clk_hw_register_fixed_factor(NULL, factors->name, pn, 0,
+					  factors->mult, factors->div);
+
+	if (factors->onecell_idx >= 0)
+		clk_data->hws[factors->onecell_idx] = hw;
+}
+
+static void m10v_reg_mux_pre(const struct m10v_clk_mux_factors *factors,
+			       struct clk_hw_onecell_data *clk_data,
+			       void __iomem *base)
+{
+	struct clk_hw *hw;
+
+	hw = m10v_clk_hw_register_mux(NULL, factors->name,
+				      factors->parent_names,
+				      factors->num_parents,
+				      CLK_SET_RATE_PARENT,
+				      base + factors->offset, factors->shift,
+				      factors->mask, factors->mux_flags,
+				      factors->table, &m10v_crglock);
+
+	if (factors->onecell_idx >= 0)
+		clk_data->hws[factors->onecell_idx] = hw;
+}
+
+static int m10v_clk_probe(struct platform_device *pdev)
+{
+	int id;
+	struct resource *res;
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	void __iomem *base;
+	const char *parent_name;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(base))
+		return PTR_ERR(base);
+
+	parent_name = of_clk_get_parent_name(np, 0);
+
+	for (id = 0; id < ARRAY_SIZE(m10v_div_factor_data); ++id)
+		m10v_reg_div_pre(&m10v_div_factor_data[id],
+				 m10v_clk_data, base);
+
+	for (id = 0; id < ARRAY_SIZE(m10v_div_fixed_data); ++id)
+		m10v_reg_fixed_pre(&m10v_div_fixed_data[id],
+				   m10v_clk_data, parent_name);
+
+	for (id = 0; id < ARRAY_SIZE(m10v_mux_factor_data); ++id)
+		m10v_reg_mux_pre(&m10v_mux_factor_data[id],
+				 m10v_clk_data, base);
+
+	for (id = 0; id < M10V_NUM_CLKS; id++) {
+		if (IS_ERR(m10v_clk_data->hws[id]))
+			return PTR_ERR(m10v_clk_data->hws[id]);
+	}
+
+	return 0;
+}
+
+static const struct of_device_id m10v_clk_dt_ids[] = {
+	{ .compatible = "socionext,milbeaut-m10v-ccu", },
+	{ }
+};
+
+static struct platform_driver m10v_clk_driver = {
+	.probe  = m10v_clk_probe,
+	.driver = {
+		.name = "m10v-ccu",
+		.of_match_table = m10v_clk_dt_ids,
+	},
+};
+builtin_platform_driver(m10v_clk_driver);
+
+static void __init m10v_cc_init(struct device_node *np)
+{
+	int id;
+	void __iomem *base;
+	const char *parent_name;
+	struct clk_hw *hw;
+
+	m10v_clk_data = kzalloc(struct_size(m10v_clk_data, hws,
+					M10V_NUM_CLKS),
+					GFP_KERNEL);
+
+	if (!m10v_clk_data)
+		return;
+
+	base = of_iomap(np, 0);
+	if (!base) {
+		kfree(m10v_clk_data);
+		return;
+	}
+
+	parent_name = of_clk_get_parent_name(np, 0);
+	if (!parent_name) {
+		kfree(m10v_clk_data);
+		iounmap(base);
+		return;
+	}
+
+	/*
+	 * This way all clocks fetched before the platform device probes,
+	 * except those we assign here for early use, will be deferred.
+	 */
+	for (id = 0; id < M10V_NUM_CLKS; id++)
+		m10v_clk_data->hws[id] = ERR_PTR(-EPROBE_DEFER);
+
+	/*
+	 * PLLs are set by bootloader so this driver registers them as the
+	 * fixed factor.
+	 */
+	for (id = 0; id < ARRAY_SIZE(m10v_pll_fixed_data); ++id)
+		m10v_reg_fixed_pre(&m10v_pll_fixed_data[id],
+				   m10v_clk_data, parent_name);
+
+	/*
+	 * timer consumes "rclk" so it needs to register here.
+	 */
+	hw = m10v_clk_hw_register_divider(NULL, "rclk", M10V_PLL10DIV2, 0,
+					base + CLKSEL(1), 0, 3, 0, rclk_table,
+					&m10v_crglock, NULL);
+	m10v_clk_data->hws[M10V_RCLK_ID] = hw;
+
+	m10v_clk_data->num = M10V_NUM_CLKS;
+	of_clk_add_hw_provider(np, of_clk_hw_onecell_get, m10v_clk_data);
+}
+CLK_OF_DECLARE_DRIVER(m10v_cc, "socionext,milbeaut-m10v-ccu", m10v_cc_init);
diff --git a/drivers/clk/clk-multiplier.c b/drivers/clk/clk-multiplier.c
index 3c86f859c199..94470b4eadf4 100644
--- a/drivers/clk/clk-multiplier.c
+++ b/drivers/clk/clk-multiplier.c
@@ -11,6 +11,22 @@
 #include <linux/of.h>
 #include <linux/slab.h>
 
+static inline u32 clk_mult_readl(struct clk_multiplier *mult)
+{
+	if (mult->flags & CLK_MULTIPLIER_BIG_ENDIAN)
+		return ioread32be(mult->reg);
+
+	return readl(mult->reg);
+}
+
+static inline void clk_mult_writel(struct clk_multiplier *mult, u32 val)
+{
+	if (mult->flags & CLK_MULTIPLIER_BIG_ENDIAN)
+		iowrite32be(val, mult->reg);
+	else
+		writel(val, mult->reg);
+}
+
 static unsigned long __get_mult(struct clk_multiplier *mult,
 				unsigned long rate,
 				unsigned long parent_rate)
@@ -27,7 +43,7 @@ static unsigned long clk_multiplier_recalc_rate(struct clk_hw *hw,
 	struct clk_multiplier *mult = to_clk_multiplier(hw);
 	unsigned long val;
 
-	val = clk_readl(mult->reg) >> mult->shift;
+	val = clk_mult_readl(mult) >> mult->shift;
 	val &= GENMASK(mult->width - 1, 0);
 
 	if (!val && mult->flags & CLK_MULTIPLIER_ZERO_BYPASS)
@@ -118,10 +134,10 @@ static int clk_multiplier_set_rate(struct clk_hw *hw, unsigned long rate,
 	else
 		__acquire(mult->lock);
 
-	val = clk_readl(mult->reg);
+	val = clk_mult_readl(mult);
 	val &= ~GENMASK(mult->width + mult->shift - 1, mult->shift);
 	val |= factor << mult->shift;
-	clk_writel(val, mult->reg);
+	clk_mult_writel(mult, val);
 
 	if (mult->lock)
 		spin_unlock_irqrestore(mult->lock, flags);
diff --git a/drivers/clk/clk-mux.c b/drivers/clk/clk-mux.c
index 2ad2df2e8909..66e91f740508 100644
--- a/drivers/clk/clk-mux.c
+++ b/drivers/clk/clk-mux.c
@@ -23,6 +23,22 @@
  * parent - parent is adjustable through clk_set_parent
  */
 
+static inline u32 clk_mux_readl(struct clk_mux *mux)
+{
+	if (mux->flags & CLK_MUX_BIG_ENDIAN)
+		return ioread32be(mux->reg);
+
+	return readl(mux->reg);
+}
+
+static inline void clk_mux_writel(struct clk_mux *mux, u32 val)
+{
+	if (mux->flags & CLK_MUX_BIG_ENDIAN)
+		iowrite32be(val, mux->reg);
+	else
+		writel(val, mux->reg);
+}
+
 int clk_mux_val_to_index(struct clk_hw *hw, u32 *table, unsigned int flags,
 			 unsigned int val)
 {
@@ -73,7 +89,7 @@ static u8 clk_mux_get_parent(struct clk_hw *hw)
 	struct clk_mux *mux = to_clk_mux(hw);
 	u32 val;
 
-	val = clk_readl(mux->reg) >> mux->shift;
+	val = clk_mux_readl(mux) >> mux->shift;
 	val &= mux->mask;
 
 	return clk_mux_val_to_index(hw, mux->table, mux->flags, val);
@@ -94,12 +110,12 @@ static int clk_mux_set_parent(struct clk_hw *hw, u8 index)
 	if (mux->flags & CLK_MUX_HIWORD_MASK) {
 		reg = mux->mask << (mux->shift + 16);
 	} else {
-		reg = clk_readl(mux->reg);
+		reg = clk_mux_readl(mux);
 		reg &= ~(mux->mask << mux->shift);
 	}
 	val = val << mux->shift;
 	reg |= val;
-	clk_writel(reg, mux->reg);
+	clk_mux_writel(mux, reg);
 
 	if (mux->lock)
 		spin_unlock_irqrestore(mux->lock, flags);
@@ -159,7 +175,7 @@ struct clk_hw *clk_hw_register_mux_table(struct device *dev, const char *name,
 		init.ops = &clk_mux_ro_ops;
 	else
 		init.ops = &clk_mux_ops;
-	init.flags = flags | CLK_IS_BASIC;
+	init.flags = flags;
 	init.parent_names = parent_names;
 	init.num_parents = num_parents;
 
diff --git a/drivers/clk/clk-pwm.c b/drivers/clk/clk-pwm.c
index 8cb9d117fdbf..02b472a1f9b0 100644
--- a/drivers/clk/clk-pwm.c
+++ b/drivers/clk/clk-pwm.c
@@ -101,7 +101,7 @@ static int clk_pwm_probe(struct platform_device *pdev)
 
 	init.name = clk_name;
 	init.ops = &clk_pwm_ops;
-	init.flags = CLK_IS_BASIC;
+	init.flags = 0;
 	init.num_parents = 0;
 
 	clk_pwm->pwm = pwm;
diff --git a/drivers/clk/clk-qoriq.c b/drivers/clk/clk-qoriq.c
index 1212a9be7e80..4739a47ec8bd 100644
--- a/drivers/clk/clk-qoriq.c
+++ b/drivers/clk/clk-qoriq.c
@@ -34,6 +34,7 @@
 #define CGA_PLL4	4	/* only on clockgen-1.0, which lacks CGB */
 #define CGB_PLL1	4
 #define CGB_PLL2	5
+#define MAX_PLL_DIV	16
 
 struct clockgen_pll_div {
 	struct clk *clk;
@@ -41,7 +42,7 @@ struct clockgen_pll_div {
 };
 
 struct clockgen_pll {
-	struct clockgen_pll_div div[8];
+	struct clockgen_pll_div div[MAX_PLL_DIV];
 };
 
 #define CLKSEL_VALID	1
@@ -79,7 +80,7 @@ struct clockgen_chipinfo {
 	const struct clockgen_muxinfo *cmux_groups[2];
 	const struct clockgen_muxinfo *hwaccel[NUM_HWACCEL];
 	void (*init_periph)(struct clockgen *cg);
-	int cmux_to_group[NUM_CMUX]; /* -1 terminates if fewer than NUM_CMUX */
+	int cmux_to_group[NUM_CMUX + 1]; /* array should be -1 terminated */
 	u32 pll_mask;	/* 1 << n bit set if PLL n is valid */
 	u32 flags;	/* CG_xxx */
 };
@@ -245,6 +246,58 @@ static const struct clockgen_muxinfo clockgen2_cmux_cgb = {
 	},
 };
 
+static const struct clockgen_muxinfo ls1028a_hwa1 = {
+	{
+		{ CLKSEL_VALID, PLATFORM_PLL, PLL_DIV1 },
+		{ CLKSEL_VALID, CGA_PLL1, PLL_DIV1 },
+		{ CLKSEL_VALID, CGA_PLL1, PLL_DIV2 },
+		{ CLKSEL_VALID, CGA_PLL1, PLL_DIV3 },
+		{ CLKSEL_VALID, CGA_PLL1, PLL_DIV4 },
+		{},
+		{ CLKSEL_VALID, CGA_PLL2, PLL_DIV2 },
+		{ CLKSEL_VALID, CGA_PLL2, PLL_DIV3 },
+	},
+};
+
+static const struct clockgen_muxinfo ls1028a_hwa2 = {
+	{
+		{ CLKSEL_VALID, PLATFORM_PLL, PLL_DIV1 },
+		{ CLKSEL_VALID, CGA_PLL2, PLL_DIV1 },
+		{ CLKSEL_VALID, CGA_PLL2, PLL_DIV2 },
+		{ CLKSEL_VALID, CGA_PLL2, PLL_DIV3 },
+		{ CLKSEL_VALID, CGA_PLL2, PLL_DIV4 },
+		{},
+		{ CLKSEL_VALID, CGA_PLL1, PLL_DIV2 },
+		{ CLKSEL_VALID, CGA_PLL1, PLL_DIV3 },
+	},
+};
+
+static const struct clockgen_muxinfo ls1028a_hwa3 = {
+	{
+		{ CLKSEL_VALID, PLATFORM_PLL, PLL_DIV1 },
+		{ CLKSEL_VALID, CGA_PLL1, PLL_DIV1 },
+		{ CLKSEL_VALID, CGA_PLL1, PLL_DIV2 },
+		{ CLKSEL_VALID, CGA_PLL1, PLL_DIV3 },
+		{ CLKSEL_VALID, CGA_PLL1, PLL_DIV4 },
+		{},
+		{ CLKSEL_VALID, CGA_PLL2, PLL_DIV2 },
+		{ CLKSEL_VALID, CGA_PLL2, PLL_DIV3 },
+	},
+};
+
+static const struct clockgen_muxinfo ls1028a_hwa4 = {
+	{
+		{ CLKSEL_VALID, PLATFORM_PLL, PLL_DIV1 },
+		{ CLKSEL_VALID, CGA_PLL2, PLL_DIV1 },
+		{ CLKSEL_VALID, CGA_PLL2, PLL_DIV2 },
+		{ CLKSEL_VALID, CGA_PLL2, PLL_DIV3 },
+		{ CLKSEL_VALID, CGA_PLL2, PLL_DIV4 },
+		{},
+		{ CLKSEL_VALID, CGA_PLL1, PLL_DIV2 },
+		{ CLKSEL_VALID, CGA_PLL1, PLL_DIV3 },
+	},
+};
+
 static const struct clockgen_muxinfo ls1043a_hwa1 = {
 	{
 		{},
@@ -508,6 +561,21 @@ static const struct clockgen_chipinfo chipinfo[] = {
 		.pll_mask = 0x03,
 	},
 	{
+		.compat = "fsl,ls1028a-clockgen",
+		.cmux_groups = {
+			&clockgen2_cmux_cga12
+		},
+		.hwaccel = {
+			&ls1028a_hwa1, &ls1028a_hwa2,
+			&ls1028a_hwa3, &ls1028a_hwa4
+		},
+		.cmux_to_group = {
+			0, 0, 0, 0, -1
+		},
+		.pll_mask = 0x07,
+		.flags = CG_VER3 | CG_LITTLE_ENDIAN,
+	},
+	{
 		.compat = "fsl,ls1043a-clockgen",
 		.init_periph = t2080_init_periph,
 		.cmux_groups = {
@@ -601,7 +669,7 @@ static const struct clockgen_chipinfo chipinfo[] = {
 			&p4080_cmux_grp1, &p4080_cmux_grp2
 		},
 		.cmux_to_group = {
-			0, 0, 0, 0, 1, 1, 1, 1
+			0, 0, 0, 0, 1, 1, 1, 1, -1
 		},
 		.pll_mask = 0x1f,
 	},
@@ -1128,7 +1196,7 @@ static void __init create_one_pll(struct clockgen *cg, int idx)
 		int ret;
 
 		/*
-		 * For platform PLL, there are 8 divider clocks.
+		 * For platform PLL, there are MAX_PLL_DIV divider clocks.
 		 * For core PLL, there are 4 divider clocks at most.
 		 */
 		if (idx != PLATFORM_PLL && i >= 4)
@@ -1423,6 +1491,7 @@ CLK_OF_DECLARE(qoriq_clockgen_b4420, "fsl,b4420-clockgen", clockgen_init);
 CLK_OF_DECLARE(qoriq_clockgen_b4860, "fsl,b4860-clockgen", clockgen_init);
 CLK_OF_DECLARE(qoriq_clockgen_ls1012a, "fsl,ls1012a-clockgen", clockgen_init);
 CLK_OF_DECLARE(qoriq_clockgen_ls1021a, "fsl,ls1021a-clockgen", clockgen_init);
+CLK_OF_DECLARE(qoriq_clockgen_ls1028a, "fsl,ls1028a-clockgen", clockgen_init);
 CLK_OF_DECLARE(qoriq_clockgen_ls1043a, "fsl,ls1043a-clockgen", clockgen_init);
 CLK_OF_DECLARE(qoriq_clockgen_ls1046a, "fsl,ls1046a-clockgen", clockgen_init);
 CLK_OF_DECLARE(qoriq_clockgen_ls1088a, "fsl,ls1088a-clockgen", clockgen_init);
diff --git a/drivers/clk/clk-stm32f4.c b/drivers/clk/clk-stm32f4.c
index cdaa567c8042..fdac33a9be2f 100644
--- a/drivers/clk/clk-stm32f4.c
+++ b/drivers/clk/clk-stm32f4.c
@@ -300,6 +300,85 @@ static const struct stm32f4_gate_data stm32f746_gates[] __initconst = {
 	{ STM32F4_RCC_APB2ENR, 26,	"ltdc",		"apb2_div" },
 };
 
+static const struct stm32f4_gate_data stm32f769_gates[] __initconst = {
+	{ STM32F4_RCC_AHB1ENR,  0,	"gpioa",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR,  1,	"gpiob",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR,  2,	"gpioc",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR,  3,	"gpiod",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR,  4,	"gpioe",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR,  5,	"gpiof",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR,  6,	"gpiog",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR,  7,	"gpioh",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR,  8,	"gpioi",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR,  9,	"gpioj",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR, 10,	"gpiok",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR, 12,	"crc",		"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR, 18,	"bkpsra",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR, 20,	"dtcmram",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR, 21,	"dma1",		"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR, 22,	"dma2",		"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR, 23,	"dma2d",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR, 25,	"ethmac",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR, 26,	"ethmactx",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR, 27,	"ethmacrx",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR, 28,	"ethmacptp",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR, 29,	"otghs",	"ahb_div" },
+	{ STM32F4_RCC_AHB1ENR, 30,	"otghsulpi",	"ahb_div" },
+
+	{ STM32F4_RCC_AHB2ENR,  0,	"dcmi",		"ahb_div" },
+	{ STM32F4_RCC_AHB2ENR,  1,	"jpeg",		"ahb_div" },
+	{ STM32F4_RCC_AHB2ENR,  4,	"cryp",		"ahb_div" },
+	{ STM32F4_RCC_AHB2ENR,  5,	"hash",		"ahb_div" },
+	{ STM32F4_RCC_AHB2ENR,  6,	"rng",		"pll48"   },
+	{ STM32F4_RCC_AHB2ENR,  7,	"otgfs",	"pll48"   },
+
+	{ STM32F4_RCC_AHB3ENR,  0,	"fmc",		"ahb_div",
+		CLK_IGNORE_UNUSED },
+	{ STM32F4_RCC_AHB3ENR,  1,	"qspi",		"ahb_div",
+		CLK_IGNORE_UNUSED },
+
+	{ STM32F4_RCC_APB1ENR,  0,	"tim2",		"apb1_mul" },
+	{ STM32F4_RCC_APB1ENR,  1,	"tim3",		"apb1_mul" },
+	{ STM32F4_RCC_APB1ENR,  2,	"tim4",		"apb1_mul" },
+	{ STM32F4_RCC_APB1ENR,  3,	"tim5",		"apb1_mul" },
+	{ STM32F4_RCC_APB1ENR,  4,	"tim6",		"apb1_mul" },
+	{ STM32F4_RCC_APB1ENR,  5,	"tim7",		"apb1_mul" },
+	{ STM32F4_RCC_APB1ENR,  6,	"tim12",	"apb1_mul" },
+	{ STM32F4_RCC_APB1ENR,  7,	"tim13",	"apb1_mul" },
+	{ STM32F4_RCC_APB1ENR,  8,	"tim14",	"apb1_mul" },
+	{ STM32F4_RCC_APB1ENR, 10,	"rtcapb",	"apb1_mul" },
+	{ STM32F4_RCC_APB1ENR, 11,	"wwdg",		"apb1_div" },
+	{ STM32F4_RCC_APB1ENR, 13,	"can3",		"apb1_div" },
+	{ STM32F4_RCC_APB1ENR, 14,	"spi2",		"apb1_div" },
+	{ STM32F4_RCC_APB1ENR, 15,	"spi3",		"apb1_div" },
+	{ STM32F4_RCC_APB1ENR, 16,	"spdifrx",	"apb1_div" },
+	{ STM32F4_RCC_APB1ENR, 25,	"can1",		"apb1_div" },
+	{ STM32F4_RCC_APB1ENR, 26,	"can2",		"apb1_div" },
+	{ STM32F4_RCC_APB1ENR, 27,	"cec",		"apb1_div" },
+	{ STM32F4_RCC_APB1ENR, 28,	"pwr",		"apb1_div" },
+	{ STM32F4_RCC_APB1ENR, 29,	"dac",		"apb1_div" },
+
+	{ STM32F4_RCC_APB2ENR,  0,	"tim1",		"apb2_mul" },
+	{ STM32F4_RCC_APB2ENR,  1,	"tim8",		"apb2_mul" },
+	{ STM32F4_RCC_APB2ENR,  7,	"sdmmc2",	"sdmux2" },
+	{ STM32F4_RCC_APB2ENR,  8,	"adc1",		"apb2_div" },
+	{ STM32F4_RCC_APB2ENR,  9,	"adc2",		"apb2_div" },
+	{ STM32F4_RCC_APB2ENR, 10,	"adc3",		"apb2_div" },
+	{ STM32F4_RCC_APB2ENR, 11,	"sdmmc1",	"sdmux1" },
+	{ STM32F4_RCC_APB2ENR, 12,	"spi1",		"apb2_div" },
+	{ STM32F4_RCC_APB2ENR, 13,	"spi4",		"apb2_div" },
+	{ STM32F4_RCC_APB2ENR, 14,	"syscfg",	"apb2_div" },
+	{ STM32F4_RCC_APB2ENR, 16,	"tim9",		"apb2_mul" },
+	{ STM32F4_RCC_APB2ENR, 17,	"tim10",	"apb2_mul" },
+	{ STM32F4_RCC_APB2ENR, 18,	"tim11",	"apb2_mul" },
+	{ STM32F4_RCC_APB2ENR, 20,	"spi5",		"apb2_div" },
+	{ STM32F4_RCC_APB2ENR, 21,	"spi6",		"apb2_div" },
+	{ STM32F4_RCC_APB2ENR, 22,	"sai1",		"apb2_div" },
+	{ STM32F4_RCC_APB2ENR, 23,	"sai2",		"apb2_div" },
+	{ STM32F4_RCC_APB2ENR, 26,	"ltdc",		"apb2_div" },
+	{ STM32F4_RCC_APB2ENR, 30,	"mdio",		"apb2_div" },
+};
+
 /*
  * This bitmask tells us which bit offsets (0..192) on STM32F4[23]xxx
  * have gate bits associated with them. Its combined hweight is 71.
@@ -318,6 +397,10 @@ static const u64 stm32f746_gate_map[MAX_GATE_MAP] = { 0x000000f17ef417ffull,
 						      0x0000000000000003ull,
 						      0x04f77f833e01c9ffull };
 
+static const u64 stm32f769_gate_map[MAX_GATE_MAP] = { 0x000000f37ef417ffull,
+						      0x0000000000000003ull,
+						      0x44F77F833E01EDFFull };
+
 static const u64 *stm32f4_gate_map;
 
 static struct clk_hw **clks;
@@ -1048,6 +1131,10 @@ static const char *rtc_parents[4] = {
 	"no-clock", "lse", "lsi", "hse-rtc"
 };
 
+static const char *pll_src = "pll-src";
+
+static const char *pllsrc_parent[2] = { "hsi", NULL };
+
 static const char *dsi_parent[2] = { NULL, "pll-r" };
 
 static const char *lcd_parent[1] = { "pllsai-r-div" };
@@ -1072,6 +1159,9 @@ static const char *uart_parents2[4] = { "apb1_div", "sys", "hsi", "lse" };
 
 static const char *i2c_parents[4] = { "apb1_div", "sys", "hsi", "no-clock" };
 
+static const char * const dfsdm1_src[] = { "apb2_div", "sys" };
+static const char * const adsfdm1_parent[] = { "sai1_clk", "sai2_clk" };
+
 struct stm32_aux_clk {
 	int idx;
 	const char *name;
@@ -1313,6 +1403,177 @@ static const struct stm32_aux_clk stm32f746_aux_clk[] = {
 	},
 };
 
+static const struct stm32_aux_clk stm32f769_aux_clk[] = {
+	{
+		CLK_LCD, "lcd-tft", lcd_parent, ARRAY_SIZE(lcd_parent),
+		NO_MUX, 0, 0,
+		STM32F4_RCC_APB2ENR, 26,
+		CLK_SET_RATE_PARENT
+	},
+	{
+		CLK_I2S, "i2s", i2s_parents, ARRAY_SIZE(i2s_parents),
+		STM32F4_RCC_CFGR, 23, 1,
+		NO_GATE, 0,
+		CLK_SET_RATE_PARENT
+	},
+	{
+		CLK_SAI1, "sai1_clk", sai_parents, ARRAY_SIZE(sai_parents),
+		STM32F4_RCC_DCKCFGR, 20, 3,
+		STM32F4_RCC_APB2ENR, 22,
+		CLK_SET_RATE_PARENT
+	},
+	{
+		CLK_SAI2, "sai2_clk", sai_parents, ARRAY_SIZE(sai_parents),
+		STM32F4_RCC_DCKCFGR, 22, 3,
+		STM32F4_RCC_APB2ENR, 23,
+		CLK_SET_RATE_PARENT
+	},
+	{
+		NO_IDX, "pll48", pll48_parents, ARRAY_SIZE(pll48_parents),
+		STM32F7_RCC_DCKCFGR2, 27, 1,
+		NO_GATE, 0,
+		0
+	},
+	{
+		NO_IDX, "sdmux1", sdmux_parents, ARRAY_SIZE(sdmux_parents),
+		STM32F7_RCC_DCKCFGR2, 28, 1,
+		NO_GATE, 0,
+		0
+	},
+	{
+		NO_IDX, "sdmux2", sdmux_parents, ARRAY_SIZE(sdmux_parents),
+		STM32F7_RCC_DCKCFGR2, 29, 1,
+		NO_GATE, 0,
+		0
+	},
+	{
+		CLK_HDMI_CEC, "hdmi-cec",
+		hdmi_parents, ARRAY_SIZE(hdmi_parents),
+		STM32F7_RCC_DCKCFGR2, 26, 1,
+		NO_GATE, 0,
+		0
+	},
+	{
+		CLK_SPDIF, "spdif-rx",
+		spdif_parent, ARRAY_SIZE(spdif_parent),
+		STM32F7_RCC_DCKCFGR2, 22, 3,
+		STM32F4_RCC_APB2ENR, 23,
+		CLK_SET_RATE_PARENT
+	},
+	{
+		CLK_USART1, "usart1",
+		uart_parents1, ARRAY_SIZE(uart_parents1),
+		STM32F7_RCC_DCKCFGR2, 0, 3,
+		STM32F4_RCC_APB2ENR, 4,
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		CLK_USART2, "usart2",
+		uart_parents2, ARRAY_SIZE(uart_parents1),
+		STM32F7_RCC_DCKCFGR2, 2, 3,
+		STM32F4_RCC_APB1ENR, 17,
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		CLK_USART3, "usart3",
+		uart_parents2, ARRAY_SIZE(uart_parents1),
+		STM32F7_RCC_DCKCFGR2, 4, 3,
+		STM32F4_RCC_APB1ENR, 18,
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		CLK_UART4, "uart4",
+		uart_parents2, ARRAY_SIZE(uart_parents1),
+		STM32F7_RCC_DCKCFGR2, 6, 3,
+		STM32F4_RCC_APB1ENR, 19,
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		CLK_UART5, "uart5",
+		uart_parents2, ARRAY_SIZE(uart_parents1),
+		STM32F7_RCC_DCKCFGR2, 8, 3,
+		STM32F4_RCC_APB1ENR, 20,
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		CLK_USART6, "usart6",
+		uart_parents1, ARRAY_SIZE(uart_parents1),
+		STM32F7_RCC_DCKCFGR2, 10, 3,
+		STM32F4_RCC_APB2ENR, 5,
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		CLK_UART7, "uart7",
+		uart_parents2, ARRAY_SIZE(uart_parents1),
+		STM32F7_RCC_DCKCFGR2, 12, 3,
+		STM32F4_RCC_APB1ENR, 30,
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		CLK_UART8, "uart8",
+		uart_parents2, ARRAY_SIZE(uart_parents1),
+		STM32F7_RCC_DCKCFGR2, 14, 3,
+		STM32F4_RCC_APB1ENR, 31,
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		CLK_I2C1, "i2c1",
+		i2c_parents, ARRAY_SIZE(i2c_parents),
+		STM32F7_RCC_DCKCFGR2, 16, 3,
+		STM32F4_RCC_APB1ENR, 21,
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		CLK_I2C2, "i2c2",
+		i2c_parents, ARRAY_SIZE(i2c_parents),
+		STM32F7_RCC_DCKCFGR2, 18, 3,
+		STM32F4_RCC_APB1ENR, 22,
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		CLK_I2C3, "i2c3",
+		i2c_parents, ARRAY_SIZE(i2c_parents),
+		STM32F7_RCC_DCKCFGR2, 20, 3,
+		STM32F4_RCC_APB1ENR, 23,
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		CLK_I2C4, "i2c4",
+		i2c_parents, ARRAY_SIZE(i2c_parents),
+		STM32F7_RCC_DCKCFGR2, 22, 3,
+		STM32F4_RCC_APB1ENR, 24,
+		CLK_SET_RATE_PARENT,
+	},
+	{
+		CLK_LPTIMER, "lptim1",
+		lptim_parent, ARRAY_SIZE(lptim_parent),
+		STM32F7_RCC_DCKCFGR2, 24, 3,
+		STM32F4_RCC_APB1ENR, 9,
+		CLK_SET_RATE_PARENT
+	},
+	{
+		CLK_F769_DSI, "dsi",
+		dsi_parent, ARRAY_SIZE(dsi_parent),
+		STM32F7_RCC_DCKCFGR2, 0, 1,
+		STM32F4_RCC_APB2ENR, 27,
+		CLK_SET_RATE_PARENT
+	},
+	{
+		CLK_DFSDM1, "dfsdm1",
+		dfsdm1_src, ARRAY_SIZE(dfsdm1_src),
+		STM32F4_RCC_DCKCFGR, 25, 1,
+		STM32F4_RCC_APB2ENR, 29,
+		CLK_SET_RATE_PARENT
+	},
+	{
+		CLK_ADFSDM1, "adfsdm1",
+		adsfdm1_parent, ARRAY_SIZE(adsfdm1_parent),
+		STM32F4_RCC_DCKCFGR, 26, 1,
+		STM32F4_RCC_APB2ENR, 29,
+		CLK_SET_RATE_PARENT
+	},
+};
+
 static const struct stm32f4_clk_data stm32f429_clk_data = {
 	.end_primary	= END_PRIMARY_CLK,
 	.gates_data	= stm32f429_gates,
@@ -1343,6 +1604,16 @@ static const struct stm32f4_clk_data stm32f746_clk_data = {
 	.aux_clk_num	= ARRAY_SIZE(stm32f746_aux_clk),
 };
 
+static const struct stm32f4_clk_data stm32f769_clk_data = {
+	.end_primary	= END_PRIMARY_CLK_F7,
+	.gates_data	= stm32f769_gates,
+	.gates_map	= stm32f769_gate_map,
+	.gates_num	= ARRAY_SIZE(stm32f769_gates),
+	.pll_data	= stm32f469_pll,
+	.aux_clk	= stm32f769_aux_clk,
+	.aux_clk_num	= ARRAY_SIZE(stm32f769_aux_clk),
+};
+
 static const struct of_device_id stm32f4_of_match[] = {
 	{
 		.compatible = "st,stm32f42xx-rcc",
@@ -1356,6 +1627,10 @@ static const struct of_device_id stm32f4_of_match[] = {
 		.compatible = "st,stm32f746-rcc",
 		.data = &stm32f746_clk_data
 	},
+	{
+		.compatible = "st,stm32f769-rcc",
+		.data = &stm32f769_clk_data
+	},
 	{}
 };
 
@@ -1427,9 +1702,8 @@ static void __init stm32f4_rcc_init(struct device_node *np)
 	int n;
 	const struct of_device_id *match;
 	const struct stm32f4_clk_data *data;
-	unsigned long pllcfgr;
-	const char *pllsrc;
 	unsigned long pllm;
+	struct clk_hw *pll_src_hw;
 
 	base = of_iomap(np, 0);
 	if (!base) {
@@ -1460,21 +1734,33 @@ static void __init stm32f4_rcc_init(struct device_node *np)
 
 	hse_clk = of_clk_get_parent_name(np, 0);
 	dsi_parent[0] = hse_clk;
+	pllsrc_parent[1] = hse_clk;
 
 	i2s_in_clk = of_clk_get_parent_name(np, 1);
 
 	i2s_parents[1] = i2s_in_clk;
 	sai_parents[2] = i2s_in_clk;
 
+	if (of_device_is_compatible(np, "st,stm32f769-rcc")) {
+		clk_hw_register_gate(NULL, "dfsdm1_apb", "apb2_div", 0,
+				     base + STM32F4_RCC_APB2ENR, 29,
+				     CLK_IGNORE_UNUSED, &stm32f4_clk_lock);
+		dsi_parent[0] = pll_src;
+		sai_parents[3] = pll_src;
+	}
+
 	clks[CLK_HSI] = clk_hw_register_fixed_rate_with_accuracy(NULL, "hsi",
 			NULL, 0, 16000000, 160000);
 
-	pllcfgr = readl(base + STM32F4_RCC_PLLCFGR);
-	pllsrc = pllcfgr & BIT(22) ? hse_clk : "hsi";
-	pllm = pllcfgr & 0x3f;
+	pll_src_hw = clk_hw_register_mux(NULL, pll_src, pllsrc_parent,
+					 ARRAY_SIZE(pllsrc_parent), 0,
+					 base + STM32F4_RCC_PLLCFGR, 22, 1, 0,
+					 &stm32f4_clk_lock);
+
+	pllm = readl(base + STM32F4_RCC_PLLCFGR) & 0x3f;
 
-	clk_hw_register_fixed_factor(NULL, "vco_in", pllsrc,
-					       0, 1, pllm);
+	clk_hw_register_fixed_factor(NULL, "vco_in", pll_src,
+				     0, 1, pllm);
 
 	stm32f4_rcc_register_pll("vco_in", &data->pll_data[0],
 			&stm32f4_clk_lock);
@@ -1612,12 +1898,16 @@ static void __init stm32f4_rcc_init(struct device_node *np)
 			clks[aux_clk->idx] = hw;
 	}
 
-	if (of_device_is_compatible(np, "st,stm32f746-rcc"))
+	if (of_device_is_compatible(np, "st,stm32f746-rcc")) {
 
 		clk_hw_register_fixed_factor(NULL, "hsi_div488", "hsi", 0,
 				1, 488);
 
+		clks[CLK_PLL_SRC] = pll_src_hw;
+	}
+
 	of_clk_add_hw_provider(np, stm32f4_rcc_lookup_clk, NULL);
+
 	return;
 fail:
 	kfree(clks);
@@ -1626,3 +1916,4 @@ fail:
 CLK_OF_DECLARE_DRIVER(stm32f42xx_rcc, "st,stm32f42xx-rcc", stm32f4_rcc_init);
 CLK_OF_DECLARE_DRIVER(stm32f46xx_rcc, "st,stm32f469-rcc", stm32f4_rcc_init);
 CLK_OF_DECLARE_DRIVER(stm32f746_rcc, "st,stm32f746-rcc", stm32f4_rcc_init);
+CLK_OF_DECLARE_DRIVER(stm32f769_rcc, "st,stm32f769-rcc", stm32f4_rcc_init);
diff --git a/drivers/clk/clk-stm32mp1.c b/drivers/clk/clk-stm32mp1.c
index a0ae8dc16909..a875649df8b8 100644
--- a/drivers/clk/clk-stm32mp1.c
+++ b/drivers/clk/clk-stm32mp1.c
@@ -1402,6 +1402,7 @@ enum {
 	G_CRYP1,
 	G_HASH1,
 	G_BKPSRAM,
+	G_DDRPERFM,
 
 	G_LAST
 };
@@ -1488,6 +1489,7 @@ static struct stm32_gate_cfg per_gate_cfg[G_LAST] = {
 	K_GATE(G_STGENRO,	RCC_APB4ENSETR, 20, 0),
 	K_MGATE(G_USBPHY,	RCC_APB4ENSETR, 16, 0),
 	K_GATE(G_IWDG2,		RCC_APB4ENSETR, 15, 0),
+	K_GATE(G_DDRPERFM,	RCC_APB4ENSETR, 8, 0),
 	K_MGATE(G_DSI,		RCC_APB4ENSETR, 4, 0),
 	K_MGATE(G_LTDC,		RCC_APB4ENSETR, 0, 0),
 
@@ -1899,6 +1901,7 @@ static const struct clock_config stm32mp1_clock_cfg[] = {
 	PCLK(CRC1, "crc1", "ck_axi", 0, G_CRC1),
 	PCLK(USBH, "usbh", "ck_axi", 0, G_USBH),
 	PCLK(ETHSTP, "ethstp", "ck_axi", 0, G_ETHSTP),
+	PCLK(DDRPERFM, "ddrperfm", "pclk4", 0, G_DDRPERFM),
 
 	/* Kernel clocks */
 	KCLK(SDMMC1_K, "sdmmc1_k", sdmmc12_src, 0, G_SDMMC1, M_SDMMC12),
diff --git a/drivers/clk/clk-xgene.c b/drivers/clk/clk-xgene.c
index 531b030d4d4e..d975465fe2a8 100644
--- a/drivers/clk/clk-xgene.c
+++ b/drivers/clk/clk-xgene.c
@@ -262,7 +262,7 @@ static unsigned long xgene_clk_pmd_recalc_rate(struct clk_hw *hw,
 	else
 		__acquire(fd->lock);
 
-	val = clk_readl(fd->reg);
+	val = readl(fd->reg);
 
 	if (fd->lock)
 		spin_unlock_irqrestore(fd->lock, flags);
@@ -333,10 +333,10 @@ static int xgene_clk_pmd_set_rate(struct clk_hw *hw, unsigned long rate,
 	else
 		__acquire(fd->lock);
 
-	val = clk_readl(fd->reg);
+	val = readl(fd->reg);
 	val &= ~fd->mask;
 	val |= (scale << fd->shift);
-	clk_writel(val, fd->reg);
+	writel(val, fd->reg);
 
 	if (fd->lock)
 		spin_unlock_irqrestore(fd->lock, flags);
diff --git a/drivers/clk/clk.c b/drivers/clk/clk.c
index 96053a96fe2f..aa51756fd4d6 100644
--- a/drivers/clk/clk.c
+++ b/drivers/clk/clk.c
@@ -39,15 +39,23 @@ static LIST_HEAD(clk_notifier_list);
 
 /***    private data structures    ***/
 
+struct clk_parent_map {
+	const struct clk_hw	*hw;
+	struct clk_core		*core;
+	const char		*fw_name;
+	const char		*name;
+	int			index;
+};
+
 struct clk_core {
 	const char		*name;
 	const struct clk_ops	*ops;
 	struct clk_hw		*hw;
 	struct module		*owner;
 	struct device		*dev;
+	struct device_node	*of_node;
 	struct clk_core		*parent;
-	const char		**parent_names;
-	struct clk_core		**parents;
+	struct clk_parent_map	*parents;
 	u8			num_parents;
 	u8			new_parent_index;
 	unsigned long		rate;
@@ -316,17 +324,102 @@ static struct clk_core *clk_core_lookup(const char *name)
 	return NULL;
 }
 
+/**
+ * clk_core_get - Find the clk_core parent of a clk
+ * @core: clk to find parent of
+ * @p_index: parent index to search for
+ *
+ * This is the preferred method for clk providers to find the parent of a
+ * clk when that parent is external to the clk controller. The parent_names
+ * array is indexed and treated as a local name matching a string in the device
+ * node's 'clock-names' property or as the 'con_id' matching the device's
+ * dev_name() in a clk_lookup. This allows clk providers to use their own
+ * namespace instead of looking for a globally unique parent string.
+ *
+ * For example the following DT snippet would allow a clock registered by the
+ * clock-controller@c001 that has a clk_init_data::parent_data array
+ * with 'xtal' in the 'name' member to find the clock provided by the
+ * clock-controller@f00abcd without needing to get the globally unique name of
+ * the xtal clk.
+ *
+ *      parent: clock-controller@f00abcd {
+ *              reg = <0xf00abcd 0xabcd>;
+ *              #clock-cells = <0>;
+ *      };
+ *
+ *      clock-controller@c001 {
+ *              reg = <0xc001 0xf00d>;
+ *              clocks = <&parent>;
+ *              clock-names = "xtal";
+ *              #clock-cells = <1>;
+ *      };
+ *
+ * Returns: -ENOENT when the provider can't be found or the clk doesn't
+ * exist in the provider. -EINVAL when the name can't be found. NULL when the
+ * provider knows about the clk but it isn't provided on this system.
+ * A valid clk_core pointer when the clk can be found in the provider.
+ */
+static struct clk_core *clk_core_get(struct clk_core *core, u8 p_index)
+{
+	const char *name = core->parents[p_index].fw_name;
+	int index = core->parents[p_index].index;
+	struct clk_hw *hw = ERR_PTR(-ENOENT);
+	struct device *dev = core->dev;
+	const char *dev_id = dev ? dev_name(dev) : NULL;
+	struct device_node *np = core->of_node;
+
+	if (np && index >= 0)
+		hw = of_clk_get_hw(np, index, name);
+
+	/*
+	 * If the DT search above couldn't find the provider or the provider
+	 * didn't know about this clk, fallback to looking up via clkdev based
+	 * clk_lookups
+	 */
+	if (PTR_ERR(hw) == -ENOENT && name)
+		hw = clk_find_hw(dev_id, name);
+
+	if (IS_ERR(hw))
+		return ERR_CAST(hw);
+
+	return hw->core;
+}
+
+static void clk_core_fill_parent_index(struct clk_core *core, u8 index)
+{
+	struct clk_parent_map *entry = &core->parents[index];
+	struct clk_core *parent = ERR_PTR(-ENOENT);
+
+	if (entry->hw) {
+		parent = entry->hw->core;
+		/*
+		 * We have a direct reference but it isn't registered yet?
+		 * Orphan it and let clk_reparent() update the orphan status
+		 * when the parent is registered.
+		 */
+		if (!parent)
+			parent = ERR_PTR(-EPROBE_DEFER);
+	} else {
+		parent = clk_core_get(core, index);
+		if (IS_ERR(parent) && PTR_ERR(parent) == -ENOENT)
+			parent = clk_core_lookup(entry->name);
+	}
+
+	/* Only cache it if it's not an error */
+	if (!IS_ERR(parent))
+		entry->core = parent;
+}
+
 static struct clk_core *clk_core_get_parent_by_index(struct clk_core *core,
 							 u8 index)
 {
-	if (!core || index >= core->num_parents)
+	if (!core || index >= core->num_parents || !core->parents)
 		return NULL;
 
-	if (!core->parents[index])
-		core->parents[index] =
-				clk_core_lookup(core->parent_names[index]);
+	if (!core->parents[index].core)
+		clk_core_fill_parent_index(core, index);
 
-	return core->parents[index];
+	return core->parents[index].core;
 }
 
 struct clk_hw *
@@ -347,23 +440,18 @@ unsigned int __clk_get_enable_count(struct clk *clk)
 
 static unsigned long clk_core_get_rate_nolock(struct clk_core *core)
 {
-	unsigned long ret;
-
-	if (!core) {
-		ret = 0;
-		goto out;
-	}
-
-	ret = core->rate;
-
-	if (!core->num_parents)
-		goto out;
+	if (!core)
+		return 0;
 
-	if (!core->parent)
-		ret = 0;
+	if (!core->num_parents || core->parent)
+		return core->rate;
 
-out:
-	return ret;
+	/*
+	 * Clk must have a parent because num_parents > 0 but the parent isn't
+	 * known yet. Best to return 0 as the rate of this clk until we can
+	 * properly recalc the rate based on the parent's rate.
+	 */
+	return 0;
 }
 
 unsigned long clk_hw_get_rate(const struct clk_hw *hw)
@@ -524,9 +612,15 @@ void clk_hw_set_rate_range(struct clk_hw *hw, unsigned long min_rate,
 EXPORT_SYMBOL_GPL(clk_hw_set_rate_range);
 
 /*
+ * __clk_mux_determine_rate - clk_ops::determine_rate implementation for a mux type clk
+ * @hw: mux type clk to determine rate on
+ * @req: rate request, also used to return preferred parent and frequencies
+ *
  * Helper for finding best parent to provide a given frequency. This can be used
  * directly as a determine_rate callback (e.g. for a mux), or from a more
  * complex clock that may combine a mux with other operations.
+ *
+ * Returns: 0 on success, -EERROR value on error
  */
 int __clk_mux_determine_rate(struct clk_hw *hw,
 			     struct clk_rate_request *req)
@@ -1519,20 +1613,37 @@ static int clk_fetch_parent_index(struct clk_core *core,
 		return -EINVAL;
 
 	for (i = 0; i < core->num_parents; i++) {
-		if (core->parents[i] == parent)
+		/* Found it first try! */
+		if (core->parents[i].core == parent)
 			return i;
 
-		if (core->parents[i])
+		/* Something else is here, so keep looking */
+		if (core->parents[i].core)
 			continue;
 
-		/* Fallback to comparing globally unique names */
-		if (!strcmp(parent->name, core->parent_names[i])) {
-			core->parents[i] = parent;
-			return i;
+		/* Maybe core hasn't been cached but the hw is all we know? */
+		if (core->parents[i].hw) {
+			if (core->parents[i].hw == parent->hw)
+				break;
+
+			/* Didn't match, but we're expecting a clk_hw */
+			continue;
 		}
+
+		/* Maybe it hasn't been cached (clk_set_parent() path) */
+		if (parent == clk_core_get(core, i))
+			break;
+
+		/* Fallback to comparing globally unique names */
+		if (!strcmp(parent->name, core->parents[i].name))
+			break;
 	}
 
-	return -EINVAL;
+	if (i == core->num_parents)
+		return -EINVAL;
+
+	core->parents[i].core = parent;
+	return i;
 }
 
 /*
@@ -2293,6 +2404,7 @@ void clk_hw_reparent(struct clk_hw *hw, struct clk_hw *new_parent)
 bool clk_has_parent(struct clk *clk, struct clk *parent)
 {
 	struct clk_core *core, *parent_core;
+	int i;
 
 	/* NULL clocks should be nops, so return success if either is NULL. */
 	if (!clk || !parent)
@@ -2305,8 +2417,11 @@ bool clk_has_parent(struct clk *clk, struct clk *parent)
 	if (core->parent == parent_core)
 		return true;
 
-	return match_string(core->parent_names, core->num_parents,
-			    parent_core->name) >= 0;
+	for (i = 0; i < core->num_parents; i++)
+		if (!strcmp(core->parents[i].name, parent_core->name))
+			return true;
+
+	return false;
 }
 EXPORT_SYMBOL_GPL(clk_has_parent);
 
@@ -2850,7 +2965,6 @@ static const struct {
 	ENTRY(CLK_SET_PARENT_GATE),
 	ENTRY(CLK_SET_RATE_PARENT),
 	ENTRY(CLK_IGNORE_UNUSED),
-	ENTRY(CLK_IS_BASIC),
 	ENTRY(CLK_GET_RATE_NOCACHE),
 	ENTRY(CLK_SET_RATE_NO_REPARENT),
 	ENTRY(CLK_GET_ACCURACY_NOCACHE),
@@ -2889,9 +3003,9 @@ static int possible_parents_show(struct seq_file *s, void *data)
 	int i;
 
 	for (i = 0; i < core->num_parents - 1; i++)
-		seq_printf(s, "%s ", core->parent_names[i]);
+		seq_printf(s, "%s ", core->parents[i].name);
 
-	seq_printf(s, "%s\n", core->parent_names[i]);
+	seq_printf(s, "%s\n", core->parents[i].name);
 
 	return 0;
 }
@@ -3025,7 +3139,7 @@ static inline void clk_debug_unregister(struct clk_core *core)
  */
 static int __clk_core_init(struct clk_core *core)
 {
-	int i, ret;
+	int ret;
 	struct clk_core *orphan;
 	struct hlist_node *tmp2;
 	unsigned long rate;
@@ -3079,12 +3193,6 @@ static int __clk_core_init(struct clk_core *core)
 		goto out;
 	}
 
-	/* throw a WARN if any entries in parent_names are NULL */
-	for (i = 0; i < core->num_parents; i++)
-		WARN(!core->parent_names[i],
-				"%s: invalid NULL in %s's .parent_names\n",
-				__func__, core->name);
-
 	core->parent = __clk_init_parent(core);
 
 	/*
@@ -3313,20 +3421,104 @@ struct clk *clk_hw_create_clk(struct device *dev, struct clk_hw *hw,
 	return clk;
 }
 
-/**
- * clk_register - allocate a new clock, register it and return an opaque cookie
- * @dev: device that is registering this clock
- * @hw: link to hardware-specific clock data
- *
- * clk_register is the primary interface for populating the clock tree with new
- * clock nodes.  It returns a pointer to the newly allocated struct clk which
- * cannot be dereferenced by driver code but may be used in conjunction with the
- * rest of the clock API.  In the event of an error clk_register will return an
- * error code; drivers must test for an error code after calling clk_register.
- */
-struct clk *clk_register(struct device *dev, struct clk_hw *hw)
+static int clk_cpy_name(const char **dst_p, const char *src, bool must_exist)
+{
+	const char *dst;
+
+	if (!src) {
+		if (must_exist)
+			return -EINVAL;
+		return 0;
+	}
+
+	*dst_p = dst = kstrdup_const(src, GFP_KERNEL);
+	if (!dst)
+		return -ENOMEM;
+
+	return 0;
+}
+
+static int clk_core_populate_parent_map(struct clk_core *core)
+{
+	const struct clk_init_data *init = core->hw->init;
+	u8 num_parents = init->num_parents;
+	const char * const *parent_names = init->parent_names;
+	const struct clk_hw **parent_hws = init->parent_hws;
+	const struct clk_parent_data *parent_data = init->parent_data;
+	int i, ret = 0;
+	struct clk_parent_map *parents, *parent;
+
+	if (!num_parents)
+		return 0;
+
+	/*
+	 * Avoid unnecessary string look-ups of clk_core's possible parents by
+	 * having a cache of names/clk_hw pointers to clk_core pointers.
+	 */
+	parents = kcalloc(num_parents, sizeof(*parents), GFP_KERNEL);
+	core->parents = parents;
+	if (!parents)
+		return -ENOMEM;
+
+	/* Copy everything over because it might be __initdata */
+	for (i = 0, parent = parents; i < num_parents; i++, parent++) {
+		parent->index = -1;
+		if (parent_names) {
+			/* throw a WARN if any entries are NULL */
+			WARN(!parent_names[i],
+				"%s: invalid NULL in %s's .parent_names\n",
+				__func__, core->name);
+			ret = clk_cpy_name(&parent->name, parent_names[i],
+					   true);
+		} else if (parent_data) {
+			parent->hw = parent_data[i].hw;
+			parent->index = parent_data[i].index;
+			ret = clk_cpy_name(&parent->fw_name,
+					   parent_data[i].fw_name, false);
+			if (!ret)
+				ret = clk_cpy_name(&parent->name,
+						   parent_data[i].name,
+						   false);
+		} else if (parent_hws) {
+			parent->hw = parent_hws[i];
+		} else {
+			ret = -EINVAL;
+			WARN(1, "Must specify parents if num_parents > 0\n");
+		}
+
+		if (ret) {
+			do {
+				kfree_const(parents[i].name);
+				kfree_const(parents[i].fw_name);
+			} while (--i >= 0);
+			kfree(parents);
+
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static void clk_core_free_parent_map(struct clk_core *core)
+{
+	int i = core->num_parents;
+
+	if (!core->num_parents)
+		return;
+
+	while (--i >= 0) {
+		kfree_const(core->parents[i].name);
+		kfree_const(core->parents[i].fw_name);
+	}
+
+	kfree(core->parents);
+}
+
+static struct clk *
+__clk_register(struct device *dev, struct device_node *np, struct clk_hw *hw)
 {
-	int i, ret;
+	int ret;
 	struct clk_core *core;
 
 	core = kzalloc(sizeof(*core), GFP_KERNEL);
@@ -3350,6 +3542,7 @@ struct clk *clk_register(struct device *dev, struct clk_hw *hw)
 	if (dev && pm_runtime_enabled(dev))
 		core->rpm_enabled = true;
 	core->dev = dev;
+	core->of_node = np;
 	if (dev && dev->driver)
 		core->owner = dev->driver->owner;
 	core->hw = hw;
@@ -3359,33 +3552,9 @@ struct clk *clk_register(struct device *dev, struct clk_hw *hw)
 	core->max_rate = ULONG_MAX;
 	hw->core = core;
 
-	/* allocate local copy in case parent_names is __initdata */
-	core->parent_names = kcalloc(core->num_parents, sizeof(char *),
-					GFP_KERNEL);
-
-	if (!core->parent_names) {
-		ret = -ENOMEM;
-		goto fail_parent_names;
-	}
-
-
-	/* copy each string name in case parent_names is __initdata */
-	for (i = 0; i < core->num_parents; i++) {
-		core->parent_names[i] = kstrdup_const(hw->init->parent_names[i],
-						GFP_KERNEL);
-		if (!core->parent_names[i]) {
-			ret = -ENOMEM;
-			goto fail_parent_names_copy;
-		}
-	}
-
-	/* avoid unnecessary string look-ups of clk_core's possible parents. */
-	core->parents = kcalloc(core->num_parents, sizeof(*core->parents),
-				GFP_KERNEL);
-	if (!core->parents) {
-		ret = -ENOMEM;
+	ret = clk_core_populate_parent_map(core);
+	if (ret)
 		goto fail_parents;
-	};
 
 	INIT_HLIST_HEAD(&core->clks);
 
@@ -3396,7 +3565,7 @@ struct clk *clk_register(struct device *dev, struct clk_hw *hw)
 	hw->clk = alloc_clk(core, NULL, NULL);
 	if (IS_ERR(hw->clk)) {
 		ret = PTR_ERR(hw->clk);
-		goto fail_parents;
+		goto fail_create_clk;
 	}
 
 	clk_core_link_consumer(hw->core, hw->clk);
@@ -3412,13 +3581,9 @@ struct clk *clk_register(struct device *dev, struct clk_hw *hw)
 	free_clk(hw->clk);
 	hw->clk = NULL;
 
+fail_create_clk:
+	clk_core_free_parent_map(core);
 fail_parents:
-	kfree(core->parents);
-fail_parent_names_copy:
-	while (--i >= 0)
-		kfree_const(core->parent_names[i]);
-	kfree(core->parent_names);
-fail_parent_names:
 fail_ops:
 	kfree_const(core->name);
 fail_name:
@@ -3426,6 +3591,24 @@ fail_name:
 fail_out:
 	return ERR_PTR(ret);
 }
+
+/**
+ * clk_register - allocate a new clock, register it and return an opaque cookie
+ * @dev: device that is registering this clock
+ * @hw: link to hardware-specific clock data
+ *
+ * clk_register is the *deprecated* interface for populating the clock tree with
+ * new clock nodes. Use clk_hw_register() instead.
+ *
+ * Returns: a pointer to the newly allocated struct clk which
+ * cannot be dereferenced by driver code but may be used in conjunction with the
+ * rest of the clock API.  In the event of an error clk_register will return an
+ * error code; drivers must test for an error code after calling clk_register.
+ */
+struct clk *clk_register(struct device *dev, struct clk_hw *hw)
+{
+	return __clk_register(dev, dev_of_node(dev), hw);
+}
 EXPORT_SYMBOL_GPL(clk_register);
 
 /**
@@ -3440,23 +3623,35 @@ EXPORT_SYMBOL_GPL(clk_register);
  */
 int clk_hw_register(struct device *dev, struct clk_hw *hw)
 {
-	return PTR_ERR_OR_ZERO(clk_register(dev, hw));
+	return PTR_ERR_OR_ZERO(__clk_register(dev, dev_of_node(dev), hw));
 }
 EXPORT_SYMBOL_GPL(clk_hw_register);
 
+/*
+ * of_clk_hw_register - register a clk_hw and return an error code
+ * @node: device_node of device that is registering this clock
+ * @hw: link to hardware-specific clock data
+ *
+ * of_clk_hw_register() is the primary interface for populating the clock tree
+ * with new clock nodes when a struct device is not available, but a struct
+ * device_node is. It returns an integer equal to zero indicating success or
+ * less than zero indicating failure. Drivers must test for an error code after
+ * calling of_clk_hw_register().
+ */
+int of_clk_hw_register(struct device_node *node, struct clk_hw *hw)
+{
+	return PTR_ERR_OR_ZERO(__clk_register(NULL, node, hw));
+}
+EXPORT_SYMBOL_GPL(of_clk_hw_register);
+
 /* Free memory allocated for a clock. */
 static void __clk_release(struct kref *ref)
 {
 	struct clk_core *core = container_of(ref, struct clk_core, ref);
-	int i = core->num_parents;
 
 	lockdep_assert_held(&prepare_lock);
 
-	kfree(core->parents);
-	while (--i >= 0)
-		kfree_const(core->parent_names[i]);
-
-	kfree(core->parent_names);
+	clk_core_free_parent_map(core);
 	kfree_const(core->name);
 	kfree(core);
 }
@@ -3575,9 +3770,10 @@ static void devm_clk_hw_release(struct device *dev, void *res)
  * @dev: device that is registering this clock
  * @hw: link to hardware-specific clock data
  *
- * Managed clk_register(). Clocks returned from this function are
- * automatically clk_unregister()ed on driver detach. See clk_register() for
- * more information.
+ * Managed clk_register(). This function is *deprecated*, use devm_clk_hw_register() instead.
+ *
+ * Clocks returned from this function are automatically clk_unregister()ed on
+ * driver detach. See clk_register() for more information.
  */
 struct clk *devm_clk_register(struct device *dev, struct clk_hw *hw)
 {
@@ -3895,6 +4091,8 @@ EXPORT_SYMBOL_GPL(of_clk_hw_onecell_get);
  * @np: Device node pointer associated with clock provider
  * @clk_src_get: callback for decoding clock
  * @data: context pointer for @clk_src_get callback.
+ *
+ * This function is *deprecated*. Use of_clk_add_hw_provider() instead.
  */
 int of_clk_add_provider(struct device_node *np,
 			struct clk *(*clk_src_get)(struct of_phandle_args *clkspec,
diff --git a/drivers/clk/clk.h b/drivers/clk/clk.h
index 553f531cc232..d8400d623b34 100644
--- a/drivers/clk/clk.h
+++ b/drivers/clk/clk.h
@@ -19,6 +19,8 @@ static inline struct clk_hw *of_clk_get_hw(struct device_node *np,
 }
 #endif
 
+struct clk_hw *clk_find_hw(const char *dev_id, const char *con_id);
+
 #ifdef CONFIG_COMMON_CLK
 struct clk *clk_hw_create_clk(struct device *dev, struct clk_hw *hw,
 			      const char *dev_id, const char *con_id);
diff --git a/drivers/clk/clkdev.c b/drivers/clk/clkdev.c
index 6e787cc9e5b9..2afc8df8acff 100644
--- a/drivers/clk/clkdev.c
+++ b/drivers/clk/clkdev.c
@@ -72,25 +72,26 @@ static struct clk_lookup *clk_find(const char *dev_id, const char *con_id)
 	return cl;
 }
 
-static struct clk *__clk_get_sys(struct device *dev, const char *dev_id,
-				 const char *con_id)
+struct clk_hw *clk_find_hw(const char *dev_id, const char *con_id)
 {
 	struct clk_lookup *cl;
-	struct clk *clk = NULL;
+	struct clk_hw *hw = ERR_PTR(-ENOENT);
 
 	mutex_lock(&clocks_mutex);
-
 	cl = clk_find(dev_id, con_id);
-	if (!cl)
-		goto out;
-
-	clk = clk_hw_create_clk(dev, cl->clk_hw, dev_id, con_id);
-	if (IS_ERR(clk))
-		cl = NULL;
-out:
+	if (cl)
+		hw = cl->clk_hw;
 	mutex_unlock(&clocks_mutex);
 
-	return cl ? clk : ERR_PTR(-ENOENT);
+	return hw;
+}
+
+static struct clk *__clk_get_sys(struct device *dev, const char *dev_id,
+				 const char *con_id)
+{
+	struct clk_hw *hw = clk_find_hw(dev_id, con_id);
+
+	return clk_hw_create_clk(dev, hw, dev_id, con_id);
 }
 
 struct clk *clk_get_sys(const char *dev_id, const char *con_id)
diff --git a/drivers/clk/davinci/da8xx-cfgchip.c b/drivers/clk/davinci/da8xx-cfgchip.c
index d1bbee19ed0f..bdc52364b421 100644
--- a/drivers/clk/davinci/da8xx-cfgchip.c
+++ b/drivers/clk/davinci/da8xx-cfgchip.c
@@ -160,10 +160,8 @@ static int __init da8xx_cfgchip_register_div4p5(struct device *dev,
 	struct da8xx_cfgchip_gate_clk *gate;
 
 	gate = da8xx_cfgchip_gate_clk_register(dev, &da8xx_div4p5ena_info, regmap);
-	if (IS_ERR(gate))
-		return PTR_ERR(gate);
 
-	return 0;
+	return PTR_ERR_OR_ZERO(gate);
 }
 
 static int __init
diff --git a/drivers/clk/davinci/pll.h b/drivers/clk/davinci/pll.h
index 7cc354dd29e2..c2a453caa131 100644
--- a/drivers/clk/davinci/pll.h
+++ b/drivers/clk/davinci/pll.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Clock driver for TI Davinci PSC controllers
  *
diff --git a/drivers/clk/davinci/psc.h b/drivers/clk/davinci/psc.h
index cc5614567a70..69070f834391 100644
--- a/drivers/clk/davinci/psc.h
+++ b/drivers/clk/davinci/psc.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Clock driver for TI Davinci PSC controllers
  *
diff --git a/drivers/clk/hisilicon/clk-hi3660.c b/drivers/clk/hisilicon/clk-hi3660.c
index f40419959656..794eeff0d5d2 100644
--- a/drivers/clk/hisilicon/clk-hi3660.c
+++ b/drivers/clk/hisilicon/clk-hi3660.c
@@ -163,8 +163,12 @@ static const struct hisi_gate_clock hi3660_crgctrl_gate_sep_clks[] = {
 	  "clk_isp_snclk_mux", CLK_SET_RATE_PARENT, 0x50, 17, 0, },
 	{ HI3660_CLK_GATE_ISP_SNCLK2, "clk_gate_isp_snclk2",
 	  "clk_isp_snclk_mux", CLK_SET_RATE_PARENT, 0x50, 18, 0, },
+	/*
+	 * clk_gate_ufs_subsys is a system bus clock, mark it as critical
+	 * clock and keep it on for system suspend and resume.
+	 */
 	{ HI3660_CLK_GATE_UFS_SUBSYS, "clk_gate_ufs_subsys", "clk_div_sysbus",
-	  CLK_SET_RATE_PARENT, 0x50, 21, 0, },
+	  CLK_SET_RATE_PARENT | CLK_IS_CRITICAL, 0x50, 21, 0, },
 	{ HI3660_PCLK_GATE_DSI0, "pclk_gate_dsi0", "clk_div_cfgbus",
 	  CLK_SET_RATE_PARENT, 0x50, 28, 0, },
 	{ HI3660_PCLK_GATE_DSI1, "pclk_gate_dsi1", "clk_div_cfgbus",
diff --git a/drivers/clk/hisilicon/clk-hisi-phase.c b/drivers/clk/hisilicon/clk-hisi-phase.c
index 5fdc267bb2da..ba6afad66a2b 100644
--- a/drivers/clk/hisilicon/clk-hisi-phase.c
+++ b/drivers/clk/hisilicon/clk-hisi-phase.c
@@ -75,10 +75,10 @@ static int hisi_clk_set_phase(struct clk_hw *hw, int degrees)
 
 	spin_lock_irqsave(phase->lock, flags);
 
-	val = clk_readl(phase->reg);
+	val = readl(phase->reg);
 	val &= ~phase->mask;
 	val |= regval << phase->shift;
-	clk_writel(val, phase->reg);
+	writel(val, phase->reg);
 
 	spin_unlock_irqrestore(phase->lock, flags);
 
diff --git a/drivers/clk/imx/Makefile b/drivers/clk/imx/Makefile
index 0d5180fbe988..05641c64b317 100644
--- a/drivers/clk/imx/Makefile
+++ b/drivers/clk/imx/Makefile
@@ -35,7 +35,7 @@ obj-$(CONFIG_SOC_IMX25)  += clk-imx25.o
 obj-$(CONFIG_SOC_IMX27)  += clk-imx27.o
 obj-$(CONFIG_SOC_IMX31)  += clk-imx31.o
 obj-$(CONFIG_SOC_IMX35)  += clk-imx35.o
-obj-$(CONFIG_SOC_IMX5)   += clk-imx51-imx53.o
+obj-$(CONFIG_SOC_IMX5)   += clk-imx5.o
 obj-$(CONFIG_SOC_IMX6Q)  += clk-imx6q.o
 obj-$(CONFIG_SOC_IMX6SL) += clk-imx6sl.o
 obj-$(CONFIG_SOC_IMX6SLL) += clk-imx6sll.o
diff --git a/drivers/clk/imx/clk-divider-gate.c b/drivers/clk/imx/clk-divider-gate.c
index df1f8429fe16..2a8352a316c7 100644
--- a/drivers/clk/imx/clk-divider-gate.c
+++ b/drivers/clk/imx/clk-divider-gate.c
@@ -29,7 +29,7 @@ static unsigned long clk_divider_gate_recalc_rate_ro(struct clk_hw *hw,
 	struct clk_divider *div = to_clk_divider(hw);
 	unsigned int val;
 
-	val = clk_readl(div->reg) >> div->shift;
+	val = readl(div->reg) >> div->shift;
 	val &= clk_div_mask(div->width);
 	if (!val)
 		return 0;
@@ -51,7 +51,7 @@ static unsigned long clk_divider_gate_recalc_rate(struct clk_hw *hw,
 	if (!clk_hw_is_enabled(hw)) {
 		val = div_gate->cached_val;
 	} else {
-		val = clk_readl(div->reg) >> div->shift;
+		val = readl(div->reg) >> div->shift;
 		val &= clk_div_mask(div->width);
 	}
 
@@ -87,10 +87,10 @@ static int clk_divider_gate_set_rate(struct clk_hw *hw, unsigned long rate,
 	spin_lock_irqsave(div->lock, flags);
 
 	if (clk_hw_is_enabled(hw)) {
-		val = clk_readl(div->reg);
+		val = readl(div->reg);
 		val &= ~(clk_div_mask(div->width) << div->shift);
 		val |= (u32)value << div->shift;
-		clk_writel(val, div->reg);
+		writel(val, div->reg);
 	} else {
 		div_gate->cached_val = value;
 	}
@@ -114,9 +114,9 @@ static int clk_divider_enable(struct clk_hw *hw)
 
 	spin_lock_irqsave(div->lock, flags);
 	/* restore div val */
-	val = clk_readl(div->reg);
+	val = readl(div->reg);
 	val |= div_gate->cached_val << div->shift;
-	clk_writel(val, div->reg);
+	writel(val, div->reg);
 
 	spin_unlock_irqrestore(div->lock, flags);
 
@@ -133,10 +133,10 @@ static void clk_divider_disable(struct clk_hw *hw)
 	spin_lock_irqsave(div->lock, flags);
 
 	/* store the current div val */
-	val = clk_readl(div->reg) >> div->shift;
+	val = readl(div->reg) >> div->shift;
 	val &= clk_div_mask(div->width);
 	div_gate->cached_val = val;
-	clk_writel(0, div->reg);
+	writel(0, div->reg);
 
 	spin_unlock_irqrestore(div->lock, flags);
 }
@@ -146,7 +146,7 @@ static int clk_divider_is_enabled(struct clk_hw *hw)
 	struct clk_divider *div = to_clk_divider(hw);
 	u32 val;
 
-	val = clk_readl(div->reg) >> div->shift;
+	val = readl(div->reg) >> div->shift;
 	val &= clk_div_mask(div->width);
 
 	return val ? 1 : 0;
@@ -206,7 +206,7 @@ struct clk_hw *imx_clk_divider_gate(const char *name, const char *parent_name,
 	div_gate->divider.hw.init = &init;
 	div_gate->divider.flags = CLK_DIVIDER_ONE_BASED | clk_divider_flags;
 	/* cache gate status */
-	val = clk_readl(reg) >> shift;
+	val = readl(reg) >> shift;
 	val &= clk_div_mask(width);
 	div_gate->cached_val = val;
 
diff --git a/drivers/clk/imx/clk-imx51-imx53.c b/drivers/clk/imx/clk-imx5.c
index e91c826bce70..c85ebd74a8a5 100644
--- a/drivers/clk/imx/clk-imx51-imx53.c
+++ b/drivers/clk/imx/clk-imx5.c
@@ -164,10 +164,6 @@ static void __init mx5_clocks_common_init(void __iomem *ccm_base)
 	clk[IMX5_CLK_CKIH1]		= imx_obtain_fixed_clock("ckih1", 0);
 	clk[IMX5_CLK_CKIH2]		= imx_obtain_fixed_clock("ckih2", 0);
 
-	clk[IMX5_CLK_PERIPH_APM]	= imx_clk_mux("periph_apm", MXC_CCM_CBCMR, 12, 2,
-						periph_apm_sel, ARRAY_SIZE(periph_apm_sel));
-	clk[IMX5_CLK_MAIN_BUS]		= imx_clk_mux("main_bus", MXC_CCM_CBCDR, 25, 1,
-						main_bus_sel, ARRAY_SIZE(main_bus_sel));
 	clk[IMX5_CLK_PER_LP_APM]	= imx_clk_mux("per_lp_apm", MXC_CCM_CBCMR, 1, 1,
 						per_lp_apm_sel, ARRAY_SIZE(per_lp_apm_sel));
 	clk[IMX5_CLK_PER_PRED1]		= imx_clk_divider("per_pred1", "per_lp_apm", MXC_CCM_CBCDR, 6, 2);
@@ -191,16 +187,10 @@ static void __init mx5_clocks_common_init(void __iomem *ccm_base)
 	clk[IMX5_CLK_UART_PRED]		= imx_clk_divider("uart_pred", "uart_sel", MXC_CCM_CSCDR1, 3, 3);
 	clk[IMX5_CLK_UART_ROOT]		= imx_clk_divider("uart_root", "uart_pred", MXC_CCM_CSCDR1, 0, 3);
 
-	clk[IMX5_CLK_ESDHC_A_SEL]	= imx_clk_mux("esdhc_a_sel", MXC_CCM_CSCMR1, 20, 2,
-						standard_pll_sel, ARRAY_SIZE(standard_pll_sel));
-	clk[IMX5_CLK_ESDHC_B_SEL]	= imx_clk_mux("esdhc_b_sel", MXC_CCM_CSCMR1, 16, 2,
-						standard_pll_sel, ARRAY_SIZE(standard_pll_sel));
 	clk[IMX5_CLK_ESDHC_A_PRED]	= imx_clk_divider("esdhc_a_pred", "esdhc_a_sel", MXC_CCM_CSCDR1, 16, 3);
 	clk[IMX5_CLK_ESDHC_A_PODF]	= imx_clk_divider("esdhc_a_podf", "esdhc_a_pred", MXC_CCM_CSCDR1, 11, 3);
 	clk[IMX5_CLK_ESDHC_B_PRED]	= imx_clk_divider("esdhc_b_pred", "esdhc_b_sel", MXC_CCM_CSCDR1, 22, 3);
 	clk[IMX5_CLK_ESDHC_B_PODF]	= imx_clk_divider("esdhc_b_podf", "esdhc_b_pred", MXC_CCM_CSCDR1, 19, 3);
-	clk[IMX5_CLK_ESDHC_C_SEL]	= imx_clk_mux("esdhc_c_sel", MXC_CCM_CSCMR1, 19, 1, esdhc_c_sel, ARRAY_SIZE(esdhc_c_sel));
-	clk[IMX5_CLK_ESDHC_D_SEL]	= imx_clk_mux("esdhc_d_sel", MXC_CCM_CSCMR1, 18, 1, esdhc_d_sel, ARRAY_SIZE(esdhc_d_sel));
 
 	clk[IMX5_CLK_EMI_SEL]		= imx_clk_mux("emi_sel", MXC_CCM_CBCDR, 26, 1,
 						emi_slow_sel, ARRAY_SIZE(emi_slow_sel));
@@ -311,10 +301,6 @@ static void __init mx5_clocks_common_init(void __iomem *ccm_base)
 	clk_register_clkdev(clk[IMX5_CLK_CPU_PODF], NULL, "cpu0");
 	clk_register_clkdev(clk[IMX5_CLK_GPC_DVFS], "gpc_dvfs", NULL);
 
-	/* Set SDHC parents to be PLL2 */
-	clk_set_parent(clk[IMX5_CLK_ESDHC_A_SEL], clk[IMX5_CLK_PLL2_SW]);
-	clk_set_parent(clk[IMX5_CLK_ESDHC_B_SEL], clk[IMX5_CLK_PLL2_SW]);
-
 	/* move usb phy clk to 24MHz */
 	clk_set_parent(clk[IMX5_CLK_USB_PHY_SEL], clk[IMX5_CLK_OSC]);
 }
@@ -342,8 +328,21 @@ static void __init mx50_clocks_init(struct device_node *np)
 
 	mx5_clocks_common_init(ccm_base);
 
+	/*
+	 * This clock is called periph_clk in the i.MX50 Reference Manual, but
+	 * it comes closest in scope to the main_bus_clk of i.MX51 and i.MX53
+	 */
+	clk[IMX5_CLK_MAIN_BUS]          = imx_clk_mux("main_bus", MXC_CCM_CBCDR, 25, 2,
+						standard_pll_sel, ARRAY_SIZE(standard_pll_sel));
+
 	clk[IMX5_CLK_LP_APM]		= imx_clk_mux("lp_apm", MXC_CCM_CCSR, 10, 1,
 						lp_apm_sel, ARRAY_SIZE(lp_apm_sel));
+	clk[IMX5_CLK_ESDHC_A_SEL]	= imx_clk_mux("esdhc_a_sel", MXC_CCM_CSCMR1, 21, 2,
+						standard_pll_sel, ARRAY_SIZE(standard_pll_sel));
+	clk[IMX5_CLK_ESDHC_B_SEL]	= imx_clk_mux("esdhc_b_sel", MXC_CCM_CSCMR1, 16, 2,
+						standard_pll_sel, ARRAY_SIZE(standard_pll_sel));
+	clk[IMX5_CLK_ESDHC_C_SEL]	= imx_clk_mux("esdhc_c_sel", MXC_CCM_CSCMR1, 20, 1, esdhc_c_sel, ARRAY_SIZE(esdhc_c_sel));
+	clk[IMX5_CLK_ESDHC_D_SEL]	= imx_clk_mux("esdhc_d_sel", MXC_CCM_CSCMR1, 19, 1, esdhc_d_sel, ARRAY_SIZE(esdhc_d_sel));
 	clk[IMX5_CLK_ESDHC1_PER_GATE]	= imx_clk_gate2("esdhc1_per_gate", "esdhc_a_podf", MXC_CCM_CCGR3, 2);
 	clk[IMX5_CLK_ESDHC2_PER_GATE]	= imx_clk_gate2("esdhc2_per_gate", "esdhc_c_sel", MXC_CCM_CCGR3, 6);
 	clk[IMX5_CLK_ESDHC3_PER_GATE]	= imx_clk_gate2("esdhc3_per_gate", "esdhc_b_podf", MXC_CCM_CCGR3, 10);
@@ -372,6 +371,10 @@ static void __init mx50_clocks_init(struct device_node *np)
 	clk_data.clk_num = ARRAY_SIZE(clk);
 	of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data);
 
+	/* Set SDHC parents to be PLL2 */
+	clk_set_parent(clk[IMX5_CLK_ESDHC_A_SEL], clk[IMX5_CLK_PLL2_SW]);
+	clk_set_parent(clk[IMX5_CLK_ESDHC_B_SEL], clk[IMX5_CLK_PLL2_SW]);
+
 	/* set SDHC root clock to 200MHZ*/
 	clk_set_rate(clk[IMX5_CLK_ESDHC_A_PODF], 200000000);
 	clk_set_rate(clk[IMX5_CLK_ESDHC_B_PODF], 200000000);
@@ -410,6 +413,10 @@ static void __init mx51_clocks_init(struct device_node *np)
 
 	mx5_clocks_common_init(ccm_base);
 
+	clk[IMX5_CLK_PERIPH_APM]	= imx_clk_mux("periph_apm", MXC_CCM_CBCMR, 12, 2,
+						periph_apm_sel, ARRAY_SIZE(periph_apm_sel));
+	clk[IMX5_CLK_MAIN_BUS]		= imx_clk_mux("main_bus", MXC_CCM_CBCDR, 25, 1,
+						main_bus_sel, ARRAY_SIZE(main_bus_sel));
 	clk[IMX5_CLK_LP_APM]		= imx_clk_mux("lp_apm", MXC_CCM_CCSR, 9, 1,
 						lp_apm_sel, ARRAY_SIZE(lp_apm_sel));
 	clk[IMX5_CLK_IPU_DI0_SEL]	= imx_clk_mux_flags("ipu_di0_sel", MXC_CCM_CSCMR2, 26, 3,
@@ -422,6 +429,12 @@ static void __init mx51_clocks_init(struct device_node *np)
 						mx51_tve_sel, ARRAY_SIZE(mx51_tve_sel));
 	clk[IMX5_CLK_TVE_GATE]		= imx_clk_gate2("tve_gate", "tve_sel", MXC_CCM_CCGR2, 30);
 	clk[IMX5_CLK_TVE_PRED]		= imx_clk_divider("tve_pred", "pll3_sw", MXC_CCM_CDCDR, 28, 3);
+	clk[IMX5_CLK_ESDHC_A_SEL]	= imx_clk_mux("esdhc_a_sel", MXC_CCM_CSCMR1, 20, 2,
+						standard_pll_sel, ARRAY_SIZE(standard_pll_sel));
+	clk[IMX5_CLK_ESDHC_B_SEL]	= imx_clk_mux("esdhc_b_sel", MXC_CCM_CSCMR1, 16, 2,
+						standard_pll_sel, ARRAY_SIZE(standard_pll_sel));
+	clk[IMX5_CLK_ESDHC_C_SEL]	= imx_clk_mux("esdhc_c_sel", MXC_CCM_CSCMR1, 19, 1, esdhc_c_sel, ARRAY_SIZE(esdhc_c_sel));
+	clk[IMX5_CLK_ESDHC_D_SEL]	= imx_clk_mux("esdhc_d_sel", MXC_CCM_CSCMR1, 18, 1, esdhc_d_sel, ARRAY_SIZE(esdhc_d_sel));
 	clk[IMX5_CLK_ESDHC1_PER_GATE]	= imx_clk_gate2("esdhc1_per_gate", "esdhc_a_podf", MXC_CCM_CCGR3, 2);
 	clk[IMX5_CLK_ESDHC2_PER_GATE]	= imx_clk_gate2("esdhc2_per_gate", "esdhc_b_podf", MXC_CCM_CCGR3, 6);
 	clk[IMX5_CLK_ESDHC3_PER_GATE]	= imx_clk_gate2("esdhc3_per_gate", "esdhc_c_sel", MXC_CCM_CCGR3, 10);
@@ -452,6 +465,10 @@ static void __init mx51_clocks_init(struct device_node *np)
 	/* set the usboh3 parent to pll2_sw */
 	clk_set_parent(clk[IMX5_CLK_USBOH3_SEL], clk[IMX5_CLK_PLL2_SW]);
 
+	/* Set SDHC parents to be PLL2 */
+	clk_set_parent(clk[IMX5_CLK_ESDHC_A_SEL], clk[IMX5_CLK_PLL2_SW]);
+	clk_set_parent(clk[IMX5_CLK_ESDHC_B_SEL], clk[IMX5_CLK_PLL2_SW]);
+
 	/* set SDHC root clock to 166.25MHZ*/
 	clk_set_rate(clk[IMX5_CLK_ESDHC_A_PODF], 166250000);
 	clk_set_rate(clk[IMX5_CLK_ESDHC_B_PODF], 166250000);
@@ -506,6 +523,10 @@ static void __init mx53_clocks_init(struct device_node *np)
 
 	mx5_clocks_common_init(ccm_base);
 
+	clk[IMX5_CLK_PERIPH_APM]	= imx_clk_mux("periph_apm", MXC_CCM_CBCMR, 12, 2,
+						periph_apm_sel, ARRAY_SIZE(periph_apm_sel));
+	clk[IMX5_CLK_MAIN_BUS]		= imx_clk_mux("main_bus", MXC_CCM_CBCDR, 25, 1,
+						main_bus_sel, ARRAY_SIZE(main_bus_sel));
 	clk[IMX5_CLK_LP_APM]		= imx_clk_mux("lp_apm", MXC_CCM_CCSR, 10, 1,
 						lp_apm_sel, ARRAY_SIZE(lp_apm_sel));
 	clk[IMX5_CLK_LDB_DI1_DIV_3_5]	= imx_clk_fixed_factor("ldb_di1_div_3_5", "ldb_di1_sel", 2, 7);
@@ -527,6 +548,12 @@ static void __init mx53_clocks_init(struct device_node *np)
 						mx53_tve_ext_sel, ARRAY_SIZE(mx53_tve_ext_sel), CLK_SET_RATE_PARENT);
 	clk[IMX5_CLK_TVE_GATE]		= imx_clk_gate2("tve_gate", "tve_pred", MXC_CCM_CCGR2, 30);
 	clk[IMX5_CLK_TVE_PRED]		= imx_clk_divider("tve_pred", "tve_ext_sel", MXC_CCM_CDCDR, 28, 3);
+	clk[IMX5_CLK_ESDHC_A_SEL]	= imx_clk_mux("esdhc_a_sel", MXC_CCM_CSCMR1, 20, 2,
+						standard_pll_sel, ARRAY_SIZE(standard_pll_sel));
+	clk[IMX5_CLK_ESDHC_B_SEL]	= imx_clk_mux("esdhc_b_sel", MXC_CCM_CSCMR1, 16, 2,
+						standard_pll_sel, ARRAY_SIZE(standard_pll_sel));
+	clk[IMX5_CLK_ESDHC_C_SEL]	= imx_clk_mux("esdhc_c_sel", MXC_CCM_CSCMR1, 19, 1, esdhc_c_sel, ARRAY_SIZE(esdhc_c_sel));
+	clk[IMX5_CLK_ESDHC_D_SEL]	= imx_clk_mux("esdhc_d_sel", MXC_CCM_CSCMR1, 18, 1, esdhc_d_sel, ARRAY_SIZE(esdhc_d_sel));
 	clk[IMX5_CLK_ESDHC1_PER_GATE]	= imx_clk_gate2("esdhc1_per_gate", "esdhc_a_podf", MXC_CCM_CCGR3, 2);
 	clk[IMX5_CLK_ESDHC2_PER_GATE]	= imx_clk_gate2("esdhc2_per_gate", "esdhc_c_sel", MXC_CCM_CCGR3, 6);
 	clk[IMX5_CLK_ESDHC3_PER_GATE]	= imx_clk_gate2("esdhc3_per_gate", "esdhc_b_podf", MXC_CCM_CCGR3, 10);
@@ -589,6 +616,10 @@ static void __init mx53_clocks_init(struct device_node *np)
 	clk_data.clk_num = ARRAY_SIZE(clk);
 	of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data);
 
+	/* Set SDHC parents to be PLL2 */
+	clk_set_parent(clk[IMX5_CLK_ESDHC_A_SEL], clk[IMX5_CLK_PLL2_SW]);
+	clk_set_parent(clk[IMX5_CLK_ESDHC_B_SEL], clk[IMX5_CLK_PLL2_SW]);
+
 	/* set SDHC root clock to 200MHZ*/
 	clk_set_rate(clk[IMX5_CLK_ESDHC_A_PODF], 200000000);
 	clk_set_rate(clk[IMX5_CLK_ESDHC_B_PODF], 200000000);
diff --git a/drivers/clk/imx/clk-imx6sll.c b/drivers/clk/imx/clk-imx6sll.c
index 3bd2044cf25c..7eea448cb9a9 100644
--- a/drivers/clk/imx/clk-imx6sll.c
+++ b/drivers/clk/imx/clk-imx6sll.c
@@ -76,6 +76,20 @@ static u32 share_count_ssi1;
 static u32 share_count_ssi2;
 static u32 share_count_ssi3;
 
+static struct clk ** const uart_clks[] __initconst = {
+	&clks[IMX6SLL_CLK_UART1_IPG],
+	&clks[IMX6SLL_CLK_UART1_SERIAL],
+	&clks[IMX6SLL_CLK_UART2_IPG],
+	&clks[IMX6SLL_CLK_UART2_SERIAL],
+	&clks[IMX6SLL_CLK_UART3_IPG],
+	&clks[IMX6SLL_CLK_UART3_SERIAL],
+	&clks[IMX6SLL_CLK_UART4_IPG],
+	&clks[IMX6SLL_CLK_UART4_SERIAL],
+	&clks[IMX6SLL_CLK_UART5_IPG],
+	&clks[IMX6SLL_CLK_UART5_SERIAL],
+	NULL
+};
+
 static void __init imx6sll_clocks_init(struct device_node *ccm_node)
 {
 	struct device_node *np;
@@ -268,7 +282,7 @@ static void __init imx6sll_clocks_init(struct device_node *ccm_node)
 	clks[IMX6SLL_CLK_GPT_BUS]	= imx_clk_gate2("gpt1_bus",	"perclk", base + 0x6c, 20);
 	clks[IMX6SLL_CLK_GPT_SERIAL]	= imx_clk_gate2("gpt1_serial",	"perclk", base + 0x6c, 22);
 	clks[IMX6SLL_CLK_UART4_IPG]	= imx_clk_gate2("uart4_ipg",	"ipg", base + 0x6c, 24);
-	clks[IMX6SLL_CLK_UART4_SERIAL]	= imx_clk_gate2("uart4_serail",	"uart_podf", base + 0x6c, 24);
+	clks[IMX6SLL_CLK_UART4_SERIAL]	= imx_clk_gate2("uart4_serial",	"uart_podf", base + 0x6c, 24);
 	clks[IMX6SLL_CLK_GPIO1]		= imx_clk_gate2("gpio1",	"ipg", base + 0x6c, 26);
 	clks[IMX6SLL_CLK_GPIO5]		= imx_clk_gate2("gpio5",	"ipg", base + 0x6c, 30);
 
@@ -334,6 +348,8 @@ static void __init imx6sll_clocks_init(struct device_node *ccm_node)
 	clk_data.clk_num = ARRAY_SIZE(clks);
 	of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data);
 
+	imx_register_uart_clocks(uart_clks);
+
 	/* Lower the AHB clock rate before changing the clock source. */
 	clk_set_rate(clks[IMX6SLL_CLK_AHB], 99000000);
 
diff --git a/drivers/clk/imx/clk-imx7d.c b/drivers/clk/imx/clk-imx7d.c
index cfbd8d4edb85..5b8a0c729f90 100644
--- a/drivers/clk/imx/clk-imx7d.c
+++ b/drivers/clk/imx/clk-imx7d.c
@@ -417,8 +417,8 @@ static void __init imx7d_clocks_init(struct device_node *ccm_node)
 	clks[IMX7D_PLL_DRAM_MAIN] = imx_clk_pllv3(IMX_PLLV3_DDR_IMX7, "pll_dram_main", "osc", base + 0x70, 0x7f);
 	clks[IMX7D_PLL_SYS_MAIN]  = imx_clk_pllv3(IMX_PLLV3_GENERIC, "pll_sys_main", "osc", base + 0xb0, 0x1);
 	clks[IMX7D_PLL_ENET_MAIN] = imx_clk_pllv3(IMX_PLLV3_ENET_IMX7, "pll_enet_main", "osc", base + 0xe0, 0x0);
-	clks[IMX7D_PLL_AUDIO_MAIN] = imx_clk_pllv3(IMX_PLLV3_AV, "pll_audio_main", "osc", base + 0xf0, 0x7f);
-	clks[IMX7D_PLL_VIDEO_MAIN] = imx_clk_pllv3(IMX_PLLV3_AV, "pll_video_main", "osc", base + 0x130, 0x7f);
+	clks[IMX7D_PLL_AUDIO_MAIN] = imx_clk_pllv3(IMX_PLLV3_AV_IMX7, "pll_audio_main", "osc", base + 0xf0, 0x7f);
+	clks[IMX7D_PLL_VIDEO_MAIN] = imx_clk_pllv3(IMX_PLLV3_AV_IMX7, "pll_video_main", "osc", base + 0x130, 0x7f);
 
 	clks[IMX7D_PLL_ARM_MAIN_BYPASS]  = imx_clk_mux_flags("pll_arm_main_bypass", base + 0x60, 16, 1, pll_arm_bypass_sel, ARRAY_SIZE(pll_arm_bypass_sel), CLK_SET_RATE_PARENT);
 	clks[IMX7D_PLL_DRAM_MAIN_BYPASS] = imx_clk_mux_flags("pll_dram_main_bypass", base + 0x70, 16, 1, pll_dram_bypass_sel, ARRAY_SIZE(pll_dram_bypass_sel), CLK_SET_RATE_PARENT);
diff --git a/drivers/clk/imx/clk-imx7ulp.c b/drivers/clk/imx/clk-imx7ulp.c
index ce306631e844..66682100f14c 100644
--- a/drivers/clk/imx/clk-imx7ulp.c
+++ b/drivers/clk/imx/clk-imx7ulp.c
@@ -151,7 +151,6 @@ static void __init imx7ulp_clk_pcc2_init(struct device_node *np)
 	clks[IMX7ULP_CLK_DMA1]		= imx_clk_hw_gate("dma1", "nic1_clk", base + 0x20, 30);
 	clks[IMX7ULP_CLK_RGPIO2P1]	= imx_clk_hw_gate("rgpio2p1", "nic1_bus_clk", base + 0x3c, 30);
 	clks[IMX7ULP_CLK_DMA_MUX1]	= imx_clk_hw_gate("dma_mux1", "nic1_bus_clk", base + 0x84, 30);
-	clks[IMX7ULP_CLK_SNVS]		= imx_clk_hw_gate("snvs", "nic1_bus_clk", base + 0x8c, 30);
 	clks[IMX7ULP_CLK_CAAM]		= imx_clk_hw_gate("caam", "nic1_clk", base + 0x90, 30);
 	clks[IMX7ULP_CLK_LPTPM4]	= imx7ulp_clk_composite("lptpm4",  periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, false, true, base + 0x94);
 	clks[IMX7ULP_CLK_LPTPM5]	= imx7ulp_clk_composite("lptpm5",  periph_bus_sels, ARRAY_SIZE(periph_bus_sels), true, false, true, base + 0x98);
diff --git a/drivers/clk/imx/clk-imx8mq.c b/drivers/clk/imx/clk-imx8mq.c
index a9b3888aef0c..daf1841b2adb 100644
--- a/drivers/clk/imx/clk-imx8mq.c
+++ b/drivers/clk/imx/clk-imx8mq.c
@@ -458,6 +458,7 @@ static int imx8mq_clocks_probe(struct platform_device *pdev)
 	clks[IMX8MQ_CLK_DSI_DBI] = imx8m_clk_composite("dsi_dbi", imx8mq_dsi_dbi_sels, base + 0xbc00);
 	clks[IMX8MQ_CLK_DSI_ESC] = imx8m_clk_composite("dsi_esc", imx8mq_dsi_esc_sels, base + 0xbc80);
 	clks[IMX8MQ_CLK_DSI_AHB] = imx8m_clk_composite("dsi_ahb", imx8mq_dsi_ahb_sels, base + 0x9200);
+	clks[IMX8MQ_CLK_DSI_IPG_DIV] = imx_clk_divider2("dsi_ipg_div", "dsi_ahb", base + 0x9280, 0, 6);
 	clks[IMX8MQ_CLK_CSI1_CORE] = imx8m_clk_composite("csi1_core", imx8mq_csi1_core_sels, base + 0xbd00);
 	clks[IMX8MQ_CLK_CSI1_PHY_REF] = imx8m_clk_composite("csi1_phy_ref", imx8mq_csi1_phy_sels, base + 0xbd80);
 	clks[IMX8MQ_CLK_CSI1_ESC] = imx8m_clk_composite("csi1_esc", imx8mq_csi1_esc_sels, base + 0xbe00);
diff --git a/drivers/clk/imx/clk-pfdv2.c b/drivers/clk/imx/clk-pfdv2.c
index 7e9134b205ab..fb567dcc2118 100644
--- a/drivers/clk/imx/clk-pfdv2.c
+++ b/drivers/clk/imx/clk-pfdv2.c
@@ -43,7 +43,7 @@ static int clk_pfdv2_wait(struct clk_pfdv2 *pfd)
 {
 	u32 val;
 
-	return readl_poll_timeout(pfd->reg, val, val & pfd->vld_bit,
+	return readl_poll_timeout(pfd->reg, val, val & (1 << pfd->vld_bit),
 				  0, LOCK_TIMEOUT_US);
 }
 
@@ -55,7 +55,7 @@ static int clk_pfdv2_enable(struct clk_hw *hw)
 
 	spin_lock_irqsave(&pfd_lock, flags);
 	val = readl_relaxed(pfd->reg);
-	val &= ~pfd->gate_bit;
+	val &= ~(1 << pfd->gate_bit);
 	writel_relaxed(val, pfd->reg);
 	spin_unlock_irqrestore(&pfd_lock, flags);
 
@@ -70,7 +70,7 @@ static void clk_pfdv2_disable(struct clk_hw *hw)
 
 	spin_lock_irqsave(&pfd_lock, flags);
 	val = readl_relaxed(pfd->reg);
-	val |= pfd->gate_bit;
+	val |= (1 << pfd->gate_bit);
 	writel_relaxed(val, pfd->reg);
 	spin_unlock_irqrestore(&pfd_lock, flags);
 }
@@ -123,7 +123,7 @@ static int clk_pfdv2_is_enabled(struct clk_hw *hw)
 {
 	struct clk_pfdv2 *pfd = to_clk_pfdv2(hw);
 
-	if (readl_relaxed(pfd->reg) & pfd->gate_bit)
+	if (readl_relaxed(pfd->reg) & (1 << pfd->gate_bit))
 		return 0;
 
 	return 1;
@@ -180,7 +180,7 @@ struct clk_hw *imx_clk_pfdv2(const char *name, const char *parent_name,
 		return ERR_PTR(-ENOMEM);
 
 	pfd->reg = reg;
-	pfd->gate_bit = 1 << ((idx + 1) * 8 - 1);
+	pfd->gate_bit = (idx + 1) * 8 - 1;
 	pfd->vld_bit = pfd->gate_bit - 1;
 	pfd->frac_off = idx * 8;
 
diff --git a/drivers/clk/imx/clk-pll14xx.c b/drivers/clk/imx/clk-pll14xx.c
index 113d71042199..b7213023b238 100644
--- a/drivers/clk/imx/clk-pll14xx.c
+++ b/drivers/clk/imx/clk-pll14xx.c
@@ -74,10 +74,9 @@ static unsigned long clk_pll1416x_recalc_rate(struct clk_hw *hw,
 						  unsigned long parent_rate)
 {
 	struct clk_pll14xx *pll = to_clk_pll14xx(hw);
-	u32 mdiv, pdiv, sdiv, pll_gnrl, pll_div;
+	u32 mdiv, pdiv, sdiv, pll_div;
 	u64 fvco = parent_rate;
 
-	pll_gnrl = readl_relaxed(pll->base);
 	pll_div = readl_relaxed(pll->base + 4);
 	mdiv = (pll_div & MDIV_MASK) >> MDIV_SHIFT;
 	pdiv = (pll_div & PDIV_MASK) >> PDIV_SHIFT;
@@ -93,11 +92,10 @@ static unsigned long clk_pll1443x_recalc_rate(struct clk_hw *hw,
 						  unsigned long parent_rate)
 {
 	struct clk_pll14xx *pll = to_clk_pll14xx(hw);
-	u32 mdiv, pdiv, sdiv, pll_gnrl, pll_div_ctl0, pll_div_ctl1;
+	u32 mdiv, pdiv, sdiv, pll_div_ctl0, pll_div_ctl1;
 	short int kdiv;
 	u64 fvco = parent_rate;
 
-	pll_gnrl = readl_relaxed(pll->base);
 	pll_div_ctl0 = readl_relaxed(pll->base + 4);
 	pll_div_ctl1 = readl_relaxed(pll->base + 8);
 	mdiv = (pll_div_ctl0 & MDIV_MASK) >> MDIV_SHIFT;
diff --git a/drivers/clk/imx/clk-pllv3.c b/drivers/clk/imx/clk-pllv3.c
index 9af62ee8f347..4110e713d259 100644
--- a/drivers/clk/imx/clk-pllv3.c
+++ b/drivers/clk/imx/clk-pllv3.c
@@ -20,6 +20,8 @@
 
 #define PLL_NUM_OFFSET		0x10
 #define PLL_DENOM_OFFSET	0x20
+#define PLL_IMX7_NUM_OFFSET	0x20
+#define PLL_IMX7_DENOM_OFFSET	0x30
 
 #define PLL_VF610_NUM_OFFSET	0x20
 #define PLL_VF610_DENOM_OFFSET	0x30
@@ -49,6 +51,8 @@ struct clk_pllv3 {
 	u32		div_mask;
 	u32		div_shift;
 	unsigned long	ref_clock;
+	u32		num_offset;
+	u32		denom_offset;
 };
 
 #define to_clk_pllv3(_hw) container_of(_hw, struct clk_pllv3, hw)
@@ -219,8 +223,8 @@ static unsigned long clk_pllv3_av_recalc_rate(struct clk_hw *hw,
 					      unsigned long parent_rate)
 {
 	struct clk_pllv3 *pll = to_clk_pllv3(hw);
-	u32 mfn = readl_relaxed(pll->base + PLL_NUM_OFFSET);
-	u32 mfd = readl_relaxed(pll->base + PLL_DENOM_OFFSET);
+	u32 mfn = readl_relaxed(pll->base + pll->num_offset);
+	u32 mfd = readl_relaxed(pll->base + pll->denom_offset);
 	u32 div = readl_relaxed(pll->base) & pll->div_mask;
 	u64 temp64 = (u64)parent_rate;
 
@@ -289,8 +293,8 @@ static int clk_pllv3_av_set_rate(struct clk_hw *hw, unsigned long rate,
 	val &= ~pll->div_mask;
 	val |= div;
 	writel_relaxed(val, pll->base);
-	writel_relaxed(mfn, pll->base + PLL_NUM_OFFSET);
-	writel_relaxed(mfd, pll->base + PLL_DENOM_OFFSET);
+	writel_relaxed(mfn, pll->base + pll->num_offset);
+	writel_relaxed(mfd, pll->base + pll->denom_offset);
 
 	return clk_pllv3_wait_lock(pll);
 }
@@ -352,8 +356,8 @@ static unsigned long clk_pllv3_vf610_recalc_rate(struct clk_hw *hw,
 	struct clk_pllv3 *pll = to_clk_pllv3(hw);
 	struct clk_pllv3_vf610_mf mf;
 
-	mf.mfn = readl_relaxed(pll->base + PLL_VF610_NUM_OFFSET);
-	mf.mfd = readl_relaxed(pll->base + PLL_VF610_DENOM_OFFSET);
+	mf.mfn = readl_relaxed(pll->base + pll->num_offset);
+	mf.mfd = readl_relaxed(pll->base + pll->denom_offset);
 	mf.mfi = (readl_relaxed(pll->base) & pll->div_mask) ? 22 : 20;
 
 	return clk_pllv3_vf610_mf_to_rate(parent_rate, mf);
@@ -382,8 +386,8 @@ static int clk_pllv3_vf610_set_rate(struct clk_hw *hw, unsigned long rate,
 		val |= pll->div_mask;	/* set bit for mfi=22 */
 	writel_relaxed(val, pll->base);
 
-	writel_relaxed(mf.mfn, pll->base + PLL_VF610_NUM_OFFSET);
-	writel_relaxed(mf.mfd, pll->base + PLL_VF610_DENOM_OFFSET);
+	writel_relaxed(mf.mfn, pll->base + pll->num_offset);
+	writel_relaxed(mf.mfd, pll->base + pll->denom_offset);
 
 	return clk_pllv3_wait_lock(pll);
 }
@@ -426,6 +430,8 @@ struct clk *imx_clk_pllv3(enum imx_pllv3_type type, const char *name,
 		return ERR_PTR(-ENOMEM);
 
 	pll->power_bit = BM_PLL_POWER;
+	pll->num_offset = PLL_NUM_OFFSET;
+	pll->denom_offset = PLL_DENOM_OFFSET;
 
 	switch (type) {
 	case IMX_PLLV3_SYS:
@@ -433,13 +439,20 @@ struct clk *imx_clk_pllv3(enum imx_pllv3_type type, const char *name,
 		break;
 	case IMX_PLLV3_SYS_VF610:
 		ops = &clk_pllv3_vf610_ops;
+		pll->num_offset = PLL_VF610_NUM_OFFSET;
+		pll->denom_offset = PLL_VF610_DENOM_OFFSET;
 		break;
 	case IMX_PLLV3_USB_VF610:
 		pll->div_shift = 1;
+		/* fall through */
 	case IMX_PLLV3_USB:
 		ops = &clk_pllv3_ops;
 		pll->powerup_set = true;
 		break;
+	case IMX_PLLV3_AV_IMX7:
+		pll->num_offset = PLL_IMX7_NUM_OFFSET;
+		pll->denom_offset = PLL_IMX7_DENOM_OFFSET;
+		/* fall through */
 	case IMX_PLLV3_AV:
 		ops = &clk_pllv3_av_ops;
 		break;
@@ -454,6 +467,8 @@ struct clk *imx_clk_pllv3(enum imx_pllv3_type type, const char *name,
 		break;
 	case IMX_PLLV3_DDR_IMX7:
 		pll->power_bit = IMX7_DDR_PLL_POWER;
+		pll->num_offset = PLL_IMX7_NUM_OFFSET;
+		pll->denom_offset = PLL_IMX7_DENOM_OFFSET;
 		ops = &clk_pllv3_av_ops;
 		break;
 	default:
diff --git a/drivers/clk/imx/clk-pllv4.c b/drivers/clk/imx/clk-pllv4.c
index d38bc9f87c1d..d7e62c3620d3 100644
--- a/drivers/clk/imx/clk-pllv4.c
+++ b/drivers/clk/imx/clk-pllv4.c
@@ -30,6 +30,9 @@
 /* PLL Denominator Register (xPLLDENOM) */
 #define PLL_DENOM_OFFSET	0x14
 
+#define MAX_MFD			0x3fffffff
+#define DEFAULT_MFD		1000000
+
 struct clk_pllv4 {
 	struct clk_hw	hw;
 	void __iomem	*base;
@@ -64,13 +67,20 @@ static unsigned long clk_pllv4_recalc_rate(struct clk_hw *hw,
 					   unsigned long parent_rate)
 {
 	struct clk_pllv4 *pll = to_clk_pllv4(hw);
-	u32 div;
+	u32 mult, mfn, mfd;
+	u64 temp64;
+
+	mult = readl_relaxed(pll->base + PLL_CFG_OFFSET);
+	mult &= BM_PLL_MULT;
+	mult >>= BP_PLL_MULT;
 
-	div = readl_relaxed(pll->base + PLL_CFG_OFFSET);
-	div &= BM_PLL_MULT;
-	div >>= BP_PLL_MULT;
+	mfn = readl_relaxed(pll->base + PLL_NUM_OFFSET);
+	mfd = readl_relaxed(pll->base + PLL_DENOM_OFFSET);
+	temp64 = parent_rate;
+	temp64 *= mfn;
+	do_div(temp64, mfd);
 
-	return parent_rate * div;
+	return (parent_rate * mult) + (u32)temp64;
 }
 
 static long clk_pllv4_round_rate(struct clk_hw *hw, unsigned long rate,
@@ -78,14 +88,46 @@ static long clk_pllv4_round_rate(struct clk_hw *hw, unsigned long rate,
 {
 	unsigned long parent_rate = *prate;
 	unsigned long round_rate, i;
+	u32 mfn, mfd = DEFAULT_MFD;
+	bool found = false;
+	u64 temp64;
 
 	for (i = 0; i < ARRAY_SIZE(pllv4_mult_table); i++) {
 		round_rate = parent_rate * pllv4_mult_table[i];
-		if (rate >= round_rate)
-			return round_rate;
+		if (rate >= round_rate) {
+			found = true;
+			break;
+		}
+	}
+
+	if (!found) {
+		pr_warn("%s: unable to round rate %lu, parent rate %lu\n",
+			clk_hw_get_name(hw), rate, parent_rate);
+		return 0;
 	}
 
-	return round_rate;
+	if (parent_rate <= MAX_MFD)
+		mfd = parent_rate;
+
+	temp64 = (u64)(rate - round_rate);
+	temp64 *= mfd;
+	do_div(temp64, parent_rate);
+	mfn = temp64;
+
+	/*
+	 * NOTE: The value of numerator must always be configured to be
+	 * less than the value of the denominator. If we can't get a proper
+	 * pair of mfn/mfd, we simply return the round_rate without using
+	 * the frac part.
+	 */
+	if (mfn >= mfd)
+		return round_rate;
+
+	temp64 = (u64)parent_rate;
+	temp64 *= mfn;
+	do_div(temp64, mfd);
+
+	return round_rate + (u32)temp64;
 }
 
 static bool clk_pllv4_is_valid_mult(unsigned int mult)
@@ -105,18 +147,30 @@ static int clk_pllv4_set_rate(struct clk_hw *hw, unsigned long rate,
 			      unsigned long parent_rate)
 {
 	struct clk_pllv4 *pll = to_clk_pllv4(hw);
-	u32 val, mult;
+	u32 val, mult, mfn, mfd = DEFAULT_MFD;
+	u64 temp64;
 
 	mult = rate / parent_rate;
 
 	if (!clk_pllv4_is_valid_mult(mult))
 		return -EINVAL;
 
+	if (parent_rate <= MAX_MFD)
+		mfd = parent_rate;
+
+	temp64 = (u64)(rate - mult * parent_rate);
+	temp64 *= mfd;
+	do_div(temp64, parent_rate);
+	mfn = temp64;
+
 	val = readl_relaxed(pll->base + PLL_CFG_OFFSET);
 	val &= ~BM_PLL_MULT;
 	val |= mult << BP_PLL_MULT;
 	writel_relaxed(val, pll->base + PLL_CFG_OFFSET);
 
+	writel_relaxed(mfn, pll->base + PLL_NUM_OFFSET);
+	writel_relaxed(mfd, pll->base + PLL_DENOM_OFFSET);
+
 	return 0;
 }
 
diff --git a/drivers/clk/imx/clk-sccg-pll.c b/drivers/clk/imx/clk-sccg-pll.c
index 9dfd03a95557..991bbe63f156 100644
--- a/drivers/clk/imx/clk-sccg-pll.c
+++ b/drivers/clk/imx/clk-sccg-pll.c
@@ -348,7 +348,7 @@ static unsigned long clk_sccg_pll_recalc_rate(struct clk_hw *hw,
 
 	temp64 = parent_rate;
 
-	val = clk_readl(pll->base + PLL_CFG0);
+	val = readl(pll->base + PLL_CFG0);
 	if (val & SSCG_PLL_BYPASS2_MASK) {
 		temp64 = parent_rate;
 	} else if (val & SSCG_PLL_BYPASS1_MASK) {
@@ -371,10 +371,10 @@ static int clk_sccg_pll_set_rate(struct clk_hw *hw, unsigned long rate,
 	u32 val;
 
 	/* set bypass here too since the parent might be the same */
-	val = clk_readl(pll->base + PLL_CFG0);
+	val = readl(pll->base + PLL_CFG0);
 	val &= ~SSCG_PLL_BYPASS_MASK;
 	val |= FIELD_PREP(SSCG_PLL_BYPASS_MASK, setup->bypass);
-	clk_writel(val, pll->base + PLL_CFG0);
+	writel(val, pll->base + PLL_CFG0);
 
 	val = readl_relaxed(pll->base + PLL_CFG2);
 	val &= ~(PLL_DIVF1_MASK | PLL_DIVF2_MASK);
@@ -395,7 +395,7 @@ static u8 clk_sccg_pll_get_parent(struct clk_hw *hw)
 	u32 val;
 	u8 ret = pll->parent;
 
-	val = clk_readl(pll->base + PLL_CFG0);
+	val = readl(pll->base + PLL_CFG0);
 	if (val & SSCG_PLL_BYPASS2_MASK)
 		ret = pll->bypass2;
 	else if (val & SSCG_PLL_BYPASS1_MASK)
@@ -408,10 +408,10 @@ static int clk_sccg_pll_set_parent(struct clk_hw *hw, u8 index)
 	struct clk_sccg_pll *pll = to_clk_sccg_pll(hw);
 	u32 val;
 
-	val = clk_readl(pll->base + PLL_CFG0);
+	val = readl(pll->base + PLL_CFG0);
 	val &= ~SSCG_PLL_BYPASS_MASK;
 	val |= FIELD_PREP(SSCG_PLL_BYPASS_MASK, pll->setup.bypass);
-	clk_writel(val, pll->base + PLL_CFG0);
+	writel(val, pll->base + PLL_CFG0);
 
 	return clk_sccg_pll_wait_lock(pll);
 }
diff --git a/drivers/clk/imx/clk.h b/drivers/clk/imx/clk.h
index 5748ec8673e4..8639a8f2153e 100644
--- a/drivers/clk/imx/clk.h
+++ b/drivers/clk/imx/clk.h
@@ -77,6 +77,7 @@ enum imx_pllv3_type {
 	IMX_PLLV3_ENET_IMX7,
 	IMX_PLLV3_SYS_VF610,
 	IMX_PLLV3_DDR_IMX7,
+	IMX_PLLV3_AV_IMX7,
 };
 
 struct clk *imx_clk_pllv3(enum imx_pllv3_type type, const char *name,
@@ -138,11 +139,6 @@ static inline struct clk_hw *imx_clk_hw_fixed(const char *name, int rate)
 	return clk_hw_register_fixed_rate(NULL, name, NULL, 0, rate);
 }
 
-static inline struct clk_hw *imx_get_clk_hw_fixed(const char *name, int rate)
-{
-	return clk_hw_register_fixed_rate(NULL, name, NULL, 0, rate);
-}
-
 static inline struct clk *imx_clk_mux_ldb(const char *name, void __iomem *reg,
 			u8 shift, u8 width, const char * const *parents,
 			int num_parents)
diff --git a/drivers/clk/ingenic/jz4725b-cgu.c b/drivers/clk/ingenic/jz4725b-cgu.c
index 584ff4ff81c7..8901ea0295b7 100644
--- a/drivers/clk/ingenic/jz4725b-cgu.c
+++ b/drivers/clk/ingenic/jz4725b-cgu.c
@@ -205,6 +205,12 @@ static const struct ingenic_cgu_clk_info jz4725b_cgu_clocks[] = {
 		.parents = { JZ4725B_CLK_EXT512, JZ4725B_CLK_OSC32K, -1, -1 },
 		.mux = { CGU_REG_OPCR, 2, 1},
 	},
+
+	[JZ4725B_CLK_UDC_PHY] = {
+		"udc_phy", CGU_CLK_GATE,
+		.parents = { JZ4725B_CLK_EXT, -1, -1, -1 },
+		.gate = { CGU_REG_OPCR, 6, true },
+	},
 };
 
 static void __init jz4725b_cgu_init(struct device_node *np)
diff --git a/drivers/clk/mediatek/Kconfig b/drivers/clk/mediatek/Kconfig
index 53edade25a1d..4d8a9aef95f6 100644
--- a/drivers/clk/mediatek/Kconfig
+++ b/drivers/clk/mediatek/Kconfig
@@ -216,4 +216,87 @@ config COMMON_CLK_MT8173
 	default ARCH_MEDIATEK
 	---help---
 	  This driver supports MediaTek MT8173 clocks.
+
+config COMMON_CLK_MT8183
+	bool "Clock driver for MediaTek MT8183"
+	depends on (ARCH_MEDIATEK && ARM64) || COMPILE_TEST
+	select COMMON_CLK_MEDIATEK
+	default ARCH_MEDIATEK && ARM64
+	help
+	  This driver supports MediaTek MT8183 basic clocks.
+
+config COMMON_CLK_MT8183_AUDIOSYS
+	bool "Clock driver for MediaTek MT8183 audiosys"
+	depends on COMMON_CLK_MT8183
+	help
+	  This driver supports MediaTek MT8183 audiosys clocks.
+
+config COMMON_CLK_MT8183_CAMSYS
+	bool "Clock driver for MediaTek MT8183 camsys"
+	depends on COMMON_CLK_MT8183
+	help
+	  This driver supports MediaTek MT8183 camsys clocks.
+
+config COMMON_CLK_MT8183_IMGSYS
+	bool "Clock driver for MediaTek MT8183 imgsys"
+	depends on COMMON_CLK_MT8183
+	help
+	  This driver supports MediaTek MT8183 imgsys clocks.
+
+config COMMON_CLK_MT8183_IPU_CORE0
+	bool "Clock driver for MediaTek MT8183 ipu_core0"
+	depends on COMMON_CLK_MT8183
+	help
+	  This driver supports MediaTek MT8183 ipu_core0 clocks.
+
+config COMMON_CLK_MT8183_IPU_CORE1
+	bool "Clock driver for MediaTek MT8183 ipu_core1"
+	depends on COMMON_CLK_MT8183
+	help
+	  This driver supports MediaTek MT8183 ipu_core1 clocks.
+
+config COMMON_CLK_MT8183_IPU_ADL
+	bool "Clock driver for MediaTek MT8183 ipu_adl"
+	depends on COMMON_CLK_MT8183
+	help
+	  This driver supports MediaTek MT8183 ipu_adl clocks.
+
+config COMMON_CLK_MT8183_IPU_CONN
+	bool "Clock driver for MediaTek MT8183 ipu_conn"
+	depends on COMMON_CLK_MT8183
+	help
+	  This driver supports MediaTek MT8183 ipu_conn clocks.
+
+config COMMON_CLK_MT8183_MFGCFG
+	bool "Clock driver for MediaTek MT8183 mfgcfg"
+	depends on COMMON_CLK_MT8183
+	help
+	  This driver supports MediaTek MT8183 mfgcfg clocks.
+
+config COMMON_CLK_MT8183_MMSYS
+	bool "Clock driver for MediaTek MT8183 mmsys"
+	depends on COMMON_CLK_MT8183
+	help
+	  This driver supports MediaTek MT8183 mmsys clocks.
+
+config COMMON_CLK_MT8183_VDECSYS
+	bool "Clock driver for MediaTek MT8183 vdecsys"
+	depends on COMMON_CLK_MT8183
+	help
+	  This driver supports MediaTek MT8183 vdecsys clocks.
+
+config COMMON_CLK_MT8183_VENCSYS
+	bool "Clock driver for MediaTek MT8183 vencsys"
+	depends on COMMON_CLK_MT8183
+	help
+	  This driver supports MediaTek MT8183 vencsys clocks.
+
+config COMMON_CLK_MT8516
+	bool "Clock driver for MediaTek MT8516"
+	depends on ARCH_MEDIATEK || COMPILE_TEST
+	select COMMON_CLK_MEDIATEK
+	default ARCH_MEDIATEK
+	help
+	  This driver supports MediaTek MT8516 clocks.
+
 endmenu
diff --git a/drivers/clk/mediatek/Makefile b/drivers/clk/mediatek/Makefile
index ee4410ff43ab..f74937b35f68 100644
--- a/drivers/clk/mediatek/Makefile
+++ b/drivers/clk/mediatek/Makefile
@@ -1,5 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0
-obj-$(CONFIG_COMMON_CLK_MEDIATEK) += clk-mtk.o clk-pll.o clk-gate.o clk-apmixed.o clk-cpumux.o reset.o
+obj-$(CONFIG_COMMON_CLK_MEDIATEK) += clk-mtk.o clk-pll.o clk-gate.o clk-apmixed.o clk-cpumux.o reset.o clk-mux.o
+
 obj-$(CONFIG_COMMON_CLK_MT6797) += clk-mt6797.o
 obj-$(CONFIG_COMMON_CLK_MT6797_IMGSYS) += clk-mt6797-img.o
 obj-$(CONFIG_COMMON_CLK_MT6797_MMSYS) += clk-mt6797-mm.o
@@ -31,3 +32,16 @@ obj-$(CONFIG_COMMON_CLK_MT7629_ETHSYS) += clk-mt7629-eth.o
 obj-$(CONFIG_COMMON_CLK_MT7629_HIFSYS) += clk-mt7629-hif.o
 obj-$(CONFIG_COMMON_CLK_MT8135) += clk-mt8135.o
 obj-$(CONFIG_COMMON_CLK_MT8173) += clk-mt8173.o
+obj-$(CONFIG_COMMON_CLK_MT8183) += clk-mt8183.o
+obj-$(CONFIG_COMMON_CLK_MT8183_AUDIOSYS) += clk-mt8183-audio.o
+obj-$(CONFIG_COMMON_CLK_MT8183_CAMSYS) += clk-mt8183-cam.o
+obj-$(CONFIG_COMMON_CLK_MT8183_IMGSYS) += clk-mt8183-img.o
+obj-$(CONFIG_COMMON_CLK_MT8183_IPU_CORE0) += clk-mt8183-ipu0.o
+obj-$(CONFIG_COMMON_CLK_MT8183_IPU_CORE1) += clk-mt8183-ipu1.o
+obj-$(CONFIG_COMMON_CLK_MT8183_IPU_ADL) += clk-mt8183-ipu_adl.o
+obj-$(CONFIG_COMMON_CLK_MT8183_IPU_CONN) += clk-mt8183-ipu_conn.o
+obj-$(CONFIG_COMMON_CLK_MT8183_MFGCFG) += clk-mt8183-mfgcfg.o
+obj-$(CONFIG_COMMON_CLK_MT8183_MMSYS) += clk-mt8183-mm.o
+obj-$(CONFIG_COMMON_CLK_MT8183_VDECSYS) += clk-mt8183-vdec.o
+obj-$(CONFIG_COMMON_CLK_MT8183_VENCSYS) += clk-mt8183-venc.o
+obj-$(CONFIG_COMMON_CLK_MT8516) += clk-mt8516.o
diff --git a/drivers/clk/mediatek/clk-gate.h b/drivers/clk/mediatek/clk-gate.h
index 9f766dfe1d57..ab240163f9f8 100644
--- a/drivers/clk/mediatek/clk-gate.h
+++ b/drivers/clk/mediatek/clk-gate.h
@@ -50,4 +50,18 @@ struct clk *mtk_clk_register_gate(
 		const struct clk_ops *ops,
 		unsigned long flags);
 
+#define GATE_MTK_FLAGS(_id, _name, _parent, _regs, _shift,	\
+			_ops, _flags) {				\
+		.id = _id,					\
+		.name = _name,					\
+		.parent_name = _parent,				\
+		.regs = _regs,					\
+		.shift = _shift,				\
+		.ops = _ops,					\
+		.flags = _flags,				\
+	}
+
+#define GATE_MTK(_id, _name, _parent, _regs, _shift, _ops)		\
+	GATE_MTK_FLAGS(_id, _name, _parent, _regs, _shift, _ops, 0)
+
 #endif /* __DRV_CLK_GATE_H */
diff --git a/drivers/clk/mediatek/clk-mt8183-audio.c b/drivers/clk/mediatek/clk-mt8183-audio.c
new file mode 100644
index 000000000000..c87450180b7b
--- /dev/null
+++ b/drivers/clk/mediatek/clk-mt8183-audio.c
@@ -0,0 +1,105 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2018 MediaTek Inc.
+// Author: Weiyi Lu <weiyi.lu@mediatek.com>
+
+#include <linux/clk-provider.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+
+#include "clk-mtk.h"
+#include "clk-gate.h"
+
+#include <dt-bindings/clock/mt8183-clk.h>
+
+static const struct mtk_gate_regs audio0_cg_regs = {
+	.set_ofs = 0x0,
+	.clr_ofs = 0x0,
+	.sta_ofs = 0x0,
+};
+
+static const struct mtk_gate_regs audio1_cg_regs = {
+	.set_ofs = 0x4,
+	.clr_ofs = 0x4,
+	.sta_ofs = 0x4,
+};
+
+#define GATE_AUDIO0(_id, _name, _parent, _shift)		\
+	GATE_MTK(_id, _name, _parent, &audio0_cg_regs, _shift,	\
+		&mtk_clk_gate_ops_no_setclr)
+
+#define GATE_AUDIO1(_id, _name, _parent, _shift)		\
+	GATE_MTK(_id, _name, _parent, &audio1_cg_regs, _shift,	\
+		&mtk_clk_gate_ops_no_setclr)
+
+static const struct mtk_gate audio_clks[] = {
+	/* AUDIO0 */
+	GATE_AUDIO0(CLK_AUDIO_AFE, "aud_afe", "audio_sel",
+		2),
+	GATE_AUDIO0(CLK_AUDIO_22M, "aud_22m", "aud_eng1_sel",
+		8),
+	GATE_AUDIO0(CLK_AUDIO_24M, "aud_24m", "aud_eng2_sel",
+		9),
+	GATE_AUDIO0(CLK_AUDIO_APLL2_TUNER, "aud_apll2_tuner", "aud_eng2_sel",
+		18),
+	GATE_AUDIO0(CLK_AUDIO_APLL_TUNER, "aud_apll_tuner", "aud_eng1_sel",
+		19),
+	GATE_AUDIO0(CLK_AUDIO_TDM, "aud_tdm", "apll12_divb",
+		20),
+	GATE_AUDIO0(CLK_AUDIO_ADC, "aud_adc", "audio_sel",
+		24),
+	GATE_AUDIO0(CLK_AUDIO_DAC, "aud_dac", "audio_sel",
+		25),
+	GATE_AUDIO0(CLK_AUDIO_DAC_PREDIS, "aud_dac_predis", "audio_sel",
+		26),
+	GATE_AUDIO0(CLK_AUDIO_TML, "aud_tml", "audio_sel",
+		27),
+	/* AUDIO1 */
+	GATE_AUDIO1(CLK_AUDIO_I2S1, "aud_i2s1", "audio_sel",
+		4),
+	GATE_AUDIO1(CLK_AUDIO_I2S2, "aud_i2s2", "audio_sel",
+		5),
+	GATE_AUDIO1(CLK_AUDIO_I2S3, "aud_i2s3", "audio_sel",
+		6),
+	GATE_AUDIO1(CLK_AUDIO_I2S4, "aud_i2s4", "audio_sel",
+		7),
+	GATE_AUDIO1(CLK_AUDIO_PDN_ADDA6_ADC, "aud_pdn_adda6_adc", "audio_sel",
+		20),
+};
+
+static int clk_mt8183_audio_probe(struct platform_device *pdev)
+{
+	struct clk_onecell_data *clk_data;
+	int r;
+	struct device_node *node = pdev->dev.of_node;
+
+	clk_data = mtk_alloc_clk_data(CLK_AUDIO_NR_CLK);
+
+	mtk_clk_register_gates(node, audio_clks, ARRAY_SIZE(audio_clks),
+			clk_data);
+
+	r = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+	if (r)
+		return r;
+
+	r = devm_of_platform_populate(&pdev->dev);
+	if (r)
+		of_clk_del_provider(node);
+
+	return r;
+}
+
+static const struct of_device_id of_match_clk_mt8183_audio[] = {
+	{ .compatible = "mediatek,mt8183-audiosys", },
+	{}
+};
+
+static struct platform_driver clk_mt8183_audio_drv = {
+	.probe = clk_mt8183_audio_probe,
+	.driver = {
+		.name = "clk-mt8183-audio",
+		.of_match_table = of_match_clk_mt8183_audio,
+	},
+};
+
+builtin_platform_driver(clk_mt8183_audio_drv);
diff --git a/drivers/clk/mediatek/clk-mt8183-cam.c b/drivers/clk/mediatek/clk-mt8183-cam.c
new file mode 100644
index 000000000000..8643802c4471
--- /dev/null
+++ b/drivers/clk/mediatek/clk-mt8183-cam.c
@@ -0,0 +1,63 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2018 MediaTek Inc.
+// Author: Weiyi Lu <weiyi.lu@mediatek.com>
+
+#include <linux/clk-provider.h>
+#include <linux/platform_device.h>
+
+#include "clk-mtk.h"
+#include "clk-gate.h"
+
+#include <dt-bindings/clock/mt8183-clk.h>
+
+static const struct mtk_gate_regs cam_cg_regs = {
+	.set_ofs = 0x4,
+	.clr_ofs = 0x8,
+	.sta_ofs = 0x0,
+};
+
+#define GATE_CAM(_id, _name, _parent, _shift)			\
+	GATE_MTK(_id, _name, _parent, &cam_cg_regs, _shift,	\
+		&mtk_clk_gate_ops_setclr)
+
+static const struct mtk_gate cam_clks[] = {
+	GATE_CAM(CLK_CAM_LARB6, "cam_larb6", "cam_sel", 0),
+	GATE_CAM(CLK_CAM_DFP_VAD, "cam_dfp_vad", "cam_sel", 1),
+	GATE_CAM(CLK_CAM_LARB3, "cam_larb3", "cam_sel", 2),
+	GATE_CAM(CLK_CAM_CAM, "cam_cam", "cam_sel", 6),
+	GATE_CAM(CLK_CAM_CAMTG, "cam_camtg", "cam_sel", 7),
+	GATE_CAM(CLK_CAM_SENINF, "cam_seninf", "cam_sel", 8),
+	GATE_CAM(CLK_CAM_CAMSV0, "cam_camsv0", "cam_sel", 9),
+	GATE_CAM(CLK_CAM_CAMSV1, "cam_camsv1", "cam_sel", 10),
+	GATE_CAM(CLK_CAM_CAMSV2, "cam_camsv2", "cam_sel", 11),
+	GATE_CAM(CLK_CAM_CCU, "cam_ccu", "cam_sel", 12),
+};
+
+static int clk_mt8183_cam_probe(struct platform_device *pdev)
+{
+	struct clk_onecell_data *clk_data;
+	struct device_node *node = pdev->dev.of_node;
+
+	clk_data = mtk_alloc_clk_data(CLK_CAM_NR_CLK);
+
+	mtk_clk_register_gates(node, cam_clks, ARRAY_SIZE(cam_clks),
+			clk_data);
+
+	return of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+}
+
+static const struct of_device_id of_match_clk_mt8183_cam[] = {
+	{ .compatible = "mediatek,mt8183-camsys", },
+	{}
+};
+
+static struct platform_driver clk_mt8183_cam_drv = {
+	.probe = clk_mt8183_cam_probe,
+	.driver = {
+		.name = "clk-mt8183-cam",
+		.of_match_table = of_match_clk_mt8183_cam,
+	},
+};
+
+builtin_platform_driver(clk_mt8183_cam_drv);
diff --git a/drivers/clk/mediatek/clk-mt8183-img.c b/drivers/clk/mediatek/clk-mt8183-img.c
new file mode 100644
index 000000000000..470d676a4a10
--- /dev/null
+++ b/drivers/clk/mediatek/clk-mt8183-img.c
@@ -0,0 +1,63 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2018 MediaTek Inc.
+// Author: Weiyi Lu <weiyi.lu@mediatek.com>
+
+#include <linux/clk-provider.h>
+#include <linux/platform_device.h>
+
+#include "clk-mtk.h"
+#include "clk-gate.h"
+
+#include <dt-bindings/clock/mt8183-clk.h>
+
+static const struct mtk_gate_regs img_cg_regs = {
+	.set_ofs = 0x4,
+	.clr_ofs = 0x8,
+	.sta_ofs = 0x0,
+};
+
+#define GATE_IMG(_id, _name, _parent, _shift)			\
+	GATE_MTK(_id, _name, _parent, &img_cg_regs, _shift,	\
+		&mtk_clk_gate_ops_setclr)
+
+static const struct mtk_gate img_clks[] = {
+	GATE_IMG(CLK_IMG_LARB5, "img_larb5", "img_sel", 0),
+	GATE_IMG(CLK_IMG_LARB2, "img_larb2", "img_sel", 1),
+	GATE_IMG(CLK_IMG_DIP, "img_dip", "img_sel", 2),
+	GATE_IMG(CLK_IMG_FDVT, "img_fdvt", "img_sel", 3),
+	GATE_IMG(CLK_IMG_DPE, "img_dpe", "img_sel", 4),
+	GATE_IMG(CLK_IMG_RSC, "img_rsc", "img_sel", 5),
+	GATE_IMG(CLK_IMG_MFB, "img_mfb", "img_sel", 6),
+	GATE_IMG(CLK_IMG_WPE_A, "img_wpe_a", "img_sel", 7),
+	GATE_IMG(CLK_IMG_WPE_B, "img_wpe_b", "img_sel", 8),
+	GATE_IMG(CLK_IMG_OWE, "img_owe", "img_sel", 9),
+};
+
+static int clk_mt8183_img_probe(struct platform_device *pdev)
+{
+	struct clk_onecell_data *clk_data;
+	struct device_node *node = pdev->dev.of_node;
+
+	clk_data = mtk_alloc_clk_data(CLK_IMG_NR_CLK);
+
+	mtk_clk_register_gates(node, img_clks, ARRAY_SIZE(img_clks),
+			clk_data);
+
+	return of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+}
+
+static const struct of_device_id of_match_clk_mt8183_img[] = {
+	{ .compatible = "mediatek,mt8183-imgsys", },
+	{}
+};
+
+static struct platform_driver clk_mt8183_img_drv = {
+	.probe = clk_mt8183_img_probe,
+	.driver = {
+		.name = "clk-mt8183-img",
+		.of_match_table = of_match_clk_mt8183_img,
+	},
+};
+
+builtin_platform_driver(clk_mt8183_img_drv);
diff --git a/drivers/clk/mediatek/clk-mt8183-ipu0.c b/drivers/clk/mediatek/clk-mt8183-ipu0.c
new file mode 100644
index 000000000000..c5cb76fc9e5e
--- /dev/null
+++ b/drivers/clk/mediatek/clk-mt8183-ipu0.c
@@ -0,0 +1,56 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2018 MediaTek Inc.
+// Author: Weiyi Lu <weiyi.lu@mediatek.com>
+
+#include <linux/clk-provider.h>
+#include <linux/platform_device.h>
+
+#include "clk-mtk.h"
+#include "clk-gate.h"
+
+#include <dt-bindings/clock/mt8183-clk.h>
+
+static const struct mtk_gate_regs ipu_core0_cg_regs = {
+	.set_ofs = 0x4,
+	.clr_ofs = 0x8,
+	.sta_ofs = 0x0,
+};
+
+#define GATE_IPU_CORE0(_id, _name, _parent, _shift)			\
+	GATE_MTK(_id, _name, _parent, &ipu_core0_cg_regs, _shift,	\
+		&mtk_clk_gate_ops_setclr)
+
+static const struct mtk_gate ipu_core0_clks[] = {
+	GATE_IPU_CORE0(CLK_IPU_CORE0_JTAG, "ipu_core0_jtag", "dsp_sel", 0),
+	GATE_IPU_CORE0(CLK_IPU_CORE0_AXI, "ipu_core0_axi", "dsp_sel", 1),
+	GATE_IPU_CORE0(CLK_IPU_CORE0_IPU, "ipu_core0_ipu", "dsp_sel", 2),
+};
+
+static int clk_mt8183_ipu_core0_probe(struct platform_device *pdev)
+{
+	struct clk_onecell_data *clk_data;
+	struct device_node *node = pdev->dev.of_node;
+
+	clk_data = mtk_alloc_clk_data(CLK_IPU_CORE0_NR_CLK);
+
+	mtk_clk_register_gates(node, ipu_core0_clks, ARRAY_SIZE(ipu_core0_clks),
+			clk_data);
+
+	return of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+}
+
+static const struct of_device_id of_match_clk_mt8183_ipu_core0[] = {
+	{ .compatible = "mediatek,mt8183-ipu_core0", },
+	{}
+};
+
+static struct platform_driver clk_mt8183_ipu_core0_drv = {
+	.probe = clk_mt8183_ipu_core0_probe,
+	.driver = {
+		.name = "clk-mt8183-ipu_core0",
+		.of_match_table = of_match_clk_mt8183_ipu_core0,
+	},
+};
+
+builtin_platform_driver(clk_mt8183_ipu_core0_drv);
diff --git a/drivers/clk/mediatek/clk-mt8183-ipu1.c b/drivers/clk/mediatek/clk-mt8183-ipu1.c
new file mode 100644
index 000000000000..8fd5fe002890
--- /dev/null
+++ b/drivers/clk/mediatek/clk-mt8183-ipu1.c
@@ -0,0 +1,56 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2018 MediaTek Inc.
+// Author: Weiyi Lu <weiyi.lu@mediatek.com>
+
+#include <linux/clk-provider.h>
+#include <linux/platform_device.h>
+
+#include "clk-mtk.h"
+#include "clk-gate.h"
+
+#include <dt-bindings/clock/mt8183-clk.h>
+
+static const struct mtk_gate_regs ipu_core1_cg_regs = {
+	.set_ofs = 0x4,
+	.clr_ofs = 0x8,
+	.sta_ofs = 0x0,
+};
+
+#define GATE_IPU_CORE1(_id, _name, _parent, _shift)			\
+	GATE_MTK(_id, _name, _parent, &ipu_core1_cg_regs, _shift,	\
+		&mtk_clk_gate_ops_setclr)
+
+static const struct mtk_gate ipu_core1_clks[] = {
+	GATE_IPU_CORE1(CLK_IPU_CORE1_JTAG, "ipu_core1_jtag", "dsp_sel", 0),
+	GATE_IPU_CORE1(CLK_IPU_CORE1_AXI, "ipu_core1_axi", "dsp_sel", 1),
+	GATE_IPU_CORE1(CLK_IPU_CORE1_IPU, "ipu_core1_ipu", "dsp_sel", 2),
+};
+
+static int clk_mt8183_ipu_core1_probe(struct platform_device *pdev)
+{
+	struct clk_onecell_data *clk_data;
+	struct device_node *node = pdev->dev.of_node;
+
+	clk_data = mtk_alloc_clk_data(CLK_IPU_CORE1_NR_CLK);
+
+	mtk_clk_register_gates(node, ipu_core1_clks, ARRAY_SIZE(ipu_core1_clks),
+			clk_data);
+
+	return of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+}
+
+static const struct of_device_id of_match_clk_mt8183_ipu_core1[] = {
+	{ .compatible = "mediatek,mt8183-ipu_core1", },
+	{}
+};
+
+static struct platform_driver clk_mt8183_ipu_core1_drv = {
+	.probe = clk_mt8183_ipu_core1_probe,
+	.driver = {
+		.name = "clk-mt8183-ipu_core1",
+		.of_match_table = of_match_clk_mt8183_ipu_core1,
+	},
+};
+
+builtin_platform_driver(clk_mt8183_ipu_core1_drv);
diff --git a/drivers/clk/mediatek/clk-mt8183-ipu_adl.c b/drivers/clk/mediatek/clk-mt8183-ipu_adl.c
new file mode 100644
index 000000000000..3f37d0ef1df1
--- /dev/null
+++ b/drivers/clk/mediatek/clk-mt8183-ipu_adl.c
@@ -0,0 +1,54 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2018 MediaTek Inc.
+// Author: Weiyi Lu <weiyi.lu@mediatek.com>
+
+#include <linux/clk-provider.h>
+#include <linux/platform_device.h>
+
+#include "clk-mtk.h"
+#include "clk-gate.h"
+
+#include <dt-bindings/clock/mt8183-clk.h>
+
+static const struct mtk_gate_regs ipu_adl_cg_regs = {
+	.set_ofs = 0x204,
+	.clr_ofs = 0x204,
+	.sta_ofs = 0x204,
+};
+
+#define GATE_IPU_ADL_I(_id, _name, _parent, _shift)		\
+	GATE_MTK(_id, _name, _parent, &ipu_adl_cg_regs, _shift,	\
+		&mtk_clk_gate_ops_no_setclr_inv)
+
+static const struct mtk_gate ipu_adl_clks[] = {
+	GATE_IPU_ADL_I(CLK_IPU_ADL_CABGEN, "ipu_adl_cabgen", "dsp_sel", 24),
+};
+
+static int clk_mt8183_ipu_adl_probe(struct platform_device *pdev)
+{
+	struct clk_onecell_data *clk_data;
+	struct device_node *node = pdev->dev.of_node;
+
+	clk_data = mtk_alloc_clk_data(CLK_IPU_ADL_NR_CLK);
+
+	mtk_clk_register_gates(node, ipu_adl_clks, ARRAY_SIZE(ipu_adl_clks),
+			clk_data);
+
+	return of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+}
+
+static const struct of_device_id of_match_clk_mt8183_ipu_adl[] = {
+	{ .compatible = "mediatek,mt8183-ipu_adl", },
+	{}
+};
+
+static struct platform_driver clk_mt8183_ipu_adl_drv = {
+	.probe = clk_mt8183_ipu_adl_probe,
+	.driver = {
+		.name = "clk-mt8183-ipu_adl",
+		.of_match_table = of_match_clk_mt8183_ipu_adl,
+	},
+};
+
+builtin_platform_driver(clk_mt8183_ipu_adl_drv);
diff --git a/drivers/clk/mediatek/clk-mt8183-ipu_conn.c b/drivers/clk/mediatek/clk-mt8183-ipu_conn.c
new file mode 100644
index 000000000000..7e0eef79c461
--- /dev/null
+++ b/drivers/clk/mediatek/clk-mt8183-ipu_conn.c
@@ -0,0 +1,123 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2018 MediaTek Inc.
+// Author: Weiyi Lu <weiyi.lu@mediatek.com>
+
+#include <linux/clk-provider.h>
+#include <linux/platform_device.h>
+
+#include "clk-mtk.h"
+#include "clk-gate.h"
+
+#include <dt-bindings/clock/mt8183-clk.h>
+
+static const struct mtk_gate_regs ipu_conn_cg_regs = {
+	.set_ofs = 0x4,
+	.clr_ofs = 0x8,
+	.sta_ofs = 0x0,
+};
+
+static const struct mtk_gate_regs ipu_conn_apb_cg_regs = {
+	.set_ofs = 0x10,
+	.clr_ofs = 0x10,
+	.sta_ofs = 0x10,
+};
+
+static const struct mtk_gate_regs ipu_conn_axi_cg_regs = {
+	.set_ofs = 0x18,
+	.clr_ofs = 0x18,
+	.sta_ofs = 0x18,
+};
+
+static const struct mtk_gate_regs ipu_conn_axi1_cg_regs = {
+	.set_ofs = 0x1c,
+	.clr_ofs = 0x1c,
+	.sta_ofs = 0x1c,
+};
+
+static const struct mtk_gate_regs ipu_conn_axi2_cg_regs = {
+	.set_ofs = 0x20,
+	.clr_ofs = 0x20,
+	.sta_ofs = 0x20,
+};
+
+#define GATE_IPU_CONN(_id, _name, _parent, _shift)			\
+	GATE_MTK(_id, _name, _parent, &ipu_conn_cg_regs, _shift,	\
+		&mtk_clk_gate_ops_setclr)
+
+#define GATE_IPU_CONN_APB(_id, _name, _parent, _shift)			\
+	GATE_MTK(_id, _name, _parent, &ipu_conn_apb_cg_regs, _shift,	\
+		&mtk_clk_gate_ops_no_setclr)
+
+#define GATE_IPU_CONN_AXI_I(_id, _name, _parent, _shift)		\
+	GATE_MTK(_id, _name, _parent, &ipu_conn_axi_cg_regs, _shift,	\
+		&mtk_clk_gate_ops_no_setclr_inv)
+
+#define GATE_IPU_CONN_AXI1_I(_id, _name, _parent, _shift)		\
+	GATE_MTK(_id, _name, _parent, &ipu_conn_axi1_cg_regs, _shift,	\
+		&mtk_clk_gate_ops_no_setclr_inv)
+
+#define GATE_IPU_CONN_AXI2_I(_id, _name, _parent, _shift)		\
+	GATE_MTK(_id, _name, _parent, &ipu_conn_axi2_cg_regs, _shift,	\
+		&mtk_clk_gate_ops_no_setclr_inv)
+
+static const struct mtk_gate ipu_conn_clks[] = {
+	GATE_IPU_CONN(CLK_IPU_CONN_IPU,
+		"ipu_conn_ipu", "dsp_sel", 0),
+	GATE_IPU_CONN(CLK_IPU_CONN_AHB,
+		"ipu_conn_ahb", "dsp_sel", 1),
+	GATE_IPU_CONN(CLK_IPU_CONN_AXI,
+		"ipu_conn_axi", "dsp_sel", 2),
+	GATE_IPU_CONN(CLK_IPU_CONN_ISP,
+		"ipu_conn_isp", "dsp_sel", 3),
+	GATE_IPU_CONN(CLK_IPU_CONN_CAM_ADL,
+		"ipu_conn_cam_adl", "dsp_sel", 4),
+	GATE_IPU_CONN(CLK_IPU_CONN_IMG_ADL,
+		"ipu_conn_img_adl", "dsp_sel", 5),
+	GATE_IPU_CONN_APB(CLK_IPU_CONN_DAP_RX,
+		"ipu_conn_dap_rx", "dsp1_sel", 0),
+	GATE_IPU_CONN_APB(CLK_IPU_CONN_APB2AXI,
+		"ipu_conn_apb2axi", "dsp1_sel", 3),
+	GATE_IPU_CONN_APB(CLK_IPU_CONN_APB2AHB,
+		"ipu_conn_apb2ahb", "dsp1_sel", 20),
+	GATE_IPU_CONN_AXI_I(CLK_IPU_CONN_IPU_CAB1TO2,
+		"ipu_conn_ipu_cab1to2", "dsp1_sel", 6),
+	GATE_IPU_CONN_AXI_I(CLK_IPU_CONN_IPU1_CAB1TO2,
+		"ipu_conn_ipu1_cab1to2", "dsp1_sel", 13),
+	GATE_IPU_CONN_AXI_I(CLK_IPU_CONN_IPU2_CAB1TO2,
+		"ipu_conn_ipu2_cab1to2", "dsp1_sel", 20),
+	GATE_IPU_CONN_AXI1_I(CLK_IPU_CONN_CAB3TO3,
+		"ipu_conn_cab3to3", "dsp1_sel", 0),
+	GATE_IPU_CONN_AXI2_I(CLK_IPU_CONN_CAB2TO1,
+		"ipu_conn_cab2to1", "dsp1_sel", 14),
+	GATE_IPU_CONN_AXI2_I(CLK_IPU_CONN_CAB3TO1_SLICE,
+		"ipu_conn_cab3to1_slice", "dsp1_sel", 17),
+};
+
+static int clk_mt8183_ipu_conn_probe(struct platform_device *pdev)
+{
+	struct clk_onecell_data *clk_data;
+	struct device_node *node = pdev->dev.of_node;
+
+	clk_data = mtk_alloc_clk_data(CLK_IPU_CONN_NR_CLK);
+
+	mtk_clk_register_gates(node, ipu_conn_clks, ARRAY_SIZE(ipu_conn_clks),
+			clk_data);
+
+	return of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+}
+
+static const struct of_device_id of_match_clk_mt8183_ipu_conn[] = {
+	{ .compatible = "mediatek,mt8183-ipu_conn", },
+	{}
+};
+
+static struct platform_driver clk_mt8183_ipu_conn_drv = {
+	.probe = clk_mt8183_ipu_conn_probe,
+	.driver = {
+		.name = "clk-mt8183-ipu_conn",
+		.of_match_table = of_match_clk_mt8183_ipu_conn,
+	},
+};
+
+builtin_platform_driver(clk_mt8183_ipu_conn_drv);
diff --git a/drivers/clk/mediatek/clk-mt8183-mfgcfg.c b/drivers/clk/mediatek/clk-mt8183-mfgcfg.c
new file mode 100644
index 000000000000..99a6b020833e
--- /dev/null
+++ b/drivers/clk/mediatek/clk-mt8183-mfgcfg.c
@@ -0,0 +1,54 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2018 MediaTek Inc.
+// Author: Weiyi Lu <weiyi.lu@mediatek.com>
+
+#include <linux/clk-provider.h>
+#include <linux/platform_device.h>
+
+#include "clk-mtk.h"
+#include "clk-gate.h"
+
+#include <dt-bindings/clock/mt8183-clk.h>
+
+static const struct mtk_gate_regs mfg_cg_regs = {
+	.set_ofs = 0x4,
+	.clr_ofs = 0x8,
+	.sta_ofs = 0x0,
+};
+
+#define GATE_MFG(_id, _name, _parent, _shift)			\
+	GATE_MTK(_id, _name, _parent, &mfg_cg_regs, _shift,	\
+		&mtk_clk_gate_ops_setclr)
+
+static const struct mtk_gate mfg_clks[] = {
+	GATE_MFG(CLK_MFG_BG3D, "mfg_bg3d", "mfg_sel", 0)
+};
+
+static int clk_mt8183_mfg_probe(struct platform_device *pdev)
+{
+	struct clk_onecell_data *clk_data;
+	struct device_node *node = pdev->dev.of_node;
+
+	clk_data = mtk_alloc_clk_data(CLK_MFG_NR_CLK);
+
+	mtk_clk_register_gates(node, mfg_clks, ARRAY_SIZE(mfg_clks),
+			clk_data);
+
+	return of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+}
+
+static const struct of_device_id of_match_clk_mt8183_mfg[] = {
+	{ .compatible = "mediatek,mt8183-mfgcfg", },
+	{}
+};
+
+static struct platform_driver clk_mt8183_mfg_drv = {
+	.probe = clk_mt8183_mfg_probe,
+	.driver = {
+		.name = "clk-mt8183-mfg",
+		.of_match_table = of_match_clk_mt8183_mfg,
+	},
+};
+
+builtin_platform_driver(clk_mt8183_mfg_drv);
diff --git a/drivers/clk/mediatek/clk-mt8183-mm.c b/drivers/clk/mediatek/clk-mt8183-mm.c
new file mode 100644
index 000000000000..720c696b506d
--- /dev/null
+++ b/drivers/clk/mediatek/clk-mt8183-mm.c
@@ -0,0 +1,111 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2018 MediaTek Inc.
+// Author: Weiyi Lu <weiyi.lu@mediatek.com>
+
+#include <linux/clk-provider.h>
+#include <linux/platform_device.h>
+
+#include "clk-mtk.h"
+#include "clk-gate.h"
+
+#include <dt-bindings/clock/mt8183-clk.h>
+
+static const struct mtk_gate_regs mm0_cg_regs = {
+	.set_ofs = 0x104,
+	.clr_ofs = 0x108,
+	.sta_ofs = 0x100,
+};
+
+static const struct mtk_gate_regs mm1_cg_regs = {
+	.set_ofs = 0x114,
+	.clr_ofs = 0x118,
+	.sta_ofs = 0x110,
+};
+
+#define GATE_MM0(_id, _name, _parent, _shift)			\
+	GATE_MTK(_id, _name, _parent, &mm0_cg_regs, _shift,	\
+		&mtk_clk_gate_ops_setclr)
+
+#define GATE_MM1(_id, _name, _parent, _shift)			\
+	GATE_MTK(_id, _name, _parent, &mm1_cg_regs, _shift,	\
+		&mtk_clk_gate_ops_setclr)
+
+static const struct mtk_gate mm_clks[] = {
+	/* MM0 */
+	GATE_MM0(CLK_MM_SMI_COMMON, "mm_smi_common", "mm_sel", 0),
+	GATE_MM0(CLK_MM_SMI_LARB0, "mm_smi_larb0", "mm_sel", 1),
+	GATE_MM0(CLK_MM_SMI_LARB1, "mm_smi_larb1", "mm_sel", 2),
+	GATE_MM0(CLK_MM_GALS_COMM0, "mm_gals_comm0", "mm_sel", 3),
+	GATE_MM0(CLK_MM_GALS_COMM1, "mm_gals_comm1", "mm_sel", 4),
+	GATE_MM0(CLK_MM_GALS_CCU2MM, "mm_gals_ccu2mm", "mm_sel", 5),
+	GATE_MM0(CLK_MM_GALS_IPU12MM, "mm_gals_ipu12mm", "mm_sel", 6),
+	GATE_MM0(CLK_MM_GALS_IMG2MM, "mm_gals_img2mm", "mm_sel", 7),
+	GATE_MM0(CLK_MM_GALS_CAM2MM, "mm_gals_cam2mm", "mm_sel", 8),
+	GATE_MM0(CLK_MM_GALS_IPU2MM, "mm_gals_ipu2mm", "mm_sel", 9),
+	GATE_MM0(CLK_MM_MDP_DL_TXCK, "mm_mdp_dl_txck", "mm_sel", 10),
+	GATE_MM0(CLK_MM_IPU_DL_TXCK, "mm_ipu_dl_txck", "mm_sel", 11),
+	GATE_MM0(CLK_MM_MDP_RDMA0, "mm_mdp_rdma0", "mm_sel", 12),
+	GATE_MM0(CLK_MM_MDP_RDMA1, "mm_mdp_rdma1", "mm_sel", 13),
+	GATE_MM0(CLK_MM_MDP_RSZ0, "mm_mdp_rsz0", "mm_sel", 14),
+	GATE_MM0(CLK_MM_MDP_RSZ1, "mm_mdp_rsz1", "mm_sel", 15),
+	GATE_MM0(CLK_MM_MDP_TDSHP, "mm_mdp_tdshp", "mm_sel", 16),
+	GATE_MM0(CLK_MM_MDP_WROT0, "mm_mdp_wrot0", "mm_sel", 17),
+	GATE_MM0(CLK_MM_MDP_WDMA0, "mm_mdp_wdma0", "mm_sel", 18),
+	GATE_MM0(CLK_MM_FAKE_ENG, "mm_fake_eng", "mm_sel", 19),
+	GATE_MM0(CLK_MM_DISP_OVL0, "mm_disp_ovl0", "mm_sel", 20),
+	GATE_MM0(CLK_MM_DISP_OVL0_2L, "mm_disp_ovl0_2l", "mm_sel", 21),
+	GATE_MM0(CLK_MM_DISP_OVL1_2L, "mm_disp_ovl1_2l", "mm_sel", 22),
+	GATE_MM0(CLK_MM_DISP_RDMA0, "mm_disp_rdma0", "mm_sel", 23),
+	GATE_MM0(CLK_MM_DISP_RDMA1, "mm_disp_rdma1", "mm_sel", 24),
+	GATE_MM0(CLK_MM_DISP_WDMA0, "mm_disp_wdma0", "mm_sel", 25),
+	GATE_MM0(CLK_MM_DISP_COLOR0, "mm_disp_color0", "mm_sel", 26),
+	GATE_MM0(CLK_MM_DISP_CCORR0, "mm_disp_ccorr0", "mm_sel", 27),
+	GATE_MM0(CLK_MM_DISP_AAL0, "mm_disp_aal0", "mm_sel", 28),
+	GATE_MM0(CLK_MM_DISP_GAMMA0, "mm_disp_gamma0", "mm_sel", 29),
+	GATE_MM0(CLK_MM_DISP_DITHER0, "mm_disp_dither0", "mm_sel", 30),
+	GATE_MM0(CLK_MM_DISP_SPLIT, "mm_disp_split", "mm_sel", 31),
+	/* MM1 */
+	GATE_MM1(CLK_MM_DSI0_MM, "mm_dsi0_mm", "mm_sel", 0),
+	GATE_MM1(CLK_MM_DSI0_IF, "mm_dsi0_if", "mm_sel", 1),
+	GATE_MM1(CLK_MM_DPI_MM, "mm_dpi_mm", "mm_sel", 2),
+	GATE_MM1(CLK_MM_DPI_IF, "mm_dpi_if", "dpi0_sel", 3),
+	GATE_MM1(CLK_MM_FAKE_ENG2, "mm_fake_eng2", "mm_sel", 4),
+	GATE_MM1(CLK_MM_MDP_DL_RX, "mm_mdp_dl_rx", "mm_sel", 5),
+	GATE_MM1(CLK_MM_IPU_DL_RX, "mm_ipu_dl_rx", "mm_sel", 6),
+	GATE_MM1(CLK_MM_26M, "mm_26m", "f_f26m_ck", 7),
+	GATE_MM1(CLK_MM_MMSYS_R2Y, "mm_mmsys_r2y", "mm_sel", 8),
+	GATE_MM1(CLK_MM_DISP_RSZ, "mm_disp_rsz", "mm_sel", 9),
+	GATE_MM1(CLK_MM_MDP_AAL, "mm_mdp_aal", "mm_sel", 10),
+	GATE_MM1(CLK_MM_MDP_CCORR, "mm_mdp_ccorr", "mm_sel", 11),
+	GATE_MM1(CLK_MM_DBI_MM, "mm_dbi_mm", "mm_sel", 12),
+	GATE_MM1(CLK_MM_DBI_IF, "mm_dbi_if", "dpi0_sel", 13),
+};
+
+static int clk_mt8183_mm_probe(struct platform_device *pdev)
+{
+	struct clk_onecell_data *clk_data;
+	struct device_node *node = pdev->dev.of_node;
+
+	clk_data = mtk_alloc_clk_data(CLK_MM_NR_CLK);
+
+	mtk_clk_register_gates(node, mm_clks, ARRAY_SIZE(mm_clks),
+			clk_data);
+
+	return of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+}
+
+static const struct of_device_id of_match_clk_mt8183_mm[] = {
+	{ .compatible = "mediatek,mt8183-mmsys", },
+	{}
+};
+
+static struct platform_driver clk_mt8183_mm_drv = {
+	.probe = clk_mt8183_mm_probe,
+	.driver = {
+		.name = "clk-mt8183-mm",
+		.of_match_table = of_match_clk_mt8183_mm,
+	},
+};
+
+builtin_platform_driver(clk_mt8183_mm_drv);
diff --git a/drivers/clk/mediatek/clk-mt8183-vdec.c b/drivers/clk/mediatek/clk-mt8183-vdec.c
new file mode 100644
index 000000000000..6250fd1e0edc
--- /dev/null
+++ b/drivers/clk/mediatek/clk-mt8183-vdec.c
@@ -0,0 +1,67 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2018 MediaTek Inc.
+// Author: Weiyi Lu <weiyi.lu@mediatek.com>
+
+#include <linux/clk-provider.h>
+#include <linux/platform_device.h>
+
+#include "clk-mtk.h"
+#include "clk-gate.h"
+
+#include <dt-bindings/clock/mt8183-clk.h>
+
+static const struct mtk_gate_regs vdec0_cg_regs = {
+	.set_ofs = 0x0,
+	.clr_ofs = 0x4,
+	.sta_ofs = 0x0,
+};
+
+static const struct mtk_gate_regs vdec1_cg_regs = {
+	.set_ofs = 0x8,
+	.clr_ofs = 0xc,
+	.sta_ofs = 0x8,
+};
+
+#define GATE_VDEC0_I(_id, _name, _parent, _shift)		\
+	GATE_MTK(_id, _name, _parent, &vdec0_cg_regs, _shift,	\
+		&mtk_clk_gate_ops_setclr_inv)
+
+#define GATE_VDEC1_I(_id, _name, _parent, _shift)		\
+	GATE_MTK(_id, _name, _parent, &vdec1_cg_regs, _shift,	\
+		&mtk_clk_gate_ops_setclr_inv)
+
+static const struct mtk_gate vdec_clks[] = {
+	/* VDEC0 */
+	GATE_VDEC0_I(CLK_VDEC_VDEC, "vdec_vdec", "mm_sel", 0),
+	/* VDEC1 */
+	GATE_VDEC1_I(CLK_VDEC_LARB1, "vdec_larb1", "mm_sel", 0),
+};
+
+static int clk_mt8183_vdec_probe(struct platform_device *pdev)
+{
+	struct clk_onecell_data *clk_data;
+	struct device_node *node = pdev->dev.of_node;
+
+	clk_data = mtk_alloc_clk_data(CLK_VDEC_NR_CLK);
+
+	mtk_clk_register_gates(node, vdec_clks, ARRAY_SIZE(vdec_clks),
+			clk_data);
+
+	return of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+}
+
+static const struct of_device_id of_match_clk_mt8183_vdec[] = {
+	{ .compatible = "mediatek,mt8183-vdecsys", },
+	{}
+};
+
+static struct platform_driver clk_mt8183_vdec_drv = {
+	.probe = clk_mt8183_vdec_probe,
+	.driver = {
+		.name = "clk-mt8183-vdec",
+		.of_match_table = of_match_clk_mt8183_vdec,
+	},
+};
+
+builtin_platform_driver(clk_mt8183_vdec_drv);
diff --git a/drivers/clk/mediatek/clk-mt8183-venc.c b/drivers/clk/mediatek/clk-mt8183-venc.c
new file mode 100644
index 000000000000..6678ef03fab2
--- /dev/null
+++ b/drivers/clk/mediatek/clk-mt8183-venc.c
@@ -0,0 +1,59 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2018 MediaTek Inc.
+// Author: Weiyi Lu <weiyi.lu@mediatek.com>
+
+#include <linux/clk-provider.h>
+#include <linux/platform_device.h>
+
+#include "clk-mtk.h"
+#include "clk-gate.h"
+
+#include <dt-bindings/clock/mt8183-clk.h>
+
+static const struct mtk_gate_regs venc_cg_regs = {
+	.set_ofs = 0x4,
+	.clr_ofs = 0x8,
+	.sta_ofs = 0x0,
+};
+
+#define GATE_VENC_I(_id, _name, _parent, _shift)		\
+	GATE_MTK(_id, _name, _parent, &venc_cg_regs, _shift,	\
+		&mtk_clk_gate_ops_setclr_inv)
+
+static const struct mtk_gate venc_clks[] = {
+	GATE_VENC_I(CLK_VENC_LARB, "venc_larb",
+		"mm_sel", 0),
+	GATE_VENC_I(CLK_VENC_VENC, "venc_venc",
+		"mm_sel", 4),
+	GATE_VENC_I(CLK_VENC_JPGENC, "venc_jpgenc",
+		"mm_sel", 8),
+};
+
+static int clk_mt8183_venc_probe(struct platform_device *pdev)
+{
+	struct clk_onecell_data *clk_data;
+	struct device_node *node = pdev->dev.of_node;
+
+	clk_data = mtk_alloc_clk_data(CLK_VENC_NR_CLK);
+
+	mtk_clk_register_gates(node, venc_clks, ARRAY_SIZE(venc_clks),
+			clk_data);
+
+	return of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+}
+
+static const struct of_device_id of_match_clk_mt8183_venc[] = {
+	{ .compatible = "mediatek,mt8183-vencsys", },
+	{}
+};
+
+static struct platform_driver clk_mt8183_venc_drv = {
+	.probe = clk_mt8183_venc_probe,
+	.driver = {
+		.name = "clk-mt8183-venc",
+		.of_match_table = of_match_clk_mt8183_venc,
+	},
+};
+
+builtin_platform_driver(clk_mt8183_venc_drv);
diff --git a/drivers/clk/mediatek/clk-mt8183.c b/drivers/clk/mediatek/clk-mt8183.c
new file mode 100644
index 000000000000..9d8651033ae9
--- /dev/null
+++ b/drivers/clk/mediatek/clk-mt8183.c
@@ -0,0 +1,1284 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2018 MediaTek Inc.
+// Author: Weiyi Lu <weiyi.lu@mediatek.com>
+
+#include <linux/delay.h>
+#include <linux/mfd/syscon.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+#include "clk-mtk.h"
+#include "clk-mux.h"
+#include "clk-gate.h"
+
+#include <dt-bindings/clock/mt8183-clk.h>
+
+static DEFINE_SPINLOCK(mt8183_clk_lock);
+
+static const struct mtk_fixed_clk top_fixed_clks[] = {
+	FIXED_CLK(CLK_TOP_CLK26M, "f_f26m_ck", "clk26m", 26000000),
+	FIXED_CLK(CLK_TOP_ULPOSC, "osc", NULL, 250000),
+	FIXED_CLK(CLK_TOP_UNIVP_192M, "univpll_192m", "univpll", 192000000),
+};
+
+static const struct mtk_fixed_factor top_divs[] = {
+	FACTOR(CLK_TOP_CLK13M, "clk13m", "clk26m", 1,
+		2),
+	FACTOR(CLK_TOP_F26M_CK_D2, "csw_f26m_ck_d2", "clk26m", 1,
+		2),
+	FACTOR(CLK_TOP_SYSPLL_CK, "syspll_ck", "mainpll", 1,
+		1),
+	FACTOR(CLK_TOP_SYSPLL_D2, "syspll_d2", "syspll_ck", 1,
+		2),
+	FACTOR(CLK_TOP_SYSPLL_D2_D2, "syspll_d2_d2", "syspll_d2", 1,
+		2),
+	FACTOR(CLK_TOP_SYSPLL_D2_D4, "syspll_d2_d4", "syspll_d2", 1,
+		4),
+	FACTOR(CLK_TOP_SYSPLL_D2_D8, "syspll_d2_d8", "syspll_d2", 1,
+		8),
+	FACTOR(CLK_TOP_SYSPLL_D2_D16, "syspll_d2_d16", "syspll_d2", 1,
+		16),
+	FACTOR(CLK_TOP_SYSPLL_D3, "syspll_d3", "mainpll", 1,
+		3),
+	FACTOR(CLK_TOP_SYSPLL_D3_D2, "syspll_d3_d2", "syspll_d3", 1,
+		2),
+	FACTOR(CLK_TOP_SYSPLL_D3_D4, "syspll_d3_d4", "syspll_d3", 1,
+		4),
+	FACTOR(CLK_TOP_SYSPLL_D3_D8, "syspll_d3_d8", "syspll_d3", 1,
+		8),
+	FACTOR(CLK_TOP_SYSPLL_D5, "syspll_d5", "mainpll", 1,
+		5),
+	FACTOR(CLK_TOP_SYSPLL_D5_D2, "syspll_d5_d2", "syspll_d5", 1,
+		2),
+	FACTOR(CLK_TOP_SYSPLL_D5_D4, "syspll_d5_d4", "syspll_d5", 1,
+		4),
+	FACTOR(CLK_TOP_SYSPLL_D7, "syspll_d7", "mainpll", 1,
+		7),
+	FACTOR(CLK_TOP_SYSPLL_D7_D2, "syspll_d7_d2", "syspll_d7", 1,
+		2),
+	FACTOR(CLK_TOP_SYSPLL_D7_D4, "syspll_d7_d4", "syspll_d7", 1,
+		4),
+	FACTOR(CLK_TOP_UNIVPLL_CK, "univpll_ck", "univpll", 1,
+		1),
+	FACTOR(CLK_TOP_UNIVPLL_D2, "univpll_d2", "univpll_ck", 1,
+		2),
+	FACTOR(CLK_TOP_UNIVPLL_D2_D2, "univpll_d2_d2", "univpll_d2", 1,
+		2),
+	FACTOR(CLK_TOP_UNIVPLL_D2_D4, "univpll_d2_d4", "univpll_d2", 1,
+		4),
+	FACTOR(CLK_TOP_UNIVPLL_D2_D8, "univpll_d2_d8", "univpll_d2", 1,
+		8),
+	FACTOR(CLK_TOP_UNIVPLL_D3, "univpll_d3", "univpll", 1,
+		3),
+	FACTOR(CLK_TOP_UNIVPLL_D3_D2, "univpll_d3_d2", "univpll_d3", 1,
+		2),
+	FACTOR(CLK_TOP_UNIVPLL_D3_D4, "univpll_d3_d4", "univpll_d3", 1,
+		4),
+	FACTOR(CLK_TOP_UNIVPLL_D3_D8, "univpll_d3_d8", "univpll_d3", 1,
+		8),
+	FACTOR(CLK_TOP_UNIVPLL_D5, "univpll_d5", "univpll", 1,
+		5),
+	FACTOR(CLK_TOP_UNIVPLL_D5_D2, "univpll_d5_d2", "univpll_d5", 1,
+		2),
+	FACTOR(CLK_TOP_UNIVPLL_D5_D4, "univpll_d5_d4", "univpll_d5", 1,
+		4),
+	FACTOR(CLK_TOP_UNIVPLL_D5_D8, "univpll_d5_d8", "univpll_d5", 1,
+		8),
+	FACTOR(CLK_TOP_UNIVPLL_D7, "univpll_d7", "univpll", 1,
+		7),
+	FACTOR(CLK_TOP_UNIVP_192M_CK, "univ_192m_ck", "univpll_192m", 1,
+		1),
+	FACTOR(CLK_TOP_UNIVP_192M_D2, "univ_192m_d2", "univ_192m_ck", 1,
+		2),
+	FACTOR(CLK_TOP_UNIVP_192M_D4, "univ_192m_d4", "univ_192m_ck", 1,
+		4),
+	FACTOR(CLK_TOP_UNIVP_192M_D8, "univ_192m_d8", "univ_192m_ck", 1,
+		8),
+	FACTOR(CLK_TOP_UNIVP_192M_D16, "univ_192m_d16", "univ_192m_ck", 1,
+		16),
+	FACTOR(CLK_TOP_UNIVP_192M_D32, "univ_192m_d32", "univ_192m_ck", 1,
+		32),
+	FACTOR(CLK_TOP_APLL1_CK, "apll1_ck", "apll1", 1,
+		1),
+	FACTOR(CLK_TOP_APLL1_D2, "apll1_d2", "apll1", 1,
+		2),
+	FACTOR(CLK_TOP_APLL1_D4, "apll1_d4", "apll1", 1,
+		4),
+	FACTOR(CLK_TOP_APLL1_D8, "apll1_d8", "apll1", 1,
+		8),
+	FACTOR(CLK_TOP_APLL2_CK, "apll2_ck", "apll2", 1,
+		1),
+	FACTOR(CLK_TOP_APLL2_D2, "apll2_d2", "apll2", 1,
+		2),
+	FACTOR(CLK_TOP_APLL2_D4, "apll2_d4", "apll2", 1,
+		4),
+	FACTOR(CLK_TOP_APLL2_D8, "apll2_d8", "apll2", 1,
+		8),
+	FACTOR(CLK_TOP_TVDPLL_CK, "tvdpll_ck", "tvdpll", 1,
+		1),
+	FACTOR(CLK_TOP_TVDPLL_D2, "tvdpll_d2", "tvdpll_ck", 1,
+		2),
+	FACTOR(CLK_TOP_TVDPLL_D4, "tvdpll_d4", "tvdpll", 1,
+		4),
+	FACTOR(CLK_TOP_TVDPLL_D8, "tvdpll_d8", "tvdpll", 1,
+		8),
+	FACTOR(CLK_TOP_TVDPLL_D16, "tvdpll_d16", "tvdpll", 1,
+		16),
+	FACTOR(CLK_TOP_MMPLL_CK, "mmpll_ck", "mmpll", 1,
+		1),
+	FACTOR(CLK_TOP_MMPLL_D4, "mmpll_d4", "mmpll", 1,
+		4),
+	FACTOR(CLK_TOP_MMPLL_D4_D2, "mmpll_d4_d2", "mmpll_d4", 1,
+		2),
+	FACTOR(CLK_TOP_MMPLL_D4_D4, "mmpll_d4_d4", "mmpll_d4", 1,
+		4),
+	FACTOR(CLK_TOP_MMPLL_D5, "mmpll_d5", "mmpll", 1,
+		5),
+	FACTOR(CLK_TOP_MMPLL_D5_D2, "mmpll_d5_d2", "mmpll_d5", 1,
+		2),
+	FACTOR(CLK_TOP_MMPLL_D5_D4, "mmpll_d5_d4", "mmpll_d5", 1,
+		4),
+	FACTOR(CLK_TOP_MMPLL_D6, "mmpll_d6", "mmpll", 1,
+		6),
+	FACTOR(CLK_TOP_MMPLL_D7, "mmpll_d7", "mmpll", 1,
+		7),
+	FACTOR(CLK_TOP_MFGPLL_CK, "mfgpll_ck", "mfgpll", 1,
+		1),
+	FACTOR(CLK_TOP_MSDCPLL_CK, "msdcpll_ck", "msdcpll", 1,
+		1),
+	FACTOR(CLK_TOP_MSDCPLL_D2, "msdcpll_d2", "msdcpll", 1,
+		2),
+	FACTOR(CLK_TOP_MSDCPLL_D4, "msdcpll_d4", "msdcpll", 1,
+		4),
+	FACTOR(CLK_TOP_MSDCPLL_D8, "msdcpll_d8", "msdcpll", 1,
+		8),
+	FACTOR(CLK_TOP_MSDCPLL_D16, "msdcpll_d16", "msdcpll", 1,
+		16),
+	FACTOR(CLK_TOP_AD_OSC_CK, "ad_osc_ck", "osc", 1,
+		1),
+	FACTOR(CLK_TOP_OSC_D2, "osc_d2", "osc", 1,
+		2),
+	FACTOR(CLK_TOP_OSC_D4, "osc_d4", "osc", 1,
+		4),
+	FACTOR(CLK_TOP_OSC_D8, "osc_d8", "osc", 1,
+		8),
+	FACTOR(CLK_TOP_OSC_D16, "osc_d16", "osc", 1,
+		16),
+	FACTOR(CLK_TOP_UNIVPLL, "univpll", "univ2pll", 1,
+		2),
+	FACTOR(CLK_TOP_UNIVPLL_D3_D16, "univpll_d3_d16", "univpll_d3", 1,
+		16),
+};
+
+static const char * const axi_parents[] = {
+	"clk26m",
+	"syspll_d2_d4",
+	"syspll_d7",
+	"osc_d4"
+};
+
+static const char * const mm_parents[] = {
+	"clk26m",
+	"mmpll_d7",
+	"syspll_d3",
+	"univpll_d2_d2",
+	"syspll_d2_d2",
+	"syspll_d3_d2"
+};
+
+static const char * const img_parents[] = {
+	"clk26m",
+	"mmpll_d6",
+	"univpll_d3",
+	"syspll_d3",
+	"univpll_d2_d2",
+	"syspll_d2_d2",
+	"univpll_d3_d2",
+	"syspll_d3_d2"
+};
+
+static const char * const cam_parents[] = {
+	"clk26m",
+	"syspll_d2",
+	"mmpll_d6",
+	"syspll_d3",
+	"mmpll_d7",
+	"univpll_d3",
+	"univpll_d2_d2",
+	"syspll_d2_d2",
+	"syspll_d3_d2",
+	"univpll_d3_d2"
+};
+
+static const char * const dsp_parents[] = {
+	"clk26m",
+	"mmpll_d6",
+	"mmpll_d7",
+	"univpll_d3",
+	"syspll_d3",
+	"univpll_d2_d2",
+	"syspll_d2_d2",
+	"univpll_d3_d2",
+	"syspll_d3_d2"
+};
+
+static const char * const dsp1_parents[] = {
+	"clk26m",
+	"mmpll_d6",
+	"mmpll_d7",
+	"univpll_d3",
+	"syspll_d3",
+	"univpll_d2_d2",
+	"syspll_d2_d2",
+	"univpll_d3_d2",
+	"syspll_d3_d2"
+};
+
+static const char * const dsp2_parents[] = {
+	"clk26m",
+	"mmpll_d6",
+	"mmpll_d7",
+	"univpll_d3",
+	"syspll_d3",
+	"univpll_d2_d2",
+	"syspll_d2_d2",
+	"univpll_d3_d2",
+	"syspll_d3_d2"
+};
+
+static const char * const ipu_if_parents[] = {
+	"clk26m",
+	"mmpll_d6",
+	"mmpll_d7",
+	"univpll_d3",
+	"syspll_d3",
+	"univpll_d2_d2",
+	"syspll_d2_d2",
+	"univpll_d3_d2",
+	"syspll_d3_d2"
+};
+
+static const char * const mfg_parents[] = {
+	"clk26m",
+	"mfgpll_ck",
+	"univpll_d3",
+	"syspll_d3"
+};
+
+static const char * const f52m_mfg_parents[] = {
+	"clk26m",
+	"univpll_d3_d2",
+	"univpll_d3_d4",
+	"univpll_d3_d8"
+};
+
+static const char * const camtg_parents[] = {
+	"clk26m",
+	"univ_192m_d8",
+	"univpll_d3_d8",
+	"univ_192m_d4",
+	"univpll_d3_d16",
+	"csw_f26m_ck_d2",
+	"univ_192m_d16",
+	"univ_192m_d32"
+};
+
+static const char * const camtg2_parents[] = {
+	"clk26m",
+	"univ_192m_d8",
+	"univpll_d3_d8",
+	"univ_192m_d4",
+	"univpll_d3_d16",
+	"csw_f26m_ck_d2",
+	"univ_192m_d16",
+	"univ_192m_d32"
+};
+
+static const char * const camtg3_parents[] = {
+	"clk26m",
+	"univ_192m_d8",
+	"univpll_d3_d8",
+	"univ_192m_d4",
+	"univpll_d3_d16",
+	"csw_f26m_ck_d2",
+	"univ_192m_d16",
+	"univ_192m_d32"
+};
+
+static const char * const camtg4_parents[] = {
+	"clk26m",
+	"univ_192m_d8",
+	"univpll_d3_d8",
+	"univ_192m_d4",
+	"univpll_d3_d16",
+	"csw_f26m_ck_d2",
+	"univ_192m_d16",
+	"univ_192m_d32"
+};
+
+static const char * const uart_parents[] = {
+	"clk26m",
+	"univpll_d3_d8"
+};
+
+static const char * const spi_parents[] = {
+	"clk26m",
+	"syspll_d5_d2",
+	"syspll_d3_d4",
+	"msdcpll_d4"
+};
+
+static const char * const msdc50_hclk_parents[] = {
+	"clk26m",
+	"syspll_d2_d2",
+	"syspll_d3_d2"
+};
+
+static const char * const msdc50_0_parents[] = {
+	"clk26m",
+	"msdcpll_ck",
+	"msdcpll_d2",
+	"univpll_d2_d4",
+	"syspll_d3_d2",
+	"univpll_d2_d2"
+};
+
+static const char * const msdc30_1_parents[] = {
+	"clk26m",
+	"univpll_d3_d2",
+	"syspll_d3_d2",
+	"syspll_d7",
+	"msdcpll_d2"
+};
+
+static const char * const msdc30_2_parents[] = {
+	"clk26m",
+	"univpll_d3_d2",
+	"syspll_d3_d2",
+	"syspll_d7",
+	"msdcpll_d2"
+};
+
+static const char * const audio_parents[] = {
+	"clk26m",
+	"syspll_d5_d4",
+	"syspll_d7_d4",
+	"syspll_d2_d16"
+};
+
+static const char * const aud_intbus_parents[] = {
+	"clk26m",
+	"syspll_d2_d4",
+	"syspll_d7_d2"
+};
+
+static const char * const pmicspi_parents[] = {
+	"clk26m",
+	"syspll_d2_d8",
+	"osc_d8"
+};
+
+static const char * const fpwrap_ulposc_parents[] = {
+	"clk26m",
+	"osc_d16",
+	"osc_d4",
+	"osc_d8"
+};
+
+static const char * const atb_parents[] = {
+	"clk26m",
+	"syspll_d2_d2",
+	"syspll_d5"
+};
+
+static const char * const sspm_parents[] = {
+	"clk26m",
+	"univpll_d2_d4",
+	"syspll_d2_d2",
+	"univpll_d2_d2",
+	"syspll_d3"
+};
+
+static const char * const dpi0_parents[] = {
+	"clk26m",
+	"tvdpll_d2",
+	"tvdpll_d4",
+	"tvdpll_d8",
+	"tvdpll_d16",
+	"univpll_d5_d2",
+	"univpll_d3_d4",
+	"syspll_d3_d4",
+	"univpll_d3_d8"
+};
+
+static const char * const scam_parents[] = {
+	"clk26m",
+	"syspll_d5_d2"
+};
+
+static const char * const disppwm_parents[] = {
+	"clk26m",
+	"univpll_d3_d4",
+	"osc_d2",
+	"osc_d4",
+	"osc_d16"
+};
+
+static const char * const usb_top_parents[] = {
+	"clk26m",
+	"univpll_d5_d4",
+	"univpll_d3_d4",
+	"univpll_d5_d2"
+};
+
+
+static const char * const ssusb_top_xhci_parents[] = {
+	"clk26m",
+	"univpll_d5_d4",
+	"univpll_d3_d4",
+	"univpll_d5_d2"
+};
+
+static const char * const spm_parents[] = {
+	"clk26m",
+	"syspll_d2_d8"
+};
+
+static const char * const i2c_parents[] = {
+	"clk26m",
+	"syspll_d2_d8",
+	"univpll_d5_d2"
+};
+
+static const char * const scp_parents[] = {
+	"clk26m",
+	"univpll_d2_d8",
+	"syspll_d5",
+	"syspll_d2_d2",
+	"univpll_d2_d2",
+	"syspll_d3",
+	"univpll_d3"
+};
+
+static const char * const seninf_parents[] = {
+	"clk26m",
+	"univpll_d2_d2",
+	"univpll_d3_d2",
+	"univpll_d2_d4"
+};
+
+static const char * const dxcc_parents[] = {
+	"clk26m",
+	"syspll_d2_d2",
+	"syspll_d2_d4",
+	"syspll_d2_d8"
+};
+
+static const char * const aud_engen1_parents[] = {
+	"clk26m",
+	"apll1_d2",
+	"apll1_d4",
+	"apll1_d8"
+};
+
+static const char * const aud_engen2_parents[] = {
+	"clk26m",
+	"apll2_d2",
+	"apll2_d4",
+	"apll2_d8"
+};
+
+static const char * const faes_ufsfde_parents[] = {
+	"clk26m",
+	"syspll_d2",
+	"syspll_d2_d2",
+	"syspll_d3",
+	"syspll_d2_d4",
+	"univpll_d3"
+};
+
+static const char * const fufs_parents[] = {
+	"clk26m",
+	"syspll_d2_d4",
+	"syspll_d2_d8",
+	"syspll_d2_d16"
+};
+
+static const char * const aud_1_parents[] = {
+	"clk26m",
+	"apll1_ck"
+};
+
+static const char * const aud_2_parents[] = {
+	"clk26m",
+	"apll2_ck"
+};
+
+/*
+ * CRITICAL CLOCK:
+ * axi_sel is the main bus clock of whole SOC.
+ * spm_sel is the clock of the always-on co-processor.
+ */
+static const struct mtk_mux top_muxes[] = {
+	/* CLK_CFG_0 */
+	MUX_GATE_CLR_SET_UPD_FLAGS(CLK_TOP_MUX_AXI, "axi_sel",
+		axi_parents, 0x40,
+		0x44, 0x48, 0, 2, 7, 0x004, 0, CLK_IS_CRITICAL),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_MM, "mm_sel",
+		mm_parents, 0x40,
+		0x44, 0x48, 8, 3, 15, 0x004, 1),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_IMG, "img_sel",
+		img_parents, 0x40,
+		0x44, 0x48, 16, 3, 23, 0x004, 2),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_CAM, "cam_sel",
+		cam_parents, 0x40,
+		0x44, 0x48, 24, 4, 31, 0x004, 3),
+	/* CLK_CFG_1 */
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_DSP, "dsp_sel",
+		dsp_parents, 0x50,
+		0x54, 0x58, 0, 4, 7, 0x004, 4),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_DSP1, "dsp1_sel",
+		dsp1_parents, 0x50,
+		0x54, 0x58, 8, 4, 15, 0x004, 5),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_DSP2, "dsp2_sel",
+		dsp2_parents, 0x50,
+		0x54, 0x58, 16, 4, 23, 0x004, 6),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_IPU_IF, "ipu_if_sel",
+		ipu_if_parents, 0x50,
+		0x54, 0x58, 24, 4, 31, 0x004, 7),
+	/* CLK_CFG_2 */
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_MFG, "mfg_sel",
+		mfg_parents, 0x60,
+		0x64, 0x68, 0, 2, 7, 0x004, 8),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_F52M_MFG, "f52m_mfg_sel",
+		f52m_mfg_parents, 0x60,
+		0x64, 0x68, 8, 2, 15, 0x004, 9),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_CAMTG, "camtg_sel",
+		camtg_parents, 0x60,
+		0x64, 0x68, 16, 3, 23, 0x004, 10),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_CAMTG2, "camtg2_sel",
+		camtg2_parents, 0x60,
+		0x64, 0x68, 24, 3, 31, 0x004, 11),
+	/* CLK_CFG_3 */
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_CAMTG3, "camtg3_sel",
+		camtg3_parents, 0x70,
+		0x74, 0x78, 0, 3, 7, 0x004, 12),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_CAMTG4, "camtg4_sel",
+		camtg4_parents, 0x70,
+		0x74, 0x78, 8, 3, 15, 0x004, 13),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_UART, "uart_sel",
+		uart_parents, 0x70,
+		0x74, 0x78, 16, 1, 23, 0x004, 14),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_SPI, "spi_sel",
+		spi_parents, 0x70,
+		0x74, 0x78, 24, 2, 31, 0x004, 15),
+	/* CLK_CFG_4 */
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_MSDC50_0_HCLK, "msdc50_hclk_sel",
+		msdc50_hclk_parents, 0x80,
+		0x84, 0x88, 0, 2, 7, 0x004, 16),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_MSDC50_0, "msdc50_0_sel",
+		msdc50_0_parents, 0x80,
+		0x84, 0x88, 8, 3, 15, 0x004, 17),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_MSDC30_1, "msdc30_1_sel",
+		msdc30_1_parents, 0x80,
+		0x84, 0x88, 16, 3, 23, 0x004, 18),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_MSDC30_2, "msdc30_2_sel",
+		msdc30_2_parents, 0x80,
+		0x84, 0x88, 24, 3, 31, 0x004, 19),
+	/* CLK_CFG_5 */
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_AUDIO, "audio_sel",
+		audio_parents, 0x90,
+		0x94, 0x98, 0, 2, 7, 0x004, 20),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_AUD_INTBUS, "aud_intbus_sel",
+		aud_intbus_parents, 0x90,
+		0x94, 0x98, 8, 2, 15, 0x004, 21),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_PMICSPI, "pmicspi_sel",
+		pmicspi_parents, 0x90,
+		0x94, 0x98, 16, 2, 23, 0x004, 22),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_FPWRAP_ULPOSC, "fpwrap_ulposc_sel",
+		fpwrap_ulposc_parents, 0x90,
+		0x94, 0x98, 24, 2, 31, 0x004, 23),
+	/* CLK_CFG_6 */
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_ATB, "atb_sel",
+		atb_parents, 0xa0,
+		0xa4, 0xa8, 0, 2, 7, 0x004, 24),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_SSPM, "sspm_sel",
+		sspm_parents, 0xa0,
+		0xa4, 0xa8, 8, 3, 15, 0x004, 25),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_DPI0, "dpi0_sel",
+		dpi0_parents, 0xa0,
+		0xa4, 0xa8, 16, 4, 23, 0x004, 26),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_SCAM, "scam_sel",
+		scam_parents, 0xa0,
+		0xa4, 0xa8, 24, 1, 31, 0x004, 27),
+	/* CLK_CFG_7 */
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_DISP_PWM, "disppwm_sel",
+		disppwm_parents, 0xb0,
+		0xb4, 0xb8, 0, 3, 7, 0x004, 28),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_USB_TOP, "usb_top_sel",
+		usb_top_parents, 0xb0,
+		0xb4, 0xb8, 8, 2, 15, 0x004, 29),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_SSUSB_TOP_XHCI, "ssusb_top_xhci_sel",
+		ssusb_top_xhci_parents, 0xb0,
+		0xb4, 0xb8, 16, 2, 23, 0x004, 30),
+	MUX_GATE_CLR_SET_UPD_FLAGS(CLK_TOP_MUX_SPM, "spm_sel",
+		spm_parents, 0xb0,
+		0xb4, 0xb8, 24, 1, 31, 0x008, 0, CLK_IS_CRITICAL),
+	/* CLK_CFG_8 */
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_I2C, "i2c_sel",
+		i2c_parents, 0xc0,
+		0xc4, 0xc8, 0, 2, 7, 0x008, 1),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_SCP, "scp_sel",
+		scp_parents, 0xc0,
+		0xc4, 0xc8, 8, 3, 15, 0x008, 2),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_SENINF, "seninf_sel",
+		seninf_parents, 0xc0,
+		0xc4, 0xc8, 16, 2, 23, 0x008, 3),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_DXCC, "dxcc_sel",
+		dxcc_parents, 0xc0,
+		0xc4, 0xc8, 24, 2, 31, 0x008, 4),
+	/* CLK_CFG_9 */
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_AUD_ENG1, "aud_eng1_sel",
+		aud_engen1_parents, 0xd0,
+		0xd4, 0xd8, 0, 2, 7, 0x008, 5),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_AUD_ENG2, "aud_eng2_sel",
+		aud_engen2_parents, 0xd0,
+		0xd4, 0xd8, 8, 2, 15, 0x008, 6),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_FAES_UFSFDE, "faes_ufsfde_sel",
+		faes_ufsfde_parents, 0xd0,
+		0xd4, 0xd8, 16, 3, 23, 0x008, 7),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_FUFS, "fufs_sel",
+		fufs_parents, 0xd0,
+		0xd4, 0xd8, 24, 2, 31, 0x008, 8),
+	/* CLK_CFG_10 */
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_AUD_1, "aud_1_sel",
+		aud_1_parents, 0xe0,
+		0xe4, 0xe8, 0, 1, 7, 0x008, 9),
+	MUX_GATE_CLR_SET_UPD(CLK_TOP_MUX_AUD_2, "aud_2_sel",
+		aud_2_parents, 0xe0,
+		0xe4, 0xe8, 8, 1, 15, 0x008, 10),
+};
+
+static const char * const apll_i2s0_parents[] = {
+	"aud_1_sel",
+	"aud_2_sel"
+};
+
+static const char * const apll_i2s1_parents[] = {
+	"aud_1_sel",
+	"aud_2_sel"
+};
+
+static const char * const apll_i2s2_parents[] = {
+	"aud_1_sel",
+	"aud_2_sel"
+};
+
+static const char * const apll_i2s3_parents[] = {
+	"aud_1_sel",
+	"aud_2_sel"
+};
+
+static const char * const apll_i2s4_parents[] = {
+	"aud_1_sel",
+	"aud_2_sel"
+};
+
+static const char * const apll_i2s5_parents[] = {
+	"aud_1_sel",
+	"aud_2_sel"
+};
+
+static struct mtk_composite top_aud_muxes[] = {
+	MUX(CLK_TOP_MUX_APLL_I2S0, "apll_i2s0_sel", apll_i2s0_parents,
+		0x320, 8, 1),
+	MUX(CLK_TOP_MUX_APLL_I2S1, "apll_i2s1_sel", apll_i2s1_parents,
+		0x320, 9, 1),
+	MUX(CLK_TOP_MUX_APLL_I2S2, "apll_i2s2_sel", apll_i2s2_parents,
+		0x320, 10, 1),
+	MUX(CLK_TOP_MUX_APLL_I2S3, "apll_i2s3_sel", apll_i2s3_parents,
+		0x320, 11, 1),
+	MUX(CLK_TOP_MUX_APLL_I2S4, "apll_i2s4_sel", apll_i2s4_parents,
+		0x320, 12, 1),
+	MUX(CLK_TOP_MUX_APLL_I2S5, "apll_i2s5_sel", apll_i2s5_parents,
+		0x328, 20, 1),
+};
+
+static const char * const mcu_mp0_parents[] = {
+	"clk26m",
+	"armpll_ll",
+	"armpll_div_pll1",
+	"armpll_div_pll2"
+};
+
+static const char * const mcu_mp2_parents[] = {
+	"clk26m",
+	"armpll_l",
+	"armpll_div_pll1",
+	"armpll_div_pll2"
+};
+
+static const char * const mcu_bus_parents[] = {
+	"clk26m",
+	"ccipll",
+	"armpll_div_pll1",
+	"armpll_div_pll2"
+};
+
+static struct mtk_composite mcu_muxes[] = {
+	/* mp0_pll_divider_cfg */
+	MUX(CLK_MCU_MP0_SEL, "mcu_mp0_sel", mcu_mp0_parents, 0x7A0, 9, 2),
+	/* mp2_pll_divider_cfg */
+	MUX(CLK_MCU_MP2_SEL, "mcu_mp2_sel", mcu_mp2_parents, 0x7A8, 9, 2),
+	/* bus_pll_divider_cfg */
+	MUX(CLK_MCU_BUS_SEL, "mcu_bus_sel", mcu_bus_parents, 0x7C0, 9, 2),
+};
+
+static struct mtk_composite top_aud_divs[] = {
+	DIV_GATE(CLK_TOP_APLL12_DIV0, "apll12_div0", "apll_i2s0_sel",
+		0x320, 2, 0x324, 8, 0),
+	DIV_GATE(CLK_TOP_APLL12_DIV1, "apll12_div1", "apll_i2s1_sel",
+		0x320, 3, 0x324, 8, 8),
+	DIV_GATE(CLK_TOP_APLL12_DIV2, "apll12_div2", "apll_i2s2_sel",
+		0x320, 4, 0x324, 8, 16),
+	DIV_GATE(CLK_TOP_APLL12_DIV3, "apll12_div3", "apll_i2s3_sel",
+		0x320, 5, 0x324, 8, 24),
+	DIV_GATE(CLK_TOP_APLL12_DIV4, "apll12_div4", "apll_i2s4_sel",
+		0x320, 6, 0x328, 8, 0),
+	DIV_GATE(CLK_TOP_APLL12_DIVB, "apll12_divb", "apll12_div4",
+		0x320, 7, 0x328, 8, 8),
+};
+
+static const struct mtk_gate_regs top_cg_regs = {
+	.set_ofs = 0x104,
+	.clr_ofs = 0x104,
+	.sta_ofs = 0x104,
+};
+
+#define GATE_TOP(_id, _name, _parent, _shift)			\
+	GATE_MTK(_id, _name, _parent, &top_cg_regs, _shift,	\
+		&mtk_clk_gate_ops_no_setclr_inv)
+
+static const struct mtk_gate top_clks[] = {
+	/* TOP */
+	GATE_TOP(CLK_TOP_ARMPLL_DIV_PLL1, "armpll_div_pll1", "mainpll", 4),
+	GATE_TOP(CLK_TOP_ARMPLL_DIV_PLL2, "armpll_div_pll2", "univpll", 5),
+};
+
+static const struct mtk_gate_regs infra0_cg_regs = {
+	.set_ofs = 0x80,
+	.clr_ofs = 0x84,
+	.sta_ofs = 0x90,
+};
+
+static const struct mtk_gate_regs infra1_cg_regs = {
+	.set_ofs = 0x88,
+	.clr_ofs = 0x8c,
+	.sta_ofs = 0x94,
+};
+
+static const struct mtk_gate_regs infra2_cg_regs = {
+	.set_ofs = 0xa4,
+	.clr_ofs = 0xa8,
+	.sta_ofs = 0xac,
+};
+
+static const struct mtk_gate_regs infra3_cg_regs = {
+	.set_ofs = 0xc0,
+	.clr_ofs = 0xc4,
+	.sta_ofs = 0xc8,
+};
+
+#define GATE_INFRA0(_id, _name, _parent, _shift)		\
+	GATE_MTK(_id, _name, _parent, &infra0_cg_regs, _shift,	\
+		&mtk_clk_gate_ops_setclr)
+
+#define GATE_INFRA1(_id, _name, _parent, _shift)		\
+	GATE_MTK(_id, _name, _parent, &infra1_cg_regs, _shift,	\
+		&mtk_clk_gate_ops_setclr)
+
+#define GATE_INFRA2(_id, _name, _parent, _shift)		\
+	GATE_MTK(_id, _name, _parent, &infra2_cg_regs, _shift,	\
+		&mtk_clk_gate_ops_setclr)
+
+#define GATE_INFRA3(_id, _name, _parent, _shift)		\
+	GATE_MTK(_id, _name, _parent, &infra3_cg_regs, _shift,	\
+		&mtk_clk_gate_ops_setclr)
+
+static const struct mtk_gate infra_clks[] = {
+	/* INFRA0 */
+	GATE_INFRA0(CLK_INFRA_PMIC_TMR, "infra_pmic_tmr",
+		"axi_sel", 0),
+	GATE_INFRA0(CLK_INFRA_PMIC_AP, "infra_pmic_ap",
+		"axi_sel", 1),
+	GATE_INFRA0(CLK_INFRA_PMIC_MD, "infra_pmic_md",
+		"axi_sel", 2),
+	GATE_INFRA0(CLK_INFRA_PMIC_CONN, "infra_pmic_conn",
+		"axi_sel", 3),
+	GATE_INFRA0(CLK_INFRA_SCPSYS, "infra_scp",
+		"scp_sel", 4),
+	GATE_INFRA0(CLK_INFRA_SEJ, "infra_sej",
+		"f_f26m_ck", 5),
+	GATE_INFRA0(CLK_INFRA_APXGPT, "infra_apxgpt",
+		"axi_sel", 6),
+	GATE_INFRA0(CLK_INFRA_ICUSB, "infra_icusb",
+		"axi_sel", 8),
+	GATE_INFRA0(CLK_INFRA_GCE, "infra_gce",
+		"axi_sel", 9),
+	GATE_INFRA0(CLK_INFRA_THERM, "infra_therm",
+		"axi_sel", 10),
+	GATE_INFRA0(CLK_INFRA_I2C0, "infra_i2c0",
+		"i2c_sel", 11),
+	GATE_INFRA0(CLK_INFRA_I2C1, "infra_i2c1",
+		"i2c_sel", 12),
+	GATE_INFRA0(CLK_INFRA_I2C2, "infra_i2c2",
+		"i2c_sel", 13),
+	GATE_INFRA0(CLK_INFRA_I2C3, "infra_i2c3",
+		"i2c_sel", 14),
+	GATE_INFRA0(CLK_INFRA_PWM_HCLK, "infra_pwm_hclk",
+		"axi_sel", 15),
+	GATE_INFRA0(CLK_INFRA_PWM1, "infra_pwm1",
+		"i2c_sel", 16),
+	GATE_INFRA0(CLK_INFRA_PWM2, "infra_pwm2",
+		"i2c_sel", 17),
+	GATE_INFRA0(CLK_INFRA_PWM3, "infra_pwm3",
+		"i2c_sel", 18),
+	GATE_INFRA0(CLK_INFRA_PWM4, "infra_pwm4",
+		"i2c_sel", 19),
+	GATE_INFRA0(CLK_INFRA_PWM, "infra_pwm",
+		"i2c_sel", 21),
+	GATE_INFRA0(CLK_INFRA_UART0, "infra_uart0",
+		"uart_sel", 22),
+	GATE_INFRA0(CLK_INFRA_UART1, "infra_uart1",
+		"uart_sel", 23),
+	GATE_INFRA0(CLK_INFRA_UART2, "infra_uart2",
+		"uart_sel", 24),
+	GATE_INFRA0(CLK_INFRA_UART3, "infra_uart3",
+		"uart_sel", 25),
+	GATE_INFRA0(CLK_INFRA_GCE_26M, "infra_gce_26m",
+		"axi_sel", 27),
+	GATE_INFRA0(CLK_INFRA_CQ_DMA_FPC, "infra_cqdma_fpc",
+		"axi_sel", 28),
+	GATE_INFRA0(CLK_INFRA_BTIF, "infra_btif",
+		"axi_sel", 31),
+	/* INFRA1 */
+	GATE_INFRA1(CLK_INFRA_SPI0, "infra_spi0",
+		"spi_sel", 1),
+	GATE_INFRA1(CLK_INFRA_MSDC0, "infra_msdc0",
+		"msdc50_hclk_sel", 2),
+	GATE_INFRA1(CLK_INFRA_MSDC1, "infra_msdc1",
+		"axi_sel", 4),
+	GATE_INFRA1(CLK_INFRA_MSDC2, "infra_msdc2",
+		"axi_sel", 5),
+	GATE_INFRA1(CLK_INFRA_MSDC0_SCK, "infra_msdc0_sck",
+		"msdc50_0_sel", 6),
+	GATE_INFRA1(CLK_INFRA_DVFSRC, "infra_dvfsrc",
+		"f_f26m_ck", 7),
+	GATE_INFRA1(CLK_INFRA_GCPU, "infra_gcpu",
+		"axi_sel", 8),
+	GATE_INFRA1(CLK_INFRA_TRNG, "infra_trng",
+		"axi_sel", 9),
+	GATE_INFRA1(CLK_INFRA_AUXADC, "infra_auxadc",
+		"f_f26m_ck", 10),
+	GATE_INFRA1(CLK_INFRA_CPUM, "infra_cpum",
+		"axi_sel", 11),
+	GATE_INFRA1(CLK_INFRA_CCIF1_AP, "infra_ccif1_ap",
+		"axi_sel", 12),
+	GATE_INFRA1(CLK_INFRA_CCIF1_MD, "infra_ccif1_md",
+		"axi_sel", 13),
+	GATE_INFRA1(CLK_INFRA_AUXADC_MD, "infra_auxadc_md",
+		"f_f26m_ck", 14),
+	GATE_INFRA1(CLK_INFRA_MSDC1_SCK, "infra_msdc1_sck",
+		"msdc30_1_sel", 16),
+	GATE_INFRA1(CLK_INFRA_MSDC2_SCK, "infra_msdc2_sck",
+		"msdc30_2_sel", 17),
+	GATE_INFRA1(CLK_INFRA_AP_DMA, "infra_apdma",
+		"axi_sel", 18),
+	GATE_INFRA1(CLK_INFRA_XIU, "infra_xiu",
+		"axi_sel", 19),
+	GATE_INFRA1(CLK_INFRA_DEVICE_APC, "infra_device_apc",
+		"axi_sel", 20),
+	GATE_INFRA1(CLK_INFRA_CCIF_AP, "infra_ccif_ap",
+		"axi_sel", 23),
+	GATE_INFRA1(CLK_INFRA_DEBUGSYS, "infra_debugsys",
+		"axi_sel", 24),
+	GATE_INFRA1(CLK_INFRA_AUDIO, "infra_audio",
+		"axi_sel", 25),
+	GATE_INFRA1(CLK_INFRA_CCIF_MD, "infra_ccif_md",
+		"axi_sel", 26),
+	GATE_INFRA1(CLK_INFRA_DXCC_SEC_CORE, "infra_dxcc_sec_core",
+		"dxcc_sel", 27),
+	GATE_INFRA1(CLK_INFRA_DXCC_AO, "infra_dxcc_ao",
+		"dxcc_sel", 28),
+	GATE_INFRA1(CLK_INFRA_DEVMPU_BCLK, "infra_devmpu_bclk",
+		"axi_sel", 30),
+	GATE_INFRA1(CLK_INFRA_DRAMC_F26M, "infra_dramc_f26m",
+		"f_f26m_ck", 31),
+	/* INFRA2 */
+	GATE_INFRA2(CLK_INFRA_IRTX, "infra_irtx",
+		"f_f26m_ck", 0),
+	GATE_INFRA2(CLK_INFRA_USB, "infra_usb",
+		"usb_top_sel", 1),
+	GATE_INFRA2(CLK_INFRA_DISP_PWM, "infra_disppwm",
+		"axi_sel", 2),
+	GATE_INFRA2(CLK_INFRA_CLDMA_BCLK, "infra_cldma_bclk",
+		"axi_sel", 3),
+	GATE_INFRA2(CLK_INFRA_AUDIO_26M_BCLK, "infra_audio_26m_bclk",
+		"f_f26m_ck", 4),
+	GATE_INFRA2(CLK_INFRA_SPI1, "infra_spi1",
+		"spi_sel", 6),
+	GATE_INFRA2(CLK_INFRA_I2C4, "infra_i2c4",
+		"i2c_sel", 7),
+	GATE_INFRA2(CLK_INFRA_MODEM_TEMP_SHARE, "infra_md_tmp_share",
+		"f_f26m_ck", 8),
+	GATE_INFRA2(CLK_INFRA_SPI2, "infra_spi2",
+		"spi_sel", 9),
+	GATE_INFRA2(CLK_INFRA_SPI3, "infra_spi3",
+		"spi_sel", 10),
+	GATE_INFRA2(CLK_INFRA_UNIPRO_SCK, "infra_unipro_sck",
+		"ssusb_top_xhci_sel", 11),
+	GATE_INFRA2(CLK_INFRA_UNIPRO_TICK, "infra_unipro_tick",
+		"fufs_sel", 12),
+	GATE_INFRA2(CLK_INFRA_UFS_MP_SAP_BCLK, "infra_ufs_mp_sap_bck",
+		"fufs_sel", 13),
+	GATE_INFRA2(CLK_INFRA_MD32_BCLK, "infra_md32_bclk",
+		"axi_sel", 14),
+	GATE_INFRA2(CLK_INFRA_SSPM, "infra_sspm",
+		"sspm_sel", 15),
+	GATE_INFRA2(CLK_INFRA_UNIPRO_MBIST, "infra_unipro_mbist",
+		"axi_sel", 16),
+	GATE_INFRA2(CLK_INFRA_SSPM_BUS_HCLK, "infra_sspm_bus_hclk",
+		"axi_sel", 17),
+	GATE_INFRA2(CLK_INFRA_I2C5, "infra_i2c5",
+		"i2c_sel", 18),
+	GATE_INFRA2(CLK_INFRA_I2C5_ARBITER, "infra_i2c5_arbiter",
+		"i2c_sel", 19),
+	GATE_INFRA2(CLK_INFRA_I2C5_IMM, "infra_i2c5_imm",
+		"i2c_sel", 20),
+	GATE_INFRA2(CLK_INFRA_I2C1_ARBITER, "infra_i2c1_arbiter",
+		"i2c_sel", 21),
+	GATE_INFRA2(CLK_INFRA_I2C1_IMM, "infra_i2c1_imm",
+		"i2c_sel", 22),
+	GATE_INFRA2(CLK_INFRA_I2C2_ARBITER, "infra_i2c2_arbiter",
+		"i2c_sel", 23),
+	GATE_INFRA2(CLK_INFRA_I2C2_IMM, "infra_i2c2_imm",
+		"i2c_sel", 24),
+	GATE_INFRA2(CLK_INFRA_SPI4, "infra_spi4",
+		"spi_sel", 25),
+	GATE_INFRA2(CLK_INFRA_SPI5, "infra_spi5",
+		"spi_sel", 26),
+	GATE_INFRA2(CLK_INFRA_CQ_DMA, "infra_cqdma",
+		"axi_sel", 27),
+	GATE_INFRA2(CLK_INFRA_UFS, "infra_ufs",
+		"fufs_sel", 28),
+	GATE_INFRA2(CLK_INFRA_AES_UFSFDE, "infra_aes_ufsfde",
+		"faes_ufsfde_sel", 29),
+	GATE_INFRA2(CLK_INFRA_UFS_TICK, "infra_ufs_tick",
+		"fufs_sel", 30),
+	/* INFRA3 */
+	GATE_INFRA3(CLK_INFRA_MSDC0_SELF, "infra_msdc0_self",
+		"msdc50_0_sel", 0),
+	GATE_INFRA3(CLK_INFRA_MSDC1_SELF, "infra_msdc1_self",
+		"msdc50_0_sel", 1),
+	GATE_INFRA3(CLK_INFRA_MSDC2_SELF, "infra_msdc2_self",
+		"msdc50_0_sel", 2),
+	GATE_INFRA3(CLK_INFRA_SSPM_26M_SELF, "infra_sspm_26m_self",
+		"f_f26m_ck", 3),
+	GATE_INFRA3(CLK_INFRA_SSPM_32K_SELF, "infra_sspm_32k_self",
+		"f_f26m_ck", 4),
+	GATE_INFRA3(CLK_INFRA_UFS_AXI, "infra_ufs_axi",
+		"axi_sel", 5),
+	GATE_INFRA3(CLK_INFRA_I2C6, "infra_i2c6",
+		"i2c_sel", 6),
+	GATE_INFRA3(CLK_INFRA_AP_MSDC0, "infra_ap_msdc0",
+		"msdc50_hclk_sel", 7),
+	GATE_INFRA3(CLK_INFRA_MD_MSDC0, "infra_md_msdc0",
+		"msdc50_hclk_sel", 8),
+	GATE_INFRA3(CLK_INFRA_CCIF2_AP, "infra_ccif2_ap",
+		"axi_sel", 16),
+	GATE_INFRA3(CLK_INFRA_CCIF2_MD, "infra_ccif2_md",
+		"axi_sel", 17),
+	GATE_INFRA3(CLK_INFRA_CCIF3_AP, "infra_ccif3_ap",
+		"axi_sel", 18),
+	GATE_INFRA3(CLK_INFRA_CCIF3_MD, "infra_ccif3_md",
+		"axi_sel", 19),
+	GATE_INFRA3(CLK_INFRA_SEJ_F13M, "infra_sej_f13m",
+		"f_f26m_ck", 20),
+	GATE_INFRA3(CLK_INFRA_AES_BCLK, "infra_aes_bclk",
+		"axi_sel", 21),
+	GATE_INFRA3(CLK_INFRA_I2C7, "infra_i2c7",
+		"i2c_sel", 22),
+	GATE_INFRA3(CLK_INFRA_I2C8, "infra_i2c8",
+		"i2c_sel", 23),
+	GATE_INFRA3(CLK_INFRA_FBIST2FPC, "infra_fbist2fpc",
+		"msdc50_0_sel", 24),
+};
+
+static const struct mtk_gate_regs apmixed_cg_regs = {
+	.set_ofs = 0x20,
+	.clr_ofs = 0x20,
+	.sta_ofs = 0x20,
+};
+
+#define GATE_APMIXED_FLAGS(_id, _name, _parent, _shift, _flags)	\
+	GATE_MTK_FLAGS(_id, _name, _parent, &apmixed_cg_regs,		\
+		_shift, &mtk_clk_gate_ops_no_setclr_inv, _flags)
+
+#define GATE_APMIXED(_id, _name, _parent, _shift)	\
+	GATE_APMIXED_FLAGS(_id, _name, _parent, _shift,	0)
+
+/*
+ * CRITICAL CLOCK:
+ * apmixed_appll26m is the toppest clock gate of all PLLs.
+ */
+static const struct mtk_gate apmixed_clks[] = {
+	/* AUDIO0 */
+	GATE_APMIXED(CLK_APMIXED_SSUSB_26M, "apmixed_ssusb26m",
+		"f_f26m_ck", 4),
+	GATE_APMIXED_FLAGS(CLK_APMIXED_APPLL_26M, "apmixed_appll26m",
+		"f_f26m_ck", 5, CLK_IS_CRITICAL),
+	GATE_APMIXED(CLK_APMIXED_MIPIC0_26M, "apmixed_mipic026m",
+		"f_f26m_ck", 6),
+	GATE_APMIXED(CLK_APMIXED_MDPLLGP_26M, "apmixed_mdpll26m",
+		"f_f26m_ck", 7),
+	GATE_APMIXED(CLK_APMIXED_MMSYS_26M, "apmixed_mmsys26m",
+		"f_f26m_ck", 8),
+	GATE_APMIXED(CLK_APMIXED_UFS_26M, "apmixed_ufs26m",
+		"f_f26m_ck", 9),
+	GATE_APMIXED(CLK_APMIXED_MIPIC1_26M, "apmixed_mipic126m",
+		"f_f26m_ck", 11),
+	GATE_APMIXED(CLK_APMIXED_MEMPLL_26M, "apmixed_mempll26m",
+		"f_f26m_ck", 13),
+	GATE_APMIXED(CLK_APMIXED_CLKSQ_LVPLL_26M, "apmixed_lvpll26m",
+		"f_f26m_ck", 14),
+	GATE_APMIXED(CLK_APMIXED_MIPID0_26M, "apmixed_mipid026m",
+		"f_f26m_ck", 16),
+	GATE_APMIXED(CLK_APMIXED_MIPID1_26M, "apmixed_mipid126m",
+		"f_f26m_ck", 17),
+};
+
+#define MT8183_PLL_FMAX		(3800UL * MHZ)
+#define MT8183_PLL_FMIN		(1500UL * MHZ)
+
+#define PLL_B(_id, _name, _reg, _pwr_reg, _en_mask, _flags,		\
+			_rst_bar_mask, _pcwbits, _pcwibits, _pd_reg,	\
+			_pd_shift, _tuner_reg,  _tuner_en_reg,		\
+			_tuner_en_bit, _pcw_reg, _pcw_shift,		\
+			_pcw_chg_reg, _div_table) {			\
+		.id = _id,						\
+		.name = _name,						\
+		.reg = _reg,						\
+		.pwr_reg = _pwr_reg,					\
+		.en_mask = _en_mask,					\
+		.flags = _flags,					\
+		.rst_bar_mask = _rst_bar_mask,				\
+		.fmax = MT8183_PLL_FMAX,				\
+		.fmin = MT8183_PLL_FMIN,				\
+		.pcwbits = _pcwbits,					\
+		.pcwibits = _pcwibits,					\
+		.pd_reg = _pd_reg,					\
+		.pd_shift = _pd_shift,					\
+		.tuner_reg = _tuner_reg,				\
+		.tuner_en_reg = _tuner_en_reg,				\
+		.tuner_en_bit = _tuner_en_bit,				\
+		.pcw_reg = _pcw_reg,					\
+		.pcw_shift = _pcw_shift,				\
+		.pcw_chg_reg = _pcw_chg_reg,				\
+		.div_table = _div_table,				\
+	}
+
+#define PLL(_id, _name, _reg, _pwr_reg, _en_mask, _flags,		\
+			_rst_bar_mask, _pcwbits, _pcwibits, _pd_reg,	\
+			_pd_shift, _tuner_reg, _tuner_en_reg,		\
+			_tuner_en_bit, _pcw_reg, _pcw_shift,		\
+			_pcw_chg_reg)					\
+		PLL_B(_id, _name, _reg, _pwr_reg, _en_mask, _flags,	\
+			_rst_bar_mask, _pcwbits, _pcwibits, _pd_reg,	\
+			_pd_shift, _tuner_reg, _tuner_en_reg,		\
+			_tuner_en_bit, _pcw_reg, _pcw_shift,		\
+			_pcw_chg_reg, NULL)
+
+static const struct mtk_pll_div_table armpll_div_table[] = {
+	{ .div = 0, .freq = MT8183_PLL_FMAX },
+	{ .div = 1, .freq = 1500 * MHZ },
+	{ .div = 2, .freq = 750 * MHZ },
+	{ .div = 3, .freq = 375 * MHZ },
+	{ .div = 4, .freq = 187500000 },
+	{ } /* sentinel */
+};
+
+static const struct mtk_pll_div_table mfgpll_div_table[] = {
+	{ .div = 0, .freq = MT8183_PLL_FMAX },
+	{ .div = 1, .freq = 1600 * MHZ },
+	{ .div = 2, .freq = 800 * MHZ },
+	{ .div = 3, .freq = 400 * MHZ },
+	{ .div = 4, .freq = 200 * MHZ },
+	{ } /* sentinel */
+};
+
+static const struct mtk_pll_data plls[] = {
+	PLL_B(CLK_APMIXED_ARMPLL_LL, "armpll_ll", 0x0200, 0x020C, 0x00000001,
+		HAVE_RST_BAR | PLL_AO, BIT(24), 22, 8, 0x0204, 24, 0x0, 0x0, 0,
+		0x0204, 0, 0, armpll_div_table),
+	PLL_B(CLK_APMIXED_ARMPLL_L, "armpll_l", 0x0210, 0x021C, 0x00000001,
+		HAVE_RST_BAR | PLL_AO, BIT(24), 22, 8, 0x0214, 24, 0x0, 0x0, 0,
+		0x0214, 0, 0, armpll_div_table),
+	PLL(CLK_APMIXED_CCIPLL, "ccipll", 0x0290, 0x029C, 0x00000001,
+		HAVE_RST_BAR | PLL_AO, BIT(24), 22, 8, 0x0294, 24, 0x0, 0x0, 0,
+		0x0294, 0, 0),
+	PLL(CLK_APMIXED_MAINPLL, "mainpll", 0x0220, 0x022C, 0x00000001,
+		HAVE_RST_BAR, BIT(24), 22, 8, 0x0224, 24, 0x0, 0x0, 0,
+		0x0224, 0, 0),
+	PLL(CLK_APMIXED_UNIV2PLL, "univ2pll", 0x0230, 0x023C, 0x00000001,
+		HAVE_RST_BAR, BIT(24), 22, 8, 0x0234, 24, 0x0, 0x0, 0,
+		0x0234, 0, 0),
+	PLL_B(CLK_APMIXED_MFGPLL, "mfgpll", 0x0240, 0x024C, 0x00000001,
+		0, 0, 22, 8, 0x0244, 24, 0x0, 0x0, 0, 0x0244, 0, 0,
+		mfgpll_div_table),
+	PLL(CLK_APMIXED_MSDCPLL, "msdcpll", 0x0250, 0x025C, 0x00000001,
+		0, 0, 22, 8, 0x0254, 24, 0x0, 0x0, 0, 0x0254, 0, 0),
+	PLL(CLK_APMIXED_TVDPLL, "tvdpll", 0x0260, 0x026C, 0x00000001,
+		0, 0, 22, 8, 0x0264, 24, 0x0, 0x0, 0, 0x0264, 0, 0),
+	PLL(CLK_APMIXED_MMPLL, "mmpll", 0x0270, 0x027C, 0x00000001,
+		HAVE_RST_BAR, BIT(23), 22, 8, 0x0274, 24, 0x0, 0x0, 0,
+		0x0274, 0, 0),
+	PLL(CLK_APMIXED_APLL1, "apll1", 0x02A0, 0x02B0, 0x00000001,
+		0, 0, 32, 8, 0x02A0, 1, 0x02A8, 0x0014, 0, 0x02A4, 0, 0x02A0),
+	PLL(CLK_APMIXED_APLL2, "apll2", 0x02b4, 0x02c4, 0x00000001,
+		0, 0, 32, 8, 0x02B4, 1, 0x02BC, 0x0014, 1, 0x02B8, 0, 0x02B4),
+};
+
+static int clk_mt8183_apmixed_probe(struct platform_device *pdev)
+{
+	struct clk_onecell_data *clk_data;
+	struct device_node *node = pdev->dev.of_node;
+
+	clk_data = mtk_alloc_clk_data(CLK_APMIXED_NR_CLK);
+
+	mtk_clk_register_plls(node, plls, ARRAY_SIZE(plls), clk_data);
+
+	mtk_clk_register_gates(node, apmixed_clks, ARRAY_SIZE(apmixed_clks),
+		clk_data);
+
+	return of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+}
+
+static int clk_mt8183_top_probe(struct platform_device *pdev)
+{
+	struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	void __iomem *base;
+	struct clk_onecell_data *clk_data;
+	struct device_node *node = pdev->dev.of_node;
+
+	base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(base))
+		return PTR_ERR(base);
+
+	clk_data = mtk_alloc_clk_data(CLK_TOP_NR_CLK);
+
+	mtk_clk_register_fixed_clks(top_fixed_clks, ARRAY_SIZE(top_fixed_clks),
+		clk_data);
+
+	mtk_clk_register_factors(top_divs, ARRAY_SIZE(top_divs), clk_data);
+
+	mtk_clk_register_muxes(top_muxes, ARRAY_SIZE(top_muxes),
+		node, &mt8183_clk_lock, clk_data);
+
+	mtk_clk_register_composites(top_aud_muxes, ARRAY_SIZE(top_aud_muxes),
+		base, &mt8183_clk_lock, clk_data);
+
+	mtk_clk_register_composites(top_aud_divs, ARRAY_SIZE(top_aud_divs),
+		base, &mt8183_clk_lock, clk_data);
+
+	mtk_clk_register_gates(node, top_clks, ARRAY_SIZE(top_clks),
+		clk_data);
+
+	return of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+}
+
+static int clk_mt8183_infra_probe(struct platform_device *pdev)
+{
+	struct clk_onecell_data *clk_data;
+	struct device_node *node = pdev->dev.of_node;
+
+	clk_data = mtk_alloc_clk_data(CLK_INFRA_NR_CLK);
+
+	mtk_clk_register_gates(node, infra_clks, ARRAY_SIZE(infra_clks),
+		clk_data);
+
+	return of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+}
+
+static int clk_mt8183_mcu_probe(struct platform_device *pdev)
+{
+	struct clk_onecell_data *clk_data;
+	struct device_node *node = pdev->dev.of_node;
+	void __iomem *base;
+	struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+
+	base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(base))
+		return PTR_ERR(base);
+
+	clk_data = mtk_alloc_clk_data(CLK_MCU_NR_CLK);
+
+	mtk_clk_register_composites(mcu_muxes, ARRAY_SIZE(mcu_muxes), base,
+			&mt8183_clk_lock, clk_data);
+
+	return of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+}
+
+static const struct of_device_id of_match_clk_mt8183[] = {
+	{
+		.compatible = "mediatek,mt8183-apmixedsys",
+		.data = clk_mt8183_apmixed_probe,
+	}, {
+		.compatible = "mediatek,mt8183-topckgen",
+		.data = clk_mt8183_top_probe,
+	}, {
+		.compatible = "mediatek,mt8183-infracfg",
+		.data = clk_mt8183_infra_probe,
+	}, {
+		.compatible = "mediatek,mt8183-mcucfg",
+		.data = clk_mt8183_mcu_probe,
+	}, {
+		/* sentinel */
+	}
+};
+
+static int clk_mt8183_probe(struct platform_device *pdev)
+{
+	int (*clk_probe)(struct platform_device *pdev);
+	int r;
+
+	clk_probe = of_device_get_match_data(&pdev->dev);
+	if (!clk_probe)
+		return -EINVAL;
+
+	r = clk_probe(pdev);
+	if (r)
+		dev_err(&pdev->dev,
+			"could not register clock provider: %s: %d\n",
+			pdev->name, r);
+
+	return r;
+}
+
+static struct platform_driver clk_mt8183_drv = {
+	.probe = clk_mt8183_probe,
+	.driver = {
+		.name = "clk-mt8183",
+		.of_match_table = of_match_clk_mt8183,
+	},
+};
+
+static int __init clk_mt8183_init(void)
+{
+	return platform_driver_register(&clk_mt8183_drv);
+}
+
+arch_initcall(clk_mt8183_init);
diff --git a/drivers/clk/mediatek/clk-mt8516.c b/drivers/clk/mediatek/clk-mt8516.c
new file mode 100644
index 000000000000..26fe43cc9ea2
--- /dev/null
+++ b/drivers/clk/mediatek/clk-mt8516.c
@@ -0,0 +1,815 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2019 MediaTek Inc.
+ * Author: James Liao <jamesjj.liao@mediatek.com>
+ *         Fabien Parent <fparent@baylibre.com>
+ */
+
+#include <linux/delay.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/slab.h>
+#include <linux/mfd/syscon.h>
+
+#include "clk-mtk.h"
+#include "clk-gate.h"
+
+#include <dt-bindings/clock/mt8516-clk.h>
+
+static DEFINE_SPINLOCK(mt8516_clk_lock);
+
+static const struct mtk_fixed_clk fixed_clks[] __initconst = {
+	FIXED_CLK(CLK_TOP_CLK_NULL, "clk_null", NULL, 0),
+	FIXED_CLK(CLK_TOP_I2S_INFRA_BCK, "i2s_infra_bck", "clk_null", 26000000),
+	FIXED_CLK(CLK_TOP_MEMPLL, "mempll", "clk26m", 800000000),
+};
+
+static const struct mtk_fixed_factor top_divs[] __initconst = {
+	FACTOR(CLK_TOP_DMPLL, "dmpll_ck", "mempll", 1, 1),
+	FACTOR(CLK_TOP_MAINPLL_D2, "mainpll_d2", "mainpll", 1, 2),
+	FACTOR(CLK_TOP_MAINPLL_D4, "mainpll_d4", "mainpll", 1, 4),
+	FACTOR(CLK_TOP_MAINPLL_D8, "mainpll_d8", "mainpll", 1, 8),
+	FACTOR(CLK_TOP_MAINPLL_D16, "mainpll_d16", "mainpll", 1, 16),
+	FACTOR(CLK_TOP_MAINPLL_D11, "mainpll_d11", "mainpll", 1, 11),
+	FACTOR(CLK_TOP_MAINPLL_D22, "mainpll_d22", "mainpll", 1, 22),
+	FACTOR(CLK_TOP_MAINPLL_D3, "mainpll_d3", "mainpll", 1, 3),
+	FACTOR(CLK_TOP_MAINPLL_D6, "mainpll_d6", "mainpll", 1, 6),
+	FACTOR(CLK_TOP_MAINPLL_D12, "mainpll_d12", "mainpll", 1, 12),
+	FACTOR(CLK_TOP_MAINPLL_D5, "mainpll_d5", "mainpll", 1, 5),
+	FACTOR(CLK_TOP_MAINPLL_D10, "mainpll_d10", "mainpll", 1, 10),
+	FACTOR(CLK_TOP_MAINPLL_D20, "mainpll_d20", "mainpll", 1, 20),
+	FACTOR(CLK_TOP_MAINPLL_D40, "mainpll_d40", "mainpll", 1, 40),
+	FACTOR(CLK_TOP_MAINPLL_D7, "mainpll_d7", "mainpll", 1, 7),
+	FACTOR(CLK_TOP_MAINPLL_D14, "mainpll_d14", "mainpll", 1, 14),
+	FACTOR(CLK_TOP_UNIVPLL_D2, "univpll_d2", "univpll", 1, 2),
+	FACTOR(CLK_TOP_UNIVPLL_D4, "univpll_d4", "univpll", 1, 4),
+	FACTOR(CLK_TOP_UNIVPLL_D8, "univpll_d8", "univpll", 1, 8),
+	FACTOR(CLK_TOP_UNIVPLL_D16, "univpll_d16", "univpll", 1, 16),
+	FACTOR(CLK_TOP_UNIVPLL_D3, "univpll_d3", "univpll", 1, 3),
+	FACTOR(CLK_TOP_UNIVPLL_D6, "univpll_d6", "univpll", 1, 6),
+	FACTOR(CLK_TOP_UNIVPLL_D12, "univpll_d12", "univpll", 1, 12),
+	FACTOR(CLK_TOP_UNIVPLL_D24, "univpll_d24", "univpll", 1, 24),
+	FACTOR(CLK_TOP_UNIVPLL_D5, "univpll_d5", "univpll", 1, 5),
+	FACTOR(CLK_TOP_UNIVPLL_D20, "univpll_d20", "univpll", 1, 20),
+	FACTOR(CLK_TOP_MMPLL380M, "mmpll380m", "mmpll", 1, 1),
+	FACTOR(CLK_TOP_MMPLL_D2, "mmpll_d2", "mmpll", 1, 2),
+	FACTOR(CLK_TOP_MMPLL_200M, "mmpll_200m", "mmpll", 1, 3),
+	FACTOR(CLK_TOP_USB_PHY48M, "usb_phy48m_ck", "univpll", 1, 26),
+	FACTOR(CLK_TOP_APLL1, "apll1_ck", "apll1", 1, 1),
+	FACTOR(CLK_TOP_APLL1_D2, "apll1_d2", "apll1_ck", 1, 2),
+	FACTOR(CLK_TOP_APLL1_D4, "apll1_d4", "rg_apll1_d2_en", 1, 2),
+	FACTOR(CLK_TOP_APLL1_D8, "apll1_d8", "rg_apll1_d4_en", 1, 2),
+	FACTOR(CLK_TOP_APLL2, "apll2_ck", "apll2", 1, 1),
+	FACTOR(CLK_TOP_APLL2_D2, "apll2_d2", "apll2_ck", 1, 2),
+	FACTOR(CLK_TOP_APLL2_D4, "apll2_d4", "rg_apll2_d2_en", 1, 2),
+	FACTOR(CLK_TOP_APLL2_D8, "apll2_d8", "rg_apll2_d4_en", 1, 2),
+	FACTOR(CLK_TOP_CLK26M, "clk26m_ck", "clk26m", 1, 1),
+	FACTOR(CLK_TOP_CLK26M_D2, "clk26m_d2", "clk26m", 1, 2),
+	FACTOR(CLK_TOP_AHB_INFRA_D2, "ahb_infra_d2", "ahb_infra_sel", 1, 2),
+	FACTOR(CLK_TOP_NFI1X, "nfi1x_ck", "nfi2x_pad_sel", 1, 2),
+	FACTOR(CLK_TOP_ETH_D2, "eth_d2_ck", "eth_sel", 1, 2),
+};
+
+static const char * const uart0_parents[] __initconst = {
+	"clk26m_ck",
+	"univpll_d24"
+};
+
+static const char * const ahb_infra_parents[] __initconst = {
+	"clk_null",
+	"clk26m_ck",
+	"mainpll_d11",
+	"clk_null",
+	"mainpll_d12",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"mainpll_d10"
+};
+
+static const char * const msdc0_parents[] __initconst = {
+	"clk26m_ck",
+	"univpll_d6",
+	"mainpll_d8",
+	"univpll_d8",
+	"mainpll_d16",
+	"mmpll_200m",
+	"mainpll_d12",
+	"mmpll_d2"
+};
+
+static const char * const uart1_parents[] __initconst = {
+	"clk26m_ck",
+	"univpll_d24"
+};
+
+static const char * const msdc1_parents[] __initconst = {
+	"clk26m_ck",
+	"univpll_d6",
+	"mainpll_d8",
+	"univpll_d8",
+	"mainpll_d16",
+	"mmpll_200m",
+	"mainpll_d12",
+	"mmpll_d2"
+};
+
+static const char * const pmicspi_parents[] __initconst = {
+	"univpll_d20",
+	"usb_phy48m_ck",
+	"univpll_d16",
+	"clk26m_ck"
+};
+
+static const char * const qaxi_aud26m_parents[] __initconst = {
+	"clk26m_ck",
+	"ahb_infra_sel"
+};
+
+static const char * const aud_intbus_parents[] __initconst = {
+	"clk_null",
+	"clk26m_ck",
+	"mainpll_d22",
+	"clk_null",
+	"mainpll_d11"
+};
+
+static const char * const nfi2x_pad_parents[] __initconst = {
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk26m_ck",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"mainpll_d12",
+	"mainpll_d8",
+	"clk_null",
+	"mainpll_d6",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"mainpll_d4",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"clk_null",
+	"mainpll_d10",
+	"mainpll_d7",
+	"clk_null",
+	"mainpll_d5"
+};
+
+static const char * const nfi1x_pad_parents[] __initconst = {
+	"ahb_infra_sel",
+	"nfi1x_ck"
+};
+
+static const char * const ddrphycfg_parents[] __initconst = {
+	"clk26m_ck",
+	"mainpll_d16"
+};
+
+static const char * const usb_78m_parents[] __initconst = {
+	"clk_null",
+	"clk26m_ck",
+	"univpll_d16",
+	"clk_null",
+	"mainpll_d20"
+};
+
+static const char * const spinor_parents[] __initconst = {
+	"clk26m_d2",
+	"clk26m_ck",
+	"mainpll_d40",
+	"univpll_d24",
+	"univpll_d20",
+	"mainpll_d20",
+	"mainpll_d16",
+	"univpll_d12"
+};
+
+static const char * const msdc2_parents[] __initconst = {
+	"clk26m_ck",
+	"univpll_d6",
+	"mainpll_d8",
+	"univpll_d8",
+	"mainpll_d16",
+	"mmpll_200m",
+	"mainpll_d12",
+	"mmpll_d2"
+};
+
+static const char * const eth_parents[] __initconst = {
+	"clk26m_ck",
+	"mainpll_d40",
+	"univpll_d24",
+	"univpll_d20",
+	"mainpll_d20"
+};
+
+static const char * const aud1_parents[] __initconst = {
+	"clk26m_ck",
+	"apll1_ck"
+};
+
+static const char * const aud2_parents[] __initconst = {
+	"clk26m_ck",
+	"apll2_ck"
+};
+
+static const char * const aud_engen1_parents[] __initconst = {
+	"clk26m_ck",
+	"rg_apll1_d2_en",
+	"rg_apll1_d4_en",
+	"rg_apll1_d8_en"
+};
+
+static const char * const aud_engen2_parents[] __initconst = {
+	"clk26m_ck",
+	"rg_apll2_d2_en",
+	"rg_apll2_d4_en",
+	"rg_apll2_d8_en"
+};
+
+static const char * const i2c_parents[] __initconst = {
+	"clk26m_ck",
+	"univpll_d20",
+	"univpll_d16",
+	"univpll_d12"
+};
+
+static const char * const aud_i2s0_m_parents[] __initconst = {
+	"rg_aud1",
+	"rg_aud2"
+};
+
+static const char * const pwm_parents[] __initconst = {
+	"clk26m_ck",
+	"univpll_d12"
+};
+
+static const char * const spi_parents[] __initconst = {
+	"clk26m_ck",
+	"univpll_d12",
+	"univpll_d8",
+	"univpll_d6"
+};
+
+static const char * const aud_spdifin_parents[] __initconst = {
+	"clk26m_ck",
+	"univpll_d2"
+};
+
+static const char * const uart2_parents[] __initconst = {
+	"clk26m_ck",
+	"univpll_d24"
+};
+
+static const char * const bsi_parents[] __initconst = {
+	"clk26m_ck",
+	"mainpll_d10",
+	"mainpll_d12",
+	"mainpll_d20"
+};
+
+static const char * const dbg_atclk_parents[] __initconst = {
+	"clk_null",
+	"clk26m_ck",
+	"mainpll_d5",
+	"clk_null",
+	"univpll_d5"
+};
+
+static const char * const csw_nfiecc_parents[] __initconst = {
+	"clk_null",
+	"mainpll_d7",
+	"mainpll_d6",
+	"clk_null",
+	"mainpll_d5"
+};
+
+static const char * const nfiecc_parents[] __initconst = {
+	"clk_null",
+	"nfi2x_pad_sel",
+	"mainpll_d4",
+	"clk_null",
+	"csw_nfiecc_sel"
+};
+
+static struct mtk_composite top_muxes[] __initdata = {
+	/* CLK_MUX_SEL0 */
+	MUX(CLK_TOP_UART0_SEL, "uart0_sel", uart0_parents,
+		0x000, 0, 1),
+	MUX(CLK_TOP_AHB_INFRA_SEL, "ahb_infra_sel", ahb_infra_parents,
+		0x000, 4, 4),
+	MUX(CLK_TOP_MSDC0_SEL, "msdc0_sel", msdc0_parents,
+		0x000, 11, 3),
+	MUX(CLK_TOP_UART1_SEL, "uart1_sel", uart1_parents,
+		0x000, 19, 1),
+	MUX(CLK_TOP_MSDC1_SEL, "msdc1_sel", msdc1_parents,
+		0x000, 20, 3),
+	MUX(CLK_TOP_PMICSPI_SEL, "pmicspi_sel", pmicspi_parents,
+		0x000, 24, 2),
+	MUX(CLK_TOP_QAXI_AUD26M_SEL, "qaxi_aud26m_sel", qaxi_aud26m_parents,
+		0x000, 26, 1),
+	MUX(CLK_TOP_AUD_INTBUS_SEL, "aud_intbus_sel", aud_intbus_parents,
+		0x000, 27, 3),
+	/* CLK_MUX_SEL1 */
+	MUX(CLK_TOP_NFI2X_PAD_SEL, "nfi2x_pad_sel", nfi2x_pad_parents,
+		0x004, 0, 7),
+	MUX(CLK_TOP_NFI1X_PAD_SEL, "nfi1x_pad_sel", nfi1x_pad_parents,
+		0x004, 7, 1),
+	MUX(CLK_TOP_USB_78M_SEL, "usb_78m_sel", usb_78m_parents,
+		0x004, 20, 3),
+	/* CLK_MUX_SEL8 */
+	MUX(CLK_TOP_SPINOR_SEL, "spinor_sel", spinor_parents,
+		0x040, 0, 3),
+	MUX(CLK_TOP_MSDC2_SEL, "msdc2_sel", msdc2_parents,
+		0x040, 3, 3),
+	MUX(CLK_TOP_ETH_SEL, "eth_sel", eth_parents,
+		0x040, 6, 3),
+	MUX(CLK_TOP_AUD1_SEL, "aud1_sel", aud1_parents,
+		0x040, 22, 1),
+	MUX(CLK_TOP_AUD2_SEL, "aud2_sel", aud2_parents,
+		0x040, 23, 1),
+	MUX(CLK_TOP_AUD_ENGEN1_SEL, "aud_engen1_sel", aud_engen1_parents,
+		0x040, 24, 2),
+	MUX(CLK_TOP_AUD_ENGEN2_SEL, "aud_engen2_sel", aud_engen2_parents,
+		0x040, 26, 2),
+	MUX(CLK_TOP_I2C_SEL, "i2c_sel", i2c_parents,
+		0x040, 28, 2),
+	/* CLK_SEL_9 */
+	MUX(CLK_TOP_AUD_I2S0_M_SEL, "aud_i2s0_m_sel", aud_i2s0_m_parents,
+		0x044, 12, 1),
+	MUX(CLK_TOP_AUD_I2S1_M_SEL, "aud_i2s1_m_sel", aud_i2s0_m_parents,
+		0x044, 13, 1),
+	MUX(CLK_TOP_AUD_I2S2_M_SEL, "aud_i2s2_m_sel", aud_i2s0_m_parents,
+		0x044, 14, 1),
+	MUX(CLK_TOP_AUD_I2S3_M_SEL, "aud_i2s3_m_sel", aud_i2s0_m_parents,
+		0x044, 15, 1),
+	MUX(CLK_TOP_AUD_I2S4_M_SEL, "aud_i2s4_m_sel", aud_i2s0_m_parents,
+		0x044, 16, 1),
+	MUX(CLK_TOP_AUD_I2S5_M_SEL, "aud_i2s5_m_sel", aud_i2s0_m_parents,
+		0x044, 17, 1),
+	MUX(CLK_TOP_AUD_SPDIF_B_SEL, "aud_spdif_b_sel", aud_i2s0_m_parents,
+		0x044, 18, 1),
+	/* CLK_MUX_SEL13 */
+	MUX(CLK_TOP_PWM_SEL, "pwm_sel", pwm_parents,
+		0x07c, 0, 1),
+	MUX(CLK_TOP_SPI_SEL, "spi_sel", spi_parents,
+		0x07c, 1, 2),
+	MUX(CLK_TOP_AUD_SPDIFIN_SEL, "aud_spdifin_sel", aud_spdifin_parents,
+		0x07c, 3, 1),
+	MUX(CLK_TOP_UART2_SEL, "uart2_sel", uart2_parents,
+		0x07c, 4, 1),
+	MUX(CLK_TOP_BSI_SEL, "bsi_sel", bsi_parents,
+		0x07c, 5, 2),
+	MUX(CLK_TOP_DBG_ATCLK_SEL, "dbg_atclk_sel", dbg_atclk_parents,
+		0x07c, 7, 3),
+	MUX(CLK_TOP_CSW_NFIECC_SEL, "csw_nfiecc_sel", csw_nfiecc_parents,
+		0x07c, 10, 3),
+	MUX(CLK_TOP_NFIECC_SEL, "nfiecc_sel", nfiecc_parents,
+		0x07c, 13, 3),
+};
+
+static const char * const ifr_mux1_parents[] __initconst = {
+	"clk26m_ck",
+	"armpll",
+	"univpll",
+	"mainpll_d2"
+};
+
+static const char * const ifr_eth_25m_parents[] __initconst = {
+	"eth_d2_ck",
+	"rg_eth"
+};
+
+static const char * const ifr_i2c0_parents[] __initconst = {
+	"ahb_infra_d2",
+	"rg_i2c"
+};
+
+static const struct mtk_composite ifr_muxes[] __initconst = {
+	MUX(CLK_IFR_MUX1_SEL, "ifr_mux1_sel", ifr_mux1_parents, 0x000,
+		2, 2),
+	MUX(CLK_IFR_ETH_25M_SEL, "ifr_eth_25m_sel", ifr_eth_25m_parents, 0x080,
+		0, 1),
+	MUX(CLK_IFR_I2C0_SEL, "ifr_i2c0_sel", ifr_i2c0_parents, 0x080,
+		1, 1),
+	MUX(CLK_IFR_I2C1_SEL, "ifr_i2c1_sel", ifr_i2c0_parents, 0x080,
+		2, 1),
+	MUX(CLK_IFR_I2C2_SEL, "ifr_i2c2_sel", ifr_i2c0_parents, 0x080,
+		3, 1),
+};
+
+#define DIV_ADJ(_id, _name, _parent, _reg, _shift, _width) {	\
+		.id = _id,					\
+		.name = _name,					\
+		.parent_name = _parent,				\
+		.div_reg = _reg,				\
+		.div_shift = _shift,				\
+		.div_width = _width,				\
+}
+
+static const struct mtk_clk_divider top_adj_divs[] = {
+	DIV_ADJ(CLK_TOP_APLL12_CK_DIV0, "apll12_ck_div0", "aud_i2s0_m_sel",
+		0x0048, 0, 8),
+	DIV_ADJ(CLK_TOP_APLL12_CK_DIV1, "apll12_ck_div1", "aud_i2s1_m_sel",
+		0x0048, 8, 8),
+	DIV_ADJ(CLK_TOP_APLL12_CK_DIV2, "apll12_ck_div2", "aud_i2s2_m_sel",
+		0x0048, 16, 8),
+	DIV_ADJ(CLK_TOP_APLL12_CK_DIV3, "apll12_ck_div3", "aud_i2s3_m_sel",
+		0x0048, 24, 8),
+	DIV_ADJ(CLK_TOP_APLL12_CK_DIV4, "apll12_ck_div4", "aud_i2s4_m_sel",
+		0x004c, 0, 8),
+	DIV_ADJ(CLK_TOP_APLL12_CK_DIV4B, "apll12_ck_div4b", "apll12_div4",
+		0x004c, 8, 8),
+	DIV_ADJ(CLK_TOP_APLL12_CK_DIV5, "apll12_ck_div5", "aud_i2s5_m_sel",
+		0x004c, 16, 8),
+	DIV_ADJ(CLK_TOP_APLL12_CK_DIV5B, "apll12_ck_div5b", "apll12_div5",
+		0x004c, 24, 8),
+	DIV_ADJ(CLK_TOP_APLL12_CK_DIV6, "apll12_ck_div6", "aud_spdif_b_sel",
+		0x0078, 0, 8),
+};
+
+static const struct mtk_gate_regs top1_cg_regs = {
+	.set_ofs = 0x54,
+	.clr_ofs = 0x84,
+	.sta_ofs = 0x24,
+};
+
+static const struct mtk_gate_regs top2_cg_regs = {
+	.set_ofs = 0x6c,
+	.clr_ofs = 0x9c,
+	.sta_ofs = 0x3c,
+};
+
+static const struct mtk_gate_regs top3_cg_regs = {
+	.set_ofs = 0xa0,
+	.clr_ofs = 0xb0,
+	.sta_ofs = 0x70,
+};
+
+static const struct mtk_gate_regs top4_cg_regs = {
+	.set_ofs = 0xa4,
+	.clr_ofs = 0xb4,
+	.sta_ofs = 0x74,
+};
+
+static const struct mtk_gate_regs top5_cg_regs = {
+	.set_ofs = 0x44,
+	.clr_ofs = 0x44,
+	.sta_ofs = 0x44,
+};
+
+#define GATE_TOP1(_id, _name, _parent, _shift) {	\
+		.id = _id,				\
+		.name = _name,				\
+		.parent_name = _parent,			\
+		.regs = &top1_cg_regs,			\
+		.shift = _shift,			\
+		.ops = &mtk_clk_gate_ops_setclr,	\
+	}
+
+#define GATE_TOP2(_id, _name, _parent, _shift) {	\
+		.id = _id,				\
+		.name = _name,				\
+		.parent_name = _parent,			\
+		.regs = &top2_cg_regs,			\
+		.shift = _shift,			\
+		.ops = &mtk_clk_gate_ops_setclr,	\
+	}
+
+#define GATE_TOP2_I(_id, _name, _parent, _shift) {	\
+		.id = _id,				\
+		.name = _name,				\
+		.parent_name = _parent,			\
+		.regs = &top2_cg_regs,			\
+		.shift = _shift,			\
+		.ops = &mtk_clk_gate_ops_setclr_inv,	\
+	}
+
+#define GATE_TOP3(_id, _name, _parent, _shift) {	\
+		.id = _id,				\
+		.name = _name,				\
+		.parent_name = _parent,			\
+		.regs = &top3_cg_regs,			\
+		.shift = _shift,			\
+		.ops = &mtk_clk_gate_ops_setclr,	\
+	}
+
+#define GATE_TOP4_I(_id, _name, _parent, _shift) {	\
+		.id = _id,				\
+		.name = _name,				\
+		.parent_name = _parent,			\
+		.regs = &top4_cg_regs,			\
+		.shift = _shift,			\
+		.ops = &mtk_clk_gate_ops_setclr_inv,	\
+	}
+
+#define GATE_TOP5(_id, _name, _parent, _shift) {	\
+		.id = _id,				\
+		.name = _name,				\
+		.parent_name = _parent,			\
+		.regs = &top5_cg_regs,			\
+		.shift = _shift,			\
+		.ops = &mtk_clk_gate_ops_no_setclr,	\
+	}
+
+static const struct mtk_gate top_clks[] __initconst = {
+	/* TOP1 */
+	GATE_TOP1(CLK_TOP_THEM, "them", "ahb_infra_sel", 1),
+	GATE_TOP1(CLK_TOP_APDMA, "apdma", "ahb_infra_sel", 2),
+	GATE_TOP1(CLK_TOP_I2C0, "i2c0", "ifr_i2c0_sel", 3),
+	GATE_TOP1(CLK_TOP_I2C1, "i2c1", "ifr_i2c1_sel", 4),
+	GATE_TOP1(CLK_TOP_AUXADC1, "auxadc1", "ahb_infra_sel", 5),
+	GATE_TOP1(CLK_TOP_NFI, "nfi", "nfi1x_pad_sel", 6),
+	GATE_TOP1(CLK_TOP_NFIECC, "nfiecc", "rg_nfiecc", 7),
+	GATE_TOP1(CLK_TOP_DEBUGSYS, "debugsys", "rg_dbg_atclk", 8),
+	GATE_TOP1(CLK_TOP_PWM, "pwm", "ahb_infra_sel", 9),
+	GATE_TOP1(CLK_TOP_UART0, "uart0", "uart0_sel", 10),
+	GATE_TOP1(CLK_TOP_UART1, "uart1", "uart1_sel", 11),
+	GATE_TOP1(CLK_TOP_BTIF, "btif", "ahb_infra_sel", 12),
+	GATE_TOP1(CLK_TOP_USB, "usb", "usb_78m", 13),
+	GATE_TOP1(CLK_TOP_FLASHIF_26M, "flashif_26m", "clk26m_ck", 14),
+	GATE_TOP1(CLK_TOP_AUXADC2, "auxadc2", "ahb_infra_sel", 15),
+	GATE_TOP1(CLK_TOP_I2C2, "i2c2", "ifr_i2c2_sel", 16),
+	GATE_TOP1(CLK_TOP_MSDC0, "msdc0", "msdc0_sel", 17),
+	GATE_TOP1(CLK_TOP_MSDC1, "msdc1", "msdc1_sel", 18),
+	GATE_TOP1(CLK_TOP_NFI2X, "nfi2x", "nfi2x_pad_sel", 19),
+	GATE_TOP1(CLK_TOP_PMICWRAP_AP, "pwrap_ap", "clk26m_ck", 20),
+	GATE_TOP1(CLK_TOP_SEJ, "sej", "ahb_infra_sel", 21),
+	GATE_TOP1(CLK_TOP_MEMSLP_DLYER, "memslp_dlyer", "clk26m_ck", 22),
+	GATE_TOP1(CLK_TOP_SPI, "spi", "spi_sel", 23),
+	GATE_TOP1(CLK_TOP_APXGPT, "apxgpt", "clk26m_ck", 24),
+	GATE_TOP1(CLK_TOP_AUDIO, "audio", "clk26m_ck", 25),
+	GATE_TOP1(CLK_TOP_PMICWRAP_MD, "pwrap_md", "clk26m_ck", 27),
+	GATE_TOP1(CLK_TOP_PMICWRAP_CONN, "pwrap_conn", "clk26m_ck", 28),
+	GATE_TOP1(CLK_TOP_PMICWRAP_26M, "pwrap_26m", "clk26m_ck", 29),
+	GATE_TOP1(CLK_TOP_AUX_ADC, "aux_adc", "clk26m_ck", 30),
+	GATE_TOP1(CLK_TOP_AUX_TP, "aux_tp", "clk26m_ck", 31),
+	/* TOP2 */
+	GATE_TOP2(CLK_TOP_MSDC2, "msdc2", "ahb_infra_sel", 0),
+	GATE_TOP2(CLK_TOP_RBIST, "rbist", "univpll_d12", 1),
+	GATE_TOP2(CLK_TOP_NFI_BUS, "nfi_bus", "ahb_infra_sel", 2),
+	GATE_TOP2(CLK_TOP_GCE, "gce", "ahb_infra_sel", 4),
+	GATE_TOP2(CLK_TOP_TRNG, "trng", "ahb_infra_sel", 5),
+	GATE_TOP2(CLK_TOP_SEJ_13M, "sej_13m", "clk26m_ck", 6),
+	GATE_TOP2(CLK_TOP_AES, "aes", "ahb_infra_sel", 7),
+	GATE_TOP2(CLK_TOP_PWM_B, "pwm_b", "rg_pwm_infra", 8),
+	GATE_TOP2(CLK_TOP_PWM1_FB, "pwm1_fb", "rg_pwm_infra", 9),
+	GATE_TOP2(CLK_TOP_PWM2_FB, "pwm2_fb", "rg_pwm_infra", 10),
+	GATE_TOP2(CLK_TOP_PWM3_FB, "pwm3_fb", "rg_pwm_infra", 11),
+	GATE_TOP2(CLK_TOP_PWM4_FB, "pwm4_fb", "rg_pwm_infra", 12),
+	GATE_TOP2(CLK_TOP_PWM5_FB, "pwm5_fb", "rg_pwm_infra", 13),
+	GATE_TOP2(CLK_TOP_USB_1P, "usb_1p", "usb_78m", 14),
+	GATE_TOP2(CLK_TOP_FLASHIF_FREERUN, "flashif_freerun", "ahb_infra_sel",
+		15),
+	GATE_TOP2(CLK_TOP_66M_ETH, "eth_66m", "ahb_infra_d2", 19),
+	GATE_TOP2(CLK_TOP_133M_ETH, "eth_133m", "ahb_infra_sel", 20),
+	GATE_TOP2(CLK_TOP_FETH_25M, "feth_25m", "ifr_eth_25m_sel", 21),
+	GATE_TOP2(CLK_TOP_FETH_50M, "feth_50m", "rg_eth", 22),
+	GATE_TOP2(CLK_TOP_FLASHIF_AXI, "flashif_axi", "ahb_infra_sel", 23),
+	GATE_TOP2(CLK_TOP_USBIF, "usbif", "ahb_infra_sel", 24),
+	GATE_TOP2(CLK_TOP_UART2, "uart2", "rg_uart2", 25),
+	GATE_TOP2(CLK_TOP_BSI, "bsi", "ahb_infra_sel", 26),
+	GATE_TOP2_I(CLK_TOP_MSDC0_INFRA, "msdc0_infra", "msdc0", 28),
+	GATE_TOP2_I(CLK_TOP_MSDC1_INFRA, "msdc1_infra", "msdc1", 29),
+	GATE_TOP2_I(CLK_TOP_MSDC2_INFRA, "msdc2_infra", "rg_msdc2", 30),
+	GATE_TOP2(CLK_TOP_USB_78M, "usb_78m", "usb_78m_sel", 31),
+	/* TOP3 */
+	GATE_TOP3(CLK_TOP_RG_SPINOR, "rg_spinor", "spinor_sel", 0),
+	GATE_TOP3(CLK_TOP_RG_MSDC2, "rg_msdc2", "msdc2_sel", 1),
+	GATE_TOP3(CLK_TOP_RG_ETH, "rg_eth", "eth_sel", 2),
+	GATE_TOP3(CLK_TOP_RG_AUD1, "rg_aud1", "aud1_sel", 8),
+	GATE_TOP3(CLK_TOP_RG_AUD2, "rg_aud2", "aud2_sel", 9),
+	GATE_TOP3(CLK_TOP_RG_AUD_ENGEN1, "rg_aud_engen1", "aud_engen1_sel", 10),
+	GATE_TOP3(CLK_TOP_RG_AUD_ENGEN2, "rg_aud_engen2", "aud_engen2_sel", 11),
+	GATE_TOP3(CLK_TOP_RG_I2C, "rg_i2c", "i2c_sel", 12),
+	GATE_TOP3(CLK_TOP_RG_PWM_INFRA, "rg_pwm_infra", "pwm_sel", 13),
+	GATE_TOP3(CLK_TOP_RG_AUD_SPDIF_IN, "rg_aud_spdif_in", "aud_spdifin_sel",
+		14),
+	GATE_TOP3(CLK_TOP_RG_UART2, "rg_uart2", "uart2_sel", 15),
+	GATE_TOP3(CLK_TOP_RG_BSI, "rg_bsi", "bsi_sel", 16),
+	GATE_TOP3(CLK_TOP_RG_DBG_ATCLK, "rg_dbg_atclk", "dbg_atclk_sel", 17),
+	GATE_TOP3(CLK_TOP_RG_NFIECC, "rg_nfiecc", "nfiecc_sel", 18),
+	/* TOP4 */
+	GATE_TOP4_I(CLK_TOP_RG_APLL1_D2_EN, "rg_apll1_d2_en", "apll1_d2", 8),
+	GATE_TOP4_I(CLK_TOP_RG_APLL1_D4_EN, "rg_apll1_d4_en", "apll1_d4", 9),
+	GATE_TOP4_I(CLK_TOP_RG_APLL1_D8_EN, "rg_apll1_d8_en", "apll1_d8", 10),
+	GATE_TOP4_I(CLK_TOP_RG_APLL2_D2_EN, "rg_apll2_d2_en", "apll2_d2", 11),
+	GATE_TOP4_I(CLK_TOP_RG_APLL2_D4_EN, "rg_apll2_d4_en", "apll2_d4", 12),
+	GATE_TOP4_I(CLK_TOP_RG_APLL2_D8_EN, "rg_apll2_d8_en", "apll2_d8", 13),
+	/* TOP5 */
+	GATE_TOP5(CLK_TOP_APLL12_DIV0, "apll12_div0", "apll12_ck_div0", 0),
+	GATE_TOP5(CLK_TOP_APLL12_DIV1, "apll12_div1", "apll12_ck_div1", 1),
+	GATE_TOP5(CLK_TOP_APLL12_DIV2, "apll12_div2", "apll12_ck_div2", 2),
+	GATE_TOP5(CLK_TOP_APLL12_DIV3, "apll12_div3", "apll12_ck_div3", 3),
+	GATE_TOP5(CLK_TOP_APLL12_DIV4, "apll12_div4", "apll12_ck_div4", 4),
+	GATE_TOP5(CLK_TOP_APLL12_DIV4B, "apll12_div4b", "apll12_ck_div4b", 5),
+	GATE_TOP5(CLK_TOP_APLL12_DIV5, "apll12_div5", "apll12_ck_div5", 6),
+	GATE_TOP5(CLK_TOP_APLL12_DIV5B, "apll12_div5b", "apll12_ck_div5b", 7),
+	GATE_TOP5(CLK_TOP_APLL12_DIV6, "apll12_div6", "apll12_ck_div6", 8),
+};
+
+static void __init mtk_topckgen_init(struct device_node *node)
+{
+	struct clk_onecell_data *clk_data;
+	int r;
+	void __iomem *base;
+
+	base = of_iomap(node, 0);
+	if (!base) {
+		pr_err("%s(): ioremap failed\n", __func__);
+		return;
+	}
+
+	clk_data = mtk_alloc_clk_data(CLK_TOP_NR_CLK);
+
+	mtk_clk_register_fixed_clks(fixed_clks, ARRAY_SIZE(fixed_clks),
+				    clk_data);
+	mtk_clk_register_gates(node, top_clks, ARRAY_SIZE(top_clks), clk_data);
+
+	mtk_clk_register_factors(top_divs, ARRAY_SIZE(top_divs), clk_data);
+	mtk_clk_register_composites(top_muxes, ARRAY_SIZE(top_muxes), base,
+		&mt8516_clk_lock, clk_data);
+	mtk_clk_register_dividers(top_adj_divs, ARRAY_SIZE(top_adj_divs),
+				base, &mt8516_clk_lock, clk_data);
+
+	r = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+	if (r)
+		pr_err("%s(): could not register clock provider: %d\n",
+			__func__, r);
+}
+CLK_OF_DECLARE(mtk_topckgen, "mediatek,mt8516-topckgen", mtk_topckgen_init);
+
+static void __init mtk_infracfg_init(struct device_node *node)
+{
+	struct clk_onecell_data *clk_data;
+	int r;
+	void __iomem *base;
+
+	base = of_iomap(node, 0);
+	if (!base) {
+		pr_err("%s(): ioremap failed\n", __func__);
+		return;
+	}
+
+	clk_data = mtk_alloc_clk_data(CLK_IFR_NR_CLK);
+
+	mtk_clk_register_composites(ifr_muxes, ARRAY_SIZE(ifr_muxes), base,
+		&mt8516_clk_lock, clk_data);
+
+	r = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+	if (r)
+		pr_err("%s(): could not register clock provider: %d\n",
+			__func__, r);
+}
+CLK_OF_DECLARE(mtk_infracfg, "mediatek,mt8516-infracfg", mtk_infracfg_init);
+
+#define MT8516_PLL_FMAX		(1502UL * MHZ)
+
+#define CON0_MT8516_RST_BAR	BIT(27)
+
+#define PLL_B(_id, _name, _reg, _pwr_reg, _en_mask, _flags, _pcwbits,	\
+			_pd_reg, _pd_shift, _tuner_reg, _pcw_reg,	\
+			_pcw_shift, _div_table) {			\
+		.id = _id,						\
+		.name = _name,						\
+		.reg = _reg,						\
+		.pwr_reg = _pwr_reg,					\
+		.en_mask = _en_mask,					\
+		.flags = _flags,					\
+		.rst_bar_mask = CON0_MT8516_RST_BAR,			\
+		.fmax = MT8516_PLL_FMAX,				\
+		.pcwbits = _pcwbits,					\
+		.pd_reg = _pd_reg,					\
+		.pd_shift = _pd_shift,					\
+		.tuner_reg = _tuner_reg,				\
+		.pcw_reg = _pcw_reg,					\
+		.pcw_shift = _pcw_shift,				\
+		.div_table = _div_table,				\
+	}
+
+#define PLL(_id, _name, _reg, _pwr_reg, _en_mask, _flags, _pcwbits,	\
+			_pd_reg, _pd_shift, _tuner_reg, _pcw_reg,	\
+			_pcw_shift)					\
+		PLL_B(_id, _name, _reg, _pwr_reg, _en_mask, _flags, _pcwbits, \
+			_pd_reg, _pd_shift, _tuner_reg, _pcw_reg, _pcw_shift, \
+			NULL)
+
+static const struct mtk_pll_div_table mmpll_div_table[] = {
+	{ .div = 0, .freq = MT8516_PLL_FMAX },
+	{ .div = 1, .freq = 1000000000 },
+	{ .div = 2, .freq = 604500000 },
+	{ .div = 3, .freq = 253500000 },
+	{ .div = 4, .freq = 126750000 },
+	{ } /* sentinel */
+};
+
+static const struct mtk_pll_data plls[] = {
+	PLL(CLK_APMIXED_ARMPLL, "armpll", 0x0100, 0x0110, 0x00000001, 0,
+		21, 0x0104, 24, 0, 0x0104, 0),
+	PLL(CLK_APMIXED_MAINPLL, "mainpll", 0x0120, 0x0130, 0x00000001,
+		HAVE_RST_BAR, 21, 0x0124, 24, 0, 0x0124, 0),
+	PLL(CLK_APMIXED_UNIVPLL, "univpll", 0x0140, 0x0150, 0x30000001,
+		HAVE_RST_BAR, 7, 0x0144, 24, 0, 0x0144, 0),
+	PLL_B(CLK_APMIXED_MMPLL, "mmpll", 0x0160, 0x0170, 0x00000001, 0,
+		21, 0x0164, 24, 0, 0x0164, 0, mmpll_div_table),
+	PLL(CLK_APMIXED_APLL1, "apll1", 0x0180, 0x0190, 0x00000001, 0,
+		31, 0x0180, 1, 0x0194, 0x0184, 0),
+	PLL(CLK_APMIXED_APLL2, "apll2", 0x01A0, 0x01B0, 0x00000001, 0,
+		31, 0x01A0, 1, 0x01B4, 0x01A4, 0),
+};
+
+static void __init mtk_apmixedsys_init(struct device_node *node)
+{
+	struct clk_onecell_data *clk_data;
+	void __iomem *base;
+	int r;
+
+	base = of_iomap(node, 0);
+	if (!base) {
+		pr_err("%s(): ioremap failed\n", __func__);
+		return;
+	}
+
+	clk_data = mtk_alloc_clk_data(CLK_APMIXED_NR_CLK);
+
+	mtk_clk_register_plls(node, plls, ARRAY_SIZE(plls), clk_data);
+
+	r = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+	if (r)
+		pr_err("%s(): could not register clock provider: %d\n",
+			__func__, r);
+
+}
+CLK_OF_DECLARE(mtk_apmixedsys, "mediatek,mt8516-apmixedsys",
+		mtk_apmixedsys_init);
diff --git a/drivers/clk/mediatek/clk-mtk.h b/drivers/clk/mediatek/clk-mtk.h
index fb27b5bf30d9..33ab1731482f 100644
--- a/drivers/clk/mediatek/clk-mtk.h
+++ b/drivers/clk/mediatek/clk-mtk.h
@@ -227,10 +227,13 @@ struct mtk_pll_data {
 	unsigned int flags;
 	const struct clk_ops *ops;
 	u32 rst_bar_mask;
+	unsigned long fmin;
 	unsigned long fmax;
 	int pcwbits;
+	int pcwibits;
 	uint32_t pcw_reg;
 	int pcw_shift;
+	uint32_t pcw_chg_reg;
 	const struct mtk_pll_div_table *div_table;
 	const char *parent_name;
 };
diff --git a/drivers/clk/mediatek/clk-mux.c b/drivers/clk/mediatek/clk-mux.c
new file mode 100644
index 000000000000..76f9cd039195
--- /dev/null
+++ b/drivers/clk/mediatek/clk-mux.c
@@ -0,0 +1,223 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018 MediaTek Inc.
+ * Author: Owen Chen <owen.chen@mediatek.com>
+ */
+
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/slab.h>
+#include <linux/mfd/syscon.h>
+
+#include "clk-mtk.h"
+#include "clk-mux.h"
+
+static inline struct mtk_clk_mux *to_mtk_clk_mux(struct clk_hw *hw)
+{
+	return container_of(hw, struct mtk_clk_mux, hw);
+}
+
+static int mtk_clk_mux_enable(struct clk_hw *hw)
+{
+	struct mtk_clk_mux *mux = to_mtk_clk_mux(hw);
+	u32 mask = BIT(mux->data->gate_shift);
+
+	return regmap_update_bits(mux->regmap, mux->data->mux_ofs,
+			mask, ~mask);
+}
+
+static void mtk_clk_mux_disable(struct clk_hw *hw)
+{
+	struct mtk_clk_mux *mux = to_mtk_clk_mux(hw);
+	u32 mask = BIT(mux->data->gate_shift);
+
+	regmap_update_bits(mux->regmap, mux->data->mux_ofs, mask, mask);
+}
+
+static int mtk_clk_mux_enable_setclr(struct clk_hw *hw)
+{
+	struct mtk_clk_mux *mux = to_mtk_clk_mux(hw);
+
+	return regmap_write(mux->regmap, mux->data->clr_ofs,
+			BIT(mux->data->gate_shift));
+}
+
+static void mtk_clk_mux_disable_setclr(struct clk_hw *hw)
+{
+	struct mtk_clk_mux *mux = to_mtk_clk_mux(hw);
+
+	regmap_write(mux->regmap, mux->data->set_ofs,
+			BIT(mux->data->gate_shift));
+}
+
+static int mtk_clk_mux_is_enabled(struct clk_hw *hw)
+{
+	struct mtk_clk_mux *mux = to_mtk_clk_mux(hw);
+	u32 val;
+
+	regmap_read(mux->regmap, mux->data->mux_ofs, &val);
+
+	return (val & BIT(mux->data->gate_shift)) == 0;
+}
+
+static u8 mtk_clk_mux_get_parent(struct clk_hw *hw)
+{
+	struct mtk_clk_mux *mux = to_mtk_clk_mux(hw);
+	u32 mask = GENMASK(mux->data->mux_width - 1, 0);
+	u32 val;
+
+	regmap_read(mux->regmap, mux->data->mux_ofs, &val);
+	val = (val >> mux->data->mux_shift) & mask;
+
+	return val;
+}
+
+static int mtk_clk_mux_set_parent_lock(struct clk_hw *hw, u8 index)
+{
+	struct mtk_clk_mux *mux = to_mtk_clk_mux(hw);
+	u32 mask = GENMASK(mux->data->mux_width - 1, 0);
+	unsigned long flags = 0;
+
+	if (mux->lock)
+		spin_lock_irqsave(mux->lock, flags);
+	else
+		__acquire(mux->lock);
+
+	regmap_update_bits(mux->regmap, mux->data->mux_ofs, mask,
+		index << mux->data->mux_shift);
+
+	if (mux->lock)
+		spin_unlock_irqrestore(mux->lock, flags);
+	else
+		__release(mux->lock);
+
+	return 0;
+}
+
+static int mtk_clk_mux_set_parent_setclr_lock(struct clk_hw *hw, u8 index)
+{
+	struct mtk_clk_mux *mux = to_mtk_clk_mux(hw);
+	u32 mask = GENMASK(mux->data->mux_width - 1, 0);
+	u32 val, orig;
+	unsigned long flags = 0;
+
+	if (mux->lock)
+		spin_lock_irqsave(mux->lock, flags);
+	else
+		__acquire(mux->lock);
+
+	regmap_read(mux->regmap, mux->data->mux_ofs, &orig);
+	val = (orig & ~(mask << mux->data->mux_shift))
+			| (index << mux->data->mux_shift);
+
+	if (val != orig) {
+		regmap_write(mux->regmap, mux->data->clr_ofs,
+				mask << mux->data->mux_shift);
+		regmap_write(mux->regmap, mux->data->set_ofs,
+				index << mux->data->mux_shift);
+
+		if (mux->data->upd_shift >= 0)
+			regmap_write(mux->regmap, mux->data->upd_ofs,
+					BIT(mux->data->upd_shift));
+	}
+
+	if (mux->lock)
+		spin_unlock_irqrestore(mux->lock, flags);
+	else
+		__release(mux->lock);
+
+	return 0;
+}
+
+const struct clk_ops mtk_mux_ops = {
+	.get_parent = mtk_clk_mux_get_parent,
+	.set_parent = mtk_clk_mux_set_parent_lock,
+};
+
+const struct clk_ops mtk_mux_clr_set_upd_ops = {
+	.get_parent = mtk_clk_mux_get_parent,
+	.set_parent = mtk_clk_mux_set_parent_setclr_lock,
+};
+
+const struct clk_ops mtk_mux_gate_ops = {
+	.enable = mtk_clk_mux_enable,
+	.disable = mtk_clk_mux_disable,
+	.is_enabled = mtk_clk_mux_is_enabled,
+	.get_parent = mtk_clk_mux_get_parent,
+	.set_parent = mtk_clk_mux_set_parent_lock,
+};
+
+const struct clk_ops mtk_mux_gate_clr_set_upd_ops = {
+	.enable = mtk_clk_mux_enable_setclr,
+	.disable = mtk_clk_mux_disable_setclr,
+	.is_enabled = mtk_clk_mux_is_enabled,
+	.get_parent = mtk_clk_mux_get_parent,
+	.set_parent = mtk_clk_mux_set_parent_setclr_lock,
+};
+
+struct clk *mtk_clk_register_mux(const struct mtk_mux *mux,
+				 struct regmap *regmap,
+				 spinlock_t *lock)
+{
+	struct mtk_clk_mux *clk_mux;
+	struct clk_init_data init;
+	struct clk *clk;
+
+	clk_mux = kzalloc(sizeof(*clk_mux), GFP_KERNEL);
+	if (!clk_mux)
+		return ERR_PTR(-ENOMEM);
+
+	init.name = mux->name;
+	init.flags = mux->flags | CLK_SET_RATE_PARENT;
+	init.parent_names = mux->parent_names;
+	init.num_parents = mux->num_parents;
+	init.ops = mux->ops;
+
+	clk_mux->regmap = regmap;
+	clk_mux->data = mux;
+	clk_mux->lock = lock;
+	clk_mux->hw.init = &init;
+
+	clk = clk_register(NULL, &clk_mux->hw);
+	if (IS_ERR(clk)) {
+		kfree(clk_mux);
+		return clk;
+	}
+
+	return clk;
+}
+
+int mtk_clk_register_muxes(const struct mtk_mux *muxes,
+			   int num, struct device_node *node,
+			   spinlock_t *lock,
+			   struct clk_onecell_data *clk_data)
+{
+	struct regmap *regmap;
+	struct clk *clk;
+	int i;
+
+	regmap = syscon_node_to_regmap(node);
+	if (IS_ERR(regmap)) {
+		pr_err("Cannot find regmap for %pOF: %ld\n", node,
+		       PTR_ERR(regmap));
+		return PTR_ERR(regmap);
+	}
+
+	for (i = 0; i < num; i++) {
+		const struct mtk_mux *mux = &muxes[i];
+
+		if (IS_ERR_OR_NULL(clk_data->clks[mux->id])) {
+			clk = mtk_clk_register_mux(mux, regmap, lock);
+
+			if (IS_ERR(clk)) {
+				pr_err("Failed to register clk %s: %ld\n",
+				       mux->name, PTR_ERR(clk));
+				continue;
+			}
+
+			clk_data->clks[mux->id] = clk;
+		}
+	}
+
+	return 0;
+}
diff --git a/drivers/clk/mediatek/clk-mux.h b/drivers/clk/mediatek/clk-mux.h
new file mode 100644
index 000000000000..f5625f4d9e6c
--- /dev/null
+++ b/drivers/clk/mediatek/clk-mux.h
@@ -0,0 +1,89 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2018 MediaTek Inc.
+ * Author: Owen Chen <owen.chen@mediatek.com>
+ */
+
+#ifndef __DRV_CLK_MTK_MUX_H
+#define __DRV_CLK_MTK_MUX_H
+
+#include <linux/clk-provider.h>
+
+struct mtk_clk_mux {
+	struct clk_hw hw;
+	struct regmap *regmap;
+	const struct mtk_mux *data;
+	spinlock_t *lock;
+};
+
+struct mtk_mux {
+	int id;
+	const char *name;
+	const char * const *parent_names;
+	unsigned int flags;
+
+	u32 mux_ofs;
+	u32 set_ofs;
+	u32 clr_ofs;
+	u32 upd_ofs;
+
+	u8 mux_shift;
+	u8 mux_width;
+	u8 gate_shift;
+	s8 upd_shift;
+
+	const struct clk_ops *ops;
+
+	signed char num_parents;
+};
+
+extern const struct clk_ops mtk_mux_ops;
+extern const struct clk_ops mtk_mux_clr_set_upd_ops;
+extern const struct clk_ops mtk_mux_gate_ops;
+extern const struct clk_ops mtk_mux_gate_clr_set_upd_ops;
+
+#define GATE_CLR_SET_UPD_FLAGS(_id, _name, _parents, _mux_ofs,		\
+			_mux_set_ofs, _mux_clr_ofs, _shift, _width,	\
+			_gate, _upd_ofs, _upd, _flags, _ops) {		\
+		.id = _id,						\
+		.name = _name,						\
+		.mux_ofs = _mux_ofs,					\
+		.set_ofs = _mux_set_ofs,				\
+		.clr_ofs = _mux_clr_ofs,				\
+		.upd_ofs = _upd_ofs,					\
+		.mux_shift = _shift,					\
+		.mux_width = _width,					\
+		.gate_shift = _gate,					\
+		.upd_shift = _upd,					\
+		.parent_names = _parents,				\
+		.num_parents = ARRAY_SIZE(_parents),			\
+		.flags = _flags,					\
+		.ops = &_ops,						\
+	}
+
+#define MUX_GATE_CLR_SET_UPD_FLAGS(_id, _name, _parents, _mux_ofs,	\
+			_mux_set_ofs, _mux_clr_ofs, _shift, _width,	\
+			_gate, _upd_ofs, _upd, _flags)			\
+		GATE_CLR_SET_UPD_FLAGS(_id, _name, _parents, _mux_ofs,	\
+			_mux_set_ofs, _mux_clr_ofs, _shift, _width,	\
+			_gate, _upd_ofs, _upd, _flags,			\
+			mtk_mux_gate_clr_set_upd_ops)
+
+#define MUX_GATE_CLR_SET_UPD(_id, _name, _parents, _mux_ofs,		\
+			_mux_set_ofs, _mux_clr_ofs, _shift, _width,	\
+			_gate, _upd_ofs, _upd)				\
+		MUX_GATE_CLR_SET_UPD_FLAGS(_id, _name, _parents,	\
+			_mux_ofs, _mux_set_ofs, _mux_clr_ofs, _shift,	\
+			_width, _gate, _upd_ofs, _upd,			\
+			CLK_SET_RATE_PARENT)
+
+struct clk *mtk_clk_register_mux(const struct mtk_mux *mux,
+				 struct regmap *regmap,
+				 spinlock_t *lock);
+
+int mtk_clk_register_muxes(const struct mtk_mux *muxes,
+			   int num, struct device_node *node,
+			   spinlock_t *lock,
+			   struct clk_onecell_data *clk_data);
+
+#endif /* __DRV_CLK_MTK_MUX_H */
diff --git a/drivers/clk/mediatek/clk-pll.c b/drivers/clk/mediatek/clk-pll.c
index f54e4015b0b1..8d556fc99fed 100644
--- a/drivers/clk/mediatek/clk-pll.c
+++ b/drivers/clk/mediatek/clk-pll.c
@@ -27,11 +27,13 @@
 #define CON0_BASE_EN		BIT(0)
 #define CON0_PWR_ON		BIT(0)
 #define CON0_ISO_EN		BIT(1)
-#define CON0_PCW_CHG		BIT(31)
+#define PCW_CHG_MASK		BIT(31)
 
 #define AUDPLL_TUNER_EN		BIT(31)
 
 #define POSTDIV_MASK		0x7
+
+/* default 7 bits integer, can be overridden with pcwibits. */
 #define INTEGER_BITS		7
 
 /*
@@ -49,6 +51,7 @@ struct mtk_clk_pll {
 	void __iomem	*tuner_addr;
 	void __iomem	*tuner_en_addr;
 	void __iomem	*pcw_addr;
+	void __iomem	*pcw_chg_addr;
 	const struct mtk_pll_data *data;
 };
 
@@ -68,12 +71,15 @@ static unsigned long __mtk_pll_recalc_rate(struct mtk_clk_pll *pll, u32 fin,
 		u32 pcw, int postdiv)
 {
 	int pcwbits = pll->data->pcwbits;
-	int pcwfbits;
+	int pcwfbits = 0;
+	int ibits;
 	u64 vco;
 	u8 c = 0;
 
 	/* The fractional part of the PLL divider. */
-	pcwfbits = pcwbits > INTEGER_BITS ? pcwbits - INTEGER_BITS : 0;
+	ibits = pll->data->pcwibits ? pll->data->pcwibits : INTEGER_BITS;
+	if (pcwbits > ibits)
+		pcwfbits = pcwbits - ibits;
 
 	vco = (u64)fin * pcw;
 
@@ -88,13 +94,39 @@ static unsigned long __mtk_pll_recalc_rate(struct mtk_clk_pll *pll, u32 fin,
 	return ((unsigned long)vco + postdiv - 1) / postdiv;
 }
 
+static void __mtk_pll_tuner_enable(struct mtk_clk_pll *pll)
+{
+	u32 r;
+
+	if (pll->tuner_en_addr) {
+		r = readl(pll->tuner_en_addr) | BIT(pll->data->tuner_en_bit);
+		writel(r, pll->tuner_en_addr);
+	} else if (pll->tuner_addr) {
+		r = readl(pll->tuner_addr) | AUDPLL_TUNER_EN;
+		writel(r, pll->tuner_addr);
+	}
+}
+
+static void __mtk_pll_tuner_disable(struct mtk_clk_pll *pll)
+{
+	u32 r;
+
+	if (pll->tuner_en_addr) {
+		r = readl(pll->tuner_en_addr) & ~BIT(pll->data->tuner_en_bit);
+		writel(r, pll->tuner_en_addr);
+	} else if (pll->tuner_addr) {
+		r = readl(pll->tuner_addr) & ~AUDPLL_TUNER_EN;
+		writel(r, pll->tuner_addr);
+	}
+}
+
 static void mtk_pll_set_rate_regs(struct mtk_clk_pll *pll, u32 pcw,
 		int postdiv)
 {
-	u32 con1, val;
-	int pll_en;
+	u32 chg, val;
 
-	pll_en = readl(pll->base_addr + REG_CON0) & CON0_BASE_EN;
+	/* disable tuner */
+	__mtk_pll_tuner_disable(pll);
 
 	/* set postdiv */
 	val = readl(pll->pd_addr);
@@ -112,18 +144,15 @@ static void mtk_pll_set_rate_regs(struct mtk_clk_pll *pll, u32 pcw,
 			pll->data->pcw_shift);
 	val |= pcw << pll->data->pcw_shift;
 	writel(val, pll->pcw_addr);
-
-	con1 = readl(pll->base_addr + REG_CON1);
-
-	if (pll_en)
-		con1 |= CON0_PCW_CHG;
-
-	writel(con1, pll->base_addr + REG_CON1);
+	chg = readl(pll->pcw_chg_addr) | PCW_CHG_MASK;
+	writel(chg, pll->pcw_chg_addr);
 	if (pll->tuner_addr)
-		writel(con1 + 1, pll->tuner_addr);
+		writel(val + 1, pll->tuner_addr);
+
+	/* restore tuner_en */
+	__mtk_pll_tuner_enable(pll);
 
-	if (pll_en)
-		udelay(20);
+	udelay(20);
 }
 
 /*
@@ -138,9 +167,10 @@ static void mtk_pll_set_rate_regs(struct mtk_clk_pll *pll, u32 pcw,
 static void mtk_pll_calc_values(struct mtk_clk_pll *pll, u32 *pcw, u32 *postdiv,
 		u32 freq, u32 fin)
 {
-	unsigned long fmin = 1000 * MHZ;
+	unsigned long fmin = pll->data->fmin ? pll->data->fmin : (1000 * MHZ);
 	const struct mtk_pll_div_table *div_table = pll->data->div_table;
 	u64 _pcw;
+	int ibits;
 	u32 val;
 
 	if (freq > pll->data->fmax)
@@ -164,7 +194,8 @@ static void mtk_pll_calc_values(struct mtk_clk_pll *pll, u32 *pcw, u32 *postdiv,
 	}
 
 	/* _pcw = freq * postdiv / fin * 2^pcwfbits */
-	_pcw = ((u64)freq << val) << (pll->data->pcwbits - INTEGER_BITS);
+	ibits = pll->data->pcwibits ? pll->data->pcwibits : INTEGER_BITS;
+	_pcw = ((u64)freq << val) << (pll->data->pcwbits - ibits);
 	do_div(_pcw, fin);
 
 	*pcw = (u32)_pcw;
@@ -228,13 +259,7 @@ static int mtk_pll_prepare(struct clk_hw *hw)
 	r |= pll->data->en_mask;
 	writel(r, pll->base_addr + REG_CON0);
 
-	if (pll->tuner_en_addr) {
-		r = readl(pll->tuner_en_addr) | BIT(pll->data->tuner_en_bit);
-		writel(r, pll->tuner_en_addr);
-	} else if (pll->tuner_addr) {
-		r = readl(pll->tuner_addr) | AUDPLL_TUNER_EN;
-		writel(r, pll->tuner_addr);
-	}
+	__mtk_pll_tuner_enable(pll);
 
 	udelay(20);
 
@@ -258,13 +283,7 @@ static void mtk_pll_unprepare(struct clk_hw *hw)
 		writel(r, pll->base_addr + REG_CON0);
 	}
 
-	if (pll->tuner_en_addr) {
-		r = readl(pll->tuner_en_addr) & ~BIT(pll->data->tuner_en_bit);
-		writel(r, pll->tuner_en_addr);
-	} else if (pll->tuner_addr) {
-		r = readl(pll->tuner_addr) & ~AUDPLL_TUNER_EN;
-		writel(r, pll->tuner_addr);
-	}
+	__mtk_pll_tuner_disable(pll);
 
 	r = readl(pll->base_addr + REG_CON0);
 	r &= ~CON0_BASE_EN;
@@ -302,6 +321,10 @@ static struct clk *mtk_clk_register_pll(const struct mtk_pll_data *data,
 	pll->pwr_addr = base + data->pwr_reg;
 	pll->pd_addr = base + data->pd_reg;
 	pll->pcw_addr = base + data->pcw_reg;
+	if (data->pcw_chg_reg)
+		pll->pcw_chg_addr = base + data->pcw_chg_reg;
+	else
+		pll->pcw_chg_addr = pll->base_addr + REG_CON1;
 	if (data->tuner_reg)
 		pll->tuner_addr = base + data->tuner_reg;
 	if (data->tuner_en_reg)
diff --git a/drivers/clk/meson/axg-audio.c b/drivers/clk/meson/axg-audio.c
index 7ab200b6c3bf..8028ff6f6610 100644
--- a/drivers/clk/meson/axg-audio.c
+++ b/drivers/clk/meson/axg-audio.c
@@ -20,18 +20,18 @@
 #include "clk-phase.h"
 #include "sclk-div.h"
 
-#define AXG_MST_IN_COUNT	8
-#define AXG_SLV_SCLK_COUNT	10
-#define AXG_SLV_LRCLK_COUNT	10
+#define AUD_MST_IN_COUNT	8
+#define AUD_SLV_SCLK_COUNT	10
+#define AUD_SLV_LRCLK_COUNT	10
 
-#define AXG_AUD_GATE(_name, _reg, _bit, _pname, _iflags)		\
-struct clk_regmap axg_##_name = {					\
+#define AUD_GATE(_name, _reg, _bit, _pname, _iflags)			\
+struct clk_regmap aud_##_name = {					\
 	.data = &(struct clk_regmap_gate_data){				\
 		.offset = (_reg),					\
 		.bit_idx = (_bit),					\
 	},								\
 	.hw.init = &(struct clk_init_data) {				\
-		.name = "axg_"#_name,					\
+		.name = "aud_"#_name,					\
 		.ops = &clk_regmap_gate_ops,				\
 		.parent_names = (const char *[]){ _pname },		\
 		.num_parents = 1,					\
@@ -39,8 +39,8 @@ struct clk_regmap axg_##_name = {					\
 	},								\
 }
 
-#define AXG_AUD_MUX(_name, _reg, _mask, _shift, _dflags, _pnames, _iflags) \
-struct clk_regmap axg_##_name = {					\
+#define AUD_MUX(_name, _reg, _mask, _shift, _dflags, _pnames, _iflags)	\
+struct clk_regmap aud_##_name = {					\
 	.data = &(struct clk_regmap_mux_data){				\
 		.offset = (_reg),					\
 		.mask = (_mask),					\
@@ -48,7 +48,7 @@ struct clk_regmap axg_##_name = {					\
 		.flags = (_dflags),					\
 	},								\
 	.hw.init = &(struct clk_init_data){				\
-		.name = "axg_"#_name,					\
+		.name = "aud_"#_name,					\
 		.ops = &clk_regmap_mux_ops,				\
 		.parent_names = (_pnames),				\
 		.num_parents = ARRAY_SIZE(_pnames),			\
@@ -56,8 +56,8 @@ struct clk_regmap axg_##_name = {					\
 	},								\
 }
 
-#define AXG_AUD_DIV(_name, _reg, _shift, _width, _dflags, _pname, _iflags) \
-struct clk_regmap axg_##_name = {					\
+#define AUD_DIV(_name, _reg, _shift, _width, _dflags, _pname, _iflags)	\
+struct clk_regmap aud_##_name = {					\
 	.data = &(struct clk_regmap_div_data){				\
 		.offset = (_reg),					\
 		.shift = (_shift),					\
@@ -65,7 +65,7 @@ struct clk_regmap axg_##_name = {					\
 		.flags = (_dflags),					\
 	},								\
 	.hw.init = &(struct clk_init_data){				\
-		.name = "axg_"#_name,					\
+		.name = "aud_"#_name,					\
 		.ops = &clk_regmap_divider_ops,				\
 		.parent_names = (const char *[]) { _pname },		\
 		.num_parents = 1,					\
@@ -73,109 +73,113 @@ struct clk_regmap axg_##_name = {					\
 	},								\
 }
 
-#define AXG_PCLK_GATE(_name, _bit)				\
-	AXG_AUD_GATE(_name, AUDIO_CLK_GATE_EN, _bit, "axg_audio_pclk", 0)
+#define AUD_PCLK_GATE(_name, _bit)				\
+	AUD_GATE(_name, AUDIO_CLK_GATE_EN, _bit, "audio_pclk", 0)
 
 /* Audio peripheral clocks */
-static AXG_PCLK_GATE(ddr_arb,	   0);
-static AXG_PCLK_GATE(pdm,	   1);
-static AXG_PCLK_GATE(tdmin_a,	   2);
-static AXG_PCLK_GATE(tdmin_b,	   3);
-static AXG_PCLK_GATE(tdmin_c,	   4);
-static AXG_PCLK_GATE(tdmin_lb,	   5);
-static AXG_PCLK_GATE(tdmout_a,	   6);
-static AXG_PCLK_GATE(tdmout_b,	   7);
-static AXG_PCLK_GATE(tdmout_c,	   8);
-static AXG_PCLK_GATE(frddr_a,	   9);
-static AXG_PCLK_GATE(frddr_b,	   10);
-static AXG_PCLK_GATE(frddr_c,	   11);
-static AXG_PCLK_GATE(toddr_a,	   12);
-static AXG_PCLK_GATE(toddr_b,	   13);
-static AXG_PCLK_GATE(toddr_c,	   14);
-static AXG_PCLK_GATE(loopback,	   15);
-static AXG_PCLK_GATE(spdifin,	   16);
-static AXG_PCLK_GATE(spdifout,	   17);
-static AXG_PCLK_GATE(resample,	   18);
-static AXG_PCLK_GATE(power_detect, 19);
+static AUD_PCLK_GATE(ddr_arb,	   0);
+static AUD_PCLK_GATE(pdm,	   1);
+static AUD_PCLK_GATE(tdmin_a,	   2);
+static AUD_PCLK_GATE(tdmin_b,	   3);
+static AUD_PCLK_GATE(tdmin_c,	   4);
+static AUD_PCLK_GATE(tdmin_lb,	   5);
+static AUD_PCLK_GATE(tdmout_a,	   6);
+static AUD_PCLK_GATE(tdmout_b,	   7);
+static AUD_PCLK_GATE(tdmout_c,	   8);
+static AUD_PCLK_GATE(frddr_a,	   9);
+static AUD_PCLK_GATE(frddr_b,	   10);
+static AUD_PCLK_GATE(frddr_c,	   11);
+static AUD_PCLK_GATE(toddr_a,	   12);
+static AUD_PCLK_GATE(toddr_b,	   13);
+static AUD_PCLK_GATE(toddr_c,	   14);
+static AUD_PCLK_GATE(loopback,	   15);
+static AUD_PCLK_GATE(spdifin,	   16);
+static AUD_PCLK_GATE(spdifout,	   17);
+static AUD_PCLK_GATE(resample,	   18);
+static AUD_PCLK_GATE(power_detect, 19);
+static AUD_PCLK_GATE(spdifout_b,   21);
 
 /* Audio Master Clocks */
 static const char * const mst_mux_parent_names[] = {
-	"axg_mst_in0", "axg_mst_in1", "axg_mst_in2", "axg_mst_in3",
-	"axg_mst_in4", "axg_mst_in5", "axg_mst_in6", "axg_mst_in7",
+	"aud_mst_in0", "aud_mst_in1", "aud_mst_in2", "aud_mst_in3",
+	"aud_mst_in4", "aud_mst_in5", "aud_mst_in6", "aud_mst_in7",
 };
 
-#define AXG_MST_MUX(_name, _reg, _flag)				\
-	AXG_AUD_MUX(_name##_sel, _reg, 0x7, 24, _flag,		\
-		    mst_mux_parent_names, CLK_SET_RATE_PARENT)
-
-#define AXG_MST_MCLK_MUX(_name, _reg)				\
-	AXG_MST_MUX(_name, _reg, CLK_MUX_ROUND_CLOSEST)
-
-#define AXG_MST_SYS_MUX(_name, _reg)				\
-	AXG_MST_MUX(_name, _reg, 0)
-
-static AXG_MST_MCLK_MUX(mst_a_mclk,   AUDIO_MCLK_A_CTRL);
-static AXG_MST_MCLK_MUX(mst_b_mclk,   AUDIO_MCLK_B_CTRL);
-static AXG_MST_MCLK_MUX(mst_c_mclk,   AUDIO_MCLK_C_CTRL);
-static AXG_MST_MCLK_MUX(mst_d_mclk,   AUDIO_MCLK_D_CTRL);
-static AXG_MST_MCLK_MUX(mst_e_mclk,   AUDIO_MCLK_E_CTRL);
-static AXG_MST_MCLK_MUX(mst_f_mclk,   AUDIO_MCLK_F_CTRL);
-static AXG_MST_MCLK_MUX(spdifout_clk, AUDIO_CLK_SPDIFOUT_CTRL);
-static AXG_MST_MCLK_MUX(pdm_dclk,     AUDIO_CLK_PDMIN_CTRL0);
-static AXG_MST_SYS_MUX(spdifin_clk,   AUDIO_CLK_SPDIFIN_CTRL);
-static AXG_MST_SYS_MUX(pdm_sysclk,    AUDIO_CLK_PDMIN_CTRL1);
-
-#define AXG_MST_DIV(_name, _reg, _flag)				\
-	AXG_AUD_DIV(_name##_div, _reg, 0, 16, _flag,		\
-		    "axg_"#_name"_sel", CLK_SET_RATE_PARENT)	\
-
-#define AXG_MST_MCLK_DIV(_name, _reg)				\
-	AXG_MST_DIV(_name, _reg, CLK_DIVIDER_ROUND_CLOSEST)
-
-#define AXG_MST_SYS_DIV(_name, _reg)				\
-	AXG_MST_DIV(_name, _reg, 0)
-
-static AXG_MST_MCLK_DIV(mst_a_mclk,   AUDIO_MCLK_A_CTRL);
-static AXG_MST_MCLK_DIV(mst_b_mclk,   AUDIO_MCLK_B_CTRL);
-static AXG_MST_MCLK_DIV(mst_c_mclk,   AUDIO_MCLK_C_CTRL);
-static AXG_MST_MCLK_DIV(mst_d_mclk,   AUDIO_MCLK_D_CTRL);
-static AXG_MST_MCLK_DIV(mst_e_mclk,   AUDIO_MCLK_E_CTRL);
-static AXG_MST_MCLK_DIV(mst_f_mclk,   AUDIO_MCLK_F_CTRL);
-static AXG_MST_MCLK_DIV(spdifout_clk, AUDIO_CLK_SPDIFOUT_CTRL);
-static AXG_MST_MCLK_DIV(pdm_dclk,     AUDIO_CLK_PDMIN_CTRL0);
-static AXG_MST_SYS_DIV(spdifin_clk,   AUDIO_CLK_SPDIFIN_CTRL);
-static AXG_MST_SYS_DIV(pdm_sysclk,    AUDIO_CLK_PDMIN_CTRL1);
-
-#define AXG_MST_MCLK_GATE(_name, _reg)				\
-	AXG_AUD_GATE(_name, _reg, 31,  "axg_"#_name"_div",	\
-		     CLK_SET_RATE_PARENT)
-
-static AXG_MST_MCLK_GATE(mst_a_mclk,   AUDIO_MCLK_A_CTRL);
-static AXG_MST_MCLK_GATE(mst_b_mclk,   AUDIO_MCLK_B_CTRL);
-static AXG_MST_MCLK_GATE(mst_c_mclk,   AUDIO_MCLK_C_CTRL);
-static AXG_MST_MCLK_GATE(mst_d_mclk,   AUDIO_MCLK_D_CTRL);
-static AXG_MST_MCLK_GATE(mst_e_mclk,   AUDIO_MCLK_E_CTRL);
-static AXG_MST_MCLK_GATE(mst_f_mclk,   AUDIO_MCLK_F_CTRL);
-static AXG_MST_MCLK_GATE(spdifout_clk, AUDIO_CLK_SPDIFOUT_CTRL);
-static AXG_MST_MCLK_GATE(spdifin_clk,  AUDIO_CLK_SPDIFIN_CTRL);
-static AXG_MST_MCLK_GATE(pdm_dclk,     AUDIO_CLK_PDMIN_CTRL0);
-static AXG_MST_MCLK_GATE(pdm_sysclk,   AUDIO_CLK_PDMIN_CTRL1);
+#define AUD_MST_MUX(_name, _reg, _flag)				\
+	AUD_MUX(_name##_sel, _reg, 0x7, 24, _flag,		\
+		mst_mux_parent_names, CLK_SET_RATE_PARENT)
+
+#define AUD_MST_MCLK_MUX(_name, _reg)				\
+	AUD_MST_MUX(_name, _reg, CLK_MUX_ROUND_CLOSEST)
+
+#define AUD_MST_SYS_MUX(_name, _reg)				\
+	AUD_MST_MUX(_name, _reg, 0)
+
+static AUD_MST_MCLK_MUX(mst_a_mclk,   AUDIO_MCLK_A_CTRL);
+static AUD_MST_MCLK_MUX(mst_b_mclk,   AUDIO_MCLK_B_CTRL);
+static AUD_MST_MCLK_MUX(mst_c_mclk,   AUDIO_MCLK_C_CTRL);
+static AUD_MST_MCLK_MUX(mst_d_mclk,   AUDIO_MCLK_D_CTRL);
+static AUD_MST_MCLK_MUX(mst_e_mclk,   AUDIO_MCLK_E_CTRL);
+static AUD_MST_MCLK_MUX(mst_f_mclk,   AUDIO_MCLK_F_CTRL);
+static AUD_MST_MCLK_MUX(spdifout_clk, AUDIO_CLK_SPDIFOUT_CTRL);
+static AUD_MST_MCLK_MUX(pdm_dclk,     AUDIO_CLK_PDMIN_CTRL0);
+static AUD_MST_SYS_MUX(spdifin_clk,   AUDIO_CLK_SPDIFIN_CTRL);
+static AUD_MST_SYS_MUX(pdm_sysclk,    AUDIO_CLK_PDMIN_CTRL1);
+static AUD_MST_MCLK_MUX(spdifout_b_clk, AUDIO_CLK_SPDIFOUT_B_CTRL);
+
+#define AUD_MST_DIV(_name, _reg, _flag)				\
+	AUD_DIV(_name##_div, _reg, 0, 16, _flag,		\
+		    "aud_"#_name"_sel", CLK_SET_RATE_PARENT)	\
+
+#define AUD_MST_MCLK_DIV(_name, _reg)				\
+	AUD_MST_DIV(_name, _reg, CLK_DIVIDER_ROUND_CLOSEST)
+
+#define AUD_MST_SYS_DIV(_name, _reg)				\
+	AUD_MST_DIV(_name, _reg, 0)
+
+static AUD_MST_MCLK_DIV(mst_a_mclk,   AUDIO_MCLK_A_CTRL);
+static AUD_MST_MCLK_DIV(mst_b_mclk,   AUDIO_MCLK_B_CTRL);
+static AUD_MST_MCLK_DIV(mst_c_mclk,   AUDIO_MCLK_C_CTRL);
+static AUD_MST_MCLK_DIV(mst_d_mclk,   AUDIO_MCLK_D_CTRL);
+static AUD_MST_MCLK_DIV(mst_e_mclk,   AUDIO_MCLK_E_CTRL);
+static AUD_MST_MCLK_DIV(mst_f_mclk,   AUDIO_MCLK_F_CTRL);
+static AUD_MST_MCLK_DIV(spdifout_clk, AUDIO_CLK_SPDIFOUT_CTRL);
+static AUD_MST_MCLK_DIV(pdm_dclk,     AUDIO_CLK_PDMIN_CTRL0);
+static AUD_MST_SYS_DIV(spdifin_clk,   AUDIO_CLK_SPDIFIN_CTRL);
+static AUD_MST_SYS_DIV(pdm_sysclk,    AUDIO_CLK_PDMIN_CTRL1);
+static AUD_MST_MCLK_DIV(spdifout_b_clk, AUDIO_CLK_SPDIFOUT_B_CTRL);
+
+#define AUD_MST_MCLK_GATE(_name, _reg)				\
+	AUD_GATE(_name, _reg, 31,  "aud_"#_name"_div",	\
+		 CLK_SET_RATE_PARENT)
+
+static AUD_MST_MCLK_GATE(mst_a_mclk,   AUDIO_MCLK_A_CTRL);
+static AUD_MST_MCLK_GATE(mst_b_mclk,   AUDIO_MCLK_B_CTRL);
+static AUD_MST_MCLK_GATE(mst_c_mclk,   AUDIO_MCLK_C_CTRL);
+static AUD_MST_MCLK_GATE(mst_d_mclk,   AUDIO_MCLK_D_CTRL);
+static AUD_MST_MCLK_GATE(mst_e_mclk,   AUDIO_MCLK_E_CTRL);
+static AUD_MST_MCLK_GATE(mst_f_mclk,   AUDIO_MCLK_F_CTRL);
+static AUD_MST_MCLK_GATE(spdifout_clk, AUDIO_CLK_SPDIFOUT_CTRL);
+static AUD_MST_MCLK_GATE(spdifin_clk,  AUDIO_CLK_SPDIFIN_CTRL);
+static AUD_MST_MCLK_GATE(pdm_dclk,     AUDIO_CLK_PDMIN_CTRL0);
+static AUD_MST_MCLK_GATE(pdm_sysclk,   AUDIO_CLK_PDMIN_CTRL1);
+static AUD_MST_MCLK_GATE(spdifout_b_clk, AUDIO_CLK_SPDIFOUT_B_CTRL);
 
 /* Sample Clocks */
-#define AXG_MST_SCLK_PRE_EN(_name, _reg)			\
-	AXG_AUD_GATE(mst_##_name##_sclk_pre_en, _reg, 31,	\
-		     "axg_mst_"#_name"_mclk", 0)
-
-static AXG_MST_SCLK_PRE_EN(a, AUDIO_MST_A_SCLK_CTRL0);
-static AXG_MST_SCLK_PRE_EN(b, AUDIO_MST_B_SCLK_CTRL0);
-static AXG_MST_SCLK_PRE_EN(c, AUDIO_MST_C_SCLK_CTRL0);
-static AXG_MST_SCLK_PRE_EN(d, AUDIO_MST_D_SCLK_CTRL0);
-static AXG_MST_SCLK_PRE_EN(e, AUDIO_MST_E_SCLK_CTRL0);
-static AXG_MST_SCLK_PRE_EN(f, AUDIO_MST_F_SCLK_CTRL0);
-
-#define AXG_AUD_SCLK_DIV(_name, _reg, _div_shift, _div_width,		\
+#define AUD_MST_SCLK_PRE_EN(_name, _reg)			\
+	AUD_GATE(mst_##_name##_sclk_pre_en, _reg, 31,		\
+		 "aud_mst_"#_name"_mclk", 0)
+
+static AUD_MST_SCLK_PRE_EN(a, AUDIO_MST_A_SCLK_CTRL0);
+static AUD_MST_SCLK_PRE_EN(b, AUDIO_MST_B_SCLK_CTRL0);
+static AUD_MST_SCLK_PRE_EN(c, AUDIO_MST_C_SCLK_CTRL0);
+static AUD_MST_SCLK_PRE_EN(d, AUDIO_MST_D_SCLK_CTRL0);
+static AUD_MST_SCLK_PRE_EN(e, AUDIO_MST_E_SCLK_CTRL0);
+static AUD_MST_SCLK_PRE_EN(f, AUDIO_MST_F_SCLK_CTRL0);
+
+#define AUD_SCLK_DIV(_name, _reg, _div_shift, _div_width,		\
 			 _hi_shift, _hi_width, _pname, _iflags)		\
-struct clk_regmap axg_##_name = {					\
+struct clk_regmap aud_##_name = {					\
 	.data = &(struct meson_sclk_div_data) {				\
 		.div = {						\
 			.reg_off = (_reg),				\
@@ -189,7 +193,7 @@ struct clk_regmap axg_##_name = {					\
 		},							\
 	},								\
 	.hw.init = &(struct clk_init_data) {				\
-		.name = "axg_"#_name,					\
+		.name = "aud_"#_name,					\
 		.ops = &meson_sclk_div_ops,				\
 		.parent_names = (const char *[]) { _pname },		\
 		.num_parents = 1,					\
@@ -197,32 +201,32 @@ struct clk_regmap axg_##_name = {					\
 	},								\
 }
 
-#define AXG_MST_SCLK_DIV(_name, _reg)					\
-	AXG_AUD_SCLK_DIV(mst_##_name##_sclk_div, _reg, 20, 10, 0, 0,	\
-			 "axg_mst_"#_name"_sclk_pre_en",		\
-			 CLK_SET_RATE_PARENT)
-
-static AXG_MST_SCLK_DIV(a, AUDIO_MST_A_SCLK_CTRL0);
-static AXG_MST_SCLK_DIV(b, AUDIO_MST_B_SCLK_CTRL0);
-static AXG_MST_SCLK_DIV(c, AUDIO_MST_C_SCLK_CTRL0);
-static AXG_MST_SCLK_DIV(d, AUDIO_MST_D_SCLK_CTRL0);
-static AXG_MST_SCLK_DIV(e, AUDIO_MST_E_SCLK_CTRL0);
-static AXG_MST_SCLK_DIV(f, AUDIO_MST_F_SCLK_CTRL0);
-
-#define AXG_MST_SCLK_POST_EN(_name, _reg)				\
-	AXG_AUD_GATE(mst_##_name##_sclk_post_en, _reg, 30,		\
-		     "axg_mst_"#_name"_sclk_div", CLK_SET_RATE_PARENT)
-
-static AXG_MST_SCLK_POST_EN(a, AUDIO_MST_A_SCLK_CTRL0);
-static AXG_MST_SCLK_POST_EN(b, AUDIO_MST_B_SCLK_CTRL0);
-static AXG_MST_SCLK_POST_EN(c, AUDIO_MST_C_SCLK_CTRL0);
-static AXG_MST_SCLK_POST_EN(d, AUDIO_MST_D_SCLK_CTRL0);
-static AXG_MST_SCLK_POST_EN(e, AUDIO_MST_E_SCLK_CTRL0);
-static AXG_MST_SCLK_POST_EN(f, AUDIO_MST_F_SCLK_CTRL0);
-
-#define AXG_AUD_TRIPHASE(_name, _reg, _width, _shift0, _shift1, _shift2, \
+#define AUD_MST_SCLK_DIV(_name, _reg)					\
+	AUD_SCLK_DIV(mst_##_name##_sclk_div, _reg, 20, 10, 0, 0,	\
+		     "aud_mst_"#_name"_sclk_pre_en",			\
+		     CLK_SET_RATE_PARENT)
+
+static AUD_MST_SCLK_DIV(a, AUDIO_MST_A_SCLK_CTRL0);
+static AUD_MST_SCLK_DIV(b, AUDIO_MST_B_SCLK_CTRL0);
+static AUD_MST_SCLK_DIV(c, AUDIO_MST_C_SCLK_CTRL0);
+static AUD_MST_SCLK_DIV(d, AUDIO_MST_D_SCLK_CTRL0);
+static AUD_MST_SCLK_DIV(e, AUDIO_MST_E_SCLK_CTRL0);
+static AUD_MST_SCLK_DIV(f, AUDIO_MST_F_SCLK_CTRL0);
+
+#define AUD_MST_SCLK_POST_EN(_name, _reg)				\
+	AUD_GATE(mst_##_name##_sclk_post_en, _reg, 30,		\
+		 "aud_mst_"#_name"_sclk_div", CLK_SET_RATE_PARENT)
+
+static AUD_MST_SCLK_POST_EN(a, AUDIO_MST_A_SCLK_CTRL0);
+static AUD_MST_SCLK_POST_EN(b, AUDIO_MST_B_SCLK_CTRL0);
+static AUD_MST_SCLK_POST_EN(c, AUDIO_MST_C_SCLK_CTRL0);
+static AUD_MST_SCLK_POST_EN(d, AUDIO_MST_D_SCLK_CTRL0);
+static AUD_MST_SCLK_POST_EN(e, AUDIO_MST_E_SCLK_CTRL0);
+static AUD_MST_SCLK_POST_EN(f, AUDIO_MST_F_SCLK_CTRL0);
+
+#define AUD_TRIPHASE(_name, _reg, _width, _shift0, _shift1, _shift2, \
 			 _pname, _iflags)				\
-struct clk_regmap axg_##_name = {					\
+struct clk_regmap aud_##_name = {					\
 	.data = &(struct meson_clk_triphase_data) {			\
 		.ph0 = {						\
 			.reg_off = (_reg),				\
@@ -241,7 +245,7 @@ struct clk_regmap axg_##_name = {					\
 		},							\
 	},								\
 	.hw.init = &(struct clk_init_data) {				\
-		.name = "axg_"#_name,					\
+		.name = "aud_"#_name,					\
 		.ops = &meson_clk_triphase_ops,				\
 		.parent_names = (const char *[]) { _pname },		\
 		.num_parents = 1,					\
@@ -249,87 +253,87 @@ struct clk_regmap axg_##_name = {					\
 	},								\
 }
 
-#define AXG_MST_SCLK(_name, _reg)					\
-	AXG_AUD_TRIPHASE(mst_##_name##_sclk, _reg, 1, 0, 2, 4,		\
-			 "axg_mst_"#_name"_sclk_post_en", CLK_SET_RATE_PARENT)
-
-static AXG_MST_SCLK(a, AUDIO_MST_A_SCLK_CTRL1);
-static AXG_MST_SCLK(b, AUDIO_MST_B_SCLK_CTRL1);
-static AXG_MST_SCLK(c, AUDIO_MST_C_SCLK_CTRL1);
-static AXG_MST_SCLK(d, AUDIO_MST_D_SCLK_CTRL1);
-static AXG_MST_SCLK(e, AUDIO_MST_E_SCLK_CTRL1);
-static AXG_MST_SCLK(f, AUDIO_MST_F_SCLK_CTRL1);
-
-#define AXG_MST_LRCLK_DIV(_name, _reg)					\
-	AXG_AUD_SCLK_DIV(mst_##_name##_lrclk_div, _reg, 0, 10, 10, 10,	\
-		    "axg_mst_"#_name"_sclk_post_en", 0)			\
-
-static AXG_MST_LRCLK_DIV(a, AUDIO_MST_A_SCLK_CTRL0);
-static AXG_MST_LRCLK_DIV(b, AUDIO_MST_B_SCLK_CTRL0);
-static AXG_MST_LRCLK_DIV(c, AUDIO_MST_C_SCLK_CTRL0);
-static AXG_MST_LRCLK_DIV(d, AUDIO_MST_D_SCLK_CTRL0);
-static AXG_MST_LRCLK_DIV(e, AUDIO_MST_E_SCLK_CTRL0);
-static AXG_MST_LRCLK_DIV(f, AUDIO_MST_F_SCLK_CTRL0);
-
-#define AXG_MST_LRCLK(_name, _reg)					\
-	AXG_AUD_TRIPHASE(mst_##_name##_lrclk, _reg, 1, 1, 3, 5,		\
-			 "axg_mst_"#_name"_lrclk_div", CLK_SET_RATE_PARENT)
-
-static AXG_MST_LRCLK(a, AUDIO_MST_A_SCLK_CTRL1);
-static AXG_MST_LRCLK(b, AUDIO_MST_B_SCLK_CTRL1);
-static AXG_MST_LRCLK(c, AUDIO_MST_C_SCLK_CTRL1);
-static AXG_MST_LRCLK(d, AUDIO_MST_D_SCLK_CTRL1);
-static AXG_MST_LRCLK(e, AUDIO_MST_E_SCLK_CTRL1);
-static AXG_MST_LRCLK(f, AUDIO_MST_F_SCLK_CTRL1);
+#define AUD_MST_SCLK(_name, _reg)					\
+	AUD_TRIPHASE(mst_##_name##_sclk, _reg, 1, 0, 2, 4,		\
+		     "aud_mst_"#_name"_sclk_post_en", CLK_SET_RATE_PARENT)
+
+static AUD_MST_SCLK(a, AUDIO_MST_A_SCLK_CTRL1);
+static AUD_MST_SCLK(b, AUDIO_MST_B_SCLK_CTRL1);
+static AUD_MST_SCLK(c, AUDIO_MST_C_SCLK_CTRL1);
+static AUD_MST_SCLK(d, AUDIO_MST_D_SCLK_CTRL1);
+static AUD_MST_SCLK(e, AUDIO_MST_E_SCLK_CTRL1);
+static AUD_MST_SCLK(f, AUDIO_MST_F_SCLK_CTRL1);
+
+#define AUD_MST_LRCLK_DIV(_name, _reg)					\
+	AUD_SCLK_DIV(mst_##_name##_lrclk_div, _reg, 0, 10, 10, 10,	\
+		     "aud_mst_"#_name"_sclk_post_en", 0)		\
+
+static AUD_MST_LRCLK_DIV(a, AUDIO_MST_A_SCLK_CTRL0);
+static AUD_MST_LRCLK_DIV(b, AUDIO_MST_B_SCLK_CTRL0);
+static AUD_MST_LRCLK_DIV(c, AUDIO_MST_C_SCLK_CTRL0);
+static AUD_MST_LRCLK_DIV(d, AUDIO_MST_D_SCLK_CTRL0);
+static AUD_MST_LRCLK_DIV(e, AUDIO_MST_E_SCLK_CTRL0);
+static AUD_MST_LRCLK_DIV(f, AUDIO_MST_F_SCLK_CTRL0);
+
+#define AUD_MST_LRCLK(_name, _reg)					\
+	AUD_TRIPHASE(mst_##_name##_lrclk, _reg, 1, 1, 3, 5,		\
+		     "aud_mst_"#_name"_lrclk_div", CLK_SET_RATE_PARENT)
+
+static AUD_MST_LRCLK(a, AUDIO_MST_A_SCLK_CTRL1);
+static AUD_MST_LRCLK(b, AUDIO_MST_B_SCLK_CTRL1);
+static AUD_MST_LRCLK(c, AUDIO_MST_C_SCLK_CTRL1);
+static AUD_MST_LRCLK(d, AUDIO_MST_D_SCLK_CTRL1);
+static AUD_MST_LRCLK(e, AUDIO_MST_E_SCLK_CTRL1);
+static AUD_MST_LRCLK(f, AUDIO_MST_F_SCLK_CTRL1);
 
 static const char * const tdm_sclk_parent_names[] = {
-	"axg_mst_a_sclk", "axg_mst_b_sclk", "axg_mst_c_sclk",
-	"axg_mst_d_sclk", "axg_mst_e_sclk", "axg_mst_f_sclk",
-	"axg_slv_sclk0", "axg_slv_sclk1", "axg_slv_sclk2",
-	"axg_slv_sclk3", "axg_slv_sclk4", "axg_slv_sclk5",
-	"axg_slv_sclk6", "axg_slv_sclk7", "axg_slv_sclk8",
-	"axg_slv_sclk9"
+	"aud_mst_a_sclk", "aud_mst_b_sclk", "aud_mst_c_sclk",
+	"aud_mst_d_sclk", "aud_mst_e_sclk", "aud_mst_f_sclk",
+	"aud_slv_sclk0", "aud_slv_sclk1", "aud_slv_sclk2",
+	"aud_slv_sclk3", "aud_slv_sclk4", "aud_slv_sclk5",
+	"aud_slv_sclk6", "aud_slv_sclk7", "aud_slv_sclk8",
+	"aud_slv_sclk9"
 };
 
-#define AXG_TDM_SCLK_MUX(_name, _reg)				\
-	AXG_AUD_MUX(tdm##_name##_sclk_sel, _reg, 0xf, 24,	\
+#define AUD_TDM_SCLK_MUX(_name, _reg)				\
+	AUD_MUX(tdm##_name##_sclk_sel, _reg, 0xf, 24,		\
 		    CLK_MUX_ROUND_CLOSEST,			\
 		    tdm_sclk_parent_names, 0)
 
-static AXG_TDM_SCLK_MUX(in_a,  AUDIO_CLK_TDMIN_A_CTRL);
-static AXG_TDM_SCLK_MUX(in_b,  AUDIO_CLK_TDMIN_B_CTRL);
-static AXG_TDM_SCLK_MUX(in_c,  AUDIO_CLK_TDMIN_C_CTRL);
-static AXG_TDM_SCLK_MUX(in_lb, AUDIO_CLK_TDMIN_LB_CTRL);
-static AXG_TDM_SCLK_MUX(out_a, AUDIO_CLK_TDMOUT_A_CTRL);
-static AXG_TDM_SCLK_MUX(out_b, AUDIO_CLK_TDMOUT_B_CTRL);
-static AXG_TDM_SCLK_MUX(out_c, AUDIO_CLK_TDMOUT_C_CTRL);
-
-#define AXG_TDM_SCLK_PRE_EN(_name, _reg)				\
-	AXG_AUD_GATE(tdm##_name##_sclk_pre_en, _reg, 31,		\
-		     "axg_tdm"#_name"_sclk_sel", CLK_SET_RATE_PARENT)
-
-static AXG_TDM_SCLK_PRE_EN(in_a,  AUDIO_CLK_TDMIN_A_CTRL);
-static AXG_TDM_SCLK_PRE_EN(in_b,  AUDIO_CLK_TDMIN_B_CTRL);
-static AXG_TDM_SCLK_PRE_EN(in_c,  AUDIO_CLK_TDMIN_C_CTRL);
-static AXG_TDM_SCLK_PRE_EN(in_lb, AUDIO_CLK_TDMIN_LB_CTRL);
-static AXG_TDM_SCLK_PRE_EN(out_a, AUDIO_CLK_TDMOUT_A_CTRL);
-static AXG_TDM_SCLK_PRE_EN(out_b, AUDIO_CLK_TDMOUT_B_CTRL);
-static AXG_TDM_SCLK_PRE_EN(out_c, AUDIO_CLK_TDMOUT_C_CTRL);
-
-#define AXG_TDM_SCLK_POST_EN(_name, _reg)				\
-	AXG_AUD_GATE(tdm##_name##_sclk_post_en, _reg, 30,		\
-		     "axg_tdm"#_name"_sclk_pre_en", CLK_SET_RATE_PARENT)
-
-static AXG_TDM_SCLK_POST_EN(in_a,  AUDIO_CLK_TDMIN_A_CTRL);
-static AXG_TDM_SCLK_POST_EN(in_b,  AUDIO_CLK_TDMIN_B_CTRL);
-static AXG_TDM_SCLK_POST_EN(in_c,  AUDIO_CLK_TDMIN_C_CTRL);
-static AXG_TDM_SCLK_POST_EN(in_lb, AUDIO_CLK_TDMIN_LB_CTRL);
-static AXG_TDM_SCLK_POST_EN(out_a, AUDIO_CLK_TDMOUT_A_CTRL);
-static AXG_TDM_SCLK_POST_EN(out_b, AUDIO_CLK_TDMOUT_B_CTRL);
-static AXG_TDM_SCLK_POST_EN(out_c, AUDIO_CLK_TDMOUT_C_CTRL);
-
-#define AXG_TDM_SCLK(_name, _reg)					\
-	struct clk_regmap axg_tdm##_name##_sclk = {			\
+static AUD_TDM_SCLK_MUX(in_a,  AUDIO_CLK_TDMIN_A_CTRL);
+static AUD_TDM_SCLK_MUX(in_b,  AUDIO_CLK_TDMIN_B_CTRL);
+static AUD_TDM_SCLK_MUX(in_c,  AUDIO_CLK_TDMIN_C_CTRL);
+static AUD_TDM_SCLK_MUX(in_lb, AUDIO_CLK_TDMIN_LB_CTRL);
+static AUD_TDM_SCLK_MUX(out_a, AUDIO_CLK_TDMOUT_A_CTRL);
+static AUD_TDM_SCLK_MUX(out_b, AUDIO_CLK_TDMOUT_B_CTRL);
+static AUD_TDM_SCLK_MUX(out_c, AUDIO_CLK_TDMOUT_C_CTRL);
+
+#define AUD_TDM_SCLK_PRE_EN(_name, _reg)				\
+	AUD_GATE(tdm##_name##_sclk_pre_en, _reg, 31,			\
+		 "aud_tdm"#_name"_sclk_sel", CLK_SET_RATE_PARENT)
+
+static AUD_TDM_SCLK_PRE_EN(in_a,  AUDIO_CLK_TDMIN_A_CTRL);
+static AUD_TDM_SCLK_PRE_EN(in_b,  AUDIO_CLK_TDMIN_B_CTRL);
+static AUD_TDM_SCLK_PRE_EN(in_c,  AUDIO_CLK_TDMIN_C_CTRL);
+static AUD_TDM_SCLK_PRE_EN(in_lb, AUDIO_CLK_TDMIN_LB_CTRL);
+static AUD_TDM_SCLK_PRE_EN(out_a, AUDIO_CLK_TDMOUT_A_CTRL);
+static AUD_TDM_SCLK_PRE_EN(out_b, AUDIO_CLK_TDMOUT_B_CTRL);
+static AUD_TDM_SCLK_PRE_EN(out_c, AUDIO_CLK_TDMOUT_C_CTRL);
+
+#define AUD_TDM_SCLK_POST_EN(_name, _reg)				\
+	AUD_GATE(tdm##_name##_sclk_post_en, _reg, 30,			\
+		 "aud_tdm"#_name"_sclk_pre_en", CLK_SET_RATE_PARENT)
+
+static AUD_TDM_SCLK_POST_EN(in_a,  AUDIO_CLK_TDMIN_A_CTRL);
+static AUD_TDM_SCLK_POST_EN(in_b,  AUDIO_CLK_TDMIN_B_CTRL);
+static AUD_TDM_SCLK_POST_EN(in_c,  AUDIO_CLK_TDMIN_C_CTRL);
+static AUD_TDM_SCLK_POST_EN(in_lb, AUDIO_CLK_TDMIN_LB_CTRL);
+static AUD_TDM_SCLK_POST_EN(out_a, AUDIO_CLK_TDMOUT_A_CTRL);
+static AUD_TDM_SCLK_POST_EN(out_b, AUDIO_CLK_TDMOUT_B_CTRL);
+static AUD_TDM_SCLK_POST_EN(out_c, AUDIO_CLK_TDMOUT_C_CTRL);
+
+#define AUD_TDM_SCLK(_name, _reg)					\
+	struct clk_regmap aud_tdm##_name##_sclk = {			\
 	.data = &(struct meson_clk_phase_data) {			\
 		.ph = {							\
 			.reg_off = (_reg),				\
@@ -338,44 +342,83 @@ static AXG_TDM_SCLK_POST_EN(out_c, AUDIO_CLK_TDMOUT_C_CTRL);
 		},							\
 	},								\
 	.hw.init = &(struct clk_init_data) {				\
-		.name = "axg_tdm"#_name"_sclk",				\
+		.name = "aud_tdm"#_name"_sclk",				\
 		.ops = &meson_clk_phase_ops,				\
 		.parent_names = (const char *[])			\
-		{ "axg_tdm"#_name"_sclk_post_en" },			\
+		{ "aud_tdm"#_name"_sclk_post_en" },			\
 		.num_parents = 1,					\
 		.flags = CLK_DUTY_CYCLE_PARENT | CLK_SET_RATE_PARENT,	\
 	},								\
 }
 
-static AXG_TDM_SCLK(in_a,  AUDIO_CLK_TDMIN_A_CTRL);
-static AXG_TDM_SCLK(in_b,  AUDIO_CLK_TDMIN_B_CTRL);
-static AXG_TDM_SCLK(in_c,  AUDIO_CLK_TDMIN_C_CTRL);
-static AXG_TDM_SCLK(in_lb, AUDIO_CLK_TDMIN_LB_CTRL);
-static AXG_TDM_SCLK(out_a, AUDIO_CLK_TDMOUT_A_CTRL);
-static AXG_TDM_SCLK(out_b, AUDIO_CLK_TDMOUT_B_CTRL);
-static AXG_TDM_SCLK(out_c, AUDIO_CLK_TDMOUT_C_CTRL);
+static AUD_TDM_SCLK(in_a,  AUDIO_CLK_TDMIN_A_CTRL);
+static AUD_TDM_SCLK(in_b,  AUDIO_CLK_TDMIN_B_CTRL);
+static AUD_TDM_SCLK(in_c,  AUDIO_CLK_TDMIN_C_CTRL);
+static AUD_TDM_SCLK(in_lb, AUDIO_CLK_TDMIN_LB_CTRL);
+static AUD_TDM_SCLK(out_a, AUDIO_CLK_TDMOUT_A_CTRL);
+static AUD_TDM_SCLK(out_b, AUDIO_CLK_TDMOUT_B_CTRL);
+static AUD_TDM_SCLK(out_c, AUDIO_CLK_TDMOUT_C_CTRL);
 
 static const char * const tdm_lrclk_parent_names[] = {
-	"axg_mst_a_lrclk", "axg_mst_b_lrclk", "axg_mst_c_lrclk",
-	"axg_mst_d_lrclk", "axg_mst_e_lrclk", "axg_mst_f_lrclk",
-	"axg_slv_lrclk0", "axg_slv_lrclk1", "axg_slv_lrclk2",
-	"axg_slv_lrclk3", "axg_slv_lrclk4", "axg_slv_lrclk5",
-	"axg_slv_lrclk6", "axg_slv_lrclk7", "axg_slv_lrclk8",
-	"axg_slv_lrclk9"
+	"aud_mst_a_lrclk", "aud_mst_b_lrclk", "aud_mst_c_lrclk",
+	"aud_mst_d_lrclk", "aud_mst_e_lrclk", "aud_mst_f_lrclk",
+	"aud_slv_lrclk0", "aud_slv_lrclk1", "aud_slv_lrclk2",
+	"aud_slv_lrclk3", "aud_slv_lrclk4", "aud_slv_lrclk5",
+	"aud_slv_lrclk6", "aud_slv_lrclk7", "aud_slv_lrclk8",
+	"aud_slv_lrclk9"
 };
 
-#define AXG_TDM_LRLCK(_name, _reg)		       \
-	AXG_AUD_MUX(tdm##_name##_lrclk, _reg, 0xf, 20, \
-		    CLK_MUX_ROUND_CLOSEST,	       \
-		    tdm_lrclk_parent_names, 0)
+#define AUD_TDM_LRLCK(_name, _reg)		       \
+	AUD_MUX(tdm##_name##_lrclk, _reg, 0xf, 20,     \
+		CLK_MUX_ROUND_CLOSEST,		       \
+		tdm_lrclk_parent_names, 0)
+
+static AUD_TDM_LRLCK(in_a,  AUDIO_CLK_TDMIN_A_CTRL);
+static AUD_TDM_LRLCK(in_b,  AUDIO_CLK_TDMIN_B_CTRL);
+static AUD_TDM_LRLCK(in_c,  AUDIO_CLK_TDMIN_C_CTRL);
+static AUD_TDM_LRLCK(in_lb, AUDIO_CLK_TDMIN_LB_CTRL);
+static AUD_TDM_LRLCK(out_a, AUDIO_CLK_TDMOUT_A_CTRL);
+static AUD_TDM_LRLCK(out_b, AUDIO_CLK_TDMOUT_B_CTRL);
+static AUD_TDM_LRLCK(out_c, AUDIO_CLK_TDMOUT_C_CTRL);
+
+/* G12a Pad control */
+#define AUD_TDM_PAD_CTRL(_name, _reg, _shift, _parents)		\
+	AUD_MUX(tdm_##_name, _reg, 0x7, _shift, 0, _parents,	\
+		CLK_SET_RATE_NO_REPARENT)
+
+static const char * const mclk_pad_ctrl_parent_names[] = {
+	"aud_mst_a_mclk", "aud_mst_b_mclk", "aud_mst_c_mclk",
+	"aud_mst_d_mclk", "aud_mst_e_mclk", "aud_mst_f_mclk",
+};
+
+static AUD_TDM_PAD_CTRL(mclk_pad_0, AUDIO_MST_PAD_CTRL0, 0,
+			mclk_pad_ctrl_parent_names);
+static AUD_TDM_PAD_CTRL(mclk_pad_1, AUDIO_MST_PAD_CTRL0, 4,
+			mclk_pad_ctrl_parent_names);
+
+static const char * const lrclk_pad_ctrl_parent_names[] = {
+	"aud_mst_a_lrclk", "aud_mst_b_lrclk", "aud_mst_c_lrclk",
+	"aud_mst_d_lrclk", "aud_mst_e_lrclk", "aud_mst_f_lrclk",
+};
 
-static AXG_TDM_LRLCK(in_a,  AUDIO_CLK_TDMIN_A_CTRL);
-static AXG_TDM_LRLCK(in_b,  AUDIO_CLK_TDMIN_B_CTRL);
-static AXG_TDM_LRLCK(in_c,  AUDIO_CLK_TDMIN_C_CTRL);
-static AXG_TDM_LRLCK(in_lb, AUDIO_CLK_TDMIN_LB_CTRL);
-static AXG_TDM_LRLCK(out_a, AUDIO_CLK_TDMOUT_A_CTRL);
-static AXG_TDM_LRLCK(out_b, AUDIO_CLK_TDMOUT_B_CTRL);
-static AXG_TDM_LRLCK(out_c, AUDIO_CLK_TDMOUT_C_CTRL);
+static AUD_TDM_PAD_CTRL(lrclk_pad_0, AUDIO_MST_PAD_CTRL1, 16,
+			lrclk_pad_ctrl_parent_names);
+static AUD_TDM_PAD_CTRL(lrclk_pad_1, AUDIO_MST_PAD_CTRL1, 20,
+			lrclk_pad_ctrl_parent_names);
+static AUD_TDM_PAD_CTRL(lrclk_pad_2, AUDIO_MST_PAD_CTRL1, 24,
+			lrclk_pad_ctrl_parent_names);
+
+static const char * const sclk_pad_ctrl_parent_names[] = {
+	"aud_mst_a_sclk", "aud_mst_b_sclk", "aud_mst_c_sclk",
+	"aud_mst_d_sclk", "aud_mst_e_sclk", "aud_mst_f_sclk",
+};
+
+static AUD_TDM_PAD_CTRL(sclk_pad_0, AUDIO_MST_PAD_CTRL1, 0,
+			sclk_pad_ctrl_parent_names);
+static AUD_TDM_PAD_CTRL(sclk_pad_1, AUDIO_MST_PAD_CTRL1, 4,
+			sclk_pad_ctrl_parent_names);
+static AUD_TDM_PAD_CTRL(sclk_pad_2, AUDIO_MST_PAD_CTRL1, 8,
+			sclk_pad_ctrl_parent_names);
 
 /*
  * Array of all clocks provided by this provider
@@ -383,255 +426,416 @@ static AXG_TDM_LRLCK(out_c, AUDIO_CLK_TDMOUT_C_CTRL);
  */
 static struct clk_hw_onecell_data axg_audio_hw_onecell_data = {
 	.hws = {
-		[AUD_CLKID_DDR_ARB]		= &axg_ddr_arb.hw,
-		[AUD_CLKID_PDM]			= &axg_pdm.hw,
-		[AUD_CLKID_TDMIN_A]		= &axg_tdmin_a.hw,
-		[AUD_CLKID_TDMIN_B]		= &axg_tdmin_b.hw,
-		[AUD_CLKID_TDMIN_C]		= &axg_tdmin_c.hw,
-		[AUD_CLKID_TDMIN_LB]		= &axg_tdmin_lb.hw,
-		[AUD_CLKID_TDMOUT_A]		= &axg_tdmout_a.hw,
-		[AUD_CLKID_TDMOUT_B]		= &axg_tdmout_b.hw,
-		[AUD_CLKID_TDMOUT_C]		= &axg_tdmout_c.hw,
-		[AUD_CLKID_FRDDR_A]		= &axg_frddr_a.hw,
-		[AUD_CLKID_FRDDR_B]		= &axg_frddr_b.hw,
-		[AUD_CLKID_FRDDR_C]		= &axg_frddr_c.hw,
-		[AUD_CLKID_TODDR_A]		= &axg_toddr_a.hw,
-		[AUD_CLKID_TODDR_B]		= &axg_toddr_b.hw,
-		[AUD_CLKID_TODDR_C]		= &axg_toddr_c.hw,
-		[AUD_CLKID_LOOPBACK]		= &axg_loopback.hw,
-		[AUD_CLKID_SPDIFIN]		= &axg_spdifin.hw,
-		[AUD_CLKID_SPDIFOUT]		= &axg_spdifout.hw,
-		[AUD_CLKID_RESAMPLE]		= &axg_resample.hw,
-		[AUD_CLKID_POWER_DETECT]	= &axg_power_detect.hw,
-		[AUD_CLKID_MST_A_MCLK_SEL]	= &axg_mst_a_mclk_sel.hw,
-		[AUD_CLKID_MST_B_MCLK_SEL]	= &axg_mst_b_mclk_sel.hw,
-		[AUD_CLKID_MST_C_MCLK_SEL]	= &axg_mst_c_mclk_sel.hw,
-		[AUD_CLKID_MST_D_MCLK_SEL]	= &axg_mst_d_mclk_sel.hw,
-		[AUD_CLKID_MST_E_MCLK_SEL]	= &axg_mst_e_mclk_sel.hw,
-		[AUD_CLKID_MST_F_MCLK_SEL]	= &axg_mst_f_mclk_sel.hw,
-		[AUD_CLKID_MST_A_MCLK_DIV]	= &axg_mst_a_mclk_div.hw,
-		[AUD_CLKID_MST_B_MCLK_DIV]	= &axg_mst_b_mclk_div.hw,
-		[AUD_CLKID_MST_C_MCLK_DIV]	= &axg_mst_c_mclk_div.hw,
-		[AUD_CLKID_MST_D_MCLK_DIV]	= &axg_mst_d_mclk_div.hw,
-		[AUD_CLKID_MST_E_MCLK_DIV]	= &axg_mst_e_mclk_div.hw,
-		[AUD_CLKID_MST_F_MCLK_DIV]	= &axg_mst_f_mclk_div.hw,
-		[AUD_CLKID_MST_A_MCLK]		= &axg_mst_a_mclk.hw,
-		[AUD_CLKID_MST_B_MCLK]		= &axg_mst_b_mclk.hw,
-		[AUD_CLKID_MST_C_MCLK]		= &axg_mst_c_mclk.hw,
-		[AUD_CLKID_MST_D_MCLK]		= &axg_mst_d_mclk.hw,
-		[AUD_CLKID_MST_E_MCLK]		= &axg_mst_e_mclk.hw,
-		[AUD_CLKID_MST_F_MCLK]		= &axg_mst_f_mclk.hw,
-		[AUD_CLKID_SPDIFOUT_CLK_SEL]	= &axg_spdifout_clk_sel.hw,
-		[AUD_CLKID_SPDIFOUT_CLK_DIV]	= &axg_spdifout_clk_div.hw,
-		[AUD_CLKID_SPDIFOUT_CLK]	= &axg_spdifout_clk.hw,
-		[AUD_CLKID_SPDIFIN_CLK_SEL]	= &axg_spdifin_clk_sel.hw,
-		[AUD_CLKID_SPDIFIN_CLK_DIV]	= &axg_spdifin_clk_div.hw,
-		[AUD_CLKID_SPDIFIN_CLK]		= &axg_spdifin_clk.hw,
-		[AUD_CLKID_PDM_DCLK_SEL]	= &axg_pdm_dclk_sel.hw,
-		[AUD_CLKID_PDM_DCLK_DIV]	= &axg_pdm_dclk_div.hw,
-		[AUD_CLKID_PDM_DCLK]		= &axg_pdm_dclk.hw,
-		[AUD_CLKID_PDM_SYSCLK_SEL]	= &axg_pdm_sysclk_sel.hw,
-		[AUD_CLKID_PDM_SYSCLK_DIV]	= &axg_pdm_sysclk_div.hw,
-		[AUD_CLKID_PDM_SYSCLK]		= &axg_pdm_sysclk.hw,
-		[AUD_CLKID_MST_A_SCLK_PRE_EN]	= &axg_mst_a_sclk_pre_en.hw,
-		[AUD_CLKID_MST_B_SCLK_PRE_EN]	= &axg_mst_b_sclk_pre_en.hw,
-		[AUD_CLKID_MST_C_SCLK_PRE_EN]	= &axg_mst_c_sclk_pre_en.hw,
-		[AUD_CLKID_MST_D_SCLK_PRE_EN]	= &axg_mst_d_sclk_pre_en.hw,
-		[AUD_CLKID_MST_E_SCLK_PRE_EN]	= &axg_mst_e_sclk_pre_en.hw,
-		[AUD_CLKID_MST_F_SCLK_PRE_EN]	= &axg_mst_f_sclk_pre_en.hw,
-		[AUD_CLKID_MST_A_SCLK_DIV]	= &axg_mst_a_sclk_div.hw,
-		[AUD_CLKID_MST_B_SCLK_DIV]	= &axg_mst_b_sclk_div.hw,
-		[AUD_CLKID_MST_C_SCLK_DIV]	= &axg_mst_c_sclk_div.hw,
-		[AUD_CLKID_MST_D_SCLK_DIV]	= &axg_mst_d_sclk_div.hw,
-		[AUD_CLKID_MST_E_SCLK_DIV]	= &axg_mst_e_sclk_div.hw,
-		[AUD_CLKID_MST_F_SCLK_DIV]	= &axg_mst_f_sclk_div.hw,
-		[AUD_CLKID_MST_A_SCLK_POST_EN]	= &axg_mst_a_sclk_post_en.hw,
-		[AUD_CLKID_MST_B_SCLK_POST_EN]	= &axg_mst_b_sclk_post_en.hw,
-		[AUD_CLKID_MST_C_SCLK_POST_EN]	= &axg_mst_c_sclk_post_en.hw,
-		[AUD_CLKID_MST_D_SCLK_POST_EN]	= &axg_mst_d_sclk_post_en.hw,
-		[AUD_CLKID_MST_E_SCLK_POST_EN]	= &axg_mst_e_sclk_post_en.hw,
-		[AUD_CLKID_MST_F_SCLK_POST_EN]	= &axg_mst_f_sclk_post_en.hw,
-		[AUD_CLKID_MST_A_SCLK]		= &axg_mst_a_sclk.hw,
-		[AUD_CLKID_MST_B_SCLK]		= &axg_mst_b_sclk.hw,
-		[AUD_CLKID_MST_C_SCLK]		= &axg_mst_c_sclk.hw,
-		[AUD_CLKID_MST_D_SCLK]		= &axg_mst_d_sclk.hw,
-		[AUD_CLKID_MST_E_SCLK]		= &axg_mst_e_sclk.hw,
-		[AUD_CLKID_MST_F_SCLK]		= &axg_mst_f_sclk.hw,
-		[AUD_CLKID_MST_A_LRCLK_DIV]	= &axg_mst_a_lrclk_div.hw,
-		[AUD_CLKID_MST_B_LRCLK_DIV]	= &axg_mst_b_lrclk_div.hw,
-		[AUD_CLKID_MST_C_LRCLK_DIV]	= &axg_mst_c_lrclk_div.hw,
-		[AUD_CLKID_MST_D_LRCLK_DIV]	= &axg_mst_d_lrclk_div.hw,
-		[AUD_CLKID_MST_E_LRCLK_DIV]	= &axg_mst_e_lrclk_div.hw,
-		[AUD_CLKID_MST_F_LRCLK_DIV]	= &axg_mst_f_lrclk_div.hw,
-		[AUD_CLKID_MST_A_LRCLK]		= &axg_mst_a_lrclk.hw,
-		[AUD_CLKID_MST_B_LRCLK]		= &axg_mst_b_lrclk.hw,
-		[AUD_CLKID_MST_C_LRCLK]		= &axg_mst_c_lrclk.hw,
-		[AUD_CLKID_MST_D_LRCLK]		= &axg_mst_d_lrclk.hw,
-		[AUD_CLKID_MST_E_LRCLK]		= &axg_mst_e_lrclk.hw,
-		[AUD_CLKID_MST_F_LRCLK]		= &axg_mst_f_lrclk.hw,
-		[AUD_CLKID_TDMIN_A_SCLK_SEL]	= &axg_tdmin_a_sclk_sel.hw,
-		[AUD_CLKID_TDMIN_B_SCLK_SEL]	= &axg_tdmin_b_sclk_sel.hw,
-		[AUD_CLKID_TDMIN_C_SCLK_SEL]	= &axg_tdmin_c_sclk_sel.hw,
-		[AUD_CLKID_TDMIN_LB_SCLK_SEL]	= &axg_tdmin_lb_sclk_sel.hw,
-		[AUD_CLKID_TDMOUT_A_SCLK_SEL]	= &axg_tdmout_a_sclk_sel.hw,
-		[AUD_CLKID_TDMOUT_B_SCLK_SEL]	= &axg_tdmout_b_sclk_sel.hw,
-		[AUD_CLKID_TDMOUT_C_SCLK_SEL]	= &axg_tdmout_c_sclk_sel.hw,
-		[AUD_CLKID_TDMIN_A_SCLK_PRE_EN]	= &axg_tdmin_a_sclk_pre_en.hw,
-		[AUD_CLKID_TDMIN_B_SCLK_PRE_EN]	= &axg_tdmin_b_sclk_pre_en.hw,
-		[AUD_CLKID_TDMIN_C_SCLK_PRE_EN]	= &axg_tdmin_c_sclk_pre_en.hw,
-		[AUD_CLKID_TDMIN_LB_SCLK_PRE_EN] = &axg_tdmin_lb_sclk_pre_en.hw,
-		[AUD_CLKID_TDMOUT_A_SCLK_PRE_EN] = &axg_tdmout_a_sclk_pre_en.hw,
-		[AUD_CLKID_TDMOUT_B_SCLK_PRE_EN] = &axg_tdmout_b_sclk_pre_en.hw,
-		[AUD_CLKID_TDMOUT_C_SCLK_PRE_EN] = &axg_tdmout_c_sclk_pre_en.hw,
-		[AUD_CLKID_TDMIN_A_SCLK_POST_EN] = &axg_tdmin_a_sclk_post_en.hw,
-		[AUD_CLKID_TDMIN_B_SCLK_POST_EN] = &axg_tdmin_b_sclk_post_en.hw,
-		[AUD_CLKID_TDMIN_C_SCLK_POST_EN] = &axg_tdmin_c_sclk_post_en.hw,
-		[AUD_CLKID_TDMIN_LB_SCLK_POST_EN] = &axg_tdmin_lb_sclk_post_en.hw,
-		[AUD_CLKID_TDMOUT_A_SCLK_POST_EN] = &axg_tdmout_a_sclk_post_en.hw,
-		[AUD_CLKID_TDMOUT_B_SCLK_POST_EN] = &axg_tdmout_b_sclk_post_en.hw,
-		[AUD_CLKID_TDMOUT_C_SCLK_POST_EN] = &axg_tdmout_c_sclk_post_en.hw,
-		[AUD_CLKID_TDMIN_A_SCLK]	= &axg_tdmin_a_sclk.hw,
-		[AUD_CLKID_TDMIN_B_SCLK]	= &axg_tdmin_b_sclk.hw,
-		[AUD_CLKID_TDMIN_C_SCLK]	= &axg_tdmin_c_sclk.hw,
-		[AUD_CLKID_TDMIN_LB_SCLK]	= &axg_tdmin_lb_sclk.hw,
-		[AUD_CLKID_TDMOUT_A_SCLK]	= &axg_tdmout_a_sclk.hw,
-		[AUD_CLKID_TDMOUT_B_SCLK]	= &axg_tdmout_b_sclk.hw,
-		[AUD_CLKID_TDMOUT_C_SCLK]	= &axg_tdmout_c_sclk.hw,
-		[AUD_CLKID_TDMIN_A_LRCLK]	= &axg_tdmin_a_lrclk.hw,
-		[AUD_CLKID_TDMIN_B_LRCLK]	= &axg_tdmin_b_lrclk.hw,
-		[AUD_CLKID_TDMIN_C_LRCLK]	= &axg_tdmin_c_lrclk.hw,
-		[AUD_CLKID_TDMIN_LB_LRCLK]	= &axg_tdmin_lb_lrclk.hw,
-		[AUD_CLKID_TDMOUT_A_LRCLK]	= &axg_tdmout_a_lrclk.hw,
-		[AUD_CLKID_TDMOUT_B_LRCLK]	= &axg_tdmout_b_lrclk.hw,
-		[AUD_CLKID_TDMOUT_C_LRCLK]	= &axg_tdmout_c_lrclk.hw,
+		[AUD_CLKID_DDR_ARB]		= &aud_ddr_arb.hw,
+		[AUD_CLKID_PDM]			= &aud_pdm.hw,
+		[AUD_CLKID_TDMIN_A]		= &aud_tdmin_a.hw,
+		[AUD_CLKID_TDMIN_B]		= &aud_tdmin_b.hw,
+		[AUD_CLKID_TDMIN_C]		= &aud_tdmin_c.hw,
+		[AUD_CLKID_TDMIN_LB]		= &aud_tdmin_lb.hw,
+		[AUD_CLKID_TDMOUT_A]		= &aud_tdmout_a.hw,
+		[AUD_CLKID_TDMOUT_B]		= &aud_tdmout_b.hw,
+		[AUD_CLKID_TDMOUT_C]		= &aud_tdmout_c.hw,
+		[AUD_CLKID_FRDDR_A]		= &aud_frddr_a.hw,
+		[AUD_CLKID_FRDDR_B]		= &aud_frddr_b.hw,
+		[AUD_CLKID_FRDDR_C]		= &aud_frddr_c.hw,
+		[AUD_CLKID_TODDR_A]		= &aud_toddr_a.hw,
+		[AUD_CLKID_TODDR_B]		= &aud_toddr_b.hw,
+		[AUD_CLKID_TODDR_C]		= &aud_toddr_c.hw,
+		[AUD_CLKID_LOOPBACK]		= &aud_loopback.hw,
+		[AUD_CLKID_SPDIFIN]		= &aud_spdifin.hw,
+		[AUD_CLKID_SPDIFOUT]		= &aud_spdifout.hw,
+		[AUD_CLKID_RESAMPLE]		= &aud_resample.hw,
+		[AUD_CLKID_POWER_DETECT]	= &aud_power_detect.hw,
+		[AUD_CLKID_MST_A_MCLK_SEL]	= &aud_mst_a_mclk_sel.hw,
+		[AUD_CLKID_MST_B_MCLK_SEL]	= &aud_mst_b_mclk_sel.hw,
+		[AUD_CLKID_MST_C_MCLK_SEL]	= &aud_mst_c_mclk_sel.hw,
+		[AUD_CLKID_MST_D_MCLK_SEL]	= &aud_mst_d_mclk_sel.hw,
+		[AUD_CLKID_MST_E_MCLK_SEL]	= &aud_mst_e_mclk_sel.hw,
+		[AUD_CLKID_MST_F_MCLK_SEL]	= &aud_mst_f_mclk_sel.hw,
+		[AUD_CLKID_MST_A_MCLK_DIV]	= &aud_mst_a_mclk_div.hw,
+		[AUD_CLKID_MST_B_MCLK_DIV]	= &aud_mst_b_mclk_div.hw,
+		[AUD_CLKID_MST_C_MCLK_DIV]	= &aud_mst_c_mclk_div.hw,
+		[AUD_CLKID_MST_D_MCLK_DIV]	= &aud_mst_d_mclk_div.hw,
+		[AUD_CLKID_MST_E_MCLK_DIV]	= &aud_mst_e_mclk_div.hw,
+		[AUD_CLKID_MST_F_MCLK_DIV]	= &aud_mst_f_mclk_div.hw,
+		[AUD_CLKID_MST_A_MCLK]		= &aud_mst_a_mclk.hw,
+		[AUD_CLKID_MST_B_MCLK]		= &aud_mst_b_mclk.hw,
+		[AUD_CLKID_MST_C_MCLK]		= &aud_mst_c_mclk.hw,
+		[AUD_CLKID_MST_D_MCLK]		= &aud_mst_d_mclk.hw,
+		[AUD_CLKID_MST_E_MCLK]		= &aud_mst_e_mclk.hw,
+		[AUD_CLKID_MST_F_MCLK]		= &aud_mst_f_mclk.hw,
+		[AUD_CLKID_SPDIFOUT_CLK_SEL]	= &aud_spdifout_clk_sel.hw,
+		[AUD_CLKID_SPDIFOUT_CLK_DIV]	= &aud_spdifout_clk_div.hw,
+		[AUD_CLKID_SPDIFOUT_CLK]	= &aud_spdifout_clk.hw,
+		[AUD_CLKID_SPDIFIN_CLK_SEL]	= &aud_spdifin_clk_sel.hw,
+		[AUD_CLKID_SPDIFIN_CLK_DIV]	= &aud_spdifin_clk_div.hw,
+		[AUD_CLKID_SPDIFIN_CLK]		= &aud_spdifin_clk.hw,
+		[AUD_CLKID_PDM_DCLK_SEL]	= &aud_pdm_dclk_sel.hw,
+		[AUD_CLKID_PDM_DCLK_DIV]	= &aud_pdm_dclk_div.hw,
+		[AUD_CLKID_PDM_DCLK]		= &aud_pdm_dclk.hw,
+		[AUD_CLKID_PDM_SYSCLK_SEL]	= &aud_pdm_sysclk_sel.hw,
+		[AUD_CLKID_PDM_SYSCLK_DIV]	= &aud_pdm_sysclk_div.hw,
+		[AUD_CLKID_PDM_SYSCLK]		= &aud_pdm_sysclk.hw,
+		[AUD_CLKID_MST_A_SCLK_PRE_EN]	= &aud_mst_a_sclk_pre_en.hw,
+		[AUD_CLKID_MST_B_SCLK_PRE_EN]	= &aud_mst_b_sclk_pre_en.hw,
+		[AUD_CLKID_MST_C_SCLK_PRE_EN]	= &aud_mst_c_sclk_pre_en.hw,
+		[AUD_CLKID_MST_D_SCLK_PRE_EN]	= &aud_mst_d_sclk_pre_en.hw,
+		[AUD_CLKID_MST_E_SCLK_PRE_EN]	= &aud_mst_e_sclk_pre_en.hw,
+		[AUD_CLKID_MST_F_SCLK_PRE_EN]	= &aud_mst_f_sclk_pre_en.hw,
+		[AUD_CLKID_MST_A_SCLK_DIV]	= &aud_mst_a_sclk_div.hw,
+		[AUD_CLKID_MST_B_SCLK_DIV]	= &aud_mst_b_sclk_div.hw,
+		[AUD_CLKID_MST_C_SCLK_DIV]	= &aud_mst_c_sclk_div.hw,
+		[AUD_CLKID_MST_D_SCLK_DIV]	= &aud_mst_d_sclk_div.hw,
+		[AUD_CLKID_MST_E_SCLK_DIV]	= &aud_mst_e_sclk_div.hw,
+		[AUD_CLKID_MST_F_SCLK_DIV]	= &aud_mst_f_sclk_div.hw,
+		[AUD_CLKID_MST_A_SCLK_POST_EN]	= &aud_mst_a_sclk_post_en.hw,
+		[AUD_CLKID_MST_B_SCLK_POST_EN]	= &aud_mst_b_sclk_post_en.hw,
+		[AUD_CLKID_MST_C_SCLK_POST_EN]	= &aud_mst_c_sclk_post_en.hw,
+		[AUD_CLKID_MST_D_SCLK_POST_EN]	= &aud_mst_d_sclk_post_en.hw,
+		[AUD_CLKID_MST_E_SCLK_POST_EN]	= &aud_mst_e_sclk_post_en.hw,
+		[AUD_CLKID_MST_F_SCLK_POST_EN]	= &aud_mst_f_sclk_post_en.hw,
+		[AUD_CLKID_MST_A_SCLK]		= &aud_mst_a_sclk.hw,
+		[AUD_CLKID_MST_B_SCLK]		= &aud_mst_b_sclk.hw,
+		[AUD_CLKID_MST_C_SCLK]		= &aud_mst_c_sclk.hw,
+		[AUD_CLKID_MST_D_SCLK]		= &aud_mst_d_sclk.hw,
+		[AUD_CLKID_MST_E_SCLK]		= &aud_mst_e_sclk.hw,
+		[AUD_CLKID_MST_F_SCLK]		= &aud_mst_f_sclk.hw,
+		[AUD_CLKID_MST_A_LRCLK_DIV]	= &aud_mst_a_lrclk_div.hw,
+		[AUD_CLKID_MST_B_LRCLK_DIV]	= &aud_mst_b_lrclk_div.hw,
+		[AUD_CLKID_MST_C_LRCLK_DIV]	= &aud_mst_c_lrclk_div.hw,
+		[AUD_CLKID_MST_D_LRCLK_DIV]	= &aud_mst_d_lrclk_div.hw,
+		[AUD_CLKID_MST_E_LRCLK_DIV]	= &aud_mst_e_lrclk_div.hw,
+		[AUD_CLKID_MST_F_LRCLK_DIV]	= &aud_mst_f_lrclk_div.hw,
+		[AUD_CLKID_MST_A_LRCLK]		= &aud_mst_a_lrclk.hw,
+		[AUD_CLKID_MST_B_LRCLK]		= &aud_mst_b_lrclk.hw,
+		[AUD_CLKID_MST_C_LRCLK]		= &aud_mst_c_lrclk.hw,
+		[AUD_CLKID_MST_D_LRCLK]		= &aud_mst_d_lrclk.hw,
+		[AUD_CLKID_MST_E_LRCLK]		= &aud_mst_e_lrclk.hw,
+		[AUD_CLKID_MST_F_LRCLK]		= &aud_mst_f_lrclk.hw,
+		[AUD_CLKID_TDMIN_A_SCLK_SEL]	= &aud_tdmin_a_sclk_sel.hw,
+		[AUD_CLKID_TDMIN_B_SCLK_SEL]	= &aud_tdmin_b_sclk_sel.hw,
+		[AUD_CLKID_TDMIN_C_SCLK_SEL]	= &aud_tdmin_c_sclk_sel.hw,
+		[AUD_CLKID_TDMIN_LB_SCLK_SEL]	= &aud_tdmin_lb_sclk_sel.hw,
+		[AUD_CLKID_TDMOUT_A_SCLK_SEL]	= &aud_tdmout_a_sclk_sel.hw,
+		[AUD_CLKID_TDMOUT_B_SCLK_SEL]	= &aud_tdmout_b_sclk_sel.hw,
+		[AUD_CLKID_TDMOUT_C_SCLK_SEL]	= &aud_tdmout_c_sclk_sel.hw,
+		[AUD_CLKID_TDMIN_A_SCLK_PRE_EN]	= &aud_tdmin_a_sclk_pre_en.hw,
+		[AUD_CLKID_TDMIN_B_SCLK_PRE_EN]	= &aud_tdmin_b_sclk_pre_en.hw,
+		[AUD_CLKID_TDMIN_C_SCLK_PRE_EN]	= &aud_tdmin_c_sclk_pre_en.hw,
+		[AUD_CLKID_TDMIN_LB_SCLK_PRE_EN] = &aud_tdmin_lb_sclk_pre_en.hw,
+		[AUD_CLKID_TDMOUT_A_SCLK_PRE_EN] = &aud_tdmout_a_sclk_pre_en.hw,
+		[AUD_CLKID_TDMOUT_B_SCLK_PRE_EN] = &aud_tdmout_b_sclk_pre_en.hw,
+		[AUD_CLKID_TDMOUT_C_SCLK_PRE_EN] = &aud_tdmout_c_sclk_pre_en.hw,
+		[AUD_CLKID_TDMIN_A_SCLK_POST_EN] = &aud_tdmin_a_sclk_post_en.hw,
+		[AUD_CLKID_TDMIN_B_SCLK_POST_EN] = &aud_tdmin_b_sclk_post_en.hw,
+		[AUD_CLKID_TDMIN_C_SCLK_POST_EN] = &aud_tdmin_c_sclk_post_en.hw,
+		[AUD_CLKID_TDMIN_LB_SCLK_POST_EN] = &aud_tdmin_lb_sclk_post_en.hw,
+		[AUD_CLKID_TDMOUT_A_SCLK_POST_EN] = &aud_tdmout_a_sclk_post_en.hw,
+		[AUD_CLKID_TDMOUT_B_SCLK_POST_EN] = &aud_tdmout_b_sclk_post_en.hw,
+		[AUD_CLKID_TDMOUT_C_SCLK_POST_EN] = &aud_tdmout_c_sclk_post_en.hw,
+		[AUD_CLKID_TDMIN_A_SCLK]	= &aud_tdmin_a_sclk.hw,
+		[AUD_CLKID_TDMIN_B_SCLK]	= &aud_tdmin_b_sclk.hw,
+		[AUD_CLKID_TDMIN_C_SCLK]	= &aud_tdmin_c_sclk.hw,
+		[AUD_CLKID_TDMIN_LB_SCLK]	= &aud_tdmin_lb_sclk.hw,
+		[AUD_CLKID_TDMOUT_A_SCLK]	= &aud_tdmout_a_sclk.hw,
+		[AUD_CLKID_TDMOUT_B_SCLK]	= &aud_tdmout_b_sclk.hw,
+		[AUD_CLKID_TDMOUT_C_SCLK]	= &aud_tdmout_c_sclk.hw,
+		[AUD_CLKID_TDMIN_A_LRCLK]	= &aud_tdmin_a_lrclk.hw,
+		[AUD_CLKID_TDMIN_B_LRCLK]	= &aud_tdmin_b_lrclk.hw,
+		[AUD_CLKID_TDMIN_C_LRCLK]	= &aud_tdmin_c_lrclk.hw,
+		[AUD_CLKID_TDMIN_LB_LRCLK]	= &aud_tdmin_lb_lrclk.hw,
+		[AUD_CLKID_TDMOUT_A_LRCLK]	= &aud_tdmout_a_lrclk.hw,
+		[AUD_CLKID_TDMOUT_B_LRCLK]	= &aud_tdmout_b_lrclk.hw,
+		[AUD_CLKID_TDMOUT_C_LRCLK]	= &aud_tdmout_c_lrclk.hw,
 		[NR_CLKS] = NULL,
 	},
 	.num = NR_CLKS,
 };
 
-/* Convenience table to populate regmap in .probe() */
-static struct clk_regmap *const axg_audio_clk_regmaps[] = {
-	&axg_ddr_arb,
-	&axg_pdm,
-	&axg_tdmin_a,
-	&axg_tdmin_b,
-	&axg_tdmin_c,
-	&axg_tdmin_lb,
-	&axg_tdmout_a,
-	&axg_tdmout_b,
-	&axg_tdmout_c,
-	&axg_frddr_a,
-	&axg_frddr_b,
-	&axg_frddr_c,
-	&axg_toddr_a,
-	&axg_toddr_b,
-	&axg_toddr_c,
-	&axg_loopback,
-	&axg_spdifin,
-	&axg_spdifout,
-	&axg_resample,
-	&axg_power_detect,
-	&axg_mst_a_mclk_sel,
-	&axg_mst_b_mclk_sel,
-	&axg_mst_c_mclk_sel,
-	&axg_mst_d_mclk_sel,
-	&axg_mst_e_mclk_sel,
-	&axg_mst_f_mclk_sel,
-	&axg_mst_a_mclk_div,
-	&axg_mst_b_mclk_div,
-	&axg_mst_c_mclk_div,
-	&axg_mst_d_mclk_div,
-	&axg_mst_e_mclk_div,
-	&axg_mst_f_mclk_div,
-	&axg_mst_a_mclk,
-	&axg_mst_b_mclk,
-	&axg_mst_c_mclk,
-	&axg_mst_d_mclk,
-	&axg_mst_e_mclk,
-	&axg_mst_f_mclk,
-	&axg_spdifout_clk_sel,
-	&axg_spdifout_clk_div,
-	&axg_spdifout_clk,
-	&axg_spdifin_clk_sel,
-	&axg_spdifin_clk_div,
-	&axg_spdifin_clk,
-	&axg_pdm_dclk_sel,
-	&axg_pdm_dclk_div,
-	&axg_pdm_dclk,
-	&axg_pdm_sysclk_sel,
-	&axg_pdm_sysclk_div,
-	&axg_pdm_sysclk,
-	&axg_mst_a_sclk_pre_en,
-	&axg_mst_b_sclk_pre_en,
-	&axg_mst_c_sclk_pre_en,
-	&axg_mst_d_sclk_pre_en,
-	&axg_mst_e_sclk_pre_en,
-	&axg_mst_f_sclk_pre_en,
-	&axg_mst_a_sclk_div,
-	&axg_mst_b_sclk_div,
-	&axg_mst_c_sclk_div,
-	&axg_mst_d_sclk_div,
-	&axg_mst_e_sclk_div,
-	&axg_mst_f_sclk_div,
-	&axg_mst_a_sclk_post_en,
-	&axg_mst_b_sclk_post_en,
-	&axg_mst_c_sclk_post_en,
-	&axg_mst_d_sclk_post_en,
-	&axg_mst_e_sclk_post_en,
-	&axg_mst_f_sclk_post_en,
-	&axg_mst_a_sclk,
-	&axg_mst_b_sclk,
-	&axg_mst_c_sclk,
-	&axg_mst_d_sclk,
-	&axg_mst_e_sclk,
-	&axg_mst_f_sclk,
-	&axg_mst_a_lrclk_div,
-	&axg_mst_b_lrclk_div,
-	&axg_mst_c_lrclk_div,
-	&axg_mst_d_lrclk_div,
-	&axg_mst_e_lrclk_div,
-	&axg_mst_f_lrclk_div,
-	&axg_mst_a_lrclk,
-	&axg_mst_b_lrclk,
-	&axg_mst_c_lrclk,
-	&axg_mst_d_lrclk,
-	&axg_mst_e_lrclk,
-	&axg_mst_f_lrclk,
-	&axg_tdmin_a_sclk_sel,
-	&axg_tdmin_b_sclk_sel,
-	&axg_tdmin_c_sclk_sel,
-	&axg_tdmin_lb_sclk_sel,
-	&axg_tdmout_a_sclk_sel,
-	&axg_tdmout_b_sclk_sel,
-	&axg_tdmout_c_sclk_sel,
-	&axg_tdmin_a_sclk_pre_en,
-	&axg_tdmin_b_sclk_pre_en,
-	&axg_tdmin_c_sclk_pre_en,
-	&axg_tdmin_lb_sclk_pre_en,
-	&axg_tdmout_a_sclk_pre_en,
-	&axg_tdmout_b_sclk_pre_en,
-	&axg_tdmout_c_sclk_pre_en,
-	&axg_tdmin_a_sclk_post_en,
-	&axg_tdmin_b_sclk_post_en,
-	&axg_tdmin_c_sclk_post_en,
-	&axg_tdmin_lb_sclk_post_en,
-	&axg_tdmout_a_sclk_post_en,
-	&axg_tdmout_b_sclk_post_en,
-	&axg_tdmout_c_sclk_post_en,
-	&axg_tdmin_a_sclk,
-	&axg_tdmin_b_sclk,
-	&axg_tdmin_c_sclk,
-	&axg_tdmin_lb_sclk,
-	&axg_tdmout_a_sclk,
-	&axg_tdmout_b_sclk,
-	&axg_tdmout_c_sclk,
-	&axg_tdmin_a_lrclk,
-	&axg_tdmin_b_lrclk,
-	&axg_tdmin_c_lrclk,
-	&axg_tdmin_lb_lrclk,
-	&axg_tdmout_a_lrclk,
-	&axg_tdmout_b_lrclk,
-	&axg_tdmout_c_lrclk,
+/*
+ * Array of all G12A clocks provided by this provider
+ * The input clocks of the controller will be populated at runtime
+ */
+static struct clk_hw_onecell_data g12a_audio_hw_onecell_data = {
+	.hws = {
+		[AUD_CLKID_DDR_ARB]		= &aud_ddr_arb.hw,
+		[AUD_CLKID_PDM]			= &aud_pdm.hw,
+		[AUD_CLKID_TDMIN_A]		= &aud_tdmin_a.hw,
+		[AUD_CLKID_TDMIN_B]		= &aud_tdmin_b.hw,
+		[AUD_CLKID_TDMIN_C]		= &aud_tdmin_c.hw,
+		[AUD_CLKID_TDMIN_LB]		= &aud_tdmin_lb.hw,
+		[AUD_CLKID_TDMOUT_A]		= &aud_tdmout_a.hw,
+		[AUD_CLKID_TDMOUT_B]		= &aud_tdmout_b.hw,
+		[AUD_CLKID_TDMOUT_C]		= &aud_tdmout_c.hw,
+		[AUD_CLKID_FRDDR_A]		= &aud_frddr_a.hw,
+		[AUD_CLKID_FRDDR_B]		= &aud_frddr_b.hw,
+		[AUD_CLKID_FRDDR_C]		= &aud_frddr_c.hw,
+		[AUD_CLKID_TODDR_A]		= &aud_toddr_a.hw,
+		[AUD_CLKID_TODDR_B]		= &aud_toddr_b.hw,
+		[AUD_CLKID_TODDR_C]		= &aud_toddr_c.hw,
+		[AUD_CLKID_LOOPBACK]		= &aud_loopback.hw,
+		[AUD_CLKID_SPDIFIN]		= &aud_spdifin.hw,
+		[AUD_CLKID_SPDIFOUT]		= &aud_spdifout.hw,
+		[AUD_CLKID_RESAMPLE]		= &aud_resample.hw,
+		[AUD_CLKID_POWER_DETECT]	= &aud_power_detect.hw,
+		[AUD_CLKID_SPDIFOUT_B]		= &aud_spdifout_b.hw,
+		[AUD_CLKID_MST_A_MCLK_SEL]	= &aud_mst_a_mclk_sel.hw,
+		[AUD_CLKID_MST_B_MCLK_SEL]	= &aud_mst_b_mclk_sel.hw,
+		[AUD_CLKID_MST_C_MCLK_SEL]	= &aud_mst_c_mclk_sel.hw,
+		[AUD_CLKID_MST_D_MCLK_SEL]	= &aud_mst_d_mclk_sel.hw,
+		[AUD_CLKID_MST_E_MCLK_SEL]	= &aud_mst_e_mclk_sel.hw,
+		[AUD_CLKID_MST_F_MCLK_SEL]	= &aud_mst_f_mclk_sel.hw,
+		[AUD_CLKID_MST_A_MCLK_DIV]	= &aud_mst_a_mclk_div.hw,
+		[AUD_CLKID_MST_B_MCLK_DIV]	= &aud_mst_b_mclk_div.hw,
+		[AUD_CLKID_MST_C_MCLK_DIV]	= &aud_mst_c_mclk_div.hw,
+		[AUD_CLKID_MST_D_MCLK_DIV]	= &aud_mst_d_mclk_div.hw,
+		[AUD_CLKID_MST_E_MCLK_DIV]	= &aud_mst_e_mclk_div.hw,
+		[AUD_CLKID_MST_F_MCLK_DIV]	= &aud_mst_f_mclk_div.hw,
+		[AUD_CLKID_MST_A_MCLK]		= &aud_mst_a_mclk.hw,
+		[AUD_CLKID_MST_B_MCLK]		= &aud_mst_b_mclk.hw,
+		[AUD_CLKID_MST_C_MCLK]		= &aud_mst_c_mclk.hw,
+		[AUD_CLKID_MST_D_MCLK]		= &aud_mst_d_mclk.hw,
+		[AUD_CLKID_MST_E_MCLK]		= &aud_mst_e_mclk.hw,
+		[AUD_CLKID_MST_F_MCLK]		= &aud_mst_f_mclk.hw,
+		[AUD_CLKID_SPDIFOUT_CLK_SEL]	= &aud_spdifout_clk_sel.hw,
+		[AUD_CLKID_SPDIFOUT_CLK_DIV]	= &aud_spdifout_clk_div.hw,
+		[AUD_CLKID_SPDIFOUT_CLK]	= &aud_spdifout_clk.hw,
+		[AUD_CLKID_SPDIFOUT_B_CLK_SEL]	= &aud_spdifout_b_clk_sel.hw,
+		[AUD_CLKID_SPDIFOUT_B_CLK_DIV]	= &aud_spdifout_b_clk_div.hw,
+		[AUD_CLKID_SPDIFOUT_B_CLK]	= &aud_spdifout_b_clk.hw,
+		[AUD_CLKID_SPDIFIN_CLK_SEL]	= &aud_spdifin_clk_sel.hw,
+		[AUD_CLKID_SPDIFIN_CLK_DIV]	= &aud_spdifin_clk_div.hw,
+		[AUD_CLKID_SPDIFIN_CLK]		= &aud_spdifin_clk.hw,
+		[AUD_CLKID_PDM_DCLK_SEL]	= &aud_pdm_dclk_sel.hw,
+		[AUD_CLKID_PDM_DCLK_DIV]	= &aud_pdm_dclk_div.hw,
+		[AUD_CLKID_PDM_DCLK]		= &aud_pdm_dclk.hw,
+		[AUD_CLKID_PDM_SYSCLK_SEL]	= &aud_pdm_sysclk_sel.hw,
+		[AUD_CLKID_PDM_SYSCLK_DIV]	= &aud_pdm_sysclk_div.hw,
+		[AUD_CLKID_PDM_SYSCLK]		= &aud_pdm_sysclk.hw,
+		[AUD_CLKID_MST_A_SCLK_PRE_EN]	= &aud_mst_a_sclk_pre_en.hw,
+		[AUD_CLKID_MST_B_SCLK_PRE_EN]	= &aud_mst_b_sclk_pre_en.hw,
+		[AUD_CLKID_MST_C_SCLK_PRE_EN]	= &aud_mst_c_sclk_pre_en.hw,
+		[AUD_CLKID_MST_D_SCLK_PRE_EN]	= &aud_mst_d_sclk_pre_en.hw,
+		[AUD_CLKID_MST_E_SCLK_PRE_EN]	= &aud_mst_e_sclk_pre_en.hw,
+		[AUD_CLKID_MST_F_SCLK_PRE_EN]	= &aud_mst_f_sclk_pre_en.hw,
+		[AUD_CLKID_MST_A_SCLK_DIV]	= &aud_mst_a_sclk_div.hw,
+		[AUD_CLKID_MST_B_SCLK_DIV]	= &aud_mst_b_sclk_div.hw,
+		[AUD_CLKID_MST_C_SCLK_DIV]	= &aud_mst_c_sclk_div.hw,
+		[AUD_CLKID_MST_D_SCLK_DIV]	= &aud_mst_d_sclk_div.hw,
+		[AUD_CLKID_MST_E_SCLK_DIV]	= &aud_mst_e_sclk_div.hw,
+		[AUD_CLKID_MST_F_SCLK_DIV]	= &aud_mst_f_sclk_div.hw,
+		[AUD_CLKID_MST_A_SCLK_POST_EN]	= &aud_mst_a_sclk_post_en.hw,
+		[AUD_CLKID_MST_B_SCLK_POST_EN]	= &aud_mst_b_sclk_post_en.hw,
+		[AUD_CLKID_MST_C_SCLK_POST_EN]	= &aud_mst_c_sclk_post_en.hw,
+		[AUD_CLKID_MST_D_SCLK_POST_EN]	= &aud_mst_d_sclk_post_en.hw,
+		[AUD_CLKID_MST_E_SCLK_POST_EN]	= &aud_mst_e_sclk_post_en.hw,
+		[AUD_CLKID_MST_F_SCLK_POST_EN]	= &aud_mst_f_sclk_post_en.hw,
+		[AUD_CLKID_MST_A_SCLK]		= &aud_mst_a_sclk.hw,
+		[AUD_CLKID_MST_B_SCLK]		= &aud_mst_b_sclk.hw,
+		[AUD_CLKID_MST_C_SCLK]		= &aud_mst_c_sclk.hw,
+		[AUD_CLKID_MST_D_SCLK]		= &aud_mst_d_sclk.hw,
+		[AUD_CLKID_MST_E_SCLK]		= &aud_mst_e_sclk.hw,
+		[AUD_CLKID_MST_F_SCLK]		= &aud_mst_f_sclk.hw,
+		[AUD_CLKID_MST_A_LRCLK_DIV]	= &aud_mst_a_lrclk_div.hw,
+		[AUD_CLKID_MST_B_LRCLK_DIV]	= &aud_mst_b_lrclk_div.hw,
+		[AUD_CLKID_MST_C_LRCLK_DIV]	= &aud_mst_c_lrclk_div.hw,
+		[AUD_CLKID_MST_D_LRCLK_DIV]	= &aud_mst_d_lrclk_div.hw,
+		[AUD_CLKID_MST_E_LRCLK_DIV]	= &aud_mst_e_lrclk_div.hw,
+		[AUD_CLKID_MST_F_LRCLK_DIV]	= &aud_mst_f_lrclk_div.hw,
+		[AUD_CLKID_MST_A_LRCLK]		= &aud_mst_a_lrclk.hw,
+		[AUD_CLKID_MST_B_LRCLK]		= &aud_mst_b_lrclk.hw,
+		[AUD_CLKID_MST_C_LRCLK]		= &aud_mst_c_lrclk.hw,
+		[AUD_CLKID_MST_D_LRCLK]		= &aud_mst_d_lrclk.hw,
+		[AUD_CLKID_MST_E_LRCLK]		= &aud_mst_e_lrclk.hw,
+		[AUD_CLKID_MST_F_LRCLK]		= &aud_mst_f_lrclk.hw,
+		[AUD_CLKID_TDMIN_A_SCLK_SEL]	= &aud_tdmin_a_sclk_sel.hw,
+		[AUD_CLKID_TDMIN_B_SCLK_SEL]	= &aud_tdmin_b_sclk_sel.hw,
+		[AUD_CLKID_TDMIN_C_SCLK_SEL]	= &aud_tdmin_c_sclk_sel.hw,
+		[AUD_CLKID_TDMIN_LB_SCLK_SEL]	= &aud_tdmin_lb_sclk_sel.hw,
+		[AUD_CLKID_TDMOUT_A_SCLK_SEL]	= &aud_tdmout_a_sclk_sel.hw,
+		[AUD_CLKID_TDMOUT_B_SCLK_SEL]	= &aud_tdmout_b_sclk_sel.hw,
+		[AUD_CLKID_TDMOUT_C_SCLK_SEL]	= &aud_tdmout_c_sclk_sel.hw,
+		[AUD_CLKID_TDMIN_A_SCLK_PRE_EN]	= &aud_tdmin_a_sclk_pre_en.hw,
+		[AUD_CLKID_TDMIN_B_SCLK_PRE_EN]	= &aud_tdmin_b_sclk_pre_en.hw,
+		[AUD_CLKID_TDMIN_C_SCLK_PRE_EN]	= &aud_tdmin_c_sclk_pre_en.hw,
+		[AUD_CLKID_TDMIN_LB_SCLK_PRE_EN] = &aud_tdmin_lb_sclk_pre_en.hw,
+		[AUD_CLKID_TDMOUT_A_SCLK_PRE_EN] = &aud_tdmout_a_sclk_pre_en.hw,
+		[AUD_CLKID_TDMOUT_B_SCLK_PRE_EN] = &aud_tdmout_b_sclk_pre_en.hw,
+		[AUD_CLKID_TDMOUT_C_SCLK_PRE_EN] = &aud_tdmout_c_sclk_pre_en.hw,
+		[AUD_CLKID_TDMIN_A_SCLK_POST_EN] = &aud_tdmin_a_sclk_post_en.hw,
+		[AUD_CLKID_TDMIN_B_SCLK_POST_EN] = &aud_tdmin_b_sclk_post_en.hw,
+		[AUD_CLKID_TDMIN_C_SCLK_POST_EN] = &aud_tdmin_c_sclk_post_en.hw,
+		[AUD_CLKID_TDMIN_LB_SCLK_POST_EN] = &aud_tdmin_lb_sclk_post_en.hw,
+		[AUD_CLKID_TDMOUT_A_SCLK_POST_EN] = &aud_tdmout_a_sclk_post_en.hw,
+		[AUD_CLKID_TDMOUT_B_SCLK_POST_EN] = &aud_tdmout_b_sclk_post_en.hw,
+		[AUD_CLKID_TDMOUT_C_SCLK_POST_EN] = &aud_tdmout_c_sclk_post_en.hw,
+		[AUD_CLKID_TDMIN_A_SCLK]	= &aud_tdmin_a_sclk.hw,
+		[AUD_CLKID_TDMIN_B_SCLK]	= &aud_tdmin_b_sclk.hw,
+		[AUD_CLKID_TDMIN_C_SCLK]	= &aud_tdmin_c_sclk.hw,
+		[AUD_CLKID_TDMIN_LB_SCLK]	= &aud_tdmin_lb_sclk.hw,
+		[AUD_CLKID_TDMOUT_A_SCLK]	= &aud_tdmout_a_sclk.hw,
+		[AUD_CLKID_TDMOUT_B_SCLK]	= &aud_tdmout_b_sclk.hw,
+		[AUD_CLKID_TDMOUT_C_SCLK]	= &aud_tdmout_c_sclk.hw,
+		[AUD_CLKID_TDMIN_A_LRCLK]	= &aud_tdmin_a_lrclk.hw,
+		[AUD_CLKID_TDMIN_B_LRCLK]	= &aud_tdmin_b_lrclk.hw,
+		[AUD_CLKID_TDMIN_C_LRCLK]	= &aud_tdmin_c_lrclk.hw,
+		[AUD_CLKID_TDMIN_LB_LRCLK]	= &aud_tdmin_lb_lrclk.hw,
+		[AUD_CLKID_TDMOUT_A_LRCLK]	= &aud_tdmout_a_lrclk.hw,
+		[AUD_CLKID_TDMOUT_B_LRCLK]	= &aud_tdmout_b_lrclk.hw,
+		[AUD_CLKID_TDMOUT_C_LRCLK]	= &aud_tdmout_c_lrclk.hw,
+		[AUD_CLKID_TDM_MCLK_PAD0]	= &aud_tdm_mclk_pad_0.hw,
+		[AUD_CLKID_TDM_MCLK_PAD1]	= &aud_tdm_mclk_pad_1.hw,
+		[AUD_CLKID_TDM_LRCLK_PAD0]	= &aud_tdm_lrclk_pad_0.hw,
+		[AUD_CLKID_TDM_LRCLK_PAD1]	= &aud_tdm_lrclk_pad_1.hw,
+		[AUD_CLKID_TDM_LRCLK_PAD2]	= &aud_tdm_lrclk_pad_2.hw,
+		[AUD_CLKID_TDM_SCLK_PAD0]	= &aud_tdm_sclk_pad_0.hw,
+		[AUD_CLKID_TDM_SCLK_PAD1]	= &aud_tdm_sclk_pad_1.hw,
+		[AUD_CLKID_TDM_SCLK_PAD2]	= &aud_tdm_sclk_pad_2.hw,
+		[NR_CLKS] = NULL,
+	},
+	.num = NR_CLKS,
+};
+
+/* Convenience table to populate regmap in .probe()
+ * Note that this table is shared between both AXG and G12A,
+ * with spdifout_b clocks being exclusive to G12A. Since those
+ * clocks are not declared within the AXG onecell table, we do not
+ * feel the need to have separate AXG/G12A regmap tables.
+ */
+static struct clk_regmap *const aud_clk_regmaps[] = {
+	&aud_ddr_arb,
+	&aud_pdm,
+	&aud_tdmin_a,
+	&aud_tdmin_b,
+	&aud_tdmin_c,
+	&aud_tdmin_lb,
+	&aud_tdmout_a,
+	&aud_tdmout_b,
+	&aud_tdmout_c,
+	&aud_frddr_a,
+	&aud_frddr_b,
+	&aud_frddr_c,
+	&aud_toddr_a,
+	&aud_toddr_b,
+	&aud_toddr_c,
+	&aud_loopback,
+	&aud_spdifin,
+	&aud_spdifout,
+	&aud_resample,
+	&aud_power_detect,
+	&aud_spdifout_b,
+	&aud_mst_a_mclk_sel,
+	&aud_mst_b_mclk_sel,
+	&aud_mst_c_mclk_sel,
+	&aud_mst_d_mclk_sel,
+	&aud_mst_e_mclk_sel,
+	&aud_mst_f_mclk_sel,
+	&aud_mst_a_mclk_div,
+	&aud_mst_b_mclk_div,
+	&aud_mst_c_mclk_div,
+	&aud_mst_d_mclk_div,
+	&aud_mst_e_mclk_div,
+	&aud_mst_f_mclk_div,
+	&aud_mst_a_mclk,
+	&aud_mst_b_mclk,
+	&aud_mst_c_mclk,
+	&aud_mst_d_mclk,
+	&aud_mst_e_mclk,
+	&aud_mst_f_mclk,
+	&aud_spdifout_clk_sel,
+	&aud_spdifout_clk_div,
+	&aud_spdifout_clk,
+	&aud_spdifin_clk_sel,
+	&aud_spdifin_clk_div,
+	&aud_spdifin_clk,
+	&aud_pdm_dclk_sel,
+	&aud_pdm_dclk_div,
+	&aud_pdm_dclk,
+	&aud_pdm_sysclk_sel,
+	&aud_pdm_sysclk_div,
+	&aud_pdm_sysclk,
+	&aud_mst_a_sclk_pre_en,
+	&aud_mst_b_sclk_pre_en,
+	&aud_mst_c_sclk_pre_en,
+	&aud_mst_d_sclk_pre_en,
+	&aud_mst_e_sclk_pre_en,
+	&aud_mst_f_sclk_pre_en,
+	&aud_mst_a_sclk_div,
+	&aud_mst_b_sclk_div,
+	&aud_mst_c_sclk_div,
+	&aud_mst_d_sclk_div,
+	&aud_mst_e_sclk_div,
+	&aud_mst_f_sclk_div,
+	&aud_mst_a_sclk_post_en,
+	&aud_mst_b_sclk_post_en,
+	&aud_mst_c_sclk_post_en,
+	&aud_mst_d_sclk_post_en,
+	&aud_mst_e_sclk_post_en,
+	&aud_mst_f_sclk_post_en,
+	&aud_mst_a_sclk,
+	&aud_mst_b_sclk,
+	&aud_mst_c_sclk,
+	&aud_mst_d_sclk,
+	&aud_mst_e_sclk,
+	&aud_mst_f_sclk,
+	&aud_mst_a_lrclk_div,
+	&aud_mst_b_lrclk_div,
+	&aud_mst_c_lrclk_div,
+	&aud_mst_d_lrclk_div,
+	&aud_mst_e_lrclk_div,
+	&aud_mst_f_lrclk_div,
+	&aud_mst_a_lrclk,
+	&aud_mst_b_lrclk,
+	&aud_mst_c_lrclk,
+	&aud_mst_d_lrclk,
+	&aud_mst_e_lrclk,
+	&aud_mst_f_lrclk,
+	&aud_tdmin_a_sclk_sel,
+	&aud_tdmin_b_sclk_sel,
+	&aud_tdmin_c_sclk_sel,
+	&aud_tdmin_lb_sclk_sel,
+	&aud_tdmout_a_sclk_sel,
+	&aud_tdmout_b_sclk_sel,
+	&aud_tdmout_c_sclk_sel,
+	&aud_tdmin_a_sclk_pre_en,
+	&aud_tdmin_b_sclk_pre_en,
+	&aud_tdmin_c_sclk_pre_en,
+	&aud_tdmin_lb_sclk_pre_en,
+	&aud_tdmout_a_sclk_pre_en,
+	&aud_tdmout_b_sclk_pre_en,
+	&aud_tdmout_c_sclk_pre_en,
+	&aud_tdmin_a_sclk_post_en,
+	&aud_tdmin_b_sclk_post_en,
+	&aud_tdmin_c_sclk_post_en,
+	&aud_tdmin_lb_sclk_post_en,
+	&aud_tdmout_a_sclk_post_en,
+	&aud_tdmout_b_sclk_post_en,
+	&aud_tdmout_c_sclk_post_en,
+	&aud_tdmin_a_sclk,
+	&aud_tdmin_b_sclk,
+	&aud_tdmin_c_sclk,
+	&aud_tdmin_lb_sclk,
+	&aud_tdmout_a_sclk,
+	&aud_tdmout_b_sclk,
+	&aud_tdmout_c_sclk,
+	&aud_tdmin_a_lrclk,
+	&aud_tdmin_b_lrclk,
+	&aud_tdmin_c_lrclk,
+	&aud_tdmin_lb_lrclk,
+	&aud_tdmout_a_lrclk,
+	&aud_tdmout_b_lrclk,
+	&aud_tdmout_c_lrclk,
+	&aud_spdifout_b_clk_sel,
+	&aud_spdifout_b_clk_div,
+	&aud_spdifout_b_clk,
+	&aud_tdm_mclk_pad_0,
+	&aud_tdm_mclk_pad_1,
+	&aud_tdm_lrclk_pad_0,
+	&aud_tdm_lrclk_pad_1,
+	&aud_tdm_lrclk_pad_2,
+	&aud_tdm_sclk_pad_0,
+	&aud_tdm_sclk_pad_1,
+	&aud_tdm_sclk_pad_2,
 };
 
 static int devm_clk_get_enable(struct device *dev, char *id)
@@ -665,14 +869,13 @@ static int devm_clk_get_enable(struct device *dev, char *id)
 }
 
 static int axg_register_clk_hw_input(struct device *dev,
-				     const char *name,
-				     unsigned int clkid)
+				     const char *name)
 {
 	char *clk_name;
 	struct clk_hw *hw;
 	int err = 0;
 
-	clk_name = kasprintf(GFP_KERNEL, "axg_%s", name);
+	clk_name = kasprintf(GFP_KERNEL, "aud_%s", name);
 	if (!clk_name)
 		return -ENOMEM;
 
@@ -686,8 +889,6 @@ static int axg_register_clk_hw_input(struct device *dev,
 			if (err != -EPROBE_DEFER)
 				dev_err(dev, "failed to get %s clock", name);
 		}
-	} else {
-		axg_audio_hw_onecell_data.hws[clkid] = hw;
 	}
 
 	kfree(clk_name);
@@ -696,8 +897,7 @@ static int axg_register_clk_hw_input(struct device *dev,
 
 static int axg_register_clk_hw_inputs(struct device *dev,
 				      const char *basename,
-				      unsigned int count,
-				      unsigned int clkid)
+				      unsigned int count)
 {
 	char *name;
 	int i, ret;
@@ -707,7 +907,7 @@ static int axg_register_clk_hw_inputs(struct device *dev,
 		if (!name)
 			return -ENOMEM;
 
-		ret = axg_register_clk_hw_input(dev, name, clkid + i);
+		ret = axg_register_clk_hw_input(dev, name);
 		kfree(name);
 		if (ret)
 			return ret;
@@ -723,15 +923,24 @@ static const struct regmap_config axg_audio_regmap_cfg = {
 	.max_register	= AUDIO_CLK_PDMIN_CTRL1,
 };
 
+struct audioclk_data {
+	struct clk_hw_onecell_data *hw_onecell_data;
+};
+
 static int axg_audio_clkc_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
+	const struct audioclk_data *data;
 	struct regmap *map;
 	struct resource *res;
 	void __iomem *regs;
 	struct clk_hw *hw;
 	int ret, i;
 
+	data = of_device_get_match_data(dev);
+	if (!data)
+		return -EINVAL;
+
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	regs = devm_ioremap_resource(dev, res);
 	if (IS_ERR(regs))
@@ -755,40 +964,35 @@ static int axg_audio_clkc_probe(struct platform_device *pdev)
 	}
 
 	/* Register the peripheral input clock */
-	hw = meson_clk_hw_register_input(dev, "pclk", "axg_audio_pclk", 0);
+	hw = meson_clk_hw_register_input(dev, "pclk", "audio_pclk", 0);
 	if (IS_ERR(hw))
 		return PTR_ERR(hw);
 
-	axg_audio_hw_onecell_data.hws[AUD_CLKID_PCLK] = hw;
-
 	/* Register optional input master clocks */
 	ret = axg_register_clk_hw_inputs(dev, "mst_in",
-					 AXG_MST_IN_COUNT,
-					 AUD_CLKID_MST0);
+					 AUD_MST_IN_COUNT);
 	if (ret)
 		return ret;
 
 	/* Register optional input slave sclks */
 	ret = axg_register_clk_hw_inputs(dev, "slv_sclk",
-					 AXG_SLV_SCLK_COUNT,
-					 AUD_CLKID_SLV_SCLK0);
+					 AUD_SLV_SCLK_COUNT);
 	if (ret)
 		return ret;
 
 	/* Register optional input slave lrclks */
 	ret = axg_register_clk_hw_inputs(dev, "slv_lrclk",
-					 AXG_SLV_LRCLK_COUNT,
-					 AUD_CLKID_SLV_LRCLK0);
+					 AUD_SLV_LRCLK_COUNT);
 	if (ret)
 		return ret;
 
 	/* Populate regmap for the regmap backed clocks */
-	for (i = 0; i < ARRAY_SIZE(axg_audio_clk_regmaps); i++)
-		axg_audio_clk_regmaps[i]->map = map;
+	for (i = 0; i < ARRAY_SIZE(aud_clk_regmaps); i++)
+		aud_clk_regmaps[i]->map = map;
 
 	/* Take care to skip the registered input clocks */
-	for (i = AUD_CLKID_DDR_ARB; i < axg_audio_hw_onecell_data.num; i++) {
-		hw = axg_audio_hw_onecell_data.hws[i];
+	for (i = AUD_CLKID_DDR_ARB; i < data->hw_onecell_data->num; i++) {
+		hw = data->hw_onecell_data->hws[i];
 		/* array might be sparse */
 		if (!hw)
 			continue;
@@ -802,12 +1006,25 @@ static int axg_audio_clkc_probe(struct platform_device *pdev)
 	}
 
 	return devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get,
-					   &axg_audio_hw_onecell_data);
+					   data->hw_onecell_data);
 }
 
+static const struct audioclk_data axg_audioclk_data = {
+	.hw_onecell_data = &axg_audio_hw_onecell_data,
+};
+
+static const struct audioclk_data g12a_audioclk_data = {
+	.hw_onecell_data = &g12a_audio_hw_onecell_data,
+};
+
 static const struct of_device_id clkc_match_table[] = {
-	{ .compatible = "amlogic,axg-audio-clkc" },
-	{}
+	{
+		.compatible = "amlogic,axg-audio-clkc",
+		.data = &axg_audioclk_data
+	}, {
+		.compatible = "amlogic,g12a-audio-clkc",
+		.data = &g12a_audioclk_data
+	}, {}
 };
 MODULE_DEVICE_TABLE(of, clkc_match_table);
 
@@ -820,6 +1037,6 @@ static struct platform_driver axg_audio_driver = {
 };
 module_platform_driver(axg_audio_driver);
 
-MODULE_DESCRIPTION("Amlogic A113x Audio Clock driver");
+MODULE_DESCRIPTION("Amlogic AXG/G12A Audio Clock driver");
 MODULE_AUTHOR("Jerome Brunet <jbrunet@baylibre.com>");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/clk/meson/axg-audio.h b/drivers/clk/meson/axg-audio.h
index 7191b39c9d65..5d972d55d6c7 100644
--- a/drivers/clk/meson/axg-audio.h
+++ b/drivers/clk/meson/axg-audio.h
@@ -20,6 +20,8 @@
 #define AUDIO_MCLK_D_CTRL	0x010
 #define AUDIO_MCLK_E_CTRL	0x014
 #define AUDIO_MCLK_F_CTRL	0x018
+#define AUDIO_MST_PAD_CTRL0	0x01c
+#define AUDIO_MST_PAD_CTRL1	0x020
 #define AUDIO_MST_A_SCLK_CTRL0	0x040
 #define AUDIO_MST_A_SCLK_CTRL1	0x044
 #define AUDIO_MST_B_SCLK_CTRL0	0x048
@@ -45,21 +47,13 @@
 #define AUDIO_CLK_LOCKER_CTRL	0x0A8
 #define AUDIO_CLK_PDMIN_CTRL0	0x0AC
 #define AUDIO_CLK_PDMIN_CTRL1	0x0B0
+#define AUDIO_CLK_SPDIFOUT_B_CTRL 0x0B4
 
 /*
  * CLKID index values
  * These indices are entirely contrived and do not map onto the hardware.
  */
 
-#define AUD_CLKID_PCLK			0
-#define AUD_CLKID_MST0			1
-#define AUD_CLKID_MST1			2
-#define AUD_CLKID_MST2			3
-#define AUD_CLKID_MST3			4
-#define AUD_CLKID_MST4			5
-#define AUD_CLKID_MST5			6
-#define AUD_CLKID_MST6			7
-#define AUD_CLKID_MST7			8
 #define AUD_CLKID_MST_A_MCLK_SEL	59
 #define AUD_CLKID_MST_B_MCLK_SEL	60
 #define AUD_CLKID_MST_C_MCLK_SEL	61
@@ -118,10 +112,12 @@
 #define AUD_CLKID_TDMOUT_A_SCLK_POST_EN	148
 #define AUD_CLKID_TDMOUT_B_SCLK_POST_EN	149
 #define AUD_CLKID_TDMOUT_C_SCLK_POST_EN	150
+#define AUD_CLKID_SPDIFOUT_B_CLK_SEL	153
+#define AUD_CLKID_SPDIFOUT_B_CLK_DIV	154
 
 /* include the CLKIDs which are part of the DT bindings */
 #include <dt-bindings/clock/axg-audio-clkc.h>
 
-#define NR_CLKS	151
+#define NR_CLKS	163
 
 #endif /*__AXG_AUDIO_CLKC_H */
diff --git a/drivers/clk/meson/clk-pll.c b/drivers/clk/meson/clk-pll.c
index 7a14ac9b2fec..ddb1e5634739 100644
--- a/drivers/clk/meson/clk-pll.c
+++ b/drivers/clk/meson/clk-pll.c
@@ -303,6 +303,16 @@ static int meson_clk_pll_is_enabled(struct clk_hw *hw)
 	return 1;
 }
 
+static int meson_clk_pcie_pll_enable(struct clk_hw *hw)
+{
+	meson_clk_pll_init(hw);
+
+	if (meson_clk_pll_wait_lock(hw))
+		return -EIO;
+
+	return 0;
+}
+
 static int meson_clk_pll_enable(struct clk_hw *hw)
 {
 	struct clk_regmap *clk = to_clk_regmap(hw);
@@ -387,6 +397,22 @@ static int meson_clk_pll_set_rate(struct clk_hw *hw, unsigned long rate,
 	return 0;
 }
 
+/*
+ * The Meson G12A PCIE PLL is fined tuned to deliver a very precise
+ * 100MHz reference clock for the PCIe Analog PHY, and thus requires
+ * a strict register sequence to enable the PLL.
+ * To simplify, re-use the _init() op to enable the PLL and keep
+ * the other ops except set_rate since the rate is fixed.
+ */
+const struct clk_ops meson_clk_pcie_pll_ops = {
+	.recalc_rate	= meson_clk_pll_recalc_rate,
+	.round_rate	= meson_clk_pll_round_rate,
+	.is_enabled	= meson_clk_pll_is_enabled,
+	.enable		= meson_clk_pcie_pll_enable,
+	.disable	= meson_clk_pll_disable
+};
+EXPORT_SYMBOL_GPL(meson_clk_pcie_pll_ops);
+
 const struct clk_ops meson_clk_pll_ops = {
 	.init		= meson_clk_pll_init,
 	.recalc_rate	= meson_clk_pll_recalc_rate,
diff --git a/drivers/clk/meson/clk-pll.h b/drivers/clk/meson/clk-pll.h
index 55af2e285b1b..367efd0f6410 100644
--- a/drivers/clk/meson/clk-pll.h
+++ b/drivers/clk/meson/clk-pll.h
@@ -45,5 +45,6 @@ struct meson_clk_pll_data {
 
 extern const struct clk_ops meson_clk_pll_ro_ops;
 extern const struct clk_ops meson_clk_pll_ops;
+extern const struct clk_ops meson_clk_pcie_pll_ops;
 
 #endif /* __MESON_CLK_PLL_H */
diff --git a/drivers/clk/meson/g12a-aoclk.h b/drivers/clk/meson/g12a-aoclk.h
index 04b0d5506641..a67c8a7cd7c4 100644
--- a/drivers/clk/meson/g12a-aoclk.h
+++ b/drivers/clk/meson/g12a-aoclk.h
@@ -16,9 +16,7 @@
  * to expose, such as the internal muxes and dividers of composite clocks,
  * will remain defined here.
  */
-#define CLKID_AO_SAR_ADC_SEL	16
 #define CLKID_AO_SAR_ADC_DIV	17
-#define CLKID_AO_CTS_OSCIN	19
 #define CLKID_AO_32K_PRE	20
 #define CLKID_AO_32K_DIV	21
 #define CLKID_AO_32K_SEL	22
diff --git a/drivers/clk/meson/g12a.c b/drivers/clk/meson/g12a.c
index f7b11e1eeebe..739f64fdf1e3 100644
--- a/drivers/clk/meson/g12a.c
+++ b/drivers/clk/meson/g12a.c
@@ -150,6 +150,318 @@ static struct clk_regmap g12a_sys_pll = {
 	},
 };
 
+static struct clk_regmap g12a_sys_pll_div16_en = {
+	.data = &(struct clk_regmap_gate_data){
+		.offset = HHI_SYS_CPU_CLK_CNTL1,
+		.bit_idx = 24,
+	},
+	.hw.init = &(struct clk_init_data) {
+		.name = "sys_pll_div16_en",
+		.ops = &clk_regmap_gate_ro_ops,
+		.parent_names = (const char *[]){ "sys_pll" },
+		.num_parents = 1,
+		/*
+		 * This clock is used to debug the sys_pll range
+		 * Linux should not change it at runtime
+		 */
+	},
+};
+
+static struct clk_fixed_factor g12a_sys_pll_div16 = {
+	.mult = 1,
+	.div = 16,
+	.hw.init = &(struct clk_init_data){
+		.name = "sys_pll_div16",
+		.ops = &clk_fixed_factor_ops,
+		.parent_names = (const char *[]){ "sys_pll_div16_en" },
+		.num_parents = 1,
+	},
+};
+
+/* Datasheet names this field as "premux0" */
+static struct clk_regmap g12a_cpu_clk_premux0 = {
+	.data = &(struct clk_regmap_mux_data){
+		.offset = HHI_SYS_CPU_CLK_CNTL0,
+		.mask = 0x3,
+		.shift = 0,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "cpu_clk_dyn0_sel",
+		.ops = &clk_regmap_mux_ro_ops,
+		.parent_names = (const char *[]){ IN_PREFIX "xtal",
+						  "fclk_div2",
+						  "fclk_div3" },
+		.num_parents = 3,
+	},
+};
+
+/* Datasheet names this field as "mux0_divn_tcnt" */
+static struct clk_regmap g12a_cpu_clk_mux0_div = {
+	.data = &(struct clk_regmap_div_data){
+		.offset = HHI_SYS_CPU_CLK_CNTL0,
+		.shift = 4,
+		.width = 6,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "cpu_clk_dyn0_div",
+		.ops = &clk_regmap_divider_ro_ops,
+		.parent_names = (const char *[]){ "cpu_clk_dyn0_sel" },
+		.num_parents = 1,
+	},
+};
+
+/* Datasheet names this field as "postmux0" */
+static struct clk_regmap g12a_cpu_clk_postmux0 = {
+	.data = &(struct clk_regmap_mux_data){
+		.offset = HHI_SYS_CPU_CLK_CNTL0,
+		.mask = 0x1,
+		.shift = 2,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "cpu_clk_dyn0",
+		.ops = &clk_regmap_mux_ro_ops,
+		.parent_names = (const char *[]){ "cpu_clk_dyn0_sel",
+						  "cpu_clk_dyn0_div" },
+		.num_parents = 2,
+	},
+};
+
+/* Datasheet names this field as "premux1" */
+static struct clk_regmap g12a_cpu_clk_premux1 = {
+	.data = &(struct clk_regmap_mux_data){
+		.offset = HHI_SYS_CPU_CLK_CNTL0,
+		.mask = 0x3,
+		.shift = 16,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "cpu_clk_dyn1_sel",
+		.ops = &clk_regmap_mux_ro_ops,
+		.parent_names = (const char *[]){ IN_PREFIX "xtal",
+						  "fclk_div2",
+						  "fclk_div3" },
+		.num_parents = 3,
+	},
+};
+
+/* Datasheet names this field as "Mux1_divn_tcnt" */
+static struct clk_regmap g12a_cpu_clk_mux1_div = {
+	.data = &(struct clk_regmap_div_data){
+		.offset = HHI_SYS_CPU_CLK_CNTL0,
+		.shift = 20,
+		.width = 6,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "cpu_clk_dyn1_div",
+		.ops = &clk_regmap_divider_ro_ops,
+		.parent_names = (const char *[]){ "cpu_clk_dyn1_sel" },
+		.num_parents = 1,
+	},
+};
+
+/* Datasheet names this field as "postmux1" */
+static struct clk_regmap g12a_cpu_clk_postmux1 = {
+	.data = &(struct clk_regmap_mux_data){
+		.offset = HHI_SYS_CPU_CLK_CNTL0,
+		.mask = 0x1,
+		.shift = 18,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "cpu_clk_dyn1",
+		.ops = &clk_regmap_mux_ro_ops,
+		.parent_names = (const char *[]){ "cpu_clk_dyn1_sel",
+						  "cpu_clk_dyn1_div" },
+		.num_parents = 2,
+	},
+};
+
+/* Datasheet names this field as "Final_dyn_mux_sel" */
+static struct clk_regmap g12a_cpu_clk_dyn = {
+	.data = &(struct clk_regmap_mux_data){
+		.offset = HHI_SYS_CPU_CLK_CNTL0,
+		.mask = 0x1,
+		.shift = 10,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "cpu_clk_dyn",
+		.ops = &clk_regmap_mux_ro_ops,
+		.parent_names = (const char *[]){ "cpu_clk_dyn0",
+						  "cpu_clk_dyn1" },
+		.num_parents = 2,
+	},
+};
+
+/* Datasheet names this field as "Final_mux_sel" */
+static struct clk_regmap g12a_cpu_clk = {
+	.data = &(struct clk_regmap_mux_data){
+		.offset = HHI_SYS_CPU_CLK_CNTL0,
+		.mask = 0x1,
+		.shift = 11,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "cpu_clk",
+		.ops = &clk_regmap_mux_ro_ops,
+		.parent_names = (const char *[]){ "cpu_clk_dyn",
+						  "sys_pll" },
+		.num_parents = 2,
+	},
+};
+
+static struct clk_regmap g12a_cpu_clk_div16_en = {
+	.data = &(struct clk_regmap_gate_data){
+		.offset = HHI_SYS_CPU_CLK_CNTL1,
+		.bit_idx = 1,
+	},
+	.hw.init = &(struct clk_init_data) {
+		.name = "cpu_clk_div16_en",
+		.ops = &clk_regmap_gate_ro_ops,
+		.parent_names = (const char *[]){ "cpu_clk" },
+		.num_parents = 1,
+		/*
+		 * This clock is used to debug the cpu_clk range
+		 * Linux should not change it at runtime
+		 */
+	},
+};
+
+static struct clk_fixed_factor g12a_cpu_clk_div16 = {
+	.mult = 1,
+	.div = 16,
+	.hw.init = &(struct clk_init_data){
+		.name = "cpu_clk_div16",
+		.ops = &clk_fixed_factor_ops,
+		.parent_names = (const char *[]){ "cpu_clk_div16_en" },
+		.num_parents = 1,
+	},
+};
+
+static struct clk_regmap g12a_cpu_clk_apb_div = {
+	.data = &(struct clk_regmap_div_data){
+		.offset = HHI_SYS_CPU_CLK_CNTL1,
+		.shift = 3,
+		.width = 3,
+		.flags = CLK_DIVIDER_POWER_OF_TWO,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "cpu_clk_apb_div",
+		.ops = &clk_regmap_divider_ro_ops,
+		.parent_names = (const char *[]){ "cpu_clk" },
+		.num_parents = 1,
+	},
+};
+
+static struct clk_regmap g12a_cpu_clk_apb = {
+	.data = &(struct clk_regmap_gate_data){
+		.offset = HHI_SYS_CPU_CLK_CNTL1,
+		.bit_idx = 1,
+	},
+	.hw.init = &(struct clk_init_data) {
+		.name = "cpu_clk_apb",
+		.ops = &clk_regmap_gate_ro_ops,
+		.parent_names = (const char *[]){ "cpu_clk_apb_div" },
+		.num_parents = 1,
+		/*
+		 * This clock is set by the ROM monitor code,
+		 * Linux should not change it at runtime
+		 */
+	},
+};
+
+static struct clk_regmap g12a_cpu_clk_atb_div = {
+	.data = &(struct clk_regmap_div_data){
+		.offset = HHI_SYS_CPU_CLK_CNTL1,
+		.shift = 6,
+		.width = 3,
+		.flags = CLK_DIVIDER_POWER_OF_TWO,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "cpu_clk_atb_div",
+		.ops = &clk_regmap_divider_ro_ops,
+		.parent_names = (const char *[]){ "cpu_clk" },
+		.num_parents = 1,
+	},
+};
+
+static struct clk_regmap g12a_cpu_clk_atb = {
+	.data = &(struct clk_regmap_gate_data){
+		.offset = HHI_SYS_CPU_CLK_CNTL1,
+		.bit_idx = 17,
+	},
+	.hw.init = &(struct clk_init_data) {
+		.name = "cpu_clk_atb",
+		.ops = &clk_regmap_gate_ro_ops,
+		.parent_names = (const char *[]){ "cpu_clk_atb_div" },
+		.num_parents = 1,
+		/*
+		 * This clock is set by the ROM monitor code,
+		 * Linux should not change it at runtime
+		 */
+	},
+};
+
+static struct clk_regmap g12a_cpu_clk_axi_div = {
+	.data = &(struct clk_regmap_div_data){
+		.offset = HHI_SYS_CPU_CLK_CNTL1,
+		.shift = 9,
+		.width = 3,
+		.flags = CLK_DIVIDER_POWER_OF_TWO,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "cpu_clk_axi_div",
+		.ops = &clk_regmap_divider_ro_ops,
+		.parent_names = (const char *[]){ "cpu_clk" },
+		.num_parents = 1,
+	},
+};
+
+static struct clk_regmap g12a_cpu_clk_axi = {
+	.data = &(struct clk_regmap_gate_data){
+		.offset = HHI_SYS_CPU_CLK_CNTL1,
+		.bit_idx = 18,
+	},
+	.hw.init = &(struct clk_init_data) {
+		.name = "cpu_clk_axi",
+		.ops = &clk_regmap_gate_ro_ops,
+		.parent_names = (const char *[]){ "cpu_clk_axi_div" },
+		.num_parents = 1,
+		/*
+		 * This clock is set by the ROM monitor code,
+		 * Linux should not change it at runtime
+		 */
+	},
+};
+
+static struct clk_regmap g12a_cpu_clk_trace_div = {
+	.data = &(struct clk_regmap_div_data){
+		.offset = HHI_SYS_CPU_CLK_CNTL1,
+		.shift = 20,
+		.width = 3,
+		.flags = CLK_DIVIDER_POWER_OF_TWO,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "cpu_clk_trace_div",
+		.ops = &clk_regmap_divider_ro_ops,
+		.parent_names = (const char *[]){ "cpu_clk" },
+		.num_parents = 1,
+	},
+};
+
+static struct clk_regmap g12a_cpu_clk_trace = {
+	.data = &(struct clk_regmap_gate_data){
+		.offset = HHI_SYS_CPU_CLK_CNTL1,
+		.bit_idx = 23,
+	},
+	.hw.init = &(struct clk_init_data) {
+		.name = "cpu_clk_trace",
+		.ops = &clk_regmap_gate_ro_ops,
+		.parent_names = (const char *[]){ "cpu_clk_trace_div" },
+		.num_parents = 1,
+		/*
+		 * This clock is set by the ROM monitor code,
+		 * Linux should not change it at runtime
+		 */
+	},
+};
+
 static const struct pll_mult_range g12a_gp0_pll_mult_range = {
 	.min = 55,
 	.max = 255,
@@ -302,6 +614,118 @@ static struct clk_regmap g12a_hifi_pll = {
 	},
 };
 
+/*
+ * The Meson G12A PCIE PLL is fined tuned to deliver a very precise
+ * 100MHz reference clock for the PCIe Analog PHY, and thus requires
+ * a strict register sequence to enable the PLL.
+ */
+static const struct reg_sequence g12a_pcie_pll_init_regs[] = {
+	{ .reg = HHI_PCIE_PLL_CNTL0,	.def = 0x20090496 },
+	{ .reg = HHI_PCIE_PLL_CNTL0,	.def = 0x30090496 },
+	{ .reg = HHI_PCIE_PLL_CNTL1,	.def = 0x00000000 },
+	{ .reg = HHI_PCIE_PLL_CNTL2,	.def = 0x00001100 },
+	{ .reg = HHI_PCIE_PLL_CNTL3,	.def = 0x10058e00 },
+	{ .reg = HHI_PCIE_PLL_CNTL4,	.def = 0x000100c0 },
+	{ .reg = HHI_PCIE_PLL_CNTL5,	.def = 0x68000048 },
+	{ .reg = HHI_PCIE_PLL_CNTL5,	.def = 0x68000068, .delay_us = 20 },
+	{ .reg = HHI_PCIE_PLL_CNTL4,	.def = 0x008100c0, .delay_us = 10 },
+	{ .reg = HHI_PCIE_PLL_CNTL0,	.def = 0x34090496 },
+	{ .reg = HHI_PCIE_PLL_CNTL0,	.def = 0x14090496, .delay_us = 10 },
+	{ .reg = HHI_PCIE_PLL_CNTL2,	.def = 0x00001000 },
+};
+
+/* Keep a single entry table for recalc/round_rate() ops */
+static const struct pll_params_table g12a_pcie_pll_table[] = {
+	PLL_PARAMS(150, 1),
+	{0, 0},
+};
+
+static struct clk_regmap g12a_pcie_pll_dco = {
+	.data = &(struct meson_clk_pll_data){
+		.en = {
+			.reg_off = HHI_PCIE_PLL_CNTL0,
+			.shift   = 28,
+			.width   = 1,
+		},
+		.m = {
+			.reg_off = HHI_PCIE_PLL_CNTL0,
+			.shift   = 0,
+			.width   = 8,
+		},
+		.n = {
+			.reg_off = HHI_PCIE_PLL_CNTL0,
+			.shift   = 10,
+			.width   = 5,
+		},
+		.frac = {
+			.reg_off = HHI_PCIE_PLL_CNTL1,
+			.shift   = 0,
+			.width   = 12,
+		},
+		.l = {
+			.reg_off = HHI_PCIE_PLL_CNTL0,
+			.shift   = 31,
+			.width   = 1,
+		},
+		.rst = {
+			.reg_off = HHI_PCIE_PLL_CNTL0,
+			.shift   = 29,
+			.width   = 1,
+		},
+		.table = g12a_pcie_pll_table,
+		.init_regs = g12a_pcie_pll_init_regs,
+		.init_count = ARRAY_SIZE(g12a_pcie_pll_init_regs),
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "pcie_pll_dco",
+		.ops = &meson_clk_pcie_pll_ops,
+		.parent_names = (const char *[]){ IN_PREFIX "xtal" },
+		.num_parents = 1,
+	},
+};
+
+static struct clk_fixed_factor g12a_pcie_pll_dco_div2 = {
+	.mult = 1,
+	.div = 2,
+	.hw.init = &(struct clk_init_data){
+		.name = "pcie_pll_dco_div2",
+		.ops = &clk_fixed_factor_ops,
+		.parent_names = (const char *[]){ "pcie_pll_dco" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap g12a_pcie_pll_od = {
+	.data = &(struct clk_regmap_div_data){
+		.offset = HHI_PCIE_PLL_CNTL0,
+		.shift = 16,
+		.width = 5,
+		.flags = CLK_DIVIDER_ROUND_CLOSEST |
+			 CLK_DIVIDER_ONE_BASED |
+			 CLK_DIVIDER_ALLOW_ZERO,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "pcie_pll_od",
+		.ops = &clk_regmap_divider_ops,
+		.parent_names = (const char *[]){ "pcie_pll_dco_div2" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_fixed_factor g12a_pcie_pll = {
+	.mult = 1,
+	.div = 2,
+	.hw.init = &(struct clk_init_data){
+		.name = "pcie_pll_pll",
+		.ops = &clk_fixed_factor_ops,
+		.parent_names = (const char *[]){ "pcie_pll_od" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
 static struct clk_regmap g12a_hdmi_pll_dco = {
 	.data = &(struct meson_clk_pll_data){
 		.en = {
@@ -1071,6 +1495,151 @@ static struct clk_regmap g12a_vpu = {
 	},
 };
 
+/* VDEC clocks */
+
+static const char * const g12a_vdec_parent_names[] = {
+	"fclk_div2p5", "fclk_div3", "fclk_div4", "fclk_div5", "fclk_div7",
+	"hifi_pll", "gp0_pll",
+};
+
+static struct clk_regmap g12a_vdec_1_sel = {
+	.data = &(struct clk_regmap_mux_data){
+		.offset = HHI_VDEC_CLK_CNTL,
+		.mask = 0x7,
+		.shift = 9,
+		.flags = CLK_MUX_ROUND_CLOSEST,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "vdec_1_sel",
+		.ops = &clk_regmap_mux_ops,
+		.parent_names = g12a_vdec_parent_names,
+		.num_parents = ARRAY_SIZE(g12a_vdec_parent_names),
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap g12a_vdec_1_div = {
+	.data = &(struct clk_regmap_div_data){
+		.offset = HHI_VDEC_CLK_CNTL,
+		.shift = 0,
+		.width = 7,
+		.flags = CLK_DIVIDER_ROUND_CLOSEST,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "vdec_1_div",
+		.ops = &clk_regmap_divider_ops,
+		.parent_names = (const char *[]){ "vdec_1_sel" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap g12a_vdec_1 = {
+	.data = &(struct clk_regmap_gate_data){
+		.offset = HHI_VDEC_CLK_CNTL,
+		.bit_idx = 8,
+	},
+	.hw.init = &(struct clk_init_data) {
+		.name = "vdec_1",
+		.ops = &clk_regmap_gate_ops,
+		.parent_names = (const char *[]){ "vdec_1_div" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap g12a_vdec_hevcf_sel = {
+	.data = &(struct clk_regmap_mux_data){
+		.offset = HHI_VDEC2_CLK_CNTL,
+		.mask = 0x7,
+		.shift = 9,
+		.flags = CLK_MUX_ROUND_CLOSEST,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "vdec_hevcf_sel",
+		.ops = &clk_regmap_mux_ops,
+		.parent_names = g12a_vdec_parent_names,
+		.num_parents = ARRAY_SIZE(g12a_vdec_parent_names),
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap g12a_vdec_hevcf_div = {
+	.data = &(struct clk_regmap_div_data){
+		.offset = HHI_VDEC2_CLK_CNTL,
+		.shift = 0,
+		.width = 7,
+		.flags = CLK_DIVIDER_ROUND_CLOSEST,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "vdec_hevcf_div",
+		.ops = &clk_regmap_divider_ops,
+		.parent_names = (const char *[]){ "vdec_hevcf_sel" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap g12a_vdec_hevcf = {
+	.data = &(struct clk_regmap_gate_data){
+		.offset = HHI_VDEC2_CLK_CNTL,
+		.bit_idx = 8,
+	},
+	.hw.init = &(struct clk_init_data) {
+		.name = "vdec_hevcf",
+		.ops = &clk_regmap_gate_ops,
+		.parent_names = (const char *[]){ "vdec_hevcf_div" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap g12a_vdec_hevc_sel = {
+	.data = &(struct clk_regmap_mux_data){
+		.offset = HHI_VDEC2_CLK_CNTL,
+		.mask = 0x7,
+		.shift = 25,
+		.flags = CLK_MUX_ROUND_CLOSEST,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "vdec_hevc_sel",
+		.ops = &clk_regmap_mux_ops,
+		.parent_names = g12a_vdec_parent_names,
+		.num_parents = ARRAY_SIZE(g12a_vdec_parent_names),
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap g12a_vdec_hevc_div = {
+	.data = &(struct clk_regmap_div_data){
+		.offset = HHI_VDEC2_CLK_CNTL,
+		.shift = 16,
+		.width = 7,
+		.flags = CLK_DIVIDER_ROUND_CLOSEST,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "vdec_hevc_div",
+		.ops = &clk_regmap_divider_ops,
+		.parent_names = (const char *[]){ "vdec_hevc_sel" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap g12a_vdec_hevc = {
+	.data = &(struct clk_regmap_gate_data){
+		.offset = HHI_VDEC2_CLK_CNTL,
+		.bit_idx = 24,
+	},
+	.hw.init = &(struct clk_init_data) {
+		.name = "vdec_hevc",
+		.ops = &clk_regmap_gate_ops,
+		.parent_names = (const char *[]){ "vdec_hevc_div" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
 /* VAPB Clock */
 
 static const char * const g12a_vapb_parent_names[] = {
@@ -2167,6 +2736,39 @@ static struct clk_hw_onecell_data g12a_hw_onecell_data = {
 		[CLKID_MALI]			= &g12a_mali.hw,
 		[CLKID_MPLL_5OM_DIV]		= &g12a_mpll_50m_div.hw,
 		[CLKID_MPLL_5OM]		= &g12a_mpll_50m.hw,
+		[CLKID_SYS_PLL_DIV16_EN]	= &g12a_sys_pll_div16_en.hw,
+		[CLKID_SYS_PLL_DIV16]		= &g12a_sys_pll_div16.hw,
+		[CLKID_CPU_CLK_DYN0_SEL]	= &g12a_cpu_clk_premux0.hw,
+		[CLKID_CPU_CLK_DYN0_DIV]	= &g12a_cpu_clk_mux0_div.hw,
+		[CLKID_CPU_CLK_DYN0]		= &g12a_cpu_clk_postmux0.hw,
+		[CLKID_CPU_CLK_DYN1_SEL]	= &g12a_cpu_clk_premux1.hw,
+		[CLKID_CPU_CLK_DYN1_DIV]	= &g12a_cpu_clk_mux1_div.hw,
+		[CLKID_CPU_CLK_DYN1]		= &g12a_cpu_clk_postmux1.hw,
+		[CLKID_CPU_CLK_DYN]		= &g12a_cpu_clk_dyn.hw,
+		[CLKID_CPU_CLK]			= &g12a_cpu_clk.hw,
+		[CLKID_CPU_CLK_DIV16_EN]	= &g12a_cpu_clk_div16_en.hw,
+		[CLKID_CPU_CLK_DIV16]		= &g12a_cpu_clk_div16.hw,
+		[CLKID_CPU_CLK_APB_DIV]		= &g12a_cpu_clk_apb_div.hw,
+		[CLKID_CPU_CLK_APB]		= &g12a_cpu_clk_apb.hw,
+		[CLKID_CPU_CLK_ATB_DIV]		= &g12a_cpu_clk_atb_div.hw,
+		[CLKID_CPU_CLK_ATB]		= &g12a_cpu_clk_atb.hw,
+		[CLKID_CPU_CLK_AXI_DIV]		= &g12a_cpu_clk_axi_div.hw,
+		[CLKID_CPU_CLK_AXI]		= &g12a_cpu_clk_axi.hw,
+		[CLKID_CPU_CLK_TRACE_DIV]	= &g12a_cpu_clk_trace_div.hw,
+		[CLKID_CPU_CLK_TRACE]		= &g12a_cpu_clk_trace.hw,
+		[CLKID_PCIE_PLL_DCO]		= &g12a_pcie_pll_dco.hw,
+		[CLKID_PCIE_PLL_DCO_DIV2]	= &g12a_pcie_pll_dco_div2.hw,
+		[CLKID_PCIE_PLL_OD]		= &g12a_pcie_pll_od.hw,
+		[CLKID_PCIE_PLL]		= &g12a_pcie_pll.hw,
+		[CLKID_VDEC_1_SEL]		= &g12a_vdec_1_sel.hw,
+		[CLKID_VDEC_1_DIV]		= &g12a_vdec_1_div.hw,
+		[CLKID_VDEC_1]			= &g12a_vdec_1.hw,
+		[CLKID_VDEC_HEVC_SEL]		= &g12a_vdec_hevc_sel.hw,
+		[CLKID_VDEC_HEVC_DIV]		= &g12a_vdec_hevc_div.hw,
+		[CLKID_VDEC_HEVC]		= &g12a_vdec_hevc.hw,
+		[CLKID_VDEC_HEVCF_SEL]		= &g12a_vdec_hevcf_sel.hw,
+		[CLKID_VDEC_HEVCF_DIV]		= &g12a_vdec_hevcf_div.hw,
+		[CLKID_VDEC_HEVCF]		= &g12a_vdec_hevcf.hw,
 		[NR_CLKS]			= NULL,
 	},
 	.num = NR_CLKS,
@@ -2335,6 +2937,35 @@ static struct clk_regmap *const g12a_clk_regmaps[] = {
 	&g12a_mali_1,
 	&g12a_mali,
 	&g12a_mpll_50m,
+	&g12a_sys_pll_div16_en,
+	&g12a_cpu_clk_premux0,
+	&g12a_cpu_clk_mux0_div,
+	&g12a_cpu_clk_postmux0,
+	&g12a_cpu_clk_premux1,
+	&g12a_cpu_clk_mux1_div,
+	&g12a_cpu_clk_postmux1,
+	&g12a_cpu_clk_dyn,
+	&g12a_cpu_clk,
+	&g12a_cpu_clk_div16_en,
+	&g12a_cpu_clk_apb_div,
+	&g12a_cpu_clk_apb,
+	&g12a_cpu_clk_atb_div,
+	&g12a_cpu_clk_atb,
+	&g12a_cpu_clk_axi_div,
+	&g12a_cpu_clk_axi,
+	&g12a_cpu_clk_trace_div,
+	&g12a_cpu_clk_trace,
+	&g12a_pcie_pll_od,
+	&g12a_pcie_pll_dco,
+	&g12a_vdec_1_sel,
+	&g12a_vdec_1_div,
+	&g12a_vdec_1,
+	&g12a_vdec_hevc_sel,
+	&g12a_vdec_hevc_div,
+	&g12a_vdec_hevc,
+	&g12a_vdec_hevcf_sel,
+	&g12a_vdec_hevcf_div,
+	&g12a_vdec_hevcf,
 };
 
 static const struct meson_eeclkc_data g12a_clkc_data = {
diff --git a/drivers/clk/meson/g12a.h b/drivers/clk/meson/g12a.h
index f399dfe1401c..39c41af70804 100644
--- a/drivers/clk/meson/g12a.h
+++ b/drivers/clk/meson/g12a.h
@@ -50,6 +50,7 @@
 #define HHI_GCLK_MPEG2			0x148
 #define HHI_GCLK_OTHER			0x150
 #define HHI_GCLK_OTHER2			0x154
+#define HHI_SYS_CPU_CLK_CNTL1		0x15c
 #define HHI_VID_CLK_DIV			0x164
 #define HHI_MPEG_CLK_CNTL		0x174
 #define HHI_AUD_CLK_CNTL		0x178
@@ -166,8 +167,36 @@
 #define CLKID_MALI_0_DIV			170
 #define CLKID_MALI_1_DIV			173
 #define CLKID_MPLL_5OM_DIV			176
+#define CLKID_SYS_PLL_DIV16_EN			178
+#define CLKID_SYS_PLL_DIV16			179
+#define CLKID_CPU_CLK_DYN0_SEL			180
+#define CLKID_CPU_CLK_DYN0_DIV			181
+#define CLKID_CPU_CLK_DYN0			182
+#define CLKID_CPU_CLK_DYN1_SEL			183
+#define CLKID_CPU_CLK_DYN1_DIV			184
+#define CLKID_CPU_CLK_DYN1			185
+#define CLKID_CPU_CLK_DYN			186
+#define CLKID_CPU_CLK_DIV16_EN			188
+#define CLKID_CPU_CLK_DIV16			189
+#define CLKID_CPU_CLK_APB_DIV			190
+#define CLKID_CPU_CLK_APB			191
+#define CLKID_CPU_CLK_ATB_DIV			192
+#define CLKID_CPU_CLK_ATB			193
+#define CLKID_CPU_CLK_AXI_DIV			194
+#define CLKID_CPU_CLK_AXI			195
+#define CLKID_CPU_CLK_TRACE_DIV			196
+#define CLKID_CPU_CLK_TRACE			197
+#define CLKID_PCIE_PLL_DCO			198
+#define CLKID_PCIE_PLL_DCO_DIV2			199
+#define CLKID_PCIE_PLL_OD			200
+#define CLKID_VDEC_1_SEL			202
+#define CLKID_VDEC_1_DIV			203
+#define CLKID_VDEC_HEVC_SEL			205
+#define CLKID_VDEC_HEVC_DIV			206
+#define CLKID_VDEC_HEVCF_SEL			208
+#define CLKID_VDEC_HEVCF_DIV			209
 
-#define NR_CLKS					178
+#define NR_CLKS					211
 
 /* include the CLKIDs that have been made part of the DT binding */
 #include <dt-bindings/clock/g12a-clkc.h>
diff --git a/drivers/clk/meson/meson8b.c b/drivers/clk/meson/meson8b.c
index 576ad42252d0..37cf0f01bb5d 100644
--- a/drivers/clk/meson/meson8b.c
+++ b/drivers/clk/meson/meson8b.c
@@ -1703,6 +1703,456 @@ static struct clk_regmap meson8b_mali = {
 	},
 };
 
+static const struct pll_params_table meson8m2_gp_pll_params_table[] = {
+	PLL_PARAMS(182, 3),
+	{ /* sentinel */ },
+};
+
+static struct clk_regmap meson8m2_gp_pll_dco = {
+	.data = &(struct meson_clk_pll_data){
+		.en = {
+			.reg_off = HHI_GP_PLL_CNTL,
+			.shift   = 30,
+			.width   = 1,
+		},
+		.m = {
+			.reg_off = HHI_GP_PLL_CNTL,
+			.shift   = 0,
+			.width   = 9,
+		},
+		.n = {
+			.reg_off = HHI_GP_PLL_CNTL,
+			.shift   = 9,
+			.width   = 5,
+		},
+		.l = {
+			.reg_off = HHI_GP_PLL_CNTL,
+			.shift   = 31,
+			.width   = 1,
+		},
+		.rst = {
+			.reg_off = HHI_GP_PLL_CNTL,
+			.shift   = 29,
+			.width   = 1,
+		},
+		.table = meson8m2_gp_pll_params_table,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "gp_pll_dco",
+		.ops = &meson_clk_pll_ops,
+		.parent_names = (const char *[]){ "xtal" },
+		.num_parents = 1,
+	},
+};
+
+static struct clk_regmap meson8m2_gp_pll = {
+	.data = &(struct clk_regmap_div_data){
+		.offset = HHI_GP_PLL_CNTL,
+		.shift = 16,
+		.width = 2,
+		.flags = CLK_DIVIDER_POWER_OF_TWO,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "gp_pll",
+		.ops = &clk_regmap_divider_ops,
+		.parent_names = (const char *[]){ "gp_pll_dco" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static const char * const mmeson8b_vpu_0_1_parent_names[] = {
+	"fclk_div4", "fclk_div3", "fclk_div5", "fclk_div7"
+};
+
+static const char * const mmeson8m2_vpu_0_1_parent_names[] = {
+	"fclk_div4", "fclk_div3", "fclk_div5", "gp_pll"
+};
+
+static struct clk_regmap meson8b_vpu_0_sel = {
+	.data = &(struct clk_regmap_mux_data){
+		.offset = HHI_VPU_CLK_CNTL,
+		.mask = 0x3,
+		.shift = 9,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "vpu_0_sel",
+		.ops = &clk_regmap_mux_ops,
+		.parent_names = mmeson8b_vpu_0_1_parent_names,
+		.num_parents = ARRAY_SIZE(mmeson8b_vpu_0_1_parent_names),
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap meson8m2_vpu_0_sel = {
+	.data = &(struct clk_regmap_mux_data){
+		.offset = HHI_VPU_CLK_CNTL,
+		.mask = 0x3,
+		.shift = 9,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "vpu_0_sel",
+		.ops = &clk_regmap_mux_ops,
+		.parent_names = mmeson8m2_vpu_0_1_parent_names,
+		.num_parents = ARRAY_SIZE(mmeson8m2_vpu_0_1_parent_names),
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap meson8b_vpu_0_div = {
+	.data = &(struct clk_regmap_div_data){
+		.offset = HHI_VPU_CLK_CNTL,
+		.shift = 0,
+		.width = 7,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "vpu_0_div",
+		.ops = &clk_regmap_divider_ops,
+		.parent_names = (const char *[]){ "vpu_0_sel" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap meson8b_vpu_0 = {
+	.data = &(struct clk_regmap_gate_data){
+		.offset = HHI_VPU_CLK_CNTL,
+		.bit_idx = 8,
+	},
+	.hw.init = &(struct clk_init_data) {
+		.name = "vpu_0",
+		.ops = &clk_regmap_gate_ops,
+		.parent_names = (const char *[]){ "vpu_0_div" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap meson8b_vpu_1_sel = {
+	.data = &(struct clk_regmap_mux_data){
+		.offset = HHI_VPU_CLK_CNTL,
+		.mask = 0x3,
+		.shift = 25,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "vpu_1_sel",
+		.ops = &clk_regmap_mux_ops,
+		.parent_names = mmeson8b_vpu_0_1_parent_names,
+		.num_parents = ARRAY_SIZE(mmeson8b_vpu_0_1_parent_names),
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap meson8m2_vpu_1_sel = {
+	.data = &(struct clk_regmap_mux_data){
+		.offset = HHI_VPU_CLK_CNTL,
+		.mask = 0x3,
+		.shift = 25,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "vpu_1_sel",
+		.ops = &clk_regmap_mux_ops,
+		.parent_names = mmeson8m2_vpu_0_1_parent_names,
+		.num_parents = ARRAY_SIZE(mmeson8m2_vpu_0_1_parent_names),
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap meson8b_vpu_1_div = {
+	.data = &(struct clk_regmap_div_data){
+		.offset = HHI_VPU_CLK_CNTL,
+		.shift = 16,
+		.width = 7,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "vpu_1_div",
+		.ops = &clk_regmap_divider_ops,
+		.parent_names = (const char *[]){ "vpu_1_sel" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap meson8b_vpu_1 = {
+	.data = &(struct clk_regmap_gate_data){
+		.offset = HHI_VPU_CLK_CNTL,
+		.bit_idx = 24,
+	},
+	.hw.init = &(struct clk_init_data) {
+		.name = "vpu_1",
+		.ops = &clk_regmap_gate_ops,
+		.parent_names = (const char *[]){ "vpu_1_div" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap meson8b_vpu = {
+	.data = &(struct clk_regmap_mux_data){
+		.offset = HHI_VPU_CLK_CNTL,
+		.mask = 1,
+		.shift = 31,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "vpu",
+		.ops = &clk_regmap_mux_ops,
+		.parent_names = (const char *[]){ "vpu_0", "vpu_1" },
+		.num_parents = 2,
+		.flags = CLK_SET_RATE_NO_REPARENT,
+	},
+};
+
+static const char * const meson8b_vdec_parent_names[] = {
+	"fclk_div4", "fclk_div3", "fclk_div5", "fclk_div7", "mpll2", "mpll1"
+};
+
+static struct clk_regmap meson8b_vdec_1_sel = {
+	.data = &(struct clk_regmap_mux_data){
+		.offset = HHI_VDEC_CLK_CNTL,
+		.mask = 0x3,
+		.shift = 9,
+		.flags = CLK_MUX_ROUND_CLOSEST,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "vdec_1_sel",
+		.ops = &clk_regmap_mux_ops,
+		.parent_names = meson8b_vdec_parent_names,
+		.num_parents = ARRAY_SIZE(meson8b_vdec_parent_names),
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap meson8b_vdec_1_1_div = {
+	.data = &(struct clk_regmap_div_data){
+		.offset = HHI_VDEC_CLK_CNTL,
+		.shift = 0,
+		.width = 7,
+		.flags = CLK_DIVIDER_ROUND_CLOSEST,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "vdec_1_1_div",
+		.ops = &clk_regmap_divider_ops,
+		.parent_names = (const char *[]){ "vdec_1_sel" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap meson8b_vdec_1_1 = {
+	.data = &(struct clk_regmap_gate_data){
+		.offset = HHI_VDEC_CLK_CNTL,
+		.bit_idx = 8,
+	},
+	.hw.init = &(struct clk_init_data) {
+		.name = "vdec_1_1",
+		.ops = &clk_regmap_gate_ops,
+		.parent_names = (const char *[]){ "vdec_1_1_div" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap meson8b_vdec_1_2_div = {
+	.data = &(struct clk_regmap_div_data){
+		.offset = HHI_VDEC3_CLK_CNTL,
+		.shift = 0,
+		.width = 7,
+		.flags = CLK_DIVIDER_ROUND_CLOSEST,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "vdec_1_2_div",
+		.ops = &clk_regmap_divider_ops,
+		.parent_names = (const char *[]){ "vdec_1_sel" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap meson8b_vdec_1_2 = {
+	.data = &(struct clk_regmap_gate_data){
+		.offset = HHI_VDEC3_CLK_CNTL,
+		.bit_idx = 8,
+	},
+	.hw.init = &(struct clk_init_data) {
+		.name = "vdec_1_2",
+		.ops = &clk_regmap_gate_ops,
+		.parent_names = (const char *[]){ "vdec_1_2_div" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap meson8b_vdec_1 = {
+	.data = &(struct clk_regmap_mux_data){
+		.offset = HHI_VDEC3_CLK_CNTL,
+		.mask = 0x1,
+		.shift = 15,
+		.flags = CLK_MUX_ROUND_CLOSEST,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "vdec_1",
+		.ops = &clk_regmap_mux_ops,
+		.parent_names = (const char *[]){ "vdec_1_1", "vdec_1_2" },
+		.num_parents = 2,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap meson8b_vdec_hcodec_sel = {
+	.data = &(struct clk_regmap_mux_data){
+		.offset = HHI_VDEC_CLK_CNTL,
+		.mask = 0x3,
+		.shift = 25,
+		.flags = CLK_MUX_ROUND_CLOSEST,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "vdec_hcodec_sel",
+		.ops = &clk_regmap_mux_ops,
+		.parent_names = meson8b_vdec_parent_names,
+		.num_parents = ARRAY_SIZE(meson8b_vdec_parent_names),
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap meson8b_vdec_hcodec_div = {
+	.data = &(struct clk_regmap_div_data){
+		.offset = HHI_VDEC_CLK_CNTL,
+		.shift = 16,
+		.width = 7,
+		.flags = CLK_DIVIDER_ROUND_CLOSEST,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "vdec_hcodec_div",
+		.ops = &clk_regmap_divider_ops,
+		.parent_names = (const char *[]){ "vdec_hcodec_sel" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap meson8b_vdec_hcodec = {
+	.data = &(struct clk_regmap_gate_data){
+		.offset = HHI_VDEC_CLK_CNTL,
+		.bit_idx = 24,
+	},
+	.hw.init = &(struct clk_init_data) {
+		.name = "vdec_hcodec",
+		.ops = &clk_regmap_gate_ops,
+		.parent_names = (const char *[]){ "vdec_hcodec_div" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap meson8b_vdec_2_sel = {
+	.data = &(struct clk_regmap_mux_data){
+		.offset = HHI_VDEC2_CLK_CNTL,
+		.mask = 0x3,
+		.shift = 9,
+		.flags = CLK_MUX_ROUND_CLOSEST,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "vdec_2_sel",
+		.ops = &clk_regmap_mux_ops,
+		.parent_names = meson8b_vdec_parent_names,
+		.num_parents = ARRAY_SIZE(meson8b_vdec_parent_names),
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap meson8b_vdec_2_div = {
+	.data = &(struct clk_regmap_div_data){
+		.offset = HHI_VDEC2_CLK_CNTL,
+		.shift = 0,
+		.width = 7,
+		.flags = CLK_DIVIDER_ROUND_CLOSEST,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "vdec_2_div",
+		.ops = &clk_regmap_divider_ops,
+		.parent_names = (const char *[]){ "vdec_2_sel" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap meson8b_vdec_2 = {
+	.data = &(struct clk_regmap_gate_data){
+		.offset = HHI_VDEC2_CLK_CNTL,
+		.bit_idx = 8,
+	},
+	.hw.init = &(struct clk_init_data) {
+		.name = "vdec_2",
+		.ops = &clk_regmap_gate_ops,
+		.parent_names = (const char *[]){ "vdec_2_div" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap meson8b_vdec_hevc_sel = {
+	.data = &(struct clk_regmap_mux_data){
+		.offset = HHI_VDEC2_CLK_CNTL,
+		.mask = 0x3,
+		.shift = 25,
+		.flags = CLK_MUX_ROUND_CLOSEST,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "vdec_hevc_sel",
+		.ops = &clk_regmap_mux_ops,
+		.parent_names = meson8b_vdec_parent_names,
+		.num_parents = ARRAY_SIZE(meson8b_vdec_parent_names),
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap meson8b_vdec_hevc_div = {
+	.data = &(struct clk_regmap_div_data){
+		.offset = HHI_VDEC2_CLK_CNTL,
+		.shift = 16,
+		.width = 7,
+		.flags = CLK_DIVIDER_ROUND_CLOSEST,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "vdec_hevc_div",
+		.ops = &clk_regmap_divider_ops,
+		.parent_names = (const char *[]){ "vdec_hevc_sel" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap meson8b_vdec_hevc_en = {
+	.data = &(struct clk_regmap_gate_data){
+		.offset = HHI_VDEC2_CLK_CNTL,
+		.bit_idx = 24,
+	},
+	.hw.init = &(struct clk_init_data) {
+		.name = "vdec_hevc_en",
+		.ops = &clk_regmap_gate_ops,
+		.parent_names = (const char *[]){ "vdec_hevc_div" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
+static struct clk_regmap meson8b_vdec_hevc = {
+	.data = &(struct clk_regmap_mux_data){
+		.offset = HHI_VDEC2_CLK_CNTL,
+		.mask = 0x1,
+		.shift = 31,
+		.flags = CLK_MUX_ROUND_CLOSEST,
+	},
+	.hw.init = &(struct clk_init_data){
+		.name = "vdec_hevc",
+		.ops = &clk_regmap_mux_ops,
+		/* TODO: The second parent is currently unknown */
+		.parent_names = (const char *[]){ "vdec_hevc_en" },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+	},
+};
+
 /* Everything Else (EE) domain gates */
 
 static MESON_GATE(meson8b_ddr, HHI_GCLK_MPEG0, 0);
@@ -1966,6 +2416,22 @@ static struct clk_hw_onecell_data meson8_hw_onecell_data = {
 		[CLKID_MALI_0_SEL]	    = &meson8b_mali_0_sel.hw,
 		[CLKID_MALI_0_DIV]	    = &meson8b_mali_0_div.hw,
 		[CLKID_MALI]		    = &meson8b_mali_0.hw,
+		[CLKID_VPU_0_SEL]	    = &meson8b_vpu_0_sel.hw,
+		[CLKID_VPU_0_DIV]	    = &meson8b_vpu_0_div.hw,
+		[CLKID_VPU]		    = &meson8b_vpu_0.hw,
+		[CLKID_VDEC_1_SEL]	    = &meson8b_vdec_1_sel.hw,
+		[CLKID_VDEC_1_1_DIV]	    = &meson8b_vdec_1_1_div.hw,
+		[CLKID_VDEC_1]	   	    = &meson8b_vdec_1_1.hw,
+		[CLKID_VDEC_HCODEC_SEL]	    = &meson8b_vdec_hcodec_sel.hw,
+		[CLKID_VDEC_HCODEC_DIV]	    = &meson8b_vdec_hcodec_div.hw,
+		[CLKID_VDEC_HCODEC]	    = &meson8b_vdec_hcodec.hw,
+		[CLKID_VDEC_2_SEL]	    = &meson8b_vdec_2_sel.hw,
+		[CLKID_VDEC_2_DIV]	    = &meson8b_vdec_2_div.hw,
+		[CLKID_VDEC_2]	    	    = &meson8b_vdec_2.hw,
+		[CLKID_VDEC_HEVC_SEL]	    = &meson8b_vdec_hevc_sel.hw,
+		[CLKID_VDEC_HEVC_DIV]	    = &meson8b_vdec_hevc_div.hw,
+		[CLKID_VDEC_HEVC_EN]	    = &meson8b_vdec_hevc_en.hw,
+		[CLKID_VDEC_HEVC]	    = &meson8b_vdec_hevc.hw,
 		[CLK_NR_CLKS]		    = NULL,
 	},
 	.num = CLK_NR_CLKS,
@@ -2152,6 +2618,240 @@ static struct clk_hw_onecell_data meson8b_hw_onecell_data = {
 		[CLKID_MALI_1_DIV]	    = &meson8b_mali_1_div.hw,
 		[CLKID_MALI_1]		    = &meson8b_mali_1.hw,
 		[CLKID_MALI]		    = &meson8b_mali.hw,
+		[CLKID_VPU_0_SEL]	    = &meson8b_vpu_0_sel.hw,
+		[CLKID_VPU_0_DIV]	    = &meson8b_vpu_0_div.hw,
+		[CLKID_VPU_0]		    = &meson8b_vpu_0.hw,
+		[CLKID_VPU_1_SEL]	    = &meson8b_vpu_1_sel.hw,
+		[CLKID_VPU_1_DIV]	    = &meson8b_vpu_1_div.hw,
+		[CLKID_VPU_1]		    = &meson8b_vpu_1.hw,
+		[CLKID_VPU]		    = &meson8b_vpu.hw,
+		[CLKID_VDEC_1_SEL]	    = &meson8b_vdec_1_sel.hw,
+		[CLKID_VDEC_1_1_DIV]	    = &meson8b_vdec_1_1_div.hw,
+		[CLKID_VDEC_1_1]	    = &meson8b_vdec_1_1.hw,
+		[CLKID_VDEC_1_2_DIV]	    = &meson8b_vdec_1_2_div.hw,
+		[CLKID_VDEC_1_2]	    = &meson8b_vdec_1_2.hw,
+		[CLKID_VDEC_1]	    	    = &meson8b_vdec_1.hw,
+		[CLKID_VDEC_HCODEC_SEL]	    = &meson8b_vdec_hcodec_sel.hw,
+		[CLKID_VDEC_HCODEC_DIV]	    = &meson8b_vdec_hcodec_div.hw,
+		[CLKID_VDEC_HCODEC]	    = &meson8b_vdec_hcodec.hw,
+		[CLKID_VDEC_2_SEL]	    = &meson8b_vdec_2_sel.hw,
+		[CLKID_VDEC_2_DIV]	    = &meson8b_vdec_2_div.hw,
+		[CLKID_VDEC_2]	    	    = &meson8b_vdec_2.hw,
+		[CLKID_VDEC_HEVC_SEL]	    = &meson8b_vdec_hevc_sel.hw,
+		[CLKID_VDEC_HEVC_DIV]	    = &meson8b_vdec_hevc_div.hw,
+		[CLKID_VDEC_HEVC_EN]	    = &meson8b_vdec_hevc_en.hw,
+		[CLKID_VDEC_HEVC]	    = &meson8b_vdec_hevc.hw,
+		[CLK_NR_CLKS]		    = NULL,
+	},
+	.num = CLK_NR_CLKS,
+};
+
+static struct clk_hw_onecell_data meson8m2_hw_onecell_data = {
+	.hws = {
+		[CLKID_XTAL] = &meson8b_xtal.hw,
+		[CLKID_PLL_FIXED] = &meson8b_fixed_pll.hw,
+		[CLKID_PLL_VID] = &meson8b_vid_pll.hw,
+		[CLKID_PLL_SYS] = &meson8b_sys_pll.hw,
+		[CLKID_FCLK_DIV2] = &meson8b_fclk_div2.hw,
+		[CLKID_FCLK_DIV3] = &meson8b_fclk_div3.hw,
+		[CLKID_FCLK_DIV4] = &meson8b_fclk_div4.hw,
+		[CLKID_FCLK_DIV5] = &meson8b_fclk_div5.hw,
+		[CLKID_FCLK_DIV7] = &meson8b_fclk_div7.hw,
+		[CLKID_CPUCLK] = &meson8b_cpu_clk.hw,
+		[CLKID_MPEG_SEL] = &meson8b_mpeg_clk_sel.hw,
+		[CLKID_MPEG_DIV] = &meson8b_mpeg_clk_div.hw,
+		[CLKID_CLK81] = &meson8b_clk81.hw,
+		[CLKID_DDR]		    = &meson8b_ddr.hw,
+		[CLKID_DOS]		    = &meson8b_dos.hw,
+		[CLKID_ISA]		    = &meson8b_isa.hw,
+		[CLKID_PL301]		    = &meson8b_pl301.hw,
+		[CLKID_PERIPHS]		    = &meson8b_periphs.hw,
+		[CLKID_SPICC]		    = &meson8b_spicc.hw,
+		[CLKID_I2C]		    = &meson8b_i2c.hw,
+		[CLKID_SAR_ADC]		    = &meson8b_sar_adc.hw,
+		[CLKID_SMART_CARD]	    = &meson8b_smart_card.hw,
+		[CLKID_RNG0]		    = &meson8b_rng0.hw,
+		[CLKID_UART0]		    = &meson8b_uart0.hw,
+		[CLKID_SDHC]		    = &meson8b_sdhc.hw,
+		[CLKID_STREAM]		    = &meson8b_stream.hw,
+		[CLKID_ASYNC_FIFO]	    = &meson8b_async_fifo.hw,
+		[CLKID_SDIO]		    = &meson8b_sdio.hw,
+		[CLKID_ABUF]		    = &meson8b_abuf.hw,
+		[CLKID_HIU_IFACE]	    = &meson8b_hiu_iface.hw,
+		[CLKID_ASSIST_MISC]	    = &meson8b_assist_misc.hw,
+		[CLKID_SPI]		    = &meson8b_spi.hw,
+		[CLKID_I2S_SPDIF]	    = &meson8b_i2s_spdif.hw,
+		[CLKID_ETH]		    = &meson8b_eth.hw,
+		[CLKID_DEMUX]		    = &meson8b_demux.hw,
+		[CLKID_AIU_GLUE]	    = &meson8b_aiu_glue.hw,
+		[CLKID_IEC958]		    = &meson8b_iec958.hw,
+		[CLKID_I2S_OUT]		    = &meson8b_i2s_out.hw,
+		[CLKID_AMCLK]		    = &meson8b_amclk.hw,
+		[CLKID_AIFIFO2]		    = &meson8b_aififo2.hw,
+		[CLKID_MIXER]		    = &meson8b_mixer.hw,
+		[CLKID_MIXER_IFACE]	    = &meson8b_mixer_iface.hw,
+		[CLKID_ADC]		    = &meson8b_adc.hw,
+		[CLKID_BLKMV]		    = &meson8b_blkmv.hw,
+		[CLKID_AIU]		    = &meson8b_aiu.hw,
+		[CLKID_UART1]		    = &meson8b_uart1.hw,
+		[CLKID_G2D]		    = &meson8b_g2d.hw,
+		[CLKID_USB0]		    = &meson8b_usb0.hw,
+		[CLKID_USB1]		    = &meson8b_usb1.hw,
+		[CLKID_RESET]		    = &meson8b_reset.hw,
+		[CLKID_NAND]		    = &meson8b_nand.hw,
+		[CLKID_DOS_PARSER]	    = &meson8b_dos_parser.hw,
+		[CLKID_USB]		    = &meson8b_usb.hw,
+		[CLKID_VDIN1]		    = &meson8b_vdin1.hw,
+		[CLKID_AHB_ARB0]	    = &meson8b_ahb_arb0.hw,
+		[CLKID_EFUSE]		    = &meson8b_efuse.hw,
+		[CLKID_BOOT_ROM]	    = &meson8b_boot_rom.hw,
+		[CLKID_AHB_DATA_BUS]	    = &meson8b_ahb_data_bus.hw,
+		[CLKID_AHB_CTRL_BUS]	    = &meson8b_ahb_ctrl_bus.hw,
+		[CLKID_HDMI_INTR_SYNC]	    = &meson8b_hdmi_intr_sync.hw,
+		[CLKID_HDMI_PCLK]	    = &meson8b_hdmi_pclk.hw,
+		[CLKID_USB1_DDR_BRIDGE]	    = &meson8b_usb1_ddr_bridge.hw,
+		[CLKID_USB0_DDR_BRIDGE]	    = &meson8b_usb0_ddr_bridge.hw,
+		[CLKID_MMC_PCLK]	    = &meson8b_mmc_pclk.hw,
+		[CLKID_DVIN]		    = &meson8b_dvin.hw,
+		[CLKID_UART2]		    = &meson8b_uart2.hw,
+		[CLKID_SANA]		    = &meson8b_sana.hw,
+		[CLKID_VPU_INTR]	    = &meson8b_vpu_intr.hw,
+		[CLKID_SEC_AHB_AHB3_BRIDGE] = &meson8b_sec_ahb_ahb3_bridge.hw,
+		[CLKID_CLK81_A9]	    = &meson8b_clk81_a9.hw,
+		[CLKID_VCLK2_VENCI0]	    = &meson8b_vclk2_venci0.hw,
+		[CLKID_VCLK2_VENCI1]	    = &meson8b_vclk2_venci1.hw,
+		[CLKID_VCLK2_VENCP0]	    = &meson8b_vclk2_vencp0.hw,
+		[CLKID_VCLK2_VENCP1]	    = &meson8b_vclk2_vencp1.hw,
+		[CLKID_GCLK_VENCI_INT]	    = &meson8b_gclk_venci_int.hw,
+		[CLKID_GCLK_VENCP_INT]	    = &meson8b_gclk_vencp_int.hw,
+		[CLKID_DAC_CLK]		    = &meson8b_dac_clk.hw,
+		[CLKID_AOCLK_GATE]	    = &meson8b_aoclk_gate.hw,
+		[CLKID_IEC958_GATE]	    = &meson8b_iec958_gate.hw,
+		[CLKID_ENC480P]		    = &meson8b_enc480p.hw,
+		[CLKID_RNG1]		    = &meson8b_rng1.hw,
+		[CLKID_GCLK_VENCL_INT]	    = &meson8b_gclk_vencl_int.hw,
+		[CLKID_VCLK2_VENCLMCC]	    = &meson8b_vclk2_venclmcc.hw,
+		[CLKID_VCLK2_VENCL]	    = &meson8b_vclk2_vencl.hw,
+		[CLKID_VCLK2_OTHER]	    = &meson8b_vclk2_other.hw,
+		[CLKID_EDP]		    = &meson8b_edp.hw,
+		[CLKID_AO_MEDIA_CPU]	    = &meson8b_ao_media_cpu.hw,
+		[CLKID_AO_AHB_SRAM]	    = &meson8b_ao_ahb_sram.hw,
+		[CLKID_AO_AHB_BUS]	    = &meson8b_ao_ahb_bus.hw,
+		[CLKID_AO_IFACE]	    = &meson8b_ao_iface.hw,
+		[CLKID_MPLL0]		    = &meson8b_mpll0.hw,
+		[CLKID_MPLL1]		    = &meson8b_mpll1.hw,
+		[CLKID_MPLL2]		    = &meson8b_mpll2.hw,
+		[CLKID_MPLL0_DIV]	    = &meson8b_mpll0_div.hw,
+		[CLKID_MPLL1_DIV]	    = &meson8b_mpll1_div.hw,
+		[CLKID_MPLL2_DIV]	    = &meson8b_mpll2_div.hw,
+		[CLKID_CPU_IN_SEL]	    = &meson8b_cpu_in_sel.hw,
+		[CLKID_CPU_IN_DIV2]	    = &meson8b_cpu_in_div2.hw,
+		[CLKID_CPU_IN_DIV3]	    = &meson8b_cpu_in_div3.hw,
+		[CLKID_CPU_SCALE_DIV]	    = &meson8b_cpu_scale_div.hw,
+		[CLKID_CPU_SCALE_OUT_SEL]   = &meson8b_cpu_scale_out_sel.hw,
+		[CLKID_MPLL_PREDIV]	    = &meson8b_mpll_prediv.hw,
+		[CLKID_FCLK_DIV2_DIV]	    = &meson8b_fclk_div2_div.hw,
+		[CLKID_FCLK_DIV3_DIV]	    = &meson8b_fclk_div3_div.hw,
+		[CLKID_FCLK_DIV4_DIV]	    = &meson8b_fclk_div4_div.hw,
+		[CLKID_FCLK_DIV5_DIV]	    = &meson8b_fclk_div5_div.hw,
+		[CLKID_FCLK_DIV7_DIV]	    = &meson8b_fclk_div7_div.hw,
+		[CLKID_NAND_SEL]	    = &meson8b_nand_clk_sel.hw,
+		[CLKID_NAND_DIV]	    = &meson8b_nand_clk_div.hw,
+		[CLKID_NAND_CLK]	    = &meson8b_nand_clk_gate.hw,
+		[CLKID_PLL_FIXED_DCO]	    = &meson8b_fixed_pll_dco.hw,
+		[CLKID_HDMI_PLL_DCO]	    = &meson8b_hdmi_pll_dco.hw,
+		[CLKID_PLL_SYS_DCO]	    = &meson8b_sys_pll_dco.hw,
+		[CLKID_CPU_CLK_DIV2]	    = &meson8b_cpu_clk_div2.hw,
+		[CLKID_CPU_CLK_DIV3]	    = &meson8b_cpu_clk_div3.hw,
+		[CLKID_CPU_CLK_DIV4]	    = &meson8b_cpu_clk_div4.hw,
+		[CLKID_CPU_CLK_DIV5]	    = &meson8b_cpu_clk_div5.hw,
+		[CLKID_CPU_CLK_DIV6]	    = &meson8b_cpu_clk_div6.hw,
+		[CLKID_CPU_CLK_DIV7]	    = &meson8b_cpu_clk_div7.hw,
+		[CLKID_CPU_CLK_DIV8]	    = &meson8b_cpu_clk_div8.hw,
+		[CLKID_APB_SEL]		    = &meson8b_apb_clk_sel.hw,
+		[CLKID_APB]		    = &meson8b_apb_clk_gate.hw,
+		[CLKID_PERIPH_SEL]	    = &meson8b_periph_clk_sel.hw,
+		[CLKID_PERIPH]		    = &meson8b_periph_clk_gate.hw,
+		[CLKID_AXI_SEL]		    = &meson8b_axi_clk_sel.hw,
+		[CLKID_AXI]		    = &meson8b_axi_clk_gate.hw,
+		[CLKID_L2_DRAM_SEL]	    = &meson8b_l2_dram_clk_sel.hw,
+		[CLKID_L2_DRAM]		    = &meson8b_l2_dram_clk_gate.hw,
+		[CLKID_HDMI_PLL_LVDS_OUT]   = &meson8b_hdmi_pll_lvds_out.hw,
+		[CLKID_HDMI_PLL_HDMI_OUT]   = &meson8b_hdmi_pll_hdmi_out.hw,
+		[CLKID_VID_PLL_IN_SEL]	    = &meson8b_vid_pll_in_sel.hw,
+		[CLKID_VID_PLL_IN_EN]	    = &meson8b_vid_pll_in_en.hw,
+		[CLKID_VID_PLL_PRE_DIV]	    = &meson8b_vid_pll_pre_div.hw,
+		[CLKID_VID_PLL_POST_DIV]    = &meson8b_vid_pll_post_div.hw,
+		[CLKID_VID_PLL_FINAL_DIV]   = &meson8b_vid_pll_final_div.hw,
+		[CLKID_VCLK_IN_SEL]	    = &meson8b_vclk_in_sel.hw,
+		[CLKID_VCLK_IN_EN]	    = &meson8b_vclk_in_en.hw,
+		[CLKID_VCLK_DIV1]	    = &meson8b_vclk_div1_gate.hw,
+		[CLKID_VCLK_DIV2_DIV]	    = &meson8b_vclk_div2_div.hw,
+		[CLKID_VCLK_DIV2]	    = &meson8b_vclk_div2_div_gate.hw,
+		[CLKID_VCLK_DIV4_DIV]	    = &meson8b_vclk_div4_div.hw,
+		[CLKID_VCLK_DIV4]	    = &meson8b_vclk_div4_div_gate.hw,
+		[CLKID_VCLK_DIV6_DIV]	    = &meson8b_vclk_div6_div.hw,
+		[CLKID_VCLK_DIV6]	    = &meson8b_vclk_div6_div_gate.hw,
+		[CLKID_VCLK_DIV12_DIV]	    = &meson8b_vclk_div12_div.hw,
+		[CLKID_VCLK_DIV12]	    = &meson8b_vclk_div12_div_gate.hw,
+		[CLKID_VCLK2_IN_SEL]	    = &meson8b_vclk2_in_sel.hw,
+		[CLKID_VCLK2_IN_EN]	    = &meson8b_vclk2_clk_in_en.hw,
+		[CLKID_VCLK2_DIV1]	    = &meson8b_vclk2_div1_gate.hw,
+		[CLKID_VCLK2_DIV2_DIV]	    = &meson8b_vclk2_div2_div.hw,
+		[CLKID_VCLK2_DIV2]	    = &meson8b_vclk2_div2_div_gate.hw,
+		[CLKID_VCLK2_DIV4_DIV]	    = &meson8b_vclk2_div4_div.hw,
+		[CLKID_VCLK2_DIV4]	    = &meson8b_vclk2_div4_div_gate.hw,
+		[CLKID_VCLK2_DIV6_DIV]	    = &meson8b_vclk2_div6_div.hw,
+		[CLKID_VCLK2_DIV6]	    = &meson8b_vclk2_div6_div_gate.hw,
+		[CLKID_VCLK2_DIV12_DIV]	    = &meson8b_vclk2_div12_div.hw,
+		[CLKID_VCLK2_DIV12]	    = &meson8b_vclk2_div12_div_gate.hw,
+		[CLKID_CTS_ENCT_SEL]	    = &meson8b_cts_enct_sel.hw,
+		[CLKID_CTS_ENCT]	    = &meson8b_cts_enct.hw,
+		[CLKID_CTS_ENCP_SEL]	    = &meson8b_cts_encp_sel.hw,
+		[CLKID_CTS_ENCP]	    = &meson8b_cts_encp.hw,
+		[CLKID_CTS_ENCI_SEL]	    = &meson8b_cts_enci_sel.hw,
+		[CLKID_CTS_ENCI]	    = &meson8b_cts_enci.hw,
+		[CLKID_HDMI_TX_PIXEL_SEL]   = &meson8b_hdmi_tx_pixel_sel.hw,
+		[CLKID_HDMI_TX_PIXEL]	    = &meson8b_hdmi_tx_pixel.hw,
+		[CLKID_CTS_ENCL_SEL]	    = &meson8b_cts_encl_sel.hw,
+		[CLKID_CTS_ENCL]	    = &meson8b_cts_encl.hw,
+		[CLKID_CTS_VDAC0_SEL]	    = &meson8b_cts_vdac0_sel.hw,
+		[CLKID_CTS_VDAC0]	    = &meson8b_cts_vdac0.hw,
+		[CLKID_HDMI_SYS_SEL]	    = &meson8b_hdmi_sys_sel.hw,
+		[CLKID_HDMI_SYS_DIV]	    = &meson8b_hdmi_sys_div.hw,
+		[CLKID_HDMI_SYS]	    = &meson8b_hdmi_sys.hw,
+		[CLKID_MALI_0_SEL]	    = &meson8b_mali_0_sel.hw,
+		[CLKID_MALI_0_DIV]	    = &meson8b_mali_0_div.hw,
+		[CLKID_MALI_0]		    = &meson8b_mali_0.hw,
+		[CLKID_MALI_1_SEL]	    = &meson8b_mali_1_sel.hw,
+		[CLKID_MALI_1_DIV]	    = &meson8b_mali_1_div.hw,
+		[CLKID_MALI_1]		    = &meson8b_mali_1.hw,
+		[CLKID_MALI]		    = &meson8b_mali.hw,
+		[CLKID_GP_PLL_DCO]	    = &meson8m2_gp_pll_dco.hw,
+		[CLKID_GP_PLL]		    = &meson8m2_gp_pll.hw,
+		[CLKID_VPU_0_SEL]	    = &meson8m2_vpu_0_sel.hw,
+		[CLKID_VPU_0_DIV]	    = &meson8b_vpu_0_div.hw,
+		[CLKID_VPU_0]		    = &meson8b_vpu_0.hw,
+		[CLKID_VPU_1_SEL]	    = &meson8m2_vpu_1_sel.hw,
+		[CLKID_VPU_1_DIV]	    = &meson8b_vpu_1_div.hw,
+		[CLKID_VPU_1]		    = &meson8b_vpu_1.hw,
+		[CLKID_VPU]		    = &meson8b_vpu.hw,
+		[CLKID_VDEC_1_SEL]	    = &meson8b_vdec_1_sel.hw,
+		[CLKID_VDEC_1_1_DIV]	    = &meson8b_vdec_1_1_div.hw,
+		[CLKID_VDEC_1_1]	    = &meson8b_vdec_1_1.hw,
+		[CLKID_VDEC_1_2_DIV]	    = &meson8b_vdec_1_2_div.hw,
+		[CLKID_VDEC_1_2]	    = &meson8b_vdec_1_2.hw,
+		[CLKID_VDEC_1]	    	    = &meson8b_vdec_1.hw,
+		[CLKID_VDEC_HCODEC_SEL]	    = &meson8b_vdec_hcodec_sel.hw,
+		[CLKID_VDEC_HCODEC_DIV]	    = &meson8b_vdec_hcodec_div.hw,
+		[CLKID_VDEC_HCODEC]	    = &meson8b_vdec_hcodec.hw,
+		[CLKID_VDEC_2_SEL]	    = &meson8b_vdec_2_sel.hw,
+		[CLKID_VDEC_2_DIV]	    = &meson8b_vdec_2_div.hw,
+		[CLKID_VDEC_2]	    	    = &meson8b_vdec_2.hw,
+		[CLKID_VDEC_HEVC_SEL]	    = &meson8b_vdec_hevc_sel.hw,
+		[CLKID_VDEC_HEVC_DIV]	    = &meson8b_vdec_hevc_div.hw,
+		[CLKID_VDEC_HEVC_EN]	    = &meson8b_vdec_hevc_en.hw,
+		[CLKID_VDEC_HEVC]	    = &meson8b_vdec_hevc.hw,
 		[CLK_NR_CLKS]		    = NULL,
 	},
 	.num = CLK_NR_CLKS,
@@ -2314,6 +3014,33 @@ static struct clk_regmap *const meson8b_clk_regmaps[] = {
 	&meson8b_mali_1_div,
 	&meson8b_mali_1,
 	&meson8b_mali,
+	&meson8m2_gp_pll_dco,
+	&meson8m2_gp_pll,
+	&meson8b_vpu_0_sel,
+	&meson8m2_vpu_0_sel,
+	&meson8b_vpu_0_div,
+	&meson8b_vpu_0,
+	&meson8b_vpu_1_sel,
+	&meson8m2_vpu_1_sel,
+	&meson8b_vpu_1_div,
+	&meson8b_vpu_1,
+	&meson8b_vpu,
+	&meson8b_vdec_1_sel,
+	&meson8b_vdec_1_1_div,
+	&meson8b_vdec_1_1,
+	&meson8b_vdec_1_2_div,
+	&meson8b_vdec_1_2,
+	&meson8b_vdec_1,
+	&meson8b_vdec_hcodec_sel,
+	&meson8b_vdec_hcodec_div,
+	&meson8b_vdec_hcodec,
+	&meson8b_vdec_2_sel,
+	&meson8b_vdec_2_div,
+	&meson8b_vdec_2,
+	&meson8b_vdec_hevc_sel,
+	&meson8b_vdec_hevc_div,
+	&meson8b_vdec_hevc_en,
+	&meson8b_vdec_hevc,
 };
 
 static const struct meson8b_clk_reset_line {
@@ -2558,9 +3285,14 @@ static void __init meson8b_clkc_init(struct device_node *np)
 	return meson8b_clkc_init_common(np, &meson8b_hw_onecell_data);
 }
 
+static void __init meson8m2_clkc_init(struct device_node *np)
+{
+	return meson8b_clkc_init_common(np, &meson8m2_hw_onecell_data);
+}
+
 CLK_OF_DECLARE_DRIVER(meson8_clkc, "amlogic,meson8-clkc",
 		      meson8_clkc_init);
 CLK_OF_DECLARE_DRIVER(meson8b_clkc, "amlogic,meson8b-clkc",
 		      meson8b_clkc_init);
 CLK_OF_DECLARE_DRIVER(meson8m2_clkc, "amlogic,meson8m2-clkc",
-		      meson8b_clkc_init);
+		      meson8m2_clkc_init);
diff --git a/drivers/clk/meson/meson8b.h b/drivers/clk/meson/meson8b.h
index b8c58faeae52..ed37196187e6 100644
--- a/drivers/clk/meson/meson8b.h
+++ b/drivers/clk/meson/meson8b.h
@@ -19,6 +19,7 @@
  *
  * [0] http://dn.odroid.com/S805/Datasheet/S805_Datasheet%20V0.8%2020150126.pdf
  */
+#define HHI_GP_PLL_CNTL			0x40  /* 0x10 offset in data sheet */
 #define HHI_VIID_CLK_DIV		0x128 /* 0x4a offset in data sheet */
 #define HHI_VIID_CLK_CNTL		0x12c /* 0x4b offset in data sheet */
 #define HHI_GCLK_MPEG0			0x140 /* 0x50 offset in data sheet */
@@ -34,7 +35,11 @@
 #define HHI_VID_DIVIDER_CNTL		0x198 /* 0x66 offset in data sheet */
 #define HHI_SYS_CPU_CLK_CNTL0		0x19c /* 0x67 offset in data sheet */
 #define HHI_MALI_CLK_CNTL		0x1b0 /* 0x6c offset in data sheet */
+#define HHI_VPU_CLK_CNTL		0x1bc /* 0x6f offset in data sheet */
 #define HHI_HDMI_CLK_CNTL		0x1cc /* 0x73 offset in data sheet */
+#define HHI_VDEC_CLK_CNTL		0x1e0 /* 0x78 offset in data sheet */
+#define HHI_VDEC2_CLK_CNTL		0x1e4 /* 0x79 offset in data sheet */
+#define HHI_VDEC3_CLK_CNTL		0x1e8 /* 0x7a offset in data sheet */
 #define HHI_NAND_CLK_CNTL		0x25c /* 0x97 offset in data sheet */
 #define HHI_MPLL_CNTL			0x280 /* 0xa0 offset in data sheet */
 #define HHI_SYS_PLL_CNTL		0x300 /* 0xc0 offset in data sheet */
@@ -146,8 +151,28 @@
 #define CLKID_MALI_1_SEL	178
 #define CLKID_MALI_1_DIV	179
 #define CLKID_MALI_1		180
+#define CLKID_GP_PLL_DCO	181
+#define CLKID_GP_PLL		182
+#define CLKID_VPU_0_SEL		183
+#define CLKID_VPU_0_DIV		184
+#define CLKID_VPU_0		185
+#define CLKID_VPU_1_SEL		186
+#define CLKID_VPU_1_DIV		187
+#define CLKID_VPU_1		189
+#define CLKID_VDEC_1_SEL	191
+#define CLKID_VDEC_1_1_DIV	192
+#define CLKID_VDEC_1_1		193
+#define CLKID_VDEC_1_2_DIV	194
+#define CLKID_VDEC_1_2		195
+#define CLKID_VDEC_HCODEC_SEL	197
+#define CLKID_VDEC_HCODEC_DIV	198
+#define CLKID_VDEC_2_SEL	200
+#define CLKID_VDEC_2_DIV	201
+#define CLKID_VDEC_HEVC_SEL	203
+#define CLKID_VDEC_HEVC_DIV	204
+#define CLKID_VDEC_HEVC_EN	205
 
-#define CLK_NR_CLKS		181
+#define CLK_NR_CLKS		207
 
 /*
  * include the CLKID and RESETID that have
diff --git a/drivers/clk/mmp/clk-gate.c b/drivers/clk/mmp/clk-gate.c
index 7355595c42e2..1755916ddef2 100644
--- a/drivers/clk/mmp/clk-gate.c
+++ b/drivers/clk/mmp/clk-gate.c
@@ -108,7 +108,7 @@ struct clk *mmp_clk_register_gate(struct device *dev, const char *name,
 
 	init.name = name;
 	init.ops = &mmp_clk_gate_ops;
-	init.flags = flags | CLK_IS_BASIC;
+	init.flags = flags;
 	init.parent_names = (parent_name ? &parent_name : NULL);
 	init.num_parents = (parent_name ? 1 : 0);
 
diff --git a/drivers/clk/mvebu/common.c b/drivers/clk/mvebu/common.c
index 6ab3c2e627c7..785dbede4835 100644
--- a/drivers/clk/mvebu/common.c
+++ b/drivers/clk/mvebu/common.c
@@ -240,7 +240,7 @@ void __init mvebu_clk_gating_setup(struct device_node *np,
 	int n;
 
 	if (ctrl) {
-		pr_err("mvebu-clk-gating: cannot instantiate more than one gatable clock device\n");
+		pr_err("mvebu-clk-gating: cannot instantiate more than one gateable clock device\n");
 		return;
 	}
 
diff --git a/drivers/clk/mvebu/cp110-system-controller.c b/drivers/clk/mvebu/cp110-system-controller.c
index 9235a331b588..b6de283f45e3 100644
--- a/drivers/clk/mvebu/cp110-system-controller.c
+++ b/drivers/clk/mvebu/cp110-system-controller.c
@@ -21,7 +21,7 @@
  *    - Equal to SDIO clock
  *    - 2/5 PLL0
  *
- * CP110 has 32 gatable clocks, for the various peripherals in the IP.
+ * CP110 has 32 gateable clocks, for the various peripherals in the IP.
  */
 
 #define pr_fmt(fmt) "cp110-system-controller: " fmt
@@ -57,7 +57,7 @@ enum {
 #define CP110_CORE_NAND			4
 #define CP110_CORE_SDIO			5
 
-/* A number of gatable clocks need special handling */
+/* A number of gateable clocks need special handling */
 #define CP110_GATE_AUDIO		0
 #define CP110_GATE_COMM_UNIT		1
 #define CP110_GATE_NAND			2
diff --git a/drivers/clk/nxp/clk-lpc18xx-ccu.c b/drivers/clk/nxp/clk-lpc18xx-ccu.c
index 27781b49eb82..5969f620607a 100644
--- a/drivers/clk/nxp/clk-lpc18xx-ccu.c
+++ b/drivers/clk/nxp/clk-lpc18xx-ccu.c
@@ -142,7 +142,7 @@ static int lpc18xx_ccu_gate_endisable(struct clk_hw *hw, bool enable)
 	 * Divider field is write only, so divider stat field must
 	 * be read so divider field can be set accordingly.
 	 */
-	val = clk_readl(gate->reg);
+	val = readl(gate->reg);
 	if (val & LPC18XX_CCU_DIVSTAT)
 		val |= LPC18XX_CCU_DIV;
 
@@ -155,12 +155,12 @@ static int lpc18xx_ccu_gate_endisable(struct clk_hw *hw, bool enable)
 		 * and the next write should clear the RUN bit.
 		 */
 		val |= LPC18XX_CCU_AUTO;
-		clk_writel(val, gate->reg);
+		writel(val, gate->reg);
 
 		val &= ~LPC18XX_CCU_RUN;
 	}
 
-	clk_writel(val, gate->reg);
+	writel(val, gate->reg);
 
 	return 0;
 }
diff --git a/drivers/clk/nxp/clk-lpc18xx-cgu.c b/drivers/clk/nxp/clk-lpc18xx-cgu.c
index 2531174b399e..f5bc8bd192b7 100644
--- a/drivers/clk/nxp/clk-lpc18xx-cgu.c
+++ b/drivers/clk/nxp/clk-lpc18xx-cgu.c
@@ -352,9 +352,9 @@ static unsigned long lpc18xx_pll0_recalc_rate(struct clk_hw *hw,
 	struct lpc18xx_pll *pll = to_lpc_pll(hw);
 	u32 ctrl, mdiv, msel, npdiv;
 
-	ctrl = clk_readl(pll->reg + LPC18XX_CGU_PLL0USB_CTRL);
-	mdiv = clk_readl(pll->reg + LPC18XX_CGU_PLL0USB_MDIV);
-	npdiv = clk_readl(pll->reg + LPC18XX_CGU_PLL0USB_NP_DIV);
+	ctrl = readl(pll->reg + LPC18XX_CGU_PLL0USB_CTRL);
+	mdiv = readl(pll->reg + LPC18XX_CGU_PLL0USB_MDIV);
+	npdiv = readl(pll->reg + LPC18XX_CGU_PLL0USB_NP_DIV);
 
 	if (ctrl & LPC18XX_PLL0_CTRL_BYPASS)
 		return parent_rate;
@@ -415,25 +415,25 @@ static int lpc18xx_pll0_set_rate(struct clk_hw *hw, unsigned long rate,
 	m |= lpc18xx_pll0_msel2seli(m) << LPC18XX_PLL0_MDIV_SELI_SHIFT;
 
 	/* Power down PLL, disable clk output and dividers */
-	ctrl = clk_readl(pll->reg + LPC18XX_CGU_PLL0USB_CTRL);
+	ctrl = readl(pll->reg + LPC18XX_CGU_PLL0USB_CTRL);
 	ctrl |= LPC18XX_PLL0_CTRL_PD;
 	ctrl &= ~(LPC18XX_PLL0_CTRL_BYPASS | LPC18XX_PLL0_CTRL_DIRECTI |
 		  LPC18XX_PLL0_CTRL_DIRECTO | LPC18XX_PLL0_CTRL_CLKEN);
-	clk_writel(ctrl, pll->reg + LPC18XX_CGU_PLL0USB_CTRL);
+	writel(ctrl, pll->reg + LPC18XX_CGU_PLL0USB_CTRL);
 
 	/* Configure new PLL settings */
-	clk_writel(m, pll->reg + LPC18XX_CGU_PLL0USB_MDIV);
-	clk_writel(LPC18XX_PLL0_NP_DIVS_1, pll->reg + LPC18XX_CGU_PLL0USB_NP_DIV);
+	writel(m, pll->reg + LPC18XX_CGU_PLL0USB_MDIV);
+	writel(LPC18XX_PLL0_NP_DIVS_1, pll->reg + LPC18XX_CGU_PLL0USB_NP_DIV);
 
 	/* Power up PLL and wait for lock */
 	ctrl &= ~LPC18XX_PLL0_CTRL_PD;
-	clk_writel(ctrl, pll->reg + LPC18XX_CGU_PLL0USB_CTRL);
+	writel(ctrl, pll->reg + LPC18XX_CGU_PLL0USB_CTRL);
 	do {
 		udelay(10);
-		stat = clk_readl(pll->reg + LPC18XX_CGU_PLL0USB_STAT);
+		stat = readl(pll->reg + LPC18XX_CGU_PLL0USB_STAT);
 		if (stat & LPC18XX_PLL0_STAT_LOCK) {
 			ctrl |= LPC18XX_PLL0_CTRL_CLKEN;
-			clk_writel(ctrl, pll->reg + LPC18XX_CGU_PLL0USB_CTRL);
+			writel(ctrl, pll->reg + LPC18XX_CGU_PLL0USB_CTRL);
 
 			return 0;
 		}
@@ -458,8 +458,8 @@ static unsigned long lpc18xx_pll1_recalc_rate(struct clk_hw *hw,
 	bool direct, fbsel;
 	u32 stat, ctrl;
 
-	stat = clk_readl(pll->reg + LPC18XX_CGU_PLL1_STAT);
-	ctrl = clk_readl(pll->reg + LPC18XX_CGU_PLL1_CTRL);
+	stat = readl(pll->reg + LPC18XX_CGU_PLL1_STAT);
+	ctrl = readl(pll->reg + LPC18XX_CGU_PLL1_CTRL);
 
 	direct = (ctrl & LPC18XX_PLL1_CTRL_DIRECT) ? true : false;
 	fbsel = (ctrl & LPC18XX_PLL1_CTRL_FBSEL) ? true : false;
diff --git a/drivers/clk/nxp/clk-lpc32xx.c b/drivers/clk/nxp/clk-lpc32xx.c
index 5eeecee17b69..7524d19fe60b 100644
--- a/drivers/clk/nxp/clk-lpc32xx.c
+++ b/drivers/clk/nxp/clk-lpc32xx.c
@@ -1085,13 +1085,12 @@ struct clk_hw_proto {
 	};
 };
 
-#define LPC32XX_DEFINE_FIXED(_idx, _rate, _flags)			\
+#define LPC32XX_DEFINE_FIXED(_idx, _rate)			\
 [CLK_PREFIX(_idx)] = {							\
 	.type = CLK_FIXED,						\
 	{								\
 		.f = {							\
 			.fixed_rate = (_rate),				\
-			.flags = (_flags),				\
 		},							\
 	},								\
 }
@@ -1225,7 +1224,7 @@ struct clk_hw_proto {
 }
 
 static struct clk_hw_proto clk_hw_proto[LPC32XX_CLK_HW_MAX] = {
-	LPC32XX_DEFINE_FIXED(RTC, 32768, 0),
+	LPC32XX_DEFINE_FIXED(RTC, 32768),
 	LPC32XX_DEFINE_PLL(PLL397X, pll_397x, HCLKPLL_CTRL, BIT(1)),
 	LPC32XX_DEFINE_PLL(HCLK_PLL, hclk_pll, HCLKPLL_CTRL, PLL_CTRL_ENABLE),
 	LPC32XX_DEFINE_PLL(USB_PLL, usb_pll, USB_CTRL, PLL_CTRL_ENABLE),
@@ -1468,7 +1467,7 @@ static struct clk * __init lpc32xx_clk_register(u32 id)
 		struct clk_fixed_rate *fixed = &clk_hw->f;
 
 		clk = clk_register_fixed_rate(NULL, lpc32xx_clk->name,
-			parents[0], fixed->flags, fixed->fixed_rate);
+			parents[0], 0, fixed->fixed_rate);
 		break;
 	}
 	default:
diff --git a/drivers/clk/qcom/Kconfig b/drivers/clk/qcom/Kconfig
index 1c04575c118f..18bdf34d5e64 100644
--- a/drivers/clk/qcom/Kconfig
+++ b/drivers/clk/qcom/Kconfig
@@ -243,6 +243,12 @@ config SDM_GCC_660
 	  Say Y if you want to use peripheral devices such as UART, SPI,
 	  i2C, USB, UFS, SDDC, PCIe, etc.
 
+config QCS_TURING_404
+	tristate "QCS404 Turing Clock Controller"
+	help
+	  Support for the Turing Clock Controller on QCS404, provides clocks
+	  and resets for the Turing subsystem.
+
 config SDM_GCC_845
 	tristate "SDM845 Global Clock Controller"
 	select QCOM_GDSC
diff --git a/drivers/clk/qcom/Makefile b/drivers/clk/qcom/Makefile
index ee8d0698e370..f0768fb1f037 100644
--- a/drivers/clk/qcom/Makefile
+++ b/drivers/clk/qcom/Makefile
@@ -42,6 +42,7 @@ obj-$(CONFIG_QCOM_CLK_RPM) += clk-rpm.o
 obj-$(CONFIG_QCOM_CLK_RPMH) += clk-rpmh.o
 obj-$(CONFIG_QCOM_CLK_SMD_RPM) += clk-smd-rpm.o
 obj-$(CONFIG_QCS_GCC_404) += gcc-qcs404.o
+obj-$(CONFIG_QCS_TURING_404) += turingcc-qcs404.o
 obj-$(CONFIG_SDM_CAMCC_845) += camcc-sdm845.o
 obj-$(CONFIG_SDM_DISPCC_845) += dispcc-sdm845.o
 obj-$(CONFIG_SDM_GCC_660) += gcc-sdm660.o
diff --git a/drivers/clk/qcom/clk-branch.c b/drivers/clk/qcom/clk-branch.c
index 99446bf630aa..f869fc6aaed6 100644
--- a/drivers/clk/qcom/clk-branch.c
+++ b/drivers/clk/qcom/clk-branch.c
@@ -146,6 +146,12 @@ const struct clk_ops clk_branch2_ops = {
 };
 EXPORT_SYMBOL_GPL(clk_branch2_ops);
 
+const struct clk_ops clk_branch2_aon_ops = {
+	.enable = clk_branch2_enable,
+	.is_enabled = clk_is_enabled_regmap,
+};
+EXPORT_SYMBOL_GPL(clk_branch2_aon_ops);
+
 const struct clk_ops clk_branch_simple_ops = {
 	.enable = clk_enable_regmap,
 	.disable = clk_disable_regmap,
diff --git a/drivers/clk/qcom/clk-branch.h b/drivers/clk/qcom/clk-branch.h
index b3561e0a3984..17a58119165e 100644
--- a/drivers/clk/qcom/clk-branch.h
+++ b/drivers/clk/qcom/clk-branch.h
@@ -40,6 +40,7 @@ struct clk_branch {
 extern const struct clk_ops clk_branch_ops;
 extern const struct clk_ops clk_branch2_ops;
 extern const struct clk_ops clk_branch_simple_ops;
+extern const struct clk_ops clk_branch2_aon_ops;
 
 #define to_clk_branch(_hw) \
 	container_of(to_clk_regmap(_hw), struct clk_branch, clkr)
diff --git a/drivers/clk/qcom/clk-regmap-mux-div.h b/drivers/clk/qcom/clk-regmap-mux-div.h
index 6cd6261be7ac..4df6c8d24c24 100644
--- a/drivers/clk/qcom/clk-regmap-mux-div.h
+++ b/drivers/clk/qcom/clk-regmap-mux-div.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (c) 2017, Linaro Limited
  * Author: Georgi Djakov <georgi.djakov@linaro.org>
diff --git a/drivers/clk/qcom/gcc-msm8998.c b/drivers/clk/qcom/gcc-msm8998.c
index c240fba794c7..033688264c7b 100644
--- a/drivers/clk/qcom/gcc-msm8998.c
+++ b/drivers/clk/qcom/gcc-msm8998.c
@@ -2161,7 +2161,7 @@ static struct clk_branch gcc_pcie_0_mstr_axi_clk = {
 
 static struct clk_branch gcc_pcie_0_pipe_clk = {
 	.halt_reg = 0x6b018,
-	.halt_check = BRANCH_HALT,
+	.halt_check = BRANCH_HALT_SKIP,
 	.clkr = {
 		.enable_reg = 0x6b018,
 		.enable_mask = BIT(0),
diff --git a/drivers/clk/qcom/gcc-qcs404.c b/drivers/clk/qcom/gcc-qcs404.c
index 5a62f64ada93..a54807eb3b28 100644
--- a/drivers/clk/qcom/gcc-qcs404.c
+++ b/drivers/clk/qcom/gcc-qcs404.c
@@ -260,6 +260,20 @@ static const char * const gcc_parent_names_15[] = {
 	"core_bi_pll_test_se",
 };
 
+static const struct parent_map gcc_parent_map_16[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0_OUT_MAIN, 1 },
+	{ P_GPLL0_OUT_AUX, 2 },
+	{ P_CORE_BI_PLL_TEST_SE, 7 },
+};
+
+static const char * const gcc_parent_names_16[] = {
+	"cxo",
+	"gpll0_out_main",
+	"gpll0_out_aux",
+	"core_bi_pll_test_se",
+};
+
 static struct clk_fixed_factor cxo = {
 	.mult = 1,
 	.div = 1,
@@ -1194,6 +1208,28 @@ static struct clk_rcg2 vsync_clk_src = {
 	},
 };
 
+static const struct freq_tbl ftbl_cdsp_bimc_clk_src[] = {
+	F(19200000, P_XO, 1, 0, 0),
+	F(133333333, P_GPLL0_OUT_MAIN, 6, 0, 0),
+	F(266666667, P_GPLL0_OUT_MAIN, 3, 0, 0),
+	F(320000000, P_GPLL0_OUT_MAIN, 2.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 cdsp_bimc_clk_src = {
+	.cmd_rcgr = 0x5e010,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_16,
+	.freq_tbl = ftbl_cdsp_bimc_clk_src,
+	.clkr.hw.init = &(struct clk_init_data) {
+		.name = "cdsp_bimc_clk_src",
+		.parent_names = gcc_parent_names_16,
+		.num_parents = 4,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
 static struct clk_branch gcc_apss_ahb_clk = {
 	.halt_reg = 0x4601c,
 	.halt_check = BRANCH_HALT_VOTED,
@@ -1255,6 +1291,24 @@ static struct clk_branch gcc_bimc_gpu_clk = {
 	},
 };
 
+static struct clk_branch gcc_bimc_cdsp_clk = {
+	.halt_reg = 0x31030,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x31030,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data) {
+			.name = "gcc_bimc_cdsp_clk",
+			.parent_names = (const char *[]) {
+				"cdsp_bimc_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
 static struct clk_branch gcc_bimc_mdss_clk = {
 	.halt_reg = 0x31038,
 	.halt_check = BRANCH_HALT,
@@ -1792,6 +1846,24 @@ static struct clk_branch gcc_gfx_tbu_clk = {
 	},
 };
 
+static struct clk_branch gcc_cdsp_tbu_clk = {
+	.halt_reg = 0x1203c,
+	.halt_check = BRANCH_VOTED,
+	.clkr = {
+		.enable_reg = 0x13020,
+		.enable_mask = BIT(9),
+		.hw.init = &(struct clk_init_data) {
+			.name = "gcc_cdsp_tbu_clk",
+			.parent_names = (const char *[]) {
+				"cdsp_bimc_clk_src",
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
 static struct clk_branch gcc_gp1_clk = {
 	.halt_reg = 0x8000,
 	.halt_check = BRANCH_HALT,
@@ -2304,6 +2376,19 @@ static struct clk_branch gcc_sdcc1_ice_core_clk = {
 	},
 };
 
+static struct clk_branch gcc_cdsp_cfg_ahb_clk = {
+	.halt_reg = 0x5e004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x5e004,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data) {
+			.name = "gcc_cdsp_cfg_ahb_cbcr",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
 static struct clk_branch gcc_sdcc2_ahb_clk = {
 	.halt_reg = 0x4301c,
 	.halt_check = BRANCH_HALT,
@@ -2548,6 +2633,7 @@ static struct clk_regmap *gcc_qcs404_clocks[] = {
 	[GCC_ESC0_CLK_SRC] = &esc0_clk_src.clkr,
 	[GCC_APSS_AHB_CLK] = &gcc_apss_ahb_clk.clkr,
 	[GCC_BIMC_GFX_CLK] = &gcc_bimc_gfx_clk.clkr,
+	[GCC_BIMC_CDSP_CLK] = &gcc_bimc_cdsp_clk.clkr,
 	[GCC_BIMC_MDSS_CLK] = &gcc_bimc_mdss_clk.clkr,
 	[GCC_BLSP1_AHB_CLK] = &gcc_blsp1_ahb_clk.clkr,
 	[GCC_BLSP1_QUP0_I2C_APPS_CLK] = &gcc_blsp1_qup0_i2c_apps_clk.clkr,
@@ -2605,6 +2691,7 @@ static struct clk_regmap *gcc_qcs404_clocks[] = {
 	[GCC_SDCC1_AHB_CLK] = &gcc_sdcc1_ahb_clk.clkr,
 	[GCC_SDCC1_APPS_CLK] = &gcc_sdcc1_apps_clk.clkr,
 	[GCC_SDCC1_ICE_CORE_CLK] = &gcc_sdcc1_ice_core_clk.clkr,
+	[GCC_CDSP_CFG_AHB_CLK] = &gcc_cdsp_cfg_ahb_clk.clkr,
 	[GCC_SDCC2_AHB_CLK] = &gcc_sdcc2_ahb_clk.clkr,
 	[GCC_SDCC2_APPS_CLK] = &gcc_sdcc2_apps_clk.clkr,
 	[GCC_SYS_NOC_USB3_CLK] = &gcc_sys_noc_usb3_clk.clkr,
@@ -2645,6 +2732,7 @@ static struct clk_regmap *gcc_qcs404_clocks[] = {
 	[GCC_USB3_PHY_AUX_CLK_SRC] = &usb3_phy_aux_clk_src.clkr,
 	[GCC_USB_HS_SYSTEM_CLK_SRC] = &usb_hs_system_clk_src.clkr,
 	[GCC_VSYNC_CLK_SRC] = &vsync_clk_src.clkr,
+	[GCC_CDSP_BIMC_CLK_SRC] = &cdsp_bimc_clk_src.clkr,
 	[GCC_USB_HS_INACTIVITY_TIMERS_CLK] =
 			&gcc_usb_hs_inactivity_timers_clk.clkr,
 	[GCC_BIMC_GPU_CLK] = &gcc_bimc_gpu_clk.clkr,
@@ -2653,6 +2741,7 @@ static struct clk_regmap *gcc_qcs404_clocks[] = {
 	[GCC_GFX_TBU_CLK] = &gcc_gfx_tbu_clk.clkr,
 	[GCC_SMMU_CFG_CLK] = &gcc_smmu_cfg_clk.clkr,
 	[GCC_APSS_TCU_CLK] = &gcc_apss_tcu_clk.clkr,
+	[GCC_CDSP_TBU_CLK] = &gcc_cdsp_tbu_clk.clkr,
 	[GCC_CRYPTO_AHB_CLK] = &gcc_crypto_ahb_clk.clkr,
 	[GCC_CRYPTO_AXI_CLK] = &gcc_crypto_axi_clk.clkr,
 	[GCC_CRYPTO_CLK] = &gcc_crypto_clk.clkr,
@@ -2664,6 +2753,7 @@ static struct clk_regmap *gcc_qcs404_clocks[] = {
 
 static const struct qcom_reset_map gcc_qcs404_resets[] = {
 	[GCC_GENI_IR_BCR] = { 0x0F000 },
+	[GCC_CDSP_RESTART] = { 0x18000 },
 	[GCC_USB_HS_BCR] = { 0x41000 },
 	[GCC_USB2_HS_PHY_ONLY_BCR] = { 0x41034 },
 	[GCC_QUSB2_PHY_BCR] = { 0x4103c },
diff --git a/drivers/clk/qcom/turingcc-qcs404.c b/drivers/clk/qcom/turingcc-qcs404.c
new file mode 100644
index 000000000000..aa859e6ec9bd
--- /dev/null
+++ b/drivers/clk/qcom/turingcc-qcs404.c
@@ -0,0 +1,170 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2019, Linaro Ltd.
+ */
+
+#include <linux/bitops.h>
+#include <linux/err.h>
+#include <linux/platform_device.h>
+#include <linux/module.h>
+#include <linux/of_address.h>
+#include <linux/pm_clock.h>
+#include <linux/pm_runtime.h>
+#include <linux/regmap.h>
+
+#include <dt-bindings/clock/qcom,turingcc-qcs404.h>
+
+#include "clk-regmap.h"
+#include "clk-branch.h"
+#include "common.h"
+#include "reset.h"
+
+static struct clk_branch turing_wrapper_aon_cbcr = {
+	.halt_reg = 0x5098,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x5098,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data) {
+			.name = "turing_wrapper_aon_clk",
+			.ops = &clk_branch2_aon_ops,
+		},
+	},
+};
+
+static struct clk_branch turing_q6ss_ahbm_aon_cbcr = {
+	.halt_reg = 0x9000,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x9000,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data) {
+			.name = "turing_q6ss_ahbm_aon_cbcr",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch turing_q6ss_q6_axim_clk = {
+	.halt_reg = 0xb000,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0xb000,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data) {
+			.name = "turing_q6ss_q6_axim_clk",
+			.ops = &clk_branch2_aon_ops,
+		},
+	},
+};
+
+static struct clk_branch turing_q6ss_ahbs_aon_cbcr = {
+	.halt_reg = 0x10000,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x10000,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data) {
+			.name = "turing_q6ss_ahbs_aon_clk",
+			.ops = &clk_branch2_aon_ops,
+		},
+	},
+};
+
+static struct clk_branch turing_wrapper_qos_ahbs_aon_cbcr = {
+	.halt_reg = 0x11014,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x11014,
+		.enable_mask = BIT(0),
+		.hw.init = &(struct clk_init_data) {
+			.name = "turing_wrapper_qos_ahbs_aon_clk",
+			.ops = &clk_branch2_aon_ops,
+		},
+	},
+};
+
+static struct clk_regmap *turingcc_clocks[] = {
+	[TURING_WRAPPER_AON_CLK] = &turing_wrapper_aon_cbcr.clkr,
+	[TURING_Q6SS_AHBM_AON_CLK] = &turing_q6ss_ahbm_aon_cbcr.clkr,
+	[TURING_Q6SS_Q6_AXIM_CLK] = &turing_q6ss_q6_axim_clk.clkr,
+	[TURING_Q6SS_AHBS_AON_CLK] = &turing_q6ss_ahbs_aon_cbcr.clkr,
+	[TURING_WRAPPER_QOS_AHBS_AON_CLK] = &turing_wrapper_qos_ahbs_aon_cbcr.clkr,
+};
+
+static const struct regmap_config turingcc_regmap_config = {
+	.reg_bits	= 32,
+	.reg_stride	= 4,
+	.val_bits	= 32,
+	.max_register	= 0x30000,
+	.fast_io	= true,
+};
+
+static const struct qcom_cc_desc turingcc_desc = {
+	.config = &turingcc_regmap_config,
+	.clks = turingcc_clocks,
+	.num_clks = ARRAY_SIZE(turingcc_clocks),
+};
+
+static int turingcc_probe(struct platform_device *pdev)
+{
+	int ret;
+
+	pm_runtime_enable(&pdev->dev);
+	ret = pm_clk_create(&pdev->dev);
+	if (ret)
+		goto disable_pm_runtime;
+
+	ret = pm_clk_add(&pdev->dev, NULL);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "failed to acquire iface clock\n");
+		goto destroy_pm_clk;
+	}
+
+	ret = qcom_cc_probe(pdev, &turingcc_desc);
+	if (ret < 0)
+		goto destroy_pm_clk;
+
+	return 0;
+
+destroy_pm_clk:
+	pm_clk_destroy(&pdev->dev);
+
+disable_pm_runtime:
+	pm_runtime_disable(&pdev->dev);
+
+	return ret;
+}
+
+static int turingcc_remove(struct platform_device *pdev)
+{
+	pm_clk_destroy(&pdev->dev);
+	pm_runtime_disable(&pdev->dev);
+
+	return 0;
+}
+
+static const struct dev_pm_ops turingcc_pm_ops = {
+	SET_RUNTIME_PM_OPS(pm_clk_suspend, pm_clk_resume, NULL)
+};
+
+static const struct of_device_id turingcc_match_table[] = {
+	{ .compatible = "qcom,qcs404-turingcc" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, turingcc_match_table);
+
+static struct platform_driver turingcc_driver = {
+	.probe		= turingcc_probe,
+	.remove		= turingcc_remove,
+	.driver		= {
+		.name	= "qcs404-turingcc",
+		.of_match_table = turingcc_match_table,
+		.pm = &turingcc_pm_ops,
+	},
+};
+
+module_platform_driver(turingcc_driver);
+
+MODULE_DESCRIPTION("Qualcomm QCS404 Turing Clock Controller");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/clk/renesas/r7s9210-cpg-mssr.c b/drivers/clk/renesas/r7s9210-cpg-mssr.c
index 57c49fe88295..cf65d4e0e116 100644
--- a/drivers/clk/renesas/r7s9210-cpg-mssr.c
+++ b/drivers/clk/renesas/r7s9210-cpg-mssr.c
@@ -11,6 +11,7 @@
 
 #include <linux/clk.h>
 #include <linux/clk-provider.h>
+#include <linux/io.h>
 #include <dt-bindings/clock/r7s9210-cpg-mssr.h>
 #include "renesas-cpg-mssr.h"
 
@@ -119,7 +120,7 @@ static void __init r7s9210_update_clk_table(struct clk *extal_clk,
 	if (clk_get_rate(extal_clk) > 12000000)
 		cpg_mode = 1;
 
-	frqcr = clk_readl(base + CPG_FRQCR) & 0xFFF;
+	frqcr = readl(base + CPG_FRQCR) & 0xFFF;
 	if (frqcr == 0x012)
 		index = 0;
 	else if (frqcr == 0x112)
diff --git a/drivers/clk/renesas/r8a774a1-cpg-mssr.c b/drivers/clk/renesas/r8a774a1-cpg-mssr.c
index 4d92b27a6153..76ed7d1bae36 100644
--- a/drivers/clk/renesas/r8a774a1-cpg-mssr.c
+++ b/drivers/clk/renesas/r8a774a1-cpg-mssr.c
@@ -71,8 +71,8 @@ static const struct cpg_core_clk r8a774a1_core_clks[] __initconst = {
 	DEF_GEN3_OSC(".r",      CLK_RINT,          CLK_EXTAL,      32),
 
 	/* Core Clock Outputs */
-	DEF_BASE("z",           R8A774A1_CLK_Z,     CLK_TYPE_GEN3_Z, CLK_PLL0),
-	DEF_BASE("z2",          R8A774A1_CLK_Z2,    CLK_TYPE_GEN3_Z2, CLK_PLL2),
+	DEF_GEN3_Z("z",		R8A774A1_CLK_Z,     CLK_TYPE_GEN3_Z,  CLK_PLL0, 2, 8),
+	DEF_GEN3_Z("z2",	R8A774A1_CLK_Z2,    CLK_TYPE_GEN3_Z,  CLK_PLL2, 2, 0),
 	DEF_FIXED("ztr",        R8A774A1_CLK_ZTR,   CLK_PLL1_DIV2,  6, 1),
 	DEF_FIXED("ztrd2",      R8A774A1_CLK_ZTRD2, CLK_PLL1_DIV2, 12, 1),
 	DEF_FIXED("zt",         R8A774A1_CLK_ZT,    CLK_PLL1_DIV2,  4, 1),
@@ -123,8 +123,8 @@ static const struct mssr_mod_clk r8a774a1_mod_clks[] __initconst = {
 	DEF_MOD("msiof2",		 209,	R8A774A1_CLK_MSO),
 	DEF_MOD("msiof1",		 210,	R8A774A1_CLK_MSO),
 	DEF_MOD("msiof0",		 211,	R8A774A1_CLK_MSO),
-	DEF_MOD("sys-dmac2",		 217,	R8A774A1_CLK_S0D3),
-	DEF_MOD("sys-dmac1",		 218,	R8A774A1_CLK_S0D3),
+	DEF_MOD("sys-dmac2",		 217,	R8A774A1_CLK_S3D1),
+	DEF_MOD("sys-dmac1",		 218,	R8A774A1_CLK_S3D1),
 	DEF_MOD("sys-dmac0",		 219,	R8A774A1_CLK_S0D3),
 	DEF_MOD("cmt3",			 300,	R8A774A1_CLK_R),
 	DEF_MOD("cmt2",			 301,	R8A774A1_CLK_R),
@@ -143,8 +143,8 @@ static const struct mssr_mod_clk r8a774a1_mod_clks[] __initconst = {
 	DEF_MOD("rwdt",			 402,	R8A774A1_CLK_R),
 	DEF_MOD("intc-ex",		 407,	R8A774A1_CLK_CP),
 	DEF_MOD("intc-ap",		 408,	R8A774A1_CLK_S0D3),
-	DEF_MOD("audmac1",		 501,	R8A774A1_CLK_S0D3),
-	DEF_MOD("audmac0",		 502,	R8A774A1_CLK_S0D3),
+	DEF_MOD("audmac1",		 501,	R8A774A1_CLK_S1D2),
+	DEF_MOD("audmac0",		 502,	R8A774A1_CLK_S1D2),
 	DEF_MOD("hscif4",		 516,	R8A774A1_CLK_S3D1),
 	DEF_MOD("hscif3",		 517,	R8A774A1_CLK_S3D1),
 	DEF_MOD("hscif2",		 518,	R8A774A1_CLK_S3D1),
@@ -165,9 +165,9 @@ static const struct mssr_mod_clk r8a774a1_mod_clks[] __initconst = {
 	DEF_MOD("vspd0",		 623,	R8A774A1_CLK_S0D2),
 	DEF_MOD("vspb",			 626,	R8A774A1_CLK_S0D1),
 	DEF_MOD("vspi0",		 631,	R8A774A1_CLK_S0D1),
-	DEF_MOD("ehci1",		 702,	R8A774A1_CLK_S3D4),
-	DEF_MOD("ehci0",		 703,	R8A774A1_CLK_S3D4),
-	DEF_MOD("hsusb",		 704,	R8A774A1_CLK_S3D4),
+	DEF_MOD("ehci1",		 702,	R8A774A1_CLK_S3D2),
+	DEF_MOD("ehci0",		 703,	R8A774A1_CLK_S3D2),
+	DEF_MOD("hsusb",		 704,	R8A774A1_CLK_S3D2),
 	DEF_MOD("csi20",		 714,	R8A774A1_CLK_CSI0),
 	DEF_MOD("csi40",		 716,	R8A774A1_CLK_CSI0),
 	DEF_MOD("du2",			 722,	R8A774A1_CLK_S2D1),
diff --git a/drivers/clk/renesas/r8a774c0-cpg-mssr.c b/drivers/clk/renesas/r8a774c0-cpg-mssr.c
index 34e274f2a273..f91e7a484753 100644
--- a/drivers/clk/renesas/r8a774c0-cpg-mssr.c
+++ b/drivers/clk/renesas/r8a774c0-cpg-mssr.c
@@ -81,6 +81,7 @@ static const struct cpg_core_clk r8a774c0_core_clks[] __initconst = {
 	/* Core Clock Outputs */
 	DEF_FIXED("za2",       R8A774C0_CLK_ZA2,   CLK_PLL0D24,    1, 1),
 	DEF_FIXED("za8",       R8A774C0_CLK_ZA8,   CLK_PLL0D8,     1, 1),
+	DEF_GEN3_Z("z2",       R8A774C0_CLK_Z2,    CLK_TYPE_GEN3_Z, CLK_PLL0, 4, 8),
 	DEF_FIXED("ztr",       R8A774C0_CLK_ZTR,   CLK_PLL1,       6, 1),
 	DEF_FIXED("zt",        R8A774C0_CLK_ZT,    CLK_PLL1,       4, 1),
 	DEF_FIXED("zx",        R8A774C0_CLK_ZX,    CLK_PLL1,       3, 1),
@@ -157,7 +158,7 @@ static const struct mssr_mod_clk r8a774c0_mod_clks[] __initconst = {
 	DEF_MOD("intc-ex",		 407,	R8A774C0_CLK_CP),
 	DEF_MOD("intc-ap",		 408,	R8A774C0_CLK_S0D3),
 
-	DEF_MOD("audmac0",		 502,	R8A774C0_CLK_S3D4),
+	DEF_MOD("audmac0",		 502,	R8A774C0_CLK_S1D2),
 	DEF_MOD("hscif4",		 516,	R8A774C0_CLK_S3D1C),
 	DEF_MOD("hscif3",		 517,	R8A774C0_CLK_S3D1C),
 	DEF_MOD("hscif2",		 518,	R8A774C0_CLK_S3D1C),
@@ -177,8 +178,8 @@ static const struct mssr_mod_clk r8a774c0_mod_clks[] __initconst = {
 	DEF_MOD("vspb",			 626,	R8A774C0_CLK_S0D1),
 	DEF_MOD("vspi0",		 631,	R8A774C0_CLK_S0D1),
 
-	DEF_MOD("ehci0",		 703,	R8A774C0_CLK_S3D4),
-	DEF_MOD("hsusb",		 704,	R8A774C0_CLK_S3D4),
+	DEF_MOD("ehci0",		 703,	R8A774C0_CLK_S3D2),
+	DEF_MOD("hsusb",		 704,	R8A774C0_CLK_S3D2),
 	DEF_MOD("csi40",		 716,	R8A774C0_CLK_CSI0),
 	DEF_MOD("du1",			 723,	R8A774C0_CLK_S1D1),
 	DEF_MOD("du0",			 724,	R8A774C0_CLK_S1D1),
diff --git a/drivers/clk/renesas/r8a7795-cpg-mssr.c b/drivers/clk/renesas/r8a7795-cpg-mssr.c
index 86842c9fd314..9e9a6f2c31e8 100644
--- a/drivers/clk/renesas/r8a7795-cpg-mssr.c
+++ b/drivers/clk/renesas/r8a7795-cpg-mssr.c
@@ -3,6 +3,7 @@
  * r8a7795 Clock Pulse Generator / Module Standby and Software Reset
  *
  * Copyright (C) 2015 Glider bvba
+ * Copyright (C) 2018-2019 Renesas Electronics Corp.
  *
  * Based on clk-rcar-gen3.c
  *
@@ -73,8 +74,8 @@ static struct cpg_core_clk r8a7795_core_clks[] __initdata = {
 	DEF_GEN3_OSC(".r",      CLK_RINT,          CLK_EXTAL,      32),
 
 	/* Core Clock Outputs */
-	DEF_BASE("z",           R8A7795_CLK_Z,     CLK_TYPE_GEN3_Z, CLK_PLL0),
-	DEF_BASE("z2",          R8A7795_CLK_Z2,    CLK_TYPE_GEN3_Z2, CLK_PLL2),
+	DEF_GEN3_Z("z",         R8A7795_CLK_Z,     CLK_TYPE_GEN3_Z,  CLK_PLL0, 2, 8),
+	DEF_GEN3_Z("z2",        R8A7795_CLK_Z2,    CLK_TYPE_GEN3_Z,  CLK_PLL2, 2, 0),
 	DEF_FIXED("ztr",        R8A7795_CLK_ZTR,   CLK_PLL1_DIV2,  6, 1),
 	DEF_FIXED("ztrd2",      R8A7795_CLK_ZTRD2, CLK_PLL1_DIV2, 12, 1),
 	DEF_FIXED("zt",         R8A7795_CLK_ZT,    CLK_PLL1_DIV2,  4, 1),
@@ -129,8 +130,8 @@ static struct mssr_mod_clk r8a7795_mod_clks[] __initdata = {
 	DEF_MOD("msiof2",		 209,	R8A7795_CLK_MSO),
 	DEF_MOD("msiof1",		 210,	R8A7795_CLK_MSO),
 	DEF_MOD("msiof0",		 211,	R8A7795_CLK_MSO),
-	DEF_MOD("sys-dmac2",		 217,	R8A7795_CLK_S0D3),
-	DEF_MOD("sys-dmac1",		 218,	R8A7795_CLK_S0D3),
+	DEF_MOD("sys-dmac2",		 217,	R8A7795_CLK_S3D1),
+	DEF_MOD("sys-dmac1",		 218,	R8A7795_CLK_S3D1),
 	DEF_MOD("sys-dmac0",		 219,	R8A7795_CLK_S0D3),
 	DEF_MOD("sceg-pub",		 229,	R8A7795_CLK_CR),
 	DEF_MOD("cmt3",			 300,	R8A7795_CLK_R),
@@ -153,16 +154,16 @@ static struct mssr_mod_clk r8a7795_mod_clks[] __initdata = {
 	DEF_MOD("rwdt",			 402,	R8A7795_CLK_R),
 	DEF_MOD("intc-ex",		 407,	R8A7795_CLK_CP),
 	DEF_MOD("intc-ap",		 408,	R8A7795_CLK_S0D3),
-	DEF_MOD("audmac1",		 501,	R8A7795_CLK_S0D3),
-	DEF_MOD("audmac0",		 502,	R8A7795_CLK_S0D3),
-	DEF_MOD("drif7",		 508,	R8A7795_CLK_S3D2),
-	DEF_MOD("drif6",		 509,	R8A7795_CLK_S3D2),
-	DEF_MOD("drif5",		 510,	R8A7795_CLK_S3D2),
-	DEF_MOD("drif4",		 511,	R8A7795_CLK_S3D2),
-	DEF_MOD("drif3",		 512,	R8A7795_CLK_S3D2),
-	DEF_MOD("drif2",		 513,	R8A7795_CLK_S3D2),
-	DEF_MOD("drif1",		 514,	R8A7795_CLK_S3D2),
-	DEF_MOD("drif0",		 515,	R8A7795_CLK_S3D2),
+	DEF_MOD("audmac1",		 501,	R8A7795_CLK_S1D2),
+	DEF_MOD("audmac0",		 502,	R8A7795_CLK_S1D2),
+	DEF_MOD("drif31",		 508,	R8A7795_CLK_S3D2),
+	DEF_MOD("drif30",		 509,	R8A7795_CLK_S3D2),
+	DEF_MOD("drif21",		 510,	R8A7795_CLK_S3D2),
+	DEF_MOD("drif20",		 511,	R8A7795_CLK_S3D2),
+	DEF_MOD("drif11",		 512,	R8A7795_CLK_S3D2),
+	DEF_MOD("drif10",		 513,	R8A7795_CLK_S3D2),
+	DEF_MOD("drif01",		 514,	R8A7795_CLK_S3D2),
+	DEF_MOD("drif00",		 515,	R8A7795_CLK_S3D2),
 	DEF_MOD("hscif4",		 516,	R8A7795_CLK_S3D1),
 	DEF_MOD("hscif3",		 517,	R8A7795_CLK_S3D1),
 	DEF_MOD("hscif2",		 518,	R8A7795_CLK_S3D1),
@@ -194,12 +195,12 @@ static struct mssr_mod_clk r8a7795_mod_clks[] __initdata = {
 	DEF_MOD("vspi2",		 629,	R8A7795_CLK_S2D1), /* ES1.x */
 	DEF_MOD("vspi1",		 630,	R8A7795_CLK_S0D1),
 	DEF_MOD("vspi0",		 631,	R8A7795_CLK_S0D1),
-	DEF_MOD("ehci3",		 700,	R8A7795_CLK_S3D4),
-	DEF_MOD("ehci2",		 701,	R8A7795_CLK_S3D4),
-	DEF_MOD("ehci1",		 702,	R8A7795_CLK_S3D4),
-	DEF_MOD("ehci0",		 703,	R8A7795_CLK_S3D4),
-	DEF_MOD("hsusb",		 704,	R8A7795_CLK_S3D4),
-	DEF_MOD("hsusb3",		 705,	R8A7795_CLK_S3D4),
+	DEF_MOD("ehci3",		 700,	R8A7795_CLK_S3D2),
+	DEF_MOD("ehci2",		 701,	R8A7795_CLK_S3D2),
+	DEF_MOD("ehci1",		 702,	R8A7795_CLK_S3D2),
+	DEF_MOD("ehci0",		 703,	R8A7795_CLK_S3D2),
+	DEF_MOD("hsusb",		 704,	R8A7795_CLK_S3D2),
+	DEF_MOD("hsusb3",		 705,	R8A7795_CLK_S3D2),
 	DEF_MOD("csi21",		 713,	R8A7795_CLK_CSI0), /* ES1.x */
 	DEF_MOD("csi20",		 714,	R8A7795_CLK_CSI0),
 	DEF_MOD("csi41",		 715,	R8A7795_CLK_CSI0),
diff --git a/drivers/clk/renesas/r8a7796-cpg-mssr.c b/drivers/clk/renesas/r8a7796-cpg-mssr.c
index 12c455859f2c..d8e9af5d9ae9 100644
--- a/drivers/clk/renesas/r8a7796-cpg-mssr.c
+++ b/drivers/clk/renesas/r8a7796-cpg-mssr.c
@@ -3,6 +3,7 @@
  * r8a7796 Clock Pulse Generator / Module Standby and Software Reset
  *
  * Copyright (C) 2016 Glider bvba
+ * Copyright (C) 2018 Renesas Electronics Corp.
  *
  * Based on r8a7795-cpg-mssr.c
  *
@@ -73,8 +74,8 @@ static const struct cpg_core_clk r8a7796_core_clks[] __initconst = {
 	DEF_GEN3_OSC(".r",      CLK_RINT,          CLK_EXTAL,      32),
 
 	/* Core Clock Outputs */
-	DEF_BASE("z",           R8A7796_CLK_Z,     CLK_TYPE_GEN3_Z, CLK_PLL0),
-	DEF_BASE("z2",          R8A7796_CLK_Z2,    CLK_TYPE_GEN3_Z2, CLK_PLL2),
+	DEF_GEN3_Z("z",         R8A7796_CLK_Z,     CLK_TYPE_GEN3_Z,  CLK_PLL0, 2, 8),
+	DEF_GEN3_Z("z2",        R8A7796_CLK_Z2,    CLK_TYPE_GEN3_Z,  CLK_PLL2, 2, 0),
 	DEF_FIXED("ztr",        R8A7796_CLK_ZTR,   CLK_PLL1_DIV2,  6, 1),
 	DEF_FIXED("ztrd2",      R8A7796_CLK_ZTRD2, CLK_PLL1_DIV2, 12, 1),
 	DEF_FIXED("zt",         R8A7796_CLK_ZT,    CLK_PLL1_DIV2,  4, 1),
@@ -126,8 +127,8 @@ static const struct mssr_mod_clk r8a7796_mod_clks[] __initconst = {
 	DEF_MOD("msiof2",		 209,	R8A7796_CLK_MSO),
 	DEF_MOD("msiof1",		 210,	R8A7796_CLK_MSO),
 	DEF_MOD("msiof0",		 211,	R8A7796_CLK_MSO),
-	DEF_MOD("sys-dmac2",		 217,	R8A7796_CLK_S0D3),
-	DEF_MOD("sys-dmac1",		 218,	R8A7796_CLK_S0D3),
+	DEF_MOD("sys-dmac2",		 217,	R8A7796_CLK_S3D1),
+	DEF_MOD("sys-dmac1",		 218,	R8A7796_CLK_S3D1),
 	DEF_MOD("sys-dmac0",		 219,	R8A7796_CLK_S0D3),
 	DEF_MOD("cmt3",			 300,	R8A7796_CLK_R),
 	DEF_MOD("cmt2",			 301,	R8A7796_CLK_R),
@@ -146,16 +147,16 @@ static const struct mssr_mod_clk r8a7796_mod_clks[] __initconst = {
 	DEF_MOD("rwdt",			 402,	R8A7796_CLK_R),
 	DEF_MOD("intc-ex",		 407,	R8A7796_CLK_CP),
 	DEF_MOD("intc-ap",		 408,	R8A7796_CLK_S0D3),
-	DEF_MOD("audmac1",		 501,	R8A7796_CLK_S0D3),
-	DEF_MOD("audmac0",		 502,	R8A7796_CLK_S0D3),
-	DEF_MOD("drif7",		 508,	R8A7796_CLK_S3D2),
-	DEF_MOD("drif6",		 509,	R8A7796_CLK_S3D2),
-	DEF_MOD("drif5",		 510,	R8A7796_CLK_S3D2),
-	DEF_MOD("drif4",		 511,	R8A7796_CLK_S3D2),
-	DEF_MOD("drif3",		 512,	R8A7796_CLK_S3D2),
-	DEF_MOD("drif2",		 513,	R8A7796_CLK_S3D2),
-	DEF_MOD("drif1",		 514,	R8A7796_CLK_S3D2),
-	DEF_MOD("drif0",		 515,	R8A7796_CLK_S3D2),
+	DEF_MOD("audmac1",		 501,	R8A7796_CLK_S1D2),
+	DEF_MOD("audmac0",		 502,	R8A7796_CLK_S1D2),
+	DEF_MOD("drif31",		 508,	R8A7796_CLK_S3D2),
+	DEF_MOD("drif30",		 509,	R8A7796_CLK_S3D2),
+	DEF_MOD("drif21",		 510,	R8A7796_CLK_S3D2),
+	DEF_MOD("drif20",		 511,	R8A7796_CLK_S3D2),
+	DEF_MOD("drif11",		 512,	R8A7796_CLK_S3D2),
+	DEF_MOD("drif10",		 513,	R8A7796_CLK_S3D2),
+	DEF_MOD("drif01",		 514,	R8A7796_CLK_S3D2),
+	DEF_MOD("drif00",		 515,	R8A7796_CLK_S3D2),
 	DEF_MOD("hscif4",		 516,	R8A7796_CLK_S3D1),
 	DEF_MOD("hscif3",		 517,	R8A7796_CLK_S3D1),
 	DEF_MOD("hscif2",		 518,	R8A7796_CLK_S3D1),
@@ -176,9 +177,9 @@ static const struct mssr_mod_clk r8a7796_mod_clks[] __initconst = {
 	DEF_MOD("vspd0",		 623,	R8A7796_CLK_S0D2),
 	DEF_MOD("vspb",			 626,	R8A7796_CLK_S0D1),
 	DEF_MOD("vspi0",		 631,	R8A7796_CLK_S0D1),
-	DEF_MOD("ehci1",		 702,	R8A7796_CLK_S3D4),
-	DEF_MOD("ehci0",		 703,	R8A7796_CLK_S3D4),
-	DEF_MOD("hsusb",		 704,	R8A7796_CLK_S3D4),
+	DEF_MOD("ehci1",		 702,	R8A7796_CLK_S3D2),
+	DEF_MOD("ehci0",		 703,	R8A7796_CLK_S3D2),
+	DEF_MOD("hsusb",		 704,	R8A7796_CLK_S3D2),
 	DEF_MOD("csi20",		 714,	R8A7796_CLK_CSI0),
 	DEF_MOD("csi40",		 716,	R8A7796_CLK_CSI0),
 	DEF_MOD("du2",			 722,	R8A7796_CLK_S2D1),
diff --git a/drivers/clk/renesas/r8a77965-cpg-mssr.c b/drivers/clk/renesas/r8a77965-cpg-mssr.c
index eb1cca58a1e1..8f87e314d949 100644
--- a/drivers/clk/renesas/r8a77965-cpg-mssr.c
+++ b/drivers/clk/renesas/r8a77965-cpg-mssr.c
@@ -3,6 +3,7 @@
  * r8a77965 Clock Pulse Generator / Module Standby and Software Reset
  *
  * Copyright (C) 2018 Jacopo Mondi <jacopo+renesas@jmondi.org>
+ * Copyright (C) 2019 Renesas Electronics Corp.
  *
  * Based on r8a7795-cpg-mssr.c
  *
@@ -71,7 +72,7 @@ static const struct cpg_core_clk r8a77965_core_clks[] __initconst = {
 	DEF_GEN3_OSC(".r",	CLK_RINT,		CLK_EXTAL,	32),
 
 	/* Core Clock Outputs */
-	DEF_BASE("z",		R8A77965_CLK_Z,		CLK_TYPE_GEN3_Z, CLK_PLL0),
+	DEF_GEN3_Z("z",		R8A77965_CLK_Z,		CLK_TYPE_GEN3_Z,  CLK_PLL0, 2, 8),
 	DEF_FIXED("ztr",	R8A77965_CLK_ZTR,	CLK_PLL1_DIV2,	6, 1),
 	DEF_FIXED("ztrd2",	R8A77965_CLK_ZTRD2,	CLK_PLL1_DIV2,	12, 1),
 	DEF_FIXED("zt",		R8A77965_CLK_ZT,	CLK_PLL1_DIV2,	4, 1),
@@ -123,8 +124,8 @@ static const struct mssr_mod_clk r8a77965_mod_clks[] __initconst = {
 	DEF_MOD("msiof2",		209,	R8A77965_CLK_MSO),
 	DEF_MOD("msiof1",		210,	R8A77965_CLK_MSO),
 	DEF_MOD("msiof0",		211,	R8A77965_CLK_MSO),
-	DEF_MOD("sys-dmac2",		217,	R8A77965_CLK_S0D3),
-	DEF_MOD("sys-dmac1",		218,	R8A77965_CLK_S0D3),
+	DEF_MOD("sys-dmac2",		217,	R8A77965_CLK_S3D1),
+	DEF_MOD("sys-dmac1",		218,	R8A77965_CLK_S3D1),
 	DEF_MOD("sys-dmac0",		219,	R8A77965_CLK_S0D3),
 
 	DEF_MOD("cmt3",			300,	R8A77965_CLK_R),
@@ -146,16 +147,16 @@ static const struct mssr_mod_clk r8a77965_mod_clks[] __initconst = {
 	DEF_MOD("intc-ex",		407,	R8A77965_CLK_CP),
 	DEF_MOD("intc-ap",		408,	R8A77965_CLK_S0D3),
 
-	DEF_MOD("audmac1",		501,	R8A77965_CLK_S0D3),
-	DEF_MOD("audmac0",		502,	R8A77965_CLK_S0D3),
-	DEF_MOD("drif7",		508,	R8A77965_CLK_S3D2),
-	DEF_MOD("drif6",		509,	R8A77965_CLK_S3D2),
-	DEF_MOD("drif5",		510,	R8A77965_CLK_S3D2),
-	DEF_MOD("drif4",		511,	R8A77965_CLK_S3D2),
-	DEF_MOD("drif3",		512,	R8A77965_CLK_S3D2),
-	DEF_MOD("drif2",		513,	R8A77965_CLK_S3D2),
-	DEF_MOD("drif1",		514,	R8A77965_CLK_S3D2),
-	DEF_MOD("drif0",		515,	R8A77965_CLK_S3D2),
+	DEF_MOD("audmac1",		501,	R8A77965_CLK_S1D2),
+	DEF_MOD("audmac0",		502,	R8A77965_CLK_S1D2),
+	DEF_MOD("drif31",		508,	R8A77965_CLK_S3D2),
+	DEF_MOD("drif30",		509,	R8A77965_CLK_S3D2),
+	DEF_MOD("drif21",		510,	R8A77965_CLK_S3D2),
+	DEF_MOD("drif20",		511,	R8A77965_CLK_S3D2),
+	DEF_MOD("drif11",		512,	R8A77965_CLK_S3D2),
+	DEF_MOD("drif10",		513,	R8A77965_CLK_S3D2),
+	DEF_MOD("drif01",		514,	R8A77965_CLK_S3D2),
+	DEF_MOD("drif00",		515,	R8A77965_CLK_S3D2),
 	DEF_MOD("hscif4",		516,	R8A77965_CLK_S3D1),
 	DEF_MOD("hscif3",		517,	R8A77965_CLK_S3D1),
 	DEF_MOD("hscif2",		518,	R8A77965_CLK_S3D1),
@@ -175,9 +176,9 @@ static const struct mssr_mod_clk r8a77965_mod_clks[] __initconst = {
 	DEF_MOD("vspb",			626,	R8A77965_CLK_S0D1),
 	DEF_MOD("vspi0",		631,	R8A77965_CLK_S0D1),
 
-	DEF_MOD("ehci1",		702,	R8A77965_CLK_S3D4),
-	DEF_MOD("ehci0",		703,	R8A77965_CLK_S3D4),
-	DEF_MOD("hsusb",		704,	R8A77965_CLK_S3D4),
+	DEF_MOD("ehci1",		702,	R8A77965_CLK_S3D2),
+	DEF_MOD("ehci0",		703,	R8A77965_CLK_S3D2),
+	DEF_MOD("hsusb",		704,	R8A77965_CLK_S3D2),
 	DEF_MOD("csi20",		714,	R8A77965_CLK_CSI0),
 	DEF_MOD("csi40",		716,	R8A77965_CLK_CSI0),
 	DEF_MOD("du3",			721,	R8A77965_CLK_S2D1),
diff --git a/drivers/clk/renesas/r8a77980-cpg-mssr.c b/drivers/clk/renesas/r8a77980-cpg-mssr.c
index f9e07fcc0d96..7227f675e61f 100644
--- a/drivers/clk/renesas/r8a77980-cpg-mssr.c
+++ b/drivers/clk/renesas/r8a77980-cpg-mssr.c
@@ -171,7 +171,7 @@ static const struct mssr_mod_clk r8a77980_mod_clks[] __initconst = {
 	DEF_MOD("gpio1",		 911,	R8A77980_CLK_CP),
 	DEF_MOD("gpio0",		 912,	R8A77980_CLK_CP),
 	DEF_MOD("can-fd",		 914,	R8A77980_CLK_S3D2),
-	DEF_MOD("rpc-if",		 917,	R8A77980_CLK_RPC),
+	DEF_MOD("rpc-if",		 917,	R8A77980_CLK_RPCD2),
 	DEF_MOD("i2c4",			 927,	R8A77980_CLK_S0D6),
 	DEF_MOD("i2c3",			 928,	R8A77980_CLK_S0D6),
 	DEF_MOD("i2c2",			 929,	R8A77980_CLK_S3D2),
diff --git a/drivers/clk/renesas/r8a77990-cpg-mssr.c b/drivers/clk/renesas/r8a77990-cpg-mssr.c
index 9a278c75c918..9570404baa58 100644
--- a/drivers/clk/renesas/r8a77990-cpg-mssr.c
+++ b/drivers/clk/renesas/r8a77990-cpg-mssr.c
@@ -2,7 +2,7 @@
 /*
  * r8a77990 Clock Pulse Generator / Module Standby and Software Reset
  *
- * Copyright (C) 2018 Renesas Electronics Corp.
+ * Copyright (C) 2018-2019 Renesas Electronics Corp.
  *
  * Based on r8a7795-cpg-mssr.c
  *
@@ -81,6 +81,7 @@ static const struct cpg_core_clk r8a77990_core_clks[] __initconst = {
 	/* Core Clock Outputs */
 	DEF_FIXED("za2",       R8A77990_CLK_ZA2,   CLK_PLL0D24,    1, 1),
 	DEF_FIXED("za8",       R8A77990_CLK_ZA8,   CLK_PLL0D8,     1, 1),
+	DEF_GEN3_Z("z2",       R8A77990_CLK_Z2,    CLK_TYPE_GEN3_Z, CLK_PLL0, 4, 8),
 	DEF_FIXED("ztr",       R8A77990_CLK_ZTR,   CLK_PLL1,       6, 1),
 	DEF_FIXED("zt",        R8A77990_CLK_ZT,    CLK_PLL1,       4, 1),
 	DEF_FIXED("zx",        R8A77990_CLK_ZX,    CLK_PLL1,       3, 1),
@@ -152,15 +153,15 @@ static const struct mssr_mod_clk r8a77990_mod_clks[] __initconst = {
 	DEF_MOD("intc-ex",		 407,	R8A77990_CLK_CP),
 	DEF_MOD("intc-ap",		 408,	R8A77990_CLK_S0D3),
 
-	DEF_MOD("audmac0",		 502,	R8A77990_CLK_S3D4),
-	DEF_MOD("drif7",		 508,	R8A77990_CLK_S3D2),
-	DEF_MOD("drif6",		 509,	R8A77990_CLK_S3D2),
-	DEF_MOD("drif5",		 510,	R8A77990_CLK_S3D2),
-	DEF_MOD("drif4",		 511,	R8A77990_CLK_S3D2),
-	DEF_MOD("drif3",		 512,	R8A77990_CLK_S3D2),
-	DEF_MOD("drif2",		 513,	R8A77990_CLK_S3D2),
-	DEF_MOD("drif1",		 514,	R8A77990_CLK_S3D2),
-	DEF_MOD("drif0",		 515,	R8A77990_CLK_S3D2),
+	DEF_MOD("audmac0",		 502,	R8A77990_CLK_S1D2),
+	DEF_MOD("drif31",		 508,	R8A77990_CLK_S3D2),
+	DEF_MOD("drif30",		 509,	R8A77990_CLK_S3D2),
+	DEF_MOD("drif21",		 510,	R8A77990_CLK_S3D2),
+	DEF_MOD("drif20",		 511,	R8A77990_CLK_S3D2),
+	DEF_MOD("drif11",		 512,	R8A77990_CLK_S3D2),
+	DEF_MOD("drif10",		 513,	R8A77990_CLK_S3D2),
+	DEF_MOD("drif01",		 514,	R8A77990_CLK_S3D2),
+	DEF_MOD("drif00",		 515,	R8A77990_CLK_S3D2),
 	DEF_MOD("hscif4",		 516,	R8A77990_CLK_S3D1C),
 	DEF_MOD("hscif3",		 517,	R8A77990_CLK_S3D1C),
 	DEF_MOD("hscif2",		 518,	R8A77990_CLK_S3D1C),
@@ -180,8 +181,8 @@ static const struct mssr_mod_clk r8a77990_mod_clks[] __initconst = {
 	DEF_MOD("vspb",			 626,	R8A77990_CLK_S0D1),
 	DEF_MOD("vspi0",		 631,	R8A77990_CLK_S0D1),
 
-	DEF_MOD("ehci0",		 703,	R8A77990_CLK_S3D4),
-	DEF_MOD("hsusb",		 704,	R8A77990_CLK_S3D4),
+	DEF_MOD("ehci0",		 703,	R8A77990_CLK_S3D2),
+	DEF_MOD("hsusb",		 704,	R8A77990_CLK_S3D2),
 	DEF_MOD("csi40",		 716,	R8A77990_CLK_CSI0),
 	DEF_MOD("du1",			 723,	R8A77990_CLK_S1D1),
 	DEF_MOD("du0",			 724,	R8A77990_CLK_S1D1),
diff --git a/drivers/clk/renesas/r8a77995-cpg-mssr.c b/drivers/clk/renesas/r8a77995-cpg-mssr.c
index eee3874865a9..68707277b17b 100644
--- a/drivers/clk/renesas/r8a77995-cpg-mssr.c
+++ b/drivers/clk/renesas/r8a77995-cpg-mssr.c
@@ -133,7 +133,7 @@ static const struct mssr_mod_clk r8a77995_mod_clks[] __initconst = {
 	DEF_MOD("rwdt",			 402,	R8A77995_CLK_R),
 	DEF_MOD("intc-ex",		 407,	R8A77995_CLK_CP),
 	DEF_MOD("intc-ap",		 408,	R8A77995_CLK_S1D2),
-	DEF_MOD("audmac0",		 502,	R8A77995_CLK_S3D1),
+	DEF_MOD("audmac0",		 502,	R8A77995_CLK_S1D2),
 	DEF_MOD("hscif3",		 517,	R8A77995_CLK_S3D1C),
 	DEF_MOD("hscif0",		 520,	R8A77995_CLK_S3D1C),
 	DEF_MOD("thermal",		 522,	R8A77995_CLK_CP),
diff --git a/drivers/clk/renesas/r9a06g032-clocks.c b/drivers/clk/renesas/r9a06g032-clocks.c
index 658cb11b6f55..97c72477cd54 100644
--- a/drivers/clk/renesas/r9a06g032-clocks.c
+++ b/drivers/clk/renesas/r9a06g032-clocks.c
@@ -170,6 +170,7 @@ static const struct r9a06g032_clkdesc r9a06g032_clocks[] __initconst = {
 	D_GATE(CLK_P6_PG2, "clk_p6_pg2", DIV_P6_PG, 0x8a3, 0x8a4, 0x8a5, 0, 0xb61, 0, 0),
 	D_GATE(CLK_P6_PG3, "clk_p6_pg3", DIV_P6_PG, 0x8a6, 0x8a7, 0x8a8, 0, 0xb62, 0, 0),
 	D_GATE(CLK_P6_PG4, "clk_p6_pg4", DIV_P6_PG, 0x8a9, 0x8aa, 0x8ab, 0, 0xb63, 0, 0),
+	D_GATE(CLK_PCI_USB, "clk_pci_usb", CLKOUT_D40, 0xe6, 0, 0, 0, 0, 0, 0),
 	D_GATE(CLK_QSPI0, "clk_qspi0", DIV_QSPI0, 0x2a4, 0x2a5, 0, 0, 0, 0, 0),
 	D_GATE(CLK_QSPI1, "clk_qspi1", DIV_QSPI1, 0x484, 0x485, 0, 0, 0, 0, 0),
 	D_GATE(CLK_RGMII_REF, "clk_rgmii_ref", CLKOUT_D8, 0x340, 0, 0, 0, 0, 0, 0),
diff --git a/drivers/clk/renesas/rcar-gen2-cpg.h b/drivers/clk/renesas/rcar-gen2-cpg.h
index bff9551c7a38..db2f57ef2f99 100644
--- a/drivers/clk/renesas/rcar-gen2-cpg.h
+++ b/drivers/clk/renesas/rcar-gen2-cpg.h
@@ -1,5 +1,5 @@
-/* SPDX-License-Identifier: GPL-2.0
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * R-Car Gen2 Clock Pulse Generator
  *
  * Copyright (C) 2016 Cogent Embedded Inc.
diff --git a/drivers/clk/renesas/rcar-gen3-cpg.c b/drivers/clk/renesas/rcar-gen3-cpg.c
index 9a8071a8114d..d25c8ba00a65 100644
--- a/drivers/clk/renesas/rcar-gen3-cpg.c
+++ b/drivers/clk/renesas/rcar-gen3-cpg.c
@@ -3,6 +3,7 @@
  * R-Car Gen3 Clock Pulse Generator
  *
  * Copyright (C) 2015-2018 Glider bvba
+ * Copyright (C) 2019 Renesas Electronics Corp.
  *
  * Based on clk-rcar-gen3.c
  *
@@ -88,14 +89,13 @@ static void cpg_simple_notifier_register(struct raw_notifier_head *notifiers,
 #define CPG_FRQCRB			0x00000004
 #define CPG_FRQCRB_KICK			BIT(31)
 #define CPG_FRQCRC			0x000000e0
-#define CPG_FRQCRC_ZFC_MASK		GENMASK(12, 8)
-#define CPG_FRQCRC_Z2FC_MASK		GENMASK(4, 0)
 
 struct cpg_z_clk {
 	struct clk_hw hw;
 	void __iomem *reg;
 	void __iomem *kick_reg;
 	unsigned long mask;
+	unsigned int fixed_div;
 };
 
 #define to_z_clk(_hw)	container_of(_hw, struct cpg_z_clk, hw)
@@ -110,17 +110,18 @@ static unsigned long cpg_z_clk_recalc_rate(struct clk_hw *hw,
 	val = readl(zclk->reg) & zclk->mask;
 	mult = 32 - (val >> __ffs(zclk->mask));
 
-	/* Factor of 2 is for fixed divider */
-	return DIV_ROUND_CLOSEST_ULL((u64)parent_rate * mult, 32 * 2);
+	return DIV_ROUND_CLOSEST_ULL((u64)parent_rate * mult,
+				     32 * zclk->fixed_div);
 }
 
 static long cpg_z_clk_round_rate(struct clk_hw *hw, unsigned long rate,
 				 unsigned long *parent_rate)
 {
-	/* Factor of 2 is for fixed divider */
-	unsigned long prate = *parent_rate / 2;
+	struct cpg_z_clk *zclk = to_z_clk(hw);
+	unsigned long prate;
 	unsigned int mult;
 
+	prate = *parent_rate / zclk->fixed_div;
 	mult = div_u64(rate * 32ULL, prate);
 	mult = clamp(mult, 1U, 32U);
 
@@ -134,8 +135,8 @@ static int cpg_z_clk_set_rate(struct clk_hw *hw, unsigned long rate,
 	unsigned int mult;
 	unsigned int i;
 
-	/* Factor of 2 is for fixed divider */
-	mult = DIV_ROUND_CLOSEST_ULL(rate * 32ULL * 2, parent_rate);
+	mult = DIV64_U64_ROUND_CLOSEST(rate * 32ULL * zclk->fixed_div,
+				       parent_rate);
 	mult = clamp(mult, 1U, 32U);
 
 	if (readl(zclk->kick_reg) & CPG_FRQCRB_KICK)
@@ -178,7 +179,8 @@ static const struct clk_ops cpg_z_clk_ops = {
 static struct clk * __init cpg_z_clk_register(const char *name,
 					      const char *parent_name,
 					      void __iomem *reg,
-					      unsigned long mask)
+					      unsigned int div,
+					      unsigned int offset)
 {
 	struct clk_init_data init;
 	struct cpg_z_clk *zclk;
@@ -197,7 +199,8 @@ static struct clk * __init cpg_z_clk_register(const char *name,
 	zclk->reg = reg + CPG_FRQCRC;
 	zclk->kick_reg = reg + CPG_FRQCRB;
 	zclk->hw.init = &init;
-	zclk->mask = mask;
+	zclk->mask = GENMASK(offset + 4, offset);
+	zclk->fixed_div = div; /* PLLVCO x 1/div x SYS-CPU divider */
 
 	clk = clk_register(NULL, &zclk->hw);
 	if (IS_ERR(clk))
@@ -234,8 +237,6 @@ struct sd_clock {
 	const struct sd_div_table *div_table;
 	struct cpg_simple_notifier csn;
 	unsigned int div_num;
-	unsigned int div_min;
-	unsigned int div_max;
 	unsigned int cur_div_idx;
 };
 
@@ -312,14 +313,20 @@ static unsigned int cpg_sd_clock_calc_div(struct sd_clock *clock,
 					  unsigned long rate,
 					  unsigned long parent_rate)
 {
-	unsigned int div;
-
-	if (!rate)
-		rate = 1;
-
-	div = DIV_ROUND_CLOSEST(parent_rate, rate);
+	unsigned long calc_rate, diff, diff_min = ULONG_MAX;
+	unsigned int i, best_div = 0;
+
+	for (i = 0; i < clock->div_num; i++) {
+		calc_rate = DIV_ROUND_CLOSEST(parent_rate,
+					      clock->div_table[i].div);
+		diff = calc_rate > rate ? calc_rate - rate : rate - calc_rate;
+		if (diff < diff_min) {
+			best_div = clock->div_table[i].div;
+			diff_min = diff;
+		}
+	}
 
-	return clamp_t(unsigned int, div, clock->div_min, clock->div_max);
+	return best_div;
 }
 
 static long cpg_sd_clock_round_rate(struct clk_hw *hw, unsigned long rate,
@@ -369,27 +376,26 @@ static u32 cpg_quirks __initdata;
 #define RCKCR_CKSEL	BIT(1)		/* Manual RCLK parent selection */
 #define SD_SKIP_FIRST	BIT(2)		/* Skip first clock in SD table */
 
-static struct clk * __init cpg_sd_clk_register(const struct cpg_core_clk *core,
-	void __iomem *base, const char *parent_name,
+static struct clk * __init cpg_sd_clk_register(const char *name,
+	void __iomem *base, unsigned int offset, const char *parent_name,
 	struct raw_notifier_head *notifiers)
 {
 	struct clk_init_data init;
 	struct sd_clock *clock;
 	struct clk *clk;
-	unsigned int i;
 	u32 val;
 
 	clock = kzalloc(sizeof(*clock), GFP_KERNEL);
 	if (!clock)
 		return ERR_PTR(-ENOMEM);
 
-	init.name = core->name;
+	init.name = name;
 	init.ops = &cpg_sd_clock_ops;
 	init.flags = CLK_SET_RATE_PARENT;
 	init.parent_names = &parent_name;
 	init.num_parents = 1;
 
-	clock->csn.reg = base + core->offset;
+	clock->csn.reg = base + offset;
 	clock->hw.init = &init;
 	clock->div_table = cpg_sd_div_table;
 	clock->div_num = ARRAY_SIZE(cpg_sd_div_table);
@@ -403,13 +409,6 @@ static struct clk * __init cpg_sd_clk_register(const struct cpg_core_clk *core,
 	val |= CPG_SD_STP_MASK | (clock->div_table[0].val & CPG_SD_FC_MASK);
 	writel(val, clock->csn.reg);
 
-	clock->div_max = clock->div_table[0].div;
-	clock->div_min = clock->div_max;
-	for (i = 1; i < clock->div_num; i++) {
-		clock->div_max = max(clock->div_max, clock->div_table[i].div);
-		clock->div_min = min(clock->div_min, clock->div_table[i].div);
-	}
-
 	clk = clk_register(NULL, &clock->hw);
 	if (IS_ERR(clk))
 		goto free_clock;
@@ -606,8 +605,8 @@ struct clk * __init rcar_gen3_cpg_clk_register(struct device *dev,
 		break;
 
 	case CLK_TYPE_GEN3_SD:
-		return cpg_sd_clk_register(core, base, __clk_get_name(parent),
-					   notifiers);
+		return cpg_sd_clk_register(core->name, base, core->offset,
+					   __clk_get_name(parent), notifiers);
 
 	case CLK_TYPE_GEN3_R:
 		if (cpg_quirks & RCKCR_CKSEL) {
@@ -658,11 +657,7 @@ struct clk * __init rcar_gen3_cpg_clk_register(struct device *dev,
 
 	case CLK_TYPE_GEN3_Z:
 		return cpg_z_clk_register(core->name, __clk_get_name(parent),
-					  base, CPG_FRQCRC_ZFC_MASK);
-
-	case CLK_TYPE_GEN3_Z2:
-		return cpg_z_clk_register(core->name, __clk_get_name(parent),
-					  base, CPG_FRQCRC_Z2FC_MASK);
+					  base, core->div, core->offset);
 
 	case CLK_TYPE_GEN3_OSC:
 		/*
diff --git a/drivers/clk/renesas/rcar-gen3-cpg.h b/drivers/clk/renesas/rcar-gen3-cpg.h
index eac1b057455a..c4ac80cac6a0 100644
--- a/drivers/clk/renesas/rcar-gen3-cpg.h
+++ b/drivers/clk/renesas/rcar-gen3-cpg.h
@@ -1,8 +1,9 @@
-/* SPDX-License-Identifier: GPL-2.0
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * R-Car Gen3 Clock Pulse Generator
  *
  * Copyright (C) 2015-2018 Glider bvba
+ * Copyright (C) 2018 Renesas Electronics Corp.
  *
  */
 
@@ -20,7 +21,6 @@ enum rcar_gen3_clk_types {
 	CLK_TYPE_GEN3_R,
 	CLK_TYPE_GEN3_MDSEL,	/* Select parent/divider using mode pin */
 	CLK_TYPE_GEN3_Z,
-	CLK_TYPE_GEN3_Z2,
 	CLK_TYPE_GEN3_OSC,	/* OSC EXTAL predivider and fixed divider */
 	CLK_TYPE_GEN3_RCKSEL,	/* Select parent/divider using RCKCR.CKSEL */
 	CLK_TYPE_GEN3_RPCSRC,
@@ -51,6 +51,9 @@ enum rcar_gen3_clk_types {
 	DEF_BASE(_name, _id, CLK_TYPE_GEN3_RCKSEL,	\
 		 (_parent0) << 16 | (_parent1),	.div = (_div0) << 16 | (_div1))
 
+#define DEF_GEN3_Z(_name, _id, _type, _parent, _div, _offset)	\
+	DEF_BASE(_name, _id, _type, _parent, .div = _div, .offset = _offset)
+
 struct rcar_gen3_cpg_pll_config {
 	u8 extal_div;
 	u8 pll1_mult;
diff --git a/drivers/clk/renesas/renesas-cpg-mssr.h b/drivers/clk/renesas/renesas-cpg-mssr.h
index c4ec9df146fd..4ddcdf3bfb95 100644
--- a/drivers/clk/renesas/renesas-cpg-mssr.h
+++ b/drivers/clk/renesas/renesas-cpg-mssr.h
@@ -1,5 +1,5 @@
-/* SPDX-License-Identifier: GPL-2.0
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Renesas Clock Pulse Generator / Module Standby and Software Reset
  *
  * Copyright (C) 2015 Glider bvba
diff --git a/drivers/clk/rockchip/clk-ddr.c b/drivers/clk/rockchip/clk-ddr.c
index ebce5260068b..09ede6920593 100644
--- a/drivers/clk/rockchip/clk-ddr.c
+++ b/drivers/clk/rockchip/clk-ddr.c
@@ -82,7 +82,7 @@ static u8 rockchip_ddrclk_get_parent(struct clk_hw *hw)
 	struct rockchip_ddrclk *ddrclk = to_rockchip_ddrclk_hw(hw);
 	u32 val;
 
-	val = clk_readl(ddrclk->reg_base +
+	val = readl(ddrclk->reg_base +
 			ddrclk->mux_offset) >> ddrclk->mux_shift;
 	val &= GENMASK(ddrclk->mux_width - 1, 0);
 
diff --git a/drivers/clk/rockchip/clk-half-divider.c b/drivers/clk/rockchip/clk-half-divider.c
index b8da6e799423..784b81e1ea7c 100644
--- a/drivers/clk/rockchip/clk-half-divider.c
+++ b/drivers/clk/rockchip/clk-half-divider.c
@@ -24,7 +24,7 @@ static unsigned long clk_half_divider_recalc_rate(struct clk_hw *hw,
 	struct clk_divider *divider = to_clk_divider(hw);
 	unsigned int val;
 
-	val = clk_readl(divider->reg) >> divider->shift;
+	val = readl(divider->reg) >> divider->shift;
 	val &= div_mask(divider->width);
 	val = val * 2 + 3;
 
@@ -124,11 +124,11 @@ static int clk_half_divider_set_rate(struct clk_hw *hw, unsigned long rate,
 	if (divider->flags & CLK_DIVIDER_HIWORD_MASK) {
 		val = div_mask(divider->width) << (divider->shift + 16);
 	} else {
-		val = clk_readl(divider->reg);
+		val = readl(divider->reg);
 		val &= ~(div_mask(divider->width) << divider->shift);
 	}
 	val |= value << divider->shift;
-	clk_writel(val, divider->reg);
+	writel(val, divider->reg);
 
 	if (divider->lock)
 		spin_unlock_irqrestore(divider->lock, flags);
diff --git a/drivers/clk/rockchip/clk-rk3288.c b/drivers/clk/rockchip/clk-rk3288.c
index 5a67b7869960..24baeb56a1b3 100644
--- a/drivers/clk/rockchip/clk-rk3288.c
+++ b/drivers/clk/rockchip/clk-rk3288.c
@@ -200,8 +200,8 @@ PNAME(mux_aclk_cpu_src_p)	= { "cpll_aclk_cpu", "gpll_aclk_cpu" };
 PNAME(mux_pll_src_cpll_gpll_p)		= { "cpll", "gpll" };
 PNAME(mux_pll_src_npll_cpll_gpll_p)	= { "npll", "cpll", "gpll" };
 PNAME(mux_pll_src_cpll_gpll_npll_p)	= { "cpll", "gpll", "npll" };
-PNAME(mux_pll_src_cpll_gpll_usb480m_p)	= { "cpll", "gpll", "usbphy480m_src" };
-PNAME(mux_pll_src_cpll_gll_usb_npll_p)	= { "cpll", "gpll", "usbphy480m_src", "npll" };
+PNAME(mux_pll_src_cpll_gpll_usb480m_p)	= { "cpll", "gpll", "unstable:usbphy480m_src" };
+PNAME(mux_pll_src_cpll_gll_usb_npll_p)	= { "cpll", "gpll", "unstable:usbphy480m_src", "npll" };
 
 PNAME(mux_mmc_src_p)	= { "cpll", "gpll", "xin24m", "xin24m" };
 PNAME(mux_i2s_pre_p)	= { "i2s_src", "i2s_frac", "ext_i2s", "xin12m" };
@@ -219,7 +219,7 @@ PNAME(mux_hsadcout_p)	= { "hsadc_src", "ext_hsadc" };
 PNAME(mux_edp_24m_p)	= { "ext_edp_24m", "xin24m" };
 PNAME(mux_tspout_p)	= { "cpll", "gpll", "npll", "xin27m" };
 
-PNAME(mux_aclk_vcodec_pre_p)	= { "aclk_vepu", "aclk_vdpu" };
+PNAME(mux_aclk_vcodec_pre_p)	= { "aclk_vdpu", "aclk_vepu" };
 PNAME(mux_usbphy480m_p)		= { "sclk_otgphy1_480m", "sclk_otgphy2_480m",
 				    "sclk_otgphy0_480m" };
 PNAME(mux_hsicphy480m_p)	= { "cpll", "gpll", "usbphy480m_src" };
@@ -313,13 +313,13 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = {
 	COMPOSITE_NOMUX(0, "aclk_core_mp", "armclk", CLK_IGNORE_UNUSED,
 			RK3288_CLKSEL_CON(0), 4, 4, DFLAGS | CLK_DIVIDER_READ_ONLY,
 			RK3288_CLKGATE_CON(12), 6, GFLAGS),
-	COMPOSITE_NOMUX(0, "atclk", "armclk", CLK_IGNORE_UNUSED,
+	COMPOSITE_NOMUX(0, "atclk", "armclk", 0,
 			RK3288_CLKSEL_CON(37), 4, 5, DFLAGS | CLK_DIVIDER_READ_ONLY,
 			RK3288_CLKGATE_CON(12), 7, GFLAGS),
 	COMPOSITE_NOMUX(0, "pclk_dbg_pre", "armclk", CLK_IGNORE_UNUSED,
 			RK3288_CLKSEL_CON(37), 9, 5, DFLAGS | CLK_DIVIDER_READ_ONLY,
 			RK3288_CLKGATE_CON(12), 8, GFLAGS),
-	GATE(0, "pclk_dbg", "pclk_dbg_pre", CLK_IGNORE_UNUSED,
+	GATE(0, "pclk_dbg", "pclk_dbg_pre", 0,
 			RK3288_CLKGATE_CON(12), 9, GFLAGS),
 	GATE(0, "cs_dbg", "pclk_dbg_pre", CLK_IGNORE_UNUSED,
 			RK3288_CLKGATE_CON(12), 10, GFLAGS),
@@ -420,7 +420,7 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = {
 	COMPOSITE(0, "aclk_vdpu", mux_pll_src_cpll_gpll_usb480m_p, 0,
 			RK3288_CLKSEL_CON(32), 14, 2, MFLAGS, 8, 5, DFLAGS,
 			RK3288_CLKGATE_CON(3), 11, GFLAGS),
-	MUXGRF(0, "aclk_vcodec_pre", mux_aclk_vcodec_pre_p, 0,
+	MUXGRF(0, "aclk_vcodec_pre", mux_aclk_vcodec_pre_p, CLK_SET_RATE_PARENT,
 			RK3288_GRF_SOC_CON(0), 7, 1, MFLAGS),
 	GATE(ACLK_VCODEC, "aclk_vcodec", "aclk_vcodec_pre", 0,
 		RK3288_CLKGATE_CON(9), 0, GFLAGS),
@@ -647,7 +647,7 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = {
 	INVERTER(SCLK_HSADC, "sclk_hsadc", "sclk_hsadc_out",
 			RK3288_CLKSEL_CON(22), 7, IFLAGS),
 
-	GATE(0, "jtag", "ext_jtag", CLK_IGNORE_UNUSED,
+	GATE(0, "jtag", "ext_jtag", 0,
 			RK3288_CLKGATE_CON(4), 14, GFLAGS),
 
 	COMPOSITE_NODIV(SCLK_USBPHY480M_SRC, "usbphy480m_src", mux_usbphy480m_p, 0,
@@ -656,7 +656,7 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = {
 	COMPOSITE_NODIV(SCLK_HSICPHY480M, "sclk_hsicphy480m", mux_hsicphy480m_p, 0,
 			RK3288_CLKSEL_CON(29), 0, 2, MFLAGS,
 			RK3288_CLKGATE_CON(3), 6, GFLAGS),
-	GATE(0, "hsicphy12m_xin12m", "xin12m", CLK_IGNORE_UNUSED,
+	GATE(0, "hsicphy12m_xin12m", "xin12m", 0,
 			RK3288_CLKGATE_CON(13), 9, GFLAGS),
 	DIV(0, "hsicphy12m_usbphy", "sclk_hsicphy480m", 0,
 			RK3288_CLKSEL_CON(11), 8, 6, DFLAGS),
@@ -697,7 +697,7 @@ static struct rockchip_clk_branch rk3288_clk_branches[] __initdata = {
 	GATE(PCLK_TZPC, "pclk_tzpc", "pclk_cpu", 0, RK3288_CLKGATE_CON(11), 3, GFLAGS),
 	GATE(PCLK_UART2, "pclk_uart2", "pclk_cpu", 0, RK3288_CLKGATE_CON(11), 9, GFLAGS),
 	GATE(PCLK_EFUSE256, "pclk_efuse_256", "pclk_cpu", 0, RK3288_CLKGATE_CON(11), 10, GFLAGS),
-	GATE(PCLK_RKPWM, "pclk_rkpwm", "pclk_cpu", CLK_IGNORE_UNUSED, RK3288_CLKGATE_CON(11), 11, GFLAGS),
+	GATE(PCLK_RKPWM, "pclk_rkpwm", "pclk_cpu", 0, RK3288_CLKGATE_CON(11), 11, GFLAGS),
 
 	/* ddrctrl [DDR Controller PHY clock] gates */
 	GATE(0, "nclk_ddrupctl0", "ddrphy", CLK_IGNORE_UNUSED, RK3288_CLKGATE_CON(11), 4, GFLAGS),
@@ -837,12 +837,9 @@ static const char *const rk3288_critical_clocks[] __initconst = {
 	"pclk_alive_niu",
 	"pclk_pd_pmu",
 	"pclk_pmu_niu",
-	"pclk_core_niu",
-	"pclk_ddrupctl0",
-	"pclk_publ0",
-	"pclk_ddrupctl1",
-	"pclk_publ1",
 	"pmu_hclk_otg0",
+	/* pwm-regulators on some boards, so handoff-critical later */
+	"pclk_rkpwm",
 };
 
 static void __iomem *rk3288_cru_base;
@@ -859,6 +856,9 @@ static const int rk3288_saved_cru_reg_ids[] = {
 	RK3288_CLKSEL_CON(10),
 	RK3288_CLKSEL_CON(33),
 	RK3288_CLKSEL_CON(37),
+
+	/* We turn aclk_dmac1 on for suspend; this will restore it */
+	RK3288_CLKGATE_CON(10),
 };
 
 static u32 rk3288_saved_cru_regs[ARRAY_SIZE(rk3288_saved_cru_reg_ids)];
@@ -875,6 +875,14 @@ static int rk3288_clk_suspend(void)
 	}
 
 	/*
+	 * Going into deep sleep (specifically setting PMU_CLR_DMA in
+	 * RK3288_PMU_PWRMODE_CON1) appears to fail unless
+	 * "aclk_dmac1" is on.
+	 */
+	writel_relaxed(1 << (12 + 16),
+		       rk3288_cru_base + RK3288_CLKGATE_CON(10));
+
+	/*
 	 * Switch PLLs other than DPLL (for SDRAM) to slow mode to
 	 * avoid crashes on resume. The Mask ROM on the system will
 	 * put APLL, CPLL, and GPLL into slow mode at resume time
diff --git a/drivers/clk/rockchip/clk-rk3328.c b/drivers/clk/rockchip/clk-rk3328.c
index 65ab5c2f48b0..f12142d9cea2 100644
--- a/drivers/clk/rockchip/clk-rk3328.c
+++ b/drivers/clk/rockchip/clk-rk3328.c
@@ -458,7 +458,7 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = {
 			RK3328_CLKSEL_CON(35), 15, 1, MFLAGS, 8, 7, DFLAGS,
 			RK3328_CLKGATE_CON(2), 12, GFLAGS),
 	COMPOSITE(SCLK_CRYPTO, "clk_crypto", mux_2plls_p, 0,
-			RK3328_CLKSEL_CON(20), 7, 1, MFLAGS, 0, 7, DFLAGS,
+			RK3328_CLKSEL_CON(20), 7, 1, MFLAGS, 0, 5, DFLAGS,
 			RK3328_CLKGATE_CON(2), 4, GFLAGS),
 	COMPOSITE_NOMUX(SCLK_TSADC, "clk_tsadc", "clk_24m", 0,
 			RK3328_CLKSEL_CON(22), 0, 10, DFLAGS,
@@ -550,15 +550,15 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = {
 	GATE(0, "hclk_rkvenc_niu", "hclk_rkvenc", 0,
 			RK3328_CLKGATE_CON(25), 1, GFLAGS),
 	GATE(ACLK_H265, "aclk_h265", "aclk_rkvenc", 0,
-			RK3328_CLKGATE_CON(25), 0, GFLAGS),
+			RK3328_CLKGATE_CON(25), 2, GFLAGS),
 	GATE(PCLK_H265, "pclk_h265", "hclk_rkvenc", 0,
-			RK3328_CLKGATE_CON(25), 1, GFLAGS),
+			RK3328_CLKGATE_CON(25), 3, GFLAGS),
 	GATE(ACLK_H264, "aclk_h264", "aclk_rkvenc", 0,
-			RK3328_CLKGATE_CON(25), 0, GFLAGS),
+			RK3328_CLKGATE_CON(25), 4, GFLAGS),
 	GATE(HCLK_H264, "hclk_h264", "hclk_rkvenc", 0,
-			RK3328_CLKGATE_CON(25), 1, GFLAGS),
+			RK3328_CLKGATE_CON(25), 5, GFLAGS),
 	GATE(ACLK_AXISRAM, "aclk_axisram", "aclk_rkvenc", CLK_IGNORE_UNUSED,
-			RK3328_CLKGATE_CON(25), 0, GFLAGS),
+			RK3328_CLKGATE_CON(25), 6, GFLAGS),
 
 	COMPOSITE(SCLK_VENC_CORE, "sclk_venc_core", mux_4plls_p, 0,
 			RK3328_CLKSEL_CON(51), 14, 2, MFLAGS, 8, 5, DFLAGS,
@@ -663,7 +663,7 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = {
 
 	/* PD_GMAC */
 	COMPOSITE(ACLK_GMAC, "aclk_gmac", mux_2plls_hdmiphy_p, 0,
-			RK3328_CLKSEL_CON(35), 6, 2, MFLAGS, 0, 5, DFLAGS,
+			RK3328_CLKSEL_CON(25), 6, 2, MFLAGS, 0, 5, DFLAGS,
 			RK3328_CLKGATE_CON(3), 2, GFLAGS),
 	COMPOSITE_NOMUX(PCLK_GMAC, "pclk_gmac", "aclk_gmac", 0,
 			RK3328_CLKSEL_CON(25), 8, 3, DFLAGS,
@@ -733,7 +733,7 @@ static struct rockchip_clk_branch rk3328_clk_branches[] __initdata = {
 
 	/* PD_PERI */
 	GATE(0, "aclk_peri_noc", "aclk_peri", CLK_IGNORE_UNUSED, RK3328_CLKGATE_CON(19), 11, GFLAGS),
-	GATE(ACLK_USB3OTG, "aclk_usb3otg", "aclk_peri", 0, RK3328_CLKGATE_CON(19), 4, GFLAGS),
+	GATE(ACLK_USB3OTG, "aclk_usb3otg", "aclk_peri", 0, RK3328_CLKGATE_CON(19), 14, GFLAGS),
 
 	GATE(HCLK_SDMMC, "hclk_sdmmc", "hclk_peri", 0, RK3328_CLKGATE_CON(19), 0, GFLAGS),
 	GATE(HCLK_SDIO, "hclk_sdio", "hclk_peri", 0, RK3328_CLKGATE_CON(19), 1, GFLAGS),
@@ -913,7 +913,7 @@ static void __init rk3328_clk_init(struct device_node *np)
 				     &rk3328_cpuclk_data, rk3328_cpuclk_rates,
 				     ARRAY_SIZE(rk3328_cpuclk_rates));
 
-	rockchip_register_softrst(np, 11, reg_base + RK3328_SOFTRST_CON(0),
+	rockchip_register_softrst(np, 12, reg_base + RK3328_SOFTRST_CON(0),
 				  ROCKCHIP_SOFTRST_HIWORD_MASK);
 
 	rockchip_register_restart_notifier(ctx, RK3328_GLB_SRST_FST, NULL);
diff --git a/drivers/clk/rockchip/clk.c b/drivers/clk/rockchip/clk.c
index c3ad92965823..0ea8e8080d1a 100644
--- a/drivers/clk/rockchip/clk.c
+++ b/drivers/clk/rockchip/clk.c
@@ -46,7 +46,7 @@ static struct clk *rockchip_clk_register_branch(const char *name,
 		const char *const *parent_names, u8 num_parents,
 		void __iomem *base,
 		int muxdiv_offset, u8 mux_shift, u8 mux_width, u8 mux_flags,
-		u8 div_shift, u8 div_width, u8 div_flags,
+		int div_offset, u8 div_shift, u8 div_width, u8 div_flags,
 		struct clk_div_table *div_table, int gate_offset,
 		u8 gate_shift, u8 gate_flags, unsigned long flags,
 		spinlock_t *lock)
@@ -95,7 +95,10 @@ static struct clk *rockchip_clk_register_branch(const char *name,
 		}
 
 		div->flags = div_flags;
-		div->reg = base + muxdiv_offset;
+		if (div_offset)
+			div->reg = base + div_offset;
+		else
+			div->reg = base + muxdiv_offset;
 		div->shift = div_shift;
 		div->width = div_width;
 		div->lock = lock;
@@ -516,7 +519,7 @@ void __init rockchip_clk_register_branches(
 				ctx->reg_base, list->muxdiv_offset,
 				list->mux_shift,
 				list->mux_width, list->mux_flags,
-				list->div_shift, list->div_width,
+				list->div_offset, list->div_shift, list->div_width,
 				list->div_flags, list->div_table,
 				list->gate_offset, list->gate_shift,
 				list->gate_flags, flags, &ctx->lock);
diff --git a/drivers/clk/rockchip/clk.h b/drivers/clk/rockchip/clk.h
index 6b53fff4cc96..1b5270755431 100644
--- a/drivers/clk/rockchip/clk.h
+++ b/drivers/clk/rockchip/clk.h
@@ -407,6 +407,7 @@ struct rockchip_clk_branch {
 	u8				mux_shift;
 	u8				mux_width;
 	u8				mux_flags;
+	int				div_offset;
 	u8				div_shift;
 	u8				div_width;
 	u8				div_flags;
@@ -438,6 +439,28 @@ struct rockchip_clk_branch {
 		.gate_flags	= gf,				\
 	}
 
+#define COMPOSITE_DIV_OFFSET(_id, cname, pnames, f, mo, ms, mw,	\
+			     mf, do, ds, dw, df, go, gs, gf)	\
+	{							\
+		.id		= _id,				\
+		.branch_type	= branch_composite,		\
+		.name		= cname,			\
+		.parent_names	= pnames,			\
+		.num_parents	= ARRAY_SIZE(pnames),		\
+		.flags		= f,				\
+		.muxdiv_offset	= mo,				\
+		.mux_shift	= ms,				\
+		.mux_width	= mw,				\
+		.mux_flags	= mf,				\
+		.div_offset	= do,				\
+		.div_shift	= ds,				\
+		.div_width	= dw,				\
+		.div_flags	= df,				\
+		.gate_offset	= go,				\
+		.gate_shift	= gs,				\
+		.gate_flags	= gf,				\
+	}
+
 #define COMPOSITE_NOMUX(_id, cname, pname, f, mo, ds, dw, df,	\
 			go, gs, gf)				\
 	{							\
diff --git a/drivers/clk/samsung/clk-exynos5410.c b/drivers/clk/samsung/clk-exynos5410.c
index 0a0b09591e6f..b2da2c8fa0c7 100644
--- a/drivers/clk/samsung/clk-exynos5410.c
+++ b/drivers/clk/samsung/clk-exynos5410.c
@@ -209,6 +209,7 @@ static const struct samsung_gate_clock exynos5410_gate_clks[] __initconst = {
 	GATE(CLK_USI1, "usi1", "aclk66", GATE_IP_PERIC, 11, 0, 0),
 	GATE(CLK_USI2, "usi2", "aclk66", GATE_IP_PERIC, 12, 0, 0),
 	GATE(CLK_USI3, "usi3", "aclk66", GATE_IP_PERIC, 13, 0, 0),
+	GATE(CLK_TSADC, "tsadc", "aclk66", GATE_IP_PERIC, 15, 0, 0),
 	GATE(CLK_PWM, "pwm", "aclk66", GATE_IP_PERIC, 24, 0, 0),
 
 	GATE(CLK_SCLK_UART0, "sclk_uart0", "div_uart0",
diff --git a/drivers/clk/sifive/Kconfig b/drivers/clk/sifive/Kconfig
new file mode 100644
index 000000000000..8db4a3eb4782
--- /dev/null
+++ b/drivers/clk/sifive/Kconfig
@@ -0,0 +1,18 @@
+# SPDX-License-Identifier: GPL-2.0
+
+menuconfig CLK_SIFIVE
+	bool "SiFive SoC driver support"
+	help
+	  SoC drivers for SiFive Linux-capable SoCs.
+
+if CLK_SIFIVE
+
+config CLK_SIFIVE_FU540_PRCI
+	bool "PRCI driver for SiFive FU540 SoCs"
+	select CLK_ANALOGBITS_WRPLL_CLN28HPC
+	help
+	  Supports the Power Reset Clock interface (PRCI) IP block found in
+	  FU540 SoCs.  If this kernel is meant to run on a SiFive FU540 SoC,
+	  enable this driver.
+
+endif
diff --git a/drivers/clk/sifive/Makefile b/drivers/clk/sifive/Makefile
new file mode 100644
index 000000000000..74d58a4c0756
--- /dev/null
+++ b/drivers/clk/sifive/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_CLK_SIFIVE_FU540_PRCI)	+= fu540-prci.o
diff --git a/drivers/clk/sifive/fu540-prci.c b/drivers/clk/sifive/fu540-prci.c
new file mode 100644
index 000000000000..0ec8bf7b4b28
--- /dev/null
+++ b/drivers/clk/sifive/fu540-prci.c
@@ -0,0 +1,626 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2018-2019 SiFive, Inc.
+ * Wesley Terpstra
+ * Paul Walmsley
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * The FU540 PRCI implements clock and reset control for the SiFive
+ * FU540-C000 chip.  This driver assumes that it has sole control
+ * over all PRCI resources.
+ *
+ * This driver is based on the PRCI driver written by Wesley Terpstra:
+ * https://github.com/riscv/riscv-linux/commit/999529edf517ed75b56659d456d221b2ee56bb60
+ *
+ * References:
+ * - SiFive FU540-C000 manual v1p0, Chapter 7 "Clocking and Reset"
+ */
+
+#include <dt-bindings/clock/sifive-fu540-prci.h>
+#include <linux/clkdev.h>
+#include <linux/clk-provider.h>
+#include <linux/clk/analogbits-wrpll-cln28hpc.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_clk.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+/*
+ * EXPECTED_CLK_PARENT_COUNT: how many parent clocks this driver expects:
+ *     hfclk and rtcclk
+ */
+#define EXPECTED_CLK_PARENT_COUNT		2
+
+/*
+ * Register offsets and bitmasks
+ */
+
+/* COREPLLCFG0 */
+#define PRCI_COREPLLCFG0_OFFSET			0x4
+# define PRCI_COREPLLCFG0_DIVR_SHIFT		0
+# define PRCI_COREPLLCFG0_DIVR_MASK		(0x3f << PRCI_COREPLLCFG0_DIVR_SHIFT)
+# define PRCI_COREPLLCFG0_DIVF_SHIFT		6
+# define PRCI_COREPLLCFG0_DIVF_MASK		(0x1ff << PRCI_COREPLLCFG0_DIVF_SHIFT)
+# define PRCI_COREPLLCFG0_DIVQ_SHIFT		15
+# define PRCI_COREPLLCFG0_DIVQ_MASK		(0x7 << PRCI_COREPLLCFG0_DIVQ_SHIFT)
+# define PRCI_COREPLLCFG0_RANGE_SHIFT		18
+# define PRCI_COREPLLCFG0_RANGE_MASK		(0x7 << PRCI_COREPLLCFG0_RANGE_SHIFT)
+# define PRCI_COREPLLCFG0_BYPASS_SHIFT		24
+# define PRCI_COREPLLCFG0_BYPASS_MASK		(0x1 << PRCI_COREPLLCFG0_BYPASS_SHIFT)
+# define PRCI_COREPLLCFG0_FSE_SHIFT		25
+# define PRCI_COREPLLCFG0_FSE_MASK		(0x1 << PRCI_COREPLLCFG0_FSE_SHIFT)
+# define PRCI_COREPLLCFG0_LOCK_SHIFT		31
+# define PRCI_COREPLLCFG0_LOCK_MASK		(0x1 << PRCI_COREPLLCFG0_LOCK_SHIFT)
+
+/* DDRPLLCFG0 */
+#define PRCI_DDRPLLCFG0_OFFSET			0xc
+# define PRCI_DDRPLLCFG0_DIVR_SHIFT		0
+# define PRCI_DDRPLLCFG0_DIVR_MASK		(0x3f << PRCI_DDRPLLCFG0_DIVR_SHIFT)
+# define PRCI_DDRPLLCFG0_DIVF_SHIFT		6
+# define PRCI_DDRPLLCFG0_DIVF_MASK		(0x1ff << PRCI_DDRPLLCFG0_DIVF_SHIFT)
+# define PRCI_DDRPLLCFG0_DIVQ_SHIFT		15
+# define PRCI_DDRPLLCFG0_DIVQ_MASK		(0x7 << PRCI_DDRPLLCFG0_DIVQ_SHIFT)
+# define PRCI_DDRPLLCFG0_RANGE_SHIFT		18
+# define PRCI_DDRPLLCFG0_RANGE_MASK		(0x7 << PRCI_DDRPLLCFG0_RANGE_SHIFT)
+# define PRCI_DDRPLLCFG0_BYPASS_SHIFT		24
+# define PRCI_DDRPLLCFG0_BYPASS_MASK		(0x1 << PRCI_DDRPLLCFG0_BYPASS_SHIFT)
+# define PRCI_DDRPLLCFG0_FSE_SHIFT		25
+# define PRCI_DDRPLLCFG0_FSE_MASK		(0x1 << PRCI_DDRPLLCFG0_FSE_SHIFT)
+# define PRCI_DDRPLLCFG0_LOCK_SHIFT		31
+# define PRCI_DDRPLLCFG0_LOCK_MASK		(0x1 << PRCI_DDRPLLCFG0_LOCK_SHIFT)
+
+/* DDRPLLCFG1 */
+#define PRCI_DDRPLLCFG1_OFFSET			0x10
+# define PRCI_DDRPLLCFG1_CKE_SHIFT		24
+# define PRCI_DDRPLLCFG1_CKE_MASK		(0x1 << PRCI_DDRPLLCFG1_CKE_SHIFT)
+
+/* GEMGXLPLLCFG0 */
+#define PRCI_GEMGXLPLLCFG0_OFFSET		0x1c
+# define PRCI_GEMGXLPLLCFG0_DIVR_SHIFT		0
+# define PRCI_GEMGXLPLLCFG0_DIVR_MASK		(0x3f << PRCI_GEMGXLPLLCFG0_DIVR_SHIFT)
+# define PRCI_GEMGXLPLLCFG0_DIVF_SHIFT		6
+# define PRCI_GEMGXLPLLCFG0_DIVF_MASK		(0x1ff << PRCI_GEMGXLPLLCFG0_DIVF_SHIFT)
+# define PRCI_GEMGXLPLLCFG0_DIVQ_SHIFT		15
+# define PRCI_GEMGXLPLLCFG0_DIVQ_MASK		(0x7 << PRCI_GEMGXLPLLCFG0_DIVQ_SHIFT)
+# define PRCI_GEMGXLPLLCFG0_RANGE_SHIFT		18
+# define PRCI_GEMGXLPLLCFG0_RANGE_MASK		(0x7 << PRCI_GEMGXLPLLCFG0_RANGE_SHIFT)
+# define PRCI_GEMGXLPLLCFG0_BYPASS_SHIFT	24
+# define PRCI_GEMGXLPLLCFG0_BYPASS_MASK		(0x1 << PRCI_GEMGXLPLLCFG0_BYPASS_SHIFT)
+# define PRCI_GEMGXLPLLCFG0_FSE_SHIFT		25
+# define PRCI_GEMGXLPLLCFG0_FSE_MASK		(0x1 << PRCI_GEMGXLPLLCFG0_FSE_SHIFT)
+# define PRCI_GEMGXLPLLCFG0_LOCK_SHIFT		31
+# define PRCI_GEMGXLPLLCFG0_LOCK_MASK		(0x1 << PRCI_GEMGXLPLLCFG0_LOCK_SHIFT)
+
+/* GEMGXLPLLCFG1 */
+#define PRCI_GEMGXLPLLCFG1_OFFSET		0x20
+# define PRCI_GEMGXLPLLCFG1_CKE_SHIFT		24
+# define PRCI_GEMGXLPLLCFG1_CKE_MASK		(0x1 << PRCI_GEMGXLPLLCFG1_CKE_SHIFT)
+
+/* CORECLKSEL */
+#define PRCI_CORECLKSEL_OFFSET			0x24
+# define PRCI_CORECLKSEL_CORECLKSEL_SHIFT	0
+# define PRCI_CORECLKSEL_CORECLKSEL_MASK	(0x1 << PRCI_CORECLKSEL_CORECLKSEL_SHIFT)
+
+/* DEVICESRESETREG */
+#define PRCI_DEVICESRESETREG_OFFSET			0x28
+# define PRCI_DEVICESRESETREG_DDR_CTRL_RST_N_SHIFT	0
+# define PRCI_DEVICESRESETREG_DDR_CTRL_RST_N_MASK	(0x1 << PRCI_DEVICESRESETREG_DDR_CTRL_RST_N_SHIFT)
+# define PRCI_DEVICESRESETREG_DDR_AXI_RST_N_SHIFT	1
+# define PRCI_DEVICESRESETREG_DDR_AXI_RST_N_MASK	(0x1 << PRCI_DEVICESRESETREG_DDR_AXI_RST_N_SHIFT)
+# define PRCI_DEVICESRESETREG_DDR_AHB_RST_N_SHIFT	2
+# define PRCI_DEVICESRESETREG_DDR_AHB_RST_N_MASK	(0x1 << PRCI_DEVICESRESETREG_DDR_AHB_RST_N_SHIFT)
+# define PRCI_DEVICESRESETREG_DDR_PHY_RST_N_SHIFT	3
+# define PRCI_DEVICESRESETREG_DDR_PHY_RST_N_MASK	(0x1 << PRCI_DEVICESRESETREG_DDR_PHY_RST_N_SHIFT)
+# define PRCI_DEVICESRESETREG_GEMGXL_RST_N_SHIFT	5
+# define PRCI_DEVICESRESETREG_GEMGXL_RST_N_MASK		(0x1 << PRCI_DEVICESRESETREG_GEMGXL_RST_N_SHIFT)
+
+/* CLKMUXSTATUSREG */
+#define PRCI_CLKMUXSTATUSREG_OFFSET			0x2c
+# define PRCI_CLKMUXSTATUSREG_TLCLKSEL_STATUS_SHIFT	1
+# define PRCI_CLKMUXSTATUSREG_TLCLKSEL_STATUS_MASK	(0x1 << PRCI_CLKMUXSTATUSREG_TLCLKSEL_STATUS_SHIFT)
+
+/*
+ * Private structures
+ */
+
+/**
+ * struct __prci_data - per-device-instance data
+ * @va: base virtual address of the PRCI IP block
+ * @hw_clks: encapsulates struct clk_hw records
+ *
+ * PRCI per-device instance data
+ */
+struct __prci_data {
+	void __iomem *va;
+	struct clk_hw_onecell_data hw_clks;
+};
+
+/**
+ * struct __prci_wrpll_data - WRPLL configuration and integration data
+ * @c: WRPLL current configuration record
+ * @enable_bypass: fn ptr to code to bypass the WRPLL (if applicable; else NULL)
+ * @disable_bypass: fn ptr to code to not bypass the WRPLL (or NULL)
+ * @cfg0_offs: WRPLL CFG0 register offset (in bytes) from the PRCI base address
+ *
+ * @enable_bypass and @disable_bypass are used for WRPLL instances
+ * that contain a separate external glitchless clock mux downstream
+ * from the PLL.  The WRPLL internal bypass mux is not glitchless.
+ */
+struct __prci_wrpll_data {
+	struct wrpll_cfg c;
+	void (*enable_bypass)(struct __prci_data *pd);
+	void (*disable_bypass)(struct __prci_data *pd);
+	u8 cfg0_offs;
+};
+
+/**
+ * struct __prci_clock - describes a clock device managed by PRCI
+ * @name: user-readable clock name string - should match the manual
+ * @parent_name: parent name for this clock
+ * @ops: struct clk_ops for the Linux clock framework to use for control
+ * @hw: Linux-private clock data
+ * @pwd: WRPLL-specific data, associated with this clock (if not NULL)
+ * @pd: PRCI-specific data associated with this clock (if not NULL)
+ *
+ * PRCI clock data.  Used by the PRCI driver to register PRCI-provided
+ * clocks to the Linux clock infrastructure.
+ */
+struct __prci_clock {
+	const char *name;
+	const char *parent_name;
+	const struct clk_ops *ops;
+	struct clk_hw hw;
+	struct __prci_wrpll_data *pwd;
+	struct __prci_data *pd;
+};
+
+#define clk_hw_to_prci_clock(pwd) container_of(pwd, struct __prci_clock, hw)
+
+/*
+ * Private functions
+ */
+
+/**
+ * __prci_readl() - read from a PRCI register
+ * @pd: PRCI context
+ * @offs: register offset to read from (in bytes, from PRCI base address)
+ *
+ * Read the register located at offset @offs from the base virtual
+ * address of the PRCI register target described by @pd, and return
+ * the value to the caller.
+ *
+ * Context: Any context.
+ *
+ * Return: the contents of the register described by @pd and @offs.
+ */
+static u32 __prci_readl(struct __prci_data *pd, u32 offs)
+{
+	return readl_relaxed(pd->va + offs);
+}
+
+static void __prci_writel(u32 v, u32 offs, struct __prci_data *pd)
+{
+	writel_relaxed(v, pd->va + offs);
+}
+
+/* WRPLL-related private functions */
+
+/**
+ * __prci_wrpll_unpack() - unpack WRPLL configuration registers into parameters
+ * @c: ptr to a struct wrpll_cfg record to write config into
+ * @r: value read from the PRCI PLL configuration register
+ *
+ * Given a value @r read from an FU540 PRCI PLL configuration register,
+ * split it into fields and populate it into the WRPLL configuration record
+ * pointed to by @c.
+ *
+ * The COREPLLCFG0 macros are used below, but the other *PLLCFG0 macros
+ * have the same register layout.
+ *
+ * Context: Any context.
+ */
+static void __prci_wrpll_unpack(struct wrpll_cfg *c, u32 r)
+{
+	u32 v;
+
+	v = r & PRCI_COREPLLCFG0_DIVR_MASK;
+	v >>= PRCI_COREPLLCFG0_DIVR_SHIFT;
+	c->divr = v;
+
+	v = r & PRCI_COREPLLCFG0_DIVF_MASK;
+	v >>= PRCI_COREPLLCFG0_DIVF_SHIFT;
+	c->divf = v;
+
+	v = r & PRCI_COREPLLCFG0_DIVQ_MASK;
+	v >>= PRCI_COREPLLCFG0_DIVQ_SHIFT;
+	c->divq = v;
+
+	v = r & PRCI_COREPLLCFG0_RANGE_MASK;
+	v >>= PRCI_COREPLLCFG0_RANGE_SHIFT;
+	c->range = v;
+
+	c->flags &= (WRPLL_FLAGS_INT_FEEDBACK_MASK |
+		     WRPLL_FLAGS_EXT_FEEDBACK_MASK);
+
+	/* external feedback mode not supported */
+	c->flags |= WRPLL_FLAGS_INT_FEEDBACK_MASK;
+}
+
+/**
+ * __prci_wrpll_pack() - pack PLL configuration parameters into a register value
+ * @c: pointer to a struct wrpll_cfg record containing the PLL's cfg
+ *
+ * Using a set of WRPLL configuration values pointed to by @c,
+ * assemble a PRCI PLL configuration register value, and return it to
+ * the caller.
+ *
+ * Context: Any context.  Caller must ensure that the contents of the
+ *          record pointed to by @c do not change during the execution
+ *          of this function.
+ *
+ * Returns: a value suitable for writing into a PRCI PLL configuration
+ *          register
+ */
+static u32 __prci_wrpll_pack(const struct wrpll_cfg *c)
+{
+	u32 r = 0;
+
+	r |= c->divr << PRCI_COREPLLCFG0_DIVR_SHIFT;
+	r |= c->divf << PRCI_COREPLLCFG0_DIVF_SHIFT;
+	r |= c->divq << PRCI_COREPLLCFG0_DIVQ_SHIFT;
+	r |= c->range << PRCI_COREPLLCFG0_RANGE_SHIFT;
+
+	/* external feedback mode not supported */
+	r |= PRCI_COREPLLCFG0_FSE_MASK;
+
+	return r;
+}
+
+/**
+ * __prci_wrpll_read_cfg() - read the WRPLL configuration from the PRCI
+ * @pd: PRCI context
+ * @pwd: PRCI WRPLL metadata
+ *
+ * Read the current configuration of the PLL identified by @pwd from
+ * the PRCI identified by @pd, and store it into the local configuration
+ * cache in @pwd.
+ *
+ * Context: Any context.  Caller must prevent the records pointed to by
+ *          @pd and @pwd from changing during execution.
+ */
+static void __prci_wrpll_read_cfg(struct __prci_data *pd,
+				  struct __prci_wrpll_data *pwd)
+{
+	__prci_wrpll_unpack(&pwd->c, __prci_readl(pd, pwd->cfg0_offs));
+}
+
+/**
+ * __prci_wrpll_write_cfg() - write WRPLL configuration into the PRCI
+ * @pd: PRCI context
+ * @pwd: PRCI WRPLL metadata
+ * @c: WRPLL configuration record to write
+ *
+ * Write the WRPLL configuration described by @c into the WRPLL
+ * configuration register identified by @pwd in the PRCI instance
+ * described by @c.  Make a cached copy of the WRPLL's current
+ * configuration so it can be used by other code.
+ *
+ * Context: Any context.  Caller must prevent the records pointed to by
+ *          @pd and @pwd from changing during execution.
+ */
+static void __prci_wrpll_write_cfg(struct __prci_data *pd,
+				   struct __prci_wrpll_data *pwd,
+				   struct wrpll_cfg *c)
+{
+	__prci_writel(__prci_wrpll_pack(c), pwd->cfg0_offs, pd);
+
+	memcpy(&pwd->c, c, sizeof(*c));
+}
+
+/* Core clock mux control */
+
+/**
+ * __prci_coreclksel_use_hfclk() - switch the CORECLK mux to output HFCLK
+ * @pd: struct __prci_data * for the PRCI containing the CORECLK mux reg
+ *
+ * Switch the CORECLK mux to the HFCLK input source; return once complete.
+ *
+ * Context: Any context.  Caller must prevent concurrent changes to the
+ *          PRCI_CORECLKSEL_OFFSET register.
+ */
+static void __prci_coreclksel_use_hfclk(struct __prci_data *pd)
+{
+	u32 r;
+
+	r = __prci_readl(pd, PRCI_CORECLKSEL_OFFSET);
+	r |= PRCI_CORECLKSEL_CORECLKSEL_MASK;
+	__prci_writel(r, PRCI_CORECLKSEL_OFFSET, pd);
+
+	r = __prci_readl(pd, PRCI_CORECLKSEL_OFFSET); /* barrier */
+}
+
+/**
+ * __prci_coreclksel_use_corepll() - switch the CORECLK mux to output COREPLL
+ * @pd: struct __prci_data * for the PRCI containing the CORECLK mux reg
+ *
+ * Switch the CORECLK mux to the PLL output clock; return once complete.
+ *
+ * Context: Any context.  Caller must prevent concurrent changes to the
+ *          PRCI_CORECLKSEL_OFFSET register.
+ */
+static void __prci_coreclksel_use_corepll(struct __prci_data *pd)
+{
+	u32 r;
+
+	r = __prci_readl(pd, PRCI_CORECLKSEL_OFFSET);
+	r &= ~PRCI_CORECLKSEL_CORECLKSEL_MASK;
+	__prci_writel(r, PRCI_CORECLKSEL_OFFSET, pd);
+
+	r = __prci_readl(pd, PRCI_CORECLKSEL_OFFSET); /* barrier */
+}
+
+/*
+ * Linux clock framework integration
+ *
+ * See the Linux clock framework documentation for more information on
+ * these functions.
+ */
+
+static unsigned long sifive_fu540_prci_wrpll_recalc_rate(struct clk_hw *hw,
+							 unsigned long parent_rate)
+{
+	struct __prci_clock *pc = clk_hw_to_prci_clock(hw);
+	struct __prci_wrpll_data *pwd = pc->pwd;
+
+	return wrpll_calc_output_rate(&pwd->c, parent_rate);
+}
+
+static long sifive_fu540_prci_wrpll_round_rate(struct clk_hw *hw,
+					       unsigned long rate,
+					       unsigned long *parent_rate)
+{
+	struct __prci_clock *pc = clk_hw_to_prci_clock(hw);
+	struct __prci_wrpll_data *pwd = pc->pwd;
+	struct wrpll_cfg c;
+
+	memcpy(&c, &pwd->c, sizeof(c));
+
+	wrpll_configure_for_rate(&c, rate, *parent_rate);
+
+	return wrpll_calc_output_rate(&c, *parent_rate);
+}
+
+static int sifive_fu540_prci_wrpll_set_rate(struct clk_hw *hw,
+					    unsigned long rate,
+					    unsigned long parent_rate)
+{
+	struct __prci_clock *pc = clk_hw_to_prci_clock(hw);
+	struct __prci_wrpll_data *pwd = pc->pwd;
+	struct __prci_data *pd = pc->pd;
+	int r;
+
+	r = wrpll_configure_for_rate(&pwd->c, rate, parent_rate);
+	if (r)
+		return r;
+
+	if (pwd->enable_bypass)
+		pwd->enable_bypass(pd);
+
+	__prci_wrpll_write_cfg(pd, pwd, &pwd->c);
+
+	udelay(wrpll_calc_max_lock_us(&pwd->c));
+
+	if (pwd->disable_bypass)
+		pwd->disable_bypass(pd);
+
+	return 0;
+}
+
+static const struct clk_ops sifive_fu540_prci_wrpll_clk_ops = {
+	.set_rate = sifive_fu540_prci_wrpll_set_rate,
+	.round_rate = sifive_fu540_prci_wrpll_round_rate,
+	.recalc_rate = sifive_fu540_prci_wrpll_recalc_rate,
+};
+
+static const struct clk_ops sifive_fu540_prci_wrpll_ro_clk_ops = {
+	.recalc_rate = sifive_fu540_prci_wrpll_recalc_rate,
+};
+
+/* TLCLKSEL clock integration */
+
+static unsigned long sifive_fu540_prci_tlclksel_recalc_rate(struct clk_hw *hw,
+							    unsigned long parent_rate)
+{
+	struct __prci_clock *pc = clk_hw_to_prci_clock(hw);
+	struct __prci_data *pd = pc->pd;
+	u32 v;
+	u8 div;
+
+	v = __prci_readl(pd, PRCI_CLKMUXSTATUSREG_OFFSET);
+	v &= PRCI_CLKMUXSTATUSREG_TLCLKSEL_STATUS_MASK;
+	div = v ? 1 : 2;
+
+	return div_u64(parent_rate, div);
+}
+
+static const struct clk_ops sifive_fu540_prci_tlclksel_clk_ops = {
+	.recalc_rate = sifive_fu540_prci_tlclksel_recalc_rate,
+};
+
+/*
+ * PRCI integration data for each WRPLL instance
+ */
+
+static struct __prci_wrpll_data __prci_corepll_data = {
+	.cfg0_offs = PRCI_COREPLLCFG0_OFFSET,
+	.enable_bypass = __prci_coreclksel_use_hfclk,
+	.disable_bypass = __prci_coreclksel_use_corepll,
+};
+
+static struct __prci_wrpll_data __prci_ddrpll_data = {
+	.cfg0_offs = PRCI_DDRPLLCFG0_OFFSET,
+};
+
+static struct __prci_wrpll_data __prci_gemgxlpll_data = {
+	.cfg0_offs = PRCI_GEMGXLPLLCFG0_OFFSET,
+};
+
+/*
+ * List of clock controls provided by the PRCI
+ */
+
+static struct __prci_clock __prci_init_clocks[] = {
+	[PRCI_CLK_COREPLL] = {
+		.name = "corepll",
+		.parent_name = "hfclk",
+		.ops = &sifive_fu540_prci_wrpll_clk_ops,
+		.pwd = &__prci_corepll_data,
+	},
+	[PRCI_CLK_DDRPLL] = {
+		.name = "ddrpll",
+		.parent_name = "hfclk",
+		.ops = &sifive_fu540_prci_wrpll_ro_clk_ops,
+		.pwd = &__prci_ddrpll_data,
+	},
+	[PRCI_CLK_GEMGXLPLL] = {
+		.name = "gemgxlpll",
+		.parent_name = "hfclk",
+		.ops = &sifive_fu540_prci_wrpll_clk_ops,
+		.pwd = &__prci_gemgxlpll_data,
+	},
+	[PRCI_CLK_TLCLK] = {
+		.name = "tlclk",
+		.parent_name = "corepll",
+		.ops = &sifive_fu540_prci_tlclksel_clk_ops,
+	},
+};
+
+/**
+ * __prci_register_clocks() - register clock controls in the PRCI with Linux
+ * @dev: Linux struct device *
+ *
+ * Register the list of clock controls described in __prci_init_plls[] with
+ * the Linux clock framework.
+ *
+ * Return: 0 upon success or a negative error code upon failure.
+ */
+static int __prci_register_clocks(struct device *dev, struct __prci_data *pd)
+{
+	struct clk_init_data init = { };
+	struct __prci_clock *pic;
+	int parent_count, i, r;
+
+	parent_count = of_clk_get_parent_count(dev->of_node);
+	if (parent_count != EXPECTED_CLK_PARENT_COUNT) {
+		dev_err(dev, "expected only two parent clocks, found %d\n",
+			parent_count);
+		return -EINVAL;
+	}
+
+	/* Register PLLs */
+	for (i = 0; i < ARRAY_SIZE(__prci_init_clocks); ++i) {
+		pic = &__prci_init_clocks[i];
+
+		init.name = pic->name;
+		init.parent_names = &pic->parent_name;
+		init.num_parents = 1;
+		init.ops = pic->ops;
+		pic->hw.init = &init;
+
+		pic->pd = pd;
+
+		if (pic->pwd)
+			__prci_wrpll_read_cfg(pd, pic->pwd);
+
+		r = devm_clk_hw_register(dev, &pic->hw);
+		if (r) {
+			dev_warn(dev, "Failed to register clock %s: %d\n",
+				 init.name, r);
+			return r;
+		}
+
+		r = clk_hw_register_clkdev(&pic->hw, pic->name, dev_name(dev));
+		if (r) {
+			dev_warn(dev, "Failed to register clkdev for %s: %d\n",
+				 init.name, r);
+			return r;
+		}
+
+		pd->hw_clks.hws[i] = &pic->hw;
+	}
+
+	pd->hw_clks.num = i;
+
+	r = devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get,
+					&pd->hw_clks);
+	if (r) {
+		dev_err(dev, "could not add hw_provider: %d\n", r);
+		return r;
+	}
+
+	return 0;
+}
+
+/*
+ * Linux device model integration
+ *
+ * See the Linux device model documentation for more information about
+ * these functions.
+ */
+static int sifive_fu540_prci_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct resource *res;
+	struct __prci_data *pd;
+	int r;
+
+	pd = devm_kzalloc(dev, sizeof(*pd), GFP_KERNEL);
+	if (!pd)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	pd->va = devm_ioremap_resource(dev, res);
+	if (IS_ERR(pd->va))
+		return PTR_ERR(pd->va);
+
+	r = __prci_register_clocks(dev, pd);
+	if (r) {
+		dev_err(dev, "could not register clocks: %d\n", r);
+		return r;
+	}
+
+	dev_dbg(dev, "SiFive FU540 PRCI probed\n");
+
+	return 0;
+}
+
+static const struct of_device_id sifive_fu540_prci_of_match[] = {
+	{ .compatible = "sifive,fu540-c000-prci", },
+	{}
+};
+MODULE_DEVICE_TABLE(of, sifive_fu540_prci_of_match);
+
+static struct platform_driver sifive_fu540_prci_driver = {
+	.driver	= {
+		.name = "sifive-fu540-prci",
+		.of_match_table = sifive_fu540_prci_of_match,
+	},
+	.probe = sifive_fu540_prci_probe,
+};
+
+static int __init sifive_fu540_prci_init(void)
+{
+	return platform_driver_register(&sifive_fu540_prci_driver);
+}
+core_initcall(sifive_fu540_prci_init);
diff --git a/drivers/clk/sprd/common.h b/drivers/clk/sprd/common.h
index abd9ff5ef448..1d077b39cef6 100644
--- a/drivers/clk/sprd/common.h
+++ b/drivers/clk/sprd/common.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 //
 // Spreadtrum clock infrastructure
 //
diff --git a/drivers/clk/sprd/composite.h b/drivers/clk/sprd/composite.h
index 0984e9e252dc..04ab3f587ee2 100644
--- a/drivers/clk/sprd/composite.h
+++ b/drivers/clk/sprd/composite.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 //
 // Spreadtrum composite clock driver
 //
diff --git a/drivers/clk/sprd/div.h b/drivers/clk/sprd/div.h
index b3033d24d431..87510e3d0e14 100644
--- a/drivers/clk/sprd/div.h
+++ b/drivers/clk/sprd/div.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 //
 // Spreadtrum divider clock driver
 //
diff --git a/drivers/clk/sprd/gate.h b/drivers/clk/sprd/gate.h
index 2e582c68a08b..dc352ea55e1f 100644
--- a/drivers/clk/sprd/gate.h
+++ b/drivers/clk/sprd/gate.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 //
 // Spreadtrum gate clock driver
 //
diff --git a/drivers/clk/sprd/mux.h b/drivers/clk/sprd/mux.h
index 548cfa0f145c..892e4191cc7f 100644
--- a/drivers/clk/sprd/mux.h
+++ b/drivers/clk/sprd/mux.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 //
 // Spreadtrum multiplexer clock driver
 //
diff --git a/drivers/clk/sprd/pll.h b/drivers/clk/sprd/pll.h
index 514175621099..e95f11e91ffe 100644
--- a/drivers/clk/sprd/pll.h
+++ b/drivers/clk/sprd/pll.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 //
 // Spreadtrum pll clock driver
 //
diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-a64.c b/drivers/clk/sunxi-ng/ccu-sun50i-a64.c
index 932836d26e2b..be0deee70182 100644
--- a/drivers/clk/sunxi-ng/ccu-sun50i-a64.c
+++ b/drivers/clk/sunxi-ng/ccu-sun50i-a64.c
@@ -531,7 +531,8 @@ static SUNXI_CCU_GATE(dram_ts_clk,	"dram-ts",	"dram",
 
 static const char * const de_parents[] = { "pll-periph0-2x", "pll-de" };
 static SUNXI_CCU_M_WITH_MUX_GATE(de_clk, "de", de_parents,
-				 0x104, 0, 4, 24, 3, BIT(31), 0);
+				 0x104, 0, 4, 24, 3, BIT(31),
+				 CLK_SET_RATE_PARENT);
 
 static const char * const tcon0_parents[] = { "pll-mipi", "pll-video0-2x" };
 static const u8 tcon0_table[] = { 0, 2, };
diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-h6.c b/drivers/clk/sunxi-ng/ccu-sun50i-h6.c
index 139e8389615c..3c32d7798f27 100644
--- a/drivers/clk/sunxi-ng/ccu-sun50i-h6.c
+++ b/drivers/clk/sunxi-ng/ccu-sun50i-h6.c
@@ -266,7 +266,7 @@ static SUNXI_CCU_M_WITH_MUX_GATE(de_clk, "de", de_parents, 0x600,
 				       0, 4,	/* M */
 				       24, 1,	/* mux */
 				       BIT(31),	/* gate */
-				       0);
+				       CLK_SET_RATE_PARENT);
 
 static SUNXI_CCU_GATE(bus_de_clk, "bus-de", "psi-ahb1-ahb2",
 		      0x60c, BIT(0), 0);
@@ -311,7 +311,7 @@ static SUNXI_CCU_M_WITH_MUX_GATE(ve_clk, "ve", ve_parents, 0x690,
 				       0, 3,	/* M */
 				       24, 1,	/* mux */
 				       BIT(31),	/* gate */
-				       0);
+				       CLK_SET_RATE_PARENT);
 
 static SUNXI_CCU_GATE(bus_ve_clk, "bus-ve", "psi-ahb1-ahb2",
 		      0x69c, BIT(0), 0);
@@ -656,6 +656,8 @@ static const char * const hdmi_cec_parents[] = { "osc32k", "pll-periph0-2x" };
 static const struct ccu_mux_fixed_prediv hdmi_cec_predivs[] = {
 	{ .index = 1, .div = 36621 },
 };
+
+#define SUN50I_H6_HDMI_CEC_CLK_REG		0xb10
 static struct ccu_mux hdmi_cec_clk = {
 	.enable		= BIT(31),
 
@@ -689,7 +691,7 @@ static SUNXI_CCU_MUX_WITH_GATE(tcon_lcd0_clk, "tcon-lcd0",
 			       tcon_lcd0_parents, 0xb60,
 			       24, 3,	/* mux */
 			       BIT(31),	/* gate */
-			       0);
+			       CLK_SET_RATE_PARENT);
 
 static SUNXI_CCU_GATE(bus_tcon_lcd0_clk, "bus-tcon-lcd0", "ahb3",
 		      0xb7c, BIT(0), 0);
@@ -704,7 +706,7 @@ static SUNXI_CCU_MP_WITH_MUX_GATE(tcon_tv0_clk, "tcon-tv0",
 				  8, 2,		/* P */
 				  24, 3,	/* mux */
 				  BIT(31),	/* gate */
-				  0);
+				  CLK_SET_RATE_PARENT);
 
 static SUNXI_CCU_GATE(bus_tcon_tv0_clk, "bus-tcon-tv0", "ahb3",
 		      0xb9c, BIT(0), 0);
@@ -1200,6 +1202,15 @@ static int sun50i_h6_ccu_probe(struct platform_device *pdev)
 	val &= ~(GENMASK(21, 16) | BIT(0));
 	writel(val | (7 << 16), reg + SUN50I_H6_PLL_AUDIO_REG);
 
+	/*
+	 * First clock parent (osc32K) is unusable for CEC. But since there
+	 * is no good way to force parent switch (both run with same frequency),
+	 * just set second clock parent here.
+	 */
+	val = readl(reg + SUN50I_H6_HDMI_CEC_CLK_REG);
+	val |= BIT(24);
+	writel(val, reg + SUN50I_H6_HDMI_CEC_CLK_REG);
+
 	return sunxi_ccu_probe(pdev->dev.of_node, reg, &sun50i_h6_ccu_desc);
 }
 
diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-h6.h b/drivers/clk/sunxi-ng/ccu-sun50i-h6.h
index 2ccfe4428260..9406f9a6a8aa 100644
--- a/drivers/clk/sunxi-ng/ccu-sun50i-h6.h
+++ b/drivers/clk/sunxi-ng/ccu-sun50i-h6.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright 2016 Icenowy Zheng <icenowy@aosc.io>
  */
diff --git a/drivers/clk/sunxi-ng/ccu-sun5i.h b/drivers/clk/sunxi-ng/ccu-sun5i.h
index 93a275fbd9a9..b66abd4fd0bf 100644
--- a/drivers/clk/sunxi-ng/ccu-sun5i.h
+++ b/drivers/clk/sunxi-ng/ccu-sun5i.h
@@ -60,10 +60,6 @@
 
 /* The rest of the module clocks are exported */
 
-#define CLK_MBUS		99
-
-/* And finally the IEP clock */
-
 #define CLK_NUMBER		(CLK_IEP + 1)
 
 #endif /* _CCU_SUN5I_H_ */
diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-a83t.c b/drivers/clk/sunxi-ng/ccu-sun8i-a83t.c
index 2d6555d73170..5f714b4d8ee4 100644
--- a/drivers/clk/sunxi-ng/ccu-sun8i-a83t.c
+++ b/drivers/clk/sunxi-ng/ccu-sun8i-a83t.c
@@ -513,8 +513,9 @@ static SUNXI_CCU_GATE(csi_misc_clk, "csi-misc", "osc24M", 0x130, BIT(16), 0);
 
 static SUNXI_CCU_GATE(mipi_csi_clk, "mipi-csi", "osc24M", 0x130, BIT(31), 0);
 
-static const char * const csi_mclk_parents[] = { "pll-de", "osc24M" };
-static const u8 csi_mclk_table[] = { 3, 5 };
+static const char * const csi_mclk_parents[] = { "pll-video0", "pll-de",
+						 "osc24M" };
+static const u8 csi_mclk_table[] = { 0, 3, 5 };
 static SUNXI_CCU_M_WITH_MUX_TABLE_GATE(csi_mclk_clk, "csi-mclk",
 				       csi_mclk_parents, csi_mclk_table,
 				       0x134,
diff --git a/drivers/clk/sunxi-ng/ccu-sun8i-v3s.c b/drivers/clk/sunxi-ng/ccu-sun8i-v3s.c
index ac12f261f8ca..eada0e291859 100644
--- a/drivers/clk/sunxi-ng/ccu-sun8i-v3s.c
+++ b/drivers/clk/sunxi-ng/ccu-sun8i-v3s.c
@@ -325,7 +325,8 @@ static SUNXI_CCU_GATE(dram_ohci_clk,	"dram-ohci",	"dram",
 
 static const char * const de_parents[] = { "pll-video", "pll-periph0" };
 static SUNXI_CCU_M_WITH_MUX_GATE(de_clk, "de", de_parents,
-				 0x104, 0, 4, 24, 2, BIT(31), 0);
+				 0x104, 0, 4, 24, 2, BIT(31),
+				 CLK_SET_RATE_PARENT);
 
 static const char * const tcon_parents[] = { "pll-video" };
 static SUNXI_CCU_M_WITH_MUX_GATE(tcon_clk, "tcon", tcon_parents,
diff --git a/drivers/clk/sunxi-ng/ccu-suniv-f1c100s.c b/drivers/clk/sunxi-ng/ccu-suniv-f1c100s.c
index a09dfbe36402..dc9f0a365664 100644
--- a/drivers/clk/sunxi-ng/ccu-suniv-f1c100s.c
+++ b/drivers/clk/sunxi-ng/ccu-suniv-f1c100s.c
@@ -240,7 +240,7 @@ static SUNXI_CCU_MUX_WITH_GATE(spdif_clk, "spdif", i2s_spdif_parents,
 /* The BSP header file has a CIR_CFG, but no mod clock uses this definition */
 
 static SUNXI_CCU_GATE(usb_phy0_clk,	"usb-phy0",	"osc24M",
-		      0x0cc, BIT(8), 0);
+		      0x0cc, BIT(1), 0);
 
 static SUNXI_CCU_GATE(dram_ve_clk,	"dram-ve",	"pll-ddr",
 		      0x100, BIT(0), 0);
diff --git a/drivers/clk/sunxi-ng/ccu-suniv-f1c100s.h b/drivers/clk/sunxi-ng/ccu-suniv-f1c100s.h
index 39d06fed55b2..b22484f1bb9a 100644
--- a/drivers/clk/sunxi-ng/ccu-suniv-f1c100s.h
+++ b/drivers/clk/sunxi-ng/ccu-suniv-f1c100s.h
@@ -1,5 +1,5 @@
-/* SPDX-License-Identifier: GPL-2.0+
- *
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
  * Copyright 2017 Icenowy Zheng <icenowy@aosc.io>
  *
  */
diff --git a/drivers/clk/sunxi/Kconfig b/drivers/clk/sunxi/Kconfig
new file mode 100644
index 000000000000..2b6207cc4eda
--- /dev/null
+++ b/drivers/clk/sunxi/Kconfig
@@ -0,0 +1,43 @@
+menuconfig CLK_SUNXI
+	bool "Legacy clock support for Allwinner SoCs"
+	depends on ARCH_SUNXI || COMPILE_TEST
+	default y
+
+if CLK_SUNXI
+
+config CLK_SUNXI_CLOCKS
+	bool "Legacy clock drivers"
+	default y
+	help
+	  Legacy clock drivers being used on older (A10, A13, A20,
+	  A23, A31, A80) SoCs. These drivers are kept around for
+	  Device Tree backward compatibility issues, in case one would
+	  still use a Device Tree with one clock provider by
+	  node. Newer Device Trees and newer SoCs use the drivers
+	  controlled by CONFIG_SUNXI_CCU.
+
+config CLK_SUNXI_PRCM_SUN6I
+	bool "Legacy A31 PRCM driver"
+	select MFD_SUN6I_PRCM
+	default y
+	help
+	  Legacy clock driver for the A31 PRCM clocks. Those are
+	  usually needed for the PMIC communication, mostly.
+
+config CLK_SUNXI_PRCM_SUN8I
+	bool "Legacy sun8i PRCM driver"
+	select MFD_SUN6I_PRCM
+	default y
+	help
+	  Legacy clock driver for the sun8i family PRCM clocks.
+	  Those are usually needed for the PMIC communication,
+	  mostly.
+
+config CLK_SUNXI_PRCM_SUN9I
+	bool "Legacy A80 PRCM driver"
+	default y
+	help
+	  Legacy clock driver for the A80 PRCM clocks. Those are
+	  usually needed for the PMIC communication, mostly.
+
+endif
diff --git a/drivers/clk/sunxi/Makefile b/drivers/clk/sunxi/Makefile
index be88368b48a1..e10824c76ae9 100644
--- a/drivers/clk/sunxi/Makefile
+++ b/drivers/clk/sunxi/Makefile
@@ -3,27 +3,32 @@
 # Makefile for sunxi specific clk
 #
 
-obj-y += clk-sunxi.o clk-factors.o
-obj-y += clk-a10-codec.o
-obj-y += clk-a10-hosc.o
-obj-y += clk-a10-mod1.o
-obj-y += clk-a10-pll2.o
-obj-y += clk-a10-ve.o
-obj-y += clk-a20-gmac.o
-obj-y += clk-mod0.o
-obj-y += clk-simple-gates.o
-obj-y += clk-sun4i-display.o
-obj-y += clk-sun4i-pll3.o
-obj-y += clk-sun4i-tcon-ch1.o
-obj-y += clk-sun8i-bus-gates.o
-obj-y += clk-sun8i-mbus.o
-obj-y += clk-sun9i-core.o
-obj-y += clk-sun9i-mmc.o
-obj-y += clk-usb.o
+obj-$(CONFIG_CLK_SUNXI) += clk-factors.o
 
-obj-$(CONFIG_MACH_SUN9I) += clk-sun8i-apb0.o
-obj-$(CONFIG_MACH_SUN9I) += clk-sun9i-cpus.o
+obj-$(CONFIG_CLK_SUNXI_CLOCKS) += clk-sunxi.o
+obj-$(CONFIG_CLK_SUNXI_CLOCKS) += clk-a10-codec.o
+obj-$(CONFIG_CLK_SUNXI_CLOCKS) += clk-a10-hosc.o
+obj-$(CONFIG_CLK_SUNXI_CLOCKS) += clk-a10-mod1.o
+obj-$(CONFIG_CLK_SUNXI_CLOCKS) += clk-a10-pll2.o
+obj-$(CONFIG_CLK_SUNXI_CLOCKS) += clk-a10-ve.o
+obj-$(CONFIG_CLK_SUNXI_CLOCKS) += clk-a20-gmac.o
+obj-$(CONFIG_CLK_SUNXI_CLOCKS) += clk-mod0.o
+obj-$(CONFIG_CLK_SUNXI_CLOCKS) += clk-simple-gates.o
+obj-$(CONFIG_CLK_SUNXI_CLOCKS) += clk-sun4i-display.o
+obj-$(CONFIG_CLK_SUNXI_CLOCKS) += clk-sun4i-pll3.o
+obj-$(CONFIG_CLK_SUNXI_CLOCKS) += clk-sun4i-tcon-ch1.o
+obj-$(CONFIG_CLK_SUNXI_CLOCKS) += clk-sun8i-bus-gates.o
+obj-$(CONFIG_CLK_SUNXI_CLOCKS) += clk-sun8i-mbus.o
+obj-$(CONFIG_CLK_SUNXI_CLOCKS) += clk-sun9i-core.o
+obj-$(CONFIG_CLK_SUNXI_CLOCKS) += clk-sun9i-mmc.o
+obj-$(CONFIG_CLK_SUNXI_CLOCKS) += clk-usb.o
 
-obj-$(CONFIG_MFD_SUN6I_PRCM) += \
-	clk-sun6i-ar100.o clk-sun6i-apb0.o clk-sun6i-apb0-gates.o \
-	clk-sun8i-apb0.o
+obj-$(CONFIG_CLK_SUNXI_CLOCKS) += clk-sun8i-apb0.o
+obj-$(CONFIG_CLK_SUNXI_CLOCKS) += clk-sun9i-cpus.o
+
+obj-$(CONFIG_CLK_SUNXI_PRCM_SUN6I)	+= clk-sun6i-apb0.o
+obj-$(CONFIG_CLK_SUNXI_PRCM_SUN6I)	+= clk-sun6i-apb0-gates.o
+obj-$(CONFIG_CLK_SUNXI_PRCM_SUN6I)	+= clk-sun6i-ar100.o
+
+obj-$(CONFIG_CLK_SUNXI_PRCM_SUN8I)	+= clk-sun8i-apb0.o
+obj-$(CONFIG_CLK_SUNXI_PRCM_SUN8I)	+= clk-sun6i-apb0-gates.o
diff --git a/drivers/clk/tegra/clk-divider.c b/drivers/clk/tegra/clk-divider.c
index 205fe8ff63f0..2a1822a22740 100644
--- a/drivers/clk/tegra/clk-divider.c
+++ b/drivers/clk/tegra/clk-divider.c
@@ -175,6 +175,7 @@ struct clk *tegra_clk_register_mc(const char *name, const char *parent_name,
 				  void __iomem *reg, spinlock_t *lock)
 {
 	return clk_register_divider_table(NULL, name, parent_name,
-					  CLK_IS_CRITICAL, reg, 16, 1, 0,
+					  CLK_IS_CRITICAL,
+					  reg, 16, 1, CLK_DIVIDER_READ_ONLY,
 					  mc_div_table, lock);
 }
diff --git a/drivers/clk/tegra/clk-emc.c b/drivers/clk/tegra/clk-emc.c
index 0621a3a82ea6..93ecb538e59b 100644
--- a/drivers/clk/tegra/clk-emc.c
+++ b/drivers/clk/tegra/clk-emc.c
@@ -121,18 +121,28 @@ static int emc_determine_rate(struct clk_hw *hw, struct clk_rate_request *req)
 	struct tegra_clk_emc *tegra;
 	u8 ram_code = tegra_read_ram_code();
 	struct emc_timing *timing = NULL;
-	int i;
+	int i, k, t;
 
 	tegra = container_of(hw, struct tegra_clk_emc, hw);
 
-	for (i = 0; i < tegra->num_timings; i++) {
-		if (tegra->timings[i].ram_code != ram_code)
-			continue;
+	for (k = 0; k < tegra->num_timings; k++) {
+		if (tegra->timings[k].ram_code == ram_code)
+			break;
+	}
+
+	for (t = k; t < tegra->num_timings; t++) {
+		if (tegra->timings[t].ram_code != ram_code)
+			break;
+	}
 
+	for (i = k; i < t; i++) {
 		timing = tegra->timings + i;
 
+		if (timing->rate < req->rate && i != t - 1)
+			continue;
+
 		if (timing->rate > req->max_rate) {
-			i = max(i, 1);
+			i = max(i, k + 1);
 			req->rate = tegra->timings[i - 1].rate;
 			return 0;
 		}
@@ -140,10 +150,8 @@ static int emc_determine_rate(struct clk_hw *hw, struct clk_rate_request *req)
 		if (timing->rate < req->min_rate)
 			continue;
 
-		if (timing->rate >= req->rate) {
-			req->rate = timing->rate;
-			return 0;
-		}
+		req->rate = timing->rate;
+		return 0;
 	}
 
 	if (timing) {
@@ -214,7 +222,10 @@ static int emc_set_timing(struct tegra_clk_emc *tegra,
 
 	if (emc_get_parent(&tegra->hw) == timing->parent_index &&
 	    clk_get_rate(timing->parent) != timing->parent_rate) {
-		BUG();
+		WARN_ONCE(1, "parent %s rate mismatch %lu %lu\n",
+			  __clk_get_name(timing->parent),
+			  clk_get_rate(timing->parent),
+			  timing->parent_rate);
 		return -EINVAL;
 	}
 
@@ -282,7 +293,7 @@ static struct emc_timing *get_backup_timing(struct tegra_clk_emc *tegra,
 	for (i = timing_index+1; i < tegra->num_timings; i++) {
 		timing = tegra->timings + i;
 		if (timing->ram_code != ram_code)
-			continue;
+			break;
 
 		if (emc_parent_clk_sources[timing->parent_index] !=
 		    emc_parent_clk_sources[
@@ -293,7 +304,7 @@ static struct emc_timing *get_backup_timing(struct tegra_clk_emc *tegra,
 	for (i = timing_index-1; i >= 0; --i) {
 		timing = tegra->timings + i;
 		if (timing->ram_code != ram_code)
-			continue;
+			break;
 
 		if (emc_parent_clk_sources[timing->parent_index] !=
 		    emc_parent_clk_sources[
@@ -433,19 +444,23 @@ static int load_timings_from_dt(struct tegra_clk_emc *tegra,
 				struct device_node *node,
 				u32 ram_code)
 {
+	struct emc_timing *timings_ptr;
 	struct device_node *child;
 	int child_count = of_get_child_count(node);
 	int i = 0, err;
+	size_t size;
 
-	tegra->timings = kcalloc(child_count, sizeof(struct emc_timing),
-				 GFP_KERNEL);
+	size = (tegra->num_timings + child_count) * sizeof(struct emc_timing);
+
+	tegra->timings = krealloc(tegra->timings, size, GFP_KERNEL);
 	if (!tegra->timings)
 		return -ENOMEM;
 
-	tegra->num_timings = child_count;
+	timings_ptr = tegra->timings + tegra->num_timings;
+	tegra->num_timings += child_count;
 
 	for_each_child_of_node(node, child) {
-		struct emc_timing *timing = tegra->timings + (i++);
+		struct emc_timing *timing = timings_ptr + (i++);
 
 		err = load_one_timing_from_dt(tegra, timing, child);
 		if (err) {
@@ -456,7 +471,7 @@ static int load_timings_from_dt(struct tegra_clk_emc *tegra,
 		timing->ram_code = ram_code;
 	}
 
-	sort(tegra->timings, tegra->num_timings, sizeof(struct emc_timing),
+	sort(timings_ptr, child_count, sizeof(struct emc_timing),
 	     cmp_timings, NULL);
 
 	return 0;
@@ -499,10 +514,10 @@ struct clk *tegra_clk_register_emc(void __iomem *base, struct device_node *np,
 		 * fuses until the apbmisc driver is loaded.
 		 */
 		err = load_timings_from_dt(tegra, node, node_ram_code);
-		of_node_put(node);
-		if (err)
+		if (err) {
+			of_node_put(node);
 			return ERR_PTR(err);
-		break;
+		}
 	}
 
 	if (tegra->num_timings == 0)
@@ -532,7 +547,5 @@ struct clk *tegra_clk_register_emc(void __iomem *base, struct device_node *np,
 	/* Allow debugging tools to see the EMC clock */
 	clk_register_clkdev(clk, "emc", "tegra-clk-debug");
 
-	clk_prepare_enable(clk);
-
 	return clk;
 };
diff --git a/drivers/clk/tegra/clk-pll.c b/drivers/clk/tegra/clk-pll.c
index b50b7460014b..6b976b2514f7 100644
--- a/drivers/clk/tegra/clk-pll.c
+++ b/drivers/clk/tegra/clk-pll.c
@@ -444,6 +444,9 @@ static int clk_pll_enable(struct clk_hw *hw)
 	unsigned long flags = 0;
 	int ret;
 
+	if (clk_pll_is_enabled(hw))
+		return 0;
+
 	if (pll->lock)
 		spin_lock_irqsave(pll->lock, flags);
 
@@ -663,8 +666,8 @@ static void _update_pll_mnp(struct tegra_clk_pll *pll,
 		pll_override_writel(val, params->pmc_divp_reg, pll);
 
 		val = pll_override_readl(params->pmc_divnm_reg, pll);
-		val &= ~(divm_mask(pll) << div_nmp->override_divm_shift) |
-			~(divn_mask(pll) << div_nmp->override_divn_shift);
+		val &= ~((divm_mask(pll) << div_nmp->override_divm_shift) |
+			(divn_mask(pll) << div_nmp->override_divn_shift));
 		val |= (cfg->m << div_nmp->override_divm_shift) |
 			(cfg->n << div_nmp->override_divn_shift);
 		pll_override_writel(val, params->pmc_divnm_reg, pll);
@@ -940,11 +943,16 @@ static int clk_plle_training(struct tegra_clk_pll *pll)
 static int clk_plle_enable(struct clk_hw *hw)
 {
 	struct tegra_clk_pll *pll = to_clk_pll(hw);
-	unsigned long input_rate = clk_hw_get_rate(clk_hw_get_parent(hw));
 	struct tegra_clk_pll_freq_table sel;
+	unsigned long input_rate;
 	u32 val;
 	int err;
 
+	if (clk_pll_is_enabled(hw))
+		return 0;
+
+	input_rate = clk_hw_get_rate(clk_hw_get_parent(hw));
+
 	if (_get_table_rate(hw, &sel, pll->params->fixed_rate, input_rate))
 		return -EINVAL;
 
@@ -1355,6 +1363,9 @@ static int clk_pllc_enable(struct clk_hw *hw)
 	int ret;
 	unsigned long flags = 0;
 
+	if (clk_pll_is_enabled(hw))
+		return 0;
+
 	if (pll->lock)
 		spin_lock_irqsave(pll->lock, flags);
 
@@ -1567,7 +1578,12 @@ static int clk_plle_tegra114_enable(struct clk_hw *hw)
 	u32 val;
 	int ret;
 	unsigned long flags = 0;
-	unsigned long input_rate = clk_hw_get_rate(clk_hw_get_parent(hw));
+	unsigned long input_rate;
+
+	if (clk_pll_is_enabled(hw))
+		return 0;
+
+	input_rate = clk_hw_get_rate(clk_hw_get_parent(hw));
 
 	if (_get_table_rate(hw, &sel, pll->params->fixed_rate, input_rate))
 		return -EINVAL;
@@ -1704,6 +1720,9 @@ static int clk_pllu_tegra114_enable(struct clk_hw *hw)
 		return -EINVAL;
 	}
 
+	if (clk_pll_is_enabled(hw))
+		return 0;
+
 	input_rate = clk_hw_get_rate(__clk_get_hw(osc));
 
 	if (pll->lock)
@@ -2379,6 +2398,16 @@ struct clk *tegra_clk_register_pllre_tegra210(const char *name,
 	return clk;
 }
 
+static int clk_plle_tegra210_is_enabled(struct clk_hw *hw)
+{
+	struct tegra_clk_pll *pll = to_clk_pll(hw);
+	u32 val;
+
+	val = pll_readl_base(pll);
+
+	return val & PLLE_BASE_ENABLE ? 1 : 0;
+}
+
 static int clk_plle_tegra210_enable(struct clk_hw *hw)
 {
 	struct tegra_clk_pll *pll = to_clk_pll(hw);
@@ -2386,7 +2415,12 @@ static int clk_plle_tegra210_enable(struct clk_hw *hw)
 	u32 val;
 	int ret = 0;
 	unsigned long flags = 0;
-	unsigned long input_rate = clk_hw_get_rate(clk_hw_get_parent(hw));
+	unsigned long input_rate;
+
+	if (clk_plle_tegra210_is_enabled(hw))
+		return 0;
+
+	input_rate = clk_hw_get_rate(clk_hw_get_parent(hw));
 
 	if (_get_table_rate(hw, &sel, pll->params->fixed_rate, input_rate))
 		return -EINVAL;
@@ -2497,16 +2531,6 @@ out:
 		spin_unlock_irqrestore(pll->lock, flags);
 }
 
-static int clk_plle_tegra210_is_enabled(struct clk_hw *hw)
-{
-	struct tegra_clk_pll *pll = to_clk_pll(hw);
-	u32 val;
-
-	val = pll_readl_base(pll);
-
-	return val & PLLE_BASE_ENABLE ? 1 : 0;
-}
-
 static const struct clk_ops tegra_clk_plle_tegra210_ops = {
 	.is_enabled =  clk_plle_tegra210_is_enabled,
 	.enable = clk_plle_tegra210_enable,
diff --git a/drivers/clk/tegra/clk-super.c b/drivers/clk/tegra/clk-super.c
index 84267cfc4433..b5ff76c663f8 100644
--- a/drivers/clk/tegra/clk-super.c
+++ b/drivers/clk/tegra/clk-super.c
@@ -121,7 +121,7 @@ out:
 	return err;
 }
 
-const struct clk_ops tegra_clk_super_mux_ops = {
+static const struct clk_ops tegra_clk_super_mux_ops = {
 	.get_parent = clk_super_get_parent,
 	.set_parent = clk_super_set_parent,
 };
diff --git a/drivers/clk/tegra/clk-tegra124.c b/drivers/clk/tegra/clk-tegra124.c
index df0018f7bf7e..d7bee144f4b7 100644
--- a/drivers/clk/tegra/clk-tegra124.c
+++ b/drivers/clk/tegra/clk-tegra124.c
@@ -413,7 +413,6 @@ static struct tegra_clk_pll_params pll_m_params = {
 	.base_reg = PLLM_BASE,
 	.misc_reg = PLLM_MISC,
 	.lock_mask = PLL_BASE_LOCK,
-	.lock_enable_bit_idx = PLL_MISC_LOCK_ENABLE,
 	.lock_delay = 300,
 	.max_p = 5,
 	.pdiv_tohw = pllm_p,
@@ -421,7 +420,7 @@ static struct tegra_clk_pll_params pll_m_params = {
 	.pmc_divnm_reg = PMC_PLLM_WB0_OVERRIDE,
 	.pmc_divp_reg = PMC_PLLM_WB0_OVERRIDE_2,
 	.freq_table = pll_m_freq_table,
-	.flags = TEGRA_PLL_USE_LOCK | TEGRA_PLL_HAS_LOCK_ENABLE,
+	.flags = TEGRA_PLL_USE_LOCK,
 };
 
 static struct tegra_clk_pll_freq_table pll_e_freq_table[] = {
@@ -1466,9 +1465,9 @@ static void __init tegra124_132_clock_init_pre(struct device_node *np)
 	tegra_pmc_clk_init(pmc_base, tegra124_clks);
 
 	/* For Tegra124 & Tegra132, PLLD is the only source for DSIA & DSIB */
-	plld_base = clk_readl(clk_base + PLLD_BASE);
+	plld_base = readl(clk_base + PLLD_BASE);
 	plld_base &= ~BIT(25);
-	clk_writel(plld_base, clk_base + PLLD_BASE);
+	writel(plld_base, clk_base + PLLD_BASE);
 }
 
 /**
diff --git a/drivers/clk/tegra/clk-tegra210.c b/drivers/clk/tegra/clk-tegra210.c
index 7545af763d7a..ed3c7df75d1e 100644
--- a/drivers/clk/tegra/clk-tegra210.c
+++ b/drivers/clk/tegra/clk-tegra210.c
@@ -3557,7 +3557,7 @@ static void __init tegra210_clock_init(struct device_node *np)
 	if (!clks)
 		return;
 
-	value = clk_readl(clk_base + SPARE_REG0) >> CLK_M_DIVISOR_SHIFT;
+	value = readl(clk_base + SPARE_REG0) >> CLK_M_DIVISOR_SHIFT;
 	clk_m_div = (value & CLK_M_DIVISOR_MASK) + 1;
 
 	if (tegra_osc_clk_init(clk_base, tegra210_clks, tegra210_input_freq,
@@ -3574,9 +3574,9 @@ static void __init tegra210_clock_init(struct device_node *np)
 	tegra_pmc_clk_init(pmc_base, tegra210_clks);
 
 	/* For Tegra210, PLLD is the only source for DSIA & DSIB */
-	value = clk_readl(clk_base + PLLD_BASE);
+	value = readl(clk_base + PLLD_BASE);
 	value &= ~BIT(25);
-	clk_writel(value, clk_base + PLLD_BASE);
+	writel(value, clk_base + PLLD_BASE);
 
 	tegra_clk_apply_init_table = tegra210_clock_apply_init_table;
 
diff --git a/drivers/clk/ti/clk-7xx-compat.c b/drivers/clk/ti/clk-7xx-compat.c
index e3cb7f0b03ae..b3cd2296f84b 100644
--- a/drivers/clk/ti/clk-7xx-compat.c
+++ b/drivers/clk/ti/clk-7xx-compat.c
@@ -362,7 +362,7 @@ static const struct omap_clkctrl_reg_data dra7_l3init_clkctrl_regs[] __initconst
 	{ DRA7_MMC2_CLKCTRL, dra7_mmc2_bit_data, CLKF_SW_SUP, "l3init_cm:clk:0010:25" },
 	{ DRA7_USB_OTG_SS2_CLKCTRL, dra7_usb_otg_ss2_bit_data, CLKF_HW_SUP, "dpll_core_h13x2_ck" },
 	{ DRA7_USB_OTG_SS3_CLKCTRL, NULL, CLKF_HW_SUP, "dpll_core_h13x2_ck" },
-	{ DRA7_USB_OTG_SS4_CLKCTRL, NULL, CLKF_HW_SUP, "dpll_core_h13x2_ck" },
+	{ DRA7_USB_OTG_SS4_CLKCTRL, NULL, CLKF_HW_SUP | CLKF_SOC_DRA74 | CLKF_SOC_DRA76, "dpll_core_h13x2_ck" },
 	{ DRA7_SATA_CLKCTRL, dra7_sata_bit_data, CLKF_SW_SUP, "func_48m_fclk" },
 	{ DRA7_PCIE1_CLKCTRL, dra7_pcie1_bit_data, CLKF_SW_SUP, "l4_root_clk_div", "pcie_clkdm" },
 	{ DRA7_PCIE2_CLKCTRL, dra7_pcie2_bit_data, CLKF_SW_SUP, "l4_root_clk_div", "pcie_clkdm" },
@@ -662,7 +662,7 @@ static const struct omap_clkctrl_reg_data dra7_l4per_clkctrl_regs[] __initconst
 	{ DRA7_AES1_CLKCTRL, NULL, CLKF_HW_SUP, "l3_iclk_div", "l4sec_clkdm" },
 	{ DRA7_AES2_CLKCTRL, NULL, CLKF_HW_SUP, "l3_iclk_div", "l4sec_clkdm" },
 	{ DRA7_DES_CLKCTRL, NULL, CLKF_HW_SUP, "l3_iclk_div", "l4sec_clkdm" },
-	{ DRA7_RNG_CLKCTRL, NULL, CLKF_HW_SUP, "l3_iclk_div", "l4sec_clkdm" },
+	{ DRA7_RNG_CLKCTRL, NULL, CLKF_HW_SUP | CLKF_SOC_NONSEC, "l3_iclk_div", "l4sec_clkdm" },
 	{ DRA7_SHAM_CLKCTRL, NULL, CLKF_HW_SUP, "l3_iclk_div", "l4sec_clkdm" },
 	{ DRA7_UART7_CLKCTRL, dra7_uart7_bit_data, CLKF_SW_SUP, "l4per_cm:clk:01d0:24", "l4per2_clkdm" },
 	{ DRA7_UART8_CLKCTRL, dra7_uart8_bit_data, CLKF_SW_SUP, "l4per_cm:clk:01e0:24", "l4per2_clkdm" },
@@ -704,7 +704,7 @@ static const struct omap_clkctrl_reg_data dra7_wkupaon_clkctrl_regs[] __initcons
 	{ DRA7_WD_TIMER2_CLKCTRL, NULL, CLKF_SW_SUP, "sys_32k_ck" },
 	{ DRA7_GPIO1_CLKCTRL, dra7_gpio1_bit_data, CLKF_HW_SUP, "wkupaon_iclk_mux" },
 	{ DRA7_TIMER1_CLKCTRL, dra7_timer1_bit_data, CLKF_SW_SUP, "wkupaon_cm:clk:0020:24" },
-	{ DRA7_TIMER12_CLKCTRL, NULL, 0, "secure_32k_clk_src_ck" },
+	{ DRA7_TIMER12_CLKCTRL, NULL, CLKF_SOC_NONSEC, "secure_32k_clk_src_ck" },
 	{ DRA7_COUNTER_32K_CLKCTRL, NULL, 0, "wkupaon_iclk_mux" },
 	{ DRA7_UART10_CLKCTRL, dra7_uart10_bit_data, CLKF_SW_SUP, "wkupaon_cm:clk:0060:24" },
 	{ DRA7_DCAN1_CLKCTRL, dra7_dcan1_bit_data, CLKF_SW_SUP, "wkupaon_cm:clk:0068:24" },
diff --git a/drivers/clk/ti/clk-7xx.c b/drivers/clk/ti/clk-7xx.c
index 597fb4a59318..79186b918d87 100644
--- a/drivers/clk/ti/clk-7xx.c
+++ b/drivers/clk/ti/clk-7xx.c
@@ -348,7 +348,7 @@ static const struct omap_clkctrl_reg_data dra7_l3init_clkctrl_regs[] __initconst
 	{ DRA7_L3INIT_MMC2_CLKCTRL, dra7_mmc2_bit_data, CLKF_SW_SUP, "l3init-clkctrl:0010:25" },
 	{ DRA7_L3INIT_USB_OTG_SS2_CLKCTRL, dra7_usb_otg_ss2_bit_data, CLKF_HW_SUP, "dpll_core_h13x2_ck" },
 	{ DRA7_L3INIT_USB_OTG_SS3_CLKCTRL, NULL, CLKF_HW_SUP, "dpll_core_h13x2_ck" },
-	{ DRA7_L3INIT_USB_OTG_SS4_CLKCTRL, NULL, CLKF_HW_SUP, "dpll_core_h13x2_ck" },
+	{ DRA7_L3INIT_USB_OTG_SS4_CLKCTRL, NULL, CLKF_HW_SUP | CLKF_SOC_DRA74 | CLKF_SOC_DRA76, "dpll_core_h13x2_ck" },
 	{ DRA7_L3INIT_SATA_CLKCTRL, dra7_sata_bit_data, CLKF_SW_SUP, "func_48m_fclk" },
 	{ DRA7_L3INIT_OCP2SCP1_CLKCTRL, NULL, CLKF_HW_SUP, "l4_root_clk_div" },
 	{ DRA7_L3INIT_OCP2SCP3_CLKCTRL, NULL, CLKF_HW_SUP, "l4_root_clk_div" },
@@ -590,7 +590,7 @@ static const struct omap_clkctrl_reg_data dra7_l4sec_clkctrl_regs[] __initconst
 	{ DRA7_L4SEC_AES1_CLKCTRL, NULL, CLKF_HW_SUP, "l3_iclk_div" },
 	{ DRA7_L4SEC_AES2_CLKCTRL, NULL, CLKF_HW_SUP, "l3_iclk_div" },
 	{ DRA7_L4SEC_DES_CLKCTRL, NULL, CLKF_HW_SUP, "l3_iclk_div" },
-	{ DRA7_L4SEC_RNG_CLKCTRL, NULL, CLKF_HW_SUP, "" },
+	{ DRA7_L4SEC_RNG_CLKCTRL, NULL, CLKF_HW_SUP | CLKF_SOC_NONSEC, "" },
 	{ DRA7_L4SEC_SHAM_CLKCTRL, NULL, CLKF_HW_SUP, "l3_iclk_div" },
 	{ 0 },
 };
@@ -757,7 +757,7 @@ static const struct omap_clkctrl_reg_data dra7_wkupaon_clkctrl_regs[] __initcons
 	{ DRA7_WKUPAON_WD_TIMER2_CLKCTRL, NULL, CLKF_SW_SUP, "sys_32k_ck" },
 	{ DRA7_WKUPAON_GPIO1_CLKCTRL, dra7_gpio1_bit_data, CLKF_HW_SUP, "wkupaon_iclk_mux" },
 	{ DRA7_WKUPAON_TIMER1_CLKCTRL, dra7_timer1_bit_data, CLKF_SW_SUP, "wkupaon-clkctrl:0020:24" },
-	{ DRA7_WKUPAON_TIMER12_CLKCTRL, NULL, 0, "secure_32k_clk_src_ck" },
+	{ DRA7_WKUPAON_TIMER12_CLKCTRL, NULL, CLKF_SOC_NONSEC, "secure_32k_clk_src_ck" },
 	{ DRA7_WKUPAON_COUNTER_32K_CLKCTRL, NULL, 0, "wkupaon_iclk_mux" },
 	{ DRA7_WKUPAON_UART10_CLKCTRL, dra7_uart10_bit_data, CLKF_SW_SUP, "wkupaon-clkctrl:0060:24" },
 	{ DRA7_WKUPAON_DCAN1_CLKCTRL, dra7_dcan1_bit_data, CLKF_SW_SUP, "wkupaon-clkctrl:0068:24" },
diff --git a/drivers/clk/ti/clkctrl.c b/drivers/clk/ti/clkctrl.c
index 639f515e08f0..96d65a1cf7be 100644
--- a/drivers/clk/ti/clkctrl.c
+++ b/drivers/clk/ti/clkctrl.c
@@ -446,6 +446,7 @@ static void __init _ti_omap4_clkctrl_setup(struct device_node *node)
 	u32 addr;
 	int ret;
 	char *c;
+	u16 soc_mask = 0;
 
 	if (!(ti_clk_get_features()->flags & TI_CLK_CLKCTRL_COMPAT) &&
 	    of_node_name_eq(node, "clk"))
@@ -469,6 +470,13 @@ static void __init _ti_omap4_clkctrl_setup(struct device_node *node)
 		else
 			data = dra7_clkctrl_data;
 	}
+
+	if (of_machine_is_compatible("ti,dra72"))
+		soc_mask = CLKF_SOC_DRA72;
+	if (of_machine_is_compatible("ti,dra74"))
+		soc_mask = CLKF_SOC_DRA74;
+	if (of_machine_is_compatible("ti,dra76"))
+		soc_mask = CLKF_SOC_DRA76;
 #endif
 #ifdef CONFIG_SOC_AM33XX
 	if (of_machine_is_compatible("ti,am33xx")) {
@@ -501,6 +509,9 @@ static void __init _ti_omap4_clkctrl_setup(struct device_node *node)
 		data = dm816_clkctrl_data;
 #endif
 
+	if (ti_clk_get_features()->flags & TI_CLK_DEVICE_TYPE_GP)
+		soc_mask |= CLKF_SOC_NONSEC;
+
 	while (data->addr) {
 		if (addr == data->addr)
 			break;
@@ -562,6 +573,12 @@ static void __init _ti_omap4_clkctrl_setup(struct device_node *node)
 	reg_data = data->regs;
 
 	while (reg_data->parent) {
+		if ((reg_data->flags & CLKF_SOC_MASK) &&
+		    (reg_data->flags & soc_mask) == 0) {
+			reg_data++;
+			continue;
+		}
+
 		hw = kzalloc(sizeof(*hw), GFP_KERNEL);
 		if (!hw)
 			return;
diff --git a/drivers/clk/ti/clock.h b/drivers/clk/ti/clock.h
index 1c0fac59d809..e4b8392ff63c 100644
--- a/drivers/clk/ti/clock.h
+++ b/drivers/clk/ti/clock.h
@@ -83,6 +83,13 @@ enum {
 #define CLKF_HW_SUP			BIT(6)
 #define CLKF_NO_IDLEST			BIT(7)
 
+#define CLKF_SOC_MASK			GENMASK(11, 8)
+
+#define CLKF_SOC_NONSEC			BIT(8)
+#define CLKF_SOC_DRA72			BIT(9)
+#define CLKF_SOC_DRA74			BIT(10)
+#define CLKF_SOC_DRA76			BIT(11)
+
 #define CLK(dev, con, ck)		\
 	{				\
 		.lk = {			\
@@ -303,7 +310,6 @@ long omap4_dpll_regm4xen_round_rate(struct clk_hw *hw,
 int omap4_dpll_regm4xen_determine_rate(struct clk_hw *hw,
 				       struct clk_rate_request *req);
 int omap2_clk_for_each(int (*fn)(struct clk_hw_omap *hw));
-bool omap2_clk_is_hw_omap(struct clk_hw *hw);
 
 extern struct ti_clk_ll_ops *ti_clk_ll_ops;
 
diff --git a/drivers/clk/ux500/clk-sysctrl.c b/drivers/clk/ux500/clk-sysctrl.c
index 7c0403b733ae..698306f4801f 100644
--- a/drivers/clk/ux500/clk-sysctrl.c
+++ b/drivers/clk/ux500/clk-sysctrl.c
@@ -42,7 +42,8 @@ static int clk_sysctrl_prepare(struct clk_hw *hw)
 				clk->reg_bits[0]);
 
 	if (!ret && clk->enable_delay_us)
-		usleep_range(clk->enable_delay_us, clk->enable_delay_us);
+		usleep_range(clk->enable_delay_us, clk->enable_delay_us +
+			     (clk->enable_delay_us >> 2));
 
 	return ret;
 }
diff --git a/drivers/clk/zynq/clkc.c b/drivers/clk/zynq/clkc.c
index d7b53ac8ad11..4b9d5c14c400 100644
--- a/drivers/clk/zynq/clkc.c
+++ b/drivers/clk/zynq/clkc.c
@@ -158,7 +158,7 @@ static void __init zynq_clk_register_fclk(enum zynq_clk fclk,
 	clks[fclk] = clk_register_gate(NULL, clk_name,
 			div1_name, CLK_SET_RATE_PARENT, fclk_gate_reg,
 			0, CLK_GATE_SET_TO_DISABLE, fclk_gate_lock);
-	enable_reg = clk_readl(fclk_gate_reg) & 1;
+	enable_reg = readl(fclk_gate_reg) & 1;
 	if (enable && !enable_reg) {
 		if (clk_prepare_enable(clks[fclk]))
 			pr_warn("%s: FCLK%u enable failed\n", __func__,
@@ -287,7 +287,7 @@ static void __init zynq_clk_setup(struct device_node *np)
 			SLCR_IOPLL_CTRL, 4, 1, 0, &iopll_lock);
 
 	/* CPU clocks */
-	tmp = clk_readl(SLCR_621_TRUE) & 1;
+	tmp = readl(SLCR_621_TRUE) & 1;
 	clk = clk_register_mux(NULL, "cpu_mux", cpu_parents, 4,
 			CLK_SET_RATE_NO_REPARENT, SLCR_ARM_CLK_CTRL, 4, 2, 0,
 			&armclk_lock);
@@ -510,7 +510,7 @@ static void __init zynq_clk_setup(struct device_node *np)
 			&dbgclk_lock);
 
 	/* leave debug clocks in the state the bootloader set them up to */
-	tmp = clk_readl(SLCR_DBG_CLK_CTRL);
+	tmp = readl(SLCR_DBG_CLK_CTRL);
 	if (tmp & DBG_CLK_CTRL_CLKACT_TRC)
 		if (clk_prepare_enable(clks[dbg_trc]))
 			pr_warn("%s: trace clk enable failed\n", __func__);
diff --git a/drivers/clk/zynq/pll.c b/drivers/clk/zynq/pll.c
index 00d72fb5c036..800b70ee19b3 100644
--- a/drivers/clk/zynq/pll.c
+++ b/drivers/clk/zynq/pll.c
@@ -90,7 +90,7 @@ static unsigned long zynq_pll_recalc_rate(struct clk_hw *hw,
 	 * makes probably sense to redundantly save fbdiv in the struct
 	 * zynq_pll to save the IO access.
 	 */
-	fbdiv = (clk_readl(clk->pll_ctrl) & PLLCTRL_FBDIV_MASK) >>
+	fbdiv = (readl(clk->pll_ctrl) & PLLCTRL_FBDIV_MASK) >>
 			PLLCTRL_FBDIV_SHIFT;
 
 	return parent_rate * fbdiv;
@@ -112,7 +112,7 @@ static int zynq_pll_is_enabled(struct clk_hw *hw)
 
 	spin_lock_irqsave(clk->lock, flags);
 
-	reg = clk_readl(clk->pll_ctrl);
+	reg = readl(clk->pll_ctrl);
 
 	spin_unlock_irqrestore(clk->lock, flags);
 
@@ -138,10 +138,10 @@ static int zynq_pll_enable(struct clk_hw *hw)
 	/* Power up PLL and wait for lock */
 	spin_lock_irqsave(clk->lock, flags);
 
-	reg = clk_readl(clk->pll_ctrl);
+	reg = readl(clk->pll_ctrl);
 	reg &= ~(PLLCTRL_RESET_MASK | PLLCTRL_PWRDWN_MASK);
-	clk_writel(reg, clk->pll_ctrl);
-	while (!(clk_readl(clk->pll_status) & (1 << clk->lockbit)))
+	writel(reg, clk->pll_ctrl);
+	while (!(readl(clk->pll_status) & (1 << clk->lockbit)))
 		;
 
 	spin_unlock_irqrestore(clk->lock, flags);
@@ -168,9 +168,9 @@ static void zynq_pll_disable(struct clk_hw *hw)
 	/* shut down PLL */
 	spin_lock_irqsave(clk->lock, flags);
 
-	reg = clk_readl(clk->pll_ctrl);
+	reg = readl(clk->pll_ctrl);
 	reg |= PLLCTRL_RESET_MASK | PLLCTRL_PWRDWN_MASK;
-	clk_writel(reg, clk->pll_ctrl);
+	writel(reg, clk->pll_ctrl);
 
 	spin_unlock_irqrestore(clk->lock, flags);
 }
@@ -223,9 +223,9 @@ struct clk *clk_register_zynq_pll(const char *name, const char *parent,
 
 	spin_lock_irqsave(pll->lock, flags);
 
-	reg = clk_readl(pll->pll_ctrl);
+	reg = readl(pll->pll_ctrl);
 	reg &= ~PLLCTRL_BPQUAL_MASK;
-	clk_writel(reg, pll->pll_ctrl);
+	writel(reg, pll->pll_ctrl);
 
 	spin_unlock_irqrestore(pll->lock, flags);
 
diff --git a/drivers/clk/zynqmp/clk-mux-zynqmp.c b/drivers/clk/zynqmp/clk-mux-zynqmp.c
index 4143f560c28d..0af8f74c5fa5 100644
--- a/drivers/clk/zynqmp/clk-mux-zynqmp.c
+++ b/drivers/clk/zynqmp/clk-mux-zynqmp.c
@@ -138,4 +138,3 @@ struct clk_hw *zynqmp_clk_register_mux(const char *name, u32 clk_id,
 
 	return hw;
 }
-EXPORT_SYMBOL_GPL(zynqmp_clk_register_mux);
diff --git a/drivers/clk/zynqmp/clk-zynqmp.h b/drivers/clk/zynqmp/clk-zynqmp.h
index 7ab163b67249..fec9a15c8786 100644
--- a/drivers/clk/zynqmp/clk-zynqmp.h
+++ b/drivers/clk/zynqmp/clk-zynqmp.h
@@ -10,12 +10,6 @@
 
 #include <linux/firmware/xlnx-zynqmp.h>
 
-/* Clock APIs payload parameters */
-#define CLK_GET_NAME_RESP_LEN				16
-#define CLK_GET_TOPOLOGY_RESP_WORDS			3
-#define CLK_GET_PARENTS_RESP_WORDS			3
-#define CLK_GET_ATTR_RESP_WORDS				1
-
 enum topology_type {
 	TYPE_INVALID,
 	TYPE_MUX,
diff --git a/drivers/clk/zynqmp/clkc.c b/drivers/clk/zynqmp/clkc.c
index b0908ec62f73..8febd2431545 100644
--- a/drivers/clk/zynqmp/clkc.c
+++ b/drivers/clk/zynqmp/clkc.c
@@ -21,24 +21,6 @@
 #define MAX_NODES			6
 #define MAX_NAME_LEN			50
 
-#define CLK_TYPE_SHIFT			2
-
-#define PM_API_PAYLOAD_LEN		3
-
-#define NA_PARENT			0xFFFFFFFF
-#define DUMMY_PARENT			0xFFFFFFFE
-
-#define CLK_TYPE_FIELD_LEN		4
-#define CLK_TOPOLOGY_NODE_OFFSET	16
-#define NODES_PER_RESP			3
-
-#define CLK_TYPE_FIELD_MASK		0xF
-#define CLK_FLAG_FIELD_MASK		GENMASK(21, 8)
-#define CLK_TYPE_FLAG_FIELD_MASK	GENMASK(31, 24)
-
-#define CLK_PARENTS_ID_LEN		16
-#define CLK_PARENTS_ID_MASK		0xFFFF
-
 /* Flags for parents */
 #define PARENT_CLK_SELF			0
 #define PARENT_CLK_NODE1		1
@@ -52,7 +34,10 @@
 #define END_OF_PARENTS			1
 #define RESERVED_CLK_NAME		""
 
-#define CLK_VALID_MASK			0x1
+#define CLK_GET_NAME_RESP_LEN		16
+#define CLK_GET_TOPOLOGY_RESP_WORDS	3
+#define CLK_GET_PARENTS_RESP_WORDS	3
+#define CLK_GET_ATTR_RESP_WORDS		1
 
 enum clk_type {
 	CLK_TYPE_OUTPUT,
@@ -80,6 +65,7 @@ struct clock_parent {
  * @num_nodes:		Number of nodes present in topology
  * @parent:		Parent of clock
  * @num_parents:	Number of parents of clock
+ * @clk_id:		Clock id
  */
 struct zynqmp_clock {
 	char clk_name[MAX_NAME_LEN];
@@ -89,6 +75,36 @@ struct zynqmp_clock {
 	u32 num_nodes;
 	struct clock_parent parent[MAX_PARENT];
 	u32 num_parents;
+	u32 clk_id;
+};
+
+struct name_resp {
+	char name[CLK_GET_NAME_RESP_LEN];
+};
+
+struct topology_resp {
+#define CLK_TOPOLOGY_TYPE		GENMASK(3, 0)
+#define CLK_TOPOLOGY_FLAGS		GENMASK(23, 8)
+#define CLK_TOPOLOGY_TYPE_FLAGS		GENMASK(31, 24)
+	u32 topology[CLK_GET_TOPOLOGY_RESP_WORDS];
+};
+
+struct parents_resp {
+#define NA_PARENT			0xFFFFFFFF
+#define DUMMY_PARENT			0xFFFFFFFE
+#define CLK_PARENTS_ID			GENMASK(15, 0)
+#define CLK_PARENTS_FLAGS		GENMASK(31, 16)
+	u32 parents[CLK_GET_PARENTS_RESP_WORDS];
+};
+
+struct attr_resp {
+#define CLK_ATTR_VALID			BIT(0)
+#define CLK_ATTR_TYPE			BIT(2)
+#define CLK_ATTR_NODE_INDEX		GENMASK(13, 0)
+#define CLK_ATTR_NODE_TYPE		GENMASK(19, 14)
+#define CLK_ATTR_NODE_SUBCLASS		GENMASK(25, 20)
+#define CLK_ATTR_NODE_CLASS		GENMASK(31, 26)
+	u32 attr[CLK_GET_ATTR_RESP_WORDS];
 };
 
 static const char clk_type_postfix[][10] = {
@@ -199,14 +215,15 @@ static int zynqmp_pm_clock_get_num_clocks(u32 *nclocks)
 /**
  * zynqmp_pm_clock_get_name() - Get the name of clock for given id
  * @clock_id:	ID of the clock to be queried
- * @name:	Name of given clock
+ * @response:	Name of the clock with the given id
  *
  * This function is used to get name of clock specified by given
  * clock ID.
  *
- * Return: Returns 0, in case of error name would be 0
+ * Return: Returns 0
  */
-static int zynqmp_pm_clock_get_name(u32 clock_id, char *name)
+static int zynqmp_pm_clock_get_name(u32 clock_id,
+				    struct name_resp *response)
 {
 	struct zynqmp_pm_query_data qdata = {0};
 	u32 ret_payload[PAYLOAD_ARG_CNT];
@@ -215,7 +232,7 @@ static int zynqmp_pm_clock_get_name(u32 clock_id, char *name)
 	qdata.arg1 = clock_id;
 
 	eemi_ops->query_data(qdata, ret_payload);
-	memcpy(name, ret_payload, CLK_GET_NAME_RESP_LEN);
+	memcpy(response, ret_payload, sizeof(*response));
 
 	return 0;
 }
@@ -224,7 +241,7 @@ static int zynqmp_pm_clock_get_name(u32 clock_id, char *name)
  * zynqmp_pm_clock_get_topology() - Get the topology of clock for given id
  * @clock_id:	ID of the clock to be queried
  * @index:	Node index of clock topology
- * @topology:	Buffer to store nodes in topology and flags
+ * @response:	Buffer used for the topology response
  *
  * This function is used to get topology information for the clock
  * specified by given clock ID.
@@ -237,7 +254,8 @@ static int zynqmp_pm_clock_get_name(u32 clock_id, char *name)
  *
  * Return: 0 on success else error+reason
  */
-static int zynqmp_pm_clock_get_topology(u32 clock_id, u32 index, u32 *topology)
+static int zynqmp_pm_clock_get_topology(u32 clock_id, u32 index,
+					struct topology_resp *response)
 {
 	struct zynqmp_pm_query_data qdata = {0};
 	u32 ret_payload[PAYLOAD_ARG_CNT];
@@ -248,7 +266,7 @@ static int zynqmp_pm_clock_get_topology(u32 clock_id, u32 index, u32 *topology)
 	qdata.arg2 = index;
 
 	ret = eemi_ops->query_data(qdata, ret_payload);
-	memcpy(topology, &ret_payload[1], CLK_GET_TOPOLOGY_RESP_WORDS * 4);
+	memcpy(response, &ret_payload[1], sizeof(*response));
 
 	return ret;
 }
@@ -297,7 +315,7 @@ struct clk_hw *zynqmp_clk_register_fixed_factor(const char *name, u32 clk_id,
  * zynqmp_pm_clock_get_parents() - Get the first 3 parents of clock for given id
  * @clock_id:	Clock ID
  * @index:	Parent index
- * @parents:	3 parents of the given clock
+ * @response:	Parents of the given clock
  *
  * This function is used to get 3 parents for the clock specified by
  * given clock ID.
@@ -310,7 +328,8 @@ struct clk_hw *zynqmp_clk_register_fixed_factor(const char *name, u32 clk_id,
  *
  * Return: 0 on success else error+reason
  */
-static int zynqmp_pm_clock_get_parents(u32 clock_id, u32 index, u32 *parents)
+static int zynqmp_pm_clock_get_parents(u32 clock_id, u32 index,
+				       struct parents_resp *response)
 {
 	struct zynqmp_pm_query_data qdata = {0};
 	u32 ret_payload[PAYLOAD_ARG_CNT];
@@ -321,7 +340,7 @@ static int zynqmp_pm_clock_get_parents(u32 clock_id, u32 index, u32 *parents)
 	qdata.arg2 = index;
 
 	ret = eemi_ops->query_data(qdata, ret_payload);
-	memcpy(parents, &ret_payload[1], CLK_GET_PARENTS_RESP_WORDS * 4);
+	memcpy(response, &ret_payload[1], sizeof(*response));
 
 	return ret;
 }
@@ -329,13 +348,14 @@ static int zynqmp_pm_clock_get_parents(u32 clock_id, u32 index, u32 *parents)
 /**
  * zynqmp_pm_clock_get_attributes() - Get the attributes of clock for given id
  * @clock_id:	Clock ID
- * @attr:	Clock attributes
+ * @response:	Clock attributes response
  *
  * This function is used to get clock's attributes(e.g. valid, clock type, etc).
  *
  * Return: 0 on success else error+reason
  */
-static int zynqmp_pm_clock_get_attributes(u32 clock_id, u32 *attr)
+static int zynqmp_pm_clock_get_attributes(u32 clock_id,
+					  struct attr_resp *response)
 {
 	struct zynqmp_pm_query_data qdata = {0};
 	u32 ret_payload[PAYLOAD_ARG_CNT];
@@ -345,7 +365,7 @@ static int zynqmp_pm_clock_get_attributes(u32 clock_id, u32 *attr)
 	qdata.arg1 = clock_id;
 
 	ret = eemi_ops->query_data(qdata, ret_payload);
-	memcpy(attr, &ret_payload[1], CLK_GET_ATTR_RESP_WORDS * 4);
+	memcpy(response, &ret_payload[1], sizeof(*response));
 
 	return ret;
 }
@@ -354,24 +374,28 @@ static int zynqmp_pm_clock_get_attributes(u32 clock_id, u32 *attr)
  * __zynqmp_clock_get_topology() - Get topology data of clock from firmware
  *				   response data
  * @topology:		Clock topology
- * @data:		Clock topology data received from firmware
+ * @response:		Clock topology data received from firmware
  * @nnodes:		Number of nodes
  *
  * Return: 0 on success else error+reason
  */
 static int __zynqmp_clock_get_topology(struct clock_topology *topology,
-				       u32 *data, u32 *nnodes)
+				       struct topology_resp *response,
+				       u32 *nnodes)
 {
 	int i;
+	u32 type;
 
-	for (i = 0; i < PM_API_PAYLOAD_LEN; i++) {
-		if (!(data[i] & CLK_TYPE_FIELD_MASK))
+	for (i = 0; i < ARRAY_SIZE(response->topology); i++) {
+		type = FIELD_GET(CLK_TOPOLOGY_TYPE, response->topology[i]);
+		if (type == TYPE_INVALID)
 			return END_OF_TOPOLOGY_NODE;
-		topology[*nnodes].type = data[i] & CLK_TYPE_FIELD_MASK;
-		topology[*nnodes].flag = FIELD_GET(CLK_FLAG_FIELD_MASK,
-						   data[i]);
+		topology[*nnodes].type = type;
+		topology[*nnodes].flag = FIELD_GET(CLK_TOPOLOGY_FLAGS,
+						   response->topology[i]);
 		topology[*nnodes].type_flag =
-				FIELD_GET(CLK_TYPE_FLAG_FIELD_MASK, data[i]);
+				FIELD_GET(CLK_TOPOLOGY_TYPE_FLAGS,
+					  response->topology[i]);
 		(*nnodes)++;
 	}
 
@@ -392,14 +416,16 @@ static int zynqmp_clock_get_topology(u32 clk_id,
 				     u32 *num_nodes)
 {
 	int j, ret;
-	u32 pm_resp[PM_API_PAYLOAD_LEN] = {0};
+	struct topology_resp response = { };
 
 	*num_nodes = 0;
-	for (j = 0; j <= MAX_NODES; j += 3) {
-		ret = zynqmp_pm_clock_get_topology(clk_id, j, pm_resp);
+	for (j = 0; j <= MAX_NODES; j += ARRAY_SIZE(response.topology)) {
+		ret = zynqmp_pm_clock_get_topology(clock[clk_id].clk_id, j,
+						   &response);
 		if (ret)
 			return ret;
-		ret = __zynqmp_clock_get_topology(topology, pm_resp, num_nodes);
+		ret = __zynqmp_clock_get_topology(topology, &response,
+						  num_nodes);
 		if (ret == END_OF_TOPOLOGY_NODE)
 			return 0;
 	}
@@ -408,31 +434,33 @@ static int zynqmp_clock_get_topology(u32 clk_id,
 }
 
 /**
- * __zynqmp_clock_get_topology() - Get parents info of clock from firmware
+ * __zynqmp_clock_get_parents() - Get parents info of clock from firmware
  *				   response data
  * @parents:		Clock parents
- * @data:		Clock parents data received from firmware
+ * @response:		Clock parents data received from firmware
  * @nparent:		Number of parent
  *
  * Return: 0 on success else error+reason
  */
-static int __zynqmp_clock_get_parents(struct clock_parent *parents, u32 *data,
+static int __zynqmp_clock_get_parents(struct clock_parent *parents,
+				      struct parents_resp *response,
 				      u32 *nparent)
 {
 	int i;
 	struct clock_parent *parent;
 
-	for (i = 0; i < PM_API_PAYLOAD_LEN; i++) {
-		if (data[i] == NA_PARENT)
+	for (i = 0; i < ARRAY_SIZE(response->parents); i++) {
+		if (response->parents[i] == NA_PARENT)
 			return END_OF_PARENTS;
 
 		parent = &parents[i];
-		parent->id = data[i] & CLK_PARENTS_ID_MASK;
-		if (data[i] == DUMMY_PARENT) {
+		parent->id = FIELD_GET(CLK_PARENTS_ID, response->parents[i]);
+		if (response->parents[i] == DUMMY_PARENT) {
 			strcpy(parent->name, "dummy_name");
 			parent->flag = 0;
 		} else {
-			parent->flag = data[i] >> CLK_PARENTS_ID_LEN;
+			parent->flag = FIELD_GET(CLK_PARENTS_FLAGS,
+						 response->parents[i]);
 			if (zynqmp_get_clock_name(parent->id, parent->name))
 				continue;
 		}
@@ -454,20 +482,21 @@ static int zynqmp_clock_get_parents(u32 clk_id, struct clock_parent *parents,
 				    u32 *num_parents)
 {
 	int j = 0, ret;
-	u32 pm_resp[PM_API_PAYLOAD_LEN] = {0};
+	struct parents_resp response = { };
 
 	*num_parents = 0;
 	do {
 		/* Get parents from firmware */
-		ret = zynqmp_pm_clock_get_parents(clk_id, j, pm_resp);
+		ret = zynqmp_pm_clock_get_parents(clock[clk_id].clk_id, j,
+						  &response);
 		if (ret)
 			return ret;
 
-		ret = __zynqmp_clock_get_parents(&parents[j], pm_resp,
+		ret = __zynqmp_clock_get_parents(&parents[j], &response,
 						 num_parents);
 		if (ret == END_OF_PARENTS)
 			return 0;
-		j += PM_API_PAYLOAD_LEN;
+		j += ARRAY_SIZE(response.parents);
 	} while (*num_parents <= MAX_PARENT);
 
 	return 0;
@@ -528,13 +557,14 @@ static struct clk_hw *zynqmp_register_clk_topology(int clk_id, char *clk_name,
 						   const char **parent_names)
 {
 	int j;
-	u32 num_nodes;
+	u32 num_nodes, clk_dev_id;
 	char *clk_out = NULL;
 	struct clock_topology *nodes;
 	struct clk_hw *hw = NULL;
 
 	nodes = clock[clk_id].node;
 	num_nodes = clock[clk_id].num_nodes;
+	clk_dev_id = clock[clk_id].clk_id;
 
 	for (j = 0; j < num_nodes; j++) {
 		/*
@@ -551,13 +581,14 @@ static struct clk_hw *zynqmp_register_clk_topology(int clk_id, char *clk_name,
 		if (!clk_topology[nodes[j].type])
 			continue;
 
-		hw = (*clk_topology[nodes[j].type])(clk_out, clk_id,
+		hw = (*clk_topology[nodes[j].type])(clk_out, clk_dev_id,
 						    parent_names,
 						    num_parents,
 						    &nodes[j]);
 		if (IS_ERR(hw))
-			pr_warn_once("%s() %s register fail with %ld\n",
-				     __func__, clk_name, PTR_ERR(hw));
+			pr_warn_once("%s() 0x%x: %s register fail with %ld\n",
+				     __func__,  clk_dev_id, clk_name,
+				     PTR_ERR(hw));
 
 		parent_names[0] = clk_out;
 	}
@@ -621,20 +652,33 @@ static int zynqmp_register_clocks(struct device_node *np)
 static void zynqmp_get_clock_info(void)
 {
 	int i, ret;
-	u32 attr, type = 0;
+	u32 type = 0;
+	u32 nodetype, subclass, class;
+	struct attr_resp attr;
+	struct name_resp name;
 
 	for (i = 0; i < clock_max_idx; i++) {
-		zynqmp_pm_clock_get_name(i, clock[i].clk_name);
-		if (!strcmp(clock[i].clk_name, RESERVED_CLK_NAME))
-			continue;
-
 		ret = zynqmp_pm_clock_get_attributes(i, &attr);
 		if (ret)
 			continue;
 
-		clock[i].valid = attr & CLK_VALID_MASK;
-		clock[i].type = attr >> CLK_TYPE_SHIFT ? CLK_TYPE_EXTERNAL :
-							CLK_TYPE_OUTPUT;
+		clock[i].valid = FIELD_GET(CLK_ATTR_VALID, attr.attr[0]);
+		clock[i].type = FIELD_GET(CLK_ATTR_TYPE, attr.attr[0]) ?
+			CLK_TYPE_EXTERNAL : CLK_TYPE_OUTPUT;
+
+		nodetype = FIELD_GET(CLK_ATTR_NODE_TYPE, attr.attr[0]);
+		subclass = FIELD_GET(CLK_ATTR_NODE_SUBCLASS, attr.attr[0]);
+		class = FIELD_GET(CLK_ATTR_NODE_CLASS, attr.attr[0]);
+
+		clock[i].clk_id = FIELD_PREP(CLK_ATTR_NODE_CLASS, class) |
+				  FIELD_PREP(CLK_ATTR_NODE_SUBCLASS, subclass) |
+				  FIELD_PREP(CLK_ATTR_NODE_TYPE, nodetype) |
+				  FIELD_PREP(CLK_ATTR_NODE_INDEX, i);
+
+		zynqmp_pm_clock_get_name(clock[i].clk_id, &name);
+		if (!strcmp(name.name, RESERVED_CLK_NAME))
+			continue;
+		strncpy(clock[i].clk_name, name.name, MAX_NAME_LEN);
 	}
 
 	/* Get topology of all clock */
diff --git a/drivers/clk/zynqmp/divider.c b/drivers/clk/zynqmp/divider.c
index a371c66e72ef..d8f5b70d2709 100644
--- a/drivers/clk/zynqmp/divider.c
+++ b/drivers/clk/zynqmp/divider.c
@@ -31,12 +31,14 @@
  * struct zynqmp_clk_divider - adjustable divider clock
  * @hw:		handle between common and hardware-specific interfaces
  * @flags:	Hardware specific flags
+ * @is_frac:	The divider is a fractional divider
  * @clk_id:	Id of clock
  * @div_type:	divisor type (TYPE_DIV1 or TYPE_DIV2)
  */
 struct zynqmp_clk_divider {
 	struct clk_hw hw;
 	u8 flags;
+	bool is_frac;
 	u32 clk_id;
 	u32 div_type;
 };
@@ -76,6 +78,13 @@ static unsigned long zynqmp_clk_divider_recalc_rate(struct clk_hw *hw,
 	else
 		value = div >> 16;
 
+	if (!value) {
+		WARN(!(divider->flags & CLK_DIVIDER_ALLOW_ZERO),
+		     "%s: Zero divisor and CLK_DIVIDER_ALLOW_ZERO not set\n",
+		     clk_name);
+		return parent_rate;
+	}
+
 	return DIV_ROUND_UP_ULL(parent_rate, value);
 }
 
@@ -116,8 +125,7 @@ static long zynqmp_clk_divider_round_rate(struct clk_hw *hw,
 
 	bestdiv = zynqmp_divider_get_val(*prate, rate);
 
-	if ((clk_hw_get_flags(hw) & CLK_SET_RATE_PARENT) &&
-	    (divider->flags & CLK_FRAC))
+	if ((clk_hw_get_flags(hw) & CLK_SET_RATE_PARENT) && divider->is_frac)
 		bestdiv = rate % *prate ? 1 : bestdiv;
 	*prate = rate * bestdiv;
 
@@ -195,11 +203,13 @@ struct clk_hw *zynqmp_clk_register_divider(const char *name,
 
 	init.name = name;
 	init.ops = &zynqmp_clk_divider_ops;
-	init.flags = nodes->flag;
+	/* CLK_FRAC is not defined in the common clk framework */
+	init.flags = nodes->flag & ~CLK_FRAC;
 	init.parent_names = parents;
 	init.num_parents = 1;
 
 	/* struct clk_divider assignments */
+	div->is_frac = !!(nodes->flag & CLK_FRAC);
 	div->flags = nodes->type_flag;
 	div->hw.init = &init;
 	div->clk_id = clk_id;
@@ -214,4 +224,3 @@ struct clk_hw *zynqmp_clk_register_divider(const char *name,
 
 	return hw;
 }
-EXPORT_SYMBOL_GPL(zynqmp_clk_register_divider);
diff --git a/drivers/clocksource/timer-ti-dm.c b/drivers/clocksource/timer-ti-dm.c
index ee8ec5a8cb16..e40b55a7086f 100644
--- a/drivers/clocksource/timer-ti-dm.c
+++ b/drivers/clocksource/timer-ti-dm.c
@@ -970,5 +970,4 @@ module_platform_driver(omap_dm_timer_driver);
 
 MODULE_DESCRIPTION("OMAP Dual-Mode Timer Driver");
 MODULE_LICENSE("GPL");
-MODULE_ALIAS("platform:" DRIVER_NAME);
 MODULE_AUTHOR("Texas Instruments Inc");
diff --git a/drivers/dma-buf/Makefile b/drivers/dma-buf/Makefile
index 0913a6ccab5a..1f006e083eb9 100644
--- a/drivers/dma-buf/Makefile
+++ b/drivers/dma-buf/Makefile
@@ -1,4 +1,5 @@
-obj-y := dma-buf.o dma-fence.o dma-fence-array.o reservation.o seqno-fence.o
+obj-y := dma-buf.o dma-fence.o dma-fence-array.o dma-fence-chain.o \
+	 reservation.o seqno-fence.o
 obj-$(CONFIG_SYNC_FILE)		+= sync_file.o
 obj-$(CONFIG_SW_SYNC)		+= sw_sync.o sync_debug.o
 obj-$(CONFIG_UDMABUF)		+= udmabuf.o
diff --git a/drivers/dma-buf/dma-fence-chain.c b/drivers/dma-buf/dma-fence-chain.c
new file mode 100644
index 000000000000..93c42078cb57
--- /dev/null
+++ b/drivers/dma-buf/dma-fence-chain.c
@@ -0,0 +1,242 @@
+/*
+ * fence-chain: chain fences together in a timeline
+ *
+ * Copyright (C) 2018 Advanced Micro Devices, Inc.
+ * Authors:
+ *	Christian König <christian.koenig@amd.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#include <linux/dma-fence-chain.h>
+
+static bool dma_fence_chain_enable_signaling(struct dma_fence *fence);
+
+/**
+ * dma_fence_chain_get_prev - use RCU to get a reference to the previous fence
+ * @chain: chain node to get the previous node from
+ *
+ * Use dma_fence_get_rcu_safe to get a reference to the previous fence of the
+ * chain node.
+ */
+static struct dma_fence *dma_fence_chain_get_prev(struct dma_fence_chain *chain)
+{
+	struct dma_fence *prev;
+
+	rcu_read_lock();
+	prev = dma_fence_get_rcu_safe(&chain->prev);
+	rcu_read_unlock();
+	return prev;
+}
+
+/**
+ * dma_fence_chain_walk - chain walking function
+ * @fence: current chain node
+ *
+ * Walk the chain to the next node. Returns the next fence or NULL if we are at
+ * the end of the chain. Garbage collects chain nodes which are already
+ * signaled.
+ */
+struct dma_fence *dma_fence_chain_walk(struct dma_fence *fence)
+{
+	struct dma_fence_chain *chain, *prev_chain;
+	struct dma_fence *prev, *replacement, *tmp;
+
+	chain = to_dma_fence_chain(fence);
+	if (!chain) {
+		dma_fence_put(fence);
+		return NULL;
+	}
+
+	while ((prev = dma_fence_chain_get_prev(chain))) {
+
+		prev_chain = to_dma_fence_chain(prev);
+		if (prev_chain) {
+			if (!dma_fence_is_signaled(prev_chain->fence))
+				break;
+
+			replacement = dma_fence_chain_get_prev(prev_chain);
+		} else {
+			if (!dma_fence_is_signaled(prev))
+				break;
+
+			replacement = NULL;
+		}
+
+		tmp = cmpxchg((void **)&chain->prev, (void *)prev, (void *)replacement);
+		if (tmp == prev)
+			dma_fence_put(tmp);
+		else
+			dma_fence_put(replacement);
+		dma_fence_put(prev);
+	}
+
+	dma_fence_put(fence);
+	return prev;
+}
+EXPORT_SYMBOL(dma_fence_chain_walk);
+
+/**
+ * dma_fence_chain_find_seqno - find fence chain node by seqno
+ * @pfence: pointer to the chain node where to start
+ * @seqno: the sequence number to search for
+ *
+ * Advance the fence pointer to the chain node which will signal this sequence
+ * number. If no sequence number is provided then this is a no-op.
+ *
+ * Returns EINVAL if the fence is not a chain node or the sequence number has
+ * not yet advanced far enough.
+ */
+int dma_fence_chain_find_seqno(struct dma_fence **pfence, uint64_t seqno)
+{
+	struct dma_fence_chain *chain;
+
+	if (!seqno)
+		return 0;
+
+	chain = to_dma_fence_chain(*pfence);
+	if (!chain || chain->base.seqno < seqno)
+		return -EINVAL;
+
+	dma_fence_chain_for_each(*pfence, &chain->base) {
+		if ((*pfence)->context != chain->base.context ||
+		    to_dma_fence_chain(*pfence)->prev_seqno < seqno)
+			break;
+	}
+	dma_fence_put(&chain->base);
+
+	return 0;
+}
+EXPORT_SYMBOL(dma_fence_chain_find_seqno);
+
+static const char *dma_fence_chain_get_driver_name(struct dma_fence *fence)
+{
+        return "dma_fence_chain";
+}
+
+static const char *dma_fence_chain_get_timeline_name(struct dma_fence *fence)
+{
+        return "unbound";
+}
+
+static void dma_fence_chain_irq_work(struct irq_work *work)
+{
+	struct dma_fence_chain *chain;
+
+	chain = container_of(work, typeof(*chain), work);
+
+	/* Try to rearm the callback */
+	if (!dma_fence_chain_enable_signaling(&chain->base))
+		/* Ok, we are done. No more unsignaled fences left */
+		dma_fence_signal(&chain->base);
+	dma_fence_put(&chain->base);
+}
+
+static void dma_fence_chain_cb(struct dma_fence *f, struct dma_fence_cb *cb)
+{
+	struct dma_fence_chain *chain;
+
+	chain = container_of(cb, typeof(*chain), cb);
+	irq_work_queue(&chain->work);
+	dma_fence_put(f);
+}
+
+static bool dma_fence_chain_enable_signaling(struct dma_fence *fence)
+{
+	struct dma_fence_chain *head = to_dma_fence_chain(fence);
+
+	dma_fence_get(&head->base);
+	dma_fence_chain_for_each(fence, &head->base) {
+		struct dma_fence_chain *chain = to_dma_fence_chain(fence);
+		struct dma_fence *f = chain ? chain->fence : fence;
+
+		dma_fence_get(f);
+		if (!dma_fence_add_callback(f, &head->cb, dma_fence_chain_cb)) {
+			dma_fence_put(fence);
+			return true;
+		}
+		dma_fence_put(f);
+	}
+	dma_fence_put(&head->base);
+	return false;
+}
+
+static bool dma_fence_chain_signaled(struct dma_fence *fence)
+{
+	dma_fence_chain_for_each(fence, fence) {
+		struct dma_fence_chain *chain = to_dma_fence_chain(fence);
+		struct dma_fence *f = chain ? chain->fence : fence;
+
+		if (!dma_fence_is_signaled(f)) {
+			dma_fence_put(fence);
+			return false;
+		}
+	}
+
+	return true;
+}
+
+static void dma_fence_chain_release(struct dma_fence *fence)
+{
+	struct dma_fence_chain *chain = to_dma_fence_chain(fence);
+
+	dma_fence_put(rcu_dereference_protected(chain->prev, true));
+	dma_fence_put(chain->fence);
+	dma_fence_free(fence);
+}
+
+const struct dma_fence_ops dma_fence_chain_ops = {
+	.use_64bit_seqno = true,
+	.get_driver_name = dma_fence_chain_get_driver_name,
+	.get_timeline_name = dma_fence_chain_get_timeline_name,
+	.enable_signaling = dma_fence_chain_enable_signaling,
+	.signaled = dma_fence_chain_signaled,
+	.release = dma_fence_chain_release,
+};
+EXPORT_SYMBOL(dma_fence_chain_ops);
+
+/**
+ * dma_fence_chain_init - initialize a fence chain
+ * @chain: the chain node to initialize
+ * @prev: the previous fence
+ * @fence: the current fence
+ *
+ * Initialize a new chain node and either start a new chain or add the node to
+ * the existing chain of the previous fence.
+ */
+void dma_fence_chain_init(struct dma_fence_chain *chain,
+			  struct dma_fence *prev,
+			  struct dma_fence *fence,
+			  uint64_t seqno)
+{
+	struct dma_fence_chain *prev_chain = to_dma_fence_chain(prev);
+	uint64_t context;
+
+	spin_lock_init(&chain->lock);
+	rcu_assign_pointer(chain->prev, prev);
+	chain->fence = fence;
+	chain->prev_seqno = 0;
+	init_irq_work(&chain->work, dma_fence_chain_irq_work);
+
+	/* Try to reuse the context of the previous chain node. */
+	if (prev_chain && __dma_fence_is_later(seqno, prev->seqno, prev->ops)) {
+		context = prev->context;
+		chain->prev_seqno = prev->seqno;
+	} else {
+		context = dma_fence_context_alloc(1);
+		/* Make sure that we always have a valid sequence number. */
+		if (prev_chain)
+			seqno = max(prev->seqno, seqno);
+	}
+
+	dma_fence_init(&chain->base, &dma_fence_chain_ops,
+		       &chain->lock, context, seqno);
+}
+EXPORT_SYMBOL(dma_fence_chain_init);
diff --git a/drivers/dma-buf/reservation.c b/drivers/dma-buf/reservation.c
index c1618335ca99..4d32e2c67862 100644
--- a/drivers/dma-buf/reservation.c
+++ b/drivers/dma-buf/reservation.c
@@ -73,6 +73,8 @@ int reservation_object_reserve_shared(struct reservation_object *obj,
 	struct reservation_object_list *old, *new;
 	unsigned int i, j, k, max;
 
+	reservation_object_assert_held(obj);
+
 	old = reservation_object_get_list(obj);
 
 	if (old && old->shared_max) {
@@ -151,6 +153,8 @@ void reservation_object_add_shared_fence(struct reservation_object *obj,
 
 	dma_fence_get(fence);
 
+	reservation_object_assert_held(obj);
+
 	fobj = reservation_object_get_list(obj);
 	count = fobj->shared_count;
 
@@ -196,6 +200,8 @@ void reservation_object_add_excl_fence(struct reservation_object *obj,
 	struct reservation_object_list *old;
 	u32 i = 0;
 
+	reservation_object_assert_held(obj);
+
 	old = reservation_object_get_list(obj);
 	if (old)
 		i = old->shared_count;
@@ -236,6 +242,8 @@ int reservation_object_copy_fences(struct reservation_object *dst,
 	size_t size;
 	unsigned i;
 
+	reservation_object_assert_held(dst);
+
 	rcu_read_lock();
 	src_list = rcu_dereference(src->fence);
 
diff --git a/drivers/dma-buf/sw_sync.c b/drivers/dma-buf/sw_sync.c
index 32dcf7b4c935..119b2ffbc2c9 100644
--- a/drivers/dma-buf/sw_sync.c
+++ b/drivers/dma-buf/sw_sync.c
@@ -161,7 +161,7 @@ static bool timeline_fence_signaled(struct dma_fence *fence)
 {
 	struct sync_timeline *parent = dma_fence_parent(fence);
 
-	return !__dma_fence_is_later(fence->seqno, parent->value);
+	return !__dma_fence_is_later(fence->seqno, parent->value, fence->ops);
 }
 
 static bool timeline_fence_enable_signaling(struct dma_fence *fence)
diff --git a/drivers/dma-buf/sync_file.c b/drivers/dma-buf/sync_file.c
index 4f6305ca52c8..ed3fb6e5224c 100644
--- a/drivers/dma-buf/sync_file.c
+++ b/drivers/dma-buf/sync_file.c
@@ -258,7 +258,8 @@ static struct sync_file *sync_file_merge(const char *name, struct sync_file *a,
 
 			i_b++;
 		} else {
-			if (__dma_fence_is_later(pt_a->seqno, pt_b->seqno))
+			if (__dma_fence_is_later(pt_a->seqno, pt_b->seqno,
+						 pt_a->ops))
 				add_fence(fences, &i, pt_a);
 			else
 				add_fence(fences, &i, pt_b);
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
index 0b1dfb5bf2d9..eaf78f4e07ce 100644
--- a/drivers/dma/Kconfig
+++ b/drivers/dma/Kconfig
@@ -99,7 +99,7 @@ config AT_XDMAC
 
 config AXI_DMAC
 	tristate "Analog Devices AXI-DMAC DMA support"
-	depends on MICROBLAZE || NIOS2 || ARCH_ZYNQ || ARCH_SOCFPGA || COMPILE_TEST
+	depends on MICROBLAZE || NIOS2 || ARCH_ZYNQ || ARCH_ZYNQMP || ARCH_SOCFPGA || COMPILE_TEST
 	select DMA_ENGINE
 	select DMA_VIRTUAL_CHANNELS
 	help
diff --git a/drivers/dma/amba-pl08x.c b/drivers/dma/amba-pl08x.c
index fc8c2bab563c..8cfc753ad4b0 100644
--- a/drivers/dma/amba-pl08x.c
+++ b/drivers/dma/amba-pl08x.c
@@ -254,6 +254,7 @@ enum pl08x_dma_chan_state {
  * @slave: whether this channel is a device (slave) or for memcpy
  * @signal: the physical DMA request signal which this channel is using
  * @mux_use: count of descriptors using this DMA request signal setting
+ * @waiting_at: time in jiffies when this channel moved to waiting state
  */
 struct pl08x_dma_chan {
 	struct virt_dma_chan vc;
@@ -267,6 +268,7 @@ struct pl08x_dma_chan {
 	bool slave;
 	int signal;
 	unsigned mux_use;
+	unsigned long waiting_at;
 };
 
 /**
@@ -875,6 +877,7 @@ static void pl08x_phy_alloc_and_start(struct pl08x_dma_chan *plchan)
 	if (!ch) {
 		dev_dbg(&pl08x->adev->dev, "no physical channel available for xfer on %s\n", plchan->name);
 		plchan->state = PL08X_CHAN_WAITING;
+		plchan->waiting_at = jiffies;
 		return;
 	}
 
@@ -913,22 +916,29 @@ static void pl08x_phy_free(struct pl08x_dma_chan *plchan)
 {
 	struct pl08x_driver_data *pl08x = plchan->host;
 	struct pl08x_dma_chan *p, *next;
-
+	unsigned long waiting_at;
  retry:
 	next = NULL;
+	waiting_at = jiffies;
 
-	/* Find a waiting virtual channel for the next transfer. */
+	/*
+	 * Find a waiting virtual channel for the next transfer.
+	 * To be fair, time when each channel reached waiting state is compared
+	 * to select channel that is waiting for the longest time.
+	 */
 	list_for_each_entry(p, &pl08x->memcpy.channels, vc.chan.device_node)
-		if (p->state == PL08X_CHAN_WAITING) {
+		if (p->state == PL08X_CHAN_WAITING &&
+		    p->waiting_at <= waiting_at) {
 			next = p;
-			break;
+			waiting_at = p->waiting_at;
 		}
 
 	if (!next && pl08x->has_slave) {
 		list_for_each_entry(p, &pl08x->slave.channels, vc.chan.device_node)
-			if (p->state == PL08X_CHAN_WAITING) {
+			if (p->state == PL08X_CHAN_WAITING &&
+			    p->waiting_at <= waiting_at) {
 				next = p;
-				break;
+				waiting_at = p->waiting_at;
 			}
 	}
 
diff --git a/drivers/dma/at_xdmac.c b/drivers/dma/at_xdmac.c
index fe69dccfa0c0..e4ae2ee46d3f 100644
--- a/drivers/dma/at_xdmac.c
+++ b/drivers/dma/at_xdmac.c
@@ -308,6 +308,11 @@ static inline int at_xdmac_csize(u32 maxburst)
 	return csize;
 };
 
+static inline bool at_xdmac_chan_is_peripheral_xfer(u32 cfg)
+{
+	return cfg & AT_XDMAC_CC_TYPE_PER_TRAN;
+}
+
 static inline u8 at_xdmac_get_dwidth(u32 cfg)
 {
 	return (cfg & AT_XDMAC_CC_DWIDTH_MASK) >> AT_XDMAC_CC_DWIDTH_OFFSET;
@@ -389,7 +394,13 @@ static void at_xdmac_start_xfer(struct at_xdmac_chan *atchan,
 		 at_xdmac_chan_read(atchan, AT_XDMAC_CUBC));
 
 	at_xdmac_chan_write(atchan, AT_XDMAC_CID, 0xffffffff);
-	reg = AT_XDMAC_CIE_RBEIE | AT_XDMAC_CIE_WBEIE | AT_XDMAC_CIE_ROIE;
+	reg = AT_XDMAC_CIE_RBEIE | AT_XDMAC_CIE_WBEIE;
+	/*
+	 * Request Overflow Error is only for peripheral synchronized transfers
+	 */
+	if (at_xdmac_chan_is_peripheral_xfer(first->lld.mbr_cfg))
+		reg |= AT_XDMAC_CIE_ROIE;
+
 	/*
 	 * There is no end of list when doing cyclic dma, we need to get
 	 * an interrupt after each periods.
@@ -1575,6 +1586,46 @@ static void at_xdmac_handle_cyclic(struct at_xdmac_chan *atchan)
 		dmaengine_desc_get_callback_invoke(txd, NULL);
 }
 
+static void at_xdmac_handle_error(struct at_xdmac_chan *atchan)
+{
+	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);
+	struct at_xdmac_desc	*bad_desc;
+
+	/*
+	 * The descriptor currently at the head of the active list is
+	 * broken. Since we don't have any way to report errors, we'll
+	 * just have to scream loudly and try to continue with other
+	 * descriptors queued (if any).
+	 */
+	if (atchan->irq_status & AT_XDMAC_CIS_RBEIS)
+		dev_err(chan2dev(&atchan->chan), "read bus error!!!");
+	if (atchan->irq_status & AT_XDMAC_CIS_WBEIS)
+		dev_err(chan2dev(&atchan->chan), "write bus error!!!");
+	if (atchan->irq_status & AT_XDMAC_CIS_ROIS)
+		dev_err(chan2dev(&atchan->chan), "request overflow error!!!");
+
+	spin_lock_bh(&atchan->lock);
+
+	/* Channel must be disabled first as it's not done automatically */
+	at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask);
+	while (at_xdmac_read(atxdmac, AT_XDMAC_GS) & atchan->mask)
+		cpu_relax();
+
+	bad_desc = list_first_entry(&atchan->xfers_list,
+				    struct at_xdmac_desc,
+				    xfer_node);
+
+	spin_unlock_bh(&atchan->lock);
+
+	/* Print bad descriptor's details if needed */
+	dev_dbg(chan2dev(&atchan->chan),
+		"%s: lld: mbr_sa=%pad, mbr_da=%pad, mbr_ubc=0x%08x\n",
+		__func__, &bad_desc->lld.mbr_sa, &bad_desc->lld.mbr_da,
+		bad_desc->lld.mbr_ubc);
+
+	/* Then continue with usual descriptor management */
+}
+
 static void at_xdmac_tasklet(unsigned long data)
 {
 	struct at_xdmac_chan	*atchan = (struct at_xdmac_chan *)data;
@@ -1594,19 +1645,19 @@ static void at_xdmac_tasklet(unsigned long data)
 		   || (atchan->irq_status & error_mask)) {
 		struct dma_async_tx_descriptor  *txd;
 
-		if (atchan->irq_status & AT_XDMAC_CIS_RBEIS)
-			dev_err(chan2dev(&atchan->chan), "read bus error!!!");
-		if (atchan->irq_status & AT_XDMAC_CIS_WBEIS)
-			dev_err(chan2dev(&atchan->chan), "write bus error!!!");
-		if (atchan->irq_status & AT_XDMAC_CIS_ROIS)
-			dev_err(chan2dev(&atchan->chan), "request overflow error!!!");
+		if (atchan->irq_status & error_mask)
+			at_xdmac_handle_error(atchan);
 
 		spin_lock(&atchan->lock);
 		desc = list_first_entry(&atchan->xfers_list,
 					struct at_xdmac_desc,
 					xfer_node);
 		dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc);
-		BUG_ON(!desc->active_xfer);
+		if (!desc->active_xfer) {
+			dev_err(chan2dev(&atchan->chan), "Xfer not active: exiting");
+			spin_unlock(&atchan->lock);
+			return;
+		}
 
 		txd = &desc->tx_dma_desc;
 
diff --git a/drivers/dma/bcm-sba-raid.c b/drivers/dma/bcm-sba-raid.c
index 72878ac5c78d..fa81d0177765 100644
--- a/drivers/dma/bcm-sba-raid.c
+++ b/drivers/dma/bcm-sba-raid.c
@@ -1459,8 +1459,7 @@ static void sba_receive_message(struct mbox_client *cl, void *msg)
 
 static int sba_debugfs_stats_show(struct seq_file *file, void *offset)
 {
-	struct platform_device *pdev = to_platform_device(file->private);
-	struct sba_device *sba = platform_get_drvdata(pdev);
+	struct sba_device *sba = dev_get_drvdata(file->private);
 
 	/* Write stats in file */
 	sba_write_stats_in_seqfile(sba, file);
diff --git a/drivers/dma/bcm2835-dma.c b/drivers/dma/bcm2835-dma.c
index 54093ffd0aef..8101ff2f05c1 100644
--- a/drivers/dma/bcm2835-dma.c
+++ b/drivers/dma/bcm2835-dma.c
@@ -891,7 +891,6 @@ static int bcm2835_dma_probe(struct platform_device *pdev)
 	dma_cap_set(DMA_SLAVE, od->ddev.cap_mask);
 	dma_cap_set(DMA_PRIVATE, od->ddev.cap_mask);
 	dma_cap_set(DMA_CYCLIC, od->ddev.cap_mask);
-	dma_cap_set(DMA_SLAVE, od->ddev.cap_mask);
 	dma_cap_set(DMA_MEMCPY, od->ddev.cap_mask);
 	od->ddev.device_alloc_chan_resources = bcm2835_dma_alloc_chan_resources;
 	od->ddev.device_free_chan_resources = bcm2835_dma_free_chan_resources;
diff --git a/drivers/dma/dma-axi-dmac.c b/drivers/dma/dma-axi-dmac.c
index ffc0adc2f6ce..f32fdf21edbd 100644
--- a/drivers/dma/dma-axi-dmac.c
+++ b/drivers/dma/dma-axi-dmac.c
@@ -166,7 +166,7 @@ static int axi_dmac_dest_is_mem(struct axi_dmac_chan *chan)
 
 static bool axi_dmac_check_len(struct axi_dmac_chan *chan, unsigned int len)
 {
-	if (len == 0 || len > chan->max_length)
+	if (len == 0)
 		return false;
 	if ((len & chan->align_mask) != 0) /* Not aligned */
 		return false;
@@ -379,6 +379,49 @@ static struct axi_dmac_desc *axi_dmac_alloc_desc(unsigned int num_sgs)
 	return desc;
 }
 
+static struct axi_dmac_sg *axi_dmac_fill_linear_sg(struct axi_dmac_chan *chan,
+	enum dma_transfer_direction direction, dma_addr_t addr,
+	unsigned int num_periods, unsigned int period_len,
+	struct axi_dmac_sg *sg)
+{
+	unsigned int num_segments, i;
+	unsigned int segment_size;
+	unsigned int len;
+
+	/* Split into multiple equally sized segments if necessary */
+	num_segments = DIV_ROUND_UP(period_len, chan->max_length);
+	segment_size = DIV_ROUND_UP(period_len, num_segments);
+	/* Take care of alignment */
+	segment_size = ((segment_size - 1) | chan->align_mask) + 1;
+
+	for (i = 0; i < num_periods; i++) {
+		len = period_len;
+
+		while (len > segment_size) {
+			if (direction == DMA_DEV_TO_MEM)
+				sg->dest_addr = addr;
+			else
+				sg->src_addr = addr;
+			sg->x_len = segment_size;
+			sg->y_len = 1;
+			sg++;
+			addr += segment_size;
+			len -= segment_size;
+		}
+
+		if (direction == DMA_DEV_TO_MEM)
+			sg->dest_addr = addr;
+		else
+			sg->src_addr = addr;
+		sg->x_len = len;
+		sg->y_len = 1;
+		sg++;
+		addr += len;
+	}
+
+	return sg;
+}
+
 static struct dma_async_tx_descriptor *axi_dmac_prep_slave_sg(
 	struct dma_chan *c, struct scatterlist *sgl,
 	unsigned int sg_len, enum dma_transfer_direction direction,
@@ -386,16 +429,24 @@ static struct dma_async_tx_descriptor *axi_dmac_prep_slave_sg(
 {
 	struct axi_dmac_chan *chan = to_axi_dmac_chan(c);
 	struct axi_dmac_desc *desc;
+	struct axi_dmac_sg *dsg;
 	struct scatterlist *sg;
+	unsigned int num_sgs;
 	unsigned int i;
 
 	if (direction != chan->direction)
 		return NULL;
 
-	desc = axi_dmac_alloc_desc(sg_len);
+	num_sgs = 0;
+	for_each_sg(sgl, sg, sg_len, i)
+		num_sgs += DIV_ROUND_UP(sg_dma_len(sg), chan->max_length);
+
+	desc = axi_dmac_alloc_desc(num_sgs);
 	if (!desc)
 		return NULL;
 
+	dsg = desc->sg;
+
 	for_each_sg(sgl, sg, sg_len, i) {
 		if (!axi_dmac_check_addr(chan, sg_dma_address(sg)) ||
 		    !axi_dmac_check_len(chan, sg_dma_len(sg))) {
@@ -403,12 +454,8 @@ static struct dma_async_tx_descriptor *axi_dmac_prep_slave_sg(
 			return NULL;
 		}
 
-		if (direction == DMA_DEV_TO_MEM)
-			desc->sg[i].dest_addr = sg_dma_address(sg);
-		else
-			desc->sg[i].src_addr = sg_dma_address(sg);
-		desc->sg[i].x_len = sg_dma_len(sg);
-		desc->sg[i].y_len = 1;
+		dsg = axi_dmac_fill_linear_sg(chan, direction, sg_dma_address(sg), 1,
+			sg_dma_len(sg), dsg);
 	}
 
 	desc->cyclic = false;
@@ -423,7 +470,7 @@ static struct dma_async_tx_descriptor *axi_dmac_prep_dma_cyclic(
 {
 	struct axi_dmac_chan *chan = to_axi_dmac_chan(c);
 	struct axi_dmac_desc *desc;
-	unsigned int num_periods, i;
+	unsigned int num_periods, num_segments;
 
 	if (direction != chan->direction)
 		return NULL;
@@ -436,20 +483,14 @@ static struct dma_async_tx_descriptor *axi_dmac_prep_dma_cyclic(
 		return NULL;
 
 	num_periods = buf_len / period_len;
+	num_segments = DIV_ROUND_UP(period_len, chan->max_length);
 
-	desc = axi_dmac_alloc_desc(num_periods);
+	desc = axi_dmac_alloc_desc(num_periods * num_segments);
 	if (!desc)
 		return NULL;
 
-	for (i = 0; i < num_periods; i++) {
-		if (direction == DMA_DEV_TO_MEM)
-			desc->sg[i].dest_addr = buf_addr;
-		else
-			desc->sg[i].src_addr = buf_addr;
-		desc->sg[i].x_len = period_len;
-		desc->sg[i].y_len = 1;
-		buf_addr += period_len;
-	}
+	axi_dmac_fill_linear_sg(chan, direction, buf_addr, num_periods,
+		period_len, desc->sg);
 
 	desc->cyclic = true;
 
@@ -485,7 +526,7 @@ static struct dma_async_tx_descriptor *axi_dmac_prep_interleaved(
 
 	if (chan->hw_2d) {
 		if (!axi_dmac_check_len(chan, xt->sgl[0].size) ||
-		    !axi_dmac_check_len(chan, xt->numf))
+		    xt->numf == 0)
 			return NULL;
 		if (xt->sgl[0].size + dst_icg > chan->max_length ||
 		    xt->sgl[0].size + src_icg > chan->max_length)
@@ -577,15 +618,6 @@ static int axi_dmac_parse_chan_dt(struct device_node *of_chan,
 		return ret;
 	chan->dest_width = val / 8;
 
-	ret = of_property_read_u32(of_chan, "adi,length-width", &val);
-	if (ret)
-		return ret;
-
-	if (val >= 32)
-		chan->max_length = UINT_MAX;
-	else
-		chan->max_length = (1ULL << val) - 1;
-
 	chan->align_mask = max(chan->dest_width, chan->src_width) - 1;
 
 	if (axi_dmac_dest_is_mem(chan) && axi_dmac_src_is_mem(chan))
@@ -597,12 +629,27 @@ static int axi_dmac_parse_chan_dt(struct device_node *of_chan,
 	else
 		chan->direction = DMA_DEV_TO_DEV;
 
-	chan->hw_cyclic = of_property_read_bool(of_chan, "adi,cyclic");
-	chan->hw_2d = of_property_read_bool(of_chan, "adi,2d");
-
 	return 0;
 }
 
+static void axi_dmac_detect_caps(struct axi_dmac *dmac)
+{
+	struct axi_dmac_chan *chan = &dmac->chan;
+
+	axi_dmac_write(dmac, AXI_DMAC_REG_FLAGS, AXI_DMAC_FLAG_CYCLIC);
+	if (axi_dmac_read(dmac, AXI_DMAC_REG_FLAGS) == AXI_DMAC_FLAG_CYCLIC)
+		chan->hw_cyclic = true;
+
+	axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, 1);
+	if (axi_dmac_read(dmac, AXI_DMAC_REG_Y_LENGTH) == 1)
+		chan->hw_2d = true;
+
+	axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, 0xffffffff);
+	chan->max_length = axi_dmac_read(dmac, AXI_DMAC_REG_X_LENGTH);
+	if (chan->max_length != UINT_MAX)
+		chan->max_length++;
+}
+
 static int axi_dmac_probe(struct platform_device *pdev)
 {
 	struct device_node *of_channels, *of_chan;
@@ -647,11 +694,12 @@ static int axi_dmac_probe(struct platform_device *pdev)
 	of_node_put(of_channels);
 
 	pdev->dev.dma_parms = &dmac->dma_parms;
-	dma_set_max_seg_size(&pdev->dev, dmac->chan.max_length);
+	dma_set_max_seg_size(&pdev->dev, UINT_MAX);
 
 	dma_dev = &dmac->dma_dev;
 	dma_cap_set(DMA_SLAVE, dma_dev->cap_mask);
 	dma_cap_set(DMA_CYCLIC, dma_dev->cap_mask);
+	dma_cap_set(DMA_INTERLEAVE, dma_dev->cap_mask);
 	dma_dev->device_free_chan_resources = axi_dmac_free_chan_resources;
 	dma_dev->device_tx_status = dma_cookie_status;
 	dma_dev->device_issue_pending = axi_dmac_issue_pending;
@@ -675,6 +723,8 @@ static int axi_dmac_probe(struct platform_device *pdev)
 	if (ret < 0)
 		return ret;
 
+	axi_dmac_detect_caps(dmac);
+
 	axi_dmac_write(dmac, AXI_DMAC_REG_IRQ_MASK, 0x00);
 
 	ret = dma_async_device_register(dma_dev);
diff --git a/drivers/dma/fsl-edma-common.h b/drivers/dma/fsl-edma-common.h
index b435d8e1e3a1..c53f76eeb4d3 100644
--- a/drivers/dma/fsl-edma-common.h
+++ b/drivers/dma/fsl-edma-common.h
@@ -136,7 +136,7 @@ struct fsl_edma_desc {
 };
 
 enum edma_version {
-	v1, /* 32ch, Vybdir, mpc57x, etc */
+	v1, /* 32ch, Vybrid, mpc57x, etc */
 	v2, /* 64ch Coldfire */
 };
 
diff --git a/drivers/dma/fsl-edma.c b/drivers/dma/fsl-edma.c
index 75e8a7ba3a22..d641ef85a634 100644
--- a/drivers/dma/fsl-edma.c
+++ b/drivers/dma/fsl-edma.c
@@ -144,21 +144,21 @@ fsl_edma_irq_init(struct platform_device *pdev, struct fsl_edma_engine *fsl_edma
 				fsl_edma_irq_handler, 0, "eDMA", fsl_edma);
 		if (ret) {
 			dev_err(&pdev->dev, "Can't register eDMA IRQ.\n");
-			 return  ret;
+			return ret;
 		}
 	} else {
 		ret = devm_request_irq(&pdev->dev, fsl_edma->txirq,
 				fsl_edma_tx_handler, 0, "eDMA tx", fsl_edma);
 		if (ret) {
 			dev_err(&pdev->dev, "Can't register eDMA tx IRQ.\n");
-			return  ret;
+			return ret;
 		}
 
 		ret = devm_request_irq(&pdev->dev, fsl_edma->errirq,
 				fsl_edma_err_handler, 0, "eDMA err", fsl_edma);
 		if (ret) {
 			dev_err(&pdev->dev, "Can't register eDMA err IRQ.\n");
-			return  ret;
+			return ret;
 		}
 	}
 
diff --git a/drivers/dma/idma64.c b/drivers/dma/idma64.c
index 0baf9797cc09..07fd4f25cdd8 100644
--- a/drivers/dma/idma64.c
+++ b/drivers/dma/idma64.c
@@ -19,10 +19,9 @@
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 
-#include "idma64.h"
+#include <linux/dma/idma64.h>
 
-/* Platform driver name */
-#define DRV_NAME		"idma64"
+#include "idma64.h"
 
 /* For now we support only two channels */
 #define IDMA64_NR_CHAN		2
@@ -592,7 +591,7 @@ static int idma64_probe(struct idma64_chip *chip)
 	idma64->dma.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
 	idma64->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
 
-	idma64->dma.dev = chip->dev;
+	idma64->dma.dev = chip->sysdev;
 
 	dma_set_max_seg_size(idma64->dma.dev, IDMA64C_CTLH_BLOCK_TS_MASK);
 
@@ -632,6 +631,7 @@ static int idma64_platform_probe(struct platform_device *pdev)
 {
 	struct idma64_chip *chip;
 	struct device *dev = &pdev->dev;
+	struct device *sysdev = dev->parent;
 	struct resource *mem;
 	int ret;
 
@@ -648,11 +648,12 @@ static int idma64_platform_probe(struct platform_device *pdev)
 	if (IS_ERR(chip->regs))
 		return PTR_ERR(chip->regs);
 
-	ret = dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
+	ret = dma_coerce_mask_and_coherent(sysdev, DMA_BIT_MASK(64));
 	if (ret)
 		return ret;
 
 	chip->dev = dev;
+	chip->sysdev = sysdev;
 
 	ret = idma64_probe(chip);
 	if (ret)
@@ -697,7 +698,7 @@ static struct platform_driver idma64_platform_driver = {
 	.probe		= idma64_platform_probe,
 	.remove		= idma64_platform_remove,
 	.driver = {
-		.name	= DRV_NAME,
+		.name	= LPSS_IDMA64_DRIVER_NAME,
 		.pm	= &idma64_dev_pm_ops,
 	},
 };
@@ -707,4 +708,4 @@ module_platform_driver(idma64_platform_driver);
 MODULE_LICENSE("GPL v2");
 MODULE_DESCRIPTION("iDMA64 core driver");
 MODULE_AUTHOR("Andy Shevchenko <andriy.shevchenko@linux.intel.com>");
-MODULE_ALIAS("platform:" DRV_NAME);
+MODULE_ALIAS("platform:" LPSS_IDMA64_DRIVER_NAME);
diff --git a/drivers/dma/idma64.h b/drivers/dma/idma64.h
index 6b816878e5e7..baa32e1425de 100644
--- a/drivers/dma/idma64.h
+++ b/drivers/dma/idma64.h
@@ -216,12 +216,14 @@ static inline void idma64_writel(struct idma64 *idma64, int offset, u32 value)
 /**
  * struct idma64_chip - representation of iDMA 64-bit controller hardware
  * @dev:		struct device of the DMA controller
+ * @sysdev:		struct device of the physical device that does DMA
  * @irq:		irq line
  * @regs:		memory mapped I/O space
  * @idma64:		struct idma64 that is filed by idma64_probe()
  */
 struct idma64_chip {
 	struct device	*dev;
+	struct device	*sysdev;
 	int		irq;
 	void __iomem	*regs;
 	struct idma64	*idma64;
diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c
index 5f3c1378b90e..99d9f431ae2c 100644
--- a/drivers/dma/imx-sdma.c
+++ b/drivers/dma/imx-sdma.c
@@ -419,6 +419,7 @@ struct sdma_driver_data {
 	int chnenbl0;
 	int num_events;
 	struct sdma_script_start_addrs	*script_addrs;
+	bool check_ratio;
 };
 
 struct sdma_engine {
@@ -557,6 +558,13 @@ static struct sdma_driver_data sdma_imx7d = {
 	.script_addrs = &sdma_script_imx7d,
 };
 
+static struct sdma_driver_data sdma_imx8mq = {
+	.chnenbl0 = SDMA_CHNENBL0_IMX35,
+	.num_events = 48,
+	.script_addrs = &sdma_script_imx7d,
+	.check_ratio = 1,
+};
+
 static const struct platform_device_id sdma_devtypes[] = {
 	{
 		.name = "imx25-sdma",
@@ -580,6 +588,9 @@ static const struct platform_device_id sdma_devtypes[] = {
 		.name = "imx7d-sdma",
 		.driver_data = (unsigned long)&sdma_imx7d,
 	}, {
+		.name = "imx8mq-sdma",
+		.driver_data = (unsigned long)&sdma_imx8mq,
+	}, {
 		/* sentinel */
 	}
 };
@@ -593,6 +604,7 @@ static const struct of_device_id sdma_dt_ids[] = {
 	{ .compatible = "fsl,imx31-sdma", .data = &sdma_imx31, },
 	{ .compatible = "fsl,imx25-sdma", .data = &sdma_imx25, },
 	{ .compatible = "fsl,imx7d-sdma", .data = &sdma_imx7d, },
+	{ .compatible = "fsl,imx8mq-sdma", .data = &sdma_imx8mq, },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, sdma_dt_ids);
@@ -1852,7 +1864,8 @@ static int sdma_init(struct sdma_engine *sdma)
 	if (ret)
 		goto disable_clk_ipg;
 
-	if (clk_get_rate(sdma->clk_ahb) == clk_get_rate(sdma->clk_ipg))
+	if (sdma->drvdata->check_ratio &&
+	    (clk_get_rate(sdma->clk_ahb) == clk_get_rate(sdma->clk_ipg)))
 		sdma->clk_ratio = 1;
 
 	/* Be sure SDMA has not started yet */
diff --git a/drivers/dma/nbpfaxi.c b/drivers/dma/nbpfaxi.c
index a67b292190f4..594409a6e975 100644
--- a/drivers/dma/nbpfaxi.c
+++ b/drivers/dma/nbpfaxi.c
@@ -1491,14 +1491,14 @@ MODULE_DEVICE_TABLE(platform, nbpf_ids);
 #ifdef CONFIG_PM
 static int nbpf_runtime_suspend(struct device *dev)
 {
-	struct nbpf_device *nbpf = platform_get_drvdata(to_platform_device(dev));
+	struct nbpf_device *nbpf = dev_get_drvdata(dev);
 	clk_disable_unprepare(nbpf->clk);
 	return 0;
 }
 
 static int nbpf_runtime_resume(struct device *dev)
 {
-	struct nbpf_device *nbpf = platform_get_drvdata(to_platform_device(dev));
+	struct nbpf_device *nbpf = dev_get_drvdata(dev);
 	return clk_prepare_enable(nbpf->clk);
 }
 #endif
diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c
index eec79fdf27a5..6e6837214210 100644
--- a/drivers/dma/pl330.c
+++ b/drivers/dma/pl330.c
@@ -11,6 +11,7 @@
  * (at your option) any later version.
  */
 
+#include <linux/debugfs.h>
 #include <linux/kernel.h>
 #include <linux/io.h>
 #include <linux/init.h>
@@ -966,6 +967,7 @@ static void _stop(struct pl330_thread *thrd)
 {
 	void __iomem *regs = thrd->dmac->base;
 	u8 insn[6] = {0, 0, 0, 0, 0, 0};
+	u32 inten = readl(regs + INTEN);
 
 	if (_state(thrd) == PL330_STATE_FAULT_COMPLETING)
 		UNTIL(thrd, PL330_STATE_FAULTING | PL330_STATE_KILLING);
@@ -978,10 +980,13 @@ static void _stop(struct pl330_thread *thrd)
 
 	_emit_KILL(0, insn);
 
-	/* Stop generating interrupts for SEV */
-	writel(readl(regs + INTEN) & ~(1 << thrd->ev), regs + INTEN);
-
 	_execute_DBGINSN(thrd, insn, is_manager(thrd));
+
+	/* clear the event */
+	if (inten & (1 << thrd->ev))
+		writel(1 << thrd->ev, regs + INTCLR);
+	/* Stop generating interrupts for SEV */
+	writel(inten & ~(1 << thrd->ev), regs + INTEN);
 }
 
 /* Start doing req 'idx' of thread 'thrd' */
@@ -2896,6 +2901,55 @@ static irqreturn_t pl330_irq_handler(int irq, void *data)
 	BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \
 	BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)
 
+#ifdef CONFIG_DEBUG_FS
+static int pl330_debugfs_show(struct seq_file *s, void *data)
+{
+	struct pl330_dmac *pl330 = s->private;
+	int chans, pchs, ch, pr;
+
+	chans = pl330->pcfg.num_chan;
+	pchs = pl330->num_peripherals;
+
+	seq_puts(s, "PL330 physical channels:\n");
+	seq_puts(s, "THREAD:\t\tCHANNEL:\n");
+	seq_puts(s, "--------\t-----\n");
+	for (ch = 0; ch < chans; ch++) {
+		struct pl330_thread *thrd = &pl330->channels[ch];
+		int found = -1;
+
+		for (pr = 0; pr < pchs; pr++) {
+			struct dma_pl330_chan *pch = &pl330->peripherals[pr];
+
+			if (!pch->thread || thrd->id != pch->thread->id)
+				continue;
+
+			found = pr;
+		}
+
+		seq_printf(s, "%d\t\t", thrd->id);
+		if (found == -1)
+			seq_puts(s, "--\n");
+		else
+			seq_printf(s, "%d\n", found);
+	}
+
+	return 0;
+}
+
+DEFINE_SHOW_ATTRIBUTE(pl330_debugfs);
+
+static inline void init_pl330_debugfs(struct pl330_dmac *pl330)
+{
+	debugfs_create_file(dev_name(pl330->ddma.dev),
+			    S_IFREG | 0444, NULL, pl330,
+			    &pl330_debugfs_fops);
+}
+#else
+static inline void init_pl330_debugfs(struct pl330_dmac *pl330)
+{
+}
+#endif
+
 /*
  * Runtime PM callbacks are provided by amba/bus.c driver.
  *
@@ -3082,6 +3136,7 @@ pl330_probe(struct amba_device *adev, const struct amba_id *id)
 		dev_err(&adev->dev, "unable to set the seg size\n");
 
 
+	init_pl330_debugfs(pl330);
 	dev_info(&adev->dev,
 		"Loaded driver for PL330 DMAC-%x\n", adev->periphid);
 	dev_info(&adev->dev,
diff --git a/drivers/dma/sh/rcar-dmac.c b/drivers/dma/sh/rcar-dmac.c
index e2a5398f89b5..33ab1b607e2b 100644
--- a/drivers/dma/sh/rcar-dmac.c
+++ b/drivers/dma/sh/rcar-dmac.c
@@ -1,8 +1,8 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
- * Renesas R-Car Gen2 DMA Controller Driver
+ * Renesas R-Car Gen2/Gen3 DMA Controller Driver
  *
- * Copyright (C) 2014 Renesas Electronics Inc.
+ * Copyright (C) 2014-2019 Renesas Electronics Inc.
  *
  * Author: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
  */
diff --git a/drivers/dma/stm32-dma.c b/drivers/dma/stm32-dma.c
index ba239b529fa9..88d9c6c4389f 100644
--- a/drivers/dma/stm32-dma.c
+++ b/drivers/dma/stm32-dma.c
@@ -1042,33 +1042,97 @@ static u32 stm32_dma_get_remaining_bytes(struct stm32_dma_chan *chan)
 	return ndtr << width;
 }
 
+/**
+ * stm32_dma_is_current_sg - check that expected sg_req is currently transferred
+ * @chan: dma channel
+ *
+ * This function called when IRQ are disable, checks that the hardware has not
+ * switched on the next transfer in double buffer mode. The test is done by
+ * comparing the next_sg memory address with the hardware related register
+ * (based on CT bit value).
+ *
+ * Returns true if expected current transfer is still running or double
+ * buffer mode is not activated.
+ */
+static bool stm32_dma_is_current_sg(struct stm32_dma_chan *chan)
+{
+	struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan);
+	struct stm32_dma_sg_req *sg_req;
+	u32 dma_scr, dma_smar, id;
+
+	id = chan->id;
+	dma_scr = stm32_dma_read(dmadev, STM32_DMA_SCR(id));
+
+	if (!(dma_scr & STM32_DMA_SCR_DBM))
+		return true;
+
+	sg_req = &chan->desc->sg_req[chan->next_sg];
+
+	if (dma_scr & STM32_DMA_SCR_CT) {
+		dma_smar = stm32_dma_read(dmadev, STM32_DMA_SM0AR(id));
+		return (dma_smar == sg_req->chan_reg.dma_sm0ar);
+	}
+
+	dma_smar = stm32_dma_read(dmadev, STM32_DMA_SM1AR(id));
+
+	return (dma_smar == sg_req->chan_reg.dma_sm1ar);
+}
+
 static size_t stm32_dma_desc_residue(struct stm32_dma_chan *chan,
 				     struct stm32_dma_desc *desc,
 				     u32 next_sg)
 {
 	u32 modulo, burst_size;
-	u32 residue = 0;
+	u32 residue;
+	u32 n_sg = next_sg;
+	struct stm32_dma_sg_req *sg_req = &chan->desc->sg_req[chan->next_sg];
 	int i;
 
 	/*
-	 * In cyclic mode, for the last period, residue = remaining bytes from
-	 * NDTR
+	 * Calculate the residue means compute the descriptors
+	 * information:
+	 * - the sg_req currently transferred
+	 * - the Hardware remaining position in this sg (NDTR bits field).
+	 *
+	 * A race condition may occur if DMA is running in cyclic or double
+	 * buffer mode, since the DMA register are automatically reloaded at end
+	 * of period transfer. The hardware may have switched to the next
+	 * transfer (CT bit updated) just before the position (SxNDTR reg) is
+	 * read.
+	 * In this case the SxNDTR reg could (or not) correspond to the new
+	 * transfer position, and not the expected one.
+	 * The strategy implemented in the stm32 driver is to:
+	 *  - read the SxNDTR register
+	 *  - crosscheck that hardware is still in current transfer.
+	 * In case of switch, we can assume that the DMA is at the beginning of
+	 * the next transfer. So we approximate the residue in consequence, by
+	 * pointing on the beginning of next transfer.
+	 *
+	 * This race condition doesn't apply for none cyclic mode, as double
+	 * buffer is not used. In such situation registers are updated by the
+	 * software.
 	 */
-	if (chan->desc->cyclic && next_sg == 0) {
-		residue = stm32_dma_get_remaining_bytes(chan);
-		goto end;
+
+	residue = stm32_dma_get_remaining_bytes(chan);
+
+	if (!stm32_dma_is_current_sg(chan)) {
+		n_sg++;
+		if (n_sg == chan->desc->num_sgs)
+			n_sg = 0;
+		residue = sg_req->len;
 	}
 
 	/*
-	 * For all other periods in cyclic mode, and in sg mode,
-	 * residue = remaining bytes from NDTR + remaining periods/sg to be
-	 * transferred
+	 * In cyclic mode, for the last period, residue = remaining bytes
+	 * from NDTR,
+	 * else for all other periods in cyclic mode, and in sg mode,
+	 * residue = remaining bytes from NDTR + remaining
+	 * periods/sg to be transferred
 	 */
-	for (i = next_sg; i < desc->num_sgs; i++)
-		residue += desc->sg_req[i].len;
-	residue += stm32_dma_get_remaining_bytes(chan);
+	if (!chan->desc->cyclic || n_sg != 0)
+		for (i = n_sg; i < desc->num_sgs; i++)
+			residue += desc->sg_req[i].len;
 
-end:
 	if (!chan->mem_burst)
 		return residue;
 
@@ -1302,13 +1366,16 @@ static int stm32_dma_probe(struct platform_device *pdev)
 
 	for (i = 0; i < STM32_DMA_MAX_CHANNELS; i++) {
 		chan = &dmadev->chan[i];
-		res = platform_get_resource(pdev, IORESOURCE_IRQ, i);
-		if (!res) {
-			ret = -EINVAL;
-			dev_err(&pdev->dev, "No irq resource for chan %d\n", i);
+		chan->irq = platform_get_irq(pdev, i);
+		ret = platform_get_irq(pdev, i);
+		if (ret < 0)  {
+			if (ret != -EPROBE_DEFER)
+				dev_err(&pdev->dev,
+					"No irq resource for chan %d\n", i);
 			goto err_unregister;
 		}
-		chan->irq = res->start;
+		chan->irq = ret;
+
 		ret = devm_request_irq(&pdev->dev, chan->irq,
 				       stm32_dma_chan_irq, 0,
 				       dev_name(chan2dev(chan)), chan);
diff --git a/drivers/dma/tegra210-adma.c b/drivers/dma/tegra210-adma.c
index 5ec0dd97b397..21f6be16d013 100644
--- a/drivers/dma/tegra210-adma.c
+++ b/drivers/dma/tegra210-adma.c
@@ -22,7 +22,6 @@
 #include <linux/of_device.h>
 #include <linux/of_dma.h>
 #include <linux/of_irq.h>
-#include <linux/pm_clock.h>
 #include <linux/pm_runtime.h>
 #include <linux/slab.h>
 
@@ -31,35 +30,33 @@
 #define ADMA_CH_CMD					0x00
 #define ADMA_CH_STATUS					0x0c
 #define ADMA_CH_STATUS_XFER_EN				BIT(0)
+#define ADMA_CH_STATUS_XFER_PAUSED			BIT(1)
 
 #define ADMA_CH_INT_STATUS				0x10
 #define ADMA_CH_INT_STATUS_XFER_DONE			BIT(0)
 
 #define ADMA_CH_INT_CLEAR				0x1c
 #define ADMA_CH_CTRL					0x24
-#define ADMA_CH_CTRL_TX_REQ(val)			(((val) & 0xf) << 28)
-#define ADMA_CH_CTRL_TX_REQ_MAX				10
-#define ADMA_CH_CTRL_RX_REQ(val)			(((val) & 0xf) << 24)
-#define ADMA_CH_CTRL_RX_REQ_MAX				10
 #define ADMA_CH_CTRL_DIR(val)				(((val) & 0xf) << 12)
 #define ADMA_CH_CTRL_DIR_AHUB2MEM			2
 #define ADMA_CH_CTRL_DIR_MEM2AHUB			4
 #define ADMA_CH_CTRL_MODE_CONTINUOUS			(2 << 8)
 #define ADMA_CH_CTRL_FLOWCTRL_EN			BIT(1)
+#define ADMA_CH_CTRL_XFER_PAUSE_SHIFT			0
 
 #define ADMA_CH_CONFIG					0x28
 #define ADMA_CH_CONFIG_SRC_BUF(val)			(((val) & 0x7) << 28)
 #define ADMA_CH_CONFIG_TRG_BUF(val)			(((val) & 0x7) << 24)
-#define ADMA_CH_CONFIG_BURST_SIZE(val)			(((val) & 0x7) << 20)
-#define ADMA_CH_CONFIG_BURST_16				5
+#define ADMA_CH_CONFIG_BURST_SIZE_SHIFT			20
+#define ADMA_CH_CONFIG_MAX_BURST_SIZE                   16
 #define ADMA_CH_CONFIG_WEIGHT_FOR_WRR(val)		((val) & 0xf)
 #define ADMA_CH_CONFIG_MAX_BUFS				8
 
 #define ADMA_CH_FIFO_CTRL				0x2c
 #define ADMA_CH_FIFO_CTRL_OVRFW_THRES(val)		(((val) & 0xf) << 24)
 #define ADMA_CH_FIFO_CTRL_STARV_THRES(val)		(((val) & 0xf) << 16)
-#define ADMA_CH_FIFO_CTRL_TX_SIZE(val)			(((val) & 0xf) << 8)
-#define ADMA_CH_FIFO_CTRL_RX_SIZE(val)			((val) & 0xf)
+#define ADMA_CH_FIFO_CTRL_TX_FIFO_SIZE_SHIFT		8
+#define ADMA_CH_FIFO_CTRL_RX_FIFO_SIZE_SHIFT		0
 
 #define ADMA_CH_LOWER_SRC_ADDR				0x34
 #define ADMA_CH_LOWER_TRG_ADDR				0x3c
@@ -69,25 +66,41 @@
 #define ADMA_CH_XFER_STATUS				0x54
 #define ADMA_CH_XFER_STATUS_COUNT_MASK			0xffff
 
-#define ADMA_GLOBAL_CMD					0xc00
-#define ADMA_GLOBAL_SOFT_RESET				0xc04
-#define ADMA_GLOBAL_INT_CLEAR				0xc20
-#define ADMA_GLOBAL_CTRL				0xc24
+#define ADMA_GLOBAL_CMD					0x00
+#define ADMA_GLOBAL_SOFT_RESET				0x04
 
-#define ADMA_CH_REG_OFFSET(a)				(a * 0x80)
+#define TEGRA_ADMA_BURST_COMPLETE_TIME			20
 
 #define ADMA_CH_FIFO_CTRL_DEFAULT	(ADMA_CH_FIFO_CTRL_OVRFW_THRES(1) | \
-					 ADMA_CH_FIFO_CTRL_STARV_THRES(1) | \
-					 ADMA_CH_FIFO_CTRL_TX_SIZE(3)     | \
-					 ADMA_CH_FIFO_CTRL_RX_SIZE(3))
+					 ADMA_CH_FIFO_CTRL_STARV_THRES(1))
+
+#define ADMA_CH_REG_FIELD_VAL(val, mask, shift)	(((val) & mask) << shift)
+
 struct tegra_adma;
 
 /*
  * struct tegra_adma_chip_data - Tegra chip specific data
+ * @global_reg_offset: Register offset of DMA global register.
+ * @global_int_clear: Register offset of DMA global interrupt clear.
+ * @ch_req_tx_shift: Register offset for AHUB transmit channel select.
+ * @ch_req_rx_shift: Register offset for AHUB receive channel select.
+ * @ch_base_offset: Reister offset of DMA channel registers.
+ * @ch_req_mask: Mask for Tx or Rx channel select.
+ * @ch_req_max: Maximum number of Tx or Rx channels available.
+ * @ch_reg_size: Size of DMA channel register space.
  * @nr_channels: Number of DMA channels available.
  */
 struct tegra_adma_chip_data {
-	int nr_channels;
+	unsigned int (*adma_get_burst_config)(unsigned int burst_size);
+	unsigned int global_reg_offset;
+	unsigned int global_int_clear;
+	unsigned int ch_req_tx_shift;
+	unsigned int ch_req_rx_shift;
+	unsigned int ch_base_offset;
+	unsigned int ch_req_mask;
+	unsigned int ch_req_max;
+	unsigned int ch_reg_size;
+	unsigned int nr_channels;
 };
 
 /*
@@ -99,6 +112,7 @@ struct tegra_adma_chan_regs {
 	unsigned int src_addr;
 	unsigned int trg_addr;
 	unsigned int fifo_ctrl;
+	unsigned int cmd;
 	unsigned int tc;
 };
 
@@ -128,6 +142,7 @@ struct tegra_adma_chan {
 	enum dma_transfer_direction	sreq_dir;
 	unsigned int			sreq_index;
 	bool				sreq_reserved;
+	struct tegra_adma_chan_regs	ch_regs;
 
 	/* Transfer count and position info */
 	unsigned int			tx_buf_count;
@@ -141,6 +156,7 @@ struct tegra_adma {
 	struct dma_device		dma_dev;
 	struct device			*dev;
 	void __iomem			*base_addr;
+	struct clk			*ahub_clk;
 	unsigned int			nr_channels;
 	unsigned long			rx_requests_reserved;
 	unsigned long			tx_requests_reserved;
@@ -148,18 +164,20 @@ struct tegra_adma {
 	/* Used to store global command register state when suspending */
 	unsigned int			global_cmd;
 
+	const struct tegra_adma_chip_data *cdata;
+
 	/* Last member of the structure */
 	struct tegra_adma_chan		channels[0];
 };
 
 static inline void tdma_write(struct tegra_adma *tdma, u32 reg, u32 val)
 {
-	writel(val, tdma->base_addr + reg);
+	writel(val, tdma->base_addr + tdma->cdata->global_reg_offset + reg);
 }
 
 static inline u32 tdma_read(struct tegra_adma *tdma, u32 reg)
 {
-	return readl(tdma->base_addr + reg);
+	return readl(tdma->base_addr + tdma->cdata->global_reg_offset + reg);
 }
 
 static inline void tdma_ch_write(struct tegra_adma_chan *tdc, u32 reg, u32 val)
@@ -209,14 +227,16 @@ static int tegra_adma_init(struct tegra_adma *tdma)
 	int ret;
 
 	/* Clear any interrupts */
-	tdma_write(tdma, ADMA_GLOBAL_INT_CLEAR, 0x1);
+	tdma_write(tdma, tdma->cdata->global_int_clear, 0x1);
 
 	/* Assert soft reset */
 	tdma_write(tdma, ADMA_GLOBAL_SOFT_RESET, 0x1);
 
 	/* Wait for reset to clear */
 	ret = readx_poll_timeout(readl,
-				 tdma->base_addr + ADMA_GLOBAL_SOFT_RESET,
+				 tdma->base_addr +
+				 tdma->cdata->global_reg_offset +
+				 ADMA_GLOBAL_SOFT_RESET,
 				 status, status == 0, 20, 10000);
 	if (ret)
 		return ret;
@@ -236,13 +256,13 @@ static int tegra_adma_request_alloc(struct tegra_adma_chan *tdc,
 	if (tdc->sreq_reserved)
 		return tdc->sreq_dir == direction ? 0 : -EINVAL;
 
+	if (sreq_index > tdma->cdata->ch_req_max) {
+		dev_err(tdma->dev, "invalid DMA request\n");
+		return -EINVAL;
+	}
+
 	switch (direction) {
 	case DMA_MEM_TO_DEV:
-		if (sreq_index > ADMA_CH_CTRL_TX_REQ_MAX) {
-			dev_err(tdma->dev, "invalid DMA request\n");
-			return -EINVAL;
-		}
-
 		if (test_and_set_bit(sreq_index, &tdma->tx_requests_reserved)) {
 			dev_err(tdma->dev, "DMA request reserved\n");
 			return -EINVAL;
@@ -250,11 +270,6 @@ static int tegra_adma_request_alloc(struct tegra_adma_chan *tdc,
 		break;
 
 	case DMA_DEV_TO_MEM:
-		if (sreq_index > ADMA_CH_CTRL_RX_REQ_MAX) {
-			dev_err(tdma->dev, "invalid DMA request\n");
-			return -EINVAL;
-		}
-
 		if (test_and_set_bit(sreq_index, &tdma->rx_requests_reserved)) {
 			dev_err(tdma->dev, "DMA request reserved\n");
 			return -EINVAL;
@@ -428,6 +443,51 @@ static void tegra_adma_issue_pending(struct dma_chan *dc)
 	spin_unlock_irqrestore(&tdc->vc.lock, flags);
 }
 
+static bool tegra_adma_is_paused(struct tegra_adma_chan *tdc)
+{
+	u32 csts;
+
+	csts = tdma_ch_read(tdc, ADMA_CH_STATUS);
+	csts &= ADMA_CH_STATUS_XFER_PAUSED;
+
+	return csts ? true : false;
+}
+
+static int tegra_adma_pause(struct dma_chan *dc)
+{
+	struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);
+	struct tegra_adma_desc *desc = tdc->desc;
+	struct tegra_adma_chan_regs *ch_regs = &desc->ch_regs;
+	int dcnt = 10;
+
+	ch_regs->ctrl = tdma_ch_read(tdc, ADMA_CH_CTRL);
+	ch_regs->ctrl |= (1 << ADMA_CH_CTRL_XFER_PAUSE_SHIFT);
+	tdma_ch_write(tdc, ADMA_CH_CTRL, ch_regs->ctrl);
+
+	while (dcnt-- && !tegra_adma_is_paused(tdc))
+		udelay(TEGRA_ADMA_BURST_COMPLETE_TIME);
+
+	if (dcnt < 0) {
+		dev_err(tdc2dev(tdc), "unable to pause DMA channel\n");
+		return -EBUSY;
+	}
+
+	return 0;
+}
+
+static int tegra_adma_resume(struct dma_chan *dc)
+{
+	struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);
+	struct tegra_adma_desc *desc = tdc->desc;
+	struct tegra_adma_chan_regs *ch_regs = &desc->ch_regs;
+
+	ch_regs->ctrl = tdma_ch_read(tdc, ADMA_CH_CTRL);
+	ch_regs->ctrl &= ~(1 << ADMA_CH_CTRL_XFER_PAUSE_SHIFT);
+	tdma_ch_write(tdc, ADMA_CH_CTRL, ch_regs->ctrl);
+
+	return 0;
+}
+
 static int tegra_adma_terminate_all(struct dma_chan *dc)
 {
 	struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc);
@@ -481,12 +541,29 @@ static enum dma_status tegra_adma_tx_status(struct dma_chan *dc,
 	return ret;
 }
 
+static unsigned int tegra210_adma_get_burst_config(unsigned int burst_size)
+{
+	if (!burst_size || burst_size > ADMA_CH_CONFIG_MAX_BURST_SIZE)
+		burst_size = ADMA_CH_CONFIG_MAX_BURST_SIZE;
+
+	return fls(burst_size) << ADMA_CH_CONFIG_BURST_SIZE_SHIFT;
+}
+
+static unsigned int tegra186_adma_get_burst_config(unsigned int burst_size)
+{
+	if (!burst_size || burst_size > ADMA_CH_CONFIG_MAX_BURST_SIZE)
+		burst_size = ADMA_CH_CONFIG_MAX_BURST_SIZE;
+
+	return (burst_size - 1) << ADMA_CH_CONFIG_BURST_SIZE_SHIFT;
+}
+
 static int tegra_adma_set_xfer_params(struct tegra_adma_chan *tdc,
 				      struct tegra_adma_desc *desc,
 				      dma_addr_t buf_addr,
 				      enum dma_transfer_direction direction)
 {
 	struct tegra_adma_chan_regs *ch_regs = &desc->ch_regs;
+	const struct tegra_adma_chip_data *cdata = tdc->tdma->cdata;
 	unsigned int burst_size, adma_dir;
 
 	if (desc->num_periods > ADMA_CH_CONFIG_MAX_BUFS)
@@ -495,17 +572,21 @@ static int tegra_adma_set_xfer_params(struct tegra_adma_chan *tdc,
 	switch (direction) {
 	case DMA_MEM_TO_DEV:
 		adma_dir = ADMA_CH_CTRL_DIR_MEM2AHUB;
-		burst_size = fls(tdc->sconfig.dst_maxburst);
+		burst_size = tdc->sconfig.dst_maxburst;
 		ch_regs->config = ADMA_CH_CONFIG_SRC_BUF(desc->num_periods - 1);
-		ch_regs->ctrl = ADMA_CH_CTRL_TX_REQ(tdc->sreq_index);
+		ch_regs->ctrl = ADMA_CH_REG_FIELD_VAL(tdc->sreq_index,
+						      cdata->ch_req_mask,
+						      cdata->ch_req_tx_shift);
 		ch_regs->src_addr = buf_addr;
 		break;
 
 	case DMA_DEV_TO_MEM:
 		adma_dir = ADMA_CH_CTRL_DIR_AHUB2MEM;
-		burst_size = fls(tdc->sconfig.src_maxburst);
+		burst_size = tdc->sconfig.src_maxburst;
 		ch_regs->config = ADMA_CH_CONFIG_TRG_BUF(desc->num_periods - 1);
-		ch_regs->ctrl = ADMA_CH_CTRL_RX_REQ(tdc->sreq_index);
+		ch_regs->ctrl = ADMA_CH_REG_FIELD_VAL(tdc->sreq_index,
+						      cdata->ch_req_mask,
+						      cdata->ch_req_rx_shift);
 		ch_regs->trg_addr = buf_addr;
 		break;
 
@@ -514,13 +595,10 @@ static int tegra_adma_set_xfer_params(struct tegra_adma_chan *tdc,
 		return -EINVAL;
 	}
 
-	if (!burst_size || burst_size > ADMA_CH_CONFIG_BURST_16)
-		burst_size = ADMA_CH_CONFIG_BURST_16;
-
 	ch_regs->ctrl |= ADMA_CH_CTRL_DIR(adma_dir) |
 			 ADMA_CH_CTRL_MODE_CONTINUOUS |
 			 ADMA_CH_CTRL_FLOWCTRL_EN;
-	ch_regs->config |= ADMA_CH_CONFIG_BURST_SIZE(burst_size);
+	ch_regs->config |= cdata->adma_get_burst_config(burst_size);
 	ch_regs->config |= ADMA_CH_CONFIG_WEIGHT_FOR_WRR(1);
 	ch_regs->fifo_ctrl = ADMA_CH_FIFO_CTRL_DEFAULT;
 	ch_regs->tc = desc->period_len & ADMA_CH_TC_COUNT_MASK;
@@ -635,32 +713,99 @@ static struct dma_chan *tegra_dma_of_xlate(struct of_phandle_args *dma_spec,
 static int tegra_adma_runtime_suspend(struct device *dev)
 {
 	struct tegra_adma *tdma = dev_get_drvdata(dev);
+	struct tegra_adma_chan_regs *ch_reg;
+	struct tegra_adma_chan *tdc;
+	int i;
 
 	tdma->global_cmd = tdma_read(tdma, ADMA_GLOBAL_CMD);
+	if (!tdma->global_cmd)
+		goto clk_disable;
+
+	for (i = 0; i < tdma->nr_channels; i++) {
+		tdc = &tdma->channels[i];
+		ch_reg = &tdc->ch_regs;
+		ch_reg->cmd = tdma_ch_read(tdc, ADMA_CH_CMD);
+		/* skip if channel is not active */
+		if (!ch_reg->cmd)
+			continue;
+		ch_reg->tc = tdma_ch_read(tdc, ADMA_CH_TC);
+		ch_reg->src_addr = tdma_ch_read(tdc, ADMA_CH_LOWER_SRC_ADDR);
+		ch_reg->trg_addr = tdma_ch_read(tdc, ADMA_CH_LOWER_TRG_ADDR);
+		ch_reg->ctrl = tdma_ch_read(tdc, ADMA_CH_CTRL);
+		ch_reg->fifo_ctrl = tdma_ch_read(tdc, ADMA_CH_FIFO_CTRL);
+		ch_reg->config = tdma_ch_read(tdc, ADMA_CH_CONFIG);
+	}
+
+clk_disable:
+	clk_disable_unprepare(tdma->ahub_clk);
 
-	return pm_clk_suspend(dev);
+	return 0;
 }
 
 static int tegra_adma_runtime_resume(struct device *dev)
 {
 	struct tegra_adma *tdma = dev_get_drvdata(dev);
-	int ret;
+	struct tegra_adma_chan_regs *ch_reg;
+	struct tegra_adma_chan *tdc;
+	int ret, i;
 
-	ret = pm_clk_resume(dev);
-	if (ret)
+	ret = clk_prepare_enable(tdma->ahub_clk);
+	if (ret) {
+		dev_err(dev, "ahub clk_enable failed: %d\n", ret);
 		return ret;
-
+	}
 	tdma_write(tdma, ADMA_GLOBAL_CMD, tdma->global_cmd);
 
+	if (!tdma->global_cmd)
+		return 0;
+
+	for (i = 0; i < tdma->nr_channels; i++) {
+		tdc = &tdma->channels[i];
+		ch_reg = &tdc->ch_regs;
+		/* skip if channel was not active earlier */
+		if (!ch_reg->cmd)
+			continue;
+		tdma_ch_write(tdc, ADMA_CH_TC, ch_reg->tc);
+		tdma_ch_write(tdc, ADMA_CH_LOWER_SRC_ADDR, ch_reg->src_addr);
+		tdma_ch_write(tdc, ADMA_CH_LOWER_TRG_ADDR, ch_reg->trg_addr);
+		tdma_ch_write(tdc, ADMA_CH_CTRL, ch_reg->ctrl);
+		tdma_ch_write(tdc, ADMA_CH_FIFO_CTRL, ch_reg->fifo_ctrl);
+		tdma_ch_write(tdc, ADMA_CH_CONFIG, ch_reg->config);
+		tdma_ch_write(tdc, ADMA_CH_CMD, ch_reg->cmd);
+	}
+
 	return 0;
 }
 
 static const struct tegra_adma_chip_data tegra210_chip_data = {
-	.nr_channels = 22,
+	.adma_get_burst_config  = tegra210_adma_get_burst_config,
+	.global_reg_offset	= 0xc00,
+	.global_int_clear	= 0x20,
+	.ch_req_tx_shift	= 28,
+	.ch_req_rx_shift	= 24,
+	.ch_base_offset		= 0,
+	.ch_req_mask		= 0xf,
+	.ch_req_max		= 10,
+	.ch_reg_size		= 0x80,
+	.nr_channels		= 22,
+};
+
+static const struct tegra_adma_chip_data tegra186_chip_data = {
+	.adma_get_burst_config  = tegra186_adma_get_burst_config,
+	.global_reg_offset	= 0,
+	.global_int_clear	= 0x402c,
+	.ch_req_tx_shift	= 27,
+	.ch_req_rx_shift	= 22,
+	.ch_base_offset		= 0x10000,
+	.ch_req_mask		= 0x1f,
+	.ch_req_max		= 20,
+	.ch_reg_size		= 0x100,
+	.nr_channels		= 32,
 };
 
 static const struct of_device_id tegra_adma_of_match[] = {
 	{ .compatible = "nvidia,tegra210-adma", .data = &tegra210_chip_data },
+	{ .compatible = "nvidia,tegra186-adma", .data = &tegra186_chip_data },
 	{ },
 };
 MODULE_DEVICE_TABLE(of, tegra_adma_of_match);
@@ -685,6 +830,7 @@ static int tegra_adma_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	tdma->dev = &pdev->dev;
+	tdma->cdata = cdata;
 	tdma->nr_channels = cdata->nr_channels;
 	platform_set_drvdata(pdev, tdma);
 
@@ -693,13 +839,11 @@ static int tegra_adma_probe(struct platform_device *pdev)
 	if (IS_ERR(tdma->base_addr))
 		return PTR_ERR(tdma->base_addr);
 
-	ret = pm_clk_create(&pdev->dev);
-	if (ret)
-		return ret;
-
-	ret = of_pm_clk_add_clk(&pdev->dev, "d_audio");
-	if (ret)
-		goto clk_destroy;
+	tdma->ahub_clk = devm_clk_get(&pdev->dev, "d_audio");
+	if (IS_ERR(tdma->ahub_clk)) {
+		dev_err(&pdev->dev, "Error: Missing ahub controller clock\n");
+		return PTR_ERR(tdma->ahub_clk);
+	}
 
 	pm_runtime_enable(&pdev->dev);
 
@@ -715,7 +859,8 @@ static int tegra_adma_probe(struct platform_device *pdev)
 	for (i = 0; i < tdma->nr_channels; i++) {
 		struct tegra_adma_chan *tdc = &tdma->channels[i];
 
-		tdc->chan_addr = tdma->base_addr + ADMA_CH_REG_OFFSET(i);
+		tdc->chan_addr = tdma->base_addr + cdata->ch_base_offset
+				 + (cdata->ch_reg_size * i);
 
 		tdc->irq = of_irq_get(pdev->dev.of_node, i);
 		if (tdc->irq <= 0) {
@@ -746,6 +891,8 @@ static int tegra_adma_probe(struct platform_device *pdev)
 	tdma->dma_dev.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
 	tdma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
 	tdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
+	tdma->dma_dev.device_pause = tegra_adma_pause;
+	tdma->dma_dev.device_resume = tegra_adma_resume;
 
 	ret = dma_async_device_register(&tdma->dma_dev);
 	if (ret < 0) {
@@ -776,8 +923,6 @@ rpm_put:
 	pm_runtime_put_sync(&pdev->dev);
 rpm_disable:
 	pm_runtime_disable(&pdev->dev);
-clk_destroy:
-	pm_clk_destroy(&pdev->dev);
 
 	return ret;
 }
@@ -787,6 +932,7 @@ static int tegra_adma_remove(struct platform_device *pdev)
 	struct tegra_adma *tdma = platform_get_drvdata(pdev);
 	int i;
 
+	of_dma_controller_free(pdev->dev.of_node);
 	dma_async_device_unregister(&tdma->dma_dev);
 
 	for (i = 0; i < tdma->nr_channels; ++i)
@@ -794,22 +940,15 @@ static int tegra_adma_remove(struct platform_device *pdev)
 
 	pm_runtime_put_sync(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
-	pm_clk_destroy(&pdev->dev);
 
 	return 0;
 }
 
-#ifdef CONFIG_PM_SLEEP
-static int tegra_adma_pm_suspend(struct device *dev)
-{
-	return pm_runtime_suspended(dev) == false;
-}
-#endif
-
 static const struct dev_pm_ops tegra_adma_dev_pm_ops = {
 	SET_RUNTIME_PM_OPS(tegra_adma_runtime_suspend,
 			   tegra_adma_runtime_resume, NULL)
-	SET_SYSTEM_SLEEP_PM_OPS(tegra_adma_pm_suspend, NULL)
+	SET_LATE_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				     pm_runtime_force_resume)
 };
 
 static struct platform_driver tegra_admac_driver = {
diff --git a/drivers/dma/xgene-dma.c b/drivers/dma/xgene-dma.c
index eafd6c4b90fe..8d174dc5dccd 100644
--- a/drivers/dma/xgene-dma.c
+++ b/drivers/dma/xgene-dma.c
@@ -703,7 +703,7 @@ static void xgene_dma_cleanup_descriptors(struct xgene_dma_chan *chan)
 
 	INIT_LIST_HEAD(&ld_completed);
 
-	spin_lock_bh(&chan->lock);
+	spin_lock(&chan->lock);
 
 	/* Clean already completed and acked descriptors */
 	xgene_dma_clean_completed_descriptor(chan);
@@ -772,7 +772,7 @@ static void xgene_dma_cleanup_descriptors(struct xgene_dma_chan *chan)
 	 */
 	xgene_chan_xfer_ld_pending(chan);
 
-	spin_unlock_bh(&chan->lock);
+	spin_unlock(&chan->lock);
 
 	/* Run the callback for each descriptor, in order */
 	list_for_each_entry_safe(desc_sw, _desc_sw, &ld_completed, node) {
@@ -797,7 +797,7 @@ static int xgene_dma_alloc_chan_resources(struct dma_chan *dchan)
 		return -ENOMEM;
 	}
 
-	chan_dbg(chan, "Allocate descripto pool\n");
+	chan_dbg(chan, "Allocate descriptor pool\n");
 
 	return 1;
 }
diff --git a/drivers/gpio/gpiolib-devprop.c b/drivers/gpio/gpiolib-devprop.c
index dd517098ab95..53781b253986 100644
--- a/drivers/gpio/gpiolib-devprop.c
+++ b/drivers/gpio/gpiolib-devprop.c
@@ -10,6 +10,7 @@
 #include <linux/slab.h>
 #include <linux/gpio/consumer.h>
 #include <linux/gpio/driver.h>
+#include <linux/export.h>
 
 #include "gpiolib.h"
 
@@ -56,3 +57,4 @@ void devprop_gpiochip_set_names(struct gpio_chip *chip,
 
 	kfree(names);
 }
+EXPORT_SYMBOL_GPL(devprop_gpiochip_set_names);
diff --git a/drivers/gpio/gpiolib.h b/drivers/gpio/gpiolib.h
index 078ab17b96bf..3243c1eb5c88 100644
--- a/drivers/gpio/gpiolib.h
+++ b/drivers/gpio/gpiolib.h
@@ -243,9 +243,6 @@ static inline int gpio_chip_hwgpio(const struct gpio_desc *desc)
 	return desc - &desc->gdev->descs[0];
 }
 
-void devprop_gpiochip_set_names(struct gpio_chip *chip,
-				const struct fwnode_handle *fwnode);
-
 /* With descriptor prefix */
 
 #define gpiod_emerg(desc, fmt, ...)					       \
diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig
index bd943a71756c..2267e84d5cb4 100644
--- a/drivers/gpu/drm/Kconfig
+++ b/drivers/gpu/drm/Kconfig
@@ -173,6 +173,12 @@ config DRM_KMS_CMA_HELPER
 	help
 	  Choose this if you need the KMS CMA helper functions
 
+config DRM_GEM_SHMEM_HELPER
+	bool
+	depends on DRM
+	help
+	  Choose this if you need the GEM shmem helper functions
+
 config DRM_VM
 	bool
 	depends on DRM && MMU
@@ -225,8 +231,6 @@ config DRM_AMDGPU
 
 source "drivers/gpu/drm/amd/amdgpu/Kconfig"
 
-source "drivers/gpu/drm/amd/lib/Kconfig"
-
 source "drivers/gpu/drm/nouveau/Kconfig"
 
 source "drivers/gpu/drm/i915/Kconfig"
@@ -251,6 +255,9 @@ config DRM_VKMS
 
 	  If M is selected the module will be called vkms.
 
+config DRM_ATI_PCIGART
+        bool
+
 source "drivers/gpu/drm/exynos/Kconfig"
 
 source "drivers/gpu/drm/rockchip/Kconfig"
@@ -329,12 +336,21 @@ source "drivers/gpu/drm/tve200/Kconfig"
 
 source "drivers/gpu/drm/xen/Kconfig"
 
+source "drivers/gpu/drm/vboxvideo/Kconfig"
+
+source "drivers/gpu/drm/lima/Kconfig"
+
+source "drivers/gpu/drm/panfrost/Kconfig"
+
+source "drivers/gpu/drm/aspeed/Kconfig"
+
 # Keep legacy drivers last
 
 menuconfig DRM_LEGACY
 	bool "Enable legacy drivers (DANGEROUS)"
 	depends on DRM && MMU
 	select DRM_VM
+	select DRM_ATI_PCIGART if PCI
 	help
 	  Enable legacy DRI1 drivers. Those drivers expose unsafe and dangerous
 	  APIs to user-space, which can be used to circumvent access
diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile
index 1ac55c65eac0..72f5036d9bfa 100644
--- a/drivers/gpu/drm/Makefile
+++ b/drivers/gpu/drm/Makefile
@@ -3,11 +3,9 @@
 # Makefile for the drm device driver.  This driver provides support for the
 # Direct Rendering Infrastructure (DRI) in XFree86 4.1.0 and higher.
 
-drm-y       :=	drm_auth.o drm_bufs.o drm_cache.o \
-		drm_context.o drm_dma.o \
+drm-y       :=	drm_auth.o drm_cache.o \
 		drm_file.o drm_gem.o drm_ioctl.o drm_irq.o \
-		drm_lock.o drm_memory.o drm_drv.o \
-		drm_scatter.o drm_pci.o \
+		drm_memory.o drm_drv.o drm_pci.o \
 		drm_sysfs.o drm_hashtab.o drm_mm.o \
 		drm_crtc.o drm_fourcc.o drm_modes.o drm_edid.o \
 		drm_encoder_slave.o \
@@ -21,11 +19,13 @@ drm-y       :=	drm_auth.o drm_bufs.o drm_cache.o \
 		drm_syncobj.o drm_lease.o drm_writeback.o drm_client.o \
 		drm_atomic_uapi.o
 
+drm-$(CONFIG_DRM_LEGACY) += drm_legacy_misc.o drm_bufs.o drm_context.o drm_dma.o drm_scatter.o drm_lock.o
 drm-$(CONFIG_DRM_LIB_RANDOM) += lib/drm_random.o
 drm-$(CONFIG_DRM_VM) += drm_vm.o
 drm-$(CONFIG_COMPAT) += drm_ioc32.o
 drm-$(CONFIG_DRM_GEM_CMA_HELPER) += drm_gem_cma_helper.o
-drm-$(CONFIG_PCI) += ati_pcigart.o
+drm-$(CONFIG_DRM_GEM_SHMEM_HELPER) += drm_gem_shmem_helper.o
+drm-$(CONFIG_DRM_ATI_PCIGART) += ati_pcigart.o
 drm-$(CONFIG_DRM_PANEL) += drm_panel.o
 drm-$(CONFIG_OF) += drm_of.o
 drm-$(CONFIG_AGP) += drm_agpsupport.o
@@ -37,7 +37,8 @@ drm_kms_helper-y := drm_crtc_helper.o drm_dp_helper.o drm_dsc.o drm_probe_helper
 		drm_kms_helper_common.o drm_dp_dual_mode_helper.o \
 		drm_simple_kms_helper.o drm_modeset_helper.o \
 		drm_scdc_helper.o drm_gem_framebuffer_helper.o \
-		drm_atomic_state_helper.o drm_damage_helper.o
+		drm_atomic_state_helper.o drm_damage_helper.o \
+		drm_format_helper.o
 
 drm_kms_helper-$(CONFIG_DRM_PANEL_BRIDGE) += bridge/panel.o
 drm_kms_helper-$(CONFIG_DRM_FBDEV_EMULATION) += drm_fb_helper.o
@@ -56,7 +57,6 @@ obj-$(CONFIG_DRM_TTM)	+= ttm/
 obj-$(CONFIG_DRM_SCHED)	+= scheduler/
 obj-$(CONFIG_DRM_TDFX)	+= tdfx/
 obj-$(CONFIG_DRM_R128)	+= r128/
-obj-y			+= amd/lib/
 obj-$(CONFIG_HSA_AMD) += amd/amdkfd/
 obj-$(CONFIG_DRM_RADEON)+= radeon/
 obj-$(CONFIG_DRM_AMDGPU)+= amd/amdgpu/
@@ -109,3 +109,7 @@ obj-$(CONFIG_DRM_TINYDRM) += tinydrm/
 obj-$(CONFIG_DRM_PL111) += pl111/
 obj-$(CONFIG_DRM_TVE200) += tve200/
 obj-$(CONFIG_DRM_XEN) += xen/
+obj-$(CONFIG_DRM_VBOXVIDEO) += vboxvideo/
+obj-$(CONFIG_DRM_LIMA)  += lima/
+obj-$(CONFIG_DRM_PANFROST) += panfrost/
+obj-$(CONFIG_DRM_ASPEED_GFX) += aspeed/
diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
index 466da5954a68..fdd0ca4b0f0b 100644
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -23,7 +23,7 @@
 # Makefile for the drm device driver.  This driver provides support for the
 # Direct Rendering Infrastructure (DRI) in XFree86 4.1.0 and higher.
 
-FULL_AMD_PATH=$(src)/..
+FULL_AMD_PATH=$(srctree)/$(src)/..
 DISPLAY_FOLDER_NAME=display
 FULL_AMD_DISPLAY_PATH = $(FULL_AMD_PATH)/$(DISPLAY_FOLDER_NAME)
 
@@ -53,7 +53,8 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
 	amdgpu_ucode.o amdgpu_bo_list.o amdgpu_ctx.o amdgpu_sync.o \
 	amdgpu_gtt_mgr.o amdgpu_vram_mgr.o amdgpu_virt.o amdgpu_atomfirmware.o \
 	amdgpu_vf_error.o amdgpu_sched.o amdgpu_debugfs.o amdgpu_ids.o \
-	amdgpu_gmc.o amdgpu_xgmi.o amdgpu_csa.o
+	amdgpu_gmc.o amdgpu_xgmi.o amdgpu_csa.o amdgpu_ras.o amdgpu_vm_cpu.o \
+	amdgpu_vm_sdma.o
 
 # add asic specific block
 amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o cik_ih.o kv_smc.o kv_dpm.o \
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 8d0d7f3dd5fb..14398f55f602 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -83,6 +83,7 @@
 #include "amdgpu_gem.h"
 #include "amdgpu_doorbell.h"
 #include "amdgpu_amdkfd.h"
+#include "amdgpu_smu.h"
 
 #define MAX_GPU_INSTANCE		16
 
@@ -156,6 +157,8 @@ extern int amdgpu_emu_mode;
 extern uint amdgpu_smu_memory_pool_size;
 extern uint amdgpu_dc_feature_mask;
 extern struct amdgpu_mgpu_info mgpu_info;
+extern int amdgpu_ras_enable;
+extern uint amdgpu_ras_mask;
 
 #ifdef CONFIG_DRM_AMDGPU_SI
 extern int amdgpu_si_support;
@@ -433,6 +436,12 @@ struct amdgpu_cs_chunk {
 	void			*kdata;
 };
 
+struct amdgpu_cs_post_dep {
+	struct drm_syncobj *syncobj;
+	struct dma_fence_chain *chain;
+	u64 point;
+};
+
 struct amdgpu_cs_parser {
 	struct amdgpu_device	*adev;
 	struct drm_file		*filp;
@@ -462,8 +471,8 @@ struct amdgpu_cs_parser {
 	/* user fence */
 	struct amdgpu_bo_list_entry	uf_entry;
 
-	unsigned num_post_dep_syncobjs;
-	struct drm_syncobj **post_dep_syncobjs;
+	unsigned			num_post_deps;
+	struct amdgpu_cs_post_dep	*post_deps;
 };
 
 static inline u32 amdgpu_get_ib_value(struct amdgpu_cs_parser *p,
@@ -702,7 +711,6 @@ enum amd_hw_ip_block_type {
 struct amd_powerplay {
 	void *pp_handle;
 	const struct amd_pm_funcs *pp_funcs;
-	uint32_t pp_feature;
 };
 
 #define AMDGPU_RESET_MAGIC_NUM 64
@@ -825,6 +833,7 @@ struct amdgpu_device {
 	/* For pre-DCE11. DCE11 and later are in "struct amdgpu_device->dm" */
 	struct work_struct		hotplug_work;
 	struct amdgpu_irq_src		crtc_irq;
+	struct amdgpu_irq_src		vupdate_irq;
 	struct amdgpu_irq_src		pageflip_irq;
 	struct amdgpu_irq_src		hpd_irq;
 
@@ -842,6 +851,9 @@ struct amdgpu_device {
 	struct amd_powerplay		powerplay;
 	bool				pp_force_state_enabled;
 
+	/* smu */
+	struct smu_context		smu;
+
 	/* dpm */
 	struct amdgpu_pm		pm;
 	u32				cg_flags;
@@ -922,6 +934,8 @@ struct amdgpu_device {
 
 	int asic_reset_res;
 	struct work_struct		xgmi_reset_work;
+
+	bool                            in_baco_reset;
 };
 
 static inline struct amdgpu_device *amdgpu_ttm_adev(struct ttm_bo_device *bdev)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
index fe1d7368c1e6..aeead072fa79 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
@@ -335,6 +335,43 @@ void amdgpu_amdkfd_free_gtt_mem(struct kgd_dev *kgd, void *mem_obj)
 	amdgpu_bo_unref(&(bo));
 }
 
+uint32_t amdgpu_amdkfd_get_fw_version(struct kgd_dev *kgd,
+				      enum kgd_engine_type type)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
+
+	switch (type) {
+	case KGD_ENGINE_PFP:
+		return adev->gfx.pfp_fw_version;
+
+	case KGD_ENGINE_ME:
+		return adev->gfx.me_fw_version;
+
+	case KGD_ENGINE_CE:
+		return adev->gfx.ce_fw_version;
+
+	case KGD_ENGINE_MEC1:
+		return adev->gfx.mec_fw_version;
+
+	case KGD_ENGINE_MEC2:
+		return adev->gfx.mec2_fw_version;
+
+	case KGD_ENGINE_RLC:
+		return adev->gfx.rlc_fw_version;
+
+	case KGD_ENGINE_SDMA1:
+		return adev->sdma.instance[0].fw_version;
+
+	case KGD_ENGINE_SDMA2:
+		return adev->sdma.instance[1].fw_version;
+
+	default:
+		return 0;
+	}
+
+	return 0;
+}
+
 void amdgpu_amdkfd_get_local_mem_info(struct kgd_dev *kgd,
 				      struct kfd_local_mem_info *mem_info)
 {
@@ -640,4 +677,8 @@ int kgd2kfd_post_reset(struct kfd_dev *kfd)
 void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry)
 {
 }
+
+void kgd2kfd_set_sram_ecc_flag(struct kfd_dev *kfd)
+{
+}
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index 0b31a1859023..4e37fa7e85b1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -81,6 +81,18 @@ struct amdgpu_kfd_dev {
 	uint64_t vram_used;
 };
 
+enum kgd_engine_type {
+	KGD_ENGINE_PFP = 1,
+	KGD_ENGINE_ME,
+	KGD_ENGINE_CE,
+	KGD_ENGINE_MEC1,
+	KGD_ENGINE_MEC2,
+	KGD_ENGINE_RLC,
+	KGD_ENGINE_SDMA1,
+	KGD_ENGINE_SDMA2,
+	KGD_ENGINE_MAX
+};
+
 struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,
 						       struct mm_struct *mm);
 bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm);
@@ -142,6 +154,8 @@ int amdgpu_amdkfd_alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
 				void **mem_obj, uint64_t *gpu_addr,
 				void **cpu_ptr, bool mqd_gfx9);
 void amdgpu_amdkfd_free_gtt_mem(struct kgd_dev *kgd, void *mem_obj);
+uint32_t amdgpu_amdkfd_get_fw_version(struct kgd_dev *kgd,
+				      enum kgd_engine_type type);
 void amdgpu_amdkfd_get_local_mem_info(struct kgd_dev *kgd,
 				      struct kfd_local_mem_info *mem_info);
 uint64_t amdgpu_amdkfd_get_gpu_clock_counter(struct kgd_dev *kgd);
@@ -230,5 +244,6 @@ int kgd2kfd_quiesce_mm(struct mm_struct *mm);
 int kgd2kfd_resume_mm(struct mm_struct *mm);
 int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm,
 					       struct dma_fence *fence);
+void kgd2kfd_set_sram_ecc_flag(struct kfd_dev *kfd);
 
 #endif /* AMDGPU_AMDKFD_H_INCLUDED */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
index ff7fac7df34b..fa09e11a600c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
@@ -22,14 +22,12 @@
 
 #include <linux/fdtable.h>
 #include <linux/uaccess.h>
-#include <linux/firmware.h>
 #include <linux/mmu_context.h>
 #include <drm/drmP.h>
 #include "amdgpu.h"
 #include "amdgpu_amdkfd.h"
 #include "cikd.h"
 #include "cik_sdma.h"
-#include "amdgpu_ucode.h"
 #include "gfx_v7_0.h"
 #include "gca/gfx_7_2_d.h"
 #include "gca/gfx_7_2_enum.h"
@@ -139,7 +137,6 @@ static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd, uint8_t vmid);
 static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd,
 							uint8_t vmid);
 
-static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type);
 static void set_scratch_backing_va(struct kgd_dev *kgd,
 					uint64_t va, uint32_t vmid);
 static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid,
@@ -191,7 +188,6 @@ static const struct kfd2kgd_calls kfd2kgd = {
 	.address_watch_get_offset = kgd_address_watch_get_offset,
 	.get_atc_vmid_pasid_mapping_pasid = get_atc_vmid_pasid_mapping_pasid,
 	.get_atc_vmid_pasid_mapping_valid = get_atc_vmid_pasid_mapping_valid,
-	.get_fw_version = get_fw_version,
 	.set_scratch_backing_va = set_scratch_backing_va,
 	.get_tile_config = get_tile_config,
 	.set_vm_context_page_table_base = set_vm_context_page_table_base,
@@ -792,63 +788,6 @@ static void set_scratch_backing_va(struct kgd_dev *kgd,
 	unlock_srbm(kgd);
 }
 
-static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type)
-{
-	struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
-	const union amdgpu_firmware_header *hdr;
-
-	switch (type) {
-	case KGD_ENGINE_PFP:
-		hdr = (const union amdgpu_firmware_header *)
-						adev->gfx.pfp_fw->data;
-		break;
-
-	case KGD_ENGINE_ME:
-		hdr = (const union amdgpu_firmware_header *)
-						adev->gfx.me_fw->data;
-		break;
-
-	case KGD_ENGINE_CE:
-		hdr = (const union amdgpu_firmware_header *)
-						adev->gfx.ce_fw->data;
-		break;
-
-	case KGD_ENGINE_MEC1:
-		hdr = (const union amdgpu_firmware_header *)
-						adev->gfx.mec_fw->data;
-		break;
-
-	case KGD_ENGINE_MEC2:
-		hdr = (const union amdgpu_firmware_header *)
-						adev->gfx.mec2_fw->data;
-		break;
-
-	case KGD_ENGINE_RLC:
-		hdr = (const union amdgpu_firmware_header *)
-						adev->gfx.rlc_fw->data;
-		break;
-
-	case KGD_ENGINE_SDMA1:
-		hdr = (const union amdgpu_firmware_header *)
-						adev->sdma.instance[0].fw->data;
-		break;
-
-	case KGD_ENGINE_SDMA2:
-		hdr = (const union amdgpu_firmware_header *)
-						adev->sdma.instance[1].fw->data;
-		break;
-
-	default:
-		return 0;
-	}
-
-	if (hdr == NULL)
-		return 0;
-
-	/* Only 12 bit in use*/
-	return hdr->common.ucode_version;
-}
-
 static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid,
 			uint64_t page_table_base)
 {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
index 56ea929f524b..fec3a6aa1de6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
@@ -23,12 +23,10 @@
 #include <linux/module.h>
 #include <linux/fdtable.h>
 #include <linux/uaccess.h>
-#include <linux/firmware.h>
 #include <linux/mmu_context.h>
 #include <drm/drmP.h>
 #include "amdgpu.h"
 #include "amdgpu_amdkfd.h"
-#include "amdgpu_ucode.h"
 #include "gfx_v8_0.h"
 #include "gca/gfx_8_0_sh_mask.h"
 #include "gca/gfx_8_0_d.h"
@@ -95,7 +93,6 @@ static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd,
 		uint8_t vmid);
 static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd,
 		uint8_t vmid);
-static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type);
 static void set_scratch_backing_va(struct kgd_dev *kgd,
 					uint64_t va, uint32_t vmid);
 static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid,
@@ -148,7 +145,6 @@ static const struct kfd2kgd_calls kfd2kgd = {
 			get_atc_vmid_pasid_mapping_pasid,
 	.get_atc_vmid_pasid_mapping_valid =
 			get_atc_vmid_pasid_mapping_valid,
-	.get_fw_version = get_fw_version,
 	.set_scratch_backing_va = set_scratch_backing_va,
 	.get_tile_config = get_tile_config,
 	.set_vm_context_page_table_base = set_vm_context_page_table_base,
@@ -751,63 +747,6 @@ static void set_scratch_backing_va(struct kgd_dev *kgd,
 	unlock_srbm(kgd);
 }
 
-static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type)
-{
-	struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
-	const union amdgpu_firmware_header *hdr;
-
-	switch (type) {
-	case KGD_ENGINE_PFP:
-		hdr = (const union amdgpu_firmware_header *)
-						adev->gfx.pfp_fw->data;
-		break;
-
-	case KGD_ENGINE_ME:
-		hdr = (const union amdgpu_firmware_header *)
-						adev->gfx.me_fw->data;
-		break;
-
-	case KGD_ENGINE_CE:
-		hdr = (const union amdgpu_firmware_header *)
-						adev->gfx.ce_fw->data;
-		break;
-
-	case KGD_ENGINE_MEC1:
-		hdr = (const union amdgpu_firmware_header *)
-						adev->gfx.mec_fw->data;
-		break;
-
-	case KGD_ENGINE_MEC2:
-		hdr = (const union amdgpu_firmware_header *)
-						adev->gfx.mec2_fw->data;
-		break;
-
-	case KGD_ENGINE_RLC:
-		hdr = (const union amdgpu_firmware_header *)
-						adev->gfx.rlc_fw->data;
-		break;
-
-	case KGD_ENGINE_SDMA1:
-		hdr = (const union amdgpu_firmware_header *)
-						adev->sdma.instance[0].fw->data;
-		break;
-
-	case KGD_ENGINE_SDMA2:
-		hdr = (const union amdgpu_firmware_header *)
-						adev->sdma.instance[1].fw->data;
-		break;
-
-	default:
-		return 0;
-	}
-
-	if (hdr == NULL)
-		return 0;
-
-	/* Only 12 bit in use*/
-	return hdr->common.ucode_version;
-}
-
 static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid,
 		uint64_t page_table_base)
 {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
index 5c51d4910650..ef3d93b995b2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c
@@ -25,12 +25,10 @@
 #include <linux/module.h>
 #include <linux/fdtable.h>
 #include <linux/uaccess.h>
-#include <linux/firmware.h>
 #include <linux/mmu_context.h>
 #include <drm/drmP.h>
 #include "amdgpu.h"
 #include "amdgpu_amdkfd.h"
-#include "amdgpu_ucode.h"
 #include "soc15_hw_ip.h"
 #include "gc/gc_9_0_offset.h"
 #include "gc/gc_9_0_sh_mask.h"
@@ -111,7 +109,6 @@ static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd,
 		uint8_t vmid);
 static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid,
 		uint64_t page_table_base);
-static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type);
 static void set_scratch_backing_va(struct kgd_dev *kgd,
 					uint64_t va, uint32_t vmid);
 static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid);
@@ -158,7 +155,6 @@ static const struct kfd2kgd_calls kfd2kgd = {
 			get_atc_vmid_pasid_mapping_pasid,
 	.get_atc_vmid_pasid_mapping_valid =
 			get_atc_vmid_pasid_mapping_valid,
-	.get_fw_version = get_fw_version,
 	.set_scratch_backing_va = set_scratch_backing_va,
 	.get_tile_config = amdgpu_amdkfd_get_tile_config,
 	.set_vm_context_page_table_base = set_vm_context_page_table_base,
@@ -874,56 +870,6 @@ static void set_scratch_backing_va(struct kgd_dev *kgd,
 	 */
 }
 
-/* FIXME: Does this need to be ASIC-specific code? */
-static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type)
-{
-	struct amdgpu_device *adev = (struct amdgpu_device *) kgd;
-	const union amdgpu_firmware_header *hdr;
-
-	switch (type) {
-	case KGD_ENGINE_PFP:
-		hdr = (const union amdgpu_firmware_header *)adev->gfx.pfp_fw->data;
-		break;
-
-	case KGD_ENGINE_ME:
-		hdr = (const union amdgpu_firmware_header *)adev->gfx.me_fw->data;
-		break;
-
-	case KGD_ENGINE_CE:
-		hdr = (const union amdgpu_firmware_header *)adev->gfx.ce_fw->data;
-		break;
-
-	case KGD_ENGINE_MEC1:
-		hdr = (const union amdgpu_firmware_header *)adev->gfx.mec_fw->data;
-		break;
-
-	case KGD_ENGINE_MEC2:
-		hdr = (const union amdgpu_firmware_header *)adev->gfx.mec2_fw->data;
-		break;
-
-	case KGD_ENGINE_RLC:
-		hdr = (const union amdgpu_firmware_header *)adev->gfx.rlc_fw->data;
-		break;
-
-	case KGD_ENGINE_SDMA1:
-		hdr = (const union amdgpu_firmware_header *)adev->sdma.instance[0].fw->data;
-		break;
-
-	case KGD_ENGINE_SDMA2:
-		hdr = (const union amdgpu_firmware_header *)adev->sdma.instance[1].fw->data;
-		break;
-
-	default:
-		return 0;
-	}
-
-	if (hdr == NULL)
-		return 0;
-
-	/* Only 12 bit in use*/
-	return hdr->common.ucode_version;
-}
-
 static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid,
 		uint64_t page_table_base)
 {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 1921dec3df7a..a6e5184d436c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -410,15 +410,7 @@ static int add_bo_to_vm(struct amdgpu_device *adev, struct kgd_mem *mem,
 	if (p_bo_va_entry)
 		*p_bo_va_entry = bo_va_entry;
 
-	/* Allocate new page tables if needed and validate
-	 * them.
-	 */
-	ret = amdgpu_vm_alloc_pts(adev, vm, va, amdgpu_bo_size(bo));
-	if (ret) {
-		pr_err("Failed to allocate pts, err=%d\n", ret);
-		goto err_alloc_pts;
-	}
-
+	/* Allocate validate page tables if needed */
 	ret = vm_validate_pt_pd_bos(vm);
 	if (ret) {
 		pr_err("validate_pt_pd_bos() failed\n");
@@ -741,13 +733,7 @@ static int update_gpuvm_pte(struct amdgpu_device *adev,
 		struct amdgpu_sync *sync)
 {
 	int ret;
-	struct amdgpu_vm *vm;
-	struct amdgpu_bo_va *bo_va;
-	struct amdgpu_bo *bo;
-
-	bo_va = entry->bo_va;
-	vm = bo_va->base.vm;
-	bo = bo_va->base.bo;
+	struct amdgpu_bo_va *bo_va = entry->bo_va;
 
 	/* Update the page tables  */
 	ret = amdgpu_vm_bo_update(adev, bo_va, false);
@@ -906,7 +892,8 @@ static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info,
 		pr_err("validate_pt_pd_bos() failed\n");
 		goto validate_pd_fail;
 	}
-	amdgpu_bo_sync_wait(vm->root.base.bo, AMDGPU_FENCE_OWNER_KFD, false);
+	ret = amdgpu_bo_sync_wait(vm->root.base.bo,
+				  AMDGPU_FENCE_OWNER_KFD, false);
 	if (ret)
 		goto wait_pd_fail;
 	amdgpu_bo_fence(vm->root.base.bo,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
index b61e1dc61b4c..f96d75c6e099 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c
@@ -28,8 +28,6 @@
 #include "atom.h"
 #include "atombios.h"
 
-#define get_index_into_master_table(master_table, table_name) (offsetof(struct master_table, table_name) / sizeof(uint16_t))
-
 bool amdgpu_atomfirmware_gpu_supports_virtualization(struct amdgpu_device *adev)
 {
 	int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
@@ -238,10 +236,71 @@ int amdgpu_atomfirmware_get_vram_type(struct amdgpu_device *adev)
 	return 0;
 }
 
+/*
+ * Return true if vbios enabled ecc by default, if umc info table is available
+ * or false if ecc is not enabled or umc info table is not available
+ */
+bool amdgpu_atomfirmware_mem_ecc_supported(struct amdgpu_device *adev)
+{
+	struct amdgpu_mode_info *mode_info = &adev->mode_info;
+	int index;
+	u16 data_offset, size;
+	union umc_info *umc_info;
+	u8 frev, crev;
+	bool ecc_default_enabled = false;
+
+	index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
+			umc_info);
+
+	if (amdgpu_atom_parse_data_header(mode_info->atom_context,
+				index, &size, &frev, &crev, &data_offset)) {
+		/* support umc_info 3.1+ */
+		if ((frev == 3 && crev >= 1) || (frev > 3)) {
+			umc_info = (union umc_info *)
+				(mode_info->atom_context->bios + data_offset);
+			ecc_default_enabled =
+				(le32_to_cpu(umc_info->v31.umc_config) &
+				 UMC_CONFIG__DEFAULT_MEM_ECC_ENABLE) ? true : false;
+		}
+	}
+
+	return ecc_default_enabled;
+}
+
 union firmware_info {
 	struct atom_firmware_info_v3_1 v31;
 };
 
+/*
+ * Return true if vbios supports sram ecc or false if not
+ */
+bool amdgpu_atomfirmware_sram_ecc_supported(struct amdgpu_device *adev)
+{
+	struct amdgpu_mode_info *mode_info = &adev->mode_info;
+	int index;
+	u16 data_offset, size;
+	union firmware_info *firmware_info;
+	u8 frev, crev;
+	bool sram_ecc_supported = false;
+
+	index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
+			firmwareinfo);
+
+	if (amdgpu_atom_parse_data_header(adev->mode_info.atom_context,
+				index, &size, &frev, &crev, &data_offset)) {
+		/* support firmware_info 3.1 + */
+		if ((frev == 3 && crev >=1) || (frev > 3)) {
+			firmware_info = (union firmware_info *)
+				(mode_info->atom_context->bios + data_offset);
+			sram_ecc_supported =
+				(le32_to_cpu(firmware_info->v31.firmware_capability) &
+				 ATOM_FIRMWARE_CAP_SRAM_ECC) ? true : false;
+		}
+	}
+
+	return sram_ecc_supported;
+}
+
 union smu_info {
 	struct atom_smu_info_v3_1 v31;
 };
@@ -346,11 +405,11 @@ int amdgpu_atomfirmware_get_gfx_info(struct amdgpu_device *adev)
 			(mode_info->atom_context->bios + data_offset);
 		switch (crev) {
 		case 4:
-			adev->gfx.config.max_shader_engines = gfx_info->v24.gc_num_se;
-			adev->gfx.config.max_cu_per_sh = gfx_info->v24.gc_num_cu_per_sh;
-			adev->gfx.config.max_sh_per_se = gfx_info->v24.gc_num_sh_per_se;
-			adev->gfx.config.max_backends_per_se = gfx_info->v24.gc_num_rb_per_se;
-			adev->gfx.config.max_texture_channel_caches = gfx_info->v24.gc_num_tccs;
+			adev->gfx.config.max_shader_engines = gfx_info->v24.max_shader_engines;
+			adev->gfx.config.max_cu_per_sh = gfx_info->v24.max_cu_per_sh;
+			adev->gfx.config.max_sh_per_se = gfx_info->v24.max_sh_per_se;
+			adev->gfx.config.max_backends_per_se = gfx_info->v24.max_backends_per_se;
+			adev->gfx.config.max_texture_channel_caches = gfx_info->v24.max_texture_channel_caches;
 			adev->gfx.config.max_gprs = le16_to_cpu(gfx_info->v24.gc_num_gprs);
 			adev->gfx.config.max_gs_threads = gfx_info->v24.gc_num_max_gs_thds;
 			adev->gfx.config.gs_vgt_table_depth = gfx_info->v24.gc_gs_table_depth;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.h
index 20f158fd3b76..5ec6f92f353c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.h
@@ -24,6 +24,8 @@
 #ifndef __AMDGPU_ATOMFIRMWARE_H__
 #define __AMDGPU_ATOMFIRMWARE_H__
 
+#define get_index_into_master_table(master_table, table_name) (offsetof(struct master_table, table_name) / sizeof(uint16_t))
+
 bool amdgpu_atomfirmware_gpu_supports_virtualization(struct amdgpu_device *adev);
 void amdgpu_atomfirmware_scratch_regs_init(struct amdgpu_device *adev);
 int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev);
@@ -31,5 +33,7 @@ int amdgpu_atomfirmware_get_vram_width(struct amdgpu_device *adev);
 int amdgpu_atomfirmware_get_vram_type(struct amdgpu_device *adev);
 int amdgpu_atomfirmware_get_clock_info(struct amdgpu_device *adev);
 int amdgpu_atomfirmware_get_gfx_info(struct amdgpu_device *adev);
+bool amdgpu_atomfirmware_mem_ecc_supported(struct amdgpu_device *adev);
+bool amdgpu_atomfirmware_sram_ecc_supported(struct amdgpu_device *adev);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index 52a5e4fdc95b..2f6239b6be6f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -215,6 +215,8 @@ static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, union drm_amdgpu_cs
 		case AMDGPU_CHUNK_ID_SYNCOBJ_IN:
 		case AMDGPU_CHUNK_ID_SYNCOBJ_OUT:
 		case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES:
+		case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT:
+		case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL:
 			break;
 
 		default:
@@ -804,9 +806,11 @@ static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error,
 		ttm_eu_backoff_reservation(&parser->ticket,
 					   &parser->validated);
 
-	for (i = 0; i < parser->num_post_dep_syncobjs; i++)
-		drm_syncobj_put(parser->post_dep_syncobjs[i]);
-	kfree(parser->post_dep_syncobjs);
+	for (i = 0; i < parser->num_post_deps; i++) {
+		drm_syncobj_put(parser->post_deps[i].syncobj);
+		kfree(parser->post_deps[i].chain);
+	}
+	kfree(parser->post_deps);
 
 	dma_fence_put(parser->fence);
 
@@ -1117,13 +1121,18 @@ static int amdgpu_cs_process_fence_dep(struct amdgpu_cs_parser *p,
 }
 
 static int amdgpu_syncobj_lookup_and_add_to_sync(struct amdgpu_cs_parser *p,
-						 uint32_t handle)
+						 uint32_t handle, u64 point,
+						 u64 flags)
 {
-	int r;
 	struct dma_fence *fence;
-	r = drm_syncobj_find_fence(p->filp, handle, 0, 0, &fence);
-	if (r)
+	int r;
+
+	r = drm_syncobj_find_fence(p->filp, handle, point, flags, &fence);
+	if (r) {
+		DRM_ERROR("syncobj %u failed to find fence @ %llu (%d)!\n",
+			  handle, point, r);
 		return r;
+	}
 
 	r = amdgpu_sync_fence(p->adev, &p->job->sync, fence, true);
 	dma_fence_put(fence);
@@ -1134,46 +1143,118 @@ static int amdgpu_syncobj_lookup_and_add_to_sync(struct amdgpu_cs_parser *p,
 static int amdgpu_cs_process_syncobj_in_dep(struct amdgpu_cs_parser *p,
 					    struct amdgpu_cs_chunk *chunk)
 {
+	struct drm_amdgpu_cs_chunk_sem *deps;
 	unsigned num_deps;
 	int i, r;
-	struct drm_amdgpu_cs_chunk_sem *deps;
 
 	deps = (struct drm_amdgpu_cs_chunk_sem *)chunk->kdata;
 	num_deps = chunk->length_dw * 4 /
 		sizeof(struct drm_amdgpu_cs_chunk_sem);
+	for (i = 0; i < num_deps; ++i) {
+		r = amdgpu_syncobj_lookup_and_add_to_sync(p, deps[i].handle,
+							  0, 0);
+		if (r)
+			return r;
+	}
+
+	return 0;
+}
+
 
+static int amdgpu_cs_process_syncobj_timeline_in_dep(struct amdgpu_cs_parser *p,
+						     struct amdgpu_cs_chunk *chunk)
+{
+	struct drm_amdgpu_cs_chunk_syncobj *syncobj_deps;
+	unsigned num_deps;
+	int i, r;
+
+	syncobj_deps = (struct drm_amdgpu_cs_chunk_syncobj *)chunk->kdata;
+	num_deps = chunk->length_dw * 4 /
+		sizeof(struct drm_amdgpu_cs_chunk_syncobj);
 	for (i = 0; i < num_deps; ++i) {
-		r = amdgpu_syncobj_lookup_and_add_to_sync(p, deps[i].handle);
+		r = amdgpu_syncobj_lookup_and_add_to_sync(p,
+							  syncobj_deps[i].handle,
+							  syncobj_deps[i].point,
+							  syncobj_deps[i].flags);
 		if (r)
 			return r;
 	}
+
 	return 0;
 }
 
 static int amdgpu_cs_process_syncobj_out_dep(struct amdgpu_cs_parser *p,
 					     struct amdgpu_cs_chunk *chunk)
 {
+	struct drm_amdgpu_cs_chunk_sem *deps;
 	unsigned num_deps;
 	int i;
-	struct drm_amdgpu_cs_chunk_sem *deps;
+
 	deps = (struct drm_amdgpu_cs_chunk_sem *)chunk->kdata;
 	num_deps = chunk->length_dw * 4 /
 		sizeof(struct drm_amdgpu_cs_chunk_sem);
 
-	p->post_dep_syncobjs = kmalloc_array(num_deps,
-					     sizeof(struct drm_syncobj *),
-					     GFP_KERNEL);
-	p->num_post_dep_syncobjs = 0;
+	p->post_deps = kmalloc_array(num_deps, sizeof(*p->post_deps),
+				     GFP_KERNEL);
+	p->num_post_deps = 0;
 
-	if (!p->post_dep_syncobjs)
+	if (!p->post_deps)
 		return -ENOMEM;
 
+
 	for (i = 0; i < num_deps; ++i) {
-		p->post_dep_syncobjs[i] = drm_syncobj_find(p->filp, deps[i].handle);
-		if (!p->post_dep_syncobjs[i])
+		p->post_deps[i].syncobj =
+			drm_syncobj_find(p->filp, deps[i].handle);
+		if (!p->post_deps[i].syncobj)
 			return -EINVAL;
-		p->num_post_dep_syncobjs++;
+		p->post_deps[i].chain = NULL;
+		p->post_deps[i].point = 0;
+		p->num_post_deps++;
 	}
+
+	return 0;
+}
+
+
+static int amdgpu_cs_process_syncobj_timeline_out_dep(struct amdgpu_cs_parser *p,
+						      struct amdgpu_cs_chunk
+						      *chunk)
+{
+	struct drm_amdgpu_cs_chunk_syncobj *syncobj_deps;
+	unsigned num_deps;
+	int i;
+
+	syncobj_deps = (struct drm_amdgpu_cs_chunk_syncobj *)chunk->kdata;
+	num_deps = chunk->length_dw * 4 /
+		sizeof(struct drm_amdgpu_cs_chunk_syncobj);
+
+	p->post_deps = kmalloc_array(num_deps, sizeof(*p->post_deps),
+				     GFP_KERNEL);
+	p->num_post_deps = 0;
+
+	if (!p->post_deps)
+		return -ENOMEM;
+
+	for (i = 0; i < num_deps; ++i) {
+		struct amdgpu_cs_post_dep *dep = &p->post_deps[i];
+
+		dep->chain = NULL;
+		if (syncobj_deps[i].point) {
+			dep->chain = kmalloc(sizeof(*dep->chain), GFP_KERNEL);
+			if (!dep->chain)
+				return -ENOMEM;
+		}
+
+		dep->syncobj = drm_syncobj_find(p->filp,
+						syncobj_deps[i].handle);
+		if (!dep->syncobj) {
+			kfree(dep->chain);
+			return -EINVAL;
+		}
+		dep->point = syncobj_deps[i].point;
+		p->num_post_deps++;
+	}
+
 	return 0;
 }
 
@@ -1187,19 +1268,33 @@ static int amdgpu_cs_dependencies(struct amdgpu_device *adev,
 
 		chunk = &p->chunks[i];
 
-		if (chunk->chunk_id == AMDGPU_CHUNK_ID_DEPENDENCIES ||
-		    chunk->chunk_id == AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES) {
+		switch (chunk->chunk_id) {
+		case AMDGPU_CHUNK_ID_DEPENDENCIES:
+		case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES:
 			r = amdgpu_cs_process_fence_dep(p, chunk);
 			if (r)
 				return r;
-		} else if (chunk->chunk_id == AMDGPU_CHUNK_ID_SYNCOBJ_IN) {
+			break;
+		case AMDGPU_CHUNK_ID_SYNCOBJ_IN:
 			r = amdgpu_cs_process_syncobj_in_dep(p, chunk);
 			if (r)
 				return r;
-		} else if (chunk->chunk_id == AMDGPU_CHUNK_ID_SYNCOBJ_OUT) {
+			break;
+		case AMDGPU_CHUNK_ID_SYNCOBJ_OUT:
 			r = amdgpu_cs_process_syncobj_out_dep(p, chunk);
 			if (r)
 				return r;
+			break;
+		case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT:
+			r = amdgpu_cs_process_syncobj_timeline_in_dep(p, chunk);
+			if (r)
+				return r;
+			break;
+		case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL:
+			r = amdgpu_cs_process_syncobj_timeline_out_dep(p, chunk);
+			if (r)
+				return r;
+			break;
 		}
 	}
 
@@ -1210,8 +1305,17 @@ static void amdgpu_cs_post_dependencies(struct amdgpu_cs_parser *p)
 {
 	int i;
 
-	for (i = 0; i < p->num_post_dep_syncobjs; ++i)
-		drm_syncobj_replace_fence(p->post_dep_syncobjs[i], p->fence);
+	for (i = 0; i < p->num_post_deps; ++i) {
+		if (p->post_deps[i].chain && p->post_deps[i].point) {
+			drm_syncobj_add_point(p->post_deps[i].syncobj,
+					      p->post_deps[i].chain,
+					      p->fence, p->post_deps[i].point);
+			p->post_deps[i].chain = NULL;
+		} else {
+			drm_syncobj_replace_fence(p->post_deps[i].syncobj,
+						  p->fence);
+		}
+	}
 }
 
 static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c
index 7e22be7ca68a..54dd02a898b9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c
@@ -92,15 +92,6 @@ int amdgpu_map_static_csa(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 		return -ENOMEM;
 	}
 
-	r = amdgpu_vm_alloc_pts(adev, (*bo_va)->base.vm, csa_addr,
-				size);
-	if (r) {
-		DRM_ERROR("failed to allocate pts for static CSA, err=%d\n", r);
-		amdgpu_vm_bo_rmv(adev, *bo_va);
-		ttm_eu_backoff_reservation(&ticket, &list);
-		return r;
-	}
-
 	r = amdgpu_vm_bo_map(adev, *bo_va, csa_addr, 0, size,
 			     AMDGPU_PTE_READABLE | AMDGPU_PTE_WRITEABLE |
 			     AMDGPU_PTE_EXECUTABLE);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
index 7b526593eb77..a28a3d722ba2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
@@ -26,6 +26,7 @@
 #include <drm/drm_auth.h>
 #include "amdgpu.h"
 #include "amdgpu_sched.h"
+#include "amdgpu_ras.h"
 
 #define to_amdgpu_ctx_entity(e)	\
 	container_of((e), struct amdgpu_ctx_entity, entity)
@@ -344,6 +345,7 @@ static int amdgpu_ctx_query2(struct amdgpu_device *adev,
 {
 	struct amdgpu_ctx *ctx;
 	struct amdgpu_ctx_mgr *mgr;
+	uint32_t ras_counter;
 
 	if (!fpriv)
 		return -EINVAL;
@@ -368,6 +370,21 @@ static int amdgpu_ctx_query2(struct amdgpu_device *adev,
 	if (atomic_read(&ctx->guilty))
 		out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_GUILTY;
 
+	/*query ue count*/
+	ras_counter = amdgpu_ras_query_error_count(adev, false);
+	/*ras counter is monotonic increasing*/
+	if (ras_counter != ctx->ras_counter_ue) {
+		out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RAS_UE;
+		ctx->ras_counter_ue = ras_counter;
+	}
+
+	/*query ce count*/
+	ras_counter = amdgpu_ras_query_error_count(adev, true);
+	if (ras_counter != ctx->ras_counter_ce) {
+		out->state.flags |= AMDGPU_CTX_QUERY2_FLAGS_RAS_CE;
+		ctx->ras_counter_ce = ras_counter;
+	}
+
 	mutex_unlock(&mgr->lock);
 	return 0;
 }
@@ -541,32 +558,26 @@ void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr)
 	idr_init(&mgr->ctx_handles);
 }
 
-void amdgpu_ctx_mgr_entity_flush(struct amdgpu_ctx_mgr *mgr)
+long amdgpu_ctx_mgr_entity_flush(struct amdgpu_ctx_mgr *mgr, long timeout)
 {
 	unsigned num_entities = amdgput_ctx_total_num_entities();
 	struct amdgpu_ctx *ctx;
 	struct idr *idp;
 	uint32_t id, i;
-	long max_wait = MAX_WAIT_SCHED_ENTITY_Q_EMPTY;
 
 	idp = &mgr->ctx_handles;
 
 	mutex_lock(&mgr->lock);
 	idr_for_each_entry(idp, ctx, id) {
-
-		if (!ctx->adev) {
-			mutex_unlock(&mgr->lock);
-			return;
-		}
-
 		for (i = 0; i < num_entities; i++) {
 			struct drm_sched_entity *entity;
 
 			entity = &ctx->entities[0][i].entity;
-			max_wait = drm_sched_entity_flush(entity, max_wait);
+			timeout = drm_sched_entity_flush(entity, timeout);
 		}
 	}
 	mutex_unlock(&mgr->lock);
+	return timeout;
 }
 
 void amdgpu_ctx_mgr_entity_fini(struct amdgpu_ctx_mgr *mgr)
@@ -579,10 +590,6 @@ void amdgpu_ctx_mgr_entity_fini(struct amdgpu_ctx_mgr *mgr)
 	idp = &mgr->ctx_handles;
 
 	idr_for_each_entry(idp, ctx, id) {
-
-		if (!ctx->adev)
-			return;
-
 		if (kref_read(&ctx->refcount) != 1) {
 			DRM_ERROR("ctx %p is still alive\n", ctx);
 			continue;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
index b3b012c0a7da..5f1b54c9bcdb 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
@@ -49,6 +49,8 @@ struct amdgpu_ctx {
 	enum drm_sched_priority		override_priority;
 	struct mutex			lock;
 	atomic_t			guilty;
+	uint32_t			ras_counter_ce;
+	uint32_t			ras_counter_ue;
 };
 
 struct amdgpu_ctx_mgr {
@@ -82,7 +84,7 @@ int amdgpu_ctx_wait_prev_fence(struct amdgpu_ctx *ctx,
 
 void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr);
 void amdgpu_ctx_mgr_entity_fini(struct amdgpu_ctx_mgr *mgr);
-void amdgpu_ctx_mgr_entity_flush(struct amdgpu_ctx_mgr *mgr);
+long amdgpu_ctx_mgr_entity_flush(struct amdgpu_ctx_mgr *mgr, long timeout);
 void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
index 4ae3ff9a1d4c..8930d66f2204 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
@@ -568,10 +568,9 @@ static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
 	idx = *pos >> 2;
 
 	valuesize = sizeof(values);
-	if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->read_sensor)
-		r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize);
-	else
-		return -EINVAL;
+	r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize);
+	if (r)
+		return r;
 
 	if (size > valuesize)
 		return -EINVAL;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 79fb302fb954..cc8ad3831982 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -60,6 +60,7 @@
 #include "amdgpu_pm.h"
 
 #include "amdgpu_xgmi.h"
+#include "amdgpu_ras.h"
 
 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
 MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
@@ -1506,7 +1507,9 @@ static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
 			return -EAGAIN;
 	}
 
-	adev->powerplay.pp_feature = amdgpu_pp_feature_mask;
+	adev->pm.pp_feature = amdgpu_pp_feature_mask;
+	if (amdgpu_sriov_vf(adev))
+		adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
 
 	for (i = 0; i < adev->num_ip_blocks; i++) {
 		if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
@@ -1638,6 +1641,10 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev)
 {
 	int i, r;
 
+	r = amdgpu_ras_init(adev);
+	if (r)
+		return r;
+
 	for (i = 0; i < adev->num_ip_blocks; i++) {
 		if (!adev->ip_blocks[i].status.valid)
 			continue;
@@ -1681,6 +1688,13 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev)
 		}
 	}
 
+	r = amdgpu_ib_pool_init(adev);
+	if (r) {
+		dev_err(adev->dev, "IB initialization failed (%d).\n", r);
+		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
+		goto init_failed;
+	}
+
 	r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/
 	if (r)
 		goto init_failed;
@@ -1869,6 +1883,8 @@ static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
 {
 	int i, r;
 
+	amdgpu_ras_pre_fini(adev);
+
 	if (adev->gmc.xgmi.num_physical_nodes > 1)
 		amdgpu_xgmi_remove_device(adev);
 
@@ -1917,6 +1933,7 @@ static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
 			amdgpu_free_static_csa(&adev->virt.csa_obj);
 			amdgpu_device_wb_fini(adev);
 			amdgpu_device_vram_scratch_fini(adev);
+			amdgpu_ib_pool_fini(adev);
 		}
 
 		r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
@@ -1937,6 +1954,8 @@ static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
 		adev->ip_blocks[i].status.late_initialized = false;
 	}
 
+	amdgpu_ras_fini(adev);
+
 	if (amdgpu_sriov_vf(adev))
 		if (amdgpu_virt_release_full_gpu(adev, false))
 			DRM_ERROR("failed to release exclusive mode on fini\n");
@@ -1999,6 +2018,10 @@ static void amdgpu_device_ip_late_init_func_handler(struct work_struct *work)
 	r = amdgpu_device_enable_mgpu_fan_boost();
 	if (r)
 		DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);
+
+	/*set to low pstate by default */
+	amdgpu_xgmi_set_pstate(adev, 0);
+
 }
 
 static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
@@ -2369,7 +2392,7 @@ static void amdgpu_device_xgmi_reset_func(struct work_struct *__work)
 
 	adev->asic_reset_res =  amdgpu_asic_reset(adev);
 	if (adev->asic_reset_res)
-		DRM_WARN("ASIC reset failed with err r, %d for drm dev, %s",
+		DRM_WARN("ASIC reset failed with error, %d for drm dev, %s",
 			 adev->asic_reset_res, adev->ddev->unique);
 }
 
@@ -2448,6 +2471,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	mutex_init(&adev->virt.vf_errors.lock);
 	hash_init(adev->mn_hash);
 	mutex_init(&adev->lock_reset);
+	mutex_init(&adev->virt.dpm_mutex);
 
 	amdgpu_device_check_arguments(adev);
 
@@ -2642,13 +2666,6 @@ fence_driver_init:
 	/* Get a log2 for easy divisions. */
 	adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
 
-	r = amdgpu_ib_pool_init(adev);
-	if (r) {
-		dev_err(adev->dev, "IB initialization failed (%d).\n", r);
-		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
-		goto failed;
-	}
-
 	amdgpu_fbdev_init(adev);
 
 	r = amdgpu_pm_sysfs_init(adev);
@@ -2694,6 +2711,9 @@ fence_driver_init:
 		goto failed;
 	}
 
+	/* must succeed. */
+	amdgpu_ras_post_init(adev);
+
 	return 0;
 
 failed:
@@ -2726,7 +2746,6 @@ void amdgpu_device_fini(struct amdgpu_device *adev)
 		else
 			drm_atomic_helper_shutdown(adev->ddev);
 	}
-	amdgpu_ib_pool_fini(adev);
 	amdgpu_fence_driver_fini(adev);
 	amdgpu_pm_sysfs_fini(adev);
 	amdgpu_fbdev_fini(adev);
@@ -3225,6 +3244,8 @@ static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
 	if (r)
 		return r;
 
+	amdgpu_amdkfd_pre_reset(adev);
+
 	/* Resume IP prior to SMC */
 	r = amdgpu_device_ip_reinit_early_sriov(adev);
 	if (r)
@@ -3244,6 +3265,7 @@ static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
 
 	amdgpu_irq_gpu_reset_resume_helper(adev);
 	r = amdgpu_ib_ring_tests(adev);
+	amdgpu_amdkfd_post_reset(adev);
 
 error:
 	amdgpu_virt_init_data_exchange(adev);
@@ -3376,7 +3398,7 @@ static int amdgpu_do_asic_reset(struct amdgpu_hive_info *hive,
 				r = amdgpu_asic_reset(tmp_adev);
 
 			if (r) {
-				DRM_ERROR("ASIC reset failed with err r, %d for drm dev, %s",
+				DRM_ERROR("ASIC reset failed with error, %d for drm dev, %s",
 					 r, tmp_adev->ddev->unique);
 				break;
 			}
@@ -3393,6 +3415,11 @@ static int amdgpu_do_asic_reset(struct amdgpu_hive_info *hive,
 						break;
 				}
 			}
+
+			list_for_each_entry(tmp_adev, device_list_handle,
+					gmc.xgmi.head) {
+				amdgpu_ras_reserve_bad_pages(tmp_adev);
+			}
 		}
 	}
 
@@ -3411,7 +3438,7 @@ static int amdgpu_do_asic_reset(struct amdgpu_hive_info *hive,
 
 				vram_lost = amdgpu_device_check_vram_lost(tmp_adev);
 				if (vram_lost) {
-					DRM_ERROR("VRAM is lost!\n");
+					DRM_INFO("VRAM is lost due to GPU reset!\n");
 					atomic_inc(&tmp_adev->vram_lost_counter);
 				}
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c
index 344967df3137..523b8ab6b04e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.c
@@ -904,3 +904,19 @@ amdgpu_get_vce_clock_state(void *handle, u32 idx)
 
 	return NULL;
 }
+
+int amdgpu_dpm_get_sclk(struct amdgpu_device *adev, bool low)
+{
+	if (is_support_sw_smu(adev))
+		return smu_get_sclk(&adev->smu, low);
+	else
+		return (adev)->powerplay.pp_funcs->get_sclk((adev)->powerplay.pp_handle, (low));
+}
+
+int amdgpu_dpm_get_mclk(struct amdgpu_device *adev, bool low)
+{
+	if (is_support_sw_smu(adev))
+		return smu_get_mclk(&adev->smu, low);
+	else
+		return (adev)->powerplay.pp_funcs->get_mclk((adev)->powerplay.pp_handle, (low));
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
index e871e022c129..dca35407879d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dpm.h
@@ -260,9 +260,6 @@ enum amdgpu_pcie_gen {
 #define amdgpu_dpm_enable_bapm(adev, e) \
 		((adev)->powerplay.pp_funcs->enable_bapm((adev)->powerplay.pp_handle, (e)))
 
-#define amdgpu_dpm_read_sensor(adev, idx, value, size) \
-		((adev)->powerplay.pp_funcs->read_sensor((adev)->powerplay.pp_handle, (idx), (value), (size)))
-
 #define amdgpu_dpm_set_fan_control_mode(adev, m) \
 		((adev)->powerplay.pp_funcs->set_fan_control_mode((adev)->powerplay.pp_handle, (m)))
 
@@ -281,18 +278,18 @@ enum amdgpu_pcie_gen {
 #define amdgpu_dpm_set_fan_speed_rpm(adev, s) \
 		((adev)->powerplay.pp_funcs->set_fan_speed_rpm)((adev)->powerplay.pp_handle, (s))
 
-#define amdgpu_dpm_get_sclk(adev, l) \
-		((adev)->powerplay.pp_funcs->get_sclk((adev)->powerplay.pp_handle, (l)))
-
-#define amdgpu_dpm_get_mclk(adev, l)  \
-		((adev)->powerplay.pp_funcs->get_mclk((adev)->powerplay.pp_handle, (l)))
-
 #define amdgpu_dpm_force_performance_level(adev, l) \
 		((adev)->powerplay.pp_funcs->force_performance_level((adev)->powerplay.pp_handle, (l)))
 
 #define amdgpu_dpm_get_current_power_state(adev) \
 		((adev)->powerplay.pp_funcs->get_current_power_state((adev)->powerplay.pp_handle))
 
+#define amdgpu_smu_get_current_power_state(adev) \
+		((adev)->smu.ppt_funcs->get_current_power_state(&((adev)->smu)))
+
+#define amdgpu_smu_set_power_state(adev) \
+		((adev)->smu.ppt_funcs->set_power_state(&((adev)->smu)))
+
 #define amdgpu_dpm_get_pp_num_states(adev, data) \
 		((adev)->powerplay.pp_funcs->get_pp_num_states((adev)->powerplay.pp_handle, data))
 
@@ -448,6 +445,9 @@ struct amdgpu_pm {
 	uint32_t                smu_prv_buffer_size;
 	struct amdgpu_bo        *smu_prv_buffer;
 	bool ac_power;
+	/* powerplay feature */
+	uint32_t pp_feature;
+
 };
 
 #define R600_SSTU_DFLT                               0
@@ -486,6 +486,8 @@ void amdgpu_dpm_print_ps_status(struct amdgpu_device *adev,
 u32 amdgpu_dpm_get_vblank_time(struct amdgpu_device *adev);
 u32 amdgpu_dpm_get_vrefresh(struct amdgpu_device *adev);
 void amdgpu_dpm_get_active_displays(struct amdgpu_device *adev);
+int amdgpu_dpm_read_sensor(struct amdgpu_device *adev, enum amd_pp_sensors sensor,
+			   void *data, uint32_t *size);
 
 bool amdgpu_is_internal_thermal_sensor(enum amdgpu_int_thermal_type sensor);
 
@@ -504,4 +506,8 @@ enum amdgpu_pcie_gen amdgpu_get_pcie_gen_support(struct amdgpu_device *adev,
 struct amd_vce_state*
 amdgpu_get_vce_clock_state(void *handle, u32 idx);
 
+extern int amdgpu_dpm_get_sclk(struct amdgpu_device *adev, bool low);
+
+extern int amdgpu_dpm_get_mclk(struct amdgpu_device *adev, bool low);
+
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index 7419ea8a388b..1e2cc9d68a05 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -74,9 +74,11 @@
  * - 3.28.0 - Add AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES
  * - 3.29.0 - Add AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID
  * - 3.30.0 - Add AMDGPU_SCHED_OP_CONTEXT_PRIORITY_OVERRIDE.
+ * - 3.31.0 - Add support for per-flip tiling attribute changes with DC
+ * - 3.32.0 - Add syncobj timeline support to AMDGPU_CS.
  */
 #define KMS_DRIVER_MAJOR	3
-#define KMS_DRIVER_MINOR	30
+#define KMS_DRIVER_MINOR	32
 #define KMS_DRIVER_PATCHLEVEL	0
 
 int amdgpu_vram_limit = 0;
@@ -117,8 +119,8 @@ uint amdgpu_pg_mask = 0xffffffff;
 uint amdgpu_sdma_phase_quantum = 32;
 char *amdgpu_disable_cu = NULL;
 char *amdgpu_virtual_display = NULL;
-/* OverDrive(bit 14),gfxoff(bit 15),stutter mode(bit 17) disabled by default*/
-uint amdgpu_pp_feature_mask = 0xfffd3fff;
+/* OverDrive(bit 14) disabled by default*/
+uint amdgpu_pp_feature_mask = 0xffffbfff;
 int amdgpu_ngg = 0;
 int amdgpu_prim_buf_per_se = 0;
 int amdgpu_pos_buf_per_se = 0;
@@ -136,6 +138,8 @@ uint amdgpu_dc_feature_mask = 0;
 struct amdgpu_mgpu_info mgpu_info = {
 	.mutex = __MUTEX_INITIALIZER(mgpu_info.mutex),
 };
+int amdgpu_ras_enable = -1;
+uint amdgpu_ras_mask = 0xffffffff;
 
 /**
  * DOC: vramlimit (int)
@@ -495,6 +499,21 @@ MODULE_PARM_DESC(emu_mode, "Emulation mode, (1 = enable, 0 = disable)");
 module_param_named(emu_mode, amdgpu_emu_mode, int, 0444);
 
 /**
+ * DOC: ras_enable (int)
+ * Enable RAS features on the GPU (0 = disable, 1 = enable, -1 = auto (default))
+ */
+MODULE_PARM_DESC(ras_enable, "Enable RAS features on the GPU (0 = disable, 1 = enable, -1 = auto (default))");
+module_param_named(ras_enable, amdgpu_ras_enable, int, 0444);
+
+/**
+ * DOC: ras_mask (uint)
+ * Mask of RAS features to enable (default 0xffffffff), only valid when ras_enable == 1
+ * See the flags in drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h
+ */
+MODULE_PARM_DESC(ras_mask, "Mask of RAS features to enable (default 0xffffffff), only valid when ras_enable == 1");
+module_param_named(ras_mask, amdgpu_ras_mask, uint, 0444);
+
+/**
  * DOC: si_support (int)
  * Set SI support driver. This parameter works after set config CONFIG_DRM_AMDGPU_SI. For SI asic, when radeon driver is enabled,
  * set value 0 to use radeon driver, while set value 1 to use amdgpu driver. The default is using radeon driver when it available,
@@ -974,6 +993,7 @@ amdgpu_pci_remove(struct pci_dev *pdev)
 
 	DRM_ERROR("Device removal is currently not supported outside of fbcon\n");
 	drm_dev_unplug(dev);
+	drm_dev_put(dev);
 	pci_disable_device(pdev);
 	pci_set_drvdata(pdev, NULL);
 }
@@ -1158,13 +1178,14 @@ static int amdgpu_flush(struct file *f, fl_owner_t id)
 {
 	struct drm_file *file_priv = f->private_data;
 	struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
+	long timeout = MAX_WAIT_SCHED_ENTITY_Q_EMPTY;
 
-	amdgpu_ctx_mgr_entity_flush(&fpriv->ctx_mgr);
+	timeout = amdgpu_ctx_mgr_entity_flush(&fpriv->ctx_mgr, timeout);
+	timeout = amdgpu_vm_wait_idle(&fpriv->vm, timeout);
 
-	return 0;
+	return timeout >= 0 ? 0 : timeout;
 }
 
-
 static const struct file_operations amdgpu_driver_kms_fops = {
 	.owner = THIS_MODULE,
 	.open = drm_open,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
index 5cbde74b97dd..e47609218839 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c
@@ -49,12 +49,11 @@
 static int
 amdgpufb_open(struct fb_info *info, int user)
 {
-	struct amdgpu_fbdev *rfbdev = info->par;
-	struct amdgpu_device *adev = rfbdev->adev;
-	int ret = pm_runtime_get_sync(adev->ddev->dev);
+	struct drm_fb_helper *fb_helper = info->par;
+	int ret = pm_runtime_get_sync(fb_helper->dev->dev);
 	if (ret < 0 && ret != -EACCES) {
-		pm_runtime_mark_last_busy(adev->ddev->dev);
-		pm_runtime_put_autosuspend(adev->ddev->dev);
+		pm_runtime_mark_last_busy(fb_helper->dev->dev);
+		pm_runtime_put_autosuspend(fb_helper->dev->dev);
 		return ret;
 	}
 	return 0;
@@ -63,11 +62,10 @@ amdgpufb_open(struct fb_info *info, int user)
 static int
 amdgpufb_release(struct fb_info *info, int user)
 {
-	struct amdgpu_fbdev *rfbdev = info->par;
-	struct amdgpu_device *adev = rfbdev->adev;
+	struct drm_fb_helper *fb_helper = info->par;
 
-	pm_runtime_mark_last_busy(adev->ddev->dev);
-	pm_runtime_put_autosuspend(adev->ddev->dev);
+	pm_runtime_mark_last_busy(fb_helper->dev->dev);
+	pm_runtime_put_autosuspend(fb_helper->dev->dev);
 	return 0;
 }
 
@@ -233,9 +231,6 @@ static int amdgpufb_create(struct drm_fb_helper *helper,
 		goto out;
 	}
 
-	info->par = rfbdev;
-	info->skip_vt_switch = true;
-
 	ret = amdgpu_display_framebuffer_init(adev->ddev, &rfbdev->rfb,
 					      &mode_cmd, gobj);
 	if (ret) {
@@ -248,10 +243,6 @@ static int amdgpufb_create(struct drm_fb_helper *helper,
 	/* setup helper */
 	rfbdev->helper.fb = fb;
 
-	strcpy(info->fix.id, "amdgpudrmfb");
-
-	drm_fb_helper_fill_fix(info, fb->pitches[0], fb->format->depth);
-
 	info->fbops = &amdgpufb_ops;
 
 	tmp = amdgpu_bo_gpu_offset(abo) - adev->gmc.vram_start;
@@ -260,7 +251,7 @@ static int amdgpufb_create(struct drm_fb_helper *helper,
 	info->screen_base = amdgpu_bo_kptr(abo);
 	info->screen_size = amdgpu_bo_size(abo);
 
-	drm_fb_helper_fill_var(info, &rfbdev->helper, sizes->fb_width, sizes->fb_height);
+	drm_fb_helper_fill_info(info, &rfbdev->helper, sizes);
 
 	/* setup aperture base/size for vesafb takeover */
 	info->apertures->ranges[0].base = adev->ddev->mode_config.fb_base;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
index ee47c11e92ce..4dee2326b29c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
@@ -136,8 +136,9 @@ int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f,
 {
 	struct amdgpu_device *adev = ring->adev;
 	struct amdgpu_fence *fence;
-	struct dma_fence *old, **ptr;
+	struct dma_fence __rcu **ptr;
 	uint32_t seq;
+	int r;
 
 	fence = kmem_cache_alloc(amdgpu_fence_slab, GFP_KERNEL);
 	if (fence == NULL)
@@ -153,15 +154,24 @@ int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f,
 			       seq, flags | AMDGPU_FENCE_FLAG_INT);
 
 	ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];
+	if (unlikely(rcu_dereference_protected(*ptr, 1))) {
+		struct dma_fence *old;
+
+		rcu_read_lock();
+		old = dma_fence_get_rcu_safe(ptr);
+		rcu_read_unlock();
+
+		if (old) {
+			r = dma_fence_wait(old, false);
+			dma_fence_put(old);
+			if (r)
+				return r;
+		}
+	}
+
 	/* This function can't be called concurrently anyway, otherwise
 	 * emitting the fence would mess up the hardware ring buffer.
 	 */
-	old = rcu_dereference_protected(*ptr, 1);
-	if (old && !dma_fence_is_signaled(old)) {
-		DRM_INFO("rcu slot is busy\n");
-		dma_fence_wait(old, false);
-	}
-
 	rcu_assign_pointer(*ptr, dma_fence_get(&fence->base));
 
 	*f = &fence->base;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index d21dd2f369da..d4fcf5475464 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -31,6 +31,7 @@
 #include <drm/amdgpu_drm.h>
 #include "amdgpu.h"
 #include "amdgpu_display.h"
+#include "amdgpu_xgmi.h"
 
 void amdgpu_gem_object_free(struct drm_gem_object *gobj)
 {
@@ -627,11 +628,6 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
 
 	switch (args->operation) {
 	case AMDGPU_VA_OP_MAP:
-		r = amdgpu_vm_alloc_pts(adev, bo_va->base.vm, args->va_address,
-					args->map_size);
-		if (r)
-			goto error_backoff;
-
 		va_flags = amdgpu_gmc_get_pte_flags(adev, args->flags);
 		r = amdgpu_vm_bo_map(adev, bo_va, args->va_address,
 				     args->offset_in_bo, args->map_size,
@@ -647,11 +643,6 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
 						args->map_size);
 		break;
 	case AMDGPU_VA_OP_REPLACE:
-		r = amdgpu_vm_alloc_pts(adev, bo_va->base.vm, args->va_address,
-					args->map_size);
-		if (r)
-			goto error_backoff;
-
 		va_flags = amdgpu_gmc_get_pte_flags(adev, args->flags);
 		r = amdgpu_vm_bo_replace_map(adev, bo_va, args->va_address,
 					     args->offset_in_bo, args->map_size,
@@ -678,6 +669,7 @@ int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data,
 	struct amdgpu_device *adev = dev->dev_private;
 	struct drm_amdgpu_gem_op *args = data;
 	struct drm_gem_object *gobj;
+	struct amdgpu_vm_bo_base *base;
 	struct amdgpu_bo *robj;
 	int r;
 
@@ -716,6 +708,15 @@ int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data,
 			amdgpu_bo_unreserve(robj);
 			break;
 		}
+		for (base = robj->vm_bo; base; base = base->next)
+			if (amdgpu_xgmi_same_hive(amdgpu_ttm_adev(robj->tbo.bdev),
+				amdgpu_ttm_adev(base->vm->root.base.bo->tbo.bdev))) {
+				r = -EINVAL;
+				amdgpu_bo_unreserve(robj);
+				goto out;
+			}
+
+
 		robj->preferred_domains = args->value & (AMDGPU_GEM_DOMAIN_VRAM |
 							AMDGPU_GEM_DOMAIN_GTT |
 							AMDGPU_GEM_DOMAIN_CPU);
@@ -745,17 +746,25 @@ int amdgpu_mode_dumb_create(struct drm_file *file_priv,
 	struct amdgpu_device *adev = dev->dev_private;
 	struct drm_gem_object *gobj;
 	uint32_t handle;
+	u64 flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
 	u32 domain;
 	int r;
 
+	/*
+	 * The buffer returned from this function should be cleared, but
+	 * it can only be done if the ring is enabled or we'll fail to
+	 * create the buffer.
+	 */
+	if (adev->mman.buffer_funcs_enabled)
+		flags |= AMDGPU_GEM_CREATE_VRAM_CLEARED;
+
 	args->pitch = amdgpu_align_pitch(adev, args->width,
 					 DIV_ROUND_UP(args->bpp, 8), 0);
 	args->size = (u64)args->pitch * args->height;
 	args->size = ALIGN(args->size, PAGE_SIZE);
 	domain = amdgpu_bo_get_preferred_pin_domain(adev,
 				amdgpu_display_supported_domains(adev));
-	r = amdgpu_gem_object_create(adev, args->size, 0, domain,
-				     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+	r = amdgpu_gem_object_create(adev, args->size, 0, domain, flags,
 				     ttm_bo_type_device, NULL, &gobj);
 	if (r)
 		return -ENOMEM;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
index 97a60da62004..997932ebbb83 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
@@ -390,7 +390,7 @@ void amdgpu_gfx_compute_mqd_sw_fini(struct amdgpu_device *adev)
 
 void amdgpu_gfx_off_ctrl(struct amdgpu_device *adev, bool enable)
 {
-	if (!(adev->powerplay.pp_feature & PP_GFXOFF_MASK))
+	if (!(adev->pm.pp_feature & PP_GFXOFF_MASK))
 		return;
 
 	if (!adev->powerplay.pp_funcs || !adev->powerplay.pp_funcs->set_powergating_by_smu)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
index f790e15bcd08..09fc53af3d35 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
@@ -258,6 +258,9 @@ struct amdgpu_gfx {
 	/* pipe reservation */
 	struct mutex			pipe_reserve_mutex;
 	DECLARE_BITMAP			(pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
+
+	/*ras */
+	struct ras_common_if		*ras_if;
 };
 
 #define amdgpu_gfx_get_gpu_clock_counter(adev) (adev)->gfx.funcs->get_gpu_clock_counter((adev))
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
index d73367cab4f3..250d9212cc38 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
@@ -80,6 +80,33 @@ uint64_t amdgpu_gmc_pd_addr(struct amdgpu_bo *bo)
 }
 
 /**
+ * amdgpu_gmc_set_pte_pde - update the page tables using CPU
+ *
+ * @adev: amdgpu_device pointer
+ * @cpu_pt_addr: cpu address of the page table
+ * @gpu_page_idx: entry in the page table to update
+ * @addr: dst addr to write into pte/pde
+ * @flags: access flags
+ *
+ * Update the page tables using CPU.
+ */
+int amdgpu_gmc_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr,
+				uint32_t gpu_page_idx, uint64_t addr,
+				uint64_t flags)
+{
+	void __iomem *ptr = (void *)cpu_pt_addr;
+	uint64_t value;
+
+	/*
+	 * The following is for PTE only. GART does not have PDEs.
+	*/
+	value = addr & 0x0000FFFFFFFFF000ULL;
+	value |= flags;
+	writeq(value, ptr + (gpu_page_idx * 8));
+	return 0;
+}
+
+/**
  * amdgpu_gmc_agp_addr - return the address in the AGP address space
  *
  * @tbo: TTM BO which needs the address, must be in GTT domain
@@ -213,3 +240,58 @@ void amdgpu_gmc_agp_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc)
 	dev_info(adev->dev, "AGP: %lluM 0x%016llX - 0x%016llX\n",
 			mc->agp_size >> 20, mc->agp_start, mc->agp_end);
 }
+
+/**
+ * amdgpu_gmc_filter_faults - filter VM faults
+ *
+ * @adev: amdgpu device structure
+ * @addr: address of the VM fault
+ * @pasid: PASID of the process causing the fault
+ * @timestamp: timestamp of the fault
+ *
+ * Returns:
+ * True if the fault was filtered and should not be processed further.
+ * False if the fault is a new one and needs to be handled.
+ */
+bool amdgpu_gmc_filter_faults(struct amdgpu_device *adev, uint64_t addr,
+			      uint16_t pasid, uint64_t timestamp)
+{
+	struct amdgpu_gmc *gmc = &adev->gmc;
+
+	uint64_t stamp, key = addr << 4 | pasid;
+	struct amdgpu_gmc_fault *fault;
+	uint32_t hash;
+
+	/* If we don't have space left in the ring buffer return immediately */
+	stamp = max(timestamp, AMDGPU_GMC_FAULT_TIMEOUT + 1) -
+		AMDGPU_GMC_FAULT_TIMEOUT;
+	if (gmc->fault_ring[gmc->last_fault].timestamp >= stamp)
+		return true;
+
+	/* Try to find the fault in the hash */
+	hash = hash_64(key, AMDGPU_GMC_FAULT_HASH_ORDER);
+	fault = &gmc->fault_ring[gmc->fault_hash[hash].idx];
+	while (fault->timestamp >= stamp) {
+		uint64_t tmp;
+
+		if (fault->key == key)
+			return true;
+
+		tmp = fault->timestamp;
+		fault = &gmc->fault_ring[fault->next];
+
+		/* Check if the entry was reused */
+		if (fault->timestamp >= tmp)
+			break;
+	}
+
+	/* Add the fault to the ring */
+	fault = &gmc->fault_ring[gmc->last_fault];
+	fault->key = key;
+	fault->timestamp = timestamp;
+
+	/* And update the hash */
+	fault->next = gmc->fault_hash[hash].idx;
+	gmc->fault_hash[hash].idx = gmc->last_fault++;
+	return false;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
index 81e6070d255b..071145ac67b5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
@@ -43,9 +43,35 @@
  */
 #define AMDGPU_GMC_HOLE_MASK	0x0000ffffffffffffULL
 
+/*
+ * Ring size as power of two for the log of recent faults.
+ */
+#define AMDGPU_GMC_FAULT_RING_ORDER	8
+#define AMDGPU_GMC_FAULT_RING_SIZE	(1 << AMDGPU_GMC_FAULT_RING_ORDER)
+
+/*
+ * Hash size as power of two for the log of recent faults
+ */
+#define AMDGPU_GMC_FAULT_HASH_ORDER	8
+#define AMDGPU_GMC_FAULT_HASH_SIZE	(1 << AMDGPU_GMC_FAULT_HASH_ORDER)
+
+/*
+ * Number of IH timestamp ticks until a fault is considered handled
+ */
+#define AMDGPU_GMC_FAULT_TIMEOUT	5000ULL
+
 struct firmware;
 
 /*
+ * GMC page fault information
+ */
+struct amdgpu_gmc_fault {
+	uint64_t	timestamp;
+	uint64_t	next:AMDGPU_GMC_FAULT_RING_ORDER;
+	uint64_t	key:52;
+};
+
+/*
  * VMHUB structures, functions & helpers
  */
 struct amdgpu_vmhub {
@@ -71,12 +97,6 @@ struct amdgpu_gmc_funcs {
 	/* Change the VMID -> PASID mapping */
 	void (*emit_pasid_mapping)(struct amdgpu_ring *ring, unsigned vmid,
 				   unsigned pasid);
-	/* write pte/pde updates using the cpu */
-	int (*set_pte_pde)(struct amdgpu_device *adev,
-			   void *cpu_pt_addr, /* cpu addr of page table */
-			   uint32_t gpu_page_idx, /* pte/pde to update */
-			   uint64_t addr, /* addr to write into pte/pde */
-			   uint64_t flags); /* access flags */
 	/* enable/disable PRT support */
 	void (*set_prt)(struct amdgpu_device *adev, bool enable);
 	/* set pte flags based per asic */
@@ -147,15 +167,22 @@ struct amdgpu_gmc {
 	struct kfd_vm_fault_info *vm_fault_info;
 	atomic_t		vm_fault_info_updated;
 
+	struct amdgpu_gmc_fault	fault_ring[AMDGPU_GMC_FAULT_RING_SIZE];
+	struct {
+		uint64_t	idx:AMDGPU_GMC_FAULT_RING_ORDER;
+	} fault_hash[AMDGPU_GMC_FAULT_HASH_SIZE];
+	uint64_t		last_fault:AMDGPU_GMC_FAULT_RING_ORDER;
+
 	const struct amdgpu_gmc_funcs	*gmc_funcs;
 
 	struct amdgpu_xgmi xgmi;
+	struct amdgpu_irq_src	ecc_irq;
+	struct ras_common_if    *ras_if;
 };
 
 #define amdgpu_gmc_flush_gpu_tlb(adev, vmid, type) (adev)->gmc.gmc_funcs->flush_gpu_tlb((adev), (vmid), (type))
 #define amdgpu_gmc_emit_flush_gpu_tlb(r, vmid, addr) (r)->adev->gmc.gmc_funcs->emit_flush_gpu_tlb((r), (vmid), (addr))
 #define amdgpu_gmc_emit_pasid_mapping(r, vmid, pasid) (r)->adev->gmc.gmc_funcs->emit_pasid_mapping((r), (vmid), (pasid))
-#define amdgpu_gmc_set_pte_pde(adev, pt, idx, addr, flags) (adev)->gmc.gmc_funcs->set_pte_pde((adev), (pt), (idx), (addr), (flags))
 #define amdgpu_gmc_get_vm_pde(adev, level, dst, flags) (adev)->gmc.gmc_funcs->get_vm_pde((adev), (level), (dst), (flags))
 #define amdgpu_gmc_get_pte_flags(adev, flags) (adev)->gmc.gmc_funcs->get_vm_pte_flags((adev),(flags))
 
@@ -189,6 +216,9 @@ static inline uint64_t amdgpu_gmc_sign_extend(uint64_t addr)
 
 void amdgpu_gmc_get_pde_for_bo(struct amdgpu_bo *bo, int level,
 			       uint64_t *addr, uint64_t *flags);
+int amdgpu_gmc_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr,
+				uint32_t gpu_page_idx, uint64_t addr,
+				uint64_t flags);
 uint64_t amdgpu_gmc_pd_addr(struct amdgpu_bo *bo);
 uint64_t amdgpu_gmc_agp_addr(struct ttm_buffer_object *bo);
 void amdgpu_gmc_vram_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc,
@@ -197,5 +227,7 @@ void amdgpu_gmc_gart_location(struct amdgpu_device *adev,
 			      struct amdgpu_gmc *mc);
 void amdgpu_gmc_agp_location(struct amdgpu_device *adev,
 			     struct amdgpu_gmc *mc);
+bool amdgpu_gmc_filter_faults(struct amdgpu_device *adev, uint64_t addr,
+			      uint16_t pasid, uint64_t timestamp);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
index da7b1b92d9cf..62591d081856 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
@@ -37,6 +37,47 @@ struct amdgpu_gtt_node {
 };
 
 /**
+ * DOC: mem_info_gtt_total
+ *
+ * The amdgpu driver provides a sysfs API for reporting current total size of
+ * the GTT.
+ * The file mem_info_gtt_total is used for this, and returns the total size of
+ * the GTT block, in bytes
+ */
+static ssize_t amdgpu_mem_info_gtt_total_show(struct device *dev,
+		struct device_attribute *attr, char *buf)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct amdgpu_device *adev = ddev->dev_private;
+
+	return snprintf(buf, PAGE_SIZE, "%llu\n",
+			(adev->mman.bdev.man[TTM_PL_TT].size) * PAGE_SIZE);
+}
+
+/**
+ * DOC: mem_info_gtt_used
+ *
+ * The amdgpu driver provides a sysfs API for reporting current total amount of
+ * used GTT.
+ * The file mem_info_gtt_used is used for this, and returns the current used
+ * size of the GTT block, in bytes
+ */
+static ssize_t amdgpu_mem_info_gtt_used_show(struct device *dev,
+		struct device_attribute *attr, char *buf)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct amdgpu_device *adev = ddev->dev_private;
+
+	return snprintf(buf, PAGE_SIZE, "%llu\n",
+			amdgpu_gtt_mgr_usage(&adev->mman.bdev.man[TTM_PL_TT]));
+}
+
+static DEVICE_ATTR(mem_info_gtt_total, S_IRUGO,
+	           amdgpu_mem_info_gtt_total_show, NULL);
+static DEVICE_ATTR(mem_info_gtt_used, S_IRUGO,
+	           amdgpu_mem_info_gtt_used_show, NULL);
+
+/**
  * amdgpu_gtt_mgr_init - init GTT manager and DRM MM
  *
  * @man: TTM memory type manager
@@ -50,6 +91,7 @@ static int amdgpu_gtt_mgr_init(struct ttm_mem_type_manager *man,
 	struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev);
 	struct amdgpu_gtt_mgr *mgr;
 	uint64_t start, size;
+	int ret;
 
 	mgr = kzalloc(sizeof(*mgr), GFP_KERNEL);
 	if (!mgr)
@@ -61,6 +103,18 @@ static int amdgpu_gtt_mgr_init(struct ttm_mem_type_manager *man,
 	spin_lock_init(&mgr->lock);
 	atomic64_set(&mgr->available, p_size);
 	man->priv = mgr;
+
+	ret = device_create_file(adev->dev, &dev_attr_mem_info_gtt_total);
+	if (ret) {
+		DRM_ERROR("Failed to create device file mem_info_gtt_total\n");
+		return ret;
+	}
+	ret = device_create_file(adev->dev, &dev_attr_mem_info_gtt_used);
+	if (ret) {
+		DRM_ERROR("Failed to create device file mem_info_gtt_used\n");
+		return ret;
+	}
+
 	return 0;
 }
 
@@ -74,12 +128,17 @@ static int amdgpu_gtt_mgr_init(struct ttm_mem_type_manager *man,
  */
 static int amdgpu_gtt_mgr_fini(struct ttm_mem_type_manager *man)
 {
+	struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev);
 	struct amdgpu_gtt_mgr *mgr = man->priv;
 	spin_lock(&mgr->lock);
 	drm_mm_takedown(&mgr->mm);
 	spin_unlock(&mgr->lock);
 	kfree(mgr);
 	man->priv = NULL;
+
+	device_remove_file(adev->dev, &dev_attr_mem_info_gtt_total);
+	device_remove_file(adev->dev, &dev_attr_mem_info_gtt_used);
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c
index 1c50be3ab8a9..934dfdcb4e73 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c
@@ -142,6 +142,7 @@ void amdgpu_ih_ring_fini(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih)
  */
 int amdgpu_ih_process(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih)
 {
+	unsigned int count = AMDGPU_IH_MAX_NUM_IVS;
 	u32 wptr;
 
 	if (!ih->enabled || adev->shutdown)
@@ -159,7 +160,7 @@ restart_ih:
 	/* Order reading of wptr vs. reading of IH ring data */
 	rmb();
 
-	while (ih->rptr != wptr) {
+	while (ih->rptr != wptr && --count) {
 		amdgpu_irq_dispatch(adev, ih);
 		ih->rptr &= ih->ptr_mask;
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h
index 113a1ba13d4a..4e0bb645176d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h
@@ -24,6 +24,9 @@
 #ifndef __AMDGPU_IH_H__
 #define __AMDGPU_IH_H__
 
+/* Maximum number of IVs processed at once */
+#define AMDGPU_IH_MAX_NUM_IVS	32
+
 struct amdgpu_device;
 struct amdgpu_iv_entry;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index e860412043bb..b17d0545728e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -39,6 +39,7 @@
 #include "amdgpu_amdkfd.h"
 #include "amdgpu_gem.h"
 #include "amdgpu_display.h"
+#include "amdgpu_ras.h"
 
 static void amdgpu_unregister_gpu_instance(struct amdgpu_device *adev)
 {
@@ -296,6 +297,17 @@ static int amdgpu_firmware_info(struct drm_amdgpu_info_firmware *fw_info,
 		fw_info->ver = adev->pm.fw_version;
 		fw_info->feature = 0;
 		break;
+	case AMDGPU_INFO_FW_TA:
+		if (query_fw->index > 1)
+			return -EINVAL;
+		if (query_fw->index == 0) {
+			fw_info->ver = adev->psp.ta_fw_version;
+			fw_info->feature = adev->psp.ta_xgmi_ucode_version;
+		} else {
+			fw_info->ver = adev->psp.ta_fw_version;
+			fw_info->feature = adev->psp.ta_ras_ucode_version;
+		}
+		break;
 	case AMDGPU_INFO_FW_SDMA:
 		if (query_fw->index >= adev->sdma.num_instances)
 			return -EINVAL;
@@ -684,6 +696,10 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
 		if (adev->pm.dpm_enabled) {
 			dev_info.max_engine_clock = amdgpu_dpm_get_sclk(adev, false) * 10;
 			dev_info.max_memory_clock = amdgpu_dpm_get_mclk(adev, false) * 10;
+		} else if (amdgpu_sriov_vf(adev) && amdgim_is_hwperf(adev) &&
+			   adev->virt.ops->get_pp_clk) {
+			dev_info.max_engine_clock = amdgpu_virt_get_sclk(adev, false) * 10;
+			dev_info.max_memory_clock = amdgpu_virt_get_mclk(adev, false) * 10;
 		} else {
 			dev_info.max_engine_clock = adev->clock.default_sclk * 10;
 			dev_info.max_memory_clock = adev->clock.default_mclk * 10;
@@ -909,6 +925,18 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
 	case AMDGPU_INFO_VRAM_LOST_COUNTER:
 		ui32 = atomic_read(&adev->vram_lost_counter);
 		return copy_to_user(out, &ui32, min(size, 4u)) ? -EFAULT : 0;
+	case AMDGPU_INFO_RAS_ENABLED_FEATURES: {
+		struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
+		uint64_t ras_mask;
+
+		if (!ras)
+			return -EINVAL;
+		ras_mask = (uint64_t)ras->supported << 32 | ras->features;
+
+		return copy_to_user(out, &ras_mask,
+				min_t(u64, size, sizeof(ras_mask))) ?
+			-EFAULT : 0;
+	}
 	default:
 		DRM_DEBUG_KMS("Invalid request %d\n", info->query);
 		return -EINVAL;
@@ -1328,6 +1356,16 @@ static int amdgpu_debugfs_firmware_info(struct seq_file *m, void *data)
 	seq_printf(m, "ASD feature version: %u, firmware version: 0x%08x\n",
 		   fw_info.feature, fw_info.ver);
 
+	query_fw.fw_type = AMDGPU_INFO_FW_TA;
+	for (i = 0; i < 2; i++) {
+		query_fw.index = i;
+		ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
+		if (ret)
+			continue;
+		seq_printf(m, "TA %s feature version: %u, firmware version: 0x%08x\n",
+				i ? "RAS" : "XGMI", fw_info.feature, fw_info.ver);
+	}
+
 	/* SMC */
 	query_fw.fw_type = AMDGPU_INFO_FW_SMC;
 	ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
index 889e443eeee7..2e9e3db778c6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
@@ -58,7 +58,7 @@ struct amdgpu_hpd;
 #define to_amdgpu_encoder(x) container_of(x, struct amdgpu_encoder, base)
 #define to_amdgpu_framebuffer(x) container_of(x, struct amdgpu_framebuffer, base)
 
-#define to_dm_plane_state(x)	container_of(x, struct dm_plane_state, base);
+#define to_dm_plane_state(x)	container_of(x, struct dm_plane_state, base)
 
 #define AMDGPU_MAX_HPD_PINS 6
 #define AMDGPU_MAX_CRTCS 6
@@ -406,7 +406,7 @@ struct amdgpu_crtc {
 	struct amdgpu_flip_work *pflip_works;
 	enum amdgpu_flip_status pflip_status;
 	int deferred_flip_completion;
-	u64 last_flip_vblank;
+	u32 last_flip_vblank;
 	/* pll sharing */
 	struct amdgpu_atom_ss ss;
 	bool ss_enabled;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index ec9e45004bff..93b2c5a48a71 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -88,12 +88,14 @@ static void amdgpu_bo_destroy(struct ttm_buffer_object *tbo)
 	if (bo->gem_base.import_attach)
 		drm_prime_gem_destroy(&bo->gem_base, bo->tbo.sg);
 	drm_gem_object_release(&bo->gem_base);
-	amdgpu_bo_unref(&bo->parent);
+	/* in case amdgpu_device_recover_vram got NULL of bo->parent */
 	if (!list_empty(&bo->shadow_list)) {
 		mutex_lock(&adev->shadow_list_lock);
 		list_del_init(&bo->shadow_list);
 		mutex_unlock(&adev->shadow_list_lock);
 	}
+	amdgpu_bo_unref(&bo->parent);
+
 	kfree(bo->metadata);
 	kfree(bo);
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
index 220a6a7b1bc1..c430e8259038 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
@@ -72,6 +72,8 @@ struct amdgpu_bo_va {
 
 	/* If the mappings are cleared or filled */
 	bool				cleared;
+
+	bool				is_xgmi;
 };
 
 struct amdgpu_bo {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
index a7adb7b6bd98..95144e49c7f9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
@@ -28,6 +28,7 @@
 #include "amdgpu_pm.h"
 #include "amdgpu_dpm.h"
 #include "amdgpu_display.h"
+#include "amdgpu_smu.h"
 #include "atom.h"
 #include <linux/power_supply.h>
 #include <linux/hwmon.h>
@@ -80,6 +81,27 @@ void amdgpu_pm_acpi_event_handler(struct amdgpu_device *adev)
 	}
 }
 
+int amdgpu_dpm_read_sensor(struct amdgpu_device *adev, enum amd_pp_sensors sensor,
+			   void *data, uint32_t *size)
+{
+	int ret = 0;
+
+	if (!data || !size)
+		return -EINVAL;
+
+	if (is_support_sw_smu(adev))
+		ret = smu_read_sensor(&adev->smu, sensor, data, size);
+	else {
+		if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->read_sensor)
+			ret = adev->powerplay.pp_funcs->read_sensor((adev)->powerplay.pp_handle,
+								    sensor, data, size);
+		else
+			ret = -EINVAL;
+	}
+
+	return ret;
+}
+
 /**
  * DOC: power_dpm_state
  *
@@ -122,7 +144,9 @@ static ssize_t amdgpu_get_dpm_state(struct device *dev,
 	struct amdgpu_device *adev = ddev->dev_private;
 	enum amd_pm_state_type pm;
 
-	if (adev->powerplay.pp_funcs->get_current_power_state)
+	if (is_support_sw_smu(adev) && adev->smu.ppt_funcs->get_current_power_state)
+		pm = amdgpu_smu_get_current_power_state(adev);
+	else if (adev->powerplay.pp_funcs->get_current_power_state)
 		pm = amdgpu_dpm_get_current_power_state(adev);
 	else
 		pm = adev->pm.dpm.user_state;
@@ -240,7 +264,9 @@ static ssize_t amdgpu_get_dpm_forced_performance_level(struct device *dev,
 	     (ddev->switch_power_state != DRM_SWITCH_POWER_ON))
 		return snprintf(buf, PAGE_SIZE, "off\n");
 
-	if (adev->powerplay.pp_funcs->get_performance_level)
+	if (is_support_sw_smu(adev))
+		level = smu_get_performance_level(&adev->smu);
+	else if (adev->powerplay.pp_funcs->get_performance_level)
 		level = amdgpu_dpm_get_performance_level(adev);
 	else
 		level = adev->pm.dpm.forced_level;
@@ -273,7 +299,9 @@ static ssize_t amdgpu_set_dpm_forced_performance_level(struct device *dev,
 	     (ddev->switch_power_state != DRM_SWITCH_POWER_ON))
 		return -EINVAL;
 
-	if (adev->powerplay.pp_funcs->get_performance_level)
+	if (is_support_sw_smu(adev))
+		current_level = smu_get_performance_level(&adev->smu);
+	else if (adev->powerplay.pp_funcs->get_performance_level)
 		current_level = amdgpu_dpm_get_performance_level(adev);
 
 	if (strncmp("low", buf, strlen("low")) == 0) {
@@ -299,10 +327,35 @@ static ssize_t amdgpu_set_dpm_forced_performance_level(struct device *dev,
 		goto fail;
 	}
 
+        if (amdgpu_sriov_vf(adev)) {
+                if (amdgim_is_hwperf(adev) &&
+                    adev->virt.ops->force_dpm_level) {
+                        mutex_lock(&adev->pm.mutex);
+                        adev->virt.ops->force_dpm_level(adev, level);
+                        mutex_unlock(&adev->pm.mutex);
+                        return count;
+                } else {
+                        return -EINVAL;
+		}
+        }
+
 	if (current_level == level)
 		return count;
 
-	if (adev->powerplay.pp_funcs->force_performance_level) {
+	if (is_support_sw_smu(adev)) {
+		mutex_lock(&adev->pm.mutex);
+		if (adev->pm.dpm.thermal_active) {
+			count = -EINVAL;
+			mutex_unlock(&adev->pm.mutex);
+			goto fail;
+		}
+		ret = smu_force_performance_level(&adev->smu, level);
+		if (ret)
+			count = -EINVAL;
+		else
+			adev->pm.dpm.forced_level = level;
+		mutex_unlock(&adev->pm.mutex);
+	} else if (adev->powerplay.pp_funcs->force_performance_level) {
 		mutex_lock(&adev->pm.mutex);
 		if (adev->pm.dpm.thermal_active) {
 			count = -EINVAL;
@@ -328,9 +381,13 @@ static ssize_t amdgpu_get_pp_num_states(struct device *dev,
 	struct drm_device *ddev = dev_get_drvdata(dev);
 	struct amdgpu_device *adev = ddev->dev_private;
 	struct pp_states_info data;
-	int i, buf_len;
+	int i, buf_len, ret;
 
-	if (adev->powerplay.pp_funcs->get_pp_num_states)
+	if (is_support_sw_smu(adev)) {
+		ret = smu_get_power_num_states(&adev->smu, &data);
+		if (ret)
+			return ret;
+	} else if (adev->powerplay.pp_funcs->get_pp_num_states)
 		amdgpu_dpm_get_pp_num_states(adev, &data);
 
 	buf_len = snprintf(buf, PAGE_SIZE, "states: %d\n", data.nums);
@@ -351,23 +408,29 @@ static ssize_t amdgpu_get_pp_cur_state(struct device *dev,
 	struct drm_device *ddev = dev_get_drvdata(dev);
 	struct amdgpu_device *adev = ddev->dev_private;
 	struct pp_states_info data;
+	struct smu_context *smu = &adev->smu;
 	enum amd_pm_state_type pm = 0;
-	int i = 0;
+	int i = 0, ret = 0;
 
-	if (adev->powerplay.pp_funcs->get_current_power_state
+	if (is_support_sw_smu(adev)) {
+		pm = smu_get_current_power_state(smu);
+		ret = smu_get_power_num_states(smu, &data);
+		if (ret)
+			return ret;
+	} else if (adev->powerplay.pp_funcs->get_current_power_state
 		 && adev->powerplay.pp_funcs->get_pp_num_states) {
 		pm = amdgpu_dpm_get_current_power_state(adev);
 		amdgpu_dpm_get_pp_num_states(adev, &data);
+	}
 
-		for (i = 0; i < data.nums; i++) {
-			if (pm == data.states[i])
-				break;
-		}
-
-		if (i == data.nums)
-			i = -EINVAL;
+	for (i = 0; i < data.nums; i++) {
+		if (pm == data.states[i])
+			break;
 	}
 
+	if (i == data.nums)
+		i = -EINVAL;
+
 	return snprintf(buf, PAGE_SIZE, "%d\n", i);
 }
 
@@ -397,6 +460,8 @@ static ssize_t amdgpu_set_pp_force_state(struct device *dev,
 
 	if (strlen(buf) == 1)
 		adev->pp_force_state_enabled = false;
+	else if (is_support_sw_smu(adev))
+		adev->pp_force_state_enabled = false;
 	else if (adev->powerplay.pp_funcs->dispatch_tasks &&
 			adev->powerplay.pp_funcs->get_pp_num_states) {
 		struct pp_states_info data;
@@ -442,7 +507,12 @@ static ssize_t amdgpu_get_pp_table(struct device *dev,
 	char *table = NULL;
 	int size;
 
-	if (adev->powerplay.pp_funcs->get_pp_table)
+	if (is_support_sw_smu(adev)) {
+		size = smu_sys_get_pp_table(&adev->smu, (void **)&table);
+		if (size < 0)
+			return size;
+	}
+	else if (adev->powerplay.pp_funcs->get_pp_table)
 		size = amdgpu_dpm_get_pp_table(adev, &table);
 	else
 		return 0;
@@ -462,8 +532,13 @@ static ssize_t amdgpu_set_pp_table(struct device *dev,
 {
 	struct drm_device *ddev = dev_get_drvdata(dev);
 	struct amdgpu_device *adev = ddev->dev_private;
+	int ret = 0;
 
-	if (adev->powerplay.pp_funcs->set_pp_table)
+	if (is_support_sw_smu(adev)) {
+		ret = smu_sys_set_pp_table(&adev->smu, (void *)buf, count);
+		if (ret)
+			return ret;
+	} else if (adev->powerplay.pp_funcs->set_pp_table)
 		amdgpu_dpm_set_pp_table(adev, buf, count);
 
 	return count;
@@ -586,19 +661,29 @@ static ssize_t amdgpu_set_pp_od_clk_voltage(struct device *dev,
 			tmp_str++;
 	}
 
-	if (adev->powerplay.pp_funcs->odn_edit_dpm_table)
-		ret = amdgpu_dpm_odn_edit_dpm_table(adev, type,
-						parameter, parameter_size);
+	if (is_support_sw_smu(adev)) {
+		ret = smu_od_edit_dpm_table(&adev->smu, type,
+					    parameter, parameter_size);
 
-	if (ret)
-		return -EINVAL;
+		if (ret)
+			return -EINVAL;
+	} else {
+		if (adev->powerplay.pp_funcs->odn_edit_dpm_table)
+			ret = amdgpu_dpm_odn_edit_dpm_table(adev, type,
+						parameter, parameter_size);
 
-	if (type == PP_OD_COMMIT_DPM_TABLE) {
-		if (adev->powerplay.pp_funcs->dispatch_tasks) {
-			amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_READJUST_POWER_STATE, NULL);
-			return count;
-		} else {
+		if (ret)
 			return -EINVAL;
+
+		if (type == PP_OD_COMMIT_DPM_TABLE) {
+			if (adev->powerplay.pp_funcs->dispatch_tasks) {
+				amdgpu_dpm_dispatch_task(adev,
+						AMD_PP_TASK_READJUST_POWER_STATE,
+						NULL);
+				return count;
+			} else {
+				return -EINVAL;
+			}
 		}
 	}
 
@@ -613,7 +698,13 @@ static ssize_t amdgpu_get_pp_od_clk_voltage(struct device *dev,
 	struct amdgpu_device *adev = ddev->dev_private;
 	uint32_t size = 0;
 
-	if (adev->powerplay.pp_funcs->print_clock_levels) {
+	if (is_support_sw_smu(adev)) {
+		size = smu_print_clk_levels(&adev->smu, OD_SCLK, buf);
+		size += smu_print_clk_levels(&adev->smu, OD_MCLK, buf+size);
+		size += smu_print_clk_levels(&adev->smu, OD_VDDC_CURVE, buf+size);
+		size += smu_print_clk_levels(&adev->smu, OD_RANGE, buf+size);
+		return size;
+	} else if (adev->powerplay.pp_funcs->print_clock_levels) {
 		size = amdgpu_dpm_print_clock_levels(adev, OD_SCLK, buf);
 		size += amdgpu_dpm_print_clock_levels(adev, OD_MCLK, buf+size);
 		size += amdgpu_dpm_print_clock_levels(adev, OD_VDDC_CURVE, buf+size);
@@ -711,7 +802,13 @@ static ssize_t amdgpu_get_pp_dpm_sclk(struct device *dev,
 	struct drm_device *ddev = dev_get_drvdata(dev);
 	struct amdgpu_device *adev = ddev->dev_private;
 
-	if (adev->powerplay.pp_funcs->print_clock_levels)
+	if (amdgpu_sriov_vf(adev) && amdgim_is_hwperf(adev) &&
+	    adev->virt.ops->get_pp_clk)
+		return adev->virt.ops->get_pp_clk(adev, PP_SCLK, buf);
+
+	if (is_support_sw_smu(adev))
+		return smu_print_clk_levels(&adev->smu, PP_SCLK, buf);
+	else if (adev->powerplay.pp_funcs->print_clock_levels)
 		return amdgpu_dpm_print_clock_levels(adev, PP_SCLK, buf);
 	else
 		return snprintf(buf, PAGE_SIZE, "\n");
@@ -767,7 +864,9 @@ static ssize_t amdgpu_set_pp_dpm_sclk(struct device *dev,
 	if (ret)
 		return ret;
 
-	if (adev->powerplay.pp_funcs->force_clock_level)
+	if (is_support_sw_smu(adev))
+		ret = smu_force_clk_levels(&adev->smu, PP_SCLK, mask);
+	else if (adev->powerplay.pp_funcs->force_clock_level)
 		ret = amdgpu_dpm_force_clock_level(adev, PP_SCLK, mask);
 
 	if (ret)
@@ -783,7 +882,9 @@ static ssize_t amdgpu_get_pp_dpm_mclk(struct device *dev,
 	struct drm_device *ddev = dev_get_drvdata(dev);
 	struct amdgpu_device *adev = ddev->dev_private;
 
-	if (adev->powerplay.pp_funcs->print_clock_levels)
+	if (is_support_sw_smu(adev))
+		return smu_print_clk_levels(&adev->smu, PP_MCLK, buf);
+	else if (adev->powerplay.pp_funcs->print_clock_levels)
 		return amdgpu_dpm_print_clock_levels(adev, PP_MCLK, buf);
 	else
 		return snprintf(buf, PAGE_SIZE, "\n");
@@ -803,7 +904,9 @@ static ssize_t amdgpu_set_pp_dpm_mclk(struct device *dev,
 	if (ret)
 		return ret;
 
-	if (adev->powerplay.pp_funcs->force_clock_level)
+	if (is_support_sw_smu(adev))
+		ret = smu_force_clk_levels(&adev->smu, PP_MCLK, mask);
+	else if (adev->powerplay.pp_funcs->force_clock_level)
 		ret = amdgpu_dpm_force_clock_level(adev, PP_MCLK, mask);
 
 	if (ret)
@@ -819,7 +922,9 @@ static ssize_t amdgpu_get_pp_dpm_socclk(struct device *dev,
 	struct drm_device *ddev = dev_get_drvdata(dev);
 	struct amdgpu_device *adev = ddev->dev_private;
 
-	if (adev->powerplay.pp_funcs->print_clock_levels)
+	if (is_support_sw_smu(adev))
+		return smu_print_clk_levels(&adev->smu, PP_SOCCLK, buf);
+	else if (adev->powerplay.pp_funcs->print_clock_levels)
 		return amdgpu_dpm_print_clock_levels(adev, PP_SOCCLK, buf);
 	else
 		return snprintf(buf, PAGE_SIZE, "\n");
@@ -839,7 +944,9 @@ static ssize_t amdgpu_set_pp_dpm_socclk(struct device *dev,
 	if (ret)
 		return ret;
 
-	if (adev->powerplay.pp_funcs->force_clock_level)
+	if (is_support_sw_smu(adev))
+		ret = smu_force_clk_levels(&adev->smu, PP_SOCCLK, mask);
+	else if (adev->powerplay.pp_funcs->force_clock_level)
 		ret = amdgpu_dpm_force_clock_level(adev, PP_SOCCLK, mask);
 
 	if (ret)
@@ -855,7 +962,9 @@ static ssize_t amdgpu_get_pp_dpm_fclk(struct device *dev,
 	struct drm_device *ddev = dev_get_drvdata(dev);
 	struct amdgpu_device *adev = ddev->dev_private;
 
-	if (adev->powerplay.pp_funcs->print_clock_levels)
+	if (is_support_sw_smu(adev))
+		return smu_print_clk_levels(&adev->smu, PP_FCLK, buf);
+	else if (adev->powerplay.pp_funcs->print_clock_levels)
 		return amdgpu_dpm_print_clock_levels(adev, PP_FCLK, buf);
 	else
 		return snprintf(buf, PAGE_SIZE, "\n");
@@ -875,7 +984,9 @@ static ssize_t amdgpu_set_pp_dpm_fclk(struct device *dev,
 	if (ret)
 		return ret;
 
-	if (adev->powerplay.pp_funcs->force_clock_level)
+	if (is_support_sw_smu(adev))
+		ret = smu_force_clk_levels(&adev->smu, PP_FCLK, mask);
+	else if (adev->powerplay.pp_funcs->force_clock_level)
 		ret = amdgpu_dpm_force_clock_level(adev, PP_FCLK, mask);
 
 	if (ret)
@@ -891,7 +1002,9 @@ static ssize_t amdgpu_get_pp_dpm_dcefclk(struct device *dev,
 	struct drm_device *ddev = dev_get_drvdata(dev);
 	struct amdgpu_device *adev = ddev->dev_private;
 
-	if (adev->powerplay.pp_funcs->print_clock_levels)
+	if (is_support_sw_smu(adev))
+		return smu_print_clk_levels(&adev->smu, PP_DCEFCLK, buf);
+	else if (adev->powerplay.pp_funcs->print_clock_levels)
 		return amdgpu_dpm_print_clock_levels(adev, PP_DCEFCLK, buf);
 	else
 		return snprintf(buf, PAGE_SIZE, "\n");
@@ -911,7 +1024,9 @@ static ssize_t amdgpu_set_pp_dpm_dcefclk(struct device *dev,
 	if (ret)
 		return ret;
 
-	if (adev->powerplay.pp_funcs->force_clock_level)
+	if (is_support_sw_smu(adev))
+		ret = smu_force_clk_levels(&adev->smu, PP_DCEFCLK, mask);
+	else if (adev->powerplay.pp_funcs->force_clock_level)
 		ret = amdgpu_dpm_force_clock_level(adev, PP_DCEFCLK, mask);
 
 	if (ret)
@@ -927,7 +1042,9 @@ static ssize_t amdgpu_get_pp_dpm_pcie(struct device *dev,
 	struct drm_device *ddev = dev_get_drvdata(dev);
 	struct amdgpu_device *adev = ddev->dev_private;
 
-	if (adev->powerplay.pp_funcs->print_clock_levels)
+	if (is_support_sw_smu(adev))
+		return smu_print_clk_levels(&adev->smu, PP_PCIE, buf);
+	else if (adev->powerplay.pp_funcs->print_clock_levels)
 		return amdgpu_dpm_print_clock_levels(adev, PP_PCIE, buf);
 	else
 		return snprintf(buf, PAGE_SIZE, "\n");
@@ -947,7 +1064,9 @@ static ssize_t amdgpu_set_pp_dpm_pcie(struct device *dev,
 	if (ret)
 		return ret;
 
-	if (adev->powerplay.pp_funcs->force_clock_level)
+	if (is_support_sw_smu(adev))
+		ret = smu_force_clk_levels(&adev->smu, PP_PCIE, mask);
+	else if (adev->powerplay.pp_funcs->force_clock_level)
 		ret = amdgpu_dpm_force_clock_level(adev, PP_PCIE, mask);
 
 	if (ret)
@@ -964,7 +1083,9 @@ static ssize_t amdgpu_get_pp_sclk_od(struct device *dev,
 	struct amdgpu_device *adev = ddev->dev_private;
 	uint32_t value = 0;
 
-	if (adev->powerplay.pp_funcs->get_sclk_od)
+	if (is_support_sw_smu(adev))
+		value = smu_get_od_percentage(&(adev->smu), OD_SCLK);
+	else if (adev->powerplay.pp_funcs->get_sclk_od)
 		value = amdgpu_dpm_get_sclk_od(adev);
 
 	return snprintf(buf, PAGE_SIZE, "%d\n", value);
@@ -986,14 +1107,19 @@ static ssize_t amdgpu_set_pp_sclk_od(struct device *dev,
 		count = -EINVAL;
 		goto fail;
 	}
-	if (adev->powerplay.pp_funcs->set_sclk_od)
-		amdgpu_dpm_set_sclk_od(adev, (uint32_t)value);
 
-	if (adev->powerplay.pp_funcs->dispatch_tasks) {
-		amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_READJUST_POWER_STATE, NULL);
+	if (is_support_sw_smu(adev)) {
+		value = smu_set_od_percentage(&(adev->smu), OD_SCLK, (uint32_t)value);
 	} else {
-		adev->pm.dpm.current_ps = adev->pm.dpm.boot_ps;
-		amdgpu_pm_compute_clocks(adev);
+		if (adev->powerplay.pp_funcs->set_sclk_od)
+			amdgpu_dpm_set_sclk_od(adev, (uint32_t)value);
+
+		if (adev->powerplay.pp_funcs->dispatch_tasks) {
+			amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_READJUST_POWER_STATE, NULL);
+		} else {
+			adev->pm.dpm.current_ps = adev->pm.dpm.boot_ps;
+			amdgpu_pm_compute_clocks(adev);
+		}
 	}
 
 fail:
@@ -1008,7 +1134,9 @@ static ssize_t amdgpu_get_pp_mclk_od(struct device *dev,
 	struct amdgpu_device *adev = ddev->dev_private;
 	uint32_t value = 0;
 
-	if (adev->powerplay.pp_funcs->get_mclk_od)
+	if (is_support_sw_smu(adev))
+		value = smu_get_od_percentage(&(adev->smu), OD_MCLK);
+	else if (adev->powerplay.pp_funcs->get_mclk_od)
 		value = amdgpu_dpm_get_mclk_od(adev);
 
 	return snprintf(buf, PAGE_SIZE, "%d\n", value);
@@ -1030,14 +1158,19 @@ static ssize_t amdgpu_set_pp_mclk_od(struct device *dev,
 		count = -EINVAL;
 		goto fail;
 	}
-	if (adev->powerplay.pp_funcs->set_mclk_od)
-		amdgpu_dpm_set_mclk_od(adev, (uint32_t)value);
 
-	if (adev->powerplay.pp_funcs->dispatch_tasks) {
-		amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_READJUST_POWER_STATE, NULL);
+	if (is_support_sw_smu(adev)) {
+		value = smu_set_od_percentage(&(adev->smu), OD_MCLK, (uint32_t)value);
 	} else {
-		adev->pm.dpm.current_ps = adev->pm.dpm.boot_ps;
-		amdgpu_pm_compute_clocks(adev);
+		if (adev->powerplay.pp_funcs->set_mclk_od)
+			amdgpu_dpm_set_mclk_od(adev, (uint32_t)value);
+
+		if (adev->powerplay.pp_funcs->dispatch_tasks) {
+			amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_READJUST_POWER_STATE, NULL);
+		} else {
+			adev->pm.dpm.current_ps = adev->pm.dpm.boot_ps;
+			amdgpu_pm_compute_clocks(adev);
+		}
 	}
 
 fail:
@@ -1071,7 +1204,9 @@ static ssize_t amdgpu_get_pp_power_profile_mode(struct device *dev,
 	struct drm_device *ddev = dev_get_drvdata(dev);
 	struct amdgpu_device *adev = ddev->dev_private;
 
-	if (adev->powerplay.pp_funcs->get_power_profile_mode)
+	if (is_support_sw_smu(adev))
+		return smu_get_power_profile_mode(&adev->smu, buf);
+	else if (adev->powerplay.pp_funcs->get_power_profile_mode)
 		return amdgpu_dpm_get_power_profile_mode(adev, buf);
 
 	return snprintf(buf, PAGE_SIZE, "\n");
@@ -1121,9 +1256,10 @@ static ssize_t amdgpu_set_pp_power_profile_mode(struct device *dev,
 		}
 	}
 	parameter[parameter_size] = profile_mode;
-	if (adev->powerplay.pp_funcs->set_power_profile_mode)
+	if (is_support_sw_smu(adev))
+		ret = smu_set_power_profile_mode(&adev->smu, parameter, parameter_size);
+	else if (adev->powerplay.pp_funcs->set_power_profile_mode)
 		ret = amdgpu_dpm_set_power_profile_mode(adev, parameter, parameter_size);
-
 	if (!ret)
 		return count;
 fail:
@@ -1146,14 +1282,10 @@ static ssize_t amdgpu_get_busy_percent(struct device *dev,
 	struct amdgpu_device *adev = ddev->dev_private;
 	int r, value, size = sizeof(value);
 
-	/* sanity check PP is enabled */
-	if (!(adev->powerplay.pp_funcs &&
-	      adev->powerplay.pp_funcs->read_sensor))
-		return -EINVAL;
-
 	/* read the IP busy sensor */
 	r = amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GPU_LOAD,
 				   (void *)&value, &size);
+
 	if (r)
 		return r;
 
@@ -1247,11 +1379,6 @@ static ssize_t amdgpu_hwmon_show_temp(struct device *dev,
 	     (ddev->switch_power_state != DRM_SWITCH_POWER_ON))
 		return -EINVAL;
 
-	/* sanity check PP is enabled */
-	if (!(adev->powerplay.pp_funcs &&
-	      adev->powerplay.pp_funcs->read_sensor))
-		return -EINVAL;
-
 	/* get the temperature */
 	r = amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GPU_TEMP,
 				   (void *)&temp, &size);
@@ -1283,11 +1410,14 @@ static ssize_t amdgpu_hwmon_get_pwm1_enable(struct device *dev,
 {
 	struct amdgpu_device *adev = dev_get_drvdata(dev);
 	u32 pwm_mode = 0;
+	if (is_support_sw_smu(adev)) {
+		pwm_mode = smu_get_fan_control_mode(&adev->smu);
+	} else {
+		if (!adev->powerplay.pp_funcs->get_fan_control_mode)
+			return -EINVAL;
 
-	if (!adev->powerplay.pp_funcs->get_fan_control_mode)
-		return -EINVAL;
-
-	pwm_mode = amdgpu_dpm_get_fan_control_mode(adev);
+		pwm_mode = amdgpu_dpm_get_fan_control_mode(adev);
+	}
 
 	return sprintf(buf, "%i\n", pwm_mode);
 }
@@ -1306,14 +1436,22 @@ static ssize_t amdgpu_hwmon_set_pwm1_enable(struct device *dev,
 	     (adev->ddev->switch_power_state != DRM_SWITCH_POWER_ON))
 		return -EINVAL;
 
-	if (!adev->powerplay.pp_funcs->set_fan_control_mode)
-		return -EINVAL;
+	if (is_support_sw_smu(adev)) {
+		err = kstrtoint(buf, 10, &value);
+		if (err)
+			return err;
 
-	err = kstrtoint(buf, 10, &value);
-	if (err)
-		return err;
+		smu_set_fan_control_mode(&adev->smu, value);
+	} else {
+		if (!adev->powerplay.pp_funcs->set_fan_control_mode)
+			return -EINVAL;
+
+		err = kstrtoint(buf, 10, &value);
+		if (err)
+			return err;
 
-	amdgpu_dpm_set_fan_control_mode(adev, value);
+		amdgpu_dpm_set_fan_control_mode(adev, value);
+	}
 
 	return count;
 }
@@ -1345,8 +1483,10 @@ static ssize_t amdgpu_hwmon_set_pwm1(struct device *dev,
 	if  ((adev->flags & AMD_IS_PX) &&
 	     (adev->ddev->switch_power_state != DRM_SWITCH_POWER_ON))
 		return -EINVAL;
-
-	pwm_mode = amdgpu_dpm_get_fan_control_mode(adev);
+	if (is_support_sw_smu(adev))
+		pwm_mode = smu_get_fan_control_mode(&adev->smu);
+	else
+		pwm_mode = amdgpu_dpm_get_fan_control_mode(adev);
 	if (pwm_mode != AMD_FAN_CTRL_MANUAL) {
 		pr_info("manual fan speed control should be enabled first\n");
 		return -EINVAL;
@@ -1358,7 +1498,11 @@ static ssize_t amdgpu_hwmon_set_pwm1(struct device *dev,
 
 	value = (value * 100) / 255;
 
-	if (adev->powerplay.pp_funcs->set_fan_speed_percent) {
+	if (is_support_sw_smu(adev)) {
+		err = smu_set_fan_speed_percent(&adev->smu, value);
+		if (err)
+			return err;
+	} else if (adev->powerplay.pp_funcs->set_fan_speed_percent) {
 		err = amdgpu_dpm_set_fan_speed_percent(adev, value);
 		if (err)
 			return err;
@@ -1380,7 +1524,11 @@ static ssize_t amdgpu_hwmon_get_pwm1(struct device *dev,
 	     (adev->ddev->switch_power_state != DRM_SWITCH_POWER_ON))
 		return -EINVAL;
 
-	if (adev->powerplay.pp_funcs->get_fan_speed_percent) {
+	if (is_support_sw_smu(adev)) {
+		err = smu_get_fan_speed_percent(&adev->smu, &speed);
+		if (err)
+			return err;
+	} else if (adev->powerplay.pp_funcs->get_fan_speed_percent) {
 		err = amdgpu_dpm_get_fan_speed_percent(adev, &speed);
 		if (err)
 			return err;
@@ -1404,7 +1552,11 @@ static ssize_t amdgpu_hwmon_get_fan1_input(struct device *dev,
 	     (adev->ddev->switch_power_state != DRM_SWITCH_POWER_ON))
 		return -EINVAL;
 
-	if (adev->powerplay.pp_funcs->get_fan_speed_rpm) {
+	if (is_support_sw_smu(adev)) {
+		err = smu_get_current_rpm(&adev->smu, &speed);
+		if (err)
+			return err;
+	} else if (adev->powerplay.pp_funcs->get_fan_speed_rpm) {
 		err = amdgpu_dpm_get_fan_speed_rpm(adev, &speed);
 		if (err)
 			return err;
@@ -1422,9 +1574,6 @@ static ssize_t amdgpu_hwmon_get_fan1_min(struct device *dev,
 	u32 size = sizeof(min_rpm);
 	int r;
 
-	if (!adev->powerplay.pp_funcs->read_sensor)
-		return -EINVAL;
-
 	r = amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_MIN_FAN_RPM,
 				   (void *)&min_rpm, &size);
 	if (r)
@@ -1442,9 +1591,6 @@ static ssize_t amdgpu_hwmon_get_fan1_max(struct device *dev,
 	u32 size = sizeof(max_rpm);
 	int r;
 
-	if (!adev->powerplay.pp_funcs->read_sensor)
-		return -EINVAL;
-
 	r = amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_MAX_FAN_RPM,
 				   (void *)&max_rpm, &size);
 	if (r)
@@ -1466,7 +1612,11 @@ static ssize_t amdgpu_hwmon_get_fan1_target(struct device *dev,
 	     (adev->ddev->switch_power_state != DRM_SWITCH_POWER_ON))
 		return -EINVAL;
 
-	if (adev->powerplay.pp_funcs->get_fan_speed_rpm) {
+	if (is_support_sw_smu(adev)) {
+		err = smu_get_current_rpm(&adev->smu, &rpm);
+		if (err)
+			return err;
+	} else if (adev->powerplay.pp_funcs->get_fan_speed_rpm) {
 		err = amdgpu_dpm_get_fan_speed_rpm(adev, &rpm);
 		if (err)
 			return err;
@@ -1484,7 +1634,11 @@ static ssize_t amdgpu_hwmon_set_fan1_target(struct device *dev,
 	u32 value;
 	u32 pwm_mode;
 
-	pwm_mode = amdgpu_dpm_get_fan_control_mode(adev);
+	if (is_support_sw_smu(adev))
+		pwm_mode = smu_get_fan_control_mode(&adev->smu);
+	else
+		pwm_mode = amdgpu_dpm_get_fan_control_mode(adev);
+
 	if (pwm_mode != AMD_FAN_CTRL_MANUAL)
 		return -ENODATA;
 
@@ -1497,7 +1651,11 @@ static ssize_t amdgpu_hwmon_set_fan1_target(struct device *dev,
 	if (err)
 		return err;
 
-	if (adev->powerplay.pp_funcs->set_fan_speed_rpm) {
+	if (is_support_sw_smu(adev)) {
+		err = smu_set_fan_speed_rpm(&adev->smu, value);
+		if (err)
+			return err;
+	} else if (adev->powerplay.pp_funcs->set_fan_speed_rpm) {
 		err = amdgpu_dpm_set_fan_speed_rpm(adev, value);
 		if (err)
 			return err;
@@ -1513,11 +1671,14 @@ static ssize_t amdgpu_hwmon_get_fan1_enable(struct device *dev,
 	struct amdgpu_device *adev = dev_get_drvdata(dev);
 	u32 pwm_mode = 0;
 
-	if (!adev->powerplay.pp_funcs->get_fan_control_mode)
-		return -EINVAL;
-
-	pwm_mode = amdgpu_dpm_get_fan_control_mode(adev);
+	if (is_support_sw_smu(adev)) {
+		pwm_mode = smu_get_fan_control_mode(&adev->smu);
+	} else {
+		if (!adev->powerplay.pp_funcs->get_fan_control_mode)
+			return -EINVAL;
 
+		pwm_mode = amdgpu_dpm_get_fan_control_mode(adev);
+	}
 	return sprintf(buf, "%i\n", pwm_mode == AMD_FAN_CTRL_AUTO ? 0 : 1);
 }
 
@@ -1536,8 +1697,6 @@ static ssize_t amdgpu_hwmon_set_fan1_enable(struct device *dev,
 	     (adev->ddev->switch_power_state != DRM_SWITCH_POWER_ON))
 		return -EINVAL;
 
-	if (!adev->powerplay.pp_funcs->set_fan_control_mode)
-		return -EINVAL;
 
 	err = kstrtoint(buf, 10, &value);
 	if (err)
@@ -1550,7 +1709,13 @@ static ssize_t amdgpu_hwmon_set_fan1_enable(struct device *dev,
 	else
 		return -EINVAL;
 
-	amdgpu_dpm_set_fan_control_mode(adev, pwm_mode);
+	if (is_support_sw_smu(adev)) {
+		smu_set_fan_control_mode(&adev->smu, pwm_mode);
+	} else {
+		if (!adev->powerplay.pp_funcs->set_fan_control_mode)
+			return -EINVAL;
+		amdgpu_dpm_set_fan_control_mode(adev, pwm_mode);
+	}
 
 	return count;
 }
@@ -1569,11 +1734,6 @@ static ssize_t amdgpu_hwmon_show_vddgfx(struct device *dev,
 	     (ddev->switch_power_state != DRM_SWITCH_POWER_ON))
 		return -EINVAL;
 
-	/* sanity check PP is enabled */
-	if (!(adev->powerplay.pp_funcs &&
-	      adev->powerplay.pp_funcs->read_sensor))
-	      return -EINVAL;
-
 	/* get the voltage */
 	r = amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_VDDGFX,
 				   (void *)&vddgfx, &size);
@@ -1608,11 +1768,6 @@ static ssize_t amdgpu_hwmon_show_vddnb(struct device *dev,
 	     (ddev->switch_power_state != DRM_SWITCH_POWER_ON))
 		return -EINVAL;
 
-	/* sanity check PP is enabled */
-	if (!(adev->powerplay.pp_funcs &&
-	      adev->powerplay.pp_funcs->read_sensor))
-	      return -EINVAL;
-
 	/* get the voltage */
 	r = amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_VDDNB,
 				   (void *)&vddnb, &size);
@@ -1644,11 +1799,6 @@ static ssize_t amdgpu_hwmon_show_power_avg(struct device *dev,
 	     (ddev->switch_power_state != DRM_SWITCH_POWER_ON))
 		return -EINVAL;
 
-	/* sanity check PP is enabled */
-	if (!(adev->powerplay.pp_funcs &&
-	      adev->powerplay.pp_funcs->read_sensor))
-	      return -EINVAL;
-
 	/* get the voltage */
 	r = amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GPU_POWER,
 				   (void *)&query, &size);
@@ -1675,7 +1825,10 @@ static ssize_t amdgpu_hwmon_show_power_cap_max(struct device *dev,
 	struct amdgpu_device *adev = dev_get_drvdata(dev);
 	uint32_t limit = 0;
 
-	if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->get_power_limit) {
+	if (is_support_sw_smu(adev)) {
+		smu_get_power_limit(&adev->smu, &limit, true);
+		return snprintf(buf, PAGE_SIZE, "%u\n", limit * 1000000);
+	} else if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->get_power_limit) {
 		adev->powerplay.pp_funcs->get_power_limit(adev->powerplay.pp_handle, &limit, true);
 		return snprintf(buf, PAGE_SIZE, "%u\n", limit * 1000000);
 	} else {
@@ -1690,7 +1843,10 @@ static ssize_t amdgpu_hwmon_show_power_cap(struct device *dev,
 	struct amdgpu_device *adev = dev_get_drvdata(dev);
 	uint32_t limit = 0;
 
-	if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->get_power_limit) {
+	if (is_support_sw_smu(adev)) {
+		smu_get_power_limit(&adev->smu, &limit, false);
+		return snprintf(buf, PAGE_SIZE, "%u\n", limit * 1000000);
+	} else if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->get_power_limit) {
 		adev->powerplay.pp_funcs->get_power_limit(adev->powerplay.pp_handle, &limit, false);
 		return snprintf(buf, PAGE_SIZE, "%u\n", limit * 1000000);
 	} else {
@@ -1713,7 +1869,9 @@ static ssize_t amdgpu_hwmon_set_power_cap(struct device *dev,
 		return err;
 
 	value = value / 1000000; /* convert to Watt */
-	if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->set_power_limit) {
+	if (is_support_sw_smu(adev)) {
+		adev->smu.funcs->set_power_limit(&adev->smu, value);
+	} else if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->set_power_limit) {
 		err = adev->powerplay.pp_funcs->set_power_limit(adev->powerplay.pp_handle, value);
 		if (err)
 			return err;
@@ -1967,18 +2125,20 @@ static umode_t hwmon_attributes_visible(struct kobject *kobj,
 	     attr == &sensor_dev_attr_fan1_enable.dev_attr.attr))
 		return 0;
 
-	/* mask fan attributes if we have no bindings for this asic to expose */
-	if ((!adev->powerplay.pp_funcs->get_fan_speed_percent &&
-	     attr == &sensor_dev_attr_pwm1.dev_attr.attr) || /* can't query fan */
-	    (!adev->powerplay.pp_funcs->get_fan_control_mode &&
-	     attr == &sensor_dev_attr_pwm1_enable.dev_attr.attr)) /* can't query state */
-		effective_mode &= ~S_IRUGO;
+	if (!is_support_sw_smu(adev)) {
+		/* mask fan attributes if we have no bindings for this asic to expose */
+		if ((!adev->powerplay.pp_funcs->get_fan_speed_percent &&
+		     attr == &sensor_dev_attr_pwm1.dev_attr.attr) || /* can't query fan */
+		    (!adev->powerplay.pp_funcs->get_fan_control_mode &&
+		     attr == &sensor_dev_attr_pwm1_enable.dev_attr.attr)) /* can't query state */
+			effective_mode &= ~S_IRUGO;
 
-	if ((!adev->powerplay.pp_funcs->set_fan_speed_percent &&
-	     attr == &sensor_dev_attr_pwm1.dev_attr.attr) || /* can't manage fan */
-	    (!adev->powerplay.pp_funcs->set_fan_control_mode &&
-	     attr == &sensor_dev_attr_pwm1_enable.dev_attr.attr)) /* can't manage state */
-		effective_mode &= ~S_IWUSR;
+		if ((!adev->powerplay.pp_funcs->set_fan_speed_percent &&
+		     attr == &sensor_dev_attr_pwm1.dev_attr.attr) || /* can't manage fan */
+		    (!adev->powerplay.pp_funcs->set_fan_control_mode &&
+		     attr == &sensor_dev_attr_pwm1_enable.dev_attr.attr)) /* can't manage state */
+			effective_mode &= ~S_IWUSR;
+	}
 
 	if ((adev->flags & AMD_IS_APU) &&
 	    (attr == &sensor_dev_attr_power1_average.dev_attr.attr ||
@@ -1987,20 +2147,22 @@ static umode_t hwmon_attributes_visible(struct kobject *kobj,
 	     attr == &sensor_dev_attr_power1_cap.dev_attr.attr))
 		return 0;
 
-	/* hide max/min values if we can't both query and manage the fan */
-	if ((!adev->powerplay.pp_funcs->set_fan_speed_percent &&
-	     !adev->powerplay.pp_funcs->get_fan_speed_percent) &&
-	     (!adev->powerplay.pp_funcs->set_fan_speed_rpm &&
-	     !adev->powerplay.pp_funcs->get_fan_speed_rpm) &&
-	    (attr == &sensor_dev_attr_pwm1_max.dev_attr.attr ||
-	     attr == &sensor_dev_attr_pwm1_min.dev_attr.attr))
-		return 0;
+	if (!is_support_sw_smu(adev)) {
+		/* hide max/min values if we can't both query and manage the fan */
+		if ((!adev->powerplay.pp_funcs->set_fan_speed_percent &&
+		     !adev->powerplay.pp_funcs->get_fan_speed_percent) &&
+		     (!adev->powerplay.pp_funcs->set_fan_speed_rpm &&
+		     !adev->powerplay.pp_funcs->get_fan_speed_rpm) &&
+		    (attr == &sensor_dev_attr_pwm1_max.dev_attr.attr ||
+		     attr == &sensor_dev_attr_pwm1_min.dev_attr.attr))
+			return 0;
 
-	if ((!adev->powerplay.pp_funcs->set_fan_speed_rpm &&
-	     !adev->powerplay.pp_funcs->get_fan_speed_rpm) &&
-	    (attr == &sensor_dev_attr_fan1_max.dev_attr.attr ||
-	     attr == &sensor_dev_attr_fan1_min.dev_attr.attr))
-		return 0;
+		if ((!adev->powerplay.pp_funcs->set_fan_speed_rpm &&
+		     !adev->powerplay.pp_funcs->get_fan_speed_rpm) &&
+		    (attr == &sensor_dev_attr_fan1_max.dev_attr.attr ||
+		     attr == &sensor_dev_attr_fan1_min.dev_attr.attr))
+			return 0;
+	}
 
 	/* only APUs have vddnb */
 	if (!(adev->flags & AMD_IS_APU) &&
@@ -2039,9 +2201,7 @@ void amdgpu_dpm_thermal_work_handler(struct work_struct *work)
 	if (!adev->pm.dpm_enabled)
 		return;
 
-	if (adev->powerplay.pp_funcs &&
-	    adev->powerplay.pp_funcs->read_sensor &&
-	    !amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GPU_TEMP,
+	if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GPU_TEMP,
 				    (void *)&temp, &size)) {
 		if (temp < adev->pm.dpm.thermal.min_temp)
 			/* switch back the user state */
@@ -2267,7 +2427,13 @@ static void amdgpu_dpm_change_power_state_locked(struct amdgpu_device *adev)
 
 void amdgpu_dpm_enable_uvd(struct amdgpu_device *adev, bool enable)
 {
-	if (adev->powerplay.pp_funcs->set_powergating_by_smu) {
+	int ret = 0;
+	if (is_support_sw_smu(adev)) {
+	    ret = smu_dpm_set_power_gate(&adev->smu, AMD_IP_BLOCK_TYPE_UVD, enable);
+	    if (ret)
+		DRM_ERROR("[SW SMU]: dpm enable uvd failed, state = %s, ret = %d. \n",
+			  enable ? "true" : "false", ret);
+	} else if (adev->powerplay.pp_funcs->set_powergating_by_smu) {
 		/* enable/disable UVD */
 		mutex_lock(&adev->pm.mutex);
 		amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_UVD, !enable);
@@ -2288,7 +2454,13 @@ void amdgpu_dpm_enable_uvd(struct amdgpu_device *adev, bool enable)
 
 void amdgpu_dpm_enable_vce(struct amdgpu_device *adev, bool enable)
 {
-	if (adev->powerplay.pp_funcs->set_powergating_by_smu) {
+	int ret = 0;
+	if (is_support_sw_smu(adev)) {
+	    ret = smu_dpm_set_power_gate(&adev->smu, AMD_IP_BLOCK_TYPE_VCE, enable);
+	    if (ret)
+		DRM_ERROR("[SW SMU]: dpm enable vce failed, state = %s, ret = %d. \n",
+			  enable ? "true" : "false", ret);
+	} else if (adev->powerplay.pp_funcs->set_powergating_by_smu) {
 		/* enable/disable VCE */
 		mutex_lock(&adev->pm.mutex);
 		amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_VCE, !enable);
@@ -2413,7 +2585,8 @@ int amdgpu_pm_sysfs_init(struct amdgpu_device *adev)
 				"pp_power_profile_mode\n");
 		return ret;
 	}
-	if (hwmgr->od_enabled) {
+	if ((is_support_sw_smu(adev) && adev->smu.od_enabled) ||
+	    (!is_support_sw_smu(adev) && hwmgr->od_enabled)) {
 		ret = device_create_file(adev->dev,
 				&dev_attr_pp_od_clk_voltage);
 		if (ret) {
@@ -2489,7 +2662,8 @@ void amdgpu_pm_sysfs_fini(struct amdgpu_device *adev)
 	device_remove_file(adev->dev, &dev_attr_pp_mclk_od);
 	device_remove_file(adev->dev,
 			&dev_attr_pp_power_profile_mode);
-	if (hwmgr->od_enabled)
+	if ((is_support_sw_smu(adev) && adev->smu.od_enabled) ||
+	    (!is_support_sw_smu(adev) && hwmgr->od_enabled))
 		device_remove_file(adev->dev,
 				&dev_attr_pp_od_clk_voltage);
 	device_remove_file(adev->dev, &dev_attr_gpu_busy_percent);
@@ -2516,28 +2690,38 @@ void amdgpu_pm_compute_clocks(struct amdgpu_device *adev)
 			amdgpu_fence_wait_empty(ring);
 	}
 
-	if (adev->powerplay.pp_funcs->dispatch_tasks) {
-		if (!amdgpu_device_has_dc_support(adev)) {
+	if (is_support_sw_smu(adev)) {
+		struct smu_context *smu = &adev->smu;
+		struct smu_dpm_context *smu_dpm = &adev->smu.smu_dpm;
+		mutex_lock(&(smu->mutex));
+		smu_handle_task(&adev->smu,
+				smu_dpm->dpm_level,
+				AMD_PP_TASK_DISPLAY_CONFIG_CHANGE);
+		mutex_unlock(&(smu->mutex));
+	} else {
+		if (adev->powerplay.pp_funcs->dispatch_tasks) {
+			if (!amdgpu_device_has_dc_support(adev)) {
+				mutex_lock(&adev->pm.mutex);
+				amdgpu_dpm_get_active_displays(adev);
+				adev->pm.pm_display_cfg.num_display = adev->pm.dpm.new_active_crtc_count;
+				adev->pm.pm_display_cfg.vrefresh = amdgpu_dpm_get_vrefresh(adev);
+				adev->pm.pm_display_cfg.min_vblank_time = amdgpu_dpm_get_vblank_time(adev);
+				/* we have issues with mclk switching with refresh rates over 120 hz on the non-DC code. */
+				if (adev->pm.pm_display_cfg.vrefresh > 120)
+					adev->pm.pm_display_cfg.min_vblank_time = 0;
+				if (adev->powerplay.pp_funcs->display_configuration_change)
+					adev->powerplay.pp_funcs->display_configuration_change(
+									adev->powerplay.pp_handle,
+									&adev->pm.pm_display_cfg);
+				mutex_unlock(&adev->pm.mutex);
+			}
+			amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_DISPLAY_CONFIG_CHANGE, NULL);
+		} else {
 			mutex_lock(&adev->pm.mutex);
 			amdgpu_dpm_get_active_displays(adev);
-			adev->pm.pm_display_cfg.num_display = adev->pm.dpm.new_active_crtc_count;
-			adev->pm.pm_display_cfg.vrefresh = amdgpu_dpm_get_vrefresh(adev);
-			adev->pm.pm_display_cfg.min_vblank_time = amdgpu_dpm_get_vblank_time(adev);
-			/* we have issues with mclk switching with refresh rates over 120 hz on the non-DC code. */
-			if (adev->pm.pm_display_cfg.vrefresh > 120)
-				adev->pm.pm_display_cfg.min_vblank_time = 0;
-			if (adev->powerplay.pp_funcs->display_configuration_change)
-				adev->powerplay.pp_funcs->display_configuration_change(
-								adev->powerplay.pp_handle,
-								&adev->pm.pm_display_cfg);
+			amdgpu_dpm_change_power_state_locked(adev);
 			mutex_unlock(&adev->pm.mutex);
 		}
-		amdgpu_dpm_dispatch_task(adev, AMD_PP_TASK_DISPLAY_CONFIG_CHANGE, NULL);
-	} else {
-		mutex_lock(&adev->pm.mutex);
-		amdgpu_dpm_get_active_displays(adev);
-		amdgpu_dpm_change_power_state_locked(adev);
-		mutex_unlock(&adev->pm.mutex);
 	}
 }
 
@@ -2553,11 +2737,6 @@ static int amdgpu_debugfs_pm_info_pp(struct seq_file *m, struct amdgpu_device *a
 	uint32_t query = 0;
 	int size;
 
-	/* sanity check PP is enabled */
-	if (!(adev->powerplay.pp_funcs &&
-	      adev->powerplay.pp_funcs->read_sensor))
-	      return -EINVAL;
-
 	/* GPU Clocks */
 	size = sizeof(value);
 	seq_printf(m, "GFX Clocks and Power:\n");
@@ -2649,7 +2828,7 @@ static int amdgpu_debugfs_pm_info(struct seq_file *m, void *data)
 	if  ((adev->flags & AMD_IS_PX) &&
 	     (ddev->switch_power_state != DRM_SWITCH_POWER_ON)) {
 		seq_printf(m, "PX asic powered off\n");
-	} else if (adev->powerplay.pp_funcs->debugfs_print_current_performance_level) {
+	} else if (!is_support_sw_smu(adev) && adev->powerplay.pp_funcs->debugfs_print_current_performance_level) {
 		mutex_lock(&adev->pm.mutex);
 		if (adev->powerplay.pp_funcs->debugfs_print_current_performance_level)
 			adev->powerplay.pp_funcs->debugfs_print_current_performance_level(adev, m);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
index 3091488cd8cc..905cce1814f3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
@@ -120,6 +120,7 @@ psp_cmd_submit_buf(struct psp_context *psp,
 {
 	int ret;
 	int index;
+	int timeout = 2000;
 
 	memset(psp->cmd_buf_mem, 0, PSP_CMD_BUFFER_SIZE);
 
@@ -133,8 +134,11 @@ psp_cmd_submit_buf(struct psp_context *psp,
 		return ret;
 	}
 
-	while (*((unsigned int *)psp->fence_buf) != index)
+	while (*((unsigned int *)psp->fence_buf) != index) {
+		if (--timeout == 0)
+			break;
 		msleep(1);
+	}
 
 	/* In some cases, psp response status is not 0 even there is no
 	 * problem while the command is submitted. Some version of PSP FW
@@ -143,12 +147,14 @@ psp_cmd_submit_buf(struct psp_context *psp,
 	 * during psp initialization to avoid breaking hw_init and it doesn't
 	 * return -EINVAL.
 	 */
-	if (psp->cmd_buf_mem->resp.status) {
+	if (psp->cmd_buf_mem->resp.status || !timeout) {
 		if (ucode)
 			DRM_WARN("failed to load ucode id (%d) ",
 				  ucode->ucode_id);
 		DRM_WARN("psp command failed and response status is (%d)\n",
 			  psp->cmd_buf_mem->resp.status);
+		if (!timeout)
+			return -EINVAL;
 	}
 
 	/* get xGMI session id from response buffer */
@@ -181,13 +187,13 @@ static int psp_tmr_init(struct psp_context *psp)
 	int ret;
 
 	/*
-	 * Allocate 3M memory aligned to 1M from Frame Buffer (local
-	 * physical).
+	 * According to HW engineer, they prefer the TMR address be "naturally
+	 * aligned" , e.g. the start address be an integer divide of TMR size.
 	 *
 	 * Note: this memory need be reserved till the driver
 	 * uninitializes.
 	 */
-	ret = amdgpu_bo_create_kernel(psp->adev, PSP_TMR_SIZE, 0x100000,
+	ret = amdgpu_bo_create_kernel(psp->adev, PSP_TMR_SIZE, PSP_TMR_SIZE,
 				      AMDGPU_GEM_DOMAIN_VRAM,
 				      &psp->tmr_bo, &psp->tmr_mc_addr, &psp->tmr_buf);
 
@@ -466,6 +472,206 @@ static int psp_xgmi_initialize(struct psp_context *psp)
 	return ret;
 }
 
+// ras begin
+static void psp_prep_ras_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
+		uint64_t ras_ta_mc, uint64_t ras_mc_shared,
+		uint32_t ras_ta_size, uint32_t shared_size)
+{
+	cmd->cmd_id = GFX_CMD_ID_LOAD_TA;
+	cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(ras_ta_mc);
+	cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(ras_ta_mc);
+	cmd->cmd.cmd_load_ta.app_len = ras_ta_size;
+
+	cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = lower_32_bits(ras_mc_shared);
+	cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = upper_32_bits(ras_mc_shared);
+	cmd->cmd.cmd_load_ta.cmd_buf_len = shared_size;
+}
+
+static int psp_ras_init_shared_buf(struct psp_context *psp)
+{
+	int ret;
+
+	/*
+	 * Allocate 16k memory aligned to 4k from Frame Buffer (local
+	 * physical) for ras ta <-> Driver
+	 */
+	ret = amdgpu_bo_create_kernel(psp->adev, PSP_RAS_SHARED_MEM_SIZE,
+			PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
+			&psp->ras.ras_shared_bo,
+			&psp->ras.ras_shared_mc_addr,
+			&psp->ras.ras_shared_buf);
+
+	return ret;
+}
+
+static int psp_ras_load(struct psp_context *psp)
+{
+	int ret;
+	struct psp_gfx_cmd_resp *cmd;
+
+	/*
+	 * TODO: bypass the loading in sriov for now
+	 */
+	if (amdgpu_sriov_vf(psp->adev))
+		return 0;
+
+	cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
+	if (!cmd)
+		return -ENOMEM;
+
+	memset(psp->fw_pri_buf, 0, PSP_1_MEG);
+	memcpy(psp->fw_pri_buf, psp->ta_ras_start_addr, psp->ta_ras_ucode_size);
+
+	psp_prep_ras_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr,
+			psp->ras.ras_shared_mc_addr,
+			psp->ta_ras_ucode_size, PSP_RAS_SHARED_MEM_SIZE);
+
+	ret = psp_cmd_submit_buf(psp, NULL, cmd,
+			psp->fence_buf_mc_addr);
+
+	if (!ret) {
+		psp->ras.ras_initialized = 1;
+		psp->ras.session_id = cmd->resp.session_id;
+	}
+
+	kfree(cmd);
+
+	return ret;
+}
+
+static void psp_prep_ras_ta_unload_cmd_buf(struct psp_gfx_cmd_resp *cmd,
+						uint32_t ras_session_id)
+{
+	cmd->cmd_id = GFX_CMD_ID_UNLOAD_TA;
+	cmd->cmd.cmd_unload_ta.session_id = ras_session_id;
+}
+
+static int psp_ras_unload(struct psp_context *psp)
+{
+	int ret;
+	struct psp_gfx_cmd_resp *cmd;
+
+	/*
+	 * TODO: bypass the unloading in sriov for now
+	 */
+	if (amdgpu_sriov_vf(psp->adev))
+		return 0;
+
+	cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
+	if (!cmd)
+		return -ENOMEM;
+
+	psp_prep_ras_ta_unload_cmd_buf(cmd, psp->ras.session_id);
+
+	ret = psp_cmd_submit_buf(psp, NULL, cmd,
+			psp->fence_buf_mc_addr);
+
+	kfree(cmd);
+
+	return ret;
+}
+
+static void psp_prep_ras_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd,
+		uint32_t ta_cmd_id,
+		uint32_t ras_session_id)
+{
+	cmd->cmd_id = GFX_CMD_ID_INVOKE_CMD;
+	cmd->cmd.cmd_invoke_cmd.session_id = ras_session_id;
+	cmd->cmd.cmd_invoke_cmd.ta_cmd_id = ta_cmd_id;
+	/* Note: cmd_invoke_cmd.buf is not used for now */
+}
+
+int psp_ras_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
+{
+	int ret;
+	struct psp_gfx_cmd_resp *cmd;
+
+	/*
+	 * TODO: bypass the loading in sriov for now
+	 */
+	if (amdgpu_sriov_vf(psp->adev))
+		return 0;
+
+	cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
+	if (!cmd)
+		return -ENOMEM;
+
+	psp_prep_ras_ta_invoke_cmd_buf(cmd, ta_cmd_id,
+			psp->ras.session_id);
+
+	ret = psp_cmd_submit_buf(psp, NULL, cmd,
+			psp->fence_buf_mc_addr);
+
+	kfree(cmd);
+
+	return ret;
+}
+
+int psp_ras_enable_features(struct psp_context *psp,
+		union ta_ras_cmd_input *info, bool enable)
+{
+	struct ta_ras_shared_memory *ras_cmd;
+	int ret;
+
+	if (!psp->ras.ras_initialized)
+		return -EINVAL;
+
+	ras_cmd = (struct ta_ras_shared_memory *)psp->ras.ras_shared_buf;
+	memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
+
+	if (enable)
+		ras_cmd->cmd_id = TA_RAS_COMMAND__ENABLE_FEATURES;
+	else
+		ras_cmd->cmd_id = TA_RAS_COMMAND__DISABLE_FEATURES;
+
+	ras_cmd->ras_in_message = *info;
+
+	ret = psp_ras_invoke(psp, ras_cmd->cmd_id);
+	if (ret)
+		return -EINVAL;
+
+	return ras_cmd->ras_status;
+}
+
+static int psp_ras_terminate(struct psp_context *psp)
+{
+	int ret;
+
+	if (!psp->ras.ras_initialized)
+		return 0;
+
+	ret = psp_ras_unload(psp);
+	if (ret)
+		return ret;
+
+	psp->ras.ras_initialized = 0;
+
+	/* free ras shared memory */
+	amdgpu_bo_free_kernel(&psp->ras.ras_shared_bo,
+			&psp->ras.ras_shared_mc_addr,
+			&psp->ras.ras_shared_buf);
+
+	return 0;
+}
+
+static int psp_ras_initialize(struct psp_context *psp)
+{
+	int ret;
+
+	if (!psp->ras.ras_initialized) {
+		ret = psp_ras_init_shared_buf(psp);
+		if (ret)
+			return ret;
+	}
+
+	ret = psp_ras_load(psp);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+// ras end
+
 static int psp_hw_start(struct psp_context *psp)
 {
 	struct amdgpu_device *adev = psp->adev;
@@ -473,25 +679,35 @@ static int psp_hw_start(struct psp_context *psp)
 
 	if (!amdgpu_sriov_vf(adev) || !adev->in_gpu_reset) {
 		ret = psp_bootloader_load_sysdrv(psp);
-		if (ret)
+		if (ret) {
+			DRM_ERROR("PSP load sysdrv failed!\n");
 			return ret;
+		}
 
 		ret = psp_bootloader_load_sos(psp);
-		if (ret)
+		if (ret) {
+			DRM_ERROR("PSP load sos failed!\n");
 			return ret;
+		}
 	}
 
 	ret = psp_ring_create(psp, PSP_RING_TYPE__KM);
-	if (ret)
+	if (ret) {
+		DRM_ERROR("PSP create ring failed!\n");
 		return ret;
+	}
 
 	ret = psp_tmr_load(psp);
-	if (ret)
+	if (ret) {
+		DRM_ERROR("PSP load tmr failed!\n");
 		return ret;
+	}
 
 	ret = psp_asd_load(psp);
-	if (ret)
+	if (ret) {
+		DRM_ERROR("PSP load asd failed!\n");
 		return ret;
+	}
 
 	if (adev->gmc.xgmi.num_physical_nodes > 1) {
 		ret = psp_xgmi_initialize(psp);
@@ -502,6 +718,15 @@ static int psp_hw_start(struct psp_context *psp)
 			dev_err(psp->adev->dev,
 				"XGMI: Failed to initialize XGMI session\n");
 	}
+
+
+	if (psp->adev->psp.ta_fw) {
+		ret = psp_ras_initialize(psp);
+		if (ret)
+			dev_err(psp->adev->dev,
+					"RAS: Failed to initialize RAS\n");
+	}
+
 	return 0;
 }
 
@@ -665,53 +890,52 @@ static int psp_load_fw(struct amdgpu_device *adev)
 					&psp->fence_buf_mc_addr,
 					&psp->fence_buf);
 	if (ret)
-		goto failed_mem2;
+		goto failed;
 
 	ret = amdgpu_bo_create_kernel(adev, PSP_CMD_BUFFER_SIZE, PAGE_SIZE,
 				      AMDGPU_GEM_DOMAIN_VRAM,
 				      &psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
 				      (void **)&psp->cmd_buf_mem);
 	if (ret)
-		goto failed_mem1;
+		goto failed;
 
 	memset(psp->fence_buf, 0, PSP_FENCE_BUFFER_SIZE);
 
 	ret = psp_ring_init(psp, PSP_RING_TYPE__KM);
-	if (ret)
-		goto failed_mem;
+	if (ret) {
+		DRM_ERROR("PSP ring init failed!\n");
+		goto failed;
+	}
 
 	ret = psp_tmr_init(psp);
-	if (ret)
-		goto failed_mem;
+	if (ret) {
+		DRM_ERROR("PSP tmr init failed!\n");
+		goto failed;
+	}
 
 	ret = psp_asd_init(psp);
-	if (ret)
-		goto failed_mem;
+	if (ret) {
+		DRM_ERROR("PSP asd init failed!\n");
+		goto failed;
+	}
 
 skip_memalloc:
 	ret = psp_hw_start(psp);
 	if (ret)
-		goto failed_mem;
+		goto failed;
 
 	ret = psp_np_fw_load(psp);
 	if (ret)
-		goto failed_mem;
+		goto failed;
 
 	return 0;
 
-failed_mem:
-	amdgpu_bo_free_kernel(&psp->cmd_buf_bo,
-			      &psp->cmd_buf_mc_addr,
-			      (void **)&psp->cmd_buf_mem);
-failed_mem1:
-	amdgpu_bo_free_kernel(&psp->fence_buf_bo,
-			      &psp->fence_buf_mc_addr, &psp->fence_buf);
-failed_mem2:
-	amdgpu_bo_free_kernel(&psp->fw_pri_bo,
-			      &psp->fw_pri_mc_addr, &psp->fw_pri_buf);
 failed:
-	kfree(psp->cmd);
-	psp->cmd = NULL;
+	/*
+	 * all cleanup jobs (xgmi terminate, ras terminate,
+	 * ring destroy, cmd/fence/fw buffers destory,
+	 * psp->cmd destory) are delayed to psp_hw_fini
+	 */
 	return ret;
 }
 
@@ -753,6 +977,9 @@ static int psp_hw_fini(void *handle)
 	    psp->xgmi_context.initialized == 1)
                 psp_xgmi_terminate(psp);
 
+	if (psp->adev->psp.ta_fw)
+		psp_ras_terminate(psp);
+
 	psp_ring_destroy(psp, PSP_RING_TYPE__KM);
 
 	amdgpu_bo_free_kernel(&psp->tmr_bo, &psp->tmr_mc_addr, &psp->tmr_buf);
@@ -786,6 +1013,14 @@ static int psp_suspend(void *handle)
 		}
 	}
 
+	if (psp->adev->psp.ta_fw) {
+		ret = psp_ras_terminate(psp);
+		if (ret) {
+			DRM_ERROR("Failed to terminate ras ta\n");
+			return ret;
+		}
+	}
+
 	ret = psp_ring_stop(psp, PSP_RING_TYPE__KM);
 	if (ret) {
 		DRM_ERROR("PSP ring stop failed\n");
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
index 2ef98cc755d6..cde113f07c96 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
@@ -28,11 +28,13 @@
 #include "amdgpu.h"
 #include "psp_gfx_if.h"
 #include "ta_xgmi_if.h"
+#include "ta_ras_if.h"
 
 #define PSP_FENCE_BUFFER_SIZE	0x1000
 #define PSP_CMD_BUFFER_SIZE	0x1000
 #define PSP_ASD_SHARED_MEM_SIZE 0x4000
 #define PSP_XGMI_SHARED_MEM_SIZE 0x4000
+#define PSP_RAS_SHARED_MEM_SIZE 0x4000
 #define PSP_1_MEG		0x100000
 #define PSP_TMR_SIZE	0x400000
 
@@ -88,6 +90,9 @@ struct psp_funcs
 	int (*xgmi_set_topology_info)(struct psp_context *psp, int number_devices,
 				      struct psp_xgmi_topology_info *topology);
 	bool (*support_vmr_ring)(struct psp_context *psp);
+	int (*ras_trigger_error)(struct psp_context *psp,
+			struct ta_ras_trigger_error_input *info);
+	int (*ras_cure_posion)(struct psp_context *psp, uint64_t *mode_ptr);
 };
 
 struct psp_xgmi_context {
@@ -98,6 +103,16 @@ struct psp_xgmi_context {
 	void                            *xgmi_shared_buf;
 };
 
+struct psp_ras_context {
+	/*ras fw*/
+	bool			ras_initialized;
+	uint32_t		session_id;
+	struct amdgpu_bo	*ras_shared_bo;
+	uint64_t		ras_shared_mc_addr;
+	void			*ras_shared_buf;
+	struct amdgpu_ras	*ras;
+};
+
 struct psp_context
 {
 	struct amdgpu_device            *adev;
@@ -150,10 +165,15 @@ struct psp_context
 
 	/* xgmi ta firmware and buffer */
 	const struct firmware		*ta_fw;
+	uint32_t			ta_fw_version;
 	uint32_t			ta_xgmi_ucode_version;
 	uint32_t			ta_xgmi_ucode_size;
 	uint8_t				*ta_xgmi_start_addr;
+	uint32_t			ta_ras_ucode_version;
+	uint32_t			ta_ras_ucode_size;
+	uint8_t				*ta_ras_start_addr;
 	struct psp_xgmi_context		xgmi_context;
+	struct psp_ras_context		ras;
 };
 
 struct amdgpu_psp_funcs {
@@ -207,6 +227,13 @@ struct psp_xgmi_topology_info {
 
 #define amdgpu_psp_check_fw_loading_status(adev, i) (adev)->firmware.funcs->check_fw_loading_status((adev), (i))
 
+#define psp_ras_trigger_error(psp, info) \
+	((psp)->funcs->ras_trigger_error ? \
+	(psp)->funcs->ras_trigger_error((psp), (info)) : -EINVAL)
+#define psp_ras_cure_posion(psp, addr) \
+	((psp)->funcs->ras_cure_posion ? \
+	(psp)->funcs->ras_cure_posion(psp, (addr)) : -EINVAL)
+
 extern const struct amd_ip_funcs psp_ip_funcs;
 
 extern const struct amdgpu_ip_block_version psp_v3_1_ip_block;
@@ -217,6 +244,11 @@ extern const struct amdgpu_ip_block_version psp_v10_0_ip_block;
 
 int psp_gpu_reset(struct amdgpu_device *adev);
 int psp_xgmi_invoke(struct psp_context *psp, uint32_t ta_cmd_id);
+
+int psp_ras_invoke(struct psp_context *psp, uint32_t ta_cmd_id);
+int psp_ras_enable_features(struct psp_context *psp,
+		union ta_ras_cmd_input *info, bool enable);
+
 extern const struct amdgpu_ip_block_version psp_v11_0_ip_block;
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
new file mode 100644
index 000000000000..22bd21efe6b1
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
@@ -0,0 +1,1482 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ *
+ */
+#include <linux/debugfs.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include "amdgpu.h"
+#include "amdgpu_ras.h"
+#include "amdgpu_atomfirmware.h"
+
+struct ras_ih_data {
+	/* interrupt bottom half */
+	struct work_struct ih_work;
+	int inuse;
+	/* IP callback */
+	ras_ih_cb cb;
+	/* full of entries */
+	unsigned char *ring;
+	unsigned int ring_size;
+	unsigned int element_size;
+	unsigned int aligned_element_size;
+	unsigned int rptr;
+	unsigned int wptr;
+};
+
+struct ras_fs_data {
+	char sysfs_name[32];
+	char debugfs_name[32];
+};
+
+struct ras_err_data {
+	unsigned long ue_count;
+	unsigned long ce_count;
+};
+
+struct ras_err_handler_data {
+	/* point to bad pages array */
+	struct {
+		unsigned long bp;
+		struct amdgpu_bo *bo;
+	} *bps;
+	/* the count of entries */
+	int count;
+	/* the space can place new entries */
+	int space_left;
+	/* last reserved entry's index + 1 */
+	int last_reserved;
+};
+
+struct ras_manager {
+	struct ras_common_if head;
+	/* reference count */
+	int use;
+	/* ras block link */
+	struct list_head node;
+	/* the device */
+	struct amdgpu_device *adev;
+	/* debugfs */
+	struct dentry *ent;
+	/* sysfs */
+	struct device_attribute sysfs_attr;
+	int attr_inuse;
+
+	/* fs node name */
+	struct ras_fs_data fs_data;
+
+	/* IH data */
+	struct ras_ih_data ih_data;
+
+	struct ras_err_data err_data;
+};
+
+const char *ras_error_string[] = {
+	"none",
+	"parity",
+	"single_correctable",
+	"multi_uncorrectable",
+	"poison",
+};
+
+const char *ras_block_string[] = {
+	"umc",
+	"sdma",
+	"gfx",
+	"mmhub",
+	"athub",
+	"pcie_bif",
+	"hdp",
+	"xgmi_wafl",
+	"df",
+	"smn",
+	"sem",
+	"mp0",
+	"mp1",
+	"fuse",
+};
+
+#define ras_err_str(i) (ras_error_string[ffs(i)])
+#define ras_block_str(i) (ras_block_string[i])
+
+#define AMDGPU_RAS_FLAG_INIT_BY_VBIOS 1
+#define RAS_DEFAULT_FLAGS (AMDGPU_RAS_FLAG_INIT_BY_VBIOS)
+
+static void amdgpu_ras_self_test(struct amdgpu_device *adev)
+{
+	/* TODO */
+}
+
+static ssize_t amdgpu_ras_debugfs_read(struct file *f, char __user *buf,
+					size_t size, loff_t *pos)
+{
+	struct ras_manager *obj = (struct ras_manager *)file_inode(f)->i_private;
+	struct ras_query_if info = {
+		.head = obj->head,
+	};
+	ssize_t s;
+	char val[128];
+
+	if (amdgpu_ras_error_query(obj->adev, &info))
+		return -EINVAL;
+
+	s = snprintf(val, sizeof(val), "%s: %lu\n%s: %lu\n",
+			"ue", info.ue_count,
+			"ce", info.ce_count);
+	if (*pos >= s)
+		return 0;
+
+	s -= *pos;
+	s = min_t(u64, s, size);
+
+
+	if (copy_to_user(buf, &val[*pos], s))
+		return -EINVAL;
+
+	*pos += s;
+
+	return s;
+}
+
+static const struct file_operations amdgpu_ras_debugfs_ops = {
+	.owner = THIS_MODULE,
+	.read = amdgpu_ras_debugfs_read,
+	.write = NULL,
+	.llseek = default_llseek
+};
+
+static int amdgpu_ras_find_block_id_by_name(const char *name, int *block_id)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(ras_block_string); i++) {
+		*block_id = i;
+		if (strcmp(name, ras_block_str(i)) == 0)
+			return 0;
+	}
+	return -EINVAL;
+}
+
+static int amdgpu_ras_debugfs_ctrl_parse_data(struct file *f,
+		const char __user *buf, size_t size,
+		loff_t *pos, struct ras_debug_if *data)
+{
+	ssize_t s = min_t(u64, 64, size);
+	char str[65];
+	char block_name[33];
+	char err[9] = "ue";
+	int op = -1;
+	int block_id;
+	u64 address, value;
+
+	if (*pos)
+		return -EINVAL;
+	*pos = size;
+
+	memset(str, 0, sizeof(str));
+	memset(data, 0, sizeof(*data));
+
+	if (copy_from_user(str, buf, s))
+		return -EINVAL;
+
+	if (sscanf(str, "disable %32s", block_name) == 1)
+		op = 0;
+	else if (sscanf(str, "enable %32s %8s", block_name, err) == 2)
+		op = 1;
+	else if (sscanf(str, "inject %32s %8s", block_name, err) == 2)
+		op = 2;
+	else if (str[0] && str[1] && str[2] && str[3])
+		/* ascii string, but commands are not matched. */
+		return -EINVAL;
+
+	if (op != -1) {
+		if (amdgpu_ras_find_block_id_by_name(block_name, &block_id))
+			return -EINVAL;
+
+		data->head.block = block_id;
+		data->head.type = memcmp("ue", err, 2) == 0 ?
+			AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE :
+			AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE;
+		data->op = op;
+
+		if (op == 2) {
+			if (sscanf(str, "%*s %*s %*s %llu %llu",
+						&address, &value) != 2)
+				if (sscanf(str, "%*s %*s %*s 0x%llx 0x%llx",
+							&address, &value) != 2)
+					return -EINVAL;
+			data->inject.address = address;
+			data->inject.value = value;
+		}
+	} else {
+		if (size < sizeof(*data))
+			return -EINVAL;
+
+		if (copy_from_user(data, buf, sizeof(*data)))
+			return -EINVAL;
+	}
+
+	return 0;
+}
+/*
+ * DOC: ras debugfs control interface
+ *
+ * It accepts struct ras_debug_if who has two members.
+ *
+ * First member: ras_debug_if::head or ras_debug_if::inject.
+ *
+ * head is used to indicate which IP block will be under control.
+ *
+ * head has four members, they are block, type, sub_block_index, name.
+ * block: which IP will be under control.
+ * type: what kind of error will be enabled/disabled/injected.
+ * sub_block_index: some IPs have subcomponets. say, GFX, sDMA.
+ * name: the name of IP.
+ *
+ * inject has two more members than head, they are address, value.
+ * As their names indicate, inject operation will write the
+ * value to the address.
+ *
+ * Second member: struct ras_debug_if::op.
+ * It has three kinds of operations.
+ *  0: disable RAS on the block. Take ::head as its data.
+ *  1: enable RAS on the block. Take ::head as its data.
+ *  2: inject errors on the block. Take ::inject as its data.
+ *
+ * How to use the interface?
+ * programs:
+ * copy the struct ras_debug_if in your codes and initialize it.
+ * write the struct to the control node.
+ *
+ * bash:
+ * echo op block [error [address value]] > .../ras/ras_ctrl
+ *	op: disable, enable, inject
+ *		disable: only block is needed
+ *		enable: block and error are needed
+ *		inject: error, address, value are needed
+ *	block: umc, smda, gfx, .........
+ *		see ras_block_string[] for details
+ *	error: ue, ce
+ *		ue: multi_uncorrectable
+ *		ce: single_correctable
+ *
+ * here are some examples for bash commands,
+ *	echo inject umc ue 0x0 0x0 > /sys/kernel/debug/dri/0/ras/ras_ctrl
+ *	echo inject umc ce 0 0 > /sys/kernel/debug/dri/0/ras/ras_ctrl
+ *	echo disable umc > /sys/kernel/debug/dri/0/ras/ras_ctrl
+ *
+ * How to check the result?
+ *
+ * For disable/enable, please check ras features at
+ * /sys/class/drm/card[0/1/2...]/device/ras/features
+ *
+ * For inject, please check corresponding err count at
+ * /sys/class/drm/card[0/1/2...]/device/ras/[gfx/sdma/...]_err_count
+ *
+ * NOTE: operation is only allowed on blocks which are supported.
+ * Please check ras mask at /sys/module/amdgpu/parameters/ras_mask
+ */
+static ssize_t amdgpu_ras_debugfs_ctrl_write(struct file *f, const char __user *buf,
+		size_t size, loff_t *pos)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private;
+	struct ras_debug_if data;
+	int ret = 0;
+
+	ret = amdgpu_ras_debugfs_ctrl_parse_data(f, buf, size, pos, &data);
+	if (ret)
+		return -EINVAL;
+
+	if (!amdgpu_ras_is_supported(adev, data.head.block))
+		return -EINVAL;
+
+	switch (data.op) {
+	case 0:
+		ret = amdgpu_ras_feature_enable(adev, &data.head, 0);
+		break;
+	case 1:
+		ret = amdgpu_ras_feature_enable(adev, &data.head, 1);
+		break;
+	case 2:
+		ret = amdgpu_ras_error_inject(adev, &data.inject);
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	};
+
+	if (ret)
+		return -EINVAL;
+
+	return size;
+}
+
+static const struct file_operations amdgpu_ras_debugfs_ctrl_ops = {
+	.owner = THIS_MODULE,
+	.read = NULL,
+	.write = amdgpu_ras_debugfs_ctrl_write,
+	.llseek = default_llseek
+};
+
+static ssize_t amdgpu_ras_sysfs_read(struct device *dev,
+		struct device_attribute *attr, char *buf)
+{
+	struct ras_manager *obj = container_of(attr, struct ras_manager, sysfs_attr);
+	struct ras_query_if info = {
+		.head = obj->head,
+	};
+
+	if (amdgpu_ras_error_query(obj->adev, &info))
+		return -EINVAL;
+
+	return snprintf(buf, PAGE_SIZE, "%s: %lu\n%s: %lu\n",
+			"ue", info.ue_count,
+			"ce", info.ce_count);
+}
+
+/* obj begin */
+
+#define get_obj(obj) do { (obj)->use++; } while (0)
+#define alive_obj(obj) ((obj)->use)
+
+static inline void put_obj(struct ras_manager *obj)
+{
+	if (obj && --obj->use == 0)
+		list_del(&obj->node);
+	if (obj && obj->use < 0) {
+		 DRM_ERROR("RAS ERROR: Unbalance obj(%s) use\n", obj->head.name);
+	}
+}
+
+/* make one obj and return it. */
+static struct ras_manager *amdgpu_ras_create_obj(struct amdgpu_device *adev,
+		struct ras_common_if *head)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+	struct ras_manager *obj;
+
+	if (!con)
+		return NULL;
+
+	if (head->block >= AMDGPU_RAS_BLOCK_COUNT)
+		return NULL;
+
+	obj = &con->objs[head->block];
+	/* already exist. return obj? */
+	if (alive_obj(obj))
+		return NULL;
+
+	obj->head = *head;
+	obj->adev = adev;
+	list_add(&obj->node, &con->head);
+	get_obj(obj);
+
+	return obj;
+}
+
+/* return an obj equal to head, or the first when head is NULL */
+static struct ras_manager *amdgpu_ras_find_obj(struct amdgpu_device *adev,
+		struct ras_common_if *head)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+	struct ras_manager *obj;
+	int i;
+
+	if (!con)
+		return NULL;
+
+	if (head) {
+		if (head->block >= AMDGPU_RAS_BLOCK_COUNT)
+			return NULL;
+
+		obj = &con->objs[head->block];
+
+		if (alive_obj(obj)) {
+			WARN_ON(head->block != obj->head.block);
+			return obj;
+		}
+	} else {
+		for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT; i++) {
+			obj = &con->objs[i];
+			if (alive_obj(obj)) {
+				WARN_ON(i != obj->head.block);
+				return obj;
+			}
+		}
+	}
+
+	return NULL;
+}
+/* obj end */
+
+/* feature ctl begin */
+static int amdgpu_ras_is_feature_allowed(struct amdgpu_device *adev,
+		struct ras_common_if *head)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+
+	return con->hw_supported & BIT(head->block);
+}
+
+static int amdgpu_ras_is_feature_enabled(struct amdgpu_device *adev,
+		struct ras_common_if *head)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+
+	return con->features & BIT(head->block);
+}
+
+/*
+ * if obj is not created, then create one.
+ * set feature enable flag.
+ */
+static int __amdgpu_ras_feature_enable(struct amdgpu_device *adev,
+		struct ras_common_if *head, int enable)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+	struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
+
+	/* If hardware does not support ras, then do not create obj.
+	 * But if hardware support ras, we can create the obj.
+	 * Ras framework checks con->hw_supported to see if it need do
+	 * corresponding initialization.
+	 * IP checks con->support to see if it need disable ras.
+	 */
+	if (!amdgpu_ras_is_feature_allowed(adev, head))
+		return 0;
+	if (!(!!enable ^ !!amdgpu_ras_is_feature_enabled(adev, head)))
+		return 0;
+
+	if (enable) {
+		if (!obj) {
+			obj = amdgpu_ras_create_obj(adev, head);
+			if (!obj)
+				return -EINVAL;
+		} else {
+			/* In case we create obj somewhere else */
+			get_obj(obj);
+		}
+		con->features |= BIT(head->block);
+	} else {
+		if (obj && amdgpu_ras_is_feature_enabled(adev, head)) {
+			con->features &= ~BIT(head->block);
+			put_obj(obj);
+		}
+	}
+
+	return 0;
+}
+
+/* wrapper of psp_ras_enable_features */
+int amdgpu_ras_feature_enable(struct amdgpu_device *adev,
+		struct ras_common_if *head, bool enable)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+	union ta_ras_cmd_input info;
+	int ret;
+
+	if (!con)
+		return -EINVAL;
+
+	if (!enable) {
+		info.disable_features = (struct ta_ras_disable_features_input) {
+			.block_id =  amdgpu_ras_block_to_ta(head->block),
+			.error_type = amdgpu_ras_error_to_ta(head->type),
+		};
+	} else {
+		info.enable_features = (struct ta_ras_enable_features_input) {
+			.block_id =  amdgpu_ras_block_to_ta(head->block),
+			.error_type = amdgpu_ras_error_to_ta(head->type),
+		};
+	}
+
+	/* Do not enable if it is not allowed. */
+	WARN_ON(enable && !amdgpu_ras_is_feature_allowed(adev, head));
+	/* Are we alerady in that state we are going to set? */
+	if (!(!!enable ^ !!amdgpu_ras_is_feature_enabled(adev, head)))
+		return 0;
+
+	ret = psp_ras_enable_features(&adev->psp, &info, enable);
+	if (ret) {
+		DRM_ERROR("RAS ERROR: %s %s feature failed ret %d\n",
+				enable ? "enable":"disable",
+				ras_block_str(head->block),
+				ret);
+		return -EINVAL;
+	}
+
+	/* setup the obj */
+	__amdgpu_ras_feature_enable(adev, head, enable);
+
+	return 0;
+}
+
+/* Only used in device probe stage and called only once. */
+int amdgpu_ras_feature_enable_on_boot(struct amdgpu_device *adev,
+		struct ras_common_if *head, bool enable)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+	int ret;
+
+	if (!con)
+		return -EINVAL;
+
+	if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) {
+		/* If ras is enabled by vbios, we set up ras object first in
+		 * both case. For enable, that is all what we need do. For
+		 * disable, we need perform a ras TA disable cmd after that.
+		 */
+		ret = __amdgpu_ras_feature_enable(adev, head, 1);
+		if (ret)
+			return ret;
+
+		if (!enable)
+			ret = amdgpu_ras_feature_enable(adev, head, 0);
+	} else
+		ret = amdgpu_ras_feature_enable(adev, head, enable);
+
+	return ret;
+}
+
+static int amdgpu_ras_disable_all_features(struct amdgpu_device *adev,
+		bool bypass)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+	struct ras_manager *obj, *tmp;
+
+	list_for_each_entry_safe(obj, tmp, &con->head, node) {
+		/* bypass psp.
+		 * aka just release the obj and corresponding flags
+		 */
+		if (bypass) {
+			if (__amdgpu_ras_feature_enable(adev, &obj->head, 0))
+				break;
+		} else {
+			if (amdgpu_ras_feature_enable(adev, &obj->head, 0))
+				break;
+		}
+	}
+
+	return con->features;
+}
+
+static int amdgpu_ras_enable_all_features(struct amdgpu_device *adev,
+		bool bypass)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+	int ras_block_count = AMDGPU_RAS_BLOCK_COUNT;
+	int i;
+	const enum amdgpu_ras_error_type default_ras_type =
+		AMDGPU_RAS_ERROR__NONE;
+
+	for (i = 0; i < ras_block_count; i++) {
+		struct ras_common_if head = {
+			.block = i,
+			.type = default_ras_type,
+			.sub_block_index = 0,
+		};
+		strcpy(head.name, ras_block_str(i));
+		if (bypass) {
+			/*
+			 * bypass psp. vbios enable ras for us.
+			 * so just create the obj
+			 */
+			if (__amdgpu_ras_feature_enable(adev, &head, 1))
+				break;
+		} else {
+			if (amdgpu_ras_feature_enable(adev, &head, 1))
+				break;
+		}
+	}
+
+	return con->features;
+}
+/* feature ctl end */
+
+/* query/inject/cure begin */
+int amdgpu_ras_error_query(struct amdgpu_device *adev,
+		struct ras_query_if *info)
+{
+	struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
+
+	if (!obj)
+		return -EINVAL;
+	/* TODO might read the register to read the count */
+
+	info->ue_count = obj->err_data.ue_count;
+	info->ce_count = obj->err_data.ce_count;
+
+	return 0;
+}
+
+/* wrapper of psp_ras_trigger_error */
+int amdgpu_ras_error_inject(struct amdgpu_device *adev,
+		struct ras_inject_if *info)
+{
+	struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
+	struct ta_ras_trigger_error_input block_info = {
+		.block_id =  amdgpu_ras_block_to_ta(info->head.block),
+		.inject_error_type = amdgpu_ras_error_to_ta(info->head.type),
+		.sub_block_index = info->head.sub_block_index,
+		.address = info->address,
+		.value = info->value,
+	};
+	int ret = 0;
+
+	if (!obj)
+		return -EINVAL;
+
+	ret = psp_ras_trigger_error(&adev->psp, &block_info);
+	if (ret)
+		DRM_ERROR("RAS ERROR: inject %s error failed ret %d\n",
+				ras_block_str(info->head.block),
+				ret);
+
+	return ret;
+}
+
+int amdgpu_ras_error_cure(struct amdgpu_device *adev,
+		struct ras_cure_if *info)
+{
+	/* psp fw has no cure interface for now. */
+	return 0;
+}
+
+/* get the total error counts on all IPs */
+int amdgpu_ras_query_error_count(struct amdgpu_device *adev,
+		bool is_ce)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+	struct ras_manager *obj;
+	struct ras_err_data data = {0, 0};
+
+	if (!con)
+		return -EINVAL;
+
+	list_for_each_entry(obj, &con->head, node) {
+		struct ras_query_if info = {
+			.head = obj->head,
+		};
+
+		if (amdgpu_ras_error_query(adev, &info))
+			return -EINVAL;
+
+		data.ce_count += info.ce_count;
+		data.ue_count += info.ue_count;
+	}
+
+	return is_ce ? data.ce_count : data.ue_count;
+}
+/* query/inject/cure end */
+
+
+/* sysfs begin */
+
+static ssize_t amdgpu_ras_sysfs_features_read(struct device *dev,
+		struct device_attribute *attr, char *buf)
+{
+	struct amdgpu_ras *con =
+		container_of(attr, struct amdgpu_ras, features_attr);
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct amdgpu_device *adev = ddev->dev_private;
+	struct ras_common_if head;
+	int ras_block_count = AMDGPU_RAS_BLOCK_COUNT;
+	int i;
+	ssize_t s;
+	struct ras_manager *obj;
+
+	s = scnprintf(buf, PAGE_SIZE, "feature mask: 0x%x\n", con->features);
+
+	for (i = 0; i < ras_block_count; i++) {
+		head.block = i;
+
+		if (amdgpu_ras_is_feature_enabled(adev, &head)) {
+			obj = amdgpu_ras_find_obj(adev, &head);
+			s += scnprintf(&buf[s], PAGE_SIZE - s,
+					"%s: %s\n",
+					ras_block_str(i),
+					ras_err_str(obj->head.type));
+		} else
+			s += scnprintf(&buf[s], PAGE_SIZE - s,
+					"%s: disabled\n",
+					ras_block_str(i));
+	}
+
+	return s;
+}
+
+static int amdgpu_ras_sysfs_create_feature_node(struct amdgpu_device *adev)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+	struct attribute *attrs[] = {
+		&con->features_attr.attr,
+		NULL
+	};
+	struct attribute_group group = {
+		.name = "ras",
+		.attrs = attrs,
+	};
+
+	con->features_attr = (struct device_attribute) {
+		.attr = {
+			.name = "features",
+			.mode = S_IRUGO,
+		},
+			.show = amdgpu_ras_sysfs_features_read,
+	};
+	sysfs_attr_init(attrs[0]);
+
+	return sysfs_create_group(&adev->dev->kobj, &group);
+}
+
+static int amdgpu_ras_sysfs_remove_feature_node(struct amdgpu_device *adev)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+	struct attribute *attrs[] = {
+		&con->features_attr.attr,
+		NULL
+	};
+	struct attribute_group group = {
+		.name = "ras",
+		.attrs = attrs,
+	};
+
+	sysfs_remove_group(&adev->dev->kobj, &group);
+
+	return 0;
+}
+
+int amdgpu_ras_sysfs_create(struct amdgpu_device *adev,
+		struct ras_fs_if *head)
+{
+	struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head->head);
+
+	if (!obj || obj->attr_inuse)
+		return -EINVAL;
+
+	get_obj(obj);
+
+	memcpy(obj->fs_data.sysfs_name,
+			head->sysfs_name,
+			sizeof(obj->fs_data.sysfs_name));
+
+	obj->sysfs_attr = (struct device_attribute){
+		.attr = {
+			.name = obj->fs_data.sysfs_name,
+			.mode = S_IRUGO,
+		},
+			.show = amdgpu_ras_sysfs_read,
+	};
+	sysfs_attr_init(&obj->sysfs_attr.attr);
+
+	if (sysfs_add_file_to_group(&adev->dev->kobj,
+				&obj->sysfs_attr.attr,
+				"ras")) {
+		put_obj(obj);
+		return -EINVAL;
+	}
+
+	obj->attr_inuse = 1;
+
+	return 0;
+}
+
+int amdgpu_ras_sysfs_remove(struct amdgpu_device *adev,
+		struct ras_common_if *head)
+{
+	struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
+
+	if (!obj || !obj->attr_inuse)
+		return -EINVAL;
+
+	sysfs_remove_file_from_group(&adev->dev->kobj,
+				&obj->sysfs_attr.attr,
+				"ras");
+	obj->attr_inuse = 0;
+	put_obj(obj);
+
+	return 0;
+}
+
+static int amdgpu_ras_sysfs_remove_all(struct amdgpu_device *adev)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+	struct ras_manager *obj, *tmp;
+
+	list_for_each_entry_safe(obj, tmp, &con->head, node) {
+		amdgpu_ras_sysfs_remove(adev, &obj->head);
+	}
+
+	amdgpu_ras_sysfs_remove_feature_node(adev);
+
+	return 0;
+}
+/* sysfs end */
+
+/* debugfs begin */
+static int amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device *adev)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+	struct drm_minor *minor = adev->ddev->primary;
+	struct dentry *root = minor->debugfs_root, *dir;
+	struct dentry *ent;
+
+	dir = debugfs_create_dir("ras", root);
+	if (IS_ERR(dir))
+		return -EINVAL;
+
+	con->dir = dir;
+
+	ent = debugfs_create_file("ras_ctrl",
+			S_IWUGO | S_IRUGO, con->dir,
+			adev, &amdgpu_ras_debugfs_ctrl_ops);
+	if (IS_ERR(ent)) {
+		debugfs_remove(con->dir);
+		return -EINVAL;
+	}
+
+	con->ent = ent;
+	return 0;
+}
+
+int amdgpu_ras_debugfs_create(struct amdgpu_device *adev,
+		struct ras_fs_if *head)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+	struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head->head);
+	struct dentry *ent;
+
+	if (!obj || obj->ent)
+		return -EINVAL;
+
+	get_obj(obj);
+
+	memcpy(obj->fs_data.debugfs_name,
+			head->debugfs_name,
+			sizeof(obj->fs_data.debugfs_name));
+
+	ent = debugfs_create_file(obj->fs_data.debugfs_name,
+			S_IWUGO | S_IRUGO, con->dir,
+			obj, &amdgpu_ras_debugfs_ops);
+
+	if (IS_ERR(ent))
+		return -EINVAL;
+
+	obj->ent = ent;
+
+	return 0;
+}
+
+int amdgpu_ras_debugfs_remove(struct amdgpu_device *adev,
+		struct ras_common_if *head)
+{
+	struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
+
+	if (!obj || !obj->ent)
+		return 0;
+
+	debugfs_remove(obj->ent);
+	obj->ent = NULL;
+	put_obj(obj);
+
+	return 0;
+}
+
+static int amdgpu_ras_debugfs_remove_all(struct amdgpu_device *adev)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+	struct ras_manager *obj, *tmp;
+
+	list_for_each_entry_safe(obj, tmp, &con->head, node) {
+		amdgpu_ras_debugfs_remove(adev, &obj->head);
+	}
+
+	debugfs_remove(con->ent);
+	debugfs_remove(con->dir);
+	con->dir = NULL;
+	con->ent = NULL;
+
+	return 0;
+}
+/* debugfs end */
+
+/* ras fs */
+
+static int amdgpu_ras_fs_init(struct amdgpu_device *adev)
+{
+	amdgpu_ras_sysfs_create_feature_node(adev);
+	amdgpu_ras_debugfs_create_ctrl_node(adev);
+
+	return 0;
+}
+
+static int amdgpu_ras_fs_fini(struct amdgpu_device *adev)
+{
+	amdgpu_ras_debugfs_remove_all(adev);
+	amdgpu_ras_sysfs_remove_all(adev);
+	return 0;
+}
+/* ras fs end */
+
+/* ih begin */
+static void amdgpu_ras_interrupt_handler(struct ras_manager *obj)
+{
+	struct ras_ih_data *data = &obj->ih_data;
+	struct amdgpu_iv_entry entry;
+	int ret;
+
+	while (data->rptr != data->wptr) {
+		rmb();
+		memcpy(&entry, &data->ring[data->rptr],
+				data->element_size);
+
+		wmb();
+		data->rptr = (data->aligned_element_size +
+				data->rptr) % data->ring_size;
+
+		/* Let IP handle its data, maybe we need get the output
+		 * from the callback to udpate the error type/count, etc
+		 */
+		if (data->cb) {
+			ret = data->cb(obj->adev, &entry);
+			/* ue will trigger an interrupt, and in that case
+			 * we need do a reset to recovery the whole system.
+			 * But leave IP do that recovery, here we just dispatch
+			 * the error.
+			 */
+			if (ret == AMDGPU_RAS_UE) {
+				obj->err_data.ue_count++;
+			}
+			/* Might need get ce count by register, but not all IP
+			 * saves ce count, some IP just use one bit or two bits
+			 * to indicate ce happened.
+			 */
+		}
+	}
+}
+
+static void amdgpu_ras_interrupt_process_handler(struct work_struct *work)
+{
+	struct ras_ih_data *data =
+		container_of(work, struct ras_ih_data, ih_work);
+	struct ras_manager *obj =
+		container_of(data, struct ras_manager, ih_data);
+
+	amdgpu_ras_interrupt_handler(obj);
+}
+
+int amdgpu_ras_interrupt_dispatch(struct amdgpu_device *adev,
+		struct ras_dispatch_if *info)
+{
+	struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
+	struct ras_ih_data *data = &obj->ih_data;
+
+	if (!obj)
+		return -EINVAL;
+
+	if (data->inuse == 0)
+		return 0;
+
+	/* Might be overflow... */
+	memcpy(&data->ring[data->wptr], info->entry,
+			data->element_size);
+
+	wmb();
+	data->wptr = (data->aligned_element_size +
+			data->wptr) % data->ring_size;
+
+	schedule_work(&data->ih_work);
+
+	return 0;
+}
+
+int amdgpu_ras_interrupt_remove_handler(struct amdgpu_device *adev,
+		struct ras_ih_if *info)
+{
+	struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
+	struct ras_ih_data *data;
+
+	if (!obj)
+		return -EINVAL;
+
+	data = &obj->ih_data;
+	if (data->inuse == 0)
+		return 0;
+
+	cancel_work_sync(&data->ih_work);
+
+	kfree(data->ring);
+	memset(data, 0, sizeof(*data));
+	put_obj(obj);
+
+	return 0;
+}
+
+int amdgpu_ras_interrupt_add_handler(struct amdgpu_device *adev,
+		struct ras_ih_if *info)
+{
+	struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
+	struct ras_ih_data *data;
+
+	if (!obj) {
+		/* in case we registe the IH before enable ras feature */
+		obj = amdgpu_ras_create_obj(adev, &info->head);
+		if (!obj)
+			return -EINVAL;
+	} else
+		get_obj(obj);
+
+	data = &obj->ih_data;
+	/* add the callback.etc */
+	*data = (struct ras_ih_data) {
+		.inuse = 0,
+		.cb = info->cb,
+		.element_size = sizeof(struct amdgpu_iv_entry),
+		.rptr = 0,
+		.wptr = 0,
+	};
+
+	INIT_WORK(&data->ih_work, amdgpu_ras_interrupt_process_handler);
+
+	data->aligned_element_size = ALIGN(data->element_size, 8);
+	/* the ring can store 64 iv entries. */
+	data->ring_size = 64 * data->aligned_element_size;
+	data->ring = kmalloc(data->ring_size, GFP_KERNEL);
+	if (!data->ring) {
+		put_obj(obj);
+		return -ENOMEM;
+	}
+
+	/* IH is ready */
+	data->inuse = 1;
+
+	return 0;
+}
+
+static int amdgpu_ras_interrupt_remove_all(struct amdgpu_device *adev)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+	struct ras_manager *obj, *tmp;
+
+	list_for_each_entry_safe(obj, tmp, &con->head, node) {
+		struct ras_ih_if info = {
+			.head = obj->head,
+		};
+		amdgpu_ras_interrupt_remove_handler(adev, &info);
+	}
+
+	return 0;
+}
+/* ih end */
+
+/* recovery begin */
+static void amdgpu_ras_do_recovery(struct work_struct *work)
+{
+	struct amdgpu_ras *ras =
+		container_of(work, struct amdgpu_ras, recovery_work);
+
+	amdgpu_device_gpu_recover(ras->adev, 0);
+	atomic_set(&ras->in_recovery, 0);
+}
+
+static int amdgpu_ras_release_vram(struct amdgpu_device *adev,
+		struct amdgpu_bo **bo_ptr)
+{
+	/* no need to free it actually. */
+	amdgpu_bo_free_kernel(bo_ptr, NULL, NULL);
+	return 0;
+}
+
+/* reserve vram with size@offset */
+static int amdgpu_ras_reserve_vram(struct amdgpu_device *adev,
+		uint64_t offset, uint64_t size,
+		struct amdgpu_bo **bo_ptr)
+{
+	struct ttm_operation_ctx ctx = { false, false };
+	struct amdgpu_bo_param bp;
+	int r = 0;
+	int i;
+	struct amdgpu_bo *bo;
+
+	if (bo_ptr)
+		*bo_ptr = NULL;
+	memset(&bp, 0, sizeof(bp));
+	bp.size = size;
+	bp.byte_align = PAGE_SIZE;
+	bp.domain = AMDGPU_GEM_DOMAIN_VRAM;
+	bp.flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
+		AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
+	bp.type = ttm_bo_type_kernel;
+	bp.resv = NULL;
+
+	r = amdgpu_bo_create(adev, &bp, &bo);
+	if (r)
+		return -EINVAL;
+
+	r = amdgpu_bo_reserve(bo, false);
+	if (r)
+		goto error_reserve;
+
+	offset = ALIGN(offset, PAGE_SIZE);
+	for (i = 0; i < bo->placement.num_placement; ++i) {
+		bo->placements[i].fpfn = offset >> PAGE_SHIFT;
+		bo->placements[i].lpfn = (offset + size) >> PAGE_SHIFT;
+	}
+
+	ttm_bo_mem_put(&bo->tbo, &bo->tbo.mem);
+	r = ttm_bo_mem_space(&bo->tbo, &bo->placement, &bo->tbo.mem, &ctx);
+	if (r)
+		goto error_pin;
+
+	r = amdgpu_bo_pin_restricted(bo,
+			AMDGPU_GEM_DOMAIN_VRAM,
+			offset,
+			offset + size);
+	if (r)
+		goto error_pin;
+
+	if (bo_ptr)
+		*bo_ptr = bo;
+
+	amdgpu_bo_unreserve(bo);
+	return r;
+
+error_pin:
+	amdgpu_bo_unreserve(bo);
+error_reserve:
+	amdgpu_bo_unref(&bo);
+	return r;
+}
+
+/* alloc/realloc bps array */
+static int amdgpu_ras_realloc_eh_data_space(struct amdgpu_device *adev,
+		struct ras_err_handler_data *data, int pages)
+{
+	unsigned int old_space = data->count + data->space_left;
+	unsigned int new_space = old_space + pages;
+	unsigned int align_space = ALIGN(new_space, 1024);
+	void *tmp = kmalloc(align_space * sizeof(*data->bps), GFP_KERNEL);
+
+	if (!tmp)
+		return -ENOMEM;
+
+	if (data->bps) {
+		memcpy(tmp, data->bps,
+				data->count * sizeof(*data->bps));
+		kfree(data->bps);
+	}
+
+	data->bps = tmp;
+	data->space_left += align_space - old_space;
+	return 0;
+}
+
+/* it deal with vram only. */
+int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev,
+		unsigned long *bps, int pages)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+	struct ras_err_handler_data *data;
+	int i = pages;
+	int ret = 0;
+
+	if (!con || !con->eh_data || !bps || pages <= 0)
+		return 0;
+
+	mutex_lock(&con->recovery_lock);
+	data = con->eh_data;
+	if (!data)
+		goto out;
+
+	if (data->space_left <= pages)
+		if (amdgpu_ras_realloc_eh_data_space(adev, data, pages)) {
+			ret = -ENOMEM;
+			goto out;
+		}
+
+	while (i--)
+		data->bps[data->count++].bp = bps[i];
+
+	data->space_left -= pages;
+out:
+	mutex_unlock(&con->recovery_lock);
+
+	return ret;
+}
+
+/* called in gpu recovery/init */
+int amdgpu_ras_reserve_bad_pages(struct amdgpu_device *adev)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+	struct ras_err_handler_data *data;
+	uint64_t bp;
+	struct amdgpu_bo *bo;
+	int i;
+
+	if (!con || !con->eh_data)
+		return 0;
+
+	mutex_lock(&con->recovery_lock);
+	data = con->eh_data;
+	if (!data)
+		goto out;
+	/* reserve vram at driver post stage. */
+	for (i = data->last_reserved; i < data->count; i++) {
+		bp = data->bps[i].bp;
+
+		if (amdgpu_ras_reserve_vram(adev, bp << PAGE_SHIFT,
+					PAGE_SIZE, &bo))
+			DRM_ERROR("RAS ERROR: reserve vram %llx fail\n", bp);
+
+		data->bps[i].bo = bo;
+		data->last_reserved = i + 1;
+	}
+out:
+	mutex_unlock(&con->recovery_lock);
+	return 0;
+}
+
+/* called when driver unload */
+static int amdgpu_ras_release_bad_pages(struct amdgpu_device *adev)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+	struct ras_err_handler_data *data;
+	struct amdgpu_bo *bo;
+	int i;
+
+	if (!con || !con->eh_data)
+		return 0;
+
+	mutex_lock(&con->recovery_lock);
+	data = con->eh_data;
+	if (!data)
+		goto out;
+
+	for (i = data->last_reserved - 1; i >= 0; i--) {
+		bo = data->bps[i].bo;
+
+		amdgpu_ras_release_vram(adev, &bo);
+
+		data->bps[i].bo = bo;
+		data->last_reserved = i;
+	}
+out:
+	mutex_unlock(&con->recovery_lock);
+	return 0;
+}
+
+static int amdgpu_ras_save_bad_pages(struct amdgpu_device *adev)
+{
+	/* TODO
+	 * write the array to eeprom when SMU disabled.
+	 */
+	return 0;
+}
+
+static int amdgpu_ras_load_bad_pages(struct amdgpu_device *adev)
+{
+	/* TODO
+	 * read the array to eeprom when SMU disabled.
+	 */
+	return 0;
+}
+
+static int amdgpu_ras_recovery_init(struct amdgpu_device *adev)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+	struct ras_err_handler_data **data = &con->eh_data;
+
+	*data = kmalloc(sizeof(**data),
+			GFP_KERNEL|__GFP_ZERO);
+	if (!*data)
+		return -ENOMEM;
+
+	mutex_init(&con->recovery_lock);
+	INIT_WORK(&con->recovery_work, amdgpu_ras_do_recovery);
+	atomic_set(&con->in_recovery, 0);
+	con->adev = adev;
+
+	amdgpu_ras_load_bad_pages(adev);
+	amdgpu_ras_reserve_bad_pages(adev);
+
+	return 0;
+}
+
+static int amdgpu_ras_recovery_fini(struct amdgpu_device *adev)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+	struct ras_err_handler_data *data = con->eh_data;
+
+	cancel_work_sync(&con->recovery_work);
+	amdgpu_ras_save_bad_pages(adev);
+	amdgpu_ras_release_bad_pages(adev);
+
+	mutex_lock(&con->recovery_lock);
+	con->eh_data = NULL;
+	kfree(data->bps);
+	kfree(data);
+	mutex_unlock(&con->recovery_lock);
+
+	return 0;
+}
+/* recovery end */
+
+/*
+ * check hardware's ras ability which will be saved in hw_supported.
+ * if hardware does not support ras, we can skip some ras initializtion and
+ * forbid some ras operations from IP.
+ * if software itself, say boot parameter, limit the ras ability. We still
+ * need allow IP do some limited operations, like disable. In such case,
+ * we have to initialize ras as normal. but need check if operation is
+ * allowed or not in each function.
+ */
+static void amdgpu_ras_check_supported(struct amdgpu_device *adev,
+		uint32_t *hw_supported, uint32_t *supported)
+{
+	*hw_supported = 0;
+	*supported = 0;
+
+	if (amdgpu_sriov_vf(adev) ||
+			adev->asic_type != CHIP_VEGA20)
+		return;
+
+	if (adev->is_atom_fw &&
+			(amdgpu_atomfirmware_mem_ecc_supported(adev) ||
+			 amdgpu_atomfirmware_sram_ecc_supported(adev)))
+		*hw_supported = AMDGPU_RAS_BLOCK_MASK;
+
+	*supported = amdgpu_ras_enable == 0 ?
+				0 : *hw_supported & amdgpu_ras_mask;
+}
+
+int amdgpu_ras_init(struct amdgpu_device *adev)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+
+	if (con)
+		return 0;
+
+	con = kmalloc(sizeof(struct amdgpu_ras) +
+			sizeof(struct ras_manager) * AMDGPU_RAS_BLOCK_COUNT,
+			GFP_KERNEL|__GFP_ZERO);
+	if (!con)
+		return -ENOMEM;
+
+	con->objs = (struct ras_manager *)(con + 1);
+
+	amdgpu_ras_set_context(adev, con);
+
+	amdgpu_ras_check_supported(adev, &con->hw_supported,
+			&con->supported);
+	con->features = 0;
+	INIT_LIST_HEAD(&con->head);
+	/* Might need get this flag from vbios. */
+	con->flags = RAS_DEFAULT_FLAGS;
+
+	if (amdgpu_ras_recovery_init(adev))
+		goto recovery_out;
+
+	amdgpu_ras_mask &= AMDGPU_RAS_BLOCK_MASK;
+
+	if (amdgpu_ras_fs_init(adev))
+		goto fs_out;
+
+	amdgpu_ras_self_test(adev);
+
+	DRM_INFO("RAS INFO: ras initialized successfully, "
+			"hardware ability[%x] ras_mask[%x]\n",
+			con->hw_supported, con->supported);
+	return 0;
+fs_out:
+	amdgpu_ras_recovery_fini(adev);
+recovery_out:
+	amdgpu_ras_set_context(adev, NULL);
+	kfree(con);
+
+	return -EINVAL;
+}
+
+/* do some init work after IP late init as dependence */
+void amdgpu_ras_post_init(struct amdgpu_device *adev)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+	struct ras_manager *obj, *tmp;
+
+	if (!con)
+		return;
+
+	if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) {
+		/* Set up all other IPs which are not implemented. There is a
+		 * tricky thing that IP's actual ras error type should be
+		 * MULTI_UNCORRECTABLE, but as driver does not handle it, so
+		 * ERROR_NONE make sense anyway.
+		 */
+		amdgpu_ras_enable_all_features(adev, 1);
+
+		/* We enable ras on all hw_supported block, but as boot
+		 * parameter might disable some of them and one or more IP has
+		 * not implemented yet. So we disable them on behalf.
+		 */
+		list_for_each_entry_safe(obj, tmp, &con->head, node) {
+			if (!amdgpu_ras_is_supported(adev, obj->head.block)) {
+				amdgpu_ras_feature_enable(adev, &obj->head, 0);
+				/* there should be no any reference. */
+				WARN_ON(alive_obj(obj));
+			}
+		}
+	}
+}
+
+/* do some fini work before IP fini as dependence */
+int amdgpu_ras_pre_fini(struct amdgpu_device *adev)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+
+	if (!con)
+		return 0;
+
+	/* Need disable ras on all IPs here before ip [hw/sw]fini */
+	amdgpu_ras_disable_all_features(adev, 0);
+	amdgpu_ras_recovery_fini(adev);
+	return 0;
+}
+
+int amdgpu_ras_fini(struct amdgpu_device *adev)
+{
+	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
+
+	if (!con)
+		return 0;
+
+	amdgpu_ras_fs_fini(adev);
+	amdgpu_ras_interrupt_remove_all(adev);
+
+	WARN(con->features, "Feature mask is not cleared");
+
+	if (con->features)
+		amdgpu_ras_disable_all_features(adev, 1);
+
+	amdgpu_ras_set_context(adev, NULL);
+	kfree(con);
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h
new file mode 100644
index 000000000000..eaef5edefc34
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h
@@ -0,0 +1,294 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ *
+ */
+#ifndef _AMDGPU_RAS_H
+#define _AMDGPU_RAS_H
+
+#include <linux/debugfs.h>
+#include <linux/list.h>
+#include "amdgpu.h"
+#include "amdgpu_psp.h"
+#include "ta_ras_if.h"
+
+enum amdgpu_ras_block {
+	AMDGPU_RAS_BLOCK__UMC = 0,
+	AMDGPU_RAS_BLOCK__SDMA,
+	AMDGPU_RAS_BLOCK__GFX,
+	AMDGPU_RAS_BLOCK__MMHUB,
+	AMDGPU_RAS_BLOCK__ATHUB,
+	AMDGPU_RAS_BLOCK__PCIE_BIF,
+	AMDGPU_RAS_BLOCK__HDP,
+	AMDGPU_RAS_BLOCK__XGMI_WAFL,
+	AMDGPU_RAS_BLOCK__DF,
+	AMDGPU_RAS_BLOCK__SMN,
+	AMDGPU_RAS_BLOCK__SEM,
+	AMDGPU_RAS_BLOCK__MP0,
+	AMDGPU_RAS_BLOCK__MP1,
+	AMDGPU_RAS_BLOCK__FUSE,
+
+	AMDGPU_RAS_BLOCK__LAST
+};
+
+#define AMDGPU_RAS_BLOCK_COUNT	AMDGPU_RAS_BLOCK__LAST
+#define AMDGPU_RAS_BLOCK_MASK	((1ULL << AMDGPU_RAS_BLOCK_COUNT) - 1)
+
+enum amdgpu_ras_error_type {
+	AMDGPU_RAS_ERROR__NONE							= 0,
+	AMDGPU_RAS_ERROR__PARITY						= 1,
+	AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE					= 2,
+	AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE					= 4,
+	AMDGPU_RAS_ERROR__POISON						= 8,
+};
+
+enum amdgpu_ras_ret {
+	AMDGPU_RAS_SUCCESS = 0,
+	AMDGPU_RAS_FAIL,
+	AMDGPU_RAS_UE,
+	AMDGPU_RAS_CE,
+	AMDGPU_RAS_PT,
+};
+
+struct ras_common_if {
+	enum amdgpu_ras_block block;
+	enum amdgpu_ras_error_type type;
+	uint32_t sub_block_index;
+	/* block name */
+	char name[32];
+};
+
+typedef int (*ras_ih_cb)(struct amdgpu_device *adev,
+		struct amdgpu_iv_entry *entry);
+
+struct amdgpu_ras {
+	/* ras infrastructure */
+	/* for ras itself. */
+	uint32_t hw_supported;
+	/* for IP to check its ras ability. */
+	uint32_t supported;
+	uint32_t features;
+	struct list_head head;
+	/* debugfs */
+	struct dentry *dir;
+	/* debugfs ctrl */
+	struct dentry *ent;
+	/* sysfs */
+	struct device_attribute features_attr;
+	/* block array */
+	struct ras_manager *objs;
+
+	/* gpu recovery */
+	struct work_struct recovery_work;
+	atomic_t in_recovery;
+	struct amdgpu_device *adev;
+	/* error handler data */
+	struct ras_err_handler_data *eh_data;
+	struct mutex recovery_lock;
+
+	uint32_t flags;
+};
+
+/* interfaces for IP */
+
+struct ras_fs_if {
+	struct ras_common_if head;
+	char sysfs_name[32];
+	char debugfs_name[32];
+};
+
+struct ras_query_if {
+	struct ras_common_if head;
+	unsigned long ue_count;
+	unsigned long ce_count;
+};
+
+struct ras_inject_if {
+	struct ras_common_if head;
+	uint64_t address;
+	uint64_t value;
+};
+
+struct ras_cure_if {
+	struct ras_common_if head;
+	uint64_t address;
+};
+
+struct ras_ih_if {
+	struct ras_common_if head;
+	ras_ih_cb cb;
+};
+
+struct ras_dispatch_if {
+	struct ras_common_if head;
+	struct amdgpu_iv_entry *entry;
+};
+
+struct ras_debug_if {
+	union {
+		struct ras_common_if head;
+		struct ras_inject_if inject;
+	};
+	int op;
+};
+/* work flow
+ * vbios
+ * 1: ras feature enable (enabled by default)
+ * psp
+ * 2: ras framework init (in ip_init)
+ * IP
+ * 3: IH add
+ * 4: debugfs/sysfs create
+ * 5: query/inject
+ * 6: debugfs/sysfs remove
+ * 7: IH remove
+ * 8: feature disable
+ */
+
+#define amdgpu_ras_get_context(adev)		((adev)->psp.ras.ras)
+#define amdgpu_ras_set_context(adev, ras_con)	((adev)->psp.ras.ras = (ras_con))
+
+/* check if ras is supported on block, say, sdma, gfx */
+static inline int amdgpu_ras_is_supported(struct amdgpu_device *adev,
+		unsigned int block)
+{
+	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
+
+	return ras && (ras->supported & (1 << block));
+}
+
+int amdgpu_ras_query_error_count(struct amdgpu_device *adev,
+		bool is_ce);
+
+/* error handling functions */
+int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev,
+		unsigned long *bps, int pages);
+
+int amdgpu_ras_reserve_bad_pages(struct amdgpu_device *adev);
+
+static inline int amdgpu_ras_reset_gpu(struct amdgpu_device *adev,
+		bool is_baco)
+{
+	/* remove me when gpu reset works on vega20 A1. */
+#if 0
+	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
+
+	if (atomic_cmpxchg(&ras->in_recovery, 0, 1) == 0)
+		schedule_work(&ras->recovery_work);
+#endif
+	return 0;
+}
+
+static inline enum ta_ras_block
+amdgpu_ras_block_to_ta(enum amdgpu_ras_block block) {
+	switch (block) {
+	case AMDGPU_RAS_BLOCK__UMC:
+		return TA_RAS_BLOCK__UMC;
+	case AMDGPU_RAS_BLOCK__SDMA:
+		return TA_RAS_BLOCK__SDMA;
+	case AMDGPU_RAS_BLOCK__GFX:
+		return TA_RAS_BLOCK__GFX;
+	case AMDGPU_RAS_BLOCK__MMHUB:
+		return TA_RAS_BLOCK__MMHUB;
+	case AMDGPU_RAS_BLOCK__ATHUB:
+		return TA_RAS_BLOCK__ATHUB;
+	case AMDGPU_RAS_BLOCK__PCIE_BIF:
+		return TA_RAS_BLOCK__PCIE_BIF;
+	case AMDGPU_RAS_BLOCK__HDP:
+		return TA_RAS_BLOCK__HDP;
+	case AMDGPU_RAS_BLOCK__XGMI_WAFL:
+		return TA_RAS_BLOCK__XGMI_WAFL;
+	case AMDGPU_RAS_BLOCK__DF:
+		return TA_RAS_BLOCK__DF;
+	case AMDGPU_RAS_BLOCK__SMN:
+		return TA_RAS_BLOCK__SMN;
+	case AMDGPU_RAS_BLOCK__SEM:
+		return TA_RAS_BLOCK__SEM;
+	case AMDGPU_RAS_BLOCK__MP0:
+		return TA_RAS_BLOCK__MP0;
+	case AMDGPU_RAS_BLOCK__MP1:
+		return TA_RAS_BLOCK__MP1;
+	case AMDGPU_RAS_BLOCK__FUSE:
+		return TA_RAS_BLOCK__FUSE;
+	default:
+		WARN_ONCE(1, "RAS ERROR: unexpected block id %d\n", block);
+		return TA_RAS_BLOCK__UMC;
+	}
+}
+
+static inline enum ta_ras_error_type
+amdgpu_ras_error_to_ta(enum amdgpu_ras_error_type error) {
+	switch (error) {
+	case AMDGPU_RAS_ERROR__NONE:
+		return TA_RAS_ERROR__NONE;
+	case AMDGPU_RAS_ERROR__PARITY:
+		return TA_RAS_ERROR__PARITY;
+	case AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE:
+		return TA_RAS_ERROR__SINGLE_CORRECTABLE;
+	case AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE:
+		return TA_RAS_ERROR__MULTI_UNCORRECTABLE;
+	case AMDGPU_RAS_ERROR__POISON:
+		return TA_RAS_ERROR__POISON;
+	default:
+		WARN_ONCE(1, "RAS ERROR: unexpected error type %d\n", error);
+		return TA_RAS_ERROR__NONE;
+	}
+}
+
+/* called in ip_init and ip_fini */
+int amdgpu_ras_init(struct amdgpu_device *adev);
+void amdgpu_ras_post_init(struct amdgpu_device *adev);
+int amdgpu_ras_fini(struct amdgpu_device *adev);
+int amdgpu_ras_pre_fini(struct amdgpu_device *adev);
+
+int amdgpu_ras_feature_enable(struct amdgpu_device *adev,
+		struct ras_common_if *head, bool enable);
+
+int amdgpu_ras_feature_enable_on_boot(struct amdgpu_device *adev,
+		struct ras_common_if *head, bool enable);
+
+int amdgpu_ras_sysfs_create(struct amdgpu_device *adev,
+		struct ras_fs_if *head);
+
+int amdgpu_ras_sysfs_remove(struct amdgpu_device *adev,
+		struct ras_common_if *head);
+
+int amdgpu_ras_debugfs_create(struct amdgpu_device *adev,
+		struct ras_fs_if *head);
+
+int amdgpu_ras_debugfs_remove(struct amdgpu_device *adev,
+		struct ras_common_if *head);
+
+int amdgpu_ras_error_query(struct amdgpu_device *adev,
+		struct ras_query_if *info);
+
+int amdgpu_ras_error_inject(struct amdgpu_device *adev,
+		struct ras_inject_if *info);
+
+int amdgpu_ras_interrupt_add_handler(struct amdgpu_device *adev,
+		struct ras_ih_if *info);
+
+int amdgpu_ras_interrupt_remove_handler(struct amdgpu_device *adev,
+		struct ras_ih_if *info);
+
+int amdgpu_ras_interrupt_dispatch(struct amdgpu_device *adev,
+		struct ras_dispatch_if *info);
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
index 335a0edf114b..8f5026c123ef 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
@@ -248,6 +248,8 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
 	 */
 	if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
 		sched_hw_submission = max(sched_hw_submission, 256);
+	else if (ring == &adev->sdma.instance[0].page)
+		sched_hw_submission = 256;
 
 	if (ring->adev == NULL) {
 		if (adev->num_rings >= AMDGPU_MAX_RINGS)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h
index 16b1a6ae5ba6..1ba9ba3b54f7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h
@@ -28,9 +28,8 @@
 #define AMDGPU_MAX_SDMA_INSTANCES		2
 
 enum amdgpu_sdma_irq {
-	AMDGPU_SDMA_IRQ_TRAP0 = 0,
-	AMDGPU_SDMA_IRQ_TRAP1,
-
+	AMDGPU_SDMA_IRQ_INSTANCE0  = 0,
+	AMDGPU_SDMA_IRQ_INSTANCE1,
 	AMDGPU_SDMA_IRQ_LAST
 };
 
@@ -49,9 +48,11 @@ struct amdgpu_sdma {
 	struct amdgpu_sdma_instance instance[AMDGPU_MAX_SDMA_INSTANCES];
 	struct amdgpu_irq_src	trap_irq;
 	struct amdgpu_irq_src	illegal_inst_irq;
+	struct amdgpu_irq_src	ecc_irq;
 	int			num_instances;
 	uint32_t                    srbm_soft_reset;
 	bool			has_page_queue;
+	struct ras_common_if	*ras_if;
 };
 
 /*
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 73e71e61dc99..0c52d1f9fe0f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -50,8 +50,6 @@
 #include "amdgpu_sdma.h"
 #include "bif/bif_4_1_d.h"
 
-#define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT)
-
 static int amdgpu_map_buffer(struct ttm_buffer_object *bo,
 			     struct ttm_mem_reg *mem, unsigned num_pages,
 			     uint64_t offset, unsigned window,
@@ -1424,6 +1422,13 @@ static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
 	struct dma_fence *f;
 	int i;
 
+	/* Don't evict VM page tables while they are busy, otherwise we can't
+	 * cleanly handle page faults.
+	 */
+	if (bo->type == ttm_bo_type_kernel &&
+	    !reservation_object_test_signaled_rcu(bo->resv, true))
+		return false;
+
 	/* If bo is a KFD BO, check if the bo belongs to the current process.
 	 * If true, then return false as any KFD process needs all its BOs to
 	 * be resident to run successfully
@@ -1671,7 +1676,6 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
 	r = ttm_bo_device_init(&adev->mman.bdev,
 			       &amdgpu_bo_driver,
 			       adev->ddev->anon_inode->i_mapping,
-			       DRM_FILE_PAGE_OFFSET,
 			       adev->need_dma32);
 	if (r) {
 		DRM_ERROR("failed initializing buffer object driver(%d).\n", r);
@@ -1877,14 +1881,9 @@ void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev, bool enable)
 
 int amdgpu_mmap(struct file *filp, struct vm_area_struct *vma)
 {
-	struct drm_file *file_priv;
-	struct amdgpu_device *adev;
-
-	if (unlikely(vma->vm_pgoff < DRM_FILE_PAGE_OFFSET))
-		return -EINVAL;
+	struct drm_file *file_priv = filp->private_data;
+	struct amdgpu_device *adev = file_priv->minor->dev->dev_private;
 
-	file_priv = filp->private_data;
-	adev = file_priv->minor->dev->dev_private;
 	if (adev == NULL)
 		return -EINVAL;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
index 462a04e0f5e6..7d484fad3909 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
@@ -36,6 +36,7 @@ void amdgpu_virt_init_setting(struct amdgpu_device *adev)
 	/* enable virtual display */
 	adev->mode_info.num_crtc = 1;
 	adev->enable_virtual_display = true;
+	adev->ddev->driver->driver_features &= ~DRIVER_ATOMIC;
 	adev->cg_flags = 0;
 	adev->pg_flags = 0;
 }
@@ -375,4 +376,53 @@ void amdgpu_virt_init_data_exchange(struct amdgpu_device *adev)
 	}
 }
 
+static uint32_t parse_clk(char *buf, bool min)
+{
+        char *ptr = buf;
+        uint32_t clk = 0;
+
+        do {
+                ptr = strchr(ptr, ':');
+                if (!ptr)
+                        break;
+                ptr+=2;
+                clk = simple_strtoul(ptr, NULL, 10);
+        } while (!min);
+
+        return clk * 100;
+}
+
+uint32_t amdgpu_virt_get_sclk(struct amdgpu_device *adev, bool lowest)
+{
+	char *buf = NULL;
+	uint32_t clk = 0;
+
+	buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	adev->virt.ops->get_pp_clk(adev, PP_SCLK, buf);
+	clk = parse_clk(buf, lowest);
+
+	kfree(buf);
+
+	return clk;
+}
+
+uint32_t amdgpu_virt_get_mclk(struct amdgpu_device *adev, bool lowest)
+{
+	char *buf = NULL;
+	uint32_t clk = 0;
+
+	buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	adev->virt.ops->get_pp_clk(adev, PP_MCLK, buf);
+	clk = parse_clk(buf, lowest);
+
+	kfree(buf);
+
+	return clk;
+}
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
index 722deefc0a7e..584947b7ccf3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
@@ -57,6 +57,8 @@ struct amdgpu_virt_ops {
 	int (*reset_gpu)(struct amdgpu_device *adev);
 	int (*wait_reset)(struct amdgpu_device *adev);
 	void (*trans_msg)(struct amdgpu_device *adev, u32 req, u32 data1, u32 data2, u32 data3);
+	int (*get_pp_clk)(struct amdgpu_device *adev, u32 type, char *buf);
+	int (*force_dpm_level)(struct amdgpu_device *adev, u32 level);
 };
 
 /*
@@ -83,6 +85,8 @@ enum AMDGIM_FEATURE_FLAG {
 	AMDGIM_FEATURE_GIM_LOAD_UCODES   = 0x2,
 	/* VRAM LOST by GIM */
 	AMDGIM_FEATURE_GIM_FLR_VRAMLOST = 0x4,
+	/* HW PERF SIM in GIM */
+	AMDGIM_FEATURE_HW_PERF_SIMULATION = (1 << 3),
 };
 
 struct amd_sriov_msg_pf2vf_info_header {
@@ -252,6 +256,8 @@ struct amdgpu_virt {
 	struct amdgpu_vf_error_buffer   vf_errors;
 	struct amdgpu_virt_fw_reserve	fw_reserve;
 	uint32_t gim_feature;
+	/* protect DPM events to GIM */
+	struct mutex                    dpm_mutex;
 };
 
 #define amdgpu_sriov_enabled(adev) \
@@ -278,6 +284,9 @@ static inline bool is_virtual_machine(void)
 #endif
 }
 
+#define amdgim_is_hwperf(adev) \
+	((adev)->virt.gim_feature & AMDGIM_FEATURE_HW_PERF_SIMULATION)
+
 bool amdgpu_virt_mmio_blocked(struct amdgpu_device *adev);
 void amdgpu_virt_init_setting(struct amdgpu_device *adev);
 uint32_t amdgpu_virt_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);
@@ -295,5 +304,7 @@ int amdgpu_virt_fw_reserve_get_checksum(void *obj, unsigned long obj_size,
 					unsigned int key,
 					unsigned int chksum);
 void amdgpu_virt_init_data_exchange(struct amdgpu_device *adev);
+uint32_t amdgpu_virt_get_sclk(struct amdgpu_device *adev, bool lowest);
+uint32_t amdgpu_virt_get_mclk(struct amdgpu_device *adev, bool lowest);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 16fcb56c232b..a07c85815b7a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -34,6 +34,7 @@
 #include "amdgpu_trace.h"
 #include "amdgpu_amdkfd.h"
 #include "amdgpu_gmc.h"
+#include "amdgpu_xgmi.h"
 
 /**
  * DOC: GPUVM
@@ -66,50 +67,6 @@ INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping, rb, uint64_t, __subtree_last,
 #undef LAST
 
 /**
- * struct amdgpu_pte_update_params - Local structure
- *
- * Encapsulate some VM table update parameters to reduce
- * the number of function parameters
- *
- */
-struct amdgpu_pte_update_params {
-
-	/**
-	 * @adev: amdgpu device we do this update for
-	 */
-	struct amdgpu_device *adev;
-
-	/**
-	 * @vm: optional amdgpu_vm we do this update for
-	 */
-	struct amdgpu_vm *vm;
-
-	/**
-	 * @src: address where to copy page table entries from
-	 */
-	uint64_t src;
-
-	/**
-	 * @ib: indirect buffer to fill with commands
-	 */
-	struct amdgpu_ib *ib;
-
-	/**
-	 * @func: Function which actually does the update
-	 */
-	void (*func)(struct amdgpu_pte_update_params *params,
-		     struct amdgpu_bo *bo, uint64_t pe,
-		     uint64_t addr, unsigned count, uint32_t incr,
-		     uint64_t flags);
-	/**
-	 * @pages_addr:
-	 *
-	 * DMA addresses to use for mapping, used during VM update by CPU
-	 */
-	dma_addr_t *pages_addr;
-};
-
-/**
  * struct amdgpu_prt_cb - Helper to disable partial resident texture feature from a fence callback
  */
 struct amdgpu_prt_cb {
@@ -183,6 +140,22 @@ static unsigned amdgpu_vm_num_entries(struct amdgpu_device *adev,
 }
 
 /**
+ * amdgpu_vm_num_ats_entries - return the number of ATS entries in the root PD
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Returns:
+ * The number of entries in the root page directory which needs the ATS setting.
+ */
+static unsigned amdgpu_vm_num_ats_entries(struct amdgpu_device *adev)
+{
+	unsigned shift;
+
+	shift = amdgpu_vm_level_shift(adev, adev->vm_manager.root_level);
+	return AMDGPU_GMC_HOLE_START >> (shift + AMDGPU_GPU_PAGE_SHIFT);
+}
+
+/**
  * amdgpu_vm_entries_mask - the mask to get the entry number of a PD/PT
  *
  * @adev: amdgpu_device pointer
@@ -333,7 +306,7 @@ static void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
 		return;
 
 	vm->bulk_moveable = false;
-	if (bo->tbo.type == ttm_bo_type_kernel)
+	if (bo->tbo.type == ttm_bo_type_kernel && bo->parent)
 		amdgpu_vm_bo_relocated(base);
 	else
 		amdgpu_vm_bo_idle(base);
@@ -505,61 +478,39 @@ static void amdgpu_vm_pt_next(struct amdgpu_device *adev,
 }
 
 /**
- * amdgpu_vm_pt_first_leaf - get first leaf PD/PT
+ * amdgpu_vm_pt_first_dfs - start a deep first search
  *
- * @adev: amdgpu_device pointer
+ * @adev: amdgpu_device structure
  * @vm: amdgpu_vm structure
- * @start: start addr of the walk
  * @cursor: state to initialize
  *
- * Start a walk and go directly to the leaf node.
- */
-static void amdgpu_vm_pt_first_leaf(struct amdgpu_device *adev,
-				    struct amdgpu_vm *vm, uint64_t start,
-				    struct amdgpu_vm_pt_cursor *cursor)
-{
-	amdgpu_vm_pt_start(adev, vm, start, cursor);
-	while (amdgpu_vm_pt_descendant(adev, cursor));
-}
-
-/**
- * amdgpu_vm_pt_next_leaf - get next leaf PD/PT
- *
- * @adev: amdgpu_device pointer
- * @cursor: current state
- *
- * Walk the PD/PT tree to the next leaf node.
+ * Starts a deep first traversal of the PD/PT tree.
  */
-static void amdgpu_vm_pt_next_leaf(struct amdgpu_device *adev,
+static void amdgpu_vm_pt_first_dfs(struct amdgpu_device *adev,
+				   struct amdgpu_vm *vm,
+				   struct amdgpu_vm_pt_cursor *start,
 				   struct amdgpu_vm_pt_cursor *cursor)
 {
-	amdgpu_vm_pt_next(adev, cursor);
-	if (cursor->pfn != ~0ll)
-		while (amdgpu_vm_pt_descendant(adev, cursor));
+	if (start)
+		*cursor = *start;
+	else
+		amdgpu_vm_pt_start(adev, vm, 0, cursor);
+	while (amdgpu_vm_pt_descendant(adev, cursor));
 }
 
 /**
- * for_each_amdgpu_vm_pt_leaf - walk over all leaf PDs/PTs in the hierarchy
- */
-#define for_each_amdgpu_vm_pt_leaf(adev, vm, start, end, cursor)		\
-	for (amdgpu_vm_pt_first_leaf((adev), (vm), (start), &(cursor));		\
-	     (cursor).pfn <= end; amdgpu_vm_pt_next_leaf((adev), &(cursor)))
-
-/**
- * amdgpu_vm_pt_first_dfs - start a deep first search
+ * amdgpu_vm_pt_continue_dfs - check if the deep first search should continue
  *
- * @adev: amdgpu_device structure
- * @vm: amdgpu_vm structure
- * @cursor: state to initialize
+ * @start: starting point for the search
+ * @entry: current entry
  *
- * Starts a deep first traversal of the PD/PT tree.
+ * Returns:
+ * True when the search should continue, false otherwise.
  */
-static void amdgpu_vm_pt_first_dfs(struct amdgpu_device *adev,
-				   struct amdgpu_vm *vm,
-				   struct amdgpu_vm_pt_cursor *cursor)
+static bool amdgpu_vm_pt_continue_dfs(struct amdgpu_vm_pt_cursor *start,
+				      struct amdgpu_vm_pt *entry)
 {
-	amdgpu_vm_pt_start(adev, vm, 0, cursor);
-	while (amdgpu_vm_pt_descendant(adev, cursor));
+	return entry && (!start || entry != start->entry);
 }
 
 /**
@@ -587,11 +538,11 @@ static void amdgpu_vm_pt_next_dfs(struct amdgpu_device *adev,
 /**
  * for_each_amdgpu_vm_pt_dfs_safe - safe deep first search of all PDs/PTs
  */
-#define for_each_amdgpu_vm_pt_dfs_safe(adev, vm, cursor, entry)			\
-	for (amdgpu_vm_pt_first_dfs((adev), (vm), &(cursor)),			\
+#define for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry)		\
+	for (amdgpu_vm_pt_first_dfs((adev), (vm), (start), &(cursor)),		\
 	     (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor));\
-	     (entry); (entry) = (cursor).entry,					\
-	     amdgpu_vm_pt_next_dfs((adev), &(cursor)))
+	     amdgpu_vm_pt_continue_dfs((start), (entry));			\
+	     (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor)))
 
 /**
  * amdgpu_vm_get_pd_bo - add the VM PD to a validation list
@@ -712,18 +663,11 @@ int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 		if (bo->tbo.type != ttm_bo_type_kernel) {
 			amdgpu_vm_bo_moved(bo_base);
 		} else {
-			if (vm->use_cpu_for_update)
-				r = amdgpu_bo_kmap(bo, NULL);
+			vm->update_funcs->map_table(bo);
+			if (bo->parent)
+				amdgpu_vm_bo_relocated(bo_base);
 			else
-				r = amdgpu_ttm_alloc_gart(&bo->tbo);
-			if (r)
-				break;
-			if (bo->shadow) {
-				r = amdgpu_ttm_alloc_gart(&bo->shadow->tbo);
-				if (r)
-					break;
-			}
-			amdgpu_vm_bo_relocated(bo_base);
+				amdgpu_vm_bo_idle(bo_base);
 		}
 	}
 
@@ -751,8 +695,6 @@ bool amdgpu_vm_ready(struct amdgpu_vm *vm)
  * @adev: amdgpu_device pointer
  * @vm: VM to clear BO from
  * @bo: BO to clear
- * @level: level this BO is at
- * @pte_support_ats: indicate ATS support from PTE
  *
  * Root PD needs to be reserved when calling this.
  *
@@ -760,99 +702,112 @@ bool amdgpu_vm_ready(struct amdgpu_vm *vm)
  * 0 on success, errno otherwise.
  */
 static int amdgpu_vm_clear_bo(struct amdgpu_device *adev,
-			      struct amdgpu_vm *vm, struct amdgpu_bo *bo,
-			      unsigned level, bool pte_support_ats)
+			      struct amdgpu_vm *vm,
+			      struct amdgpu_bo *bo)
 {
 	struct ttm_operation_ctx ctx = { true, false };
-	struct dma_fence *fence = NULL;
+	unsigned level = adev->vm_manager.root_level;
+	struct amdgpu_vm_update_params params;
+	struct amdgpu_bo *ancestor = bo;
 	unsigned entries, ats_entries;
-	struct amdgpu_ring *ring;
-	struct amdgpu_job *job;
 	uint64_t addr;
 	int r;
 
+	/* Figure out our place in the hierarchy */
+	if (ancestor->parent) {
+		++level;
+		while (ancestor->parent->parent) {
+			++level;
+			ancestor = ancestor->parent;
+		}
+	}
+
 	entries = amdgpu_bo_size(bo) / 8;
+	if (!vm->pte_support_ats) {
+		ats_entries = 0;
+
+	} else if (!bo->parent) {
+		ats_entries = amdgpu_vm_num_ats_entries(adev);
+		ats_entries = min(ats_entries, entries);
+		entries -= ats_entries;
+
+	} else {
+		struct amdgpu_vm_pt *pt;
 
-	if (pte_support_ats) {
-		if (level == adev->vm_manager.root_level) {
-			ats_entries = amdgpu_vm_level_shift(adev, level);
-			ats_entries += AMDGPU_GPU_PAGE_SHIFT;
-			ats_entries = AMDGPU_GMC_HOLE_START >> ats_entries;
-			ats_entries = min(ats_entries, entries);
-			entries -= ats_entries;
+		pt = container_of(ancestor->vm_bo, struct amdgpu_vm_pt, base);
+		ats_entries = amdgpu_vm_num_ats_entries(adev);
+		if ((pt - vm->root.entries) >= ats_entries) {
+			ats_entries = 0;
 		} else {
 			ats_entries = entries;
 			entries = 0;
 		}
-	} else {
-		ats_entries = 0;
 	}
 
-	ring = container_of(vm->entity.rq->sched, struct amdgpu_ring, sched);
-
 	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 	if (r)
-		goto error;
+		return r;
+
+	if (bo->shadow) {
+		r = ttm_bo_validate(&bo->shadow->tbo, &bo->shadow->placement,
+				    &ctx);
+		if (r)
+			return r;
+	}
 
-	r = amdgpu_ttm_alloc_gart(&bo->tbo);
+	r = vm->update_funcs->map_table(bo);
 	if (r)
 		return r;
 
-	r = amdgpu_job_alloc_with_ib(adev, 64, &job);
+	memset(&params, 0, sizeof(params));
+	params.adev = adev;
+	params.vm = vm;
+
+	r = vm->update_funcs->prepare(&params, AMDGPU_FENCE_OWNER_KFD, NULL);
 	if (r)
-		goto error;
+		return r;
 
-	addr = amdgpu_bo_gpu_offset(bo);
+	addr = 0;
 	if (ats_entries) {
-		uint64_t ats_value;
+		uint64_t value = 0, flags;
 
-		ats_value = AMDGPU_PTE_DEFAULT_ATC;
-		if (level != AMDGPU_VM_PTB)
-			ats_value |= AMDGPU_PDE_PTE;
+		flags = AMDGPU_PTE_DEFAULT_ATC;
+		if (level != AMDGPU_VM_PTB) {
+			/* Handle leaf PDEs as PTEs */
+			flags |= AMDGPU_PDE_PTE;
+			amdgpu_gmc_get_vm_pde(adev, level, &value, &flags);
+		}
+
+		r = vm->update_funcs->update(&params, bo, addr, 0, ats_entries,
+					     value, flags);
+		if (r)
+			return r;
 
-		amdgpu_vm_set_pte_pde(adev, &job->ibs[0], addr, 0,
-				      ats_entries, 0, ats_value);
 		addr += ats_entries * 8;
 	}
 
 	if (entries) {
-		uint64_t value = 0;
-
-		/* Workaround for fault priority problem on GMC9 */
-		if (level == AMDGPU_VM_PTB && adev->asic_type >= CHIP_VEGA10)
-			value = AMDGPU_PTE_EXECUTABLE;
+		uint64_t value = 0, flags = 0;
+
+		if (adev->asic_type >= CHIP_VEGA10) {
+			if (level != AMDGPU_VM_PTB) {
+				/* Handle leaf PDEs as PTEs */
+				flags |= AMDGPU_PDE_PTE;
+				amdgpu_gmc_get_vm_pde(adev, level,
+						      &value, &flags);
+			} else {
+				/* Workaround for fault priority problem on GMC9 */
+				flags = AMDGPU_PTE_EXECUTABLE;
+			}
+		}
 
-		amdgpu_vm_set_pte_pde(adev, &job->ibs[0], addr, 0,
-				      entries, 0, value);
+		r = vm->update_funcs->update(&params, bo, addr, 0, entries,
+					     value, flags);
+		if (r)
+			return r;
 	}
 
-	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
-
-	WARN_ON(job->ibs[0].length_dw > 64);
-	r = amdgpu_sync_resv(adev, &job->sync, bo->tbo.resv,
-			     AMDGPU_FENCE_OWNER_KFD, false);
-	if (r)
-		goto error_free;
-
-	r = amdgpu_job_submit(job, &vm->entity, AMDGPU_FENCE_OWNER_UNDEFINED,
-			      &fence);
-	if (r)
-		goto error_free;
-
-	amdgpu_bo_fence(bo, fence, true);
-	dma_fence_put(fence);
-
-	if (bo->shadow)
-		return amdgpu_vm_clear_bo(adev, vm, bo->shadow,
-					  level, pte_support_ats);
-
-	return 0;
-
-error_free:
-	amdgpu_job_free(job);
-
-error:
-	return r;
+	return vm->update_funcs->commit(&params, NULL);
 }
 
 /**
@@ -883,89 +838,56 @@ static void amdgpu_vm_bo_param(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 }
 
 /**
- * amdgpu_vm_alloc_pts - Allocate page tables.
+ * amdgpu_vm_alloc_pts - Allocate a specific page table
  *
  * @adev: amdgpu_device pointer
  * @vm: VM to allocate page tables for
- * @saddr: Start address which needs to be allocated
- * @size: Size from start address we need.
+ * @cursor: Which page table to allocate
  *
- * Make sure the page directories and page tables are allocated
+ * Make sure a specific page table or directory is allocated.
  *
  * Returns:
- * 0 on success, errno otherwise.
+ * 1 if page table needed to be allocated, 0 if page table was already
+ * allocated, negative errno if an error occurred.
  */
-int amdgpu_vm_alloc_pts(struct amdgpu_device *adev,
-			struct amdgpu_vm *vm,
-			uint64_t saddr, uint64_t size)
+static int amdgpu_vm_alloc_pts(struct amdgpu_device *adev,
+			       struct amdgpu_vm *vm,
+			       struct amdgpu_vm_pt_cursor *cursor)
 {
-	struct amdgpu_vm_pt_cursor cursor;
+	struct amdgpu_vm_pt *entry = cursor->entry;
+	struct amdgpu_bo_param bp;
 	struct amdgpu_bo *pt;
-	bool ats = false;
-	uint64_t eaddr;
 	int r;
 
-	/* validate the parameters */
-	if (saddr & AMDGPU_GPU_PAGE_MASK || size & AMDGPU_GPU_PAGE_MASK)
-		return -EINVAL;
-
-	eaddr = saddr + size - 1;
-
-	if (vm->pte_support_ats)
-		ats = saddr < AMDGPU_GMC_HOLE_START;
-
-	saddr /= AMDGPU_GPU_PAGE_SIZE;
-	eaddr /= AMDGPU_GPU_PAGE_SIZE;
+	if (cursor->level < AMDGPU_VM_PTB && !entry->entries) {
+		unsigned num_entries;
 
-	if (eaddr >= adev->vm_manager.max_pfn) {
-		dev_err(adev->dev, "va above limit (0x%08llX >= 0x%08llX)\n",
-			eaddr, adev->vm_manager.max_pfn);
-		return -EINVAL;
+		num_entries = amdgpu_vm_num_entries(adev, cursor->level);
+		entry->entries = kvmalloc_array(num_entries,
+						sizeof(*entry->entries),
+						GFP_KERNEL | __GFP_ZERO);
+		if (!entry->entries)
+			return -ENOMEM;
 	}
 
-	for_each_amdgpu_vm_pt_leaf(adev, vm, saddr, eaddr, cursor) {
-		struct amdgpu_vm_pt *entry = cursor.entry;
-		struct amdgpu_bo_param bp;
-
-		if (cursor.level < AMDGPU_VM_PTB) {
-			unsigned num_entries;
-
-			num_entries = amdgpu_vm_num_entries(adev, cursor.level);
-			entry->entries = kvmalloc_array(num_entries,
-							sizeof(*entry->entries),
-							GFP_KERNEL |
-							__GFP_ZERO);
-			if (!entry->entries)
-				return -ENOMEM;
-		}
-
-
-		if (entry->base.bo)
-			continue;
-
-		amdgpu_vm_bo_param(adev, vm, cursor.level, &bp);
+	if (entry->base.bo)
+		return 0;
 
-		r = amdgpu_bo_create(adev, &bp, &pt);
-		if (r)
-			return r;
+	amdgpu_vm_bo_param(adev, vm, cursor->level, &bp);
 
-		if (vm->use_cpu_for_update) {
-			r = amdgpu_bo_kmap(pt, NULL);
-			if (r)
-				goto error_free_pt;
-		}
-
-		/* Keep a reference to the root directory to avoid
-		* freeing them up in the wrong order.
-		*/
-		pt->parent = amdgpu_bo_ref(cursor.parent->base.bo);
+	r = amdgpu_bo_create(adev, &bp, &pt);
+	if (r)
+		return r;
 
-		amdgpu_vm_bo_base_init(&entry->base, vm, pt);
+	/* Keep a reference to the root directory to avoid
+	 * freeing them up in the wrong order.
+	 */
+	pt->parent = amdgpu_bo_ref(cursor->parent->base.bo);
+	amdgpu_vm_bo_base_init(&entry->base, vm, pt);
 
-		r = amdgpu_vm_clear_bo(adev, vm, pt, cursor.level, ats);
-		if (r)
-			goto error_free_pt;
-	}
+	r = amdgpu_vm_clear_bo(adev, vm, pt);
+	if (r)
+		goto error_free_pt;
 
 	return 0;
 
@@ -976,31 +898,45 @@ error_free_pt:
 }
 
 /**
+ * amdgpu_vm_free_table - fre one PD/PT
+ *
+ * @entry: PDE to free
+ */
+static void amdgpu_vm_free_table(struct amdgpu_vm_pt *entry)
+{
+	if (entry->base.bo) {
+		entry->base.bo->vm_bo = NULL;
+		list_del(&entry->base.vm_status);
+		amdgpu_bo_unref(&entry->base.bo->shadow);
+		amdgpu_bo_unref(&entry->base.bo);
+	}
+	kvfree(entry->entries);
+	entry->entries = NULL;
+}
+
+/**
  * amdgpu_vm_free_pts - free PD/PT levels
  *
  * @adev: amdgpu device structure
  * @vm: amdgpu vm structure
+ * @start: optional cursor where to start freeing PDs/PTs
  *
  * Free the page directory or page table level and all sub levels.
  */
 static void amdgpu_vm_free_pts(struct amdgpu_device *adev,
-			       struct amdgpu_vm *vm)
+			       struct amdgpu_vm *vm,
+			       struct amdgpu_vm_pt_cursor *start)
 {
 	struct amdgpu_vm_pt_cursor cursor;
 	struct amdgpu_vm_pt *entry;
 
-	for_each_amdgpu_vm_pt_dfs_safe(adev, vm, cursor, entry) {
+	vm->bulk_moveable = false;
 
-		if (entry->base.bo) {
-			entry->base.bo->vm_bo = NULL;
-			list_del(&entry->base.vm_status);
-			amdgpu_bo_unref(&entry->base.bo->shadow);
-			amdgpu_bo_unref(&entry->base.bo);
-		}
-		kvfree(entry->entries);
-	}
+	for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry)
+		amdgpu_vm_free_table(entry);
 
-	BUG_ON(vm->root.base.bo);
+	if (start)
+		amdgpu_vm_free_table(start->entry);
 }
 
 /**
@@ -1212,66 +1148,6 @@ struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
 }
 
 /**
- * amdgpu_vm_do_set_ptes - helper to call the right asic function
- *
- * @params: see amdgpu_pte_update_params definition
- * @bo: PD/PT to update
- * @pe: addr of the page entry
- * @addr: dst addr to write into pe
- * @count: number of page entries to update
- * @incr: increase next addr by incr bytes
- * @flags: hw access flags
- *
- * Traces the parameters and calls the right asic functions
- * to setup the page table using the DMA.
- */
-static void amdgpu_vm_do_set_ptes(struct amdgpu_pte_update_params *params,
-				  struct amdgpu_bo *bo,
-				  uint64_t pe, uint64_t addr,
-				  unsigned count, uint32_t incr,
-				  uint64_t flags)
-{
-	pe += amdgpu_bo_gpu_offset(bo);
-	trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags);
-
-	if (count < 3) {
-		amdgpu_vm_write_pte(params->adev, params->ib, pe,
-				    addr | flags, count, incr);
-
-	} else {
-		amdgpu_vm_set_pte_pde(params->adev, params->ib, pe, addr,
-				      count, incr, flags);
-	}
-}
-
-/**
- * amdgpu_vm_do_copy_ptes - copy the PTEs from the GART
- *
- * @params: see amdgpu_pte_update_params definition
- * @bo: PD/PT to update
- * @pe: addr of the page entry
- * @addr: dst addr to write into pe
- * @count: number of page entries to update
- * @incr: increase next addr by incr bytes
- * @flags: hw access flags
- *
- * Traces the parameters and calls the DMA function to copy the PTEs.
- */
-static void amdgpu_vm_do_copy_ptes(struct amdgpu_pte_update_params *params,
-				   struct amdgpu_bo *bo,
-				   uint64_t pe, uint64_t addr,
-				   unsigned count, uint32_t incr,
-				   uint64_t flags)
-{
-	uint64_t src = (params->src + (addr >> 12) * 8);
-
-	pe += amdgpu_bo_gpu_offset(bo);
-	trace_amdgpu_vm_copy_ptes(pe, src, count);
-
-	amdgpu_vm_copy_pte(params->adev, params->ib, pe, src, count);
-}
-
-/**
  * amdgpu_vm_map_gart - Resolve gart mapping of addr
  *
  * @pages_addr: optional DMA address to use for lookup
@@ -1283,7 +1159,7 @@ static void amdgpu_vm_do_copy_ptes(struct amdgpu_pte_update_params *params,
  * Returns:
  * The pointer for the page table entry.
  */
-static uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr)
+uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr)
 {
 	uint64_t result;
 
@@ -1298,88 +1174,31 @@ static uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr)
 	return result;
 }
 
-/**
- * amdgpu_vm_cpu_set_ptes - helper to update page tables via CPU
- *
- * @params: see amdgpu_pte_update_params definition
- * @bo: PD/PT to update
- * @pe: kmap addr of the page entry
- * @addr: dst addr to write into pe
- * @count: number of page entries to update
- * @incr: increase next addr by incr bytes
- * @flags: hw access flags
- *
- * Write count number of PT/PD entries directly.
- */
-static void amdgpu_vm_cpu_set_ptes(struct amdgpu_pte_update_params *params,
-				   struct amdgpu_bo *bo,
-				   uint64_t pe, uint64_t addr,
-				   unsigned count, uint32_t incr,
-				   uint64_t flags)
-{
-	unsigned int i;
-	uint64_t value;
-
-	pe += (unsigned long)amdgpu_bo_kptr(bo);
-
-	trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags);
-
-	for (i = 0; i < count; i++) {
-		value = params->pages_addr ?
-			amdgpu_vm_map_gart(params->pages_addr, addr) :
-			addr;
-		amdgpu_gmc_set_pte_pde(params->adev, (void *)(uintptr_t)pe,
-				       i, value, flags);
-		addr += incr;
-	}
-}
-
-/**
- * amdgpu_vm_update_func - helper to call update function
- *
- * Calls the update function for both the given BO as well as its shadow.
- */
-static void amdgpu_vm_update_func(struct amdgpu_pte_update_params *params,
-				  struct amdgpu_bo *bo,
-				  uint64_t pe, uint64_t addr,
-				  unsigned count, uint32_t incr,
-				  uint64_t flags)
-{
-	if (bo->shadow)
-		params->func(params, bo->shadow, pe, addr, count, incr, flags);
-	params->func(params, bo, pe, addr, count, incr, flags);
-}
-
 /*
  * amdgpu_vm_update_pde - update a single level in the hierarchy
  *
  * @param: parameters for the update
  * @vm: requested vm
- * @parent: parent directory
  * @entry: entry to update
  *
  * Makes sure the requested entry in parent is up to date.
  */
-static void amdgpu_vm_update_pde(struct amdgpu_pte_update_params *params,
-				 struct amdgpu_vm *vm,
-				 struct amdgpu_vm_pt *parent,
-				 struct amdgpu_vm_pt *entry)
+static int amdgpu_vm_update_pde(struct amdgpu_vm_update_params *params,
+				struct amdgpu_vm *vm,
+				struct amdgpu_vm_pt *entry)
 {
+	struct amdgpu_vm_pt *parent = amdgpu_vm_pt_parent(entry);
 	struct amdgpu_bo *bo = parent->base.bo, *pbo;
 	uint64_t pde, pt, flags;
 	unsigned level;
 
-	/* Don't update huge pages here */
-	if (entry->huge)
-		return;
-
 	for (level = 0, pbo = bo->parent; pbo; ++level)
 		pbo = pbo->parent;
 
 	level += params->adev->vm_manager.root_level;
 	amdgpu_gmc_get_pde_for_bo(entry->base.bo, level, &pt, &flags);
 	pde = (entry - parent->entries) * 8;
-	amdgpu_vm_update_func(params, bo, pde, pt, 1, 0, flags);
+	return vm->update_funcs->update(params, bo, pde, pt, 1, 0, flags);
 }
 
 /*
@@ -1396,7 +1215,7 @@ static void amdgpu_vm_invalidate_pds(struct amdgpu_device *adev,
 	struct amdgpu_vm_pt_cursor cursor;
 	struct amdgpu_vm_pt *entry;
 
-	for_each_amdgpu_vm_pt_dfs_safe(adev, vm, cursor, entry)
+	for_each_amdgpu_vm_pt_dfs_safe(adev, vm, NULL, cursor, entry)
 		if (entry->base.bo && !entry->base.moved)
 			amdgpu_vm_bo_relocated(&entry->base);
 }
@@ -1415,89 +1234,39 @@ static void amdgpu_vm_invalidate_pds(struct amdgpu_device *adev,
 int amdgpu_vm_update_directories(struct amdgpu_device *adev,
 				 struct amdgpu_vm *vm)
 {
-	struct amdgpu_pte_update_params params;
-	struct amdgpu_job *job;
-	unsigned ndw = 0;
-	int r = 0;
+	struct amdgpu_vm_update_params params;
+	int r;
 
 	if (list_empty(&vm->relocated))
 		return 0;
 
-restart:
 	memset(&params, 0, sizeof(params));
 	params.adev = adev;
+	params.vm = vm;
 
-	if (vm->use_cpu_for_update) {
-		r = amdgpu_bo_sync_wait(vm->root.base.bo,
-					AMDGPU_FENCE_OWNER_VM, true);
-		if (unlikely(r))
-			return r;
-
-		params.func = amdgpu_vm_cpu_set_ptes;
-	} else {
-		ndw = 512 * 8;
-		r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
-		if (r)
-			return r;
-
-		params.ib = &job->ibs[0];
-		params.func = amdgpu_vm_do_set_ptes;
-	}
+	r = vm->update_funcs->prepare(&params, AMDGPU_FENCE_OWNER_VM, NULL);
+	if (r)
+		return r;
 
 	while (!list_empty(&vm->relocated)) {
-		struct amdgpu_vm_pt *pt, *entry;
+		struct amdgpu_vm_pt *entry;
 
 		entry = list_first_entry(&vm->relocated, struct amdgpu_vm_pt,
 					 base.vm_status);
 		amdgpu_vm_bo_idle(&entry->base);
 
-		pt = amdgpu_vm_pt_parent(entry);
-		if (!pt)
-			continue;
-
-		amdgpu_vm_update_pde(&params, vm, pt, entry);
-
-		if (!vm->use_cpu_for_update &&
-		    (ndw - params.ib->length_dw) < 32)
-			break;
-	}
-
-	if (vm->use_cpu_for_update) {
-		/* Flush HDP */
-		mb();
-		amdgpu_asic_flush_hdp(adev, NULL);
-	} else if (params.ib->length_dw == 0) {
-		amdgpu_job_free(job);
-	} else {
-		struct amdgpu_bo *root = vm->root.base.bo;
-		struct amdgpu_ring *ring;
-		struct dma_fence *fence;
-
-		ring = container_of(vm->entity.rq->sched, struct amdgpu_ring,
-				    sched);
-
-		amdgpu_ring_pad_ib(ring, params.ib);
-		amdgpu_sync_resv(adev, &job->sync, root->tbo.resv,
-				 AMDGPU_FENCE_OWNER_VM, false);
-		WARN_ON(params.ib->length_dw > ndw);
-		r = amdgpu_job_submit(job, &vm->entity, AMDGPU_FENCE_OWNER_VM,
-				      &fence);
+		r = amdgpu_vm_update_pde(&params, vm, entry);
 		if (r)
 			goto error;
-
-		amdgpu_bo_fence(root, fence, true);
-		dma_fence_put(vm->last_update);
-		vm->last_update = fence;
 	}
 
-	if (!list_empty(&vm->relocated))
-		goto restart;
-
+	r = vm->update_funcs->commit(&params, &vm->last_update);
+	if (r)
+		goto error;
 	return 0;
 
 error:
 	amdgpu_vm_invalidate_pds(adev, vm);
-	amdgpu_job_free(job);
 	return r;
 }
 
@@ -1506,7 +1275,7 @@ error:
  *
  * Make sure to set the right flags for the PTEs at the desired level.
  */
-static void amdgpu_vm_update_flags(struct amdgpu_pte_update_params *params,
+static void amdgpu_vm_update_flags(struct amdgpu_vm_update_params *params,
 				   struct amdgpu_bo *bo, unsigned level,
 				   uint64_t pe, uint64_t addr,
 				   unsigned count, uint32_t incr,
@@ -1525,13 +1294,14 @@ static void amdgpu_vm_update_flags(struct amdgpu_pte_update_params *params,
 		flags |= AMDGPU_PTE_EXECUTABLE;
 	}
 
-	amdgpu_vm_update_func(params, bo, pe, addr, count, incr, flags);
+	params->vm->update_funcs->update(params, bo, pe, addr, count, incr,
+					 flags);
 }
 
 /**
  * amdgpu_vm_fragment - get fragment for PTEs
  *
- * @params: see amdgpu_pte_update_params definition
+ * @params: see amdgpu_vm_update_params definition
  * @start: first PTE to handle
  * @end: last PTE to handle
  * @flags: hw mapping flags
@@ -1540,7 +1310,7 @@ static void amdgpu_vm_update_flags(struct amdgpu_pte_update_params *params,
  *
  * Returns the first possible fragment for the start and end address.
  */
-static void amdgpu_vm_fragment(struct amdgpu_pte_update_params *params,
+static void amdgpu_vm_fragment(struct amdgpu_vm_update_params *params,
 			       uint64_t start, uint64_t end, uint64_t flags,
 			       unsigned int *frag, uint64_t *frag_end)
 {
@@ -1573,7 +1343,7 @@ static void amdgpu_vm_fragment(struct amdgpu_pte_update_params *params,
 		max_frag = 31;
 
 	/* system pages are non continuously */
-	if (params->src) {
+	if (params->pages_addr) {
 		*frag = 0;
 		*frag_end = end;
 		return;
@@ -1592,7 +1362,7 @@ static void amdgpu_vm_fragment(struct amdgpu_pte_update_params *params,
 /**
  * amdgpu_vm_update_ptes - make sure that page tables are valid
  *
- * @params: see amdgpu_pte_update_params definition
+ * @params: see amdgpu_vm_update_params definition
  * @start: start of GPU address range
  * @end: end of GPU address range
  * @dst: destination address to map to, the next dst inside the function
@@ -1603,7 +1373,7 @@ static void amdgpu_vm_fragment(struct amdgpu_pte_update_params *params,
  * Returns:
  * 0 for success, -EINVAL for failure.
  */
-static int amdgpu_vm_update_ptes(struct amdgpu_pte_update_params *params,
+static int amdgpu_vm_update_ptes(struct amdgpu_vm_update_params *params,
 				 uint64_t start, uint64_t end,
 				 uint64_t dst, uint64_t flags)
 {
@@ -1611,6 +1381,7 @@ static int amdgpu_vm_update_ptes(struct amdgpu_pte_update_params *params,
 	struct amdgpu_vm_pt_cursor cursor;
 	uint64_t frag_start = start, frag_end;
 	unsigned int frag;
+	int r;
 
 	/* figure out the initial fragment */
 	amdgpu_vm_fragment(params, frag_start, end, flags, &frag, &frag_end);
@@ -1618,12 +1389,15 @@ static int amdgpu_vm_update_ptes(struct amdgpu_pte_update_params *params,
 	/* walk over the address space and update the PTs */
 	amdgpu_vm_pt_start(adev, params->vm, start, &cursor);
 	while (cursor.pfn < end) {
-		struct amdgpu_bo *pt = cursor.entry->base.bo;
 		unsigned shift, parent_shift, mask;
 		uint64_t incr, entry_end, pe_start;
+		struct amdgpu_bo *pt;
 
-		if (!pt)
-			return -ENOENT;
+		r = amdgpu_vm_alloc_pts(params->adev, params->vm, &cursor);
+		if (r)
+			return r;
+
+		pt = cursor.entry->base.bo;
 
 		/* The root level can't be a huge page */
 		if (cursor.level == adev->vm_manager.root_level) {
@@ -1632,16 +1406,10 @@ static int amdgpu_vm_update_ptes(struct amdgpu_pte_update_params *params,
 			continue;
 		}
 
-		/* If it isn't already handled it can't be a huge page */
-		if (cursor.entry->huge) {
-			/* Add the entry to the relocated list to update it. */
-			cursor.entry->huge = false;
-			amdgpu_vm_bo_relocated(&cursor.entry->base);
-		}
-
 		shift = amdgpu_vm_level_shift(adev, cursor.level);
 		parent_shift = amdgpu_vm_level_shift(adev, cursor.level - 1);
-		if (adev->asic_type < CHIP_VEGA10) {
+		if (adev->asic_type < CHIP_VEGA10 &&
+		    (flags & AMDGPU_PTE_VALID)) {
 			/* No huge page support before GMC v9 */
 			if (cursor.level != AMDGPU_VM_PTB) {
 				if (!amdgpu_vm_pt_descendant(adev, &cursor))
@@ -1697,9 +1465,9 @@ static int amdgpu_vm_update_ptes(struct amdgpu_pte_update_params *params,
 		} while (frag_start < entry_end);
 
 		if (amdgpu_vm_pt_descendant(adev, &cursor)) {
-			/* Mark all child entries as huge */
+			/* Free all child entries */
 			while (cursor.pfn < frag_start) {
-				cursor.entry->huge = true;
+				amdgpu_vm_free_pts(adev, params->vm, &cursor);
 				amdgpu_vm_pt_next(adev, &cursor);
 			}
 
@@ -1738,137 +1506,28 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
 				       uint64_t flags, uint64_t addr,
 				       struct dma_fence **fence)
 {
-	struct amdgpu_ring *ring;
+	struct amdgpu_vm_update_params params;
 	void *owner = AMDGPU_FENCE_OWNER_VM;
-	unsigned nptes, ncmds, ndw;
-	struct amdgpu_job *job;
-	struct amdgpu_pte_update_params params;
-	struct dma_fence *f = NULL;
 	int r;
 
 	memset(&params, 0, sizeof(params));
 	params.adev = adev;
 	params.vm = vm;
+	params.pages_addr = pages_addr;
 
 	/* sync to everything except eviction fences on unmapping */
 	if (!(flags & AMDGPU_PTE_VALID))
 		owner = AMDGPU_FENCE_OWNER_KFD;
 
-	if (vm->use_cpu_for_update) {
-		/* params.src is used as flag to indicate system Memory */
-		if (pages_addr)
-			params.src = ~0;
-
-		/* Wait for PT BOs to be idle. PTs share the same resv. object
-		 * as the root PD BO
-		 */
-		r = amdgpu_bo_sync_wait(vm->root.base.bo, owner, true);
-		if (unlikely(r))
-			return r;
-
-		/* Wait for any BO move to be completed */
-		if (exclusive) {
-			r = dma_fence_wait(exclusive, true);
-			if (unlikely(r))
-				return r;
-		}
-
-		params.func = amdgpu_vm_cpu_set_ptes;
-		params.pages_addr = pages_addr;
-		return amdgpu_vm_update_ptes(&params, start, last + 1,
-					     addr, flags);
-	}
-
-	ring = container_of(vm->entity.rq->sched, struct amdgpu_ring, sched);
-
-	nptes = last - start + 1;
-
-	/*
-	 * reserve space for two commands every (1 << BLOCK_SIZE)
-	 *  entries or 2k dwords (whatever is smaller)
-	 */
-	ncmds = ((nptes >> min(adev->vm_manager.block_size, 11u)) + 1);
-
-	/* The second command is for the shadow pagetables. */
-	if (vm->root.base.bo->shadow)
-		ncmds *= 2;
-
-	/* padding, etc. */
-	ndw = 64;
-
-	if (pages_addr) {
-		/* copy commands needed */
-		ndw += ncmds * adev->vm_manager.vm_pte_funcs->copy_pte_num_dw;
-
-		/* and also PTEs */
-		ndw += nptes * 2;
-
-		params.func = amdgpu_vm_do_copy_ptes;
-
-	} else {
-		/* set page commands needed */
-		ndw += ncmds * 10;
-
-		/* extra commands for begin/end fragments */
-		ncmds = 2 * adev->vm_manager.fragment_size;
-		if (vm->root.base.bo->shadow)
-			ncmds *= 2;
-
-		ndw += 10 * ncmds;
-
-		params.func = amdgpu_vm_do_set_ptes;
-	}
-
-	r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
+	r = vm->update_funcs->prepare(&params, owner, exclusive);
 	if (r)
 		return r;
 
-	params.ib = &job->ibs[0];
-
-	if (pages_addr) {
-		uint64_t *pte;
-		unsigned i;
-
-		/* Put the PTEs at the end of the IB. */
-		i = ndw - nptes * 2;
-		pte= (uint64_t *)&(job->ibs->ptr[i]);
-		params.src = job->ibs->gpu_addr + i * 4;
-
-		for (i = 0; i < nptes; ++i) {
-			pte[i] = amdgpu_vm_map_gart(pages_addr, addr + i *
-						    AMDGPU_GPU_PAGE_SIZE);
-			pte[i] |= flags;
-		}
-		addr = 0;
-	}
-
-	r = amdgpu_sync_fence(adev, &job->sync, exclusive, false);
-	if (r)
-		goto error_free;
-
-	r = amdgpu_sync_resv(adev, &job->sync, vm->root.base.bo->tbo.resv,
-			     owner, false);
-	if (r)
-		goto error_free;
-
 	r = amdgpu_vm_update_ptes(&params, start, last + 1, addr, flags);
 	if (r)
-		goto error_free;
-
-	amdgpu_ring_pad_ib(ring, params.ib);
-	WARN_ON(params.ib->length_dw > ndw);
-	r = amdgpu_job_submit(job, &vm->entity, AMDGPU_FENCE_OWNER_VM, &f);
-	if (r)
-		goto error_free;
-
-	amdgpu_bo_fence(vm->root.base.bo, f, true);
-	dma_fence_put(*fence);
-	*fence = f;
-	return 0;
+		return r;
 
-error_free:
-	amdgpu_job_free(job);
-	return r;
+	return vm->update_funcs->commit(&params, fence);
 }
 
 /**
@@ -1880,6 +1539,7 @@ error_free:
  * @vm: requested vm
  * @mapping: mapped range and flags to use for the update
  * @flags: HW flags for the mapping
+ * @bo_adev: amdgpu_device pointer that bo actually been allocated
  * @nodes: array of drm_mm_nodes with the MC addresses
  * @fence: optional resulting fence
  *
@@ -1895,6 +1555,7 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
 				      struct amdgpu_vm *vm,
 				      struct amdgpu_bo_va_mapping *mapping,
 				      uint64_t flags,
+				      struct amdgpu_device *bo_adev,
 				      struct drm_mm_node *nodes,
 				      struct dma_fence **fence)
 {
@@ -1949,7 +1610,6 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
 		if (pages_addr) {
 			uint64_t count;
 
-			max_entries = min(max_entries, 16ull * 1024ull);
 			for (count = 1;
 			     count < max_entries / AMDGPU_GPU_PAGES_IN_CPU_PAGE;
 			     ++count) {
@@ -1969,7 +1629,7 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
 			}
 
 		} else if (flags & AMDGPU_PTE_VALID) {
-			addr += adev->vm_manager.vram_base_offset;
+			addr += bo_adev->vm_manager.vram_base_offset;
 			addr += pfn << PAGE_SHIFT;
 		}
 
@@ -2016,6 +1676,7 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev,
 	struct drm_mm_node *nodes;
 	struct dma_fence *exclusive, **last_update;
 	uint64_t flags;
+	struct amdgpu_device *bo_adev = adev;
 	int r;
 
 	if (clear || !bo) {
@@ -2034,10 +1695,12 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev,
 		exclusive = reservation_object_get_excl(bo->tbo.resv);
 	}
 
-	if (bo)
+	if (bo) {
 		flags = amdgpu_ttm_tt_pte_flags(adev, bo->tbo.ttm, mem);
-	else
+		bo_adev = amdgpu_ttm_adev(bo->tbo.bdev);
+	} else {
 		flags = 0x0;
+	}
 
 	if (clear || (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv))
 		last_update = &vm->last_update;
@@ -2054,7 +1717,7 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev,
 
 	list_for_each_entry(mapping, &bo_va->invalids, list) {
 		r = amdgpu_vm_bo_split_mapping(adev, exclusive, pages_addr, vm,
-					       mapping, flags, nodes,
+					       mapping, flags, bo_adev, nodes,
 					       last_update);
 		if (r)
 			return r;
@@ -2374,6 +2037,16 @@ struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
 	INIT_LIST_HEAD(&bo_va->valids);
 	INIT_LIST_HEAD(&bo_va->invalids);
 
+	if (bo && amdgpu_xgmi_same_hive(adev, amdgpu_ttm_adev(bo->tbo.bdev)) &&
+	    (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM)) {
+		bo_va->is_xgmi = true;
+		mutex_lock(&adev->vm_manager.lock_pstate);
+		/* Power up XGMI if it can be potentially used */
+		if (++adev->vm_manager.xgmi_map_counter == 1)
+			amdgpu_xgmi_set_pstate(adev, 1);
+		mutex_unlock(&adev->vm_manager.lock_pstate);
+	}
+
 	return bo_va;
 }
 
@@ -2792,6 +2465,14 @@ void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
 	}
 
 	dma_fence_put(bo_va->last_pt_update);
+
+	if (bo && bo_va->is_xgmi) {
+		mutex_lock(&adev->vm_manager.lock_pstate);
+		if (--adev->vm_manager.xgmi_map_counter == 0)
+			amdgpu_xgmi_set_pstate(adev, 0);
+		mutex_unlock(&adev->vm_manager.lock_pstate);
+	}
+
 	kfree(bo_va);
 }
 
@@ -2949,20 +2630,16 @@ void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t min_vm_size,
 		 adev->vm_manager.fragment_size);
 }
 
-static struct amdgpu_retryfault_hashtable *init_fault_hash(void)
+/**
+ * amdgpu_vm_wait_idle - wait for the VM to become idle
+ *
+ * @vm: VM object to wait for
+ * @timeout: timeout to wait for VM to become idle
+ */
+long amdgpu_vm_wait_idle(struct amdgpu_vm *vm, long timeout)
 {
-	struct amdgpu_retryfault_hashtable *fault_hash;
-
-	fault_hash = kmalloc(sizeof(*fault_hash), GFP_KERNEL);
-	if (!fault_hash)
-		return fault_hash;
-
-	INIT_CHASH_TABLE(fault_hash->hash,
-			AMDGPU_PAGEFAULT_HASH_BITS, 8, 0);
-	spin_lock_init(&fault_hash->lock);
-	fault_hash->count = 0;
-
-	return fault_hash;
+	return reservation_object_wait_timeout_rcu(vm->root.base.bo->tbo.resv,
+						   true, true, timeout);
 }
 
 /**
@@ -3018,6 +2695,11 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 			 vm->use_cpu_for_update ? "CPU" : "SDMA");
 	WARN_ONCE((vm->use_cpu_for_update && !amdgpu_gmc_vram_full_visible(&adev->gmc)),
 		  "CPU update of VM recommended only for large BAR system\n");
+
+	if (vm->use_cpu_for_update)
+		vm->update_funcs = &amdgpu_vm_cpu_funcs;
+	else
+		vm->update_funcs = &amdgpu_vm_sdma_funcs;
 	vm->last_update = NULL;
 
 	amdgpu_vm_bo_param(adev, vm, adev->vm_manager.root_level, &bp);
@@ -3037,9 +2719,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 
 	amdgpu_vm_bo_base_init(&vm->root.base, vm, root);
 
-	r = amdgpu_vm_clear_bo(adev, vm, root,
-			       adev->vm_manager.root_level,
-			       vm->pte_support_ats);
+	r = amdgpu_vm_clear_bo(adev, vm, root);
 	if (r)
 		goto error_unreserve;
 
@@ -3058,12 +2738,6 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 		vm->pasid = pasid;
 	}
 
-	vm->fault_hash = init_fault_hash();
-	if (!vm->fault_hash) {
-		r = -ENOMEM;
-		goto error_free_root;
-	}
-
 	INIT_KFIFO(vm->faults);
 
 	return 0;
@@ -3134,9 +2808,8 @@ int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm, uns
 	 * changing any other state, in case it fails.
 	 */
 	if (pte_support_ats != vm->pte_support_ats) {
-		r = amdgpu_vm_clear_bo(adev, vm, vm->root.base.bo,
-			       adev->vm_manager.root_level,
-			       pte_support_ats);
+		vm->pte_support_ats = pte_support_ats;
+		r = amdgpu_vm_clear_bo(adev, vm, vm->root.base.bo);
 		if (r)
 			goto free_idr;
 	}
@@ -3144,7 +2817,6 @@ int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm, uns
 	/* Update VM state */
 	vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
 				    AMDGPU_VM_USE_CPU_FOR_COMPUTE);
-	vm->pte_support_ats = pte_support_ats;
 	DRM_DEBUG_DRIVER("VM update mode is %s\n",
 			 vm->use_cpu_for_update ? "CPU" : "SDMA");
 	WARN_ONCE((vm->use_cpu_for_update && !amdgpu_gmc_vram_full_visible(&adev->gmc)),
@@ -3219,15 +2891,10 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 	struct amdgpu_bo_va_mapping *mapping, *tmp;
 	bool prt_fini_needed = !!adev->gmc.gmc_funcs->set_prt;
 	struct amdgpu_bo *root;
-	u64 fault;
 	int i, r;
 
 	amdgpu_amdkfd_gpuvm_destroy_cb(adev, vm);
 
-	/* Clear pending page faults from IH when the VM is destroyed */
-	while (kfifo_get(&vm->faults, &fault))
-		amdgpu_vm_clear_fault(vm->fault_hash, fault);
-
 	if (vm->pasid) {
 		unsigned long flags;
 
@@ -3236,9 +2903,6 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 		spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
 	}
 
-	kfree(vm->fault_hash);
-	vm->fault_hash = NULL;
-
 	drm_sched_entity_destroy(&vm->entity);
 
 	if (!RB_EMPTY_ROOT(&vm->va.rb_root)) {
@@ -3267,10 +2931,11 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 	if (r) {
 		dev_err(adev->dev, "Leaking page tables because BO reservation failed\n");
 	} else {
-		amdgpu_vm_free_pts(adev, vm);
+		amdgpu_vm_free_pts(adev, vm, NULL);
 		amdgpu_bo_unreserve(root);
 	}
 	amdgpu_bo_unref(&root);
+	WARN_ON(vm->root.base.bo);
 	dma_fence_put(vm->last_update);
 	for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
 		amdgpu_vmid_free_reserved(adev, vm, i);
@@ -3315,6 +2980,9 @@ void amdgpu_vm_manager_init(struct amdgpu_device *adev)
 
 	idr_init(&adev->vm_manager.pasid_idr);
 	spin_lock_init(&adev->vm_manager.pasid_lock);
+
+	adev->vm_manager.xgmi_map_counter = 0;
+	mutex_init(&adev->vm_manager.lock_pstate);
 }
 
 /**
@@ -3405,78 +3073,3 @@ void amdgpu_vm_set_task_info(struct amdgpu_vm *vm)
 		}
 	}
 }
-
-/**
- * amdgpu_vm_add_fault - Add a page fault record to fault hash table
- *
- * @fault_hash: fault hash table
- * @key: 64-bit encoding of PASID and address
- *
- * This should be called when a retry page fault interrupt is
- * received. If this is a new page fault, it will be added to a hash
- * table. The return value indicates whether this is a new fault, or
- * a fault that was already known and is already being handled.
- *
- * If there are too many pending page faults, this will fail. Retry
- * interrupts should be ignored in this case until there is enough
- * free space.
- *
- * Returns 0 if the fault was added, 1 if the fault was already known,
- * -ENOSPC if there are too many pending faults.
- */
-int amdgpu_vm_add_fault(struct amdgpu_retryfault_hashtable *fault_hash, u64 key)
-{
-	unsigned long flags;
-	int r = -ENOSPC;
-
-	if (WARN_ON_ONCE(!fault_hash))
-		/* Should be allocated in amdgpu_vm_init
-		 */
-		return r;
-
-	spin_lock_irqsave(&fault_hash->lock, flags);
-
-	/* Only let the hash table fill up to 50% for best performance */
-	if (fault_hash->count >= (1 << (AMDGPU_PAGEFAULT_HASH_BITS-1)))
-		goto unlock_out;
-
-	r = chash_table_copy_in(&fault_hash->hash, key, NULL);
-	if (!r)
-		fault_hash->count++;
-
-	/* chash_table_copy_in should never fail unless we're losing count */
-	WARN_ON_ONCE(r < 0);
-
-unlock_out:
-	spin_unlock_irqrestore(&fault_hash->lock, flags);
-	return r;
-}
-
-/**
- * amdgpu_vm_clear_fault - Remove a page fault record
- *
- * @fault_hash: fault hash table
- * @key: 64-bit encoding of PASID and address
- *
- * This should be called when a page fault has been handled. Any
- * future interrupt with this key will be processed as a new
- * page fault.
- */
-void amdgpu_vm_clear_fault(struct amdgpu_retryfault_hashtable *fault_hash, u64 key)
-{
-	unsigned long flags;
-	int r;
-
-	if (!fault_hash)
-		return;
-
-	spin_lock_irqsave(&fault_hash->lock, flags);
-
-	r = chash_table_remove(&fault_hash->hash, key, NULL);
-	if (!WARN_ON_ONCE(r < 0)) {
-		fault_hash->count--;
-		WARN_ON_ONCE(fault_hash->count < 0);
-	}
-
-	spin_unlock_irqrestore(&fault_hash->lock, flags);
-}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index 81ff8177f092..91baf95212a6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -30,7 +30,6 @@
 #include <drm/gpu_scheduler.h>
 #include <drm/drm_file.h>
 #include <drm/ttm/ttm_bo_driver.h>
-#include <linux/chash.h>
 
 #include "amdgpu_sync.h"
 #include "amdgpu_ring.h"
@@ -140,7 +139,6 @@ struct amdgpu_vm_bo_base {
 
 struct amdgpu_vm_pt {
 	struct amdgpu_vm_bo_base	base;
-	bool				huge;
 
 	/* array of page tables, one for each directory entry */
 	struct amdgpu_vm_pt		*entries;
@@ -167,11 +165,6 @@ struct amdgpu_vm_pte_funcs {
 			    uint32_t incr, uint64_t flags);
 };
 
-#define AMDGPU_VM_FAULT(pasid, addr) (((u64)(pasid) << 48) | (addr))
-#define AMDGPU_VM_FAULT_PASID(fault) ((u64)(fault) >> 48)
-#define AMDGPU_VM_FAULT_ADDR(fault)  ((u64)(fault) & 0xfffffffff000ULL)
-
-
 struct amdgpu_task_info {
 	char	process_name[TASK_COMM_LEN];
 	char	task_name[TASK_COMM_LEN];
@@ -179,11 +172,52 @@ struct amdgpu_task_info {
 	pid_t	tgid;
 };
 
-#define AMDGPU_PAGEFAULT_HASH_BITS 8
-struct amdgpu_retryfault_hashtable {
-	DECLARE_CHASH_TABLE(hash, AMDGPU_PAGEFAULT_HASH_BITS, 8, 0);
-	spinlock_t	lock;
-	int		count;
+/**
+ * struct amdgpu_vm_update_params
+ *
+ * Encapsulate some VM table update parameters to reduce
+ * the number of function parameters
+ *
+ */
+struct amdgpu_vm_update_params {
+
+	/**
+	 * @adev: amdgpu device we do this update for
+	 */
+	struct amdgpu_device *adev;
+
+	/**
+	 * @vm: optional amdgpu_vm we do this update for
+	 */
+	struct amdgpu_vm *vm;
+
+	/**
+	 * @pages_addr:
+	 *
+	 * DMA addresses to use for mapping
+	 */
+	dma_addr_t *pages_addr;
+
+	/**
+	 * @job: job to used for hw submission
+	 */
+	struct amdgpu_job *job;
+
+	/**
+	 * @num_dw_left: number of dw left for the IB
+	 */
+	unsigned int num_dw_left;
+};
+
+struct amdgpu_vm_update_funcs {
+	int (*map_table)(struct amdgpu_bo *bo);
+	int (*prepare)(struct amdgpu_vm_update_params *p, void * owner,
+		       struct dma_fence *exclusive);
+	int (*update)(struct amdgpu_vm_update_params *p,
+		      struct amdgpu_bo *bo, uint64_t pe, uint64_t addr,
+		      unsigned count, uint32_t incr, uint64_t flags);
+	int (*commit)(struct amdgpu_vm_update_params *p,
+		      struct dma_fence **fence);
 };
 
 struct amdgpu_vm {
@@ -221,7 +255,10 @@ struct amdgpu_vm {
 	struct amdgpu_vmid	*reserved_vmid[AMDGPU_MAX_VMHUBS];
 
 	/* Flag to indicate if VM tables are updated by CPU or GPU (SDMA) */
-	bool                    use_cpu_for_update;
+	bool					use_cpu_for_update;
+
+	/* Functions to use for VM table updates */
+	const struct amdgpu_vm_update_funcs	*update_funcs;
 
 	/* Flag to indicate ATS support from PTE for GFX9 */
 	bool			pte_support_ats;
@@ -245,7 +282,6 @@ struct amdgpu_vm {
 	struct ttm_lru_bulk_move lru_bulk_move;
 	/* mark whether can do the bulk move */
 	bool			bulk_moveable;
-	struct amdgpu_retryfault_hashtable *fault_hash;
 };
 
 struct amdgpu_vm_manager {
@@ -267,6 +303,7 @@ struct amdgpu_vm_manager {
 	const struct amdgpu_vm_pte_funcs	*vm_pte_funcs;
 	struct drm_sched_rq			*vm_pte_rqs[AMDGPU_MAX_RINGS];
 	unsigned				vm_pte_num_rqs;
+	struct amdgpu_ring			*page_fault;
 
 	/* partial resident texture handling */
 	spinlock_t				prt_lock;
@@ -283,14 +320,23 @@ struct amdgpu_vm_manager {
 	 */
 	struct idr				pasid_idr;
 	spinlock_t				pasid_lock;
+
+	/* counter of mapped memory through xgmi */
+	uint32_t				xgmi_map_counter;
+	struct mutex				lock_pstate;
 };
 
 #define amdgpu_vm_copy_pte(adev, ib, pe, src, count) ((adev)->vm_manager.vm_pte_funcs->copy_pte((ib), (pe), (src), (count)))
 #define amdgpu_vm_write_pte(adev, ib, pe, value, count, incr) ((adev)->vm_manager.vm_pte_funcs->write_pte((ib), (pe), (value), (count), (incr)))
 #define amdgpu_vm_set_pte_pde(adev, ib, pe, addr, count, incr, flags) ((adev)->vm_manager.vm_pte_funcs->set_pte_pde((ib), (pe), (addr), (count), (incr), (flags)))
 
+extern const struct amdgpu_vm_update_funcs amdgpu_vm_cpu_funcs;
+extern const struct amdgpu_vm_update_funcs amdgpu_vm_sdma_funcs;
+
 void amdgpu_vm_manager_init(struct amdgpu_device *adev);
 void amdgpu_vm_manager_fini(struct amdgpu_device *adev);
+
+long amdgpu_vm_wait_idle(struct amdgpu_vm *vm, long timeout);
 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 		   int vm_context, unsigned int pasid);
 int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm, unsigned int pasid);
@@ -303,9 +349,6 @@ bool amdgpu_vm_ready(struct amdgpu_vm *vm);
 int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 			      int (*callback)(void *p, struct amdgpu_bo *bo),
 			      void *param);
-int amdgpu_vm_alloc_pts(struct amdgpu_device *adev,
-			struct amdgpu_vm *vm,
-			uint64_t saddr, uint64_t size);
 int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_pipe_sync);
 int amdgpu_vm_update_directories(struct amdgpu_device *adev,
 				 struct amdgpu_vm *vm);
@@ -319,6 +362,7 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev,
 			bool clear);
 void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
 			     struct amdgpu_bo *bo, bool evicted);
+uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr);
 struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
 				       struct amdgpu_bo *bo);
 struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
@@ -358,11 +402,6 @@ void amdgpu_vm_set_task_info(struct amdgpu_vm *vm);
 
 void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev,
 				struct amdgpu_vm *vm);
-
-int amdgpu_vm_add_fault(struct amdgpu_retryfault_hashtable *fault_hash, u64 key);
-
-void amdgpu_vm_clear_fault(struct amdgpu_retryfault_hashtable *fault_hash, u64 key);
-
 void amdgpu_vm_del_from_lru_notify(struct ttm_buffer_object *bo);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c
new file mode 100644
index 000000000000..5222d165abfc
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c
@@ -0,0 +1,127 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "amdgpu_vm.h"
+#include "amdgpu_object.h"
+#include "amdgpu_trace.h"
+
+/**
+ * amdgpu_vm_cpu_map_table - make sure new PDs/PTs are kmapped
+ *
+ * @table: newly allocated or validated PD/PT
+ */
+static int amdgpu_vm_cpu_map_table(struct amdgpu_bo *table)
+{
+	return amdgpu_bo_kmap(table, NULL);
+}
+
+/**
+ * amdgpu_vm_cpu_prepare - prepare page table update with the CPU
+ *
+ * @p: see amdgpu_vm_update_params definition
+ * @owner: owner we need to sync to
+ * @exclusive: exclusive move fence we need to sync to
+ *
+ * Returns:
+ * Negativ errno, 0 for success.
+ */
+static int amdgpu_vm_cpu_prepare(struct amdgpu_vm_update_params *p, void *owner,
+				 struct dma_fence *exclusive)
+{
+	int r;
+
+	/* Wait for PT BOs to be idle. PTs share the same resv. object
+	 * as the root PD BO
+	 */
+	r = amdgpu_bo_sync_wait(p->vm->root.base.bo, owner, true);
+	if (unlikely(r))
+		return r;
+
+	/* Wait for any BO move to be completed */
+	if (exclusive) {
+		r = dma_fence_wait(exclusive, true);
+		if (unlikely(r))
+			return r;
+	}
+
+	return 0;
+}
+
+/**
+ * amdgpu_vm_cpu_update - helper to update page tables via CPU
+ *
+ * @p: see amdgpu_vm_update_params definition
+ * @bo: PD/PT to update
+ * @pe: kmap addr of the page entry
+ * @addr: dst addr to write into pe
+ * @count: number of page entries to update
+ * @incr: increase next addr by incr bytes
+ * @flags: hw access flags
+ *
+ * Write count number of PT/PD entries directly.
+ */
+static int amdgpu_vm_cpu_update(struct amdgpu_vm_update_params *p,
+				struct amdgpu_bo *bo, uint64_t pe,
+				uint64_t addr, unsigned count, uint32_t incr,
+				uint64_t flags)
+{
+	unsigned int i;
+	uint64_t value;
+
+	pe += (unsigned long)amdgpu_bo_kptr(bo);
+
+	trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags);
+
+	for (i = 0; i < count; i++) {
+		value = p->pages_addr ?
+			amdgpu_vm_map_gart(p->pages_addr, addr) :
+			addr;
+		amdgpu_gmc_set_pte_pde(p->adev, (void *)(uintptr_t)pe,
+				       i, value, flags);
+		addr += incr;
+	}
+	return 0;
+}
+
+/**
+ * amdgpu_vm_cpu_commit - commit page table update to the HW
+ *
+ * @p: see amdgpu_vm_update_params definition
+ * @fence: unused
+ *
+ * Make sure that the hardware sees the page table updates.
+ */
+static int amdgpu_vm_cpu_commit(struct amdgpu_vm_update_params *p,
+				struct dma_fence **fence)
+{
+	/* Flush HDP */
+	mb();
+	amdgpu_asic_flush_hdp(p->adev, NULL);
+	return 0;
+}
+
+const struct amdgpu_vm_update_funcs amdgpu_vm_cpu_funcs = {
+	.map_table = amdgpu_vm_cpu_map_table,
+	.prepare = amdgpu_vm_cpu_prepare,
+	.update = amdgpu_vm_cpu_update,
+	.commit = amdgpu_vm_cpu_commit
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
new file mode 100644
index 000000000000..ddd181f5ed37
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
@@ -0,0 +1,270 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "amdgpu_vm.h"
+#include "amdgpu_job.h"
+#include "amdgpu_object.h"
+#include "amdgpu_trace.h"
+
+#define AMDGPU_VM_SDMA_MIN_NUM_DW	256u
+#define AMDGPU_VM_SDMA_MAX_NUM_DW	(16u * 1024u)
+
+/**
+ * amdgpu_vm_sdma_map_table - make sure new PDs/PTs are GTT mapped
+ *
+ * @table: newly allocated or validated PD/PT
+ */
+static int amdgpu_vm_sdma_map_table(struct amdgpu_bo *table)
+{
+	int r;
+
+	r = amdgpu_ttm_alloc_gart(&table->tbo);
+	if (r)
+		return r;
+
+	if (table->shadow)
+		r = amdgpu_ttm_alloc_gart(&table->shadow->tbo);
+
+	return r;
+}
+
+/**
+ * amdgpu_vm_sdma_prepare - prepare SDMA command submission
+ *
+ * @p: see amdgpu_vm_update_params definition
+ * @owner: owner we need to sync to
+ * @exclusive: exclusive move fence we need to sync to
+ *
+ * Returns:
+ * Negativ errno, 0 for success.
+ */
+static int amdgpu_vm_sdma_prepare(struct amdgpu_vm_update_params *p,
+				  void *owner, struct dma_fence *exclusive)
+{
+	struct amdgpu_bo *root = p->vm->root.base.bo;
+	unsigned int ndw = AMDGPU_VM_SDMA_MIN_NUM_DW;
+	int r;
+
+	r = amdgpu_job_alloc_with_ib(p->adev, ndw * 4, &p->job);
+	if (r)
+		return r;
+
+	r = amdgpu_sync_fence(p->adev, &p->job->sync, exclusive, false);
+	if (r)
+		return r;
+
+	r = amdgpu_sync_resv(p->adev, &p->job->sync, root->tbo.resv,
+			     owner, false);
+	if (r)
+		return r;
+
+	p->num_dw_left = ndw;
+	return 0;
+}
+
+/**
+ * amdgpu_vm_sdma_commit - commit SDMA command submission
+ *
+ * @p: see amdgpu_vm_update_params definition
+ * @fence: resulting fence
+ *
+ * Returns:
+ * Negativ errno, 0 for success.
+ */
+static int amdgpu_vm_sdma_commit(struct amdgpu_vm_update_params *p,
+				 struct dma_fence **fence)
+{
+	struct amdgpu_bo *root = p->vm->root.base.bo;
+	struct amdgpu_ib *ib = p->job->ibs;
+	struct amdgpu_ring *ring;
+	struct dma_fence *f;
+	int r;
+
+	ring = container_of(p->vm->entity.rq->sched, struct amdgpu_ring, sched);
+
+	WARN_ON(ib->length_dw == 0);
+	amdgpu_ring_pad_ib(ring, ib);
+	WARN_ON(ib->length_dw > p->num_dw_left);
+	r = amdgpu_job_submit(p->job, &p->vm->entity,
+			      AMDGPU_FENCE_OWNER_VM, &f);
+	if (r)
+		goto error;
+
+	amdgpu_bo_fence(root, f, true);
+	if (fence)
+		swap(*fence, f);
+	dma_fence_put(f);
+	return 0;
+
+error:
+	amdgpu_job_free(p->job);
+	return r;
+}
+
+
+/**
+ * amdgpu_vm_sdma_copy_ptes - copy the PTEs from mapping
+ *
+ * @p: see amdgpu_vm_update_params definition
+ * @bo: PD/PT to update
+ * @pe: addr of the page entry
+ * @count: number of page entries to copy
+ *
+ * Traces the parameters and calls the DMA function to copy the PTEs.
+ */
+static void amdgpu_vm_sdma_copy_ptes(struct amdgpu_vm_update_params *p,
+				     struct amdgpu_bo *bo, uint64_t pe,
+				     unsigned count)
+{
+	struct amdgpu_ib *ib = p->job->ibs;
+	uint64_t src = ib->gpu_addr;
+
+	src += p->num_dw_left * 4;
+
+	pe += amdgpu_bo_gpu_offset(bo);
+	trace_amdgpu_vm_copy_ptes(pe, src, count);
+
+	amdgpu_vm_copy_pte(p->adev, ib, pe, src, count);
+}
+
+/**
+ * amdgpu_vm_sdma_set_ptes - helper to call the right asic function
+ *
+ * @p: see amdgpu_vm_update_params definition
+ * @bo: PD/PT to update
+ * @pe: addr of the page entry
+ * @addr: dst addr to write into pe
+ * @count: number of page entries to update
+ * @incr: increase next addr by incr bytes
+ * @flags: hw access flags
+ *
+ * Traces the parameters and calls the right asic functions
+ * to setup the page table using the DMA.
+ */
+static void amdgpu_vm_sdma_set_ptes(struct amdgpu_vm_update_params *p,
+				    struct amdgpu_bo *bo, uint64_t pe,
+				    uint64_t addr, unsigned count,
+				    uint32_t incr, uint64_t flags)
+{
+	struct amdgpu_ib *ib = p->job->ibs;
+
+	pe += amdgpu_bo_gpu_offset(bo);
+	trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags);
+	if (count < 3) {
+		amdgpu_vm_write_pte(p->adev, ib, pe, addr | flags,
+				    count, incr);
+	} else {
+		amdgpu_vm_set_pte_pde(p->adev, ib, pe, addr,
+				      count, incr, flags);
+	}
+}
+
+/**
+ * amdgpu_vm_sdma_update - execute VM update
+ *
+ * @p: see amdgpu_vm_update_params definition
+ * @bo: PD/PT to update
+ * @pe: addr of the page entry
+ * @addr: dst addr to write into pe
+ * @count: number of page entries to update
+ * @incr: increase next addr by incr bytes
+ * @flags: hw access flags
+ *
+ * Reserve space in the IB, setup mapping buffer on demand and write commands to
+ * the IB.
+ */
+static int amdgpu_vm_sdma_update(struct amdgpu_vm_update_params *p,
+				 struct amdgpu_bo *bo, uint64_t pe,
+				 uint64_t addr, unsigned count, uint32_t incr,
+				 uint64_t flags)
+{
+	unsigned int i, ndw, nptes;
+	uint64_t *pte;
+	int r;
+
+	do {
+		ndw = p->num_dw_left;
+		ndw -= p->job->ibs->length_dw;
+
+		if (ndw < 32) {
+			r = amdgpu_vm_sdma_commit(p, NULL);
+			if (r)
+				return r;
+
+			/* estimate how many dw we need */
+			ndw = 32;
+			if (p->pages_addr)
+				ndw += count * 2;
+			ndw = max(ndw, AMDGPU_VM_SDMA_MIN_NUM_DW);
+			ndw = min(ndw, AMDGPU_VM_SDMA_MAX_NUM_DW);
+
+			r = amdgpu_job_alloc_with_ib(p->adev, ndw * 4, &p->job);
+			if (r)
+				return r;
+
+			p->num_dw_left = ndw;
+		}
+
+		if (!p->pages_addr) {
+			/* set page commands needed */
+			if (bo->shadow)
+				amdgpu_vm_sdma_set_ptes(p, bo->shadow, pe, addr,
+							count, incr, flags);
+			amdgpu_vm_sdma_set_ptes(p, bo, pe, addr, count,
+						incr, flags);
+			return 0;
+		}
+
+		/* copy commands needed */
+		ndw -= p->adev->vm_manager.vm_pte_funcs->copy_pte_num_dw *
+			(bo->shadow ? 2 : 1);
+
+		/* for padding */
+		ndw -= 7;
+
+		nptes = min(count, ndw / 2);
+
+		/* Put the PTEs at the end of the IB. */
+		p->num_dw_left -= nptes * 2;
+		pte = (uint64_t *)&(p->job->ibs->ptr[p->num_dw_left]);
+		for (i = 0; i < nptes; ++i, addr += incr) {
+			pte[i] = amdgpu_vm_map_gart(p->pages_addr, addr);
+			pte[i] |= flags;
+		}
+
+		if (bo->shadow)
+			amdgpu_vm_sdma_copy_ptes(p, bo->shadow, pe, nptes);
+		amdgpu_vm_sdma_copy_ptes(p, bo, pe, nptes);
+
+		pe += nptes * 8;
+		count -= nptes;
+	} while (count);
+
+	return 0;
+}
+
+const struct amdgpu_vm_update_funcs amdgpu_vm_sdma_funcs = {
+	.map_table = amdgpu_vm_sdma_map_table,
+	.prepare = amdgpu_vm_sdma_prepare,
+	.update = amdgpu_vm_sdma_update,
+	.commit = amdgpu_vm_sdma_commit
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
index 3f9d5d00c9b3..ec9ea3fdbb4a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
@@ -33,6 +33,85 @@ struct amdgpu_vram_mgr {
 };
 
 /**
+ * DOC: mem_info_vram_total
+ *
+ * The amdgpu driver provides a sysfs API for reporting current total VRAM
+ * available on the device
+ * The file mem_info_vram_total is used for this and returns the total
+ * amount of VRAM in bytes
+ */
+static ssize_t amdgpu_mem_info_vram_total_show(struct device *dev,
+		struct device_attribute *attr, char *buf)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct amdgpu_device *adev = ddev->dev_private;
+
+	return snprintf(buf, PAGE_SIZE, "%llu\n", adev->gmc.real_vram_size);
+}
+
+/**
+ * DOC: mem_info_vis_vram_total
+ *
+ * The amdgpu driver provides a sysfs API for reporting current total
+ * visible VRAM available on the device
+ * The file mem_info_vis_vram_total is used for this and returns the total
+ * amount of visible VRAM in bytes
+ */
+static ssize_t amdgpu_mem_info_vis_vram_total_show(struct device *dev,
+		struct device_attribute *attr, char *buf)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct amdgpu_device *adev = ddev->dev_private;
+
+	return snprintf(buf, PAGE_SIZE, "%llu\n", adev->gmc.visible_vram_size);
+}
+
+/**
+ * DOC: mem_info_vram_used
+ *
+ * The amdgpu driver provides a sysfs API for reporting current total VRAM
+ * available on the device
+ * The file mem_info_vram_used is used for this and returns the total
+ * amount of currently used VRAM in bytes
+ */
+static ssize_t amdgpu_mem_info_vram_used_show(struct device *dev,
+		struct device_attribute *attr, char *buf)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct amdgpu_device *adev = ddev->dev_private;
+
+	return snprintf(buf, PAGE_SIZE, "%llu\n",
+		amdgpu_vram_mgr_usage(&adev->mman.bdev.man[TTM_PL_VRAM]));
+}
+
+/**
+ * DOC: mem_info_vis_vram_used
+ *
+ * The amdgpu driver provides a sysfs API for reporting current total of
+ * used visible VRAM
+ * The file mem_info_vis_vram_used is used for this and returns the total
+ * amount of currently used visible VRAM in bytes
+ */
+static ssize_t amdgpu_mem_info_vis_vram_used_show(struct device *dev,
+		struct device_attribute *attr, char *buf)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct amdgpu_device *adev = ddev->dev_private;
+
+	return snprintf(buf, PAGE_SIZE, "%llu\n",
+		amdgpu_vram_mgr_vis_usage(&adev->mman.bdev.man[TTM_PL_VRAM]));
+}
+
+static DEVICE_ATTR(mem_info_vram_total, S_IRUGO,
+		   amdgpu_mem_info_vram_total_show, NULL);
+static DEVICE_ATTR(mem_info_vis_vram_total, S_IRUGO,
+		   amdgpu_mem_info_vis_vram_total_show,NULL);
+static DEVICE_ATTR(mem_info_vram_used, S_IRUGO,
+		   amdgpu_mem_info_vram_used_show, NULL);
+static DEVICE_ATTR(mem_info_vis_vram_used, S_IRUGO,
+		   amdgpu_mem_info_vis_vram_used_show, NULL);
+
+/**
  * amdgpu_vram_mgr_init - init VRAM manager and DRM MM
  *
  * @man: TTM memory type manager
@@ -43,7 +122,9 @@ struct amdgpu_vram_mgr {
 static int amdgpu_vram_mgr_init(struct ttm_mem_type_manager *man,
 				unsigned long p_size)
 {
+	struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev);
 	struct amdgpu_vram_mgr *mgr;
+	int ret;
 
 	mgr = kzalloc(sizeof(*mgr), GFP_KERNEL);
 	if (!mgr)
@@ -52,6 +133,29 @@ static int amdgpu_vram_mgr_init(struct ttm_mem_type_manager *man,
 	drm_mm_init(&mgr->mm, 0, p_size);
 	spin_lock_init(&mgr->lock);
 	man->priv = mgr;
+
+	/* Add the two VRAM-related sysfs files */
+	ret = device_create_file(adev->dev, &dev_attr_mem_info_vram_total);
+	if (ret) {
+		DRM_ERROR("Failed to create device file mem_info_vram_total\n");
+		return ret;
+	}
+	ret = device_create_file(adev->dev, &dev_attr_mem_info_vis_vram_total);
+	if (ret) {
+		DRM_ERROR("Failed to create device file mem_info_vis_vram_total\n");
+		return ret;
+	}
+	ret = device_create_file(adev->dev, &dev_attr_mem_info_vram_used);
+	if (ret) {
+		DRM_ERROR("Failed to create device file mem_info_vram_used\n");
+		return ret;
+	}
+	ret = device_create_file(adev->dev, &dev_attr_mem_info_vis_vram_used);
+	if (ret) {
+		DRM_ERROR("Failed to create device file mem_info_vis_vram_used\n");
+		return ret;
+	}
+
 	return 0;
 }
 
@@ -65,6 +169,7 @@ static int amdgpu_vram_mgr_init(struct ttm_mem_type_manager *man,
  */
 static int amdgpu_vram_mgr_fini(struct ttm_mem_type_manager *man)
 {
+	struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev);
 	struct amdgpu_vram_mgr *mgr = man->priv;
 
 	spin_lock(&mgr->lock);
@@ -72,6 +177,10 @@ static int amdgpu_vram_mgr_fini(struct ttm_mem_type_manager *man)
 	spin_unlock(&mgr->lock);
 	kfree(mgr);
 	man->priv = NULL;
+	device_remove_file(adev->dev, &dev_attr_mem_info_vram_total);
+	device_remove_file(adev->dev, &dev_attr_mem_info_vis_vram_total);
+	device_remove_file(adev->dev, &dev_attr_mem_info_vram_used);
+	device_remove_file(adev->dev, &dev_attr_mem_info_vis_vram_used);
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
index 407dd16cc35c..a48c84c51775 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
@@ -24,6 +24,7 @@
 #include <linux/list.h>
 #include "amdgpu.h"
 #include "amdgpu_xgmi.h"
+#include "amdgpu_smu.h"
 
 
 static DEFINE_MUTEX(xgmi_mutex);
@@ -34,12 +35,132 @@ static DEFINE_MUTEX(xgmi_mutex);
 static struct amdgpu_hive_info xgmi_hives[AMDGPU_MAX_XGMI_HIVE];
 static unsigned hive_count = 0;
 
-
 void *amdgpu_xgmi_hive_try_lock(struct amdgpu_hive_info *hive)
 {
 	return &hive->device_list;
 }
 
+static ssize_t amdgpu_xgmi_show_hive_id(struct device *dev,
+		struct device_attribute *attr, char *buf)
+{
+	struct amdgpu_hive_info *hive =
+			container_of(attr, struct amdgpu_hive_info, dev_attr);
+
+	return snprintf(buf, PAGE_SIZE, "%llu\n", hive->hive_id);
+}
+
+static int amdgpu_xgmi_sysfs_create(struct amdgpu_device *adev,
+				    struct amdgpu_hive_info *hive)
+{
+	int ret = 0;
+
+	if (WARN_ON(hive->kobj))
+		return -EINVAL;
+
+	hive->kobj = kobject_create_and_add("xgmi_hive_info", &adev->dev->kobj);
+	if (!hive->kobj) {
+		dev_err(adev->dev, "XGMI: Failed to allocate sysfs entry!\n");
+		return -EINVAL;
+	}
+
+	hive->dev_attr = (struct device_attribute) {
+		.attr = {
+			.name = "xgmi_hive_id",
+			.mode = S_IRUGO,
+
+		},
+		.show = amdgpu_xgmi_show_hive_id,
+	};
+
+	ret = sysfs_create_file(hive->kobj, &hive->dev_attr.attr);
+	if (ret) {
+		dev_err(adev->dev, "XGMI: Failed to create device file xgmi_hive_id\n");
+		kobject_del(hive->kobj);
+		kobject_put(hive->kobj);
+		hive->kobj = NULL;
+	}
+
+	return ret;
+}
+
+static void amdgpu_xgmi_sysfs_destroy(struct amdgpu_device *adev,
+				    struct amdgpu_hive_info *hive)
+{
+	sysfs_remove_file(hive->kobj, &hive->dev_attr.attr);
+	kobject_del(hive->kobj);
+	kobject_put(hive->kobj);
+	hive->kobj = NULL;
+}
+
+static ssize_t amdgpu_xgmi_show_device_id(struct device *dev,
+				     struct device_attribute *attr,
+				     char *buf)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct amdgpu_device *adev = ddev->dev_private;
+
+	return snprintf(buf, PAGE_SIZE, "%llu\n", adev->gmc.xgmi.node_id);
+
+}
+
+
+static DEVICE_ATTR(xgmi_device_id, S_IRUGO, amdgpu_xgmi_show_device_id, NULL);
+
+
+static int amdgpu_xgmi_sysfs_add_dev_info(struct amdgpu_device *adev,
+					 struct amdgpu_hive_info *hive)
+{
+	int ret = 0;
+	char node[10] = { 0 };
+
+	/* Create xgmi device id file */
+	ret = device_create_file(adev->dev, &dev_attr_xgmi_device_id);
+	if (ret) {
+		dev_err(adev->dev, "XGMI: Failed to create device file xgmi_device_id\n");
+		return ret;
+	}
+
+	/* Create sysfs link to hive info folder on the first device */
+	if (adev != hive->adev) {
+		ret = sysfs_create_link(&adev->dev->kobj, hive->kobj,
+					"xgmi_hive_info");
+		if (ret) {
+			dev_err(adev->dev, "XGMI: Failed to create link to hive info");
+			goto remove_file;
+		}
+	}
+
+	sprintf(node, "node%d", hive->number_devices);
+	/* Create sysfs link form the hive folder to yourself */
+	ret = sysfs_create_link(hive->kobj, &adev->dev->kobj, node);
+	if (ret) {
+		dev_err(adev->dev, "XGMI: Failed to create link from hive info");
+		goto remove_link;
+	}
+
+	goto success;
+
+
+remove_link:
+	sysfs_remove_link(&adev->dev->kobj, adev->ddev->unique);
+
+remove_file:
+	device_remove_file(adev->dev, &dev_attr_xgmi_device_id);
+
+success:
+	return ret;
+}
+
+static void amdgpu_xgmi_sysfs_rem_dev_info(struct amdgpu_device *adev,
+					  struct amdgpu_hive_info *hive)
+{
+	device_remove_file(adev->dev, &dev_attr_xgmi_device_id);
+	sysfs_remove_link(&adev->dev->kobj, adev->ddev->unique);
+	sysfs_remove_link(hive->kobj, adev->ddev->unique);
+}
+
+
+
 struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev, int lock)
 {
 	int i;
@@ -66,18 +187,50 @@ struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev, int lo
 
 	/* initialize new hive if not exist */
 	tmp = &xgmi_hives[hive_count++];
+
+	if (amdgpu_xgmi_sysfs_create(adev, tmp)) {
+		mutex_unlock(&xgmi_mutex);
+		return NULL;
+	}
+
+	tmp->adev = adev;
 	tmp->hive_id = adev->gmc.xgmi.hive_id;
 	INIT_LIST_HEAD(&tmp->device_list);
 	mutex_init(&tmp->hive_lock);
 	mutex_init(&tmp->reset_lock);
+
 	if (lock)
 		mutex_lock(&tmp->hive_lock);
-
+	tmp->pstate = -1;
 	mutex_unlock(&xgmi_mutex);
 
 	return tmp;
 }
 
+int amdgpu_xgmi_set_pstate(struct amdgpu_device *adev, int pstate)
+{
+	int ret = 0;
+	struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev, 0);
+
+	if (!hive)
+		return 0;
+
+	if (hive->pstate == pstate)
+		return 0;
+
+	dev_dbg(adev->dev, "Set xgmi pstate %d.\n", pstate);
+
+	if (is_support_sw_smu(adev))
+		ret = smu_set_xgmi_pstate(&adev->smu, pstate);
+	if (ret)
+		dev_err(adev->dev,
+			"XGMI: Set pstate failure on device %llx, hive %llx, ret %d",
+			adev->gmc.xgmi.node_id,
+			adev->gmc.xgmi.hive_id, ret);
+
+	return ret;
+}
+
 int amdgpu_xgmi_update_topology(struct amdgpu_hive_info *hive, struct amdgpu_device *adev)
 {
 	int ret = -EINVAL;
@@ -156,8 +309,17 @@ int amdgpu_xgmi_add_device(struct amdgpu_device *adev)
 			break;
 	}
 
-	dev_info(adev->dev, "XGMI: Add node %d, hive 0x%llx.\n",
-		 adev->gmc.xgmi.physical_node_id, adev->gmc.xgmi.hive_id);
+	if (!ret)
+		ret = amdgpu_xgmi_sysfs_add_dev_info(adev, hive);
+
+	if (!ret)
+		dev_info(adev->dev, "XGMI: Add node %d, hive 0x%llx.\n",
+			 adev->gmc.xgmi.physical_node_id, adev->gmc.xgmi.hive_id);
+	else
+		dev_err(adev->dev, "XGMI: Failed to add node %d, hive 0x%llx ret: %d\n",
+			adev->gmc.xgmi.physical_node_id, adev->gmc.xgmi.hive_id,
+			ret);
+
 
 	mutex_unlock(&hive->hive_lock);
 exit:
@@ -176,9 +338,11 @@ void amdgpu_xgmi_remove_device(struct amdgpu_device *adev)
 		return;
 
 	if (!(hive->number_devices--)) {
+		amdgpu_xgmi_sysfs_destroy(adev, hive);
 		mutex_destroy(&hive->hive_lock);
 		mutex_destroy(&hive->reset_lock);
 	} else {
+		amdgpu_xgmi_sysfs_rem_dev_info(adev, hive);
 		mutex_unlock(&hive->hive_lock);
 	}
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
index 14bc60664159..3e9c91e9a4bf 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
@@ -29,13 +29,25 @@ struct amdgpu_hive_info {
 	struct list_head	device_list;
 	struct psp_xgmi_topology_info	topology_info;
 	int number_devices;
-	struct mutex hive_lock,
-		     reset_lock;
+	struct mutex hive_lock, reset_lock;
+	struct kobject *kobj;
+	struct device_attribute dev_attr;
+	struct amdgpu_device *adev;
+	int pstate; /*0 -- low , 1 -- high , -1 unknown*/
 };
 
 struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev, int lock);
 int amdgpu_xgmi_update_topology(struct amdgpu_hive_info *hive, struct amdgpu_device *adev);
 int amdgpu_xgmi_add_device(struct amdgpu_device *adev);
 void amdgpu_xgmi_remove_device(struct amdgpu_device *adev);
+int amdgpu_xgmi_set_pstate(struct amdgpu_device *adev, int pstate);
+
+static inline bool amdgpu_xgmi_same_hive(struct amdgpu_device *adev,
+		struct amdgpu_device *bo_adev)
+{
+	return (adev != bo_adev &&
+		adev->gmc.xgmi.hive_id &&
+		adev->gmc.xgmi.hive_id == bo_adev->gmc.xgmi.hive_id);
+}
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
index 189599b694e8..d42808b05971 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
@@ -977,8 +977,8 @@ static int cik_sdma_sw_init(void *handle)
 		r = amdgpu_ring_init(adev, ring, 1024,
 				     &adev->sdma.trap_irq,
 				     (i == 0) ?
-				     AMDGPU_SDMA_IRQ_TRAP0 :
-				     AMDGPU_SDMA_IRQ_TRAP1);
+				     AMDGPU_SDMA_IRQ_INSTANCE0 :
+				     AMDGPU_SDMA_IRQ_INSTANCE1);
 		if (r)
 			return r;
 	}
@@ -1114,7 +1114,7 @@ static int cik_sdma_set_trap_irq_state(struct amdgpu_device *adev,
 	u32 sdma_cntl;
 
 	switch (type) {
-	case AMDGPU_SDMA_IRQ_TRAP0:
+	case AMDGPU_SDMA_IRQ_INSTANCE0:
 		switch (state) {
 		case AMDGPU_IRQ_STATE_DISABLE:
 			sdma_cntl = RREG32(mmSDMA0_CNTL + SDMA0_REGISTER_OFFSET);
@@ -1130,7 +1130,7 @@ static int cik_sdma_set_trap_irq_state(struct amdgpu_device *adev,
 			break;
 		}
 		break;
-	case AMDGPU_SDMA_IRQ_TRAP1:
+	case AMDGPU_SDMA_IRQ_INSTANCE1:
 		switch (state) {
 		case AMDGPU_IRQ_STATE_DISABLE:
 			sdma_cntl = RREG32(mmSDMA0_CNTL + SDMA1_REGISTER_OFFSET);
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
index 305276c7e4bf..c0cb244f58cd 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
@@ -782,6 +782,25 @@ static void gfx_v6_0_tiling_mode_table_init(struct amdgpu_device *adev)
 				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2);
+		tilemode[18] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
+				ARRAY_MODE(ARRAY_1D_TILED_THICK) |
+				PIPE_CONFIG(ADDR_SURF_P4_8x16);
+		tilemode[19] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
+				ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
+				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+				NUM_BANKS(ADDR_SURF_16_BANK) |
+				TILE_SPLIT(split_equal_to_row_size);
+		tilemode[20] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
+				ARRAY_MODE(ARRAY_2D_TILED_THICK) |
+				PIPE_CONFIG(ADDR_SURF_P4_8x16) |
+				BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+				NUM_BANKS(ADDR_SURF_16_BANK) |
+				TILE_SPLIT(split_equal_to_row_size);
 		tilemode[21] =  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
 				ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
 				PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
index b8e50a34bdb3..02955e6e9dd9 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
@@ -3236,6 +3236,7 @@ static void gfx_v8_0_tiling_mode_table_init(struct amdgpu_device *adev)
 		dev_warn(adev->dev,
 			 "Unknown chip type (%d) in function gfx_v8_0_tiling_mode_table_init() falling through to CHIP_CARRIZO\n",
 			 adev->asic_type);
+		/* fall through */
 
 	case CHIP_CARRIZO:
 		modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
index a11db2b1a63f..ba67d1023264 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
@@ -40,6 +40,8 @@
 
 #include "ivsrcid/gfx/irqsrcs_gfx_9_0.h"
 
+#include "amdgpu_ras.h"
+
 #define GFX9_NUM_GFX_RINGS     1
 #define GFX9_MEC_HPD_SIZE 4096
 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L
@@ -576,6 +578,27 @@ static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev)
 	}
 }
 
+static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev)
+{
+	switch (adev->asic_type) {
+	case CHIP_VEGA10:
+	case CHIP_VEGA12:
+	case CHIP_VEGA20:
+		break;
+	case CHIP_RAVEN:
+		if (adev->rev_id >= 0x8 || adev->pdev->device == 0x15d8)
+			break;
+		if ((adev->gfx.rlc_fw_version < 531) ||
+		    (adev->gfx.rlc_fw_version == 53815) ||
+		    (adev->gfx.rlc_feature_version < 1) ||
+		    !adev->gfx.rlc.is_rlc_v2_1)
+			adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
+		break;
+	default:
+		break;
+	}
+}
+
 static int gfx_v9_0_init_microcode(struct amdgpu_device *adev)
 {
 	const char *chip_name;
@@ -828,6 +851,7 @@ static int gfx_v9_0_init_microcode(struct amdgpu_device *adev)
 	}
 
 out:
+	gfx_v9_0_check_if_need_gfxoff(adev);
 	gfx_v9_0_check_fw_write_wait(adev);
 	if (err) {
 		dev_err(adev->dev,
@@ -1639,6 +1663,18 @@ static int gfx_v9_0_sw_init(void *handle)
 	if (r)
 		return r;
 
+	/* ECC error */
+	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_ECC_ERROR,
+			      &adev->gfx.cp_ecc_error_irq);
+	if (r)
+		return r;
+
+	/* FUE error */
+	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_FUE_ERROR,
+			      &adev->gfx.cp_ecc_error_irq);
+	if (r)
+		return r;
+
 	adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
 
 	gfx_v9_0_scratch_init(adev);
@@ -1731,6 +1767,20 @@ static int gfx_v9_0_sw_fini(void *handle)
 	int i;
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
+	if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX) &&
+			adev->gfx.ras_if) {
+		struct ras_common_if *ras_if = adev->gfx.ras_if;
+		struct ras_ih_if ih_info = {
+			.head = *ras_if,
+		};
+
+		amdgpu_ras_debugfs_remove(adev, ras_if);
+		amdgpu_ras_sysfs_remove(adev, ras_if);
+		amdgpu_ras_interrupt_remove_handler(adev,  &ih_info);
+		amdgpu_ras_feature_enable(adev, ras_if, 0);
+		kfree(ras_if);
+	}
+
 	amdgpu_bo_free_kernel(&adev->gds.oa_gfx_bo, NULL, NULL);
 	amdgpu_bo_free_kernel(&adev->gds.gws_gfx_bo, NULL, NULL);
 	amdgpu_bo_free_kernel(&adev->gds.gds_gfx_bo, NULL, NULL);
@@ -3303,6 +3353,7 @@ static int gfx_v9_0_hw_fini(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
+	amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0);
 	amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
 	amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
 
@@ -3492,6 +3543,80 @@ static int gfx_v9_0_early_init(void *handle)
 	return 0;
 }
 
+static int gfx_v9_0_process_ras_data_cb(struct amdgpu_device *adev,
+		struct amdgpu_iv_entry *entry);
+
+static int gfx_v9_0_ecc_late_init(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	struct ras_common_if **ras_if = &adev->gfx.ras_if;
+	struct ras_ih_if ih_info = {
+		.cb = gfx_v9_0_process_ras_data_cb,
+	};
+	struct ras_fs_if fs_info = {
+		.sysfs_name = "gfx_err_count",
+		.debugfs_name = "gfx_err_inject",
+	};
+	struct ras_common_if ras_block = {
+		.block = AMDGPU_RAS_BLOCK__GFX,
+		.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE,
+		.sub_block_index = 0,
+		.name = "gfx",
+	};
+	int r;
+
+	if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) {
+		amdgpu_ras_feature_enable_on_boot(adev, &ras_block, 0);
+		return 0;
+	}
+
+	if (*ras_if)
+		goto resume;
+
+	*ras_if = kmalloc(sizeof(**ras_if), GFP_KERNEL);
+	if (!*ras_if)
+		return -ENOMEM;
+
+	**ras_if = ras_block;
+
+	r = amdgpu_ras_feature_enable_on_boot(adev, *ras_if, 1);
+	if (r)
+		goto feature;
+
+	ih_info.head = **ras_if;
+	fs_info.head = **ras_if;
+
+	r = amdgpu_ras_interrupt_add_handler(adev, &ih_info);
+	if (r)
+		goto interrupt;
+
+	r = amdgpu_ras_debugfs_create(adev, &fs_info);
+	if (r)
+		goto debugfs;
+
+	r = amdgpu_ras_sysfs_create(adev, &fs_info);
+	if (r)
+		goto sysfs;
+resume:
+	r = amdgpu_irq_get(adev, &adev->gfx.cp_ecc_error_irq, 0);
+	if (r)
+		goto irq;
+
+	return 0;
+irq:
+	amdgpu_ras_sysfs_remove(adev, *ras_if);
+sysfs:
+	amdgpu_ras_debugfs_remove(adev, *ras_if);
+debugfs:
+	amdgpu_ras_interrupt_remove_handler(adev, &ih_info);
+interrupt:
+	amdgpu_ras_feature_enable(adev, *ras_if, 0);
+feature:
+	kfree(*ras_if);
+	*ras_if = NULL;
+	return -EINVAL;
+}
+
 static int gfx_v9_0_late_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
@@ -3505,6 +3630,10 @@ static int gfx_v9_0_late_init(void *handle)
 	if (r)
 		return r;
 
+	r = gfx_v9_0_ecc_late_init(handle);
+	if (r)
+		return r;
+
 	return 0;
 }
 
@@ -4541,6 +4670,45 @@ static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
 	return 0;
 }
 
+#define ENABLE_ECC_ON_ME_PIPE(me, pipe)				\
+	WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
+			CP_ECC_ERROR_INT_ENABLE, 1)
+
+#define DISABLE_ECC_ON_ME_PIPE(me, pipe)			\
+	WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
+			CP_ECC_ERROR_INT_ENABLE, 0)
+
+static int gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device *adev,
+					      struct amdgpu_irq_src *source,
+					      unsigned type,
+					      enum amdgpu_interrupt_state state)
+{
+	switch (state) {
+	case AMDGPU_IRQ_STATE_DISABLE:
+		WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
+				CP_ECC_ERROR_INT_ENABLE, 0);
+		DISABLE_ECC_ON_ME_PIPE(1, 0);
+		DISABLE_ECC_ON_ME_PIPE(1, 1);
+		DISABLE_ECC_ON_ME_PIPE(1, 2);
+		DISABLE_ECC_ON_ME_PIPE(1, 3);
+		break;
+
+	case AMDGPU_IRQ_STATE_ENABLE:
+		WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
+				CP_ECC_ERROR_INT_ENABLE, 1);
+		ENABLE_ECC_ON_ME_PIPE(1, 0);
+		ENABLE_ECC_ON_ME_PIPE(1, 1);
+		ENABLE_ECC_ON_ME_PIPE(1, 2);
+		ENABLE_ECC_ON_ME_PIPE(1, 3);
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+
 static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev,
 					    struct amdgpu_irq_src *src,
 					    unsigned type,
@@ -4657,6 +4825,34 @@ static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev,
 	return 0;
 }
 
+static int gfx_v9_0_process_ras_data_cb(struct amdgpu_device *adev,
+		struct amdgpu_iv_entry *entry)
+{
+	/* TODO ue will trigger an interrupt. */
+	kgd2kfd_set_sram_ecc_flag(adev->kfd.dev);
+	amdgpu_ras_reset_gpu(adev, 0);
+	return AMDGPU_RAS_UE;
+}
+
+static int gfx_v9_0_cp_ecc_error_irq(struct amdgpu_device *adev,
+				  struct amdgpu_irq_src *source,
+				  struct amdgpu_iv_entry *entry)
+{
+	struct ras_common_if *ras_if = adev->gfx.ras_if;
+	struct ras_dispatch_if ih_data = {
+		.entry = entry,
+	};
+
+	if (!ras_if)
+		return 0;
+
+	ih_data.head = *ras_if;
+
+	DRM_ERROR("CP ECC ERROR IRQ\n");
+	amdgpu_ras_interrupt_dispatch(adev, &ih_data);
+	return 0;
+}
+
 static const struct amd_ip_funcs gfx_v9_0_ip_funcs = {
 	.name = "gfx_v9_0",
 	.early_init = gfx_v9_0_early_init,
@@ -4818,6 +5014,12 @@ static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = {
 	.process = gfx_v9_0_priv_inst_irq,
 };
 
+static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = {
+	.set = gfx_v9_0_set_cp_ecc_error_state,
+	.process = gfx_v9_0_cp_ecc_error_irq,
+};
+
+
 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev)
 {
 	adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
@@ -4828,6 +5030,9 @@ static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev)
 
 	adev->gfx.priv_inst_irq.num_types = 1;
 	adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs;
+
+	adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/
+	adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs;
 }
 
 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
index f5edddf3b29d..7bb5359d0bbd 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c
@@ -143,7 +143,7 @@ static void gfxhub_v1_0_init_cache_regs(struct amdgpu_device *adev)
 	/* XXX for emulation, Refer to closed source code.*/
 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, L2_PDE0_CACHE_TAG_GENERATION_MODE,
 			    0);
-	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, PDE_FAULT_CLASSIFICATION, 1);
+	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, PDE_FAULT_CLASSIFICATION, 0);
 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1);
 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, IDENTITY_MODE_FRAGMENT_SIZE, 0);
 	WREG32_SOC15(GC, 0, mmVM_L2_CNTL, tmp);
@@ -236,7 +236,7 @@ static void gfxhub_v1_0_setup_vmid_config(struct amdgpu_device *adev)
 				    block_size);
 		/* Send no-retry XNACK on fault to suppress VM fault storm. */
 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
-				    RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0);
+				    RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 1);
 		WREG32_SOC15_OFFSET(GC, 0, mmVM_CONTEXT1_CNTL, i, tmp);
 		WREG32_SOC15_OFFSET(GC, 0, mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, i*2, 0);
 		WREG32_SOC15_OFFSET(GC, 0, mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32, i*2, 0);
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
index 9fc3296592fe..b06d876da2d9 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
@@ -225,7 +225,7 @@ static void gmc_v6_0_vram_gtt_location(struct amdgpu_device *adev,
 	u64 base = RREG32(mmMC_VM_FB_LOCATION) & 0xFFFF;
 	base <<= 24;
 
-	amdgpu_gmc_vram_location(adev, &adev->gmc, base);
+	amdgpu_gmc_vram_location(adev, mc, base);
 	amdgpu_gmc_gart_location(adev, mc);
 }
 
@@ -383,20 +383,6 @@ static uint64_t gmc_v6_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
 	return pd_addr;
 }
 
-static int gmc_v6_0_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr,
-				uint32_t gpu_page_idx, uint64_t addr,
-				uint64_t flags)
-{
-	void __iomem *ptr = (void *)cpu_pt_addr;
-	uint64_t value;
-
-	value = addr & 0xFFFFFFFFFFFFF000ULL;
-	value |= flags;
-	writeq(value, ptr + (gpu_page_idx * 8));
-
-	return 0;
-}
-
 static uint64_t gmc_v6_0_get_vm_pte_flags(struct amdgpu_device *adev,
 					  uint32_t flags)
 {
@@ -886,7 +872,7 @@ static int gmc_v6_0_sw_init(void *handle)
 		pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(32));
 		dev_warn(adev->dev, "amdgpu: No coherent DMA available.\n");
 	}
-	adev->need_swiotlb = drm_get_max_iomem() > ((u64)1 << dma_bits);
+	adev->need_swiotlb = drm_need_swiotlb(dma_bits);
 
 	r = gmc_v6_0_init_microcode(adev);
 	if (r) {
@@ -1169,7 +1155,6 @@ static const struct amd_ip_funcs gmc_v6_0_ip_funcs = {
 static const struct amdgpu_gmc_funcs gmc_v6_0_gmc_funcs = {
 	.flush_gpu_tlb = gmc_v6_0_flush_gpu_tlb,
 	.emit_flush_gpu_tlb = gmc_v6_0_emit_flush_gpu_tlb,
-	.set_pte_pde = gmc_v6_0_set_pte_pde,
 	.set_prt = gmc_v6_0_set_prt,
 	.get_vm_pde = gmc_v6_0_get_vm_pde,
 	.get_vm_pte_flags = gmc_v6_0_get_vm_pte_flags
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
index 761dcfb2fec0..75aa3332aee2 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
@@ -242,7 +242,7 @@ static void gmc_v7_0_vram_gtt_location(struct amdgpu_device *adev,
 	u64 base = RREG32(mmMC_VM_FB_LOCATION) & 0xFFFF;
 	base <<= 24;
 
-	amdgpu_gmc_vram_location(adev, &adev->gmc, base);
+	amdgpu_gmc_vram_location(adev, mc, base);
 	amdgpu_gmc_gart_location(adev, mc);
 }
 
@@ -460,31 +460,6 @@ static void gmc_v7_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid,
 	amdgpu_ring_emit_wreg(ring, mmIH_VMID_0_LUT + vmid, pasid);
 }
 
-/**
- * gmc_v7_0_set_pte_pde - update the page tables using MMIO
- *
- * @adev: amdgpu_device pointer
- * @cpu_pt_addr: cpu address of the page table
- * @gpu_page_idx: entry in the page table to update
- * @addr: dst addr to write into pte/pde
- * @flags: access flags
- *
- * Update the page tables using the CPU.
- */
-static int gmc_v7_0_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr,
-				 uint32_t gpu_page_idx, uint64_t addr,
-				 uint64_t flags)
-{
-	void __iomem *ptr = (void *)cpu_pt_addr;
-	uint64_t value;
-
-	value = addr & 0xFFFFFFFFFFFFF000ULL;
-	value |= flags;
-	writeq(value, ptr + (gpu_page_idx * 8));
-
-	return 0;
-}
-
 static uint64_t gmc_v7_0_get_vm_pte_flags(struct amdgpu_device *adev,
 					  uint32_t flags)
 {
@@ -1030,7 +1005,7 @@ static int gmc_v7_0_sw_init(void *handle)
 		pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(32));
 		pr_warn("amdgpu: No coherent DMA available\n");
 	}
-	adev->need_swiotlb = drm_get_max_iomem() > ((u64)1 << dma_bits);
+	adev->need_swiotlb = drm_need_swiotlb(dma_bits);
 
 	r = gmc_v7_0_init_microcode(adev);
 	if (r) {
@@ -1376,7 +1351,6 @@ static const struct amdgpu_gmc_funcs gmc_v7_0_gmc_funcs = {
 	.flush_gpu_tlb = gmc_v7_0_flush_gpu_tlb,
 	.emit_flush_gpu_tlb = gmc_v7_0_emit_flush_gpu_tlb,
 	.emit_pasid_mapping = gmc_v7_0_emit_pasid_mapping,
-	.set_pte_pde = gmc_v7_0_set_pte_pde,
 	.set_prt = gmc_v7_0_set_prt,
 	.get_vm_pte_flags = gmc_v7_0_get_vm_pte_flags,
 	.get_vm_pde = gmc_v7_0_get_vm_pde
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
index 34440672f938..8a3b5e6fc6c9 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
@@ -433,7 +433,7 @@ static void gmc_v8_0_vram_gtt_location(struct amdgpu_device *adev,
 		base = RREG32(mmMC_VM_FB_LOCATION) & 0xFFFF;
 	base <<= 24;
 
-	amdgpu_gmc_vram_location(adev, &adev->gmc, base);
+	amdgpu_gmc_vram_location(adev, mc, base);
 	amdgpu_gmc_gart_location(adev, mc);
 }
 
@@ -662,50 +662,26 @@ static void gmc_v8_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid,
 	amdgpu_ring_emit_wreg(ring, mmIH_VMID_0_LUT + vmid, pasid);
 }
 
-/**
- * gmc_v8_0_set_pte_pde - update the page tables using MMIO
- *
- * @adev: amdgpu_device pointer
- * @cpu_pt_addr: cpu address of the page table
- * @gpu_page_idx: entry in the page table to update
- * @addr: dst addr to write into pte/pde
- * @flags: access flags
+/*
+ * PTE format on VI:
+ * 63:40 reserved
+ * 39:12 4k physical page base address
+ * 11:7 fragment
+ * 6 write
+ * 5 read
+ * 4 exe
+ * 3 reserved
+ * 2 snooped
+ * 1 system
+ * 0 valid
  *
- * Update the page tables using the CPU.
+ * PDE format on VI:
+ * 63:59 block fragment size
+ * 58:40 reserved
+ * 39:1 physical base address of PTE
+ * bits 5:1 must be 0.
+ * 0 valid
  */
-static int gmc_v8_0_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr,
-				uint32_t gpu_page_idx, uint64_t addr,
-				uint64_t flags)
-{
-	void __iomem *ptr = (void *)cpu_pt_addr;
-	uint64_t value;
-
-	/*
-	 * PTE format on VI:
-	 * 63:40 reserved
-	 * 39:12 4k physical page base address
-	 * 11:7 fragment
-	 * 6 write
-	 * 5 read
-	 * 4 exe
-	 * 3 reserved
-	 * 2 snooped
-	 * 1 system
-	 * 0 valid
-	 *
-	 * PDE format on VI:
-	 * 63:59 block fragment size
-	 * 58:40 reserved
-	 * 39:1 physical base address of PTE
-	 * bits 5:1 must be 0.
-	 * 0 valid
-	 */
-	value = addr & 0x000000FFFFFFF000ULL;
-	value |= flags;
-	writeq(value, ptr + (gpu_page_idx * 8));
-
-	return 0;
-}
 
 static uint64_t gmc_v8_0_get_vm_pte_flags(struct amdgpu_device *adev,
 					  uint32_t flags)
@@ -1155,7 +1131,7 @@ static int gmc_v8_0_sw_init(void *handle)
 		pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(32));
 		pr_warn("amdgpu: No coherent DMA available\n");
 	}
-	adev->need_swiotlb = drm_get_max_iomem() > ((u64)1 << dma_bits);
+	adev->need_swiotlb = drm_need_swiotlb(dma_bits);
 
 	r = gmc_v8_0_init_microcode(adev);
 	if (r) {
@@ -1743,7 +1719,6 @@ static const struct amdgpu_gmc_funcs gmc_v8_0_gmc_funcs = {
 	.flush_gpu_tlb = gmc_v8_0_flush_gpu_tlb,
 	.emit_flush_gpu_tlb = gmc_v8_0_emit_flush_gpu_tlb,
 	.emit_pasid_mapping = gmc_v8_0_emit_pasid_mapping,
-	.set_pte_pde = gmc_v8_0_set_pte_pde,
 	.set_prt = gmc_v8_0_set_prt,
 	.get_vm_pte_flags = gmc_v8_0_get_vm_pte_flags,
 	.get_vm_pde = gmc_v8_0_get_vm_pde
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index 2fe8397241ea..3fd79e07944d 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
@@ -47,6 +47,8 @@
 
 #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h"
 
+#include "amdgpu_ras.h"
+
 /* add these here since we already include dce12 headers and these are for DCN */
 #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION                                                          0x055d
 #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX                                                 2
@@ -84,121 +86,182 @@ static const struct soc15_reg_golden golden_settings_athub_1_0_0[] =
 	SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL2, 0x00ff00ff, 0x00080008)
 };
 
-/* Ecc related register addresses, (BASE + reg offset) */
-/* Universal Memory Controller caps (may be fused). */
-/* UMCCH:UmcLocalCap */
-#define UMCLOCALCAPS_ADDR0	(0x00014306 + 0x00000000)
-#define UMCLOCALCAPS_ADDR1	(0x00014306 + 0x00000800)
-#define UMCLOCALCAPS_ADDR2	(0x00014306 + 0x00001000)
-#define UMCLOCALCAPS_ADDR3	(0x00014306 + 0x00001800)
-#define UMCLOCALCAPS_ADDR4	(0x00054306 + 0x00000000)
-#define UMCLOCALCAPS_ADDR5	(0x00054306 + 0x00000800)
-#define UMCLOCALCAPS_ADDR6	(0x00054306 + 0x00001000)
-#define UMCLOCALCAPS_ADDR7	(0x00054306 + 0x00001800)
-#define UMCLOCALCAPS_ADDR8	(0x00094306 + 0x00000000)
-#define UMCLOCALCAPS_ADDR9	(0x00094306 + 0x00000800)
-#define UMCLOCALCAPS_ADDR10	(0x00094306 + 0x00001000)
-#define UMCLOCALCAPS_ADDR11	(0x00094306 + 0x00001800)
-#define UMCLOCALCAPS_ADDR12	(0x000d4306 + 0x00000000)
-#define UMCLOCALCAPS_ADDR13	(0x000d4306 + 0x00000800)
-#define UMCLOCALCAPS_ADDR14	(0x000d4306 + 0x00001000)
-#define UMCLOCALCAPS_ADDR15	(0x000d4306 + 0x00001800)
-
-/* Universal Memory Controller Channel config. */
-/* UMCCH:UMC_CONFIG */
-#define UMCCH_UMC_CONFIG_ADDR0	(0x00014040 + 0x00000000)
-#define UMCCH_UMC_CONFIG_ADDR1	(0x00014040 + 0x00000800)
-#define UMCCH_UMC_CONFIG_ADDR2	(0x00014040 + 0x00001000)
-#define UMCCH_UMC_CONFIG_ADDR3	(0x00014040 + 0x00001800)
-#define UMCCH_UMC_CONFIG_ADDR4	(0x00054040 + 0x00000000)
-#define UMCCH_UMC_CONFIG_ADDR5	(0x00054040 + 0x00000800)
-#define UMCCH_UMC_CONFIG_ADDR6	(0x00054040 + 0x00001000)
-#define UMCCH_UMC_CONFIG_ADDR7	(0x00054040 + 0x00001800)
-#define UMCCH_UMC_CONFIG_ADDR8	(0x00094040 + 0x00000000)
-#define UMCCH_UMC_CONFIG_ADDR9	(0x00094040 + 0x00000800)
-#define UMCCH_UMC_CONFIG_ADDR10	(0x00094040 + 0x00001000)
-#define UMCCH_UMC_CONFIG_ADDR11	(0x00094040 + 0x00001800)
-#define UMCCH_UMC_CONFIG_ADDR12	(0x000d4040 + 0x00000000)
-#define UMCCH_UMC_CONFIG_ADDR13	(0x000d4040 + 0x00000800)
-#define UMCCH_UMC_CONFIG_ADDR14	(0x000d4040 + 0x00001000)
-#define UMCCH_UMC_CONFIG_ADDR15	(0x000d4040 + 0x00001800)
-
-/* Universal Memory Controller Channel Ecc config. */
-/* UMCCH:EccCtrl */
-#define UMCCH_ECCCTRL_ADDR0	(0x00014053 + 0x00000000)
-#define UMCCH_ECCCTRL_ADDR1	(0x00014053 + 0x00000800)
-#define UMCCH_ECCCTRL_ADDR2	(0x00014053 + 0x00001000)
-#define UMCCH_ECCCTRL_ADDR3	(0x00014053 + 0x00001800)
-#define UMCCH_ECCCTRL_ADDR4	(0x00054053 + 0x00000000)
-#define UMCCH_ECCCTRL_ADDR5	(0x00054053 + 0x00000800)
-#define UMCCH_ECCCTRL_ADDR6	(0x00054053 + 0x00001000)
-#define UMCCH_ECCCTRL_ADDR7	(0x00054053 + 0x00001800)
-#define UMCCH_ECCCTRL_ADDR8	(0x00094053 + 0x00000000)
-#define UMCCH_ECCCTRL_ADDR9	(0x00094053 + 0x00000800)
-#define UMCCH_ECCCTRL_ADDR10	(0x00094053 + 0x00001000)
-#define UMCCH_ECCCTRL_ADDR11	(0x00094053 + 0x00001800)
-#define UMCCH_ECCCTRL_ADDR12	(0x000d4053 + 0x00000000)
-#define UMCCH_ECCCTRL_ADDR13	(0x000d4053 + 0x00000800)
-#define UMCCH_ECCCTRL_ADDR14	(0x000d4053 + 0x00001000)
-#define UMCCH_ECCCTRL_ADDR15	(0x000d4053 + 0x00001800)
-
-static const uint32_t ecc_umclocalcap_addrs[] = {
-	UMCLOCALCAPS_ADDR0,
-	UMCLOCALCAPS_ADDR1,
-	UMCLOCALCAPS_ADDR2,
-	UMCLOCALCAPS_ADDR3,
-	UMCLOCALCAPS_ADDR4,
-	UMCLOCALCAPS_ADDR5,
-	UMCLOCALCAPS_ADDR6,
-	UMCLOCALCAPS_ADDR7,
-	UMCLOCALCAPS_ADDR8,
-	UMCLOCALCAPS_ADDR9,
-	UMCLOCALCAPS_ADDR10,
-	UMCLOCALCAPS_ADDR11,
-	UMCLOCALCAPS_ADDR12,
-	UMCLOCALCAPS_ADDR13,
-	UMCLOCALCAPS_ADDR14,
-	UMCLOCALCAPS_ADDR15,
+static const uint32_t ecc_umc_mcumc_ctrl_addrs[] = {
+	(0x000143c0 + 0x00000000),
+	(0x000143c0 + 0x00000800),
+	(0x000143c0 + 0x00001000),
+	(0x000143c0 + 0x00001800),
+	(0x000543c0 + 0x00000000),
+	(0x000543c0 + 0x00000800),
+	(0x000543c0 + 0x00001000),
+	(0x000543c0 + 0x00001800),
+	(0x000943c0 + 0x00000000),
+	(0x000943c0 + 0x00000800),
+	(0x000943c0 + 0x00001000),
+	(0x000943c0 + 0x00001800),
+	(0x000d43c0 + 0x00000000),
+	(0x000d43c0 + 0x00000800),
+	(0x000d43c0 + 0x00001000),
+	(0x000d43c0 + 0x00001800),
+	(0x001143c0 + 0x00000000),
+	(0x001143c0 + 0x00000800),
+	(0x001143c0 + 0x00001000),
+	(0x001143c0 + 0x00001800),
+	(0x001543c0 + 0x00000000),
+	(0x001543c0 + 0x00000800),
+	(0x001543c0 + 0x00001000),
+	(0x001543c0 + 0x00001800),
+	(0x001943c0 + 0x00000000),
+	(0x001943c0 + 0x00000800),
+	(0x001943c0 + 0x00001000),
+	(0x001943c0 + 0x00001800),
+	(0x001d43c0 + 0x00000000),
+	(0x001d43c0 + 0x00000800),
+	(0x001d43c0 + 0x00001000),
+	(0x001d43c0 + 0x00001800),
 };
 
-static const uint32_t ecc_umcch_umc_config_addrs[] = {
-	UMCCH_UMC_CONFIG_ADDR0,
-	UMCCH_UMC_CONFIG_ADDR1,
-	UMCCH_UMC_CONFIG_ADDR2,
-	UMCCH_UMC_CONFIG_ADDR3,
-	UMCCH_UMC_CONFIG_ADDR4,
-	UMCCH_UMC_CONFIG_ADDR5,
-	UMCCH_UMC_CONFIG_ADDR6,
-	UMCCH_UMC_CONFIG_ADDR7,
-	UMCCH_UMC_CONFIG_ADDR8,
-	UMCCH_UMC_CONFIG_ADDR9,
-	UMCCH_UMC_CONFIG_ADDR10,
-	UMCCH_UMC_CONFIG_ADDR11,
-	UMCCH_UMC_CONFIG_ADDR12,
-	UMCCH_UMC_CONFIG_ADDR13,
-	UMCCH_UMC_CONFIG_ADDR14,
-	UMCCH_UMC_CONFIG_ADDR15,
+static const uint32_t ecc_umc_mcumc_ctrl_mask_addrs[] = {
+	(0x000143e0 + 0x00000000),
+	(0x000143e0 + 0x00000800),
+	(0x000143e0 + 0x00001000),
+	(0x000143e0 + 0x00001800),
+	(0x000543e0 + 0x00000000),
+	(0x000543e0 + 0x00000800),
+	(0x000543e0 + 0x00001000),
+	(0x000543e0 + 0x00001800),
+	(0x000943e0 + 0x00000000),
+	(0x000943e0 + 0x00000800),
+	(0x000943e0 + 0x00001000),
+	(0x000943e0 + 0x00001800),
+	(0x000d43e0 + 0x00000000),
+	(0x000d43e0 + 0x00000800),
+	(0x000d43e0 + 0x00001000),
+	(0x000d43e0 + 0x00001800),
+	(0x001143e0 + 0x00000000),
+	(0x001143e0 + 0x00000800),
+	(0x001143e0 + 0x00001000),
+	(0x001143e0 + 0x00001800),
+	(0x001543e0 + 0x00000000),
+	(0x001543e0 + 0x00000800),
+	(0x001543e0 + 0x00001000),
+	(0x001543e0 + 0x00001800),
+	(0x001943e0 + 0x00000000),
+	(0x001943e0 + 0x00000800),
+	(0x001943e0 + 0x00001000),
+	(0x001943e0 + 0x00001800),
+	(0x001d43e0 + 0x00000000),
+	(0x001d43e0 + 0x00000800),
+	(0x001d43e0 + 0x00001000),
+	(0x001d43e0 + 0x00001800),
 };
 
-static const uint32_t ecc_umcch_eccctrl_addrs[] = {
-	UMCCH_ECCCTRL_ADDR0,
-	UMCCH_ECCCTRL_ADDR1,
-	UMCCH_ECCCTRL_ADDR2,
-	UMCCH_ECCCTRL_ADDR3,
-	UMCCH_ECCCTRL_ADDR4,
-	UMCCH_ECCCTRL_ADDR5,
-	UMCCH_ECCCTRL_ADDR6,
-	UMCCH_ECCCTRL_ADDR7,
-	UMCCH_ECCCTRL_ADDR8,
-	UMCCH_ECCCTRL_ADDR9,
-	UMCCH_ECCCTRL_ADDR10,
-	UMCCH_ECCCTRL_ADDR11,
-	UMCCH_ECCCTRL_ADDR12,
-	UMCCH_ECCCTRL_ADDR13,
-	UMCCH_ECCCTRL_ADDR14,
-	UMCCH_ECCCTRL_ADDR15,
+static const uint32_t ecc_umc_mcumc_status_addrs[] = {
+	(0x000143c2 + 0x00000000),
+	(0x000143c2 + 0x00000800),
+	(0x000143c2 + 0x00001000),
+	(0x000143c2 + 0x00001800),
+	(0x000543c2 + 0x00000000),
+	(0x000543c2 + 0x00000800),
+	(0x000543c2 + 0x00001000),
+	(0x000543c2 + 0x00001800),
+	(0x000943c2 + 0x00000000),
+	(0x000943c2 + 0x00000800),
+	(0x000943c2 + 0x00001000),
+	(0x000943c2 + 0x00001800),
+	(0x000d43c2 + 0x00000000),
+	(0x000d43c2 + 0x00000800),
+	(0x000d43c2 + 0x00001000),
+	(0x000d43c2 + 0x00001800),
+	(0x001143c2 + 0x00000000),
+	(0x001143c2 + 0x00000800),
+	(0x001143c2 + 0x00001000),
+	(0x001143c2 + 0x00001800),
+	(0x001543c2 + 0x00000000),
+	(0x001543c2 + 0x00000800),
+	(0x001543c2 + 0x00001000),
+	(0x001543c2 + 0x00001800),
+	(0x001943c2 + 0x00000000),
+	(0x001943c2 + 0x00000800),
+	(0x001943c2 + 0x00001000),
+	(0x001943c2 + 0x00001800),
+	(0x001d43c2 + 0x00000000),
+	(0x001d43c2 + 0x00000800),
+	(0x001d43c2 + 0x00001000),
+	(0x001d43c2 + 0x00001800),
 };
 
+static int gmc_v9_0_ecc_interrupt_state(struct amdgpu_device *adev,
+		struct amdgpu_irq_src *src,
+		unsigned type,
+		enum amdgpu_interrupt_state state)
+{
+	u32 bits, i, tmp, reg;
+
+	bits = 0x7f;
+
+	switch (state) {
+	case AMDGPU_IRQ_STATE_DISABLE:
+		for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) {
+			reg = ecc_umc_mcumc_ctrl_addrs[i];
+			tmp = RREG32(reg);
+			tmp &= ~bits;
+			WREG32(reg, tmp);
+		}
+		for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) {
+			reg = ecc_umc_mcumc_ctrl_mask_addrs[i];
+			tmp = RREG32(reg);
+			tmp &= ~bits;
+			WREG32(reg, tmp);
+		}
+		break;
+	case AMDGPU_IRQ_STATE_ENABLE:
+		for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) {
+			reg = ecc_umc_mcumc_ctrl_addrs[i];
+			tmp = RREG32(reg);
+			tmp |= bits;
+			WREG32(reg, tmp);
+		}
+		for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) {
+			reg = ecc_umc_mcumc_ctrl_mask_addrs[i];
+			tmp = RREG32(reg);
+			tmp |= bits;
+			WREG32(reg, tmp);
+		}
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static int gmc_v9_0_process_ras_data_cb(struct amdgpu_device *adev,
+		struct amdgpu_iv_entry *entry)
+{
+	kgd2kfd_set_sram_ecc_flag(adev->kfd.dev);
+	amdgpu_ras_reset_gpu(adev, 0);
+	return AMDGPU_RAS_UE;
+}
+
+static int gmc_v9_0_process_ecc_irq(struct amdgpu_device *adev,
+		struct amdgpu_irq_src *source,
+		struct amdgpu_iv_entry *entry)
+{
+	struct ras_common_if *ras_if = adev->gmc.ras_if;
+	struct ras_dispatch_if ih_data = {
+		.entry = entry,
+	};
+
+	if (!ras_if)
+		return 0;
+
+	ih_data.head = *ras_if;
+
+	amdgpu_ras_interrupt_dispatch(adev, &ih_data);
+	return 0;
+}
+
 static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
 					struct amdgpu_irq_src *src,
 					unsigned type,
@@ -244,62 +307,6 @@ static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
 	return 0;
 }
 
-/**
- * vega10_ih_prescreen_iv - prescreen an interrupt vector
- *
- * @adev: amdgpu_device pointer
- *
- * Returns true if the interrupt vector should be further processed.
- */
-static bool gmc_v9_0_prescreen_iv(struct amdgpu_device *adev,
-				  struct amdgpu_iv_entry *entry,
-				  uint64_t addr)
-{
-	struct amdgpu_vm *vm;
-	u64 key;
-	int r;
-
-	/* No PASID, can't identify faulting process */
-	if (!entry->pasid)
-		return true;
-
-	/* Not a retry fault */
-	if (!(entry->src_data[1] & 0x80))
-		return true;
-
-	/* Track retry faults in per-VM fault FIFO. */
-	spin_lock(&adev->vm_manager.pasid_lock);
-	vm = idr_find(&adev->vm_manager.pasid_idr, entry->pasid);
-	if (!vm) {
-		/* VM not found, process it normally */
-		spin_unlock(&adev->vm_manager.pasid_lock);
-		return true;
-	}
-
-	key = AMDGPU_VM_FAULT(entry->pasid, addr);
-	r = amdgpu_vm_add_fault(vm->fault_hash, key);
-
-	/* Hash table is full or the fault is already being processed,
-	 * ignore further page faults
-	 */
-	if (r != 0) {
-		spin_unlock(&adev->vm_manager.pasid_lock);
-		return false;
-	}
-	/* No locking required with single writer and single reader */
-	r = kfifo_put(&vm->faults, key);
-	if (!r) {
-		/* FIFO is full. Ignore it until there is space */
-		amdgpu_vm_clear_fault(vm->fault_hash, key);
-		spin_unlock(&adev->vm_manager.pasid_lock);
-		return false;
-	}
-
-	spin_unlock(&adev->vm_manager.pasid_lock);
-	/* It's the first fault for this address, process it normally */
-	return true;
-}
-
 static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev,
 				struct amdgpu_irq_src *source,
 				struct amdgpu_iv_entry *entry)
@@ -312,9 +319,11 @@ static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev,
 	addr = (u64)entry->src_data[0] << 12;
 	addr |= ((u64)entry->src_data[1] & 0xf) << 44;
 
-	if (!gmc_v9_0_prescreen_iv(adev, entry, addr))
+	if (retry_fault && amdgpu_gmc_filter_faults(adev, addr, entry->pasid,
+						    entry->timestamp))
 		return 1; /* This also prevents sending it to KFD */
 
+	/* If it's the first fault for this address, process it normally */
 	if (!amdgpu_sriov_vf(adev)) {
 		status = RREG32(hub->vm_l2_pro_fault_status);
 		WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1);
@@ -350,10 +359,19 @@ static const struct amdgpu_irq_src_funcs gmc_v9_0_irq_funcs = {
 	.process = gmc_v9_0_process_interrupt,
 };
 
+
+static const struct amdgpu_irq_src_funcs gmc_v9_0_ecc_funcs = {
+	.set = gmc_v9_0_ecc_interrupt_state,
+	.process = gmc_v9_0_process_ecc_irq,
+};
+
 static void gmc_v9_0_set_irq_funcs(struct amdgpu_device *adev)
 {
 	adev->gmc.vm_fault.num_types = 1;
 	adev->gmc.vm_fault.funcs = &gmc_v9_0_irq_funcs;
+
+	adev->gmc.ecc_irq.num_types = 1;
+	adev->gmc.ecc_irq.funcs = &gmc_v9_0_ecc_funcs;
 }
 
 static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid,
@@ -466,64 +484,37 @@ static void gmc_v9_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid,
 	amdgpu_ring_emit_wreg(ring, reg, pasid);
 }
 
-/**
- * gmc_v9_0_set_pte_pde - update the page tables using MMIO
- *
- * @adev: amdgpu_device pointer
- * @cpu_pt_addr: cpu address of the page table
- * @gpu_page_idx: entry in the page table to update
- * @addr: dst addr to write into pte/pde
- * @flags: access flags
+/*
+ * PTE format on VEGA 10:
+ * 63:59 reserved
+ * 58:57 mtype
+ * 56 F
+ * 55 L
+ * 54 P
+ * 53 SW
+ * 52 T
+ * 50:48 reserved
+ * 47:12 4k physical page base address
+ * 11:7 fragment
+ * 6 write
+ * 5 read
+ * 4 exe
+ * 3 Z
+ * 2 snooped
+ * 1 system
+ * 0 valid
  *
- * Update the page tables using the CPU.
+ * PDE format on VEGA 10:
+ * 63:59 block fragment size
+ * 58:55 reserved
+ * 54 P
+ * 53:48 reserved
+ * 47:6 physical base address of PD or PTE
+ * 5:3 reserved
+ * 2 C
+ * 1 system
+ * 0 valid
  */
-static int gmc_v9_0_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr,
-				uint32_t gpu_page_idx, uint64_t addr,
-				uint64_t flags)
-{
-	void __iomem *ptr = (void *)cpu_pt_addr;
-	uint64_t value;
-
-	/*
-	 * PTE format on VEGA 10:
-	 * 63:59 reserved
-	 * 58:57 mtype
-	 * 56 F
-	 * 55 L
-	 * 54 P
-	 * 53 SW
-	 * 52 T
-	 * 50:48 reserved
-	 * 47:12 4k physical page base address
-	 * 11:7 fragment
-	 * 6 write
-	 * 5 read
-	 * 4 exe
-	 * 3 Z
-	 * 2 snooped
-	 * 1 system
-	 * 0 valid
-	 *
-	 * PDE format on VEGA 10:
-	 * 63:59 block fragment size
-	 * 58:55 reserved
-	 * 54 P
-	 * 53:48 reserved
-	 * 47:6 physical base address of PD or PTE
-	 * 5:3 reserved
-	 * 2 C
-	 * 1 system
-	 * 0 valid
-	 */
-
-	/*
-	 * The following is for PTE only. GART does not have PDEs.
-	*/
-	value = addr & 0x0000FFFFFFFFF000ULL;
-	value |= flags;
-	writeq(value, ptr + (gpu_page_idx * 8));
-	return 0;
-}
 
 static uint64_t gmc_v9_0_get_vm_pte_flags(struct amdgpu_device *adev,
 						uint32_t flags)
@@ -593,7 +584,6 @@ static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = {
 	.flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb,
 	.emit_flush_gpu_tlb = gmc_v9_0_emit_flush_gpu_tlb,
 	.emit_pasid_mapping = gmc_v9_0_emit_pasid_mapping,
-	.set_pte_pde = gmc_v9_0_set_pte_pde,
 	.get_vm_pte_flags = gmc_v9_0_get_vm_pte_flags,
 	.get_vm_pde = gmc_v9_0_get_vm_pde
 };
@@ -620,85 +610,6 @@ static int gmc_v9_0_early_init(void *handle)
 	return 0;
 }
 
-static int gmc_v9_0_ecc_available(struct amdgpu_device *adev)
-{
-	uint32_t reg_val;
-	uint32_t reg_addr;
-	uint32_t field_val;
-	size_t i;
-	uint32_t fv2;
-	size_t lost_sheep;
-
-	DRM_DEBUG("ecc: gmc_v9_0_ecc_available()\n");
-
-	lost_sheep = 0;
-	for (i = 0; i < ARRAY_SIZE(ecc_umclocalcap_addrs); ++i) {
-		reg_addr = ecc_umclocalcap_addrs[i];
-		DRM_DEBUG("ecc: "
-			  "UMCCH_UmcLocalCap[%zu]: reg_addr: 0x%08x\n",
-			  i, reg_addr);
-		reg_val = RREG32(reg_addr);
-		field_val = REG_GET_FIELD(reg_val, UMCCH0_0_UmcLocalCap,
-					  EccDis);
-		DRM_DEBUG("ecc: "
-			  "reg_val: 0x%08x, "
-			  "EccDis: 0x%08x, ",
-			  reg_val, field_val);
-		if (field_val) {
-			DRM_ERROR("ecc: UmcLocalCap:EccDis is set.\n");
-			++lost_sheep;
-		}
-	}
-
-	for (i = 0; i < ARRAY_SIZE(ecc_umcch_umc_config_addrs); ++i) {
-		reg_addr = ecc_umcch_umc_config_addrs[i];
-		DRM_DEBUG("ecc: "
-			  "UMCCH0_0_UMC_CONFIG[%zu]: reg_addr: 0x%08x",
-			  i, reg_addr);
-		reg_val = RREG32(reg_addr);
-		field_val = REG_GET_FIELD(reg_val, UMCCH0_0_UMC_CONFIG,
-					  DramReady);
-		DRM_DEBUG("ecc: "
-			  "reg_val: 0x%08x, "
-			  "DramReady: 0x%08x\n",
-			  reg_val, field_val);
-
-		if (!field_val) {
-			DRM_ERROR("ecc: UMC_CONFIG:DramReady is not set.\n");
-			++lost_sheep;
-		}
-	}
-
-	for (i = 0; i < ARRAY_SIZE(ecc_umcch_eccctrl_addrs); ++i) {
-		reg_addr = ecc_umcch_eccctrl_addrs[i];
-		DRM_DEBUG("ecc: "
-			  "UMCCH_EccCtrl[%zu]: reg_addr: 0x%08x, ",
-			  i, reg_addr);
-		reg_val = RREG32(reg_addr);
-		field_val = REG_GET_FIELD(reg_val, UMCCH0_0_EccCtrl,
-					  WrEccEn);
-		fv2 = REG_GET_FIELD(reg_val, UMCCH0_0_EccCtrl,
-				    RdEccEn);
-		DRM_DEBUG("ecc: "
-			  "reg_val: 0x%08x, "
-			  "WrEccEn: 0x%08x, "
-			  "RdEccEn: 0x%08x\n",
-			  reg_val, field_val, fv2);
-
-		if (!field_val) {
-			DRM_DEBUG("ecc: WrEccEn is not set\n");
-			++lost_sheep;
-		}
-		if (!fv2) {
-			DRM_DEBUG("ecc: RdEccEn is not set\n");
-			++lost_sheep;
-		}
-	}
-
-	DRM_DEBUG("ecc: lost_sheep: %zu\n", lost_sheep);
-	return lost_sheep == 0;
-}
-
 static bool gmc_v9_0_keep_stolen_memory(struct amdgpu_device *adev)
 {
 
@@ -751,31 +662,119 @@ static int gmc_v9_0_allocate_vm_inv_eng(struct amdgpu_device *adev)
 	return 0;
 }
 
-static int gmc_v9_0_late_init(void *handle)
+static int gmc_v9_0_ecc_late_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	struct ras_common_if **ras_if = &adev->gmc.ras_if;
+	struct ras_ih_if ih_info = {
+		.cb = gmc_v9_0_process_ras_data_cb,
+	};
+	struct ras_fs_if fs_info = {
+		.sysfs_name = "umc_err_count",
+		.debugfs_name = "umc_err_inject",
+	};
+	struct ras_common_if ras_block = {
+		.block = AMDGPU_RAS_BLOCK__UMC,
+		.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE,
+		.sub_block_index = 0,
+		.name = "umc",
+	};
 	int r;
 
+	if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__UMC)) {
+		amdgpu_ras_feature_enable_on_boot(adev, &ras_block, 0);
+		return 0;
+	}
+	/* handle resume path. */
+	if (*ras_if)
+		goto resume;
+
+	*ras_if = kmalloc(sizeof(**ras_if), GFP_KERNEL);
+	if (!*ras_if)
+		return -ENOMEM;
+
+	**ras_if = ras_block;
+
+	r = amdgpu_ras_feature_enable_on_boot(adev, *ras_if, 1);
+	if (r)
+		goto feature;
+
+	ih_info.head = **ras_if;
+	fs_info.head = **ras_if;
+
+	r = amdgpu_ras_interrupt_add_handler(adev, &ih_info);
+	if (r)
+		goto interrupt;
+
+	r = amdgpu_ras_debugfs_create(adev, &fs_info);
+	if (r)
+		goto debugfs;
+
+	r = amdgpu_ras_sysfs_create(adev, &fs_info);
+	if (r)
+		goto sysfs;
+resume:
+	r = amdgpu_irq_get(adev, &adev->gmc.ecc_irq, 0);
+	if (r)
+		goto irq;
+
+	return 0;
+irq:
+	amdgpu_ras_sysfs_remove(adev, *ras_if);
+sysfs:
+	amdgpu_ras_debugfs_remove(adev, *ras_if);
+debugfs:
+	amdgpu_ras_interrupt_remove_handler(adev, &ih_info);
+interrupt:
+	amdgpu_ras_feature_enable(adev, *ras_if, 0);
+feature:
+	kfree(*ras_if);
+	*ras_if = NULL;
+	return -EINVAL;
+}
+
+
+static int gmc_v9_0_late_init(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	bool r;
+
 	if (!gmc_v9_0_keep_stolen_memory(adev))
 		amdgpu_bo_late_init(adev);
 
 	r = gmc_v9_0_allocate_vm_inv_eng(adev);
 	if (r)
 		return r;
+	/* Check if ecc is available */
+	if (!amdgpu_sriov_vf(adev)) {
+		switch (adev->asic_type) {
+		case CHIP_VEGA10:
+		case CHIP_VEGA20:
+			r = amdgpu_atomfirmware_mem_ecc_supported(adev);
+			if (!r) {
+				DRM_INFO("ECC is not present.\n");
+				if (adev->df_funcs->enable_ecc_force_par_wr_rmw)
+					adev->df_funcs->enable_ecc_force_par_wr_rmw(adev, false);
+			} else {
+				DRM_INFO("ECC is active.\n");
+			}
 
-	if (adev->asic_type == CHIP_VEGA10 && !amdgpu_sriov_vf(adev)) {
-		r = gmc_v9_0_ecc_available(adev);
-		if (r == 1) {
-			DRM_INFO("ECC is active.\n");
-		} else if (r == 0) {
-			DRM_INFO("ECC is not present.\n");
-			adev->df_funcs->enable_ecc_force_par_wr_rmw(adev, false);
-		} else {
-			DRM_ERROR("gmc_v9_0_ecc_available() failed. r: %d\n", r);
-			return r;
+			r = amdgpu_atomfirmware_sram_ecc_supported(adev);
+			if (!r) {
+				DRM_INFO("SRAM ECC is not present.\n");
+			} else {
+				DRM_INFO("SRAM ECC is active.\n");
+			}
+			break;
+		default:
+			break;
 		}
 	}
 
+	r = gmc_v9_0_ecc_late_init(handle);
+	if (r)
+		return r;
+
 	return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0);
 }
 
@@ -787,7 +786,7 @@ static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev,
 		base = mmhub_v1_0_get_fb_location(adev);
 	/* add the xgmi offset of the physical node */
 	base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
-	amdgpu_gmc_vram_location(adev, &adev->gmc, base);
+	amdgpu_gmc_vram_location(adev, mc, base);
 	amdgpu_gmc_gart_location(adev, mc);
 	if (!amdgpu_sriov_vf(adev))
 		amdgpu_gmc_agp_location(adev, mc);
@@ -987,6 +986,12 @@ static int gmc_v9_0_sw_init(void *handle)
 	if (r)
 		return r;
 
+	/* interrupt sent to DF. */
+	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DF, 0,
+			&adev->gmc.ecc_irq);
+	if (r)
+		return r;
+
 	/* Set the internal MC address mask
 	 * This is the max address of the GPU's
 	 * internal address space.
@@ -1011,7 +1016,7 @@ static int gmc_v9_0_sw_init(void *handle)
 		pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(32));
 		printk(KERN_WARNING "amdgpu: No coherent DMA available.\n");
 	}
-	adev->need_swiotlb = drm_get_max_iomem() > ((u64)1 << dma_bits);
+	adev->need_swiotlb = drm_need_swiotlb(dma_bits);
 
 	if (adev->gmc.xgmi.supported) {
 		r = gfxhub_v1_1_get_xgmi_info(adev);
@@ -1052,6 +1057,22 @@ static int gmc_v9_0_sw_fini(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
+	if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__UMC) &&
+			adev->gmc.ras_if) {
+		struct ras_common_if *ras_if = adev->gmc.ras_if;
+		struct ras_ih_if ih_info = {
+			.head = *ras_if,
+		};
+
+		/*remove fs first*/
+		amdgpu_ras_debugfs_remove(adev, ras_if);
+		amdgpu_ras_sysfs_remove(adev, ras_if);
+		/*remove the IH*/
+		amdgpu_ras_interrupt_remove_handler(adev, &ih_info);
+		amdgpu_ras_feature_enable(adev, ras_if, 0);
+		kfree(ras_if);
+	}
+
 	amdgpu_gem_force_release(adev);
 	amdgpu_vm_manager_fini(adev);
 
@@ -1198,6 +1219,7 @@ static int gmc_v9_0_hw_fini(void *handle)
 		return 0;
 	}
 
+	amdgpu_irq_put(adev, &adev->gmc.ecc_irq, 0);
 	amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0);
 	gmc_v9_0_gart_disable(adev);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/kv_dpm.c b/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
index 0c9a2c03504e..f2e6b148ccad 100644
--- a/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
@@ -2824,7 +2824,7 @@ static int kv_dpm_init(struct amdgpu_device *adev)
 		pi->caps_tcp_ramping = true;
 	}
 
-	if (adev->powerplay.pp_feature & PP_SCLK_DEEP_SLEEP_MASK)
+	if (adev->pm.pp_feature & PP_SCLK_DEEP_SLEEP_MASK)
 		pi->caps_sclk_ds = true;
 	else
 		pi->caps_sclk_ds = false;
diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c
index 1696644ec022..41a9a5779623 100644
--- a/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c
@@ -163,7 +163,7 @@ static void mmhub_v1_0_init_cache_regs(struct amdgpu_device *adev)
 	/* XXX for emulation, Refer to closed source code.*/
 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, L2_PDE0_CACHE_TAG_GENERATION_MODE,
 			    0);
-	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, PDE_FAULT_CLASSIFICATION, 1);
+	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, PDE_FAULT_CLASSIFICATION, 0);
 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1);
 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, IDENTITY_MODE_FRAGMENT_SIZE, 0);
 	WREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL, tmp);
@@ -256,7 +256,7 @@ static void mmhub_v1_0_setup_vmid_config(struct amdgpu_device *adev)
 				    block_size);
 		/* Send no-retry XNACK on fault to suppress VM fault storm. */
 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
-				    RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0);
+				    RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 1);
 		WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT1_CNTL, i, tmp);
 		WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, i*2, 0);
 		WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32, i*2, 0);
diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
index 73851ebb3833..8dbad496b29f 100644
--- a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
@@ -157,6 +157,82 @@ static void xgpu_ai_mailbox_trans_msg (struct amdgpu_device *adev,
 	xgpu_ai_mailbox_set_valid(adev, false);
 }
 
+static int xgpu_ai_get_pp_clk(struct amdgpu_device *adev, u32 type, char *buf)
+{
+        int r = 0;
+        u32 req, val, size;
+
+        if (!amdgim_is_hwperf(adev) || buf == NULL)
+                return -EBADRQC;
+
+        switch(type) {
+        case PP_SCLK:
+                req = IDH_IRQ_GET_PP_SCLK;
+                break;
+        case PP_MCLK:
+                req = IDH_IRQ_GET_PP_MCLK;
+                break;
+        default:
+                return -EBADRQC;
+        }
+
+        mutex_lock(&adev->virt.dpm_mutex);
+
+        xgpu_ai_mailbox_trans_msg(adev, req, 0, 0, 0);
+
+        r = xgpu_ai_poll_msg(adev, IDH_SUCCESS);
+        if (!r && adev->fw_vram_usage.va != NULL) {
+                val = RREG32_NO_KIQ(
+                        SOC15_REG_OFFSET(NBIO, 0,
+                                         mmBIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW1));
+                size = strnlen((((char *)adev->virt.fw_reserve.p_pf2vf) +
+                                val), PAGE_SIZE);
+
+                if (size < PAGE_SIZE)
+                        strcpy(buf,((char *)adev->virt.fw_reserve.p_pf2vf + val));
+                else
+                        size = 0;
+
+                r = size;
+                goto out;
+        }
+
+        r = xgpu_ai_poll_msg(adev, IDH_FAIL);
+        if(r)
+                pr_info("%s DPM request failed",
+                        (type == PP_SCLK)? "SCLK" : "MCLK");
+
+out:
+        mutex_unlock(&adev->virt.dpm_mutex);
+        return r;
+}
+
+static int xgpu_ai_force_dpm_level(struct amdgpu_device *adev, u32 level)
+{
+        int r = 0;
+        u32 req = IDH_IRQ_FORCE_DPM_LEVEL;
+
+        if (!amdgim_is_hwperf(adev))
+                return -EBADRQC;
+
+        mutex_lock(&adev->virt.dpm_mutex);
+        xgpu_ai_mailbox_trans_msg(adev, req, level, 0, 0);
+
+        r = xgpu_ai_poll_msg(adev, IDH_SUCCESS);
+        if (!r)
+                goto out;
+
+        r = xgpu_ai_poll_msg(adev, IDH_FAIL);
+        if (!r)
+                pr_info("DPM request failed");
+        else
+                pr_info("Mailbox is broken");
+
+out:
+        mutex_unlock(&adev->virt.dpm_mutex);
+        return r;
+}
+
 static int xgpu_ai_send_access_requests(struct amdgpu_device *adev,
 					enum idh_request req)
 {
@@ -375,4 +451,6 @@ const struct amdgpu_virt_ops xgpu_ai_virt_ops = {
 	.reset_gpu = xgpu_ai_request_reset,
 	.wait_reset = NULL,
 	.trans_msg = xgpu_ai_mailbox_trans_msg,
+	.get_pp_clk = xgpu_ai_get_pp_clk,
+	.force_dpm_level = xgpu_ai_force_dpm_level,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.h b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.h
index b4a9ceea334b..39d151b79153 100644
--- a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.h
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.h
@@ -35,6 +35,10 @@ enum idh_request {
 	IDH_REL_GPU_FINI_ACCESS,
 	IDH_REQ_GPU_RESET_ACCESS,
 
+	IDH_IRQ_FORCE_DPM_LEVEL = 10,
+	IDH_IRQ_GET_PP_SCLK,
+	IDH_IRQ_GET_PP_MCLK,
+
 	IDH_LOG_VF_ERROR       = 200,
 };
 
@@ -43,6 +47,8 @@ enum idh_event {
 	IDH_READY_TO_ACCESS_GPU,
 	IDH_FLR_NOTIFICATION,
 	IDH_FLR_NOTIFICATION_CMPL,
+	IDH_SUCCESS,
+	IDH_FAIL,
 	IDH_EVENT_MAX
 };
 
diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
index 6a0fcd67662a..aef9d059ae52 100644
--- a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
@@ -515,7 +515,7 @@ static void xgpu_vi_mailbox_flr_work(struct work_struct *work)
 
 	/* wait until RCV_MSG become 3 */
 	if (xgpu_vi_poll_msg(adev, IDH_FLR_NOTIFICATION_CMPL)) {
-		pr_err("failed to recieve FLR_CMPL\n");
+		pr_err("failed to receive FLR_CMPL\n");
 		return;
 	}
 
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c b/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c
index cc967dbfd631..6590143c3f75 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c
@@ -118,7 +118,8 @@ static void nbio_v6_1_ih_doorbell_range(struct amdgpu_device *adev,
 
 	if (use_doorbell) {
 		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range, BIF_IH_DOORBELL_RANGE, OFFSET, doorbell_index);
-		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range, BIF_IH_DOORBELL_RANGE, SIZE, 2);
+		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
+						  BIF_IH_DOORBELL_RANGE, SIZE, 6);
 	} else
 		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range, BIF_IH_DOORBELL_RANGE, SIZE, 0);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h b/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h
index f3a7d207af07..2f79765b4bdb 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h
+++ b/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h
@@ -43,6 +43,7 @@ enum psp_gfx_crtl_cmd_id
     GFX_CTRL_CMD_ID_ENABLE_INT      = 0x00050000,   /* enable PSP-to-Gfx interrupt */
     GFX_CTRL_CMD_ID_DISABLE_INT     = 0x00060000,   /* disable PSP-to-Gfx interrupt */
     GFX_CTRL_CMD_ID_MODE1_RST       = 0x00070000,   /* trigger the Mode 1 reset */
+    GFX_CTRL_CMD_ID_GBR_IH_SET      = 0x00080000,   /* set Gbr IH_RB_CNTL registers */
     GFX_CTRL_CMD_ID_CONSUME_CMD     = 0x000A0000,   /* send interrupt to psp for updating write pointer of vf */
     GFX_CTRL_CMD_ID_DESTROY_GPCOM_RING = 0x000C0000, /* destroy GPCOM ring */
 
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c
index 860b70d80d3c..b91df7bd1d98 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c
@@ -33,6 +33,9 @@
 #include "sdma0/sdma0_4_0_offset.h"
 #include "nbio/nbio_7_4_offset.h"
 
+#include "oss/osssys_4_0_offset.h"
+#include "oss/osssys_4_0_sh_mask.h"
+
 MODULE_FIRMWARE("amdgpu/vega20_sos.bin");
 MODULE_FIRMWARE("amdgpu/vega20_asd.bin");
 MODULE_FIRMWARE("amdgpu/vega20_ta.bin");
@@ -113,6 +116,13 @@ static int psp_v11_0_init_microcode(struct psp_context *psp)
 		adev->psp.ta_xgmi_ucode_size = le32_to_cpu(ta_hdr->ta_xgmi_size_bytes);
 		adev->psp.ta_xgmi_start_addr = (uint8_t *)ta_hdr +
 			le32_to_cpu(ta_hdr->header.ucode_array_offset_bytes);
+
+		adev->psp.ta_fw_version = le32_to_cpu(ta_hdr->header.ucode_version);
+
+		adev->psp.ta_ras_ucode_version = le32_to_cpu(ta_hdr->ta_ras_ucode_version);
+		adev->psp.ta_ras_ucode_size = le32_to_cpu(ta_hdr->ta_ras_size_bytes);
+		adev->psp.ta_ras_start_addr = (uint8_t *)adev->psp.ta_xgmi_start_addr +
+			le32_to_cpu(ta_hdr->ta_ras_offset_bytes);
 	}
 
 	return 0;
@@ -217,6 +227,37 @@ static int psp_v11_0_bootloader_load_sos(struct psp_context *psp)
 	return ret;
 }
 
+static void psp_v11_0_reroute_ih(struct psp_context *psp)
+{
+	struct amdgpu_device *adev = psp->adev;
+	uint32_t tmp;
+
+	/* Change IH ring for VMC */
+	tmp = REG_SET_FIELD(0, IH_CLIENT_CFG_DATA, CREDIT_RETURN_ADDR, 0x1244b);
+	tmp = REG_SET_FIELD(tmp, IH_CLIENT_CFG_DATA, CLIENT_TYPE, 1);
+	tmp = REG_SET_FIELD(tmp, IH_CLIENT_CFG_DATA, RING_ID, 1);
+
+	WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_69, 3);
+	WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_70, tmp);
+	WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_64, GFX_CTRL_CMD_ID_GBR_IH_SET);
+
+	mdelay(20);
+	psp_wait_for(psp, SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_64),
+		     0x80000000, 0x8000FFFF, false);
+
+	/* Change IH ring for UMC */
+	tmp = REG_SET_FIELD(0, IH_CLIENT_CFG_DATA, CREDIT_RETURN_ADDR, 0x1216b);
+	tmp = REG_SET_FIELD(tmp, IH_CLIENT_CFG_DATA, RING_ID, 1);
+
+	WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_69, 4);
+	WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_70, tmp);
+	WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_64, GFX_CTRL_CMD_ID_GBR_IH_SET);
+
+	mdelay(20);
+	psp_wait_for(psp, SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_64),
+		     0x80000000, 0x8000FFFF, false);
+}
+
 static int psp_v11_0_ring_init(struct psp_context *psp,
 			      enum psp_ring_type ring_type)
 {
@@ -224,6 +265,8 @@ static int psp_v11_0_ring_init(struct psp_context *psp,
 	struct psp_ring *ring;
 	struct amdgpu_device *adev = psp->adev;
 
+	psp_v11_0_reroute_ih(psp);
+
 	ring = &psp->km_ring;
 
 	ring->ring_type = ring_type;
@@ -631,7 +674,7 @@ static int psp_v11_0_xgmi_set_topology_info(struct psp_context *psp,
 	for (i = 0; i < topology_info_input->num_nodes; i++) {
 		topology_info_input->nodes[i].node_id = topology->nodes[i].node_id;
 		topology_info_input->nodes[i].num_hops = topology->nodes[i].num_hops;
-		topology_info_input->nodes[i].is_sharing_enabled = topology->nodes[i].is_sharing_enabled;
+		topology_info_input->nodes[i].is_sharing_enabled = 1;
 		topology_info_input->nodes[i].sdma_engine = topology->nodes[i].sdma_engine;
 	}
 
@@ -679,6 +722,54 @@ static int psp_v11_0_xgmi_get_node_id(struct psp_context *psp, uint64_t *node_id
 	return 0;
 }
 
+static int psp_v11_0_ras_trigger_error(struct psp_context *psp,
+		struct ta_ras_trigger_error_input *info)
+{
+	struct ta_ras_shared_memory *ras_cmd;
+	int ret;
+
+	if (!psp->ras.ras_initialized)
+		return -EINVAL;
+
+	ras_cmd = (struct ta_ras_shared_memory *)psp->ras.ras_shared_buf;
+	memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
+
+	ras_cmd->cmd_id = TA_RAS_COMMAND__TRIGGER_ERROR;
+	ras_cmd->ras_in_message.trigger_error = *info;
+
+	ret = psp_ras_invoke(psp, ras_cmd->cmd_id);
+	if (ret)
+		return -EINVAL;
+
+	return ras_cmd->ras_status;
+}
+
+static int psp_v11_0_ras_cure_posion(struct psp_context *psp, uint64_t *mode_ptr)
+{
+#if 0
+	// not support yet.
+	struct ta_ras_shared_memory *ras_cmd;
+	int ret;
+
+	if (!psp->ras.ras_initialized)
+		return -EINVAL;
+
+	ras_cmd = (struct ta_ras_shared_memory *)psp->ras.ras_shared_buf;
+	memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
+
+	ras_cmd->cmd_id = TA_RAS_COMMAND__CURE_POISON;
+	ras_cmd->ras_in_message.cure_poison.mode_ptr = mode_ptr;
+
+	ret = psp_ras_invoke(psp, ras_cmd->cmd_id);
+	if (ret)
+		return -EINVAL;
+
+	return ras_cmd->ras_status;
+#else
+	return -EINVAL;
+#endif
+}
+
 static const struct psp_funcs psp_v11_0_funcs = {
 	.init_microcode = psp_v11_0_init_microcode,
 	.bootloader_load_sysdrv = psp_v11_0_bootloader_load_sysdrv,
@@ -695,6 +786,8 @@ static const struct psp_funcs psp_v11_0_funcs = {
 	.xgmi_get_hive_id = psp_v11_0_xgmi_get_hive_id,
 	.xgmi_get_node_id = psp_v11_0_xgmi_get_node_id,
 	.support_vmr_ring = psp_v11_0_support_vmr_ring,
+	.ras_trigger_error = psp_v11_0_ras_trigger_error,
+	.ras_cure_posion = psp_v11_0_ras_cure_posion,
 };
 
 void psp_v11_0_set_psp_funcs(struct psp_context *psp)
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c
index 0487e3a4e9e7..143f0fae69d5 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c
@@ -37,6 +37,9 @@
 #include "sdma0/sdma0_4_0_offset.h"
 #include "nbio/nbio_6_1_offset.h"
 
+#include "oss/osssys_4_0_offset.h"
+#include "oss/osssys_4_0_sh_mask.h"
+
 MODULE_FIRMWARE("amdgpu/vega10_sos.bin");
 MODULE_FIRMWARE("amdgpu/vega10_asd.bin");
 MODULE_FIRMWARE("amdgpu/vega12_sos.bin");
@@ -252,6 +255,37 @@ static int psp_v3_1_ring_init(struct psp_context *psp,
 	return 0;
 }
 
+static void psp_v3_1_reroute_ih(struct psp_context *psp)
+{
+	struct amdgpu_device *adev = psp->adev;
+	uint32_t tmp;
+
+	/* Change IH ring for VMC */
+	tmp = REG_SET_FIELD(0, IH_CLIENT_CFG_DATA, CREDIT_RETURN_ADDR, 0x1244b);
+	tmp = REG_SET_FIELD(tmp, IH_CLIENT_CFG_DATA, CLIENT_TYPE, 1);
+	tmp = REG_SET_FIELD(tmp, IH_CLIENT_CFG_DATA, RING_ID, 1);
+
+	WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_69, 3);
+	WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_70, tmp);
+	WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_64, GFX_CTRL_CMD_ID_GBR_IH_SET);
+
+	mdelay(20);
+	psp_wait_for(psp, SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_64),
+		     0x80000000, 0x8000FFFF, false);
+
+	/* Change IH ring for UMC */
+	tmp = REG_SET_FIELD(0, IH_CLIENT_CFG_DATA, CREDIT_RETURN_ADDR, 0x1216b);
+	tmp = REG_SET_FIELD(tmp, IH_CLIENT_CFG_DATA, RING_ID, 1);
+
+	WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_69, 4);
+	WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_70, tmp);
+	WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_64, GFX_CTRL_CMD_ID_GBR_IH_SET);
+
+	mdelay(20);
+	psp_wait_for(psp, SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_64),
+		     0x80000000, 0x8000FFFF, false);
+}
+
 static int psp_v3_1_ring_create(struct psp_context *psp,
 				enum psp_ring_type ring_type)
 {
@@ -260,6 +294,8 @@ static int psp_v3_1_ring_create(struct psp_context *psp,
 	struct psp_ring *ring = &psp->km_ring;
 	struct amdgpu_device *adev = psp->adev;
 
+	psp_v3_1_reroute_ih(psp);
+
 	/* Write low address of the ring to C2PMSG_69 */
 	psp_ring_reg = lower_32_bits(ring->ring_mem_mc_addr);
 	WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_69, psp_ring_reg);
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
index cca3552b36ed..36196372e8db 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
@@ -870,8 +870,8 @@ static int sdma_v2_4_sw_init(void *handle)
 		r = amdgpu_ring_init(adev, ring, 1024,
 				     &adev->sdma.trap_irq,
 				     (i == 0) ?
-				     AMDGPU_SDMA_IRQ_TRAP0 :
-				     AMDGPU_SDMA_IRQ_TRAP1);
+				     AMDGPU_SDMA_IRQ_INSTANCE0 :
+				     AMDGPU_SDMA_IRQ_INSTANCE1);
 		if (r)
 			return r;
 	}
@@ -1006,7 +1006,7 @@ static int sdma_v2_4_set_trap_irq_state(struct amdgpu_device *adev,
 	u32 sdma_cntl;
 
 	switch (type) {
-	case AMDGPU_SDMA_IRQ_TRAP0:
+	case AMDGPU_SDMA_IRQ_INSTANCE0:
 		switch (state) {
 		case AMDGPU_IRQ_STATE_DISABLE:
 			sdma_cntl = RREG32(mmSDMA0_CNTL + SDMA0_REGISTER_OFFSET);
@@ -1022,7 +1022,7 @@ static int sdma_v2_4_set_trap_irq_state(struct amdgpu_device *adev,
 			break;
 		}
 		break;
-	case AMDGPU_SDMA_IRQ_TRAP1:
+	case AMDGPU_SDMA_IRQ_INSTANCE1:
 		switch (state) {
 		case AMDGPU_IRQ_STATE_DISABLE:
 			sdma_cntl = RREG32(mmSDMA0_CNTL + SDMA1_REGISTER_OFFSET);
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
index 0ce8331baeb2..6d39544e7829 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
@@ -1154,8 +1154,8 @@ static int sdma_v3_0_sw_init(void *handle)
 		r = amdgpu_ring_init(adev, ring, 1024,
 				     &adev->sdma.trap_irq,
 				     (i == 0) ?
-				     AMDGPU_SDMA_IRQ_TRAP0 :
-				     AMDGPU_SDMA_IRQ_TRAP1);
+				     AMDGPU_SDMA_IRQ_INSTANCE0 :
+				     AMDGPU_SDMA_IRQ_INSTANCE1);
 		if (r)
 			return r;
 	}
@@ -1340,7 +1340,7 @@ static int sdma_v3_0_set_trap_irq_state(struct amdgpu_device *adev,
 	u32 sdma_cntl;
 
 	switch (type) {
-	case AMDGPU_SDMA_IRQ_TRAP0:
+	case AMDGPU_SDMA_IRQ_INSTANCE0:
 		switch (state) {
 		case AMDGPU_IRQ_STATE_DISABLE:
 			sdma_cntl = RREG32(mmSDMA0_CNTL + SDMA0_REGISTER_OFFSET);
@@ -1356,7 +1356,7 @@ static int sdma_v3_0_set_trap_irq_state(struct amdgpu_device *adev,
 			break;
 		}
 		break;
-	case AMDGPU_SDMA_IRQ_TRAP1:
+	case AMDGPU_SDMA_IRQ_INSTANCE1:
 		switch (state) {
 		case AMDGPU_IRQ_STATE_DISABLE:
 			sdma_cntl = RREG32(mmSDMA0_CNTL + SDMA1_REGISTER_OFFSET);
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
index c816e55d43a9..9c88ce513d78 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
@@ -41,6 +41,8 @@
 #include "ivsrcid/sdma0/irqsrcs_sdma0_4_0.h"
 #include "ivsrcid/sdma1/irqsrcs_sdma1_4_0.h"
 
+#include "amdgpu_ras.h"
+
 MODULE_FIRMWARE("amdgpu/vega10_sdma.bin");
 MODULE_FIRMWARE("amdgpu/vega10_sdma1.bin");
 MODULE_FIRMWARE("amdgpu/vega12_sdma.bin");
@@ -154,7 +156,6 @@ static const struct soc15_reg_golden golden_settings_sdma0_4_2[] =
 	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC7_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),
 	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_RLC7_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
 	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_PAGE, 0x000003ff, 0x000003c0),
-	SOC15_REG_GOLDEN_VALUE(SDMA0, 0, mmSDMA0_UTCL1_WATERMK, 0xFE000000, 0x00000000),
 };
 
 static const struct soc15_reg_golden golden_settings_sdma1_4_2[] = {
@@ -184,7 +185,6 @@ static const struct soc15_reg_golden golden_settings_sdma1_4_2[] = {
 	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC7_RB_RPTR_ADDR_LO, 0xfffffffd, 0x00000001),
 	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_RLC7_RB_WPTR_POLL_CNTL, 0xfffffff7, 0x00403000),
 	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_UTCL1_PAGE, 0x000003ff, 0x000003c0),
-	SOC15_REG_GOLDEN_VALUE(SDMA1, 0, mmSDMA1_UTCL1_WATERMK, 0xFE000000, 0x00000000),
 };
 
 static const struct soc15_reg_golden golden_settings_sdma_rv1[] =
@@ -849,7 +849,7 @@ static void sdma_v4_0_gfx_resume(struct amdgpu_device *adev, unsigned int i)
 	wptr_poll_cntl = RREG32_SDMA(i, mmSDMA0_GFX_RB_WPTR_POLL_CNTL);
 	wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl,
 				       SDMA0_GFX_RB_WPTR_POLL_CNTL,
-				       F32_POLL_ENABLE, amdgpu_sriov_vf(adev));
+				       F32_POLL_ENABLE, amdgpu_sriov_vf(adev)? 1 : 0);
 	WREG32_SDMA(i, mmSDMA0_GFX_RB_WPTR_POLL_CNTL, wptr_poll_cntl);
 
 	/* enable DMA RB */
@@ -940,7 +940,7 @@ static void sdma_v4_0_page_resume(struct amdgpu_device *adev, unsigned int i)
 	wptr_poll_cntl = RREG32_SDMA(i, mmSDMA0_PAGE_RB_WPTR_POLL_CNTL);
 	wptr_poll_cntl = REG_SET_FIELD(wptr_poll_cntl,
 				       SDMA0_PAGE_RB_WPTR_POLL_CNTL,
-				       F32_POLL_ENABLE, amdgpu_sriov_vf(adev));
+				       F32_POLL_ENABLE, amdgpu_sriov_vf(adev)? 1 : 0);
 	WREG32_SDMA(i, mmSDMA0_PAGE_RB_WPTR_POLL_CNTL, wptr_poll_cntl);
 
 	/* enable DMA RB */
@@ -1493,6 +1493,87 @@ static int sdma_v4_0_early_init(void *handle)
 	return 0;
 }
 
+static int sdma_v4_0_process_ras_data_cb(struct amdgpu_device *adev,
+		struct amdgpu_iv_entry *entry);
+
+static int sdma_v4_0_late_init(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	struct ras_common_if **ras_if = &adev->sdma.ras_if;
+	struct ras_ih_if ih_info = {
+		.cb = sdma_v4_0_process_ras_data_cb,
+	};
+	struct ras_fs_if fs_info = {
+		.sysfs_name = "sdma_err_count",
+		.debugfs_name = "sdma_err_inject",
+	};
+	struct ras_common_if ras_block = {
+		.block = AMDGPU_RAS_BLOCK__SDMA,
+		.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE,
+		.sub_block_index = 0,
+		.name = "sdma",
+	};
+	int r;
+
+	if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA)) {
+		amdgpu_ras_feature_enable_on_boot(adev, &ras_block, 0);
+		return 0;
+	}
+
+	/* handle resume path. */
+	if (*ras_if)
+		goto resume;
+
+	*ras_if = kmalloc(sizeof(**ras_if), GFP_KERNEL);
+	if (!*ras_if)
+		return -ENOMEM;
+
+	**ras_if = ras_block;
+
+	r = amdgpu_ras_feature_enable_on_boot(adev, *ras_if, 1);
+	if (r)
+		goto feature;
+
+	ih_info.head = **ras_if;
+	fs_info.head = **ras_if;
+
+	r = amdgpu_ras_interrupt_add_handler(adev, &ih_info);
+	if (r)
+		goto interrupt;
+
+	r = amdgpu_ras_debugfs_create(adev, &fs_info);
+	if (r)
+		goto debugfs;
+
+	r = amdgpu_ras_sysfs_create(adev, &fs_info);
+	if (r)
+		goto sysfs;
+resume:
+	r = amdgpu_irq_get(adev, &adev->sdma.ecc_irq, AMDGPU_SDMA_IRQ_INSTANCE0);
+	if (r)
+		goto irq;
+
+	r = amdgpu_irq_get(adev, &adev->sdma.ecc_irq, AMDGPU_SDMA_IRQ_INSTANCE1);
+	if (r) {
+		amdgpu_irq_put(adev, &adev->sdma.ecc_irq, AMDGPU_SDMA_IRQ_INSTANCE0);
+		goto irq;
+	}
+
+	return 0;
+irq:
+	amdgpu_ras_sysfs_remove(adev, *ras_if);
+sysfs:
+	amdgpu_ras_debugfs_remove(adev, *ras_if);
+debugfs:
+	amdgpu_ras_interrupt_remove_handler(adev, &ih_info);
+interrupt:
+	amdgpu_ras_feature_enable(adev, *ras_if, 0);
+feature:
+	kfree(*ras_if);
+	*ras_if = NULL;
+	return -EINVAL;
+}
+
 static int sdma_v4_0_sw_init(void *handle)
 {
 	struct amdgpu_ring *ring;
@@ -1511,6 +1592,18 @@ static int sdma_v4_0_sw_init(void *handle)
 	if (r)
 		return r;
 
+	/* SDMA SRAM ECC event */
+	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_SDMA0, SDMA0_4_0__SRCID__SDMA_SRAM_ECC,
+			&adev->sdma.ecc_irq);
+	if (r)
+		return r;
+
+	/* SDMA SRAM ECC event */
+	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_SDMA1, SDMA1_4_0__SRCID__SDMA_SRAM_ECC,
+			&adev->sdma.ecc_irq);
+	if (r)
+		return r;
+
 	for (i = 0; i < adev->sdma.num_instances; i++) {
 		ring = &adev->sdma.instance[i].ring;
 		ring->ring_obj = NULL;
@@ -1526,8 +1619,8 @@ static int sdma_v4_0_sw_init(void *handle)
 		r = amdgpu_ring_init(adev, ring, 1024,
 				     &adev->sdma.trap_irq,
 				     (i == 0) ?
-				     AMDGPU_SDMA_IRQ_TRAP0 :
-				     AMDGPU_SDMA_IRQ_TRAP1);
+				     AMDGPU_SDMA_IRQ_INSTANCE0 :
+				     AMDGPU_SDMA_IRQ_INSTANCE1);
 		if (r)
 			return r;
 
@@ -1546,8 +1639,8 @@ static int sdma_v4_0_sw_init(void *handle)
 			r = amdgpu_ring_init(adev, ring, 1024,
 					     &adev->sdma.trap_irq,
 					     (i == 0) ?
-					     AMDGPU_SDMA_IRQ_TRAP0 :
-					     AMDGPU_SDMA_IRQ_TRAP1);
+					     AMDGPU_SDMA_IRQ_INSTANCE0 :
+					     AMDGPU_SDMA_IRQ_INSTANCE1);
 			if (r)
 				return r;
 		}
@@ -1561,6 +1654,22 @@ static int sdma_v4_0_sw_fini(void *handle)
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 	int i;
 
+	if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA) &&
+			adev->sdma.ras_if) {
+		struct ras_common_if *ras_if = adev->sdma.ras_if;
+		struct ras_ih_if ih_info = {
+			.head = *ras_if,
+		};
+
+		/*remove fs first*/
+		amdgpu_ras_debugfs_remove(adev, ras_if);
+		amdgpu_ras_sysfs_remove(adev, ras_if);
+		/*remove the IH*/
+		amdgpu_ras_interrupt_remove_handler(adev, &ih_info);
+		amdgpu_ras_feature_enable(adev, ras_if, 0);
+		kfree(ras_if);
+	}
+
 	for (i = 0; i < adev->sdma.num_instances; i++) {
 		amdgpu_ring_fini(&adev->sdma.instance[i].ring);
 		if (adev->sdma.has_page_queue)
@@ -1598,6 +1707,9 @@ static int sdma_v4_0_hw_fini(void *handle)
 	if (amdgpu_sriov_vf(adev))
 		return 0;
 
+	amdgpu_irq_put(adev, &adev->sdma.ecc_irq, AMDGPU_SDMA_IRQ_INSTANCE0);
+	amdgpu_irq_put(adev, &adev->sdma.ecc_irq, AMDGPU_SDMA_IRQ_INSTANCE1);
+
 	sdma_v4_0_ctx_switch_enable(adev, false);
 	sdma_v4_0_enable(adev, false);
 
@@ -1666,13 +1778,12 @@ static int sdma_v4_0_set_trap_irq_state(struct amdgpu_device *adev,
 					unsigned type,
 					enum amdgpu_interrupt_state state)
 {
-	unsigned int instance = (type == AMDGPU_SDMA_IRQ_TRAP0) ? 0 : 1;
 	u32 sdma_cntl;
 
-	sdma_cntl = RREG32_SDMA(instance, mmSDMA0_CNTL);
+	sdma_cntl = RREG32_SDMA(type, mmSDMA0_CNTL);
 	sdma_cntl = REG_SET_FIELD(sdma_cntl, SDMA0_CNTL, TRAP_ENABLE,
 		       state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
-	WREG32_SDMA(instance, mmSDMA0_CNTL, sdma_cntl);
+	WREG32_SDMA(type, mmSDMA0_CNTL, sdma_cntl);
 
 	return 0;
 }
@@ -1714,6 +1825,58 @@ static int sdma_v4_0_process_trap_irq(struct amdgpu_device *adev,
 	return 0;
 }
 
+static int sdma_v4_0_process_ras_data_cb(struct amdgpu_device *adev,
+		struct amdgpu_iv_entry *entry)
+{
+	uint32_t instance, err_source;
+
+	switch (entry->client_id) {
+	case SOC15_IH_CLIENTID_SDMA0:
+		instance = 0;
+		break;
+	case SOC15_IH_CLIENTID_SDMA1:
+		instance = 1;
+		break;
+	default:
+		return 0;
+	}
+
+	switch (entry->src_id) {
+	case SDMA0_4_0__SRCID__SDMA_SRAM_ECC:
+		err_source = 0;
+		break;
+	case SDMA0_4_0__SRCID__SDMA_ECC:
+		err_source = 1;
+		break;
+	default:
+		return 0;
+	}
+
+	kgd2kfd_set_sram_ecc_flag(adev->kfd.dev);
+
+	amdgpu_ras_reset_gpu(adev, 0);
+
+	return AMDGPU_RAS_UE;
+}
+
+static int sdma_v4_0_process_ecc_irq(struct amdgpu_device *adev,
+				      struct amdgpu_irq_src *source,
+				      struct amdgpu_iv_entry *entry)
+{
+	struct ras_common_if *ras_if = adev->sdma.ras_if;
+	struct ras_dispatch_if ih_data = {
+		.entry = entry,
+	};
+
+	if (!ras_if)
+		return 0;
+
+	ih_data.head = *ras_if;
+
+	amdgpu_ras_interrupt_dispatch(adev, &ih_data);
+	return 0;
+}
+
 static int sdma_v4_0_process_illegal_inst_irq(struct amdgpu_device *adev,
 					      struct amdgpu_irq_src *source,
 					      struct amdgpu_iv_entry *entry)
@@ -1741,6 +1904,25 @@ static int sdma_v4_0_process_illegal_inst_irq(struct amdgpu_device *adev,
 	return 0;
 }
 
+static int sdma_v4_0_set_ecc_irq_state(struct amdgpu_device *adev,
+					struct amdgpu_irq_src *source,
+					unsigned type,
+					enum amdgpu_interrupt_state state)
+{
+	u32 sdma_edc_config;
+
+	u32 reg_offset = (type == AMDGPU_SDMA_IRQ_INSTANCE0) ?
+		sdma_v4_0_get_reg_offset(adev, 0, mmSDMA0_EDC_CONFIG) :
+		sdma_v4_0_get_reg_offset(adev, 1, mmSDMA0_EDC_CONFIG);
+
+	sdma_edc_config = RREG32(reg_offset);
+	sdma_edc_config = REG_SET_FIELD(sdma_edc_config, SDMA0_EDC_CONFIG, ECC_INT_ENABLE,
+		       state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
+	WREG32(reg_offset, sdma_edc_config);
+
+	return 0;
+}
+
 static void sdma_v4_0_update_medium_grain_clock_gating(
 		struct amdgpu_device *adev,
 		bool enable)
@@ -1906,7 +2088,7 @@ static void sdma_v4_0_get_clockgating_state(void *handle, u32 *flags)
 const struct amd_ip_funcs sdma_v4_0_ip_funcs = {
 	.name = "sdma_v4_0",
 	.early_init = sdma_v4_0_early_init,
-	.late_init = NULL,
+	.late_init = sdma_v4_0_late_init,
 	.sw_init = sdma_v4_0_sw_init,
 	.sw_fini = sdma_v4_0_sw_fini,
 	.hw_init = sdma_v4_0_hw_init,
@@ -2008,11 +2190,20 @@ static const struct amdgpu_irq_src_funcs sdma_v4_0_illegal_inst_irq_funcs = {
 	.process = sdma_v4_0_process_illegal_inst_irq,
 };
 
+static const struct amdgpu_irq_src_funcs sdma_v4_0_ecc_irq_funcs = {
+	.set = sdma_v4_0_set_ecc_irq_state,
+	.process = sdma_v4_0_process_ecc_irq,
+};
+
+
+
 static void sdma_v4_0_set_irq_funcs(struct amdgpu_device *adev)
 {
 	adev->sdma.trap_irq.num_types = AMDGPU_SDMA_IRQ_LAST;
 	adev->sdma.trap_irq.funcs = &sdma_v4_0_trap_irq_funcs;
 	adev->sdma.illegal_inst_irq.funcs = &sdma_v4_0_illegal_inst_irq_funcs;
+	adev->sdma.ecc_irq.num_types = AMDGPU_SDMA_IRQ_LAST;
+	adev->sdma.ecc_irq.funcs = &sdma_v4_0_ecc_irq_funcs;
 }
 
 /**
@@ -2077,8 +2268,8 @@ static const struct amdgpu_buffer_funcs sdma_v4_0_buffer_funcs = {
 static void sdma_v4_0_set_buffer_funcs(struct amdgpu_device *adev)
 {
 	adev->mman.buffer_funcs = &sdma_v4_0_buffer_funcs;
-	if (adev->sdma.has_page_queue)
-		adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].page;
+	if (adev->sdma.has_page_queue && adev->sdma.num_instances > 1)
+		adev->mman.buffer_funcs_ring = &adev->sdma.instance[1].page;
 	else
 		adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
 }
@@ -2097,15 +2288,22 @@ static void sdma_v4_0_set_vm_pte_funcs(struct amdgpu_device *adev)
 	unsigned i;
 
 	adev->vm_manager.vm_pte_funcs = &sdma_v4_0_vm_pte_funcs;
-	for (i = 0; i < adev->sdma.num_instances; i++) {
-		if (adev->sdma.has_page_queue)
+	if (adev->sdma.has_page_queue && adev->sdma.num_instances > 1) {
+		for (i = 1; i < adev->sdma.num_instances; i++) {
 			sched = &adev->sdma.instance[i].page.sched;
-		else
+			adev->vm_manager.vm_pte_rqs[i - 1] =
+				&sched->sched_rq[DRM_SCHED_PRIORITY_KERNEL];
+		}
+		adev->vm_manager.vm_pte_num_rqs = adev->sdma.num_instances - 1;
+		adev->vm_manager.page_fault = &adev->sdma.instance[0].page;
+	} else {
+		for (i = 0; i < adev->sdma.num_instances; i++) {
 			sched = &adev->sdma.instance[i].ring.sched;
-		adev->vm_manager.vm_pte_rqs[i] =
-			&sched->sched_rq[DRM_SCHED_PRIORITY_KERNEL];
+			adev->vm_manager.vm_pte_rqs[i] =
+				&sched->sched_rq[DRM_SCHED_PRIORITY_KERNEL];
+		}
+		adev->vm_manager.vm_pte_num_rqs = adev->sdma.num_instances;
 	}
-	adev->vm_manager.vm_pte_num_rqs = adev->sdma.num_instances;
 }
 
 const struct amdgpu_ip_block_version sdma_v4_0_ip_block = {
diff --git a/drivers/gpu/drm/amd/amdgpu/si_dma.c b/drivers/gpu/drm/amd/amdgpu/si_dma.c
index f15f196684ba..3eeefd40dae0 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_dma.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_dma.c
@@ -503,8 +503,8 @@ static int si_dma_sw_init(void *handle)
 		r = amdgpu_ring_init(adev, ring, 1024,
 				     &adev->sdma.trap_irq,
 				     (i == 0) ?
-				     AMDGPU_SDMA_IRQ_TRAP0 :
-				     AMDGPU_SDMA_IRQ_TRAP1);
+				     AMDGPU_SDMA_IRQ_INSTANCE0 :
+				     AMDGPU_SDMA_IRQ_INSTANCE1);
 		if (r)
 			return r;
 	}
@@ -591,7 +591,7 @@ static int si_dma_set_trap_irq_state(struct amdgpu_device *adev,
 	u32 sdma_cntl;
 
 	switch (type) {
-	case AMDGPU_SDMA_IRQ_TRAP0:
+	case AMDGPU_SDMA_IRQ_INSTANCE0:
 		switch (state) {
 		case AMDGPU_IRQ_STATE_DISABLE:
 			sdma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET);
@@ -607,7 +607,7 @@ static int si_dma_set_trap_irq_state(struct amdgpu_device *adev,
 			break;
 		}
 		break;
-	case AMDGPU_SDMA_IRQ_TRAP1:
+	case AMDGPU_SDMA_IRQ_INSTANCE1:
 		switch (state) {
 		case AMDGPU_IRQ_STATE_DISABLE:
 			sdma_cntl = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET);
diff --git a/drivers/gpu/drm/amd/amdgpu/si_dpm.c b/drivers/gpu/drm/amd/amdgpu/si_dpm.c
index 41e01a7f57a4..d57e75e5c71f 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_dpm.c
@@ -4098,14 +4098,13 @@ static int si_notify_smc_display_change(struct amdgpu_device *adev,
 
 static void si_program_response_times(struct amdgpu_device *adev)
 {
-	u32 voltage_response_time, backbias_response_time, acpi_delay_time, vbi_time_out;
+	u32 voltage_response_time, acpi_delay_time, vbi_time_out;
 	u32 vddc_dly, acpi_dly, vbi_dly;
 	u32 reference_clock;
 
 	si_write_smc_soft_register(adev, SI_SMC_SOFT_REGISTER_mvdd_chg_time, 1);
 
 	voltage_response_time = (u32)adev->pm.dpm.voltage_response_time;
-	backbias_response_time = (u32)adev->pm.dpm.backbias_response_time;
 
 	if (voltage_response_time == 0)
 		voltage_response_time = 1000;
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c
index ed89a101f73f..4900e4958dec 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
@@ -63,6 +63,7 @@
 #include "vcn_v1_0.h"
 #include "dce_virtual.h"
 #include "mxgpu_ai.h"
+#include "amdgpu_smu.h"
 
 #define mmMP0_MISC_CGTT_CTRL0                                                                   0x01b9
 #define mmMP0_MISC_CGTT_CTRL0_BASE_IDX                                                          0
@@ -392,6 +393,7 @@ void soc15_program_register_sequence(struct amdgpu_device *adev,
 static int soc15_asic_mode1_reset(struct amdgpu_device *adev)
 {
 	u32 i;
+	int ret = 0;
 
 	amdgpu_atombios_scratch_regs_engine_hung(adev, true);
 
@@ -402,7 +404,9 @@ static int soc15_asic_mode1_reset(struct amdgpu_device *adev)
 
 	pci_save_state(adev->pdev);
 
-	psp_gpu_reset(adev);
+	ret = psp_gpu_reset(adev);
+	if (ret)
+		dev_err(adev->dev, "GPU mode1 reset failed\n");
 
 	pci_restore_state(adev->pdev);
 
@@ -417,7 +421,7 @@ static int soc15_asic_mode1_reset(struct amdgpu_device *adev)
 
 	amdgpu_atombios_scratch_regs_engine_hung(adev, false);
 
-	return 0;
+	return ret;
 }
 
 static int soc15_asic_get_baco_capability(struct amdgpu_device *adev, bool *cap)
@@ -451,6 +455,8 @@ static int soc15_asic_baco_reset(struct amdgpu_device *adev)
 
 	dev_info(adev->dev, "GPU BACO reset\n");
 
+	adev->in_baco_reset = 1;
+
 	return 0;
 }
 
@@ -461,8 +467,15 @@ static int soc15_asic_reset(struct amdgpu_device *adev)
 
 	switch (adev->asic_type) {
 	case CHIP_VEGA10:
+	case CHIP_VEGA12:
 		soc15_asic_get_baco_capability(adev, &baco_reset);
 		break;
+	case CHIP_VEGA20:
+		if (adev->psp.sos_fw_version >= 0x80067)
+			soc15_asic_get_baco_capability(adev, &baco_reset);
+		else
+			baco_reset = false;
+		break;
 	default:
 		baco_reset = false;
 		break;
@@ -602,8 +615,12 @@ int soc15_set_ip_blocks(struct amdgpu_device *adev)
 		}
 		amdgpu_device_ip_block_add(adev, &gfx_v9_0_ip_block);
 		amdgpu_device_ip_block_add(adev, &sdma_v4_0_ip_block);
-		if (!amdgpu_sriov_vf(adev))
-			amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
+		if (!amdgpu_sriov_vf(adev)) {
+			if (is_support_sw_smu(adev))
+				amdgpu_device_ip_block_add(adev, &smu_v11_0_ip_block);
+			else
+				amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
+		}
 		if (adev->enable_virtual_display || amdgpu_sriov_vf(adev))
 			amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
 #if defined(CONFIG_DRM_AMD_DC)
@@ -884,7 +901,8 @@ static int soc15_common_early_init(void *handle)
 
 			adev->pg_flags = AMD_PG_SUPPORT_SDMA | AMD_PG_SUPPORT_VCN;
 		} else if (adev->pdev->device == 0x15d8) {
-			adev->cg_flags = AMD_CG_SUPPORT_GFX_MGLS |
+			adev->cg_flags = AMD_CG_SUPPORT_GFX_MGCG |
+				AMD_CG_SUPPORT_GFX_MGLS |
 				AMD_CG_SUPPORT_GFX_CP_LS |
 				AMD_CG_SUPPORT_GFX_3D_CGCG |
 				AMD_CG_SUPPORT_GFX_3D_CGLS |
@@ -927,7 +945,7 @@ static int soc15_common_early_init(void *handle)
 			adev->pg_flags = AMD_PG_SUPPORT_SDMA | AMD_PG_SUPPORT_VCN;
 		}
 
-		if (adev->powerplay.pp_feature & PP_GFXOFF_MASK)
+		if (adev->pm.pp_feature & PP_GFXOFF_MASK)
 			adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
 				AMD_PG_SUPPORT_CP |
 				AMD_PG_SUPPORT_RLC_SMU_HS;
diff --git a/drivers/gpu/drm/amd/amdgpu/ta_ras_if.h b/drivers/gpu/drm/amd/amdgpu/ta_ras_if.h
new file mode 100644
index 000000000000..0b4e7b55595a
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/ta_ras_if.h
@@ -0,0 +1,108 @@
+/****************************************************************************\
+* 
+*  File Name      ta_ras_if.h
+*  Project        AMD PSP SW IP Module
+*
+*  Description    Interface to the RAS Trusted Application
+*
+*  Copyright 2019 Advanced Micro Devices, Inc.
+*
+* Permission is hereby granted, free of charge, to any person obtaining a copy of this software 
+* and associated documentation files (the "Software"), to deal in the Software without restriction,
+* including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense,
+* and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so,
+* subject to the following conditions:
+*
+* The above copyright notice and this permission notice shall be included in all copies or substantial
+* portions of the Software.
+*
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+* OTHER DEALINGS IN THE SOFTWARE.
+*/
+#ifndef _TA_RAS_IF_H
+#define _TA_RAS_IF_H
+
+/* Responses have bit 31 set */
+#define RSP_ID_MASK (1U << 31)
+#define RSP_ID(cmdId) (((uint32_t)(cmdId)) | RSP_ID_MASK)
+
+#define TA_NUM_BLOCK_MAX		14
+
+enum ras_command {
+	TA_RAS_COMMAND__ENABLE_FEATURES = 0,
+	TA_RAS_COMMAND__DISABLE_FEATURES,
+	TA_RAS_COMMAND__TRIGGER_ERROR,
+};
+
+enum ta_ras_status {
+	TA_RAS_STATUS__SUCCESS				= 0x00,
+	TA_RAS_STATUS__RESET_NEEDED			= 0x01,
+	TA_RAS_STATUS__ERROR_INVALID_PARAMETER		= 0x02,
+	TA_RAS_STATUS__ERROR_RAS_NOT_AVAILABLE		= 0x03,
+	TA_RAS_STATUS__ERROR_RAS_DUPLICATE_CMD		= 0x04,
+	TA_RAS_STATUS__ERROR_INJECTION_FAILED		= 0x05
+};
+
+enum ta_ras_block {
+	TA_RAS_BLOCK__UMC = 0,
+	TA_RAS_BLOCK__SDMA,
+	TA_RAS_BLOCK__GFX,
+	TA_RAS_BLOCK__MMHUB,
+	TA_RAS_BLOCK__ATHUB,
+	TA_RAS_BLOCK__PCIE_BIF,
+	TA_RAS_BLOCK__HDP,
+	TA_RAS_BLOCK__XGMI_WAFL,
+	TA_RAS_BLOCK__DF,
+	TA_RAS_BLOCK__SMN,
+	TA_RAS_BLOCK__SEM,
+	TA_RAS_BLOCK__MP0,
+	TA_RAS_BLOCK__MP1,
+	TA_RAS_BLOCK__FUSE = (TA_NUM_BLOCK_MAX - 1),
+};
+
+enum ta_ras_error_type {
+	TA_RAS_ERROR__NONE				= 0,
+	TA_RAS_ERROR__PARITY				= 1,
+	TA_RAS_ERROR__SINGLE_CORRECTABLE		= 2,
+	TA_RAS_ERROR__MULTI_UNCORRECTABLE		= 4,
+	TA_RAS_ERROR__POISON				= 8
+};
+
+struct ta_ras_enable_features_input {
+	enum ta_ras_block       block_id;
+	enum ta_ras_error_type  error_type;
+};
+
+struct ta_ras_disable_features_input {
+	enum ta_ras_block       block_id;
+	enum ta_ras_error_type  error_type;
+};
+
+struct ta_ras_trigger_error_input {
+	enum ta_ras_block		block_id;
+	enum ta_ras_error_type		inject_error_type;
+	uint32_t			sub_block_index;
+	uint64_t			address;
+	uint64_t			value;
+};
+
+union ta_ras_cmd_input {
+	struct ta_ras_enable_features_input	enable_features;
+	struct ta_ras_disable_features_input	disable_features;
+	struct ta_ras_trigger_error_input	trigger_error;
+};
+
+struct ta_ras_shared_memory {
+	uint32_t			cmd_id;
+	uint32_t			resp_id;
+	enum ta_ras_status		ras_status;
+	uint32_t			reserved;
+	union ta_ras_cmd_input		ras_in_message;
+};
+
+#endif // TL_RAS_IF_H_
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
index bed78a778e3f..40363ca6c5f1 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
@@ -283,7 +283,7 @@ static int vce_v2_0_stop(struct amdgpu_device *adev)
 	}
 
 	if (vce_v2_0_wait_for_idle(adev)) {
-		DRM_INFO("VCE is busy, Can't set clock gateing");
+		DRM_INFO("VCE is busy, Can't set clock gating");
 		return 0;
 	}
 
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
index aadc3e66ebd7..f3f5938430d4 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
@@ -382,6 +382,7 @@ static int vce_v4_0_start(struct amdgpu_device *adev)
 static int vce_v4_0_stop(struct amdgpu_device *adev)
 {
 
+	/* Disable VCPU */
 	WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CNTL), 0, ~0x200001);
 
 	/* hold on ECPU */
@@ -389,8 +390,8 @@ static int vce_v4_0_stop(struct amdgpu_device *adev)
 			VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK,
 			~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);
 
-	/* clear BUSY flag */
-	WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_STATUS), 0, ~VCE_STATUS__JOB_BUSY_MASK);
+	/* clear VCE_STATUS */
+	WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_STATUS), 0);
 
 	/* Set Clock-Gating off */
 	/* if (adev->cg_flags & AMD_CG_SUPPORT_VCE_MGCG)
@@ -922,6 +923,7 @@ static int vce_v4_0_set_clockgating_state(void *handle,
 
 	return 0;
 }
+#endif
 
 static int vce_v4_0_set_powergating_state(void *handle,
 					  enum amd_powergating_state state)
@@ -935,16 +937,11 @@ static int vce_v4_0_set_powergating_state(void *handle,
 	 */
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-	if (!(adev->pg_flags & AMD_PG_SUPPORT_VCE))
-		return 0;
-
 	if (state == AMD_PG_STATE_GATE)
-		/* XXX do we need a vce_v4_0_stop()? */
-		return 0;
+		return vce_v4_0_stop(adev);
 	else
 		return vce_v4_0_start(adev);
 }
-#endif
 
 static void vce_v4_0_ring_emit_ib(struct amdgpu_ring *ring, struct amdgpu_job *job,
 					struct amdgpu_ib *ib, uint32_t flags)
@@ -1059,7 +1056,7 @@ const struct amd_ip_funcs vce_v4_0_ip_funcs = {
 	.soft_reset = NULL /* vce_v4_0_soft_reset */,
 	.post_soft_reset = NULL /* vce_v4_0_post_soft_reset */,
 	.set_clockgating_state = vce_v4_0_set_clockgating_state,
-	.set_powergating_state = NULL /* vce_v4_0_set_powergating_state */,
+	.set_powergating_state = vce_v4_0_set_powergating_state,
 };
 
 static const struct amdgpu_ring_funcs vce_v4_0_ring_vm_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/vega10_ih.c b/drivers/gpu/drm/amd/amdgpu/vega10_ih.c
index 6d1f804277f8..1b2f69a9a24e 100644
--- a/drivers/gpu/drm/amd/amdgpu/vega10_ih.c
+++ b/drivers/gpu/drm/amd/amdgpu/vega10_ih.c
@@ -136,6 +136,25 @@ static uint32_t vega10_ih_rb_cntl(struct amdgpu_ih_ring *ih, uint32_t ih_rb_cntl
 	return ih_rb_cntl;
 }
 
+static uint32_t vega10_ih_doorbell_rptr(struct amdgpu_ih_ring *ih)
+{
+	u32 ih_doorbell_rtpr = 0;
+
+	if (ih->use_doorbell) {
+		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr,
+						 IH_DOORBELL_RPTR, OFFSET,
+						 ih->doorbell_index);
+		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr,
+						 IH_DOORBELL_RPTR,
+						 ENABLE, 1);
+	} else {
+		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr,
+						 IH_DOORBELL_RPTR,
+						 ENABLE, 0);
+	}
+	return ih_doorbell_rtpr;
+}
+
 /**
  * vega10_ih_irq_init - init and enable the interrupt ring
  *
@@ -150,8 +169,8 @@ static uint32_t vega10_ih_rb_cntl(struct amdgpu_ih_ring *ih, uint32_t ih_rb_cntl
 static int vega10_ih_irq_init(struct amdgpu_device *adev)
 {
 	struct amdgpu_ih_ring *ih;
+	u32 ih_rb_cntl;
 	int ret = 0;
-	u32 ih_rb_cntl, ih_doorbell_rtpr;
 	u32 tmp;
 
 	/* disable irqs */
@@ -177,23 +196,11 @@ static int vega10_ih_irq_init(struct amdgpu_device *adev)
 		     upper_32_bits(ih->wptr_addr) & 0xFFFF);
 
 	/* set rptr, wptr to 0 */
-	WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0);
 	WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0);
+	WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0);
 
-	ih_doorbell_rtpr = RREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR);
-	if (adev->irq.ih.use_doorbell) {
-		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr,
-						 IH_DOORBELL_RPTR, OFFSET,
-						 adev->irq.ih.doorbell_index);
-		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr,
-						 IH_DOORBELL_RPTR,
-						 ENABLE, 1);
-	} else {
-		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr,
-						 IH_DOORBELL_RPTR,
-						 ENABLE, 0);
-	}
-	WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR, ih_doorbell_rtpr);
+	WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR,
+		     vega10_ih_doorbell_rptr(ih));
 
 	ih = &adev->irq.ih1;
 	if (ih->ring_size) {
@@ -203,11 +210,18 @@ static int vega10_ih_irq_init(struct amdgpu_device *adev)
 
 		ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1);
 		ih_rb_cntl = vega10_ih_rb_cntl(ih, ih_rb_cntl);
+		ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
+					   WPTR_OVERFLOW_ENABLE, 0);
+		ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
+					   RB_FULL_DRAIN_ENABLE, 1);
 		WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl);
 
 		/* set rptr, wptr to 0 */
-		WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING1, 0);
 		WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING1, 0);
+		WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING1, 0);
+
+		WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING1,
+			     vega10_ih_doorbell_rptr(ih));
 	}
 
 	ih = &adev->irq.ih2;
@@ -216,13 +230,16 @@ static int vega10_ih_irq_init(struct amdgpu_device *adev)
 		WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI_RING2,
 			     (ih->gpu_addr >> 40) & 0xff);
 
-		ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1);
+		ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2);
 		ih_rb_cntl = vega10_ih_rb_cntl(ih, ih_rb_cntl);
 		WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl);
 
 		/* set rptr, wptr to 0 */
-		WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING2, 0);
 		WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING2, 0);
+		WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING2, 0);
+
+		WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING2,
+			     vega10_ih_doorbell_rptr(ih));
 	}
 
 	tmp = RREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL);
@@ -449,20 +466,23 @@ static int vega10_ih_sw_init(void *handle)
 	if (r)
 		return r;
 
-	if (adev->asic_type == CHIP_VEGA10) {
-		r = amdgpu_ih_ring_init(adev, &adev->irq.ih1, PAGE_SIZE, true);
-		if (r)
-			return r;
-
-		r = amdgpu_ih_ring_init(adev, &adev->irq.ih2, PAGE_SIZE, true);
-		if (r)
-			return r;
-	}
-
-	/* TODO add doorbell for IH1 & IH2 as well */
 	adev->irq.ih.use_doorbell = true;
 	adev->irq.ih.doorbell_index = adev->doorbell_index.ih << 1;
 
+	r = amdgpu_ih_ring_init(adev, &adev->irq.ih1, PAGE_SIZE, true);
+	if (r)
+		return r;
+
+	adev->irq.ih1.use_doorbell = true;
+	adev->irq.ih1.doorbell_index = (adev->doorbell_index.ih + 1) << 1;
+
+	r = amdgpu_ih_ring_init(adev, &adev->irq.ih2, PAGE_SIZE, true);
+	if (r)
+		return r;
+
+	adev->irq.ih2.use_doorbell = true;
+	adev->irq.ih2.doorbell_index = (adev->doorbell_index.ih + 2) << 1;
+
 	r = amdgpu_irq_init(adev);
 
 	return r;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
index cf9a49f49d3a..c1e4d44d6137 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
@@ -467,6 +467,8 @@ struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd,
 	memset(&kfd->doorbell_available_index, 0,
 		sizeof(kfd->doorbell_available_index));
 
+	atomic_set(&kfd->sram_ecc_flag, 0);
+
 	return kfd;
 }
 
@@ -492,9 +494,9 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
 {
 	unsigned int size;
 
-	kfd->mec_fw_version = kfd->kfd2kgd->get_fw_version(kfd->kgd,
+	kfd->mec_fw_version = amdgpu_amdkfd_get_fw_version(kfd->kgd,
 			KGD_ENGINE_MEC1);
-	kfd->sdma_fw_version = kfd->kfd2kgd->get_fw_version(kfd->kgd,
+	kfd->sdma_fw_version = amdgpu_amdkfd_get_fw_version(kfd->kgd,
 			KGD_ENGINE_SDMA1);
 	kfd->shared_resources = *gpu_resources;
 
@@ -662,6 +664,9 @@ int kgd2kfd_post_reset(struct kfd_dev *kfd)
 		return ret;
 	count = atomic_dec_return(&kfd_locked);
 	WARN_ONCE(count != 0, "KFD reset ref. error");
+
+	atomic_set(&kfd->sram_ecc_flag, 0);
+
 	return 0;
 }
 
@@ -1025,6 +1030,12 @@ int kfd_gtt_sa_free(struct kfd_dev *kfd, struct kfd_mem_obj *mem_obj)
 	return 0;
 }
 
+void kgd2kfd_set_sram_ecc_flag(struct kfd_dev *kfd)
+{
+	if (kfd)
+		atomic_inc(&kfd->sram_ecc_flag);
+}
+
 #if defined(CONFIG_DEBUG_FS)
 
 /* This function will send a package to HIQ to hang the HWS
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
index e9f0e0a1b41c..6e1d41c5bf86 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_events.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
@@ -1011,25 +1011,41 @@ void kfd_signal_vm_fault_event(struct kfd_dev *dev, unsigned int pasid,
 void kfd_signal_reset_event(struct kfd_dev *dev)
 {
 	struct kfd_hsa_hw_exception_data hw_exception_data;
+	struct kfd_hsa_memory_exception_data memory_exception_data;
 	struct kfd_process *p;
 	struct kfd_event *ev;
 	unsigned int temp;
 	uint32_t id, idx;
+	int reset_cause = atomic_read(&dev->sram_ecc_flag) ?
+			KFD_HW_EXCEPTION_ECC :
+			KFD_HW_EXCEPTION_GPU_HANG;
 
 	/* Whole gpu reset caused by GPU hang and memory is lost */
 	memset(&hw_exception_data, 0, sizeof(hw_exception_data));
 	hw_exception_data.gpu_id = dev->id;
 	hw_exception_data.memory_lost = 1;
+	hw_exception_data.reset_cause = reset_cause;
+
+	memset(&memory_exception_data, 0, sizeof(memory_exception_data));
+	memory_exception_data.ErrorType = KFD_MEM_ERR_SRAM_ECC;
+	memory_exception_data.gpu_id = dev->id;
+	memory_exception_data.failure.imprecise = true;
 
 	idx = srcu_read_lock(&kfd_processes_srcu);
 	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
 		mutex_lock(&p->event_mutex);
 		id = KFD_FIRST_NONSIGNAL_EVENT_ID;
-		idr_for_each_entry_continue(&p->event_idr, ev, id)
+		idr_for_each_entry_continue(&p->event_idr, ev, id) {
 			if (ev->type == KFD_EVENT_TYPE_HW_EXCEPTION) {
 				ev->hw_exception_data = hw_exception_data;
 				set_event(ev);
 			}
+			if (ev->type == KFD_EVENT_TYPE_MEMORY &&
+			    reset_cause == KFD_HW_EXCEPTION_ECC) {
+				ev->memory_exception_data = memory_exception_data;
+				set_event(ev);
+			}
+		}
 		mutex_unlock(&p->event_mutex);
 	}
 	srcu_read_unlock(&kfd_processes_srcu, idx);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 0eeee3c6d6dc..9e0230965675 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -276,6 +276,9 @@ struct kfd_dev {
 	uint64_t hive_id;
 
 	bool pci_atomic_requested;
+
+	/* SRAM ECC flag */
+	atomic_t sram_ecc_flag;
 };
 
 enum kfd_mempool {
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
index 09da91644f9f..2cb09e088dce 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
@@ -37,6 +37,7 @@
 #include "kfd_device_queue_manager.h"
 #include "kfd_iommu.h"
 #include "amdgpu_amdkfd.h"
+#include "amdgpu_ras.h"
 
 /* topology_device_list - Master list of all topology devices */
 static struct list_head topology_device_list;
@@ -1197,6 +1198,7 @@ int kfd_topology_add_device(struct kfd_dev *gpu)
 	void *crat_image = NULL;
 	size_t image_size = 0;
 	int proximity_domain;
+	struct amdgpu_ras *ctx;
 
 	INIT_LIST_HEAD(&temp_topology_device_list);
 
@@ -1328,6 +1330,20 @@ int kfd_topology_add_device(struct kfd_dev *gpu)
 		dev->node_props.capability |= HSA_CAP_ATS_PRESENT;
 	}
 
+	ctx = amdgpu_ras_get_context((struct amdgpu_device *)(dev->gpu->kgd));
+	if (ctx) {
+		/* kfd only concerns sram ecc on GFX/SDMA and HBM ecc on UMC */
+		dev->node_props.capability |=
+			(((ctx->features & BIT(AMDGPU_RAS_BLOCK__SDMA)) != 0) ||
+			 ((ctx->features & BIT(AMDGPU_RAS_BLOCK__GFX)) != 0)) ?
+			HSA_CAP_SRAM_EDCSUPPORTED : 0;
+		dev->node_props.capability |= ((ctx->features & BIT(AMDGPU_RAS_BLOCK__UMC)) != 0) ?
+			HSA_CAP_MEM_EDCSUPPORTED : 0;
+
+		dev->node_props.capability |= (ctx->features != 0) ?
+			HSA_CAP_RASEVENTNOTIFY : 0;
+	}
+
 	kfd_debug_print_topology();
 
 	if (!res)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
index 92a19be07344..84710cfd23c2 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h
@@ -48,6 +48,10 @@
 #define HSA_CAP_DOORBELL_TYPE_2_0		0x2
 #define HSA_CAP_AQL_QUEUE_DOUBLE_MAP		0x00004000
 
+#define HSA_CAP_SRAM_EDCSUPPORTED		0x00080000
+#define HSA_CAP_MEM_EDCSUPPORTED		0x00100000
+#define HSA_CAP_RASEVENTNOTIFY			0x00200000
+
 struct kfd_node_properties {
 	uint64_t hive_id;
 	uint32_t cpu_cores_count;
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
index 3082b55b1e77..1854506e3e8f 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
@@ -111,7 +111,8 @@ amdgpu_dm_update_connector_after_detect(struct amdgpu_dm_connector *aconnector);
 
 static int amdgpu_dm_plane_init(struct amdgpu_display_manager *dm,
 				struct drm_plane *plane,
-				unsigned long possible_crtcs);
+				unsigned long possible_crtcs,
+				const struct dc_plane_cap *plane_cap);
 static int amdgpu_dm_crtc_init(struct amdgpu_display_manager *dm,
 			       struct drm_plane *plane,
 			       uint32_t link_index);
@@ -137,30 +138,6 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 static void handle_cursor_update(struct drm_plane *plane,
 				 struct drm_plane_state *old_plane_state);
 
-
-
-static const enum drm_plane_type dm_plane_type_default[AMDGPU_MAX_PLANES] = {
-	DRM_PLANE_TYPE_PRIMARY,
-	DRM_PLANE_TYPE_PRIMARY,
-	DRM_PLANE_TYPE_PRIMARY,
-	DRM_PLANE_TYPE_PRIMARY,
-	DRM_PLANE_TYPE_PRIMARY,
-	DRM_PLANE_TYPE_PRIMARY,
-};
-
-static const enum drm_plane_type dm_plane_type_carizzo[AMDGPU_MAX_PLANES] = {
-	DRM_PLANE_TYPE_PRIMARY,
-	DRM_PLANE_TYPE_PRIMARY,
-	DRM_PLANE_TYPE_PRIMARY,
-	DRM_PLANE_TYPE_OVERLAY,/* YUV Capable Underlay */
-};
-
-static const enum drm_plane_type dm_plane_type_stoney[AMDGPU_MAX_PLANES] = {
-	DRM_PLANE_TYPE_PRIMARY,
-	DRM_PLANE_TYPE_PRIMARY,
-	DRM_PLANE_TYPE_OVERLAY, /* YUV Capable Underlay */
-};
-
 /*
  * dm_vblank_get_counter
  *
@@ -275,12 +252,22 @@ get_crtc_by_otg_inst(struct amdgpu_device *adev,
 	return NULL;
 }
 
+static inline bool amdgpu_dm_vrr_active(struct dm_crtc_state *dm_state)
+{
+	return dm_state->freesync_config.state == VRR_STATE_ACTIVE_VARIABLE ||
+	       dm_state->freesync_config.state == VRR_STATE_ACTIVE_FIXED;
+}
+
 static void dm_pflip_high_irq(void *interrupt_params)
 {
 	struct amdgpu_crtc *amdgpu_crtc;
 	struct common_irq_params *irq_params = interrupt_params;
 	struct amdgpu_device *adev = irq_params->adev;
 	unsigned long flags;
+	struct drm_pending_vblank_event *e;
+	struct dm_crtc_state *acrtc_state;
+	uint32_t vpos, hpos, v_blank_start, v_blank_end;
+	bool vrr_active;
 
 	amdgpu_crtc = get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_PFLIP);
 
@@ -303,26 +290,116 @@ static void dm_pflip_high_irq(void *interrupt_params)
 		return;
 	}
 
-	/* Update to correct count(s) if racing with vblank irq */
-	amdgpu_crtc->last_flip_vblank = drm_crtc_accurate_vblank_count(&amdgpu_crtc->base);
+	/* page flip completed. */
+	e = amdgpu_crtc->event;
+	amdgpu_crtc->event = NULL;
 
-	/* wake up userspace */
-	if (amdgpu_crtc->event) {
-		drm_crtc_send_vblank_event(&amdgpu_crtc->base, amdgpu_crtc->event);
+	if (!e)
+		WARN_ON(1);
 
-		/* page flip completed. clean up */
-		amdgpu_crtc->event = NULL;
+	acrtc_state = to_dm_crtc_state(amdgpu_crtc->base.state);
+	vrr_active = amdgpu_dm_vrr_active(acrtc_state);
+
+	/* Fixed refresh rate, or VRR scanout position outside front-porch? */
+	if (!vrr_active ||
+	    !dc_stream_get_scanoutpos(acrtc_state->stream, &v_blank_start,
+				      &v_blank_end, &hpos, &vpos) ||
+	    (vpos < v_blank_start)) {
+		/* Update to correct count and vblank timestamp if racing with
+		 * vblank irq. This also updates to the correct vblank timestamp
+		 * even in VRR mode, as scanout is past the front-porch atm.
+		 */
+		drm_crtc_accurate_vblank_count(&amdgpu_crtc->base);
 
-	} else
-		WARN_ON(1);
+		/* Wake up userspace by sending the pageflip event with proper
+		 * count and timestamp of vblank of flip completion.
+		 */
+		if (e) {
+			drm_crtc_send_vblank_event(&amdgpu_crtc->base, e);
+
+			/* Event sent, so done with vblank for this flip */
+			drm_crtc_vblank_put(&amdgpu_crtc->base);
+		}
+	} else if (e) {
+		/* VRR active and inside front-porch: vblank count and
+		 * timestamp for pageflip event will only be up to date after
+		 * drm_crtc_handle_vblank() has been executed from late vblank
+		 * irq handler after start of back-porch (vline 0). We queue the
+		 * pageflip event for send-out by drm_crtc_handle_vblank() with
+		 * updated timestamp and count, once it runs after us.
+		 *
+		 * We need to open-code this instead of using the helper
+		 * drm_crtc_arm_vblank_event(), as that helper would
+		 * call drm_crtc_accurate_vblank_count(), which we must
+		 * not call in VRR mode while we are in front-porch!
+		 */
+
+		/* sequence will be replaced by real count during send-out. */
+		e->sequence = drm_crtc_vblank_count(&amdgpu_crtc->base);
+		e->pipe = amdgpu_crtc->crtc_id;
+
+		list_add_tail(&e->base.link, &adev->ddev->vblank_event_list);
+		e = NULL;
+	}
+
+	/* Keep track of vblank of this flip for flip throttling. We use the
+	 * cooked hw counter, as that one incremented at start of this vblank
+	 * of pageflip completion, so last_flip_vblank is the forbidden count
+	 * for queueing new pageflips if vsync + VRR is enabled.
+	 */
+	amdgpu_crtc->last_flip_vblank = amdgpu_get_vblank_counter_kms(adev->ddev,
+							amdgpu_crtc->crtc_id);
 
 	amdgpu_crtc->pflip_status = AMDGPU_FLIP_NONE;
 	spin_unlock_irqrestore(&adev->ddev->event_lock, flags);
 
-	DRM_DEBUG_DRIVER("%s - crtc :%d[%p], pflip_stat:AMDGPU_FLIP_NONE\n",
-					__func__, amdgpu_crtc->crtc_id, amdgpu_crtc);
+	DRM_DEBUG_DRIVER("crtc:%d[%p], pflip_stat:AMDGPU_FLIP_NONE, vrr[%d]-fp %d\n",
+			 amdgpu_crtc->crtc_id, amdgpu_crtc,
+			 vrr_active, (int) !e);
+}
+
+static void dm_vupdate_high_irq(void *interrupt_params)
+{
+	struct common_irq_params *irq_params = interrupt_params;
+	struct amdgpu_device *adev = irq_params->adev;
+	struct amdgpu_crtc *acrtc;
+	struct dm_crtc_state *acrtc_state;
+	unsigned long flags;
+
+	acrtc = get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VUPDATE);
+
+	if (acrtc) {
+		acrtc_state = to_dm_crtc_state(acrtc->base.state);
+
+		DRM_DEBUG_DRIVER("crtc:%d, vupdate-vrr:%d\n", acrtc->crtc_id,
+				 amdgpu_dm_vrr_active(acrtc_state));
 
-	drm_crtc_vblank_put(&amdgpu_crtc->base);
+		/* Core vblank handling is done here after end of front-porch in
+		 * vrr mode, as vblank timestamping will give valid results
+		 * while now done after front-porch. This will also deliver
+		 * page-flip completion events that have been queued to us
+		 * if a pageflip happened inside front-porch.
+		 */
+		if (amdgpu_dm_vrr_active(acrtc_state)) {
+			drm_crtc_handle_vblank(&acrtc->base);
+
+			/* BTR processing for pre-DCE12 ASICs */
+			if (acrtc_state->stream &&
+			    adev->family < AMDGPU_FAMILY_AI) {
+				spin_lock_irqsave(&adev->ddev->event_lock, flags);
+				mod_freesync_handle_v_update(
+				    adev->dm.freesync_module,
+				    acrtc_state->stream,
+				    &acrtc_state->vrr_params);
+
+				dc_stream_adjust_vmin_vmax(
+				    adev->dm.dc,
+				    acrtc_state->stream,
+				    &acrtc_state->vrr_params.adjust);
+				spin_unlock_irqrestore(&adev->ddev->event_lock, flags);
+			}
+		}
+	}
 }
 
 static void dm_crtc_high_irq(void *interrupt_params)
@@ -331,18 +408,33 @@ static void dm_crtc_high_irq(void *interrupt_params)
 	struct amdgpu_device *adev = irq_params->adev;
 	struct amdgpu_crtc *acrtc;
 	struct dm_crtc_state *acrtc_state;
+	unsigned long flags;
 
 	acrtc = get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VBLANK);
 
 	if (acrtc) {
-		drm_crtc_handle_vblank(&acrtc->base);
-		amdgpu_dm_crtc_handle_crc_irq(&acrtc->base);
-
 		acrtc_state = to_dm_crtc_state(acrtc->base.state);
 
-		if (acrtc_state->stream &&
+		DRM_DEBUG_DRIVER("crtc:%d, vupdate-vrr:%d\n", acrtc->crtc_id,
+				 amdgpu_dm_vrr_active(acrtc_state));
+
+		/* Core vblank handling at start of front-porch is only possible
+		 * in non-vrr mode, as only there vblank timestamping will give
+		 * valid results while done in front-porch. Otherwise defer it
+		 * to dm_vupdate_high_irq after end of front-porch.
+		 */
+		if (!amdgpu_dm_vrr_active(acrtc_state))
+			drm_crtc_handle_vblank(&acrtc->base);
+
+		/* Following stuff must happen at start of vblank, for crc
+		 * computation and below-the-range btr support in vrr mode.
+		 */
+		amdgpu_dm_crtc_handle_crc_irq(&acrtc->base);
+
+		if (acrtc_state->stream && adev->family >= AMDGPU_FAMILY_AI &&
 		    acrtc_state->vrr_params.supported &&
 		    acrtc_state->freesync_config.state == VRR_STATE_ACTIVE_VARIABLE) {
+			spin_lock_irqsave(&adev->ddev->event_lock, flags);
 			mod_freesync_handle_v_update(
 				adev->dm.freesync_module,
 				acrtc_state->stream,
@@ -352,6 +444,7 @@ static void dm_crtc_high_irq(void *interrupt_params)
 				adev->dm.dc,
 				acrtc_state->stream,
 				&acrtc_state->vrr_params.adjust);
+			spin_unlock_irqrestore(&adev->ddev->event_lock, flags);
 		}
 	}
 }
@@ -462,6 +555,8 @@ static int amdgpu_dm_init(struct amdgpu_device *adev)
 	if (amdgpu_dc_feature_mask & DC_FBC_MASK)
 		init_data.flags.fbc_support = true;
 
+	init_data.flags.power_down_display_on_boot = true;
+
 	/* Display Core create. */
 	adev->dm.dc = dc_create(&init_data);
 
@@ -912,9 +1007,16 @@ static int dm_resume(void *handle)
 	struct drm_plane *plane;
 	struct drm_plane_state *new_plane_state;
 	struct dm_plane_state *dm_new_plane_state;
+	struct dm_atomic_state *dm_state = to_dm_atomic_state(dm->atomic_obj.state);
 	enum dc_connection_type new_connection_type = dc_connection_none;
 	int i;
 
+	/* Recreate dc_state - DC invalidates it when setting power state to S3. */
+	dc_release_state(dm_state->context);
+	dm_state->context = dc_create_state(dm->dc);
+	/* TODO: Remove dc_state->dccg, use dc->dccg directly. */
+	dc_resource_state_construct(dm->dc, dm_state->context);
+
 	/* power on hardware */
 	dc_set_power_state(dm->dc, DC_ACPI_CM_POWER_STATE_D0);
 
@@ -1457,6 +1559,27 @@ static int dce110_register_irq_handlers(struct amdgpu_device *adev)
 				dm_crtc_high_irq, c_irq_params);
 	}
 
+	/* Use VUPDATE interrupt */
+	for (i = VISLANDS30_IV_SRCID_D1_V_UPDATE_INT; i <= VISLANDS30_IV_SRCID_D6_V_UPDATE_INT; i += 2) {
+		r = amdgpu_irq_add_id(adev, client_id, i, &adev->vupdate_irq);
+		if (r) {
+			DRM_ERROR("Failed to add vupdate irq id!\n");
+			return r;
+		}
+
+		int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
+		int_params.irq_source =
+			dc_interrupt_to_irq_source(dc, i, 0);
+
+		c_irq_params = &adev->dm.vupdate_params[int_params.irq_source - DC_IRQ_SOURCE_VUPDATE1];
+
+		c_irq_params->adev = adev;
+		c_irq_params->irq_src = int_params.irq_source;
+
+		amdgpu_dm_irq_register_interrupt(adev, &int_params,
+				dm_vupdate_high_irq, c_irq_params);
+	}
+
 	/* Use GRPH_PFLIP interrupt */
 	for (i = VISLANDS30_IV_SRCID_D1_GRPH_PFLIP;
 			i <= VISLANDS30_IV_SRCID_D6_GRPH_PFLIP; i += 2) {
@@ -1542,6 +1665,34 @@ static int dcn10_register_irq_handlers(struct amdgpu_device *adev)
 				dm_crtc_high_irq, c_irq_params);
 	}
 
+	/* Use VUPDATE_NO_LOCK interrupt on DCN, which seems to correspond to
+	 * the regular VUPDATE interrupt on DCE. We want DC_IRQ_SOURCE_VUPDATEx
+	 * to trigger at end of each vblank, regardless of state of the lock,
+	 * matching DCE behaviour.
+	 */
+	for (i = DCN_1_0__SRCID__OTG0_IHC_V_UPDATE_NO_LOCK_INTERRUPT;
+	     i <= DCN_1_0__SRCID__OTG0_IHC_V_UPDATE_NO_LOCK_INTERRUPT + adev->mode_info.num_crtc - 1;
+	     i++) {
+		r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, i, &adev->vupdate_irq);
+
+		if (r) {
+			DRM_ERROR("Failed to add vupdate irq id!\n");
+			return r;
+		}
+
+		int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
+		int_params.irq_source =
+			dc_interrupt_to_irq_source(dc, i, 0);
+
+		c_irq_params = &adev->dm.vupdate_params[int_params.irq_source - DC_IRQ_SOURCE_VUPDATE1];
+
+		c_irq_params->adev = adev;
+		c_irq_params->irq_src = int_params.irq_source;
+
+		amdgpu_dm_irq_register_interrupt(adev, &int_params,
+				dm_vupdate_high_irq, c_irq_params);
+	}
+
 	/* Use GRPH_PFLIP interrupt */
 	for (i = DCN_1_0__SRCID__HUBP0_FLIP_INTERRUPT;
 			i <= DCN_1_0__SRCID__HUBP0_FLIP_INTERRUPT + adev->mode_info.num_crtc - 1;
@@ -1593,15 +1744,10 @@ static int dm_atomic_get_state(struct drm_atomic_state *state,
 	struct amdgpu_device *adev = dev->dev_private;
 	struct amdgpu_display_manager *dm = &adev->dm;
 	struct drm_private_state *priv_state;
-	int ret;
 
 	if (*dm_state)
 		return 0;
 
-	ret = drm_modeset_lock(&dm->atomic_obj_lock, state->acquire_ctx);
-	if (ret)
-		return ret;
-
 	priv_state = drm_atomic_get_private_obj_state(state, &dm->atomic_obj);
 	if (IS_ERR(priv_state))
 		return PTR_ERR(priv_state);
@@ -1658,17 +1804,16 @@ dm_atomic_duplicate_state(struct drm_private_obj *obj)
 
 	__drm_atomic_helper_private_obj_duplicate_state(obj, &new_state->base);
 
-	new_state->context = dc_create_state();
+	old_state = to_dm_atomic_state(obj->state);
+
+	if (old_state && old_state->context)
+		new_state->context = dc_copy_state(old_state->context);
+
 	if (!new_state->context) {
 		kfree(new_state);
 		return NULL;
 	}
 
-	old_state = to_dm_atomic_state(obj->state);
-	if (old_state && old_state->context)
-		dc_resource_state_copy_construct(old_state->context,
-						 new_state->context);
-
 	return &new_state->base;
 }
 
@@ -1708,13 +1853,11 @@ static int amdgpu_dm_mode_config_init(struct amdgpu_device *adev)
 
 	adev->ddev->mode_config.fb_base = adev->gmc.aper_base;
 
-	drm_modeset_lock_init(&adev->dm.atomic_obj_lock);
-
 	state = kzalloc(sizeof(*state), GFP_KERNEL);
 	if (!state)
 		return -ENOMEM;
 
-	state->context = dc_create_state();
+	state->context = dc_create_state(adev->dm.dc);
 	if (!state->context) {
 		kfree(state);
 		return -ENOMEM;
@@ -1841,39 +1984,42 @@ amdgpu_dm_register_backlight_device(struct amdgpu_display_manager *dm)
 #endif
 
 static int initialize_plane(struct amdgpu_display_manager *dm,
-			     struct amdgpu_mode_info *mode_info,
-			     int plane_id)
+			    struct amdgpu_mode_info *mode_info, int plane_id,
+			    enum drm_plane_type plane_type,
+			    const struct dc_plane_cap *plane_cap)
 {
 	struct drm_plane *plane;
 	unsigned long possible_crtcs;
 	int ret = 0;
 
 	plane = kzalloc(sizeof(struct drm_plane), GFP_KERNEL);
-	mode_info->planes[plane_id] = plane;
-
 	if (!plane) {
 		DRM_ERROR("KMS: Failed to allocate plane\n");
 		return -ENOMEM;
 	}
-	plane->type = mode_info->plane_type[plane_id];
+	plane->type = plane_type;
 
 	/*
-	 * HACK: IGT tests expect that each plane can only have
-	 * one possible CRTC. For now, set one CRTC for each
-	 * plane that is not an underlay, but still allow multiple
-	 * CRTCs for underlay planes.
+	 * HACK: IGT tests expect that the primary plane for a CRTC
+	 * can only have one possible CRTC. Only expose support for
+	 * any CRTC if they're not going to be used as a primary plane
+	 * for a CRTC - like overlay or underlay planes.
 	 */
 	possible_crtcs = 1 << plane_id;
 	if (plane_id >= dm->dc->caps.max_streams)
 		possible_crtcs = 0xff;
 
-	ret = amdgpu_dm_plane_init(dm, mode_info->planes[plane_id], possible_crtcs);
+	ret = amdgpu_dm_plane_init(dm, plane, possible_crtcs, plane_cap);
 
 	if (ret) {
 		DRM_ERROR("KMS: Failed to initialize plane\n");
+		kfree(plane);
 		return ret;
 	}
 
+	if (mode_info)
+		mode_info->planes[plane_id] = plane;
+
 	return ret;
 }
 
@@ -1916,8 +2062,9 @@ static int amdgpu_dm_initialize_drm_device(struct amdgpu_device *adev)
 	struct amdgpu_encoder *aencoder = NULL;
 	struct amdgpu_mode_info *mode_info = &adev->mode_info;
 	uint32_t link_cnt;
-	int32_t total_overlay_planes, total_primary_planes;
+	int32_t primary_planes;
 	enum dc_connection_type new_connection_type = dc_connection_none;
+	const struct dc_plane_cap *plane;
 
 	link_cnt = dm->dc->caps.max_links;
 	if (amdgpu_dm_mode_config_init(dm->adev)) {
@@ -1925,24 +2072,53 @@ static int amdgpu_dm_initialize_drm_device(struct amdgpu_device *adev)
 		return -EINVAL;
 	}
 
-	/* Identify the number of planes to be initialized */
-	total_overlay_planes = dm->dc->caps.max_slave_planes;
-	total_primary_planes = dm->dc->caps.max_planes - dm->dc->caps.max_slave_planes;
+	/* There is one primary plane per CRTC */
+	primary_planes = dm->dc->caps.max_streams;
+	ASSERT(primary_planes <= AMDGPU_MAX_PLANES);
 
-	/* First initialize overlay planes, index starting after primary planes */
-	for (i = (total_overlay_planes - 1); i >= 0; i--) {
-		if (initialize_plane(dm, mode_info, (total_primary_planes + i))) {
-			DRM_ERROR("KMS: Failed to initialize overlay plane\n");
+	/*
+	 * Initialize primary planes, implicit planes for legacy IOCTLS.
+	 * Order is reversed to match iteration order in atomic check.
+	 */
+	for (i = (primary_planes - 1); i >= 0; i--) {
+		plane = &dm->dc->caps.planes[i];
+
+		if (initialize_plane(dm, mode_info, i,
+				     DRM_PLANE_TYPE_PRIMARY, plane)) {
+			DRM_ERROR("KMS: Failed to initialize primary plane\n");
 			goto fail;
 		}
 	}
 
-	/* Initialize primary planes */
-	for (i = (total_primary_planes - 1); i >= 0; i--) {
-		if (initialize_plane(dm, mode_info, i)) {
-			DRM_ERROR("KMS: Failed to initialize primary plane\n");
+	/*
+	 * Initialize overlay planes, index starting after primary planes.
+	 * These planes have a higher DRM index than the primary planes since
+	 * they should be considered as having a higher z-order.
+	 * Order is reversed to match iteration order in atomic check.
+	 *
+	 * Only support DCN for now, and only expose one so we don't encourage
+	 * userspace to use up all the pipes.
+	 */
+	for (i = 0; i < dm->dc->caps.max_planes; ++i) {
+		struct dc_plane_cap *plane = &dm->dc->caps.planes[i];
+
+		if (plane->type != DC_PLANE_TYPE_DCN_UNIVERSAL)
+			continue;
+
+		if (!plane->blends_with_above || !plane->blends_with_below)
+			continue;
+
+		if (!plane->pixel_format_support.argb8888)
+			continue;
+
+		if (initialize_plane(dm, NULL, primary_planes + i,
+				     DRM_PLANE_TYPE_OVERLAY, plane)) {
+			DRM_ERROR("KMS: Failed to initialize overlay plane\n");
 			goto fail;
 		}
+
+		/* Only create one overlay plane. */
+		break;
 	}
 
 	for (i = 0; i < dm->dc->caps.max_streams; i++)
@@ -2042,8 +2218,7 @@ static int amdgpu_dm_initialize_drm_device(struct amdgpu_device *adev)
 fail:
 	kfree(aencoder);
 	kfree(aconnector);
-	for (i = 0; i < dm->dc->caps.max_planes; i++)
-		kfree(mode_info->planes[i]);
+
 	return -EINVAL;
 }
 
@@ -2124,53 +2299,45 @@ static int dm_early_init(void *handle)
 		adev->mode_info.num_crtc = 6;
 		adev->mode_info.num_hpd = 6;
 		adev->mode_info.num_dig = 6;
-		adev->mode_info.plane_type = dm_plane_type_default;
 		break;
 	case CHIP_KAVERI:
 		adev->mode_info.num_crtc = 4;
 		adev->mode_info.num_hpd = 6;
 		adev->mode_info.num_dig = 7;
-		adev->mode_info.plane_type = dm_plane_type_default;
 		break;
 	case CHIP_KABINI:
 	case CHIP_MULLINS:
 		adev->mode_info.num_crtc = 2;
 		adev->mode_info.num_hpd = 6;
 		adev->mode_info.num_dig = 6;
-		adev->mode_info.plane_type = dm_plane_type_default;
 		break;
 	case CHIP_FIJI:
 	case CHIP_TONGA:
 		adev->mode_info.num_crtc = 6;
 		adev->mode_info.num_hpd = 6;
 		adev->mode_info.num_dig = 7;
-		adev->mode_info.plane_type = dm_plane_type_default;
 		break;
 	case CHIP_CARRIZO:
 		adev->mode_info.num_crtc = 3;
 		adev->mode_info.num_hpd = 6;
 		adev->mode_info.num_dig = 9;
-		adev->mode_info.plane_type = dm_plane_type_carizzo;
 		break;
 	case CHIP_STONEY:
 		adev->mode_info.num_crtc = 2;
 		adev->mode_info.num_hpd = 6;
 		adev->mode_info.num_dig = 9;
-		adev->mode_info.plane_type = dm_plane_type_stoney;
 		break;
 	case CHIP_POLARIS11:
 	case CHIP_POLARIS12:
 		adev->mode_info.num_crtc = 5;
 		adev->mode_info.num_hpd = 5;
 		adev->mode_info.num_dig = 5;
-		adev->mode_info.plane_type = dm_plane_type_default;
 		break;
 	case CHIP_POLARIS10:
 	case CHIP_VEGAM:
 		adev->mode_info.num_crtc = 6;
 		adev->mode_info.num_hpd = 6;
 		adev->mode_info.num_dig = 6;
-		adev->mode_info.plane_type = dm_plane_type_default;
 		break;
 	case CHIP_VEGA10:
 	case CHIP_VEGA12:
@@ -2178,14 +2345,12 @@ static int dm_early_init(void *handle)
 		adev->mode_info.num_crtc = 6;
 		adev->mode_info.num_hpd = 6;
 		adev->mode_info.num_dig = 6;
-		adev->mode_info.plane_type = dm_plane_type_default;
 		break;
 #if defined(CONFIG_DRM_AMD_DC_DCN1_0)
 	case CHIP_RAVEN:
 		adev->mode_info.num_crtc = 4;
 		adev->mode_info.num_hpd = 4;
 		adev->mode_info.num_dig = 4;
-		adev->mode_info.plane_type = dm_plane_type_default;
 		break;
 #endif
 	default:
@@ -2243,56 +2408,63 @@ static const struct drm_encoder_funcs amdgpu_dm_encoder_funcs = {
 	.destroy = amdgpu_dm_encoder_destroy,
 };
 
-static bool fill_rects_from_plane_state(const struct drm_plane_state *state,
-					struct dc_plane_state *plane_state)
+
+static int fill_dc_scaling_info(const struct drm_plane_state *state,
+				struct dc_scaling_info *scaling_info)
 {
-	plane_state->src_rect.x = state->src_x >> 16;
-	plane_state->src_rect.y = state->src_y >> 16;
-	/* we ignore the mantissa for now and do not deal with floating pixels :( */
-	plane_state->src_rect.width = state->src_w >> 16;
+	int scale_w, scale_h;
 
-	if (plane_state->src_rect.width == 0)
-		return false;
+	memset(scaling_info, 0, sizeof(*scaling_info));
 
-	plane_state->src_rect.height = state->src_h >> 16;
-	if (plane_state->src_rect.height == 0)
-		return false;
+	/* Source is fixed 16.16 but we ignore mantissa for now... */
+	scaling_info->src_rect.x = state->src_x >> 16;
+	scaling_info->src_rect.y = state->src_y >> 16;
 
-	plane_state->dst_rect.x = state->crtc_x;
-	plane_state->dst_rect.y = state->crtc_y;
+	scaling_info->src_rect.width = state->src_w >> 16;
+	if (scaling_info->src_rect.width == 0)
+		return -EINVAL;
+
+	scaling_info->src_rect.height = state->src_h >> 16;
+	if (scaling_info->src_rect.height == 0)
+		return -EINVAL;
+
+	scaling_info->dst_rect.x = state->crtc_x;
+	scaling_info->dst_rect.y = state->crtc_y;
 
 	if (state->crtc_w == 0)
-		return false;
+		return -EINVAL;
 
-	plane_state->dst_rect.width = state->crtc_w;
+	scaling_info->dst_rect.width = state->crtc_w;
 
 	if (state->crtc_h == 0)
-		return false;
+		return -EINVAL;
 
-	plane_state->dst_rect.height = state->crtc_h;
+	scaling_info->dst_rect.height = state->crtc_h;
 
-	plane_state->clip_rect = plane_state->dst_rect;
+	/* DRM doesn't specify clipping on destination output. */
+	scaling_info->clip_rect = scaling_info->dst_rect;
 
-	switch (state->rotation & DRM_MODE_ROTATE_MASK) {
-	case DRM_MODE_ROTATE_0:
-		plane_state->rotation = ROTATION_ANGLE_0;
-		break;
-	case DRM_MODE_ROTATE_90:
-		plane_state->rotation = ROTATION_ANGLE_90;
-		break;
-	case DRM_MODE_ROTATE_180:
-		plane_state->rotation = ROTATION_ANGLE_180;
-		break;
-	case DRM_MODE_ROTATE_270:
-		plane_state->rotation = ROTATION_ANGLE_270;
-		break;
-	default:
-		plane_state->rotation = ROTATION_ANGLE_0;
-		break;
-	}
+	/* TODO: Validate scaling per-format with DC plane caps */
+	scale_w = scaling_info->dst_rect.width * 1000 /
+		  scaling_info->src_rect.width;
 
-	return true;
+	if (scale_w < 250 || scale_w > 16000)
+		return -EINVAL;
+
+	scale_h = scaling_info->dst_rect.height * 1000 /
+		  scaling_info->src_rect.height;
+
+	if (scale_h < 250 || scale_h > 16000)
+		return -EINVAL;
+
+	/*
+	 * The "scaling_quality" can be ignored for now, quality = 0 has DC
+	 * assume reasonable defaults based on the format.
+	 */
+
+	return 0;
 }
+
 static int get_fb_info(const struct amdgpu_framebuffer *amdgpu_fb,
 		       uint64_t *tiling_flags)
 {
@@ -2321,10 +2493,16 @@ static inline uint64_t get_dcc_address(uint64_t address, uint64_t tiling_flags)
 	return offset ? (address + offset * 256) : 0;
 }
 
-static bool fill_plane_dcc_attributes(struct amdgpu_device *adev,
-				      const struct amdgpu_framebuffer *afb,
-				      struct dc_plane_state *plane_state,
-				      uint64_t info)
+static int
+fill_plane_dcc_attributes(struct amdgpu_device *adev,
+			  const struct amdgpu_framebuffer *afb,
+			  const enum surface_pixel_format format,
+			  const enum dc_rotation_angle rotation,
+			  const union plane_size *plane_size,
+			  const union dc_tiling_info *tiling_info,
+			  const uint64_t info,
+			  struct dc_plane_dcc_param *dcc,
+			  struct dc_plane_address *address)
 {
 	struct dc *dc = adev->dm.dc;
 	struct dc_dcc_surface_param input;
@@ -2337,133 +2515,103 @@ static bool fill_plane_dcc_attributes(struct amdgpu_device *adev,
 	memset(&output, 0, sizeof(output));
 
 	if (!offset)
-		return false;
+		return 0;
+
+	if (format >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN)
+		return 0;
 
 	if (!dc->cap_funcs.get_dcc_compression_cap)
-		return false;
+		return -EINVAL;
 
-	input.format = plane_state->format;
-	input.surface_size.width =
-		plane_state->plane_size.grph.surface_size.width;
-	input.surface_size.height =
-		plane_state->plane_size.grph.surface_size.height;
-	input.swizzle_mode = plane_state->tiling_info.gfx9.swizzle;
+	input.format = format;
+	input.surface_size.width = plane_size->grph.surface_size.width;
+	input.surface_size.height = plane_size->grph.surface_size.height;
+	input.swizzle_mode = tiling_info->gfx9.swizzle;
 
-	if (plane_state->rotation == ROTATION_ANGLE_0 ||
-	    plane_state->rotation == ROTATION_ANGLE_180)
+	if (rotation == ROTATION_ANGLE_0 || rotation == ROTATION_ANGLE_180)
 		input.scan = SCAN_DIRECTION_HORIZONTAL;
-	else if (plane_state->rotation == ROTATION_ANGLE_90 ||
-		 plane_state->rotation == ROTATION_ANGLE_270)
+	else if (rotation == ROTATION_ANGLE_90 || rotation == ROTATION_ANGLE_270)
 		input.scan = SCAN_DIRECTION_VERTICAL;
 
 	if (!dc->cap_funcs.get_dcc_compression_cap(dc, &input, &output))
-		return false;
+		return -EINVAL;
 
 	if (!output.capable)
-		return false;
+		return -EINVAL;
 
 	if (i64b == 0 && output.grph.rgb.independent_64b_blks != 0)
-		return false;
+		return -EINVAL;
 
-	plane_state->dcc.enable = 1;
-	plane_state->dcc.grph.meta_pitch =
+	dcc->enable = 1;
+	dcc->grph.meta_pitch =
 		AMDGPU_TILING_GET(info, DCC_PITCH_MAX) + 1;
-	plane_state->dcc.grph.independent_64b_blks = i64b;
+	dcc->grph.independent_64b_blks = i64b;
 
 	dcc_address = get_dcc_address(afb->address, info);
-	plane_state->address.grph.meta_addr.low_part =
-		lower_32_bits(dcc_address);
-	plane_state->address.grph.meta_addr.high_part =
-		upper_32_bits(dcc_address);
+	address->grph.meta_addr.low_part = lower_32_bits(dcc_address);
+	address->grph.meta_addr.high_part = upper_32_bits(dcc_address);
 
-	return true;
+	return 0;
 }
 
-static int fill_plane_attributes_from_fb(struct amdgpu_device *adev,
-					 struct dc_plane_state *plane_state,
-					 const struct amdgpu_framebuffer *amdgpu_fb)
-{
-	uint64_t tiling_flags;
-	unsigned int awidth;
-	const struct drm_framebuffer *fb = &amdgpu_fb->base;
-	int ret = 0;
-	struct drm_format_name_buf format_name;
-
-	ret = get_fb_info(
-		amdgpu_fb,
-		&tiling_flags);
-
-	if (ret)
-		return ret;
-
-	switch (fb->format->format) {
-	case DRM_FORMAT_C8:
-		plane_state->format = SURFACE_PIXEL_FORMAT_GRPH_PALETA_256_COLORS;
-		break;
-	case DRM_FORMAT_RGB565:
-		plane_state->format = SURFACE_PIXEL_FORMAT_GRPH_RGB565;
-		break;
-	case DRM_FORMAT_XRGB8888:
-	case DRM_FORMAT_ARGB8888:
-		plane_state->format = SURFACE_PIXEL_FORMAT_GRPH_ARGB8888;
-		break;
-	case DRM_FORMAT_XRGB2101010:
-	case DRM_FORMAT_ARGB2101010:
-		plane_state->format = SURFACE_PIXEL_FORMAT_GRPH_ARGB2101010;
-		break;
-	case DRM_FORMAT_XBGR2101010:
-	case DRM_FORMAT_ABGR2101010:
-		plane_state->format = SURFACE_PIXEL_FORMAT_GRPH_ABGR2101010;
-		break;
-	case DRM_FORMAT_XBGR8888:
-	case DRM_FORMAT_ABGR8888:
-		plane_state->format = SURFACE_PIXEL_FORMAT_GRPH_ABGR8888;
-		break;
-	case DRM_FORMAT_NV21:
-		plane_state->format = SURFACE_PIXEL_FORMAT_VIDEO_420_YCbCr;
-		break;
-	case DRM_FORMAT_NV12:
-		plane_state->format = SURFACE_PIXEL_FORMAT_VIDEO_420_YCrCb;
-		break;
-	default:
-		DRM_ERROR("Unsupported screen format %s\n",
-			  drm_get_format_name(fb->format->format, &format_name));
-		return -EINVAL;
-	}
-
-	memset(&plane_state->address, 0, sizeof(plane_state->address));
-	memset(&plane_state->tiling_info, 0, sizeof(plane_state->tiling_info));
-	memset(&plane_state->dcc, 0, sizeof(plane_state->dcc));
-
-	if (plane_state->format < SURFACE_PIXEL_FORMAT_VIDEO_BEGIN) {
-		plane_state->address.type = PLN_ADDR_TYPE_GRAPHICS;
-		plane_state->plane_size.grph.surface_size.x = 0;
-		plane_state->plane_size.grph.surface_size.y = 0;
-		plane_state->plane_size.grph.surface_size.width = fb->width;
-		plane_state->plane_size.grph.surface_size.height = fb->height;
-		plane_state->plane_size.grph.surface_pitch =
-				fb->pitches[0] / fb->format->cpp[0];
-		/* TODO: unhardcode */
-		plane_state->color_space = COLOR_SPACE_SRGB;
+static int
+fill_plane_buffer_attributes(struct amdgpu_device *adev,
+			     const struct amdgpu_framebuffer *afb,
+			     const enum surface_pixel_format format,
+			     const enum dc_rotation_angle rotation,
+			     const uint64_t tiling_flags,
+			     union dc_tiling_info *tiling_info,
+			     union plane_size *plane_size,
+			     struct dc_plane_dcc_param *dcc,
+			     struct dc_plane_address *address)
+{
+	const struct drm_framebuffer *fb = &afb->base;
+	int ret;
 
+	memset(tiling_info, 0, sizeof(*tiling_info));
+	memset(plane_size, 0, sizeof(*plane_size));
+	memset(dcc, 0, sizeof(*dcc));
+	memset(address, 0, sizeof(*address));
+
+	if (format < SURFACE_PIXEL_FORMAT_VIDEO_BEGIN) {
+		plane_size->grph.surface_size.x = 0;
+		plane_size->grph.surface_size.y = 0;
+		plane_size->grph.surface_size.width = fb->width;
+		plane_size->grph.surface_size.height = fb->height;
+		plane_size->grph.surface_pitch =
+			fb->pitches[0] / fb->format->cpp[0];
+
+		address->type = PLN_ADDR_TYPE_GRAPHICS;
+		address->grph.addr.low_part = lower_32_bits(afb->address);
+		address->grph.addr.high_part = upper_32_bits(afb->address);
 	} else {
-		awidth = ALIGN(fb->width, 64);
-		plane_state->address.type = PLN_ADDR_TYPE_VIDEO_PROGRESSIVE;
-		plane_state->plane_size.video.luma_size.x = 0;
-		plane_state->plane_size.video.luma_size.y = 0;
-		plane_state->plane_size.video.luma_size.width = awidth;
-		plane_state->plane_size.video.luma_size.height = fb->height;
-		/* TODO: unhardcode */
-		plane_state->plane_size.video.luma_pitch = awidth;
-
-		plane_state->plane_size.video.chroma_size.x = 0;
-		plane_state->plane_size.video.chroma_size.y = 0;
-		plane_state->plane_size.video.chroma_size.width = awidth;
-		plane_state->plane_size.video.chroma_size.height = fb->height;
-		plane_state->plane_size.video.chroma_pitch = awidth / 2;
-
-		/* TODO: unhardcode */
-		plane_state->color_space = COLOR_SPACE_YCBCR709;
+		uint64_t chroma_addr = afb->address + fb->offsets[1];
+
+		plane_size->video.luma_size.x = 0;
+		plane_size->video.luma_size.y = 0;
+		plane_size->video.luma_size.width = fb->width;
+		plane_size->video.luma_size.height = fb->height;
+		plane_size->video.luma_pitch =
+			fb->pitches[0] / fb->format->cpp[0];
+
+		plane_size->video.chroma_size.x = 0;
+		plane_size->video.chroma_size.y = 0;
+		/* TODO: set these based on surface format */
+		plane_size->video.chroma_size.width = fb->width / 2;
+		plane_size->video.chroma_size.height = fb->height / 2;
+
+		plane_size->video.chroma_pitch =
+			fb->pitches[1] / fb->format->cpp[1];
+
+		address->type = PLN_ADDR_TYPE_VIDEO_PROGRESSIVE;
+		address->video_progressive.luma_addr.low_part =
+			lower_32_bits(afb->address);
+		address->video_progressive.luma_addr.high_part =
+			upper_32_bits(afb->address);
+		address->video_progressive.chroma_addr.low_part =
+			lower_32_bits(chroma_addr);
+		address->video_progressive.chroma_addr.high_part =
+			upper_32_bits(chroma_addr);
 	}
 
 	/* Fill GFX8 params */
@@ -2477,21 +2625,21 @@ static int fill_plane_attributes_from_fb(struct amdgpu_device *adev,
 		num_banks = AMDGPU_TILING_GET(tiling_flags, NUM_BANKS);
 
 		/* XXX fix me for VI */
-		plane_state->tiling_info.gfx8.num_banks = num_banks;
-		plane_state->tiling_info.gfx8.array_mode =
+		tiling_info->gfx8.num_banks = num_banks;
+		tiling_info->gfx8.array_mode =
 				DC_ARRAY_2D_TILED_THIN1;
-		plane_state->tiling_info.gfx8.tile_split = tile_split;
-		plane_state->tiling_info.gfx8.bank_width = bankw;
-		plane_state->tiling_info.gfx8.bank_height = bankh;
-		plane_state->tiling_info.gfx8.tile_aspect = mtaspect;
-		plane_state->tiling_info.gfx8.tile_mode =
+		tiling_info->gfx8.tile_split = tile_split;
+		tiling_info->gfx8.bank_width = bankw;
+		tiling_info->gfx8.bank_height = bankh;
+		tiling_info->gfx8.tile_aspect = mtaspect;
+		tiling_info->gfx8.tile_mode =
 				DC_ADDR_SURF_MICRO_TILING_DISPLAY;
 	} else if (AMDGPU_TILING_GET(tiling_flags, ARRAY_MODE)
 			== DC_ARRAY_1D_TILED_THIN1) {
-		plane_state->tiling_info.gfx8.array_mode = DC_ARRAY_1D_TILED_THIN1;
+		tiling_info->gfx8.array_mode = DC_ARRAY_1D_TILED_THIN1;
 	}
 
-	plane_state->tiling_info.gfx8.pipe_config =
+	tiling_info->gfx8.pipe_config =
 			AMDGPU_TILING_GET(tiling_flags, PIPE_CONFIG);
 
 	if (adev->asic_type == CHIP_VEGA10 ||
@@ -2499,60 +2647,249 @@ static int fill_plane_attributes_from_fb(struct amdgpu_device *adev,
 	    adev->asic_type == CHIP_VEGA20 ||
 	    adev->asic_type == CHIP_RAVEN) {
 		/* Fill GFX9 params */
-		plane_state->tiling_info.gfx9.num_pipes =
+		tiling_info->gfx9.num_pipes =
 			adev->gfx.config.gb_addr_config_fields.num_pipes;
-		plane_state->tiling_info.gfx9.num_banks =
+		tiling_info->gfx9.num_banks =
 			adev->gfx.config.gb_addr_config_fields.num_banks;
-		plane_state->tiling_info.gfx9.pipe_interleave =
+		tiling_info->gfx9.pipe_interleave =
 			adev->gfx.config.gb_addr_config_fields.pipe_interleave_size;
-		plane_state->tiling_info.gfx9.num_shader_engines =
+		tiling_info->gfx9.num_shader_engines =
 			adev->gfx.config.gb_addr_config_fields.num_se;
-		plane_state->tiling_info.gfx9.max_compressed_frags =
+		tiling_info->gfx9.max_compressed_frags =
 			adev->gfx.config.gb_addr_config_fields.max_compress_frags;
-		plane_state->tiling_info.gfx9.num_rb_per_se =
+		tiling_info->gfx9.num_rb_per_se =
 			adev->gfx.config.gb_addr_config_fields.num_rb_per_se;
-		plane_state->tiling_info.gfx9.swizzle =
+		tiling_info->gfx9.swizzle =
 			AMDGPU_TILING_GET(tiling_flags, SWIZZLE_MODE);
-		plane_state->tiling_info.gfx9.shaderEnable = 1;
+		tiling_info->gfx9.shaderEnable = 1;
 
-		fill_plane_dcc_attributes(adev, amdgpu_fb, plane_state,
-					  tiling_flags);
+		ret = fill_plane_dcc_attributes(adev, afb, format, rotation,
+						plane_size, tiling_info,
+						tiling_flags, dcc, address);
+		if (ret)
+			return ret;
 	}
 
-	plane_state->visible = true;
-	plane_state->scaling_quality.h_taps_c = 0;
-	plane_state->scaling_quality.v_taps_c = 0;
+	return 0;
+}
 
-	/* is this needed? is plane_state zeroed at allocation? */
-	plane_state->scaling_quality.h_taps = 0;
-	plane_state->scaling_quality.v_taps = 0;
-	plane_state->stereo_format = PLANE_STEREO_FORMAT_NONE;
+static void
+fill_blending_from_plane_state(const struct drm_plane_state *plane_state,
+			       bool *per_pixel_alpha, bool *global_alpha,
+			       int *global_alpha_value)
+{
+	*per_pixel_alpha = false;
+	*global_alpha = false;
+	*global_alpha_value = 0xff;
 
-	return ret;
+	if (plane_state->plane->type != DRM_PLANE_TYPE_OVERLAY)
+		return;
 
+	if (plane_state->pixel_blend_mode == DRM_MODE_BLEND_PREMULTI) {
+		static const uint32_t alpha_formats[] = {
+			DRM_FORMAT_ARGB8888,
+			DRM_FORMAT_RGBA8888,
+			DRM_FORMAT_ABGR8888,
+		};
+		uint32_t format = plane_state->fb->format->format;
+		unsigned int i;
+
+		for (i = 0; i < ARRAY_SIZE(alpha_formats); ++i) {
+			if (format == alpha_formats[i]) {
+				*per_pixel_alpha = true;
+				break;
+			}
+		}
+	}
+
+	if (plane_state->alpha < 0xffff) {
+		*global_alpha = true;
+		*global_alpha_value = plane_state->alpha >> 8;
+	}
 }
 
-static int fill_plane_attributes(struct amdgpu_device *adev,
-				 struct dc_plane_state *dc_plane_state,
-				 struct drm_plane_state *plane_state,
-				 struct drm_crtc_state *crtc_state)
+static int
+fill_plane_color_attributes(const struct drm_plane_state *plane_state,
+			    const enum surface_pixel_format format,
+			    enum dc_color_space *color_space)
 {
-	const struct amdgpu_framebuffer *amdgpu_fb =
+	bool full_range;
+
+	*color_space = COLOR_SPACE_SRGB;
+
+	/* DRM color properties only affect non-RGB formats. */
+	if (format < SURFACE_PIXEL_FORMAT_VIDEO_BEGIN)
+		return 0;
+
+	full_range = (plane_state->color_range == DRM_COLOR_YCBCR_FULL_RANGE);
+
+	switch (plane_state->color_encoding) {
+	case DRM_COLOR_YCBCR_BT601:
+		if (full_range)
+			*color_space = COLOR_SPACE_YCBCR601;
+		else
+			*color_space = COLOR_SPACE_YCBCR601_LIMITED;
+		break;
+
+	case DRM_COLOR_YCBCR_BT709:
+		if (full_range)
+			*color_space = COLOR_SPACE_YCBCR709;
+		else
+			*color_space = COLOR_SPACE_YCBCR709_LIMITED;
+		break;
+
+	case DRM_COLOR_YCBCR_BT2020:
+		if (full_range)
+			*color_space = COLOR_SPACE_2020_YCBCR;
+		else
+			return -EINVAL;
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+fill_dc_plane_info_and_addr(struct amdgpu_device *adev,
+			    const struct drm_plane_state *plane_state,
+			    const uint64_t tiling_flags,
+			    struct dc_plane_info *plane_info,
+			    struct dc_plane_address *address)
+{
+	const struct drm_framebuffer *fb = plane_state->fb;
+	const struct amdgpu_framebuffer *afb =
 		to_amdgpu_framebuffer(plane_state->fb);
-	const struct drm_crtc *crtc = plane_state->crtc;
-	int ret = 0;
+	struct drm_format_name_buf format_name;
+	int ret;
 
-	if (!fill_rects_from_plane_state(plane_state, dc_plane_state))
+	memset(plane_info, 0, sizeof(*plane_info));
+
+	switch (fb->format->format) {
+	case DRM_FORMAT_C8:
+		plane_info->format =
+			SURFACE_PIXEL_FORMAT_GRPH_PALETA_256_COLORS;
+		break;
+	case DRM_FORMAT_RGB565:
+		plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_RGB565;
+		break;
+	case DRM_FORMAT_XRGB8888:
+	case DRM_FORMAT_ARGB8888:
+		plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_ARGB8888;
+		break;
+	case DRM_FORMAT_XRGB2101010:
+	case DRM_FORMAT_ARGB2101010:
+		plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_ARGB2101010;
+		break;
+	case DRM_FORMAT_XBGR2101010:
+	case DRM_FORMAT_ABGR2101010:
+		plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_ABGR2101010;
+		break;
+	case DRM_FORMAT_XBGR8888:
+	case DRM_FORMAT_ABGR8888:
+		plane_info->format = SURFACE_PIXEL_FORMAT_GRPH_ABGR8888;
+		break;
+	case DRM_FORMAT_NV21:
+		plane_info->format = SURFACE_PIXEL_FORMAT_VIDEO_420_YCbCr;
+		break;
+	case DRM_FORMAT_NV12:
+		plane_info->format = SURFACE_PIXEL_FORMAT_VIDEO_420_YCrCb;
+		break;
+	default:
+		DRM_ERROR(
+			"Unsupported screen format %s\n",
+			drm_get_format_name(fb->format->format, &format_name));
 		return -EINVAL;
+	}
+
+	switch (plane_state->rotation & DRM_MODE_ROTATE_MASK) {
+	case DRM_MODE_ROTATE_0:
+		plane_info->rotation = ROTATION_ANGLE_0;
+		break;
+	case DRM_MODE_ROTATE_90:
+		plane_info->rotation = ROTATION_ANGLE_90;
+		break;
+	case DRM_MODE_ROTATE_180:
+		plane_info->rotation = ROTATION_ANGLE_180;
+		break;
+	case DRM_MODE_ROTATE_270:
+		plane_info->rotation = ROTATION_ANGLE_270;
+		break;
+	default:
+		plane_info->rotation = ROTATION_ANGLE_0;
+		break;
+	}
 
-	ret = fill_plane_attributes_from_fb(
-		crtc->dev->dev_private,
-		dc_plane_state,
-		amdgpu_fb);
+	plane_info->visible = true;
+	plane_info->stereo_format = PLANE_STEREO_FORMAT_NONE;
 
+	ret = fill_plane_color_attributes(plane_state, plane_info->format,
+					  &plane_info->color_space);
 	if (ret)
 		return ret;
 
+	ret = fill_plane_buffer_attributes(adev, afb, plane_info->format,
+					   plane_info->rotation, tiling_flags,
+					   &plane_info->tiling_info,
+					   &plane_info->plane_size,
+					   &plane_info->dcc, address);
+	if (ret)
+		return ret;
+
+	fill_blending_from_plane_state(
+		plane_state, &plane_info->per_pixel_alpha,
+		&plane_info->global_alpha, &plane_info->global_alpha_value);
+
+	return 0;
+}
+
+static int fill_dc_plane_attributes(struct amdgpu_device *adev,
+				    struct dc_plane_state *dc_plane_state,
+				    struct drm_plane_state *plane_state,
+				    struct drm_crtc_state *crtc_state)
+{
+	const struct amdgpu_framebuffer *amdgpu_fb =
+		to_amdgpu_framebuffer(plane_state->fb);
+	struct dc_scaling_info scaling_info;
+	struct dc_plane_info plane_info;
+	uint64_t tiling_flags;
+	int ret;
+
+	ret = fill_dc_scaling_info(plane_state, &scaling_info);
+	if (ret)
+		return ret;
+
+	dc_plane_state->src_rect = scaling_info.src_rect;
+	dc_plane_state->dst_rect = scaling_info.dst_rect;
+	dc_plane_state->clip_rect = scaling_info.clip_rect;
+	dc_plane_state->scaling_quality = scaling_info.scaling_quality;
+
+	ret = get_fb_info(amdgpu_fb, &tiling_flags);
+	if (ret)
+		return ret;
+
+	ret = fill_dc_plane_info_and_addr(adev, plane_state, tiling_flags,
+					  &plane_info,
+					  &dc_plane_state->address);
+	if (ret)
+		return ret;
+
+	dc_plane_state->format = plane_info.format;
+	dc_plane_state->color_space = plane_info.color_space;
+	dc_plane_state->format = plane_info.format;
+	dc_plane_state->plane_size = plane_info.plane_size;
+	dc_plane_state->rotation = plane_info.rotation;
+	dc_plane_state->horizontal_mirror = plane_info.horizontal_mirror;
+	dc_plane_state->stereo_format = plane_info.stereo_format;
+	dc_plane_state->tiling_info = plane_info.tiling_info;
+	dc_plane_state->visible = plane_info.visible;
+	dc_plane_state->per_pixel_alpha = plane_info.per_pixel_alpha;
+	dc_plane_state->global_alpha = plane_info.global_alpha;
+	dc_plane_state->global_alpha_value = plane_info.global_alpha_value;
+	dc_plane_state->dcc = plane_info.dcc;
+
 	/*
 	 * Always set input transfer function, since plane state is refreshed
 	 * every time.
@@ -3124,6 +3461,8 @@ dm_crtc_duplicate_state(struct drm_crtc *crtc)
 		dc_stream_retain(state->stream);
 	}
 
+	state->active_planes = cur->active_planes;
+	state->interrupts_enabled = cur->interrupts_enabled;
 	state->vrr_params = cur->vrr_params;
 	state->vrr_infopacket = cur->vrr_infopacket;
 	state->abm_level = cur->abm_level;
@@ -3136,12 +3475,41 @@ dm_crtc_duplicate_state(struct drm_crtc *crtc)
 	return &state->base;
 }
 
+static inline int dm_set_vupdate_irq(struct drm_crtc *crtc, bool enable)
+{
+	enum dc_irq_source irq_source;
+	struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc);
+	struct amdgpu_device *adev = crtc->dev->dev_private;
+	int rc;
+
+	irq_source = IRQ_TYPE_VUPDATE + acrtc->otg_inst;
+
+	rc = dc_interrupt_set(adev->dm.dc, irq_source, enable) ? 0 : -EBUSY;
+
+	DRM_DEBUG_DRIVER("crtc %d - vupdate irq %sabling: r=%d\n",
+			 acrtc->crtc_id, enable ? "en" : "dis", rc);
+	return rc;
+}
 
 static inline int dm_set_vblank(struct drm_crtc *crtc, bool enable)
 {
 	enum dc_irq_source irq_source;
 	struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc);
 	struct amdgpu_device *adev = crtc->dev->dev_private;
+	struct dm_crtc_state *acrtc_state = to_dm_crtc_state(crtc->state);
+	int rc = 0;
+
+	if (enable) {
+		/* vblank irq on -> Only need vupdate irq in vrr mode */
+		if (amdgpu_dm_vrr_active(acrtc_state))
+			rc = dm_set_vupdate_irq(crtc, true);
+	} else {
+		/* vblank irq off -> vupdate irq off */
+		rc = dm_set_vupdate_irq(crtc, false);
+	}
+
+	if (rc)
+		return rc;
 
 	irq_source = IRQ_TYPE_VBLANK + acrtc->otg_inst;
 	return dc_interrupt_set(adev->dm.dc, irq_source, enable) ? 0 : -EBUSY;
@@ -3519,6 +3887,76 @@ static void dm_crtc_helper_disable(struct drm_crtc *crtc)
 {
 }
 
+static bool does_crtc_have_active_cursor(struct drm_crtc_state *new_crtc_state)
+{
+	struct drm_device *dev = new_crtc_state->crtc->dev;
+	struct drm_plane *plane;
+
+	drm_for_each_plane_mask(plane, dev, new_crtc_state->plane_mask) {
+		if (plane->type == DRM_PLANE_TYPE_CURSOR)
+			return true;
+	}
+
+	return false;
+}
+
+static int count_crtc_active_planes(struct drm_crtc_state *new_crtc_state)
+{
+	struct drm_atomic_state *state = new_crtc_state->state;
+	struct drm_plane *plane;
+	int num_active = 0;
+
+	drm_for_each_plane_mask(plane, state->dev, new_crtc_state->plane_mask) {
+		struct drm_plane_state *new_plane_state;
+
+		/* Cursor planes are "fake". */
+		if (plane->type == DRM_PLANE_TYPE_CURSOR)
+			continue;
+
+		new_plane_state = drm_atomic_get_new_plane_state(state, plane);
+
+		if (!new_plane_state) {
+			/*
+			 * The plane is enable on the CRTC and hasn't changed
+			 * state. This means that it previously passed
+			 * validation and is therefore enabled.
+			 */
+			num_active += 1;
+			continue;
+		}
+
+		/* We need a framebuffer to be considered enabled. */
+		num_active += (new_plane_state->fb != NULL);
+	}
+
+	return num_active;
+}
+
+/*
+ * Sets whether interrupts should be enabled on a specific CRTC.
+ * We require that the stream be enabled and that there exist active
+ * DC planes on the stream.
+ */
+static void
+dm_update_crtc_interrupt_state(struct drm_crtc *crtc,
+			       struct drm_crtc_state *new_crtc_state)
+{
+	struct dm_crtc_state *dm_new_crtc_state =
+		to_dm_crtc_state(new_crtc_state);
+
+	dm_new_crtc_state->active_planes = 0;
+	dm_new_crtc_state->interrupts_enabled = false;
+
+	if (!dm_new_crtc_state->stream)
+		return;
+
+	dm_new_crtc_state->active_planes =
+		count_crtc_active_planes(new_crtc_state);
+
+	dm_new_crtc_state->interrupts_enabled =
+		dm_new_crtc_state->active_planes > 0;
+}
+
 static int dm_crtc_helper_atomic_check(struct drm_crtc *crtc,
 				       struct drm_crtc_state *state)
 {
@@ -3527,6 +3965,14 @@ static int dm_crtc_helper_atomic_check(struct drm_crtc *crtc,
 	struct dm_crtc_state *dm_crtc_state = to_dm_crtc_state(state);
 	int ret = -EINVAL;
 
+	/*
+	 * Update interrupt state for the CRTC. This needs to happen whenever
+	 * the CRTC has changed or whenever any of its planes have changed.
+	 * Atomic check satisfies both of these requirements since the CRTC
+	 * is added to the state by DRM during drm_atomic_helper_check_planes.
+	 */
+	dm_update_crtc_interrupt_state(crtc, state);
+
 	if (unlikely(!dm_crtc_state->stream &&
 		     modeset_required(state, NULL, dm_crtc_state->stream))) {
 		WARN_ON(1);
@@ -3537,6 +3983,15 @@ static int dm_crtc_helper_atomic_check(struct drm_crtc *crtc,
 	if (!dm_crtc_state->stream)
 		return 0;
 
+	/*
+	 * We want at least one hardware plane enabled to use
+	 * the stream with a cursor enabled.
+	 */
+	if (state->enable && state->active &&
+	    does_crtc_have_active_cursor(state) &&
+	    dm_crtc_state->active_planes == 0)
+		return -EINVAL;
+
 	if (dc_validate_stream(dc, dm_crtc_state->stream) == DC_OK)
 		return 0;
 
@@ -3583,11 +4038,8 @@ static void dm_drm_plane_reset(struct drm_plane *plane)
 	amdgpu_state = kzalloc(sizeof(*amdgpu_state), GFP_KERNEL);
 	WARN_ON(amdgpu_state == NULL);
 
-	if (amdgpu_state) {
-		plane->state = &amdgpu_state->base;
-		plane->state->plane = plane;
-		plane->state->rotation = DRM_MODE_ROTATE_0;
-	}
+	if (amdgpu_state)
+		__drm_atomic_helper_plane_reset(plane, &amdgpu_state->base);
 }
 
 static struct drm_plane_state *
@@ -3637,10 +4089,8 @@ static int dm_plane_helper_prepare_fb(struct drm_plane *plane,
 	struct drm_gem_object *obj;
 	struct amdgpu_device *adev;
 	struct amdgpu_bo *rbo;
-	uint64_t chroma_addr = 0;
 	struct dm_plane_state *dm_plane_state_new, *dm_plane_state_old;
-	uint64_t tiling_flags, dcc_address;
-	unsigned int awidth;
+	uint64_t tiling_flags;
 	uint32_t domain;
 	int r;
 
@@ -3693,29 +4143,11 @@ static int dm_plane_helper_prepare_fb(struct drm_plane *plane,
 			dm_plane_state_old->dc_state != dm_plane_state_new->dc_state) {
 		struct dc_plane_state *plane_state = dm_plane_state_new->dc_state;
 
-		if (plane_state->format < SURFACE_PIXEL_FORMAT_VIDEO_BEGIN) {
-			plane_state->address.grph.addr.low_part = lower_32_bits(afb->address);
-			plane_state->address.grph.addr.high_part = upper_32_bits(afb->address);
-
-			dcc_address =
-				get_dcc_address(afb->address, tiling_flags);
-			plane_state->address.grph.meta_addr.low_part =
-				lower_32_bits(dcc_address);
-			plane_state->address.grph.meta_addr.high_part =
-				upper_32_bits(dcc_address);
-		} else {
-			awidth = ALIGN(new_state->fb->width, 64);
-			plane_state->address.type = PLN_ADDR_TYPE_VIDEO_PROGRESSIVE;
-			plane_state->address.video_progressive.luma_addr.low_part
-							= lower_32_bits(afb->address);
-			plane_state->address.video_progressive.luma_addr.high_part
-							= upper_32_bits(afb->address);
-			chroma_addr = afb->address + (u64)awidth * new_state->fb->height;
-			plane_state->address.video_progressive.chroma_addr.low_part
-							= lower_32_bits(chroma_addr);
-			plane_state->address.video_progressive.chroma_addr.high_part
-							= upper_32_bits(chroma_addr);
-		}
+		fill_plane_buffer_attributes(
+			adev, afb, plane_state->format, plane_state->rotation,
+			tiling_flags, &plane_state->tiling_info,
+			&plane_state->plane_size, &plane_state->dcc,
+			&plane_state->address);
 	}
 
 	return 0;
@@ -3747,13 +4179,18 @@ static int dm_plane_atomic_check(struct drm_plane *plane,
 {
 	struct amdgpu_device *adev = plane->dev->dev_private;
 	struct dc *dc = adev->dm.dc;
-	struct dm_plane_state *dm_plane_state = to_dm_plane_state(state);
+	struct dm_plane_state *dm_plane_state;
+	struct dc_scaling_info scaling_info;
+	int ret;
+
+	dm_plane_state = to_dm_plane_state(state);
 
 	if (!dm_plane_state->dc_state)
 		return 0;
 
-	if (!fill_rects_from_plane_state(state, dm_plane_state->dc_state))
-		return -EINVAL;
+	ret = fill_dc_scaling_info(state, &scaling_info);
+	if (ret)
+		return ret;
 
 	if (dc_validate_plane(dc, dm_plane_state->dc_state) == DC_OK)
 		return 0;
@@ -3826,64 +4263,115 @@ static const uint32_t rgb_formats[] = {
 	DRM_FORMAT_ABGR2101010,
 	DRM_FORMAT_XBGR8888,
 	DRM_FORMAT_ABGR8888,
+	DRM_FORMAT_RGB565,
 };
 
-static const uint32_t yuv_formats[] = {
-	DRM_FORMAT_NV12,
-	DRM_FORMAT_NV21,
+static const uint32_t overlay_formats[] = {
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_RGBA8888,
+	DRM_FORMAT_XBGR8888,
+	DRM_FORMAT_ABGR8888,
+	DRM_FORMAT_RGB565
 };
 
 static const u32 cursor_formats[] = {
 	DRM_FORMAT_ARGB8888
 };
 
-static int amdgpu_dm_plane_init(struct amdgpu_display_manager *dm,
-				struct drm_plane *plane,
-				unsigned long possible_crtcs)
+static int get_plane_formats(const struct drm_plane *plane,
+			     const struct dc_plane_cap *plane_cap,
+			     uint32_t *formats, int max_formats)
 {
-	int res = -EPERM;
+	int i, num_formats = 0;
+
+	/*
+	 * TODO: Query support for each group of formats directly from
+	 * DC plane caps. This will require adding more formats to the
+	 * caps list.
+	 */
 
 	switch (plane->type) {
 	case DRM_PLANE_TYPE_PRIMARY:
-		res = drm_universal_plane_init(
-				dm->adev->ddev,
-				plane,
-				possible_crtcs,
-				&dm_plane_funcs,
-				rgb_formats,
-				ARRAY_SIZE(rgb_formats),
-				NULL, plane->type, NULL);
+		for (i = 0; i < ARRAY_SIZE(rgb_formats); ++i) {
+			if (num_formats >= max_formats)
+				break;
+
+			formats[num_formats++] = rgb_formats[i];
+		}
+
+		if (plane_cap && plane_cap->pixel_format_support.nv12)
+			formats[num_formats++] = DRM_FORMAT_NV12;
 		break;
+
 	case DRM_PLANE_TYPE_OVERLAY:
-		res = drm_universal_plane_init(
-				dm->adev->ddev,
-				plane,
-				possible_crtcs,
-				&dm_plane_funcs,
-				yuv_formats,
-				ARRAY_SIZE(yuv_formats),
-				NULL, plane->type, NULL);
+		for (i = 0; i < ARRAY_SIZE(overlay_formats); ++i) {
+			if (num_formats >= max_formats)
+				break;
+
+			formats[num_formats++] = overlay_formats[i];
+		}
 		break;
+
 	case DRM_PLANE_TYPE_CURSOR:
-		res = drm_universal_plane_init(
-				dm->adev->ddev,
-				plane,
-				possible_crtcs,
-				&dm_plane_funcs,
-				cursor_formats,
-				ARRAY_SIZE(cursor_formats),
-				NULL, plane->type, NULL);
+		for (i = 0; i < ARRAY_SIZE(cursor_formats); ++i) {
+			if (num_formats >= max_formats)
+				break;
+
+			formats[num_formats++] = cursor_formats[i];
+		}
 		break;
 	}
 
+	return num_formats;
+}
+
+static int amdgpu_dm_plane_init(struct amdgpu_display_manager *dm,
+				struct drm_plane *plane,
+				unsigned long possible_crtcs,
+				const struct dc_plane_cap *plane_cap)
+{
+	uint32_t formats[32];
+	int num_formats;
+	int res = -EPERM;
+
+	num_formats = get_plane_formats(plane, plane_cap, formats,
+					ARRAY_SIZE(formats));
+
+	res = drm_universal_plane_init(dm->adev->ddev, plane, possible_crtcs,
+				       &dm_plane_funcs, formats, num_formats,
+				       NULL, plane->type, NULL);
+	if (res)
+		return res;
+
+	if (plane->type == DRM_PLANE_TYPE_OVERLAY &&
+	    plane_cap && plane_cap->per_pixel_alpha) {
+		unsigned int blend_caps = BIT(DRM_MODE_BLEND_PIXEL_NONE) |
+					  BIT(DRM_MODE_BLEND_PREMULTI);
+
+		drm_plane_create_alpha_property(plane);
+		drm_plane_create_blend_mode_property(plane, blend_caps);
+	}
+
+	if (plane->type == DRM_PLANE_TYPE_PRIMARY &&
+	    plane_cap && plane_cap->pixel_format_support.nv12) {
+		/* This only affects YUV formats. */
+		drm_plane_create_color_properties(
+			plane,
+			BIT(DRM_COLOR_YCBCR_BT601) |
+			BIT(DRM_COLOR_YCBCR_BT709),
+			BIT(DRM_COLOR_YCBCR_LIMITED_RANGE) |
+			BIT(DRM_COLOR_YCBCR_FULL_RANGE),
+			DRM_COLOR_YCBCR_BT709, DRM_COLOR_YCBCR_LIMITED_RANGE);
+	}
+
 	drm_plane_helper_add(plane, &dm_plane_helper_funcs);
 
 	/* Create (reset) the plane state */
 	if (plane->funcs->reset)
 		plane->funcs->reset(plane);
 
-
-	return res;
+	return 0;
 }
 
 static int amdgpu_dm_crtc_init(struct amdgpu_display_manager *dm,
@@ -3900,7 +4388,7 @@ static int amdgpu_dm_crtc_init(struct amdgpu_display_manager *dm,
 		goto fail;
 
 	cursor_plane->type = DRM_PLANE_TYPE_CURSOR;
-	res = amdgpu_dm_plane_init(dm, cursor_plane, 0);
+	res = amdgpu_dm_plane_init(dm, cursor_plane, 0, NULL);
 
 	acrtc = kzalloc(sizeof(struct amdgpu_crtc), GFP_KERNEL);
 	if (!acrtc)
@@ -4334,6 +4822,8 @@ static int amdgpu_dm_connector_init(struct amdgpu_display_manager *dm,
 		DRM_ERROR("Failed to create debugfs for connector");
 		goto out_free;
 	}
+	aconnector->debugfs_dpcd_address = 0;
+	aconnector->debugfs_dpcd_size = 0;
 #endif
 
 	if (connector_type == DRM_MODE_CONNECTOR_DisplayPort
@@ -4473,9 +4963,13 @@ static int get_cursor_position(struct drm_plane *plane, struct drm_crtc *crtc,
 
 	x = plane->state->crtc_x;
 	y = plane->state->crtc_y;
-	/* avivo cursor are offset into the total surface */
-	x += crtc->primary->state->src_x >> 16;
-	y += crtc->primary->state->src_y >> 16;
+
+	if (crtc->primary->state) {
+		/* avivo cursor are offset into the total surface */
+		x += crtc->primary->state->src_x >> 16;
+		y += crtc->primary->state->src_y >> 16;
+	}
+
 	if (x < 0) {
 		xorigin = min(-x, amdgpu_crtc->max_cursor_width - 1);
 		x = 0;
@@ -4582,9 +5076,10 @@ static void update_freesync_state_on_stream(
 	struct dc_plane_state *surface,
 	u32 flip_timestamp_in_us)
 {
-	struct mod_vrr_params vrr_params = new_crtc_state->vrr_params;
+	struct mod_vrr_params vrr_params;
 	struct dc_info_packet vrr_infopacket = {0};
-	struct mod_freesync_config config = new_crtc_state->freesync_config;
+	struct amdgpu_device *adev = dm->adev;
+	unsigned long flags;
 
 	if (!new_stream)
 		return;
@@ -4597,19 +5092,8 @@ static void update_freesync_state_on_stream(
 	if (!new_stream->timing.h_total || !new_stream->timing.v_total)
 		return;
 
-	if (new_crtc_state->vrr_supported &&
-	    config.min_refresh_in_uhz &&
-	    config.max_refresh_in_uhz) {
-		config.state = new_crtc_state->base.vrr_enabled ?
-			VRR_STATE_ACTIVE_VARIABLE :
-			VRR_STATE_INACTIVE;
-	} else {
-		config.state = VRR_STATE_UNSUPPORTED;
-	}
-
-	mod_freesync_build_vrr_params(dm->freesync_module,
-				      new_stream,
-				      &config, &vrr_params);
+	spin_lock_irqsave(&adev->ddev->event_lock, flags);
+	vrr_params = new_crtc_state->vrr_params;
 
 	if (surface) {
 		mod_freesync_handle_preflip(
@@ -4618,6 +5102,12 @@ static void update_freesync_state_on_stream(
 			new_stream,
 			flip_timestamp_in_us,
 			&vrr_params);
+
+		if (adev->family < AMDGPU_FAMILY_AI &&
+		    amdgpu_dm_vrr_active(new_crtc_state)) {
+			mod_freesync_handle_v_update(dm->freesync_module,
+						     new_stream, &vrr_params);
+		}
 	}
 
 	mod_freesync_build_vrr_infopacket(
@@ -4649,6 +5139,100 @@ static void update_freesync_state_on_stream(
 			      new_crtc_state->base.crtc->base.id,
 			      (int)new_crtc_state->base.vrr_enabled,
 			      (int)vrr_params.state);
+
+	spin_unlock_irqrestore(&adev->ddev->event_lock, flags);
+}
+
+static void pre_update_freesync_state_on_stream(
+	struct amdgpu_display_manager *dm,
+	struct dm_crtc_state *new_crtc_state)
+{
+	struct dc_stream_state *new_stream = new_crtc_state->stream;
+	struct mod_vrr_params vrr_params;
+	struct mod_freesync_config config = new_crtc_state->freesync_config;
+	struct amdgpu_device *adev = dm->adev;
+	unsigned long flags;
+
+	if (!new_stream)
+		return;
+
+	/*
+	 * TODO: Determine why min/max totals and vrefresh can be 0 here.
+	 * For now it's sufficient to just guard against these conditions.
+	 */
+	if (!new_stream->timing.h_total || !new_stream->timing.v_total)
+		return;
+
+	spin_lock_irqsave(&adev->ddev->event_lock, flags);
+	vrr_params = new_crtc_state->vrr_params;
+
+	if (new_crtc_state->vrr_supported &&
+	    config.min_refresh_in_uhz &&
+	    config.max_refresh_in_uhz) {
+		config.state = new_crtc_state->base.vrr_enabled ?
+			VRR_STATE_ACTIVE_VARIABLE :
+			VRR_STATE_INACTIVE;
+	} else {
+		config.state = VRR_STATE_UNSUPPORTED;
+	}
+
+	mod_freesync_build_vrr_params(dm->freesync_module,
+				      new_stream,
+				      &config, &vrr_params);
+
+	new_crtc_state->freesync_timing_changed |=
+		(memcmp(&new_crtc_state->vrr_params.adjust,
+			&vrr_params.adjust,
+			sizeof(vrr_params.adjust)) != 0);
+
+	new_crtc_state->vrr_params = vrr_params;
+	spin_unlock_irqrestore(&adev->ddev->event_lock, flags);
+}
+
+static void amdgpu_dm_handle_vrr_transition(struct dm_crtc_state *old_state,
+					    struct dm_crtc_state *new_state)
+{
+	bool old_vrr_active = amdgpu_dm_vrr_active(old_state);
+	bool new_vrr_active = amdgpu_dm_vrr_active(new_state);
+
+	if (!old_vrr_active && new_vrr_active) {
+		/* Transition VRR inactive -> active:
+		 * While VRR is active, we must not disable vblank irq, as a
+		 * reenable after disable would compute bogus vblank/pflip
+		 * timestamps if it likely happened inside display front-porch.
+		 *
+		 * We also need vupdate irq for the actual core vblank handling
+		 * at end of vblank.
+		 */
+		dm_set_vupdate_irq(new_state->base.crtc, true);
+		drm_crtc_vblank_get(new_state->base.crtc);
+		DRM_DEBUG_DRIVER("%s: crtc=%u VRR off->on: Get vblank ref\n",
+				 __func__, new_state->base.crtc->base.id);
+	} else if (old_vrr_active && !new_vrr_active) {
+		/* Transition VRR active -> inactive:
+		 * Allow vblank irq disable again for fixed refresh rate.
+		 */
+		dm_set_vupdate_irq(new_state->base.crtc, false);
+		drm_crtc_vblank_put(new_state->base.crtc);
+		DRM_DEBUG_DRIVER("%s: crtc=%u VRR on->off: Drop vblank ref\n",
+				 __func__, new_state->base.crtc->base.id);
+	}
+}
+
+static void amdgpu_dm_commit_cursors(struct drm_atomic_state *state)
+{
+	struct drm_plane *plane;
+	struct drm_plane_state *old_plane_state, *new_plane_state;
+	int i;
+
+	/*
+	 * TODO: Make this per-stream so we don't issue redundant updates for
+	 * commits with multiple streams.
+	 */
+	for_each_oldnew_plane_in_state(state, plane, old_plane_state,
+				       new_plane_state, i)
+		if (plane->type == DRM_PLANE_TYPE_CURSOR)
+			handle_cursor_update(plane, old_plane_state);
 }
 
 static void amdgpu_dm_commit_planes(struct drm_atomic_state *state,
@@ -4656,7 +5240,7 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state,
 				    struct drm_device *dev,
 				    struct amdgpu_display_manager *dm,
 				    struct drm_crtc *pcrtc,
-				    bool *wait_for_vblank)
+				    bool wait_for_vblank)
 {
 	uint32_t i, r;
 	uint64_t timestamp_ns;
@@ -4668,42 +5252,42 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state,
 	struct dm_crtc_state *acrtc_state = to_dm_crtc_state(new_pcrtc_state);
 	struct dm_crtc_state *dm_old_crtc_state =
 			to_dm_crtc_state(drm_atomic_get_old_crtc_state(state, pcrtc));
-	int flip_count = 0, planes_count = 0, vpos, hpos;
+	int planes_count = 0, vpos, hpos;
 	unsigned long flags;
 	struct amdgpu_bo *abo;
-	uint64_t tiling_flags, dcc_address;
-	uint32_t target, target_vblank;
-	uint64_t last_flip_vblank;
-	bool vrr_active = acrtc_state->freesync_config.state == VRR_STATE_ACTIVE_VARIABLE;
-
-	struct {
-		struct dc_surface_update surface_updates[MAX_SURFACES];
-		struct dc_flip_addrs flip_addrs[MAX_SURFACES];
-		struct dc_stream_update stream_update;
-	} *flip;
-
+	uint64_t tiling_flags;
+	uint32_t target_vblank, last_flip_vblank;
+	bool vrr_active = amdgpu_dm_vrr_active(acrtc_state);
+	bool pflip_present = false;
 	struct {
 		struct dc_surface_update surface_updates[MAX_SURFACES];
 		struct dc_plane_info plane_infos[MAX_SURFACES];
 		struct dc_scaling_info scaling_infos[MAX_SURFACES];
+		struct dc_flip_addrs flip_addrs[MAX_SURFACES];
 		struct dc_stream_update stream_update;
-	} *full;
+	} *bundle;
 
-	flip = kzalloc(sizeof(*flip), GFP_KERNEL);
-	full = kzalloc(sizeof(*full), GFP_KERNEL);
+	bundle = kzalloc(sizeof(*bundle), GFP_KERNEL);
 
-	if (!flip || !full) {
-		dm_error("Failed to allocate update bundles\n");
+	if (!bundle) {
+		dm_error("Failed to allocate update bundle\n");
 		goto cleanup;
 	}
 
+	/*
+	 * Disable the cursor first if we're disabling all the planes.
+	 * It'll remain on the screen after the planes are re-enabled
+	 * if we don't.
+	 */
+	if (acrtc_state->active_planes == 0)
+		amdgpu_dm_commit_cursors(state);
+
 	/* update planes when needed */
 	for_each_oldnew_plane_in_state(state, plane, old_plane_state, new_plane_state, i) {
 		struct drm_crtc *crtc = new_plane_state->crtc;
 		struct drm_crtc_state *new_crtc_state;
 		struct drm_framebuffer *fb = new_plane_state->fb;
-		struct amdgpu_framebuffer *afb = to_amdgpu_framebuffer(fb);
-		bool pflip_needed;
+		bool plane_needs_flip;
 		struct dc_plane_state *dc_plane;
 		struct dm_plane_state *dm_new_plane_state = to_dm_plane_state(new_plane_state);
 
@@ -4718,122 +5302,96 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state,
 		if (!new_crtc_state->active)
 			continue;
 
-		pflip_needed = old_plane_state->fb &&
-			old_plane_state->fb != new_plane_state->fb;
-
 		dc_plane = dm_new_plane_state->dc_state;
 
-		if (pflip_needed) {
-			/*
-			 * Assume even ONE crtc with immediate flip means
-			 * entire can't wait for VBLANK
-			 * TODO Check if it's correct
-			 */
-			if (new_pcrtc_state->pageflip_flags & DRM_MODE_PAGE_FLIP_ASYNC)
-				*wait_for_vblank = false;
+		bundle->surface_updates[planes_count].surface = dc_plane;
+		if (new_pcrtc_state->color_mgmt_changed) {
+			bundle->surface_updates[planes_count].gamma = dc_plane->gamma_correction;
+			bundle->surface_updates[planes_count].in_transfer_func = dc_plane->in_transfer_func;
+		}
 
-			/*
-			 * TODO This might fail and hence better not used, wait
-			 * explicitly on fences instead
-			 * and in general should be called for
-			 * blocking commit to as per framework helpers
-			 */
-			abo = gem_to_amdgpu_bo(fb->obj[0]);
-			r = amdgpu_bo_reserve(abo, true);
-			if (unlikely(r != 0))
-				DRM_ERROR("failed to reserve buffer before flip\n");
+		fill_dc_scaling_info(new_plane_state,
+				     &bundle->scaling_infos[planes_count]);
 
-			/*
-			 * Wait for all fences on this FB. Do limited wait to avoid
-			 * deadlock during GPU reset when this fence will not signal
-			 * but we hold reservation lock for the BO.
-			 */
-			r = reservation_object_wait_timeout_rcu(abo->tbo.resv,
-								true, false,
-								msecs_to_jiffies(5000));
-			if (unlikely(r == 0))
-				DRM_ERROR("Waiting for fences timed out.");
+		bundle->surface_updates[planes_count].scaling_info =
+			&bundle->scaling_infos[planes_count];
 
+		plane_needs_flip = old_plane_state->fb && new_plane_state->fb;
 
+		pflip_present = pflip_present || plane_needs_flip;
 
-			amdgpu_bo_get_tiling_flags(abo, &tiling_flags);
+		if (!plane_needs_flip) {
+			planes_count += 1;
+			continue;
+		}
 
-			amdgpu_bo_unreserve(abo);
+		abo = gem_to_amdgpu_bo(fb->obj[0]);
 
-			flip->flip_addrs[flip_count].address.grph.addr.low_part = lower_32_bits(afb->address);
-			flip->flip_addrs[flip_count].address.grph.addr.high_part = upper_32_bits(afb->address);
+		/*
+		 * Wait for all fences on this FB. Do limited wait to avoid
+		 * deadlock during GPU reset when this fence will not signal
+		 * but we hold reservation lock for the BO.
+		 */
+		r = reservation_object_wait_timeout_rcu(abo->tbo.resv, true,
+							false,
+							msecs_to_jiffies(5000));
+		if (unlikely(r <= 0))
+			DRM_ERROR("Waiting for fences timed out or interrupted!");
 
-			dcc_address = get_dcc_address(afb->address, tiling_flags);
-			flip->flip_addrs[flip_count].address.grph.meta_addr.low_part = lower_32_bits(dcc_address);
-			flip->flip_addrs[flip_count].address.grph.meta_addr.high_part = upper_32_bits(dcc_address);
+		/*
+		 * TODO This might fail and hence better not used, wait
+		 * explicitly on fences instead
+		 * and in general should be called for
+		 * blocking commit to as per framework helpers
+		 */
+		r = amdgpu_bo_reserve(abo, true);
+		if (unlikely(r != 0))
+			DRM_ERROR("failed to reserve buffer before flip\n");
 
-			flip->flip_addrs[flip_count].flip_immediate =
-					(crtc->state->pageflip_flags & DRM_MODE_PAGE_FLIP_ASYNC) != 0;
+		amdgpu_bo_get_tiling_flags(abo, &tiling_flags);
 
-			timestamp_ns = ktime_get_ns();
-			flip->flip_addrs[flip_count].flip_timestamp_in_us = div_u64(timestamp_ns, 1000);
-			flip->surface_updates[flip_count].flip_addr = &flip->flip_addrs[flip_count];
-			flip->surface_updates[flip_count].surface = dc_plane;
+		amdgpu_bo_unreserve(abo);
 
-			if (!flip->surface_updates[flip_count].surface) {
-				DRM_ERROR("No surface for CRTC: id=%d\n",
-						acrtc_attach->crtc_id);
-				continue;
-			}
+		fill_dc_plane_info_and_addr(
+			dm->adev, new_plane_state, tiling_flags,
+			&bundle->plane_infos[planes_count],
+			&bundle->flip_addrs[planes_count].address);
 
-			if (plane == pcrtc->primary)
-				update_freesync_state_on_stream(
-					dm,
-					acrtc_state,
-					acrtc_state->stream,
-					dc_plane,
-					flip->flip_addrs[flip_count].flip_timestamp_in_us);
+		bundle->surface_updates[planes_count].plane_info =
+			&bundle->plane_infos[planes_count];
 
-			DRM_DEBUG_DRIVER("%s Flipping to hi: 0x%x, low: 0x%x\n",
-					 __func__,
-					 flip->flip_addrs[flip_count].address.grph.addr.high_part,
-					 flip->flip_addrs[flip_count].address.grph.addr.low_part);
+		bundle->flip_addrs[planes_count].flip_immediate =
+				(crtc->state->pageflip_flags & DRM_MODE_PAGE_FLIP_ASYNC) != 0;
 
-			flip_count += 1;
-		}
+		timestamp_ns = ktime_get_ns();
+		bundle->flip_addrs[planes_count].flip_timestamp_in_us = div_u64(timestamp_ns, 1000);
+		bundle->surface_updates[planes_count].flip_addr = &bundle->flip_addrs[planes_count];
+		bundle->surface_updates[planes_count].surface = dc_plane;
 
-		full->surface_updates[planes_count].surface = dc_plane;
-		if (new_pcrtc_state->color_mgmt_changed) {
-			full->surface_updates[planes_count].gamma = dc_plane->gamma_correction;
-			full->surface_updates[planes_count].in_transfer_func = dc_plane->in_transfer_func;
+		if (!bundle->surface_updates[planes_count].surface) {
+			DRM_ERROR("No surface for CRTC: id=%d\n",
+					acrtc_attach->crtc_id);
+			continue;
 		}
 
+		if (plane == pcrtc->primary)
+			update_freesync_state_on_stream(
+				dm,
+				acrtc_state,
+				acrtc_state->stream,
+				dc_plane,
+				bundle->flip_addrs[planes_count].flip_timestamp_in_us);
 
-		full->scaling_infos[planes_count].scaling_quality = dc_plane->scaling_quality;
-		full->scaling_infos[planes_count].src_rect = dc_plane->src_rect;
-		full->scaling_infos[planes_count].dst_rect = dc_plane->dst_rect;
-		full->scaling_infos[planes_count].clip_rect = dc_plane->clip_rect;
-		full->surface_updates[planes_count].scaling_info = &full->scaling_infos[planes_count];
-
-
-		full->plane_infos[planes_count].color_space = dc_plane->color_space;
-		full->plane_infos[planes_count].format = dc_plane->format;
-		full->plane_infos[planes_count].plane_size = dc_plane->plane_size;
-		full->plane_infos[planes_count].rotation = dc_plane->rotation;
-		full->plane_infos[planes_count].horizontal_mirror = dc_plane->horizontal_mirror;
-		full->plane_infos[planes_count].stereo_format = dc_plane->stereo_format;
-		full->plane_infos[planes_count].tiling_info = dc_plane->tiling_info;
-		full->plane_infos[planes_count].visible = dc_plane->visible;
-		full->plane_infos[planes_count].per_pixel_alpha = dc_plane->per_pixel_alpha;
-		full->plane_infos[planes_count].dcc = dc_plane->dcc;
-		full->surface_updates[planes_count].plane_info = &full->plane_infos[planes_count];
+		DRM_DEBUG_DRIVER("%s Flipping to hi: 0x%x, low: 0x%x\n",
+				 __func__,
+				 bundle->flip_addrs[planes_count].address.grph.addr.high_part,
+				 bundle->flip_addrs[planes_count].address.grph.addr.low_part);
 
 		planes_count += 1;
 
 	}
 
-	/*
-	 * TODO: For proper atomic behaviour, we should be calling into DC once with
-	 * all the changes.  However, DC refuses to do pageflips and non-pageflip
-	 * changes in the same call.  Change DC to respect atomic behaviour,
-	 * hopefully eliminating dc_*_update structs in their entirety.
-	 */
-	if (flip_count) {
+	if (pflip_present) {
 		if (!vrr_active) {
 			/* Use old throttling in non-vrr fixed refresh rate mode
 			 * to keep flip scheduling based on target vblank counts
@@ -4841,7 +5399,7 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state,
 			 * clients using the GLX_OML_sync_control extension or
 			 * DRI3/Present extension with defined target_msc.
 			 */
-			last_flip_vblank = drm_crtc_vblank_count(pcrtc);
+			last_flip_vblank = amdgpu_get_vblank_counter_kms(dm->ddev, acrtc_attach->crtc_id);
 		}
 		else {
 			/* For variable refresh rate mode only:
@@ -4857,11 +5415,7 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state,
 			spin_unlock_irqrestore(&pcrtc->dev->event_lock, flags);
 		}
 
-		target = (uint32_t)last_flip_vblank + *wait_for_vblank;
-
-		/* Prepare wait for target vblank early - before the fence-waits */
-		target_vblank = target - (uint32_t)drm_crtc_vblank_count(pcrtc) +
-				amdgpu_get_vblank_counter_kms(pcrtc->dev, acrtc_attach->crtc_id);
+		target_vblank = last_flip_vblank + wait_for_vblank;
 
 		/*
 		 * Wait until we're out of the vertical blank period before the one
@@ -4892,54 +5446,102 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state,
 		if (acrtc_state->stream) {
 
 			if (acrtc_state->freesync_timing_changed)
-				flip->stream_update.adjust =
+				bundle->stream_update.adjust =
 					&acrtc_state->stream->adjust;
 
 			if (acrtc_state->freesync_vrr_info_changed)
-				flip->stream_update.vrr_infopacket =
+				bundle->stream_update.vrr_infopacket =
 					&acrtc_state->stream->vrr_infopacket;
 		}
-
-		mutex_lock(&dm->dc_lock);
-		dc_commit_updates_for_stream(dm->dc,
-						     flip->surface_updates,
-						     flip_count,
-						     acrtc_state->stream,
-						     &flip->stream_update,
-						     dc_state);
-		mutex_unlock(&dm->dc_lock);
 	}
 
-	if (planes_count) {
+	/* Update the planes if changed or disable if we don't have any. */
+	if (planes_count || acrtc_state->active_planes == 0) {
 		if (new_pcrtc_state->mode_changed) {
-			full->stream_update.src = acrtc_state->stream->src;
-			full->stream_update.dst = acrtc_state->stream->dst;
+			bundle->stream_update.src = acrtc_state->stream->src;
+			bundle->stream_update.dst = acrtc_state->stream->dst;
 		}
 
 		if (new_pcrtc_state->color_mgmt_changed)
-			full->stream_update.out_transfer_func = acrtc_state->stream->out_transfer_func;
+			bundle->stream_update.out_transfer_func = acrtc_state->stream->out_transfer_func;
 
 		acrtc_state->stream->abm_level = acrtc_state->abm_level;
 		if (acrtc_state->abm_level != dm_old_crtc_state->abm_level)
-			full->stream_update.abm_level = &acrtc_state->abm_level;
+			bundle->stream_update.abm_level = &acrtc_state->abm_level;
 
 		mutex_lock(&dm->dc_lock);
 		dc_commit_updates_for_stream(dm->dc,
-						     full->surface_updates,
+						     bundle->surface_updates,
 						     planes_count,
 						     acrtc_state->stream,
-						     &full->stream_update,
+						     &bundle->stream_update,
 						     dc_state);
 		mutex_unlock(&dm->dc_lock);
 	}
 
-	for_each_oldnew_plane_in_state(state, plane, old_plane_state, new_plane_state, i)
-		if (plane->type == DRM_PLANE_TYPE_CURSOR)
-			handle_cursor_update(plane, old_plane_state);
+	/*
+	 * Update cursor state *after* programming all the planes.
+	 * This avoids redundant programming in the case where we're going
+	 * to be disabling a single plane - those pipes are being disabled.
+	 */
+	if (acrtc_state->active_planes)
+		amdgpu_dm_commit_cursors(state);
 
 cleanup:
-	kfree(flip);
-	kfree(full);
+	kfree(bundle);
+}
+
+/*
+ * Enable interrupts on CRTCs that are newly active, undergone
+ * a modeset, or have active planes again.
+ *
+ * Done in two passes, based on the for_modeset flag:
+ * Pass 1: For CRTCs going through modeset
+ * Pass 2: For CRTCs going from 0 to n active planes
+ *
+ * Interrupts can only be enabled after the planes are programmed,
+ * so this requires a two-pass approach since we don't want to
+ * just defer the interrupts until after commit planes every time.
+ */
+static void amdgpu_dm_enable_crtc_interrupts(struct drm_device *dev,
+					     struct drm_atomic_state *state,
+					     bool for_modeset)
+{
+	struct amdgpu_device *adev = dev->dev_private;
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
+	int i;
+
+	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state,
+				      new_crtc_state, i) {
+		struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc);
+		struct dm_crtc_state *dm_new_crtc_state =
+			to_dm_crtc_state(new_crtc_state);
+		struct dm_crtc_state *dm_old_crtc_state =
+			to_dm_crtc_state(old_crtc_state);
+		bool modeset = drm_atomic_crtc_needs_modeset(new_crtc_state);
+		bool run_pass;
+
+		run_pass = (for_modeset && modeset) ||
+			   (!for_modeset && !modeset &&
+			    !dm_old_crtc_state->interrupts_enabled);
+
+		if (!run_pass)
+			continue;
+
+		if (!dm_new_crtc_state->interrupts_enabled)
+			continue;
+
+		manage_dm_interrupts(adev, acrtc, true);
+
+#ifdef CONFIG_DEBUG_FS
+		/* The stream has changed so CRC capture needs to re-enabled. */
+		if (dm_new_crtc_state->crc_enabled) {
+			dm_new_crtc_state->crc_enabled = false;
+			amdgpu_dm_crtc_set_crc_source(crtc, "auto");
+		}
+#endif
+	}
 }
 
 /*
@@ -4953,8 +5555,7 @@ cleanup:
 static void amdgpu_dm_crtc_copy_transient_flags(struct drm_crtc_state *crtc_state,
 						struct dc_stream_state *stream_state)
 {
-	stream_state->mode_changed =
-		crtc_state->mode_changed || crtc_state->active_changed;
+	stream_state->mode_changed = drm_atomic_crtc_needs_modeset(crtc_state);
 }
 
 static int amdgpu_dm_atomic_commit(struct drm_device *dev,
@@ -4967,30 +5568,41 @@ static int amdgpu_dm_atomic_commit(struct drm_device *dev,
 	int i;
 
 	/*
-	 * We evade vblanks and pflips on crtc that
-	 * should be changed. We do it here to flush & disable
-	 * interrupts before drm_swap_state is called in drm_atomic_helper_commit
-	 * it will update crtc->dm_crtc_state->stream pointer which is used in
-	 * the ISRs.
+	 * We evade vblank and pflip interrupts on CRTCs that are undergoing
+	 * a modeset, being disabled, or have no active planes.
+	 *
+	 * It's done in atomic commit rather than commit tail for now since
+	 * some of these interrupt handlers access the current CRTC state and
+	 * potentially the stream pointer itself.
+	 *
+	 * Since the atomic state is swapped within atomic commit and not within
+	 * commit tail this would leave to new state (that hasn't been committed yet)
+	 * being accesssed from within the handlers.
+	 *
+	 * TODO: Fix this so we can do this in commit tail and not have to block
+	 * in atomic check.
 	 */
 	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
 		struct dm_crtc_state *dm_old_crtc_state = to_dm_crtc_state(old_crtc_state);
 		struct dm_crtc_state *dm_new_crtc_state = to_dm_crtc_state(new_crtc_state);
 		struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc);
 
-		if (drm_atomic_crtc_needs_modeset(new_crtc_state)
-		    && dm_old_crtc_state->stream) {
+		if (dm_old_crtc_state->interrupts_enabled &&
+		    (!dm_new_crtc_state->interrupts_enabled ||
+		     drm_atomic_crtc_needs_modeset(new_crtc_state))) {
 			/*
-			 * If the stream is removed and CRC capture was
-			 * enabled on the CRTC the extra vblank reference
-			 * needs to be dropped since CRC capture will be
-			 * disabled.
+			 * Drop the extra vblank reference added by CRC
+			 * capture if applicable.
 			 */
-			if (!dm_new_crtc_state->stream
-			    && dm_new_crtc_state->crc_enabled) {
+			if (dm_new_crtc_state->crc_enabled)
 				drm_crtc_vblank_put(crtc);
+
+			/*
+			 * Only keep CRC capture enabled if there's
+			 * still a stream for the CRTC.
+			 */
+			if (!dm_new_crtc_state->stream)
 				dm_new_crtc_state->crc_enabled = false;
-			}
 
 			manage_dm_interrupts(adev, acrtc, false);
 		}
@@ -5037,7 +5649,7 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state)
 		dc_state = dm_state->context;
 	} else {
 		/* No state changes, retain current state. */
-		dc_state_temp = dc_create_state();
+		dc_state_temp = dc_create_state(dm->dc);
 		ASSERT(dc_state_temp);
 		dc_state = dc_state_temp;
 		dc_resource_state_copy_construct_current(dm->dc, dc_state);
@@ -5206,45 +5818,41 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state)
 		mutex_unlock(&dm->dc_lock);
 	}
 
+	/* Count number of newly disabled CRTCs for dropping PM refs later. */
 	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state,
-			new_crtc_state, i) {
-		/*
-		 * loop to enable interrupts on newly arrived crtc
-		 */
-		struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc);
-		bool modeset_needed;
-
+				      new_crtc_state, i) {
 		if (old_crtc_state->active && !new_crtc_state->active)
 			crtc_disable_count++;
 
 		dm_new_crtc_state = to_dm_crtc_state(new_crtc_state);
 		dm_old_crtc_state = to_dm_crtc_state(old_crtc_state);
-		modeset_needed = modeset_required(
-				new_crtc_state,
-				dm_new_crtc_state->stream,
-				dm_old_crtc_state->stream);
 
-		if (dm_new_crtc_state->stream == NULL || !modeset_needed)
-			continue;
-
-		manage_dm_interrupts(adev, acrtc, true);
+		/* Update freesync active state. */
+		pre_update_freesync_state_on_stream(dm, dm_new_crtc_state);
 
-#ifdef CONFIG_DEBUG_FS
-		/* The stream has changed so CRC capture needs to re-enabled. */
-		if (dm_new_crtc_state->crc_enabled)
-			amdgpu_dm_crtc_set_crc_source(crtc, "auto");
-#endif
+		/* Handle vrr on->off / off->on transitions */
+		amdgpu_dm_handle_vrr_transition(dm_old_crtc_state,
+						dm_new_crtc_state);
 	}
 
+	/* Enable interrupts for CRTCs going through a modeset. */
+	amdgpu_dm_enable_crtc_interrupts(dev, state, true);
+
+	for_each_new_crtc_in_state(state, crtc, new_crtc_state, j)
+		if (new_crtc_state->pageflip_flags & DRM_MODE_PAGE_FLIP_ASYNC)
+			wait_for_vblank = false;
+
 	/* update planes when needed per crtc*/
 	for_each_new_crtc_in_state(state, crtc, new_crtc_state, j) {
 		dm_new_crtc_state = to_dm_crtc_state(new_crtc_state);
 
 		if (dm_new_crtc_state->stream)
 			amdgpu_dm_commit_planes(state, dc_state, dev,
-						dm, crtc, &wait_for_vblank);
+						dm, crtc, wait_for_vblank);
 	}
 
+	/* Enable interrupts for CRTCs going from 0 to n active planes. */
+	amdgpu_dm_enable_crtc_interrupts(dev, state, false);
 
 	/*
 	 * send vblank event on all events not handled in flip and
@@ -5484,21 +6092,12 @@ static int dm_update_crtc_state(struct amdgpu_display_manager *dm,
 	struct amdgpu_dm_connector *aconnector = NULL;
 	struct drm_connector_state *drm_new_conn_state = NULL, *drm_old_conn_state = NULL;
 	struct dm_connector_state *dm_new_conn_state = NULL, *dm_old_conn_state = NULL;
-	struct drm_plane_state *new_plane_state = NULL;
 
 	new_stream = NULL;
 
 	dm_old_crtc_state = to_dm_crtc_state(old_crtc_state);
 	dm_new_crtc_state = to_dm_crtc_state(new_crtc_state);
 	acrtc = to_amdgpu_crtc(crtc);
-
-	new_plane_state = drm_atomic_get_new_plane_state(state, new_crtc_state->crtc->primary);
-
-	if (new_crtc_state->enable && new_plane_state && !new_plane_state->fb) {
-		ret = -EINVAL;
-		goto fail;
-	}
-
 	aconnector = amdgpu_dm_find_first_crtc_matching_connector(state, crtc);
 
 	/* TODO This hack should go away */
@@ -5661,6 +6260,9 @@ skip_modeset:
 		update_stream_scaling_settings(
 			&new_crtc_state->mode, dm_new_conn_state, dm_new_crtc_state->stream);
 
+	/* ABM settings */
+	dm_new_crtc_state->abm_level = dm_new_conn_state->abm_level;
+
 	/*
 	 * Color management settings. We also update color properties
 	 * when a modeset is needed, to ensure it gets reprogrammed.
@@ -5685,6 +6287,69 @@ fail:
 	return ret;
 }
 
+static bool should_reset_plane(struct drm_atomic_state *state,
+			       struct drm_plane *plane,
+			       struct drm_plane_state *old_plane_state,
+			       struct drm_plane_state *new_plane_state)
+{
+	struct drm_plane *other;
+	struct drm_plane_state *old_other_state, *new_other_state;
+	struct drm_crtc_state *new_crtc_state;
+	int i;
+
+	/*
+	 * TODO: Remove this hack once the checks below are sufficient
+	 * enough to determine when we need to reset all the planes on
+	 * the stream.
+	 */
+	if (state->allow_modeset)
+		return true;
+
+	/* Exit early if we know that we're adding or removing the plane. */
+	if (old_plane_state->crtc != new_plane_state->crtc)
+		return true;
+
+	/* old crtc == new_crtc == NULL, plane not in context. */
+	if (!new_plane_state->crtc)
+		return false;
+
+	new_crtc_state =
+		drm_atomic_get_new_crtc_state(state, new_plane_state->crtc);
+
+	if (!new_crtc_state)
+		return true;
+
+	if (drm_atomic_crtc_needs_modeset(new_crtc_state))
+		return true;
+
+	/*
+	 * If there are any new primary or overlay planes being added or
+	 * removed then the z-order can potentially change. To ensure
+	 * correct z-order and pipe acquisition the current DC architecture
+	 * requires us to remove and recreate all existing planes.
+	 *
+	 * TODO: Come up with a more elegant solution for this.
+	 */
+	for_each_oldnew_plane_in_state(state, other, old_other_state, new_other_state, i) {
+		if (other->type == DRM_PLANE_TYPE_CURSOR)
+			continue;
+
+		if (old_other_state->crtc != new_plane_state->crtc &&
+		    new_other_state->crtc != new_plane_state->crtc)
+			continue;
+
+		if (old_other_state->crtc != new_other_state->crtc)
+			return true;
+
+		/* TODO: Remove this once we can handle fast format changes. */
+		if (old_other_state->fb && new_other_state->fb &&
+		    old_other_state->fb->format != new_other_state->fb->format)
+			return true;
+	}
+
+	return false;
+}
+
 static int dm_update_plane_state(struct dc *dc,
 				 struct drm_atomic_state *state,
 				 struct drm_plane *plane,
@@ -5699,8 +6364,7 @@ static int dm_update_plane_state(struct dc *dc,
 	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
 	struct dm_crtc_state *dm_new_crtc_state, *dm_old_crtc_state;
 	struct dm_plane_state *dm_new_plane_state, *dm_old_plane_state;
-	/* TODO return page_flip_needed() function */
-	bool pflip_needed  = !state->allow_modeset;
+	bool needs_reset;
 	int ret = 0;
 
 
@@ -5713,10 +6377,12 @@ static int dm_update_plane_state(struct dc *dc,
 	if (plane->type == DRM_PLANE_TYPE_CURSOR)
 		return 0;
 
+	needs_reset = should_reset_plane(state, plane, old_plane_state,
+					 new_plane_state);
+
 	/* Remove any changed/removed planes */
 	if (!enable) {
-		if (pflip_needed &&
-		    plane->type != DRM_PLANE_TYPE_OVERLAY)
+		if (!needs_reset)
 			return 0;
 
 		if (!old_plane_crtc)
@@ -5767,7 +6433,7 @@ static int dm_update_plane_state(struct dc *dc,
 		if (!dm_new_crtc_state->stream)
 			return 0;
 
-		if (pflip_needed && plane->type != DRM_PLANE_TYPE_OVERLAY)
+		if (!needs_reset)
 			return 0;
 
 		WARN_ON(dm_new_plane_state->dc_state);
@@ -5779,7 +6445,7 @@ static int dm_update_plane_state(struct dc *dc,
 		DRM_DEBUG_DRIVER("Enabling DRM plane: %d on DRM crtc %d\n",
 				plane->base.id, new_plane_crtc->base.id);
 
-		ret = fill_plane_attributes(
+		ret = fill_dc_plane_attributes(
 			new_plane_crtc->dev->dev_private,
 			dc_new_plane_state,
 			new_plane_state,
@@ -5827,10 +6493,11 @@ static int dm_update_plane_state(struct dc *dc,
 }
 
 static int
-dm_determine_update_type_for_commit(struct dc *dc,
+dm_determine_update_type_for_commit(struct amdgpu_display_manager *dm,
 				    struct drm_atomic_state *state,
 				    enum surface_update_type *out_type)
 {
+	struct dc *dc = dm->dc;
 	struct dm_atomic_state *dm_state = NULL, *old_dm_state = NULL;
 	int i, j, num_plane, ret = 0;
 	struct drm_plane_state *old_plane_state, *new_plane_state;
@@ -5844,21 +6511,22 @@ dm_determine_update_type_for_commit(struct dc *dc,
 	struct dc_stream_status *status = NULL;
 
 	struct dc_surface_update *updates;
-	struct dc_plane_state *surface;
 	enum surface_update_type update_type = UPDATE_TYPE_FAST;
 
 	updates = kcalloc(MAX_SURFACES, sizeof(*updates), GFP_KERNEL);
-	surface = kcalloc(MAX_SURFACES, sizeof(*surface), GFP_KERNEL);
 
-	if (!updates || !surface) {
-		DRM_ERROR("Plane or surface update failed to allocate");
+	if (!updates) {
+		DRM_ERROR("Failed to allocate plane updates\n");
 		/* Set type to FULL to avoid crashing in DC*/
 		update_type = UPDATE_TYPE_FULL;
 		goto cleanup;
 	}
 
 	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
-		struct dc_stream_update stream_update = { 0 };
+		struct dc_scaling_info scaling_info;
+		struct dc_stream_update stream_update;
+
+		memset(&stream_update, 0, sizeof(stream_update));
 
 		new_dm_crtc_state = to_dm_crtc_state(new_crtc_state);
 		old_dm_crtc_state = to_dm_crtc_state(old_crtc_state);
@@ -5886,23 +6554,12 @@ dm_determine_update_type_for_commit(struct dc *dc,
 				goto cleanup;
 			}
 
-			if (!state->allow_modeset)
-				continue;
-
 			if (crtc != new_plane_crtc)
 				continue;
 
-			updates[num_plane].surface = &surface[num_plane];
+			updates[num_plane].surface = new_dm_plane_state->dc_state;
 
 			if (new_crtc_state->mode_changed) {
-				updates[num_plane].surface->src_rect =
-						new_dm_plane_state->dc_state->src_rect;
-				updates[num_plane].surface->dst_rect =
-						new_dm_plane_state->dc_state->dst_rect;
-				updates[num_plane].surface->rotation =
-						new_dm_plane_state->dc_state->rotation;
-				updates[num_plane].surface->in_transfer_func =
-						new_dm_plane_state->dc_state->in_transfer_func;
 				stream_update.dst = new_dm_crtc_state->stream->dst;
 				stream_update.src = new_dm_crtc_state->stream->src;
 			}
@@ -5918,6 +6575,13 @@ dm_determine_update_type_for_commit(struct dc *dc,
 						new_dm_crtc_state->stream->out_transfer_func;
 			}
 
+			ret = fill_dc_scaling_info(new_plane_state,
+						   &scaling_info);
+			if (ret)
+				goto cleanup;
+
+			updates[num_plane].scaling_info = &scaling_info;
+
 			num_plane++;
 		}
 
@@ -5937,8 +6601,14 @@ dm_determine_update_type_for_commit(struct dc *dc,
 		status = dc_stream_get_status_from_state(old_dm_state->context,
 							 new_dm_crtc_state->stream);
 
+		/*
+		 * TODO: DC modifies the surface during this call so we need
+		 * to lock here - find a way to do this without locking.
+		 */
+		mutex_lock(&dm->dc_lock);
 		update_type = dc_check_update_surfaces_for_stream(dc, updates, num_plane,
 								  &stream_update, status);
+		mutex_unlock(&dm->dc_lock);
 
 		if (update_type > UPDATE_TYPE_MED) {
 			update_type = UPDATE_TYPE_FULL;
@@ -5948,7 +6618,6 @@ dm_determine_update_type_for_commit(struct dc *dc,
 
 cleanup:
 	kfree(updates);
-	kfree(surface);
 
 	*out_type = update_type;
 	return ret;
@@ -6132,7 +6801,7 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 		lock_and_validation_needed = true;
 	}
 
-	ret = dm_determine_update_type_for_commit(dc, state, &update_type);
+	ret = dm_determine_update_type_for_commit(&adev->dm, state, &update_type);
 	if (ret)
 		goto fail;
 
@@ -6147,9 +6816,6 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 	 */
 	if (lock_and_validation_needed && overall_update_type <= UPDATE_TYPE_FAST)
 		WARN(1, "Global lock should be Set, overall_update_type should be UPDATE_TYPE_MED or UPDATE_TYPE_FULL");
-	else if (!lock_and_validation_needed && overall_update_type > UPDATE_TYPE_FAST)
-		WARN(1, "Global lock should NOT be set, overall_update_type should be UPDATE_TYPE_FAST");
-
 
 	if (overall_update_type > UPDATE_TYPE_FAST) {
 		ret = dm_atomic_get_state(state, &dm_state);
@@ -6160,7 +6826,7 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 		if (ret)
 			goto fail;
 
-		if (dc_validate_global_state(dc, dm_state->context) != DC_OK) {
+		if (dc_validate_global_state(dc, dm_state->context, false) != DC_OK) {
 			ret = -EINVAL;
 			goto fail;
 		}
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
index fbd161ddc3f4..978ff14a7d45 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
@@ -132,8 +132,6 @@ struct amdgpu_display_manager {
 	 */
 	struct drm_private_obj atomic_obj;
 
-	struct drm_modeset_lock atomic_obj_lock;
-
 	/**
 	 * @dc_lock:
 	 *
@@ -184,6 +182,15 @@ struct amdgpu_display_manager {
 	struct common_irq_params
 	vblank_params[DC_IRQ_SOURCE_VBLANK6 - DC_IRQ_SOURCE_VBLANK1 + 1];
 
+	/**
+	 * @vupdate_params:
+	 *
+	 * Vertical update IRQ parameters, passed to registered handlers when
+	 * triggered.
+	 */
+	struct common_irq_params
+	vupdate_params[DC_IRQ_SOURCE_VUPDATE6 - DC_IRQ_SOURCE_VUPDATE1 + 1];
+
 	spinlock_t irq_handler_list_table_lock;
 
 	struct backlight_device *backlight_dev;
@@ -240,6 +247,10 @@ struct amdgpu_dm_connector {
 	struct mutex hpd_lock;
 
 	bool fake_enable;
+#ifdef CONFIG_DEBUG_FS
+	uint32_t debugfs_dpcd_address;
+	uint32_t debugfs_dpcd_size;
+#endif
 };
 
 #define to_amdgpu_dm_connector(x) container_of(x, struct amdgpu_dm_connector, base)
@@ -260,6 +271,9 @@ struct dm_crtc_state {
 	struct drm_crtc_state base;
 	struct dc_stream_state *stream;
 
+	int active_planes;
+	bool interrupts_enabled;
+
 	int crc_skip_count;
 	bool crc_enabled;
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c
index 216e48cec716..7258c992a2bf 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c
@@ -126,46 +126,51 @@ int amdgpu_dm_set_regamma_lut(struct dm_crtc_state *crtc)
 		crtc->base.state->dev->dev_private;
 	struct drm_color_lut *lut;
 	uint32_t lut_size;
-	struct dc_gamma *gamma;
+	struct dc_gamma *gamma = NULL;
 	enum dc_transfer_func_type old_type = stream->out_transfer_func->type;
 
 	bool ret;
 
-	if (!blob) {
+	if (!blob && adev->asic_type <= CHIP_RAVEN) {
 		/* By default, use the SRGB predefined curve.*/
 		stream->out_transfer_func->type = TF_TYPE_PREDEFINED;
 		stream->out_transfer_func->tf = TRANSFER_FUNCTION_SRGB;
 		return 0;
 	}
 
-	lut = (struct drm_color_lut *)blob->data;
-	lut_size = blob->length / sizeof(struct drm_color_lut);
-
-	gamma = dc_create_gamma();
-	if (!gamma)
-		return -ENOMEM;
+	if (blob) {
+		lut = (struct drm_color_lut *)blob->data;
+		lut_size = blob->length / sizeof(struct drm_color_lut);
+
+		gamma = dc_create_gamma();
+		if (!gamma)
+			return -ENOMEM;
+
+		gamma->num_entries = lut_size;
+		if (gamma->num_entries == MAX_COLOR_LEGACY_LUT_ENTRIES)
+			gamma->type = GAMMA_RGB_256;
+		else if (gamma->num_entries == MAX_COLOR_LUT_ENTRIES)
+			gamma->type = GAMMA_CS_TFM_1D;
+		else {
+			/* Invalid lut size */
+			dc_gamma_release(&gamma);
+			return -EINVAL;
+		}
 
-	gamma->num_entries = lut_size;
-	if (gamma->num_entries == MAX_COLOR_LEGACY_LUT_ENTRIES)
-		gamma->type = GAMMA_RGB_256;
-	else if (gamma->num_entries == MAX_COLOR_LUT_ENTRIES)
-		gamma->type = GAMMA_CS_TFM_1D;
-	else {
-		/* Invalid lut size */
-		dc_gamma_release(&gamma);
-		return -EINVAL;
+		/* Convert drm_lut into dc_gamma */
+		__drm_lut_to_dc_gamma(lut, gamma, gamma->type == GAMMA_RGB_256);
 	}
 
-	/* Convert drm_lut into dc_gamma */
-	__drm_lut_to_dc_gamma(lut, gamma, gamma->type == GAMMA_RGB_256);
-
-	/* Call color module to translate into something DC understands. Namely
-	 * a transfer function.
+	/* predefined gamma ROM only exist for RAVEN and pre-RAVEN ASIC,
+	 * set canRomBeUsed accordingly
 	 */
 	stream->out_transfer_func->type = TF_TYPE_DISTRIBUTED_POINTS;
 	ret = mod_color_calculate_regamma_params(stream->out_transfer_func,
-						 gamma, true, adev->asic_type <= CHIP_RAVEN, NULL);
-	dc_gamma_release(&gamma);
+			gamma, true, adev->asic_type <= CHIP_RAVEN, NULL);
+
+	if (gamma)
+		dc_gamma_release(&gamma);
+
 	if (!ret) {
 		stream->out_transfer_func->type = old_type;
 		DRM_ERROR("Out of memory when calculating regamma params\n");
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c
index 4a55cde027cf..1d5fc5ad3bee 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c
@@ -29,6 +29,7 @@
 #include "amdgpu.h"
 #include "amdgpu_dm.h"
 #include "amdgpu_dm_debugfs.h"
+#include "dm_helpers.h"
 
 /* function description
  * get/ set DP configuration: lane_count, link_rate, spread_spectrum
@@ -688,8 +689,131 @@ static int vrr_range_show(struct seq_file *m, void *data)
 
 	return 0;
 }
+
+/* function description
+ *
+ * generic SDP message access for testing
+ *
+ * debugfs sdp_message is located at /syskernel/debug/dri/0/DP-x
+ *
+ * SDP header
+ * Hb0 : Secondary-Data Packet ID
+ * Hb1 : Secondary-Data Packet type
+ * Hb2 : Secondary-Data-packet-specific header, Byte 0
+ * Hb3 : Secondary-Data-packet-specific header, Byte 1
+ *
+ * for using custom sdp message: input 4 bytes SDP header and 32 bytes raw data
+ */
+static ssize_t dp_sdp_message_debugfs_write(struct file *f, const char __user *buf,
+				 size_t size, loff_t *pos)
+{
+	int r;
+	uint8_t data[36];
+	struct amdgpu_dm_connector *connector = file_inode(f)->i_private;
+	struct dm_crtc_state *acrtc_state;
+	uint32_t write_size = 36;
+
+	if (connector->base.status != connector_status_connected)
+		return -ENODEV;
+
+	if (size == 0)
+		return 0;
+
+	acrtc_state = to_dm_crtc_state(connector->base.state->crtc->state);
+
+	r = copy_from_user(data, buf, write_size);
+
+	write_size -= r;
+
+	dc_stream_send_dp_sdp(acrtc_state->stream, data, write_size);
+
+	return write_size;
+}
+
 DEFINE_SHOW_ATTRIBUTE(vrr_range);
 
+static ssize_t dp_dpcd_address_write(struct file *f, const char __user *buf,
+				 size_t size, loff_t *pos)
+{
+	int r;
+	struct amdgpu_dm_connector *connector = file_inode(f)->i_private;
+
+	if (size < sizeof(connector->debugfs_dpcd_address))
+		return 0;
+
+	r = copy_from_user(&connector->debugfs_dpcd_address,
+			buf, sizeof(connector->debugfs_dpcd_address));
+
+	return size - r;
+}
+
+static ssize_t dp_dpcd_size_write(struct file *f, const char __user *buf,
+				 size_t size, loff_t *pos)
+{
+	int r;
+	struct amdgpu_dm_connector *connector = file_inode(f)->i_private;
+
+	if (size < sizeof(connector->debugfs_dpcd_size))
+		return 0;
+
+	r = copy_from_user(&connector->debugfs_dpcd_size,
+			buf, sizeof(connector->debugfs_dpcd_size));
+
+	if (connector->debugfs_dpcd_size > 256)
+		connector->debugfs_dpcd_size = 0;
+
+	return size - r;
+}
+
+static ssize_t dp_dpcd_data_write(struct file *f, const char __user *buf,
+				 size_t size, loff_t *pos)
+{
+	int r;
+	char *data;
+	struct amdgpu_dm_connector *connector = file_inode(f)->i_private;
+	struct dc_link *link = connector->dc_link;
+	uint32_t write_size = connector->debugfs_dpcd_size;
+
+	if (size < write_size)
+		return 0;
+
+	data = kzalloc(write_size, GFP_KERNEL);
+	if (!data)
+		return 0;
+
+	r = copy_from_user(data, buf, write_size);
+
+	dm_helpers_dp_write_dpcd(link->ctx, link,
+			connector->debugfs_dpcd_address, data, write_size - r);
+	kfree(data);
+	return write_size - r;
+}
+
+static ssize_t dp_dpcd_data_read(struct file *f, char __user *buf,
+				 size_t size, loff_t *pos)
+{
+	int r;
+	char *data;
+	struct amdgpu_dm_connector *connector = file_inode(f)->i_private;
+	struct dc_link *link = connector->dc_link;
+	uint32_t read_size = connector->debugfs_dpcd_size;
+
+	if (size < read_size)
+		return 0;
+
+	data = kzalloc(read_size, GFP_KERNEL);
+	if (!data)
+		return 0;
+
+	dm_helpers_dp_read_dpcd(link->ctx, link,
+			connector->debugfs_dpcd_address, data, read_size);
+
+	r = copy_to_user(buf, data, read_size);
+
+	kfree(data);
+	return read_size - r;
+}
+
 static const struct file_operations dp_link_settings_debugfs_fops = {
 	.owner = THIS_MODULE,
 	.read = dp_link_settings_read,
@@ -710,6 +834,31 @@ static const struct file_operations dp_phy_test_pattern_fops = {
 	.llseek = default_llseek
 };
 
+static const struct file_operations sdp_message_fops = {
+	.owner = THIS_MODULE,
+	.write = dp_sdp_message_debugfs_write,
+	.llseek = default_llseek
+};
+
+static const struct file_operations dp_dpcd_address_debugfs_fops = {
+	.owner = THIS_MODULE,
+	.write = dp_dpcd_address_write,
+	.llseek = default_llseek
+};
+
+static const struct file_operations dp_dpcd_size_debugfs_fops = {
+	.owner = THIS_MODULE,
+	.write = dp_dpcd_size_write,
+	.llseek = default_llseek
+};
+
+static const struct file_operations dp_dpcd_data_debugfs_fops = {
+	.owner = THIS_MODULE,
+	.read = dp_dpcd_data_read,
+	.write = dp_dpcd_data_write,
+	.llseek = default_llseek
+};
+
 static const struct {
 	char *name;
 	const struct file_operations *fops;
@@ -717,7 +866,11 @@ static const struct {
 		{"link_settings", &dp_link_settings_debugfs_fops},
 		{"phy_settings", &dp_phy_settings_debugfs_fop},
 		{"test_pattern", &dp_phy_test_pattern_fops},
-		{"vrr_range", &vrr_range_fops}
+		{"vrr_range", &vrr_range_fops},
+		{"sdp_message", &sdp_message_fops},
+		{"aux_dpcd_address", &dp_dpcd_address_debugfs_fops},
+		{"aux_dpcd_size", &dp_dpcd_size_debugfs_fops},
+		{"aux_dpcd_data", &dp_dpcd_data_debugfs_fops}
 };
 
 int connector_debugfs_init(struct amdgpu_dm_connector *connector)
@@ -842,6 +995,35 @@ static const struct drm_info_list amdgpu_dm_debugfs_list[] = {
 	{"amdgpu_target_backlight_pwm", &target_backlight_read},
 };
 
+/*
+ * Sets the DC visual confirm debug option from the given string.
+ * Example usage: echo 1 > /sys/kernel/debug/dri/0/amdgpu_visual_confirm
+ */
+static int visual_confirm_set(void *data, u64 val)
+{
+	struct amdgpu_device *adev = data;
+
+	adev->dm.dc->debug.visual_confirm = (enum visual_confirm)val;
+
+	return 0;
+}
+
+/*
+ * Reads the DC visual confirm debug option value into the given buffer.
+ * Example usage: cat /sys/kernel/debug/dri/0/amdgpu_dm_visual_confirm
+ */
+static int visual_confirm_get(void *data, u64 *val)
+{
+	struct amdgpu_device *adev = data;
+
+	*val = adev->dm.dc->debug.visual_confirm;
+
+	return 0;
+}
+
+DEFINE_DEBUGFS_ATTRIBUTE(visual_confirm_fops, visual_confirm_get,
+			 visual_confirm_set, "%llu\n");
+
 int dtn_debugfs_init(struct amdgpu_device *adev)
 {
 	static const struct file_operations dtn_log_fops = {
@@ -867,5 +1049,13 @@ int dtn_debugfs_init(struct amdgpu_device *adev)
 		adev,
 		&dtn_log_fops);
 
-	return PTR_ERR_OR_ZERO(ent);
+	if (IS_ERR(ent))
+		return PTR_ERR(ent);
+
+	ent = debugfs_create_file_unsafe("amdgpu_dm_visual_confirm", 0644, root,
+					 adev, &visual_confirm_fops);
+	if (IS_ERR(ent))
+		return PTR_ERR(ent);
+
+	return 0;
 }
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
index b39766bd2840..e6cd67342df8 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
@@ -264,7 +264,7 @@ bool dm_helpers_dp_mst_write_payload_allocation_table(
 }
 
 /*
- * poll pending down reply before clear payload allocation table
+ * poll pending down reply
  */
 void dm_helpers_dp_mst_poll_pending_down_reply(
 	struct dc_context *ctx,
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c
index cd10f77cdeb0..fd22b4474dbf 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c
@@ -674,11 +674,30 @@ static int amdgpu_dm_set_crtc_irq_state(struct amdgpu_device *adev,
 		__func__);
 }
 
+static int amdgpu_dm_set_vupdate_irq_state(struct amdgpu_device *adev,
+					   struct amdgpu_irq_src *source,
+					   unsigned int crtc_id,
+					   enum amdgpu_interrupt_state state)
+{
+	return dm_irq_state(
+		adev,
+		source,
+		crtc_id,
+		state,
+		IRQ_TYPE_VUPDATE,
+		__func__);
+}
+
 static const struct amdgpu_irq_src_funcs dm_crtc_irq_funcs = {
 	.set = amdgpu_dm_set_crtc_irq_state,
 	.process = amdgpu_dm_irq_handler,
 };
 
+static const struct amdgpu_irq_src_funcs dm_vupdate_irq_funcs = {
+	.set = amdgpu_dm_set_vupdate_irq_state,
+	.process = amdgpu_dm_irq_handler,
+};
+
 static const struct amdgpu_irq_src_funcs dm_pageflip_irq_funcs = {
 	.set = amdgpu_dm_set_pflip_irq_state,
 	.process = amdgpu_dm_irq_handler,
@@ -695,6 +714,9 @@ void amdgpu_dm_set_irq_funcs(struct amdgpu_device *adev)
 	adev->crtc_irq.num_types = adev->mode_info.num_crtc;
 	adev->crtc_irq.funcs = &dm_crtc_irq_funcs;
 
+	adev->vupdate_irq.num_types = adev->mode_info.num_crtc;
+	adev->vupdate_irq.funcs = &dm_vupdate_irq_funcs;
+
 	adev->pageflip_irq.num_types = adev->mode_info.num_crtc;
 	adev->pageflip_irq.funcs = &dm_pageflip_irq_funcs;
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c
index c4ea3a91f17a..6e205ee36ac3 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c
@@ -84,6 +84,7 @@ static ssize_t dm_dp_aux_transfer(struct drm_dp_aux *aux,
 {
 	ssize_t result = 0;
 	struct aux_payload payload;
+	enum aux_channel_operation_result operation_result;
 
 	if (WARN_ON(msg->size > 16))
 		return -E2BIG;
@@ -97,13 +98,27 @@ static ssize_t dm_dp_aux_transfer(struct drm_dp_aux *aux,
 	payload.mot = (msg->request & DP_AUX_I2C_MOT) != 0;
 	payload.defer_delay = 0;
 
-	result = dc_link_aux_transfer(TO_DM_AUX(aux)->ddc_service, &payload);
+	result = dc_link_aux_transfer_raw(TO_DM_AUX(aux)->ddc_service, &payload,
+				      &operation_result);
 
 	if (payload.write)
 		result = msg->size;
 
-	if (result < 0) /* DC doesn't know about kernel error codes */
-		result = -EIO;
+	if (result < 0)
+		switch (operation_result) {
+		case AUX_CHANNEL_OPERATION_SUCCEEDED:
+			break;
+		case AUX_CHANNEL_OPERATION_FAILED_HPD_DISCON:
+		case AUX_CHANNEL_OPERATION_FAILED_REASON_UNKNOWN:
+			result = -EIO;
+			break;
+		case AUX_CHANNEL_OPERATION_FAILED_INVALID_REPLY:
+			result = -EBUSY;
+			break;
+		case AUX_CHANNEL_OPERATION_FAILED_TIMEOUT:
+			result = -ETIMEDOUT;
+			break;
+		}
 
 	return result;
 }
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_pp_smu.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_pp_smu.c
index a114954d6a5b..350e7a620d45 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_pp_smu.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_pp_smu.c
@@ -33,6 +33,7 @@
 #include "amdgpu_dm_irq.h"
 #include "amdgpu_pm.h"
 #include "dm_pp_smu.h"
+#include "amdgpu_smu.h"
 
 
 bool dm_pp_apply_display_requirements(
@@ -40,6 +41,7 @@ bool dm_pp_apply_display_requirements(
 		const struct dm_pp_display_configuration *pp_display_cfg)
 {
 	struct amdgpu_device *adev = ctx->driver_context;
+	struct smu_context *smu = &adev->smu;
 	int i;
 
 	if (adev->pm.dpm_enabled) {
@@ -105,6 +107,9 @@ bool dm_pp_apply_display_requirements(
 			adev->powerplay.pp_funcs->display_configuration_change(
 				adev->powerplay.pp_handle,
 				&adev->pm.pm_display_cfg);
+		else
+			smu_display_configuration_change(smu,
+							 &adev->pm.pm_display_cfg);
 
 		amdgpu_pm_compute_clocks(adev);
 	}
@@ -308,6 +313,12 @@ bool dm_pp_get_clock_levels_by_type(
 		if (adev->powerplay.pp_funcs->get_clock_by_type(pp_handle,
 			dc_to_pp_clock_type(clk_type), &pp_clks)) {
 		/* Error in pplib. Provide default values. */
+			return true;
+		}
+	} else if (adev->smu.funcs && adev->smu.funcs->get_clock_by_type) {
+		if (smu_get_clock_by_type(&adev->smu,
+					  dc_to_pp_clock_type(clk_type),
+					  &pp_clks)) {
 			get_default_clock_levels(clk_type, dc_clks);
 			return true;
 		}
@@ -324,6 +335,13 @@ bool dm_pp_get_clock_levels_by_type(
 			validation_clks.memory_max_clock = 80000;
 			validation_clks.level = 0;
 		}
+	} else if (adev->smu.funcs && adev->smu.funcs->get_max_high_clocks) {
+		if (smu_get_max_high_clocks(&adev->smu, &validation_clks)) {
+			DRM_INFO("DM_PPLIB: Warning: using default validation clocks!\n");
+			validation_clks.engine_max_clock = 72000;
+			validation_clks.memory_max_clock = 80000;
+			validation_clks.level = 0;
+		}
 	}
 
 	DRM_INFO("DM_PPLIB: Validation clocks:\n");
@@ -374,14 +392,21 @@ bool dm_pp_get_clock_levels_by_type_with_latency(
 	void *pp_handle = adev->powerplay.pp_handle;
 	struct pp_clock_levels_with_latency pp_clks = { 0 };
 	const struct amd_pm_funcs *pp_funcs = adev->powerplay.pp_funcs;
+	int ret;
+
+	if (pp_funcs && pp_funcs->get_clock_by_type_with_latency) {
+		ret = pp_funcs->get_clock_by_type_with_latency(pp_handle,
+						dc_to_pp_clock_type(clk_type),
+						&pp_clks);
+		if (ret)
+			return false;
+	} else if (adev->smu.ppt_funcs && adev->smu.ppt_funcs->get_clock_by_type_with_latency) {
+		if (smu_get_clock_by_type_with_latency(&adev->smu,
+						       dc_to_pp_clock_type(clk_type),
+						       &pp_clks))
+			return false;
+	}
 
-	if (!pp_funcs || !pp_funcs->get_clock_by_type_with_latency)
-		return false;
-
-	if (pp_funcs->get_clock_by_type_with_latency(pp_handle,
-						     dc_to_pp_clock_type(clk_type),
-						     &pp_clks))
-		return false;
 
 	pp_to_dc_clock_levels_with_latency(&pp_clks, clk_level_info, clk_type);
 
@@ -397,14 +422,20 @@ bool dm_pp_get_clock_levels_by_type_with_voltage(
 	void *pp_handle = adev->powerplay.pp_handle;
 	struct pp_clock_levels_with_voltage pp_clk_info = {0};
 	const struct amd_pm_funcs *pp_funcs = adev->powerplay.pp_funcs;
-
-	if (!pp_funcs || !pp_funcs->get_clock_by_type_with_voltage)
-		return false;
-
-	if (pp_funcs->get_clock_by_type_with_voltage(pp_handle,
-						     dc_to_pp_clock_type(clk_type),
-						     &pp_clk_info))
-		return false;
+	int ret;
+
+	if (pp_funcs && pp_funcs->get_clock_by_type_with_voltage) {
+		ret = pp_funcs->get_clock_by_type_with_voltage(pp_handle,
+						dc_to_pp_clock_type(clk_type),
+						&pp_clk_info);
+		if (ret)
+			return false;
+	} else if (adev->smu.ppt_funcs && adev->smu.ppt_funcs->get_clock_by_type_with_voltage) {
+		if (smu_get_clock_by_type_with_voltage(&adev->smu,
+						       dc_to_pp_clock_type(clk_type),
+						       &pp_clk_info))
+			return false;
+	}
 
 	pp_to_dc_clock_levels_with_voltage(&pp_clk_info, clk_level_info, clk_type);
 
@@ -445,6 +476,10 @@ bool dm_pp_apply_clock_for_voltage_request(
 		ret = adev->powerplay.pp_funcs->display_clock_voltage_request(
 			adev->powerplay.pp_handle,
 			&pp_clock_request);
+	else if (adev->smu.funcs &&
+		 adev->smu.funcs->display_clock_voltage_request)
+		ret = smu_display_clock_voltage_request(&adev->smu,
+							&pp_clock_request);
 	if (ret)
 		return false;
 	return true;
@@ -462,6 +497,8 @@ bool dm_pp_get_static_clocks(
 		ret = adev->powerplay.pp_funcs->get_current_clocks(
 			adev->powerplay.pp_handle,
 			&pp_clk_info);
+	else if (adev->smu.funcs)
+		ret = smu_get_current_clocks(&adev->smu, &pp_clk_info);
 	if (ret)
 		return false;
 
@@ -472,27 +509,6 @@ bool dm_pp_get_static_clocks(
 	return true;
 }
 
-void pp_rv_set_display_requirement(struct pp_smu *pp,
-		struct pp_smu_display_requirement_rv *req)
-{
-	const struct dc_context *ctx = pp->dm;
-	struct amdgpu_device *adev = ctx->driver_context;
-	void *pp_handle = adev->powerplay.pp_handle;
-	const struct amd_pm_funcs *pp_funcs = adev->powerplay.pp_funcs;
-	struct pp_display_clock_request clock = {0};
-
-	if (!pp_funcs || !pp_funcs->display_clock_voltage_request)
-		return;
-
-	clock.clock_type = amd_pp_dcf_clock;
-	clock.clock_freq_in_khz = req->hard_min_dcefclk_mhz * 1000;
-	pp_funcs->display_clock_voltage_request(pp_handle, &clock);
-
-	clock.clock_type = amd_pp_f_clock;
-	clock.clock_freq_in_khz = req->hard_min_fclk_mhz * 1000;
-	pp_funcs->display_clock_voltage_request(pp_handle, &clock);
-}
-
 void pp_rv_set_wm_ranges(struct pp_smu *pp,
 		struct pp_smu_wm_range_sets *ranges)
 {
@@ -508,9 +524,6 @@ void pp_rv_set_wm_ranges(struct pp_smu *pp,
 	wm_with_clock_ranges.num_wm_dmif_sets = ranges->num_reader_wm_sets;
 	wm_with_clock_ranges.num_wm_mcif_sets = ranges->num_writer_wm_sets;
 
-	if (!pp_funcs || !pp_funcs->set_watermarks_for_clocks_ranges)
-		return;
-
 	for (i = 0; i < wm_with_clock_ranges.num_wm_dmif_sets; i++) {
 		if (ranges->reader_wm_sets[i].wm_inst > 3)
 			wm_dce_clocks[i].wm_set_id = WM_SET_A;
@@ -543,7 +556,13 @@ void pp_rv_set_wm_ranges(struct pp_smu *pp,
 				ranges->writer_wm_sets[i].min_drain_clk_mhz * 1000;
 	}
 
-	pp_funcs->set_watermarks_for_clocks_ranges(pp_handle, &wm_with_clock_ranges);
+	if (pp_funcs && pp_funcs->set_watermarks_for_clocks_ranges)
+		pp_funcs->set_watermarks_for_clocks_ranges(pp_handle,
+							   &wm_with_clock_ranges);
+	else if (adev->smu.funcs &&
+		 adev->smu.funcs->set_watermarks_for_clock_ranges)
+		smu_set_watermarks_for_clock_ranges(&adev->smu,
+						    &wm_with_clock_ranges);
 }
 
 void pp_rv_set_pme_wa_enable(struct pp_smu *pp)
@@ -553,10 +572,10 @@ void pp_rv_set_pme_wa_enable(struct pp_smu *pp)
 	void *pp_handle = adev->powerplay.pp_handle;
 	const struct amd_pm_funcs *pp_funcs = adev->powerplay.pp_funcs;
 
-	if (!pp_funcs || !pp_funcs->notify_smu_enable_pwe)
-		return;
-
-	pp_funcs->notify_smu_enable_pwe(pp_handle);
+	if (pp_funcs && pp_funcs->notify_smu_enable_pwe)
+		pp_funcs->notify_smu_enable_pwe(pp_handle);
+	else if (adev->smu.funcs)
+		smu_notify_smu_enable_pwe(&adev->smu);
 }
 
 void pp_rv_set_active_display_count(struct pp_smu *pp, int count)
@@ -611,17 +630,16 @@ void pp_rv_set_hard_min_fclk_by_freq(struct pp_smu *pp, int mhz)
 	pp_funcs->set_hard_min_fclk_by_freq(pp_handle, mhz);
 }
 
-void dm_pp_get_funcs_rv(
+void dm_pp_get_funcs(
 		struct dc_context *ctx,
-		struct pp_smu_funcs_rv *funcs)
+		struct pp_smu_funcs *funcs)
 {
-	funcs->pp_smu.dm = ctx;
-	funcs->set_display_requirement = pp_rv_set_display_requirement;
-	funcs->set_wm_ranges = pp_rv_set_wm_ranges;
-	funcs->set_pme_wa_enable = pp_rv_set_pme_wa_enable;
-	funcs->set_display_count = pp_rv_set_active_display_count;
-	funcs->set_min_deep_sleep_dcfclk = pp_rv_set_min_deep_sleep_dcfclk;
-	funcs->set_hard_min_dcfclk_by_freq = pp_rv_set_hard_min_dcefclk_by_freq;
-	funcs->set_hard_min_fclk_by_freq = pp_rv_set_hard_min_fclk_by_freq;
+	funcs->rv_funcs.pp_smu.dm = ctx;
+	funcs->rv_funcs.set_wm_ranges = pp_rv_set_wm_ranges;
+	funcs->rv_funcs.set_pme_wa_enable = pp_rv_set_pme_wa_enable;
+	funcs->rv_funcs.set_display_count = pp_rv_set_active_display_count;
+	funcs->rv_funcs.set_min_deep_sleep_dcfclk = pp_rv_set_min_deep_sleep_dcfclk;
+	funcs->rv_funcs.set_hard_min_dcfclk_by_freq = pp_rv_set_hard_min_dcefclk_by_freq;
+	funcs->rv_funcs.set_hard_min_fclk_by_freq = pp_rv_set_hard_min_fclk_by_freq;
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/basics/fixpt31_32.c b/drivers/gpu/drm/amd/display/dc/basics/fixpt31_32.c
index f28989860fd8..1e9a2d352068 100644
--- a/drivers/gpu/drm/amd/display/dc/basics/fixpt31_32.c
+++ b/drivers/gpu/drm/amd/display/dc/basics/fixpt31_32.c
@@ -449,6 +449,11 @@ static inline unsigned int clamp_ux_dy(
 		return min_clamp;
 }
 
+unsigned int dc_fixpt_u4d19(struct fixed31_32 arg)
+{
+	return ux_dy(arg.value, 4, 19);
+}
+
 unsigned int dc_fixpt_u3d19(struct fixed31_32 arg)
 {
 	return ux_dy(arg.value, 3, 19);
diff --git a/drivers/gpu/drm/amd/display/dc/calcs/dcn_calcs.c b/drivers/gpu/drm/amd/display/dc/calcs/dcn_calcs.c
index eb62d10bb65c..1b4b51657f5e 100644
--- a/drivers/gpu/drm/amd/display/dc/calcs/dcn_calcs.c
+++ b/drivers/gpu/drm/amd/display/dc/calcs/dcn_calcs.c
@@ -247,6 +247,53 @@ static enum dcn_bw_defs tl_pixel_format_to_bw_defs(enum surface_pixel_format for
 	}
 }
 
+enum source_macro_tile_size swizzle_mode_to_macro_tile_size(enum swizzle_mode_values sw_mode)
+{
+	switch (sw_mode) {
+	/* for 4/8/16 high tiles */
+	case DC_SW_LINEAR:
+		return dm_4k_tile;
+	case DC_SW_4KB_S:
+	case DC_SW_4KB_S_X:
+		return dm_4k_tile;
+	case DC_SW_64KB_S:
+	case DC_SW_64KB_S_X:
+	case DC_SW_64KB_S_T:
+		return dm_64k_tile;
+	case DC_SW_VAR_S:
+	case DC_SW_VAR_S_X:
+		return dm_256k_tile;
+
+	/* For 64bpp 2 high tiles */
+	case DC_SW_4KB_D:
+	case DC_SW_4KB_D_X:
+		return dm_4k_tile;
+	case DC_SW_64KB_D:
+	case DC_SW_64KB_D_X:
+	case DC_SW_64KB_D_T:
+		return dm_64k_tile;
+	case DC_SW_VAR_D:
+	case DC_SW_VAR_D_X:
+		return dm_256k_tile;
+
+	case DC_SW_4KB_R:
+	case DC_SW_4KB_R_X:
+		return dm_4k_tile;
+	case DC_SW_64KB_R:
+	case DC_SW_64KB_R_X:
+		return dm_64k_tile;
+	case DC_SW_VAR_R:
+	case DC_SW_VAR_R_X:
+		return dm_256k_tile;
+
+	/* Unsupported swizzle modes for dcn */
+	case DC_SW_256B_S:
+	default:
+		ASSERT(0); /* Not supported */
+		return 0;
+	}
+}
+
 static void pipe_ctx_to_e2e_pipe_params (
 		const struct pipe_ctx *pipe,
 		struct _vcs_dpi_display_pipe_params_st *input)
@@ -287,46 +334,7 @@ static void pipe_ctx_to_e2e_pipe_params (
 	input->src.cur0_src_width      = 128; /* TODO: Cursor calcs, not curently stored */
 	input->src.cur0_bpp            = 32;
 
-	switch (pipe->plane_state->tiling_info.gfx9.swizzle) {
-	/* for 4/8/16 high tiles */
-	case DC_SW_LINEAR:
-		input->src.macro_tile_size = dm_4k_tile;
-		break;
-	case DC_SW_4KB_S:
-	case DC_SW_4KB_S_X:
-		input->src.macro_tile_size = dm_4k_tile;
-		break;
-	case DC_SW_64KB_S:
-	case DC_SW_64KB_S_X:
-	case DC_SW_64KB_S_T:
-		input->src.macro_tile_size = dm_64k_tile;
-		break;
-	case DC_SW_VAR_S:
-	case DC_SW_VAR_S_X:
-		input->src.macro_tile_size = dm_256k_tile;
-		break;
-
-	/* For 64bpp 2 high tiles */
-	case DC_SW_4KB_D:
-	case DC_SW_4KB_D_X:
-		input->src.macro_tile_size = dm_4k_tile;
-		break;
-	case DC_SW_64KB_D:
-	case DC_SW_64KB_D_X:
-	case DC_SW_64KB_D_T:
-		input->src.macro_tile_size = dm_64k_tile;
-		break;
-	case DC_SW_VAR_D:
-	case DC_SW_VAR_D_X:
-		input->src.macro_tile_size = dm_256k_tile;
-		break;
-
-	/* Unsupported swizzle modes for dcn */
-	case DC_SW_256B_S:
-	default:
-		ASSERT(0); /* Not supported */
-		break;
-	}
+	input->src.macro_tile_size = swizzle_mode_to_macro_tile_size(pipe->plane_state->tiling_info.gfx9.swizzle);
 
 	switch (pipe->plane_state->rotation) {
 	case ROTATION_ANGLE_0:
@@ -466,7 +474,7 @@ static void dcn_bw_calc_rq_dlg_ttu(
 	input.clks_cfg.dcfclk_mhz = v->dcfclk;
 	input.clks_cfg.dispclk_mhz = v->dispclk;
 	input.clks_cfg.dppclk_mhz = v->dppclk;
-	input.clks_cfg.refclk_mhz = dc->res_pool->ref_clock_inKhz / 1000.0;
+	input.clks_cfg.refclk_mhz = dc->res_pool->ref_clocks.dchub_ref_clock_inKhz / 1000.0;
 	input.clks_cfg.socclk_mhz = v->socclk;
 	input.clks_cfg.voltage = v->voltage_level;
 //	dc->dml.logger = pool->base.logger;
@@ -536,28 +544,28 @@ static void calc_wm_sets_and_perf_params(
 		v->fabric_and_dram_bandwidth = v->fabric_and_dram_bandwidth_vnom0p8;
 		dispclkdppclkdcfclk_deep_sleep_prefetch_parameters_watermarks_and_performance_calculation(v);
 
-		context->bw.dcn.watermarks.b.cstate_pstate.cstate_exit_ns =
+		context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.cstate_exit_ns =
 			v->stutter_exit_watermark * 1000;
-		context->bw.dcn.watermarks.b.cstate_pstate.cstate_enter_plus_exit_ns =
+		context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.cstate_enter_plus_exit_ns =
 				v->stutter_enter_plus_exit_watermark * 1000;
-		context->bw.dcn.watermarks.b.cstate_pstate.pstate_change_ns =
+		context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.pstate_change_ns =
 				v->dram_clock_change_watermark * 1000;
-		context->bw.dcn.watermarks.b.pte_meta_urgent_ns = v->ptemeta_urgent_watermark * 1000;
-		context->bw.dcn.watermarks.b.urgent_ns = v->urgent_watermark * 1000;
+		context->bw_ctx.bw.dcn.watermarks.b.pte_meta_urgent_ns = v->ptemeta_urgent_watermark * 1000;
+		context->bw_ctx.bw.dcn.watermarks.b.urgent_ns = v->urgent_watermark * 1000;
 
 		v->dcfclk_per_state[1] = v->dcfclkv_nom0p8;
 		v->dcfclk_per_state[0] = v->dcfclkv_nom0p8;
 		v->dcfclk = v->dcfclkv_nom0p8;
 		dispclkdppclkdcfclk_deep_sleep_prefetch_parameters_watermarks_and_performance_calculation(v);
 
-		context->bw.dcn.watermarks.c.cstate_pstate.cstate_exit_ns =
+		context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.cstate_exit_ns =
 			v->stutter_exit_watermark * 1000;
-		context->bw.dcn.watermarks.c.cstate_pstate.cstate_enter_plus_exit_ns =
+		context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.cstate_enter_plus_exit_ns =
 				v->stutter_enter_plus_exit_watermark * 1000;
-		context->bw.dcn.watermarks.c.cstate_pstate.pstate_change_ns =
+		context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.pstate_change_ns =
 				v->dram_clock_change_watermark * 1000;
-		context->bw.dcn.watermarks.c.pte_meta_urgent_ns = v->ptemeta_urgent_watermark * 1000;
-		context->bw.dcn.watermarks.c.urgent_ns = v->urgent_watermark * 1000;
+		context->bw_ctx.bw.dcn.watermarks.c.pte_meta_urgent_ns = v->ptemeta_urgent_watermark * 1000;
+		context->bw_ctx.bw.dcn.watermarks.c.urgent_ns = v->urgent_watermark * 1000;
 	}
 
 	if (v->voltage_level < 3) {
@@ -571,14 +579,14 @@ static void calc_wm_sets_and_perf_params(
 		v->dcfclk = v->dcfclkv_max0p9;
 		dispclkdppclkdcfclk_deep_sleep_prefetch_parameters_watermarks_and_performance_calculation(v);
 
-		context->bw.dcn.watermarks.d.cstate_pstate.cstate_exit_ns =
+		context->bw_ctx.bw.dcn.watermarks.d.cstate_pstate.cstate_exit_ns =
 			v->stutter_exit_watermark * 1000;
-		context->bw.dcn.watermarks.d.cstate_pstate.cstate_enter_plus_exit_ns =
+		context->bw_ctx.bw.dcn.watermarks.d.cstate_pstate.cstate_enter_plus_exit_ns =
 				v->stutter_enter_plus_exit_watermark * 1000;
-		context->bw.dcn.watermarks.d.cstate_pstate.pstate_change_ns =
+		context->bw_ctx.bw.dcn.watermarks.d.cstate_pstate.pstate_change_ns =
 				v->dram_clock_change_watermark * 1000;
-		context->bw.dcn.watermarks.d.pte_meta_urgent_ns = v->ptemeta_urgent_watermark * 1000;
-		context->bw.dcn.watermarks.d.urgent_ns = v->urgent_watermark * 1000;
+		context->bw_ctx.bw.dcn.watermarks.d.pte_meta_urgent_ns = v->ptemeta_urgent_watermark * 1000;
+		context->bw_ctx.bw.dcn.watermarks.d.urgent_ns = v->urgent_watermark * 1000;
 	}
 
 	v->fabric_and_dram_bandwidth_per_state[2] = v->fabric_and_dram_bandwidth_vnom0p8;
@@ -591,20 +599,20 @@ static void calc_wm_sets_and_perf_params(
 	v->dcfclk = v->dcfclk_per_state[v->voltage_level];
 	dispclkdppclkdcfclk_deep_sleep_prefetch_parameters_watermarks_and_performance_calculation(v);
 
-	context->bw.dcn.watermarks.a.cstate_pstate.cstate_exit_ns =
+	context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_exit_ns =
 		v->stutter_exit_watermark * 1000;
-	context->bw.dcn.watermarks.a.cstate_pstate.cstate_enter_plus_exit_ns =
+	context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_enter_plus_exit_ns =
 			v->stutter_enter_plus_exit_watermark * 1000;
-	context->bw.dcn.watermarks.a.cstate_pstate.pstate_change_ns =
+	context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.pstate_change_ns =
 			v->dram_clock_change_watermark * 1000;
-	context->bw.dcn.watermarks.a.pte_meta_urgent_ns = v->ptemeta_urgent_watermark * 1000;
-	context->bw.dcn.watermarks.a.urgent_ns = v->urgent_watermark * 1000;
+	context->bw_ctx.bw.dcn.watermarks.a.pte_meta_urgent_ns = v->ptemeta_urgent_watermark * 1000;
+	context->bw_ctx.bw.dcn.watermarks.a.urgent_ns = v->urgent_watermark * 1000;
 	if (v->voltage_level >= 2) {
-		context->bw.dcn.watermarks.b = context->bw.dcn.watermarks.a;
-		context->bw.dcn.watermarks.c = context->bw.dcn.watermarks.a;
+		context->bw_ctx.bw.dcn.watermarks.b = context->bw_ctx.bw.dcn.watermarks.a;
+		context->bw_ctx.bw.dcn.watermarks.c = context->bw_ctx.bw.dcn.watermarks.a;
 	}
 	if (v->voltage_level >= 3)
-		context->bw.dcn.watermarks.d = context->bw.dcn.watermarks.a;
+		context->bw_ctx.bw.dcn.watermarks.d = context->bw_ctx.bw.dcn.watermarks.a;
 }
 #endif
 
@@ -693,8 +701,15 @@ static void hack_bounding_box(struct dcn_bw_internal_vars *v,
 
 bool dcn_validate_bandwidth(
 		struct dc *dc,
-		struct dc_state *context)
+		struct dc_state *context,
+		bool fast_validate)
 {
+	/*
+	 * we want a breakdown of the various stages of validation, which the
+	 * perf_trace macro doesn't support
+	 */
+	BW_VAL_TRACE_SETUP();
+
 	const struct resource_pool *pool = dc->res_pool;
 	struct dcn_bw_internal_vars *v = &context->dcn_bw_vars;
 	int i, input_idx;
@@ -703,6 +718,9 @@ bool dcn_validate_bandwidth(
 	float bw_limit;
 
 	PERFORMANCE_TRACE_START();
+
+	BW_VAL_TRACE_COUNT();
+
 	if (dcn_bw_apply_registry_override(dc))
 		dcn_bw_sync_calcs_and_dml(dc);
 
@@ -1000,13 +1018,16 @@ bool dcn_validate_bandwidth(
 				dc->debug.sr_enter_plus_exit_time_dpm0_ns / 1000.0f;
 		if (dc->debug.sr_exit_time_dpm0_ns)
 			v->sr_exit_time =  dc->debug.sr_exit_time_dpm0_ns / 1000.0f;
-		dc->dml.soc.sr_enter_plus_exit_time_us = v->sr_enter_plus_exit_time;
-		dc->dml.soc.sr_exit_time_us = v->sr_exit_time;
+		context->bw_ctx.dml.soc.sr_enter_plus_exit_time_us = v->sr_enter_plus_exit_time;
+		context->bw_ctx.dml.soc.sr_exit_time_us = v->sr_exit_time;
 		mode_support_and_system_configuration(v);
 	}
 
-	if (v->voltage_level != 5) {
+	BW_VAL_TRACE_END_VOLTAGE_LEVEL();
+
+	if (v->voltage_level != number_of_states_plus_one && !fast_validate) {
 		float bw_consumed = v->total_bandwidth_consumed_gbyte_per_second;
+
 		if (bw_consumed < v->fabric_and_dram_bandwidth_vmin0p65)
 			bw_consumed = v->fabric_and_dram_bandwidth_vmin0p65;
 		else if (bw_consumed < v->fabric_and_dram_bandwidth_vmid0p72)
@@ -1027,58 +1048,60 @@ bool dcn_validate_bandwidth(
 		 */
 		dispclkdppclkdcfclk_deep_sleep_prefetch_parameters_watermarks_and_performance_calculation(v);
 
-		context->bw.dcn.watermarks.a.cstate_pstate.cstate_exit_ns =
+		context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_exit_ns =
 			v->stutter_exit_watermark * 1000;
-		context->bw.dcn.watermarks.a.cstate_pstate.cstate_enter_plus_exit_ns =
+		context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_enter_plus_exit_ns =
 				v->stutter_enter_plus_exit_watermark * 1000;
-		context->bw.dcn.watermarks.a.cstate_pstate.pstate_change_ns =
+		context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.pstate_change_ns =
 				v->dram_clock_change_watermark * 1000;
-		context->bw.dcn.watermarks.a.pte_meta_urgent_ns = v->ptemeta_urgent_watermark * 1000;
-		context->bw.dcn.watermarks.a.urgent_ns = v->urgent_watermark * 1000;
-		context->bw.dcn.watermarks.b = context->bw.dcn.watermarks.a;
-		context->bw.dcn.watermarks.c = context->bw.dcn.watermarks.a;
-		context->bw.dcn.watermarks.d = context->bw.dcn.watermarks.a;
+		context->bw_ctx.bw.dcn.watermarks.a.pte_meta_urgent_ns = v->ptemeta_urgent_watermark * 1000;
+		context->bw_ctx.bw.dcn.watermarks.a.urgent_ns = v->urgent_watermark * 1000;
+		context->bw_ctx.bw.dcn.watermarks.b = context->bw_ctx.bw.dcn.watermarks.a;
+		context->bw_ctx.bw.dcn.watermarks.c = context->bw_ctx.bw.dcn.watermarks.a;
+		context->bw_ctx.bw.dcn.watermarks.d = context->bw_ctx.bw.dcn.watermarks.a;
 
-		context->bw.dcn.clk.fclk_khz = (int)(bw_consumed * 1000000 /
+		context->bw_ctx.bw.dcn.clk.fclk_khz = (int)(bw_consumed * 1000000 /
 				(ddr4_dram_factor_single_Channel * v->number_of_channels));
 		if (bw_consumed == v->fabric_and_dram_bandwidth_vmin0p65) {
-			context->bw.dcn.clk.fclk_khz = (int)(bw_consumed * 1000000 / 32);
+			context->bw_ctx.bw.dcn.clk.fclk_khz = (int)(bw_consumed * 1000000 / 32);
 		}
 
-		context->bw.dcn.clk.dcfclk_deep_sleep_khz = (int)(v->dcf_clk_deep_sleep * 1000);
-		context->bw.dcn.clk.dcfclk_khz = (int)(v->dcfclk * 1000);
+		context->bw_ctx.bw.dcn.clk.dcfclk_deep_sleep_khz = (int)(v->dcf_clk_deep_sleep * 1000);
+		context->bw_ctx.bw.dcn.clk.dcfclk_khz = (int)(v->dcfclk * 1000);
 
-		context->bw.dcn.clk.dispclk_khz = (int)(v->dispclk * 1000);
+		context->bw_ctx.bw.dcn.clk.dispclk_khz = (int)(v->dispclk * 1000);
 		if (dc->debug.max_disp_clk == true)
-			context->bw.dcn.clk.dispclk_khz = (int)(dc->dcn_soc->max_dispclk_vmax0p9 * 1000);
+			context->bw_ctx.bw.dcn.clk.dispclk_khz = (int)(dc->dcn_soc->max_dispclk_vmax0p9 * 1000);
 
-		if (context->bw.dcn.clk.dispclk_khz <
+		if (context->bw_ctx.bw.dcn.clk.dispclk_khz <
 				dc->debug.min_disp_clk_khz) {
-			context->bw.dcn.clk.dispclk_khz =
+			context->bw_ctx.bw.dcn.clk.dispclk_khz =
 					dc->debug.min_disp_clk_khz;
 		}
 
-		context->bw.dcn.clk.dppclk_khz = context->bw.dcn.clk.dispclk_khz / v->dispclk_dppclk_ratio;
-		context->bw.dcn.clk.phyclk_khz = v->phyclk_per_state[v->voltage_level];
+		context->bw_ctx.bw.dcn.clk.dppclk_khz = context->bw_ctx.bw.dcn.clk.dispclk_khz / v->dispclk_dppclk_ratio;
+		context->bw_ctx.bw.dcn.clk.phyclk_khz = v->phyclk_per_state[v->voltage_level];
 		switch (v->voltage_level) {
 		case 0:
-			context->bw.dcn.clk.max_supported_dppclk_khz =
+			context->bw_ctx.bw.dcn.clk.max_supported_dppclk_khz =
 					(int)(dc->dcn_soc->max_dppclk_vmin0p65 * 1000);
 			break;
 		case 1:
-			context->bw.dcn.clk.max_supported_dppclk_khz =
+			context->bw_ctx.bw.dcn.clk.max_supported_dppclk_khz =
 					(int)(dc->dcn_soc->max_dppclk_vmid0p72 * 1000);
 			break;
 		case 2:
-			context->bw.dcn.clk.max_supported_dppclk_khz =
+			context->bw_ctx.bw.dcn.clk.max_supported_dppclk_khz =
 					(int)(dc->dcn_soc->max_dppclk_vnom0p8 * 1000);
 			break;
 		default:
-			context->bw.dcn.clk.max_supported_dppclk_khz =
+			context->bw_ctx.bw.dcn.clk.max_supported_dppclk_khz =
 					(int)(dc->dcn_soc->max_dppclk_vmax0p9 * 1000);
 			break;
 		}
 
+		BW_VAL_TRACE_END_WATERMARKS();
+
 		for (i = 0, input_idx = 0; i < pool->pipe_count; i++) {
 			struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
 
@@ -1141,7 +1164,7 @@ bool dcn_validate_bandwidth(
 						hsplit_pipe->pipe_dlg_param.vblank_end = pipe->pipe_dlg_param.vblank_end;
 					} else {
 						/* pipe not split previously needs split */
-						hsplit_pipe = find_idle_secondary_pipe(&context->res_ctx, pool);
+						hsplit_pipe = find_idle_secondary_pipe(&context->res_ctx, pool, pipe);
 						ASSERT(hsplit_pipe);
 						split_stream_across_pipes(
 							&context->res_ctx, pool,
@@ -1169,13 +1192,17 @@ bool dcn_validate_bandwidth(
 
 			input_idx++;
 		}
+	} else if (v->voltage_level == number_of_states_plus_one) {
+		BW_VAL_TRACE_SKIP(fail);
+	} else if (fast_validate) {
+		BW_VAL_TRACE_SKIP(fast);
 	}
 
 	if (v->voltage_level == 0) {
 
-		dc->dml.soc.sr_enter_plus_exit_time_us =
+		context->bw_ctx.dml.soc.sr_enter_plus_exit_time_us =
 				dc->dcn_soc->sr_enter_plus_exit_time;
-		dc->dml.soc.sr_exit_time_us = dc->dcn_soc->sr_exit_time;
+		context->bw_ctx.dml.soc.sr_exit_time_us = dc->dcn_soc->sr_exit_time;
 	}
 
 	/*
@@ -1188,6 +1215,7 @@ bool dcn_validate_bandwidth(
 	kernel_fpu_end();
 
 	PERFORMANCE_TRACE_END();
+	BW_VAL_TRACE_FINISH();
 
 	if (bw_limit_pass && v->voltage_level != 5)
 		return true;
@@ -1395,12 +1423,14 @@ void dcn_bw_update_from_pplib(struct dc *dc)
 
 void dcn_bw_notify_pplib_of_wm_ranges(struct dc *dc)
 {
-	struct pp_smu_funcs_rv *pp = dc->res_pool->pp_smu;
+	struct pp_smu_funcs_rv *pp = NULL;
 	struct pp_smu_wm_range_sets ranges = {0};
 	int min_fclk_khz, min_dcfclk_khz, socclk_khz;
 	const int overdrive = 5000000; /* 5 GHz to cover Overdrive */
 
-	if (!pp->set_wm_ranges)
+	if (dc->res_pool->pp_smu)
+		pp = &dc->res_pool->pp_smu->rv_funcs;
+	if (!pp || !pp->set_wm_ranges)
 		return;
 
 	kernel_fpu_begin();
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c
index a6cda201c964..18c775a950cc 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc.c
@@ -524,6 +524,14 @@ void dc_link_set_preferred_link_settings(struct dc *dc,
 	struct dc_stream_state *link_stream;
 	struct dc_link_settings store_settings = *link_setting;
 
+	link->preferred_link_setting = store_settings;
+
+	/* Retrain with preferred link settings only relevant for
+	 * DP signal type
+	 */
+	if (!dc_is_dp_signal(link->connector_signal))
+		return;
+
 	for (i = 0; i < MAX_PIPES; i++) {
 		pipe = &dc->current_state->res_ctx.pipe_ctx[i];
 		if (pipe->stream && pipe->stream->link) {
@@ -538,7 +546,10 @@ void dc_link_set_preferred_link_settings(struct dc *dc,
 
 	link_stream = link->dc->current_state->res_ctx.pipe_ctx[i].stream;
 
-	link->preferred_link_setting = store_settings;
+	/* Cannot retrain link if backend is off */
+	if (link_stream->dpms_off)
+		return;
+
 	if (link_stream)
 		decide_link_settings(link_stream, &store_settings);
 
@@ -573,6 +584,28 @@ void dc_link_set_test_pattern(struct dc_link *link,
 			cust_pattern_size);
 }
 
+uint32_t dc_link_bandwidth_kbps(
+	const struct dc_link *link,
+	const struct dc_link_settings *link_setting)
+{
+	uint32_t link_bw_kbps = link_setting->link_rate * LINK_RATE_REF_FREQ_IN_KHZ; /* bytes per sec */
+
+	link_bw_kbps *= 8;   /* 8 bits per byte*/
+	link_bw_kbps *= link_setting->lane_count;
+
+	return link_bw_kbps;
+
+}
+
+const struct dc_link_settings *dc_link_get_link_cap(
+		const struct dc_link *link)
+{
+	if (link->preferred_link_setting.lane_count != LANE_COUNT_UNKNOWN &&
+			link->preferred_link_setting.link_rate != LINK_RATE_UNKNOWN)
+		return &link->preferred_link_setting;
+	return &link->verified_link_cap;
+}
+
 static void destruct(struct dc *dc)
 {
 	dc_release_state(dc->current_state);
@@ -621,6 +654,10 @@ static bool construct(struct dc *dc,
 #endif
 
 	enum dce_version dc_version = DCE_VERSION_UNKNOWN;
+	dc->config = init_params->flags;
+
+	memcpy(&dc->bb_overrides, &init_params->bb_overrides, sizeof(dc->bb_overrides));
+
 	dc_dceip = kzalloc(sizeof(*dc_dceip), GFP_KERNEL);
 	if (!dc_dceip) {
 		dm_error("%s: failed to create dceip\n", __func__);
@@ -668,13 +705,6 @@ static bool construct(struct dc *dc,
 	dc_ctx->dc_stream_id_count = 0;
 	dc->ctx = dc_ctx;
 
-	dc->current_state = dc_create_state();
-
-	if (!dc->current_state) {
-		dm_error("%s: failed to create validate ctx\n", __func__);
-		goto fail;
-	}
-
 	/* Create logger */
 
 	dc_ctx->dce_environment = init_params->dce_environment;
@@ -722,14 +752,22 @@ static bool construct(struct dc *dc,
 		goto fail;
 	}
 
-	dc->res_pool = dc_create_resource_pool(
-			dc,
-			init_params->num_virtual_links,
-			dc_version,
-			init_params->asic_id);
+	dc->res_pool = dc_create_resource_pool(dc, init_params, dc_version);
 	if (!dc->res_pool)
 		goto fail;
 
+	/* Creation of current_state must occur after dc->dml
+	 * is initialized in dc_create_resource_pool because
+	 * on creation it copies the contents of dc->dml
+	 */
+
+	dc->current_state = dc_create_state(dc);
+
+	if (!dc->current_state) {
+		dm_error("%s: failed to create validate ctx\n", __func__);
+		goto fail;
+	}
+
 	dc_resource_state_construct(dc, dc->current_state);
 
 	if (!create_links(dc, init_params->num_virtual_links))
@@ -746,7 +784,7 @@ fail:
 static void disable_dangling_plane(struct dc *dc, struct dc_state *context)
 {
 	int i, j;
-	struct dc_state *dangling_context = dc_create_state();
+	struct dc_state *dangling_context = dc_create_state(dc);
 	struct dc_state *current_ctx;
 
 	if (dangling_context == NULL)
@@ -811,8 +849,6 @@ struct dc *dc_create(const struct dc_init_data *init_params)
 	if (dc->res_pool->dmcu != NULL)
 		dc->versions.dmcu_version = dc->res_pool->dmcu->dmcu_version;
 
-	dc->config = init_params->flags;
-
 	dc->build_id = DC_BUILD_ID;
 
 	DC_LOG_DC("Display Core initialized\n");
@@ -969,7 +1005,7 @@ static bool context_changed(
 	return false;
 }
 
-bool dc_validate_seamless_boot_timing(struct dc *dc,
+bool dc_validate_seamless_boot_timing(const struct dc *dc,
 				const struct dc_sink *sink,
 				struct dc_crtc_timing *crtc_timing)
 {
@@ -1060,7 +1096,13 @@ static enum dc_status dc_commit_state_no_check(struct dc *dc, struct dc_state *c
 	if (!dcb->funcs->is_accelerated_mode(dcb))
 		dc->hwss.enable_accelerated_mode(dc, context);
 
-	dc->hwss.prepare_bandwidth(dc, context);
+	for (i = 0; i < context->stream_count; i++) {
+		if (context->streams[i]->apply_seamless_boot_optimization)
+			dc->optimize_seamless_boot = true;
+	}
+
+	if (!dc->optimize_seamless_boot)
+		dc->hwss.prepare_bandwidth(dc, context);
 
 	/* re-program planes for existing stream, in case we need to
 	 * free up plane resource for later use
@@ -1135,12 +1177,15 @@ static enum dc_status dc_commit_state_no_check(struct dc *dc, struct dc_state *c
 
 	dc_enable_stereo(dc, context, dc_streams, context->stream_count);
 
-	/* pplib is notified if disp_num changed */
-	dc->hwss.optimize_bandwidth(dc, context);
+	if (!dc->optimize_seamless_boot)
+		/* pplib is notified if disp_num changed */
+		dc->hwss.optimize_bandwidth(dc, context);
 
 	for (i = 0; i < context->stream_count; i++)
 		context->streams[i]->mode_changed = false;
 
+	memset(&context->commit_hints, 0, sizeof(context->commit_hints));
+
 	dc_release_state(dc->current_state);
 
 	dc->current_state = context;
@@ -1177,7 +1222,7 @@ bool dc_post_update_surfaces_to_stream(struct dc *dc)
 	int i;
 	struct dc_state *context = dc->current_state;
 
-	if (dc->optimized_required == false)
+	if (!dc->optimized_required || dc->optimize_seamless_boot)
 		return true;
 
 	post_surface_trace(dc);
@@ -1195,18 +1240,60 @@ bool dc_post_update_surfaces_to_stream(struct dc *dc)
 	return true;
 }
 
-struct dc_state *dc_create_state(void)
+struct dc_state *dc_create_state(struct dc *dc)
 {
 	struct dc_state *context = kzalloc(sizeof(struct dc_state),
 					   GFP_KERNEL);
 
 	if (!context)
 		return NULL;
+	/* Each context must have their own instance of VBA and in order to
+	 * initialize and obtain IP and SOC the base DML instance from DC is
+	 * initially copied into every context
+	 */
+#ifdef CONFIG_DRM_AMD_DC_DCN1_0
+	memcpy(&context->bw_ctx.dml, &dc->dml, sizeof(struct display_mode_lib));
+#endif
 
 	kref_init(&context->refcount);
+
 	return context;
 }
 
+struct dc_state *dc_copy_state(struct dc_state *src_ctx)
+{
+	int i, j;
+	struct dc_state *new_ctx = kzalloc(sizeof(struct dc_state),
+					   GFP_KERNEL);
+
+	if (!new_ctx)
+		return NULL;
+
+	memcpy(new_ctx, src_ctx, sizeof(struct dc_state));
+
+	for (i = 0; i < MAX_PIPES; i++) {
+			struct pipe_ctx *cur_pipe = &new_ctx->res_ctx.pipe_ctx[i];
+
+			if (cur_pipe->top_pipe)
+				cur_pipe->top_pipe =  &new_ctx->res_ctx.pipe_ctx[cur_pipe->top_pipe->pipe_idx];
+
+			if (cur_pipe->bottom_pipe)
+				cur_pipe->bottom_pipe = &new_ctx->res_ctx.pipe_ctx[cur_pipe->bottom_pipe->pipe_idx];
+
+	}
+
+	for (i = 0; i < new_ctx->stream_count; i++) {
+			dc_stream_retain(new_ctx->streams[i]);
+			for (j = 0; j < new_ctx->stream_status[i].plane_count; j++)
+				dc_plane_state_retain(
+					new_ctx->stream_status[i].plane_states[j]);
+	}
+
+	kref_init(&new_ctx->refcount);
+
+	return new_ctx;
+}
+
 void dc_retain_state(struct dc_state *context)
 {
 	kref_get(&context->refcount);
@@ -1666,6 +1753,7 @@ static void commit_planes_do_stream_update(struct dc *dc,
 				continue;
 
 			if (stream_update->dpms_off) {
+				dc->hwss.pipe_control_lock(dc, pipe_ctx, true);
 				if (*stream_update->dpms_off) {
 					core_link_disable_stream(pipe_ctx, KEEP_ACQUIRED_RESOURCE);
 					dc->hwss.optimize_bandwidth(dc, dc->current_state);
@@ -1673,6 +1761,7 @@ static void commit_planes_do_stream_update(struct dc *dc,
 					dc->hwss.prepare_bandwidth(dc, dc->current_state);
 					core_link_enable_stream(dc->current_state, pipe_ctx);
 				}
+				dc->hwss.pipe_control_lock(dc, pipe_ctx, false);
 			}
 
 			if (stream_update->abm_level && pipe_ctx->stream_res.abm) {
@@ -1700,7 +1789,16 @@ static void commit_planes_for_stream(struct dc *dc,
 	int i, j;
 	struct pipe_ctx *top_pipe_to_program = NULL;
 
-	if (update_type == UPDATE_TYPE_FULL) {
+	if (dc->optimize_seamless_boot && surface_count > 0) {
+		/* Optimize seamless boot flag keeps clocks and watermarks high until
+		 * first flip. After first flip, optimization is required to lower
+		 * bandwidth.
+		 */
+		dc->optimize_seamless_boot = false;
+		dc->optimized_required = true;
+	}
+
+	if (update_type == UPDATE_TYPE_FULL && !dc->optimize_seamless_boot) {
 		dc->hwss.prepare_bandwidth(dc, context);
 		context_clock_trace(dc, context);
 	}
@@ -1800,7 +1898,7 @@ void dc_commit_updates_for_stream(struct dc *dc,
 	if (update_type >= UPDATE_TYPE_FULL) {
 
 		/* initialize scratch memory for building context */
-		context = dc_create_state();
+		context = dc_create_state(dc);
 		if (context == NULL) {
 			DC_ERROR("Failed to allocate new validate context!\n");
 			return;
@@ -2099,13 +2197,13 @@ void dc_link_remove_remote_sink(struct dc_link *link, struct dc_sink *sink)
 
 void get_clock_requirements_for_state(struct dc_state *state, struct AsicStateEx *info)
 {
-	info->displayClock				= (unsigned int)state->bw.dcn.clk.dispclk_khz;
-	info->engineClock				= (unsigned int)state->bw.dcn.clk.dcfclk_khz;
-	info->memoryClock				= (unsigned int)state->bw.dcn.clk.dramclk_khz;
-	info->maxSupportedDppClock		= (unsigned int)state->bw.dcn.clk.max_supported_dppclk_khz;
-	info->dppClock					= (unsigned int)state->bw.dcn.clk.dppclk_khz;
-	info->socClock					= (unsigned int)state->bw.dcn.clk.socclk_khz;
-	info->dcfClockDeepSleep			= (unsigned int)state->bw.dcn.clk.dcfclk_deep_sleep_khz;
-	info->fClock					= (unsigned int)state->bw.dcn.clk.fclk_khz;
-	info->phyClock					= (unsigned int)state->bw.dcn.clk.phyclk_khz;
+	info->displayClock				= (unsigned int)state->bw_ctx.bw.dcn.clk.dispclk_khz;
+	info->engineClock				= (unsigned int)state->bw_ctx.bw.dcn.clk.dcfclk_khz;
+	info->memoryClock				= (unsigned int)state->bw_ctx.bw.dcn.clk.dramclk_khz;
+	info->maxSupportedDppClock		= (unsigned int)state->bw_ctx.bw.dcn.clk.max_supported_dppclk_khz;
+	info->dppClock					= (unsigned int)state->bw_ctx.bw.dcn.clk.dppclk_khz;
+	info->socClock					= (unsigned int)state->bw_ctx.bw.dcn.clk.socclk_khz;
+	info->dcfClockDeepSleep			= (unsigned int)state->bw_ctx.bw.dcn.clk.dcfclk_deep_sleep_khz;
+	info->fClock					= (unsigned int)state->bw_ctx.bw.dcn.clk.fclk_khz;
+	info->phyClock					= (unsigned int)state->bw_ctx.bw.dcn.clk.phyclk_khz;
 }
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_debug.c b/drivers/gpu/drm/amd/display/dc/core/dc_debug.c
index 73d049506618..5903e7822f98 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_debug.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_debug.c
@@ -351,19 +351,19 @@ void context_clock_trace(
 	DC_LOGGER_INIT(dc->ctx->logger);
 	CLOCK_TRACE("Current: dispclk_khz:%d  max_dppclk_khz:%d  dcfclk_khz:%d\n"
 			"dcfclk_deep_sleep_khz:%d  fclk_khz:%d  socclk_khz:%d\n",
-			context->bw.dcn.clk.dispclk_khz,
-			context->bw.dcn.clk.dppclk_khz,
-			context->bw.dcn.clk.dcfclk_khz,
-			context->bw.dcn.clk.dcfclk_deep_sleep_khz,
-			context->bw.dcn.clk.fclk_khz,
-			context->bw.dcn.clk.socclk_khz);
+			context->bw_ctx.bw.dcn.clk.dispclk_khz,
+			context->bw_ctx.bw.dcn.clk.dppclk_khz,
+			context->bw_ctx.bw.dcn.clk.dcfclk_khz,
+			context->bw_ctx.bw.dcn.clk.dcfclk_deep_sleep_khz,
+			context->bw_ctx.bw.dcn.clk.fclk_khz,
+			context->bw_ctx.bw.dcn.clk.socclk_khz);
 	CLOCK_TRACE("Calculated: dispclk_khz:%d  max_dppclk_khz:%d  dcfclk_khz:%d\n"
 			"dcfclk_deep_sleep_khz:%d  fclk_khz:%d  socclk_khz:%d\n",
-			context->bw.dcn.clk.dispclk_khz,
-			context->bw.dcn.clk.dppclk_khz,
-			context->bw.dcn.clk.dcfclk_khz,
-			context->bw.dcn.clk.dcfclk_deep_sleep_khz,
-			context->bw.dcn.clk.fclk_khz,
-			context->bw.dcn.clk.socclk_khz);
+			context->bw_ctx.bw.dcn.clk.dispclk_khz,
+			context->bw_ctx.bw.dcn.clk.dppclk_khz,
+			context->bw_ctx.bw.dcn.clk.dcfclk_khz,
+			context->bw_ctx.bw.dcn.clk.dcfclk_deep_sleep_khz,
+			context->bw_ctx.bw.dcn.clk.fclk_khz,
+			context->bw_ctx.bw.dcn.clk.socclk_khz);
 #endif
 }
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link.c b/drivers/gpu/drm/amd/display/dc/core/dc_link.c
index ea18e9c2d8ce..b37ecc3ede61 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_link.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_link.c
@@ -58,7 +58,6 @@
  ******************************************************************************/
 
 enum {
-	LINK_RATE_REF_FREQ_IN_MHZ = 27,
 	PEAK_FACTOR_X1000 = 1006,
 	/*
 	* Some receivers fail to train on first try and are good
@@ -515,6 +514,40 @@ static void link_disconnect_remap(struct dc_sink *prev_sink, struct dc_link *lin
 }
 
 
+static void read_edp_current_link_settings_on_detect(struct dc_link *link)
+{
+	union lane_count_set lane_count_set = { {0} };
+	uint8_t link_bw_set;
+	uint8_t link_rate_set;
+
+	// Read DPCD 00101h to find out the number of lanes currently set
+	core_link_read_dpcd(link, DP_LANE_COUNT_SET,
+			&lane_count_set.raw, sizeof(lane_count_set));
+	link->cur_link_settings.lane_count = lane_count_set.bits.LANE_COUNT_SET;
+
+	// Read DPCD 00100h to find if standard link rates are set
+	core_link_read_dpcd(link, DP_LINK_BW_SET,
+			&link_bw_set, sizeof(link_bw_set));
+
+	if (link_bw_set == 0) {
+		/* If standard link rates are not being used,
+		 * Read DPCD 00115h to find the link rate set used
+		 */
+		core_link_read_dpcd(link, DP_LINK_RATE_SET,
+				&link_rate_set, sizeof(link_rate_set));
+
+		if (link_rate_set < link->dpcd_caps.edp_supported_link_rates_count) {
+			link->cur_link_settings.link_rate =
+				link->dpcd_caps.edp_supported_link_rates[link_rate_set];
+			link->cur_link_settings.link_rate_set = link_rate_set;
+			link->cur_link_settings.use_link_rate_set = true;
+		}
+	} else {
+		link->cur_link_settings.link_rate = link_bw_set;
+		link->cur_link_settings.use_link_rate_set = false;
+	}
+}
+
 static bool detect_dp(
 	struct dc_link *link,
 	struct display_sink_capability *sink_caps,
@@ -640,7 +673,8 @@ bool dc_link_detect(struct dc_link *link, enum dc_detect_reason reason)
 	bool same_dpcd = true;
 	enum dc_connection_type new_connection_type = dc_connection_none;
 	DC_LOGGER_INIT(link->ctx->logger);
-	if (link->connector_signal == SIGNAL_TYPE_VIRTUAL)
+
+	if (dc_is_virtual_signal(link->connector_signal))
 		return false;
 
 	if (false == dc_link_detect_sink(link, &new_connection_type)) {
@@ -648,9 +682,14 @@ bool dc_link_detect(struct dc_link *link, enum dc_detect_reason reason)
 		return false;
 	}
 
-	if (link->connector_signal == SIGNAL_TYPE_EDP &&
-			link->local_sink)
-		return true;
+	if (link->connector_signal == SIGNAL_TYPE_EDP) {
+		/* On detect, we want to make sure current link settings are
+		 * up to date, especially if link was powered on by GOP.
+		 */
+		read_edp_current_link_settings_on_detect(link);
+		if (link->local_sink)
+			return true;
+	}
 
 	if (link->connector_signal == SIGNAL_TYPE_LVDS &&
 			link->local_sink)
@@ -720,9 +759,8 @@ bool dc_link_detect(struct dc_link *link, enum dc_detect_reason reason)
 					same_dpcd = false;
 			}
 			/* Active dongle plug in without display or downstream unplug*/
-			if (link->type == dc_connection_active_dongle
-					&& link->dpcd_caps.sink_count.
-					bits.SINK_COUNT == 0) {
+			if (link->type == dc_connection_active_dongle &&
+				link->dpcd_caps.sink_count.bits.SINK_COUNT == 0) {
 				if (prev_sink != NULL) {
 					/* Downstream unplug */
 					dc_sink_release(prev_sink);
@@ -1172,8 +1210,6 @@ static bool construct(
 		goto create_fail;
 	}
 
-
-
 	/* TODO: #DAL3 Implement id to str function.*/
 	LINK_INFO("Connector[%d] description:"
 			"signal %d\n",
@@ -1207,7 +1243,7 @@ static bool construct(
 	link->link_enc = link->dc->res_pool->funcs->link_enc_create(
 								&enc_init_data);
 
-	if( link->link_enc == NULL) {
+	if (link->link_enc == NULL) {
 		DC_ERROR("Failed to create link encoder!\n");
 		goto link_enc_create_fail;
 	}
@@ -1399,9 +1435,24 @@ static enum dc_status enable_link_dp(
 	/* get link settings for video mode timing */
 	decide_link_settings(stream, &link_settings);
 
+	if (pipe_ctx->stream->signal == SIGNAL_TYPE_EDP) {
+		/* If link settings are different than current and link already enabled
+		 * then need to disable before programming to new rate.
+		 */
+		if (link->link_status.link_active &&
+			(link->cur_link_settings.lane_count != link_settings.lane_count ||
+			 link->cur_link_settings.link_rate != link_settings.link_rate)) {
+			dp_disable_link_phy(link, pipe_ctx->stream->signal);
+		}
+
+		/*in case it is not on*/
+		link->dc->hwss.edp_power_control(link, true);
+		link->dc->hwss.edp_wait_for_hpd_ready(link, true);
+	}
+
 	pipe_ctx->stream_res.pix_clk_params.requested_sym_clk =
 			link_settings.link_rate * LINK_RATE_REF_FREQ_IN_KHZ;
-	state->dccg->funcs->update_clocks(state->dccg, state, false);
+	state->clk_mgr->funcs->update_clocks(state->clk_mgr, state, false);
 
 	dp_enable_link_phy(
 		link,
@@ -1442,15 +1493,9 @@ static enum dc_status enable_link_edp(
 		struct pipe_ctx *pipe_ctx)
 {
 	enum dc_status status;
-	struct dc_stream_state *stream = pipe_ctx->stream;
-	struct dc_link *link = stream->link;
-	/*in case it is not on*/
-	link->dc->hwss.edp_power_control(link, true);
-	link->dc->hwss.edp_wait_for_hpd_ready(link, true);
 
 	status = enable_link_dp(state, pipe_ctx);
 
-
 	return status;
 }
 
@@ -1466,14 +1511,14 @@ static enum dc_status enable_link_dp_mst(
 	if (link->cur_link_settings.lane_count != LANE_COUNT_UNKNOWN)
 		return DC_OK;
 
+	/* clear payload table */
+	dm_helpers_dp_mst_clear_payload_allocation_table(link->ctx, link);
+
 	/* to make sure the pending down rep can be processed
-	 * before clear payload table
+	 * before enabling the link
 	 */
 	dm_helpers_dp_mst_poll_pending_down_reply(link->ctx, link);
 
-	/* clear payload table */
-	dm_helpers_dp_mst_clear_payload_allocation_table(link->ctx, link);
-
 	/* set the sink to MST mode before enabling the link */
 	dp_enable_mst_on_sink(link, true);
 
@@ -1982,7 +2027,7 @@ static void enable_link_hdmi(struct pipe_ctx *pipe_ctx)
 			pipe_ctx->stream->signal,
 			stream->phy_pix_clk);
 
-	if (pipe_ctx->stream->signal == SIGNAL_TYPE_HDMI_TYPE_A)
+	if (dc_is_hdmi_signal(pipe_ctx->stream->signal))
 		dal_ddc_service_read_scdc_data(link->ddc);
 }
 
@@ -2074,11 +2119,28 @@ static void disable_link(struct dc_link *link, enum signal_type signal)
 	}
 }
 
+static uint32_t get_timing_pixel_clock_100hz(const struct dc_crtc_timing *timing)
+{
+
+	uint32_t pxl_clk = timing->pix_clk_100hz;
+
+	if (timing->pixel_encoding == PIXEL_ENCODING_YCBCR420)
+		pxl_clk /= 2;
+	else if (timing->pixel_encoding == PIXEL_ENCODING_YCBCR422)
+		pxl_clk = pxl_clk * 2 / 3;
+
+	if (timing->display_color_depth == COLOR_DEPTH_101010)
+		pxl_clk = pxl_clk * 10 / 8;
+	else if (timing->display_color_depth == COLOR_DEPTH_121212)
+		pxl_clk = pxl_clk * 12 / 8;
+
+	return pxl_clk;
+}
+
 static bool dp_active_dongle_validate_timing(
 		const struct dc_crtc_timing *timing,
 		const struct dpcd_caps *dpcd_caps)
 {
-	unsigned int required_pix_clk_100hz = timing->pix_clk_100hz;
 	const struct dc_dongle_caps *dongle_caps = &dpcd_caps->dongle_caps;
 
 	switch (dpcd_caps->dongle_type) {
@@ -2115,13 +2177,6 @@ static bool dp_active_dongle_validate_timing(
 		return false;
 	}
 
-
-	/* Check Color Depth and Pixel Clock */
-	if (timing->pixel_encoding == PIXEL_ENCODING_YCBCR420)
-		required_pix_clk_100hz /= 2;
-	else if (timing->pixel_encoding == PIXEL_ENCODING_YCBCR422)
-		required_pix_clk_100hz = required_pix_clk_100hz * 2 / 3;
-
 	switch (timing->display_color_depth) {
 	case COLOR_DEPTH_666:
 	case COLOR_DEPTH_888:
@@ -2130,14 +2185,11 @@ static bool dp_active_dongle_validate_timing(
 	case COLOR_DEPTH_101010:
 		if (dongle_caps->dp_hdmi_max_bpc < 10)
 			return false;
-		required_pix_clk_100hz = required_pix_clk_100hz * 10 / 8;
 		break;
 	case COLOR_DEPTH_121212:
 		if (dongle_caps->dp_hdmi_max_bpc < 12)
 			return false;
-		required_pix_clk_100hz = required_pix_clk_100hz * 12 / 8;
 		break;
-
 	case COLOR_DEPTH_141414:
 	case COLOR_DEPTH_161616:
 	default:
@@ -2145,7 +2197,7 @@ static bool dp_active_dongle_validate_timing(
 		return false;
 	}
 
-	if (required_pix_clk_100hz > (dongle_caps->dp_hdmi_max_pixel_clk * 10))
+	if (get_timing_pixel_clock_100hz(timing) > (dongle_caps->dp_hdmi_max_pixel_clk_in_khz * 10))
 		return false;
 
 	return true;
@@ -2166,7 +2218,7 @@ enum dc_status dc_link_validate_mode_timing(
 		return DC_OK;
 
 	/* Passive Dongle */
-	if (0 != max_pix_clk && timing->pix_clk_100hz > max_pix_clk)
+	if (max_pix_clk != 0 && get_timing_pixel_clock_100hz(timing) > max_pix_clk)
 		return DC_EXCEED_DONGLE_CAP;
 
 	/* Active Dongle*/
@@ -2284,14 +2336,13 @@ void core_link_resume(struct dc_link *link)
 
 static struct fixed31_32 get_pbn_per_slot(struct dc_stream_state *stream)
 {
-	struct dc_link_settings *link_settings =
-			&stream->link->cur_link_settings;
-	uint32_t link_rate_in_mbps =
-			link_settings->link_rate * LINK_RATE_REF_FREQ_IN_MHZ;
-	struct fixed31_32 mbps = dc_fixpt_from_int(
-			link_rate_in_mbps * link_settings->lane_count);
-
-	return dc_fixpt_div_int(mbps, 54);
+	struct fixed31_32 mbytes_per_sec;
+	uint32_t link_rate_in_mbytes_per_sec = dc_link_bandwidth_kbps(stream->link, &stream->link->cur_link_settings);
+	link_rate_in_mbytes_per_sec /= 8000; /* Kbits to MBytes */
+
+	mbytes_per_sec = dc_fixpt_from_int(link_rate_in_mbytes_per_sec);
+
+	return dc_fixpt_div_int(mbytes_per_sec, 54);
 }
 
 static int get_color_depth(enum dc_color_depth color_depth)
@@ -2316,7 +2367,7 @@ static struct fixed31_32 get_pbn_from_timing(struct pipe_ctx *pipe_ctx)
 	uint32_t denominator;
 
 	bpc = get_color_depth(pipe_ctx->stream_res.pix_clk_params.color_depth);
-	kbps = pipe_ctx->stream_res.pix_clk_params.requested_pix_clk_100hz / 10 * bpc * 3;
+	kbps = dc_bandwidth_in_kbps_from_timing(&pipe_ctx->stream->timing);
 
 	/*
 	 * margin 5300ppm + 300ppm ~ 0.6% as per spec, factor is 1.006
@@ -2551,12 +2602,12 @@ void core_link_enable_stream(
 		struct dc_state *state,
 		struct pipe_ctx *pipe_ctx)
 {
-	struct dc  *core_dc = pipe_ctx->stream->ctx->dc;
+	struct dc *core_dc = pipe_ctx->stream->ctx->dc;
 	struct dc_stream_state *stream = pipe_ctx->stream;
 	enum dc_status status;
 	DC_LOGGER_INIT(pipe_ctx->stream->ctx->logger);
 
-	if (pipe_ctx->stream->signal != SIGNAL_TYPE_VIRTUAL) {
+	if (!dc_is_virtual_signal(pipe_ctx->stream->signal)) {
 		stream->link->link_enc->funcs->setup(
 			stream->link->link_enc,
 			pipe_ctx->stream->signal);
@@ -2570,9 +2621,10 @@ void core_link_enable_stream(
 		pipe_ctx->stream_res.stream_enc->funcs->dp_set_stream_attribute(
 			pipe_ctx->stream_res.stream_enc,
 			&stream->timing,
-			stream->output_color_space);
+			stream->output_color_space,
+			stream->link->dpcd_caps.dprx_feature.bits.SST_SPLIT_SDP_CAP);
 
-	if (dc_is_hdmi_signal(pipe_ctx->stream->signal))
+	if (dc_is_hdmi_tmds_signal(pipe_ctx->stream->signal))
 		pipe_ctx->stream_res.stream_enc->funcs->hdmi_set_stream_attribute(
 			pipe_ctx->stream_res.stream_enc,
 			&stream->timing,
@@ -2736,3 +2788,49 @@ void dc_link_enable_hpd_filter(struct dc_link *link, bool enable)
 	}
 }
 
+uint32_t dc_bandwidth_in_kbps_from_timing(
+	const struct dc_crtc_timing *timing)
+{
+	uint32_t bits_per_channel = 0;
+	uint32_t kbps;
+
+	switch (timing->display_color_depth) {
+	case COLOR_DEPTH_666:
+		bits_per_channel = 6;
+		break;
+	case COLOR_DEPTH_888:
+		bits_per_channel = 8;
+		break;
+	case COLOR_DEPTH_101010:
+		bits_per_channel = 10;
+		break;
+	case COLOR_DEPTH_121212:
+		bits_per_channel = 12;
+		break;
+	case COLOR_DEPTH_141414:
+		bits_per_channel = 14;
+		break;
+	case COLOR_DEPTH_161616:
+		bits_per_channel = 16;
+		break;
+	default:
+		break;
+	}
+
+	ASSERT(bits_per_channel != 0);
+
+	kbps = timing->pix_clk_100hz / 10;
+	kbps *= bits_per_channel;
+
+	if (timing->flags.Y_ONLY != 1) {
+		/*Only YOnly make reduce bandwidth by 1/3 compares to RGB*/
+		kbps *= 3;
+		if (timing->pixel_encoding == PIXEL_ENCODING_YCBCR420)
+			kbps /= 2;
+		else if (timing->pixel_encoding == PIXEL_ENCODING_YCBCR422)
+			kbps = kbps * 2 / 3;
+	}
+
+	return kbps;
+
+}
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_ddc.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_ddc.c
index b7ee63cd8dc7..f02092a0dc76 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_link_ddc.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_ddc.c
@@ -573,12 +573,28 @@ bool dal_ddc_service_query_ddc_data(
 	return ret;
 }
 
-int dc_link_aux_transfer(struct ddc_service *ddc,
-		struct aux_payload *payload)
+/* dc_link_aux_transfer_raw() - Attempt to transfer
+ * the given aux payload.  This function does not perform
+ * retries or handle error states.  The reply is returned
+ * in the payload->reply and the result through
+ * *operation_result.  Returns the number of bytes transferred,
+ * or -1 on a failure.
+ */
+int dc_link_aux_transfer_raw(struct ddc_service *ddc,
+		struct aux_payload *payload,
+		enum aux_channel_operation_result *operation_result)
 {
-	return dce_aux_transfer(ddc, payload);
+	return dce_aux_transfer_raw(ddc, payload, operation_result);
 }
 
+/* dc_link_aux_transfer_with_retries() - Attempt to submit an
+ * aux payload, retrying on timeouts, defers, and busy states
+ * as outlined in the DP spec.  Returns true if the request
+ * was successful.
+ *
+ * Unless you want to implement your own retry semantics, this
+ * is probably the one you want.
+ */
 bool dc_link_aux_transfer_with_retries(struct ddc_service *ddc,
 		struct aux_payload *payload)
 {
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c
index 09d301216076..1ee544a32ebb 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c
@@ -93,12 +93,10 @@ static void dpcd_set_link_settings(
 	struct dc_link *link,
 	const struct link_training_settings *lt_settings)
 {
-	uint8_t rate = (uint8_t)
-	(lt_settings->link_settings.link_rate);
+	uint8_t rate;
 
 	union down_spread_ctrl downspread = { {0} };
 	union lane_count_set lane_count_set = { {0} };
-	uint8_t link_set_buffer[2];
 
 	downspread.raw = (uint8_t)
 	(lt_settings->link_settings.link_spread);
@@ -111,29 +109,42 @@ static void dpcd_set_link_settings(
 	lane_count_set.bits.POST_LT_ADJ_REQ_GRANTED =
 		link->dpcd_caps.max_ln_count.bits.POST_LT_ADJ_REQ_SUPPORTED;
 
-	link_set_buffer[0] = rate;
-	link_set_buffer[1] = lane_count_set.raw;
-
-	core_link_write_dpcd(link, DP_LINK_BW_SET,
-	link_set_buffer, 2);
 	core_link_write_dpcd(link, DP_DOWNSPREAD_CTRL,
 	&downspread.raw, sizeof(downspread));
 
+	core_link_write_dpcd(link, DP_LANE_COUNT_SET,
+	&lane_count_set.raw, 1);
+
 	if (link->dpcd_caps.dpcd_rev.raw >= DPCD_REV_14 &&
-		(link->dpcd_caps.link_rate_set >= 1 &&
-		link->dpcd_caps.link_rate_set <= 8)) {
+			lt_settings->link_settings.use_link_rate_set == true) {
+		rate = 0;
+		core_link_write_dpcd(link, DP_LINK_BW_SET, &rate, 1);
 		core_link_write_dpcd(link, DP_LINK_RATE_SET,
-		&link->dpcd_caps.link_rate_set, 1);
+				&lt_settings->link_settings.link_rate_set, 1);
+	} else {
+		rate = (uint8_t) (lt_settings->link_settings.link_rate);
+		core_link_write_dpcd(link, DP_LINK_BW_SET, &rate, 1);
 	}
 
-	DC_LOG_HW_LINK_TRAINING("%s\n %x rate = %x\n %x lane = %x\n %x spread = %x\n",
-		__func__,
-		DP_LINK_BW_SET,
-		lt_settings->link_settings.link_rate,
-		DP_LANE_COUNT_SET,
-		lt_settings->link_settings.lane_count,
-		DP_DOWNSPREAD_CTRL,
-		lt_settings->link_settings.link_spread);
+	if (rate) {
+		DC_LOG_HW_LINK_TRAINING("%s\n %x rate = %x\n %x lane = %x\n %x spread = %x\n",
+			__func__,
+			DP_LINK_BW_SET,
+			lt_settings->link_settings.link_rate,
+			DP_LANE_COUNT_SET,
+			lt_settings->link_settings.lane_count,
+			DP_DOWNSPREAD_CTRL,
+			lt_settings->link_settings.link_spread);
+	} else {
+		DC_LOG_HW_LINK_TRAINING("%s\n %x rate set = %x\n %x lane = %x\n %x spread = %x\n",
+			__func__,
+			DP_LINK_RATE_SET,
+			lt_settings->link_settings.link_rate_set,
+			DP_LANE_COUNT_SET,
+			lt_settings->link_settings.lane_count,
+			DP_DOWNSPREAD_CTRL,
+			lt_settings->link_settings.link_spread);
+	}
 
 }
 
@@ -952,6 +963,8 @@ enum link_training_result dc_link_dp_perform_link_training(
 
 	lt_settings.link_settings.link_rate = link_setting->link_rate;
 	lt_settings.link_settings.lane_count = link_setting->lane_count;
+	lt_settings.link_settings.use_link_rate_set = link_setting->use_link_rate_set;
+	lt_settings.link_settings.link_rate_set = link_setting->link_rate_set;
 
 	/*@todo[vdevulap] move SS to LS, should not be handled by displaypath*/
 
@@ -1075,7 +1088,7 @@ static struct dc_link_settings get_max_link_cap(struct dc_link *link)
 {
 	/* Set Default link settings */
 	struct dc_link_settings max_link_cap = {LANE_COUNT_FOUR, LINK_RATE_HIGH,
-			LINK_SPREAD_05_DOWNSPREAD_30KHZ};
+			LINK_SPREAD_05_DOWNSPREAD_30KHZ, false, 0};
 
 	/* Higher link settings based on feature supported */
 	if (link->link_enc->features.flags.bits.IS_HBR2_CAPABLE)
@@ -1520,69 +1533,6 @@ static bool decide_fallback_link_setting(
 	return true;
 }
 
-static uint32_t bandwidth_in_kbps_from_timing(
-	const struct dc_crtc_timing *timing)
-{
-	uint32_t bits_per_channel = 0;
-	uint32_t kbps;
-
-	switch (timing->display_color_depth) {
-	case COLOR_DEPTH_666:
-		bits_per_channel = 6;
-		break;
-	case COLOR_DEPTH_888:
-		bits_per_channel = 8;
-		break;
-	case COLOR_DEPTH_101010:
-		bits_per_channel = 10;
-		break;
-	case COLOR_DEPTH_121212:
-		bits_per_channel = 12;
-		break;
-	case COLOR_DEPTH_141414:
-		bits_per_channel = 14;
-		break;
-	case COLOR_DEPTH_161616:
-		bits_per_channel = 16;
-		break;
-	default:
-		break;
-	}
-
-	ASSERT(bits_per_channel != 0);
-
-	kbps = timing->pix_clk_100hz / 10;
-	kbps *= bits_per_channel;
-
-	if (timing->flags.Y_ONLY != 1) {
-		/*Only YOnly make reduce bandwidth by 1/3 compares to RGB*/
-		kbps *= 3;
-		if (timing->pixel_encoding == PIXEL_ENCODING_YCBCR420)
-			kbps /= 2;
-		else if (timing->pixel_encoding == PIXEL_ENCODING_YCBCR422)
-			kbps = kbps * 2 / 3;
-	}
-
-	return kbps;
-
-}
-
-static uint32_t bandwidth_in_kbps_from_link_settings(
-	const struct dc_link_settings *link_setting)
-{
-	uint32_t link_rate_in_kbps = link_setting->link_rate *
-		LINK_RATE_REF_FREQ_IN_KHZ;
-
-	uint32_t lane_count  = link_setting->lane_count;
-	uint32_t kbps = link_rate_in_kbps;
-
-	kbps *= lane_count;
-	kbps *= 8;   /* 8 bits per byte*/
-
-	return kbps;
-
-}
-
 bool dp_validate_mode_timing(
 	struct dc_link *link,
 	const struct dc_crtc_timing *timing)
@@ -1598,8 +1548,7 @@ bool dp_validate_mode_timing(
 		timing->v_addressable == (uint32_t) 480)
 		return true;
 
-	/* We always use verified link settings */
-	link_setting = &link->verified_link_cap;
+	link_setting = dc_link_get_link_cap(link);
 
 	/* TODO: DYNAMIC_VALIDATION needs to be implemented */
 	/*if (flags.DYNAMIC_VALIDATION == 1 &&
@@ -1607,8 +1556,8 @@ bool dp_validate_mode_timing(
 		link_setting = &link->verified_link_cap;
 	*/
 
-	req_bw = bandwidth_in_kbps_from_timing(timing);
-	max_bw = bandwidth_in_kbps_from_link_settings(link_setting);
+	req_bw = dc_bandwidth_in_kbps_from_timing(timing);
+	max_bw = dc_link_bandwidth_kbps(link, link_setting);
 
 	if (req_bw <= max_bw) {
 		/* remember the biggest mode here, during
@@ -1629,58 +1578,78 @@ bool dp_validate_mode_timing(
 		return false;
 }
 
-void decide_link_settings(struct dc_stream_state *stream,
-	struct dc_link_settings *link_setting)
+static bool decide_dp_link_settings(struct dc_link *link, struct dc_link_settings *link_setting, uint32_t req_bw)
 {
-
 	struct dc_link_settings initial_link_setting = {
-		LANE_COUNT_ONE, LINK_RATE_LOW, LINK_SPREAD_DISABLED};
+		LANE_COUNT_ONE, LINK_RATE_LOW, LINK_SPREAD_DISABLED, false, 0};
 	struct dc_link_settings current_link_setting =
 			initial_link_setting;
-	struct dc_link *link;
-	uint32_t req_bw;
 	uint32_t link_bw;
 
-	req_bw = bandwidth_in_kbps_from_timing(&stream->timing);
-
-	link = stream->link;
-
-	/* if preferred is specified through AMDDP, use it, if it's enough
-	 * to drive the mode
+	/* search for the minimum link setting that:
+	 * 1. is supported according to the link training result
+	 * 2. could support the b/w requested by the timing
 	 */
-	if (link->preferred_link_setting.lane_count !=
-			LANE_COUNT_UNKNOWN &&
-			link->preferred_link_setting.link_rate !=
-					LINK_RATE_UNKNOWN) {
-		*link_setting =  link->preferred_link_setting;
-		return;
-	}
+	while (current_link_setting.link_rate <=
+			link->verified_link_cap.link_rate) {
+		link_bw = dc_link_bandwidth_kbps(
+				link,
+				&current_link_setting);
+		if (req_bw <= link_bw) {
+			*link_setting = current_link_setting;
+			return true;
+		}
 
-	/* MST doesn't perform link training for now
-	 * TODO: add MST specific link training routine
-	 */
-	if (stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST) {
-		*link_setting = link->verified_link_cap;
-		return;
+		if (current_link_setting.lane_count <
+				link->verified_link_cap.lane_count) {
+			current_link_setting.lane_count =
+					increase_lane_count(
+							current_link_setting.lane_count);
+		} else {
+			current_link_setting.link_rate =
+					increase_link_rate(
+							current_link_setting.link_rate);
+			current_link_setting.lane_count =
+					initial_link_setting.lane_count;
+		}
 	}
 
-	/* EDP use the link cap setting */
-	if (link->connector_signal == SIGNAL_TYPE_EDP) {
+	return false;
+}
+
+static bool decide_edp_link_settings(struct dc_link *link, struct dc_link_settings *link_setting, uint32_t req_bw)
+{
+	struct dc_link_settings initial_link_setting;
+	struct dc_link_settings current_link_setting;
+	uint32_t link_bw;
+
+	if (link->dpcd_caps.dpcd_rev.raw < DPCD_REV_14 ||
+			link->dpcd_caps.edp_supported_link_rates_count == 0 ||
+			link->dc->config.optimize_edp_link_rate == false) {
 		*link_setting = link->verified_link_cap;
-		return;
+		return true;
 	}
 
+	memset(&initial_link_setting, 0, sizeof(initial_link_setting));
+	initial_link_setting.lane_count = LANE_COUNT_ONE;
+	initial_link_setting.link_rate = link->dpcd_caps.edp_supported_link_rates[0];
+	initial_link_setting.link_spread = LINK_SPREAD_DISABLED;
+	initial_link_setting.use_link_rate_set = true;
+	initial_link_setting.link_rate_set = 0;
+	current_link_setting = initial_link_setting;
+
 	/* search for the minimum link setting that:
 	 * 1. is supported according to the link training result
 	 * 2. could support the b/w requested by the timing
 	 */
 	while (current_link_setting.link_rate <=
 			link->verified_link_cap.link_rate) {
-		link_bw = bandwidth_in_kbps_from_link_settings(
+		link_bw = dc_link_bandwidth_kbps(
+				link,
 				&current_link_setting);
 		if (req_bw <= link_bw) {
 			*link_setting = current_link_setting;
-			return;
+			return true;
 		}
 
 		if (current_link_setting.lane_count <
@@ -1689,13 +1658,53 @@ void decide_link_settings(struct dc_stream_state *stream,
 					increase_lane_count(
 							current_link_setting.lane_count);
 		} else {
-			current_link_setting.link_rate =
-					increase_link_rate(
-							current_link_setting.link_rate);
-			current_link_setting.lane_count =
-					initial_link_setting.lane_count;
+			if (current_link_setting.link_rate_set < link->dpcd_caps.edp_supported_link_rates_count) {
+				current_link_setting.link_rate_set++;
+				current_link_setting.link_rate =
+					link->dpcd_caps.edp_supported_link_rates[current_link_setting.link_rate_set];
+				current_link_setting.lane_count =
+									initial_link_setting.lane_count;
+			} else
+				break;
 		}
 	}
+	return false;
+}
+
+void decide_link_settings(struct dc_stream_state *stream,
+	struct dc_link_settings *link_setting)
+{
+	struct dc_link *link;
+	uint32_t req_bw;
+
+	req_bw = dc_bandwidth_in_kbps_from_timing(&stream->timing);
+
+	link = stream->link;
+
+	/* if preferred is specified through AMDDP, use it, if it's enough
+	 * to drive the mode
+	 */
+	if (link->preferred_link_setting.lane_count !=
+			LANE_COUNT_UNKNOWN &&
+			link->preferred_link_setting.link_rate !=
+					LINK_RATE_UNKNOWN) {
+		*link_setting =  link->preferred_link_setting;
+		return;
+	}
+
+	/* MST doesn't perform link training for now
+	 * TODO: add MST specific link training routine
+	 */
+	if (stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST) {
+		*link_setting = link->verified_link_cap;
+		return;
+	}
+
+	if (link->connector_signal == SIGNAL_TYPE_EDP) {
+		if (decide_edp_link_settings(link, link_setting, req_bw))
+			return;
+	} else if (decide_dp_link_settings(link, link_setting, req_bw))
+		return;
 
 	BREAK_TO_DEBUGGER();
 	ASSERT(link->verified_link_cap.lane_count != LANE_COUNT_UNKNOWN);
@@ -2155,11 +2164,7 @@ bool is_mst_supported(struct dc_link *link)
 
 bool is_dp_active_dongle(const struct dc_link *link)
 {
-	enum display_dongle_type dongle_type = link->dpcd_caps.dongle_type;
-
-	return (dongle_type == DISPLAY_DONGLE_DP_VGA_CONVERTER) ||
-			(dongle_type == DISPLAY_DONGLE_DP_DVI_CONVERTER) ||
-			(dongle_type == DISPLAY_DONGLE_DP_HDMI_CONVERTER);
+	return link->dpcd_caps.is_branch_dev;
 }
 
 static int translate_dpcd_max_bpc(enum dpcd_downstream_port_max_bpc bpc)
@@ -2180,6 +2185,30 @@ static int translate_dpcd_max_bpc(enum dpcd_downstream_port_max_bpc bpc)
 	return -1;
 }
 
+static void read_dp_device_vendor_id(struct dc_link *link)
+{
+	struct dp_device_vendor_id dp_id;
+
+	/* read IEEE branch device id */
+	core_link_read_dpcd(
+		link,
+		DP_BRANCH_OUI,
+		(uint8_t *)&dp_id,
+		sizeof(dp_id));
+
+	link->dpcd_caps.branch_dev_id =
+		(dp_id.ieee_oui[0] << 16) +
+		(dp_id.ieee_oui[1] << 8) +
+		dp_id.ieee_oui[2];
+
+	memmove(
+		link->dpcd_caps.branch_dev_name,
+		dp_id.ieee_device_id,
+		sizeof(dp_id.ieee_device_id));
+}
+
+
+
 static void get_active_converter_info(
 	uint8_t data, struct dc_link *link)
 {
@@ -2193,6 +2222,9 @@ static void get_active_converter_info(
 		return;
 	}
 
+	/* DPCD 0x5 bit 0 = 1, it indicate it's branch device */
+	link->dpcd_caps.is_branch_dev = ds_port.fields.PORT_PRESENT;
+
 	switch (ds_port.fields.PORT_TYPE) {
 	case DOWNSTREAM_VGA:
 		link->dpcd_caps.dongle_type = DISPLAY_DONGLE_DP_VGA_CONVERTER;
@@ -2234,8 +2266,8 @@ static void get_active_converter_info(
 					hdmi_caps = {.raw = det_caps[3] };
 				union dwnstream_port_caps_byte2
 					hdmi_color_caps = {.raw = det_caps[2] };
-				link->dpcd_caps.dongle_caps.dp_hdmi_max_pixel_clk =
-					det_caps[1] * 25000;
+				link->dpcd_caps.dongle_caps.dp_hdmi_max_pixel_clk_in_khz =
+					det_caps[1] * 2500;
 
 				link->dpcd_caps.dongle_caps.is_dp_hdmi_s3d_converter =
 					hdmi_caps.bits.FRAME_SEQ_TO_FRAME_PACK;
@@ -2252,7 +2284,7 @@ static void get_active_converter_info(
 					translate_dpcd_max_bpc(
 						hdmi_color_caps.bits.MAX_BITS_PER_COLOR_COMPONENT);
 
-				if (link->dpcd_caps.dongle_caps.dp_hdmi_max_pixel_clk != 0)
+				if (link->dpcd_caps.dongle_caps.dp_hdmi_max_pixel_clk_in_khz != 0)
 					link->dpcd_caps.dongle_caps.extendedCapValid = true;
 			}
 
@@ -2263,27 +2295,6 @@ static void get_active_converter_info(
 	ddc_service_set_dongle_type(link->ddc, link->dpcd_caps.dongle_type);
 
 	{
-		struct dp_device_vendor_id dp_id;
-
-		/* read IEEE branch device id */
-		core_link_read_dpcd(
-			link,
-			DP_BRANCH_OUI,
-			(uint8_t *)&dp_id,
-			sizeof(dp_id));
-
-		link->dpcd_caps.branch_dev_id =
-			(dp_id.ieee_oui[0] << 16) +
-			(dp_id.ieee_oui[1] << 8) +
-			dp_id.ieee_oui[2];
-
-		memmove(
-			link->dpcd_caps.branch_dev_name,
-			dp_id.ieee_device_id,
-			sizeof(dp_id.ieee_device_id));
-	}
-
-	{
 		struct dp_sink_hw_fw_revision dp_hw_fw_revision;
 
 		core_link_read_dpcd(
@@ -2347,6 +2358,10 @@ static bool retrieve_link_cap(struct dc_link *link)
 {
 	uint8_t dpcd_data[DP_ADAPTER_CAP - DP_DPCD_REV + 1];
 
+	/*Only need to read 1 byte starting from DP_DPRX_FEATURE_ENUMERATION_LIST.
+	 */
+	uint8_t dpcd_dprx_data = '\0';
+
 	struct dp_device_vendor_id sink_id;
 	union down_stream_port_count down_strm_port_count;
 	union edp_configuration_cap edp_config_cap;
@@ -2383,7 +2398,10 @@ static bool retrieve_link_cap(struct dc_link *link)
 		aux_rd_interval.raw =
 			dpcd_data[DP_TRAINING_AUX_RD_INTERVAL];
 
-		if (aux_rd_interval.bits.EXT_RECIEVER_CAP_FIELD_PRESENT == 1) {
+		link->dpcd_caps.ext_receiver_cap_field_present =
+				aux_rd_interval.bits.EXT_RECEIVER_CAP_FIELD_PRESENT == 1 ? true:false;
+
+		if (aux_rd_interval.bits.EXT_RECEIVER_CAP_FIELD_PRESENT == 1) {
 			uint8_t ext_cap_data[16];
 
 			memset(ext_cap_data, '\0', sizeof(ext_cap_data));
@@ -2404,11 +2422,44 @@ static bool retrieve_link_cap(struct dc_link *link)
 	}
 
 	link->dpcd_caps.dpcd_rev.raw =
-		dpcd_data[DP_DPCD_REV - DP_DPCD_REV];
+			dpcd_data[DP_DPCD_REV - DP_DPCD_REV];
+
+	if (link->dpcd_caps.dpcd_rev.raw >= 0x14) {
+		for (i = 0; i < read_dpcd_retry_cnt; i++) {
+			status = core_link_read_dpcd(
+					link,
+					DP_DPRX_FEATURE_ENUMERATION_LIST,
+					&dpcd_dprx_data,
+					sizeof(dpcd_dprx_data));
+			if (status == DC_OK)
+				break;
+		}
+
+		link->dpcd_caps.dprx_feature.raw = dpcd_dprx_data;
+
+		if (status != DC_OK)
+			dm_error("%s: Read DPRX caps data failed.\n", __func__);
+	}
+
+	else {
+		link->dpcd_caps.dprx_feature.raw = 0;
+	}
+
+
+	/* Error condition checking...
+	 * It is impossible for Sink to report Max Lane Count = 0.
+	 * It is possible for Sink to report Max Link Rate = 0, if it is
+	 * an eDP device that is reporting specialized link rates in the
+	 * SUPPORTED_LINK_RATE table.
+	 */
+	if (dpcd_data[DP_MAX_LANE_COUNT - DP_DPCD_REV] == 0)
+		return false;
 
 	ds_port.byte = dpcd_data[DP_DOWNSTREAMPORT_PRESENT -
 				 DP_DPCD_REV];
 
+	read_dp_device_vendor_id(link);
+
 	get_active_converter_info(ds_port.byte, link);
 
 	dp_wa_power_up_0010FA(link, dpcd_data, sizeof(dpcd_data));
@@ -2536,31 +2587,31 @@ enum dc_link_rate linkRateInKHzToLinkRateMultiplier(uint32_t link_rate_in_khz)
 
 void detect_edp_sink_caps(struct dc_link *link)
 {
-	uint8_t supported_link_rates[16] = {0};
+	uint8_t supported_link_rates[16];
 	uint32_t entry;
 	uint32_t link_rate_in_khz;
 	enum dc_link_rate link_rate = LINK_RATE_UNKNOWN;
 
 	retrieve_link_cap(link);
+	link->dpcd_caps.edp_supported_link_rates_count = 0;
+	memset(supported_link_rates, 0, sizeof(supported_link_rates));
 
-	if (link->dpcd_caps.dpcd_rev.raw >= DPCD_REV_14) {
+	if (link->dpcd_caps.dpcd_rev.raw >= DPCD_REV_14 &&
+			link->dc->config.optimize_edp_link_rate) {
 		// Read DPCD 00010h - 0001Fh 16 bytes at one shot
 		core_link_read_dpcd(link, DP_SUPPORTED_LINK_RATES,
 							supported_link_rates, sizeof(supported_link_rates));
 
-		link->dpcd_caps.link_rate_set = 0;
 		for (entry = 0; entry < 16; entry += 2) {
 			// DPCD register reports per-lane link rate = 16-bit link rate capability
-			// value X 200 kHz. Need multipler to find link rate in kHz.
+			// value X 200 kHz. Need multiplier to find link rate in kHz.
 			link_rate_in_khz = (supported_link_rates[entry+1] * 0x100 +
 										supported_link_rates[entry]) * 200;
 
 			if (link_rate_in_khz != 0) {
 				link_rate = linkRateInKHzToLinkRateMultiplier(link_rate_in_khz);
-				if (link->reported_link_cap.link_rate < link_rate) {
-					link->reported_link_cap.link_rate = link_rate;
-					link->dpcd_caps.link_rate_set = entry;
-				}
+				link->dpcd_caps.edp_supported_link_rates[link->dpcd_caps.edp_supported_link_rates_count] = link_rate;
+				link->dpcd_caps.edp_supported_link_rates_count++;
 			}
 		}
 	}
@@ -2601,6 +2652,7 @@ static void set_crtc_test_pattern(struct dc_link *link,
 	enum dc_color_depth color_depth = pipe_ctx->
 		stream->timing.display_color_depth;
 	struct bit_depth_reduction_params params;
+	struct output_pixel_processor *opp = pipe_ctx->stream_res.opp;
 
 	memset(&params, 0, sizeof(params));
 
@@ -2640,8 +2692,7 @@ static void set_crtc_test_pattern(struct dc_link *link,
 	{
 		/* disable bit depth reduction */
 		pipe_ctx->stream->bit_depth_params = params;
-		pipe_ctx->stream_res.opp->funcs->
-			opp_program_bit_depth_reduction(pipe_ctx->stream_res.opp, &params);
+		opp->funcs->opp_program_bit_depth_reduction(opp, &params);
 		if (pipe_ctx->stream_res.tg->funcs->set_test_pattern)
 			pipe_ctx->stream_res.tg->funcs->set_test_pattern(pipe_ctx->stream_res.tg,
 				controller_test_pattern, color_depth);
@@ -2650,11 +2701,9 @@ static void set_crtc_test_pattern(struct dc_link *link,
 	case DP_TEST_PATTERN_VIDEO_MODE:
 	{
 		/* restore bitdepth reduction */
-		resource_build_bit_depth_reduction_params(pipe_ctx->stream,
-					&params);
+		resource_build_bit_depth_reduction_params(pipe_ctx->stream, &params);
 		pipe_ctx->stream->bit_depth_params = params;
-		pipe_ctx->stream_res.opp->funcs->
-			opp_program_bit_depth_reduction(pipe_ctx->stream_res.opp, &params);
+		opp->funcs->opp_program_bit_depth_reduction(opp, &params);
 		if (pipe_ctx->stream_res.tg->funcs->set_test_pattern)
 			pipe_ctx->stream_res.tg->funcs->set_test_pattern(pipe_ctx->stream_res.tg,
 				CONTROLLER_DP_TEST_PATTERN_VIDEOMODE,
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_hwss.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_hwss.c
index f7f7515f65f4..b0dea759cd86 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_link_hwss.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_hwss.c
@@ -58,6 +58,8 @@ void dp_enable_link_phy(
 	const struct dc_link_settings *link_settings)
 {
 	struct link_encoder *link_enc = link->link_enc;
+	struct dc  *core_dc = link->ctx->dc;
+	struct dmcu *dmcu = core_dc->res_pool->dmcu;
 
 	struct pipe_ctx *pipes =
 			link->dc->current_state->res_ctx.pipe_ctx;
@@ -84,6 +86,9 @@ void dp_enable_link_phy(
 		}
 	}
 
+	if (dmcu != NULL && dmcu->funcs->lock_phy)
+		dmcu->funcs->lock_phy(dmcu);
+
 	if (dc_is_dp_sst_signal(signal)) {
 		link_enc->funcs->enable_dp_output(
 						link_enc,
@@ -95,6 +100,10 @@ void dp_enable_link_phy(
 						link_settings,
 						clock_source);
 	}
+
+	if (dmcu != NULL && dmcu->funcs->unlock_phy)
+		dmcu->funcs->unlock_phy(dmcu);
+
 	link->cur_link_settings = *link_settings;
 
 	dp_receiver_power_ctrl(link, true);
@@ -150,15 +159,25 @@ bool edp_receiver_ready_T7(struct dc_link *link)
 
 void dp_disable_link_phy(struct dc_link *link, enum signal_type signal)
 {
+	struct dc  *core_dc = link->ctx->dc;
+	struct dmcu *dmcu = core_dc->res_pool->dmcu;
+
 	if (!link->wa_flags.dp_keep_receiver_powered)
 		dp_receiver_power_ctrl(link, false);
 
 	if (signal == SIGNAL_TYPE_EDP) {
 		link->link_enc->funcs->disable_output(link->link_enc, signal);
 		link->dc->hwss.edp_power_control(link, false);
-	} else
+	} else {
+		if (dmcu != NULL && dmcu->funcs->lock_phy)
+			dmcu->funcs->lock_phy(dmcu);
+
 		link->link_enc->funcs->disable_output(link->link_enc, signal);
 
+		if (dmcu != NULL && dmcu->funcs->unlock_phy)
+			dmcu->funcs->unlock_phy(dmcu);
+	}
+
 	/* Clear current link setting.*/
 	memset(&link->cur_link_settings, 0,
 			sizeof(link->cur_link_settings));
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
index 349ab8017776..eac7186e4f08 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
@@ -31,6 +31,8 @@
 #include "opp.h"
 #include "timing_generator.h"
 #include "transform.h"
+#include "dccg.h"
+#include "dchubbub.h"
 #include "dpp.h"
 #include "core_types.h"
 #include "set_mode_types.h"
@@ -104,44 +106,43 @@ enum dce_version resource_parse_asic_id(struct hw_asic_id asic_id)
 	return dc_version;
 }
 
-struct resource_pool *dc_create_resource_pool(
-				struct dc  *dc,
-				int num_virtual_links,
-				enum dce_version dc_version,
-				struct hw_asic_id asic_id)
+struct resource_pool *dc_create_resource_pool(struct dc  *dc,
+					      const struct dc_init_data *init_data,
+					      enum dce_version dc_version)
 {
 	struct resource_pool *res_pool = NULL;
 
 	switch (dc_version) {
 	case DCE_VERSION_8_0:
 		res_pool = dce80_create_resource_pool(
-			num_virtual_links, dc);
+				init_data->num_virtual_links, dc);
 		break;
 	case DCE_VERSION_8_1:
 		res_pool = dce81_create_resource_pool(
-			num_virtual_links, dc);
+				init_data->num_virtual_links, dc);
 		break;
 	case DCE_VERSION_8_3:
 		res_pool = dce83_create_resource_pool(
-			num_virtual_links, dc);
+				init_data->num_virtual_links, dc);
 		break;
 	case DCE_VERSION_10_0:
 		res_pool = dce100_create_resource_pool(
-				num_virtual_links, dc);
+				init_data->num_virtual_links, dc);
 		break;
 	case DCE_VERSION_11_0:
 		res_pool = dce110_create_resource_pool(
-			num_virtual_links, dc, asic_id);
+				init_data->num_virtual_links, dc,
+				init_data->asic_id);
 		break;
 	case DCE_VERSION_11_2:
 	case DCE_VERSION_11_22:
 		res_pool = dce112_create_resource_pool(
-			num_virtual_links, dc);
+				init_data->num_virtual_links, dc);
 		break;
 	case DCE_VERSION_12_0:
 	case DCE_VERSION_12_1:
 		res_pool = dce120_create_resource_pool(
-			num_virtual_links, dc);
+				init_data->num_virtual_links, dc);
 		break;
 
 #if defined(CONFIG_DRM_AMD_DC_DCN1_0)
@@ -149,8 +150,7 @@ struct resource_pool *dc_create_resource_pool(
 #if defined(CONFIG_DRM_AMD_DC_DCN1_01)
 	case DCN_VERSION_1_01:
 #endif
-		res_pool = dcn10_create_resource_pool(
-				num_virtual_links, dc);
+		res_pool = dcn10_create_resource_pool(init_data, dc);
 		break;
 #endif
 
@@ -163,7 +163,28 @@ struct resource_pool *dc_create_resource_pool(
 
 		if (dc->ctx->dc_bios->funcs->get_firmware_info(
 				dc->ctx->dc_bios, &fw_info) == BP_RESULT_OK) {
-				res_pool->ref_clock_inKhz = fw_info.pll_info.crystal_frequency;
+				res_pool->ref_clocks.xtalin_clock_inKhz = fw_info.pll_info.crystal_frequency;
+
+				if (IS_FPGA_MAXIMUS_DC(dc->ctx->dce_environment)) {
+					// On FPGA these dividers are currently not configured by GDB
+					res_pool->ref_clocks.dccg_ref_clock_inKhz = res_pool->ref_clocks.xtalin_clock_inKhz;
+					res_pool->ref_clocks.dchub_ref_clock_inKhz = res_pool->ref_clocks.xtalin_clock_inKhz;
+				} else if (res_pool->dccg && res_pool->hubbub) {
+					// If DCCG reference frequency cannot be determined (usually means not set to xtalin) then this is a critical error
+					// as this value must be known for DCHUB programming
+					(res_pool->dccg->funcs->get_dccg_ref_freq)(res_pool->dccg,
+							fw_info.pll_info.crystal_frequency,
+							&res_pool->ref_clocks.dccg_ref_clock_inKhz);
+
+					// Similarly, if DCHUB reference frequency cannot be determined, then it is also a critical error
+					(res_pool->hubbub->funcs->get_dchub_ref_freq)(res_pool->hubbub,
+							res_pool->ref_clocks.dccg_ref_clock_inKhz,
+							&res_pool->ref_clocks.dchub_ref_clock_inKhz);
+				} else {
+					// Not all ASICs have DCCG sw component
+					res_pool->ref_clocks.dccg_ref_clock_inKhz = res_pool->ref_clocks.xtalin_clock_inKhz;
+					res_pool->ref_clocks.dchub_ref_clock_inKhz = res_pool->ref_clocks.xtalin_clock_inKhz;
+				}
 			} else
 				ASSERT_CRITICAL(false);
 	}
@@ -260,6 +281,7 @@ bool resource_construct(
 			pool->stream_enc_count++;
 		}
 	}
+
 	dc->caps.dynamic_audio = false;
 	if (pool->audio_count < pool->stream_enc_count) {
 		dc->caps.dynamic_audio = true;
@@ -1014,24 +1036,60 @@ enum dc_status resource_build_scaling_params_for_context(
 
 struct pipe_ctx *find_idle_secondary_pipe(
 		struct resource_context *res_ctx,
-		const struct resource_pool *pool)
+		const struct resource_pool *pool,
+		const struct pipe_ctx *primary_pipe)
 {
 	int i;
 	struct pipe_ctx *secondary_pipe = NULL;
 
 	/*
-	 * search backwards for the second pipe to keep pipe
-	 * assignment more consistent
+	 * We add a preferred pipe mapping to avoid the chance that
+	 * MPCCs already in use will need to be reassigned to other trees.
+	 * For example, if we went with the strict, assign backwards logic:
+	 *
+	 * (State 1)
+	 * Display A on, no surface, top pipe = 0
+	 * Display B on, no surface, top pipe = 1
+	 *
+	 * (State 2)
+	 * Display A on, no surface, top pipe = 0
+	 * Display B on, surface enable, top pipe = 1, bottom pipe = 5
+	 *
+	 * (State 3)
+	 * Display A on, surface enable, top pipe = 0, bottom pipe = 5
+	 * Display B on, surface enable, top pipe = 1, bottom pipe = 4
+	 *
+	 * The state 2->3 transition requires remapping MPCC 5 from display B
+	 * to display A.
+	 *
+	 * However, with the preferred pipe logic, state 2 would look like:
+	 *
+	 * (State 2)
+	 * Display A on, no surface, top pipe = 0
+	 * Display B on, surface enable, top pipe = 1, bottom pipe = 4
+	 *
+	 * This would then cause 2->3 to not require remapping any MPCCs.
 	 */
-
-	for (i = pool->pipe_count - 1; i >= 0; i--) {
-		if (res_ctx->pipe_ctx[i].stream == NULL) {
-			secondary_pipe = &res_ctx->pipe_ctx[i];
-			secondary_pipe->pipe_idx = i;
-			break;
+	if (primary_pipe) {
+		int preferred_pipe_idx = (pool->pipe_count - 1) - primary_pipe->pipe_idx;
+		if (res_ctx->pipe_ctx[preferred_pipe_idx].stream == NULL) {
+			secondary_pipe = &res_ctx->pipe_ctx[preferred_pipe_idx];
+			secondary_pipe->pipe_idx = preferred_pipe_idx;
 		}
 	}
 
+	/*
+	 * search backwards for the second pipe to keep pipe
+	 * assignment more consistent
+	 */
+	if (!secondary_pipe)
+		for (i = pool->pipe_count - 1; i >= 0; i--) {
+			if (res_ctx->pipe_ctx[i].stream == NULL) {
+				secondary_pipe = &res_ctx->pipe_ctx[i];
+				secondary_pipe->pipe_idx = i;
+				break;
+			}
+		}
 
 	return secondary_pipe;
 }
@@ -1214,6 +1272,9 @@ bool dc_add_plane_to_context(
 		free_pipe->clock_source = tail_pipe->clock_source;
 		free_pipe->top_pipe = tail_pipe;
 		tail_pipe->bottom_pipe = free_pipe;
+	} else if (free_pipe->bottom_pipe && free_pipe->bottom_pipe->plane_state == NULL) {
+		ASSERT(free_pipe->bottom_pipe->stream_res.opp != free_pipe->stream_res.opp);
+		free_pipe->bottom_pipe->plane_state = plane_state;
 	}
 
 	/* assign new surfaces*/
@@ -1224,6 +1285,35 @@ bool dc_add_plane_to_context(
 	return true;
 }
 
+struct pipe_ctx *dc_res_get_odm_bottom_pipe(struct pipe_ctx *pipe_ctx)
+{
+	struct pipe_ctx *bottom_pipe = pipe_ctx->bottom_pipe;
+
+	/* ODM should only be updated once per otg */
+	if (pipe_ctx->top_pipe)
+		return NULL;
+
+	while (bottom_pipe) {
+		if (bottom_pipe->stream_res.opp != pipe_ctx->stream_res.opp)
+			break;
+		bottom_pipe = bottom_pipe->bottom_pipe;
+	}
+
+	return bottom_pipe;
+}
+
+bool dc_res_is_odm_head_pipe(struct pipe_ctx *pipe_ctx)
+{
+	struct pipe_ctx *top_pipe = pipe_ctx->top_pipe;
+
+	if (!top_pipe)
+		return false;
+	if (top_pipe && top_pipe->stream_res.opp == pipe_ctx->stream_res.opp)
+		return false;
+
+	return true;
+}
+
 bool dc_remove_plane_from_context(
 		const struct dc *dc,
 		struct dc_stream_state *stream,
@@ -1247,10 +1337,14 @@ bool dc_remove_plane_from_context(
 
 	/* release pipe for plane*/
 	for (i = pool->pipe_count - 1; i >= 0; i--) {
-		struct pipe_ctx *pipe_ctx;
+		struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
 
-		if (context->res_ctx.pipe_ctx[i].plane_state == plane_state) {
-			pipe_ctx = &context->res_ctx.pipe_ctx[i];
+		if (pipe_ctx->plane_state == plane_state) {
+			if (dc_res_is_odm_head_pipe(pipe_ctx)) {
+				pipe_ctx->plane_state = NULL;
+				pipe_ctx->bottom_pipe = NULL;
+				continue;
+			}
 
 			if (pipe_ctx->top_pipe)
 				pipe_ctx->top_pipe->bottom_pipe = pipe_ctx->bottom_pipe;
@@ -1268,8 +1362,9 @@ bool dc_remove_plane_from_context(
 			 */
 			if (!pipe_ctx->top_pipe) {
 				pipe_ctx->plane_state = NULL;
-				pipe_ctx->bottom_pipe = NULL;
-			} else  {
+				if (!dc_res_get_odm_bottom_pipe(pipe_ctx))
+					pipe_ctx->bottom_pipe = NULL;
+			} else {
 				memset(pipe_ctx, 0, sizeof(*pipe_ctx));
 			}
 		}
@@ -1674,6 +1769,9 @@ enum dc_status dc_remove_stream_from_ctx(
 	for (i = 0; i < MAX_PIPES; i++) {
 		if (new_ctx->res_ctx.pipe_ctx[i].stream == stream &&
 				!new_ctx->res_ctx.pipe_ctx[i].top_pipe) {
+			struct pipe_ctx *odm_pipe =
+					dc_res_get_odm_bottom_pipe(&new_ctx->res_ctx.pipe_ctx[i]);
+
 			del_pipe = &new_ctx->res_ctx.pipe_ctx[i];
 
 			ASSERT(del_pipe->stream_res.stream_enc);
@@ -1698,6 +1796,8 @@ enum dc_status dc_remove_stream_from_ctx(
 				dc->res_pool->funcs->remove_stream_from_ctx(dc, new_ctx, stream);
 
 			memset(del_pipe, 0, sizeof(*del_pipe));
+			if (odm_pipe)
+				memset(odm_pipe, 0, sizeof(*odm_pipe));
 
 			break;
 		}
@@ -1855,6 +1955,7 @@ enum dc_status resource_map_pool_resources(
 	struct dc_context *dc_ctx = dc->ctx;
 	struct pipe_ctx *pipe_ctx = NULL;
 	int pipe_idx = -1;
+	struct dc_bios *dcb = dc->ctx->dc_bios;
 
 	/* TODO Check if this is needed */
 	/*if (!resource_is_stream_unchanged(old_context, stream)) {
@@ -1869,6 +1970,13 @@ enum dc_status resource_map_pool_resources(
 
 	calculate_phy_pix_clks(stream);
 
+	/* TODO: Check Linux */
+	if (dc->config.allow_seamless_boot_optimization &&
+			!dcb->funcs->is_accelerated_mode(dcb)) {
+		if (dc_validate_seamless_boot_timing(dc, stream->sink, &stream->timing))
+			stream->apply_seamless_boot_optimization = true;
+	}
+
 	if (stream->apply_seamless_boot_optimization)
 		pipe_idx = acquire_resource_from_hw_enabled_state(
 				&context->res_ctx,
@@ -1951,7 +2059,7 @@ void dc_resource_state_construct(
 		const struct dc *dc,
 		struct dc_state *dst_ctx)
 {
-	dst_ctx->dccg = dc->res_pool->clk_mgr;
+	dst_ctx->clk_mgr = dc->res_pool->clk_mgr;
 }
 
 /**
@@ -1959,12 +2067,14 @@ void dc_resource_state_construct(
  * Checks HW resource availability and bandwidth requirement.
  * @dc: dc struct for this driver
  * @new_ctx: state to be validated
+ * @fast_validate: set to true if only yes/no to support matters
  *
  * Return: DC_OK if the result can be programmed.  Otherwise, an error code.
  */
 enum dc_status dc_validate_global_state(
 		struct dc *dc,
-		struct dc_state *new_ctx)
+		struct dc_state *new_ctx,
+		bool fast_validate)
 {
 	enum dc_status result = DC_ERROR_UNEXPECTED;
 	int i, j;
@@ -2019,7 +2129,7 @@ enum dc_status dc_validate_global_state(
 	result = resource_build_scaling_params_for_context(dc, new_ctx);
 
 	if (result == DC_OK)
-		if (!dc->res_pool->funcs->validate_bandwidth(dc, new_ctx))
+		if (!dc->res_pool->funcs->validate_bandwidth(dc, new_ctx, fast_validate))
 			result = DC_FAIL_BANDWIDTH_VALIDATE;
 
 	return result;
@@ -2315,6 +2425,21 @@ static void set_spd_info_packet(
 	*info_packet = stream->vrr_infopacket;
 }
 
+static void set_dp_sdp_info_packet(
+		struct dc_info_packet *info_packet,
+		struct dc_stream_state *stream)
+{
+	/* SPD info packet for custom sdp message */
+
+	/* Return if false. If true,
+	 * set the corresponding bit in the info packet
+	 */
+	if (!stream->dpsdp_infopacket.valid)
+		return;
+
+	*info_packet = stream->dpsdp_infopacket;
+}
+
 static void set_hdr_static_info_packet(
 		struct dc_info_packet *info_packet,
 		struct dc_stream_state *stream)
@@ -2411,6 +2536,7 @@ void resource_build_info_frame(struct pipe_ctx *pipe_ctx)
 	info->spd.valid = false;
 	info->hdrsmd.valid = false;
 	info->vsc.valid = false;
+	info->dpsdp.valid = false;
 
 	signal = pipe_ctx->stream->signal;
 
@@ -2430,6 +2556,8 @@ void resource_build_info_frame(struct pipe_ctx *pipe_ctx)
 		set_spd_info_packet(&info->spd, pipe_ctx->stream);
 
 		set_hdr_static_info_packet(&info->hdrsmd, pipe_ctx->stream);
+
+		set_dp_sdp_info_packet(&info->dpsdp, pipe_ctx->stream);
 	}
 
 	patch_gamut_packet_checksum(&info->gamut);
@@ -2657,10 +2785,11 @@ enum dc_status dc_validate_stream(struct dc *dc, struct dc_stream_state *stream)
 	if (!tg->funcs->validate_timing(tg, &stream->timing))
 		res = DC_FAIL_CONTROLLER_VALIDATE;
 
-	if (res == DC_OK)
+	if (res == DC_OK) {
 		if (!link->link_enc->funcs->validate_output_with_stream(
 						link->link_enc, stream))
 			res = DC_FAIL_ENC_VALIDATE;
+	}
 
 	/* TODO: validate audio ASIC caps, encoder */
 
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c
index 996298c35f42..96e97d25d639 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c
@@ -29,6 +29,9 @@
 #include "resource.h"
 #include "ipp.h"
 #include "timing_generator.h"
+#if defined(CONFIG_DRM_AMD_DC_DCN1_0)
+#include "dcn10/dcn10_hw_sequencer.h"
+#endif
 
 #define DC_LOGGER dc->ctx->logger
 
@@ -160,6 +163,27 @@ struct dc_stream_state *dc_create_stream_for_sink(
 	return stream;
 }
 
+struct dc_stream_state *dc_copy_stream(const struct dc_stream_state *stream)
+{
+	struct dc_stream_state *new_stream;
+
+	new_stream = kzalloc(sizeof(struct dc_stream_state), GFP_KERNEL);
+	if (!new_stream)
+		return NULL;
+
+	memcpy(new_stream, stream, sizeof(struct dc_stream_state));
+
+	if (new_stream->sink)
+		dc_sink_retain(new_stream->sink);
+
+	if (new_stream->out_transfer_func)
+		dc_transfer_func_retain(new_stream->out_transfer_func);
+
+	kref_init(&new_stream->refcount);
+
+	return new_stream;
+}
+
 /**
  * dc_stream_get_status_from_state - Get stream status from given dc state
  * @state: DC state to find the stream status in
@@ -196,6 +220,35 @@ struct dc_stream_status *dc_stream_get_status(
 	return dc_stream_get_status_from_state(dc->current_state, stream);
 }
 
+static void delay_cursor_until_vupdate(struct pipe_ctx *pipe_ctx, struct dc *dc)
+{
+#if defined(CONFIG_DRM_AMD_DC_DCN1_0)
+	unsigned int vupdate_line;
+	unsigned int lines_to_vupdate, us_to_vupdate, vpos, nvpos;
+	struct dc_stream_state *stream = pipe_ctx->stream;
+	unsigned int us_per_line;
+
+	if (stream->ctx->asic_id.chip_family == FAMILY_RV &&
+			ASIC_REV_IS_RAVEN(stream->ctx->asic_id.hw_internal_rev)) {
+
+		vupdate_line = get_vupdate_offset_from_vsync(pipe_ctx);
+		if (!dc_stream_get_crtc_position(dc, &stream, 1, &vpos, &nvpos))
+			return;
+
+		if (vpos >= vupdate_line)
+			return;
+
+		us_per_line = stream->timing.h_total * 10000 / stream->timing.pix_clk_100hz;
+		lines_to_vupdate = vupdate_line - vpos;
+		us_to_vupdate = lines_to_vupdate * us_per_line;
+
+		/* 70 us is a conservative estimate of cursor update time*/
+		if (us_to_vupdate < 70)
+			udelay(us_to_vupdate);
+	}
+#endif
+}
+
 /**
  * dc_stream_set_cursor_attributes() - Update cursor attributes and set cursor surface address
  */
@@ -234,6 +287,8 @@ bool dc_stream_set_cursor_attributes(
 
 		if (!pipe_to_program) {
 			pipe_to_program = pipe_ctx;
+
+			delay_cursor_until_vupdate(pipe_ctx, core_dc);
 			core_dc->hwss.pipe_control_lock(core_dc, pipe_to_program, true);
 		}
 
@@ -278,11 +333,13 @@ bool dc_stream_set_cursor_position(
 				(!pipe_ctx->plane_res.mi  && !pipe_ctx->plane_res.hubp) ||
 				!pipe_ctx->plane_state ||
 				(!pipe_ctx->plane_res.xfm && !pipe_ctx->plane_res.dpp) ||
-				!pipe_ctx->plane_res.ipp)
+				(!pipe_ctx->plane_res.ipp && !pipe_ctx->plane_res.dpp))
 			continue;
 
 		if (!pipe_to_program) {
 			pipe_to_program = pipe_ctx;
+
+			delay_cursor_until_vupdate(pipe_ctx, core_dc);
 			core_dc->hwss.pipe_control_lock(core_dc, pipe_to_program, true);
 		}
 
@@ -314,6 +371,68 @@ uint32_t dc_stream_get_vblank_counter(const struct dc_stream_state *stream)
 	return 0;
 }
 
+static void build_dp_sdp_info_frame(struct pipe_ctx *pipe_ctx,
+		const uint8_t  *custom_sdp_message,
+		unsigned int sdp_message_size)
+{
+	uint8_t i;
+	struct encoder_info_frame *info = &pipe_ctx->stream_res.encoder_info_frame;
+
+	/* set valid info */
+	info->dpsdp.valid = true;
+
+	/* set sdp message header */
+	info->dpsdp.hb0 = custom_sdp_message[0]; /* package id */
+	info->dpsdp.hb1 = custom_sdp_message[1]; /* package type */
+	info->dpsdp.hb2 = custom_sdp_message[2]; /* package specific byte 0 any data */
+	info->dpsdp.hb3 = custom_sdp_message[3]; /* package specific byte 0 any data */
+
+	/* set sdp message data */
+	for (i = 0; i < 32; i++)
+		info->dpsdp.sb[i] = (custom_sdp_message[i+4]);
+
+}
+
+static void invalid_dp_sdp_info_frame(struct pipe_ctx *pipe_ctx)
+{
+	struct encoder_info_frame *info = &pipe_ctx->stream_res.encoder_info_frame;
+
+	/* in-valid info */
+	info->dpsdp.valid = false;
+}
+
+bool dc_stream_send_dp_sdp(const struct dc_stream_state *stream,
+		const uint8_t *custom_sdp_message,
+		unsigned int sdp_message_size)
+{
+	int i;
+	struct dc  *core_dc;
+	struct resource_context *res_ctx;
+
+	if (stream == NULL) {
+		dm_error("DC: dc_stream is NULL!\n");
+		return false;
+	}
+
+	core_dc = stream->ctx->dc;
+	res_ctx = &core_dc->current_state->res_ctx;
+
+	for (i = 0; i < MAX_PIPES; i++) {
+		struct pipe_ctx *pipe_ctx = &res_ctx->pipe_ctx[i];
+
+		if (pipe_ctx->stream != stream)
+			continue;
+
+		build_dp_sdp_info_frame(pipe_ctx, custom_sdp_message, sdp_message_size);
+
+		core_dc->hwss.update_info_frame(pipe_ctx);
+
+		invalid_dp_sdp_info_frame(pipe_ctx);
+	}
+
+	return true;
+}
+
 bool dc_stream_get_scanoutpos(const struct dc_stream_state *stream,
 				  uint32_t *v_blank_start,
 				  uint32_t *v_blank_end,
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_surface.c b/drivers/gpu/drm/amd/display/dc/core/dc_surface.c
index ee6bd50f60b8..a5e86f9b148f 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_surface.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_surface.c
@@ -119,6 +119,19 @@ const struct dc_plane_status *dc_plane_get_status(
 	if (core_dc->current_state == NULL)
 		return NULL;
 
+	/* Find the current plane state and set its pending bit to false */
+	for (i = 0; i < core_dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe_ctx =
+				&core_dc->current_state->res_ctx.pipe_ctx[i];
+
+		if (pipe_ctx->plane_state != plane_state)
+			continue;
+
+		pipe_ctx->plane_state->status.is_flip_pending = false;
+
+		break;
+	}
+
 	for (i = 0; i < core_dc->res_pool->pipe_count; i++) {
 		struct pipe_ctx *pipe_ctx =
 				&core_dc->current_state->res_ctx.pipe_ctx[i];
diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h
index 0515095574e7..44e4b0465587 100644
--- a/drivers/gpu/drm/amd/display/dc/dc.h
+++ b/drivers/gpu/drm/amd/display/dc/dc.h
@@ -39,9 +39,10 @@
 #include "inc/hw/dmcu.h"
 #include "dml/display_mode_lib.h"
 
-#define DC_VER "3.2.17"
+#define DC_VER "3.2.27"
 
 #define MAX_SURFACES 3
+#define MAX_PLANES 6
 #define MAX_STREAMS 6
 #define MAX_SINKS_PER_LINK 4
 
@@ -53,6 +54,41 @@ struct dc_versions {
 	struct dmcu_version dmcu_version;
 };
 
+enum dc_plane_type {
+	DC_PLANE_TYPE_INVALID,
+	DC_PLANE_TYPE_DCE_RGB,
+	DC_PLANE_TYPE_DCE_UNDERLAY,
+	DC_PLANE_TYPE_DCN_UNIVERSAL,
+};
+
+struct dc_plane_cap {
+	enum dc_plane_type type;
+	uint32_t blends_with_above : 1;
+	uint32_t blends_with_below : 1;
+	uint32_t per_pixel_alpha : 1;
+	struct {
+		uint32_t argb8888 : 1;
+		uint32_t nv12 : 1;
+		uint32_t fp16 : 1;
+	} pixel_format_support;
+	// max upscaling factor x1000
+	// upscaling factors are always >= 1
+	// for example, 1080p -> 8K is 4.0, or 4000 raw value
+	struct {
+		uint32_t argb8888;
+		uint32_t nv12;
+		uint32_t fp16;
+	} max_upscale_factor;
+	// max downscale factor x1000
+	// downscale factors are always <= 1
+	// for example, 8K -> 1080p is 0.25, or 250 raw value
+	struct {
+		uint32_t argb8888;
+		uint32_t nv12;
+		uint32_t fp16;
+	} max_downscale_factor;
+};
+
 struct dc_caps {
 	uint32_t max_streams;
 	uint32_t max_links;
@@ -73,6 +109,7 @@ struct dc_caps {
 	bool force_dp_tps4_for_cp2520;
 	bool disable_dp_clk_share;
 	bool psp_setup_panel_mode;
+	struct dc_plane_cap planes[MAX_PLANES];
 };
 
 struct dc_dcc_surface_param {
@@ -164,6 +201,10 @@ struct dc_config {
 	bool gpu_vm_support;
 	bool disable_disp_pll_sharing;
 	bool fbc_support;
+	bool optimize_edp_link_rate;
+	bool disable_fractional_pwm;
+	bool allow_seamless_boot_optimization;
+	bool power_down_display_on_boot;
 };
 
 enum visual_confirm {
@@ -203,7 +244,59 @@ struct dc_clocks {
 	int fclk_khz;
 	int phyclk_khz;
 	int dramclk_khz;
-};
+	bool p_state_change_support;
+};
+
+struct dc_bw_validation_profile {
+	bool enable;
+
+	unsigned long long total_ticks;
+	unsigned long long voltage_level_ticks;
+	unsigned long long watermark_ticks;
+	unsigned long long rq_dlg_ticks;
+
+	unsigned long long total_count;
+	unsigned long long skip_fast_count;
+	unsigned long long skip_pass_count;
+	unsigned long long skip_fail_count;
+};
+
+#define BW_VAL_TRACE_SETUP() \
+		unsigned long long end_tick = 0; \
+		unsigned long long voltage_level_tick = 0; \
+		unsigned long long watermark_tick = 0; \
+		unsigned long long start_tick = dc->debug.bw_val_profile.enable ? \
+				dm_get_timestamp(dc->ctx) : 0
+
+#define BW_VAL_TRACE_COUNT() \
+		if (dc->debug.bw_val_profile.enable) \
+			dc->debug.bw_val_profile.total_count++
+
+#define BW_VAL_TRACE_SKIP(status) \
+		if (dc->debug.bw_val_profile.enable) { \
+			if (!voltage_level_tick) \
+				voltage_level_tick = dm_get_timestamp(dc->ctx); \
+			dc->debug.bw_val_profile.skip_ ## status ## _count++; \
+		}
+
+#define BW_VAL_TRACE_END_VOLTAGE_LEVEL() \
+		if (dc->debug.bw_val_profile.enable) \
+			voltage_level_tick = dm_get_timestamp(dc->ctx)
+
+#define BW_VAL_TRACE_END_WATERMARKS() \
+		if (dc->debug.bw_val_profile.enable) \
+			watermark_tick = dm_get_timestamp(dc->ctx)
+
+#define BW_VAL_TRACE_FINISH() \
+		if (dc->debug.bw_val_profile.enable) { \
+			end_tick = dm_get_timestamp(dc->ctx); \
+			dc->debug.bw_val_profile.total_ticks += end_tick - start_tick; \
+			dc->debug.bw_val_profile.voltage_level_ticks += voltage_level_tick - start_tick; \
+			if (watermark_tick) { \
+				dc->debug.bw_val_profile.watermark_ticks += watermark_tick - voltage_level_tick; \
+				dc->debug.bw_val_profile.rq_dlg_ticks += end_tick - watermark_tick; \
+			} \
+		}
 
 struct dc_debug_options {
 	enum visual_confirm visual_confirm;
@@ -257,6 +350,8 @@ struct dc_debug_options {
 	bool skip_detection_link_training;
 	unsigned int force_odm_combine; //bit vector based on otg inst
 	unsigned int force_fclk_khz;
+	bool disable_tri_buf;
+	struct dc_bw_validation_profile bw_val_profile;
 };
 
 struct dc_debug_data {
@@ -265,6 +360,14 @@ struct dc_debug_data {
 	uint32_t auxErrorCount;
 };
 
+struct dc_bounding_box_overrides {
+	int sr_exit_time_ns;
+	int sr_enter_plus_exit_time_ns;
+	int urgent_latency_ns;
+	int percent_of_ideal_drambw;
+	int dram_clock_change_latency_ns;
+};
+
 struct dc_state;
 struct resource_pool;
 struct dce_hwseq;
@@ -274,6 +377,7 @@ struct dc {
 	struct dc_cap_funcs cap_funcs;
 	struct dc_config config;
 	struct dc_debug_options debug;
+	struct dc_bounding_box_overrides bb_overrides;
 	struct dc_context *ctx;
 
 	uint8_t link_count;
@@ -298,8 +402,12 @@ struct dc {
 	struct hw_sequencer_funcs hwss;
 	struct dce_hwseq *hwseq;
 
+	/* Require to optimize clocks and bandwidth for added/removed planes */
 	bool optimized_required;
 
+	/* Require to maintain clocks and bandwidth for UEFI enabled HW */
+	bool optimize_seamless_boot;
+
 	/* FBC compressor */
 	struct compressor *fbc_compressor;
 
@@ -327,6 +435,7 @@ struct dc_init_data {
 	struct hw_asic_id asic_id;
 	void *driver; /* ctx */
 	struct cgs_device *cgs_device;
+	struct dc_bounding_box_overrides bb_overrides;
 
 	int num_virtual_links;
 	/*
@@ -597,7 +706,7 @@ struct dc_validation_set {
 	uint8_t plane_count;
 };
 
-bool dc_validate_seamless_boot_timing(struct dc *dc,
+bool dc_validate_seamless_boot_timing(const struct dc *dc,
 				const struct dc_sink *sink,
 				struct dc_crtc_timing *crtc_timing);
 
@@ -605,9 +714,14 @@ enum dc_status dc_validate_plane(struct dc *dc, const struct dc_plane_state *pla
 
 void get_clock_requirements_for_state(struct dc_state *state, struct AsicStateEx *info);
 
+/*
+ * fast_validate: we return after determining if we can support the new state,
+ * but before we populate the programming info
+ */
 enum dc_status dc_validate_global_state(
 		struct dc *dc,
-		struct dc_state *new_ctx);
+		struct dc_state *new_ctx,
+		bool fast_validate);
 
 
 void dc_resource_state_construct(
@@ -636,7 +750,8 @@ void dc_resource_state_destruct(struct dc_state *context);
 bool dc_commit_state(struct dc *dc, struct dc_state *context);
 
 
-struct dc_state *dc_create_state(void);
+struct dc_state *dc_create_state(struct dc *dc);
+struct dc_state *dc_copy_state(struct dc_state *src_ctx);
 void dc_retain_state(struct dc_state *context);
 void dc_release_state(struct dc_state *context);
 
@@ -648,9 +763,16 @@ struct dpcd_caps {
 	union dpcd_rev dpcd_rev;
 	union max_lane_count max_ln_count;
 	union max_down_spread max_down_spread;
+	union dprx_feature dprx_feature;
+
+	/* valid only for eDP v1.4 or higher*/
+	uint8_t edp_supported_link_rates_count;
+	enum dc_link_rate edp_supported_link_rates[8];
 
 	/* dongle type (DP converter, CV smart dongle) */
 	enum display_dongle_type dongle_type;
+	/* branch device or sink device */
+	bool is_branch_dev;
 	/* Dongle's downstream count. */
 	union sink_count sink_count;
 	/* If dongle_type == DISPLAY_DONGLE_DP_HDMI_CONVERTER,
@@ -666,11 +788,11 @@ struct dpcd_caps {
 	int8_t branch_dev_name[6];
 	int8_t branch_hw_revision;
 	int8_t branch_fw_revision[2];
-	uint8_t link_rate_set;
 
 	bool allow_invalid_MSA_timing_param;
 	bool panel_mode_edp;
 	bool dpcd_display_control_capable;
+	bool ext_receiver_cap_field_present;
 };
 
 #include "dc_link.h"
diff --git a/drivers/gpu/drm/amd/display/dc/dc_ddc_types.h b/drivers/gpu/drm/amd/display/dc/dc_ddc_types.h
index 05c8c31d8b31..4ef97f65e55d 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_ddc_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_ddc_types.h
@@ -68,6 +68,8 @@ enum aux_transaction_reply {
 	AUX_TRANSACTION_REPLY_AUX_ACK = 0x00,
 	AUX_TRANSACTION_REPLY_AUX_NACK = 0x01,
 	AUX_TRANSACTION_REPLY_AUX_DEFER = 0x02,
+	AUX_TRANSACTION_REPLY_I2C_OVER_AUX_NACK = 0x04,
+	AUX_TRANSACTION_REPLY_I2C_OVER_AUX_DEFER = 0x08,
 
 	AUX_TRANSACTION_REPLY_I2C_ACK = 0x00,
 	AUX_TRANSACTION_REPLY_I2C_NACK = 0x10,
diff --git a/drivers/gpu/drm/amd/display/dc/dc_dp_types.h b/drivers/gpu/drm/amd/display/dc/dc_dp_types.h
index d4eab33c453b..11c68a399267 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_dp_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_dp_types.h
@@ -94,6 +94,8 @@ struct dc_link_settings {
 	enum dc_lane_count lane_count;
 	enum dc_link_rate link_rate;
 	enum dc_link_spread link_spread;
+	bool use_link_rate_set;
+	uint8_t link_rate_set;
 };
 
 struct dc_lane_settings {
@@ -420,10 +422,24 @@ union edp_configuration_cap {
 	uint8_t raw;
 };
 
+union dprx_feature {
+	struct {
+		uint8_t GTC_CAP:1;                             // bit 0: DP 1.3+
+		uint8_t SST_SPLIT_SDP_CAP:1;                   // bit 1: DP 1.4
+		uint8_t AV_SYNC_CAP:1;                         // bit 2: DP 1.3+
+		uint8_t VSC_SDP_COLORIMETRY_SUPPORTED:1;       // bit 3: DP 1.3+
+		uint8_t VSC_EXT_VESA_SDP_SUPPORTED:1;          // bit 4: DP 1.4
+		uint8_t VSC_EXT_VESA_SDP_CHAINING_SUPPORTED:1; // bit 5: DP 1.4
+		uint8_t VSC_EXT_CEA_SDP_SUPPORTED:1;           // bit 6: DP 1.4
+		uint8_t VSC_EXT_CEA_SDP_CHAINING_SUPPORTED:1;  // bit 7: DP 1.4
+	} bits;
+	uint8_t raw;
+};
+
 union training_aux_rd_interval {
 	struct {
 		uint8_t TRAINIG_AUX_RD_INTERVAL:7;
-		uint8_t EXT_RECIEVER_CAP_FIELD_PRESENT:1;
+		uint8_t EXT_RECEIVER_CAP_FIELD_PRESENT:1;
 	} bits;
 	uint8_t raw;
 };
diff --git a/drivers/gpu/drm/amd/display/dc/dc_helper.c b/drivers/gpu/drm/amd/display/dc/dc_helper.c
index 597d38393379..5e6c5eff49cf 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_helper.c
+++ b/drivers/gpu/drm/amd/display/dc/dc_helper.c
@@ -51,20 +51,16 @@ static inline void set_reg_field_value_masks(
 	field_value_mask->mask = field_value_mask->mask | mask;
 }
 
-uint32_t generic_reg_update_ex(const struct dc_context *ctx,
-		uint32_t addr, uint32_t reg_val, int n,
+static void set_reg_field_values(struct dc_reg_value_masks *field_value_mask,
+		uint32_t addr, int n,
 		uint8_t shift1, uint32_t mask1, uint32_t field_value1,
-		...)
+		va_list ap)
 {
-	struct dc_reg_value_masks field_value_mask = {0};
 	uint32_t shift, mask, field_value;
 	int i = 1;
 
-	va_list ap;
-	va_start(ap, field_value1);
-
 	/* gather all bits value/mask getting updated in this register */
-	set_reg_field_value_masks(&field_value_mask,
+	set_reg_field_value_masks(field_value_mask,
 			field_value1, mask1, shift1);
 
 	while (i < n) {
@@ -72,10 +68,48 @@ uint32_t generic_reg_update_ex(const struct dc_context *ctx,
 		mask = va_arg(ap, uint32_t);
 		field_value = va_arg(ap, uint32_t);
 
-		set_reg_field_value_masks(&field_value_mask,
+		set_reg_field_value_masks(field_value_mask,
 				field_value, mask, shift);
 		i++;
 	}
+}
+
+uint32_t generic_reg_update_ex(const struct dc_context *ctx,
+		uint32_t addr, int n,
+		uint8_t shift1, uint32_t mask1, uint32_t field_value1,
+		...)
+{
+	struct dc_reg_value_masks field_value_mask = {0};
+	uint32_t reg_val;
+	va_list ap;
+
+	va_start(ap, field_value1);
+
+	set_reg_field_values(&field_value_mask, addr, n, shift1, mask1,
+			field_value1, ap);
+
+	va_end(ap);
+
+	/* mmio write directly */
+	reg_val = dm_read_reg(ctx, addr);
+	reg_val = (reg_val & ~field_value_mask.mask) | field_value_mask.value;
+	dm_write_reg(ctx, addr, reg_val);
+	return reg_val;
+}
+
+uint32_t generic_reg_set_ex(const struct dc_context *ctx,
+		uint32_t addr, uint32_t reg_val, int n,
+		uint8_t shift1, uint32_t mask1, uint32_t field_value1,
+		...)
+{
+	struct dc_reg_value_masks field_value_mask = {0};
+	va_list ap;
+
+	va_start(ap, field_value1);
+
+	set_reg_field_values(&field_value_mask, addr, n, shift1, mask1,
+			field_value1, ap);
+
 	va_end(ap);
 
 
@@ -85,6 +119,24 @@ uint32_t generic_reg_update_ex(const struct dc_context *ctx,
 	return reg_val;
 }
 
+uint32_t dm_read_reg_func(
+	const struct dc_context *ctx,
+	uint32_t address,
+	const char *func_name)
+{
+	uint32_t value;
+#ifdef DM_CHECK_ADDR_0
+	if (address == 0) {
+		DC_ERR("invalid register read; address = 0\n");
+		return 0;
+	}
+#endif
+	value = cgs_read_register(ctx->cgs_device, address);
+	trace_amdgpu_dc_rreg(&ctx->perf_trace->read_count, address, value);
+
+	return value;
+}
+
 uint32_t generic_reg_get(const struct dc_context *ctx, uint32_t addr,
 		uint8_t shift, uint32_t mask, uint32_t *field_value)
 {
@@ -235,7 +287,7 @@ uint32_t generic_reg_get(const struct dc_context *ctx,
 }
 */
 
-uint32_t generic_reg_wait(const struct dc_context *ctx,
+void generic_reg_wait(const struct dc_context *ctx,
 	uint32_t addr, uint32_t shift, uint32_t mask, uint32_t condition_value,
 	unsigned int delay_between_poll_us, unsigned int time_out_num_tries,
 	const char *func_name, int line)
@@ -265,7 +317,7 @@ uint32_t generic_reg_wait(const struct dc_context *ctx,
 				DC_LOG_DC("REG_WAIT taking a while: %dms in %s line:%d\n",
 						delay_between_poll_us * i / 1000,
 						func_name, line);
-			return reg_val;
+			return;
 		}
 	}
 
@@ -275,8 +327,6 @@ uint32_t generic_reg_wait(const struct dc_context *ctx,
 
 	if (!IS_FPGA_MAXIMUS_DC(ctx->dce_environment))
 		BREAK_TO_DEBUGGER();
-
-	return reg_val;
 }
 
 void generic_write_indirect_reg(const struct dc_context *ctx,
diff --git a/drivers/gpu/drm/amd/display/dc/dc_link.h b/drivers/gpu/drm/amd/display/dc/dc_link.h
index 8fc223defed4..7b9429e30d82 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_link.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_link.h
@@ -120,6 +120,7 @@ struct dc_link {
 	/* MST record stream using this link */
 	struct link_flags {
 		bool dp_keep_receiver_powered;
+		bool dp_skip_DID2;
 	} wa_flags;
 	struct link_mst_stream_allocation_table mst_stream_alloc_table;
 
@@ -246,10 +247,18 @@ void dc_link_set_test_pattern(struct dc_link *link,
 			const struct link_training_settings *p_link_settings,
 			const unsigned char *p_custom_pattern,
 			unsigned int cust_pattern_size);
+uint32_t dc_link_bandwidth_kbps(
+	const struct dc_link *link,
+	const struct dc_link_settings *link_setting);
+
+const struct dc_link_settings *dc_link_get_link_cap(
+		const struct dc_link *link);
 
 bool dc_submit_i2c(
 		struct dc *dc,
 		uint32_t link_index,
 		struct i2c_command *cmd);
 
+uint32_t dc_bandwidth_in_kbps_from_timing(
+	const struct dc_crtc_timing *timing);
 #endif /* DC_LINK_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dc_stream.h b/drivers/gpu/drm/amd/display/dc/dc_stream.h
index 5657cb3a2ad3..189bdab929a5 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_stream.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_stream.h
@@ -80,6 +80,7 @@ struct dc_stream_state {
 	struct dc_info_packet vrr_infopacket;
 	struct dc_info_packet vsc_infopacket;
 	struct dc_info_packet vsp_infopacket;
+	struct dc_info_packet dpsdp_infopacket;
 
 	struct rect src; /* composition area */
 	struct rect dst; /* stream addressable area */
@@ -221,6 +222,13 @@ struct dc_stream_state *dc_get_stream_at_index(struct dc *dc, uint8_t i);
  */
 uint32_t dc_stream_get_vblank_counter(const struct dc_stream_state *stream);
 
+/*
+ * Send dp sdp message.
+ */
+bool dc_stream_send_dp_sdp(const struct dc_stream_state *stream,
+		const uint8_t *custom_sdp_message,
+		unsigned int sdp_message_size);
+
 /* TODO: Return parsed values rather than direct register read
  * This has a dependency on the caller (amdgpu_display_get_crtc_scanoutpos)
  * being refactored properly to be dce-specific
@@ -299,6 +307,8 @@ enum surface_update_type dc_check_update_surfaces_for_stream(
  */
 struct dc_stream_state *dc_create_stream_for_sink(struct dc_sink *dc_sink);
 
+struct dc_stream_state *dc_copy_stream(const struct dc_stream_state *stream);
+
 void update_stream_signal(struct dc_stream_state *stream, struct dc_sink *sink);
 
 void dc_stream_retain(struct dc_stream_state *dc_stream);
diff --git a/drivers/gpu/drm/amd/display/dc/dc_types.h b/drivers/gpu/drm/amd/display/dc/dc_types.h
index da2009a108cf..6c2a3d9a4c2e 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_types.h
@@ -103,7 +103,7 @@ struct dc_context {
 };
 
 
-#define DC_MAX_EDID_BUFFER_SIZE 512
+#define DC_MAX_EDID_BUFFER_SIZE 1024
 #define EDID_BLOCK_SIZE 128
 #define MAX_SURFACE_NUM 4
 #define NUM_PIXEL_FORMATS 10
@@ -395,7 +395,7 @@ struct dc_dongle_caps {
 	bool is_dp_hdmi_ycbcr422_converter;
 	bool is_dp_hdmi_ycbcr420_converter;
 	uint32_t dp_hdmi_max_bpc;
-	uint32_t dp_hdmi_max_pixel_clk;
+	uint32_t dp_hdmi_max_pixel_clk_in_khz;
 };
 /* Scaling format */
 enum scaling_transformation {
@@ -550,9 +550,9 @@ struct psr_config {
 	unsigned char psr_version;
 	unsigned int psr_rfb_setup_time;
 	bool psr_exit_link_training_required;
-
 	bool psr_frame_capture_indication_req;
 	unsigned int psr_sdp_transmit_line_num_deadline;
+	bool allow_smu_optimizations;
 };
 
 union dmcu_psr_level {
@@ -654,6 +654,7 @@ struct psr_context {
 	 * continuing powerd own
 	 */
 	unsigned int frame_delay;
+	bool allow_smu_optimizations;
 };
 
 struct colorspace_transform {
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c
index 4fe3664fb495..bd33c47183fc 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c
@@ -171,24 +171,24 @@ static void submit_channel_request(
 		 (request->action == I2CAUX_TRANSACTION_ACTION_I2C_WRITE_MOT)));
 	if (REG(AUXN_IMPCAL)) {
 		/* clear_aux_error */
-		REG_UPDATE_SEQ(AUXN_IMPCAL, AUXN_CALOUT_ERROR_AK,
-				1,
-				0);
+		REG_UPDATE_SEQ_2(AUXN_IMPCAL,
+				AUXN_CALOUT_ERROR_AK, 1,
+				AUXN_CALOUT_ERROR_AK, 0);
 
-		REG_UPDATE_SEQ(AUXP_IMPCAL, AUXP_CALOUT_ERROR_AK,
-				1,
-				0);
+		REG_UPDATE_SEQ_2(AUXP_IMPCAL,
+				AUXP_CALOUT_ERROR_AK, 1,
+				AUXP_CALOUT_ERROR_AK, 0);
 
 		/* force_default_calibrate */
-		REG_UPDATE_1BY1_2(AUXN_IMPCAL,
+		REG_UPDATE_SEQ_2(AUXN_IMPCAL,
 				AUXN_IMPCAL_ENABLE, 1,
 				AUXN_IMPCAL_OVERRIDE_ENABLE, 0);
 
 		/* bug? why AUXN update EN and OVERRIDE_EN 1 by 1 while AUX P toggles OVERRIDE? */
 
-		REG_UPDATE_SEQ(AUXP_IMPCAL, AUXP_IMPCAL_OVERRIDE_ENABLE,
-				1,
-				0);
+		REG_UPDATE_SEQ_2(AUXP_IMPCAL,
+				AUXP_IMPCAL_OVERRIDE_ENABLE, 1,
+				AUXP_IMPCAL_OVERRIDE_ENABLE, 0);
 	}
 
 	REG_UPDATE(AUX_INTERRUPT_CONTROL, AUX_SW_DONE_ACK, 1);
@@ -270,7 +270,7 @@ static int read_channel_reply(struct dce_aux *engine, uint32_t size,
 	if (!bytes_replied)
 		return -1;
 
-	REG_UPDATE_1BY1_3(AUX_SW_DATA,
+	REG_UPDATE_SEQ_3(AUX_SW_DATA,
 			  AUX_SW_INDEX, 0,
 			  AUX_SW_AUTOINCREMENT_DISABLE, 1,
 			  AUX_SW_DATA_RW, 1);
@@ -320,9 +320,10 @@ static enum aux_channel_operation_result get_channel_status(
 	*returned_bytes = 0;
 
 	/* poll to make sure that SW_DONE is asserted */
-	value = REG_WAIT(AUX_SW_STATUS, AUX_SW_DONE, 1,
+	REG_WAIT(AUX_SW_STATUS, AUX_SW_DONE, 1,
 				10, aux110->timeout_period/10);
 
+	value = REG_READ(AUX_SW_STATUS);
 	/* in case HPD is LOW, exit AUX transaction */
 	if ((value & AUX_SW_STATUS__AUX_SW_HPD_DISCON_MASK))
 		return AUX_CHANNEL_OPERATION_FAILED_HPD_DISCON;
@@ -377,7 +378,6 @@ static bool acquire(
 	struct dce_aux *engine,
 	struct ddc *ddc)
 {
-
 	enum gpio_result result;
 
 	if (!is_engine_available(engine))
@@ -442,12 +442,12 @@ static enum i2caux_transaction_action i2caux_action_from_payload(struct aux_payl
 	return I2CAUX_TRANSACTION_ACTION_DP_READ;
 }
 
-int dce_aux_transfer(struct ddc_service *ddc,
-		struct aux_payload *payload)
+int dce_aux_transfer_raw(struct ddc_service *ddc,
+		struct aux_payload *payload,
+		enum aux_channel_operation_result *operation_result)
 {
 	struct ddc *ddc_pin = ddc->ddc_pin;
 	struct dce_aux *aux_engine;
-	enum aux_channel_operation_result operation_result;
 	struct aux_request_transaction_data aux_req;
 	struct aux_reply_transaction_data aux_rep;
 	uint8_t returned_bytes = 0;
@@ -458,7 +458,8 @@ int dce_aux_transfer(struct ddc_service *ddc,
 	memset(&aux_rep, 0, sizeof(aux_rep));
 
 	aux_engine = ddc->ctx->dc->res_pool->engines[ddc_pin->pin_data->en];
-	acquire(aux_engine, ddc_pin);
+	if (!acquire(aux_engine, ddc_pin))
+		return -1;
 
 	if (payload->i2c_over_aux)
 		aux_req.type = AUX_TRANSACTION_TYPE_I2C;
@@ -473,28 +474,26 @@ int dce_aux_transfer(struct ddc_service *ddc,
 	aux_req.data = payload->data;
 
 	submit_channel_request(aux_engine, &aux_req);
-	operation_result = get_channel_status(aux_engine, &returned_bytes);
-
-	switch (operation_result) {
-	case AUX_CHANNEL_OPERATION_SUCCEEDED:
-		res = read_channel_reply(aux_engine, payload->length,
-							payload->data, payload->reply,
-							&status);
-		break;
-	case AUX_CHANNEL_OPERATION_FAILED_HPD_DISCON:
-		res = 0;
-		break;
-	case AUX_CHANNEL_OPERATION_FAILED_REASON_UNKNOWN:
-	case AUX_CHANNEL_OPERATION_FAILED_INVALID_REPLY:
-	case AUX_CHANNEL_OPERATION_FAILED_TIMEOUT:
+	*operation_result = get_channel_status(aux_engine, &returned_bytes);
+
+	if (*operation_result == AUX_CHANNEL_OPERATION_SUCCEEDED) {
+		read_channel_reply(aux_engine, payload->length,
+					 payload->data, payload->reply,
+					 &status);
+		res = returned_bytes;
+	} else {
 		res = -1;
-		break;
 	}
+
 	release_engine(aux_engine);
 	return res;
 }
 
-#define AUX_RETRY_MAX 7
+#define AUX_MAX_RETRIES 7
+#define AUX_MAX_DEFER_RETRIES 7
+#define AUX_MAX_I2C_DEFER_RETRIES 7
+#define AUX_MAX_INVALID_REPLY_RETRIES 2
+#define AUX_MAX_TIMEOUT_RETRIES 3
 
 bool dce_aux_transfer_with_retries(struct ddc_service *ddc,
 		struct aux_payload *payload)
@@ -502,24 +501,85 @@ bool dce_aux_transfer_with_retries(struct ddc_service *ddc,
 	int i, ret = 0;
 	uint8_t reply;
 	bool payload_reply = true;
+	enum aux_channel_operation_result operation_result;
+	int aux_ack_retries = 0,
+		aux_defer_retries = 0,
+		aux_i2c_defer_retries = 0,
+		aux_timeout_retries = 0,
+		aux_invalid_reply_retries = 0;
 
 	if (!payload->reply) {
 		payload_reply = false;
 		payload->reply = &reply;
 	}
 
-	for (i = 0; i < AUX_RETRY_MAX; i++) {
-		ret = dce_aux_transfer(ddc, payload);
-
-		if (ret >= 0) {
-			if (*payload->reply == 0) {
-				if (!payload_reply)
-					payload->reply = NULL;
-				return true;
+	for (i = 0; i < AUX_MAX_RETRIES; i++) {
+		ret = dce_aux_transfer_raw(ddc, payload, &operation_result);
+		switch (operation_result) {
+		case AUX_CHANNEL_OPERATION_SUCCEEDED:
+			aux_timeout_retries = 0;
+			aux_invalid_reply_retries = 0;
+
+			switch (*payload->reply) {
+			case AUX_TRANSACTION_REPLY_AUX_ACK:
+				if (!payload->write && payload->length != ret) {
+					if (++aux_ack_retries >= AUX_MAX_RETRIES)
+						goto fail;
+					else
+						udelay(300);
+				} else
+					return true;
+			break;
+
+			case AUX_TRANSACTION_REPLY_AUX_DEFER:
+			case AUX_TRANSACTION_REPLY_I2C_OVER_AUX_NACK:
+			case AUX_TRANSACTION_REPLY_I2C_OVER_AUX_DEFER:
+				if (++aux_defer_retries >= AUX_MAX_DEFER_RETRIES)
+					goto fail;
+				break;
+
+			case AUX_TRANSACTION_REPLY_I2C_DEFER:
+				aux_defer_retries = 0;
+				if (++aux_i2c_defer_retries >= AUX_MAX_I2C_DEFER_RETRIES)
+					goto fail;
+				break;
+
+			case AUX_TRANSACTION_REPLY_AUX_NACK:
+			case AUX_TRANSACTION_REPLY_HPD_DISCON:
+			default:
+				goto fail;
 			}
-		}
+			break;
+
+		case AUX_CHANNEL_OPERATION_FAILED_INVALID_REPLY:
+			if (++aux_invalid_reply_retries >= AUX_MAX_INVALID_REPLY_RETRIES)
+				goto fail;
+			else
+				udelay(400);
+			break;
+
+		case AUX_CHANNEL_OPERATION_FAILED_TIMEOUT:
+			if (++aux_timeout_retries >= AUX_MAX_TIMEOUT_RETRIES)
+				goto fail;
+			else {
+				/*
+				 * DP 1.4, 2.8.2:  AUX Transaction Response/Reply Timeouts
+				 * According to the DP spec there should be 3 retries total
+				 * with a 400us wait inbetween each. Hardware already waits
+				 * for 550us therefore no wait is required here.
+				 */
+			}
+			break;
 
-		udelay(1000);
+		case AUX_CHANNEL_OPERATION_FAILED_HPD_DISCON:
+		case AUX_CHANNEL_OPERATION_FAILED_REASON_UNKNOWN:
+		default:
+			goto fail;
+		}
 	}
+
+fail:
+	if (!payload_reply)
+		payload->reply = NULL;
 	return false;
 }
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_aux.h b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.h
index e28ed6a00ff4..ce6a26d189b0 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_aux.h
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.h
@@ -123,8 +123,9 @@ bool dce110_aux_engine_acquire(
 	struct dce_aux *aux_engine,
 	struct ddc *ddc);
 
-int dce_aux_transfer(struct ddc_service *ddc,
-		struct aux_payload *cmd);
+int dce_aux_transfer_raw(struct ddc_service *ddc,
+		struct aux_payload *cmd,
+		enum aux_channel_operation_result *operation_result);
 
 bool dce_aux_transfer_with_retries(struct ddc_service *ddc,
 		struct aux_payload *cmd);
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/dce/dce_clk_mgr.c
index 6e142c2db986..963686380738 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_clk_mgr.c
@@ -222,7 +222,7 @@ static enum dm_pp_clocks_state dce_get_required_clocks_state(
 	 * all required clocks
 	 */
 	for (i = clk_mgr_dce->max_clks_state; i >= DM_PP_CLOCKS_STATE_ULTRA_LOW; i--)
-		if (context->bw.dce.dispclk_khz >
+		if (context->bw_ctx.bw.dce.dispclk_khz >
 				clk_mgr_dce->max_clks_by_state[i].display_clk_khz
 			|| max_pix_clk >
 				clk_mgr_dce->max_clks_by_state[i].pixel_clk_khz)
@@ -232,7 +232,7 @@ static enum dm_pp_clocks_state dce_get_required_clocks_state(
 	if (low_req_clk > clk_mgr_dce->max_clks_state) {
 		/* set max clock state for high phyclock, invalid on exceeding display clock */
 		if (clk_mgr_dce->max_clks_by_state[clk_mgr_dce->max_clks_state].display_clk_khz
-				< context->bw.dce.dispclk_khz)
+				< context->bw_ctx.bw.dce.dispclk_khz)
 			low_req_clk = DM_PP_CLOCKS_STATE_INVALID;
 		else
 			low_req_clk = clk_mgr_dce->max_clks_state;
@@ -610,22 +610,22 @@ static void dce11_pplib_apply_display_requirements(
 	struct dm_pp_display_configuration *pp_display_cfg = &context->pp_display_cfg;
 
 	pp_display_cfg->all_displays_in_sync =
-		context->bw.dce.all_displays_in_sync;
+		context->bw_ctx.bw.dce.all_displays_in_sync;
 	pp_display_cfg->nb_pstate_switch_disable =
-			context->bw.dce.nbp_state_change_enable == false;
+			context->bw_ctx.bw.dce.nbp_state_change_enable == false;
 	pp_display_cfg->cpu_cc6_disable =
-			context->bw.dce.cpuc_state_change_enable == false;
+			context->bw_ctx.bw.dce.cpuc_state_change_enable == false;
 	pp_display_cfg->cpu_pstate_disable =
-			context->bw.dce.cpup_state_change_enable == false;
+			context->bw_ctx.bw.dce.cpup_state_change_enable == false;
 	pp_display_cfg->cpu_pstate_separation_time =
-			context->bw.dce.blackout_recovery_time_us;
+			context->bw_ctx.bw.dce.blackout_recovery_time_us;
 
-	pp_display_cfg->min_memory_clock_khz = context->bw.dce.yclk_khz
+	pp_display_cfg->min_memory_clock_khz = context->bw_ctx.bw.dce.yclk_khz
 		/ MEMORY_TYPE_MULTIPLIER_CZ;
 
 	pp_display_cfg->min_engine_clock_khz = determine_sclk_from_bounding_box(
 			dc,
-			context->bw.dce.sclk_khz);
+			context->bw_ctx.bw.dce.sclk_khz);
 
 	/*
 	 * As workaround for >4x4K lightup set dcfclock to min_engine_clock value.
@@ -638,7 +638,7 @@ static void dce11_pplib_apply_display_requirements(
 			pp_display_cfg->min_engine_clock_khz : 0;
 
 	pp_display_cfg->min_engine_clock_deep_sleep_khz
-			= context->bw.dce.sclk_deep_sleep_khz;
+			= context->bw_ctx.bw.dce.sclk_deep_sleep_khz;
 
 	pp_display_cfg->avail_mclk_switch_time_us =
 						dce110_get_min_vblank_time_us(context);
@@ -669,7 +669,7 @@ static void dce_update_clocks(struct clk_mgr *clk_mgr,
 {
 	struct dce_clk_mgr *clk_mgr_dce = TO_DCE_CLK_MGR(clk_mgr);
 	struct dm_pp_power_level_change_request level_change_req;
-	int patched_disp_clk = context->bw.dce.dispclk_khz;
+	int patched_disp_clk = context->bw_ctx.bw.dce.dispclk_khz;
 
 	/*TODO: W/A for dal3 linux, investigate why this works */
 	if (!clk_mgr_dce->dfs_bypass_active)
@@ -696,7 +696,7 @@ static void dce11_update_clocks(struct clk_mgr *clk_mgr,
 {
 	struct dce_clk_mgr *clk_mgr_dce = TO_DCE_CLK_MGR(clk_mgr);
 	struct dm_pp_power_level_change_request level_change_req;
-	int patched_disp_clk = context->bw.dce.dispclk_khz;
+	int patched_disp_clk = context->bw_ctx.bw.dce.dispclk_khz;
 
 	/*TODO: W/A for dal3 linux, investigate why this works */
 	if (!clk_mgr_dce->dfs_bypass_active)
@@ -711,7 +711,7 @@ static void dce11_update_clocks(struct clk_mgr *clk_mgr,
 	}
 
 	if (should_set_clock(safe_to_lower, patched_disp_clk, clk_mgr->clks.dispclk_khz)) {
-		context->bw.dce.dispclk_khz = dce_set_clock(clk_mgr, patched_disp_clk);
+		context->bw_ctx.bw.dce.dispclk_khz = dce_set_clock(clk_mgr, patched_disp_clk);
 		clk_mgr->clks.dispclk_khz = patched_disp_clk;
 	}
 	dce11_pplib_apply_display_requirements(clk_mgr->ctx->dc, context);
@@ -723,7 +723,7 @@ static void dce112_update_clocks(struct clk_mgr *clk_mgr,
 {
 	struct dce_clk_mgr *clk_mgr_dce = TO_DCE_CLK_MGR(clk_mgr);
 	struct dm_pp_power_level_change_request level_change_req;
-	int patched_disp_clk = context->bw.dce.dispclk_khz;
+	int patched_disp_clk = context->bw_ctx.bw.dce.dispclk_khz;
 
 	/*TODO: W/A for dal3 linux, investigate why this works */
 	if (!clk_mgr_dce->dfs_bypass_active)
@@ -751,7 +751,7 @@ static void dce12_update_clocks(struct clk_mgr *clk_mgr,
 	struct dce_clk_mgr *clk_mgr_dce = TO_DCE_CLK_MGR(clk_mgr);
 	struct dm_pp_clock_for_voltage_req clock_voltage_req = {0};
 	int max_pix_clk = get_max_pixel_clock_for_all_paths(context);
-	int patched_disp_clk = context->bw.dce.dispclk_khz;
+	int patched_disp_clk = context->bw_ctx.bw.dce.dispclk_khz;
 
 	/*TODO: W/A for dal3 linux, investigate why this works */
 	if (!clk_mgr_dce->dfs_bypass_active)
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c b/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c
index 71d5777de961..f70437aae8e0 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c
@@ -978,7 +978,7 @@ static bool dce110_clock_source_power_down(
 }
 
 static bool get_pixel_clk_frequency_100hz(
-		struct clock_source *clock_source,
+		const struct clock_source *clock_source,
 		unsigned int inst,
 		unsigned int *pixel_clk_khz)
 {
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c b/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c
index c2926cf19dee..818536eea00a 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c
@@ -51,6 +51,9 @@
 #define PSR_SET_WAITLOOP 0x31
 #define MCP_INIT_DMCU 0x88
 #define MCP_INIT_IRAM 0x89
+#define MCP_SYNC_PHY_LOCK 0x90
+#define MCP_SYNC_PHY_UNLOCK 0x91
+#define MCP_BL_SET_PWM_FRAC 0x6A  /* Enable or disable Fractional PWM */
 #define MASTER_COMM_CNTL_REG__MASTER_COMM_INTERRUPT_MASK   0x00000001L
 
 static bool dce_dmcu_init(struct dmcu *dmcu)
@@ -213,9 +216,6 @@ static bool dce_dmcu_setup_psr(struct dmcu *dmcu,
 	link->link_enc->funcs->psr_program_secondary_packet(link->link_enc,
 			psr_context->sdpTransmitLineNumDeadline);
 
-	if (psr_context->psr_level.bits.SKIP_SMU_NOTIFICATION)
-		REG_UPDATE(SMU_INTERRUPT_CONTROL, DC_SMU_INT_ENABLE, 1);
-
 	/* waitDMCUReadyForCmd */
 	REG_WAIT(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 0,
 					dmcu_wait_reg_ready_interval,
@@ -342,9 +342,32 @@ static void dcn10_get_dmcu_version(struct dmcu *dmcu)
 			IRAM_RD_ADDR_AUTO_INC, 0);
 }
 
+static void dcn10_dmcu_enable_fractional_pwm(struct dmcu *dmcu,
+		uint32_t fractional_pwm)
+{
+	struct dce_dmcu *dmcu_dce = TO_DCE_DMCU(dmcu);
+
+	/* Wait until microcontroller is ready to process interrupt */
+	REG_WAIT(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 0, 100, 800);
+
+	/* Set PWM fractional enable/disable */
+	REG_WRITE(MASTER_COMM_DATA_REG1, fractional_pwm);
+
+	/* Set command to enable or disable fractional PWM microcontroller */
+	REG_UPDATE(MASTER_COMM_CMD_REG, MASTER_COMM_CMD_REG_BYTE0,
+			MCP_BL_SET_PWM_FRAC);
+
+	/* Notify microcontroller of new command */
+	REG_UPDATE(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 1);
+
+	/* Ensure command has been executed before continuing */
+	REG_WAIT(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 0, 100, 800);
+}
+
 static bool dcn10_dmcu_init(struct dmcu *dmcu)
 {
 	struct dce_dmcu *dmcu_dce = TO_DCE_DMCU(dmcu);
+	const struct dc_config *config = &dmcu->ctx->dc->config;
 	bool status = false;
 
 	/*  Definition of DC_DMCU_SCRATCH
@@ -382,9 +405,14 @@ static bool dcn10_dmcu_init(struct dmcu *dmcu)
 		if (dmcu->dmcu_state == DMCU_RUNNING) {
 			/* Retrieve and cache the DMCU firmware version. */
 			dcn10_get_dmcu_version(dmcu);
+
+			/* Initialize DMCU to use fractional PWM or not */
+			dcn10_dmcu_enable_fractional_pwm(dmcu,
+				(config->disable_fractional_pwm == false) ? 1 : 0);
 			status = true;
-		} else
+		} else {
 			status = false;
+		}
 
 		break;
 	case DMCU_RUNNING:
@@ -594,7 +622,7 @@ static bool dcn10_dmcu_setup_psr(struct dmcu *dmcu,
 	link->link_enc->funcs->psr_program_secondary_packet(link->link_enc,
 			psr_context->sdpTransmitLineNumDeadline);
 
-	if (psr_context->psr_level.bits.SKIP_SMU_NOTIFICATION)
+	if (psr_context->allow_smu_optimizations)
 		REG_UPDATE(SMU_INTERRUPT_CONTROL, DC_SMU_INT_ENABLE, 1);
 
 	/* waitDMCUReadyForCmd */
@@ -615,6 +643,7 @@ static bool dcn10_dmcu_setup_psr(struct dmcu *dmcu,
 			psr_context->psrFrameCaptureIndicationReq;
 	masterCmdData1.bits.aux_chan = psr_context->channel;
 	masterCmdData1.bits.aux_repeat = psr_context->aux_repeats;
+	masterCmdData1.bits.allow_smu_optimizations = psr_context->allow_smu_optimizations;
 	dm_write_reg(dmcu->ctx, REG(MASTER_COMM_DATA_REG1),
 					masterCmdData1.u32All);
 
@@ -635,6 +664,7 @@ static bool dcn10_dmcu_setup_psr(struct dmcu *dmcu,
 	dm_write_reg(dmcu->ctx, REG(MASTER_COMM_DATA_REG3),
 			masterCmdData3.u32All);
 
+
 	/* setDMCUParam_Cmd */
 	REG_UPDATE(MASTER_COMM_CMD_REG,
 			MASTER_COMM_CMD_REG_BYTE0, PSR_SET);
@@ -691,7 +721,7 @@ static bool dcn10_is_dmcu_initialized(struct dmcu *dmcu)
 	return true;
 }
 
-#endif
+#endif //(CONFIG_DRM_AMD_DC_DCN1_0)
 
 static const struct dmcu_funcs dce_funcs = {
 	.dmcu_init = dce_dmcu_init,
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.h b/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.h
index c24c0e5ea44e..60ce56f60ae3 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.h
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.h
@@ -199,16 +199,16 @@ struct dce_dmcu {
  ******************************************************************/
 union dce_dmcu_psr_config_data_reg1 {
 	struct {
-		unsigned int timehyst_frames:8;    /*[7:0]*/
-		unsigned int hyst_lines:7;         /*[14:8]*/
-		unsigned int rfb_update_auto_en:1; /*[15:15]*/
-		unsigned int dp_port_num:3;        /*[18:16]*/
-		unsigned int dcp_sel:3;            /*[21:19]*/
-		unsigned int phy_type:1;           /*[22:22]*/
-		unsigned int frame_cap_ind:1;      /*[23:23]*/
-		unsigned int aux_chan:3;           /*[26:24]*/
-		unsigned int aux_repeat:4;         /*[30:27]*/
-		unsigned int reserved:1;           /*[31:31]*/
+		unsigned int timehyst_frames:8;                  /*[7:0]*/
+		unsigned int hyst_lines:7;                       /*[14:8]*/
+		unsigned int rfb_update_auto_en:1;               /*[15:15]*/
+		unsigned int dp_port_num:3;                      /*[18:16]*/
+		unsigned int dcp_sel:3;                          /*[21:19]*/
+		unsigned int phy_type:1;                         /*[22:22]*/
+		unsigned int frame_cap_ind:1;                    /*[23:23]*/
+		unsigned int aux_chan:3;                         /*[26:24]*/
+		unsigned int aux_repeat:4;                       /*[30:27]*/
+		unsigned int allow_smu_optimizations:1;         /*[31:31]*/
 	} bits;
 	unsigned int u32All;
 };
@@ -236,7 +236,7 @@ union dce_dmcu_psr_config_data_reg3 {
 	struct {
 		unsigned int psr_level:16;      /*[15:0]*/
 		unsigned int link_rate:4;       /*[19:16]*/
-		unsigned int reserved:12;       /*[31:20]*/
+		unsigned int reserved:12;        /*[31:20]*/
 	} bits;
 	unsigned int u32All;
 };
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.c b/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.c
index 40f2d6e0b122..cd26161bcc4d 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.c
@@ -346,6 +346,16 @@ static void release_engine(
 
 }
 
+static bool is_engine_available(struct dce_i2c_hw *dce_i2c_hw)
+{
+	unsigned int arbitrate;
+
+	REG_GET(DC_I2C_ARBITRATION, DC_I2C_REG_RW_CNTL_STATUS, &arbitrate);
+	if (arbitrate == DC_I2C_REG_RW_CNTL_STATUS_DMCU_ONLY)
+		return false;
+	return true;
+}
+
 struct dce_i2c_hw *acquire_i2c_hw_engine(
 	struct resource_pool *pool,
 	struct ddc *ddc)
@@ -368,7 +378,7 @@ struct dce_i2c_hw *acquire_i2c_hw_engine(
 	if (!dce_i2c_hw)
 		return NULL;
 
-	if (pool->i2c_hw_buffer_in_use)
+	if (pool->i2c_hw_buffer_in_use || !is_engine_available(dce_i2c_hw))
 		return NULL;
 
 	do {
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.h b/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.h
index 7f19bb439665..575500755b2e 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.h
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.h
@@ -29,7 +29,8 @@
 enum dc_i2c_status {
 	DC_I2C_STATUS__DC_I2C_STATUS_IDLE,
 	DC_I2C_STATUS__DC_I2C_STATUS_USED_BY_SW,
-	DC_I2C_STATUS__DC_I2C_STATUS_USED_BY_HW
+	DC_I2C_STATUS__DC_I2C_STATUS_USED_BY_HW,
+	DC_I2C_REG_RW_CNTL_STATUS_DMCU_ONLY = 2,
 };
 
 enum dc_i2c_arbitration {
@@ -129,7 +130,8 @@ enum {
 	I2C_SF(DC_I2C_DATA, DC_I2C_DATA, mask_sh),\
 	I2C_SF(DC_I2C_DATA, DC_I2C_INDEX, mask_sh),\
 	I2C_SF(DC_I2C_DATA, DC_I2C_INDEX_WRITE, mask_sh),\
-	I2C_SF(MICROSECOND_TIME_BASE_DIV, XTAL_REF_DIV, mask_sh)
+	I2C_SF(MICROSECOND_TIME_BASE_DIV, XTAL_REF_DIV, mask_sh),\
+	I2C_SF(DC_I2C_ARBITRATION, DC_I2C_REG_RW_CNTL_STATUS, mask_sh)
 
 #define I2C_COMMON_MASK_SH_LIST_DCE110(mask_sh)\
 	I2C_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(mask_sh),\
@@ -170,6 +172,7 @@ struct dce_i2c_shift {
 	uint8_t DC_I2C_INDEX;
 	uint8_t DC_I2C_INDEX_WRITE;
 	uint8_t XTAL_REF_DIV;
+	uint8_t DC_I2C_REG_RW_CNTL_STATUS;
 };
 
 struct dce_i2c_mask {
@@ -207,6 +210,7 @@ struct dce_i2c_mask {
 	uint32_t DC_I2C_INDEX;
 	uint32_t DC_I2C_INDEX_WRITE;
 	uint32_t XTAL_REF_DIV;
+	uint32_t DC_I2C_REG_RW_CNTL_STATUS;
 };
 
 struct dce_i2c_registers {
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.c
index 1fa2d4fd7a35..14309fe6f2e6 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.c
@@ -272,7 +272,8 @@ static void dce110_update_hdmi_info_packet(
 static void dce110_stream_encoder_dp_set_stream_attribute(
 	struct stream_encoder *enc,
 	struct dc_crtc_timing *crtc_timing,
-	enum dc_color_space output_color_space)
+	enum dc_color_space output_color_space,
+	uint32_t enable_sdp_splitting)
 {
 #if defined(CONFIG_DRM_AMD_DC_DCN1_0)
 	uint32_t h_active_start;
@@ -977,7 +978,7 @@ static void dce110_stream_encoder_dp_unblank(
 
 		uint64_t m_vid_l = n_vid;
 
-		m_vid_l *= param->pixel_clk_khz;
+		m_vid_l *= param->timing.pix_clk_100hz / 10;
 		m_vid_l = div_u64(m_vid_l,
 			param->link_settings.link_rate
 				* LINK_RATE_REF_FREQ_IN_KHZ);
diff --git a/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.c b/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.c
index 23044e6723e8..e938bf9986d3 100644
--- a/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.c
@@ -378,6 +378,28 @@ static const struct resource_caps res_cap = {
 	.num_ddc = 6,
 };
 
+static const struct dc_plane_cap plane_cap = {
+	.type = DC_PLANE_TYPE_DCE_RGB,
+
+	.pixel_format_support = {
+			.argb8888 = true,
+			.nv12 = false,
+			.fp16 = false
+	},
+
+	.max_upscale_factor = {
+			.argb8888 = 16000,
+			.nv12 = 1,
+			.fp16 = 1
+	},
+
+	.max_downscale_factor = {
+			.argb8888 = 250,
+			.nv12 = 1,
+			.fp16 = 1
+	}
+};
+
 #define CTX  ctx
 #define REG(reg) mm ## reg
 
@@ -756,7 +778,8 @@ static enum dc_status build_mapped_resource(
 
 bool dce100_validate_bandwidth(
 	struct dc  *dc,
-	struct dc_state *context)
+	struct dc_state *context,
+	bool fast_validate)
 {
 	int i;
 	bool at_least_one_pipe = false;
@@ -768,11 +791,11 @@ bool dce100_validate_bandwidth(
 
 	if (at_least_one_pipe) {
 		/* TODO implement when needed but for now hardcode max value*/
-		context->bw.dce.dispclk_khz = 681000;
-		context->bw.dce.yclk_khz = 250000 * MEMORY_TYPE_MULTIPLIER_CZ;
+		context->bw_ctx.bw.dce.dispclk_khz = 681000;
+		context->bw_ctx.bw.dce.yclk_khz = 250000 * MEMORY_TYPE_MULTIPLIER_CZ;
 	} else {
-		context->bw.dce.dispclk_khz = 0;
-		context->bw.dce.yclk_khz = 0;
+		context->bw_ctx.bw.dce.dispclk_khz = 0;
+		context->bw_ctx.bw.dce.yclk_khz = 0;
 	}
 
 	return true;
@@ -1023,6 +1046,9 @@ static bool construct(
 
 	dc->caps.max_planes =  pool->base.pipe_count;
 
+	for (i = 0; i < dc->caps.max_planes; ++i)
+		dc->caps.planes[i] = plane_cap;
+
 	if (!resource_construct(num_virtual_links, dc, &pool->base,
 			&res_create_funcs))
 		goto res_create_fail;
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c
index 5e4db3712eef..7ac50ab1b762 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c
+++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c
@@ -616,7 +616,7 @@ dce110_set_output_transfer_func(struct pipe_ctx *pipe_ctx,
 
 void dce110_update_info_frame(struct pipe_ctx *pipe_ctx)
 {
-	bool is_hdmi;
+	bool is_hdmi_tmds;
 	bool is_dp;
 
 	ASSERT(pipe_ctx->stream);
@@ -624,13 +624,13 @@ void dce110_update_info_frame(struct pipe_ctx *pipe_ctx)
 	if (pipe_ctx->stream_res.stream_enc == NULL)
 		return;  /* this is not root pipe */
 
-	is_hdmi = dc_is_hdmi_signal(pipe_ctx->stream->signal);
+	is_hdmi_tmds = dc_is_hdmi_tmds_signal(pipe_ctx->stream->signal);
 	is_dp = dc_is_dp_signal(pipe_ctx->stream->signal);
 
-	if (!is_hdmi && !is_dp)
+	if (!is_hdmi_tmds && !is_dp)
 		return;
 
-	if (is_hdmi)
+	if (is_hdmi_tmds)
 		pipe_ctx->stream_res.stream_enc->funcs->update_hdmi_info_packets(
 			pipe_ctx->stream_res.stream_enc,
 			&pipe_ctx->stream_res.encoder_info_frame);
@@ -935,13 +935,31 @@ void hwss_edp_backlight_control(
 		edp_receiver_ready_T9(link);
 }
 
+// Static helper function which calls the correct function
+// based on pp_smu version
+static void set_pme_wa_enable_by_version(struct dc *dc)
+{
+	struct pp_smu_funcs *pp_smu = NULL;
+
+	if (dc->res_pool->pp_smu)
+		pp_smu = dc->res_pool->pp_smu;
+
+	if (pp_smu) {
+		if (pp_smu->ctx.ver == PP_SMU_VER_RV && pp_smu->rv_funcs.set_pme_wa_enable)
+			pp_smu->rv_funcs.set_pme_wa_enable(&(pp_smu->ctx));
+	}
+}
+
 void dce110_enable_audio_stream(struct pipe_ctx *pipe_ctx)
 {
-	struct dc *core_dc = pipe_ctx->stream->ctx->dc;
 	/* notify audio driver for audio modes of monitor */
-	struct pp_smu_funcs_rv *pp_smu = core_dc->res_pool->pp_smu;
+	struct dc *core_dc = pipe_ctx->stream->ctx->dc;
+	struct pp_smu_funcs *pp_smu = NULL;
 	unsigned int i, num_audio = 1;
 
+	if (core_dc->res_pool->pp_smu)
+		pp_smu = core_dc->res_pool->pp_smu;
+
 	if (pipe_ctx->stream_res.audio) {
 		for (i = 0; i < MAX_PIPES; i++) {
 			/*current_state not updated yet*/
@@ -951,30 +969,31 @@ void dce110_enable_audio_stream(struct pipe_ctx *pipe_ctx)
 
 		pipe_ctx->stream_res.audio->funcs->az_enable(pipe_ctx->stream_res.audio);
 
-		if (num_audio >= 1 && pp_smu != NULL && pp_smu->set_pme_wa_enable != NULL)
+		if (num_audio >= 1 && pp_smu != NULL)
 			/*this is the first audio. apply the PME w/a in order to wake AZ from D3*/
-			pp_smu->set_pme_wa_enable(&pp_smu->pp_smu);
+			set_pme_wa_enable_by_version(core_dc);
 		/* un-mute audio */
 		/* TODO: audio should be per stream rather than per link */
 		pipe_ctx->stream_res.stream_enc->funcs->audio_mute_control(
-			pipe_ctx->stream_res.stream_enc, false);
+					pipe_ctx->stream_res.stream_enc, false);
 	}
 }
 
 void dce110_disable_audio_stream(struct pipe_ctx *pipe_ctx, int option)
 {
 	struct dc *dc = pipe_ctx->stream->ctx->dc;
+	struct pp_smu_funcs *pp_smu = NULL;
 
 	pipe_ctx->stream_res.stream_enc->funcs->audio_mute_control(
 			pipe_ctx->stream_res.stream_enc, true);
 	if (pipe_ctx->stream_res.audio) {
-		struct pp_smu_funcs_rv *pp_smu = dc->res_pool->pp_smu;
+		if (dc->res_pool->pp_smu)
+			pp_smu = dc->res_pool->pp_smu;
 
 		if (option != KEEP_ACQUIRED_RESOURCE ||
-				!dc->debug.az_endpoint_mute_only) {
+				!dc->debug.az_endpoint_mute_only)
 			/*only disalbe az_endpoint if power down or free*/
 			pipe_ctx->stream_res.audio->funcs->az_disable(pipe_ctx->stream_res.audio);
-		}
 
 		if (dc_is_dp_signal(pipe_ctx->stream->signal))
 			pipe_ctx->stream_res.stream_enc->funcs->dp_audio_disable(
@@ -989,9 +1008,9 @@ void dce110_disable_audio_stream(struct pipe_ctx *pipe_ctx, int option)
 			update_audio_usage(&dc->current_state->res_ctx, dc->res_pool, pipe_ctx->stream_res.audio, false);
 			pipe_ctx->stream_res.audio = NULL;
 		}
-		if (pp_smu != NULL && pp_smu->set_pme_wa_enable != NULL)
+		if (pp_smu != NULL)
 			/*this is the first audio. apply the PME w/a in order to wake AZ from D3*/
-			pp_smu->set_pme_wa_enable(&pp_smu->pp_smu);
+			set_pme_wa_enable_by_version(dc);
 
 		/* TODO: notify audio driver for if audio modes list changed
 		 * add audio mode list change flag */
@@ -1007,7 +1026,7 @@ void dce110_disable_stream(struct pipe_ctx *pipe_ctx, int option)
 	struct dc_link *link = stream->link;
 	struct dc *dc = pipe_ctx->stream->ctx->dc;
 
-	if (dc_is_hdmi_signal(pipe_ctx->stream->signal))
+	if (dc_is_hdmi_tmds_signal(pipe_ctx->stream->signal))
 		pipe_ctx->stream_res.stream_enc->funcs->stop_hdmi_info_packets(
 			pipe_ctx->stream_res.stream_enc);
 
@@ -1032,7 +1051,7 @@ void dce110_unblank_stream(struct pipe_ctx *pipe_ctx,
 	struct dc_link *link = stream->link;
 
 	/* only 3 items below are used by unblank */
-	params.pixel_clk_khz = pipe_ctx->stream->timing.pix_clk_100hz / 10;
+	params.timing = pipe_ctx->stream->timing;
 	params.link_settings.link_rate = link_settings->link_rate;
 
 	if (dc_is_dp_signal(pipe_ctx->stream->signal))
@@ -1147,8 +1166,8 @@ static void build_audio_output(
 	if (pipe_ctx->stream->signal == SIGNAL_TYPE_DISPLAY_PORT ||
 			pipe_ctx->stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST) {
 		audio_output->pll_info.dp_dto_source_clock_in_khz =
-				state->dccg->funcs->get_dp_ref_clk_frequency(
-						state->dccg);
+				state->clk_mgr->funcs->get_dp_ref_clk_frequency(
+						state->clk_mgr);
 	}
 
 	audio_output->pll_info.feed_back_divider =
@@ -1349,7 +1368,7 @@ static enum dc_status apply_single_controller_ctx_to_hw(
 		pipe_ctx->stream_res.tg->funcs->set_static_screen_control(
 				pipe_ctx->stream_res.tg, event_triggers);
 
-	if (pipe_ctx->stream->signal != SIGNAL_TYPE_VIRTUAL)
+	if (!dc_is_virtual_signal(pipe_ctx->stream->signal))
 		pipe_ctx->stream_res.stream_enc->funcs->dig_connect_to_otg(
 			pipe_ctx->stream_res.stream_enc,
 			pipe_ctx->stream_res.tg->inst);
@@ -1358,7 +1377,7 @@ static enum dc_status apply_single_controller_ctx_to_hw(
 			pipe_ctx->stream_res.opp,
 			COLOR_SPACE_YCBCR601,
 			stream->timing.display_color_depth,
-			pipe_ctx->stream->signal);
+			stream->signal);
 
 	pipe_ctx->stream_res.opp->funcs->opp_program_fmt(
 		pipe_ctx->stream_res.opp,
@@ -1532,6 +1551,9 @@ void dce110_enable_accelerated_mode(struct dc *dc, struct dc_state *context)
 		}
 	}
 
+	if (dc->hwss.init_pipes)
+		dc->hwss.init_pipes(dc, context);
+
 	if (edp_link) {
 		/* this seems to cause blank screens on DCE8 */
 		if ((dc->ctx->dce_version == DCE_VERSION_8_0) ||
@@ -1608,18 +1630,18 @@ static void dce110_set_displaymarks(
 			dc->bw_vbios->blackout_duration, pipe_ctx->stream);
 		pipe_ctx->plane_res.mi->funcs->mem_input_program_display_marks(
 			pipe_ctx->plane_res.mi,
-			context->bw.dce.nbp_state_change_wm_ns[num_pipes],
-			context->bw.dce.stutter_exit_wm_ns[num_pipes],
-			context->bw.dce.stutter_entry_wm_ns[num_pipes],
-			context->bw.dce.urgent_wm_ns[num_pipes],
+			context->bw_ctx.bw.dce.nbp_state_change_wm_ns[num_pipes],
+			context->bw_ctx.bw.dce.stutter_exit_wm_ns[num_pipes],
+			context->bw_ctx.bw.dce.stutter_entry_wm_ns[num_pipes],
+			context->bw_ctx.bw.dce.urgent_wm_ns[num_pipes],
 			total_dest_line_time_ns);
 		if (i == underlay_idx) {
 			num_pipes++;
 			pipe_ctx->plane_res.mi->funcs->mem_input_program_chroma_display_marks(
 				pipe_ctx->plane_res.mi,
-				context->bw.dce.nbp_state_change_wm_ns[num_pipes],
-				context->bw.dce.stutter_exit_wm_ns[num_pipes],
-				context->bw.dce.urgent_wm_ns[num_pipes],
+				context->bw_ctx.bw.dce.nbp_state_change_wm_ns[num_pipes],
+				context->bw_ctx.bw.dce.stutter_exit_wm_ns[num_pipes],
+				context->bw_ctx.bw.dce.urgent_wm_ns[num_pipes],
 				total_dest_line_time_ns);
 		}
 		num_pipes++;
@@ -2612,7 +2634,7 @@ void dce110_set_cursor_position(struct pipe_ctx *pipe_ctx)
 	struct mem_input *mi = pipe_ctx->plane_res.mi;
 	struct dc_cursor_mi_param param = {
 		.pixel_clk_khz = pipe_ctx->stream->timing.pix_clk_100hz / 10,
-		.ref_clk_khz = pipe_ctx->stream->ctx->dc->res_pool->ref_clock_inKhz,
+		.ref_clk_khz = pipe_ctx->stream->ctx->dc->res_pool->ref_clocks.xtalin_clock_inKhz,
 		.viewport = pipe_ctx->plane_res.scl_data.viewport,
 		.h_scale_ratio = pipe_ctx->plane_res.scl_data.ratios.horz,
 		.v_scale_ratio = pipe_ctx->plane_res.scl_data.ratios.vert,
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.c
index 7549adaa1542..dcd04e9ea76b 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.c
@@ -392,6 +392,55 @@ static const struct resource_caps stoney_resource_cap = {
 		.num_ddc = 3,
 };
 
+static const struct dc_plane_cap plane_cap = {
+		.type = DC_PLANE_TYPE_DCE_RGB,
+		.blends_with_below = true,
+		.blends_with_above = true,
+		.per_pixel_alpha = 1,
+
+		.pixel_format_support = {
+				.argb8888 = true,
+				.nv12 = false,
+				.fp16 = false
+		},
+
+		.max_upscale_factor = {
+				.argb8888 = 16000,
+				.nv12 = 1,
+				.fp16 = 1
+		},
+
+		.max_downscale_factor = {
+				.argb8888 = 250,
+				.nv12 = 1,
+				.fp16 = 1
+		}
+};
+
+static const struct dc_plane_cap underlay_plane_cap = {
+		.type = DC_PLANE_TYPE_DCE_UNDERLAY,
+		.blends_with_above = true,
+		.per_pixel_alpha = 1,
+
+		.pixel_format_support = {
+				.argb8888 = false,
+				.nv12 = true,
+				.fp16 = false
+		},
+
+		.max_upscale_factor = {
+				.argb8888 = 1,
+				.nv12 = 16000,
+				.fp16 = 1
+		},
+
+		.max_downscale_factor = {
+				.argb8888 = 1,
+				.nv12 = 250,
+				.fp16 = 1
+		}
+};
+
 #define CTX  ctx
 #define REG(reg) mm ## reg
 
@@ -854,7 +903,8 @@ static enum dc_status build_mapped_resource(
 
 static bool dce110_validate_bandwidth(
 	struct dc *dc,
-	struct dc_state *context)
+	struct dc_state *context,
+	bool fast_validate)
 {
 	bool result = false;
 
@@ -868,7 +918,7 @@ static bool dce110_validate_bandwidth(
 			dc->bw_vbios,
 			context->res_ctx.pipe_ctx,
 			dc->res_pool->pipe_count,
-			&context->bw.dce))
+			&context->bw_ctx.bw.dce))
 		result =  true;
 
 	if (!result)
@@ -878,8 +928,8 @@ static bool dce110_validate_bandwidth(
 			context->streams[0]->timing.v_addressable,
 			context->streams[0]->timing.pix_clk_100hz / 10);
 
-	if (memcmp(&dc->current_state->bw.dce,
-			&context->bw.dce, sizeof(context->bw.dce))) {
+	if (memcmp(&dc->current_state->bw_ctx.bw.dce,
+			&context->bw_ctx.bw.dce, sizeof(context->bw_ctx.bw.dce))) {
 
 		DC_LOG_BANDWIDTH_CALCS(
 			"%s: finish,\n"
@@ -893,34 +943,34 @@ static bool dce110_validate_bandwidth(
 			"sclk: %d sclk_sleep: %d yclk: %d blackout_recovery_time_us: %d\n"
 			,
 			__func__,
-			context->bw.dce.nbp_state_change_wm_ns[0].b_mark,
-			context->bw.dce.nbp_state_change_wm_ns[0].a_mark,
-			context->bw.dce.urgent_wm_ns[0].b_mark,
-			context->bw.dce.urgent_wm_ns[0].a_mark,
-			context->bw.dce.stutter_exit_wm_ns[0].b_mark,
-			context->bw.dce.stutter_exit_wm_ns[0].a_mark,
-			context->bw.dce.nbp_state_change_wm_ns[1].b_mark,
-			context->bw.dce.nbp_state_change_wm_ns[1].a_mark,
-			context->bw.dce.urgent_wm_ns[1].b_mark,
-			context->bw.dce.urgent_wm_ns[1].a_mark,
-			context->bw.dce.stutter_exit_wm_ns[1].b_mark,
-			context->bw.dce.stutter_exit_wm_ns[1].a_mark,
-			context->bw.dce.nbp_state_change_wm_ns[2].b_mark,
-			context->bw.dce.nbp_state_change_wm_ns[2].a_mark,
-			context->bw.dce.urgent_wm_ns[2].b_mark,
-			context->bw.dce.urgent_wm_ns[2].a_mark,
-			context->bw.dce.stutter_exit_wm_ns[2].b_mark,
-			context->bw.dce.stutter_exit_wm_ns[2].a_mark,
-			context->bw.dce.stutter_mode_enable,
-			context->bw.dce.cpuc_state_change_enable,
-			context->bw.dce.cpup_state_change_enable,
-			context->bw.dce.nbp_state_change_enable,
-			context->bw.dce.all_displays_in_sync,
-			context->bw.dce.dispclk_khz,
-			context->bw.dce.sclk_khz,
-			context->bw.dce.sclk_deep_sleep_khz,
-			context->bw.dce.yclk_khz,
-			context->bw.dce.blackout_recovery_time_us);
+			context->bw_ctx.bw.dce.nbp_state_change_wm_ns[0].b_mark,
+			context->bw_ctx.bw.dce.nbp_state_change_wm_ns[0].a_mark,
+			context->bw_ctx.bw.dce.urgent_wm_ns[0].b_mark,
+			context->bw_ctx.bw.dce.urgent_wm_ns[0].a_mark,
+			context->bw_ctx.bw.dce.stutter_exit_wm_ns[0].b_mark,
+			context->bw_ctx.bw.dce.stutter_exit_wm_ns[0].a_mark,
+			context->bw_ctx.bw.dce.nbp_state_change_wm_ns[1].b_mark,
+			context->bw_ctx.bw.dce.nbp_state_change_wm_ns[1].a_mark,
+			context->bw_ctx.bw.dce.urgent_wm_ns[1].b_mark,
+			context->bw_ctx.bw.dce.urgent_wm_ns[1].a_mark,
+			context->bw_ctx.bw.dce.stutter_exit_wm_ns[1].b_mark,
+			context->bw_ctx.bw.dce.stutter_exit_wm_ns[1].a_mark,
+			context->bw_ctx.bw.dce.nbp_state_change_wm_ns[2].b_mark,
+			context->bw_ctx.bw.dce.nbp_state_change_wm_ns[2].a_mark,
+			context->bw_ctx.bw.dce.urgent_wm_ns[2].b_mark,
+			context->bw_ctx.bw.dce.urgent_wm_ns[2].a_mark,
+			context->bw_ctx.bw.dce.stutter_exit_wm_ns[2].b_mark,
+			context->bw_ctx.bw.dce.stutter_exit_wm_ns[2].a_mark,
+			context->bw_ctx.bw.dce.stutter_mode_enable,
+			context->bw_ctx.bw.dce.cpuc_state_change_enable,
+			context->bw_ctx.bw.dce.cpup_state_change_enable,
+			context->bw_ctx.bw.dce.nbp_state_change_enable,
+			context->bw_ctx.bw.dce.all_displays_in_sync,
+			context->bw_ctx.bw.dce.dispclk_khz,
+			context->bw_ctx.bw.dce.sclk_khz,
+			context->bw_ctx.bw.dce.sclk_deep_sleep_khz,
+			context->bw_ctx.bw.dce.yclk_khz,
+			context->bw_ctx.bw.dce.blackout_recovery_time_us);
 	}
 	return result;
 }
@@ -1371,6 +1421,11 @@ static bool construct(
 
 	dc->caps.max_planes =  pool->base.pipe_count;
 
+	for (i = 0; i < pool->base.underlay_pipe_index; ++i)
+		dc->caps.planes[i] = plane_cap;
+
+	dc->caps.planes[pool->base.underlay_pipe_index] = underlay_plane_cap;
+
 	bw_calcs_init(dc->bw_dceip, dc->bw_vbios, dc->ctx->asic_id);
 
 	bw_calcs_data_update_from_pplib(dc);
diff --git a/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.c b/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.c
index ea3065d63372..a480b15f6885 100644
--- a/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.c
@@ -397,6 +397,28 @@ static const struct resource_caps polaris_11_resource_cap = {
 		.num_ddc = 5,
 };
 
+static const struct dc_plane_cap plane_cap = {
+	.type = DC_PLANE_TYPE_DCE_RGB,
+
+	.pixel_format_support = {
+			.argb8888 = true,
+			.nv12 = false,
+			.fp16 = false
+	},
+
+	.max_upscale_factor = {
+			.argb8888 = 16000,
+			.nv12 = 1,
+			.fp16 = 1
+	},
+
+	.max_downscale_factor = {
+			.argb8888 = 250,
+			.nv12 = 1,
+			.fp16 = 1
+	}
+};
+
 #define CTX  ctx
 #define REG(reg) mm ## reg
 
@@ -804,7 +826,8 @@ static enum dc_status build_mapped_resource(
 
 bool dce112_validate_bandwidth(
 	struct dc *dc,
-	struct dc_state *context)
+	struct dc_state *context,
+	bool fast_validate)
 {
 	bool result = false;
 
@@ -818,7 +841,7 @@ bool dce112_validate_bandwidth(
 			dc->bw_vbios,
 			context->res_ctx.pipe_ctx,
 			dc->res_pool->pipe_count,
-			&context->bw.dce))
+			&context->bw_ctx.bw.dce))
 		result = true;
 
 	if (!result)
@@ -826,8 +849,8 @@ bool dce112_validate_bandwidth(
 			"%s: Bandwidth validation failed!",
 			__func__);
 
-	if (memcmp(&dc->current_state->bw.dce,
-			&context->bw.dce, sizeof(context->bw.dce))) {
+	if (memcmp(&dc->current_state->bw_ctx.bw.dce,
+			&context->bw_ctx.bw.dce, sizeof(context->bw_ctx.bw.dce))) {
 
 		DC_LOG_BANDWIDTH_CALCS(
 			"%s: finish,\n"
@@ -841,34 +864,34 @@ bool dce112_validate_bandwidth(
 			"sclk: %d sclk_sleep: %d yclk: %d blackout_recovery_time_us: %d\n"
 			,
 			__func__,
-			context->bw.dce.nbp_state_change_wm_ns[0].b_mark,
-			context->bw.dce.nbp_state_change_wm_ns[0].a_mark,
-			context->bw.dce.urgent_wm_ns[0].b_mark,
-			context->bw.dce.urgent_wm_ns[0].a_mark,
-			context->bw.dce.stutter_exit_wm_ns[0].b_mark,
-			context->bw.dce.stutter_exit_wm_ns[0].a_mark,
-			context->bw.dce.nbp_state_change_wm_ns[1].b_mark,
-			context->bw.dce.nbp_state_change_wm_ns[1].a_mark,
-			context->bw.dce.urgent_wm_ns[1].b_mark,
-			context->bw.dce.urgent_wm_ns[1].a_mark,
-			context->bw.dce.stutter_exit_wm_ns[1].b_mark,
-			context->bw.dce.stutter_exit_wm_ns[1].a_mark,
-			context->bw.dce.nbp_state_change_wm_ns[2].b_mark,
-			context->bw.dce.nbp_state_change_wm_ns[2].a_mark,
-			context->bw.dce.urgent_wm_ns[2].b_mark,
-			context->bw.dce.urgent_wm_ns[2].a_mark,
-			context->bw.dce.stutter_exit_wm_ns[2].b_mark,
-			context->bw.dce.stutter_exit_wm_ns[2].a_mark,
-			context->bw.dce.stutter_mode_enable,
-			context->bw.dce.cpuc_state_change_enable,
-			context->bw.dce.cpup_state_change_enable,
-			context->bw.dce.nbp_state_change_enable,
-			context->bw.dce.all_displays_in_sync,
-			context->bw.dce.dispclk_khz,
-			context->bw.dce.sclk_khz,
-			context->bw.dce.sclk_deep_sleep_khz,
-			context->bw.dce.yclk_khz,
-			context->bw.dce.blackout_recovery_time_us);
+			context->bw_ctx.bw.dce.nbp_state_change_wm_ns[0].b_mark,
+			context->bw_ctx.bw.dce.nbp_state_change_wm_ns[0].a_mark,
+			context->bw_ctx.bw.dce.urgent_wm_ns[0].b_mark,
+			context->bw_ctx.bw.dce.urgent_wm_ns[0].a_mark,
+			context->bw_ctx.bw.dce.stutter_exit_wm_ns[0].b_mark,
+			context->bw_ctx.bw.dce.stutter_exit_wm_ns[0].a_mark,
+			context->bw_ctx.bw.dce.nbp_state_change_wm_ns[1].b_mark,
+			context->bw_ctx.bw.dce.nbp_state_change_wm_ns[1].a_mark,
+			context->bw_ctx.bw.dce.urgent_wm_ns[1].b_mark,
+			context->bw_ctx.bw.dce.urgent_wm_ns[1].a_mark,
+			context->bw_ctx.bw.dce.stutter_exit_wm_ns[1].b_mark,
+			context->bw_ctx.bw.dce.stutter_exit_wm_ns[1].a_mark,
+			context->bw_ctx.bw.dce.nbp_state_change_wm_ns[2].b_mark,
+			context->bw_ctx.bw.dce.nbp_state_change_wm_ns[2].a_mark,
+			context->bw_ctx.bw.dce.urgent_wm_ns[2].b_mark,
+			context->bw_ctx.bw.dce.urgent_wm_ns[2].a_mark,
+			context->bw_ctx.bw.dce.stutter_exit_wm_ns[2].b_mark,
+			context->bw_ctx.bw.dce.stutter_exit_wm_ns[2].a_mark,
+			context->bw_ctx.bw.dce.stutter_mode_enable,
+			context->bw_ctx.bw.dce.cpuc_state_change_enable,
+			context->bw_ctx.bw.dce.cpup_state_change_enable,
+			context->bw_ctx.bw.dce.nbp_state_change_enable,
+			context->bw_ctx.bw.dce.all_displays_in_sync,
+			context->bw_ctx.bw.dce.dispclk_khz,
+			context->bw_ctx.bw.dce.sclk_khz,
+			context->bw_ctx.bw.dce.sclk_deep_sleep_khz,
+			context->bw_ctx.bw.dce.yclk_khz,
+			context->bw_ctx.bw.dce.blackout_recovery_time_us);
 	}
 	return result;
 }
@@ -887,7 +910,7 @@ enum dc_status resource_map_phy_clock_resources(
 		return DC_ERROR_UNEXPECTED;
 
 	if (dc_is_dp_signal(pipe_ctx->stream->signal)
-		|| pipe_ctx->stream->signal == SIGNAL_TYPE_VIRTUAL)
+		|| dc_is_virtual_signal(pipe_ctx->stream->signal))
 		pipe_ctx->clock_source =
 				dc->res_pool->dp_clock_source;
 	else
@@ -1310,6 +1333,9 @@ static bool construct(
 
 	dc->caps.max_planes =  pool->base.pipe_count;
 
+	for (i = 0; i < dc->caps.max_planes; ++i)
+		dc->caps.planes[i] = plane_cap;
+
 	/* Create hardware sequencer */
 	dce112_hw_sequencer_construct(dc);
 
diff --git a/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.h b/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.h
index 95a403396219..1f57ebc6f9b4 100644
--- a/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.h
+++ b/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.h
@@ -44,7 +44,8 @@ enum dc_status dce112_validate_with_context(
 
 bool dce112_validate_bandwidth(
 	struct dc *dc,
-	struct dc_state *context);
+	struct dc_state *context,
+	bool fast_validate);
 
 enum dc_status dce112_add_stream_to_ctx(
 		struct dc *dc,
diff --git a/drivers/gpu/drm/amd/display/dc/dce120/dce120_resource.c b/drivers/gpu/drm/amd/display/dc/dce120/dce120_resource.c
index 312a0aebf91f..6d49c7143c67 100644
--- a/drivers/gpu/drm/amd/display/dc/dce120/dce120_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dce120/dce120_resource.c
@@ -454,6 +454,28 @@ static const struct resource_caps res_cap = {
 		.num_ddc = 6,
 };
 
+static const struct dc_plane_cap plane_cap = {
+	.type = DC_PLANE_TYPE_DCE_RGB,
+
+	.pixel_format_support = {
+			.argb8888 = true,
+			.nv12 = false,
+			.fp16 = false
+	},
+
+	.max_upscale_factor = {
+			.argb8888 = 16000,
+			.nv12 = 1,
+			.fp16 = 1
+	},
+
+	.max_downscale_factor = {
+			.argb8888 = 250,
+			.nv12 = 1,
+			.fp16 = 1
+	}
+};
+
 static const struct dc_debug_options debug_defaults = {
 		.disable_clock_gate = true,
 };
@@ -1171,6 +1193,9 @@ static bool construct(
 
 	dc->caps.max_planes =  pool->base.pipe_count;
 
+	for (i = 0; i < dc->caps.max_planes; ++i)
+		dc->caps.planes[i] = plane_cap;
+
 	bw_calcs_init(dc->bw_dceip, dc->bw_vbios, dc->ctx->asic_id);
 
 	bw_calcs_data_update_from_pplib(dc);
diff --git a/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c b/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c
index c109ace96be9..27d0cc394963 100644
--- a/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c
@@ -387,6 +387,28 @@ static const struct resource_caps res_cap_83 = {
 		.num_ddc = 2,
 };
 
+static const struct dc_plane_cap plane_cap = {
+	.type = DC_PLANE_TYPE_DCE_RGB,
+
+	.pixel_format_support = {
+			.argb8888 = true,
+			.nv12 = false,
+			.fp16 = false
+	},
+
+	.max_upscale_factor = {
+			.argb8888 = 16000,
+			.nv12 = 1,
+			.fp16 = 1
+	},
+
+	.max_downscale_factor = {
+			.argb8888 = 250,
+			.nv12 = 1,
+			.fp16 = 1
+	}
+};
+
 static const struct dce_dmcu_registers dmcu_regs = {
 		DMCU_DCE80_REG_LIST()
 };
@@ -790,7 +812,8 @@ static void destruct(struct dce110_resource_pool *pool)
 
 bool dce80_validate_bandwidth(
 	struct dc *dc,
-	struct dc_state *context)
+	struct dc_state *context,
+	bool fast_validate)
 {
 	int i;
 	bool at_least_one_pipe = false;
@@ -802,11 +825,11 @@ bool dce80_validate_bandwidth(
 
 	if (at_least_one_pipe) {
 		/* TODO implement when needed but for now hardcode max value*/
-		context->bw.dce.dispclk_khz = 681000;
-		context->bw.dce.yclk_khz = 250000 * MEMORY_TYPE_MULTIPLIER_CZ;
+		context->bw_ctx.bw.dce.dispclk_khz = 681000;
+		context->bw_ctx.bw.dce.yclk_khz = 250000 * MEMORY_TYPE_MULTIPLIER_CZ;
 	} else {
-		context->bw.dce.dispclk_khz = 0;
-		context->bw.dce.yclk_khz = 0;
+		context->bw_ctx.bw.dce.dispclk_khz = 0;
+		context->bw_ctx.bw.dce.yclk_khz = 0;
 	}
 
 	return true;
@@ -1032,6 +1055,10 @@ static bool dce80_construct(
 	}
 
 	dc->caps.max_planes =  pool->base.pipe_count;
+
+	for (i = 0; i < dc->caps.max_planes; ++i)
+		dc->caps.planes[i] = plane_cap;
+
 	dc->caps.disable_dp_clk_share = true;
 
 	if (!resource_construct(num_virtual_links, dc, &pool->base,
@@ -1237,6 +1264,10 @@ static bool dce81_construct(
 	}
 
 	dc->caps.max_planes =  pool->base.pipe_count;
+
+	for (i = 0; i < dc->caps.max_planes; ++i)
+		dc->caps.planes[i] = plane_cap;
+
 	dc->caps.disable_dp_clk_share = true;
 
 	if (!resource_construct(num_virtual_links, dc, &pool->base,
@@ -1438,6 +1469,10 @@ static bool dce83_construct(
 	}
 
 	dc->caps.max_planes =  pool->base.pipe_count;
+
+	for (i = 0; i < dc->caps.max_planes; ++i)
+		dc->caps.planes[i] = plane_cap;
+
 	dc->caps.disable_dp_clk_share = true;
 
 	if (!resource_construct(num_virtual_links, dc, &pool->base,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_clk_mgr.c
index afe8c42211cd..2b2de1d913c9 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_clk_mgr.c
@@ -43,23 +43,6 @@
 #define DC_LOGGER \
 	clk_mgr->ctx->logger
 
-void dcn1_pplib_apply_display_requirements(
-	struct dc *dc,
-	struct dc_state *context)
-{
-	struct dm_pp_display_configuration *pp_display_cfg = &context->pp_display_cfg;
-
-	pp_display_cfg->min_engine_clock_khz = dc->res_pool->clk_mgr->clks.dcfclk_khz;
-	pp_display_cfg->min_memory_clock_khz = dc->res_pool->clk_mgr->clks.fclk_khz;
-	pp_display_cfg->min_engine_clock_deep_sleep_khz = dc->res_pool->clk_mgr->clks.dcfclk_deep_sleep_khz;
-	pp_display_cfg->min_dcfc_deep_sleep_clock_khz = dc->res_pool->clk_mgr->clks.dcfclk_deep_sleep_khz;
-	pp_display_cfg->min_dcfclock_khz = dc->res_pool->clk_mgr->clks.dcfclk_khz;
-	pp_display_cfg->disp_clk_khz = dc->res_pool->clk_mgr->clks.dispclk_khz;
-	dce110_fill_display_configs(context, pp_display_cfg);
-
-	dm_pp_apply_display_requirements(dc->ctx, pp_display_cfg);
-}
-
 static int dcn1_determine_dppclk_threshold(struct clk_mgr *clk_mgr, struct dc_clocks *new_clocks)
 {
 	bool request_dpp_div = new_clocks->dispclk_khz > new_clocks->dppclk_khz;
@@ -167,11 +150,8 @@ static void dcn1_update_clocks(struct clk_mgr *clk_mgr,
 {
 	struct dc *dc = clk_mgr->ctx->dc;
 	struct dc_debug_options *debug = &dc->debug;
-	struct dc_clocks *new_clocks = &context->bw.dcn.clk;
-	struct pp_smu_display_requirement_rv *smu_req_cur =
-			&dc->res_pool->pp_smu_req;
-	struct pp_smu_display_requirement_rv smu_req = *smu_req_cur;
-	struct pp_smu_funcs_rv *pp_smu = dc->res_pool->pp_smu;
+	struct dc_clocks *new_clocks = &context->bw_ctx.bw.dcn.clk;
+	struct pp_smu_funcs_rv *pp_smu = NULL;
 	bool send_request_to_increase = false;
 	bool send_request_to_lower = false;
 	int display_count;
@@ -179,7 +159,8 @@ static void dcn1_update_clocks(struct clk_mgr *clk_mgr,
 	bool enter_display_off = false;
 
 	display_count = get_active_display_cnt(dc, context);
-
+	if (dc->res_pool->pp_smu)
+		pp_smu = &dc->res_pool->pp_smu->rv_funcs;
 	if (display_count == 0)
 		enter_display_off = true;
 
@@ -189,10 +170,8 @@ static void dcn1_update_clocks(struct clk_mgr *clk_mgr,
 		 * if function pointer not set up, this message is
 		 * sent as part of pplib_apply_display_requirements.
 		 */
-		if (pp_smu->set_display_count)
+		if (pp_smu && pp_smu->set_display_count)
 			pp_smu->set_display_count(&pp_smu->pp_smu, display_count);
-
-		smu_req.display_count = display_count;
 	}
 
 	if (new_clocks->dispclk_khz > clk_mgr->clks.dispclk_khz
@@ -203,7 +182,6 @@ static void dcn1_update_clocks(struct clk_mgr *clk_mgr,
 
 	if (should_set_clock(safe_to_lower, new_clocks->phyclk_khz, clk_mgr->clks.phyclk_khz)) {
 		clk_mgr->clks.phyclk_khz = new_clocks->phyclk_khz;
-
 		send_request_to_lower = true;
 	}
 
@@ -213,24 +191,18 @@ static void dcn1_update_clocks(struct clk_mgr *clk_mgr,
 
 	if (should_set_clock(safe_to_lower, new_clocks->fclk_khz, clk_mgr->clks.fclk_khz)) {
 		clk_mgr->clks.fclk_khz = new_clocks->fclk_khz;
-		smu_req.hard_min_fclk_mhz = new_clocks->fclk_khz / 1000;
-
 		send_request_to_lower = true;
 	}
 
 	//DCF Clock
 	if (should_set_clock(safe_to_lower, new_clocks->dcfclk_khz, clk_mgr->clks.dcfclk_khz)) {
 		clk_mgr->clks.dcfclk_khz = new_clocks->dcfclk_khz;
-		smu_req.hard_min_dcefclk_mhz = new_clocks->dcfclk_khz / 1000;
-
 		send_request_to_lower = true;
 	}
 
 	if (should_set_clock(safe_to_lower,
 			new_clocks->dcfclk_deep_sleep_khz, clk_mgr->clks.dcfclk_deep_sleep_khz)) {
 		clk_mgr->clks.dcfclk_deep_sleep_khz = new_clocks->dcfclk_deep_sleep_khz;
-		smu_req.min_deep_sleep_dcefclk_mhz = (new_clocks->dcfclk_deep_sleep_khz + 999) / 1000;
-
 		send_request_to_lower = true;
 	}
 
@@ -239,17 +211,13 @@ static void dcn1_update_clocks(struct clk_mgr *clk_mgr,
 	 */
 	if (send_request_to_increase) {
 		/*use dcfclk to request voltage*/
-		if (pp_smu->set_hard_min_fclk_by_freq &&
+		if (pp_smu && pp_smu->set_hard_min_fclk_by_freq &&
 				pp_smu->set_hard_min_dcfclk_by_freq &&
 				pp_smu->set_min_deep_sleep_dcfclk) {
 
-			pp_smu->set_hard_min_fclk_by_freq(&pp_smu->pp_smu, smu_req.hard_min_fclk_mhz);
-			pp_smu->set_hard_min_dcfclk_by_freq(&pp_smu->pp_smu, smu_req.hard_min_dcefclk_mhz);
-			pp_smu->set_min_deep_sleep_dcfclk(&pp_smu->pp_smu, smu_req.min_deep_sleep_dcefclk_mhz);
-		} else {
-			if (pp_smu->set_display_requirement)
-				pp_smu->set_display_requirement(&pp_smu->pp_smu, &smu_req);
-			dcn1_pplib_apply_display_requirements(dc, context);
+			pp_smu->set_hard_min_fclk_by_freq(&pp_smu->pp_smu, new_clocks->fclk_khz / 1000);
+			pp_smu->set_hard_min_dcfclk_by_freq(&pp_smu->pp_smu, new_clocks->dcfclk_khz / 1000);
+			pp_smu->set_min_deep_sleep_dcfclk(&pp_smu->pp_smu, (new_clocks->dcfclk_deep_sleep_khz + 999) / 1000);
 		}
 	}
 
@@ -259,27 +227,20 @@ static void dcn1_update_clocks(struct clk_mgr *clk_mgr,
 			|| new_clocks->dispclk_khz == clk_mgr->clks.dispclk_khz) {
 		dcn1_ramp_up_dispclk_with_dpp(clk_mgr, new_clocks);
 		clk_mgr->clks.dispclk_khz = new_clocks->dispclk_khz;
-
 		send_request_to_lower = true;
 	}
 
 	if (!send_request_to_increase && send_request_to_lower) {
 		/*use dcfclk to request voltage*/
-		if (pp_smu->set_hard_min_fclk_by_freq &&
+		if (pp_smu && pp_smu->set_hard_min_fclk_by_freq &&
 				pp_smu->set_hard_min_dcfclk_by_freq &&
 				pp_smu->set_min_deep_sleep_dcfclk) {
 
-			pp_smu->set_hard_min_fclk_by_freq(&pp_smu->pp_smu, smu_req.hard_min_fclk_mhz);
-			pp_smu->set_hard_min_dcfclk_by_freq(&pp_smu->pp_smu, smu_req.hard_min_dcefclk_mhz);
-			pp_smu->set_min_deep_sleep_dcfclk(&pp_smu->pp_smu, smu_req.min_deep_sleep_dcefclk_mhz);
-		} else {
-			if (pp_smu->set_display_requirement)
-				pp_smu->set_display_requirement(&pp_smu->pp_smu, &smu_req);
-			dcn1_pplib_apply_display_requirements(dc, context);
+			pp_smu->set_hard_min_fclk_by_freq(&pp_smu->pp_smu, new_clocks->fclk_khz / 1000);
+			pp_smu->set_hard_min_dcfclk_by_freq(&pp_smu->pp_smu, new_clocks->dcfclk_khz / 1000);
+			pp_smu->set_min_deep_sleep_dcfclk(&pp_smu->pp_smu, (new_clocks->dcfclk_deep_sleep_khz + 999) / 1000);
 		}
 	}
-
-	*smu_req_cur = smu_req;
 }
 static const struct clk_mgr_funcs dcn1_funcs = {
 	.get_dp_ref_clk_frequency = dce12_get_dp_ref_freq_khz,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_clk_mgr.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_clk_mgr.h
index a995eda443a3..97007cf33665 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_clk_mgr.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_clk_mgr.h
@@ -34,10 +34,6 @@ struct clk_bypass {
 	uint32_t dprefclk_bypass;
 };
 
-void dcn1_pplib_apply_display_requirements(
-	struct dc *dc,
-	struct dc_state *context);
-
 struct clk_mgr *dcn1_clk_mgr_create(struct dc_context *ctx);
 
 #endif //__DCN10_CLK_MGR_H__
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.c
index cd1ebe57ed59..6f4b24756323 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.c
@@ -91,13 +91,6 @@ enum dscl_mode_sel {
 	DSCL_MODE_DSCL_BYPASS = 6
 };
 
-enum gamut_remap_select {
-	GAMUT_REMAP_BYPASS = 0,
-	GAMUT_REMAP_COEFF,
-	GAMUT_REMAP_COMA_COEFF,
-	GAMUT_REMAP_COMB_COEFF
-};
-
 void dpp_read_state(struct dpp *dpp_base,
 		struct dcn_dpp_state *s)
 {
@@ -392,6 +385,10 @@ void dpp1_cnv_setup (
 	default:
 		break;
 	}
+
+	/* Set default color space based on format if none is given. */
+	color_space = input_color_space ? input_color_space : color_space;
+
 	REG_SET(CNVC_SURFACE_PIXEL_FORMAT, 0,
 			CNVC_SURFACE_PIXEL_FORMAT, pixel_format);
 	REG_UPDATE(FORMAT_CONTROL, FORMAT_CONTROL__ALPHA_EN, alpha_en);
@@ -403,7 +400,7 @@ void dpp1_cnv_setup (
 		for (i = 0; i < 12; i++)
 			tbl_entry.regval[i] = input_csc_color_matrix.matrix[i];
 
-		tbl_entry.color_space = input_color_space;
+		tbl_entry.color_space = color_space;
 
 		if (color_space >= COLOR_SPACE_YCBCR601)
 			select = INPUT_CSC_SELECT_ICSC;
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_cm.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_cm.c
index 41f0f4c912e7..882bcc5a40f6 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_cm.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_cm.c
@@ -88,13 +88,6 @@ enum dscl_mode_sel {
 	DSCL_MODE_DSCL_BYPASS = 6
 };
 
-enum gamut_remap_select {
-	GAMUT_REMAP_BYPASS = 0,
-	GAMUT_REMAP_COEFF,
-	GAMUT_REMAP_COMA_COEFF,
-	GAMUT_REMAP_COMB_COEFF
-};
-
 static const struct dpp_input_csc_matrix dpp_input_csc_matrix[] = {
 	{COLOR_SPACE_SRGB,
 		{0x2000, 0, 0, 0, 0, 0x2000, 0, 0, 0, 0, 0x2000, 0} },
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_dscl.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_dscl.c
index c7642e748297..ce21a290bf3e 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_dscl.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_dscl.c
@@ -406,15 +406,25 @@ void dpp1_dscl_calc_lb_num_partitions(
 		int *num_part_y,
 		int *num_part_c)
 {
+	int lb_memory_size, lb_memory_size_c, lb_memory_size_a, num_partitions_a,
+	lb_bpc, memory_line_size_y, memory_line_size_c, memory_line_size_a;
+
 	int line_size = scl_data->viewport.width < scl_data->recout.width ?
 			scl_data->viewport.width : scl_data->recout.width;
 	int line_size_c = scl_data->viewport_c.width < scl_data->recout.width ?
 			scl_data->viewport_c.width : scl_data->recout.width;
-	int lb_bpc = dpp1_dscl_get_lb_depth_bpc(scl_data->lb_params.depth);
-	int memory_line_size_y = (line_size * lb_bpc + 71) / 72; /* +71 to ceil */
-	int memory_line_size_c = (line_size_c * lb_bpc + 71) / 72; /* +71 to ceil */
-	int memory_line_size_a = (line_size + 5) / 6; /* +5 to ceil */
-	int lb_memory_size, lb_memory_size_c, lb_memory_size_a, num_partitions_a;
+
+	if (line_size == 0)
+		line_size = 1;
+
+	if (line_size_c == 0)
+		line_size_c = 1;
+
+
+	lb_bpc = dpp1_dscl_get_lb_depth_bpc(scl_data->lb_params.depth);
+	memory_line_size_y = (line_size * lb_bpc + 71) / 72; /* +71 to ceil */
+	memory_line_size_c = (line_size_c * lb_bpc + 71) / 72; /* +71 to ceil */
+	memory_line_size_a = (line_size + 5) / 6; /* +5 to ceil */
 
 	if (lb_config == LB_MEMORY_CONFIG_1) {
 		lb_memory_size = 816;
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.c
index e161ad836812..0db2a6e96fc0 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.c
@@ -258,8 +258,9 @@ void hubbub1_wm_change_req_wa(struct hubbub *hubbub)
 {
 	struct dcn10_hubbub *hubbub1 = TO_DCN10_HUBBUB(hubbub);
 
-	REG_UPDATE_SEQ(DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL,
-			DCHUBBUB_ARB_WATERMARK_CHANGE_REQUEST, 0, 1);
+	REG_UPDATE_SEQ_2(DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL,
+			DCHUBBUB_ARB_WATERMARK_CHANGE_REQUEST, 0,
+			DCHUBBUB_ARB_WATERMARK_CHANGE_REQUEST, 1);
 }
 
 void hubbub1_program_watermarks(
@@ -282,7 +283,8 @@ void hubbub1_program_watermarks(
 		hubbub1->watermarks.a.urgent_ns = watermarks->a.urgent_ns;
 		prog_wm_value = convert_and_clamp(watermarks->a.urgent_ns,
 				refclk_mhz, 0x1fffff);
-		REG_WRITE(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A, prog_wm_value);
+		REG_SET(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A, 0,
+				DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A, prog_wm_value);
 
 		DC_LOG_BANDWIDTH_CALCS("URGENCY_WATERMARK_A calculated =%d\n"
 			"HW register value = 0x%x\n",
@@ -309,7 +311,8 @@ void hubbub1_program_watermarks(
 			prog_wm_value = convert_and_clamp(
 					watermarks->a.cstate_pstate.cstate_enter_plus_exit_ns,
 					refclk_mhz, 0x1fffff);
-			REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A, prog_wm_value);
+			REG_SET(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A, 0,
+					DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A, prog_wm_value);
 			DC_LOG_BANDWIDTH_CALCS("SR_ENTER_EXIT_WATERMARK_A calculated =%d\n"
 				"HW register value = 0x%x\n",
 				watermarks->a.cstate_pstate.cstate_enter_plus_exit_ns, prog_wm_value);
@@ -322,7 +325,8 @@ void hubbub1_program_watermarks(
 			prog_wm_value = convert_and_clamp(
 					watermarks->a.cstate_pstate.cstate_exit_ns,
 					refclk_mhz, 0x1fffff);
-			REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A, prog_wm_value);
+			REG_SET(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A, 0,
+					DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A, prog_wm_value);
 			DC_LOG_BANDWIDTH_CALCS("SR_EXIT_WATERMARK_A calculated =%d\n"
 				"HW register value = 0x%x\n",
 				watermarks->a.cstate_pstate.cstate_exit_ns, prog_wm_value);
@@ -336,7 +340,8 @@ void hubbub1_program_watermarks(
 		prog_wm_value = convert_and_clamp(
 				watermarks->a.cstate_pstate.pstate_change_ns,
 				refclk_mhz, 0x1fffff);
-		REG_WRITE(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_A, prog_wm_value);
+		REG_SET(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_A, 0,
+				DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_A, prog_wm_value);
 		DC_LOG_BANDWIDTH_CALCS("DRAM_CLK_CHANGE_WATERMARK_A calculated =%d\n"
 			"HW register value = 0x%x\n\n",
 			watermarks->a.cstate_pstate.pstate_change_ns, prog_wm_value);
@@ -347,7 +352,8 @@ void hubbub1_program_watermarks(
 		hubbub1->watermarks.b.urgent_ns = watermarks->b.urgent_ns;
 		prog_wm_value = convert_and_clamp(watermarks->b.urgent_ns,
 				refclk_mhz, 0x1fffff);
-		REG_WRITE(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B, prog_wm_value);
+		REG_SET(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B, 0,
+				DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B, prog_wm_value);
 
 		DC_LOG_BANDWIDTH_CALCS("URGENCY_WATERMARK_B calculated =%d\n"
 			"HW register value = 0x%x\n",
@@ -374,7 +380,8 @@ void hubbub1_program_watermarks(
 			prog_wm_value = convert_and_clamp(
 					watermarks->b.cstate_pstate.cstate_enter_plus_exit_ns,
 					refclk_mhz, 0x1fffff);
-			REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B, prog_wm_value);
+			REG_SET(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B, 0,
+					DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B, prog_wm_value);
 			DC_LOG_BANDWIDTH_CALCS("SR_ENTER_EXIT_WATERMARK_B calculated =%d\n"
 				"HW register value = 0x%x\n",
 				watermarks->b.cstate_pstate.cstate_enter_plus_exit_ns, prog_wm_value);
@@ -387,7 +394,8 @@ void hubbub1_program_watermarks(
 			prog_wm_value = convert_and_clamp(
 					watermarks->b.cstate_pstate.cstate_exit_ns,
 					refclk_mhz, 0x1fffff);
-			REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B, prog_wm_value);
+			REG_SET(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B, 0,
+					DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B, prog_wm_value);
 			DC_LOG_BANDWIDTH_CALCS("SR_EXIT_WATERMARK_B calculated =%d\n"
 				"HW register value = 0x%x\n",
 				watermarks->b.cstate_pstate.cstate_exit_ns, prog_wm_value);
@@ -401,7 +409,8 @@ void hubbub1_program_watermarks(
 		prog_wm_value = convert_and_clamp(
 				watermarks->b.cstate_pstate.pstate_change_ns,
 				refclk_mhz, 0x1fffff);
-		REG_WRITE(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_B, prog_wm_value);
+		REG_SET(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_B, 0,
+				DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_B, prog_wm_value);
 		DC_LOG_BANDWIDTH_CALCS("DRAM_CLK_CHANGE_WATERMARK_B calculated =%d\n"
 			"HW register value = 0x%x\n\n",
 			watermarks->b.cstate_pstate.pstate_change_ns, prog_wm_value);
@@ -412,7 +421,8 @@ void hubbub1_program_watermarks(
 		hubbub1->watermarks.c.urgent_ns = watermarks->c.urgent_ns;
 		prog_wm_value = convert_and_clamp(watermarks->c.urgent_ns,
 				refclk_mhz, 0x1fffff);
-		REG_WRITE(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C, prog_wm_value);
+		REG_SET(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C, 0,
+				DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C, prog_wm_value);
 
 		DC_LOG_BANDWIDTH_CALCS("URGENCY_WATERMARK_C calculated =%d\n"
 			"HW register value = 0x%x\n",
@@ -439,7 +449,8 @@ void hubbub1_program_watermarks(
 			prog_wm_value = convert_and_clamp(
 					watermarks->c.cstate_pstate.cstate_enter_plus_exit_ns,
 					refclk_mhz, 0x1fffff);
-			REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C, prog_wm_value);
+			REG_SET(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C, 0,
+					DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C, prog_wm_value);
 			DC_LOG_BANDWIDTH_CALCS("SR_ENTER_EXIT_WATERMARK_C calculated =%d\n"
 				"HW register value = 0x%x\n",
 				watermarks->c.cstate_pstate.cstate_enter_plus_exit_ns, prog_wm_value);
@@ -452,7 +463,8 @@ void hubbub1_program_watermarks(
 			prog_wm_value = convert_and_clamp(
 					watermarks->c.cstate_pstate.cstate_exit_ns,
 					refclk_mhz, 0x1fffff);
-			REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C, prog_wm_value);
+			REG_SET(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C, 0,
+					DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C, prog_wm_value);
 			DC_LOG_BANDWIDTH_CALCS("SR_EXIT_WATERMARK_C calculated =%d\n"
 				"HW register value = 0x%x\n",
 				watermarks->c.cstate_pstate.cstate_exit_ns, prog_wm_value);
@@ -466,7 +478,8 @@ void hubbub1_program_watermarks(
 		prog_wm_value = convert_and_clamp(
 				watermarks->c.cstate_pstate.pstate_change_ns,
 				refclk_mhz, 0x1fffff);
-		REG_WRITE(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_C, prog_wm_value);
+		REG_SET(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_C, 0,
+				DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_C, prog_wm_value);
 		DC_LOG_BANDWIDTH_CALCS("DRAM_CLK_CHANGE_WATERMARK_C calculated =%d\n"
 			"HW register value = 0x%x\n\n",
 			watermarks->c.cstate_pstate.pstate_change_ns, prog_wm_value);
@@ -477,7 +490,8 @@ void hubbub1_program_watermarks(
 		hubbub1->watermarks.d.urgent_ns = watermarks->d.urgent_ns;
 		prog_wm_value = convert_and_clamp(watermarks->d.urgent_ns,
 				refclk_mhz, 0x1fffff);
-		REG_WRITE(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D, prog_wm_value);
+		REG_SET(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D, 0,
+				DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D, prog_wm_value);
 
 		DC_LOG_BANDWIDTH_CALCS("URGENCY_WATERMARK_D calculated =%d\n"
 			"HW register value = 0x%x\n",
@@ -504,7 +518,8 @@ void hubbub1_program_watermarks(
 			prog_wm_value = convert_and_clamp(
 					watermarks->d.cstate_pstate.cstate_enter_plus_exit_ns,
 					refclk_mhz, 0x1fffff);
-			REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D, prog_wm_value);
+			REG_SET(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D, 0,
+					DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D, prog_wm_value);
 			DC_LOG_BANDWIDTH_CALCS("SR_ENTER_EXIT_WATERMARK_D calculated =%d\n"
 				"HW register value = 0x%x\n",
 				watermarks->d.cstate_pstate.cstate_enter_plus_exit_ns, prog_wm_value);
@@ -517,7 +532,8 @@ void hubbub1_program_watermarks(
 			prog_wm_value = convert_and_clamp(
 					watermarks->d.cstate_pstate.cstate_exit_ns,
 					refclk_mhz, 0x1fffff);
-			REG_WRITE(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D, prog_wm_value);
+			REG_SET(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D, 0,
+					DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D, prog_wm_value);
 			DC_LOG_BANDWIDTH_CALCS("SR_EXIT_WATERMARK_D calculated =%d\n"
 				"HW register value = 0x%x\n",
 				watermarks->d.cstate_pstate.cstate_exit_ns, prog_wm_value);
@@ -531,7 +547,8 @@ void hubbub1_program_watermarks(
 		prog_wm_value = convert_and_clamp(
 				watermarks->d.cstate_pstate.pstate_change_ns,
 				refclk_mhz, 0x1fffff);
-		REG_WRITE(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_D, prog_wm_value);
+		REG_SET(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_D, 0,
+				DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_D, prog_wm_value);
 		DC_LOG_BANDWIDTH_CALCS("DRAM_CLK_CHANGE_WATERMARK_D calculated =%d\n"
 			"HW register value = 0x%x\n\n",
 			watermarks->d.cstate_pstate.pstate_change_ns, prog_wm_value);
@@ -866,6 +883,7 @@ static const struct hubbub_funcs hubbub1_funcs = {
 	.dcc_support_pixel_format = hubbub1_dcc_support_pixel_format,
 	.get_dcc_compression_cap = hubbub1_get_dcc_compression_cap,
 	.wm_read_state = hubbub1_wm_read_state,
+	.program_watermarks = hubbub1_program_watermarks,
 };
 
 void hubbub1_construct(struct hubbub *hubbub,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.h
index 9cd4a5194154..85811b24a497 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubbub.h
@@ -32,18 +32,14 @@
 #define TO_DCN10_HUBBUB(hubbub)\
 	container_of(hubbub, struct dcn10_hubbub, base)
 
-#define HUBHUB_REG_LIST_DCN()\
+#define HUBBUB_REG_LIST_DCN_COMMON()\
 	SR(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A),\
-	SR(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_A),\
 	SR(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_A),\
 	SR(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B),\
-	SR(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_B),\
 	SR(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_B),\
 	SR(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C),\
-	SR(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_C),\
 	SR(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_C),\
 	SR(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D),\
-	SR(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_D),\
 	SR(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_D),\
 	SR(DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL),\
 	SR(DCHUBBUB_ARB_DRAM_STATE_CNTL),\
@@ -54,6 +50,12 @@
 	SR(DCHUBBUB_TEST_DEBUG_DATA),\
 	SR(DCHUBBUB_SOFT_RESET)
 
+#define HUBBUB_VM_REG_LIST() \
+	SR(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_A),\
+	SR(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_B),\
+	SR(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_C),\
+	SR(DCHUBBUB_ARB_PTE_META_URGENCY_WATERMARK_D)
+
 #define HUBBUB_SR_WATERMARK_REG_LIST()\
 	SR(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A),\
 	SR(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A),\
@@ -65,7 +67,8 @@
 	SR(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D)
 
 #define HUBBUB_REG_LIST_DCN10(id)\
-	HUBHUB_REG_LIST_DCN(), \
+	HUBBUB_REG_LIST_DCN_COMMON(), \
+	HUBBUB_VM_REG_LIST(), \
 	HUBBUB_SR_WATERMARK_REG_LIST(), \
 	SR(DCHUBBUB_SDPIF_FB_TOP),\
 	SR(DCHUBBUB_SDPIF_FB_BASE),\
@@ -122,8 +125,7 @@ struct dcn_hubbub_registers {
 #define HUBBUB_SF(reg_name, field_name, post_fix)\
 	.field_name = reg_name ## __ ## field_name ## post_fix
 
-
-#define HUBBUB_MASK_SH_LIST_DCN(mask_sh)\
+#define HUBBUB_MASK_SH_LIST_DCN_COMMON(mask_sh)\
 		HUBBUB_SF(DCHUBBUB_GLOBAL_TIMER_CNTL, DCHUBBUB_GLOBAL_TIMER_ENABLE, mask_sh), \
 		HUBBUB_SF(DCHUBBUB_SOFT_RESET, DCHUBBUB_GLOBAL_SOFT_RESET, mask_sh), \
 		HUBBUB_SF(DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL, DCHUBBUB_ARB_WATERMARK_CHANGE_REQUEST, mask_sh), \
@@ -133,10 +135,29 @@ struct dcn_hubbub_registers {
 		HUBBUB_SF(DCHUBBUB_ARB_DRAM_STATE_CNTL, DCHUBBUB_ARB_ALLOW_PSTATE_CHANGE_FORCE_VALUE, mask_sh), \
 		HUBBUB_SF(DCHUBBUB_ARB_DRAM_STATE_CNTL, DCHUBBUB_ARB_ALLOW_PSTATE_CHANGE_FORCE_ENABLE, mask_sh), \
 		HUBBUB_SF(DCHUBBUB_ARB_SAT_LEVEL, DCHUBBUB_ARB_SAT_LEVEL, mask_sh), \
-		HUBBUB_SF(DCHUBBUB_ARB_DF_REQ_OUTSTAND, DCHUBBUB_ARB_MIN_REQ_OUTSTAND, mask_sh)
+		HUBBUB_SF(DCHUBBUB_ARB_DF_REQ_OUTSTAND, DCHUBBUB_ARB_MIN_REQ_OUTSTAND, mask_sh), \
+		HUBBUB_SF(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A, DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A, mask_sh), \
+		HUBBUB_SF(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B, DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B, mask_sh), \
+		HUBBUB_SF(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C, DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C, mask_sh), \
+		HUBBUB_SF(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D, DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D, mask_sh), \
+		HUBBUB_SF(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_A, DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_A, mask_sh), \
+		HUBBUB_SF(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_B, DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_B, mask_sh), \
+		HUBBUB_SF(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_C, DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_C, mask_sh), \
+		HUBBUB_SF(DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_D, DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_D, mask_sh)
+
+#define HUBBUB_MASK_SH_LIST_STUTTER(mask_sh) \
+		HUBBUB_SF(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A, DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A, mask_sh), \
+		HUBBUB_SF(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B, DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B, mask_sh), \
+		HUBBUB_SF(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C, DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C, mask_sh), \
+		HUBBUB_SF(DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D, DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D, mask_sh), \
+		HUBBUB_SF(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A, DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A, mask_sh), \
+		HUBBUB_SF(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B, DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B, mask_sh), \
+		HUBBUB_SF(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C, DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C, mask_sh), \
+		HUBBUB_SF(DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D, DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D, mask_sh)
 
 #define HUBBUB_MASK_SH_LIST_DCN10(mask_sh)\
-		HUBBUB_MASK_SH_LIST_DCN(mask_sh), \
+		HUBBUB_MASK_SH_LIST_DCN_COMMON(mask_sh), \
+		HUBBUB_MASK_SH_LIST_STUTTER(mask_sh), \
 		HUBBUB_SF(DCHUBBUB_SDPIF_FB_TOP, SDPIF_FB_TOP, mask_sh), \
 		HUBBUB_SF(DCHUBBUB_SDPIF_FB_BASE, SDPIF_FB_BASE, mask_sh), \
 		HUBBUB_SF(DCHUBBUB_SDPIF_FB_OFFSET, SDPIF_FB_OFFSET, mask_sh), \
@@ -167,15 +188,35 @@ struct dcn_hubbub_registers {
 		type FB_OFFSET;\
 		type AGP_BOT;\
 		type AGP_TOP;\
-		type AGP_BASE
+		type AGP_BASE;\
+		type DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A;\
+		type DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B;\
+		type DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C;\
+		type DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D;\
+		type DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_A;\
+		type DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_B;\
+		type DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_C;\
+		type DCHUBBUB_ARB_ALLOW_DRAM_CLK_CHANGE_WATERMARK_D
+
+#define HUBBUB_STUTTER_REG_FIELD_LIST(type) \
+		type DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_A;\
+		type DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_B;\
+		type DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_C;\
+		type DCHUBBUB_ARB_ALLOW_SR_ENTER_WATERMARK_D;\
+		type DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_A;\
+		type DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_B;\
+		type DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_C;\
+		type DCHUBBUB_ARB_ALLOW_SR_EXIT_WATERMARK_D
 
 
 struct dcn_hubbub_shift {
 	DCN_HUBBUB_REG_FIELD_LIST(uint8_t);
+	HUBBUB_STUTTER_REG_FIELD_LIST(uint8_t);
 };
 
 struct dcn_hubbub_mask {
 	DCN_HUBBUB_REG_FIELD_LIST(uint32_t);
+	HUBBUB_STUTTER_REG_FIELD_LIST(uint32_t);
 };
 
 struct dc;
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.c
index 0ba68d41b9c3..54b219a710d8 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.c
@@ -1178,6 +1178,10 @@ void hubp1_vtg_sel(struct hubp *hubp, uint32_t otg_inst)
 	REG_UPDATE(DCHUBP_CNTL, HUBP_VTG_SEL, otg_inst);
 }
 
+void hubp1_init(struct hubp *hubp)
+{
+	//do nothing
+}
 static const struct hubp_funcs dcn10_hubp_funcs = {
 	.hubp_program_surface_flip_and_addr =
 			hubp1_program_surface_flip_and_addr,
@@ -1201,7 +1205,7 @@ static const struct hubp_funcs dcn10_hubp_funcs = {
 	.hubp_clear_underflow = hubp1_clear_underflow,
 	.hubp_disable_control =  hubp1_disable_control,
 	.hubp_get_underflow_status = hubp1_get_underflow_status,
-
+	.hubp_init = hubp1_init,
 };
 
 /*****************************************/
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.h
index a6d6dfe00617..99d2b7e2a578 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.h
@@ -34,6 +34,7 @@
 #define HUBP_REG_LIST_DCN(id)\
 	SRI(DCHUBP_CNTL, HUBP, id),\
 	SRI(HUBPREQ_DEBUG_DB, HUBP, id),\
+	SRI(HUBPREQ_DEBUG, HUBP, id),\
 	SRI(DCSURF_ADDR_CONFIG, HUBP, id),\
 	SRI(DCSURF_TILING_CONFIG, HUBP, id),\
 	SRI(DCSURF_SURFACE_PITCH, HUBPREQ, id),\
@@ -138,6 +139,7 @@
 #define HUBP_COMMON_REG_VARIABLE_LIST \
 	uint32_t DCHUBP_CNTL; \
 	uint32_t HUBPREQ_DEBUG_DB; \
+	uint32_t HUBPREQ_DEBUG; \
 	uint32_t DCSURF_ADDR_CONFIG; \
 	uint32_t DCSURF_TILING_CONFIG; \
 	uint32_t DCSURF_SURFACE_PITCH; \
@@ -247,7 +249,7 @@
 	.field_name = reg_name ## __ ## field_name ## post_fix
 
 /* Mask/shift struct generation macro for all ASICs (including those with reduced functionality) */
-#define HUBP_MASK_SH_LIST_DCN(mask_sh)\
+#define HUBP_MASK_SH_LIST_DCN_COMMON(mask_sh)\
 	HUBP_SF(HUBP0_DCHUBP_CNTL, HUBP_BLANK_EN, mask_sh),\
 	HUBP_SF(HUBP0_DCHUBP_CNTL, HUBP_TTU_DISABLE, mask_sh),\
 	HUBP_SF(HUBP0_DCHUBP_CNTL, HUBP_UNDERFLOW_STATUS, mask_sh),\
@@ -331,7 +333,6 @@
 	HUBP_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG, META_CHUNK_SIZE, mask_sh),\
 	HUBP_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG, MIN_META_CHUNK_SIZE, mask_sh),\
 	HUBP_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG, DPTE_GROUP_SIZE, mask_sh),\
-	HUBP_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG, MPTE_GROUP_SIZE, mask_sh),\
 	HUBP_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG, SWATH_HEIGHT, mask_sh),\
 	HUBP_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG, PTE_ROW_HEIGHT_LINEAR, mask_sh),\
 	HUBP_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG_C, CHUNK_SIZE_C, mask_sh),\
@@ -339,7 +340,6 @@
 	HUBP_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG_C, META_CHUNK_SIZE_C, mask_sh),\
 	HUBP_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG_C, MIN_META_CHUNK_SIZE_C, mask_sh),\
 	HUBP_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG_C, DPTE_GROUP_SIZE_C, mask_sh),\
-	HUBP_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG_C, MPTE_GROUP_SIZE_C, mask_sh),\
 	HUBP_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG_C, SWATH_HEIGHT_C, mask_sh),\
 	HUBP_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG_C, PTE_ROW_HEIGHT_LINEAR_C, mask_sh),\
 	HUBP_SF(HUBPREQ0_BLANK_OFFSET_0, REFCYC_H_BLANK_END, mask_sh),\
@@ -373,6 +373,11 @@
 	HUBP_SF(HUBPREQ0_DCN_SURF0_TTU_CNTL1, REFCYC_PER_REQ_DELIVERY_PRE, mask_sh),\
 	HUBP_SF(HUBP0_HUBP_CLK_CNTL, HUBP_CLOCK_ENABLE, mask_sh)
 
+#define HUBP_MASK_SH_LIST_DCN(mask_sh)\
+	HUBP_MASK_SH_LIST_DCN_COMMON(mask_sh),\
+	HUBP_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG, MPTE_GROUP_SIZE, mask_sh),\
+	HUBP_SF(HUBP0_DCHUBP_REQ_SIZE_CONFIG_C, MPTE_GROUP_SIZE_C, mask_sh)
+
 /* Mask/shift struct generation macro for ASICs with VM */
 #define HUBP_MASK_SH_LIST_DCN_VM(mask_sh)\
 	HUBP_SF(HUBPREQ0_NOM_PARAMETERS_0, DST_Y_PER_PTE_ROW_NOM_L, mask_sh),\
@@ -595,6 +600,9 @@
 	type AGP_BASE;\
 	type AGP_BOT;\
 	type AGP_TOP;\
+	type DCN_VM_SYSTEM_APERTURE_DEFAULT_SYSTEM;\
+	type DCN_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB;\
+	type DCN_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB;\
 	/* todo:  get these from GVM instead of reading registers ourselves */\
 	type PAGE_DIRECTORY_ENTRY_HI32;\
 	type PAGE_DIRECTORY_ENTRY_LO32;\
@@ -743,4 +751,6 @@ enum cursor_pitch hubp1_get_cursor_pitch(unsigned int pitch);
 void hubp1_vready_workaround(struct hubp *hubp,
 		struct _vcs_dpi_display_pipe_dest_params_st *pipe_dest);
 
+void hubp1_init(struct hubp *hubp);
+
 #endif
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c
index d1a8f1c302a9..33d311cea28c 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c
@@ -65,7 +65,7 @@ void print_microsec(struct dc_context *dc_ctx,
 	struct dc_log_buffer_ctx *log_ctx,
 	uint32_t ref_cycle)
 {
-	const uint32_t ref_clk_mhz = dc_ctx->dc->res_pool->ref_clock_inKhz / 1000;
+	const uint32_t ref_clk_mhz = dc_ctx->dc->res_pool->ref_clocks.dchub_ref_clock_inKhz / 1000;
 	static const unsigned int frac = 1000;
 	uint32_t us_x10 = (ref_cycle * frac) / ref_clk_mhz;
 
@@ -345,13 +345,13 @@ void dcn10_log_hw_state(struct dc *dc,
 
 	DTN_INFO("\nCALCULATED Clocks: dcfclk_khz:%d  dcfclk_deep_sleep_khz:%d  dispclk_khz:%d\n"
 		"dppclk_khz:%d  max_supported_dppclk_khz:%d  fclk_khz:%d  socclk_khz:%d\n\n",
-			dc->current_state->bw.dcn.clk.dcfclk_khz,
-			dc->current_state->bw.dcn.clk.dcfclk_deep_sleep_khz,
-			dc->current_state->bw.dcn.clk.dispclk_khz,
-			dc->current_state->bw.dcn.clk.dppclk_khz,
-			dc->current_state->bw.dcn.clk.max_supported_dppclk_khz,
-			dc->current_state->bw.dcn.clk.fclk_khz,
-			dc->current_state->bw.dcn.clk.socclk_khz);
+			dc->current_state->bw_ctx.bw.dcn.clk.dcfclk_khz,
+			dc->current_state->bw_ctx.bw.dcn.clk.dcfclk_deep_sleep_khz,
+			dc->current_state->bw_ctx.bw.dcn.clk.dispclk_khz,
+			dc->current_state->bw_ctx.bw.dcn.clk.dppclk_khz,
+			dc->current_state->bw_ctx.bw.dcn.clk.max_supported_dppclk_khz,
+			dc->current_state->bw_ctx.bw.dcn.clk.fclk_khz,
+			dc->current_state->bw_ctx.bw.dcn.clk.socclk_khz);
 
 	log_mpc_crc(dc, log_ctx);
 
@@ -714,7 +714,7 @@ static enum dc_status dcn10_enable_stream_timing(
 	return DC_OK;
 }
 
-static void reset_back_end_for_pipe(
+static void dcn10_reset_back_end_for_pipe(
 		struct dc *dc,
 		struct pipe_ctx *pipe_ctx,
 		struct dc_state *context)
@@ -889,22 +889,23 @@ void hwss1_plane_atomic_disconnect(struct dc *dc, struct pipe_ctx *pipe_ctx)
 		dcn10_verify_allow_pstate_change_high(dc);
 }
 
-static void plane_atomic_power_down(struct dc *dc, struct pipe_ctx *pipe_ctx)
+static void plane_atomic_power_down(struct dc *dc,
+		struct dpp *dpp,
+		struct hubp *hubp)
 {
 	struct dce_hwseq *hws = dc->hwseq;
-	struct dpp *dpp = pipe_ctx->plane_res.dpp;
 	DC_LOGGER_INIT(dc->ctx->logger);
 
 	if (REG(DC_IP_REQUEST_CNTL)) {
 		REG_SET(DC_IP_REQUEST_CNTL, 0,
 				IP_REQUEST_EN, 1);
 		dpp_pg_control(hws, dpp->inst, false);
-		hubp_pg_control(hws, pipe_ctx->plane_res.hubp->inst, false);
+		hubp_pg_control(hws, hubp->inst, false);
 		dpp->funcs->dpp_reset(dpp);
 		REG_SET(DC_IP_REQUEST_CNTL, 0,
 				IP_REQUEST_EN, 0);
 		DC_LOG_DEBUG(
-				"Power gated front end %d\n", pipe_ctx->pipe_idx);
+				"Power gated front end %d\n", hubp->inst);
 	}
 }
 
@@ -931,7 +932,9 @@ static void plane_atomic_disable(struct dc *dc, struct pipe_ctx *pipe_ctx)
 	hubp->power_gated = true;
 	dc->optimized_required = false; /* We're powering off, no need to optimize */
 
-	plane_atomic_power_down(dc, pipe_ctx);
+	plane_atomic_power_down(dc,
+			pipe_ctx->plane_res.dpp,
+			pipe_ctx->plane_res.hubp);
 
 	pipe_ctx->stream = NULL;
 	memset(&pipe_ctx->stream_res, 0, sizeof(pipe_ctx->stream_res));
@@ -976,16 +979,14 @@ static void dcn10_init_pipes(struct dc *dc, struct dc_state *context)
 		 * to non-preferred front end. If pipe_ctx->stream is not NULL,
 		 * we will use the pipe, so don't disable
 		 */
-		if (pipe_ctx->stream != NULL)
+		if (pipe_ctx->stream != NULL && can_apply_seamless_boot)
 			continue;
 
-		if (tg->funcs->is_tg_enabled(tg))
-			tg->funcs->lock(tg);
-
 		/* Blank controller using driver code instead of
 		 * command table.
 		 */
 		if (tg->funcs->is_tg_enabled(tg)) {
+			tg->funcs->lock(tg);
 			tg->funcs->set_blank(tg, true);
 			hwss_wait_for_blank_complete(tg);
 		}
@@ -1001,16 +1002,19 @@ static void dcn10_init_pipes(struct dc *dc, struct dc_state *context)
 		struct dpp *dpp = dc->res_pool->dpps[i];
 		struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
 
-		// W/A for issue with dc_post_update_surfaces_to_stream
-		hubp->power_gated = true;
-
 		/* There is assumption that pipe_ctx is not mapping irregularly
 		 * to non-preferred front end. If pipe_ctx->stream is not NULL,
 		 * we will use the pipe, so don't disable
 		 */
-		if (pipe_ctx->stream != NULL)
+		if (can_apply_seamless_boot &&
+			pipe_ctx->stream != NULL &&
+			pipe_ctx->stream_res.tg->funcs->is_tg_enabled(
+				pipe_ctx->stream_res.tg))
 			continue;
 
+		/* Disable on the current state so the new one isn't cleared. */
+		pipe_ctx = &dc->current_state->res_ctx.pipe_ctx[i];
+
 		dpp->funcs->dpp_reset(dpp);
 
 		pipe_ctx->stream_res.tg = tg;
@@ -1108,6 +1112,25 @@ static void dcn10_init_hw(struct dc *dc)
 			link->link_status.link_active = true;
 	}
 
+	/* If taking control over from VBIOS, we may want to optimize our first
+	 * mode set, so we need to skip powering down pipes until we know which
+	 * pipes we want to use.
+	 * Otherwise, if taking control is not possible, we need to power
+	 * everything down.
+	 */
+	if (dcb->funcs->is_accelerated_mode(dcb) || dc->config.power_down_display_on_boot) {
+		for (i = 0; i < dc->res_pool->pipe_count; i++) {
+			struct hubp *hubp = dc->res_pool->hubps[i];
+			struct dpp *dpp = dc->res_pool->dpps[i];
+
+			hubp->funcs->hubp_init(hubp);
+			dc->res_pool->opps[i]->mpc_tree_params.opp_id = dc->res_pool->opps[i]->inst;
+			plane_atomic_power_down(dc, dpp, hubp);
+		}
+
+		apply_DEGVIDCN10_253_wa(dc);
+	}
+
 	for (i = 0; i < dc->res_pool->audio_count; i++) {
 		struct audio *audio = dc->res_pool->audios[i];
 
@@ -1137,12 +1160,9 @@ static void dcn10_init_hw(struct dc *dc)
 	enable_power_gating_plane(dc->hwseq, true);
 
 	memset(&dc->res_pool->clk_mgr->clks, 0, sizeof(dc->res_pool->clk_mgr->clks));
-
-	if (dc->hwss.init_pipes)
-		dc->hwss.init_pipes(dc, dc->current_state);
 }
 
-static void reset_hw_ctx_wrap(
+static void dcn10_reset_hw_ctx_wrap(
 		struct dc *dc,
 		struct dc_state *context)
 {
@@ -1164,10 +1184,9 @@ static void reset_hw_ctx_wrap(
 				pipe_need_reprogram(pipe_ctx_old, pipe_ctx)) {
 			struct clock_source *old_clk = pipe_ctx_old->clock_source;
 
-			reset_back_end_for_pipe(dc, pipe_ctx_old, dc->current_state);
-			if (dc->hwss.enable_stream_gating) {
+			dcn10_reset_back_end_for_pipe(dc, pipe_ctx_old, dc->current_state);
+			if (dc->hwss.enable_stream_gating)
 				dc->hwss.enable_stream_gating(dc, pipe_ctx);
-			}
 			if (old_clk)
 				old_clk->funcs->cs_power_down(old_clk);
 		}
@@ -1836,7 +1855,7 @@ void dcn10_get_hdr_visual_confirm_color(
 
 	switch (top_pipe_ctx->plane_res.scl_data.format) {
 	case PIXEL_FORMAT_ARGB2101010:
-		if (top_pipe_ctx->stream->out_transfer_func->tf == TRANSFER_FUNCTION_UNITY) {
+		if (top_pipe_ctx->stream->out_transfer_func->tf == TRANSFER_FUNCTION_PQ) {
 			/* HDR10, ARGB2101010 - set boarder color to red */
 			color->color_r_cr = color_value;
 		}
@@ -1931,7 +1950,7 @@ static void update_dpp(struct dpp *dpp, struct dc_plane_state *plane_state)
 			plane_state->format,
 			EXPANSION_MODE_ZERO,
 			plane_state->input_csc_color_matrix,
-			COLOR_SPACE_YCBCR601_LIMITED);
+			plane_state->color_space);
 
 	//set scale and bias registers
 	build_prescale_params(&bns_params, plane_state);
@@ -2051,7 +2070,7 @@ void update_dchubp_dpp(
 	 * divided by 2
 	 */
 	if (plane_state->update_flags.bits.full_update) {
-		bool should_divided_by_2 = context->bw.dcn.clk.dppclk_khz <=
+		bool should_divided_by_2 = context->bw_ctx.bw.dcn.clk.dppclk_khz <=
 				dc->res_pool->clk_mgr->clks.dispclk_khz / 2;
 
 		dpp->funcs->dpp_dppclk_control(
@@ -2120,6 +2139,9 @@ void update_dchubp_dpp(
 	if (pipe_ctx->stream->cursor_attributes.address.quad_part != 0) {
 		dc->hwss.set_cursor_position(pipe_ctx);
 		dc->hwss.set_cursor_attribute(pipe_ctx);
+
+		if (dc->hwss.set_cursor_sdr_white_level)
+			dc->hwss.set_cursor_sdr_white_level(pipe_ctx);
 	}
 
 	if (plane_state->update_flags.bits.full_update) {
@@ -2310,6 +2332,7 @@ static void dcn10_apply_ctx_for_surface(
 	int i;
 	struct timing_generator *tg;
 	bool removed_pipe[4] = { false };
+	bool interdependent_update = false;
 	struct pipe_ctx *top_pipe_to_program =
 			find_top_pipe_for_stream(dc, context, stream);
 	DC_LOGGER_INIT(dc->ctx->logger);
@@ -2319,7 +2342,13 @@ static void dcn10_apply_ctx_for_surface(
 
 	tg = top_pipe_to_program->stream_res.tg;
 
-	dcn10_pipe_control_lock(dc, top_pipe_to_program, true);
+	interdependent_update = top_pipe_to_program->plane_state &&
+		top_pipe_to_program->plane_state->update_flags.bits.full_update;
+
+	if (interdependent_update)
+		lock_all_pipes(dc, context, true);
+	else
+		dcn10_pipe_control_lock(dc, top_pipe_to_program, true);
 
 	if (num_planes == 0) {
 		/* OTG blank before remove all front end */
@@ -2339,15 +2368,9 @@ static void dcn10_apply_ctx_for_surface(
 		 */
 		if (pipe_ctx->plane_state && !old_pipe_ctx->plane_state) {
 			if (old_pipe_ctx->stream_res.tg == tg &&
-				old_pipe_ctx->plane_res.hubp &&
-				old_pipe_ctx->plane_res.hubp->opp_id != 0xf) {
+			    old_pipe_ctx->plane_res.hubp &&
+			    old_pipe_ctx->plane_res.hubp->opp_id != 0xf)
 				dcn10_disable_plane(dc, old_pipe_ctx);
-				/*
-				 * power down fe will unlock when calling reset, need
-				 * to lock it back here. Messy, need rework.
-				 */
-				pipe_ctx->stream_res.tg->funcs->lock(pipe_ctx->stream_res.tg);
-			}
 		}
 
 		if ((!pipe_ctx->plane_state ||
@@ -2366,29 +2389,25 @@ static void dcn10_apply_ctx_for_surface(
 	if (num_planes > 0)
 		program_all_pipe_in_tree(dc, top_pipe_to_program, context);
 
-	dcn10_pipe_control_lock(dc, top_pipe_to_program, false);
-
-	if (top_pipe_to_program->plane_state &&
-			top_pipe_to_program->plane_state->update_flags.bits.full_update)
+	if (interdependent_update)
 		for (i = 0; i < dc->res_pool->pipe_count; i++) {
 			struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
-			tg = pipe_ctx->stream_res.tg;
 			/* Skip inactive pipes and ones already updated */
-			if (!pipe_ctx->stream || pipe_ctx->stream == stream
-					|| !pipe_ctx->plane_state
-					|| !tg->funcs->is_tg_enabled(tg))
+			if (!pipe_ctx->stream || pipe_ctx->stream == stream ||
+			    !pipe_ctx->plane_state || !tg->funcs->is_tg_enabled(tg))
 				continue;
 
-			tg->funcs->lock(tg);
-
 			pipe_ctx->plane_res.hubp->funcs->hubp_setup_interdependent(
 				pipe_ctx->plane_res.hubp,
 				&pipe_ctx->dlg_regs,
 				&pipe_ctx->ttu_regs);
-
-			tg->funcs->unlock(tg);
 		}
 
+	if (interdependent_update)
+		lock_all_pipes(dc, context, false);
+	else
+		dcn10_pipe_control_lock(dc, top_pipe_to_program, false);
+
 	if (num_planes == 0)
 		false_optc_underflow_wa(dc, stream, tg);
 
@@ -2420,12 +2439,14 @@ static void dcn10_prepare_bandwidth(
 		struct dc *dc,
 		struct dc_state *context)
 {
+	struct hubbub *hubbub = dc->res_pool->hubbub;
+
 	if (dc->debug.sanity_checks)
 		dcn10_verify_allow_pstate_change_high(dc);
 
 	if (!IS_FPGA_MAXIMUS_DC(dc->ctx->dce_environment)) {
 		if (context->stream_count == 0)
-			context->bw.dcn.clk.phyclk_khz = 0;
+			context->bw_ctx.bw.dcn.clk.phyclk_khz = 0;
 
 		dc->res_pool->clk_mgr->funcs->update_clocks(
 				dc->res_pool->clk_mgr,
@@ -2433,9 +2454,9 @@ static void dcn10_prepare_bandwidth(
 				false);
 	}
 
-	hubbub1_program_watermarks(dc->res_pool->hubbub,
-			&context->bw.dcn.watermarks,
-			dc->res_pool->ref_clock_inKhz / 1000,
+	hubbub->funcs->program_watermarks(hubbub,
+			&context->bw_ctx.bw.dcn.watermarks,
+			dc->res_pool->ref_clocks.dchub_ref_clock_inKhz / 1000,
 			true);
 	dcn10_stereo_hw_frame_pack_wa(dc, context);
 
@@ -2450,12 +2471,14 @@ static void dcn10_optimize_bandwidth(
 		struct dc *dc,
 		struct dc_state *context)
 {
+	struct hubbub *hubbub = dc->res_pool->hubbub;
+
 	if (dc->debug.sanity_checks)
 		dcn10_verify_allow_pstate_change_high(dc);
 
 	if (!IS_FPGA_MAXIMUS_DC(dc->ctx->dce_environment)) {
 		if (context->stream_count == 0)
-			context->bw.dcn.clk.phyclk_khz = 0;
+			context->bw_ctx.bw.dcn.clk.phyclk_khz = 0;
 
 		dc->res_pool->clk_mgr->funcs->update_clocks(
 				dc->res_pool->clk_mgr,
@@ -2463,9 +2486,9 @@ static void dcn10_optimize_bandwidth(
 				true);
 	}
 
-	hubbub1_program_watermarks(dc->res_pool->hubbub,
-			&context->bw.dcn.watermarks,
-			dc->res_pool->ref_clock_inKhz / 1000,
+	hubbub->funcs->program_watermarks(hubbub,
+			&context->bw_ctx.bw.dcn.watermarks,
+			dc->res_pool->ref_clocks.dchub_ref_clock_inKhz / 1000,
 			true);
 	dcn10_stereo_hw_frame_pack_wa(dc, context);
 
@@ -2654,7 +2677,7 @@ static void dcn10_update_pending_status(struct pipe_ctx *pipe_ctx)
 	flip_pending = pipe_ctx->plane_res.hubp->funcs->hubp_is_flip_pending(
 					pipe_ctx->plane_res.hubp);
 
-	plane_state->status.is_flip_pending = flip_pending;
+	plane_state->status.is_flip_pending = plane_state->status.is_flip_pending || flip_pending;
 
 	if (!flip_pending)
 		plane_state->status.current_address = plane_state->status.requested_address;
@@ -2685,16 +2708,22 @@ static void dcn10_set_cursor_position(struct pipe_ctx *pipe_ctx)
 	struct dpp *dpp = pipe_ctx->plane_res.dpp;
 	struct dc_cursor_mi_param param = {
 		.pixel_clk_khz = pipe_ctx->stream->timing.pix_clk_100hz / 10,
-		.ref_clk_khz = pipe_ctx->stream->ctx->dc->res_pool->ref_clock_inKhz,
+		.ref_clk_khz = pipe_ctx->stream->ctx->dc->res_pool->ref_clocks.dchub_ref_clock_inKhz,
 		.viewport = pipe_ctx->plane_res.scl_data.viewport,
 		.h_scale_ratio = pipe_ctx->plane_res.scl_data.ratios.horz,
 		.v_scale_ratio = pipe_ctx->plane_res.scl_data.ratios.vert,
 		.rotation = pipe_ctx->plane_state->rotation,
 		.mirror = pipe_ctx->plane_state->horizontal_mirror
 	};
+	uint32_t x_plane = pipe_ctx->plane_state->dst_rect.x;
+	uint32_t y_plane = pipe_ctx->plane_state->dst_rect.y;
+	uint32_t x_offset = min(x_plane, pos_cpy.x);
+	uint32_t y_offset = min(y_plane, pos_cpy.y);
 
-	pos_cpy.x_hotspot += pipe_ctx->plane_state->dst_rect.x;
-	pos_cpy.y_hotspot += pipe_ctx->plane_state->dst_rect.y;
+	pos_cpy.x -= x_offset;
+	pos_cpy.y -= y_offset;
+	pos_cpy.x_hotspot += (x_plane - x_offset);
+	pos_cpy.y_hotspot += (y_plane - y_offset);
 
 	if (pipe_ctx->plane_state->address.type
 			== PLN_ADDR_TYPE_VIDEO_PROGRESSIVE)
@@ -2789,6 +2818,33 @@ int get_vupdate_offset_from_vsync(struct pipe_ctx *pipe_ctx)
 	return vertical_line_start;
 }
 
+void lock_all_pipes(struct dc *dc,
+	struct dc_state *context,
+	bool lock)
+{
+	struct pipe_ctx *pipe_ctx;
+	struct timing_generator *tg;
+	int i;
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		pipe_ctx = &context->res_ctx.pipe_ctx[i];
+		tg = pipe_ctx->stream_res.tg;
+		/*
+		 * Only lock the top pipe's tg to prevent redundant
+		 * (un)locking. Also skip if pipe is disabled.
+		 */
+		if (pipe_ctx->top_pipe ||
+		    !pipe_ctx->stream || !pipe_ctx->plane_state ||
+		    !tg->funcs->is_tg_enabled(tg))
+			continue;
+
+		if (lock)
+			tg->funcs->lock(tg);
+		else
+			tg->funcs->unlock(tg);
+	}
+}
+
 static void calc_vupdate_position(
 		struct pipe_ctx *pipe_ctx,
 		uint32_t *start_line,
@@ -2882,6 +2938,29 @@ static void dcn10_setup_vupdate_interrupt(struct pipe_ctx *pipe_ctx)
 		tg->funcs->setup_vertical_interrupt2(tg, start_line);
 }
 
+static void dcn10_unblank_stream(struct pipe_ctx *pipe_ctx,
+		struct dc_link_settings *link_settings)
+{
+	struct encoder_unblank_param params = { { 0 } };
+	struct dc_stream_state *stream = pipe_ctx->stream;
+	struct dc_link *link = stream->link;
+
+	/* only 3 items below are used by unblank */
+	params.timing = pipe_ctx->stream->timing;
+
+	params.link_settings.link_rate = link_settings->link_rate;
+
+	if (dc_is_dp_signal(pipe_ctx->stream->signal)) {
+		if (params.timing.pixel_encoding == PIXEL_ENCODING_YCBCR420)
+			params.timing.pix_clk_100hz /= 2;
+		pipe_ctx->stream_res.stream_enc->funcs->dp_unblank(pipe_ctx->stream_res.stream_enc, &params);
+	}
+
+	if (link->local_sink && link->local_sink->sink_signal == SIGNAL_TYPE_EDP) {
+		link->dc->hwss.edp_backlight_control(link, true);
+	}
+}
+
 static const struct hw_sequencer_funcs dcn10_funcs = {
 	.program_gamut_remap = program_gamut_remap,
 	.init_hw = dcn10_init_hw,
@@ -2903,7 +2982,7 @@ static const struct hw_sequencer_funcs dcn10_funcs = {
 	.update_info_frame = dce110_update_info_frame,
 	.enable_stream = dce110_enable_stream,
 	.disable_stream = dce110_disable_stream,
-	.unblank_stream = dce110_unblank_stream,
+	.unblank_stream = dcn10_unblank_stream,
 	.blank_stream = dce110_blank_stream,
 	.enable_audio_stream = dce110_enable_audio_stream,
 	.disable_audio_stream = dce110_disable_audio_stream,
@@ -2913,7 +2992,7 @@ static const struct hw_sequencer_funcs dcn10_funcs = {
 	.pipe_control_lock = dcn10_pipe_control_lock,
 	.prepare_bandwidth = dcn10_prepare_bandwidth,
 	.optimize_bandwidth = dcn10_optimize_bandwidth,
-	.reset_hw_ctx_wrap = reset_hw_ctx_wrap,
+	.reset_hw_ctx_wrap = dcn10_reset_hw_ctx_wrap,
 	.enable_stream_timing = dcn10_enable_stream_timing,
 	.set_drr = set_drr,
 	.get_position = get_position,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h
index 6d66084df55f..4b3b27a5d23b 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h
@@ -83,4 +83,8 @@ struct pipe_ctx *find_top_pipe_for_stream(
 
 int get_vupdate_offset_from_vsync(struct pipe_ctx *pipe_ctx);
 
+void lock_all_pipes(struct dc *dc,
+	struct dc_state *context,
+	bool lock);
+
 #endif /* __DC_HWSS_DCN10_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer_debug.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer_debug.c
index 98f41d250978..991622da9ed5 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer_debug.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer_debug.c
@@ -77,7 +77,7 @@ static unsigned int dcn10_get_hubbub_state(struct dc *dc, char *pBuf, unsigned i
 	unsigned int chars_printed = 0;
 	unsigned int remaining_buffer = bufSize;
 
-	const uint32_t ref_clk_mhz = dc_ctx->dc->res_pool->ref_clock_inKhz / 1000;
+	const uint32_t ref_clk_mhz = dc_ctx->dc->res_pool->ref_clocks.dchub_ref_clock_inKhz / 1000;
 	static const unsigned int frac = 1000;
 
 	memset(&wm, 0, sizeof(struct dcn_hubbub_wm));
@@ -115,7 +115,7 @@ static unsigned int dcn10_get_hubp_states(struct dc *dc, char *pBuf, unsigned in
 	unsigned int chars_printed = 0;
 	unsigned int remaining_buffer = bufSize;
 
-	const uint32_t ref_clk_mhz = dc_ctx->dc->res_pool->ref_clock_inKhz / 1000;
+	const uint32_t ref_clk_mhz = dc_ctx->dc->res_pool->ref_clocks.dchub_ref_clock_inKhz / 1000;
 	static const unsigned int frac = 1000;
 
 	if (invarOnly)
@@ -472,12 +472,12 @@ static unsigned int dcn10_get_clock_states(struct dc *dc, char *pBuf, unsigned i
 	chars_printed = snprintf_count(pBuf, bufSize, "dcfclk,dcfclk_deep_sleep,dispclk,"
 		"dppclk,fclk,socclk\n"
 		"%d,%d,%d,%d,%d,%d\n",
-		dc->current_state->bw.dcn.clk.dcfclk_khz,
-		dc->current_state->bw.dcn.clk.dcfclk_deep_sleep_khz,
-		dc->current_state->bw.dcn.clk.dispclk_khz,
-		dc->current_state->bw.dcn.clk.dppclk_khz,
-		dc->current_state->bw.dcn.clk.fclk_khz,
-		dc->current_state->bw.dcn.clk.socclk_khz);
+		dc->current_state->bw_ctx.bw.dcn.clk.dcfclk_khz,
+		dc->current_state->bw_ctx.bw.dcn.clk.dcfclk_deep_sleep_khz,
+		dc->current_state->bw_ctx.bw.dcn.clk.dispclk_khz,
+		dc->current_state->bw_ctx.bw.dcn.clk.dppclk_khz,
+		dc->current_state->bw_ctx.bw.dcn.clk.fclk_khz,
+		dc->current_state->bw_ctx.bw.dcn.clk.socclk_khz);
 
 	remaining_buffer -= chars_printed;
 	pBuf += chars_printed;
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.c
index a9db372688ff..0126a44ba012 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.c
@@ -1304,7 +1304,6 @@ void dcn10_link_encoder_connect_dig_be_to_fe(
 #define HPD_REG_UPDATE_N(reg_name, n, ...)	\
 		generic_reg_update_ex(CTX, \
 				HPD_REG(reg_name), \
-				HPD_REG_READ(reg_name), \
 				n, __VA_ARGS__)
 
 #define HPD_REG_UPDATE(reg_name, field, val)	\
@@ -1337,7 +1336,6 @@ void dcn10_link_encoder_disable_hpd(struct link_encoder *enc)
 #define AUX_REG_UPDATE_N(reg_name, n, ...)	\
 		generic_reg_update_ex(CTX, \
 				AUX_REG(reg_name), \
-				AUX_REG_READ(reg_name), \
 				n, __VA_ARGS__)
 
 #define AUX_REG_UPDATE(reg_name, field, val)	\
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c
index 09d74070a49b..7eccb54c421d 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c
@@ -516,6 +516,31 @@ static const struct resource_caps rv2_res_cap = {
 };
 #endif
 
+static const struct dc_plane_cap plane_cap = {
+	.type = DC_PLANE_TYPE_DCN_UNIVERSAL,
+	.blends_with_above = true,
+	.blends_with_below = true,
+	.per_pixel_alpha = true,
+
+	.pixel_format_support = {
+			.argb8888 = true,
+			.nv12 = true,
+			.fp16 = true
+	},
+
+	.max_upscale_factor = {
+			.argb8888 = 16000,
+			.nv12 = 16000,
+			.fp16 = 1
+	},
+
+	.max_downscale_factor = {
+			.argb8888 = 250,
+			.nv12 = 250,
+			.fp16 = 1
+	}
+};
+
 static const struct dc_debug_options debug_defaults_drv = {
 		.sanity_checks = true,
 		.disable_dmcu = true,
@@ -848,14 +873,14 @@ void dcn10_clock_source_destroy(struct clock_source **clk_src)
 	*clk_src = NULL;
 }
 
-static struct pp_smu_funcs_rv *dcn10_pp_smu_create(struct dc_context *ctx)
+static struct pp_smu_funcs *dcn10_pp_smu_create(struct dc_context *ctx)
 {
-	struct pp_smu_funcs_rv *pp_smu = kzalloc(sizeof(*pp_smu), GFP_KERNEL);
+	struct pp_smu_funcs *pp_smu = kzalloc(sizeof(*pp_smu), GFP_KERNEL);
 
 	if (!pp_smu)
 		return pp_smu;
 
-	dm_pp_get_funcs_rv(ctx, pp_smu);
+	dm_pp_get_funcs(ctx, pp_smu);
 	return pp_smu;
 }
 
@@ -865,10 +890,7 @@ static void destruct(struct dcn10_resource_pool *pool)
 
 	for (i = 0; i < pool->base.stream_enc_count; i++) {
 		if (pool->base.stream_enc[i] != NULL) {
-			/* TODO: free dcn version of stream encoder once implemented
-			 * rather than using virtual stream encoder
-			 */
-			kfree(pool->base.stream_enc[i]);
+			kfree(DCN10STRENC_FROM_STRENC(pool->base.stream_enc[i]));
 			pool->base.stream_enc[i] = NULL;
 		}
 	}
@@ -921,9 +943,6 @@ static void destruct(struct dcn10_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.stream_enc_count; i++)
-		kfree(pool->base.stream_enc[i]);
-
 	for (i = 0; i < pool->base.audio_count; i++) {
 		if (pool->base.audios[i])
 			dce_aud_destroy(&pool->base.audios[i]);
@@ -1078,7 +1097,7 @@ static struct pipe_ctx *dcn10_acquire_idle_pipe_for_layer(
 {
 	struct resource_context *res_ctx = &context->res_ctx;
 	struct pipe_ctx *head_pipe = resource_get_head_pipe_for_stream(res_ctx, stream);
-	struct pipe_ctx *idle_pipe = find_idle_secondary_pipe(res_ctx, pool);
+	struct pipe_ctx *idle_pipe = find_idle_secondary_pipe(res_ctx, pool, head_pipe);
 
 	if (!head_pipe) {
 		ASSERT(0);
@@ -1143,7 +1162,7 @@ static enum dc_status dcn10_validate_global(struct dc *dc, struct dc_state *cont
 			continue;
 
 		if (context->stream_status[i].plane_count > 2)
-			return false;
+			return DC_FAIL_UNSUPPORTED_1;
 
 		for (j = 0; j < context->stream_status[i].plane_count; j++) {
 			struct dc_plane_state *plane =
@@ -1351,7 +1370,7 @@ static bool construct(
 		goto fail;
 	}
 
-	dml_init_instance(&dc->dml, DML_PROJECT_RAVEN1);
+	dml_init_instance(&dc->dml, &dcn1_0_soc, &dcn1_0_ip, DML_PROJECT_RAVEN1);
 	memcpy(dc->dcn_ip, &dcn10_ip_defaults, sizeof(dcn10_ip_defaults));
 	memcpy(dc->dcn_soc, &dcn10_soc_defaults, sizeof(dcn10_soc_defaults));
 
@@ -1510,6 +1529,9 @@ static bool construct(
 	dcn10_hw_sequencer_construct(dc);
 	dc->caps.max_planes =  pool->base.pipe_count;
 
+	for (i = 0; i < dc->caps.max_planes; ++i)
+		dc->caps.planes[i] = plane_cap;
+
 	dc->cap_funcs = cap_funcs;
 
 	return true;
@@ -1522,7 +1544,7 @@ fail:
 }
 
 struct resource_pool *dcn10_create_resource_pool(
-		uint8_t num_virtual_links,
+		const struct dc_init_data *init_data,
 		struct dc *dc)
 {
 	struct dcn10_resource_pool *pool =
@@ -1531,7 +1553,7 @@ struct resource_pool *dcn10_create_resource_pool(
 	if (!pool)
 		return NULL;
 
-	if (construct(num_virtual_links, dc, pool))
+	if (construct(init_data->num_virtual_links, dc, pool))
 		return &pool->base;
 
 	BREAK_TO_DEBUGGER();
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.h
index 8f71225bc61b..999c684a0b36 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.h
@@ -39,7 +39,7 @@ struct dcn10_resource_pool {
 	struct resource_pool base;
 };
 struct resource_pool *dcn10_create_resource_pool(
-		uint8_t num_virtual_links,
+		const struct dc_init_data *init_data,
 		struct dc *dc);
 
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.c
index b08254121251..8ee9f6dc1d62 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.c
@@ -245,7 +245,8 @@ static void enc1_update_hdmi_info_packet(
 void enc1_stream_encoder_dp_set_stream_attribute(
 	struct stream_encoder *enc,
 	struct dc_crtc_timing *crtc_timing,
-	enum dc_color_space output_color_space)
+	enum dc_color_space output_color_space,
+	uint32_t enable_sdp_splitting)
 {
 	uint32_t h_active_start;
 	uint32_t v_active_start;
@@ -298,7 +299,6 @@ void enc1_stream_encoder_dp_set_stream_attribute(
 		break;
 	case PIXEL_ENCODING_YCBCR420:
 		dp_pixel_encoding = DP_PIXEL_ENCODING_TYPE_YCBCR420;
-		REG_UPDATE(DP_VID_TIMING, DP_VID_N_MUL, 1);
 		break;
 	default:
 		dp_pixel_encoding = DP_PIXEL_ENCODING_TYPE_RGB444;
@@ -726,13 +726,19 @@ void enc1_stream_encoder_update_dp_info_packets(
 				3,  /* packetIndex */
 				&info_frame->hdrsmd);
 
+	if (info_frame->dpsdp.valid)
+		enc1_update_generic_info_packet(
+				enc1,
+				4,/* packetIndex */
+				&info_frame->dpsdp);
+
 	/* enable/disable transmission of packet(s).
 	 * If enabled, packet transmission begins on the next frame
 	 */
 	REG_UPDATE(DP_SEC_CNTL, DP_SEC_GSP0_ENABLE, info_frame->vsc.valid);
 	REG_UPDATE(DP_SEC_CNTL, DP_SEC_GSP2_ENABLE, info_frame->spd.valid);
 	REG_UPDATE(DP_SEC_CNTL, DP_SEC_GSP3_ENABLE, info_frame->hdrsmd.valid);
-
+	REG_UPDATE(DP_SEC_CNTL, DP_SEC_GSP4_ENABLE, info_frame->dpsdp.valid);
 
 	/* This bit is the master enable bit.
 	 * When enabling secondary stream engine,
@@ -797,10 +803,10 @@ void enc1_stream_encoder_dp_blank(
 	 */
 	REG_UPDATE(DP_VID_STREAM_CNTL, DP_VID_STREAM_DIS_DEFER, 2);
 	/* Larger delay to wait until VBLANK - use max retry of
-	 * 10us*3000=30ms. This covers 16.6ms of typical 60 Hz mode +
+	 * 10us*5000=50ms. This covers 41.7ms of minimum 24 Hz mode +
 	 * a little more because we may not trust delay accuracy.
 	 */
-	max_retries = DP_BLANK_MAX_RETRY * 150;
+	max_retries = DP_BLANK_MAX_RETRY * 250;
 
 	/* disable DP stream */
 	REG_UPDATE(DP_VID_STREAM_CNTL, DP_VID_STREAM_ENABLE, 0);
@@ -833,14 +839,19 @@ void enc1_stream_encoder_dp_unblank(
 	if (param->link_settings.link_rate != LINK_RATE_UNKNOWN) {
 		uint32_t n_vid = 0x8000;
 		uint32_t m_vid;
+		uint32_t n_multiply = 0;
+		uint64_t m_vid_l = n_vid;
 
+		/* YCbCr 4:2:0 : Computed VID_M will be 2X the input rate */
+		if (param->timing.pixel_encoding == PIXEL_ENCODING_YCBCR420) {
+			/*this param->pixel_clk_khz is half of 444 rate for 420 already*/
+			n_multiply = 1;
+		}
 		/* M / N = Fstream / Flink
 		 * m_vid / n_vid = pixel rate / link rate
 		 */
 
-		uint64_t m_vid_l = n_vid;
-
-		m_vid_l *= param->pixel_clk_khz;
+		m_vid_l *= param->timing.pix_clk_100hz / 10;
 		m_vid_l = div_u64(m_vid_l,
 			param->link_settings.link_rate
 				* LINK_RATE_REF_FREQ_IN_KHZ);
@@ -859,7 +870,9 @@ void enc1_stream_encoder_dp_unblank(
 
 		REG_UPDATE(DP_VID_M, DP_VID_M, m_vid);
 
-		REG_UPDATE(DP_VID_TIMING, DP_VID_M_N_GEN_EN, 1);
+		REG_UPDATE_2(DP_VID_TIMING,
+				DP_VID_M_N_GEN_EN, 1,
+				DP_VID_N_MUL, n_multiply);
 	}
 
 	/* set DIG_START to 0x1 to resync FIFO */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.h
index b7c800e10a32..e654c2f55971 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_stream_encoder.h
@@ -462,7 +462,8 @@ void enc1_update_generic_info_packet(
 void enc1_stream_encoder_dp_set_stream_attribute(
 	struct stream_encoder *enc,
 	struct dc_crtc_timing *crtc_timing,
-	enum dc_color_space output_color_space);
+	enum dc_color_space output_color_space,
+	uint32_t enable_sdp_splitting);
 
 void enc1_stream_encoder_hdmi_set_stream_attribute(
 	struct stream_encoder *enc,
diff --git a/drivers/gpu/drm/amd/display/dc/dm_helpers.h b/drivers/gpu/drm/amd/display/dc/dm_helpers.h
index e81b24374bcb..ccbfe9680d27 100644
--- a/drivers/gpu/drm/amd/display/dc/dm_helpers.h
+++ b/drivers/gpu/drm/amd/display/dc/dm_helpers.h
@@ -58,7 +58,7 @@ bool dm_helpers_dp_mst_write_payload_allocation_table(
 		bool enable);
 
 /*
- * poll pending down reply before clear payload allocation table
+ * poll pending down reply
  */
 void dm_helpers_dp_mst_poll_pending_down_reply(
 	struct dc_context *ctx,
diff --git a/drivers/gpu/drm/amd/display/dc/dm_pp_smu.h b/drivers/gpu/drm/amd/display/dc/dm_pp_smu.h
index 14bed5b1fa97..4fc4208d1472 100644
--- a/drivers/gpu/drm/amd/display/dc/dm_pp_smu.h
+++ b/drivers/gpu/drm/amd/display/dc/dm_pp_smu.h
@@ -30,6 +30,8 @@
  * interface to PPLIB/SMU to setup clocks and pstate requirements on SoC
  */
 
+typedef bool BOOLEAN;
+
 enum pp_smu_ver {
 	/*
 	 * PP_SMU_INTERFACE_X should be interpreted as the interface defined
@@ -72,29 +74,6 @@ struct pp_smu_wm_range_sets {
 	struct pp_smu_wm_set_range writer_wm_sets[MAX_WATERMARK_SETS];
 };
 
-struct pp_smu_display_requirement_rv {
-	/* PPSMC_MSG_SetDisplayCount: count
-	 *  0 triggers S0i2 optimization
-	 */
-	unsigned int display_count;
-
-	/* PPSMC_MSG_SetHardMinFclkByFreq: mhz
-	 *  FCLK will vary with DPM, but never below requested hard min
-	 */
-	unsigned int hard_min_fclk_mhz;
-
-	/* PPSMC_MSG_SetHardMinDcefclkByFreq: mhz
-	 *  fixed clock at requested freq, either from FCH bypass or DFS
-	 */
-	unsigned int hard_min_dcefclk_mhz;
-
-	/* PPSMC_MSG_SetMinDeepSleepDcefclk: mhz
-	 *  when DF is in cstate, dcf clock is further divided down
-	 *  to just above given frequency
-	 */
-	unsigned int min_deep_sleep_dcefclk_mhz;
-};
-
 struct pp_smu_funcs_rv {
 	struct pp_smu pp_smu;
 
@@ -137,12 +116,6 @@ struct pp_smu_funcs_rv {
 	/* PME w/a */
 	void (*set_pme_wa_enable)(struct pp_smu *pp);
 
-	/*
-	 * Legacy functions.  Used for backwards comp. with existing
-	 * PPlib code.
-	 */
-	void (*set_display_requirement)(struct pp_smu *pp,
-			struct pp_smu_display_requirement_rv *req);
 };
 
 struct pp_smu_funcs {
diff --git a/drivers/gpu/drm/amd/display/dc/dm_services.h b/drivers/gpu/drm/amd/display/dc/dm_services.h
index 1961cc6d9143..b426ba02b793 100644
--- a/drivers/gpu/drm/amd/display/dc/dm_services.h
+++ b/drivers/gpu/drm/amd/display/dc/dm_services.h
@@ -52,30 +52,17 @@ irq_handler_idx dm_register_interrupt(
  * GPU registers access
  *
  */
-
+uint32_t dm_read_reg_func(
+	const struct dc_context *ctx,
+	uint32_t address,
+	const char *func_name);
 /* enable for debugging new code, this adds 50k to the driver size. */
 /* #define DM_CHECK_ADDR_0 */
 
 #define dm_read_reg(ctx, address)	\
 		dm_read_reg_func(ctx, address, __func__)
 
-static inline uint32_t dm_read_reg_func(
-	const struct dc_context *ctx,
-	uint32_t address,
-	const char *func_name)
-{
-	uint32_t value;
-#ifdef DM_CHECK_ADDR_0
-	if (address == 0) {
-		DC_ERR("invalid register read; address = 0\n");
-		return 0;
-	}
-#endif
-	value = cgs_read_register(ctx->cgs_device, address);
-	trace_amdgpu_dc_rreg(&ctx->perf_trace->read_count, address, value);
 
-	return value;
-}
 
 #define dm_write_reg(ctx, address, value)	\
 	dm_write_reg_func(ctx, address, value, __func__)
@@ -144,10 +131,14 @@ static inline uint32_t set_reg_field_value_ex(
 		reg_name ## __ ## reg_field ## _MASK,\
 		reg_name ## __ ## reg_field ## __SHIFT)
 
-uint32_t generic_reg_update_ex(const struct dc_context *ctx,
+uint32_t generic_reg_set_ex(const struct dc_context *ctx,
 		uint32_t addr, uint32_t reg_val, int n,
 		uint8_t shift1, uint32_t mask1, uint32_t field_value1, ...);
 
+uint32_t generic_reg_update_ex(const struct dc_context *ctx,
+		uint32_t addr, int n,
+		uint8_t shift1, uint32_t mask1, uint32_t field_value1, ...);
+
 #define FD(reg_field)	reg_field ## __SHIFT, \
 						reg_field ## _MASK
 
@@ -155,7 +146,7 @@ uint32_t generic_reg_update_ex(const struct dc_context *ctx,
  * return number of poll before condition is met
  * return 0 if condition is not meet after specified time out tries
  */
-unsigned int generic_reg_wait(const struct dc_context *ctx,
+void generic_reg_wait(const struct dc_context *ctx,
 	uint32_t addr, uint32_t mask, uint32_t shift, uint32_t condition_value,
 	unsigned int delay_between_poll_us, unsigned int time_out_num_tries,
 	const char *func_name, int line);
@@ -172,11 +163,10 @@ unsigned int generic_reg_wait(const struct dc_context *ctx,
 
 #define generic_reg_update_soc15(ctx, inst_offset, reg_name, n, ...)\
 		generic_reg_update_ex(ctx, DCE_BASE.instance[0].segment[mm##reg_name##_BASE_IDX] +  mm##reg_name + inst_offset, \
-		dm_read_reg_func(ctx, mm##reg_name + DCE_BASE.instance[0].segment[mm##reg_name##_BASE_IDX] + inst_offset, __func__), \
 		n, __VA_ARGS__)
 
 #define generic_reg_set_soc15(ctx, inst_offset, reg_name, n, ...)\
-		generic_reg_update_ex(ctx, DCE_BASE.instance[0].segment[mm##reg_name##_BASE_IDX] + mm##reg_name + inst_offset, 0, \
+		generic_reg_set_ex(ctx, DCE_BASE.instance[0].segment[mm##reg_name##_BASE_IDX] + mm##reg_name + inst_offset, 0, \
 		n, __VA_ARGS__)
 
 #define get_reg_field_value_soc15(reg_value, block, reg_num, reg_name, reg_field)\
@@ -223,8 +213,8 @@ bool dm_pp_notify_wm_clock_changes(
 	const struct dc_context *ctx,
 	struct dm_pp_wm_sets_with_clock_ranges *wm_with_clock_ranges);
 
-void dm_pp_get_funcs_rv(struct dc_context *ctx,
-		struct pp_smu_funcs_rv *funcs);
+void dm_pp_get_funcs(struct dc_context *ctx,
+		struct pp_smu_funcs *funcs);
 
 /* DAL calls this function to notify PP about completion of Mode Set.
  * For PP it means that current DCE clocks are those which were returned
diff --git a/drivers/gpu/drm/amd/display/dc/dm_services_types.h b/drivers/gpu/drm/amd/display/dc/dm_services_types.h
index 77200711abbe..a3d1be20dd9d 100644
--- a/drivers/gpu/drm/amd/display/dc/dm_services_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dm_services_types.h
@@ -29,7 +29,7 @@
 #include "os_types.h"
 #include "dc_types.h"
 
-struct pp_smu_funcs_rv;
+struct pp_smu_funcs;
 
 struct dm_pp_clock_range {
 	int min_khz;
diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.c b/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.c
index d303b789adfe..80ffd7d958b2 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.c
@@ -26,40 +26,14 @@
 #include "display_mode_lib.h"
 #include "dc_features.h"
 
-extern const struct _vcs_dpi_ip_params_st dcn1_0_ip;
-extern const struct _vcs_dpi_soc_bounding_box_st dcn1_0_soc;
-
-static void set_soc_bounding_box(struct _vcs_dpi_soc_bounding_box_st *soc, enum dml_project project)
-{
-	switch (project) {
-	case DML_PROJECT_RAVEN1:
-		*soc = dcn1_0_soc;
-		break;
-	default:
-		ASSERT(0);
-		break;
-	}
-}
-
-static void set_ip_params(struct _vcs_dpi_ip_params_st *ip, enum dml_project project)
+void dml_init_instance(struct display_mode_lib *lib,
+		const struct _vcs_dpi_soc_bounding_box_st *soc_bb,
+		const struct _vcs_dpi_ip_params_st *ip_params,
+		enum dml_project project)
 {
-	switch (project) {
-	case DML_PROJECT_RAVEN1:
-		*ip = dcn1_0_ip;
-		break;
-	default:
-		ASSERT(0);
-		break;
-	}
-}
-
-void dml_init_instance(struct display_mode_lib *lib, enum dml_project project)
-{
-	if (lib->project != project) {
-		set_soc_bounding_box(&lib->soc, project);
-		set_ip_params(&lib->ip, project);
-		lib->project = project;
-	}
+	lib->soc = *soc_bb;
+	lib->ip = *ip_params;
+	lib->project = project;
 }
 
 const char *dml_get_status_message(enum dm_validation_status status)
diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.h b/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.h
index a730e0209c05..1b546dba34bd 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.h
+++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_lib.h
@@ -41,7 +41,10 @@ struct display_mode_lib {
 	struct dal_logger *logger;
 };
 
-void dml_init_instance(struct display_mode_lib *lib, enum dml_project project);
+void dml_init_instance(struct display_mode_lib *lib,
+		const struct _vcs_dpi_soc_bounding_box_st *soc_bb,
+		const struct _vcs_dpi_ip_params_st *ip_params,
+		enum dml_project project);
 
 const char *dml_get_status_message(enum dm_validation_status status);
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h b/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h
index 391183e3428f..c5b791d158a7 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h
+++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h
@@ -25,6 +25,8 @@
 #ifndef __DISPLAY_MODE_STRUCTS_H__
 #define __DISPLAY_MODE_STRUCTS_H__
 
+#define MAX_CLOCK_LIMIT_STATES 8
+
 typedef struct _vcs_dpi_voltage_scaling_st voltage_scaling_st;
 typedef struct _vcs_dpi_soc_bounding_box_st soc_bounding_box_st;
 typedef struct _vcs_dpi_ip_params_st ip_params_st;
@@ -103,7 +105,7 @@ struct _vcs_dpi_soc_bounding_box_st {
 	double xfc_xbuf_latency_tolerance_us;
 	int use_urgent_burst_bw;
 	unsigned int num_states;
-	struct _vcs_dpi_voltage_scaling_st clock_limits[8];
+	struct _vcs_dpi_voltage_scaling_st clock_limits[MAX_CLOCK_LIMIT_STATES];
 };
 
 struct _vcs_dpi_ip_params_st {
@@ -416,6 +418,7 @@ struct _vcs_dpi_display_dlg_regs_st {
 	unsigned int refcyc_per_vm_group_flip;
 	unsigned int refcyc_per_vm_req_vblank;
 	unsigned int refcyc_per_vm_req_flip;
+	unsigned int refcyc_per_vm_dmdata;
 };
 
 struct _vcs_dpi_display_ttu_regs_st {
diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_rq_dlg_helpers.c b/drivers/gpu/drm/amd/display/dc/dml/display_rq_dlg_helpers.c
index 48400d642610..e2d82aacd3bc 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/display_rq_dlg_helpers.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/display_rq_dlg_helpers.c
@@ -321,6 +321,9 @@ void print__dlg_regs_st(struct display_mode_lib *mode_lib, display_dlg_regs_st d
 	dml_print(
 			"DML_RQ_DLG_CALC:    xfc_reg_remote_surface_flip_latency = 0x%0x\n",
 			dlg_regs.xfc_reg_remote_surface_flip_latency);
+	dml_print(
+			"DML_RQ_DLG_CALC:    refcyc_per_vm_dmdata            = 0x%0x\n",
+			dlg_regs.refcyc_per_vm_dmdata);
 
 	dml_print("DML_RQ_DLG_CALC: =====================================\n");
 }
diff --git a/drivers/gpu/drm/amd/display/dc/inc/clock_source.h b/drivers/gpu/drm/amd/display/dc/inc/clock_source.h
index fe6301cb8681..1b01a9a58d14 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/clock_source.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/clock_source.h
@@ -167,7 +167,7 @@ struct clock_source_funcs {
 			struct pixel_clk_params *,
 			struct pll_settings *);
 	bool (*get_pixel_clk_frequency_100hz)(
-			struct clock_source *clock_source,
+			const struct clock_source *clock_source,
 			unsigned int inst,
 			unsigned int *pixel_clk_khz);
 };
diff --git a/drivers/gpu/drm/amd/display/dc/inc/core_status.h b/drivers/gpu/drm/amd/display/dc/inc/core_status.h
index 2e61a22ef4b2..8dca3b7700e5 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/core_status.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/core_status.h
@@ -43,7 +43,7 @@ enum dc_status {
 	DC_FAIL_BANDWIDTH_VALIDATE = 13, /* BW and Watermark validation */
 	DC_FAIL_SCALING = 14,
 	DC_FAIL_DP_LINK_TRAINING = 15,
-
+	DC_FAIL_UNSUPPORTED_1 = 18,
 	DC_ERROR_UNEXPECTED = -1
 };
 
diff --git a/drivers/gpu/drm/amd/display/dc/inc/core_types.h b/drivers/gpu/drm/amd/display/dc/inc/core_types.h
index 986ed1728644..6f5ab05d6467 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/core_types.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/core_types.h
@@ -95,10 +95,10 @@ struct resource_funcs {
 	void (*link_init)(struct dc_link *link);
 	struct link_encoder *(*link_enc_create)(
 			const struct encoder_init_data *init);
-
 	bool (*validate_bandwidth)(
 					struct dc *dc,
-					struct dc_state *context);
+					struct dc_state *context,
+					bool fast_validate);
 
 	enum dc_status (*validate_global)(
 		struct dc *dc,
@@ -144,8 +144,7 @@ struct resource_pool {
 	struct stream_encoder *stream_enc[MAX_PIPES * 2];
 	struct hubbub *hubbub;
 	struct mpc *mpc;
-	struct pp_smu_funcs_rv *pp_smu;
-	struct pp_smu_display_requirement_rv pp_smu_req;
+	struct pp_smu_funcs *pp_smu;
 	struct dce_aux *engines[MAX_PIPES];
 	struct dce_i2c_hw *hw_i2cs[MAX_PIPES];
 	struct dce_i2c_sw *sw_i2cs[MAX_PIPES];
@@ -154,7 +153,12 @@ struct resource_pool {
 	unsigned int pipe_count;
 	unsigned int underlay_pipe_index;
 	unsigned int stream_enc_count;
-	unsigned int ref_clock_inKhz;
+
+	struct {
+		unsigned int xtalin_clock_inKhz;
+		unsigned int dccg_ref_clock_inKhz;
+		unsigned int dchub_ref_clock_inKhz;
+	} ref_clocks;
 	unsigned int timing_generator_count;
 
 	/*
@@ -262,18 +266,22 @@ struct dcn_bw_output {
 	struct dcn_watermark_set watermarks;
 };
 
-union bw_context {
+union bw_output {
 	struct dcn_bw_output dcn;
 	struct dce_bw_output dce;
 };
 
+struct bw_context {
+	union bw_output bw;
+	struct display_mode_lib dml;
+};
 /**
  * struct dc_state - The full description of a state requested by a user
  *
  * @streams: Stream properties
  * @stream_status: The planes on a given stream
  * @res_ctx: Persistent state of resources
- * @bw: The output from bandwidth and watermark calculations
+ * @bw_ctx: The output from bandwidth and watermark calculations and the DML
  * @pp_display_cfg: PowerPlay clocks and settings
  * @dcn_bw_vars: non-stack memory to support bandwidth calculations
  *
@@ -285,7 +293,7 @@ struct dc_state {
 
 	struct resource_context res_ctx;
 
-	union bw_context bw;
+	struct bw_context bw_ctx;
 
 	/* Note: these are big structures, do *not* put on stack! */
 	struct dm_pp_display_configuration pp_display_cfg;
@@ -293,7 +301,11 @@ struct dc_state {
 	struct dcn_bw_internal_vars dcn_bw_vars;
 #endif
 
-	struct clk_mgr *dccg;
+	struct clk_mgr *clk_mgr;
+
+	struct {
+		bool full_update_needed : 1;
+	} commit_hints;
 
 	struct kref refcount;
 };
diff --git a/drivers/gpu/drm/amd/display/dc/inc/dc_link_ddc.h b/drivers/gpu/drm/amd/display/dc/inc/dc_link_ddc.h
index 16fd4dc6c4dd..b1fab251c09b 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/dc_link_ddc.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/dc_link_ddc.h
@@ -95,8 +95,9 @@ bool dal_ddc_service_query_ddc_data(
 		uint8_t *read_buf,
 		uint32_t read_size);
 
-int dc_link_aux_transfer(struct ddc_service *ddc,
-		struct aux_payload *payload);
+int dc_link_aux_transfer_raw(struct ddc_service *ddc,
+		struct aux_payload *payload,
+		enum aux_channel_operation_result *operation_result);
 
 bool dc_link_aux_transfer_with_retries(struct ddc_service *ddc,
 		struct aux_payload *payload);
diff --git a/drivers/gpu/drm/amd/display/dc/inc/dcn_calcs.h b/drivers/gpu/drm/amd/display/dc/inc/dcn_calcs.h
index ece954a40a8e..263c09630c06 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/dcn_calcs.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/dcn_calcs.h
@@ -621,7 +621,8 @@ extern const struct dcn_ip_params dcn10_ip_defaults;
 
 bool dcn_validate_bandwidth(
 		struct dc *dc,
-		struct dc_state *context);
+		struct dc_state *context,
+		bool fast_validate);
 
 unsigned int dcn_find_dcfclk_suits_all(
 	const struct dc *dc,
@@ -631,5 +632,7 @@ void dcn_bw_update_from_pplib(struct dc *dc);
 void dcn_bw_notify_pplib_of_wm_ranges(struct dc *dc);
 void dcn_bw_sync_calcs_and_dml(struct dc *dc);
 
+enum source_macro_tile_size swizzle_mode_to_macro_tile_size(enum swizzle_mode_values sw_mode);
+
 #endif /* __DCN_CALCS_H__ */
 
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h b/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h
index 23a4b18e5fee..31bd6d5183ab 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h
@@ -42,6 +42,8 @@ struct clk_mgr_funcs {
 			bool safe_to_lower);
 
 	int (*get_dp_ref_clk_frequency)(struct clk_mgr *clk_mgr);
+
+	void (*init_clocks)(struct clk_mgr *clk_mgr);
 };
 
 #endif /* __DAL_CLK_MGR_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/dccg.h b/drivers/gpu/drm/amd/display/dc/inc/hw/dccg.h
index 95a56d012626..05ee5295d2c1 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/dccg.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/dccg.h
@@ -39,6 +39,10 @@ struct dccg_funcs {
 	void (*update_dpp_dto)(struct dccg *dccg,
 			int dpp_inst,
 			int req_dppclk);
+	void (*get_dccg_ref_freq)(struct dccg *dccg,
+			unsigned int xtalin_freq_inKhz,
+			unsigned int *dccg_ref_freq_inKhz);
+	void (*dccg_init)(struct dccg *dccg);
 };
 
 #endif //__DAL_DCCG_H__
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/dchubbub.h b/drivers/gpu/drm/amd/display/dc/inc/hw/dchubbub.h
index 9d2d8e51306c..93667e8b23b3 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/dchubbub.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/dchubbub.h
@@ -73,6 +73,16 @@ struct hubbub_funcs {
 
 	void (*wm_read_state)(struct hubbub *hubbub,
 			struct dcn_hubbub_wm *wm);
+
+	void (*get_dchub_ref_freq)(struct hubbub *hubbub,
+			unsigned int dccg_ref_freq_inKhz,
+			unsigned int *dchub_ref_freq_inKhz);
+
+	void (*program_watermarks)(
+			struct hubbub *hubbub,
+			struct dcn_watermark_set *watermarks,
+			unsigned int refclk_mhz,
+			bool safe_to_lower);
 };
 
 struct hubbub {
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/dmcu.h b/drivers/gpu/drm/amd/display/dc/inc/hw/dmcu.h
index cbaa43853611..c68f0ce346c7 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/dmcu.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/dmcu.h
@@ -70,6 +70,8 @@ struct dmcu_funcs {
 	void (*get_psr_wait_loop)(struct dmcu *dmcu,
 			unsigned int *psr_wait_loop_number);
 	bool (*is_dmcu_initialized)(struct dmcu *dmcu);
+	bool (*lock_phy)(struct dmcu *dmcu);
+	bool (*unlock_phy)(struct dmcu *dmcu);
 };
 
 #endif
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h b/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h
index 1cd07e94ee63..455df4999797 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h
@@ -130,6 +130,7 @@ struct hubp_funcs {
 	void (*hubp_clear_underflow)(struct hubp *hubp);
 	void (*hubp_disable_control)(struct hubp *hubp, bool disable_hubp);
 	unsigned int (*hubp_get_underflow_status)(struct hubp *hubp);
+	void (*hubp_init)(struct hubp *hubp);
 
 };
 
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h b/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h
index da85537a4488..4c8e2c6fb6db 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h
@@ -146,6 +146,12 @@ struct out_csc_color_matrix {
 	uint16_t regval[12];
 };
 
+enum gamut_remap_select {
+	GAMUT_REMAP_BYPASS = 0,
+	GAMUT_REMAP_COEFF,
+	GAMUT_REMAP_COMA_COEFF,
+	GAMUT_REMAP_COMB_COEFF
+};
 
 enum opp_regamma {
 	OPP_REGAMMA_BYPASS = 0,
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/stream_encoder.h b/drivers/gpu/drm/amd/display/dc/inc/hw/stream_encoder.h
index 4051493557bc..49854eb73d1d 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/stream_encoder.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/stream_encoder.h
@@ -63,11 +63,13 @@ struct encoder_info_frame {
 	struct dc_info_packet vsc;
 	/* HDR Static MetaData */
 	struct dc_info_packet hdrsmd;
+	/* custom sdp message */
+	struct dc_info_packet dpsdp;
 };
 
 struct encoder_unblank_param {
 	struct dc_link_settings link_settings;
-	unsigned int pixel_clk_khz;
+	struct dc_crtc_timing timing;
 };
 
 struct encoder_set_dp_phy_pattern_param {
@@ -88,7 +90,8 @@ struct stream_encoder_funcs {
 	void (*dp_set_stream_attribute)(
 		struct stream_encoder *enc,
 		struct dc_crtc_timing *crtc_timing,
-		enum dc_color_space output_color_space);
+		enum dc_color_space output_color_space,
+		uint32_t enable_sdp_splitting);
 
 	void (*hdmi_set_stream_attribute)(
 		struct stream_encoder *enc,
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h b/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h
index c25f7df7b5e3..067d53caf28a 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h
@@ -187,8 +187,10 @@ struct timing_generator_funcs {
 	bool (*did_triggered_reset_occur)(struct timing_generator *tg);
 	void (*setup_global_swap_lock)(struct timing_generator *tg,
 							const struct dcp_gsl_params *gsl_params);
+	void (*setup_global_lock)(struct timing_generator *tg);
 	void (*unlock)(struct timing_generator *tg);
 	void (*lock)(struct timing_generator *tg);
+	void (*lock_global)(struct timing_generator *tg);
 	void (*enable_reset_trigger)(struct timing_generator *tg,
 				     int source_tg_inst);
 	void (*enable_crtc_reset)(struct timing_generator *tg,
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h
index 7676f25216b1..33905468e2b9 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h
@@ -176,6 +176,10 @@ struct hw_sequencer_funcs {
 				struct dc *dc,
 				struct pipe_ctx *pipe,
 				bool lock);
+	void (*pipe_control_lock_global)(
+				struct dc *dc,
+				struct pipe_ctx *pipe,
+				bool lock);
 	void (*blank_pixel_data)(
 			struct dc *dc,
 			struct pipe_ctx *pipe_ctx,
diff --git a/drivers/gpu/drm/amd/display/dc/inc/reg_helper.h b/drivers/gpu/drm/amd/display/dc/inc/reg_helper.h
index cf5a84b9e27c..8503d9cc4763 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/reg_helper.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/reg_helper.h
@@ -52,7 +52,7 @@
 
 /* macro to set register fields. */
 #define REG_SET_N(reg_name, n, initial_val, ...)	\
-		generic_reg_update_ex(CTX, \
+		generic_reg_set_ex(CTX, \
 				REG(reg_name), \
 				initial_val, \
 				n, __VA_ARGS__)
@@ -225,7 +225,6 @@
 #define REG_UPDATE_N(reg_name, n, ...)	\
 		generic_reg_update_ex(CTX, \
 				REG(reg_name), \
-				REG_READ(reg_name), \
 				n, __VA_ARGS__)
 
 #define REG_UPDATE(reg_name, field, val)	\
@@ -380,16 +379,11 @@
 /* macro to update a register field to specified values in given sequences.
  * useful when toggling bits
  */
-#define REG_UPDATE_SEQ(reg, field, value1, value2) \
-{	uint32_t val = REG_UPDATE(reg, field, value1); \
-	REG_SET(reg, val, field, value2); }
-
-/* macro to update fields in register 1 field at a time in given order */
-#define REG_UPDATE_1BY1_2(reg, f1, v1, f2, v2) \
+#define REG_UPDATE_SEQ_2(reg, f1, v1, f2, v2) \
 {	uint32_t val = REG_UPDATE(reg, f1, v1); \
 	REG_SET(reg, val, f2, v2); }
 
-#define REG_UPDATE_1BY1_3(reg, f1, v1, f2, v2, f3, v3) \
+#define REG_UPDATE_SEQ_3(reg, f1, v1, f2, v2, f3, v3) \
 {	uint32_t val = REG_UPDATE(reg, f1, v1); \
 	val = REG_SET(reg, val, f2, v2); \
 	REG_SET(reg, val, f3, v3); }
diff --git a/drivers/gpu/drm/amd/display/dc/inc/resource.h b/drivers/gpu/drm/amd/display/dc/inc/resource.h
index 0086a2f1d21a..3ce0a4fc5822 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/resource.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/resource.h
@@ -70,11 +70,9 @@ bool resource_construct(
 	struct resource_pool *pool,
 	const struct resource_create_funcs *create_funcs);
 
-struct resource_pool *dc_create_resource_pool(
-				struct dc *dc,
-				int num_virtual_links,
-				enum dce_version dc_version,
-				struct hw_asic_id asic_id);
+struct resource_pool *dc_create_resource_pool(struct dc  *dc,
+					      const struct dc_init_data *init_data,
+					      enum dce_version dc_version);
 
 void dc_destroy_resource_pool(struct dc *dc);
 
@@ -131,7 +129,8 @@ bool resource_attach_surfaces_to_context(
 
 struct pipe_ctx *find_idle_secondary_pipe(
 		struct resource_context *res_ctx,
-		const struct resource_pool *pool);
+		const struct resource_pool *pool,
+		const struct pipe_ctx *primary_pipe);
 
 bool resource_is_stream_unchanged(
 	struct dc_state *old_context, struct dc_stream_state *stream);
@@ -172,4 +171,7 @@ void update_audio_usage(
 
 unsigned int resource_pixel_format_to_bpp(enum surface_pixel_format format);
 
+struct pipe_ctx *dc_res_get_odm_bottom_pipe(struct pipe_ctx *pipe_ctx);
+bool dc_res_is_odm_head_pipe(struct pipe_ctx *pipe_ctx);
+
 #endif /* DRIVERS_GPU_DRM_AMD_DC_DEV_DC_INC_RESOURCE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/irq/dce110/irq_service_dce110.c b/drivers/gpu/drm/amd/display/dc/irq/dce110/irq_service_dce110.c
index afe0876fe6f8..86987f5e8bd5 100644
--- a/drivers/gpu/drm/amd/display/dc/irq/dce110/irq_service_dce110.c
+++ b/drivers/gpu/drm/amd/display/dc/irq/dce110/irq_service_dce110.c
@@ -81,6 +81,11 @@ static const struct irq_source_info_funcs vblank_irq_info_funcs = {
 	.ack = NULL
 };
 
+static const struct irq_source_info_funcs vupdate_irq_info_funcs = {
+	.set = NULL,
+	.ack = NULL
+};
+
 #define hpd_int_entry(reg_num)\
 	[DC_IRQ_SOURCE_HPD1 + reg_num] = {\
 		.enable_reg = mmHPD ## reg_num ## _DC_HPD_INT_CONTROL,\
@@ -137,7 +142,7 @@ static const struct irq_source_info_funcs vblank_irq_info_funcs = {
 		CRTC_V_UPDATE_INT_STATUS__CRTC_V_UPDATE_INT_CLEAR_MASK,\
 		.ack_value =\
 		CRTC_V_UPDATE_INT_STATUS__CRTC_V_UPDATE_INT_CLEAR_MASK,\
-		.funcs = &vblank_irq_info_funcs\
+		.funcs = &vupdate_irq_info_funcs\
 	}
 
 #define vblank_int_entry(reg_num)\
diff --git a/drivers/gpu/drm/amd/display/dc/irq/dce120/irq_service_dce120.c b/drivers/gpu/drm/amd/display/dc/irq/dce120/irq_service_dce120.c
index 1ea7256ec89b..750ba0ab4106 100644
--- a/drivers/gpu/drm/amd/display/dc/irq/dce120/irq_service_dce120.c
+++ b/drivers/gpu/drm/amd/display/dc/irq/dce120/irq_service_dce120.c
@@ -84,6 +84,11 @@ static const struct irq_source_info_funcs vblank_irq_info_funcs = {
 	.ack = NULL
 };
 
+static const struct irq_source_info_funcs vupdate_irq_info_funcs = {
+	.set = NULL,
+	.ack = NULL
+};
+
 #define BASE_INNER(seg) \
 	DCE_BASE__INST0_SEG ## seg
 
@@ -140,7 +145,7 @@ static const struct irq_source_info_funcs vblank_irq_info_funcs = {
 		IRQ_REG_ENTRY(CRTC, reg_num,\
 			CRTC_INTERRUPT_CONTROL, CRTC_V_UPDATE_INT_MSK,\
 			CRTC_V_UPDATE_INT_STATUS, CRTC_V_UPDATE_INT_CLEAR),\
-		.funcs = &vblank_irq_info_funcs\
+		.funcs = &vupdate_irq_info_funcs\
 	}
 
 #define vblank_int_entry(reg_num)\
diff --git a/drivers/gpu/drm/amd/display/dc/irq/dce80/irq_service_dce80.c b/drivers/gpu/drm/amd/display/dc/irq/dce80/irq_service_dce80.c
index 8a2066c313fe..de218fe84a43 100644
--- a/drivers/gpu/drm/amd/display/dc/irq/dce80/irq_service_dce80.c
+++ b/drivers/gpu/drm/amd/display/dc/irq/dce80/irq_service_dce80.c
@@ -84,6 +84,10 @@ static const struct irq_source_info_funcs vblank_irq_info_funcs = {
 	.ack = NULL
 };
 
+static const struct irq_source_info_funcs vupdate_irq_info_funcs = {
+	.set = NULL,
+	.ack = NULL
+};
 
 #define hpd_int_entry(reg_num)\
 	[DC_IRQ_SOURCE_INVALID + reg_num] = {\
@@ -142,7 +146,7 @@ static const struct irq_source_info_funcs vblank_irq_info_funcs = {
 		CRTC_V_UPDATE_INT_STATUS__CRTC_V_UPDATE_INT_CLEAR_MASK,\
 		.ack_value =\
 		CRTC_V_UPDATE_INT_STATUS__CRTC_V_UPDATE_INT_CLEAR_MASK,\
-		.funcs = &vblank_irq_info_funcs\
+		.funcs = &vupdate_irq_info_funcs\
 	}
 
 #define vblank_int_entry(reg_num)\
diff --git a/drivers/gpu/drm/amd/display/dc/irq/dcn10/irq_service_dcn10.c b/drivers/gpu/drm/amd/display/dc/irq/dcn10/irq_service_dcn10.c
index e04ae49243f6..10ac6deff5ff 100644
--- a/drivers/gpu/drm/amd/display/dc/irq/dcn10/irq_service_dcn10.c
+++ b/drivers/gpu/drm/amd/display/dc/irq/dcn10/irq_service_dcn10.c
@@ -56,6 +56,18 @@ enum dc_irq_source to_dal_irq_source_dcn10(
 		return DC_IRQ_SOURCE_VBLANK5;
 	case DCN_1_0__SRCID__DC_D6_OTG_VSTARTUP:
 		return DC_IRQ_SOURCE_VBLANK6;
+	case DCN_1_0__SRCID__OTG0_IHC_V_UPDATE_NO_LOCK_INTERRUPT:
+		return DC_IRQ_SOURCE_VUPDATE1;
+	case DCN_1_0__SRCID__OTG1_IHC_V_UPDATE_NO_LOCK_INTERRUPT:
+		return DC_IRQ_SOURCE_VUPDATE2;
+	case DCN_1_0__SRCID__OTG2_IHC_V_UPDATE_NO_LOCK_INTERRUPT:
+		return DC_IRQ_SOURCE_VUPDATE3;
+	case DCN_1_0__SRCID__OTG3_IHC_V_UPDATE_NO_LOCK_INTERRUPT:
+		return DC_IRQ_SOURCE_VUPDATE4;
+	case DCN_1_0__SRCID__OTG4_IHC_V_UPDATE_NO_LOCK_INTERRUPT:
+		return DC_IRQ_SOURCE_VUPDATE5;
+	case DCN_1_0__SRCID__OTG5_IHC_V_UPDATE_NO_LOCK_INTERRUPT:
+		return DC_IRQ_SOURCE_VUPDATE6;
 	case DCN_1_0__SRCID__HUBP0_FLIP_INTERRUPT:
 		return DC_IRQ_SOURCE_PFLIP1;
 	case DCN_1_0__SRCID__HUBP1_FLIP_INTERRUPT:
@@ -153,6 +165,11 @@ static const struct irq_source_info_funcs vblank_irq_info_funcs = {
 	.ack = NULL
 };
 
+static const struct irq_source_info_funcs vupdate_no_lock_irq_info_funcs = {
+	.set = NULL,
+	.ack = NULL
+};
+
 #define BASE_INNER(seg) \
 	DCE_BASE__INST0_SEG ## seg
 
@@ -203,12 +220,15 @@ static const struct irq_source_info_funcs vblank_irq_info_funcs = {
 		.funcs = &pflip_irq_info_funcs\
 	}
 
-#define vupdate_int_entry(reg_num)\
+/* vupdate_no_lock_int_entry maps to DC_IRQ_SOURCE_VUPDATEx, to match semantic
+ * of DCE's DC_IRQ_SOURCE_VUPDATEx.
+ */
+#define vupdate_no_lock_int_entry(reg_num)\
 	[DC_IRQ_SOURCE_VUPDATE1 + reg_num] = {\
 		IRQ_REG_ENTRY(OTG, reg_num,\
-			OTG_GLOBAL_SYNC_STATUS, VUPDATE_INT_EN,\
-			OTG_GLOBAL_SYNC_STATUS, VUPDATE_EVENT_CLEAR),\
-		.funcs = &vblank_irq_info_funcs\
+			OTG_GLOBAL_SYNC_STATUS, VUPDATE_NO_LOCK_INT_EN,\
+			OTG_GLOBAL_SYNC_STATUS, VUPDATE_NO_LOCK_EVENT_CLEAR),\
+		.funcs = &vupdate_no_lock_irq_info_funcs\
 	}
 
 #define vblank_int_entry(reg_num)\
@@ -315,12 +335,12 @@ irq_source_info_dcn10[DAL_IRQ_SOURCES_NUMBER] = {
 	dc_underflow_int_entry(6),
 	[DC_IRQ_SOURCE_DMCU_SCP] = dummy_irq_entry(),
 	[DC_IRQ_SOURCE_VBIOS_SW] = dummy_irq_entry(),
-	vupdate_int_entry(0),
-	vupdate_int_entry(1),
-	vupdate_int_entry(2),
-	vupdate_int_entry(3),
-	vupdate_int_entry(4),
-	vupdate_int_entry(5),
+	vupdate_no_lock_int_entry(0),
+	vupdate_no_lock_int_entry(1),
+	vupdate_no_lock_int_entry(2),
+	vupdate_no_lock_int_entry(3),
+	vupdate_no_lock_int_entry(4),
+	vupdate_no_lock_int_entry(5),
 	vblank_int_entry(0),
 	vblank_int_entry(1),
 	vblank_int_entry(2),
diff --git a/drivers/gpu/drm/amd/display/dc/virtual/virtual_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/virtual/virtual_stream_encoder.c
index 3dc1733eea20..fdcf9e66d852 100644
--- a/drivers/gpu/drm/amd/display/dc/virtual/virtual_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/virtual/virtual_stream_encoder.c
@@ -29,7 +29,8 @@
 static void virtual_stream_encoder_dp_set_stream_attribute(
 	struct stream_encoder *enc,
 	struct dc_crtc_timing *crtc_timing,
-	enum dc_color_space output_color_space) {}
+	enum dc_color_space output_color_space,
+	uint32_t enable_sdp_splitting) {}
 
 static void virtual_stream_encoder_hdmi_set_stream_attribute(
 	struct stream_encoder *enc,
diff --git a/drivers/gpu/drm/amd/display/include/fixed31_32.h b/drivers/gpu/drm/amd/display/include/fixed31_32.h
index 52a73332befb..89ef9f6860e5 100644
--- a/drivers/gpu/drm/amd/display/include/fixed31_32.h
+++ b/drivers/gpu/drm/amd/display/include/fixed31_32.h
@@ -503,6 +503,8 @@ static inline int dc_fixpt_ceil(struct fixed31_32 arg)
  * fractional
  */
 
+unsigned int dc_fixpt_u4d19(struct fixed31_32 arg);
+
 unsigned int dc_fixpt_u3d19(struct fixed31_32 arg);
 
 unsigned int dc_fixpt_u2d19(struct fixed31_32 arg);
diff --git a/drivers/gpu/drm/amd/display/include/signal_types.h b/drivers/gpu/drm/amd/display/include/signal_types.h
index f56d2891475f..beed70179bb5 100644
--- a/drivers/gpu/drm/amd/display/include/signal_types.h
+++ b/drivers/gpu/drm/amd/display/include/signal_types.h
@@ -45,6 +45,11 @@ enum signal_type {
 };
 
 /* help functions for signal types manipulation */
+static inline bool dc_is_hdmi_tmds_signal(enum signal_type signal)
+{
+	return (signal == SIGNAL_TYPE_HDMI_TYPE_A);
+}
+
 static inline bool dc_is_hdmi_signal(enum signal_type signal)
 {
 	return (signal == SIGNAL_TYPE_HDMI_TYPE_A);
diff --git a/drivers/gpu/drm/amd/display/modules/color/color_gamma.c b/drivers/gpu/drm/amd/display/modules/color/color_gamma.c
index 0fbc8fbc3541..a1055413bade 100644
--- a/drivers/gpu/drm/amd/display/modules/color/color_gamma.c
+++ b/drivers/gpu/drm/amd/display/modules/color/color_gamma.c
@@ -1854,6 +1854,8 @@ bool mod_color_calculate_degamma_params(struct dc_transfer_func *input_tf,
 			coordinates_x, axis_x, curve,
 			MAX_HW_POINTS, tf_pts,
 			mapUserRamp && ramp && ramp->type == GAMMA_RGB_256);
+	if (ramp->type == GAMMA_CUSTOM)
+		apply_lut_1d(ramp, MAX_HW_POINTS, tf_pts);
 
 	ret = true;
 
diff --git a/drivers/gpu/drm/amd/display/modules/freesync/freesync.c b/drivers/gpu/drm/amd/display/modules/freesync/freesync.c
index bfd27f10879e..19b1eaebe484 100644
--- a/drivers/gpu/drm/amd/display/modules/freesync/freesync.c
+++ b/drivers/gpu/drm/amd/display/modules/freesync/freesync.c
@@ -37,6 +37,8 @@
 #define RENDER_TIMES_MAX_COUNT 10
 /* Threshold to exit BTR (to avoid frequent enter-exits at the lower limit) */
 #define BTR_EXIT_MARGIN 2000
+/* Threshold to change BTR multiplier (to avoid frequent changes) */
+#define BTR_DRIFT_MARGIN 2000
 /*Threshold to exit fixed refresh rate*/
 #define FIXED_REFRESH_EXIT_MARGIN_IN_HZ 4
 /* Number of consecutive frames to check before entering/exiting fixed refresh*/
@@ -48,6 +50,93 @@ struct core_freesync {
 	struct dc *dc;
 };
 
+void setFieldWithMask(unsigned char *dest, unsigned int mask, unsigned int value)
+{
+	unsigned int shift = 0;
+
+	if (!mask || !dest)
+		return;
+
+	while (!((mask >> shift) & 1))
+		shift++;
+
+	//reset
+	*dest = *dest & ~mask;
+	//set
+	//dont let value span past mask
+	value = value & (mask >> shift);
+	//insert value
+	*dest = *dest | (value << shift);
+}
+
+// VTEM Byte Offset
+#define VRR_VTEM_PB0		0
+#define VRR_VTEM_PB1		1
+#define VRR_VTEM_PB2		2
+#define VRR_VTEM_PB3		3
+#define VRR_VTEM_PB4		4
+#define VRR_VTEM_PB5		5
+#define VRR_VTEM_PB6		6
+
+#define VRR_VTEM_MD0		7
+#define VRR_VTEM_MD1		8
+#define VRR_VTEM_MD2		9
+#define VRR_VTEM_MD3		10
+
+
+// VTEM Byte Masks
+//PB0
+#define MASK__VRR_VTEM_PB0__RESERVED0  0x01
+#define MASK__VRR_VTEM_PB0__SYNC       0x02
+#define MASK__VRR_VTEM_PB0__VFR        0x04
+#define MASK__VRR_VTEM_PB0__AFR        0x08
+#define MASK__VRR_VTEM_PB0__DS_TYPE    0x30
+	//0: Periodic pseudo-static EM Data Set
+	//1: Periodic dynamic EM Data Set
+	//2: Unique EM Data Set
+	//3: Reserved
+#define MASK__VRR_VTEM_PB0__END        0x40
+#define MASK__VRR_VTEM_PB0__NEW        0x80
+
+//PB1
+#define MASK__VRR_VTEM_PB1__RESERVED1 0xFF
+
+//PB2
+#define MASK__VRR_VTEM_PB2__ORGANIZATION_ID 0xFF
+	//0: This is a Vendor Specific EM Data Set
+	//1: This EM Data Set is defined by This Specification (HDMI 2.1 r102.clean)
+	//2: This EM Data Set is defined by CTA-861-G
+	//3: This EM Data Set is defined by VESA
+//PB3
+#define MASK__VRR_VTEM_PB3__DATA_SET_TAG_MSB    0xFF
+//PB4
+#define MASK__VRR_VTEM_PB4__DATA_SET_TAG_LSB    0xFF
+//PB5
+#define MASK__VRR_VTEM_PB5__DATA_SET_LENGTH_MSB 0xFF
+//PB6
+#define MASK__VRR_VTEM_PB6__DATA_SET_LENGTH_LSB 0xFF
+
+
+
+//PB7-27 (20 bytes):
+//PB7 = MD0
+#define MASK__VRR_VTEM_MD0__VRR_EN         0x01
+#define MASK__VRR_VTEM_MD0__M_CONST        0x02
+#define MASK__VRR_VTEM_MD0__RESERVED2      0x0C
+#define MASK__VRR_VTEM_MD0__FVA_FACTOR_M1  0xF0
+
+//MD1
+#define MASK__VRR_VTEM_MD1__BASE_VFRONT    0xFF
+
+//MD2
+#define MASK__VRR_VTEM_MD2__BASE_REFRESH_RATE_98  0x03
+#define MASK__VRR_VTEM_MD2__RB                    0x04
+#define MASK__VRR_VTEM_MD2__RESERVED3             0xF8
+
+//MD3
+#define MASK__VRR_VTEM_MD3__BASE_REFRESH_RATE_07  0xFF
+
+
 #define MOD_FREESYNC_TO_CORE(mod_freesync)\
 		container_of(mod_freesync, struct core_freesync, public)
 
@@ -248,6 +337,7 @@ static void apply_below_the_range(struct core_freesync *core_freesync,
 	unsigned int frames_to_insert = 0;
 	unsigned int min_frame_duration_in_ns = 0;
 	unsigned int max_render_time_in_us = in_out_vrr->max_duration_in_us;
+	unsigned int delta_from_mid_point_delta_in_us;
 
 	min_frame_duration_in_ns = ((unsigned int) (div64_u64(
 		(1000000000ULL * 1000000),
@@ -318,10 +408,27 @@ static void apply_below_the_range(struct core_freesync *core_freesync,
 		/* Choose number of frames to insert based on how close it
 		 * can get to the mid point of the variable range.
 		 */
-		if (delta_from_mid_point_in_us_1 < delta_from_mid_point_in_us_2)
+		if (delta_from_mid_point_in_us_1 < delta_from_mid_point_in_us_2) {
 			frames_to_insert = mid_point_frames_ceil;
-		else
+			delta_from_mid_point_delta_in_us = delta_from_mid_point_in_us_2 -
+					delta_from_mid_point_in_us_1;
+		} else {
 			frames_to_insert = mid_point_frames_floor;
+			delta_from_mid_point_delta_in_us = delta_from_mid_point_in_us_1 -
+					delta_from_mid_point_in_us_2;
+		}
+
+		/* Prefer current frame multiplier when BTR is enabled unless it drifts
+		 * too far from the midpoint
+		 */
+		if (in_out_vrr->btr.frames_to_insert != 0 &&
+				delta_from_mid_point_delta_in_us < BTR_DRIFT_MARGIN) {
+			if (((last_render_time_in_us / in_out_vrr->btr.frames_to_insert) <
+					in_out_vrr->max_duration_in_us) &&
+				((last_render_time_in_us / in_out_vrr->btr.frames_to_insert) >
+					in_out_vrr->min_duration_in_us))
+				frames_to_insert = in_out_vrr->btr.frames_to_insert;
+		}
 
 		/* Either we've calculated the number of frames to insert,
 		 * or we need to insert min duration frames
@@ -330,10 +437,8 @@ static void apply_below_the_range(struct core_freesync *core_freesync,
 			inserted_frame_duration_in_us = last_render_time_in_us /
 							frames_to_insert;
 
-		if (inserted_frame_duration_in_us <
-			(1000000 / in_out_vrr->max_refresh_in_uhz))
-			inserted_frame_duration_in_us =
-				(1000000 / in_out_vrr->max_refresh_in_uhz);
+		if (inserted_frame_duration_in_us < in_out_vrr->min_duration_in_us)
+			inserted_frame_duration_in_us = in_out_vrr->min_duration_in_us;
 
 		/* Cache the calculated variables */
 		in_out_vrr->btr.inserted_duration_in_us =
@@ -469,16 +574,14 @@ static void build_vrr_infopacket_header_vtem(enum signal_type signal,
 	// HB0, HB1, HB2 indicates PacketType VTEMPacket
 	infopacket->hb0 = 0x7F;
 	infopacket->hb1 = 0xC0;
-	infopacket->hb2 = 0x00;
-	/* HB3 Bit Fields
-	 * Reserved :1 = 0
-	 * Sync     :1 = 0
-	 * VFR      :1 = 1
-	 * Ds_Type  :2 = 0
-	 * End      :1 = 0
-	 * New      :1 = 0
-	 */
-	infopacket->hb3 = 0x20;
+	infopacket->hb2 = 0x00; //sequence_index
+
+	setFieldWithMask(&infopacket->sb[VRR_VTEM_PB0], MASK__VRR_VTEM_PB0__VFR, 1);
+	setFieldWithMask(&infopacket->sb[VRR_VTEM_PB2], MASK__VRR_VTEM_PB2__ORGANIZATION_ID, 1);
+	setFieldWithMask(&infopacket->sb[VRR_VTEM_PB3], MASK__VRR_VTEM_PB3__DATA_SET_TAG_MSB, 0);
+	setFieldWithMask(&infopacket->sb[VRR_VTEM_PB4], MASK__VRR_VTEM_PB4__DATA_SET_TAG_LSB, 1);
+	setFieldWithMask(&infopacket->sb[VRR_VTEM_PB5], MASK__VRR_VTEM_PB5__DATA_SET_LENGTH_MSB, 0);
+	setFieldWithMask(&infopacket->sb[VRR_VTEM_PB6], MASK__VRR_VTEM_PB6__DATA_SET_LENGTH_LSB, 4);
 }
 
 static void build_vrr_infopacket_header_v1(enum signal_type signal,
@@ -583,45 +686,36 @@ static void build_vrr_vtem_infopacket_data(const struct dc_stream_state *stream,
 		const struct mod_vrr_params *vrr,
 		struct dc_info_packet *infopacket)
 {
-	/* dc_info_packet to VtemPacket Translation of Bit-fields,
-	 * SB[6]
-	 * unsigned char VRR_EN        :1
-	 * unsigned char M_CONST       :1
-	 * unsigned char Reserved2     :2
-	 * unsigned char FVA_Factor_M1 :4
-	 * SB[7]
-	 * unsigned char Base_Vfront   :8
-	 * SB[8]
-	 * unsigned char Base_Refresh_Rate_98 :2
-	 * unsigned char RB                   :1
-	 * unsigned char Reserved3            :5
-	 * SB[9]
-	 * unsigned char Base_RefreshRate_07  :8
-	 */
 	unsigned int fieldRateInHz;
 
 	if (vrr->state == VRR_STATE_ACTIVE_VARIABLE ||
-				vrr->state == VRR_STATE_ACTIVE_FIXED){
-		infopacket->sb[6] |= 0x80; //VRR_EN Bit = 1
+				vrr->state == VRR_STATE_ACTIVE_FIXED) {
+		setFieldWithMask(&infopacket->sb[VRR_VTEM_MD0], MASK__VRR_VTEM_MD0__VRR_EN, 1);
 	} else {
-		infopacket->sb[6] &= 0x7F; //VRR_EN Bit = 0
+		setFieldWithMask(&infopacket->sb[VRR_VTEM_MD0], MASK__VRR_VTEM_MD0__VRR_EN, 0);
 	}
 
 	if (!stream->timing.vic) {
-		infopacket->sb[7] = stream->timing.v_front_porch;
+		setFieldWithMask(&infopacket->sb[VRR_VTEM_MD1], MASK__VRR_VTEM_MD1__BASE_VFRONT,
+				stream->timing.v_front_porch);
+
 
 		/* TODO: In dal2, we check mode flags for a reduced blanking timing.
 		 * Need a way to relay that information to this function.
 		 * if("ReducedBlanking")
 		 * {
-		 *   infopacket->sb[8] |= 0x20; //Set 3rd bit to 1
+		 *   setFieldWithMask(&infopacket->sb[VRR_VTEM_MD2], MASK__VRR_VTEM_MD2__RB, 1;
 		 * }
 		 */
+
+		//TODO: DAL2 does FixPoint and rounding. Here we might need to account for that
 		fieldRateInHz = (stream->timing.pix_clk_100hz * 100)/
-				(stream->timing.h_total * stream->timing.v_total);
+			(stream->timing.h_total * stream->timing.v_total);
 
-		infopacket->sb[8] |= ((fieldRateInHz & 0x300) >> 2);
-		infopacket->sb[9] |= fieldRateInHz & 0xFF;
+		setFieldWithMask(&infopacket->sb[VRR_VTEM_MD2],  MASK__VRR_VTEM_MD2__BASE_REFRESH_RATE_98,
+				fieldRateInHz >> 8);
+		setFieldWithMask(&infopacket->sb[VRR_VTEM_MD3], MASK__VRR_VTEM_MD3__BASE_REFRESH_RATE_07,
+				fieldRateInHz);
 
 	}
 	infopacket->valid = true;
@@ -745,6 +839,8 @@ static void build_vrr_infopacket_vtem(const struct dc_stream_state *stream,
 {
 	//VTEM info packet for HdmiVrr
 
+	memset(infopacket, 0, sizeof(struct dc_info_packet));
+
 	//VTEM Packet is structured differently
 	build_vrr_infopacket_header_vtem(stream->signal, infopacket);
 	build_vrr_vtem_infopacket_data(stream, vrr, infopacket);
diff --git a/drivers/gpu/drm/amd/display/modules/power/power_helpers.c b/drivers/gpu/drm/amd/display/modules/power/power_helpers.c
index 038b88221c5f..b3810b864676 100644
--- a/drivers/gpu/drm/amd/display/modules/power/power_helpers.c
+++ b/drivers/gpu/drm/amd/display/modules/power/power_helpers.c
@@ -41,9 +41,12 @@ static const unsigned char min_reduction_table[13] = {
 static const unsigned char max_reduction_table[13] = {
 0xf5, 0xe5, 0xd9, 0xcd, 0xb1, 0xa5, 0xa5, 0x80, 0x65, 0x4d, 0x4d, 0x4d, 0x32};
 
-/* ABM 2.2 Min Reduction effectively disabled (100% for all configs)*/
+/* Possible ABM 2.2 Min Reduction configs from least aggressive to most aggressive
+ *  0    1     2     3     4     5     6     7     8     9     10    11   12
+ * 100  100   100   100   100   100   100   100  100  92.2  83.1  75.3  75.3 %
+ */
 static const unsigned char min_reduction_table_v_2_2[13] = {
-0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xeb, 0xd4, 0xc0, 0xc0};
 
 /* Possible ABM 2.2 Max Reduction configs from least aggressive to most aggressive
  *  0    1     2     3     4     5     6     7     8     9     10    11   12
@@ -408,9 +411,9 @@ void fill_iram_v_2_2(struct iram_table_v_2_2 *ram_table, struct dmcu_iram_parame
 	ram_table->flags = 0x0;
 
 	ram_table->deviation_gain[0] = 0xb3;
-	ram_table->deviation_gain[1] = 0xb3;
-	ram_table->deviation_gain[2] = 0xb3;
-	ram_table->deviation_gain[3] = 0xb3;
+	ram_table->deviation_gain[1] = 0xa8;
+	ram_table->deviation_gain[2] = 0x98;
+	ram_table->deviation_gain[3] = 0x68;
 
 	ram_table->min_reduction[0][0] = min_reduction_table_v_2_2[abm_config[set][0]];
 	ram_table->min_reduction[1][0] = min_reduction_table_v_2_2[abm_config[set][0]];
@@ -505,7 +508,7 @@ void fill_iram_v_2_2(struct iram_table_v_2_2 *ram_table, struct dmcu_iram_parame
 
 	ram_table->contrastFactor[0] = 0x99;
 	ram_table->contrastFactor[1] = 0x99;
-	ram_table->contrastFactor[2] = 0x99;
+	ram_table->contrastFactor[2] = 0x90;
 	ram_table->contrastFactor[3] = 0x80;
 
 	ram_table->iir_curve[0] = 0x65;
diff --git a/drivers/gpu/drm/amd/include/amd_shared.h b/drivers/gpu/drm/amd/include/amd_shared.h
index 470d7b89071a..574bf6e70763 100644
--- a/drivers/gpu/drm/amd/include/amd_shared.h
+++ b/drivers/gpu/drm/amd/include/amd_shared.h
@@ -137,6 +137,7 @@ enum DC_FEATURE_MASK {
 	DC_FBC_MASK = 0x1,
 };
 
+enum amd_dpm_forced_level;
 /**
  * struct amd_ip_funcs - general hooks for managing amdgpu IP Blocks
  */
@@ -186,6 +187,8 @@ struct amd_ip_funcs {
 				     enum amd_powergating_state state);
 	/** @get_clockgating_state: get current clockgating status */
 	void (*get_clockgating_state)(void *handle, u32 *flags);
+	/** @enable_umd_pstate: enable UMD powerstate */
+	int (*enable_umd_pstate)(void *handle, enum amd_dpm_forced_level *level);
 };
 
 
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_1_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_1_0_offset.h
index 721c61171045..5a44e614ab7e 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_1_0_offset.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_1_0_offset.h
@@ -2347,6 +2347,8 @@
 #define mmHUBP0_DCHUBP_VMPG_CONFIG_BASE_IDX                                                            2
 #define mmHUBP0_HUBPREQ_DEBUG_DB                                                                       0x0569
 #define mmHUBP0_HUBPREQ_DEBUG_DB_BASE_IDX                                                              2
+#define mmHUBP0_HUBPREQ_DEBUG                                                                          0x056a
+#define mmHUBP0_HUBPREQ_DEBUG_BASE_IDX                                                                 2
 #define mmHUBP0_HUBP_MEASURE_WIN_CTRL_DCFCLK                                                           0x056e
 #define mmHUBP0_HUBP_MEASURE_WIN_CTRL_DCFCLK_BASE_IDX                                                  2
 #define mmHUBP0_HUBP_MEASURE_WIN_CTRL_DPPCLK                                                           0x056f
@@ -2631,6 +2633,8 @@
 #define mmHUBP1_DCHUBP_VMPG_CONFIG_BASE_IDX                                                            2
 #define mmHUBP1_HUBPREQ_DEBUG_DB                                                                       0x062d
 #define mmHUBP1_HUBPREQ_DEBUG_DB_BASE_IDX                                                              2
+#define mmHUBP1_HUBPREQ_DEBUG                                                                          0x062e
+#define mmHUBP1_HUBPREQ_DEBUG_BASE_IDX                                                                 2
 #define mmHUBP1_HUBP_MEASURE_WIN_CTRL_DCFCLK                                                           0x0632
 #define mmHUBP1_HUBP_MEASURE_WIN_CTRL_DCFCLK_BASE_IDX                                                  2
 #define mmHUBP1_HUBP_MEASURE_WIN_CTRL_DPPCLK                                                           0x0633
@@ -2915,6 +2919,8 @@
 #define mmHUBP2_DCHUBP_VMPG_CONFIG_BASE_IDX                                                            2
 #define mmHUBP2_HUBPREQ_DEBUG_DB                                                                       0x06f1
 #define mmHUBP2_HUBPREQ_DEBUG_DB_BASE_IDX                                                              2
+#define mmHUBP2_HUBPREQ_DEBUG                                                                          0x06f2
+#define mmHUBP2_HUBPREQ_DEBUG_BASE_IDX                                                                 2
 #define mmHUBP2_HUBP_MEASURE_WIN_CTRL_DCFCLK                                                           0x06f6
 #define mmHUBP2_HUBP_MEASURE_WIN_CTRL_DCFCLK_BASE_IDX                                                  2
 #define mmHUBP2_HUBP_MEASURE_WIN_CTRL_DPPCLK                                                           0x06f7
@@ -3199,6 +3205,8 @@
 #define mmHUBP3_DCHUBP_VMPG_CONFIG_BASE_IDX                                                            2
 #define mmHUBP3_HUBPREQ_DEBUG_DB                                                                       0x07b5
 #define mmHUBP3_HUBPREQ_DEBUG_DB_BASE_IDX                                                              2
+#define mmHUBP3_HUBPREQ_DEBUG                                                                          0x07b6
+#define mmHUBP3_HUBPREQ_DEBUG_BASE_IDX                                                                 2
 #define mmHUBP3_HUBP_MEASURE_WIN_CTRL_DCFCLK                                                           0x07ba
 #define mmHUBP3_HUBP_MEASURE_WIN_CTRL_DCFCLK_BASE_IDX                                                  2
 #define mmHUBP3_HUBP_MEASURE_WIN_CTRL_DPPCLK                                                           0x07bb
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vcn/vcn_1_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/vcn/vcn_1_0_offset.h
index 442ca7c471a5..6109f5ad25ad 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/vcn/vcn_1_0_offset.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/vcn/vcn_1_0_offset.h
@@ -141,6 +141,8 @@
 #define mmUVD_GPCOM_VCPU_DATA0_BASE_IDX                                                                1
 #define mmUVD_GPCOM_VCPU_DATA1                                                                         0x03c5
 #define mmUVD_GPCOM_VCPU_DATA1_BASE_IDX                                                                1
+#define mmUVD_ENGINE_CNTL                                                                              0x03c6
+#define mmUVD_ENGINE_CNTL_BASE_IDX                                                                     1
 #define mmUVD_UDEC_DBW_UV_ADDR_CONFIG                                                                  0x03d2
 #define mmUVD_UDEC_DBW_UV_ADDR_CONFIG_BASE_IDX                                                         1
 #define mmUVD_UDEC_ADDR_CONFIG                                                                         0x03d3
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vcn/vcn_1_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/vcn/vcn_1_0_sh_mask.h
index 63457f9df4c5..f84bed6eecb9 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/vcn/vcn_1_0_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/vcn/vcn_1_0_sh_mask.h
@@ -312,6 +312,11 @@
 //UVD_GPCOM_VCPU_DATA1
 #define UVD_GPCOM_VCPU_DATA1__DATA1__SHIFT                                                                    0x0
 #define UVD_GPCOM_VCPU_DATA1__DATA1_MASK                                                                      0xFFFFFFFFL
+//UVD_ENGINE_CNTL
+#define UVD_ENGINE_CNTL__ENGINE_START_MASK 0x1
+#define UVD_ENGINE_CNTL__ENGINE_START__SHIFT 0x0
+#define UVD_ENGINE_CNTL__ENGINE_START_MODE_MASK 0x2
+#define UVD_ENGINE_CNTL__ENGINE_START_MODE__SHIFT 0x1
 //UVD_UDEC_DBW_UV_ADDR_CONFIG
 #define UVD_UDEC_DBW_UV_ADDR_CONFIG__NUM_PIPES__SHIFT                                                         0x0
 #define UVD_UDEC_DBW_UV_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE__SHIFT                                              0x3
diff --git a/drivers/gpu/drm/amd/include/atomfirmware.h b/drivers/gpu/drm/amd/include/atomfirmware.h
index 8eb0bb241210..d3075adb3297 100644
--- a/drivers/gpu/drm/amd/include/atomfirmware.h
+++ b/drivers/gpu/drm/amd/include/atomfirmware.h
@@ -494,6 +494,9 @@ enum atombios_firmware_capability
   ATOM_FIRMWARE_CAP_FIRMWARE_POSTED = 0x00000001,
   ATOM_FIRMWARE_CAP_GPU_VIRTUALIZATION  = 0x00000002,
   ATOM_FIRMWARE_CAP_WMI_SUPPORT  = 0x00000040,
+  ATOM_FIRMWARE_CAP_HWEMU_ENABLE  = 0x00000080,
+  ATOM_FIRMWARE_CAP_HWEMU_UMC_CFG = 0x00000100,
+  ATOM_FIRMWARE_CAP_SRAM_ECC      = 0x00000200,
 };
 
 enum atom_cooling_solution_id{
@@ -528,6 +531,35 @@ struct atom_firmware_info_v3_2 {
   uint32_t reserved2[3];
 };
 
+struct atom_firmware_info_v3_3
+{
+  struct atom_common_table_header table_header;
+  uint32_t firmware_revision;
+  uint32_t bootup_sclk_in10khz;
+  uint32_t bootup_mclk_in10khz;
+  uint32_t firmware_capability;             // enum atombios_firmware_capability
+  uint32_t main_call_parser_entry;          /* direct address of main parser call in VBIOS binary. */
+  uint32_t bios_scratch_reg_startaddr;      // 1st bios scratch register dword address
+  uint16_t bootup_vddc_mv;
+  uint16_t bootup_vddci_mv;
+  uint16_t bootup_mvddc_mv;
+  uint16_t bootup_vddgfx_mv;
+  uint8_t  mem_module_id;
+  uint8_t  coolingsolution_id;              /*0: Air cooling; 1: Liquid cooling ... */
+  uint8_t  reserved1[2];
+  uint32_t mc_baseaddr_high;
+  uint32_t mc_baseaddr_low;
+  uint8_t  board_i2c_feature_id;            // enum of atom_board_i2c_feature_id_def
+  uint8_t  board_i2c_feature_gpio_id;       // i2c id find in gpio_lut data table gpio_id
+  uint8_t  board_i2c_feature_slave_addr;
+  uint8_t  reserved3;
+  uint16_t bootup_mvddq_mv;
+  uint16_t bootup_mvpp_mv;
+  uint32_t zfbstartaddrin16mb;
+  uint32_t pplib_pptable_id;                // if pplib_pptable_id!=0, pplib get powerplay table inside driver instead of from VBIOS
+  uint32_t reserved2[2];
+};
+
 /* 
   ***************************************************************************
     Data Table lcd_info  structure
@@ -686,6 +718,7 @@ enum atom_encoder_caps_def
   ATOM_ENCODER_CAP_RECORD_HBR2_EN               =0x02,         // DP1.2 HBR2 setting is qualified and HBR2 can be enabled 
   ATOM_ENCODER_CAP_RECORD_HDMI6Gbps_EN          =0x04,         // HDMI2.0 6Gbps enable or not. 
   ATOM_ENCODER_CAP_RECORD_HBR3_EN               =0x08,         // DP1.3 HBR3 is supported by board. 
+  ATOM_ENCODER_CAP_RECORD_USB_C_TYPE            =0x100,        // the DP connector is a USB-C type.
 };
 
 struct  atom_encoder_caps_record
@@ -1226,16 +1259,17 @@ struct  atom_gfx_info_v2_3 {
   uint32_t rm21_sram_vmin_value;
 };
 
-struct  atom_gfx_info_v2_4 {
+struct  atom_gfx_info_v2_4
+{
   struct  atom_common_table_header  table_header;
   uint8_t gfxip_min_ver;
   uint8_t gfxip_max_ver;
-  uint8_t gc_num_se;
-  uint8_t max_tile_pipes;
-  uint8_t gc_num_cu_per_sh;
-  uint8_t gc_num_sh_per_se;
-  uint8_t gc_num_rb_per_se;
-  uint8_t gc_num_tccs;
+  uint8_t max_shader_engines;
+  uint8_t reserved;
+  uint8_t max_cu_per_sh;
+  uint8_t max_sh_per_se;
+  uint8_t max_backends_per_se;
+  uint8_t max_texture_channel_caches;
   uint32_t regaddr_cp_dma_src_addr;
   uint32_t regaddr_cp_dma_src_addr_hi;
   uint32_t regaddr_cp_dma_dst_addr;
@@ -1780,6 +1814,56 @@ struct atom_umc_info_v3_1
   uint32_t mem_refclk_10khz;
 };
 
+// umc_info.umc_config
+enum atom_umc_config_def {
+  UMC_CONFIG__ENABLE_1KB_INTERLEAVE_MODE  =   0x00000001,
+  UMC_CONFIG__DEFAULT_MEM_ECC_ENABLE      =   0x00000002,
+  UMC_CONFIG__ENABLE_HBM_LANE_REPAIR      =   0x00000004,
+  UMC_CONFIG__ENABLE_BANK_HARVESTING      =   0x00000008,
+  UMC_CONFIG__ENABLE_PHY_REINIT           =   0x00000010,
+  UMC_CONFIG__DISABLE_UCODE_CHKSTATUS     =   0x00000020,
+};
+
+struct atom_umc_info_v3_2
+{
+  struct  atom_common_table_header  table_header;
+  uint32_t ucode_version;
+  uint32_t ucode_rom_startaddr;
+  uint32_t ucode_length;
+  uint16_t umc_reg_init_offset;
+  uint16_t customer_ucode_name_offset;
+  uint16_t mclk_ss_percentage;
+  uint16_t mclk_ss_rate_10hz;
+  uint8_t umcip_min_ver;
+  uint8_t umcip_max_ver;
+  uint8_t vram_type;              //enum of atom_dgpu_vram_type
+  uint8_t umc_config;
+  uint32_t mem_refclk_10khz;
+  uint32_t pstate_uclk_10khz[4];
+  uint16_t umcgoldenoffset;
+  uint16_t densitygoldenoffset;
+};
+
+struct atom_umc_info_v3_3
+{
+  struct  atom_common_table_header  table_header;
+  uint32_t ucode_reserved;
+  uint32_t ucode_rom_startaddr;
+  uint32_t ucode_length;
+  uint16_t umc_reg_init_offset;
+  uint16_t customer_ucode_name_offset;
+  uint16_t mclk_ss_percentage;
+  uint16_t mclk_ss_rate_10hz;
+  uint8_t umcip_min_ver;
+  uint8_t umcip_max_ver;
+  uint8_t vram_type;              //enum of atom_dgpu_vram_type
+  uint8_t umc_config;
+  uint32_t mem_refclk_10khz;
+  uint32_t pstate_uclk_10khz[4];
+  uint16_t umcgoldenoffset;
+  uint16_t densitygoldenoffset;
+  uint32_t reserved[4];
+};
 
 /* 
   ***************************************************************************
diff --git a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
index 5f3c10ebff08..b897aca9b4c9 100644
--- a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
+++ b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
@@ -85,18 +85,6 @@ enum kgd_memory_pool {
 	KGD_POOL_FRAMEBUFFER = 3,
 };
 
-enum kgd_engine_type {
-	KGD_ENGINE_PFP = 1,
-	KGD_ENGINE_ME,
-	KGD_ENGINE_CE,
-	KGD_ENGINE_MEC1,
-	KGD_ENGINE_MEC2,
-	KGD_ENGINE_RLC,
-	KGD_ENGINE_SDMA1,
-	KGD_ENGINE_SDMA2,
-	KGD_ENGINE_MAX
-};
-
 /**
  * enum kfd_sched_policy
  *
@@ -230,8 +218,6 @@ struct tile_config {
  * @hqd_sdma_destroy: Destructs and preempts the SDMA queue assigned to that
  * SDMA hqd slot.
  *
- * @get_fw_version: Returns FW versions from the header
- *
  * @set_scratch_backing_va: Sets VA for scratch backing memory of a VMID.
  * Only used for no cp scheduling mode
  *
@@ -311,8 +297,6 @@ struct kfd2kgd_calls {
 					struct kgd_dev *kgd,
 					uint8_t vmid);
 
-	uint16_t (*get_fw_version)(struct kgd_dev *kgd,
-				enum kgd_engine_type type);
 	void (*set_scratch_backing_va)(struct kgd_dev *kgd,
 				uint64_t va, uint32_t vmid);
 	int (*get_tile_config)(struct kgd_dev *kgd, struct tile_config *config);
diff --git a/drivers/gpu/drm/amd/include/linux/chash.h b/drivers/gpu/drm/amd/include/linux/chash.h
deleted file mode 100644
index 6dc159924ed1..000000000000
--- a/drivers/gpu/drm/amd/include/linux/chash.h
+++ /dev/null
@@ -1,366 +0,0 @@
-/*
- * Copyright 2017 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#ifndef _LINUX_CHASH_H
-#define _LINUX_CHASH_H
-
-#include <linux/types.h>
-#include <linux/hash.h>
-#include <linux/bug.h>
-#include <asm/bitsperlong.h>
-
-#if BITS_PER_LONG == 32
-# define _CHASH_LONG_SHIFT 5
-#elif BITS_PER_LONG == 64
-# define _CHASH_LONG_SHIFT 6
-#else
-# error "Unexpected BITS_PER_LONG"
-#endif
-
-struct __chash_table {
-	u8 bits;
-	u8 key_size;
-	unsigned int value_size;
-	u32 size_mask;
-	unsigned long *occup_bitmap, *valid_bitmap;
-	union {
-		u32 *keys32;
-		u64 *keys64;
-	};
-	u8 *values;
-
-#ifdef CONFIG_CHASH_STATS
-	u64 hits, hits_steps, hits_time_ns;
-	u64 miss, miss_steps, miss_time_ns;
-	u64 relocs, reloc_dist;
-#endif
-};
-
-#define __CHASH_BITMAP_SIZE(bits)				\
-	(((1 << (bits)) + BITS_PER_LONG - 1) / BITS_PER_LONG)
-#define __CHASH_ARRAY_SIZE(bits, size)				\
-	((((size) << (bits)) + sizeof(long) - 1) / sizeof(long))
-
-#define __CHASH_DATA_SIZE(bits, key_size, value_size)	\
-	(__CHASH_BITMAP_SIZE(bits) * 2 +		\
-	 __CHASH_ARRAY_SIZE(bits, key_size) +		\
-	 __CHASH_ARRAY_SIZE(bits, value_size))
-
-#define STRUCT_CHASH_TABLE(bits, key_size, value_size)			\
-	struct {							\
-		struct __chash_table table;				\
-		unsigned long data					\
-			[__CHASH_DATA_SIZE(bits, key_size, value_size)];\
-	}
-
-/**
- * struct chash_table - Dynamically allocated closed hash table
- *
- * Use this struct for dynamically allocated hash tables (using
- * chash_table_alloc and chash_table_free), where the size is
- * determined at runtime.
- */
-struct chash_table {
-	struct __chash_table table;
-	unsigned long *data;
-};
-
-/**
- * DECLARE_CHASH_TABLE - macro to declare a closed hash table
- * @table: name of the declared hash table
- * @bts: Table size will be 2^bits entries
- * @key_sz: Size of hash keys in bytes, 4 or 8
- * @val_sz: Size of data values in bytes, can be 0
- *
- * This declares the hash table variable with a static size.
- *
- * The closed hash table stores key-value pairs with low memory and
- * lookup overhead. In operation it performs no dynamic memory
- * management. The data being stored does not require any
- * list_heads. The hash table performs best with small @val_sz and as
- * long as some space (about 50%) is left free in the table. But the
- * table can still work reasonably efficiently even when filled up to
- * about 90%. If bigger data items need to be stored and looked up,
- * store the pointer to it as value in the hash table.
- *
- * @val_sz may be 0. This can be useful when all the stored
- * information is contained in the key itself and the fact that it is
- * in the hash table (or not).
- */
-#define DECLARE_CHASH_TABLE(table, bts, key_sz, val_sz)		\
-	STRUCT_CHASH_TABLE(bts, key_sz, val_sz) table
-
-#ifdef CONFIG_CHASH_STATS
-#define __CHASH_STATS_INIT(prefix),		\
-		prefix.hits = 0,		\
-		prefix.hits_steps = 0,		\
-		prefix.hits_time_ns = 0,	\
-		prefix.miss = 0,		\
-		prefix.miss_steps = 0,		\
-		prefix.miss_time_ns = 0,	\
-		prefix.relocs = 0,		\
-		prefix.reloc_dist = 0
-#else
-#define __CHASH_STATS_INIT(prefix)
-#endif
-
-#define __CHASH_TABLE_INIT(prefix, data, bts, key_sz, val_sz)	\
-	prefix.bits = (bts),					\
-		prefix.key_size = (key_sz),			\
-		prefix.value_size = (val_sz),			\
-		prefix.size_mask = ((1 << bts) - 1),		\
-		prefix.occup_bitmap = &data[0],			\
-		prefix.valid_bitmap = &data			\
-			[__CHASH_BITMAP_SIZE(bts)],		\
-		prefix.keys64 = (u64 *)&data			\
-			[__CHASH_BITMAP_SIZE(bts) * 2],		\
-		prefix.values = (u8 *)&data			\
-			[__CHASH_BITMAP_SIZE(bts) * 2 +		\
-			 __CHASH_ARRAY_SIZE(bts, key_sz)]	\
-		__CHASH_STATS_INIT(prefix)
-
-/**
- * DEFINE_CHASH_TABLE - macro to define and initialize a closed hash table
- * @tbl: name of the declared hash table
- * @bts: Table size will be 2^bits entries
- * @key_sz: Size of hash keys in bytes, 4 or 8
- * @val_sz: Size of data values in bytes, can be 0
- *
- * Note: the macro can be used for global and local hash table variables.
- */
-#define DEFINE_CHASH_TABLE(tbl, bts, key_sz, val_sz)			\
-	DECLARE_CHASH_TABLE(tbl, bts, key_sz, val_sz) = {		\
-		.table = {						\
-			__CHASH_TABLE_INIT(, (tbl).data, bts, key_sz, val_sz) \
-		},							\
-		.data = {0}						\
-	}
-
-/**
- * INIT_CHASH_TABLE - Initialize a hash table declared by DECLARE_CHASH_TABLE
- * @tbl: name of the declared hash table
- * @bts: Table size will be 2^bits entries
- * @key_sz: Size of hash keys in bytes, 4 or 8
- * @val_sz: Size of data values in bytes, can be 0
- */
-#define INIT_CHASH_TABLE(tbl, bts, key_sz, val_sz)			\
-	__CHASH_TABLE_INIT(((tbl).table), (tbl).data, bts, key_sz, val_sz)
-
-int chash_table_alloc(struct chash_table *table, u8 bits, u8 key_size,
-		      unsigned int value_size, gfp_t gfp_mask);
-void chash_table_free(struct chash_table *table);
-
-/**
- * chash_table_dump_stats - Dump statistics of a closed hash table
- * @tbl: Pointer to the table structure
- *
- * Dumps some performance statistics of the table gathered in operation
- * in the kernel log using pr_debug. If CONFIG_DYNAMIC_DEBUG is enabled,
- * user must turn on messages for chash.c (file chash.c +p).
- */
-#ifdef CONFIG_CHASH_STATS
-#define chash_table_dump_stats(tbl) __chash_table_dump_stats(&(*tbl).table)
-
-void __chash_table_dump_stats(struct __chash_table *table);
-#else
-#define chash_table_dump_stats(tbl)
-#endif
-
-/**
- * chash_table_reset_stats - Reset statistics of a closed hash table
- * @tbl: Pointer to the table structure
- */
-#ifdef CONFIG_CHASH_STATS
-#define chash_table_reset_stats(tbl) __chash_table_reset_stats(&(*tbl).table)
-
-static inline void __chash_table_reset_stats(struct __chash_table *table)
-{
-	(void)table __CHASH_STATS_INIT((*table));
-}
-#else
-#define chash_table_reset_stats(tbl)
-#endif
-
-/**
- * chash_table_copy_in - Copy a new value into the hash table
- * @tbl: Pointer to the table structure
- * @key: Key of the entry to add or update
- * @value: Pointer to value to copy, may be NULL
- *
- * If @key already has an entry, its value is replaced. Otherwise a
- * new entry is added. If @value is NULL, the value is left unchanged
- * or uninitialized. Returns 1 if an entry already existed, 0 if a new
- * entry was added or %-ENOMEM if there was no free space in the
- * table.
- */
-#define chash_table_copy_in(tbl, key, value)			\
-	__chash_table_copy_in(&(*tbl).table, key, value)
-
-int __chash_table_copy_in(struct __chash_table *table, u64 key,
-			  const void *value);
-
-/**
- * chash_table_copy_out - Copy a value out of the hash table
- * @tbl: Pointer to the table structure
- * @key: Key of the entry to find
- * @value: Pointer to value to copy, may be NULL
- *
- * If @value is not NULL and the table has a non-0 value_size, the
- * value at @key is copied to @value. Returns the slot index of the
- * entry or %-EINVAL if @key was not found.
- */
-#define chash_table_copy_out(tbl, key, value)			\
-	__chash_table_copy_out(&(*tbl).table, key, value, false)
-
-int __chash_table_copy_out(struct __chash_table *table, u64 key,
-			   void *value, bool remove);
-
-/**
- * chash_table_remove - Remove an entry from the hash table
- * @tbl: Pointer to the table structure
- * @key: Key of the entry to find
- * @value: Pointer to value to copy, may be NULL
- *
- * If @value is not NULL and the table has a non-0 value_size, the
- * value at @key is copied to @value. The entry is removed from the
- * table. Returns the slot index of the removed entry or %-EINVAL if
- * @key was not found.
- */
-#define chash_table_remove(tbl, key, value)			\
-	__chash_table_copy_out(&(*tbl).table, key, value, true)
-
-/*
- * Low level iterator API used internally by the above functions.
- */
-struct chash_iter {
-	struct __chash_table *table;
-	unsigned long mask;
-	int slot;
-};
-
-/**
- * CHASH_ITER_INIT - Initialize a hash table iterator
- * @tbl: Pointer to hash table to iterate over
- * @s: Initial slot number
- */
-#define CHASH_ITER_INIT(table, s) {			\
-		table,					\
-		1UL << ((s) & (BITS_PER_LONG - 1)),	\
-		s					\
-	}
-/**
- * CHASH_ITER_SET - Set hash table iterator to new slot
- * @iter: Iterator
- * @s: Slot number
- */
-#define CHASH_ITER_SET(iter, s)					\
-	(iter).mask = 1UL << ((s) & (BITS_PER_LONG - 1)),	\
-	(iter).slot = (s)
-/**
- * CHASH_ITER_INC - Increment hash table iterator
- * @table: Hash table to iterate over
- *
- * Wraps around at the end.
- */
-#define CHASH_ITER_INC(iter) do {					\
-		(iter).mask = (iter).mask << 1 |			\
-			(iter).mask >> (BITS_PER_LONG - 1);		\
-		(iter).slot = ((iter).slot + 1) & (iter).table->size_mask; \
-	} while (0)
-
-static inline bool chash_iter_is_valid(const struct chash_iter iter)
-{
-	BUG_ON((unsigned)iter.slot >= (1 << iter.table->bits));
-	return !!(iter.table->valid_bitmap[iter.slot >> _CHASH_LONG_SHIFT] &
-		  iter.mask);
-}
-static inline bool chash_iter_is_empty(const struct chash_iter iter)
-{
-	BUG_ON((unsigned)iter.slot >= (1 << iter.table->bits));
-	return !(iter.table->occup_bitmap[iter.slot >> _CHASH_LONG_SHIFT] &
-		 iter.mask);
-}
-
-static inline void chash_iter_set_valid(const struct chash_iter iter)
-{
-	BUG_ON((unsigned)iter.slot >= (1 << iter.table->bits));
-	iter.table->valid_bitmap[iter.slot >> _CHASH_LONG_SHIFT] |= iter.mask;
-	iter.table->occup_bitmap[iter.slot >> _CHASH_LONG_SHIFT] |= iter.mask;
-}
-static inline void chash_iter_set_invalid(const struct chash_iter iter)
-{
-	BUG_ON((unsigned)iter.slot >= (1 << iter.table->bits));
-	iter.table->valid_bitmap[iter.slot >> _CHASH_LONG_SHIFT] &= ~iter.mask;
-}
-static inline void chash_iter_set_empty(const struct chash_iter iter)
-{
-	BUG_ON((unsigned)iter.slot >= (1 << iter.table->bits));
-	iter.table->occup_bitmap[iter.slot >> _CHASH_LONG_SHIFT] &= ~iter.mask;
-}
-
-static inline u32 chash_iter_key32(const struct chash_iter iter)
-{
-	BUG_ON(iter.table->key_size != 4);
-	BUG_ON((unsigned)iter.slot >= (1 << iter.table->bits));
-	return iter.table->keys32[iter.slot];
-}
-static inline u64 chash_iter_key64(const struct chash_iter iter)
-{
-	BUG_ON(iter.table->key_size != 8);
-	BUG_ON((unsigned)iter.slot >= (1 << iter.table->bits));
-	return iter.table->keys64[iter.slot];
-}
-static inline u64 chash_iter_key(const struct chash_iter iter)
-{
-	BUG_ON((unsigned)iter.slot >= (1 << iter.table->bits));
-	return (iter.table->key_size == 4) ?
-		iter.table->keys32[iter.slot] : iter.table->keys64[iter.slot];
-}
-
-static inline u32 chash_iter_hash32(const struct chash_iter iter)
-{
-	BUG_ON(iter.table->key_size != 4);
-	return hash_32(chash_iter_key32(iter), iter.table->bits);
-}
-
-static inline u32 chash_iter_hash64(const struct chash_iter iter)
-{
-	BUG_ON(iter.table->key_size != 8);
-	return hash_64(chash_iter_key64(iter), iter.table->bits);
-}
-
-static inline u32 chash_iter_hash(const struct chash_iter iter)
-{
-	return (iter.table->key_size == 4) ?
-		hash_32(chash_iter_key32(iter), iter.table->bits) :
-		hash_64(chash_iter_key64(iter), iter.table->bits);
-}
-
-static inline void *chash_iter_value(const struct chash_iter iter)
-{
-	BUG_ON((unsigned)iter.slot >= (1 << iter.table->bits));
-	return iter.table->values +
-		((unsigned long)iter.slot * iter.table->value_size);
-}
-
-#endif /* _LINUX_CHASH_H */
diff --git a/drivers/gpu/drm/amd/lib/Kconfig b/drivers/gpu/drm/amd/lib/Kconfig
deleted file mode 100644
index 776ef3434c10..000000000000
--- a/drivers/gpu/drm/amd/lib/Kconfig
+++ /dev/null
@@ -1,28 +0,0 @@
-menu "AMD Library routines"
-
-#
-# Closed hash table
-#
-config CHASH
-	tristate
-	default DRM_AMDGPU
-	help
-	 Statically sized closed hash table implementation with low
-	 memory and CPU overhead.
-
-config CHASH_STATS
-	bool "Closed hash table performance statistics"
-	depends on CHASH
-	default n
-	help
-	 Enable collection of performance statistics for closed hash tables.
-
-config CHASH_SELFTEST
-	bool "Closed hash table self test"
-	depends on CHASH
-	default n
-	help
-	 Runs a selftest during module load. Several module parameters
-	 are available to modify the behaviour of the test.
-
-endmenu
diff --git a/drivers/gpu/drm/amd/lib/Makefile b/drivers/gpu/drm/amd/lib/Makefile
deleted file mode 100644
index 690243001e1a..000000000000
--- a/drivers/gpu/drm/amd/lib/Makefile
+++ /dev/null
@@ -1,32 +0,0 @@
-#
-# Copyright 2017 Advanced Micro Devices, Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining a
-# copy of this software and associated documentation files (the "Software"),
-# to deal in the Software without restriction, including without limitation
-# the rights to use, copy, modify, merge, publish, distribute, sublicense,
-# and/or sell copies of the Software, and to permit persons to whom the
-# Software is furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
-# THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
-# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
-# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-# OTHER DEALINGS IN THE SOFTWARE.
-#
-#
-# Makefile for AMD library routines, which are used by AMD driver
-# components.
-#
-# This is for common library routines that can be shared between AMD
-# driver components or later moved to kernel/lib for sharing with
-# other drivers.
-
-ccflags-y := -I$(src)/../include
-
-obj-$(CONFIG_CHASH) += chash.o
diff --git a/drivers/gpu/drm/amd/lib/chash.c b/drivers/gpu/drm/amd/lib/chash.c
deleted file mode 100644
index b8e45f356a1c..000000000000
--- a/drivers/gpu/drm/amd/lib/chash.c
+++ /dev/null
@@ -1,638 +0,0 @@
-/*
- * Copyright 2017 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#include <linux/types.h>
-#include <linux/hash.h>
-#include <linux/bug.h>
-#include <linux/slab.h>
-#include <linux/module.h>
-#include <linux/sched/clock.h>
-#include <asm/div64.h>
-#include <linux/chash.h>
-
-/**
- * chash_table_alloc - Allocate closed hash table
- * @table: Pointer to the table structure
- * @bits: Table size will be 2^bits entries
- * @key_size: Size of hash keys in bytes, 4 or 8
- * @value_size: Size of data values in bytes, can be 0
- */
-int chash_table_alloc(struct chash_table *table, u8 bits, u8 key_size,
-		      unsigned int value_size, gfp_t gfp_mask)
-{
-	if (bits > 31)
-		return -EINVAL;
-
-	if (key_size != 4 && key_size != 8)
-		return -EINVAL;
-
-	table->data = kcalloc(__CHASH_DATA_SIZE(bits, key_size, value_size),
-		       sizeof(long), gfp_mask);
-	if (!table->data)
-		return -ENOMEM;
-
-	__CHASH_TABLE_INIT(table->table, table->data,
-			   bits, key_size, value_size);
-
-	return 0;
-}
-EXPORT_SYMBOL(chash_table_alloc);
-
-/**
- * chash_table_free - Free closed hash table
- * @table: Pointer to the table structure
- */
-void chash_table_free(struct chash_table *table)
-{
-	kfree(table->data);
-}
-EXPORT_SYMBOL(chash_table_free);
-
-#ifdef CONFIG_CHASH_STATS
-
-#define DIV_FRAC(nom, denom, quot, frac, frac_digits) do {		\
-		u64 __nom = (nom);					\
-		u64 __denom = (denom);					\
-		u64 __quot, __frac;					\
-		u32 __rem;						\
-									\
-		while (__denom >> 32) {					\
-			__nom   >>= 1;					\
-			__denom >>= 1;					\
-		}							\
-		__quot = __nom;						\
-		__rem  = do_div(__quot, __denom);			\
-		__frac = __rem * (frac_digits) + (__denom >> 1);	\
-		do_div(__frac, __denom);				\
-		(quot) = __quot;					\
-		(frac) = __frac;					\
-	} while (0)
-
-void __chash_table_dump_stats(struct __chash_table *table)
-{
-	struct chash_iter iter = CHASH_ITER_INIT(table, 0);
-	u32 filled = 0, empty = 0, tombstones = 0;
-	u64 quot1, quot2;
-	u32 frac1, frac2;
-
-	do {
-		if (chash_iter_is_valid(iter))
-			filled++;
-		else if (chash_iter_is_empty(iter))
-			empty++;
-		else
-			tombstones++;
-		CHASH_ITER_INC(iter);
-	} while (iter.slot);
-
-	pr_debug("chash: key size %u, value size %u\n",
-		 table->key_size, table->value_size);
-	pr_debug("  Slots total/filled/empty/tombstones: %u / %u / %u / %u\n",
-		 1 << table->bits, filled, empty, tombstones);
-	if (table->hits > 0) {
-		DIV_FRAC(table->hits_steps, table->hits, quot1, frac1, 1000);
-		DIV_FRAC(table->hits * 1000, table->hits_time_ns,
-			 quot2, frac2, 1000);
-	} else {
-		quot1 = quot2 = 0;
-		frac1 = frac2 = 0;
-	}
-	pr_debug("  Hits   (avg.cost, rate): %llu (%llu.%03u, %llu.%03u M/s)\n",
-		 table->hits, quot1, frac1, quot2, frac2);
-	if (table->miss > 0) {
-		DIV_FRAC(table->miss_steps, table->miss, quot1, frac1, 1000);
-		DIV_FRAC(table->miss * 1000, table->miss_time_ns,
-			 quot2, frac2, 1000);
-	} else {
-		quot1 = quot2 = 0;
-		frac1 = frac2 = 0;
-	}
-	pr_debug("  Misses (avg.cost, rate): %llu (%llu.%03u, %llu.%03u M/s)\n",
-		 table->miss, quot1, frac1, quot2, frac2);
-	if (table->hits + table->miss > 0) {
-		DIV_FRAC(table->hits_steps + table->miss_steps,
-			 table->hits + table->miss, quot1, frac1, 1000);
-		DIV_FRAC((table->hits + table->miss) * 1000,
-			 (table->hits_time_ns + table->miss_time_ns),
-			 quot2, frac2, 1000);
-	} else {
-		quot1 = quot2 = 0;
-		frac1 = frac2 = 0;
-	}
-	pr_debug("  Total  (avg.cost, rate): %llu (%llu.%03u, %llu.%03u M/s)\n",
-		 table->hits + table->miss, quot1, frac1, quot2, frac2);
-	if (table->relocs > 0) {
-		DIV_FRAC(table->hits + table->miss, table->relocs,
-			 quot1, frac1, 1000);
-		DIV_FRAC(table->reloc_dist, table->relocs, quot2, frac2, 1000);
-		pr_debug("  Relocations (freq, avg.dist): %llu (1:%llu.%03u, %llu.%03u)\n",
-			 table->relocs, quot1, frac1, quot2, frac2);
-	} else {
-		pr_debug("  No relocations\n");
-	}
-}
-EXPORT_SYMBOL(__chash_table_dump_stats);
-
-#undef DIV_FRAC
-#endif
-
-#define CHASH_INC(table, a) ((a) = ((a) + 1) & (table)->size_mask)
-#define CHASH_ADD(table, a, b) (((a) + (b)) & (table)->size_mask)
-#define CHASH_SUB(table, a, b) (((a) - (b)) & (table)->size_mask)
-#define CHASH_IN_RANGE(table, slot, first, last) \
-	(CHASH_SUB(table, slot, first) <= CHASH_SUB(table, last, first))
-
-/*#define CHASH_DEBUG Uncomment this to enable verbose debug output*/
-#ifdef CHASH_DEBUG
-static void chash_table_dump(struct __chash_table *table)
-{
-	struct chash_iter iter = CHASH_ITER_INIT(table, 0);
-
-	do {
-		if ((iter.slot & 3) == 0)
-			pr_debug("%04x: ", iter.slot);
-
-		if (chash_iter_is_valid(iter))
-			pr_debug("[%016llx] ", chash_iter_key(iter));
-		else if (chash_iter_is_empty(iter))
-			pr_debug("[    <empty>     ] ");
-		else
-			pr_debug("[  <tombstone>   ] ");
-
-		if ((iter.slot & 3) == 3)
-			pr_debug("\n");
-
-		CHASH_ITER_INC(iter);
-	} while (iter.slot);
-
-	if ((iter.slot & 3) != 0)
-		pr_debug("\n");
-}
-
-static int chash_table_check(struct __chash_table *table)
-{
-	u32 hash;
-	struct chash_iter iter = CHASH_ITER_INIT(table, 0);
-	struct chash_iter cur = CHASH_ITER_INIT(table, 0);
-
-	do {
-		if (!chash_iter_is_valid(iter)) {
-			CHASH_ITER_INC(iter);
-			continue;
-		}
-
-		hash = chash_iter_hash(iter);
-		CHASH_ITER_SET(cur, hash);
-		while (cur.slot != iter.slot) {
-			if (chash_iter_is_empty(cur)) {
-				pr_err("Path to element at %x with hash %x broken at slot %x\n",
-				       iter.slot, hash, cur.slot);
-				chash_table_dump(table);
-				return -EINVAL;
-			}
-			CHASH_ITER_INC(cur);
-		}
-
-		CHASH_ITER_INC(iter);
-	} while (iter.slot);
-
-	return 0;
-}
-#endif
-
-static void chash_iter_relocate(struct chash_iter dst, struct chash_iter src)
-{
-	BUG_ON(src.table == dst.table && src.slot == dst.slot);
-	BUG_ON(src.table->key_size != dst.table->key_size);
-	BUG_ON(src.table->value_size != dst.table->value_size);
-
-	if (dst.table->key_size == 4)
-		dst.table->keys32[dst.slot] = src.table->keys32[src.slot];
-	else
-		dst.table->keys64[dst.slot] = src.table->keys64[src.slot];
-
-	if (dst.table->value_size)
-		memcpy(chash_iter_value(dst), chash_iter_value(src),
-		       dst.table->value_size);
-
-	chash_iter_set_valid(dst);
-	chash_iter_set_invalid(src);
-
-#ifdef CONFIG_CHASH_STATS
-	if (src.table == dst.table) {
-		dst.table->relocs++;
-		dst.table->reloc_dist +=
-			CHASH_SUB(dst.table, src.slot, dst.slot);
-	}
-#endif
-}
-
-/**
- * __chash_table_find - Helper for looking up a hash table entry
- * @iter: Pointer to hash table iterator
- * @key: Key of the entry to find
- * @for_removal: set to true if the element will be removed soon
- *
- * Searches for an entry in the hash table with a given key. iter must
- * be initialized by the caller to point to the home position of the
- * hypothetical entry, i.e. it must be initialized with the hash table
- * and the key's hash as the initial slot for the search.
- *
- * This function also does some local clean-up to speed up future
- * look-ups by relocating entries to better slots and removing
- * tombstones that are no longer needed.
- *
- * If @for_removal is true, the function avoids relocating the entry
- * that is being returned.
- *
- * Returns 0 if the search is successful. In this case iter is updated
- * to point to the found entry. Otherwise %-EINVAL is returned and the
- * iter is updated to point to the first available slot for the given
- * key. If the table is full, the slot is set to -1.
- */
-static int chash_table_find(struct chash_iter *iter, u64 key,
-			    bool for_removal)
-{
-#ifdef CONFIG_CHASH_STATS
-	u64 ts1 = local_clock();
-#endif
-	u32 hash = iter->slot;
-	struct chash_iter first_redundant = CHASH_ITER_INIT(iter->table, -1);
-	int first_avail = (for_removal ? -2 : -1);
-
-	while (!chash_iter_is_valid(*iter) || chash_iter_key(*iter) != key) {
-		if (chash_iter_is_empty(*iter)) {
-			/* Found an empty slot, which ends the
-			 * search. Clean up any preceding tombstones
-			 * that are no longer needed because they lead
-			 * to no-where
-			 */
-			if ((int)first_redundant.slot < 0)
-				goto not_found;
-			while (first_redundant.slot != iter->slot) {
-				if (!chash_iter_is_valid(first_redundant))
-					chash_iter_set_empty(first_redundant);
-				CHASH_ITER_INC(first_redundant);
-			}
-#ifdef CHASH_DEBUG
-			chash_table_check(iter->table);
-#endif
-			goto not_found;
-		} else if (!chash_iter_is_valid(*iter)) {
-			/* Found a tombstone. Remember it as candidate
-			 * for relocating the entry we're looking for
-			 * or for adding a new entry with the given key
-			 */
-			if (first_avail == -1)
-				first_avail = iter->slot;
-			/* Or mark it as the start of a series of
-			 * potentially redundant tombstones
-			 */
-			else if (first_redundant.slot == -1)
-				CHASH_ITER_SET(first_redundant, iter->slot);
-		} else if (first_redundant.slot >= 0) {
-			/* Found a valid, occupied slot with a
-			 * preceding series of tombstones. Relocate it
-			 * to a better position that no longer depends
-			 * on those tombstones
-			 */
-			u32 cur_hash = chash_iter_hash(*iter);
-
-			if (!CHASH_IN_RANGE(iter->table, cur_hash,
-					    first_redundant.slot + 1,
-					    iter->slot)) {
-				/* This entry has a hash at or before
-				 * the first tombstone we found. We
-				 * can relocate it to that tombstone
-				 * and advance to the next tombstone
-				 */
-				chash_iter_relocate(first_redundant, *iter);
-				do {
-					CHASH_ITER_INC(first_redundant);
-				} while (chash_iter_is_valid(first_redundant));
-			} else if (cur_hash != iter->slot) {
-				/* Relocate entry to its home position
-				 * or as close as possible so it no
-				 * longer depends on any preceding
-				 * tombstones
-				 */
-				struct chash_iter new_iter =
-					CHASH_ITER_INIT(iter->table, cur_hash);
-
-				while (new_iter.slot != iter->slot &&
-				       chash_iter_is_valid(new_iter))
-					CHASH_ITER_INC(new_iter);
-
-				if (new_iter.slot != iter->slot)
-					chash_iter_relocate(new_iter, *iter);
-			}
-		}
-
-		CHASH_ITER_INC(*iter);
-		if (iter->slot == hash) {
-			iter->slot = -1;
-			goto not_found;
-		}
-	}
-
-#ifdef CONFIG_CHASH_STATS
-	iter->table->hits++;
-	iter->table->hits_steps += CHASH_SUB(iter->table, iter->slot, hash) + 1;
-#endif
-
-	if (first_avail >= 0) {
-		CHASH_ITER_SET(first_redundant, first_avail);
-		chash_iter_relocate(first_redundant, *iter);
-		iter->slot = first_redundant.slot;
-		iter->mask = first_redundant.mask;
-	}
-
-#ifdef CONFIG_CHASH_STATS
-	iter->table->hits_time_ns += local_clock() - ts1;
-#endif
-
-	return 0;
-
-not_found:
-#ifdef CONFIG_CHASH_STATS
-	iter->table->miss++;
-	iter->table->miss_steps += (iter->slot < 0) ?
-		(1 << iter->table->bits) :
-		CHASH_SUB(iter->table, iter->slot, hash) + 1;
-#endif
-
-	if (first_avail >= 0)
-		CHASH_ITER_SET(*iter, first_avail);
-
-#ifdef CONFIG_CHASH_STATS
-	iter->table->miss_time_ns += local_clock() - ts1;
-#endif
-
-	return -EINVAL;
-}
-
-int __chash_table_copy_in(struct __chash_table *table, u64 key,
-			  const void *value)
-{
-	u32 hash = (table->key_size == 4) ?
-		hash_32(key, table->bits) : hash_64(key, table->bits);
-	struct chash_iter iter = CHASH_ITER_INIT(table, hash);
-	int r = chash_table_find(&iter, key, false);
-
-	/* Found an existing entry */
-	if (!r) {
-		if (value && table->value_size)
-			memcpy(chash_iter_value(iter), value,
-			       table->value_size);
-		return 1;
-	}
-
-	/* Is there a place to add a new entry? */
-	if (iter.slot < 0) {
-		pr_err("Hash table overflow\n");
-		return -ENOMEM;
-	}
-
-	chash_iter_set_valid(iter);
-
-	if (table->key_size == 4)
-		table->keys32[iter.slot] = key;
-	else
-		table->keys64[iter.slot] = key;
-	if (value && table->value_size)
-		memcpy(chash_iter_value(iter), value, table->value_size);
-
-	return 0;
-}
-EXPORT_SYMBOL(__chash_table_copy_in);
-
-int __chash_table_copy_out(struct __chash_table *table, u64 key,
-			   void *value, bool remove)
-{
-	u32 hash = (table->key_size == 4) ?
-		hash_32(key, table->bits) : hash_64(key, table->bits);
-	struct chash_iter iter = CHASH_ITER_INIT(table, hash);
-	int r = chash_table_find(&iter, key, remove);
-
-	if (r < 0)
-		return r;
-
-	if (value && table->value_size)
-		memcpy(value, chash_iter_value(iter), table->value_size);
-
-	if (remove)
-		chash_iter_set_invalid(iter);
-
-	return iter.slot;
-}
-EXPORT_SYMBOL(__chash_table_copy_out);
-
-#ifdef CONFIG_CHASH_SELFTEST
-/**
- * chash_self_test - Run a self-test of the hash table implementation
- * @bits: Table size will be 2^bits entries
- * @key_size: Size of hash keys in bytes, 4 or 8
- * @min_fill: Minimum fill level during the test
- * @max_fill: Maximum fill level during the test
- * @iterations: Number of test iterations
- *
- * The test adds and removes entries from a hash table, cycling the
- * fill level between min_fill and max_fill entries. Also tests lookup
- * and value retrieval.
- */
-static int __init chash_self_test(u8 bits, u8 key_size,
-				  int min_fill, int max_fill,
-				  u64 iterations)
-{
-	struct chash_table table;
-	int ret;
-	u64 add_count, rmv_count;
-	u64 value;
-
-	if (key_size == 4 && iterations > 0xffffffff)
-		return -EINVAL;
-	if (min_fill >= max_fill)
-		return -EINVAL;
-
-	ret = chash_table_alloc(&table, bits, key_size, sizeof(u64),
-				GFP_KERNEL);
-	if (ret) {
-		pr_err("chash_table_alloc failed: %d\n", ret);
-		return ret;
-	}
-
-	for (add_count = 0, rmv_count = 0; add_count < iterations;
-	     add_count++) {
-		/* When we hit the max_fill level, remove entries down
-		 * to min_fill
-		 */
-		if (add_count - rmv_count == max_fill) {
-			u64 find_count = rmv_count;
-
-			/* First try to find all entries that we're
-			 * about to remove, confirm their value, test
-			 * writing them back a second time.
-			 */
-			for (; add_count - find_count > min_fill;
-			     find_count++) {
-				ret = chash_table_copy_out(&table, find_count,
-							   &value);
-				if (ret < 0) {
-					pr_err("chash_table_copy_out failed: %d\n",
-					       ret);
-					goto out;
-				}
-				if (value != ~find_count) {
-					pr_err("Wrong value retrieved for key 0x%llx, expected 0x%llx got 0x%llx\n",
-					       find_count, ~find_count, value);
-#ifdef CHASH_DEBUG
-					chash_table_dump(&table.table);
-#endif
-					ret = -EFAULT;
-					goto out;
-				}
-				ret = chash_table_copy_in(&table, find_count,
-							  &value);
-				if (ret != 1) {
-					pr_err("copy_in second time returned %d, expected 1\n",
-					       ret);
-					ret = -EFAULT;
-					goto out;
-				}
-			}
-			/* Remove them until we hit min_fill level */
-			for (; add_count - rmv_count > min_fill; rmv_count++) {
-				ret = chash_table_remove(&table, rmv_count,
-							 NULL);
-				if (ret < 0) {
-					pr_err("chash_table_remove failed: %d\n",
-					       ret);
-					goto out;
-				}
-			}
-		}
-
-		/* Add a new value */
-		value = ~add_count;
-		ret = chash_table_copy_in(&table, add_count, &value);
-		if (ret != 0) {
-			pr_err("copy_in first time returned %d, expected 0\n",
-			       ret);
-			ret = -EFAULT;
-			goto out;
-		}
-	}
-
-	chash_table_dump_stats(&table);
-	chash_table_reset_stats(&table);
-
-out:
-	chash_table_free(&table);
-	return ret;
-}
-
-static unsigned int chash_test_bits = 10;
-MODULE_PARM_DESC(test_bits,
-		 "Selftest number of hash bits ([4..20], default=10)");
-module_param_named(test_bits, chash_test_bits, uint, 0444);
-
-static unsigned int chash_test_keysize = 8;
-MODULE_PARM_DESC(test_keysize, "Selftest keysize (4 or 8, default=8)");
-module_param_named(test_keysize, chash_test_keysize, uint, 0444);
-
-static unsigned int chash_test_minfill;
-MODULE_PARM_DESC(test_minfill, "Selftest minimum #entries (default=50%)");
-module_param_named(test_minfill, chash_test_minfill, uint, 0444);
-
-static unsigned int chash_test_maxfill;
-MODULE_PARM_DESC(test_maxfill, "Selftest maximum #entries (default=80%)");
-module_param_named(test_maxfill, chash_test_maxfill, uint, 0444);
-
-static unsigned long chash_test_iters;
-MODULE_PARM_DESC(test_iters, "Selftest iterations (default=1000 x #entries)");
-module_param_named(test_iters, chash_test_iters, ulong, 0444);
-
-static int __init chash_init(void)
-{
-	int ret;
-	u64 ts1_ns;
-
-	/* Skip self test on user errors */
-	if (chash_test_bits < 4 || chash_test_bits > 20) {
-		pr_err("chash: test_bits out of range [4..20].\n");
-		return 0;
-	}
-	if (chash_test_keysize != 4 && chash_test_keysize != 8) {
-		pr_err("chash: test_keysize invalid. Must be 4 or 8.\n");
-		return 0;
-	}
-
-	if (!chash_test_minfill)
-		chash_test_minfill = (1 << chash_test_bits) / 2;
-	if (!chash_test_maxfill)
-		chash_test_maxfill = (1 << chash_test_bits) * 4 / 5;
-	if (!chash_test_iters)
-		chash_test_iters = (1 << chash_test_bits) * 1000;
-
-	if (chash_test_minfill >= (1 << chash_test_bits)) {
-		pr_err("chash: test_minfill too big. Must be < table size.\n");
-		return 0;
-	}
-	if (chash_test_maxfill >= (1 << chash_test_bits)) {
-		pr_err("chash: test_maxfill too big. Must be < table size.\n");
-		return 0;
-	}
-	if (chash_test_minfill >= chash_test_maxfill) {
-		pr_err("chash: test_minfill must be < test_maxfill.\n");
-		return 0;
-	}
-	if (chash_test_keysize == 4 && chash_test_iters > 0xffffffff) {
-		pr_err("chash: test_iters must be < 4G for 4 byte keys.\n");
-		return 0;
-	}
-
-	ts1_ns = local_clock();
-	ret = chash_self_test(chash_test_bits, chash_test_keysize,
-			      chash_test_minfill, chash_test_maxfill,
-			      chash_test_iters);
-	if (!ret) {
-		u64 ts_delta_us = local_clock() - ts1_ns;
-		u64 iters_per_second = (u64)chash_test_iters * 1000000;
-
-		do_div(ts_delta_us, 1000);
-		do_div(iters_per_second, ts_delta_us);
-		pr_info("chash: self test took %llu us, %llu iterations/s\n",
-			ts_delta_us, iters_per_second);
-	} else {
-		pr_err("chash: self test failed: %d\n", ret);
-	}
-
-	return ret;
-}
-
-module_init(chash_init);
-
-#endif /* CONFIG_CHASH_SELFTEST */
-
-MODULE_DESCRIPTION("Closed hash table");
-MODULE_LICENSE("GPL and additional rights");
diff --git a/drivers/gpu/drm/amd/powerplay/Makefile b/drivers/gpu/drm/amd/powerplay/Makefile
index 231785a9e24c..ec87b3430d12 100644
--- a/drivers/gpu/drm/amd/powerplay/Makefile
+++ b/drivers/gpu/drm/amd/powerplay/Makefile
@@ -35,7 +35,7 @@ AMD_POWERPLAY = $(addsuffix /Makefile,$(addprefix $(FULL_AMD_PATH)/powerplay/,$(
 
 include $(AMD_POWERPLAY)
 
-POWER_MGR = amd_powerplay.o
+POWER_MGR = amd_powerplay.o amdgpu_smu.o smu_v11_0.o vega20_ppt.o
 
 AMD_PP_POWER = $(addprefix $(AMD_PP_PATH)/,$(POWER_MGR))
 
diff --git a/drivers/gpu/drm/amd/powerplay/amd_powerplay.c b/drivers/gpu/drm/amd/powerplay/amd_powerplay.c
index 3f73f7cd18b9..bea1587d352d 100644
--- a/drivers/gpu/drm/amd/powerplay/amd_powerplay.c
+++ b/drivers/gpu/drm/amd/powerplay/amd_powerplay.c
@@ -53,7 +53,7 @@ static int amd_powerplay_create(struct amdgpu_device *adev)
 	mutex_init(&hwmgr->smu_lock);
 	hwmgr->chip_family = adev->family;
 	hwmgr->chip_id = adev->asic_type;
-	hwmgr->feature_mask = adev->powerplay.pp_feature;
+	hwmgr->feature_mask = adev->pm.pp_feature;
 	hwmgr->display_config = &adev->pm.pm_display_cfg;
 	adev->powerplay.pp_handle = hwmgr;
 	adev->powerplay.pp_funcs = &pp_dpm_funcs;
@@ -1304,7 +1304,7 @@ static int pp_notify_smu_enable_pwe(void *handle)
 
 	if (hwmgr->hwmgr_func->smus_notify_pwe == NULL) {
 		pr_info_ratelimited("%s was not implemented.\n", __func__);
-		return -EINVAL;;
+		return -EINVAL;
 	}
 
 	mutex_lock(&hwmgr->smu_lock);
@@ -1341,7 +1341,7 @@ static int pp_set_min_deep_sleep_dcefclk(void *handle, uint32_t clock)
 
 	if (hwmgr->hwmgr_func->set_min_deep_sleep_dcefclk == NULL) {
 		pr_debug("%s was not implemented.\n", __func__);
-		return -EINVAL;;
+		return -EINVAL;
 	}
 
 	mutex_lock(&hwmgr->smu_lock);
@@ -1360,7 +1360,7 @@ static int pp_set_hard_min_dcefclk_by_freq(void *handle, uint32_t clock)
 
 	if (hwmgr->hwmgr_func->set_hard_min_dcefclk_by_freq == NULL) {
 		pr_debug("%s was not implemented.\n", __func__);
-		return -EINVAL;;
+		return -EINVAL;
 	}
 
 	mutex_lock(&hwmgr->smu_lock);
@@ -1379,7 +1379,7 @@ static int pp_set_hard_min_fclk_by_freq(void *handle, uint32_t clock)
 
 	if (hwmgr->hwmgr_func->set_hard_min_fclk_by_freq == NULL) {
 		pr_debug("%s was not implemented.\n", __func__);
-		return -EINVAL;;
+		return -EINVAL;
 	}
 
 	mutex_lock(&hwmgr->smu_lock);
diff --git a/drivers/gpu/drm/amd/powerplay/amdgpu_smu.c b/drivers/gpu/drm/amd/powerplay/amdgpu_smu.c
new file mode 100644
index 000000000000..c058c784180e
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/amdgpu_smu.c
@@ -0,0 +1,1253 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "pp_debug.h"
+#include <linux/firmware.h>
+#include <drm/drmP.h>
+#include "amdgpu.h"
+#include "amdgpu_smu.h"
+#include "soc15_common.h"
+#include "smu_v11_0.h"
+#include "atom.h"
+#include "amd_pcie.h"
+
+int smu_dpm_set_power_gate(struct smu_context *smu, uint32_t block_type,
+			   bool gate)
+{
+	int ret = 0;
+
+	switch (block_type) {
+	case AMD_IP_BLOCK_TYPE_UVD:
+		ret = smu_dpm_set_uvd_enable(smu, gate);
+		break;
+	case AMD_IP_BLOCK_TYPE_VCE:
+		ret = smu_dpm_set_vce_enable(smu, gate);
+		break;
+	default:
+		break;
+	}
+
+	return ret;
+}
+
+enum amd_pm_state_type smu_get_current_power_state(struct smu_context *smu)
+{
+	/* not support power state */
+	return POWER_STATE_TYPE_DEFAULT;
+}
+
+int smu_get_power_num_states(struct smu_context *smu,
+			     struct pp_states_info *state_info)
+{
+	if (!state_info)
+		return -EINVAL;
+
+	/* not support power state */
+	memset(state_info, 0, sizeof(struct pp_states_info));
+	state_info->nums = 0;
+
+	return 0;
+}
+
+int smu_common_read_sensor(struct smu_context *smu, enum amd_pp_sensors sensor,
+			   void *data, uint32_t *size)
+{
+	int ret = 0;
+
+	switch (sensor) {
+	case AMDGPU_PP_SENSOR_STABLE_PSTATE_SCLK:
+		*((uint32_t *)data) = smu->pstate_sclk;
+		*size = 4;
+		break;
+	case AMDGPU_PP_SENSOR_STABLE_PSTATE_MCLK:
+		*((uint32_t *)data) = smu->pstate_mclk;
+		*size = 4;
+		break;
+	case AMDGPU_PP_SENSOR_ENABLED_SMC_FEATURES_MASK:
+		ret = smu_feature_get_enabled_mask(smu, (uint32_t *)data, 2);
+		*size = 8;
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	if (ret)
+		*size = 0;
+
+	return ret;
+}
+
+int smu_update_table_with_arg(struct smu_context *smu, uint16_t table_id, uint16_t exarg,
+		     void *table_data, bool drv2smu)
+{
+	struct smu_table_context *smu_table = &smu->smu_table;
+	struct smu_table *table = NULL;
+	int ret = 0;
+	uint32_t table_index;
+
+	if (!table_data || table_id >= smu_table->table_count)
+		return -EINVAL;
+
+	table_index = (exarg << 16) | table_id;
+
+	table = &smu_table->tables[table_id];
+
+	if (drv2smu)
+		memcpy(table->cpu_addr, table_data, table->size);
+
+	ret = smu_send_smc_msg_with_param(smu, SMU_MSG_SetDriverDramAddrHigh,
+					  upper_32_bits(table->mc_address));
+	if (ret)
+		return ret;
+	ret = smu_send_smc_msg_with_param(smu, SMU_MSG_SetDriverDramAddrLow,
+					  lower_32_bits(table->mc_address));
+	if (ret)
+		return ret;
+	ret = smu_send_smc_msg_with_param(smu, drv2smu ?
+					  SMU_MSG_TransferTableDram2Smu :
+					  SMU_MSG_TransferTableSmu2Dram,
+					  table_index);
+	if (ret)
+		return ret;
+
+	if (!drv2smu)
+		memcpy(table_data, table->cpu_addr, table->size);
+
+	return ret;
+}
+
+bool is_support_sw_smu(struct amdgpu_device *adev)
+{
+	if (amdgpu_dpm != 1)
+		return false;
+
+	if (adev->asic_type >= CHIP_VEGA20 && adev->asic_type != CHIP_RAVEN)
+		return true;
+
+	return false;
+}
+
+int smu_sys_get_pp_table(struct smu_context *smu, void **table)
+{
+	struct smu_table_context *smu_table = &smu->smu_table;
+
+	if (!smu_table->power_play_table && !smu_table->hardcode_pptable)
+		return -EINVAL;
+
+	if (smu_table->hardcode_pptable)
+		*table = smu_table->hardcode_pptable;
+	else
+		*table = smu_table->power_play_table;
+
+	return smu_table->power_play_table_size;
+}
+
+int smu_sys_set_pp_table(struct smu_context *smu,  void *buf, size_t size)
+{
+	struct smu_table_context *smu_table = &smu->smu_table;
+	ATOM_COMMON_TABLE_HEADER *header = (ATOM_COMMON_TABLE_HEADER *)buf;
+	int ret = 0;
+
+	if (header->usStructureSize != size) {
+		pr_err("pp table size not matched !\n");
+		return -EIO;
+	}
+
+	mutex_lock(&smu->mutex);
+	if (!smu_table->hardcode_pptable)
+		smu_table->hardcode_pptable = kzalloc(size, GFP_KERNEL);
+	if (!smu_table->hardcode_pptable) {
+		ret = -ENOMEM;
+		goto failed;
+	}
+
+	memcpy(smu_table->hardcode_pptable, buf, size);
+	smu_table->power_play_table = smu_table->hardcode_pptable;
+	smu_table->power_play_table_size = size;
+	mutex_unlock(&smu->mutex);
+
+	ret = smu_reset(smu);
+	if (ret)
+		pr_info("smu reset failed, ret = %d\n", ret);
+
+	return ret;
+
+failed:
+	mutex_unlock(&smu->mutex);
+	return ret;
+}
+
+int smu_feature_init_dpm(struct smu_context *smu)
+{
+	struct smu_feature *feature = &smu->smu_feature;
+	int ret = 0;
+	uint32_t unallowed_feature_mask[SMU_FEATURE_MAX/32];
+
+	mutex_lock(&feature->mutex);
+	bitmap_fill(feature->allowed, SMU_FEATURE_MAX);
+	mutex_unlock(&feature->mutex);
+
+	ret = smu_get_unallowed_feature_mask(smu, unallowed_feature_mask,
+					     SMU_FEATURE_MAX/32);
+	if (ret)
+		return ret;
+
+	mutex_lock(&feature->mutex);
+	bitmap_andnot(feature->allowed, feature->allowed,
+		      (unsigned long *)unallowed_feature_mask,
+		      feature->feature_num);
+	mutex_unlock(&feature->mutex);
+
+	return ret;
+}
+
+int smu_feature_is_enabled(struct smu_context *smu, int feature_id)
+{
+	struct smu_feature *feature = &smu->smu_feature;
+	int ret = 0;
+
+	WARN_ON(feature_id > feature->feature_num);
+
+	mutex_lock(&feature->mutex);
+	ret = test_bit(feature_id, feature->enabled);
+	mutex_unlock(&feature->mutex);
+
+	return ret;
+}
+
+int smu_feature_set_enabled(struct smu_context *smu, int feature_id, bool enable)
+{
+	struct smu_feature *feature = &smu->smu_feature;
+	int ret = 0;
+
+	WARN_ON(feature_id > feature->feature_num);
+
+	mutex_lock(&feature->mutex);
+	ret = smu_feature_update_enable_state(smu, feature_id, enable);
+	if (ret)
+		goto failed;
+
+	if (enable)
+		test_and_set_bit(feature_id, feature->enabled);
+	else
+		test_and_clear_bit(feature_id, feature->enabled);
+
+failed:
+	mutex_unlock(&feature->mutex);
+
+	return ret;
+}
+
+int smu_feature_is_supported(struct smu_context *smu, int feature_id)
+{
+	struct smu_feature *feature = &smu->smu_feature;
+	int ret = 0;
+
+	WARN_ON(feature_id > feature->feature_num);
+
+	mutex_lock(&feature->mutex);
+	ret = test_bit(feature_id, feature->supported);
+	mutex_unlock(&feature->mutex);
+
+	return ret;
+}
+
+int smu_feature_set_supported(struct smu_context *smu, int feature_id,
+			      bool enable)
+{
+	struct smu_feature *feature = &smu->smu_feature;
+	int ret = 0;
+
+	WARN_ON(feature_id > feature->feature_num);
+
+	mutex_unlock(&feature->mutex);
+	if (enable)
+		test_and_set_bit(feature_id, feature->supported);
+	else
+		test_and_clear_bit(feature_id, feature->supported);
+	mutex_unlock(&feature->mutex);
+
+	return ret;
+}
+
+static int smu_set_funcs(struct amdgpu_device *adev)
+{
+	struct smu_context *smu = &adev->smu;
+
+	switch (adev->asic_type) {
+	case CHIP_VEGA20:
+		adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
+		if (adev->pm.pp_feature & PP_OVERDRIVE_MASK)
+			smu->od_enabled = true;
+		smu_v11_0_set_smu_funcs(smu);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int smu_early_init(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	struct smu_context *smu = &adev->smu;
+
+	smu->adev = adev;
+	mutex_init(&smu->mutex);
+
+	return smu_set_funcs(adev);
+}
+
+static int smu_late_init(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	struct smu_context *smu = &adev->smu;
+	mutex_lock(&smu->mutex);
+	smu_handle_task(&adev->smu,
+			smu->smu_dpm.dpm_level,
+			AMD_PP_TASK_COMPLETE_INIT);
+	mutex_unlock(&smu->mutex);
+
+	return 0;
+}
+
+int smu_get_atom_data_table(struct smu_context *smu, uint32_t table,
+			    uint16_t *size, uint8_t *frev, uint8_t *crev,
+			    uint8_t **addr)
+{
+	struct amdgpu_device *adev = smu->adev;
+	uint16_t data_start;
+
+	if (!amdgpu_atom_parse_data_header(adev->mode_info.atom_context, table,
+					   size, frev, crev, &data_start))
+		return -EINVAL;
+
+	*addr = (uint8_t *)adev->mode_info.atom_context->bios + data_start;
+
+	return 0;
+}
+
+static int smu_initialize_pptable(struct smu_context *smu)
+{
+	/* TODO */
+	return 0;
+}
+
+static int smu_smc_table_sw_init(struct smu_context *smu)
+{
+	int ret;
+
+	ret = smu_initialize_pptable(smu);
+	if (ret) {
+		pr_err("Failed to init smu_initialize_pptable!\n");
+		return ret;
+	}
+
+	/**
+	 * Create smu_table structure, and init smc tables such as
+	 * TABLE_PPTABLE, TABLE_WATERMARKS, TABLE_SMU_METRICS, and etc.
+	 */
+	ret = smu_init_smc_tables(smu);
+	if (ret) {
+		pr_err("Failed to init smc tables!\n");
+		return ret;
+	}
+
+	/**
+	 * Create smu_power_context structure, and allocate smu_dpm_context and
+	 * context size to fill the smu_power_context data.
+	 */
+	ret = smu_init_power(smu);
+	if (ret) {
+		pr_err("Failed to init smu_init_power!\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static int smu_smc_table_sw_fini(struct smu_context *smu)
+{
+	int ret;
+
+	ret = smu_fini_smc_tables(smu);
+	if (ret) {
+		pr_err("Failed to smu_fini_smc_tables!\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static int smu_sw_init(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	struct smu_context *smu = &adev->smu;
+	int ret;
+
+	if (!is_support_sw_smu(adev))
+		return -EINVAL;
+
+	smu->pool_size = adev->pm.smu_prv_buffer_size;
+	smu->smu_feature.feature_num = SMU_FEATURE_MAX;
+	mutex_init(&smu->smu_feature.mutex);
+	bitmap_zero(smu->smu_feature.supported, SMU_FEATURE_MAX);
+	bitmap_zero(smu->smu_feature.enabled, SMU_FEATURE_MAX);
+	bitmap_zero(smu->smu_feature.allowed, SMU_FEATURE_MAX);
+	smu->watermarks_bitmap = 0;
+	smu->power_profile_mode = PP_SMC_POWER_PROFILE_BOOTUP_DEFAULT;
+	smu->default_power_profile_mode = PP_SMC_POWER_PROFILE_BOOTUP_DEFAULT;
+
+	smu->workload_mask = 1 << smu->workload_prority[PP_SMC_POWER_PROFILE_BOOTUP_DEFAULT];
+	smu->workload_prority[PP_SMC_POWER_PROFILE_BOOTUP_DEFAULT] = 0;
+	smu->workload_prority[PP_SMC_POWER_PROFILE_FULLSCREEN3D] = 1;
+	smu->workload_prority[PP_SMC_POWER_PROFILE_POWERSAVING] = 2;
+	smu->workload_prority[PP_SMC_POWER_PROFILE_VIDEO] = 3;
+	smu->workload_prority[PP_SMC_POWER_PROFILE_VR] = 4;
+	smu->workload_prority[PP_SMC_POWER_PROFILE_COMPUTE] = 5;
+	smu->workload_prority[PP_SMC_POWER_PROFILE_CUSTOM] = 6;
+
+	smu->workload_setting[0] = PP_SMC_POWER_PROFILE_BOOTUP_DEFAULT;
+	smu->workload_setting[1] = PP_SMC_POWER_PROFILE_FULLSCREEN3D;
+	smu->workload_setting[2] = PP_SMC_POWER_PROFILE_POWERSAVING;
+	smu->workload_setting[3] = PP_SMC_POWER_PROFILE_VIDEO;
+	smu->workload_setting[4] = PP_SMC_POWER_PROFILE_VR;
+	smu->workload_setting[5] = PP_SMC_POWER_PROFILE_COMPUTE;
+	smu->workload_setting[6] = PP_SMC_POWER_PROFILE_CUSTOM;
+	smu->display_config = &adev->pm.pm_display_cfg;
+
+	smu->smu_dpm.dpm_level = AMD_DPM_FORCED_LEVEL_AUTO;
+	smu->smu_dpm.requested_dpm_level = AMD_DPM_FORCED_LEVEL_AUTO;
+	ret = smu_init_microcode(smu);
+	if (ret) {
+		pr_err("Failed to load smu firmware!\n");
+		return ret;
+	}
+
+	ret = smu_smc_table_sw_init(smu);
+	if (ret) {
+		pr_err("Failed to sw init smc table!\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static int smu_sw_fini(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	struct smu_context *smu = &adev->smu;
+	int ret;
+
+	if (!is_support_sw_smu(adev))
+		return -EINVAL;
+
+	ret = smu_smc_table_sw_fini(smu);
+	if (ret) {
+		pr_err("Failed to sw fini smc table!\n");
+		return ret;
+	}
+
+	ret = smu_fini_power(smu);
+	if (ret) {
+		pr_err("Failed to init smu_fini_power!\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static int smu_init_fb_allocations(struct smu_context *smu)
+{
+	struct amdgpu_device *adev = smu->adev;
+	struct smu_table_context *smu_table = &smu->smu_table;
+	struct smu_table *tables = smu_table->tables;
+	uint32_t table_count = smu_table->table_count;
+	uint32_t i = 0;
+	int32_t ret = 0;
+
+	if (table_count <= 0)
+		return -EINVAL;
+
+	for (i = 0 ; i < table_count; i++) {
+		if (tables[i].size == 0)
+			continue;
+		ret = amdgpu_bo_create_kernel(adev,
+					      tables[i].size,
+					      tables[i].align,
+					      tables[i].domain,
+					      &tables[i].bo,
+					      &tables[i].mc_address,
+					      &tables[i].cpu_addr);
+		if (ret)
+			goto failed;
+	}
+
+	return 0;
+failed:
+	for (; i > 0; i--) {
+		if (tables[i].size == 0)
+			continue;
+		amdgpu_bo_free_kernel(&tables[i].bo,
+				      &tables[i].mc_address,
+				      &tables[i].cpu_addr);
+
+	}
+	return ret;
+}
+
+static int smu_fini_fb_allocations(struct smu_context *smu)
+{
+	struct smu_table_context *smu_table = &smu->smu_table;
+	struct smu_table *tables = smu_table->tables;
+	uint32_t table_count = smu_table->table_count;
+	uint32_t i = 0;
+
+	if (table_count == 0 || tables == NULL)
+		return 0;
+
+	for (i = 0 ; i < table_count; i++) {
+		if (tables[i].size == 0)
+			continue;
+		amdgpu_bo_free_kernel(&tables[i].bo,
+				      &tables[i].mc_address,
+				      &tables[i].cpu_addr);
+	}
+
+	return 0;
+}
+
+static int smu_override_pcie_parameters(struct smu_context *smu)
+{
+	struct amdgpu_device *adev = smu->adev;
+	uint32_t pcie_gen = 0, pcie_width = 0, smu_pcie_arg;
+	int ret;
+
+	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4)
+		pcie_gen = 3;
+	else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
+		pcie_gen = 2;
+	else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
+		pcie_gen = 1;
+	else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1)
+		pcie_gen = 0;
+
+	/* Bit 31:16: LCLK DPM level. 0 is DPM0, and 1 is DPM1
+	 * Bit 15:8:  PCIE GEN, 0 to 3 corresponds to GEN1 to GEN4
+	 * Bit 7:0:   PCIE lane width, 1 to 7 corresponds is x1 to x32
+	 */
+	if (adev->pm.pcie_mlw_mask & CAIL_PCIE_LINK_WIDTH_SUPPORT_X16)
+		pcie_width = 6;
+	else if (adev->pm.pcie_mlw_mask & CAIL_PCIE_LINK_WIDTH_SUPPORT_X12)
+		pcie_width = 5;
+	else if (adev->pm.pcie_mlw_mask & CAIL_PCIE_LINK_WIDTH_SUPPORT_X8)
+		pcie_width = 4;
+	else if (adev->pm.pcie_mlw_mask & CAIL_PCIE_LINK_WIDTH_SUPPORT_X4)
+		pcie_width = 3;
+	else if (adev->pm.pcie_mlw_mask & CAIL_PCIE_LINK_WIDTH_SUPPORT_X2)
+		pcie_width = 2;
+	else if (adev->pm.pcie_mlw_mask & CAIL_PCIE_LINK_WIDTH_SUPPORT_X1)
+		pcie_width = 1;
+
+	smu_pcie_arg = (1 << 16) | (pcie_gen << 8) | pcie_width;
+	ret = smu_send_smc_msg_with_param(smu,
+					  SMU_MSG_OverridePcieParameters,
+					  smu_pcie_arg);
+	if (ret)
+		pr_err("[%s] Attempt to override pcie params failed!\n", __func__);
+	return ret;
+}
+
+static int smu_smc_table_hw_init(struct smu_context *smu,
+				 bool initialize)
+{
+	struct amdgpu_device *adev = smu->adev;
+	int ret;
+
+	if (smu_is_dpm_running(smu) && adev->in_suspend) {
+		pr_info("dpm has been enabled\n");
+		return 0;
+	}
+
+	ret = smu_init_display(smu);
+	if (ret)
+		return ret;
+
+	if (initialize) {
+		ret = smu_read_pptable_from_vbios(smu);
+		if (ret)
+			return ret;
+
+		/* get boot_values from vbios to set revision, gfxclk, and etc. */
+		ret = smu_get_vbios_bootup_values(smu);
+		if (ret)
+			return ret;
+
+		ret = smu_get_clk_info_from_vbios(smu);
+		if (ret)
+			return ret;
+
+		/*
+		 * check if the format_revision in vbios is up to pptable header
+		 * version, and the structure size is not 0.
+		 */
+		ret = smu_get_clk_info_from_vbios(smu);
+		if (ret)
+			return ret;
+
+		ret = smu_check_pptable(smu);
+		if (ret)
+			return ret;
+
+		/*
+		 * allocate vram bos to store smc table contents.
+		 */
+		ret = smu_init_fb_allocations(smu);
+		if (ret)
+			return ret;
+
+		/*
+		 * Parse pptable format and fill PPTable_t smc_pptable to
+		 * smu_table_context structure. And read the smc_dpm_table from vbios,
+		 * then fill it into smc_pptable.
+		 */
+		ret = smu_parse_pptable(smu);
+		if (ret)
+			return ret;
+
+		/*
+		 * Send msg GetDriverIfVersion to check if the return value is equal
+		 * with DRIVER_IF_VERSION of smc header.
+		 */
+		ret = smu_check_fw_version(smu);
+		if (ret)
+			return ret;
+	}
+
+	/*
+	 * Copy pptable bo in the vram to smc with SMU MSGs such as
+	 * SetDriverDramAddr and TransferTableDram2Smu.
+	 */
+	ret = smu_write_pptable(smu);
+	if (ret)
+		return ret;
+
+	/* issue RunAfllBtc msg */
+	ret = smu_run_afll_btc(smu);
+	if (ret)
+		return ret;
+
+	ret = smu_feature_set_allowed_mask(smu);
+	if (ret)
+		return ret;
+
+	ret = smu_system_features_control(smu, true);
+	if (ret)
+		return ret;
+
+	ret = smu_override_pcie_parameters(smu);
+	if (ret)
+		return ret;
+
+	ret = smu_notify_display_change(smu);
+	if (ret)
+		return ret;
+
+	/*
+	 * Set min deep sleep dce fclk with bootup value from vbios via
+	 * SetMinDeepSleepDcefclk MSG.
+	 */
+	ret = smu_set_min_dcef_deep_sleep(smu);
+	if (ret)
+		return ret;
+
+	/*
+	 * Set initialized values (get from vbios) to dpm tables context such as
+	 * gfxclk, memclk, dcefclk, and etc. And enable the DPM feature for each
+	 * type of clks.
+	 */
+	if (initialize) {
+		ret = smu_populate_smc_pptable(smu);
+		if (ret)
+			return ret;
+
+		ret = smu_init_max_sustainable_clocks(smu);
+		if (ret)
+			return ret;
+	}
+
+	ret = smu_set_od8_default_settings(smu, initialize);
+	if (ret)
+		return ret;
+
+	if (initialize) {
+		ret = smu_populate_umd_state_clk(smu);
+		if (ret)
+			return ret;
+
+		ret = smu_get_power_limit(smu, &smu->default_power_limit, false);
+		if (ret)
+			return ret;
+	}
+
+	/*
+	 * Set PMSTATUSLOG table bo address with SetToolsDramAddr MSG for tools.
+	 */
+	ret = smu_set_tool_table_location(smu);
+
+	return ret;
+}
+
+/**
+ * smu_alloc_memory_pool - allocate memory pool in the system memory
+ *
+ * @smu: amdgpu_device pointer
+ *
+ * This memory pool will be used for SMC use and msg SetSystemVirtualDramAddr
+ * and DramLogSetDramAddr can notify it changed.
+ *
+ * Returns 0 on success, error on failure.
+ */
+static int smu_alloc_memory_pool(struct smu_context *smu)
+{
+	struct amdgpu_device *adev = smu->adev;
+	struct smu_table_context *smu_table = &smu->smu_table;
+	struct smu_table *memory_pool = &smu_table->memory_pool;
+	uint64_t pool_size = smu->pool_size;
+	int ret = 0;
+
+	if (pool_size == SMU_MEMORY_POOL_SIZE_ZERO)
+		return ret;
+
+	memory_pool->size = pool_size;
+	memory_pool->align = PAGE_SIZE;
+	memory_pool->domain = AMDGPU_GEM_DOMAIN_GTT;
+
+	switch (pool_size) {
+	case SMU_MEMORY_POOL_SIZE_256_MB:
+	case SMU_MEMORY_POOL_SIZE_512_MB:
+	case SMU_MEMORY_POOL_SIZE_1_GB:
+	case SMU_MEMORY_POOL_SIZE_2_GB:
+		ret = amdgpu_bo_create_kernel(adev,
+					      memory_pool->size,
+					      memory_pool->align,
+					      memory_pool->domain,
+					      &memory_pool->bo,
+					      &memory_pool->mc_address,
+					      &memory_pool->cpu_addr);
+		break;
+	default:
+		break;
+	}
+
+	return ret;
+}
+
+static int smu_free_memory_pool(struct smu_context *smu)
+{
+	struct smu_table_context *smu_table = &smu->smu_table;
+	struct smu_table *memory_pool = &smu_table->memory_pool;
+	int ret = 0;
+
+	if (memory_pool->size == SMU_MEMORY_POOL_SIZE_ZERO)
+		return ret;
+
+	amdgpu_bo_free_kernel(&memory_pool->bo,
+			      &memory_pool->mc_address,
+			      &memory_pool->cpu_addr);
+
+	memset(memory_pool, 0, sizeof(struct smu_table));
+
+	return ret;
+}
+
+static int smu_hw_init(void *handle)
+{
+	int ret;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	struct smu_context *smu = &adev->smu;
+
+	if (!is_support_sw_smu(adev))
+		return -EINVAL;
+
+	if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
+		ret = smu_load_microcode(smu);
+		if (ret)
+			return ret;
+	}
+
+	ret = smu_check_fw_status(smu);
+	if (ret) {
+		pr_err("SMC firmware status is not correct\n");
+		return ret;
+	}
+
+	mutex_lock(&smu->mutex);
+
+	ret = smu_feature_init_dpm(smu);
+	if (ret)
+		goto failed;
+
+	ret = smu_smc_table_hw_init(smu, true);
+	if (ret)
+		goto failed;
+
+	ret = smu_alloc_memory_pool(smu);
+	if (ret)
+		goto failed;
+
+	/*
+	 * Use msg SetSystemVirtualDramAddr and DramLogSetDramAddr can notify
+	 * pool location.
+	 */
+	ret = smu_notify_memory_pool_location(smu);
+	if (ret)
+		goto failed;
+
+	ret = smu_start_thermal_control(smu);
+	if (ret)
+		goto failed;
+
+	mutex_unlock(&smu->mutex);
+
+	adev->pm.dpm_enabled = true;
+
+	pr_info("SMU is initialized successfully!\n");
+
+	return 0;
+
+failed:
+	mutex_unlock(&smu->mutex);
+	return ret;
+}
+
+static int smu_hw_fini(void *handle)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	struct smu_context *smu = &adev->smu;
+	struct smu_table_context *table_context = &smu->smu_table;
+	int ret = 0;
+
+	if (!is_support_sw_smu(adev))
+		return -EINVAL;
+
+	kfree(table_context->driver_pptable);
+	table_context->driver_pptable = NULL;
+
+	kfree(table_context->max_sustainable_clocks);
+	table_context->max_sustainable_clocks = NULL;
+
+	kfree(table_context->od_feature_capabilities);
+	table_context->od_feature_capabilities = NULL;
+
+	kfree(table_context->od_settings_max);
+	table_context->od_settings_max = NULL;
+
+	kfree(table_context->od_settings_min);
+	table_context->od_settings_min = NULL;
+
+	kfree(table_context->overdrive_table);
+	table_context->overdrive_table = NULL;
+
+	kfree(table_context->od8_settings);
+	table_context->od8_settings = NULL;
+
+	ret = smu_fini_fb_allocations(smu);
+	if (ret)
+		return ret;
+
+	ret = smu_free_memory_pool(smu);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+int smu_reset(struct smu_context *smu)
+{
+	struct amdgpu_device *adev = smu->adev;
+	int ret = 0;
+
+	ret = smu_hw_fini(adev);
+	if (ret)
+		return ret;
+
+	ret = smu_hw_init(adev);
+	if (ret)
+		return ret;
+
+	return ret;
+}
+
+static int smu_suspend(void *handle)
+{
+	int ret;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	struct smu_context *smu = &adev->smu;
+
+	if (!is_support_sw_smu(adev))
+		return -EINVAL;
+
+	ret = smu_system_features_control(smu, false);
+	if (ret)
+		return ret;
+
+	smu->watermarks_bitmap &= ~(WATERMARKS_LOADED);
+
+	return 0;
+}
+
+static int smu_resume(void *handle)
+{
+	int ret;
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	struct smu_context *smu = &adev->smu;
+
+	if (!is_support_sw_smu(adev))
+		return -EINVAL;
+
+	pr_info("SMU is resuming...\n");
+
+	mutex_lock(&smu->mutex);
+
+	ret = smu_smc_table_hw_init(smu, false);
+	if (ret)
+		goto failed;
+
+	ret = smu_start_thermal_control(smu);
+	if (ret)
+		goto failed;
+
+	mutex_unlock(&smu->mutex);
+
+	pr_info("SMU is resumed successfully!\n");
+
+	return 0;
+failed:
+	mutex_unlock(&smu->mutex);
+	return ret;
+}
+
+int smu_display_configuration_change(struct smu_context *smu,
+				     const struct amd_pp_display_configuration *display_config)
+{
+	int index = 0;
+	int num_of_active_display = 0;
+
+	if (!is_support_sw_smu(smu->adev))
+		return -EINVAL;
+
+	if (!display_config)
+		return -EINVAL;
+
+	mutex_lock(&smu->mutex);
+
+	smu_set_deep_sleep_dcefclk(smu,
+				   display_config->min_dcef_deep_sleep_set_clk / 100);
+
+	for (index = 0; index < display_config->num_path_including_non_display; index++) {
+		if (display_config->displays[index].controller_id != 0)
+			num_of_active_display++;
+	}
+
+	smu_set_active_display_count(smu, num_of_active_display);
+
+	smu_store_cc6_data(smu, display_config->cpu_pstate_separation_time,
+			   display_config->cpu_cc6_disable,
+			   display_config->cpu_pstate_disable,
+			   display_config->nb_pstate_switch_disable);
+
+	mutex_unlock(&smu->mutex);
+
+	return 0;
+}
+
+static int smu_get_clock_info(struct smu_context *smu,
+			      struct smu_clock_info *clk_info,
+			      enum smu_perf_level_designation designation)
+{
+	int ret;
+	struct smu_performance_level level = {0};
+
+	if (!clk_info)
+		return -EINVAL;
+
+	ret = smu_get_perf_level(smu, PERF_LEVEL_ACTIVITY, &level);
+	if (ret)
+		return -EINVAL;
+
+	clk_info->min_mem_clk = level.memory_clock;
+	clk_info->min_eng_clk = level.core_clock;
+	clk_info->min_bus_bandwidth = level.non_local_mem_freq * level.non_local_mem_width;
+
+	ret = smu_get_perf_level(smu, designation, &level);
+	if (ret)
+		return -EINVAL;
+
+	clk_info->min_mem_clk = level.memory_clock;
+	clk_info->min_eng_clk = level.core_clock;
+	clk_info->min_bus_bandwidth = level.non_local_mem_freq * level.non_local_mem_width;
+
+	return 0;
+}
+
+int smu_get_current_clocks(struct smu_context *smu,
+			   struct amd_pp_clock_info *clocks)
+{
+	struct amd_pp_simple_clock_info simple_clocks = {0};
+	struct smu_clock_info hw_clocks;
+	int ret = 0;
+
+	if (!is_support_sw_smu(smu->adev))
+		return -EINVAL;
+
+	mutex_lock(&smu->mutex);
+
+	smu_get_dal_power_level(smu, &simple_clocks);
+
+	if (smu->support_power_containment)
+		ret = smu_get_clock_info(smu, &hw_clocks,
+					 PERF_LEVEL_POWER_CONTAINMENT);
+	else
+		ret = smu_get_clock_info(smu, &hw_clocks, PERF_LEVEL_ACTIVITY);
+
+	if (ret) {
+		pr_err("Error in smu_get_clock_info\n");
+		goto failed;
+	}
+
+	clocks->min_engine_clock = hw_clocks.min_eng_clk;
+	clocks->max_engine_clock = hw_clocks.max_eng_clk;
+	clocks->min_memory_clock = hw_clocks.min_mem_clk;
+	clocks->max_memory_clock = hw_clocks.max_mem_clk;
+	clocks->min_bus_bandwidth = hw_clocks.min_bus_bandwidth;
+	clocks->max_bus_bandwidth = hw_clocks.max_bus_bandwidth;
+	clocks->max_engine_clock_in_sr = hw_clocks.max_eng_clk;
+	clocks->min_engine_clock_in_sr = hw_clocks.min_eng_clk;
+
+        if (simple_clocks.level == 0)
+                clocks->max_clocks_state = PP_DAL_POWERLEVEL_7;
+        else
+                clocks->max_clocks_state = simple_clocks.level;
+
+        if (!smu_get_current_shallow_sleep_clocks(smu, &hw_clocks)) {
+                clocks->max_engine_clock_in_sr = hw_clocks.max_eng_clk;
+                clocks->min_engine_clock_in_sr = hw_clocks.min_eng_clk;
+        }
+
+failed:
+	mutex_unlock(&smu->mutex);
+	return ret;
+}
+
+static int smu_set_clockgating_state(void *handle,
+				     enum amd_clockgating_state state)
+{
+	return 0;
+}
+
+static int smu_set_powergating_state(void *handle,
+				     enum amd_powergating_state state)
+{
+	return 0;
+}
+
+static int smu_enable_umd_pstate(void *handle,
+		      enum amd_dpm_forced_level *level)
+{
+	uint32_t profile_mode_mask = AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD |
+					AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK |
+					AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK |
+					AMD_DPM_FORCED_LEVEL_PROFILE_PEAK;
+
+	struct smu_context *smu = (struct smu_context*)(handle);
+	struct smu_dpm_context *smu_dpm_ctx = &(smu->smu_dpm);
+	if (!smu_dpm_ctx->dpm_context)
+		return -EINVAL;
+
+	if (!(smu_dpm_ctx->dpm_level & profile_mode_mask)) {
+		/* enter umd pstate, save current level, disable gfx cg*/
+		if (*level & profile_mode_mask) {
+			smu_dpm_ctx->saved_dpm_level = smu_dpm_ctx->dpm_level;
+			smu_dpm_ctx->enable_umd_pstate = true;
+			amdgpu_device_ip_set_clockgating_state(smu->adev,
+							       AMD_IP_BLOCK_TYPE_GFX,
+							       AMD_CG_STATE_UNGATE);
+			amdgpu_device_ip_set_powergating_state(smu->adev,
+							       AMD_IP_BLOCK_TYPE_GFX,
+							       AMD_PG_STATE_UNGATE);
+		}
+	} else {
+		/* exit umd pstate, restore level, enable gfx cg*/
+		if (!(*level & profile_mode_mask)) {
+			if (*level == AMD_DPM_FORCED_LEVEL_PROFILE_EXIT)
+				*level = smu_dpm_ctx->saved_dpm_level;
+			smu_dpm_ctx->enable_umd_pstate = false;
+			amdgpu_device_ip_set_clockgating_state(smu->adev,
+							       AMD_IP_BLOCK_TYPE_GFX,
+							       AMD_CG_STATE_GATE);
+			amdgpu_device_ip_set_powergating_state(smu->adev,
+							       AMD_IP_BLOCK_TYPE_GFX,
+							       AMD_PG_STATE_GATE);
+		}
+	}
+
+	return 0;
+}
+
+int smu_adjust_power_state_dynamic(struct smu_context *smu,
+				   enum amd_dpm_forced_level level,
+				   bool skip_display_settings)
+{
+	int ret = 0;
+	int index = 0;
+	uint32_t sclk_mask, mclk_mask, soc_mask;
+	long workload;
+	struct smu_dpm_context *smu_dpm_ctx = &(smu->smu_dpm);
+
+	if (!skip_display_settings) {
+		ret = smu_display_config_changed(smu);
+		if (ret) {
+			pr_err("Failed to change display config!");
+			return ret;
+		}
+	}
+
+	ret = smu_apply_clocks_adjust_rules(smu);
+	if (ret) {
+		pr_err("Failed to apply clocks adjust rules!");
+		return ret;
+	}
+
+	if (!skip_display_settings) {
+		ret = smu_notify_smc_dispaly_config(smu);
+		if (ret) {
+			pr_err("Failed to notify smc display config!");
+			return ret;
+		}
+	}
+
+	if (smu_dpm_ctx->dpm_level != level) {
+		switch (level) {
+		case AMD_DPM_FORCED_LEVEL_HIGH:
+			ret = smu_force_dpm_limit_value(smu, true);
+			break;
+		case AMD_DPM_FORCED_LEVEL_LOW:
+			ret = smu_force_dpm_limit_value(smu, false);
+			break;
+
+		case AMD_DPM_FORCED_LEVEL_AUTO:
+			ret = smu_unforce_dpm_levels(smu);
+			break;
+
+		case AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD:
+		case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK:
+		case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK:
+		case AMD_DPM_FORCED_LEVEL_PROFILE_PEAK:
+			ret = smu_get_profiling_clk_mask(smu, level,
+							 &sclk_mask,
+							 &mclk_mask,
+							 &soc_mask);
+			if (ret)
+				return ret;
+			smu_force_clk_levels(smu, PP_SCLK, 1 << sclk_mask);
+			smu_force_clk_levels(smu, PP_MCLK, 1 << mclk_mask);
+			break;
+
+		case AMD_DPM_FORCED_LEVEL_MANUAL:
+		case AMD_DPM_FORCED_LEVEL_PROFILE_EXIT:
+		default:
+			break;
+		}
+
+		if (!ret)
+			smu_dpm_ctx->dpm_level = level;
+	}
+
+	if (smu_dpm_ctx->dpm_level != AMD_DPM_FORCED_LEVEL_MANUAL) {
+		index = fls(smu->workload_mask);
+		index = index > 0 && index <= WORKLOAD_POLICY_MAX ? index - 1 : 0;
+		workload = smu->workload_setting[index];
+
+		if (smu->power_profile_mode != workload)
+			smu_set_power_profile_mode(smu, &workload, 0);
+	}
+
+	return ret;
+}
+
+int smu_handle_task(struct smu_context *smu,
+		    enum amd_dpm_forced_level level,
+		    enum amd_pp_task task_id)
+{
+	int ret = 0;
+
+	switch (task_id) {
+	case AMD_PP_TASK_DISPLAY_CONFIG_CHANGE:
+		ret = smu_pre_display_config_changed(smu);
+		if (ret)
+			return ret;
+		ret = smu_set_cpu_power_state(smu);
+		if (ret)
+			return ret;
+		ret = smu_adjust_power_state_dynamic(smu, level, false);
+		break;
+	case AMD_PP_TASK_COMPLETE_INIT:
+	case AMD_PP_TASK_READJUST_POWER_STATE:
+		ret = smu_adjust_power_state_dynamic(smu, level, true);
+		break;
+	default:
+		break;
+	}
+
+	return ret;
+}
+
+const struct amd_ip_funcs smu_ip_funcs = {
+	.name = "smu",
+	.early_init = smu_early_init,
+	.late_init = smu_late_init,
+	.sw_init = smu_sw_init,
+	.sw_fini = smu_sw_fini,
+	.hw_init = smu_hw_init,
+	.hw_fini = smu_hw_fini,
+	.suspend = smu_suspend,
+	.resume = smu_resume,
+	.is_idle = NULL,
+	.check_soft_reset = NULL,
+	.wait_for_idle = NULL,
+	.soft_reset = NULL,
+	.set_clockgating_state = smu_set_clockgating_state,
+	.set_powergating_state = smu_set_powergating_state,
+	.enable_umd_pstate = smu_enable_umd_pstate,
+};
+
+const struct amdgpu_ip_block_version smu_v11_0_ip_block =
+{
+	.type = AMD_IP_BLOCK_TYPE_SMC,
+	.major = 11,
+	.minor = 0,
+	.rev = 0,
+	.funcs = &smu_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/Makefile b/drivers/gpu/drm/amd/powerplay/hwmgr/Makefile
index 0b3c6d1d52e4..cc63705920dc 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/Makefile
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/Makefile
@@ -35,7 +35,8 @@ HARDWARE_MGR = hwmgr.o processpptables.o \
 		vega12_thermal.o \
 		pp_overdriver.o smu_helper.o \
 		vega20_processpptables.o vega20_hwmgr.o vega20_powertune.o \
-		vega20_thermal.o common_baco.o vega10_baco.o  vega20_baco.o
+		vega20_thermal.o common_baco.o vega10_baco.o  vega20_baco.o \
+		vega12_baco.o smu9_baco.o
 
 AMD_PP_HWMGR = $(addprefix $(AMD_PP_PATH)/hwmgr/,$(HARDWARE_MGR))
 
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c b/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c
index c1c51c115e57..70f7f47a2fcf 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c
@@ -76,7 +76,7 @@ int phm_set_power_state(struct pp_hwmgr *hwmgr,
 int phm_enable_dynamic_state_management(struct pp_hwmgr *hwmgr)
 {
 	struct amdgpu_device *adev = NULL;
-	int ret = -EINVAL;;
+	int ret = -EINVAL;
 	PHM_FUNC_CHECK(hwmgr);
 	adev = hwmgr->adev;
 
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu10_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/smu10_hwmgr.c
index 0ad8fe4a6277..9a595f7525e6 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu10_hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu10_hwmgr.c
@@ -35,6 +35,7 @@
 #include "smu10_hwmgr.h"
 #include "power_state.h"
 #include "soc15_common.h"
+#include "smu10.h"
 
 #define SMU10_MAX_DEEPSLEEP_DIVIDER_ID     5
 #define SMU10_MINIMUM_ENGINE_CLOCK         800   /* 8Mhz, the low boundary of engine clock allowed on this chip */
@@ -114,11 +115,6 @@ static int smu10_initialize_dpm_defaults(struct pp_hwmgr *hwmgr)
 	smu10_data->num_active_display = 0;
 	smu10_data->deep_sleep_dcefclk = 0;
 
-	if (hwmgr->feature_mask & PP_GFXOFF_MASK)
-		smu10_data->gfx_off_controled_by_driver = true;
-	else
-		smu10_data->gfx_off_controled_by_driver = false;
-
 	phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
 					PHM_PlatformCaps_SclkDeepSleep);
 
@@ -209,18 +205,13 @@ static int smu10_set_clock_limit(struct pp_hwmgr *hwmgr, const void *input)
 	return 0;
 }
 
-static inline uint32_t convert_10k_to_mhz(uint32_t clock)
-{
-	return (clock + 99) / 100;
-}
-
 static int smu10_set_min_deep_sleep_dcefclk(struct pp_hwmgr *hwmgr, uint32_t clock)
 {
 	struct smu10_hwmgr *smu10_data = (struct smu10_hwmgr *)(hwmgr->backend);
 
 	if (smu10_data->need_min_deep_sleep_dcefclk &&
-		smu10_data->deep_sleep_dcefclk != convert_10k_to_mhz(clock)) {
-		smu10_data->deep_sleep_dcefclk = convert_10k_to_mhz(clock);
+		smu10_data->deep_sleep_dcefclk != clock) {
+		smu10_data->deep_sleep_dcefclk = clock;
 		smum_send_msg_to_smc_with_parameter(hwmgr,
 					PPSMC_MSG_SetMinDeepSleepDcefclk,
 					smu10_data->deep_sleep_dcefclk);
@@ -233,8 +224,8 @@ static int smu10_set_hard_min_dcefclk_by_freq(struct pp_hwmgr *hwmgr, uint32_t c
 	struct smu10_hwmgr *smu10_data = (struct smu10_hwmgr *)(hwmgr->backend);
 
 	if (smu10_data->dcf_actual_hard_min_freq &&
-		smu10_data->dcf_actual_hard_min_freq != convert_10k_to_mhz(clock)) {
-		smu10_data->dcf_actual_hard_min_freq = convert_10k_to_mhz(clock);
+		smu10_data->dcf_actual_hard_min_freq != clock) {
+		smu10_data->dcf_actual_hard_min_freq = clock;
 		smum_send_msg_to_smc_with_parameter(hwmgr,
 					PPSMC_MSG_SetHardMinDcefclkByFreq,
 					smu10_data->dcf_actual_hard_min_freq);
@@ -247,8 +238,8 @@ static int smu10_set_hard_min_fclk_by_freq(struct pp_hwmgr *hwmgr, uint32_t cloc
 	struct smu10_hwmgr *smu10_data = (struct smu10_hwmgr *)(hwmgr->backend);
 
 	if (smu10_data->f_actual_hard_min_freq &&
-		smu10_data->f_actual_hard_min_freq != convert_10k_to_mhz(clock)) {
-		smu10_data->f_actual_hard_min_freq = convert_10k_to_mhz(clock);
+		smu10_data->f_actual_hard_min_freq != clock) {
+		smu10_data->f_actual_hard_min_freq = clock;
 		smum_send_msg_to_smc_with_parameter(hwmgr,
 					PPSMC_MSG_SetHardMinFclkByFreq,
 					smu10_data->f_actual_hard_min_freq);
@@ -330,9 +321,9 @@ static bool smu10_is_gfx_on(struct pp_hwmgr *hwmgr)
 
 static int smu10_disable_gfx_off(struct pp_hwmgr *hwmgr)
 {
-	struct smu10_hwmgr *smu10_data = (struct smu10_hwmgr *)(hwmgr->backend);
+	struct amdgpu_device *adev = hwmgr->adev;
 
-	if (smu10_data->gfx_off_controled_by_driver) {
+	if (adev->pm.pp_feature & PP_GFXOFF_MASK) {
 		smum_send_msg_to_smc(hwmgr, PPSMC_MSG_DisableGfxOff);
 
 		/* confirm gfx is back to "on" state */
@@ -350,9 +341,9 @@ static int smu10_disable_dpm_tasks(struct pp_hwmgr *hwmgr)
 
 static int smu10_enable_gfx_off(struct pp_hwmgr *hwmgr)
 {
-	struct smu10_hwmgr *smu10_data = (struct smu10_hwmgr *)(hwmgr->backend);
+	struct amdgpu_device *adev = hwmgr->adev;
 
-	if (smu10_data->gfx_off_controled_by_driver)
+	if (adev->pm.pp_feature & PP_GFXOFF_MASK)
 		smum_send_msg_to_smc(hwmgr, PPSMC_MSG_EnableGfxOff);
 
 	return 0;
@@ -577,7 +568,6 @@ static int smu10_dpm_force_dpm_level(struct pp_hwmgr *hwmgr,
 				enum amd_dpm_forced_level level)
 {
 	struct smu10_hwmgr *data = hwmgr->backend;
-	struct amdgpu_device *adev = hwmgr->adev;
 	uint32_t min_sclk = hwmgr->display_config->min_core_set_clock;
 	uint32_t min_mclk = hwmgr->display_config->min_mem_set_clock/100;
 
@@ -586,11 +576,6 @@ static int smu10_dpm_force_dpm_level(struct pp_hwmgr *hwmgr,
 		return 0;
 	}
 
-	/* Disable UMDPSTATE support on rv2 temporarily */
-	if ((adev->asic_type == CHIP_RAVEN) &&
-	    (adev->rev_id >= 8))
-		return 0;
-
 	if (min_sclk < data->gfx_min_freq_limit)
 		min_sclk = data->gfx_min_freq_limit;
 
@@ -1205,6 +1190,94 @@ static void smu10_powergate_vcn(struct pp_hwmgr *hwmgr, bool bgate)
 	}
 }
 
+static int conv_power_profile_to_pplib_workload(int power_profile)
+{
+	int pplib_workload = 0;
+
+	switch (power_profile) {
+	case PP_SMC_POWER_PROFILE_BOOTUP_DEFAULT:
+		pplib_workload = WORKLOAD_DEFAULT_BIT;
+		break;
+	case PP_SMC_POWER_PROFILE_FULLSCREEN3D:
+		pplib_workload = WORKLOAD_PPLIB_FULL_SCREEN_3D_BIT;
+		break;
+	case PP_SMC_POWER_PROFILE_POWERSAVING:
+		pplib_workload = WORKLOAD_PPLIB_POWER_SAVING_BIT;
+		break;
+	case PP_SMC_POWER_PROFILE_VIDEO:
+		pplib_workload = WORKLOAD_PPLIB_VIDEO_BIT;
+		break;
+	case PP_SMC_POWER_PROFILE_VR:
+		pplib_workload = WORKLOAD_PPLIB_VR_BIT;
+		break;
+	case PP_SMC_POWER_PROFILE_COMPUTE:
+		pplib_workload = WORKLOAD_PPLIB_COMPUTE_BIT;
+		break;
+	}
+
+	return pplib_workload;
+}
+
+static int smu10_get_power_profile_mode(struct pp_hwmgr *hwmgr, char *buf)
+{
+	uint32_t i, size = 0;
+	static const uint8_t
+		profile_mode_setting[6][4] = {{70, 60, 0, 0,},
+						{70, 60, 1, 3,},
+						{90, 60, 0, 0,},
+						{70, 60, 0, 0,},
+						{70, 90, 0, 0,},
+						{30, 60, 0, 6,},
+						};
+	static const char *profile_name[6] = {
+					"BOOTUP_DEFAULT",
+					"3D_FULL_SCREEN",
+					"POWER_SAVING",
+					"VIDEO",
+					"VR",
+					"COMPUTE"};
+	static const char *title[6] = {"NUM",
+			"MODE_NAME",
+			"BUSY_SET_POINT",
+			"FPS",
+			"USE_RLC_BUSY",
+			"MIN_ACTIVE_LEVEL"};
+
+	if (!buf)
+		return -EINVAL;
+
+	size += sprintf(buf + size, "%s %16s %s %s %s %s\n",title[0],
+			title[1], title[2], title[3], title[4], title[5]);
+
+	for (i = 0; i <= PP_SMC_POWER_PROFILE_COMPUTE; i++)
+		size += sprintf(buf + size, "%3d %14s%s: %14d %3d %10d %14d\n",
+			i, profile_name[i], (i == hwmgr->power_profile_mode) ? "*" : " ",
+			profile_mode_setting[i][0], profile_mode_setting[i][1],
+			profile_mode_setting[i][2], profile_mode_setting[i][3]);
+
+	return size;
+}
+
+static int smu10_set_power_profile_mode(struct pp_hwmgr *hwmgr, long *input, uint32_t size)
+{
+	int workload_type = 0;
+
+	if (input[size] > PP_SMC_POWER_PROFILE_COMPUTE) {
+		pr_err("Invalid power profile mode %ld\n", input[size]);
+		return -EINVAL;
+	}
+	hwmgr->power_profile_mode = input[size];
+
+	/* conv PP_SMC_POWER_PROFILE* to WORKLOAD_PPLIB_*_BIT */
+	workload_type =
+		conv_power_profile_to_pplib_workload(hwmgr->power_profile_mode);
+	smum_send_msg_to_smc_with_parameter(hwmgr, PPSMC_MSG_ActiveProcessNotify,
+						1 << workload_type);
+
+	return 0;
+}
+
+
 static const struct pp_hwmgr_func smu10_hwmgr_funcs = {
 	.backend_init = smu10_hwmgr_backend_init,
 	.backend_fini = smu10_hwmgr_backend_fini,
@@ -1246,6 +1319,8 @@ static const struct pp_hwmgr_func smu10_hwmgr_funcs = {
 	.powergate_sdma = smu10_powergate_sdma,
 	.set_hard_min_dcefclk_by_freq = smu10_set_hard_min_dcefclk_by_freq,
 	.set_hard_min_fclk_by_freq = smu10_set_hard_min_fclk_by_freq,
+	.get_power_profile_mode = smu10_get_power_profile_mode,
+	.set_power_profile_mode = smu10_set_power_profile_mode,
 };
 
 int smu10_init_function_pointers(struct pp_hwmgr *hwmgr)
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c
index 83d3d935f3ac..048757e8f494 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c
@@ -77,7 +77,7 @@
 #define PCIE_BUS_CLK                10000
 #define TCLK                        (PCIE_BUS_CLK / 10)
 
-static const struct profile_mode_setting smu7_profiling[7] =
+static struct profile_mode_setting smu7_profiling[7] =
 					{{0, 0, 0, 0, 0, 0, 0, 0},
 					 {1, 0, 100, 30, 1, 0, 100, 10},
 					 {1, 10, 0, 30, 0, 0, 0, 0},
@@ -4984,17 +4984,27 @@ static int smu7_set_power_profile_mode(struct pp_hwmgr *hwmgr, long *input, uint
 	mode = input[size];
 	switch (mode) {
 	case PP_SMC_POWER_PROFILE_CUSTOM:
-		if (size < 8)
+		if (size < 8 && size != 0)
 			return -EINVAL;
-
-		tmp.bupdate_sclk = input[0];
-		tmp.sclk_up_hyst = input[1];
-		tmp.sclk_down_hyst = input[2];
-		tmp.sclk_activity = input[3];
-		tmp.bupdate_mclk = input[4];
-		tmp.mclk_up_hyst = input[5];
-		tmp.mclk_down_hyst = input[6];
-		tmp.mclk_activity = input[7];
+		/* If only CUSTOM is passed in, use the saved values. Check
+		 * that we actually have a CUSTOM profile by ensuring that
+		 * the "use sclk" or the "use mclk" bits are set
+		 */
+		tmp = smu7_profiling[PP_SMC_POWER_PROFILE_CUSTOM];
+		if (size == 0) {
+			if (tmp.bupdate_sclk == 0 && tmp.bupdate_mclk == 0)
+				return -EINVAL;
+		} else {
+			tmp.bupdate_sclk = input[0];
+			tmp.sclk_up_hyst = input[1];
+			tmp.sclk_down_hyst = input[2];
+			tmp.sclk_activity = input[3];
+			tmp.bupdate_mclk = input[4];
+			tmp.mclk_up_hyst = input[5];
+			tmp.mclk_down_hyst = input[6];
+			tmp.mclk_activity = input[7];
+			smu7_profiling[PP_SMC_POWER_PROFILE_CUSTOM] = tmp;
+		}
 		if (!smum_update_dpm_settings(hwmgr, &tmp)) {
 			memcpy(&data->current_profile_setting, &tmp, sizeof(struct profile_mode_setting));
 			hwmgr->power_profile_mode = mode;
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu9_baco.c b/drivers/gpu/drm/amd/powerplay/hwmgr/smu9_baco.c
new file mode 100644
index 000000000000..de0a37f7c632
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu9_baco.c
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "amdgpu.h"
+#include "soc15.h"
+#include "soc15_hw_ip.h"
+#include "vega10_ip_offset.h"
+#include "soc15_common.h"
+#include "vega10_inc.h"
+#include "smu9_baco.h"
+
+int smu9_baco_get_capability(struct pp_hwmgr *hwmgr, bool *cap)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)(hwmgr->adev);
+	uint32_t reg, data;
+
+	*cap = false;
+	if (!phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_BACO))
+		return 0;
+
+	WREG32(0x12074, 0xFFF0003B);
+	data = RREG32(0x12075);
+
+	if (data == 0x1) {
+		reg = RREG32_SOC15(NBIF, 0, mmRCC_BIF_STRAP0);
+
+		if (reg & RCC_BIF_STRAP0__STRAP_PX_CAPABLE_MASK)
+			*cap = true;
+	}
+
+	return 0;
+}
+
+int smu9_baco_get_state(struct pp_hwmgr *hwmgr, enum BACO_STATE *state)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)(hwmgr->adev);
+	uint32_t reg;
+
+	reg = RREG32_SOC15(NBIF, 0, mmBACO_CNTL);
+
+	if (reg & BACO_CNTL__BACO_MODE_MASK)
+		/* gfx has already entered BACO state */
+		*state = BACO_STATE_IN;
+	else
+		*state = BACO_STATE_OUT;
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu9_baco.h b/drivers/gpu/drm/amd/powerplay/hwmgr/smu9_baco.h
new file mode 100644
index 000000000000..84e90f801ac3
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu9_baco.h
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __SMU9_BACO_H__
+#define __SMU9_BACO_H__
+#include "hwmgr.h"
+#include "common_baco.h"
+
+extern int smu9_baco_get_capability(struct pp_hwmgr *hwmgr, bool *cap);
+extern int smu9_baco_get_state(struct pp_hwmgr *hwmgr, enum BACO_STATE *state);
+
+#endif
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_baco.c b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_baco.c
index 7337be5602e4..d168af4a4d78 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_baco.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_baco.c
@@ -85,48 +85,11 @@ static const struct soc15_baco_cmd_entry clean_baco_tbl[] =
 	{CMD_WRITE, SOC15_REG_ENTRY(NBIF, 0, mmBIOS_SCRATCH_7), 0, 0, 0, 0},
 };
 
-int vega10_baco_get_capability(struct pp_hwmgr *hwmgr, bool *cap)
-{
-	struct amdgpu_device *adev = (struct amdgpu_device *)(hwmgr->adev);
-	uint32_t reg, data;
-
-	*cap = false;
-	if (!phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_BACO))
-		return 0;
-
-	WREG32(0x12074, 0xFFF0003B);
-	data = RREG32(0x12075);
-
-	if (data == 0x1) {
-		reg = RREG32_SOC15(NBIF, 0, mmRCC_BIF_STRAP0);
-
-		if (reg & RCC_BIF_STRAP0__STRAP_PX_CAPABLE_MASK)
-			*cap = true;
-	}
-
-	return 0;
-}
-
-int vega10_baco_get_state(struct pp_hwmgr *hwmgr, enum BACO_STATE *state)
-{
-	struct amdgpu_device *adev = (struct amdgpu_device *)(hwmgr->adev);
-	uint32_t reg;
-
-	reg = RREG32_SOC15(NBIF, 0, mmBACO_CNTL);
-
-	if (reg & BACO_CNTL__BACO_MODE_MASK)
-		/* gfx has already entered BACO state */
-		*state = BACO_STATE_IN;
-	else
-		*state = BACO_STATE_OUT;
-	return 0;
-}
-
 int vega10_baco_set_state(struct pp_hwmgr *hwmgr, enum BACO_STATE state)
 {
 	enum BACO_STATE cur_state;
 
-	vega10_baco_get_state(hwmgr, &cur_state);
+	smu9_baco_get_state(hwmgr, &cur_state);
 
 	if (cur_state == state)
 		/* aisc already in the target state */
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_baco.h b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_baco.h
index f7a3ffa744b3..96d793f026a5 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_baco.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_baco.h
@@ -22,11 +22,8 @@
  */
 #ifndef __VEGA10_BACO_H__
 #define __VEGA10_BACO_H__
-#include "hwmgr.h"
-#include "common_baco.h"
+#include "smu9_baco.h"
 
-extern int vega10_baco_get_capability(struct pp_hwmgr *hwmgr, bool *cap);
-extern int vega10_baco_get_state(struct pp_hwmgr *hwmgr, enum BACO_STATE *state);
 extern int vega10_baco_set_state(struct pp_hwmgr *hwmgr, enum BACO_STATE state);
 
 #endif
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.c
index 5c4f701939ea..384c37875cd0 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega10_hwmgr.c
@@ -1427,6 +1427,15 @@ static int vega10_setup_default_dpm_tables(struct pp_hwmgr *hwmgr)
 
 	vega10_setup_default_pcie_table(hwmgr);
 
+	/* Zero out the saved copy of the CUSTOM profile
+	 * This will be checked when trying to set the profile
+	 * and will require that new values be passed in
+	 */
+	data->custom_profile_mode[0] = 0;
+	data->custom_profile_mode[1] = 0;
+	data->custom_profile_mode[2] = 0;
+	data->custom_profile_mode[3] = 0;
+
 	/* save a copy of the default DPM table */
 	memcpy(&(data->golden_dpm_table), &(data->dpm_table),
 			sizeof(struct vega10_dpm_table));
@@ -4904,16 +4913,23 @@ static int vega10_set_power_profile_mode(struct pp_hwmgr *hwmgr, long *input, ui
 	uint8_t FPS;
 	uint8_t use_rlc_busy;
 	uint8_t min_active_level;
+	uint32_t power_profile_mode = input[size];
 
-	hwmgr->power_profile_mode = input[size];
-
-	smum_send_msg_to_smc_with_parameter(hwmgr, PPSMC_MSG_SetWorkloadMask,
-						1<<hwmgr->power_profile_mode);
-
-	if (hwmgr->power_profile_mode == PP_SMC_POWER_PROFILE_CUSTOM) {
-		if (size == 0 || size > 4)
+	if (power_profile_mode == PP_SMC_POWER_PROFILE_CUSTOM) {
+		if (size != 0 && size != 4)
 			return -EINVAL;
 
+		/* If size = 0 and the CUSTOM profile has been set already
+		 * then just apply the profile. The copy stored in the hwmgr
+		 * is zeroed out on init
+		 */
+		if (size == 0) {
+			if (data->custom_profile_mode[0] != 0)
+				goto out;
+			else
+				return -EINVAL;
+		}
+
 		data->custom_profile_mode[0] = busy_set_point = input[0];
 		data->custom_profile_mode[1] = FPS = input[1];
 		data->custom_profile_mode[2] = use_rlc_busy = input[2];
@@ -4924,6 +4940,11 @@ static int vega10_set_power_profile_mode(struct pp_hwmgr *hwmgr, long *input, ui
 					use_rlc_busy << 16 | min_active_level<<24);
 	}
 
+out:
+	smum_send_msg_to_smc_with_parameter(hwmgr, PPSMC_MSG_SetWorkloadMask,
+						1 << power_profile_mode);
+	hwmgr->power_profile_mode = power_profile_mode;
+
 	return 0;
 }
 
@@ -5170,8 +5191,8 @@ static const struct pp_hwmgr_func vega10_hwmgr_funcs = {
 	.set_power_limit = vega10_set_power_limit,
 	.odn_edit_dpm_table = vega10_odn_edit_dpm_table,
 	.get_performance_level = vega10_get_performance_level,
-	.get_asic_baco_capability = vega10_baco_get_capability,
-	.get_asic_baco_state = vega10_baco_get_state,
+	.get_asic_baco_capability = smu9_baco_get_capability,
+	.get_asic_baco_state = smu9_baco_get_state,
 	.set_asic_baco_state = vega10_baco_set_state,
 	.enable_mgpu_fan_boost = vega10_enable_mgpu_fan_boost,
 	.get_ppfeature_status = vega10_get_ppfeature_status,
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_baco.c b/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_baco.c
new file mode 100644
index 000000000000..9d8ca94a8f0c
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_baco.c
@@ -0,0 +1,119 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "amdgpu.h"
+#include "soc15.h"
+#include "soc15_hw_ip.h"
+#include "vega10_ip_offset.h"
+#include "soc15_common.h"
+#include "vega12_inc.h"
+#include "vega12_ppsmc.h"
+#include "vega12_baco.h"
+
+static const struct soc15_baco_cmd_entry  pre_baco_tbl[] =
+{
+	{ CMD_READMODIFYWRITE, NBIF_HWID, 0, mmBIF_DOORBELL_CNTL_BASE_IDX, mmBIF_DOORBELL_CNTL, BIF_DOORBELL_CNTL__DOORBELL_MONITOR_EN_MASK, BIF_DOORBELL_CNTL__DOORBELL_MONITOR_EN__SHIFT, 0, 0 },
+	{ CMD_WRITE, NBIF_HWID, 0, mmBIF_FB_EN_BASE_IDX, mmBIF_FB_EN, 0, 0, 0, 0 },
+	{ CMD_READMODIFYWRITE, NBIF_HWID, 0, mmRCC_BACO_CNTL_MISC_BASE_IDX, mmBACO_CNTL, BACO_CNTL__BACO_DSTATE_BYPASS_MASK, BACO_CNTL__BACO_DSTATE_BYPASS__SHIFT, 0, 1 },
+	{ CMD_READMODIFYWRITE, NBIF_HWID, 0, mmRCC_BACO_CNTL_MISC_BASE_IDX, mmBACO_CNTL, BACO_CNTL__BACO_RST_INTR_MASK_MASK, BACO_CNTL__BACO_RST_INTR_MASK__SHIFT, 0, 1 }
+};
+
+static const struct soc15_baco_cmd_entry enter_baco_tbl[] =
+{
+	{ CMD_WAITFOR, THM_HWID, 0, mmTHM_BACO_CNTL_BASE_IDX, mmTHM_BACO_CNTL, THM_BACO_CNTL__SOC_DOMAIN_IDLE_MASK, THM_BACO_CNTL__SOC_DOMAIN_IDLE__SHIFT, 0xffffffff, 0x80000000 },
+	{ CMD_READMODIFYWRITE, NBIF_HWID, 0, mmRCC_BACO_CNTL_MISC_BASE_IDX, mmBACO_CNTL, BACO_CNTL__BACO_EN_MASK, BACO_CNTL__BACO_EN__SHIFT, 0, 1 },
+	{ CMD_READMODIFYWRITE, NBIF_HWID, 0, mmRCC_BACO_CNTL_MISC_BASE_IDX, mmBACO_CNTL, BACO_CNTL__BACO_BIF_LCLK_SWITCH_MASK, BACO_CNTL__BACO_BIF_LCLK_SWITCH__SHIFT, 0, 1 },
+	{ CMD_READMODIFYWRITE, NBIF_HWID, 0, mmRCC_BACO_CNTL_MISC_BASE_IDX, mmBACO_CNTL, BACO_CNTL__BACO_DUMMY_EN_MASK, BACO_CNTL__BACO_DUMMY_EN__SHIFT, 0, 1 },
+	{ CMD_READMODIFYWRITE, THM_HWID, 0, mmTHM_BACO_CNTL_BASE_IDX, mmTHM_BACO_CNTL, THM_BACO_CNTL__BACO_SOC_VDCI_RESET_MASK, THM_BACO_CNTL__BACO_SOC_VDCI_RESET__SHIFT, 0, 1 },
+	{ CMD_READMODIFYWRITE, THM_HWID, 0, mmTHM_BACO_CNTL_BASE_IDX, mmTHM_BACO_CNTL, THM_BACO_CNTL__BACO_SMNCLK_MUX_MASK, THM_BACO_CNTL__BACO_SMNCLK_MUX__SHIFT, 0, 1 },
+	{ CMD_READMODIFYWRITE, THM_HWID, 0, mmTHM_BACO_CNTL_BASE_IDX, mmTHM_BACO_CNTL, THM_BACO_CNTL__BACO_ISO_EN_MASK, THM_BACO_CNTL__BACO_ISO_EN__SHIFT, 0, 1 },
+	{ CMD_READMODIFYWRITE, THM_HWID, 0, mmTHM_BACO_CNTL_BASE_IDX, mmTHM_BACO_CNTL, THM_BACO_CNTL__BACO_AEB_ISO_EN_MASK, THM_BACO_CNTL__BACO_AEB_ISO_EN__SHIFT, 0, 1 },
+	{ CMD_READMODIFYWRITE, THM_HWID, 0, mmTHM_BACO_CNTL_BASE_IDX, mmTHM_BACO_CNTL, THM_BACO_CNTL__BACO_ANA_ISO_EN_MASK, THM_BACO_CNTL__BACO_ANA_ISO_EN__SHIFT, 0, 1 },
+	{ CMD_READMODIFYWRITE, THM_HWID, 0, mmTHM_BACO_CNTL_BASE_IDX, mmTHM_BACO_CNTL, THM_BACO_CNTL__BACO_SOC_REFCLK_OFF_MASK, THM_BACO_CNTL__BACO_SOC_REFCLK_OFF__SHIFT, 0, 1 },
+	{ CMD_READMODIFYWRITE, NBIF_HWID, 0, mmRCC_BACO_CNTL_MISC_BASE_IDX, mmBACO_CNTL, BACO_CNTL__BACO_POWER_OFF_MASK, BACO_CNTL__BACO_POWER_OFF__SHIFT, 0, 1 },
+	{ CMD_DELAY_MS, 0, 0, 0, 5, 0 },
+	{ CMD_READMODIFYWRITE, THM_HWID, 0, mmTHM_BACO_CNTL_BASE_IDX, mmTHM_BACO_CNTL, THM_BACO_CNTL__BACO_RESET_EN_MASK, THM_BACO_CNTL__BACO_RESET_EN__SHIFT, 0, 1 },
+	{ CMD_READMODIFYWRITE, THM_HWID, 0, mmTHM_BACO_CNTL_BASE_IDX, mmTHM_BACO_CNTL, THM_BACO_CNTL__BACO_PWROKRAW_CNTL_MASK, THM_BACO_CNTL__BACO_PWROKRAW_CNTL__SHIFT, 0, 0 },
+	{ CMD_WAITFOR, NBIF_HWID, 0, mmRCC_BACO_CNTL_MISC_BASE_IDX, mmBACO_CNTL, BACO_CNTL__BACO_MODE_MASK, BACO_CNTL__BACO_MODE__SHIFT, 0xffffffff, 0x100 }
+};
+
+static const struct soc15_baco_cmd_entry exit_baco_tbl[] =
+{
+	{ CMD_READMODIFYWRITE, NBIF_HWID, 0, mmRCC_BACO_CNTL_MISC_BASE_IDX, mmBACO_CNTL, BACO_CNTL__BACO_POWER_OFF_MASK, BACO_CNTL__BACO_POWER_OFF__SHIFT, 0, 0 },
+	{ CMD_DELAY_MS, 0, 0, 0, 0, 0, 0, 10, 0 },
+	{ CMD_READMODIFYWRITE, THM_HWID, 0, mmTHM_BACO_CNTL_BASE_IDX, mmTHM_BACO_CNTL, THM_BACO_CNTL__BACO_SOC_REFCLK_OFF_MASK, THM_BACO_CNTL__BACO_SOC_REFCLK_OFF__SHIFT, 0, 0 },
+	{ CMD_READMODIFYWRITE, THM_HWID, 0, mmTHM_BACO_CNTL_BASE_IDX, mmTHM_BACO_CNTL, THM_BACO_CNTL__BACO_ANA_ISO_EN_MASK, THM_BACO_CNTL__BACO_ANA_ISO_EN__SHIFT, 0, 0 },
+	{ CMD_READMODIFYWRITE, THM_HWID, 0, mmTHM_BACO_CNTL_BASE_IDX, mmTHM_BACO_CNTL, THM_BACO_CNTL__BACO_AEB_ISO_EN_MASK, THM_BACO_CNTL__BACO_AEB_ISO_EN__SHIFT, 0, 0 },
+	{ CMD_READMODIFYWRITE, THM_HWID, 0, mmTHM_BACO_CNTL_BASE_IDX, mmTHM_BACO_CNTL, THM_BACO_CNTL__BACO_ISO_EN_MASK, THM_BACO_CNTL__BACO_ISO_EN__SHIFT, 0, 0 },
+	{ CMD_READMODIFYWRITE, THM_HWID, 0, mmTHM_BACO_CNTL_BASE_IDX, mmTHM_BACO_CNTL, THM_BACO_CNTL__BACO_PWROKRAW_CNTL_MASK, THM_BACO_CNTL__BACO_PWROKRAW_CNTL__SHIFT, 0, 1 },
+	{ CMD_READMODIFYWRITE, THM_HWID, 0, mmTHM_BACO_CNTL_BASE_IDX, mmTHM_BACO_CNTL, THM_BACO_CNTL__BACO_SMNCLK_MUX_MASK, THM_BACO_CNTL__BACO_SMNCLK_MUX__SHIFT, 0, 0 },
+	{ CMD_READMODIFYWRITE, THM_HWID, 0, mmTHM_BACO_CNTL_BASE_IDX, mmTHM_BACO_CNTL, THM_BACO_CNTL__BACO_SOC_VDCI_RESET_MASK, THM_BACO_CNTL__BACO_SOC_VDCI_RESET__SHIFT, 0, 0 },
+	{ CMD_READMODIFYWRITE, THM_HWID, 0, mmTHM_BACO_CNTL_BASE_IDX, mmTHM_BACO_CNTL, THM_BACO_CNTL__BACO_EXIT_MASK, THM_BACO_CNTL__BACO_EXIT__SHIFT, 0, 1 },
+	{ CMD_READMODIFYWRITE, THM_HWID, 0, mmTHM_BACO_CNTL_BASE_IDX, mmTHM_BACO_CNTL, THM_BACO_CNTL__BACO_RESET_EN_MASK, THM_BACO_CNTL__BACO_RESET_EN__SHIFT, 0, 0 },
+	{ CMD_WAITFOR, THM_HWID, 0, mmTHM_BACO_CNTL_BASE_IDX, mmTHM_BACO_CNTL, THM_BACO_CNTL__BACO_EXIT_MASK, 0, 0xffffffff, 0 },
+	{ CMD_READMODIFYWRITE, THM_HWID, 0, mmTHM_BACO_CNTL_BASE_IDX, mmTHM_BACO_CNTL, THM_BACO_CNTL__BACO_SB_AXI_FENCE_MASK, THM_BACO_CNTL__BACO_SB_AXI_FENCE__SHIFT, 0, 0 },
+	{ CMD_READMODIFYWRITE, NBIF_HWID, 0, mmRCC_BACO_CNTL_MISC_BASE_IDX, mmBACO_CNTL, BACO_CNTL__BACO_DUMMY_EN_MASK, BACO_CNTL__BACO_DUMMY_EN__SHIFT, 0, 0 },
+	{ CMD_READMODIFYWRITE, NBIF_HWID, 0, mmRCC_BACO_CNTL_MISC_BASE_IDX, mmBACO_CNTL, BACO_CNTL__BACO_BIF_LCLK_SWITCH_MASK, BACO_CNTL__BACO_BIF_LCLK_SWITCH__SHIFT, 0, 0 },
+	{ CMD_READMODIFYWRITE, NBIF_HWID, 0, mmRCC_BACO_CNTL_MISC_BASE_IDX, mmBACO_CNTL, BACO_CNTL__BACO_EN_MASK, BACO_CNTL__BACO_EN__SHIFT, 0, 0 },
+	{ CMD_WAITFOR, NBIF_HWID, 0, mmRCC_BACO_CNTL_MISC_BASE_IDX, mmBACO_CNTL, BACO_CNTL__BACO_MODE_MASK, 0, 0xffffffff, 0 }
+};
+
+static const struct soc15_baco_cmd_entry clean_baco_tbl[] =
+{
+	{ CMD_WRITE, NBIF_HWID, 0, mmBIOS_SCRATCH_6_BASE_IDX, mmBIOS_SCRATCH_6, 0, 0, 0, 0 },
+	{ CMD_WRITE, NBIF_HWID, 0, mmBIOS_SCRATCH_7_BASE_IDX, mmBIOS_SCRATCH_7, 0, 0, 0, 0 }
+};
+
+int vega12_baco_set_state(struct pp_hwmgr *hwmgr, enum BACO_STATE state)
+{
+	enum BACO_STATE cur_state;
+
+	smu9_baco_get_state(hwmgr, &cur_state);
+
+	if (cur_state == state)
+		/* aisc already in the target state */
+		return 0;
+
+	if (state == BACO_STATE_IN) {
+		if (soc15_baco_program_registers(hwmgr, pre_baco_tbl,
+					     ARRAY_SIZE(pre_baco_tbl))) {
+			if (smum_send_msg_to_smc_with_parameter(hwmgr, PPSMC_MSG_EnterBaco, 0))
+				return -EINVAL;
+
+			if (soc15_baco_program_registers(hwmgr, enter_baco_tbl,
+						   ARRAY_SIZE(enter_baco_tbl)))
+				return 0;
+		}
+	} else if (state == BACO_STATE_OUT) {
+		/* HW requires at least 20ms between regulator off and on */
+		msleep(20);
+		/* Execute Hardware BACO exit sequence */
+		if (soc15_baco_program_registers(hwmgr, exit_baco_tbl,
+					     ARRAY_SIZE(exit_baco_tbl))) {
+			if (soc15_baco_program_registers(hwmgr, clean_baco_tbl,
+						     ARRAY_SIZE(clean_baco_tbl)))
+				return 0;
+		}
+	}
+
+	return -EINVAL;
+}
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_baco.h b/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_baco.h
new file mode 100644
index 000000000000..57b72e5a95ae
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_baco.h
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __VEGA12_BACO_H__
+#define __VEGA12_BACO_H__
+#include "smu9_baco.h"
+
+extern int vega12_baco_set_state(struct pp_hwmgr *hwmgr, enum BACO_STATE state);
+
+#endif
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_hwmgr.c
index bdb48e94eff6..707cd4b0357f 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_hwmgr.c
@@ -45,6 +45,7 @@
 #include "ppinterrupt.h"
 #include "pp_overdriver.h"
 #include "pp_thermal.h"
+#include "vega12_baco.h"
 
 
 static int vega12_force_clock_level(struct pp_hwmgr *hwmgr,
@@ -2626,8 +2627,12 @@ static const struct pp_hwmgr_func vega12_hwmgr_funcs = {
 	.start_thermal_controller = vega12_start_thermal_controller,
 	.powergate_gfx = vega12_gfx_off_control,
 	.get_performance_level = vega12_get_performance_level,
+	.get_asic_baco_capability = smu9_baco_get_capability,
+	.get_asic_baco_state = smu9_baco_get_state,
+	.set_asic_baco_state = vega12_baco_set_state,
 	.get_ppfeature_status = vega12_get_ppfeature_status,
 	.set_ppfeature_status = vega12_set_ppfeature_status,
+
 };
 
 int vega12_hwmgr_init(struct pp_hwmgr *hwmgr)
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_inc.h b/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_inc.h
index 30b278c50222..e6d9e84059e1 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_inc.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega12_inc.h
@@ -35,5 +35,7 @@
 #include "asic_reg/gc/gc_9_2_1_sh_mask.h"
 
 #include "asic_reg/nbio/nbio_6_1_offset.h"
+#include "asic_reg/nbio/nbio_6_1_offset.h"
+#include "asic_reg/nbio/nbio_6_1_sh_mask.h"
 
 #endif
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_baco.c b/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_baco.c
index 5e8602a79b1c..df6ff9252401 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_baco.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_baco.c
@@ -27,6 +27,7 @@
 #include "vega20_inc.h"
 #include "vega20_ppsmc.h"
 #include "vega20_baco.h"
+#include "vega20_smumgr.h"
 
 
 
@@ -101,3 +102,14 @@ int vega20_baco_set_state(struct pp_hwmgr *hwmgr, enum BACO_STATE state)
 
 	return 0;
 }
+
+int vega20_baco_apply_vdci_flush_workaround(struct pp_hwmgr *hwmgr)
+{
+	int ret = 0;
+
+	ret = vega20_set_pptable_driver_address(hwmgr);
+	if (ret)
+		return ret;
+
+	return smum_send_msg_to_smc(hwmgr, PPSMC_MSG_BacoWorkAroundFlushVDCI);
+}
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_baco.h b/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_baco.h
index 51c7f8392925..f06471e712dc 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_baco.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_baco.h
@@ -28,5 +28,6 @@
 extern int vega20_baco_get_capability(struct pp_hwmgr *hwmgr, bool *cap);
 extern int vega20_baco_get_state(struct pp_hwmgr *hwmgr, enum BACO_STATE *state);
 extern int vega20_baco_set_state(struct pp_hwmgr *hwmgr, enum BACO_STATE state);
+extern int vega20_baco_apply_vdci_flush_workaround(struct pp_hwmgr *hwmgr);
 
 #endif
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_hwmgr.c
index 23b5b94a4939..9b9f87b84910 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_hwmgr.c
@@ -434,6 +434,7 @@ static int vega20_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
 	hwmgr->platform_descriptor.clockStep.memoryClock = 500;
 
 	data->total_active_cus = adev->gfx.cu_info.number;
+	data->is_custom_profile_set = false;
 
 	return 0;
 }
@@ -450,6 +451,7 @@ static int vega20_init_sclk_threshold(struct pp_hwmgr *hwmgr)
 
 static int vega20_setup_asic_task(struct pp_hwmgr *hwmgr)
 {
+	struct amdgpu_device *adev = (struct amdgpu_device *)(hwmgr->adev);
 	int ret = 0;
 
 	ret = vega20_init_sclk_threshold(hwmgr);
@@ -457,7 +459,15 @@ static int vega20_setup_asic_task(struct pp_hwmgr *hwmgr)
 			"Failed to init sclk threshold!",
 			return ret);
 
-	return 0;
+	if (adev->in_baco_reset) {
+		adev->in_baco_reset = 0;
+
+		ret = vega20_baco_apply_vdci_flush_workaround(hwmgr);
+		if (ret)
+			pr_err("Failed to apply vega20 baco workaround!\n");
+	}
+
+	return ret;
 }
 
 /*
@@ -3450,7 +3460,18 @@ static void vega20_power_gate_vce(struct pp_hwmgr *hwmgr, bool bgate)
 		return ;
 
 	data->vce_power_gated = bgate;
-	vega20_enable_disable_vce_dpm(hwmgr, !bgate);
+	if (bgate) {
+		vega20_enable_disable_vce_dpm(hwmgr, !bgate);
+		amdgpu_device_ip_set_powergating_state(hwmgr->adev,
+						AMD_IP_BLOCK_TYPE_VCE,
+						AMD_PG_STATE_GATE);
+	} else {
+		amdgpu_device_ip_set_powergating_state(hwmgr->adev,
+						AMD_IP_BLOCK_TYPE_VCE,
+						AMD_PG_STATE_UNGATE);
+		vega20_enable_disable_vce_dpm(hwmgr, !bgate);
+	}
+
 }
 
 static void vega20_power_gate_uvd(struct pp_hwmgr *hwmgr, bool bgate)
@@ -3826,16 +3847,19 @@ static int vega20_set_power_profile_mode(struct pp_hwmgr *hwmgr, long *input, ui
 {
 	DpmActivityMonitorCoeffInt_t activity_monitor;
 	int workload_type, result = 0;
+	uint32_t power_profile_mode = input[size];
 
-	hwmgr->power_profile_mode = input[size];
-
-	if (hwmgr->power_profile_mode > PP_SMC_POWER_PROFILE_CUSTOM) {
-		pr_err("Invalid power profile mode %d\n", hwmgr->power_profile_mode);
+	if (power_profile_mode > PP_SMC_POWER_PROFILE_CUSTOM) {
+		pr_err("Invalid power profile mode %d\n", power_profile_mode);
 		return -EINVAL;
 	}
 
-	if (hwmgr->power_profile_mode == PP_SMC_POWER_PROFILE_CUSTOM) {
-		if (size < 10)
+	if (power_profile_mode == PP_SMC_POWER_PROFILE_CUSTOM) {
+		struct vega20_hwmgr *data =
+			(struct vega20_hwmgr *)(hwmgr->backend);
+		if (size == 0 && !data->is_custom_profile_set)
+			return -EINVAL;
+		if (size < 10 && size != 0)
 			return -EINVAL;
 
 		result = vega20_get_activity_monitor_coeff(hwmgr,
@@ -3845,6 +3869,13 @@ static int vega20_set_power_profile_mode(struct pp_hwmgr *hwmgr, long *input, ui
 				"[SetPowerProfile] Failed to get activity monitor!",
 				return result);
 
+		/* If size==0, then we want to apply the already-configured
+		 * CUSTOM profile again. Just apply it, since we checked its
+		 * validity above
+		 */
+		if (size == 0)
+			goto out;
+
 		switch (input[0]) {
 		case 0: /* Gfxclk */
 			activity_monitor.Gfx_FPS = input[1];
@@ -3895,17 +3926,21 @@ static int vega20_set_power_profile_mode(struct pp_hwmgr *hwmgr, long *input, ui
 		result = vega20_set_activity_monitor_coeff(hwmgr,
 				(uint8_t *)(&activity_monitor),
 				WORKLOAD_PPLIB_CUSTOM_BIT);
+		data->is_custom_profile_set = true;
 		PP_ASSERT_WITH_CODE(!result,
 				"[SetPowerProfile] Failed to set activity monitor!",
 				return result);
 	}
 
+out:
 	/* conv PP_SMC_POWER_PROFILE* to WORKLOAD_PPLIB_*_BIT */
 	workload_type =
-		conv_power_profile_to_pplib_workload(hwmgr->power_profile_mode);
+		conv_power_profile_to_pplib_workload(power_profile_mode);
 	smum_send_msg_to_smc_with_parameter(hwmgr, PPSMC_MSG_SetWorkloadMask,
 						1 << workload_type);
 
+	hwmgr->power_profile_mode = power_profile_mode;
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_hwmgr.h b/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_hwmgr.h
index ac2a3118a0ae..2c3125f82b24 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_hwmgr.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/vega20_hwmgr.h
@@ -531,6 +531,8 @@ struct vega20_hwmgr {
 	bool                           pcie_parameters_override;
 	uint32_t                       pcie_gen_level1;
 	uint32_t                       pcie_width_level1;
+
+	bool                           is_custom_profile_set;
 };
 
 #define VEGA20_DPM2_NEAR_TDP_DEC                      10
diff --git a/drivers/gpu/drm/amd/powerplay/inc/amdgpu_smu.h b/drivers/gpu/drm/amd/powerplay/inc/amdgpu_smu.h
new file mode 100644
index 000000000000..c8b168b3413b
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/inc/amdgpu_smu.h
@@ -0,0 +1,770 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef __AMDGPU_SMU_H__
+#define __AMDGPU_SMU_H__
+
+#include "amdgpu.h"
+#include "kgd_pp_interface.h"
+#include "dm_pp_interface.h"
+
+struct smu_hw_power_state {
+	unsigned int magic;
+};
+
+struct smu_power_state;
+
+enum smu_state_ui_label {
+	SMU_STATE_UI_LABEL_NONE,
+	SMU_STATE_UI_LABEL_BATTERY,
+	SMU_STATE_UI_TABEL_MIDDLE_LOW,
+	SMU_STATE_UI_LABEL_BALLANCED,
+	SMU_STATE_UI_LABEL_MIDDLE_HIGHT,
+	SMU_STATE_UI_LABEL_PERFORMANCE,
+	SMU_STATE_UI_LABEL_BACO,
+};
+
+enum smu_state_classification_flag {
+	SMU_STATE_CLASSIFICATION_FLAG_BOOT                     = 0x0001,
+	SMU_STATE_CLASSIFICATION_FLAG_THERMAL                  = 0x0002,
+	SMU_STATE_CLASSIFICATIN_FLAG_LIMITED_POWER_SOURCE      = 0x0004,
+	SMU_STATE_CLASSIFICATION_FLAG_RESET                    = 0x0008,
+	SMU_STATE_CLASSIFICATION_FLAG_FORCED                   = 0x0010,
+	SMU_STATE_CLASSIFICATION_FLAG_USER_3D_PERFORMANCE      = 0x0020,
+	SMU_STATE_CLASSIFICATION_FLAG_USER_2D_PERFORMANCE      = 0x0040,
+	SMU_STATE_CLASSIFICATION_FLAG_3D_PERFORMANCE           = 0x0080,
+	SMU_STATE_CLASSIFICATION_FLAG_AC_OVERDIRVER_TEMPLATE   = 0x0100,
+	SMU_STATE_CLASSIFICATION_FLAG_UVD                      = 0x0200,
+	SMU_STATE_CLASSIFICATION_FLAG_3D_PERFORMANCE_LOW       = 0x0400,
+	SMU_STATE_CLASSIFICATION_FLAG_ACPI                     = 0x0800,
+	SMU_STATE_CLASSIFICATION_FLAG_HD2                      = 0x1000,
+	SMU_STATE_CLASSIFICATION_FLAG_UVD_HD                   = 0x2000,
+	SMU_STATE_CLASSIFICATION_FLAG_UVD_SD                   = 0x4000,
+	SMU_STATE_CLASSIFICATION_FLAG_USER_DC_PERFORMANCE      = 0x8000,
+	SMU_STATE_CLASSIFICATION_FLAG_DC_OVERDIRVER_TEMPLATE   = 0x10000,
+	SMU_STATE_CLASSIFICATION_FLAG_BACO                     = 0x20000,
+	SMU_STATE_CLASSIFICATIN_FLAG_LIMITED_POWER_SOURCE2      = 0x40000,
+	SMU_STATE_CLASSIFICATION_FLAG_ULV                      = 0x80000,
+	SMU_STATE_CLASSIFICATION_FLAG_UVD_MVC                  = 0x100000,
+};
+
+struct smu_state_classification_block {
+	enum smu_state_ui_label         ui_label;
+	enum smu_state_classification_flag  flags;
+	int                          bios_index;
+	bool                      temporary_state;
+	bool                      to_be_deleted;
+};
+
+struct smu_state_pcie_block {
+	unsigned int lanes;
+};
+
+enum smu_refreshrate_source {
+	SMU_REFRESHRATE_SOURCE_EDID,
+	SMU_REFRESHRATE_SOURCE_EXPLICIT
+};
+
+struct smu_state_display_block {
+	bool              disable_frame_modulation;
+	bool              limit_refreshrate;
+	enum smu_refreshrate_source refreshrate_source;
+	int                  explicit_refreshrate;
+	int                  edid_refreshrate_index;
+	bool              enable_vari_bright;
+};
+
+struct smu_state_memroy_block {
+	bool              dll_off;
+	uint8_t                 m3arb;
+	uint8_t                 unused[3];
+};
+
+struct smu_state_software_algorithm_block {
+	bool disable_load_balancing;
+	bool enable_sleep_for_timestamps;
+};
+
+struct smu_temperature_range {
+	int min;
+	int max;
+};
+
+struct smu_state_validation_block {
+	bool single_display_only;
+	bool disallow_on_dc;
+	uint8_t supported_power_levels;
+};
+
+struct smu_uvd_clocks {
+	uint32_t vclk;
+	uint32_t dclk;
+};
+
+/**
+* Structure to hold a SMU Power State.
+*/
+struct smu_power_state {
+	uint32_t                                      id;
+	struct list_head                              ordered_list;
+	struct list_head                              all_states_list;
+
+	struct smu_state_classification_block         classification;
+	struct smu_state_validation_block             validation;
+	struct smu_state_pcie_block                   pcie;
+	struct smu_state_display_block                display;
+	struct smu_state_memroy_block                 memory;
+	struct smu_temperature_range                  temperatures;
+	struct smu_state_software_algorithm_block     software;
+	struct smu_uvd_clocks                         uvd_clocks;
+	struct smu_hw_power_state                     hardware;
+};
+
+enum smu_message_type
+{
+	SMU_MSG_TestMessage = 0,
+	SMU_MSG_GetSmuVersion,
+	SMU_MSG_GetDriverIfVersion,
+	SMU_MSG_SetAllowedFeaturesMaskLow,
+	SMU_MSG_SetAllowedFeaturesMaskHigh,
+	SMU_MSG_EnableAllSmuFeatures,
+	SMU_MSG_DisableAllSmuFeatures,
+	SMU_MSG_EnableSmuFeaturesLow,
+	SMU_MSG_EnableSmuFeaturesHigh,
+	SMU_MSG_DisableSmuFeaturesLow,
+	SMU_MSG_DisableSmuFeaturesHigh,
+	SMU_MSG_GetEnabledSmuFeaturesLow,
+	SMU_MSG_GetEnabledSmuFeaturesHigh,
+	SMU_MSG_SetWorkloadMask,
+	SMU_MSG_SetPptLimit,
+	SMU_MSG_SetDriverDramAddrHigh,
+	SMU_MSG_SetDriverDramAddrLow,
+	SMU_MSG_SetToolsDramAddrHigh,
+	SMU_MSG_SetToolsDramAddrLow,
+	SMU_MSG_TransferTableSmu2Dram,
+	SMU_MSG_TransferTableDram2Smu,
+	SMU_MSG_UseDefaultPPTable,
+	SMU_MSG_UseBackupPPTable,
+	SMU_MSG_RunBtc,
+	SMU_MSG_RequestI2CBus,
+	SMU_MSG_ReleaseI2CBus,
+	SMU_MSG_SetFloorSocVoltage,
+	SMU_MSG_SoftReset,
+	SMU_MSG_StartBacoMonitor,
+	SMU_MSG_CancelBacoMonitor,
+	SMU_MSG_EnterBaco,
+	SMU_MSG_SetSoftMinByFreq,
+	SMU_MSG_SetSoftMaxByFreq,
+	SMU_MSG_SetHardMinByFreq,
+	SMU_MSG_SetHardMaxByFreq,
+	SMU_MSG_GetMinDpmFreq,
+	SMU_MSG_GetMaxDpmFreq,
+	SMU_MSG_GetDpmFreqByIndex,
+	SMU_MSG_GetDpmClockFreq,
+	SMU_MSG_GetSsVoltageByDpm,
+	SMU_MSG_SetMemoryChannelConfig,
+	SMU_MSG_SetGeminiMode,
+	SMU_MSG_SetGeminiApertureHigh,
+	SMU_MSG_SetGeminiApertureLow,
+	SMU_MSG_SetMinLinkDpmByIndex,
+	SMU_MSG_OverridePcieParameters,
+	SMU_MSG_OverDriveSetPercentage,
+	SMU_MSG_SetMinDeepSleepDcefclk,
+	SMU_MSG_ReenableAcDcInterrupt,
+	SMU_MSG_NotifyPowerSource,
+	SMU_MSG_SetUclkFastSwitch,
+	SMU_MSG_SetUclkDownHyst,
+	SMU_MSG_GfxDeviceDriverReset,
+	SMU_MSG_GetCurrentRpm,
+	SMU_MSG_SetVideoFps,
+	SMU_MSG_SetTjMax,
+	SMU_MSG_SetFanTemperatureTarget,
+	SMU_MSG_PrepareMp1ForUnload,
+	SMU_MSG_DramLogSetDramAddrHigh,
+	SMU_MSG_DramLogSetDramAddrLow,
+	SMU_MSG_DramLogSetDramSize,
+	SMU_MSG_SetFanMaxRpm,
+	SMU_MSG_SetFanMinPwm,
+	SMU_MSG_ConfigureGfxDidt,
+	SMU_MSG_NumOfDisplays,
+	SMU_MSG_RemoveMargins,
+	SMU_MSG_ReadSerialNumTop32,
+	SMU_MSG_ReadSerialNumBottom32,
+	SMU_MSG_SetSystemVirtualDramAddrHigh,
+	SMU_MSG_SetSystemVirtualDramAddrLow,
+	SMU_MSG_WaflTest,
+	SMU_MSG_SetFclkGfxClkRatio,
+	SMU_MSG_AllowGfxOff,
+	SMU_MSG_DisallowGfxOff,
+	SMU_MSG_GetPptLimit,
+	SMU_MSG_GetDcModeMaxDpmFreq,
+	SMU_MSG_GetDebugData,
+	SMU_MSG_SetXgmiMode,
+	SMU_MSG_RunAfllBtc,
+	SMU_MSG_ExitBaco,
+	SMU_MSG_PrepareMp1ForReset,
+	SMU_MSG_PrepareMp1ForShutdown,
+	SMU_MSG_SetMGpuFanBoostLimitRpm,
+	SMU_MSG_GetAVFSVoltageByDpm,
+	SMU_MSG_MAX_COUNT,
+};
+
+enum smu_memory_pool_size
+{
+    SMU_MEMORY_POOL_SIZE_ZERO   = 0,
+    SMU_MEMORY_POOL_SIZE_256_MB = 0x10000000,
+    SMU_MEMORY_POOL_SIZE_512_MB = 0x20000000,
+    SMU_MEMORY_POOL_SIZE_1_GB   = 0x40000000,
+    SMU_MEMORY_POOL_SIZE_2_GB   = 0x80000000,
+};
+
+#define SMU_TABLE_INIT(tables, table_id, s, a, d)	\
+	do {						\
+		tables[table_id].size = s;		\
+		tables[table_id].align = a;		\
+		tables[table_id].domain = d;		\
+	} while (0)
+
+struct smu_table {
+	uint64_t size;
+	uint32_t align;
+	uint8_t domain;
+	uint64_t mc_address;
+	void *cpu_addr;
+	struct amdgpu_bo *bo;
+};
+
+enum smu_perf_level_designation {
+	PERF_LEVEL_ACTIVITY,
+	PERF_LEVEL_POWER_CONTAINMENT,
+};
+
+struct smu_performance_level {
+	uint32_t core_clock;
+	uint32_t memory_clock;
+	uint32_t vddc;
+	uint32_t vddci;
+	uint32_t non_local_mem_freq;
+	uint32_t non_local_mem_width;
+};
+
+struct smu_clock_info {
+	uint32_t min_mem_clk;
+	uint32_t max_mem_clk;
+	uint32_t min_eng_clk;
+	uint32_t max_eng_clk;
+	uint32_t min_bus_bandwidth;
+	uint32_t max_bus_bandwidth;
+};
+
+struct smu_bios_boot_up_values
+{
+	uint32_t			revision;
+	uint32_t			gfxclk;
+	uint32_t			uclk;
+	uint32_t			socclk;
+	uint32_t			dcefclk;
+	uint32_t			eclk;
+	uint32_t			vclk;
+	uint32_t			dclk;
+	uint16_t			vddc;
+	uint16_t			vddci;
+	uint16_t			mvddc;
+	uint16_t			vdd_gfx;
+	uint8_t				cooling_id;
+	uint32_t			pp_table_id;
+};
+
+struct smu_table_context
+{
+	void				*power_play_table;
+	uint32_t			power_play_table_size;
+	void				*hardcode_pptable;
+
+	void				*max_sustainable_clocks;
+	struct smu_bios_boot_up_values	boot_values;
+	void                            *driver_pptable;
+	struct smu_table		*tables;
+	uint32_t			table_count;
+	struct smu_table		memory_pool;
+	uint16_t                        software_shutdown_temp;
+	uint8_t                         thermal_controller_type;
+	uint16_t			TDPODLimit;
+
+	uint8_t				*od_feature_capabilities;
+	uint32_t			*od_settings_max;
+	uint32_t			*od_settings_min;
+	void				*overdrive_table;
+	void				*od8_settings;
+	bool				od_gfxclk_update;
+	bool				od_memclk_update;
+};
+
+struct smu_dpm_context {
+	uint32_t dpm_context_size;
+	void *dpm_context;
+	void *golden_dpm_context;
+	bool enable_umd_pstate;
+	enum amd_dpm_forced_level dpm_level;
+	enum amd_dpm_forced_level saved_dpm_level;
+	enum amd_dpm_forced_level requested_dpm_level;
+	struct smu_power_state *dpm_request_power_state;
+	struct smu_power_state *dpm_current_power_state;
+	struct mclock_latency_table *mclk_latency_table;
+};
+
+struct smu_power_context {
+	void *power_context;
+	uint32_t power_context_size;
+};
+
+
+#define SMU_FEATURE_MAX	(64)
+struct smu_feature
+{
+	uint32_t feature_num;
+	DECLARE_BITMAP(supported, SMU_FEATURE_MAX);
+	DECLARE_BITMAP(allowed, SMU_FEATURE_MAX);
+	DECLARE_BITMAP(enabled, SMU_FEATURE_MAX);
+	struct mutex mutex;
+};
+
+struct smu_clocks {
+	uint32_t engine_clock;
+	uint32_t memory_clock;
+	uint32_t bus_bandwidth;
+	uint32_t engine_clock_in_sr;
+	uint32_t dcef_clock;
+	uint32_t dcef_clock_in_sr;
+};
+
+#define MAX_REGULAR_DPM_NUM 16
+struct mclk_latency_entries {
+	uint32_t  frequency;
+	uint32_t  latency;
+};
+struct mclock_latency_table {
+	uint32_t  count;
+	struct mclk_latency_entries  entries[MAX_REGULAR_DPM_NUM];
+};
+
+#define WORKLOAD_POLICY_MAX 7
+struct smu_context
+{
+	struct amdgpu_device            *adev;
+
+	const struct smu_funcs		*funcs;
+	const struct pptable_funcs	*ppt_funcs;
+	struct mutex			mutex;
+	uint64_t pool_size;
+
+	struct smu_table_context	smu_table;
+	struct smu_dpm_context		smu_dpm;
+	struct smu_power_context	smu_power;
+	struct smu_feature		smu_feature;
+	struct amd_pp_display_configuration  *display_config;
+
+	uint32_t pstate_sclk;
+	uint32_t pstate_mclk;
+
+	bool od_enabled;
+	uint32_t power_limit;
+	uint32_t default_power_limit;
+
+	bool support_power_containment;
+	bool disable_watermark;
+
+#define WATERMARKS_EXIST	(1 << 0)
+#define WATERMARKS_LOADED	(1 << 1)
+	uint32_t watermarks_bitmap;
+
+	uint32_t workload_mask;
+	uint32_t workload_prority[WORKLOAD_POLICY_MAX];
+	uint32_t workload_setting[WORKLOAD_POLICY_MAX];
+	uint32_t power_profile_mode;
+	uint32_t default_power_profile_mode;
+
+	uint32_t smc_if_version;
+};
+
+struct pptable_funcs {
+	int (*alloc_dpm_context)(struct smu_context *smu);
+	int (*store_powerplay_table)(struct smu_context *smu);
+	int (*check_powerplay_table)(struct smu_context *smu);
+	int (*append_powerplay_table)(struct smu_context *smu);
+	int (*get_smu_msg_index)(struct smu_context *smu, uint32_t index);
+	int (*run_afll_btc)(struct smu_context *smu);
+	int (*get_unallowed_feature_mask)(struct smu_context *smu, uint32_t *feature_mask, uint32_t num);
+	enum amd_pm_state_type (*get_current_power_state)(struct smu_context *smu);
+	int (*set_default_dpm_table)(struct smu_context *smu);
+	int (*set_power_state)(struct smu_context *smu);
+	int (*populate_umd_state_clk)(struct smu_context *smu);
+	int (*print_clk_levels)(struct smu_context *smu, enum pp_clock_type type, char *buf);
+	int (*force_clk_levels)(struct smu_context *smu, enum pp_clock_type type, uint32_t mask);
+	int (*set_default_od8_settings)(struct smu_context *smu);
+	int (*update_specified_od8_value)(struct smu_context *smu,
+					  uint32_t index,
+					  uint32_t value);
+	int (*get_od_percentage)(struct smu_context *smu, enum pp_clock_type type);
+	int (*set_od_percentage)(struct smu_context *smu,
+				 enum pp_clock_type type,
+				 uint32_t value);
+	int (*od_edit_dpm_table)(struct smu_context *smu,
+				 enum PP_OD_DPM_TABLE_COMMAND type,
+				 long *input, uint32_t size);
+	int (*get_clock_by_type_with_latency)(struct smu_context *smu,
+					      enum amd_pp_clock_type type,
+					      struct
+					      pp_clock_levels_with_latency
+					      *clocks);
+	int (*get_clock_by_type_with_voltage)(struct smu_context *smu,
+					      enum amd_pp_clock_type type,
+					      struct
+					      pp_clock_levels_with_voltage
+					      *clocks);
+	int (*get_power_profile_mode)(struct smu_context *smu, char *buf);
+	int (*set_power_profile_mode)(struct smu_context *smu, long *input, uint32_t size);
+	enum amd_dpm_forced_level (*get_performance_level)(struct smu_context *smu);
+	int (*force_performance_level)(struct smu_context *smu, enum amd_dpm_forced_level level);
+	int (*pre_display_config_changed)(struct smu_context *smu);
+	int (*display_config_changed)(struct smu_context *smu);
+	int (*apply_clocks_adjust_rules)(struct smu_context *smu);
+	int (*notify_smc_dispaly_config)(struct smu_context *smu);
+	int (*force_dpm_limit_value)(struct smu_context *smu, bool highest);
+	int (*unforce_dpm_levels)(struct smu_context *smu);
+	int (*upload_dpm_level)(struct smu_context *smu, bool max,
+				uint32_t feature_mask);
+	int (*get_profiling_clk_mask)(struct smu_context *smu,
+				      enum amd_dpm_forced_level level,
+				      uint32_t *sclk_mask,
+				      uint32_t *mclk_mask,
+				      uint32_t *soc_mask);
+	int (*set_cpu_power_state)(struct smu_context *smu);
+};
+
+struct smu_funcs
+{
+	int (*init_microcode)(struct smu_context *smu);
+	int (*init_smc_tables)(struct smu_context *smu);
+	int (*fini_smc_tables)(struct smu_context *smu);
+	int (*init_power)(struct smu_context *smu);
+	int (*fini_power)(struct smu_context *smu);
+	int (*load_microcode)(struct smu_context *smu);
+	int (*check_fw_status)(struct smu_context *smu);
+	int (*read_pptable_from_vbios)(struct smu_context *smu);
+	int (*get_vbios_bootup_values)(struct smu_context *smu);
+	int (*get_clk_info_from_vbios)(struct smu_context *smu);
+	int (*check_pptable)(struct smu_context *smu);
+	int (*parse_pptable)(struct smu_context *smu);
+	int (*populate_smc_pptable)(struct smu_context *smu);
+	int (*check_fw_version)(struct smu_context *smu);
+	int (*write_pptable)(struct smu_context *smu);
+	int (*set_min_dcef_deep_sleep)(struct smu_context *smu);
+	int (*set_tool_table_location)(struct smu_context *smu);
+	int (*notify_memory_pool_location)(struct smu_context *smu);
+	int (*write_watermarks_table)(struct smu_context *smu);
+	int (*set_last_dcef_min_deep_sleep_clk)(struct smu_context *smu);
+	int (*system_features_control)(struct smu_context *smu, bool en);
+	int (*send_smc_msg)(struct smu_context *smu, uint16_t msg);
+	int (*send_smc_msg_with_param)(struct smu_context *smu, uint16_t msg, uint32_t param);
+	int (*read_smc_arg)(struct smu_context *smu, uint32_t *arg);
+	int (*init_display)(struct smu_context *smu);
+	int (*set_allowed_mask)(struct smu_context *smu);
+	int (*get_enabled_mask)(struct smu_context *smu, uint32_t *feature_mask, uint32_t num);
+	bool (*is_dpm_running)(struct smu_context *smu);
+	int (*update_feature_enable_state)(struct smu_context *smu, uint32_t feature_id, bool enabled);
+	int (*notify_display_change)(struct smu_context *smu);
+	int (*get_power_limit)(struct smu_context *smu, uint32_t *limit, bool def);
+	int (*set_power_limit)(struct smu_context *smu, uint32_t n);
+	int (*get_current_clk_freq)(struct smu_context *smu, uint32_t clk_id, uint32_t *value);
+	int (*init_max_sustainable_clocks)(struct smu_context *smu);
+	int (*start_thermal_control)(struct smu_context *smu);
+	int (*read_sensor)(struct smu_context *smu, enum amd_pp_sensors sensor,
+			   void *data, uint32_t *size);
+	int (*set_deep_sleep_dcefclk)(struct smu_context *smu, uint32_t clk);
+	int (*set_active_display_count)(struct smu_context *smu, uint32_t count);
+	int (*store_cc6_data)(struct smu_context *smu, uint32_t separation_time,
+			      bool cc6_disable, bool pstate_disable,
+			      bool pstate_switch_disable);
+	int (*get_clock_by_type)(struct smu_context *smu,
+				 enum amd_pp_clock_type type,
+				 struct amd_pp_clocks *clocks);
+	int (*get_max_high_clocks)(struct smu_context *smu,
+				   struct amd_pp_simple_clock_info *clocks);
+	int (*display_clock_voltage_request)(struct smu_context *smu, struct
+					     pp_display_clock_request
+					     *clock_req);
+	int (*get_dal_power_level)(struct smu_context *smu,
+				   struct amd_pp_simple_clock_info *clocks);
+	int (*get_perf_level)(struct smu_context *smu,
+			      enum smu_perf_level_designation designation,
+			      struct smu_performance_level *level);
+	int (*get_current_shallow_sleep_clocks)(struct smu_context *smu,
+						struct smu_clock_info *clocks);
+	int (*notify_smu_enable_pwe)(struct smu_context *smu);
+	int (*set_watermarks_for_clock_ranges)(struct smu_context *smu,
+					       struct dm_pp_wm_sets_with_clock_ranges_soc15 *clock_ranges);
+	int (*set_od8_default_settings)(struct smu_context *smu,
+					bool initialize);
+	int (*conv_power_profile_to_pplib_workload)(int power_profile);
+	int (*get_power_profile_mode)(struct smu_context *smu, char *buf);
+	int (*set_power_profile_mode)(struct smu_context *smu, long *input, uint32_t size);
+	int (*update_od8_settings)(struct smu_context *smu,
+				   uint32_t index,
+				   uint32_t value);
+	int (*dpm_set_uvd_enable)(struct smu_context *smu, bool enable);
+	int (*dpm_set_vce_enable)(struct smu_context *smu, bool enable);
+	uint32_t (*get_sclk)(struct smu_context *smu, bool low);
+	uint32_t (*get_mclk)(struct smu_context *smu, bool low);
+	int (*get_current_rpm)(struct smu_context *smu, uint32_t *speed);
+	uint32_t (*get_fan_control_mode)(struct smu_context *smu);
+	int (*set_fan_control_mode)(struct smu_context *smu, uint32_t mode);
+	int (*get_fan_speed_percent)(struct smu_context *smu, uint32_t *speed);
+	int (*set_fan_speed_percent)(struct smu_context *smu, uint32_t speed);
+	int (*set_fan_speed_rpm)(struct smu_context *smu, uint32_t speed);
+	int (*set_xgmi_pstate)(struct smu_context *smu, uint32_t pstate);
+
+};
+
+#define smu_init_microcode(smu) \
+	((smu)->funcs->init_microcode ? (smu)->funcs->init_microcode((smu)) : 0)
+#define smu_init_smc_tables(smu) \
+	((smu)->funcs->init_smc_tables ? (smu)->funcs->init_smc_tables((smu)) : 0)
+#define smu_fini_smc_tables(smu) \
+	((smu)->funcs->fini_smc_tables ? (smu)->funcs->fini_smc_tables((smu)) : 0)
+#define smu_init_power(smu) \
+	((smu)->funcs->init_power ? (smu)->funcs->init_power((smu)) : 0)
+#define smu_fini_power(smu) \
+	((smu)->funcs->fini_power ? (smu)->funcs->fini_power((smu)) : 0)
+#define smu_load_microcode(smu) \
+	((smu)->funcs->load_microcode ? (smu)->funcs->load_microcode((smu)) : 0)
+#define smu_check_fw_status(smu) \
+	((smu)->funcs->check_fw_status ? (smu)->funcs->check_fw_status((smu)) : 0)
+#define smu_read_pptable_from_vbios(smu) \
+	((smu)->funcs->read_pptable_from_vbios ? (smu)->funcs->read_pptable_from_vbios((smu)) : 0)
+#define smu_get_vbios_bootup_values(smu) \
+	((smu)->funcs->get_vbios_bootup_values ? (smu)->funcs->get_vbios_bootup_values((smu)) : 0)
+#define smu_get_clk_info_from_vbios(smu) \
+	((smu)->funcs->get_clk_info_from_vbios ? (smu)->funcs->get_clk_info_from_vbios((smu)) : 0)
+#define smu_check_pptable(smu) \
+	((smu)->funcs->check_pptable ? (smu)->funcs->check_pptable((smu)) : 0)
+#define smu_parse_pptable(smu) \
+	((smu)->funcs->parse_pptable ? (smu)->funcs->parse_pptable((smu)) : 0)
+#define smu_populate_smc_pptable(smu) \
+	((smu)->funcs->populate_smc_pptable ? (smu)->funcs->populate_smc_pptable((smu)) : 0)
+#define smu_check_fw_version(smu) \
+	((smu)->funcs->check_fw_version ? (smu)->funcs->check_fw_version((smu)) : 0)
+#define smu_write_pptable(smu) \
+	((smu)->funcs->write_pptable ? (smu)->funcs->write_pptable((smu)) : 0)
+#define smu_set_min_dcef_deep_sleep(smu) \
+	((smu)->funcs->set_min_dcef_deep_sleep ? (smu)->funcs->set_min_dcef_deep_sleep((smu)) : 0)
+#define smu_set_tool_table_location(smu) \
+	((smu)->funcs->set_tool_table_location ? (smu)->funcs->set_tool_table_location((smu)) : 0)
+#define smu_notify_memory_pool_location(smu) \
+	((smu)->funcs->notify_memory_pool_location ? (smu)->funcs->notify_memory_pool_location((smu)) : 0)
+#define smu_write_watermarks_table(smu) \
+	((smu)->funcs->write_watermarks_table ? (smu)->funcs->write_watermarks_table((smu)) : 0)
+#define smu_set_last_dcef_min_deep_sleep_clk(smu) \
+	((smu)->funcs->set_last_dcef_min_deep_sleep_clk ? (smu)->funcs->set_last_dcef_min_deep_sleep_clk((smu)) : 0)
+#define smu_system_features_control(smu, en) \
+	((smu)->funcs->system_features_control ? (smu)->funcs->system_features_control((smu), (en)) : 0)
+#define smu_init_max_sustainable_clocks(smu) \
+	((smu)->funcs->init_max_sustainable_clocks ? (smu)->funcs->init_max_sustainable_clocks((smu)) : 0)
+#define smu_set_od8_default_settings(smu, initialize) \
+	((smu)->funcs->set_od8_default_settings ? (smu)->funcs->set_od8_default_settings((smu), (initialize)) : 0)
+#define smu_update_od8_settings(smu, index, value) \
+	((smu)->funcs->update_od8_settings ? (smu)->funcs->update_od8_settings((smu), (index), (value)) : 0)
+#define smu_get_current_rpm(smu, speed) \
+	((smu)->funcs->get_current_rpm ? (smu)->funcs->get_current_rpm((smu), (speed)) : 0)
+#define smu_set_fan_speed_rpm(smu, speed) \
+	((smu)->funcs->set_fan_speed_rpm ? (smu)->funcs->set_fan_speed_rpm((smu), (speed)) : 0)
+#define smu_send_smc_msg(smu, msg) \
+	((smu)->funcs->send_smc_msg? (smu)->funcs->send_smc_msg((smu), (msg)) : 0)
+#define smu_send_smc_msg_with_param(smu, msg, param) \
+	((smu)->funcs->send_smc_msg_with_param? (smu)->funcs->send_smc_msg_with_param((smu), (msg), (param)) : 0)
+#define smu_read_smc_arg(smu, arg) \
+	((smu)->funcs->read_smc_arg? (smu)->funcs->read_smc_arg((smu), (arg)) : 0)
+#define smu_alloc_dpm_context(smu) \
+	((smu)->ppt_funcs->alloc_dpm_context ? (smu)->ppt_funcs->alloc_dpm_context((smu)) : 0)
+#define smu_init_display(smu) \
+	((smu)->funcs->init_display ? (smu)->funcs->init_display((smu)) : 0)
+#define smu_feature_set_allowed_mask(smu) \
+	((smu)->funcs->set_allowed_mask? (smu)->funcs->set_allowed_mask((smu)) : 0)
+#define smu_feature_get_enabled_mask(smu, mask, num) \
+	((smu)->funcs->get_enabled_mask? (smu)->funcs->get_enabled_mask((smu), (mask), (num)) : 0)
+#define smu_is_dpm_running(smu) \
+	((smu)->funcs->is_dpm_running ? (smu)->funcs->is_dpm_running((smu)) : 0)
+#define smu_feature_update_enable_state(smu, feature_id, enabled) \
+	((smu)->funcs->update_feature_enable_state? (smu)->funcs->update_feature_enable_state((smu), (feature_id), (enabled)) : 0)
+#define smu_notify_display_change(smu) \
+	((smu)->funcs->notify_display_change? (smu)->funcs->notify_display_change((smu)) : 0)
+#define smu_store_powerplay_table(smu) \
+	((smu)->ppt_funcs->store_powerplay_table ? (smu)->ppt_funcs->store_powerplay_table((smu)) : 0)
+#define smu_check_powerplay_table(smu) \
+	((smu)->ppt_funcs->check_powerplay_table ? (smu)->ppt_funcs->check_powerplay_table((smu)) : 0)
+#define smu_append_powerplay_table(smu) \
+	((smu)->ppt_funcs->append_powerplay_table ? (smu)->ppt_funcs->append_powerplay_table((smu)) : 0)
+#define smu_set_default_dpm_table(smu) \
+	((smu)->ppt_funcs->set_default_dpm_table ? (smu)->ppt_funcs->set_default_dpm_table((smu)) : 0)
+#define smu_populate_umd_state_clk(smu) \
+	((smu)->ppt_funcs->populate_umd_state_clk ? (smu)->ppt_funcs->populate_umd_state_clk((smu)) : 0)
+#define smu_set_default_od8_settings(smu) \
+	((smu)->ppt_funcs->set_default_od8_settings ? (smu)->ppt_funcs->set_default_od8_settings((smu)) : 0)
+#define smu_update_specified_od8_value(smu, index, value) \
+	((smu)->ppt_funcs->update_specified_od8_value ? (smu)->ppt_funcs->update_specified_od8_value((smu), (index), (value)) : 0)
+#define smu_get_power_limit(smu, limit, def) \
+	((smu)->funcs->get_power_limit ? (smu)->funcs->get_power_limit((smu), (limit), (def)) : 0)
+#define smu_set_power_limit(smu, limit) \
+	((smu)->funcs->set_power_limit ? (smu)->funcs->set_power_limit((smu), (limit)) : 0)
+#define smu_get_current_clk_freq(smu, clk_id, value) \
+	((smu)->funcs->get_current_clk_freq? (smu)->funcs->get_current_clk_freq((smu), (clk_id), (value)) : 0)
+#define smu_print_clk_levels(smu, type, buf) \
+	((smu)->ppt_funcs->print_clk_levels ? (smu)->ppt_funcs->print_clk_levels((smu), (type), (buf)) : 0)
+#define smu_force_clk_levels(smu, type, level) \
+	((smu)->ppt_funcs->force_clk_levels ? (smu)->ppt_funcs->force_clk_levels((smu), (type), (level)) : 0)
+#define smu_get_od_percentage(smu, type) \
+	((smu)->ppt_funcs->get_od_percentage ? (smu)->ppt_funcs->get_od_percentage((smu), (type)) : 0)
+#define smu_set_od_percentage(smu, type, value) \
+	((smu)->ppt_funcs->set_od_percentage ? (smu)->ppt_funcs->set_od_percentage((smu), (type), (value)) : 0)
+#define smu_od_edit_dpm_table(smu, type, input, size) \
+	((smu)->ppt_funcs->od_edit_dpm_table ? (smu)->ppt_funcs->od_edit_dpm_table((smu), (type), (input), (size)) : 0)
+#define smu_start_thermal_control(smu) \
+	((smu)->funcs->start_thermal_control? (smu)->funcs->start_thermal_control((smu)) : 0)
+#define smu_read_sensor(smu, sensor, data, size) \
+	((smu)->funcs->read_sensor? (smu)->funcs->read_sensor((smu), (sensor), (data), (size)) : 0)
+#define smu_get_power_profile_mode(smu, buf) \
+	((smu)->funcs->get_power_profile_mode ? (smu)->funcs->get_power_profile_mode((smu), buf) : 0)
+#define smu_set_power_profile_mode(smu, param, param_size) \
+	((smu)->funcs->set_power_profile_mode ? (smu)->funcs->set_power_profile_mode((smu), (param), (param_size)) : 0)
+#define smu_get_performance_level(smu) \
+	((smu)->ppt_funcs->get_performance_level ? (smu)->ppt_funcs->get_performance_level((smu)) : 0)
+#define smu_force_performance_level(smu, level) \
+	((smu)->ppt_funcs->force_performance_level ? (smu)->ppt_funcs->force_performance_level((smu), (level)) : 0)
+#define smu_pre_display_config_changed(smu) \
+	((smu)->ppt_funcs->pre_display_config_changed ? (smu)->ppt_funcs->pre_display_config_changed((smu)) : 0)
+#define smu_display_config_changed(smu) \
+	((smu)->ppt_funcs->display_config_changed ? (smu)->ppt_funcs->display_config_changed((smu)) : 0)
+#define smu_apply_clocks_adjust_rules(smu) \
+	((smu)->ppt_funcs->apply_clocks_adjust_rules ? (smu)->ppt_funcs->apply_clocks_adjust_rules((smu)) : 0)
+#define smu_notify_smc_dispaly_config(smu) \
+	((smu)->ppt_funcs->notify_smc_dispaly_config ? (smu)->ppt_funcs->notify_smc_dispaly_config((smu)) : 0)
+#define smu_force_dpm_limit_value(smu, highest) \
+	((smu)->ppt_funcs->force_dpm_limit_value ? (smu)->ppt_funcs->force_dpm_limit_value((smu), (highest)) : 0)
+#define smu_unforce_dpm_levels(smu) \
+	((smu)->ppt_funcs->unforce_dpm_levels ? (smu)->ppt_funcs->unforce_dpm_levels((smu)) : 0)
+#define smu_upload_dpm_level(smu, max, feature_mask) \
+	((smu)->ppt_funcs->upload_dpm_level ? (smu)->ppt_funcs->upload_dpm_level((smu), (max), (feature_mask)) : 0)
+#define smu_get_profiling_clk_mask(smu, level, sclk_mask, mclk_mask, soc_mask) \
+	((smu)->ppt_funcs->get_profiling_clk_mask ? (smu)->ppt_funcs->get_profiling_clk_mask((smu), (level), (sclk_mask), (mclk_mask), (soc_mask)) : 0)
+#define smu_set_cpu_power_state(smu) \
+	((smu)->ppt_funcs->set_cpu_power_state ? (smu)->ppt_funcs->set_cpu_power_state((smu)) : 0)
+#define smu_get_fan_control_mode(smu) \
+	((smu)->funcs->get_fan_control_mode ? (smu)->funcs->get_fan_control_mode((smu)) : 0)
+#define smu_set_fan_control_mode(smu, value) \
+	((smu)->funcs->set_fan_control_mode ? (smu)->funcs->set_fan_control_mode((smu), (value)) : 0)
+#define smu_get_fan_speed_percent(smu, speed) \
+	((smu)->funcs->get_fan_speed_percent ? (smu)->funcs->get_fan_speed_percent((smu), (speed)) : 0)
+#define smu_set_fan_speed_percent(smu, speed) \
+	((smu)->funcs->set_fan_speed_percent ? (smu)->funcs->set_fan_speed_percent((smu), (speed)) : 0)
+
+#define smu_msg_get_index(smu, msg) \
+	((smu)->ppt_funcs? ((smu)->ppt_funcs->get_smu_msg_index? (smu)->ppt_funcs->get_smu_msg_index((smu), (msg)) : -EINVAL) : -EINVAL)
+#define smu_run_afll_btc(smu) \
+	((smu)->ppt_funcs? ((smu)->ppt_funcs->run_afll_btc? (smu)->ppt_funcs->run_afll_btc((smu)) : 0) : 0)
+#define smu_get_unallowed_feature_mask(smu, feature_mask, num) \
+	((smu)->ppt_funcs? ((smu)->ppt_funcs->get_unallowed_feature_mask? (smu)->ppt_funcs->get_unallowed_feature_mask((smu), (feature_mask), (num)) : 0) : 0)
+#define smu_set_deep_sleep_dcefclk(smu, clk) \
+	((smu)->funcs->set_deep_sleep_dcefclk ? (smu)->funcs->set_deep_sleep_dcefclk((smu), (clk)) : 0)
+#define smu_set_active_display_count(smu, count) \
+	((smu)->funcs->set_active_display_count ? (smu)->funcs->set_active_display_count((smu), (count)) : 0)
+#define smu_store_cc6_data(smu, st, cc6_dis, pst_dis, pst_sw_dis) \
+	((smu)->funcs->store_cc6_data ? (smu)->funcs->store_cc6_data((smu), (st), (cc6_dis), (pst_dis), (pst_sw_dis)) : 0)
+#define smu_get_clock_by_type(smu, type, clocks) \
+	((smu)->funcs->get_clock_by_type ? (smu)->funcs->get_clock_by_type((smu), (type), (clocks)) : 0)
+#define smu_get_max_high_clocks(smu, clocks) \
+	((smu)->funcs->get_max_high_clocks ? (smu)->funcs->get_max_high_clocks((smu), (clocks)) : 0)
+#define smu_get_clock_by_type_with_latency(smu, type, clocks) \
+	((smu)->ppt_funcs->get_clock_by_type_with_latency ? (smu)->ppt_funcs->get_clock_by_type_with_latency((smu), (type), (clocks)) : 0)
+#define smu_get_clock_by_type_with_voltage(smu, type, clocks) \
+	((smu)->ppt_funcs->get_clock_by_type_with_voltage ? (smu)->ppt_funcs->get_clock_by_type_with_voltage((smu), (type), (clocks)) : 0)
+#define smu_display_clock_voltage_request(smu, clock_req) \
+	((smu)->funcs->display_clock_voltage_request ? (smu)->funcs->display_clock_voltage_request((smu), (clock_req)) : 0)
+#define smu_get_dal_power_level(smu, clocks) \
+	((smu)->funcs->get_dal_power_level ? (smu)->funcs->get_dal_power_level((smu), (clocks)) : 0)
+#define smu_get_perf_level(smu, designation, level) \
+	((smu)->funcs->get_perf_level ? (smu)->funcs->get_perf_level((smu), (designation), (level)) : 0)
+#define smu_get_current_shallow_sleep_clocks(smu, clocks) \
+	((smu)->funcs->get_current_shallow_sleep_clocks ? (smu)->funcs->get_current_shallow_sleep_clocks((smu), (clocks)) : 0)
+#define smu_notify_smu_enable_pwe(smu) \
+	((smu)->funcs->notify_smu_enable_pwe ? (smu)->funcs->notify_smu_enable_pwe((smu)) : 0)
+#define smu_set_watermarks_for_clock_ranges(smu, clock_ranges) \
+	((smu)->funcs->set_watermarks_for_clock_ranges ? (smu)->funcs->set_watermarks_for_clock_ranges((smu), (clock_ranges)) : 0)
+#define smu_dpm_set_uvd_enable(smu, enable) \
+	((smu)->funcs->dpm_set_uvd_enable ? (smu)->funcs->dpm_set_uvd_enable((smu), (enable)) : 0)
+#define smu_dpm_set_vce_enable(smu, enable) \
+	((smu)->funcs->dpm_set_vce_enable ? (smu)->funcs->dpm_set_vce_enable((smu), (enable)) : 0)
+#define smu_get_sclk(smu, low) \
+	((smu)->funcs->get_sclk ? (smu)->funcs->get_sclk((smu), (low)) : 0)
+#define smu_get_mclk(smu, low) \
+	((smu)->funcs->get_mclk ? (smu)->funcs->get_mclk((smu), (low)) : 0)
+#define smu_set_xgmi_pstate(smu, pstate) \
+		((smu)->funcs->set_xgmi_pstate ? (smu)->funcs->set_xgmi_pstate((smu), (pstate)) : 0)
+
+
+extern int smu_get_atom_data_table(struct smu_context *smu, uint32_t table,
+				   uint16_t *size, uint8_t *frev, uint8_t *crev,
+				   uint8_t **addr);
+
+extern const struct amd_ip_funcs smu_ip_funcs;
+
+extern const struct amdgpu_ip_block_version smu_v11_0_ip_block;
+extern int smu_feature_init_dpm(struct smu_context *smu);
+
+extern int smu_feature_is_enabled(struct smu_context *smu, int feature_id);
+extern int smu_feature_set_enabled(struct smu_context *smu, int feature_id, bool enable);
+extern int smu_feature_is_supported(struct smu_context *smu, int feature_id);
+extern int smu_feature_set_supported(struct smu_context *smu, int feature_id, bool enable);
+
+int smu_update_table_with_arg(struct smu_context *smu, uint16_t table_id, uint16_t exarg,
+		     void *table_data, bool drv2smu);
+#define smu_update_table(smu, table_id, table_data, drv2smu) \
+	smu_update_table_with_arg((smu), (table_id), 0, (table_data), (drv2smu))
+
+bool is_support_sw_smu(struct amdgpu_device *adev);
+int smu_reset(struct smu_context *smu);
+int smu_common_read_sensor(struct smu_context *smu, enum amd_pp_sensors sensor,
+			   void *data, uint32_t *size);
+int smu_sys_get_pp_table(struct smu_context *smu, void **table);
+int smu_sys_set_pp_table(struct smu_context *smu,  void *buf, size_t size);
+int smu_get_power_num_states(struct smu_context *smu, struct pp_states_info *state_info);
+enum amd_pm_state_type smu_get_current_power_state(struct smu_context *smu);
+
+/* smu to display interface */
+extern int smu_display_configuration_change(struct smu_context *smu, const
+					    struct amd_pp_display_configuration
+					    *display_config);
+extern int smu_get_current_clocks(struct smu_context *smu,
+				  struct amd_pp_clock_info *clocks);
+extern int smu_dpm_set_power_gate(struct smu_context *smu,uint32_t block_type, bool gate);
+extern int smu_handle_task(struct smu_context *smu,
+			   enum amd_dpm_forced_level level,
+			   enum amd_pp_task task_id);
+#endif
diff --git a/drivers/gpu/drm/amd/powerplay/inc/rv_ppsmc.h b/drivers/gpu/drm/amd/powerplay/inc/rv_ppsmc.h
index a2991fa2e6f8..90879e4092a3 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/rv_ppsmc.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/rv_ppsmc.h
@@ -85,7 +85,6 @@
 #define PPSMC_MSG_SetRccPfcPmeRestoreRegister   0x36
 #define PPSMC_Message_Count                     0x37
 
-
 typedef uint16_t PPSMC_Result;
 typedef int      PPSMC_Msg;
 
diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu10.h b/drivers/gpu/drm/amd/powerplay/inc/smu10.h
index 9e837a5014c5..b96520528240 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/smu10.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/smu10.h
@@ -136,12 +136,14 @@
 #define FEATURE_CORE_CSTATES_MASK     (1 << FEATURE_CORE_CSTATES_BIT)
 
 /* Workload bits */
-#define WORKLOAD_PPLIB_FULL_SCREEN_3D_BIT 0
-#define WORKLOAD_PPLIB_VIDEO_BIT          2
-#define WORKLOAD_PPLIB_VR_BIT             3
-#define WORKLOAD_PPLIB_COMPUTE_BIT        4
-#define WORKLOAD_PPLIB_CUSTOM_BIT         5
-#define WORKLOAD_PPLIB_COUNT              6
+#define WORKLOAD_DEFAULT_BIT              0
+#define WORKLOAD_PPLIB_FULL_SCREEN_3D_BIT 1
+#define WORKLOAD_PPLIB_POWER_SAVING_BIT   2
+#define WORKLOAD_PPLIB_VIDEO_BIT          3
+#define WORKLOAD_PPLIB_VR_BIT             4
+#define WORKLOAD_PPLIB_COMPUTE_BIT        5
+#define WORKLOAD_PPLIB_CUSTOM_BIT         6
+#define WORKLOAD_PPLIB_COUNT              7
 
 typedef struct {
 	/* MP1_EXT_SCRATCH0 */
diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu_v11_0.h b/drivers/gpu/drm/amd/powerplay/inc/smu_v11_0.h
new file mode 100644
index 000000000000..aa8d81f4111e
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/inc/smu_v11_0.h
@@ -0,0 +1,89 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __SMU_V11_0_H__
+#define __SMU_V11_0_H__
+
+#include "amdgpu_smu.h"
+
+/* MP Apertures */
+#define MP0_Public			0x03800000
+#define MP0_SRAM			0x03900000
+#define MP1_Public			0x03b00000
+#define MP1_SRAM			0x03c00004
+
+/* address block */
+#define smnMP1_FIRMWARE_FLAGS		0x3010024
+#define smnMP0_FW_INTF			0x30101c0
+#define smnMP1_PUB_CTRL			0x3010b14
+
+struct smu_11_0_max_sustainable_clocks {
+	uint32_t display_clock;
+	uint32_t phy_clock;
+	uint32_t pixel_clock;
+	uint32_t uclock;
+	uint32_t dcef_clock;
+	uint32_t soc_clock;
+};
+
+struct smu_11_0_dpm_table {
+	uint32_t    min;        /* MHz */
+	uint32_t    max;        /* MHz */
+};
+
+struct smu_11_0_dpm_tables {
+	struct smu_11_0_dpm_table        soc_table;
+	struct smu_11_0_dpm_table        gfx_table;
+	struct smu_11_0_dpm_table        uclk_table;
+	struct smu_11_0_dpm_table        eclk_table;
+	struct smu_11_0_dpm_table        vclk_table;
+	struct smu_11_0_dpm_table        dclk_table;
+	struct smu_11_0_dpm_table        dcef_table;
+	struct smu_11_0_dpm_table        pixel_table;
+	struct smu_11_0_dpm_table        display_table;
+	struct smu_11_0_dpm_table        phy_table;
+	struct smu_11_0_dpm_table        fclk_table;
+};
+
+struct smu_11_0_dpm_context {
+	struct smu_11_0_dpm_tables  dpm_tables;
+	uint32_t                    workload_policy_mask;
+	uint32_t                    dcef_min_ds_clk;
+};
+
+enum smu_11_0_power_state {
+	SMU_11_0_POWER_STATE__D0 = 0,
+	SMU_11_0_POWER_STATE__D1,
+	SMU_11_0_POWER_STATE__D3, /* Sleep*/
+	SMU_11_0_POWER_STATE__D4, /* Hibernate*/
+	SMU_11_0_POWER_STATE__D5, /* Power off*/
+};
+
+struct smu_11_0_power_context {
+	uint32_t	power_source;
+	uint8_t		in_power_limit_boost_mode;
+	enum smu_11_0_power_state power_state;
+};
+
+void smu_v11_0_set_smu_funcs(struct smu_context *smu);
+
+#endif
diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu_v11_0_ppsmc.h b/drivers/gpu/drm/amd/powerplay/inc/smu_v11_0_ppsmc.h
new file mode 100644
index 000000000000..f466f624ad32
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/inc/smu_v11_0_ppsmc.h
@@ -0,0 +1,128 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef SMU_V11_0_PPSMC_H
+#define SMU_V11_0_PPSMC_H
+
+// SMU Response Codes:
+#define PPSMC_Result_OK                    0x1
+#define PPSMC_Result_Failed                0xFF
+#define PPSMC_Result_UnknownCmd            0xFE
+#define PPSMC_Result_CmdRejectedPrereq     0xFD
+#define PPSMC_Result_CmdRejectedBusy       0xFC
+
+// Message Definitions:
+// BASIC
+#define PPSMC_MSG_TestMessage                    0x1
+#define PPSMC_MSG_GetSmuVersion                  0x2
+#define PPSMC_MSG_GetDriverIfVersion             0x3
+#define PPSMC_MSG_SetAllowedFeaturesMaskLow      0x4
+#define PPSMC_MSG_SetAllowedFeaturesMaskHigh     0x5
+#define PPSMC_MSG_EnableAllSmuFeatures           0x6
+#define PPSMC_MSG_DisableAllSmuFeatures          0x7
+#define PPSMC_MSG_EnableSmuFeaturesLow           0x8
+#define PPSMC_MSG_EnableSmuFeaturesHigh          0x9
+#define PPSMC_MSG_DisableSmuFeaturesLow          0xA
+#define PPSMC_MSG_DisableSmuFeaturesHigh         0xB
+#define PPSMC_MSG_GetEnabledSmuFeaturesLow       0xC
+#define PPSMC_MSG_GetEnabledSmuFeaturesHigh      0xD
+#define PPSMC_MSG_SetDriverDramAddrHigh          0xE
+#define PPSMC_MSG_SetDriverDramAddrLow           0xF
+#define PPSMC_MSG_SetToolsDramAddrHigh           0x10
+#define PPSMC_MSG_SetToolsDramAddrLow            0x11
+#define PPSMC_MSG_TransferTableSmu2Dram          0x12
+#define PPSMC_MSG_TransferTableDram2Smu          0x13
+#define PPSMC_MSG_UseDefaultPPTable              0x14
+#define PPSMC_MSG_UseBackupPPTable               0x15
+#define PPSMC_MSG_SetSystemVirtualDramAddrHigh   0x16
+#define PPSMC_MSG_SetSystemVirtualDramAddrLow    0x17
+
+//BACO/BAMACO/BOMACO
+#define PPSMC_MSG_EnterBaco                      0x18
+#define PPSMC_MSG_ExitBaco                       0x19
+
+//DPM
+#define PPSMC_MSG_SetSoftMinByFreq               0x1A
+#define PPSMC_MSG_SetSoftMaxByFreq               0x1B
+#define PPSMC_MSG_SetHardMinByFreq               0x1C
+#define PPSMC_MSG_SetHardMaxByFreq               0x1D 
+#define PPSMC_MSG_GetMinDpmFreq                  0x1E
+#define PPSMC_MSG_GetMaxDpmFreq                  0x1F
+#define PPSMC_MSG_GetDpmFreqByIndex              0x20
+#define PPSMC_MSG_OverridePcieParameters         0x21
+#define PPSMC_MSG_SetMinDeepSleepDcefclk         0x22
+#define PPSMC_MSG_SetWorkloadMask                0x23 
+#define PPSMC_MSG_SetUclkFastSwitch              0x24
+#define PPSMC_MSG_GetAvfsVoltageByDpm            0x25
+#define PPSMC_MSG_SetVideoFps                    0x26
+#define PPSMC_MSG_GetDcModeMaxDpmFreq            0x27
+
+//Power Gating
+#define PPSMC_MSG_AllowGfxOff                    0x28
+#define PPSMC_MSG_DisallowGfxOff                 0x29
+#define PPSMC_MSG_PowerUpVcn					 0x2A
+#define PPSMC_MSG_PowerDownVcn					 0x2B	
+#define PPSMC_MSG_PowerUpJpeg                    0x2C
+#define PPSMC_MSG_PowerDownJpeg					 0x2D
+//reserve 0x2A to 0x2F for PG harvesting TBD
+
+//I2C Interface
+#define PPSMC_RequestI2cTransaction              0x30
+
+//Resets
+#define PPSMC_MSG_SoftReset                      0x31  //FIXME Need confirmation from driver
+#define PPSMC_MSG_PrepareMp1ForUnload            0x32
+#define PPSMC_MSG_PrepareMp1ForReset             0x33
+#define PPSMC_MSG_PrepareMp1ForShutdown          0x34
+
+//ACDC Power Source
+#define PPSMC_MSG_SetPptLimit                    0x35
+#define PPSMC_MSG_GetPptLimit                    0x36
+#define PPSMC_MSG_ReenableAcDcInterrupt          0x37
+#define PPSMC_MSG_NotifyPowerSource              0x38
+//#define PPSMC_MSG_GfxDeviceDriverReset           0x39 //FIXME mode1 and 2 resets will go directly go PSP
+
+//BTC
+#define PPSMC_MSG_RunBtc                         0x3A
+
+//Debug
+#define PPSMC_MSG_DramLogSetDramAddrHigh         0x3B
+#define PPSMC_MSG_DramLogSetDramAddrLow          0x3C
+#define PPSMC_MSG_DramLogSetDramSize             0x3D
+#define PPSMC_MSG_GetDebugData                   0x3E
+
+//Others
+#define PPSMC_MSG_ConfigureGfxDidt               0x3F
+#define PPSMC_MSG_NumOfDisplays                  0x40
+
+#define PPSMC_MSG_SetMemoryChannelConfig         0x41 
+#define PPSMC_MSG_SetGeminiMode                  0x42
+#define PPSMC_MSG_SetGeminiApertureHigh          0x43
+#define PPSMC_MSG_SetGeminiApertureLow           0x44
+
+#define PPSMC_Message_Count                      0x45
+
+typedef uint32_t PPSMC_Result;
+typedef uint32_t PPSMC_Msg;
+
+#endif
diff --git a/drivers/gpu/drm/amd/powerplay/inc/smu_v11_0_pptable.h b/drivers/gpu/drm/amd/powerplay/inc/smu_v11_0_pptable.h
new file mode 100644
index 000000000000..92c65b80bde2
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/inc/smu_v11_0_pptable.h
@@ -0,0 +1,147 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef SMU_11_0_PPTABLE_H
+#define SMU_11_0_PPTABLE_H
+
+
+#define SMU_11_0_TABLE_FORMAT_REVISION                  12
+
+//// POWERPLAYTABLE::ulPlatformCaps
+#define SMU_11_0_PP_PLATFORM_CAP_POWERPLAY              0x1
+#define SMU_11_0_PP_PLATFORM_CAP_SBIOSPOWERSOURCE       0x2
+#define SMU_11_0_PP_PLATFORM_CAP_HARDWAREDC             0x4
+#define SMU_11_0_PP_PLATFORM_CAP_BACO                   0x8
+#define SMU_11_0_PP_PLATFORM_CAP_MACO                   0x10
+#define SMU_11_0_PP_PLATFORM_CAP_SHADOWPSTATE           0x20
+
+// SMU_11_0_PP_THERMALCONTROLLER - Thermal Controller Type
+#define SMU_11_0_PP_THERMALCONTROLLER_NONE              0
+
+#define SMU_11_0_PP_OVERDRIVE_VERSION                   0x0800
+#define SMU_11_0_PP_POWERSAVINGCLOCK_VERSION            0x0100
+
+enum SMU_11_0_ODFEATURE_ID {
+    SMU_11_0_ODFEATURE_GFXCLK_LIMITS        = 1 << 0,         //GFXCLK Limit feature
+    SMU_11_0_ODFEATURE_GFXCLK_CURVE         = 1 << 1,         //GFXCLK Curve feature
+    SMU_11_0_ODFEATURE_UCLK_MAX             = 1 << 2,         //UCLK Limit feature
+    SMU_11_0_ODFEATURE_POWER_LIMIT          = 1 << 3,         //Power Limit feature
+    SMU_11_0_ODFEATURE_FAN_ACOUSTIC_LIMIT   = 1 << 4,         //Fan Acoustic RPM feature
+    SMU_11_0_ODFEATURE_FAN_SPEED_MIN        = 1 << 5,         //Minimum Fan Speed feature
+    SMU_11_0_ODFEATURE_TEMPERATURE_FAN      = 1 << 6,         //Fan Target Temperature Limit feature
+    SMU_11_0_ODFEATURE_TEMPERATURE_SYSTEM   = 1 << 7,         //Operating Temperature Limit feature
+    SMU_11_0_ODFEATURE_MEMORY_TIMING_TUNE   = 1 << 8,         //AC Timing Tuning feature
+    SMU_11_0_ODFEATURE_FAN_ZERO_RPM_CONTROL = 1 << 9,         //Zero RPM feature
+    SMU_11_0_ODFEATURE_AUTO_UV_ENGINE       = 1 << 10,        //Auto Under Volt GFXCLK feature
+    SMU_11_0_ODFEATURE_AUTO_OC_ENGINE       = 1 << 11,        //Auto Over Clock GFXCLK feature
+    SMU_11_0_ODFEATURE_AUTO_OC_MEMORY       = 1 << 12,        //Auto Over Clock MCLK feature
+    SMU_11_0_ODFEATURE_FAN_CURVE            = 1 << 13,        //VICTOR TODO
+    SMU_11_0_ODFEATURE_COUNT                = 14,
+};
+#define SMU_11_0_MAX_ODFEATURE    32          //Maximum Number of OD Features
+
+enum SMU_11_0_ODSETTING_ID {
+    SMU_11_0_ODSETTING_GFXCLKFMAX = 0,
+    SMU_11_0_ODSETTING_GFXCLKFMIN,
+    SMU_11_0_ODSETTING_VDDGFXCURVEFREQ_P1,
+    SMU_11_0_ODSETTING_VDDGFXCURVEVOLTAGE_P1,
+    SMU_11_0_ODSETTING_VDDGFXCURVEFREQ_P2,
+    SMU_11_0_ODSETTING_VDDGFXCURVEVOLTAGE_P2,
+    SMU_11_0_ODSETTING_VDDGFXCURVEFREQ_P3,
+    SMU_11_0_ODSETTING_VDDGFXCURVEVOLTAGE_P3,
+    SMU_11_0_ODSETTING_UCLKFMAX,
+    SMU_11_0_ODSETTING_POWERPERCENTAGE,
+    SMU_11_0_ODSETTING_FANRPMMIN,
+    SMU_11_0_ODSETTING_FANRPMACOUSTICLIMIT,
+    SMU_11_0_ODSETTING_FANTARGETTEMPERATURE,
+    SMU_11_0_ODSETTING_OPERATINGTEMPMAX,
+    SMU_11_0_ODSETTING_ACTIMING,
+    SMU_11_0_ODSETTING_FAN_ZERO_RPM_CONTROL,
+    SMU_11_0_ODSETTING_AUTOUVENGINE,
+    SMU_11_0_ODSETTING_AUTOOCENGINE,
+    SMU_11_0_ODSETTING_AUTOOCMEMORY,
+    SMU_11_0_ODSETTING_COUNT,
+};
+#define SMU_11_0_MAX_ODSETTING    32          //Maximum Number of ODSettings
+
+struct smu_11_0_overdrive_table
+{
+    uint8_t  revision;                                        //Revision = SMU_11_0_PP_OVERDRIVE_VERSION
+    uint8_t  reserve[3];                                      //Zero filled field reserved for future use
+    uint32_t feature_count;                                   //Total number of supported features
+    uint32_t setting_count;                                   //Total number of supported settings
+    uint8_t  cap[SMU_11_0_MAX_ODFEATURE];                     //OD feature support flags
+    uint32_t max[SMU_11_0_MAX_ODSETTING];                     //default maximum settings
+    uint32_t min[SMU_11_0_MAX_ODSETTING];                     //default minimum settings
+} __attribute__((packed));
+
+enum SMU_11_0_PPCLOCK_ID {
+    SMU_11_0_PPCLOCK_GFXCLK = 0,
+    SMU_11_0_PPCLOCK_VCLK,
+    SMU_11_0_PPCLOCK_DCLK,
+    SMU_11_0_PPCLOCK_ECLK,
+    SMU_11_0_PPCLOCK_SOCCLK,
+    SMU_11_0_PPCLOCK_UCLK,
+    SMU_11_0_PPCLOCK_DCEFCLK,
+    SMU_11_0_PPCLOCK_DISPCLK,
+    SMU_11_0_PPCLOCK_PIXCLK,
+    SMU_11_0_PPCLOCK_PHYCLK,
+    SMU_11_0_PPCLOCK_COUNT,
+};
+#define SMU_11_0_MAX_PPCLOCK      16          //Maximum Number of PP Clocks
+
+struct smu_11_0_power_saving_clock_table
+{
+    uint8_t  revision;                                        //Revision = SMU_11_0_PP_POWERSAVINGCLOCK_VERSION
+    uint8_t  reserve[3];                                      //Zero filled field reserved for future use
+    uint32_t count;                                           //power_saving_clock_count = SMU_11_0_PPCLOCK_COUNT
+    uint32_t max[SMU_11_0_MAX_PPCLOCK];                       //PowerSavingClock Mode Clock Maximum array In MHz
+    uint32_t min[SMU_11_0_MAX_PPCLOCK];                       //PowerSavingClock Mode Clock Minimum array In MHz
+} __attribute__((packed));
+
+struct smu_11_0_powerplay_table
+{
+      struct atom_common_table_header header;
+      uint8_t  table_revision;
+      uint32_t table_size;                          //Driver portion table size. The offset to smc_pptable including header size
+      uint32_t golden_pp_id;
+      uint32_t golden_revision;
+      uint16_t format_id;
+      uint32_t platform_caps;                       //POWERPLAYABLE::ulPlatformCaps
+                                                    
+      uint8_t  thermal_controller_type;             //one of SMU_11_0_PP_THERMALCONTROLLER
+
+      uint16_t small_power_limit1;
+      uint16_t small_power_limit2;
+      uint16_t boost_power_limit;
+      uint16_t od_turbo_power_limit;                //Power limit setting for Turbo mode in Performance UI Tuning. 
+      uint16_t od_power_save_power_limit;           //Power limit setting for PowerSave/Optimal mode in Performance UI Tuning. 
+      uint16_t software_shutdown_temp;
+
+      uint16_t reserve[6];                          //Zero filled field reserved for future use
+
+      struct smu_11_0_power_saving_clock_table      power_saving_clock;
+      struct smu_11_0_overdrive_table               overdrive_table;
+
+      PPTable_t smc_pptable;                        //PPTable_t in smu11_driver_if.h
+} __attribute__((packed));
+
+#endif
diff --git a/drivers/gpu/drm/amd/powerplay/inc/vega20_ppsmc.h b/drivers/gpu/drm/amd/powerplay/inc/vega20_ppsmc.h
index 4f63a736ea0e..a0883038f3c3 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/vega20_ppsmc.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/vega20_ppsmc.h
@@ -119,7 +119,8 @@
 #define PPSMC_MSG_PrepareMp1ForShutdown          0x5A
 #define PPSMC_MSG_SetMGpuFanBoostLimitRpm        0x5D
 #define PPSMC_MSG_GetAVFSVoltageByDpm            0x5F
-#define PPSMC_Message_Count                      0x60
+#define PPSMC_MSG_BacoWorkAroundFlushVDCI        0x60
+#define PPSMC_Message_Count                      0x61
 
 typedef uint32_t PPSMC_Result;
 typedef uint32_t PPSMC_Msg;
diff --git a/drivers/gpu/drm/amd/powerplay/smu_v11_0.c b/drivers/gpu/drm/amd/powerplay/smu_v11_0.c
new file mode 100644
index 000000000000..92903a4cc4d8
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/smu_v11_0.c
@@ -0,0 +1,1977 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "pp_debug.h"
+#include <linux/firmware.h>
+#include "amdgpu.h"
+#include "amdgpu_smu.h"
+#include "atomfirmware.h"
+#include "amdgpu_atomfirmware.h"
+#include "smu_v11_0.h"
+#include "smu11_driver_if.h"
+#include "soc15_common.h"
+#include "atom.h"
+#include "vega20_ppt.h"
+#include "pp_thermal.h"
+
+#include "asic_reg/thm/thm_11_0_2_offset.h"
+#include "asic_reg/thm/thm_11_0_2_sh_mask.h"
+#include "asic_reg/mp/mp_9_0_offset.h"
+#include "asic_reg/mp/mp_9_0_sh_mask.h"
+#include "asic_reg/nbio/nbio_7_4_offset.h"
+#include "asic_reg/smuio/smuio_9_0_offset.h"
+#include "asic_reg/smuio/smuio_9_0_sh_mask.h"
+
+MODULE_FIRMWARE("amdgpu/vega20_smc.bin");
+
+#define SMU11_TOOL_SIZE		0x19000
+#define SMU11_THERMAL_MINIMUM_ALERT_TEMP      0
+#define SMU11_THERMAL_MAXIMUM_ALERT_TEMP      255
+
+#define SMU11_TEMPERATURE_UNITS_PER_CENTIGRADES 1000
+#define SMU11_VOLTAGE_SCALE 4
+
+#define SMC_DPM_FEATURE (FEATURE_DPM_PREFETCHER_MASK | \
+			 FEATURE_DPM_GFXCLK_MASK | \
+			 FEATURE_DPM_UCLK_MASK | \
+			 FEATURE_DPM_SOCCLK_MASK | \
+			 FEATURE_DPM_UVD_MASK | \
+			 FEATURE_DPM_VCE_MASK | \
+			 FEATURE_DPM_MP0CLK_MASK | \
+			 FEATURE_DPM_LINK_MASK | \
+			 FEATURE_DPM_DCEFCLK_MASK)
+
+static int smu_v11_0_send_msg_without_waiting(struct smu_context *smu,
+					      uint16_t msg)
+{
+	struct amdgpu_device *adev = smu->adev;
+	WREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_66, msg);
+	return 0;
+}
+
+static int smu_v11_0_read_arg(struct smu_context *smu, uint32_t *arg)
+{
+	struct amdgpu_device *adev = smu->adev;
+
+	*arg = RREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_82);
+	return 0;
+}
+
+static int smu_v11_0_wait_for_response(struct smu_context *smu)
+{
+	struct amdgpu_device *adev = smu->adev;
+	uint32_t cur_value, i;
+
+	for (i = 0; i < adev->usec_timeout; i++) {
+		cur_value = RREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_90);
+		if ((cur_value & MP1_C2PMSG_90__CONTENT_MASK) != 0)
+			break;
+		udelay(1);
+	}
+
+	/* timeout means wrong logic */
+	if (i == adev->usec_timeout)
+		return -ETIME;
+
+	return RREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_90) == 0x1 ? 0 : -EIO;
+}
+
+static int smu_v11_0_send_msg(struct smu_context *smu, uint16_t msg)
+{
+	struct amdgpu_device *adev = smu->adev;
+	int ret = 0, index = 0;
+
+	index = smu_msg_get_index(smu, msg);
+	if (index < 0)
+		return index;
+
+	smu_v11_0_wait_for_response(smu);
+
+	WREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_90, 0);
+
+	smu_v11_0_send_msg_without_waiting(smu, (uint16_t)index);
+
+	ret = smu_v11_0_wait_for_response(smu);
+
+	if (ret)
+		pr_err("Failed to send message 0x%x, response 0x%x\n", index,
+		       ret);
+
+	return ret;
+
+}
+
+static int
+smu_v11_0_send_msg_with_param(struct smu_context *smu, uint16_t msg,
+			      uint32_t param)
+{
+
+	struct amdgpu_device *adev = smu->adev;
+	int ret = 0, index = 0;
+
+	index = smu_msg_get_index(smu, msg);
+	if (index < 0)
+		return index;
+
+	ret = smu_v11_0_wait_for_response(smu);
+	if (ret)
+		pr_err("Failed to send message 0x%x, response 0x%x, param 0x%x\n",
+		       index, ret, param);
+
+	WREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_90, 0);
+
+	WREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_82, param);
+
+	smu_v11_0_send_msg_without_waiting(smu, (uint16_t)index);
+
+	ret = smu_v11_0_wait_for_response(smu);
+	if (ret)
+		pr_err("Failed to send message 0x%x, response 0x%x param 0x%x\n",
+		       index, ret, param);
+
+	return ret;
+}
+
+static int smu_v11_0_init_microcode(struct smu_context *smu)
+{
+	struct amdgpu_device *adev = smu->adev;
+	const char *chip_name;
+	char fw_name[30];
+	int err = 0;
+	const struct smc_firmware_header_v1_0 *hdr;
+	const struct common_firmware_header *header;
+	struct amdgpu_firmware_info *ucode = NULL;
+
+	switch (adev->asic_type) {
+	case CHIP_VEGA20:
+		chip_name = "vega20";
+		break;
+	default:
+		BUG();
+	}
+
+	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_smc.bin", chip_name);
+
+	err = request_firmware(&adev->pm.fw, fw_name, adev->dev);
+	if (err)
+		goto out;
+	err = amdgpu_ucode_validate(adev->pm.fw);
+	if (err)
+		goto out;
+
+	hdr = (const struct smc_firmware_header_v1_0 *) adev->pm.fw->data;
+	amdgpu_ucode_print_smc_hdr(&hdr->header);
+	adev->pm.fw_version = le32_to_cpu(hdr->header.ucode_version);
+
+	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
+		ucode = &adev->firmware.ucode[AMDGPU_UCODE_ID_SMC];
+		ucode->ucode_id = AMDGPU_UCODE_ID_SMC;
+		ucode->fw = adev->pm.fw;
+		header = (const struct common_firmware_header *)ucode->fw->data;
+		adev->firmware.fw_size +=
+			ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
+	}
+
+out:
+	if (err) {
+		DRM_ERROR("smu_v11_0: Failed to load firmware \"%s\"\n",
+			  fw_name);
+		release_firmware(adev->pm.fw);
+		adev->pm.fw = NULL;
+	}
+	return err;
+}
+
+static int smu_v11_0_load_microcode(struct smu_context *smu)
+{
+	return 0;
+}
+
+static int smu_v11_0_check_fw_status(struct smu_context *smu)
+{
+	struct amdgpu_device *adev = smu->adev;
+	uint32_t mp1_fw_flags;
+
+	mp1_fw_flags = RREG32_PCIE(MP1_Public |
+				   (smnMP1_FIRMWARE_FLAGS & 0xffffffff));
+
+	if ((mp1_fw_flags & MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED_MASK) >>
+	    MP1_FIRMWARE_FLAGS__INTERRUPTS_ENABLED__SHIFT)
+		return 0;
+
+	return -EIO;
+}
+
+static int smu_v11_0_check_fw_version(struct smu_context *smu)
+{
+	uint32_t smu_version = 0xff;
+	int ret = 0;
+
+	ret = smu_send_smc_msg(smu, SMU_MSG_GetDriverIfVersion);
+	if (ret)
+		goto err;
+
+	ret = smu_read_smc_arg(smu, &smu_version);
+	if (ret)
+		goto err;
+
+	if (smu_version != smu->smc_if_version)
+		ret = -EINVAL;
+err:
+	return ret;
+}
+
+static int smu_v11_0_read_pptable_from_vbios(struct smu_context *smu)
+{
+	int ret, index;
+	uint16_t size;
+	uint8_t frev, crev;
+	void *table;
+
+	index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
+					    powerplayinfo);
+
+	ret = smu_get_atom_data_table(smu, index, &size, &frev, &crev,
+				      (uint8_t **)&table);
+	if (ret)
+		return ret;
+
+	if (!smu->smu_table.power_play_table)
+		smu->smu_table.power_play_table = table;
+	if (!smu->smu_table.power_play_table_size)
+		smu->smu_table.power_play_table_size = size;
+
+	return 0;
+}
+
+static int smu_v11_0_init_dpm_context(struct smu_context *smu)
+{
+	struct smu_dpm_context *smu_dpm = &smu->smu_dpm;
+
+	if (smu_dpm->dpm_context || smu_dpm->dpm_context_size != 0)
+		return -EINVAL;
+
+	return smu_alloc_dpm_context(smu);
+}
+
+static int smu_v11_0_fini_dpm_context(struct smu_context *smu)
+{
+	struct smu_dpm_context *smu_dpm = &smu->smu_dpm;
+
+	if (!smu_dpm->dpm_context || smu_dpm->dpm_context_size == 0)
+		return -EINVAL;
+
+	kfree(smu_dpm->dpm_context);
+	kfree(smu_dpm->golden_dpm_context);
+	kfree(smu_dpm->dpm_current_power_state);
+	kfree(smu_dpm->dpm_request_power_state);
+	smu_dpm->dpm_context = NULL;
+	smu_dpm->golden_dpm_context = NULL;
+	smu_dpm->dpm_context_size = 0;
+	smu_dpm->dpm_current_power_state = NULL;
+	smu_dpm->dpm_request_power_state = NULL;
+
+	return 0;
+}
+
+static int smu_v11_0_init_smc_tables(struct smu_context *smu)
+{
+	struct smu_table_context *smu_table = &smu->smu_table;
+	struct smu_table *tables = NULL;
+	int ret = 0;
+
+	if (smu_table->tables || smu_table->table_count != 0)
+		return -EINVAL;
+
+	tables = kcalloc(TABLE_COUNT, sizeof(struct smu_table), GFP_KERNEL);
+	if (!tables)
+		return -ENOMEM;
+
+	smu_table->tables = tables;
+	smu_table->table_count = TABLE_COUNT;
+
+	SMU_TABLE_INIT(tables, TABLE_PPTABLE, sizeof(PPTable_t),
+		       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM);
+	SMU_TABLE_INIT(tables, TABLE_WATERMARKS, sizeof(Watermarks_t),
+		       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM);
+	SMU_TABLE_INIT(tables, TABLE_SMU_METRICS, sizeof(SmuMetrics_t),
+		       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM);
+	SMU_TABLE_INIT(tables, TABLE_OVERDRIVE, sizeof(OverDriveTable_t),
+		       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM);
+	SMU_TABLE_INIT(tables, TABLE_PMSTATUSLOG, SMU11_TOOL_SIZE, PAGE_SIZE,
+		       AMDGPU_GEM_DOMAIN_VRAM);
+	SMU_TABLE_INIT(tables, TABLE_ACTIVITY_MONITOR_COEFF,
+		       sizeof(DpmActivityMonitorCoeffInt_t),
+		       PAGE_SIZE,
+		       AMDGPU_GEM_DOMAIN_VRAM);
+
+	ret = smu_v11_0_init_dpm_context(smu);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int smu_v11_0_fini_smc_tables(struct smu_context *smu)
+{
+	struct smu_table_context *smu_table = &smu->smu_table;
+	int ret = 0;
+
+	if (!smu_table->tables || smu_table->table_count == 0)
+		return -EINVAL;
+
+	kfree(smu_table->tables);
+	smu_table->tables = NULL;
+	smu_table->table_count = 0;
+
+	ret = smu_v11_0_fini_dpm_context(smu);
+	if (ret)
+		return ret;
+	return 0;
+}
+
+static int smu_v11_0_init_power(struct smu_context *smu)
+{
+	struct smu_power_context *smu_power = &smu->smu_power;
+
+	if (smu_power->power_context || smu_power->power_context_size != 0)
+		return -EINVAL;
+
+	smu_power->power_context = kzalloc(sizeof(struct smu_11_0_dpm_context),
+					   GFP_KERNEL);
+	if (!smu_power->power_context)
+		return -ENOMEM;
+	smu_power->power_context_size = sizeof(struct smu_11_0_dpm_context);
+
+	return 0;
+}
+
+static int smu_v11_0_fini_power(struct smu_context *smu)
+{
+	struct smu_power_context *smu_power = &smu->smu_power;
+
+	if (!smu_power->power_context || smu_power->power_context_size == 0)
+		return -EINVAL;
+
+	kfree(smu_power->power_context);
+	smu_power->power_context = NULL;
+	smu_power->power_context_size = 0;
+
+	return 0;
+}
+
+int smu_v11_0_get_vbios_bootup_values(struct smu_context *smu)
+{
+	int ret, index;
+	uint16_t size;
+	uint8_t frev, crev;
+	struct atom_common_table_header *header;
+	struct atom_firmware_info_v3_3 *v_3_3;
+	struct atom_firmware_info_v3_1 *v_3_1;
+
+	index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
+					    firmwareinfo);
+
+	ret = smu_get_atom_data_table(smu, index, &size, &frev, &crev,
+				      (uint8_t **)&header);
+	if (ret)
+		return ret;
+
+	if (header->format_revision != 3) {
+		pr_err("unknown atom_firmware_info version! for smu11\n");
+		return -EINVAL;
+	}
+
+	switch (header->content_revision) {
+	case 0:
+	case 1:
+	case 2:
+		v_3_1 = (struct atom_firmware_info_v3_1 *)header;
+		smu->smu_table.boot_values.revision = v_3_1->firmware_revision;
+		smu->smu_table.boot_values.gfxclk = v_3_1->bootup_sclk_in10khz;
+		smu->smu_table.boot_values.uclk = v_3_1->bootup_mclk_in10khz;
+		smu->smu_table.boot_values.socclk = 0;
+		smu->smu_table.boot_values.dcefclk = 0;
+		smu->smu_table.boot_values.vddc = v_3_1->bootup_vddc_mv;
+		smu->smu_table.boot_values.vddci = v_3_1->bootup_vddci_mv;
+		smu->smu_table.boot_values.mvddc = v_3_1->bootup_mvddc_mv;
+		smu->smu_table.boot_values.vdd_gfx = v_3_1->bootup_vddgfx_mv;
+		smu->smu_table.boot_values.cooling_id = v_3_1->coolingsolution_id;
+		smu->smu_table.boot_values.pp_table_id = 0;
+		break;
+	case 3:
+	default:
+		v_3_3 = (struct atom_firmware_info_v3_3 *)header;
+		smu->smu_table.boot_values.revision = v_3_3->firmware_revision;
+		smu->smu_table.boot_values.gfxclk = v_3_3->bootup_sclk_in10khz;
+		smu->smu_table.boot_values.uclk = v_3_3->bootup_mclk_in10khz;
+		smu->smu_table.boot_values.socclk = 0;
+		smu->smu_table.boot_values.dcefclk = 0;
+		smu->smu_table.boot_values.vddc = v_3_3->bootup_vddc_mv;
+		smu->smu_table.boot_values.vddci = v_3_3->bootup_vddci_mv;
+		smu->smu_table.boot_values.mvddc = v_3_3->bootup_mvddc_mv;
+		smu->smu_table.boot_values.vdd_gfx = v_3_3->bootup_vddgfx_mv;
+		smu->smu_table.boot_values.cooling_id = v_3_3->coolingsolution_id;
+		smu->smu_table.boot_values.pp_table_id = v_3_3->pplib_pptable_id;
+	}
+
+	return 0;
+}
+
+static int smu_v11_0_get_clk_info_from_vbios(struct smu_context *smu)
+{
+	int ret, index;
+	struct amdgpu_device *adev = smu->adev;
+	struct atom_get_smu_clock_info_parameters_v3_1 input = {0};
+	struct atom_get_smu_clock_info_output_parameters_v3_1 *output;
+
+	input.clk_id = SMU11_SYSPLL0_SOCCLK_ID;
+	input.command = GET_SMU_CLOCK_INFO_V3_1_GET_CLOCK_FREQ;
+	index = get_index_into_master_table(atom_master_list_of_command_functions_v2_1,
+					    getsmuclockinfo);
+
+	ret = amdgpu_atom_execute_table(adev->mode_info.atom_context, index,
+					(uint32_t *)&input);
+	if (ret)
+		return -EINVAL;
+
+	output = (struct atom_get_smu_clock_info_output_parameters_v3_1 *)&input;
+	smu->smu_table.boot_values.socclk = le32_to_cpu(output->atom_smu_outputclkfreq.smu_clock_freq_hz) / 10000;
+
+	memset(&input, 0, sizeof(input));
+	input.clk_id = SMU11_SYSPLL0_DCEFCLK_ID;
+	input.command = GET_SMU_CLOCK_INFO_V3_1_GET_CLOCK_FREQ;
+	index = get_index_into_master_table(atom_master_list_of_command_functions_v2_1,
+					    getsmuclockinfo);
+
+	ret = amdgpu_atom_execute_table(adev->mode_info.atom_context, index,
+					(uint32_t *)&input);
+	if (ret)
+		return -EINVAL;
+
+	output = (struct atom_get_smu_clock_info_output_parameters_v3_1 *)&input;
+	smu->smu_table.boot_values.dcefclk = le32_to_cpu(output->atom_smu_outputclkfreq.smu_clock_freq_hz) / 10000;
+
+	memset(&input, 0, sizeof(input));
+	input.clk_id = SMU11_SYSPLL0_ECLK_ID;
+	input.command = GET_SMU_CLOCK_INFO_V3_1_GET_CLOCK_FREQ;
+	index = get_index_into_master_table(atom_master_list_of_command_functions_v2_1,
+					    getsmuclockinfo);
+
+	ret = amdgpu_atom_execute_table(adev->mode_info.atom_context, index,
+					(uint32_t *)&input);
+	if (ret)
+		return -EINVAL;
+
+	output = (struct atom_get_smu_clock_info_output_parameters_v3_1 *)&input;
+	smu->smu_table.boot_values.eclk = le32_to_cpu(output->atom_smu_outputclkfreq.smu_clock_freq_hz) / 10000;
+
+	memset(&input, 0, sizeof(input));
+	input.clk_id = SMU11_SYSPLL0_VCLK_ID;
+	input.command = GET_SMU_CLOCK_INFO_V3_1_GET_CLOCK_FREQ;
+	index = get_index_into_master_table(atom_master_list_of_command_functions_v2_1,
+					    getsmuclockinfo);
+
+	ret = amdgpu_atom_execute_table(adev->mode_info.atom_context, index,
+					(uint32_t *)&input);
+	if (ret)
+		return -EINVAL;
+
+	output = (struct atom_get_smu_clock_info_output_parameters_v3_1 *)&input;
+	smu->smu_table.boot_values.vclk = le32_to_cpu(output->atom_smu_outputclkfreq.smu_clock_freq_hz) / 10000;
+
+	memset(&input, 0, sizeof(input));
+	input.clk_id = SMU11_SYSPLL0_DCLK_ID;
+	input.command = GET_SMU_CLOCK_INFO_V3_1_GET_CLOCK_FREQ;
+	index = get_index_into_master_table(atom_master_list_of_command_functions_v2_1,
+					    getsmuclockinfo);
+
+	ret = amdgpu_atom_execute_table(adev->mode_info.atom_context, index,
+					(uint32_t *)&input);
+	if (ret)
+		return -EINVAL;
+
+	output = (struct atom_get_smu_clock_info_output_parameters_v3_1 *)&input;
+	smu->smu_table.boot_values.dclk = le32_to_cpu(output->atom_smu_outputclkfreq.smu_clock_freq_hz) / 10000;
+
+	return 0;
+}
+
+static int smu_v11_0_notify_memory_pool_location(struct smu_context *smu)
+{
+	struct smu_table_context *smu_table = &smu->smu_table;
+	struct smu_table *memory_pool = &smu_table->memory_pool;
+	int ret = 0;
+	uint64_t address;
+	uint32_t address_low, address_high;
+
+	if (memory_pool->size == 0 || memory_pool->cpu_addr == NULL)
+		return ret;
+
+	address = (uintptr_t)memory_pool->cpu_addr;
+	address_high = (uint32_t)upper_32_bits(address);
+	address_low  = (uint32_t)lower_32_bits(address);
+
+	ret = smu_send_smc_msg_with_param(smu,
+					  SMU_MSG_SetSystemVirtualDramAddrHigh,
+					  address_high);
+	if (ret)
+		return ret;
+	ret = smu_send_smc_msg_with_param(smu,
+					  SMU_MSG_SetSystemVirtualDramAddrLow,
+					  address_low);
+	if (ret)
+		return ret;
+
+	address = memory_pool->mc_address;
+	address_high = (uint32_t)upper_32_bits(address);
+	address_low  = (uint32_t)lower_32_bits(address);
+
+	ret = smu_send_smc_msg_with_param(smu, SMU_MSG_DramLogSetDramAddrHigh,
+					  address_high);
+	if (ret)
+		return ret;
+	ret = smu_send_smc_msg_with_param(smu, SMU_MSG_DramLogSetDramAddrLow,
+					  address_low);
+	if (ret)
+		return ret;
+	ret = smu_send_smc_msg_with_param(smu, SMU_MSG_DramLogSetDramSize,
+					  (uint32_t)memory_pool->size);
+	if (ret)
+		return ret;
+
+	return ret;
+}
+
+static int smu_v11_0_check_pptable(struct smu_context *smu)
+{
+	int ret;
+
+	ret = smu_check_powerplay_table(smu);
+	return ret;
+}
+
+static int smu_v11_0_parse_pptable(struct smu_context *smu)
+{
+	int ret;
+
+	struct smu_table_context *table_context = &smu->smu_table;
+
+	if (table_context->driver_pptable)
+		return -EINVAL;
+
+	table_context->driver_pptable = kzalloc(sizeof(PPTable_t), GFP_KERNEL);
+
+	if (!table_context->driver_pptable)
+		return -ENOMEM;
+
+	ret = smu_store_powerplay_table(smu);
+	if (ret)
+		return -EINVAL;
+
+	ret = smu_append_powerplay_table(smu);
+
+	return ret;
+}
+
+static int smu_v11_0_populate_smc_pptable(struct smu_context *smu)
+{
+	int ret;
+
+	ret = smu_set_default_dpm_table(smu);
+
+	return ret;
+}
+
+static int smu_v11_0_write_pptable(struct smu_context *smu)
+{
+	struct smu_table_context *table_context = &smu->smu_table;
+	int ret = 0;
+
+	ret = smu_update_table(smu, TABLE_PPTABLE, table_context->driver_pptable, true);
+
+	return ret;
+}
+
+static int smu_v11_0_write_watermarks_table(struct smu_context *smu)
+{
+	return smu_update_table(smu, TABLE_WATERMARKS,
+				smu->smu_table.tables[TABLE_WATERMARKS].cpu_addr, true);
+}
+
+static int smu_v11_0_set_deep_sleep_dcefclk(struct smu_context *smu, uint32_t clk)
+{
+	int ret;
+
+	ret = smu_send_smc_msg_with_param(smu,
+					  SMU_MSG_SetMinDeepSleepDcefclk, clk);
+	if (ret)
+		pr_err("SMU11 attempt to set divider for DCEFCLK Failed!");
+
+	return ret;
+}
+
+static int smu_v11_0_set_min_dcef_deep_sleep(struct smu_context *smu)
+{
+	struct smu_table_context *table_context = &smu->smu_table;
+
+	if (!table_context)
+		return -EINVAL;
+
+	return smu_set_deep_sleep_dcefclk(smu,
+					  table_context->boot_values.dcefclk / 100);
+}
+
+static int smu_v11_0_set_tool_table_location(struct smu_context *smu)
+{
+	int ret = 0;
+	struct smu_table *tool_table = &smu->smu_table.tables[TABLE_PMSTATUSLOG];
+
+	if (tool_table->mc_address) {
+		ret = smu_send_smc_msg_with_param(smu,
+				SMU_MSG_SetToolsDramAddrHigh,
+				upper_32_bits(tool_table->mc_address));
+		if (!ret)
+			ret = smu_send_smc_msg_with_param(smu,
+				SMU_MSG_SetToolsDramAddrLow,
+				lower_32_bits(tool_table->mc_address));
+	}
+
+	return ret;
+}
+
+static int smu_v11_0_init_display(struct smu_context *smu)
+{
+	int ret = 0;
+	ret = smu_send_smc_msg_with_param(smu, SMU_MSG_NumOfDisplays, 0);
+	return ret;
+}
+
+static int smu_v11_0_update_feature_enable_state(struct smu_context *smu, uint32_t feature_id, bool enabled)
+{
+	uint32_t feature_low = 0, feature_high = 0;
+	int ret = 0;
+
+	if (feature_id >= 0 && feature_id < 31)
+		feature_low = (1 << feature_id);
+	else if (feature_id > 31 && feature_id < 63)
+		feature_high = (1 << feature_id);
+	else
+		return -EINVAL;
+
+	if (enabled) {
+		ret = smu_send_smc_msg_with_param(smu, SMU_MSG_EnableSmuFeaturesLow,
+						  feature_low);
+		if (ret)
+			return ret;
+		ret = smu_send_smc_msg_with_param(smu, SMU_MSG_EnableSmuFeaturesHigh,
+						  feature_high);
+		if (ret)
+			return ret;
+
+	} else {
+		ret = smu_send_smc_msg_with_param(smu, SMU_MSG_DisableSmuFeaturesLow,
+						  feature_low);
+		if (ret)
+			return ret;
+		ret = smu_send_smc_msg_with_param(smu, SMU_MSG_DisableSmuFeaturesHigh,
+						  feature_high);
+		if (ret)
+			return ret;
+
+	}
+
+	return ret;
+}
+
+static int smu_v11_0_set_allowed_mask(struct smu_context *smu)
+{
+	struct smu_feature *feature = &smu->smu_feature;
+	int ret = 0;
+	uint32_t feature_mask[2];
+
+	mutex_lock(&feature->mutex);
+	if (bitmap_empty(feature->allowed, SMU_FEATURE_MAX) || feature->feature_num < 64)
+		goto failed;
+
+	bitmap_copy((unsigned long *)feature_mask, feature->allowed, 64);
+
+	ret = smu_send_smc_msg_with_param(smu, SMU_MSG_SetAllowedFeaturesMaskHigh,
+					  feature_mask[1]);
+	if (ret)
+		goto failed;
+
+	ret = smu_send_smc_msg_with_param(smu, SMU_MSG_SetAllowedFeaturesMaskLow,
+					  feature_mask[0]);
+	if (ret)
+		goto failed;
+
+failed:
+	mutex_unlock(&feature->mutex);
+	return ret;
+}
+
+static int smu_v11_0_get_enabled_mask(struct smu_context *smu,
+				      uint32_t *feature_mask, uint32_t num)
+{
+	uint32_t feature_mask_high = 0, feature_mask_low = 0;
+	int ret = 0;
+
+	if (!feature_mask || num < 2)
+		return -EINVAL;
+
+	ret = smu_send_smc_msg(smu, SMU_MSG_GetEnabledSmuFeaturesHigh);
+	if (ret)
+		return ret;
+	ret = smu_read_smc_arg(smu, &feature_mask_high);
+	if (ret)
+		return ret;
+
+	ret = smu_send_smc_msg(smu, SMU_MSG_GetEnabledSmuFeaturesLow);
+	if (ret)
+		return ret;
+	ret = smu_read_smc_arg(smu, &feature_mask_low);
+	if (ret)
+		return ret;
+
+	feature_mask[0] = feature_mask_low;
+	feature_mask[1] = feature_mask_high;
+
+	return ret;
+}
+
+static bool smu_v11_0_is_dpm_running(struct smu_context *smu)
+{
+	int ret = 0;
+	uint32_t feature_mask[2];
+	unsigned long feature_enabled;
+	ret = smu_v11_0_get_enabled_mask(smu, feature_mask, 2);
+	feature_enabled = (unsigned long)((uint64_t)feature_mask[0] |
+			   ((uint64_t)feature_mask[1] << 32));
+	return !!(feature_enabled & SMC_DPM_FEATURE);
+}
+
+static int smu_v11_0_system_features_control(struct smu_context *smu,
+					     bool en)
+{
+	struct smu_feature *feature = &smu->smu_feature;
+	uint32_t feature_mask[2];
+	int ret = 0;
+
+	ret = smu_send_smc_msg(smu, (en ? SMU_MSG_EnableAllSmuFeatures :
+				     SMU_MSG_DisableAllSmuFeatures));
+	if (ret)
+		return ret;
+	ret = smu_feature_get_enabled_mask(smu, feature_mask, 2);
+	if (ret)
+		return ret;
+
+	bitmap_copy(feature->enabled, (unsigned long *)&feature_mask,
+		    feature->feature_num);
+	bitmap_copy(feature->supported, (unsigned long *)&feature_mask,
+		    feature->feature_num);
+
+	return ret;
+}
+
+static int smu_v11_0_notify_display_change(struct smu_context *smu)
+{
+	int ret = 0;
+
+	if (smu_feature_is_enabled(smu, FEATURE_DPM_UCLK_BIT))
+	    ret = smu_send_smc_msg_with_param(smu, SMU_MSG_SetUclkFastSwitch, 1);
+
+	return ret;
+}
+
+static int
+smu_v11_0_get_max_sustainable_clock(struct smu_context *smu, uint32_t *clock,
+				    PPCLK_e clock_select)
+{
+	int ret = 0;
+
+	ret = smu_send_smc_msg_with_param(smu, SMU_MSG_GetDcModeMaxDpmFreq,
+					  clock_select << 16);
+	if (ret) {
+		pr_err("[GetMaxSustainableClock] Failed to get max DC clock from SMC!");
+		return ret;
+	}
+
+	ret = smu_read_smc_arg(smu, clock);
+	if (ret)
+		return ret;
+
+	if (*clock != 0)
+		return 0;
+
+	/* if DC limit is zero, return AC limit */
+	ret = smu_send_smc_msg_with_param(smu, SMU_MSG_GetMaxDpmFreq,
+					  clock_select << 16);
+	if (ret) {
+		pr_err("[GetMaxSustainableClock] failed to get max AC clock from SMC!");
+		return ret;
+	}
+
+	ret = smu_read_smc_arg(smu, clock);
+
+	return ret;
+}
+
+static int smu_v11_0_init_max_sustainable_clocks(struct smu_context *smu)
+{
+	struct smu_11_0_max_sustainable_clocks *max_sustainable_clocks;
+	int ret = 0;
+
+	max_sustainable_clocks = kzalloc(sizeof(struct smu_11_0_max_sustainable_clocks),
+					 GFP_KERNEL);
+	smu->smu_table.max_sustainable_clocks = (void *)max_sustainable_clocks;
+
+	max_sustainable_clocks->uclock = smu->smu_table.boot_values.uclk / 100;
+	max_sustainable_clocks->soc_clock = smu->smu_table.boot_values.socclk / 100;
+	max_sustainable_clocks->dcef_clock = smu->smu_table.boot_values.dcefclk / 100;
+	max_sustainable_clocks->display_clock = 0xFFFFFFFF;
+	max_sustainable_clocks->phy_clock = 0xFFFFFFFF;
+	max_sustainable_clocks->pixel_clock = 0xFFFFFFFF;
+
+	if (smu_feature_is_enabled(smu, FEATURE_DPM_UCLK_BIT)) {
+		ret = smu_v11_0_get_max_sustainable_clock(smu,
+							  &(max_sustainable_clocks->uclock),
+							  PPCLK_UCLK);
+		if (ret) {
+			pr_err("[%s] failed to get max UCLK from SMC!",
+			       __func__);
+			return ret;
+		}
+	}
+
+	if (smu_feature_is_enabled(smu, FEATURE_DPM_SOCCLK_BIT)) {
+		ret = smu_v11_0_get_max_sustainable_clock(smu,
+							  &(max_sustainable_clocks->soc_clock),
+							  PPCLK_SOCCLK);
+		if (ret) {
+			pr_err("[%s] failed to get max SOCCLK from SMC!",
+			       __func__);
+			return ret;
+		}
+	}
+
+	if (smu_feature_is_enabled(smu, FEATURE_DPM_DCEFCLK_BIT)) {
+		ret = smu_v11_0_get_max_sustainable_clock(smu,
+							  &(max_sustainable_clocks->dcef_clock),
+							  PPCLK_DCEFCLK);
+		if (ret) {
+			pr_err("[%s] failed to get max DCEFCLK from SMC!",
+			       __func__);
+			return ret;
+		}
+
+		ret = smu_v11_0_get_max_sustainable_clock(smu,
+							  &(max_sustainable_clocks->display_clock),
+							  PPCLK_DISPCLK);
+		if (ret) {
+			pr_err("[%s] failed to get max DISPCLK from SMC!",
+			       __func__);
+			return ret;
+		}
+		ret = smu_v11_0_get_max_sustainable_clock(smu,
+							  &(max_sustainable_clocks->phy_clock),
+							  PPCLK_PHYCLK);
+		if (ret) {
+			pr_err("[%s] failed to get max PHYCLK from SMC!",
+			       __func__);
+			return ret;
+		}
+		ret = smu_v11_0_get_max_sustainable_clock(smu,
+							  &(max_sustainable_clocks->pixel_clock),
+							  PPCLK_PIXCLK);
+		if (ret) {
+			pr_err("[%s] failed to get max PIXCLK from SMC!",
+			       __func__);
+			return ret;
+		}
+	}
+
+	if (max_sustainable_clocks->soc_clock < max_sustainable_clocks->uclock)
+		max_sustainable_clocks->uclock = max_sustainable_clocks->soc_clock;
+
+	return 0;
+}
+
+static int smu_v11_0_get_power_limit(struct smu_context *smu,
+				     uint32_t *limit,
+				     bool get_default)
+{
+	int ret = 0;
+
+	if (get_default) {
+		mutex_lock(&smu->mutex);
+		*limit = smu->default_power_limit;
+		if (smu->od_enabled) {
+			*limit *= (100 + smu->smu_table.TDPODLimit);
+			*limit /= 100;
+		}
+		mutex_unlock(&smu->mutex);
+	} else {
+		ret = smu_send_smc_msg_with_param(smu, SMU_MSG_GetPptLimit,
+						  POWER_SOURCE_AC << 16);
+		if (ret) {
+			pr_err("[%s] get PPT limit failed!", __func__);
+			return ret;
+		}
+		smu_read_smc_arg(smu, limit);
+		smu->power_limit = *limit;
+	}
+
+	return ret;
+}
+
+static int smu_v11_0_set_power_limit(struct smu_context *smu, uint32_t n)
+{
+	uint32_t max_power_limit;
+	int ret = 0;
+
+	if (n == 0)
+		n = smu->default_power_limit;
+
+	max_power_limit = smu->default_power_limit;
+
+	if (smu->od_enabled) {
+		max_power_limit *= (100 + smu->smu_table.TDPODLimit);
+		max_power_limit /= 100;
+	}
+
+	if (smu_feature_is_enabled(smu, FEATURE_PPT_BIT))
+		ret = smu_send_smc_msg_with_param(smu, SMU_MSG_SetPptLimit, n);
+	if (ret) {
+		pr_err("[%s] Set power limit Failed!", __func__);
+		return ret;
+	}
+
+	return ret;
+}
+
+static int smu_v11_0_get_current_clk_freq(struct smu_context *smu, uint32_t clk_id, uint32_t *value)
+{
+	int ret = 0;
+	uint32_t freq;
+
+	if (clk_id >= PPCLK_COUNT || !value)
+		return -EINVAL;
+
+	ret = smu_send_smc_msg_with_param(smu,
+			SMU_MSG_GetDpmClockFreq, (clk_id << 16));
+	if (ret)
+		return ret;
+
+	ret = smu_read_smc_arg(smu, &freq);
+	if (ret)
+		return ret;
+
+	freq *= 100;
+	*value = freq;
+
+	return ret;
+}
+
+static int smu_v11_0_get_thermal_range(struct smu_context *smu,
+				struct PP_TemperatureRange *range)
+{
+	memcpy(range, &SMU7ThermalWithDelayPolicy[0], sizeof(struct PP_TemperatureRange));
+
+	range->max = smu->smu_table.software_shutdown_temp *
+		PP_TEMPERATURE_UNITS_PER_CENTIGRADES;
+
+	return 0;
+}
+
+static int smu_v11_0_set_thermal_range(struct smu_context *smu,
+			struct PP_TemperatureRange *range)
+{
+	struct amdgpu_device *adev = smu->adev;
+	int low = SMU11_THERMAL_MINIMUM_ALERT_TEMP *
+		PP_TEMPERATURE_UNITS_PER_CENTIGRADES;
+	int high = SMU11_THERMAL_MAXIMUM_ALERT_TEMP *
+		PP_TEMPERATURE_UNITS_PER_CENTIGRADES;
+	uint32_t val;
+
+	if (low < range->min)
+		low = range->min;
+	if (high > range->max)
+		high = range->max;
+
+	if (low > high)
+		return -EINVAL;
+
+	val = RREG32_SOC15(THM, 0, mmTHM_THERMAL_INT_CTRL);
+	val = REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, MAX_IH_CREDIT, 5);
+	val = REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, THERM_IH_HW_ENA, 1);
+	val = REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, DIG_THERM_INTH, (high / PP_TEMPERATURE_UNITS_PER_CENTIGRADES));
+	val = REG_SET_FIELD(val, THM_THERMAL_INT_CTRL, DIG_THERM_INTL, (low / PP_TEMPERATURE_UNITS_PER_CENTIGRADES));
+	val = val & (~THM_THERMAL_INT_CTRL__THERM_TRIGGER_MASK_MASK);
+
+	WREG32_SOC15(THM, 0, mmTHM_THERMAL_INT_CTRL, val);
+
+	return 0;
+}
+
+static int smu_v11_0_enable_thermal_alert(struct smu_context *smu)
+{
+	struct amdgpu_device *adev = smu->adev;
+	uint32_t val = 0;
+
+	val |= (1 << THM_THERMAL_INT_ENA__THERM_INTH_CLR__SHIFT);
+	val |= (1 << THM_THERMAL_INT_ENA__THERM_INTL_CLR__SHIFT);
+	val |= (1 << THM_THERMAL_INT_ENA__THERM_TRIGGER_CLR__SHIFT);
+
+	WREG32_SOC15(THM, 0, mmTHM_THERMAL_INT_ENA, val);
+
+	return 0;
+}
+
+static int smu_v11_0_set_thermal_fan_table(struct smu_context *smu)
+{
+	int ret;
+	struct smu_table_context *table_context = &smu->smu_table;
+	PPTable_t *pptable = table_context->driver_pptable;
+
+	ret = smu_send_smc_msg_with_param(smu, SMU_MSG_SetFanTemperatureTarget,
+			(uint32_t)pptable->FanTargetTemperature);
+
+	return ret;
+}
+
+static int smu_v11_0_start_thermal_control(struct smu_context *smu)
+{
+	int ret = 0;
+	struct PP_TemperatureRange range;
+	struct amdgpu_device *adev = smu->adev;
+
+	smu_v11_0_get_thermal_range(smu, &range);
+
+	if (smu->smu_table.thermal_controller_type) {
+		ret = smu_v11_0_set_thermal_range(smu, &range);
+		if (ret)
+			return ret;
+
+		ret = smu_v11_0_enable_thermal_alert(smu);
+		if (ret)
+			return ret;
+		ret = smu_v11_0_set_thermal_fan_table(smu);
+		if (ret)
+			return ret;
+	}
+
+	adev->pm.dpm.thermal.min_temp = range.min;
+	adev->pm.dpm.thermal.max_temp = range.max;
+
+	return ret;
+}
+
+static int smu_v11_0_get_current_activity_percent(struct smu_context *smu,
+						  uint32_t *value)
+{
+	int ret = 0;
+	SmuMetrics_t metrics;
+
+	if (!value)
+		return -EINVAL;
+
+	ret = smu_update_table(smu, TABLE_SMU_METRICS, (void *)&metrics, false);
+	if (ret)
+		return ret;
+
+	*value = metrics.AverageGfxActivity;
+
+	return 0;
+}
+
+static int smu_v11_0_thermal_get_temperature(struct smu_context *smu, uint32_t *value)
+{
+	struct amdgpu_device *adev = smu->adev;
+	uint32_t temp = 0;
+
+	if (!value)
+		return -EINVAL;
+
+	temp = RREG32_SOC15(THM, 0, mmCG_MULT_THERMAL_STATUS);
+	temp = (temp & CG_MULT_THERMAL_STATUS__CTF_TEMP_MASK) >>
+			CG_MULT_THERMAL_STATUS__CTF_TEMP__SHIFT;
+
+	temp = temp & 0x1ff;
+	temp *= SMU11_TEMPERATURE_UNITS_PER_CENTIGRADES;
+
+	*value = temp;
+
+	return 0;
+}
+
+static int smu_v11_0_get_gpu_power(struct smu_context *smu, uint32_t *value)
+{
+	int ret = 0;
+	SmuMetrics_t metrics;
+
+	if (!value)
+		return -EINVAL;
+
+	ret = smu_update_table(smu, TABLE_SMU_METRICS, (void *)&metrics, false);
+	if (ret)
+		return ret;
+
+	*value = metrics.CurrSocketPower << 8;
+
+	return 0;
+}
+
+static uint16_t convert_to_vddc(uint8_t vid)
+{
+	return (uint16_t) ((6200 - (vid * 25)) / SMU11_VOLTAGE_SCALE);
+}
+
+static int smu_v11_0_get_gfx_vdd(struct smu_context *smu, uint32_t *value)
+{
+	struct amdgpu_device *adev = smu->adev;
+	uint32_t vdd = 0, val_vid = 0;
+
+	if (!value)
+		return -EINVAL;
+	val_vid = (RREG32_SOC15(SMUIO, 0, mmSMUSVI0_TEL_PLANE0) &
+		SMUSVI0_TEL_PLANE0__SVI0_PLANE0_VDDCOR_MASK) >>
+		SMUSVI0_TEL_PLANE0__SVI0_PLANE0_VDDCOR__SHIFT;
+
+	vdd = (uint32_t)convert_to_vddc((uint8_t)val_vid);
+
+	*value = vdd;
+
+	return 0;
+
+}
+
+static int smu_v11_0_read_sensor(struct smu_context *smu,
+				 enum amd_pp_sensors sensor,
+				 void *data, uint32_t *size)
+{
+	struct smu_table_context *table_context = &smu->smu_table;
+	PPTable_t *pptable = table_context->driver_pptable;
+	int ret = 0;
+	switch (sensor) {
+	case AMDGPU_PP_SENSOR_GPU_LOAD:
+		ret = smu_v11_0_get_current_activity_percent(smu,
+							     (uint32_t *)data);
+		*size = 4;
+		break;
+	case AMDGPU_PP_SENSOR_GFX_MCLK:
+		ret = smu_get_current_clk_freq(smu, PPCLK_UCLK, (uint32_t *)data);
+		*size = 4;
+		break;
+	case AMDGPU_PP_SENSOR_GFX_SCLK:
+		ret = smu_get_current_clk_freq(smu, PPCLK_GFXCLK, (uint32_t *)data);
+		*size = 4;
+		break;
+	case AMDGPU_PP_SENSOR_GPU_TEMP:
+		ret = smu_v11_0_thermal_get_temperature(smu, (uint32_t *)data);
+		*size = 4;
+		break;
+	case AMDGPU_PP_SENSOR_GPU_POWER:
+		ret = smu_v11_0_get_gpu_power(smu, (uint32_t *)data);
+		*size = 4;
+		break;
+	case AMDGPU_PP_SENSOR_VDDGFX:
+		ret = smu_v11_0_get_gfx_vdd(smu, (uint32_t *)data);
+		*size = 4;
+		break;
+	case AMDGPU_PP_SENSOR_UVD_POWER:
+		*(uint32_t *)data = smu_feature_is_enabled(smu, FEATURE_DPM_UVD_BIT) ? 1 : 0;
+		*size = 4;
+		break;
+	case AMDGPU_PP_SENSOR_VCE_POWER:
+		*(uint32_t *)data = smu_feature_is_enabled(smu, FEATURE_DPM_VCE_BIT) ? 1 : 0;
+		*size = 4;
+		break;
+	case AMDGPU_PP_SENSOR_MIN_FAN_RPM:
+		*(uint32_t *)data = 0;
+		*size = 4;
+		break;
+	case AMDGPU_PP_SENSOR_MAX_FAN_RPM:
+		*(uint32_t *)data = pptable->FanMaximumRpm;
+		*size = 4;
+		break;
+	default:
+		ret = smu_common_read_sensor(smu, sensor, data, size);
+		break;
+	}
+
+	if (ret)
+		*size = 0;
+
+	return ret;
+}
+
+static int
+smu_v11_0_display_clock_voltage_request(struct smu_context *smu,
+					struct pp_display_clock_request
+					*clock_req)
+{
+	enum amd_pp_clock_type clk_type = clock_req->clock_type;
+	int ret = 0;
+	PPCLK_e clk_select = 0;
+	uint32_t clk_freq = clock_req->clock_freq_in_khz / 1000;
+
+	if (smu_feature_is_enabled(smu, FEATURE_DPM_DCEFCLK_BIT)) {
+		switch (clk_type) {
+		case amd_pp_dcef_clock:
+			clk_select = PPCLK_DCEFCLK;
+			break;
+		case amd_pp_disp_clock:
+			clk_select = PPCLK_DISPCLK;
+			break;
+		case amd_pp_pixel_clock:
+			clk_select = PPCLK_PIXCLK;
+			break;
+		case amd_pp_phy_clock:
+			clk_select = PPCLK_PHYCLK;
+			break;
+		default:
+			pr_info("[%s] Invalid Clock Type!", __func__);
+			ret = -EINVAL;
+			break;
+		}
+
+		if (ret)
+			goto failed;
+
+		ret = smu_send_smc_msg_with_param(smu, SMU_MSG_SetHardMinByFreq,
+						  (clk_select << 16) | clk_freq);
+	}
+
+failed:
+	return ret;
+}
+
+static int smu_v11_0_set_watermarks_table(struct smu_context *smu,
+					  Watermarks_t *table, struct
+					  dm_pp_wm_sets_with_clock_ranges_soc15
+					  *clock_ranges)
+{
+	int i;
+
+	if (!table || !clock_ranges)
+		return -EINVAL;
+
+	if (clock_ranges->num_wm_dmif_sets > 4 ||
+	    clock_ranges->num_wm_mcif_sets > 4)
+                return -EINVAL;
+
+        for (i = 0; i < clock_ranges->num_wm_dmif_sets; i++) {
+		table->WatermarkRow[1][i].MinClock =
+			cpu_to_le16((uint16_t)
+			(clock_ranges->wm_dmif_clocks_ranges[i].wm_min_dcfclk_clk_in_khz /
+			1000));
+		table->WatermarkRow[1][i].MaxClock =
+			cpu_to_le16((uint16_t)
+			(clock_ranges->wm_dmif_clocks_ranges[i].wm_max_dcfclk_clk_in_khz /
+			1000));
+		table->WatermarkRow[1][i].MinUclk =
+			cpu_to_le16((uint16_t)
+			(clock_ranges->wm_dmif_clocks_ranges[i].wm_min_mem_clk_in_khz /
+			1000));
+		table->WatermarkRow[1][i].MaxUclk =
+			cpu_to_le16((uint16_t)
+			(clock_ranges->wm_dmif_clocks_ranges[i].wm_max_mem_clk_in_khz /
+			1000));
+		table->WatermarkRow[1][i].WmSetting = (uint8_t)
+				clock_ranges->wm_dmif_clocks_ranges[i].wm_set_id;
+        }
+
+	for (i = 0; i < clock_ranges->num_wm_mcif_sets; i++) {
+		table->WatermarkRow[0][i].MinClock =
+			cpu_to_le16((uint16_t)
+			(clock_ranges->wm_mcif_clocks_ranges[i].wm_min_socclk_clk_in_khz /
+			1000));
+		table->WatermarkRow[0][i].MaxClock =
+			cpu_to_le16((uint16_t)
+			(clock_ranges->wm_mcif_clocks_ranges[i].wm_max_socclk_clk_in_khz /
+			1000));
+		table->WatermarkRow[0][i].MinUclk =
+			cpu_to_le16((uint16_t)
+			(clock_ranges->wm_mcif_clocks_ranges[i].wm_min_mem_clk_in_khz /
+			1000));
+		table->WatermarkRow[0][i].MaxUclk =
+			cpu_to_le16((uint16_t)
+			(clock_ranges->wm_mcif_clocks_ranges[i].wm_max_mem_clk_in_khz /
+			1000));
+		table->WatermarkRow[0][i].WmSetting = (uint8_t)
+				clock_ranges->wm_mcif_clocks_ranges[i].wm_set_id;
+        }
+
+	return 0;
+}
+
+static int
+smu_v11_0_set_watermarks_for_clock_ranges(struct smu_context *smu, struct
+					  dm_pp_wm_sets_with_clock_ranges_soc15
+					  *clock_ranges)
+{
+	int ret = 0;
+	struct smu_table *watermarks = &smu->smu_table.tables[TABLE_WATERMARKS];
+	Watermarks_t *table = watermarks->cpu_addr;
+
+	if (!smu->disable_watermark &&
+	    smu_feature_is_enabled(smu, FEATURE_DPM_DCEFCLK_BIT) &&
+	    smu_feature_is_enabled(smu, FEATURE_DPM_SOCCLK_BIT)) {
+		smu_v11_0_set_watermarks_table(smu, table, clock_ranges);
+		smu->watermarks_bitmap |= WATERMARKS_EXIST;
+		smu->watermarks_bitmap &= ~WATERMARKS_LOADED;
+	}
+
+	return ret;
+}
+
+static int smu_v11_0_get_clock_ranges(struct smu_context *smu,
+				      uint32_t *clock,
+				      PPCLK_e clock_select,
+				      bool max)
+{
+	int ret;
+	*clock = 0;
+	if (max) {
+		ret = smu_send_smc_msg_with_param(smu, SMU_MSG_GetMaxDpmFreq,
+					    (clock_select << 16));
+		if (ret) {
+			pr_err("[GetClockRanges] Failed to get max clock from SMC!\n");
+			return ret;
+		}
+		smu_read_smc_arg(smu, clock);
+	} else {
+		ret = smu_send_smc_msg_with_param(smu, SMU_MSG_GetMinDpmFreq,
+					    (clock_select << 16));
+		if (ret) {
+			pr_err("[GetClockRanges] Failed to get min clock from SMC!\n");
+			return ret;
+		}
+		smu_read_smc_arg(smu, clock);
+	}
+
+	return 0;
+}
+
+static uint32_t smu_v11_0_dpm_get_sclk(struct smu_context *smu, bool low)
+{
+	uint32_t gfx_clk;
+	int ret;
+
+	if (!smu_feature_is_enabled(smu, FEATURE_DPM_GFXCLK_BIT)) {
+		pr_err("[GetSclks]: gfxclk dpm not enabled!\n");
+		return -EPERM;
+	}
+
+	if (low) {
+		ret = smu_v11_0_get_clock_ranges(smu, &gfx_clk, PPCLK_GFXCLK, false);
+		if (ret) {
+			pr_err("[GetSclks]: fail to get min PPCLK_GFXCLK\n");
+			return ret;
+		}
+	} else {
+		ret = smu_v11_0_get_clock_ranges(smu, &gfx_clk, PPCLK_GFXCLK, true);
+		if (ret) {
+			pr_err("[GetSclks]: fail to get max PPCLK_GFXCLK\n");
+			return ret;
+		}
+	}
+
+	return (gfx_clk * 100);
+}
+
+static uint32_t smu_v11_0_dpm_get_mclk(struct smu_context *smu, bool low)
+{
+	uint32_t mem_clk;
+	int ret;
+
+	if (!smu_feature_is_enabled(smu, FEATURE_DPM_UCLK_BIT)) {
+		pr_err("[GetMclks]: memclk dpm not enabled!\n");
+		return -EPERM;
+	}
+
+	if (low) {
+		ret = smu_v11_0_get_clock_ranges(smu, &mem_clk, PPCLK_UCLK, false);
+		if (ret) {
+			pr_err("[GetMclks]: fail to get min PPCLK_UCLK\n");
+			return ret;
+		}
+	} else {
+		ret = smu_v11_0_get_clock_ranges(smu, &mem_clk, PPCLK_GFXCLK, true);
+		if (ret) {
+			pr_err("[GetMclks]: fail to get max PPCLK_UCLK\n");
+			return ret;
+		}
+	}
+
+	return (mem_clk * 100);
+}
+
+static int smu_v11_0_set_od8_default_settings(struct smu_context *smu,
+					      bool initialize)
+{
+	struct smu_table_context *table_context = &smu->smu_table;
+	int ret;
+
+	if (initialize) {
+		if (table_context->overdrive_table)
+			return -EINVAL;
+
+		table_context->overdrive_table = kzalloc(sizeof(OverDriveTable_t), GFP_KERNEL);
+
+		if (!table_context->overdrive_table)
+			return -ENOMEM;
+
+		ret = smu_update_table(smu, TABLE_OVERDRIVE, table_context->overdrive_table, false);
+		if (ret) {
+			pr_err("Failed to export over drive table!\n");
+			return ret;
+		}
+
+		smu_set_default_od8_settings(smu);
+	}
+
+	ret = smu_update_table(smu, TABLE_OVERDRIVE, table_context->overdrive_table, true);
+	if (ret) {
+		pr_err("Failed to import over drive table!\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static int smu_v11_0_conv_power_profile_to_pplib_workload(int power_profile)
+{
+	int pplib_workload = 0;
+
+	switch (power_profile) {
+	case PP_SMC_POWER_PROFILE_BOOTUP_DEFAULT:
+	     pplib_workload = WORKLOAD_DEFAULT_BIT;
+	     break;
+	case PP_SMC_POWER_PROFILE_FULLSCREEN3D:
+	     pplib_workload = WORKLOAD_PPLIB_FULL_SCREEN_3D_BIT;
+	     break;
+	case PP_SMC_POWER_PROFILE_POWERSAVING:
+	     pplib_workload = WORKLOAD_PPLIB_POWER_SAVING_BIT;
+	     break;
+	case PP_SMC_POWER_PROFILE_VIDEO:
+	     pplib_workload = WORKLOAD_PPLIB_VIDEO_BIT;
+	     break;
+	case PP_SMC_POWER_PROFILE_VR:
+	     pplib_workload = WORKLOAD_PPLIB_VR_BIT;
+	     break;
+	case PP_SMC_POWER_PROFILE_COMPUTE:
+	     pplib_workload = WORKLOAD_PPLIB_COMPUTE_BIT;
+	     break;
+	case PP_SMC_POWER_PROFILE_CUSTOM:
+		pplib_workload = WORKLOAD_PPLIB_CUSTOM_BIT;
+		break;
+	}
+
+	return pplib_workload;
+}
+
+static int smu_v11_0_get_power_profile_mode(struct smu_context *smu, char *buf)
+{
+	DpmActivityMonitorCoeffInt_t activity_monitor;
+	uint32_t i, size = 0;
+	uint16_t workload_type = 0;
+	static const char *profile_name[] = {
+					"BOOTUP_DEFAULT",
+					"3D_FULL_SCREEN",
+					"POWER_SAVING",
+					"VIDEO",
+					"VR",
+					"COMPUTE",
+					"CUSTOM"};
+	static const char *title[] = {
+			"PROFILE_INDEX(NAME)",
+			"CLOCK_TYPE(NAME)",
+			"FPS",
+			"UseRlcBusy",
+			"MinActiveFreqType",
+			"MinActiveFreq",
+			"BoosterFreqType",
+			"BoosterFreq",
+			"PD_Data_limit_c",
+			"PD_Data_error_coeff",
+			"PD_Data_error_rate_coeff"};
+	int result = 0;
+
+	if (!buf)
+		return -EINVAL;
+
+	size += sprintf(buf + size, "%16s %s %s %s %s %s %s %s %s %s %s\n",
+			title[0], title[1], title[2], title[3], title[4], title[5],
+			title[6], title[7], title[8], title[9], title[10]);
+
+	for (i = 0; i <= PP_SMC_POWER_PROFILE_CUSTOM; i++) {
+		/* conv PP_SMC_POWER_PROFILE* to WORKLOAD_PPLIB_*_BIT */
+		workload_type = smu_v11_0_conv_power_profile_to_pplib_workload(i);
+		result = smu_update_table_with_arg(smu, TABLE_ACTIVITY_MONITOR_COEFF,
+						   workload_type, &activity_monitor, false);
+		if (result) {
+			pr_err("[%s] Failed to get activity monitor!", __func__);
+			return result;
+		}
+
+		size += sprintf(buf + size, "%2d %14s%s:\n",
+			i, profile_name[i], (i == smu->power_profile_mode) ? "*" : " ");
+
+		size += sprintf(buf + size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d %7d\n",
+			" ",
+			0,
+			"GFXCLK",
+			activity_monitor.Gfx_FPS,
+			activity_monitor.Gfx_UseRlcBusy,
+			activity_monitor.Gfx_MinActiveFreqType,
+			activity_monitor.Gfx_MinActiveFreq,
+			activity_monitor.Gfx_BoosterFreqType,
+			activity_monitor.Gfx_BoosterFreq,
+			activity_monitor.Gfx_PD_Data_limit_c,
+			activity_monitor.Gfx_PD_Data_error_coeff,
+			activity_monitor.Gfx_PD_Data_error_rate_coeff);
+
+		size += sprintf(buf + size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d %7d\n",
+			" ",
+			1,
+			"SOCCLK",
+			activity_monitor.Soc_FPS,
+			activity_monitor.Soc_UseRlcBusy,
+			activity_monitor.Soc_MinActiveFreqType,
+			activity_monitor.Soc_MinActiveFreq,
+			activity_monitor.Soc_BoosterFreqType,
+			activity_monitor.Soc_BoosterFreq,
+			activity_monitor.Soc_PD_Data_limit_c,
+			activity_monitor.Soc_PD_Data_error_coeff,
+			activity_monitor.Soc_PD_Data_error_rate_coeff);
+
+		size += sprintf(buf + size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d %7d\n",
+			" ",
+			2,
+			"UCLK",
+			activity_monitor.Mem_FPS,
+			activity_monitor.Mem_UseRlcBusy,
+			activity_monitor.Mem_MinActiveFreqType,
+			activity_monitor.Mem_MinActiveFreq,
+			activity_monitor.Mem_BoosterFreqType,
+			activity_monitor.Mem_BoosterFreq,
+			activity_monitor.Mem_PD_Data_limit_c,
+			activity_monitor.Mem_PD_Data_error_coeff,
+			activity_monitor.Mem_PD_Data_error_rate_coeff);
+
+		size += sprintf(buf + size, "%19s %d(%13s) %7d %7d %7d %7d %7d %7d %7d %7d %7d\n",
+			" ",
+			3,
+			"FCLK",
+			activity_monitor.Fclk_FPS,
+			activity_monitor.Fclk_UseRlcBusy,
+			activity_monitor.Fclk_MinActiveFreqType,
+			activity_monitor.Fclk_MinActiveFreq,
+			activity_monitor.Fclk_BoosterFreqType,
+			activity_monitor.Fclk_BoosterFreq,
+			activity_monitor.Fclk_PD_Data_limit_c,
+			activity_monitor.Fclk_PD_Data_error_coeff,
+			activity_monitor.Fclk_PD_Data_error_rate_coeff);
+	}
+
+	return size;
+}
+
+static int smu_v11_0_set_power_profile_mode(struct smu_context *smu, long *input, uint32_t size)
+{
+	DpmActivityMonitorCoeffInt_t activity_monitor;
+	int workload_type = 0, ret = 0;
+
+	smu->power_profile_mode = input[size];
+
+	if (smu->power_profile_mode > PP_SMC_POWER_PROFILE_CUSTOM) {
+		pr_err("Invalid power profile mode %d\n", smu->power_profile_mode);
+		return -EINVAL;
+	}
+
+	if (smu->power_profile_mode == PP_SMC_POWER_PROFILE_CUSTOM) {
+		ret = smu_update_table_with_arg(smu, TABLE_ACTIVITY_MONITOR_COEFF,
+						WORKLOAD_PPLIB_CUSTOM_BIT, &activity_monitor, false);
+		if (ret) {
+			pr_err("[%s] Failed to get activity monitor!", __func__);
+			return ret;
+		}
+
+		switch (input[0]) {
+		case 0: /* Gfxclk */
+			activity_monitor.Gfx_FPS = input[1];
+			activity_monitor.Gfx_UseRlcBusy = input[2];
+			activity_monitor.Gfx_MinActiveFreqType = input[3];
+			activity_monitor.Gfx_MinActiveFreq = input[4];
+			activity_monitor.Gfx_BoosterFreqType = input[5];
+			activity_monitor.Gfx_BoosterFreq = input[6];
+			activity_monitor.Gfx_PD_Data_limit_c = input[7];
+			activity_monitor.Gfx_PD_Data_error_coeff = input[8];
+			activity_monitor.Gfx_PD_Data_error_rate_coeff = input[9];
+			break;
+		case 1: /* Socclk */
+			activity_monitor.Soc_FPS = input[1];
+			activity_monitor.Soc_UseRlcBusy = input[2];
+			activity_monitor.Soc_MinActiveFreqType = input[3];
+			activity_monitor.Soc_MinActiveFreq = input[4];
+			activity_monitor.Soc_BoosterFreqType = input[5];
+			activity_monitor.Soc_BoosterFreq = input[6];
+			activity_monitor.Soc_PD_Data_limit_c = input[7];
+			activity_monitor.Soc_PD_Data_error_coeff = input[8];
+			activity_monitor.Soc_PD_Data_error_rate_coeff = input[9];
+			break;
+		case 2: /* Uclk */
+			activity_monitor.Mem_FPS = input[1];
+			activity_monitor.Mem_UseRlcBusy = input[2];
+			activity_monitor.Mem_MinActiveFreqType = input[3];
+			activity_monitor.Mem_MinActiveFreq = input[4];
+			activity_monitor.Mem_BoosterFreqType = input[5];
+			activity_monitor.Mem_BoosterFreq = input[6];
+			activity_monitor.Mem_PD_Data_limit_c = input[7];
+			activity_monitor.Mem_PD_Data_error_coeff = input[8];
+			activity_monitor.Mem_PD_Data_error_rate_coeff = input[9];
+			break;
+		case 3: /* Fclk */
+			activity_monitor.Fclk_FPS = input[1];
+			activity_monitor.Fclk_UseRlcBusy = input[2];
+			activity_monitor.Fclk_MinActiveFreqType = input[3];
+			activity_monitor.Fclk_MinActiveFreq = input[4];
+			activity_monitor.Fclk_BoosterFreqType = input[5];
+			activity_monitor.Fclk_BoosterFreq = input[6];
+			activity_monitor.Fclk_PD_Data_limit_c = input[7];
+			activity_monitor.Fclk_PD_Data_error_coeff = input[8];
+			activity_monitor.Fclk_PD_Data_error_rate_coeff = input[9];
+			break;
+		}
+
+		ret = smu_update_table_with_arg(smu, TABLE_ACTIVITY_MONITOR_COEFF,
+						WORKLOAD_PPLIB_COMPUTE_BIT, &activity_monitor, true);
+		if (ret) {
+			pr_err("[%s] Failed to set activity monitor!", __func__);
+			return ret;
+		}
+	}
+
+	/* conv PP_SMC_POWER_PROFILE* to WORKLOAD_PPLIB_*_BIT */
+	workload_type =
+		smu_v11_0_conv_power_profile_to_pplib_workload(smu->power_profile_mode);
+	smu_send_smc_msg_with_param(smu, SMU_MSG_SetWorkloadMask,
+				    1 << workload_type);
+
+	return ret;
+}
+
+static int smu_v11_0_update_od8_settings(struct smu_context *smu,
+					uint32_t index,
+					uint32_t value)
+{
+	struct smu_table_context *table_context = &smu->smu_table;
+	int ret;
+
+	ret = smu_update_table(smu, TABLE_OVERDRIVE,
+			       table_context->overdrive_table, false);
+	if (ret) {
+		pr_err("Failed to export over drive table!\n");
+		return ret;
+	}
+
+	smu_update_specified_od8_value(smu, index, value);
+
+	ret = smu_update_table(smu, TABLE_OVERDRIVE,
+			       table_context->overdrive_table, true);
+	if (ret) {
+		pr_err("Failed to import over drive table!\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static int smu_v11_0_dpm_set_uvd_enable(struct smu_context *smu, bool enable)
+{
+	if (!smu_feature_is_supported(smu, FEATURE_DPM_VCE_BIT))
+		return 0;
+
+	if (enable == smu_feature_is_enabled(smu, FEATURE_DPM_VCE_BIT))
+		return 0;
+
+	return smu_feature_set_enabled(smu, FEATURE_DPM_VCE_BIT, enable);
+}
+
+static int smu_v11_0_dpm_set_vce_enable(struct smu_context *smu, bool enable)
+{
+	if (!smu_feature_is_supported(smu, FEATURE_DPM_UVD_BIT))
+		return 0;
+
+	if (enable == smu_feature_is_enabled(smu, FEATURE_DPM_UVD_BIT))
+		return 0;
+
+	return smu_feature_set_enabled(smu, FEATURE_DPM_UVD_BIT, enable);
+}
+
+static int smu_v11_0_get_current_rpm(struct smu_context *smu,
+				     uint32_t *current_rpm)
+{
+	int ret;
+
+	ret = smu_send_smc_msg(smu, SMU_MSG_GetCurrentRpm);
+
+	if (ret) {
+		pr_err("Attempt to get current RPM from SMC Failed!\n");
+		return ret;
+	}
+
+	smu_read_smc_arg(smu, current_rpm);
+
+	return 0;
+}
+
+static uint32_t
+smu_v11_0_get_fan_control_mode(struct smu_context *smu)
+{
+	if (!smu_feature_is_enabled(smu, FEATURE_FAN_CONTROL_BIT))
+		return AMD_FAN_CTRL_MANUAL;
+	else
+		return AMD_FAN_CTRL_AUTO;
+}
+
+static int
+smu_v11_0_get_fan_speed_percent(struct smu_context *smu,
+					   uint32_t *speed)
+{
+	int ret = 0;
+	uint32_t percent = 0;
+	uint32_t current_rpm;
+	PPTable_t *pptable = smu->smu_table.driver_pptable;
+
+	ret = smu_v11_0_get_current_rpm(smu, &current_rpm);
+	percent = current_rpm * 100 / pptable->FanMaximumRpm;
+	*speed = percent > 100 ? 100 : percent;
+
+	return ret;
+}
+
+static int
+smu_v11_0_smc_fan_control(struct smu_context *smu, bool start)
+{
+	int ret = 0;
+
+	if (smu_feature_is_supported(smu, FEATURE_FAN_CONTROL_BIT))
+		return 0;
+
+	ret = smu_feature_set_enabled(smu, FEATURE_FAN_CONTROL_BIT, start);
+	if (ret)
+		pr_err("[%s]%s smc FAN CONTROL feature failed!",
+		       __func__, (start ? "Start" : "Stop"));
+
+	return ret;
+}
+
+static int
+smu_v11_0_set_fan_static_mode(struct smu_context *smu, uint32_t mode)
+{
+	struct amdgpu_device *adev = smu->adev;
+
+	WREG32_SOC15(THM, 0, mmCG_FDO_CTRL2,
+		     REG_SET_FIELD(RREG32_SOC15(THM, 0, mmCG_FDO_CTRL2),
+				   CG_FDO_CTRL2, TMIN, 0));
+	WREG32_SOC15(THM, 0, mmCG_FDO_CTRL2,
+		     REG_SET_FIELD(RREG32_SOC15(THM, 0, mmCG_FDO_CTRL2),
+				   CG_FDO_CTRL2, FDO_PWM_MODE, mode));
+
+	return 0;
+}
+
+static int
+smu_v11_0_set_fan_speed_percent(struct smu_context *smu, uint32_t speed)
+{
+	struct amdgpu_device *adev = smu->adev;
+	uint32_t duty100;
+	uint32_t duty;
+	uint64_t tmp64;
+	bool stop = 0;
+
+	if (speed > 100)
+		speed = 100;
+
+	if (smu_v11_0_smc_fan_control(smu, stop))
+		return -EINVAL;
+	duty100 = REG_GET_FIELD(RREG32_SOC15(THM, 0, mmCG_FDO_CTRL1),
+				CG_FDO_CTRL1, FMAX_DUTY100);
+	if (!duty100)
+		return -EINVAL;
+
+	tmp64 = (uint64_t)speed * duty100;
+	do_div(tmp64, 100);
+	duty = (uint32_t)tmp64;
+
+	WREG32_SOC15(THM, 0, mmCG_FDO_CTRL0,
+		     REG_SET_FIELD(RREG32_SOC15(THM, 0, mmCG_FDO_CTRL0),
+				   CG_FDO_CTRL0, FDO_STATIC_DUTY, duty));
+
+	return smu_v11_0_set_fan_static_mode(smu, FDO_PWM_MODE_STATIC);
+}
+
+static int
+smu_v11_0_set_fan_control_mode(struct smu_context *smu,
+			       uint32_t mode)
+{
+	int ret = 0;
+	bool start = 1;
+	bool stop  = 0;
+
+	switch (mode) {
+	case AMD_FAN_CTRL_NONE:
+		ret = smu_v11_0_set_fan_speed_percent(smu, 100);
+		break;
+	case AMD_FAN_CTRL_MANUAL:
+		ret = smu_v11_0_smc_fan_control(smu, stop);
+		break;
+	case AMD_FAN_CTRL_AUTO:
+		ret = smu_v11_0_smc_fan_control(smu, start);
+		break;
+	default:
+		break;
+	}
+
+	if (ret) {
+		pr_err("[%s]Set fan control mode failed!", __func__);
+		return -EINVAL;
+	}
+
+	return ret;
+}
+
+static int smu_v11_0_set_fan_speed_rpm(struct smu_context *smu,
+				       uint32_t speed)
+{
+	struct amdgpu_device *adev = smu->adev;
+	int ret;
+	uint32_t tach_period, crystal_clock_freq;
+	bool stop = 0;
+
+	if (!speed)
+		return -EINVAL;
+
+	mutex_lock(&(smu->mutex));
+	ret = smu_v11_0_smc_fan_control(smu, stop);
+	if (ret)
+		goto set_fan_speed_rpm_failed;
+
+	crystal_clock_freq = amdgpu_asic_get_xclk(adev);
+	tach_period = 60 * crystal_clock_freq * 10000 / (8 * speed);
+	WREG32_SOC15(THM, 0, mmCG_TACH_CTRL,
+		     REG_SET_FIELD(RREG32_SOC15(THM, 0, mmCG_TACH_CTRL),
+				   CG_TACH_CTRL, TARGET_PERIOD,
+				   tach_period));
+
+	ret = smu_v11_0_set_fan_static_mode(smu, FDO_PWM_MODE_STATIC_RPM);
+
+set_fan_speed_rpm_failed:
+	mutex_unlock(&(smu->mutex));
+	return ret;
+}
+
+static int smu_v11_0_set_xgmi_pstate(struct smu_context *smu,
+				     uint32_t pstate)
+{
+	int ret = 0;
+	mutex_lock(&(smu->mutex));
+	ret = smu_send_smc_msg_with_param(smu,
+					  SMU_MSG_SetXgmiMode,
+					  pstate ? XGMI_STATE_D0 : XGMI_STATE_D3);
+	mutex_unlock(&(smu->mutex));
+	return ret;
+}
+
+static const struct smu_funcs smu_v11_0_funcs = {
+	.init_microcode = smu_v11_0_init_microcode,
+	.load_microcode = smu_v11_0_load_microcode,
+	.check_fw_status = smu_v11_0_check_fw_status,
+	.check_fw_version = smu_v11_0_check_fw_version,
+	.send_smc_msg = smu_v11_0_send_msg,
+	.send_smc_msg_with_param = smu_v11_0_send_msg_with_param,
+	.read_smc_arg = smu_v11_0_read_arg,
+	.read_pptable_from_vbios = smu_v11_0_read_pptable_from_vbios,
+	.init_smc_tables = smu_v11_0_init_smc_tables,
+	.fini_smc_tables = smu_v11_0_fini_smc_tables,
+	.init_power = smu_v11_0_init_power,
+	.fini_power = smu_v11_0_fini_power,
+	.get_vbios_bootup_values = smu_v11_0_get_vbios_bootup_values,
+	.get_clk_info_from_vbios = smu_v11_0_get_clk_info_from_vbios,
+	.notify_memory_pool_location = smu_v11_0_notify_memory_pool_location,
+	.check_pptable = smu_v11_0_check_pptable,
+	.parse_pptable = smu_v11_0_parse_pptable,
+	.populate_smc_pptable = smu_v11_0_populate_smc_pptable,
+	.write_pptable = smu_v11_0_write_pptable,
+	.write_watermarks_table = smu_v11_0_write_watermarks_table,
+	.set_min_dcef_deep_sleep = smu_v11_0_set_min_dcef_deep_sleep,
+	.set_tool_table_location = smu_v11_0_set_tool_table_location,
+	.init_display = smu_v11_0_init_display,
+	.set_allowed_mask = smu_v11_0_set_allowed_mask,
+	.get_enabled_mask = smu_v11_0_get_enabled_mask,
+	.is_dpm_running = smu_v11_0_is_dpm_running,
+	.system_features_control = smu_v11_0_system_features_control,
+	.update_feature_enable_state = smu_v11_0_update_feature_enable_state,
+	.notify_display_change = smu_v11_0_notify_display_change,
+	.get_power_limit = smu_v11_0_get_power_limit,
+	.set_power_limit = smu_v11_0_set_power_limit,
+	.get_current_clk_freq = smu_v11_0_get_current_clk_freq,
+	.init_max_sustainable_clocks = smu_v11_0_init_max_sustainable_clocks,
+	.start_thermal_control = smu_v11_0_start_thermal_control,
+	.read_sensor = smu_v11_0_read_sensor,
+	.set_deep_sleep_dcefclk = smu_v11_0_set_deep_sleep_dcefclk,
+	.display_clock_voltage_request = smu_v11_0_display_clock_voltage_request,
+	.set_watermarks_for_clock_ranges = smu_v11_0_set_watermarks_for_clock_ranges,
+	.get_sclk = smu_v11_0_dpm_get_sclk,
+	.get_mclk = smu_v11_0_dpm_get_mclk,
+	.set_od8_default_settings = smu_v11_0_set_od8_default_settings,
+	.conv_power_profile_to_pplib_workload = smu_v11_0_conv_power_profile_to_pplib_workload,
+	.get_power_profile_mode = smu_v11_0_get_power_profile_mode,
+	.set_power_profile_mode = smu_v11_0_set_power_profile_mode,
+	.update_od8_settings = smu_v11_0_update_od8_settings,
+	.dpm_set_uvd_enable = smu_v11_0_dpm_set_uvd_enable,
+	.dpm_set_vce_enable = smu_v11_0_dpm_set_vce_enable,
+	.get_current_rpm = smu_v11_0_get_current_rpm,
+	.get_fan_control_mode = smu_v11_0_get_fan_control_mode,
+	.set_fan_control_mode = smu_v11_0_set_fan_control_mode,
+	.get_fan_speed_percent = smu_v11_0_get_fan_speed_percent,
+	.set_fan_speed_percent = smu_v11_0_set_fan_speed_percent,
+	.set_fan_speed_rpm = smu_v11_0_set_fan_speed_rpm,
+	.set_xgmi_pstate = smu_v11_0_set_xgmi_pstate,
+};
+
+void smu_v11_0_set_smu_funcs(struct smu_context *smu)
+{
+	struct amdgpu_device *adev = smu->adev;
+
+	smu->funcs = &smu_v11_0_funcs;
+	switch (adev->asic_type) {
+	case CHIP_VEGA20:
+		vega20_set_ppt_funcs(smu);
+		break;
+	default:
+		pr_warn("Unknown asic for smu11\n");
+	}
+}
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smu10_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/smu10_smumgr.c
index d111dd4e03d7..6d11076a79ba 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/smu10_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/smu10_smumgr.c
@@ -212,6 +212,10 @@ static int smu10_start_smu(struct pp_hwmgr *hwmgr)
 	hwmgr->smu_version = smu10_read_arg_from_smc(hwmgr);
 	adev->pm.fw_version = hwmgr->smu_version >> 8;
 
+	if (adev->rev_id < 0x8 && adev->pdev->device != 0x15d8 &&
+	    adev->pm.fw_version < 0x1e45)
+		adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
+
 	if (smu10_verify_smc_interface(hwmgr))
 		return -EINVAL;
 
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/vega20_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/vega20_smumgr.c
index ba00744c3413..f301a73f6df1 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/vega20_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/vega20_smumgr.c
@@ -367,6 +367,26 @@ static int vega20_set_tools_address(struct pp_hwmgr *hwmgr)
 	return ret;
 }
 
+int vega20_set_pptable_driver_address(struct pp_hwmgr *hwmgr)
+{
+	struct vega20_smumgr *priv =
+			(struct vega20_smumgr *)(hwmgr->smu_backend);
+	int ret = 0;
+
+	PP_ASSERT_WITH_CODE((ret = vega20_send_msg_to_smc_with_parameter(hwmgr,
+			PPSMC_MSG_SetDriverDramAddrHigh,
+			upper_32_bits(priv->smu_tables.entry[TABLE_PPTABLE].mc_addr))) == 0,
+			"[SetPPtabeDriverAddress] Attempt to Set Dram Addr High Failed!",
+			return ret);
+	PP_ASSERT_WITH_CODE((ret = vega20_send_msg_to_smc_with_parameter(hwmgr,
+			PPSMC_MSG_SetDriverDramAddrLow,
+			lower_32_bits(priv->smu_tables.entry[TABLE_PPTABLE].mc_addr))) == 0,
+			"[SetPPtabeDriverAddress] Attempt to Set Dram Addr Low Failed!",
+			return ret);
+
+	return ret;
+}
+
 static int vega20_smu_init(struct pp_hwmgr *hwmgr)
 {
 	struct vega20_smumgr *priv;
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/vega20_smumgr.h b/drivers/gpu/drm/amd/powerplay/smumgr/vega20_smumgr.h
index 77349c3f0162..ec953ab13e87 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/vega20_smumgr.h
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/vega20_smumgr.h
@@ -55,6 +55,7 @@ int vega20_set_activity_monitor_coeff(struct pp_hwmgr *hwmgr,
 		uint8_t *table, uint16_t workload_type);
 int vega20_get_activity_monitor_coeff(struct pp_hwmgr *hwmgr,
 		uint8_t *table, uint16_t workload_type);
+int vega20_set_pptable_driver_address(struct pp_hwmgr *hwmgr);
 
 #endif
 
diff --git a/drivers/gpu/drm/amd/powerplay/vega20_ppt.c b/drivers/gpu/drm/amd/powerplay/vega20_ppt.c
new file mode 100644
index 000000000000..8fafcbdb1dfd
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/vega20_ppt.c
@@ -0,0 +1,2413 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "pp_debug.h"
+#include <linux/firmware.h>
+#include "amdgpu.h"
+#include "amdgpu_smu.h"
+#include "atomfirmware.h"
+#include "amdgpu_atomfirmware.h"
+#include "smu_v11_0.h"
+#include "smu11_driver_if.h"
+#include "soc15_common.h"
+#include "atom.h"
+#include "power_state.h"
+#include "vega20_ppt.h"
+#include "vega20_pptable.h"
+#include "vega20_ppsmc.h"
+#include "nbio/nbio_7_4_sh_mask.h"
+
+#define smnPCIE_LC_SPEED_CNTL			0x11140290
+#define smnPCIE_LC_LINK_WIDTH_CNTL		0x11140288
+
+#define MSG_MAP(msg) \
+	[SMU_MSG_##msg] = PPSMC_MSG_##msg
+
+static int vega20_message_map[SMU_MSG_MAX_COUNT] = {
+	MSG_MAP(TestMessage),
+	MSG_MAP(GetSmuVersion),
+	MSG_MAP(GetDriverIfVersion),
+	MSG_MAP(SetAllowedFeaturesMaskLow),
+	MSG_MAP(SetAllowedFeaturesMaskHigh),
+	MSG_MAP(EnableAllSmuFeatures),
+	MSG_MAP(DisableAllSmuFeatures),
+	MSG_MAP(EnableSmuFeaturesLow),
+	MSG_MAP(EnableSmuFeaturesHigh),
+	MSG_MAP(DisableSmuFeaturesLow),
+	MSG_MAP(DisableSmuFeaturesHigh),
+	MSG_MAP(GetEnabledSmuFeaturesLow),
+	MSG_MAP(GetEnabledSmuFeaturesHigh),
+	MSG_MAP(SetWorkloadMask),
+	MSG_MAP(SetPptLimit),
+	MSG_MAP(SetDriverDramAddrHigh),
+	MSG_MAP(SetDriverDramAddrLow),
+	MSG_MAP(SetToolsDramAddrHigh),
+	MSG_MAP(SetToolsDramAddrLow),
+	MSG_MAP(TransferTableSmu2Dram),
+	MSG_MAP(TransferTableDram2Smu),
+	MSG_MAP(UseDefaultPPTable),
+	MSG_MAP(UseBackupPPTable),
+	MSG_MAP(RunBtc),
+	MSG_MAP(RequestI2CBus),
+	MSG_MAP(ReleaseI2CBus),
+	MSG_MAP(SetFloorSocVoltage),
+	MSG_MAP(SoftReset),
+	MSG_MAP(StartBacoMonitor),
+	MSG_MAP(CancelBacoMonitor),
+	MSG_MAP(EnterBaco),
+	MSG_MAP(SetSoftMinByFreq),
+	MSG_MAP(SetSoftMaxByFreq),
+	MSG_MAP(SetHardMinByFreq),
+	MSG_MAP(SetHardMaxByFreq),
+	MSG_MAP(GetMinDpmFreq),
+	MSG_MAP(GetMaxDpmFreq),
+	MSG_MAP(GetDpmFreqByIndex),
+	MSG_MAP(GetDpmClockFreq),
+	MSG_MAP(GetSsVoltageByDpm),
+	MSG_MAP(SetMemoryChannelConfig),
+	MSG_MAP(SetGeminiMode),
+	MSG_MAP(SetGeminiApertureHigh),
+	MSG_MAP(SetGeminiApertureLow),
+	MSG_MAP(SetMinLinkDpmByIndex),
+	MSG_MAP(OverridePcieParameters),
+	MSG_MAP(OverDriveSetPercentage),
+	MSG_MAP(SetMinDeepSleepDcefclk),
+	MSG_MAP(ReenableAcDcInterrupt),
+	MSG_MAP(NotifyPowerSource),
+	MSG_MAP(SetUclkFastSwitch),
+	MSG_MAP(SetUclkDownHyst),
+	MSG_MAP(GetCurrentRpm),
+	MSG_MAP(SetVideoFps),
+	MSG_MAP(SetTjMax),
+	MSG_MAP(SetFanTemperatureTarget),
+	MSG_MAP(PrepareMp1ForUnload),
+	MSG_MAP(DramLogSetDramAddrHigh),
+	MSG_MAP(DramLogSetDramAddrLow),
+	MSG_MAP(DramLogSetDramSize),
+	MSG_MAP(SetFanMaxRpm),
+	MSG_MAP(SetFanMinPwm),
+	MSG_MAP(ConfigureGfxDidt),
+	MSG_MAP(NumOfDisplays),
+	MSG_MAP(RemoveMargins),
+	MSG_MAP(ReadSerialNumTop32),
+	MSG_MAP(ReadSerialNumBottom32),
+	MSG_MAP(SetSystemVirtualDramAddrHigh),
+	MSG_MAP(SetSystemVirtualDramAddrLow),
+	MSG_MAP(WaflTest),
+	MSG_MAP(SetFclkGfxClkRatio),
+	MSG_MAP(AllowGfxOff),
+	MSG_MAP(DisallowGfxOff),
+	MSG_MAP(GetPptLimit),
+	MSG_MAP(GetDcModeMaxDpmFreq),
+	MSG_MAP(GetDebugData),
+	MSG_MAP(SetXgmiMode),
+	MSG_MAP(RunAfllBtc),
+	MSG_MAP(ExitBaco),
+	MSG_MAP(PrepareMp1ForReset),
+	MSG_MAP(PrepareMp1ForShutdown),
+	MSG_MAP(SetMGpuFanBoostLimitRpm),
+	MSG_MAP(GetAVFSVoltageByDpm),
+};
+
+static int vega20_get_smu_msg_index(struct smu_context *smc, uint32_t index)
+{
+	int val;
+
+	if (index >= SMU_MSG_MAX_COUNT)
+		return -EINVAL;
+
+	val = vega20_message_map[index];
+	if (val > PPSMC_Message_Count)
+		return -EINVAL;
+
+	return val;
+}
+
+static int vega20_allocate_dpm_context(struct smu_context *smu)
+{
+	struct smu_dpm_context *smu_dpm = &smu->smu_dpm;
+
+	if (smu_dpm->dpm_context)
+		return -EINVAL;
+
+	smu_dpm->dpm_context = kzalloc(sizeof(struct vega20_dpm_table),
+				       GFP_KERNEL);
+	if (!smu_dpm->dpm_context)
+		return -ENOMEM;
+
+	if (smu_dpm->golden_dpm_context)
+		return -EINVAL;
+
+	smu_dpm->golden_dpm_context = kzalloc(sizeof(struct vega20_dpm_table),
+					      GFP_KERNEL);
+	if (!smu_dpm->golden_dpm_context)
+		return -ENOMEM;
+
+	smu_dpm->dpm_context_size = sizeof(struct vega20_dpm_table);
+
+	smu_dpm->dpm_current_power_state = kzalloc(sizeof(struct smu_power_state),
+				       GFP_KERNEL);
+	if (!smu_dpm->dpm_current_power_state)
+		return -ENOMEM;
+
+	smu_dpm->dpm_request_power_state = kzalloc(sizeof(struct smu_power_state),
+				       GFP_KERNEL);
+	if (!smu_dpm->dpm_request_power_state)
+		return -ENOMEM;
+
+	return 0;
+}
+
+static int vega20_setup_od8_information(struct smu_context *smu)
+{
+	ATOM_Vega20_POWERPLAYTABLE *powerplay_table = NULL;
+	struct smu_table_context *table_context = &smu->smu_table;
+
+	uint32_t od_feature_count, od_feature_array_size,
+		 od_setting_count, od_setting_array_size;
+
+	if (!table_context->power_play_table)
+		return -EINVAL;
+
+	powerplay_table = table_context->power_play_table;
+
+	if (powerplay_table->OverDrive8Table.ucODTableRevision == 1) {
+		/* Setup correct ODFeatureCount, and store ODFeatureArray from
+		 * powerplay table to od_feature_capabilities */
+		od_feature_count =
+			(le32_to_cpu(powerplay_table->OverDrive8Table.ODFeatureCount) >
+			 ATOM_VEGA20_ODFEATURE_COUNT) ?
+			ATOM_VEGA20_ODFEATURE_COUNT :
+			le32_to_cpu(powerplay_table->OverDrive8Table.ODFeatureCount);
+
+		od_feature_array_size = sizeof(uint8_t) * od_feature_count;
+
+		if (table_context->od_feature_capabilities)
+			return -EINVAL;
+
+		table_context->od_feature_capabilities = kmemdup(&powerplay_table->OverDrive8Table.ODFeatureCapabilities,
+								 od_feature_array_size,
+								 GFP_KERNEL);
+		if (!table_context->od_feature_capabilities)
+			return -ENOMEM;
+
+		/* Setup correct ODSettingCount, and store ODSettingArray from
+		 * powerplay table to od_settings_max and od_setting_min */
+		od_setting_count =
+			(le32_to_cpu(powerplay_table->OverDrive8Table.ODSettingCount) >
+			 ATOM_VEGA20_ODSETTING_COUNT) ?
+			ATOM_VEGA20_ODSETTING_COUNT :
+			le32_to_cpu(powerplay_table->OverDrive8Table.ODSettingCount);
+
+		od_setting_array_size = sizeof(uint32_t) * od_setting_count;
+
+		if (table_context->od_settings_max)
+			return -EINVAL;
+
+		table_context->od_settings_max = kmemdup(&powerplay_table->OverDrive8Table.ODSettingsMax,
+							 od_setting_array_size,
+							 GFP_KERNEL);
+
+		if (!table_context->od_settings_max) {
+			kfree(table_context->od_feature_capabilities);
+			table_context->od_feature_capabilities = NULL;
+			return -ENOMEM;
+		}
+
+		if (table_context->od_settings_min)
+			return -EINVAL;
+
+		table_context->od_settings_min = kmemdup(&powerplay_table->OverDrive8Table.ODSettingsMin,
+							 od_setting_array_size,
+							 GFP_KERNEL);
+
+		if (!table_context->od_settings_min) {
+			kfree(table_context->od_feature_capabilities);
+			table_context->od_feature_capabilities = NULL;
+			kfree(table_context->od_settings_max);
+			table_context->od_settings_max = NULL;
+			return -ENOMEM;
+		}
+	}
+
+	return 0;
+}
+
+static int vega20_store_powerplay_table(struct smu_context *smu)
+{
+	ATOM_Vega20_POWERPLAYTABLE *powerplay_table = NULL;
+	struct smu_table_context *table_context = &smu->smu_table;
+	int ret;
+
+	if (!table_context->power_play_table)
+		return -EINVAL;
+
+	powerplay_table = table_context->power_play_table;
+
+	memcpy(table_context->driver_pptable, &powerplay_table->smcPPTable,
+	       sizeof(PPTable_t));
+
+	table_context->software_shutdown_temp = powerplay_table->usSoftwareShutdownTemp;
+	table_context->thermal_controller_type = powerplay_table->ucThermalControllerType;
+	table_context->TDPODLimit = le32_to_cpu(powerplay_table->OverDrive8Table.ODSettingsMax[ATOM_VEGA20_ODSETTING_POWERPERCENTAGE]);
+
+	ret = vega20_setup_od8_information(smu);
+
+	return ret;
+}
+
+static int vega20_append_powerplay_table(struct smu_context *smu)
+{
+	struct smu_table_context *table_context = &smu->smu_table;
+	PPTable_t *smc_pptable = table_context->driver_pptable;
+	struct atom_smc_dpm_info_v4_4 *smc_dpm_table;
+	int index, i, ret;
+
+	index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1,
+					   smc_dpm_info);
+
+	ret = smu_get_atom_data_table(smu, index, NULL, NULL, NULL,
+				      (uint8_t **)&smc_dpm_table);
+	if (ret)
+		return ret;
+
+	smc_pptable->MaxVoltageStepGfx = smc_dpm_table->maxvoltagestepgfx;
+	smc_pptable->MaxVoltageStepSoc = smc_dpm_table->maxvoltagestepsoc;
+
+	smc_pptable->VddGfxVrMapping = smc_dpm_table->vddgfxvrmapping;
+	smc_pptable->VddSocVrMapping = smc_dpm_table->vddsocvrmapping;
+	smc_pptable->VddMem0VrMapping = smc_dpm_table->vddmem0vrmapping;
+	smc_pptable->VddMem1VrMapping = smc_dpm_table->vddmem1vrmapping;
+
+	smc_pptable->GfxUlvPhaseSheddingMask = smc_dpm_table->gfxulvphasesheddingmask;
+	smc_pptable->SocUlvPhaseSheddingMask = smc_dpm_table->soculvphasesheddingmask;
+	smc_pptable->ExternalSensorPresent = smc_dpm_table->externalsensorpresent;
+
+	smc_pptable->GfxMaxCurrent = smc_dpm_table->gfxmaxcurrent;
+	smc_pptable->GfxOffset = smc_dpm_table->gfxoffset;
+	smc_pptable->Padding_TelemetryGfx = smc_dpm_table->padding_telemetrygfx;
+
+	smc_pptable->SocMaxCurrent = smc_dpm_table->socmaxcurrent;
+	smc_pptable->SocOffset = smc_dpm_table->socoffset;
+	smc_pptable->Padding_TelemetrySoc = smc_dpm_table->padding_telemetrysoc;
+
+	smc_pptable->Mem0MaxCurrent = smc_dpm_table->mem0maxcurrent;
+	smc_pptable->Mem0Offset = smc_dpm_table->mem0offset;
+	smc_pptable->Padding_TelemetryMem0 = smc_dpm_table->padding_telemetrymem0;
+
+	smc_pptable->Mem1MaxCurrent = smc_dpm_table->mem1maxcurrent;
+	smc_pptable->Mem1Offset = smc_dpm_table->mem1offset;
+	smc_pptable->Padding_TelemetryMem1 = smc_dpm_table->padding_telemetrymem1;
+
+	smc_pptable->AcDcGpio = smc_dpm_table->acdcgpio;
+	smc_pptable->AcDcPolarity = smc_dpm_table->acdcpolarity;
+	smc_pptable->VR0HotGpio = smc_dpm_table->vr0hotgpio;
+	smc_pptable->VR0HotPolarity = smc_dpm_table->vr0hotpolarity;
+
+	smc_pptable->VR1HotGpio = smc_dpm_table->vr1hotgpio;
+	smc_pptable->VR1HotPolarity = smc_dpm_table->vr1hotpolarity;
+	smc_pptable->Padding1 = smc_dpm_table->padding1;
+	smc_pptable->Padding2 = smc_dpm_table->padding2;
+
+	smc_pptable->LedPin0 = smc_dpm_table->ledpin0;
+	smc_pptable->LedPin1 = smc_dpm_table->ledpin1;
+	smc_pptable->LedPin2 = smc_dpm_table->ledpin2;
+
+	smc_pptable->PllGfxclkSpreadEnabled = smc_dpm_table->pllgfxclkspreadenabled;
+	smc_pptable->PllGfxclkSpreadPercent = smc_dpm_table->pllgfxclkspreadpercent;
+	smc_pptable->PllGfxclkSpreadFreq = smc_dpm_table->pllgfxclkspreadfreq;
+
+	smc_pptable->UclkSpreadEnabled = 0;
+	smc_pptable->UclkSpreadPercent = smc_dpm_table->uclkspreadpercent;
+	smc_pptable->UclkSpreadFreq = smc_dpm_table->uclkspreadfreq;
+
+	smc_pptable->FclkSpreadEnabled = smc_dpm_table->fclkspreadenabled;
+	smc_pptable->FclkSpreadPercent = smc_dpm_table->fclkspreadpercent;
+	smc_pptable->FclkSpreadFreq = smc_dpm_table->fclkspreadfreq;
+
+	smc_pptable->FllGfxclkSpreadEnabled = smc_dpm_table->fllgfxclkspreadenabled;
+	smc_pptable->FllGfxclkSpreadPercent = smc_dpm_table->fllgfxclkspreadpercent;
+	smc_pptable->FllGfxclkSpreadFreq = smc_dpm_table->fllgfxclkspreadfreq;
+
+	for (i = 0; i < I2C_CONTROLLER_NAME_COUNT; i++) {
+		smc_pptable->I2cControllers[i].Enabled =
+			smc_dpm_table->i2ccontrollers[i].enabled;
+		smc_pptable->I2cControllers[i].SlaveAddress =
+			smc_dpm_table->i2ccontrollers[i].slaveaddress;
+		smc_pptable->I2cControllers[i].ControllerPort =
+			smc_dpm_table->i2ccontrollers[i].controllerport;
+		smc_pptable->I2cControllers[i].ThermalThrottler =
+			smc_dpm_table->i2ccontrollers[i].thermalthrottler;
+		smc_pptable->I2cControllers[i].I2cProtocol =
+			smc_dpm_table->i2ccontrollers[i].i2cprotocol;
+		smc_pptable->I2cControllers[i].I2cSpeed =
+			smc_dpm_table->i2ccontrollers[i].i2cspeed;
+	}
+
+	return 0;
+}
+
+static int vega20_check_powerplay_table(struct smu_context *smu)
+{
+	ATOM_Vega20_POWERPLAYTABLE *powerplay_table = NULL;
+	struct smu_table_context *table_context = &smu->smu_table;
+
+	powerplay_table = table_context->power_play_table;
+
+	if (powerplay_table->sHeader.format_revision < ATOM_VEGA20_TABLE_REVISION_VEGA20) {
+		pr_err("Unsupported PPTable format!");
+		return -EINVAL;
+	}
+
+	if (!powerplay_table->sHeader.structuresize) {
+		pr_err("Invalid PowerPlay Table!");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int vega20_run_btc_afll(struct smu_context *smu)
+{
+	return smu_send_smc_msg(smu, SMU_MSG_RunAfllBtc);
+}
+
+static int
+vega20_get_unallowed_feature_mask(struct smu_context *smu,
+				  uint32_t *feature_mask, uint32_t num)
+{
+	if (num > 2)
+		return -EINVAL;
+
+	feature_mask[0] = 0xE0041C00;
+	feature_mask[1] = 0xFFFFFFFE; /* bit32~bit63 is Unsupported */
+
+	return 0;
+}
+
+static enum
+amd_pm_state_type vega20_get_current_power_state(struct smu_context *smu)
+{
+	enum amd_pm_state_type pm_type;
+	struct smu_dpm_context *smu_dpm_ctx = &(smu->smu_dpm);
+
+	if (!smu_dpm_ctx->dpm_context ||
+	    !smu_dpm_ctx->dpm_current_power_state)
+		return -EINVAL;
+
+	mutex_lock(&(smu->mutex));
+	switch (smu_dpm_ctx->dpm_current_power_state->classification.ui_label) {
+	case SMU_STATE_UI_LABEL_BATTERY:
+		pm_type = POWER_STATE_TYPE_BATTERY;
+		break;
+	case SMU_STATE_UI_LABEL_BALLANCED:
+		pm_type = POWER_STATE_TYPE_BALANCED;
+		break;
+	case SMU_STATE_UI_LABEL_PERFORMANCE:
+		pm_type = POWER_STATE_TYPE_PERFORMANCE;
+		break;
+	default:
+		if (smu_dpm_ctx->dpm_current_power_state->classification.flags & SMU_STATE_CLASSIFICATION_FLAG_BOOT)
+			pm_type = POWER_STATE_TYPE_INTERNAL_BOOT;
+		else
+			pm_type = POWER_STATE_TYPE_DEFAULT;
+		break;
+	}
+	mutex_unlock(&(smu->mutex));
+
+	return pm_type;
+}
+
+static int
+vega20_set_single_dpm_table(struct smu_context *smu,
+			    struct vega20_single_dpm_table *single_dpm_table,
+			    PPCLK_e clk_id)
+{
+	int ret = 0;
+	uint32_t i, num_of_levels = 0, clk;
+
+	ret = smu_send_smc_msg_with_param(smu,
+			SMU_MSG_GetDpmFreqByIndex,
+			(clk_id << 16 | 0xFF));
+	if (ret) {
+		pr_err("[GetNumOfDpmLevel] failed to get dpm levels!");
+		return ret;
+	}
+
+	smu_read_smc_arg(smu, &num_of_levels);
+	if (!num_of_levels) {
+		pr_err("[GetNumOfDpmLevel] number of clk levels is invalid!");
+		return -EINVAL;
+	}
+
+	single_dpm_table->count = num_of_levels;
+
+	for (i = 0; i < num_of_levels; i++) {
+		ret = smu_send_smc_msg_with_param(smu,
+				SMU_MSG_GetDpmFreqByIndex,
+				(clk_id << 16 | i));
+		if (ret) {
+			pr_err("[GetDpmFreqByIndex] failed to get dpm freq by index!");
+			return ret;
+		}
+		smu_read_smc_arg(smu, &clk);
+		if (!clk) {
+			pr_err("[GetDpmFreqByIndex] clk value is invalid!");
+			return -EINVAL;
+		}
+		single_dpm_table->dpm_levels[i].value = clk;
+		single_dpm_table->dpm_levels[i].enabled = true;
+	}
+	return 0;
+}
+
+static void vega20_init_single_dpm_state(struct vega20_dpm_state *dpm_state)
+{
+	dpm_state->soft_min_level = 0x0;
+	dpm_state->soft_max_level = 0xffff;
+        dpm_state->hard_min_level = 0x0;
+        dpm_state->hard_max_level = 0xffff;
+}
+
+static int vega20_set_default_dpm_table(struct smu_context *smu)
+{
+	int ret;
+
+	struct smu_dpm_context *smu_dpm = &smu->smu_dpm;
+	struct vega20_dpm_table *dpm_table = NULL;
+	struct vega20_single_dpm_table *single_dpm_table;
+
+	dpm_table = smu_dpm->dpm_context;
+
+	/* socclk */
+	single_dpm_table = &(dpm_table->soc_table);
+
+	if (smu_feature_is_enabled(smu, FEATURE_DPM_SOCCLK_BIT)) {
+		ret = vega20_set_single_dpm_table(smu, single_dpm_table,
+						  PPCLK_SOCCLK);
+		if (ret) {
+			pr_err("[SetupDefaultDpmTable] failed to get socclk dpm levels!");
+			return ret;
+		}
+	} else {
+		single_dpm_table->count = 1;
+		single_dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.socclk / 100;
+	}
+	vega20_init_single_dpm_state(&(single_dpm_table->dpm_state));
+
+	/* gfxclk */
+	single_dpm_table = &(dpm_table->gfx_table);
+
+	if (smu_feature_is_enabled(smu, FEATURE_DPM_GFXCLK_BIT)) {
+		ret = vega20_set_single_dpm_table(smu, single_dpm_table,
+						  PPCLK_GFXCLK);
+		if (ret) {
+			pr_err("[SetupDefaultDpmTable] failed to get gfxclk dpm levels!");
+			return ret;
+		}
+	} else {
+		single_dpm_table->count = 1;
+		single_dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.gfxclk / 100;
+	}
+	vega20_init_single_dpm_state(&(single_dpm_table->dpm_state));
+
+	/* memclk */
+	single_dpm_table = &(dpm_table->mem_table);
+
+	if (smu_feature_is_enabled(smu, FEATURE_DPM_UCLK_BIT)) {
+		ret = vega20_set_single_dpm_table(smu, single_dpm_table,
+						  PPCLK_UCLK);
+		if (ret) {
+			pr_err("[SetupDefaultDpmTable] failed to get memclk dpm levels!");
+			return ret;
+		}
+	} else {
+		single_dpm_table->count = 1;
+		single_dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.uclk / 100;
+	}
+	vega20_init_single_dpm_state(&(single_dpm_table->dpm_state));
+
+	/* eclk */
+	single_dpm_table = &(dpm_table->eclk_table);
+
+	if (smu_feature_is_enabled(smu, FEATURE_DPM_VCE_BIT)) {
+		ret = vega20_set_single_dpm_table(smu, single_dpm_table, PPCLK_ECLK);
+		if (ret) {
+			pr_err("[SetupDefaultDpmTable] failed to get eclk dpm levels!");
+			return ret;
+		}
+	} else {
+		single_dpm_table->count = 1;
+		single_dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.eclk / 100;
+	}
+	vega20_init_single_dpm_state(&(single_dpm_table->dpm_state));
+
+	/* vclk */
+	single_dpm_table = &(dpm_table->vclk_table);
+
+	if (smu_feature_is_enabled(smu, FEATURE_DPM_UVD_BIT)) {
+		ret = vega20_set_single_dpm_table(smu, single_dpm_table, PPCLK_VCLK);
+		if (ret) {
+			pr_err("[SetupDefaultDpmTable] failed to get vclk dpm levels!");
+			return ret;
+		}
+	} else {
+		single_dpm_table->count = 1;
+		single_dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.vclk / 100;
+	}
+	vega20_init_single_dpm_state(&(single_dpm_table->dpm_state));
+
+	/* dclk */
+	single_dpm_table = &(dpm_table->dclk_table);
+
+	if (smu_feature_is_enabled(smu, FEATURE_DPM_UVD_BIT)) {
+		ret = vega20_set_single_dpm_table(smu, single_dpm_table, PPCLK_DCLK);
+		if (ret) {
+			pr_err("[SetupDefaultDpmTable] failed to get dclk dpm levels!");
+			return ret;
+		}
+	} else {
+		single_dpm_table->count = 1;
+		single_dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.dclk / 100;
+	}
+	vega20_init_single_dpm_state(&(single_dpm_table->dpm_state));
+
+	/* dcefclk */
+	single_dpm_table = &(dpm_table->dcef_table);
+
+	if (smu_feature_is_enabled(smu, FEATURE_DPM_DCEFCLK_BIT)) {
+		ret = vega20_set_single_dpm_table(smu, single_dpm_table,
+						  PPCLK_DCEFCLK);
+		if (ret) {
+			pr_err("[SetupDefaultDpmTable] failed to get dcefclk dpm levels!");
+			return ret;
+		}
+	} else {
+		single_dpm_table->count = 1;
+		single_dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.dcefclk / 100;
+	}
+	vega20_init_single_dpm_state(&(single_dpm_table->dpm_state));
+
+	/* pixclk */
+	single_dpm_table = &(dpm_table->pixel_table);
+
+	if (smu_feature_is_enabled(smu, FEATURE_DPM_DCEFCLK_BIT)) {
+		ret = vega20_set_single_dpm_table(smu, single_dpm_table,
+						  PPCLK_PIXCLK);
+		if (ret) {
+			pr_err("[SetupDefaultDpmTable] failed to get pixclk dpm levels!");
+			return ret;
+		}
+	} else {
+		single_dpm_table->count = 0;
+	}
+	vega20_init_single_dpm_state(&(single_dpm_table->dpm_state));
+
+	/* dispclk */
+	single_dpm_table = &(dpm_table->display_table);
+
+	if (smu_feature_is_enabled(smu, FEATURE_DPM_DCEFCLK_BIT)) {
+		ret = vega20_set_single_dpm_table(smu, single_dpm_table,
+						  PPCLK_DISPCLK);
+		if (ret) {
+			pr_err("[SetupDefaultDpmTable] failed to get dispclk dpm levels!");
+			return ret;
+		}
+	} else {
+		single_dpm_table->count = 0;
+	}
+	vega20_init_single_dpm_state(&(single_dpm_table->dpm_state));
+
+	/* phyclk */
+	single_dpm_table = &(dpm_table->phy_table);
+
+	if (smu_feature_is_enabled(smu, FEATURE_DPM_DCEFCLK_BIT)) {
+		ret = vega20_set_single_dpm_table(smu, single_dpm_table,
+						  PPCLK_PHYCLK);
+		if (ret) {
+			pr_err("[SetupDefaultDpmTable] failed to get phyclk dpm levels!");
+			return ret;
+		}
+	} else {
+		single_dpm_table->count = 0;
+	}
+	vega20_init_single_dpm_state(&(single_dpm_table->dpm_state));
+
+	/* fclk */
+	single_dpm_table = &(dpm_table->fclk_table);
+
+	if (smu_feature_is_enabled(smu,FEATURE_DPM_FCLK_BIT)) {
+		ret = vega20_set_single_dpm_table(smu, single_dpm_table,
+						  PPCLK_FCLK);
+		if (ret) {
+			pr_err("[SetupDefaultDpmTable] failed to get fclk dpm levels!");
+			return ret;
+		}
+	} else {
+		single_dpm_table->count = 0;
+	}
+	vega20_init_single_dpm_state(&(single_dpm_table->dpm_state));
+
+	memcpy(smu_dpm->golden_dpm_context, dpm_table,
+	       sizeof(struct vega20_dpm_table));
+
+	return 0;
+}
+
+static int vega20_populate_umd_state_clk(struct smu_context *smu)
+{
+	struct smu_dpm_context *smu_dpm = &smu->smu_dpm;
+	struct vega20_dpm_table *dpm_table = NULL;
+	struct vega20_single_dpm_table *gfx_table = NULL;
+	struct vega20_single_dpm_table *mem_table = NULL;
+
+	dpm_table = smu_dpm->dpm_context;
+	gfx_table = &(dpm_table->gfx_table);
+	mem_table = &(dpm_table->mem_table);
+
+	smu->pstate_sclk = gfx_table->dpm_levels[0].value;
+	smu->pstate_mclk = mem_table->dpm_levels[0].value;
+
+	if (gfx_table->count > VEGA20_UMD_PSTATE_GFXCLK_LEVEL &&
+	    mem_table->count > VEGA20_UMD_PSTATE_MCLK_LEVEL) {
+		smu->pstate_sclk = gfx_table->dpm_levels[VEGA20_UMD_PSTATE_GFXCLK_LEVEL].value;
+		smu->pstate_mclk = mem_table->dpm_levels[VEGA20_UMD_PSTATE_MCLK_LEVEL].value;
+	}
+
+	smu->pstate_sclk = smu->pstate_sclk * 100;
+	smu->pstate_mclk = smu->pstate_mclk * 100;
+
+	return 0;
+}
+
+static int vega20_get_clk_table(struct smu_context *smu,
+			struct pp_clock_levels_with_latency *clocks,
+			struct vega20_single_dpm_table *dpm_table)
+{
+	int i, count;
+
+	count = (dpm_table->count > MAX_NUM_CLOCKS) ? MAX_NUM_CLOCKS : dpm_table->count;
+	clocks->num_levels = count;
+
+	for (i = 0; i < count; i++) {
+		clocks->data[i].clocks_in_khz =
+			dpm_table->dpm_levels[i].value * 1000;
+		clocks->data[i].latency_in_us = 0;
+	}
+
+	return 0;
+}
+
+static int vega20_print_clk_levels(struct smu_context *smu,
+			enum pp_clock_type type, char *buf)
+{
+	int i, now, size = 0;
+	int ret = 0;
+	uint32_t gen_speed, lane_width;
+	struct amdgpu_device *adev = smu->adev;
+	struct pp_clock_levels_with_latency clocks;
+	struct vega20_single_dpm_table *single_dpm_table;
+	struct smu_table_context *table_context = &smu->smu_table;
+	struct smu_dpm_context *smu_dpm = &smu->smu_dpm;
+	struct vega20_dpm_table *dpm_table = NULL;
+	struct vega20_od8_settings *od8_settings =
+		(struct vega20_od8_settings *)table_context->od8_settings;
+	OverDriveTable_t *od_table =
+		(OverDriveTable_t *)(table_context->overdrive_table);
+	PPTable_t *pptable = (PPTable_t *)table_context->driver_pptable;
+
+	dpm_table = smu_dpm->dpm_context;
+
+	switch (type) {
+	case PP_SCLK:
+		ret = smu_get_current_clk_freq(smu, PPCLK_GFXCLK, &now);
+		if (ret) {
+			pr_err("Attempt to get current gfx clk Failed!");
+			return ret;
+		}
+
+		single_dpm_table = &(dpm_table->gfx_table);
+		ret = vega20_get_clk_table(smu, &clocks, single_dpm_table);
+		if (ret) {
+			pr_err("Attempt to get gfx clk levels Failed!");
+			return ret;
+		}
+
+		for (i = 0; i < clocks.num_levels; i++)
+			size += sprintf(buf + size, "%d: %uMhz %s\n", i,
+					clocks.data[i].clocks_in_khz / 1000,
+					(clocks.data[i].clocks_in_khz == now * 10)
+					? "*" : "");
+		break;
+
+	case PP_MCLK:
+		ret = smu_get_current_clk_freq(smu, PPCLK_UCLK, &now);
+		if (ret) {
+			pr_err("Attempt to get current mclk Failed!");
+			return ret;
+		}
+
+		single_dpm_table = &(dpm_table->mem_table);
+		ret = vega20_get_clk_table(smu, &clocks, single_dpm_table);
+		if (ret) {
+			pr_err("Attempt to get memory clk levels Failed!");
+			return ret;
+		}
+
+		for (i = 0; i < clocks.num_levels; i++)
+			size += sprintf(buf + size, "%d: %uMhz %s\n",
+				i, clocks.data[i].clocks_in_khz / 1000,
+				(clocks.data[i].clocks_in_khz == now * 10)
+				? "*" : "");
+		break;
+
+	case PP_SOCCLK:
+		ret = smu_get_current_clk_freq(smu, PPCLK_SOCCLK, &now);
+		if (ret) {
+			pr_err("Attempt to get current socclk Failed!");
+			return ret;
+		}
+
+		single_dpm_table = &(dpm_table->soc_table);
+		ret = vega20_get_clk_table(smu, &clocks, single_dpm_table);
+		if (ret) {
+			pr_err("Attempt to get socclk levels Failed!");
+			return ret;
+		}
+
+		for (i = 0; i < clocks.num_levels; i++)
+			size += sprintf(buf + size, "%d: %uMhz %s\n",
+				i, clocks.data[i].clocks_in_khz / 1000,
+				(clocks.data[i].clocks_in_khz == now * 10)
+				? "*" : "");
+		break;
+
+	case PP_FCLK:
+		ret = smu_get_current_clk_freq(smu, PPCLK_FCLK, &now);
+		if (ret) {
+			pr_err("Attempt to get current fclk Failed!");
+			return ret;
+		}
+
+		single_dpm_table = &(dpm_table->fclk_table);
+		for (i = 0; i < single_dpm_table->count; i++)
+			size += sprintf(buf + size, "%d: %uMhz %s\n",
+				i, single_dpm_table->dpm_levels[i].value,
+				(single_dpm_table->dpm_levels[i].value == now / 100)
+				? "*" : "");
+		break;
+
+	case PP_DCEFCLK:
+		ret = smu_get_current_clk_freq(smu, PPCLK_DCEFCLK, &now);
+		if (ret) {
+			pr_err("Attempt to get current dcefclk Failed!");
+			return ret;
+		}
+
+		single_dpm_table = &(dpm_table->dcef_table);
+		ret = vega20_get_clk_table(smu, &clocks, single_dpm_table);
+		if (ret) {
+			pr_err("Attempt to get dcefclk levels Failed!");
+			return ret;
+		}
+
+		for (i = 0; i < clocks.num_levels; i++)
+			size += sprintf(buf + size, "%d: %uMhz %s\n",
+				i, clocks.data[i].clocks_in_khz / 1000,
+				(clocks.data[i].clocks_in_khz == now * 10) ? "*" : "");
+		break;
+
+	case PP_PCIE:
+		gen_speed = (RREG32_PCIE(smnPCIE_LC_SPEED_CNTL) &
+			     PSWUSP0_PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK)
+			>> PSWUSP0_PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT;
+		lane_width = (RREG32_PCIE(smnPCIE_LC_LINK_WIDTH_CNTL) &
+			      PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_RD_MASK)
+			>> PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_RD__SHIFT;
+		for (i = 0; i < NUM_LINK_LEVELS; i++)
+			size += sprintf(buf + size, "%d: %s %s %dMhz %s\n", i,
+					(pptable->PcieGenSpeed[i] == 0) ? "2.5GT/s," :
+					(pptable->PcieGenSpeed[i] == 1) ? "5.0GT/s," :
+					(pptable->PcieGenSpeed[i] == 2) ? "8.0GT/s," :
+					(pptable->PcieGenSpeed[i] == 3) ? "16.0GT/s," : "",
+					(pptable->PcieLaneCount[i] == 1) ? "x1" :
+					(pptable->PcieLaneCount[i] == 2) ? "x2" :
+					(pptable->PcieLaneCount[i] == 3) ? "x4" :
+					(pptable->PcieLaneCount[i] == 4) ? "x8" :
+					(pptable->PcieLaneCount[i] == 5) ? "x12" :
+					(pptable->PcieLaneCount[i] == 6) ? "x16" : "",
+					pptable->LclkFreq[i],
+					(gen_speed == pptable->PcieGenSpeed[i]) &&
+					(lane_width == pptable->PcieLaneCount[i]) ?
+					"*" : "");
+		break;
+
+	case OD_SCLK:
+		if (od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FMIN].feature_id &&
+		    od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FMAX].feature_id) {
+			size = sprintf(buf, "%s:\n", "OD_SCLK");
+			size += sprintf(buf + size, "0: %10uMhz\n",
+					od_table->GfxclkFmin);
+			size += sprintf(buf + size, "1: %10uMhz\n",
+					od_table->GfxclkFmax);
+		}
+
+		break;
+
+	case OD_MCLK:
+		if (od8_settings->od8_settings_array[OD8_SETTING_UCLK_FMAX].feature_id) {
+			size = sprintf(buf, "%s:\n", "OD_MCLK");
+			size += sprintf(buf + size, "1: %10uMhz\n",
+					 od_table->UclkFmax);
+		}
+
+		break;
+
+	case OD_VDDC_CURVE:
+		if (od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FREQ1].feature_id &&
+		    od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FREQ2].feature_id &&
+		    od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FREQ3].feature_id &&
+		    od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_VOLTAGE1].feature_id &&
+		    od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_VOLTAGE2].feature_id &&
+		    od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_VOLTAGE3].feature_id) {
+			size = sprintf(buf, "%s:\n", "OD_VDDC_CURVE");
+			size += sprintf(buf + size, "0: %10uMhz %10dmV\n",
+					od_table->GfxclkFreq1,
+					od_table->GfxclkVolt1 / VOLTAGE_SCALE);
+			size += sprintf(buf + size, "1: %10uMhz %10dmV\n",
+					od_table->GfxclkFreq2,
+					od_table->GfxclkVolt2 / VOLTAGE_SCALE);
+			size += sprintf(buf + size, "2: %10uMhz %10dmV\n",
+					od_table->GfxclkFreq3,
+					od_table->GfxclkVolt3 / VOLTAGE_SCALE);
+		}
+
+		break;
+
+	case OD_RANGE:
+		size = sprintf(buf, "%s:\n", "OD_RANGE");
+
+		if (od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FMIN].feature_id &&
+		    od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FMAX].feature_id) {
+			size += sprintf(buf + size, "SCLK: %7uMhz %10uMhz\n",
+					od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FMIN].min_value,
+					od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FMAX].max_value);
+		}
+
+		if (od8_settings->od8_settings_array[OD8_SETTING_UCLK_FMAX].feature_id) {
+			single_dpm_table = &(dpm_table->mem_table);
+			ret = vega20_get_clk_table(smu, &clocks, single_dpm_table);
+			if (ret) {
+				pr_err("Attempt to get memory clk levels Failed!");
+				return ret;
+			}
+
+			size += sprintf(buf + size, "MCLK: %7uMhz %10uMhz\n",
+					clocks.data[0].clocks_in_khz / 1000,
+					od8_settings->od8_settings_array[OD8_SETTING_UCLK_FMAX].max_value);
+		}
+
+		if (od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FREQ1].feature_id &&
+		    od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FREQ2].feature_id &&
+		    od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FREQ3].feature_id &&
+		    od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_VOLTAGE1].feature_id &&
+		    od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_VOLTAGE2].feature_id &&
+		    od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_VOLTAGE3].feature_id) {
+			size += sprintf(buf + size, "VDDC_CURVE_SCLK[0]: %7uMhz %10uMhz\n",
+					od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FREQ1].min_value,
+					od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FREQ1].max_value);
+			size += sprintf(buf + size, "VDDC_CURVE_VOLT[0]: %7dmV %11dmV\n",
+					od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_VOLTAGE1].min_value,
+					od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_VOLTAGE1].max_value);
+			size += sprintf(buf + size, "VDDC_CURVE_SCLK[1]: %7uMhz %10uMhz\n",
+					od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FREQ2].min_value,
+					od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FREQ2].max_value);
+			size += sprintf(buf + size, "VDDC_CURVE_VOLT[1]: %7dmV %11dmV\n",
+					od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_VOLTAGE2].min_value,
+					od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_VOLTAGE2].max_value);
+			size += sprintf(buf + size, "VDDC_CURVE_SCLK[2]: %7uMhz %10uMhz\n",
+					od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FREQ3].min_value,
+					od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FREQ3].max_value);
+			size += sprintf(buf + size, "VDDC_CURVE_VOLT[2]: %7dmV %11dmV\n",
+					od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_VOLTAGE3].min_value,
+					od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_VOLTAGE3].max_value);
+		}
+
+		break;
+
+	default:
+		break;
+	}
+	return size;
+}
+
+static int vega20_upload_dpm_level(struct smu_context *smu, bool max,
+				   uint32_t feature_mask)
+{
+	struct vega20_dpm_table *dpm_table;
+	struct vega20_single_dpm_table *single_dpm_table;
+	uint32_t freq;
+	int ret = 0;
+
+	dpm_table = smu->smu_dpm.dpm_context;
+
+	if (smu_feature_is_enabled(smu, FEATURE_DPM_GFXCLK_BIT) &&
+	    (feature_mask & FEATURE_DPM_GFXCLK_MASK)) {
+		single_dpm_table = &(dpm_table->gfx_table);
+		freq = max ? single_dpm_table->dpm_state.soft_max_level :
+			single_dpm_table->dpm_state.soft_min_level;
+		ret = smu_send_smc_msg_with_param(smu,
+			(max ? SMU_MSG_SetSoftMaxByFreq : SMU_MSG_SetSoftMinByFreq),
+			(PPCLK_GFXCLK << 16) | (freq & 0xffff));
+		if (ret) {
+			pr_err("Failed to set soft %s gfxclk !\n",
+						max ? "max" : "min");
+			return ret;
+		}
+	}
+
+	if (smu_feature_is_enabled(smu, FEATURE_DPM_UCLK_BIT) &&
+	    (feature_mask & FEATURE_DPM_UCLK_MASK)) {
+		single_dpm_table = &(dpm_table->mem_table);
+		freq = max ? single_dpm_table->dpm_state.soft_max_level :
+			single_dpm_table->dpm_state.soft_min_level;
+		ret = smu_send_smc_msg_with_param(smu,
+			(max ? SMU_MSG_SetSoftMaxByFreq : SMU_MSG_SetSoftMinByFreq),
+			(PPCLK_UCLK << 16) | (freq & 0xffff));
+		if (ret) {
+			pr_err("Failed to set soft %s memclk !\n",
+						max ? "max" : "min");
+			return ret;
+		}
+	}
+
+	if (smu_feature_is_enabled(smu, FEATURE_DPM_SOCCLK_BIT) &&
+	    (feature_mask & FEATURE_DPM_SOCCLK_MASK)) {
+		single_dpm_table = &(dpm_table->soc_table);
+		freq = max ? single_dpm_table->dpm_state.soft_max_level :
+			single_dpm_table->dpm_state.soft_min_level;
+		ret = smu_send_smc_msg_with_param(smu,
+			(max ? SMU_MSG_SetSoftMaxByFreq : SMU_MSG_SetSoftMinByFreq),
+			(PPCLK_SOCCLK << 16) | (freq & 0xffff));
+		if (ret) {
+			pr_err("Failed to set soft %s socclk !\n",
+						max ? "max" : "min");
+			return ret;
+		}
+	}
+
+	if (smu_feature_is_enabled(smu, FEATURE_DPM_FCLK_BIT) &&
+	    (feature_mask & FEATURE_DPM_FCLK_MASK)) {
+		single_dpm_table = &(dpm_table->fclk_table);
+		freq = max ? single_dpm_table->dpm_state.soft_max_level :
+			single_dpm_table->dpm_state.soft_min_level;
+		ret = smu_send_smc_msg_with_param(smu,
+			(max ? SMU_MSG_SetSoftMaxByFreq : SMU_MSG_SetSoftMinByFreq),
+			(PPCLK_FCLK << 16) | (freq & 0xffff));
+		if (ret) {
+			pr_err("Failed to set soft %s fclk !\n",
+						max ? "max" : "min");
+			return ret;
+		}
+	}
+
+	if (smu_feature_is_enabled(smu, FEATURE_DPM_DCEFCLK_BIT) &&
+	    (feature_mask & FEATURE_DPM_DCEFCLK_MASK)) {
+		single_dpm_table = &(dpm_table->dcef_table);
+		freq = single_dpm_table->dpm_state.hard_min_level;
+		if (!max) {
+			ret = smu_send_smc_msg_with_param(smu,
+				SMU_MSG_SetHardMinByFreq,
+				(PPCLK_DCEFCLK << 16) | (freq & 0xffff));
+			if (ret) {
+				pr_err("Failed to set hard min dcefclk !\n");
+				return ret;
+			}
+		}
+	}
+
+	return ret;
+}
+
+static int vega20_force_clk_levels(struct smu_context *smu,
+			enum pp_clock_type type, uint32_t mask)
+{
+	struct vega20_dpm_table *dpm_table;
+	struct vega20_single_dpm_table *single_dpm_table;
+	uint32_t soft_min_level, soft_max_level, hard_min_level;
+	struct smu_dpm_context *smu_dpm = &smu->smu_dpm;
+	int ret = 0;
+
+	if (smu_dpm->dpm_level != AMD_DPM_FORCED_LEVEL_MANUAL) {
+		pr_info("force clock level is for dpm manual mode only.\n");
+		return -EINVAL;
+	}
+
+	mutex_lock(&(smu->mutex));
+
+	soft_min_level = mask ? (ffs(mask) - 1) : 0;
+	soft_max_level = mask ? (fls(mask) - 1) : 0;
+
+	dpm_table = smu->smu_dpm.dpm_context;
+
+	switch (type) {
+	case PP_SCLK:
+		single_dpm_table = &(dpm_table->gfx_table);
+
+		if (soft_max_level >= single_dpm_table->count) {
+			pr_err("Clock level specified %d is over max allowed %d\n",
+					soft_max_level, single_dpm_table->count - 1);
+			ret = -EINVAL;
+			break;
+		}
+
+		single_dpm_table->dpm_state.soft_min_level =
+			single_dpm_table->dpm_levels[soft_min_level].value;
+		single_dpm_table->dpm_state.soft_max_level =
+			single_dpm_table->dpm_levels[soft_max_level].value;
+
+		ret = vega20_upload_dpm_level(smu, false, FEATURE_DPM_GFXCLK_MASK);
+		if (ret) {
+			pr_err("Failed to upload boot level to lowest!\n");
+			break;
+		}
+
+		ret = vega20_upload_dpm_level(smu, true, FEATURE_DPM_GFXCLK_MASK);
+		if (ret)
+			pr_err("Failed to upload dpm max level to highest!\n");
+
+		break;
+
+	case PP_MCLK:
+		single_dpm_table = &(dpm_table->mem_table);
+
+		if (soft_max_level >= single_dpm_table->count) {
+			pr_err("Clock level specified %d is over max allowed %d\n",
+					soft_max_level, single_dpm_table->count - 1);
+			ret = -EINVAL;
+			break;
+		}
+
+		single_dpm_table->dpm_state.soft_min_level =
+			single_dpm_table->dpm_levels[soft_min_level].value;
+		single_dpm_table->dpm_state.soft_max_level =
+			single_dpm_table->dpm_levels[soft_max_level].value;
+
+		ret = vega20_upload_dpm_level(smu, false, FEATURE_DPM_UCLK_MASK);
+		if (ret) {
+			pr_err("Failed to upload boot level to lowest!\n");
+			break;
+		}
+
+		ret = vega20_upload_dpm_level(smu, true, FEATURE_DPM_UCLK_MASK);
+		if (ret)
+			pr_err("Failed to upload dpm max level to highest!\n");
+
+		break;
+
+	case PP_SOCCLK:
+		single_dpm_table = &(dpm_table->soc_table);
+
+		if (soft_max_level >= single_dpm_table->count) {
+			pr_err("Clock level specified %d is over max allowed %d\n",
+					soft_max_level, single_dpm_table->count - 1);
+			ret = -EINVAL;
+			break;
+		}
+
+		single_dpm_table->dpm_state.soft_min_level =
+			single_dpm_table->dpm_levels[soft_min_level].value;
+		single_dpm_table->dpm_state.soft_max_level =
+			single_dpm_table->dpm_levels[soft_max_level].value;
+
+		ret = vega20_upload_dpm_level(smu, false, FEATURE_DPM_SOCCLK_MASK);
+		if (ret) {
+			pr_err("Failed to upload boot level to lowest!\n");
+			break;
+		}
+
+		ret = vega20_upload_dpm_level(smu, true, FEATURE_DPM_SOCCLK_MASK);
+		if (ret)
+			pr_err("Failed to upload dpm max level to highest!\n");
+
+		break;
+
+	case PP_FCLK:
+		single_dpm_table = &(dpm_table->fclk_table);
+
+		if (soft_max_level >= single_dpm_table->count) {
+			pr_err("Clock level specified %d is over max allowed %d\n",
+					soft_max_level, single_dpm_table->count - 1);
+			ret = -EINVAL;
+			break;
+		}
+
+		single_dpm_table->dpm_state.soft_min_level =
+			single_dpm_table->dpm_levels[soft_min_level].value;
+		single_dpm_table->dpm_state.soft_max_level =
+			single_dpm_table->dpm_levels[soft_max_level].value;
+
+		ret = vega20_upload_dpm_level(smu, false, FEATURE_DPM_FCLK_MASK);
+		if (ret) {
+			pr_err("Failed to upload boot level to lowest!\n");
+			break;
+		}
+
+		ret = vega20_upload_dpm_level(smu, true, FEATURE_DPM_FCLK_MASK);
+		if (ret)
+			pr_err("Failed to upload dpm max level to highest!\n");
+
+		break;
+
+	case PP_DCEFCLK:
+		hard_min_level = soft_min_level;
+		single_dpm_table = &(dpm_table->dcef_table);
+
+		if (hard_min_level >= single_dpm_table->count) {
+			pr_err("Clock level specified %d is over max allowed %d\n",
+					hard_min_level, single_dpm_table->count - 1);
+			ret = -EINVAL;
+			break;
+		}
+
+		single_dpm_table->dpm_state.hard_min_level =
+			single_dpm_table->dpm_levels[hard_min_level].value;
+
+		ret = vega20_upload_dpm_level(smu, false, FEATURE_DPM_DCEFCLK_MASK);
+		if (ret)
+			pr_err("Failed to upload boot level to lowest!\n");
+
+		break;
+
+	case PP_PCIE:
+		if (soft_min_level >= NUM_LINK_LEVELS ||
+		    soft_max_level >= NUM_LINK_LEVELS) {
+			ret = -EINVAL;
+			break;
+		}
+
+		ret = smu_send_smc_msg_with_param(smu,
+				SMU_MSG_SetMinLinkDpmByIndex, soft_min_level);
+		if (ret)
+			pr_err("Failed to set min link dpm level!\n");
+
+		break;
+
+	default:
+		break;
+	}
+
+	mutex_unlock(&(smu->mutex));
+	return ret;
+}
+
+static int vega20_get_clock_by_type_with_latency(struct smu_context *smu,
+						 enum amd_pp_clock_type type,
+						 struct pp_clock_levels_with_latency *clocks)
+{
+	int ret;
+	struct vega20_single_dpm_table *single_dpm_table;
+	struct smu_dpm_context *smu_dpm = &smu->smu_dpm;
+	struct vega20_dpm_table *dpm_table = NULL;
+
+	dpm_table = smu_dpm->dpm_context;
+
+	mutex_lock(&smu->mutex);
+
+	switch (type) {
+	case amd_pp_sys_clock:
+		single_dpm_table = &(dpm_table->gfx_table);
+		ret = vega20_get_clk_table(smu, clocks, single_dpm_table);
+		break;
+	case amd_pp_mem_clock:
+		single_dpm_table = &(dpm_table->mem_table);
+		ret = vega20_get_clk_table(smu, clocks, single_dpm_table);
+		break;
+	case amd_pp_dcef_clock:
+		single_dpm_table = &(dpm_table->dcef_table);
+		ret = vega20_get_clk_table(smu, clocks, single_dpm_table);
+		break;
+	case amd_pp_soc_clock:
+		single_dpm_table = &(dpm_table->soc_table);
+		ret = vega20_get_clk_table(smu, clocks, single_dpm_table);
+		break;
+	default:
+		ret = -EINVAL;
+	}
+
+	mutex_unlock(&smu->mutex);
+	return ret;
+}
+
+static int vega20_overdrive_get_gfx_clk_base_voltage(struct smu_context *smu,
+						     uint32_t *voltage,
+						     uint32_t freq)
+{
+	int ret;
+
+	ret = smu_send_smc_msg_with_param(smu,
+			SMU_MSG_GetAVFSVoltageByDpm,
+			((AVFS_CURVE << 24) | (OD8_HOTCURVE_TEMPERATURE << 16) | freq));
+	if (ret) {
+		pr_err("[GetBaseVoltage] failed to get GFXCLK AVFS voltage from SMU!");
+		return ret;
+	}
+
+	smu_read_smc_arg(smu, voltage);
+	*voltage = *voltage / VOLTAGE_SCALE;
+
+	return 0;
+}
+
+static int vega20_set_default_od8_setttings(struct smu_context *smu)
+{
+	struct smu_table_context *table_context = &smu->smu_table;
+	OverDriveTable_t *od_table = (OverDriveTable_t *)(table_context->overdrive_table);
+	struct vega20_od8_settings *od8_settings = NULL;
+	PPTable_t *smc_pptable = table_context->driver_pptable;
+	int i, ret;
+
+	if (table_context->od8_settings)
+		return -EINVAL;
+
+	table_context->od8_settings = kzalloc(sizeof(struct vega20_od8_settings), GFP_KERNEL);
+
+	if (!table_context->od8_settings)
+		return -ENOMEM;
+
+	memset(table_context->od8_settings, 0, sizeof(struct vega20_od8_settings));
+	od8_settings = (struct vega20_od8_settings *)table_context->od8_settings;
+
+	if (smu_feature_is_enabled(smu, FEATURE_DPM_SOCCLK_BIT)) {
+		if (table_context->od_feature_capabilities[ATOM_VEGA20_ODFEATURE_GFXCLK_LIMITS] &&
+		    table_context->od_settings_max[OD8_SETTING_GFXCLK_FMAX] > 0 &&
+		    table_context->od_settings_min[OD8_SETTING_GFXCLK_FMIN] > 0 &&
+		    (table_context->od_settings_max[OD8_SETTING_GFXCLK_FMAX] >=
+		     table_context->od_settings_min[OD8_SETTING_GFXCLK_FMIN])) {
+			od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FMIN].feature_id =
+				OD8_GFXCLK_LIMITS;
+			od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FMAX].feature_id =
+				OD8_GFXCLK_LIMITS;
+			od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FMIN].default_value =
+				od_table->GfxclkFmin;
+			od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FMAX].default_value =
+				od_table->GfxclkFmax;
+		}
+
+		if (table_context->od_feature_capabilities[ATOM_VEGA20_ODFEATURE_GFXCLK_CURVE] &&
+		    (table_context->od_settings_min[OD8_SETTING_GFXCLK_VOLTAGE1] >=
+		     smc_pptable->MinVoltageGfx / VOLTAGE_SCALE) &&
+		    (table_context->od_settings_max[OD8_SETTING_GFXCLK_VOLTAGE3] <=
+		     smc_pptable->MaxVoltageGfx / VOLTAGE_SCALE) &&
+		    (table_context->od_settings_min[OD8_SETTING_GFXCLK_VOLTAGE1] <=
+		     table_context->od_settings_max[OD8_SETTING_GFXCLK_VOLTAGE3])) {
+			od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FREQ1].feature_id =
+				OD8_GFXCLK_CURVE;
+			od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_VOLTAGE1].feature_id =
+				OD8_GFXCLK_CURVE;
+			od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FREQ2].feature_id =
+				OD8_GFXCLK_CURVE;
+			od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_VOLTAGE2].feature_id =
+				OD8_GFXCLK_CURVE;
+			od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FREQ3].feature_id =
+				OD8_GFXCLK_CURVE;
+			od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_VOLTAGE3].feature_id =
+				OD8_GFXCLK_CURVE;
+
+			od_table->GfxclkFreq1 = od_table->GfxclkFmin;
+			od_table->GfxclkFreq2 = (od_table->GfxclkFmin + od_table->GfxclkFmax) / 2;
+			od_table->GfxclkFreq3 = od_table->GfxclkFmax;
+			od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FREQ1].default_value =
+				od_table->GfxclkFreq1;
+			od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FREQ2].default_value =
+				od_table->GfxclkFreq2;
+			od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FREQ3].default_value =
+				od_table->GfxclkFreq3;
+
+			ret = vega20_overdrive_get_gfx_clk_base_voltage(smu,
+				&od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_VOLTAGE1].default_value,
+				od_table->GfxclkFreq1);
+			if (ret)
+				od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_VOLTAGE1].default_value = 0;
+			od_table->GfxclkVolt1 =
+				od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_VOLTAGE1].default_value
+				* VOLTAGE_SCALE;
+			ret = vega20_overdrive_get_gfx_clk_base_voltage(smu,
+				&od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_VOLTAGE2].default_value,
+				od_table->GfxclkFreq2);
+			if (ret)
+				od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_VOLTAGE2].default_value = 0;
+			od_table->GfxclkVolt2 =
+				od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_VOLTAGE2].default_value
+				* VOLTAGE_SCALE;
+			ret = vega20_overdrive_get_gfx_clk_base_voltage(smu,
+				&od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_VOLTAGE3].default_value,
+				od_table->GfxclkFreq3);
+			if (ret)
+				od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_VOLTAGE3].default_value = 0;
+			od_table->GfxclkVolt3 =
+				od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_VOLTAGE3].default_value
+				* VOLTAGE_SCALE;
+		}
+	}
+
+	if (smu_feature_is_enabled(smu, FEATURE_DPM_UCLK_BIT)) {
+		if (table_context->od_feature_capabilities[ATOM_VEGA20_ODFEATURE_UCLK_MAX] &&
+		    table_context->od_settings_min[OD8_SETTING_UCLK_FMAX] > 0 &&
+		    table_context->od_settings_max[OD8_SETTING_UCLK_FMAX] > 0 &&
+		    (table_context->od_settings_max[OD8_SETTING_UCLK_FMAX] >=
+		     table_context->od_settings_min[OD8_SETTING_UCLK_FMAX])) {
+			od8_settings->od8_settings_array[OD8_SETTING_UCLK_FMAX].feature_id =
+				OD8_UCLK_MAX;
+			od8_settings->od8_settings_array[OD8_SETTING_UCLK_FMAX].default_value =
+				od_table->UclkFmax;
+		}
+	}
+
+	if (table_context->od_feature_capabilities[ATOM_VEGA20_ODFEATURE_POWER_LIMIT] &&
+	    table_context->od_settings_min[OD8_SETTING_POWER_PERCENTAGE] > 0 &&
+	    table_context->od_settings_min[OD8_SETTING_POWER_PERCENTAGE] <= 100 &&
+	    table_context->od_settings_max[OD8_SETTING_POWER_PERCENTAGE] > 0 &&
+	    table_context->od_settings_max[OD8_SETTING_POWER_PERCENTAGE] <= 100) {
+		od8_settings->od8_settings_array[OD8_SETTING_POWER_PERCENTAGE].feature_id =
+			OD8_POWER_LIMIT;
+		od8_settings->od8_settings_array[OD8_SETTING_POWER_PERCENTAGE].default_value =
+			od_table->OverDrivePct;
+	}
+
+	if (smu_feature_is_enabled(smu, FEATURE_FAN_CONTROL_BIT)) {
+		if (table_context->od_feature_capabilities[ATOM_VEGA20_ODFEATURE_FAN_ACOUSTIC_LIMIT] &&
+		    table_context->od_settings_min[OD8_SETTING_FAN_ACOUSTIC_LIMIT] > 0 &&
+		    table_context->od_settings_max[OD8_SETTING_FAN_ACOUSTIC_LIMIT] > 0 &&
+		    (table_context->od_settings_max[OD8_SETTING_FAN_ACOUSTIC_LIMIT] >=
+		     table_context->od_settings_min[OD8_SETTING_FAN_ACOUSTIC_LIMIT])) {
+			od8_settings->od8_settings_array[OD8_SETTING_FAN_ACOUSTIC_LIMIT].feature_id =
+				OD8_ACOUSTIC_LIMIT_SCLK;
+			od8_settings->od8_settings_array[OD8_SETTING_FAN_ACOUSTIC_LIMIT].default_value =
+				od_table->FanMaximumRpm;
+		}
+
+		if (table_context->od_feature_capabilities[ATOM_VEGA20_ODFEATURE_FAN_SPEED_MIN] &&
+		    table_context->od_settings_min[OD8_SETTING_FAN_MIN_SPEED] > 0 &&
+		    table_context->od_settings_max[OD8_SETTING_FAN_MIN_SPEED] > 0 &&
+		    (table_context->od_settings_max[OD8_SETTING_FAN_MIN_SPEED] >=
+		     table_context->od_settings_min[OD8_SETTING_FAN_MIN_SPEED])) {
+			od8_settings->od8_settings_array[OD8_SETTING_FAN_MIN_SPEED].feature_id =
+				OD8_FAN_SPEED_MIN;
+			od8_settings->od8_settings_array[OD8_SETTING_FAN_MIN_SPEED].default_value =
+				od_table->FanMinimumPwm * smc_pptable->FanMaximumRpm / 100;
+		}
+	}
+
+	if (smu_feature_is_enabled(smu, FEATURE_THERMAL_BIT)) {
+		if (table_context->od_feature_capabilities[ATOM_VEGA20_ODFEATURE_TEMPERATURE_FAN] &&
+		    table_context->od_settings_min[OD8_SETTING_FAN_TARGET_TEMP] > 0 &&
+		    table_context->od_settings_max[OD8_SETTING_FAN_TARGET_TEMP] > 0 &&
+		    (table_context->od_settings_max[OD8_SETTING_FAN_TARGET_TEMP] >=
+		     table_context->od_settings_min[OD8_SETTING_FAN_TARGET_TEMP])) {
+			od8_settings->od8_settings_array[OD8_SETTING_FAN_TARGET_TEMP].feature_id =
+				OD8_TEMPERATURE_FAN;
+			od8_settings->od8_settings_array[OD8_SETTING_FAN_TARGET_TEMP].default_value =
+				od_table->FanTargetTemperature;
+		}
+
+		if (table_context->od_feature_capabilities[ATOM_VEGA20_ODFEATURE_TEMPERATURE_SYSTEM] &&
+		    table_context->od_settings_min[OD8_SETTING_OPERATING_TEMP_MAX] > 0 &&
+		    table_context->od_settings_max[OD8_SETTING_OPERATING_TEMP_MAX] > 0 &&
+		    (table_context->od_settings_max[OD8_SETTING_OPERATING_TEMP_MAX] >=
+		     table_context->od_settings_min[OD8_SETTING_OPERATING_TEMP_MAX])) {
+			od8_settings->od8_settings_array[OD8_SETTING_OPERATING_TEMP_MAX].feature_id =
+				OD8_TEMPERATURE_SYSTEM;
+			od8_settings->od8_settings_array[OD8_SETTING_OPERATING_TEMP_MAX].default_value =
+				od_table->MaxOpTemp;
+		}
+	}
+
+	for (i = 0; i < OD8_SETTING_COUNT; i++) {
+		if (od8_settings->od8_settings_array[i].feature_id) {
+			od8_settings->od8_settings_array[i].min_value =
+				table_context->od_settings_min[i];
+			od8_settings->od8_settings_array[i].max_value =
+				table_context->od_settings_max[i];
+			od8_settings->od8_settings_array[i].current_value =
+				od8_settings->od8_settings_array[i].default_value;
+		} else {
+			od8_settings->od8_settings_array[i].min_value = 0;
+			od8_settings->od8_settings_array[i].max_value = 0;
+			od8_settings->od8_settings_array[i].current_value = 0;
+		}
+	}
+
+	return 0;
+}
+
+static int vega20_get_od_percentage(struct smu_context *smu,
+				    enum pp_clock_type type)
+{
+	struct smu_dpm_context *smu_dpm = &smu->smu_dpm;
+	struct vega20_dpm_table *dpm_table = NULL;
+	struct vega20_dpm_table *golden_table = NULL;
+	struct vega20_single_dpm_table *single_dpm_table;
+	struct vega20_single_dpm_table *golden_dpm_table;
+	int value, golden_value;
+
+	dpm_table = smu_dpm->dpm_context;
+	golden_table = smu_dpm->golden_dpm_context;
+
+	switch (type) {
+	case OD_SCLK:
+		single_dpm_table = &(dpm_table->gfx_table);
+		golden_dpm_table = &(golden_table->gfx_table);
+		break;
+	case OD_MCLK:
+		single_dpm_table = &(dpm_table->mem_table);
+		golden_dpm_table = &(golden_table->mem_table);
+		break;
+	default:
+		return -EINVAL;
+		break;
+	}
+
+	value = single_dpm_table->dpm_levels[single_dpm_table->count - 1].value;
+	golden_value = golden_dpm_table->dpm_levels[golden_dpm_table->count - 1].value;
+
+	value -= golden_value;
+	value = DIV_ROUND_UP(value * 100, golden_value);
+
+	return value;
+}
+
+static int
+vega20_get_profiling_clk_mask(struct smu_context *smu,
+			      enum amd_dpm_forced_level level,
+			      uint32_t *sclk_mask,
+			      uint32_t *mclk_mask,
+			      uint32_t *soc_mask)
+{
+	struct vega20_dpm_table *dpm_table = (struct vega20_dpm_table *)smu->smu_dpm.dpm_context;
+	struct vega20_single_dpm_table *gfx_dpm_table;
+	struct vega20_single_dpm_table *mem_dpm_table;
+	struct vega20_single_dpm_table *soc_dpm_table;
+
+	if (!smu->smu_dpm.dpm_context)
+		return -EINVAL;
+
+	gfx_dpm_table = &dpm_table->gfx_table;
+	mem_dpm_table = &dpm_table->mem_table;
+	soc_dpm_table = &dpm_table->soc_table;
+
+	*sclk_mask = 0;
+	*mclk_mask = 0;
+	*soc_mask  = 0;
+
+	if (gfx_dpm_table->count > VEGA20_UMD_PSTATE_GFXCLK_LEVEL &&
+	    mem_dpm_table->count > VEGA20_UMD_PSTATE_MCLK_LEVEL &&
+	    soc_dpm_table->count > VEGA20_UMD_PSTATE_SOCCLK_LEVEL) {
+		*sclk_mask = VEGA20_UMD_PSTATE_GFXCLK_LEVEL;
+		*mclk_mask = VEGA20_UMD_PSTATE_MCLK_LEVEL;
+		*soc_mask  = VEGA20_UMD_PSTATE_SOCCLK_LEVEL;
+	}
+
+	if (level == AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK) {
+		*sclk_mask = 0;
+	} else if (level == AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK) {
+		*mclk_mask = 0;
+	} else if (level == AMD_DPM_FORCED_LEVEL_PROFILE_PEAK) {
+		*sclk_mask = gfx_dpm_table->count - 1;
+		*mclk_mask = mem_dpm_table->count - 1;
+		*soc_mask  = soc_dpm_table->count - 1;
+	}
+
+	return 0;
+}
+
+static int
+vega20_set_uclk_to_highest_dpm_level(struct smu_context *smu,
+				     struct vega20_single_dpm_table *dpm_table)
+{
+	int ret = 0;
+	struct smu_dpm_context *smu_dpm_ctx = &(smu->smu_dpm);
+	if (!smu_dpm_ctx->dpm_context)
+		return -EINVAL;
+
+	if (smu_feature_is_enabled(smu, FEATURE_DPM_UCLK_BIT)) {
+		if (dpm_table->count <= 0) {
+			pr_err("[%s] Dpm table has no entry!", __func__);
+				return -EINVAL;
+		}
+
+		if (dpm_table->count > NUM_UCLK_DPM_LEVELS) {
+			pr_err("[%s] Dpm table has too many entries!", __func__);
+				return -EINVAL;
+		}
+
+		dpm_table->dpm_state.hard_min_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+		ret = smu_send_smc_msg_with_param(smu,
+				SMU_MSG_SetHardMinByFreq,
+				(PPCLK_UCLK << 16) | dpm_table->dpm_state.hard_min_level);
+		if (ret) {
+			pr_err("[%s] Set hard min uclk failed!", __func__);
+				return ret;
+		}
+	}
+
+	return ret;
+}
+
+static int vega20_pre_display_config_changed(struct smu_context *smu)
+{
+	int ret = 0;
+	struct vega20_dpm_table *dpm_table = smu->smu_dpm.dpm_context;
+
+	if (!smu->smu_dpm.dpm_context)
+		return -EINVAL;
+
+	smu_send_smc_msg_with_param(smu, SMU_MSG_NumOfDisplays, 0);
+	ret = vega20_set_uclk_to_highest_dpm_level(smu,
+						   &dpm_table->mem_table);
+	if (ret)
+		pr_err("Failed to set uclk to highest dpm level");
+	return ret;
+}
+
+static int vega20_display_config_changed(struct smu_context *smu)
+{
+	int ret = 0;
+
+	if (!smu->funcs)
+		return -EINVAL;
+
+	if (!smu->smu_dpm.dpm_context ||
+	    !smu->smu_table.tables ||
+	    !smu->smu_table.tables[TABLE_WATERMARKS].cpu_addr)
+		return -EINVAL;
+
+	if ((smu->watermarks_bitmap & WATERMARKS_EXIST) &&
+	    !(smu->watermarks_bitmap & WATERMARKS_LOADED)) {
+		ret = smu->funcs->write_watermarks_table(smu);
+		if (ret) {
+			pr_err("Failed to update WMTABLE!");
+			return ret;
+		}
+		smu->watermarks_bitmap |= WATERMARKS_LOADED;
+	}
+
+	if ((smu->watermarks_bitmap & WATERMARKS_EXIST) &&
+	    smu_feature_is_supported(smu, FEATURE_DPM_DCEFCLK_BIT) &&
+	    smu_feature_is_supported(smu, FEATURE_DPM_SOCCLK_BIT)) {
+		smu_send_smc_msg_with_param(smu,
+					    SMU_MSG_NumOfDisplays,
+					    smu->display_config->num_display);
+	}
+
+	return ret;
+}
+
+static int vega20_apply_clocks_adjust_rules(struct smu_context *smu)
+{
+	struct smu_dpm_context *smu_dpm_ctx = &(smu->smu_dpm);
+	struct vega20_dpm_table *dpm_ctx = (struct vega20_dpm_table *)(smu_dpm_ctx->dpm_context);
+	struct vega20_single_dpm_table *dpm_table;
+	bool vblank_too_short = false;
+	bool disable_mclk_switching;
+	uint32_t i, latency;
+
+	disable_mclk_switching = ((1 < smu->display_config->num_display) &&
+				  !smu->display_config->multi_monitor_in_sync) || vblank_too_short;
+	latency = smu->display_config->dce_tolerable_mclk_in_active_latency;
+
+	/* gfxclk */
+	dpm_table = &(dpm_ctx->gfx_table);
+	dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[0].value;
+	dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+	dpm_table->dpm_state.hard_min_level = dpm_table->dpm_levels[0].value;
+	dpm_table->dpm_state.hard_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+
+		if (VEGA20_UMD_PSTATE_GFXCLK_LEVEL < dpm_table->count) {
+			dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[VEGA20_UMD_PSTATE_GFXCLK_LEVEL].value;
+			dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[VEGA20_UMD_PSTATE_GFXCLK_LEVEL].value;
+		}
+
+		if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK) {
+			dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[0].value;
+			dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[0].value;
+		}
+
+		if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_PROFILE_PEAK) {
+			dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+			dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+		}
+
+	/* memclk */
+	dpm_table = &(dpm_ctx->mem_table);
+	dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[0].value;
+	dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+	dpm_table->dpm_state.hard_min_level = dpm_table->dpm_levels[0].value;
+	dpm_table->dpm_state.hard_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+
+		if (VEGA20_UMD_PSTATE_MCLK_LEVEL < dpm_table->count) {
+			dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[VEGA20_UMD_PSTATE_MCLK_LEVEL].value;
+			dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[VEGA20_UMD_PSTATE_MCLK_LEVEL].value;
+		}
+
+		if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK) {
+			dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[0].value;
+			dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[0].value;
+		}
+
+		if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_PROFILE_PEAK) {
+			dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+			dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+		}
+
+	/* honour DAL's UCLK Hardmin */
+	if (dpm_table->dpm_state.hard_min_level < (smu->display_config->min_mem_set_clock / 100))
+		dpm_table->dpm_state.hard_min_level = smu->display_config->min_mem_set_clock / 100;
+
+	/* Hardmin is dependent on displayconfig */
+	if (disable_mclk_switching) {
+		dpm_table->dpm_state.hard_min_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+		for (i = 0; i < smu_dpm_ctx->mclk_latency_table->count - 1; i++) {
+			if (smu_dpm_ctx->mclk_latency_table->entries[i].latency <= latency) {
+				if (dpm_table->dpm_levels[i].value >= (smu->display_config->min_mem_set_clock / 100)) {
+					dpm_table->dpm_state.hard_min_level = dpm_table->dpm_levels[i].value;
+					break;
+				}
+			}
+		}
+	}
+
+	if (smu->display_config->nb_pstate_switch_disable)
+		dpm_table->dpm_state.hard_min_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+
+	/* vclk */
+	dpm_table = &(dpm_ctx->vclk_table);
+	dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[0].value;
+	dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+	dpm_table->dpm_state.hard_min_level = dpm_table->dpm_levels[0].value;
+	dpm_table->dpm_state.hard_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+
+		if (VEGA20_UMD_PSTATE_UVDCLK_LEVEL < dpm_table->count) {
+			dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[VEGA20_UMD_PSTATE_UVDCLK_LEVEL].value;
+			dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[VEGA20_UMD_PSTATE_UVDCLK_LEVEL].value;
+		}
+
+		if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_PROFILE_PEAK) {
+			dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+			dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+		}
+
+	/* dclk */
+	dpm_table = &(dpm_ctx->dclk_table);
+	dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[0].value;
+	dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+	dpm_table->dpm_state.hard_min_level = dpm_table->dpm_levels[0].value;
+	dpm_table->dpm_state.hard_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+
+		if (VEGA20_UMD_PSTATE_UVDCLK_LEVEL < dpm_table->count) {
+			dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[VEGA20_UMD_PSTATE_UVDCLK_LEVEL].value;
+			dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[VEGA20_UMD_PSTATE_UVDCLK_LEVEL].value;
+		}
+
+		if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_PROFILE_PEAK) {
+			dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+			dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+		}
+
+	/* socclk */
+	dpm_table = &(dpm_ctx->soc_table);
+	dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[0].value;
+	dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+	dpm_table->dpm_state.hard_min_level = dpm_table->dpm_levels[0].value;
+	dpm_table->dpm_state.hard_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+
+		if (VEGA20_UMD_PSTATE_SOCCLK_LEVEL < dpm_table->count) {
+			dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[VEGA20_UMD_PSTATE_SOCCLK_LEVEL].value;
+			dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[VEGA20_UMD_PSTATE_SOCCLK_LEVEL].value;
+		}
+
+		if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_PROFILE_PEAK) {
+			dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+			dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+		}
+
+	/* eclk */
+	dpm_table = &(dpm_ctx->eclk_table);
+	dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[0].value;
+	dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+	dpm_table->dpm_state.hard_min_level = dpm_table->dpm_levels[0].value;
+	dpm_table->dpm_state.hard_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+
+		if (VEGA20_UMD_PSTATE_VCEMCLK_LEVEL < dpm_table->count) {
+			dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[VEGA20_UMD_PSTATE_VCEMCLK_LEVEL].value;
+			dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[VEGA20_UMD_PSTATE_VCEMCLK_LEVEL].value;
+		}
+
+		if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_PROFILE_PEAK) {
+			dpm_table->dpm_state.soft_min_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+			dpm_table->dpm_state.soft_max_level = dpm_table->dpm_levels[dpm_table->count - 1].value;
+		}
+	return 0;
+}
+
+static int
+vega20_notify_smc_dispaly_config(struct smu_context *smu)
+{
+	struct vega20_dpm_table *dpm_table = smu->smu_dpm.dpm_context;
+	struct vega20_single_dpm_table *memtable = &dpm_table->mem_table;
+	struct smu_clocks min_clocks = {0};
+	struct pp_display_clock_request clock_req;
+	int ret = 0;
+
+	min_clocks.dcef_clock = smu->display_config->min_dcef_set_clk;
+	min_clocks.dcef_clock_in_sr = smu->display_config->min_dcef_deep_sleep_set_clk;
+	min_clocks.memory_clock = smu->display_config->min_mem_set_clock;
+
+	if (smu_feature_is_supported(smu, FEATURE_DPM_DCEFCLK_BIT)) {
+		clock_req.clock_type = amd_pp_dcef_clock;
+		clock_req.clock_freq_in_khz = min_clocks.dcef_clock * 10;
+		if (!smu->funcs->display_clock_voltage_request(smu, &clock_req)) {
+			if (smu_feature_is_supported(smu, FEATURE_DS_DCEFCLK_BIT)) {
+				ret = smu_send_smc_msg_with_param(smu,
+								  SMU_MSG_SetMinDeepSleepDcefclk,
+								  min_clocks.dcef_clock_in_sr/100);
+				if (ret) {
+					pr_err("Attempt to set divider for DCEFCLK Failed!");
+					return ret;
+				}
+			}
+		} else {
+			pr_info("Attempt to set Hard Min for DCEFCLK Failed!");
+		}
+	}
+
+	if (smu_feature_is_enabled(smu, FEATURE_DPM_UCLK_BIT)) {
+		memtable->dpm_state.hard_min_level = min_clocks.memory_clock/100;
+		ret = smu_send_smc_msg_with_param(smu,
+						  SMU_MSG_SetHardMinByFreq,
+						  (PPCLK_UCLK << 16) | memtable->dpm_state.hard_min_level);
+		if (ret) {
+			pr_err("[%s] Set hard min uclk failed!", __func__);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static uint32_t vega20_find_lowest_dpm_level(struct vega20_single_dpm_table *table)
+{
+	uint32_t i;
+
+	for (i = 0; i < table->count; i++) {
+		if (table->dpm_levels[i].enabled)
+			break;
+	}
+	if (i >= table->count) {
+		i = 0;
+		table->dpm_levels[i].enabled = true;
+	}
+
+	return i;
+}
+
+static uint32_t vega20_find_highest_dpm_level(struct vega20_single_dpm_table *table)
+{
+	int i = 0;
+
+	if (!table) {
+		pr_err("[%s] DPM Table does not exist!", __func__);
+		return 0;
+	}
+	if (table->count <= 0) {
+		pr_err("[%s] DPM Table has no entry!", __func__);
+		return 0;
+	}
+	if (table->count > MAX_REGULAR_DPM_NUMBER) {
+		pr_err("[%s] DPM Table has too many entries!", __func__);
+		return MAX_REGULAR_DPM_NUMBER - 1;
+	}
+
+	for (i = table->count - 1; i >= 0; i--) {
+		if (table->dpm_levels[i].enabled)
+			break;
+	}
+	if (i < 0) {
+		i = 0;
+		table->dpm_levels[i].enabled = true;
+	}
+
+	return i;
+}
+
+static int vega20_force_dpm_limit_value(struct smu_context *smu, bool highest)
+{
+	uint32_t soft_level;
+	int ret = 0;
+	struct vega20_dpm_table *dpm_table =
+		(struct vega20_dpm_table *)smu->smu_dpm.dpm_context;
+
+	if (highest)
+		soft_level = vega20_find_highest_dpm_level(&(dpm_table->gfx_table));
+	else
+		soft_level = vega20_find_lowest_dpm_level(&(dpm_table->gfx_table));
+
+	dpm_table->gfx_table.dpm_state.soft_min_level =
+		dpm_table->gfx_table.dpm_state.soft_max_level =
+		dpm_table->gfx_table.dpm_levels[soft_level].value;
+
+	if (highest)
+		soft_level = vega20_find_highest_dpm_level(&(dpm_table->mem_table));
+	else
+		soft_level = vega20_find_lowest_dpm_level(&(dpm_table->mem_table));
+
+	dpm_table->mem_table.dpm_state.soft_min_level =
+		dpm_table->mem_table.dpm_state.soft_max_level =
+		dpm_table->mem_table.dpm_levels[soft_level].value;
+
+	if (highest)
+		soft_level = vega20_find_highest_dpm_level(&(dpm_table->soc_table));
+	else
+		soft_level = vega20_find_lowest_dpm_level(&(dpm_table->soc_table));
+
+	dpm_table->soc_table.dpm_state.soft_min_level =
+		dpm_table->soc_table.dpm_state.soft_max_level =
+		dpm_table->soc_table.dpm_levels[soft_level].value;
+
+	ret = vega20_upload_dpm_level(smu, false, 0xFFFFFFFF);
+	if (ret) {
+		pr_err("Failed to upload boot level to %s!\n",
+				highest ? "highest" : "lowest");
+		return ret;
+	}
+
+	ret = vega20_upload_dpm_level(smu, true, 0xFFFFFFFF);
+	if (ret) {
+		pr_err("Failed to upload dpm max level to %s!\n!",
+				highest ? "highest" : "lowest");
+		return ret;
+	}
+
+	return ret;
+}
+
+static int vega20_unforce_dpm_levels(struct smu_context *smu)
+{
+	uint32_t soft_min_level, soft_max_level;
+	int ret = 0;
+	struct vega20_dpm_table *dpm_table =
+		(struct vega20_dpm_table *)smu->smu_dpm.dpm_context;
+
+	soft_min_level = vega20_find_lowest_dpm_level(&(dpm_table->gfx_table));
+	soft_max_level = vega20_find_highest_dpm_level(&(dpm_table->gfx_table));
+	dpm_table->gfx_table.dpm_state.soft_min_level =
+		dpm_table->gfx_table.dpm_levels[soft_min_level].value;
+	dpm_table->gfx_table.dpm_state.soft_max_level =
+		dpm_table->gfx_table.dpm_levels[soft_max_level].value;
+
+	soft_min_level = vega20_find_lowest_dpm_level(&(dpm_table->mem_table));
+	soft_max_level = vega20_find_highest_dpm_level(&(dpm_table->mem_table));
+	dpm_table->mem_table.dpm_state.soft_min_level =
+		dpm_table->gfx_table.dpm_levels[soft_min_level].value;
+	dpm_table->mem_table.dpm_state.soft_max_level =
+		dpm_table->gfx_table.dpm_levels[soft_max_level].value;
+
+	soft_min_level = vega20_find_lowest_dpm_level(&(dpm_table->soc_table));
+	soft_max_level = vega20_find_highest_dpm_level(&(dpm_table->soc_table));
+	dpm_table->soc_table.dpm_state.soft_min_level =
+		dpm_table->soc_table.dpm_levels[soft_min_level].value;
+	dpm_table->soc_table.dpm_state.soft_max_level =
+		dpm_table->soc_table.dpm_levels[soft_max_level].value;
+
+	ret = smu_upload_dpm_level(smu, false, 0xFFFFFFFF);
+	if (ret) {
+		pr_err("Failed to upload DPM Bootup Levels!");
+		return ret;
+	}
+
+	ret = smu_upload_dpm_level(smu, true, 0xFFFFFFFF);
+	if (ret) {
+		pr_err("Failed to upload DPM Max Levels!");
+		return ret;
+	}
+
+	return ret;
+}
+
+static enum amd_dpm_forced_level vega20_get_performance_level(struct smu_context *smu)
+{
+	struct smu_dpm_context *smu_dpm_ctx = &(smu->smu_dpm);
+	if (!smu_dpm_ctx->dpm_context)
+		return -EINVAL;
+
+	if (smu_dpm_ctx->dpm_level != smu_dpm_ctx->saved_dpm_level) {
+		mutex_lock(&(smu->mutex));
+		smu_dpm_ctx->saved_dpm_level = smu_dpm_ctx->dpm_level;
+		mutex_unlock(&(smu->mutex));
+	}
+	return smu_dpm_ctx->dpm_level;
+}
+
+static int
+vega20_force_performance_level(struct smu_context *smu, enum amd_dpm_forced_level level)
+{
+	int ret = 0;
+	int i;
+	struct smu_dpm_context *smu_dpm_ctx = &(smu->smu_dpm);
+
+	if (!smu_dpm_ctx->dpm_context)
+		return -EINVAL;
+
+	for (i = 0; i < smu->adev->num_ip_blocks; i++) {
+		if (smu->adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC)
+			break;
+	}
+
+	mutex_lock(&smu->mutex);
+
+	smu->adev->ip_blocks[i].version->funcs->enable_umd_pstate(smu, &level);
+	ret = smu_handle_task(smu, level,
+			      AMD_PP_TASK_READJUST_POWER_STATE);
+
+	mutex_unlock(&smu->mutex);
+
+	return ret;
+}
+
+static int vega20_update_specified_od8_value(struct smu_context *smu,
+					     uint32_t index,
+					     uint32_t value)
+{
+	struct smu_table_context *table_context = &smu->smu_table;
+	OverDriveTable_t *od_table =
+		(OverDriveTable_t *)(table_context->overdrive_table);
+	struct vega20_od8_settings *od8_settings =
+		(struct vega20_od8_settings *)table_context->od8_settings;
+
+	switch (index) {
+	case OD8_SETTING_GFXCLK_FMIN:
+		od_table->GfxclkFmin = (uint16_t)value;
+		break;
+
+	case OD8_SETTING_GFXCLK_FMAX:
+		if (value < od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FMAX].min_value ||
+		    value > od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FMAX].max_value)
+			return -EINVAL;
+		od_table->GfxclkFmax = (uint16_t)value;
+		break;
+
+	case OD8_SETTING_GFXCLK_FREQ1:
+		od_table->GfxclkFreq1 = (uint16_t)value;
+		break;
+
+	case OD8_SETTING_GFXCLK_VOLTAGE1:
+		od_table->GfxclkVolt1 = (uint16_t)value;
+		break;
+
+	case OD8_SETTING_GFXCLK_FREQ2:
+		od_table->GfxclkFreq2 = (uint16_t)value;
+		break;
+
+	case OD8_SETTING_GFXCLK_VOLTAGE2:
+		od_table->GfxclkVolt2 = (uint16_t)value;
+		break;
+
+	case OD8_SETTING_GFXCLK_FREQ3:
+		od_table->GfxclkFreq3 = (uint16_t)value;
+		break;
+
+	case OD8_SETTING_GFXCLK_VOLTAGE3:
+		od_table->GfxclkVolt3 = (uint16_t)value;
+		break;
+
+	case OD8_SETTING_UCLK_FMAX:
+		if (value < od8_settings->od8_settings_array[OD8_SETTING_UCLK_FMAX].min_value ||
+		    value > od8_settings->od8_settings_array[OD8_SETTING_UCLK_FMAX].max_value)
+			return -EINVAL;
+		od_table->UclkFmax = (uint16_t)value;
+		break;
+
+	case OD8_SETTING_POWER_PERCENTAGE:
+		od_table->OverDrivePct = (int16_t)value;
+		break;
+
+	case OD8_SETTING_FAN_ACOUSTIC_LIMIT:
+		od_table->FanMaximumRpm = (uint16_t)value;
+		break;
+
+	case OD8_SETTING_FAN_MIN_SPEED:
+		od_table->FanMinimumPwm = (uint16_t)value;
+		break;
+
+	case OD8_SETTING_FAN_TARGET_TEMP:
+		od_table->FanTargetTemperature = (uint16_t)value;
+		break;
+
+	case OD8_SETTING_OPERATING_TEMP_MAX:
+		od_table->MaxOpTemp = (uint16_t)value;
+		break;
+	}
+
+	return 0;
+}
+
+static int vega20_set_od_percentage(struct smu_context *smu,
+				    enum pp_clock_type type,
+				    uint32_t value)
+{
+	struct smu_dpm_context *smu_dpm = &smu->smu_dpm;
+	struct vega20_dpm_table *dpm_table = NULL;
+	struct vega20_dpm_table *golden_table = NULL;
+	struct vega20_single_dpm_table *single_dpm_table;
+	struct vega20_single_dpm_table *golden_dpm_table;
+	uint32_t od_clk, index;
+	int ret = 0;
+	int feature_enabled;
+	PPCLK_e clk_id;
+
+	mutex_lock(&(smu->mutex));
+
+	dpm_table = smu_dpm->dpm_context;
+	golden_table = smu_dpm->golden_dpm_context;
+
+	switch (type) {
+	case OD_SCLK:
+		single_dpm_table = &(dpm_table->gfx_table);
+		golden_dpm_table = &(golden_table->gfx_table);
+		feature_enabled = smu_feature_is_enabled(smu, FEATURE_DPM_GFXCLK_BIT);
+		clk_id = PPCLK_GFXCLK;
+		index = OD8_SETTING_GFXCLK_FMAX;
+		break;
+	case OD_MCLK:
+		single_dpm_table = &(dpm_table->mem_table);
+		golden_dpm_table = &(golden_table->mem_table);
+		feature_enabled = smu_feature_is_enabled(smu, FEATURE_DPM_UCLK_BIT);
+		clk_id = PPCLK_UCLK;
+		index = OD8_SETTING_UCLK_FMAX;
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	if (ret)
+		goto set_od_failed;
+
+	od_clk = golden_dpm_table->dpm_levels[golden_dpm_table->count - 1].value * value;
+	od_clk /= 100;
+	od_clk += golden_dpm_table->dpm_levels[golden_dpm_table->count - 1].value;
+
+	ret = smu_update_od8_settings(smu, index, od_clk);
+	if (ret) {
+		pr_err("[Setoverdrive] failed to set od clk!\n");
+		goto set_od_failed;
+	}
+
+	if (feature_enabled) {
+		ret = vega20_set_single_dpm_table(smu, single_dpm_table,
+						  clk_id);
+		if (ret) {
+			pr_err("[Setoverdrive] failed to refresh dpm table!\n");
+			goto set_od_failed;
+		}
+	} else {
+		single_dpm_table->count = 1;
+		single_dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.gfxclk / 100;
+	}
+
+	ret = smu_handle_task(smu, smu_dpm->dpm_level,
+			      AMD_PP_TASK_READJUST_POWER_STATE);
+
+set_od_failed:
+	mutex_unlock(&(smu->mutex));
+
+	return ret;
+}
+
+static int vega20_odn_edit_dpm_table(struct smu_context *smu,
+				     enum PP_OD_DPM_TABLE_COMMAND type,
+				     long *input, uint32_t size)
+{
+	struct smu_table_context *table_context = &smu->smu_table;
+	OverDriveTable_t *od_table =
+		(OverDriveTable_t *)(table_context->overdrive_table);
+	struct smu_dpm_context *smu_dpm = &smu->smu_dpm;
+	struct vega20_dpm_table *dpm_table = NULL;
+	struct vega20_single_dpm_table *single_dpm_table;
+	struct vega20_od8_settings *od8_settings =
+		(struct vega20_od8_settings *)table_context->od8_settings;
+	struct pp_clock_levels_with_latency clocks;
+	int32_t input_index, input_clk, input_vol, i;
+	int od8_id;
+	int ret = 0;
+
+	dpm_table = smu_dpm->dpm_context;
+
+	if (!input) {
+		pr_warn("NULL user input for clock and voltage\n");
+		return -EINVAL;
+	}
+
+	switch (type) {
+	case PP_OD_EDIT_SCLK_VDDC_TABLE:
+		if (!(od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FMIN].feature_id &&
+		      od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FMAX].feature_id)) {
+			pr_info("Sclk min/max frequency overdrive not supported\n");
+			return -EOPNOTSUPP;
+		}
+
+		for (i = 0; i < size; i += 2) {
+			if (i + 2 > size) {
+				pr_info("invalid number of input parameters %d\n", size);
+				return -EINVAL;
+			}
+
+			input_index = input[i];
+			input_clk = input[i + 1];
+
+			if (input_index != 0 && input_index != 1) {
+				pr_info("Invalid index %d\n", input_index);
+				pr_info("Support min/max sclk frequency settingonly which index by 0/1\n");
+				return -EINVAL;
+			}
+
+			if (input_clk < od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FMIN].min_value ||
+			    input_clk > od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FMAX].max_value) {
+				pr_info("clock freq %d is not within allowed range [%d - %d]\n",
+					input_clk,
+					od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FMIN].min_value,
+					od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FMAX].max_value);
+				return -EINVAL;
+			}
+
+			if (input_index == 0 && od_table->GfxclkFmin != input_clk) {
+				od_table->GfxclkFmin = input_clk;
+				table_context->od_gfxclk_update = true;
+			} else if (input_index == 1 && od_table->GfxclkFmax != input_clk) {
+				od_table->GfxclkFmax = input_clk;
+				table_context->od_gfxclk_update = true;
+			}
+		}
+
+		break;
+
+	case PP_OD_EDIT_MCLK_VDDC_TABLE:
+		if (!od8_settings->od8_settings_array[OD8_SETTING_UCLK_FMAX].feature_id) {
+			pr_info("Mclk max frequency overdrive not supported\n");
+			return -EOPNOTSUPP;
+		}
+
+		single_dpm_table = &(dpm_table->mem_table);
+		ret = vega20_get_clk_table(smu, &clocks, single_dpm_table);
+		if (ret) {
+			pr_err("Attempt to get memory clk levels Failed!");
+			return ret;
+		}
+
+		for (i = 0; i < size; i += 2) {
+			if (i + 2 > size) {
+				pr_info("invalid number of input parameters %d\n",
+					 size);
+				return -EINVAL;
+			}
+
+			input_index = input[i];
+			input_clk = input[i + 1];
+
+			if (input_index != 1) {
+				pr_info("Invalid index %d\n", input_index);
+				pr_info("Support max Mclk frequency setting only which index by 1\n");
+				return -EINVAL;
+			}
+
+			if (input_clk < clocks.data[0].clocks_in_khz / 1000 ||
+			    input_clk > od8_settings->od8_settings_array[OD8_SETTING_UCLK_FMAX].max_value) {
+				pr_info("clock freq %d is not within allowed range [%d - %d]\n",
+					input_clk,
+					clocks.data[0].clocks_in_khz / 1000,
+					od8_settings->od8_settings_array[OD8_SETTING_UCLK_FMAX].max_value);
+				return -EINVAL;
+			}
+
+			if (input_index == 1 && od_table->UclkFmax != input_clk) {
+				table_context->od_gfxclk_update = true;
+				od_table->UclkFmax = input_clk;
+			}
+		}
+
+		break;
+
+	case PP_OD_EDIT_VDDC_CURVE:
+		if (!(od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FREQ1].feature_id &&
+		      od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FREQ2].feature_id &&
+		      od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_FREQ3].feature_id &&
+		      od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_VOLTAGE1].feature_id &&
+		      od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_VOLTAGE2].feature_id &&
+		      od8_settings->od8_settings_array[OD8_SETTING_GFXCLK_VOLTAGE3].feature_id)) {
+			pr_info("Voltage curve calibrate not supported\n");
+			return -EOPNOTSUPP;
+		}
+
+		for (i = 0; i < size; i += 3) {
+			if (i + 3 > size) {
+				pr_info("invalid number of input parameters %d\n",
+					size);
+				return -EINVAL;
+			}
+
+			input_index = input[i];
+			input_clk = input[i + 1];
+			input_vol = input[i + 2];
+
+			if (input_index > 2) {
+				pr_info("Setting for point %d is not supported\n",
+					input_index + 1);
+				pr_info("Three supported points index by 0, 1, 2\n");
+				return -EINVAL;
+			}
+
+			od8_id = OD8_SETTING_GFXCLK_FREQ1 + 2 * input_index;
+			if (input_clk < od8_settings->od8_settings_array[od8_id].min_value ||
+			    input_clk > od8_settings->od8_settings_array[od8_id].max_value) {
+				pr_info("clock freq %d is not within allowed range [%d - %d]\n",
+					input_clk,
+					od8_settings->od8_settings_array[od8_id].min_value,
+					od8_settings->od8_settings_array[od8_id].max_value);
+				return -EINVAL;
+			}
+
+			od8_id = OD8_SETTING_GFXCLK_VOLTAGE1 + 2 * input_index;
+			if (input_vol < od8_settings->od8_settings_array[od8_id].min_value ||
+			    input_vol > od8_settings->od8_settings_array[od8_id].max_value) {
+				pr_info("clock voltage %d is not within allowed range [%d- %d]\n",
+					input_vol,
+					od8_settings->od8_settings_array[od8_id].min_value,
+					od8_settings->od8_settings_array[od8_id].max_value);
+				return -EINVAL;
+			}
+
+			switch (input_index) {
+			case 0:
+				od_table->GfxclkFreq1 = input_clk;
+				od_table->GfxclkVolt1 = input_vol * VOLTAGE_SCALE;
+				break;
+			case 1:
+				od_table->GfxclkFreq2 = input_clk;
+				od_table->GfxclkVolt2 = input_vol * VOLTAGE_SCALE;
+				break;
+			case 2:
+				od_table->GfxclkFreq3 = input_clk;
+				od_table->GfxclkVolt3 = input_vol * VOLTAGE_SCALE;
+				break;
+			}
+		}
+
+		break;
+
+	case PP_OD_RESTORE_DEFAULT_TABLE:
+		ret = smu_update_table(smu, TABLE_OVERDRIVE, table_context->overdrive_table, false);
+		if (ret) {
+			pr_err("Failed to export over drive table!\n");
+			return ret;
+		}
+
+		break;
+
+	case PP_OD_COMMIT_DPM_TABLE:
+		ret = smu_update_table(smu, TABLE_OVERDRIVE, table_context->overdrive_table, true);
+		if (ret) {
+			pr_err("Failed to import over drive table!\n");
+			return ret;
+		}
+
+		/* retrieve updated gfxclk table */
+		if (table_context->od_gfxclk_update) {
+			table_context->od_gfxclk_update = false;
+			single_dpm_table = &(dpm_table->gfx_table);
+
+			if (smu_feature_is_enabled(smu, FEATURE_DPM_GFXCLK_BIT)) {
+				ret = vega20_set_single_dpm_table(smu, single_dpm_table,
+								  PPCLK_GFXCLK);
+				if (ret) {
+					pr_err("[Setoverdrive] failed to refresh dpm table!\n");
+					return ret;
+				}
+			} else {
+				single_dpm_table->count = 1;
+				single_dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.gfxclk / 100;
+			}
+		}
+
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	if (type == PP_OD_COMMIT_DPM_TABLE) {
+		mutex_lock(&(smu->mutex));
+		ret = smu_handle_task(smu, smu_dpm->dpm_level,
+				      AMD_PP_TASK_READJUST_POWER_STATE);
+		mutex_unlock(&(smu->mutex));
+	}
+
+	return ret;
+}
+
+static const struct pptable_funcs vega20_ppt_funcs = {
+	.alloc_dpm_context = vega20_allocate_dpm_context,
+	.store_powerplay_table = vega20_store_powerplay_table,
+	.check_powerplay_table = vega20_check_powerplay_table,
+	.append_powerplay_table = vega20_append_powerplay_table,
+	.get_smu_msg_index = vega20_get_smu_msg_index,
+	.run_afll_btc = vega20_run_btc_afll,
+	.get_unallowed_feature_mask = vega20_get_unallowed_feature_mask,
+	.get_current_power_state = vega20_get_current_power_state,
+	.set_default_dpm_table = vega20_set_default_dpm_table,
+	.set_power_state = NULL,
+	.populate_umd_state_clk = vega20_populate_umd_state_clk,
+	.print_clk_levels = vega20_print_clk_levels,
+	.force_clk_levels = vega20_force_clk_levels,
+	.get_clock_by_type_with_latency = vega20_get_clock_by_type_with_latency,
+	.set_default_od8_settings = vega20_set_default_od8_setttings,
+	.get_od_percentage = vega20_get_od_percentage,
+	.get_performance_level = vega20_get_performance_level,
+	.force_performance_level = vega20_force_performance_level,
+	.update_specified_od8_value = vega20_update_specified_od8_value,
+	.set_od_percentage = vega20_set_od_percentage,
+	.od_edit_dpm_table = vega20_odn_edit_dpm_table,
+	.pre_display_config_changed = vega20_pre_display_config_changed,
+	.display_config_changed = vega20_display_config_changed,
+	.apply_clocks_adjust_rules = vega20_apply_clocks_adjust_rules,
+	.notify_smc_dispaly_config = vega20_notify_smc_dispaly_config,
+	.force_dpm_limit_value = vega20_force_dpm_limit_value,
+	.unforce_dpm_levels = vega20_unforce_dpm_levels,
+	.upload_dpm_level = vega20_upload_dpm_level,
+	.get_profiling_clk_mask = vega20_get_profiling_clk_mask,
+};
+
+void vega20_set_ppt_funcs(struct smu_context *smu)
+{
+	smu->ppt_funcs = &vega20_ppt_funcs;
+	smu->smc_if_version = SMU11_DRIVER_IF_VERSION;
+}
diff --git a/drivers/gpu/drm/amd/powerplay/vega20_ppt.h b/drivers/gpu/drm/amd/powerplay/vega20_ppt.h
new file mode 100644
index 000000000000..5a0d2af63173
--- /dev/null
+++ b/drivers/gpu/drm/amd/powerplay/vega20_ppt.h
@@ -0,0 +1,129 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef __VEGA20_PPT_H__
+#define __VEGA20_PPT_H__
+
+#define VEGA20_UMD_PSTATE_GFXCLK_LEVEL         0x3
+#define VEGA20_UMD_PSTATE_SOCCLK_LEVEL         0x3
+#define VEGA20_UMD_PSTATE_MCLK_LEVEL           0x2
+#define VEGA20_UMD_PSTATE_UVDCLK_LEVEL         0x3
+#define VEGA20_UMD_PSTATE_VCEMCLK_LEVEL        0x3
+
+#define MAX_REGULAR_DPM_NUMBER 16
+#define MAX_PCIE_CONF 2
+
+#define VOLTAGE_SCALE 4
+#define AVFS_CURVE 0
+#define OD8_HOTCURVE_TEMPERATURE 85
+
+struct vega20_dpm_level {
+        bool            enabled;
+        uint32_t        value;
+        uint32_t        param1;
+};
+
+struct vega20_dpm_state {
+        uint32_t  soft_min_level;
+        uint32_t  soft_max_level;
+        uint32_t  hard_min_level;
+        uint32_t  hard_max_level;
+};
+
+struct vega20_single_dpm_table {
+        uint32_t                count;
+        struct vega20_dpm_state dpm_state;
+        struct vega20_dpm_level dpm_levels[MAX_REGULAR_DPM_NUMBER];
+};
+
+struct vega20_pcie_table {
+        uint16_t count;
+        uint8_t  pcie_gen[MAX_PCIE_CONF];
+        uint8_t  pcie_lane[MAX_PCIE_CONF];
+        uint32_t lclk[MAX_PCIE_CONF];
+};
+
+struct vega20_dpm_table {
+	struct vega20_single_dpm_table  soc_table;
+        struct vega20_single_dpm_table  gfx_table;
+        struct vega20_single_dpm_table  mem_table;
+        struct vega20_single_dpm_table  eclk_table;
+        struct vega20_single_dpm_table  vclk_table;
+        struct vega20_single_dpm_table  dclk_table;
+        struct vega20_single_dpm_table  dcef_table;
+        struct vega20_single_dpm_table  pixel_table;
+        struct vega20_single_dpm_table  display_table;
+        struct vega20_single_dpm_table  phy_table;
+        struct vega20_single_dpm_table  fclk_table;
+        struct vega20_pcie_table        pcie_table;
+};
+
+enum OD8_FEATURE_ID
+{
+	OD8_GFXCLK_LIMITS               = 1 << 0,
+	OD8_GFXCLK_CURVE                = 1 << 1,
+	OD8_UCLK_MAX                    = 1 << 2,
+	OD8_POWER_LIMIT                 = 1 << 3,
+	OD8_ACOUSTIC_LIMIT_SCLK         = 1 << 4,   //FanMaximumRpm
+	OD8_FAN_SPEED_MIN               = 1 << 5,   //FanMinimumPwm
+	OD8_TEMPERATURE_FAN             = 1 << 6,   //FanTargetTemperature
+	OD8_TEMPERATURE_SYSTEM          = 1 << 7,   //MaxOpTemp
+	OD8_MEMORY_TIMING_TUNE          = 1 << 8,
+	OD8_FAN_ZERO_RPM_CONTROL        = 1 << 9
+};
+
+enum OD8_SETTING_ID
+{
+	OD8_SETTING_GFXCLK_FMIN = 0,
+	OD8_SETTING_GFXCLK_FMAX,
+	OD8_SETTING_GFXCLK_FREQ1,
+	OD8_SETTING_GFXCLK_VOLTAGE1,
+	OD8_SETTING_GFXCLK_FREQ2,
+	OD8_SETTING_GFXCLK_VOLTAGE2,
+	OD8_SETTING_GFXCLK_FREQ3,
+	OD8_SETTING_GFXCLK_VOLTAGE3,
+	OD8_SETTING_UCLK_FMAX,
+	OD8_SETTING_POWER_PERCENTAGE,
+	OD8_SETTING_FAN_ACOUSTIC_LIMIT,
+	OD8_SETTING_FAN_MIN_SPEED,
+	OD8_SETTING_FAN_TARGET_TEMP,
+	OD8_SETTING_OPERATING_TEMP_MAX,
+	OD8_SETTING_AC_TIMING,
+	OD8_SETTING_FAN_ZERO_RPM_CONTROL,
+	OD8_SETTING_COUNT
+};
+
+struct vega20_od8_single_setting {
+	uint32_t	feature_id;
+	int32_t		min_value;
+	int32_t		max_value;
+	int32_t		current_value;
+	int32_t		default_value;
+};
+
+struct vega20_od8_settings {
+	struct vega20_od8_single_setting	od8_settings_array[OD8_SETTING_COUNT];
+};
+
+extern void vega20_set_ppt_funcs(struct smu_context *smu);
+
+#endif
diff --git a/drivers/gpu/drm/arm/display/include/malidp_product.h b/drivers/gpu/drm/arm/display/include/malidp_product.h
index b35fc5db866b..1053b11352eb 100644
--- a/drivers/gpu/drm/arm/display/include/malidp_product.h
+++ b/drivers/gpu/drm/arm/display/include/malidp_product.h
@@ -20,4 +20,16 @@
 /* Mali-display product IDs */
 #define MALIDP_D71_PRODUCT_ID   0x0071
 
+union komeda_config_id {
+	struct {
+		__u32	max_line_sz:16,
+			n_pipelines:2,
+			n_scalers:2, /* number of scalers per pipeline */
+			n_layers:3, /* number of layers per pipeline */
+			n_richs:3, /* number of rich layers per pipeline */
+			reserved_bits:6;
+	};
+	__u32 value;
+};
+
 #endif /* _MALIDP_PRODUCT_H_ */
diff --git a/drivers/gpu/drm/arm/display/include/malidp_utils.h b/drivers/gpu/drm/arm/display/include/malidp_utils.h
index 63cc47cefcf8..8cfd91196e15 100644
--- a/drivers/gpu/drm/arm/display/include/malidp_utils.h
+++ b/drivers/gpu/drm/arm/display/include/malidp_utils.h
@@ -7,10 +7,41 @@
 #ifndef _MALIDP_UTILS_
 #define _MALIDP_UTILS_
 
+#include <linux/delay.h>
+
 #define has_bit(nr, mask)	(BIT(nr) & (mask))
 #define has_bits(bits, mask)	(((bits) & (mask)) == (bits))
 
 #define dp_for_each_set_bit(bit, mask) \
 	for_each_set_bit((bit), ((unsigned long *)&(mask)), sizeof(mask) * 8)
 
+#define dp_wait_cond(__cond, __tries, __min_range, __max_range)	\
+({							\
+	int num_tries = __tries;			\
+	while (!__cond && (num_tries > 0)) {		\
+		usleep_range(__min_range, __max_range);	\
+		if (__cond)				\
+			break;				\
+		num_tries--;				\
+	}						\
+	num_tries;					\
+})
+
+/* the restriction of range is [start, end] */
+struct malidp_range {
+	u32 start;
+	u32 end;
+};
+
+static inline void set_range(struct malidp_range *rg, u32 start, u32 end)
+{
+	rg->start = start;
+	rg->end   = end;
+}
+
+static inline bool in_range(struct malidp_range *rg, u32 v)
+{
+	return (v >= rg->start) && (v <= rg->end);
+}
+
 #endif /* _MALIDP_UTILS_ */
diff --git a/drivers/gpu/drm/arm/display/komeda/Makefile b/drivers/gpu/drm/arm/display/komeda/Makefile
index 1b875e5dc0f6..412eeba8c39f 100644
--- a/drivers/gpu/drm/arm/display/komeda/Makefile
+++ b/drivers/gpu/drm/arm/display/komeda/Makefile
@@ -1,14 +1,15 @@
 # SPDX-License-Identifier: GPL-2.0
 
 ccflags-y := \
-	-I$(src)/../include \
-	-I$(src)
+	-I $(srctree)/$(src)/../include \
+	-I $(srctree)/$(src)
 
 komeda-y := \
 	komeda_drv.o \
 	komeda_dev.o \
 	komeda_format_caps.o \
 	komeda_pipeline.o \
+	komeda_pipeline_state.o \
 	komeda_framebuffer.o \
 	komeda_kms.o \
 	komeda_crtc.o \
@@ -16,6 +17,7 @@ komeda-y := \
 	komeda_private_obj.o
 
 komeda-y += \
-	d71/d71_dev.o
+	d71/d71_dev.o \
+	d71/d71_component.o
 
 obj-$(CONFIG_DRM_KOMEDA) += komeda.o
diff --git a/drivers/gpu/drm/arm/display/komeda/d71/d71_component.c b/drivers/gpu/drm/arm/display/komeda/d71/d71_component.c
new file mode 100644
index 000000000000..031e5f305a3c
--- /dev/null
+++ b/drivers/gpu/drm/arm/display/komeda/d71/d71_component.c
@@ -0,0 +1,685 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * (C) COPYRIGHT 2018 ARM Limited. All rights reserved.
+ * Author: James.Qian.Wang <james.qian.wang@arm.com>
+ *
+ */
+
+#include <drm/drm_print.h>
+#include "d71_dev.h"
+#include "komeda_kms.h"
+#include "malidp_io.h"
+#include "komeda_framebuffer.h"
+
+static void get_resources_id(u32 hw_id, u32 *pipe_id, u32 *comp_id)
+{
+	u32 id = BLOCK_INFO_BLK_ID(hw_id);
+	u32 pipe = id;
+
+	switch (BLOCK_INFO_BLK_TYPE(hw_id)) {
+	case D71_BLK_TYPE_LPU_WB_LAYER:
+		id = KOMEDA_COMPONENT_WB_LAYER;
+		break;
+	case D71_BLK_TYPE_CU_SPLITTER:
+		id = KOMEDA_COMPONENT_SPLITTER;
+		break;
+	case D71_BLK_TYPE_CU_SCALER:
+		pipe = id / D71_PIPELINE_MAX_SCALERS;
+		id %= D71_PIPELINE_MAX_SCALERS;
+		id += KOMEDA_COMPONENT_SCALER0;
+		break;
+	case D71_BLK_TYPE_CU:
+		id += KOMEDA_COMPONENT_COMPIZ0;
+		break;
+	case D71_BLK_TYPE_LPU_LAYER:
+		pipe = id / D71_PIPELINE_MAX_LAYERS;
+		id %= D71_PIPELINE_MAX_LAYERS;
+		id += KOMEDA_COMPONENT_LAYER0;
+		break;
+	case D71_BLK_TYPE_DOU_IPS:
+		id += KOMEDA_COMPONENT_IPS0;
+		break;
+	case D71_BLK_TYPE_CU_MERGER:
+		id = KOMEDA_COMPONENT_MERGER;
+		break;
+	case D71_BLK_TYPE_DOU:
+		id = KOMEDA_COMPONENT_TIMING_CTRLR;
+		break;
+	default:
+		id = 0xFFFFFFFF;
+	}
+
+	if (comp_id)
+		*comp_id = id;
+
+	if (pipe_id)
+		*pipe_id = pipe;
+}
+
+static u32 get_valid_inputs(struct block_header *blk)
+{
+	u32 valid_inputs = 0, comp_id;
+	int i;
+
+	for (i = 0; i < PIPELINE_INFO_N_VALID_INPUTS(blk->pipeline_info); i++) {
+		get_resources_id(blk->input_ids[i], NULL, &comp_id);
+		if (comp_id == 0xFFFFFFFF)
+			continue;
+		valid_inputs |= BIT(comp_id);
+	}
+
+	return valid_inputs;
+}
+
+static void get_values_from_reg(void __iomem *reg, u32 offset,
+				u32 count, u32 *val)
+{
+	u32 i, addr;
+
+	for (i = 0; i < count; i++) {
+		addr = offset + (i << 2);
+		/* 0xA4 is WO register */
+		if (addr != 0xA4)
+			val[i] = malidp_read32(reg, addr);
+		else
+			val[i] = 0xDEADDEAD;
+	}
+}
+
+static void dump_block_header(struct seq_file *sf, void __iomem *reg)
+{
+	struct block_header hdr;
+	u32 i, n_input, n_output;
+
+	d71_read_block_header(reg, &hdr);
+	seq_printf(sf, "BLOCK_INFO:\t\t0x%X\n", hdr.block_info);
+	seq_printf(sf, "PIPELINE_INFO:\t\t0x%X\n", hdr.pipeline_info);
+
+	n_output = PIPELINE_INFO_N_OUTPUTS(hdr.pipeline_info);
+	n_input  = PIPELINE_INFO_N_VALID_INPUTS(hdr.pipeline_info);
+
+	for (i = 0; i < n_input; i++)
+		seq_printf(sf, "VALID_INPUT_ID%u:\t0x%X\n",
+			   i, hdr.input_ids[i]);
+
+	for (i = 0; i < n_output; i++)
+		seq_printf(sf, "OUTPUT_ID%u:\t\t0x%X\n",
+			   i, hdr.output_ids[i]);
+}
+
+static u32 to_rot_ctrl(u32 rot)
+{
+	u32 lr_ctrl = 0;
+
+	switch (rot & DRM_MODE_ROTATE_MASK) {
+	case DRM_MODE_ROTATE_0:
+		lr_ctrl |= L_ROT(L_ROT_R0);
+		break;
+	case DRM_MODE_ROTATE_90:
+		lr_ctrl |= L_ROT(L_ROT_R90);
+		break;
+	case DRM_MODE_ROTATE_180:
+		lr_ctrl |= L_ROT(L_ROT_R180);
+		break;
+	case DRM_MODE_ROTATE_270:
+		lr_ctrl |= L_ROT(L_ROT_R270);
+		break;
+	}
+
+	if (rot & DRM_MODE_REFLECT_X)
+		lr_ctrl |= L_HFLIP;
+	if (rot & DRM_MODE_REFLECT_Y)
+		lr_ctrl |= L_VFLIP;
+
+	return lr_ctrl;
+}
+
+static inline u32 to_d71_input_id(struct komeda_component_output *output)
+{
+	struct komeda_component *comp = output->component;
+
+	return comp ? (comp->hw_id + output->output_port) : 0;
+}
+
+static void d71_layer_disable(struct komeda_component *c)
+{
+	malidp_write32_mask(c->reg, BLK_CONTROL, L_EN, 0);
+}
+
+static void d71_layer_update(struct komeda_component *c,
+			     struct komeda_component_state *state)
+{
+	struct komeda_layer_state *st = to_layer_st(state);
+	struct drm_plane_state *plane_st = state->plane->state;
+	struct drm_framebuffer *fb = plane_st->fb;
+	struct komeda_fb *kfb = to_kfb(fb);
+	u32 __iomem *reg = c->reg;
+	u32 ctrl_mask = L_EN | L_ROT(L_ROT_R270) | L_HFLIP | L_VFLIP | L_TBU_EN;
+	u32 ctrl = L_EN | to_rot_ctrl(st->rot);
+	int i;
+
+	for (i = 0; i < fb->format->num_planes; i++) {
+		malidp_write32(reg,
+			       BLK_P0_PTR_LOW + i * LAYER_PER_PLANE_REGS * 4,
+			       lower_32_bits(st->addr[i]));
+		malidp_write32(reg,
+			       BLK_P0_PTR_HIGH + i * LAYER_PER_PLANE_REGS * 4,
+			       upper_32_bits(st->addr[i]));
+		if (i >= 2)
+			break;
+
+		malidp_write32(reg,
+			       BLK_P0_STRIDE + i * LAYER_PER_PLANE_REGS * 4,
+			       fb->pitches[i] & 0xFFFF);
+	}
+
+	malidp_write32(reg, LAYER_FMT, kfb->format_caps->hw_id);
+	malidp_write32(reg, BLK_IN_SIZE, HV_SIZE(st->hsize, st->vsize));
+
+	malidp_write32_mask(reg, BLK_CONTROL, ctrl_mask, ctrl);
+}
+
+static void d71_layer_dump(struct komeda_component *c, struct seq_file *sf)
+{
+	u32 v[15], i;
+	bool rich, rgb2rgb;
+	char *prefix;
+
+	get_values_from_reg(c->reg, LAYER_INFO, 1, &v[14]);
+	if (v[14] & 0x1) {
+		rich = true;
+		prefix = "LR_";
+	} else {
+		rich = false;
+		prefix = "LS_";
+	}
+
+	rgb2rgb = !!(v[14] & L_INFO_CM);
+
+	dump_block_header(sf, c->reg);
+
+	seq_printf(sf, "%sLAYER_INFO:\t\t0x%X\n", prefix, v[14]);
+
+	get_values_from_reg(c->reg, 0xD0, 1, v);
+	seq_printf(sf, "%sCONTROL:\t\t0x%X\n", prefix, v[0]);
+	if (rich) {
+		get_values_from_reg(c->reg, 0xD4, 1, v);
+		seq_printf(sf, "LR_RICH_CONTROL:\t0x%X\n", v[0]);
+	}
+	get_values_from_reg(c->reg, 0xD8, 4, v);
+	seq_printf(sf, "%sFORMAT:\t\t0x%X\n", prefix, v[0]);
+	seq_printf(sf, "%sIT_COEFFTAB:\t\t0x%X\n", prefix, v[1]);
+	seq_printf(sf, "%sIN_SIZE:\t\t0x%X\n", prefix, v[2]);
+	seq_printf(sf, "%sPALPHA:\t\t0x%X\n", prefix, v[3]);
+
+	get_values_from_reg(c->reg, 0x100, 3, v);
+	seq_printf(sf, "%sP0_PTR_LOW:\t\t0x%X\n", prefix, v[0]);
+	seq_printf(sf, "%sP0_PTR_HIGH:\t\t0x%X\n", prefix, v[1]);
+	seq_printf(sf, "%sP0_STRIDE:\t\t0x%X\n", prefix, v[2]);
+
+	get_values_from_reg(c->reg, 0x110, 2, v);
+	seq_printf(sf, "%sP1_PTR_LOW:\t\t0x%X\n", prefix, v[0]);
+	seq_printf(sf, "%sP1_PTR_HIGH:\t\t0x%X\n", prefix, v[1]);
+	if (rich) {
+		get_values_from_reg(c->reg, 0x118, 1, v);
+		seq_printf(sf, "LR_P1_STRIDE:\t\t0x%X\n", v[0]);
+
+		get_values_from_reg(c->reg, 0x120, 2, v);
+		seq_printf(sf, "LR_P2_PTR_LOW:\t\t0x%X\n", v[0]);
+		seq_printf(sf, "LR_P2_PTR_HIGH:\t\t0x%X\n", v[1]);
+
+		get_values_from_reg(c->reg, 0x130, 12, v);
+		for (i = 0; i < 12; i++)
+			seq_printf(sf, "LR_YUV_RGB_COEFF%u:\t0x%X\n", i, v[i]);
+	}
+
+	if (rgb2rgb) {
+		get_values_from_reg(c->reg, LAYER_RGB_RGB_COEFF0, 12, v);
+		for (i = 0; i < 12; i++)
+			seq_printf(sf, "LS_RGB_RGB_COEFF%u:\t0x%X\n", i, v[i]);
+	}
+
+	get_values_from_reg(c->reg, 0x160, 3, v);
+	seq_printf(sf, "%sAD_CONTROL:\t\t0x%X\n", prefix, v[0]);
+	seq_printf(sf, "%sAD_H_CROP:\t\t0x%X\n", prefix, v[1]);
+	seq_printf(sf, "%sAD_V_CROP:\t\t0x%X\n", prefix, v[2]);
+}
+
+static struct komeda_component_funcs d71_layer_funcs = {
+	.update		= d71_layer_update,
+	.disable	= d71_layer_disable,
+	.dump_register	= d71_layer_dump,
+};
+
+static int d71_layer_init(struct d71_dev *d71,
+			  struct block_header *blk, u32 __iomem *reg)
+{
+	struct komeda_component *c;
+	struct komeda_layer *layer;
+	u32 pipe_id, layer_id, layer_info;
+
+	get_resources_id(blk->block_info, &pipe_id, &layer_id);
+	c = komeda_component_add(&d71->pipes[pipe_id]->base, sizeof(*layer),
+				 layer_id,
+				 BLOCK_INFO_INPUT_ID(blk->block_info),
+				 &d71_layer_funcs, 0,
+				 get_valid_inputs(blk),
+				 1, reg, "LPU%d_LAYER%d", pipe_id, layer_id);
+	if (IS_ERR(c)) {
+		DRM_ERROR("Failed to add layer component\n");
+		return PTR_ERR(c);
+	}
+
+	layer = to_layer(c);
+	layer_info = malidp_read32(reg, LAYER_INFO);
+
+	if (layer_info & L_INFO_RF)
+		layer->layer_type = KOMEDA_FMT_RICH_LAYER;
+	else
+		layer->layer_type = KOMEDA_FMT_SIMPLE_LAYER;
+
+	set_range(&layer->hsize_in, 4, d71->max_line_size);
+	set_range(&layer->vsize_in, 4, d71->max_vsize);
+
+	malidp_write32(reg, LAYER_PALPHA, D71_PALPHA_DEF_MAP);
+
+	layer->supported_rots = DRM_MODE_ROTATE_MASK | DRM_MODE_REFLECT_MASK;
+
+	return 0;
+}
+
+static int d71_wb_layer_init(struct d71_dev *d71,
+			     struct block_header *blk, u32 __iomem *reg)
+{
+	DRM_DEBUG("Detect D71_Wb_Layer.\n");
+
+	return 0;
+}
+
+static void d71_component_disable(struct komeda_component *c)
+{
+	u32 __iomem *reg = c->reg;
+	u32 i;
+
+	malidp_write32(reg, BLK_CONTROL, 0);
+
+	for (i = 0; i < c->max_active_inputs; i++)
+		malidp_write32(reg, BLK_INPUT_ID0 + (i << 2), 0);
+}
+
+static void compiz_enable_input(u32 __iomem *id_reg,
+				u32 __iomem *cfg_reg,
+				u32 input_hw_id,
+				struct komeda_compiz_input_cfg *cin)
+{
+	u32 ctrl = CU_INPUT_CTRL_EN;
+	u8 blend = cin->pixel_blend_mode;
+
+	if (blend == DRM_MODE_BLEND_PIXEL_NONE)
+		ctrl |= CU_INPUT_CTRL_PAD;
+	else if (blend == DRM_MODE_BLEND_PREMULTI)
+		ctrl |= CU_INPUT_CTRL_PMUL;
+
+	ctrl |= CU_INPUT_CTRL_ALPHA(cin->layer_alpha);
+
+	malidp_write32(id_reg, BLK_INPUT_ID0, input_hw_id);
+
+	malidp_write32(cfg_reg, CU_INPUT0_SIZE,
+		       HV_SIZE(cin->hsize, cin->vsize));
+	malidp_write32(cfg_reg, CU_INPUT0_OFFSET,
+		       HV_OFFSET(cin->hoffset, cin->voffset));
+	malidp_write32(cfg_reg, CU_INPUT0_CONTROL, ctrl);
+}
+
+static void d71_compiz_update(struct komeda_component *c,
+			      struct komeda_component_state *state)
+{
+	struct komeda_compiz_state *st = to_compiz_st(state);
+	u32 __iomem *reg = c->reg;
+	u32 __iomem *id_reg, *cfg_reg;
+	u32 index, input_hw_id;
+
+	for_each_changed_input(state, index) {
+		id_reg = reg + index;
+		cfg_reg = reg + index * CU_PER_INPUT_REGS;
+		input_hw_id = to_d71_input_id(&state->inputs[index]);
+		if (state->active_inputs & BIT(index)) {
+			compiz_enable_input(id_reg, cfg_reg,
+					    input_hw_id, &st->cins[index]);
+		} else {
+			malidp_write32(id_reg, BLK_INPUT_ID0, 0);
+			malidp_write32(cfg_reg, CU_INPUT0_CONTROL, 0);
+		}
+	}
+
+	malidp_write32(reg, BLK_SIZE, HV_SIZE(st->hsize, st->vsize));
+}
+
+static void d71_compiz_dump(struct komeda_component *c, struct seq_file *sf)
+{
+	u32 v[8], i;
+
+	dump_block_header(sf, c->reg);
+
+	get_values_from_reg(c->reg, 0x80, 5, v);
+	for (i = 0; i < 5; i++)
+		seq_printf(sf, "CU_INPUT_ID%u:\t\t0x%X\n", i, v[i]);
+
+	get_values_from_reg(c->reg, 0xA0, 5, v);
+	seq_printf(sf, "CU_IRQ_RAW_STATUS:\t0x%X\n", v[0]);
+	seq_printf(sf, "CU_IRQ_CLEAR:\t\t0x%X\n", v[1]);
+	seq_printf(sf, "CU_IRQ_MASK:\t\t0x%X\n", v[2]);
+	seq_printf(sf, "CU_IRQ_STATUS:\t\t0x%X\n", v[3]);
+	seq_printf(sf, "CU_STATUS:\t\t0x%X\n", v[4]);
+
+	get_values_from_reg(c->reg, 0xD0, 2, v);
+	seq_printf(sf, "CU_CONTROL:\t\t0x%X\n", v[0]);
+	seq_printf(sf, "CU_SIZE:\t\t0x%X\n", v[1]);
+
+	get_values_from_reg(c->reg, 0xDC, 1, v);
+	seq_printf(sf, "CU_BG_COLOR:\t\t0x%X\n", v[0]);
+
+	for (i = 0, v[4] = 0xE0; i < 5; i++, v[4] += 0x10) {
+		get_values_from_reg(c->reg, v[4], 3, v);
+		seq_printf(sf, "CU_INPUT%u_SIZE:\t\t0x%X\n", i, v[0]);
+		seq_printf(sf, "CU_INPUT%u_OFFSET:\t0x%X\n", i, v[1]);
+		seq_printf(sf, "CU_INPUT%u_CONTROL:\t0x%X\n", i, v[2]);
+	}
+
+	get_values_from_reg(c->reg, 0x130, 2, v);
+	seq_printf(sf, "CU_USER_LOW:\t\t0x%X\n", v[0]);
+	seq_printf(sf, "CU_USER_HIGH:\t\t0x%X\n", v[1]);
+}
+
+static struct komeda_component_funcs d71_compiz_funcs = {
+	.update		= d71_compiz_update,
+	.disable	= d71_component_disable,
+	.dump_register	= d71_compiz_dump,
+};
+
+static int d71_compiz_init(struct d71_dev *d71,
+			   struct block_header *blk, u32 __iomem *reg)
+{
+	struct komeda_component *c;
+	struct komeda_compiz *compiz;
+	u32 pipe_id, comp_id;
+
+	get_resources_id(blk->block_info, &pipe_id, &comp_id);
+
+	c = komeda_component_add(&d71->pipes[pipe_id]->base, sizeof(*compiz),
+				 comp_id,
+				 BLOCK_INFO_INPUT_ID(blk->block_info),
+				 &d71_compiz_funcs,
+				 CU_NUM_INPUT_IDS, get_valid_inputs(blk),
+				 CU_NUM_OUTPUT_IDS, reg,
+				 "CU%d", pipe_id);
+	if (IS_ERR(c))
+		return PTR_ERR(c);
+
+	compiz = to_compiz(c);
+
+	set_range(&compiz->hsize, D71_MIN_LINE_SIZE, d71->max_line_size);
+	set_range(&compiz->vsize, D71_MIN_VERTICAL_SIZE, d71->max_vsize);
+
+	return 0;
+}
+
+static void d71_improc_update(struct komeda_component *c,
+			      struct komeda_component_state *state)
+{
+	struct komeda_improc_state *st = to_improc_st(state);
+	u32 __iomem *reg = c->reg;
+	u32 index, input_hw_id;
+
+	for_each_changed_input(state, index) {
+		input_hw_id = state->active_inputs & BIT(index) ?
+			      to_d71_input_id(&state->inputs[index]) : 0;
+		malidp_write32(reg, BLK_INPUT_ID0 + index * 4, input_hw_id);
+	}
+
+	malidp_write32(reg, BLK_SIZE, HV_SIZE(st->hsize, st->vsize));
+}
+
+static void d71_improc_dump(struct komeda_component *c, struct seq_file *sf)
+{
+	u32 v[12], i;
+
+	dump_block_header(sf, c->reg);
+
+	get_values_from_reg(c->reg, 0x80, 2, v);
+	seq_printf(sf, "IPS_INPUT_ID0:\t\t0x%X\n", v[0]);
+	seq_printf(sf, "IPS_INPUT_ID1:\t\t0x%X\n", v[1]);
+
+	get_values_from_reg(c->reg, 0xC0, 1, v);
+	seq_printf(sf, "IPS_INFO:\t\t0x%X\n", v[0]);
+
+	get_values_from_reg(c->reg, 0xD0, 3, v);
+	seq_printf(sf, "IPS_CONTROL:\t\t0x%X\n", v[0]);
+	seq_printf(sf, "IPS_SIZE:\t\t0x%X\n", v[1]);
+	seq_printf(sf, "IPS_DEPTH:\t\t0x%X\n", v[2]);
+
+	get_values_from_reg(c->reg, 0x130, 12, v);
+	for (i = 0; i < 12; i++)
+		seq_printf(sf, "IPS_RGB_RGB_COEFF%u:\t0x%X\n", i, v[i]);
+
+	get_values_from_reg(c->reg, 0x170, 12, v);
+	for (i = 0; i < 12; i++)
+		seq_printf(sf, "IPS_RGB_YUV_COEFF%u:\t0x%X\n", i, v[i]);
+}
+
+static struct komeda_component_funcs d71_improc_funcs = {
+	.update		= d71_improc_update,
+	.disable	= d71_component_disable,
+	.dump_register	= d71_improc_dump,
+};
+
+static int d71_improc_init(struct d71_dev *d71,
+			   struct block_header *blk, u32 __iomem *reg)
+{
+	struct komeda_component *c;
+	struct komeda_improc *improc;
+	u32 pipe_id, comp_id, value;
+
+	get_resources_id(blk->block_info, &pipe_id, &comp_id);
+
+	c = komeda_component_add(&d71->pipes[pipe_id]->base, sizeof(*improc),
+				 comp_id,
+				 BLOCK_INFO_INPUT_ID(blk->block_info),
+				 &d71_improc_funcs, IPS_NUM_INPUT_IDS,
+				 get_valid_inputs(blk),
+				 IPS_NUM_OUTPUT_IDS, reg, "DOU%d_IPS", pipe_id);
+	if (IS_ERR(c)) {
+		DRM_ERROR("Failed to add improc component\n");
+		return PTR_ERR(c);
+	}
+
+	improc = to_improc(c);
+	improc->supported_color_depths = BIT(8) | BIT(10);
+	improc->supported_color_formats = DRM_COLOR_FORMAT_RGB444 |
+					  DRM_COLOR_FORMAT_YCRCB444 |
+					  DRM_COLOR_FORMAT_YCRCB422;
+	value = malidp_read32(reg, BLK_INFO);
+	if (value & IPS_INFO_CHD420)
+		improc->supported_color_formats |= DRM_COLOR_FORMAT_YCRCB420;
+
+	improc->supports_csc = true;
+	improc->supports_gamma = true;
+
+	return 0;
+}
+
+static void d71_timing_ctrlr_disable(struct komeda_component *c)
+{
+	malidp_write32_mask(c->reg, BLK_CONTROL, BS_CTRL_EN, 0);
+}
+
+static void d71_timing_ctrlr_update(struct komeda_component *c,
+				    struct komeda_component_state *state)
+{
+	struct drm_crtc_state *crtc_st = state->crtc->state;
+	u32 __iomem *reg = c->reg;
+	struct videomode vm;
+	u32 value;
+
+	drm_display_mode_to_videomode(&crtc_st->adjusted_mode, &vm);
+
+	malidp_write32(reg, BS_ACTIVESIZE, HV_SIZE(vm.hactive, vm.vactive));
+	malidp_write32(reg, BS_HINTERVALS, BS_H_INTVALS(vm.hfront_porch,
+							vm.hback_porch));
+	malidp_write32(reg, BS_VINTERVALS, BS_V_INTVALS(vm.vfront_porch,
+							vm.vback_porch));
+
+	value = BS_SYNC_VSW(vm.vsync_len) | BS_SYNC_HSW(vm.hsync_len);
+	value |= vm.flags & DISPLAY_FLAGS_VSYNC_HIGH ? BS_SYNC_VSP : 0;
+	value |= vm.flags & DISPLAY_FLAGS_HSYNC_HIGH ? BS_SYNC_HSP : 0;
+	malidp_write32(reg, BS_SYNC, value);
+
+	malidp_write32(reg, BS_PROG_LINE, D71_DEFAULT_PREPRETCH_LINE - 1);
+	malidp_write32(reg, BS_PREFETCH_LINE, D71_DEFAULT_PREPRETCH_LINE);
+
+	/* configure bs control register */
+	value = BS_CTRL_EN | BS_CTRL_VM;
+
+	malidp_write32(reg, BLK_CONTROL, value);
+}
+
+static void d71_timing_ctrlr_dump(struct komeda_component *c,
+				  struct seq_file *sf)
+{
+	u32 v[8], i;
+
+	dump_block_header(sf, c->reg);
+
+	get_values_from_reg(c->reg, 0xC0, 1, v);
+	seq_printf(sf, "BS_INFO:\t\t0x%X\n", v[0]);
+
+	get_values_from_reg(c->reg, 0xD0, 8, v);
+	seq_printf(sf, "BS_CONTROL:\t\t0x%X\n", v[0]);
+	seq_printf(sf, "BS_PROG_LINE:\t\t0x%X\n", v[1]);
+	seq_printf(sf, "BS_PREFETCH_LINE:\t0x%X\n", v[2]);
+	seq_printf(sf, "BS_BG_COLOR:\t\t0x%X\n", v[3]);
+	seq_printf(sf, "BS_ACTIVESIZE:\t\t0x%X\n", v[4]);
+	seq_printf(sf, "BS_HINTERVALS:\t\t0x%X\n", v[5]);
+	seq_printf(sf, "BS_VINTERVALS:\t\t0x%X\n", v[6]);
+	seq_printf(sf, "BS_SYNC:\t\t0x%X\n", v[7]);
+
+	get_values_from_reg(c->reg, 0x100, 3, v);
+	seq_printf(sf, "BS_DRIFT_TO:\t\t0x%X\n", v[0]);
+	seq_printf(sf, "BS_FRAME_TO:\t\t0x%X\n", v[1]);
+	seq_printf(sf, "BS_TE_TO:\t\t0x%X\n", v[2]);
+
+	get_values_from_reg(c->reg, 0x110, 3, v);
+	for (i = 0; i < 3; i++)
+		seq_printf(sf, "BS_T%u_INTERVAL:\t\t0x%X\n", i, v[i]);
+
+	get_values_from_reg(c->reg, 0x120, 5, v);
+	for (i = 0; i < 2; i++) {
+		seq_printf(sf, "BS_CRC%u_LOW:\t\t0x%X\n", i, v[i << 1]);
+		seq_printf(sf, "BS_CRC%u_HIGH:\t\t0x%X\n", i, v[(i << 1) + 1]);
+	}
+	seq_printf(sf, "BS_USER:\t\t0x%X\n", v[4]);
+}
+
+static struct komeda_component_funcs d71_timing_ctrlr_funcs = {
+	.update		= d71_timing_ctrlr_update,
+	.disable	= d71_timing_ctrlr_disable,
+	.dump_register	= d71_timing_ctrlr_dump,
+};
+
+static int d71_timing_ctrlr_init(struct d71_dev *d71,
+				 struct block_header *blk, u32 __iomem *reg)
+{
+	struct komeda_component *c;
+	struct komeda_timing_ctrlr *ctrlr;
+	u32 pipe_id, comp_id;
+
+	get_resources_id(blk->block_info, &pipe_id, &comp_id);
+
+	c = komeda_component_add(&d71->pipes[pipe_id]->base, sizeof(*ctrlr),
+				 KOMEDA_COMPONENT_TIMING_CTRLR,
+				 BLOCK_INFO_INPUT_ID(blk->block_info),
+				 &d71_timing_ctrlr_funcs,
+				 1, BIT(KOMEDA_COMPONENT_IPS0 + pipe_id),
+				 BS_NUM_OUTPUT_IDS, reg, "DOU%d_BS", pipe_id);
+	if (IS_ERR(c)) {
+		DRM_ERROR("Failed to add display_ctrl component\n");
+		return PTR_ERR(c);
+	}
+
+	ctrlr = to_ctrlr(c);
+
+	ctrlr->supports_dual_link = true;
+
+	return 0;
+}
+
+int d71_probe_block(struct d71_dev *d71,
+		    struct block_header *blk, u32 __iomem *reg)
+{
+	struct d71_pipeline *pipe;
+	int blk_id = BLOCK_INFO_BLK_ID(blk->block_info);
+
+	int err = 0;
+
+	switch (BLOCK_INFO_BLK_TYPE(blk->block_info)) {
+	case D71_BLK_TYPE_GCU:
+		break;
+
+	case D71_BLK_TYPE_LPU:
+		pipe = d71->pipes[blk_id];
+		pipe->lpu_addr = reg;
+		break;
+
+	case D71_BLK_TYPE_LPU_LAYER:
+		err = d71_layer_init(d71, blk, reg);
+		break;
+
+	case D71_BLK_TYPE_LPU_WB_LAYER:
+		err = d71_wb_layer_init(d71, blk, reg);
+		break;
+
+	case D71_BLK_TYPE_CU:
+		pipe = d71->pipes[blk_id];
+		pipe->cu_addr = reg;
+		err = d71_compiz_init(d71, blk, reg);
+		break;
+
+	case D71_BLK_TYPE_CU_SPLITTER:
+	case D71_BLK_TYPE_CU_SCALER:
+	case D71_BLK_TYPE_CU_MERGER:
+		break;
+
+	case D71_BLK_TYPE_DOU:
+		pipe = d71->pipes[blk_id];
+		pipe->dou_addr = reg;
+		break;
+
+	case D71_BLK_TYPE_DOU_IPS:
+		err = d71_improc_init(d71, blk, reg);
+		break;
+
+	case D71_BLK_TYPE_DOU_FT_COEFF:
+		pipe = d71->pipes[blk_id];
+		pipe->dou_ft_coeff_addr = reg;
+		break;
+
+	case D71_BLK_TYPE_DOU_BS:
+		err = d71_timing_ctrlr_init(d71, blk, reg);
+		break;
+
+	case D71_BLK_TYPE_GLB_LT_COEFF:
+		break;
+
+	case D71_BLK_TYPE_GLB_SCL_COEFF:
+		d71->glb_scl_coeff_addr[blk_id] = reg;
+		break;
+
+	default:
+		DRM_ERROR("Unknown block (block_info: 0x%x) is found\n",
+			  blk->block_info);
+		err = -EINVAL;
+		break;
+	}
+
+	return err;
+}
diff --git a/drivers/gpu/drm/arm/display/komeda/d71/d71_dev.c b/drivers/gpu/drm/arm/display/komeda/d71/d71_dev.c
index edbf9daa1545..34506ef7ad40 100644
--- a/drivers/gpu/drm/arm/display/komeda/d71/d71_dev.c
+++ b/drivers/gpu/drm/arm/display/komeda/d71/d71_dev.c
@@ -4,13 +4,425 @@
  * Author: James.Qian.Wang <james.qian.wang@arm.com>
  *
  */
+
+#include <drm/drm_print.h>
+#include "d71_dev.h"
 #include "malidp_io.h"
-#include "komeda_dev.h"
+
+static u64 get_lpu_event(struct d71_pipeline *d71_pipeline)
+{
+	u32 __iomem *reg = d71_pipeline->lpu_addr;
+	u32 status, raw_status;
+	u64 evts = 0ULL;
+
+	raw_status = malidp_read32(reg, BLK_IRQ_RAW_STATUS);
+	if (raw_status & LPU_IRQ_IBSY)
+		evts |= KOMEDA_EVENT_IBSY;
+	if (raw_status & LPU_IRQ_EOW)
+		evts |= KOMEDA_EVENT_EOW;
+
+	if (raw_status & (LPU_IRQ_ERR | LPU_IRQ_IBSY)) {
+		u32 restore = 0, tbu_status;
+		/* Check error of LPU status */
+		status = malidp_read32(reg, BLK_STATUS);
+		if (status & LPU_STATUS_AXIE) {
+			restore |= LPU_STATUS_AXIE;
+			evts |= KOMEDA_ERR_AXIE;
+		}
+		if (status & LPU_STATUS_ACE0) {
+			restore |= LPU_STATUS_ACE0;
+			evts |= KOMEDA_ERR_ACE0;
+		}
+		if (status & LPU_STATUS_ACE1) {
+			restore |= LPU_STATUS_ACE1;
+			evts |= KOMEDA_ERR_ACE1;
+		}
+		if (status & LPU_STATUS_ACE2) {
+			restore |= LPU_STATUS_ACE2;
+			evts |= KOMEDA_ERR_ACE2;
+		}
+		if (status & LPU_STATUS_ACE3) {
+			restore |= LPU_STATUS_ACE3;
+			evts |= KOMEDA_ERR_ACE3;
+		}
+		if (restore != 0)
+			malidp_write32_mask(reg, BLK_STATUS, restore, 0);
+
+		restore = 0;
+		/* Check errors of TBU status */
+		tbu_status = malidp_read32(reg, LPU_TBU_STATUS);
+		if (tbu_status & LPU_TBU_STATUS_TCF) {
+			restore |= LPU_TBU_STATUS_TCF;
+			evts |= KOMEDA_ERR_TCF;
+		}
+		if (tbu_status & LPU_TBU_STATUS_TTNG) {
+			restore |= LPU_TBU_STATUS_TTNG;
+			evts |= KOMEDA_ERR_TTNG;
+		}
+		if (tbu_status & LPU_TBU_STATUS_TITR) {
+			restore |= LPU_TBU_STATUS_TITR;
+			evts |= KOMEDA_ERR_TITR;
+		}
+		if (tbu_status & LPU_TBU_STATUS_TEMR) {
+			restore |= LPU_TBU_STATUS_TEMR;
+			evts |= KOMEDA_ERR_TEMR;
+		}
+		if (tbu_status & LPU_TBU_STATUS_TTF) {
+			restore |= LPU_TBU_STATUS_TTF;
+			evts |= KOMEDA_ERR_TTF;
+		}
+		if (restore != 0)
+			malidp_write32_mask(reg, LPU_TBU_STATUS, restore, 0);
+	}
+
+	malidp_write32(reg, BLK_IRQ_CLEAR, raw_status);
+	return evts;
+}
+
+static u64 get_cu_event(struct d71_pipeline *d71_pipeline)
+{
+	u32 __iomem *reg = d71_pipeline->cu_addr;
+	u32 status, raw_status;
+	u64 evts = 0ULL;
+
+	raw_status = malidp_read32(reg, BLK_IRQ_RAW_STATUS);
+	if (raw_status & CU_IRQ_OVR)
+		evts |= KOMEDA_EVENT_OVR;
+
+	if (raw_status & (CU_IRQ_ERR | CU_IRQ_OVR)) {
+		status = malidp_read32(reg, BLK_STATUS) & 0x7FFFFFFF;
+		if (status & CU_STATUS_CPE)
+			evts |= KOMEDA_ERR_CPE;
+		if (status & CU_STATUS_ZME)
+			evts |= KOMEDA_ERR_ZME;
+		if (status & CU_STATUS_CFGE)
+			evts |= KOMEDA_ERR_CFGE;
+		if (status)
+			malidp_write32_mask(reg, BLK_STATUS, status, 0);
+	}
+
+	malidp_write32(reg, BLK_IRQ_CLEAR, raw_status);
+
+	return evts;
+}
+
+static u64 get_dou_event(struct d71_pipeline *d71_pipeline)
+{
+	u32 __iomem *reg = d71_pipeline->dou_addr;
+	u32 status, raw_status;
+	u64 evts = 0ULL;
+
+	raw_status = malidp_read32(reg, BLK_IRQ_RAW_STATUS);
+	if (raw_status & DOU_IRQ_PL0)
+		evts |= KOMEDA_EVENT_VSYNC;
+	if (raw_status & DOU_IRQ_UND)
+		evts |= KOMEDA_EVENT_URUN;
+
+	if (raw_status & (DOU_IRQ_ERR | DOU_IRQ_UND)) {
+		u32 restore  = 0;
+
+		status = malidp_read32(reg, BLK_STATUS);
+		if (status & DOU_STATUS_DRIFTTO) {
+			restore |= DOU_STATUS_DRIFTTO;
+			evts |= KOMEDA_ERR_DRIFTTO;
+		}
+		if (status & DOU_STATUS_FRAMETO) {
+			restore |= DOU_STATUS_FRAMETO;
+			evts |= KOMEDA_ERR_FRAMETO;
+		}
+		if (status & DOU_STATUS_TETO) {
+			restore |= DOU_STATUS_TETO;
+			evts |= KOMEDA_ERR_TETO;
+		}
+		if (status & DOU_STATUS_CSCE) {
+			restore |= DOU_STATUS_CSCE;
+			evts |= KOMEDA_ERR_CSCE;
+		}
+
+		if (restore != 0)
+			malidp_write32_mask(reg, BLK_STATUS, restore, 0);
+	}
+
+	malidp_write32(reg, BLK_IRQ_CLEAR, raw_status);
+	return evts;
+}
+
+static u64 get_pipeline_event(struct d71_pipeline *d71_pipeline, u32 gcu_status)
+{
+	u32 evts = 0ULL;
+
+	if (gcu_status & (GLB_IRQ_STATUS_LPU0 | GLB_IRQ_STATUS_LPU1))
+		evts |= get_lpu_event(d71_pipeline);
+
+	if (gcu_status & (GLB_IRQ_STATUS_CU0 | GLB_IRQ_STATUS_CU1))
+		evts |= get_cu_event(d71_pipeline);
+
+	if (gcu_status & (GLB_IRQ_STATUS_DOU0 | GLB_IRQ_STATUS_DOU1))
+		evts |= get_dou_event(d71_pipeline);
+
+	return evts;
+}
+
+static irqreturn_t
+d71_irq_handler(struct komeda_dev *mdev, struct komeda_events *evts)
+{
+	struct d71_dev *d71 = mdev->chip_data;
+	u32 status, gcu_status, raw_status;
+
+	gcu_status = malidp_read32(d71->gcu_addr, GLB_IRQ_STATUS);
+
+	if (gcu_status & GLB_IRQ_STATUS_GCU) {
+		raw_status = malidp_read32(d71->gcu_addr, BLK_IRQ_RAW_STATUS);
+		if (raw_status & GCU_IRQ_CVAL0)
+			evts->pipes[0] |= KOMEDA_EVENT_FLIP;
+		if (raw_status & GCU_IRQ_CVAL1)
+			evts->pipes[1] |= KOMEDA_EVENT_FLIP;
+		if (raw_status & GCU_IRQ_ERR) {
+			status = malidp_read32(d71->gcu_addr, BLK_STATUS);
+			if (status & GCU_STATUS_MERR) {
+				evts->global |= KOMEDA_ERR_MERR;
+				malidp_write32_mask(d71->gcu_addr, BLK_STATUS,
+						    GCU_STATUS_MERR, 0);
+			}
+		}
+
+		malidp_write32(d71->gcu_addr, BLK_IRQ_CLEAR, raw_status);
+	}
+
+	if (gcu_status & GLB_IRQ_STATUS_PIPE0)
+		evts->pipes[0] |= get_pipeline_event(d71->pipes[0], gcu_status);
+
+	if (gcu_status & GLB_IRQ_STATUS_PIPE1)
+		evts->pipes[1] |= get_pipeline_event(d71->pipes[1], gcu_status);
+
+	return gcu_status ? IRQ_HANDLED : IRQ_NONE;
+}
+
+#define ENABLED_GCU_IRQS	(GCU_IRQ_CVAL0 | GCU_IRQ_CVAL1 | \
+				 GCU_IRQ_MODE | GCU_IRQ_ERR)
+#define ENABLED_LPU_IRQS	(LPU_IRQ_IBSY | LPU_IRQ_ERR | LPU_IRQ_EOW)
+#define ENABLED_CU_IRQS		(CU_IRQ_OVR | CU_IRQ_ERR)
+#define ENABLED_DOU_IRQS	(DOU_IRQ_UND | DOU_IRQ_ERR)
+
+static int d71_enable_irq(struct komeda_dev *mdev)
+{
+	struct d71_dev *d71 = mdev->chip_data;
+	struct d71_pipeline *pipe;
+	u32 i;
+
+	malidp_write32_mask(d71->gcu_addr, BLK_IRQ_MASK,
+			    ENABLED_GCU_IRQS, ENABLED_GCU_IRQS);
+	for (i = 0; i < d71->num_pipelines; i++) {
+		pipe = d71->pipes[i];
+		malidp_write32_mask(pipe->cu_addr,  BLK_IRQ_MASK,
+				    ENABLED_CU_IRQS, ENABLED_CU_IRQS);
+		malidp_write32_mask(pipe->lpu_addr, BLK_IRQ_MASK,
+				    ENABLED_LPU_IRQS, ENABLED_LPU_IRQS);
+		malidp_write32_mask(pipe->dou_addr, BLK_IRQ_MASK,
+				    ENABLED_DOU_IRQS, ENABLED_DOU_IRQS);
+	}
+	return 0;
+}
+
+static int d71_disable_irq(struct komeda_dev *mdev)
+{
+	struct d71_dev *d71 = mdev->chip_data;
+	struct d71_pipeline *pipe;
+	u32 i;
+
+	malidp_write32_mask(d71->gcu_addr, BLK_IRQ_MASK, ENABLED_GCU_IRQS, 0);
+	for (i = 0; i < d71->num_pipelines; i++) {
+		pipe = d71->pipes[i];
+		malidp_write32_mask(pipe->cu_addr,  BLK_IRQ_MASK,
+				    ENABLED_CU_IRQS, 0);
+		malidp_write32_mask(pipe->lpu_addr, BLK_IRQ_MASK,
+				    ENABLED_LPU_IRQS, 0);
+		malidp_write32_mask(pipe->dou_addr, BLK_IRQ_MASK,
+				    ENABLED_DOU_IRQS, 0);
+	}
+	return 0;
+}
+
+static void d71_on_off_vblank(struct komeda_dev *mdev, int master_pipe, bool on)
+{
+	struct d71_dev *d71 = mdev->chip_data;
+	struct d71_pipeline *pipe = d71->pipes[master_pipe];
+
+	malidp_write32_mask(pipe->dou_addr, BLK_IRQ_MASK,
+			    DOU_IRQ_PL0, on ? DOU_IRQ_PL0 : 0);
+}
+
+static int to_d71_opmode(int core_mode)
+{
+	switch (core_mode) {
+	case KOMEDA_MODE_DISP0:
+		return DO0_ACTIVE_MODE;
+	case KOMEDA_MODE_DISP1:
+		return DO1_ACTIVE_MODE;
+	case KOMEDA_MODE_DUAL_DISP:
+		return DO01_ACTIVE_MODE;
+	case KOMEDA_MODE_INACTIVE:
+		return INACTIVE_MODE;
+	default:
+		WARN(1, "Unknown operation mode");
+		return INACTIVE_MODE;
+	}
+}
+
+static int d71_change_opmode(struct komeda_dev *mdev, int new_mode)
+{
+	struct d71_dev *d71 = mdev->chip_data;
+	u32 opmode = to_d71_opmode(new_mode);
+	int ret;
+
+	malidp_write32_mask(d71->gcu_addr, BLK_CONTROL, 0x7, opmode);
+
+	ret = dp_wait_cond(((malidp_read32(d71->gcu_addr, BLK_CONTROL) & 0x7) == opmode),
+			   100, 1000, 10000);
+
+	return ret > 0 ? 0 : -ETIMEDOUT;
+}
+
+static void d71_flush(struct komeda_dev *mdev,
+		      int master_pipe, u32 active_pipes)
+{
+	struct d71_dev *d71 = mdev->chip_data;
+	u32 reg_offset = (master_pipe == 0) ?
+			 GCU_CONFIG_VALID0 : GCU_CONFIG_VALID1;
+
+	malidp_write32(d71->gcu_addr, reg_offset, GCU_CONFIG_CVAL);
+}
+
+static int d71_reset(struct d71_dev *d71)
+{
+	u32 __iomem *gcu = d71->gcu_addr;
+	int ret;
+
+	malidp_write32_mask(gcu, BLK_CONTROL,
+			    GCU_CONTROL_SRST, GCU_CONTROL_SRST);
+
+	ret = dp_wait_cond(!(malidp_read32(gcu, BLK_CONTROL) & GCU_CONTROL_SRST),
+			   100, 1000, 10000);
+
+	return ret > 0 ? 0 : -ETIMEDOUT;
+}
+
+void d71_read_block_header(u32 __iomem *reg, struct block_header *blk)
+{
+	int i;
+
+	blk->block_info = malidp_read32(reg, BLK_BLOCK_INFO);
+	if (BLOCK_INFO_BLK_TYPE(blk->block_info) == D71_BLK_TYPE_RESERVED)
+		return;
+
+	blk->pipeline_info = malidp_read32(reg, BLK_PIPELINE_INFO);
+
+	/* get valid input and output ids */
+	for (i = 0; i < PIPELINE_INFO_N_VALID_INPUTS(blk->pipeline_info); i++)
+		blk->input_ids[i] = malidp_read32(reg + i, BLK_VALID_INPUT_ID0);
+	for (i = 0; i < PIPELINE_INFO_N_OUTPUTS(blk->pipeline_info); i++)
+		blk->output_ids[i] = malidp_read32(reg + i, BLK_OUTPUT_ID0);
+}
+
+static void d71_cleanup(struct komeda_dev *mdev)
+{
+	struct d71_dev *d71 = mdev->chip_data;
+
+	if (!d71)
+		return;
+
+	devm_kfree(mdev->dev, d71);
+	mdev->chip_data = NULL;
+}
 
 static int d71_enum_resources(struct komeda_dev *mdev)
 {
-	/* TODO add enum resources */
-	return -1;
+	struct d71_dev *d71;
+	struct komeda_pipeline *pipe;
+	struct block_header blk;
+	u32 __iomem *blk_base;
+	u32 i, value, offset;
+	int err;
+
+	d71 = devm_kzalloc(mdev->dev, sizeof(*d71), GFP_KERNEL);
+	if (!d71)
+		return -ENOMEM;
+
+	mdev->chip_data = d71;
+	d71->mdev = mdev;
+	d71->gcu_addr = mdev->reg_base;
+	d71->periph_addr = mdev->reg_base + (D71_BLOCK_OFFSET_PERIPH >> 2);
+
+	err = d71_reset(d71);
+	if (err) {
+		DRM_ERROR("Fail to reset d71 device.\n");
+		goto err_cleanup;
+	}
+
+	/* probe GCU */
+	value = malidp_read32(d71->gcu_addr, GLB_CORE_INFO);
+	d71->num_blocks = value & 0xFF;
+	d71->num_pipelines = (value >> 8) & 0x7;
+
+	if (d71->num_pipelines > D71_MAX_PIPELINE) {
+		DRM_ERROR("d71 supports %d pipelines, but got: %d.\n",
+			  D71_MAX_PIPELINE, d71->num_pipelines);
+		err = -EINVAL;
+		goto err_cleanup;
+	}
+
+	/* probe PERIPH */
+	value = malidp_read32(d71->periph_addr, BLK_BLOCK_INFO);
+	if (BLOCK_INFO_BLK_TYPE(value) != D71_BLK_TYPE_PERIPH) {
+		DRM_ERROR("access blk periph but got blk: %d.\n",
+			  BLOCK_INFO_BLK_TYPE(value));
+		err = -EINVAL;
+		goto err_cleanup;
+	}
+
+	value = malidp_read32(d71->periph_addr, PERIPH_CONFIGURATION_ID);
+
+	d71->max_line_size	= value & PERIPH_MAX_LINE_SIZE ? 4096 : 2048;
+	d71->max_vsize		= 4096;
+	d71->num_rich_layers	= value & PERIPH_NUM_RICH_LAYERS ? 2 : 1;
+	d71->supports_dual_link	= value & PERIPH_SPLIT_EN ? true : false;
+	d71->integrates_tbu	= value & PERIPH_TBU_EN ? true : false;
+
+	for (i = 0; i < d71->num_pipelines; i++) {
+		pipe = komeda_pipeline_add(mdev, sizeof(struct d71_pipeline),
+					   NULL);
+		if (IS_ERR(pipe)) {
+			err = PTR_ERR(pipe);
+			goto err_cleanup;
+		}
+		d71->pipes[i] = to_d71_pipeline(pipe);
+	}
+
+	/* loop the register blks and probe */
+	i = 2; /* exclude GCU and PERIPH */
+	offset = D71_BLOCK_SIZE; /* skip GCU */
+	while (i < d71->num_blocks) {
+		blk_base = mdev->reg_base + (offset >> 2);
+
+		d71_read_block_header(blk_base, &blk);
+		if (BLOCK_INFO_BLK_TYPE(blk.block_info) != D71_BLK_TYPE_RESERVED) {
+			err = d71_probe_block(d71, &blk, blk_base);
+			if (err)
+				goto err_cleanup;
+			i++;
+		}
+
+		offset += D71_BLOCK_SIZE;
+	}
+
+	DRM_DEBUG("total %d (out of %d) blocks are found.\n",
+		  i, d71->num_blocks);
+
+	return 0;
+
+err_cleanup:
+	d71_cleanup(mdev);
+	return err;
 }
 
 #define __HW_ID(__group, __format) \
@@ -93,19 +505,22 @@ static void d71_init_fmt_tbl(struct komeda_dev *mdev)
 static struct komeda_dev_funcs d71_chip_funcs = {
 	.init_format_table = d71_init_fmt_tbl,
 	.enum_resources	= d71_enum_resources,
-	.cleanup	= NULL,
+	.cleanup	= d71_cleanup,
+	.irq_handler	= d71_irq_handler,
+	.enable_irq	= d71_enable_irq,
+	.disable_irq	= d71_disable_irq,
+	.on_off_vblank	= d71_on_off_vblank,
+	.change_opmode	= d71_change_opmode,
+	.flush		= d71_flush,
 };
 
-#define GLB_ARCH_ID		0x000
-#define GLB_CORE_ID		0x004
-#define GLB_CORE_INFO		0x008
-
 struct komeda_dev_funcs *
 d71_identify(u32 __iomem *reg_base, struct komeda_chip_info *chip)
 {
 	chip->arch_id	= malidp_read32(reg_base, GLB_ARCH_ID);
 	chip->core_id	= malidp_read32(reg_base, GLB_CORE_ID);
 	chip->core_info	= malidp_read32(reg_base, GLB_CORE_INFO);
+	chip->bus_width	= D71_BUS_WIDTH_16_BYTES;
 
 	return &d71_chip_funcs;
 }
diff --git a/drivers/gpu/drm/arm/display/komeda/d71/d71_dev.h b/drivers/gpu/drm/arm/display/komeda/d71/d71_dev.h
new file mode 100644
index 000000000000..7465c57d9774
--- /dev/null
+++ b/drivers/gpu/drm/arm/display/komeda/d71/d71_dev.h
@@ -0,0 +1,50 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * (C) COPYRIGHT 2018 ARM Limited. All rights reserved.
+ * Author: James.Qian.Wang <james.qian.wang@arm.com>
+ *
+ */
+#ifndef _D71_DEV_H_
+#define _D71_DEV_H_
+
+#include "komeda_dev.h"
+#include "komeda_pipeline.h"
+#include "d71_regs.h"
+
+struct d71_pipeline {
+	struct komeda_pipeline base;
+
+	/* d71 private pipeline blocks */
+	u32 __iomem	*lpu_addr;
+	u32 __iomem	*cu_addr;
+	u32 __iomem	*dou_addr;
+	u32 __iomem	*dou_ft_coeff_addr; /* forward transform coeffs table */
+};
+
+struct d71_dev {
+	struct komeda_dev *mdev;
+
+	int	num_blocks;
+	int	num_pipelines;
+	int	num_rich_layers;
+	u32	max_line_size;
+	u32	max_vsize;
+	u32	supports_dual_link : 1;
+	u32	integrates_tbu : 1;
+
+	/* global register blocks */
+	u32 __iomem	*gcu_addr;
+	/* scaling coeffs table */
+	u32 __iomem	*glb_scl_coeff_addr[D71_MAX_GLB_SCL_COEFF];
+	u32 __iomem	*periph_addr;
+
+	struct d71_pipeline *pipes[D71_MAX_PIPELINE];
+};
+
+#define to_d71_pipeline(x)	container_of(x, struct d71_pipeline, base)
+
+int d71_probe_block(struct d71_dev *d71,
+		    struct block_header *blk, u32 __iomem *reg);
+void d71_read_block_header(u32 __iomem *reg, struct block_header *blk);
+
+#endif /* !_D71_DEV_H_ */
diff --git a/drivers/gpu/drm/arm/display/komeda/d71/d71_regs.h b/drivers/gpu/drm/arm/display/komeda/d71/d71_regs.h
new file mode 100644
index 000000000000..2d5e6d00b42c
--- /dev/null
+++ b/drivers/gpu/drm/arm/display/komeda/d71/d71_regs.h
@@ -0,0 +1,530 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * (C) COPYRIGHT 2018 ARM Limited. All rights reserved.
+ * Author: James.Qian.Wang <james.qian.wang@arm.com>
+ *
+ */
+#ifndef _D71_REG_H_
+#define _D71_REG_H_
+
+/* Common block registers offset */
+#define BLK_BLOCK_INFO		0x000
+#define BLK_PIPELINE_INFO	0x004
+#define BLK_VALID_INPUT_ID0	0x020
+#define BLK_OUTPUT_ID0		0x060
+#define BLK_INPUT_ID0		0x080
+#define BLK_IRQ_RAW_STATUS	0x0A0
+#define BLK_IRQ_CLEAR		0x0A4
+#define BLK_IRQ_MASK		0x0A8
+#define BLK_IRQ_STATUS		0x0AC
+#define BLK_STATUS		0x0B0
+#define BLK_INFO		0x0C0
+#define BLK_CONTROL		0x0D0
+#define BLK_SIZE		0x0D4
+#define BLK_IN_SIZE		0x0E0
+
+#define BLK_P0_PTR_LOW		0x100
+#define BLK_P0_PTR_HIGH		0x104
+#define BLK_P0_STRIDE		0x108
+#define BLK_P1_PTR_LOW		0x110
+#define BLK_P1_PTR_HIGH		0x114
+#define BLK_P1_STRIDE		0x118
+#define BLK_P2_PTR_LOW		0x120
+#define BLK_P2_PTR_HIGH		0x124
+
+#define BLOCK_INFO_N_SUBBLKS(x)	((x) & 0x000F)
+#define BLOCK_INFO_BLK_ID(x)	(((x) & 0x00F0) >> 4)
+#define BLOCK_INFO_BLK_TYPE(x)	(((x) & 0xFF00) >> 8)
+#define BLOCK_INFO_INPUT_ID(x)	((x) & 0xFFF0)
+#define BLOCK_INFO_TYPE_ID(x)	(((x) & 0x0FF0) >> 4)
+
+#define PIPELINE_INFO_N_OUTPUTS(x)	((x) & 0x000F)
+#define PIPELINE_INFO_N_VALID_INPUTS(x)	(((x) & 0x0F00) >> 8)
+
+/* Common block control register bits */
+#define BLK_CTRL_EN		BIT(0)
+/* Common size macro */
+#define HV_SIZE(h, v)		(((h) & 0x1FFF) + (((v) & 0x1FFF) << 16))
+#define HV_OFFSET(h, v)		(((h) & 0xFFF) + (((v) & 0xFFF) << 16))
+#define HV_CROP(h, v)		(((h) & 0xFFF) + (((v) & 0xFFF) << 16))
+
+/* AD_CONTROL register */
+#define AD_CONTROL		0x160
+
+/* AD_CONTROL register bits */
+#define AD_AEN			BIT(0)
+#define AD_YT			BIT(1)
+#define AD_BS			BIT(2)
+#define AD_WB			BIT(3)
+#define AD_TH			BIT(4)
+
+/* Global Control Unit */
+#define GLB_ARCH_ID		0x000
+#define GLB_CORE_ID		0x004
+#define GLB_CORE_INFO		0x008
+#define GLB_IRQ_STATUS		0x010
+
+#define GCU_CONFIG_VALID0	0x0D4
+#define GCU_CONFIG_VALID1	0x0D8
+
+/* GCU_CONTROL_BITS */
+#define GCU_CONTROL_MODE(x)	((x) & 0x7)
+#define GCU_CONTROL_SRST	BIT(16)
+
+/* GCU opmode */
+#define INACTIVE_MODE		0
+#define TBU_CONNECT_MODE	1
+#define TBU_DISCONNECT_MODE	2
+#define DO0_ACTIVE_MODE		3
+#define DO1_ACTIVE_MODE		4
+#define DO01_ACTIVE_MODE	5
+
+/* GLB_IRQ_STATUS bits */
+#define GLB_IRQ_STATUS_GCU	BIT(0)
+#define GLB_IRQ_STATUS_LPU0	BIT(8)
+#define GLB_IRQ_STATUS_LPU1	BIT(9)
+#define GLB_IRQ_STATUS_ATU0	BIT(10)
+#define GLB_IRQ_STATUS_ATU1	BIT(11)
+#define GLB_IRQ_STATUS_ATU2	BIT(12)
+#define GLB_IRQ_STATUS_ATU3	BIT(13)
+#define GLB_IRQ_STATUS_CU0	BIT(16)
+#define GLB_IRQ_STATUS_CU1	BIT(17)
+#define GLB_IRQ_STATUS_DOU0	BIT(24)
+#define GLB_IRQ_STATUS_DOU1	BIT(25)
+
+#define GLB_IRQ_STATUS_PIPE0	(GLB_IRQ_STATUS_LPU0 |\
+				 GLB_IRQ_STATUS_ATU0 |\
+				 GLB_IRQ_STATUS_ATU1 |\
+				 GLB_IRQ_STATUS_CU0 |\
+				 GLB_IRQ_STATUS_DOU0)
+
+#define GLB_IRQ_STATUS_PIPE1	(GLB_IRQ_STATUS_LPU1 |\
+				 GLB_IRQ_STATUS_ATU2 |\
+				 GLB_IRQ_STATUS_ATU3 |\
+				 GLB_IRQ_STATUS_CU1 |\
+				 GLB_IRQ_STATUS_DOU1)
+
+#define GLB_IRQ_STATUS_ATU	(GLB_IRQ_STATUS_ATU0 |\
+				 GLB_IRQ_STATUS_ATU1 |\
+				 GLB_IRQ_STATUS_ATU2 |\
+				 GLB_IRQ_STATUS_ATU3)
+
+/* GCU_IRQ_BITS */
+#define GCU_IRQ_CVAL0		BIT(0)
+#define GCU_IRQ_CVAL1		BIT(1)
+#define GCU_IRQ_MODE		BIT(4)
+#define GCU_IRQ_ERR		BIT(11)
+
+/* GCU_STATUS_BITS */
+#define GCU_STATUS_MODE(x)	((x) & 0x7)
+#define GCU_STATUS_MERR		BIT(4)
+#define GCU_STATUS_TCS0		BIT(8)
+#define GCU_STATUS_TCS1		BIT(9)
+#define GCU_STATUS_ACTIVE	BIT(31)
+
+/* GCU_CONFIG_VALIDx BITS */
+#define GCU_CONFIG_CVAL		BIT(0)
+
+/* PERIPHERAL registers */
+#define PERIPH_MAX_LINE_SIZE	BIT(0)
+#define PERIPH_NUM_RICH_LAYERS	BIT(4)
+#define PERIPH_SPLIT_EN		BIT(8)
+#define PERIPH_TBU_EN		BIT(12)
+#define PERIPH_AFBC_DMA_EN	BIT(16)
+#define PERIPH_CONFIGURATION_ID	0x1D4
+
+/* LPU register */
+#define LPU_TBU_STATUS		0x0B4
+#define LPU_RAXI_CONTROL	0x0D0
+#define LPU_WAXI_CONTROL	0x0D4
+#define LPU_TBU_CONTROL		0x0D8
+
+/* LPU_xAXI_CONTROL_BITS */
+#define TO_RAXI_AOUTSTDCAPB(x)	(x)
+#define TO_RAXI_BOUTSTDCAPB(x)	((x) << 8)
+#define TO_RAXI_BEN(x)		((x) << 15)
+#define TO_xAXI_BURSTLEN(x)	((x) << 16)
+#define TO_xAXI_AxQOS(x)	((x) << 24)
+#define TO_xAXI_ORD(x)		((x) << 31)
+#define TO_WAXI_OUTSTDCAPB(x)	(x)
+
+#define RAXI_AOUTSTDCAPB_MASK	0x7F
+#define RAXI_BOUTSTDCAPB_MASK	0x7F00
+#define RAXI_BEN_MASK		BIT(15)
+#define xAXI_BURSTLEN_MASK	0x3F0000
+#define xAXI_AxQOS_MASK		0xF000000
+#define xAXI_ORD_MASK		BIT(31)
+#define WAXI_OUTSTDCAPB_MASK	0x3F
+
+/* LPU_TBU_CONTROL BITS */
+#define TO_TBU_DOUTSTDCAPB(x)	(x)
+#define TBU_DOUTSTDCAPB_MASK	0x3F
+
+/* LPU_IRQ_BITS */
+#define LPU_IRQ_IBSY		BIT(10)
+#define LPU_IRQ_ERR		BIT(11)
+#define LPU_IRQ_EOW		BIT(12)
+#define LPU_IRQ_PL0		BIT(13)
+
+/* LPU_STATUS_BITS */
+#define LPU_STATUS_AXIED(x)	((x) & 0xF)
+#define LPU_STATUS_AXIE		BIT(4)
+#define LPU_STATUS_AXIRP	BIT(5)
+#define LPU_STATUS_AXIWP	BIT(6)
+#define LPU_STATUS_ACE0		BIT(16)
+#define LPU_STATUS_ACE1		BIT(17)
+#define LPU_STATUS_ACE2		BIT(18)
+#define LPU_STATUS_ACE3		BIT(19)
+#define LPU_STATUS_ACTIVE	BIT(31)
+
+#define AXIEID_MASK		0xF
+#define AXIE_MASK		LPU_STATUS_AXIE
+#define AXIRP_MASK		LPU_STATUS_AXIRP
+#define AXIWP_MASK		LPU_STATUS_AXIWP
+
+#define FROM_AXIEID(reg)	((reg) & AXIEID_MASK)
+#define TO_AXIE(x)		((x) << 4)
+#define FROM_AXIRP(reg)		(((reg) & AXIRP_MASK) >> 5)
+#define FROM_AXIWP(reg)		(((reg) & AXIWP_MASK) >> 6)
+
+/* LPU_TBU_STATUS_BITS */
+#define LPU_TBU_STATUS_TCF	BIT(1)
+#define LPU_TBU_STATUS_TTNG	BIT(2)
+#define LPU_TBU_STATUS_TITR	BIT(8)
+#define LPU_TBU_STATUS_TEMR	BIT(16)
+#define LPU_TBU_STATUS_TTF	BIT(31)
+
+/* LPU_TBU_CONTROL BITS */
+#define LPU_TBU_CTRL_TLBPEN	BIT(16)
+
+/* CROSSBAR CONTROL BITS */
+#define CBU_INPUT_CTRL_EN	BIT(0)
+#define CBU_NUM_INPUT_IDS	5
+#define CBU_NUM_OUTPUT_IDS	5
+
+/* CU register */
+#define CU_BG_COLOR		0x0DC
+#define CU_INPUT0_SIZE		0x0E0
+#define CU_INPUT0_OFFSET	0x0E4
+#define CU_INPUT0_CONTROL	0x0E8
+#define CU_INPUT1_SIZE		0x0F0
+#define CU_INPUT1_OFFSET	0x0F4
+#define CU_INPUT1_CONTROL	0x0F8
+#define CU_INPUT2_SIZE		0x100
+#define CU_INPUT2_OFFSET	0x104
+#define CU_INPUT2_CONTROL	0x108
+#define CU_INPUT3_SIZE		0x110
+#define CU_INPUT3_OFFSET	0x114
+#define CU_INPUT3_CONTROL	0x118
+#define CU_INPUT4_SIZE		0x120
+#define CU_INPUT4_OFFSET	0x124
+#define CU_INPUT4_CONTROL	0x128
+
+#define CU_PER_INPUT_REGS	4
+
+#define CU_NUM_INPUT_IDS	5
+#define CU_NUM_OUTPUT_IDS	1
+
+/* CU control register bits */
+#define CU_CTRL_COPROC		BIT(0)
+
+/* CU_IRQ_BITS */
+#define CU_IRQ_OVR		BIT(9)
+#define CU_IRQ_ERR		BIT(11)
+
+/* CU_STATUS_BITS */
+#define CU_STATUS_CPE		BIT(0)
+#define CU_STATUS_ZME		BIT(1)
+#define CU_STATUS_CFGE		BIT(2)
+#define CU_STATUS_ACTIVE	BIT(31)
+
+/* CU input control register bits */
+#define CU_INPUT_CTRL_EN	BIT(0)
+#define CU_INPUT_CTRL_PAD	BIT(1)
+#define CU_INPUT_CTRL_PMUL	BIT(2)
+#define CU_INPUT_CTRL_ALPHA(x)	(((x) & 0xFF) << 8)
+
+/* DOU register */
+
+/* DOU_IRQ_BITS */
+#define DOU_IRQ_UND		BIT(8)
+#define DOU_IRQ_ERR		BIT(11)
+#define DOU_IRQ_PL0		BIT(13)
+#define DOU_IRQ_PL1		BIT(14)
+
+/* DOU_STATUS_BITS */
+#define DOU_STATUS_DRIFTTO	BIT(0)
+#define DOU_STATUS_FRAMETO	BIT(1)
+#define DOU_STATUS_TETO		BIT(2)
+#define DOU_STATUS_CSCE		BIT(8)
+#define DOU_STATUS_ACTIVE	BIT(31)
+
+/* Layer registers */
+#define LAYER_INFO		0x0C0
+#define LAYER_R_CONTROL		0x0D4
+#define LAYER_FMT		0x0D8
+#define LAYER_LT_COEFFTAB	0x0DC
+#define LAYER_PALPHA		0x0E4
+
+#define LAYER_YUV_RGB_COEFF0	0x130
+
+#define LAYER_AD_H_CROP		0x164
+#define LAYER_AD_V_CROP		0x168
+
+#define LAYER_RGB_RGB_COEFF0	0x170
+
+/* L_CONTROL_BITS */
+#define L_EN			BIT(0)
+#define L_IT			BIT(4)
+#define L_R2R			BIT(5)
+#define L_FT			BIT(6)
+#define L_ROT(x)		(((x) & 3) << 8)
+#define L_HFLIP			BIT(10)
+#define L_VFLIP			BIT(11)
+#define L_TBU_EN		BIT(16)
+#define L_A_RCACHE(x)		(((x) & 0xF) << 28)
+#define L_ROT_R0		0
+#define L_ROT_R90		1
+#define L_ROT_R180		2
+#define L_ROT_R270		3
+
+/* LAYER_R_CONTROL BITS */
+#define LR_CHI422_BILINEAR	0
+#define LR_CHI422_REPLICATION	1
+#define LR_CHI420_JPEG		(0 << 2)
+#define LR_CHI420_MPEG		(1 << 2)
+
+#define L_ITSEL(x)		((x) & 0xFFF)
+#define L_FTSEL(x)		(((x) & 0xFFF) << 16)
+
+#define LAYER_PER_PLANE_REGS	4
+
+/* Layer_WR registers */
+#define LAYER_WR_PROG_LINE	0x0D4
+#define LAYER_WR_FORMAT		0x0D8
+
+/* Layer_WR control bits */
+#define LW_OFM			BIT(4)
+#define LW_LALPHA(x)		(((x) & 0xFF) << 8)
+#define LW_A_WCACHE(x)		(((x) & 0xF) << 28)
+#define LW_TBU_EN		BIT(16)
+
+#define AxCACHE_MASK		0xF0000000
+
+/* Layer AXI R/W cache setting */
+#define AxCACHE_B		BIT(0)	/* Bufferable */
+#define AxCACHE_M		BIT(1)	/* Modifiable */
+#define AxCACHE_RA		BIT(2)	/* Read-Allocate */
+#define AxCACHE_WA		BIT(3)	/* Write-Allocate */
+
+/* Layer info bits */
+#define L_INFO_RF		BIT(0)
+#define L_INFO_CM		BIT(1)
+#define L_INFO_ABUF_SIZE(x)	(((x) >> 4) & 0x7)
+
+/* Scaler registers */
+#define SC_COEFFTAB		0x0DC
+#define SC_OUT_SIZE		0x0E4
+#define SC_H_CROP		0x0E8
+#define SC_V_CROP		0x0EC
+#define SC_H_INIT_PH		0x0F0
+#define SC_H_DELTA_PH		0x0F4
+#define SC_V_INIT_PH		0x0F8
+#define SC_V_DELTA_PH		0x0FC
+#define SC_ENH_LIMITS		0x130
+#define SC_ENH_COEFF0		0x134
+
+#define SC_MAX_ENH_COEFF	9
+
+/* SC_CTRL_BITS */
+#define SC_CTRL_SCL		BIT(0)
+#define SC_CTRL_LS		BIT(1)
+#define SC_CTRL_AP		BIT(4)
+#define SC_CTRL_IENH		BIT(8)
+#define SC_CTRL_RGBSM		BIT(16)
+#define SC_CTRL_ASM		BIT(17)
+
+#define SC_VTSEL(vtal)		((vtal) << 16)
+
+#define SC_NUM_INPUTS_IDS	1
+#define SC_NUM_OUTPUTS_IDS	1
+
+#define MG_NUM_INPUTS_IDS	2
+#define MG_NUM_OUTPUTS_IDS	1
+
+/* Merger registers */
+#define MG_INPUT_ID0		BLK_INPUT_ID0
+#define MG_INPUT_ID1		(MG_INPUT_ID0 + 4)
+#define MG_SIZE			BLK_SIZE
+
+/* Splitter registers */
+#define SP_OVERLAP_SIZE		0xD8
+
+/* Backend registers */
+#define BS_INFO			0x0C0
+#define BS_PROG_LINE		0x0D4
+#define BS_PREFETCH_LINE	0x0D8
+#define BS_BG_COLOR		0x0DC
+#define BS_ACTIVESIZE		0x0E0
+#define BS_HINTERVALS		0x0E4
+#define BS_VINTERVALS		0x0E8
+#define BS_SYNC			0x0EC
+#define BS_DRIFT_TO		0x100
+#define BS_FRAME_TO		0x104
+#define BS_TE_TO		0x108
+#define BS_T0_INTERVAL		0x110
+#define BS_T1_INTERVAL		0x114
+#define BS_T2_INTERVAL		0x118
+#define BS_CRC0_LOW		0x120
+#define BS_CRC0_HIGH		0x124
+#define BS_CRC1_LOW		0x128
+#define BS_CRC1_HIGH		0x12C
+#define BS_USER			0x130
+
+/* BS control register bits */
+#define BS_CTRL_EN		BIT(0)
+#define BS_CTRL_VM		BIT(1)
+#define BS_CTRL_BM		BIT(2)
+#define BS_CTRL_HMASK		BIT(4)
+#define BS_CTRL_VD		BIT(5)
+#define BS_CTRL_TE		BIT(8)
+#define BS_CTRL_TS		BIT(9)
+#define BS_CTRL_TM		BIT(12)
+#define BS_CTRL_DL		BIT(16)
+#define BS_CTRL_SBS		BIT(17)
+#define BS_CTRL_CRC		BIT(18)
+#define BS_CTRL_PM		BIT(20)
+
+/* BS active size/intervals */
+#define BS_H_INTVALS(hfp, hbp)	(((hfp) & 0xFFF) + (((hbp) & 0x3FF) << 16))
+#define BS_V_INTVALS(vfp, vbp)  (((vfp) & 0x3FFF) + (((vbp) & 0xFF) << 16))
+
+/* BS_SYNC bits */
+#define BS_SYNC_HSW(x)		((x) & 0x3FF)
+#define BS_SYNC_HSP		BIT(12)
+#define BS_SYNC_VSW(x)		(((x) & 0xFF) << 16)
+#define BS_SYNC_VSP		BIT(28)
+
+#define BS_NUM_INPUT_IDS	0
+#define BS_NUM_OUTPUT_IDS	0
+
+/* Image process registers */
+#define IPS_DEPTH		0x0D8
+#define IPS_RGB_RGB_COEFF0	0x130
+#define IPS_RGB_YUV_COEFF0	0x170
+
+#define IPS_DEPTH_MARK		0xF
+
+/* IPS control register bits */
+#define IPS_CTRL_RGB		BIT(0)
+#define IPS_CTRL_FT		BIT(4)
+#define IPS_CTRL_YUV		BIT(8)
+#define IPS_CTRL_CHD422		BIT(9)
+#define IPS_CTRL_CHD420		BIT(10)
+#define IPS_CTRL_LPF		BIT(11)
+#define IPS_CTRL_DITH		BIT(12)
+#define IPS_CTRL_CLAMP		BIT(16)
+#define IPS_CTRL_SBS		BIT(17)
+
+/* IPS info register bits */
+#define IPS_INFO_CHD420		BIT(10)
+
+#define IPS_NUM_INPUT_IDS	2
+#define IPS_NUM_OUTPUT_IDS	1
+
+/* FT_COEFF block registers */
+#define FT_COEFF0		0x80
+#define GLB_IT_COEFF		0x80
+
+/* GLB_SC_COEFF registers */
+#define GLB_SC_COEFF_ADDR	0x0080
+#define GLB_SC_COEFF_DATA	0x0084
+#define GLB_LT_COEFF_DATA	0x0080
+
+#define GLB_SC_COEFF_MAX_NUM	1024
+#define GLB_LT_COEFF_NUM	65
+/* GLB_SC_ADDR */
+#define SC_COEFF_R_ADDR		BIT(18)
+#define SC_COEFF_G_ADDR		BIT(17)
+#define SC_COEFF_B_ADDR		BIT(16)
+
+#define SC_COEFF_DATA(x, y)	(((y) & 0xFFFF) | (((x) & 0xFFFF) << 16))
+
+enum d71_blk_type {
+	D71_BLK_TYPE_GCU		= 0x00,
+	D71_BLK_TYPE_LPU		= 0x01,
+	D71_BLK_TYPE_CU			= 0x02,
+	D71_BLK_TYPE_DOU		= 0x03,
+	D71_BLK_TYPE_AEU		= 0x04,
+	D71_BLK_TYPE_GLB_LT_COEFF	= 0x05,
+	D71_BLK_TYPE_GLB_SCL_COEFF	= 0x06, /* SH/SV scaler coeff */
+	D71_BLK_TYPE_GLB_SC_COEFF	= 0x07,
+	D71_BLK_TYPE_PERIPH		= 0x08,
+	D71_BLK_TYPE_LPU_TRUSTED	= 0x09,
+	D71_BLK_TYPE_AEU_TRUSTED	= 0x0A,
+	D71_BLK_TYPE_LPU_LAYER		= 0x10,
+	D71_BLK_TYPE_LPU_WB_LAYER	= 0x11,
+	D71_BLK_TYPE_CU_SPLITTER	= 0x20,
+	D71_BLK_TYPE_CU_SCALER		= 0x21,
+	D71_BLK_TYPE_CU_MERGER		= 0x22,
+	D71_BLK_TYPE_DOU_IPS		= 0x30,
+	D71_BLK_TYPE_DOU_BS		= 0x31,
+	D71_BLK_TYPE_DOU_FT_COEFF	= 0x32,
+	D71_BLK_TYPE_AEU_DS		= 0x40,
+	D71_BLK_TYPE_AEU_AES		= 0x41,
+	D71_BLK_TYPE_RESERVED		= 0xFF
+};
+
+/* Constant of components */
+#define D71_MAX_PIPELINE		2
+#define D71_PIPELINE_MAX_SCALERS	2
+#define D71_PIPELINE_MAX_LAYERS		4
+
+#define D71_MAX_GLB_IT_COEFF		3
+#define D71_MAX_GLB_SCL_COEFF		4
+
+#define D71_MAX_LAYERS_PER_LPU		4
+#define D71_BLOCK_MAX_INPUT		9
+#define D71_BLOCK_MAX_OUTPUT		5
+#define D71_MAX_SC_PER_CU		2
+
+#define D71_BLOCK_OFFSET_PERIPH		0xFE00
+#define D71_BLOCK_SIZE			0x0200
+
+#define D71_DEFAULT_PREPRETCH_LINE	5
+#define D71_BUS_WIDTH_16_BYTES		16
+
+#define D71_MIN_LINE_SIZE		64
+#define D71_MIN_VERTICAL_SIZE		64
+#define D71_SC_MIN_LIN_SIZE		4
+#define D71_SC_MIN_VERTICAL_SIZE	4
+#define D71_SC_MAX_LIN_SIZE		2048
+#define D71_SC_MAX_VERTICAL_SIZE	4096
+
+#define D71_SC_MAX_UPSCALING		64
+#define D71_SC_MAX_DOWNSCALING		6
+#define D71_SC_SPLIT_OVERLAP		8
+#define D71_SC_ENH_SPLIT_OVERLAP	1
+
+#define D71_MG_MIN_MERGED_SIZE		4
+#define D71_MG_MAX_MERGED_HSIZE		4032
+#define D71_MG_MAX_MERGED_VSIZE		4096
+
+#define D71_PALPHA_DEF_MAP		0xFFAA5500
+#define D71_LAYER_CONTROL_DEFAULT	0x30000000
+#define D71_WB_LAYER_CONTROL_DEFAULT	0x3000FF00
+#define D71_BS_CONTROL_DEFAULT		0x00000002
+
+struct block_header {
+	u32 block_info;
+	u32 pipeline_info;
+	u32 input_ids[D71_BLOCK_MAX_INPUT];
+	u32 output_ids[D71_BLOCK_MAX_OUTPUT];
+};
+
+static inline u32 get_block_type(struct block_header *blk)
+{
+	return BLOCK_INFO_BLK_TYPE(blk->block_info);
+}
+
+#endif /* !_D71_REG_H_ */
diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_crtc.c b/drivers/gpu/drm/arm/display/komeda/komeda_crtc.c
index 3ca5718aa0c2..62fad59f5a6a 100644
--- a/drivers/gpu/drm/arm/display/komeda/komeda_crtc.c
+++ b/drivers/gpu/drm/arm/display/komeda/komeda_crtc.c
@@ -18,10 +18,415 @@
 #include "komeda_dev.h"
 #include "komeda_kms.h"
 
-struct drm_crtc_helper_funcs komeda_crtc_helper_funcs = {
+/**
+ * komeda_crtc_atomic_check - build display output data flow
+ * @crtc: DRM crtc
+ * @state: the crtc state object
+ *
+ * crtc_atomic_check is the final check stage, so beside build a display data
+ * pipeline according to the crtc_state, but still needs to release or disable
+ * the unclaimed pipeline resources.
+ *
+ * RETURNS:
+ * Zero for success or -errno
+ */
+static int
+komeda_crtc_atomic_check(struct drm_crtc *crtc,
+			 struct drm_crtc_state *state)
+{
+	struct komeda_crtc *kcrtc = to_kcrtc(crtc);
+	struct komeda_crtc_state *kcrtc_st = to_kcrtc_st(state);
+	int err;
+
+	if (state->active) {
+		err = komeda_build_display_data_flow(kcrtc, kcrtc_st);
+		if (err)
+			return err;
+	}
+
+	/* release unclaimed pipeline resources */
+	err = komeda_release_unclaimed_resources(kcrtc->master, kcrtc_st);
+	if (err)
+		return err;
+
+	return 0;
+}
+
+static u32 komeda_calc_mclk(struct komeda_crtc_state *kcrtc_st)
+{
+	unsigned long mclk = kcrtc_st->base.adjusted_mode.clock * 1000;
+
+	return mclk;
+}
+
+/* For active a crtc, mainly need two parts of preparation
+ * 1. adjust display operation mode.
+ * 2. enable needed clk
+ */
+static int
+komeda_crtc_prepare(struct komeda_crtc *kcrtc)
+{
+	struct komeda_dev *mdev = kcrtc->base.dev->dev_private;
+	struct komeda_pipeline *master = kcrtc->master;
+	struct komeda_crtc_state *kcrtc_st = to_kcrtc_st(kcrtc->base.state);
+	unsigned long pxlclk_rate = kcrtc_st->base.adjusted_mode.clock * 1000;
+	u32 new_mode;
+	int err;
+
+	mutex_lock(&mdev->lock);
+
+	new_mode = mdev->dpmode | BIT(master->id);
+	if (WARN_ON(new_mode == mdev->dpmode)) {
+		err = 0;
+		goto unlock;
+	}
+
+	err = mdev->funcs->change_opmode(mdev, new_mode);
+	if (err) {
+		DRM_ERROR("failed to change opmode: 0x%x -> 0x%x.\n,",
+			  mdev->dpmode, new_mode);
+		goto unlock;
+	}
+
+	mdev->dpmode = new_mode;
+	/* Only need to enable mclk on single display mode, but no need to
+	 * enable mclk it on dual display mode, since the dual mode always
+	 * switch from single display mode, the mclk already enabled, no need
+	 * to enable it again.
+	 */
+	if (new_mode != KOMEDA_MODE_DUAL_DISP) {
+		err = clk_set_rate(mdev->mclk, komeda_calc_mclk(kcrtc_st));
+		if (err)
+			DRM_ERROR("failed to set mclk.\n");
+		err = clk_prepare_enable(mdev->mclk);
+		if (err)
+			DRM_ERROR("failed to enable mclk.\n");
+	}
+
+	err = clk_prepare_enable(master->aclk);
+	if (err)
+		DRM_ERROR("failed to enable axi clk for pipe%d.\n", master->id);
+	err = clk_set_rate(master->pxlclk, pxlclk_rate);
+	if (err)
+		DRM_ERROR("failed to set pxlclk for pipe%d\n", master->id);
+	err = clk_prepare_enable(master->pxlclk);
+	if (err)
+		DRM_ERROR("failed to enable pxl clk for pipe%d.\n", master->id);
+
+unlock:
+	mutex_unlock(&mdev->lock);
+
+	return err;
+}
+
+static int
+komeda_crtc_unprepare(struct komeda_crtc *kcrtc)
+{
+	struct komeda_dev *mdev = kcrtc->base.dev->dev_private;
+	struct komeda_pipeline *master = kcrtc->master;
+	u32 new_mode;
+	int err;
+
+	mutex_lock(&mdev->lock);
+
+	new_mode = mdev->dpmode & (~BIT(master->id));
+
+	if (WARN_ON(new_mode == mdev->dpmode)) {
+		err = 0;
+		goto unlock;
+	}
+
+	err = mdev->funcs->change_opmode(mdev, new_mode);
+	if (err) {
+		DRM_ERROR("failed to change opmode: 0x%x -> 0x%x.\n,",
+			  mdev->dpmode, new_mode);
+		goto unlock;
+	}
+
+	mdev->dpmode = new_mode;
+
+	clk_disable_unprepare(master->pxlclk);
+	clk_disable_unprepare(master->aclk);
+	if (new_mode == KOMEDA_MODE_INACTIVE)
+		clk_disable_unprepare(mdev->mclk);
+
+unlock:
+	mutex_unlock(&mdev->lock);
+
+	return err;
+}
+
+void komeda_crtc_handle_event(struct komeda_crtc   *kcrtc,
+			      struct komeda_events *evts)
+{
+	struct drm_crtc *crtc = &kcrtc->base;
+	u32 events = evts->pipes[kcrtc->master->id];
+
+	if (events & KOMEDA_EVENT_VSYNC)
+		drm_crtc_handle_vblank(crtc);
+
+	/* will handle it together with the write back support */
+	if (events & KOMEDA_EVENT_EOW)
+		DRM_DEBUG("EOW.\n");
+
+	if (events & KOMEDA_EVENT_FLIP) {
+		unsigned long flags;
+		struct drm_pending_vblank_event *event;
+
+		spin_lock_irqsave(&crtc->dev->event_lock, flags);
+		if (kcrtc->disable_done) {
+			complete_all(kcrtc->disable_done);
+			kcrtc->disable_done = NULL;
+		} else if (crtc->state->event) {
+			event = crtc->state->event;
+			/*
+			 * Consume event before notifying drm core that flip
+			 * happened.
+			 */
+			crtc->state->event = NULL;
+			drm_crtc_send_vblank_event(crtc, event);
+		} else {
+			DRM_WARN("CRTC[%d]: FLIP happen but no pending commit.\n",
+				 drm_crtc_index(&kcrtc->base));
+		}
+		spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
+	}
+}
+
+static void
+komeda_crtc_do_flush(struct drm_crtc *crtc,
+		     struct drm_crtc_state *old)
+{
+	struct komeda_crtc *kcrtc = to_kcrtc(crtc);
+	struct komeda_crtc_state *kcrtc_st = to_kcrtc_st(crtc->state);
+	struct komeda_dev *mdev = kcrtc->base.dev->dev_private;
+	struct komeda_pipeline *master = kcrtc->master;
+
+	DRM_DEBUG_ATOMIC("CRTC%d_FLUSH: active_pipes: 0x%x, affected: 0x%x.\n",
+			 drm_crtc_index(crtc),
+			 kcrtc_st->active_pipes, kcrtc_st->affected_pipes);
+
+	/* step 1: update the pipeline/component state to HW */
+	if (has_bit(master->id, kcrtc_st->affected_pipes))
+		komeda_pipeline_update(master, old->state);
+
+	/* step 2: notify the HW to kickoff the update */
+	mdev->funcs->flush(mdev, master->id, kcrtc_st->active_pipes);
+}
+
+static void
+komeda_crtc_atomic_enable(struct drm_crtc *crtc,
+			  struct drm_crtc_state *old)
+{
+	komeda_crtc_prepare(to_kcrtc(crtc));
+	drm_crtc_vblank_on(crtc);
+	komeda_crtc_do_flush(crtc, old);
+}
+
+static void
+komeda_crtc_atomic_disable(struct drm_crtc *crtc,
+			   struct drm_crtc_state *old)
+{
+	struct komeda_crtc *kcrtc = to_kcrtc(crtc);
+	struct komeda_crtc_state *old_st = to_kcrtc_st(old);
+	struct komeda_dev *mdev = crtc->dev->dev_private;
+	struct komeda_pipeline *master = kcrtc->master;
+	struct completion *disable_done = &crtc->state->commit->flip_done;
+	struct completion temp;
+	int timeout;
+
+	DRM_DEBUG_ATOMIC("CRTC%d_DISABLE: active_pipes: 0x%x, affected: 0x%x.\n",
+			 drm_crtc_index(crtc),
+			 old_st->active_pipes, old_st->affected_pipes);
+
+	if (has_bit(master->id, old_st->active_pipes))
+		komeda_pipeline_disable(master, old->state);
+
+	/* crtc_disable has two scenarios according to the state->active switch.
+	 * 1. active -> inactive
+	 *    this commit is a disable commit. and the commit will be finished
+	 *    or done after the disable operation. on this case we can directly
+	 *    use the crtc->state->event to tracking the HW disable operation.
+	 * 2. active -> active
+	 *    the crtc->commit is not for disable, but a modeset operation when
+	 *    crtc is active, such commit actually has been completed by 3
+	 *    DRM operations:
+	 *    crtc_disable, update_planes(crtc_flush), crtc_enable
+	 *    so on this case the crtc->commit is for the whole process.
+	 *    we can not use it for tracing the disable, we need a temporary
+	 *    flip_done for tracing the disable. and crtc->state->event for
+	 *    the crtc_enable operation.
+	 *    That's also the reason why skip modeset commit in
+	 *    komeda_crtc_atomic_flush()
+	 */
+	if (crtc->state->active) {
+		struct komeda_pipeline_state *pipe_st;
+		/* clear the old active_comps to zero */
+		pipe_st = komeda_pipeline_get_old_state(master, old->state);
+		pipe_st->active_comps = 0;
+
+		init_completion(&temp);
+		kcrtc->disable_done = &temp;
+		disable_done = &temp;
+	}
+
+	mdev->funcs->flush(mdev, master->id, 0);
+
+	/* wait the disable take affect.*/
+	timeout = wait_for_completion_timeout(disable_done, HZ);
+	if (timeout == 0) {
+		DRM_ERROR("disable pipeline%d timeout.\n", kcrtc->master->id);
+		if (crtc->state->active) {
+			unsigned long flags;
+
+			spin_lock_irqsave(&crtc->dev->event_lock, flags);
+			kcrtc->disable_done = NULL;
+			spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
+		}
+	}
+
+	drm_crtc_vblank_off(crtc);
+	komeda_crtc_unprepare(kcrtc);
+}
+
+static void
+komeda_crtc_atomic_flush(struct drm_crtc *crtc,
+			 struct drm_crtc_state *old)
+{
+	/* commit with modeset will be handled in enable/disable */
+	if (drm_atomic_crtc_needs_modeset(crtc->state))
+		return;
+
+	komeda_crtc_do_flush(crtc, old);
+}
+
+static enum drm_mode_status
+komeda_crtc_mode_valid(struct drm_crtc *crtc, const struct drm_display_mode *m)
+{
+	struct komeda_dev *mdev = crtc->dev->dev_private;
+	struct komeda_crtc *kcrtc = to_kcrtc(crtc);
+	struct komeda_pipeline *master = kcrtc->master;
+	long mode_clk, pxlclk;
+
+	if (m->flags & DRM_MODE_FLAG_INTERLACE)
+		return MODE_NO_INTERLACE;
+
+	/* main clock/AXI clk must be faster than pxlclk*/
+	mode_clk = m->clock * 1000;
+	pxlclk = clk_round_rate(master->pxlclk, mode_clk);
+	if (pxlclk != mode_clk) {
+		DRM_DEBUG_ATOMIC("pxlclk doesn't support %ld Hz\n", mode_clk);
+
+		return MODE_NOCLOCK;
+	}
+
+	if (clk_round_rate(mdev->mclk, mode_clk) < pxlclk) {
+		DRM_DEBUG_ATOMIC("mclk can't satisfy the requirement of %s-clk: %ld.\n",
+				 m->name, pxlclk);
+
+		return MODE_CLOCK_HIGH;
+	}
+
+	if (clk_round_rate(master->aclk, mode_clk) < pxlclk) {
+		DRM_DEBUG_ATOMIC("aclk can't satisfy the requirement of %s-clk: %ld.\n",
+				 m->name, pxlclk);
+
+		return MODE_CLOCK_HIGH;
+	}
+
+	return MODE_OK;
+}
+
+static bool komeda_crtc_mode_fixup(struct drm_crtc *crtc,
+				   const struct drm_display_mode *m,
+				   struct drm_display_mode *adjusted_mode)
+{
+	struct komeda_crtc *kcrtc = to_kcrtc(crtc);
+	struct komeda_pipeline *master = kcrtc->master;
+	long mode_clk = m->clock * 1000;
+
+	adjusted_mode->clock = clk_round_rate(master->pxlclk, mode_clk) / 1000;
+
+	return true;
+}
+
+static struct drm_crtc_helper_funcs komeda_crtc_helper_funcs = {
+	.atomic_check	= komeda_crtc_atomic_check,
+	.atomic_flush	= komeda_crtc_atomic_flush,
+	.atomic_enable	= komeda_crtc_atomic_enable,
+	.atomic_disable	= komeda_crtc_atomic_disable,
+	.mode_valid	= komeda_crtc_mode_valid,
+	.mode_fixup	= komeda_crtc_mode_fixup,
 };
 
+static void komeda_crtc_reset(struct drm_crtc *crtc)
+{
+	struct komeda_crtc_state *state;
+
+	if (crtc->state)
+		__drm_atomic_helper_crtc_destroy_state(crtc->state);
+
+	kfree(to_kcrtc_st(crtc->state));
+	crtc->state = NULL;
+
+	state = kzalloc(sizeof(*state), GFP_KERNEL);
+	if (state) {
+		crtc->state = &state->base;
+		crtc->state->crtc = crtc;
+	}
+}
+
+static struct drm_crtc_state *
+komeda_crtc_atomic_duplicate_state(struct drm_crtc *crtc)
+{
+	struct komeda_crtc_state *old = to_kcrtc_st(crtc->state);
+	struct komeda_crtc_state *new;
+
+	new = kzalloc(sizeof(*new), GFP_KERNEL);
+	if (!new)
+		return NULL;
+
+	__drm_atomic_helper_crtc_duplicate_state(crtc, &new->base);
+
+	new->affected_pipes = old->active_pipes;
+
+	return &new->base;
+}
+
+static void komeda_crtc_atomic_destroy_state(struct drm_crtc *crtc,
+					     struct drm_crtc_state *state)
+{
+	__drm_atomic_helper_crtc_destroy_state(state);
+	kfree(to_kcrtc_st(state));
+}
+
+static int komeda_crtc_vblank_enable(struct drm_crtc *crtc)
+{
+	struct komeda_dev *mdev = crtc->dev->dev_private;
+	struct komeda_crtc *kcrtc = to_kcrtc(crtc);
+
+	mdev->funcs->on_off_vblank(mdev, kcrtc->master->id, true);
+	return 0;
+}
+
+static void komeda_crtc_vblank_disable(struct drm_crtc *crtc)
+{
+	struct komeda_dev *mdev = crtc->dev->dev_private;
+	struct komeda_crtc *kcrtc = to_kcrtc(crtc);
+
+	mdev->funcs->on_off_vblank(mdev, kcrtc->master->id, false);
+}
+
 static const struct drm_crtc_funcs komeda_crtc_funcs = {
+	.gamma_set		= drm_atomic_helper_legacy_gamma_set,
+	.destroy		= drm_crtc_cleanup,
+	.set_config		= drm_atomic_helper_set_config,
+	.page_flip		= drm_atomic_helper_page_flip,
+	.reset			= komeda_crtc_reset,
+	.atomic_duplicate_state	= komeda_crtc_atomic_duplicate_state,
+	.atomic_destroy_state	= komeda_crtc_atomic_destroy_state,
+	.enable_vblank		= komeda_crtc_vblank_enable,
+	.disable_vblank		= komeda_crtc_vblank_disable,
 };
 
 int komeda_kms_setup_crtcs(struct komeda_kms_dev *kms,
diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_dev.c b/drivers/gpu/drm/arm/display/komeda/komeda_dev.c
index 70e9bb7fa30c..ca3599e4a4d3 100644
--- a/drivers/gpu/drm/arm/display/komeda/komeda_dev.c
+++ b/drivers/gpu/drm/arm/display/komeda/komeda_dev.c
@@ -8,11 +8,99 @@
 #include <linux/of_device.h>
 #include <linux/of_graph.h>
 #include <linux/platform_device.h>
+#ifdef CONFIG_DEBUG_FS
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
+#endif
 
 #include <drm/drm_print.h>
 
 #include "komeda_dev.h"
 
+static int komeda_register_show(struct seq_file *sf, void *x)
+{
+	struct komeda_dev *mdev = sf->private;
+	int i;
+
+	if (mdev->funcs->dump_register)
+		mdev->funcs->dump_register(mdev, sf);
+
+	for (i = 0; i < mdev->n_pipelines; i++)
+		komeda_pipeline_dump_register(mdev->pipelines[i], sf);
+
+	return 0;
+}
+
+static int komeda_register_open(struct inode *inode, struct file *filp)
+{
+	return single_open(filp, komeda_register_show, inode->i_private);
+}
+
+static const struct file_operations komeda_register_fops = {
+	.owner		= THIS_MODULE,
+	.open		= komeda_register_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+};
+
+#ifdef CONFIG_DEBUG_FS
+static void komeda_debugfs_init(struct komeda_dev *mdev)
+{
+	if (!debugfs_initialized())
+		return;
+
+	mdev->debugfs_root = debugfs_create_dir("komeda", NULL);
+	if (IS_ERR_OR_NULL(mdev->debugfs_root))
+		return;
+
+	debugfs_create_file("register", 0444, mdev->debugfs_root,
+			    mdev, &komeda_register_fops);
+}
+#endif
+
+static ssize_t
+core_id_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct komeda_dev *mdev = dev_to_mdev(dev);
+
+	return snprintf(buf, PAGE_SIZE, "0x%08x\n", mdev->chip.core_id);
+}
+static DEVICE_ATTR_RO(core_id);
+
+static ssize_t
+config_id_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct komeda_dev *mdev = dev_to_mdev(dev);
+	struct komeda_pipeline *pipe = mdev->pipelines[0];
+	union komeda_config_id config_id;
+	int i;
+
+	memset(&config_id, 0, sizeof(config_id));
+
+	config_id.max_line_sz = pipe->layers[0]->hsize_in.end;
+	config_id.n_pipelines = mdev->n_pipelines;
+	config_id.n_scalers = pipe->n_scalers;
+	config_id.n_layers = pipe->n_layers;
+	config_id.n_richs = 0;
+	for (i = 0; i < pipe->n_layers; i++) {
+		if (pipe->layers[i]->layer_type == KOMEDA_FMT_RICH_LAYER)
+			config_id.n_richs++;
+	}
+	return snprintf(buf, PAGE_SIZE, "0x%08x\n", config_id.value);
+}
+static DEVICE_ATTR_RO(config_id);
+
+static struct attribute *komeda_sysfs_entries[] = {
+	&dev_attr_core_id.attr,
+	&dev_attr_config_id.attr,
+	NULL,
+};
+
+static struct attribute_group komeda_sysfs_attr_group = {
+	.attrs = komeda_sysfs_entries,
+};
+
 static int komeda_parse_pipe_dt(struct komeda_dev *mdev, struct device_node *np)
 {
 	struct komeda_pipeline *pipe;
@@ -53,6 +141,7 @@ static int komeda_parse_pipe_dt(struct komeda_dev *mdev, struct device_node *np)
 
 static int komeda_parse_dt(struct device *dev, struct komeda_dev *mdev)
 {
+	struct platform_device *pdev = to_platform_device(dev);
 	struct device_node *child, *np = dev->of_node;
 	struct clk *clk;
 	int ret;
@@ -62,6 +151,11 @@ static int komeda_parse_dt(struct device *dev, struct komeda_dev *mdev)
 		return PTR_ERR(clk);
 
 	mdev->mclk = clk;
+	mdev->irq  = platform_get_irq(pdev, 0);
+	if (mdev->irq < 0) {
+		DRM_ERROR("could not get IRQ number.\n");
+		return mdev->irq;
+	}
 
 	for_each_available_child_of_node(np, child) {
 		if (of_node_cmp(child->name, "pipeline") == 0) {
@@ -99,6 +193,8 @@ struct komeda_dev *komeda_dev_create(struct device *dev)
 	if (!mdev)
 		return ERR_PTR(-ENOMEM);
 
+	mutex_init(&mdev->lock);
+
 	mdev->dev = dev;
 	mdev->reg_base = devm_ioremap_resource(dev, io_res);
 	if (IS_ERR(mdev->reg_base)) {
@@ -147,6 +243,22 @@ struct komeda_dev *komeda_dev_create(struct device *dev)
 		goto err_cleanup;
 	}
 
+	err = komeda_assemble_pipelines(mdev);
+	if (err) {
+		DRM_ERROR("assemble display pipelines failed.\n");
+		goto err_cleanup;
+	}
+
+	err = sysfs_create_group(&dev->kobj, &komeda_sysfs_attr_group);
+	if (err) {
+		DRM_ERROR("create sysfs group failed.\n");
+		goto err_cleanup;
+	}
+
+#ifdef CONFIG_DEBUG_FS
+	komeda_debugfs_init(mdev);
+#endif
+
 	return mdev;
 
 err_cleanup:
@@ -160,6 +272,12 @@ void komeda_dev_destroy(struct komeda_dev *mdev)
 	struct komeda_dev_funcs *funcs = mdev->funcs;
 	int i;
 
+	sysfs_remove_group(&dev->kobj, &komeda_sysfs_attr_group);
+
+#ifdef CONFIG_DEBUG_FS
+	debugfs_remove_recursive(mdev->debugfs_root);
+#endif
+
 	for (i = 0; i < mdev->n_pipelines; i++) {
 		komeda_pipeline_destroy(mdev, mdev->pipelines[i]);
 		mdev->pipelines[i] = NULL;
diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_dev.h b/drivers/gpu/drm/arm/display/komeda/komeda_dev.h
index 0f77dead6a23..29e03c4e1ffc 100644
--- a/drivers/gpu/drm/arm/display/komeda/komeda_dev.h
+++ b/drivers/gpu/drm/arm/display/komeda/komeda_dev.h
@@ -13,6 +13,33 @@
 #include "malidp_product.h"
 #include "komeda_format_caps.h"
 
+#define KOMEDA_EVENT_VSYNC		BIT_ULL(0)
+#define KOMEDA_EVENT_FLIP		BIT_ULL(1)
+#define KOMEDA_EVENT_URUN		BIT_ULL(2)
+#define KOMEDA_EVENT_IBSY		BIT_ULL(3)
+#define KOMEDA_EVENT_OVR		BIT_ULL(4)
+#define KOMEDA_EVENT_EOW		BIT_ULL(5)
+#define KOMEDA_EVENT_MODE		BIT_ULL(6)
+
+#define KOMEDA_ERR_TETO			BIT_ULL(14)
+#define KOMEDA_ERR_TEMR			BIT_ULL(15)
+#define KOMEDA_ERR_TITR			BIT_ULL(16)
+#define KOMEDA_ERR_CPE			BIT_ULL(17)
+#define KOMEDA_ERR_CFGE			BIT_ULL(18)
+#define KOMEDA_ERR_AXIE			BIT_ULL(19)
+#define KOMEDA_ERR_ACE0			BIT_ULL(20)
+#define KOMEDA_ERR_ACE1			BIT_ULL(21)
+#define KOMEDA_ERR_ACE2			BIT_ULL(22)
+#define KOMEDA_ERR_ACE3			BIT_ULL(23)
+#define KOMEDA_ERR_DRIFTTO		BIT_ULL(24)
+#define KOMEDA_ERR_FRAMETO		BIT_ULL(25)
+#define KOMEDA_ERR_CSCE			BIT_ULL(26)
+#define KOMEDA_ERR_ZME			BIT_ULL(27)
+#define KOMEDA_ERR_MERR			BIT_ULL(28)
+#define KOMEDA_ERR_TCF			BIT_ULL(29)
+#define KOMEDA_ERR_TTNG			BIT_ULL(30)
+#define KOMEDA_ERR_TTF			BIT_ULL(31)
+
 /* malidp device id */
 enum {
 	MALI_D71 = 0,
@@ -39,6 +66,11 @@ struct komeda_product_data {
 
 struct komeda_dev;
 
+struct komeda_events {
+	u64 global;
+	u64 pipes[KOMEDA_MAX_PIPELINES];
+};
+
 /**
  * struct komeda_dev_funcs
  *
@@ -60,6 +92,49 @@ struct komeda_dev_funcs {
 	int (*enum_resources)(struct komeda_dev *mdev);
 	/** @cleanup: call to chip to cleanup komeda_dev->chip data */
 	void (*cleanup)(struct komeda_dev *mdev);
+	/**
+	 * @irq_handler:
+	 *
+	 * for CORE to get the HW event from the CHIP when interrupt happened.
+	 */
+	irqreturn_t (*irq_handler)(struct komeda_dev *mdev,
+				   struct komeda_events *events);
+	/** @enable_irq: enable irq */
+	int (*enable_irq)(struct komeda_dev *mdev);
+	/** @disable_irq: disable irq */
+	int (*disable_irq)(struct komeda_dev *mdev);
+	/** @on_off_vblank: notify HW to on/off vblank */
+	void (*on_off_vblank)(struct komeda_dev *mdev,
+			      int master_pipe, bool on);
+
+	/** @dump_register: Optional, dump registers to seq_file */
+	void (*dump_register)(struct komeda_dev *mdev, struct seq_file *seq);
+	/**
+	 * @change_opmode:
+	 *
+	 * Notify HW to switch to a new display operation mode.
+	 */
+	int (*change_opmode)(struct komeda_dev *mdev, int new_mode);
+	/** @flush: Notify the HW to flush or kickoff the update */
+	void (*flush)(struct komeda_dev *mdev,
+		      int master_pipe, u32 active_pipes);
+};
+
+/*
+ * DISPLAY_MODE describes how many display been enabled, and which will be
+ * passed to CHIP by &komeda_dev_funcs->change_opmode(), then CHIP can do the
+ * pipeline resources assignment according to this usage hint.
+ * -   KOMEDA_MODE_DISP0: Only one display enabled, pipeline-0 work as master.
+ * -   KOMEDA_MODE_DISP1: Only one display enabled, pipeline-0 work as master.
+ * -   KOMEDA_MODE_DUAL_DISP: Dual display mode, both display has been enabled.
+ * And D71 supports assign two pipelines to one single display on mode
+ * KOMEDA_MODE_DISP0/DISP1
+ */
+enum {
+	KOMEDA_MODE_INACTIVE	= 0,
+	KOMEDA_MODE_DISP0	= BIT(0),
+	KOMEDA_MODE_DISP1	= BIT(1),
+	KOMEDA_MODE_DUAL_DISP	= KOMEDA_MODE_DISP0 | KOMEDA_MODE_DISP1,
 };
 
 /**
@@ -70,18 +145,31 @@ struct komeda_dev_funcs {
  * control-abilites of device.
  */
 struct komeda_dev {
+	/** @dev: the base device structure */
 	struct device *dev;
+	/** @reg_base: the base address of komeda io space */
 	u32 __iomem   *reg_base;
 
+	/** @chip: the basic chip information */
 	struct komeda_chip_info chip;
 	/** @fmt_tbl: initialized by &komeda_dev_funcs->init_format_table */
 	struct komeda_format_caps_table fmt_tbl;
 	/** @pclk: APB clock for register access */
 	struct clk *pclk;
-	/** @mck: HW main engine clk */
+	/** @mclk: HW main engine clk */
 	struct clk *mclk;
 
+	/** @irq: irq number */
+	int irq;
+
+	/** @lock: used to protect dpmode */
+	struct mutex lock;
+	/** @dpmode: current display mode */
+	u32 dpmode;
+
+	/** @n_pipelines: the number of pipe in @pipelines */
 	int n_pipelines;
+	/** @pipelines: the komeda pipelines */
 	struct komeda_pipeline *pipelines[KOMEDA_MAX_PIPELINES];
 
 	/** @funcs: chip funcs to access to HW */
@@ -93,6 +181,9 @@ struct komeda_dev {
 	 * destroyed by &komeda_dev_funcs.cleanup()
 	 */
 	void *chip_data;
+
+	/** @debugfs_root: root directory of komeda debugfs */
+	struct dentry *debugfs_root;
 };
 
 static inline bool
@@ -107,4 +198,6 @@ d71_identify(u32 __iomem *reg, struct komeda_chip_info *chip);
 struct komeda_dev *komeda_dev_create(struct device *dev);
 void komeda_dev_destroy(struct komeda_dev *mdev);
 
+struct komeda_dev *dev_to_mdev(struct device *dev);
+
 #endif /*_KOMEDA_DEV_H_*/
diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_drv.c b/drivers/gpu/drm/arm/display/komeda/komeda_drv.c
index 2bdd189b041d..cfa5068d9d1e 100644
--- a/drivers/gpu/drm/arm/display/komeda/komeda_drv.c
+++ b/drivers/gpu/drm/arm/display/komeda/komeda_drv.c
@@ -17,6 +17,13 @@ struct komeda_drv {
 	struct komeda_kms_dev *kms;
 };
 
+struct komeda_dev *dev_to_mdev(struct device *dev)
+{
+	struct komeda_drv *mdrv = dev_get_drvdata(dev);
+
+	return mdrv ? mdrv->mdev : NULL;
+}
+
 static void komeda_unbind(struct device *dev)
 {
 	struct komeda_drv *mdrv = dev_get_drvdata(dev);
@@ -120,7 +127,7 @@ static const struct komeda_product_data komeda_products[] = {
 	},
 };
 
-const struct of_device_id komeda_of_match[] = {
+static const struct of_device_id komeda_of_match[] = {
 	{ .compatible = "arm,mali-d71", .data = &komeda_products[MALI_D71], },
 	{},
 };
diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_framebuffer.h b/drivers/gpu/drm/arm/display/komeda/komeda_framebuffer.h
index 0de2e4a2afd2..ea2fe190c1e3 100644
--- a/drivers/gpu/drm/arm/display/komeda/komeda_framebuffer.h
+++ b/drivers/gpu/drm/arm/display/komeda/komeda_framebuffer.h
@@ -10,11 +10,16 @@
 #include <drm/drm_framebuffer.h>
 #include "komeda_format_caps.h"
 
-/** struct komeda_fb - entend drm_framebuffer with komeda attribute */
+/**
+ * struct komeda_fb - Entending drm_framebuffer with komeda attribute
+ */
 struct komeda_fb {
 	/** @base: &drm_framebuffer */
 	struct drm_framebuffer base;
-	/* @format_caps: &komeda_format_caps */
+	/**
+	 * @format_caps:
+	 * extends drm_format_info for komeda specific information
+	 */
 	const struct komeda_format_caps *format_caps;
 	/** @aligned_w: aligned frame buffer width */
 	u32 aligned_w;
diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_kms.c b/drivers/gpu/drm/arm/display/komeda/komeda_kms.c
index 47a58ab20434..86f6542afb40 100644
--- a/drivers/gpu/drm/arm/display/komeda/komeda_kms.c
+++ b/drivers/gpu/drm/arm/display/komeda/komeda_kms.c
@@ -13,6 +13,7 @@
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_gem_cma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
+#include <drm/drm_irq.h>
 #include <drm/drm_vblank.h>
 
 #include "komeda_dev.h"
@@ -25,18 +26,39 @@ static int komeda_gem_cma_dumb_create(struct drm_file *file,
 				      struct drm_device *dev,
 				      struct drm_mode_create_dumb *args)
 {
-	u32 alignment = 16; /* TODO get alignment from dev */
+	struct komeda_dev *mdev = dev->dev_private;
+	u32 pitch = DIV_ROUND_UP(args->width * args->bpp, 8);
 
-	args->pitch = ALIGN(DIV_ROUND_UP(args->width * args->bpp, 8),
-			    alignment);
+	args->pitch = ALIGN(pitch, mdev->chip.bus_width);
 
 	return drm_gem_cma_dumb_create_internal(file, dev, args);
 }
 
+static irqreturn_t komeda_kms_irq_handler(int irq, void *data)
+{
+	struct drm_device *drm = data;
+	struct komeda_dev *mdev = drm->dev_private;
+	struct komeda_kms_dev *kms = to_kdev(drm);
+	struct komeda_events evts;
+	irqreturn_t status;
+	u32 i;
+
+	/* Call into the CHIP to recognize events */
+	memset(&evts, 0, sizeof(evts));
+	status = mdev->funcs->irq_handler(mdev, &evts);
+
+	/* Notify the crtc to handle the events */
+	for (i = 0; i < kms->n_crtcs; i++)
+		komeda_crtc_handle_event(&kms->crtcs[i], &evts);
+
+	return status;
+}
+
 static struct drm_driver komeda_kms_driver = {
 	.driver_features = DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC |
-			   DRIVER_PRIME,
+			   DRIVER_PRIME | DRIVER_HAVE_IRQ,
 	.lastclose			= drm_fb_helper_lastclose,
+	.irq_handler			= komeda_kms_irq_handler,
 	.gem_free_object_unlocked	= drm_gem_cma_free_object,
 	.gem_vm_ops			= &drm_gem_cma_vm_ops,
 	.dumb_create			= komeda_gem_cma_dumb_create,
@@ -78,9 +100,37 @@ static const struct drm_mode_config_helper_funcs komeda_mode_config_helpers = {
 	.atomic_commit_tail = komeda_kms_commit_tail,
 };
 
+static int komeda_kms_check(struct drm_device *dev,
+			    struct drm_atomic_state *state)
+{
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *old_crtc_st, *new_crtc_st;
+	int i, err;
+
+	err = drm_atomic_helper_check_modeset(dev, state);
+	if (err)
+		return err;
+
+	/* komeda need to re-calculate resource assumption in every commit
+	 * so need to add all affected_planes (even unchanged) to
+	 * drm_atomic_state.
+	 */
+	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_st, new_crtc_st, i) {
+		err = drm_atomic_add_affected_planes(state, crtc);
+		if (err)
+			return err;
+	}
+
+	err = drm_atomic_helper_check_planes(dev, state);
+	if (err)
+		return err;
+
+	return 0;
+}
+
 static const struct drm_mode_config_funcs komeda_mode_config_funcs = {
 	.fb_create		= komeda_fb_create,
-	.atomic_check		= drm_atomic_helper_check,
+	.atomic_check		= komeda_kms_check,
 	.atomic_commit		= drm_atomic_helper_commit,
 };
 
@@ -144,14 +194,25 @@ struct komeda_kms_dev *komeda_kms_attach(struct komeda_dev *mdev)
 
 	drm_mode_config_reset(drm);
 
-	err = drm_dev_register(drm, 0);
+	err = drm_irq_install(drm, mdev->irq);
 	if (err)
 		goto cleanup_mode_config;
 
+	err = mdev->funcs->enable_irq(mdev);
+	if (err)
+		goto uninstall_irq;
+
+	err = drm_dev_register(drm, 0);
+	if (err)
+		goto uninstall_irq;
+
 	return kms;
 
+uninstall_irq:
+	drm_irq_uninstall(drm);
 cleanup_mode_config:
 	drm_mode_config_cleanup(drm);
+	komeda_kms_cleanup_private_objs(kms);
 free_kms:
 	kfree(kms);
 	return ERR_PTR(err);
@@ -162,9 +223,11 @@ void komeda_kms_detach(struct komeda_kms_dev *kms)
 	struct drm_device *drm = &kms->base;
 	struct komeda_dev *mdev = drm->dev_private;
 
+	mdev->funcs->disable_irq(mdev);
 	drm_dev_unregister(drm);
+	drm_irq_uninstall(drm);
 	component_unbind_all(mdev->dev, drm);
-	komeda_kms_cleanup_private_objs(mdev);
+	komeda_kms_cleanup_private_objs(kms);
 	drm_mode_config_cleanup(drm);
 	drm->dev_private = NULL;
 	drm_dev_put(drm);
diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_kms.h b/drivers/gpu/drm/arm/display/komeda/komeda_kms.h
index 874e9c9f0749..ac3d9209b4d9 100644
--- a/drivers/gpu/drm/arm/display/komeda/komeda_kms.h
+++ b/drivers/gpu/drm/arm/display/komeda/komeda_kms.h
@@ -12,8 +12,12 @@
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_device.h>
 #include <drm/drm_writeback.h>
+#include <video/videomode.h>
+#include <video/display_timing.h>
 
-/** struct komeda_plane - komeda instance of drm_plane */
+/**
+ * struct komeda_plane - komeda instance of drm_plane
+ */
 struct komeda_plane {
 	/** @base: &drm_plane */
 	struct drm_plane base;
@@ -68,9 +72,14 @@ struct komeda_crtc {
 	 * merge into the master.
 	 */
 	struct komeda_pipeline *slave;
+
+	/** @disable_done: this flip_done is for tracing the disable */
+	struct completion *disable_done;
 };
 
-/** struct komeda_crtc_state */
+/**
+ * struct komeda_crtc_state
+ */
 struct komeda_crtc_state {
 	/** @base: &drm_crtc_state */
 	struct drm_crtc_state base;
@@ -78,7 +87,15 @@ struct komeda_crtc_state {
 	/* private properties */
 
 	/* computed state which are used by validate/check */
+	/**
+	 * @affected_pipes:
+	 * the affected pipelines in once display instance
+	 */
 	u32 affected_pipes;
+	/**
+	 * @active_pipes:
+	 * the active pipelines in once display instance
+	 */
 	u32 active_pipes;
 };
 
@@ -106,7 +123,10 @@ int komeda_kms_add_crtcs(struct komeda_kms_dev *kms, struct komeda_dev *mdev);
 int komeda_kms_add_planes(struct komeda_kms_dev *kms, struct komeda_dev *mdev);
 int komeda_kms_add_private_objs(struct komeda_kms_dev *kms,
 				struct komeda_dev *mdev);
-void komeda_kms_cleanup_private_objs(struct komeda_dev *mdev);
+void komeda_kms_cleanup_private_objs(struct komeda_kms_dev *kms);
+
+void komeda_crtc_handle_event(struct komeda_crtc   *kcrtc,
+			      struct komeda_events *evts);
 
 struct komeda_kms_dev *komeda_kms_attach(struct komeda_dev *mdev);
 void komeda_kms_detach(struct komeda_kms_dev *kms);
diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_pipeline.c b/drivers/gpu/drm/arm/display/komeda/komeda_pipeline.c
index f1908e9ef128..c379439c6194 100644
--- a/drivers/gpu/drm/arm/display/komeda/komeda_pipeline.c
+++ b/drivers/gpu/drm/arm/display/komeda/komeda_pipeline.c
@@ -19,17 +19,17 @@ komeda_pipeline_add(struct komeda_dev *mdev, size_t size,
 	if (mdev->n_pipelines + 1 > KOMEDA_MAX_PIPELINES) {
 		DRM_ERROR("Exceed max support %d pipelines.\n",
 			  KOMEDA_MAX_PIPELINES);
-		return NULL;
+		return ERR_PTR(-ENOSPC);
 	}
 
 	if (size < sizeof(*pipe)) {
 		DRM_ERROR("Request pipeline size too small.\n");
-		return NULL;
+		return ERR_PTR(-EINVAL);
 	}
 
 	pipe = devm_kzalloc(mdev->dev, size, GFP_KERNEL);
 	if (!pipe)
-		return NULL;
+		return ERR_PTR(-ENOMEM);
 
 	pipe->mdev = mdev;
 	pipe->id   = mdev->n_pipelines;
@@ -62,7 +62,7 @@ void komeda_pipeline_destroy(struct komeda_dev *mdev,
 	devm_kfree(mdev->dev, pipe);
 }
 
-struct komeda_component **
+static struct komeda_component **
 komeda_pipeline_get_component_pos(struct komeda_pipeline *pipe, int id)
 {
 	struct komeda_dev *mdev = pipe->mdev;
@@ -142,32 +142,32 @@ komeda_component_add(struct komeda_pipeline *pipe,
 	if (max_active_inputs > KOMEDA_COMPONENT_N_INPUTS) {
 		WARN(1, "please large KOMEDA_COMPONENT_N_INPUTS to %d.\n",
 		     max_active_inputs);
-		return NULL;
+		return ERR_PTR(-ENOSPC);
 	}
 
 	pos = komeda_pipeline_get_component_pos(pipe, id);
 	if (!pos || (*pos))
-		return NULL;
+		return ERR_PTR(-EINVAL);
 
 	if (has_bit(id, KOMEDA_PIPELINE_LAYERS)) {
 		idx = id - KOMEDA_COMPONENT_LAYER0;
 		num = &pipe->n_layers;
 		if (idx != pipe->n_layers) {
 			DRM_ERROR("please add Layer by id sequence.\n");
-			return NULL;
+			return ERR_PTR(-EINVAL);
 		}
 	} else if (has_bit(id,  KOMEDA_PIPELINE_SCALERS)) {
 		idx = id - KOMEDA_COMPONENT_SCALER0;
 		num = &pipe->n_scalers;
 		if (idx != pipe->n_scalers) {
 			DRM_ERROR("please add Scaler by id sequence.\n");
-			return NULL;
+			return ERR_PTR(-EINVAL);
 		}
 	}
 
 	c = devm_kzalloc(pipe->mdev->dev, comp_sz, GFP_KERNEL);
 	if (!c)
-		return NULL;
+		return ERR_PTR(-ENOMEM);
 
 	c->id = id;
 	c->hw_id = hw_id;
@@ -200,3 +200,98 @@ void komeda_component_destroy(struct komeda_dev *mdev,
 {
 	devm_kfree(mdev->dev, c);
 }
+
+static void komeda_component_dump(struct komeda_component *c)
+{
+	if (!c)
+		return;
+
+	DRM_DEBUG("	%s: ID %d-0x%08lx.\n",
+		  c->name, c->id, BIT(c->id));
+	DRM_DEBUG("		max_active_inputs:%d, supported_inputs: 0x%08x.\n",
+		  c->max_active_inputs, c->supported_inputs);
+	DRM_DEBUG("		max_active_outputs:%d, supported_outputs: 0x%08x.\n",
+		  c->max_active_outputs, c->supported_outputs);
+}
+
+static void komeda_pipeline_dump(struct komeda_pipeline *pipe)
+{
+	struct komeda_component *c;
+	int id;
+
+	DRM_INFO("Pipeline-%d: n_layers: %d, n_scalers: %d, output: %s\n",
+		 pipe->id, pipe->n_layers, pipe->n_scalers,
+		 pipe->of_output_dev ? pipe->of_output_dev->full_name : "none");
+
+	dp_for_each_set_bit(id, pipe->avail_comps) {
+		c = komeda_pipeline_get_component(pipe, id);
+
+		komeda_component_dump(c);
+	}
+}
+
+static void komeda_component_verify_inputs(struct komeda_component *c)
+{
+	struct komeda_pipeline *pipe = c->pipeline;
+	struct komeda_component *input;
+	int id;
+
+	dp_for_each_set_bit(id, c->supported_inputs) {
+		input = komeda_pipeline_get_component(pipe, id);
+		if (!input) {
+			c->supported_inputs &= ~(BIT(id));
+			DRM_WARN("Can not find input(ID-%d) for component: %s.\n",
+				 id, c->name);
+			continue;
+		}
+
+		input->supported_outputs |= BIT(c->id);
+	}
+}
+
+static void komeda_pipeline_assemble(struct komeda_pipeline *pipe)
+{
+	struct komeda_component *c;
+	int id;
+
+	dp_for_each_set_bit(id, pipe->avail_comps) {
+		c = komeda_pipeline_get_component(pipe, id);
+
+		komeda_component_verify_inputs(c);
+	}
+}
+
+int komeda_assemble_pipelines(struct komeda_dev *mdev)
+{
+	struct komeda_pipeline *pipe;
+	int i;
+
+	for (i = 0; i < mdev->n_pipelines; i++) {
+		pipe = mdev->pipelines[i];
+
+		komeda_pipeline_assemble(pipe);
+		komeda_pipeline_dump(pipe);
+	}
+
+	return 0;
+}
+
+void komeda_pipeline_dump_register(struct komeda_pipeline *pipe,
+				   struct seq_file *sf)
+{
+	struct komeda_component *c;
+	u32 id;
+
+	seq_printf(sf, "\n======== Pipeline-%d ==========\n", pipe->id);
+
+	if (pipe->funcs && pipe->funcs->dump_register)
+		pipe->funcs->dump_register(pipe, sf);
+
+	dp_for_each_set_bit(id, pipe->avail_comps) {
+		c = komeda_pipeline_get_component(pipe, id);
+
+		seq_printf(sf, "\n------%s------\n", c->name);
+		if (c->funcs->dump_register)
+			c->funcs->dump_register(c, sf);
+	}
+}
diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_pipeline.h b/drivers/gpu/drm/arm/display/komeda/komeda_pipeline.h
index 8c950bc8ae96..b1f813a349a4 100644
--- a/drivers/gpu/drm/arm/display/komeda/komeda_pipeline.h
+++ b/drivers/gpu/drm/arm/display/komeda/komeda_pipeline.h
@@ -90,32 +90,35 @@ struct komeda_component {
 	u32 __iomem *reg;
 	/** @id: component id */
 	u32 id;
-	/** @hw_ic: component hw id,
-	 *  which is initialized by chip and used by chip only
+	/**
+	 * @hw_id: component hw id,
+	 * which is initialized by chip and used by chip only
 	 */
 	u32 hw_id;
 
 	/**
 	 * @max_active_inputs:
-	 * @max_active_outpus:
+	 * @max_active_outputs:
 	 *
-	 * maximum number of inputs/outputs that can be active in the same time
+	 * maximum number of inputs/outputs that can be active at the same time
 	 * Note:
 	 * the number isn't the bit number of @supported_inputs or
 	 * @supported_outputs, but may be less than it, since component may not
 	 * support enabling all @supported_inputs/outputs at the same time.
 	 */
 	u8 max_active_inputs;
+	/** @max_active_outputs: maximum number of outputs */
 	u8 max_active_outputs;
 	/**
 	 * @supported_inputs:
 	 * @supported_outputs:
 	 *
-	 * bitmask of BIT(component->id) for the supported inputs/outputs
+	 * bitmask of BIT(component->id) for the supported inputs/outputs,
 	 * describes the possibilities of how a component is linked into a
 	 * pipeline.
 	 */
 	u32 supported_inputs;
+	/** @supported_outputs: bitmask of supported output componenet ids */
 	u32 supported_outputs;
 
 	/**
@@ -134,7 +137,8 @@ struct komeda_component {
 struct komeda_component_output {
 	/** @component: indicate which component the data comes from */
 	struct komeda_component *component;
-	/** @output_port:
+	/**
+	 * @output_port:
 	 * the output port of the &komeda_component_output.component
 	 */
 	u8 output_port;
@@ -150,11 +154,12 @@ struct komeda_component_output {
 struct komeda_component_state {
 	/** @obj: tracking component_state by drm_atomic_state */
 	struct drm_private_state obj;
+	/** @component: backpointer to the component */
 	struct komeda_component *component;
 	/**
 	 * @binding_user:
-	 * currently bound user, the user can be crtc/plane/wb_conn, which is
-	 * valid decided by @component and @inputs
+	 * currently bound user, the user can be @crtc, @plane or @wb_conn,
+	 * which is valid decided by @component and @inputs
 	 *
 	 * -  Layer: its user always is plane.
 	 * -  compiz/improc/timing_ctrlr: the user is crtc.
@@ -162,20 +167,24 @@ struct komeda_component_state {
 	 * -  scaler: plane when input is layer, wb_conn if input is compiz.
 	 */
 	union {
+		/** @crtc: backpointer for user crtc */
 		struct drm_crtc *crtc;
+		/** @plane: backpointer for user plane */
 		struct drm_plane *plane;
+		/** @wb_conn: backpointer for user wb_connector  */
 		struct drm_connector *wb_conn;
 		void *binding_user;
 	};
+
 	/**
 	 * @active_inputs:
 	 *
 	 * active_inputs is bitmask of @inputs index
 	 *
-	 * -  active_inputs = changed_active_inputs + unchanged_active_inputs
-	 * -  affected_inputs = old->active_inputs + new->active_inputs;
+	 * -  active_inputs = changed_active_inputs | unchanged_active_inputs
+	 * -  affected_inputs = old->active_inputs | new->active_inputs;
 	 * -  disabling_inputs = affected_inputs ^ active_inputs;
-	 * -  changed_inputs = disabling_inputs + changed_active_inputs;
+	 * -  changed_inputs = disabling_inputs | changed_active_inputs;
 	 *
 	 * NOTE:
 	 * changed_inputs doesn't include all active_input but only
@@ -183,7 +192,9 @@ struct komeda_component_state {
 	 * level for dirty update.
 	 */
 	u16 active_inputs;
+	/** @changed_active_inputs: bitmask of the changed @active_inputs */
 	u16 changed_active_inputs;
+	/** @affected_inputs: bitmask for affected @inputs */
 	u16 affected_inputs;
 	/**
 	 * @inputs:
@@ -204,57 +215,96 @@ static inline u16 component_changed_inputs(struct komeda_component_state *st)
 	return component_disabling_inputs(st) | st->changed_active_inputs;
 }
 
+#define for_each_changed_input(st, i)	\
+	for ((i) = 0; (i) < (st)->component->max_active_inputs; (i)++)	\
+		if (has_bit((i), component_changed_inputs(st)))
+
 #define to_comp(__c)	(((__c) == NULL) ? NULL : &((__c)->base))
 #define to_cpos(__c)	((struct komeda_component **)&(__c))
 
-/* these structures are going to be filled in in uture patches */
 struct komeda_layer {
 	struct komeda_component base;
-	/* layer specific features and caps */
-	int layer_type; /* RICH, SIMPLE or WB */
+	/* accepted h/v input range before rotation */
+	struct malidp_range hsize_in, vsize_in;
+	u32 layer_type; /* RICH, SIMPLE or WB */
+	u32 supported_rots;
 };
 
 struct komeda_layer_state {
 	struct komeda_component_state base;
 	/* layer specific configuration state */
+	u16 hsize, vsize;
+	u32 rot;
+	dma_addr_t addr[3];
 };
 
-struct komeda_compiz {
+struct komeda_scaler {
 	struct komeda_component base;
-	/* compiz specific features and caps */
+	/* scaler features and caps */
 };
 
-struct komeda_compiz_state {
+struct komeda_scaler_state {
 	struct komeda_component_state base;
-	/* compiz specific configuration state */
 };
 
-struct komeda_scaler {
+struct komeda_compiz {
 	struct komeda_component base;
-	/* scaler features and caps */
+	struct malidp_range hsize, vsize;
 };
 
-struct komeda_scaler_state {
+struct komeda_compiz_input_cfg {
+	u16 hsize, vsize;
+	u16 hoffset, voffset;
+	u8 pixel_blend_mode, layer_alpha;
+};
+
+struct komeda_compiz_state {
 	struct komeda_component_state base;
+	/* composition size */
+	u16 hsize, vsize;
+	struct komeda_compiz_input_cfg cins[KOMEDA_COMPONENT_N_INPUTS];
 };
 
 struct komeda_improc {
 	struct komeda_component base;
+	u32 supported_color_formats;  /* DRM_RGB/YUV444/YUV420*/
+	u32 supported_color_depths; /* BIT(8) | BIT(10)*/
+	u8 supports_degamma : 1;
+	u8 supports_csc : 1;
+	u8 supports_gamma : 1;
 };
 
 struct komeda_improc_state {
 	struct komeda_component_state base;
+	u16 hsize, vsize;
 };
 
 /* display timing controller */
 struct komeda_timing_ctrlr {
 	struct komeda_component base;
+	u8 supports_dual_link : 1;
 };
 
 struct komeda_timing_ctrlr_state {
 	struct komeda_component_state base;
 };
 
+/* Why define A separated structure but not use plane_state directly ?
+ * 1. Komeda supports layer_split which means a plane_state can be split and
+ *    handled by two layers, one layer only handle half of plane image.
+ * 2. Fix up the user properties according to HW's capabilities, like user
+ *    set rotation to R180, but HW only supports REFLECT_X+Y. the rot here is
+ *    after drm_rotation_simplify()
+ */
+struct komeda_data_flow_cfg {
+	struct komeda_component_output input;
+	u16 in_x, in_y, in_w, in_h;
+	u32 out_x, out_y, out_w, out_h;
+	u32 rot;
+	int blending_zorder;
+	u8 pixel_blend_mode, layer_alpha;
+};
+
 /** struct komeda_pipeline_funcs */
 struct komeda_pipeline_funcs {
 	/* dump_register: Optional, dump registers to seq_file */
@@ -280,14 +330,23 @@ struct komeda_pipeline {
 	int id;
 	/** @avail_comps: available components mask of pipeline */
 	u32 avail_comps;
+	/** @n_layers: the number of layer on @layers */
 	int n_layers;
+	/** @layers: the pipeline layers */
 	struct komeda_layer *layers[KOMEDA_PIPELINE_MAX_LAYERS];
+	/** @n_scalers: the number of scaler on @scalers */
 	int n_scalers;
+	/** @scalers: the pipeline scalers */
 	struct komeda_scaler *scalers[KOMEDA_PIPELINE_MAX_SCALERS];
+	/** @compiz: compositor */
 	struct komeda_compiz *compiz;
+	/** @wb_layer: writeback layer */
 	struct komeda_layer  *wb_layer;
+	/** @improc: post image processor */
 	struct komeda_improc *improc;
+	/** @ctrlr: timing controller */
 	struct komeda_timing_ctrlr *ctrlr;
+	/** @funcs: chip pipeline functions */
 	struct komeda_pipeline_funcs *funcs; /* private pipeline functions */
 
 	/** @of_node: pipeline dt node */
@@ -308,6 +367,7 @@ struct komeda_pipeline {
 struct komeda_pipeline_state {
 	/** @obj: tracking pipeline_state by drm_atomic_state */
 	struct drm_private_state obj;
+	/** @pipe: backpointer to the pipeline */
 	struct komeda_pipeline *pipe;
 	/** @crtc: currently bound crtc */
 	struct drm_crtc *crtc;
@@ -340,10 +400,13 @@ komeda_pipeline_add(struct komeda_dev *mdev, size_t size,
 		    struct komeda_pipeline_funcs *funcs);
 void komeda_pipeline_destroy(struct komeda_dev *mdev,
 			     struct komeda_pipeline *pipe);
-
+int komeda_assemble_pipelines(struct komeda_dev *mdev);
 struct komeda_component *
 komeda_pipeline_get_component(struct komeda_pipeline *pipe, int id);
 
+void komeda_pipeline_dump_register(struct komeda_pipeline *pipe,
+				   struct seq_file *sf);
+
 /* component APIs */
 struct komeda_component *
 komeda_component_add(struct komeda_pipeline *pipe,
@@ -356,4 +419,26 @@ komeda_component_add(struct komeda_pipeline *pipe,
 void komeda_component_destroy(struct komeda_dev *mdev,
 			      struct komeda_component *c);
 
+struct komeda_plane_state;
+struct komeda_crtc_state;
+struct komeda_crtc;
+
+int komeda_build_layer_data_flow(struct komeda_layer *layer,
+				 struct komeda_plane_state *kplane_st,
+				 struct komeda_crtc_state *kcrtc_st,
+				 struct komeda_data_flow_cfg *dflow);
+int komeda_build_display_data_flow(struct komeda_crtc *kcrtc,
+				   struct komeda_crtc_state *kcrtc_st);
+
+int komeda_release_unclaimed_resources(struct komeda_pipeline *pipe,
+				       struct komeda_crtc_state *kcrtc_st);
+
+struct komeda_pipeline_state *
+komeda_pipeline_get_old_state(struct komeda_pipeline *pipe,
+			      struct drm_atomic_state *state);
+void komeda_pipeline_disable(struct komeda_pipeline *pipe,
+			     struct drm_atomic_state *old_state);
+void komeda_pipeline_update(struct komeda_pipeline *pipe,
+			    struct drm_atomic_state *old_state);
+
 #endif /* _KOMEDA_PIPELINE_H_*/
diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_pipeline_state.c b/drivers/gpu/drm/arm/display/komeda/komeda_pipeline_state.c
new file mode 100644
index 000000000000..36570d7dad61
--- /dev/null
+++ b/drivers/gpu/drm/arm/display/komeda/komeda_pipeline_state.c
@@ -0,0 +1,610 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * (C) COPYRIGHT 2018 ARM Limited. All rights reserved.
+ * Author: James.Qian.Wang <james.qian.wang@arm.com>
+ *
+ */
+
+#include <drm/drm_print.h>
+#include <linux/clk.h>
+#include "komeda_dev.h"
+#include "komeda_kms.h"
+#include "komeda_pipeline.h"
+#include "komeda_framebuffer.h"
+
+static inline bool is_switching_user(void *old, void *new)
+{
+	if (!old || !new)
+		return false;
+
+	return old != new;
+}
+
+static struct komeda_pipeline_state *
+komeda_pipeline_get_state(struct komeda_pipeline *pipe,
+			  struct drm_atomic_state *state)
+{
+	struct drm_private_state *priv_st;
+
+	priv_st = drm_atomic_get_private_obj_state(state, &pipe->obj);
+	if (IS_ERR(priv_st))
+		return ERR_CAST(priv_st);
+
+	return priv_to_pipe_st(priv_st);
+}
+
+struct komeda_pipeline_state *
+komeda_pipeline_get_old_state(struct komeda_pipeline *pipe,
+			      struct drm_atomic_state *state)
+{
+	struct drm_private_state *priv_st;
+
+	priv_st = drm_atomic_get_old_private_obj_state(state, &pipe->obj);
+	if (priv_st)
+		return priv_to_pipe_st(priv_st);
+	return NULL;
+}
+
+static struct komeda_pipeline_state *
+komeda_pipeline_get_new_state(struct komeda_pipeline *pipe,
+			      struct drm_atomic_state *state)
+{
+	struct drm_private_state *priv_st;
+
+	priv_st = drm_atomic_get_new_private_obj_state(state, &pipe->obj);
+	if (priv_st)
+		return priv_to_pipe_st(priv_st);
+	return NULL;
+}
+
+/* Assign pipeline for crtc */
+static struct komeda_pipeline_state *
+komeda_pipeline_get_state_and_set_crtc(struct komeda_pipeline *pipe,
+				       struct drm_atomic_state *state,
+				       struct drm_crtc *crtc)
+{
+	struct komeda_pipeline_state *st;
+
+	st = komeda_pipeline_get_state(pipe, state);
+	if (IS_ERR(st))
+		return st;
+
+	if (is_switching_user(crtc, st->crtc)) {
+		DRM_DEBUG_ATOMIC("CRTC%d required pipeline%d is busy.\n",
+				 drm_crtc_index(crtc), pipe->id);
+		return ERR_PTR(-EBUSY);
+	}
+
+	/* pipeline only can be disabled when the it is free or unused */
+	if (!crtc && st->active_comps) {
+		DRM_DEBUG_ATOMIC("Disabling a busy pipeline:%d.\n", pipe->id);
+		return ERR_PTR(-EBUSY);
+	}
+
+	st->crtc = crtc;
+
+	if (crtc) {
+		struct komeda_crtc_state *kcrtc_st;
+
+		kcrtc_st = to_kcrtc_st(drm_atomic_get_new_crtc_state(state,
+								     crtc));
+
+		kcrtc_st->active_pipes |= BIT(pipe->id);
+		kcrtc_st->affected_pipes |= BIT(pipe->id);
+	}
+	return st;
+}
+
+static struct komeda_component_state *
+komeda_component_get_state(struct komeda_component *c,
+			   struct drm_atomic_state *state)
+{
+	struct drm_private_state *priv_st;
+
+	WARN_ON(!drm_modeset_is_locked(&c->pipeline->obj.lock));
+
+	priv_st = drm_atomic_get_private_obj_state(state, &c->obj);
+	if (IS_ERR(priv_st))
+		return ERR_CAST(priv_st);
+
+	return priv_to_comp_st(priv_st);
+}
+
+static struct komeda_component_state *
+komeda_component_get_old_state(struct komeda_component *c,
+			       struct drm_atomic_state *state)
+{
+	struct drm_private_state *priv_st;
+
+	priv_st = drm_atomic_get_old_private_obj_state(state, &c->obj);
+	if (priv_st)
+		return priv_to_comp_st(priv_st);
+	return NULL;
+}
+
+/**
+ * komeda_component_get_state_and_set_user()
+ *
+ * @c: component to get state and set user
+ * @state: global atomic state
+ * @user: direct user, the binding user
+ * @crtc: the CRTC user, the big boss :)
+ *
+ * This function accepts two users:
+ * -   The direct user: can be plane/crtc/wb_connector depends on component
+ * -   The big boss (CRTC)
+ * CRTC is the big boss (the final user), because all component resources
+ * eventually will be assigned to CRTC, like the layer will be binding to
+ * kms_plane, but kms plane will be binding to a CRTC eventually.
+ *
+ * The big boss (CRTC) is for pipeline assignment, since &komeda_component isn't
+ * independent and can be assigned to CRTC freely, but belongs to a specific
+ * pipeline, only pipeline can be shared between crtc, and pipeline as a whole
+ * (include all the internal components) assigned to a specific CRTC.
+ *
+ * So when set a user to komeda_component, need first to check the status of
+ * component->pipeline to see if the pipeline is available on this specific
+ * CRTC. if the pipeline is busy (assigned to another CRTC), even the required
+ * component is free, the component still cannot be assigned to the direct user.
+ */
+static struct komeda_component_state *
+komeda_component_get_state_and_set_user(struct komeda_component *c,
+					struct drm_atomic_state *state,
+					void *user,
+					struct drm_crtc *crtc)
+{
+	struct komeda_pipeline_state *pipe_st;
+	struct komeda_component_state *st;
+
+	/* First check if the pipeline is available */
+	pipe_st = komeda_pipeline_get_state_and_set_crtc(c->pipeline,
+							 state, crtc);
+	if (IS_ERR(pipe_st))
+		return ERR_CAST(pipe_st);
+
+	st = komeda_component_get_state(c, state);
+	if (IS_ERR(st))
+		return st;
+
+	/* check if the component has been occupied */
+	if (is_switching_user(user, st->binding_user)) {
+		DRM_DEBUG_ATOMIC("required %s is busy.\n", c->name);
+		return ERR_PTR(-EBUSY);
+	}
+
+	st->binding_user = user;
+	/* mark the component as active if user is valid */
+	if (st->binding_user)
+		pipe_st->active_comps |= BIT(c->id);
+
+	return st;
+}
+
+static void
+komeda_component_add_input(struct komeda_component_state *state,
+			   struct komeda_component_output *input,
+			   int idx)
+{
+	struct komeda_component *c = state->component;
+
+	WARN_ON((idx < 0 || idx >= c->max_active_inputs));
+
+	/* since the inputs[i] is only valid when it is active. So if a input[i]
+	 * is a newly enabled input which switches from disable to enable, then
+	 * the old inputs[i] is undefined (NOT zeroed), we can not rely on
+	 * memcmp, but directly mark it changed
+	 */
+	if (!has_bit(idx, state->affected_inputs) ||
+	    memcmp(&state->inputs[idx], input, sizeof(*input))) {
+		memcpy(&state->inputs[idx], input, sizeof(*input));
+		state->changed_active_inputs |= BIT(idx);
+	}
+	state->active_inputs |= BIT(idx);
+	state->affected_inputs |= BIT(idx);
+}
+
+static int
+komeda_component_check_input(struct komeda_component_state *state,
+			     struct komeda_component_output *input,
+			     int idx)
+{
+	struct komeda_component *c = state->component;
+
+	if ((idx < 0) || (idx >= c->max_active_inputs)) {
+		DRM_DEBUG_ATOMIC("%s invalid input id: %d.\n", c->name, idx);
+		return -EINVAL;
+	}
+
+	if (has_bit(idx, state->active_inputs)) {
+		DRM_DEBUG_ATOMIC("%s required input_id: %d has been occupied already.\n",
+				 c->name, idx);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static void
+komeda_component_set_output(struct komeda_component_output *output,
+			    struct komeda_component *comp,
+			    u8 output_port)
+{
+	output->component = comp;
+	output->output_port = output_port;
+}
+
+static int
+komeda_component_validate_private(struct komeda_component *c,
+				  struct komeda_component_state *st)
+{
+	int err;
+
+	if (!c->funcs->validate)
+		return 0;
+
+	err = c->funcs->validate(c, st);
+	if (err)
+		DRM_DEBUG_ATOMIC("%s validate private failed.\n", c->name);
+
+	return err;
+}
+
+static int
+komeda_layer_check_cfg(struct komeda_layer *layer,
+		       struct komeda_plane_state *kplane_st,
+		       struct komeda_data_flow_cfg *dflow)
+{
+	if (!in_range(&layer->hsize_in, dflow->in_w)) {
+		DRM_DEBUG_ATOMIC("src_w: %d is out of range.\n", dflow->in_w);
+		return -EINVAL;
+	}
+
+	if (!in_range(&layer->vsize_in, dflow->in_h)) {
+		DRM_DEBUG_ATOMIC("src_h: %d is out of range.\n", dflow->in_h);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+komeda_layer_validate(struct komeda_layer *layer,
+		      struct komeda_plane_state *kplane_st,
+		      struct komeda_data_flow_cfg *dflow)
+{
+	struct drm_plane_state *plane_st = &kplane_st->base;
+	struct drm_framebuffer *fb = plane_st->fb;
+	struct komeda_fb *kfb = to_kfb(fb);
+	struct komeda_component_state *c_st;
+	struct komeda_layer_state *st;
+	int i, err;
+
+	err = komeda_layer_check_cfg(layer, kplane_st, dflow);
+	if (err)
+		return err;
+
+	c_st = komeda_component_get_state_and_set_user(&layer->base,
+			plane_st->state, plane_st->plane, plane_st->crtc);
+	if (IS_ERR(c_st))
+		return PTR_ERR(c_st);
+
+	st = to_layer_st(c_st);
+
+	st->rot = dflow->rot;
+	st->hsize = kfb->aligned_w;
+	st->vsize = kfb->aligned_h;
+
+	for (i = 0; i < fb->format->num_planes; i++)
+		st->addr[i] = komeda_fb_get_pixel_addr(kfb, dflow->in_x,
+						       dflow->in_y, i);
+
+	err = komeda_component_validate_private(&layer->base, c_st);
+	if (err)
+		return err;
+
+	/* update the data flow for the next stage */
+	komeda_component_set_output(&dflow->input, &layer->base, 0);
+
+	return 0;
+}
+
+static void pipeline_composition_size(struct komeda_crtc_state *kcrtc_st,
+				      u16 *hsize, u16 *vsize)
+{
+	struct drm_display_mode *m = &kcrtc_st->base.adjusted_mode;
+
+	if (hsize)
+		*hsize = m->hdisplay;
+	if (vsize)
+		*vsize = m->vdisplay;
+}
+
+static int
+komeda_compiz_set_input(struct komeda_compiz *compiz,
+			struct komeda_crtc_state *kcrtc_st,
+			struct komeda_data_flow_cfg *dflow)
+{
+	struct drm_atomic_state *drm_st = kcrtc_st->base.state;
+	struct komeda_component_state *c_st, *old_st;
+	struct komeda_compiz_input_cfg *cin;
+	u16 compiz_w, compiz_h;
+	int idx = dflow->blending_zorder;
+
+	pipeline_composition_size(kcrtc_st, &compiz_w, &compiz_h);
+	/* check display rect */
+	if ((dflow->out_x + dflow->out_w > compiz_w) ||
+	    (dflow->out_y + dflow->out_h > compiz_h) ||
+	     dflow->out_w == 0 || dflow->out_h == 0) {
+		DRM_DEBUG_ATOMIC("invalid disp rect [x=%d, y=%d, w=%d, h=%d]\n",
+				 dflow->out_x, dflow->out_y,
+				 dflow->out_w, dflow->out_h);
+		return -EINVAL;
+	}
+
+	c_st = komeda_component_get_state_and_set_user(&compiz->base, drm_st,
+			kcrtc_st->base.crtc, kcrtc_st->base.crtc);
+	if (IS_ERR(c_st))
+		return PTR_ERR(c_st);
+
+	if (komeda_component_check_input(c_st, &dflow->input, idx))
+		return -EINVAL;
+
+	cin = &(to_compiz_st(c_st)->cins[idx]);
+
+	cin->hsize   = dflow->out_w;
+	cin->vsize   = dflow->out_h;
+	cin->hoffset = dflow->out_x;
+	cin->voffset = dflow->out_y;
+	cin->pixel_blend_mode = dflow->pixel_blend_mode;
+	cin->layer_alpha = dflow->layer_alpha;
+
+	old_st = komeda_component_get_old_state(&compiz->base, drm_st);
+	WARN_ON(!old_st);
+
+	/* compare with old to check if this input has been changed */
+	if (memcmp(&(to_compiz_st(old_st)->cins[idx]), cin, sizeof(*cin)))
+		c_st->changed_active_inputs |= BIT(idx);
+
+	komeda_component_add_input(c_st, &dflow->input, idx);
+
+	return 0;
+}
+
+static int
+komeda_compiz_validate(struct komeda_compiz *compiz,
+		       struct komeda_crtc_state *state,
+		       struct komeda_data_flow_cfg *dflow)
+{
+	struct komeda_component_state *c_st;
+	struct komeda_compiz_state *st;
+
+	c_st = komeda_component_get_state_and_set_user(&compiz->base,
+			state->base.state, state->base.crtc, state->base.crtc);
+	if (IS_ERR(c_st))
+		return PTR_ERR(c_st);
+
+	st = to_compiz_st(c_st);
+
+	pipeline_composition_size(state, &st->hsize, &st->vsize);
+
+	komeda_component_set_output(&dflow->input, &compiz->base, 0);
+
+	/* compiz output dflow will be fed to the next pipeline stage, prepare
+	 * the data flow configuration for the next stage
+	 */
+	if (dflow) {
+		dflow->in_w = st->hsize;
+		dflow->in_h = st->vsize;
+		dflow->out_w = dflow->in_w;
+		dflow->out_h = dflow->in_h;
+		/* the output data of compiz doesn't have alpha, it only can be
+		 * used as bottom layer when blend it with master layers
+		 */
+		dflow->pixel_blend_mode = DRM_MODE_BLEND_PIXEL_NONE;
+		dflow->layer_alpha = 0xFF;
+		dflow->blending_zorder = 0;
+	}
+
+	return 0;
+}
+
+static int
+komeda_improc_validate(struct komeda_improc *improc,
+		       struct komeda_crtc_state *kcrtc_st,
+		       struct komeda_data_flow_cfg *dflow)
+{
+	struct drm_crtc *crtc = kcrtc_st->base.crtc;
+	struct komeda_component_state *c_st;
+	struct komeda_improc_state *st;
+
+	c_st = komeda_component_get_state_and_set_user(&improc->base,
+			kcrtc_st->base.state, crtc, crtc);
+	if (IS_ERR(c_st))
+		return PTR_ERR(c_st);
+
+	st = to_improc_st(c_st);
+
+	st->hsize = dflow->in_w;
+	st->vsize = dflow->in_h;
+
+	komeda_component_add_input(&st->base, &dflow->input, 0);
+	komeda_component_set_output(&dflow->input, &improc->base, 0);
+
+	return 0;
+}
+
+static int
+komeda_timing_ctrlr_validate(struct komeda_timing_ctrlr *ctrlr,
+			     struct komeda_crtc_state *kcrtc_st,
+			     struct komeda_data_flow_cfg *dflow)
+{
+	struct drm_crtc *crtc = kcrtc_st->base.crtc;
+	struct komeda_timing_ctrlr_state *st;
+	struct komeda_component_state *c_st;
+
+	c_st = komeda_component_get_state_and_set_user(&ctrlr->base,
+			kcrtc_st->base.state, crtc, crtc);
+	if (IS_ERR(c_st))
+		return PTR_ERR(c_st);
+
+	st = to_ctrlr_st(c_st);
+
+	komeda_component_add_input(&st->base, &dflow->input, 0);
+	komeda_component_set_output(&dflow->input, &ctrlr->base, 0);
+
+	return 0;
+}
+
+int komeda_build_layer_data_flow(struct komeda_layer *layer,
+				 struct komeda_plane_state *kplane_st,
+				 struct komeda_crtc_state *kcrtc_st,
+				 struct komeda_data_flow_cfg *dflow)
+{
+	struct drm_plane *plane = kplane_st->base.plane;
+	struct komeda_pipeline *pipe = layer->base.pipeline;
+	int err;
+
+	DRM_DEBUG_ATOMIC("%s handling [PLANE:%d:%s]: src[x/y:%d/%d, w/h:%d/%d] disp[x/y:%d/%d, w/h:%d/%d]",
+			 layer->base.name, plane->base.id, plane->name,
+			 dflow->in_x, dflow->in_y, dflow->in_w, dflow->in_h,
+			 dflow->out_x, dflow->out_y, dflow->out_w, dflow->out_h);
+
+	err = komeda_layer_validate(layer, kplane_st, dflow);
+	if (err)
+		return err;
+
+	err = komeda_compiz_set_input(pipe->compiz, kcrtc_st, dflow);
+
+	return err;
+}
+
+/* build display output data flow, the data path is:
+ * compiz -> improc -> timing_ctrlr
+ */
+int komeda_build_display_data_flow(struct komeda_crtc *kcrtc,
+				   struct komeda_crtc_state *kcrtc_st)
+{
+	struct komeda_pipeline *master = kcrtc->master;
+	struct komeda_data_flow_cfg m_dflow; /* master data flow */
+	int err;
+
+	memset(&m_dflow, 0, sizeof(m_dflow));
+
+	err = komeda_compiz_validate(master->compiz, kcrtc_st, &m_dflow);
+	if (err)
+		return err;
+
+	err = komeda_improc_validate(master->improc, kcrtc_st, &m_dflow);
+	if (err)
+		return err;
+
+	err = komeda_timing_ctrlr_validate(master->ctrlr, kcrtc_st, &m_dflow);
+	if (err)
+		return err;
+
+	return 0;
+}
+
+static void
+komeda_pipeline_unbound_components(struct komeda_pipeline *pipe,
+				   struct komeda_pipeline_state *new)
+{
+	struct drm_atomic_state *drm_st = new->obj.state;
+	struct komeda_pipeline_state *old = priv_to_pipe_st(pipe->obj.state);
+	struct komeda_component_state *c_st;
+	struct komeda_component *c;
+	u32 disabling_comps, id;
+
+	WARN_ON(!old);
+
+	disabling_comps = (~new->active_comps) & old->active_comps;
+
+	/* unbound all disabling component */
+	dp_for_each_set_bit(id, disabling_comps) {
+		c = komeda_pipeline_get_component(pipe, id);
+		c_st = komeda_component_get_state_and_set_user(c,
+				drm_st, NULL, new->crtc);
+		WARN_ON(IS_ERR(c_st));
+	}
+}
+
+/* release unclaimed pipeline resource */
+int komeda_release_unclaimed_resources(struct komeda_pipeline *pipe,
+				       struct komeda_crtc_state *kcrtc_st)
+{
+	struct drm_atomic_state *drm_st = kcrtc_st->base.state;
+	struct komeda_pipeline_state *st;
+
+	/* ignore the pipeline which is not affected */
+	if (!pipe || !has_bit(pipe->id, kcrtc_st->affected_pipes))
+		return 0;
+
+	if (has_bit(pipe->id, kcrtc_st->active_pipes))
+		st = komeda_pipeline_get_new_state(pipe, drm_st);
+	else
+		st = komeda_pipeline_get_state_and_set_crtc(pipe, drm_st, NULL);
+
+	if (WARN_ON(IS_ERR_OR_NULL(st)))
+		return -EINVAL;
+
+	komeda_pipeline_unbound_components(pipe, st);
+
+	return 0;
+}
+
+void komeda_pipeline_disable(struct komeda_pipeline *pipe,
+			     struct drm_atomic_state *old_state)
+{
+	struct komeda_pipeline_state *old;
+	struct komeda_component *c;
+	struct komeda_component_state *c_st;
+	u32 id, disabling_comps = 0;
+
+	old = komeda_pipeline_get_old_state(pipe, old_state);
+
+	disabling_comps = old->active_comps;
+	DRM_DEBUG_ATOMIC("PIPE%d: disabling_comps: 0x%x.\n",
+			 pipe->id, disabling_comps);
+
+	dp_for_each_set_bit(id, disabling_comps) {
+		c = komeda_pipeline_get_component(pipe, id);
+		c_st = priv_to_comp_st(c->obj.state);
+
+		/*
+		 * If we disabled a component then all active_inputs should be
+		 * put in the list of changed_active_inputs, so they get
+		 * re-enabled.
+		 * This usually happens during a modeset when the pipeline is
+		 * first disabled and then the actual state gets committed
+		 * again.
+		 */
+		c_st->changed_active_inputs |= c_st->active_inputs;
+
+		c->funcs->disable(c);
+	}
+}
+
+void komeda_pipeline_update(struct komeda_pipeline *pipe,
+			    struct drm_atomic_state *old_state)
+{
+	struct komeda_pipeline_state *new = priv_to_pipe_st(pipe->obj.state);
+	struct komeda_pipeline_state *old;
+	struct komeda_component *c;
+	u32 id, changed_comps = 0;
+
+	old = komeda_pipeline_get_old_state(pipe, old_state);
+
+	changed_comps = new->active_comps | old->active_comps;
+
+	DRM_DEBUG_ATOMIC("PIPE%d: active_comps: 0x%x, changed: 0x%x.\n",
+			 pipe->id, new->active_comps, changed_comps);
+
+	dp_for_each_set_bit(id, changed_comps) {
+		c = komeda_pipeline_get_component(pipe, id);
+
+		if (new->active_comps & BIT(c->id))
+			c->funcs->update(c, priv_to_comp_st(c->obj.state));
+		else
+			c->funcs->disable(c);
+	}
+}
diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_plane.c b/drivers/gpu/drm/arm/display/komeda/komeda_plane.c
index 0a4953a9a909..07ed0cc1bc44 100644
--- a/drivers/gpu/drm/arm/display/komeda/komeda_plane.c
+++ b/drivers/gpu/drm/arm/display/komeda/komeda_plane.c
@@ -7,10 +7,96 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_plane_helper.h>
+#include <drm/drm_print.h>
 #include "komeda_dev.h"
 #include "komeda_kms.h"
 
+static int
+komeda_plane_init_data_flow(struct drm_plane_state *st,
+			    struct komeda_data_flow_cfg *dflow)
+{
+	struct drm_framebuffer *fb = st->fb;
+
+	memset(dflow, 0, sizeof(*dflow));
+
+	dflow->blending_zorder = st->zpos;
+
+	/* if format doesn't have alpha, fix blend mode to PIXEL_NONE */
+	dflow->pixel_blend_mode = fb->format->has_alpha ?
+		st->pixel_blend_mode : DRM_MODE_BLEND_PIXEL_NONE;
+	dflow->layer_alpha = st->alpha >> 8;
+
+	dflow->out_x = st->crtc_x;
+	dflow->out_y = st->crtc_y;
+	dflow->out_w = st->crtc_w;
+	dflow->out_h = st->crtc_h;
+
+	dflow->in_x = st->src_x >> 16;
+	dflow->in_y = st->src_y >> 16;
+	dflow->in_w = st->src_w >> 16;
+	dflow->in_h = st->src_h >> 16;
+
+	return 0;
+}
+
+/**
+ * komeda_plane_atomic_check - build input data flow
+ * @plane: DRM plane
+ * @state: the plane state object
+ *
+ * RETURNS:
+ * Zero for success or -errno
+ */
+static int
+komeda_plane_atomic_check(struct drm_plane *plane,
+			  struct drm_plane_state *state)
+{
+	struct komeda_plane *kplane = to_kplane(plane);
+	struct komeda_plane_state *kplane_st = to_kplane_st(state);
+	struct komeda_layer *layer = kplane->layer;
+	struct drm_crtc_state *crtc_st;
+	struct komeda_crtc *kcrtc;
+	struct komeda_crtc_state *kcrtc_st;
+	struct komeda_data_flow_cfg dflow;
+	int err;
+
+	if (!state->crtc || !state->fb)
+		return 0;
+
+	crtc_st = drm_atomic_get_crtc_state(state->state, state->crtc);
+	if (!crtc_st->enable) {
+		DRM_DEBUG_ATOMIC("Cannot update plane on a disabled CRTC.\n");
+		return -EINVAL;
+	}
+
+	/* crtc is inactive, skip the resource assignment */
+	if (!crtc_st->active)
+		return 0;
+
+	kcrtc = to_kcrtc(state->crtc);
+	kcrtc_st = to_kcrtc_st(crtc_st);
+
+	err = komeda_plane_init_data_flow(state, &dflow);
+	if (err)
+		return err;
+
+	err = komeda_build_layer_data_flow(layer, kplane_st, kcrtc_st, &dflow);
+
+	return err;
+}
+
+/* plane doesn't represent a real HW, so there is no HW update for plane.
+ * komeda handles all the HW update in crtc->atomic_flush
+ */
+static void
+komeda_plane_atomic_update(struct drm_plane *plane,
+			   struct drm_plane_state *old_state)
+{
+}
+
 static const struct drm_plane_helper_funcs komeda_plane_helper_funcs = {
+	.atomic_check	= komeda_plane_atomic_check,
+	.atomic_update	= komeda_plane_atomic_update,
 };
 
 static void komeda_plane_destroy(struct drm_plane *plane)
@@ -20,7 +106,60 @@ static void komeda_plane_destroy(struct drm_plane *plane)
 	kfree(to_kplane(plane));
 }
 
+static void komeda_plane_reset(struct drm_plane *plane)
+{
+	struct komeda_plane_state *state;
+	struct komeda_plane *kplane = to_kplane(plane);
+
+	if (plane->state)
+		__drm_atomic_helper_plane_destroy_state(plane->state);
+
+	kfree(plane->state);
+	plane->state = NULL;
+
+	state = kzalloc(sizeof(*state), GFP_KERNEL);
+	if (state) {
+		state->base.rotation = DRM_MODE_ROTATE_0;
+		state->base.pixel_blend_mode = DRM_MODE_BLEND_PREMULTI;
+		state->base.alpha = DRM_BLEND_ALPHA_OPAQUE;
+		state->base.zpos = kplane->layer->base.id;
+		plane->state = &state->base;
+		plane->state->plane = plane;
+	}
+}
+
+static struct drm_plane_state *
+komeda_plane_atomic_duplicate_state(struct drm_plane *plane)
+{
+	struct komeda_plane_state *new;
+
+	if (WARN_ON(!plane->state))
+		return NULL;
+
+	new = kzalloc(sizeof(*new), GFP_KERNEL);
+	if (!new)
+		return NULL;
+
+	__drm_atomic_helper_plane_duplicate_state(plane, &new->base);
+
+	return &new->base;
+}
+
+static void
+komeda_plane_atomic_destroy_state(struct drm_plane *plane,
+				  struct drm_plane_state *state)
+{
+	__drm_atomic_helper_plane_destroy_state(state);
+	kfree(to_kplane_st(state));
+}
+
 static const struct drm_plane_funcs komeda_plane_funcs = {
+	.update_plane		= drm_atomic_helper_update_plane,
+	.disable_plane		= drm_atomic_helper_disable_plane,
+	.destroy		= komeda_plane_destroy,
+	.reset			= komeda_plane_reset,
+	.atomic_duplicate_state	= komeda_plane_atomic_duplicate_state,
+	.atomic_destroy_state	= komeda_plane_atomic_destroy_state,
 };
 
 /* for komeda, which is pipeline can be share between crtcs */
diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_private_obj.c b/drivers/gpu/drm/arm/display/komeda/komeda_private_obj.c
index f1c9e3fefa86..a54878cbd6e4 100644
--- a/drivers/gpu/drm/arm/display/komeda/komeda_private_obj.c
+++ b/drivers/gpu/drm/arm/display/komeda/komeda_private_obj.c
@@ -7,6 +7,188 @@
 #include "komeda_dev.h"
 #include "komeda_kms.h"
 
+static void
+komeda_component_state_reset(struct komeda_component_state *st)
+{
+	st->binding_user = NULL;
+	st->affected_inputs = st->active_inputs;
+	st->active_inputs = 0;
+	st->changed_active_inputs = 0;
+}
+
+static struct drm_private_state *
+komeda_layer_atomic_duplicate_state(struct drm_private_obj *obj)
+{
+	struct komeda_layer_state *st;
+
+	st = kmemdup(obj->state, sizeof(*st), GFP_KERNEL);
+	if (!st)
+		return NULL;
+
+	komeda_component_state_reset(&st->base);
+	__drm_atomic_helper_private_obj_duplicate_state(obj, &st->base.obj);
+
+	return &st->base.obj;
+}
+
+static void
+komeda_layer_atomic_destroy_state(struct drm_private_obj *obj,
+				  struct drm_private_state *state)
+{
+	struct komeda_layer_state *st = to_layer_st(priv_to_comp_st(state));
+
+	kfree(st);
+}
+
+static const struct drm_private_state_funcs komeda_layer_obj_funcs = {
+	.atomic_duplicate_state	= komeda_layer_atomic_duplicate_state,
+	.atomic_destroy_state	= komeda_layer_atomic_destroy_state,
+};
+
+static int komeda_layer_obj_add(struct komeda_kms_dev *kms,
+				struct komeda_layer *layer)
+{
+	struct komeda_layer_state *st;
+
+	st = kzalloc(sizeof(*st), GFP_KERNEL);
+	if (!st)
+		return -ENOMEM;
+
+	st->base.component = &layer->base;
+	drm_atomic_private_obj_init(&kms->base, &layer->base.obj, &st->base.obj,
+				    &komeda_layer_obj_funcs);
+	return 0;
+}
+
+static struct drm_private_state *
+komeda_compiz_atomic_duplicate_state(struct drm_private_obj *obj)
+{
+	struct komeda_compiz_state *st;
+
+	st = kmemdup(obj->state, sizeof(*st), GFP_KERNEL);
+	if (!st)
+		return NULL;
+
+	komeda_component_state_reset(&st->base);
+	__drm_atomic_helper_private_obj_duplicate_state(obj, &st->base.obj);
+
+	return &st->base.obj;
+}
+
+static void
+komeda_compiz_atomic_destroy_state(struct drm_private_obj *obj,
+				   struct drm_private_state *state)
+{
+	kfree(to_compiz_st(priv_to_comp_st(state)));
+}
+
+static const struct drm_private_state_funcs komeda_compiz_obj_funcs = {
+	.atomic_duplicate_state	= komeda_compiz_atomic_duplicate_state,
+	.atomic_destroy_state	= komeda_compiz_atomic_destroy_state,
+};
+
+static int komeda_compiz_obj_add(struct komeda_kms_dev *kms,
+				 struct komeda_compiz *compiz)
+{
+	struct komeda_compiz_state *st;
+
+	st = kzalloc(sizeof(*st), GFP_KERNEL);
+	if (!st)
+		return -ENOMEM;
+
+	st->base.component = &compiz->base;
+	drm_atomic_private_obj_init(&kms->base, &compiz->base.obj, &st->base.obj,
+				    &komeda_compiz_obj_funcs);
+
+	return 0;
+}
+
+static struct drm_private_state *
+komeda_improc_atomic_duplicate_state(struct drm_private_obj *obj)
+{
+	struct komeda_improc_state *st;
+
+	st = kmemdup(obj->state, sizeof(*st), GFP_KERNEL);
+	if (!st)
+		return NULL;
+
+	komeda_component_state_reset(&st->base);
+	__drm_atomic_helper_private_obj_duplicate_state(obj, &st->base.obj);
+
+	return &st->base.obj;
+}
+
+static void
+komeda_improc_atomic_destroy_state(struct drm_private_obj *obj,
+				   struct drm_private_state *state)
+{
+	kfree(to_improc_st(priv_to_comp_st(state)));
+}
+
+static const struct drm_private_state_funcs komeda_improc_obj_funcs = {
+	.atomic_duplicate_state	= komeda_improc_atomic_duplicate_state,
+	.atomic_destroy_state	= komeda_improc_atomic_destroy_state,
+};
+
+static int komeda_improc_obj_add(struct komeda_kms_dev *kms,
+				 struct komeda_improc *improc)
+{
+	struct komeda_improc_state *st;
+
+	st = kzalloc(sizeof(*st), GFP_KERNEL);
+	if (!st)
+		return -ENOMEM;
+
+	st->base.component = &improc->base;
+	drm_atomic_private_obj_init(&kms->base, &improc->base.obj, &st->base.obj,
+				    &komeda_improc_obj_funcs);
+
+	return 0;
+}
+
+static struct drm_private_state *
+komeda_timing_ctrlr_atomic_duplicate_state(struct drm_private_obj *obj)
+{
+	struct komeda_timing_ctrlr_state *st;
+
+	st = kmemdup(obj->state, sizeof(*st), GFP_KERNEL);
+	if (!st)
+		return NULL;
+
+	komeda_component_state_reset(&st->base);
+	__drm_atomic_helper_private_obj_duplicate_state(obj, &st->base.obj);
+
+	return &st->base.obj;
+}
+
+static void
+komeda_timing_ctrlr_atomic_destroy_state(struct drm_private_obj *obj,
+					 struct drm_private_state *state)
+{
+	kfree(to_ctrlr_st(priv_to_comp_st(state)));
+}
+
+static const struct drm_private_state_funcs komeda_timing_ctrlr_obj_funcs = {
+	.atomic_duplicate_state	= komeda_timing_ctrlr_atomic_duplicate_state,
+	.atomic_destroy_state	= komeda_timing_ctrlr_atomic_destroy_state,
+};
+
+static int komeda_timing_ctrlr_obj_add(struct komeda_kms_dev *kms,
+				       struct komeda_timing_ctrlr *ctrlr)
+{
+	struct komeda_compiz_state *st;
+
+	st = kzalloc(sizeof(*st), GFP_KERNEL);
+	if (!st)
+		return -ENOMEM;
+
+	st->base.component = &ctrlr->base;
+	drm_atomic_private_obj_init(&kms->base, &ctrlr->base.obj, &st->base.obj,
+				    &komeda_timing_ctrlr_obj_funcs);
+
+	return 0;
+}
+
 static struct drm_private_state *
 komeda_pipeline_atomic_duplicate_state(struct drm_private_obj *obj)
 {
@@ -55,7 +237,7 @@ int komeda_kms_add_private_objs(struct komeda_kms_dev *kms,
 				struct komeda_dev *mdev)
 {
 	struct komeda_pipeline *pipe;
-	int i, err;
+	int i, j, err;
 
 	for (i = 0; i < mdev->n_pipelines; i++) {
 		pipe = mdev->pipelines[i];
@@ -64,25 +246,33 @@ int komeda_kms_add_private_objs(struct komeda_kms_dev *kms,
 		if (err)
 			return err;
 
-		/* Add component */
+		for (j = 0; j < pipe->n_layers; j++) {
+			err = komeda_layer_obj_add(kms, pipe->layers[j]);
+			if (err)
+				return err;
+		}
+
+		err = komeda_compiz_obj_add(kms, pipe->compiz);
+		if (err)
+			return err;
+
+		err = komeda_improc_obj_add(kms, pipe->improc);
+		if (err)
+			return err;
+
+		err = komeda_timing_ctrlr_obj_add(kms, pipe->ctrlr);
+		if (err)
+			return err;
 	}
 
 	return 0;
 }
 
-void komeda_kms_cleanup_private_objs(struct komeda_dev *mdev)
+void komeda_kms_cleanup_private_objs(struct komeda_kms_dev *kms)
 {
-	struct komeda_pipeline *pipe;
-	struct komeda_component *c;
-	int i, id;
+	struct drm_mode_config *config = &kms->base.mode_config;
+	struct drm_private_obj *obj, *next;
 
-	for (i = 0; i < mdev->n_pipelines; i++) {
-		pipe = mdev->pipelines[i];
-		dp_for_each_set_bit(id, pipe->avail_comps) {
-			c = komeda_pipeline_get_component(pipe, id);
-
-			drm_atomic_private_obj_fini(&c->obj);
-		}
-		drm_atomic_private_obj_fini(&pipe->obj);
-	}
+	list_for_each_entry_safe(obj, next, &config->privobj_list, head)
+		drm_atomic_private_obj_fini(obj);
 }
diff --git a/drivers/gpu/drm/arm/malidp_drv.c b/drivers/gpu/drm/arm/malidp_drv.c
index ab50ad06e271..21725c9b9f5e 100644
--- a/drivers/gpu/drm/arm/malidp_drv.c
+++ b/drivers/gpu/drm/arm/malidp_drv.c
@@ -264,37 +264,17 @@ static bool
 malidp_verify_afbc_framebuffer_caps(struct drm_device *dev,
 				    const struct drm_mode_fb_cmd2 *mode_cmd)
 {
-	const struct drm_format_info *info;
-
-	if ((mode_cmd->modifier[0] >> 56) != DRM_FORMAT_MOD_VENDOR_ARM) {
-		DRM_DEBUG_KMS("Unknown modifier (not Arm)\n");
-		return false;
-	}
-
-	if (mode_cmd->modifier[0] &
-	    ~DRM_FORMAT_MOD_ARM_AFBC(AFBC_MOD_VALID_BITS)) {
-		DRM_DEBUG_KMS("Unsupported modifiers\n");
-		return false;
-	}
-
-	info = drm_get_format_info(dev, mode_cmd);
-	if (!info) {
-		DRM_DEBUG_KMS("Unable to get the format information\n");
+	if (malidp_format_mod_supported(dev, mode_cmd->pixel_format,
+					mode_cmd->modifier[0]) == false)
 		return false;
-	}
-
-	if (info->num_planes != 1) {
-		DRM_DEBUG_KMS("AFBC buffers expect one plane\n");
-		return false;
-	}
 
 	if (mode_cmd->offsets[0] != 0) {
 		DRM_DEBUG_KMS("AFBC buffers' plane offset should be 0\n");
 		return false;
 	}
 
-	switch (mode_cmd->modifier[0] & AFBC_FORMAT_MOD_BLOCK_SIZE_MASK) {
-	case AFBC_FORMAT_MOD_BLOCK_SIZE_16x16:
+	switch (mode_cmd->modifier[0] & AFBC_SIZE_MASK) {
+	case AFBC_SIZE_16X16:
 		if ((mode_cmd->width % 16) || (mode_cmd->height % 16)) {
 			DRM_DEBUG_KMS("AFBC buffers must be aligned to 16 pixels\n");
 			return false;
@@ -318,9 +298,10 @@ malidp_verify_afbc_framebuffer_size(struct drm_device *dev,
 	struct drm_gem_object *objs = NULL;
 	u32 afbc_superblock_size = 0, afbc_superblock_height = 0;
 	u32 afbc_superblock_width = 0, afbc_size = 0;
+	int bpp = 0;
 
-	switch (mode_cmd->modifier[0] & AFBC_FORMAT_MOD_BLOCK_SIZE_MASK) {
-	case AFBC_FORMAT_MOD_BLOCK_SIZE_16x16:
+	switch (mode_cmd->modifier[0] & AFBC_SIZE_MASK) {
+	case AFBC_SIZE_16X16:
 		afbc_superblock_height = 16;
 		afbc_superblock_width = 16;
 		break;
@@ -334,15 +315,19 @@ malidp_verify_afbc_framebuffer_size(struct drm_device *dev,
 	n_superblocks = (mode_cmd->width / afbc_superblock_width) *
 		(mode_cmd->height / afbc_superblock_height);
 
-	afbc_superblock_size = info->cpp[0] * afbc_superblock_width *
-		afbc_superblock_height;
+	bpp = malidp_format_get_bpp(info->format);
+
+	afbc_superblock_size = (bpp * afbc_superblock_width * afbc_superblock_height)
+				/ BITS_PER_BYTE;
 
 	afbc_size = ALIGN(n_superblocks * AFBC_HEADER_SIZE, AFBC_SUPERBLK_ALIGNMENT);
 	afbc_size += n_superblocks * ALIGN(afbc_superblock_size, AFBC_SUPERBLK_ALIGNMENT);
 
-	if (mode_cmd->width * info->cpp[0] != mode_cmd->pitches[0]) {
-		DRM_DEBUG_KMS("Invalid value of pitch (=%u) should be same as width (=%u) * cpp (=%u)\n",
-			      mode_cmd->pitches[0], mode_cmd->width, info->cpp[0]);
+	if ((mode_cmd->width * bpp) != (mode_cmd->pitches[0] * BITS_PER_BYTE)) {
+		DRM_DEBUG_KMS("Invalid value of (pitch * BITS_PER_BYTE) (=%u) "
+			      "should be same as width (=%u) * bpp (=%u)\n",
+			      (mode_cmd->pitches[0] * BITS_PER_BYTE),
+			      mode_cmd->width, bpp);
 		return false;
 	}
 
@@ -406,6 +391,7 @@ static int malidp_init(struct drm_device *drm)
 	drm->mode_config.max_height = hwdev->max_line_size;
 	drm->mode_config.funcs = &malidp_mode_config_funcs;
 	drm->mode_config.helper_private = &malidp_mode_config_helpers;
+	drm->mode_config.allow_fb_modifiers = true;
 
 	ret = malidp_crtc_init(drm);
 	if (ret)
diff --git a/drivers/gpu/drm/arm/malidp_drv.h b/drivers/gpu/drm/arm/malidp_drv.h
index b76c86f18a56..019a682b2716 100644
--- a/drivers/gpu/drm/arm/malidp_drv.h
+++ b/drivers/gpu/drm/arm/malidp_drv.h
@@ -90,6 +90,12 @@ struct malidp_crtc_state {
 int malidp_de_planes_init(struct drm_device *drm);
 int malidp_crtc_init(struct drm_device *drm);
 
+bool malidp_hw_format_is_linear_only(u32 format);
+bool malidp_hw_format_is_afbc_only(u32 format);
+
+bool malidp_format_mod_supported(struct drm_device *drm,
+				 u32 format, u64 modifier);
+
 #ifdef CONFIG_DEBUG_FS
 void malidp_error(struct malidp_drm *malidp,
 		  struct malidp_error_stats *error_stats, u32 status,
diff --git a/drivers/gpu/drm/arm/malidp_hw.c b/drivers/gpu/drm/arm/malidp_hw.c
index b9bed1138fa3..8df12e9a33bb 100644
--- a/drivers/gpu/drm/arm/malidp_hw.c
+++ b/drivers/gpu/drm/arm/malidp_hw.c
@@ -49,11 +49,19 @@ static const struct malidp_format_id malidp500_de_formats[] = {
 	{ DRM_FORMAT_YUYV, DE_VIDEO1, 13 },
 	{ DRM_FORMAT_NV12, DE_VIDEO1 | SE_MEMWRITE, 14 },
 	{ DRM_FORMAT_YUV420, DE_VIDEO1, 15 },
+	{ DRM_FORMAT_XYUV8888, DE_VIDEO1, 16 },
+	/* These are supported with AFBC only */
+	{ DRM_FORMAT_YUV420_8BIT, DE_VIDEO1, 14 },
+	{ DRM_FORMAT_VUY888, DE_VIDEO1, 16 },
+	{ DRM_FORMAT_VUY101010, DE_VIDEO1, 17 },
+	{ DRM_FORMAT_YUV420_10BIT, DE_VIDEO1, 18 }
 };
 
 #define MALIDP_ID(__group, __format) \
 	((((__group) & 0x7) << 3) | ((__format) & 0x7))
 
+#define AFBC_YUV_422_FORMAT_ID	MALIDP_ID(5, 1)
+
 #define MALIDP_COMMON_FORMATS \
 	/*    fourcc,   layers supporting the format,      internal id   */ \
 	{ DRM_FORMAT_ARGB2101010, DE_VIDEO1 | DE_GRAPHICS1 | DE_VIDEO2 | SE_MEMWRITE, MALIDP_ID(0, 0) }, \
@@ -74,11 +82,25 @@ static const struct malidp_format_id malidp500_de_formats[] = {
 	{ DRM_FORMAT_ABGR1555, DE_VIDEO1 | DE_GRAPHICS1 | DE_VIDEO2, MALIDP_ID(4, 1) }, \
 	{ DRM_FORMAT_RGB565, DE_VIDEO1 | DE_GRAPHICS1 | DE_VIDEO2, MALIDP_ID(4, 2) }, \
 	{ DRM_FORMAT_BGR565, DE_VIDEO1 | DE_GRAPHICS1 | DE_VIDEO2, MALIDP_ID(4, 3) }, \
+	/* This is only supported with linear modifier */	\
+	{ DRM_FORMAT_XYUV8888, DE_VIDEO1 | DE_VIDEO2, MALIDP_ID(5, 0) },\
+	/* This is only supported with AFBC modifier */		\
+	{ DRM_FORMAT_VUY888, DE_VIDEO1 | DE_VIDEO2, MALIDP_ID(5, 0) }, \
 	{ DRM_FORMAT_YUYV, DE_VIDEO1 | DE_VIDEO2, MALIDP_ID(5, 2) },	\
+	/* This is only supported with linear modifier */ \
 	{ DRM_FORMAT_UYVY, DE_VIDEO1 | DE_VIDEO2, MALIDP_ID(5, 3) },	\
 	{ DRM_FORMAT_NV12, DE_VIDEO1 | DE_VIDEO2 | SE_MEMWRITE, MALIDP_ID(5, 6) },	\
+	/* This is only supported with AFBC modifier */ \
+	{ DRM_FORMAT_YUV420_8BIT, DE_VIDEO1 | DE_VIDEO2, MALIDP_ID(5, 6) }, \
 	{ DRM_FORMAT_YUV420, DE_VIDEO1 | DE_VIDEO2, MALIDP_ID(5, 7) }, \
-	{ DRM_FORMAT_X0L2, DE_VIDEO1 | DE_VIDEO2, MALIDP_ID(6, 6)}
+	/* This is only supported with linear modifier */ \
+	{ DRM_FORMAT_XVYU2101010, DE_VIDEO1 | DE_VIDEO2, MALIDP_ID(6, 0)}, \
+	/* This is only supported with AFBC modifier */ \
+	{ DRM_FORMAT_VUY101010, DE_VIDEO1 | DE_VIDEO2, MALIDP_ID(6, 0)}, \
+	{ DRM_FORMAT_X0L2, DE_VIDEO1 | DE_VIDEO2, MALIDP_ID(6, 6)}, \
+	/* This is only supported with AFBC modifier */ \
+	{ DRM_FORMAT_YUV420_10BIT, DE_VIDEO1 | DE_VIDEO2, MALIDP_ID(6, 7)}, \
+	{ DRM_FORMAT_P010, DE_VIDEO1 | DE_VIDEO2, MALIDP_ID(6, 7)}
 
 static const struct malidp_format_id malidp550_de_formats[] = {
 	MALIDP_COMMON_FORMATS,
@@ -94,11 +116,14 @@ static const struct malidp_layer malidp500_layers[] = {
 	 *	yuv2rgb matrix offset, mmu control register offset, rotation_features
 	 */
 	{ DE_VIDEO1, MALIDP500_DE_LV_BASE, MALIDP500_DE_LV_PTR_BASE,
-		MALIDP_DE_LV_STRIDE0, MALIDP500_LV_YUV2RGB, 0, ROTATE_ANY },
+		MALIDP_DE_LV_STRIDE0, MALIDP500_LV_YUV2RGB, 0, ROTATE_ANY,
+		MALIDP500_DE_LV_AD_CTRL },
 	{ DE_GRAPHICS1, MALIDP500_DE_LG1_BASE, MALIDP500_DE_LG1_PTR_BASE,
-		MALIDP_DE_LG_STRIDE, 0, 0, ROTATE_ANY },
+		MALIDP_DE_LG_STRIDE, 0, 0, ROTATE_ANY,
+		MALIDP500_DE_LG1_AD_CTRL },
 	{ DE_GRAPHICS2, MALIDP500_DE_LG2_BASE, MALIDP500_DE_LG2_PTR_BASE,
-		MALIDP_DE_LG_STRIDE, 0, 0, ROTATE_ANY },
+		MALIDP_DE_LG_STRIDE, 0, 0, ROTATE_ANY,
+		MALIDP500_DE_LG2_AD_CTRL },
 };
 
 static const struct malidp_layer malidp550_layers[] = {
@@ -106,13 +131,16 @@ static const struct malidp_layer malidp550_layers[] = {
 	 *	yuv2rgb matrix offset, mmu control register offset, rotation_features
 	 */
 	{ DE_VIDEO1, MALIDP550_DE_LV1_BASE, MALIDP550_DE_LV1_PTR_BASE,
-		MALIDP_DE_LV_STRIDE0, MALIDP550_LV_YUV2RGB, 0, ROTATE_ANY },
+		MALIDP_DE_LV_STRIDE0, MALIDP550_LV_YUV2RGB, 0, ROTATE_ANY,
+		MALIDP550_DE_LV1_AD_CTRL },
 	{ DE_GRAPHICS1, MALIDP550_DE_LG_BASE, MALIDP550_DE_LG_PTR_BASE,
-		MALIDP_DE_LG_STRIDE, 0, 0, ROTATE_ANY },
+		MALIDP_DE_LG_STRIDE, 0, 0, ROTATE_ANY,
+		MALIDP550_DE_LG_AD_CTRL },
 	{ DE_VIDEO2, MALIDP550_DE_LV2_BASE, MALIDP550_DE_LV2_PTR_BASE,
-		MALIDP_DE_LV_STRIDE0, MALIDP550_LV_YUV2RGB, 0, ROTATE_ANY },
+		MALIDP_DE_LV_STRIDE0, MALIDP550_LV_YUV2RGB, 0, ROTATE_ANY,
+		MALIDP550_DE_LV2_AD_CTRL },
 	{ DE_SMART, MALIDP550_DE_LS_BASE, MALIDP550_DE_LS_PTR_BASE,
-		MALIDP550_DE_LS_R1_STRIDE, 0, 0, ROTATE_NONE },
+		MALIDP550_DE_LS_R1_STRIDE, 0, 0, ROTATE_NONE, 0 },
 };
 
 static const struct malidp_layer malidp650_layers[] = {
@@ -122,16 +150,44 @@ static const struct malidp_layer malidp650_layers[] = {
 	 */
 	{ DE_VIDEO1, MALIDP550_DE_LV1_BASE, MALIDP550_DE_LV1_PTR_BASE,
 		MALIDP_DE_LV_STRIDE0, MALIDP550_LV_YUV2RGB,
-		MALIDP650_DE_LV_MMU_CTRL, ROTATE_ANY },
+		MALIDP650_DE_LV_MMU_CTRL, ROTATE_ANY,
+		MALIDP550_DE_LV1_AD_CTRL },
 	{ DE_GRAPHICS1, MALIDP550_DE_LG_BASE, MALIDP550_DE_LG_PTR_BASE,
 		MALIDP_DE_LG_STRIDE, 0, MALIDP650_DE_LG_MMU_CTRL,
-		ROTATE_COMPRESSED },
+		ROTATE_COMPRESSED, MALIDP550_DE_LG_AD_CTRL },
 	{ DE_VIDEO2, MALIDP550_DE_LV2_BASE, MALIDP550_DE_LV2_PTR_BASE,
 		MALIDP_DE_LV_STRIDE0, MALIDP550_LV_YUV2RGB,
-		MALIDP650_DE_LV_MMU_CTRL, ROTATE_ANY },
+		MALIDP650_DE_LV_MMU_CTRL, ROTATE_ANY,
+		MALIDP550_DE_LV2_AD_CTRL },
 	{ DE_SMART, MALIDP550_DE_LS_BASE, MALIDP550_DE_LS_PTR_BASE,
 		MALIDP550_DE_LS_R1_STRIDE, 0, MALIDP650_DE_LS_MMU_CTRL,
-		ROTATE_NONE },
+		ROTATE_NONE, 0 },
+};
+
+const u64 malidp_format_modifiers[] = {
+	/* All RGB formats (except XRGB, RGBX, XBGR, BGRX) */
+	DRM_FORMAT_MOD_ARM_AFBC(AFBC_SIZE_16X16 | AFBC_YTR | AFBC_SPARSE),
+	DRM_FORMAT_MOD_ARM_AFBC(AFBC_SIZE_16X16 | AFBC_YTR),
+
+	/* All RGB formats > 16bpp (except XRGB, RGBX, XBGR, BGRX) */
+	DRM_FORMAT_MOD_ARM_AFBC(AFBC_SIZE_16X16 | AFBC_YTR | AFBC_SPARSE | AFBC_SPLIT),
+
+	/* All 8 or 10 bit YUV 444 formats. */
+	/* In DP550, 10 bit YUV 420 format also supported */
+	DRM_FORMAT_MOD_ARM_AFBC(AFBC_SIZE_16X16 | AFBC_SPARSE | AFBC_SPLIT),
+
+	/* YUV 420, 422 P1 8 bit and YUV 444 8 bit/10 bit formats */
+	DRM_FORMAT_MOD_ARM_AFBC(AFBC_SIZE_16X16 | AFBC_SPARSE),
+	DRM_FORMAT_MOD_ARM_AFBC(AFBC_SIZE_16X16),
+
+	/* YUV 420, 422 P1 8, 10 bit formats */
+	DRM_FORMAT_MOD_ARM_AFBC(AFBC_SIZE_16X16 | AFBC_CBR | AFBC_SPARSE),
+	DRM_FORMAT_MOD_ARM_AFBC(AFBC_SIZE_16X16 | AFBC_CBR),
+
+	/* All formats */
+	DRM_FORMAT_MOD_LINEAR,
+
+	DRM_FORMAT_MOD_INVALID
 };
 
 #define SE_N_SCALING_COEFFS	96
@@ -324,14 +380,39 @@ static void malidp500_modeset(struct malidp_hw_device *hwdev, struct videomode *
 		malidp_hw_clearbits(hwdev, MALIDP_DISP_FUNC_ILACED, MALIDP_DE_DISPLAY_FUNC);
 }
 
-static int malidp500_rotmem_required(struct malidp_hw_device *hwdev, u16 w, u16 h, u32 fmt)
+int malidp_format_get_bpp(u32 fmt)
+{
+	int bpp = drm_format_plane_cpp(fmt, 0) * 8;
+
+	if (bpp == 0) {
+		switch (fmt) {
+		case DRM_FORMAT_VUY101010:
+			bpp = 30;
+		case DRM_FORMAT_YUV420_10BIT:
+			bpp = 15;
+			break;
+		case DRM_FORMAT_YUV420_8BIT:
+			bpp = 12;
+			break;
+		default:
+			bpp = 0;
+		}
+	}
+
+	return bpp;
+}
+
+static int malidp500_rotmem_required(struct malidp_hw_device *hwdev, u16 w,
+				     u16 h, u32 fmt, bool has_modifier)
 {
 	/*
 	 * Each layer needs enough rotation memory to fit 8 lines
 	 * worth of pixel data. Required size is then:
 	 *    size = rotated_width * (bpp / 8) * 8;
 	 */
-	return w * drm_format_plane_cpp(fmt, 0) * 8;
+	int bpp = malidp_format_get_bpp(fmt);
+
+	return w * bpp;
 }
 
 static void malidp500_se_write_pp_coefftab(struct malidp_hw_device *hwdev,
@@ -609,9 +690,9 @@ static void malidp550_modeset(struct malidp_hw_device *hwdev, struct videomode *
 		malidp_hw_clearbits(hwdev, MALIDP_DISP_FUNC_ILACED, MALIDP_DE_DISPLAY_FUNC);
 }
 
-static int malidp550_rotmem_required(struct malidp_hw_device *hwdev, u16 w, u16 h, u32 fmt)
+static int malidpx50_get_bytes_per_column(u32 fmt)
 {
-	u32 bytes_per_col;
+	u32 bytes_per_column;
 
 	switch (fmt) {
 	/* 8 lines at 4 bytes per pixel */
@@ -637,19 +718,77 @@ static int malidp550_rotmem_required(struct malidp_hw_device *hwdev, u16 w, u16
 	case DRM_FORMAT_UYVY:
 	case DRM_FORMAT_YUYV:
 	case DRM_FORMAT_X0L0:
-	case DRM_FORMAT_X0L2:
-		bytes_per_col = 32;
+		bytes_per_column = 32;
 		break;
 	/* 16 lines at 1.5 bytes per pixel */
 	case DRM_FORMAT_NV12:
 	case DRM_FORMAT_YUV420:
-		bytes_per_col = 24;
+	/* 8 lines at 3 bytes per pixel */
+	case DRM_FORMAT_VUY888:
+	/* 16 lines at 12 bits per pixel */
+	case DRM_FORMAT_YUV420_8BIT:
+	/* 8 lines at 3 bytes per pixel */
+	case DRM_FORMAT_P010:
+		bytes_per_column = 24;
+		break;
+	/* 8 lines at 30 bits per pixel */
+	case DRM_FORMAT_VUY101010:
+	/* 16 lines at 15 bits per pixel */
+	case DRM_FORMAT_YUV420_10BIT:
+		bytes_per_column = 30;
 		break;
 	default:
 		return -EINVAL;
 	}
 
-	return w * bytes_per_col;
+	return bytes_per_column;
+}
+
+static int malidp550_rotmem_required(struct malidp_hw_device *hwdev, u16 w,
+				     u16 h, u32 fmt, bool has_modifier)
+{
+	int bytes_per_column = 0;
+
+	switch (fmt) {
+	/* 8 lines at 15 bits per pixel */
+	case DRM_FORMAT_YUV420_10BIT:
+		bytes_per_column = 15;
+		break;
+	/* Uncompressed YUV 420 10 bit single plane cannot be rotated */
+	case DRM_FORMAT_X0L2:
+		if (has_modifier)
+			bytes_per_column = 8;
+		else
+			return -EINVAL;
+		break;
+	default:
+		bytes_per_column = malidpx50_get_bytes_per_column(fmt);
+	}
+
+	if (bytes_per_column == -EINVAL)
+		return bytes_per_column;
+
+	return w * bytes_per_column;
+}
+
+static int malidp650_rotmem_required(struct malidp_hw_device *hwdev, u16 w,
+				     u16 h, u32 fmt, bool has_modifier)
+{
+	int bytes_per_column = 0;
+
+	switch (fmt) {
+	/* 16 lines at 2 bytes per pixel */
+	case DRM_FORMAT_X0L2:
+		bytes_per_column = 32;
+		break;
+	default:
+		bytes_per_column = malidpx50_get_bytes_per_column(fmt);
+	}
+
+	if (bytes_per_column == -EINVAL)
+		return bytes_per_column;
+
+	return w * bytes_per_column;
 }
 
 static int malidp550_se_set_scaling_coeffs(struct malidp_hw_device *hwdev,
@@ -838,7 +977,10 @@ const struct malidp_hw malidp_device[MALIDP_MAX_DEVICES] = {
 			.se_base = MALIDP550_SE_BASE,
 			.dc_base = MALIDP550_DC_BASE,
 			.out_depth_base = MALIDP550_DE_OUTPUT_DEPTH,
-			.features = MALIDP_REGMAP_HAS_CLEARIRQ,
+			.features = MALIDP_REGMAP_HAS_CLEARIRQ |
+				    MALIDP_DEVICE_AFBC_SUPPORT_SPLIT |
+				    MALIDP_DEVICE_AFBC_YUV_420_10_SUPPORT_SPLIT |
+				    MALIDP_DEVICE_AFBC_YUYV_USE_422_P2,
 			.n_layers = ARRAY_SIZE(malidp550_layers),
 			.layers = malidp550_layers,
 			.de_irq_map = {
@@ -884,7 +1026,9 @@ const struct malidp_hw malidp_device[MALIDP_MAX_DEVICES] = {
 			.se_base = MALIDP550_SE_BASE,
 			.dc_base = MALIDP550_DC_BASE,
 			.out_depth_base = MALIDP550_DE_OUTPUT_DEPTH,
-			.features = MALIDP_REGMAP_HAS_CLEARIRQ,
+			.features = MALIDP_REGMAP_HAS_CLEARIRQ |
+				    MALIDP_DEVICE_AFBC_SUPPORT_SPLIT |
+				    MALIDP_DEVICE_AFBC_YUYV_USE_422_P2,
 			.n_layers = ARRAY_SIZE(malidp650_layers),
 			.layers = malidp650_layers,
 			.de_irq_map = {
@@ -923,7 +1067,7 @@ const struct malidp_hw malidp_device[MALIDP_MAX_DEVICES] = {
 		.in_config_mode = malidp550_in_config_mode,
 		.set_config_valid = malidp550_set_config_valid,
 		.modeset = malidp550_modeset,
-		.rotmem_required = malidp550_rotmem_required,
+		.rotmem_required = malidp650_rotmem_required,
 		.se_set_scaling_coeffs = malidp550_se_set_scaling_coeffs,
 		.se_calc_mclk = malidp550_se_calc_mclk,
 		.enable_memwrite = malidp550_enable_memwrite,
@@ -933,19 +1077,72 @@ const struct malidp_hw malidp_device[MALIDP_MAX_DEVICES] = {
 };
 
 u8 malidp_hw_get_format_id(const struct malidp_hw_regmap *map,
-			   u8 layer_id, u32 format)
+			   u8 layer_id, u32 format, bool has_modifier)
 {
 	unsigned int i;
 
 	for (i = 0; i < map->n_pixel_formats; i++) {
 		if (((map->pixel_formats[i].layer & layer_id) == layer_id) &&
-		    (map->pixel_formats[i].format == format))
-			return map->pixel_formats[i].id;
+		    (map->pixel_formats[i].format == format)) {
+			/*
+			 * In some DP550 and DP650, DRM_FORMAT_YUYV + AFBC modifier
+			 * is supported by a different h/w format id than
+			 * DRM_FORMAT_YUYV (only).
+			 */
+			if (format == DRM_FORMAT_YUYV &&
+			    (has_modifier) &&
+			    (map->features & MALIDP_DEVICE_AFBC_YUYV_USE_422_P2))
+				return AFBC_YUV_422_FORMAT_ID;
+			else
+				return map->pixel_formats[i].id;
+		}
 	}
 
 	return MALIDP_INVALID_FORMAT_ID;
 }
 
+bool malidp_hw_format_is_linear_only(u32 format)
+{
+	switch (format) {
+	case DRM_FORMAT_ARGB2101010:
+	case DRM_FORMAT_RGBA1010102:
+	case DRM_FORMAT_BGRA1010102:
+	case DRM_FORMAT_ARGB8888:
+	case DRM_FORMAT_RGBA8888:
+	case DRM_FORMAT_BGRA8888:
+	case DRM_FORMAT_XBGR8888:
+	case DRM_FORMAT_XRGB8888:
+	case DRM_FORMAT_RGBX8888:
+	case DRM_FORMAT_BGRX8888:
+	case DRM_FORMAT_RGB888:
+	case DRM_FORMAT_RGB565:
+	case DRM_FORMAT_ARGB1555:
+	case DRM_FORMAT_RGBA5551:
+	case DRM_FORMAT_BGRA5551:
+	case DRM_FORMAT_UYVY:
+	case DRM_FORMAT_XYUV8888:
+	case DRM_FORMAT_XVYU2101010:
+	case DRM_FORMAT_X0L2:
+	case DRM_FORMAT_X0L0:
+		return true;
+	default:
+		return false;
+	}
+}
+
+bool malidp_hw_format_is_afbc_only(u32 format)
+{
+	switch (format) {
+	case DRM_FORMAT_VUY888:
+	case DRM_FORMAT_VUY101010:
+	case DRM_FORMAT_YUV420_8BIT:
+	case DRM_FORMAT_YUV420_10BIT:
+		return true;
+	default:
+		return false;
+	}
+}
+
 static void malidp_hw_clear_irq(struct malidp_hw_device *hwdev, u8 block, u32 irq)
 {
 	u32 base = malidp_get_block_base(hwdev, block);
diff --git a/drivers/gpu/drm/arm/malidp_hw.h b/drivers/gpu/drm/arm/malidp_hw.h
index 40155e2ea9d9..207c3ce52f1a 100644
--- a/drivers/gpu/drm/arm/malidp_hw.h
+++ b/drivers/gpu/drm/arm/malidp_hw.h
@@ -70,6 +70,8 @@ struct malidp_layer {
 	s16 yuv2rgb_offset;	/* offset to the YUV->RGB matrix entries */
 	u16 mmu_ctrl_offset;    /* offset to the MMU control register */
 	enum rotation_features rot;	/* type of rotation supported */
+	/* address offset for the AFBC decoder registers */
+	u16 afbc_decoder_offset;
 };
 
 enum malidp_scaling_coeff_set {
@@ -93,7 +95,10 @@ struct malidp_se_config {
 };
 
 /* regmap features */
-#define MALIDP_REGMAP_HAS_CLEARIRQ	(1 << 0)
+#define MALIDP_REGMAP_HAS_CLEARIRQ				BIT(0)
+#define MALIDP_DEVICE_AFBC_SUPPORT_SPLIT			BIT(1)
+#define MALIDP_DEVICE_AFBC_YUV_420_10_SUPPORT_SPLIT		BIT(2)
+#define MALIDP_DEVICE_AFBC_YUYV_USE_422_P2			BIT(3)
 
 struct malidp_hw_regmap {
 	/* address offset of the DE register bank */
@@ -179,7 +184,8 @@ struct malidp_hw {
 	 * Calculate the required rotation memory given the active area
 	 * and the buffer format.
 	 */
-	int (*rotmem_required)(struct malidp_hw_device *hwdev, u16 w, u16 h, u32 fmt);
+	int (*rotmem_required)(struct malidp_hw_device *hwdev, u16 w, u16 h,
+			       u32 fmt, bool has_modifier);
 
 	int (*se_set_scaling_coeffs)(struct malidp_hw_device *hwdev,
 				     struct malidp_se_config *se_config,
@@ -319,7 +325,9 @@ int malidp_se_irq_init(struct drm_device *drm, int irq);
 void malidp_se_irq_fini(struct malidp_hw_device *hwdev);
 
 u8 malidp_hw_get_format_id(const struct malidp_hw_regmap *map,
-			   u8 layer_id, u32 format);
+			   u8 layer_id, u32 format, bool has_modifier);
+
+int malidp_format_get_bpp(u32 fmt);
 
 static inline u8 malidp_hw_get_pitch_align(struct malidp_hw_device *hwdev, bool rotated)
 {
@@ -388,9 +396,18 @@ static inline void malidp_se_set_enh_coeffs(struct malidp_hw_device *hwdev)
 
 #define MALIDP_GAMMA_LUT_SIZE		4096
 
-#define AFBC_MOD_VALID_BITS (AFBC_FORMAT_MOD_BLOCK_SIZE_MASK | \
-			AFBC_FORMAT_MOD_YTR | AFBC_FORMAT_MOD_SPLIT | \
-			AFBC_FORMAT_MOD_SPARSE | AFBC_FORMAT_MOD_CBR | \
-			AFBC_FORMAT_MOD_TILED | AFBC_FORMAT_MOD_SC)
+#define AFBC_SIZE_MASK		AFBC_FORMAT_MOD_BLOCK_SIZE_MASK
+#define AFBC_SIZE_16X16		AFBC_FORMAT_MOD_BLOCK_SIZE_16x16
+#define AFBC_YTR		AFBC_FORMAT_MOD_YTR
+#define AFBC_SPARSE		AFBC_FORMAT_MOD_SPARSE
+#define AFBC_CBR		AFBC_FORMAT_MOD_CBR
+#define AFBC_SPLIT		AFBC_FORMAT_MOD_SPLIT
+#define AFBC_TILED		AFBC_FORMAT_MOD_TILED
+#define AFBC_SC			AFBC_FORMAT_MOD_SC
+
+#define AFBC_MOD_VALID_BITS	(AFBC_SIZE_MASK | AFBC_YTR | AFBC_SPLIT | \
+				 AFBC_SPARSE | AFBC_CBR | AFBC_TILED | AFBC_SC)
+
+extern const u64 malidp_format_modifiers[];
 
 #endif  /* __MALIDP_HW_H__ */
diff --git a/drivers/gpu/drm/arm/malidp_mw.c b/drivers/gpu/drm/arm/malidp_mw.c
index 041a64dc7167..5f102bdaf841 100644
--- a/drivers/gpu/drm/arm/malidp_mw.c
+++ b/drivers/gpu/drm/arm/malidp_mw.c
@@ -141,9 +141,14 @@ malidp_mw_encoder_atomic_check(struct drm_encoder *encoder,
 		return -EINVAL;
 	}
 
+	if (fb->modifier) {
+		DRM_DEBUG_KMS("Writeback framebuffer does not support modifiers\n");
+		return -EINVAL;
+	}
+
 	mw_state->format =
 		malidp_hw_get_format_id(&malidp->dev->hw->map, SE_MEMWRITE,
-					fb->format->format);
+					fb->format->format, !!fb->modifier);
 	if (mw_state->format == MALIDP_INVALID_FORMAT_ID) {
 		struct drm_format_name_buf format_name;
 
@@ -252,8 +257,7 @@ void malidp_mw_atomic_commit(struct drm_device *drm,
 				     &mw_state->addrs[0],
 				     mw_state->format);
 
-		drm_writeback_queue_job(mw_conn, conn_state->writeback_job);
-		conn_state->writeback_job = NULL;
+		drm_writeback_queue_job(mw_conn, conn_state);
 		hwdev->hw->enable_memwrite(hwdev, mw_state->addrs,
 					   mw_state->pitches, mw_state->n_planes,
 					   fb->width, fb->height, mw_state->format,
diff --git a/drivers/gpu/drm/arm/malidp_planes.c b/drivers/gpu/drm/arm/malidp_planes.c
index c9a6d3e0cada..d42e0ea9a303 100644
--- a/drivers/gpu/drm/arm/malidp_planes.c
+++ b/drivers/gpu/drm/arm/malidp_planes.c
@@ -52,6 +52,8 @@
 #define MALIDP550_LS_ENABLE		0x01c
 #define MALIDP550_LS_R1_IN_SIZE		0x020
 
+#define MODIFIERS_COUNT_MAX		15
+
 /*
  * This 4-entry look-up-table is used to determine the full 8-bit alpha value
  * for formats with 1- or 2-bit alpha channels.
@@ -145,6 +147,119 @@ static void malidp_plane_atomic_print_state(struct drm_printer *p,
 	drm_printf(p, "\tmmu_prefetch_pgsize=%d\n", ms->mmu_prefetch_pgsize);
 }
 
+bool malidp_format_mod_supported(struct drm_device *drm,
+				 u32 format, u64 modifier)
+{
+	const struct drm_format_info *info;
+	const u64 *modifiers;
+	struct malidp_drm *malidp = drm->dev_private;
+	const struct malidp_hw_regmap *map = &malidp->dev->hw->map;
+
+	if (WARN_ON(modifier == DRM_FORMAT_MOD_INVALID))
+		return false;
+
+	/* Some pixel formats are supported without any modifier */
+	if (modifier == DRM_FORMAT_MOD_LINEAR) {
+		/*
+		 * However these pixel formats need to be supported with
+		 * modifiers only
+		 */
+		return !malidp_hw_format_is_afbc_only(format);
+	}
+
+	if ((modifier >> 56) != DRM_FORMAT_MOD_VENDOR_ARM) {
+		DRM_ERROR("Unknown modifier (not Arm)\n");
+		return false;
+	}
+
+	if (modifier &
+	    ~DRM_FORMAT_MOD_ARM_AFBC(AFBC_MOD_VALID_BITS)) {
+		DRM_DEBUG_KMS("Unsupported modifiers\n");
+		return false;
+	}
+
+	modifiers = malidp_format_modifiers;
+
+	/* SPLIT buffers must use SPARSE layout */
+	if (WARN_ON_ONCE((modifier & AFBC_SPLIT) && !(modifier & AFBC_SPARSE)))
+		return false;
+
+	/* CBR only applies to YUV formats, where YTR should be always 0 */
+	if (WARN_ON_ONCE((modifier & AFBC_CBR) && (modifier & AFBC_YTR)))
+		return false;
+
+	while (*modifiers != DRM_FORMAT_MOD_INVALID) {
+		if (*modifiers == modifier)
+			break;
+
+		modifiers++;
+	}
+
+	/* return false, if the modifier was not found */
+	if (*modifiers == DRM_FORMAT_MOD_INVALID) {
+		DRM_DEBUG_KMS("Unsupported modifier\n");
+		return false;
+	}
+
+	info = drm_format_info(format);
+
+	if (info->num_planes != 1) {
+		DRM_DEBUG_KMS("AFBC buffers expect one plane\n");
+		return false;
+	}
+
+	if (malidp_hw_format_is_linear_only(format) == true) {
+		DRM_DEBUG_KMS("Given format (0x%x) is supported is linear mode only\n",
+			      format);
+		return false;
+	}
+
+	/*
+	 * RGB formats need to provide YTR modifier and YUV formats should not
+	 * provide YTR modifier.
+	 */
+	if (!(info->is_yuv) != !!(modifier & AFBC_FORMAT_MOD_YTR)) {
+		DRM_DEBUG_KMS("AFBC_FORMAT_MOD_YTR is %s for %s formats\n",
+			      info->is_yuv ? "disallowed" : "mandatory",
+			      info->is_yuv ? "YUV" : "RGB");
+		return false;
+	}
+
+	if (modifier & AFBC_SPLIT) {
+		if (!info->is_yuv) {
+			if (drm_format_plane_cpp(format, 0) <= 2) {
+				DRM_DEBUG_KMS("RGB formats <= 16bpp are not supported with SPLIT\n");
+				return false;
+			}
+		}
+
+		if ((drm_format_horz_chroma_subsampling(format) != 1) ||
+		    (drm_format_vert_chroma_subsampling(format) != 1)) {
+			if (!(format == DRM_FORMAT_YUV420_10BIT &&
+			      (map->features & MALIDP_DEVICE_AFBC_YUV_420_10_SUPPORT_SPLIT))) {
+				DRM_DEBUG_KMS("Formats which are sub-sampled should never be split\n");
+				return false;
+			}
+		}
+	}
+
+	if (modifier & AFBC_CBR) {
+		if ((drm_format_horz_chroma_subsampling(format) == 1) ||
+		    (drm_format_vert_chroma_subsampling(format) == 1)) {
+			DRM_DEBUG_KMS("Formats which are not sub-sampled should not have CBR set\n");
+			return false;
+		}
+	}
+
+	return true;
+}
+
+static bool malidp_format_mod_supported_per_plane(struct drm_plane *plane,
+						  u32 format, u64 modifier)
+{
+	return malidp_format_mod_supported(plane->dev, format, modifier);
+}
+
 static const struct drm_plane_funcs malidp_de_plane_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
@@ -153,6 +268,7 @@ static const struct drm_plane_funcs malidp_de_plane_funcs = {
 	.atomic_duplicate_state = malidp_duplicate_plane_state,
 	.atomic_destroy_state = malidp_destroy_plane_state,
 	.atomic_print_state = malidp_plane_atomic_print_state,
+	.format_mod_supported = malidp_format_mod_supported_per_plane,
 };
 
 static int malidp_se_check_scaling(struct malidp_plane *mp,
@@ -406,8 +522,8 @@ static int malidp_de_plane_check(struct drm_plane *plane,
 	fb = state->fb;
 
 	ms->format = malidp_hw_get_format_id(&mp->hwdev->hw->map,
-					     mp->layer->id,
-					     fb->format->format);
+					     mp->layer->id, fb->format->format,
+					     !!fb->modifier);
 	if (ms->format == MALIDP_INVALID_FORMAT_ID)
 		return -EINVAL;
 
@@ -415,8 +531,8 @@ static int malidp_de_plane_check(struct drm_plane *plane,
 	for (i = 0; i < ms->n_planes; i++) {
 		u8 alignment = malidp_hw_get_pitch_align(mp->hwdev, rotated);
 
-		if ((fb->pitches[i] * drm_format_info_block_height(fb->format, i))
-				& (alignment - 1)) {
+		if (((fb->pitches[i] * drm_format_info_block_height(fb->format, i))
+				& (alignment - 1)) && !(fb->modifier)) {
 			DRM_DEBUG_KMS("Invalid pitch %u for plane %d\n",
 				      fb->pitches[i], i);
 			return -EINVAL;
@@ -469,13 +585,20 @@ static int malidp_de_plane_check(struct drm_plane *plane,
 			return -EINVAL;
 	}
 
+	/* SMART layer does not support AFBC */
+	if (mp->layer->id == DE_SMART && fb->modifier) {
+		DRM_ERROR("AFBC framebuffer not supported in SMART layer");
+		return -EINVAL;
+	}
+
 	ms->rotmem_size = 0;
 	if (state->rotation & MALIDP_ROTATED_MASK) {
 		int val;
 
 		val = mp->hwdev->hw->rotmem_required(mp->hwdev, state->crtc_w,
 						     state->crtc_h,
-						     fb->format->format);
+						     fb->format->format,
+						     !!(fb->modifier));
 		if (val < 0)
 			return val;
 
@@ -592,6 +715,83 @@ static void malidp_de_set_mmu_control(struct malidp_plane *mp,
 			mp->layer->base + mp->layer->mmu_ctrl_offset);
 }
 
+static void malidp_set_plane_base_addr(struct drm_framebuffer *fb,
+				       struct malidp_plane *mp,
+				       int plane_index)
+{
+	dma_addr_t paddr;
+	u16 ptr;
+	struct drm_plane *plane = &mp->base;
+	bool afbc = fb->modifier ? true : false;
+
+	ptr = mp->layer->ptr + (plane_index << 4);
+
+	/*
+	 * drm_fb_cma_get_gem_addr() alters the physical base address of the
+	 * framebuffer as per the plane's src_x, src_y co-ordinates (ie to
+	 * take care of source cropping).
+	 * For AFBC, this is not needed as the cropping is handled by _AD_CROP_H
+	 * and _AD_CROP_V registers.
+	 */
+	if (!afbc) {
+		paddr = drm_fb_cma_get_gem_addr(fb, plane->state,
+						plane_index);
+	} else {
+		struct drm_gem_cma_object *obj;
+
+		obj = drm_fb_cma_get_gem_obj(fb, plane_index);
+
+		if (WARN_ON(!obj))
+			return;
+		paddr = obj->paddr;
+	}
+
+	malidp_hw_write(mp->hwdev, lower_32_bits(paddr), ptr);
+	malidp_hw_write(mp->hwdev, upper_32_bits(paddr), ptr + 4);
+}
+
+static void malidp_de_set_plane_afbc(struct drm_plane *plane)
+{
+	struct malidp_plane *mp;
+	u32 src_w, src_h, val = 0, src_x, src_y;
+	struct drm_framebuffer *fb = plane->state->fb;
+
+	mp = to_malidp_plane(plane);
+
+	/* no afbc_decoder_offset means AFBC is not supported on this plane */
+	if (!mp->layer->afbc_decoder_offset)
+		return;
+
+	if (!fb->modifier) {
+		malidp_hw_write(mp->hwdev, 0, mp->layer->afbc_decoder_offset);
+		return;
+	}
+
+	/* convert src values from Q16 fixed point to integer */
+	src_w = plane->state->src_w >> 16;
+	src_h = plane->state->src_h >> 16;
+	src_x = plane->state->src_x >> 16;
+	src_y = plane->state->src_y >> 16;
+
+	val = ((fb->width - (src_x + src_w)) << MALIDP_AD_CROP_RIGHT_OFFSET) |
+		   src_x;
+	malidp_hw_write(mp->hwdev, val,
+			mp->layer->afbc_decoder_offset + MALIDP_AD_CROP_H);
+
+	val = ((fb->height - (src_y + src_h)) << MALIDP_AD_CROP_BOTTOM_OFFSET) |
+		   src_y;
+	malidp_hw_write(mp->hwdev, val,
+			mp->layer->afbc_decoder_offset + MALIDP_AD_CROP_V);
+
+	val = MALIDP_AD_EN;
+	if (fb->modifier & AFBC_FORMAT_MOD_SPLIT)
+		val |= MALIDP_AD_BS;
+	if (fb->modifier & AFBC_FORMAT_MOD_YTR)
+		val |= MALIDP_AD_YTR;
+
+	malidp_hw_write(mp->hwdev, val, mp->layer->afbc_decoder_offset);
+}
+
 static void malidp_de_plane_update(struct drm_plane *plane,
 				   struct drm_plane_state *old_state)
 {
@@ -602,12 +802,23 @@ static void malidp_de_plane_update(struct drm_plane *plane,
 	u8 plane_alpha = state->alpha >> 8;
 	u32 src_w, src_h, dest_w, dest_h, val;
 	int i;
+	struct drm_framebuffer *fb = plane->state->fb;
 
 	mp = to_malidp_plane(plane);
 
-	/* convert src values from Q16 fixed point to integer */
-	src_w = state->src_w >> 16;
-	src_h = state->src_h >> 16;
+	/*
+	 * For AFBC framebuffer, use the framebuffer width and height for
+	 * configuring layer input size register.
+	 */
+	if (fb->modifier) {
+		src_w = fb->width;
+		src_h = fb->height;
+	} else {
+		/* convert src values from Q16 fixed point to integer */
+		src_w = state->src_w >> 16;
+		src_h = state->src_h >> 16;
+	}
+
 	dest_w = state->crtc_w;
 	dest_h = state->crtc_h;
 
@@ -615,15 +826,8 @@ static void malidp_de_plane_update(struct drm_plane *plane,
 	val = (val & ~LAYER_FORMAT_MASK) | ms->format;
 	malidp_hw_write(mp->hwdev, val, mp->layer->base);
 
-	for (i = 0; i < ms->n_planes; i++) {
-		/* calculate the offset for the layer's plane registers */
-		u16 ptr = mp->layer->ptr + (i << 4);
-		dma_addr_t fb_addr = drm_fb_cma_get_gem_addr(state->fb,
-							     state, i);
-
-		malidp_hw_write(mp->hwdev, lower_32_bits(fb_addr), ptr);
-		malidp_hw_write(mp->hwdev, upper_32_bits(fb_addr), ptr + 4);
-	}
+	for (i = 0; i < ms->n_planes; i++)
+		malidp_set_plane_base_addr(fb, mp, i);
 
 	malidp_de_set_mmu_control(mp, ms);
 
@@ -657,6 +861,8 @@ static void malidp_de_plane_update(struct drm_plane *plane,
 				mp->layer->base + MALIDP550_LS_R1_IN_SIZE);
 	}
 
+	malidp_de_set_plane_afbc(plane);
+
 	/* first clear the rotation bits */
 	val = malidp_hw_read(mp->hwdev, mp->layer->base + MALIDP_LAYER_CONTROL);
 	val &= ~LAYER_ROT_MASK;
@@ -733,7 +939,26 @@ int malidp_de_planes_init(struct drm_device *drm)
 				  BIT(DRM_MODE_BLEND_PREMULTI)   |
 				  BIT(DRM_MODE_BLEND_COVERAGE);
 	u32 *formats;
-	int ret, i, j, n;
+	int ret, i = 0, j = 0, n;
+	u64 supported_modifiers[MODIFIERS_COUNT_MAX];
+	const u64 *modifiers;
+
+	modifiers = malidp_format_modifiers;
+
+	if (!(map->features & MALIDP_DEVICE_AFBC_SUPPORT_SPLIT)) {
+		/*
+		 * Since our hardware does not support SPLIT, so build the list
+		 * of supported modifiers excluding SPLIT ones.
+		 */
+		while (*modifiers != DRM_FORMAT_MOD_INVALID) {
+			if (!(*modifiers & AFBC_SPLIT))
+				supported_modifiers[j++] = *modifiers;
+
+			modifiers++;
+		}
+		supported_modifiers[j++] = DRM_FORMAT_MOD_INVALID;
+		modifiers = supported_modifiers;
+	}
 
 	formats = kcalloc(map->n_pixel_formats, sizeof(*formats), GFP_KERNEL);
 	if (!formats) {
@@ -758,9 +983,15 @@ int malidp_de_planes_init(struct drm_device *drm)
 
 		plane_type = (i == 0) ? DRM_PLANE_TYPE_PRIMARY :
 					DRM_PLANE_TYPE_OVERLAY;
+
+		/*
+		 * All the layers except smart layer supports AFBC modifiers.
+		 */
 		ret = drm_universal_plane_init(drm, &plane->base, crtcs,
-					       &malidp_de_plane_funcs, formats,
-					       n, NULL, plane_type, NULL);
+				&malidp_de_plane_funcs, formats, n,
+				(id == DE_SMART) ? NULL : modifiers, plane_type,
+				NULL);
+
 		if (ret < 0)
 			goto cleanup;
 
diff --git a/drivers/gpu/drm/arm/malidp_regs.h b/drivers/gpu/drm/arm/malidp_regs.h
index 7ce3e141464d..a0dd6e1676a8 100644
--- a/drivers/gpu/drm/arm/malidp_regs.h
+++ b/drivers/gpu/drm/arm/malidp_regs.h
@@ -198,10 +198,13 @@
 #define MALIDP500_LV_YUV2RGB		((s16)(-0xB8))
 #define MALIDP500_DE_LV_BASE		0x00100
 #define MALIDP500_DE_LV_PTR_BASE	0x00124
+#define MALIDP500_DE_LV_AD_CTRL		0x00400
 #define MALIDP500_DE_LG1_BASE		0x00200
 #define MALIDP500_DE_LG1_PTR_BASE	0x0021c
+#define MALIDP500_DE_LG1_AD_CTRL	0x0040c
 #define MALIDP500_DE_LG2_BASE		0x00300
 #define MALIDP500_DE_LG2_PTR_BASE	0x0031c
+#define MALIDP500_DE_LG2_AD_CTRL	0x00418
 #define MALIDP500_SE_BASE		0x00c00
 #define MALIDP500_SE_CONTROL		0x00c0c
 #define MALIDP500_SE_MEMWRITE_OUT_SIZE	0x00c2c
@@ -228,10 +231,13 @@
 #define MALIDP550_LV_YUV2RGB		0x00084
 #define MALIDP550_DE_LV1_BASE		0x00100
 #define MALIDP550_DE_LV1_PTR_BASE	0x00124
+#define MALIDP550_DE_LV1_AD_CTRL	0x001B8
 #define MALIDP550_DE_LV2_BASE		0x00200
 #define MALIDP550_DE_LV2_PTR_BASE	0x00224
+#define MALIDP550_DE_LV2_AD_CTRL	0x002B8
 #define MALIDP550_DE_LG_BASE		0x00300
 #define MALIDP550_DE_LG_PTR_BASE	0x0031c
+#define MALIDP550_DE_LG_AD_CTRL		0x00330
 #define MALIDP550_DE_LS_BASE		0x00400
 #define MALIDP550_DE_LS_PTR_BASE	0x0042c
 #define MALIDP550_DE_PERF_BASE		0x00500
@@ -258,6 +264,20 @@
 #define MALIDP_MMU_CTRL_PX_PS(x)	(1 << (8 + (x)))
 #define MALIDP_MMU_CTRL_PP_NUM_REQ(x)	(((x) & 0x7f) << 12)
 
+/* AFBC register offsets relative to MALIDPXXX_DE_LX_AD_CTRL */
+/* The following register offsets are common for DP500, DP550 and DP650 */
+#define MALIDP_AD_CROP_H                0x4
+#define MALIDP_AD_CROP_V                0x8
+#define MALIDP_AD_END_PTR_LOW           0xc
+#define MALIDP_AD_END_PTR_HIGH          0x10
+
+/* AFBC decoder Registers */
+#define MALIDP_AD_EN                    BIT(0)
+#define MALIDP_AD_YTR                   BIT(4)
+#define MALIDP_AD_BS                    BIT(8)
+#define MALIDP_AD_CROP_RIGHT_OFFSET     16
+#define MALIDP_AD_CROP_BOTTOM_OFFSET    16
+
 /*
  * Starting with DP550 the register map blocks has been standardised to the
  * following layout:
diff --git a/drivers/gpu/drm/armada/armada_fbdev.c b/drivers/gpu/drm/armada/armada_fbdev.c
index 8d23700848df..1e7140f005a5 100644
--- a/drivers/gpu/drm/armada/armada_fbdev.c
+++ b/drivers/gpu/drm/armada/armada_fbdev.c
@@ -78,8 +78,6 @@ static int armada_fbdev_create(struct drm_fb_helper *fbh,
 		goto err_fballoc;
 	}
 
-	strlcpy(info->fix.id, "armada-drmfb", sizeof(info->fix.id));
-	info->par = fbh;
 	info->fbops = &armada_fb_ops;
 	info->fix.smem_start = obj->phys_addr;
 	info->fix.smem_len = obj->obj.size;
@@ -87,9 +85,7 @@ static int armada_fbdev_create(struct drm_fb_helper *fbh,
 	info->screen_base = ptr;
 	fbh->fb = &dfb->fb;
 
-	drm_fb_helper_fill_fix(info, dfb->fb.pitches[0],
-			       dfb->fb.format->depth);
-	drm_fb_helper_fill_var(info, fbh, sizes->fb_width, sizes->fb_height);
+	drm_fb_helper_fill_info(info, fbh, sizes);
 
 	DRM_DEBUG_KMS("allocated %dx%d %dbpp fb: 0x%08llx\n",
 		dfb->fb.width, dfb->fb.height, dfb->fb.format->cpp[0] * 8,
diff --git a/drivers/gpu/drm/aspeed/Kconfig b/drivers/gpu/drm/aspeed/Kconfig
new file mode 100644
index 000000000000..cccab520e02f
--- /dev/null
+++ b/drivers/gpu/drm/aspeed/Kconfig
@@ -0,0 +1,14 @@
+config DRM_ASPEED_GFX
+	tristate "ASPEED BMC Display Controller"
+	depends on DRM && OF
+	depends on (COMPILE_TEST || ARCH_ASPEED)
+	select DRM_KMS_HELPER
+	select DRM_KMS_CMA_HELPER
+	select DMA_CMA if HAVE_DMA_CONTIGUOUS
+	select CMA if HAVE_DMA_CONTIGUOUS
+	select MFD_SYSCON
+	help
+	  Chose this option if you have an ASPEED AST2500 SOC Display
+	  Controller (aka GFX).
+
+	  If M is selected this module will be called aspeed_gfx.
diff --git a/drivers/gpu/drm/aspeed/Makefile b/drivers/gpu/drm/aspeed/Makefile
new file mode 100644
index 000000000000..6e194cd790d8
--- /dev/null
+++ b/drivers/gpu/drm/aspeed/Makefile
@@ -0,0 +1,3 @@
+aspeed_gfx-y := aspeed_gfx_drv.o aspeed_gfx_crtc.o aspeed_gfx_out.o
+
+obj-$(CONFIG_DRM_ASPEED_GFX) += aspeed_gfx.o
diff --git a/drivers/gpu/drm/aspeed/aspeed_gfx.h b/drivers/gpu/drm/aspeed/aspeed_gfx.h
new file mode 100644
index 000000000000..a10358bb61ec
--- /dev/null
+++ b/drivers/gpu/drm/aspeed/aspeed_gfx.h
@@ -0,0 +1,104 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/* Copyright 2018 IBM Corporation */
+
+#include <drm/drm_device.h>
+#include <drm/drm_simple_kms_helper.h>
+
+struct aspeed_gfx {
+	void __iomem			*base;
+	struct clk			*clk;
+	struct reset_control		*rst;
+	struct regmap			*scu;
+
+	struct drm_simple_display_pipe	pipe;
+	struct drm_connector		connector;
+	struct drm_fbdev_cma		*fbdev;
+};
+
+int aspeed_gfx_create_pipe(struct drm_device *drm);
+int aspeed_gfx_create_output(struct drm_device *drm);
+
+#define CRT_CTRL1		0x60 /* CRT Control I */
+#define CRT_CTRL2		0x64 /* CRT Control II */
+#define CRT_STATUS		0x68 /* CRT Status */
+#define CRT_MISC		0x6c /* CRT Misc Setting */
+#define CRT_HORIZ0		0x70 /* CRT Horizontal Total & Display Enable End */
+#define CRT_HORIZ1		0x74 /* CRT Horizontal Retrace Start & End */
+#define CRT_VERT0		0x78 /* CRT Vertical Total & Display Enable End */
+#define CRT_VERT1		0x7C /* CRT Vertical Retrace Start & End */
+#define CRT_ADDR		0x80 /* CRT Display Starting Address */
+#define CRT_OFFSET		0x84 /* CRT Display Offset & Terminal Count */
+#define CRT_THROD		0x88 /* CRT Threshold */
+#define CRT_XSCALE		0x8C /* CRT Scaling-Up Factor */
+#define CRT_CURSOR0		0x90 /* CRT Hardware Cursor X & Y Offset */
+#define CRT_CURSOR1		0x94 /* CRT Hardware Cursor X & Y Position */
+#define CRT_CURSOR2		0x98 /* CRT Hardware Cursor Pattern Address */
+#define CRT_9C			0x9C
+#define CRT_OSD_H		0xA0 /* CRT OSD Horizontal Start/End */
+#define CRT_OSD_V		0xA4 /* CRT OSD Vertical Start/End */
+#define CRT_OSD_ADDR		0xA8 /* CRT OSD Pattern Address */
+#define CRT_OSD_DISP		0xAC /* CRT OSD Offset */
+#define CRT_OSD_THRESH		0xB0 /* CRT OSD Threshold & Alpha */
+#define CRT_B4			0xB4
+#define CRT_STS_V		0xB8 /* CRT Status V */
+#define CRT_SCRATCH		0xBC /* Scratchpad */
+#define CRT_BB0_ADDR		0xD0 /* CRT Display BB0 Starting Address */
+#define CRT_BB1_ADDR		0xD4 /* CRT Display BB1 Starting Address */
+#define CRT_BB_COUNT		0xD8 /* CRT Display BB Terminal Count */
+#define OSD_COLOR1		0xE0 /* OSD Color Palette Index 1 & 0 */
+#define OSD_COLOR2		0xE4 /* OSD Color Palette Index 3 & 2 */
+#define OSD_COLOR3		0xE8 /* OSD Color Palette Index 5 & 4 */
+#define OSD_COLOR4		0xEC /* OSD Color Palette Index 7 & 6 */
+#define OSD_COLOR5		0xF0 /* OSD Color Palette Index 9 & 8 */
+#define OSD_COLOR6		0xF4 /* OSD Color Palette Index 11 & 10 */
+#define OSD_COLOR7		0xF8 /* OSD Color Palette Index 13 & 12 */
+#define OSD_COLOR8		0xFC /* OSD Color Palette Index 15 & 14 */
+
+/* CTRL1 */
+#define CRT_CTRL_EN			BIT(0)
+#define CRT_CTRL_HW_CURSOR_EN		BIT(1)
+#define CRT_CTRL_OSD_EN			BIT(2)
+#define CRT_CTRL_INTERLACED		BIT(3)
+#define CRT_CTRL_COLOR_RGB565		(0 << 7)
+#define CRT_CTRL_COLOR_YUV444		(1 << 7)
+#define CRT_CTRL_COLOR_XRGB8888		(2 << 7)
+#define CRT_CTRL_COLOR_RGB888		(3 << 7)
+#define CRT_CTRL_COLOR_YUV444_2RGB	(5 << 7)
+#define CRT_CTRL_COLOR_YUV422		(7 << 7)
+#define CRT_CTRL_COLOR_MASK		GENMASK(9, 7)
+#define CRT_CTRL_HSYNC_NEGATIVE		BIT(16)
+#define CRT_CTRL_VSYNC_NEGATIVE		BIT(17)
+#define CRT_CTRL_VERTICAL_INTR_EN	BIT(30)
+#define CRT_CTRL_VERTICAL_INTR_STS	BIT(31)
+
+/* CTRL2 */
+#define CRT_CTRL_DAC_EN			BIT(0)
+#define CRT_CTRL_VBLANK_LINE(x)		(((x) << 20) & CRT_CTRL_VBLANK_LINE_MASK)
+#define CRT_CTRL_VBLANK_LINE_MASK	GENMASK(20, 31)
+
+/* CRT_HORIZ0 */
+#define CRT_H_TOTAL(x)			(x)
+#define CRT_H_DE(x)			((x) << 16)
+
+/* CRT_HORIZ1 */
+#define CRT_H_RS_START(x)		(x)
+#define CRT_H_RS_END(x)			((x) << 16)
+
+/* CRT_VIRT0 */
+#define CRT_V_TOTAL(x)			(x)
+#define CRT_V_DE(x)			((x) << 16)
+
+/* CRT_VIRT1 */
+#define CRT_V_RS_START(x)		(x)
+#define CRT_V_RS_END(x)			((x) << 16)
+
+/* CRT_OFFSET */
+#define CRT_DISP_OFFSET(x)		(x)
+#define CRT_TERM_COUNT(x)		((x) << 16)
+
+/* CRT_THROD */
+#define CRT_THROD_LOW(x)		(x)
+#define CRT_THROD_HIGH(x)		((x) << 8)
+
+/* Default Threshold Seting */
+#define G5_CRT_THROD_VAL	(CRT_THROD_LOW(0x24) | CRT_THROD_HIGH(0x3C))
diff --git a/drivers/gpu/drm/aspeed/aspeed_gfx_crtc.c b/drivers/gpu/drm/aspeed/aspeed_gfx_crtc.c
new file mode 100644
index 000000000000..15db9e426ec4
--- /dev/null
+++ b/drivers/gpu/drm/aspeed/aspeed_gfx_crtc.c
@@ -0,0 +1,241 @@
+// SPDX-License-Identifier: GPL-2.0+
+// Copyright 2018 IBM Corporation
+
+#include <linux/clk.h>
+#include <linux/reset.h>
+#include <linux/regmap.h>
+
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_device.h>
+#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fourcc.h>
+#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_framebuffer_helper.h>
+#include <drm/drm_panel.h>
+#include <drm/drm_simple_kms_helper.h>
+#include <drm/drm_vblank.h>
+
+#include "aspeed_gfx.h"
+
+static struct aspeed_gfx *
+drm_pipe_to_aspeed_gfx(struct drm_simple_display_pipe *pipe)
+{
+	return container_of(pipe, struct aspeed_gfx, pipe);
+}
+
+static int aspeed_gfx_set_pixel_fmt(struct aspeed_gfx *priv, u32 *bpp)
+{
+	struct drm_crtc *crtc = &priv->pipe.crtc;
+	struct drm_device *drm = crtc->dev;
+	const u32 format = crtc->primary->state->fb->format->format;
+	u32 ctrl1;
+
+	ctrl1 = readl(priv->base + CRT_CTRL1);
+	ctrl1 &= ~CRT_CTRL_COLOR_MASK;
+
+	switch (format) {
+	case DRM_FORMAT_RGB565:
+		dev_dbg(drm->dev, "Setting up RGB565 mode\n");
+		ctrl1 |= CRT_CTRL_COLOR_RGB565;
+		*bpp = 16;
+		break;
+	case DRM_FORMAT_XRGB8888:
+		dev_dbg(drm->dev, "Setting up XRGB8888 mode\n");
+		ctrl1 |= CRT_CTRL_COLOR_XRGB8888;
+		*bpp = 32;
+		break;
+	default:
+		dev_err(drm->dev, "Unhandled pixel format %08x\n", format);
+		return -EINVAL;
+	}
+
+	writel(ctrl1, priv->base + CRT_CTRL1);
+
+	return 0;
+}
+
+static void aspeed_gfx_enable_controller(struct aspeed_gfx *priv)
+{
+	u32 ctrl1 = readl(priv->base + CRT_CTRL1);
+	u32 ctrl2 = readl(priv->base + CRT_CTRL2);
+
+	/* SCU2C: set DAC source for display output to Graphics CRT (GFX) */
+	regmap_update_bits(priv->scu, 0x2c, BIT(16), BIT(16));
+
+	writel(ctrl1 | CRT_CTRL_EN, priv->base + CRT_CTRL1);
+	writel(ctrl2 | CRT_CTRL_DAC_EN, priv->base + CRT_CTRL2);
+}
+
+static void aspeed_gfx_disable_controller(struct aspeed_gfx *priv)
+{
+	u32 ctrl1 = readl(priv->base + CRT_CTRL1);
+	u32 ctrl2 = readl(priv->base + CRT_CTRL2);
+
+	writel(ctrl1 & ~CRT_CTRL_EN, priv->base + CRT_CTRL1);
+	writel(ctrl2 & ~CRT_CTRL_DAC_EN, priv->base + CRT_CTRL2);
+
+	regmap_update_bits(priv->scu, 0x2c, BIT(16), 0);
+}
+
+static void aspeed_gfx_crtc_mode_set_nofb(struct aspeed_gfx *priv)
+{
+	struct drm_display_mode *m = &priv->pipe.crtc.state->adjusted_mode;
+	u32 ctrl1, d_offset, t_count, bpp;
+	int err;
+
+	err = aspeed_gfx_set_pixel_fmt(priv, &bpp);
+	if (err)
+		return;
+
+#if 0
+	/* TODO: we have only been able to test with the 40MHz USB clock. The
+	 * clock is fixed, so we cannot adjust it here. */
+	clk_set_rate(priv->pixel_clk, m->crtc_clock * 1000);
+#endif
+
+	ctrl1 = readl(priv->base + CRT_CTRL1);
+	ctrl1 &= ~(CRT_CTRL_INTERLACED |
+			CRT_CTRL_HSYNC_NEGATIVE |
+			CRT_CTRL_VSYNC_NEGATIVE);
+
+	if (m->flags & DRM_MODE_FLAG_INTERLACE)
+		ctrl1 |= CRT_CTRL_INTERLACED;
+
+	if (!(m->flags & DRM_MODE_FLAG_PHSYNC))
+		ctrl1 |= CRT_CTRL_HSYNC_NEGATIVE;
+
+	if (!(m->flags & DRM_MODE_FLAG_PVSYNC))
+		ctrl1 |= CRT_CTRL_VSYNC_NEGATIVE;
+
+	writel(ctrl1, priv->base + CRT_CTRL1);
+
+	/* Horizontal timing */
+	writel(CRT_H_TOTAL(m->htotal - 1) | CRT_H_DE(m->hdisplay - 1),
+			priv->base + CRT_HORIZ0);
+	writel(CRT_H_RS_START(m->hsync_start - 1) | CRT_H_RS_END(m->hsync_end),
+			priv->base + CRT_HORIZ1);
+
+
+	/* Vertical timing */
+	writel(CRT_V_TOTAL(m->vtotal - 1) | CRT_V_DE(m->vdisplay - 1),
+			priv->base + CRT_VERT0);
+	writel(CRT_V_RS_START(m->vsync_start) | CRT_V_RS_END(m->vsync_end),
+			priv->base + CRT_VERT1);
+
+	/*
+	 * Display Offset: address difference between consecutive scan lines
+	 * Terminal Count: memory size of one scan line
+	 */
+	d_offset = m->hdisplay * bpp / 8;
+	t_count = (m->hdisplay * bpp + 127) / 128;
+	writel(CRT_DISP_OFFSET(d_offset) | CRT_TERM_COUNT(t_count),
+			priv->base + CRT_OFFSET);
+
+	/*
+	 * Threshold: FIFO thresholds of refill and stop (16 byte chunks
+	 * per line, rounded up)
+	 */
+	writel(G5_CRT_THROD_VAL, priv->base + CRT_THROD);
+}
+
+static void aspeed_gfx_pipe_enable(struct drm_simple_display_pipe *pipe,
+			      struct drm_crtc_state *crtc_state,
+			      struct drm_plane_state *plane_state)
+{
+	struct aspeed_gfx *priv = drm_pipe_to_aspeed_gfx(pipe);
+	struct drm_crtc *crtc = &pipe->crtc;
+
+	aspeed_gfx_crtc_mode_set_nofb(priv);
+	aspeed_gfx_enable_controller(priv);
+	drm_crtc_vblank_on(crtc);
+}
+
+static void aspeed_gfx_pipe_disable(struct drm_simple_display_pipe *pipe)
+{
+	struct aspeed_gfx *priv = drm_pipe_to_aspeed_gfx(pipe);
+	struct drm_crtc *crtc = &pipe->crtc;
+
+	drm_crtc_vblank_off(crtc);
+	aspeed_gfx_disable_controller(priv);
+}
+
+static void aspeed_gfx_pipe_update(struct drm_simple_display_pipe *pipe,
+				   struct drm_plane_state *plane_state)
+{
+	struct aspeed_gfx *priv = drm_pipe_to_aspeed_gfx(pipe);
+	struct drm_crtc *crtc = &pipe->crtc;
+	struct drm_framebuffer *fb = pipe->plane.state->fb;
+	struct drm_pending_vblank_event *event;
+	struct drm_gem_cma_object *gem;
+
+	spin_lock_irq(&crtc->dev->event_lock);
+	event = crtc->state->event;
+	if (event) {
+		crtc->state->event = NULL;
+
+		if (drm_crtc_vblank_get(crtc) == 0)
+			drm_crtc_arm_vblank_event(crtc, event);
+		else
+			drm_crtc_send_vblank_event(crtc, event);
+	}
+	spin_unlock_irq(&crtc->dev->event_lock);
+
+	if (!fb)
+		return;
+
+	gem = drm_fb_cma_get_gem_obj(fb, 0);
+	if (!gem)
+		return;
+	writel(gem->paddr, priv->base + CRT_ADDR);
+}
+
+static int aspeed_gfx_enable_vblank(struct drm_simple_display_pipe *pipe)
+{
+	struct aspeed_gfx *priv = drm_pipe_to_aspeed_gfx(pipe);
+	u32 reg = readl(priv->base + CRT_CTRL1);
+
+	/* Clear pending VBLANK IRQ */
+	writel(reg | CRT_CTRL_VERTICAL_INTR_STS, priv->base + CRT_CTRL1);
+
+	reg |= CRT_CTRL_VERTICAL_INTR_EN;
+	writel(reg, priv->base + CRT_CTRL1);
+
+	return 0;
+}
+
+static void aspeed_gfx_disable_vblank(struct drm_simple_display_pipe *pipe)
+{
+	struct aspeed_gfx *priv = drm_pipe_to_aspeed_gfx(pipe);
+	u32 reg = readl(priv->base + CRT_CTRL1);
+
+	reg &= ~CRT_CTRL_VERTICAL_INTR_EN;
+	writel(reg, priv->base + CRT_CTRL1);
+
+	/* Clear pending VBLANK IRQ */
+	writel(reg | CRT_CTRL_VERTICAL_INTR_STS, priv->base + CRT_CTRL1);
+}
+
+static struct drm_simple_display_pipe_funcs aspeed_gfx_funcs = {
+	.enable		= aspeed_gfx_pipe_enable,
+	.disable	= aspeed_gfx_pipe_disable,
+	.update		= aspeed_gfx_pipe_update,
+	.prepare_fb	= drm_gem_fb_simple_display_pipe_prepare_fb,
+	.enable_vblank	= aspeed_gfx_enable_vblank,
+	.disable_vblank	= aspeed_gfx_disable_vblank,
+};
+
+static const uint32_t aspeed_gfx_formats[] = {
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_RGB565,
+};
+
+int aspeed_gfx_create_pipe(struct drm_device *drm)
+{
+	struct aspeed_gfx *priv = drm->dev_private;
+
+	return drm_simple_display_pipe_init(drm, &priv->pipe, &aspeed_gfx_funcs,
+					    aspeed_gfx_formats,
+					    ARRAY_SIZE(aspeed_gfx_formats),
+					    NULL,
+					    &priv->connector);
+}
diff --git a/drivers/gpu/drm/aspeed/aspeed_gfx_drv.c b/drivers/gpu/drm/aspeed/aspeed_gfx_drv.c
new file mode 100644
index 000000000000..eeb22eccd1fc
--- /dev/null
+++ b/drivers/gpu/drm/aspeed/aspeed_gfx_drv.c
@@ -0,0 +1,269 @@
+// SPDX-License-Identifier: GPL-2.0+
+// Copyright 2018 IBM Corporation
+
+#include <linux/clk.h>
+#include <linux/dma-mapping.h>
+#include <linux/irq.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_reserved_mem.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/reset.h>
+
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_device.h>
+#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_helper.h>
+#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_framebuffer_helper.h>
+#include <drm/drm_probe_helper.h>
+#include <drm/drm_simple_kms_helper.h>
+#include <drm/drm_vblank.h>
+#include <drm/drm_drv.h>
+
+#include "aspeed_gfx.h"
+
+/**
+ * DOC: ASPEED GFX Driver
+ *
+ * This driver is for the ASPEED BMC SoC's 'GFX' display hardware, also called
+ * the 'SOC Display Controller' in the datasheet. This driver runs on the ARM
+ * based BMC systems, unlike the ast driver which runs on a host CPU and is for
+ * a PCIe graphics device.
+ *
+ * The AST2500 supports a total of 3 output paths:
+ *
+ *   1. VGA output, the output target can choose either or both to the DAC
+ *   or DVO interface.
+ *
+ *   2. Graphics CRT output, the output target can choose either or both to
+ *   the DAC or DVO interface.
+ *
+ *   3. Video input from DVO, the video input can be used for video engine
+ *   capture or DAC display output.
+ *
+ * Output options are selected in SCU2C.
+ *
+ * The "VGA mode" device is the PCI attached controller. The "Graphics CRT"
+ * is the ARM's internal display controller.
+ *
+ * The driver only supports a simple configuration consisting of a 40MHz
+ * pixel clock, fixed by hardware limitations, and the VGA output path.
+ *
+ * The driver was written with the 'AST2500 Software Programming Guide' v17,
+ * which is available under NDA from ASPEED.
+ */
+
+static const struct drm_mode_config_funcs aspeed_gfx_mode_config_funcs = {
+	.fb_create		= drm_gem_fb_create,
+	.atomic_check		= drm_atomic_helper_check,
+	.atomic_commit		= drm_atomic_helper_commit,
+};
+
+static void aspeed_gfx_setup_mode_config(struct drm_device *drm)
+{
+	drm_mode_config_init(drm);
+
+	drm->mode_config.min_width = 0;
+	drm->mode_config.min_height = 0;
+	drm->mode_config.max_width = 800;
+	drm->mode_config.max_height = 600;
+	drm->mode_config.funcs = &aspeed_gfx_mode_config_funcs;
+}
+
+static irqreturn_t aspeed_gfx_irq_handler(int irq, void *data)
+{
+	struct drm_device *drm = data;
+	struct aspeed_gfx *priv = drm->dev_private;
+	u32 reg;
+
+	reg = readl(priv->base + CRT_CTRL1);
+
+	if (reg & CRT_CTRL_VERTICAL_INTR_STS) {
+		drm_crtc_handle_vblank(&priv->pipe.crtc);
+		writel(reg, priv->base + CRT_CTRL1);
+		return IRQ_HANDLED;
+	}
+
+	return IRQ_NONE;
+}
+
+
+
+static int aspeed_gfx_load(struct drm_device *drm)
+{
+	struct platform_device *pdev = to_platform_device(drm->dev);
+	struct aspeed_gfx *priv;
+	struct resource *res;
+	int ret;
+
+	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+	drm->dev_private = priv;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	priv->base = devm_ioremap_resource(drm->dev, res);
+	if (IS_ERR(priv->base))
+		return PTR_ERR(priv->base);
+
+	priv->scu = syscon_regmap_lookup_by_compatible("aspeed,ast2500-scu");
+	if (IS_ERR(priv->scu)) {
+		dev_err(&pdev->dev, "failed to find SCU regmap\n");
+		return PTR_ERR(priv->scu);
+	}
+
+	ret = of_reserved_mem_device_init(drm->dev);
+	if (ret) {
+		dev_err(&pdev->dev,
+			"failed to initialize reserved mem: %d\n", ret);
+		return ret;
+	}
+
+	ret = dma_set_mask_and_coherent(drm->dev, DMA_BIT_MASK(32));
+	if (ret) {
+		dev_err(&pdev->dev, "failed to set DMA mask: %d\n", ret);
+		return ret;
+	}
+
+	priv->rst = devm_reset_control_get_exclusive(&pdev->dev, NULL);
+	if (IS_ERR(priv->rst)) {
+		dev_err(&pdev->dev,
+			"missing or invalid reset controller device tree entry");
+		return PTR_ERR(priv->rst);
+	}
+	reset_control_deassert(priv->rst);
+
+	priv->clk = devm_clk_get(drm->dev, NULL);
+	if (IS_ERR(priv->clk)) {
+		dev_err(&pdev->dev,
+			"missing or invalid clk device tree entry");
+		return PTR_ERR(priv->clk);
+	}
+	clk_prepare_enable(priv->clk);
+
+	/* Sanitize control registers */
+	writel(0, priv->base + CRT_CTRL1);
+	writel(0, priv->base + CRT_CTRL2);
+
+	aspeed_gfx_setup_mode_config(drm);
+
+	ret = drm_vblank_init(drm, 1);
+	if (ret < 0) {
+		dev_err(drm->dev, "Failed to initialise vblank\n");
+		return ret;
+	}
+
+	ret = aspeed_gfx_create_output(drm);
+	if (ret < 0) {
+		dev_err(drm->dev, "Failed to create outputs\n");
+		return ret;
+	}
+
+	ret = aspeed_gfx_create_pipe(drm);
+	if (ret < 0) {
+		dev_err(drm->dev, "Cannot setup simple display pipe\n");
+		return ret;
+	}
+
+	ret = devm_request_irq(drm->dev, platform_get_irq(pdev, 0),
+			       aspeed_gfx_irq_handler, 0, "aspeed gfx", drm);
+	if (ret < 0) {
+		dev_err(drm->dev, "Failed to install IRQ handler\n");
+		return ret;
+	}
+
+	drm_mode_config_reset(drm);
+
+	drm_fbdev_generic_setup(drm, 32);
+
+	return 0;
+}
+
+static void aspeed_gfx_unload(struct drm_device *drm)
+{
+	drm_kms_helper_poll_fini(drm);
+	drm_mode_config_cleanup(drm);
+
+	drm->dev_private = NULL;
+}
+
+DEFINE_DRM_GEM_CMA_FOPS(fops);
+
+static struct drm_driver aspeed_gfx_driver = {
+	.driver_features        = DRIVER_GEM | DRIVER_MODESET |
+				DRIVER_PRIME | DRIVER_ATOMIC,
+	.gem_create_object	= drm_cma_gem_create_object_default_funcs,
+	.dumb_create		= drm_gem_cma_dumb_create,
+	.prime_handle_to_fd	= drm_gem_prime_handle_to_fd,
+	.prime_fd_to_handle	= drm_gem_prime_fd_to_handle,
+	.gem_prime_import_sg_table = drm_gem_cma_prime_import_sg_table,
+	.gem_prime_mmap		= drm_gem_prime_mmap,
+	.fops = &fops,
+	.name = "aspeed-gfx-drm",
+	.desc = "ASPEED GFX DRM",
+	.date = "20180319",
+	.major = 1,
+	.minor = 0,
+};
+
+static const struct of_device_id aspeed_gfx_match[] = {
+	{ .compatible = "aspeed,ast2500-gfx" },
+	{ }
+};
+
+static int aspeed_gfx_probe(struct platform_device *pdev)
+{
+	struct drm_device *drm;
+	int ret;
+
+	drm = drm_dev_alloc(&aspeed_gfx_driver, &pdev->dev);
+	if (IS_ERR(drm))
+		return PTR_ERR(drm);
+
+	ret = aspeed_gfx_load(drm);
+	if (ret)
+		goto err_free;
+
+	ret = drm_dev_register(drm, 0);
+	if (ret)
+		goto err_unload;
+
+	return 0;
+
+err_unload:
+	aspeed_gfx_unload(drm);
+err_free:
+	drm_dev_put(drm);
+
+	return ret;
+}
+
+static int aspeed_gfx_remove(struct platform_device *pdev)
+{
+	struct drm_device *drm = platform_get_drvdata(pdev);
+
+	drm_dev_unregister(drm);
+	aspeed_gfx_unload(drm);
+	drm_dev_put(drm);
+
+	return 0;
+}
+
+static struct platform_driver aspeed_gfx_platform_driver = {
+	.probe		= aspeed_gfx_probe,
+	.remove		= aspeed_gfx_remove,
+	.driver = {
+		.name = "aspeed_gfx",
+		.of_match_table = aspeed_gfx_match,
+	},
+};
+
+module_platform_driver(aspeed_gfx_platform_driver);
+
+MODULE_AUTHOR("Joel Stanley <joel@jms.id.au>");
+MODULE_DESCRIPTION("ASPEED BMC DRM/KMS driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/aspeed/aspeed_gfx_out.c b/drivers/gpu/drm/aspeed/aspeed_gfx_out.c
new file mode 100644
index 000000000000..67ee5fa10055
--- /dev/null
+++ b/drivers/gpu/drm/aspeed/aspeed_gfx_out.c
@@ -0,0 +1,42 @@
+// SPDX-License-Identifier: GPL-2.0+
+// Copyright 2018 IBM Corporation
+
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_connector.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_probe_helper.h>
+
+#include "aspeed_gfx.h"
+
+static int aspeed_gfx_get_modes(struct drm_connector *connector)
+{
+	return drm_add_modes_noedid(connector, 800, 600);
+}
+
+static const struct
+drm_connector_helper_funcs aspeed_gfx_connector_helper_funcs = {
+	.get_modes = aspeed_gfx_get_modes,
+};
+
+static const struct drm_connector_funcs aspeed_gfx_connector_funcs = {
+	.fill_modes		= drm_helper_probe_single_connector_modes,
+	.destroy		= drm_connector_cleanup,
+	.reset			= drm_atomic_helper_connector_reset,
+	.atomic_duplicate_state	= drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state	= drm_atomic_helper_connector_destroy_state,
+};
+
+int aspeed_gfx_create_output(struct drm_device *drm)
+{
+	struct aspeed_gfx *priv = drm->dev_private;
+	int ret;
+
+	priv->connector.dpms = DRM_MODE_DPMS_OFF;
+	priv->connector.polled = 0;
+	drm_connector_helper_add(&priv->connector,
+				 &aspeed_gfx_connector_helper_funcs);
+	ret = drm_connector_init(drm, &priv->connector,
+				 &aspeed_gfx_connector_funcs,
+				 DRM_MODE_CONNECTOR_Unknown);
+	return ret;
+}
diff --git a/drivers/gpu/drm/ast/ast_drv.h b/drivers/gpu/drm/ast/ast_drv.h
index bfc65040dfcb..1cf0c75e411d 100644
--- a/drivers/gpu/drm/ast/ast_drv.h
+++ b/drivers/gpu/drm/ast/ast_drv.h
@@ -259,7 +259,7 @@ struct ast_framebuffer {
 };
 
 struct ast_fbdev {
-	struct drm_fb_helper helper;
+	struct drm_fb_helper helper; /* must be first */
 	struct ast_framebuffer afb;
 	void *sysram;
 	int size;
@@ -353,8 +353,6 @@ extern int ast_dumb_mmap_offset(struct drm_file *file,
 				uint32_t handle,
 				uint64_t *offset);
 
-#define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT)
-
 int ast_mm_init(struct ast_private *ast);
 void ast_mm_fini(struct ast_private *ast);
 
diff --git a/drivers/gpu/drm/ast/ast_fb.c b/drivers/gpu/drm/ast/ast_fb.c
index 2c9f8dd9733a..e718d0f60d6b 100644
--- a/drivers/gpu/drm/ast/ast_fb.c
+++ b/drivers/gpu/drm/ast/ast_fb.c
@@ -217,8 +217,6 @@ static int astfb_create(struct drm_fb_helper *helper,
 		ret = PTR_ERR(info);
 		goto out;
 	}
-	info->par = afbdev;
-
 	ret = ast_framebuffer_init(dev, &afbdev->afb, &mode_cmd, gobj);
 	if (ret)
 		goto out;
@@ -229,15 +227,12 @@ static int astfb_create(struct drm_fb_helper *helper,
 	fb = &afbdev->afb.base;
 	afbdev->helper.fb = fb;
 
-	strcpy(info->fix.id, "astdrmfb");
-
 	info->fbops = &astfb_ops;
 
 	info->apertures->ranges[0].base = pci_resource_start(dev->pdev, 0);
 	info->apertures->ranges[0].size = pci_resource_len(dev->pdev, 0);
 
-	drm_fb_helper_fill_fix(info, fb->pitches[0], fb->format->depth);
-	drm_fb_helper_fill_var(info, &afbdev->helper, sizes->fb_width, sizes->fb_height);
+	drm_fb_helper_fill_info(info, &afbdev->helper, sizes);
 
 	info->screen_base = sysram;
 	info->screen_size = size;
diff --git a/drivers/gpu/drm/ast/ast_ttm.c b/drivers/gpu/drm/ast/ast_ttm.c
index c168d62fe8f9..75d477b37854 100644
--- a/drivers/gpu/drm/ast/ast_ttm.c
+++ b/drivers/gpu/drm/ast/ast_ttm.c
@@ -178,7 +178,6 @@ int ast_mm_init(struct ast_private *ast)
 	ret = ttm_bo_device_init(&ast->ttm.bdev,
 				 &ast_bo_driver,
 				 dev->anon_inode->i_mapping,
-				 DRM_FILE_PAGE_OFFSET,
 				 true);
 	if (ret) {
 		DRM_ERROR("Error initialising bo driver; %d\n", ret);
@@ -344,13 +343,8 @@ int ast_bo_push_sysram(struct ast_bo *bo)
 
 int ast_mmap(struct file *filp, struct vm_area_struct *vma)
 {
-	struct drm_file *file_priv;
-	struct ast_private *ast;
+	struct drm_file *file_priv = filp->private_data;
+	struct ast_private *ast = file_priv->minor->dev->dev_private;
 
-	if (unlikely(vma->vm_pgoff < DRM_FILE_PAGE_OFFSET))
-		return -EINVAL;
-
-	file_priv = filp->private_data;
-	ast = file_priv->minor->dev->dev_private;
 	return ttm_bo_mmap(filp, vma, &ast->ttm.bdev);
 }
diff --git a/drivers/gpu/drm/bochs/bochs.h b/drivers/gpu/drm/bochs/bochs.h
index 03711394f1ed..341cc9d1bab4 100644
--- a/drivers/gpu/drm/bochs/bochs.h
+++ b/drivers/gpu/drm/bochs/bochs.h
@@ -7,6 +7,7 @@
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_encoder.h>
 #include <drm/drm_fb_helper.h>
+#include <drm/drm_simple_kms_helper.h>
 
 #include <drm/drm_gem.h>
 
@@ -69,11 +70,9 @@ struct bochs_device {
 	struct edid *edid;
 
 	/* drm */
-	struct drm_device  *dev;
-	struct drm_crtc crtc;
-	struct drm_encoder encoder;
+	struct drm_device *dev;
+	struct drm_simple_display_pipe pipe;
 	struct drm_connector connector;
-	bool mode_config_initialized;
 
 	/* ttm */
 	struct {
@@ -101,8 +100,6 @@ static inline struct bochs_bo *gem_to_bochs_bo(struct drm_gem_object *gem)
 	return container_of(gem, struct bochs_bo, gem);
 }
 
-#define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT)
-
 static inline u64 bochs_bo_mmap_offset(struct bochs_bo *bo)
 {
 	return drm_vma_node_offset_addr(&bo->bo.vma_node);
diff --git a/drivers/gpu/drm/bochs/bochs_kms.c b/drivers/gpu/drm/bochs/bochs_kms.c
index 9cd82e3631fb..5e905f50449d 100644
--- a/drivers/gpu/drm/bochs/bochs_kms.c
+++ b/drivers/gpu/drm/bochs/bochs_kms.c
@@ -22,76 +22,55 @@ MODULE_PARM_DESC(defy, "default y resolution");
 
 /* ---------------------------------------------------------------------- */
 
-static void bochs_crtc_mode_set_nofb(struct drm_crtc *crtc)
+static const uint32_t bochs_formats[] = {
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_BGRX8888,
+};
+
+static void bochs_plane_update(struct bochs_device *bochs,
+			       struct drm_plane_state *state)
 {
-	struct bochs_device *bochs =
-		container_of(crtc, struct bochs_device, crtc);
+	struct bochs_bo *bo;
 
-	bochs_hw_setmode(bochs, &crtc->mode);
+	if (!state->fb || !bochs->stride)
+		return;
+
+	bo = gem_to_bochs_bo(state->fb->obj[0]);
+	bochs_hw_setbase(bochs,
+			 state->crtc_x,
+			 state->crtc_y,
+			 bo->bo.offset);
+	bochs_hw_setformat(bochs, state->fb->format);
 }
 
-static void bochs_crtc_atomic_enable(struct drm_crtc *crtc,
-				     struct drm_crtc_state *old_crtc_state)
+static void bochs_pipe_enable(struct drm_simple_display_pipe *pipe,
+			      struct drm_crtc_state *crtc_state,
+			      struct drm_plane_state *plane_state)
 {
+	struct bochs_device *bochs = pipe->crtc.dev->dev_private;
+
+	bochs_hw_setmode(bochs, &crtc_state->mode);
+	bochs_plane_update(bochs, plane_state);
 }
 
-static void bochs_crtc_atomic_flush(struct drm_crtc *crtc,
-				    struct drm_crtc_state *old_crtc_state)
+static void bochs_pipe_update(struct drm_simple_display_pipe *pipe,
+			      struct drm_plane_state *old_state)
 {
-	struct drm_device *dev = crtc->dev;
-	struct drm_pending_vblank_event *event;
+	struct bochs_device *bochs = pipe->crtc.dev->dev_private;
+	struct drm_crtc *crtc = &pipe->crtc;
 
-	if (crtc->state && crtc->state->event) {
-		unsigned long irqflags;
+	bochs_plane_update(bochs, pipe->plane.state);
 
-		spin_lock_irqsave(&dev->event_lock, irqflags);
-		event = crtc->state->event;
+	if (crtc->state->event) {
+		spin_lock_irq(&crtc->dev->event_lock);
+		drm_crtc_send_vblank_event(crtc, crtc->state->event);
 		crtc->state->event = NULL;
-		drm_crtc_send_vblank_event(crtc, event);
-		spin_unlock_irqrestore(&dev->event_lock, irqflags);
+		spin_unlock_irq(&crtc->dev->event_lock);
 	}
 }
 
-
-/* These provide the minimum set of functions required to handle a CRTC */
-static const struct drm_crtc_funcs bochs_crtc_funcs = {
-	.set_config = drm_atomic_helper_set_config,
-	.destroy = drm_crtc_cleanup,
-	.page_flip = drm_atomic_helper_page_flip,
-	.reset = drm_atomic_helper_crtc_reset,
-	.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
-	.atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
-};
-
-static const struct drm_crtc_helper_funcs bochs_helper_funcs = {
-	.mode_set_nofb = bochs_crtc_mode_set_nofb,
-	.atomic_enable = bochs_crtc_atomic_enable,
-	.atomic_flush = bochs_crtc_atomic_flush,
-};
-
-static const uint32_t bochs_formats[] = {
-	DRM_FORMAT_XRGB8888,
-	DRM_FORMAT_BGRX8888,
-};
-
-static void bochs_plane_atomic_update(struct drm_plane *plane,
-				      struct drm_plane_state *old_state)
-{
-	struct bochs_device *bochs = plane->dev->dev_private;
-	struct bochs_bo *bo;
-
-	if (!plane->state->fb)
-		return;
-	bo = gem_to_bochs_bo(plane->state->fb->obj[0]);
-	bochs_hw_setbase(bochs,
-			 plane->state->crtc_x,
-			 plane->state->crtc_y,
-			 bo->bo.offset);
-	bochs_hw_setformat(bochs, plane->state->fb->format);
-}
-
-static int bochs_plane_prepare_fb(struct drm_plane *plane,
-				struct drm_plane_state *new_state)
+static int bochs_pipe_prepare_fb(struct drm_simple_display_pipe *pipe,
+				 struct drm_plane_state *new_state)
 {
 	struct bochs_bo *bo;
 
@@ -101,8 +80,8 @@ static int bochs_plane_prepare_fb(struct drm_plane *plane,
 	return bochs_bo_pin(bo, TTM_PL_FLAG_VRAM);
 }
 
-static void bochs_plane_cleanup_fb(struct drm_plane *plane,
-				   struct drm_plane_state *old_state)
+static void bochs_pipe_cleanup_fb(struct drm_simple_display_pipe *pipe,
+				  struct drm_plane_state *old_state)
 {
 	struct bochs_bo *bo;
 
@@ -112,73 +91,13 @@ static void bochs_plane_cleanup_fb(struct drm_plane *plane,
 	bochs_bo_unpin(bo);
 }
 
-static const struct drm_plane_helper_funcs bochs_plane_helper_funcs = {
-	.atomic_update = bochs_plane_atomic_update,
-	.prepare_fb = bochs_plane_prepare_fb,
-	.cleanup_fb = bochs_plane_cleanup_fb,
-};
-
-static const struct drm_plane_funcs bochs_plane_funcs = {
-       .update_plane   = drm_atomic_helper_update_plane,
-       .disable_plane  = drm_atomic_helper_disable_plane,
-       .destroy        = drm_primary_helper_destroy,
-       .reset          = drm_atomic_helper_plane_reset,
-       .atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
-       .atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
-};
-
-static struct drm_plane *bochs_primary_plane(struct drm_device *dev)
-{
-	struct drm_plane *primary;
-	int ret;
-
-	primary = kzalloc(sizeof(*primary), GFP_KERNEL);
-	if (primary == NULL) {
-		DRM_DEBUG_KMS("Failed to allocate primary plane\n");
-		return NULL;
-	}
-
-	ret = drm_universal_plane_init(dev, primary, 0,
-				       &bochs_plane_funcs,
-				       bochs_formats,
-				       ARRAY_SIZE(bochs_formats),
-				       NULL,
-				       DRM_PLANE_TYPE_PRIMARY, NULL);
-	if (ret) {
-		kfree(primary);
-		return NULL;
-	}
-
-	drm_plane_helper_add(primary, &bochs_plane_helper_funcs);
-	return primary;
-}
-
-static void bochs_crtc_init(struct drm_device *dev)
-{
-	struct bochs_device *bochs = dev->dev_private;
-	struct drm_crtc *crtc = &bochs->crtc;
-	struct drm_plane *primary = bochs_primary_plane(dev);
-
-	drm_crtc_init_with_planes(dev, crtc, primary, NULL,
-				  &bochs_crtc_funcs, NULL);
-	drm_crtc_helper_add(crtc, &bochs_helper_funcs);
-}
-
-static const struct drm_encoder_funcs bochs_encoder_encoder_funcs = {
-	.destroy = drm_encoder_cleanup,
+static const struct drm_simple_display_pipe_funcs bochs_pipe_funcs = {
+	.enable	    = bochs_pipe_enable,
+	.update	    = bochs_pipe_update,
+	.prepare_fb = bochs_pipe_prepare_fb,
+	.cleanup_fb = bochs_pipe_cleanup_fb,
 };
 
-static void bochs_encoder_init(struct drm_device *dev)
-{
-	struct bochs_device *bochs = dev->dev_private;
-	struct drm_encoder *encoder = &bochs->encoder;
-
-	encoder->possible_crtcs = 0x1;
-	drm_encoder_init(dev, encoder, &bochs_encoder_encoder_funcs,
-			 DRM_MODE_ENCODER_DAC, NULL);
-}
-
-
 static int bochs_connector_get_modes(struct drm_connector *connector)
 {
 	struct bochs_device *bochs =
@@ -214,20 +133,9 @@ static enum drm_mode_status bochs_connector_mode_valid(struct drm_connector *con
 	return MODE_OK;
 }
 
-static struct drm_encoder *
-bochs_connector_best_encoder(struct drm_connector *connector)
-{
-	int enc_id = connector->encoder_ids[0];
-	/* pick the encoder ids */
-	if (enc_id)
-		return drm_encoder_find(connector->dev, NULL, enc_id);
-	return NULL;
-}
-
 static const struct drm_connector_helper_funcs bochs_connector_connector_helper_funcs = {
 	.get_modes = bochs_connector_get_modes,
 	.mode_valid = bochs_connector_mode_valid,
-	.best_encoder = bochs_connector_best_encoder,
 };
 
 static const struct drm_connector_funcs bochs_connector_connector_funcs = {
@@ -278,7 +186,6 @@ const struct drm_mode_config_funcs bochs_mode_funcs = {
 int bochs_kms_init(struct bochs_device *bochs)
 {
 	drm_mode_config_init(bochs->dev);
-	bochs->mode_config_initialized = true;
 
 	bochs->dev->mode_config.max_width = 8192;
 	bochs->dev->mode_config.max_height = 8192;
@@ -290,11 +197,14 @@ int bochs_kms_init(struct bochs_device *bochs)
 
 	bochs->dev->mode_config.funcs = &bochs_mode_funcs;
 
-	bochs_crtc_init(bochs->dev);
-	bochs_encoder_init(bochs->dev);
 	bochs_connector_init(bochs->dev);
-	drm_connector_attach_encoder(&bochs->connector,
-					  &bochs->encoder);
+	drm_simple_display_pipe_init(bochs->dev,
+				     &bochs->pipe,
+				     &bochs_pipe_funcs,
+				     bochs_formats,
+				     ARRAY_SIZE(bochs_formats),
+				     NULL,
+				     &bochs->connector);
 
 	drm_mode_config_reset(bochs->dev);
 
@@ -303,8 +213,6 @@ int bochs_kms_init(struct bochs_device *bochs)
 
 void bochs_kms_fini(struct bochs_device *bochs)
 {
-	if (bochs->mode_config_initialized) {
-		drm_mode_config_cleanup(bochs->dev);
-		bochs->mode_config_initialized = false;
-	}
+	drm_atomic_helper_shutdown(bochs->dev);
+	drm_mode_config_cleanup(bochs->dev);
 }
diff --git a/drivers/gpu/drm/bochs/bochs_mm.c b/drivers/gpu/drm/bochs/bochs_mm.c
index 49463348a07a..4a40308169c4 100644
--- a/drivers/gpu/drm/bochs/bochs_mm.c
+++ b/drivers/gpu/drm/bochs/bochs_mm.c
@@ -156,7 +156,6 @@ int bochs_mm_init(struct bochs_device *bochs)
 	ret = ttm_bo_device_init(&bochs->ttm.bdev,
 				 &bochs_bo_driver,
 				 bochs->dev->anon_inode->i_mapping,
-				 DRM_FILE_PAGE_OFFSET,
 				 true);
 	if (ret) {
 		DRM_ERROR("Error initialising bo driver; %d\n", ret);
@@ -264,14 +263,9 @@ int bochs_bo_unpin(struct bochs_bo *bo)
 
 int bochs_mmap(struct file *filp, struct vm_area_struct *vma)
 {
-	struct drm_file *file_priv;
-	struct bochs_device *bochs;
+	struct drm_file *file_priv = filp->private_data;
+	struct bochs_device *bochs = file_priv->minor->dev->dev_private;
 
-	if (unlikely(vma->vm_pgoff < DRM_FILE_PAGE_OFFSET))
-		return -EINVAL;
-
-	file_priv = filp->private_data;
-	bochs = file_priv->minor->dev->dev_private;
 	return ttm_bo_mmap(filp, vma, &bochs->ttm.bdev);
 }
 
diff --git a/drivers/gpu/drm/bridge/dumb-vga-dac.c b/drivers/gpu/drm/bridge/dumb-vga-dac.c
index 0805801f4e94..e64736c39a9f 100644
--- a/drivers/gpu/drm/bridge/dumb-vga-dac.c
+++ b/drivers/gpu/drm/bridge/dumb-vga-dac.c
@@ -234,7 +234,7 @@ static int dumb_vga_remove(struct platform_device *pdev)
  */
 static const struct drm_bridge_timings default_dac_timings = {
 	/* Timing specifications, datasheet page 7 */
-	.sampling_edge = DRM_BUS_FLAG_PIXDATA_POSEDGE,
+	.input_bus_flags = DRM_BUS_FLAG_PIXDATA_SAMPLE_POSEDGE,
 	.setup_time_ps = 500,
 	.hold_time_ps = 1500,
 };
@@ -245,7 +245,7 @@ static const struct drm_bridge_timings default_dac_timings = {
  */
 static const struct drm_bridge_timings ti_ths8134_dac_timings = {
 	/* From timing diagram, datasheet page 9 */
-	.sampling_edge = DRM_BUS_FLAG_PIXDATA_POSEDGE,
+	.input_bus_flags = DRM_BUS_FLAG_PIXDATA_SAMPLE_POSEDGE,
 	/* From datasheet, page 12 */
 	.setup_time_ps = 3000,
 	/* I guess this means latched input */
@@ -258,7 +258,7 @@ static const struct drm_bridge_timings ti_ths8134_dac_timings = {
  */
 static const struct drm_bridge_timings ti_ths8135_dac_timings = {
 	/* From timing diagram, datasheet page 14 */
-	.sampling_edge = DRM_BUS_FLAG_PIXDATA_POSEDGE,
+	.input_bus_flags = DRM_BUS_FLAG_PIXDATA_SAMPLE_POSEDGE,
 	/* From datasheet, page 16 */
 	.setup_time_ps = 2000,
 	.hold_time_ps = 500,
diff --git a/drivers/gpu/drm/bridge/tc358767.c b/drivers/gpu/drm/bridge/tc358767.c
index 888980d4bc74..e570c9dee180 100644
--- a/drivers/gpu/drm/bridge/tc358767.c
+++ b/drivers/gpu/drm/bridge/tc358767.c
@@ -1222,8 +1222,8 @@ static int tc_bridge_attach(struct drm_bridge *bridge)
 					 &bus_format, 1);
 	tc->connector.display_info.bus_flags =
 		DRM_BUS_FLAG_DE_HIGH |
-		DRM_BUS_FLAG_PIXDATA_NEGEDGE |
-		DRM_BUS_FLAG_SYNC_NEGEDGE;
+		DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE |
+		DRM_BUS_FLAG_SYNC_DRIVE_NEGEDGE;
 	drm_connector_attach_encoder(&tc->connector, tc->bridge.encoder);
 
 	return 0;
diff --git a/drivers/gpu/drm/bridge/ti-tfp410.c b/drivers/gpu/drm/bridge/ti-tfp410.c
index 7bfb4f338813..8b0e71bd3ca7 100644
--- a/drivers/gpu/drm/bridge/ti-tfp410.c
+++ b/drivers/gpu/drm/bridge/ti-tfp410.c
@@ -27,10 +27,16 @@
 struct tfp410 {
 	struct drm_bridge	bridge;
 	struct drm_connector	connector;
+	unsigned int		connector_type;
 
+	u32			bus_format;
 	struct i2c_adapter	*ddc;
 	struct gpio_desc	*hpd;
+	int			hpd_irq;
 	struct delayed_work	hpd_work;
+	struct gpio_desc	*powerdown;
+
+	struct drm_bridge_timings timings;
 
 	struct device *dev;
 };
@@ -120,26 +126,47 @@ static int tfp410_attach(struct drm_bridge *bridge)
 		return -ENODEV;
 	}
 
-	if (dvi->hpd)
+	if (dvi->hpd_irq >= 0)
 		dvi->connector.polled = DRM_CONNECTOR_POLL_HPD;
+	else
+		dvi->connector.polled = DRM_CONNECTOR_POLL_CONNECT | DRM_CONNECTOR_POLL_DISCONNECT;
 
 	drm_connector_helper_add(&dvi->connector,
 				 &tfp410_con_helper_funcs);
 	ret = drm_connector_init(bridge->dev, &dvi->connector,
-				 &tfp410_con_funcs, DRM_MODE_CONNECTOR_HDMIA);
+				 &tfp410_con_funcs, dvi->connector_type);
 	if (ret) {
 		dev_err(dvi->dev, "drm_connector_init() failed: %d\n", ret);
 		return ret;
 	}
 
+	drm_display_info_set_bus_formats(&dvi->connector.display_info,
+					 &dvi->bus_format, 1);
+
 	drm_connector_attach_encoder(&dvi->connector,
 					  bridge->encoder);
 
 	return 0;
 }
 
+static void tfp410_enable(struct drm_bridge *bridge)
+{
+	struct tfp410 *dvi = drm_bridge_to_tfp410(bridge);
+
+	gpiod_set_value_cansleep(dvi->powerdown, 0);
+}
+
+static void tfp410_disable(struct drm_bridge *bridge)
+{
+	struct tfp410 *dvi = drm_bridge_to_tfp410(bridge);
+
+	gpiod_set_value_cansleep(dvi->powerdown, 1);
+}
+
 static const struct drm_bridge_funcs tfp410_bridge_funcs = {
 	.attach		= tfp410_attach,
+	.enable		= tfp410_enable,
+	.disable	= tfp410_disable,
 };
 
 static void tfp410_hpd_work_func(struct work_struct *work)
@@ -162,6 +189,83 @@ static irqreturn_t tfp410_hpd_irq_thread(int irq, void *arg)
 	return IRQ_HANDLED;
 }
 
+static const struct drm_bridge_timings tfp410_default_timings = {
+	.input_bus_flags = DRM_BUS_FLAG_PIXDATA_SAMPLE_POSEDGE
+			 | DRM_BUS_FLAG_DE_HIGH,
+	.setup_time_ps = 1200,
+	.hold_time_ps = 1300,
+};
+
+static int tfp410_parse_timings(struct tfp410 *dvi, bool i2c)
+{
+	struct drm_bridge_timings *timings = &dvi->timings;
+	struct device_node *ep;
+	u32 pclk_sample = 0;
+	u32 bus_width = 24;
+	s32 deskew = 0;
+
+	/* Start with defaults. */
+	*timings = tfp410_default_timings;
+
+	if (i2c)
+		/*
+		 * In I2C mode timings are configured through the I2C interface.
+		 * As the driver doesn't support I2C configuration yet, we just
+		 * go with the defaults (BSEL=1, DSEL=1, DKEN=0, EDGE=1).
+		 */
+		return 0;
+
+	/*
+	 * In non-I2C mode, timings are configured through the BSEL, DSEL, DKEN
+	 * and EDGE pins. They are specified in DT through endpoint properties
+	 * and vendor-specific properties.
+	 */
+	ep = of_graph_get_endpoint_by_regs(dvi->dev->of_node, 0, 0);
+	if (!ep)
+		return -EINVAL;
+
+	/* Get the sampling edge from the endpoint. */
+	of_property_read_u32(ep, "pclk-sample", &pclk_sample);
+	of_property_read_u32(ep, "bus-width", &bus_width);
+	of_node_put(ep);
+
+	timings->input_bus_flags = DRM_BUS_FLAG_DE_HIGH;
+
+	switch (pclk_sample) {
+	case 0:
+		timings->input_bus_flags |= DRM_BUS_FLAG_PIXDATA_SAMPLE_NEGEDGE
+					 |  DRM_BUS_FLAG_SYNC_SAMPLE_NEGEDGE;
+		break;
+	case 1:
+		timings->input_bus_flags |= DRM_BUS_FLAG_PIXDATA_SAMPLE_POSEDGE
+					 |  DRM_BUS_FLAG_SYNC_SAMPLE_POSEDGE;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	switch (bus_width) {
+	case 12:
+		dvi->bus_format = MEDIA_BUS_FMT_RGB888_2X12_LE;
+		break;
+	case 24:
+		dvi->bus_format = MEDIA_BUS_FMT_RGB888_1X24;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	/* Get the setup and hold time from vendor-specific properties. */
+	of_property_read_u32(dvi->dev->of_node, "ti,deskew", (u32 *)&deskew);
+	if (deskew < -4 || deskew > 3)
+		return -EINVAL;
+
+	timings->setup_time_ps = min(0, 1200 - 350 * deskew);
+	timings->hold_time_ps = min(0, 1300 + 350 * deskew);
+
+	return 0;
+}
+
 static int tfp410_get_connector_properties(struct tfp410 *dvi)
 {
 	struct device_node *connector_node, *ddc_phandle;
@@ -172,6 +276,11 @@ static int tfp410_get_connector_properties(struct tfp410 *dvi)
 	if (!connector_node)
 		return -ENODEV;
 
+	if (of_device_is_compatible(connector_node, "hdmi-connector"))
+		dvi->connector_type = DRM_MODE_CONNECTOR_HDMIA;
+	else
+		dvi->connector_type = DRM_MODE_CONNECTOR_DVID;
+
 	dvi->hpd = fwnode_get_named_gpiod(&connector_node->fwnode,
 					"hpd-gpios", 0, GPIOD_IN, "hpd");
 	if (IS_ERR(dvi->hpd)) {
@@ -200,7 +309,7 @@ fail:
 	return ret;
 }
 
-static int tfp410_init(struct device *dev)
+static int tfp410_init(struct device *dev, bool i2c)
 {
 	struct tfp410 *dvi;
 	int ret;
@@ -217,16 +326,33 @@ static int tfp410_init(struct device *dev)
 
 	dvi->bridge.funcs = &tfp410_bridge_funcs;
 	dvi->bridge.of_node = dev->of_node;
+	dvi->bridge.timings = &dvi->timings;
 	dvi->dev = dev;
 
+	ret = tfp410_parse_timings(dvi, i2c);
+	if (ret)
+		goto fail;
+
 	ret = tfp410_get_connector_properties(dvi);
 	if (ret)
 		goto fail;
 
-	if (dvi->hpd) {
+	dvi->powerdown = devm_gpiod_get_optional(dev, "powerdown",
+						 GPIOD_OUT_HIGH);
+	if (IS_ERR(dvi->powerdown)) {
+		dev_err(dev, "failed to parse powerdown gpio\n");
+		return PTR_ERR(dvi->powerdown);
+	}
+
+	if (dvi->hpd)
+		dvi->hpd_irq = gpiod_to_irq(dvi->hpd);
+	else
+		dvi->hpd_irq = -ENXIO;
+
+	if (dvi->hpd_irq >= 0) {
 		INIT_DELAYED_WORK(&dvi->hpd_work, tfp410_hpd_work_func);
 
-		ret = devm_request_threaded_irq(dev, gpiod_to_irq(dvi->hpd),
+		ret = devm_request_threaded_irq(dev, dvi->hpd_irq,
 			NULL, tfp410_hpd_irq_thread, IRQF_TRIGGER_RISING |
 			IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
 			"hdmi-hpd", dvi);
@@ -264,7 +390,7 @@ static int tfp410_fini(struct device *dev)
 
 static int tfp410_probe(struct platform_device *pdev)
 {
-	return tfp410_init(&pdev->dev);
+	return tfp410_init(&pdev->dev, false);
 }
 
 static int tfp410_remove(struct platform_device *pdev)
@@ -301,7 +427,7 @@ static int tfp410_i2c_probe(struct i2c_client *client,
 		return -ENXIO;
 	}
 
-	return tfp410_init(&client->dev);
+	return tfp410_init(&client->dev, true);
 }
 
 static int tfp410_i2c_remove(struct i2c_client *client)
diff --git a/drivers/gpu/drm/cirrus/Kconfig b/drivers/gpu/drm/cirrus/Kconfig
index fc78c90ee931..dd4f52a0bc1c 100644
--- a/drivers/gpu/drm/cirrus/Kconfig
+++ b/drivers/gpu/drm/cirrus/Kconfig
@@ -2,7 +2,7 @@ config DRM_CIRRUS_QEMU
 	tristate "Cirrus driver for QEMU emulated device"
 	depends on DRM && PCI && MMU
 	select DRM_KMS_HELPER
-	select DRM_TTM
+	select DRM_GEM_SHMEM_HELPER
 	help
 	 This is a KMS driver for emulated cirrus device in qemu.
 	 It is *NOT* intended for real cirrus devices. This requires
diff --git a/drivers/gpu/drm/cirrus/Makefile b/drivers/gpu/drm/cirrus/Makefile
index 919c0a336c97..acf8971d37a1 100644
--- a/drivers/gpu/drm/cirrus/Makefile
+++ b/drivers/gpu/drm/cirrus/Makefile
@@ -1,4 +1 @@
-cirrus-y  := cirrus_main.o cirrus_mode.o \
-	cirrus_drv.o cirrus_fbdev.o cirrus_ttm.o
-
 obj-$(CONFIG_DRM_CIRRUS_QEMU) += cirrus.o
diff --git a/drivers/gpu/drm/cirrus/cirrus.c b/drivers/gpu/drm/cirrus/cirrus.c
new file mode 100644
index 000000000000..be4ea370ba31
--- /dev/null
+++ b/drivers/gpu/drm/cirrus/cirrus.c
@@ -0,0 +1,657 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright 2012-2019 Red Hat
+ *
+ * This file is subject to the terms and conditions of the GNU General
+ * Public License version 2. See the file COPYING in the main
+ * directory of this archive for more details.
+ *
+ * Authors: Matthew Garrett
+ *	    Dave Airlie
+ *	    Gerd Hoffmann
+ *
+ * Portions of this code derived from cirrusfb.c:
+ * drivers/video/cirrusfb.c - driver for Cirrus Logic chipsets
+ *
+ * Copyright 1999-2001 Jeff Garzik <jgarzik@pobox.com>
+ */
+
+#include <linux/console.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+
+#include <video/cirrus.h>
+#include <video/vga.h>
+
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_atomic_state_helper.h>
+#include <drm/drm_connector.h>
+#include <drm/drm_damage_helper.h>
+#include <drm/drm_drv.h>
+#include <drm/drm_fb_helper.h>
+#include <drm/drm_file.h>
+#include <drm/drm_format_helper.h>
+#include <drm/drm_fourcc.h>
+#include <drm/drm_gem_shmem_helper.h>
+#include <drm/drm_gem_framebuffer_helper.h>
+#include <drm/drm_ioctl.h>
+#include <drm/drm_modeset_helper_vtables.h>
+#include <drm/drm_probe_helper.h>
+#include <drm/drm_simple_kms_helper.h>
+#include <drm/drm_vblank.h>
+
+#define DRIVER_NAME "cirrus"
+#define DRIVER_DESC "qemu cirrus vga"
+#define DRIVER_DATE "2019"
+#define DRIVER_MAJOR 2
+#define DRIVER_MINOR 0
+
+#define CIRRUS_MAX_PITCH (0x1FF << 3)      /* (4096 - 1) & ~111b bytes */
+#define CIRRUS_VRAM_SIZE (4 * 1024 * 1024) /* 4 MB */
+
+struct cirrus_device {
+	struct drm_device	       dev;
+	struct drm_simple_display_pipe pipe;
+	struct drm_connector	       conn;
+	unsigned int		       cpp;
+	unsigned int		       pitch;
+	void __iomem		       *vram;
+	void __iomem		       *mmio;
+};
+
+/* ------------------------------------------------------------------ */
+/*
+ * The meat of this driver. The core passes us a mode and we have to program
+ * it. The modesetting here is the bare minimum required to satisfy the qemu
+ * emulation of this hardware, and running this against a real device is
+ * likely to result in an inadequately programmed mode. We've already had
+ * the opportunity to modify the mode, so whatever we receive here should
+ * be something that can be correctly programmed and displayed
+ */
+
+#define SEQ_INDEX 4
+#define SEQ_DATA 5
+
+static u8 rreg_seq(struct cirrus_device *cirrus, u8 reg)
+{
+	iowrite8(reg, cirrus->mmio + SEQ_INDEX);
+	return ioread8(cirrus->mmio + SEQ_DATA);
+}
+
+static void wreg_seq(struct cirrus_device *cirrus, u8 reg, u8 val)
+{
+	iowrite8(reg, cirrus->mmio + SEQ_INDEX);
+	iowrite8(val, cirrus->mmio + SEQ_DATA);
+}
+
+#define CRT_INDEX 0x14
+#define CRT_DATA 0x15
+
+static u8 rreg_crt(struct cirrus_device *cirrus, u8 reg)
+{
+	iowrite8(reg, cirrus->mmio + CRT_INDEX);
+	return ioread8(cirrus->mmio + CRT_DATA);
+}
+
+static void wreg_crt(struct cirrus_device *cirrus, u8 reg, u8 val)
+{
+	iowrite8(reg, cirrus->mmio + CRT_INDEX);
+	iowrite8(val, cirrus->mmio + CRT_DATA);
+}
+
+#define GFX_INDEX 0xe
+#define GFX_DATA 0xf
+
+static void wreg_gfx(struct cirrus_device *cirrus, u8 reg, u8 val)
+{
+	iowrite8(reg, cirrus->mmio + GFX_INDEX);
+	iowrite8(val, cirrus->mmio + GFX_DATA);
+}
+
+#define VGA_DAC_MASK  0x06
+
+static void wreg_hdr(struct cirrus_device *cirrus, u8 val)
+{
+	ioread8(cirrus->mmio + VGA_DAC_MASK);
+	ioread8(cirrus->mmio + VGA_DAC_MASK);
+	ioread8(cirrus->mmio + VGA_DAC_MASK);
+	ioread8(cirrus->mmio + VGA_DAC_MASK);
+	iowrite8(val, cirrus->mmio + VGA_DAC_MASK);
+}
+
+static int cirrus_convert_to(struct drm_framebuffer *fb)
+{
+	if (fb->format->cpp[0] == 4 && fb->pitches[0] > CIRRUS_MAX_PITCH) {
+		if (fb->width * 3 <= CIRRUS_MAX_PITCH)
+			/* convert from XR24 to RG24 */
+			return 3;
+		else
+			/* convert from XR24 to RG16 */
+			return 2;
+	}
+	return 0;
+}
+
+static int cirrus_cpp(struct drm_framebuffer *fb)
+{
+	int convert_cpp = cirrus_convert_to(fb);
+
+	if (convert_cpp)
+		return convert_cpp;
+	return fb->format->cpp[0];
+}
+
+static int cirrus_pitch(struct drm_framebuffer *fb)
+{
+	int convert_cpp = cirrus_convert_to(fb);
+
+	if (convert_cpp)
+		return convert_cpp * fb->width;
+	return fb->pitches[0];
+}
+
+static void cirrus_set_start_address(struct cirrus_device *cirrus, u32 offset)
+{
+	u32 addr;
+	u8 tmp;
+
+	addr = offset >> 2;
+	wreg_crt(cirrus, 0x0c, (u8)((addr >> 8) & 0xff));
+	wreg_crt(cirrus, 0x0d, (u8)(addr & 0xff));
+
+	tmp = rreg_crt(cirrus, 0x1b);
+	tmp &= 0xf2;
+	tmp |= (addr >> 16) & 0x01;
+	tmp |= (addr >> 15) & 0x0c;
+	wreg_crt(cirrus, 0x1b, tmp);
+
+	tmp = rreg_crt(cirrus, 0x1d);
+	tmp &= 0x7f;
+	tmp |= (addr >> 12) & 0x80;
+	wreg_crt(cirrus, 0x1d, tmp);
+}
+
+static int cirrus_mode_set(struct cirrus_device *cirrus,
+			   struct drm_display_mode *mode,
+			   struct drm_framebuffer *fb)
+{
+	int hsyncstart, hsyncend, htotal, hdispend;
+	int vtotal, vdispend;
+	int tmp;
+	int sr07 = 0, hdr = 0;
+
+	htotal = mode->htotal / 8;
+	hsyncend = mode->hsync_end / 8;
+	hsyncstart = mode->hsync_start / 8;
+	hdispend = mode->hdisplay / 8;
+
+	vtotal = mode->vtotal;
+	vdispend = mode->vdisplay;
+
+	vdispend -= 1;
+	vtotal -= 2;
+
+	htotal -= 5;
+	hdispend -= 1;
+	hsyncstart += 1;
+	hsyncend += 1;
+
+	wreg_crt(cirrus, VGA_CRTC_V_SYNC_END, 0x20);
+	wreg_crt(cirrus, VGA_CRTC_H_TOTAL, htotal);
+	wreg_crt(cirrus, VGA_CRTC_H_DISP, hdispend);
+	wreg_crt(cirrus, VGA_CRTC_H_SYNC_START, hsyncstart);
+	wreg_crt(cirrus, VGA_CRTC_H_SYNC_END, hsyncend);
+	wreg_crt(cirrus, VGA_CRTC_V_TOTAL, vtotal & 0xff);
+	wreg_crt(cirrus, VGA_CRTC_V_DISP_END, vdispend & 0xff);
+
+	tmp = 0x40;
+	if ((vdispend + 1) & 512)
+		tmp |= 0x20;
+	wreg_crt(cirrus, VGA_CRTC_MAX_SCAN, tmp);
+
+	/*
+	 * Overflow bits for values that don't fit in the standard registers
+	 */
+	tmp = 0x10;
+	if (vtotal & 0x100)
+		tmp |= 0x01;
+	if (vdispend & 0x100)
+		tmp |= 0x02;
+	if ((vdispend + 1) & 0x100)
+		tmp |= 0x08;
+	if (vtotal & 0x200)
+		tmp |= 0x20;
+	if (vdispend & 0x200)
+		tmp |= 0x40;
+	wreg_crt(cirrus, VGA_CRTC_OVERFLOW, tmp);
+
+	tmp = 0;
+
+	/* More overflow bits */
+
+	if ((htotal + 5) & 0x40)
+		tmp |= 0x10;
+	if ((htotal + 5) & 0x80)
+		tmp |= 0x20;
+	if (vtotal & 0x100)
+		tmp |= 0x40;
+	if (vtotal & 0x200)
+		tmp |= 0x80;
+
+	wreg_crt(cirrus, CL_CRT1A, tmp);
+
+	/* Disable Hercules/CGA compatibility */
+	wreg_crt(cirrus, VGA_CRTC_MODE, 0x03);
+
+	sr07 = rreg_seq(cirrus, 0x07);
+	sr07 &= 0xe0;
+	hdr = 0;
+
+	cirrus->cpp = cirrus_cpp(fb);
+	switch (cirrus->cpp * 8) {
+	case 8:
+		sr07 |= 0x11;
+		break;
+	case 16:
+		sr07 |= 0x17;
+		hdr = 0xc1;
+		break;
+	case 24:
+		sr07 |= 0x15;
+		hdr = 0xc5;
+		break;
+	case 32:
+		sr07 |= 0x19;
+		hdr = 0xc5;
+		break;
+	default:
+		return -1;
+	}
+
+	wreg_seq(cirrus, 0x7, sr07);
+
+	/* Program the pitch */
+	cirrus->pitch = cirrus_pitch(fb);
+	tmp = cirrus->pitch / 8;
+	wreg_crt(cirrus, VGA_CRTC_OFFSET, tmp);
+
+	/* Enable extended blanking and pitch bits, and enable full memory */
+	tmp = 0x22;
+	tmp |= (cirrus->pitch >> 7) & 0x10;
+	tmp |= (cirrus->pitch >> 6) & 0x40;
+	wreg_crt(cirrus, 0x1b, tmp);
+
+	/* Enable high-colour modes */
+	wreg_gfx(cirrus, VGA_GFX_MODE, 0x40);
+
+	/* And set graphics mode */
+	wreg_gfx(cirrus, VGA_GFX_MISC, 0x01);
+
+	wreg_hdr(cirrus, hdr);
+
+	cirrus_set_start_address(cirrus, 0);
+
+	/* Unblank (needed on S3 resume, vgabios doesn't do it then) */
+	outb(0x20, 0x3c0);
+	return 0;
+}
+
+static int cirrus_fb_blit_rect(struct drm_framebuffer *fb,
+			       struct drm_rect *rect)
+{
+	struct cirrus_device *cirrus = fb->dev->dev_private;
+	void *vmap;
+
+	vmap = drm_gem_shmem_vmap(fb->obj[0]);
+	if (!vmap)
+		return -ENOMEM;
+
+	if (cirrus->cpp == fb->format->cpp[0])
+		drm_fb_memcpy_dstclip(cirrus->vram,
+				      vmap, fb, rect);
+
+	else if (fb->format->cpp[0] == 4 && cirrus->cpp == 2)
+		drm_fb_xrgb8888_to_rgb565_dstclip(cirrus->vram,
+						  cirrus->pitch,
+						  vmap, fb, rect, false);
+
+	else if (fb->format->cpp[0] == 4 && cirrus->cpp == 3)
+		drm_fb_xrgb8888_to_rgb888_dstclip(cirrus->vram,
+						  cirrus->pitch,
+						  vmap, fb, rect);
+
+	else
+		WARN_ON_ONCE("cpp mismatch");
+
+	drm_gem_shmem_vunmap(fb->obj[0], vmap);
+	return 0;
+}
+
+static int cirrus_fb_blit_fullscreen(struct drm_framebuffer *fb)
+{
+	struct drm_rect fullscreen = {
+		.x1 = 0,
+		.x2 = fb->width,
+		.y1 = 0,
+		.y2 = fb->height,
+	};
+	return cirrus_fb_blit_rect(fb, &fullscreen);
+}
+
+static int cirrus_check_size(int width, int height,
+			     struct drm_framebuffer *fb)
+{
+	int pitch = width * 2;
+
+	if (fb)
+		pitch = cirrus_pitch(fb);
+
+	if (pitch > CIRRUS_MAX_PITCH)
+		return -EINVAL;
+	if (pitch * height > CIRRUS_VRAM_SIZE)
+		return -EINVAL;
+	return 0;
+}
+
+/* ------------------------------------------------------------------ */
+/* cirrus connector						      */
+
+static int cirrus_conn_get_modes(struct drm_connector *conn)
+{
+	int count;
+
+	count = drm_add_modes_noedid(conn,
+				     conn->dev->mode_config.max_width,
+				     conn->dev->mode_config.max_height);
+	drm_set_preferred_mode(conn, 1024, 768);
+	return count;
+}
+
+static const struct drm_connector_helper_funcs cirrus_conn_helper_funcs = {
+	.get_modes = cirrus_conn_get_modes,
+};
+
+static const struct drm_connector_funcs cirrus_conn_funcs = {
+	.fill_modes = drm_helper_probe_single_connector_modes,
+	.destroy = drm_connector_cleanup,
+	.reset = drm_atomic_helper_connector_reset,
+	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+};
+
+static int cirrus_conn_init(struct cirrus_device *cirrus)
+{
+	drm_connector_helper_add(&cirrus->conn, &cirrus_conn_helper_funcs);
+	return drm_connector_init(&cirrus->dev, &cirrus->conn,
+				  &cirrus_conn_funcs, DRM_MODE_CONNECTOR_VGA);
+
+}
+
+/* ------------------------------------------------------------------ */
+/* cirrus (simple) display pipe					      */
+
+static enum drm_mode_status cirrus_pipe_mode_valid(struct drm_crtc *crtc,
+						   const struct drm_display_mode *mode)
+{
+	if (cirrus_check_size(mode->hdisplay, mode->vdisplay, NULL) < 0)
+		return MODE_BAD;
+	return MODE_OK;
+}
+
+static int cirrus_pipe_check(struct drm_simple_display_pipe *pipe,
+			     struct drm_plane_state *plane_state,
+			     struct drm_crtc_state *crtc_state)
+{
+	struct drm_framebuffer *fb = plane_state->fb;
+
+	if (!fb)
+		return 0;
+	return cirrus_check_size(fb->width, fb->height, fb);
+}
+
+static void cirrus_pipe_enable(struct drm_simple_display_pipe *pipe,
+			       struct drm_crtc_state *crtc_state,
+			       struct drm_plane_state *plane_state)
+{
+	struct cirrus_device *cirrus = pipe->crtc.dev->dev_private;
+
+	cirrus_mode_set(cirrus, &crtc_state->mode, plane_state->fb);
+	cirrus_fb_blit_fullscreen(plane_state->fb);
+}
+
+static void cirrus_pipe_update(struct drm_simple_display_pipe *pipe,
+			       struct drm_plane_state *old_state)
+{
+	struct cirrus_device *cirrus = pipe->crtc.dev->dev_private;
+	struct drm_plane_state *state = pipe->plane.state;
+	struct drm_crtc *crtc = &pipe->crtc;
+	struct drm_rect rect;
+
+	if (pipe->plane.state->fb &&
+	    cirrus->cpp != cirrus_cpp(pipe->plane.state->fb))
+		cirrus_mode_set(cirrus, &crtc->mode,
+				pipe->plane.state->fb);
+
+	if (drm_atomic_helper_damage_merged(old_state, state, &rect))
+		cirrus_fb_blit_rect(pipe->plane.state->fb, &rect);
+
+	if (crtc->state->event) {
+		spin_lock_irq(&crtc->dev->event_lock);
+		drm_crtc_send_vblank_event(crtc, crtc->state->event);
+		crtc->state->event = NULL;
+		spin_unlock_irq(&crtc->dev->event_lock);
+	}
+}
+
+static const struct drm_simple_display_pipe_funcs cirrus_pipe_funcs = {
+	.mode_valid = cirrus_pipe_mode_valid,
+	.check	    = cirrus_pipe_check,
+	.enable	    = cirrus_pipe_enable,
+	.update	    = cirrus_pipe_update,
+};
+
+static const uint32_t cirrus_formats[] = {
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_RGB888,
+	DRM_FORMAT_XRGB8888,
+};
+
+static const uint64_t cirrus_modifiers[] = {
+	DRM_FORMAT_MOD_LINEAR,
+	DRM_FORMAT_MOD_INVALID
+};
+
+static int cirrus_pipe_init(struct cirrus_device *cirrus)
+{
+	return drm_simple_display_pipe_init(&cirrus->dev,
+					    &cirrus->pipe,
+					    &cirrus_pipe_funcs,
+					    cirrus_formats,
+					    ARRAY_SIZE(cirrus_formats),
+					    cirrus_modifiers,
+					    &cirrus->conn);
+}
+
+/* ------------------------------------------------------------------ */
+/* cirrus framebuffers & mode config				      */
+
+static struct drm_framebuffer*
+cirrus_fb_create(struct drm_device *dev, struct drm_file *file_priv,
+		 const struct drm_mode_fb_cmd2 *mode_cmd)
+{
+	if (mode_cmd->pixel_format != DRM_FORMAT_RGB565 &&
+	    mode_cmd->pixel_format != DRM_FORMAT_RGB888 &&
+	    mode_cmd->pixel_format != DRM_FORMAT_XRGB8888)
+		return ERR_PTR(-EINVAL);
+	if (cirrus_check_size(mode_cmd->width, mode_cmd->height, NULL) < 0)
+		return ERR_PTR(-EINVAL);
+	return drm_gem_fb_create_with_dirty(dev, file_priv, mode_cmd);
+}
+
+static const struct drm_mode_config_funcs cirrus_mode_config_funcs = {
+	.fb_create = cirrus_fb_create,
+	.atomic_check = drm_atomic_helper_check,
+	.atomic_commit = drm_atomic_helper_commit,
+};
+
+static void cirrus_mode_config_init(struct cirrus_device *cirrus)
+{
+	struct drm_device *dev = &cirrus->dev;
+
+	drm_mode_config_init(dev);
+	dev->mode_config.min_width = 0;
+	dev->mode_config.min_height = 0;
+	dev->mode_config.max_width = CIRRUS_MAX_PITCH / 2;
+	dev->mode_config.max_height = 1024;
+	dev->mode_config.preferred_depth = 16;
+	dev->mode_config.prefer_shadow = 0;
+	dev->mode_config.funcs = &cirrus_mode_config_funcs;
+}
+
+/* ------------------------------------------------------------------ */
+
+DEFINE_DRM_GEM_SHMEM_FOPS(cirrus_fops);
+
+static struct drm_driver cirrus_driver = {
+	.driver_features = DRIVER_MODESET | DRIVER_GEM | DRIVER_ATOMIC | DRIVER_PRIME,
+
+	.name		 = DRIVER_NAME,
+	.desc		 = DRIVER_DESC,
+	.date		 = DRIVER_DATE,
+	.major		 = DRIVER_MAJOR,
+	.minor		 = DRIVER_MINOR,
+
+	.fops		 = &cirrus_fops,
+	DRM_GEM_SHMEM_DRIVER_OPS,
+};
+
+static int cirrus_pci_probe(struct pci_dev *pdev,
+			    const struct pci_device_id *ent)
+{
+	struct drm_device *dev;
+	struct cirrus_device *cirrus;
+	int ret;
+
+	ret = drm_fb_helper_remove_conflicting_pci_framebuffers(pdev, 0, "cirrusdrmfb");
+	if (ret)
+		return ret;
+
+	ret = pci_enable_device(pdev);
+	if (ret)
+		return ret;
+
+	ret = pci_request_regions(pdev, DRIVER_NAME);
+	if (ret)
+		return ret;
+
+	ret = -ENOMEM;
+	cirrus = kzalloc(sizeof(*cirrus), GFP_KERNEL);
+	if (cirrus == NULL)
+		goto err_pci_release;
+
+	dev = &cirrus->dev;
+	ret = drm_dev_init(dev, &cirrus_driver, &pdev->dev);
+	if (ret)
+		goto err_free_cirrus;
+	dev->dev_private = cirrus;
+
+	ret = -ENOMEM;
+	cirrus->vram = ioremap(pci_resource_start(pdev, 0),
+			       pci_resource_len(pdev, 0));
+	if (cirrus->vram == NULL)
+		goto err_dev_put;
+
+	cirrus->mmio = ioremap(pci_resource_start(pdev, 1),
+			       pci_resource_len(pdev, 1));
+	if (cirrus->mmio == NULL)
+		goto err_unmap_vram;
+
+	cirrus_mode_config_init(cirrus);
+
+	ret = cirrus_conn_init(cirrus);
+	if (ret < 0)
+		goto err_cleanup;
+
+	ret = cirrus_pipe_init(cirrus);
+	if (ret < 0)
+		goto err_cleanup;
+
+	drm_mode_config_reset(dev);
+
+	dev->pdev = pdev;
+	pci_set_drvdata(pdev, dev);
+	ret = drm_dev_register(dev, 0);
+	if (ret)
+		goto err_cleanup;
+
+	drm_fbdev_generic_setup(dev, dev->mode_config.preferred_depth);
+	return 0;
+
+err_cleanup:
+	drm_mode_config_cleanup(dev);
+	iounmap(cirrus->mmio);
+err_unmap_vram:
+	iounmap(cirrus->vram);
+err_dev_put:
+	drm_dev_put(dev);
+err_free_cirrus:
+	kfree(cirrus);
+err_pci_release:
+	pci_release_regions(pdev);
+	return ret;
+}
+
+static void cirrus_pci_remove(struct pci_dev *pdev)
+{
+	struct drm_device *dev = pci_get_drvdata(pdev);
+	struct cirrus_device *cirrus = dev->dev_private;
+
+	drm_dev_unregister(dev);
+	drm_mode_config_cleanup(dev);
+	iounmap(cirrus->mmio);
+	iounmap(cirrus->vram);
+	drm_dev_put(dev);
+	kfree(cirrus);
+	pci_release_regions(pdev);
+}
+
+static const struct pci_device_id pciidlist[] = {
+	{
+		.vendor    = PCI_VENDOR_ID_CIRRUS,
+		.device    = PCI_DEVICE_ID_CIRRUS_5446,
+		/* only bind to the cirrus chip in qemu */
+		.subvendor = PCI_SUBVENDOR_ID_REDHAT_QUMRANET,
+		.subdevice = PCI_SUBDEVICE_ID_QEMU,
+	}, {
+		.vendor    = PCI_VENDOR_ID_CIRRUS,
+		.device    = PCI_DEVICE_ID_CIRRUS_5446,
+		.subvendor = PCI_VENDOR_ID_XEN,
+		.subdevice = 0x0001,
+	},
+	{ /* end if list */ }
+};
+
+static struct pci_driver cirrus_pci_driver = {
+	.name = DRIVER_NAME,
+	.id_table = pciidlist,
+	.probe = cirrus_pci_probe,
+	.remove = cirrus_pci_remove,
+};
+
+static int __init cirrus_init(void)
+{
+	if (vgacon_text_force())
+		return -EINVAL;
+	return pci_register_driver(&cirrus_pci_driver);
+}
+
+static void __exit cirrus_exit(void)
+{
+	pci_unregister_driver(&cirrus_pci_driver);
+}
+
+module_init(cirrus_init);
+module_exit(cirrus_exit);
+
+MODULE_DEVICE_TABLE(pci, pciidlist);
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/cirrus/cirrus_drv.c b/drivers/gpu/drm/cirrus/cirrus_drv.c
deleted file mode 100644
index 8ec880f3a322..000000000000
--- a/drivers/gpu/drm/cirrus/cirrus_drv.c
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- * Copyright 2012 Red Hat <mjg@redhat.com>
- *
- * This file is subject to the terms and conditions of the GNU General
- * Public License version 2. See the file COPYING in the main
- * directory of this archive for more details.
- *
- * Authors: Matthew Garrett
- *          Dave Airlie
- */
-#include <linux/module.h>
-#include <linux/console.h>
-#include <drm/drmP.h>
-#include <drm/drm_crtc_helper.h>
-#include <drm/drm_probe_helper.h>
-
-#include "cirrus_drv.h"
-
-int cirrus_modeset = -1;
-int cirrus_bpp = 16;
-
-MODULE_PARM_DESC(modeset, "Disable/Enable modesetting");
-module_param_named(modeset, cirrus_modeset, int, 0400);
-MODULE_PARM_DESC(bpp, "Max bits-per-pixel (default:16)");
-module_param_named(bpp, cirrus_bpp, int, 0400);
-
-/*
- * This is the generic driver code. This binds the driver to the drm core,
- * which then performs further device association and calls our graphics init
- * functions
- */
-
-static struct drm_driver driver;
-
-/* only bind to the cirrus chip in qemu */
-static const struct pci_device_id pciidlist[] = {
-	{ PCI_VENDOR_ID_CIRRUS, PCI_DEVICE_ID_CIRRUS_5446,
-	  PCI_SUBVENDOR_ID_REDHAT_QUMRANET, PCI_SUBDEVICE_ID_QEMU,
-	  0, 0, 0 },
-	{ PCI_VENDOR_ID_CIRRUS, PCI_DEVICE_ID_CIRRUS_5446, PCI_VENDOR_ID_XEN,
-	  0x0001, 0, 0, 0 },
-	{0,}
-};
-
-
-static int cirrus_pci_probe(struct pci_dev *pdev,
-			    const struct pci_device_id *ent)
-{
-	int ret;
-
-	ret = drm_fb_helper_remove_conflicting_pci_framebuffers(pdev, 0, "cirrusdrmfb");
-	if (ret)
-		return ret;
-
-	return drm_get_pci_dev(pdev, ent, &driver);
-}
-
-static void cirrus_pci_remove(struct pci_dev *pdev)
-{
-	struct drm_device *dev = pci_get_drvdata(pdev);
-
-	drm_put_dev(dev);
-}
-
-#ifdef CONFIG_PM_SLEEP
-static int cirrus_pm_suspend(struct device *dev)
-{
-	struct pci_dev *pdev = to_pci_dev(dev);
-	struct drm_device *drm_dev = pci_get_drvdata(pdev);
-	struct cirrus_device *cdev = drm_dev->dev_private;
-
-	drm_kms_helper_poll_disable(drm_dev);
-
-	if (cdev->mode_info.gfbdev) {
-		console_lock();
-		drm_fb_helper_set_suspend(&cdev->mode_info.gfbdev->helper, 1);
-		console_unlock();
-	}
-
-	return 0;
-}
-
-static int cirrus_pm_resume(struct device *dev)
-{
-	struct pci_dev *pdev = to_pci_dev(dev);
-	struct drm_device *drm_dev = pci_get_drvdata(pdev);
-	struct cirrus_device *cdev = drm_dev->dev_private;
-
-	drm_helper_resume_force_mode(drm_dev);
-
-	if (cdev->mode_info.gfbdev) {
-		console_lock();
-		drm_fb_helper_set_suspend(&cdev->mode_info.gfbdev->helper, 0);
-		console_unlock();
-	}
-
-	drm_kms_helper_poll_enable(drm_dev);
-	return 0;
-}
-#endif
-
-static const struct file_operations cirrus_driver_fops = {
-	.owner = THIS_MODULE,
-	.open = drm_open,
-	.release = drm_release,
-	.unlocked_ioctl = drm_ioctl,
-	.mmap = cirrus_mmap,
-	.poll = drm_poll,
-	.compat_ioctl = drm_compat_ioctl,
-};
-static struct drm_driver driver = {
-	.driver_features = DRIVER_MODESET | DRIVER_GEM,
-	.load = cirrus_driver_load,
-	.unload = cirrus_driver_unload,
-	.fops = &cirrus_driver_fops,
-	.name = DRIVER_NAME,
-	.desc = DRIVER_DESC,
-	.date = DRIVER_DATE,
-	.major = DRIVER_MAJOR,
-	.minor = DRIVER_MINOR,
-	.patchlevel = DRIVER_PATCHLEVEL,
-	.gem_free_object_unlocked = cirrus_gem_free_object,
-	.dumb_create = cirrus_dumb_create,
-	.dumb_map_offset = cirrus_dumb_mmap_offset,
-};
-
-static const struct dev_pm_ops cirrus_pm_ops = {
-	SET_SYSTEM_SLEEP_PM_OPS(cirrus_pm_suspend,
-				cirrus_pm_resume)
-};
-
-static struct pci_driver cirrus_pci_driver = {
-	.name = DRIVER_NAME,
-	.id_table = pciidlist,
-	.probe = cirrus_pci_probe,
-	.remove = cirrus_pci_remove,
-	.driver.pm = &cirrus_pm_ops,
-};
-
-static int __init cirrus_init(void)
-{
-	if (vgacon_text_force() && cirrus_modeset == -1)
-		return -EINVAL;
-
-	if (cirrus_modeset == 0)
-		return -EINVAL;
-	return pci_register_driver(&cirrus_pci_driver);
-}
-
-static void __exit cirrus_exit(void)
-{
-	pci_unregister_driver(&cirrus_pci_driver);
-}
-
-module_init(cirrus_init);
-module_exit(cirrus_exit);
-
-MODULE_DEVICE_TABLE(pci, pciidlist);
-MODULE_AUTHOR(DRIVER_AUTHOR);
-MODULE_DESCRIPTION(DRIVER_DESC);
-MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/cirrus/cirrus_drv.h b/drivers/gpu/drm/cirrus/cirrus_drv.h
index f2b2e0d169fa..1bd816be3aae 100644
--- a/drivers/gpu/drm/cirrus/cirrus_drv.h
+++ b/drivers/gpu/drm/cirrus/cirrus_drv.h
@@ -101,7 +101,6 @@ struct cirrus_crtc {
 
 struct cirrus_fbdev;
 struct cirrus_mode_info {
-	bool				mode_config_initialized;
 	struct cirrus_crtc		*crtc;
 	/* pointer to fbdev info structure */
 	struct cirrus_fbdev		*gfbdev;
@@ -143,7 +142,7 @@ struct cirrus_device {
 
 
 struct cirrus_fbdev {
-	struct drm_fb_helper helper;
+	struct drm_fb_helper helper; /* must be first */
 	struct drm_framebuffer *gfb;
 	void *sysram;
 	int size;
@@ -169,7 +168,6 @@ cirrus_bo(struct ttm_buffer_object *bo)
 
 
 #define to_cirrus_obj(x) container_of(x, struct cirrus_gem_object, base)
-#define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT)
 
 				/* cirrus_main.c */
 int cirrus_device_init(struct cirrus_device *cdev,
diff --git a/drivers/gpu/drm/cirrus/cirrus_fbdev.c b/drivers/gpu/drm/cirrus/cirrus_fbdev.c
deleted file mode 100644
index 39df62acac69..000000000000
--- a/drivers/gpu/drm/cirrus/cirrus_fbdev.c
+++ /dev/null
@@ -1,315 +0,0 @@
-/*
- * Copyright 2012 Red Hat
- *
- * This file is subject to the terms and conditions of the GNU General
- * Public License version 2. See the file COPYING in the main
- * directory of this archive for more details.
- *
- * Authors: Matthew Garrett
- *          Dave Airlie
- */
-#include <linux/module.h>
-#include <drm/drmP.h>
-#include <drm/drm_util.h>
-#include <drm/drm_fb_helper.h>
-#include <drm/drm_crtc_helper.h>
-
-#include "cirrus_drv.h"
-
-static void cirrus_dirty_update(struct cirrus_fbdev *afbdev,
-			     int x, int y, int width, int height)
-{
-	int i;
-	struct drm_gem_object *obj;
-	struct cirrus_bo *bo;
-	int src_offset, dst_offset;
-	int bpp = afbdev->gfb->format->cpp[0];
-	int ret = -EBUSY;
-	bool unmap = false;
-	bool store_for_later = false;
-	int x2, y2;
-	unsigned long flags;
-
-	obj = afbdev->gfb->obj[0];
-	bo = gem_to_cirrus_bo(obj);
-
-	/*
-	 * try and reserve the BO, if we fail with busy
-	 * then the BO is being moved and we should
-	 * store up the damage until later.
-	 */
-	if (drm_can_sleep())
-		ret = cirrus_bo_reserve(bo, true);
-	if (ret) {
-		if (ret != -EBUSY)
-			return;
-		store_for_later = true;
-	}
-
-	x2 = x + width - 1;
-	y2 = y + height - 1;
-	spin_lock_irqsave(&afbdev->dirty_lock, flags);
-
-	if (afbdev->y1 < y)
-		y = afbdev->y1;
-	if (afbdev->y2 > y2)
-		y2 = afbdev->y2;
-	if (afbdev->x1 < x)
-		x = afbdev->x1;
-	if (afbdev->x2 > x2)
-		x2 = afbdev->x2;
-
-	if (store_for_later) {
-		afbdev->x1 = x;
-		afbdev->x2 = x2;
-		afbdev->y1 = y;
-		afbdev->y2 = y2;
-		spin_unlock_irqrestore(&afbdev->dirty_lock, flags);
-		return;
-	}
-
-	afbdev->x1 = afbdev->y1 = INT_MAX;
-	afbdev->x2 = afbdev->y2 = 0;
-	spin_unlock_irqrestore(&afbdev->dirty_lock, flags);
-
-	if (!bo->kmap.virtual) {
-		ret = ttm_bo_kmap(&bo->bo, 0, bo->bo.num_pages, &bo->kmap);
-		if (ret) {
-			DRM_ERROR("failed to kmap fb updates\n");
-			cirrus_bo_unreserve(bo);
-			return;
-		}
-		unmap = true;
-	}
-	for (i = y; i < y + height; i++) {
-		/* assume equal stride for now */
-		src_offset = dst_offset = i * afbdev->gfb->pitches[0] + (x * bpp);
-		memcpy_toio(bo->kmap.virtual + src_offset, afbdev->sysram + src_offset, width * bpp);
-
-	}
-	if (unmap)
-		ttm_bo_kunmap(&bo->kmap);
-
-	cirrus_bo_unreserve(bo);
-}
-
-static void cirrus_fillrect(struct fb_info *info,
-			 const struct fb_fillrect *rect)
-{
-	struct cirrus_fbdev *afbdev = info->par;
-	drm_fb_helper_sys_fillrect(info, rect);
-	cirrus_dirty_update(afbdev, rect->dx, rect->dy, rect->width,
-			 rect->height);
-}
-
-static void cirrus_copyarea(struct fb_info *info,
-			 const struct fb_copyarea *area)
-{
-	struct cirrus_fbdev *afbdev = info->par;
-	drm_fb_helper_sys_copyarea(info, area);
-	cirrus_dirty_update(afbdev, area->dx, area->dy, area->width,
-			 area->height);
-}
-
-static void cirrus_imageblit(struct fb_info *info,
-			  const struct fb_image *image)
-{
-	struct cirrus_fbdev *afbdev = info->par;
-	drm_fb_helper_sys_imageblit(info, image);
-	cirrus_dirty_update(afbdev, image->dx, image->dy, image->width,
-			 image->height);
-}
-
-
-static struct fb_ops cirrusfb_ops = {
-	.owner = THIS_MODULE,
-	.fb_check_var = drm_fb_helper_check_var,
-	.fb_set_par = drm_fb_helper_set_par,
-	.fb_fillrect = cirrus_fillrect,
-	.fb_copyarea = cirrus_copyarea,
-	.fb_imageblit = cirrus_imageblit,
-	.fb_pan_display = drm_fb_helper_pan_display,
-	.fb_blank = drm_fb_helper_blank,
-	.fb_setcmap = drm_fb_helper_setcmap,
-};
-
-static int cirrusfb_create_object(struct cirrus_fbdev *afbdev,
-			       const struct drm_mode_fb_cmd2 *mode_cmd,
-			       struct drm_gem_object **gobj_p)
-{
-	struct drm_device *dev = afbdev->helper.dev;
-	struct cirrus_device *cdev = dev->dev_private;
-	u32 bpp;
-	u32 size;
-	struct drm_gem_object *gobj;
-	int ret = 0;
-
-	bpp = drm_format_plane_cpp(mode_cmd->pixel_format, 0) * 8;
-
-	if (!cirrus_check_framebuffer(cdev, mode_cmd->width, mode_cmd->height,
-				      bpp, mode_cmd->pitches[0]))
-		return -EINVAL;
-
-	size = mode_cmd->pitches[0] * mode_cmd->height;
-	ret = cirrus_gem_create(dev, size, true, &gobj);
-	if (ret)
-		return ret;
-
-	*gobj_p = gobj;
-	return ret;
-}
-
-static int cirrusfb_create(struct drm_fb_helper *helper,
-			   struct drm_fb_helper_surface_size *sizes)
-{
-	struct cirrus_fbdev *gfbdev =
-		container_of(helper, struct cirrus_fbdev, helper);
-	struct cirrus_device *cdev = gfbdev->helper.dev->dev_private;
-	struct fb_info *info;
-	struct drm_framebuffer *fb;
-	struct drm_mode_fb_cmd2 mode_cmd;
-	void *sysram;
-	struct drm_gem_object *gobj = NULL;
-	int size, ret;
-
-	mode_cmd.width = sizes->surface_width;
-	mode_cmd.height = sizes->surface_height;
-	mode_cmd.pitches[0] = mode_cmd.width * ((sizes->surface_bpp + 7) / 8);
-	mode_cmd.pixel_format = drm_mode_legacy_fb_format(sizes->surface_bpp,
-							  sizes->surface_depth);
-	size = mode_cmd.pitches[0] * mode_cmd.height;
-
-	ret = cirrusfb_create_object(gfbdev, &mode_cmd, &gobj);
-	if (ret) {
-		DRM_ERROR("failed to create fbcon backing object %d\n", ret);
-		return ret;
-	}
-
-	sysram = vmalloc(size);
-	if (!sysram)
-		return -ENOMEM;
-
-	info = drm_fb_helper_alloc_fbi(helper);
-	if (IS_ERR(info)) {
-		ret = PTR_ERR(info);
-		goto err_vfree;
-	}
-
-	info->par = gfbdev;
-
-	fb = kzalloc(sizeof(*fb), GFP_KERNEL);
-	if (!fb) {
-		ret = -ENOMEM;
-		goto err_drm_gem_object_put_unlocked;
-	}
-
-	ret = cirrus_framebuffer_init(cdev->dev, fb, &mode_cmd, gobj);
-	if (ret)
-		goto err_kfree;
-
-	gfbdev->sysram = sysram;
-	gfbdev->size = size;
-	gfbdev->gfb = fb;
-
-	/* setup helper */
-	gfbdev->helper.fb = fb;
-
-	strcpy(info->fix.id, "cirrusdrmfb");
-
-	info->fbops = &cirrusfb_ops;
-
-	drm_fb_helper_fill_fix(info, fb->pitches[0], fb->format->depth);
-	drm_fb_helper_fill_var(info, &gfbdev->helper, sizes->fb_width,
-			       sizes->fb_height);
-
-	/* setup aperture base/size for vesafb takeover */
-	info->apertures->ranges[0].base = cdev->dev->mode_config.fb_base;
-	info->apertures->ranges[0].size = cdev->mc.vram_size;
-
-	info->fix.smem_start = cdev->dev->mode_config.fb_base;
-	info->fix.smem_len = cdev->mc.vram_size;
-
-	info->screen_base = sysram;
-	info->screen_size = size;
-
-	info->fix.mmio_start = 0;
-	info->fix.mmio_len = 0;
-
-	DRM_INFO("fb mappable at 0x%lX\n", info->fix.smem_start);
-	DRM_INFO("vram aper at 0x%lX\n", (unsigned long)info->fix.smem_start);
-	DRM_INFO("size %lu\n", (unsigned long)info->fix.smem_len);
-	DRM_INFO("fb depth is %d\n", fb->format->depth);
-	DRM_INFO("   pitch is %d\n", fb->pitches[0]);
-
-	return 0;
-
-err_kfree:
-	kfree(fb);
-err_drm_gem_object_put_unlocked:
-	drm_gem_object_put_unlocked(gobj);
-err_vfree:
-	vfree(sysram);
-	return ret;
-}
-
-static int cirrus_fbdev_destroy(struct drm_device *dev,
-				struct cirrus_fbdev *gfbdev)
-{
-	struct drm_framebuffer *gfb = gfbdev->gfb;
-
-	drm_helper_force_disable_all(dev);
-
-	drm_fb_helper_unregister_fbi(&gfbdev->helper);
-
-	vfree(gfbdev->sysram);
-	drm_fb_helper_fini(&gfbdev->helper);
-	if (gfb)
-		drm_framebuffer_put(gfb);
-
-	return 0;
-}
-
-static const struct drm_fb_helper_funcs cirrus_fb_helper_funcs = {
-	.fb_probe = cirrusfb_create,
-};
-
-int cirrus_fbdev_init(struct cirrus_device *cdev)
-{
-	struct cirrus_fbdev *gfbdev;
-	int ret;
-
-	/*bpp_sel = 8;*/
-	gfbdev = kzalloc(sizeof(struct cirrus_fbdev), GFP_KERNEL);
-	if (!gfbdev)
-		return -ENOMEM;
-
-	cdev->mode_info.gfbdev = gfbdev;
-	spin_lock_init(&gfbdev->dirty_lock);
-
-	drm_fb_helper_prepare(cdev->dev, &gfbdev->helper,
-			      &cirrus_fb_helper_funcs);
-
-	ret = drm_fb_helper_init(cdev->dev, &gfbdev->helper,
-				 CIRRUSFB_CONN_LIMIT);
-	if (ret)
-		return ret;
-
-	ret = drm_fb_helper_single_add_all_connectors(&gfbdev->helper);
-	if (ret)
-		return ret;
-
-	/* disable all the possible outputs/crtcs before entering KMS mode */
-	drm_helper_disable_unused_functions(cdev->dev);
-
-	return drm_fb_helper_initial_config(&gfbdev->helper, cirrus_bpp);
-}
-
-void cirrus_fbdev_fini(struct cirrus_device *cdev)
-{
-	if (!cdev->mode_info.gfbdev)
-		return;
-
-	cirrus_fbdev_destroy(cdev->dev, cdev->mode_info.gfbdev);
-	kfree(cdev->mode_info.gfbdev);
-	cdev->mode_info.gfbdev = NULL;
-}
diff --git a/drivers/gpu/drm/cirrus/cirrus_main.c b/drivers/gpu/drm/cirrus/cirrus_main.c
deleted file mode 100644
index 57f8fe6d020b..000000000000
--- a/drivers/gpu/drm/cirrus/cirrus_main.c
+++ /dev/null
@@ -1,328 +0,0 @@
-/*
- * Copyright 2012 Red Hat
- *
- * This file is subject to the terms and conditions of the GNU General
- * Public License version 2. See the file COPYING in the main
- * directory of this archive for more details.
- *
- * Authors: Matthew Garrett
- *          Dave Airlie
- */
-#include <drm/drmP.h>
-#include <drm/drm_crtc_helper.h>
-#include <drm/drm_gem_framebuffer_helper.h>
-
-#include "cirrus_drv.h"
-
-static const struct drm_framebuffer_funcs cirrus_fb_funcs = {
-	.create_handle = drm_gem_fb_create_handle,
-	.destroy = drm_gem_fb_destroy,
-};
-
-int cirrus_framebuffer_init(struct drm_device *dev,
-			    struct drm_framebuffer *gfb,
-			    const struct drm_mode_fb_cmd2 *mode_cmd,
-			    struct drm_gem_object *obj)
-{
-	int ret;
-
-	drm_helper_mode_fill_fb_struct(dev, gfb, mode_cmd);
-	gfb->obj[0] = obj;
-	ret = drm_framebuffer_init(dev, gfb, &cirrus_fb_funcs);
-	if (ret) {
-		DRM_ERROR("drm_framebuffer_init failed: %d\n", ret);
-		return ret;
-	}
-	return 0;
-}
-
-static struct drm_framebuffer *
-cirrus_user_framebuffer_create(struct drm_device *dev,
-			       struct drm_file *filp,
-			       const struct drm_mode_fb_cmd2 *mode_cmd)
-{
-	struct cirrus_device *cdev = dev->dev_private;
-	struct drm_gem_object *obj;
-	struct drm_framebuffer *fb;
-	u32 bpp;
-	int ret;
-
-	bpp = drm_format_plane_cpp(mode_cmd->pixel_format, 0) * 8;
-
-	if (!cirrus_check_framebuffer(cdev, mode_cmd->width, mode_cmd->height,
-				      bpp, mode_cmd->pitches[0]))
-		return ERR_PTR(-EINVAL);
-
-	obj = drm_gem_object_lookup(filp, mode_cmd->handles[0]);
-	if (obj == NULL)
-		return ERR_PTR(-ENOENT);
-
-	fb = kzalloc(sizeof(*fb), GFP_KERNEL);
-	if (!fb) {
-		drm_gem_object_put_unlocked(obj);
-		return ERR_PTR(-ENOMEM);
-	}
-
-	ret = cirrus_framebuffer_init(dev, fb, mode_cmd, obj);
-	if (ret) {
-		drm_gem_object_put_unlocked(obj);
-		kfree(fb);
-		return ERR_PTR(ret);
-	}
-	return fb;
-}
-
-static const struct drm_mode_config_funcs cirrus_mode_funcs = {
-	.fb_create = cirrus_user_framebuffer_create,
-};
-
-/* Unmap the framebuffer from the core and release the memory */
-static void cirrus_vram_fini(struct cirrus_device *cdev)
-{
-	iounmap(cdev->rmmio);
-	cdev->rmmio = NULL;
-	if (cdev->mc.vram_base)
-		release_mem_region(cdev->mc.vram_base, cdev->mc.vram_size);
-}
-
-/* Map the framebuffer from the card and configure the core */
-static int cirrus_vram_init(struct cirrus_device *cdev)
-{
-	/* BAR 0 is VRAM */
-	cdev->mc.vram_base = pci_resource_start(cdev->dev->pdev, 0);
-	cdev->mc.vram_size = pci_resource_len(cdev->dev->pdev, 0);
-
-	if (!request_mem_region(cdev->mc.vram_base, cdev->mc.vram_size,
-				"cirrusdrmfb_vram")) {
-		DRM_ERROR("can't reserve VRAM\n");
-		return -ENXIO;
-	}
-
-	return 0;
-}
-
-/*
- * Our emulated hardware has two sets of memory. One is video RAM and can
- * simply be used as a linear framebuffer - the other provides mmio access
- * to the display registers. The latter can also be accessed via IO port
- * access, but we map the range and use mmio to program them instead
- */
-
-int cirrus_device_init(struct cirrus_device *cdev,
-		       struct drm_device *ddev,
-		       struct pci_dev *pdev, uint32_t flags)
-{
-	int ret;
-
-	cdev->dev = ddev;
-	cdev->flags = flags;
-
-	/* Hardcode the number of CRTCs to 1 */
-	cdev->num_crtc = 1;
-
-	/* BAR 0 is the framebuffer, BAR 1 contains registers */
-	cdev->rmmio_base = pci_resource_start(cdev->dev->pdev, 1);
-	cdev->rmmio_size = pci_resource_len(cdev->dev->pdev, 1);
-
-	if (!request_mem_region(cdev->rmmio_base, cdev->rmmio_size,
-				"cirrusdrmfb_mmio")) {
-		DRM_ERROR("can't reserve mmio registers\n");
-		return -ENOMEM;
-	}
-
-	cdev->rmmio = ioremap(cdev->rmmio_base, cdev->rmmio_size);
-
-	if (cdev->rmmio == NULL)
-		return -ENOMEM;
-
-	ret = cirrus_vram_init(cdev);
-	if (ret) {
-		release_mem_region(cdev->rmmio_base, cdev->rmmio_size);
-		return ret;
-	}
-
-	return 0;
-}
-
-void cirrus_device_fini(struct cirrus_device *cdev)
-{
-	release_mem_region(cdev->rmmio_base, cdev->rmmio_size);
-	cirrus_vram_fini(cdev);
-}
-
-/*
- * Functions here will be called by the core once it's bound the driver to
- * a PCI device
- */
-
-int cirrus_driver_load(struct drm_device *dev, unsigned long flags)
-{
-	struct cirrus_device *cdev;
-	int r;
-
-	cdev = kzalloc(sizeof(struct cirrus_device), GFP_KERNEL);
-	if (cdev == NULL)
-		return -ENOMEM;
-	dev->dev_private = (void *)cdev;
-
-	r = cirrus_device_init(cdev, dev, dev->pdev, flags);
-	if (r) {
-		dev_err(&dev->pdev->dev, "Fatal error during GPU init: %d\n", r);
-		goto out;
-	}
-
-	r = cirrus_mm_init(cdev);
-	if (r) {
-		dev_err(&dev->pdev->dev, "fatal err on mm init\n");
-		goto out;
-	}
-
-	/*
-	 * cirrus_modeset_init() is initializing/registering the emulated fbdev
-	 * and DRM internals can access/test some of the fields in
-	 * mode_config->funcs as part of the fbdev registration process.
-	 * Make sure dev->mode_config.funcs is properly set to avoid
-	 * dereferencing a NULL pointer.
-	 * FIXME: mode_config.funcs assignment should probably be done in
-	 * cirrus_modeset_init() (that's a common pattern seen in other DRM
-	 * drivers).
-	 */
-	dev->mode_config.funcs = &cirrus_mode_funcs;
-	r = cirrus_modeset_init(cdev);
-	if (r) {
-		dev_err(&dev->pdev->dev, "Fatal error during modeset init: %d\n", r);
-		goto out;
-	}
-
-	return 0;
-out:
-	cirrus_driver_unload(dev);
-	return r;
-}
-
-void cirrus_driver_unload(struct drm_device *dev)
-{
-	struct cirrus_device *cdev = dev->dev_private;
-
-	if (cdev == NULL)
-		return;
-	cirrus_modeset_fini(cdev);
-	cirrus_mm_fini(cdev);
-	cirrus_device_fini(cdev);
-	kfree(cdev);
-	dev->dev_private = NULL;
-}
-
-int cirrus_gem_create(struct drm_device *dev,
-		   u32 size, bool iskernel,
-		   struct drm_gem_object **obj)
-{
-	struct cirrus_bo *cirrusbo;
-	int ret;
-
-	*obj = NULL;
-
-	size = roundup(size, PAGE_SIZE);
-	if (size == 0)
-		return -EINVAL;
-
-	ret = cirrus_bo_create(dev, size, 0, 0, &cirrusbo);
-	if (ret) {
-		if (ret != -ERESTARTSYS)
-			DRM_ERROR("failed to allocate GEM object\n");
-		return ret;
-	}
-	*obj = &cirrusbo->gem;
-	return 0;
-}
-
-int cirrus_dumb_create(struct drm_file *file,
-		    struct drm_device *dev,
-		    struct drm_mode_create_dumb *args)
-{
-	int ret;
-	struct drm_gem_object *gobj;
-	u32 handle;
-
-	args->pitch = args->width * ((args->bpp + 7) / 8);
-	args->size = args->pitch * args->height;
-
-	ret = cirrus_gem_create(dev, args->size, false,
-			     &gobj);
-	if (ret)
-		return ret;
-
-	ret = drm_gem_handle_create(file, gobj, &handle);
-	drm_gem_object_put_unlocked(gobj);
-	if (ret)
-		return ret;
-
-	args->handle = handle;
-	return 0;
-}
-
-static void cirrus_bo_unref(struct cirrus_bo **bo)
-{
-	struct ttm_buffer_object *tbo;
-
-	if ((*bo) == NULL)
-		return;
-
-	tbo = &((*bo)->bo);
-	ttm_bo_put(tbo);
-	*bo = NULL;
-}
-
-void cirrus_gem_free_object(struct drm_gem_object *obj)
-{
-	struct cirrus_bo *cirrus_bo = gem_to_cirrus_bo(obj);
-
-	cirrus_bo_unref(&cirrus_bo);
-}
-
-
-static inline u64 cirrus_bo_mmap_offset(struct cirrus_bo *bo)
-{
-	return drm_vma_node_offset_addr(&bo->bo.vma_node);
-}
-
-int
-cirrus_dumb_mmap_offset(struct drm_file *file,
-		     struct drm_device *dev,
-		     uint32_t handle,
-		     uint64_t *offset)
-{
-	struct drm_gem_object *obj;
-	struct cirrus_bo *bo;
-
-	obj = drm_gem_object_lookup(file, handle);
-	if (obj == NULL)
-		return -ENOENT;
-
-	bo = gem_to_cirrus_bo(obj);
-	*offset = cirrus_bo_mmap_offset(bo);
-
-	drm_gem_object_put_unlocked(obj);
-
-	return 0;
-}
-
-bool cirrus_check_framebuffer(struct cirrus_device *cdev, int width, int height,
-			      int bpp, int pitch)
-{
-	const int max_pitch = 0x1FF << 3; /* (4096 - 1) & ~111b bytes */
-	const int max_size = cdev->mc.vram_size;
-
-	if (bpp > cirrus_bpp)
-		return false;
-	if (bpp > 32)
-		return false;
-
-	if (pitch > max_pitch)
-		return false;
-
-	if (pitch * height > max_size)
-		return false;
-
-	return true;
-}
diff --git a/drivers/gpu/drm/cirrus/cirrus_mode.c b/drivers/gpu/drm/cirrus/cirrus_mode.c
deleted file mode 100644
index 7f9bc32af685..000000000000
--- a/drivers/gpu/drm/cirrus/cirrus_mode.c
+++ /dev/null
@@ -1,621 +0,0 @@
-
-/*
- * Copyright 2012 Red Hat
- *
- * This file is subject to the terms and conditions of the GNU General
- * Public License version 2. See the file COPYING in the main
- * directory of this archive for more details.
- *
- * Authors: Matthew Garrett
- *          Dave Airlie
- *
- * Portions of this code derived from cirrusfb.c:
- * drivers/video/cirrusfb.c - driver for Cirrus Logic chipsets
- *
- * Copyright 1999-2001 Jeff Garzik <jgarzik@pobox.com>
- */
-#include <drm/drmP.h>
-#include <drm/drm_crtc_helper.h>
-#include <drm/drm_plane_helper.h>
-#include <drm/drm_probe_helper.h>
-
-#include <video/cirrus.h>
-
-#include "cirrus_drv.h"
-
-#define CIRRUS_LUT_SIZE 256
-
-#define PALETTE_INDEX 0x8
-#define PALETTE_DATA 0x9
-
-/*
- * This file contains setup code for the CRTC.
- */
-
-/*
- * The DRM core requires DPMS functions, but they make little sense in our
- * case and so are just stubs
- */
-
-static void cirrus_crtc_dpms(struct drm_crtc *crtc, int mode)
-{
-	struct drm_device *dev = crtc->dev;
-	struct cirrus_device *cdev = dev->dev_private;
-	u8 sr01, gr0e;
-
-	switch (mode) {
-	case DRM_MODE_DPMS_ON:
-		sr01 = 0x00;
-		gr0e = 0x00;
-		break;
-	case DRM_MODE_DPMS_STANDBY:
-		sr01 = 0x20;
-		gr0e = 0x02;
-		break;
-	case DRM_MODE_DPMS_SUSPEND:
-		sr01 = 0x20;
-		gr0e = 0x04;
-		break;
-	case DRM_MODE_DPMS_OFF:
-		sr01 = 0x20;
-		gr0e = 0x06;
-		break;
-	default:
-		return;
-	}
-
-	WREG8(SEQ_INDEX, 0x1);
-	sr01 |= RREG8(SEQ_DATA) & ~0x20;
-	WREG_SEQ(0x1, sr01);
-
-	WREG8(GFX_INDEX, 0xe);
-	gr0e |= RREG8(GFX_DATA) & ~0x06;
-	WREG_GFX(0xe, gr0e);
-}
-
-static void cirrus_set_start_address(struct drm_crtc *crtc, unsigned offset)
-{
-	struct cirrus_device *cdev = crtc->dev->dev_private;
-	u32 addr;
-	u8 tmp;
-
-	addr = offset >> 2;
-	WREG_CRT(0x0c, (u8)((addr >> 8) & 0xff));
-	WREG_CRT(0x0d, (u8)(addr & 0xff));
-
-	WREG8(CRT_INDEX, 0x1b);
-	tmp = RREG8(CRT_DATA);
-	tmp &= 0xf2;
-	tmp |= (addr >> 16) & 0x01;
-	tmp |= (addr >> 15) & 0x0c;
-	WREG_CRT(0x1b, tmp);
-	WREG8(CRT_INDEX, 0x1d);
-	tmp = RREG8(CRT_DATA);
-	tmp &= 0x7f;
-	tmp |= (addr >> 12) & 0x80;
-	WREG_CRT(0x1d, tmp);
-}
-
-/* cirrus is different - we will force move buffers out of VRAM */
-static int cirrus_crtc_do_set_base(struct drm_crtc *crtc,
-				struct drm_framebuffer *fb,
-				int x, int y, int atomic)
-{
-	struct cirrus_device *cdev = crtc->dev->dev_private;
-	struct cirrus_bo *bo;
-	int ret;
-	u64 gpu_addr;
-
-	/* push the previous fb to system ram */
-	if (!atomic && fb) {
-		bo = gem_to_cirrus_bo(fb->obj[0]);
-		ret = cirrus_bo_reserve(bo, false);
-		if (ret)
-			return ret;
-		cirrus_bo_push_sysram(bo);
-		cirrus_bo_unreserve(bo);
-	}
-
-	bo = gem_to_cirrus_bo(crtc->primary->fb->obj[0]);
-
-	ret = cirrus_bo_reserve(bo, false);
-	if (ret)
-		return ret;
-
-	ret = cirrus_bo_pin(bo, TTM_PL_FLAG_VRAM, &gpu_addr);
-	if (ret) {
-		cirrus_bo_unreserve(bo);
-		return ret;
-	}
-
-	if (cdev->mode_info.gfbdev->gfb == crtc->primary->fb) {
-		/* if pushing console in kmap it */
-		ret = ttm_bo_kmap(&bo->bo, 0, bo->bo.num_pages, &bo->kmap);
-		if (ret)
-			DRM_ERROR("failed to kmap fbcon\n");
-	}
-	cirrus_bo_unreserve(bo);
-
-	cirrus_set_start_address(crtc, (u32)gpu_addr);
-	return 0;
-}
-
-static int cirrus_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
-			     struct drm_framebuffer *old_fb)
-{
-	return cirrus_crtc_do_set_base(crtc, old_fb, x, y, 0);
-}
-
-/*
- * The meat of this driver. The core passes us a mode and we have to program
- * it. The modesetting here is the bare minimum required to satisfy the qemu
- * emulation of this hardware, and running this against a real device is
- * likely to result in an inadequately programmed mode. We've already had
- * the opportunity to modify the mode, so whatever we receive here should
- * be something that can be correctly programmed and displayed
- */
-static int cirrus_crtc_mode_set(struct drm_crtc *crtc,
-				struct drm_display_mode *mode,
-				struct drm_display_mode *adjusted_mode,
-				int x, int y, struct drm_framebuffer *old_fb)
-{
-	struct drm_device *dev = crtc->dev;
-	struct cirrus_device *cdev = dev->dev_private;
-	const struct drm_framebuffer *fb = crtc->primary->fb;
-	int hsyncstart, hsyncend, htotal, hdispend;
-	int vtotal, vdispend;
-	int tmp;
-	int sr07 = 0, hdr = 0;
-
-	htotal = mode->htotal / 8;
-	hsyncend = mode->hsync_end / 8;
-	hsyncstart = mode->hsync_start / 8;
-	hdispend = mode->hdisplay / 8;
-
-	vtotal = mode->vtotal;
-	vdispend = mode->vdisplay;
-
-	vdispend -= 1;
-	vtotal -= 2;
-
-	htotal -= 5;
-	hdispend -= 1;
-	hsyncstart += 1;
-	hsyncend += 1;
-
-	WREG_CRT(VGA_CRTC_V_SYNC_END, 0x20);
-	WREG_CRT(VGA_CRTC_H_TOTAL, htotal);
-	WREG_CRT(VGA_CRTC_H_DISP, hdispend);
-	WREG_CRT(VGA_CRTC_H_SYNC_START, hsyncstart);
-	WREG_CRT(VGA_CRTC_H_SYNC_END, hsyncend);
-	WREG_CRT(VGA_CRTC_V_TOTAL, vtotal & 0xff);
-	WREG_CRT(VGA_CRTC_V_DISP_END, vdispend & 0xff);
-
-	tmp = 0x40;
-	if ((vdispend + 1) & 512)
-		tmp |= 0x20;
-	WREG_CRT(VGA_CRTC_MAX_SCAN, tmp);
-
-	/*
-	 * Overflow bits for values that don't fit in the standard registers
-	 */
-	tmp = 16;
-	if (vtotal & 256)
-		tmp |= 1;
-	if (vdispend & 256)
-		tmp |= 2;
-	if ((vdispend + 1) & 256)
-		tmp |= 8;
-	if (vtotal & 512)
-		tmp |= 32;
-	if (vdispend & 512)
-		tmp |= 64;
-	WREG_CRT(VGA_CRTC_OVERFLOW, tmp);
-
-	tmp = 0;
-
-	/* More overflow bits */
-
-	if ((htotal + 5) & 64)
-		tmp |= 16;
-	if ((htotal + 5) & 128)
-		tmp |= 32;
-	if (vtotal & 256)
-		tmp |= 64;
-	if (vtotal & 512)
-		tmp |= 128;
-
-	WREG_CRT(CL_CRT1A, tmp);
-
-	/* Disable Hercules/CGA compatibility */
-	WREG_CRT(VGA_CRTC_MODE, 0x03);
-
-	WREG8(SEQ_INDEX, 0x7);
-	sr07 = RREG8(SEQ_DATA);
-	sr07 &= 0xe0;
-	hdr = 0;
-	switch (fb->format->cpp[0] * 8) {
-	case 8:
-		sr07 |= 0x11;
-		break;
-	case 16:
-		sr07 |= 0x17;
-		hdr = 0xc1;
-		break;
-	case 24:
-		sr07 |= 0x15;
-		hdr = 0xc5;
-		break;
-	case 32:
-		sr07 |= 0x19;
-		hdr = 0xc5;
-		break;
-	default:
-		return -1;
-	}
-
-	WREG_SEQ(0x7, sr07);
-
-	/* Program the pitch */
-	tmp = fb->pitches[0] / 8;
-	WREG_CRT(VGA_CRTC_OFFSET, tmp);
-
-	/* Enable extended blanking and pitch bits, and enable full memory */
-	tmp = 0x22;
-	tmp |= (fb->pitches[0] >> 7) & 0x10;
-	tmp |= (fb->pitches[0] >> 6) & 0x40;
-	WREG_CRT(0x1b, tmp);
-
-	/* Enable high-colour modes */
-	WREG_GFX(VGA_GFX_MODE, 0x40);
-
-	/* And set graphics mode */
-	WREG_GFX(VGA_GFX_MISC, 0x01);
-
-	WREG_HDR(hdr);
-	cirrus_crtc_do_set_base(crtc, old_fb, x, y, 0);
-
-	/* Unblank (needed on S3 resume, vgabios doesn't do it then) */
-	outb(0x20, 0x3c0);
-	return 0;
-}
-
-/*
- * This is called before a mode is programmed. A typical use might be to
- * enable DPMS during the programming to avoid seeing intermediate stages,
- * but that's not relevant to us
- */
-static void cirrus_crtc_prepare(struct drm_crtc *crtc)
-{
-}
-
-static void cirrus_crtc_load_lut(struct drm_crtc *crtc)
-{
-	struct drm_device *dev = crtc->dev;
-	struct cirrus_device *cdev = dev->dev_private;
-	u16 *r, *g, *b;
-	int i;
-
-	if (!crtc->enabled)
-		return;
-
-	r = crtc->gamma_store;
-	g = r + crtc->gamma_size;
-	b = g + crtc->gamma_size;
-
-	for (i = 0; i < CIRRUS_LUT_SIZE; i++) {
-		/* VGA registers */
-		WREG8(PALETTE_INDEX, i);
-		WREG8(PALETTE_DATA, *r++ >> 8);
-		WREG8(PALETTE_DATA, *g++ >> 8);
-		WREG8(PALETTE_DATA, *b++ >> 8);
-	}
-}
-
-/*
- * This is called after a mode is programmed. It should reverse anything done
- * by the prepare function
- */
-static void cirrus_crtc_commit(struct drm_crtc *crtc)
-{
-	cirrus_crtc_load_lut(crtc);
-}
-
-/*
- * The core can pass us a set of gamma values to program. We actually only
- * use this for 8-bit mode so can't perform smooth fades on deeper modes,
- * but it's a requirement that we provide the function
- */
-static int cirrus_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
-				 u16 *blue, uint32_t size,
-				 struct drm_modeset_acquire_ctx *ctx)
-{
-	cirrus_crtc_load_lut(crtc);
-
-	return 0;
-}
-
-/* Simple cleanup function */
-static void cirrus_crtc_destroy(struct drm_crtc *crtc)
-{
-	struct cirrus_crtc *cirrus_crtc = to_cirrus_crtc(crtc);
-
-	drm_crtc_cleanup(crtc);
-	kfree(cirrus_crtc);
-}
-
-/* These provide the minimum set of functions required to handle a CRTC */
-static const struct drm_crtc_funcs cirrus_crtc_funcs = {
-	.gamma_set = cirrus_crtc_gamma_set,
-	.set_config = drm_crtc_helper_set_config,
-	.destroy = cirrus_crtc_destroy,
-};
-
-static const struct drm_crtc_helper_funcs cirrus_helper_funcs = {
-	.dpms = cirrus_crtc_dpms,
-	.mode_set = cirrus_crtc_mode_set,
-	.mode_set_base = cirrus_crtc_mode_set_base,
-	.prepare = cirrus_crtc_prepare,
-	.commit = cirrus_crtc_commit,
-};
-
-/* CRTC setup */
-static const uint32_t cirrus_formats_16[] = {
-	DRM_FORMAT_RGB565,
-};
-
-static const uint32_t cirrus_formats_24[] = {
-	DRM_FORMAT_RGB888,
-	DRM_FORMAT_RGB565,
-};
-
-static const uint32_t cirrus_formats_32[] = {
-	DRM_FORMAT_XRGB8888,
-	DRM_FORMAT_ARGB8888,
-	DRM_FORMAT_RGB888,
-	DRM_FORMAT_RGB565,
-};
-
-static struct drm_plane *cirrus_primary_plane(struct drm_device *dev)
-{
-	const uint32_t *formats;
-	uint32_t nformats;
-	struct drm_plane *primary;
-	int ret;
-
-	switch (cirrus_bpp) {
-	case 16:
-		formats = cirrus_formats_16;
-		nformats = ARRAY_SIZE(cirrus_formats_16);
-		break;
-	case 24:
-		formats = cirrus_formats_24;
-		nformats = ARRAY_SIZE(cirrus_formats_24);
-		break;
-	case 32:
-		formats = cirrus_formats_32;
-		nformats = ARRAY_SIZE(cirrus_formats_32);
-		break;
-	default:
-		return NULL;
-	}
-
-	primary = kzalloc(sizeof(*primary), GFP_KERNEL);
-	if (primary == NULL) {
-		DRM_DEBUG_KMS("Failed to allocate primary plane\n");
-		return NULL;
-	}
-
-	ret = drm_universal_plane_init(dev, primary, 0,
-				       &drm_primary_helper_funcs,
-				       formats, nformats,
-				       NULL,
-				       DRM_PLANE_TYPE_PRIMARY, NULL);
-	if (ret) {
-		kfree(primary);
-		primary = NULL;
-	}
-
-	return primary;
-}
-
-static void cirrus_crtc_init(struct drm_device *dev)
-{
-	struct cirrus_device *cdev = dev->dev_private;
-	struct cirrus_crtc *cirrus_crtc;
-	struct drm_plane *primary;
-
-	cirrus_crtc = kzalloc(sizeof(struct cirrus_crtc) +
-			      (CIRRUSFB_CONN_LIMIT * sizeof(struct drm_connector *)),
-			      GFP_KERNEL);
-
-	if (cirrus_crtc == NULL)
-		return;
-
-	primary = cirrus_primary_plane(dev);
-	if (primary == NULL) {
-		kfree(cirrus_crtc);
-		return;
-	}
-
-	drm_crtc_init_with_planes(dev, &cirrus_crtc->base,
-				  primary, NULL,
-				  &cirrus_crtc_funcs, NULL);
-
-	drm_mode_crtc_set_gamma_size(&cirrus_crtc->base, CIRRUS_LUT_SIZE);
-	cdev->mode_info.crtc = cirrus_crtc;
-
-	drm_crtc_helper_add(&cirrus_crtc->base, &cirrus_helper_funcs);
-}
-
-static void cirrus_encoder_mode_set(struct drm_encoder *encoder,
-				struct drm_display_mode *mode,
-				struct drm_display_mode *adjusted_mode)
-{
-}
-
-static void cirrus_encoder_dpms(struct drm_encoder *encoder, int state)
-{
-	return;
-}
-
-static void cirrus_encoder_prepare(struct drm_encoder *encoder)
-{
-}
-
-static void cirrus_encoder_commit(struct drm_encoder *encoder)
-{
-}
-
-static void cirrus_encoder_destroy(struct drm_encoder *encoder)
-{
-	struct cirrus_encoder *cirrus_encoder = to_cirrus_encoder(encoder);
-	drm_encoder_cleanup(encoder);
-	kfree(cirrus_encoder);
-}
-
-static const struct drm_encoder_helper_funcs cirrus_encoder_helper_funcs = {
-	.dpms = cirrus_encoder_dpms,
-	.mode_set = cirrus_encoder_mode_set,
-	.prepare = cirrus_encoder_prepare,
-	.commit = cirrus_encoder_commit,
-};
-
-static const struct drm_encoder_funcs cirrus_encoder_encoder_funcs = {
-	.destroy = cirrus_encoder_destroy,
-};
-
-static struct drm_encoder *cirrus_encoder_init(struct drm_device *dev)
-{
-	struct drm_encoder *encoder;
-	struct cirrus_encoder *cirrus_encoder;
-
-	cirrus_encoder = kzalloc(sizeof(struct cirrus_encoder), GFP_KERNEL);
-	if (!cirrus_encoder)
-		return NULL;
-
-	encoder = &cirrus_encoder->base;
-	encoder->possible_crtcs = 0x1;
-
-	drm_encoder_init(dev, encoder, &cirrus_encoder_encoder_funcs,
-			 DRM_MODE_ENCODER_DAC, NULL);
-	drm_encoder_helper_add(encoder, &cirrus_encoder_helper_funcs);
-
-	return encoder;
-}
-
-
-static int cirrus_vga_get_modes(struct drm_connector *connector)
-{
-	int count;
-
-	/* Just add a static list of modes */
-	if (cirrus_bpp <= 24) {
-		count = drm_add_modes_noedid(connector, 1280, 1024);
-		drm_set_preferred_mode(connector, 1024, 768);
-	} else {
-		count = drm_add_modes_noedid(connector, 800, 600);
-		drm_set_preferred_mode(connector, 800, 600);
-	}
-	return count;
-}
-
-static struct drm_encoder *cirrus_connector_best_encoder(struct drm_connector
-						  *connector)
-{
-	int enc_id = connector->encoder_ids[0];
-	/* pick the encoder ids */
-	if (enc_id)
-		return drm_encoder_find(connector->dev, NULL, enc_id);
-	return NULL;
-}
-
-static void cirrus_connector_destroy(struct drm_connector *connector)
-{
-	drm_connector_cleanup(connector);
-	kfree(connector);
-}
-
-static const struct drm_connector_helper_funcs cirrus_vga_connector_helper_funcs = {
-	.get_modes = cirrus_vga_get_modes,
-	.best_encoder = cirrus_connector_best_encoder,
-};
-
-static const struct drm_connector_funcs cirrus_vga_connector_funcs = {
-	.dpms = drm_helper_connector_dpms,
-	.fill_modes = drm_helper_probe_single_connector_modes,
-	.destroy = cirrus_connector_destroy,
-};
-
-static struct drm_connector *cirrus_vga_init(struct drm_device *dev)
-{
-	struct drm_connector *connector;
-	struct cirrus_connector *cirrus_connector;
-
-	cirrus_connector = kzalloc(sizeof(struct cirrus_connector), GFP_KERNEL);
-	if (!cirrus_connector)
-		return NULL;
-
-	connector = &cirrus_connector->base;
-
-	drm_connector_init(dev, connector,
-			   &cirrus_vga_connector_funcs, DRM_MODE_CONNECTOR_VGA);
-
-	drm_connector_helper_add(connector, &cirrus_vga_connector_helper_funcs);
-
-	drm_connector_register(connector);
-	return connector;
-}
-
-
-int cirrus_modeset_init(struct cirrus_device *cdev)
-{
-	struct drm_encoder *encoder;
-	struct drm_connector *connector;
-	int ret;
-
-	drm_mode_config_init(cdev->dev);
-	cdev->mode_info.mode_config_initialized = true;
-
-	cdev->dev->mode_config.max_width = CIRRUS_MAX_FB_WIDTH;
-	cdev->dev->mode_config.max_height = CIRRUS_MAX_FB_HEIGHT;
-
-	cdev->dev->mode_config.fb_base = cdev->mc.vram_base;
-	cdev->dev->mode_config.preferred_depth = cirrus_bpp;
-	/* don't prefer a shadow on virt GPU */
-	cdev->dev->mode_config.prefer_shadow = 0;
-
-	cirrus_crtc_init(cdev->dev);
-
-	encoder = cirrus_encoder_init(cdev->dev);
-	if (!encoder) {
-		DRM_ERROR("cirrus_encoder_init failed\n");
-		return -1;
-	}
-
-	connector = cirrus_vga_init(cdev->dev);
-	if (!connector) {
-		DRM_ERROR("cirrus_vga_init failed\n");
-		return -1;
-	}
-
-	drm_connector_attach_encoder(connector, encoder);
-
-	ret = cirrus_fbdev_init(cdev);
-	if (ret) {
-		DRM_ERROR("cirrus_fbdev_init failed\n");
-		return ret;
-	}
-
-	return 0;
-}
-
-void cirrus_modeset_fini(struct cirrus_device *cdev)
-{
-	cirrus_fbdev_fini(cdev);
-
-	if (cdev->mode_info.mode_config_initialized) {
-		drm_mode_config_cleanup(cdev->dev);
-		cdev->mode_info.mode_config_initialized = false;
-	}
-}
diff --git a/drivers/gpu/drm/cirrus/cirrus_ttm.c b/drivers/gpu/drm/cirrus/cirrus_ttm.c
index e075810b4bd4..e6b98467a428 100644
--- a/drivers/gpu/drm/cirrus/cirrus_ttm.c
+++ b/drivers/gpu/drm/cirrus/cirrus_ttm.c
@@ -178,7 +178,6 @@ int cirrus_mm_init(struct cirrus_device *cirrus)
 	ret = ttm_bo_device_init(&cirrus->ttm.bdev,
 				 &cirrus_bo_driver,
 				 dev->anon_inode->i_mapping,
-				 DRM_FILE_PAGE_OFFSET,
 				 true);
 	if (ret) {
 		DRM_ERROR("Error initialising bo driver; %d\n", ret);
@@ -331,13 +330,8 @@ int cirrus_bo_push_sysram(struct cirrus_bo *bo)
 
 int cirrus_mmap(struct file *filp, struct vm_area_struct *vma)
 {
-	struct drm_file *file_priv;
-	struct cirrus_device *cirrus;
+	struct drm_file *file_priv = filp->private_data;
+	struct cirrus_device *cirrus = file_priv->minor->dev->dev_private;
 
-	if (unlikely(vma->vm_pgoff < DRM_FILE_PAGE_OFFSET))
-		return -EINVAL;
-
-	file_priv = filp->private_data;
-	cirrus = file_priv->minor->dev->dev_private;
 	return ttm_bo_mmap(filp, vma, &cirrus->ttm.bdev);
 }
diff --git a/drivers/gpu/drm/drm_atomic.c b/drivers/gpu/drm/drm_atomic.c
index 5eb40130fafb..f4924cb7f495 100644
--- a/drivers/gpu/drm/drm_atomic.c
+++ b/drivers/gpu/drm/drm_atomic.c
@@ -798,6 +798,50 @@ drm_atomic_get_private_obj_state(struct drm_atomic_state *state,
 EXPORT_SYMBOL(drm_atomic_get_private_obj_state);
 
 /**
+ * drm_atomic_get_old_private_obj_state
+ * @state: global atomic state object
+ * @obj: private_obj to grab
+ *
+ * This function returns the old private object state for the given private_obj,
+ * or NULL if the private_obj is not part of the global atomic state.
+ */
+struct drm_private_state *
+drm_atomic_get_old_private_obj_state(struct drm_atomic_state *state,
+				     struct drm_private_obj *obj)
+{
+	int i;
+
+	for (i = 0; i < state->num_private_objs; i++)
+		if (obj == state->private_objs[i].ptr)
+			return state->private_objs[i].old_state;
+
+	return NULL;
+}
+EXPORT_SYMBOL(drm_atomic_get_old_private_obj_state);
+
+/**
+ * drm_atomic_get_new_private_obj_state
+ * @state: global atomic state object
+ * @obj: private_obj to grab
+ *
+ * This function returns the new private object state for the given private_obj,
+ * or NULL if the private_obj is not part of the global atomic state.
+ */
+struct drm_private_state *
+drm_atomic_get_new_private_obj_state(struct drm_atomic_state *state,
+				     struct drm_private_obj *obj)
+{
+	int i;
+
+	for (i = 0; i < state->num_private_objs; i++)
+		if (obj == state->private_objs[i].ptr)
+			return state->private_objs[i].new_state;
+
+	return NULL;
+}
+EXPORT_SYMBOL(drm_atomic_get_new_private_obj_state);
+
+/**
  * drm_atomic_get_connector_state - get connector state
  * @state: global atomic state object
  * @connector: connector to get state object for
@@ -1236,4 +1280,3 @@ int drm_atomic_debugfs_init(struct drm_minor *minor)
 			minor->debugfs_root, minor);
 }
 #endif
-
diff --git a/drivers/gpu/drm/drm_atomic_helper.c b/drivers/gpu/drm/drm_atomic_helper.c
index fbb76332cc9f..2e0cb4246cbd 100644
--- a/drivers/gpu/drm/drm_atomic_helper.c
+++ b/drivers/gpu/drm/drm_atomic_helper.c
@@ -495,7 +495,7 @@ mode_fixup(struct drm_atomic_state *state)
 static enum drm_mode_status mode_valid_path(struct drm_connector *connector,
 					    struct drm_encoder *encoder,
 					    struct drm_crtc *crtc,
-					    struct drm_display_mode *mode)
+					    const struct drm_display_mode *mode)
 {
 	enum drm_mode_status ret;
 
@@ -534,7 +534,7 @@ mode_valid(struct drm_atomic_state *state)
 		struct drm_crtc *crtc = conn_state->crtc;
 		struct drm_crtc_state *crtc_state;
 		enum drm_mode_status mode_status;
-		struct drm_display_mode *mode;
+		const struct drm_display_mode *mode;
 
 		if (!crtc || !encoder)
 			continue;
@@ -1751,7 +1751,7 @@ int drm_atomic_helper_commit(struct drm_device *dev,
 	 *
 	 * NOTE: Commit work has multiple phases, first hardware commit, then
 	 * cleanup. We want them to overlap, hence need system_unbound_wq to
-	 * make sure work items don't artifically stall on each another.
+	 * make sure work items don't artificially stall on each another.
 	 */
 
 	drm_atomic_state_get(state);
@@ -1785,7 +1785,7 @@ EXPORT_SYMBOL(drm_atomic_helper_commit);
  *
  * Asynchronous workers need to have sufficient parallelism to be able to run
  * different atomic commits on different CRTCs in parallel. The simplest way to
- * achive this is by running them on the &system_unbound_wq work queue. Note
+ * achieve this is by running them on the &system_unbound_wq work queue. Note
  * that drivers are not required to split up atomic commits and run an
  * individual commit in parallel - userspace is supposed to do that if it cares.
  * But it might be beneficial to do that for modesets, since those necessarily
@@ -2260,10 +2260,21 @@ EXPORT_SYMBOL(drm_atomic_helper_commit_cleanup_done);
 int drm_atomic_helper_prepare_planes(struct drm_device *dev,
 				     struct drm_atomic_state *state)
 {
+	struct drm_connector *connector;
+	struct drm_connector_state *new_conn_state;
 	struct drm_plane *plane;
 	struct drm_plane_state *new_plane_state;
 	int ret, i, j;
 
+	for_each_new_connector_in_state(state, connector, new_conn_state, i) {
+		if (!new_conn_state->writeback_job)
+			continue;
+
+		ret = drm_writeback_prepare_job(new_conn_state->writeback_job);
+		if (ret < 0)
+			return ret;
+	}
+
 	for_each_new_plane_in_state(state, plane, new_plane_state, i) {
 		const struct drm_plane_helper_funcs *funcs;
 
diff --git a/drivers/gpu/drm/drm_atomic_state_helper.c b/drivers/gpu/drm/drm_atomic_state_helper.c
index 4985384e51f6..59ffb6b9c745 100644
--- a/drivers/gpu/drm/drm_atomic_state_helper.c
+++ b/drivers/gpu/drm/drm_atomic_state_helper.c
@@ -30,6 +30,7 @@
 #include <drm/drm_connector.h>
 #include <drm/drm_atomic.h>
 #include <drm/drm_device.h>
+#include <drm/drm_writeback.h>
 
 #include <linux/slab.h>
 #include <linux/dma-fence.h>
@@ -412,6 +413,9 @@ __drm_atomic_helper_connector_destroy_state(struct drm_connector_state *state)
 
 	if (state->commit)
 		drm_crtc_commit_put(state->commit);
+
+	if (state->writeback_job)
+		drm_writeback_cleanup_job(state->writeback_job);
 }
 EXPORT_SYMBOL(__drm_atomic_helper_connector_destroy_state);
 
diff --git a/drivers/gpu/drm/drm_atomic_uapi.c b/drivers/gpu/drm/drm_atomic_uapi.c
index 0aabd401d3ca..428d82662dc4 100644
--- a/drivers/gpu/drm/drm_atomic_uapi.c
+++ b/drivers/gpu/drm/drm_atomic_uapi.c
@@ -512,8 +512,8 @@ drm_atomic_crtc_get_property(struct drm_crtc *crtc,
 }
 
 static int drm_atomic_plane_set_property(struct drm_plane *plane,
-		struct drm_plane_state *state, struct drm_property *property,
-		uint64_t val)
+		struct drm_plane_state *state, struct drm_file *file_priv,
+		struct drm_property *property, uint64_t val)
 {
 	struct drm_device *dev = plane->dev;
 	struct drm_mode_config *config = &dev->mode_config;
@@ -521,7 +521,8 @@ static int drm_atomic_plane_set_property(struct drm_plane *plane,
 	int ret;
 
 	if (property == config->prop_fb_id) {
-		struct drm_framebuffer *fb = drm_framebuffer_lookup(dev, NULL, val);
+		struct drm_framebuffer *fb;
+		fb = drm_framebuffer_lookup(dev, file_priv, val);
 		drm_atomic_set_fb_for_plane(state, fb);
 		if (fb)
 			drm_framebuffer_put(fb);
@@ -537,7 +538,9 @@ static int drm_atomic_plane_set_property(struct drm_plane *plane,
 			return -EINVAL;
 
 	} else if (property == config->prop_crtc_id) {
-		struct drm_crtc *crtc = drm_crtc_find(dev, NULL, val);
+		struct drm_crtc *crtc = drm_crtc_find(dev, file_priv, val);
+		if (val && !crtc)
+			return -EACCES;
 		return drm_atomic_set_crtc_for_plane(state, crtc);
 	} else if (property == config->prop_crtc_x) {
 		state->crtc_x = U642I64(val);
@@ -647,28 +650,15 @@ drm_atomic_plane_get_property(struct drm_plane *plane,
 	return 0;
 }
 
-static struct drm_writeback_job *
-drm_atomic_get_writeback_job(struct drm_connector_state *conn_state)
-{
-	WARN_ON(conn_state->connector->connector_type != DRM_MODE_CONNECTOR_WRITEBACK);
-
-	if (!conn_state->writeback_job)
-		conn_state->writeback_job =
-			kzalloc(sizeof(*conn_state->writeback_job), GFP_KERNEL);
-
-	return conn_state->writeback_job;
-}
-
 static int drm_atomic_set_writeback_fb_for_connector(
 		struct drm_connector_state *conn_state,
 		struct drm_framebuffer *fb)
 {
-	struct drm_writeback_job *job =
-		drm_atomic_get_writeback_job(conn_state);
-	if (!job)
-		return -ENOMEM;
+	int ret;
 
-	drm_framebuffer_assign(&job->fb, fb);
+	ret = drm_writeback_set_fb(conn_state, fb);
+	if (ret < 0)
+		return ret;
 
 	if (fb)
 		DRM_DEBUG_ATOMIC("Set [FB:%d] for connector state %p\n",
@@ -681,14 +671,16 @@ static int drm_atomic_set_writeback_fb_for_connector(
 }
 
 static int drm_atomic_connector_set_property(struct drm_connector *connector,
-		struct drm_connector_state *state, struct drm_property *property,
-		uint64_t val)
+		struct drm_connector_state *state, struct drm_file *file_priv,
+		struct drm_property *property, uint64_t val)
 {
 	struct drm_device *dev = connector->dev;
 	struct drm_mode_config *config = &dev->mode_config;
 
 	if (property == config->prop_crtc_id) {
-		struct drm_crtc *crtc = drm_crtc_find(dev, NULL, val);
+		struct drm_crtc *crtc = drm_crtc_find(dev, file_priv, val);
+		if (val && !crtc)
+			return -EACCES;
 		return drm_atomic_set_crtc_for_connector(state, crtc);
 	} else if (property == config->dpms_property) {
 		/* setting DPMS property requires special handling, which
@@ -746,9 +738,13 @@ static int drm_atomic_connector_set_property(struct drm_connector *connector,
 			return -EINVAL;
 		}
 		state->content_protection = val;
+	} else if (property == connector->colorspace_property) {
+		state->colorspace = val;
 	} else if (property == config->writeback_fb_id_property) {
-		struct drm_framebuffer *fb = drm_framebuffer_lookup(dev, NULL, val);
-		int ret = drm_atomic_set_writeback_fb_for_connector(state, fb);
+		struct drm_framebuffer *fb;
+		int ret;
+		fb = drm_framebuffer_lookup(dev, file_priv, val);
+		ret = drm_atomic_set_writeback_fb_for_connector(state, fb);
 		if (fb)
 			drm_framebuffer_put(fb);
 		return ret;
@@ -814,6 +810,8 @@ drm_atomic_connector_get_property(struct drm_connector *connector,
 		*val = state->picture_aspect_ratio;
 	} else if (property == config->content_type_property) {
 		*val = state->content_type;
+	} else if (property == connector->colorspace_property) {
+		*val = state->colorspace;
 	} else if (property == connector->scaling_mode_property) {
 		*val = state->scaling_mode;
 	} else if (property == connector->content_protection_property) {
@@ -943,6 +941,7 @@ out:
 }
 
 int drm_atomic_set_property(struct drm_atomic_state *state,
+			    struct drm_file *file_priv,
 			    struct drm_mode_object *obj,
 			    struct drm_property *prop,
 			    uint64_t prop_value)
@@ -965,7 +964,8 @@ int drm_atomic_set_property(struct drm_atomic_state *state,
 		}
 
 		ret = drm_atomic_connector_set_property(connector,
-				connector_state, prop, prop_value);
+				connector_state, file_priv,
+				prop, prop_value);
 		break;
 	}
 	case DRM_MODE_OBJECT_CRTC: {
@@ -993,7 +993,8 @@ int drm_atomic_set_property(struct drm_atomic_state *state,
 		}
 
 		ret = drm_atomic_plane_set_property(plane,
-				plane_state, prop, prop_value);
+				plane_state, file_priv,
+				prop, prop_value);
 		break;
 	}
 	default:
@@ -1158,19 +1159,17 @@ static int prepare_signaling(struct drm_device *dev,
 
 	for_each_new_connector_in_state(state, conn, conn_state, i) {
 		struct drm_writeback_connector *wb_conn;
-		struct drm_writeback_job *job;
 		struct drm_out_fence_state *f;
 		struct dma_fence *fence;
 		s32 __user *fence_ptr;
 
+		if (!conn_state->writeback_job)
+			continue;
+
 		fence_ptr = get_out_fence_for_connector(state, conn);
 		if (!fence_ptr)
 			continue;
 
-		job = drm_atomic_get_writeback_job(conn_state);
-		if (!job)
-			return -ENOMEM;
-
 		f = krealloc(*fence_state, sizeof(**fence_state) *
 			     (*num_fences + 1), GFP_KERNEL);
 		if (!f)
@@ -1192,7 +1191,7 @@ static int prepare_signaling(struct drm_device *dev,
 			return ret;
 		}
 
-		job->out_fence = fence;
+		conn_state->writeback_job->out_fence = fence;
 	}
 
 	/*
@@ -1365,8 +1364,8 @@ retry:
 				goto out;
 			}
 
-			ret = drm_atomic_set_property(state, obj, prop,
-						      prop_value);
+			ret = drm_atomic_set_property(state, file_priv,
+						      obj, prop, prop_value);
 			if (ret) {
 				drm_mode_object_put(obj);
 				goto out;
diff --git a/drivers/gpu/drm/drm_auth.c b/drivers/gpu/drm/drm_auth.c
index 1669c42c40ed..22c7a104b802 100644
--- a/drivers/gpu/drm/drm_auth.c
+++ b/drivers/gpu/drm/drm_auth.c
@@ -103,14 +103,11 @@ struct drm_master *drm_master_create(struct drm_device *dev)
 		return NULL;
 
 	kref_init(&master->refcount);
-	spin_lock_init(&master->lock.spinlock);
-	init_waitqueue_head(&master->lock.lock_queue);
+	drm_master_legacy_init(master);
 	idr_init(&master->magic_map);
 	master->dev = dev;
 
 	/* initialize the tree of output resource lessees */
-	master->lessor = NULL;
-	master->lessee_id = 0;
 	INIT_LIST_HEAD(&master->lessees);
 	INIT_LIST_HEAD(&master->lessee_list);
 	idr_init(&master->leases);
@@ -274,21 +271,7 @@ void drm_master_release(struct drm_file *file_priv)
 	if (!drm_is_current_master(file_priv))
 		goto out;
 
-	if (drm_core_check_feature(dev, DRIVER_LEGACY)) {
-		/*
-		 * Since the master is disappearing, so is the
-		 * possibility to lock.
-		 */
-		mutex_lock(&dev->struct_mutex);
-		if (master->lock.hw_lock) {
-			if (dev->sigdata.lock == master->lock.hw_lock)
-				dev->sigdata.lock = NULL;
-			master->lock.hw_lock = NULL;
-			master->lock.file_priv = NULL;
-			wake_up_interruptible_all(&master->lock.lock_queue);
-		}
-		mutex_unlock(&dev->struct_mutex);
-	}
+	drm_legacy_lock_master_cleanup(dev, master);
 
 	if (dev->master == file_priv->master)
 		drm_drop_master(dev, file_priv);
diff --git a/drivers/gpu/drm/drm_bufs.c b/drivers/gpu/drm/drm_bufs.c
index e407adb033e7..bfc419ed9d6c 100644
--- a/drivers/gpu/drm/drm_bufs.c
+++ b/drivers/gpu/drm/drm_bufs.c
@@ -584,6 +584,14 @@ void drm_legacy_master_rmmaps(struct drm_device *dev, struct drm_master *master)
 	mutex_unlock(&dev->struct_mutex);
 }
 
+void drm_legacy_rmmaps(struct drm_device *dev)
+{
+	struct drm_map_list *r_list, *list_temp;
+
+	list_for_each_entry_safe(r_list, list_temp, &dev->maplist, head)
+		drm_legacy_rmmap(dev, r_list->map);
+}
+
 /* The rmmap ioctl appears to be unnecessary.  All mappings are torn down on
  * the last close of the device, and this is necessary for cleanup when things
  * exit uncleanly.  Therefore, having userland manually remove mappings seems
diff --git a/drivers/gpu/drm/drm_client.c b/drivers/gpu/drm/drm_client.c
index 9b2bd28dde0a..f20d1dda3961 100644
--- a/drivers/gpu/drm/drm_client.c
+++ b/drivers/gpu/drm/drm_client.c
@@ -69,7 +69,8 @@ EXPORT_SYMBOL(drm_client_close);
  * @name: Client name
  * @funcs: DRM client functions (optional)
  *
- * This initialises the client and opens a &drm_file. Use drm_client_add() to complete the process.
+ * This initialises the client and opens a &drm_file.
+ * Use drm_client_register() to complete the process.
  * The caller needs to hold a reference on @dev before calling this function.
  * The client is freed when the &drm_device is unregistered. See drm_client_release().
  *
@@ -108,16 +109,16 @@ err_put_module:
 EXPORT_SYMBOL(drm_client_init);
 
 /**
- * drm_client_add - Add client to the device list
+ * drm_client_register - Register client
  * @client: DRM client
  *
  * Add the client to the &drm_device client list to activate its callbacks.
  * @client must be initialized by a call to drm_client_init(). After
- * drm_client_add() it is no longer permissible to call drm_client_release()
+ * drm_client_register() it is no longer permissible to call drm_client_release()
  * directly (outside the unregister callback), instead cleanup will happen
  * automatically on driver unload.
  */
-void drm_client_add(struct drm_client_dev *client)
+void drm_client_register(struct drm_client_dev *client)
 {
 	struct drm_device *dev = client->dev;
 
@@ -125,7 +126,7 @@ void drm_client_add(struct drm_client_dev *client)
 	list_add(&client->list, &dev->clientlist);
 	mutex_unlock(&dev->clientlist_mutex);
 }
-EXPORT_SYMBOL(drm_client_add);
+EXPORT_SYMBOL(drm_client_register);
 
 /**
  * drm_client_release - Release DRM client resources
diff --git a/drivers/gpu/drm/drm_connector.c b/drivers/gpu/drm/drm_connector.c
index dd40eff0911c..b34c3d38bf15 100644
--- a/drivers/gpu/drm/drm_connector.c
+++ b/drivers/gpu/drm/drm_connector.c
@@ -245,6 +245,7 @@ int drm_connector_init(struct drm_device *dev,
 	INIT_LIST_HEAD(&connector->modes);
 	mutex_init(&connector->mutex);
 	connector->edid_blob_ptr = NULL;
+	connector->tile_blob_ptr = NULL;
 	connector->status = connector_status_unknown;
 	connector->display_info.panel_orientation =
 		DRM_MODE_PANEL_ORIENTATION_UNKNOWN;
@@ -272,6 +273,9 @@ int drm_connector_init(struct drm_device *dev,
 	drm_object_attach_property(&connector->base,
 				   config->non_desktop_property,
 				   0);
+	drm_object_attach_property(&connector->base,
+				   config->tile_property,
+				   0);
 
 	if (drm_core_check_feature(dev, DRIVER_ATOMIC)) {
 		drm_object_attach_property(&connector->base, config->prop_crtc_id, 0);
@@ -826,6 +830,33 @@ static struct drm_prop_enum_list drm_cp_enum_list[] = {
 };
 DRM_ENUM_NAME_FN(drm_get_content_protection_name, drm_cp_enum_list)
 
+static const struct drm_prop_enum_list hdmi_colorspaces[] = {
+	/* For Default case, driver will set the colorspace */
+	{ DRM_MODE_COLORIMETRY_DEFAULT, "Default" },
+	/* Standard Definition Colorimetry based on CEA 861 */
+	{ DRM_MODE_COLORIMETRY_SMPTE_170M_YCC, "SMPTE_170M_YCC" },
+	{ DRM_MODE_COLORIMETRY_BT709_YCC, "BT709_YCC" },
+	/* Standard Definition Colorimetry based on IEC 61966-2-4 */
+	{ DRM_MODE_COLORIMETRY_XVYCC_601, "XVYCC_601" },
+	/* High Definition Colorimetry based on IEC 61966-2-4 */
+	{ DRM_MODE_COLORIMETRY_XVYCC_709, "XVYCC_709" },
+	/* Colorimetry based on IEC 61966-2-1/Amendment 1 */
+	{ DRM_MODE_COLORIMETRY_SYCC_601, "SYCC_601" },
+	/* Colorimetry based on IEC 61966-2-5 [33] */
+	{ DRM_MODE_COLORIMETRY_OPYCC_601, "opYCC_601" },
+	/* Colorimetry based on IEC 61966-2-5 */
+	{ DRM_MODE_COLORIMETRY_OPRGB, "opRGB" },
+	/* Colorimetry based on ITU-R BT.2020 */
+	{ DRM_MODE_COLORIMETRY_BT2020_CYCC, "BT2020_CYCC" },
+	/* Colorimetry based on ITU-R BT.2020 */
+	{ DRM_MODE_COLORIMETRY_BT2020_RGB, "BT2020_RGB" },
+	/* Colorimetry based on ITU-R BT.2020 */
+	{ DRM_MODE_COLORIMETRY_BT2020_YCC, "BT2020_YCC" },
+	/* Added as part of Additional Colorimetry Extension in 861.G */
+	{ DRM_MODE_COLORIMETRY_DCI_P3_RGB_D65, "DCI-P3_RGB_D65" },
+	{ DRM_MODE_COLORIMETRY_DCI_P3_RGB_THEATER, "DCI-P3_RGB_Theater" },
+};
+
 /**
  * DOC: standard connector properties
  *
@@ -1385,12 +1416,6 @@ EXPORT_SYMBOL(drm_mode_create_scaling_mode_property);
  *
  *	The driver may place further restrictions within these minimum
  *	and maximum bounds.
- *
- *	The semantics for the vertical blank timestamp differ when
- *	variable refresh rate is active. The vertical blank timestamp
- *	is defined to be an estimate using the current mode's fixed
- *	refresh rate timings. The semantics for the page-flip event
- *	timestamp remain the same.
  */
 
 /**
@@ -1548,6 +1573,57 @@ int drm_mode_create_aspect_ratio_property(struct drm_device *dev)
 EXPORT_SYMBOL(drm_mode_create_aspect_ratio_property);
 
 /**
+ * DOC: standard connector properties
+ *
+ * Colorspace:
+ *     drm_mode_create_colorspace_property - create colorspace property
+ *     This property helps select a suitable colorspace based on the sink
+ *     capability. Modern sink devices support wider gamut like BT2020.
+ *     This helps switch to BT2020 mode if the BT2020 encoded video stream
+ *     is being played by the user, same for any other colorspace. Thereby
+ *     giving a good visual experience to users.
+ *
+ *     The expectation from userspace is that it should parse the EDID
+ *     and get supported colorspaces. Use this property and switch to the
+ *     one supported. Sink supported colorspaces should be retrieved by
+ *     userspace from EDID and driver will not explicitly expose them.
+ *
+ *     Basically the expectation from userspace is:
+ *      - Set up CRTC DEGAMMA/CTM/GAMMA to convert to some sink
+ *        colorspace
+ *      - Set this new property to let the sink know what it
+ *        converted the CRTC output to.
+ *      - This property is just to inform sink what colorspace
+ *        source is trying to drive.
+ *
+ * Called by a driver the first time it's needed, must be attached to desired
+ * connectors.
+ */
+int drm_mode_create_colorspace_property(struct drm_connector *connector)
+{
+	struct drm_device *dev = connector->dev;
+	struct drm_property *prop;
+
+	if (connector->connector_type == DRM_MODE_CONNECTOR_HDMIA ||
+	    connector->connector_type == DRM_MODE_CONNECTOR_HDMIB) {
+		prop = drm_property_create_enum(dev, DRM_MODE_PROP_ENUM,
+						"Colorspace",
+						hdmi_colorspaces,
+						ARRAY_SIZE(hdmi_colorspaces));
+		if (!prop)
+			return -ENOMEM;
+	} else {
+		DRM_DEBUG_KMS("Colorspace property not supported\n");
+		return 0;
+	}
+
+	connector->colorspace_property = prop;
+
+	return 0;
+}
+EXPORT_SYMBOL(drm_mode_create_colorspace_property);
+
+/**
  * drm_mode_create_content_type_property - create content type property
  * @dev: DRM device
  *
@@ -1634,6 +1710,8 @@ EXPORT_SYMBOL(drm_connector_set_path_property);
  * This looks up the tile information for a connector, and creates a
  * property for userspace to parse if it exists. The property is of
  * the form of 8 integers using ':' as a separator.
+ * This is used for dual port tiled displays with DisplayPort SST
+ * or DisplayPort MST connectors.
  *
  * Returns:
  * Zero on success, errno on failure.
@@ -1677,6 +1755,9 @@ EXPORT_SYMBOL(drm_connector_set_tile_property);
  *
  * This function creates a new blob modeset object and assigns its id to the
  * connector's edid property.
+ * Since we also parse tile information from EDID's displayID block, we also
+ * set the connector's tile property here. See drm_connector_set_tile_property()
+ * for more details.
  *
  * Returns:
  * Zero on success, negative errno on failure.
@@ -1718,7 +1799,9 @@ int drm_connector_update_edid_property(struct drm_connector *connector,
 	                                       edid,
 	                                       &connector->base,
 	                                       dev->mode_config.edid_property);
-	return ret;
+	if (ret)
+		return ret;
+	return drm_connector_set_tile_property(connector);
 }
 EXPORT_SYMBOL(drm_connector_update_edid_property);
 
diff --git a/drivers/gpu/drm/drm_crtc.c b/drivers/gpu/drm/drm_crtc.c
index 7dabbaf033a1..790ba5941954 100644
--- a/drivers/gpu/drm/drm_crtc.c
+++ b/drivers/gpu/drm/drm_crtc.c
@@ -559,6 +559,10 @@ int drm_mode_setcrtc(struct drm_device *dev, void *data,
 
 	plane = crtc->primary;
 
+	/* allow disabling with the primary plane leased */
+	if (crtc_req->mode_valid && !drm_lease_held(file_priv, plane->base.id))
+		return -EACCES;
+
 	mutex_lock(&crtc->dev->mode_config.mutex);
 	DRM_MODESET_LOCK_ALL_BEGIN(dev, ctx,
 				   DRM_MODESET_ACQUIRE_INTERRUPTIBLE, ret);
diff --git a/drivers/gpu/drm/drm_crtc_internal.h b/drivers/gpu/drm/drm_crtc_internal.h
index 216f2a9ee3d4..0719a235d6cc 100644
--- a/drivers/gpu/drm/drm_crtc_internal.h
+++ b/drivers/gpu/drm/drm_crtc_internal.h
@@ -214,6 +214,7 @@ int drm_atomic_connector_commit_dpms(struct drm_atomic_state *state,
 				     struct drm_connector *connector,
 				     int mode);
 int drm_atomic_set_property(struct drm_atomic_state *state,
+			    struct drm_file *file_priv,
 			    struct drm_mode_object *obj,
 			    struct drm_property *prop,
 			    uint64_t prop_value);
diff --git a/drivers/gpu/drm/drm_dp_mst_topology.c b/drivers/gpu/drm/drm_dp_mst_topology.c
index dc7ac0c60547..c630ed157994 100644
--- a/drivers/gpu/drm/drm_dp_mst_topology.c
+++ b/drivers/gpu/drm/drm_dp_mst_topology.c
@@ -3022,7 +3022,6 @@ struct edid *drm_dp_mst_get_edid(struct drm_connector *connector, struct drm_dp_
 		edid = drm_edid_duplicate(port->cached_edid);
 	else {
 		edid = drm_get_edid(connector, &port->aux.ddc);
-		drm_connector_set_tile_property(connector);
 	}
 	port->has_audio = drm_detect_monitor_audio(edid);
 	drm_dp_mst_topology_put_port(port);
diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c
index 05bbc2b622fc..862621494a93 100644
--- a/drivers/gpu/drm/drm_drv.c
+++ b/drivers/gpu/drm/drm_drv.c
@@ -286,6 +286,138 @@ void drm_minor_release(struct drm_minor *minor)
  * Note that the lifetime rules for &drm_device instance has still a lot of
  * historical baggage. Hence use the reference counting provided by
  * drm_dev_get() and drm_dev_put() only carefully.
+ *
+ * Display driver example
+ * ~~~~~~~~~~~~~~~~~~~~~~
+ *
+ * The following example shows a typical structure of a DRM display driver.
+ * The example focus on the probe() function and the other functions that is
+ * almost always present and serves as a demonstration of devm_drm_dev_init()
+ * usage with its accompanying drm_driver->release callback.
+ *
+ * .. code-block:: c
+ *
+ *	struct driver_device {
+ *		struct drm_device drm;
+ *		void *userspace_facing;
+ *		struct clk *pclk;
+ *	};
+ *
+ *	static void driver_drm_release(struct drm_device *drm)
+ *	{
+ *		struct driver_device *priv = container_of(...);
+ *
+ *		drm_mode_config_cleanup(drm);
+ *		drm_dev_fini(drm);
+ *		kfree(priv->userspace_facing);
+ *		kfree(priv);
+ *	}
+ *
+ *	static struct drm_driver driver_drm_driver = {
+ *		[...]
+ *		.release = driver_drm_release,
+ *	};
+ *
+ *	static int driver_probe(struct platform_device *pdev)
+ *	{
+ *		struct driver_device *priv;
+ *		struct drm_device *drm;
+ *		int ret;
+ *
+ *		[
+ *		  devm_kzalloc() can't be used here because the drm_device
+ *		  lifetime can exceed the device lifetime if driver unbind
+ *		  happens when userspace still has open file descriptors.
+ *		]
+ *		priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+ *		if (!priv)
+ *			return -ENOMEM;
+ *
+ *		drm = &priv->drm;
+ *
+ *		ret = devm_drm_dev_init(&pdev->dev, drm, &driver_drm_driver);
+ *		if (ret) {
+ *			kfree(drm);
+ *			return ret;
+ *		}
+ *
+ *		drm_mode_config_init(drm);
+ *
+ *		priv->userspace_facing = kzalloc(..., GFP_KERNEL);
+ *		if (!priv->userspace_facing)
+ *			return -ENOMEM;
+ *
+ *		priv->pclk = devm_clk_get(dev, "PCLK");
+ *		if (IS_ERR(priv->pclk))
+ *			return PTR_ERR(priv->pclk);
+ *
+ *		[ Further setup, display pipeline etc ]
+ *
+ *		platform_set_drvdata(pdev, drm);
+ *
+ *		drm_mode_config_reset(drm);
+ *
+ *		ret = drm_dev_register(drm);
+ *		if (ret)
+ *			return ret;
+ *
+ *		drm_fbdev_generic_setup(drm, 32);
+ *
+ *		return 0;
+ *	}
+ *
+ *	[ This function is called before the devm_ resources are released ]
+ *	static int driver_remove(struct platform_device *pdev)
+ *	{
+ *		struct drm_device *drm = platform_get_drvdata(pdev);
+ *
+ *		drm_dev_unregister(drm);
+ *		drm_atomic_helper_shutdown(drm)
+ *
+ *		return 0;
+ *	}
+ *
+ *	[ This function is called on kernel restart and shutdown ]
+ *	static void driver_shutdown(struct platform_device *pdev)
+ *	{
+ *		drm_atomic_helper_shutdown(platform_get_drvdata(pdev));
+ *	}
+ *
+ *	static int __maybe_unused driver_pm_suspend(struct device *dev)
+ *	{
+ *		return drm_mode_config_helper_suspend(dev_get_drvdata(dev));
+ *	}
+ *
+ *	static int __maybe_unused driver_pm_resume(struct device *dev)
+ *	{
+ *		drm_mode_config_helper_resume(dev_get_drvdata(dev));
+ *
+ *		return 0;
+ *	}
+ *
+ *	static const struct dev_pm_ops driver_pm_ops = {
+ *		SET_SYSTEM_SLEEP_PM_OPS(driver_pm_suspend, driver_pm_resume)
+ *	};
+ *
+ *	static struct platform_driver driver_driver = {
+ *		.driver = {
+ *			[...]
+ *			.pm = &driver_pm_ops,
+ *		},
+ *		.probe = driver_probe,
+ *		.remove = driver_remove,
+ *		.shutdown = driver_shutdown,
+ *	};
+ *	module_platform_driver(driver_driver);
+ *
+ * Drivers that want to support device unplugging (USB, DT overlay unload) should
+ * use drm_dev_unplug() instead of drm_dev_unregister(). The driver must protect
+ * regions that is accessing device resources to prevent use after they're
+ * released. This is done using drm_dev_enter() and drm_dev_exit(). There is one
+ * shortcoming however, drm_dev_unplug() marks the drm_device as unplugged before
+ * drm_atomic_helper_shutdown() is called. This means that if the disable code
+ * paths are protected, they will not run on regular driver module unload,
+ * possibily leaving the hardware enabled.
  */
 
 /**
@@ -376,7 +508,6 @@ void drm_dev_unplug(struct drm_device *dev)
 	synchronize_srcu(&drm_unplug_srcu);
 
 	drm_dev_unregister(dev);
-	drm_dev_put(dev);
 }
 EXPORT_SYMBOL(drm_dev_unplug);
 
@@ -453,6 +584,31 @@ static void drm_fs_inode_free(struct inode *inode)
 }
 
 /**
+ * DOC: component helper usage recommendations
+ *
+ * DRM drivers that drive hardware where a logical device consists of a pile of
+ * independent hardware blocks are recommended to use the :ref:`component helper
+ * library<component>`. For consistency and better options for code reuse the
+ * following guidelines apply:
+ *
+ *  - The entire device initialization procedure should be run from the
+ *    &component_master_ops.master_bind callback, starting with drm_dev_init(),
+ *    then binding all components with component_bind_all() and finishing with
+ *    drm_dev_register().
+ *
+ *  - The opaque pointer passed to all components through component_bind_all()
+ *    should point at &struct drm_device of the device instance, not some driver
+ *    specific private structure.
+ *
+ *  - The component helper fills the niche where further standardization of
+ *    interfaces is not practical. When there already is, or will be, a
+ *    standardized interface like &drm_bridge or &drm_panel, providing its own
+ *    functions to find such components at driver load time, like
+ *    drm_of_find_panel_or_bridge(), then the component helper should not be
+ *    used.
+ */
+
+/**
  * drm_dev_init - Initialise new DRM device
  * @dev: DRM device
  * @driver: DRM driver
@@ -497,26 +653,22 @@ int drm_dev_init(struct drm_device *dev,
 	BUG_ON(!parent);
 
 	kref_init(&dev->ref);
-	dev->dev = parent;
+	dev->dev = get_device(parent);
 	dev->driver = driver;
 
 	/* no per-device feature limits by default */
 	dev->driver_features = ~0u;
 
+	drm_legacy_init_members(dev);
 	INIT_LIST_HEAD(&dev->filelist);
 	INIT_LIST_HEAD(&dev->filelist_internal);
 	INIT_LIST_HEAD(&dev->clientlist);
-	INIT_LIST_HEAD(&dev->ctxlist);
-	INIT_LIST_HEAD(&dev->vmalist);
-	INIT_LIST_HEAD(&dev->maplist);
 	INIT_LIST_HEAD(&dev->vblank_event_list);
 
-	spin_lock_init(&dev->buf_lock);
 	spin_lock_init(&dev->event_lock);
 	mutex_init(&dev->struct_mutex);
 	mutex_init(&dev->filelist_mutex);
 	mutex_init(&dev->clientlist_mutex);
-	mutex_init(&dev->ctxlist_mutex);
 	mutex_init(&dev->master_mutex);
 
 	dev->anon_inode = drm_fs_inode_new();
@@ -536,7 +688,7 @@ int drm_dev_init(struct drm_device *dev,
 	if (ret)
 		goto err_minors;
 
-	ret = drm_ht_create(&dev->map_hash, 12);
+	ret = drm_legacy_create_map_hash(dev);
 	if (ret)
 		goto err_minors;
 
@@ -561,21 +713,61 @@ err_setunique:
 		drm_gem_destroy(dev);
 err_ctxbitmap:
 	drm_legacy_ctxbitmap_cleanup(dev);
-	drm_ht_remove(&dev->map_hash);
+	drm_legacy_remove_map_hash(dev);
 err_minors:
 	drm_minor_free(dev, DRM_MINOR_PRIMARY);
 	drm_minor_free(dev, DRM_MINOR_RENDER);
 	drm_fs_inode_free(dev->anon_inode);
 err_free:
+	put_device(dev->dev);
 	mutex_destroy(&dev->master_mutex);
-	mutex_destroy(&dev->ctxlist_mutex);
 	mutex_destroy(&dev->clientlist_mutex);
 	mutex_destroy(&dev->filelist_mutex);
 	mutex_destroy(&dev->struct_mutex);
+	drm_legacy_destroy_members(dev);
 	return ret;
 }
 EXPORT_SYMBOL(drm_dev_init);
 
+static void devm_drm_dev_init_release(void *data)
+{
+	drm_dev_put(data);
+}
+
+/**
+ * devm_drm_dev_init - Resource managed drm_dev_init()
+ * @parent: Parent device object
+ * @dev: DRM device
+ * @driver: DRM driver
+ *
+ * Managed drm_dev_init(). The DRM device initialized with this function is
+ * automatically put on driver detach using drm_dev_put(). You must supply a
+ * &drm_driver.release callback to control the finalization explicitly.
+ *
+ * RETURNS:
+ * 0 on success, or error code on failure.
+ */
+int devm_drm_dev_init(struct device *parent,
+		      struct drm_device *dev,
+		      struct drm_driver *driver)
+{
+	int ret;
+
+	if (WARN_ON(!parent || !driver->release))
+		return -EINVAL;
+
+	ret = drm_dev_init(dev, driver, parent);
+	if (ret)
+		return ret;
+
+	ret = devm_add_action(parent, devm_drm_dev_init_release, dev);
+	if (ret)
+		devm_drm_dev_init_release(dev);
+
+	return ret;
+}
+EXPORT_SYMBOL(devm_drm_dev_init);
+
 /**
  * drm_dev_fini - Finalize a dead DRM device
  * @dev: DRM device
@@ -596,17 +788,19 @@ void drm_dev_fini(struct drm_device *dev)
 		drm_gem_destroy(dev);
 
 	drm_legacy_ctxbitmap_cleanup(dev);
-	drm_ht_remove(&dev->map_hash);
+	drm_legacy_remove_map_hash(dev);
 	drm_fs_inode_free(dev->anon_inode);
 
 	drm_minor_free(dev, DRM_MINOR_PRIMARY);
 	drm_minor_free(dev, DRM_MINOR_RENDER);
 
+	put_device(dev->dev);
+
 	mutex_destroy(&dev->master_mutex);
-	mutex_destroy(&dev->ctxlist_mutex);
 	mutex_destroy(&dev->clientlist_mutex);
 	mutex_destroy(&dev->filelist_mutex);
 	mutex_destroy(&dev->struct_mutex);
+	drm_legacy_destroy_members(dev);
 	kfree(dev->unique);
 }
 EXPORT_SYMBOL(drm_dev_fini);
@@ -840,8 +1034,6 @@ EXPORT_SYMBOL(drm_dev_register);
  */
 void drm_dev_unregister(struct drm_device *dev)
 {
-	struct drm_map_list *r_list, *list_temp;
-
 	if (drm_core_check_feature(dev, DRIVER_LEGACY))
 		drm_lastclose(dev);
 
@@ -858,8 +1050,7 @@ void drm_dev_unregister(struct drm_device *dev)
 	if (dev->agp)
 		drm_pci_agp_destroy(dev);
 
-	list_for_each_entry_safe(r_list, list_temp, &dev->maplist, head)
-		drm_legacy_rmmap(dev, r_list->map);
+	drm_legacy_rmmaps(dev);
 
 	remove_compat_control_link(dev);
 	drm_minor_unregister(dev, DRM_MINOR_PRIMARY);
diff --git a/drivers/gpu/drm/drm_dsc.c b/drivers/gpu/drm/drm_dsc.c
index bce99f95c1a3..77f4e5ae4197 100644
--- a/drivers/gpu/drm/drm_dsc.c
+++ b/drivers/gpu/drm/drm_dsc.c
@@ -11,6 +11,7 @@
 #include <linux/init.h>
 #include <linux/errno.h>
 #include <linux/byteorder/generic.h>
+#include <drm/drm_print.h>
 #include <drm/drm_dp_helper.h>
 #include <drm/drm_dsc.h>
 
@@ -31,75 +32,74 @@
 /**
  * drm_dsc_dp_pps_header_init() - Initializes the PPS Header
  * for DisplayPort as per the DP 1.4 spec.
- * @pps_sdp: Secondary data packet for DSC Picture Parameter Set
- *           as defined in &struct drm_dsc_pps_infoframe
+ * @pps_header: Secondary data packet header for DSC Picture
+ *              Parameter Set as defined in &struct dp_sdp_header
  *
  * DP 1.4 spec defines the secondary data packet for sending the
  * picture parameter infoframes from the source to the sink.
- * This function populates the pps header defined in
- * &struct drm_dsc_pps_infoframe as per the header bytes defined
- * in &struct dp_sdp_header.
+ * This function populates the SDP header defined in
+ * &struct dp_sdp_header.
  */
-void drm_dsc_dp_pps_header_init(struct drm_dsc_pps_infoframe *pps_sdp)
+void drm_dsc_dp_pps_header_init(struct dp_sdp_header *pps_header)
 {
-	memset(&pps_sdp->pps_header, 0, sizeof(pps_sdp->pps_header));
+	memset(pps_header, 0, sizeof(*pps_header));
 
-	pps_sdp->pps_header.HB1 = DP_SDP_PPS;
-	pps_sdp->pps_header.HB2 = DP_SDP_PPS_HEADER_PAYLOAD_BYTES_MINUS_1;
+	pps_header->HB1 = DP_SDP_PPS;
+	pps_header->HB2 = DP_SDP_PPS_HEADER_PAYLOAD_BYTES_MINUS_1;
 }
 EXPORT_SYMBOL(drm_dsc_dp_pps_header_init);
 
 /**
- * drm_dsc_pps_infoframe_pack() - Populates the DSC PPS infoframe
+ * drm_dsc_pps_payload_pack() - Populates the DSC PPS
  *
- * @pps_sdp:
- * Secondary data packet for DSC Picture Parameter Set. This is defined
- * by &struct drm_dsc_pps_infoframe
+ * @pps_payload:
+ * Bitwise struct for DSC Picture Parameter Set. This is defined
+ * by &struct drm_dsc_picture_parameter_set
  * @dsc_cfg:
  * DSC Configuration data filled by driver as defined by
  * &struct drm_dsc_config
  *
- * DSC source device sends a secondary data packet filled with all the
- * picture parameter set (PPS) information required by the sink to decode
- * the compressed frame. Driver populates the dsC PPS infoframe using the DSC
- * configuration parameters in the order expected by the DSC Display Sink
- * device. For the DSC, the sink device expects the PPS payload in the big
- * endian format for the fields that span more than 1 byte.
+ * DSC source device sends a picture parameter set (PPS) containing the
+ * information required by the sink to decode the compressed frame. Driver
+ * populates the DSC PPS struct using the DSC configuration parameters in
+ * the order expected by the DSC Display Sink device. For the DSC, the sink
+ * device expects the PPS payload in big endian format for fields
+ * that span more than 1 byte.
  */
-void drm_dsc_pps_infoframe_pack(struct drm_dsc_pps_infoframe *pps_sdp,
+void drm_dsc_pps_payload_pack(struct drm_dsc_picture_parameter_set *pps_payload,
 				const struct drm_dsc_config *dsc_cfg)
 {
 	int i;
 
 	/* Protect against someone accidently changing struct size */
-	BUILD_BUG_ON(sizeof(pps_sdp->pps_payload) !=
+	BUILD_BUG_ON(sizeof(*pps_payload) !=
 		     DP_SDP_PPS_HEADER_PAYLOAD_BYTES_MINUS_1 + 1);
 
-	memset(&pps_sdp->pps_payload, 0, sizeof(pps_sdp->pps_payload));
+	memset(pps_payload, 0, sizeof(*pps_payload));
 
 	/* PPS 0 */
-	pps_sdp->pps_payload.dsc_version =
+	pps_payload->dsc_version =
 		dsc_cfg->dsc_version_minor |
 		dsc_cfg->dsc_version_major << DSC_PPS_VERSION_MAJOR_SHIFT;
 
 	/* PPS 1, 2 is 0 */
 
 	/* PPS 3 */
-	pps_sdp->pps_payload.pps_3 =
+	pps_payload->pps_3 =
 		dsc_cfg->line_buf_depth |
 		dsc_cfg->bits_per_component << DSC_PPS_BPC_SHIFT;
 
 	/* PPS 4 */
-	pps_sdp->pps_payload.pps_4 =
+	pps_payload->pps_4 =
 		((dsc_cfg->bits_per_pixel & DSC_PPS_BPP_HIGH_MASK) >>
 		 DSC_PPS_MSB_SHIFT) |
 		dsc_cfg->vbr_enable << DSC_PPS_VBR_EN_SHIFT |
-		dsc_cfg->enable422 << DSC_PPS_SIMPLE422_SHIFT |
+		dsc_cfg->simple_422 << DSC_PPS_SIMPLE422_SHIFT |
 		dsc_cfg->convert_rgb << DSC_PPS_CONVERT_RGB_SHIFT |
 		dsc_cfg->block_pred_enable << DSC_PPS_BLOCK_PRED_EN_SHIFT;
 
 	/* PPS 5 */
-	pps_sdp->pps_payload.bits_per_pixel_low =
+	pps_payload->bits_per_pixel_low =
 		(dsc_cfg->bits_per_pixel & DSC_PPS_LSB_MASK);
 
 	/*
@@ -110,103 +110,103 @@ void drm_dsc_pps_infoframe_pack(struct drm_dsc_pps_infoframe *pps_sdp,
 	 */
 
 	/* PPS 6, 7 */
-	pps_sdp->pps_payload.pic_height = cpu_to_be16(dsc_cfg->pic_height);
+	pps_payload->pic_height = cpu_to_be16(dsc_cfg->pic_height);
 
 	/* PPS 8, 9 */
-	pps_sdp->pps_payload.pic_width = cpu_to_be16(dsc_cfg->pic_width);
+	pps_payload->pic_width = cpu_to_be16(dsc_cfg->pic_width);
 
 	/* PPS 10, 11 */
-	pps_sdp->pps_payload.slice_height = cpu_to_be16(dsc_cfg->slice_height);
+	pps_payload->slice_height = cpu_to_be16(dsc_cfg->slice_height);
 
 	/* PPS 12, 13 */
-	pps_sdp->pps_payload.slice_width = cpu_to_be16(dsc_cfg->slice_width);
+	pps_payload->slice_width = cpu_to_be16(dsc_cfg->slice_width);
 
 	/* PPS 14, 15 */
-	pps_sdp->pps_payload.chunk_size = cpu_to_be16(dsc_cfg->slice_chunk_size);
+	pps_payload->chunk_size = cpu_to_be16(dsc_cfg->slice_chunk_size);
 
 	/* PPS 16 */
-	pps_sdp->pps_payload.initial_xmit_delay_high =
+	pps_payload->initial_xmit_delay_high =
 		((dsc_cfg->initial_xmit_delay &
 		  DSC_PPS_INIT_XMIT_DELAY_HIGH_MASK) >>
 		 DSC_PPS_MSB_SHIFT);
 
 	/* PPS 17 */
-	pps_sdp->pps_payload.initial_xmit_delay_low =
+	pps_payload->initial_xmit_delay_low =
 		(dsc_cfg->initial_xmit_delay & DSC_PPS_LSB_MASK);
 
 	/* PPS 18, 19 */
-	pps_sdp->pps_payload.initial_dec_delay =
+	pps_payload->initial_dec_delay =
 		cpu_to_be16(dsc_cfg->initial_dec_delay);
 
 	/* PPS 20 is 0 */
 
 	/* PPS 21 */
-	pps_sdp->pps_payload.initial_scale_value =
+	pps_payload->initial_scale_value =
 		dsc_cfg->initial_scale_value;
 
 	/* PPS 22, 23 */
-	pps_sdp->pps_payload.scale_increment_interval =
+	pps_payload->scale_increment_interval =
 		cpu_to_be16(dsc_cfg->scale_increment_interval);
 
 	/* PPS 24 */
-	pps_sdp->pps_payload.scale_decrement_interval_high =
+	pps_payload->scale_decrement_interval_high =
 		((dsc_cfg->scale_decrement_interval &
 		  DSC_PPS_SCALE_DEC_INT_HIGH_MASK) >>
 		 DSC_PPS_MSB_SHIFT);
 
 	/* PPS 25 */
-	pps_sdp->pps_payload.scale_decrement_interval_low =
+	pps_payload->scale_decrement_interval_low =
 		(dsc_cfg->scale_decrement_interval & DSC_PPS_LSB_MASK);
 
 	/* PPS 26[7:0], PPS 27[7:5] RESERVED */
 
 	/* PPS 27 */
-	pps_sdp->pps_payload.first_line_bpg_offset =
+	pps_payload->first_line_bpg_offset =
 		dsc_cfg->first_line_bpg_offset;
 
 	/* PPS 28, 29 */
-	pps_sdp->pps_payload.nfl_bpg_offset =
+	pps_payload->nfl_bpg_offset =
 		cpu_to_be16(dsc_cfg->nfl_bpg_offset);
 
 	/* PPS 30, 31 */
-	pps_sdp->pps_payload.slice_bpg_offset =
+	pps_payload->slice_bpg_offset =
 		cpu_to_be16(dsc_cfg->slice_bpg_offset);
 
 	/* PPS 32, 33 */
-	pps_sdp->pps_payload.initial_offset =
+	pps_payload->initial_offset =
 		cpu_to_be16(dsc_cfg->initial_offset);
 
 	/* PPS 34, 35 */
-	pps_sdp->pps_payload.final_offset = cpu_to_be16(dsc_cfg->final_offset);
+	pps_payload->final_offset = cpu_to_be16(dsc_cfg->final_offset);
 
 	/* PPS 36 */
-	pps_sdp->pps_payload.flatness_min_qp = dsc_cfg->flatness_min_qp;
+	pps_payload->flatness_min_qp = dsc_cfg->flatness_min_qp;
 
 	/* PPS 37 */
-	pps_sdp->pps_payload.flatness_max_qp = dsc_cfg->flatness_max_qp;
+	pps_payload->flatness_max_qp = dsc_cfg->flatness_max_qp;
 
 	/* PPS 38, 39 */
-	pps_sdp->pps_payload.rc_model_size =
+	pps_payload->rc_model_size =
 		cpu_to_be16(DSC_RC_MODEL_SIZE_CONST);
 
 	/* PPS 40 */
-	pps_sdp->pps_payload.rc_edge_factor = DSC_RC_EDGE_FACTOR_CONST;
+	pps_payload->rc_edge_factor = DSC_RC_EDGE_FACTOR_CONST;
 
 	/* PPS 41 */
-	pps_sdp->pps_payload.rc_quant_incr_limit0 =
+	pps_payload->rc_quant_incr_limit0 =
 		dsc_cfg->rc_quant_incr_limit0;
 
 	/* PPS 42 */
-	pps_sdp->pps_payload.rc_quant_incr_limit1 =
+	pps_payload->rc_quant_incr_limit1 =
 		dsc_cfg->rc_quant_incr_limit1;
 
 	/* PPS 43 */
-	pps_sdp->pps_payload.rc_tgt_offset = DSC_RC_TGT_OFFSET_LO_CONST |
+	pps_payload->rc_tgt_offset = DSC_RC_TGT_OFFSET_LO_CONST |
 		DSC_RC_TGT_OFFSET_HI_CONST << DSC_PPS_RC_TGT_OFFSET_HI_SHIFT;
 
 	/* PPS 44 - 57 */
 	for (i = 0; i < DSC_NUM_BUF_RANGES - 1; i++)
-		pps_sdp->pps_payload.rc_buf_thresh[i] =
+		pps_payload->rc_buf_thresh[i] =
 			dsc_cfg->rc_buf_thresh[i];
 
 	/* PPS 58 - 87 */
@@ -215,32 +215,181 @@ void drm_dsc_pps_infoframe_pack(struct drm_dsc_pps_infoframe *pps_sdp,
 	 * are as follows: Min_qp[15:11], max_qp[10:6], offset[5:0]
 	 */
 	for (i = 0; i < DSC_NUM_BUF_RANGES; i++) {
-		pps_sdp->pps_payload.rc_range_parameters[i] =
+		pps_payload->rc_range_parameters[i] =
 			((dsc_cfg->rc_range_params[i].range_min_qp <<
 			  DSC_PPS_RC_RANGE_MINQP_SHIFT) |
 			 (dsc_cfg->rc_range_params[i].range_max_qp <<
 			  DSC_PPS_RC_RANGE_MAXQP_SHIFT) |
 			 (dsc_cfg->rc_range_params[i].range_bpg_offset));
-		pps_sdp->pps_payload.rc_range_parameters[i] =
-			cpu_to_be16(pps_sdp->pps_payload.rc_range_parameters[i]);
+		pps_payload->rc_range_parameters[i] =
+			cpu_to_be16(pps_payload->rc_range_parameters[i]);
 	}
 
 	/* PPS 88 */
-	pps_sdp->pps_payload.native_422_420 = dsc_cfg->native_422 |
+	pps_payload->native_422_420 = dsc_cfg->native_422 |
 		dsc_cfg->native_420 << DSC_PPS_NATIVE_420_SHIFT;
 
 	/* PPS 89 */
-	pps_sdp->pps_payload.second_line_bpg_offset =
+	pps_payload->second_line_bpg_offset =
 		dsc_cfg->second_line_bpg_offset;
 
 	/* PPS 90, 91 */
-	pps_sdp->pps_payload.nsl_bpg_offset =
+	pps_payload->nsl_bpg_offset =
 		cpu_to_be16(dsc_cfg->nsl_bpg_offset);
 
 	/* PPS 92, 93 */
-	pps_sdp->pps_payload.second_line_offset_adj =
+	pps_payload->second_line_offset_adj =
 		cpu_to_be16(dsc_cfg->second_line_offset_adj);
 
 	/* PPS 94 - 127 are O */
 }
-EXPORT_SYMBOL(drm_dsc_pps_infoframe_pack);
+EXPORT_SYMBOL(drm_dsc_pps_payload_pack);
+
+/**
+ * drm_dsc_compute_rc_parameters() - Write rate control
+ * parameters to the dsc configuration defined in
+ * &struct drm_dsc_config in accordance with the DSC 1.2
+ * specification. Some configuration fields must be present
+ * beforehand.
+ *
+ * @vdsc_cfg:
+ * DSC Configuration data partially filled by driver
+ */
+int drm_dsc_compute_rc_parameters(struct drm_dsc_config *vdsc_cfg)
+{
+	unsigned long groups_per_line = 0;
+	unsigned long groups_total = 0;
+	unsigned long num_extra_mux_bits = 0;
+	unsigned long slice_bits = 0;
+	unsigned long hrd_delay = 0;
+	unsigned long final_scale = 0;
+	unsigned long rbs_min = 0;
+
+	if (vdsc_cfg->native_420 || vdsc_cfg->native_422) {
+		/* Number of groups used to code each line of a slice */
+		groups_per_line = DIV_ROUND_UP(vdsc_cfg->slice_width / 2,
+					       DSC_RC_PIXELS_PER_GROUP);
+
+		/* chunksize in Bytes */
+		vdsc_cfg->slice_chunk_size = DIV_ROUND_UP(vdsc_cfg->slice_width / 2 *
+							  vdsc_cfg->bits_per_pixel,
+							  (8 * 16));
+	} else {
+		/* Number of groups used to code each line of a slice */
+		groups_per_line = DIV_ROUND_UP(vdsc_cfg->slice_width,
+					       DSC_RC_PIXELS_PER_GROUP);
+
+		/* chunksize in Bytes */
+		vdsc_cfg->slice_chunk_size = DIV_ROUND_UP(vdsc_cfg->slice_width *
+							  vdsc_cfg->bits_per_pixel,
+							  (8 * 16));
+	}
+
+	if (vdsc_cfg->convert_rgb)
+		num_extra_mux_bits = 3 * (vdsc_cfg->mux_word_size +
+					  (4 * vdsc_cfg->bits_per_component + 4)
+					  - 2);
+	else if (vdsc_cfg->native_422)
+		num_extra_mux_bits = 4 * vdsc_cfg->mux_word_size +
+			(4 * vdsc_cfg->bits_per_component + 4) +
+			3 * (4 * vdsc_cfg->bits_per_component) - 2;
+	else
+		num_extra_mux_bits = 3 * vdsc_cfg->mux_word_size +
+			(4 * vdsc_cfg->bits_per_component + 4) +
+			2 * (4 * vdsc_cfg->bits_per_component) - 2;
+	/* Number of bits in one Slice */
+	slice_bits = 8 * vdsc_cfg->slice_chunk_size * vdsc_cfg->slice_height;
+
+	while ((num_extra_mux_bits > 0) &&
+	       ((slice_bits - num_extra_mux_bits) % vdsc_cfg->mux_word_size))
+		num_extra_mux_bits--;
+
+	if (groups_per_line < vdsc_cfg->initial_scale_value - 8)
+		vdsc_cfg->initial_scale_value = groups_per_line + 8;
+
+	/* scale_decrement_interval calculation according to DSC spec 1.11 */
+	if (vdsc_cfg->initial_scale_value > 8)
+		vdsc_cfg->scale_decrement_interval = groups_per_line /
+			(vdsc_cfg->initial_scale_value - 8);
+	else
+		vdsc_cfg->scale_decrement_interval = DSC_SCALE_DECREMENT_INTERVAL_MAX;
+
+	vdsc_cfg->final_offset = vdsc_cfg->rc_model_size -
+		(vdsc_cfg->initial_xmit_delay *
+		 vdsc_cfg->bits_per_pixel + 8) / 16 + num_extra_mux_bits;
+
+	if (vdsc_cfg->final_offset >= vdsc_cfg->rc_model_size) {
+		DRM_DEBUG_KMS("FinalOfs < RcModelSze for this InitialXmitDelay\n");
+		return -ERANGE;
+	}
+
+	final_scale = (vdsc_cfg->rc_model_size * 8) /
+		(vdsc_cfg->rc_model_size - vdsc_cfg->final_offset);
+	if (vdsc_cfg->slice_height > 1)
+		/*
+		 * NflBpgOffset is 16 bit value with 11 fractional bits
+		 * hence we multiply by 2^11 for preserving the
+		 * fractional part
+		 */
+		vdsc_cfg->nfl_bpg_offset = DIV_ROUND_UP((vdsc_cfg->first_line_bpg_offset << 11),
+							(vdsc_cfg->slice_height - 1));
+	else
+		vdsc_cfg->nfl_bpg_offset = 0;
+
+	/* 2^16 - 1 */
+	if (vdsc_cfg->nfl_bpg_offset > 65535) {
+		DRM_DEBUG_KMS("NflBpgOffset is too large for this slice height\n");
+		return -ERANGE;
+	}
+
+	/* Number of groups used to code the entire slice */
+	groups_total = groups_per_line * vdsc_cfg->slice_height;
+
+	/* slice_bpg_offset is 16 bit value with 11 fractional bits */
+	vdsc_cfg->slice_bpg_offset = DIV_ROUND_UP(((vdsc_cfg->rc_model_size -
+						    vdsc_cfg->initial_offset +
+						    num_extra_mux_bits) << 11),
+						  groups_total);
+
+	if (final_scale > 9) {
+		/*
+		 * ScaleIncrementInterval =
+		 * finaloffset/((NflBpgOffset + SliceBpgOffset)*8(finalscale - 1.125))
+		 * as (NflBpgOffset + SliceBpgOffset) has 11 bit fractional value,
+		 * we need divide by 2^11 from pstDscCfg values
+		 */
+		vdsc_cfg->scale_increment_interval =
+				(vdsc_cfg->final_offset * (1 << 11)) /
+				((vdsc_cfg->nfl_bpg_offset +
+				vdsc_cfg->slice_bpg_offset) *
+				(final_scale - 9));
+	} else {
+		/*
+		 * If finalScaleValue is less than or equal to 9, a value of 0 should
+		 * be used to disable the scale increment at the end of the slice
+		 */
+		vdsc_cfg->scale_increment_interval = 0;
+	}
+
+	if (vdsc_cfg->scale_increment_interval > 65535) {
+		DRM_DEBUG_KMS("ScaleIncrementInterval is large for slice height\n");
+		return -ERANGE;
+	}
+
+	/*
+	 * DSC spec mentions that bits_per_pixel specifies the target
+	 * bits/pixel (bpp) rate that is used by the encoder,
+	 * in steps of 1/16 of a bit per pixel
+	 */
+	rbs_min = vdsc_cfg->rc_model_size - vdsc_cfg->initial_offset +
+		DIV_ROUND_UP(vdsc_cfg->initial_xmit_delay *
+			     vdsc_cfg->bits_per_pixel, 16) +
+		groups_per_line * vdsc_cfg->first_line_bpg_offset;
+
+	hrd_delay = DIV_ROUND_UP((rbs_min * 16), vdsc_cfg->bits_per_pixel);
+	vdsc_cfg->rc_bits = (hrd_delay * vdsc_cfg->bits_per_pixel) / 16;
+	vdsc_cfg->initial_dec_delay = hrd_delay - vdsc_cfg->initial_xmit_delay;
+
+	return 0;
+}
+EXPORT_SYMBOL(drm_dsc_compute_rc_parameters);
diff --git a/drivers/gpu/drm/drm_edid.c b/drivers/gpu/drm/drm_edid.c
index 990b1909f9d7..649cfd8b4200 100644
--- a/drivers/gpu/drm/drm_edid.c
+++ b/drivers/gpu/drm/drm_edid.c
@@ -68,8 +68,6 @@
  * maximum size and use that.
  */
 #define EDID_QUIRK_DETAILED_USE_MAXIMUM_SIZE	(1 << 4)
-/* Monitor forgot to set the first detailed is preferred bit. */
-#define EDID_QUIRK_FIRST_DETAILED_PREFERRED	(1 << 5)
 /* use +hsync +vsync for detailed mode */
 #define EDID_QUIRK_DETAILED_SYNC_PP		(1 << 6)
 /* Force reduced-blanking timings for detailed modes */
@@ -107,8 +105,6 @@ static const struct edid_quirk {
 	{ "ACR", 44358, EDID_QUIRK_PREFER_LARGE_60 },
 	/* Acer F51 */
 	{ "API", 0x7602, EDID_QUIRK_PREFER_LARGE_60 },
-	/* Unknown Acer */
-	{ "ACR", 2423, EDID_QUIRK_FIRST_DETAILED_PREFERRED },
 
 	/* AEO model 0 reports 8 bpc, but is a 6 bpc panel */
 	{ "AEO", 0, EDID_QUIRK_FORCE_6BPC },
@@ -145,12 +141,6 @@ static const struct edid_quirk {
 	{ "LPL", 0, EDID_QUIRK_DETAILED_USE_MAXIMUM_SIZE },
 	{ "LPL", 0x2a00, EDID_QUIRK_DETAILED_USE_MAXIMUM_SIZE },
 
-	/* Philips 107p5 CRT */
-	{ "PHL", 57364, EDID_QUIRK_FIRST_DETAILED_PREFERRED },
-
-	/* Proview AY765C */
-	{ "PTS", 765, EDID_QUIRK_FIRST_DETAILED_PREFERRED },
-
 	/* Samsung SyncMaster 205BW.  Note: irony */
 	{ "SAM", 541, EDID_QUIRK_DETAILED_SYNC_PP },
 	/* Samsung SyncMaster 22[5-6]BW */
@@ -172,6 +162,25 @@ static const struct edid_quirk {
 	/* Rotel RSX-1058 forwards sink's EDID but only does HDMI 1.1*/
 	{ "ETR", 13896, EDID_QUIRK_FORCE_8BPC },
 
+	/* Valve Index Headset */
+	{ "VLV", 0x91a8, EDID_QUIRK_NON_DESKTOP },
+	{ "VLV", 0x91b0, EDID_QUIRK_NON_DESKTOP },
+	{ "VLV", 0x91b1, EDID_QUIRK_NON_DESKTOP },
+	{ "VLV", 0x91b2, EDID_QUIRK_NON_DESKTOP },
+	{ "VLV", 0x91b3, EDID_QUIRK_NON_DESKTOP },
+	{ "VLV", 0x91b4, EDID_QUIRK_NON_DESKTOP },
+	{ "VLV", 0x91b5, EDID_QUIRK_NON_DESKTOP },
+	{ "VLV", 0x91b6, EDID_QUIRK_NON_DESKTOP },
+	{ "VLV", 0x91b7, EDID_QUIRK_NON_DESKTOP },
+	{ "VLV", 0x91b8, EDID_QUIRK_NON_DESKTOP },
+	{ "VLV", 0x91b9, EDID_QUIRK_NON_DESKTOP },
+	{ "VLV", 0x91ba, EDID_QUIRK_NON_DESKTOP },
+	{ "VLV", 0x91bb, EDID_QUIRK_NON_DESKTOP },
+	{ "VLV", 0x91bc, EDID_QUIRK_NON_DESKTOP },
+	{ "VLV", 0x91bd, EDID_QUIRK_NON_DESKTOP },
+	{ "VLV", 0x91be, EDID_QUIRK_NON_DESKTOP },
+	{ "VLV", 0x91bf, EDID_QUIRK_NON_DESKTOP },
+
 	/* HTC Vive and Vive Pro VR Headsets */
 	{ "HVR", 0xaa01, EDID_QUIRK_NON_DESKTOP },
 	{ "HVR", 0xaa02, EDID_QUIRK_NON_DESKTOP },
@@ -193,6 +202,12 @@ static const struct edid_quirk {
 
 	/* Sony PlayStation VR Headset */
 	{ "SNY", 0x0704, EDID_QUIRK_NON_DESKTOP },
+
+	/* Sensics VR Headsets */
+	{ "SEN", 0x1019, EDID_QUIRK_NON_DESKTOP },
+
+	/* OSVR HDK and HDK2 VR Headsets */
+	{ "SVR", 0x1019, EDID_QUIRK_NON_DESKTOP },
 };
 
 /*
@@ -4924,6 +4939,76 @@ drm_hdmi_avi_infoframe_from_display_mode(struct hdmi_avi_infoframe *frame,
 }
 EXPORT_SYMBOL(drm_hdmi_avi_infoframe_from_display_mode);
 
+/* HDMI Colorspace Spec Definitions */
+#define FULL_COLORIMETRY_MASK		0x1FF
+#define NORMAL_COLORIMETRY_MASK		0x3
+#define EXTENDED_COLORIMETRY_MASK	0x7
+#define EXTENDED_ACE_COLORIMETRY_MASK	0xF
+
+#define C(x) ((x) << 0)
+#define EC(x) ((x) << 2)
+#define ACE(x) ((x) << 5)
+
+#define HDMI_COLORIMETRY_NO_DATA		0x0
+#define HDMI_COLORIMETRY_SMPTE_170M_YCC		(C(1) | EC(0) | ACE(0))
+#define HDMI_COLORIMETRY_BT709_YCC		(C(2) | EC(0) | ACE(0))
+#define HDMI_COLORIMETRY_XVYCC_601		(C(3) | EC(0) | ACE(0))
+#define HDMI_COLORIMETRY_XVYCC_709		(C(3) | EC(1) | ACE(0))
+#define HDMI_COLORIMETRY_SYCC_601		(C(3) | EC(2) | ACE(0))
+#define HDMI_COLORIMETRY_OPYCC_601		(C(3) | EC(3) | ACE(0))
+#define HDMI_COLORIMETRY_OPRGB			(C(3) | EC(4) | ACE(0))
+#define HDMI_COLORIMETRY_BT2020_CYCC		(C(3) | EC(5) | ACE(0))
+#define HDMI_COLORIMETRY_BT2020_RGB		(C(3) | EC(6) | ACE(0))
+#define HDMI_COLORIMETRY_BT2020_YCC		(C(3) | EC(6) | ACE(0))
+#define HDMI_COLORIMETRY_DCI_P3_RGB_D65		(C(3) | EC(7) | ACE(0))
+#define HDMI_COLORIMETRY_DCI_P3_RGB_THEATER	(C(3) | EC(7) | ACE(1))
+
+static const u32 hdmi_colorimetry_val[] = {
+	[DRM_MODE_COLORIMETRY_NO_DATA] = HDMI_COLORIMETRY_NO_DATA,
+	[DRM_MODE_COLORIMETRY_SMPTE_170M_YCC] = HDMI_COLORIMETRY_SMPTE_170M_YCC,
+	[DRM_MODE_COLORIMETRY_BT709_YCC] = HDMI_COLORIMETRY_BT709_YCC,
+	[DRM_MODE_COLORIMETRY_XVYCC_601] = HDMI_COLORIMETRY_XVYCC_601,
+	[DRM_MODE_COLORIMETRY_XVYCC_709] = HDMI_COLORIMETRY_XVYCC_709,
+	[DRM_MODE_COLORIMETRY_SYCC_601] = HDMI_COLORIMETRY_SYCC_601,
+	[DRM_MODE_COLORIMETRY_OPYCC_601] = HDMI_COLORIMETRY_OPYCC_601,
+	[DRM_MODE_COLORIMETRY_OPRGB] = HDMI_COLORIMETRY_OPRGB,
+	[DRM_MODE_COLORIMETRY_BT2020_CYCC] = HDMI_COLORIMETRY_BT2020_CYCC,
+	[DRM_MODE_COLORIMETRY_BT2020_RGB] = HDMI_COLORIMETRY_BT2020_RGB,
+	[DRM_MODE_COLORIMETRY_BT2020_YCC] = HDMI_COLORIMETRY_BT2020_YCC,
+};
+
+#undef C
+#undef EC
+#undef ACE
+
+/**
+ * drm_hdmi_avi_infoframe_colorspace() - fill the HDMI AVI infoframe
+ *                                       colorspace information
+ * @frame: HDMI AVI infoframe
+ * @conn_state: connector state
+ */
+void
+drm_hdmi_avi_infoframe_colorspace(struct hdmi_avi_infoframe *frame,
+				  const struct drm_connector_state *conn_state)
+{
+	u32 colorimetry_val;
+	u32 colorimetry_index = conn_state->colorspace & FULL_COLORIMETRY_MASK;
+
+	if (colorimetry_index >= ARRAY_SIZE(hdmi_colorimetry_val))
+		colorimetry_val = HDMI_COLORIMETRY_NO_DATA;
+	else
+		colorimetry_val = hdmi_colorimetry_val[colorimetry_index];
+
+	frame->colorimetry = colorimetry_val & NORMAL_COLORIMETRY_MASK;
+	/*
+	 * ToDo: Extend it for ACE formats as well. Modify the infoframe
+	 * structure and extend it in drivers/video/hdmi
+	 */
+	frame->extended_colorimetry = (colorimetry_val >> 2) &
+					EXTENDED_COLORIMETRY_MASK;
+}
+EXPORT_SYMBOL(drm_hdmi_avi_infoframe_colorspace);
+
 /**
  * drm_hdmi_avi_infoframe_quant_range() - fill the HDMI AVI infoframe
  *                                        quantization range information
diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c
index af2ab640cadb..498f95c3e81d 100644
--- a/drivers/gpu/drm/drm_fb_helper.c
+++ b/drivers/gpu/drm/drm_fb_helper.c
@@ -639,20 +639,19 @@ static struct sysrq_key_op sysrq_drm_fb_helper_restore_op = { };
 static void dpms_legacy(struct drm_fb_helper *fb_helper, int dpms_mode)
 {
 	struct drm_device *dev = fb_helper->dev;
-	struct drm_crtc *crtc;
 	struct drm_connector *connector;
+	struct drm_mode_set *modeset;
 	int i, j;
 
 	drm_modeset_lock_all(dev);
 	for (i = 0; i < fb_helper->crtc_count; i++) {
-		crtc = fb_helper->crtc_info[i].mode_set.crtc;
+		modeset = &fb_helper->crtc_info[i].mode_set;
 
-		if (!crtc->enabled)
+		if (!modeset->crtc->enabled)
 			continue;
 
-		/* Walk the connectors & encoders on this fb turning them on/off */
-		drm_fb_helper_for_each_connector(fb_helper, j) {
-			connector = fb_helper->connector_info[j]->connector;
+		for (j = 0; j < modeset->num_connectors; j++) {
+			connector = modeset->connectors[j];
 			connector->funcs->dpms(connector, dpms_mode);
 			drm_object_property_set_value(&connector->base,
 				dev->mode_config.dpms_property, dpms_mode);
@@ -934,6 +933,7 @@ struct fb_info *drm_fb_helper_alloc_fbi(struct drm_fb_helper *fb_helper)
 	}
 
 	fb_helper->fbdev = info;
+	info->skip_vt_switch = true;
 
 	return info;
 
@@ -1873,7 +1873,6 @@ static int drm_fb_helper_single_fb_probe(struct drm_fb_helper *fb_helper,
 	int crtc_count = 0;
 	int i;
 	struct drm_fb_helper_surface_size sizes;
-	int gamma_size = 0;
 	int best_depth = 0;
 
 	memset(&sizes, 0, sizeof(struct drm_fb_helper_surface_size));
@@ -1889,7 +1888,6 @@ static int drm_fb_helper_single_fb_probe(struct drm_fb_helper *fb_helper,
 	if (preferred_bpp != sizes.surface_bpp)
 		sizes.surface_depth = sizes.surface_bpp = preferred_bpp;
 
-	/* first up get a count of crtcs now in use and new min/maxes width/heights */
 	drm_fb_helper_for_each_connector(fb_helper, i) {
 		struct drm_fb_helper_connector *fb_helper_conn = fb_helper->connector_info[i];
 		struct drm_cmdline_mode *cmdline_mode;
@@ -1969,6 +1967,7 @@ static int drm_fb_helper_single_fb_probe(struct drm_fb_helper *fb_helper,
 		sizes.surface_depth = best_depth;
 	}
 
+	/* first up get a count of crtcs now in use and new min/maxes width/heights */
 	crtc_count = 0;
 	for (i = 0; i < fb_helper->crtc_count; i++) {
 		struct drm_display_mode *desired_mode;
@@ -1991,9 +1990,6 @@ static int drm_fb_helper_single_fb_probe(struct drm_fb_helper *fb_helper,
 		x = fb_helper->crtc_info[i].x;
 		y = fb_helper->crtc_info[i].y;
 
-		if (gamma_size == 0)
-			gamma_size = fb_helper->crtc_info[i].mode_set.crtc->gamma_size;
-
 		sizes.surface_width  = max_t(u32, desired_mode->hdisplay + x, sizes.surface_width);
 		sizes.surface_height = max_t(u32, desired_mode->vdisplay + y, sizes.surface_height);
 
@@ -2036,21 +2032,8 @@ static int drm_fb_helper_single_fb_probe(struct drm_fb_helper *fb_helper,
 	return 0;
 }
 
-/**
- * drm_fb_helper_fill_fix - initializes fixed fbdev information
- * @info: fbdev registered by the helper
- * @pitch: desired pitch
- * @depth: desired depth
- *
- * Helper to fill in the fixed fbdev information useful for a non-accelerated
- * fbdev emulations. Drivers which support acceleration methods which impose
- * additional constraints need to set up their own limits.
- *
- * Drivers should call this (or their equivalent setup code) from their
- * &drm_fb_helper_funcs.fb_probe callback.
- */
-void drm_fb_helper_fill_fix(struct fb_info *info, uint32_t pitch,
-			    uint32_t depth)
+static void drm_fb_helper_fill_fix(struct fb_info *info, uint32_t pitch,
+				   uint32_t depth)
 {
 	info->fix.type = FB_TYPE_PACKED_PIXELS;
 	info->fix.visual = depth == 8 ? FB_VISUAL_PSEUDOCOLOR :
@@ -2065,24 +2048,10 @@ void drm_fb_helper_fill_fix(struct fb_info *info, uint32_t pitch,
 
 	info->fix.line_length = pitch;
 }
-EXPORT_SYMBOL(drm_fb_helper_fill_fix);
 
-/**
- * drm_fb_helper_fill_var - initalizes variable fbdev information
- * @info: fbdev instance to set up
- * @fb_helper: fb helper instance to use as template
- * @fb_width: desired fb width
- * @fb_height: desired fb height
- *
- * Sets up the variable fbdev metainformation from the given fb helper instance
- * and the drm framebuffer allocated in &drm_fb_helper.fb.
- *
- * Drivers should call this (or their equivalent setup code) from their
- * &drm_fb_helper_funcs.fb_probe callback after having allocated the fbdev
- * backing storage framebuffer.
- */
-void drm_fb_helper_fill_var(struct fb_info *info, struct drm_fb_helper *fb_helper,
-			    uint32_t fb_width, uint32_t fb_height)
+static void drm_fb_helper_fill_var(struct fb_info *info,
+				   struct drm_fb_helper *fb_helper,
+				   uint32_t fb_width, uint32_t fb_height)
 {
 	struct drm_framebuffer *fb = fb_helper->fb;
 
@@ -2102,7 +2071,36 @@ void drm_fb_helper_fill_var(struct fb_info *info, struct drm_fb_helper *fb_helpe
 	info->var.xres = fb_width;
 	info->var.yres = fb_height;
 }
-EXPORT_SYMBOL(drm_fb_helper_fill_var);
+
+/**
+ * drm_fb_helper_fill_info - initializes fbdev information
+ * @info: fbdev instance to set up
+ * @fb_helper: fb helper instance to use as template
+ * @sizes: describes fbdev size and scanout surface size
+ *
+ * Sets up the variable and fixed fbdev metainformation from the given fb helper
+ * instance and the drm framebuffer allocated in &drm_fb_helper.fb.
+ *
+ * Drivers should call this (or their equivalent setup code) from their
+ * &drm_fb_helper_funcs.fb_probe callback after having allocated the fbdev
+ * backing storage framebuffer.
+ */
+void drm_fb_helper_fill_info(struct fb_info *info,
+			     struct drm_fb_helper *fb_helper,
+			     struct drm_fb_helper_surface_size *sizes)
+{
+	struct drm_framebuffer *fb = fb_helper->fb;
+
+	drm_fb_helper_fill_fix(info, fb->pitches[0], fb->format->depth);
+	drm_fb_helper_fill_var(info, fb_helper,
+			       sizes->fb_width, sizes->fb_height);
+
+	info->par = fb_helper;
+	snprintf(info->fix.id, sizeof(info->fix.id), "%sdrmfb",
+		 fb_helper->dev->driver->name);
+
+}
+EXPORT_SYMBOL(drm_fb_helper_fill_info);
 
 static int drm_fb_helper_probe_connector_modes(struct drm_fb_helper *fb_helper,
 						uint32_t maxX,
@@ -2561,6 +2559,195 @@ static void drm_setup_crtc_rotation(struct drm_fb_helper *fb_helper,
 	fb_helper->sw_rotations |= DRM_MODE_ROTATE_0;
 }
 
+static struct drm_fb_helper_crtc *
+drm_fb_helper_crtc(struct drm_fb_helper *fb_helper, struct drm_crtc *crtc)
+{
+	int i;
+
+	for (i = 0; i < fb_helper->crtc_count; i++)
+		if (fb_helper->crtc_info[i].mode_set.crtc == crtc)
+			return &fb_helper->crtc_info[i];
+
+	return NULL;
+}
+
+/* Try to read the BIOS display configuration and use it for the initial config */
+static bool drm_fb_helper_firmware_config(struct drm_fb_helper *fb_helper,
+					  struct drm_fb_helper_crtc **crtcs,
+					  struct drm_display_mode **modes,
+					  struct drm_fb_offset *offsets,
+					  bool *enabled, int width, int height)
+{
+	struct drm_device *dev = fb_helper->dev;
+	unsigned int count = min(fb_helper->connector_count, BITS_PER_LONG);
+	unsigned long conn_configured, conn_seq, mask;
+	int i, j;
+	bool *save_enabled;
+	bool fallback = true, ret = true;
+	int num_connectors_enabled = 0;
+	int num_connectors_detected = 0;
+	struct drm_modeset_acquire_ctx ctx;
+
+	if (!drm_drv_uses_atomic_modeset(dev))
+		return false;
+
+	save_enabled = kcalloc(count, sizeof(bool), GFP_KERNEL);
+	if (!save_enabled)
+		return false;
+
+	drm_modeset_acquire_init(&ctx, 0);
+
+	while (drm_modeset_lock_all_ctx(dev, &ctx) != 0)
+		drm_modeset_backoff(&ctx);
+
+	memcpy(save_enabled, enabled, count);
+	mask = GENMASK(count - 1, 0);
+	conn_configured = 0;
+retry:
+	conn_seq = conn_configured;
+	for (i = 0; i < count; i++) {
+		struct drm_fb_helper_connector *fb_conn;
+		struct drm_connector *connector;
+		struct drm_encoder *encoder;
+		struct drm_fb_helper_crtc *new_crtc;
+
+		fb_conn = fb_helper->connector_info[i];
+		connector = fb_conn->connector;
+
+		if (conn_configured & BIT(i))
+			continue;
+
+		if (conn_seq == 0 && !connector->has_tile)
+			continue;
+
+		if (connector->status == connector_status_connected)
+			num_connectors_detected++;
+
+		if (!enabled[i]) {
+			DRM_DEBUG_KMS("connector %s not enabled, skipping\n",
+				      connector->name);
+			conn_configured |= BIT(i);
+			continue;
+		}
+
+		if (connector->force == DRM_FORCE_OFF) {
+			DRM_DEBUG_KMS("connector %s is disabled by user, skipping\n",
+				      connector->name);
+			enabled[i] = false;
+			continue;
+		}
+
+		encoder = connector->state->best_encoder;
+		if (!encoder || WARN_ON(!connector->state->crtc)) {
+			if (connector->force > DRM_FORCE_OFF)
+				goto bail;
+
+			DRM_DEBUG_KMS("connector %s has no encoder or crtc, skipping\n",
+				      connector->name);
+			enabled[i] = false;
+			conn_configured |= BIT(i);
+			continue;
+		}
+
+		num_connectors_enabled++;
+
+		new_crtc = drm_fb_helper_crtc(fb_helper, connector->state->crtc);
+
+		/*
+		 * Make sure we're not trying to drive multiple connectors
+		 * with a single CRTC, since our cloning support may not
+		 * match the BIOS.
+		 */
+		for (j = 0; j < count; j++) {
+			if (crtcs[j] == new_crtc) {
+				DRM_DEBUG_KMS("fallback: cloned configuration\n");
+				goto bail;
+			}
+		}
+
+		DRM_DEBUG_KMS("looking for cmdline mode on connector %s\n",
+			      connector->name);
+
+		/* go for command line mode first */
+		modes[i] = drm_pick_cmdline_mode(fb_conn);
+
+		/* try for preferred next */
+		if (!modes[i]) {
+			DRM_DEBUG_KMS("looking for preferred mode on connector %s %d\n",
+				      connector->name, connector->has_tile);
+			modes[i] = drm_has_preferred_mode(fb_conn, width,
+							  height);
+		}
+
+		/* No preferred mode marked by the EDID? Are there any modes? */
+		if (!modes[i] && !list_empty(&connector->modes)) {
+			DRM_DEBUG_KMS("using first mode listed on connector %s\n",
+				      connector->name);
+			modes[i] = list_first_entry(&connector->modes,
+						    struct drm_display_mode,
+						    head);
+		}
+
+		/* last resort: use current mode */
+		if (!modes[i]) {
+			/*
+			 * IMPORTANT: We want to use the adjusted mode (i.e.
+			 * after the panel fitter upscaling) as the initial
+			 * config, not the input mode, which is what crtc->mode
+			 * usually contains. But since our current
+			 * code puts a mode derived from the post-pfit timings
+			 * into crtc->mode this works out correctly.
+			 *
+			 * This is crtc->mode and not crtc->state->mode for the
+			 * fastboot check to work correctly.
+			 */
+			DRM_DEBUG_KMS("looking for current mode on connector %s\n",
+				      connector->name);
+			modes[i] = &connector->state->crtc->mode;
+		}
+		crtcs[i] = new_crtc;
+
+		DRM_DEBUG_KMS("connector %s on [CRTC:%d:%s]: %dx%d%s\n",
+			      connector->name,
+			      connector->state->crtc->base.id,
+			      connector->state->crtc->name,
+			      modes[i]->hdisplay, modes[i]->vdisplay,
+			      modes[i]->flags & DRM_MODE_FLAG_INTERLACE ? "i" : "");
+
+		fallback = false;
+		conn_configured |= BIT(i);
+	}
+
+	if ((conn_configured & mask) != mask && conn_configured != conn_seq)
+		goto retry;
+
+	/*
+	 * If the BIOS didn't enable everything it could, fall back to have the
+	 * same user experiencing of lighting up as much as possible like the
+	 * fbdev helper library.
+	 */
+	if (num_connectors_enabled != num_connectors_detected &&
+	    num_connectors_enabled < dev->mode_config.num_crtc) {
+		DRM_DEBUG_KMS("fallback: Not all outputs enabled\n");
+		DRM_DEBUG_KMS("Enabled: %i, detected: %i\n", num_connectors_enabled,
+			      num_connectors_detected);
+		fallback = true;
+	}
+
+	if (fallback) {
+bail:
+		DRM_DEBUG_KMS("Not using firmware configuration\n");
+		memcpy(enabled, save_enabled, count);
+		ret = false;
+	}
+
+	drm_modeset_drop_locks(&ctx);
+	drm_modeset_acquire_fini(&ctx);
+
+	kfree(save_enabled);
+	return ret;
+}
+
 static void drm_setup_crtcs(struct drm_fb_helper *fb_helper,
 			    u32 width, u32 height)
 {
@@ -2593,10 +2780,8 @@ static void drm_setup_crtcs(struct drm_fb_helper *fb_helper,
 		DRM_DEBUG_KMS("No connectors reported connected with modes\n");
 	drm_enable_connectors(fb_helper, enabled);
 
-	if (!(fb_helper->funcs->initial_config &&
-	      fb_helper->funcs->initial_config(fb_helper, crtcs, modes,
-					       offsets,
-					       enabled, width, height))) {
+	if (!drm_fb_helper_firmware_config(fb_helper, crtcs, modes, offsets,
+					   enabled, width, height)) {
 		memset(modes, 0, fb_helper->connector_count*sizeof(modes[0]));
 		memset(crtcs, 0, fb_helper->connector_count*sizeof(crtcs[0]));
 		memset(offsets, 0, fb_helper->connector_count*sizeof(offsets[0]));
@@ -2780,9 +2965,8 @@ __drm_fb_helper_initial_config_and_unlock(struct drm_fb_helper *fb_helper,
  *
  * This function will call down into the &drm_fb_helper_funcs.fb_probe callback
  * to let the driver allocate and initialize the fbdev info structure and the
- * drm framebuffer used to back the fbdev. drm_fb_helper_fill_var() and
- * drm_fb_helper_fill_fix() are provided as helpers to setup simple default
- * values for the fbdev info structure.
+ * drm framebuffer used to back the fbdev. drm_fb_helper_fill_info() is provided
+ * as a helper to setup simple default values for the fbdev info structure.
  *
  * HANG DEBUGGING:
  *
@@ -3024,7 +3208,8 @@ static int drm_fbdev_fb_open(struct fb_info *info, int user)
 {
 	struct drm_fb_helper *fb_helper = info->par;
 
-	if (!try_module_get(fb_helper->dev->driver->fops->owner))
+	/* No need to take a ref for fbcon because it unbinds on unregister */
+	if (user && !try_module_get(fb_helper->dev->driver->fops->owner))
 		return -ENODEV;
 
 	return 0;
@@ -3034,7 +3219,8 @@ static int drm_fbdev_fb_release(struct fb_info *info, int user)
 {
 	struct drm_fb_helper *fb_helper = info->par;
 
-	module_put(fb_helper->dev->driver->fops->owner);
+	if (user)
+		module_put(fb_helper->dev->driver->fops->owner);
 
 	return 0;
 }
@@ -3149,7 +3335,6 @@ int drm_fb_helper_generic_probe(struct drm_fb_helper *fb_helper,
 	if (IS_ERR(fbi))
 		return PTR_ERR(fbi);
 
-	fbi->par = fb_helper;
 	fbi->fbops = &drm_fbdev_fb_ops;
 	fbi->screen_size = fb->height * fb->pitches[0];
 	fbi->fix.smem_len = fbi->screen_size;
@@ -3160,10 +3345,7 @@ int drm_fb_helper_generic_probe(struct drm_fb_helper *fb_helper,
 		fbi->fix.smem_start =
 			page_to_phys(virt_to_page(fbi->screen_buffer));
 #endif
-	strcpy(fbi->fix.id, "DRM emulated");
-
-	drm_fb_helper_fill_fix(fbi, fb->pitches[0], fb->format->depth);
-	drm_fb_helper_fill_var(fbi, fb_helper, sizes->fb_width, sizes->fb_height);
+	drm_fb_helper_fill_info(fbi, fb_helper, sizes);
 
 	if (fb->funcs->dirty) {
 		struct fb_ops *fbops;
@@ -3317,8 +3499,6 @@ int drm_fbdev_generic_setup(struct drm_device *dev, unsigned int preferred_bpp)
 		return ret;
 	}
 
-	drm_client_add(&fb_helper->client);
-
 	if (!preferred_bpp)
 		preferred_bpp = dev->mode_config.preferred_depth;
 	if (!preferred_bpp)
@@ -3329,6 +3509,8 @@ int drm_fbdev_generic_setup(struct drm_device *dev, unsigned int preferred_bpp)
 	if (ret)
 		DRM_DEV_DEBUG(dev->dev, "client hotplug ret=%d\n", ret);
 
+	drm_client_register(&fb_helper->client);
+
 	return 0;
 }
 EXPORT_SYMBOL(drm_fbdev_generic_setup);
diff --git a/drivers/gpu/drm/drm_file.c b/drivers/gpu/drm/drm_file.c
index 7caa3c7ed978..233f114d2186 100644
--- a/drivers/gpu/drm/drm_file.c
+++ b/drivers/gpu/drm/drm_file.c
@@ -128,7 +128,6 @@ struct drm_file *drm_file_alloc(struct drm_minor *minor)
 
 	/* for compatibility root is always authenticated */
 	file->authenticated = capable(CAP_SYS_ADMIN);
-	file->lock_count = 0;
 
 	INIT_LIST_HEAD(&file->lhead);
 	INIT_LIST_HEAD(&file->fbs);
@@ -425,30 +424,6 @@ static int drm_open_helper(struct file *filp, struct drm_minor *minor)
 	return 0;
 }
 
-static void drm_legacy_dev_reinit(struct drm_device *dev)
-{
-	if (dev->irq_enabled)
-		drm_irq_uninstall(dev);
-
-	mutex_lock(&dev->struct_mutex);
-
-	drm_legacy_agp_clear(dev);
-
-	drm_legacy_sg_cleanup(dev);
-	drm_legacy_vma_flush(dev);
-	drm_legacy_dma_takedown(dev);
-
-	mutex_unlock(&dev->struct_mutex);
-
-	dev->sigdata.lock = NULL;
-
-	dev->context_flag = 0;
-	dev->last_context = 0;
-	dev->if_version = 0;
-
-	DRM_DEBUG("lastclose completed\n");
-}
-
 void drm_lastclose(struct drm_device * dev)
 {
 	DRM_DEBUG("\n");
@@ -577,6 +552,7 @@ put_back_event:
 				file_priv->event_space -= length;
 				list_add(&e->link, &file_priv->event_list);
 				spin_unlock_irq(&dev->event_lock);
+				wake_up_interruptible(&file_priv->event_wait);
 				break;
 			}
 
diff --git a/drivers/gpu/drm/drm_format_helper.c b/drivers/gpu/drm/drm_format_helper.c
new file mode 100644
index 000000000000..a18da35145b7
--- /dev/null
+++ b/drivers/gpu/drm/drm_format_helper.c
@@ -0,0 +1,324 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2016 Noralf Trønnes
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/io.h>
+
+#include <drm/drm_format_helper.h>
+#include <drm/drm_framebuffer.h>
+#include <drm/drm_fourcc.h>
+#include <drm/drm_rect.h>
+
+static unsigned int clip_offset(struct drm_rect *clip,
+				unsigned int pitch, unsigned int cpp)
+{
+	return clip->y1 * pitch + clip->x1 * cpp;
+}
+
+/**
+ * drm_fb_memcpy - Copy clip buffer
+ * @dst: Destination buffer
+ * @vaddr: Source buffer
+ * @fb: DRM framebuffer
+ * @clip: Clip rectangle area to copy
+ *
+ * This function does not apply clipping on dst, i.e. the destination
+ * is a small buffer containing the clip rect only.
+ */
+void drm_fb_memcpy(void *dst, void *vaddr, struct drm_framebuffer *fb,
+		   struct drm_rect *clip)
+{
+	unsigned int cpp = drm_format_plane_cpp(fb->format->format, 0);
+	size_t len = (clip->x2 - clip->x1) * cpp;
+	unsigned int y, lines = clip->y2 - clip->y1;
+
+	vaddr += clip_offset(clip, fb->pitches[0], cpp);
+	for (y = 0; y < lines; y++) {
+		memcpy(dst, vaddr, len);
+		vaddr += fb->pitches[0];
+		dst += len;
+	}
+}
+EXPORT_SYMBOL(drm_fb_memcpy);
+
+/**
+ * drm_fb_memcpy_dstclip - Copy clip buffer
+ * @dst: Destination buffer (iomem)
+ * @vaddr: Source buffer
+ * @fb: DRM framebuffer
+ * @clip: Clip rectangle area to copy
+ *
+ * This function applies clipping on dst, i.e. the destination is a
+ * full (iomem) framebuffer but only the clip rect content is copied over.
+ */
+void drm_fb_memcpy_dstclip(void __iomem *dst, void *vaddr,
+			   struct drm_framebuffer *fb,
+			   struct drm_rect *clip)
+{
+	unsigned int cpp = drm_format_plane_cpp(fb->format->format, 0);
+	unsigned int offset = clip_offset(clip, fb->pitches[0], cpp);
+	size_t len = (clip->x2 - clip->x1) * cpp;
+	unsigned int y, lines = clip->y2 - clip->y1;
+
+	vaddr += offset;
+	dst += offset;
+	for (y = 0; y < lines; y++) {
+		memcpy_toio(dst, vaddr, len);
+		vaddr += fb->pitches[0];
+		dst += fb->pitches[0];
+	}
+}
+EXPORT_SYMBOL(drm_fb_memcpy_dstclip);
+
+/**
+ * drm_fb_swab16 - Swap bytes into clip buffer
+ * @dst: RGB565 destination buffer
+ * @vaddr: RGB565 source buffer
+ * @fb: DRM framebuffer
+ * @clip: Clip rectangle area to copy
+ */
+void drm_fb_swab16(u16 *dst, void *vaddr, struct drm_framebuffer *fb,
+		   struct drm_rect *clip)
+{
+	size_t len = (clip->x2 - clip->x1) * sizeof(u16);
+	unsigned int x, y;
+	u16 *src, *buf;
+
+	/*
+	 * The cma memory is write-combined so reads are uncached.
+	 * Speed up by fetching one line at a time.
+	 */
+	buf = kmalloc(len, GFP_KERNEL);
+	if (!buf)
+		return;
+
+	for (y = clip->y1; y < clip->y2; y++) {
+		src = vaddr + (y * fb->pitches[0]);
+		src += clip->x1;
+		memcpy(buf, src, len);
+		src = buf;
+		for (x = clip->x1; x < clip->x2; x++)
+			*dst++ = swab16(*src++);
+	}
+
+	kfree(buf);
+}
+EXPORT_SYMBOL(drm_fb_swab16);
+
+static void drm_fb_xrgb8888_to_rgb565_line(u16 *dbuf, u32 *sbuf,
+					   unsigned int pixels,
+					   bool swab)
+{
+	unsigned int x;
+	u16 val16;
+
+	for (x = 0; x < pixels; x++) {
+		val16 = ((sbuf[x] & 0x00F80000) >> 8) |
+			((sbuf[x] & 0x0000FC00) >> 5) |
+			((sbuf[x] & 0x000000F8) >> 3);
+		if (swab)
+			dbuf[x] = swab16(val16);
+		else
+			dbuf[x] = val16;
+	}
+}
+
+/**
+ * drm_fb_xrgb8888_to_rgb565 - Convert XRGB8888 to RGB565 clip buffer
+ * @dst: RGB565 destination buffer
+ * @vaddr: XRGB8888 source buffer
+ * @fb: DRM framebuffer
+ * @clip: Clip rectangle area to copy
+ * @swab: Swap bytes
+ *
+ * Drivers can use this function for RGB565 devices that don't natively
+ * support XRGB8888.
+ *
+ * This function does not apply clipping on dst, i.e. the destination
+ * is a small buffer containing the clip rect only.
+ */
+void drm_fb_xrgb8888_to_rgb565(void *dst, void *vaddr,
+			       struct drm_framebuffer *fb,
+			       struct drm_rect *clip, bool swab)
+{
+	size_t linepixels = clip->x2 - clip->x1;
+	size_t src_len = linepixels * sizeof(u32);
+	size_t dst_len = linepixels * sizeof(u16);
+	unsigned y, lines = clip->y2 - clip->y1;
+	void *sbuf;
+
+	/*
+	 * The cma memory is write-combined so reads are uncached.
+	 * Speed up by fetching one line at a time.
+	 */
+	sbuf = kmalloc(src_len, GFP_KERNEL);
+	if (!sbuf)
+		return;
+
+	vaddr += clip_offset(clip, fb->pitches[0], sizeof(u32));
+	for (y = 0; y < lines; y++) {
+		memcpy(sbuf, vaddr, src_len);
+		drm_fb_xrgb8888_to_rgb565_line(dst, sbuf, linepixels, swab);
+		vaddr += fb->pitches[0];
+		dst += dst_len;
+	}
+
+	kfree(sbuf);
+}
+EXPORT_SYMBOL(drm_fb_xrgb8888_to_rgb565);
+
+/**
+ * drm_fb_xrgb8888_to_rgb565_dstclip - Convert XRGB8888 to RGB565 clip buffer
+ * @dst: RGB565 destination buffer (iomem)
+ * @dst_pitch: destination buffer pitch
+ * @vaddr: XRGB8888 source buffer
+ * @fb: DRM framebuffer
+ * @clip: Clip rectangle area to copy
+ * @swab: Swap bytes
+ *
+ * Drivers can use this function for RGB565 devices that don't natively
+ * support XRGB8888.
+ *
+ * This function applies clipping on dst, i.e. the destination is a
+ * full (iomem) framebuffer but only the clip rect content is copied over.
+ */
+void drm_fb_xrgb8888_to_rgb565_dstclip(void __iomem *dst, unsigned int dst_pitch,
+				       void *vaddr, struct drm_framebuffer *fb,
+				       struct drm_rect *clip, bool swab)
+{
+	size_t linepixels = clip->x2 - clip->x1;
+	size_t dst_len = linepixels * sizeof(u16);
+	unsigned y, lines = clip->y2 - clip->y1;
+	void *dbuf;
+
+	dbuf = kmalloc(dst_len, GFP_KERNEL);
+	if (!dbuf)
+		return;
+
+	vaddr += clip_offset(clip, fb->pitches[0], sizeof(u32));
+	dst += clip_offset(clip, dst_pitch, sizeof(u16));
+	for (y = 0; y < lines; y++) {
+		drm_fb_xrgb8888_to_rgb565_line(dbuf, vaddr, linepixels, swab);
+		memcpy_toio(dst, dbuf, dst_len);
+		vaddr += fb->pitches[0];
+		dst += dst_len;
+	}
+
+	kfree(dbuf);
+}
+EXPORT_SYMBOL(drm_fb_xrgb8888_to_rgb565_dstclip);
+
+static void drm_fb_xrgb8888_to_rgb888_line(u8 *dbuf, u32 *sbuf,
+					   unsigned int pixels)
+{
+	unsigned int x;
+
+	for (x = 0; x < pixels; x++) {
+		*dbuf++ = (sbuf[x] & 0x000000FF) >>  0;
+		*dbuf++ = (sbuf[x] & 0x0000FF00) >>  8;
+		*dbuf++ = (sbuf[x] & 0x00FF0000) >> 16;
+	}
+}
+
+/**
+ * drm_fb_xrgb8888_to_rgb888_dstclip - Convert XRGB8888 to RGB888 clip buffer
+ * @dst: RGB565 destination buffer (iomem)
+ * @dst_pitch: destination buffer pitch
+ * @vaddr: XRGB8888 source buffer
+ * @fb: DRM framebuffer
+ * @clip: Clip rectangle area to copy
+ *
+ * Drivers can use this function for RGB888 devices that don't natively
+ * support XRGB8888.
+ *
+ * This function applies clipping on dst, i.e. the destination is a
+ * full (iomem) framebuffer but only the clip rect content is copied over.
+ */
+void drm_fb_xrgb8888_to_rgb888_dstclip(void __iomem *dst, unsigned int dst_pitch,
+				       void *vaddr, struct drm_framebuffer *fb,
+				       struct drm_rect *clip)
+{
+	size_t linepixels = clip->x2 - clip->x1;
+	size_t dst_len = linepixels * 3;
+	unsigned y, lines = clip->y2 - clip->y1;
+	void *dbuf;
+
+	dbuf = kmalloc(dst_len, GFP_KERNEL);
+	if (!dbuf)
+		return;
+
+	vaddr += clip_offset(clip, fb->pitches[0], sizeof(u32));
+	dst += clip_offset(clip, dst_pitch, sizeof(u16));
+	for (y = 0; y < lines; y++) {
+		drm_fb_xrgb8888_to_rgb888_line(dbuf, vaddr, linepixels);
+		memcpy_toio(dst, dbuf, dst_len);
+		vaddr += fb->pitches[0];
+		dst += dst_len;
+	}
+
+	kfree(dbuf);
+}
+EXPORT_SYMBOL(drm_fb_xrgb8888_to_rgb888_dstclip);
+
+/**
+ * drm_fb_xrgb8888_to_gray8 - Convert XRGB8888 to grayscale
+ * @dst: 8-bit grayscale destination buffer
+ * @vaddr: XRGB8888 source buffer
+ * @fb: DRM framebuffer
+ * @clip: Clip rectangle area to copy
+ *
+ * Drm doesn't have native monochrome or grayscale support.
+ * Such drivers can announce the commonly supported XR24 format to userspace
+ * and use this function to convert to the native format.
+ *
+ * Monochrome drivers will use the most significant bit,
+ * where 1 means foreground color and 0 background color.
+ *
+ * ITU BT.601 is used for the RGB -> luma (brightness) conversion.
+ */
+void drm_fb_xrgb8888_to_gray8(u8 *dst, void *vaddr, struct drm_framebuffer *fb,
+			       struct drm_rect *clip)
+{
+	unsigned int len = (clip->x2 - clip->x1) * sizeof(u32);
+	unsigned int x, y;
+	void *buf;
+	u32 *src;
+
+	if (WARN_ON(fb->format->format != DRM_FORMAT_XRGB8888))
+		return;
+	/*
+	 * The cma memory is write-combined so reads are uncached.
+	 * Speed up by fetching one line at a time.
+	 */
+	buf = kmalloc(len, GFP_KERNEL);
+	if (!buf)
+		return;
+
+	for (y = clip->y1; y < clip->y2; y++) {
+		src = vaddr + (y * fb->pitches[0]);
+		src += clip->x1;
+		memcpy(buf, src, len);
+		src = buf;
+		for (x = clip->x1; x < clip->x2; x++) {
+			u8 r = (*src & 0x00ff0000) >> 16;
+			u8 g = (*src & 0x0000ff00) >> 8;
+			u8 b =  *src & 0x000000ff;
+
+			/* ITU BT.601: Y = 0.299 R + 0.587 G + 0.114 B */
+			*dst++ = (3 * r + 6 * g + b) / 10;
+			src++;
+		}
+	}
+
+	kfree(buf);
+}
+EXPORT_SYMBOL(drm_fb_xrgb8888_to_gray8);
+
diff --git a/drivers/gpu/drm/drm_fourcc.c b/drivers/gpu/drm/drm_fourcc.c
index ba7e19d4336c..6ea55fb4526d 100644
--- a/drivers/gpu/drm/drm_fourcc.c
+++ b/drivers/gpu/drm/drm_fourcc.c
@@ -198,6 +198,10 @@ const struct drm_format_info *__drm_format_info(u32 format)
 		{ .format = DRM_FORMAT_ABGR8888,	.depth = 32, .num_planes = 1, .cpp = { 4, 0, 0 }, .hsub = 1, .vsub = 1, .has_alpha = true },
 		{ .format = DRM_FORMAT_RGBA8888,	.depth = 32, .num_planes = 1, .cpp = { 4, 0, 0 }, .hsub = 1, .vsub = 1, .has_alpha = true },
 		{ .format = DRM_FORMAT_BGRA8888,	.depth = 32, .num_planes = 1, .cpp = { 4, 0, 0 }, .hsub = 1, .vsub = 1, .has_alpha = true },
+		{ .format = DRM_FORMAT_XRGB16161616F,	.depth = 0,  .num_planes = 1, .cpp = { 8, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_XBGR16161616F,	.depth = 0,  .num_planes = 1, .cpp = { 8, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_ARGB16161616F,	.depth = 0,  .num_planes = 1, .cpp = { 8, 0, 0 }, .hsub = 1, .vsub = 1, .has_alpha = true },
+		{ .format = DRM_FORMAT_ABGR16161616F,	.depth = 0,  .num_planes = 1, .cpp = { 8, 0, 0 }, .hsub = 1, .vsub = 1, .has_alpha = true },
 		{ .format = DRM_FORMAT_RGB888_A8,	.depth = 32, .num_planes = 2, .cpp = { 3, 1, 0 }, .hsub = 1, .vsub = 1, .has_alpha = true },
 		{ .format = DRM_FORMAT_BGR888_A8,	.depth = 32, .num_planes = 2, .cpp = { 3, 1, 0 }, .hsub = 1, .vsub = 1, .has_alpha = true },
 		{ .format = DRM_FORMAT_XRGB8888_A8,	.depth = 32, .num_planes = 2, .cpp = { 4, 1, 0 }, .hsub = 1, .vsub = 1, .has_alpha = true },
@@ -225,7 +229,17 @@ const struct drm_format_info *__drm_format_info(u32 format)
 		{ .format = DRM_FORMAT_UYVY,		.depth = 0,  .num_planes = 1, .cpp = { 2, 0, 0 }, .hsub = 2, .vsub = 1, .is_yuv = true },
 		{ .format = DRM_FORMAT_VYUY,		.depth = 0,  .num_planes = 1, .cpp = { 2, 0, 0 }, .hsub = 2, .vsub = 1, .is_yuv = true },
 		{ .format = DRM_FORMAT_XYUV8888,	.depth = 0,  .num_planes = 1, .cpp = { 4, 0, 0 }, .hsub = 1, .vsub = 1, .is_yuv = true },
+		{ .format = DRM_FORMAT_VUY888,          .depth = 0,  .num_planes = 1, .cpp = { 3, 0, 0 }, .hsub = 1, .vsub = 1, .is_yuv = true },
 		{ .format = DRM_FORMAT_AYUV,		.depth = 0,  .num_planes = 1, .cpp = { 4, 0, 0 }, .hsub = 1, .vsub = 1, .has_alpha = true, .is_yuv = true },
+		{ .format = DRM_FORMAT_Y210,            .depth = 0,  .num_planes = 1, .cpp = { 4, 0, 0 }, .hsub = 2, .vsub = 1, .is_yuv = true },
+		{ .format = DRM_FORMAT_Y212,            .depth = 0,  .num_planes = 1, .cpp = { 4, 0, 0 }, .hsub = 2, .vsub = 1, .is_yuv = true },
+		{ .format = DRM_FORMAT_Y216,            .depth = 0,  .num_planes = 1, .cpp = { 4, 0, 0 }, .hsub = 2, .vsub = 1, .is_yuv = true },
+		{ .format = DRM_FORMAT_Y410,            .depth = 0,  .num_planes = 1, .cpp = { 4, 0, 0 }, .hsub = 1, .vsub = 1, .has_alpha = true, .is_yuv = true },
+		{ .format = DRM_FORMAT_Y412,            .depth = 0,  .num_planes = 1, .cpp = { 8, 0, 0 }, .hsub = 1, .vsub = 1, .has_alpha = true, .is_yuv = true },
+		{ .format = DRM_FORMAT_Y416,            .depth = 0,  .num_planes = 1, .cpp = { 8, 0, 0 }, .hsub = 1, .vsub = 1, .has_alpha = true, .is_yuv = true },
+		{ .format = DRM_FORMAT_XVYU2101010,	.depth = 0,  .num_planes = 1, .cpp = { 4, 0, 0 }, .hsub = 1, .vsub = 1, .is_yuv = true },
+		{ .format = DRM_FORMAT_XVYU12_16161616,	.depth = 0,  .num_planes = 1, .cpp = { 8, 0, 0 }, .hsub = 1, .vsub = 1, .is_yuv = true },
+		{ .format = DRM_FORMAT_XVYU16161616,	.depth = 0,  .num_planes = 1, .cpp = { 8, 0, 0 }, .hsub = 1, .vsub = 1, .is_yuv = true },
 		{ .format = DRM_FORMAT_Y0L0,		.depth = 0,  .num_planes = 1,
 		  .char_per_block = { 8, 0, 0 }, .block_w = { 2, 0, 0 }, .block_h = { 2, 0, 0 },
 		  .hsub = 2, .vsub = 2, .has_alpha = true, .is_yuv = true },
@@ -247,6 +261,19 @@ const struct drm_format_info *__drm_format_info(u32 format)
 		{ .format = DRM_FORMAT_P016,		.depth = 0,  .num_planes = 2,
 		  .char_per_block = { 2, 4, 0 }, .block_w = { 1, 0, 0 }, .block_h = { 1, 0, 0 },
 		  .hsub = 2, .vsub = 2, .is_yuv = true},
+		{ .format = DRM_FORMAT_P210,		.depth = 0,
+		  .num_planes = 2, .char_per_block = { 2, 4, 0 },
+		  .block_w = { 1, 0, 0 }, .block_h = { 1, 0, 0 }, .hsub = 2,
+		  .vsub = 1, .is_yuv = true },
+		{ .format = DRM_FORMAT_VUY101010,	.depth = 0,
+		  .num_planes = 1, .cpp = { 0, 0, 0 }, .hsub = 1, .vsub = 1,
+		  .is_yuv = true },
+		{ .format = DRM_FORMAT_YUV420_8BIT,     .depth = 0,
+		  .num_planes = 1, .cpp = { 0, 0, 0 }, .hsub = 2, .vsub = 2,
+		  .is_yuv = true },
+		{ .format = DRM_FORMAT_YUV420_10BIT,    .depth = 0,
+		  .num_planes = 1, .cpp = { 0, 0, 0 }, .hsub = 2, .vsub = 2,
+		  .is_yuv = true },
 	};
 
 	unsigned int i;
diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c
index d0b9f6a9953f..50de138c89e0 100644
--- a/drivers/gpu/drm/drm_gem.c
+++ b/drivers/gpu/drm/drm_gem.c
@@ -72,23 +72,6 @@
  * up at a later date, and as our interface with shmfs for memory allocation.
  */
 
-/*
- * We make up offsets for buffer objects so we can recognize them at
- * mmap time.
- */
-
-/* pgoff in mmap is an unsigned long, so we need to make sure that
- * the faked up offset will fit
- */
-
-#if BITS_PER_LONG == 64
-#define DRM_FILE_PAGE_OFFSET_START ((0xFFFFFFFFUL >> PAGE_SHIFT) + 1)
-#define DRM_FILE_PAGE_OFFSET_SIZE ((0xFFFFFFFFUL >> PAGE_SHIFT) * 16)
-#else
-#define DRM_FILE_PAGE_OFFSET_START ((0xFFFFFFFUL >> PAGE_SHIFT) + 1)
-#define DRM_FILE_PAGE_OFFSET_SIZE ((0xFFFFFFFUL >> PAGE_SHIFT) * 16)
-#endif
-
 /**
  * drm_gem_init - Initialize the GEM device fields
  * @dev: drm_devic structure to initialize
@@ -171,6 +154,10 @@ void drm_gem_private_object_init(struct drm_device *dev,
 	kref_init(&obj->refcount);
 	obj->handle_count = 0;
 	obj->size = size;
+	reservation_object_init(&obj->_resv);
+	if (!obj->resv)
+		obj->resv = &obj->_resv;
+
 	drm_vma_node_reset(&obj->vma_node);
 }
 EXPORT_SYMBOL(drm_gem_private_object_init);
@@ -659,6 +646,85 @@ void drm_gem_put_pages(struct drm_gem_object *obj, struct page **pages,
 }
 EXPORT_SYMBOL(drm_gem_put_pages);
 
+static int objects_lookup(struct drm_file *filp, u32 *handle, int count,
+			  struct drm_gem_object **objs)
+{
+	int i, ret = 0;
+	struct drm_gem_object *obj;
+
+	spin_lock(&filp->table_lock);
+
+	for (i = 0; i < count; i++) {
+		/* Check if we currently have a reference on the object */
+		obj = idr_find(&filp->object_idr, handle[i]);
+		if (!obj) {
+			ret = -ENOENT;
+			break;
+		}
+		drm_gem_object_get(obj);
+		objs[i] = obj;
+	}
+	spin_unlock(&filp->table_lock);
+
+	return ret;
+}
+
+/**
+ * drm_gem_objects_lookup - look up GEM objects from an array of handles
+ * @filp: DRM file private date
+ * @bo_handles: user pointer to array of userspace handle
+ * @count: size of handle array
+ * @objs_out: returned pointer to array of drm_gem_object pointers
+ *
+ * Takes an array of userspace handles and returns a newly allocated array of
+ * GEM objects.
+ *
+ * For a single handle lookup, use drm_gem_object_lookup().
+ *
+ * Returns:
+ *
+ * @objs filled in with GEM object pointers. Returned GEM objects need to be
+ * released with drm_gem_object_put(). -ENOENT is returned on a lookup
+ * failure. 0 is returned on success.
+ *
+ */
+int drm_gem_objects_lookup(struct drm_file *filp, void __user *bo_handles,
+			   int count, struct drm_gem_object ***objs_out)
+{
+	int ret;
+	u32 *handles;
+	struct drm_gem_object **objs;
+
+	if (!count)
+		return 0;
+
+	objs = kvmalloc_array(count, sizeof(struct drm_gem_object *),
+			     GFP_KERNEL | __GFP_ZERO);
+	if (!objs)
+		return -ENOMEM;
+
+	handles = kvmalloc_array(count, sizeof(u32), GFP_KERNEL);
+	if (!handles) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	if (copy_from_user(handles, bo_handles, count * sizeof(u32))) {
+		ret = -EFAULT;
+		DRM_DEBUG("Failed to copy in GEM handles\n");
+		goto out;
+	}
+
+	ret = objects_lookup(filp, handles, count, objs);
+	*objs_out = objs;
+
+out:
+	kvfree(handles);
+	return ret;
+
+}
+EXPORT_SYMBOL(drm_gem_objects_lookup);
+
 /**
  * drm_gem_object_lookup - look up a GEM object from its handle
  * @filp: DRM file private date
@@ -668,24 +734,56 @@ EXPORT_SYMBOL(drm_gem_put_pages);
  *
  * A reference to the object named by the handle if such exists on @filp, NULL
  * otherwise.
+ *
+ * If looking up an array of handles, use drm_gem_objects_lookup().
  */
 struct drm_gem_object *
 drm_gem_object_lookup(struct drm_file *filp, u32 handle)
 {
+	struct drm_gem_object *obj = NULL;
+
+	objects_lookup(filp, &handle, 1, &obj);
+	return obj;
+}
+EXPORT_SYMBOL(drm_gem_object_lookup);
+
+/**
+ * drm_gem_reservation_object_wait - Wait on GEM object's reservation's objects
+ * shared and/or exclusive fences.
+ * @filep: DRM file private date
+ * @handle: userspace handle
+ * @wait_all: if true, wait on all fences, else wait on just exclusive fence
+ * @timeout: timeout value in jiffies or zero to return immediately
+ *
+ * Returns:
+ *
+ * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or
+ * greater than 0 on success.
+ */
+long drm_gem_reservation_object_wait(struct drm_file *filep, u32 handle,
+				    bool wait_all, unsigned long timeout)
+{
+	long ret;
 	struct drm_gem_object *obj;
 
-	spin_lock(&filp->table_lock);
+	obj = drm_gem_object_lookup(filep, handle);
+	if (!obj) {
+		DRM_DEBUG("Failed to look up GEM BO %d\n", handle);
+		return -EINVAL;
+	}
 
-	/* Check if we currently have a reference on the object */
-	obj = idr_find(&filp->object_idr, handle);
-	if (obj)
-		drm_gem_object_get(obj);
+	ret = reservation_object_wait_timeout_rcu(obj->resv, wait_all,
+						  true, timeout);
+	if (ret == 0)
+		ret = -ETIME;
+	else if (ret > 0)
+		ret = 0;
 
-	spin_unlock(&filp->table_lock);
+	drm_gem_object_put_unlocked(obj);
 
-	return obj;
+	return ret;
 }
-EXPORT_SYMBOL(drm_gem_object_lookup);
+EXPORT_SYMBOL(drm_gem_reservation_object_wait);
 
 /**
  * drm_gem_close_ioctl - implementation of the GEM_CLOSE ioctl
@@ -851,6 +949,7 @@ drm_gem_object_release(struct drm_gem_object *obj)
 	if (obj->filp)
 		fput(obj->filp);
 
+	reservation_object_fini(&obj->_resv);
 	drm_gem_free_mmap_offset(obj);
 }
 EXPORT_SYMBOL(drm_gem_object_release);
@@ -1190,3 +1289,174 @@ void drm_gem_vunmap(struct drm_gem_object *obj, void *vaddr)
 		obj->dev->driver->gem_prime_vunmap(obj, vaddr);
 }
 EXPORT_SYMBOL(drm_gem_vunmap);
+
+/**
+ * drm_gem_lock_reservations - Sets up the ww context and acquires
+ * the lock on an array of GEM objects.
+ *
+ * Once you've locked your reservations, you'll want to set up space
+ * for your shared fences (if applicable), submit your job, then
+ * drm_gem_unlock_reservations().
+ *
+ * @objs: drm_gem_objects to lock
+ * @count: Number of objects in @objs
+ * @acquire_ctx: struct ww_acquire_ctx that will be initialized as
+ * part of tracking this set of locked reservations.
+ */
+int
+drm_gem_lock_reservations(struct drm_gem_object **objs, int count,
+			  struct ww_acquire_ctx *acquire_ctx)
+{
+	int contended = -1;
+	int i, ret;
+
+	ww_acquire_init(acquire_ctx, &reservation_ww_class);
+
+retry:
+	if (contended != -1) {
+		struct drm_gem_object *obj = objs[contended];
+
+		ret = ww_mutex_lock_slow_interruptible(&obj->resv->lock,
+						       acquire_ctx);
+		if (ret) {
+			ww_acquire_done(acquire_ctx);
+			return ret;
+		}
+	}
+
+	for (i = 0; i < count; i++) {
+		if (i == contended)
+			continue;
+
+		ret = ww_mutex_lock_interruptible(&objs[i]->resv->lock,
+						  acquire_ctx);
+		if (ret) {
+			int j;
+
+			for (j = 0; j < i; j++)
+				ww_mutex_unlock(&objs[j]->resv->lock);
+
+			if (contended != -1 && contended >= i)
+				ww_mutex_unlock(&objs[contended]->resv->lock);
+
+			if (ret == -EDEADLK) {
+				contended = i;
+				goto retry;
+			}
+
+			ww_acquire_done(acquire_ctx);
+			return ret;
+		}
+	}
+
+	ww_acquire_done(acquire_ctx);
+
+	return 0;
+}
+EXPORT_SYMBOL(drm_gem_lock_reservations);
+
+void
+drm_gem_unlock_reservations(struct drm_gem_object **objs, int count,
+			    struct ww_acquire_ctx *acquire_ctx)
+{
+	int i;
+
+	for (i = 0; i < count; i++)
+		ww_mutex_unlock(&objs[i]->resv->lock);
+
+	ww_acquire_fini(acquire_ctx);
+}
+EXPORT_SYMBOL(drm_gem_unlock_reservations);
+
+/**
+ * drm_gem_fence_array_add - Adds the fence to an array of fences to be
+ * waited on, deduplicating fences from the same context.
+ *
+ * @fence_array: array of dma_fence * for the job to block on.
+ * @fence: the dma_fence to add to the list of dependencies.
+ *
+ * Returns:
+ * 0 on success, or an error on failing to expand the array.
+ */
+int drm_gem_fence_array_add(struct xarray *fence_array,
+			    struct dma_fence *fence)
+{
+	struct dma_fence *entry;
+	unsigned long index;
+	u32 id = 0;
+	int ret;
+
+	if (!fence)
+		return 0;
+
+	/* Deduplicate if we already depend on a fence from the same context.
+	 * This lets the size of the array of deps scale with the number of
+	 * engines involved, rather than the number of BOs.
+	 */
+	xa_for_each(fence_array, index, entry) {
+		if (entry->context != fence->context)
+			continue;
+
+		if (dma_fence_is_later(fence, entry)) {
+			dma_fence_put(entry);
+			xa_store(fence_array, index, fence, GFP_KERNEL);
+		} else {
+			dma_fence_put(fence);
+		}
+		return 0;
+	}
+
+	ret = xa_alloc(fence_array, &id, fence, xa_limit_32b, GFP_KERNEL);
+	if (ret != 0)
+		dma_fence_put(fence);
+
+	return ret;
+}
+EXPORT_SYMBOL(drm_gem_fence_array_add);
+
+/**
+ * drm_gem_fence_array_add_implicit - Adds the implicit dependencies tracked
+ * in the GEM object's reservation object to an array of dma_fences for use in
+ * scheduling a rendering job.
+ *
+ * This should be called after drm_gem_lock_reservations() on your array of
+ * GEM objects used in the job but before updating the reservations with your
+ * own fences.
+ *
+ * @fence_array: array of dma_fence * for the job to block on.
+ * @obj: the gem object to add new dependencies from.
+ * @write: whether the job might write the object (so we need to depend on
+ * shared fences in the reservation object).
+ */
+int drm_gem_fence_array_add_implicit(struct xarray *fence_array,
+				     struct drm_gem_object *obj,
+				     bool write)
+{
+	int ret;
+	struct dma_fence **fences;
+	unsigned int i, fence_count;
+
+	if (!write) {
+		struct dma_fence *fence =
+			reservation_object_get_excl_rcu(obj->resv);
+
+		return drm_gem_fence_array_add(fence_array, fence);
+	}
+
+	ret = reservation_object_get_fences_rcu(obj->resv, NULL,
+						&fence_count, &fences);
+	if (ret || !fence_count)
+		return ret;
+
+	for (i = 0; i < fence_count; i++) {
+		ret = drm_gem_fence_array_add(fence_array, fences[i]);
+		if (ret)
+			break;
+	}
+
+	for (; i < fence_count; i++)
+		dma_fence_put(fences[i]);
+	kfree(fences);
+	return ret;
+}
+EXPORT_SYMBOL(drm_gem_fence_array_add_implicit);
diff --git a/drivers/gpu/drm/drm_gem_cma_helper.c b/drivers/gpu/drm/drm_gem_cma_helper.c
index cc26625b4b33..e01ceed09e67 100644
--- a/drivers/gpu/drm/drm_gem_cma_helper.c
+++ b/drivers/gpu/drm/drm_gem_cma_helper.c
@@ -186,13 +186,13 @@ void drm_gem_cma_free_object(struct drm_gem_object *gem_obj)
 
 	cma_obj = to_drm_gem_cma_obj(gem_obj);
 
-	if (cma_obj->vaddr) {
-		dma_free_wc(gem_obj->dev->dev, cma_obj->base.size,
-			    cma_obj->vaddr, cma_obj->paddr);
-	} else if (gem_obj->import_attach) {
+	if (gem_obj->import_attach) {
 		if (cma_obj->vaddr)
 			dma_buf_vunmap(gem_obj->import_attach->dmabuf, cma_obj->vaddr);
 		drm_prime_gem_destroy(gem_obj, cma_obj->sgt);
+	} else if (cma_obj->vaddr) {
+		dma_free_wc(gem_obj->dev->dev, cma_obj->base.size,
+			    cma_obj->vaddr, cma_obj->paddr);
 	}
 
 	drm_gem_object_release(gem_obj);
diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c
new file mode 100644
index 000000000000..1ee208c2c85e
--- /dev/null
+++ b/drivers/gpu/drm/drm_gem_shmem_helper.c
@@ -0,0 +1,625 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2018 Noralf Trønnes
+ */
+
+#include <linux/dma-buf.h>
+#include <linux/export.h>
+#include <linux/mutex.h>
+#include <linux/shmem_fs.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+
+#include <drm/drm_device.h>
+#include <drm/drm_drv.h>
+#include <drm/drm_gem_shmem_helper.h>
+#include <drm/drm_prime.h>
+#include <drm/drm_print.h>
+
+/**
+ * DOC: overview
+ *
+ * This library provides helpers for GEM objects backed by shmem buffers
+ * allocated using anonymous pageable memory.
+ */
+
+static const struct drm_gem_object_funcs drm_gem_shmem_funcs = {
+	.free = drm_gem_shmem_free_object,
+	.print_info = drm_gem_shmem_print_info,
+	.pin = drm_gem_shmem_pin,
+	.unpin = drm_gem_shmem_unpin,
+	.get_sg_table = drm_gem_shmem_get_sg_table,
+	.vmap = drm_gem_shmem_vmap,
+	.vunmap = drm_gem_shmem_vunmap,
+	.vm_ops = &drm_gem_shmem_vm_ops,
+};
+
+/**
+ * drm_gem_shmem_create - Allocate an object with the given size
+ * @dev: DRM device
+ * @size: Size of the object to allocate
+ *
+ * This function creates a shmem GEM object.
+ *
+ * Returns:
+ * A struct drm_gem_shmem_object * on success or an ERR_PTR()-encoded negative
+ * error code on failure.
+ */
+struct drm_gem_shmem_object *drm_gem_shmem_create(struct drm_device *dev, size_t size)
+{
+	struct drm_gem_shmem_object *shmem;
+	struct drm_gem_object *obj;
+	int ret;
+
+	size = PAGE_ALIGN(size);
+
+	if (dev->driver->gem_create_object)
+		obj = dev->driver->gem_create_object(dev, size);
+	else
+		obj = kzalloc(sizeof(*shmem), GFP_KERNEL);
+	if (!obj)
+		return ERR_PTR(-ENOMEM);
+
+	if (!obj->funcs)
+		obj->funcs = &drm_gem_shmem_funcs;
+
+	ret = drm_gem_object_init(dev, obj, size);
+	if (ret)
+		goto err_free;
+
+	ret = drm_gem_create_mmap_offset(obj);
+	if (ret)
+		goto err_release;
+
+	shmem = to_drm_gem_shmem_obj(obj);
+	mutex_init(&shmem->pages_lock);
+	mutex_init(&shmem->vmap_lock);
+
+	/*
+	 * Our buffers are kept pinned, so allocating them
+	 * from the MOVABLE zone is a really bad idea, and
+	 * conflicts with CMA. See comments above new_inode()
+	 * why this is required _and_ expected if you're
+	 * going to pin these pages.
+	 */
+	mapping_set_gfp_mask(obj->filp->f_mapping, GFP_HIGHUSER |
+			     __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
+
+	return shmem;
+
+err_release:
+	drm_gem_object_release(obj);
+err_free:
+	kfree(obj);
+
+	return ERR_PTR(ret);
+}
+EXPORT_SYMBOL_GPL(drm_gem_shmem_create);
+
+/**
+ * drm_gem_shmem_free_object - Free resources associated with a shmem GEM object
+ * @obj: GEM object to free
+ *
+ * This function cleans up the GEM object state and frees the memory used to
+ * store the object itself.
+ */
+void drm_gem_shmem_free_object(struct drm_gem_object *obj)
+{
+	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
+
+	WARN_ON(shmem->vmap_use_count);
+
+	if (obj->import_attach) {
+		shmem->pages_use_count--;
+		drm_prime_gem_destroy(obj, shmem->sgt);
+		kvfree(shmem->pages);
+	} else {
+		if (shmem->sgt) {
+			dma_unmap_sg(obj->dev->dev, shmem->sgt->sgl,
+				     shmem->sgt->nents, DMA_BIDIRECTIONAL);
+
+			drm_gem_shmem_put_pages(shmem);
+			sg_free_table(shmem->sgt);
+			kfree(shmem->sgt);
+		}
+	}
+
+	WARN_ON(shmem->pages_use_count);
+
+	drm_gem_object_release(obj);
+	mutex_destroy(&shmem->pages_lock);
+	mutex_destroy(&shmem->vmap_lock);
+	kfree(shmem);
+}
+EXPORT_SYMBOL_GPL(drm_gem_shmem_free_object);
+
+static int drm_gem_shmem_get_pages_locked(struct drm_gem_shmem_object *shmem)
+{
+	struct drm_gem_object *obj = &shmem->base;
+	struct page **pages;
+
+	if (shmem->pages_use_count++ > 0)
+		return 0;
+
+	pages = drm_gem_get_pages(obj);
+	if (IS_ERR(pages)) {
+		DRM_DEBUG_KMS("Failed to get pages (%ld)\n", PTR_ERR(pages));
+		shmem->pages_use_count = 0;
+		return PTR_ERR(pages);
+	}
+
+	shmem->pages = pages;
+
+	return 0;
+}
+
+/*
+ * drm_gem_shmem_get_pages - Allocate backing pages for a shmem GEM object
+ * @shmem: shmem GEM object
+ *
+ * This function makes sure that backing pages exists for the shmem GEM object
+ * and increases the use count.
+ *
+ * Returns:
+ * 0 on success or a negative error code on failure.
+ */
+int drm_gem_shmem_get_pages(struct drm_gem_shmem_object *shmem)
+{
+	int ret;
+
+	ret = mutex_lock_interruptible(&shmem->pages_lock);
+	if (ret)
+		return ret;
+	ret = drm_gem_shmem_get_pages_locked(shmem);
+	mutex_unlock(&shmem->pages_lock);
+
+	return ret;
+}
+EXPORT_SYMBOL(drm_gem_shmem_get_pages);
+
+static void drm_gem_shmem_put_pages_locked(struct drm_gem_shmem_object *shmem)
+{
+	struct drm_gem_object *obj = &shmem->base;
+
+	if (WARN_ON_ONCE(!shmem->pages_use_count))
+		return;
+
+	if (--shmem->pages_use_count > 0)
+		return;
+
+	drm_gem_put_pages(obj, shmem->pages,
+			  shmem->pages_mark_dirty_on_put,
+			  shmem->pages_mark_accessed_on_put);
+	shmem->pages = NULL;
+}
+
+/*
+ * drm_gem_shmem_put_pages - Decrease use count on the backing pages for a shmem GEM object
+ * @shmem: shmem GEM object
+ *
+ * This function decreases the use count and puts the backing pages when use drops to zero.
+ */
+void drm_gem_shmem_put_pages(struct drm_gem_shmem_object *shmem)
+{
+	mutex_lock(&shmem->pages_lock);
+	drm_gem_shmem_put_pages_locked(shmem);
+	mutex_unlock(&shmem->pages_lock);
+}
+EXPORT_SYMBOL(drm_gem_shmem_put_pages);
+
+/**
+ * drm_gem_shmem_pin - Pin backing pages for a shmem GEM object
+ * @obj: GEM object
+ *
+ * This function makes sure the backing pages are pinned in memory while the
+ * buffer is exported.
+ *
+ * Returns:
+ * 0 on success or a negative error code on failure.
+ */
+int drm_gem_shmem_pin(struct drm_gem_object *obj)
+{
+	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
+
+	return drm_gem_shmem_get_pages(shmem);
+}
+EXPORT_SYMBOL(drm_gem_shmem_pin);
+
+/**
+ * drm_gem_shmem_unpin - Unpin backing pages for a shmem GEM object
+ * @obj: GEM object
+ *
+ * This function removes the requirement that the backing pages are pinned in
+ * memory.
+ */
+void drm_gem_shmem_unpin(struct drm_gem_object *obj)
+{
+	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
+
+	drm_gem_shmem_put_pages(shmem);
+}
+EXPORT_SYMBOL(drm_gem_shmem_unpin);
+
+static void *drm_gem_shmem_vmap_locked(struct drm_gem_shmem_object *shmem)
+{
+	struct drm_gem_object *obj = &shmem->base;
+	int ret;
+
+	if (shmem->vmap_use_count++ > 0)
+		return shmem->vaddr;
+
+	ret = drm_gem_shmem_get_pages(shmem);
+	if (ret)
+		goto err_zero_use;
+
+	if (obj->import_attach)
+		shmem->vaddr = dma_buf_vmap(obj->import_attach->dmabuf);
+	else
+		shmem->vaddr = vmap(shmem->pages, obj->size >> PAGE_SHIFT, VM_MAP, PAGE_KERNEL);
+
+	if (!shmem->vaddr) {
+		DRM_DEBUG_KMS("Failed to vmap pages\n");
+		ret = -ENOMEM;
+		goto err_put_pages;
+	}
+
+	return shmem->vaddr;
+
+err_put_pages:
+	drm_gem_shmem_put_pages(shmem);
+err_zero_use:
+	shmem->vmap_use_count = 0;
+
+	return ERR_PTR(ret);
+}
+
+/*
+ * drm_gem_shmem_vmap - Create a virtual mapping for a shmem GEM object
+ * @shmem: shmem GEM object
+ *
+ * This function makes sure that a virtual address exists for the buffer backing
+ * the shmem GEM object.
+ *
+ * Returns:
+ * 0 on success or a negative error code on failure.
+ */
+void *drm_gem_shmem_vmap(struct drm_gem_object *obj)
+{
+	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
+	void *vaddr;
+	int ret;
+
+	ret = mutex_lock_interruptible(&shmem->vmap_lock);
+	if (ret)
+		return ERR_PTR(ret);
+	vaddr = drm_gem_shmem_vmap_locked(shmem);
+	mutex_unlock(&shmem->vmap_lock);
+
+	return vaddr;
+}
+EXPORT_SYMBOL(drm_gem_shmem_vmap);
+
+static void drm_gem_shmem_vunmap_locked(struct drm_gem_shmem_object *shmem)
+{
+	struct drm_gem_object *obj = &shmem->base;
+
+	if (WARN_ON_ONCE(!shmem->vmap_use_count))
+		return;
+
+	if (--shmem->vmap_use_count > 0)
+		return;
+
+	if (obj->import_attach)
+		dma_buf_vunmap(obj->import_attach->dmabuf, shmem->vaddr);
+	else
+		vunmap(shmem->vaddr);
+
+	shmem->vaddr = NULL;
+	drm_gem_shmem_put_pages(shmem);
+}
+
+/*
+ * drm_gem_shmem_vunmap - Unmap a virtual mapping fo a shmem GEM object
+ * @shmem: shmem GEM object
+ *
+ * This function removes the virtual address when use count drops to zero.
+ */
+void drm_gem_shmem_vunmap(struct drm_gem_object *obj, void *vaddr)
+{
+	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
+
+	mutex_lock(&shmem->vmap_lock);
+	drm_gem_shmem_vunmap_locked(shmem);
+	mutex_unlock(&shmem->vmap_lock);
+}
+EXPORT_SYMBOL(drm_gem_shmem_vunmap);
+
+struct drm_gem_shmem_object *
+drm_gem_shmem_create_with_handle(struct drm_file *file_priv,
+				 struct drm_device *dev, size_t size,
+				 uint32_t *handle)
+{
+	struct drm_gem_shmem_object *shmem;
+	int ret;
+
+	shmem = drm_gem_shmem_create(dev, size);
+	if (IS_ERR(shmem))
+		return shmem;
+
+	/*
+	 * Allocate an id of idr table where the obj is registered
+	 * and handle has the id what user can see.
+	 */
+	ret = drm_gem_handle_create(file_priv, &shmem->base, handle);
+	/* drop reference from allocate - handle holds it now. */
+	drm_gem_object_put_unlocked(&shmem->base);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return shmem;
+}
+EXPORT_SYMBOL(drm_gem_shmem_create_with_handle);
+
+/**
+ * drm_gem_shmem_dumb_create - Create a dumb shmem buffer object
+ * @file: DRM file structure to create the dumb buffer for
+ * @dev: DRM device
+ * @args: IOCTL data
+ *
+ * This function computes the pitch of the dumb buffer and rounds it up to an
+ * integer number of bytes per pixel. Drivers for hardware that doesn't have
+ * any additional restrictions on the pitch can directly use this function as
+ * their &drm_driver.dumb_create callback.
+ *
+ * For hardware with additional restrictions, drivers can adjust the fields
+ * set up by userspace before calling into this function.
+ *
+ * Returns:
+ * 0 on success or a negative error code on failure.
+ */
+int drm_gem_shmem_dumb_create(struct drm_file *file, struct drm_device *dev,
+			      struct drm_mode_create_dumb *args)
+{
+	u32 min_pitch = DIV_ROUND_UP(args->width * args->bpp, 8);
+	struct drm_gem_shmem_object *shmem;
+
+	if (!args->pitch || !args->size) {
+		args->pitch = min_pitch;
+		args->size = args->pitch * args->height;
+	} else {
+		/* ensure sane minimum values */
+		if (args->pitch < min_pitch)
+			args->pitch = min_pitch;
+		if (args->size < args->pitch * args->height)
+			args->size = args->pitch * args->height;
+	}
+
+	shmem = drm_gem_shmem_create_with_handle(file, dev, args->size, &args->handle);
+
+	return PTR_ERR_OR_ZERO(shmem);
+}
+EXPORT_SYMBOL_GPL(drm_gem_shmem_dumb_create);
+
+static vm_fault_t drm_gem_shmem_fault(struct vm_fault *vmf)
+{
+	struct vm_area_struct *vma = vmf->vma;
+	struct drm_gem_object *obj = vma->vm_private_data;
+	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
+	loff_t num_pages = obj->size >> PAGE_SHIFT;
+	struct page *page;
+
+	if (vmf->pgoff >= num_pages || WARN_ON_ONCE(!shmem->pages))
+		return VM_FAULT_SIGBUS;
+
+	page = shmem->pages[vmf->pgoff];
+
+	return vmf_insert_page(vma, vmf->address, page);
+}
+
+static void drm_gem_shmem_vm_open(struct vm_area_struct *vma)
+{
+	struct drm_gem_object *obj = vma->vm_private_data;
+	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
+	int ret;
+
+	ret = drm_gem_shmem_get_pages(shmem);
+	WARN_ON_ONCE(ret != 0);
+
+	drm_gem_vm_open(vma);
+}
+
+static void drm_gem_shmem_vm_close(struct vm_area_struct *vma)
+{
+	struct drm_gem_object *obj = vma->vm_private_data;
+	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
+
+	drm_gem_shmem_put_pages(shmem);
+	drm_gem_vm_close(vma);
+}
+
+const struct vm_operations_struct drm_gem_shmem_vm_ops = {
+	.fault = drm_gem_shmem_fault,
+	.open = drm_gem_shmem_vm_open,
+	.close = drm_gem_shmem_vm_close,
+};
+EXPORT_SYMBOL_GPL(drm_gem_shmem_vm_ops);
+
+/**
+ * drm_gem_shmem_mmap - Memory-map a shmem GEM object
+ * @filp: File object
+ * @vma: VMA for the area to be mapped
+ *
+ * This function implements an augmented version of the GEM DRM file mmap
+ * operation for shmem objects. Drivers which employ the shmem helpers should
+ * use this function as their &file_operations.mmap handler in the DRM device file's
+ * file_operations structure.
+ *
+ * Instead of directly referencing this function, drivers should use the
+ * DEFINE_DRM_GEM_SHMEM_FOPS() macro.
+ *
+ * Returns:
+ * 0 on success or a negative error code on failure.
+ */
+int drm_gem_shmem_mmap(struct file *filp, struct vm_area_struct *vma)
+{
+	struct drm_gem_shmem_object *shmem;
+	int ret;
+
+	ret = drm_gem_mmap(filp, vma);
+	if (ret)
+		return ret;
+
+	shmem = to_drm_gem_shmem_obj(vma->vm_private_data);
+
+	ret = drm_gem_shmem_get_pages(shmem);
+	if (ret) {
+		drm_gem_vm_close(vma);
+		return ret;
+	}
+
+	/* VM_PFNMAP was set by drm_gem_mmap() */
+	vma->vm_flags &= ~VM_PFNMAP;
+	vma->vm_flags |= VM_MIXEDMAP;
+
+	/* Remove the fake offset */
+	vma->vm_pgoff -= drm_vma_node_start(&shmem->base.vma_node);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(drm_gem_shmem_mmap);
+
+/**
+ * drm_gem_shmem_print_info() - Print &drm_gem_shmem_object info for debugfs
+ * @p: DRM printer
+ * @indent: Tab indentation level
+ * @obj: GEM object
+ */
+void drm_gem_shmem_print_info(struct drm_printer *p, unsigned int indent,
+			      const struct drm_gem_object *obj)
+{
+	const struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
+
+	drm_printf_indent(p, indent, "pages_use_count=%u\n", shmem->pages_use_count);
+	drm_printf_indent(p, indent, "vmap_use_count=%u\n", shmem->vmap_use_count);
+	drm_printf_indent(p, indent, "vaddr=%p\n", shmem->vaddr);
+}
+EXPORT_SYMBOL(drm_gem_shmem_print_info);
+
+/**
+ * drm_gem_shmem_get_sg_table - Provide a scatter/gather table of pinned
+ *                              pages for a shmem GEM object
+ * @obj: GEM object
+ *
+ * This function exports a scatter/gather table suitable for PRIME usage by
+ * calling the standard DMA mapping API.
+ *
+ * Returns:
+ * A pointer to the scatter/gather table of pinned pages or NULL on failure.
+ */
+struct sg_table *drm_gem_shmem_get_sg_table(struct drm_gem_object *obj)
+{
+	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
+
+	return drm_prime_pages_to_sg(shmem->pages, obj->size >> PAGE_SHIFT);
+}
+EXPORT_SYMBOL_GPL(drm_gem_shmem_get_sg_table);
+
+/**
+ * drm_gem_shmem_get_pages_sgt - Pin pages, dma map them, and return a
+ *				 scatter/gather table for a shmem GEM object.
+ * @obj: GEM object
+ *
+ * This function returns a scatter/gather table suitable for driver usage. If
+ * the sg table doesn't exist, the pages are pinned, dma-mapped, and a sg
+ * table created.
+ *
+ * Returns:
+ * A pointer to the scatter/gather table of pinned pages or errno on failure.
+ */
+struct sg_table *drm_gem_shmem_get_pages_sgt(struct drm_gem_object *obj)
+{
+	int ret;
+	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
+	struct sg_table *sgt;
+
+	if (shmem->sgt)
+		return shmem->sgt;
+
+	WARN_ON(obj->import_attach);
+
+	ret = drm_gem_shmem_get_pages(shmem);
+	if (ret)
+		return ERR_PTR(ret);
+
+	sgt = drm_gem_shmem_get_sg_table(&shmem->base);
+	if (IS_ERR(sgt)) {
+		ret = PTR_ERR(sgt);
+		goto err_put_pages;
+	}
+	/* Map the pages for use by the h/w. */
+	dma_map_sg(obj->dev->dev, sgt->sgl, sgt->nents, DMA_BIDIRECTIONAL);
+
+	shmem->sgt = sgt;
+
+	return sgt;
+
+err_put_pages:
+	drm_gem_shmem_put_pages(shmem);
+	return ERR_PTR(ret);
+}
+EXPORT_SYMBOL_GPL(drm_gem_shmem_get_pages_sgt);
+
+/**
+ * drm_gem_shmem_prime_import_sg_table - Produce a shmem GEM object from
+ *                 another driver's scatter/gather table of pinned pages
+ * @dev: Device to import into
+ * @attach: DMA-BUF attachment
+ * @sgt: Scatter/gather table of pinned pages
+ *
+ * This function imports a scatter/gather table exported via DMA-BUF by
+ * another driver. Drivers that use the shmem helpers should set this as their
+ * &drm_driver.gem_prime_import_sg_table callback.
+ *
+ * Returns:
+ * A pointer to a newly created GEM object or an ERR_PTR-encoded negative
+ * error code on failure.
+ */
+struct drm_gem_object *
+drm_gem_shmem_prime_import_sg_table(struct drm_device *dev,
+				    struct dma_buf_attachment *attach,
+				    struct sg_table *sgt)
+{
+	size_t size = PAGE_ALIGN(attach->dmabuf->size);
+	size_t npages = size >> PAGE_SHIFT;
+	struct drm_gem_shmem_object *shmem;
+	int ret;
+
+	shmem = drm_gem_shmem_create(dev, size);
+	if (IS_ERR(shmem))
+		return ERR_CAST(shmem);
+
+	shmem->pages = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL);
+	if (!shmem->pages) {
+		ret = -ENOMEM;
+		goto err_free_gem;
+	}
+
+	ret = drm_prime_sg_to_page_addr_arrays(sgt, shmem->pages, NULL, npages);
+	if (ret < 0)
+		goto err_free_array;
+
+	shmem->sgt = sgt;
+	shmem->pages_use_count = 1; /* Permanently pinned from our point of view */
+
+	DRM_DEBUG_PRIME("size = %zu\n", size);
+
+	return &shmem->base;
+
+err_free_array:
+	kvfree(shmem->pages);
+err_free_gem:
+	drm_gem_object_put_unlocked(&shmem->base);
+
+	return ERR_PTR(ret);
+}
+EXPORT_SYMBOL_GPL(drm_gem_shmem_prime_import_sg_table);
diff --git a/drivers/gpu/drm/drm_internal.h b/drivers/gpu/drm/drm_internal.h
index 251d67e04c2d..e19ac7ca602d 100644
--- a/drivers/gpu/drm/drm_internal.h
+++ b/drivers/gpu/drm/drm_internal.h
@@ -71,8 +71,10 @@ int drm_legacy_modeset_ctl_ioctl(struct drm_device *dev, void *data,
 /* drm_irq.c */
 
 /* IOCTLS */
+#if IS_ENABLED(CONFIG_DRM_LEGACY)
 int drm_legacy_irq_control(struct drm_device *dev, void *data,
 			   struct drm_file *file_priv);
+#endif
 
 int drm_crtc_get_sequence_ioctl(struct drm_device *dev, void *data,
 				struct drm_file *filp);
@@ -180,12 +182,20 @@ int drm_syncobj_handle_to_fd_ioctl(struct drm_device *dev, void *data,
 				   struct drm_file *file_private);
 int drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data,
 				   struct drm_file *file_private);
+int drm_syncobj_transfer_ioctl(struct drm_device *dev, void *data,
+			       struct drm_file *file_private);
 int drm_syncobj_wait_ioctl(struct drm_device *dev, void *data,
 			   struct drm_file *file_private);
+int drm_syncobj_timeline_wait_ioctl(struct drm_device *dev, void *data,
+				    struct drm_file *file_private);
 int drm_syncobj_reset_ioctl(struct drm_device *dev, void *data,
 			    struct drm_file *file_private);
 int drm_syncobj_signal_ioctl(struct drm_device *dev, void *data,
 			     struct drm_file *file_private);
+int drm_syncobj_timeline_signal_ioctl(struct drm_device *dev, void *data,
+				      struct drm_file *file_private);
+int drm_syncobj_query_ioctl(struct drm_device *dev, void *data,
+			    struct drm_file *file_private);
 
 /* drm_framebuffer.c */
 void drm_framebuffer_print_info(struct drm_printer *p, unsigned int indent,
diff --git a/drivers/gpu/drm/drm_ioc32.c b/drivers/gpu/drm/drm_ioc32.c
index 0e3043e08c69..374b372da58a 100644
--- a/drivers/gpu/drm/drm_ioc32.c
+++ b/drivers/gpu/drm/drm_ioc32.c
@@ -156,6 +156,7 @@ static int compat_drm_setunique(struct file *file, unsigned int cmd,
 	return -EINVAL;
 }
 
+#if IS_ENABLED(CONFIG_DRM_LEGACY)
 typedef struct drm_map32 {
 	u32 offset;		/* Requested physical address (0 for SAREA) */
 	u32 size;		/* Requested physical size (bytes) */
@@ -239,6 +240,7 @@ static int compat_drm_rmmap(struct file *file, unsigned int cmd,
 	map.handle = compat_ptr(handle);
 	return drm_ioctl_kernel(file, drm_legacy_rmmap_ioctl, &map, DRM_AUTH);
 }
+#endif
 
 typedef struct drm_client32 {
 	int idx;	/* Which client desired? */
@@ -301,6 +303,7 @@ static int compat_drm_getstats(struct file *file, unsigned int cmd,
 	return 0;
 }
 
+#if IS_ENABLED(CONFIG_DRM_LEGACY)
 typedef struct drm_buf_desc32 {
 	int count;		 /* Number of buffers of this size */
 	int size;		 /* Size in bytes */
@@ -604,6 +607,7 @@ static int compat_drm_dma(struct file *file, unsigned int cmd,
 
 	return 0;
 }
+#endif
 
 #if IS_ENABLED(CONFIG_AGP)
 typedef struct drm_agp_mode32 {
@@ -748,6 +752,7 @@ static int compat_drm_agp_unbind(struct file *file, unsigned int cmd,
 }
 #endif /* CONFIG_AGP */
 
+#if IS_ENABLED(CONFIG_DRM_LEGACY)
 typedef struct drm_scatter_gather32 {
 	u32 size;	/**< In bytes -- will round to page boundary */
 	u32 handle;	/**< Used for mapping / unmapping */
@@ -788,7 +793,7 @@ static int compat_drm_sg_free(struct file *file, unsigned int cmd,
 	return drm_ioctl_kernel(file, drm_legacy_sg_free, &request,
 				DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY);
 }
-
+#endif
 #if defined(CONFIG_X86)
 typedef struct drm_update_draw32 {
 	drm_drawable_t handle;
@@ -903,10 +908,13 @@ static struct {
 #define DRM_IOCTL32_DEF(n, f) [DRM_IOCTL_NR(n##32)] = {.fn = f, .name = #n}
 	DRM_IOCTL32_DEF(DRM_IOCTL_VERSION, compat_drm_version),
 	DRM_IOCTL32_DEF(DRM_IOCTL_GET_UNIQUE, compat_drm_getunique),
+#if IS_ENABLED(CONFIG_DRM_LEGACY)
 	DRM_IOCTL32_DEF(DRM_IOCTL_GET_MAP, compat_drm_getmap),
+#endif
 	DRM_IOCTL32_DEF(DRM_IOCTL_GET_CLIENT, compat_drm_getclient),
 	DRM_IOCTL32_DEF(DRM_IOCTL_GET_STATS, compat_drm_getstats),
 	DRM_IOCTL32_DEF(DRM_IOCTL_SET_UNIQUE, compat_drm_setunique),
+#if IS_ENABLED(CONFIG_DRM_LEGACY)
 	DRM_IOCTL32_DEF(DRM_IOCTL_ADD_MAP, compat_drm_addmap),
 	DRM_IOCTL32_DEF(DRM_IOCTL_ADD_BUFS, compat_drm_addbufs),
 	DRM_IOCTL32_DEF(DRM_IOCTL_MARK_BUFS, compat_drm_markbufs),
@@ -918,6 +926,7 @@ static struct {
 	DRM_IOCTL32_DEF(DRM_IOCTL_GET_SAREA_CTX, compat_drm_getsareactx),
 	DRM_IOCTL32_DEF(DRM_IOCTL_RES_CTX, compat_drm_resctx),
 	DRM_IOCTL32_DEF(DRM_IOCTL_DMA, compat_drm_dma),
+#endif
 #if IS_ENABLED(CONFIG_AGP)
 	DRM_IOCTL32_DEF(DRM_IOCTL_AGP_ENABLE, compat_drm_agp_enable),
 	DRM_IOCTL32_DEF(DRM_IOCTL_AGP_INFO, compat_drm_agp_info),
@@ -926,8 +935,10 @@ static struct {
 	DRM_IOCTL32_DEF(DRM_IOCTL_AGP_BIND, compat_drm_agp_bind),
 	DRM_IOCTL32_DEF(DRM_IOCTL_AGP_UNBIND, compat_drm_agp_unbind),
 #endif
+#if IS_ENABLED(CONFIG_DRM_LEGACY)
 	DRM_IOCTL32_DEF(DRM_IOCTL_SG_ALLOC, compat_drm_sg_alloc),
 	DRM_IOCTL32_DEF(DRM_IOCTL_SG_FREE, compat_drm_sg_free),
+#endif
 #if defined(CONFIG_X86) || defined(CONFIG_IA64)
 	DRM_IOCTL32_DEF(DRM_IOCTL_UPDATE_DRAW, compat_drm_update_draw),
 #endif
diff --git a/drivers/gpu/drm/drm_ioctl.c b/drivers/gpu/drm/drm_ioctl.c
index 687943df58e1..2263e3ddd822 100644
--- a/drivers/gpu/drm/drm_ioctl.c
+++ b/drivers/gpu/drm/drm_ioctl.c
@@ -245,6 +245,9 @@ static int drm_getcap(struct drm_device *dev, void *data, struct drm_file *file_
 	case DRM_CAP_SYNCOBJ:
 		req->value = drm_core_check_feature(dev, DRIVER_SYNCOBJ);
 		return 0;
+	case DRM_CAP_SYNCOBJ_TIMELINE:
+		req->value = drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE);
+		return 0;
 	}
 
 	/* Other caps only work with KMS drivers */
@@ -508,13 +511,6 @@ int drm_version(struct drm_device *dev, void *data,
 	return err;
 }
 
-static inline bool
-drm_render_driver_and_ioctl(const struct drm_device *dev, u32 flags)
-{
-	return drm_core_check_feature(dev, DRIVER_RENDER) &&
-		(flags & DRM_RENDER_ALLOW);
-}
-
 /**
  * drm_ioctl_permit - Check ioctl permissions against caller
  *
@@ -529,19 +525,14 @@ drm_render_driver_and_ioctl(const struct drm_device *dev, u32 flags)
  */
 int drm_ioctl_permit(u32 flags, struct drm_file *file_priv)
 {
-	const struct drm_device *dev = file_priv->minor->dev;
-
 	/* ROOT_ONLY is only for CAP_SYS_ADMIN */
 	if (unlikely((flags & DRM_ROOT_ONLY) && !capable(CAP_SYS_ADMIN)))
 		return -EACCES;
 
-	/* AUTH is only for master ... */
-	if (unlikely((flags & DRM_AUTH) && drm_is_primary_client(file_priv))) {
-		/* authenticated ones, or render capable on DRM_RENDER_ALLOW. */
-		if (!file_priv->authenticated &&
-		    !drm_render_driver_and_ioctl(dev, flags))
-			return -EACCES;
-	}
+	/* AUTH is only for authenticated or render client */
+	if (unlikely((flags & DRM_AUTH) && !drm_is_render_client(file_priv) &&
+		     !file_priv->authenticated))
+		return -EACCES;
 
 	/* MASTER is only for master or control clients */
 	if (unlikely((flags & DRM_MASTER) &&
@@ -565,6 +556,12 @@ EXPORT_SYMBOL(drm_ioctl_permit);
 		.name = #ioctl			\
 	}
 
+#if IS_ENABLED(CONFIG_DRM_LEGACY)
+#define DRM_LEGACY_IOCTL_DEF(ioctl, _func, _flags)  DRM_IOCTL_DEF(ioctl, _func, _flags)
+#else
+#define DRM_LEGACY_IOCTL_DEF(ioctl, _func, _flags) DRM_IOCTL_DEF(ioctl, drm_invalid_op, _flags)
+#endif
+
 /* Ioctl table */
 static const struct drm_ioctl_desc drm_ioctls[] = {
 	DRM_IOCTL_DEF(DRM_IOCTL_VERSION, drm_version,
@@ -572,7 +569,9 @@ static const struct drm_ioctl_desc drm_ioctls[] = {
 	DRM_IOCTL_DEF(DRM_IOCTL_GET_UNIQUE, drm_getunique, DRM_UNLOCKED),
 	DRM_IOCTL_DEF(DRM_IOCTL_GET_MAGIC, drm_getmagic, DRM_UNLOCKED),
 	DRM_IOCTL_DEF(DRM_IOCTL_IRQ_BUSID, drm_irq_by_busid, DRM_MASTER|DRM_ROOT_ONLY),
-	DRM_IOCTL_DEF(DRM_IOCTL_GET_MAP, drm_legacy_getmap_ioctl, DRM_UNLOCKED),
+
+	DRM_LEGACY_IOCTL_DEF(DRM_IOCTL_GET_MAP, drm_legacy_getmap_ioctl, DRM_UNLOCKED),
+
 	DRM_IOCTL_DEF(DRM_IOCTL_GET_CLIENT, drm_getclient, DRM_UNLOCKED),
 	DRM_IOCTL_DEF(DRM_IOCTL_GET_STATS, drm_getstats, DRM_UNLOCKED),
 	DRM_IOCTL_DEF(DRM_IOCTL_GET_CAP, drm_getcap, DRM_UNLOCKED|DRM_RENDER_ALLOW),
@@ -584,39 +583,38 @@ static const struct drm_ioctl_desc drm_ioctls[] = {
 	DRM_IOCTL_DEF(DRM_IOCTL_UNBLOCK, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
 	DRM_IOCTL_DEF(DRM_IOCTL_AUTH_MAGIC, drm_authmagic, DRM_UNLOCKED|DRM_MASTER),
 
-	DRM_IOCTL_DEF(DRM_IOCTL_ADD_MAP, drm_legacy_addmap_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
-	DRM_IOCTL_DEF(DRM_IOCTL_RM_MAP, drm_legacy_rmmap_ioctl, DRM_AUTH),
+	DRM_LEGACY_IOCTL_DEF(DRM_IOCTL_ADD_MAP, drm_legacy_addmap_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_LEGACY_IOCTL_DEF(DRM_IOCTL_RM_MAP, drm_legacy_rmmap_ioctl, DRM_AUTH),
 
-	DRM_IOCTL_DEF(DRM_IOCTL_SET_SAREA_CTX, drm_legacy_setsareactx, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
-	DRM_IOCTL_DEF(DRM_IOCTL_GET_SAREA_CTX, drm_legacy_getsareactx, DRM_AUTH),
+	DRM_LEGACY_IOCTL_DEF(DRM_IOCTL_SET_SAREA_CTX, drm_legacy_setsareactx, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_LEGACY_IOCTL_DEF(DRM_IOCTL_GET_SAREA_CTX, drm_legacy_getsareactx, DRM_AUTH),
 
 	DRM_IOCTL_DEF(DRM_IOCTL_SET_MASTER, drm_setmaster_ioctl, DRM_UNLOCKED|DRM_ROOT_ONLY),
 	DRM_IOCTL_DEF(DRM_IOCTL_DROP_MASTER, drm_dropmaster_ioctl, DRM_UNLOCKED|DRM_ROOT_ONLY),
 
-	DRM_IOCTL_DEF(DRM_IOCTL_ADD_CTX, drm_legacy_addctx, DRM_AUTH|DRM_ROOT_ONLY),
-	DRM_IOCTL_DEF(DRM_IOCTL_RM_CTX, drm_legacy_rmctx, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
-	DRM_IOCTL_DEF(DRM_IOCTL_MOD_CTX, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
-	DRM_IOCTL_DEF(DRM_IOCTL_GET_CTX, drm_legacy_getctx, DRM_AUTH),
-	DRM_IOCTL_DEF(DRM_IOCTL_SWITCH_CTX, drm_legacy_switchctx, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
-	DRM_IOCTL_DEF(DRM_IOCTL_NEW_CTX, drm_legacy_newctx, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
-	DRM_IOCTL_DEF(DRM_IOCTL_RES_CTX, drm_legacy_resctx, DRM_AUTH),
+	DRM_LEGACY_IOCTL_DEF(DRM_IOCTL_ADD_CTX, drm_legacy_addctx, DRM_AUTH|DRM_ROOT_ONLY),
+	DRM_LEGACY_IOCTL_DEF(DRM_IOCTL_RM_CTX, drm_legacy_rmctx, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_LEGACY_IOCTL_DEF(DRM_IOCTL_MOD_CTX, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_LEGACY_IOCTL_DEF(DRM_IOCTL_GET_CTX, drm_legacy_getctx, DRM_AUTH),
+	DRM_LEGACY_IOCTL_DEF(DRM_IOCTL_SWITCH_CTX, drm_legacy_switchctx, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_LEGACY_IOCTL_DEF(DRM_IOCTL_NEW_CTX, drm_legacy_newctx, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_LEGACY_IOCTL_DEF(DRM_IOCTL_RES_CTX, drm_legacy_resctx, DRM_AUTH),
 
 	DRM_IOCTL_DEF(DRM_IOCTL_ADD_DRAW, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
 	DRM_IOCTL_DEF(DRM_IOCTL_RM_DRAW, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
 
-	DRM_IOCTL_DEF(DRM_IOCTL_LOCK, drm_legacy_lock, DRM_AUTH),
-	DRM_IOCTL_DEF(DRM_IOCTL_UNLOCK, drm_legacy_unlock, DRM_AUTH),
+	DRM_LEGACY_IOCTL_DEF(DRM_IOCTL_LOCK, drm_legacy_lock, DRM_AUTH),
+	DRM_LEGACY_IOCTL_DEF(DRM_IOCTL_UNLOCK, drm_legacy_unlock, DRM_AUTH),
 
 	DRM_IOCTL_DEF(DRM_IOCTL_FINISH, drm_noop, DRM_AUTH),
 
-	DRM_IOCTL_DEF(DRM_IOCTL_ADD_BUFS, drm_legacy_addbufs, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
-	DRM_IOCTL_DEF(DRM_IOCTL_MARK_BUFS, drm_legacy_markbufs, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
-	DRM_IOCTL_DEF(DRM_IOCTL_INFO_BUFS, drm_legacy_infobufs, DRM_AUTH),
-	DRM_IOCTL_DEF(DRM_IOCTL_MAP_BUFS, drm_legacy_mapbufs, DRM_AUTH),
-	DRM_IOCTL_DEF(DRM_IOCTL_FREE_BUFS, drm_legacy_freebufs, DRM_AUTH),
-	DRM_IOCTL_DEF(DRM_IOCTL_DMA, drm_legacy_dma_ioctl, DRM_AUTH),
-
-	DRM_IOCTL_DEF(DRM_IOCTL_CONTROL, drm_legacy_irq_control, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_LEGACY_IOCTL_DEF(DRM_IOCTL_ADD_BUFS, drm_legacy_addbufs, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_LEGACY_IOCTL_DEF(DRM_IOCTL_MARK_BUFS, drm_legacy_markbufs, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_LEGACY_IOCTL_DEF(DRM_IOCTL_INFO_BUFS, drm_legacy_infobufs, DRM_AUTH),
+	DRM_LEGACY_IOCTL_DEF(DRM_IOCTL_MAP_BUFS, drm_legacy_mapbufs, DRM_AUTH),
+	DRM_LEGACY_IOCTL_DEF(DRM_IOCTL_FREE_BUFS, drm_legacy_freebufs, DRM_AUTH),
+	DRM_LEGACY_IOCTL_DEF(DRM_IOCTL_DMA, drm_legacy_dma_ioctl, DRM_AUTH),
+	DRM_LEGACY_IOCTL_DEF(DRM_IOCTL_CONTROL, drm_legacy_irq_control, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
 
 #if IS_ENABLED(CONFIG_AGP)
 	DRM_IOCTL_DEF(DRM_IOCTL_AGP_ACQUIRE, drm_agp_acquire_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
@@ -629,8 +627,8 @@ static const struct drm_ioctl_desc drm_ioctls[] = {
 	DRM_IOCTL_DEF(DRM_IOCTL_AGP_UNBIND, drm_agp_unbind_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
 #endif
 
-	DRM_IOCTL_DEF(DRM_IOCTL_SG_ALLOC, drm_legacy_sg_alloc, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
-	DRM_IOCTL_DEF(DRM_IOCTL_SG_FREE, drm_legacy_sg_free, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_LEGACY_IOCTL_DEF(DRM_IOCTL_SG_ALLOC, drm_legacy_sg_alloc, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+	DRM_LEGACY_IOCTL_DEF(DRM_IOCTL_SG_FREE, drm_legacy_sg_free, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
 
 	DRM_IOCTL_DEF(DRM_IOCTL_WAIT_VBLANK, drm_wait_vblank_ioctl, DRM_UNLOCKED),
 
@@ -686,12 +684,20 @@ static const struct drm_ioctl_desc drm_ioctls[] = {
 		      DRM_UNLOCKED|DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF(DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE, drm_syncobj_fd_to_handle_ioctl,
 		      DRM_UNLOCKED|DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF(DRM_IOCTL_SYNCOBJ_TRANSFER, drm_syncobj_transfer_ioctl,
+		      DRM_UNLOCKED|DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF(DRM_IOCTL_SYNCOBJ_WAIT, drm_syncobj_wait_ioctl,
 		      DRM_UNLOCKED|DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF(DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT, drm_syncobj_timeline_wait_ioctl,
+		      DRM_UNLOCKED|DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF(DRM_IOCTL_SYNCOBJ_RESET, drm_syncobj_reset_ioctl,
 		      DRM_UNLOCKED|DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF(DRM_IOCTL_SYNCOBJ_SIGNAL, drm_syncobj_signal_ioctl,
 		      DRM_UNLOCKED|DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF(DRM_IOCTL_SYNCOBJ_TIMELINE_SIGNAL, drm_syncobj_timeline_signal_ioctl,
+		      DRM_UNLOCKED|DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF(DRM_IOCTL_SYNCOBJ_QUERY, drm_syncobj_query_ioctl,
+		      DRM_UNLOCKED|DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF(DRM_IOCTL_CRTC_GET_SEQUENCE, drm_crtc_get_sequence_ioctl, DRM_UNLOCKED),
 	DRM_IOCTL_DEF(DRM_IOCTL_CRTC_QUEUE_SEQUENCE, drm_crtc_queue_sequence_ioctl, DRM_UNLOCKED),
 	DRM_IOCTL_DEF(DRM_IOCTL_MODE_CREATE_LEASE, drm_mode_create_lease_ioctl, DRM_MASTER|DRM_UNLOCKED),
diff --git a/drivers/gpu/drm/drm_irq.c b/drivers/gpu/drm/drm_irq.c
index 9bd8908d5fd8..02f38cc9f468 100644
--- a/drivers/gpu/drm/drm_irq.c
+++ b/drivers/gpu/drm/drm_irq.c
@@ -213,6 +213,7 @@ int drm_irq_uninstall(struct drm_device *dev)
 }
 EXPORT_SYMBOL(drm_irq_uninstall);
 
+#if IS_ENABLED(CONFIG_DRM_LEGACY)
 int drm_legacy_irq_control(struct drm_device *dev, void *data,
 			   struct drm_file *file_priv)
 {
@@ -253,3 +254,4 @@ int drm_legacy_irq_control(struct drm_device *dev, void *data,
 		return -EINVAL;
 	}
 }
+#endif
diff --git a/drivers/gpu/drm/drm_kms_helper_common.c b/drivers/gpu/drm/drm_kms_helper_common.c
index 93e2b30fe1a5..9c5ae825c507 100644
--- a/drivers/gpu/drm/drm_kms_helper_common.c
+++ b/drivers/gpu/drm/drm_kms_helper_common.c
@@ -39,7 +39,7 @@ MODULE_LICENSE("GPL and additional rights");
 /* Backward compatibility for drm_kms_helper.edid_firmware */
 static int edid_firmware_set(const char *val, const struct kernel_param *kp)
 {
-	DRM_NOTE("drm_kms_firmware.edid_firmware is deprecated, please use drm.edid_firmware intead.\n");
+	DRM_NOTE("drm_kms_firmware.edid_firmware is deprecated, please use drm.edid_firmware instead.\n");
 
 	return __drm_set_edid_firmware_path(val);
 }
diff --git a/drivers/gpu/drm/drm_lease.c b/drivers/gpu/drm/drm_lease.c
index 603b0bd9c5ce..694ff363a90b 100644
--- a/drivers/gpu/drm/drm_lease.c
+++ b/drivers/gpu/drm/drm_lease.c
@@ -111,7 +111,7 @@ static bool _drm_has_leased(struct drm_master *master, int id)
  */
 bool _drm_lease_held(struct drm_file *file_priv, int id)
 {
-	if (file_priv == NULL || file_priv->master == NULL)
+	if (!file_priv || !file_priv->master)
 		return true;
 
 	return _drm_lease_held_master(file_priv->master, id);
@@ -133,7 +133,7 @@ bool drm_lease_held(struct drm_file *file_priv, int id)
 	struct drm_master *master;
 	bool ret;
 
-	if (file_priv == NULL || file_priv->master == NULL)
+	if (!file_priv || !file_priv->master || !file_priv->master->lessor)
 		return true;
 
 	master = file_priv->master;
@@ -159,7 +159,7 @@ uint32_t drm_lease_filter_crtcs(struct drm_file *file_priv, uint32_t crtcs_in)
 	int count_in, count_out;
 	uint32_t crtcs_out = 0;
 
-	if (file_priv == NULL || file_priv->master == NULL)
+	if (!file_priv || !file_priv->master || !file_priv->master->lessor)
 		return crtcs_in;
 
 	master = file_priv->master;
@@ -220,8 +220,6 @@ static struct drm_master *drm_lease_create(struct drm_master *lessor, struct idr
 		error = 0;
 		if (!idr_find(&dev->mode_config.object_idr, object))
 			error = -ENOENT;
-		else if (!_drm_lease_held_master(lessor, object))
-			error = -EACCES;
 		else if (_drm_has_leased(lessor, object))
 			error = -EBUSY;
 
@@ -403,11 +401,6 @@ static int fill_object_idr(struct drm_device *dev,
 	/* step one - get references to all the mode objects
 	   and check for validity. */
 	for (o = 0; o < object_count; o++) {
-		if ((int) object_ids[o] < 0) {
-			ret = -EINVAL;
-			goto out_free_objects;
-		}
-
 		objects[o] = drm_mode_object_find(dev, lessor_priv,
 						  object_ids[o],
 						  DRM_MODE_OBJECT_ANY);
diff --git a/drivers/gpu/drm/drm_legacy.h b/drivers/gpu/drm/drm_legacy.h
index 280fbeb846ff..51f1fabfa145 100644
--- a/drivers/gpu/drm/drm_legacy.h
+++ b/drivers/gpu/drm/drm_legacy.h
@@ -42,11 +42,19 @@ struct drm_file;
 #define DRM_KERNEL_CONTEXT		0
 #define DRM_RESERVED_CONTEXTS		1
 
+#if IS_ENABLED(CONFIG_DRM_LEGACY)
 void drm_legacy_ctxbitmap_init(struct drm_device *dev);
 void drm_legacy_ctxbitmap_cleanup(struct drm_device *dev);
-void drm_legacy_ctxbitmap_free(struct drm_device *dev, int ctx_handle);
 void drm_legacy_ctxbitmap_flush(struct drm_device *dev, struct drm_file *file);
+#else
+static inline void drm_legacy_ctxbitmap_init(struct drm_device *dev) {}
+static inline void drm_legacy_ctxbitmap_cleanup(struct drm_device *dev) {}
+static inline void drm_legacy_ctxbitmap_flush(struct drm_device *dev, struct drm_file *file) {}
+#endif
 
+void drm_legacy_ctxbitmap_free(struct drm_device *dev, int ctx_handle);
+
+#if IS_ENABLED(CONFIG_DRM_LEGACY)
 int drm_legacy_resctx(struct drm_device *d, void *v, struct drm_file *f);
 int drm_legacy_addctx(struct drm_device *d, void *v, struct drm_file *f);
 int drm_legacy_getctx(struct drm_device *d, void *v, struct drm_file *f);
@@ -56,6 +64,7 @@ int drm_legacy_rmctx(struct drm_device *d, void *v, struct drm_file *f);
 
 int drm_legacy_setsareactx(struct drm_device *d, void *v, struct drm_file *f);
 int drm_legacy_getsareactx(struct drm_device *d, void *v, struct drm_file *f);
+#endif
 
 /*
  * Generic Buffer Management
@@ -63,16 +72,39 @@ int drm_legacy_getsareactx(struct drm_device *d, void *v, struct drm_file *f);
 
 #define DRM_MAP_HASH_OFFSET 0x10000000
 
+#if IS_ENABLED(CONFIG_DRM_LEGACY)
+static inline int drm_legacy_create_map_hash(struct drm_device *dev)
+{
+	return drm_ht_create(&dev->map_hash, 12);
+}
+
+static inline void drm_legacy_remove_map_hash(struct drm_device *dev)
+{
+	drm_ht_remove(&dev->map_hash);
+}
+#else
+static inline int drm_legacy_create_map_hash(struct drm_device *dev)
+{
+	return 0;
+}
+
+static inline void drm_legacy_remove_map_hash(struct drm_device *dev) {}
+#endif
+
+
+#if IS_ENABLED(CONFIG_DRM_LEGACY)
 int drm_legacy_getmap_ioctl(struct drm_device *dev, void *data,
 			    struct drm_file *file_priv);
 int drm_legacy_addmap_ioctl(struct drm_device *d, void *v, struct drm_file *f);
 int drm_legacy_rmmap_ioctl(struct drm_device *d, void *v, struct drm_file *f);
+
 int drm_legacy_addbufs(struct drm_device *d, void *v, struct drm_file *f);
 int drm_legacy_infobufs(struct drm_device *d, void *v, struct drm_file *f);
 int drm_legacy_markbufs(struct drm_device *d, void *v, struct drm_file *f);
 int drm_legacy_freebufs(struct drm_device *d, void *v, struct drm_file *f);
 int drm_legacy_mapbufs(struct drm_device *d, void *v, struct drm_file *f);
 int drm_legacy_dma_ioctl(struct drm_device *d, void *v, struct drm_file *f);
+#endif
 
 int __drm_legacy_infobufs(struct drm_device *, void *, int *,
 			  int (*)(void *, int, struct drm_buf_entry *));
@@ -81,7 +113,17 @@ int __drm_legacy_mapbufs(struct drm_device *, void *, int *,
 			  int (*)(void *, int, unsigned long, struct drm_buf *),
 			  struct drm_file *);
 
-#ifdef CONFIG_DRM_VM
+#if IS_ENABLED(CONFIG_DRM_LEGACY)
+void drm_legacy_master_rmmaps(struct drm_device *dev,
+			      struct drm_master *master);
+void drm_legacy_rmmaps(struct drm_device *dev);
+#else
+static inline void drm_legacy_master_rmmaps(struct drm_device *dev,
+					    struct drm_master *master) {}
+static inline void drm_legacy_rmmaps(struct drm_device *dev) {}
+#endif
+
+#if IS_ENABLED(CONFIG_DRM_VM) && IS_ENABLED(CONFIG_DRM_LEGACY)
 void drm_legacy_vma_flush(struct drm_device *d);
 #else
 static inline void drm_legacy_vma_flush(struct drm_device *d)
@@ -103,23 +145,64 @@ struct drm_agp_mem {
 };
 
 /* drm_lock.c */
+#if IS_ENABLED(CONFIG_DRM_LEGACY)
 int drm_legacy_lock(struct drm_device *d, void *v, struct drm_file *f);
 int drm_legacy_unlock(struct drm_device *d, void *v, struct drm_file *f);
 void drm_legacy_lock_release(struct drm_device *dev, struct file *filp);
+#else
+static inline void drm_legacy_lock_release(struct drm_device *dev, struct file *filp) {}
+#endif
 
 /* DMA support */
+#if IS_ENABLED(CONFIG_DRM_LEGACY)
 int drm_legacy_dma_setup(struct drm_device *dev);
 void drm_legacy_dma_takedown(struct drm_device *dev);
+#else
+static inline int drm_legacy_dma_setup(struct drm_device *dev)
+{
+	return 0;
+}
+#endif
+
 void drm_legacy_free_buffer(struct drm_device *dev,
 			    struct drm_buf * buf);
+#if IS_ENABLED(CONFIG_DRM_LEGACY)
 void drm_legacy_reclaim_buffers(struct drm_device *dev,
 				struct drm_file *filp);
+#else
+static inline void drm_legacy_reclaim_buffers(struct drm_device *dev,
+					      struct drm_file *filp) {}
+#endif
 
 /* Scatter Gather Support */
+#if IS_ENABLED(CONFIG_DRM_LEGACY)
 void drm_legacy_sg_cleanup(struct drm_device *dev);
 int drm_legacy_sg_alloc(struct drm_device *dev, void *data,
 			struct drm_file *file_priv);
 int drm_legacy_sg_free(struct drm_device *dev, void *data,
 		       struct drm_file *file_priv);
+#endif
+
+#if IS_ENABLED(CONFIG_DRM_LEGACY)
+void drm_legacy_init_members(struct drm_device *dev);
+void drm_legacy_destroy_members(struct drm_device *dev);
+void drm_legacy_dev_reinit(struct drm_device *dev);
+#else
+static inline void drm_legacy_init_members(struct drm_device *dev) {}
+static inline void drm_legacy_destroy_members(struct drm_device *dev) {}
+static inline void drm_legacy_dev_reinit(struct drm_device *dev) {}
+#endif
+
+#if IS_ENABLED(CONFIG_DRM_LEGACY)
+void drm_legacy_lock_master_cleanup(struct drm_device *dev, struct drm_master *master);
+#else
+static inline void drm_legacy_lock_master_cleanup(struct drm_device *dev, struct drm_master *master) {}
+#endif
+
+#if IS_ENABLED(CONFIG_DRM_LEGACY)
+void drm_master_legacy_init(struct drm_master *master);
+#else
+static inline void drm_master_legacy_init(struct drm_master *master) {}
+#endif
 
 #endif /* __DRM_LEGACY_H__ */
diff --git a/drivers/gpu/drm/drm_legacy_misc.c b/drivers/gpu/drm/drm_legacy_misc.c
new file mode 100644
index 000000000000..2fe786839ca8
--- /dev/null
+++ b/drivers/gpu/drm/drm_legacy_misc.c
@@ -0,0 +1,82 @@
+/**
+ * \file drm_legacy_misc.c
+ * Misc legacy support functions.
+ *
+ * \author Rickard E. (Rik) Faith <faith@valinux.com>
+ * \author Gareth Hughes <gareth@valinux.com>
+ */
+
+/*
+ * Created: Tue Feb  2 08:37:54 1999 by faith@valinux.com
+ *
+ * Copyright 1999 Precision Insight, Inc., Cedar Park, Texas.
+ * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <drm/drmP.h>
+#include "drm_internal.h"
+#include "drm_legacy.h"
+
+void drm_legacy_init_members(struct drm_device *dev)
+{
+	INIT_LIST_HEAD(&dev->ctxlist);
+	INIT_LIST_HEAD(&dev->vmalist);
+	INIT_LIST_HEAD(&dev->maplist);
+	spin_lock_init(&dev->buf_lock);
+	mutex_init(&dev->ctxlist_mutex);
+}
+
+void drm_legacy_destroy_members(struct drm_device *dev)
+{
+	mutex_destroy(&dev->ctxlist_mutex);
+}
+
+void drm_legacy_dev_reinit(struct drm_device *dev)
+{
+	if (dev->irq_enabled)
+		drm_irq_uninstall(dev);
+
+	mutex_lock(&dev->struct_mutex);
+
+	drm_legacy_agp_clear(dev);
+
+	drm_legacy_sg_cleanup(dev);
+	drm_legacy_vma_flush(dev);
+	drm_legacy_dma_takedown(dev);
+
+	mutex_unlock(&dev->struct_mutex);
+
+	dev->sigdata.lock = NULL;
+
+	dev->context_flag = 0;
+	dev->last_context = 0;
+	dev->if_version = 0;
+
+	DRM_DEBUG("lastclose completed\n");
+}
+
+void drm_master_legacy_init(struct drm_master *master)
+{
+	spin_lock_init(&master->lock.spinlock);
+	init_waitqueue_head(&master->lock.lock_queue);
+}
diff --git a/drivers/gpu/drm/drm_lock.c b/drivers/gpu/drm/drm_lock.c
index 67a1a2ca7174..b70058e77a28 100644
--- a/drivers/gpu/drm/drm_lock.c
+++ b/drivers/gpu/drm/drm_lock.c
@@ -347,3 +347,22 @@ void drm_legacy_lock_release(struct drm_device *dev, struct file *filp)
 				     _DRM_LOCKING_CONTEXT(file_priv->master->lock.hw_lock->lock));
 	}
 }
+
+void drm_legacy_lock_master_cleanup(struct drm_device *dev, struct drm_master *master)
+{
+	if (!drm_core_check_feature(dev, DRIVER_LEGACY))
+		return;
+
+	/*
+	 * Since the master is disappearing, so is the
+	 * possibility to lock.
+	 */	mutex_lock(&dev->struct_mutex);
+	if (master->lock.hw_lock) {
+		if (dev->sigdata.lock == master->lock.hw_lock)
+			dev->sigdata.lock = NULL;
+		master->lock.hw_lock = NULL;
+		master->lock.file_priv = NULL;
+		wake_up_interruptible_all(&master->lock.lock_queue);
+	}
+	mutex_unlock(&dev->struct_mutex);
+}
diff --git a/drivers/gpu/drm/drm_memory.c b/drivers/gpu/drm/drm_memory.c
index 40c4349cb939..132fef8ff1b6 100644
--- a/drivers/gpu/drm/drm_memory.c
+++ b/drivers/gpu/drm/drm_memory.c
@@ -35,6 +35,7 @@
 
 #include <linux/highmem.h>
 #include <linux/export.h>
+#include <xen/xen.h>
 #include <drm/drmP.h>
 #include "drm_legacy.h"
 
@@ -150,15 +151,34 @@ void drm_legacy_ioremapfree(struct drm_local_map *map, struct drm_device *dev)
 }
 EXPORT_SYMBOL(drm_legacy_ioremapfree);
 
-u64 drm_get_max_iomem(void)
+bool drm_need_swiotlb(int dma_bits)
 {
 	struct resource *tmp;
 	resource_size_t max_iomem = 0;
 
+	/*
+	 * Xen paravirtual hosts require swiotlb regardless of requested dma
+	 * transfer size.
+	 *
+	 * NOTE: Really, what it requires is use of the dma_alloc_coherent
+	 *       allocator used in ttm_dma_populate() instead of
+	 *       ttm_populate_and_map_pages(), which bounce buffers so much in
+	 *       Xen it leads to swiotlb buffer exhaustion.
+	 */
+	if (xen_pv_domain())
+		return true;
+
+	/*
+	 * Enforce dma_alloc_coherent when memory encryption is active as well
+	 * for the same reasons as for Xen paravirtual hosts.
+	 */
+	if (mem_encrypt_active())
+		return true;
+
 	for (tmp = iomem_resource.child; tmp; tmp = tmp->sibling) {
 		max_iomem = max(max_iomem,  tmp->end);
 	}
 
-	return max_iomem;
+	return max_iomem > ((u64)1 << dma_bits);
 }
-EXPORT_SYMBOL(drm_get_max_iomem);
+EXPORT_SYMBOL(drm_need_swiotlb);
diff --git a/drivers/gpu/drm/drm_mode_config.c b/drivers/gpu/drm/drm_mode_config.c
index 4a1c2023ccf0..1a346ae1599d 100644
--- a/drivers/gpu/drm/drm_mode_config.c
+++ b/drivers/gpu/drm/drm_mode_config.c
@@ -297,8 +297,9 @@ static int drm_mode_create_standard_properties(struct drm_device *dev)
 		return -ENOMEM;
 	dev->mode_config.prop_crtc_id = prop;
 
-	prop = drm_property_create(dev, DRM_MODE_PROP_BLOB, "FB_DAMAGE_CLIPS",
-				   0);
+	prop = drm_property_create(dev,
+			DRM_MODE_PROP_ATOMIC | DRM_MODE_PROP_BLOB,
+			"FB_DAMAGE_CLIPS", 0);
 	if (!prop)
 		return -ENOMEM;
 	dev->mode_config.prop_fb_damage_clips = prop;
diff --git a/drivers/gpu/drm/drm_mode_object.c b/drivers/gpu/drm/drm_mode_object.c
index a9005c1c2384..f32507e65b79 100644
--- a/drivers/gpu/drm/drm_mode_object.c
+++ b/drivers/gpu/drm/drm_mode_object.c
@@ -451,6 +451,7 @@ static int set_property_legacy(struct drm_mode_object *obj,
 }
 
 static int set_property_atomic(struct drm_mode_object *obj,
+			       struct drm_file *file_priv,
 			       struct drm_property *prop,
 			       uint64_t prop_value)
 {
@@ -477,7 +478,7 @@ retry:
 						       obj_to_connector(obj),
 						       prop_value);
 	} else {
-		ret = drm_atomic_set_property(state, obj, prop, prop_value);
+		ret = drm_atomic_set_property(state, file_priv, obj, prop, prop_value);
 		if (ret)
 			goto out;
 		ret = drm_atomic_commit(state);
@@ -520,7 +521,7 @@ int drm_mode_obj_set_property_ioctl(struct drm_device *dev, void *data,
 		goto out_unref;
 
 	if (drm_drv_uses_atomic_modeset(property->dev))
-		ret = set_property_atomic(arg_obj, property, arg->value);
+		ret = set_property_atomic(arg_obj, file_priv, property, arg->value);
 	else
 		ret = set_property_legacy(arg_obj, property, arg->value);
 
diff --git a/drivers/gpu/drm/drm_modes.c b/drivers/gpu/drm/drm_modes.c
index 869ac6f4671e..56f92a0bba62 100644
--- a/drivers/gpu/drm/drm_modes.c
+++ b/drivers/gpu/drm/drm_modes.c
@@ -655,22 +655,22 @@ EXPORT_SYMBOL_GPL(drm_display_mode_to_videomode);
  * @bus_flags: information about pixelclk, sync and DE polarity will be stored
  * here
  *
- * Sets DRM_BUS_FLAG_DE_(LOW|HIGH),  DRM_BUS_FLAG_PIXDATA_(POS|NEG)EDGE and
- * DISPLAY_FLAGS_SYNC_(POS|NEG)EDGE in @bus_flags according to DISPLAY_FLAGS
+ * Sets DRM_BUS_FLAG_DE_(LOW|HIGH),  DRM_BUS_FLAG_PIXDATA_DRIVE_(POS|NEG)EDGE
+ * and DISPLAY_FLAGS_SYNC_(POS|NEG)EDGE in @bus_flags according to DISPLAY_FLAGS
  * found in @vm
  */
 void drm_bus_flags_from_videomode(const struct videomode *vm, u32 *bus_flags)
 {
 	*bus_flags = 0;
 	if (vm->flags & DISPLAY_FLAGS_PIXDATA_POSEDGE)
-		*bus_flags |= DRM_BUS_FLAG_PIXDATA_POSEDGE;
+		*bus_flags |= DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE;
 	if (vm->flags & DISPLAY_FLAGS_PIXDATA_NEGEDGE)
-		*bus_flags |= DRM_BUS_FLAG_PIXDATA_NEGEDGE;
+		*bus_flags |= DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE;
 
 	if (vm->flags & DISPLAY_FLAGS_SYNC_POSEDGE)
-		*bus_flags |= DRM_BUS_FLAG_SYNC_POSEDGE;
+		*bus_flags |= DRM_BUS_FLAG_SYNC_DRIVE_POSEDGE;
 	if (vm->flags & DISPLAY_FLAGS_SYNC_NEGEDGE)
-		*bus_flags |= DRM_BUS_FLAG_SYNC_NEGEDGE;
+		*bus_flags |= DRM_BUS_FLAG_SYNC_DRIVE_NEGEDGE;
 
 	if (vm->flags & DISPLAY_FLAGS_DE_LOW)
 		*bus_flags |= DRM_BUS_FLAG_DE_LOW;
diff --git a/drivers/gpu/drm/drm_panel_orientation_quirks.c b/drivers/gpu/drm/drm_panel_orientation_quirks.c
index 52e445bb1aa5..521aff99b08a 100644
--- a/drivers/gpu/drm/drm_panel_orientation_quirks.c
+++ b/drivers/gpu/drm/drm_panel_orientation_quirks.c
@@ -80,6 +80,12 @@ static const struct drm_dmi_panel_orientation_data lcd800x1280_rightside_up = {
 	.orientation = DRM_MODE_PANEL_ORIENTATION_RIGHT_UP,
 };
 
+static const struct drm_dmi_panel_orientation_data lcd1200x1920_rightside_up = {
+	.width = 1200,
+	.height = 1920,
+	.orientation = DRM_MODE_PANEL_ORIENTATION_RIGHT_UP,
+};
+
 static const struct dmi_system_id orientation_data[] = {
 	{	/* Acer One 10 (S1003) */
 		.matches = {
@@ -148,6 +154,13 @@ static const struct dmi_system_id orientation_data[] = {
 		  DMI_EXACT_MATCH(DMI_PRODUCT_VERSION, "Lenovo MIIX 320-10ICR"),
 		},
 		.driver_data = (void *)&lcd800x1280_rightside_up,
+	}, {	/* Lenovo Ideapad D330 */
+		.matches = {
+		  DMI_EXACT_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+		  DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "81H3"),
+		  DMI_EXACT_MATCH(DMI_PRODUCT_VERSION, "Lenovo ideapad D330-10IGM"),
+		},
+		.driver_data = (void *)&lcd1200x1920_rightside_up,
 	}, {	/* VIOS LTH17 */
 		.matches = {
 		  DMI_EXACT_MATCH(DMI_SYS_VENDOR, "VIOS"),
diff --git a/drivers/gpu/drm/drm_plane.c b/drivers/gpu/drm/drm_plane.c
index 4cfb56893b7f..d6ad60ab0d38 100644
--- a/drivers/gpu/drm/drm_plane.c
+++ b/drivers/gpu/drm/drm_plane.c
@@ -960,6 +960,11 @@ retry:
 		if (ret)
 			goto out;
 
+		if (!drm_lease_held(file_priv, crtc->cursor->base.id)) {
+			ret = -EACCES;
+			goto out;
+		}
+
 		ret = drm_mode_cursor_universal(crtc, req, file_priv, &ctx);
 		goto out;
 	}
@@ -1062,6 +1067,9 @@ int drm_mode_page_flip_ioctl(struct drm_device *dev,
 
 	plane = crtc->primary;
 
+	if (!drm_lease_held(file_priv, plane->base.id))
+		return -EACCES;
+
 	if (crtc->funcs->page_flip_target) {
 		u32 current_vblank;
 		int r;
diff --git a/drivers/gpu/drm/drm_prime.c b/drivers/gpu/drm/drm_prime.c
index 231e3f6d5f41..dc079efb3b0f 100644
--- a/drivers/gpu/drm/drm_prime.c
+++ b/drivers/gpu/drm/drm_prime.c
@@ -504,6 +504,7 @@ struct dma_buf *drm_gem_prime_export(struct drm_device *dev,
 		.size = obj->size,
 		.flags = flags,
 		.priv = obj,
+		.resv = obj->resv,
 	};
 
 	if (dev->driver->gem_prime_res_obj)
diff --git a/drivers/gpu/drm/drm_print.c b/drivers/gpu/drm/drm_print.c
index 0e7fc3e7dfb4..f5cb0aabfe35 100644
--- a/drivers/gpu/drm/drm_print.c
+++ b/drivers/gpu/drm/drm_print.c
@@ -253,3 +253,31 @@ void drm_err(const char *format, ...)
 	va_end(args);
 }
 EXPORT_SYMBOL(drm_err);
+
+/**
+ * drm_print_regset32 - print the contents of registers to a
+ * &drm_printer stream.
+ *
+ * @p: the &drm printer
+ * @regset: the list of registers to print.
+ *
+ * Often in driver debug, it's useful to be able to either capture the
+ * contents of registers in the steady state using debugfs or at
+ * specific points during operation.  This lets the driver have a
+ * single list of registers for both.
+ */
+void drm_print_regset32(struct drm_printer *p, struct debugfs_regset32 *regset)
+{
+	int namelen = 0;
+	int i;
+
+	for (i = 0; i < regset->nregs; i++)
+		namelen = max(namelen, (int)strlen(regset->regs[i].name));
+
+	for (i = 0; i < regset->nregs; i++) {
+		drm_printf(p, "%*s = 0x%08x\n",
+			   namelen, regset->regs[i].name,
+			   readl(regset->base + regset->regs[i].offset));
+	}
+}
+EXPORT_SYMBOL(drm_print_regset32);
diff --git a/drivers/gpu/drm/drm_syncobj.c b/drivers/gpu/drm/drm_syncobj.c
index 8bdb4a3bd7bf..3d400905100b 100644
--- a/drivers/gpu/drm/drm_syncobj.c
+++ b/drivers/gpu/drm/drm_syncobj.c
@@ -61,6 +61,7 @@ struct syncobj_wait_entry {
 	struct task_struct *task;
 	struct dma_fence *fence;
 	struct dma_fence_cb fence_cb;
+	u64    point;
 };
 
 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
@@ -95,6 +96,8 @@ EXPORT_SYMBOL(drm_syncobj_find);
 static void drm_syncobj_fence_add_wait(struct drm_syncobj *syncobj,
 				       struct syncobj_wait_entry *wait)
 {
+	struct dma_fence *fence;
+
 	if (wait->fence)
 		return;
 
@@ -103,11 +106,15 @@ static void drm_syncobj_fence_add_wait(struct drm_syncobj *syncobj,
 	 * have the lock, try one more time just to be sure we don't add a
 	 * callback when a fence has already been set.
 	 */
-	if (syncobj->fence)
-		wait->fence = dma_fence_get(
-			rcu_dereference_protected(syncobj->fence, 1));
-	else
+	fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1));
+	if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) {
+		dma_fence_put(fence);
 		list_add_tail(&wait->node, &syncobj->cb_list);
+	} else if (!fence) {
+		wait->fence = dma_fence_get_stub();
+	} else {
+		wait->fence = fence;
+	}
 	spin_unlock(&syncobj->lock);
 }
 
@@ -123,6 +130,44 @@ static void drm_syncobj_remove_wait(struct drm_syncobj *syncobj,
 }
 
 /**
+ * drm_syncobj_add_point - add new timeline point to the syncobj
+ * @syncobj: sync object to add timeline point do
+ * @chain: chain node to use to add the point
+ * @fence: fence to encapsulate in the chain node
+ * @point: sequence number to use for the point
+ *
+ * Add the chain node as new timeline point to the syncobj.
+ */
+void drm_syncobj_add_point(struct drm_syncobj *syncobj,
+			   struct dma_fence_chain *chain,
+			   struct dma_fence *fence,
+			   uint64_t point)
+{
+	struct syncobj_wait_entry *cur, *tmp;
+	struct dma_fence *prev;
+
+	dma_fence_get(fence);
+
+	spin_lock(&syncobj->lock);
+
+	prev = drm_syncobj_fence_get(syncobj);
+	/* You are adding an unorder point to timeline, which could cause payload returned from query_ioctl is 0! */
+	if (prev && prev->seqno >= point)
+		DRM_ERROR("You are adding an unorder point to timeline!\n");
+	dma_fence_chain_init(chain, prev, fence, point);
+	rcu_assign_pointer(syncobj->fence, &chain->base);
+
+	list_for_each_entry_safe(cur, tmp, &syncobj->cb_list, node)
+		syncobj_wait_syncobj_func(syncobj, cur);
+	spin_unlock(&syncobj->lock);
+
+	/* Walk the chain once to trigger garbage collection */
+	dma_fence_chain_for_each(fence, prev);
+	dma_fence_put(prev);
+}
+EXPORT_SYMBOL(drm_syncobj_add_point);
+
+/**
  * drm_syncobj_replace_fence - replace fence in a sync object.
  * @syncobj: Sync object to replace fence in
  * @fence: fence to install in sync file.
@@ -145,10 +190,8 @@ void drm_syncobj_replace_fence(struct drm_syncobj *syncobj,
 	rcu_assign_pointer(syncobj->fence, fence);
 
 	if (fence != old_fence) {
-		list_for_each_entry_safe(cur, tmp, &syncobj->cb_list, node) {
-			list_del_init(&cur->node);
+		list_for_each_entry_safe(cur, tmp, &syncobj->cb_list, node)
 			syncobj_wait_syncobj_func(syncobj, cur);
-		}
 	}
 
 	spin_unlock(&syncobj->lock);
@@ -171,6 +214,8 @@ static void drm_syncobj_assign_null_handle(struct drm_syncobj *syncobj)
 	dma_fence_put(fence);
 }
 
+/* 5s default for wait submission */
+#define DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT 5000000000ULL
 /**
  * drm_syncobj_find_fence - lookup and reference the fence in a sync object
  * @file_private: drm file private pointer
@@ -191,16 +236,58 @@ int drm_syncobj_find_fence(struct drm_file *file_private,
 			   struct dma_fence **fence)
 {
 	struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
-	int ret = 0;
+	struct syncobj_wait_entry wait;
+	u64 timeout = nsecs_to_jiffies64(DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT);
+	int ret;
 
 	if (!syncobj)
 		return -ENOENT;
 
 	*fence = drm_syncobj_fence_get(syncobj);
-	if (!*fence) {
+	drm_syncobj_put(syncobj);
+
+	if (*fence) {
+		ret = dma_fence_chain_find_seqno(fence, point);
+		if (!ret)
+			return 0;
+		dma_fence_put(*fence);
+	} else {
 		ret = -EINVAL;
 	}
-	drm_syncobj_put(syncobj);
+
+	if (!(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
+		return ret;
+
+	memset(&wait, 0, sizeof(wait));
+	wait.task = current;
+	wait.point = point;
+	drm_syncobj_fence_add_wait(syncobj, &wait);
+
+	do {
+		set_current_state(TASK_INTERRUPTIBLE);
+		if (wait.fence) {
+			ret = 0;
+			break;
+		}
+                if (timeout == 0) {
+                        ret = -ETIME;
+                        break;
+                }
+
+		if (signal_pending(current)) {
+			ret = -ERESTARTSYS;
+			break;
+		}
+
+                timeout = schedule_timeout(timeout);
+	} while (1);
+
+	__set_current_state(TASK_RUNNING);
+	*fence = wait.fence;
+
+	if (wait.node.next)
+		drm_syncobj_remove_wait(syncobj, &wait);
+
 	return ret;
 }
 EXPORT_SYMBOL(drm_syncobj_find_fence);
@@ -592,6 +679,80 @@ drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data,
 					&args->handle);
 }
 
+static int drm_syncobj_transfer_to_timeline(struct drm_file *file_private,
+					    struct drm_syncobj_transfer *args)
+{
+	struct drm_syncobj *timeline_syncobj = NULL;
+	struct dma_fence *fence;
+	struct dma_fence_chain *chain;
+	int ret;
+
+	timeline_syncobj = drm_syncobj_find(file_private, args->dst_handle);
+	if (!timeline_syncobj) {
+		return -ENOENT;
+	}
+	ret = drm_syncobj_find_fence(file_private, args->src_handle,
+				     args->src_point, args->flags,
+				     &fence);
+	if (ret)
+		goto err;
+	chain = kzalloc(sizeof(struct dma_fence_chain), GFP_KERNEL);
+	if (!chain) {
+		ret = -ENOMEM;
+		goto err1;
+	}
+	drm_syncobj_add_point(timeline_syncobj, chain, fence, args->dst_point);
+err1:
+	dma_fence_put(fence);
+err:
+	drm_syncobj_put(timeline_syncobj);
+
+	return ret;
+}
+
+static int
+drm_syncobj_transfer_to_binary(struct drm_file *file_private,
+			       struct drm_syncobj_transfer *args)
+{
+	struct drm_syncobj *binary_syncobj = NULL;
+	struct dma_fence *fence;
+	int ret;
+
+	binary_syncobj = drm_syncobj_find(file_private, args->dst_handle);
+	if (!binary_syncobj)
+		return -ENOENT;
+	ret = drm_syncobj_find_fence(file_private, args->src_handle,
+				     args->src_point, args->flags, &fence);
+	if (ret)
+		goto err;
+	drm_syncobj_replace_fence(binary_syncobj, fence);
+	dma_fence_put(fence);
+err:
+	drm_syncobj_put(binary_syncobj);
+
+	return ret;
+}
+int
+drm_syncobj_transfer_ioctl(struct drm_device *dev, void *data,
+			   struct drm_file *file_private)
+{
+	struct drm_syncobj_transfer *args = data;
+	int ret;
+
+	if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
+		return -EOPNOTSUPP;
+
+	if (args->pad)
+		return -EINVAL;
+
+	if (args->dst_point)
+		ret = drm_syncobj_transfer_to_timeline(file_private, args);
+	else
+		ret = drm_syncobj_transfer_to_binary(file_private, args);
+
+	return ret;
+}
+
 static void syncobj_wait_fence_func(struct dma_fence *fence,
 				    struct dma_fence_cb *cb)
 {
@@ -604,13 +765,27 @@ static void syncobj_wait_fence_func(struct dma_fence *fence,
 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
 				      struct syncobj_wait_entry *wait)
 {
+	struct dma_fence *fence;
+
 	/* This happens inside the syncobj lock */
-	wait->fence = dma_fence_get(rcu_dereference_protected(syncobj->fence,
-							      lockdep_is_held(&syncobj->lock)));
+	fence = rcu_dereference_protected(syncobj->fence,
+					  lockdep_is_held(&syncobj->lock));
+	dma_fence_get(fence);
+	if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) {
+		dma_fence_put(fence);
+		return;
+	} else if (!fence) {
+		wait->fence = dma_fence_get_stub();
+	} else {
+		wait->fence = fence;
+	}
+
 	wake_up_process(wait->task);
+	list_del_init(&wait->node);
 }
 
 static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs,
+						  void __user *user_points,
 						  uint32_t count,
 						  uint32_t flags,
 						  signed long timeout,
@@ -618,12 +793,27 @@ static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs,
 {
 	struct syncobj_wait_entry *entries;
 	struct dma_fence *fence;
+	uint64_t *points;
 	uint32_t signaled_count, i;
 
-	entries = kcalloc(count, sizeof(*entries), GFP_KERNEL);
-	if (!entries)
+	points = kmalloc_array(count, sizeof(*points), GFP_KERNEL);
+	if (points == NULL)
 		return -ENOMEM;
 
+	if (!user_points) {
+		memset(points, 0, count * sizeof(uint64_t));
+
+	} else if (copy_from_user(points, user_points,
+				  sizeof(uint64_t) * count)) {
+		timeout = -EFAULT;
+		goto err_free_points;
+	}
+
+	entries = kcalloc(count, sizeof(*entries), GFP_KERNEL);
+	if (!entries) {
+		timeout = -ENOMEM;
+		goto err_free_points;
+	}
 	/* Walk the list of sync objects and initialize entries.  We do
 	 * this up-front so that we can properly return -EINVAL if there is
 	 * a syncobj with a missing fence and then never have the chance of
@@ -631,9 +821,13 @@ static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs,
 	 */
 	signaled_count = 0;
 	for (i = 0; i < count; ++i) {
+		struct dma_fence *fence;
+
 		entries[i].task = current;
-		entries[i].fence = drm_syncobj_fence_get(syncobjs[i]);
-		if (!entries[i].fence) {
+		entries[i].point = points[i];
+		fence = drm_syncobj_fence_get(syncobjs[i]);
+		if (!fence || dma_fence_chain_find_seqno(&fence, points[i])) {
+			dma_fence_put(fence);
 			if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
 				continue;
 			} else {
@@ -642,7 +836,13 @@ static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs,
 			}
 		}
 
-		if (dma_fence_is_signaled(entries[i].fence)) {
+		if (fence)
+			entries[i].fence = fence;
+		else
+			entries[i].fence = dma_fence_get_stub();
+
+		if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) ||
+		    dma_fence_is_signaled(entries[i].fence)) {
 			if (signaled_count == 0 && idx)
 				*idx = i;
 			signaled_count++;
@@ -675,7 +875,8 @@ static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs,
 			if (!fence)
 				continue;
 
-			if (dma_fence_is_signaled(fence) ||
+			if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) ||
+			    dma_fence_is_signaled(fence) ||
 			    (!entries[i].fence_cb.func &&
 			     dma_fence_add_callback(fence,
 						    &entries[i].fence_cb,
@@ -720,6 +921,9 @@ cleanup_entries:
 	}
 	kfree(entries);
 
+err_free_points:
+	kfree(points);
+
 	return timeout;
 }
 
@@ -730,7 +934,7 @@ cleanup_entries:
  *
  * Calculate the timeout in jiffies from an absolute time in sec/nsec.
  */
-static signed long drm_timeout_abs_to_jiffies(int64_t timeout_nsec)
+signed long drm_timeout_abs_to_jiffies(int64_t timeout_nsec)
 {
 	ktime_t abs_timeout, now;
 	u64 timeout_ns, timeout_jiffies64;
@@ -754,23 +958,38 @@ static signed long drm_timeout_abs_to_jiffies(int64_t timeout_nsec)
 
 	return timeout_jiffies64 + 1;
 }
+EXPORT_SYMBOL(drm_timeout_abs_to_jiffies);
 
 static int drm_syncobj_array_wait(struct drm_device *dev,
 				  struct drm_file *file_private,
 				  struct drm_syncobj_wait *wait,
-				  struct drm_syncobj **syncobjs)
+				  struct drm_syncobj_timeline_wait *timeline_wait,
+				  struct drm_syncobj **syncobjs, bool timeline)
 {
-	signed long timeout = drm_timeout_abs_to_jiffies(wait->timeout_nsec);
+	signed long timeout = 0;
 	uint32_t first = ~0;
 
-	timeout = drm_syncobj_array_wait_timeout(syncobjs,
-						 wait->count_handles,
-						 wait->flags,
-						 timeout, &first);
-	if (timeout < 0)
-		return timeout;
-
-	wait->first_signaled = first;
+	if (!timeline) {
+		timeout = drm_timeout_abs_to_jiffies(wait->timeout_nsec);
+		timeout = drm_syncobj_array_wait_timeout(syncobjs,
+							 NULL,
+							 wait->count_handles,
+							 wait->flags,
+							 timeout, &first);
+		if (timeout < 0)
+			return timeout;
+		wait->first_signaled = first;
+	} else {
+		timeout = drm_timeout_abs_to_jiffies(timeline_wait->timeout_nsec);
+		timeout = drm_syncobj_array_wait_timeout(syncobjs,
+							 u64_to_user_ptr(timeline_wait->points),
+							 timeline_wait->count_handles,
+							 timeline_wait->flags,
+							 timeout, &first);
+		if (timeout < 0)
+			return timeout;
+		timeline_wait->first_signaled = first;
+	}
 	return 0;
 }
 
@@ -856,13 +1075,48 @@ drm_syncobj_wait_ioctl(struct drm_device *dev, void *data,
 		return ret;
 
 	ret = drm_syncobj_array_wait(dev, file_private,
-				     args, syncobjs);
+				     args, NULL, syncobjs, false);
+
+	drm_syncobj_array_free(syncobjs, args->count_handles);
+
+	return ret;
+}
+
+int
+drm_syncobj_timeline_wait_ioctl(struct drm_device *dev, void *data,
+				struct drm_file *file_private)
+{
+	struct drm_syncobj_timeline_wait *args = data;
+	struct drm_syncobj **syncobjs;
+	int ret = 0;
+
+	if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
+		return -EOPNOTSUPP;
+
+	if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
+			    DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |
+			    DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE))
+		return -EINVAL;
+
+	if (args->count_handles == 0)
+		return -EINVAL;
+
+	ret = drm_syncobj_array_find(file_private,
+				     u64_to_user_ptr(args->handles),
+				     args->count_handles,
+				     &syncobjs);
+	if (ret < 0)
+		return ret;
+
+	ret = drm_syncobj_array_wait(dev, file_private,
+				     NULL, args, syncobjs, true);
 
 	drm_syncobj_array_free(syncobjs, args->count_handles);
 
 	return ret;
 }
 
+
 int
 drm_syncobj_reset_ioctl(struct drm_device *dev, void *data,
 			struct drm_file *file_private)
@@ -928,3 +1182,138 @@ drm_syncobj_signal_ioctl(struct drm_device *dev, void *data,
 
 	return ret;
 }
+
+int
+drm_syncobj_timeline_signal_ioctl(struct drm_device *dev, void *data,
+				  struct drm_file *file_private)
+{
+	struct drm_syncobj_timeline_array *args = data;
+	struct drm_syncobj **syncobjs;
+	struct dma_fence_chain **chains;
+	uint64_t *points;
+	uint32_t i, j;
+	int ret;
+
+	if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
+		return -EOPNOTSUPP;
+
+	if (args->pad != 0)
+		return -EINVAL;
+
+	if (args->count_handles == 0)
+		return -EINVAL;
+
+	ret = drm_syncobj_array_find(file_private,
+				     u64_to_user_ptr(args->handles),
+				     args->count_handles,
+				     &syncobjs);
+	if (ret < 0)
+		return ret;
+
+	points = kmalloc_array(args->count_handles, sizeof(*points),
+			       GFP_KERNEL);
+	if (!points) {
+		ret = -ENOMEM;
+		goto out;
+	}
+	if (!u64_to_user_ptr(args->points)) {
+		memset(points, 0, args->count_handles * sizeof(uint64_t));
+	} else if (copy_from_user(points, u64_to_user_ptr(args->points),
+				  sizeof(uint64_t) * args->count_handles)) {
+		ret = -EFAULT;
+		goto err_points;
+	}
+
+	chains = kmalloc_array(args->count_handles, sizeof(void *), GFP_KERNEL);
+	if (!chains) {
+		ret = -ENOMEM;
+		goto err_points;
+	}
+	for (i = 0; i < args->count_handles; i++) {
+		chains[i] = kzalloc(sizeof(struct dma_fence_chain), GFP_KERNEL);
+		if (!chains[i]) {
+			for (j = 0; j < i; j++)
+				kfree(chains[j]);
+			ret = -ENOMEM;
+			goto err_chains;
+		}
+	}
+
+	for (i = 0; i < args->count_handles; i++) {
+		struct dma_fence *fence = dma_fence_get_stub();
+
+		drm_syncobj_add_point(syncobjs[i], chains[i],
+				      fence, points[i]);
+		dma_fence_put(fence);
+	}
+err_chains:
+	kfree(chains);
+err_points:
+	kfree(points);
+out:
+	drm_syncobj_array_free(syncobjs, args->count_handles);
+
+	return ret;
+}
+
+int drm_syncobj_query_ioctl(struct drm_device *dev, void *data,
+			    struct drm_file *file_private)
+{
+	struct drm_syncobj_timeline_array *args = data;
+	struct drm_syncobj **syncobjs;
+	uint64_t __user *points = u64_to_user_ptr(args->points);
+	uint32_t i;
+	int ret;
+
+	if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
+		return -EOPNOTSUPP;
+
+	if (args->pad != 0)
+		return -EINVAL;
+
+	if (args->count_handles == 0)
+		return -EINVAL;
+
+	ret = drm_syncobj_array_find(file_private,
+				     u64_to_user_ptr(args->handles),
+				     args->count_handles,
+				     &syncobjs);
+	if (ret < 0)
+		return ret;
+
+	for (i = 0; i < args->count_handles; i++) {
+		struct dma_fence_chain *chain;
+		struct dma_fence *fence;
+		uint64_t point;
+
+		fence = drm_syncobj_fence_get(syncobjs[i]);
+		chain = to_dma_fence_chain(fence);
+		if (chain) {
+			struct dma_fence *iter, *last_signaled = NULL;
+
+			dma_fence_chain_for_each(iter, fence) {
+				if (!iter)
+					break;
+				dma_fence_put(last_signaled);
+				last_signaled = dma_fence_get(iter);
+				if (!to_dma_fence_chain(last_signaled)->prev_seqno)
+					/* It is most likely that timeline has
+					 * unorder points. */
+					break;
+			}
+			point = dma_fence_is_signaled(last_signaled) ?
+				last_signaled->seqno :
+				to_dma_fence_chain(last_signaled)->prev_seqno;
+			dma_fence_put(last_signaled);
+		} else {
+			point = 0;
+		}
+		ret = copy_to_user(&points[i], &point, sizeof(uint64_t));
+		ret = ret ? -EFAULT : 0;
+		if (ret)
+			break;
+	}
+	drm_syncobj_array_free(syncobjs, args->count_handles);
+
+	return ret;
+}
diff --git a/drivers/gpu/drm/drm_vm.c b/drivers/gpu/drm/drm_vm.c
index c3301046dfaa..10cf83d569e1 100644
--- a/drivers/gpu/drm/drm_vm.c
+++ b/drivers/gpu/drm/drm_vm.c
@@ -584,8 +584,8 @@ static int drm_mmap_locked(struct file *filp, struct vm_area_struct *vma)
 			vma->vm_ops = &drm_vm_ops;
 			break;
 		}
-		/* fall through to _DRM_FRAME_BUFFER... */
 #endif
+		/* fall through - to _DRM_FRAME_BUFFER... */
 	case _DRM_FRAME_BUFFER:
 	case _DRM_REGISTERS:
 		offset = drm_core_get_reg_ofs(dev);
@@ -610,7 +610,7 @@ static int drm_mmap_locked(struct file *filp, struct vm_area_struct *vma)
 		    vma->vm_end - vma->vm_start, vma->vm_page_prot))
 			return -EAGAIN;
 		vma->vm_page_prot = drm_dma_prot(map->type, vma);
-	/* fall through to _DRM_SHM */
+		/* fall through - to _DRM_SHM */
 	case _DRM_SHM:
 		vma->vm_ops = &drm_vm_shm_ops;
 		vma->vm_private_data = (void *)map;
@@ -646,6 +646,7 @@ int drm_legacy_mmap(struct file *filp, struct vm_area_struct *vma)
 }
 EXPORT_SYMBOL(drm_legacy_mmap);
 
+#if IS_ENABLED(CONFIG_DRM_LEGACY)
 void drm_legacy_vma_flush(struct drm_device *dev)
 {
 	struct drm_vma_entry *vma, *vma_temp;
@@ -656,3 +657,4 @@ void drm_legacy_vma_flush(struct drm_device *dev)
 		kfree(vma);
 	}
 }
+#endif
diff --git a/drivers/gpu/drm/drm_writeback.c b/drivers/gpu/drm/drm_writeback.c
index c20e6fe00cb3..79ac014701c8 100644
--- a/drivers/gpu/drm/drm_writeback.c
+++ b/drivers/gpu/drm/drm_writeback.c
@@ -239,14 +239,52 @@ fail:
 }
 EXPORT_SYMBOL(drm_writeback_connector_init);
 
+int drm_writeback_set_fb(struct drm_connector_state *conn_state,
+			 struct drm_framebuffer *fb)
+{
+	WARN_ON(conn_state->connector->connector_type != DRM_MODE_CONNECTOR_WRITEBACK);
+
+	if (!conn_state->writeback_job) {
+		conn_state->writeback_job =
+			kzalloc(sizeof(*conn_state->writeback_job), GFP_KERNEL);
+		if (!conn_state->writeback_job)
+			return -ENOMEM;
+
+		conn_state->writeback_job->connector =
+			drm_connector_to_writeback(conn_state->connector);
+	}
+
+	drm_framebuffer_assign(&conn_state->writeback_job->fb, fb);
+	return 0;
+}
+
+int drm_writeback_prepare_job(struct drm_writeback_job *job)
+{
+	struct drm_writeback_connector *connector = job->connector;
+	const struct drm_connector_helper_funcs *funcs =
+		connector->base.helper_private;
+	int ret;
+
+	if (funcs->prepare_writeback_job) {
+		ret = funcs->prepare_writeback_job(connector, job);
+		if (ret < 0)
+			return ret;
+	}
+
+	job->prepared = true;
+	return 0;
+}
+EXPORT_SYMBOL(drm_writeback_prepare_job);
+
 /**
  * drm_writeback_queue_job - Queue a writeback job for later signalling
  * @wb_connector: The writeback connector to queue a job on
- * @job: The job to queue
+ * @conn_state: The connector state containing the job to queue
  *
- * This function adds a job to the job_queue for a writeback connector. It
- * should be considered to take ownership of the writeback job, and so any other
- * references to the job must be cleared after calling this function.
+ * This function adds the job contained in @conn_state to the job_queue for a
+ * writeback connector. It takes ownership of the writeback job and sets the
+ * @conn_state->writeback_job to NULL, and so no access to the job may be
+ * performed by the caller after this function returns.
  *
  * Drivers must ensure that for a given writeback connector, jobs are queued in
  * exactly the same order as they will be completed by the hardware (and
@@ -258,16 +296,36 @@ EXPORT_SYMBOL(drm_writeback_connector_init);
  * See also: drm_writeback_signal_completion()
  */
 void drm_writeback_queue_job(struct drm_writeback_connector *wb_connector,
-			     struct drm_writeback_job *job)
+			     struct drm_connector_state *conn_state)
 {
+	struct drm_writeback_job *job;
 	unsigned long flags;
 
+	job = conn_state->writeback_job;
+	conn_state->writeback_job = NULL;
+
 	spin_lock_irqsave(&wb_connector->job_lock, flags);
 	list_add_tail(&job->list_entry, &wb_connector->job_queue);
 	spin_unlock_irqrestore(&wb_connector->job_lock, flags);
 }
 EXPORT_SYMBOL(drm_writeback_queue_job);
 
+void drm_writeback_cleanup_job(struct drm_writeback_job *job)
+{
+	struct drm_writeback_connector *connector = job->connector;
+	const struct drm_connector_helper_funcs *funcs =
+		connector->base.helper_private;
+
+	if (job->prepared && funcs->cleanup_writeback_job)
+		funcs->cleanup_writeback_job(connector, job);
+
+	if (job->fb)
+		drm_framebuffer_put(job->fb);
+
+	kfree(job);
+}
+EXPORT_SYMBOL(drm_writeback_cleanup_job);
+
 /*
  * @cleanup_work: deferred cleanup of a writeback job
  *
@@ -280,10 +338,9 @@ static void cleanup_work(struct work_struct *work)
 	struct drm_writeback_job *job = container_of(work,
 						     struct drm_writeback_job,
 						     cleanup_work);
-	drm_framebuffer_put(job->fb);
-	kfree(job);
-}
 
+	drm_writeback_cleanup_job(job);
+}
 
 /**
  * drm_writeback_signal_completion - Signal the completion of a writeback job
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
index 18c27f795cf6..7eb7cf9c3fa8 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
@@ -473,7 +473,6 @@ static struct drm_driver etnaviv_drm_driver = {
 	.prime_fd_to_handle = drm_gem_prime_fd_to_handle,
 	.gem_prime_export   = drm_gem_prime_export,
 	.gem_prime_import   = drm_gem_prime_import,
-	.gem_prime_res_obj  = etnaviv_gem_prime_res_obj,
 	.gem_prime_pin      = etnaviv_gem_prime_pin,
 	.gem_prime_unpin    = etnaviv_gem_prime_unpin,
 	.gem_prime_get_sg_table = etnaviv_gem_prime_get_sg_table,
@@ -515,6 +514,9 @@ static int etnaviv_bind(struct device *dev)
 	}
 	drm->dev_private = priv;
 
+	dev->dma_parms = &priv->dma_parms;
+	dma_set_max_seg_size(dev, SZ_2G);
+
 	mutex_init(&priv->gem_lock);
 	INIT_LIST_HEAD(&priv->gem_list);
 	priv->num_gpus = 0;
@@ -552,6 +554,8 @@ static void etnaviv_unbind(struct device *dev)
 
 	component_unbind_all(dev, drm);
 
+	dev->dma_parms = NULL;
+
 	drm->dev_private = NULL;
 	kfree(priv);
 
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.h b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
index a6a7ded37ef1..8798423705e1 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_drv.h
+++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
@@ -42,6 +42,7 @@ struct etnaviv_file_private {
 
 struct etnaviv_drm_private {
 	int num_gpus;
+	struct device_dma_parameters dma_parms;
 	struct etnaviv_gpu *gpu[ETNA_MAX_PIPES];
 
 	/* list of GEM objects: */
@@ -60,7 +61,6 @@ void *etnaviv_gem_prime_vmap(struct drm_gem_object *obj);
 void etnaviv_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr);
 int etnaviv_gem_prime_mmap(struct drm_gem_object *obj,
 			   struct vm_area_struct *vma);
-struct reservation_object *etnaviv_gem_prime_res_obj(struct drm_gem_object *obj);
 struct drm_gem_object *etnaviv_gem_prime_import_sg_table(struct drm_device *dev,
 	struct dma_buf_attachment *attach, struct sg_table *sg);
 int etnaviv_gem_prime_pin(struct drm_gem_object *obj);
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.c b/drivers/gpu/drm/etnaviv/etnaviv_gem.c
index 5c48915f492d..e8778ebb72e6 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gem.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.c
@@ -397,13 +397,13 @@ int etnaviv_gem_cpu_prep(struct drm_gem_object *obj, u32 op,
 	}
 
 	if (op & ETNA_PREP_NOSYNC) {
-		if (!reservation_object_test_signaled_rcu(etnaviv_obj->resv,
+		if (!reservation_object_test_signaled_rcu(obj->resv,
 							  write))
 			return -EBUSY;
 	} else {
 		unsigned long remain = etnaviv_timeout_to_jiffies(timeout);
 
-		ret = reservation_object_wait_timeout_rcu(etnaviv_obj->resv,
+		ret = reservation_object_wait_timeout_rcu(obj->resv,
 							  write, true, remain);
 		if (ret <= 0)
 			return ret == 0 ? -ETIMEDOUT : ret;
@@ -459,7 +459,7 @@ static void etnaviv_gem_describe_fence(struct dma_fence *fence,
 static void etnaviv_gem_describe(struct drm_gem_object *obj, struct seq_file *m)
 {
 	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
-	struct reservation_object *robj = etnaviv_obj->resv;
+	struct reservation_object *robj = obj->resv;
 	struct reservation_object_list *fobj;
 	struct dma_fence *fence;
 	unsigned long off = drm_vma_node_start(&obj->vma_node);
@@ -549,8 +549,6 @@ void etnaviv_gem_free_object(struct drm_gem_object *obj)
 
 	drm_gem_free_mmap_offset(obj);
 	etnaviv_obj->ops->release(etnaviv_obj);
-	if (etnaviv_obj->resv == &etnaviv_obj->_resv)
-		reservation_object_fini(&etnaviv_obj->_resv);
 	drm_gem_object_release(obj);
 
 	kfree(etnaviv_obj);
@@ -596,12 +594,8 @@ static int etnaviv_gem_new_impl(struct drm_device *dev, u32 size, u32 flags,
 
 	etnaviv_obj->flags = flags;
 	etnaviv_obj->ops = ops;
-	if (robj) {
-		etnaviv_obj->resv = robj;
-	} else {
-		etnaviv_obj->resv = &etnaviv_obj->_resv;
-		reservation_object_init(&etnaviv_obj->_resv);
-	}
+	if (robj)
+		etnaviv_obj->base.resv = robj;
 
 	mutex_init(&etnaviv_obj->lock);
 	INIT_LIST_HEAD(&etnaviv_obj->vram_list);
@@ -628,24 +622,18 @@ int etnaviv_gem_new_handle(struct drm_device *dev, struct drm_file *file,
 	lockdep_set_class(&to_etnaviv_bo(obj)->lock, &etnaviv_shm_lock_class);
 
 	ret = drm_gem_object_init(dev, obj, size);
-	if (ret == 0) {
-		struct address_space *mapping;
-
-		/*
-		 * Our buffers are kept pinned, so allocating them
-		 * from the MOVABLE zone is a really bad idea, and
-		 * conflicts with CMA. See comments above new_inode()
-		 * why this is required _and_ expected if you're
-		 * going to pin these pages.
-		 */
-		mapping = obj->filp->f_mapping;
-		mapping_set_gfp_mask(mapping, GFP_HIGHUSER |
-				     __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
-	}
-
 	if (ret)
 		goto fail;
 
+	/*
+	 * Our buffers are kept pinned, so allocating them from the MOVABLE
+	 * zone is a really bad idea, and conflicts with CMA. See comments
+	 * above new_inode() why this is required _and_ expected if you're
+	 * going to pin these pages.
+	 */
+	mapping_set_gfp_mask(obj->filp->f_mapping, GFP_HIGHUSER |
+			     __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
+
 	etnaviv_gem_obj_add(dev, obj);
 
 	ret = drm_gem_handle_create(file, obj, handle);
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.h b/drivers/gpu/drm/etnaviv/etnaviv_gem.h
index f0abb744ef95..753c458497d0 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gem.h
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.h
@@ -47,10 +47,6 @@ struct etnaviv_gem_object {
 	struct sg_table *sgt;
 	void *vaddr;
 
-	/* normally (resv == &_resv) except for imported bo's */
-	struct reservation_object *resv;
-	struct reservation_object _resv;
-
 	struct list_head vram_list;
 
 	/* cache maintenance */
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
index f21529e635e3..00e8b6a817e3 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c
@@ -139,10 +139,3 @@ fail:
 
 	return ERR_PTR(ret);
 }
-
-struct reservation_object *etnaviv_gem_prime_res_obj(struct drm_gem_object *obj)
-{
-	struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
-
-	return etnaviv_obj->resv;
-}
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c
index b2fe3446bfbc..e054f09ac828 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c
@@ -108,9 +108,9 @@ out_unlock:
 static void submit_unlock_object(struct etnaviv_gem_submit *submit, int i)
 {
 	if (submit->bos[i].flags & BO_LOCKED) {
-		struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj;
+		struct drm_gem_object *obj = &submit->bos[i].obj->base;
 
-		ww_mutex_unlock(&etnaviv_obj->resv->lock);
+		ww_mutex_unlock(&obj->resv->lock);
 		submit->bos[i].flags &= ~BO_LOCKED;
 	}
 }
@@ -122,7 +122,7 @@ static int submit_lock_objects(struct etnaviv_gem_submit *submit,
 
 retry:
 	for (i = 0; i < submit->nr_bos; i++) {
-		struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj;
+		struct drm_gem_object *obj = &submit->bos[i].obj->base;
 
 		if (slow_locked == i)
 			slow_locked = -1;
@@ -130,7 +130,7 @@ retry:
 		contended = i;
 
 		if (!(submit->bos[i].flags & BO_LOCKED)) {
-			ret = ww_mutex_lock_interruptible(&etnaviv_obj->resv->lock,
+			ret = ww_mutex_lock_interruptible(&obj->resv->lock,
 							  ticket);
 			if (ret == -EALREADY)
 				DRM_ERROR("BO at index %u already on submit list\n",
@@ -153,12 +153,12 @@ fail:
 		submit_unlock_object(submit, slow_locked);
 
 	if (ret == -EDEADLK) {
-		struct etnaviv_gem_object *etnaviv_obj;
+		struct drm_gem_object *obj;
 
-		etnaviv_obj = submit->bos[contended].obj;
+		obj = &submit->bos[contended].obj->base;
 
 		/* we lost out in a seqno race, lock and retry.. */
-		ret = ww_mutex_lock_slow_interruptible(&etnaviv_obj->resv->lock,
+		ret = ww_mutex_lock_slow_interruptible(&obj->resv->lock,
 						       ticket);
 		if (!ret) {
 			submit->bos[contended].flags |= BO_LOCKED;
@@ -176,7 +176,7 @@ static int submit_fence_sync(struct etnaviv_gem_submit *submit)
 
 	for (i = 0; i < submit->nr_bos; i++) {
 		struct etnaviv_gem_submit_bo *bo = &submit->bos[i];
-		struct reservation_object *robj = bo->obj->resv;
+		struct reservation_object *robj = bo->obj->base.resv;
 
 		if (!(bo->flags & ETNA_SUBMIT_BO_WRITE)) {
 			ret = reservation_object_reserve_shared(robj, 1);
@@ -207,13 +207,13 @@ static void submit_attach_object_fences(struct etnaviv_gem_submit *submit)
 	int i;
 
 	for (i = 0; i < submit->nr_bos; i++) {
-		struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj;
+		struct drm_gem_object *obj = &submit->bos[i].obj->base;
 
 		if (submit->bos[i].flags & ETNA_SUBMIT_BO_WRITE)
-			reservation_object_add_excl_fence(etnaviv_obj->resv,
+			reservation_object_add_excl_fence(obj->resv,
 							  submit->out_fence);
 		else
-			reservation_object_add_shared_fence(etnaviv_obj->resv,
+			reservation_object_add_shared_fence(obj->resv,
 							    submit->out_fence);
 
 		submit_unlock_object(submit, i);
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
index 6904535475de..72d01e873160 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
@@ -365,6 +365,7 @@ static void etnaviv_hw_identify(struct etnaviv_gpu *gpu)
 	dev_info(gpu->dev, "model: GC%x, revision: %x\n",
 		 gpu->identity.model, gpu->identity.revision);
 
+	gpu->idle_mask = ~VIVS_HI_IDLE_STATE_AXI_LP;
 	/*
 	 * If there is a match in the HWDB, we aren't interested in the
 	 * remaining register values, as they might be wrong.
@@ -412,7 +413,7 @@ static void etnaviv_hw_identify(struct etnaviv_gpu *gpu)
 	}
 
 	/* GC600 idle register reports zero bits where modules aren't present */
-	if (gpu->identity.model == chipModel_GC600) {
+	if (gpu->identity.model == chipModel_GC600)
 		gpu->idle_mask = VIVS_HI_IDLE_STATE_TX |
 				 VIVS_HI_IDLE_STATE_RA |
 				 VIVS_HI_IDLE_STATE_SE |
@@ -421,9 +422,6 @@ static void etnaviv_hw_identify(struct etnaviv_gpu *gpu)
 				 VIVS_HI_IDLE_STATE_PE |
 				 VIVS_HI_IDLE_STATE_DE |
 				 VIVS_HI_IDLE_STATE_FE;
-	} else {
-		gpu->idle_mask = ~VIVS_HI_IDLE_STATE_AXI_LP;
-	}
 
 	etnaviv_hw_specs(gpu);
 }
diff --git a/drivers/gpu/drm/exynos/exynos5433_drm_decon.c b/drivers/gpu/drm/exynos/exynos5433_drm_decon.c
index 5b4e0e8b23bc..73b318a7ef49 100644
--- a/drivers/gpu/drm/exynos/exynos5433_drm_decon.c
+++ b/drivers/gpu/drm/exynos/exynos5433_drm_decon.c
@@ -188,7 +188,7 @@ static void decon_setup_trigger(struct decon_context *ctx)
 
 	if (regmap_update_bits(ctx->sysreg, DSD_CFG_MUX,
 			       DSD_CFG_MUX_TE_UNMASK_GLOBAL, ~0))
-		DRM_ERROR("Cannot update sysreg.\n");
+		DRM_DEV_ERROR(ctx->dev, "Cannot update sysreg.\n");
 }
 
 static void decon_commit(struct exynos_drm_crtc *crtc)
@@ -356,7 +356,7 @@ static void decon_win_set_pixfmt(struct decon_context *ctx, unsigned int win,
 		break;
 	}
 
-	DRM_DEBUG_KMS("cpp = %u\n", fb->format->cpp[0]);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "cpp = %u\n", fb->format->cpp[0]);
 
 	/*
 	 * In case of exynos, setting dma-burst to 16Word causes permanent
@@ -561,8 +561,6 @@ static void decon_clear_channels(struct exynos_drm_crtc *crtc)
 	struct decon_context *ctx = crtc->ctx;
 	int win, i, ret;
 
-	DRM_DEBUG_KMS("%s\n", __FILE__);
-
 	for (i = 0; i < ARRAY_SIZE(decon_clks_name); i++) {
 		ret = clk_prepare_enable(ctx->clks[i]);
 		if (ret < 0)
diff --git a/drivers/gpu/drm/exynos/exynos7_drm_decon.c b/drivers/gpu/drm/exynos/exynos7_drm_decon.c
index 381aa3d60e37..0217ee9a118d 100644
--- a/drivers/gpu/drm/exynos/exynos7_drm_decon.c
+++ b/drivers/gpu/drm/exynos/exynos7_drm_decon.c
@@ -99,7 +99,7 @@ static void decon_wait_for_vblank(struct exynos_drm_crtc *crtc)
 	if (!wait_event_timeout(ctx->wait_vsync_queue,
 				!atomic_read(&ctx->wait_vsync_event),
 				HZ/20))
-		DRM_DEBUG_KMS("vblank wait timed out.\n");
+		DRM_DEV_DEBUG_KMS(ctx->dev, "vblank wait timed out.\n");
 }
 
 static void decon_clear_channels(struct exynos_drm_crtc *crtc)
@@ -107,8 +107,6 @@ static void decon_clear_channels(struct exynos_drm_crtc *crtc)
 	struct decon_context *ctx = crtc->ctx;
 	unsigned int win, ch_enabled = 0;
 
-	DRM_DEBUG_KMS("%s\n", __FILE__);
-
 	/* Check if any channel is enabled. */
 	for (win = 0; win < WINDOWS_NR; win++) {
 		u32 val = readl(ctx->regs + WINCON(win));
@@ -315,7 +313,7 @@ static void decon_win_set_pixfmt(struct decon_context *ctx, unsigned int win,
 		break;
 	}
 
-	DRM_DEBUG_KMS("cpp = %d\n", fb->format->cpp[0]);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "cpp = %d\n", fb->format->cpp[0]);
 
 	/*
 	 * In case of exynos, setting dma-burst to 16Word causes permanent
@@ -422,9 +420,9 @@ static void decon_update_plane(struct exynos_drm_crtc *crtc,
 	writel(state->src.x, ctx->regs + VIDW_OFFSET_X(win));
 	writel(state->src.y, ctx->regs + VIDW_OFFSET_Y(win));
 
-	DRM_DEBUG_KMS("start addr = 0x%lx\n",
+	DRM_DEV_DEBUG_KMS(ctx->dev, "start addr = 0x%lx\n",
 			(unsigned long)val);
-	DRM_DEBUG_KMS("ovl_width = %d, ovl_height = %d\n",
+	DRM_DEV_DEBUG_KMS(ctx->dev, "ovl_width = %d, ovl_height = %d\n",
 			state->crtc.w, state->crtc.h);
 
 	val = VIDOSDxA_TOPLEFT_X(state->crtc.x) |
@@ -442,7 +440,7 @@ static void decon_update_plane(struct exynos_drm_crtc *crtc,
 
 	writel(val, ctx->regs + VIDOSD_B(win));
 
-	DRM_DEBUG_KMS("osd pos: tx = %d, ty = %d, bx = %d, by = %d\n",
+	DRM_DEV_DEBUG_KMS(ctx->dev, "osd pos: tx = %d, ty = %d, bx = %d, by = %d\n",
 			state->crtc.x, state->crtc.y, last_x, last_y);
 
 	/* OSD alpha */
@@ -622,7 +620,7 @@ static int decon_bind(struct device *dev, struct device *master, void *data)
 
 	ret = decon_ctx_initialize(ctx, drm_dev);
 	if (ret) {
-		DRM_ERROR("decon_ctx_initialize failed.\n");
+		DRM_DEV_ERROR(dev, "decon_ctx_initialize failed.\n");
 		return ret;
 	}
 
@@ -802,25 +800,29 @@ static int exynos7_decon_resume(struct device *dev)
 
 	ret = clk_prepare_enable(ctx->pclk);
 	if (ret < 0) {
-		DRM_ERROR("Failed to prepare_enable the pclk [%d]\n", ret);
+		DRM_DEV_ERROR(dev, "Failed to prepare_enable the pclk [%d]\n",
+			      ret);
 		return ret;
 	}
 
 	ret = clk_prepare_enable(ctx->aclk);
 	if (ret < 0) {
-		DRM_ERROR("Failed to prepare_enable the aclk [%d]\n", ret);
+		DRM_DEV_ERROR(dev, "Failed to prepare_enable the aclk [%d]\n",
+			      ret);
 		return ret;
 	}
 
 	ret = clk_prepare_enable(ctx->eclk);
 	if  (ret < 0) {
-		DRM_ERROR("Failed to prepare_enable the eclk [%d]\n", ret);
+		DRM_DEV_ERROR(dev, "Failed to prepare_enable the eclk [%d]\n",
+			      ret);
 		return ret;
 	}
 
 	ret = clk_prepare_enable(ctx->vclk);
 	if  (ret < 0) {
-		DRM_ERROR("Failed to prepare_enable the vclk [%d]\n", ret);
+		DRM_DEV_ERROR(dev, "Failed to prepare_enable the vclk [%d]\n",
+			      ret);
 		return ret;
 	}
 
diff --git a/drivers/gpu/drm/exynos/exynos_dp.c b/drivers/gpu/drm/exynos/exynos_dp.c
index 471242a5e580..b0288cf85701 100644
--- a/drivers/gpu/drm/exynos/exynos_dp.c
+++ b/drivers/gpu/drm/exynos/exynos_dp.c
@@ -83,7 +83,8 @@ static int exynos_dp_get_modes(struct analogix_dp_plat_data *plat_data,
 
 	mode = drm_mode_create(connector->dev);
 	if (!mode) {
-		DRM_ERROR("failed to create a new display mode.\n");
+		DRM_DEV_ERROR(dp->dev,
+			      "failed to create a new display mode.\n");
 		return num_modes;
 	}
 
@@ -111,7 +112,8 @@ static int exynos_dp_bridge_attach(struct analogix_dp_plat_data *plat_data,
 	if (dp->ptn_bridge) {
 		ret = drm_bridge_attach(&dp->encoder, dp->ptn_bridge, bridge);
 		if (ret) {
-			DRM_ERROR("Failed to attach bridge to drm\n");
+			DRM_DEV_ERROR(dp->dev,
+				      "Failed to attach bridge to drm\n");
 			bridge->next = NULL;
 			return ret;
 		}
@@ -147,7 +149,8 @@ static int exynos_dp_dt_parse_panel(struct exynos_dp_device *dp)
 
 	ret = of_get_videomode(dp->dev->of_node, &dp->vm, OF_USE_NATIVE_MODE);
 	if (ret) {
-		DRM_ERROR("failed: of_get_videomode() : %d\n", ret);
+		DRM_DEV_ERROR(dp->dev,
+			      "failed: of_get_videomode() : %d\n", ret);
 		return ret;
 	}
 	return 0;
diff --git a/drivers/gpu/drm/exynos/exynos_drm_dma.c b/drivers/gpu/drm/exynos/exynos_drm_dma.c
index 3432c5ee9f0c..bef8bc3c8e00 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_dma.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_dma.c
@@ -62,7 +62,7 @@ static int drm_iommu_attach_device(struct drm_device *drm_dev,
 	int ret;
 
 	if (get_dma_ops(priv->dma_dev) != get_dma_ops(subdrv_dev)) {
-		DRM_ERROR("Device %s lacks support for IOMMU\n",
+		DRM_DEV_ERROR(subdrv_dev, "Device %s lacks support for IOMMU\n",
 			  dev_name(subdrv_dev));
 		return -EINVAL;
 	}
diff --git a/drivers/gpu/drm/exynos/exynos_drm_dpi.c b/drivers/gpu/drm/exynos/exynos_drm_dpi.c
index ae425c9a3f7b..6ea92173db9f 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_dpi.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_dpi.c
@@ -77,7 +77,8 @@ static int exynos_dpi_get_modes(struct drm_connector *connector)
 
 		mode = drm_mode_create(connector->dev);
 		if (!mode) {
-			DRM_ERROR("failed to create a new display mode\n");
+			DRM_DEV_ERROR(ctx->dev,
+				      "failed to create a new display mode\n");
 			return 0;
 		}
 		drm_display_mode_from_videomode(ctx->vm, mode);
@@ -108,7 +109,8 @@ static int exynos_dpi_create_connector(struct drm_encoder *encoder)
 				 &exynos_dpi_connector_funcs,
 				 DRM_MODE_CONNECTOR_VGA);
 	if (ret) {
-		DRM_ERROR("failed to initialize connector with drm\n");
+		DRM_DEV_ERROR(ctx->dev,
+			      "failed to initialize connector with drm\n");
 		return ret;
 	}
 
@@ -213,7 +215,8 @@ int exynos_dpi_bind(struct drm_device *dev, struct drm_encoder *encoder)
 
 	ret = exynos_dpi_create_connector(encoder);
 	if (ret) {
-		DRM_ERROR("failed to create connector ret = %d\n", ret);
+		DRM_DEV_ERROR(encoder_to_dpi(encoder)->dev,
+			      "failed to create connector ret = %d\n", ret);
 		drm_encoder_cleanup(encoder);
 		return ret;
 	}
diff --git a/drivers/gpu/drm/exynos/exynos_drm_dsi.c b/drivers/gpu/drm/exynos/exynos_drm_dsi.c
index a4253dd55f86..63a4b5074a99 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_dsi.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_dsi.c
@@ -1483,7 +1483,8 @@ static int exynos_dsi_create_connector(struct drm_encoder *encoder)
 	ret = drm_connector_init(drm, connector, &exynos_dsi_connector_funcs,
 				 DRM_MODE_CONNECTOR_DSI);
 	if (ret) {
-		DRM_ERROR("Failed to initialize connector with drm\n");
+		DRM_DEV_ERROR(dsi->dev,
+			      "Failed to initialize connector with drm\n");
 		return ret;
 	}
 
@@ -1527,7 +1528,9 @@ static int exynos_dsi_host_attach(struct mipi_dsi_host *host,
 		int ret = exynos_dsi_create_connector(encoder);
 
 		if (ret) {
-			DRM_ERROR("failed to create connector ret = %d\n", ret);
+			DRM_DEV_ERROR(dsi->dev,
+				      "failed to create connector ret = %d\n",
+				      ret);
 			drm_encoder_cleanup(encoder);
 			return ret;
 		}
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fb.c b/drivers/gpu/drm/exynos/exynos_drm_fb.c
index 1f11ab0f8e9d..832d22f57b4b 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fb.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_fb.c
@@ -45,7 +45,8 @@ static int check_fb_gem_memory_type(struct drm_device *drm_dev,
 	 * supported without IOMMU.
 	 */
 	if (IS_NONCONTIG_BUFFER(flags)) {
-		DRM_ERROR("Non-contiguous GEM memory is not supported.\n");
+		DRM_DEV_ERROR(drm_dev->dev,
+			      "Non-contiguous GEM memory is not supported.\n");
 		return -EINVAL;
 	}
 
@@ -83,7 +84,8 @@ exynos_drm_framebuffer_init(struct drm_device *dev,
 
 	ret = drm_framebuffer_init(dev, fb, &exynos_drm_fb_funcs);
 	if (ret < 0) {
-		DRM_ERROR("failed to initialize framebuffer\n");
+		DRM_DEV_ERROR(dev->dev,
+			      "failed to initialize framebuffer\n");
 		goto err;
 	}
 
@@ -113,7 +115,8 @@ exynos_user_fb_create(struct drm_device *dev, struct drm_file *file_priv,
 		exynos_gem[i] = exynos_drm_gem_get(file_priv,
 						   mode_cmd->handles[i]);
 		if (!exynos_gem[i]) {
-			DRM_ERROR("failed to lookup gem object\n");
+			DRM_DEV_ERROR(dev->dev,
+				      "failed to lookup gem object\n");
 			ret = -ENOENT;
 			goto err;
 		}
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fbdev.c b/drivers/gpu/drm/exynos/exynos_drm_fbdev.c
index c30dd88cdb25..724cb52a374a 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fbdev.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_fbdev.c
@@ -55,7 +55,7 @@ static int exynos_drm_fb_mmap(struct fb_info *info,
 			     exynos_gem->dma_addr, exynos_gem->size,
 			     exynos_gem->dma_attrs);
 	if (ret < 0) {
-		DRM_ERROR("failed to mmap.\n");
+		DRM_DEV_ERROR(to_dma_dev(helper->dev), "failed to mmap.\n");
 		return ret;
 	}
 
@@ -83,22 +83,22 @@ static int exynos_drm_fbdev_update(struct drm_fb_helper *helper,
 
 	fbi = drm_fb_helper_alloc_fbi(helper);
 	if (IS_ERR(fbi)) {
-		DRM_ERROR("failed to allocate fb info.\n");
+		DRM_DEV_ERROR(to_dma_dev(helper->dev),
+			      "failed to allocate fb info.\n");
 		return PTR_ERR(fbi);
 	}
 
-	fbi->par = helper;
 	fbi->fbops = &exynos_drm_fb_ops;
 
-	drm_fb_helper_fill_fix(fbi, fb->pitches[0], fb->format->depth);
-	drm_fb_helper_fill_var(fbi, helper, sizes->fb_width, sizes->fb_height);
+	drm_fb_helper_fill_info(fbi, helper, sizes);
 
 	nr_pages = exynos_gem->size >> PAGE_SHIFT;
 
 	exynos_gem->kvaddr = (void __iomem *) vmap(exynos_gem->pages, nr_pages,
 				VM_MAP, pgprot_writecombine(PAGE_KERNEL));
 	if (!exynos_gem->kvaddr) {
-		DRM_ERROR("failed to map pages to kernel space.\n");
+		DRM_DEV_ERROR(to_dma_dev(helper->dev),
+			      "failed to map pages to kernel space.\n");
 		return -EIO;
 	}
 
@@ -122,9 +122,10 @@ static int exynos_drm_fbdev_create(struct drm_fb_helper *helper,
 	unsigned long size;
 	int ret;
 
-	DRM_DEBUG_KMS("surface width(%d), height(%d) and bpp(%d\n",
-			sizes->surface_width, sizes->surface_height,
-			sizes->surface_bpp);
+	DRM_DEV_DEBUG_KMS(dev->dev,
+			  "surface width(%d), height(%d) and bpp(%d\n",
+			  sizes->surface_width, sizes->surface_height,
+			  sizes->surface_bpp);
 
 	mode_cmd.width = sizes->surface_width;
 	mode_cmd.height = sizes->surface_height;
@@ -154,7 +155,7 @@ static int exynos_drm_fbdev_create(struct drm_fb_helper *helper,
 	helper->fb =
 		exynos_drm_framebuffer_init(dev, &mode_cmd, &exynos_gem, 1);
 	if (IS_ERR(helper->fb)) {
-		DRM_ERROR("failed to create drm framebuffer.\n");
+		DRM_DEV_ERROR(dev->dev, "failed to create drm framebuffer.\n");
 		ret = PTR_ERR(helper->fb);
 		goto err_destroy_gem;
 	}
@@ -203,20 +204,23 @@ int exynos_drm_fbdev_init(struct drm_device *dev)
 
 	ret = drm_fb_helper_init(dev, helper, MAX_CONNECTOR);
 	if (ret < 0) {
-		DRM_ERROR("failed to initialize drm fb helper.\n");
+		DRM_DEV_ERROR(dev->dev,
+			      "failed to initialize drm fb helper.\n");
 		goto err_init;
 	}
 
 	ret = drm_fb_helper_single_add_all_connectors(helper);
 	if (ret < 0) {
-		DRM_ERROR("failed to register drm_fb_helper_connector.\n");
+		DRM_DEV_ERROR(dev->dev,
+			      "failed to register drm_fb_helper_connector.\n");
 		goto err_setup;
 
 	}
 
 	ret = drm_fb_helper_initial_config(helper, PREFERRED_BPP);
 	if (ret < 0) {
-		DRM_ERROR("failed to set up hw configuration.\n");
+		DRM_DEV_ERROR(dev->dev,
+			      "failed to set up hw configuration.\n");
 		goto err_setup;
 	}
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fimc.c b/drivers/gpu/drm/exynos/exynos_drm_fimc.c
index 90dfea0aec4d..c50b0f9270a4 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fimc.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_fimc.c
@@ -186,7 +186,7 @@ static void fimc_handle_jpeg(struct fimc_context *ctx, bool enable)
 {
 	u32 cfg;
 
-	DRM_DEBUG_KMS("enable[%d]\n", enable);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "enable[%d]\n", enable);
 
 	cfg = fimc_read(ctx, EXYNOS_CIGCTRL);
 	if (enable)
@@ -201,7 +201,7 @@ static void fimc_mask_irq(struct fimc_context *ctx, bool enable)
 {
 	u32 cfg;
 
-	DRM_DEBUG_KMS("enable[%d]\n", enable);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "enable[%d]\n", enable);
 
 	cfg = fimc_read(ctx, EXYNOS_CIGCTRL);
 	if (enable) {
@@ -225,15 +225,16 @@ static bool fimc_check_ovf(struct fimc_context *ctx)
 	flag = EXYNOS_CISTATUS_OVFIY | EXYNOS_CISTATUS_OVFICB |
 		EXYNOS_CISTATUS_OVFICR;
 
-	DRM_DEBUG_KMS("flag[0x%x]\n", flag);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "flag[0x%x]\n", flag);
 
 	if (status & flag) {
 		fimc_set_bits(ctx, EXYNOS_CIWDOFST,
 			EXYNOS_CIWDOFST_CLROVFIY | EXYNOS_CIWDOFST_CLROVFICB |
 			EXYNOS_CIWDOFST_CLROVFICR);
 
-		dev_err(ctx->dev, "occurred overflow at %d, status 0x%x.\n",
-			ctx->id, status);
+		DRM_DEV_ERROR(ctx->dev,
+			      "occurred overflow at %d, status 0x%x.\n",
+			      ctx->id, status);
 		return true;
 	}
 
@@ -246,7 +247,7 @@ static bool fimc_check_frame_end(struct fimc_context *ctx)
 
 	cfg = fimc_read(ctx, EXYNOS_CISTATUS);
 
-	DRM_DEBUG_KMS("cfg[0x%x]\n", cfg);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "cfg[0x%x]\n", cfg);
 
 	if (!(cfg & EXYNOS_CISTATUS_FRAMEEND))
 		return false;
@@ -268,17 +269,17 @@ static int fimc_get_buf_id(struct fimc_context *ctx)
 	if (frame_cnt == 0)
 		frame_cnt = EXYNOS_CISTATUS2_GET_FRAMECOUNT_PRESENT(cfg);
 
-	DRM_DEBUG_KMS("present[%d]before[%d]\n",
-		EXYNOS_CISTATUS2_GET_FRAMECOUNT_PRESENT(cfg),
-		EXYNOS_CISTATUS2_GET_FRAMECOUNT_BEFORE(cfg));
+	DRM_DEV_DEBUG_KMS(ctx->dev, "present[%d]before[%d]\n",
+			  EXYNOS_CISTATUS2_GET_FRAMECOUNT_PRESENT(cfg),
+			  EXYNOS_CISTATUS2_GET_FRAMECOUNT_BEFORE(cfg));
 
 	if (frame_cnt == 0) {
-		DRM_ERROR("failed to get frame count.\n");
+		DRM_DEV_ERROR(ctx->dev, "failed to get frame count.\n");
 		return -EIO;
 	}
 
 	buf_id = frame_cnt - 1;
-	DRM_DEBUG_KMS("buf_id[%d]\n", buf_id);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "buf_id[%d]\n", buf_id);
 
 	return buf_id;
 }
@@ -287,7 +288,7 @@ static void fimc_handle_lastend(struct fimc_context *ctx, bool enable)
 {
 	u32 cfg;
 
-	DRM_DEBUG_KMS("enable[%d]\n", enable);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "enable[%d]\n", enable);
 
 	cfg = fimc_read(ctx, EXYNOS_CIOCTRL);
 	if (enable)
@@ -302,7 +303,7 @@ static void fimc_src_set_fmt_order(struct fimc_context *ctx, u32 fmt)
 {
 	u32 cfg;
 
-	DRM_DEBUG_KMS("fmt[0x%x]\n", fmt);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "fmt[0x%x]\n", fmt);
 
 	/* RGB */
 	cfg = fimc_read(ctx, EXYNOS_CISCCTRL);
@@ -367,7 +368,7 @@ static void fimc_src_set_fmt(struct fimc_context *ctx, u32 fmt, bool tiled)
 {
 	u32 cfg;
 
-	DRM_DEBUG_KMS("fmt[0x%x]\n", fmt);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "fmt[0x%x]\n", fmt);
 
 	cfg = fimc_read(ctx, EXYNOS_MSCTRL);
 	cfg &= ~EXYNOS_MSCTRL_INFORMAT_RGB;
@@ -420,7 +421,7 @@ static void fimc_src_set_transf(struct fimc_context *ctx, unsigned int rotation)
 	unsigned int degree = rotation & DRM_MODE_ROTATE_MASK;
 	u32 cfg1, cfg2;
 
-	DRM_DEBUG_KMS("rotation[%x]\n", rotation);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "rotation[%x]\n", rotation);
 
 	cfg1 = fimc_read(ctx, EXYNOS_MSCTRL);
 	cfg1 &= ~(EXYNOS_MSCTRL_FLIP_X_MIRROR |
@@ -478,10 +479,11 @@ static void fimc_set_window(struct fimc_context *ctx,
 	v1 = buf->rect.y;
 	v2 = buf->buf.height - buf->rect.h - buf->rect.y;
 
-	DRM_DEBUG_KMS("x[%d]y[%d]w[%d]h[%d]hsize[%d]vsize[%d]\n",
-		buf->rect.x, buf->rect.y, buf->rect.w, buf->rect.h,
-		real_width, buf->buf.height);
-	DRM_DEBUG_KMS("h1[%d]h2[%d]v1[%d]v2[%d]\n", h1, h2, v1, v2);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "x[%d]y[%d]w[%d]h[%d]hsize[%d]vsize[%d]\n",
+			  buf->rect.x, buf->rect.y, buf->rect.w, buf->rect.h,
+			  real_width, buf->buf.height);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "h1[%d]h2[%d]v1[%d]v2[%d]\n", h1, h2, v1,
+			  v2);
 
 	/*
 	 * set window offset 1, 2 size
@@ -506,7 +508,8 @@ static void fimc_src_set_size(struct fimc_context *ctx,
 	unsigned int real_width = buf->buf.pitch[0] / buf->format->cpp[0];
 	u32 cfg;
 
-	DRM_DEBUG_KMS("hsize[%d]vsize[%d]\n", real_width, buf->buf.height);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "hsize[%d]vsize[%d]\n", real_width,
+			  buf->buf.height);
 
 	/* original size */
 	cfg = (EXYNOS_ORGISIZE_HORIZONTAL(real_width) |
@@ -514,8 +517,8 @@ static void fimc_src_set_size(struct fimc_context *ctx,
 
 	fimc_write(ctx, cfg, EXYNOS_ORGISIZE);
 
-	DRM_DEBUG_KMS("x[%d]y[%d]w[%d]h[%d]\n", buf->rect.x, buf->rect.y,
-		buf->rect.w, buf->rect.h);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "x[%d]y[%d]w[%d]h[%d]\n", buf->rect.x,
+			  buf->rect.y, buf->rect.w, buf->rect.h);
 
 	/* set input DMA image size */
 	cfg = fimc_read(ctx, EXYNOS_CIREAL_ISIZE);
@@ -560,7 +563,7 @@ static void fimc_dst_set_fmt_order(struct fimc_context *ctx, u32 fmt)
 {
 	u32 cfg;
 
-	DRM_DEBUG_KMS("fmt[0x%x]\n", fmt);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "fmt[0x%x]\n", fmt);
 
 	/* RGB */
 	cfg = fimc_read(ctx, EXYNOS_CISCCTRL);
@@ -631,7 +634,7 @@ static void fimc_dst_set_fmt(struct fimc_context *ctx, u32 fmt, bool tiled)
 {
 	u32 cfg;
 
-	DRM_DEBUG_KMS("fmt[0x%x]\n", fmt);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "fmt[0x%x]\n", fmt);
 
 	cfg = fimc_read(ctx, EXYNOS_CIEXTEN);
 
@@ -691,7 +694,7 @@ static void fimc_dst_set_transf(struct fimc_context *ctx, unsigned int rotation)
 	unsigned int degree = rotation & DRM_MODE_ROTATE_MASK;
 	u32 cfg;
 
-	DRM_DEBUG_KMS("rotation[0x%x]\n", rotation);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "rotation[0x%x]\n", rotation);
 
 	cfg = fimc_read(ctx, EXYNOS_CITRGFMT);
 	cfg &= ~EXYNOS_CITRGFMT_FLIP_MASK;
@@ -775,19 +778,20 @@ static int fimc_set_prescaler(struct fimc_context *ctx, struct fimc_scaler *sc,
 
 	pre_dst_width = src_w >> hfactor;
 	pre_dst_height = src_h >> vfactor;
-	DRM_DEBUG_KMS("pre_dst_width[%d]pre_dst_height[%d]\n",
-		pre_dst_width, pre_dst_height);
-	DRM_DEBUG_KMS("hfactor[%d]vfactor[%d]\n", hfactor, vfactor);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "pre_dst_width[%d]pre_dst_height[%d]\n",
+			  pre_dst_width, pre_dst_height);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "hfactor[%d]vfactor[%d]\n", hfactor,
+			  vfactor);
 
 	sc->hratio = (src_w << 14) / (dst_w << hfactor);
 	sc->vratio = (src_h << 14) / (dst_h << vfactor);
 	sc->up_h = (dst_w >= src_w) ? true : false;
 	sc->up_v = (dst_h >= src_h) ? true : false;
-	DRM_DEBUG_KMS("hratio[%d]vratio[%d]up_h[%d]up_v[%d]\n",
-		sc->hratio, sc->vratio, sc->up_h, sc->up_v);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "hratio[%d]vratio[%d]up_h[%d]up_v[%d]\n",
+			  sc->hratio, sc->vratio, sc->up_h, sc->up_v);
 
 	shfactor = FIMC_SHFACTOR - (hfactor + vfactor);
-	DRM_DEBUG_KMS("shfactor[%d]\n", shfactor);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "shfactor[%d]\n", shfactor);
 
 	cfg = (EXYNOS_CISCPRERATIO_SHFACTOR(shfactor) |
 		EXYNOS_CISCPRERATIO_PREHORRATIO(1 << hfactor) |
@@ -805,10 +809,10 @@ static void fimc_set_scaler(struct fimc_context *ctx, struct fimc_scaler *sc)
 {
 	u32 cfg, cfg_ext;
 
-	DRM_DEBUG_KMS("range[%d]bypass[%d]up_h[%d]up_v[%d]\n",
-		sc->range, sc->bypass, sc->up_h, sc->up_v);
-	DRM_DEBUG_KMS("hratio[%d]vratio[%d]\n",
-		sc->hratio, sc->vratio);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "range[%d]bypass[%d]up_h[%d]up_v[%d]\n",
+			  sc->range, sc->bypass, sc->up_h, sc->up_v);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "hratio[%d]vratio[%d]\n",
+			  sc->hratio, sc->vratio);
 
 	cfg = fimc_read(ctx, EXYNOS_CISCCTRL);
 	cfg &= ~(EXYNOS_CISCCTRL_SCALERBYPASS |
@@ -846,7 +850,8 @@ static void fimc_dst_set_size(struct fimc_context *ctx,
 	unsigned int real_width = buf->buf.pitch[0] / buf->format->cpp[0];
 	u32 cfg, cfg_ext;
 
-	DRM_DEBUG_KMS("hsize[%d]vsize[%d]\n", real_width, buf->buf.height);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "hsize[%d]vsize[%d]\n", real_width,
+			  buf->buf.height);
 
 	/* original size */
 	cfg = (EXYNOS_ORGOSIZE_HORIZONTAL(real_width) |
@@ -854,8 +859,9 @@ static void fimc_dst_set_size(struct fimc_context *ctx,
 
 	fimc_write(ctx, cfg, EXYNOS_ORGOSIZE);
 
-	DRM_DEBUG_KMS("x[%d]y[%d]w[%d]h[%d]\n", buf->rect.x, buf->rect.y,
-		buf->rect.w, buf->rect.h);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "x[%d]y[%d]w[%d]h[%d]\n", buf->rect.x,
+			  buf->rect.y,
+			  buf->rect.w, buf->rect.h);
 
 	/* CSC ITU */
 	cfg = fimc_read(ctx, EXYNOS_CIGCTRL);
@@ -905,7 +911,7 @@ static void fimc_dst_set_buf_seq(struct fimc_context *ctx, u32 buf_id,
 	u32 buf_num;
 	u32 cfg;
 
-	DRM_DEBUG_KMS("buf_id[%d]enqueu[%d]\n", buf_id, enqueue);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "buf_id[%d]enqueu[%d]\n", buf_id, enqueue);
 
 	spin_lock_irqsave(&ctx->lock, flags);
 
@@ -945,7 +951,7 @@ static irqreturn_t fimc_irq_handler(int irq, void *dev_id)
 	struct fimc_context *ctx = dev_id;
 	int buf_id;
 
-	DRM_DEBUG_KMS("fimc id[%d]\n", ctx->id);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "fimc id[%d]\n", ctx->id);
 
 	fimc_clear_irq(ctx);
 	if (fimc_check_ovf(ctx))
@@ -958,7 +964,7 @@ static irqreturn_t fimc_irq_handler(int irq, void *dev_id)
 	if (buf_id < 0)
 		return IRQ_HANDLED;
 
-	DRM_DEBUG_KMS("buf_id[%d]\n", buf_id);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "buf_id[%d]\n", buf_id);
 
 	if (ctx->task) {
 		struct exynos_drm_ipp_task *task = ctx->task;
@@ -1128,9 +1134,10 @@ static int fimc_bind(struct device *dev, struct device *master, void *data)
 	struct exynos_drm_ipp *ipp = &ctx->ipp;
 
 	ctx->drm_dev = drm_dev;
+	ipp->drm_dev = drm_dev;
 	exynos_drm_register_dma(drm_dev, dev);
 
-	exynos_drm_ipp_register(drm_dev, ipp, &ipp_funcs,
+	exynos_drm_ipp_register(dev, ipp, &ipp_funcs,
 			DRM_EXYNOS_IPP_CAP_CROP | DRM_EXYNOS_IPP_CAP_ROTATE |
 			DRM_EXYNOS_IPP_CAP_SCALE | DRM_EXYNOS_IPP_CAP_CONVERT,
 			ctx->formats, ctx->num_formats, "fimc");
@@ -1147,7 +1154,7 @@ static void fimc_unbind(struct device *dev, struct device *master,
 	struct drm_device *drm_dev = data;
 	struct exynos_drm_ipp *ipp = &ctx->ipp;
 
-	exynos_drm_ipp_unregister(drm_dev, ipp);
+	exynos_drm_ipp_unregister(dev, ipp);
 	exynos_drm_unregister_dma(drm_dev, dev);
 }
 
@@ -1380,7 +1387,7 @@ static int fimc_runtime_suspend(struct device *dev)
 {
 	struct fimc_context *ctx = get_fimc_context(dev);
 
-	DRM_DEBUG_KMS("id[%d]\n", ctx->id);
+	DRM_DEV_DEBUG_KMS(dev, "id[%d]\n", ctx->id);
 	clk_disable_unprepare(ctx->clocks[FIMC_CLK_GATE]);
 	return 0;
 }
@@ -1389,7 +1396,7 @@ static int fimc_runtime_resume(struct device *dev)
 {
 	struct fimc_context *ctx = get_fimc_context(dev);
 
-	DRM_DEBUG_KMS("id[%d]\n", ctx->id);
+	DRM_DEV_DEBUG_KMS(dev, "id[%d]\n", ctx->id);
 	return clk_prepare_enable(ctx->clocks[FIMC_CLK_GATE]);
 }
 #endif
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fimd.c b/drivers/gpu/drm/exynos/exynos_drm_fimd.c
index 786a8ee6f10f..8039e1a3671d 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fimd.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_fimd.c
@@ -315,7 +315,7 @@ static void fimd_wait_for_vblank(struct exynos_drm_crtc *crtc)
 	if (!wait_event_timeout(ctx->wait_vsync_queue,
 				!atomic_read(&ctx->wait_vsync_event),
 				HZ/20))
-		DRM_DEBUG_KMS("vblank wait timed out.\n");
+		DRM_DEV_DEBUG_KMS(ctx->dev, "vblank wait timed out.\n");
 }
 
 static void fimd_enable_video_output(struct fimd_context *ctx, unsigned int win,
@@ -350,8 +350,6 @@ static void fimd_clear_channels(struct exynos_drm_crtc *crtc)
 	struct fimd_context *ctx = crtc->ctx;
 	unsigned int win, ch_enabled = 0;
 
-	DRM_DEBUG_KMS("%s\n", __FILE__);
-
 	/* Hardware is in unknown state, so ensure it gets enabled properly */
 	pm_runtime_get_sync(ctx->dev);
 
@@ -400,7 +398,7 @@ static int fimd_atomic_check(struct exynos_drm_crtc *crtc,
 	u32 clkdiv;
 
 	if (mode->clock == 0) {
-		DRM_INFO("Mode has zero clock value.\n");
+		DRM_DEV_ERROR(ctx->dev, "Mode has zero clock value.\n");
 		return -EINVAL;
 	}
 
@@ -416,15 +414,17 @@ static int fimd_atomic_check(struct exynos_drm_crtc *crtc,
 
 	lcd_rate = clk_get_rate(ctx->lcd_clk);
 	if (2 * lcd_rate < ideal_clk) {
-		DRM_INFO("sclk_fimd clock too low(%lu) for requested pixel clock(%lu)\n",
-			 lcd_rate, ideal_clk);
+		DRM_DEV_ERROR(ctx->dev,
+			      "sclk_fimd clock too low(%lu) for requested pixel clock(%lu)\n",
+			      lcd_rate, ideal_clk);
 		return -EINVAL;
 	}
 
 	/* Find the clock divider value that gets us closest to ideal_clk */
 	clkdiv = DIV_ROUND_CLOSEST(lcd_rate, ideal_clk);
 	if (clkdiv >= 0x200) {
-		DRM_INFO("requested pixel clock(%lu) too low\n", ideal_clk);
+		DRM_DEV_ERROR(ctx->dev, "requested pixel clock(%lu) too low\n",
+			      ideal_clk);
 		return -EINVAL;
 	}
 
@@ -481,7 +481,8 @@ static void fimd_commit(struct exynos_drm_crtc *crtc)
 					driver_data->lcdblk_offset,
 					0x3 << driver_data->lcdblk_vt_shift,
 					0x1 << driver_data->lcdblk_vt_shift)) {
-			DRM_ERROR("Failed to update sysreg for I80 i/f.\n");
+			DRM_DEV_ERROR(ctx->dev,
+				      "Failed to update sysreg for I80 i/f.\n");
 			return;
 		}
 	} else {
@@ -525,7 +526,8 @@ static void fimd_commit(struct exynos_drm_crtc *crtc)
 				driver_data->lcdblk_offset,
 				0x1 << driver_data->lcdblk_bypass_shift,
 				0x1 << driver_data->lcdblk_bypass_shift)) {
-		DRM_ERROR("Failed to update sysreg for bypass setting.\n");
+		DRM_DEV_ERROR(ctx->dev,
+			      "Failed to update sysreg for bypass setting.\n");
 		return;
 	}
 
@@ -537,7 +539,8 @@ static void fimd_commit(struct exynos_drm_crtc *crtc)
 				driver_data->lcdblk_offset,
 				0x1 << driver_data->lcdblk_mic_bypass_shift,
 				0x1 << driver_data->lcdblk_mic_bypass_shift)) {
-		DRM_ERROR("Failed to update sysreg for bypass mic.\n");
+		DRM_DEV_ERROR(ctx->dev,
+			      "Failed to update sysreg for bypass mic.\n");
 		return;
 	}
 
@@ -814,10 +817,11 @@ static void fimd_update_plane(struct exynos_drm_crtc *crtc,
 	val = (unsigned long)(dma_addr + size);
 	writel(val, ctx->regs + VIDWx_BUF_END(win, 0));
 
-	DRM_DEBUG_KMS("start addr = 0x%lx, end addr = 0x%lx, size = 0x%lx\n",
-			(unsigned long)dma_addr, val, size);
-	DRM_DEBUG_KMS("ovl_width = %d, ovl_height = %d\n",
-			state->crtc.w, state->crtc.h);
+	DRM_DEV_DEBUG_KMS(ctx->dev,
+			  "start addr = 0x%lx, end addr = 0x%lx, size = 0x%lx\n",
+			  (unsigned long)dma_addr, val, size);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "ovl_width = %d, ovl_height = %d\n",
+			  state->crtc.w, state->crtc.h);
 
 	/* buffer size */
 	buf_offsize = pitch - (state->crtc.w * cpp);
@@ -847,8 +851,9 @@ static void fimd_update_plane(struct exynos_drm_crtc *crtc,
 
 	writel(val, ctx->regs + VIDOSD_B(win));
 
-	DRM_DEBUG_KMS("osd pos: tx = %d, ty = %d, bx = %d, by = %d\n",
-			state->crtc.x, state->crtc.y, last_x, last_y);
+	DRM_DEV_DEBUG_KMS(ctx->dev,
+			  "osd pos: tx = %d, ty = %d, bx = %d, by = %d\n",
+			  state->crtc.x, state->crtc.y, last_x, last_y);
 
 	/* OSD size */
 	if (win != 3 && win != 4) {
@@ -858,7 +863,8 @@ static void fimd_update_plane(struct exynos_drm_crtc *crtc,
 		val = state->crtc.w * state->crtc.h;
 		writel(val, ctx->regs + offset);
 
-		DRM_DEBUG_KMS("osd size = 0x%x\n", (unsigned int)val);
+		DRM_DEV_DEBUG_KMS(ctx->dev, "osd size = 0x%x\n",
+				  (unsigned int)val);
 	}
 
 	fimd_win_set_pixfmt(ctx, win, fb, state->src.w);
@@ -1252,13 +1258,17 @@ static int exynos_fimd_resume(struct device *dev)
 
 	ret = clk_prepare_enable(ctx->bus_clk);
 	if (ret < 0) {
-		DRM_ERROR("Failed to prepare_enable the bus clk [%d]\n", ret);
+		DRM_DEV_ERROR(dev,
+			      "Failed to prepare_enable the bus clk [%d]\n",
+			      ret);
 		return ret;
 	}
 
 	ret = clk_prepare_enable(ctx->lcd_clk);
 	if  (ret < 0) {
-		DRM_ERROR("Failed to prepare_enable the lcd clk [%d]\n", ret);
+		DRM_DEV_ERROR(dev,
+			      "Failed to prepare_enable the lcd clk [%d]\n",
+			      ret);
 		return ret;
 	}
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_g2d.c b/drivers/gpu/drm/exynos/exynos_drm_g2d.c
index 24c536d6d9cf..c20b3a759370 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_g2d.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_g2d.c
@@ -429,7 +429,7 @@ static dma_addr_t *g2d_userptr_get_dma_addr(struct g2d_data *g2d,
 	int ret;
 
 	if (!size) {
-		DRM_ERROR("invalid userptr size.\n");
+		DRM_DEV_ERROR(g2d->dev, "invalid userptr size.\n");
 		return ERR_PTR(-EINVAL);
 	}
 
@@ -482,7 +482,8 @@ static dma_addr_t *g2d_userptr_get_dma_addr(struct g2d_data *g2d,
 	ret = get_vaddr_frames(start, npages, FOLL_FORCE | FOLL_WRITE,
 		g2d_userptr->vec);
 	if (ret != npages) {
-		DRM_ERROR("failed to get user pages from userptr.\n");
+		DRM_DEV_ERROR(g2d->dev,
+			      "failed to get user pages from userptr.\n");
 		if (ret < 0)
 			goto err_destroy_framevec;
 		ret = -EFAULT;
@@ -503,7 +504,7 @@ static dma_addr_t *g2d_userptr_get_dma_addr(struct g2d_data *g2d,
 					frame_vector_pages(g2d_userptr->vec),
 					npages, offset, size, GFP_KERNEL);
 	if (ret < 0) {
-		DRM_ERROR("failed to get sgt from pages.\n");
+		DRM_DEV_ERROR(g2d->dev, "failed to get sgt from pages.\n");
 		goto err_free_sgt;
 	}
 
@@ -511,7 +512,7 @@ static dma_addr_t *g2d_userptr_get_dma_addr(struct g2d_data *g2d,
 
 	if (!dma_map_sg(to_dma_dev(g2d->drm_dev), sgt->sgl, sgt->nents,
 				DMA_BIDIRECTIONAL)) {
-		DRM_ERROR("failed to map sgt with dma region.\n");
+		DRM_DEV_ERROR(g2d->dev, "failed to map sgt with dma region.\n");
 		ret = -ENOMEM;
 		goto err_sg_free_table;
 	}
@@ -560,7 +561,7 @@ static void g2d_userptr_free_all(struct g2d_data *g2d, struct drm_file *filp)
 	g2d->current_pool = 0;
 }
 
-static enum g2d_reg_type g2d_get_reg_type(int reg_offset)
+static enum g2d_reg_type g2d_get_reg_type(struct g2d_data *g2d, int reg_offset)
 {
 	enum g2d_reg_type reg_type;
 
@@ -593,7 +594,8 @@ static enum g2d_reg_type g2d_get_reg_type(int reg_offset)
 		break;
 	default:
 		reg_type = REG_TYPE_NONE;
-		DRM_ERROR("Unknown register offset![%d]\n", reg_offset);
+		DRM_DEV_ERROR(g2d->dev, "Unknown register offset![%d]\n",
+			      reg_offset);
 		break;
 	}
 
@@ -627,9 +629,10 @@ static unsigned long g2d_get_buf_bpp(unsigned int format)
 	return bpp;
 }
 
-static bool g2d_check_buf_desc_is_valid(struct g2d_buf_desc *buf_desc,
-						enum g2d_reg_type reg_type,
-						unsigned long size)
+static bool g2d_check_buf_desc_is_valid(struct g2d_data *g2d,
+					struct g2d_buf_desc *buf_desc,
+					enum g2d_reg_type reg_type,
+					unsigned long size)
 {
 	int width, height;
 	unsigned long bpp, last_pos;
@@ -644,14 +647,15 @@ static bool g2d_check_buf_desc_is_valid(struct g2d_buf_desc *buf_desc,
 	/* This check also makes sure that right_x > left_x. */
 	width = (int)buf_desc->right_x - (int)buf_desc->left_x;
 	if (width < G2D_LEN_MIN || width > G2D_LEN_MAX) {
-		DRM_ERROR("width[%d] is out of range!\n", width);
+		DRM_DEV_ERROR(g2d->dev, "width[%d] is out of range!\n", width);
 		return false;
 	}
 
 	/* This check also makes sure that bottom_y > top_y. */
 	height = (int)buf_desc->bottom_y - (int)buf_desc->top_y;
 	if (height < G2D_LEN_MIN || height > G2D_LEN_MAX) {
-		DRM_ERROR("height[%d] is out of range!\n", height);
+		DRM_DEV_ERROR(g2d->dev,
+			      "height[%d] is out of range!\n", height);
 		return false;
 	}
 
@@ -670,8 +674,8 @@ static bool g2d_check_buf_desc_is_valid(struct g2d_buf_desc *buf_desc,
 	 */
 
 	if (last_pos >= size) {
-		DRM_ERROR("last engine access position [%lu] "
-			"is out of range [%lu]!\n", last_pos, size);
+		DRM_DEV_ERROR(g2d->dev, "last engine access position [%lu] "
+			      "is out of range [%lu]!\n", last_pos, size);
 		return false;
 	}
 
@@ -701,7 +705,7 @@ static int g2d_map_cmdlist_gem(struct g2d_data *g2d,
 		offset = cmdlist->data[reg_pos];
 		handle = cmdlist->data[reg_pos + 1];
 
-		reg_type = g2d_get_reg_type(offset);
+		reg_type = g2d_get_reg_type(g2d, offset);
 		if (reg_type == REG_TYPE_NONE) {
 			ret = -EFAULT;
 			goto err;
@@ -718,7 +722,7 @@ static int g2d_map_cmdlist_gem(struct g2d_data *g2d,
 				goto err;
 			}
 
-			if (!g2d_check_buf_desc_is_valid(buf_desc,
+			if (!g2d_check_buf_desc_is_valid(g2d, buf_desc,
 							 reg_type, exynos_gem->size)) {
 				exynos_drm_gem_put(exynos_gem);
 				ret = -EFAULT;
@@ -736,8 +740,9 @@ static int g2d_map_cmdlist_gem(struct g2d_data *g2d,
 				goto err;
 			}
 
-			if (!g2d_check_buf_desc_is_valid(buf_desc, reg_type,
-							g2d_userptr.size)) {
+			if (!g2d_check_buf_desc_is_valid(g2d, buf_desc,
+							 reg_type,
+							 g2d_userptr.size)) {
 				ret = -EFAULT;
 				goto err;
 			}
@@ -845,7 +850,7 @@ static void g2d_free_runqueue_node(struct g2d_data *g2d,
  *
  * Has to be called under runqueue lock.
  */
-static void g2d_remove_runqueue_nodes(struct g2d_data *g2d, struct drm_file* file)
+static void g2d_remove_runqueue_nodes(struct g2d_data *g2d, struct drm_file *file)
 {
 	struct g2d_runqueue_node *node, *n;
 
@@ -1044,7 +1049,7 @@ static int g2d_check_reg_offset(struct g2d_data *g2d,
 			if (!for_addr)
 				goto err;
 
-			reg_type = g2d_get_reg_type(reg_offset);
+			reg_type = g2d_get_reg_type(g2d, reg_offset);
 
 			/* check userptr buffer type. */
 			if ((cmdlist->data[index] & ~0x7fffffff) >> 31) {
@@ -1058,7 +1063,7 @@ static int g2d_check_reg_offset(struct g2d_data *g2d,
 			if (for_addr)
 				goto err;
 
-			reg_type = g2d_get_reg_type(reg_offset);
+			reg_type = g2d_get_reg_type(g2d, reg_offset);
 
 			buf_desc = &buf_info->descs[reg_type];
 			buf_desc->stride = cmdlist->data[index + 1];
@@ -1068,7 +1073,7 @@ static int g2d_check_reg_offset(struct g2d_data *g2d,
 			if (for_addr)
 				goto err;
 
-			reg_type = g2d_get_reg_type(reg_offset);
+			reg_type = g2d_get_reg_type(g2d, reg_offset);
 
 			buf_desc = &buf_info->descs[reg_type];
 			value = cmdlist->data[index + 1];
@@ -1080,7 +1085,7 @@ static int g2d_check_reg_offset(struct g2d_data *g2d,
 			if (for_addr)
 				goto err;
 
-			reg_type = g2d_get_reg_type(reg_offset);
+			reg_type = g2d_get_reg_type(g2d, reg_offset);
 
 			buf_desc = &buf_info->descs[reg_type];
 			value = cmdlist->data[index + 1];
@@ -1093,7 +1098,7 @@ static int g2d_check_reg_offset(struct g2d_data *g2d,
 			if (for_addr)
 				goto err;
 
-			reg_type = g2d_get_reg_type(reg_offset);
+			reg_type = g2d_get_reg_type(g2d, reg_offset);
 
 			buf_desc = &buf_info->descs[reg_type];
 			value = cmdlist->data[index + 1];
diff --git a/drivers/gpu/drm/exynos/exynos_drm_gem.c b/drivers/gpu/drm/exynos/exynos_drm_gem.c
index df66c383a877..a55f5ac41bf3 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_gem.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_gem.c
@@ -29,7 +29,7 @@ static int exynos_drm_alloc_buf(struct exynos_drm_gem *exynos_gem)
 	int ret = -ENOMEM;
 
 	if (exynos_gem->dma_addr) {
-		DRM_DEBUG_KMS("already allocated.\n");
+		DRM_DEV_DEBUG_KMS(to_dma_dev(dev), "already allocated.\n");
 		return 0;
 	}
 
@@ -61,7 +61,7 @@ static int exynos_drm_alloc_buf(struct exynos_drm_gem *exynos_gem)
 	exynos_gem->pages = kvmalloc_array(nr_pages, sizeof(struct page *),
 			GFP_KERNEL | __GFP_ZERO);
 	if (!exynos_gem->pages) {
-		DRM_ERROR("failed to allocate pages.\n");
+		DRM_DEV_ERROR(to_dma_dev(dev), "failed to allocate pages.\n");
 		return -ENOMEM;
 	}
 
@@ -69,7 +69,7 @@ static int exynos_drm_alloc_buf(struct exynos_drm_gem *exynos_gem)
 					     &exynos_gem->dma_addr, GFP_KERNEL,
 					     exynos_gem->dma_attrs);
 	if (!exynos_gem->cookie) {
-		DRM_ERROR("failed to allocate buffer.\n");
+		DRM_DEV_ERROR(to_dma_dev(dev), "failed to allocate buffer.\n");
 		goto err_free;
 	}
 
@@ -77,20 +77,20 @@ static int exynos_drm_alloc_buf(struct exynos_drm_gem *exynos_gem)
 				    exynos_gem->dma_addr, exynos_gem->size,
 				    exynos_gem->dma_attrs);
 	if (ret < 0) {
-		DRM_ERROR("failed to get sgtable.\n");
+		DRM_DEV_ERROR(to_dma_dev(dev), "failed to get sgtable.\n");
 		goto err_dma_free;
 	}
 
 	if (drm_prime_sg_to_page_addr_arrays(&sgt, exynos_gem->pages, NULL,
 					     nr_pages)) {
-		DRM_ERROR("invalid sgtable.\n");
+		DRM_DEV_ERROR(to_dma_dev(dev), "invalid sgtable.\n");
 		ret = -EINVAL;
 		goto err_sgt_free;
 	}
 
 	sg_free_table(&sgt);
 
-	DRM_DEBUG_KMS("dma_addr(0x%lx), size(0x%lx)\n",
+	DRM_DEV_DEBUG_KMS(to_dma_dev(dev), "dma_addr(0x%lx), size(0x%lx)\n",
 			(unsigned long)exynos_gem->dma_addr, exynos_gem->size);
 
 	return 0;
@@ -111,11 +111,11 @@ static void exynos_drm_free_buf(struct exynos_drm_gem *exynos_gem)
 	struct drm_device *dev = exynos_gem->base.dev;
 
 	if (!exynos_gem->dma_addr) {
-		DRM_DEBUG_KMS("dma_addr is invalid.\n");
+		DRM_DEV_DEBUG_KMS(dev->dev, "dma_addr is invalid.\n");
 		return;
 	}
 
-	DRM_DEBUG_KMS("dma_addr(0x%lx), size(0x%lx)\n",
+	DRM_DEV_DEBUG_KMS(dev->dev, "dma_addr(0x%lx), size(0x%lx)\n",
 			(unsigned long)exynos_gem->dma_addr, exynos_gem->size);
 
 	dma_free_attrs(to_dma_dev(dev), exynos_gem->size, exynos_gem->cookie,
@@ -139,7 +139,7 @@ static int exynos_drm_gem_handle_create(struct drm_gem_object *obj,
 	if (ret)
 		return ret;
 
-	DRM_DEBUG_KMS("gem handle = 0x%x\n", *handle);
+	DRM_DEV_DEBUG_KMS(to_dma_dev(obj->dev), "gem handle = 0x%x\n", *handle);
 
 	/* drop reference from allocate - handle holds it now. */
 	drm_gem_object_put_unlocked(obj);
@@ -151,7 +151,8 @@ void exynos_drm_gem_destroy(struct exynos_drm_gem *exynos_gem)
 {
 	struct drm_gem_object *obj = &exynos_gem->base;
 
-	DRM_DEBUG_KMS("handle count = %d\n", obj->handle_count);
+	DRM_DEV_DEBUG_KMS(to_dma_dev(obj->dev), "handle count = %d\n",
+			  obj->handle_count);
 
 	/*
 	 * do not release memory region from exporter.
@@ -186,7 +187,7 @@ static struct exynos_drm_gem *exynos_drm_gem_init(struct drm_device *dev,
 
 	ret = drm_gem_object_init(dev, obj, size);
 	if (ret < 0) {
-		DRM_ERROR("failed to initialize gem object\n");
+		DRM_DEV_ERROR(dev->dev, "failed to initialize gem object\n");
 		kfree(exynos_gem);
 		return ERR_PTR(ret);
 	}
@@ -198,7 +199,7 @@ static struct exynos_drm_gem *exynos_drm_gem_init(struct drm_device *dev,
 		return ERR_PTR(ret);
 	}
 
-	DRM_DEBUG_KMS("created file object = %pK\n", obj->filp);
+	DRM_DEV_DEBUG_KMS(dev->dev, "created file object = %pK\n", obj->filp);
 
 	return exynos_gem;
 }
@@ -211,12 +212,13 @@ struct exynos_drm_gem *exynos_drm_gem_create(struct drm_device *dev,
 	int ret;
 
 	if (flags & ~(EXYNOS_BO_MASK)) {
-		DRM_ERROR("invalid GEM buffer flags: %u\n", flags);
+		DRM_DEV_ERROR(dev->dev,
+			      "invalid GEM buffer flags: %u\n", flags);
 		return ERR_PTR(-EINVAL);
 	}
 
 	if (!size) {
-		DRM_ERROR("invalid GEM buffer size: %lu\n", size);
+		DRM_DEV_ERROR(dev->dev, "invalid GEM buffer size: %lu\n", size);
 		return ERR_PTR(-EINVAL);
 	}
 
@@ -325,7 +327,7 @@ int exynos_drm_gem_get_ioctl(struct drm_device *dev, void *data,
 
 	obj = drm_gem_object_lookup(file_priv, args->handle);
 	if (!obj) {
-		DRM_ERROR("failed to lookup gem object.\n");
+		DRM_DEV_ERROR(dev->dev, "failed to lookup gem object.\n");
 		return -EINVAL;
 	}
 
@@ -408,7 +410,8 @@ static int exynos_drm_gem_mmap_obj(struct drm_gem_object *obj,
 	struct exynos_drm_gem *exynos_gem = to_exynos_gem(obj);
 	int ret;
 
-	DRM_DEBUG_KMS("flags = 0x%x\n", exynos_gem->flags);
+	DRM_DEV_DEBUG_KMS(to_dma_dev(obj->dev), "flags = 0x%x\n",
+			  exynos_gem->flags);
 
 	/* non-cachable as default. */
 	if (exynos_gem->flags & EXYNOS_BO_CACHABLE)
diff --git a/drivers/gpu/drm/exynos/exynos_drm_gsc.c b/drivers/gpu/drm/exynos/exynos_drm_gsc.c
index f048d97fe9e2..0bfb5e9f6e91 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_gsc.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_gsc.c
@@ -395,7 +395,7 @@ static int gsc_sw_reset(struct gsc_context *ctx)
 	}
 
 	if (cfg) {
-		DRM_ERROR("failed to reset gsc h/w.\n");
+		DRM_DEV_ERROR(ctx->dev, "failed to reset gsc h/w.\n");
 		return -EBUSY;
 	}
 
@@ -422,8 +422,8 @@ static void gsc_handle_irq(struct gsc_context *ctx, bool enable,
 {
 	u32 cfg;
 
-	DRM_DEBUG_KMS("enable[%d]overflow[%d]level[%d]\n",
-			enable, overflow, done);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "enable[%d]overflow[%d]level[%d]\n",
+			  enable, overflow, done);
 
 	cfg = gsc_read(GSC_IRQ);
 	cfg |= (GSC_IRQ_OR_MASK | GSC_IRQ_FRMDONE_MASK);
@@ -451,7 +451,7 @@ static void gsc_src_set_fmt(struct gsc_context *ctx, u32 fmt, bool tiled)
 {
 	u32 cfg;
 
-	DRM_DEBUG_KMS("fmt[0x%x]\n", fmt);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "fmt[0x%x]\n", fmt);
 
 	cfg = gsc_read(GSC_IN_CON);
 	cfg &= ~(GSC_IN_RGB_TYPE_MASK | GSC_IN_YUV422_1P_ORDER_MASK |
@@ -638,7 +638,7 @@ static void gsc_dst_set_fmt(struct gsc_context *ctx, u32 fmt, bool tiled)
 {
 	u32 cfg;
 
-	DRM_DEBUG_KMS("fmt[0x%x]\n", fmt);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "fmt[0x%x]\n", fmt);
 
 	cfg = gsc_read(GSC_OUT_CON);
 	cfg &= ~(GSC_OUT_RGB_TYPE_MASK | GSC_OUT_YUV422_1P_ORDER_MASK |
@@ -706,12 +706,13 @@ static void gsc_dst_set_fmt(struct gsc_context *ctx, u32 fmt, bool tiled)
 	gsc_write(cfg, GSC_OUT_CON);
 }
 
-static int gsc_get_ratio_shift(u32 src, u32 dst, u32 *ratio)
+static int gsc_get_ratio_shift(struct gsc_context *ctx, u32 src, u32 dst,
+			       u32 *ratio)
 {
-	DRM_DEBUG_KMS("src[%d]dst[%d]\n", src, dst);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "src[%d]dst[%d]\n", src, dst);
 
 	if (src >= dst * 8) {
-		DRM_ERROR("failed to make ratio and shift.\n");
+		DRM_DEV_ERROR(ctx->dev, "failed to make ratio and shift.\n");
 		return -EINVAL;
 	} else if (src >= dst * 4)
 		*ratio = 4;
@@ -759,31 +760,31 @@ static int gsc_set_prescaler(struct gsc_context *ctx, struct gsc_scaler *sc,
 		dst_h = dst->h;
 	}
 
-	ret = gsc_get_ratio_shift(src_w, dst_w, &sc->pre_hratio);
+	ret = gsc_get_ratio_shift(ctx, src_w, dst_w, &sc->pre_hratio);
 	if (ret) {
-		dev_err(ctx->dev, "failed to get ratio horizontal.\n");
+		DRM_DEV_ERROR(ctx->dev, "failed to get ratio horizontal.\n");
 		return ret;
 	}
 
-	ret = gsc_get_ratio_shift(src_h, dst_h, &sc->pre_vratio);
+	ret = gsc_get_ratio_shift(ctx, src_h, dst_h, &sc->pre_vratio);
 	if (ret) {
-		dev_err(ctx->dev, "failed to get ratio vertical.\n");
+		DRM_DEV_ERROR(ctx->dev, "failed to get ratio vertical.\n");
 		return ret;
 	}
 
-	DRM_DEBUG_KMS("pre_hratio[%d]pre_vratio[%d]\n",
-		sc->pre_hratio, sc->pre_vratio);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "pre_hratio[%d]pre_vratio[%d]\n",
+			  sc->pre_hratio, sc->pre_vratio);
 
 	sc->main_hratio = (src_w << 16) / dst_w;
 	sc->main_vratio = (src_h << 16) / dst_h;
 
-	DRM_DEBUG_KMS("main_hratio[%ld]main_vratio[%ld]\n",
-		sc->main_hratio, sc->main_vratio);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "main_hratio[%ld]main_vratio[%ld]\n",
+			  sc->main_hratio, sc->main_vratio);
 
 	gsc_get_prescaler_shfactor(sc->pre_hratio, sc->pre_vratio,
 		&sc->pre_shfactor);
 
-	DRM_DEBUG_KMS("pre_shfactor[%d]\n", sc->pre_shfactor);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "pre_shfactor[%d]\n", sc->pre_shfactor);
 
 	cfg = (GSC_PRESC_SHFACTOR(sc->pre_shfactor) |
 		GSC_PRESC_H_RATIO(sc->pre_hratio) |
@@ -849,8 +850,8 @@ static void gsc_set_scaler(struct gsc_context *ctx, struct gsc_scaler *sc)
 {
 	u32 cfg;
 
-	DRM_DEBUG_KMS("main_hratio[%ld]main_vratio[%ld]\n",
-		sc->main_hratio, sc->main_vratio);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "main_hratio[%ld]main_vratio[%ld]\n",
+			  sc->main_hratio, sc->main_vratio);
 
 	gsc_set_h_coef(ctx, sc->main_hratio);
 	cfg = GSC_MAIN_H_RATIO_VALUE(sc->main_hratio);
@@ -916,7 +917,7 @@ static int gsc_dst_get_buf_seq(struct gsc_context *ctx)
 		if (cfg & (mask << i))
 			buf_num--;
 
-	DRM_DEBUG_KMS("buf_num[%d]\n", buf_num);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "buf_num[%d]\n", buf_num);
 
 	return buf_num;
 }
@@ -963,7 +964,7 @@ static int gsc_get_src_buf_index(struct gsc_context *ctx)
 	u32 cfg, curr_index, i;
 	u32 buf_id = GSC_MAX_SRC;
 
-	DRM_DEBUG_KMS("gsc id[%d]\n", ctx->id);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "gsc id[%d]\n", ctx->id);
 
 	cfg = gsc_read(GSC_IN_BASE_ADDR_Y_MASK);
 	curr_index = GSC_IN_CURR_GET_INDEX(cfg);
@@ -975,11 +976,11 @@ static int gsc_get_src_buf_index(struct gsc_context *ctx)
 		}
 	}
 
-	DRM_DEBUG_KMS("cfg[0x%x]curr_index[%d]buf_id[%d]\n", cfg,
-		curr_index, buf_id);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "cfg[0x%x]curr_index[%d]buf_id[%d]\n", cfg,
+			  curr_index, buf_id);
 
 	if (buf_id == GSC_MAX_SRC) {
-		DRM_ERROR("failed to get in buffer index.\n");
+		DRM_DEV_ERROR(ctx->dev, "failed to get in buffer index.\n");
 		return -EINVAL;
 	}
 
@@ -993,7 +994,7 @@ static int gsc_get_dst_buf_index(struct gsc_context *ctx)
 	u32 cfg, curr_index, i;
 	u32 buf_id = GSC_MAX_DST;
 
-	DRM_DEBUG_KMS("gsc id[%d]\n", ctx->id);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "gsc id[%d]\n", ctx->id);
 
 	cfg = gsc_read(GSC_OUT_BASE_ADDR_Y_MASK);
 	curr_index = GSC_OUT_CURR_GET_INDEX(cfg);
@@ -1006,14 +1007,14 @@ static int gsc_get_dst_buf_index(struct gsc_context *ctx)
 	}
 
 	if (buf_id == GSC_MAX_DST) {
-		DRM_ERROR("failed to get out buffer index.\n");
+		DRM_DEV_ERROR(ctx->dev, "failed to get out buffer index.\n");
 		return -EINVAL;
 	}
 
 	gsc_dst_set_buf_seq(ctx, buf_id, false);
 
-	DRM_DEBUG_KMS("cfg[0x%x]curr_index[%d]buf_id[%d]\n", cfg,
-		curr_index, buf_id);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "cfg[0x%x]curr_index[%d]buf_id[%d]\n", cfg,
+			  curr_index, buf_id);
 
 	return buf_id;
 }
@@ -1024,7 +1025,7 @@ static irqreturn_t gsc_irq_handler(int irq, void *dev_id)
 	u32 status;
 	int err = 0;
 
-	DRM_DEBUG_KMS("gsc id[%d]\n", ctx->id);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "gsc id[%d]\n", ctx->id);
 
 	status = gsc_read(GSC_IRQ);
 	if (status & GSC_IRQ_STATUS_OR_IRQ) {
@@ -1042,8 +1043,8 @@ static irqreturn_t gsc_irq_handler(int irq, void *dev_id)
 		src_buf_id = gsc_get_src_buf_index(ctx);
 		dst_buf_id = gsc_get_dst_buf_index(ctx);
 
-		DRM_DEBUG_KMS("buf_id_src[%d]buf_id_dst[%d]\n",	src_buf_id,
-			      dst_buf_id);
+		DRM_DEV_DEBUG_KMS(ctx->dev, "buf_id_src[%d]buf_id_dst[%d]\n",
+				  src_buf_id, dst_buf_id);
 
 		if (src_buf_id < 0 || dst_buf_id < 0)
 			err = -EINVAL;
@@ -1169,9 +1170,10 @@ static int gsc_bind(struct device *dev, struct device *master, void *data)
 	struct exynos_drm_ipp *ipp = &ctx->ipp;
 
 	ctx->drm_dev = drm_dev;
+	ctx->drm_dev = drm_dev;
 	exynos_drm_register_dma(drm_dev, dev);
 
-	exynos_drm_ipp_register(drm_dev, ipp, &ipp_funcs,
+	exynos_drm_ipp_register(dev, ipp, &ipp_funcs,
 			DRM_EXYNOS_IPP_CAP_CROP | DRM_EXYNOS_IPP_CAP_ROTATE |
 			DRM_EXYNOS_IPP_CAP_SCALE | DRM_EXYNOS_IPP_CAP_CONVERT,
 			ctx->formats, ctx->num_formats, "gsc");
@@ -1188,7 +1190,7 @@ static void gsc_unbind(struct device *dev, struct device *master,
 	struct drm_device *drm_dev = data;
 	struct exynos_drm_ipp *ipp = &ctx->ipp;
 
-	exynos_drm_ipp_unregister(drm_dev, ipp);
+	exynos_drm_ipp_unregister(dev, ipp);
 	exynos_drm_unregister_dma(drm_dev, dev);
 }
 
@@ -1324,7 +1326,7 @@ static int __maybe_unused gsc_runtime_suspend(struct device *dev)
 	struct gsc_context *ctx = get_gsc_context(dev);
 	int i;
 
-	DRM_DEBUG_KMS("id[%d]\n", ctx->id);
+	DRM_DEV_DEBUG_KMS(dev, "id[%d]\n", ctx->id);
 
 	for (i = ctx->num_clocks - 1; i >= 0; i--)
 		clk_disable_unprepare(ctx->clocks[i]);
@@ -1337,7 +1339,7 @@ static int __maybe_unused gsc_runtime_resume(struct device *dev)
 	struct gsc_context *ctx = get_gsc_context(dev);
 	int i, ret;
 
-	DRM_DEBUG_KMS("id[%d]\n", ctx->id);
+	DRM_DEV_DEBUG_KMS(dev, "id[%d]\n", ctx->id);
 
 	for (i = 0; i < ctx->num_clocks; i++) {
 		ret = clk_prepare_enable(ctx->clocks[i]);
diff --git a/drivers/gpu/drm/exynos/exynos_drm_ipp.c b/drivers/gpu/drm/exynos/exynos_drm_ipp.c
index 23226a0212e8..c862099723a0 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_ipp.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_ipp.c
@@ -43,7 +43,7 @@ static LIST_HEAD(ipp_list);
  * Returns:
  * Zero on success, error code on failure.
  */
-int exynos_drm_ipp_register(struct drm_device *dev, struct exynos_drm_ipp *ipp,
+int exynos_drm_ipp_register(struct device *dev, struct exynos_drm_ipp *ipp,
 		const struct exynos_drm_ipp_funcs *funcs, unsigned int caps,
 		const struct exynos_drm_ipp_formats *formats,
 		unsigned int num_formats, const char *name)
@@ -67,7 +67,7 @@ int exynos_drm_ipp_register(struct drm_device *dev, struct exynos_drm_ipp *ipp,
 	list_add_tail(&ipp->head, &ipp_list);
 	ipp->id = num_ipp++;
 
-	DRM_DEBUG_DRIVER("Registered ipp %d\n", ipp->id);
+	DRM_DEV_DEBUG_DRIVER(dev, "Registered ipp %d\n", ipp->id);
 
 	return 0;
 }
@@ -77,7 +77,7 @@ int exynos_drm_ipp_register(struct drm_device *dev, struct exynos_drm_ipp *ipp,
  * @dev: DRM device
  * @ipp: ipp module
  */
-void exynos_drm_ipp_unregister(struct drm_device *dev,
+void exynos_drm_ipp_unregister(struct device *dev,
 			       struct exynos_drm_ipp *ipp)
 {
 	WARN_ON(ipp->task);
@@ -268,7 +268,7 @@ static inline struct exynos_drm_ipp_task *
 	task->src.rect.h = task->dst.rect.h = UINT_MAX;
 	task->transform.rotation = DRM_MODE_ROTATE_0;
 
-	DRM_DEBUG_DRIVER("Allocated task %pK\n", task);
+	DRM_DEV_DEBUG_DRIVER(task->dev, "Allocated task %pK\n", task);
 
 	return task;
 }
@@ -335,7 +335,9 @@ static int exynos_drm_ipp_task_set(struct exynos_drm_ipp_task *task,
 		size -= map[i].size;
 	}
 
-	DRM_DEBUG_DRIVER("Got task %pK configuration from userspace\n", task);
+	DRM_DEV_DEBUG_DRIVER(task->dev,
+			     "Got task %pK configuration from userspace\n",
+			     task);
 	return 0;
 }
 
@@ -389,12 +391,12 @@ static void exynos_drm_ipp_task_release_buf(struct exynos_drm_ipp_buffer *buf)
 static void exynos_drm_ipp_task_free(struct exynos_drm_ipp *ipp,
 				 struct exynos_drm_ipp_task *task)
 {
-	DRM_DEBUG_DRIVER("Freeing task %pK\n", task);
+	DRM_DEV_DEBUG_DRIVER(task->dev, "Freeing task %pK\n", task);
 
 	exynos_drm_ipp_task_release_buf(&task->src);
 	exynos_drm_ipp_task_release_buf(&task->dst);
 	if (task->event)
-		drm_event_cancel_free(ipp->dev, &task->event->base);
+		drm_event_cancel_free(ipp->drm_dev, &task->event->base);
 	kfree(task);
 }
 
@@ -553,8 +555,9 @@ static int exynos_drm_ipp_check_format(struct exynos_drm_ipp_task *task,
 			       buf == src ? DRM_EXYNOS_IPP_FORMAT_SOURCE :
 					    DRM_EXYNOS_IPP_FORMAT_DESTINATION);
 	if (!fmt) {
-		DRM_DEBUG_DRIVER("Task %pK: %s format not supported\n", task,
-				 buf == src ? "src" : "dst");
+		DRM_DEV_DEBUG_DRIVER(task->dev,
+				     "Task %pK: %s format not supported\n",
+				     task, buf == src ? "src" : "dst");
 		return -EINVAL;
 	}
 
@@ -603,7 +606,7 @@ static int exynos_drm_ipp_task_check(struct exynos_drm_ipp_task *task)
 	bool rotate = (rotation != DRM_MODE_ROTATE_0);
 	bool scale = false;
 
-	DRM_DEBUG_DRIVER("Checking task %pK\n", task);
+	DRM_DEV_DEBUG_DRIVER(task->dev, "Checking task %pK\n", task);
 
 	if (src->rect.w == UINT_MAX)
 		src->rect.w = src->buf.width;
@@ -618,8 +621,9 @@ static int exynos_drm_ipp_task_check(struct exynos_drm_ipp_task *task)
 	    src->rect.y + src->rect.h > (src->buf.height) ||
 	    dst->rect.x + dst->rect.w > (dst->buf.width) ||
 	    dst->rect.y + dst->rect.h > (dst->buf.height)) {
-		DRM_DEBUG_DRIVER("Task %pK: defined area is outside provided buffers\n",
-				 task);
+		DRM_DEV_DEBUG_DRIVER(task->dev,
+				     "Task %pK: defined area is outside provided buffers\n",
+				     task);
 		return -EINVAL;
 	}
 
@@ -635,7 +639,8 @@ static int exynos_drm_ipp_task_check(struct exynos_drm_ipp_task *task)
 	    (!(ipp->capabilities & DRM_EXYNOS_IPP_CAP_SCALE) && scale) ||
 	    (!(ipp->capabilities & DRM_EXYNOS_IPP_CAP_CONVERT) &&
 	     src->buf.fourcc != dst->buf.fourcc)) {
-		DRM_DEBUG_DRIVER("Task %pK: hw capabilities exceeded\n", task);
+		DRM_DEV_DEBUG_DRIVER(task->dev, "Task %pK: hw capabilities exceeded\n",
+				     task);
 		return -EINVAL;
 	}
 
@@ -647,7 +652,8 @@ static int exynos_drm_ipp_task_check(struct exynos_drm_ipp_task *task)
 	if (ret)
 		return ret;
 
-	DRM_DEBUG_DRIVER("Task %pK: all checks done.\n", task);
+	DRM_DEV_DEBUG_DRIVER(ipp->dev, "Task %pK: all checks done.\n",
+			     task);
 
 	return ret;
 }
@@ -658,20 +664,26 @@ static int exynos_drm_ipp_task_setup_buffers(struct exynos_drm_ipp_task *task,
 	struct exynos_drm_ipp_buffer *src = &task->src, *dst = &task->dst;
 	int ret = 0;
 
-	DRM_DEBUG_DRIVER("Setting buffer for task %pK\n", task);
+	DRM_DEV_DEBUG_DRIVER(task->dev, "Setting buffer for task %pK\n",
+			     task);
 
 	ret = exynos_drm_ipp_task_setup_buffer(src, filp);
 	if (ret) {
-		DRM_DEBUG_DRIVER("Task %pK: src buffer setup failed\n", task);
+		DRM_DEV_DEBUG_DRIVER(task->dev,
+				     "Task %pK: src buffer setup failed\n",
+				     task);
 		return ret;
 	}
 	ret = exynos_drm_ipp_task_setup_buffer(dst, filp);
 	if (ret) {
-		DRM_DEBUG_DRIVER("Task %pK: dst buffer setup failed\n", task);
+		DRM_DEV_DEBUG_DRIVER(task->dev,
+				     "Task %pK: dst buffer setup failed\n",
+				     task);
 		return ret;
 	}
 
-	DRM_DEBUG_DRIVER("Task %pK: buffers prepared.\n", task);
+	DRM_DEV_DEBUG_DRIVER(task->dev, "Task %pK: buffers prepared.\n",
+			     task);
 
 	return ret;
 }
@@ -691,7 +703,7 @@ static int exynos_drm_ipp_event_create(struct exynos_drm_ipp_task *task,
 	e->event.base.length = sizeof(e->event);
 	e->event.user_data = user_data;
 
-	ret = drm_event_reserve_init(task->dev, file_priv, &e->base,
+	ret = drm_event_reserve_init(task->ipp->drm_dev, file_priv, &e->base,
 				     &e->event.base);
 	if (ret)
 		goto free;
@@ -712,7 +724,7 @@ static void exynos_drm_ipp_event_send(struct exynos_drm_ipp_task *task)
 	task->event->event.tv_usec = now.tv_nsec / NSEC_PER_USEC;
 	task->event->event.sequence = atomic_inc_return(&task->ipp->sequence);
 
-	drm_send_event(task->dev, &task->event->base);
+	drm_send_event(task->ipp->drm_dev, &task->event->base);
 }
 
 static int exynos_drm_ipp_task_cleanup(struct exynos_drm_ipp_task *task)
@@ -749,7 +761,8 @@ void exynos_drm_ipp_task_done(struct exynos_drm_ipp_task *task, int ret)
 	struct exynos_drm_ipp *ipp = task->ipp;
 	unsigned long flags;
 
-	DRM_DEBUG_DRIVER("ipp: %d, task %pK done: %d\n", ipp->id, task, ret);
+	DRM_DEV_DEBUG_DRIVER(task->dev, "ipp: %d, task %pK done: %d\n",
+			     ipp->id, task, ret);
 
 	spin_lock_irqsave(&ipp->lock, flags);
 	if (ipp->task == task)
@@ -773,7 +786,8 @@ static void exynos_drm_ipp_next_task(struct exynos_drm_ipp *ipp)
 	unsigned long flags;
 	int ret;
 
-	DRM_DEBUG_DRIVER("ipp: %d, try to run new task\n", ipp->id);
+	DRM_DEV_DEBUG_DRIVER(ipp->dev, "ipp: %d, try to run new task\n",
+			     ipp->id);
 
 	spin_lock_irqsave(&ipp->lock, flags);
 
@@ -789,7 +803,9 @@ static void exynos_drm_ipp_next_task(struct exynos_drm_ipp *ipp)
 
 	spin_unlock_irqrestore(&ipp->lock, flags);
 
-	DRM_DEBUG_DRIVER("ipp: %d, selected task %pK to run\n", ipp->id, task);
+	DRM_DEV_DEBUG_DRIVER(ipp->dev,
+			     "ipp: %d, selected task %pK to run\n", ipp->id,
+			     task);
 
 	ret = ipp->funcs->commit(ipp, task);
 	if (ret)
@@ -897,15 +913,16 @@ int exynos_drm_ipp_commit_ioctl(struct drm_device *dev, void *data,
 	 * then freed after exynos_drm_ipp_task_done()
 	 */
 	if (arg->flags & DRM_EXYNOS_IPP_FLAG_NONBLOCK) {
-		DRM_DEBUG_DRIVER("ipp: %d, nonblocking processing task %pK\n",
-				 ipp->id, task);
+		DRM_DEV_DEBUG_DRIVER(ipp->dev,
+				     "ipp: %d, nonblocking processing task %pK\n",
+				     ipp->id, task);
 
 		task->flags |= DRM_EXYNOS_IPP_TASK_ASYNC;
 		exynos_drm_ipp_schedule_task(task->ipp, task);
 		ret = 0;
 	} else {
-		DRM_DEBUG_DRIVER("ipp: %d, processing task %pK\n", ipp->id,
-				 task);
+		DRM_DEV_DEBUG_DRIVER(ipp->dev, "ipp: %d, processing task %pK\n",
+				     ipp->id, task);
 		exynos_drm_ipp_schedule_task(ipp, task);
 		ret = wait_event_interruptible(ipp->done_wq,
 					task->flags & DRM_EXYNOS_IPP_TASK_DONE);
diff --git a/drivers/gpu/drm/exynos/exynos_drm_ipp.h b/drivers/gpu/drm/exynos/exynos_drm_ipp.h
index 0b27d4a9bf94..5524c457a947 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_ipp.h
+++ b/drivers/gpu/drm/exynos/exynos_drm_ipp.h
@@ -54,7 +54,8 @@ struct exynos_drm_ipp_funcs {
  * struct exynos_drm_ipp - central picture processor module structure
  */
 struct exynos_drm_ipp {
-	struct drm_device *dev;
+	struct drm_device *drm_dev;
+	struct device *dev;
 	struct list_head head;
 	unsigned int id;
 
@@ -85,7 +86,7 @@ struct exynos_drm_ipp_buffer {
  * has to be performed by the picture processor hardware module
  */
 struct exynos_drm_ipp_task {
-	struct drm_device *dev;
+	struct device *dev;
 	struct exynos_drm_ipp *ipp;
 	struct list_head head;
 
@@ -129,11 +130,11 @@ struct exynos_drm_ipp_formats {
 #define IPP_SCALE_LIMIT(val...)		\
 	.type = (DRM_EXYNOS_IPP_LIMIT_TYPE_SCALE), val
 
-int exynos_drm_ipp_register(struct drm_device *dev, struct exynos_drm_ipp *ipp,
+int exynos_drm_ipp_register(struct device *dev, struct exynos_drm_ipp *ipp,
 		const struct exynos_drm_ipp_funcs *funcs, unsigned int caps,
 		const struct exynos_drm_ipp_formats *formats,
 		unsigned int num_formats, const char *name);
-void exynos_drm_ipp_unregister(struct drm_device *dev,
+void exynos_drm_ipp_unregister(struct device *dev,
 			       struct exynos_drm_ipp *ipp);
 
 void exynos_drm_ipp_task_done(struct exynos_drm_ipp_task *task, int ret);
diff --git a/drivers/gpu/drm/exynos/exynos_drm_mic.c b/drivers/gpu/drm/exynos/exynos_drm_mic.c
index dd02e8a323ef..d1c8411ae7d4 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_mic.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_mic.c
@@ -113,7 +113,8 @@ static void mic_set_path(struct exynos_mic *mic, bool enable)
 
 	ret = regmap_read(mic->sysreg, DSD_CFG_MUX, &val);
 	if (ret) {
-		DRM_ERROR("mic: Failed to read system register\n");
+		DRM_DEV_ERROR(mic->dev,
+			      "mic: Failed to read system register\n");
 		return;
 	}
 
@@ -129,7 +130,8 @@ static void mic_set_path(struct exynos_mic *mic, bool enable)
 
 	ret = regmap_write(mic->sysreg, DSD_CFG_MUX, val);
 	if (ret)
-		DRM_ERROR("mic: Failed to read system register\n");
+		DRM_DEV_ERROR(mic->dev,
+			      "mic: Failed to read system register\n");
 }
 
 static int mic_sw_reset(struct exynos_mic *mic)
@@ -190,7 +192,7 @@ static void mic_set_output_timing(struct exynos_mic *mic)
 	struct videomode vm = mic->vm;
 	u32 reg, bs_size_2d;
 
-	DRM_DEBUG("w: %u, h: %u\n", vm.hactive, vm.vactive);
+	DRM_DEV_DEBUG(mic->dev, "w: %u, h: %u\n", vm.hactive, vm.vactive);
 	bs_size_2d = ((vm.hactive >> 2) << 1) + (vm.vactive % 4);
 	reg = MIC_BS_SIZE_2D(bs_size_2d);
 	writel(reg, mic->reg + MIC_2D_OUTPUT_TIMING_2);
@@ -274,7 +276,7 @@ static void mic_pre_enable(struct drm_bridge *bridge)
 
 	ret = mic_sw_reset(mic);
 	if (ret) {
-		DRM_ERROR("Failed to reset\n");
+		DRM_DEV_ERROR(mic->dev, "Failed to reset\n");
 		goto turn_off;
 	}
 
@@ -354,8 +356,8 @@ static int exynos_mic_resume(struct device *dev)
 	for (i = 0; i < NUM_CLKS; i++) {
 		ret = clk_prepare_enable(mic->clks[i]);
 		if (ret < 0) {
-			DRM_ERROR("Failed to enable clock (%s)\n",
-							clk_names[i]);
+			DRM_DEV_ERROR(dev, "Failed to enable clock (%s)\n",
+				      clk_names[i]);
 			while (--i > -1)
 				clk_disable_unprepare(mic->clks[i]);
 			return ret;
@@ -380,7 +382,8 @@ static int exynos_mic_probe(struct platform_device *pdev)
 
 	mic = devm_kzalloc(dev, sizeof(*mic), GFP_KERNEL);
 	if (!mic) {
-		DRM_ERROR("mic: Failed to allocate memory for MIC object\n");
+		DRM_DEV_ERROR(dev,
+			      "mic: Failed to allocate memory for MIC object\n");
 		ret = -ENOMEM;
 		goto err;
 	}
@@ -389,12 +392,12 @@ static int exynos_mic_probe(struct platform_device *pdev)
 
 	ret = of_address_to_resource(dev->of_node, 0, &res);
 	if (ret) {
-		DRM_ERROR("mic: Failed to get mem region for MIC\n");
+		DRM_DEV_ERROR(dev, "mic: Failed to get mem region for MIC\n");
 		goto err;
 	}
 	mic->reg = devm_ioremap(dev, res.start, resource_size(&res));
 	if (!mic->reg) {
-		DRM_ERROR("mic: Failed to remap for MIC\n");
+		DRM_DEV_ERROR(dev, "mic: Failed to remap for MIC\n");
 		ret = -ENOMEM;
 		goto err;
 	}
@@ -402,7 +405,7 @@ static int exynos_mic_probe(struct platform_device *pdev)
 	mic->sysreg = syscon_regmap_lookup_by_phandle(dev->of_node,
 							"samsung,disp-syscon");
 	if (IS_ERR(mic->sysreg)) {
-		DRM_ERROR("mic: Failed to get system register.\n");
+		DRM_DEV_ERROR(dev, "mic: Failed to get system register.\n");
 		ret = PTR_ERR(mic->sysreg);
 		goto err;
 	}
@@ -410,8 +413,8 @@ static int exynos_mic_probe(struct platform_device *pdev)
 	for (i = 0; i < NUM_CLKS; i++) {
 		mic->clks[i] = devm_clk_get(dev, clk_names[i]);
 		if (IS_ERR(mic->clks[i])) {
-			DRM_ERROR("mic: Failed to get clock (%s)\n",
-								clk_names[i]);
+			DRM_DEV_ERROR(dev, "mic: Failed to get clock (%s)\n",
+				      clk_names[i]);
 			ret = PTR_ERR(mic->clks[i]);
 			goto err;
 		}
@@ -430,7 +433,7 @@ static int exynos_mic_probe(struct platform_device *pdev)
 	if (ret)
 		goto err_pm;
 
-	DRM_DEBUG_KMS("MIC has been probed\n");
+	DRM_DEV_DEBUG_KMS(dev, "MIC has been probed\n");
 
 	return 0;
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_plane.c b/drivers/gpu/drm/exynos/exynos_drm_plane.c
index df0508e0e49e..e18babb25170 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_plane.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_plane.c
@@ -119,9 +119,10 @@ static void exynos_plane_mode_set(struct exynos_drm_plane_state *exynos_state)
 	exynos_state->crtc.w = actual_w;
 	exynos_state->crtc.h = actual_h;
 
-	DRM_DEBUG_KMS("plane : offset_x/y(%d,%d), width/height(%d,%d)",
-			exynos_state->crtc.x, exynos_state->crtc.y,
-			exynos_state->crtc.w, exynos_state->crtc.h);
+	DRM_DEV_DEBUG_KMS(crtc->dev->dev,
+			  "plane : offset_x/y(%d,%d), width/height(%d,%d)",
+			  exynos_state->crtc.x, exynos_state->crtc.y,
+			  exynos_state->crtc.w, exynos_state->crtc.h);
 }
 
 static void exynos_drm_plane_reset(struct drm_plane *plane)
@@ -181,6 +182,7 @@ exynos_drm_plane_check_format(const struct exynos_drm_plane_config *config,
 			      struct exynos_drm_plane_state *state)
 {
 	struct drm_framebuffer *fb = state->base.fb;
+	struct drm_device *dev = fb->dev;
 
 	switch (fb->modifier) {
 	case DRM_FORMAT_MOD_SAMSUNG_64_32_TILE:
@@ -192,7 +194,7 @@ exynos_drm_plane_check_format(const struct exynos_drm_plane_config *config,
 		break;
 
 	default:
-		DRM_ERROR("unsupported pixel format modifier");
+		DRM_DEV_ERROR(dev->dev, "unsupported pixel format modifier");
 		return -ENOTSUPP;
 	}
 
@@ -203,6 +205,7 @@ static int
 exynos_drm_plane_check_size(const struct exynos_drm_plane_config *config,
 			    struct exynos_drm_plane_state *state)
 {
+	struct drm_crtc *crtc = state->base.crtc;
 	bool width_ok = false, height_ok = false;
 
 	if (config->capabilities & EXYNOS_DRM_PLANE_CAP_SCALE)
@@ -225,7 +228,7 @@ exynos_drm_plane_check_size(const struct exynos_drm_plane_config *config,
 	if (width_ok && height_ok)
 		return 0;
 
-	DRM_DEBUG_KMS("scaling mode is not supported");
+	DRM_DEV_DEBUG_KMS(crtc->dev->dev, "scaling mode is not supported");
 	return -ENOTSUPP;
 }
 
@@ -310,7 +313,7 @@ int exynos_plane_init(struct drm_device *dev,
 				       config->num_pixel_formats,
 				       NULL, config->type, NULL);
 	if (err) {
-		DRM_ERROR("failed to initialize plane\n");
+		DRM_DEV_ERROR(dev->dev, "failed to initialize plane\n");
 		return err;
 	}
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_rotator.c b/drivers/gpu/drm/exynos/exynos_drm_rotator.c
index 05abfed6f7f8..b6586fa95ad1 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_rotator.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_rotator.c
@@ -243,9 +243,10 @@ static int rotator_bind(struct device *dev, struct device *master, void *data)
 	struct exynos_drm_ipp *ipp = &rot->ipp;
 
 	rot->drm_dev = drm_dev;
+	ipp->drm_dev = drm_dev;
 	exynos_drm_register_dma(drm_dev, dev);
 
-	exynos_drm_ipp_register(drm_dev, ipp, &ipp_funcs,
+	exynos_drm_ipp_register(dev, ipp, &ipp_funcs,
 			   DRM_EXYNOS_IPP_CAP_CROP | DRM_EXYNOS_IPP_CAP_ROTATE,
 			   rot->formats, rot->num_formats, "rotator");
 
@@ -258,10 +259,9 @@ static void rotator_unbind(struct device *dev, struct device *master,
 			void *data)
 {
 	struct rot_context *rot = dev_get_drvdata(dev);
-	struct drm_device *drm_dev = data;
 	struct exynos_drm_ipp *ipp = &rot->ipp;
 
-	exynos_drm_ipp_unregister(drm_dev, ipp);
+	exynos_drm_ipp_unregister(dev, ipp);
 	exynos_drm_unregister_dma(rot->drm_dev, rot->dev);
 }
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_scaler.c b/drivers/gpu/drm/exynos/exynos_drm_scaler.c
index ed1dd1aec902..f1cbdd1e6e3c 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_scaler.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_scaler.c
@@ -451,9 +451,10 @@ static int scaler_bind(struct device *dev, struct device *master, void *data)
 	struct exynos_drm_ipp *ipp = &scaler->ipp;
 
 	scaler->drm_dev = drm_dev;
+	ipp->drm_dev = drm_dev;
 	exynos_drm_register_dma(drm_dev, dev);
 
-	exynos_drm_ipp_register(drm_dev, ipp, &ipp_funcs,
+	exynos_drm_ipp_register(dev, ipp, &ipp_funcs,
 			DRM_EXYNOS_IPP_CAP_CROP | DRM_EXYNOS_IPP_CAP_ROTATE |
 			DRM_EXYNOS_IPP_CAP_SCALE | DRM_EXYNOS_IPP_CAP_CONVERT,
 			scaler->scaler_data->formats,
@@ -468,10 +469,9 @@ static void scaler_unbind(struct device *dev, struct device *master,
 			void *data)
 {
 	struct scaler_context *scaler = dev_get_drvdata(dev);
-	struct drm_device *drm_dev = data;
 	struct exynos_drm_ipp *ipp = &scaler->ipp;
 
-	exynos_drm_ipp_unregister(drm_dev, ipp);
+	exynos_drm_ipp_unregister(dev, ipp);
 	exynos_drm_unregister_dma(scaler->drm_dev, scaler->dev);
 }
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_vidi.c b/drivers/gpu/drm/exynos/exynos_drm_vidi.c
index 29f4c1932aed..44bcb2d60bb2 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_vidi.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_vidi.c
@@ -40,8 +40,8 @@
 
 struct vidi_context {
 	struct drm_encoder		encoder;
-	struct platform_device		*pdev;
 	struct drm_device		*drm_dev;
+	struct device			*dev;
 	struct exynos_drm_crtc		*crtc;
 	struct drm_connector		connector;
 	struct exynos_drm_plane		planes[WINDOWS_NR];
@@ -123,7 +123,7 @@ static void vidi_update_plane(struct exynos_drm_crtc *crtc,
 		return;
 
 	addr = exynos_drm_fb_dma_addr(state->fb, 0);
-	DRM_DEBUG_KMS("dma_addr = %pad\n", &addr);
+	DRM_DEV_DEBUG_KMS(ctx->dev, "dma_addr = %pad\n", &addr);
 }
 
 static void vidi_enable(struct exynos_drm_crtc *crtc)
@@ -205,11 +205,11 @@ static ssize_t vidi_store_connection(struct device *dev,
 
 	/* if raw_edid isn't same as fake data then it can't be tested. */
 	if (ctx->raw_edid != (struct edid *)fake_edid_info) {
-		DRM_DEBUG_KMS("edid data is not fake data.\n");
+		DRM_DEV_DEBUG_KMS(dev, "edid data is not fake data.\n");
 		return -EINVAL;
 	}
 
-	DRM_DEBUG_KMS("requested connection.\n");
+	DRM_DEV_DEBUG_KMS(dev, "requested connection.\n");
 
 	drm_helper_hpd_irq_event(ctx->drm_dev);
 
@@ -226,17 +226,20 @@ int vidi_connection_ioctl(struct drm_device *drm_dev, void *data,
 	struct drm_exynos_vidi_connection *vidi = data;
 
 	if (!vidi) {
-		DRM_DEBUG_KMS("user data for vidi is null.\n");
+		DRM_DEV_DEBUG_KMS(ctx->dev,
+				  "user data for vidi is null.\n");
 		return -EINVAL;
 	}
 
 	if (vidi->connection > 1) {
-		DRM_DEBUG_KMS("connection should be 0 or 1.\n");
+		DRM_DEV_DEBUG_KMS(ctx->dev,
+				  "connection should be 0 or 1.\n");
 		return -EINVAL;
 	}
 
 	if (ctx->connected == vidi->connection) {
-		DRM_DEBUG_KMS("same connection request.\n");
+		DRM_DEV_DEBUG_KMS(ctx->dev,
+				  "same connection request.\n");
 		return -EINVAL;
 	}
 
@@ -245,12 +248,14 @@ int vidi_connection_ioctl(struct drm_device *drm_dev, void *data,
 
 		raw_edid = (struct edid *)(unsigned long)vidi->edid;
 		if (!drm_edid_is_valid(raw_edid)) {
-			DRM_DEBUG_KMS("edid data is invalid.\n");
+			DRM_DEV_DEBUG_KMS(ctx->dev,
+					  "edid data is invalid.\n");
 			return -EINVAL;
 		}
 		ctx->raw_edid = drm_edid_duplicate(raw_edid);
 		if (!ctx->raw_edid) {
-			DRM_DEBUG_KMS("failed to allocate raw_edid.\n");
+			DRM_DEV_DEBUG_KMS(ctx->dev,
+					  "failed to allocate raw_edid.\n");
 			return -ENOMEM;
 		}
 	} else {
@@ -308,14 +313,14 @@ static int vidi_get_modes(struct drm_connector *connector)
 	 * to ctx->raw_edid through specific ioctl.
 	 */
 	if (!ctx->raw_edid) {
-		DRM_DEBUG_KMS("raw_edid is null.\n");
+		DRM_DEV_DEBUG_KMS(ctx->dev, "raw_edid is null.\n");
 		return -EFAULT;
 	}
 
 	edid_len = (1 + ctx->raw_edid->extensions) * EDID_LENGTH;
 	edid = kmemdup(ctx->raw_edid, edid_len, GFP_KERNEL);
 	if (!edid) {
-		DRM_DEBUG_KMS("failed to allocate edid\n");
+		DRM_DEV_DEBUG_KMS(ctx->dev, "failed to allocate edid\n");
 		return -ENOMEM;
 	}
 
@@ -339,7 +344,8 @@ static int vidi_create_connector(struct drm_encoder *encoder)
 	ret = drm_connector_init(ctx->drm_dev, connector,
 			&vidi_connector_funcs, DRM_MODE_CONNECTOR_VIRTUAL);
 	if (ret) {
-		DRM_ERROR("Failed to initialize connector with drm\n");
+		DRM_DEV_ERROR(ctx->dev,
+			      "Failed to initialize connector with drm\n");
 		return ret;
 	}
 
@@ -402,7 +408,7 @@ static int vidi_bind(struct device *dev, struct device *master, void *data)
 	ctx->crtc = exynos_drm_crtc_create(drm_dev, &exynos_plane->base,
 			EXYNOS_DISPLAY_TYPE_VIDI, &vidi_crtc_ops, ctx);
 	if (IS_ERR(ctx->crtc)) {
-		DRM_ERROR("failed to create crtc.\n");
+		DRM_DEV_ERROR(dev, "failed to create crtc.\n");
 		return PTR_ERR(ctx->crtc);
 	}
 
@@ -417,7 +423,8 @@ static int vidi_bind(struct device *dev, struct device *master, void *data)
 
 	ret = vidi_create_connector(encoder);
 	if (ret) {
-		DRM_ERROR("failed to create connector ret = %d\n", ret);
+		DRM_DEV_ERROR(dev, "failed to create connector ret = %d\n",
+			      ret);
 		drm_encoder_cleanup(encoder);
 		return ret;
 	}
@@ -441,13 +448,14 @@ static const struct component_ops vidi_component_ops = {
 static int vidi_probe(struct platform_device *pdev)
 {
 	struct vidi_context *ctx;
+	struct device *dev = &pdev->dev;
 	int ret;
 
-	ctx = devm_kzalloc(&pdev->dev, sizeof(*ctx), GFP_KERNEL);
+	ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
 	if (!ctx)
 		return -ENOMEM;
 
-	ctx->pdev = pdev;
+	ctx->dev = dev;
 
 	timer_setup(&ctx->timer, vidi_fake_vblank_timer, 0);
 
@@ -455,20 +463,21 @@ static int vidi_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, ctx);
 
-	ret = device_create_file(&pdev->dev, &dev_attr_connection);
+	ret = device_create_file(dev, &dev_attr_connection);
 	if (ret < 0) {
-		DRM_ERROR("failed to create connection sysfs.\n");
+		DRM_DEV_ERROR(dev,
+			      "failed to create connection sysfs.\n");
 		return ret;
 	}
 
-	ret = component_add(&pdev->dev, &vidi_component_ops);
+	ret = component_add(dev, &vidi_component_ops);
 	if (ret)
 		goto err_remove_file;
 
 	return ret;
 
 err_remove_file:
-	device_remove_file(&pdev->dev, &dev_attr_connection);
+	device_remove_file(dev, &dev_attr_connection);
 
 	return ret;
 }
diff --git a/drivers/gpu/drm/exynos/exynos_hdmi.c b/drivers/gpu/drm/exynos/exynos_hdmi.c
index 8e2c02fc66e8..19c252f659dd 100644
--- a/drivers/gpu/drm/exynos/exynos_hdmi.c
+++ b/drivers/gpu/drm/exynos/exynos_hdmi.c
@@ -885,9 +885,9 @@ static int hdmi_get_modes(struct drm_connector *connector)
 		return -ENODEV;
 
 	hdata->dvi_mode = !drm_detect_hdmi_monitor(edid);
-	DRM_DEBUG_KMS("%s : width[%d] x height[%d]\n",
-		(hdata->dvi_mode ? "dvi monitor" : "hdmi monitor"),
-		edid->width_cm, edid->height_cm);
+	DRM_DEV_DEBUG_KMS(hdata->dev, "%s : width[%d] x height[%d]\n",
+			  (hdata->dvi_mode ? "dvi monitor" : "hdmi monitor"),
+			  edid->width_cm, edid->height_cm);
 
 	drm_connector_update_edid_property(connector, edid);
 	cec_notifier_set_phys_addr_from_edid(hdata->notifier, edid);
@@ -908,7 +908,8 @@ static int hdmi_find_phy_conf(struct hdmi_context *hdata, u32 pixel_clock)
 		if (confs->data[i].pixel_clock == pixel_clock)
 			return i;
 
-	DRM_DEBUG_KMS("Could not find phy config for %d\n", pixel_clock);
+	DRM_DEV_DEBUG_KMS(hdata->dev, "Could not find phy config for %d\n",
+			  pixel_clock);
 	return -EINVAL;
 }
 
@@ -918,10 +919,11 @@ static int hdmi_mode_valid(struct drm_connector *connector,
 	struct hdmi_context *hdata = connector_to_hdmi(connector);
 	int ret;
 
-	DRM_DEBUG_KMS("xres=%d, yres=%d, refresh=%d, intl=%d clock=%d\n",
-		mode->hdisplay, mode->vdisplay, mode->vrefresh,
-		(mode->flags & DRM_MODE_FLAG_INTERLACE) ? true :
-		false, mode->clock * 1000);
+	DRM_DEV_DEBUG_KMS(hdata->dev,
+			  "xres=%d, yres=%d, refresh=%d, intl=%d clock=%d\n",
+			  mode->hdisplay, mode->vdisplay, mode->vrefresh,
+			  (mode->flags & DRM_MODE_FLAG_INTERLACE) ? true :
+			  false, mode->clock * 1000);
 
 	ret = hdmi_find_phy_conf(hdata, mode->clock * 1000);
 	if (ret < 0)
@@ -947,7 +949,8 @@ static int hdmi_create_connector(struct drm_encoder *encoder)
 	ret = drm_connector_init(hdata->drm_dev, connector,
 			&hdmi_connector_funcs, DRM_MODE_CONNECTOR_HDMIA);
 	if (ret) {
-		DRM_ERROR("Failed to initialize connector with drm\n");
+		DRM_DEV_ERROR(hdata->dev,
+			      "Failed to initialize connector with drm\n");
 		return ret;
 	}
 
@@ -957,7 +960,7 @@ static int hdmi_create_connector(struct drm_encoder *encoder)
 	if (hdata->bridge) {
 		ret = drm_bridge_attach(encoder, hdata->bridge, NULL);
 		if (ret)
-			DRM_ERROR("Failed to attach bridge\n");
+			DRM_DEV_ERROR(hdata->dev, "Failed to attach bridge\n");
 	}
 
 	return ret;
@@ -1002,8 +1005,10 @@ static bool hdmi_mode_fixup(struct drm_encoder *encoder,
 			DRM_INFO("desired mode doesn't exist so\n");
 			DRM_INFO("use the most suitable mode among modes.\n");
 
-			DRM_DEBUG_KMS("Adjusted Mode: [%d]x[%d] [%d]Hz\n",
-				m->hdisplay, m->vdisplay, m->vrefresh);
+			DRM_DEV_DEBUG_KMS(dev->dev,
+					  "Adjusted Mode: [%d]x[%d] [%d]Hz\n",
+					  m->hdisplay, m->vdisplay,
+					  m->vrefresh);
 
 			drm_mode_copy(adjusted_mode, m);
 			break;
@@ -1169,13 +1174,15 @@ static void hdmiphy_wait_for_pll(struct hdmi_context *hdata)
 		u32 val = hdmi_reg_read(hdata, HDMI_PHY_STATUS);
 
 		if (val & HDMI_PHY_STATUS_READY) {
-			DRM_DEBUG_KMS("PLL stabilized after %d tries\n", tries);
+			DRM_DEV_DEBUG_KMS(hdata->dev,
+					  "PLL stabilized after %d tries\n",
+					  tries);
 			return;
 		}
 		usleep_range(10, 20);
 	}
 
-	DRM_ERROR("PLL could not reach steady state\n");
+	DRM_DEV_ERROR(hdata->dev, "PLL could not reach steady state\n");
 }
 
 static void hdmi_v13_mode_apply(struct hdmi_context *hdata)
@@ -1411,7 +1418,7 @@ static void hdmiphy_conf_apply(struct hdmi_context *hdata)
 
 	ret = hdmi_find_phy_conf(hdata, m->clock * 1000);
 	if (ret < 0) {
-		DRM_ERROR("failed to find hdmiphy conf\n");
+		DRM_DEV_ERROR(hdata->dev, "failed to find hdmiphy conf\n");
 		return;
 	}
 	phy_conf = hdata->drv_data->phy_confs.data[ret].conf;
@@ -1423,7 +1430,7 @@ static void hdmiphy_conf_apply(struct hdmi_context *hdata)
 	hdmiphy_enable_mode_set(hdata, true);
 	ret = hdmiphy_reg_write_buf(hdata, 0, phy_conf, 32);
 	if (ret) {
-		DRM_ERROR("failed to configure hdmiphy\n");
+		DRM_DEV_ERROR(hdata->dev, "failed to configure hdmiphy\n");
 		return;
 	}
 	hdmiphy_enable_mode_set(hdata, false);
@@ -1460,7 +1467,8 @@ static void hdmiphy_enable(struct hdmi_context *hdata)
 	pm_runtime_get_sync(hdata->dev);
 
 	if (regulator_bulk_enable(ARRAY_SIZE(supply), hdata->regul_bulk))
-		DRM_DEBUG_KMS("failed to enable regulator bulk\n");
+		DRM_DEV_DEBUG_KMS(hdata->dev,
+				  "failed to enable regulator bulk\n");
 
 	regmap_update_bits(hdata->pmureg, PMU_HDMI_PHY_CONTROL,
 			PMU_HDMI_PHY_ENABLE_BIT, 1);
@@ -1734,7 +1742,7 @@ static int hdmi_bridge_init(struct hdmi_context *hdata)
 	np = of_graph_get_remote_port_parent(ep);
 	of_node_put(ep);
 	if (!np) {
-		DRM_ERROR("failed to get remote port parent");
+		DRM_DEV_ERROR(dev, "failed to get remote port parent");
 		return -EINVAL;
 	}
 
@@ -1752,17 +1760,17 @@ static int hdmi_resources_init(struct hdmi_context *hdata)
 	struct device *dev = hdata->dev;
 	int i, ret;
 
-	DRM_DEBUG_KMS("HDMI resource init\n");
+	DRM_DEV_DEBUG_KMS(dev, "HDMI resource init\n");
 
 	hdata->hpd_gpio = devm_gpiod_get(dev, "hpd", GPIOD_IN);
 	if (IS_ERR(hdata->hpd_gpio)) {
-		DRM_ERROR("cannot get hpd gpio property\n");
+		DRM_DEV_ERROR(dev, "cannot get hpd gpio property\n");
 		return PTR_ERR(hdata->hpd_gpio);
 	}
 
 	hdata->irq = gpiod_to_irq(hdata->hpd_gpio);
 	if (hdata->irq < 0) {
-		DRM_ERROR("failed to get GPIO irq\n");
+		DRM_DEV_ERROR(dev, "failed to get GPIO irq\n");
 		return  hdata->irq;
 	}
 
@@ -1780,7 +1788,7 @@ static int hdmi_resources_init(struct hdmi_context *hdata)
 	ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(supply), hdata->regul_bulk);
 	if (ret) {
 		if (ret != -EPROBE_DEFER)
-			DRM_ERROR("failed to get regulators\n");
+			DRM_DEV_ERROR(dev, "failed to get regulators\n");
 		return ret;
 	}
 
@@ -1792,7 +1800,8 @@ static int hdmi_resources_init(struct hdmi_context *hdata)
 
 		ret = regulator_enable(hdata->reg_hdmi_en);
 		if (ret) {
-			DRM_ERROR("failed to enable hdmi-en regulator\n");
+			DRM_DEV_ERROR(dev,
+				      "failed to enable hdmi-en regulator\n");
 			return ret;
 		}
 	}
@@ -1845,7 +1854,8 @@ static int hdmi_bind(struct device *dev, struct device *master, void *data)
 
 	ret = hdmi_create_connector(encoder);
 	if (ret) {
-		DRM_ERROR("failed to create connector ret = %d\n", ret);
+		DRM_DEV_ERROR(dev, "failed to create connector ret = %d\n",
+			      ret);
 		drm_encoder_cleanup(encoder);
 		return ret;
 	}
@@ -1875,7 +1885,8 @@ static int hdmi_get_ddc_adapter(struct hdmi_context *hdata)
 		np = of_parse_phandle(hdata->dev->of_node, "ddc", 0);
 
 	if (!np) {
-		DRM_ERROR("Failed to find ddc node in device tree\n");
+		DRM_DEV_ERROR(hdata->dev,
+			      "Failed to find ddc node in device tree\n");
 		return -ENODEV;
 	}
 
@@ -1902,7 +1913,8 @@ static int hdmi_get_phy_io(struct hdmi_context *hdata)
 	if (!np) {
 		np = of_parse_phandle(hdata->dev->of_node, "phy", 0);
 		if (!np) {
-			DRM_ERROR("Failed to find hdmiphy node in device tree\n");
+			DRM_DEV_ERROR(hdata->dev,
+				      "Failed to find hdmiphy node in device tree\n");
 			return -ENODEV;
 		}
 	}
@@ -1910,7 +1922,8 @@ static int hdmi_get_phy_io(struct hdmi_context *hdata)
 	if (hdata->drv_data->is_apb_phy) {
 		hdata->regs_hdmiphy = of_iomap(np, 0);
 		if (!hdata->regs_hdmiphy) {
-			DRM_ERROR("failed to ioremap hdmi phy\n");
+			DRM_DEV_ERROR(hdata->dev,
+				      "failed to ioremap hdmi phy\n");
 			ret = -ENOMEM;
 			goto out;
 		}
@@ -1951,7 +1964,7 @@ static int hdmi_probe(struct platform_device *pdev)
 	ret = hdmi_resources_init(hdata);
 	if (ret) {
 		if (ret != -EPROBE_DEFER)
-			DRM_ERROR("hdmi_resources_init failed\n");
+			DRM_DEV_ERROR(dev, "hdmi_resources_init failed\n");
 		return ret;
 	}
 
@@ -1977,14 +1990,14 @@ static int hdmi_probe(struct platform_device *pdev)
 			IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
 			"hdmi", hdata);
 	if (ret) {
-		DRM_ERROR("failed to register hdmi interrupt\n");
+		DRM_DEV_ERROR(dev, "failed to register hdmi interrupt\n");
 		goto err_hdmiphy;
 	}
 
 	hdata->pmureg = syscon_regmap_lookup_by_phandle(dev->of_node,
 			"samsung,syscon-phandle");
 	if (IS_ERR(hdata->pmureg)) {
-		DRM_ERROR("syscon regmap lookup failed.\n");
+		DRM_DEV_ERROR(dev, "syscon regmap lookup failed.\n");
 		ret = -EPROBE_DEFER;
 		goto err_hdmiphy;
 	}
@@ -1993,7 +2006,7 @@ static int hdmi_probe(struct platform_device *pdev)
 		hdata->sysreg = syscon_regmap_lookup_by_phandle(dev->of_node,
 				"samsung,sysreg-phandle");
 		if (IS_ERR(hdata->sysreg)) {
-			DRM_ERROR("sysreg regmap lookup failed.\n");
+			DRM_DEV_ERROR(dev, "sysreg regmap lookup failed.\n");
 			ret = -EPROBE_DEFER;
 			goto err_hdmiphy;
 		}
diff --git a/drivers/gpu/drm/exynos/exynos_mixer.c b/drivers/gpu/drm/exynos/exynos_mixer.c
index f35e4ab55b27..b8415e53964d 100644
--- a/drivers/gpu/drm/exynos/exynos_mixer.c
+++ b/drivers/gpu/drm/exynos/exynos_mixer.c
@@ -228,8 +228,8 @@ static void mixer_regs_dump(struct mixer_context *ctx)
 {
 #define DUMPREG(reg_id) \
 do { \
-	DRM_DEBUG_KMS(#reg_id " = %08x\n", \
-		(u32)readl(ctx->mixer_regs + reg_id)); \
+	DRM_DEV_DEBUG_KMS(ctx->dev, #reg_id " = %08x\n", \
+			 (u32)readl(ctx->mixer_regs + reg_id)); \
 } while (0)
 
 	DUMPREG(MXR_STATUS);
@@ -260,8 +260,8 @@ static void vp_regs_dump(struct mixer_context *ctx)
 {
 #define DUMPREG(reg_id) \
 do { \
-	DRM_DEBUG_KMS(#reg_id " = %08x\n", \
-		(u32) readl(ctx->vp_regs + reg_id)); \
+	DRM_DEV_DEBUG_KMS(ctx->dev, #reg_id " = %08x\n", \
+			 (u32) readl(ctx->vp_regs + reg_id)); \
 } while (0)
 
 	DUMPREG(VP_ENABLE);
@@ -885,7 +885,8 @@ static int mixer_initialize(struct mixer_context *mixer_ctx,
 	/* acquire resources: regs, irqs, clocks */
 	ret = mixer_resources_init(mixer_ctx);
 	if (ret) {
-		DRM_ERROR("mixer_resources_init failed ret=%d\n", ret);
+		DRM_DEV_ERROR(mixer_ctx->dev,
+			      "mixer_resources_init failed ret=%d\n", ret);
 		return ret;
 	}
 
@@ -893,7 +894,8 @@ static int mixer_initialize(struct mixer_context *mixer_ctx,
 		/* acquire vp resources: regs, irqs, clocks */
 		ret = vp_resources_init(mixer_ctx);
 		if (ret) {
-			DRM_ERROR("vp_resources_init failed ret=%d\n", ret);
+			DRM_DEV_ERROR(mixer_ctx->dev,
+				      "vp_resources_init failed ret=%d\n", ret);
 			return ret;
 		}
 	}
@@ -952,7 +954,7 @@ static void mixer_update_plane(struct exynos_drm_crtc *crtc,
 {
 	struct mixer_context *mixer_ctx = crtc->ctx;
 
-	DRM_DEBUG_KMS("win: %d\n", plane->index);
+	DRM_DEV_DEBUG_KMS(mixer_ctx->dev, "win: %d\n", plane->index);
 
 	if (!test_bit(MXR_BIT_POWERED, &mixer_ctx->flags))
 		return;
@@ -969,7 +971,7 @@ static void mixer_disable_plane(struct exynos_drm_crtc *crtc,
 	struct mixer_context *mixer_ctx = crtc->ctx;
 	unsigned long flags;
 
-	DRM_DEBUG_KMS("win: %d\n", plane->index);
+	DRM_DEV_DEBUG_KMS(mixer_ctx->dev, "win: %d\n", plane->index);
 
 	if (!test_bit(MXR_BIT_POWERED, &mixer_ctx->flags))
 		return;
@@ -1046,8 +1048,9 @@ static int mixer_mode_valid(struct exynos_drm_crtc *crtc,
 	struct mixer_context *ctx = crtc->ctx;
 	u32 w = mode->hdisplay, h = mode->vdisplay;
 
-	DRM_DEBUG_KMS("xres=%d, yres=%d, refresh=%d, intl=%d\n", w, h,
-		mode->vrefresh, !!(mode->flags & DRM_MODE_FLAG_INTERLACE));
+	DRM_DEV_DEBUG_KMS(ctx->dev, "xres=%d, yres=%d, refresh=%d, intl=%d\n",
+			  w, h, mode->vrefresh,
+			  !!(mode->flags & DRM_MODE_FLAG_INTERLACE));
 
 	if (ctx->mxr_ver == MXR_VER_128_0_0_184)
 		return MODE_OK;
@@ -1227,7 +1230,7 @@ static int mixer_probe(struct platform_device *pdev)
 
 	ctx = devm_kzalloc(&pdev->dev, sizeof(*ctx), GFP_KERNEL);
 	if (!ctx) {
-		DRM_ERROR("failed to alloc mixer context.\n");
+		DRM_DEV_ERROR(dev, "failed to alloc mixer context.\n");
 		return -ENOMEM;
 	}
 
@@ -1282,27 +1285,33 @@ static int __maybe_unused exynos_mixer_resume(struct device *dev)
 
 	ret = clk_prepare_enable(ctx->mixer);
 	if (ret < 0) {
-		DRM_ERROR("Failed to prepare_enable the mixer clk [%d]\n", ret);
+		DRM_DEV_ERROR(ctx->dev,
+			      "Failed to prepare_enable the mixer clk [%d]\n",
+			      ret);
 		return ret;
 	}
 	ret = clk_prepare_enable(ctx->hdmi);
 	if (ret < 0) {
-		DRM_ERROR("Failed to prepare_enable the hdmi clk [%d]\n", ret);
+		DRM_DEV_ERROR(dev,
+			      "Failed to prepare_enable the hdmi clk [%d]\n",
+			      ret);
 		return ret;
 	}
 	if (test_bit(MXR_BIT_VP_ENABLED, &ctx->flags)) {
 		ret = clk_prepare_enable(ctx->vp);
 		if (ret < 0) {
-			DRM_ERROR("Failed to prepare_enable the vp clk [%d]\n",
-				  ret);
+			DRM_DEV_ERROR(dev,
+				      "Failed to prepare_enable the vp clk [%d]\n",
+				      ret);
 			return ret;
 		}
 		if (test_bit(MXR_BIT_HAS_SCLK, &ctx->flags)) {
 			ret = clk_prepare_enable(ctx->sclk_mixer);
 			if (ret < 0) {
-				DRM_ERROR("Failed to prepare_enable the " \
+				DRM_DEV_ERROR(dev,
+					   "Failed to prepare_enable the " \
 					   "sclk_mixer clk [%d]\n",
-					  ret);
+					   ret);
 				return ret;
 			}
 		}
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_crtc.c b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_crtc.c
index bf256971063d..83c841b50272 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_crtc.c
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_crtc.c
@@ -94,7 +94,7 @@ static void fsl_dcu_drm_crtc_mode_set_nofb(struct drm_crtc *crtc)
 	drm_display_mode_to_videomode(mode, &vm);
 
 	/* INV_PXCK as default (most display sample data on rising edge) */
-	if (!(con->display_info.bus_flags & DRM_BUS_FLAG_PIXDATA_POSEDGE))
+	if (!(con->display_info.bus_flags & DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE))
 		pol |= DCU_SYN_POL_INV_PXCK;
 
 	if (vm.flags & DISPLAY_FLAGS_HSYNC_LOW)
diff --git a/drivers/gpu/drm/gma500/framebuffer.c b/drivers/gpu/drm/gma500/framebuffer.c
index c934b3df1f81..a9d3a4a30ab8 100644
--- a/drivers/gpu/drm/gma500/framebuffer.c
+++ b/drivers/gpu/drm/gma500/framebuffer.c
@@ -389,7 +389,6 @@ static int psbfb_create(struct psb_fbdev *fbdev,
 		ret = PTR_ERR(info);
 		goto out;
 	}
-	info->par = fbdev;
 
 	mode_cmd.pixel_format = drm_mode_legacy_fb_format(bpp, depth);
 
@@ -402,9 +401,6 @@ static int psbfb_create(struct psb_fbdev *fbdev,
 
 	fbdev->psb_fb_helper.fb = fb;
 
-	drm_fb_helper_fill_fix(info, fb->pitches[0], fb->format->depth);
-	strcpy(info->fix.id, "psbdrmfb");
-
 	if (dev_priv->ops->accel_2d && pitch_lines > 8)	/* 2D engine */
 		info->fbops = &psbfb_ops;
 	else if (gtt_roll) {	/* GTT rolling seems best */
@@ -427,8 +423,7 @@ static int psbfb_create(struct psb_fbdev *fbdev,
 		info->apertures->ranges[0].size = dev_priv->gtt.stolen_size;
 	}
 
-	drm_fb_helper_fill_var(info, &fbdev->psb_fb_helper,
-				sizes->fb_width, sizes->fb_height);
+	drm_fb_helper_fill_info(info, &fbdev->psb_fb_helper, sizes);
 
 	info->fix.mmio_start = pci_resource_start(dev->pdev, 0);
 	info->fix.mmio_len = pci_resource_len(dev->pdev, 0);
diff --git a/drivers/gpu/drm/gma500/framebuffer.h b/drivers/gpu/drm/gma500/framebuffer.h
index 23dc3c5f8f0d..e8e6357f033b 100644
--- a/drivers/gpu/drm/gma500/framebuffer.h
+++ b/drivers/gpu/drm/gma500/framebuffer.h
@@ -34,7 +34,7 @@ struct psb_framebuffer {
 };
 
 struct psb_fbdev {
-	struct drm_fb_helper psb_fb_helper;
+	struct drm_fb_helper psb_fb_helper; /* must be first */
 	struct psb_framebuffer pfb;
 };
 
diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.h b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.h
index 3c168ae77b0c..0a381c22de26 100644
--- a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.h
+++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.h
@@ -31,7 +31,7 @@ struct hibmc_framebuffer {
 };
 
 struct hibmc_fbdev {
-	struct drm_fb_helper helper;
+	struct drm_fb_helper helper; /* must be first */
 	struct hibmc_framebuffer *fb;
 	int size;
 };
diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_fbdev.c b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_fbdev.c
index de9d7cc97e44..8026859aa07d 100644
--- a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_fbdev.c
+++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_fbdev.c
@@ -116,8 +116,6 @@ static int hibmc_drm_fb_create(struct drm_fb_helper *helper,
 		goto out_release_fbi;
 	}
 
-	info->par = hi_fbdev;
-
 	hi_fbdev->fb = hibmc_framebuffer_init(priv->dev, &mode_cmd, gobj);
 	if (IS_ERR(hi_fbdev->fb)) {
 		ret = PTR_ERR(hi_fbdev->fb);
@@ -129,14 +127,9 @@ static int hibmc_drm_fb_create(struct drm_fb_helper *helper,
 	priv->fbdev->size = size;
 	hi_fbdev->helper.fb = &hi_fbdev->fb->fb;
 
-	strcpy(info->fix.id, "hibmcdrmfb");
-
 	info->fbops = &hibmc_drm_fb_ops;
 
-	drm_fb_helper_fill_fix(info, hi_fbdev->fb->fb.pitches[0],
-			       hi_fbdev->fb->fb.format->depth);
-	drm_fb_helper_fill_var(info, &priv->fbdev->helper, sizes->fb_width,
-			       sizes->fb_height);
+	drm_fb_helper_fill_info(info, &priv->fbdev->helper, sizes);
 
 	info->screen_base = bo->kmap.virtual;
 	info->screen_size = size;
diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_vdac.c b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_vdac.c
index d2cf7317930a..8c2f9b9cafb3 100644
--- a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_vdac.c
+++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_vdac.c
@@ -33,17 +33,10 @@ static enum drm_mode_status hibmc_connector_mode_valid(struct drm_connector *con
 	return MODE_OK;
 }
 
-static struct drm_encoder *
-hibmc_connector_best_encoder(struct drm_connector *connector)
-{
-	return drm_encoder_find(connector->dev, NULL, connector->encoder_ids[0]);
-}
-
 static const struct drm_connector_helper_funcs
 	hibmc_connector_helper_funcs = {
 	.get_modes = hibmc_connector_get_modes,
 	.mode_valid = hibmc_connector_mode_valid,
-	.best_encoder = hibmc_connector_best_encoder,
 };
 
 static const struct drm_connector_funcs hibmc_connector_funcs = {
diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_ttm.c b/drivers/gpu/drm/hisilicon/hibmc/hibmc_ttm.c
index dd383267884c..6093c421daff 100644
--- a/drivers/gpu/drm/hisilicon/hibmc/hibmc_ttm.c
+++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_ttm.c
@@ -21,8 +21,6 @@
 
 #include "hibmc_drm_drv.h"
 
-#define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT)
-
 static inline struct hibmc_drm_private *
 hibmc_bdev(struct ttm_bo_device *bd)
 {
@@ -191,7 +189,6 @@ int hibmc_mm_init(struct hibmc_drm_private *hibmc)
 	ret = ttm_bo_device_init(&hibmc->bdev,
 				 &hibmc_bo_driver,
 				 dev->anon_inode->i_mapping,
-				 DRM_FILE_PAGE_OFFSET,
 				 true);
 	if (ret) {
 		DRM_ERROR("error initializing bo driver: %d\n", ret);
@@ -322,14 +319,9 @@ int hibmc_bo_unpin(struct hibmc_bo *bo)
 
 int hibmc_mmap(struct file *filp, struct vm_area_struct *vma)
 {
-	struct drm_file *file_priv;
-	struct hibmc_drm_private *hibmc;
-
-	if (unlikely(vma->vm_pgoff < DRM_FILE_PAGE_OFFSET))
-		return -EINVAL;
+	struct drm_file *file_priv = filp->private_data;
+	struct hibmc_drm_private *hibmc = file_priv->minor->dev->dev_private;
 
-	file_priv = filp->private_data;
-	hibmc = file_priv->minor->dev->dev_private;
 	return ttm_bo_mmap(filp, vma, &hibmc->bdev);
 }
 
diff --git a/drivers/gpu/drm/i915/.gitignore b/drivers/gpu/drm/i915/.gitignore
new file mode 100644
index 000000000000..cff45d81f42f
--- /dev/null
+++ b/drivers/gpu/drm/i915/.gitignore
@@ -0,0 +1 @@
+header_test_*.c
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index 1787e1299b1b..fbcb0904f4a8 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -32,10 +32,13 @@ CFLAGS_intel_fbdev.o = $(call cc-disable-warning, override-init)
 subdir-ccflags-y += \
 	$(call as-instr,movntdqa (%eax)$(comma)%xmm0,-DCONFIG_AS_MOVNTDQA)
 
+# Extra header tests
+include $(src)/Makefile.header-test
+
 # Please keep these build lists sorted!
 
 # core driver code
-i915-y := i915_drv.o \
+i915-y += i915_drv.o \
 	  i915_irq.o \
 	  i915_memcpy.o \
 	  i915_mm.o \
@@ -46,6 +49,7 @@ i915-y := i915_drv.o \
 	  i915_sw_fence.o \
 	  i915_syncmap.o \
 	  i915_sysfs.o \
+	  i915_user_extensions.o \
 	  intel_csr.o \
 	  intel_device_info.o \
 	  intel_pm.o \
@@ -77,6 +81,7 @@ i915-y += \
 	  i915_gem_tiling.o \
 	  i915_gem_userptr.o \
 	  i915_gemfs.o \
+	  i915_globals.o \
 	  i915_query.o \
 	  i915_request.o \
 	  i915_scheduler.o \
@@ -84,6 +89,7 @@ i915-y += \
 	  i915_trace_points.o \
 	  i915_vma.o \
 	  intel_breadcrumbs.o \
+	  intel_context.o \
 	  intel_engine_cs.o \
 	  intel_hangcheck.o \
 	  intel_lrc.o \
diff --git a/drivers/gpu/drm/i915/Makefile.header-test b/drivers/gpu/drm/i915/Makefile.header-test
new file mode 100644
index 000000000000..c1c391816fa7
--- /dev/null
+++ b/drivers/gpu/drm/i915/Makefile.header-test
@@ -0,0 +1,47 @@
+# SPDX-License-Identifier: MIT
+# Copyright © 2019 Intel Corporation
+
+# Test the headers are compilable as standalone units
+header_test := \
+	i915_active_types.h \
+	i915_gem_context_types.h \
+	i915_priolist_types.h \
+	i915_scheduler_types.h \
+	i915_timeline_types.h \
+	intel_atomic_plane.h \
+	intel_audio.h \
+	intel_cdclk.h \
+	intel_color.h \
+	intel_connector.h \
+	intel_context_types.h \
+	intel_crt.h \
+	intel_csr.h \
+	intel_ddi.h \
+	intel_dp.h \
+	intel_dvo.h \
+	intel_engine_types.h \
+	intel_fbc.h \
+	intel_fbdev.h \
+	intel_frontbuffer.h \
+	intel_hdcp.h \
+	intel_hdmi.h \
+	intel_lspcon.h \
+	intel_lvds.h \
+	intel_panel.h \
+	intel_pipe_crc.h \
+	intel_pm.h \
+	intel_psr.h \
+	intel_sdvo.h \
+	intel_sprite.h \
+	intel_tv.h \
+	intel_workarounds_types.h
+
+quiet_cmd_header_test = HDRTEST $@
+      cmd_header_test = echo "\#include \"$(<F)\"" > $@
+
+header_test_%.c: %.h
+	$(call cmd,header_test)
+
+i915-$(CONFIG_DRM_I915_WERROR) += $(foreach h,$(header_test),$(patsubst %.h,header_test_%.o,$(h)))
+
+clean-files += $(foreach h,$(header_test),$(patsubst %.h,header_test_%.c,$(h)))
diff --git a/drivers/gpu/drm/i915/gvt/Makefile b/drivers/gpu/drm/i915/gvt/Makefile
index 271fb46d4dd0..ea8324abc784 100644
--- a/drivers/gpu/drm/i915/gvt/Makefile
+++ b/drivers/gpu/drm/i915/gvt/Makefile
@@ -5,5 +5,5 @@ GVT_SOURCE := gvt.o aperture_gm.o handlers.o vgpu.o trace_points.o firmware.o \
 	execlist.o scheduler.o sched_policy.o mmio_context.o cmd_parser.o debugfs.o \
 	fb_decoder.o dmabuf.o page_track.o
 
-ccflags-y				+= -I$(src) -I$(src)/$(GVT_DIR)
+ccflags-y				+= -I $(srctree)/$(src) -I $(srctree)/$(src)/$(GVT_DIR)/
 i915-y					+= $(addprefix $(GVT_DIR)/, $(GVT_SOURCE))
diff --git a/drivers/gpu/drm/i915/gvt/cmd_parser.c b/drivers/gpu/drm/i915/gvt/cmd_parser.c
index 3592d04c33b2..ab002cfd3cab 100644
--- a/drivers/gpu/drm/i915/gvt/cmd_parser.c
+++ b/drivers/gpu/drm/i915/gvt/cmd_parser.c
@@ -391,12 +391,12 @@ struct cmd_info {
 #define F_POST_HANDLE	(1<<2)
 	u32 flag;
 
-#define R_RCS	(1 << RCS)
-#define R_VCS1  (1 << VCS)
-#define R_VCS2  (1 << VCS2)
+#define R_RCS	BIT(RCS0)
+#define R_VCS1  BIT(VCS0)
+#define R_VCS2  BIT(VCS1)
 #define R_VCS	(R_VCS1 | R_VCS2)
-#define R_BCS	(1 << BCS)
-#define R_VECS	(1 << VECS)
+#define R_BCS	BIT(BCS0)
+#define R_VECS	BIT(VECS0)
 #define R_ALL (R_RCS | R_VCS | R_BCS | R_VECS)
 	/* rings that support this cmd: BLT/RCS/VCS/VECS */
 	u16 rings;
@@ -558,7 +558,7 @@ static const struct decode_info decode_info_vebox = {
 };
 
 static const struct decode_info *ring_decode_info[I915_NUM_ENGINES][8] = {
-	[RCS] = {
+	[RCS0] = {
 		&decode_info_mi,
 		NULL,
 		NULL,
@@ -569,7 +569,7 @@ static const struct decode_info *ring_decode_info[I915_NUM_ENGINES][8] = {
 		NULL,
 	},
 
-	[VCS] = {
+	[VCS0] = {
 		&decode_info_mi,
 		NULL,
 		NULL,
@@ -580,7 +580,7 @@ static const struct decode_info *ring_decode_info[I915_NUM_ENGINES][8] = {
 		NULL,
 	},
 
-	[BCS] = {
+	[BCS0] = {
 		&decode_info_mi,
 		NULL,
 		&decode_info_2d,
@@ -591,7 +591,7 @@ static const struct decode_info *ring_decode_info[I915_NUM_ENGINES][8] = {
 		NULL,
 	},
 
-	[VECS] = {
+	[VECS0] = {
 		&decode_info_mi,
 		NULL,
 		NULL,
@@ -602,7 +602,7 @@ static const struct decode_info *ring_decode_info[I915_NUM_ENGINES][8] = {
 		NULL,
 	},
 
-	[VCS2] = {
+	[VCS1] = {
 		&decode_info_mi,
 		NULL,
 		NULL,
@@ -631,8 +631,7 @@ static inline const struct cmd_info *find_cmd_entry(struct intel_gvt *gvt,
 	struct cmd_entry *e;
 
 	hash_for_each_possible(gvt->cmd_table, e, hlist, opcode) {
-		if ((opcode == e->info->opcode) &&
-				(e->info->rings & (1 << ring_id)))
+		if (opcode == e->info->opcode && e->info->rings & BIT(ring_id))
 			return e->info;
 	}
 	return NULL;
@@ -943,15 +942,12 @@ static int cmd_handler_lri(struct parser_exec_state *s)
 	struct intel_gvt *gvt = s->vgpu->gvt;
 
 	for (i = 1; i < cmd_len; i += 2) {
-		if (IS_BROADWELL(gvt->dev_priv) &&
-				(s->ring_id != RCS)) {
-			if (s->ring_id == BCS &&
-					cmd_reg(s, i) ==
-					i915_mmio_reg_offset(DERRMR))
+		if (IS_BROADWELL(gvt->dev_priv) && s->ring_id != RCS0) {
+			if (s->ring_id == BCS0 &&
+			    cmd_reg(s, i) == i915_mmio_reg_offset(DERRMR))
 				ret |= 0;
 			else
-				ret |= (cmd_reg_inhibit(s, i)) ?
-					-EBADRQC : 0;
+				ret |= cmd_reg_inhibit(s, i) ? -EBADRQC : 0;
 		}
 		if (ret)
 			break;
@@ -1047,27 +1043,27 @@ struct cmd_interrupt_event {
 };
 
 static struct cmd_interrupt_event cmd_interrupt_events[] = {
-	[RCS] = {
+	[RCS0] = {
 		.pipe_control_notify = RCS_PIPE_CONTROL,
 		.mi_flush_dw = INTEL_GVT_EVENT_RESERVED,
 		.mi_user_interrupt = RCS_MI_USER_INTERRUPT,
 	},
-	[BCS] = {
+	[BCS0] = {
 		.pipe_control_notify = INTEL_GVT_EVENT_RESERVED,
 		.mi_flush_dw = BCS_MI_FLUSH_DW,
 		.mi_user_interrupt = BCS_MI_USER_INTERRUPT,
 	},
-	[VCS] = {
+	[VCS0] = {
 		.pipe_control_notify = INTEL_GVT_EVENT_RESERVED,
 		.mi_flush_dw = VCS_MI_FLUSH_DW,
 		.mi_user_interrupt = VCS_MI_USER_INTERRUPT,
 	},
-	[VCS2] = {
+	[VCS1] = {
 		.pipe_control_notify = INTEL_GVT_EVENT_RESERVED,
 		.mi_flush_dw = VCS2_MI_FLUSH_DW,
 		.mi_user_interrupt = VCS2_MI_USER_INTERRUPT,
 	},
-	[VECS] = {
+	[VECS0] = {
 		.pipe_control_notify = INTEL_GVT_EVENT_RESERVED,
 		.mi_flush_dw = VECS_MI_FLUSH_DW,
 		.mi_user_interrupt = VECS_MI_USER_INTERRUPT,
@@ -1081,6 +1077,7 @@ static int cmd_handler_pipe_control(struct parser_exec_state *s)
 	bool index_mode = false;
 	unsigned int post_sync;
 	int ret = 0;
+	u32 hws_pga, val;
 
 	post_sync = (cmd_val(s, 1) & PIPE_CONTROL_POST_SYNC_OP_MASK) >> 14;
 
@@ -1104,6 +1101,15 @@ static int cmd_handler_pipe_control(struct parser_exec_state *s)
 					index_mode = true;
 				ret |= cmd_address_audit(s, gma, sizeof(u64),
 						index_mode);
+				if (ret)
+					return ret;
+				if (index_mode) {
+					hws_pga = s->vgpu->hws_pga[s->ring_id];
+					gma = hws_pga + gma;
+					patch_value(s, cmd_ptr(s, 2), gma);
+					val = cmd_val(s, 1) & (~(1 << 21));
+					patch_value(s, cmd_ptr(s, 1), val);
+				}
 			}
 		}
 	}
@@ -1321,8 +1327,14 @@ static int gen8_update_plane_mmio_from_mi_display_flip(
 			      info->tile_val << 10);
 	}
 
-	vgpu_vreg_t(vgpu, PIPE_FRMCOUNT_G4X(info->pipe))++;
-	intel_vgpu_trigger_virtual_event(vgpu, info->event);
+	if (info->plane == PLANE_PRIMARY)
+		vgpu_vreg_t(vgpu, PIPE_FLIPCOUNT_G4X(info->pipe))++;
+
+	if (info->async_flip)
+		intel_vgpu_trigger_virtual_event(vgpu, info->event);
+	else
+		set_bit(info->event, vgpu->irq.flip_done_event[info->pipe]);
+
 	return 0;
 }
 
@@ -1567,6 +1579,7 @@ static int cmd_handler_mi_flush_dw(struct parser_exec_state *s)
 	unsigned long gma;
 	bool index_mode = false;
 	int ret = 0;
+	u32 hws_pga, val;
 
 	/* Check post-sync and ppgtt bit */
 	if (((cmd_val(s, 0) >> 14) & 0x3) && (cmd_val(s, 1) & (1 << 2))) {
@@ -1577,6 +1590,15 @@ static int cmd_handler_mi_flush_dw(struct parser_exec_state *s)
 		if (cmd_val(s, 0) & (1 << 21))
 			index_mode = true;
 		ret = cmd_address_audit(s, gma, sizeof(u64), index_mode);
+		if (ret)
+			return ret;
+		if (index_mode) {
+			hws_pga = s->vgpu->hws_pga[s->ring_id];
+			gma = hws_pga + gma;
+			patch_value(s, cmd_ptr(s, 1), gma);
+			val = cmd_val(s, 0) & (~(1 << 21));
+			patch_value(s, cmd_ptr(s, 0), val);
+		}
 	}
 	/* Check notify bit */
 	if ((cmd_val(s, 0) & (1 << 8)))
diff --git a/drivers/gpu/drm/i915/gvt/display.c b/drivers/gpu/drm/i915/gvt/display.c
index e3f9caa7839f..e1c313da6c00 100644
--- a/drivers/gpu/drm/i915/gvt/display.c
+++ b/drivers/gpu/drm/i915/gvt/display.c
@@ -407,7 +407,6 @@ static void emulate_vblank_on_pipe(struct intel_vgpu *vgpu, int pipe)
 		if (!pipe_is_enabled(vgpu, pipe))
 			continue;
 
-		vgpu_vreg_t(vgpu, PIPE_FLIPCOUNT_G4X(pipe))++;
 		intel_vgpu_trigger_virtual_event(vgpu, event);
 	}
 
diff --git a/drivers/gpu/drm/i915/gvt/dmabuf.c b/drivers/gpu/drm/i915/gvt/dmabuf.c
index 69a9a1b2ea4a..4e1e425189ba 100644
--- a/drivers/gpu/drm/i915/gvt/dmabuf.c
+++ b/drivers/gpu/drm/i915/gvt/dmabuf.c
@@ -153,7 +153,7 @@ static struct drm_i915_gem_object *vgpu_create_gem(struct drm_device *dev,
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct drm_i915_gem_object *obj;
 
-	obj = i915_gem_object_alloc(dev_priv);
+	obj = i915_gem_object_alloc();
 	if (obj == NULL)
 		return NULL;
 
diff --git a/drivers/gpu/drm/i915/gvt/execlist.c b/drivers/gpu/drm/i915/gvt/execlist.c
index 70494e394d2c..f21b8fb5b37e 100644
--- a/drivers/gpu/drm/i915/gvt/execlist.c
+++ b/drivers/gpu/drm/i915/gvt/execlist.c
@@ -47,17 +47,16 @@
 		((a)->lrca == (b)->lrca))
 
 static int context_switch_events[] = {
-	[RCS] = RCS_AS_CONTEXT_SWITCH,
-	[BCS] = BCS_AS_CONTEXT_SWITCH,
-	[VCS] = VCS_AS_CONTEXT_SWITCH,
-	[VCS2] = VCS2_AS_CONTEXT_SWITCH,
-	[VECS] = VECS_AS_CONTEXT_SWITCH,
+	[RCS0]  = RCS_AS_CONTEXT_SWITCH,
+	[BCS0]  = BCS_AS_CONTEXT_SWITCH,
+	[VCS0]  = VCS_AS_CONTEXT_SWITCH,
+	[VCS1]  = VCS2_AS_CONTEXT_SWITCH,
+	[VECS0] = VECS_AS_CONTEXT_SWITCH,
 };
 
-static int ring_id_to_context_switch_event(int ring_id)
+static int ring_id_to_context_switch_event(unsigned int ring_id)
 {
-	if (WARN_ON(ring_id < RCS ||
-		    ring_id >= ARRAY_SIZE(context_switch_events)))
+	if (WARN_ON(ring_id >= ARRAY_SIZE(context_switch_events)))
 		return -EINVAL;
 
 	return context_switch_events[ring_id];
@@ -411,7 +410,7 @@ static int complete_execlist_workload(struct intel_vgpu_workload *workload)
 	gvt_dbg_el("complete workload %p status %d\n", workload,
 			workload->status);
 
-	if (workload->status || (vgpu->resetting_eng & ENGINE_MASK(ring_id)))
+	if (workload->status || (vgpu->resetting_eng & BIT(ring_id)))
 		goto out;
 
 	if (!list_empty(workload_q_head(vgpu, ring_id))) {
@@ -527,12 +526,13 @@ static void init_vgpu_execlist(struct intel_vgpu *vgpu, int ring_id)
 	vgpu_vreg(vgpu, ctx_status_ptr_reg) = ctx_status_ptr.dw;
 }
 
-static void clean_execlist(struct intel_vgpu *vgpu, unsigned long engine_mask)
+static void clean_execlist(struct intel_vgpu *vgpu,
+			   intel_engine_mask_t engine_mask)
 {
-	unsigned int tmp;
 	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
 	struct intel_engine_cs *engine;
 	struct intel_vgpu_submission *s = &vgpu->submission;
+	intel_engine_mask_t tmp;
 
 	for_each_engine_masked(engine, dev_priv, engine_mask, tmp) {
 		kfree(s->ring_scan_buffer[engine->id]);
@@ -542,18 +542,18 @@ static void clean_execlist(struct intel_vgpu *vgpu, unsigned long engine_mask)
 }
 
 static void reset_execlist(struct intel_vgpu *vgpu,
-		unsigned long engine_mask)
+			   intel_engine_mask_t engine_mask)
 {
 	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
 	struct intel_engine_cs *engine;
-	unsigned int tmp;
+	intel_engine_mask_t tmp;
 
 	for_each_engine_masked(engine, dev_priv, engine_mask, tmp)
 		init_vgpu_execlist(vgpu, engine->id);
 }
 
 static int init_execlist(struct intel_vgpu *vgpu,
-			 unsigned long engine_mask)
+			 intel_engine_mask_t engine_mask)
 {
 	reset_execlist(vgpu, engine_mask);
 	return 0;
diff --git a/drivers/gpu/drm/i915/gvt/execlist.h b/drivers/gpu/drm/i915/gvt/execlist.h
index 714d709829a2..5ccc2c695848 100644
--- a/drivers/gpu/drm/i915/gvt/execlist.h
+++ b/drivers/gpu/drm/i915/gvt/execlist.h
@@ -180,6 +180,6 @@ int intel_vgpu_init_execlist(struct intel_vgpu *vgpu);
 int intel_vgpu_submit_execlist(struct intel_vgpu *vgpu, int ring_id);
 
 void intel_vgpu_reset_execlist(struct intel_vgpu *vgpu,
-		unsigned long engine_mask);
+			       intel_engine_mask_t engine_mask);
 
 #endif /*_GVT_EXECLIST_H_*/
diff --git a/drivers/gpu/drm/i915/gvt/gtt.c b/drivers/gpu/drm/i915/gvt/gtt.c
index 9814773882ec..c2f7d20f6346 100644
--- a/drivers/gpu/drm/i915/gvt/gtt.c
+++ b/drivers/gpu/drm/i915/gvt/gtt.c
@@ -2504,6 +2504,7 @@ static void clean_spt_oos(struct intel_gvt *gvt)
 	list_for_each_safe(pos, n, &gtt->oos_page_free_list_head) {
 		oos_page = container_of(pos, struct intel_vgpu_oos_page, list);
 		list_del(&oos_page->list);
+		free_page((unsigned long)oos_page->mem);
 		kfree(oos_page);
 	}
 }
@@ -2524,6 +2525,12 @@ static int setup_spt_oos(struct intel_gvt *gvt)
 			ret = -ENOMEM;
 			goto fail;
 		}
+		oos_page->mem = (void *)__get_free_pages(GFP_KERNEL, 0);
+		if (!oos_page->mem) {
+			ret = -ENOMEM;
+			kfree(oos_page);
+			goto fail;
+		}
 
 		INIT_LIST_HEAD(&oos_page->list);
 		INIT_LIST_HEAD(&oos_page->vm_list);
diff --git a/drivers/gpu/drm/i915/gvt/gtt.h b/drivers/gpu/drm/i915/gvt/gtt.h
index edb610dc5d86..32c573aea494 100644
--- a/drivers/gpu/drm/i915/gvt/gtt.h
+++ b/drivers/gpu/drm/i915/gvt/gtt.h
@@ -222,7 +222,7 @@ struct intel_vgpu_oos_page {
 	struct list_head list;
 	struct list_head vm_list;
 	int id;
-	unsigned char mem[I915_GTT_PAGE_SIZE];
+	void *mem;
 };
 
 #define GTT_ENTRY_NUM_IN_ONE_PAGE 512
diff --git a/drivers/gpu/drm/i915/gvt/gvt.h b/drivers/gpu/drm/i915/gvt/gvt.h
index 8bce09de4b82..f5a328b5290a 100644
--- a/drivers/gpu/drm/i915/gvt/gvt.h
+++ b/drivers/gpu/drm/i915/gvt/gvt.h
@@ -94,7 +94,6 @@ struct intel_vgpu_fence {
 
 struct intel_vgpu_mmio {
 	void *vreg;
-	void *sreg;
 };
 
 #define INTEL_GVT_MAX_BAR_NUM 4
@@ -111,11 +110,9 @@ struct intel_vgpu_cfg_space {
 
 #define vgpu_cfg_space(vgpu) ((vgpu)->cfg_space.virtual_cfg_space)
 
-#define INTEL_GVT_MAX_PIPE 4
-
 struct intel_vgpu_irq {
 	bool irq_warn_once[INTEL_GVT_EVENT_MAX];
-	DECLARE_BITMAP(flip_done_event[INTEL_GVT_MAX_PIPE],
+	DECLARE_BITMAP(flip_done_event[I915_MAX_PIPES],
 		       INTEL_GVT_EVENT_MAX);
 };
 
@@ -144,9 +141,9 @@ enum {
 
 struct intel_vgpu_submission_ops {
 	const char *name;
-	int (*init)(struct intel_vgpu *vgpu, unsigned long engine_mask);
-	void (*clean)(struct intel_vgpu *vgpu, unsigned long engine_mask);
-	void (*reset)(struct intel_vgpu *vgpu, unsigned long engine_mask);
+	int (*init)(struct intel_vgpu *vgpu, intel_engine_mask_t engine_mask);
+	void (*clean)(struct intel_vgpu *vgpu, intel_engine_mask_t engine_mask);
+	void (*reset)(struct intel_vgpu *vgpu, intel_engine_mask_t engine_mask);
 };
 
 struct intel_vgpu_submission {
@@ -449,10 +446,6 @@ void intel_vgpu_write_fence(struct intel_vgpu *vgpu,
 	(*(u64 *)(vgpu->mmio.vreg + i915_mmio_reg_offset(reg)))
 #define vgpu_vreg64(vgpu, offset) \
 	(*(u64 *)(vgpu->mmio.vreg + (offset)))
-#define vgpu_sreg_t(vgpu, reg) \
-	(*(u32 *)(vgpu->mmio.sreg + i915_mmio_reg_offset(reg)))
-#define vgpu_sreg(vgpu, offset) \
-	(*(u32 *)(vgpu->mmio.sreg + (offset)))
 
 #define for_each_active_vgpu(gvt, vgpu, id) \
 	idr_for_each_entry((&(gvt)->vgpu_idr), (vgpu), (id)) \
@@ -488,7 +481,7 @@ struct intel_vgpu *intel_gvt_create_vgpu(struct intel_gvt *gvt,
 void intel_gvt_destroy_vgpu(struct intel_vgpu *vgpu);
 void intel_gvt_release_vgpu(struct intel_vgpu *vgpu);
 void intel_gvt_reset_vgpu_locked(struct intel_vgpu *vgpu, bool dmlr,
-				 unsigned int engine_mask);
+				 intel_engine_mask_t engine_mask);
 void intel_gvt_reset_vgpu(struct intel_vgpu *vgpu);
 void intel_gvt_activate_vgpu(struct intel_vgpu *vgpu);
 void intel_gvt_deactivate_vgpu(struct intel_vgpu *vgpu);
diff --git a/drivers/gpu/drm/i915/gvt/handlers.c b/drivers/gpu/drm/i915/gvt/handlers.c
index bc64b810e0d5..18f01eeb2510 100644
--- a/drivers/gpu/drm/i915/gvt/handlers.c
+++ b/drivers/gpu/drm/i915/gvt/handlers.c
@@ -311,7 +311,7 @@ static int mul_force_wake_write(struct intel_vgpu *vgpu,
 static int gdrst_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
 			    void *p_data, unsigned int bytes)
 {
-	unsigned int engine_mask = 0;
+	intel_engine_mask_t engine_mask = 0;
 	u32 data;
 
 	write_vreg(vgpu, offset, p_data, bytes);
@@ -323,25 +323,25 @@ static int gdrst_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
 	} else {
 		if (data & GEN6_GRDOM_RENDER) {
 			gvt_dbg_mmio("vgpu%d: request RCS reset\n", vgpu->id);
-			engine_mask |= (1 << RCS);
+			engine_mask |= BIT(RCS0);
 		}
 		if (data & GEN6_GRDOM_MEDIA) {
 			gvt_dbg_mmio("vgpu%d: request VCS reset\n", vgpu->id);
-			engine_mask |= (1 << VCS);
+			engine_mask |= BIT(VCS0);
 		}
 		if (data & GEN6_GRDOM_BLT) {
 			gvt_dbg_mmio("vgpu%d: request BCS Reset\n", vgpu->id);
-			engine_mask |= (1 << BCS);
+			engine_mask |= BIT(BCS0);
 		}
 		if (data & GEN6_GRDOM_VECS) {
 			gvt_dbg_mmio("vgpu%d: request VECS Reset\n", vgpu->id);
-			engine_mask |= (1 << VECS);
+			engine_mask |= BIT(VECS0);
 		}
 		if (data & GEN8_GRDOM_MEDIA2) {
 			gvt_dbg_mmio("vgpu%d: request VCS2 Reset\n", vgpu->id);
-			if (HAS_BSD2(vgpu->gvt->dev_priv))
-				engine_mask |= (1 << VCS2);
+			engine_mask |= BIT(VCS1);
 		}
+		engine_mask &= INTEL_INFO(vgpu->gvt->dev_priv)->engine_mask;
 	}
 
 	/* vgpu_lock already hold by emulate mmio r/w */
@@ -750,18 +750,19 @@ static int pri_surf_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
 		void *p_data, unsigned int bytes)
 {
 	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
-	unsigned int index = DSPSURF_TO_PIPE(offset);
-	i915_reg_t surflive_reg = DSPSURFLIVE(index);
-	int flip_event[] = {
-		[PIPE_A] = PRIMARY_A_FLIP_DONE,
-		[PIPE_B] = PRIMARY_B_FLIP_DONE,
-		[PIPE_C] = PRIMARY_C_FLIP_DONE,
-	};
+	u32 pipe = DSPSURF_TO_PIPE(offset);
+	int event = SKL_FLIP_EVENT(pipe, PLANE_PRIMARY);
 
 	write_vreg(vgpu, offset, p_data, bytes);
-	vgpu_vreg_t(vgpu, surflive_reg) = vgpu_vreg(vgpu, offset);
+	vgpu_vreg_t(vgpu, DSPSURFLIVE(pipe)) = vgpu_vreg(vgpu, offset);
+
+	vgpu_vreg_t(vgpu, PIPE_FLIPCOUNT_G4X(pipe))++;
+
+	if (vgpu_vreg_t(vgpu, DSPCNTR(pipe)) & PLANE_CTL_ASYNC_FLIP)
+		intel_vgpu_trigger_virtual_event(vgpu, event);
+	else
+		set_bit(event, vgpu->irq.flip_done_event[pipe]);
 
-	set_bit(flip_event[index], vgpu->irq.flip_done_event[index]);
 	return 0;
 }
 
@@ -771,18 +772,42 @@ static int pri_surf_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
 static int spr_surf_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
 		void *p_data, unsigned int bytes)
 {
-	unsigned int index = SPRSURF_TO_PIPE(offset);
-	i915_reg_t surflive_reg = SPRSURFLIVE(index);
-	int flip_event[] = {
-		[PIPE_A] = SPRITE_A_FLIP_DONE,
-		[PIPE_B] = SPRITE_B_FLIP_DONE,
-		[PIPE_C] = SPRITE_C_FLIP_DONE,
-	};
+	u32 pipe = SPRSURF_TO_PIPE(offset);
+	int event = SKL_FLIP_EVENT(pipe, PLANE_SPRITE0);
+
+	write_vreg(vgpu, offset, p_data, bytes);
+	vgpu_vreg_t(vgpu, SPRSURFLIVE(pipe)) = vgpu_vreg(vgpu, offset);
+
+	if (vgpu_vreg_t(vgpu, SPRCTL(pipe)) & PLANE_CTL_ASYNC_FLIP)
+		intel_vgpu_trigger_virtual_event(vgpu, event);
+	else
+		set_bit(event, vgpu->irq.flip_done_event[pipe]);
+
+	return 0;
+}
+
+static int reg50080_mmio_write(struct intel_vgpu *vgpu,
+			       unsigned int offset, void *p_data,
+			       unsigned int bytes)
+{
+	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+	enum pipe pipe = REG_50080_TO_PIPE(offset);
+	enum plane_id plane = REG_50080_TO_PLANE(offset);
+	int event = SKL_FLIP_EVENT(pipe, plane);
 
 	write_vreg(vgpu, offset, p_data, bytes);
-	vgpu_vreg_t(vgpu, surflive_reg) = vgpu_vreg(vgpu, offset);
+	if (plane == PLANE_PRIMARY) {
+		vgpu_vreg_t(vgpu, DSPSURFLIVE(pipe)) = vgpu_vreg(vgpu, offset);
+		vgpu_vreg_t(vgpu, PIPE_FLIPCOUNT_G4X(pipe))++;
+	} else {
+		vgpu_vreg_t(vgpu, SPRSURFLIVE(pipe)) = vgpu_vreg(vgpu, offset);
+	}
+
+	if ((vgpu_vreg(vgpu, offset) & REG50080_FLIP_TYPE_MASK) == REG50080_FLIP_TYPE_ASYNC)
+		intel_vgpu_trigger_virtual_event(vgpu, event);
+	else
+		set_bit(event, vgpu->irq.flip_done_event[pipe]);
 
-	set_bit(flip_event[index], vgpu->irq.flip_done_event[index]);
 	return 0;
 }
 
@@ -1704,7 +1729,7 @@ static int ring_mode_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
 			return 0;
 
 		ret = intel_vgpu_select_submission_ops(vgpu,
-			       ENGINE_MASK(ring_id),
+			       BIT(ring_id),
 			       INTEL_VGPU_EXECLIST_SUBMISSION);
 		if (ret)
 			return ret;
@@ -1724,19 +1749,19 @@ static int gvt_reg_tlb_control_handler(struct intel_vgpu *vgpu,
 
 	switch (offset) {
 	case 0x4260:
-		id = RCS;
+		id = RCS0;
 		break;
 	case 0x4264:
-		id = VCS;
+		id = VCS0;
 		break;
 	case 0x4268:
-		id = VCS2;
+		id = VCS1;
 		break;
 	case 0x426c:
-		id = BCS;
+		id = BCS0;
 		break;
 	case 0x4270:
-		id = VECS;
+		id = VECS0;
 		break;
 	default:
 		return -EINVAL;
@@ -1793,7 +1818,7 @@ static int ring_reset_ctl_write(struct intel_vgpu *vgpu,
 	MMIO_F(prefix(BLT_RING_BASE), s, f, am, rm, d, r, w); \
 	MMIO_F(prefix(GEN6_BSD_RING_BASE), s, f, am, rm, d, r, w); \
 	MMIO_F(prefix(VEBOX_RING_BASE), s, f, am, rm, d, r, w); \
-	if (HAS_BSD2(dev_priv)) \
+	if (HAS_ENGINE(dev_priv, VCS1)) \
 		MMIO_F(prefix(GEN8_BSD2_RING_BASE), s, f, am, rm, d, r, w); \
 } while (0)
 
@@ -1848,7 +1873,7 @@ static int init_generic_mmio_info(struct intel_gvt *gvt)
 	MMIO_DH(GEN7_SC_INSTDONE, D_BDW_PLUS, mmio_read_from_hw, NULL);
 
 	MMIO_GM_RDR(_MMIO(0x2148), D_ALL, NULL, NULL);
-	MMIO_GM_RDR(CCID, D_ALL, NULL, NULL);
+	MMIO_GM_RDR(CCID(RENDER_RING_BASE), D_ALL, NULL, NULL);
 	MMIO_GM_RDR(_MMIO(0x12198), D_ALL, NULL, NULL);
 	MMIO_D(GEN7_CXT_SIZE, D_ALL);
 
@@ -1969,6 +1994,8 @@ static int init_generic_mmio_info(struct intel_gvt *gvt)
 	MMIO_DH(DSPSURF(PIPE_A), D_ALL, NULL, pri_surf_mmio_write);
 	MMIO_D(DSPOFFSET(PIPE_A), D_ALL);
 	MMIO_D(DSPSURFLIVE(PIPE_A), D_ALL);
+	MMIO_DH(REG_50080(PIPE_A, PLANE_PRIMARY), D_ALL, NULL,
+		reg50080_mmio_write);
 
 	MMIO_D(DSPCNTR(PIPE_B), D_ALL);
 	MMIO_D(DSPADDR(PIPE_B), D_ALL);
@@ -1978,6 +2005,8 @@ static int init_generic_mmio_info(struct intel_gvt *gvt)
 	MMIO_DH(DSPSURF(PIPE_B), D_ALL, NULL, pri_surf_mmio_write);
 	MMIO_D(DSPOFFSET(PIPE_B), D_ALL);
 	MMIO_D(DSPSURFLIVE(PIPE_B), D_ALL);
+	MMIO_DH(REG_50080(PIPE_B, PLANE_PRIMARY), D_ALL, NULL,
+		reg50080_mmio_write);
 
 	MMIO_D(DSPCNTR(PIPE_C), D_ALL);
 	MMIO_D(DSPADDR(PIPE_C), D_ALL);
@@ -1987,6 +2016,8 @@ static int init_generic_mmio_info(struct intel_gvt *gvt)
 	MMIO_DH(DSPSURF(PIPE_C), D_ALL, NULL, pri_surf_mmio_write);
 	MMIO_D(DSPOFFSET(PIPE_C), D_ALL);
 	MMIO_D(DSPSURFLIVE(PIPE_C), D_ALL);
+	MMIO_DH(REG_50080(PIPE_C, PLANE_PRIMARY), D_ALL, NULL,
+		reg50080_mmio_write);
 
 	MMIO_D(SPRCTL(PIPE_A), D_ALL);
 	MMIO_D(SPRLINOFF(PIPE_A), D_ALL);
@@ -2000,6 +2031,8 @@ static int init_generic_mmio_info(struct intel_gvt *gvt)
 	MMIO_D(SPROFFSET(PIPE_A), D_ALL);
 	MMIO_D(SPRSCALE(PIPE_A), D_ALL);
 	MMIO_D(SPRSURFLIVE(PIPE_A), D_ALL);
+	MMIO_DH(REG_50080(PIPE_A, PLANE_SPRITE0), D_ALL, NULL,
+		reg50080_mmio_write);
 
 	MMIO_D(SPRCTL(PIPE_B), D_ALL);
 	MMIO_D(SPRLINOFF(PIPE_B), D_ALL);
@@ -2013,6 +2046,8 @@ static int init_generic_mmio_info(struct intel_gvt *gvt)
 	MMIO_D(SPROFFSET(PIPE_B), D_ALL);
 	MMIO_D(SPRSCALE(PIPE_B), D_ALL);
 	MMIO_D(SPRSURFLIVE(PIPE_B), D_ALL);
+	MMIO_DH(REG_50080(PIPE_B, PLANE_SPRITE0), D_ALL, NULL,
+		reg50080_mmio_write);
 
 	MMIO_D(SPRCTL(PIPE_C), D_ALL);
 	MMIO_D(SPRLINOFF(PIPE_C), D_ALL);
@@ -2026,6 +2061,8 @@ static int init_generic_mmio_info(struct intel_gvt *gvt)
 	MMIO_D(SPROFFSET(PIPE_C), D_ALL);
 	MMIO_D(SPRSCALE(PIPE_C), D_ALL);
 	MMIO_D(SPRSURFLIVE(PIPE_C), D_ALL);
+	MMIO_DH(REG_50080(PIPE_C, PLANE_SPRITE0), D_ALL, NULL,
+		reg50080_mmio_write);
 
 	MMIO_D(HTOTAL(TRANSCODER_A), D_ALL);
 	MMIO_D(HBLANK(TRANSCODER_A), D_ALL);
@@ -2827,26 +2864,26 @@ static int init_skl_mmio_info(struct intel_gvt *gvt)
 
 	MMIO_DH(DBUF_CTL, D_SKL_PLUS, NULL, gen9_dbuf_ctl_mmio_write);
 
-	MMIO_D(_MMIO(0xa210), D_SKL_PLUS);
+	MMIO_D(GEN9_PG_ENABLE, D_SKL_PLUS);
 	MMIO_D(GEN9_MEDIA_PG_IDLE_HYSTERESIS, D_SKL_PLUS);
 	MMIO_D(GEN9_RENDER_PG_IDLE_HYSTERESIS, D_SKL_PLUS);
 	MMIO_DFH(GEN9_GAMT_ECO_REG_RW_IA, D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DH(_MMIO(0x4ddc), D_SKL_PLUS, NULL, NULL);
-	MMIO_DH(_MMIO(0x42080), D_SKL_PLUS, NULL, NULL);
-	MMIO_D(_MMIO(0x45504), D_SKL_PLUS);
-	MMIO_D(_MMIO(0x45520), D_SKL_PLUS);
-	MMIO_D(_MMIO(0x46000), D_SKL_PLUS);
-	MMIO_DH(_MMIO(0x46010), D_SKL_PLUS, NULL, skl_lcpll_write);
-	MMIO_DH(_MMIO(0x46014), D_SKL_PLUS, NULL, skl_lcpll_write);
-	MMIO_D(_MMIO(0x6C040), D_SKL_PLUS);
-	MMIO_D(_MMIO(0x6C048), D_SKL_PLUS);
-	MMIO_D(_MMIO(0x6C050), D_SKL_PLUS);
-	MMIO_D(_MMIO(0x6C044), D_SKL_PLUS);
-	MMIO_D(_MMIO(0x6C04C), D_SKL_PLUS);
-	MMIO_D(_MMIO(0x6C054), D_SKL_PLUS);
-	MMIO_D(_MMIO(0x6c058), D_SKL_PLUS);
-	MMIO_D(_MMIO(0x6c05c), D_SKL_PLUS);
-	MMIO_DH(_MMIO(0x6c060), D_SKL_PLUS, dpll_status_read, NULL);
+	MMIO_DH(MMCD_MISC_CTRL, D_SKL_PLUS, NULL, NULL);
+	MMIO_DH(CHICKEN_PAR1_1, D_SKL_PLUS, NULL, NULL);
+	MMIO_D(DC_STATE_EN, D_SKL_PLUS);
+	MMIO_D(DC_STATE_DEBUG, D_SKL_PLUS);
+	MMIO_D(CDCLK_CTL, D_SKL_PLUS);
+	MMIO_DH(LCPLL1_CTL, D_SKL_PLUS, NULL, skl_lcpll_write);
+	MMIO_DH(LCPLL2_CTL, D_SKL_PLUS, NULL, skl_lcpll_write);
+	MMIO_D(_MMIO(_DPLL1_CFGCR1), D_SKL_PLUS);
+	MMIO_D(_MMIO(_DPLL2_CFGCR1), D_SKL_PLUS);
+	MMIO_D(_MMIO(_DPLL3_CFGCR1), D_SKL_PLUS);
+	MMIO_D(_MMIO(_DPLL1_CFGCR2), D_SKL_PLUS);
+	MMIO_D(_MMIO(_DPLL2_CFGCR2), D_SKL_PLUS);
+	MMIO_D(_MMIO(_DPLL3_CFGCR2), D_SKL_PLUS);
+	MMIO_D(DPLL_CTRL1, D_SKL_PLUS);
+	MMIO_D(DPLL_CTRL2, D_SKL_PLUS);
+	MMIO_DH(DPLL_STATUS, D_SKL_PLUS, dpll_status_read, NULL);
 
 	MMIO_DH(SKL_PS_WIN_POS(PIPE_A, 0), D_SKL_PLUS, NULL, pf_write);
 	MMIO_DH(SKL_PS_WIN_POS(PIPE_A, 1), D_SKL_PLUS, NULL, pf_write);
@@ -2965,40 +3002,41 @@ static int init_skl_mmio_info(struct intel_gvt *gvt)
 	MMIO_DH(_MMIO(_REG_701C4(PIPE_C, 3)), D_SKL_PLUS, NULL, NULL);
 	MMIO_DH(_MMIO(_REG_701C4(PIPE_C, 4)), D_SKL_PLUS, NULL, NULL);
 
-	MMIO_D(_MMIO(0x70380), D_SKL_PLUS);
-	MMIO_D(_MMIO(0x71380), D_SKL_PLUS);
+	MMIO_D(_MMIO(_PLANE_CTL_3_A), D_SKL_PLUS);
+	MMIO_D(_MMIO(_PLANE_CTL_3_B), D_SKL_PLUS);
 	MMIO_D(_MMIO(0x72380), D_SKL_PLUS);
 	MMIO_D(_MMIO(0x7239c), D_SKL_PLUS);
-	MMIO_D(_MMIO(0x7039c), D_SKL_PLUS);
+	MMIO_D(_MMIO(_PLANE_SURF_3_A), D_SKL_PLUS);
 
-	MMIO_D(_MMIO(0x8f074), D_SKL_PLUS);
-	MMIO_D(_MMIO(0x8f004), D_SKL_PLUS);
-	MMIO_D(_MMIO(0x8f034), D_SKL_PLUS);
+	MMIO_D(CSR_SSP_BASE, D_SKL_PLUS);
+	MMIO_D(CSR_HTP_SKL, D_SKL_PLUS);
+	MMIO_D(CSR_LAST_WRITE, D_SKL_PLUS);
 
-	MMIO_D(_MMIO(0xb11c), D_SKL_PLUS);
+	MMIO_D(BDW_SCRATCH1, D_SKL_PLUS);
 
-	MMIO_D(_MMIO(0x51000), D_SKL_PLUS);
-	MMIO_D(_MMIO(0x6c00c), D_SKL_PLUS);
+	MMIO_D(SKL_DFSM, D_SKL_PLUS);
+	MMIO_D(DISPIO_CR_TX_BMU_CR0, D_SKL_PLUS);
 
-	MMIO_F(_MMIO(0xc800), 0x7f8, F_CMD_ACCESS, 0, 0, D_SKL_PLUS,
+	MMIO_F(GEN9_GFX_MOCS(0), 0x7f8, F_CMD_ACCESS, 0, 0, D_SKL_PLUS,
 		NULL, NULL);
-	MMIO_F(_MMIO(0xb020), 0x80, F_CMD_ACCESS, 0, 0, D_SKL_PLUS,
+	MMIO_F(GEN7_L3CNTLREG2, 0x80, F_CMD_ACCESS, 0, 0, D_SKL_PLUS,
 		NULL, NULL);
 
 	MMIO_D(RPM_CONFIG0, D_SKL_PLUS);
 	MMIO_D(_MMIO(0xd08), D_SKL_PLUS);
 	MMIO_D(RC6_LOCATION, D_SKL_PLUS);
-	MMIO_DFH(_MMIO(0x20e0), D_SKL_PLUS, F_MODE_MASK, NULL, NULL);
-	MMIO_DFH(_MMIO(0x20ec), D_SKL_PLUS, F_MODE_MASK | F_CMD_ACCESS,
+	MMIO_DFH(GEN7_FF_SLICE_CS_CHICKEN1, D_SKL_PLUS, F_MODE_MASK,
+		NULL, NULL);
+	MMIO_DFH(GEN9_CS_DEBUG_MODE1, D_SKL_PLUS, F_MODE_MASK | F_CMD_ACCESS,
 		NULL, NULL);
 
 	/* TRTT */
-	MMIO_DFH(_MMIO(0x4de0), D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(_MMIO(0x4de4), D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(_MMIO(0x4de8), D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(_MMIO(0x4dec), D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(_MMIO(0x4df0), D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL);
-	MMIO_DFH(_MMIO(0x4df4), D_SKL_PLUS, F_CMD_ACCESS,
+	MMIO_DFH(TRVATTL3PTRDW(0), D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(TRVATTL3PTRDW(1), D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(TRVATTL3PTRDW(2), D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(TRVATTL3PTRDW(3), D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(TRVADR, D_SKL_PLUS, F_CMD_ACCESS, NULL, NULL);
+	MMIO_DFH(TRTTE, D_SKL_PLUS, F_CMD_ACCESS,
 		NULL, gen9_trtte_write);
 	MMIO_DH(_MMIO(0x4dfc), D_SKL_PLUS, NULL, gen9_trtt_chicken_write);
 
@@ -3011,7 +3049,7 @@ static int init_skl_mmio_info(struct intel_gvt *gvt)
 	MMIO_DH(DMA_CTRL, D_SKL_PLUS, NULL, dma_ctrl_write);
 
 	MMIO_D(_MMIO(0x65900), D_SKL_PLUS);
-	MMIO_D(_MMIO(0x1082c0), D_SKL_PLUS);
+	MMIO_D(GEN6_STOLEN_RESERVED, D_SKL_PLUS);
 	MMIO_D(_MMIO(0x4068), D_SKL_PLUS);
 	MMIO_D(_MMIO(0x67054), D_SKL_PLUS);
 	MMIO_D(_MMIO(0x6e560), D_SKL_PLUS);
@@ -3042,8 +3080,8 @@ static int init_skl_mmio_info(struct intel_gvt *gvt)
 	MMIO_DFH(GEN9_WM_CHICKEN3, D_SKL_PLUS, F_MODE_MASK | F_CMD_ACCESS,
 		 NULL, NULL);
 
-	MMIO_D(_MMIO(0x4ab8), D_KBL | D_CFL);
-	MMIO_D(_MMIO(0x2248), D_SKL_PLUS);
+	MMIO_D(GAMT_CHKN_BIT_REG, D_KBL);
+	MMIO_D(GEN9_CTX_PREEMPT_REG, D_KBL | D_SKL);
 
 	return 0;
 }
@@ -3265,7 +3303,7 @@ void intel_gvt_clean_mmio_info(struct intel_gvt *gvt)
 /* Special MMIO blocks. */
 static struct gvt_mmio_block mmio_blocks[] = {
 	{D_SKL_PLUS, _MMIO(CSR_MMIO_START_RANGE), 0x3000, NULL, NULL},
-	{D_ALL, _MMIO(MCHBAR_MIRROR_BASE_SNB), 0x40000, NULL, NULL},
+	{D_ALL, MCHBAR_MIRROR_REG_BASE, 0x4000, NULL, NULL},
 	{D_ALL, _MMIO(VGT_PVINFO_PAGE), VGT_PVINFO_SIZE,
 		pvinfo_mmio_read, pvinfo_mmio_write},
 	{D_ALL, LGC_PALETTE(PIPE_A, 0), 1024, NULL, NULL},
@@ -3489,12 +3527,11 @@ int intel_vgpu_mmio_reg_rw(struct intel_vgpu *vgpu, unsigned int offset,
 		return mmio_info->read(vgpu, offset, pdata, bytes);
 	else {
 		u64 ro_mask = mmio_info->ro_mask;
-		u32 old_vreg = 0, old_sreg = 0;
+		u32 old_vreg = 0;
 		u64 data = 0;
 
 		if (intel_gvt_mmio_has_mode_mask(gvt, mmio_info->offset)) {
 			old_vreg = vgpu_vreg(vgpu, offset);
-			old_sreg = vgpu_sreg(vgpu, offset);
 		}
 
 		if (likely(!ro_mask))
@@ -3516,8 +3553,6 @@ int intel_vgpu_mmio_reg_rw(struct intel_vgpu *vgpu, unsigned int offset,
 
 			vgpu_vreg(vgpu, offset) = (old_vreg & ~mask)
 					| (vgpu_vreg(vgpu, offset) & mask);
-			vgpu_sreg(vgpu, offset) = (old_sreg & ~mask)
-					| (vgpu_sreg(vgpu, offset) & mask);
 		}
 	}
 
diff --git a/drivers/gpu/drm/i915/gvt/interrupt.c b/drivers/gpu/drm/i915/gvt/interrupt.c
index 67125c5eec6e..951681813230 100644
--- a/drivers/gpu/drm/i915/gvt/interrupt.c
+++ b/drivers/gpu/drm/i915/gvt/interrupt.c
@@ -536,7 +536,7 @@ static void gen8_init_irq(
 	SET_BIT_INFO(irq, 4, VCS_MI_FLUSH_DW, INTEL_GVT_IRQ_INFO_GT1);
 	SET_BIT_INFO(irq, 8, VCS_AS_CONTEXT_SWITCH, INTEL_GVT_IRQ_INFO_GT1);
 
-	if (HAS_BSD2(gvt->dev_priv)) {
+	if (HAS_ENGINE(gvt->dev_priv, VCS1)) {
 		SET_BIT_INFO(irq, 16, VCS2_MI_USER_INTERRUPT,
 			INTEL_GVT_IRQ_INFO_GT1);
 		SET_BIT_INFO(irq, 20, VCS2_MI_FLUSH_DW,
diff --git a/drivers/gpu/drm/i915/gvt/mmio.c b/drivers/gpu/drm/i915/gvt/mmio.c
index ed4df2f6d60b..a55178884d67 100644
--- a/drivers/gpu/drm/i915/gvt/mmio.c
+++ b/drivers/gpu/drm/i915/gvt/mmio.c
@@ -239,7 +239,6 @@ void intel_vgpu_reset_mmio(struct intel_vgpu *vgpu, bool dmlr)
 
 	if (dmlr) {
 		memcpy(vgpu->mmio.vreg, mmio, info->mmio_size);
-		memcpy(vgpu->mmio.sreg, mmio, info->mmio_size);
 
 		vgpu_vreg_t(vgpu, GEN6_GT_THREAD_STATUS_REG) = 0;
 
@@ -280,7 +279,6 @@ void intel_vgpu_reset_mmio(struct intel_vgpu *vgpu, bool dmlr)
 		 * touched
 		 */
 		memcpy(vgpu->mmio.vreg, mmio, GVT_GEN8_MMIO_RESET_OFFSET);
-		memcpy(vgpu->mmio.sreg, mmio, GVT_GEN8_MMIO_RESET_OFFSET);
 	}
 
 }
@@ -296,12 +294,10 @@ int intel_vgpu_init_mmio(struct intel_vgpu *vgpu)
 {
 	const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;
 
-	vgpu->mmio.vreg = vzalloc(array_size(info->mmio_size, 2));
+	vgpu->mmio.vreg = vzalloc(info->mmio_size);
 	if (!vgpu->mmio.vreg)
 		return -ENOMEM;
 
-	vgpu->mmio.sreg = vgpu->mmio.vreg + info->mmio_size;
-
 	intel_vgpu_reset_mmio(vgpu, true);
 
 	return 0;
@@ -315,5 +311,5 @@ int intel_vgpu_init_mmio(struct intel_vgpu *vgpu)
 void intel_vgpu_clean_mmio(struct intel_vgpu *vgpu)
 {
 	vfree(vgpu->mmio.vreg);
-	vgpu->mmio.vreg = vgpu->mmio.sreg = NULL;
+	vgpu->mmio.vreg = NULL;
 }
diff --git a/drivers/gpu/drm/i915/gvt/mmio_context.c b/drivers/gpu/drm/i915/gvt/mmio_context.c
index 7902fb162d09..e7e14c842be4 100644
--- a/drivers/gpu/drm/i915/gvt/mmio_context.c
+++ b/drivers/gpu/drm/i915/gvt/mmio_context.c
@@ -41,103 +41,102 @@
 
 /* Raw offset is appened to each line for convenience. */
 static struct engine_mmio gen8_engine_mmio_list[] __cacheline_aligned = {
-	{RCS, GFX_MODE_GEN7, 0xffff, false}, /* 0x229c */
-	{RCS, GEN9_CTX_PREEMPT_REG, 0x0, false}, /* 0x2248 */
-	{RCS, HWSTAM, 0x0, false}, /* 0x2098 */
-	{RCS, INSTPM, 0xffff, true}, /* 0x20c0 */
-	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 0), 0, false}, /* 0x24d0 */
-	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 1), 0, false}, /* 0x24d4 */
-	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 2), 0, false}, /* 0x24d8 */
-	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 3), 0, false}, /* 0x24dc */
-	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 4), 0, false}, /* 0x24e0 */
-	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 5), 0, false}, /* 0x24e4 */
-	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 6), 0, false}, /* 0x24e8 */
-	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 7), 0, false}, /* 0x24ec */
-	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 8), 0, false}, /* 0x24f0 */
-	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 9), 0, false}, /* 0x24f4 */
-	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 10), 0, false}, /* 0x24f8 */
-	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 11), 0, false}, /* 0x24fc */
-	{RCS, CACHE_MODE_1, 0xffff, true}, /* 0x7004 */
-	{RCS, GEN7_GT_MODE, 0xffff, true}, /* 0x7008 */
-	{RCS, CACHE_MODE_0_GEN7, 0xffff, true}, /* 0x7000 */
-	{RCS, GEN7_COMMON_SLICE_CHICKEN1, 0xffff, true}, /* 0x7010 */
-	{RCS, HDC_CHICKEN0, 0xffff, true}, /* 0x7300 */
-	{RCS, VF_GUARDBAND, 0xffff, true}, /* 0x83a4 */
-
-	{BCS, RING_GFX_MODE(BLT_RING_BASE), 0xffff, false}, /* 0x2229c */
-	{BCS, RING_MI_MODE(BLT_RING_BASE), 0xffff, false}, /* 0x2209c */
-	{BCS, RING_INSTPM(BLT_RING_BASE), 0xffff, false}, /* 0x220c0 */
-	{BCS, RING_HWSTAM(BLT_RING_BASE), 0x0, false}, /* 0x22098 */
-	{BCS, RING_EXCC(BLT_RING_BASE), 0x0, false}, /* 0x22028 */
-	{RCS, INVALID_MMIO_REG, 0, false } /* Terminated */
+	{RCS0, GFX_MODE_GEN7, 0xffff, false}, /* 0x229c */
+	{RCS0, GEN9_CTX_PREEMPT_REG, 0x0, false}, /* 0x2248 */
+	{RCS0, HWSTAM, 0x0, false}, /* 0x2098 */
+	{RCS0, INSTPM, 0xffff, true}, /* 0x20c0 */
+	{RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 0), 0, false}, /* 0x24d0 */
+	{RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 1), 0, false}, /* 0x24d4 */
+	{RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 2), 0, false}, /* 0x24d8 */
+	{RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 3), 0, false}, /* 0x24dc */
+	{RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 4), 0, false}, /* 0x24e0 */
+	{RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 5), 0, false}, /* 0x24e4 */
+	{RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 6), 0, false}, /* 0x24e8 */
+	{RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 7), 0, false}, /* 0x24ec */
+	{RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 8), 0, false}, /* 0x24f0 */
+	{RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 9), 0, false}, /* 0x24f4 */
+	{RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 10), 0, false}, /* 0x24f8 */
+	{RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 11), 0, false}, /* 0x24fc */
+	{RCS0, CACHE_MODE_1, 0xffff, true}, /* 0x7004 */
+	{RCS0, GEN7_GT_MODE, 0xffff, true}, /* 0x7008 */
+	{RCS0, CACHE_MODE_0_GEN7, 0xffff, true}, /* 0x7000 */
+	{RCS0, GEN7_COMMON_SLICE_CHICKEN1, 0xffff, true}, /* 0x7010 */
+	{RCS0, HDC_CHICKEN0, 0xffff, true}, /* 0x7300 */
+	{RCS0, VF_GUARDBAND, 0xffff, true}, /* 0x83a4 */
+
+	{BCS0, RING_GFX_MODE(BLT_RING_BASE), 0xffff, false}, /* 0x2229c */
+	{BCS0, RING_MI_MODE(BLT_RING_BASE), 0xffff, false}, /* 0x2209c */
+	{BCS0, RING_INSTPM(BLT_RING_BASE), 0xffff, false}, /* 0x220c0 */
+	{BCS0, RING_HWSTAM(BLT_RING_BASE), 0x0, false}, /* 0x22098 */
+	{BCS0, RING_EXCC(BLT_RING_BASE), 0xffff, false}, /* 0x22028 */
+	{RCS0, INVALID_MMIO_REG, 0, false } /* Terminated */
 };
 
 static struct engine_mmio gen9_engine_mmio_list[] __cacheline_aligned = {
-	{RCS, GFX_MODE_GEN7, 0xffff, false}, /* 0x229c */
-	{RCS, GEN9_CTX_PREEMPT_REG, 0x0, false}, /* 0x2248 */
-	{RCS, HWSTAM, 0x0, false}, /* 0x2098 */
-	{RCS, INSTPM, 0xffff, true}, /* 0x20c0 */
-	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 0), 0, false}, /* 0x24d0 */
-	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 1), 0, false}, /* 0x24d4 */
-	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 2), 0, false}, /* 0x24d8 */
-	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 3), 0, false}, /* 0x24dc */
-	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 4), 0, false}, /* 0x24e0 */
-	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 5), 0, false}, /* 0x24e4 */
-	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 6), 0, false}, /* 0x24e8 */
-	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 7), 0, false}, /* 0x24ec */
-	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 8), 0, false}, /* 0x24f0 */
-	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 9), 0, false}, /* 0x24f4 */
-	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 10), 0, false}, /* 0x24f8 */
-	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 11), 0, false}, /* 0x24fc */
-	{RCS, CACHE_MODE_1, 0xffff, true}, /* 0x7004 */
-	{RCS, GEN7_GT_MODE, 0xffff, true}, /* 0x7008 */
-	{RCS, CACHE_MODE_0_GEN7, 0xffff, true}, /* 0x7000 */
-	{RCS, GEN7_COMMON_SLICE_CHICKEN1, 0xffff, true}, /* 0x7010 */
-	{RCS, HDC_CHICKEN0, 0xffff, true}, /* 0x7300 */
-	{RCS, VF_GUARDBAND, 0xffff, true}, /* 0x83a4 */
-
-	{RCS, GEN8_PRIVATE_PAT_LO, 0, false}, /* 0x40e0 */
-	{RCS, GEN8_PRIVATE_PAT_HI, 0, false}, /* 0x40e4 */
-	{RCS, GEN8_CS_CHICKEN1, 0xffff, true}, /* 0x2580 */
-	{RCS, COMMON_SLICE_CHICKEN2, 0xffff, true}, /* 0x7014 */
-	{RCS, GEN9_CS_DEBUG_MODE1, 0xffff, false}, /* 0x20ec */
-	{RCS, GEN8_L3SQCREG4, 0, false}, /* 0xb118 */
-	{RCS, GEN7_HALF_SLICE_CHICKEN1, 0xffff, true}, /* 0xe100 */
-	{RCS, HALF_SLICE_CHICKEN2, 0xffff, true}, /* 0xe180 */
-	{RCS, HALF_SLICE_CHICKEN3, 0xffff, true}, /* 0xe184 */
-	{RCS, GEN9_HALF_SLICE_CHICKEN5, 0xffff, true}, /* 0xe188 */
-	{RCS, GEN9_HALF_SLICE_CHICKEN7, 0xffff, true}, /* 0xe194 */
-	{RCS, GEN8_ROW_CHICKEN, 0xffff, true}, /* 0xe4f0 */
-	{RCS, TRVATTL3PTRDW(0), 0, false}, /* 0x4de0 */
-	{RCS, TRVATTL3PTRDW(1), 0, false}, /* 0x4de4 */
-	{RCS, TRNULLDETCT, 0, false}, /* 0x4de8 */
-	{RCS, TRINVTILEDETCT, 0, false}, /* 0x4dec */
-	{RCS, TRVADR, 0, false}, /* 0x4df0 */
-	{RCS, TRTTE, 0, false}, /* 0x4df4 */
-
-	{BCS, RING_GFX_MODE(BLT_RING_BASE), 0xffff, false}, /* 0x2229c */
-	{BCS, RING_MI_MODE(BLT_RING_BASE), 0xffff, false}, /* 0x2209c */
-	{BCS, RING_INSTPM(BLT_RING_BASE), 0xffff, false}, /* 0x220c0 */
-	{BCS, RING_HWSTAM(BLT_RING_BASE), 0x0, false}, /* 0x22098 */
-	{BCS, RING_EXCC(BLT_RING_BASE), 0x0, false}, /* 0x22028 */
-
-	{VCS2, RING_EXCC(GEN8_BSD2_RING_BASE), 0xffff, false}, /* 0x1c028 */
-
-	{VECS, RING_EXCC(VEBOX_RING_BASE), 0xffff, false}, /* 0x1a028 */
-
-	{RCS, GEN8_HDC_CHICKEN1, 0xffff, true}, /* 0x7304 */
-	{RCS, GEN9_CTX_PREEMPT_REG, 0x0, false}, /* 0x2248 */
-	{RCS, GEN7_UCGCTL4, 0x0, false}, /* 0x940c */
-	{RCS, GAMT_CHKN_BIT_REG, 0x0, false}, /* 0x4ab8 */
-
-	{RCS, GEN9_GAMT_ECO_REG_RW_IA, 0x0, false}, /* 0x4ab0 */
-	{RCS, GEN9_CSFE_CHICKEN1_RCS, 0xffff, false}, /* 0x20d4 */
-	{RCS, _MMIO(0x20D8), 0xffff, true}, /* 0x20d8 */
-
-	{RCS, GEN8_GARBCNTL, 0x0, false}, /* 0xb004 */
-	{RCS, GEN7_FF_THREAD_MODE, 0x0, false}, /* 0x20a0 */
-	{RCS, FF_SLICE_CS_CHICKEN2, 0xffff, false}, /* 0x20e4 */
-	{RCS, INVALID_MMIO_REG, 0, false } /* Terminated */
+	{RCS0, GFX_MODE_GEN7, 0xffff, false}, /* 0x229c */
+	{RCS0, GEN9_CTX_PREEMPT_REG, 0x0, false}, /* 0x2248 */
+	{RCS0, HWSTAM, 0x0, false}, /* 0x2098 */
+	{RCS0, INSTPM, 0xffff, true}, /* 0x20c0 */
+	{RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 0), 0, false}, /* 0x24d0 */
+	{RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 1), 0, false}, /* 0x24d4 */
+	{RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 2), 0, false}, /* 0x24d8 */
+	{RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 3), 0, false}, /* 0x24dc */
+	{RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 4), 0, false}, /* 0x24e0 */
+	{RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 5), 0, false}, /* 0x24e4 */
+	{RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 6), 0, false}, /* 0x24e8 */
+	{RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 7), 0, false}, /* 0x24ec */
+	{RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 8), 0, false}, /* 0x24f0 */
+	{RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 9), 0, false}, /* 0x24f4 */
+	{RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 10), 0, false}, /* 0x24f8 */
+	{RCS0, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 11), 0, false}, /* 0x24fc */
+	{RCS0, CACHE_MODE_1, 0xffff, true}, /* 0x7004 */
+	{RCS0, GEN7_GT_MODE, 0xffff, true}, /* 0x7008 */
+	{RCS0, CACHE_MODE_0_GEN7, 0xffff, true}, /* 0x7000 */
+	{RCS0, GEN7_COMMON_SLICE_CHICKEN1, 0xffff, true}, /* 0x7010 */
+	{RCS0, HDC_CHICKEN0, 0xffff, true}, /* 0x7300 */
+	{RCS0, VF_GUARDBAND, 0xffff, true}, /* 0x83a4 */
+
+	{RCS0, GEN8_PRIVATE_PAT_LO, 0, false}, /* 0x40e0 */
+	{RCS0, GEN8_PRIVATE_PAT_HI, 0, false}, /* 0x40e4 */
+	{RCS0, GEN8_CS_CHICKEN1, 0xffff, true}, /* 0x2580 */
+	{RCS0, COMMON_SLICE_CHICKEN2, 0xffff, true}, /* 0x7014 */
+	{RCS0, GEN9_CS_DEBUG_MODE1, 0xffff, false}, /* 0x20ec */
+	{RCS0, GEN8_L3SQCREG4, 0, false}, /* 0xb118 */
+	{RCS0, GEN7_HALF_SLICE_CHICKEN1, 0xffff, true}, /* 0xe100 */
+	{RCS0, HALF_SLICE_CHICKEN2, 0xffff, true}, /* 0xe180 */
+	{RCS0, HALF_SLICE_CHICKEN3, 0xffff, true}, /* 0xe184 */
+	{RCS0, GEN9_HALF_SLICE_CHICKEN5, 0xffff, true}, /* 0xe188 */
+	{RCS0, GEN9_HALF_SLICE_CHICKEN7, 0xffff, true}, /* 0xe194 */
+	{RCS0, GEN8_ROW_CHICKEN, 0xffff, true}, /* 0xe4f0 */
+	{RCS0, TRVATTL3PTRDW(0), 0, false}, /* 0x4de0 */
+	{RCS0, TRVATTL3PTRDW(1), 0, false}, /* 0x4de4 */
+	{RCS0, TRNULLDETCT, 0, false}, /* 0x4de8 */
+	{RCS0, TRINVTILEDETCT, 0, false}, /* 0x4dec */
+	{RCS0, TRVADR, 0, false}, /* 0x4df0 */
+	{RCS0, TRTTE, 0, false}, /* 0x4df4 */
+
+	{BCS0, RING_GFX_MODE(BLT_RING_BASE), 0xffff, false}, /* 0x2229c */
+	{BCS0, RING_MI_MODE(BLT_RING_BASE), 0xffff, false}, /* 0x2209c */
+	{BCS0, RING_INSTPM(BLT_RING_BASE), 0xffff, false}, /* 0x220c0 */
+	{BCS0, RING_HWSTAM(BLT_RING_BASE), 0x0, false}, /* 0x22098 */
+	{BCS0, RING_EXCC(BLT_RING_BASE), 0xffff, false}, /* 0x22028 */
+
+	{VCS1, RING_EXCC(GEN8_BSD2_RING_BASE), 0xffff, false}, /* 0x1c028 */
+
+	{VECS0, RING_EXCC(VEBOX_RING_BASE), 0xffff, false}, /* 0x1a028 */
+
+	{RCS0, GEN8_HDC_CHICKEN1, 0xffff, true}, /* 0x7304 */
+	{RCS0, GEN9_CTX_PREEMPT_REG, 0x0, false}, /* 0x2248 */
+	{RCS0, GEN7_UCGCTL4, 0x0, false}, /* 0x940c */
+	{RCS0, GAMT_CHKN_BIT_REG, 0x0, false}, /* 0x4ab8 */
+
+	{RCS0, GEN9_GAMT_ECO_REG_RW_IA, 0x0, false}, /* 0x4ab0 */
+	{RCS0, GEN9_CSFE_CHICKEN1_RCS, 0xffff, false}, /* 0x20d4 */
+
+	{RCS0, GEN8_GARBCNTL, 0x0, false}, /* 0xb004 */
+	{RCS0, GEN7_FF_THREAD_MODE, 0x0, false}, /* 0x20a0 */
+	{RCS0, FF_SLICE_CS_CHICKEN2, 0xffff, false}, /* 0x20e4 */
+	{RCS0, INVALID_MMIO_REG, 0, false } /* Terminated */
 };
 
 static struct {
@@ -150,11 +149,11 @@ static void load_render_mocs(struct drm_i915_private *dev_priv)
 {
 	i915_reg_t offset;
 	u32 regs[] = {
-		[RCS] = 0xc800,
-		[VCS] = 0xc900,
-		[VCS2] = 0xca00,
-		[BCS] = 0xcc00,
-		[VECS] = 0xcb00,
+		[RCS0]  = 0xc800,
+		[VCS0]  = 0xc900,
+		[VCS1]  = 0xca00,
+		[BCS0]  = 0xcc00,
+		[VECS0] = 0xcb00,
 	};
 	int ring_id, i;
 
@@ -302,7 +301,7 @@ int intel_vgpu_restore_inhibit_context(struct intel_vgpu *vgpu,
 		goto out;
 
 	/* no MOCS register in context except render engine */
-	if (req->engine->id != RCS)
+	if (req->engine->id != RCS0)
 		goto out;
 
 	ret = restore_render_mocs_control_for_inhibit(vgpu, req);
@@ -328,15 +327,16 @@ out:
 static void handle_tlb_pending_event(struct intel_vgpu *vgpu, int ring_id)
 {
 	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+	struct intel_uncore *uncore = &dev_priv->uncore;
 	struct intel_vgpu_submission *s = &vgpu->submission;
 	enum forcewake_domains fw;
 	i915_reg_t reg;
 	u32 regs[] = {
-		[RCS] = 0x4260,
-		[VCS] = 0x4264,
-		[VCS2] = 0x4268,
-		[BCS] = 0x426c,
-		[VECS] = 0x4270,
+		[RCS0]  = 0x4260,
+		[VCS0]  = 0x4264,
+		[VCS1]  = 0x4268,
+		[BCS0]  = 0x426c,
+		[VECS0] = 0x4270,
 	};
 
 	if (WARN_ON(ring_id >= ARRAY_SIZE(regs)))
@@ -352,21 +352,21 @@ static void handle_tlb_pending_event(struct intel_vgpu *vgpu, int ring_id)
 	 * otherwise device can go to RC6 state and interrupt invalidation
 	 * process
 	 */
-	fw = intel_uncore_forcewake_for_reg(dev_priv, reg,
+	fw = intel_uncore_forcewake_for_reg(uncore, reg,
 					    FW_REG_READ | FW_REG_WRITE);
-	if (ring_id == RCS && (INTEL_GEN(dev_priv) >= 9))
+	if (ring_id == RCS0 && INTEL_GEN(dev_priv) >= 9)
 		fw |= FORCEWAKE_RENDER;
 
-	intel_uncore_forcewake_get(dev_priv, fw);
+	intel_uncore_forcewake_get(uncore, fw);
 
-	I915_WRITE_FW(reg, 0x1);
+	intel_uncore_write_fw(uncore, reg, 0x1);
 
-	if (wait_for_atomic((I915_READ_FW(reg) == 0), 50))
+	if (wait_for_atomic((intel_uncore_read_fw(uncore, reg) == 0), 50))
 		gvt_vgpu_err("timeout in invalidate ring (%d) tlb\n", ring_id);
 	else
 		vgpu_vreg_t(vgpu, reg) = 0;
 
-	intel_uncore_forcewake_put(dev_priv, fw);
+	intel_uncore_forcewake_put(uncore, fw);
 
 	gvt_dbg_core("invalidate TLB for ring %d\n", ring_id);
 }
@@ -379,11 +379,11 @@ static void switch_mocs(struct intel_vgpu *pre, struct intel_vgpu *next,
 	u32 old_v, new_v;
 
 	u32 regs[] = {
-		[RCS] = 0xc800,
-		[VCS] = 0xc900,
-		[VCS2] = 0xca00,
-		[BCS] = 0xcc00,
-		[VECS] = 0xcb00,
+		[RCS0]  = 0xc800,
+		[VCS0]  = 0xc900,
+		[VCS1]  = 0xca00,
+		[BCS0]  = 0xcc00,
+		[VECS0] = 0xcb00,
 	};
 	int i;
 
@@ -391,8 +391,10 @@ static void switch_mocs(struct intel_vgpu *pre, struct intel_vgpu *next,
 	if (WARN_ON(ring_id >= ARRAY_SIZE(regs)))
 		return;
 
-	if ((IS_KABYLAKE(dev_priv)  || IS_BROXTON(dev_priv)
-		|| IS_COFFEELAKE(dev_priv)) && ring_id == RCS)
+	if (ring_id == RCS0 &&
+	    (IS_KABYLAKE(dev_priv) ||
+	     IS_BROXTON(dev_priv) ||
+	     IS_COFFEELAKE(dev_priv)))
 		return;
 
 	if (!pre && !gen9_render_mocs.initialized)
@@ -415,7 +417,7 @@ static void switch_mocs(struct intel_vgpu *pre, struct intel_vgpu *next,
 		offset.reg += 4;
 	}
 
-	if (ring_id == RCS) {
+	if (ring_id == RCS0) {
 		l3_offset.reg = 0xb020;
 		for (i = 0; i < GEN9_MOCS_SIZE / 2; i++) {
 			if (pre)
@@ -493,7 +495,8 @@ static void switch_mmio(struct intel_vgpu *pre,
 			 * itself.
 			 */
 			if (mmio->in_context &&
-			    !is_inhibit_context(&s->shadow_ctx->__engine[ring_id]))
+			    !is_inhibit_context(intel_context_lookup(s->shadow_ctx,
+								     dev_priv->engine[ring_id])))
 				continue;
 
 			if (mmio->mask)
@@ -550,9 +553,9 @@ void intel_gvt_switch_mmio(struct intel_vgpu *pre,
 	 * performace for batch mmio read/write, so we need
 	 * handle forcewake mannually.
 	 */
-	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
 	switch_mmio(pre, next, ring_id);
-	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
 }
 
 /**
diff --git a/drivers/gpu/drm/i915/gvt/reg.h b/drivers/gpu/drm/i915/gvt/reg.h
index 428d252344f1..3de5b643b266 100644
--- a/drivers/gpu/drm/i915/gvt/reg.h
+++ b/drivers/gpu/drm/i915/gvt/reg.h
@@ -60,6 +60,37 @@
 #define _REG_701C0(pipe, plane) (0x701c0 + pipe * 0x1000 + (plane - 1) * 0x100)
 #define _REG_701C4(pipe, plane) (0x701c4 + pipe * 0x1000 + (plane - 1) * 0x100)
 
+#define SKL_FLIP_EVENT(pipe, plane) (PRIMARY_A_FLIP_DONE + (plane) * 3 + (pipe))
+
+#define PLANE_CTL_ASYNC_FLIP		(1 << 9)
+#define REG50080_FLIP_TYPE_MASK	0x3
+#define REG50080_FLIP_TYPE_ASYNC	0x1
+
+#define REG_50080(_pipe, _plane) ({ \
+	typeof(_pipe) (p) = (_pipe); \
+	typeof(_plane) (q) = (_plane); \
+	(((p) == PIPE_A) ? (((q) == PLANE_PRIMARY) ? (_MMIO(0x50080)) : \
+		(_MMIO(0x50090))) : \
+	(((p) == PIPE_B) ? (((q) == PLANE_PRIMARY) ? (_MMIO(0x50088)) : \
+		(_MMIO(0x50098))) : \
+	(((p) == PIPE_C) ? (((q) == PLANE_PRIMARY) ? (_MMIO(0x5008C)) : \
+		(_MMIO(0x5009C))) : \
+		(_MMIO(0x50080))))); })
+
+#define REG_50080_TO_PIPE(_reg) ({ \
+	typeof(_reg) (reg) = (_reg); \
+	(((reg) == 0x50080 || (reg) == 0x50090) ? (PIPE_A) : \
+	(((reg) == 0x50088 || (reg) == 0x50098) ? (PIPE_B) : \
+	(((reg) == 0x5008C || (reg) == 0x5009C) ? (PIPE_C) : \
+	(INVALID_PIPE)))); })
+
+#define REG_50080_TO_PLANE(_reg) ({ \
+	typeof(_reg) (reg) = (_reg); \
+	(((reg) == 0x50080 || (reg) == 0x50088 || (reg) == 0x5008C) ? \
+		(PLANE_PRIMARY) : \
+	(((reg) == 0x50090 || (reg) == 0x50098 || (reg) == 0x5009C) ? \
+		(PLANE_SPRITE0) : (I915_MAX_PLANES))); })
+
 #define GFX_MODE_BIT_SET_IN_MASK(val, bit) \
 		((((bit) & 0xffff0000) == 0) && !!((val) & (((bit) << 16))))
 
@@ -95,4 +126,7 @@
 #define RING_GFX_MODE(base)	_MMIO((base) + 0x29c)
 #define VF_GUARDBAND		_MMIO(0x83a4)
 
+/* define the effective range of MCHBAR register on Sandybridge+ */
+#define MCHBAR_MIRROR_REG_BASE	_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x4000)
+
 #endif
diff --git a/drivers/gpu/drm/i915/gvt/scheduler.c b/drivers/gpu/drm/i915/gvt/scheduler.c
index 05b953793316..8998fa5ab198 100644
--- a/drivers/gpu/drm/i915/gvt/scheduler.c
+++ b/drivers/gpu/drm/i915/gvt/scheduler.c
@@ -93,7 +93,7 @@ static void sr_oa_regs(struct intel_vgpu_workload *workload,
 		i915_mmio_reg_offset(EU_PERF_CNTL6),
 	};
 
-	if (workload->ring_id != RCS)
+	if (workload->ring_id != RCS0)
 		return;
 
 	if (save) {
@@ -149,7 +149,7 @@ static int populate_shadow_context(struct intel_vgpu_workload *workload)
 	COPY_REG_MASKED(ctx_ctrl);
 	COPY_REG(ctx_timestamp);
 
-	if (ring_id == RCS) {
+	if (ring_id == RCS0) {
 		COPY_REG(bb_per_ctx_ptr);
 		COPY_REG(rcs_indirect_ctx);
 		COPY_REG(rcs_indirect_ctx_offset);
@@ -177,7 +177,7 @@ static int populate_shadow_context(struct intel_vgpu_workload *workload)
 
 	context_page_num = context_page_num >> PAGE_SHIFT;
 
-	if (IS_BROADWELL(gvt->dev_priv) && ring_id == RCS)
+	if (IS_BROADWELL(gvt->dev_priv) && ring_id == RCS0)
 		context_page_num = 19;
 
 	i = 2;
@@ -434,8 +434,7 @@ int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload)
 	if (ret)
 		goto err_unpin;
 
-	if ((workload->ring_id == RCS) &&
-	    (workload->wa_ctx.indirect_ctx.size != 0)) {
+	if (workload->ring_id == RCS0 && workload->wa_ctx.indirect_ctx.size) {
 		ret = intel_gvt_scan_and_shadow_wa_ctx(&workload->wa_ctx);
 		if (ret)
 			goto err_shadow;
@@ -803,7 +802,7 @@ static void update_guest_context(struct intel_vgpu_workload *workload)
 	context_page_num = rq->engine->context_size;
 	context_page_num = context_page_num >> PAGE_SHIFT;
 
-	if (IS_BROADWELL(gvt->dev_priv) && rq->engine->id == RCS)
+	if (IS_BROADWELL(gvt->dev_priv) && rq->engine->id == RCS0)
 		context_page_num = 19;
 
 	i = 2;
@@ -851,13 +850,13 @@ static void update_guest_context(struct intel_vgpu_workload *workload)
 }
 
 void intel_vgpu_clean_workloads(struct intel_vgpu *vgpu,
-				unsigned long engine_mask)
+				intel_engine_mask_t engine_mask)
 {
 	struct intel_vgpu_submission *s = &vgpu->submission;
 	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
 	struct intel_engine_cs *engine;
 	struct intel_vgpu_workload *pos, *n;
-	unsigned int tmp;
+	intel_engine_mask_t tmp;
 
 	/* free the unsubmited workloads in the queues. */
 	for_each_engine_masked(engine, dev_priv, engine_mask, tmp) {
@@ -903,8 +902,8 @@ static void complete_current_workload(struct intel_gvt *gvt, int ring_id)
 				workload->status = 0;
 		}
 
-		if (!workload->status && !(vgpu->resetting_eng &
-					   ENGINE_MASK(ring_id))) {
+		if (!workload->status &&
+		    !(vgpu->resetting_eng & BIT(ring_id))) {
 			update_guest_context(workload);
 
 			for_each_set_bit(event, workload->pending_events,
@@ -927,7 +926,7 @@ static void complete_current_workload(struct intel_gvt *gvt, int ring_id)
 
 	list_del_init(&workload->list);
 
-	if (workload->status || (vgpu->resetting_eng & ENGINE_MASK(ring_id))) {
+	if (workload->status || vgpu->resetting_eng & BIT(ring_id)) {
 		/* if workload->status is not successful means HW GPU
 		 * has occurred GPU hang or something wrong with i915/GVT,
 		 * and GVT won't inject context switch interrupt to guest.
@@ -941,7 +940,7 @@ static void complete_current_workload(struct intel_gvt *gvt, int ring_id)
 		 * cleaned up during the resetting process later, so doing
 		 * the workload clean up here doesn't have any impact.
 		 **/
-		intel_vgpu_clean_workloads(vgpu, ENGINE_MASK(ring_id));
+		intel_vgpu_clean_workloads(vgpu, BIT(ring_id));
 	}
 
 	workload->complete(workload);
@@ -1001,7 +1000,7 @@ static int workload_thread(void *priv)
 				workload->ring_id, workload);
 
 		if (need_force_wake)
-			intel_uncore_forcewake_get(gvt->dev_priv,
+			intel_uncore_forcewake_get(&gvt->dev_priv->uncore,
 					FORCEWAKE_ALL);
 
 		ret = dispatch_workload(workload);
@@ -1023,7 +1022,7 @@ complete:
 		complete_current_workload(gvt, ring_id);
 
 		if (need_force_wake)
-			intel_uncore_forcewake_put(gvt->dev_priv,
+			intel_uncore_forcewake_put(&gvt->dev_priv->uncore,
 					FORCEWAKE_ALL);
 
 		intel_runtime_pm_put_unchecked(gvt->dev_priv);
@@ -1114,9 +1113,9 @@ i915_context_ppgtt_root_restore(struct intel_vgpu_submission *s)
 	struct i915_hw_ppgtt *i915_ppgtt = s->shadow_ctx->ppgtt;
 	int i;
 
-	if (i915_vm_is_48bit(&i915_ppgtt->vm))
+	if (i915_vm_is_4lvl(&i915_ppgtt->vm)) {
 		px_dma(&i915_ppgtt->pml4) = s->i915_context_pml4;
-	else {
+	} else {
 		for (i = 0; i < GEN8_3LVL_PDPES; i++)
 			px_dma(i915_ppgtt->pdp.page_directory[i]) =
 						s->i915_context_pdps[i];
@@ -1150,7 +1149,7 @@ void intel_vgpu_clean_submission(struct intel_vgpu *vgpu)
  *
  */
 void intel_vgpu_reset_submission(struct intel_vgpu *vgpu,
-		unsigned long engine_mask)
+				 intel_engine_mask_t engine_mask)
 {
 	struct intel_vgpu_submission *s = &vgpu->submission;
 
@@ -1167,7 +1166,7 @@ i915_context_ppgtt_root_save(struct intel_vgpu_submission *s)
 	struct i915_hw_ppgtt *i915_ppgtt = s->shadow_ctx->ppgtt;
 	int i;
 
-	if (i915_vm_is_48bit(&i915_ppgtt->vm))
+	if (i915_vm_is_4lvl(&i915_ppgtt->vm))
 		s->i915_context_pml4 = px_dma(&i915_ppgtt->pml4);
 	else {
 		for (i = 0; i < GEN8_3LVL_PDPES; i++)
@@ -1240,7 +1239,7 @@ out_shadow_ctx:
  *
  */
 int intel_vgpu_select_submission_ops(struct intel_vgpu *vgpu,
-				     unsigned long engine_mask,
+				     intel_engine_mask_t engine_mask,
 				     unsigned int interface)
 {
 	struct intel_vgpu_submission *s = &vgpu->submission;
@@ -1450,7 +1449,7 @@ intel_vgpu_create_workload(struct intel_vgpu *vgpu, int ring_id,
 	workload->rb_start = start;
 	workload->rb_ctl = ctl;
 
-	if (ring_id == RCS) {
+	if (ring_id == RCS0) {
 		intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
 			RING_CTX_OFF(bb_per_ctx_ptr.val), &per_ctx, 4);
 		intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
diff --git a/drivers/gpu/drm/i915/gvt/scheduler.h b/drivers/gpu/drm/i915/gvt/scheduler.h
index 0635b2c4bed7..90c6756f5453 100644
--- a/drivers/gpu/drm/i915/gvt/scheduler.h
+++ b/drivers/gpu/drm/i915/gvt/scheduler.h
@@ -142,12 +142,12 @@ void intel_gvt_wait_vgpu_idle(struct intel_vgpu *vgpu);
 int intel_vgpu_setup_submission(struct intel_vgpu *vgpu);
 
 void intel_vgpu_reset_submission(struct intel_vgpu *vgpu,
-				 unsigned long engine_mask);
+				 intel_engine_mask_t engine_mask);
 
 void intel_vgpu_clean_submission(struct intel_vgpu *vgpu);
 
 int intel_vgpu_select_submission_ops(struct intel_vgpu *vgpu,
-				     unsigned long engine_mask,
+				     intel_engine_mask_t engine_mask,
 				     unsigned int interface);
 
 extern const struct intel_vgpu_submission_ops
@@ -160,6 +160,6 @@ intel_vgpu_create_workload(struct intel_vgpu *vgpu, int ring_id,
 void intel_vgpu_destroy_workload(struct intel_vgpu_workload *workload);
 
 void intel_vgpu_clean_workloads(struct intel_vgpu *vgpu,
-				unsigned long engine_mask);
+				intel_engine_mask_t engine_mask);
 
 #endif
diff --git a/drivers/gpu/drm/i915/gvt/vgpu.c b/drivers/gpu/drm/i915/gvt/vgpu.c
index 720e2b10adaa..44ce3c2b9ac1 100644
--- a/drivers/gpu/drm/i915/gvt/vgpu.c
+++ b/drivers/gpu/drm/i915/gvt/vgpu.c
@@ -44,7 +44,7 @@ void populate_pvinfo_page(struct intel_vgpu *vgpu)
 	vgpu_vreg_t(vgpu, vgtif_reg(display_ready)) = 0;
 	vgpu_vreg_t(vgpu, vgtif_reg(vgt_id)) = vgpu->id;
 
-	vgpu_vreg_t(vgpu, vgtif_reg(vgt_caps)) = VGT_CAPS_FULL_48BIT_PPGTT;
+	vgpu_vreg_t(vgpu, vgtif_reg(vgt_caps)) = VGT_CAPS_FULL_PPGTT;
 	vgpu_vreg_t(vgpu, vgtif_reg(vgt_caps)) |= VGT_CAPS_HWSP_EMULATION;
 	vgpu_vreg_t(vgpu, vgtif_reg(vgt_caps)) |= VGT_CAPS_HUGE_GTT;
 
@@ -526,11 +526,11 @@ struct intel_vgpu *intel_gvt_create_vgpu(struct intel_gvt *gvt,
  * GPU engines. For FLR, engine_mask is ignored.
  */
 void intel_gvt_reset_vgpu_locked(struct intel_vgpu *vgpu, bool dmlr,
-				 unsigned int engine_mask)
+				 intel_engine_mask_t engine_mask)
 {
 	struct intel_gvt *gvt = vgpu->gvt;
 	struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
-	unsigned int resetting_eng = dmlr ? ALL_ENGINES : engine_mask;
+	intel_engine_mask_t resetting_eng = dmlr ? ALL_ENGINES : engine_mask;
 
 	gvt_dbg_core("------------------------------------------\n");
 	gvt_dbg_core("resseting vgpu%d, dmlr %d, engine_mask %08x\n",
diff --git a/drivers/gpu/drm/i915/i915_active.c b/drivers/gpu/drm/i915/i915_active.c
index db7bb5bd5add..863ae12707ba 100644
--- a/drivers/gpu/drm/i915/i915_active.c
+++ b/drivers/gpu/drm/i915/i915_active.c
@@ -6,6 +6,7 @@
 
 #include "i915_drv.h"
 #include "i915_active.h"
+#include "i915_globals.h"
 
 #define BKL(ref) (&(ref)->i915->drm.struct_mutex)
 
@@ -17,6 +18,7 @@
  * nodes from a local slab cache to hopefully reduce the fragmentation.
  */
 static struct i915_global_active {
+	struct i915_global base;
 	struct kmem_cache *slab_cache;
 } global;
 
@@ -285,16 +287,27 @@ void i915_active_retire_noop(struct i915_active_request *active,
 #include "selftests/i915_active.c"
 #endif
 
+static void i915_global_active_shrink(void)
+{
+	kmem_cache_shrink(global.slab_cache);
+}
+
+static void i915_global_active_exit(void)
+{
+	kmem_cache_destroy(global.slab_cache);
+}
+
+static struct i915_global_active global = { {
+	.shrink = i915_global_active_shrink,
+	.exit = i915_global_active_exit,
+} };
+
 int __init i915_global_active_init(void)
 {
 	global.slab_cache = KMEM_CACHE(active_node, SLAB_HWCACHE_ALIGN);
 	if (!global.slab_cache)
 		return -ENOMEM;
 
+	i915_global_register(&global.base);
 	return 0;
 }
-
-void __exit i915_global_active_exit(void)
-{
-	kmem_cache_destroy(global.slab_cache);
-}
diff --git a/drivers/gpu/drm/i915/i915_active.h b/drivers/gpu/drm/i915/i915_active.h
index 12b5c1d287d1..7d758719ce39 100644
--- a/drivers/gpu/drm/i915/i915_active.h
+++ b/drivers/gpu/drm/i915/i915_active.h
@@ -108,19 +108,6 @@ i915_active_request_set_retire_fn(struct i915_active_request *active,
 	active->retire = fn ?: i915_active_retire_noop;
 }
 
-static inline struct i915_request *
-__i915_active_request_peek(const struct i915_active_request *active)
-{
-	/*
-	 * Inside the error capture (running with the driver in an unknown
-	 * state), we want to bend the rules slightly (a lot).
-	 *
-	 * Work is in progress to make it safer, in the meantime this keeps
-	 * the known issue from spamming the logs.
-	 */
-	return rcu_dereference_protected(active->request, 1);
-}
-
 /**
  * i915_active_request_raw - return the active request
  * @active - the active tracker
@@ -419,7 +406,4 @@ void i915_active_fini(struct i915_active *ref);
 static inline void i915_active_fini(struct i915_active *ref) { }
 #endif
 
-int i915_global_active_init(void);
-void i915_global_active_exit(void);
-
 #endif /* _I915_ACTIVE_H_ */
diff --git a/drivers/gpu/drm/i915/i915_cmd_parser.c b/drivers/gpu/drm/i915/i915_cmd_parser.c
index 33e8eed64423..503d548a55f7 100644
--- a/drivers/gpu/drm/i915/i915_cmd_parser.c
+++ b/drivers/gpu/drm/i915/i915_cmd_parser.c
@@ -868,8 +868,8 @@ void intel_engine_init_cmd_parser(struct intel_engine_cs *engine)
 	if (!IS_GEN(engine->i915, 7))
 		return;
 
-	switch (engine->id) {
-	case RCS:
+	switch (engine->class) {
+	case RENDER_CLASS:
 		if (IS_HASWELL(engine->i915)) {
 			cmd_tables = hsw_render_ring_cmds;
 			cmd_table_count =
@@ -889,12 +889,12 @@ void intel_engine_init_cmd_parser(struct intel_engine_cs *engine)
 
 		engine->get_cmd_length_mask = gen7_render_get_cmd_length_mask;
 		break;
-	case VCS:
+	case VIDEO_DECODE_CLASS:
 		cmd_tables = gen7_video_cmds;
 		cmd_table_count = ARRAY_SIZE(gen7_video_cmds);
 		engine->get_cmd_length_mask = gen7_bsd_get_cmd_length_mask;
 		break;
-	case BCS:
+	case COPY_ENGINE_CLASS:
 		if (IS_HASWELL(engine->i915)) {
 			cmd_tables = hsw_blt_ring_cmds;
 			cmd_table_count = ARRAY_SIZE(hsw_blt_ring_cmds);
@@ -913,14 +913,14 @@ void intel_engine_init_cmd_parser(struct intel_engine_cs *engine)
 
 		engine->get_cmd_length_mask = gen7_blt_get_cmd_length_mask;
 		break;
-	case VECS:
+	case VIDEO_ENHANCEMENT_CLASS:
 		cmd_tables = hsw_vebox_cmds;
 		cmd_table_count = ARRAY_SIZE(hsw_vebox_cmds);
 		/* VECS can use the same length_mask function as VCS */
 		engine->get_cmd_length_mask = gen7_bsd_get_cmd_length_mask;
 		break;
 	default:
-		MISSING_CASE(engine->id);
+		MISSING_CASE(engine->class);
 		return;
 	}
 
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index f6f6e5b78e97..5823ffb17821 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -26,14 +26,21 @@
  *
  */
 
-#include <linux/sort.h>
 #include <linux/sched/mm.h>
+#include <linux/sort.h>
+
 #include <drm/drm_debugfs.h>
 #include <drm/drm_fourcc.h>
-#include "intel_drv.h"
-#include "intel_guc_submission.h"
 
 #include "i915_reset.h"
+#include "intel_dp.h"
+#include "intel_drv.h"
+#include "intel_fbc.h"
+#include "intel_guc_submission.h"
+#include "intel_hdcp.h"
+#include "intel_hdmi.h"
+#include "intel_pm.h"
+#include "intel_psr.h"
 
 static inline struct drm_i915_private *node_to_i915(struct drm_info_node *node)
 {
@@ -388,12 +395,9 @@ static void print_context_stats(struct seq_file *m,
 	struct i915_gem_context *ctx;
 
 	list_for_each_entry(ctx, &i915->contexts.list, link) {
-		struct intel_engine_cs *engine;
-		enum intel_engine_id id;
-
-		for_each_engine(engine, i915, id) {
-			struct intel_context *ce = to_intel_context(ctx, engine);
+		struct intel_context *ce;
 
+		list_for_each_entry(ce, &ctx->active_engines, active_link) {
 			if (ce->state)
 				per_file_stats(0, ce->state->obj, &kstats);
 			if (ce->ring)
@@ -412,9 +416,8 @@ static void print_context_stats(struct seq_file *m,
 
 			rcu_read_lock();
 			task = pid_task(ctx->pid ?: file->pid, PIDTYPE_PID);
-			snprintf(name, sizeof(name), "%s/%d",
-				 task ? task->comm : "<unknown>",
-				 ctx->user_handle);
+			snprintf(name, sizeof(name), "%s",
+				 task ? task->comm : "<unknown>");
 			rcu_read_unlock();
 
 			print_file_stats(m, name, stats);
@@ -830,11 +833,11 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
 
 	} else if (!HAS_PCH_SPLIT(dev_priv)) {
 		seq_printf(m, "Interrupt enable:    %08x\n",
-			   I915_READ(IER));
+			   I915_READ(GEN2_IER));
 		seq_printf(m, "Interrupt identity:  %08x\n",
-			   I915_READ(IIR));
+			   I915_READ(GEN2_IIR));
 		seq_printf(m, "Interrupt mask:      %08x\n",
-			   I915_READ(IMR));
+			   I915_READ(GEN2_IMR));
 		for_each_pipe(dev_priv, pipe)
 			seq_printf(m, "Pipe %c stat:         %08x\n",
 				   pipe_name(pipe),
@@ -884,7 +887,7 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
 		for_each_engine(engine, dev_priv, id) {
 			seq_printf(m,
 				   "Graphics Interrupt mask (%s):	%08x\n",
-				   engine->name, I915_READ_IMR(engine));
+				   engine->name, ENGINE_READ(engine, RING_IMR));
 		}
 	}
 
@@ -1097,7 +1100,7 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
 		}
 
 		/* RPSTAT1 is in the GT power well */
-		intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
+		intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
 
 		reqf = I915_READ(GEN6_RPNSWREQ);
 		if (INTEL_GEN(dev_priv) >= 9)
@@ -1125,7 +1128,7 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
 		cagf = intel_gpu_freq(dev_priv,
 				      intel_get_cagf(dev_priv, rpstat));
 
-		intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+		intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
 
 		if (INTEL_GEN(dev_priv) >= 11) {
 			pm_ier = I915_READ(GEN11_GPM_WGBOXPERF_INTR_ENABLE);
@@ -1281,14 +1284,11 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused)
 	intel_wakeref_t wakeref;
 	enum intel_engine_id id;
 
+	seq_printf(m, "Reset flags: %lx\n", dev_priv->gpu_error.flags);
 	if (test_bit(I915_WEDGED, &dev_priv->gpu_error.flags))
-		seq_puts(m, "Wedged\n");
+		seq_puts(m, "\tWedged\n");
 	if (test_bit(I915_RESET_BACKOFF, &dev_priv->gpu_error.flags))
-		seq_puts(m, "Reset in progress: struct_mutex backoff\n");
-	if (waitqueue_active(&dev_priv->gpu_error.wait_queue))
-		seq_puts(m, "Waiter holding struct mutex\n");
-	if (waitqueue_active(&dev_priv->gpu_error.reset_queue))
-		seq_puts(m, "struct_mutex blocked for reset\n");
+		seq_puts(m, "\tDevice (global) reset in progress\n");
 
 	if (!i915_modparams.enable_hangcheck) {
 		seq_puts(m, "Hangcheck disabled\n");
@@ -1298,10 +1298,10 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused)
 	with_intel_runtime_pm(dev_priv, wakeref) {
 		for_each_engine(engine, dev_priv, id) {
 			acthd[id] = intel_engine_get_active_head(engine);
-			seqno[id] = intel_engine_get_seqno(engine);
+			seqno[id] = intel_engine_get_hangcheck_seqno(engine);
 		}
 
-		intel_engine_get_instdone(dev_priv->engine[RCS], &instdone);
+		intel_engine_get_instdone(dev_priv->engine[RCS0], &instdone);
 	}
 
 	if (timer_pending(&dev_priv->gpu_error.hangcheck_work.timer))
@@ -1318,8 +1318,9 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused)
 	for_each_engine(engine, dev_priv, id) {
 		seq_printf(m, "%s:\n", engine->name);
 		seq_printf(m, "\tseqno = %x [current %x, last %x], %dms ago\n",
-			   engine->hangcheck.seqno, seqno[id],
-			   intel_engine_last_submit(engine),
+			   engine->hangcheck.last_seqno,
+			   seqno[id],
+			   engine->hangcheck.next_seqno,
 			   jiffies_to_msecs(jiffies -
 					    engine->hangcheck.action_timestamp));
 
@@ -1327,7 +1328,7 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused)
 			   (long long)engine->hangcheck.acthd,
 			   (long long)acthd[id]);
 
-		if (engine->id == RCS) {
+		if (engine->id == RCS0) {
 			seq_puts(m, "\tinstdone read =\n");
 
 			i915_instdone_info(dev_priv, m, &instdone);
@@ -1419,13 +1420,14 @@ static int ironlake_drpc_info(struct seq_file *m)
 static int i915_forcewake_domains(struct seq_file *m, void *data)
 {
 	struct drm_i915_private *i915 = node_to_i915(m->private);
+	struct intel_uncore *uncore = &i915->uncore;
 	struct intel_uncore_forcewake_domain *fw_domain;
 	unsigned int tmp;
 
 	seq_printf(m, "user.bypass_count = %u\n",
-		   i915->uncore.user_forcewake.count);
+		   uncore->user_forcewake.count);
 
-	for_each_fw_domain(fw_domain, i915, tmp)
+	for_each_fw_domain(fw_domain, uncore, tmp)
 		seq_printf(m, "%s.wake_count = %u\n",
 			   intel_uncore_forcewake_domain_to_str(fw_domain->id),
 			   READ_ONCE(fw_domain->wake_count));
@@ -1882,9 +1884,7 @@ static int i915_context_status(struct seq_file *m, void *unused)
 {
 	struct drm_i915_private *dev_priv = node_to_i915(m->private);
 	struct drm_device *dev = &dev_priv->drm;
-	struct intel_engine_cs *engine;
 	struct i915_gem_context *ctx;
-	enum intel_engine_id id;
 	int ret;
 
 	ret = mutex_lock_interruptible(&dev->struct_mutex);
@@ -1892,6 +1892,8 @@ static int i915_context_status(struct seq_file *m, void *unused)
 		return ret;
 
 	list_for_each_entry(ctx, &dev_priv->contexts.list, link) {
+		struct intel_context *ce;
+
 		seq_puts(m, "HW context ");
 		if (!list_empty(&ctx->hw_id_link))
 			seq_printf(m, "%x [pin %u]", ctx->hw_id,
@@ -1914,11 +1916,8 @@ static int i915_context_status(struct seq_file *m, void *unused)
 		seq_putc(m, ctx->remap_slice ? 'R' : 'r');
 		seq_putc(m, '\n');
 
-		for_each_engine(engine, dev_priv, id) {
-			struct intel_context *ce =
-				to_intel_context(ctx, engine);
-
-			seq_printf(m, "%s: ", engine->name);
+		list_for_each_entry(ce, &ctx->active_engines, active_link) {
+			seq_printf(m, "%s: ", ce->engine->name);
 			if (ce->state)
 				describe_obj(m, ce->state->obj);
 			if (ce->ring)
@@ -2023,11 +2022,9 @@ static const char *rps_power_to_str(unsigned int power)
 static int i915_rps_boost_info(struct seq_file *m, void *data)
 {
 	struct drm_i915_private *dev_priv = node_to_i915(m->private);
-	struct drm_device *dev = &dev_priv->drm;
 	struct intel_rps *rps = &dev_priv->gt_pm.rps;
 	u32 act_freq = rps->cur_freq;
 	intel_wakeref_t wakeref;
-	struct drm_file *file;
 
 	with_intel_runtime_pm_if_in_use(dev_priv, wakeref) {
 		if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
@@ -2061,22 +2058,7 @@ static int i915_rps_boost_info(struct seq_file *m, void *data)
 		   intel_gpu_freq(dev_priv, rps->efficient_freq),
 		   intel_gpu_freq(dev_priv, rps->boost_freq));
 
-	mutex_lock(&dev->filelist_mutex);
-	list_for_each_entry_reverse(file, &dev->filelist, lhead) {
-		struct drm_i915_file_private *file_priv = file->driver_priv;
-		struct task_struct *task;
-
-		rcu_read_lock();
-		task = pid_task(file->pid, PIDTYPE_PID);
-		seq_printf(m, "%s [%d]: %d boosts\n",
-			   task ? task->comm : "<unknown>",
-			   task ? task->pid : -1,
-			   atomic_read(&file_priv->rps_client.boosts));
-		rcu_read_unlock();
-	}
-	seq_printf(m, "Kernel (anonymous) boosts: %d\n",
-		   atomic_read(&rps->boosts));
-	mutex_unlock(&dev->filelist_mutex);
+	seq_printf(m, "Wait boosts: %d\n", atomic_read(&rps->boosts));
 
 	if (INTEL_GEN(dev_priv) >= 6 &&
 	    rps->enabled &&
@@ -2084,12 +2066,12 @@ static int i915_rps_boost_info(struct seq_file *m, void *data)
 		u32 rpup, rpupei;
 		u32 rpdown, rpdownei;
 
-		intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
+		intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
 		rpup = I915_READ_FW(GEN6_RP_CUR_UP) & GEN6_RP_EI_MASK;
 		rpupei = I915_READ_FW(GEN6_RP_CUR_UP_EI) & GEN6_RP_EI_MASK;
 		rpdown = I915_READ_FW(GEN6_RP_CUR_DOWN) & GEN6_RP_EI_MASK;
 		rpdownei = I915_READ_FW(GEN6_RP_CUR_DOWN_EI) & GEN6_RP_EI_MASK;
-		intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+		intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
 
 		seq_printf(m, "\nRPS Autotuning (current \"%s\" window):\n",
 			   rps_power_to_str(rps->power.mode));
@@ -2112,8 +2094,8 @@ static int i915_llc(struct seq_file *m, void *data)
 	const bool edram = INTEL_GEN(dev_priv) > 8;
 
 	seq_printf(m, "LLC: %s\n", yesno(HAS_LLC(dev_priv)));
-	seq_printf(m, "%s: %lluMB\n", edram ? "eDRAM" : "eLLC",
-		   intel_uncore_edram_size(dev_priv)/1024/1024);
+	seq_printf(m, "%s: %uMB\n", edram ? "eDRAM" : "eLLC",
+		   dev_priv->edram_size_mb);
 
 	return 0;
 }
@@ -2270,7 +2252,7 @@ static int i915_guc_stage_pool(struct seq_file *m, void *data)
 	const struct intel_guc *guc = &dev_priv->guc;
 	struct guc_stage_desc *desc = guc->stage_desc_pool_vaddr;
 	struct intel_guc_client *client = guc->execbuf_client;
-	unsigned int tmp;
+	intel_engine_mask_t tmp;
 	int index;
 
 	if (!USES_GUC_SUBMISSION(dev_priv))
@@ -2607,7 +2589,6 @@ static int
 i915_edp_psr_debug_set(void *data, u64 val)
 {
 	struct drm_i915_private *dev_priv = data;
-	struct drm_modeset_acquire_ctx ctx;
 	intel_wakeref_t wakeref;
 	int ret;
 
@@ -2618,18 +2599,7 @@ i915_edp_psr_debug_set(void *data, u64 val)
 
 	wakeref = intel_runtime_pm_get(dev_priv);
 
-	drm_modeset_acquire_init(&ctx, DRM_MODESET_ACQUIRE_INTERRUPTIBLE);
-
-retry:
-	ret = intel_psr_set_debugfs_mode(dev_priv, &ctx, val);
-	if (ret == -EDEADLK) {
-		ret = drm_modeset_backoff(&ctx);
-		if (!ret)
-			goto retry;
-	}
-
-	drm_modeset_drop_locks(&ctx);
-	drm_modeset_acquire_fini(&ctx);
+	ret = intel_psr_debug_set(dev_priv, val);
 
 	intel_runtime_pm_put(dev_priv, wakeref);
 
@@ -2686,8 +2656,7 @@ static int i915_runtime_pm_status(struct seq_file *m, void *unused)
 	seq_printf(m, "Runtime power status: %s\n",
 		   enableddisabled(!dev_priv->power_domains.wakeref));
 
-	seq_printf(m, "GPU idle: %s (epoch %u)\n",
-		   yesno(!dev_priv->gt.awake), dev_priv->gt.epoch);
+	seq_printf(m, "GPU idle: %s\n", yesno(!dev_priv->gt.awake));
 	seq_printf(m, "IRQs disabled: %s\n",
 		   yesno(!intel_irqs_enabled(dev_priv)));
 #ifdef CONFIG_PM
@@ -2904,7 +2873,6 @@ static void intel_connector_info(struct seq_file *m,
 	if (connector->status == connector_status_disconnected)
 		return;
 
-	seq_printf(m, "\tname: %s\n", connector->display_info.name);
 	seq_printf(m, "\tphysical dimensions: %dx%dmm\n",
 		   connector->display_info.width_mm,
 		   connector->display_info.height_mm);
@@ -3123,8 +3091,7 @@ static int i915_engine_info(struct seq_file *m, void *unused)
 
 	wakeref = intel_runtime_pm_get(dev_priv);
 
-	seq_printf(m, "GT awake? %s (epoch %u)\n",
-		   yesno(dev_priv->gt.awake), dev_priv->gt.epoch);
+	seq_printf(m, "GT awake? %s\n", yesno(dev_priv->gt.awake));
 	seq_printf(m, "Global active requests: %d\n",
 		   dev_priv->gt.active_requests);
 	seq_printf(m, "CS timestamp frequency: %u kHz\n",
@@ -3211,7 +3178,7 @@ static int i915_shared_dplls_info(struct seq_file *m, void *unused)
 static int i915_wa_registers(struct seq_file *m, void *unused)
 {
 	struct drm_i915_private *i915 = node_to_i915(m->private);
-	const struct i915_wa_list *wal = &i915->engine[RCS]->ctx_wa_list;
+	const struct i915_wa_list *wal = &i915->engine[RCS0]->ctx_wa_list;
 	struct i915_wa *wa;
 	unsigned int i;
 
@@ -3865,11 +3832,18 @@ static const struct file_operations i915_cur_wm_latency_fops = {
 static int
 i915_wedged_get(void *data, u64 *val)
 {
-	struct drm_i915_private *dev_priv = data;
-
-	*val = i915_terminally_wedged(&dev_priv->gpu_error);
+	int ret = i915_terminally_wedged(data);
 
-	return 0;
+	switch (ret) {
+	case -EIO:
+		*val = 1;
+		return 0;
+	case 0:
+		*val = 0;
+		return 0;
+	default:
+		return ret;
+	}
 }
 
 static int
@@ -3877,16 +3851,9 @@ i915_wedged_set(void *data, u64 val)
 {
 	struct drm_i915_private *i915 = data;
 
-	/*
-	 * There is no safeguard against this debugfs entry colliding
-	 * with the hangcheck calling same i915_handle_error() in
-	 * parallel, causing an explosion. For now we assume that the
-	 * test harness is responsible enough not to inject gpu hangs
-	 * while it is writing to 'i915_wedged'
-	 */
-
-	if (i915_reset_backoff(&i915->gpu_error))
-		return -EAGAIN;
+	/* Flush any previous reset before applying for a new one */
+	wait_event(i915->gpu_error.reset_queue,
+		   !test_bit(I915_RESET_BACKOFF, &i915->gpu_error.flags));
 
 	i915_handle_error(i915, val, I915_ERROR_CAPTURE,
 			  "Manually set wedged engine mask = %llx", val);
@@ -3927,12 +3894,9 @@ static int
 i915_drop_caches_set(void *data, u64 val)
 {
 	struct drm_i915_private *i915 = data;
-	intel_wakeref_t wakeref;
-	int ret = 0;
 
 	DRM_DEBUG("Dropping caches: 0x%08llx [0x%08llx]\n",
 		  val, val & DROP_ALL);
-	wakeref = intel_runtime_pm_get(i915);
 
 	if (val & DROP_RESET_ACTIVE &&
 	    wait_for(intel_engines_are_idle(i915), I915_IDLE_ENGINES_TIMEOUT))
@@ -3941,9 +3905,11 @@ i915_drop_caches_set(void *data, u64 val)
 	/* No need to check and wait for gpu resets, only libdrm auto-restarts
 	 * on ioctls on -EAGAIN. */
 	if (val & (DROP_ACTIVE | DROP_RETIRE | DROP_RESET_SEQNO)) {
+		int ret;
+
 		ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
 		if (ret)
-			goto out;
+			return ret;
 
 		if (val & DROP_ACTIVE)
 			ret = i915_gem_wait_for_idle(i915,
@@ -3957,7 +3923,7 @@ i915_drop_caches_set(void *data, u64 val)
 		mutex_unlock(&i915->drm.struct_mutex);
 	}
 
-	if (val & DROP_RESET_ACTIVE && i915_terminally_wedged(&i915->gpu_error))
+	if (val & DROP_RESET_ACTIVE && i915_terminally_wedged(i915))
 		i915_handle_error(i915, ALL_ENGINES, 0, NULL);
 
 	fs_reclaim_acquire(GFP_KERNEL);
@@ -3982,10 +3948,7 @@ i915_drop_caches_set(void *data, u64 val)
 	if (val & DROP_FREED)
 		i915_gem_drain_freed_objects(i915);
 
-out:
-	intel_runtime_pm_put(i915, wakeref);
-
-	return ret;
+	return 0;
 }
 
 DEFINE_SIMPLE_ATTRIBUTE(i915_drop_caches_fops,
@@ -4293,7 +4256,7 @@ static int i915_forcewake_open(struct inode *inode, struct file *file)
 		return 0;
 
 	file->private_data = (void *)(uintptr_t)intel_runtime_pm_get(i915);
-	intel_uncore_forcewake_user_get(i915);
+	intel_uncore_forcewake_user_get(&i915->uncore);
 
 	return 0;
 }
@@ -4305,7 +4268,7 @@ static int i915_forcewake_release(struct inode *inode, struct file *file)
 	if (INTEL_GEN(i915) < 6)
 		return 0;
 
-	intel_uncore_forcewake_user_put(i915);
+	intel_uncore_forcewake_user_put(&i915->uncore);
 	intel_runtime_pm_put(i915,
 			     (intel_wakeref_t)(uintptr_t)file->private_data);
 
@@ -4858,6 +4821,8 @@ static int i915_dsc_fec_support_show(struct seq_file *m, void *data)
 			   yesno(crtc_state->dsc_params.compression_enable));
 		seq_printf(m, "DSC_Sink_Support: %s\n",
 			   yesno(drm_dp_sink_supports_dsc(intel_dp->dsc_dpcd)));
+		seq_printf(m, "Force_DSC_Enable: %s\n",
+			   yesno(intel_dp->force_dsc_en));
 		if (!intel_dp_is_edp(intel_dp))
 			seq_printf(m, "FEC_Sink_Support: %s\n",
 				   yesno(drm_dp_sink_supports_fec(intel_dp->fec_capable)));
diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
index 9df65d386d11..1ad88e6d7c04 100644
--- a/drivers/gpu/drm/i915/i915_drv.c
+++ b/drivers/gpu/drm/i915/i915_drv.c
@@ -48,12 +48,19 @@
 #include <drm/i915_drm.h>
 
 #include "i915_drv.h"
-#include "i915_trace.h"
 #include "i915_pmu.h"
-#include "i915_reset.h"
 #include "i915_query.h"
+#include "i915_reset.h"
+#include "i915_trace.h"
 #include "i915_vgpu.h"
+#include "intel_audio.h"
+#include "intel_cdclk.h"
+#include "intel_csr.h"
+#include "intel_dp.h"
 #include "intel_drv.h"
+#include "intel_fbdev.h"
+#include "intel_pm.h"
+#include "intel_sprite.h"
 #include "intel_uc.h"
 #include "intel_workarounds.h"
 
@@ -188,6 +195,11 @@ intel_pch_type(const struct drm_i915_private *dev_priv, unsigned short id)
 		DRM_DEBUG_KMS("Found Cannon Lake LP PCH (CNP-LP)\n");
 		WARN_ON(!IS_CANNONLAKE(dev_priv) && !IS_COFFEELAKE(dev_priv));
 		return PCH_CNP;
+	case INTEL_PCH_CMP_DEVICE_ID_TYPE:
+		DRM_DEBUG_KMS("Found Comet Lake PCH (CMP)\n");
+		WARN_ON(!IS_COFFEELAKE(dev_priv));
+		/* CometPoint is CNP Compatible */
+		return PCH_CNP;
 	case INTEL_PCH_ICP_DEVICE_ID_TYPE:
 		DRM_DEBUG_KMS("Found Ice Lake PCH\n");
 		WARN_ON(!IS_ICELAKE(dev_priv));
@@ -219,20 +231,20 @@ intel_virt_detect_pch(const struct drm_i915_private *dev_priv)
 	 * make an educated guess as to which PCH is really there.
 	 */
 
-	if (IS_GEN(dev_priv, 5))
-		id = INTEL_PCH_IBX_DEVICE_ID_TYPE;
-	else if (IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv))
-		id = INTEL_PCH_CPT_DEVICE_ID_TYPE;
+	if (IS_ICELAKE(dev_priv))
+		id = INTEL_PCH_ICP_DEVICE_ID_TYPE;
+	else if (IS_CANNONLAKE(dev_priv) || IS_COFFEELAKE(dev_priv))
+		id = INTEL_PCH_CNP_DEVICE_ID_TYPE;
+	else if (IS_KABYLAKE(dev_priv) || IS_SKYLAKE(dev_priv))
+		id = INTEL_PCH_SPT_DEVICE_ID_TYPE;
 	else if (IS_HSW_ULT(dev_priv) || IS_BDW_ULT(dev_priv))
 		id = INTEL_PCH_LPT_LP_DEVICE_ID_TYPE;
 	else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
 		id = INTEL_PCH_LPT_DEVICE_ID_TYPE;
-	else if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))
-		id = INTEL_PCH_SPT_DEVICE_ID_TYPE;
-	else if (IS_COFFEELAKE(dev_priv) || IS_CANNONLAKE(dev_priv))
-		id = INTEL_PCH_CNP_DEVICE_ID_TYPE;
-	else if (IS_ICELAKE(dev_priv))
-		id = INTEL_PCH_ICP_DEVICE_ID_TYPE;
+	else if (IS_GEN(dev_priv, 6) || IS_IVYBRIDGE(dev_priv))
+		id = INTEL_PCH_CPT_DEVICE_ID_TYPE;
+	else if (IS_GEN(dev_priv, 5))
+		id = INTEL_PCH_IBX_DEVICE_ID_TYPE;
 
 	if (id)
 		DRM_DEBUG_KMS("Assuming PCH ID %04x\n", id);
@@ -330,16 +342,16 @@ static int i915_getparam_ioctl(struct drm_device *dev, void *data,
 		value = dev_priv->overlay ? 1 : 0;
 		break;
 	case I915_PARAM_HAS_BSD:
-		value = !!dev_priv->engine[VCS];
+		value = !!dev_priv->engine[VCS0];
 		break;
 	case I915_PARAM_HAS_BLT:
-		value = !!dev_priv->engine[BCS];
+		value = !!dev_priv->engine[BCS0];
 		break;
 	case I915_PARAM_HAS_VEBOX:
-		value = !!dev_priv->engine[VECS];
+		value = !!dev_priv->engine[VECS0];
 		break;
 	case I915_PARAM_HAS_BSD2:
-		value = !!dev_priv->engine[VCS2];
+		value = !!dev_priv->engine[VCS1];
 		break;
 	case I915_PARAM_HAS_LLC:
 		value = HAS_LLC(dev_priv);
@@ -348,10 +360,10 @@ static int i915_getparam_ioctl(struct drm_device *dev, void *data,
 		value = HAS_WT(dev_priv);
 		break;
 	case I915_PARAM_HAS_ALIASING_PPGTT:
-		value = min_t(int, INTEL_PPGTT(dev_priv), I915_GEM_PPGTT_FULL);
+		value = INTEL_PPGTT(dev_priv);
 		break;
 	case I915_PARAM_HAS_SEMAPHORES:
-		value = 0;
+		value = !!(dev_priv->caps.scheduler & I915_SCHEDULER_CAP_SEMAPHORES);
 		break;
 	case I915_PARAM_HAS_SECURE_BATCHES:
 		value = capable(CAP_SYS_ADMIN);
@@ -714,8 +726,7 @@ static int i915_load_modeset_init(struct drm_device *dev)
 	return 0;
 
 cleanup_gem:
-	if (i915_gem_suspend(dev_priv))
-		DRM_ERROR("failed to idle hardware; continuing to unload!\n");
+	i915_gem_suspend(dev_priv);
 	i915_gem_fini(dev_priv);
 cleanup_modeset:
 	intel_modeset_cleanup(dev);
@@ -864,15 +875,19 @@ static int i915_driver_init_early(struct drm_i915_private *dev_priv)
 	if (i915_inject_load_failure())
 		return -ENODEV;
 
+	intel_device_info_subplatform_init(dev_priv);
+
+	intel_uncore_init_early(&dev_priv->uncore);
+
 	spin_lock_init(&dev_priv->irq_lock);
 	spin_lock_init(&dev_priv->gpu_error.lock);
 	mutex_init(&dev_priv->backlight_lock);
-	spin_lock_init(&dev_priv->uncore.lock);
 
 	mutex_init(&dev_priv->sb_lock);
 	mutex_init(&dev_priv->av_mutex);
 	mutex_init(&dev_priv->wm.wm_mutex);
 	mutex_init(&dev_priv->pps_mutex);
+	mutex_init(&dev_priv->hdcp_comp_mutex);
 
 	i915_memcpy_init_early(dev_priv);
 	intel_runtime_pm_init_early(dev_priv);
@@ -930,46 +945,6 @@ static void i915_driver_cleanup_early(struct drm_i915_private *dev_priv)
 	i915_engines_cleanup(dev_priv);
 }
 
-static int i915_mmio_setup(struct drm_i915_private *dev_priv)
-{
-	struct pci_dev *pdev = dev_priv->drm.pdev;
-	int mmio_bar;
-	int mmio_size;
-
-	mmio_bar = IS_GEN(dev_priv, 2) ? 1 : 0;
-	/*
-	 * Before gen4, the registers and the GTT are behind different BARs.
-	 * However, from gen4 onwards, the registers and the GTT are shared
-	 * in the same BAR, so we want to restrict this ioremap from
-	 * clobbering the GTT which we want ioremap_wc instead. Fortunately,
-	 * the register BAR remains the same size for all the earlier
-	 * generations up to Ironlake.
-	 */
-	if (INTEL_GEN(dev_priv) < 5)
-		mmio_size = 512 * 1024;
-	else
-		mmio_size = 2 * 1024 * 1024;
-	dev_priv->regs = pci_iomap(pdev, mmio_bar, mmio_size);
-	if (dev_priv->regs == NULL) {
-		DRM_ERROR("failed to map registers\n");
-
-		return -EIO;
-	}
-
-	/* Try to make sure MCHBAR is enabled before poking at it */
-	intel_setup_mchbar(dev_priv);
-
-	return 0;
-}
-
-static void i915_mmio_cleanup(struct drm_i915_private *dev_priv)
-{
-	struct pci_dev *pdev = dev_priv->drm.pdev;
-
-	intel_teardown_mchbar(dev_priv);
-	pci_iounmap(pdev, dev_priv->regs);
-}
-
 /**
  * i915_driver_init_mmio - setup device MMIO
  * @dev_priv: device private
@@ -989,15 +964,16 @@ static int i915_driver_init_mmio(struct drm_i915_private *dev_priv)
 	if (i915_get_bridge_dev(dev_priv))
 		return -EIO;
 
-	ret = i915_mmio_setup(dev_priv);
+	ret = intel_uncore_init_mmio(&dev_priv->uncore);
 	if (ret < 0)
 		goto err_bridge;
 
-	intel_uncore_init(dev_priv);
+	/* Try to make sure MCHBAR is enabled before poking at it */
+	intel_setup_mchbar(dev_priv);
 
 	intel_device_info_init_mmio(dev_priv);
 
-	intel_uncore_prune(dev_priv);
+	intel_uncore_prune_mmio_domains(&dev_priv->uncore);
 
 	intel_uc_init_mmio(dev_priv);
 
@@ -1010,8 +986,8 @@ static int i915_driver_init_mmio(struct drm_i915_private *dev_priv)
 	return 0;
 
 err_uncore:
-	intel_uncore_fini(dev_priv);
-	i915_mmio_cleanup(dev_priv);
+	intel_teardown_mchbar(dev_priv);
+	intel_uncore_fini_mmio(&dev_priv->uncore);
 err_bridge:
 	pci_dev_put(dev_priv->bridge_dev);
 
@@ -1024,8 +1000,8 @@ err_bridge:
  */
 static void i915_driver_cleanup_mmio(struct drm_i915_private *dev_priv)
 {
-	intel_uncore_fini(dev_priv);
-	i915_mmio_cleanup(dev_priv);
+	intel_teardown_mchbar(dev_priv);
+	intel_uncore_fini_mmio(&dev_priv->uncore);
 	pci_dev_put(dev_priv->bridge_dev);
 }
 
@@ -1034,110 +1010,180 @@ static void intel_sanitize_options(struct drm_i915_private *dev_priv)
 	intel_gvt_sanitize_options(dev_priv);
 }
 
-static enum dram_rank skl_get_dimm_rank(u8 size, u32 rank)
+#define DRAM_TYPE_STR(type) [INTEL_DRAM_ ## type] = #type
+
+static const char *intel_dram_type_str(enum intel_dram_type type)
 {
-	if (size == 0)
-		return I915_DRAM_RANK_INVALID;
-	if (rank == SKL_DRAM_RANK_SINGLE)
-		return I915_DRAM_RANK_SINGLE;
-	else if (rank == SKL_DRAM_RANK_DUAL)
-		return I915_DRAM_RANK_DUAL;
+	static const char * const str[] = {
+		DRAM_TYPE_STR(UNKNOWN),
+		DRAM_TYPE_STR(DDR3),
+		DRAM_TYPE_STR(DDR4),
+		DRAM_TYPE_STR(LPDDR3),
+		DRAM_TYPE_STR(LPDDR4),
+	};
+
+	if (type >= ARRAY_SIZE(str))
+		type = INTEL_DRAM_UNKNOWN;
 
-	return I915_DRAM_RANK_INVALID;
+	return str[type];
 }
 
-static bool
-skl_is_16gb_dimm(enum dram_rank rank, u8 size, u8 width)
+#undef DRAM_TYPE_STR
+
+static int intel_dimm_num_devices(const struct dram_dimm_info *dimm)
 {
-	if (rank == I915_DRAM_RANK_SINGLE && width == 8 && size == 16)
-		return true;
-	else if (rank == I915_DRAM_RANK_DUAL && width == 8 && size == 32)
-		return true;
-	else if (rank == SKL_DRAM_RANK_SINGLE && width == 16 && size == 8)
-		return true;
-	else if (rank == SKL_DRAM_RANK_DUAL && width == 16 && size == 16)
-		return true;
+	return dimm->ranks * 64 / (dimm->width ?: 1);
+}
 
-	return false;
+/* Returns total GB for the whole DIMM */
+static int skl_get_dimm_size(u16 val)
+{
+	return val & SKL_DRAM_SIZE_MASK;
 }
 
-static int
-skl_dram_get_channel_info(struct dram_channel_info *ch, u32 val)
+static int skl_get_dimm_width(u16 val)
 {
-	u32 tmp_l, tmp_s;
-	u32 s_val = val >> SKL_DRAM_S_SHIFT;
+	if (skl_get_dimm_size(val) == 0)
+		return 0;
 
-	if (!val)
-		return -EINVAL;
+	switch (val & SKL_DRAM_WIDTH_MASK) {
+	case SKL_DRAM_WIDTH_X8:
+	case SKL_DRAM_WIDTH_X16:
+	case SKL_DRAM_WIDTH_X32:
+		val = (val & SKL_DRAM_WIDTH_MASK) >> SKL_DRAM_WIDTH_SHIFT;
+		return 8 << val;
+	default:
+		MISSING_CASE(val);
+		return 0;
+	}
+}
+
+static int skl_get_dimm_ranks(u16 val)
+{
+	if (skl_get_dimm_size(val) == 0)
+		return 0;
+
+	val = (val & SKL_DRAM_RANK_MASK) >> SKL_DRAM_RANK_SHIFT;
+
+	return val + 1;
+}
+
+/* Returns total GB for the whole DIMM */
+static int cnl_get_dimm_size(u16 val)
+{
+	return (val & CNL_DRAM_SIZE_MASK) / 2;
+}
+
+static int cnl_get_dimm_width(u16 val)
+{
+	if (cnl_get_dimm_size(val) == 0)
+		return 0;
+
+	switch (val & CNL_DRAM_WIDTH_MASK) {
+	case CNL_DRAM_WIDTH_X8:
+	case CNL_DRAM_WIDTH_X16:
+	case CNL_DRAM_WIDTH_X32:
+		val = (val & CNL_DRAM_WIDTH_MASK) >> CNL_DRAM_WIDTH_SHIFT;
+		return 8 << val;
+	default:
+		MISSING_CASE(val);
+		return 0;
+	}
+}
+
+static int cnl_get_dimm_ranks(u16 val)
+{
+	if (cnl_get_dimm_size(val) == 0)
+		return 0;
+
+	val = (val & CNL_DRAM_RANK_MASK) >> CNL_DRAM_RANK_SHIFT;
+
+	return val + 1;
+}
 
-	tmp_l = val & SKL_DRAM_SIZE_MASK;
-	tmp_s = s_val & SKL_DRAM_SIZE_MASK;
+static bool
+skl_is_16gb_dimm(const struct dram_dimm_info *dimm)
+{
+	/* Convert total GB to Gb per DRAM device */
+	return 8 * dimm->size / (intel_dimm_num_devices(dimm) ?: 1) == 16;
+}
+
+static void
+skl_dram_get_dimm_info(struct drm_i915_private *dev_priv,
+		       struct dram_dimm_info *dimm,
+		       int channel, char dimm_name, u16 val)
+{
+	if (INTEL_GEN(dev_priv) >= 10) {
+		dimm->size = cnl_get_dimm_size(val);
+		dimm->width = cnl_get_dimm_width(val);
+		dimm->ranks = cnl_get_dimm_ranks(val);
+	} else {
+		dimm->size = skl_get_dimm_size(val);
+		dimm->width = skl_get_dimm_width(val);
+		dimm->ranks = skl_get_dimm_ranks(val);
+	}
+
+	DRM_DEBUG_KMS("CH%u DIMM %c size: %u GB, width: X%u, ranks: %u, 16Gb DIMMs: %s\n",
+		      channel, dimm_name, dimm->size, dimm->width, dimm->ranks,
+		      yesno(skl_is_16gb_dimm(dimm)));
+}
+
+static int
+skl_dram_get_channel_info(struct drm_i915_private *dev_priv,
+			  struct dram_channel_info *ch,
+			  int channel, u32 val)
+{
+	skl_dram_get_dimm_info(dev_priv, &ch->dimm_l,
+			       channel, 'L', val & 0xffff);
+	skl_dram_get_dimm_info(dev_priv, &ch->dimm_s,
+			       channel, 'S', val >> 16);
 
-	if (tmp_l == 0 && tmp_s == 0)
+	if (ch->dimm_l.size == 0 && ch->dimm_s.size == 0) {
+		DRM_DEBUG_KMS("CH%u not populated\n", channel);
 		return -EINVAL;
+	}
 
-	ch->l_info.size = tmp_l;
-	ch->s_info.size = tmp_s;
-
-	tmp_l = (val & SKL_DRAM_WIDTH_MASK) >> SKL_DRAM_WIDTH_SHIFT;
-	tmp_s = (s_val & SKL_DRAM_WIDTH_MASK) >> SKL_DRAM_WIDTH_SHIFT;
-	ch->l_info.width = (1 << tmp_l) * 8;
-	ch->s_info.width = (1 << tmp_s) * 8;
-
-	tmp_l = val & SKL_DRAM_RANK_MASK;
-	tmp_s = s_val & SKL_DRAM_RANK_MASK;
-	ch->l_info.rank = skl_get_dimm_rank(ch->l_info.size, tmp_l);
-	ch->s_info.rank = skl_get_dimm_rank(ch->s_info.size, tmp_s);
-
-	if (ch->l_info.rank == I915_DRAM_RANK_DUAL ||
-	    ch->s_info.rank == I915_DRAM_RANK_DUAL)
-		ch->rank = I915_DRAM_RANK_DUAL;
-	else if (ch->l_info.rank == I915_DRAM_RANK_SINGLE &&
-		 ch->s_info.rank == I915_DRAM_RANK_SINGLE)
-		ch->rank = I915_DRAM_RANK_DUAL;
+	if (ch->dimm_l.ranks == 2 || ch->dimm_s.ranks == 2)
+		ch->ranks = 2;
+	else if (ch->dimm_l.ranks == 1 && ch->dimm_s.ranks == 1)
+		ch->ranks = 2;
 	else
-		ch->rank = I915_DRAM_RANK_SINGLE;
+		ch->ranks = 1;
 
-	ch->is_16gb_dimm = skl_is_16gb_dimm(ch->l_info.rank, ch->l_info.size,
-					    ch->l_info.width) ||
-			   skl_is_16gb_dimm(ch->s_info.rank, ch->s_info.size,
-					    ch->s_info.width);
+	ch->is_16gb_dimm =
+		skl_is_16gb_dimm(&ch->dimm_l) ||
+		skl_is_16gb_dimm(&ch->dimm_s);
 
-	DRM_DEBUG_KMS("(size:width:rank) L(%dGB:X%d:%s) S(%dGB:X%d:%s)\n",
-		      ch->l_info.size, ch->l_info.width,
-		      ch->l_info.rank ? "dual" : "single",
-		      ch->s_info.size, ch->s_info.width,
-		      ch->s_info.rank ? "dual" : "single");
+	DRM_DEBUG_KMS("CH%u ranks: %u, 16Gb DIMMs: %s\n",
+		      channel, ch->ranks, yesno(ch->is_16gb_dimm));
 
 	return 0;
 }
 
 static bool
-intel_is_dram_symmetric(u32 val_ch0, u32 val_ch1,
-			struct dram_channel_info *ch0)
+intel_is_dram_symmetric(const struct dram_channel_info *ch0,
+			const struct dram_channel_info *ch1)
 {
-	return (val_ch0 == val_ch1 &&
-		(ch0->s_info.size == 0 ||
-		 (ch0->l_info.size == ch0->s_info.size &&
-		  ch0->l_info.width == ch0->s_info.width &&
-		  ch0->l_info.rank == ch0->s_info.rank)));
+	return !memcmp(ch0, ch1, sizeof(*ch0)) &&
+		(ch0->dimm_s.size == 0 ||
+		 !memcmp(&ch0->dimm_l, &ch0->dimm_s, sizeof(ch0->dimm_l)));
 }
 
 static int
 skl_dram_get_channels_info(struct drm_i915_private *dev_priv)
 {
 	struct dram_info *dram_info = &dev_priv->dram_info;
-	struct dram_channel_info ch0, ch1;
-	u32 val_ch0, val_ch1;
+	struct dram_channel_info ch0 = {}, ch1 = {};
+	u32 val;
 	int ret;
 
-	val_ch0 = I915_READ(SKL_MAD_DIMM_CH0_0_0_0_MCHBAR_MCMAIN);
-	ret = skl_dram_get_channel_info(&ch0, val_ch0);
+	val = I915_READ(SKL_MAD_DIMM_CH0_0_0_0_MCHBAR_MCMAIN);
+	ret = skl_dram_get_channel_info(dev_priv, &ch0, 0, val);
 	if (ret == 0)
 		dram_info->num_channels++;
 
-	val_ch1 = I915_READ(SKL_MAD_DIMM_CH1_0_0_0_MCHBAR_MCMAIN);
-	ret = skl_dram_get_channel_info(&ch1, val_ch1);
+	val = I915_READ(SKL_MAD_DIMM_CH1_0_0_0_MCHBAR_MCMAIN);
+	ret = skl_dram_get_channel_info(dev_priv, &ch1, 1, val);
 	if (ret == 0)
 		dram_info->num_channels++;
 
@@ -1151,28 +1197,47 @@ skl_dram_get_channels_info(struct drm_i915_private *dev_priv)
 	 * will be same as if single rank memory, so consider single rank
 	 * memory.
 	 */
-	if (ch0.rank == I915_DRAM_RANK_SINGLE ||
-	    ch1.rank == I915_DRAM_RANK_SINGLE)
-		dram_info->rank = I915_DRAM_RANK_SINGLE;
+	if (ch0.ranks == 1 || ch1.ranks == 1)
+		dram_info->ranks = 1;
 	else
-		dram_info->rank = max(ch0.rank, ch1.rank);
+		dram_info->ranks = max(ch0.ranks, ch1.ranks);
 
-	if (dram_info->rank == I915_DRAM_RANK_INVALID) {
+	if (dram_info->ranks == 0) {
 		DRM_INFO("couldn't get memory rank information\n");
 		return -EINVAL;
 	}
 
 	dram_info->is_16gb_dimm = ch0.is_16gb_dimm || ch1.is_16gb_dimm;
 
-	dev_priv->dram_info.symmetric_memory = intel_is_dram_symmetric(val_ch0,
-								       val_ch1,
-								       &ch0);
+	dram_info->symmetric_memory = intel_is_dram_symmetric(&ch0, &ch1);
 
-	DRM_DEBUG_KMS("memory configuration is %sSymmetric memory\n",
-		      dev_priv->dram_info.symmetric_memory ? "" : "not ");
+	DRM_DEBUG_KMS("Memory configuration is symmetric? %s\n",
+		      yesno(dram_info->symmetric_memory));
 	return 0;
 }
 
+static enum intel_dram_type
+skl_get_dram_type(struct drm_i915_private *dev_priv)
+{
+	u32 val;
+
+	val = I915_READ(SKL_MAD_INTER_CHANNEL_0_0_0_MCHBAR_MCMAIN);
+
+	switch (val & SKL_DRAM_DDR_TYPE_MASK) {
+	case SKL_DRAM_DDR_TYPE_DDR3:
+		return INTEL_DRAM_DDR3;
+	case SKL_DRAM_DDR_TYPE_DDR4:
+		return INTEL_DRAM_DDR4;
+	case SKL_DRAM_DDR_TYPE_LPDDR3:
+		return INTEL_DRAM_LPDDR3;
+	case SKL_DRAM_DDR_TYPE_LPDDR4:
+		return INTEL_DRAM_LPDDR4;
+	default:
+		MISSING_CASE(val);
+		return INTEL_DRAM_UNKNOWN;
+	}
+}
+
 static int
 skl_get_dram_info(struct drm_i915_private *dev_priv)
 {
@@ -1180,6 +1245,9 @@ skl_get_dram_info(struct drm_i915_private *dev_priv)
 	u32 mem_freq_khz, val;
 	int ret;
 
+	dram_info->type = skl_get_dram_type(dev_priv);
+	DRM_DEBUG_KMS("DRAM type: %s\n", intel_dram_type_str(dram_info->type));
+
 	ret = skl_dram_get_channels_info(dev_priv);
 	if (ret)
 		return ret;
@@ -1200,6 +1268,85 @@ skl_get_dram_info(struct drm_i915_private *dev_priv)
 	return 0;
 }
 
+/* Returns Gb per DRAM device */
+static int bxt_get_dimm_size(u32 val)
+{
+	switch (val & BXT_DRAM_SIZE_MASK) {
+	case BXT_DRAM_SIZE_4GBIT:
+		return 4;
+	case BXT_DRAM_SIZE_6GBIT:
+		return 6;
+	case BXT_DRAM_SIZE_8GBIT:
+		return 8;
+	case BXT_DRAM_SIZE_12GBIT:
+		return 12;
+	case BXT_DRAM_SIZE_16GBIT:
+		return 16;
+	default:
+		MISSING_CASE(val);
+		return 0;
+	}
+}
+
+static int bxt_get_dimm_width(u32 val)
+{
+	if (!bxt_get_dimm_size(val))
+		return 0;
+
+	val = (val & BXT_DRAM_WIDTH_MASK) >> BXT_DRAM_WIDTH_SHIFT;
+
+	return 8 << val;
+}
+
+static int bxt_get_dimm_ranks(u32 val)
+{
+	if (!bxt_get_dimm_size(val))
+		return 0;
+
+	switch (val & BXT_DRAM_RANK_MASK) {
+	case BXT_DRAM_RANK_SINGLE:
+		return 1;
+	case BXT_DRAM_RANK_DUAL:
+		return 2;
+	default:
+		MISSING_CASE(val);
+		return 0;
+	}
+}
+
+static enum intel_dram_type bxt_get_dimm_type(u32 val)
+{
+	if (!bxt_get_dimm_size(val))
+		return INTEL_DRAM_UNKNOWN;
+
+	switch (val & BXT_DRAM_TYPE_MASK) {
+	case BXT_DRAM_TYPE_DDR3:
+		return INTEL_DRAM_DDR3;
+	case BXT_DRAM_TYPE_LPDDR3:
+		return INTEL_DRAM_LPDDR3;
+	case BXT_DRAM_TYPE_DDR4:
+		return INTEL_DRAM_DDR4;
+	case BXT_DRAM_TYPE_LPDDR4:
+		return INTEL_DRAM_LPDDR4;
+	default:
+		MISSING_CASE(val);
+		return INTEL_DRAM_UNKNOWN;
+	}
+}
+
+static void bxt_get_dimm_info(struct dram_dimm_info *dimm,
+			      u32 val)
+{
+	dimm->width = bxt_get_dimm_width(val);
+	dimm->ranks = bxt_get_dimm_ranks(val);
+
+	/*
+	 * Size in register is Gb per DRAM device. Convert to total
+	 * GB to match the way we report this for non-LP platforms.
+	 */
+	dimm->size = bxt_get_dimm_size(val) * intel_dimm_num_devices(dimm) / 8;
+}
+
 static int
 bxt_get_dram_info(struct drm_i915_private *dev_priv)
 {
@@ -1228,57 +1375,44 @@ bxt_get_dram_info(struct drm_i915_private *dev_priv)
 	 * Now read each DUNIT8/9/10/11 to check the rank of each dimms.
 	 */
 	for (i = BXT_D_CR_DRP0_DUNIT_START; i <= BXT_D_CR_DRP0_DUNIT_END; i++) {
-		u8 size, width;
-		enum dram_rank rank;
-		u32 tmp;
+		struct dram_dimm_info dimm;
+		enum intel_dram_type type;
 
 		val = I915_READ(BXT_D_CR_DRP0_DUNIT(i));
 		if (val == 0xFFFFFFFF)
 			continue;
 
 		dram_info->num_channels++;
-		tmp = val & BXT_DRAM_RANK_MASK;
-
-		if (tmp == BXT_DRAM_RANK_SINGLE)
-			rank = I915_DRAM_RANK_SINGLE;
-		else if (tmp == BXT_DRAM_RANK_DUAL)
-			rank = I915_DRAM_RANK_DUAL;
-		else
-			rank = I915_DRAM_RANK_INVALID;
-
-		tmp = val & BXT_DRAM_SIZE_MASK;
-		if (tmp == BXT_DRAM_SIZE_4GB)
-			size = 4;
-		else if (tmp == BXT_DRAM_SIZE_6GB)
-			size = 6;
-		else if (tmp == BXT_DRAM_SIZE_8GB)
-			size = 8;
-		else if (tmp == BXT_DRAM_SIZE_12GB)
-			size = 12;
-		else if (tmp == BXT_DRAM_SIZE_16GB)
-			size = 16;
-		else
-			size = 0;
-
-		tmp = (val & BXT_DRAM_WIDTH_MASK) >> BXT_DRAM_WIDTH_SHIFT;
-		width = (1 << tmp) * 8;
-		DRM_DEBUG_KMS("dram size:%dGB width:X%d rank:%s\n", size,
-			      width, rank == I915_DRAM_RANK_SINGLE ? "single" :
-			      rank == I915_DRAM_RANK_DUAL ? "dual" : "unknown");
+
+		bxt_get_dimm_info(&dimm, val);
+		type = bxt_get_dimm_type(val);
+
+		WARN_ON(type != INTEL_DRAM_UNKNOWN &&
+			dram_info->type != INTEL_DRAM_UNKNOWN &&
+			dram_info->type != type);
+
+		DRM_DEBUG_KMS("CH%u DIMM size: %u GB, width: X%u, ranks: %u, type: %s\n",
+			      i - BXT_D_CR_DRP0_DUNIT_START,
+			      dimm.size, dimm.width, dimm.ranks,
+			      intel_dram_type_str(type));
 
 		/*
 		 * If any of the channel is single rank channel,
 		 * worst case output will be same as if single rank
 		 * memory, so consider single rank memory.
 		 */
-		if (dram_info->rank == I915_DRAM_RANK_INVALID)
-			dram_info->rank = rank;
-		else if (rank == I915_DRAM_RANK_SINGLE)
-			dram_info->rank = I915_DRAM_RANK_SINGLE;
+		if (dram_info->ranks == 0)
+			dram_info->ranks = dimm.ranks;
+		else if (dimm.ranks == 1)
+			dram_info->ranks = 1;
+
+		if (type != INTEL_DRAM_UNKNOWN)
+			dram_info->type = type;
 	}
 
-	if (dram_info->rank == I915_DRAM_RANK_INVALID) {
-		DRM_INFO("couldn't get memory rank information\n");
+	if (dram_info->type == INTEL_DRAM_UNKNOWN ||
+	    dram_info->ranks == 0) {
+		DRM_INFO("couldn't get memory information\n");
 		return -EINVAL;
 	}
 
@@ -1290,14 +1424,8 @@ static void
 intel_get_dram_info(struct drm_i915_private *dev_priv)
 {
 	struct dram_info *dram_info = &dev_priv->dram_info;
-	char bandwidth_str[32];
 	int ret;
 
-	dram_info->valid = false;
-	dram_info->rank = I915_DRAM_RANK_INVALID;
-	dram_info->bandwidth_kbps = 0;
-	dram_info->num_channels = 0;
-
 	/*
 	 * Assume 16Gb DIMMs are present until proven otherwise.
 	 * This is only used for the level 0 watermark latency
@@ -1305,28 +1433,61 @@ intel_get_dram_info(struct drm_i915_private *dev_priv)
 	 */
 	dram_info->is_16gb_dimm = !IS_GEN9_LP(dev_priv);
 
-	if (INTEL_GEN(dev_priv) < 9 || IS_GEMINILAKE(dev_priv))
+	if (INTEL_GEN(dev_priv) < 9)
 		return;
 
-	/* Need to calculate bandwidth only for Gen9 */
-	if (IS_BROXTON(dev_priv))
+	if (IS_GEN9_LP(dev_priv))
 		ret = bxt_get_dram_info(dev_priv);
-	else if (IS_GEN(dev_priv, 9))
-		ret = skl_get_dram_info(dev_priv);
 	else
-		ret = skl_dram_get_channels_info(dev_priv);
+		ret = skl_get_dram_info(dev_priv);
 	if (ret)
 		return;
 
-	if (dram_info->bandwidth_kbps)
-		sprintf(bandwidth_str, "%d KBps", dram_info->bandwidth_kbps);
+	DRM_DEBUG_KMS("DRAM bandwidth: %u kBps, channels: %u\n",
+		      dram_info->bandwidth_kbps,
+		      dram_info->num_channels);
+
+	DRM_DEBUG_KMS("DRAM ranks: %u, 16Gb DIMMs: %s\n",
+		      dram_info->ranks, yesno(dram_info->is_16gb_dimm));
+}
+
+static u32 gen9_edram_size_mb(struct drm_i915_private *dev_priv, u32 cap)
+{
+	const unsigned int ways[8] = { 4, 8, 12, 16, 16, 16, 16, 16 };
+	const unsigned int sets[4] = { 1, 1, 2, 2 };
+
+	return EDRAM_NUM_BANKS(cap) *
+		ways[EDRAM_WAYS_IDX(cap)] *
+		sets[EDRAM_SETS_IDX(cap)];
+}
+
+static void edram_detect(struct drm_i915_private *dev_priv)
+{
+	u32 edram_cap = 0;
+
+	if (!(IS_HASWELL(dev_priv) ||
+	      IS_BROADWELL(dev_priv) ||
+	      INTEL_GEN(dev_priv) >= 9))
+		return;
+
+	edram_cap = __raw_uncore_read32(&dev_priv->uncore, HSW_EDRAM_CAP);
+
+	/* NB: We can't write IDICR yet because we don't have gt funcs set up */
+
+	if (!(edram_cap & EDRAM_ENABLED))
+		return;
+
+	/*
+	 * The needed capability bits for size calculation are not there with
+	 * pre gen9 so return 128MB always.
+	 */
+	if (INTEL_GEN(dev_priv) < 9)
+		dev_priv->edram_size_mb = 128;
 	else
-		sprintf(bandwidth_str, "unknown");
-	DRM_DEBUG_KMS("DRAM bandwidth:%s, total-channels: %u\n",
-		      bandwidth_str, dram_info->num_channels);
-	DRM_DEBUG_KMS("DRAM rank: %s rank 16GB-dimm:%s\n",
-		      (dram_info->rank == I915_DRAM_RANK_DUAL) ?
-		      "dual" : "single", yesno(dram_info->is_16gb_dimm));
+		dev_priv->edram_size_mb =
+			gen9_edram_size_mb(dev_priv, edram_cap);
+
+	DRM_INFO("Found %uMB of eDRAM\n", dev_priv->edram_size_mb);
 }
 
 /**
@@ -1348,7 +1509,7 @@ static int i915_driver_init_hw(struct drm_i915_private *dev_priv)
 
 	if (HAS_PPGTT(dev_priv)) {
 		if (intel_vgpu_active(dev_priv) &&
-		    !intel_vgpu_has_full_48bit_ppgtt(dev_priv)) {
+		    !intel_vgpu_has_full_ppgtt(dev_priv)) {
 			i915_report_error(dev_priv,
 					  "incompatible vGPU found, support for isolated ppGTT required\n");
 			return -ENXIO;
@@ -1371,6 +1532,9 @@ static int i915_driver_init_hw(struct drm_i915_private *dev_priv)
 
 	intel_sanitize_options(dev_priv);
 
+	/* needs to be done before ggtt probe */
+	edram_detect(dev_priv);
+
 	i915_perf_init(dev_priv);
 
 	ret = i915_ggtt_probe_hw(dev_priv);
@@ -1606,10 +1770,12 @@ static void i915_welcome_messages(struct drm_i915_private *dev_priv)
 	if (drm_debug & DRM_UT_DRIVER) {
 		struct drm_printer p = drm_debug_printer("i915 device info:");
 
-		drm_printf(&p, "pciid=0x%04x rev=0x%02x platform=%s gen=%i\n",
+		drm_printf(&p, "pciid=0x%04x rev=0x%02x platform=%s (subplatform=0x%x) gen=%i\n",
 			   INTEL_DEVID(dev_priv),
 			   INTEL_REVID(dev_priv),
 			   intel_platform_name(INTEL_INFO(dev_priv)->platform),
+			   intel_subplatform(RUNTIME_INFO(dev_priv),
+					     INTEL_INFO(dev_priv)->platform),
 			   INTEL_GEN(dev_priv));
 
 		intel_device_info_dump_flags(INTEL_INFO(dev_priv), &p);
@@ -1652,8 +1818,6 @@ i915_driver_create(struct pci_dev *pdev, const struct pci_device_id *ent)
 	memcpy(device_info, match_info, sizeof(*device_info));
 	RUNTIME_INFO(i915)->device_id = pdev->device;
 
-	BUILD_BUG_ON(INTEL_MAX_PLATFORMS >
-		     BITS_PER_TYPE(device_info->platform_mask));
 	BUG_ON(device_info->gen > BITS_PER_TYPE(device_info->gen_mask));
 
 	return i915;
@@ -1750,11 +1914,17 @@ void i915_driver_unload(struct drm_device *dev)
 
 	i915_driver_unregister(dev_priv);
 
+	/*
+	 * After unregistering the device to prevent any new users, cancel
+	 * all in-flight requests so that we can quickly unbind the active
+	 * resources.
+	 */
+	i915_gem_set_wedged(dev_priv);
+
 	/* Flush any external code that still may be under the RCU lock */
 	synchronize_rcu();
 
-	if (i915_gem_suspend(dev_priv))
-		DRM_ERROR("failed to idle hardware; continuing to unload!\n");
+	i915_gem_suspend(dev_priv);
 
 	drm_atomic_helper_shutdown(dev);
 
@@ -1862,7 +2032,6 @@ static bool suspend_to_idle(struct drm_i915_private *dev_priv)
 static int i915_drm_prepare(struct drm_device *dev)
 {
 	struct drm_i915_private *i915 = to_i915(dev);
-	int err;
 
 	/*
 	 * NB intel_display_suspend() may issue new requests after we've
@@ -1870,12 +2039,9 @@ static int i915_drm_prepare(struct drm_device *dev)
 	 * split out that work and pull it forward so that after point,
 	 * the GPU is not woken again.
 	 */
-	err = i915_gem_suspend(i915);
-	if (err)
-		dev_err(&i915->drm.pdev->dev,
-			"GEM idle failed, suspend/resume might fail\n");
+	i915_gem_suspend(i915);
 
-	return err;
+	return 0;
 }
 
 static int i915_drm_suspend(struct drm_device *dev)
@@ -1945,7 +2111,7 @@ static int i915_drm_suspend_late(struct drm_device *dev, bool hibernation)
 
 	i915_gem_suspend_late(dev_priv);
 
-	intel_uncore_suspend(dev_priv);
+	intel_uncore_suspend(&dev_priv->uncore);
 
 	intel_power_domains_suspend(dev_priv,
 				    get_suspend_mode(dev_priv, hibernation));
@@ -2141,7 +2307,9 @@ static int i915_drm_resume_early(struct drm_device *dev)
 		DRM_ERROR("Resume prepare failed: %d, continuing anyway\n",
 			  ret);
 
-	intel_uncore_resume_early(dev_priv);
+	intel_uncore_resume_early(&dev_priv->uncore);
+
+	i915_check_and_clear_faults(dev_priv);
 
 	if (INTEL_GEN(dev_priv) >= 11 || IS_GEN9_LP(dev_priv)) {
 		gen9_sanitize_dc_state(dev_priv);
@@ -2545,7 +2713,7 @@ int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool force_on)
 	if (!force_on)
 		return 0;
 
-	err = intel_wait_for_register(dev_priv,
+	err = intel_wait_for_register(&dev_priv->uncore,
 				      VLV_GTLC_SURVIVABILITY_REG,
 				      VLV_GFX_CLK_STATUS_BIT,
 				      VLV_GFX_CLK_STATUS_BIT,
@@ -2711,7 +2879,7 @@ static int intel_runtime_suspend(struct device *kdev)
 
 	intel_runtime_pm_disable_interrupts(dev_priv);
 
-	intel_uncore_suspend(dev_priv);
+	intel_uncore_suspend(&dev_priv->uncore);
 
 	ret = 0;
 	if (INTEL_GEN(dev_priv) >= 11) {
@@ -2728,7 +2896,7 @@ static int intel_runtime_suspend(struct device *kdev)
 
 	if (ret) {
 		DRM_ERROR("Runtime suspend failed, disabling it (%d)\n", ret);
-		intel_uncore_runtime_resume(dev_priv);
+		intel_uncore_runtime_resume(&dev_priv->uncore);
 
 		intel_runtime_pm_enable_interrupts(dev_priv);
 
@@ -2745,7 +2913,7 @@ static int intel_runtime_suspend(struct device *kdev)
 	enable_rpm_wakeref_asserts(dev_priv);
 	intel_runtime_pm_cleanup(dev_priv);
 
-	if (intel_uncore_arm_unclaimed_mmio_detection(dev_priv))
+	if (intel_uncore_arm_unclaimed_mmio_detection(&dev_priv->uncore))
 		DRM_ERROR("Unclaimed access detected prior to suspending\n");
 
 	dev_priv->runtime_pm.suspended = true;
@@ -2773,7 +2941,7 @@ static int intel_runtime_suspend(struct device *kdev)
 		intel_opregion_notify_adapter(dev_priv, PCI_D1);
 	}
 
-	assert_forcewakes_inactive(dev_priv);
+	assert_forcewakes_inactive(&dev_priv->uncore);
 
 	if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
 		intel_hpd_poll_init(dev_priv);
@@ -2799,7 +2967,7 @@ static int intel_runtime_resume(struct device *kdev)
 
 	intel_opregion_notify_adapter(dev_priv, PCI_D0);
 	dev_priv->runtime_pm.suspended = false;
-	if (intel_uncore_unclaimed_mmio(dev_priv))
+	if (intel_uncore_unclaimed_mmio(&dev_priv->uncore))
 		DRM_DEBUG_DRIVER("Unclaimed access during suspend, bios?\n");
 
 	if (INTEL_GEN(dev_priv) >= 11) {
@@ -2825,7 +2993,7 @@ static int intel_runtime_resume(struct device *kdev)
 		ret = vlv_resume_prepare(dev_priv, true);
 	}
 
-	intel_uncore_runtime_resume(dev_priv);
+	intel_uncore_runtime_resume(&dev_priv->uncore);
 
 	intel_runtime_pm_enable_interrupts(dev_priv);
 
@@ -2969,7 +3137,7 @@ static const struct drm_ioctl_desc i915_ioctls[] = {
 	DRM_IOCTL_DEF_DRV(I915_SET_SPRITE_COLORKEY, intel_sprite_set_colorkey_ioctl, DRM_MASTER),
 	DRM_IOCTL_DEF_DRV(I915_GET_SPRITE_COLORKEY, drm_noop, DRM_MASTER),
 	DRM_IOCTL_DEF_DRV(I915_GEM_WAIT, i915_gem_wait_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
-	DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_CREATE, i915_gem_context_create_ioctl, DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_CREATE_EXT, i915_gem_context_create_ioctl, DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_DESTROY, i915_gem_context_destroy_ioctl, DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF_DRV(I915_REG_READ, i915_reg_read_ioctl, DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF_DRV(I915_GET_RESET_STATS, i915_gem_context_reset_stats_ioctl, DRM_RENDER_ALLOW),
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index a67a63b5aa84..066fd2a12851 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -55,6 +55,7 @@
 #include <drm/drm_util.h>
 #include <drm/drm_dsc.h>
 #include <drm/drm_connector.h>
+#include <drm/i915_mei_hdcp_interface.h>
 
 #include "i915_fixed.h"
 #include "i915_params.h"
@@ -65,13 +66,14 @@
 #include "intel_device_info.h"
 #include "intel_display.h"
 #include "intel_dpll_mgr.h"
+#include "intel_frontbuffer.h"
 #include "intel_lrc.h"
 #include "intel_opregion.h"
 #include "intel_ringbuffer.h"
+#include "intel_uc.h"
 #include "intel_uncore.h"
 #include "intel_wopcm.h"
 #include "intel_workarounds.h"
-#include "intel_uc.h"
 
 #include "i915_gem.h"
 #include "i915_gem_context.h"
@@ -91,8 +93,8 @@
 
 #define DRIVER_NAME		"i915"
 #define DRIVER_DESC		"Intel Graphics"
-#define DRIVER_DATE		"20190207"
-#define DRIVER_TIMESTAMP	1549572331
+#define DRIVER_DATE		"20190417"
+#define DRIVER_TIMESTAMP	1555492067
 
 /* Use I915_STATE_WARN(x) and I915_STATE_WARN_ON() (rather than WARN() and
  * WARN_ON()) for hw state sanity checks to check for unexpected conditions
@@ -215,11 +217,12 @@ struct drm_i915_file_private {
  */
 #define DRM_I915_THROTTLE_JIFFIES msecs_to_jiffies(20)
 	} mm;
+
 	struct idr context_idr;
+	struct mutex context_idr_lock; /* guards context_idr */
 
-	struct intel_rps_client {
-		atomic_t boosts;
-	} rps_client;
+	struct idr vm_idr;
+	struct mutex vm_idr_lock; /* guards vm_idr */
 
 	unsigned int bsd_engine;
 
@@ -280,7 +283,8 @@ struct drm_i915_display_funcs {
 	void (*get_cdclk)(struct drm_i915_private *dev_priv,
 			  struct intel_cdclk_state *cdclk_state);
 	void (*set_cdclk)(struct drm_i915_private *dev_priv,
-			  const struct intel_cdclk_state *cdclk_state);
+			  const struct intel_cdclk_state *cdclk_state,
+			  enum pipe pipe);
 	int (*get_fifo_size)(struct drm_i915_private *dev_priv,
 			     enum i9xx_plane_id i9xx_plane);
 	int (*compute_pipe_wm)(struct intel_crtc_state *cstate);
@@ -323,6 +327,7 @@ struct drm_i915_display_funcs {
 	/* display clock increase/decrease */
 	/* pll clock increase/decrease */
 
+	int (*color_check)(struct intel_crtc_state *crtc_state);
 	/*
 	 * Program double buffered color management registers during
 	 * vblank evasion. The registers should then latch during the
@@ -371,14 +376,6 @@ enum i915_cache_level {
 
 #define I915_COLOR_UNEVICTABLE (-1) /* a non-vma sharing the address space */
 
-enum fb_op_origin {
-	ORIGIN_GTT,
-	ORIGIN_CPU,
-	ORIGIN_CS,
-	ORIGIN_FLIP,
-	ORIGIN_DIRTYFB,
-};
-
 struct intel_fbc {
 	/* This is always the inner lock when overlapping with struct_mutex and
 	 * it's the outer lock when overlapping with stolen_lock. */
@@ -508,7 +505,7 @@ struct i915_psr {
 
 	u32 debug;
 	bool sink_support;
-	bool prepared, enabled;
+	bool enabled;
 	struct intel_dp *dp;
 	enum pipe pipe;
 	bool active;
@@ -526,16 +523,22 @@ struct i915_psr {
 	u16 su_x_granularity;
 };
 
+/*
+ * Sorted by south display engine compatibility.
+ * If the new PCH comes with a south display engine that is not
+ * inherited from the latest item, please do not add it to the
+ * end. Instead, add it right after its "parent" PCH.
+ */
 enum intel_pch {
+	PCH_NOP = -1,	/* PCH without south display */
 	PCH_NONE = 0,	/* No PCH present */
 	PCH_IBX,	/* Ibexpeak PCH */
 	PCH_CPT,	/* Cougarpoint/Pantherpoint PCH */
 	PCH_LPT,	/* Lynxpoint/Wildcatpoint PCH */
 	PCH_SPT,        /* Sunrisepoint PCH */
 	PCH_KBP,        /* Kaby Lake PCH */
-	PCH_CNP,        /* Cannon Lake PCH */
+	PCH_CNP,        /* Cannon/Comet Lake PCH */
 	PCH_ICP,	/* Ice Lake PCH */
-	PCH_NOP,	/* PCH without south display */
 };
 
 enum intel_sbi_destination {
@@ -949,6 +952,7 @@ struct ddi_vbt_port_info {
 #define HDMI_LEVEL_SHIFT_UNKNOWN	0xff
 	u8 hdmi_level_shift;
 
+	u8 present:1;
 	u8 supports_dvi:1;
 	u8 supports_hdmi:1;
 	u8 supports_dp:1;
@@ -1009,6 +1013,7 @@ struct intel_vbt_data {
 		enum psr_lines_to_wait lines_to_wait;
 		int tp1_wakeup_time_us;
 		int tp2_tp3_wakeup_time_us;
+		int psr2_tp2_tp3_wakeup_time_us;
 	} psr;
 
 	struct {
@@ -1130,6 +1135,7 @@ struct skl_wm_level {
 	u16 plane_res_b;
 	u8 plane_res_l;
 	bool plane_en;
+	bool ignore_lines;
 };
 
 /* Stores plane specific WM parameters */
@@ -1200,7 +1206,11 @@ enum intel_pipe_crc_source {
 	INTEL_PIPE_CRC_SOURCE_NONE,
 	INTEL_PIPE_CRC_SOURCE_PLANE1,
 	INTEL_PIPE_CRC_SOURCE_PLANE2,
-	INTEL_PIPE_CRC_SOURCE_PF,
+	INTEL_PIPE_CRC_SOURCE_PLANE3,
+	INTEL_PIPE_CRC_SOURCE_PLANE4,
+	INTEL_PIPE_CRC_SOURCE_PLANE5,
+	INTEL_PIPE_CRC_SOURCE_PLANE6,
+	INTEL_PIPE_CRC_SOURCE_PLANE7,
 	INTEL_PIPE_CRC_SOURCE_PIPE,
 	/* TV/DP on pre-gen5/vlv can't use the pipe source. */
 	INTEL_PIPE_CRC_SOURCE_TV,
@@ -1468,13 +1478,6 @@ struct intel_cdclk_state {
 struct drm_i915_private {
 	struct drm_device drm;
 
-	struct kmem_cache *objects;
-	struct kmem_cache *vmas;
-	struct kmem_cache *luts;
-	struct kmem_cache *requests;
-	struct kmem_cache *dependencies;
-	struct kmem_cache *priorities;
-
 	const struct intel_device_info __info; /* Use INTEL_INFO() to access. */
 	struct intel_runtime_info __runtime; /* Use RUNTIME_INFO() to access. */
 	struct intel_driver_caps caps;
@@ -1503,8 +1506,6 @@ struct drm_i915_private {
 	 */
 	resource_size_t stolen_usable_size;	/* Total size minus reserved ranges */
 
-	void __iomem *regs;
-
 	struct intel_uncore uncore;
 
 	struct i915_virtual_gpu vgpu;
@@ -1622,6 +1623,8 @@ struct drm_i915_private {
 		struct intel_cdclk_state actual;
 		/* The current hardware cdclk state */
 		struct intel_cdclk_state hw;
+
+		int force_min_cdclk;
 	} cdclk;
 
 	/**
@@ -1700,8 +1703,11 @@ struct drm_i915_private {
 
 	struct intel_l3_parity l3_parity;
 
-	/* Cannot be determined by PCIID. You must always read a register. */
-	u32 edram_cap;
+	/*
+	 * edram size in MB.
+	 * Cannot be determined by PCIID. You must always read a register.
+	 */
+	u32 edram_size_mb;
 
 	/*
 	 * Protects RPS/RC6 register access and PCU communication.
@@ -1741,6 +1747,7 @@ struct drm_i915_private {
 	 *
 	 */
 	struct mutex av_mutex;
+	int audio_power_refcount;
 
 	struct {
 		struct mutex mutex;
@@ -1831,13 +1838,16 @@ struct drm_i915_private {
 		bool valid;
 		bool is_16gb_dimm;
 		u8 num_channels;
-		enum dram_rank {
-			I915_DRAM_RANK_INVALID = 0,
-			I915_DRAM_RANK_SINGLE,
-			I915_DRAM_RANK_DUAL
-		} rank;
+		u8 ranks;
 		u32 bandwidth_kbps;
 		bool symmetric_memory;
+		enum intel_dram_type {
+			INTEL_DRAM_UNKNOWN,
+			INTEL_DRAM_DDR3,
+			INTEL_DRAM_DDR4,
+			INTEL_DRAM_LPDDR3,
+			INTEL_DRAM_LPDDR4
+		} type;
 	} dram_info;
 
 	struct i915_runtime_pm runtime_pm;
@@ -1985,7 +1995,6 @@ struct drm_i915_private {
 
 	/* Abstract the submission mechanism (legacy ringbuffer or execlists) away */
 	struct {
-		void (*resume)(struct drm_i915_private *);
 		void (*cleanup_engine)(struct intel_engine_cs *engine);
 
 		struct i915_gt_timelines {
@@ -1997,6 +2006,7 @@ struct drm_i915_private {
 			struct list_head hwsp_free_list;
 		} timelines;
 
+		intel_engine_mask_t active_engines;
 		struct list_head active_rings;
 		struct list_head closed_vma;
 		u32 active_requests;
@@ -2011,12 +2021,6 @@ struct drm_i915_private {
 		intel_wakeref_t awake;
 
 		/**
-		 * The number of times we have woken up.
-		 */
-		unsigned int epoch;
-#define I915_EPOCH_INVALID 0
-
-		/**
 		 * We leave the user IRQ off as much as possible,
 		 * but this means that requests will finish and never
 		 * be retired once the system goes idle. Set a timer to
@@ -2039,6 +2043,14 @@ struct drm_i915_private {
 		struct i915_vma *scratch;
 	} gt;
 
+	/* For i945gm vblank irq vs. C3 workaround */
+	struct {
+		struct work_struct work;
+		struct pm_qos_request pm_qos;
+		u8 c3_disable_latency;
+		u8 enabled;
+	} i945gm_vblank;
+
 	/* perform PHY state sanity checks? */
 	bool chv_phy_assert[2];
 
@@ -2055,18 +2067,25 @@ struct drm_i915_private {
 
 	struct i915_pmu pmu;
 
+	struct i915_hdcp_comp_master *hdcp_master;
+	bool hdcp_comp_added;
+
+	/* Mutex to protect the above hdcp component related values. */
+	struct mutex hdcp_comp_mutex;
+
 	/*
 	 * NOTE: This is the dri1/ums dungeon, don't add stuff here. Your patch
 	 * will be rejected. Instead look for a better place.
 	 */
 };
 
+struct dram_dimm_info {
+	u8 size, width, ranks;
+};
+
 struct dram_channel_info {
-	struct info {
-		u8 size, width;
-		enum dram_rank rank;
-	} l_info, s_info;
-	enum dram_rank rank;
+	struct dram_dimm_info dimm_l, dimm_s;
+	u8 ranks;
 	bool is_16gb_dimm;
 };
 
@@ -2095,6 +2114,11 @@ static inline struct drm_i915_private *huc_to_i915(struct intel_huc *huc)
 	return container_of(huc, struct drm_i915_private, huc);
 }
 
+static inline struct drm_i915_private *uncore_to_i915(struct intel_uncore *uncore)
+{
+	return container_of(uncore, struct drm_i915_private, uncore);
+}
+
 /* Simple iterator over all initialised engines */
 #define for_each_engine(engine__, dev_priv__, id__) \
 	for ((id__) = 0; \
@@ -2104,7 +2128,7 @@ static inline struct drm_i915_private *huc_to_i915(struct intel_huc *huc)
 
 /* Iterator over subset of engines selected by mask */
 #define for_each_engine_masked(engine__, dev_priv__, mask__, tmp__) \
-	for ((tmp__) = (mask__) & INTEL_INFO(dev_priv__)->ring_mask; \
+	for ((tmp__) = (mask__) & INTEL_INFO(dev_priv__)->engine_mask; \
 	     (tmp__) ? \
 	     ((engine__) = (dev_priv__)->engine[__mask_next_bit(tmp__)]), 1 : \
 	     0;)
@@ -2274,7 +2298,69 @@ static inline unsigned int i915_sg_segment_size(void)
 #define IS_REVID(p, since, until) \
 	(INTEL_REVID(p) >= (since) && INTEL_REVID(p) <= (until))
 
-#define IS_PLATFORM(dev_priv, p) (INTEL_INFO(dev_priv)->platform_mask & BIT(p))
+static __always_inline unsigned int
+__platform_mask_index(const struct intel_runtime_info *info,
+		      enum intel_platform p)
+{
+	const unsigned int pbits =
+		BITS_PER_TYPE(info->platform_mask[0]) - INTEL_SUBPLATFORM_BITS;
+
+	/* Expand the platform_mask array if this fails. */
+	BUILD_BUG_ON(INTEL_MAX_PLATFORMS >
+		     pbits * ARRAY_SIZE(info->platform_mask));
+
+	return p / pbits;
+}
+
+static __always_inline unsigned int
+__platform_mask_bit(const struct intel_runtime_info *info,
+		    enum intel_platform p)
+{
+	const unsigned int pbits =
+		BITS_PER_TYPE(info->platform_mask[0]) - INTEL_SUBPLATFORM_BITS;
+
+	return p % pbits + INTEL_SUBPLATFORM_BITS;
+}
+
+static inline u32
+intel_subplatform(const struct intel_runtime_info *info, enum intel_platform p)
+{
+	const unsigned int pi = __platform_mask_index(info, p);
+
+	return info->platform_mask[pi] & INTEL_SUBPLATFORM_BITS;
+}
+
+static __always_inline bool
+IS_PLATFORM(const struct drm_i915_private *i915, enum intel_platform p)
+{
+	const struct intel_runtime_info *info = RUNTIME_INFO(i915);
+	const unsigned int pi = __platform_mask_index(info, p);
+	const unsigned int pb = __platform_mask_bit(info, p);
+
+	BUILD_BUG_ON(!__builtin_constant_p(p));
+
+	return info->platform_mask[pi] & BIT(pb);
+}
+
+static __always_inline bool
+IS_SUBPLATFORM(const struct drm_i915_private *i915,
+	       enum intel_platform p, unsigned int s)
+{
+	const struct intel_runtime_info *info = RUNTIME_INFO(i915);
+	const unsigned int pi = __platform_mask_index(info, p);
+	const unsigned int pb = __platform_mask_bit(info, p);
+	const unsigned int msb = BITS_PER_TYPE(info->platform_mask[0]) - 1;
+	const u32 mask = info->platform_mask[pi];
+
+	BUILD_BUG_ON(!__builtin_constant_p(p));
+	BUILD_BUG_ON(!__builtin_constant_p(s));
+	BUILD_BUG_ON((s) >= INTEL_SUBPLATFORM_BITS);
+
+	/* Shift and test on the MSB position so sign flag can be used. */
+	return ((mask << (msb - pb)) & (mask << (msb - s))) & BIT(msb);
+}
+
+#define IS_MOBILE(dev_priv)	(INTEL_INFO(dev_priv)->is_mobile)
 
 #define IS_I830(dev_priv)	IS_PLATFORM(dev_priv, INTEL_I830)
 #define IS_I845G(dev_priv)	IS_PLATFORM(dev_priv, INTEL_I845G)
@@ -2289,11 +2375,11 @@ static inline unsigned int i915_sg_segment_size(void)
 #define IS_G45(dev_priv)	IS_PLATFORM(dev_priv, INTEL_G45)
 #define IS_GM45(dev_priv)	IS_PLATFORM(dev_priv, INTEL_GM45)
 #define IS_G4X(dev_priv)	(IS_G45(dev_priv) || IS_GM45(dev_priv))
-#define IS_PINEVIEW_G(dev_priv)	(INTEL_DEVID(dev_priv) == 0xa001)
-#define IS_PINEVIEW_M(dev_priv)	(INTEL_DEVID(dev_priv) == 0xa011)
 #define IS_PINEVIEW(dev_priv)	IS_PLATFORM(dev_priv, INTEL_PINEVIEW)
 #define IS_G33(dev_priv)	IS_PLATFORM(dev_priv, INTEL_G33)
-#define IS_IRONLAKE_M(dev_priv)	(INTEL_DEVID(dev_priv) == 0x0046)
+#define IS_IRONLAKE(dev_priv)	IS_PLATFORM(dev_priv, INTEL_IRONLAKE)
+#define IS_IRONLAKE_M(dev_priv) \
+	(IS_PLATFORM(dev_priv, INTEL_IRONLAKE) && IS_MOBILE(dev_priv))
 #define IS_IVYBRIDGE(dev_priv)	IS_PLATFORM(dev_priv, INTEL_IVYBRIDGE)
 #define IS_IVB_GT1(dev_priv)	(IS_IVYBRIDGE(dev_priv) && \
 				 INTEL_INFO(dev_priv)->gt == 1)
@@ -2308,46 +2394,35 @@ static inline unsigned int i915_sg_segment_size(void)
 #define IS_COFFEELAKE(dev_priv)	IS_PLATFORM(dev_priv, INTEL_COFFEELAKE)
 #define IS_CANNONLAKE(dev_priv)	IS_PLATFORM(dev_priv, INTEL_CANNONLAKE)
 #define IS_ICELAKE(dev_priv)	IS_PLATFORM(dev_priv, INTEL_ICELAKE)
-#define IS_MOBILE(dev_priv)	(INTEL_INFO(dev_priv)->is_mobile)
+#define IS_ELKHARTLAKE(dev_priv)	IS_PLATFORM(dev_priv, INTEL_ELKHARTLAKE)
 #define IS_HSW_EARLY_SDV(dev_priv) (IS_HASWELL(dev_priv) && \
 				    (INTEL_DEVID(dev_priv) & 0xFF00) == 0x0C00)
-#define IS_BDW_ULT(dev_priv)	(IS_BROADWELL(dev_priv) && \
-				 ((INTEL_DEVID(dev_priv) & 0xf) == 0x6 ||	\
-				 (INTEL_DEVID(dev_priv) & 0xf) == 0xb ||	\
-				 (INTEL_DEVID(dev_priv) & 0xf) == 0xe))
-/* ULX machines are also considered ULT. */
-#define IS_BDW_ULX(dev_priv)	(IS_BROADWELL(dev_priv) && \
-				 (INTEL_DEVID(dev_priv) & 0xf) == 0xe)
+#define IS_BDW_ULT(dev_priv) \
+	IS_SUBPLATFORM(dev_priv, INTEL_BROADWELL, INTEL_SUBPLATFORM_ULT)
+#define IS_BDW_ULX(dev_priv) \
+	IS_SUBPLATFORM(dev_priv, INTEL_BROADWELL, INTEL_SUBPLATFORM_ULX)
 #define IS_BDW_GT3(dev_priv)	(IS_BROADWELL(dev_priv) && \
 				 INTEL_INFO(dev_priv)->gt == 3)
-#define IS_HSW_ULT(dev_priv)	(IS_HASWELL(dev_priv) && \
-				 (INTEL_DEVID(dev_priv) & 0xFF00) == 0x0A00)
+#define IS_HSW_ULT(dev_priv) \
+	IS_SUBPLATFORM(dev_priv, INTEL_HASWELL, INTEL_SUBPLATFORM_ULT)
 #define IS_HSW_GT3(dev_priv)	(IS_HASWELL(dev_priv) && \
 				 INTEL_INFO(dev_priv)->gt == 3)
 #define IS_HSW_GT1(dev_priv)	(IS_HASWELL(dev_priv) && \
 				 INTEL_INFO(dev_priv)->gt == 1)
 /* ULX machines are also considered ULT. */
-#define IS_HSW_ULX(dev_priv)	(INTEL_DEVID(dev_priv) == 0x0A0E || \
-				 INTEL_DEVID(dev_priv) == 0x0A1E)
-#define IS_SKL_ULT(dev_priv)	(INTEL_DEVID(dev_priv) == 0x1906 || \
-				 INTEL_DEVID(dev_priv) == 0x1913 || \
-				 INTEL_DEVID(dev_priv) == 0x1916 || \
-				 INTEL_DEVID(dev_priv) == 0x1921 || \
-				 INTEL_DEVID(dev_priv) == 0x1926)
-#define IS_SKL_ULX(dev_priv)	(INTEL_DEVID(dev_priv) == 0x190E || \
-				 INTEL_DEVID(dev_priv) == 0x1915 || \
-				 INTEL_DEVID(dev_priv) == 0x191E)
-#define IS_KBL_ULT(dev_priv)	(INTEL_DEVID(dev_priv) == 0x5906 || \
-				 INTEL_DEVID(dev_priv) == 0x5913 || \
-				 INTEL_DEVID(dev_priv) == 0x5916 || \
-				 INTEL_DEVID(dev_priv) == 0x5921 || \
-				 INTEL_DEVID(dev_priv) == 0x5926)
-#define IS_KBL_ULX(dev_priv)	(INTEL_DEVID(dev_priv) == 0x590E || \
-				 INTEL_DEVID(dev_priv) == 0x5915 || \
-				 INTEL_DEVID(dev_priv) == 0x591E)
-#define IS_AML_ULX(dev_priv)	(INTEL_DEVID(dev_priv) == 0x591C || \
-				 INTEL_DEVID(dev_priv) == 0x87C0 || \
-				 INTEL_DEVID(dev_priv) == 0x87CA)
+#define IS_HSW_ULX(dev_priv) \
+	IS_SUBPLATFORM(dev_priv, INTEL_HASWELL, INTEL_SUBPLATFORM_ULX)
+#define IS_SKL_ULT(dev_priv) \
+	IS_SUBPLATFORM(dev_priv, INTEL_SKYLAKE, INTEL_SUBPLATFORM_ULT)
+#define IS_SKL_ULX(dev_priv) \
+	IS_SUBPLATFORM(dev_priv, INTEL_SKYLAKE, INTEL_SUBPLATFORM_ULX)
+#define IS_KBL_ULT(dev_priv) \
+	IS_SUBPLATFORM(dev_priv, INTEL_KABYLAKE, INTEL_SUBPLATFORM_ULT)
+#define IS_KBL_ULX(dev_priv) \
+	IS_SUBPLATFORM(dev_priv, INTEL_KABYLAKE, INTEL_SUBPLATFORM_ULX)
+#define IS_AML_ULX(dev_priv) \
+	(IS_SUBPLATFORM(dev_priv, INTEL_KABYLAKE, INTEL_SUBPLATFORM_AML) || \
+	 IS_SUBPLATFORM(dev_priv, INTEL_COFFEELAKE, INTEL_SUBPLATFORM_AML))
 #define IS_SKL_GT2(dev_priv)	(IS_SKYLAKE(dev_priv) && \
 				 INTEL_INFO(dev_priv)->gt == 2)
 #define IS_SKL_GT3(dev_priv)	(IS_SKYLAKE(dev_priv) && \
@@ -2358,16 +2433,16 @@ static inline unsigned int i915_sg_segment_size(void)
 				 INTEL_INFO(dev_priv)->gt == 2)
 #define IS_KBL_GT3(dev_priv)	(IS_KABYLAKE(dev_priv) && \
 				 INTEL_INFO(dev_priv)->gt == 3)
-#define IS_CFL_ULT(dev_priv)	(IS_COFFEELAKE(dev_priv) && \
-				 (INTEL_DEVID(dev_priv) & 0x00F0) == 0x00A0)
+#define IS_CFL_ULT(dev_priv) \
+	IS_SUBPLATFORM(dev_priv, INTEL_COFFEELAKE, INTEL_SUBPLATFORM_ULT)
 #define IS_CFL_GT2(dev_priv)	(IS_COFFEELAKE(dev_priv) && \
 				 INTEL_INFO(dev_priv)->gt == 2)
 #define IS_CFL_GT3(dev_priv)	(IS_COFFEELAKE(dev_priv) && \
 				 INTEL_INFO(dev_priv)->gt == 3)
-#define IS_CNL_WITH_PORT_F(dev_priv)   (IS_CANNONLAKE(dev_priv) && \
-					(INTEL_DEVID(dev_priv) & 0x0004) == 0x0004)
-#define IS_ICL_WITH_PORT_F(dev_priv)   (IS_ICELAKE(dev_priv) && \
-					INTEL_DEVID(dev_priv) != 0x8A51)
+#define IS_CNL_WITH_PORT_F(dev_priv) \
+	IS_SUBPLATFORM(dev_priv, INTEL_CANNONLAKE, INTEL_SUBPLATFORM_PORTF)
+#define IS_ICL_WITH_PORT_F(dev_priv) \
+	IS_SUBPLATFORM(dev_priv, INTEL_ICELAKE, INTEL_SUBPLATFORM_PORTF)
 
 #define IS_ALPHA_SUPPORT(intel_info) ((intel_info)->is_alpha_support)
 
@@ -2426,28 +2501,22 @@ static inline unsigned int i915_sg_segment_size(void)
 #define IS_GEN9_LP(dev_priv)	(IS_GEN(dev_priv, 9) && IS_LP(dev_priv))
 #define IS_GEN9_BC(dev_priv)	(IS_GEN(dev_priv, 9) && !IS_LP(dev_priv))
 
-#define ENGINE_MASK(id)	BIT(id)
-#define RENDER_RING	ENGINE_MASK(RCS)
-#define BSD_RING	ENGINE_MASK(VCS)
-#define BLT_RING	ENGINE_MASK(BCS)
-#define VEBOX_RING	ENGINE_MASK(VECS)
-#define BSD2_RING	ENGINE_MASK(VCS2)
-#define BSD3_RING	ENGINE_MASK(VCS3)
-#define BSD4_RING	ENGINE_MASK(VCS4)
-#define VEBOX2_RING	ENGINE_MASK(VECS2)
-#define ALL_ENGINES	(~0)
-
-#define HAS_ENGINE(dev_priv, id) \
-	(!!(INTEL_INFO(dev_priv)->ring_mask & ENGINE_MASK(id)))
-
-#define HAS_BSD(dev_priv)	HAS_ENGINE(dev_priv, VCS)
-#define HAS_BSD2(dev_priv)	HAS_ENGINE(dev_priv, VCS2)
-#define HAS_BLT(dev_priv)	HAS_ENGINE(dev_priv, BCS)
-#define HAS_VEBOX(dev_priv)	HAS_ENGINE(dev_priv, VECS)
+#define HAS_ENGINE(dev_priv, id) (INTEL_INFO(dev_priv)->engine_mask & BIT(id))
+
+#define ENGINE_INSTANCES_MASK(dev_priv, first, count) ({		\
+	unsigned int first__ = (first);					\
+	unsigned int count__ = (count);					\
+	(INTEL_INFO(dev_priv)->engine_mask &				\
+	 GENMASK(first__ + count__ - 1, first__)) >> first__;		\
+})
+#define VDBOX_MASK(dev_priv) \
+	ENGINE_INSTANCES_MASK(dev_priv, VCS0, I915_MAX_VCS)
+#define VEBOX_MASK(dev_priv) \
+	ENGINE_INSTANCES_MASK(dev_priv, VECS0, I915_MAX_VECS)
 
 #define HAS_LLC(dev_priv)	(INTEL_INFO(dev_priv)->has_llc)
 #define HAS_SNOOP(dev_priv)	(INTEL_INFO(dev_priv)->has_snoop)
-#define HAS_EDRAM(dev_priv)	(!!((dev_priv)->edram_cap & EDRAM_ENABLED))
+#define HAS_EDRAM(dev_priv)	((dev_priv)->edram_size_mb)
 #define HAS_WT(dev_priv)	((IS_HASWELL(dev_priv) || \
 				 IS_BROADWELL(dev_priv)) && HAS_EDRAM(dev_priv))
 
@@ -2462,13 +2531,11 @@ static inline unsigned int i915_sg_segment_size(void)
 
 #define HAS_EXECLISTS(dev_priv) HAS_LOGICAL_RING_CONTEXTS(dev_priv)
 
-#define INTEL_PPGTT(dev_priv) (INTEL_INFO(dev_priv)->ppgtt)
+#define INTEL_PPGTT(dev_priv) (INTEL_INFO(dev_priv)->ppgtt_type)
 #define HAS_PPGTT(dev_priv) \
 	(INTEL_PPGTT(dev_priv) != INTEL_PPGTT_NONE)
 #define HAS_FULL_PPGTT(dev_priv) \
 	(INTEL_PPGTT(dev_priv) >= INTEL_PPGTT_FULL)
-#define HAS_FULL_48BIT_PPGTT(dev_priv)	\
-	(INTEL_PPGTT(dev_priv) >= INTEL_PPGTT_FULL_4LVL)
 
 #define HAS_PAGE_SIZES(dev_priv, sizes) ({ \
 	GEM_BUG_ON((sizes) == 0); \
@@ -2512,6 +2579,7 @@ static inline unsigned int i915_sg_segment_size(void)
 #define HAS_DDI(dev_priv)		 (INTEL_INFO(dev_priv)->display.has_ddi)
 #define HAS_FPGA_DBG_UNCLAIMED(dev_priv) (INTEL_INFO(dev_priv)->has_fpga_dbg)
 #define HAS_PSR(dev_priv)		 (INTEL_INFO(dev_priv)->display.has_psr)
+#define HAS_TRANSCODER_EDP(dev_priv)	 (INTEL_INFO(dev_priv)->trans_offsets[TRANSCODER_EDP] != 0)
 
 #define HAS_RC6(dev_priv)		 (INTEL_INFO(dev_priv)->has_rc6)
 #define HAS_RC6p(dev_priv)		 (INTEL_INFO(dev_priv)->has_rc6p)
@@ -2558,6 +2626,7 @@ static inline unsigned int i915_sg_segment_size(void)
 #define INTEL_PCH_KBP_DEVICE_ID_TYPE		0xA280
 #define INTEL_PCH_CNP_DEVICE_ID_TYPE		0xA300
 #define INTEL_PCH_CNP_LP_DEVICE_ID_TYPE		0x9D80
+#define INTEL_PCH_CMP_DEVICE_ID_TYPE		0x0280
 #define INTEL_PCH_ICP_DEVICE_ID_TYPE		0x3480
 #define INTEL_PCH_P2X_DEVICE_ID_TYPE		0x7100
 #define INTEL_PCH_P3X_DEVICE_ID_TYPE		0x7000
@@ -2567,8 +2636,6 @@ static inline unsigned int i915_sg_segment_size(void)
 #define INTEL_PCH_ID(dev_priv) ((dev_priv)->pch_id)
 #define HAS_PCH_ICP(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_ICP)
 #define HAS_PCH_CNP(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_CNP)
-#define HAS_PCH_CNP_LP(dev_priv) \
-	(INTEL_PCH_ID(dev_priv) == INTEL_PCH_CNP_LP_DEVICE_ID_TYPE)
 #define HAS_PCH_KBP(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_KBP)
 #define HAS_PCH_SPT(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_SPT)
 #define HAS_PCH_LPT(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_LPT)
@@ -2800,8 +2867,6 @@ void i915_gem_load_init_fences(struct drm_i915_private *dev_priv);
 int i915_gem_freeze(struct drm_i915_private *dev_priv);
 int i915_gem_freeze_late(struct drm_i915_private *dev_priv);
 
-void *i915_gem_object_alloc(struct drm_i915_private *dev_priv);
-void i915_gem_object_free(struct drm_i915_gem_object *obj);
 void i915_gem_object_init(struct drm_i915_gem_object *obj,
 			 const struct drm_i915_gem_object_ops *ops);
 struct drm_i915_gem_object *
@@ -2844,6 +2909,7 @@ static inline void i915_gem_drain_workqueue(struct drm_i915_private *i915)
 	int pass = 2;
 	do {
 		rcu_barrier();
+		i915_gem_drain_freed_objects(i915);
 		drain_workqueue(i915->wq);
 	} while (--pass);
 }
@@ -2974,6 +3040,14 @@ i915_coherent_map_type(struct drm_i915_private *i915)
 void *__must_check i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
 					   enum i915_map_type type);
 
+void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj,
+				 unsigned long offset,
+				 unsigned long size);
+static inline void i915_gem_object_flush_map(struct drm_i915_gem_object *obj)
+{
+	__i915_gem_object_flush_map(obj, 0, obj->base.size);
+}
+
 /**
  * i915_gem_object_unpin_map - releases an earlier mapping
  * @obj: the object to unmap
@@ -3002,7 +3076,12 @@ i915_gem_obj_finish_shmem_access(struct drm_i915_gem_object *obj)
 	i915_gem_object_unpin_pages(obj);
 }
 
-int __must_check i915_mutex_lock_interruptible(struct drm_device *dev);
+static inline int __must_check
+i915_mutex_lock_interruptible(struct drm_device *dev)
+{
+	return mutex_lock_interruptible(&dev->struct_mutex);
+}
+
 int i915_gem_dumb_create(struct drm_file *file_priv,
 			 struct drm_device *dev,
 			 struct drm_mode_create_dumb *args);
@@ -3016,22 +3095,14 @@ void i915_gem_track_fb(struct drm_i915_gem_object *old,
 
 int __must_check i915_gem_set_global_seqno(struct drm_device *dev, u32 seqno);
 
-struct i915_request *
-i915_gem_find_active_request(struct intel_engine_cs *engine);
-
-static inline bool i915_reset_backoff(struct i915_gpu_error *error)
-{
-	return unlikely(test_bit(I915_RESET_BACKOFF, &error->flags));
-}
-
-static inline bool i915_terminally_wedged(struct i915_gpu_error *error)
+static inline bool __i915_wedged(struct i915_gpu_error *error)
 {
 	return unlikely(test_bit(I915_WEDGED, &error->flags));
 }
 
-static inline bool i915_reset_backoff_or_wedged(struct i915_gpu_error *error)
+static inline bool i915_reset_failed(struct drm_i915_private *i915)
 {
-	return i915_reset_backoff(error) | i915_terminally_wedged(error);
+	return __i915_wedged(&i915->gpu_error);
 }
 
 static inline u32 i915_reset_count(struct i915_gpu_error *error)
@@ -3056,14 +3127,13 @@ void i915_gem_fini(struct drm_i915_private *dev_priv);
 void i915_gem_cleanup_engines(struct drm_i915_private *dev_priv);
 int i915_gem_wait_for_idle(struct drm_i915_private *dev_priv,
 			   unsigned int flags, long timeout);
-int __must_check i915_gem_suspend(struct drm_i915_private *dev_priv);
+void i915_gem_suspend(struct drm_i915_private *dev_priv);
 void i915_gem_suspend_late(struct drm_i915_private *dev_priv);
 void i915_gem_resume(struct drm_i915_private *dev_priv);
 vm_fault_t i915_gem_fault(struct vm_fault *vmf);
 int i915_gem_object_wait(struct drm_i915_gem_object *obj,
 			 unsigned int flags,
-			 long timeout,
-			 struct intel_rps_client *rps);
+			 long timeout);
 int i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
 				  unsigned int flags,
 				  const struct i915_sched_attr *attr);
@@ -3106,7 +3176,6 @@ struct drm_i915_fence_reg *
 i915_reserve_fence(struct drm_i915_private *dev_priv);
 void i915_unreserve_fence(struct drm_i915_fence_reg *fence);
 
-void i915_gem_revoke_fences(struct drm_i915_private *dev_priv);
 void i915_gem_restore_fences(struct drm_i915_private *dev_priv);
 
 void i915_gem_detect_bit_6_swizzle(struct drm_i915_private *dev_priv);
@@ -3142,7 +3211,7 @@ int i915_perf_add_config_ioctl(struct drm_device *dev, void *data,
 int i915_perf_remove_config_ioctl(struct drm_device *dev, void *data,
 				  struct drm_file *file);
 void i915_oa_init_reg_state(struct intel_engine_cs *engine,
-			    struct i915_gem_context *ctx,
+			    struct intel_context *ce,
 			    u32 *reg_state);
 
 /* i915_gem_evict.c */
@@ -3457,18 +3526,21 @@ static inline u64 intel_rc6_residency_us(struct drm_i915_private *dev_priv,
 	return DIV_ROUND_UP_ULL(intel_rc6_residency_ns(dev_priv, reg), 1000);
 }
 
-#define I915_READ8(reg)		dev_priv->uncore.funcs.mmio_readb(dev_priv, (reg), true)
-#define I915_WRITE8(reg, val)	dev_priv->uncore.funcs.mmio_writeb(dev_priv, (reg), (val), true)
+#define __I915_REG_OP(op__, dev_priv__, ...) \
+	intel_uncore_##op__(&(dev_priv__)->uncore, __VA_ARGS__)
+
+#define I915_READ8(reg__)	  __I915_REG_OP(read8, dev_priv, (reg__))
+#define I915_WRITE8(reg__, val__) __I915_REG_OP(write8, dev_priv, (reg__), (val__))
 
-#define I915_READ16(reg)	dev_priv->uncore.funcs.mmio_readw(dev_priv, (reg), true)
-#define I915_WRITE16(reg, val)	dev_priv->uncore.funcs.mmio_writew(dev_priv, (reg), (val), true)
-#define I915_READ16_NOTRACE(reg)	dev_priv->uncore.funcs.mmio_readw(dev_priv, (reg), false)
-#define I915_WRITE16_NOTRACE(reg, val)	dev_priv->uncore.funcs.mmio_writew(dev_priv, (reg), (val), false)
+#define I915_READ16(reg__)	   __I915_REG_OP(read16, dev_priv, (reg__))
+#define I915_WRITE16(reg__, val__) __I915_REG_OP(write16, dev_priv, (reg__), (val__))
+#define I915_READ16_NOTRACE(reg__)	   __I915_REG_OP(read16_notrace, dev_priv, (reg__))
+#define I915_WRITE16_NOTRACE(reg__, val__) __I915_REG_OP(write16_notrace, dev_priv, (reg__), (val__))
 
-#define I915_READ(reg)		dev_priv->uncore.funcs.mmio_readl(dev_priv, (reg), true)
-#define I915_WRITE(reg, val)	dev_priv->uncore.funcs.mmio_writel(dev_priv, (reg), (val), true)
-#define I915_READ_NOTRACE(reg)		dev_priv->uncore.funcs.mmio_readl(dev_priv, (reg), false)
-#define I915_WRITE_NOTRACE(reg, val)	dev_priv->uncore.funcs.mmio_writel(dev_priv, (reg), (val), false)
+#define I915_READ(reg__)	 __I915_REG_OP(read, dev_priv, (reg__))
+#define I915_WRITE(reg__, val__) __I915_REG_OP(write, dev_priv, (reg__), (val__))
+#define I915_READ_NOTRACE(reg__)	 __I915_REG_OP(read_notrace, dev_priv, (reg__))
+#define I915_WRITE_NOTRACE(reg__, val__) __I915_REG_OP(write_notrace, dev_priv, (reg__), (val__))
 
 /* Be very careful with read/write 64-bit values. On 32-bit machines, they
  * will be implemented using 2 32-bit writes in an arbitrary order with
@@ -3484,46 +3556,12 @@ static inline u64 intel_rc6_residency_us(struct drm_i915_private *dev_priv,
  *
  * You have been warned.
  */
-#define I915_READ64(reg)	dev_priv->uncore.funcs.mmio_readq(dev_priv, (reg), true)
-
-#define I915_READ64_2x32(lower_reg, upper_reg) ({			\
-	u32 upper, lower, old_upper, loop = 0;				\
-	upper = I915_READ(upper_reg);					\
-	do {								\
-		old_upper = upper;					\
-		lower = I915_READ(lower_reg);				\
-		upper = I915_READ(upper_reg);				\
-	} while (upper != old_upper && loop++ < 2);			\
-	(u64)upper << 32 | lower; })
-
-#define POSTING_READ(reg)	(void)I915_READ_NOTRACE(reg)
-#define POSTING_READ16(reg)	(void)I915_READ16_NOTRACE(reg)
-
-#define __raw_read(x, s) \
-static inline uint##x##_t __raw_i915_read##x(const struct drm_i915_private *dev_priv, \
-					     i915_reg_t reg) \
-{ \
-	return read##s(dev_priv->regs + i915_mmio_reg_offset(reg)); \
-}
-
-#define __raw_write(x, s) \
-static inline void __raw_i915_write##x(const struct drm_i915_private *dev_priv, \
-				       i915_reg_t reg, uint##x##_t val) \
-{ \
-	write##s(val, dev_priv->regs + i915_mmio_reg_offset(reg)); \
-}
-__raw_read(8, b)
-__raw_read(16, w)
-__raw_read(32, l)
-__raw_read(64, q)
-
-__raw_write(8, b)
-__raw_write(16, w)
-__raw_write(32, l)
-__raw_write(64, q)
+#define I915_READ64(reg__)	__I915_REG_OP(read64, dev_priv, (reg__))
+#define I915_READ64_2x32(lower_reg__, upper_reg__) \
+	__I915_REG_OP(read64_2x32, dev_priv, (lower_reg__), (upper_reg__))
 
-#undef __raw_read
-#undef __raw_write
+#define POSTING_READ(reg__)	__I915_REG_OP(posting_read, dev_priv, (reg__))
+#define POSTING_READ16(reg__)	__I915_REG_OP(posting_read16, dev_priv, (reg__))
 
 /* These are untraced mmio-accessors that are only valid to be used inside
  * critical sections, such as inside IRQ handlers, where forcewake is explicitly
@@ -3551,10 +3589,10 @@ __raw_write(64, q)
  * therefore generally be serialised, by either the dev_priv->uncore.lock or
  * a more localised lock guarding all access to that bank of registers.
  */
-#define I915_READ_FW(reg__) __raw_i915_read32(dev_priv, (reg__))
-#define I915_WRITE_FW(reg__, val__) __raw_i915_write32(dev_priv, (reg__), (val__))
-#define I915_WRITE64_FW(reg__, val__) __raw_i915_write64(dev_priv, (reg__), (val__))
-#define POSTING_READ_FW(reg__) (void)I915_READ_FW(reg__)
+#define I915_READ_FW(reg__) __I915_REG_OP(read_fw, dev_priv, (reg__))
+#define I915_WRITE_FW(reg__, val__) __I915_REG_OP(write_fw, dev_priv, (reg__), (val__))
+#define I915_WRITE64_FW(reg__, val__) __I915_REG_OP(write64_fw, dev_priv, (reg__), (val__))
+#define POSTING_READ_FW(reg__) __I915_REG_OP(posting_read_fw, dev_priv, (reg__))
 
 /* "Broadcast RGB" property */
 #define INTEL_BROADCAST_RGB_AUTO 0
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 3c724cc949a5..ad01c92aaf74 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -42,6 +42,7 @@
 #include "i915_drv.h"
 #include "i915_gem_clflush.h"
 #include "i915_gemfs.h"
+#include "i915_globals.h"
 #include "i915_reset.h"
 #include "i915_trace.h"
 #include "i915_vgpu.h"
@@ -49,6 +50,7 @@
 #include "intel_drv.h"
 #include "intel_frontbuffer.h"
 #include "intel_mocs.h"
+#include "intel_pm.h"
 #include "intel_workarounds.h"
 
 static void i915_gem_flush_free_objects(struct drm_i915_private *i915);
@@ -100,48 +102,7 @@ static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
 	spin_unlock(&dev_priv->mm.object_stat_lock);
 }
 
-static int
-i915_gem_wait_for_error(struct i915_gpu_error *error)
-{
-	int ret;
-
-	might_sleep();
-
-	/*
-	 * Only wait 10 seconds for the gpu reset to complete to avoid hanging
-	 * userspace. If it takes that long something really bad is going on and
-	 * we should simply try to bail out and fail as gracefully as possible.
-	 */
-	ret = wait_event_interruptible_timeout(error->reset_queue,
-					       !i915_reset_backoff(error),
-					       I915_RESET_TIMEOUT);
-	if (ret == 0) {
-		DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
-		return -EIO;
-	} else if (ret < 0) {
-		return ret;
-	} else {
-		return 0;
-	}
-}
-
-int i915_mutex_lock_interruptible(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	int ret;
-
-	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
-	if (ret)
-		return ret;
-
-	ret = mutex_lock_interruptible(&dev->struct_mutex);
-	if (ret)
-		return ret;
-
-	return 0;
-}
-
-static u32 __i915_gem_park(struct drm_i915_private *i915)
+static void __i915_gem_park(struct drm_i915_private *i915)
 {
 	intel_wakeref_t wakeref;
 
@@ -152,9 +113,7 @@ static u32 __i915_gem_park(struct drm_i915_private *i915)
 	GEM_BUG_ON(!list_empty(&i915->gt.active_rings));
 
 	if (!i915->gt.awake)
-		return I915_EPOCH_INVALID;
-
-	GEM_BUG_ON(i915->gt.epoch == I915_EPOCH_INVALID);
+		return;
 
 	/*
 	 * Be paranoid and flush a concurrent interrupt to make sure
@@ -183,7 +142,7 @@ static u32 __i915_gem_park(struct drm_i915_private *i915)
 
 	intel_display_power_put(i915, POWER_DOMAIN_GT_IRQ, wakeref);
 
-	return i915->gt.epoch;
+	i915_globals_park();
 }
 
 void i915_gem_park(struct drm_i915_private *i915)
@@ -225,8 +184,7 @@ void i915_gem_unpark(struct drm_i915_private *i915)
 	i915->gt.awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);
 	GEM_BUG_ON(!i915->gt.awake);
 
-	if (unlikely(++i915->gt.epoch == 0)) /* keep 0 as invalid */
-		i915->gt.epoch = 1;
+	i915_globals_unpark();
 
 	intel_enable_gt_powersave(i915);
 	i915_update_gfx_val(i915);
@@ -351,7 +309,7 @@ static void __start_cpu_write(struct drm_i915_gem_object *obj)
 		obj->cache_dirty = true;
 }
 
-static void
+void
 __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj,
 				struct sg_table *pages,
 				bool needs_clflush)
@@ -459,8 +417,7 @@ int i915_gem_object_unbind(struct drm_i915_gem_object *obj)
 static long
 i915_gem_object_wait_fence(struct dma_fence *fence,
 			   unsigned int flags,
-			   long timeout,
-			   struct intel_rps_client *rps_client)
+			   long timeout)
 {
 	struct i915_request *rq;
 
@@ -478,27 +435,6 @@ i915_gem_object_wait_fence(struct dma_fence *fence,
 	if (i915_request_completed(rq))
 		goto out;
 
-	/*
-	 * This client is about to stall waiting for the GPU. In many cases
-	 * this is undesirable and limits the throughput of the system, as
-	 * many clients cannot continue processing user input/output whilst
-	 * blocked. RPS autotuning may take tens of milliseconds to respond
-	 * to the GPU load and thus incurs additional latency for the client.
-	 * We can circumvent that by promoting the GPU frequency to maximum
-	 * before we wait. This makes the GPU throttle up much more quickly
-	 * (good for benchmarks and user experience, e.g. window animations),
-	 * but at a cost of spending more power processing the workload
-	 * (bad for battery). Not all clients even want their results
-	 * immediately and for them we should just let the GPU select its own
-	 * frequency to maximise efficiency. To prevent a single client from
-	 * forcing the clocks too high for the whole system, we only allow
-	 * each client to waitboost once in a busy period.
-	 */
-	if (rps_client && !i915_request_started(rq)) {
-		if (INTEL_GEN(rq->i915) >= 6)
-			gen6_rps_boost(rq, rps_client);
-	}
-
 	timeout = i915_request_wait(rq, flags, timeout);
 
 out:
@@ -511,8 +447,7 @@ out:
 static long
 i915_gem_object_wait_reservation(struct reservation_object *resv,
 				 unsigned int flags,
-				 long timeout,
-				 struct intel_rps_client *rps_client)
+				 long timeout)
 {
 	unsigned int seq = __read_seqcount_begin(&resv->seq);
 	struct dma_fence *excl;
@@ -530,8 +465,7 @@ i915_gem_object_wait_reservation(struct reservation_object *resv,
 
 		for (i = 0; i < count; i++) {
 			timeout = i915_gem_object_wait_fence(shared[i],
-							     flags, timeout,
-							     rps_client);
+							     flags, timeout);
 			if (timeout < 0)
 				break;
 
@@ -557,8 +491,7 @@ i915_gem_object_wait_reservation(struct reservation_object *resv,
 	}
 
 	if (excl && timeout >= 0)
-		timeout = i915_gem_object_wait_fence(excl, flags, timeout,
-						     rps_client);
+		timeout = i915_gem_object_wait_fence(excl, flags, timeout);
 
 	dma_fence_put(excl);
 
@@ -652,30 +585,19 @@ i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
  * @obj: i915 gem object
  * @flags: how to wait (under a lock, for all rendering or just for writes etc)
  * @timeout: how long to wait
- * @rps_client: client (user process) to charge for any waitboosting
  */
 int
 i915_gem_object_wait(struct drm_i915_gem_object *obj,
 		     unsigned int flags,
-		     long timeout,
-		     struct intel_rps_client *rps_client)
+		     long timeout)
 {
 	might_sleep();
 	GEM_BUG_ON(timeout < 0);
 
-	timeout = i915_gem_object_wait_reservation(obj->resv,
-						   flags, timeout,
-						   rps_client);
+	timeout = i915_gem_object_wait_reservation(obj->resv, flags, timeout);
 	return timeout < 0 ? timeout : 0;
 }
 
-static struct intel_rps_client *to_rps_client(struct drm_file *file)
-{
-	struct drm_i915_file_private *fpriv = file->driver_priv;
-
-	return &fpriv->rps_client;
-}
-
 static int
 i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
 		     struct drm_i915_gem_pwrite *args,
@@ -698,28 +620,18 @@ i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
 	return 0;
 }
 
-void *i915_gem_object_alloc(struct drm_i915_private *dev_priv)
-{
-	return kmem_cache_zalloc(dev_priv->objects, GFP_KERNEL);
-}
-
-void i915_gem_object_free(struct drm_i915_gem_object *obj)
-{
-	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
-	kmem_cache_free(dev_priv->objects, obj);
-}
-
 static int
 i915_gem_create(struct drm_file *file,
 		struct drm_i915_private *dev_priv,
-		u64 size,
+		u64 *size_p,
 		u32 *handle_p)
 {
 	struct drm_i915_gem_object *obj;
-	int ret;
 	u32 handle;
+	u64 size;
+	int ret;
 
-	size = roundup(size, PAGE_SIZE);
+	size = round_up(*size_p, PAGE_SIZE);
 	if (size == 0)
 		return -EINVAL;
 
@@ -735,6 +647,7 @@ i915_gem_create(struct drm_file *file,
 		return ret;
 
 	*handle_p = handle;
+	*size_p = size;
 	return 0;
 }
 
@@ -747,7 +660,7 @@ i915_gem_dumb_create(struct drm_file *file,
 	args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
 	args->size = args->pitch * args->height;
 	return i915_gem_create(file, to_i915(dev),
-			       args->size, &args->handle);
+			       &args->size, &args->handle);
 }
 
 static bool gpu_write_needs_clflush(struct drm_i915_gem_object *obj)
@@ -772,7 +685,7 @@ i915_gem_create_ioctl(struct drm_device *dev, void *data,
 	i915_gem_flush_free_objects(dev_priv);
 
 	return i915_gem_create(file, dev_priv,
-			       args->size, &args->handle);
+			       &args->size, &args->handle);
 }
 
 static inline enum fb_op_origin
@@ -881,8 +794,7 @@ int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
 	ret = i915_gem_object_wait(obj,
 				   I915_WAIT_INTERRUPTIBLE |
 				   I915_WAIT_LOCKED,
-				   MAX_SCHEDULE_TIMEOUT,
-				   NULL);
+				   MAX_SCHEDULE_TIMEOUT);
 	if (ret)
 		return ret;
 
@@ -934,8 +846,7 @@ int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj,
 				   I915_WAIT_INTERRUPTIBLE |
 				   I915_WAIT_LOCKED |
 				   I915_WAIT_ALL,
-				   MAX_SCHEDULE_TIMEOUT,
-				   NULL);
+				   MAX_SCHEDULE_TIMEOUT);
 	if (ret)
 		return ret;
 
@@ -1197,8 +1108,7 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 
 	ret = i915_gem_object_wait(obj,
 				   I915_WAIT_INTERRUPTIBLE,
-				   MAX_SCHEDULE_TIMEOUT,
-				   to_rps_client(file));
+				   MAX_SCHEDULE_TIMEOUT);
 	if (ret)
 		goto out;
 
@@ -1497,8 +1407,7 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 	ret = i915_gem_object_wait(obj,
 				   I915_WAIT_INTERRUPTIBLE |
 				   I915_WAIT_ALL,
-				   MAX_SCHEDULE_TIMEOUT,
-				   to_rps_client(file));
+				   MAX_SCHEDULE_TIMEOUT);
 	if (ret)
 		goto err;
 
@@ -1578,17 +1487,37 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 	if ((write_domain | read_domains) & I915_GEM_GPU_DOMAINS)
 		return -EINVAL;
 
-	/* Having something in the write domain implies it's in the read
+	/*
+	 * Having something in the write domain implies it's in the read
 	 * domain, and only that read domain.  Enforce that in the request.
 	 */
-	if (write_domain != 0 && read_domains != write_domain)
+	if (write_domain && read_domains != write_domain)
 		return -EINVAL;
 
+	if (!read_domains)
+		return 0;
+
 	obj = i915_gem_object_lookup(file, args->handle);
 	if (!obj)
 		return -ENOENT;
 
-	/* Try to flush the object off the GPU without holding the lock.
+	/*
+	 * Already in the desired write domain? Nothing for us to do!
+	 *
+	 * We apply a little bit of cunning here to catch a broader set of
+	 * no-ops. If obj->write_domain is set, we must be in the same
+	 * obj->read_domains, and only that domain. Therefore, if that
+	 * obj->write_domain matches the request read_domains, we are
+	 * already in the same read/write domain and can skip the operation,
+	 * without having to further check the requested write_domain.
+	 */
+	if (READ_ONCE(obj->write_domain) == read_domains) {
+		err = 0;
+		goto out;
+	}
+
+	/*
+	 * Try to flush the object off the GPU without holding the lock.
 	 * We will repeat the flush holding the lock in the normal manner
 	 * to catch cases where we are gazumped.
 	 */
@@ -1596,8 +1525,7 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 				   I915_WAIT_INTERRUPTIBLE |
 				   I915_WAIT_PRIORITY |
 				   (write_domain ? I915_WAIT_ALL : 0),
-				   MAX_SCHEDULE_TIMEOUT,
-				   to_rps_client(file));
+				   MAX_SCHEDULE_TIMEOUT);
 	if (err)
 		goto out;
 
@@ -1808,6 +1736,9 @@ static unsigned int tile_row_pages(const struct drm_i915_gem_object *obj)
  * 2 - Recognise WC as a separate cache domain so that we can flush the
  *     delayed writes via GTT before performing direct access via WC.
  *
+ * 3 - Remove implicit set-domain(GTT) and synchronisation on initial
+ *     pagefault; swapin remains transparent.
+ *
  * Restrictions:
  *
  *  * snoopable objects cannot be accessed via the GTT. It can cause machine
@@ -1835,7 +1766,7 @@ static unsigned int tile_row_pages(const struct drm_i915_gem_object *obj)
  */
 int i915_gem_mmap_gtt_version(void)
 {
-	return 2;
+	return 3;
 }
 
 static inline struct i915_ggtt_view
@@ -1891,6 +1822,7 @@ vm_fault_t i915_gem_fault(struct vm_fault *vmf)
 	intel_wakeref_t wakeref;
 	struct i915_vma *vma;
 	pgoff_t page_offset;
+	int srcu;
 	int ret;
 
 	/* Sanity check that we allow writing into this object */
@@ -1902,27 +1834,21 @@ vm_fault_t i915_gem_fault(struct vm_fault *vmf)
 
 	trace_i915_gem_object_fault(obj, page_offset, true, write);
 
-	/* Try to flush the object off the GPU first without holding the lock.
-	 * Upon acquiring the lock, we will perform our sanity checks and then
-	 * repeat the flush holding the lock in the normal manner to catch cases
-	 * where we are gazumped.
-	 */
-	ret = i915_gem_object_wait(obj,
-				   I915_WAIT_INTERRUPTIBLE,
-				   MAX_SCHEDULE_TIMEOUT,
-				   NULL);
-	if (ret)
-		goto err;
-
 	ret = i915_gem_object_pin_pages(obj);
 	if (ret)
 		goto err;
 
 	wakeref = intel_runtime_pm_get(dev_priv);
 
+	srcu = i915_reset_trylock(dev_priv);
+	if (srcu < 0) {
+		ret = srcu;
+		goto err_rpm;
+	}
+
 	ret = i915_mutex_lock_interruptible(dev);
 	if (ret)
-		goto err_rpm;
+		goto err_reset;
 
 	/* Access to snoopable pages through the GTT is incoherent. */
 	if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev_priv)) {
@@ -1930,7 +1856,6 @@ vm_fault_t i915_gem_fault(struct vm_fault *vmf)
 		goto err_unlock;
 	}
 
-
 	/* Now pin it into the GTT as needed */
 	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
 				       PIN_MAPPABLE |
@@ -1964,10 +1889,6 @@ vm_fault_t i915_gem_fault(struct vm_fault *vmf)
 		goto err_unlock;
 	}
 
-	ret = i915_gem_object_set_to_gtt_domain(obj, write);
-	if (ret)
-		goto err_unpin;
-
 	ret = i915_vma_pin_fence(vma);
 	if (ret)
 		goto err_unpin;
@@ -1995,6 +1916,8 @@ err_unpin:
 	__i915_vma_unpin(vma);
 err_unlock:
 	mutex_unlock(&dev->struct_mutex);
+err_reset:
+	i915_reset_unlock(dev_priv, srcu);
 err_rpm:
 	intel_runtime_pm_put(dev_priv, wakeref);
 	i915_gem_object_unpin_pages(obj);
@@ -2007,7 +1930,7 @@ err:
 		 * fail). But any other -EIO isn't ours (e.g. swap in failure)
 		 * and so needs to be reported.
 		 */
-		if (!i915_terminally_wedged(&dev_priv->gpu_error))
+		if (!i915_terminally_wedged(dev_priv))
 			return VM_FAULT_SIGBUS;
 		/* else: fall through */
 	case -EAGAIN:
@@ -2280,7 +2203,6 @@ i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj,
 	struct page *page;
 
 	__i915_gem_object_release_shmem(obj, pages, true);
-
 	i915_gem_gtt_finish_pages(obj, pages);
 
 	if (i915_gem_object_needs_bit17_swizzle(obj))
@@ -2488,7 +2410,7 @@ rebuild_st:
 		do {
 			cond_resched();
 			page = shmem_read_mapping_page_gfp(mapping, i, gfp);
-			if (likely(!IS_ERR(page)))
+			if (!IS_ERR(page))
 				break;
 
 			if (!*s) {
@@ -2622,6 +2544,14 @@ void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
 
 	lockdep_assert_held(&obj->mm.lock);
 
+	/* Make the pages coherent with the GPU (flushing any swapin). */
+	if (obj->cache_dirty) {
+		obj->write_domain = 0;
+		if (i915_gem_object_has_struct_page(obj))
+			drm_clflush_sg(pages);
+		obj->cache_dirty = false;
+	}
+
 	obj->mm.get_page.sg_pos = pages->sgl;
 	obj->mm.get_page.sg_idx = 0;
 
@@ -2823,6 +2753,33 @@ err_unlock:
 	goto out_unlock;
 }
 
+void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj,
+				 unsigned long offset,
+				 unsigned long size)
+{
+	enum i915_map_type has_type;
+	void *ptr;
+
+	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
+	GEM_BUG_ON(range_overflows_t(typeof(obj->base.size),
+				     offset, size, obj->base.size));
+
+	obj->mm.dirty = true;
+
+	if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE)
+		return;
+
+	ptr = page_unpack_bits(obj->mm.mapping, &has_type);
+	if (has_type == I915_MAP_WC)
+		return;
+
+	drm_clflush_virt_range(ptr + offset, size);
+	if (size == obj->base.size) {
+		obj->write_domain &= ~I915_GEM_DOMAIN_CPU;
+		obj->cache_dirty = false;
+	}
+}
+
 static int
 i915_gem_object_pwrite_gtt(struct drm_i915_gem_object *obj,
 			   const struct drm_i915_gem_pwrite *arg)
@@ -2832,7 +2789,11 @@ i915_gem_object_pwrite_gtt(struct drm_i915_gem_object *obj,
 	u64 remain, offset;
 	unsigned int pg;
 
-	/* Before we instantiate/pin the backing store for our use, we
+	/* Caller already validated user args */
+	GEM_BUG_ON(!access_ok(user_data, arg->size));
+
+	/*
+	 * Before we instantiate/pin the backing store for our use, we
 	 * can prepopulate the shmemfs filp efficiently using a write into
 	 * the pagecache. We avoid the penalty of instantiating all the
 	 * pages, important if the user is just writing to a few and never
@@ -2846,7 +2807,8 @@ i915_gem_object_pwrite_gtt(struct drm_i915_gem_object *obj,
 	if (obj->mm.madv != I915_MADV_WILLNEED)
 		return -EFAULT;
 
-	/* Before the pages are instantiated the object is treated as being
+	/*
+	 * Before the pages are instantiated the object is treated as being
 	 * in the CPU domain. The pages will be clflushed as required before
 	 * use, and we can freely write into the pages directly. If userspace
 	 * races pwrite with any other operation; corruption will ensue -
@@ -2862,20 +2824,32 @@ i915_gem_object_pwrite_gtt(struct drm_i915_gem_object *obj,
 		struct page *page;
 		void *data, *vaddr;
 		int err;
+		char c;
 
 		len = PAGE_SIZE - pg;
 		if (len > remain)
 			len = remain;
 
+		/* Prefault the user page to reduce potential recursion */
+		err = __get_user(c, user_data);
+		if (err)
+			return err;
+
+		err = __get_user(c, user_data + len - 1);
+		if (err)
+			return err;
+
 		err = pagecache_write_begin(obj->base.filp, mapping,
 					    offset, len, 0,
 					    &page, &data);
 		if (err < 0)
 			return err;
 
-		vaddr = kmap(page);
-		unwritten = copy_from_user(vaddr + pg, user_data, len);
-		kunmap(page);
+		vaddr = kmap_atomic(page);
+		unwritten = __copy_from_user_inatomic(vaddr + pg,
+						      user_data,
+						      len);
+		kunmap_atomic(vaddr);
 
 		err = pagecache_write_end(obj->base.filp, mapping,
 					  offset, len, len - unwritten,
@@ -2883,8 +2857,9 @@ i915_gem_object_pwrite_gtt(struct drm_i915_gem_object *obj,
 		if (err < 0)
 			return err;
 
+		/* We don't handle -EFAULT, leave it to the caller to check */
 		if (unwritten)
-			return -EFAULT;
+			return -ENODEV;
 
 		remain -= len;
 		user_data += len;
@@ -2895,51 +2870,6 @@ i915_gem_object_pwrite_gtt(struct drm_i915_gem_object *obj,
 	return 0;
 }
 
-static bool match_ring(struct i915_request *rq)
-{
-	struct drm_i915_private *dev_priv = rq->i915;
-	u32 ring = I915_READ(RING_START(rq->engine->mmio_base));
-
-	return ring == i915_ggtt_offset(rq->ring->vma);
-}
-
-struct i915_request *
-i915_gem_find_active_request(struct intel_engine_cs *engine)
-{
-	struct i915_request *request, *active = NULL;
-	unsigned long flags;
-
-	/*
-	 * We are called by the error capture, reset and to dump engine
-	 * state at random points in time. In particular, note that neither is
-	 * crucially ordered with an interrupt. After a hang, the GPU is dead
-	 * and we assume that no more writes can happen (we waited long enough
-	 * for all writes that were in transaction to be flushed) - adding an
-	 * extra delay for a recent interrupt is pointless. Hence, we do
-	 * not need an engine->irq_seqno_barrier() before the seqno reads.
-	 * At all other times, we must assume the GPU is still running, but
-	 * we only care about the snapshot of this moment.
-	 */
-	spin_lock_irqsave(&engine->timeline.lock, flags);
-	list_for_each_entry(request, &engine->timeline.requests, link) {
-		if (i915_request_completed(request))
-			continue;
-
-		if (!i915_request_started(request))
-			break;
-
-		/* More than one preemptible request may match! */
-		if (!match_ring(request))
-			break;
-
-		active = request;
-		break;
-	}
-	spin_unlock_irqrestore(&engine->timeline.lock, flags);
-
-	return active;
-}
-
 static void
 i915_gem_retire_work_handler(struct work_struct *work)
 {
@@ -2964,180 +2894,105 @@ i915_gem_retire_work_handler(struct work_struct *work)
 				   round_jiffies_up_relative(HZ));
 }
 
-static void shrink_caches(struct drm_i915_private *i915)
+static bool switch_to_kernel_context_sync(struct drm_i915_private *i915,
+					  unsigned long mask)
 {
-	/*
-	 * kmem_cache_shrink() discards empty slabs and reorders partially
-	 * filled slabs to prioritise allocating from the mostly full slabs,
-	 * with the aim of reducing fragmentation.
-	 */
-	kmem_cache_shrink(i915->priorities);
-	kmem_cache_shrink(i915->dependencies);
-	kmem_cache_shrink(i915->requests);
-	kmem_cache_shrink(i915->luts);
-	kmem_cache_shrink(i915->vmas);
-	kmem_cache_shrink(i915->objects);
-}
-
-struct sleep_rcu_work {
-	union {
-		struct rcu_head rcu;
-		struct work_struct work;
-	};
-	struct drm_i915_private *i915;
-	unsigned int epoch;
-};
+	bool result = true;
 
-static inline bool
-same_epoch(struct drm_i915_private *i915, unsigned int epoch)
-{
 	/*
-	 * There is a small chance that the epoch wrapped since we started
-	 * sleeping. If we assume that epoch is at least a u32, then it will
-	 * take at least 2^32 * 100ms for it to wrap, or about 326 years.
+	 * Even if we fail to switch, give whatever is running a small chance
+	 * to save itself before we report the failure. Yes, this may be a
+	 * false positive due to e.g. ENOMEM, caveat emptor!
 	 */
-	return epoch == READ_ONCE(i915->gt.epoch);
-}
-
-static void __sleep_work(struct work_struct *work)
-{
-	struct sleep_rcu_work *s = container_of(work, typeof(*s), work);
-	struct drm_i915_private *i915 = s->i915;
-	unsigned int epoch = s->epoch;
-
-	kfree(s);
-	if (same_epoch(i915, epoch))
-		shrink_caches(i915);
-}
+	if (i915_gem_switch_to_kernel_context(i915, mask))
+		result = false;
 
-static void __sleep_rcu(struct rcu_head *rcu)
-{
-	struct sleep_rcu_work *s = container_of(rcu, typeof(*s), rcu);
-	struct drm_i915_private *i915 = s->i915;
-
-	destroy_rcu_head(&s->rcu);
+	if (i915_gem_wait_for_idle(i915,
+				   I915_WAIT_LOCKED |
+				   I915_WAIT_FOR_IDLE_BOOST,
+				   I915_GEM_IDLE_TIMEOUT))
+		result = false;
+
+	if (!result) {
+		if (i915_modparams.reset) { /* XXX hide warning from gem_eio */
+			dev_err(i915->drm.dev,
+				"Failed to idle engines, declaring wedged!\n");
+			GEM_TRACE_DUMP();
+		}
 
-	if (same_epoch(i915, s->epoch)) {
-		INIT_WORK(&s->work, __sleep_work);
-		queue_work(i915->wq, &s->work);
-	} else {
-		kfree(s);
+		/* Forcibly cancel outstanding work and leave the gpu quiet. */
+		i915_gem_set_wedged(i915);
 	}
-}
 
-static inline bool
-new_requests_since_last_retire(const struct drm_i915_private *i915)
-{
-	return (READ_ONCE(i915->gt.active_requests) ||
-		work_pending(&i915->gt.idle_work.work));
+	i915_retire_requests(i915); /* ensure we flush after wedging */
+	return result;
 }
 
-static void assert_kernel_context_is_current(struct drm_i915_private *i915)
+static bool load_power_context(struct drm_i915_private *i915)
 {
-	struct intel_engine_cs *engine;
-	enum intel_engine_id id;
+	/* Force loading the kernel context on all engines */
+	if (!switch_to_kernel_context_sync(i915, ALL_ENGINES))
+		return false;
 
-	if (i915_terminally_wedged(&i915->gpu_error))
-		return;
+	/*
+	 * Immediately park the GPU so that we enable powersaving and
+	 * treat it as idle. The next time we issue a request, we will
+	 * unpark and start using the engine->pinned_default_state, otherwise
+	 * it is in limbo and an early reset may fail.
+	 */
+	__i915_gem_park(i915);
 
-	GEM_BUG_ON(i915->gt.active_requests);
-	for_each_engine(engine, i915, id) {
-		GEM_BUG_ON(__i915_active_request_peek(&engine->timeline.last_request));
-		GEM_BUG_ON(engine->last_retired_context !=
-			   to_intel_context(i915->kernel_context, engine));
-	}
+	return true;
 }
 
 static void
 i915_gem_idle_work_handler(struct work_struct *work)
 {
-	struct drm_i915_private *dev_priv =
-		container_of(work, typeof(*dev_priv), gt.idle_work.work);
-	unsigned int epoch = I915_EPOCH_INVALID;
+	struct drm_i915_private *i915 =
+		container_of(work, typeof(*i915), gt.idle_work.work);
 	bool rearm_hangcheck;
 
-	if (!READ_ONCE(dev_priv->gt.awake))
+	if (!READ_ONCE(i915->gt.awake))
 		return;
 
-	if (READ_ONCE(dev_priv->gt.active_requests))
+	if (READ_ONCE(i915->gt.active_requests))
 		return;
 
-	/*
-	 * Flush out the last user context, leaving only the pinned
-	 * kernel context resident. When we are idling on the kernel_context,
-	 * no more new requests (with a context switch) are emitted and we
-	 * can finally rest. A consequence is that the idle work handler is
-	 * always called at least twice before idling (and if the system is
-	 * idle that implies a round trip through the retire worker).
-	 */
-	mutex_lock(&dev_priv->drm.struct_mutex);
-	i915_gem_switch_to_kernel_context(dev_priv);
-	mutex_unlock(&dev_priv->drm.struct_mutex);
-
-	GEM_TRACE("active_requests=%d (after switch-to-kernel-context)\n",
-		  READ_ONCE(dev_priv->gt.active_requests));
-
-	/*
-	 * Wait for last execlists context complete, but bail out in case a
-	 * new request is submitted. As we don't trust the hardware, we
-	 * continue on if the wait times out. This is necessary to allow
-	 * the machine to suspend even if the hardware dies, and we will
-	 * try to recover in resume (after depriving the hardware of power,
-	 * it may be in a better mmod).
-	 */
-	__wait_for(if (new_requests_since_last_retire(dev_priv)) return,
-		   intel_engines_are_idle(dev_priv),
-		   I915_IDLE_ENGINES_TIMEOUT * 1000,
-		   10, 500);
-
 	rearm_hangcheck =
-		cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
+		cancel_delayed_work_sync(&i915->gpu_error.hangcheck_work);
 
-	if (!mutex_trylock(&dev_priv->drm.struct_mutex)) {
+	if (!mutex_trylock(&i915->drm.struct_mutex)) {
 		/* Currently busy, come back later */
-		mod_delayed_work(dev_priv->wq,
-				 &dev_priv->gt.idle_work,
+		mod_delayed_work(i915->wq,
+				 &i915->gt.idle_work,
 				 msecs_to_jiffies(50));
 		goto out_rearm;
 	}
 
 	/*
-	 * New request retired after this work handler started, extend active
-	 * period until next instance of the work.
+	 * Flush out the last user context, leaving only the pinned
+	 * kernel context resident. Should anything unfortunate happen
+	 * while we are idle (such as the GPU being power cycled), no users
+	 * will be harmed.
 	 */
-	if (new_requests_since_last_retire(dev_priv))
-		goto out_unlock;
+	if (!work_pending(&i915->gt.idle_work.work) &&
+	    !i915->gt.active_requests) {
+		++i915->gt.active_requests; /* don't requeue idle */
 
-	epoch = __i915_gem_park(dev_priv);
+		switch_to_kernel_context_sync(i915, i915->gt.active_engines);
 
-	assert_kernel_context_is_current(dev_priv);
+		if (!--i915->gt.active_requests) {
+			__i915_gem_park(i915);
+			rearm_hangcheck = false;
+		}
+	}
 
-	rearm_hangcheck = false;
-out_unlock:
-	mutex_unlock(&dev_priv->drm.struct_mutex);
+	mutex_unlock(&i915->drm.struct_mutex);
 
 out_rearm:
 	if (rearm_hangcheck) {
-		GEM_BUG_ON(!dev_priv->gt.awake);
-		i915_queue_hangcheck(dev_priv);
-	}
-
-	/*
-	 * When we are idle, it is an opportune time to reap our caches.
-	 * However, we have many objects that utilise RCU and the ordered
-	 * i915->wq that this work is executing on. To try and flush any
-	 * pending frees now we are idle, we first wait for an RCU grace
-	 * period, and then queue a task (that will run last on the wq) to
-	 * shrink and re-optimize the caches.
-	 */
-	if (same_epoch(dev_priv, epoch)) {
-		struct sleep_rcu_work *s = kmalloc(sizeof(*s), GFP_KERNEL);
-		if (s) {
-			init_rcu_head(&s->rcu);
-			s->i915 = dev_priv;
-			s->epoch = epoch;
-			call_rcu(&s->rcu, __sleep_rcu);
-		}
+		GEM_BUG_ON(!i915->gt.awake);
+		i915_queue_hangcheck(i915);
 	}
 }
 
@@ -3171,7 +3026,7 @@ void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file)
 		list_del(&lut->obj_link);
 		list_del(&lut->ctx_link);
 
-		kmem_cache_free(i915->luts, lut);
+		i915_lut_handle_free(lut);
 		__i915_gem_object_release_unless_active(obj);
 	}
 
@@ -3234,8 +3089,7 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 				   I915_WAIT_INTERRUPTIBLE |
 				   I915_WAIT_PRIORITY |
 				   I915_WAIT_ALL,
-				   to_wait_timeout(args->timeout_ns),
-				   to_rps_client(file));
+				   to_wait_timeout(args->timeout_ns));
 
 	if (args->timeout_ns > 0) {
 		args->timeout_ns -= ktime_to_ns(ktime_sub(ktime_get(), start));
@@ -3304,7 +3158,7 @@ wait_for_timelines(struct drm_i915_private *i915,
 		 * stalls, so allow the gpu to boost to maximum clocks.
 		 */
 		if (flags & I915_WAIT_FOR_IDLE_BOOST)
-			gen6_rps_boost(rq, NULL);
+			gen6_rps_boost(rq);
 
 		timeout = i915_request_wait(rq, flags, timeout);
 		i915_request_put(rq);
@@ -3340,19 +3194,11 @@ int i915_gem_wait_for_idle(struct drm_i915_private *i915,
 
 		lockdep_assert_held(&i915->drm.struct_mutex);
 
-		if (GEM_SHOW_DEBUG() && !timeout) {
-			/* Presume that timeout was non-zero to begin with! */
-			dev_warn(&i915->drm.pdev->dev,
-				 "Missed idle-completion interrupt!\n");
-			GEM_TRACE_DUMP();
-		}
-
 		err = wait_for_engines(i915);
 		if (err)
 			return err;
 
 		i915_retire_requests(i915);
-		GEM_BUG_ON(i915->gt.active_requests);
 	}
 
 	return 0;
@@ -3399,8 +3245,7 @@ i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write)
 				   I915_WAIT_INTERRUPTIBLE |
 				   I915_WAIT_LOCKED |
 				   (write ? I915_WAIT_ALL : 0),
-				   MAX_SCHEDULE_TIMEOUT,
-				   NULL);
+				   MAX_SCHEDULE_TIMEOUT);
 	if (ret)
 		return ret;
 
@@ -3462,8 +3307,7 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
 				   I915_WAIT_INTERRUPTIBLE |
 				   I915_WAIT_LOCKED |
 				   (write ? I915_WAIT_ALL : 0),
-				   MAX_SCHEDULE_TIMEOUT,
-				   NULL);
+				   MAX_SCHEDULE_TIMEOUT);
 	if (ret)
 		return ret;
 
@@ -3578,8 +3422,7 @@ restart:
 					   I915_WAIT_INTERRUPTIBLE |
 					   I915_WAIT_LOCKED |
 					   I915_WAIT_ALL,
-					   MAX_SCHEDULE_TIMEOUT,
-					   NULL);
+					   MAX_SCHEDULE_TIMEOUT);
 		if (ret)
 			return ret;
 
@@ -3717,8 +3560,7 @@ int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
 
 	ret = i915_gem_object_wait(obj,
 				   I915_WAIT_INTERRUPTIBLE,
-				   MAX_SCHEDULE_TIMEOUT,
-				   to_rps_client(file));
+				   MAX_SCHEDULE_TIMEOUT);
 	if (ret)
 		goto out;
 
@@ -3844,8 +3686,7 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
 				   I915_WAIT_INTERRUPTIBLE |
 				   I915_WAIT_LOCKED |
 				   (write ? I915_WAIT_ALL : 0),
-				   MAX_SCHEDULE_TIMEOUT,
-				   NULL);
+				   MAX_SCHEDULE_TIMEOUT);
 	if (ret)
 		return ret;
 
@@ -3891,8 +3732,9 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
 	long ret;
 
 	/* ABI: return -EIO if already wedged */
-	if (i915_terminally_wedged(&dev_priv->gpu_error))
-		return -EIO;
+	ret = i915_terminally_wedged(dev_priv);
+	if (ret)
+		return ret;
 
 	spin_lock(&file_priv->mm.lock);
 	list_for_each_entry(request, &file_priv->mm.request_list, client_link) {
@@ -3968,7 +3810,7 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
 	}
 
 	vma = i915_vma_instance(obj, vm, view);
-	if (unlikely(IS_ERR(vma)))
+	if (IS_ERR(vma))
 		return vma;
 
 	if (i915_vma_misplaced(vma, size, alignment, flags)) {
@@ -4000,22 +3842,19 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
 	return vma;
 }
 
-static __always_inline unsigned int __busy_read_flag(unsigned int id)
+static __always_inline u32 __busy_read_flag(u8 id)
 {
-	/* Note that we could alias engines in the execbuf API, but
-	 * that would be very unwise as it prevents userspace from
-	 * fine control over engine selection. Ahem.
-	 *
-	 * This should be something like EXEC_MAX_ENGINE instead of
-	 * I915_NUM_ENGINES.
-	 */
-	BUILD_BUG_ON(I915_NUM_ENGINES > 16);
-	return 0x10000 << id;
+	if (id == (u8)I915_ENGINE_CLASS_INVALID)
+		return 0xffff0000u;
+
+	GEM_BUG_ON(id >= 16);
+	return 0x10000u << id;
 }
 
-static __always_inline unsigned int __busy_write_id(unsigned int id)
+static __always_inline u32 __busy_write_id(u8 id)
 {
-	/* The uABI guarantees an active writer is also amongst the read
+	/*
+	 * The uABI guarantees an active writer is also amongst the read
 	 * engines. This would be true if we accessed the activity tracking
 	 * under the lock, but as we perform the lookup of the object and
 	 * its activity locklessly we can not guarantee that the last_write
@@ -4023,16 +3862,19 @@ static __always_inline unsigned int __busy_write_id(unsigned int id)
 	 * last_read - hence we always set both read and write busy for
 	 * last_write.
 	 */
-	return id | __busy_read_flag(id);
+	if (id == (u8)I915_ENGINE_CLASS_INVALID)
+		return 0xffffffffu;
+
+	return (id + 1) | __busy_read_flag(id);
 }
 
 static __always_inline unsigned int
-__busy_set_if_active(const struct dma_fence *fence,
-		     unsigned int (*flag)(unsigned int id))
+__busy_set_if_active(const struct dma_fence *fence, u32 (*flag)(u8 id))
 {
-	struct i915_request *rq;
+	const struct i915_request *rq;
 
-	/* We have to check the current hw status of the fence as the uABI
+	/*
+	 * We have to check the current hw status of the fence as the uABI
 	 * guarantees forward progress. We could rely on the idle worker
 	 * to eventually flush us, but to minimise latency just ask the
 	 * hardware.
@@ -4043,11 +3885,13 @@ __busy_set_if_active(const struct dma_fence *fence,
 		return 0;
 
 	/* opencode to_request() in order to avoid const warnings */
-	rq = container_of(fence, struct i915_request, fence);
+	rq = container_of(fence, const struct i915_request, fence);
 	if (i915_request_completed(rq))
 		return 0;
 
-	return flag(rq->engine->uabi_id);
+	/* Beware type-expansion follies! */
+	BUILD_BUG_ON(!typecheck(u8, rq->engine->uabi_class));
+	return flag(rq->engine->uabi_class);
 }
 
 static __always_inline unsigned int
@@ -4081,7 +3925,8 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data,
 	if (!obj)
 		goto out;
 
-	/* A discrepancy here is that we do not report the status of
+	/*
+	 * A discrepancy here is that we do not report the status of
 	 * non-i915 fences, i.e. even though we may report the object as idle,
 	 * a call to set-domain may still stall waiting for foreign rendering.
 	 * This also means that wait-ioctl may report an object as busy,
@@ -4281,7 +4126,7 @@ i915_gem_object_create(struct drm_i915_private *dev_priv, u64 size)
 	if (overflows_type(size, obj->base.size))
 		return ERR_PTR(-E2BIG);
 
-	obj = i915_gem_object_alloc(dev_priv);
+	obj = i915_gem_object_alloc();
 	if (obj == NULL)
 		return ERR_PTR(-ENOMEM);
 
@@ -4414,7 +4259,7 @@ static void __i915_gem_free_objects(struct drm_i915_private *i915,
 		drm_gem_object_release(&obj->base);
 		i915_gem_info_remove_obj(i915, obj->base.size);
 
-		kfree(obj->bit_17);
+		bitmap_free(obj->bit_17);
 		i915_gem_object_free(obj);
 
 		GEM_BUG_ON(!atomic_read(&i915->mm.free_count));
@@ -4537,7 +4382,7 @@ void i915_gem_sanitize(struct drm_i915_private *i915)
 	GEM_TRACE("\n");
 
 	wakeref = intel_runtime_pm_get(i915);
-	intel_uncore_forcewake_get(i915, FORCEWAKE_ALL);
+	intel_uncore_forcewake_get(&i915->uncore, FORCEWAKE_ALL);
 
 	/*
 	 * As we have just resumed the machine and woken the device up from
@@ -4545,7 +4390,7 @@ void i915_gem_sanitize(struct drm_i915_private *i915)
 	 * back to defaults, recovering from whatever wedged state we left it
 	 * in and so worth trying to use the device once more.
 	 */
-	if (i915_terminally_wedged(&i915->gpu_error))
+	if (i915_terminally_wedged(i915))
 		i915_gem_unset_wedged(i915);
 
 	/*
@@ -4558,7 +4403,7 @@ void i915_gem_sanitize(struct drm_i915_private *i915)
 	 */
 	intel_engines_sanitize(i915, false);
 
-	intel_uncore_forcewake_put(i915, FORCEWAKE_ALL);
+	intel_uncore_forcewake_put(&i915->uncore, FORCEWAKE_ALL);
 	intel_runtime_pm_put(i915, wakeref);
 
 	mutex_lock(&i915->drm.struct_mutex);
@@ -4566,15 +4411,13 @@ void i915_gem_sanitize(struct drm_i915_private *i915)
 	mutex_unlock(&i915->drm.struct_mutex);
 }
 
-int i915_gem_suspend(struct drm_i915_private *i915)
+void i915_gem_suspend(struct drm_i915_private *i915)
 {
 	intel_wakeref_t wakeref;
-	int ret;
 
 	GEM_TRACE("\n");
 
 	wakeref = intel_runtime_pm_get(i915);
-	intel_suspend_gt_powersave(i915);
 
 	flush_workqueue(i915->wq);
 
@@ -4589,22 +4432,7 @@ int i915_gem_suspend(struct drm_i915_private *i915)
 	 * state. Fortunately, the kernel_context is disposable and we do
 	 * not rely on its state.
 	 */
-	if (!i915_terminally_wedged(&i915->gpu_error)) {
-		ret = i915_gem_switch_to_kernel_context(i915);
-		if (ret)
-			goto err_unlock;
-
-		ret = i915_gem_wait_for_idle(i915,
-					     I915_WAIT_INTERRUPTIBLE |
-					     I915_WAIT_LOCKED |
-					     I915_WAIT_FOR_IDLE_BOOST,
-					     MAX_SCHEDULE_TIMEOUT);
-		if (ret && ret != -EIO)
-			goto err_unlock;
-
-		assert_kernel_context_is_current(i915);
-	}
-	i915_retire_requests(i915); /* ensure we flush after wedging */
+	switch_to_kernel_context_sync(i915, i915->gt.active_engines);
 
 	mutex_unlock(&i915->drm.struct_mutex);
 	i915_reset_flush(i915);
@@ -4617,23 +4445,15 @@ int i915_gem_suspend(struct drm_i915_private *i915)
 	 */
 	drain_delayed_work(&i915->gt.idle_work);
 
-	intel_uc_suspend(i915);
-
 	/*
 	 * Assert that we successfully flushed all the work and
 	 * reset the GPU back to its idle, low power state.
 	 */
-	WARN_ON(i915->gt.awake);
-	if (WARN_ON(!intel_engines_are_idle(i915)))
-		i915_gem_set_wedged(i915); /* no hope, discard everything */
+	GEM_BUG_ON(i915->gt.awake);
 
-	intel_runtime_pm_put(i915, wakeref);
-	return 0;
+	intel_uc_suspend(i915);
 
-err_unlock:
-	mutex_unlock(&i915->drm.struct_mutex);
 	intel_runtime_pm_put(i915, wakeref);
-	return ret;
 }
 
 void i915_gem_suspend_late(struct drm_i915_private *i915)
@@ -4683,7 +4503,7 @@ void i915_gem_resume(struct drm_i915_private *i915)
 	WARN_ON(i915->gt.awake);
 
 	mutex_lock(&i915->drm.struct_mutex);
-	intel_uncore_forcewake_get(i915, FORCEWAKE_ALL);
+	intel_uncore_forcewake_get(&i915->uncore, FORCEWAKE_ALL);
 
 	i915_gem_restore_gtt_mappings(i915);
 	i915_gem_restore_fences(i915);
@@ -4693,7 +4513,7 @@ void i915_gem_resume(struct drm_i915_private *i915)
 	 * guarantee that the context image is complete. So let's just reset
 	 * it and start again.
 	 */
-	i915->gt.resume(i915);
+	intel_gt_resume(i915);
 
 	if (i915_gem_init_hw(i915))
 		goto err_wedged;
@@ -4701,17 +4521,18 @@ void i915_gem_resume(struct drm_i915_private *i915)
 	intel_uc_resume(i915);
 
 	/* Always reload a context for powersaving. */
-	if (i915_gem_switch_to_kernel_context(i915))
+	if (!load_power_context(i915))
 		goto err_wedged;
 
 out_unlock:
-	intel_uncore_forcewake_put(i915, FORCEWAKE_ALL);
+	intel_uncore_forcewake_put(&i915->uncore, FORCEWAKE_ALL);
 	mutex_unlock(&i915->drm.struct_mutex);
 	return;
 
 err_wedged:
-	if (!i915_terminally_wedged(&i915->gpu_error)) {
-		DRM_ERROR("failed to re-initialize GPU, declaring wedged!\n");
+	if (!i915_reset_failed(i915)) {
+		dev_err(i915->drm.dev,
+			"Failed to re-initialize GPU, declaring it wedged!\n");
 		i915_gem_set_wedged(i915);
 	}
 	goto out_unlock;
@@ -4781,6 +4602,8 @@ static int __i915_gem_restart_engines(void *data)
 		}
 	}
 
+	intel_engines_set_scheduler_caps(i915);
+
 	return 0;
 }
 
@@ -4791,7 +4614,7 @@ int i915_gem_init_hw(struct drm_i915_private *dev_priv)
 	dev_priv->gt.last_init_time = ktime_get();
 
 	/* Double layer security blanket, see i915_gem_init() */
-	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
 
 	if (HAS_EDRAM(dev_priv) && INTEL_GEN(dev_priv) < 9)
 		I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf));
@@ -4816,10 +4639,9 @@ int i915_gem_init_hw(struct drm_i915_private *dev_priv)
 	init_unused_rings(dev_priv);
 
 	BUG_ON(!dev_priv->kernel_context);
-	if (i915_terminally_wedged(&dev_priv->gpu_error)) {
-		ret = -EIO;
+	ret = i915_terminally_wedged(dev_priv);
+	if (ret)
 		goto out;
-	}
 
 	ret = i915_ppgtt_init_hw(dev_priv);
 	if (ret) {
@@ -4847,14 +4669,14 @@ int i915_gem_init_hw(struct drm_i915_private *dev_priv)
 	if (ret)
 		goto cleanup_uc;
 
-	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
 
 	return 0;
 
 cleanup_uc:
 	intel_uc_fini_hw(dev_priv);
 out:
-	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
 
 	return ret;
 }
@@ -4864,7 +4686,7 @@ static int __intel_engines_record_defaults(struct drm_i915_private *i915)
 	struct i915_gem_context *ctx;
 	struct intel_engine_cs *engine;
 	enum intel_engine_id id;
-	int err;
+	int err = 0;
 
 	/*
 	 * As we reset the gpu during very early sanitisation, the current
@@ -4897,36 +4719,27 @@ static int __intel_engines_record_defaults(struct drm_i915_private *i915)
 			goto err_active;
 	}
 
-	err = i915_gem_switch_to_kernel_context(i915);
-	if (err)
-		goto err_active;
-
-	if (i915_gem_wait_for_idle(i915, I915_WAIT_LOCKED, HZ / 5)) {
-		i915_gem_set_wedged(i915);
-		err = -EIO; /* Caller will declare us wedged */
+	/* Flush the default context image to memory, and enable powersaving. */
+	if (!load_power_context(i915)) {
+		err = -EIO;
 		goto err_active;
 	}
 
-	assert_kernel_context_is_current(i915);
-
-	/*
-	 * Immediately park the GPU so that we enable powersaving and
-	 * treat it as idle. The next time we issue a request, we will
-	 * unpark and start using the engine->pinned_default_state, otherwise
-	 * it is in limbo and an early reset may fail.
-	 */
-	__i915_gem_park(i915);
-
 	for_each_engine(engine, i915, id) {
+		struct intel_context *ce;
 		struct i915_vma *state;
 		void *vaddr;
 
-		GEM_BUG_ON(to_intel_context(ctx, engine)->pin_count);
+		ce = intel_context_lookup(ctx, engine);
+		if (!ce)
+			continue;
 
-		state = to_intel_context(ctx, engine)->state;
+		state = ce->state;
 		if (!state)
 			continue;
 
+		GEM_BUG_ON(intel_context_is_pinned(ce));
+
 		/*
 		 * As we will hold a reference to the logical state, it will
 		 * not be torn down with the context, and importantly the
@@ -4944,6 +4757,8 @@ static int __intel_engines_record_defaults(struct drm_i915_private *i915)
 			goto err_active;
 
 		engine->default_state = i915_gem_object_get(state->obj);
+		i915_gem_object_set_cache_coherency(engine->default_state,
+						    I915_CACHE_LLC);
 
 		/* Check we can acquire the image of the context state */
 		vaddr = i915_gem_object_pin_map(engine->default_state,
@@ -4982,19 +4797,10 @@ out_ctx:
 err_active:
 	/*
 	 * If we have to abandon now, we expect the engines to be idle
-	 * and ready to be torn-down. First try to flush any remaining
-	 * request, ensure we are pointing at the kernel context and
-	 * then remove it.
+	 * and ready to be torn-down. The quickest way we can accomplish
+	 * this is by declaring ourselves wedged.
 	 */
-	if (WARN_ON(i915_gem_switch_to_kernel_context(i915)))
-		goto out_ctx;
-
-	if (WARN_ON(i915_gem_wait_for_idle(i915,
-					   I915_WAIT_LOCKED,
-					   MAX_SCHEDULE_TIMEOUT)))
-		goto out_ctx;
-
-	i915_gem_contexts_lost(i915);
+	i915_gem_set_wedged(i915);
 	goto out_ctx;
 }
 
@@ -5047,13 +4853,10 @@ int i915_gem_init(struct drm_i915_private *dev_priv)
 
 	dev_priv->mm.unordered_timeline = dma_fence_context_alloc(1);
 
-	if (HAS_LOGICAL_RING_CONTEXTS(dev_priv)) {
-		dev_priv->gt.resume = intel_lr_context_resume;
+	if (HAS_LOGICAL_RING_CONTEXTS(dev_priv))
 		dev_priv->gt.cleanup_engine = intel_logical_ring_cleanup;
-	} else {
-		dev_priv->gt.resume = intel_legacy_submission_resume;
+	else
 		dev_priv->gt.cleanup_engine = intel_engine_cleanup;
-	}
 
 	i915_timelines_init(dev_priv);
 
@@ -5076,7 +4879,7 @@ int i915_gem_init(struct drm_i915_private *dev_priv)
 	 * just magically go away.
 	 */
 	mutex_lock(&dev_priv->drm.struct_mutex);
-	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
 
 	ret = i915_gem_init_ggtt(dev_priv);
 	if (ret) {
@@ -5138,7 +4941,7 @@ int i915_gem_init(struct drm_i915_private *dev_priv)
 		goto err_init_hw;
 	}
 
-	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
 	mutex_unlock(&dev_priv->drm.struct_mutex);
 
 	return 0;
@@ -5152,7 +4955,7 @@ int i915_gem_init(struct drm_i915_private *dev_priv)
 err_init_hw:
 	mutex_unlock(&dev_priv->drm.struct_mutex);
 
-	WARN_ON(i915_gem_suspend(dev_priv));
+	i915_gem_suspend(dev_priv);
 	i915_gem_suspend_late(dev_priv);
 
 	i915_gem_drain_workqueue(dev_priv);
@@ -5173,7 +4976,7 @@ err_scratch:
 	i915_gem_fini_scratch(dev_priv);
 err_ggtt:
 err_unlock:
-	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
 	mutex_unlock(&dev_priv->drm.struct_mutex);
 
 err_uc_misc:
@@ -5192,7 +4995,7 @@ err_uc_misc:
 		 * wedged. But we only want to do this where the GPU is angry,
 		 * for all other failure, such as an allocation failure, bail.
 		 */
-		if (!i915_terminally_wedged(&dev_priv->gpu_error)) {
+		if (!i915_reset_failed(dev_priv)) {
 			i915_load_error(dev_priv,
 					"Failed to initialize GPU, declaring it wedged!\n");
 			i915_gem_set_wedged(dev_priv);
@@ -5305,36 +5108,7 @@ static void i915_gem_init__mm(struct drm_i915_private *i915)
 
 int i915_gem_init_early(struct drm_i915_private *dev_priv)
 {
-	int err = -ENOMEM;
-
-	dev_priv->objects = KMEM_CACHE(drm_i915_gem_object, SLAB_HWCACHE_ALIGN);
-	if (!dev_priv->objects)
-		goto err_out;
-
-	dev_priv->vmas = KMEM_CACHE(i915_vma, SLAB_HWCACHE_ALIGN);
-	if (!dev_priv->vmas)
-		goto err_objects;
-
-	dev_priv->luts = KMEM_CACHE(i915_lut_handle, 0);
-	if (!dev_priv->luts)
-		goto err_vmas;
-
-	dev_priv->requests = KMEM_CACHE(i915_request,
-					SLAB_HWCACHE_ALIGN |
-					SLAB_RECLAIM_ACCOUNT |
-					SLAB_TYPESAFE_BY_RCU);
-	if (!dev_priv->requests)
-		goto err_luts;
-
-	dev_priv->dependencies = KMEM_CACHE(i915_dependency,
-					    SLAB_HWCACHE_ALIGN |
-					    SLAB_RECLAIM_ACCOUNT);
-	if (!dev_priv->dependencies)
-		goto err_requests;
-
-	dev_priv->priorities = KMEM_CACHE(i915_priolist, SLAB_HWCACHE_ALIGN);
-	if (!dev_priv->priorities)
-		goto err_dependencies;
+	int err;
 
 	INIT_LIST_HEAD(&dev_priv->gt.active_rings);
 	INIT_LIST_HEAD(&dev_priv->gt.closed_vma);
@@ -5348,6 +5122,7 @@ int i915_gem_init_early(struct drm_i915_private *dev_priv)
 	init_waitqueue_head(&dev_priv->gpu_error.wait_queue);
 	init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
 	mutex_init(&dev_priv->gpu_error.wedge_mutex);
+	init_srcu_struct(&dev_priv->gpu_error.reset_backoff_srcu);
 
 	atomic_set(&dev_priv->mm.bsd_engine_dispatch_index, 0);
 
@@ -5358,19 +5133,6 @@ int i915_gem_init_early(struct drm_i915_private *dev_priv)
 		DRM_NOTE("Unable to create a private tmpfs mount, hugepage support will be disabled(%d).\n", err);
 
 	return 0;
-
-err_dependencies:
-	kmem_cache_destroy(dev_priv->dependencies);
-err_requests:
-	kmem_cache_destroy(dev_priv->requests);
-err_luts:
-	kmem_cache_destroy(dev_priv->luts);
-err_vmas:
-	kmem_cache_destroy(dev_priv->vmas);
-err_objects:
-	kmem_cache_destroy(dev_priv->objects);
-err_out:
-	return err;
 }
 
 void i915_gem_cleanup_early(struct drm_i915_private *dev_priv)
@@ -5380,15 +5142,7 @@ void i915_gem_cleanup_early(struct drm_i915_private *dev_priv)
 	GEM_BUG_ON(atomic_read(&dev_priv->mm.free_count));
 	WARN_ON(dev_priv->mm.object_count);
 
-	kmem_cache_destroy(dev_priv->priorities);
-	kmem_cache_destroy(dev_priv->dependencies);
-	kmem_cache_destroy(dev_priv->requests);
-	kmem_cache_destroy(dev_priv->luts);
-	kmem_cache_destroy(dev_priv->vmas);
-	kmem_cache_destroy(dev_priv->objects);
-
-	/* And ensure that our DESTROY_BY_RCU slabs are truly destroyed */
-	rcu_barrier();
+	cleanup_srcu_struct(&dev_priv->gpu_error.reset_backoff_srcu);
 
 	i915_gemfs_fini(dev_priv);
 }
diff --git a/drivers/gpu/drm/i915/i915_gem.h b/drivers/gpu/drm/i915/i915_gem.h
index b0e4b976880c..9074eb1e843f 100644
--- a/drivers/gpu/drm/i915/i915_gem.h
+++ b/drivers/gpu/drm/i915/i915_gem.h
@@ -73,14 +73,14 @@ struct drm_i915_private;
 #define GEM_TRACE_DUMP_ON(expr) BUILD_BUG_ON_INVALID(expr)
 #endif
 
-#define I915_NUM_ENGINES 8
+#define I915_GEM_IDLE_TIMEOUT (HZ / 5)
 
 void i915_gem_park(struct drm_i915_private *i915);
 void i915_gem_unpark(struct drm_i915_private *i915);
 
 static inline void __tasklet_disable_sync_once(struct tasklet_struct *t)
 {
-	if (atomic_inc_return(&t->count) == 1)
+	if (!atomic_fetch_inc(&t->count))
 		tasklet_unlock_wait(t);
 }
 
@@ -89,4 +89,9 @@ static inline bool __tasklet_is_enabled(const struct tasklet_struct *t)
 	return !atomic_read(&t->count);
 }
 
+static inline bool __tasklet_enable(struct tasklet_struct *t)
+{
+	return atomic_dec_and_test(&t->count);
+}
+
 #endif /* __I915_GEM_H__ */
diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c
index 280813a4bf82..dd728b26b5aa 100644
--- a/drivers/gpu/drm/i915/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/i915_gem_context.c
@@ -88,12 +88,32 @@
 #include <linux/log2.h>
 #include <drm/i915_drm.h>
 #include "i915_drv.h"
+#include "i915_globals.h"
 #include "i915_trace.h"
+#include "i915_user_extensions.h"
 #include "intel_lrc_reg.h"
 #include "intel_workarounds.h"
 
+#define I915_CONTEXT_CREATE_FLAGS_SINGLE_TIMELINE (1 << 1)
+#define I915_CONTEXT_PARAM_VM 0x9
+
 #define ALL_L3_SLICES(dev) (1 << NUM_L3_SLICES(dev)) - 1
 
+static struct i915_global_gem_context {
+	struct i915_global base;
+	struct kmem_cache *slab_luts;
+} global;
+
+struct i915_lut_handle *i915_lut_handle_alloc(void)
+{
+	return kmem_cache_alloc(global.slab_luts, GFP_KERNEL);
+}
+
+void i915_lut_handle_free(struct i915_lut_handle *lut)
+{
+	return kmem_cache_free(global.slab_luts, lut);
+}
+
 static void lut_close(struct i915_gem_context *ctx)
 {
 	struct i915_lut_handle *lut, *ln;
@@ -102,14 +122,17 @@ static void lut_close(struct i915_gem_context *ctx)
 
 	list_for_each_entry_safe(lut, ln, &ctx->handles_list, ctx_link) {
 		list_del(&lut->obj_link);
-		kmem_cache_free(ctx->i915->luts, lut);
+		i915_lut_handle_free(lut);
 	}
+	INIT_LIST_HEAD(&ctx->handles_list);
 
 	rcu_read_lock();
 	radix_tree_for_each_slot(slot, &ctx->handles_vma, &iter, 0) {
 		struct i915_vma *vma = rcu_dereference_raw(*slot);
 
 		radix_tree_iter_delete(&ctx->handles_vma, &iter, slot);
+
+		vma->open_count--;
 		__i915_gem_object_release_unless_active(vma->obj);
 	}
 	rcu_read_unlock();
@@ -206,25 +229,26 @@ static void release_hw_id(struct i915_gem_context *ctx)
 
 static void i915_gem_context_free(struct i915_gem_context *ctx)
 {
-	unsigned int n;
+	struct intel_context *it, *n;
 
 	lockdep_assert_held(&ctx->i915->drm.struct_mutex);
 	GEM_BUG_ON(!i915_gem_context_is_closed(ctx));
+	GEM_BUG_ON(!list_empty(&ctx->active_engines));
 
 	release_hw_id(ctx);
 	i915_ppgtt_put(ctx->ppgtt);
 
-	for (n = 0; n < ARRAY_SIZE(ctx->__engine); n++) {
-		struct intel_context *ce = &ctx->__engine[n];
+	rbtree_postorder_for_each_entry_safe(it, n, &ctx->hw_contexts, node)
+		intel_context_put(it);
 
-		if (ce->ops)
-			ce->ops->destroy(ce);
-	}
+	if (ctx->timeline)
+		i915_timeline_put(ctx->timeline);
 
 	kfree(ctx->name);
 	put_pid(ctx->pid);
 
 	list_del(&ctx->link);
+	mutex_destroy(&ctx->mutex);
 
 	kfree_rcu(ctx, rcu);
 }
@@ -291,8 +315,6 @@ static void context_close(struct i915_gem_context *ctx)
 	 * the ppgtt).
 	 */
 	lut_close(ctx);
-	if (ctx->ppgtt)
-		i915_ppgtt_close(&ctx->ppgtt->vm);
 
 	ctx->file_priv = ERR_PTR(-EBADF);
 	i915_gem_context_put(ctx);
@@ -307,7 +329,7 @@ static u32 default_desc_template(const struct drm_i915_private *i915,
 	desc = GEN8_CTX_VALID | GEN8_CTX_PRIVILEGE;
 
 	address_mode = INTEL_LEGACY_32B_CONTEXT;
-	if (ppgtt && i915_vm_is_48bit(&ppgtt->vm))
+	if (ppgtt && i915_vm_is_4lvl(&ppgtt->vm))
 		address_mode = INTEL_LEGACY_64B_CONTEXT;
 	desc |= address_mode << GEN8_CTX_ADDRESSING_MODE_SHIFT;
 
@@ -322,134 +344,115 @@ static u32 default_desc_template(const struct drm_i915_private *i915,
 	return desc;
 }
 
-static void intel_context_retire(struct i915_active_request *active,
-				 struct i915_request *rq)
-{
-	struct intel_context *ce =
-		container_of(active, typeof(*ce), active_tracker);
-
-	intel_context_unpin(ce);
-}
-
-void
-intel_context_init(struct intel_context *ce,
-		   struct i915_gem_context *ctx,
-		   struct intel_engine_cs *engine)
-{
-	ce->gem_context = ctx;
-
-	INIT_LIST_HEAD(&ce->signal_link);
-	INIT_LIST_HEAD(&ce->signals);
-
-	/* Use the whole device by default */
-	ce->sseu = intel_device_default_sseu(ctx->i915);
-
-	i915_active_request_init(&ce->active_tracker,
-				 NULL, intel_context_retire);
-}
-
 static struct i915_gem_context *
-__create_hw_context(struct drm_i915_private *dev_priv,
-		    struct drm_i915_file_private *file_priv)
+__create_context(struct drm_i915_private *dev_priv)
 {
 	struct i915_gem_context *ctx;
-	unsigned int n;
-	int ret;
+	int i;
 
 	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
-	if (ctx == NULL)
+	if (!ctx)
 		return ERR_PTR(-ENOMEM);
 
 	kref_init(&ctx->ref);
 	list_add_tail(&ctx->link, &dev_priv->contexts.list);
 	ctx->i915 = dev_priv;
 	ctx->sched.priority = I915_USER_PRIORITY(I915_PRIORITY_NORMAL);
+	INIT_LIST_HEAD(&ctx->active_engines);
+	mutex_init(&ctx->mutex);
 
-	for (n = 0; n < ARRAY_SIZE(ctx->__engine); n++)
-		intel_context_init(&ctx->__engine[n], ctx, dev_priv->engine[n]);
+	ctx->hw_contexts = RB_ROOT;
+	spin_lock_init(&ctx->hw_contexts_lock);
 
 	INIT_RADIX_TREE(&ctx->handles_vma, GFP_KERNEL);
 	INIT_LIST_HEAD(&ctx->handles_list);
 	INIT_LIST_HEAD(&ctx->hw_id_link);
 
-	/* Default context will never have a file_priv */
-	ret = DEFAULT_CONTEXT_HANDLE;
-	if (file_priv) {
-		ret = idr_alloc(&file_priv->context_idr, ctx,
-				DEFAULT_CONTEXT_HANDLE, 0, GFP_KERNEL);
-		if (ret < 0)
-			goto err_lut;
-	}
-	ctx->user_handle = ret;
-
-	ctx->file_priv = file_priv;
-	if (file_priv) {
-		ctx->pid = get_task_pid(current, PIDTYPE_PID);
-		ctx->name = kasprintf(GFP_KERNEL, "%s[%d]/%x",
-				      current->comm,
-				      pid_nr(ctx->pid),
-				      ctx->user_handle);
-		if (!ctx->name) {
-			ret = -ENOMEM;
-			goto err_pid;
-		}
-	}
-
 	/* NB: Mark all slices as needing a remap so that when the context first
 	 * loads it will restore whatever remap state already exists. If there
 	 * is no remap info, it will be a NOP. */
 	ctx->remap_slice = ALL_L3_SLICES(dev_priv);
 
 	i915_gem_context_set_bannable(ctx);
+	i915_gem_context_set_recoverable(ctx);
+
 	ctx->ring_size = 4 * PAGE_SIZE;
 	ctx->desc_template =
 		default_desc_template(dev_priv, dev_priv->mm.aliasing_ppgtt);
 
+	for (i = 0; i < ARRAY_SIZE(ctx->hang_timestamp); i++)
+		ctx->hang_timestamp[i] = jiffies - CONTEXT_FAST_HANG_JIFFIES;
+
 	return ctx;
+}
 
-err_pid:
-	put_pid(ctx->pid);
-	idr_remove(&file_priv->context_idr, ctx->user_handle);
-err_lut:
-	context_close(ctx);
-	return ERR_PTR(ret);
+static struct i915_hw_ppgtt *
+__set_ppgtt(struct i915_gem_context *ctx, struct i915_hw_ppgtt *ppgtt)
+{
+	struct i915_hw_ppgtt *old = ctx->ppgtt;
+
+	ctx->ppgtt = i915_ppgtt_get(ppgtt);
+	ctx->desc_template = default_desc_template(ctx->i915, ppgtt);
+
+	return old;
 }
 
-static void __destroy_hw_context(struct i915_gem_context *ctx,
-				 struct drm_i915_file_private *file_priv)
+static void __assign_ppgtt(struct i915_gem_context *ctx,
+			   struct i915_hw_ppgtt *ppgtt)
 {
-	idr_remove(&file_priv->context_idr, ctx->user_handle);
-	context_close(ctx);
+	if (ppgtt == ctx->ppgtt)
+		return;
+
+	ppgtt = __set_ppgtt(ctx, ppgtt);
+	if (ppgtt)
+		i915_ppgtt_put(ppgtt);
 }
 
 static struct i915_gem_context *
-i915_gem_create_context(struct drm_i915_private *dev_priv,
-			struct drm_i915_file_private *file_priv)
+i915_gem_create_context(struct drm_i915_private *dev_priv, unsigned int flags)
 {
 	struct i915_gem_context *ctx;
 
 	lockdep_assert_held(&dev_priv->drm.struct_mutex);
 
+	BUILD_BUG_ON(I915_CONTEXT_CREATE_FLAGS_SINGLE_TIMELINE &
+		     ~I915_CONTEXT_CREATE_FLAGS_UNKNOWN);
+	if (flags & I915_CONTEXT_CREATE_FLAGS_SINGLE_TIMELINE &&
+	    !HAS_EXECLISTS(dev_priv))
+		return ERR_PTR(-EINVAL);
+
 	/* Reap the most stale context */
 	contexts_free_first(dev_priv);
 
-	ctx = __create_hw_context(dev_priv, file_priv);
+	ctx = __create_context(dev_priv);
 	if (IS_ERR(ctx))
 		return ctx;
 
 	if (HAS_FULL_PPGTT(dev_priv)) {
 		struct i915_hw_ppgtt *ppgtt;
 
-		ppgtt = i915_ppgtt_create(dev_priv, file_priv);
+		ppgtt = i915_ppgtt_create(dev_priv);
 		if (IS_ERR(ppgtt)) {
 			DRM_DEBUG_DRIVER("PPGTT setup failed (%ld)\n",
 					 PTR_ERR(ppgtt));
-			__destroy_hw_context(ctx, file_priv);
+			context_close(ctx);
 			return ERR_CAST(ppgtt);
 		}
 
-		ctx->ppgtt = ppgtt;
-		ctx->desc_template = default_desc_template(dev_priv, ppgtt);
+		__assign_ppgtt(ctx, ppgtt);
+		i915_ppgtt_put(ppgtt);
+	}
+
+	if (flags & I915_CONTEXT_CREATE_FLAGS_SINGLE_TIMELINE) {
+		struct i915_timeline *timeline;
+
+		timeline = i915_timeline_create(dev_priv, NULL);
+		if (IS_ERR(timeline)) {
+			context_close(ctx);
+			return ERR_CAST(timeline);
+		}
+
+		ctx->timeline = timeline;
 	}
 
 	trace_i915_context_create(ctx);
@@ -480,10 +483,17 @@ i915_gem_context_create_gvt(struct drm_device *dev)
 	if (ret)
 		return ERR_PTR(ret);
 
-	ctx = i915_gem_create_context(to_i915(dev), NULL);
+	ctx = i915_gem_create_context(to_i915(dev), 0);
 	if (IS_ERR(ctx))
 		goto out;
 
+	ret = i915_gem_context_pin_hw_id(ctx);
+	if (ret) {
+		context_close(ctx);
+		ctx = ERR_PTR(ret);
+		goto out;
+	}
+
 	ctx->file_priv = ERR_PTR(-EBADF);
 	i915_gem_context_set_closed(ctx); /* not user accessible */
 	i915_gem_context_clear_bannable(ctx);
@@ -516,7 +526,7 @@ i915_gem_context_create_kernel(struct drm_i915_private *i915, int prio)
 	struct i915_gem_context *ctx;
 	int err;
 
-	ctx = i915_gem_create_context(i915, NULL);
+	ctx = i915_gem_create_context(i915, 0);
 	if (IS_ERR(ctx))
 		return ctx;
 
@@ -552,7 +562,7 @@ static void init_contexts(struct drm_i915_private *i915)
 
 static bool needs_preempt_context(struct drm_i915_private *i915)
 {
-	return HAS_LOGICAL_RING_PREEMPTION(i915);
+	return HAS_EXECLISTS(i915);
 }
 
 int i915_gem_contexts_init(struct drm_i915_private *dev_priv)
@@ -563,7 +573,7 @@ int i915_gem_contexts_init(struct drm_i915_private *dev_priv)
 	GEM_BUG_ON(dev_priv->kernel_context);
 	GEM_BUG_ON(dev_priv->preempt_context);
 
-	intel_engine_init_ctx_wa(dev_priv->engine[RCS]);
+	intel_engine_init_ctx_wa(dev_priv->engine[RCS0]);
 	init_contexts(dev_priv);
 
 	/* lowest priority; idle task */
@@ -624,31 +634,87 @@ void i915_gem_contexts_fini(struct drm_i915_private *i915)
 
 static int context_idr_cleanup(int id, void *p, void *data)
 {
-	struct i915_gem_context *ctx = p;
+	context_close(p);
+	return 0;
+}
 
-	context_close(ctx);
+static int vm_idr_cleanup(int id, void *p, void *data)
+{
+	i915_ppgtt_put(p);
 	return 0;
 }
 
+static int gem_context_register(struct i915_gem_context *ctx,
+				struct drm_i915_file_private *fpriv)
+{
+	int ret;
+
+	ctx->file_priv = fpriv;
+	if (ctx->ppgtt)
+		ctx->ppgtt->vm.file = fpriv;
+
+	ctx->pid = get_task_pid(current, PIDTYPE_PID);
+	ctx->name = kasprintf(GFP_KERNEL, "%s[%d]",
+			      current->comm, pid_nr(ctx->pid));
+	if (!ctx->name) {
+		ret = -ENOMEM;
+		goto err_pid;
+	}
+
+	/* And finally expose ourselves to userspace via the idr */
+	mutex_lock(&fpriv->context_idr_lock);
+	ret = idr_alloc(&fpriv->context_idr, ctx, 0, 0, GFP_KERNEL);
+	mutex_unlock(&fpriv->context_idr_lock);
+	if (ret >= 0)
+		goto out;
+
+	kfree(fetch_and_zero(&ctx->name));
+err_pid:
+	put_pid(fetch_and_zero(&ctx->pid));
+out:
+	return ret;
+}
+
 int i915_gem_context_open(struct drm_i915_private *i915,
 			  struct drm_file *file)
 {
 	struct drm_i915_file_private *file_priv = file->driver_priv;
 	struct i915_gem_context *ctx;
+	int err;
+
+	mutex_init(&file_priv->context_idr_lock);
+	mutex_init(&file_priv->vm_idr_lock);
 
 	idr_init(&file_priv->context_idr);
+	idr_init_base(&file_priv->vm_idr, 1);
 
 	mutex_lock(&i915->drm.struct_mutex);
-	ctx = i915_gem_create_context(i915, file_priv);
+	ctx = i915_gem_create_context(i915, 0);
 	mutex_unlock(&i915->drm.struct_mutex);
 	if (IS_ERR(ctx)) {
-		idr_destroy(&file_priv->context_idr);
-		return PTR_ERR(ctx);
+		err = PTR_ERR(ctx);
+		goto err;
 	}
 
+	err = gem_context_register(ctx, file_priv);
+	if (err < 0)
+		goto err_ctx;
+
 	GEM_BUG_ON(i915_gem_context_is_kernel(ctx));
+	GEM_BUG_ON(err > 0);
 
 	return 0;
+
+err_ctx:
+	mutex_lock(&i915->drm.struct_mutex);
+	context_close(ctx);
+	mutex_unlock(&i915->drm.struct_mutex);
+err:
+	idr_destroy(&file_priv->vm_idr);
+	idr_destroy(&file_priv->context_idr);
+	mutex_destroy(&file_priv->vm_idr_lock);
+	mutex_destroy(&file_priv->context_idr_lock);
+	return err;
 }
 
 void i915_gem_context_close(struct drm_file *file)
@@ -659,6 +725,100 @@ void i915_gem_context_close(struct drm_file *file)
 
 	idr_for_each(&file_priv->context_idr, context_idr_cleanup, NULL);
 	idr_destroy(&file_priv->context_idr);
+	mutex_destroy(&file_priv->context_idr_lock);
+
+	idr_for_each(&file_priv->vm_idr, vm_idr_cleanup, NULL);
+	idr_destroy(&file_priv->vm_idr);
+	mutex_destroy(&file_priv->vm_idr_lock);
+}
+
+int i915_gem_vm_create_ioctl(struct drm_device *dev, void *data,
+			     struct drm_file *file)
+{
+	struct drm_i915_private *i915 = to_i915(dev);
+	struct drm_i915_gem_vm_control *args = data;
+	struct drm_i915_file_private *file_priv = file->driver_priv;
+	struct i915_hw_ppgtt *ppgtt;
+	int err;
+
+	if (!HAS_FULL_PPGTT(i915))
+		return -ENODEV;
+
+	if (args->flags)
+		return -EINVAL;
+
+	ppgtt = i915_ppgtt_create(i915);
+	if (IS_ERR(ppgtt))
+		return PTR_ERR(ppgtt);
+
+	ppgtt->vm.file = file_priv;
+
+	if (args->extensions) {
+		err = i915_user_extensions(u64_to_user_ptr(args->extensions),
+					   NULL, 0,
+					   ppgtt);
+		if (err)
+			goto err_put;
+	}
+
+	err = mutex_lock_interruptible(&file_priv->vm_idr_lock);
+	if (err)
+		goto err_put;
+
+	err = idr_alloc(&file_priv->vm_idr, ppgtt, 0, 0, GFP_KERNEL);
+	if (err < 0)
+		goto err_unlock;
+
+	GEM_BUG_ON(err == 0); /* reserved for default/unassigned ppgtt */
+	ppgtt->user_handle = err;
+
+	mutex_unlock(&file_priv->vm_idr_lock);
+
+	args->vm_id = err;
+	return 0;
+
+err_unlock:
+	mutex_unlock(&file_priv->vm_idr_lock);
+err_put:
+	i915_ppgtt_put(ppgtt);
+	return err;
+}
+
+int i915_gem_vm_destroy_ioctl(struct drm_device *dev, void *data,
+			      struct drm_file *file)
+{
+	struct drm_i915_file_private *file_priv = file->driver_priv;
+	struct drm_i915_gem_vm_control *args = data;
+	struct i915_hw_ppgtt *ppgtt;
+	int err;
+	u32 id;
+
+	if (args->flags)
+		return -EINVAL;
+
+	if (args->extensions)
+		return -EINVAL;
+
+	id = args->vm_id;
+	if (!id)
+		return -ENOENT;
+
+	err = mutex_lock_interruptible(&file_priv->vm_idr_lock);
+	if (err)
+		return err;
+
+	ppgtt = idr_remove(&file_priv->vm_idr, id);
+	if (ppgtt) {
+		GEM_BUG_ON(ppgtt->user_handle != id);
+		ppgtt->user_handle = 0;
+	}
+
+	mutex_unlock(&file_priv->vm_idr_lock);
+	if (!ppgtt)
+		return -ENOENT;
+
+	i915_ppgtt_put(ppgtt);
+	return 0;
 }
 
 static struct i915_request *
@@ -671,10 +831,9 @@ last_request_on_engine(struct i915_timeline *timeline,
 
 	rq = i915_active_request_raw(&timeline->last_request,
 				     &engine->i915->drm.struct_mutex);
-	if (rq && rq->engine == engine) {
-		GEM_TRACE("last request for %s on engine %s: %llx:%llu\n",
-			  timeline->name, engine->name,
-			  rq->fence.context, rq->fence.seqno);
+	if (rq && rq->engine->mask & engine->mask) {
+		GEM_TRACE("last request on engine %s: %llx:%llu\n",
+			  engine->name, rq->fence.context, rq->fence.seqno);
 		GEM_BUG_ON(rq->timeline != timeline);
 		return rq;
 	}
@@ -682,81 +841,104 @@ last_request_on_engine(struct i915_timeline *timeline,
 	return NULL;
 }
 
-static bool engine_has_kernel_context_barrier(struct intel_engine_cs *engine)
+struct context_barrier_task {
+	struct i915_active base;
+	void (*task)(void *data);
+	void *data;
+};
+
+static void cb_retire(struct i915_active *base)
+{
+	struct context_barrier_task *cb = container_of(base, typeof(*cb), base);
+
+	if (cb->task)
+		cb->task(cb->data);
+
+	i915_active_fini(&cb->base);
+	kfree(cb);
+}
+
+I915_SELFTEST_DECLARE(static intel_engine_mask_t context_barrier_inject_fault);
+static int context_barrier_task(struct i915_gem_context *ctx,
+				intel_engine_mask_t engines,
+				int (*emit)(struct i915_request *rq, void *data),
+				void (*task)(void *data),
+				void *data)
 {
-	struct drm_i915_private *i915 = engine->i915;
-	const struct intel_context * const ce =
-		to_intel_context(i915->kernel_context, engine);
-	struct i915_timeline *barrier = ce->ring->timeline;
-	struct intel_ring *ring;
-	bool any_active = false;
+	struct drm_i915_private *i915 = ctx->i915;
+	struct context_barrier_task *cb;
+	struct intel_context *ce, *next;
+	intel_wakeref_t wakeref;
+	int err = 0;
 
 	lockdep_assert_held(&i915->drm.struct_mutex);
-	list_for_each_entry(ring, &i915->gt.active_rings, active_link) {
-		struct i915_request *rq;
+	GEM_BUG_ON(!task);
 
-		rq = last_request_on_engine(ring->timeline, engine);
-		if (!rq)
-			continue;
+	cb = kmalloc(sizeof(*cb), GFP_KERNEL);
+	if (!cb)
+		return -ENOMEM;
 
-		any_active = true;
+	i915_active_init(i915, &cb->base, cb_retire);
+	i915_active_acquire(&cb->base);
 
-		if (rq->hw_context == ce)
+	wakeref = intel_runtime_pm_get(i915);
+	rbtree_postorder_for_each_entry_safe(ce, next, &ctx->hw_contexts, node) {
+		struct intel_engine_cs *engine = ce->engine;
+		struct i915_request *rq;
+
+		if (!(engine->mask & engines))
 			continue;
 
-		/*
-		 * Was this request submitted after the previous
-		 * switch-to-kernel-context?
-		 */
-		if (!i915_timeline_sync_is_later(barrier, &rq->fence)) {
-			GEM_TRACE("%s needs barrier for %llx:%lld\n",
-				  ring->timeline->name,
-				  rq->fence.context,
-				  rq->fence.seqno);
-			return false;
+		if (I915_SELFTEST_ONLY(context_barrier_inject_fault &
+				       engine->mask)) {
+			err = -ENXIO;
+			break;
+		}
+
+		rq = i915_request_alloc(engine, ctx);
+		if (IS_ERR(rq)) {
+			err = PTR_ERR(rq);
+			break;
 		}
 
-		GEM_TRACE("%s has barrier after %llx:%lld\n",
-			  ring->timeline->name,
-			  rq->fence.context,
-			  rq->fence.seqno);
+		err = 0;
+		if (emit)
+			err = emit(rq, data);
+		if (err == 0)
+			err = i915_active_ref(&cb->base, rq->fence.context, rq);
+
+		i915_request_add(rq);
+		if (err)
+			break;
 	}
+	intel_runtime_pm_put(i915, wakeref);
 
-	/*
-	 * If any other timeline was still active and behind the last barrier,
-	 * then our last switch-to-kernel-context must still be queued and
-	 * will run last (leaving the engine in the kernel context when it
-	 * eventually idles).
-	 */
-	if (any_active)
-		return true;
+	cb->task = err ? NULL : task; /* caller needs to unwind instead */
+	cb->data = data;
 
-	/* The engine is idle; check that it is idling in the kernel context. */
-	return engine->last_retired_context == ce;
+	i915_active_release(&cb->base);
+
+	return err;
 }
 
-int i915_gem_switch_to_kernel_context(struct drm_i915_private *i915)
+int i915_gem_switch_to_kernel_context(struct drm_i915_private *i915,
+				      intel_engine_mask_t mask)
 {
 	struct intel_engine_cs *engine;
-	enum intel_engine_id id;
 
 	GEM_TRACE("awake?=%s\n", yesno(i915->gt.awake));
 
 	lockdep_assert_held(&i915->drm.struct_mutex);
 	GEM_BUG_ON(!i915->kernel_context);
 
-	i915_retire_requests(i915);
+	/* Inoperable, so presume the GPU is safely pointing into the void! */
+	if (i915_terminally_wedged(i915))
+		return 0;
 
-	for_each_engine(engine, i915, id) {
+	for_each_engine_masked(engine, i915, mask, mask) {
 		struct intel_ring *ring;
 		struct i915_request *rq;
 
-		GEM_BUG_ON(!to_intel_context(i915->kernel_context, engine));
-		if (engine_has_kernel_context_barrier(engine))
-			continue;
-
-		GEM_TRACE("emit barrier on %s\n", engine->name);
-
 		rq = i915_request_alloc(engine, i915->kernel_context);
 		if (IS_ERR(rq))
 			return PTR_ERR(rq);
@@ -779,7 +961,6 @@ int i915_gem_switch_to_kernel_context(struct drm_i915_private *i915)
 			i915_sw_fence_await_sw_fence_gfp(&rq->submit,
 							 &prev->submit,
 							 I915_FENCE_GFP);
-			i915_timeline_sync_set(rq->timeline, &prev->fence);
 		}
 
 		i915_request_add(rq);
@@ -788,183 +969,173 @@ int i915_gem_switch_to_kernel_context(struct drm_i915_private *i915)
 	return 0;
 }
 
-static bool client_is_banned(struct drm_i915_file_private *file_priv)
+static int get_ppgtt(struct drm_i915_file_private *file_priv,
+		     struct i915_gem_context *ctx,
+		     struct drm_i915_gem_context_param *args)
 {
-	return atomic_read(&file_priv->ban_score) >= I915_CLIENT_SCORE_BANNED;
-}
-
-int i915_gem_context_create_ioctl(struct drm_device *dev, void *data,
-				  struct drm_file *file)
-{
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct drm_i915_gem_context_create *args = data;
-	struct drm_i915_file_private *file_priv = file->driver_priv;
-	struct i915_gem_context *ctx;
+	struct i915_hw_ppgtt *ppgtt;
 	int ret;
 
-	if (!DRIVER_CAPS(dev_priv)->has_logical_contexts)
+	return -EINVAL; /* nothing to see here; please move along */
+
+	if (!ctx->ppgtt)
 		return -ENODEV;
 
-	if (args->pad != 0)
-		return -EINVAL;
+	/* XXX rcu acquire? */
+	ret = mutex_lock_interruptible(&ctx->i915->drm.struct_mutex);
+	if (ret)
+		return ret;
 
-	if (client_is_banned(file_priv)) {
-		DRM_DEBUG("client %s[%d] banned from creating ctx\n",
-			  current->comm,
-			  pid_nr(get_task_pid(current, PIDTYPE_PID)));
+	ppgtt = i915_ppgtt_get(ctx->ppgtt);
+	mutex_unlock(&ctx->i915->drm.struct_mutex);
 
-		return -EIO;
+	ret = mutex_lock_interruptible(&file_priv->vm_idr_lock);
+	if (ret)
+		goto err_put;
+
+	if (!ppgtt->user_handle) {
+		ret = idr_alloc(&file_priv->vm_idr, ppgtt, 0, 0, GFP_KERNEL);
+		GEM_BUG_ON(!ret);
+		if (ret < 0)
+			goto err_unlock;
+
+		ppgtt->user_handle = ret;
+		i915_ppgtt_get(ppgtt);
 	}
 
-	ret = i915_mutex_lock_interruptible(dev);
-	if (ret)
-		return ret;
+	args->size = 0;
+	args->value = ppgtt->user_handle;
 
-	ctx = i915_gem_create_context(dev_priv, file_priv);
-	mutex_unlock(&dev->struct_mutex);
-	if (IS_ERR(ctx))
-		return PTR_ERR(ctx);
+	ret = 0;
+err_unlock:
+	mutex_unlock(&file_priv->vm_idr_lock);
+err_put:
+	i915_ppgtt_put(ppgtt);
+	return ret;
+}
 
-	GEM_BUG_ON(i915_gem_context_is_kernel(ctx));
+static void set_ppgtt_barrier(void *data)
+{
+	struct i915_hw_ppgtt *old = data;
 
-	args->ctx_id = ctx->user_handle;
-	DRM_DEBUG("HW context %d created\n", args->ctx_id);
+	if (INTEL_GEN(old->vm.i915) < 8)
+		gen6_ppgtt_unpin_all(old);
 
-	return 0;
+	i915_ppgtt_put(old);
 }
 
-int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data,
-				   struct drm_file *file)
+static int emit_ppgtt_update(struct i915_request *rq, void *data)
 {
-	struct drm_i915_gem_context_destroy *args = data;
-	struct drm_i915_file_private *file_priv = file->driver_priv;
-	struct i915_gem_context *ctx;
-	int ret;
+	struct i915_hw_ppgtt *ppgtt = rq->gem_context->ppgtt;
+	struct intel_engine_cs *engine = rq->engine;
+	u32 base = engine->mmio_base;
+	u32 *cs;
+	int i;
 
-	if (args->pad != 0)
-		return -EINVAL;
+	if (i915_vm_is_4lvl(&ppgtt->vm)) {
+		const dma_addr_t pd_daddr = px_dma(&ppgtt->pml4);
 
-	if (args->ctx_id == DEFAULT_CONTEXT_HANDLE)
-		return -ENOENT;
+		cs = intel_ring_begin(rq, 6);
+		if (IS_ERR(cs))
+			return PTR_ERR(cs);
 
-	ctx = i915_gem_context_lookup(file_priv, args->ctx_id);
-	if (!ctx)
-		return -ENOENT;
+		*cs++ = MI_LOAD_REGISTER_IMM(2);
 
-	ret = mutex_lock_interruptible(&dev->struct_mutex);
-	if (ret)
-		goto out;
+		*cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(base, 0));
+		*cs++ = upper_32_bits(pd_daddr);
+		*cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(base, 0));
+		*cs++ = lower_32_bits(pd_daddr);
 
-	__destroy_hw_context(ctx, file_priv);
-	mutex_unlock(&dev->struct_mutex);
+		*cs++ = MI_NOOP;
+		intel_ring_advance(rq, cs);
+	} else if (HAS_LOGICAL_RING_CONTEXTS(engine->i915)) {
+		cs = intel_ring_begin(rq, 4 * GEN8_3LVL_PDPES + 2);
+		if (IS_ERR(cs))
+			return PTR_ERR(cs);
+
+		*cs++ = MI_LOAD_REGISTER_IMM(2 * GEN8_3LVL_PDPES);
+		for (i = GEN8_3LVL_PDPES; i--; ) {
+			const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i);
+
+			*cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(base, i));
+			*cs++ = upper_32_bits(pd_daddr);
+			*cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(base, i));
+			*cs++ = lower_32_bits(pd_daddr);
+		}
+		*cs++ = MI_NOOP;
+		intel_ring_advance(rq, cs);
+	} else {
+		/* ppGTT is not part of the legacy context image */
+		gen6_ppgtt_pin(ppgtt);
+	}
 
-out:
-	i915_gem_context_put(ctx);
 	return 0;
 }
 
-static int get_sseu(struct i915_gem_context *ctx,
-		    struct drm_i915_gem_context_param *args)
+static int set_ppgtt(struct drm_i915_file_private *file_priv,
+		     struct i915_gem_context *ctx,
+		     struct drm_i915_gem_context_param *args)
 {
-	struct drm_i915_gem_context_param_sseu user_sseu;
-	struct intel_engine_cs *engine;
-	struct intel_context *ce;
-	int ret;
-
-	if (args->size == 0)
-		goto out;
-	else if (args->size < sizeof(user_sseu))
-		return -EINVAL;
-
-	if (copy_from_user(&user_sseu, u64_to_user_ptr(args->value),
-			   sizeof(user_sseu)))
-		return -EFAULT;
+	struct i915_hw_ppgtt *ppgtt, *old;
+	int err;
 
-	if (user_sseu.flags || user_sseu.rsvd)
-		return -EINVAL;
+	return -EINVAL; /* nothing to see here; please move along */
 
-	engine = intel_engine_lookup_user(ctx->i915,
-					  user_sseu.engine_class,
-					  user_sseu.engine_instance);
-	if (!engine)
+	if (args->size)
 		return -EINVAL;
 
-	/* Only use for mutex here is to serialize get_param and set_param. */
-	ret = mutex_lock_interruptible(&ctx->i915->drm.struct_mutex);
-	if (ret)
-		return ret;
-
-	ce = to_intel_context(ctx, engine);
-
-	user_sseu.slice_mask = ce->sseu.slice_mask;
-	user_sseu.subslice_mask = ce->sseu.subslice_mask;
-	user_sseu.min_eus_per_subslice = ce->sseu.min_eus_per_subslice;
-	user_sseu.max_eus_per_subslice = ce->sseu.max_eus_per_subslice;
+	if (!ctx->ppgtt)
+		return -ENODEV;
 
-	mutex_unlock(&ctx->i915->drm.struct_mutex);
+	if (upper_32_bits(args->value))
+		return -ENOENT;
 
-	if (copy_to_user(u64_to_user_ptr(args->value), &user_sseu,
-			 sizeof(user_sseu)))
-		return -EFAULT;
+	err = mutex_lock_interruptible(&file_priv->vm_idr_lock);
+	if (err)
+		return err;
 
-out:
-	args->size = sizeof(user_sseu);
+	ppgtt = idr_find(&file_priv->vm_idr, args->value);
+	if (ppgtt) {
+		GEM_BUG_ON(ppgtt->user_handle != args->value);
+		i915_ppgtt_get(ppgtt);
+	}
+	mutex_unlock(&file_priv->vm_idr_lock);
+	if (!ppgtt)
+		return -ENOENT;
 
-	return 0;
-}
+	err = mutex_lock_interruptible(&ctx->i915->drm.struct_mutex);
+	if (err)
+		goto out;
 
-int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data,
-				    struct drm_file *file)
-{
-	struct drm_i915_file_private *file_priv = file->driver_priv;
-	struct drm_i915_gem_context_param *args = data;
-	struct i915_gem_context *ctx;
-	int ret = 0;
+	if (ppgtt == ctx->ppgtt)
+		goto unlock;
 
-	ctx = i915_gem_context_lookup(file_priv, args->ctx_id);
-	if (!ctx)
-		return -ENOENT;
+	/* Teardown the existing obj:vma cache, it will have to be rebuilt. */
+	lut_close(ctx);
 
-	switch (args->param) {
-	case I915_CONTEXT_PARAM_BAN_PERIOD:
-		ret = -EINVAL;
-		break;
-	case I915_CONTEXT_PARAM_NO_ZEROMAP:
-		args->size = 0;
-		args->value = test_bit(UCONTEXT_NO_ZEROMAP, &ctx->user_flags);
-		break;
-	case I915_CONTEXT_PARAM_GTT_SIZE:
-		args->size = 0;
+	old = __set_ppgtt(ctx, ppgtt);
 
-		if (ctx->ppgtt)
-			args->value = ctx->ppgtt->vm.total;
-		else if (to_i915(dev)->mm.aliasing_ppgtt)
-			args->value = to_i915(dev)->mm.aliasing_ppgtt->vm.total;
-		else
-			args->value = to_i915(dev)->ggtt.vm.total;
-		break;
-	case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE:
-		args->size = 0;
-		args->value = i915_gem_context_no_error_capture(ctx);
-		break;
-	case I915_CONTEXT_PARAM_BANNABLE:
-		args->size = 0;
-		args->value = i915_gem_context_is_bannable(ctx);
-		break;
-	case I915_CONTEXT_PARAM_PRIORITY:
-		args->size = 0;
-		args->value = ctx->sched.priority >> I915_USER_PRIORITY_SHIFT;
-		break;
-	case I915_CONTEXT_PARAM_SSEU:
-		ret = get_sseu(ctx, args);
-		break;
-	default:
-		ret = -EINVAL;
-		break;
+	/*
+	 * We need to flush any requests using the current ppgtt before
+	 * we release it as the requests do not hold a reference themselves,
+	 * only indirectly through the context.
+	 */
+	err = context_barrier_task(ctx, ALL_ENGINES,
+				   emit_ppgtt_update,
+				   set_ppgtt_barrier,
+				   old);
+	if (err) {
+		ctx->ppgtt = old;
+		ctx->desc_template = default_desc_template(ctx->i915, old);
+		i915_ppgtt_put(ppgtt);
 	}
 
-	i915_gem_context_put(ctx);
-	return ret;
+unlock:
+	mutex_unlock(&ctx->i915->drm.struct_mutex);
+
+out:
+	i915_ppgtt_put(ppgtt);
+	return err;
 }
 
 static int gen8_emit_rpcs_config(struct i915_request *rq,
@@ -993,39 +1164,35 @@ static int gen8_emit_rpcs_config(struct i915_request *rq,
 }
 
 static int
-gen8_modify_rpcs_gpu(struct intel_context *ce,
-		     struct intel_engine_cs *engine,
-		     struct intel_sseu sseu)
+gen8_modify_rpcs(struct intel_context *ce, struct intel_sseu sseu)
 {
-	struct drm_i915_private *i915 = engine->i915;
-	struct i915_request *rq, *prev;
+	struct drm_i915_private *i915 = ce->engine->i915;
+	struct i915_request *rq;
 	intel_wakeref_t wakeref;
 	int ret;
 
-	GEM_BUG_ON(!ce->pin_count);
+	lockdep_assert_held(&ce->pin_mutex);
 
-	lockdep_assert_held(&i915->drm.struct_mutex);
+	/*
+	 * If the context is not idle, we have to submit an ordered request to
+	 * modify its context image via the kernel context (writing to our own
+	 * image, or into the registers directory, does not stick). Pristine
+	 * and idle contexts will be configured on pinning.
+	 */
+	if (!intel_context_is_pinned(ce))
+		return 0;
 
 	/* Submitting requests etc needs the hw awake. */
 	wakeref = intel_runtime_pm_get(i915);
 
-	rq = i915_request_alloc(engine, i915->kernel_context);
+	rq = i915_request_alloc(ce->engine, i915->kernel_context);
 	if (IS_ERR(rq)) {
 		ret = PTR_ERR(rq);
 		goto out_put;
 	}
 
 	/* Queue this switch after all other activity by this context. */
-	prev = i915_active_request_raw(&ce->ring->timeline->last_request,
-				       &i915->drm.struct_mutex);
-	if (prev && !i915_request_completed(prev)) {
-		ret = i915_request_await_dma_fence(rq, &prev->fence);
-		if (ret < 0)
-			goto out_add;
-	}
-
-	/* Order all following requests to be after. */
-	ret = i915_timeline_set_barrier(ce->ring->timeline, rq);
+	ret = i915_active_request_set(&ce->ring->timeline->last_request, rq);
 	if (ret)
 		goto out_add;
 
@@ -1057,27 +1224,26 @@ __i915_gem_context_reconfigure_sseu(struct i915_gem_context *ctx,
 				    struct intel_engine_cs *engine,
 				    struct intel_sseu sseu)
 {
-	struct intel_context *ce = to_intel_context(ctx, engine);
+	struct intel_context *ce;
 	int ret = 0;
 
 	GEM_BUG_ON(INTEL_GEN(ctx->i915) < 8);
-	GEM_BUG_ON(engine->id != RCS);
+	GEM_BUG_ON(engine->id != RCS0);
+
+	ce = intel_context_pin_lock(ctx, engine);
+	if (IS_ERR(ce))
+		return PTR_ERR(ce);
 
 	/* Nothing to do if unmodified. */
 	if (!memcmp(&ce->sseu, &sseu, sizeof(sseu)))
-		return 0;
-
-	/*
-	 * If context is not idle we have to submit an ordered request to modify
-	 * its context image via the kernel context. Pristine and idle contexts
-	 * will be configured on pinning.
-	 */
-	if (ce->pin_count)
-		ret = gen8_modify_rpcs_gpu(ce, engine, sseu);
+		goto unlock;
 
+	ret = gen8_modify_rpcs(ce, sseu);
 	if (!ret)
 		ce->sseu = sseu;
 
+unlock:
+	intel_context_pin_unlock(ce);
 	return ret;
 }
 
@@ -1220,8 +1386,8 @@ static int set_sseu(struct i915_gem_context *ctx,
 		return -EINVAL;
 
 	engine = intel_engine_lookup_user(i915,
-					  user_sseu.engine_class,
-					  user_sseu.engine_instance);
+					  user_sseu.engine.engine_class,
+					  user_sseu.engine.engine_instance);
 	if (!engine)
 		return -EINVAL;
 
@@ -1242,22 +1408,13 @@ static int set_sseu(struct i915_gem_context *ctx,
 	return 0;
 }
 
-int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data,
-				    struct drm_file *file)
+static int ctx_setparam(struct drm_i915_file_private *fpriv,
+			struct i915_gem_context *ctx,
+			struct drm_i915_gem_context_param *args)
 {
-	struct drm_i915_file_private *file_priv = file->driver_priv;
-	struct drm_i915_gem_context_param *args = data;
-	struct i915_gem_context *ctx;
 	int ret = 0;
 
-	ctx = i915_gem_context_lookup(file_priv, args->ctx_id);
-	if (!ctx)
-		return -ENOENT;
-
 	switch (args->param) {
-	case I915_CONTEXT_PARAM_BAN_PERIOD:
-		ret = -EINVAL;
-		break;
 	case I915_CONTEXT_PARAM_NO_ZEROMAP:
 		if (args->size)
 			ret = -EINVAL;
@@ -1266,6 +1423,7 @@ int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data,
 		else
 			clear_bit(UCONTEXT_NO_ZEROMAP, &ctx->user_flags);
 		break;
+
 	case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE:
 		if (args->size)
 			ret = -EINVAL;
@@ -1274,6 +1432,7 @@ int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data,
 		else
 			i915_gem_context_clear_no_error_capture(ctx);
 		break;
+
 	case I915_CONTEXT_PARAM_BANNABLE:
 		if (args->size)
 			ret = -EINVAL;
@@ -1285,13 +1444,22 @@ int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data,
 			i915_gem_context_clear_bannable(ctx);
 		break;
 
+	case I915_CONTEXT_PARAM_RECOVERABLE:
+		if (args->size)
+			ret = -EINVAL;
+		else if (args->value)
+			i915_gem_context_set_recoverable(ctx);
+		else
+			i915_gem_context_clear_recoverable(ctx);
+		break;
+
 	case I915_CONTEXT_PARAM_PRIORITY:
 		{
 			s64 priority = args->value;
 
 			if (args->size)
 				ret = -EINVAL;
-			else if (!(to_i915(dev)->caps.scheduler & I915_SCHEDULER_CAP_PRIORITY))
+			else if (!(ctx->i915->caps.scheduler & I915_SCHEDULER_CAP_PRIORITY))
 				ret = -ENODEV;
 			else if (priority > I915_CONTEXT_MAX_USER_PRIORITY ||
 				 priority < I915_CONTEXT_MIN_USER_PRIORITY)
@@ -1304,14 +1472,266 @@ int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data,
 					I915_USER_PRIORITY(priority);
 		}
 		break;
+
 	case I915_CONTEXT_PARAM_SSEU:
 		ret = set_sseu(ctx, args);
 		break;
+
+	case I915_CONTEXT_PARAM_VM:
+		ret = set_ppgtt(fpriv, ctx, args);
+		break;
+
+	case I915_CONTEXT_PARAM_BAN_PERIOD:
 	default:
 		ret = -EINVAL;
 		break;
 	}
 
+	return ret;
+}
+
+struct create_ext {
+	struct i915_gem_context *ctx;
+	struct drm_i915_file_private *fpriv;
+};
+
+static int create_setparam(struct i915_user_extension __user *ext, void *data)
+{
+	struct drm_i915_gem_context_create_ext_setparam local;
+	const struct create_ext *arg = data;
+
+	if (copy_from_user(&local, ext, sizeof(local)))
+		return -EFAULT;
+
+	if (local.param.ctx_id)
+		return -EINVAL;
+
+	return ctx_setparam(arg->fpriv, arg->ctx, &local.param);
+}
+
+static const i915_user_extension_fn create_extensions[] = {
+	[I915_CONTEXT_CREATE_EXT_SETPARAM] = create_setparam,
+};
+
+static bool client_is_banned(struct drm_i915_file_private *file_priv)
+{
+	return atomic_read(&file_priv->ban_score) >= I915_CLIENT_SCORE_BANNED;
+}
+
+int i915_gem_context_create_ioctl(struct drm_device *dev, void *data,
+				  struct drm_file *file)
+{
+	struct drm_i915_private *i915 = to_i915(dev);
+	struct drm_i915_gem_context_create_ext *args = data;
+	struct create_ext ext_data;
+	int ret;
+
+	if (!DRIVER_CAPS(i915)->has_logical_contexts)
+		return -ENODEV;
+
+	if (args->flags & I915_CONTEXT_CREATE_FLAGS_UNKNOWN)
+		return -EINVAL;
+
+	ret = i915_terminally_wedged(i915);
+	if (ret)
+		return ret;
+
+	ext_data.fpriv = file->driver_priv;
+	if (client_is_banned(ext_data.fpriv)) {
+		DRM_DEBUG("client %s[%d] banned from creating ctx\n",
+			  current->comm,
+			  pid_nr(get_task_pid(current, PIDTYPE_PID)));
+		return -EIO;
+	}
+
+	ret = i915_mutex_lock_interruptible(dev);
+	if (ret)
+		return ret;
+
+	ext_data.ctx = i915_gem_create_context(i915, args->flags);
+	mutex_unlock(&dev->struct_mutex);
+	if (IS_ERR(ext_data.ctx))
+		return PTR_ERR(ext_data.ctx);
+
+	if (args->flags & I915_CONTEXT_CREATE_FLAGS_USE_EXTENSIONS) {
+		ret = i915_user_extensions(u64_to_user_ptr(args->extensions),
+					   create_extensions,
+					   ARRAY_SIZE(create_extensions),
+					   &ext_data);
+		if (ret)
+			goto err_ctx;
+	}
+
+	ret = gem_context_register(ext_data.ctx, ext_data.fpriv);
+	if (ret < 0)
+		goto err_ctx;
+
+	args->ctx_id = ret;
+	DRM_DEBUG("HW context %d created\n", args->ctx_id);
+
+	return 0;
+
+err_ctx:
+	mutex_lock(&dev->struct_mutex);
+	context_close(ext_data.ctx);
+	mutex_unlock(&dev->struct_mutex);
+	return ret;
+}
+
+int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data,
+				   struct drm_file *file)
+{
+	struct drm_i915_gem_context_destroy *args = data;
+	struct drm_i915_file_private *file_priv = file->driver_priv;
+	struct i915_gem_context *ctx;
+
+	if (args->pad != 0)
+		return -EINVAL;
+
+	if (!args->ctx_id)
+		return -ENOENT;
+
+	if (mutex_lock_interruptible(&file_priv->context_idr_lock))
+		return -EINTR;
+
+	ctx = idr_remove(&file_priv->context_idr, args->ctx_id);
+	mutex_unlock(&file_priv->context_idr_lock);
+	if (!ctx)
+		return -ENOENT;
+
+	mutex_lock(&dev->struct_mutex);
+	context_close(ctx);
+	mutex_unlock(&dev->struct_mutex);
+
+	return 0;
+}
+
+static int get_sseu(struct i915_gem_context *ctx,
+		    struct drm_i915_gem_context_param *args)
+{
+	struct drm_i915_gem_context_param_sseu user_sseu;
+	struct intel_engine_cs *engine;
+	struct intel_context *ce;
+
+	if (args->size == 0)
+		goto out;
+	else if (args->size < sizeof(user_sseu))
+		return -EINVAL;
+
+	if (copy_from_user(&user_sseu, u64_to_user_ptr(args->value),
+			   sizeof(user_sseu)))
+		return -EFAULT;
+
+	if (user_sseu.flags || user_sseu.rsvd)
+		return -EINVAL;
+
+	engine = intel_engine_lookup_user(ctx->i915,
+					  user_sseu.engine.engine_class,
+					  user_sseu.engine.engine_instance);
+	if (!engine)
+		return -EINVAL;
+
+	ce = intel_context_pin_lock(ctx, engine); /* serialises with set_sseu */
+	if (IS_ERR(ce))
+		return PTR_ERR(ce);
+
+	user_sseu.slice_mask = ce->sseu.slice_mask;
+	user_sseu.subslice_mask = ce->sseu.subslice_mask;
+	user_sseu.min_eus_per_subslice = ce->sseu.min_eus_per_subslice;
+	user_sseu.max_eus_per_subslice = ce->sseu.max_eus_per_subslice;
+
+	intel_context_pin_unlock(ce);
+
+	if (copy_to_user(u64_to_user_ptr(args->value), &user_sseu,
+			 sizeof(user_sseu)))
+		return -EFAULT;
+
+out:
+	args->size = sizeof(user_sseu);
+
+	return 0;
+}
+
+int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data,
+				    struct drm_file *file)
+{
+	struct drm_i915_file_private *file_priv = file->driver_priv;
+	struct drm_i915_gem_context_param *args = data;
+	struct i915_gem_context *ctx;
+	int ret = 0;
+
+	ctx = i915_gem_context_lookup(file_priv, args->ctx_id);
+	if (!ctx)
+		return -ENOENT;
+
+	switch (args->param) {
+	case I915_CONTEXT_PARAM_NO_ZEROMAP:
+		args->size = 0;
+		args->value = test_bit(UCONTEXT_NO_ZEROMAP, &ctx->user_flags);
+		break;
+
+	case I915_CONTEXT_PARAM_GTT_SIZE:
+		args->size = 0;
+		if (ctx->ppgtt)
+			args->value = ctx->ppgtt->vm.total;
+		else if (to_i915(dev)->mm.aliasing_ppgtt)
+			args->value = to_i915(dev)->mm.aliasing_ppgtt->vm.total;
+		else
+			args->value = to_i915(dev)->ggtt.vm.total;
+		break;
+
+	case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE:
+		args->size = 0;
+		args->value = i915_gem_context_no_error_capture(ctx);
+		break;
+
+	case I915_CONTEXT_PARAM_BANNABLE:
+		args->size = 0;
+		args->value = i915_gem_context_is_bannable(ctx);
+		break;
+
+	case I915_CONTEXT_PARAM_RECOVERABLE:
+		args->size = 0;
+		args->value = i915_gem_context_is_recoverable(ctx);
+		break;
+
+	case I915_CONTEXT_PARAM_PRIORITY:
+		args->size = 0;
+		args->value = ctx->sched.priority >> I915_USER_PRIORITY_SHIFT;
+		break;
+
+	case I915_CONTEXT_PARAM_SSEU:
+		ret = get_sseu(ctx, args);
+		break;
+
+	case I915_CONTEXT_PARAM_VM:
+		ret = get_ppgtt(file_priv, ctx, args);
+		break;
+
+	case I915_CONTEXT_PARAM_BAN_PERIOD:
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	i915_gem_context_put(ctx);
+	return ret;
+}
+
+int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data,
+				    struct drm_file *file)
+{
+	struct drm_i915_file_private *file_priv = file->driver_priv;
+	struct drm_i915_gem_context_param *args = data;
+	struct i915_gem_context *ctx;
+	int ret;
+
+	ctx = i915_gem_context_lookup(file_priv, args->ctx_id);
+	if (!ctx)
+		return -ENOENT;
+
+	ret = ctx_setparam(file_priv, ctx, args);
+
 	i915_gem_context_put(ctx);
 	return ret;
 }
@@ -1385,3 +1805,28 @@ out_unlock:
 #include "selftests/mock_context.c"
 #include "selftests/i915_gem_context.c"
 #endif
+
+static void i915_global_gem_context_shrink(void)
+{
+	kmem_cache_shrink(global.slab_luts);
+}
+
+static void i915_global_gem_context_exit(void)
+{
+	kmem_cache_destroy(global.slab_luts);
+}
+
+static struct i915_global_gem_context global = { {
+	.shrink = i915_global_gem_context_shrink,
+	.exit = i915_global_gem_context_exit,
+} };
+
+int __init i915_global_gem_context_init(void)
+{
+	global.slab_luts = KMEM_CACHE(i915_lut_handle, 0);
+	if (!global.slab_luts)
+		return -ENOMEM;
+
+	i915_global_register(&global.base);
+	return 0;
+}
diff --git a/drivers/gpu/drm/i915/i915_gem_context.h b/drivers/gpu/drm/i915/i915_gem_context.h
index ca150a764c24..23dcb01bfd82 100644
--- a/drivers/gpu/drm/i915/i915_gem_context.h
+++ b/drivers/gpu/drm/i915/i915_gem_context.h
@@ -25,210 +25,16 @@
 #ifndef __I915_GEM_CONTEXT_H__
 #define __I915_GEM_CONTEXT_H__
 
-#include <linux/bitops.h>
-#include <linux/list.h>
-#include <linux/radix-tree.h>
+#include "i915_gem_context_types.h"
 
 #include "i915_gem.h"
 #include "i915_scheduler.h"
+#include "intel_context.h"
 #include "intel_device_info.h"
 
-struct pid;
-
 struct drm_device;
 struct drm_file;
 
-struct drm_i915_private;
-struct drm_i915_file_private;
-struct i915_hw_ppgtt;
-struct i915_request;
-struct i915_vma;
-struct intel_ring;
-
-#define DEFAULT_CONTEXT_HANDLE 0
-
-struct intel_context;
-
-struct intel_context_ops {
-	void (*unpin)(struct intel_context *ce);
-	void (*destroy)(struct intel_context *ce);
-};
-
-/*
- * Powergating configuration for a particular (context,engine).
- */
-struct intel_sseu {
-	u8 slice_mask;
-	u8 subslice_mask;
-	u8 min_eus_per_subslice;
-	u8 max_eus_per_subslice;
-};
-
-/**
- * struct i915_gem_context - client state
- *
- * The struct i915_gem_context represents the combined view of the driver and
- * logical hardware state for a particular client.
- */
-struct i915_gem_context {
-	/** i915: i915 device backpointer */
-	struct drm_i915_private *i915;
-
-	/** file_priv: owning file descriptor */
-	struct drm_i915_file_private *file_priv;
-
-	/**
-	 * @ppgtt: unique address space (GTT)
-	 *
-	 * In full-ppgtt mode, each context has its own address space ensuring
-	 * complete seperation of one client from all others.
-	 *
-	 * In other modes, this is a NULL pointer with the expectation that
-	 * the caller uses the shared global GTT.
-	 */
-	struct i915_hw_ppgtt *ppgtt;
-
-	/**
-	 * @pid: process id of creator
-	 *
-	 * Note that who created the context may not be the principle user,
-	 * as the context may be shared across a local socket. However,
-	 * that should only affect the default context, all contexts created
-	 * explicitly by the client are expected to be isolated.
-	 */
-	struct pid *pid;
-
-	/**
-	 * @name: arbitrary name
-	 *
-	 * A name is constructed for the context from the creator's process
-	 * name, pid and user handle in order to uniquely identify the
-	 * context in messages.
-	 */
-	const char *name;
-
-	/** link: place with &drm_i915_private.context_list */
-	struct list_head link;
-	struct llist_node free_link;
-
-	/**
-	 * @ref: reference count
-	 *
-	 * A reference to a context is held by both the client who created it
-	 * and on each request submitted to the hardware using the request
-	 * (to ensure the hardware has access to the state until it has
-	 * finished all pending writes). See i915_gem_context_get() and
-	 * i915_gem_context_put() for access.
-	 */
-	struct kref ref;
-
-	/**
-	 * @rcu: rcu_head for deferred freeing.
-	 */
-	struct rcu_head rcu;
-
-	/**
-	 * @user_flags: small set of booleans controlled by the user
-	 */
-	unsigned long user_flags;
-#define UCONTEXT_NO_ZEROMAP		0
-#define UCONTEXT_NO_ERROR_CAPTURE	1
-#define UCONTEXT_BANNABLE		2
-
-	/**
-	 * @flags: small set of booleans
-	 */
-	unsigned long flags;
-#define CONTEXT_BANNED			0
-#define CONTEXT_CLOSED			1
-#define CONTEXT_FORCE_SINGLE_SUBMISSION	2
-
-	/**
-	 * @hw_id: - unique identifier for the context
-	 *
-	 * The hardware needs to uniquely identify the context for a few
-	 * functions like fault reporting, PASID, scheduling. The
-	 * &drm_i915_private.context_hw_ida is used to assign a unqiue
-	 * id for the lifetime of the context.
-	 *
-	 * @hw_id_pin_count: - number of times this context had been pinned
-	 * for use (should be, at most, once per engine).
-	 *
-	 * @hw_id_link: - all contexts with an assigned id are tracked
-	 * for possible repossession.
-	 */
-	unsigned int hw_id;
-	atomic_t hw_id_pin_count;
-	struct list_head hw_id_link;
-
-	/**
-	 * @user_handle: userspace identifier
-	 *
-	 * A unique per-file identifier is generated from
-	 * &drm_i915_file_private.contexts.
-	 */
-	u32 user_handle;
-
-	struct i915_sched_attr sched;
-
-	/** engine: per-engine logical HW state */
-	struct intel_context {
-		struct i915_gem_context *gem_context;
-		struct intel_engine_cs *active;
-		struct list_head signal_link;
-		struct list_head signals;
-		struct i915_vma *state;
-		struct intel_ring *ring;
-		u32 *lrc_reg_state;
-		u64 lrc_desc;
-		int pin_count;
-
-		/**
-		 * active_tracker: Active tracker for the external rq activity
-		 * on this intel_context object.
-		 */
-		struct i915_active_request active_tracker;
-
-		const struct intel_context_ops *ops;
-
-		/** sseu: Control eu/slice partitioning */
-		struct intel_sseu sseu;
-	} __engine[I915_NUM_ENGINES];
-
-	/** ring_size: size for allocating the per-engine ring buffer */
-	u32 ring_size;
-	/** desc_template: invariant fields for the HW context descriptor */
-	u32 desc_template;
-
-	/** guilty_count: How many times this context has caused a GPU hang. */
-	atomic_t guilty_count;
-	/**
-	 * @active_count: How many times this context was active during a GPU
-	 * hang, but did not cause it.
-	 */
-	atomic_t active_count;
-
-#define CONTEXT_SCORE_GUILTY		10
-#define CONTEXT_SCORE_BAN_THRESHOLD	40
-	/** ban_score: Accumulated score of all hangs caused by this context. */
-	atomic_t ban_score;
-
-	/** remap_slice: Bitmask of cache lines that need remapping */
-	u8 remap_slice;
-
-	/** handles_vma: rbtree to look up our context specific obj/vma for
-	 * the user handle. (user handles are per fd, but the binding is
-	 * per vm, which may be one per context or shared with the global GTT)
-	 */
-	struct radix_tree_root handles_vma;
-
-	/** handles_list: reverse list of all the rbtree entries in use for
-	 * this context, which allows us to free all the allocations on
-	 * context close.
-	 */
-	struct list_head handles_list;
-};
-
 static inline bool i915_gem_context_is_closed(const struct i915_gem_context *ctx)
 {
 	return test_bit(CONTEXT_CLOSED, &ctx->flags);
@@ -270,6 +76,21 @@ static inline void i915_gem_context_clear_bannable(struct i915_gem_context *ctx)
 	clear_bit(UCONTEXT_BANNABLE, &ctx->user_flags);
 }
 
+static inline bool i915_gem_context_is_recoverable(const struct i915_gem_context *ctx)
+{
+	return test_bit(UCONTEXT_RECOVERABLE, &ctx->user_flags);
+}
+
+static inline void i915_gem_context_set_recoverable(struct i915_gem_context *ctx)
+{
+	set_bit(UCONTEXT_RECOVERABLE, &ctx->user_flags);
+}
+
+static inline void i915_gem_context_clear_recoverable(struct i915_gem_context *ctx)
+{
+	clear_bit(UCONTEXT_RECOVERABLE, &ctx->user_flags);
+}
+
 static inline bool i915_gem_context_is_banned(const struct i915_gem_context *ctx)
 {
 	return test_bit(CONTEXT_BANNED, &ctx->flags);
@@ -305,45 +126,11 @@ static inline void i915_gem_context_unpin_hw_id(struct i915_gem_context *ctx)
 	atomic_dec(&ctx->hw_id_pin_count);
 }
 
-static inline bool i915_gem_context_is_default(const struct i915_gem_context *c)
-{
-	return c->user_handle == DEFAULT_CONTEXT_HANDLE;
-}
-
 static inline bool i915_gem_context_is_kernel(struct i915_gem_context *ctx)
 {
 	return !ctx->file_priv;
 }
 
-static inline struct intel_context *
-to_intel_context(struct i915_gem_context *ctx,
-		 const struct intel_engine_cs *engine)
-{
-	return &ctx->__engine[engine->id];
-}
-
-static inline struct intel_context *
-intel_context_pin(struct i915_gem_context *ctx, struct intel_engine_cs *engine)
-{
-	return engine->context_pin(engine, ctx);
-}
-
-static inline void __intel_context_pin(struct intel_context *ce)
-{
-	GEM_BUG_ON(!ce->pin_count);
-	ce->pin_count++;
-}
-
-static inline void intel_context_unpin(struct intel_context *ce)
-{
-	GEM_BUG_ON(!ce->pin_count);
-	if (--ce->pin_count)
-		return;
-
-	GEM_BUG_ON(!ce->ops);
-	ce->ops->unpin(ce);
-}
-
 /* i915_gem_context.c */
 int __must_check i915_gem_contexts_init(struct drm_i915_private *dev_priv);
 void i915_gem_contexts_lost(struct drm_i915_private *dev_priv);
@@ -354,12 +141,18 @@ int i915_gem_context_open(struct drm_i915_private *i915,
 void i915_gem_context_close(struct drm_file *file);
 
 int i915_switch_context(struct i915_request *rq);
-int i915_gem_switch_to_kernel_context(struct drm_i915_private *dev_priv);
+int i915_gem_switch_to_kernel_context(struct drm_i915_private *i915,
+				      intel_engine_mask_t engine_mask);
 
 void i915_gem_context_release(struct kref *ctx_ref);
 struct i915_gem_context *
 i915_gem_context_create_gvt(struct drm_device *dev);
 
+int i915_gem_vm_create_ioctl(struct drm_device *dev, void *data,
+			     struct drm_file *file);
+int i915_gem_vm_destroy_ioctl(struct drm_device *dev, void *data,
+			      struct drm_file *file);
+
 int i915_gem_context_create_ioctl(struct drm_device *dev, void *data,
 				  struct drm_file *file);
 int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data,
@@ -386,8 +179,7 @@ static inline void i915_gem_context_put(struct i915_gem_context *ctx)
 	kref_put(&ctx->ref, i915_gem_context_release);
 }
 
-void intel_context_init(struct intel_context *ce,
-			struct i915_gem_context *ctx,
-			struct intel_engine_cs *engine);
+struct i915_lut_handle *i915_lut_handle_alloc(void);
+void i915_lut_handle_free(struct i915_lut_handle *lut);
 
 #endif /* !__I915_GEM_CONTEXT_H__ */
diff --git a/drivers/gpu/drm/i915/i915_gem_context_types.h b/drivers/gpu/drm/i915/i915_gem_context_types.h
new file mode 100644
index 000000000000..e2ec58b10fb2
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_gem_context_types.h
@@ -0,0 +1,175 @@
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2019 Intel Corporation
+ */
+
+#ifndef __I915_GEM_CONTEXT_TYPES_H__
+#define __I915_GEM_CONTEXT_TYPES_H__
+
+#include <linux/atomic.h>
+#include <linux/list.h>
+#include <linux/llist.h>
+#include <linux/kref.h>
+#include <linux/mutex.h>
+#include <linux/radix-tree.h>
+#include <linux/rbtree.h>
+#include <linux/rcupdate.h>
+#include <linux/types.h>
+
+#include "i915_scheduler.h"
+#include "intel_context_types.h"
+
+struct pid;
+
+struct drm_i915_private;
+struct drm_i915_file_private;
+struct i915_hw_ppgtt;
+struct i915_timeline;
+struct intel_ring;
+
+/**
+ * struct i915_gem_context - client state
+ *
+ * The struct i915_gem_context represents the combined view of the driver and
+ * logical hardware state for a particular client.
+ */
+struct i915_gem_context {
+	/** i915: i915 device backpointer */
+	struct drm_i915_private *i915;
+
+	/** file_priv: owning file descriptor */
+	struct drm_i915_file_private *file_priv;
+
+	struct i915_timeline *timeline;
+
+	/**
+	 * @ppgtt: unique address space (GTT)
+	 *
+	 * In full-ppgtt mode, each context has its own address space ensuring
+	 * complete seperation of one client from all others.
+	 *
+	 * In other modes, this is a NULL pointer with the expectation that
+	 * the caller uses the shared global GTT.
+	 */
+	struct i915_hw_ppgtt *ppgtt;
+
+	/**
+	 * @pid: process id of creator
+	 *
+	 * Note that who created the context may not be the principle user,
+	 * as the context may be shared across a local socket. However,
+	 * that should only affect the default context, all contexts created
+	 * explicitly by the client are expected to be isolated.
+	 */
+	struct pid *pid;
+
+	/**
+	 * @name: arbitrary name
+	 *
+	 * A name is constructed for the context from the creator's process
+	 * name, pid and user handle in order to uniquely identify the
+	 * context in messages.
+	 */
+	const char *name;
+
+	/** link: place with &drm_i915_private.context_list */
+	struct list_head link;
+	struct llist_node free_link;
+
+	/**
+	 * @ref: reference count
+	 *
+	 * A reference to a context is held by both the client who created it
+	 * and on each request submitted to the hardware using the request
+	 * (to ensure the hardware has access to the state until it has
+	 * finished all pending writes). See i915_gem_context_get() and
+	 * i915_gem_context_put() for access.
+	 */
+	struct kref ref;
+
+	/**
+	 * @rcu: rcu_head for deferred freeing.
+	 */
+	struct rcu_head rcu;
+
+	/**
+	 * @user_flags: small set of booleans controlled by the user
+	 */
+	unsigned long user_flags;
+#define UCONTEXT_NO_ZEROMAP		0
+#define UCONTEXT_NO_ERROR_CAPTURE	1
+#define UCONTEXT_BANNABLE		2
+#define UCONTEXT_RECOVERABLE		3
+
+	/**
+	 * @flags: small set of booleans
+	 */
+	unsigned long flags;
+#define CONTEXT_BANNED			0
+#define CONTEXT_CLOSED			1
+#define CONTEXT_FORCE_SINGLE_SUBMISSION	2
+
+	/**
+	 * @hw_id: - unique identifier for the context
+	 *
+	 * The hardware needs to uniquely identify the context for a few
+	 * functions like fault reporting, PASID, scheduling. The
+	 * &drm_i915_private.context_hw_ida is used to assign a unqiue
+	 * id for the lifetime of the context.
+	 *
+	 * @hw_id_pin_count: - number of times this context had been pinned
+	 * for use (should be, at most, once per engine).
+	 *
+	 * @hw_id_link: - all contexts with an assigned id are tracked
+	 * for possible repossession.
+	 */
+	unsigned int hw_id;
+	atomic_t hw_id_pin_count;
+	struct list_head hw_id_link;
+
+	struct list_head active_engines;
+	struct mutex mutex;
+
+	struct i915_sched_attr sched;
+
+	/** hw_contexts: per-engine logical HW state */
+	struct rb_root hw_contexts;
+	spinlock_t hw_contexts_lock;
+
+	/** ring_size: size for allocating the per-engine ring buffer */
+	u32 ring_size;
+	/** desc_template: invariant fields for the HW context descriptor */
+	u32 desc_template;
+
+	/** guilty_count: How many times this context has caused a GPU hang. */
+	atomic_t guilty_count;
+	/**
+	 * @active_count: How many times this context was active during a GPU
+	 * hang, but did not cause it.
+	 */
+	atomic_t active_count;
+
+	/**
+	 * @hang_timestamp: The last time(s) this context caused a GPU hang
+	 */
+	unsigned long hang_timestamp[2];
+#define CONTEXT_FAST_HANG_JIFFIES (120 * HZ) /* 3 hangs within 120s? Banned! */
+
+	/** remap_slice: Bitmask of cache lines that need remapping */
+	u8 remap_slice;
+
+	/** handles_vma: rbtree to look up our context specific obj/vma for
+	 * the user handle. (user handles are per fd, but the binding is
+	 * per vm, which may be one per context or shared with the global GTT)
+	 */
+	struct radix_tree_root handles_vma;
+
+	/** handles_list: reverse list of all the rbtree entries in use for
+	 * this context, which allows us to free all the allocations on
+	 * context close.
+	 */
+	struct list_head handles_list;
+};
+
+#endif /* __I915_GEM_CONTEXT_TYPES_H__ */
diff --git a/drivers/gpu/drm/i915/i915_gem_dmabuf.c b/drivers/gpu/drm/i915/i915_gem_dmabuf.c
index 02f7298bfe57..5a101a9462d8 100644
--- a/drivers/gpu/drm/i915/i915_gem_dmabuf.c
+++ b/drivers/gpu/drm/i915/i915_gem_dmabuf.c
@@ -107,6 +107,7 @@ static void i915_gem_dmabuf_vunmap(struct dma_buf *dma_buf, void *vaddr)
 {
 	struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf);
 
+	i915_gem_object_flush_map(obj);
 	i915_gem_object_unpin_map(obj);
 }
 
@@ -300,7 +301,7 @@ struct drm_gem_object *i915_gem_prime_import(struct drm_device *dev,
 
 	get_dma_buf(dma_buf);
 
-	obj = i915_gem_object_alloc(to_i915(dev));
+	obj = i915_gem_object_alloc();
 	if (obj == NULL) {
 		ret = -ENOMEM;
 		goto fail_detach;
diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c
index 68d74c50ac39..060f5903544a 100644
--- a/drivers/gpu/drm/i915/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/i915_gem_evict.c
@@ -38,31 +38,21 @@ I915_SELFTEST_DECLARE(static struct igt_evict_ctl {
 
 static bool ggtt_is_idle(struct drm_i915_private *i915)
 {
-       struct intel_engine_cs *engine;
-       enum intel_engine_id id;
-
-       if (i915->gt.active_requests)
-	       return false;
-
-       for_each_engine(engine, i915, id) {
-	       if (!intel_engine_has_kernel_context(engine))
-		       return false;
-       }
-
-       return true;
+	return !i915->gt.active_requests;
 }
 
 static int ggtt_flush(struct drm_i915_private *i915)
 {
 	int err;
 
-	/* Not everything in the GGTT is tracked via vma (otherwise we
+	/*
+	 * Not everything in the GGTT is tracked via vma (otherwise we
 	 * could evict as required with minimal stalling) so we are forced
 	 * to idle the GPU and explicitly retire outstanding requests in
 	 * the hopes that we can then remove contexts and the like only
 	 * bound by their active reference.
 	 */
-	err = i915_gem_switch_to_kernel_context(i915);
+	err = i915_gem_switch_to_kernel_context(i915, i915->gt.active_engines);
 	if (err)
 		return err;
 
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
index 16f80a448820..c83d2a195d15 100644
--- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -794,8 +794,8 @@ static int eb_wait_for_ring(const struct i915_execbuffer *eb)
 	 * keeping all of their resources pinned.
 	 */
 
-	ce = to_intel_context(eb->ctx, eb->engine);
-	if (!ce->ring) /* first use, assume empty! */
+	ce = intel_context_lookup(eb->ctx, eb->engine);
+	if (!ce || !ce->ring) /* first use, assume empty! */
 		return 0;
 
 	rq = __eb_wait_for_ring(ce->ring);
@@ -849,12 +849,12 @@ static int eb_lookup_vmas(struct i915_execbuffer *eb)
 		}
 
 		vma = i915_vma_instance(obj, eb->vm, NULL);
-		if (unlikely(IS_ERR(vma))) {
+		if (IS_ERR(vma)) {
 			err = PTR_ERR(vma);
 			goto err_obj;
 		}
 
-		lut = kmem_cache_alloc(eb->i915->luts, GFP_KERNEL);
+		lut = i915_lut_handle_alloc();
 		if (unlikely(!lut)) {
 			err = -ENOMEM;
 			goto err_obj;
@@ -862,7 +862,7 @@ static int eb_lookup_vmas(struct i915_execbuffer *eb)
 
 		err = radix_tree_insert(handles_vma, handle, vma);
 		if (unlikely(err)) {
-			kmem_cache_free(eb->i915->luts, lut);
+			i915_lut_handle_free(lut);
 			goto err_obj;
 		}
 
@@ -1001,7 +1001,10 @@ static void reloc_gpu_flush(struct reloc_cache *cache)
 {
 	GEM_BUG_ON(cache->rq_size >= cache->rq->batch->obj->base.size / sizeof(u32));
 	cache->rq_cmd[cache->rq_size] = MI_BATCH_BUFFER_END;
+
+	__i915_gem_object_flush_map(cache->rq->batch->obj, 0, cache->rq_size);
 	i915_gem_object_unpin_map(cache->rq->batch->obj);
+
 	i915_gem_chipset_flush(cache->rq->i915);
 
 	i915_request_add(cache->rq);
@@ -1214,10 +1217,6 @@ static int __reloc_gpu_alloc(struct i915_execbuffer *eb,
 	if (IS_ERR(cmd))
 		return PTR_ERR(cmd);
 
-	err = i915_gem_object_set_to_wc_domain(obj, false);
-	if (err)
-		goto err_unmap;
-
 	batch = i915_vma_instance(obj, vma->vm, NULL);
 	if (IS_ERR(batch)) {
 		err = PTR_ERR(batch);
@@ -1958,7 +1957,7 @@ static int i915_reset_gen7_sol_offsets(struct i915_request *rq)
 	u32 *cs;
 	int i;
 
-	if (!IS_GEN(rq->i915, 7) || rq->engine->id != RCS) {
+	if (!IS_GEN(rq->i915, 7) || rq->engine->id != RCS0) {
 		DRM_DEBUG("sol reset is gen7/rcs only\n");
 		return -EINVAL;
 	}
@@ -2083,11 +2082,11 @@ gen8_dispatch_bsd_engine(struct drm_i915_private *dev_priv,
 #define I915_USER_RINGS (4)
 
 static const enum intel_engine_id user_ring_map[I915_USER_RINGS + 1] = {
-	[I915_EXEC_DEFAULT]	= RCS,
-	[I915_EXEC_RENDER]	= RCS,
-	[I915_EXEC_BLT]		= BCS,
-	[I915_EXEC_BSD]		= VCS,
-	[I915_EXEC_VEBOX]	= VECS
+	[I915_EXEC_DEFAULT]	= RCS0,
+	[I915_EXEC_RENDER]	= RCS0,
+	[I915_EXEC_BLT]		= BCS0,
+	[I915_EXEC_BSD]		= VCS0,
+	[I915_EXEC_VEBOX]	= VECS0
 };
 
 static struct intel_engine_cs *
@@ -2110,7 +2109,7 @@ eb_select_engine(struct drm_i915_private *dev_priv,
 		return NULL;
 	}
 
-	if (user_ring_id == I915_EXEC_BSD && HAS_BSD2(dev_priv)) {
+	if (user_ring_id == I915_EXEC_BSD && HAS_ENGINE(dev_priv, VCS1)) {
 		unsigned int bsd_idx = args->flags & I915_EXEC_BSD_MASK;
 
 		if (bsd_idx == I915_EXEC_BSD_DEFAULT) {
@@ -2313,10 +2312,6 @@ i915_gem_do_execbuffer(struct drm_device *dev,
 	if (args->flags & I915_EXEC_IS_PINNED)
 		eb.batch_flags |= I915_DISPATCH_PINNED;
 
-	eb.engine = eb_select_engine(eb.i915, file, args);
-	if (!eb.engine)
-		return -EINVAL;
-
 	if (args->flags & I915_EXEC_FENCE_IN) {
 		in_fence = sync_file_get_fence(lower_32_bits(args->rsvd2));
 		if (!in_fence)
@@ -2341,6 +2336,12 @@ i915_gem_do_execbuffer(struct drm_device *dev,
 	if (unlikely(err))
 		goto err_destroy;
 
+	eb.engine = eb_select_engine(eb.i915, file, args);
+	if (!eb.engine) {
+		err = -EINVAL;
+		goto err_engine;
+	}
+
 	/*
 	 * Take a local wakeref for preparing to dispatch the execbuf as
 	 * we expect to access the hardware fairly frequently in the
@@ -2506,6 +2507,7 @@ err_unlock:
 	mutex_unlock(&dev->struct_mutex);
 err_rpm:
 	intel_runtime_pm_put(eb.i915, wakeref);
+err_engine:
 	i915_gem_context_put(eb.ctx);
 err_destroy:
 	eb_destroy(&eb);
diff --git a/drivers/gpu/drm/i915/i915_gem_fence_reg.c b/drivers/gpu/drm/i915/i915_gem_fence_reg.c
index e037e94792f3..3084f52e3372 100644
--- a/drivers/gpu/drm/i915/i915_gem_fence_reg.c
+++ b/drivers/gpu/drm/i915/i915_gem_fence_reg.c
@@ -210,6 +210,7 @@ static int fence_update(struct drm_i915_fence_reg *fence,
 			struct i915_vma *vma)
 {
 	intel_wakeref_t wakeref;
+	struct i915_vma *old;
 	int ret;
 
 	if (vma) {
@@ -229,49 +230,55 @@ static int fence_update(struct drm_i915_fence_reg *fence,
 			return ret;
 	}
 
-	if (fence->vma) {
-		struct i915_vma *old = fence->vma;
-
+	old = xchg(&fence->vma, NULL);
+	if (old) {
 		ret = i915_active_request_retire(&old->last_fence,
 					     &old->obj->base.dev->struct_mutex);
-		if (ret)
+		if (ret) {
+			fence->vma = old;
 			return ret;
+		}
 
 		i915_vma_flush_writes(old);
-	}
 
-	if (fence->vma && fence->vma != vma) {
-		/* Ensure that all userspace CPU access is completed before
+		/*
+		 * Ensure that all userspace CPU access is completed before
 		 * stealing the fence.
 		 */
-		GEM_BUG_ON(fence->vma->fence != fence);
-		i915_vma_revoke_mmap(fence->vma);
-
-		fence->vma->fence = NULL;
-		fence->vma = NULL;
+		if (old != vma) {
+			GEM_BUG_ON(old->fence != fence);
+			i915_vma_revoke_mmap(old);
+			old->fence = NULL;
+		}
 
 		list_move(&fence->link, &fence->i915->mm.fence_list);
 	}
 
-	/* We only need to update the register itself if the device is awake.
+	/*
+	 * We only need to update the register itself if the device is awake.
 	 * If the device is currently powered down, we will defer the write
 	 * to the runtime resume, see i915_gem_restore_fences().
+	 *
+	 * This only works for removing the fence register, on acquisition
+	 * the caller must hold the rpm wakeref. The fence register must
+	 * be cleared before we can use any other fences to ensure that
+	 * the new fences do not overlap the elided clears, confusing HW.
 	 */
 	wakeref = intel_runtime_pm_get_if_in_use(fence->i915);
-	if (wakeref) {
-		fence_write(fence, vma);
-		intel_runtime_pm_put(fence->i915, wakeref);
+	if (!wakeref) {
+		GEM_BUG_ON(vma);
+		return 0;
 	}
 
-	if (vma) {
-		if (fence->vma != vma) {
-			vma->fence = fence;
-			fence->vma = vma;
-		}
+	WRITE_ONCE(fence->vma, vma);
+	fence_write(fence, vma);
 
+	if (vma) {
+		vma->fence = fence;
 		list_move_tail(&fence->link, &fence->i915->mm.fence_list);
 	}
 
+	intel_runtime_pm_put(fence->i915, wakeref);
 	return 0;
 }
 
@@ -436,32 +443,6 @@ void i915_unreserve_fence(struct drm_i915_fence_reg *fence)
 }
 
 /**
- * i915_gem_revoke_fences - revoke fence state
- * @dev_priv: i915 device private
- *
- * Removes all GTT mmappings via the fence registers. This forces any user
- * of the fence to reacquire that fence before continuing with their access.
- * One use is during GPU reset where the fence register is lost and we need to
- * revoke concurrent userspace access via GTT mmaps until the hardware has been
- * reset and the fence registers have been restored.
- */
-void i915_gem_revoke_fences(struct drm_i915_private *dev_priv)
-{
-	int i;
-
-	lockdep_assert_held(&dev_priv->drm.struct_mutex);
-
-	for (i = 0; i < dev_priv->num_fence_regs; i++) {
-		struct drm_i915_fence_reg *fence = &dev_priv->fence_regs[i];
-
-		GEM_BUG_ON(fence->vma && fence->vma->fence != fence);
-
-		if (fence->vma)
-			i915_vma_revoke_mmap(fence->vma);
-	}
-}
-
-/**
  * i915_gem_restore_fences - restore fence state
  * @dev_priv: i915 device private
  *
@@ -473,9 +454,10 @@ void i915_gem_restore_fences(struct drm_i915_private *dev_priv)
 {
 	int i;
 
+	rcu_read_lock(); /* keep obj alive as we dereference */
 	for (i = 0; i < dev_priv->num_fence_regs; i++) {
 		struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
-		struct i915_vma *vma = reg->vma;
+		struct i915_vma *vma = READ_ONCE(reg->vma);
 
 		GEM_BUG_ON(vma && vma->fence != reg);
 
@@ -483,18 +465,12 @@ void i915_gem_restore_fences(struct drm_i915_private *dev_priv)
 		 * Commit delayed tiling changes if we have an object still
 		 * attached to the fence, otherwise just clear the fence.
 		 */
-		if (vma && !i915_gem_object_is_tiled(vma->obj)) {
-			GEM_BUG_ON(!reg->dirty);
-			GEM_BUG_ON(i915_vma_has_userfault(vma));
-
-			list_move(&reg->link, &dev_priv->mm.fence_list);
-			vma->fence = NULL;
+		if (vma && !i915_gem_object_is_tiled(vma->obj))
 			vma = NULL;
-		}
 
 		fence_write(reg, vma);
-		reg->vma = vma;
 	}
+	rcu_read_unlock();
 }
 
 /**
@@ -609,8 +585,38 @@ i915_gem_detect_bit_6_swizzle(struct drm_i915_private *dev_priv)
 		 */
 		swizzle_x = I915_BIT_6_SWIZZLE_NONE;
 		swizzle_y = I915_BIT_6_SWIZZLE_NONE;
-	} else if (IS_MOBILE(dev_priv) ||
-		   IS_I915G(dev_priv) || IS_I945G(dev_priv)) {
+	} else if (IS_G45(dev_priv) || IS_I965G(dev_priv) || IS_G33(dev_priv)) {
+		/* The 965, G33, and newer, have a very flexible memory
+		 * configuration.  It will enable dual-channel mode
+		 * (interleaving) on as much memory as it can, and the GPU
+		 * will additionally sometimes enable different bit 6
+		 * swizzling for tiled objects from the CPU.
+		 *
+		 * Here's what I found on the G965:
+		 *    slot fill         memory size  swizzling
+		 * 0A   0B   1A   1B    1-ch   2-ch
+		 * 512  0    0    0     512    0     O
+		 * 512  0    512  0     16     1008  X
+		 * 512  0    0    512   16     1008  X
+		 * 0    512  0    512   16     1008  X
+		 * 1024 1024 1024 0     2048   1024  O
+		 *
+		 * We could probably detect this based on either the DRB
+		 * matching, which was the case for the swizzling required in
+		 * the table above, or from the 1-ch value being less than
+		 * the minimum size of a rank.
+		 *
+		 * Reports indicate that the swizzling actually
+		 * varies depending upon page placement inside the
+		 * channels, i.e. we see swizzled pages where the
+		 * banks of memory are paired and unswizzled on the
+		 * uneven portion, so leave that as unknown.
+		 */
+		if (I915_READ16(C0DRB3) == I915_READ16(C1DRB3)) {
+			swizzle_x = I915_BIT_6_SWIZZLE_9_10;
+			swizzle_y = I915_BIT_6_SWIZZLE_9;
+		}
+	} else {
 		u32 dcc;
 
 		/* On 9xx chipsets, channel interleave by the CPU is
@@ -660,37 +666,6 @@ i915_gem_detect_bit_6_swizzle(struct drm_i915_private *dev_priv)
 			swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN;
 			swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN;
 		}
-	} else {
-		/* The 965, G33, and newer, have a very flexible memory
-		 * configuration.  It will enable dual-channel mode
-		 * (interleaving) on as much memory as it can, and the GPU
-		 * will additionally sometimes enable different bit 6
-		 * swizzling for tiled objects from the CPU.
-		 *
-		 * Here's what I found on the G965:
-		 *    slot fill         memory size  swizzling
-		 * 0A   0B   1A   1B    1-ch   2-ch
-		 * 512  0    0    0     512    0     O
-		 * 512  0    512  0     16     1008  X
-		 * 512  0    0    512   16     1008  X
-		 * 0    512  0    512   16     1008  X
-		 * 1024 1024 1024 0     2048   1024  O
-		 *
-		 * We could probably detect this based on either the DRB
-		 * matching, which was the case for the swizzling required in
-		 * the table above, or from the 1-ch value being less than
-		 * the minimum size of a rank.
-		 *
-		 * Reports indicate that the swizzling actually
-		 * varies depending upon page placement inside the
-		 * channels, i.e. we see swizzled pages where the
-		 * banks of memory are paired and unswizzled on the
-		 * uneven portion, so leave that as unknown.
-		 */
-		if (I915_READ16(C0DRB3) == I915_READ16(C1DRB3)) {
-			swizzle_x = I915_BIT_6_SWIZZLE_9_10;
-			swizzle_y = I915_BIT_6_SWIZZLE_9;
-		}
 	}
 
 	if (swizzle_x == I915_BIT_6_SWIZZLE_UNKNOWN ||
@@ -790,8 +765,7 @@ i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj,
 	int i;
 
 	if (obj->bit_17 == NULL) {
-		obj->bit_17 = kcalloc(BITS_TO_LONGS(page_count),
-				      sizeof(long), GFP_KERNEL);
+		obj->bit_17 = bitmap_zalloc(page_count, GFP_KERNEL);
 		if (obj->bit_17 == NULL) {
 			DRM_ERROR("Failed to allocate memory for bit 17 "
 				  "record\n");
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index d646d37eec2f..8f460cc4cc1f 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -584,7 +584,7 @@ setup_scratch_page(struct i915_address_space *vm, gfp_t gfp)
 	 * for all.
 	 */
 	size = I915_GTT_PAGE_SIZE_4K;
-	if (i915_vm_is_48bit(vm) &&
+	if (i915_vm_is_4lvl(vm) &&
 	    HAS_PAGE_SIZES(vm->i915, I915_GTT_PAGE_SIZE_64K)) {
 		size = I915_GTT_PAGE_SIZE_64K;
 		gfp |= __GFP_NOWARN;
@@ -613,7 +613,7 @@ setup_scratch_page(struct i915_address_space *vm, gfp_t gfp)
 
 		vm->scratch_page.page = page;
 		vm->scratch_page.daddr = addr;
-		vm->scratch_page.order = order;
+		vm->scratch_order = order;
 		return 0;
 
 unmap_page:
@@ -632,10 +632,11 @@ skip:
 static void cleanup_scratch_page(struct i915_address_space *vm)
 {
 	struct i915_page_dma *p = &vm->scratch_page;
+	int order = vm->scratch_order;
 
-	dma_unmap_page(vm->dma, p->daddr, BIT(p->order) << PAGE_SHIFT,
+	dma_unmap_page(vm->dma, p->daddr, BIT(order) << PAGE_SHIFT,
 		       PCI_DMA_BIDIRECTIONAL);
-	__free_pages(p->page, p->order);
+	__free_pages(p->page, order);
 }
 
 static struct i915_page_table *alloc_pt(struct i915_address_space *vm)
@@ -726,18 +727,13 @@ static void __pdp_fini(struct i915_page_directory_pointer *pdp)
 	pdp->page_directory = NULL;
 }
 
-static inline bool use_4lvl(const struct i915_address_space *vm)
-{
-	return i915_vm_is_48bit(vm);
-}
-
 static struct i915_page_directory_pointer *
 alloc_pdp(struct i915_address_space *vm)
 {
 	struct i915_page_directory_pointer *pdp;
 	int ret = -ENOMEM;
 
-	GEM_BUG_ON(!use_4lvl(vm));
+	GEM_BUG_ON(!i915_vm_is_4lvl(vm));
 
 	pdp = kzalloc(sizeof(*pdp), GFP_KERNEL);
 	if (!pdp)
@@ -766,7 +762,7 @@ static void free_pdp(struct i915_address_space *vm,
 {
 	__pdp_fini(pdp);
 
-	if (!use_4lvl(vm))
+	if (!i915_vm_is_4lvl(vm))
 		return;
 
 	cleanup_px(vm, pdp);
@@ -791,14 +787,15 @@ static void gen8_initialize_pml4(struct i915_address_space *vm,
 	memset_p((void **)pml4->pdps, vm->scratch_pdp, GEN8_PML4ES_PER_PML4);
 }
 
-/* PDE TLBs are a pain to invalidate on GEN8+. When we modify
+/*
+ * PDE TLBs are a pain to invalidate on GEN8+. When we modify
  * the page table structures, we mark them dirty so that
  * context switching/execlist queuing code takes extra steps
  * to ensure that tlbs are flushed.
  */
 static void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt)
 {
-	ppgtt->pd_dirty_rings = INTEL_INFO(ppgtt->vm.i915)->ring_mask;
+	ppgtt->pd_dirty_engines = ALL_ENGINES;
 }
 
 /* Removes entries from a single page table, releasing it if it's empty.
@@ -809,8 +806,6 @@ static bool gen8_ppgtt_clear_pt(const struct i915_address_space *vm,
 				u64 start, u64 length)
 {
 	unsigned int num_entries = gen8_pte_count(start, length);
-	unsigned int pte = gen8_pte_index(start);
-	unsigned int pte_end = pte + num_entries;
 	gen8_pte_t *vaddr;
 
 	GEM_BUG_ON(num_entries > pt->used_ptes);
@@ -820,8 +815,7 @@ static bool gen8_ppgtt_clear_pt(const struct i915_address_space *vm,
 		return true;
 
 	vaddr = kmap_atomic_px(pt);
-	while (pte < pte_end)
-		vaddr[pte++] = vm->scratch_pte;
+	memset64(vaddr + gen8_pte_index(start), vm->scratch_pte, num_entries);
 	kunmap_atomic(vaddr);
 
 	return false;
@@ -872,7 +866,7 @@ static void gen8_ppgtt_set_pdpe(struct i915_address_space *vm,
 	gen8_ppgtt_pdpe_t *vaddr;
 
 	pdp->page_directory[pdpe] = pd;
-	if (!use_4lvl(vm))
+	if (!i915_vm_is_4lvl(vm))
 		return;
 
 	vaddr = kmap_atomic_px(pdp);
@@ -937,7 +931,7 @@ static void gen8_ppgtt_clear_4lvl(struct i915_address_space *vm,
 	struct i915_page_directory_pointer *pdp;
 	unsigned int pml4e;
 
-	GEM_BUG_ON(!use_4lvl(vm));
+	GEM_BUG_ON(!i915_vm_is_4lvl(vm));
 
 	gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
 		GEM_BUG_ON(pdp == vm->scratch_pdp);
@@ -1219,7 +1213,7 @@ static int gen8_init_scratch(struct i915_address_space *vm)
 
 		GEM_BUG_ON(!clone->has_read_only);
 
-		vm->scratch_page.order = clone->scratch_page.order;
+		vm->scratch_order = clone->scratch_order;
 		vm->scratch_pte = clone->scratch_pte;
 		vm->scratch_pt  = clone->scratch_pt;
 		vm->scratch_pd  = clone->scratch_pd;
@@ -1234,7 +1228,7 @@ static int gen8_init_scratch(struct i915_address_space *vm)
 	vm->scratch_pte =
 		gen8_pte_encode(vm->scratch_page.daddr,
 				I915_CACHE_LLC,
-				PTE_READ_ONLY);
+				vm->has_read_only);
 
 	vm->scratch_pt = alloc_pt(vm);
 	if (IS_ERR(vm->scratch_pt)) {
@@ -1248,7 +1242,7 @@ static int gen8_init_scratch(struct i915_address_space *vm)
 		goto free_pt;
 	}
 
-	if (use_4lvl(vm)) {
+	if (i915_vm_is_4lvl(vm)) {
 		vm->scratch_pdp = alloc_pdp(vm);
 		if (IS_ERR(vm->scratch_pdp)) {
 			ret = PTR_ERR(vm->scratch_pdp);
@@ -1258,7 +1252,7 @@ static int gen8_init_scratch(struct i915_address_space *vm)
 
 	gen8_initialize_pt(vm, vm->scratch_pt);
 	gen8_initialize_pd(vm, vm->scratch_pd);
-	if (use_4lvl(vm))
+	if (i915_vm_is_4lvl(vm))
 		gen8_initialize_pdp(vm, vm->scratch_pdp);
 
 	return 0;
@@ -1280,7 +1274,7 @@ static int gen8_ppgtt_notify_vgt(struct i915_hw_ppgtt *ppgtt, bool create)
 	enum vgt_g2v_type msg;
 	int i;
 
-	if (use_4lvl(vm)) {
+	if (i915_vm_is_4lvl(vm)) {
 		const u64 daddr = px_dma(&ppgtt->pml4);
 
 		I915_WRITE(vgtif_reg(pdp[0].lo), lower_32_bits(daddr));
@@ -1310,7 +1304,7 @@ static void gen8_free_scratch(struct i915_address_space *vm)
 	if (!vm->scratch_page.daddr)
 		return;
 
-	if (use_4lvl(vm))
+	if (i915_vm_is_4lvl(vm))
 		free_pdp(vm, vm->scratch_pdp);
 	free_pd(vm, vm->scratch_pd);
 	free_pt(vm, vm->scratch_pt);
@@ -1356,7 +1350,7 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
 	if (intel_vgpu_active(dev_priv))
 		gen8_ppgtt_notify_vgt(ppgtt, false);
 
-	if (use_4lvl(vm))
+	if (i915_vm_is_4lvl(vm))
 		gen8_ppgtt_cleanup_4lvl(ppgtt);
 	else
 		gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, &ppgtt->pdp);
@@ -1519,6 +1513,23 @@ unwind:
 	return -ENOMEM;
 }
 
+static void ppgtt_init(struct drm_i915_private *i915,
+		       struct i915_hw_ppgtt *ppgtt)
+{
+	kref_init(&ppgtt->ref);
+
+	ppgtt->vm.i915 = i915;
+	ppgtt->vm.dma = &i915->drm.pdev->dev;
+	ppgtt->vm.total = BIT_ULL(INTEL_INFO(i915)->ppgtt_size);
+
+	i915_address_space_init(&ppgtt->vm, VM_CLASS_PPGTT);
+
+	ppgtt->vm.vma_ops.bind_vma    = ppgtt_bind_vma;
+	ppgtt->vm.vma_ops.unbind_vma  = ppgtt_unbind_vma;
+	ppgtt->vm.vma_ops.set_pages   = ppgtt_set_pages;
+	ppgtt->vm.vma_ops.clear_pages = clear_pages;
+}
+
 /*
  * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers
  * with a net effect resembling a 2-level page table in normal x86 terms. Each
@@ -1535,19 +1546,15 @@ static struct i915_hw_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915)
 	if (!ppgtt)
 		return ERR_PTR(-ENOMEM);
 
-	kref_init(&ppgtt->ref);
+	ppgtt_init(i915, ppgtt);
 
-	ppgtt->vm.i915 = i915;
-	ppgtt->vm.dma = &i915->drm.pdev->dev;
-
-	ppgtt->vm.total = HAS_FULL_48BIT_PPGTT(i915) ?
-		1ULL << 48 :
-		1ULL << 32;
-
-	/* From bdw, there is support for read-only pages in the PPGTT. */
-	ppgtt->vm.has_read_only = true;
-
-	i915_address_space_init(&ppgtt->vm, VM_CLASS_PPGTT);
+	/*
+	 * From bdw, there is hw support for read-only pages in the PPGTT.
+	 *
+	 * Gen11 has HSDES#:1807136187 unresolved. Disable ro support
+	 * for now.
+	 */
+	ppgtt->vm.has_read_only = INTEL_GEN(i915) != 11;
 
 	/* There are only few exceptions for gen >=6. chv and bxt.
 	 * And we are not sure about the latter so play safe for now.
@@ -1559,7 +1566,7 @@ static struct i915_hw_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915)
 	if (err)
 		goto err_free;
 
-	if (use_4lvl(&ppgtt->vm)) {
+	if (i915_vm_is_4lvl(&ppgtt->vm)) {
 		err = setup_px(&ppgtt->vm, &ppgtt->pml4);
 		if (err)
 			goto err_scratch;
@@ -1592,11 +1599,6 @@ static struct i915_hw_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915)
 
 	ppgtt->vm.cleanup = gen8_ppgtt_cleanup;
 
-	ppgtt->vm.vma_ops.bind_vma    = ppgtt_bind_vma;
-	ppgtt->vm.vma_ops.unbind_vma  = ppgtt_unbind_vma;
-	ppgtt->vm.vma_ops.set_pages   = ppgtt_set_pages;
-	ppgtt->vm.vma_ops.clear_pages = clear_pages;
-
 	return ppgtt;
 
 err_scratch:
@@ -1672,8 +1674,7 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
 
 	while (num_entries) {
 		struct i915_page_table *pt = ppgtt->base.pd.page_table[pde++];
-		const unsigned int end = min(pte + num_entries, GEN6_PTES);
-		const unsigned int count = end - pte;
+		const unsigned int count = min(num_entries, GEN6_PTES - pte);
 		gen6_pte_t *vaddr;
 
 		GEM_BUG_ON(pt == vm->scratch_pt);
@@ -1693,9 +1694,7 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
 		 */
 
 		vaddr = kmap_atomic_px(pt);
-		do {
-			vaddr[pte++] = scratch_pte;
-		} while (pte < end);
+		memset32(vaddr + pte, scratch_pte, count);
 		kunmap_atomic(vaddr);
 
 		pte = 0;
@@ -1913,7 +1912,7 @@ static struct i915_vma *pd_vma_create(struct gen6_hw_ppgtt *ppgtt, int size)
 	GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
 	GEM_BUG_ON(size > ggtt->vm.total);
 
-	vma = kmem_cache_zalloc(i915->vmas, GFP_KERNEL);
+	vma = i915_vma_alloc();
 	if (!vma)
 		return ERR_PTR(-ENOMEM);
 
@@ -1943,6 +1942,8 @@ int gen6_ppgtt_pin(struct i915_hw_ppgtt *base)
 	struct gen6_hw_ppgtt *ppgtt = to_gen6_ppgtt(base);
 	int err;
 
+	GEM_BUG_ON(ppgtt->base.vm.closed);
+
 	/*
 	 * Workaround the limited maximum vma->pin_count and the aliasing_ppgtt
 	 * which will be pinned into every active context.
@@ -1981,6 +1982,17 @@ void gen6_ppgtt_unpin(struct i915_hw_ppgtt *base)
 	i915_vma_unpin(ppgtt->vma);
 }
 
+void gen6_ppgtt_unpin_all(struct i915_hw_ppgtt *base)
+{
+	struct gen6_hw_ppgtt *ppgtt = to_gen6_ppgtt(base);
+
+	if (!ppgtt->pin_count)
+		return;
+
+	ppgtt->pin_count = 0;
+	i915_vma_unpin(ppgtt->vma);
+}
+
 static struct i915_hw_ppgtt *gen6_ppgtt_create(struct drm_i915_private *i915)
 {
 	struct i915_ggtt * const ggtt = &i915->ggtt;
@@ -1991,25 +2003,13 @@ static struct i915_hw_ppgtt *gen6_ppgtt_create(struct drm_i915_private *i915)
 	if (!ppgtt)
 		return ERR_PTR(-ENOMEM);
 
-	kref_init(&ppgtt->base.ref);
-
-	ppgtt->base.vm.i915 = i915;
-	ppgtt->base.vm.dma = &i915->drm.pdev->dev;
-
-	ppgtt->base.vm.total = I915_PDES * GEN6_PTES * I915_GTT_PAGE_SIZE;
-
-	i915_address_space_init(&ppgtt->base.vm, VM_CLASS_PPGTT);
+	ppgtt_init(i915, &ppgtt->base);
 
 	ppgtt->base.vm.allocate_va_range = gen6_alloc_va_range;
 	ppgtt->base.vm.clear_range = gen6_ppgtt_clear_range;
 	ppgtt->base.vm.insert_entries = gen6_ppgtt_insert_entries;
 	ppgtt->base.vm.cleanup = gen6_ppgtt_cleanup;
 
-	ppgtt->base.vm.vma_ops.bind_vma    = ppgtt_bind_vma;
-	ppgtt->base.vm.vma_ops.unbind_vma  = ppgtt_unbind_vma;
-	ppgtt->base.vm.vma_ops.set_pages   = ppgtt_set_pages;
-	ppgtt->base.vm.vma_ops.clear_pages = clear_pages;
-
 	ppgtt->base.vm.pte_encode = ggtt->vm.pte_encode;
 
 	err = gen6_ppgtt_init_scratch(ppgtt);
@@ -2087,8 +2087,7 @@ __hw_ppgtt_create(struct drm_i915_private *i915)
 }
 
 struct i915_hw_ppgtt *
-i915_ppgtt_create(struct drm_i915_private *i915,
-		  struct drm_i915_file_private *fpriv)
+i915_ppgtt_create(struct drm_i915_private *i915)
 {
 	struct i915_hw_ppgtt *ppgtt;
 
@@ -2096,19 +2095,11 @@ i915_ppgtt_create(struct drm_i915_private *i915,
 	if (IS_ERR(ppgtt))
 		return ppgtt;
 
-	ppgtt->vm.file = fpriv;
-
 	trace_i915_ppgtt_create(&ppgtt->vm);
 
 	return ppgtt;
 }
 
-void i915_ppgtt_close(struct i915_address_space *vm)
-{
-	GEM_BUG_ON(vm->closed);
-	vm->closed = true;
-}
-
 static void ppgtt_destroy_vma(struct i915_address_space *vm)
 {
 	struct list_head *phases[] = {
@@ -2675,7 +2666,7 @@ int i915_gem_init_aliasing_ppgtt(struct drm_i915_private *i915)
 	struct i915_hw_ppgtt *ppgtt;
 	int err;
 
-	ppgtt = i915_ppgtt_create(i915, ERR_PTR(-EPERM));
+	ppgtt = i915_ppgtt_create(i915);
 	if (IS_ERR(ppgtt))
 		return PTR_ERR(ppgtt);
 
@@ -3701,7 +3692,7 @@ i915_get_ggtt_vma_pages(struct i915_vma *vma)
 	}
 
 	ret = 0;
-	if (unlikely(IS_ERR(vma->pages))) {
+	if (IS_ERR(vma->pages)) {
 		ret = PTR_ERR(vma->pages);
 		vma->pages = NULL;
 		DRM_ERROR("Failed to get pages for VMA view type %u (%d)!\n",
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h
index 03ade71b8d9a..f597f35b109b 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.h
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.h
@@ -213,7 +213,6 @@ struct i915_vma;
 
 struct i915_page_dma {
 	struct page *page;
-	int order;
 	union {
 		dma_addr_t daddr;
 
@@ -293,6 +292,7 @@ struct i915_address_space {
 #define VM_CLASS_PPGTT 1
 
 	u64 scratch_pte;
+	int scratch_order;
 	struct i915_page_dma scratch_page;
 	struct i915_page_table *scratch_pt;
 	struct i915_page_directory *scratch_pd;
@@ -348,7 +348,7 @@ struct i915_address_space {
 #define i915_is_ggtt(vm) ((vm)->is_ggtt)
 
 static inline bool
-i915_vm_is_48bit(const struct i915_address_space *vm)
+i915_vm_is_4lvl(const struct i915_address_space *vm)
 {
 	return (vm->total - 1) >> 32;
 }
@@ -356,7 +356,7 @@ i915_vm_is_48bit(const struct i915_address_space *vm)
 static inline bool
 i915_vm_has_scratch_64K(struct i915_address_space *vm)
 {
-	return vm->scratch_page.order == get_order(I915_GTT_PAGE_SIZE_64K);
+	return vm->scratch_order == get_order(I915_GTT_PAGE_SIZE_64K);
 }
 
 /* The Graphics Translation Table is the way in which GEN hardware translates a
@@ -390,12 +390,14 @@ struct i915_hw_ppgtt {
 	struct i915_address_space vm;
 	struct kref ref;
 
-	unsigned long pd_dirty_rings;
+	intel_engine_mask_t pd_dirty_engines;
 	union {
 		struct i915_pml4 pml4;		/* GEN8+ & 48b PPGTT */
 		struct i915_page_directory_pointer pdp;	/* GEN8+ */
 		struct i915_page_directory pd;		/* GEN6-7 */
 	};
+
+	u32 user_handle;
 };
 
 struct gen6_hw_ppgtt {
@@ -488,7 +490,7 @@ static inline u32 gen6_pde_index(u32 addr)
 static inline unsigned int
 i915_pdpes_per_pdp(const struct i915_address_space *vm)
 {
-	if (i915_vm_is_48bit(vm))
+	if (i915_vm_is_4lvl(vm))
 		return GEN8_PML4ES_PER_PML4;
 
 	return GEN8_3LVL_PDPES;
@@ -603,15 +605,16 @@ int i915_gem_init_ggtt(struct drm_i915_private *dev_priv);
 void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv);
 
 int i915_ppgtt_init_hw(struct drm_i915_private *dev_priv);
+
+struct i915_hw_ppgtt *i915_ppgtt_create(struct drm_i915_private *dev_priv);
 void i915_ppgtt_release(struct kref *kref);
-struct i915_hw_ppgtt *i915_ppgtt_create(struct drm_i915_private *dev_priv,
-					struct drm_i915_file_private *fpriv);
-void i915_ppgtt_close(struct i915_address_space *vm);
-static inline void i915_ppgtt_get(struct i915_hw_ppgtt *ppgtt)
+
+static inline struct i915_hw_ppgtt *i915_ppgtt_get(struct i915_hw_ppgtt *ppgtt)
 {
-	if (ppgtt)
-		kref_get(&ppgtt->ref);
+	kref_get(&ppgtt->ref);
+	return ppgtt;
 }
+
 static inline void i915_ppgtt_put(struct i915_hw_ppgtt *ppgtt)
 {
 	if (ppgtt)
@@ -620,6 +623,7 @@ static inline void i915_ppgtt_put(struct i915_hw_ppgtt *ppgtt)
 
 int gen6_ppgtt_pin(struct i915_hw_ppgtt *base);
 void gen6_ppgtt_unpin(struct i915_hw_ppgtt *base);
+void gen6_ppgtt_unpin_all(struct i915_hw_ppgtt *base);
 
 void i915_check_and_clear_faults(struct drm_i915_private *dev_priv);
 void i915_gem_suspend_gtt_mappings(struct drm_i915_private *dev_priv);
diff --git a/drivers/gpu/drm/i915/i915_gem_internal.c b/drivers/gpu/drm/i915/i915_gem_internal.c
index fddde1033e74..ab627ed1269c 100644
--- a/drivers/gpu/drm/i915/i915_gem_internal.c
+++ b/drivers/gpu/drm/i915/i915_gem_internal.c
@@ -193,7 +193,7 @@ i915_gem_object_create_internal(struct drm_i915_private *i915,
 	if (overflows_type(size, obj->base.size))
 		return ERR_PTR(-E2BIG);
 
-	obj = i915_gem_object_alloc(i915);
+	obj = i915_gem_object_alloc();
 	if (!obj)
 		return ERR_PTR(-ENOMEM);
 
diff --git a/drivers/gpu/drm/i915/i915_gem_object.c b/drivers/gpu/drm/i915/i915_gem_object.c
index aab8cdd80e6d..ac6a5ab84586 100644
--- a/drivers/gpu/drm/i915/i915_gem_object.c
+++ b/drivers/gpu/drm/i915/i915_gem_object.c
@@ -24,6 +24,22 @@
 
 #include "i915_drv.h"
 #include "i915_gem_object.h"
+#include "i915_globals.h"
+
+static struct i915_global_object {
+	struct i915_global base;
+	struct kmem_cache *slab_objects;
+} global;
+
+struct drm_i915_gem_object *i915_gem_object_alloc(void)
+{
+	return kmem_cache_zalloc(global.slab_objects, GFP_KERNEL);
+}
+
+void i915_gem_object_free(struct drm_i915_gem_object *obj)
+{
+	return kmem_cache_free(global.slab_objects, obj);
+}
 
 /**
  * Mark up the object's coherency levels for a given cache_level
@@ -46,3 +62,29 @@ void i915_gem_object_set_cache_coherency(struct drm_i915_gem_object *obj,
 	obj->cache_dirty =
 		!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE);
 }
+
+static void i915_global_objects_shrink(void)
+{
+	kmem_cache_shrink(global.slab_objects);
+}
+
+static void i915_global_objects_exit(void)
+{
+	kmem_cache_destroy(global.slab_objects);
+}
+
+static struct i915_global_object global = { {
+	.shrink = i915_global_objects_shrink,
+	.exit = i915_global_objects_exit,
+} };
+
+int __init i915_global_objects_init(void)
+{
+	global.slab_objects =
+		KMEM_CACHE(drm_i915_gem_object, SLAB_HWCACHE_ALIGN);
+	if (!global.slab_objects)
+		return -ENOMEM;
+
+	i915_global_register(&global.base);
+	return 0;
+}
diff --git a/drivers/gpu/drm/i915/i915_gem_object.h b/drivers/gpu/drm/i915/i915_gem_object.h
index fab040331cdb..ca93a40c0c87 100644
--- a/drivers/gpu/drm/i915/i915_gem_object.h
+++ b/drivers/gpu/drm/i915/i915_gem_object.h
@@ -304,6 +304,9 @@ to_intel_bo(struct drm_gem_object *gem)
 	return container_of(gem, struct drm_i915_gem_object, base);
 }
 
+struct drm_i915_gem_object *i915_gem_object_alloc(void);
+void i915_gem_object_free(struct drm_i915_gem_object *obj);
+
 /**
  * i915_gem_object_lookup_rcu - look up a temporary GEM object from its handle
  * @filp: DRM file private date
@@ -499,5 +502,8 @@ void i915_gem_object_set_cache_coherency(struct drm_i915_gem_object *obj,
 					 unsigned int cache_level);
 void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj);
 
-#endif
+void __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj,
+				     struct sg_table *pages,
+				     bool needs_clflush);
 
+#endif
diff --git a/drivers/gpu/drm/i915/i915_gem_render_state.c b/drivers/gpu/drm/i915/i915_gem_render_state.c
index 90baf9086d0a..9440024c763f 100644
--- a/drivers/gpu/drm/i915/i915_gem_render_state.c
+++ b/drivers/gpu/drm/i915/i915_gem_render_state.c
@@ -42,7 +42,7 @@ struct intel_render_state {
 static const struct intel_renderstate_rodata *
 render_state_get_rodata(const struct intel_engine_cs *engine)
 {
-	if (engine->id != RCS)
+	if (engine->id != RCS0)
 		return NULL;
 
 	switch (INTEL_GEN(engine->i915)) {
@@ -164,7 +164,7 @@ static int render_state_setup(struct intel_render_state *so,
 		drm_clflush_virt_range(d, i * sizeof(u32));
 	kunmap_atomic(d);
 
-	ret = i915_gem_object_set_to_gtt_domain(so->obj, false);
+	ret = 0;
 out:
 	i915_gem_obj_finish_shmem_access(so->obj);
 	return ret;
diff --git a/drivers/gpu/drm/i915/i915_gem_stolen.c b/drivers/gpu/drm/i915/i915_gem_stolen.c
index 74a9661479ca..0a8082cfc761 100644
--- a/drivers/gpu/drm/i915/i915_gem_stolen.c
+++ b/drivers/gpu/drm/i915/i915_gem_stolen.c
@@ -565,7 +565,7 @@ _i915_gem_object_create_stolen(struct drm_i915_private *dev_priv,
 	struct drm_i915_gem_object *obj;
 	unsigned int cache_level;
 
-	obj = i915_gem_object_alloc(dev_priv);
+	obj = i915_gem_object_alloc();
 	if (obj == NULL)
 		return NULL;
 
diff --git a/drivers/gpu/drm/i915/i915_gem_tiling.c b/drivers/gpu/drm/i915/i915_gem_tiling.c
index 16cc9ddbce34..a9b5329dae3b 100644
--- a/drivers/gpu/drm/i915/i915_gem_tiling.c
+++ b/drivers/gpu/drm/i915/i915_gem_tiling.c
@@ -301,11 +301,11 @@ i915_gem_object_set_tiling(struct drm_i915_gem_object *obj,
 	/* Try to preallocate memory required to save swizzling on put-pages */
 	if (i915_gem_object_needs_bit17_swizzle(obj)) {
 		if (!obj->bit_17) {
-			obj->bit_17 = kcalloc(BITS_TO_LONGS(obj->base.size >> PAGE_SHIFT),
-					      sizeof(long), GFP_KERNEL);
+			obj->bit_17 = bitmap_zalloc(obj->base.size >> PAGE_SHIFT,
+						    GFP_KERNEL);
 		}
 	} else {
-		kfree(obj->bit_17);
+		bitmap_free(obj->bit_17);
 		obj->bit_17 = NULL;
 	}
 
diff --git a/drivers/gpu/drm/i915/i915_gem_userptr.c b/drivers/gpu/drm/i915/i915_gem_userptr.c
index 1d3f9a31ad61..215bf3fef10c 100644
--- a/drivers/gpu/drm/i915/i915_gem_userptr.c
+++ b/drivers/gpu/drm/i915/i915_gem_userptr.c
@@ -673,9 +673,7 @@ i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj,
 	if (!pages)
 		return;
 
-	if (obj->mm.madv != I915_MADV_WILLNEED)
-		obj->mm.dirty = false;
-
+	__i915_gem_object_release_shmem(obj, pages, true);
 	i915_gem_gtt_finish_pages(obj, pages);
 
 	for_each_sgt_page(page, sgt_iter, pages) {
@@ -795,7 +793,7 @@ i915_gem_userptr_ioctl(struct drm_device *dev,
 			return -ENODEV;
 	}
 
-	obj = i915_gem_object_alloc(dev_priv);
+	obj = i915_gem_object_alloc();
 	if (obj == NULL)
 		return -ENOMEM;
 
diff --git a/drivers/gpu/drm/i915/i915_globals.c b/drivers/gpu/drm/i915/i915_globals.c
new file mode 100644
index 000000000000..81e5c2ce336b
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_globals.c
@@ -0,0 +1,125 @@
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2019 Intel Corporation
+ */
+
+#include <linux/slab.h>
+#include <linux/workqueue.h>
+
+#include "i915_active.h"
+#include "i915_gem_context.h"
+#include "i915_gem_object.h"
+#include "i915_globals.h"
+#include "i915_request.h"
+#include "i915_scheduler.h"
+#include "i915_vma.h"
+
+static LIST_HEAD(globals);
+
+static atomic_t active;
+static atomic_t epoch;
+static struct park_work {
+	struct rcu_work work;
+	int epoch;
+} park;
+
+static void i915_globals_shrink(void)
+{
+	struct i915_global *global;
+
+	/*
+	 * kmem_cache_shrink() discards empty slabs and reorders partially
+	 * filled slabs to prioritise allocating from the mostly full slabs,
+	 * with the aim of reducing fragmentation.
+	 */
+	list_for_each_entry(global, &globals, link)
+		global->shrink();
+}
+
+static void __i915_globals_park(struct work_struct *work)
+{
+	/* Confirm nothing woke up in the last grace period */
+	if (park.epoch == atomic_read(&epoch))
+		i915_globals_shrink();
+}
+
+void __init i915_global_register(struct i915_global *global)
+{
+	GEM_BUG_ON(!global->shrink);
+	GEM_BUG_ON(!global->exit);
+
+	list_add_tail(&global->link, &globals);
+}
+
+static void __i915_globals_cleanup(void)
+{
+	struct i915_global *global, *next;
+
+	list_for_each_entry_safe_reverse(global, next, &globals, link)
+		global->exit();
+}
+
+static __initconst int (* const initfn[])(void) = {
+	i915_global_active_init,
+	i915_global_context_init,
+	i915_global_gem_context_init,
+	i915_global_objects_init,
+	i915_global_request_init,
+	i915_global_scheduler_init,
+	i915_global_vma_init,
+};
+
+int __init i915_globals_init(void)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(initfn); i++) {
+		int err;
+
+		err = initfn[i]();
+		if (err) {
+			__i915_globals_cleanup();
+			return err;
+		}
+	}
+
+	INIT_RCU_WORK(&park.work, __i915_globals_park);
+	return 0;
+}
+
+void i915_globals_park(void)
+{
+	/*
+	 * Defer shrinking the global slab caches (and other work) until
+	 * after a RCU grace period has completed with no activity. This
+	 * is to try and reduce the latency impact on the consumers caused
+	 * by us shrinking the caches the same time as they are trying to
+	 * allocate, with the assumption being that if we idle long enough
+	 * for an RCU grace period to elapse since the last use, it is likely
+	 * to be longer until we need the caches again.
+	 */
+	if (!atomic_dec_and_test(&active))
+		return;
+
+	park.epoch = atomic_inc_return(&epoch);
+	queue_rcu_work(system_wq, &park.work);
+}
+
+void i915_globals_unpark(void)
+{
+	atomic_inc(&epoch);
+	atomic_inc(&active);
+}
+
+void __exit i915_globals_exit(void)
+{
+	/* Flush any residual park_work */
+	atomic_inc(&epoch);
+	flush_rcu_work(&park.work);
+
+	__i915_globals_cleanup();
+
+	/* And ensure that our DESTROY_BY_RCU slabs are truly destroyed */
+	rcu_barrier();
+}
diff --git a/drivers/gpu/drm/i915/i915_globals.h b/drivers/gpu/drm/i915/i915_globals.h
new file mode 100644
index 000000000000..04c1ce107fc0
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_globals.h
@@ -0,0 +1,35 @@
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2019 Intel Corporation
+ */
+
+#ifndef _I915_GLOBALS_H_
+#define _I915_GLOBALS_H_
+
+typedef void (*i915_global_func_t)(void);
+
+struct i915_global {
+	struct list_head link;
+
+	i915_global_func_t shrink;
+	i915_global_func_t exit;
+};
+
+void i915_global_register(struct i915_global *global);
+
+int i915_globals_init(void);
+void i915_globals_park(void);
+void i915_globals_unpark(void);
+void i915_globals_exit(void);
+
+/* constructors */
+int i915_global_active_init(void);
+int i915_global_context_init(void);
+int i915_global_gem_context_init(void);
+int i915_global_objects_init(void);
+int i915_global_request_init(void);
+int i915_global_scheduler_init(void);
+int i915_global_vma_init(void);
+
+#endif /* _I915_GLOBALS_H_ */
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c
index aa6791255252..f51ff683dd2e 100644
--- a/drivers/gpu/drm/i915/i915_gpu_error.c
+++ b/drivers/gpu/drm/i915/i915_gpu_error.c
@@ -380,19 +380,16 @@ static void print_error_buffers(struct drm_i915_error_state_buf *m,
 	err_printf(m, "%s [%d]:\n", name, count);
 
 	while (count--) {
-		err_printf(m, "    %08x_%08x %8u %02x %02x %02x",
+		err_printf(m, "    %08x_%08x %8u %02x %02x",
 			   upper_32_bits(err->gtt_offset),
 			   lower_32_bits(err->gtt_offset),
 			   err->size,
 			   err->read_domains,
-			   err->write_domain,
-			   err->wseqno);
+			   err->write_domain);
 		err_puts(m, tiling_flag(err->tiling));
 		err_puts(m, dirty_flag(err->dirty));
 		err_puts(m, purgeable_flag(err->purgeable));
 		err_puts(m, err->userptr ? " userptr" : "");
-		err_puts(m, err->engine != -1 ? " " : "");
-		err_puts(m, engine_name(m->i915, err->engine));
 		err_puts(m, i915_cache_level_str(m->i915, err->cache_level));
 
 		if (err->name)
@@ -414,7 +411,7 @@ static void error_print_instdone(struct drm_i915_error_state_buf *m,
 	err_printf(m, "  INSTDONE: 0x%08x\n",
 		   ee->instdone.instdone);
 
-	if (ee->engine_id != RCS || INTEL_GEN(m->i915) <= 3)
+	if (ee->engine_id != RCS0 || INTEL_GEN(m->i915) <= 3)
 		return;
 
 	err_printf(m, "  SC_INSTDONE: 0x%08x\n",
@@ -434,11 +431,6 @@ static void error_print_instdone(struct drm_i915_error_state_buf *m,
 			   ee->instdone.row[slice][subslice]);
 }
 
-static const char *bannable(const struct drm_i915_error_context *ctx)
-{
-	return ctx->bannable ? "" : " (unbannable)";
-}
-
 static void error_print_request(struct drm_i915_error_state_buf *m,
 				const char *prefix,
 				const struct drm_i915_error_request *erq,
@@ -447,9 +439,8 @@ static void error_print_request(struct drm_i915_error_state_buf *m,
 	if (!erq->seqno)
 		return;
 
-	err_printf(m, "%s pid %d, ban score %d, seqno %8x:%08x%s%s, prio %d, emitted %dms, start %08x, head %08x, tail %08x\n",
-		   prefix, erq->pid, erq->ban_score,
-		   erq->context, erq->seqno,
+	err_printf(m, "%s pid %d, seqno %8x:%08x%s%s, prio %d, emitted %dms, start %08x, head %08x, tail %08x\n",
+		   prefix, erq->pid, erq->context, erq->seqno,
 		   test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
 			    &erq->flags) ? "!" : "",
 		   test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
@@ -463,10 +454,9 @@ static void error_print_context(struct drm_i915_error_state_buf *m,
 				const char *header,
 				const struct drm_i915_error_context *ctx)
 {
-	err_printf(m, "%s%s[%d] user_handle %d hw_id %d, prio %d, ban score %d%s guilty %d active %d\n",
-		   header, ctx->comm, ctx->pid, ctx->handle, ctx->hw_id,
-		   ctx->sched_attr.priority, ctx->ban_score, bannable(ctx),
-		   ctx->guilty, ctx->active);
+	err_printf(m, "%s%s[%d] hw_id %d, prio %d, guilty %d active %d\n",
+		   header, ctx->comm, ctx->pid, ctx->hw_id,
+		   ctx->sched_attr.priority, ctx->guilty, ctx->active);
 }
 
 static void error_print_engine(struct drm_i915_error_state_buf *m,
@@ -512,13 +502,6 @@ static void error_print_engine(struct drm_i915_error_state_buf *m,
 	if (INTEL_GEN(m->i915) >= 6) {
 		err_printf(m, "  RC PSMI: 0x%08x\n", ee->rc_psmi);
 		err_printf(m, "  FAULT_REG: 0x%08x\n", ee->fault_reg);
-		err_printf(m, "  SYNC_0: 0x%08x\n",
-			   ee->semaphore_mboxes[0]);
-		err_printf(m, "  SYNC_1: 0x%08x\n",
-			   ee->semaphore_mboxes[1]);
-		if (HAS_VEBOX(m->i915))
-			err_printf(m, "  SYNC_2: 0x%08x\n",
-				   ee->semaphore_mboxes[2]);
 	}
 	if (HAS_PPGTT(m->i915)) {
 		err_printf(m, "  GFX_MODE: 0x%08x\n", ee->vm_info.gfx_mode);
@@ -533,8 +516,6 @@ static void error_print_engine(struct drm_i915_error_state_buf *m,
 				   ee->vm_info.pp_dir_base);
 		}
 	}
-	err_printf(m, "  seqno: 0x%08x\n", ee->seqno);
-	err_printf(m, "  last_seqno: 0x%08x\n", ee->last_seqno);
 	err_printf(m, "  ring->head: 0x%08x\n", ee->cpu_ring_head);
 	err_printf(m, "  ring->tail: 0x%08x\n", ee->cpu_ring_tail);
 	err_printf(m, "  hangcheck timestamp: %dms (%lu%s)\n",
@@ -688,16 +669,17 @@ static void __err_print_to_sgl(struct drm_i915_error_state_buf *m,
 		if (!error->engine[i].context.pid)
 			continue;
 
-		err_printf(m, "Active process (on ring %s): %s [%d], score %d%s\n",
+		err_printf(m, "Active process (on ring %s): %s [%d]\n",
 			   engine_name(m->i915, i),
 			   error->engine[i].context.comm,
-			   error->engine[i].context.pid,
-			   error->engine[i].context.ban_score,
-			   bannable(&error->engine[i].context));
+			   error->engine[i].context.pid);
 	}
 	err_printf(m, "Reset count: %u\n", error->reset_count);
 	err_printf(m, "Suspend count: %u\n", error->suspend_count);
 	err_printf(m, "Platform: %s\n", intel_platform_name(error->device_info.platform));
+	err_printf(m, "Subplatform: 0x%x\n",
+		   intel_subplatform(&error->runtime_info,
+				     error->device_info.platform));
 	err_print_pciid(m, m->i915);
 
 	err_printf(m, "IOMMU enabled?: %d\n", error->iommu);
@@ -779,13 +761,9 @@ static void __err_print_to_sgl(struct drm_i915_error_state_buf *m,
 		if (obj) {
 			err_puts(m, m->i915->engine[i]->name);
 			if (ee->context.pid)
-				err_printf(m, " (submitted by %s [%d], ctx %d [%d], score %d%s)",
+				err_printf(m, " (submitted by %s [%d])",
 					   ee->context.comm,
-					   ee->context.pid,
-					   ee->context.handle,
-					   ee->context.hw_id,
-					   ee->context.ban_score,
-					   bannable(&ee->context));
+					   ee->context.pid);
 			err_printf(m, " --- gtt_offset = 0x%08x %08x\n",
 				   upper_32_bits(obj->gtt_offset),
 				   lower_32_bits(obj->gtt_offset));
@@ -1061,27 +1039,6 @@ i915_error_object_create(struct drm_i915_private *i915,
 	return dst;
 }
 
-/* The error capture is special as tries to run underneath the normal
- * locking rules - so we use the raw version of the i915_active_request lookup.
- */
-static inline u32
-__active_get_seqno(struct i915_active_request *active)
-{
-	struct i915_request *request;
-
-	request = __i915_active_request_peek(active);
-	return request ? request->global_seqno : 0;
-}
-
-static inline int
-__active_get_engine_id(struct i915_active_request *active)
-{
-	struct i915_request *request;
-
-	request = __i915_active_request_peek(active);
-	return request ? request->engine->id : -1;
-}
-
 static void capture_bo(struct drm_i915_error_buffer *err,
 		       struct i915_vma *vma)
 {
@@ -1090,9 +1047,6 @@ static void capture_bo(struct drm_i915_error_buffer *err,
 	err->size = obj->base.size;
 	err->name = obj->base.name;
 
-	err->wseqno = __active_get_seqno(&obj->frontbuffer_write);
-	err->engine = __active_get_engine_id(&obj->frontbuffer_write);
-
 	err->gtt_offset = vma->node.start;
 	err->read_domains = obj->read_domains;
 	err->write_domain = obj->write_domain;
@@ -1142,7 +1096,7 @@ static u32 capture_error_bo(struct drm_i915_error_buffer *err,
  * It's only a small step better than a random number in its current form.
  */
 static u32 i915_error_generate_code(struct i915_gpu_state *error,
-				    unsigned long engine_mask)
+				    intel_engine_mask_t engine_mask)
 {
 	/*
 	 * IPEHR would be an ideal way to detect errors, as it's the gross
@@ -1178,18 +1132,6 @@ static void gem_record_fences(struct i915_gpu_state *error)
 	error->nfence = i;
 }
 
-static void gen6_record_semaphore_state(struct intel_engine_cs *engine,
-					struct drm_i915_error_engine *ee)
-{
-	struct drm_i915_private *dev_priv = engine->i915;
-
-	ee->semaphore_mboxes[0] = I915_READ(RING_SYNC_0(engine->mmio_base));
-	ee->semaphore_mboxes[1] = I915_READ(RING_SYNC_1(engine->mmio_base));
-	if (HAS_VEBOX(dev_priv))
-		ee->semaphore_mboxes[2] =
-			I915_READ(RING_SYNC_2(engine->mmio_base));
-}
-
 static void error_record_engine_registers(struct i915_gpu_state *error,
 					  struct intel_engine_cs *engine,
 					  struct drm_i915_error_engine *ee)
@@ -1197,44 +1139,40 @@ static void error_record_engine_registers(struct i915_gpu_state *error,
 	struct drm_i915_private *dev_priv = engine->i915;
 
 	if (INTEL_GEN(dev_priv) >= 6) {
-		ee->rc_psmi = I915_READ(RING_PSMI_CTL(engine->mmio_base));
-		if (INTEL_GEN(dev_priv) >= 8) {
+		ee->rc_psmi = ENGINE_READ(engine, RING_PSMI_CTL);
+		if (INTEL_GEN(dev_priv) >= 8)
 			ee->fault_reg = I915_READ(GEN8_RING_FAULT_REG);
-		} else {
-			gen6_record_semaphore_state(engine, ee);
+		else
 			ee->fault_reg = I915_READ(RING_FAULT_REG(engine));
-		}
 	}
 
 	if (INTEL_GEN(dev_priv) >= 4) {
-		ee->faddr = I915_READ(RING_DMA_FADD(engine->mmio_base));
-		ee->ipeir = I915_READ(RING_IPEIR(engine->mmio_base));
-		ee->ipehr = I915_READ(RING_IPEHR(engine->mmio_base));
-		ee->instps = I915_READ(RING_INSTPS(engine->mmio_base));
-		ee->bbaddr = I915_READ(RING_BBADDR(engine->mmio_base));
+		ee->faddr = ENGINE_READ(engine, RING_DMA_FADD);
+		ee->ipeir = ENGINE_READ(engine, RING_IPEIR);
+		ee->ipehr = ENGINE_READ(engine, RING_IPEHR);
+		ee->instps = ENGINE_READ(engine, RING_INSTPS);
+		ee->bbaddr = ENGINE_READ(engine, RING_BBADDR);
 		if (INTEL_GEN(dev_priv) >= 8) {
-			ee->faddr |= (u64) I915_READ(RING_DMA_FADD_UDW(engine->mmio_base)) << 32;
-			ee->bbaddr |= (u64) I915_READ(RING_BBADDR_UDW(engine->mmio_base)) << 32;
+			ee->faddr |= (u64)ENGINE_READ(engine, RING_DMA_FADD_UDW) << 32;
+			ee->bbaddr |= (u64)ENGINE_READ(engine, RING_BBADDR_UDW) << 32;
 		}
-		ee->bbstate = I915_READ(RING_BBSTATE(engine->mmio_base));
+		ee->bbstate = ENGINE_READ(engine, RING_BBSTATE);
 	} else {
-		ee->faddr = I915_READ(DMA_FADD_I8XX);
-		ee->ipeir = I915_READ(IPEIR);
-		ee->ipehr = I915_READ(IPEHR);
+		ee->faddr = ENGINE_READ(engine, DMA_FADD_I8XX);
+		ee->ipeir = ENGINE_READ(engine, IPEIR);
+		ee->ipehr = ENGINE_READ(engine, IPEHR);
 	}
 
 	intel_engine_get_instdone(engine, &ee->instdone);
 
-	ee->instpm = I915_READ(RING_INSTPM(engine->mmio_base));
+	ee->instpm = ENGINE_READ(engine, RING_INSTPM);
 	ee->acthd = intel_engine_get_active_head(engine);
-	ee->seqno = intel_engine_get_seqno(engine);
-	ee->last_seqno = intel_engine_last_submit(engine);
-	ee->start = I915_READ_START(engine);
-	ee->head = I915_READ_HEAD(engine);
-	ee->tail = I915_READ_TAIL(engine);
-	ee->ctl = I915_READ_CTL(engine);
+	ee->start = ENGINE_READ(engine, RING_START);
+	ee->head = ENGINE_READ(engine, RING_HEAD);
+	ee->tail = ENGINE_READ(engine, RING_TAIL);
+	ee->ctl = ENGINE_READ(engine, RING_CTL);
 	if (INTEL_GEN(dev_priv) > 2)
-		ee->mode = I915_READ_MODE(engine);
+		ee->mode = ENGINE_READ(engine, RING_MI_MODE);
 
 	if (!HWS_NEEDS_PHYSICAL(dev_priv)) {
 		i915_reg_t mmio;
@@ -1242,16 +1180,17 @@ static void error_record_engine_registers(struct i915_gpu_state *error,
 		if (IS_GEN(dev_priv, 7)) {
 			switch (engine->id) {
 			default:
-			case RCS:
+				MISSING_CASE(engine->id);
+			case RCS0:
 				mmio = RENDER_HWS_PGA_GEN7;
 				break;
-			case BCS:
+			case BCS0:
 				mmio = BLT_HWS_PGA_GEN7;
 				break;
-			case VCS:
+			case VCS0:
 				mmio = BSD_HWS_PGA_GEN7;
 				break;
-			case VECS:
+			case VECS0:
 				mmio = VEBOX_HWS_PGA_GEN7;
 				break;
 			}
@@ -1276,20 +1215,23 @@ static void error_record_engine_registers(struct i915_gpu_state *error,
 
 		ee->vm_info.gfx_mode = I915_READ(RING_MODE_GEN7(engine));
 
-		if (IS_GEN(dev_priv, 6))
+		if (IS_GEN(dev_priv, 6)) {
 			ee->vm_info.pp_dir_base =
-				I915_READ(RING_PP_DIR_BASE_READ(engine));
-		else if (IS_GEN(dev_priv, 7))
+				ENGINE_READ(engine, RING_PP_DIR_BASE_READ);
+		} else if (IS_GEN(dev_priv, 7)) {
 			ee->vm_info.pp_dir_base =
-				I915_READ(RING_PP_DIR_BASE(engine));
-		else if (INTEL_GEN(dev_priv) >= 8)
+				ENGINE_READ(engine, RING_PP_DIR_BASE);
+		} else if (INTEL_GEN(dev_priv) >= 8) {
+			u32 base = engine->mmio_base;
+
 			for (i = 0; i < 4; i++) {
 				ee->vm_info.pdp[i] =
-					I915_READ(GEN8_RING_PDP_UDW(engine, i));
+					I915_READ(GEN8_RING_PDP_UDW(base, i));
 				ee->vm_info.pdp[i] <<= 32;
 				ee->vm_info.pdp[i] |=
-					I915_READ(GEN8_RING_PDP_LDW(engine, i));
+					I915_READ(GEN8_RING_PDP_LDW(base, i));
 			}
+		}
 	}
 }
 
@@ -1299,10 +1241,9 @@ static void record_request(struct i915_request *request,
 	struct i915_gem_context *ctx = request->gem_context;
 
 	erq->flags = request->fence.flags;
-	erq->context = ctx->hw_id;
+	erq->context = request->fence.context;
+	erq->seqno = request->fence.seqno;
 	erq->sched_attr = request->sched.attr;
-	erq->ban_score = atomic_read(&ctx->ban_score);
-	erq->seqno = request->global_seqno;
 	erq->jiffies = request->emitted_jiffies;
 	erq->start = i915_ggtt_offset(request->ring->vma);
 	erq->head = request->head;
@@ -1393,11 +1334,8 @@ static void record_context(struct drm_i915_error_context *e,
 		rcu_read_unlock();
 	}
 
-	e->handle = ctx->user_handle;
 	e->hw_id = ctx->hw_id;
 	e->sched_attr = ctx->sched;
-	e->ban_score = atomic_read(&ctx->ban_score);
-	e->bannable = i915_gem_context_is_bannable(ctx);
 	e->guilty = atomic_read(&ctx->guilty_count);
 	e->active = atomic_read(&ctx->active_count);
 }
@@ -1476,7 +1414,7 @@ static void gem_record_rings(struct i915_gpu_state *error)
 		error_record_engine_registers(error, engine, ee);
 		error_record_engine_execlists(engine, ee);
 
-		request = i915_gem_find_active_request(engine);
+		request = intel_engine_find_active_request(engine);
 		if (request) {
 			struct i915_gem_context *ctx = request->gem_context;
 			struct intel_ring *ring;
@@ -1669,7 +1607,7 @@ static void capture_reg_state(struct i915_gpu_state *error)
 	}
 
 	if (INTEL_GEN(dev_priv) >= 5)
-		error->ccid = I915_READ(CCID);
+		error->ccid = I915_READ(CCID(RENDER_RING_BASE));
 
 	/* 3: Feature specific registers */
 	if (IS_GEN_RANGE(dev_priv, 6, 7)) {
@@ -1697,16 +1635,17 @@ static void capture_reg_state(struct i915_gpu_state *error)
 		error->gtier[0] = I915_READ(GTIER);
 		error->ngtier = 1;
 	} else if (IS_GEN(dev_priv, 2)) {
-		error->ier = I915_READ16(IER);
+		error->ier = I915_READ16(GEN2_IER);
 	} else if (!IS_VALLEYVIEW(dev_priv)) {
-		error->ier = I915_READ(IER);
+		error->ier = I915_READ(GEN2_IER);
 	}
 	error->eir = I915_READ(EIR);
 	error->pgtbl_er = I915_READ(PGTBL_ER);
 }
 
 static const char *
-error_msg(struct i915_gpu_state *error, unsigned long engines, const char *msg)
+error_msg(struct i915_gpu_state *error,
+	  intel_engine_mask_t engines, const char *msg)
 {
 	int len;
 	int i;
@@ -1716,7 +1655,7 @@ error_msg(struct i915_gpu_state *error, unsigned long engines, const char *msg)
 			engines &= ~BIT(i);
 
 	len = scnprintf(error->error_msg, sizeof(error->error_msg),
-			"GPU HANG: ecode %d:%lx:0x%08x",
+			"GPU HANG: ecode %d:%x:0x%08x",
 			INTEL_GEN(error->i915), engines,
 			i915_error_generate_code(error, engines));
 	if (engines) {
@@ -1855,7 +1794,7 @@ i915_capture_gpu_state(struct drm_i915_private *i915)
  * to pick up.
  */
 void i915_capture_error_state(struct drm_i915_private *i915,
-			      unsigned long engine_mask,
+			      intel_engine_mask_t engine_mask,
 			      const char *msg)
 {
 	static bool warned;
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.h b/drivers/gpu/drm/i915/i915_gpu_error.h
index 53b1f22dd365..5dc761e85d9d 100644
--- a/drivers/gpu/drm/i915/i915_gpu_error.h
+++ b/drivers/gpu/drm/i915/i915_gpu_error.h
@@ -94,8 +94,6 @@ struct i915_gpu_state {
 		u32 cpu_ring_head;
 		u32 cpu_ring_tail;
 
-		u32 last_seqno;
-
 		/* Register state */
 		u32 start;
 		u32 tail;
@@ -108,24 +106,19 @@ struct i915_gpu_state {
 		u32 bbstate;
 		u32 instpm;
 		u32 instps;
-		u32 seqno;
 		u64 bbaddr;
 		u64 acthd;
 		u32 fault_reg;
 		u64 faddr;
 		u32 rc_psmi; /* sleep state */
-		u32 semaphore_mboxes[I915_NUM_ENGINES - 1];
 		struct intel_instdone instdone;
 
 		struct drm_i915_error_context {
 			char comm[TASK_COMM_LEN];
 			pid_t pid;
-			u32 handle;
 			u32 hw_id;
-			int ban_score;
 			int active;
 			int guilty;
-			bool bannable;
 			struct i915_sched_attr sched_attr;
 		} context;
 
@@ -149,7 +142,6 @@ struct i915_gpu_state {
 			long jiffies;
 			pid_t pid;
 			u32 context;
-			int ban_score;
 			u32 seqno;
 			u32 start;
 			u32 head;
@@ -170,7 +162,6 @@ struct i915_gpu_state {
 	struct drm_i915_error_buffer {
 		u32 size;
 		u32 name;
-		u32 wseqno;
 		u64 gtt_offset;
 		u32 read_domains;
 		u32 write_domain;
@@ -179,7 +170,6 @@ struct i915_gpu_state {
 		u32 dirty:1;
 		u32 purgeable:1;
 		u32 userptr:1;
-		s32 engine:4;
 		u32 cache_level:3;
 	} *active_bo[I915_NUM_ENGINES], *pinned_bo;
 	u32 active_bo_count[I915_NUM_ENGINES], pinned_bo_count;
@@ -205,38 +195,12 @@ struct i915_gpu_error {
 	atomic_t pending_fb_pin;
 
 	/**
-	 * State variable controlling the reset flow and count
-	 *
-	 * This is a counter which gets incremented when reset is triggered,
-	 *
-	 * Before the reset commences, the I915_RESET_BACKOFF bit is set
-	 * meaning that any waiters holding onto the struct_mutex should
-	 * relinquish the lock immediately in order for the reset to start.
-	 *
-	 * If reset is not completed successfully, the I915_WEDGE bit is
-	 * set meaning that hardware is terminally sour and there is no
-	 * recovery. All waiters on the reset_queue will be woken when
-	 * that happens.
-	 *
-	 * This counter is used by the wait_seqno code to notice that reset
-	 * event happened and it needs to restart the entire ioctl (since most
-	 * likely the seqno it waited for won't ever signal anytime soon).
-	 *
-	 * This is important for lock-free wait paths, where no contended lock
-	 * naturally enforces the correct ordering between the bail-out of the
-	 * waiter and the gpu reset work code.
-	 */
-	unsigned long reset_count;
-
-	/**
 	 * flags: Control various stages of the GPU reset
 	 *
-	 * #I915_RESET_BACKOFF - When we start a reset, we want to stop any
-	 * other users acquiring the struct_mutex. To do this we set the
-	 * #I915_RESET_BACKOFF bit in the error flags when we detect a reset
-	 * and then check for that bit before acquiring the struct_mutex (in
-	 * i915_mutex_lock_interruptible()?). I915_RESET_BACKOFF serves a
-	 * secondary role in preventing two concurrent global reset attempts.
+	 * #I915_RESET_BACKOFF - When we start a global reset, we need to
+	 * serialise with any other users attempting to do the same, and
+	 * any global resources that may be clobber by the reset (such as
+	 * FENCE registers).
 	 *
 	 * #I915_RESET_ENGINE[num_engines] - Since the driver doesn't need to
 	 * acquire the struct_mutex to reset an engine, we need an explicit
@@ -255,6 +219,9 @@ struct i915_gpu_error {
 #define I915_RESET_ENGINE	2
 #define I915_WEDGED		(BITS_PER_LONG - 1)
 
+	/** Number of times the device has been reset (global) */
+	u32 reset_count;
+
 	/** Number of times an engine has been reset */
 	u32 reset_engine_count[I915_NUM_ENGINES];
 
@@ -272,6 +239,8 @@ struct i915_gpu_error {
 	 */
 	wait_queue_head_t reset_queue;
 
+	struct srcu_struct reset_backoff_srcu;
+
 	struct i915_gpu_restart *restart;
 };
 
@@ -294,7 +263,7 @@ void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...);
 
 struct i915_gpu_state *i915_capture_gpu_state(struct drm_i915_private *i915);
 void i915_capture_error_state(struct drm_i915_private *dev_priv,
-			      unsigned long engine_mask,
+			      intel_engine_mask_t engine_mask,
 			      const char *error_msg);
 
 static inline struct i915_gpu_state *
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index 441d2674b272..b92cfd69134b 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -28,15 +28,19 @@
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
-#include <linux/sysrq.h>
-#include <linux/slab.h>
 #include <linux/circ_buf.h>
-#include <drm/drm_irq.h>
+#include <linux/cpuidle.h>
+#include <linux/slab.h>
+#include <linux/sysrq.h>
+
 #include <drm/drm_drv.h>
+#include <drm/drm_irq.h>
 #include <drm/i915_drm.h>
+
 #include "i915_drv.h"
 #include "i915_trace.h"
 #include "intel_drv.h"
+#include "intel_psr.h"
 
 /**
  * DOC: interrupt handling
@@ -132,92 +136,120 @@ static const u32 hpd_icp[HPD_NUM_PINS] = {
 	[HPD_PORT_F] = SDE_TC4_HOTPLUG_ICP
 };
 
-/* IIR can theoretically queue up two events. Be paranoid. */
-#define GEN8_IRQ_RESET_NDX(type, which) do { \
-	I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \
-	POSTING_READ(GEN8_##type##_IMR(which)); \
-	I915_WRITE(GEN8_##type##_IER(which), 0); \
-	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
-	POSTING_READ(GEN8_##type##_IIR(which)); \
-	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
-	POSTING_READ(GEN8_##type##_IIR(which)); \
-} while (0)
-
-#define GEN3_IRQ_RESET(type) do { \
-	I915_WRITE(type##IMR, 0xffffffff); \
-	POSTING_READ(type##IMR); \
-	I915_WRITE(type##IER, 0); \
-	I915_WRITE(type##IIR, 0xffffffff); \
-	POSTING_READ(type##IIR); \
-	I915_WRITE(type##IIR, 0xffffffff); \
-	POSTING_READ(type##IIR); \
-} while (0)
-
-#define GEN2_IRQ_RESET(type) do { \
-	I915_WRITE16(type##IMR, 0xffff); \
-	POSTING_READ16(type##IMR); \
-	I915_WRITE16(type##IER, 0); \
-	I915_WRITE16(type##IIR, 0xffff); \
-	POSTING_READ16(type##IIR); \
-	I915_WRITE16(type##IIR, 0xffff); \
-	POSTING_READ16(type##IIR); \
-} while (0)
+static void gen3_irq_reset(struct intel_uncore *uncore, i915_reg_t imr,
+			   i915_reg_t iir, i915_reg_t ier)
+{
+	intel_uncore_write(uncore, imr, 0xffffffff);
+	intel_uncore_posting_read(uncore, imr);
+
+	intel_uncore_write(uncore, ier, 0);
+
+	/* IIR can theoretically queue up two events. Be paranoid. */
+	intel_uncore_write(uncore, iir, 0xffffffff);
+	intel_uncore_posting_read(uncore, iir);
+	intel_uncore_write(uncore, iir, 0xffffffff);
+	intel_uncore_posting_read(uncore, iir);
+}
+
+static void gen2_irq_reset(struct intel_uncore *uncore)
+{
+	intel_uncore_write16(uncore, GEN2_IMR, 0xffff);
+	intel_uncore_posting_read16(uncore, GEN2_IMR);
+
+	intel_uncore_write16(uncore, GEN2_IER, 0);
+
+	/* IIR can theoretically queue up two events. Be paranoid. */
+	intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
+	intel_uncore_posting_read16(uncore, GEN2_IIR);
+	intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
+	intel_uncore_posting_read16(uncore, GEN2_IIR);
+}
+
+#define GEN8_IRQ_RESET_NDX(uncore, type, which) \
+({ \
+	unsigned int which_ = which; \
+	gen3_irq_reset((uncore), GEN8_##type##_IMR(which_), \
+		       GEN8_##type##_IIR(which_), GEN8_##type##_IER(which_)); \
+})
+
+#define GEN3_IRQ_RESET(uncore, type) \
+	gen3_irq_reset((uncore), type##IMR, type##IIR, type##IER)
+
+#define GEN2_IRQ_RESET(uncore) \
+	gen2_irq_reset(uncore)
 
 /*
  * We should clear IMR at preinstall/uninstall, and just check at postinstall.
  */
-static void gen3_assert_iir_is_zero(struct drm_i915_private *dev_priv,
-				    i915_reg_t reg)
+static void gen3_assert_iir_is_zero(struct intel_uncore *uncore, i915_reg_t reg)
 {
-	u32 val = I915_READ(reg);
+	u32 val = intel_uncore_read(uncore, reg);
 
 	if (val == 0)
 		return;
 
 	WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n",
 	     i915_mmio_reg_offset(reg), val);
-	I915_WRITE(reg, 0xffffffff);
-	POSTING_READ(reg);
-	I915_WRITE(reg, 0xffffffff);
-	POSTING_READ(reg);
+	intel_uncore_write(uncore, reg, 0xffffffff);
+	intel_uncore_posting_read(uncore, reg);
+	intel_uncore_write(uncore, reg, 0xffffffff);
+	intel_uncore_posting_read(uncore, reg);
 }
 
-static void gen2_assert_iir_is_zero(struct drm_i915_private *dev_priv,
-				    i915_reg_t reg)
+static void gen2_assert_iir_is_zero(struct intel_uncore *uncore)
 {
-	u16 val = I915_READ16(reg);
+	u16 val = intel_uncore_read16(uncore, GEN2_IIR);
 
 	if (val == 0)
 		return;
 
 	WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n",
-	     i915_mmio_reg_offset(reg), val);
-	I915_WRITE16(reg, 0xffff);
-	POSTING_READ16(reg);
-	I915_WRITE16(reg, 0xffff);
-	POSTING_READ16(reg);
-}
-
-#define GEN8_IRQ_INIT_NDX(type, which, imr_val, ier_val) do { \
-	gen3_assert_iir_is_zero(dev_priv, GEN8_##type##_IIR(which)); \
-	I915_WRITE(GEN8_##type##_IER(which), (ier_val)); \
-	I915_WRITE(GEN8_##type##_IMR(which), (imr_val)); \
-	POSTING_READ(GEN8_##type##_IMR(which)); \
-} while (0)
-
-#define GEN3_IRQ_INIT(type, imr_val, ier_val) do { \
-	gen3_assert_iir_is_zero(dev_priv, type##IIR); \
-	I915_WRITE(type##IER, (ier_val)); \
-	I915_WRITE(type##IMR, (imr_val)); \
-	POSTING_READ(type##IMR); \
-} while (0)
-
-#define GEN2_IRQ_INIT(type, imr_val, ier_val) do { \
-	gen2_assert_iir_is_zero(dev_priv, type##IIR); \
-	I915_WRITE16(type##IER, (ier_val)); \
-	I915_WRITE16(type##IMR, (imr_val)); \
-	POSTING_READ16(type##IMR); \
-} while (0)
+	     i915_mmio_reg_offset(GEN2_IIR), val);
+	intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
+	intel_uncore_posting_read16(uncore, GEN2_IIR);
+	intel_uncore_write16(uncore, GEN2_IIR, 0xffff);
+	intel_uncore_posting_read16(uncore, GEN2_IIR);
+}
+
+static void gen3_irq_init(struct intel_uncore *uncore,
+			  i915_reg_t imr, u32 imr_val,
+			  i915_reg_t ier, u32 ier_val,
+			  i915_reg_t iir)
+{
+	gen3_assert_iir_is_zero(uncore, iir);
+
+	intel_uncore_write(uncore, ier, ier_val);
+	intel_uncore_write(uncore, imr, imr_val);
+	intel_uncore_posting_read(uncore, imr);
+}
+
+static void gen2_irq_init(struct intel_uncore *uncore,
+			  u32 imr_val, u32 ier_val)
+{
+	gen2_assert_iir_is_zero(uncore);
+
+	intel_uncore_write16(uncore, GEN2_IER, ier_val);
+	intel_uncore_write16(uncore, GEN2_IMR, imr_val);
+	intel_uncore_posting_read16(uncore, GEN2_IMR);
+}
+
+#define GEN8_IRQ_INIT_NDX(uncore, type, which, imr_val, ier_val) \
+({ \
+	unsigned int which_ = which; \
+	gen3_irq_init((uncore), \
+		      GEN8_##type##_IMR(which_), imr_val, \
+		      GEN8_##type##_IER(which_), ier_val, \
+		      GEN8_##type##_IIR(which_)); \
+})
+
+#define GEN3_IRQ_INIT(uncore, type, imr_val, ier_val) \
+	gen3_irq_init((uncore), \
+		      type##IMR, imr_val, \
+		      type##IER, ier_val, \
+		      type##IIR)
+
+#define GEN2_IRQ_INIT(uncore, imr_val, ier_val) \
+	gen2_irq_init((uncore), imr_val, ier_val)
 
 static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);
 static void gen9_guc_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);
@@ -268,7 +300,7 @@ static bool gen11_reset_one_iir(struct drm_i915_private * const i915,
 				const unsigned int bank,
 				const unsigned int bit)
 {
-	void __iomem * const regs = i915->regs;
+	void __iomem * const regs = i915->uncore.regs;
 	u32 dw;
 
 	lockdep_assert_held(&i915->irq_lock);
@@ -365,24 +397,41 @@ static i915_reg_t gen6_pm_iir(struct drm_i915_private *dev_priv)
 	return INTEL_GEN(dev_priv) >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR;
 }
 
-static i915_reg_t gen6_pm_imr(struct drm_i915_private *dev_priv)
+static void write_pm_imr(struct drm_i915_private *dev_priv)
 {
-	if (INTEL_GEN(dev_priv) >= 11)
-		return GEN11_GPM_WGBOXPERF_INTR_MASK;
-	else if (INTEL_GEN(dev_priv) >= 8)
-		return GEN8_GT_IMR(2);
-	else
-		return GEN6_PMIMR;
+	i915_reg_t reg;
+	u32 mask = dev_priv->pm_imr;
+
+	if (INTEL_GEN(dev_priv) >= 11) {
+		reg = GEN11_GPM_WGBOXPERF_INTR_MASK;
+		/* pm is in upper half */
+		mask = mask << 16;
+	} else if (INTEL_GEN(dev_priv) >= 8) {
+		reg = GEN8_GT_IMR(2);
+	} else {
+		reg = GEN6_PMIMR;
+	}
+
+	I915_WRITE(reg, mask);
+	POSTING_READ(reg);
 }
 
-static i915_reg_t gen6_pm_ier(struct drm_i915_private *dev_priv)
+static void write_pm_ier(struct drm_i915_private *dev_priv)
 {
-	if (INTEL_GEN(dev_priv) >= 11)
-		return GEN11_GPM_WGBOXPERF_INTR_ENABLE;
-	else if (INTEL_GEN(dev_priv) >= 8)
-		return GEN8_GT_IER(2);
-	else
-		return GEN6_PMIER;
+	i915_reg_t reg;
+	u32 mask = dev_priv->pm_ier;
+
+	if (INTEL_GEN(dev_priv) >= 11) {
+		reg = GEN11_GPM_WGBOXPERF_INTR_ENABLE;
+		/* pm is in upper half */
+		mask = mask << 16;
+	} else if (INTEL_GEN(dev_priv) >= 8) {
+		reg = GEN8_GT_IER(2);
+	} else {
+		reg = GEN6_PMIER;
+	}
+
+	I915_WRITE(reg, mask);
 }
 
 /**
@@ -407,8 +456,7 @@ static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
 
 	if (new_val != dev_priv->pm_imr) {
 		dev_priv->pm_imr = new_val;
-		I915_WRITE(gen6_pm_imr(dev_priv), dev_priv->pm_imr);
-		POSTING_READ(gen6_pm_imr(dev_priv));
+		write_pm_imr(dev_priv);
 	}
 }
 
@@ -449,7 +497,7 @@ static void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, u32 enable_mas
 	lockdep_assert_held(&dev_priv->irq_lock);
 
 	dev_priv->pm_ier |= enable_mask;
-	I915_WRITE(gen6_pm_ier(dev_priv), dev_priv->pm_ier);
+	write_pm_ier(dev_priv);
 	gen6_unmask_pm_irq(dev_priv, enable_mask);
 	/* unmask_pm_irq provides an implicit barrier (POSTING_READ) */
 }
@@ -460,7 +508,7 @@ static void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, u32 disable_m
 
 	dev_priv->pm_ier &= ~disable_mask;
 	__gen6_mask_pm_irq(dev_priv, disable_mask);
-	I915_WRITE(gen6_pm_ier(dev_priv), dev_priv->pm_ier);
+	write_pm_ier(dev_priv);
 	/* though a barrier is missing here, but don't really need a one */
 }
 
@@ -748,13 +796,21 @@ void i915_disable_pipestat(struct drm_i915_private *dev_priv,
 	POSTING_READ(reg);
 }
 
+static bool i915_has_asle(struct drm_i915_private *dev_priv)
+{
+	if (!dev_priv->opregion.asle)
+		return false;
+
+	return IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv);
+}
+
 /**
  * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
  * @dev_priv: i915 device private
  */
 static void i915_enable_asle_pipestat(struct drm_i915_private *dev_priv)
 {
-	if (!dev_priv->opregion.asle || !IS_MOBILE(dev_priv))
+	if (!i915_has_asle(dev_priv))
 		return;
 
 	spin_lock_irq(&dev_priv->irq_lock);
@@ -1288,6 +1344,18 @@ static void gen6_pm_rps_work(struct work_struct *work)
 
 	rps->last_adj = adj;
 
+	/*
+	 * Limit deboosting and boosting to keep ourselves at the extremes
+	 * when in the respective power modes (i.e. slowly decrease frequencies
+	 * while in the HIGH_POWER zone and slowly increase frequencies while
+	 * in the LOW_POWER zone). On idle, we will hit the timeout and drop
+	 * to the next level quickly, and conversely if busy we expect to
+	 * hit a waitboost and rapidly switch into max power.
+	 */
+	if ((adj < 0 && rps->power.mode == HIGH_POWER) ||
+	    (adj > 0 && rps->power.mode == LOW_POWER))
+		rps->last_adj = 0;
+
 	/* sysfs frequency interfaces may have snuck in while servicing the
 	 * interrupt
 	 */
@@ -1415,20 +1483,20 @@ static void ilk_gt_irq_handler(struct drm_i915_private *dev_priv,
 			       u32 gt_iir)
 {
 	if (gt_iir & GT_RENDER_USER_INTERRUPT)
-		intel_engine_breadcrumbs_irq(dev_priv->engine[RCS]);
+		intel_engine_breadcrumbs_irq(dev_priv->engine[RCS0]);
 	if (gt_iir & ILK_BSD_USER_INTERRUPT)
-		intel_engine_breadcrumbs_irq(dev_priv->engine[VCS]);
+		intel_engine_breadcrumbs_irq(dev_priv->engine[VCS0]);
 }
 
 static void snb_gt_irq_handler(struct drm_i915_private *dev_priv,
 			       u32 gt_iir)
 {
 	if (gt_iir & GT_RENDER_USER_INTERRUPT)
-		intel_engine_breadcrumbs_irq(dev_priv->engine[RCS]);
+		intel_engine_breadcrumbs_irq(dev_priv->engine[RCS0]);
 	if (gt_iir & GT_BSD_USER_INTERRUPT)
-		intel_engine_breadcrumbs_irq(dev_priv->engine[VCS]);
+		intel_engine_breadcrumbs_irq(dev_priv->engine[VCS0]);
 	if (gt_iir & GT_BLT_USER_INTERRUPT)
-		intel_engine_breadcrumbs_irq(dev_priv->engine[BCS]);
+		intel_engine_breadcrumbs_irq(dev_priv->engine[BCS0]);
 
 	if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
 		      GT_BSD_CS_ERROR_INTERRUPT |
@@ -1449,7 +1517,7 @@ gen8_cs_irq_handler(struct intel_engine_cs *engine, u32 iir)
 
 	if (iir & GT_RENDER_USER_INTERRUPT) {
 		intel_engine_breadcrumbs_irq(engine);
-		tasklet |= USES_GUC_SUBMISSION(engine->i915);
+		tasklet |= intel_engine_needs_breadcrumb_tasklet(engine);
 	}
 
 	if (tasklet)
@@ -1459,12 +1527,12 @@ gen8_cs_irq_handler(struct intel_engine_cs *engine, u32 iir)
 static void gen8_gt_irq_ack(struct drm_i915_private *i915,
 			    u32 master_ctl, u32 gt_iir[4])
 {
-	void __iomem * const regs = i915->regs;
+	void __iomem * const regs = i915->uncore.regs;
 
 #define GEN8_GT_IRQS (GEN8_GT_RCS_IRQ | \
 		      GEN8_GT_BCS_IRQ | \
+		      GEN8_GT_VCS0_IRQ | \
 		      GEN8_GT_VCS1_IRQ | \
-		      GEN8_GT_VCS2_IRQ | \
 		      GEN8_GT_VECS_IRQ | \
 		      GEN8_GT_PM_IRQ | \
 		      GEN8_GT_GUC_IRQ)
@@ -1475,7 +1543,7 @@ static void gen8_gt_irq_ack(struct drm_i915_private *i915,
 			raw_reg_write(regs, GEN8_GT_IIR(0), gt_iir[0]);
 	}
 
-	if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) {
+	if (master_ctl & (GEN8_GT_VCS0_IRQ | GEN8_GT_VCS1_IRQ)) {
 		gt_iir[1] = raw_reg_read(regs, GEN8_GT_IIR(1));
 		if (likely(gt_iir[1]))
 			raw_reg_write(regs, GEN8_GT_IIR(1), gt_iir[1]);
@@ -1498,21 +1566,21 @@ static void gen8_gt_irq_handler(struct drm_i915_private *i915,
 				u32 master_ctl, u32 gt_iir[4])
 {
 	if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
-		gen8_cs_irq_handler(i915->engine[RCS],
+		gen8_cs_irq_handler(i915->engine[RCS0],
 				    gt_iir[0] >> GEN8_RCS_IRQ_SHIFT);
-		gen8_cs_irq_handler(i915->engine[BCS],
+		gen8_cs_irq_handler(i915->engine[BCS0],
 				    gt_iir[0] >> GEN8_BCS_IRQ_SHIFT);
 	}
 
-	if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) {
-		gen8_cs_irq_handler(i915->engine[VCS],
+	if (master_ctl & (GEN8_GT_VCS0_IRQ | GEN8_GT_VCS1_IRQ)) {
+		gen8_cs_irq_handler(i915->engine[VCS0],
+				    gt_iir[1] >> GEN8_VCS0_IRQ_SHIFT);
+		gen8_cs_irq_handler(i915->engine[VCS1],
 				    gt_iir[1] >> GEN8_VCS1_IRQ_SHIFT);
-		gen8_cs_irq_handler(i915->engine[VCS2],
-				    gt_iir[1] >> GEN8_VCS2_IRQ_SHIFT);
 	}
 
 	if (master_ctl & GEN8_GT_VECS_IRQ) {
-		gen8_cs_irq_handler(i915->engine[VECS],
+		gen8_cs_irq_handler(i915->engine[VECS0],
 				    gt_iir[3] >> GEN8_VECS_IRQ_SHIFT);
 	}
 
@@ -1693,7 +1761,9 @@ static void display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
 {
 	struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
 	struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
-	u32 crcs[5];
+	u32 crcs[5] = { crc0, crc1, crc2, crc3, crc4 };
+
+	trace_intel_pipe_crc(crtc, crcs);
 
 	spin_lock(&pipe_crc->lock);
 	/*
@@ -1712,11 +1782,6 @@ static void display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
 	}
 	spin_unlock(&pipe_crc->lock);
 
-	crcs[0] = crc0;
-	crcs[1] = crc1;
-	crcs[2] = crc2;
-	crcs[3] = crc3;
-	crcs[4] = crc4;
 	drm_crtc_add_crc_entry(&crtc->base, true,
 				drm_crtc_accurate_vblank_count(&crtc->base),
 				crcs);
@@ -1775,6 +1840,25 @@ static void i9xx_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
 /* The RPS events need forcewake, so we add them to a work queue and mask their
  * IMR bits until the work is done. Other interrupts can be processed without
  * the work queue. */
+static void gen11_rps_irq_handler(struct drm_i915_private *i915, u32 pm_iir)
+{
+	struct intel_rps *rps = &i915->gt_pm.rps;
+	const u32 events = i915->pm_rps_events & pm_iir;
+
+	lockdep_assert_held(&i915->irq_lock);
+
+	if (unlikely(!events))
+		return;
+
+	gen6_mask_pm_irq(i915, events);
+
+	if (!rps->interrupts_enabled)
+		return;
+
+	rps->pm_iir |= events;
+	schedule_work(&rps->work);
+}
+
 static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
 {
 	struct intel_rps *rps = &dev_priv->gt_pm.rps;
@@ -1792,13 +1876,11 @@ static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
 	if (INTEL_GEN(dev_priv) >= 8)
 		return;
 
-	if (HAS_VEBOX(dev_priv)) {
-		if (pm_iir & PM_VEBOX_USER_INTERRUPT)
-			intel_engine_breadcrumbs_irq(dev_priv->engine[VECS]);
+	if (pm_iir & PM_VEBOX_USER_INTERRUPT)
+		intel_engine_breadcrumbs_irq(dev_priv->engine[VECS0]);
 
-		if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
-			DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
-	}
+	if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
+		DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
 }
 
 static void gen9_guc_irq_handler(struct drm_i915_private *dev_priv, u32 gt_iir)
@@ -2667,6 +2749,25 @@ static void gen11_hpd_irq_handler(struct drm_i915_private *dev_priv, u32 iir)
 		DRM_ERROR("Unexpected DE HPD interrupt 0x%08x\n", iir);
 }
 
+static u32 gen8_de_port_aux_mask(struct drm_i915_private *dev_priv)
+{
+	u32 mask = GEN8_AUX_CHANNEL_A;
+
+	if (INTEL_GEN(dev_priv) >= 9)
+		mask |= GEN9_AUX_CHANNEL_B |
+			GEN9_AUX_CHANNEL_C |
+			GEN9_AUX_CHANNEL_D;
+
+	if (IS_CNL_WITH_PORT_F(dev_priv))
+		mask |= CNL_AUX_CHANNEL_F;
+
+	if (INTEL_GEN(dev_priv) >= 11)
+		mask |= ICL_AUX_CHANNEL_E |
+			CNL_AUX_CHANNEL_F;
+
+	return mask;
+}
+
 static irqreturn_t
 gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
 {
@@ -2722,20 +2823,7 @@ gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
 			I915_WRITE(GEN8_DE_PORT_IIR, iir);
 			ret = IRQ_HANDLED;
 
-			tmp_mask = GEN8_AUX_CHANNEL_A;
-			if (INTEL_GEN(dev_priv) >= 9)
-				tmp_mask |= GEN9_AUX_CHANNEL_B |
-					    GEN9_AUX_CHANNEL_C |
-					    GEN9_AUX_CHANNEL_D;
-
-			if (INTEL_GEN(dev_priv) >= 11)
-				tmp_mask |= ICL_AUX_CHANNEL_E;
-
-			if (IS_CNL_WITH_PORT_F(dev_priv) ||
-			    INTEL_GEN(dev_priv) >= 11)
-				tmp_mask |= CNL_AUX_CHANNEL_F;
-
-			if (iir & tmp_mask) {
+			if (iir & gen8_de_port_aux_mask(dev_priv)) {
 				dp_aux_irq_handler(dev_priv);
 				found = true;
 			}
@@ -2816,11 +2904,9 @@ gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
 			I915_WRITE(SDEIIR, iir);
 			ret = IRQ_HANDLED;
 
-			if (HAS_PCH_ICP(dev_priv))
+			if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
 				icp_irq_handler(dev_priv, iir);
-			else if (HAS_PCH_SPT(dev_priv) ||
-				 HAS_PCH_KBP(dev_priv) ||
-				 HAS_PCH_CNP(dev_priv))
+			else if (INTEL_PCH_TYPE(dev_priv) >= PCH_SPT)
 				spt_irq_handler(dev_priv, iir);
 			else
 				cpt_irq_handler(dev_priv, iir);
@@ -2857,7 +2943,7 @@ static inline void gen8_master_intr_enable(void __iomem * const regs)
 static irqreturn_t gen8_irq_handler(int irq, void *arg)
 {
 	struct drm_i915_private *dev_priv = to_i915(arg);
-	void __iomem * const regs = dev_priv->regs;
+	void __iomem * const regs = dev_priv->uncore.regs;
 	u32 master_ctl;
 	u32 gt_iir[4];
 
@@ -2891,7 +2977,7 @@ static u32
 gen11_gt_engine_identity(struct drm_i915_private * const i915,
 			 const unsigned int bank, const unsigned int bit)
 {
-	void __iomem * const regs = i915->regs;
+	void __iomem * const regs = i915->uncore.regs;
 	u32 timeout_ts;
 	u32 ident;
 
@@ -2926,7 +3012,7 @@ gen11_other_irq_handler(struct drm_i915_private * const i915,
 			const u8 instance, const u16 iir)
 {
 	if (instance == OTHER_GTPM_INSTANCE)
-		return gen6_rps_irq_handler(i915, iir);
+		return gen11_rps_irq_handler(i915, iir);
 
 	WARN_ONCE(1, "unhandled other interrupt instance=0x%x, iir=0x%x\n",
 		  instance, iir);
@@ -2975,7 +3061,7 @@ static void
 gen11_gt_bank_handler(struct drm_i915_private * const i915,
 		      const unsigned int bank)
 {
-	void __iomem * const regs = i915->regs;
+	void __iomem * const regs = i915->uncore.regs;
 	unsigned long intr_dw;
 	unsigned int bit;
 
@@ -2983,14 +3069,8 @@ gen11_gt_bank_handler(struct drm_i915_private * const i915,
 
 	intr_dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank));
 
-	if (unlikely(!intr_dw)) {
-		DRM_ERROR("GT_INTR_DW%u blank!\n", bank);
-		return;
-	}
-
 	for_each_set_bit(bit, &intr_dw, 32) {
-		const u32 ident = gen11_gt_engine_identity(i915,
-							   bank, bit);
+		const u32 ident = gen11_gt_engine_identity(i915, bank, bit);
 
 		gen11_gt_identity_handler(i915, ident);
 	}
@@ -3018,7 +3098,7 @@ gen11_gt_irq_handler(struct drm_i915_private * const i915,
 static u32
 gen11_gu_misc_irq_ack(struct drm_i915_private *dev_priv, const u32 master_ctl)
 {
-	void __iomem * const regs = dev_priv->regs;
+	void __iomem * const regs = dev_priv->uncore.regs;
 	u32 iir;
 
 	if (!(master_ctl & GEN11_GU_MISC_IRQ))
@@ -3059,7 +3139,7 @@ static inline void gen11_master_intr_enable(void __iomem * const regs)
 static irqreturn_t gen11_irq_handler(int irq, void *arg)
 {
 	struct drm_i915_private * const i915 = to_i915(arg);
-	void __iomem * const regs = i915->regs;
+	void __iomem * const regs = i915->uncore.regs;
 	u32 master_ctl;
 	u32 gu_misc_iir;
 
@@ -3112,6 +3192,16 @@ static int i8xx_enable_vblank(struct drm_device *dev, unsigned int pipe)
 	return 0;
 }
 
+static int i945gm_enable_vblank(struct drm_device *dev, unsigned int pipe)
+{
+	struct drm_i915_private *dev_priv = to_i915(dev);
+
+	if (dev_priv->i945gm_vblank.enabled++ == 0)
+		schedule_work(&dev_priv->i945gm_vblank.work);
+
+	return i8xx_enable_vblank(dev, pipe);
+}
+
 static int i965_enable_vblank(struct drm_device *dev, unsigned int pipe)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
@@ -3176,6 +3266,16 @@ static void i8xx_disable_vblank(struct drm_device *dev, unsigned int pipe)
 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 }
 
+static void i945gm_disable_vblank(struct drm_device *dev, unsigned int pipe)
+{
+	struct drm_i915_private *dev_priv = to_i915(dev);
+
+	i8xx_disable_vblank(dev, pipe);
+
+	if (--dev_priv->i945gm_vblank.enabled == 0)
+		schedule_work(&dev_priv->i945gm_vblank.work);
+}
+
 static void i965_disable_vblank(struct drm_device *dev, unsigned int pipe)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
@@ -3209,12 +3309,68 @@ static void gen8_disable_vblank(struct drm_device *dev, unsigned int pipe)
 	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 }
 
+static void i945gm_vblank_work_func(struct work_struct *work)
+{
+	struct drm_i915_private *dev_priv =
+		container_of(work, struct drm_i915_private, i945gm_vblank.work);
+
+	/*
+	 * Vblank interrupts fail to wake up the device from C3,
+	 * hence we want to prevent C3 usage while vblank interrupts
+	 * are enabled.
+	 */
+	pm_qos_update_request(&dev_priv->i945gm_vblank.pm_qos,
+			      READ_ONCE(dev_priv->i945gm_vblank.enabled) ?
+			      dev_priv->i945gm_vblank.c3_disable_latency :
+			      PM_QOS_DEFAULT_VALUE);
+}
+
+static int cstate_disable_latency(const char *name)
+{
+	const struct cpuidle_driver *drv;
+	int i;
+
+	drv = cpuidle_get_driver();
+	if (!drv)
+		return 0;
+
+	for (i = 0; i < drv->state_count; i++) {
+		const struct cpuidle_state *state = &drv->states[i];
+
+		if (!strcmp(state->name, name))
+			return state->exit_latency ?
+				state->exit_latency - 1 : 0;
+	}
+
+	return 0;
+}
+
+static void i945gm_vblank_work_init(struct drm_i915_private *dev_priv)
+{
+	INIT_WORK(&dev_priv->i945gm_vblank.work,
+		  i945gm_vblank_work_func);
+
+	dev_priv->i945gm_vblank.c3_disable_latency =
+		cstate_disable_latency("C3");
+	pm_qos_add_request(&dev_priv->i945gm_vblank.pm_qos,
+			   PM_QOS_CPU_DMA_LATENCY,
+			   PM_QOS_DEFAULT_VALUE);
+}
+
+static void i945gm_vblank_work_fini(struct drm_i915_private *dev_priv)
+{
+	cancel_work_sync(&dev_priv->i945gm_vblank.work);
+	pm_qos_remove_request(&dev_priv->i945gm_vblank.pm_qos);
+}
+
 static void ibx_irq_reset(struct drm_i915_private *dev_priv)
 {
+	struct intel_uncore *uncore = &dev_priv->uncore;
+
 	if (HAS_PCH_NOP(dev_priv))
 		return;
 
-	GEN3_IRQ_RESET(SDE);
+	GEN3_IRQ_RESET(uncore, SDE);
 
 	if (HAS_PCH_CPT(dev_priv) || HAS_PCH_LPT(dev_priv))
 		I915_WRITE(SERR_INT, 0xffffffff);
@@ -3242,13 +3398,17 @@ static void ibx_irq_pre_postinstall(struct drm_device *dev)
 
 static void gen5_gt_irq_reset(struct drm_i915_private *dev_priv)
 {
-	GEN3_IRQ_RESET(GT);
+	struct intel_uncore *uncore = &dev_priv->uncore;
+
+	GEN3_IRQ_RESET(uncore, GT);
 	if (INTEL_GEN(dev_priv) >= 6)
-		GEN3_IRQ_RESET(GEN6_PM);
+		GEN3_IRQ_RESET(uncore, GEN6_PM);
 }
 
 static void vlv_display_irq_reset(struct drm_i915_private *dev_priv)
 {
+	struct intel_uncore *uncore = &dev_priv->uncore;
+
 	if (IS_CHERRYVIEW(dev_priv))
 		I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK_CHV);
 	else
@@ -3259,12 +3419,14 @@ static void vlv_display_irq_reset(struct drm_i915_private *dev_priv)
 
 	i9xx_pipestat_irq_reset(dev_priv);
 
-	GEN3_IRQ_RESET(VLV_);
+	GEN3_IRQ_RESET(uncore, VLV_);
 	dev_priv->irq_mask = ~0u;
 }
 
 static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
 {
+	struct intel_uncore *uncore = &dev_priv->uncore;
+
 	u32 pipestat_mask;
 	u32 enable_mask;
 	enum pipe pipe;
@@ -3289,7 +3451,7 @@ static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
 
 	dev_priv->irq_mask = ~enable_mask;
 
-	GEN3_IRQ_INIT(VLV_, dev_priv->irq_mask, enable_mask);
+	GEN3_IRQ_INIT(uncore, VLV_, dev_priv->irq_mask, enable_mask);
 }
 
 /* drm_dma.h hooks
@@ -3297,8 +3459,9 @@ static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
 static void ironlake_irq_reset(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct intel_uncore *uncore = &dev_priv->uncore;
 
-	GEN3_IRQ_RESET(DE);
+	GEN3_IRQ_RESET(uncore, DE);
 	if (IS_GEN(dev_priv, 7))
 		I915_WRITE(GEN7_ERR_INT, 0xffffffff);
 
@@ -3329,18 +3492,21 @@ static void valleyview_irq_reset(struct drm_device *dev)
 
 static void gen8_gt_irq_reset(struct drm_i915_private *dev_priv)
 {
-	GEN8_IRQ_RESET_NDX(GT, 0);
-	GEN8_IRQ_RESET_NDX(GT, 1);
-	GEN8_IRQ_RESET_NDX(GT, 2);
-	GEN8_IRQ_RESET_NDX(GT, 3);
+	struct intel_uncore *uncore = &dev_priv->uncore;
+
+	GEN8_IRQ_RESET_NDX(uncore, GT, 0);
+	GEN8_IRQ_RESET_NDX(uncore, GT, 1);
+	GEN8_IRQ_RESET_NDX(uncore, GT, 2);
+	GEN8_IRQ_RESET_NDX(uncore, GT, 3);
 }
 
 static void gen8_irq_reset(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct intel_uncore *uncore = &dev_priv->uncore;
 	int pipe;
 
-	gen8_master_intr_disable(dev_priv->regs);
+	gen8_master_intr_disable(dev_priv->uncore.regs);
 
 	gen8_gt_irq_reset(dev_priv);
 
@@ -3350,11 +3516,11 @@ static void gen8_irq_reset(struct drm_device *dev)
 	for_each_pipe(dev_priv, pipe)
 		if (intel_display_power_is_enabled(dev_priv,
 						   POWER_DOMAIN_PIPE(pipe)))
-			GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
+			GEN8_IRQ_RESET_NDX(uncore, DE_PIPE, pipe);
 
-	GEN3_IRQ_RESET(GEN8_DE_PORT_);
-	GEN3_IRQ_RESET(GEN8_DE_MISC_);
-	GEN3_IRQ_RESET(GEN8_PCU_);
+	GEN3_IRQ_RESET(uncore, GEN8_DE_PORT_);
+	GEN3_IRQ_RESET(uncore, GEN8_DE_MISC_);
+	GEN3_IRQ_RESET(uncore, GEN8_PCU_);
 
 	if (HAS_PCH_SPLIT(dev_priv))
 		ibx_irq_reset(dev_priv);
@@ -3380,9 +3546,10 @@ static void gen11_gt_irq_reset(struct drm_i915_private *dev_priv)
 static void gen11_irq_reset(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct intel_uncore *uncore = &dev_priv->uncore;
 	int pipe;
 
-	gen11_master_intr_disable(dev_priv->regs);
+	gen11_master_intr_disable(dev_priv->uncore.regs);
 
 	gen11_gt_irq_reset(dev_priv);
 
@@ -3394,21 +3561,23 @@ static void gen11_irq_reset(struct drm_device *dev)
 	for_each_pipe(dev_priv, pipe)
 		if (intel_display_power_is_enabled(dev_priv,
 						   POWER_DOMAIN_PIPE(pipe)))
-			GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
+			GEN8_IRQ_RESET_NDX(uncore, DE_PIPE, pipe);
 
-	GEN3_IRQ_RESET(GEN8_DE_PORT_);
-	GEN3_IRQ_RESET(GEN8_DE_MISC_);
-	GEN3_IRQ_RESET(GEN11_DE_HPD_);
-	GEN3_IRQ_RESET(GEN11_GU_MISC_);
-	GEN3_IRQ_RESET(GEN8_PCU_);
+	GEN3_IRQ_RESET(uncore, GEN8_DE_PORT_);
+	GEN3_IRQ_RESET(uncore, GEN8_DE_MISC_);
+	GEN3_IRQ_RESET(uncore, GEN11_DE_HPD_);
+	GEN3_IRQ_RESET(uncore, GEN11_GU_MISC_);
+	GEN3_IRQ_RESET(uncore, GEN8_PCU_);
 
-	if (HAS_PCH_ICP(dev_priv))
-		GEN3_IRQ_RESET(SDE);
+	if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
+		GEN3_IRQ_RESET(uncore, SDE);
 }
 
 void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
 				     u8 pipe_mask)
 {
+	struct intel_uncore *uncore = &dev_priv->uncore;
+
 	u32 extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN;
 	enum pipe pipe;
 
@@ -3420,7 +3589,7 @@ void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
 	}
 
 	for_each_pipe_masked(dev_priv, pipe, pipe_mask)
-		GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
+		GEN8_IRQ_INIT_NDX(uncore, DE_PIPE, pipe,
 				  dev_priv->de_irq_mask[pipe],
 				  ~dev_priv->de_irq_mask[pipe] | extra_ier);
 
@@ -3430,6 +3599,7 @@ void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
 void gen8_irq_power_well_pre_disable(struct drm_i915_private *dev_priv,
 				     u8 pipe_mask)
 {
+	struct intel_uncore *uncore = &dev_priv->uncore;
 	enum pipe pipe;
 
 	spin_lock_irq(&dev_priv->irq_lock);
@@ -3440,7 +3610,7 @@ void gen8_irq_power_well_pre_disable(struct drm_i915_private *dev_priv,
 	}
 
 	for_each_pipe_masked(dev_priv, pipe, pipe_mask)
-		GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
+		GEN8_IRQ_RESET_NDX(uncore, DE_PIPE, pipe);
 
 	spin_unlock_irq(&dev_priv->irq_lock);
 
@@ -3451,13 +3621,14 @@ void gen8_irq_power_well_pre_disable(struct drm_i915_private *dev_priv,
 static void cherryview_irq_reset(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct intel_uncore *uncore = &dev_priv->uncore;
 
 	I915_WRITE(GEN8_MASTER_IRQ, 0);
 	POSTING_READ(GEN8_MASTER_IRQ);
 
 	gen8_gt_irq_reset(dev_priv);
 
-	GEN3_IRQ_RESET(GEN8_PCU_);
+	GEN3_IRQ_RESET(uncore, GEN8_PCU_);
 
 	spin_lock_irq(&dev_priv->irq_lock);
 	if (dev_priv->display_irqs_enabled)
@@ -3583,7 +3754,7 @@ static void gen11_hpd_irq_setup(struct drm_i915_private *dev_priv)
 
 	gen11_hpd_detection_setup(dev_priv);
 
-	if (HAS_PCH_ICP(dev_priv))
+	if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
 		icp_hpd_irq_setup(dev_priv);
 }
 
@@ -3729,7 +3900,7 @@ static void ibx_irq_postinstall(struct drm_device *dev)
 	else
 		mask = SDE_GMBUS_CPT;
 
-	gen3_assert_iir_is_zero(dev_priv, SDEIIR);
+	gen3_assert_iir_is_zero(&dev_priv->uncore, SDEIIR);
 	I915_WRITE(SDEIMR, ~mask);
 
 	if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv) ||
@@ -3742,6 +3913,7 @@ static void ibx_irq_postinstall(struct drm_device *dev)
 static void gen5_gt_irq_postinstall(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct intel_uncore *uncore = &dev_priv->uncore;
 	u32 pm_irqs, gt_irqs;
 
 	pm_irqs = gt_irqs = 0;
@@ -3760,26 +3932,27 @@ static void gen5_gt_irq_postinstall(struct drm_device *dev)
 		gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
 	}
 
-	GEN3_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs);
+	GEN3_IRQ_INIT(uncore, GT, dev_priv->gt_irq_mask, gt_irqs);
 
 	if (INTEL_GEN(dev_priv) >= 6) {
 		/*
 		 * RPS interrupts will get enabled/disabled on demand when RPS
 		 * itself is enabled/disabled.
 		 */
-		if (HAS_VEBOX(dev_priv)) {
+		if (HAS_ENGINE(dev_priv, VECS0)) {
 			pm_irqs |= PM_VEBOX_USER_INTERRUPT;
 			dev_priv->pm_ier |= PM_VEBOX_USER_INTERRUPT;
 		}
 
 		dev_priv->pm_imr = 0xffffffff;
-		GEN3_IRQ_INIT(GEN6_PM, dev_priv->pm_imr, pm_irqs);
+		GEN3_IRQ_INIT(uncore, GEN6_PM, dev_priv->pm_imr, pm_irqs);
 	}
 }
 
 static int ironlake_irq_postinstall(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct intel_uncore *uncore = &dev_priv->uncore;
 	u32 display_mask, extra_mask;
 
 	if (INTEL_GEN(dev_priv) >= 7) {
@@ -3798,7 +3971,7 @@ static int ironlake_irq_postinstall(struct drm_device *dev)
 	}
 
 	if (IS_HASWELL(dev_priv)) {
-		gen3_assert_iir_is_zero(dev_priv, EDP_PSR_IIR);
+		gen3_assert_iir_is_zero(uncore, EDP_PSR_IIR);
 		intel_psr_irq_control(dev_priv, dev_priv->psr.debug);
 		display_mask |= DE_EDP_PSR_INT_HSW;
 	}
@@ -3807,7 +3980,8 @@ static int ironlake_irq_postinstall(struct drm_device *dev)
 
 	ibx_irq_pre_postinstall(dev);
 
-	GEN3_IRQ_INIT(DE, dev_priv->irq_mask, display_mask | extra_mask);
+	GEN3_IRQ_INIT(uncore, DE, dev_priv->irq_mask,
+		      display_mask | extra_mask);
 
 	gen5_gt_irq_postinstall(dev);
 
@@ -3877,35 +4051,42 @@ static int valleyview_irq_postinstall(struct drm_device *dev)
 
 static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv)
 {
+	struct intel_uncore *uncore = &dev_priv->uncore;
+
 	/* These are interrupts we'll toggle with the ring mask register */
 	u32 gt_interrupts[] = {
-		GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
-			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
-			GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT |
-			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT,
-		GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
-			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
-			GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT |
-			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT,
+		(GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
+		 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
+		 GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT |
+		 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT),
+
+		(GT_RENDER_USER_INTERRUPT << GEN8_VCS0_IRQ_SHIFT |
+		 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS0_IRQ_SHIFT |
+		 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
+		 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT),
+
 		0,
-		GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT |
-			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT
-		};
+
+		(GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT |
+		 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT)
+	};
 
 	dev_priv->pm_ier = 0x0;
 	dev_priv->pm_imr = ~dev_priv->pm_ier;
-	GEN8_IRQ_INIT_NDX(GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
-	GEN8_IRQ_INIT_NDX(GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
+	GEN8_IRQ_INIT_NDX(uncore, GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
+	GEN8_IRQ_INIT_NDX(uncore, GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
 	/*
 	 * RPS interrupts will get enabled/disabled on demand when RPS itself
 	 * is enabled/disabled. Same wil be the case for GuC interrupts.
 	 */
-	GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_imr, dev_priv->pm_ier);
-	GEN8_IRQ_INIT_NDX(GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
+	GEN8_IRQ_INIT_NDX(uncore, GT, 2, dev_priv->pm_imr, dev_priv->pm_ier);
+	GEN8_IRQ_INIT_NDX(uncore, GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
 }
 
 static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
 {
+	struct intel_uncore *uncore = &dev_priv->uncore;
+
 	u32 de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE;
 	u32 de_pipe_enables;
 	u32 de_port_masked = GEN8_AUX_CHANNEL_A;
@@ -3941,7 +4122,7 @@ static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
 	else if (IS_BROADWELL(dev_priv))
 		de_port_enables |= GEN8_PORT_DP_A_HOTPLUG;
 
-	gen3_assert_iir_is_zero(dev_priv, EDP_PSR_IIR);
+	gen3_assert_iir_is_zero(uncore, EDP_PSR_IIR);
 	intel_psr_irq_control(dev_priv, dev_priv->psr.debug);
 
 	for_each_pipe(dev_priv, pipe) {
@@ -3949,20 +4130,21 @@ static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
 
 		if (intel_display_power_is_enabled(dev_priv,
 				POWER_DOMAIN_PIPE(pipe)))
-			GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
+			GEN8_IRQ_INIT_NDX(uncore, DE_PIPE, pipe,
 					  dev_priv->de_irq_mask[pipe],
 					  de_pipe_enables);
 	}
 
-	GEN3_IRQ_INIT(GEN8_DE_PORT_, ~de_port_masked, de_port_enables);
-	GEN3_IRQ_INIT(GEN8_DE_MISC_, ~de_misc_masked, de_misc_masked);
+	GEN3_IRQ_INIT(uncore, GEN8_DE_PORT_, ~de_port_masked, de_port_enables);
+	GEN3_IRQ_INIT(uncore, GEN8_DE_MISC_, ~de_misc_masked, de_misc_masked);
 
 	if (INTEL_GEN(dev_priv) >= 11) {
 		u32 de_hpd_masked = 0;
 		u32 de_hpd_enables = GEN11_DE_TC_HOTPLUG_MASK |
 				     GEN11_DE_TBT_HOTPLUG_MASK;
 
-		GEN3_IRQ_INIT(GEN11_DE_HPD_, ~de_hpd_masked, de_hpd_enables);
+		GEN3_IRQ_INIT(uncore, GEN11_DE_HPD_, ~de_hpd_masked,
+			      de_hpd_enables);
 		gen11_hpd_detection_setup(dev_priv);
 	} else if (IS_GEN9_LP(dev_priv)) {
 		bxt_hpd_detection_setup(dev_priv);
@@ -3984,7 +4166,7 @@ static int gen8_irq_postinstall(struct drm_device *dev)
 	if (HAS_PCH_SPLIT(dev_priv))
 		ibx_irq_postinstall(dev);
 
-	gen8_master_intr_enable(dev_priv->regs);
+	gen8_master_intr_enable(dev_priv->uncore.regs);
 
 	return 0;
 }
@@ -4025,7 +4207,7 @@ static void icp_irq_postinstall(struct drm_device *dev)
 	I915_WRITE(SDEIER, 0xffffffff);
 	POSTING_READ(SDEIER);
 
-	gen3_assert_iir_is_zero(dev_priv, SDEIIR);
+	gen3_assert_iir_is_zero(&dev_priv->uncore, SDEIIR);
 	I915_WRITE(SDEIMR, ~mask);
 
 	icp_hpd_detection_setup(dev_priv);
@@ -4034,19 +4216,20 @@ static void icp_irq_postinstall(struct drm_device *dev)
 static int gen11_irq_postinstall(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct intel_uncore *uncore = &dev_priv->uncore;
 	u32 gu_misc_masked = GEN11_GU_MISC_GSE;
 
-	if (HAS_PCH_ICP(dev_priv))
+	if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
 		icp_irq_postinstall(dev);
 
 	gen11_gt_irq_postinstall(dev_priv);
 	gen8_de_irq_postinstall(dev_priv);
 
-	GEN3_IRQ_INIT(GEN11_GU_MISC_, ~gu_misc_masked, gu_misc_masked);
+	GEN3_IRQ_INIT(uncore, GEN11_GU_MISC_, ~gu_misc_masked, gu_misc_masked);
 
 	I915_WRITE(GEN11_DISPLAY_INT_CTL, GEN11_DISPLAY_IRQ_ENABLE);
 
-	gen11_master_intr_enable(dev_priv->regs);
+	gen11_master_intr_enable(dev_priv->uncore.regs);
 	POSTING_READ(GEN11_GFX_MSTR_IRQ);
 
 	return 0;
@@ -4072,15 +4255,17 @@ static int cherryview_irq_postinstall(struct drm_device *dev)
 static void i8xx_irq_reset(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct intel_uncore *uncore = &dev_priv->uncore;
 
 	i9xx_pipestat_irq_reset(dev_priv);
 
-	GEN2_IRQ_RESET();
+	GEN2_IRQ_RESET(uncore);
 }
 
 static int i8xx_irq_postinstall(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct intel_uncore *uncore = &dev_priv->uncore;
 	u16 enable_mask;
 
 	I915_WRITE16(EMR, ~(I915_ERROR_PAGE_TABLE |
@@ -4098,7 +4283,7 @@ static int i8xx_irq_postinstall(struct drm_device *dev)
 		I915_MASTER_ERROR_INTERRUPT |
 		I915_USER_INTERRUPT;
 
-	GEN2_IRQ_INIT(, dev_priv->irq_mask, enable_mask);
+	GEN2_IRQ_INIT(uncore, dev_priv->irq_mask, enable_mask);
 
 	/* Interrupt setup is already guaranteed to be single-threaded, this is
 	 * just to make the assert_spin_locked check happy. */
@@ -4202,7 +4387,7 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg)
 		u16 eir = 0, eir_stuck = 0;
 		u16 iir;
 
-		iir = I915_READ16(IIR);
+		iir = I915_READ16(GEN2_IIR);
 		if (iir == 0)
 			break;
 
@@ -4215,10 +4400,10 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg)
 		if (iir & I915_MASTER_ERROR_INTERRUPT)
 			i8xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
 
-		I915_WRITE16(IIR, iir);
+		I915_WRITE16(GEN2_IIR, iir);
 
 		if (iir & I915_USER_INTERRUPT)
-			intel_engine_breadcrumbs_irq(dev_priv->engine[RCS]);
+			intel_engine_breadcrumbs_irq(dev_priv->engine[RCS0]);
 
 		if (iir & I915_MASTER_ERROR_INTERRUPT)
 			i8xx_error_irq_handler(dev_priv, eir, eir_stuck);
@@ -4234,6 +4419,7 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg)
 static void i915_irq_reset(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct intel_uncore *uncore = &dev_priv->uncore;
 
 	if (I915_HAS_HOTPLUG(dev_priv)) {
 		i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
@@ -4242,12 +4428,13 @@ static void i915_irq_reset(struct drm_device *dev)
 
 	i9xx_pipestat_irq_reset(dev_priv);
 
-	GEN3_IRQ_RESET();
+	GEN3_IRQ_RESET(uncore, GEN2_);
 }
 
 static int i915_irq_postinstall(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct intel_uncore *uncore = &dev_priv->uncore;
 	u32 enable_mask;
 
 	I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE |
@@ -4274,7 +4461,7 @@ static int i915_irq_postinstall(struct drm_device *dev)
 		dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
 	}
 
-	GEN3_IRQ_INIT(, dev_priv->irq_mask, enable_mask);
+	GEN3_IRQ_INIT(uncore, GEN2_, dev_priv->irq_mask, enable_mask);
 
 	/* Interrupt setup is already guaranteed to be single-threaded, this is
 	 * just to make the assert_spin_locked check happy. */
@@ -4306,7 +4493,7 @@ static irqreturn_t i915_irq_handler(int irq, void *arg)
 		u32 hotplug_status = 0;
 		u32 iir;
 
-		iir = I915_READ(IIR);
+		iir = I915_READ(GEN2_IIR);
 		if (iir == 0)
 			break;
 
@@ -4323,10 +4510,10 @@ static irqreturn_t i915_irq_handler(int irq, void *arg)
 		if (iir & I915_MASTER_ERROR_INTERRUPT)
 			i9xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
 
-		I915_WRITE(IIR, iir);
+		I915_WRITE(GEN2_IIR, iir);
 
 		if (iir & I915_USER_INTERRUPT)
-			intel_engine_breadcrumbs_irq(dev_priv->engine[RCS]);
+			intel_engine_breadcrumbs_irq(dev_priv->engine[RCS0]);
 
 		if (iir & I915_MASTER_ERROR_INTERRUPT)
 			i9xx_error_irq_handler(dev_priv, eir, eir_stuck);
@@ -4345,18 +4532,20 @@ static irqreturn_t i915_irq_handler(int irq, void *arg)
 static void i965_irq_reset(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct intel_uncore *uncore = &dev_priv->uncore;
 
 	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
 	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
 
 	i9xx_pipestat_irq_reset(dev_priv);
 
-	GEN3_IRQ_RESET();
+	GEN3_IRQ_RESET(uncore, GEN2_);
 }
 
 static int i965_irq_postinstall(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct intel_uncore *uncore = &dev_priv->uncore;
 	u32 enable_mask;
 	u32 error_mask;
 
@@ -4394,7 +4583,7 @@ static int i965_irq_postinstall(struct drm_device *dev)
 	if (IS_G4X(dev_priv))
 		enable_mask |= I915_BSD_USER_INTERRUPT;
 
-	GEN3_IRQ_INIT(, dev_priv->irq_mask, enable_mask);
+	GEN3_IRQ_INIT(uncore, GEN2_, dev_priv->irq_mask, enable_mask);
 
 	/* Interrupt setup is already guaranteed to be single-threaded, this is
 	 * just to make the assert_spin_locked check happy. */
@@ -4452,7 +4641,7 @@ static irqreturn_t i965_irq_handler(int irq, void *arg)
 		u32 hotplug_status = 0;
 		u32 iir;
 
-		iir = I915_READ(IIR);
+		iir = I915_READ(GEN2_IIR);
 		if (iir == 0)
 			break;
 
@@ -4468,13 +4657,13 @@ static irqreturn_t i965_irq_handler(int irq, void *arg)
 		if (iir & I915_MASTER_ERROR_INTERRUPT)
 			i9xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
 
-		I915_WRITE(IIR, iir);
+		I915_WRITE(GEN2_IIR, iir);
 
 		if (iir & I915_USER_INTERRUPT)
-			intel_engine_breadcrumbs_irq(dev_priv->engine[RCS]);
+			intel_engine_breadcrumbs_irq(dev_priv->engine[RCS0]);
 
 		if (iir & I915_BSD_USER_INTERRUPT)
-			intel_engine_breadcrumbs_irq(dev_priv->engine[VCS]);
+			intel_engine_breadcrumbs_irq(dev_priv->engine[VCS0]);
 
 		if (iir & I915_MASTER_ERROR_INTERRUPT)
 			i9xx_error_irq_handler(dev_priv, eir, eir_stuck);
@@ -4503,6 +4692,9 @@ void intel_irq_init(struct drm_i915_private *dev_priv)
 	struct intel_rps *rps = &dev_priv->gt_pm.rps;
 	int i;
 
+	if (IS_I945GM(dev_priv))
+		i945gm_vblank_work_init(dev_priv);
+
 	intel_hpd_init_work(dev_priv);
 
 	INIT_WORK(&rps->work, gen6_pm_rps_work);
@@ -4523,6 +4715,10 @@ void intel_irq_init(struct drm_i915_private *dev_priv)
 					   GEN6_PM_RP_DOWN_THRESHOLD |
 					   GEN6_PM_RP_DOWN_TIMEOUT);
 
+	/* We share the register with other engine */
+	if (INTEL_GEN(dev_priv) > 9)
+		GEM_WARN_ON(dev_priv->pm_rps_events & 0xffff0000);
+
 	rps->pm_intrmsk_mbz = 0;
 
 	/*
@@ -4542,13 +4738,7 @@ void intel_irq_init(struct drm_i915_private *dev_priv)
 	else if (INTEL_GEN(dev_priv) >= 3)
 		dev->driver->get_vblank_counter = i915_get_vblank_counter;
 
-	/*
-	 * Opt out of the vblank disable timer on everything except gen2.
-	 * Gen2 doesn't have a hardware frame counter and so depends on
-	 * vblank interrupts to produce sane vblank seuquence numbers.
-	 */
-	if (!IS_GEN(dev_priv, 2))
-		dev->vblank_disable_immediate = true;
+	dev->vblank_disable_immediate = true;
 
 	/* Most platforms treat the display irq block as an always-on
 	 * power domain. vlv/chv can disable it at runtime and need
@@ -4605,8 +4795,7 @@ void intel_irq_init(struct drm_i915_private *dev_priv)
 		dev->driver->disable_vblank = gen8_disable_vblank;
 		if (IS_GEN9_LP(dev_priv))
 			dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup;
-		else if (HAS_PCH_SPT(dev_priv) || HAS_PCH_KBP(dev_priv) ||
-			 HAS_PCH_CNP(dev_priv))
+		else if (INTEL_PCH_TYPE(dev_priv) >= PCH_SPT)
 			dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup;
 		else
 			dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
@@ -4626,6 +4815,13 @@ void intel_irq_init(struct drm_i915_private *dev_priv)
 			dev->driver->irq_uninstall = i8xx_irq_reset;
 			dev->driver->enable_vblank = i8xx_enable_vblank;
 			dev->driver->disable_vblank = i8xx_disable_vblank;
+		} else if (IS_I945GM(dev_priv)) {
+			dev->driver->irq_preinstall = i915_irq_reset;
+			dev->driver->irq_postinstall = i915_irq_postinstall;
+			dev->driver->irq_uninstall = i915_irq_reset;
+			dev->driver->irq_handler = i915_irq_handler;
+			dev->driver->enable_vblank = i945gm_enable_vblank;
+			dev->driver->disable_vblank = i945gm_disable_vblank;
 		} else if (IS_GEN(dev_priv, 3)) {
 			dev->driver->irq_preinstall = i915_irq_reset;
 			dev->driver->irq_postinstall = i915_irq_postinstall;
@@ -4656,6 +4852,9 @@ void intel_irq_fini(struct drm_i915_private *i915)
 {
 	int i;
 
+	if (IS_I945GM(i915))
+		i945gm_vblank_work_fini(i915);
+
 	for (i = 0; i < MAX_L3_SLICES; ++i)
 		kfree(i915->l3_parity.remap_info[i]);
 }
diff --git a/drivers/gpu/drm/i915/i915_pci.c b/drivers/gpu/drm/i915/i915_pci.c
index 66f82f3f050f..f893c2cbce15 100644
--- a/drivers/gpu/drm/i915/i915_pci.c
+++ b/drivers/gpu/drm/i915/i915_pci.c
@@ -28,14 +28,45 @@
 
 #include <drm/drm_drv.h>
 
-#include "i915_active.h"
 #include "i915_drv.h"
+#include "i915_globals.h"
 #include "i915_selftest.h"
+#include "intel_fbdev.h"
 
-#define PLATFORM(x) .platform = (x), .platform_mask = BIT(x)
+#define PLATFORM(x) .platform = (x)
 #define GEN(x) .gen = (x), .gen_mask = BIT((x) - 1)
 
-#define GEN_DEFAULT_PIPEOFFSETS \
+#define I845_PIPE_OFFSETS \
+	.pipe_offsets = { \
+		[TRANSCODER_A] = PIPE_A_OFFSET,	\
+	}, \
+	.trans_offsets = { \
+		[TRANSCODER_A] = TRANSCODER_A_OFFSET, \
+	}
+
+#define I9XX_PIPE_OFFSETS \
+	.pipe_offsets = { \
+		[TRANSCODER_A] = PIPE_A_OFFSET,	\
+		[TRANSCODER_B] = PIPE_B_OFFSET, \
+	}, \
+	.trans_offsets = { \
+		[TRANSCODER_A] = TRANSCODER_A_OFFSET, \
+		[TRANSCODER_B] = TRANSCODER_B_OFFSET, \
+	}
+
+#define IVB_PIPE_OFFSETS \
+	.pipe_offsets = { \
+		[TRANSCODER_A] = PIPE_A_OFFSET,	\
+		[TRANSCODER_B] = PIPE_B_OFFSET, \
+		[TRANSCODER_C] = PIPE_C_OFFSET, \
+	}, \
+	.trans_offsets = { \
+		[TRANSCODER_A] = TRANSCODER_A_OFFSET, \
+		[TRANSCODER_B] = TRANSCODER_B_OFFSET, \
+		[TRANSCODER_C] = TRANSCODER_C_OFFSET, \
+	}
+
+#define HSW_PIPE_OFFSETS \
 	.pipe_offsets = { \
 		[TRANSCODER_A] = PIPE_A_OFFSET,	\
 		[TRANSCODER_B] = PIPE_B_OFFSET, \
@@ -49,7 +80,7 @@
 		[TRANSCODER_EDP] = TRANSCODER_EDP_OFFSET, \
 	}
 
-#define GEN_CHV_PIPEOFFSETS \
+#define CHV_PIPE_OFFSETS \
 	.pipe_offsets = { \
 		[TRANSCODER_A] = PIPE_A_OFFSET, \
 		[TRANSCODER_B] = PIPE_B_OFFSET, \
@@ -61,21 +92,48 @@
 		[TRANSCODER_C] = CHV_TRANSCODER_C_OFFSET, \
 	}
 
-#define CURSOR_OFFSETS \
-	.cursor_offsets = { CURSOR_A_OFFSET, CURSOR_B_OFFSET, CHV_CURSOR_C_OFFSET }
+#define I845_CURSOR_OFFSETS \
+	.cursor_offsets = { \
+		[PIPE_A] = CURSOR_A_OFFSET, \
+	}
+
+#define I9XX_CURSOR_OFFSETS \
+	.cursor_offsets = { \
+		[PIPE_A] = CURSOR_A_OFFSET, \
+		[PIPE_B] = CURSOR_B_OFFSET, \
+	}
+
+#define CHV_CURSOR_OFFSETS \
+	.cursor_offsets = { \
+		[PIPE_A] = CURSOR_A_OFFSET, \
+		[PIPE_B] = CURSOR_B_OFFSET, \
+		[PIPE_C] = CHV_CURSOR_C_OFFSET, \
+	}
 
 #define IVB_CURSOR_OFFSETS \
-	.cursor_offsets = { CURSOR_A_OFFSET, IVB_CURSOR_B_OFFSET, IVB_CURSOR_C_OFFSET }
+	.cursor_offsets = { \
+		[PIPE_A] = CURSOR_A_OFFSET, \
+		[PIPE_B] = IVB_CURSOR_B_OFFSET, \
+		[PIPE_C] = IVB_CURSOR_C_OFFSET, \
+	}
 
-#define BDW_COLORS \
-	.color = { .degamma_lut_size = 512, .gamma_lut_size = 512 }
+#define I9XX_COLORS \
+	.color = { .gamma_lut_size = 256 }
+#define I965_COLORS \
+	.color = { .gamma_lut_size = 129, \
+		   .gamma_lut_tests = DRM_COLOR_LUT_NON_DECREASING, \
+	}
+#define ILK_COLORS \
+	.color = { .gamma_lut_size = 1024 }
+#define IVB_COLORS \
+	.color = { .degamma_lut_size = 1024, .gamma_lut_size = 1024 }
 #define CHV_COLORS \
 	.color = { .degamma_lut_size = 65, .gamma_lut_size = 257, \
 		   .degamma_lut_tests = DRM_COLOR_LUT_NON_DECREASING, \
 		   .gamma_lut_tests = DRM_COLOR_LUT_NON_DECREASING, \
 	}
 #define GLK_COLORS \
-	.color = { .degamma_lut_size = 0, .gamma_lut_size = 1024, \
+	.color = { .degamma_lut_size = 33, .gamma_lut_size = 1024, \
 		   .degamma_lut_tests = DRM_COLOR_LUT_NON_DECREASING | \
 					DRM_COLOR_LUT_EQUAL_CHANNELS, \
 	}
@@ -85,7 +143,26 @@
 #define GEN_DEFAULT_PAGE_SIZES \
 	.page_sizes = I915_GTT_PAGE_SIZE_4K
 
-#define GEN2_FEATURES \
+#define I830_FEATURES \
+	GEN(2), \
+	.is_mobile = 1, \
+	.num_pipes = 2, \
+	.display.has_overlay = 1, \
+	.display.cursor_needs_physical = 1, \
+	.display.overlay_needs_physical = 1, \
+	.display.has_gmch = 1, \
+	.gpu_reset_clobbers_display = true, \
+	.hws_needs_physical = 1, \
+	.unfenced_needs_alignment = 1, \
+	.engine_mask = BIT(RCS0), \
+	.has_snoop = true, \
+	.has_coherent_ggtt = false, \
+	I9XX_PIPE_OFFSETS, \
+	I9XX_CURSOR_OFFSETS, \
+	I9XX_COLORS, \
+	GEN_DEFAULT_PAGE_SIZES
+
+#define I845_FEATURES \
 	GEN(2), \
 	.num_pipes = 1, \
 	.display.has_overlay = 1, \
@@ -94,37 +171,32 @@
 	.gpu_reset_clobbers_display = true, \
 	.hws_needs_physical = 1, \
 	.unfenced_needs_alignment = 1, \
-	.ring_mask = RENDER_RING, \
+	.engine_mask = BIT(RCS0), \
 	.has_snoop = true, \
 	.has_coherent_ggtt = false, \
-	GEN_DEFAULT_PIPEOFFSETS, \
-	GEN_DEFAULT_PAGE_SIZES, \
-	CURSOR_OFFSETS
+	I845_PIPE_OFFSETS, \
+	I845_CURSOR_OFFSETS, \
+	I9XX_COLORS, \
+	GEN_DEFAULT_PAGE_SIZES
 
 static const struct intel_device_info intel_i830_info = {
-	GEN2_FEATURES,
+	I830_FEATURES,
 	PLATFORM(INTEL_I830),
-	.is_mobile = 1,
-	.display.cursor_needs_physical = 1,
-	.num_pipes = 2, /* legal, last one wins */
 };
 
 static const struct intel_device_info intel_i845g_info = {
-	GEN2_FEATURES,
+	I845_FEATURES,
 	PLATFORM(INTEL_I845G),
 };
 
 static const struct intel_device_info intel_i85x_info = {
-	GEN2_FEATURES,
+	I830_FEATURES,
 	PLATFORM(INTEL_I85X),
-	.is_mobile = 1,
-	.num_pipes = 2, /* legal, last one wins */
-	.display.cursor_needs_physical = 1,
 	.display.has_fbc = 1,
 };
 
 static const struct intel_device_info intel_i865g_info = {
-	GEN2_FEATURES,
+	I845_FEATURES,
 	PLATFORM(INTEL_I865G),
 };
 
@@ -133,12 +205,13 @@ static const struct intel_device_info intel_i865g_info = {
 	.num_pipes = 2, \
 	.display.has_gmch = 1, \
 	.gpu_reset_clobbers_display = true, \
-	.ring_mask = RENDER_RING, \
+	.engine_mask = BIT(RCS0), \
 	.has_snoop = true, \
 	.has_coherent_ggtt = true, \
-	GEN_DEFAULT_PIPEOFFSETS, \
-	GEN_DEFAULT_PAGE_SIZES, \
-	CURSOR_OFFSETS
+	I9XX_PIPE_OFFSETS, \
+	I9XX_CURSOR_OFFSETS, \
+	I9XX_COLORS, \
+	GEN_DEFAULT_PAGE_SIZES
 
 static const struct intel_device_info intel_i915g_info = {
 	GEN3_FEATURES,
@@ -196,7 +269,14 @@ static const struct intel_device_info intel_g33_info = {
 	.display.has_overlay = 1,
 };
 
-static const struct intel_device_info intel_pineview_info = {
+static const struct intel_device_info intel_pineview_g_info = {
+	GEN3_FEATURES,
+	PLATFORM(INTEL_PINEVIEW),
+	.display.has_hotplug = 1,
+	.display.has_overlay = 1,
+};
+
+static const struct intel_device_info intel_pineview_m_info = {
 	GEN3_FEATURES,
 	PLATFORM(INTEL_PINEVIEW),
 	.is_mobile = 1,
@@ -210,12 +290,13 @@ static const struct intel_device_info intel_pineview_info = {
 	.display.has_hotplug = 1, \
 	.display.has_gmch = 1, \
 	.gpu_reset_clobbers_display = true, \
-	.ring_mask = RENDER_RING, \
+	.engine_mask = BIT(RCS0), \
 	.has_snoop = true, \
 	.has_coherent_ggtt = true, \
-	GEN_DEFAULT_PIPEOFFSETS, \
-	GEN_DEFAULT_PAGE_SIZES, \
-	CURSOR_OFFSETS
+	I9XX_PIPE_OFFSETS, \
+	I9XX_CURSOR_OFFSETS, \
+	I965_COLORS, \
+	GEN_DEFAULT_PAGE_SIZES
 
 static const struct intel_device_info intel_i965g_info = {
 	GEN4_FEATURES,
@@ -239,7 +320,7 @@ static const struct intel_device_info intel_i965gm_info = {
 static const struct intel_device_info intel_g45_info = {
 	GEN4_FEATURES,
 	PLATFORM(INTEL_G45),
-	.ring_mask = RENDER_RING | BSD_RING,
+	.engine_mask = BIT(RCS0) | BIT(VCS0),
 	.gpu_reset_clobbers_display = false,
 };
 
@@ -249,7 +330,7 @@ static const struct intel_device_info intel_gm45_info = {
 	.is_mobile = 1,
 	.display.has_fbc = 1,
 	.display.supports_tv = 1,
-	.ring_mask = RENDER_RING | BSD_RING,
+	.engine_mask = BIT(RCS0) | BIT(VCS0),
 	.gpu_reset_clobbers_display = false,
 };
 
@@ -257,14 +338,15 @@ static const struct intel_device_info intel_gm45_info = {
 	GEN(5), \
 	.num_pipes = 2, \
 	.display.has_hotplug = 1, \
-	.ring_mask = RENDER_RING | BSD_RING, \
+	.engine_mask = BIT(RCS0) | BIT(VCS0), \
 	.has_snoop = true, \
 	.has_coherent_ggtt = true, \
 	/* ilk does support rc6, but we do not implement [power] contexts */ \
 	.has_rc6 = 0, \
-	GEN_DEFAULT_PIPEOFFSETS, \
-	GEN_DEFAULT_PAGE_SIZES, \
-	CURSOR_OFFSETS
+	I9XX_PIPE_OFFSETS, \
+	I9XX_CURSOR_OFFSETS, \
+	ILK_COLORS, \
+	GEN_DEFAULT_PAGE_SIZES
 
 static const struct intel_device_info intel_ironlake_d_info = {
 	GEN5_FEATURES,
@@ -283,15 +365,17 @@ static const struct intel_device_info intel_ironlake_m_info = {
 	.num_pipes = 2, \
 	.display.has_hotplug = 1, \
 	.display.has_fbc = 1, \
-	.ring_mask = RENDER_RING | BSD_RING | BLT_RING, \
+	.engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0), \
 	.has_coherent_ggtt = true, \
 	.has_llc = 1, \
 	.has_rc6 = 1, \
 	.has_rc6p = 1, \
-	.ppgtt = INTEL_PPGTT_ALIASING, \
-	GEN_DEFAULT_PIPEOFFSETS, \
-	GEN_DEFAULT_PAGE_SIZES, \
-	CURSOR_OFFSETS
+	.ppgtt_type = INTEL_PPGTT_ALIASING, \
+	.ppgtt_size = 31, \
+	I9XX_PIPE_OFFSETS, \
+	I9XX_CURSOR_OFFSETS, \
+	ILK_COLORS, \
+	GEN_DEFAULT_PAGE_SIZES
 
 #define SNB_D_PLATFORM \
 	GEN6_FEATURES, \
@@ -328,15 +412,17 @@ static const struct intel_device_info intel_sandybridge_m_gt2_info = {
 	.num_pipes = 3, \
 	.display.has_hotplug = 1, \
 	.display.has_fbc = 1, \
-	.ring_mask = RENDER_RING | BSD_RING | BLT_RING, \
+	.engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0), \
 	.has_coherent_ggtt = true, \
 	.has_llc = 1, \
 	.has_rc6 = 1, \
 	.has_rc6p = 1, \
-	.ppgtt = INTEL_PPGTT_FULL, \
-	GEN_DEFAULT_PIPEOFFSETS, \
-	GEN_DEFAULT_PAGE_SIZES, \
-	IVB_CURSOR_OFFSETS
+	.ppgtt_type = INTEL_PPGTT_FULL, \
+	.ppgtt_size = 31, \
+	IVB_PIPE_OFFSETS, \
+	IVB_CURSOR_OFFSETS, \
+	IVB_COLORS, \
+	GEN_DEFAULT_PAGE_SIZES
 
 #define IVB_D_PLATFORM \
 	GEN7_FEATURES, \
@@ -386,24 +472,27 @@ static const struct intel_device_info intel_valleyview_info = {
 	.has_rc6 = 1,
 	.display.has_gmch = 1,
 	.display.has_hotplug = 1,
-	.ppgtt = INTEL_PPGTT_FULL,
+	.ppgtt_type = INTEL_PPGTT_FULL,
+	.ppgtt_size = 31,
 	.has_snoop = true,
 	.has_coherent_ggtt = false,
-	.ring_mask = RENDER_RING | BSD_RING | BLT_RING,
+	.engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0),
 	.display_mmio_offset = VLV_DISPLAY_BASE,
+	I9XX_PIPE_OFFSETS,
+	I9XX_CURSOR_OFFSETS,
+	I965_COLORS,
 	GEN_DEFAULT_PAGE_SIZES,
-	GEN_DEFAULT_PIPEOFFSETS,
-	CURSOR_OFFSETS
 };
 
 #define G75_FEATURES  \
 	GEN7_FEATURES, \
-	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING, \
+	.engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0), \
 	.display.has_ddi = 1, \
 	.has_fpga_dbg = 1, \
 	.display.has_psr = 1, \
 	.display.has_dp_mst = 1, \
 	.has_rc6p = 0 /* RC6p removed-by HSW */, \
+	HSW_PIPE_OFFSETS, \
 	.has_runtime_pm = 1
 
 #define HSW_PLATFORM \
@@ -429,11 +518,11 @@ static const struct intel_device_info intel_haswell_gt3_info = {
 #define GEN8_FEATURES \
 	G75_FEATURES, \
 	GEN(8), \
-	BDW_COLORS, \
 	.page_sizes = I915_GTT_PAGE_SIZE_4K | \
 		      I915_GTT_PAGE_SIZE_2M, \
 	.has_logical_ring_contexts = 1, \
-	.ppgtt = INTEL_PPGTT_FULL_4LVL, \
+	.ppgtt_type = INTEL_PPGTT_FULL, \
+	.ppgtt_size = 48, \
 	.has_64bit_reloc = 1, \
 	.has_reset_engine = 1
 
@@ -462,7 +551,8 @@ static const struct intel_device_info intel_broadwell_rsvd_info = {
 static const struct intel_device_info intel_broadwell_gt3_info = {
 	BDW_PLATFORM,
 	.gt = 3,
-	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING,
+	.engine_mask =
+		BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS1),
 };
 
 static const struct intel_device_info intel_cherryview_info = {
@@ -471,21 +561,22 @@ static const struct intel_device_info intel_cherryview_info = {
 	.num_pipes = 3,
 	.display.has_hotplug = 1,
 	.is_lp = 1,
-	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
+	.engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0),
 	.has_64bit_reloc = 1,
 	.has_runtime_pm = 1,
 	.has_rc6 = 1,
 	.has_logical_ring_contexts = 1,
 	.display.has_gmch = 1,
-	.ppgtt = INTEL_PPGTT_FULL,
+	.ppgtt_type = INTEL_PPGTT_FULL,
+	.ppgtt_size = 32,
 	.has_reset_engine = 1,
 	.has_snoop = true,
 	.has_coherent_ggtt = false,
 	.display_mmio_offset = VLV_DISPLAY_BASE,
-	GEN_DEFAULT_PAGE_SIZES,
-	GEN_CHV_PIPEOFFSETS,
-	CURSOR_OFFSETS,
+	CHV_PIPE_OFFSETS,
+	CHV_CURSOR_OFFSETS,
 	CHV_COLORS,
+	GEN_DEFAULT_PAGE_SIZES,
 };
 
 #define GEN9_DEFAULT_PAGE_SIZES \
@@ -521,7 +612,8 @@ static const struct intel_device_info intel_skylake_gt2_info = {
 
 #define SKL_GT3_PLUS_PLATFORM \
 	SKL_PLATFORM, \
-	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING
+	.engine_mask = \
+		BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS1)
 
 
 static const struct intel_device_info intel_skylake_gt3_info = {
@@ -538,7 +630,7 @@ static const struct intel_device_info intel_skylake_gt4_info = {
 	GEN(9), \
 	.is_lp = 1, \
 	.display.has_hotplug = 1, \
-	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING, \
+	.engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0), \
 	.num_pipes = 3, \
 	.has_64bit_reloc = 1, \
 	.display.has_ddi = 1, \
@@ -552,15 +644,16 @@ static const struct intel_device_info intel_skylake_gt4_info = {
 	.has_logical_ring_contexts = 1, \
 	.has_logical_ring_preemption = 1, \
 	.has_guc = 1, \
-	.ppgtt = INTEL_PPGTT_FULL_4LVL, \
+	.ppgtt_type = INTEL_PPGTT_FULL, \
+	.ppgtt_size = 48, \
 	.has_reset_engine = 1, \
 	.has_snoop = true, \
 	.has_coherent_ggtt = false, \
 	.display.has_ipc = 1, \
-	GEN9_DEFAULT_PAGE_SIZES, \
-	GEN_DEFAULT_PIPEOFFSETS, \
+	HSW_PIPE_OFFSETS, \
 	IVB_CURSOR_OFFSETS, \
-	BDW_COLORS
+	IVB_COLORS, \
+	GEN9_DEFAULT_PAGE_SIZES
 
 static const struct intel_device_info intel_broxton_info = {
 	GEN9_LP_FEATURES,
@@ -592,7 +685,8 @@ static const struct intel_device_info intel_kabylake_gt2_info = {
 static const struct intel_device_info intel_kabylake_gt3_info = {
 	KBL_PLATFORM,
 	.gt = 3,
-	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING,
+	.engine_mask =
+		BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS1),
 };
 
 #define CFL_PLATFORM \
@@ -612,7 +706,8 @@ static const struct intel_device_info intel_coffeelake_gt2_info = {
 static const struct intel_device_info intel_coffeelake_gt3_info = {
 	CFL_PLATFORM,
 	.gt = 3,
-	.ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING,
+	.engine_mask =
+		BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS1),
 };
 
 #define GEN10_FEATURES \
@@ -648,13 +743,22 @@ static const struct intel_device_info intel_cannonlake_info = {
 	}, \
 	GEN(11), \
 	.ddb_size = 2048, \
-	.has_logical_ring_elsq = 1
+	.has_logical_ring_elsq = 1, \
+	.color = { .degamma_lut_size = 33, .gamma_lut_size = 1024 }
 
 static const struct intel_device_info intel_icelake_11_info = {
 	GEN11_FEATURES,
 	PLATFORM(INTEL_ICELAKE),
+	.engine_mask =
+		BIT(RCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS0) | BIT(VCS2),
+};
+
+static const struct intel_device_info intel_elkhartlake_info = {
+	GEN11_FEATURES,
+	PLATFORM(INTEL_ELKHARTLAKE),
 	.is_alpha_support = 1,
-	.ring_mask = RENDER_RING | BLT_RING | VEBOX_RING | BSD_RING | BSD3_RING,
+	.engine_mask = BIT(RCS0) | BIT(BCS0) | BIT(VCS0),
+	.ppgtt_size = 36,
 };
 
 #undef GEN
@@ -680,7 +784,8 @@ static const struct pci_device_id pciidlist[] = {
 	INTEL_I965GM_IDS(&intel_i965gm_info),
 	INTEL_GM45_IDS(&intel_gm45_info),
 	INTEL_G45_IDS(&intel_g45_info),
-	INTEL_PINEVIEW_IDS(&intel_pineview_info),
+	INTEL_PINEVIEW_G_IDS(&intel_pineview_g_info),
+	INTEL_PINEVIEW_M_IDS(&intel_pineview_m_info),
 	INTEL_IRONLAKE_D_IDS(&intel_ironlake_d_info),
 	INTEL_IRONLAKE_M_IDS(&intel_ironlake_m_info),
 	INTEL_SNB_D_GT1_IDS(&intel_sandybridge_d_gt1_info),
@@ -722,8 +827,11 @@ static const struct pci_device_id pciidlist[] = {
 	INTEL_WHL_U_GT2_IDS(&intel_coffeelake_gt2_info),
 	INTEL_AML_CFL_GT2_IDS(&intel_coffeelake_gt2_info),
 	INTEL_WHL_U_GT3_IDS(&intel_coffeelake_gt3_info),
+	INTEL_CML_GT1_IDS(&intel_coffeelake_gt1_info),
+	INTEL_CML_GT2_IDS(&intel_coffeelake_gt2_info),
 	INTEL_CNL_IDS(&intel_cannonlake_info),
 	INTEL_ICL_11_IDS(&intel_icelake_11_info),
+	INTEL_EHL_IDS(&intel_elkhartlake_info),
 	{0, 0, 0}
 };
 MODULE_DEVICE_TABLE(pci, pciidlist);
@@ -801,7 +909,9 @@ static int __init i915_init(void)
 	bool use_kms = true;
 	int err;
 
-	i915_global_active_init();
+	err = i915_globals_init();
+	if (err)
+		return err;
 
 	err = i915_mock_selftests();
 	if (err)
@@ -834,7 +944,7 @@ static void __exit i915_exit(void)
 		return;
 
 	pci_unregister_driver(&i915_pci_driver);
-	i915_global_active_exit();
+	i915_globals_exit();
 }
 
 module_init(i915_init);
diff --git a/drivers/gpu/drm/i915/i915_perf.c b/drivers/gpu/drm/i915/i915_perf.c
index 9ebf99f3d8d3..39a4804091d7 100644
--- a/drivers/gpu/drm/i915/i915_perf.c
+++ b/drivers/gpu/drm/i915/i915_perf.c
@@ -1202,7 +1202,7 @@ static int i915_oa_read(struct i915_perf_stream *stream,
 static struct intel_context *oa_pin_context(struct drm_i915_private *i915,
 					    struct i915_gem_context *ctx)
 {
-	struct intel_engine_cs *engine = i915->engine[RCS];
+	struct intel_engine_cs *engine = i915->engine[RCS0];
 	struct intel_context *ce;
 	int ret;
 
@@ -1364,7 +1364,7 @@ static void i915_oa_stream_destroy(struct i915_perf_stream *stream)
 
 	free_oa_buffer(dev_priv);
 
-	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
 	intel_runtime_pm_put(dev_priv, stream->wakeref);
 
 	if (stream->ctx)
@@ -1509,9 +1509,7 @@ static int alloc_oa_buffer(struct drm_i915_private *dev_priv)
 		goto unlock;
 	}
 
-	ret = i915_gem_object_set_cache_level(bo, I915_CACHE_LLC);
-	if (ret)
-		goto err_unref;
+	i915_gem_object_set_cache_coherency(bo, I915_CACHE_LLC);
 
 	/* PreHSW required 512K alignment, HSW requires 16M */
 	vma = i915_gem_object_ggtt_pin(bo, NULL, 0, SZ_16M, 0);
@@ -1629,13 +1627,14 @@ static void hsw_disable_metric_set(struct drm_i915_private *dev_priv)
  * It's fine to put out-of-date values into these per-context registers
  * in the case that the OA unit has been disabled.
  */
-static void gen8_update_reg_state_unlocked(struct i915_gem_context *ctx,
-					   u32 *reg_state,
-					   const struct i915_oa_config *oa_config)
+static void
+gen8_update_reg_state_unlocked(struct intel_context *ce,
+			       u32 *reg_state,
+			       const struct i915_oa_config *oa_config)
 {
-	struct drm_i915_private *dev_priv = ctx->i915;
-	u32 ctx_oactxctrl = dev_priv->perf.oa.ctx_oactxctrl_offset;
-	u32 ctx_flexeu0 = dev_priv->perf.oa.ctx_flexeu0_offset;
+	struct drm_i915_private *i915 = ce->gem_context->i915;
+	u32 ctx_oactxctrl = i915->perf.oa.ctx_oactxctrl_offset;
+	u32 ctx_flexeu0 = i915->perf.oa.ctx_flexeu0_offset;
 	/* The MMIO offsets for Flex EU registers aren't contiguous */
 	i915_reg_t flex_regs[] = {
 		EU_PERF_CNTL0,
@@ -1649,8 +1648,8 @@ static void gen8_update_reg_state_unlocked(struct i915_gem_context *ctx,
 	int i;
 
 	CTX_REG(reg_state, ctx_oactxctrl, GEN8_OACTXCONTROL,
-		(dev_priv->perf.oa.period_exponent << GEN8_OA_TIMER_PERIOD_SHIFT) |
-		(dev_priv->perf.oa.periodic ? GEN8_OA_TIMER_ENABLE : 0) |
+		(i915->perf.oa.period_exponent << GEN8_OA_TIMER_PERIOD_SHIFT) |
+		(i915->perf.oa.periodic ? GEN8_OA_TIMER_ENABLE : 0) |
 		GEN8_OA_COUNTER_RESUME);
 
 	for (i = 0; i < ARRAY_SIZE(flex_regs); i++) {
@@ -1678,10 +1677,9 @@ static void gen8_update_reg_state_unlocked(struct i915_gem_context *ctx,
 		CTX_REG(reg_state, state_offset, flex_regs[i], value);
 	}
 
-	CTX_REG(reg_state, CTX_R_PWR_CLK_STATE, GEN8_R_PWR_CLK_STATE,
-		gen8_make_rpcs(dev_priv,
-			       &to_intel_context(ctx,
-						 dev_priv->engine[RCS])->sseu));
+	CTX_REG(reg_state,
+		CTX_R_PWR_CLK_STATE, GEN8_R_PWR_CLK_STATE,
+		gen8_make_rpcs(i915, &ce->sseu));
 }
 
 /*
@@ -1711,7 +1709,7 @@ static void gen8_update_reg_state_unlocked(struct i915_gem_context *ctx,
 static int gen8_configure_all_contexts(struct drm_i915_private *dev_priv,
 				       const struct i915_oa_config *oa_config)
 {
-	struct intel_engine_cs *engine = dev_priv->engine[RCS];
+	struct intel_engine_cs *engine = dev_priv->engine[RCS0];
 	unsigned int map_type = i915_coherent_map_type(dev_priv);
 	struct i915_gem_context *ctx;
 	struct i915_request *rq;
@@ -1740,11 +1738,11 @@ static int gen8_configure_all_contexts(struct drm_i915_private *dev_priv,
 
 	/* Update all contexts now that we've stalled the submission. */
 	list_for_each_entry(ctx, &dev_priv->contexts.list, link) {
-		struct intel_context *ce = to_intel_context(ctx, engine);
+		struct intel_context *ce = intel_context_lookup(ctx, engine);
 		u32 *regs;
 
 		/* OA settings will be set upon first use */
-		if (!ce->state)
+		if (!ce || !ce->state)
 			continue;
 
 		regs = i915_gem_object_pin_map(ce->state->obj, map_type);
@@ -1754,7 +1752,7 @@ static int gen8_configure_all_contexts(struct drm_i915_private *dev_priv,
 		ce->state->obj->mm.dirty = true;
 		regs += LRC_STATE_PN * PAGE_SIZE / sizeof(*regs);
 
-		gen8_update_reg_state_unlocked(ctx, regs, oa_config);
+		gen8_update_reg_state_unlocked(ce, regs, oa_config);
 
 		i915_gem_object_unpin_map(ce->state->obj);
 	}
@@ -1922,10 +1920,10 @@ static void i915_oa_stream_enable(struct i915_perf_stream *stream)
 
 static void gen7_oa_disable(struct i915_perf_stream *stream)
 {
-	struct drm_i915_private *dev_priv = stream->dev_priv;
+	struct intel_uncore *uncore = &stream->dev_priv->uncore;
 
-	I915_WRITE(GEN7_OACONTROL, 0);
-	if (intel_wait_for_register(dev_priv,
+	intel_uncore_write(uncore, GEN7_OACONTROL, 0);
+	if (intel_wait_for_register(uncore,
 				    GEN7_OACONTROL, GEN7_OACONTROL_ENABLE, 0,
 				    50))
 		DRM_ERROR("wait for OA to be disabled timed out\n");
@@ -1933,10 +1931,10 @@ static void gen7_oa_disable(struct i915_perf_stream *stream)
 
 static void gen8_oa_disable(struct i915_perf_stream *stream)
 {
-	struct drm_i915_private *dev_priv = stream->dev_priv;
+	struct intel_uncore *uncore = &stream->dev_priv->uncore;
 
-	I915_WRITE(GEN8_OACONTROL, 0);
-	if (intel_wait_for_register(dev_priv,
+	intel_uncore_write(uncore, GEN8_OACONTROL, 0);
+	if (intel_wait_for_register(uncore,
 				    GEN8_OACONTROL, GEN8_OA_COUNTER_ENABLE, 0,
 				    50))
 		DRM_ERROR("wait for OA to be disabled timed out\n");
@@ -2093,7 +2091,7 @@ static int i915_oa_stream_init(struct i915_perf_stream *stream,
 	 *   references will effectively disable RC6.
 	 */
 	stream->wakeref = intel_runtime_pm_get(dev_priv);
-	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
 
 	ret = alloc_oa_buffer(dev_priv);
 	if (ret)
@@ -2127,7 +2125,7 @@ err_lock:
 err_oa_buf_alloc:
 	put_oa_config(dev_priv, stream->oa_config);
 
-	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
 	intel_runtime_pm_put(dev_priv, stream->wakeref);
 
 err_config:
@@ -2138,17 +2136,17 @@ err_config:
 }
 
 void i915_oa_init_reg_state(struct intel_engine_cs *engine,
-			    struct i915_gem_context *ctx,
-			    u32 *reg_state)
+			    struct intel_context *ce,
+			    u32 *regs)
 {
 	struct i915_perf_stream *stream;
 
-	if (engine->id != RCS)
+	if (engine->class != RENDER_CLASS)
 		return;
 
 	stream = engine->i915->perf.oa.exclusive_stream;
 	if (stream)
-		gen8_update_reg_state_unlocked(ctx, reg_state, stream->oa_config);
+		gen8_update_reg_state_unlocked(ce, regs, stream->oa_config);
 }
 
 /**
@@ -2881,12 +2879,24 @@ void i915_perf_register(struct drm_i915_private *dev_priv)
 
 	sysfs_attr_init(&dev_priv->perf.oa.test_config.sysfs_metric_id.attr);
 
-	if (IS_HASWELL(dev_priv)) {
-		i915_perf_load_test_config_hsw(dev_priv);
-	} else if (IS_BROADWELL(dev_priv)) {
-		i915_perf_load_test_config_bdw(dev_priv);
-	} else if (IS_CHERRYVIEW(dev_priv)) {
-		i915_perf_load_test_config_chv(dev_priv);
+	if (INTEL_GEN(dev_priv) >= 11) {
+		i915_perf_load_test_config_icl(dev_priv);
+	} else if (IS_CANNONLAKE(dev_priv)) {
+		i915_perf_load_test_config_cnl(dev_priv);
+	} else if (IS_COFFEELAKE(dev_priv)) {
+		if (IS_CFL_GT2(dev_priv))
+			i915_perf_load_test_config_cflgt2(dev_priv);
+		if (IS_CFL_GT3(dev_priv))
+			i915_perf_load_test_config_cflgt3(dev_priv);
+	} else if (IS_GEMINILAKE(dev_priv)) {
+		i915_perf_load_test_config_glk(dev_priv);
+	} else if (IS_KABYLAKE(dev_priv)) {
+		if (IS_KBL_GT2(dev_priv))
+			i915_perf_load_test_config_kblgt2(dev_priv);
+		else if (IS_KBL_GT3(dev_priv))
+			i915_perf_load_test_config_kblgt3(dev_priv);
+	} else if (IS_BROXTON(dev_priv)) {
+		i915_perf_load_test_config_bxt(dev_priv);
 	} else if (IS_SKYLAKE(dev_priv)) {
 		if (IS_SKL_GT2(dev_priv))
 			i915_perf_load_test_config_sklgt2(dev_priv);
@@ -2894,25 +2904,13 @@ void i915_perf_register(struct drm_i915_private *dev_priv)
 			i915_perf_load_test_config_sklgt3(dev_priv);
 		else if (IS_SKL_GT4(dev_priv))
 			i915_perf_load_test_config_sklgt4(dev_priv);
-	} else if (IS_BROXTON(dev_priv)) {
-		i915_perf_load_test_config_bxt(dev_priv);
-	} else if (IS_KABYLAKE(dev_priv)) {
-		if (IS_KBL_GT2(dev_priv))
-			i915_perf_load_test_config_kblgt2(dev_priv);
-		else if (IS_KBL_GT3(dev_priv))
-			i915_perf_load_test_config_kblgt3(dev_priv);
-	} else if (IS_GEMINILAKE(dev_priv)) {
-		i915_perf_load_test_config_glk(dev_priv);
-	} else if (IS_COFFEELAKE(dev_priv)) {
-		if (IS_CFL_GT2(dev_priv))
-			i915_perf_load_test_config_cflgt2(dev_priv);
-		if (IS_CFL_GT3(dev_priv))
-			i915_perf_load_test_config_cflgt3(dev_priv);
-	} else if (IS_CANNONLAKE(dev_priv)) {
-		i915_perf_load_test_config_cnl(dev_priv);
-	} else if (IS_ICELAKE(dev_priv)) {
-		i915_perf_load_test_config_icl(dev_priv);
-	}
+	} else if (IS_CHERRYVIEW(dev_priv)) {
+		i915_perf_load_test_config_chv(dev_priv);
+	} else if (IS_BROADWELL(dev_priv)) {
+		i915_perf_load_test_config_bdw(dev_priv);
+	} else if (IS_HASWELL(dev_priv)) {
+		i915_perf_load_test_config_hsw(dev_priv);
+}
 
 	if (dev_priv->perf.oa.test_config.id == 0)
 		goto sysfs_error;
diff --git a/drivers/gpu/drm/i915/i915_pmu.c b/drivers/gpu/drm/i915/i915_pmu.c
index b745c49a5af6..46a52da3db29 100644
--- a/drivers/gpu/drm/i915/i915_pmu.c
+++ b/drivers/gpu/drm/i915/i915_pmu.c
@@ -102,7 +102,7 @@ static bool pmu_needs_timer(struct drm_i915_private *i915, bool gpu_active)
 	 *
 	 * Use RCS as proxy for all engines.
 	 */
-	else if (intel_engine_supports_stats(i915->engine[RCS]))
+	else if (intel_engine_supports_stats(i915->engine[RCS0]))
 		enable &= ~BIT(I915_SAMPLE_BUSY);
 
 	/*
@@ -149,14 +149,6 @@ void i915_pmu_gt_unparked(struct drm_i915_private *i915)
 	spin_unlock_irq(&i915->pmu.lock);
 }
 
-static bool grab_forcewake(struct drm_i915_private *i915, bool fw)
-{
-	if (!fw)
-		intel_uncore_forcewake_get(i915, FORCEWAKE_ALL);
-
-	return true;
-}
-
 static void
 add_sample(struct i915_pmu_sample *sample, u32 val)
 {
@@ -169,49 +161,48 @@ engines_sample(struct drm_i915_private *dev_priv, unsigned int period_ns)
 	struct intel_engine_cs *engine;
 	enum intel_engine_id id;
 	intel_wakeref_t wakeref;
-	bool fw = false;
+	unsigned long flags;
 
 	if ((dev_priv->pmu.enable & ENGINE_SAMPLE_MASK) == 0)
 		return;
 
-	if (!dev_priv->gt.awake)
-		return;
-
-	wakeref = intel_runtime_pm_get_if_in_use(dev_priv);
+	wakeref = 0;
+	if (READ_ONCE(dev_priv->gt.awake))
+		wakeref = intel_runtime_pm_get_if_in_use(dev_priv);
 	if (!wakeref)
 		return;
 
+	spin_lock_irqsave(&dev_priv->uncore.lock, flags);
 	for_each_engine(engine, dev_priv, id) {
-		u32 current_seqno = intel_engine_get_seqno(engine);
-		u32 last_seqno = intel_engine_last_submit(engine);
+		struct intel_engine_pmu *pmu = &engine->pmu;
+		bool busy;
 		u32 val;
 
-		val = !i915_seqno_passed(current_seqno, last_seqno);
-
-		if (val)
-			add_sample(&engine->pmu.sample[I915_SAMPLE_BUSY],
-				   period_ns);
-
-		if (val && (engine->pmu.enable &
-		    (BIT(I915_SAMPLE_WAIT) | BIT(I915_SAMPLE_SEMA)))) {
-			fw = grab_forcewake(dev_priv, fw);
-
-			val = I915_READ_FW(RING_CTL(engine->mmio_base));
-		} else {
-			val = 0;
-		}
+		val = I915_READ_FW(RING_CTL(engine->mmio_base));
+		if (val == 0) /* powerwell off => engine idle */
+			continue;
 
 		if (val & RING_WAIT)
-			add_sample(&engine->pmu.sample[I915_SAMPLE_WAIT],
-				   period_ns);
-
+			add_sample(&pmu->sample[I915_SAMPLE_WAIT], period_ns);
 		if (val & RING_WAIT_SEMAPHORE)
-			add_sample(&engine->pmu.sample[I915_SAMPLE_SEMA],
-				   period_ns);
-	}
+			add_sample(&pmu->sample[I915_SAMPLE_SEMA], period_ns);
 
-	if (fw)
-		intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+		/*
+		 * While waiting on a semaphore or event, MI_MODE reports the
+		 * ring as idle. However, previously using the seqno, and with
+		 * execlists sampling, we account for the ring waiting as the
+		 * engine being busy. Therefore, we record the sample as being
+		 * busy if either waiting or !idle.
+		 */
+		busy = val & (RING_WAIT_SEMAPHORE | RING_WAIT);
+		if (!busy) {
+			val = I915_READ_FW(RING_MI_MODE(engine->mmio_base));
+			busy = !(val & MODE_IDLE);
+		}
+		if (busy)
+			add_sample(&pmu->sample[I915_SAMPLE_BUSY], period_ns);
+	}
+	spin_unlock_irqrestore(&dev_priv->uncore.lock, flags);
 
 	intel_runtime_pm_put(dev_priv, wakeref);
 }
diff --git a/drivers/gpu/drm/i915/i915_priolist_types.h b/drivers/gpu/drm/i915/i915_priolist_types.h
new file mode 100644
index 000000000000..cc44ebd3b553
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_priolist_types.h
@@ -0,0 +1,42 @@
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2018 Intel Corporation
+ */
+
+#ifndef _I915_PRIOLIST_TYPES_H_
+#define _I915_PRIOLIST_TYPES_H_
+
+#include <linux/list.h>
+#include <linux/rbtree.h>
+
+#include <uapi/drm/i915_drm.h>
+
+enum {
+	I915_PRIORITY_MIN = I915_CONTEXT_MIN_USER_PRIORITY - 1,
+	I915_PRIORITY_NORMAL = I915_CONTEXT_DEFAULT_PRIORITY,
+	I915_PRIORITY_MAX = I915_CONTEXT_MAX_USER_PRIORITY + 1,
+
+	I915_PRIORITY_INVALID = INT_MIN
+};
+
+#define I915_USER_PRIORITY_SHIFT 3
+#define I915_USER_PRIORITY(x) ((x) << I915_USER_PRIORITY_SHIFT)
+
+#define I915_PRIORITY_COUNT BIT(I915_USER_PRIORITY_SHIFT)
+#define I915_PRIORITY_MASK (I915_PRIORITY_COUNT - 1)
+
+#define I915_PRIORITY_WAIT		((u8)BIT(0))
+#define I915_PRIORITY_NEWCLIENT		((u8)BIT(1))
+#define I915_PRIORITY_NOSEMAPHORE	((u8)BIT(2))
+
+#define __NO_PREEMPTION (I915_PRIORITY_WAIT)
+
+struct i915_priolist {
+	struct list_head requests[I915_PRIORITY_COUNT];
+	struct rb_node node;
+	unsigned long used;
+	int priority;
+};
+
+#endif /* _I915_PRIOLIST_TYPES_H_ */
diff --git a/drivers/gpu/drm/i915/i915_pvinfo.h b/drivers/gpu/drm/i915/i915_pvinfo.h
index eeaa3d506d95..969e514916ab 100644
--- a/drivers/gpu/drm/i915/i915_pvinfo.h
+++ b/drivers/gpu/drm/i915/i915_pvinfo.h
@@ -52,7 +52,7 @@ enum vgt_g2v_type {
 /*
  * VGT capabilities type
  */
-#define VGT_CAPS_FULL_48BIT_PPGTT	BIT(2)
+#define VGT_CAPS_FULL_PPGTT		BIT(2)
 #define VGT_CAPS_HWSP_EMULATION		BIT(3)
 #define VGT_CAPS_HUGE_GTT		BIT(4)
 
diff --git a/drivers/gpu/drm/i915/i915_query.c b/drivers/gpu/drm/i915/i915_query.c
index cbcb957b7141..782183b78f49 100644
--- a/drivers/gpu/drm/i915/i915_query.c
+++ b/drivers/gpu/drm/i915/i915_query.c
@@ -10,12 +10,34 @@
 #include "i915_query.h"
 #include <uapi/drm/i915_drm.h>
 
+static int copy_query_item(void *query_hdr, size_t query_sz,
+			   u32 total_length,
+			   struct drm_i915_query_item *query_item)
+{
+	if (query_item->length == 0)
+		return total_length;
+
+	if (query_item->length < total_length)
+		return -EINVAL;
+
+	if (copy_from_user(query_hdr, u64_to_user_ptr(query_item->data_ptr),
+			   query_sz))
+		return -EFAULT;
+
+	if (!access_ok(u64_to_user_ptr(query_item->data_ptr),
+		       total_length))
+		return -EFAULT;
+
+	return 0;
+}
+
 static int query_topology_info(struct drm_i915_private *dev_priv,
 			       struct drm_i915_query_item *query_item)
 {
 	const struct sseu_dev_info *sseu = &RUNTIME_INFO(dev_priv)->sseu;
 	struct drm_i915_query_topology_info topo;
 	u32 slice_length, subslice_length, eu_length, total_length;
+	int ret;
 
 	if (query_item->flags != 0)
 		return -EINVAL;
@@ -33,23 +55,14 @@ static int query_topology_info(struct drm_i915_private *dev_priv,
 
 	total_length = sizeof(topo) + slice_length + subslice_length + eu_length;
 
-	if (query_item->length == 0)
-		return total_length;
-
-	if (query_item->length < total_length)
-		return -EINVAL;
-
-	if (copy_from_user(&topo, u64_to_user_ptr(query_item->data_ptr),
-			   sizeof(topo)))
-		return -EFAULT;
+	ret = copy_query_item(&topo, sizeof(topo), total_length,
+			      query_item);
+	if (ret != 0)
+		return ret;
 
 	if (topo.flags != 0)
 		return -EINVAL;
 
-	if (!access_ok(u64_to_user_ptr(query_item->data_ptr),
-		       total_length))
-		return -EFAULT;
-
 	memset(&topo, 0, sizeof(topo));
 	topo.max_slices = sseu->max_slices;
 	topo.max_subslices = sseu->max_subslices;
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 047855dd8c6b..b74824f0b5b1 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -25,6 +25,9 @@
 #ifndef _I915_REG_H_
 #define _I915_REG_H_
 
+#include <linux/bitfield.h>
+#include <linux/bits.h>
+
 /**
  * DOC: The i915 register macro definition style guide
  *
@@ -59,15 +62,13 @@
  * significant to least significant bit. Indent the register content macros
  * using two extra spaces between ``#define`` and the macro name.
  *
- * For bit fields, define a ``_MASK`` and a ``_SHIFT`` macro. Define bit field
- * contents so that they are already shifted in place, and can be directly
- * OR'd. For convenience, function-like macros may be used to define bit fields,
- * but do note that the macros may be needed to read as well as write the
- * register contents.
+ * Define bit fields using ``REG_GENMASK(h, l)``. Define bit field contents
+ * using ``REG_FIELD_PREP(mask, value)``. This will define the values already
+ * shifted in place, so they can be directly OR'd together. For convenience,
+ * function-like macros may be used to define bit fields, but do note that the
+ * macros may be needed to read as well as write the register contents.
  *
- * Define bits using ``(1 << N)`` instead of ``BIT(N)``. We may change this in
- * the future, but this is the prevailing style. Do **not** add ``_BIT`` suffix
- * to the name.
+ * Define bits using ``REG_BIT(N)``. Do **not** add ``_BIT`` suffix to the name.
  *
  * Group the register and its contents together without blank lines, separate
  * from other registers and their contents with one blank line.
@@ -105,17 +106,78 @@
  *  #define _FOO_A                      0xf000
  *  #define _FOO_B                      0xf001
  *  #define FOO(pipe)                   _MMIO_PIPE(pipe, _FOO_A, _FOO_B)
- *  #define   FOO_ENABLE                (1 << 31)
- *  #define   FOO_MODE_MASK             (0xf << 16)
- *  #define   FOO_MODE_SHIFT            16
- *  #define   FOO_MODE_BAR              (0 << 16)
- *  #define   FOO_MODE_BAZ              (1 << 16)
- *  #define   FOO_MODE_QUX_SNB          (2 << 16)
+ *  #define   FOO_ENABLE                REG_BIT(31)
+ *  #define   FOO_MODE_MASK             REG_GENMASK(19, 16)
+ *  #define   FOO_MODE_BAR              REG_FIELD_PREP(FOO_MODE_MASK, 0)
+ *  #define   FOO_MODE_BAZ              REG_FIELD_PREP(FOO_MODE_MASK, 1)
+ *  #define   FOO_MODE_QUX_SNB          REG_FIELD_PREP(FOO_MODE_MASK, 2)
  *
  *  #define BAR                         _MMIO(0xb000)
  *  #define GEN8_BAR                    _MMIO(0xb888)
  */
 
+/**
+ * REG_BIT() - Prepare a u32 bit value
+ * @__n: 0-based bit number
+ *
+ * Local wrapper for BIT() to force u32, with compile time checks.
+ *
+ * @return: Value with bit @__n set.
+ */
+#define REG_BIT(__n)							\
+	((u32)(BIT(__n) +						\
+	       BUILD_BUG_ON_ZERO(__builtin_constant_p(__n) &&		\
+				 ((__n) < 0 || (__n) > 31))))
+
+/**
+ * REG_GENMASK() - Prepare a continuous u32 bitmask
+ * @__high: 0-based high bit
+ * @__low: 0-based low bit
+ *
+ * Local wrapper for GENMASK() to force u32, with compile time checks.
+ *
+ * @return: Continuous bitmask from @__high to @__low, inclusive.
+ */
+#define REG_GENMASK(__high, __low)					\
+	((u32)(GENMASK(__high, __low) +					\
+	       BUILD_BUG_ON_ZERO(__builtin_constant_p(__high) &&	\
+				 __builtin_constant_p(__low) &&		\
+				 ((__low) < 0 || (__high) > 31 || (__low) > (__high)))))
+
+/*
+ * Local integer constant expression version of is_power_of_2().
+ */
+#define IS_POWER_OF_2(__x)		((__x) && (((__x) & ((__x) - 1)) == 0))
+
+/**
+ * REG_FIELD_PREP() - Prepare a u32 bitfield value
+ * @__mask: shifted mask defining the field's length and position
+ * @__val: value to put in the field
+
+ * Local copy of FIELD_PREP() to generate an integer constant expression, force
+ * u32 and for consistency with REG_FIELD_GET(), REG_BIT() and REG_GENMASK().
+ *
+ * @return: @__val masked and shifted into the field defined by @__mask.
+ */
+#define REG_FIELD_PREP(__mask, __val)						\
+	((u32)((((typeof(__mask))(__val) << __bf_shf(__mask)) & (__mask)) +	\
+	       BUILD_BUG_ON_ZERO(!__is_constexpr(__mask)) +		\
+	       BUILD_BUG_ON_ZERO((__mask) == 0 || (__mask) > U32_MAX) +		\
+	       BUILD_BUG_ON_ZERO(!IS_POWER_OF_2((__mask) + (1ULL << __bf_shf(__mask)))) + \
+	       BUILD_BUG_ON_ZERO(__builtin_choose_expr(__is_constexpr(__val), (~((__mask) >> __bf_shf(__mask)) & (__val)), 0))))
+
+/**
+ * REG_FIELD_GET() - Extract a u32 bitfield value
+ * @__mask: shifted mask defining the field's length and position
+ * @__val: value to extract the bitfield value from
+ *
+ * Local wrapper for FIELD_GET() to force u32 and for consistency with
+ * REG_FIELD_PREP(), REG_BIT() and REG_GENMASK().
+ *
+ * @return: Masked and shifted value of the field defined by @__mask in @__val.
+ */
+#define REG_FIELD_GET(__mask, __val)	((u32)FIELD_GET(__mask, __val))
+
 typedef struct {
 	u32 reg;
 } i915_reg_t;
@@ -210,14 +272,14 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 
 /* Engine ID */
 
-#define RCS_HW		0
-#define VCS_HW		1
-#define BCS_HW		2
-#define VECS_HW		3
-#define VCS2_HW		4
-#define VCS3_HW		6
-#define VCS4_HW		7
-#define VECS2_HW	12
+#define RCS0_HW		0
+#define VCS0_HW		1
+#define BCS0_HW		2
+#define VECS0_HW	3
+#define VCS1_HW		4
+#define VCS2_HW		6
+#define VCS3_HW		7
+#define VECS1_HW	12
 
 /* Engine class */
 
@@ -372,13 +434,13 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define GEN11_VECS_SFC_USAGE(engine)		_MMIO((engine)->mmio_base + 0x2014)
 #define   GEN11_VECS_SFC_USAGE_BIT		(1 << 0)
 
-#define RING_PP_DIR_BASE(engine)	_MMIO((engine)->mmio_base + 0x228)
-#define RING_PP_DIR_BASE_READ(engine)	_MMIO((engine)->mmio_base + 0x518)
-#define RING_PP_DIR_DCLV(engine)	_MMIO((engine)->mmio_base + 0x220)
+#define RING_PP_DIR_BASE(base)		_MMIO((base) + 0x228)
+#define RING_PP_DIR_BASE_READ(base)	_MMIO((base) + 0x518)
+#define RING_PP_DIR_DCLV(base)		_MMIO((base) + 0x220)
 #define   PP_DIR_DCLV_2G		0xffffffff
 
-#define GEN8_RING_PDP_UDW(engine, n)	_MMIO((engine)->mmio_base + 0x270 + (n) * 8 + 4)
-#define GEN8_RING_PDP_LDW(engine, n)	_MMIO((engine)->mmio_base + 0x270 + (n) * 8)
+#define GEN8_RING_PDP_UDW(base, n)	_MMIO((base) + 0x270 + (n) * 8 + 4)
+#define GEN8_RING_PDP_LDW(base, n)	_MMIO((base) + 0x270 + (n) * 8)
 
 #define GEN8_R_PWR_CLK_STATE		_MMIO(0x20C8)
 #define   GEN8_RPCS_ENABLE		(1 << 31)
@@ -1044,7 +1106,32 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 /* See configdb bunit SB addr map */
 #define BUNIT_REG_BISOC				0x11
 
-#define PUNIT_REG_DSPFREQ			0x36
+/* PUNIT_REG_*SSPM0 */
+#define   _SSPM0_SSC(val)			((val) << 0)
+#define   SSPM0_SSC_MASK			_SSPM0_SSC(0x3)
+#define   SSPM0_SSC_PWR_ON			_SSPM0_SSC(0x0)
+#define   SSPM0_SSC_CLK_GATE			_SSPM0_SSC(0x1)
+#define   SSPM0_SSC_RESET			_SSPM0_SSC(0x2)
+#define   SSPM0_SSC_PWR_GATE			_SSPM0_SSC(0x3)
+#define   _SSPM0_SSS(val)			((val) << 24)
+#define   SSPM0_SSS_MASK			_SSPM0_SSS(0x3)
+#define   SSPM0_SSS_PWR_ON			_SSPM0_SSS(0x0)
+#define   SSPM0_SSS_CLK_GATE			_SSPM0_SSS(0x1)
+#define   SSPM0_SSS_RESET			_SSPM0_SSS(0x2)
+#define   SSPM0_SSS_PWR_GATE			_SSPM0_SSS(0x3)
+
+/* PUNIT_REG_*SSPM1 */
+#define   SSPM1_FREQSTAT_SHIFT			24
+#define   SSPM1_FREQSTAT_MASK			(0x1f << SSPM1_FREQSTAT_SHIFT)
+#define   SSPM1_FREQGUAR_SHIFT			8
+#define   SSPM1_FREQGUAR_MASK			(0x1f << SSPM1_FREQGUAR_SHIFT)
+#define   SSPM1_FREQ_SHIFT			0
+#define   SSPM1_FREQ_MASK			(0x1f << SSPM1_FREQ_SHIFT)
+
+#define PUNIT_REG_VEDSSPM0			0x32
+#define PUNIT_REG_VEDSSPM1			0x33
+
+#define PUNIT_REG_DSPSSPM			0x36
 #define   DSPFREQSTAT_SHIFT_CHV			24
 #define   DSPFREQSTAT_MASK_CHV			(0x1f << DSPFREQSTAT_SHIFT_CHV)
 #define   DSPFREQGUAR_SHIFT_CHV			8
@@ -1069,6 +1156,9 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define   DP_SSS_RESET(pipe)			_DP_SSS(0x2, (pipe))
 #define   DP_SSS_PWR_GATE(pipe)			_DP_SSS(0x3, (pipe))
 
+#define PUNIT_REG_ISPSSPM0			0x39
+#define PUNIT_REG_ISPSSPM1			0x3a
+
 /*
  * i915_power_well_id:
  *
@@ -1860,13 +1950,13 @@ enum i915_power_well_id {
 #define _CNL_PORT_TX_DW4_LN1_AE		0x1624D0
 #define CNL_PORT_TX_DW4_GRP(port)	_MMIO(_CNL_PORT_TX_DW_GRP(4, (port)))
 #define CNL_PORT_TX_DW4_LN0(port)	_MMIO(_CNL_PORT_TX_DW_LN0(4, (port)))
-#define CNL_PORT_TX_DW4_LN(port, ln)   _MMIO(_CNL_PORT_TX_DW_LN0(4, (port)) + \
+#define CNL_PORT_TX_DW4_LN(ln, port)   _MMIO(_CNL_PORT_TX_DW_LN0(4, (port)) + \
 					   ((ln) * (_CNL_PORT_TX_DW4_LN1_AE - \
 						    _CNL_PORT_TX_DW4_LN0_AE)))
 #define ICL_PORT_TX_DW4_AUX(port)	_MMIO(_ICL_PORT_TX_DW_AUX(4, port))
 #define ICL_PORT_TX_DW4_GRP(port)	_MMIO(_ICL_PORT_TX_DW_GRP(4, port))
 #define ICL_PORT_TX_DW4_LN0(port)	_MMIO(_ICL_PORT_TX_DW_LN(4, 0, port))
-#define ICL_PORT_TX_DW4_LN(port, ln)	_MMIO(_ICL_PORT_TX_DW_LN(4, ln, port))
+#define ICL_PORT_TX_DW4_LN(ln, port)	_MMIO(_ICL_PORT_TX_DW_LN(4, ln, port))
 #define   LOADGEN_SELECT		(1 << 31)
 #define   POST_CURSOR_1(x)		((x) << 12)
 #define   POST_CURSOR_1_MASK		(0x3F << 12)
@@ -1893,11 +1983,11 @@ enum i915_power_well_id {
 #define ICL_PORT_TX_DW7_AUX(port)	_MMIO(_ICL_PORT_TX_DW_AUX(7, port))
 #define ICL_PORT_TX_DW7_GRP(port)	_MMIO(_ICL_PORT_TX_DW_GRP(7, port))
 #define ICL_PORT_TX_DW7_LN0(port)	_MMIO(_ICL_PORT_TX_DW_LN(7, 0, port))
-#define ICL_PORT_TX_DW7_LN(port, ln)	_MMIO(_ICL_PORT_TX_DW_LN(7, ln, port))
+#define ICL_PORT_TX_DW7_LN(ln, port)	_MMIO(_ICL_PORT_TX_DW_LN(7, ln, port))
 #define   N_SCALAR(x)			((x) << 24)
 #define   N_SCALAR_MASK			(0x7F << 24)
 
-#define MG_PHY_PORT_LN(port, ln, ln0p1, ln0p2, ln1p1) \
+#define MG_PHY_PORT_LN(ln, port, ln0p1, ln0p2, ln1p1) \
 	_MMIO(_PORT((port) - PORT_C, ln0p1, ln0p2) + (ln) * ((ln1p1) - (ln0p1)))
 
 #define MG_TX_LINK_PARAMS_TX1LN0_PORT1		0x16812C
@@ -1908,8 +1998,8 @@ enum i915_power_well_id {
 #define MG_TX_LINK_PARAMS_TX1LN1_PORT3		0x16A52C
 #define MG_TX_LINK_PARAMS_TX1LN0_PORT4		0x16B12C
 #define MG_TX_LINK_PARAMS_TX1LN1_PORT4		0x16B52C
-#define MG_TX1_LINK_PARAMS(port, ln) \
-	MG_PHY_PORT_LN(port, ln, MG_TX_LINK_PARAMS_TX1LN0_PORT1, \
+#define MG_TX1_LINK_PARAMS(ln, port) \
+	MG_PHY_PORT_LN(ln, port, MG_TX_LINK_PARAMS_TX1LN0_PORT1, \
 				 MG_TX_LINK_PARAMS_TX1LN0_PORT2, \
 				 MG_TX_LINK_PARAMS_TX1LN1_PORT1)
 
@@ -1921,8 +2011,8 @@ enum i915_power_well_id {
 #define MG_TX_LINK_PARAMS_TX2LN1_PORT3		0x16A4AC
 #define MG_TX_LINK_PARAMS_TX2LN0_PORT4		0x16B0AC
 #define MG_TX_LINK_PARAMS_TX2LN1_PORT4		0x16B4AC
-#define MG_TX2_LINK_PARAMS(port, ln) \
-	MG_PHY_PORT_LN(port, ln, MG_TX_LINK_PARAMS_TX2LN0_PORT1, \
+#define MG_TX2_LINK_PARAMS(ln, port) \
+	MG_PHY_PORT_LN(ln, port, MG_TX_LINK_PARAMS_TX2LN0_PORT1, \
 				 MG_TX_LINK_PARAMS_TX2LN0_PORT2, \
 				 MG_TX_LINK_PARAMS_TX2LN1_PORT1)
 #define   CRI_USE_FS32			(1 << 5)
@@ -1935,8 +2025,8 @@ enum i915_power_well_id {
 #define MG_TX_PISO_READLOAD_TX1LN1_PORT3		0x16A54C
 #define MG_TX_PISO_READLOAD_TX1LN0_PORT4		0x16B14C
 #define MG_TX_PISO_READLOAD_TX1LN1_PORT4		0x16B54C
-#define MG_TX1_PISO_READLOAD(port, ln) \
-	MG_PHY_PORT_LN(port, ln, MG_TX_PISO_READLOAD_TX1LN0_PORT1, \
+#define MG_TX1_PISO_READLOAD(ln, port) \
+	MG_PHY_PORT_LN(ln, port, MG_TX_PISO_READLOAD_TX1LN0_PORT1, \
 				 MG_TX_PISO_READLOAD_TX1LN0_PORT2, \
 				 MG_TX_PISO_READLOAD_TX1LN1_PORT1)
 
@@ -1948,8 +2038,8 @@ enum i915_power_well_id {
 #define MG_TX_PISO_READLOAD_TX2LN1_PORT3		0x16A4CC
 #define MG_TX_PISO_READLOAD_TX2LN0_PORT4		0x16B0CC
 #define MG_TX_PISO_READLOAD_TX2LN1_PORT4		0x16B4CC
-#define MG_TX2_PISO_READLOAD(port, ln) \
-	MG_PHY_PORT_LN(port, ln, MG_TX_PISO_READLOAD_TX2LN0_PORT1, \
+#define MG_TX2_PISO_READLOAD(ln, port) \
+	MG_PHY_PORT_LN(ln, port, MG_TX_PISO_READLOAD_TX2LN0_PORT1, \
 				 MG_TX_PISO_READLOAD_TX2LN0_PORT2, \
 				 MG_TX_PISO_READLOAD_TX2LN1_PORT1)
 #define   CRI_CALCINIT					(1 << 1)
@@ -1962,8 +2052,8 @@ enum i915_power_well_id {
 #define MG_TX_SWINGCTRL_TX1LN1_PORT3		0x16A548
 #define MG_TX_SWINGCTRL_TX1LN0_PORT4		0x16B148
 #define MG_TX_SWINGCTRL_TX1LN1_PORT4		0x16B548
-#define MG_TX1_SWINGCTRL(port, ln) \
-	MG_PHY_PORT_LN(port, ln, MG_TX_SWINGCTRL_TX1LN0_PORT1, \
+#define MG_TX1_SWINGCTRL(ln, port) \
+	MG_PHY_PORT_LN(ln, port, MG_TX_SWINGCTRL_TX1LN0_PORT1, \
 				 MG_TX_SWINGCTRL_TX1LN0_PORT2, \
 				 MG_TX_SWINGCTRL_TX1LN1_PORT1)
 
@@ -1975,8 +2065,8 @@ enum i915_power_well_id {
 #define MG_TX_SWINGCTRL_TX2LN1_PORT3		0x16A4C8
 #define MG_TX_SWINGCTRL_TX2LN0_PORT4		0x16B0C8
 #define MG_TX_SWINGCTRL_TX2LN1_PORT4		0x16B4C8
-#define MG_TX2_SWINGCTRL(port, ln) \
-	MG_PHY_PORT_LN(port, ln, MG_TX_SWINGCTRL_TX2LN0_PORT1, \
+#define MG_TX2_SWINGCTRL(ln, port) \
+	MG_PHY_PORT_LN(ln, port, MG_TX_SWINGCTRL_TX2LN0_PORT1, \
 				 MG_TX_SWINGCTRL_TX2LN0_PORT2, \
 				 MG_TX_SWINGCTRL_TX2LN1_PORT1)
 #define   CRI_TXDEEMPH_OVERRIDE_17_12(x)		((x) << 0)
@@ -1990,8 +2080,8 @@ enum i915_power_well_id {
 #define MG_TX_DRVCTRL_TX1LN1_TXPORT3			0x16A544
 #define MG_TX_DRVCTRL_TX1LN0_TXPORT4			0x16B144
 #define MG_TX_DRVCTRL_TX1LN1_TXPORT4			0x16B544
-#define MG_TX1_DRVCTRL(port, ln) \
-	MG_PHY_PORT_LN(port, ln, MG_TX_DRVCTRL_TX1LN0_TXPORT1, \
+#define MG_TX1_DRVCTRL(ln, port) \
+	MG_PHY_PORT_LN(ln, port, MG_TX_DRVCTRL_TX1LN0_TXPORT1, \
 				 MG_TX_DRVCTRL_TX1LN0_TXPORT2, \
 				 MG_TX_DRVCTRL_TX1LN1_TXPORT1)
 
@@ -2003,8 +2093,8 @@ enum i915_power_well_id {
 #define MG_TX_DRVCTRL_TX2LN1_PORT3			0x16A4C4
 #define MG_TX_DRVCTRL_TX2LN0_PORT4			0x16B0C4
 #define MG_TX_DRVCTRL_TX2LN1_PORT4			0x16B4C4
-#define MG_TX2_DRVCTRL(port, ln) \
-	MG_PHY_PORT_LN(port, ln, MG_TX_DRVCTRL_TX2LN0_PORT1, \
+#define MG_TX2_DRVCTRL(ln, port) \
+	MG_PHY_PORT_LN(ln, port, MG_TX_DRVCTRL_TX2LN0_PORT1, \
 				 MG_TX_DRVCTRL_TX2LN0_PORT2, \
 				 MG_TX_DRVCTRL_TX2LN1_PORT1)
 #define   CRI_TXDEEMPH_OVERRIDE_11_6(x)			((x) << 24)
@@ -2023,8 +2113,8 @@ enum i915_power_well_id {
 #define MG_CLKHUB_LN1_PORT3			0x16A79C
 #define MG_CLKHUB_LN0_PORT4			0x16B39C
 #define MG_CLKHUB_LN1_PORT4			0x16B79C
-#define MG_CLKHUB(port, ln) \
-	MG_PHY_PORT_LN(port, ln, MG_CLKHUB_LN0_PORT1, \
+#define MG_CLKHUB(ln, port) \
+	MG_PHY_PORT_LN(ln, port, MG_CLKHUB_LN0_PORT1, \
 				 MG_CLKHUB_LN0_PORT2, \
 				 MG_CLKHUB_LN1_PORT1)
 #define   CFG_LOW_RATE_LKREN_EN				(1 << 11)
@@ -2037,8 +2127,8 @@ enum i915_power_well_id {
 #define MG_TX_DCC_TX1LN1_PORT3			0x16A510
 #define MG_TX_DCC_TX1LN0_PORT4			0x16B110
 #define MG_TX_DCC_TX1LN1_PORT4			0x16B510
-#define MG_TX1_DCC(port, ln) \
-	MG_PHY_PORT_LN(port, ln, MG_TX_DCC_TX1LN0_PORT1, \
+#define MG_TX1_DCC(ln, port) \
+	MG_PHY_PORT_LN(ln, port, MG_TX_DCC_TX1LN0_PORT1, \
 				 MG_TX_DCC_TX1LN0_PORT2, \
 				 MG_TX_DCC_TX1LN1_PORT1)
 #define MG_TX_DCC_TX2LN0_PORT1			0x168090
@@ -2049,8 +2139,8 @@ enum i915_power_well_id {
 #define MG_TX_DCC_TX2LN1_PORT3			0x16A490
 #define MG_TX_DCC_TX2LN0_PORT4			0x16B090
 #define MG_TX_DCC_TX2LN1_PORT4			0x16B490
-#define MG_TX2_DCC(port, ln) \
-	MG_PHY_PORT_LN(port, ln, MG_TX_DCC_TX2LN0_PORT1, \
+#define MG_TX2_DCC(ln, port) \
+	MG_PHY_PORT_LN(ln, port, MG_TX_DCC_TX2LN0_PORT1, \
 				 MG_TX_DCC_TX2LN0_PORT2, \
 				 MG_TX_DCC_TX2LN1_PORT1)
 #define   CFG_AMI_CK_DIV_OVERRIDE_VAL(x)	((x) << 25)
@@ -2065,8 +2155,8 @@ enum i915_power_well_id {
 #define MG_DP_MODE_LN1_ACU_PORT3			0x16A7A0
 #define MG_DP_MODE_LN0_ACU_PORT4			0x16B3A0
 #define MG_DP_MODE_LN1_ACU_PORT4			0x16B7A0
-#define MG_DP_MODE(port, ln)	\
-	MG_PHY_PORT_LN(port, ln, MG_DP_MODE_LN0_ACU_PORT1, \
+#define MG_DP_MODE(ln, port)	\
+	MG_PHY_PORT_LN(ln, port, MG_DP_MODE_LN0_ACU_PORT1, \
 				 MG_DP_MODE_LN0_ACU_PORT2, \
 				 MG_DP_MODE_LN1_ACU_PORT1)
 #define   MG_DP_MODE_CFG_DP_X2_MODE			(1 << 7)
@@ -2356,8 +2446,10 @@ enum i915_power_well_id {
 #define RING_HWS_PGA(base)	_MMIO((base) + 0x80)
 #define RING_HWS_PGA_GEN6(base)	_MMIO((base) + 0x2080)
 #define RING_RESET_CTL(base)	_MMIO((base) + 0xd0)
-#define   RESET_CTL_REQUEST_RESET  (1 << 0)
-#define   RESET_CTL_READY_TO_RESET (1 << 1)
+#define   RESET_CTL_CAT_ERROR	   REG_BIT(2)
+#define   RESET_CTL_READY_TO_RESET REG_BIT(1)
+#define   RESET_CTL_REQUEST_RESET  REG_BIT(0)
+
 #define RING_SEMA_WAIT_POLL(base) _MMIO((base) + 0x24c)
 
 #define HSW_GTT_CACHE_EN	_MMIO(0x4024)
@@ -2478,12 +2570,12 @@ enum i915_power_well_id {
 #define HWS_START_ADDRESS_SHIFT	4
 #define PWRCTXA		_MMIO(0x2088) /* 965GM+ only */
 #define   PWRCTX_EN	(1 << 0)
-#define IPEIR		_MMIO(0x2088)
-#define IPEHR		_MMIO(0x208c)
+#define IPEIR(base)	_MMIO((base) + 0x88)
+#define IPEHR(base)	_MMIO((base) + 0x8c)
 #define GEN2_INSTDONE	_MMIO(0x2090)
 #define NOPID		_MMIO(0x2094)
 #define HWSTAM		_MMIO(0x2098)
-#define DMA_FADD_I8XX	_MMIO(0x20d0)
+#define DMA_FADD_I8XX(base)	_MMIO((base) + 0xd0)
 #define RING_BBSTATE(base)	_MMIO((base) + 0x110)
 #define   RING_BB_PPGTT		(1 << 5)
 #define RING_SBBADDR(base)	_MMIO((base) + 0x114) /* hsw+ */
@@ -2623,10 +2715,10 @@ enum i915_power_well_id {
 #define VLV_GU_CTL0	_MMIO(VLV_DISPLAY_BASE + 0x2030)
 #define VLV_GU_CTL1	_MMIO(VLV_DISPLAY_BASE + 0x2034)
 #define SCPD0		_MMIO(0x209c) /* 915+ only */
-#define IER		_MMIO(0x20a0)
-#define IIR		_MMIO(0x20a4)
-#define IMR		_MMIO(0x20a8)
-#define ISR		_MMIO(0x20ac)
+#define GEN2_IER	_MMIO(0x20a0)
+#define GEN2_IIR	_MMIO(0x20a4)
+#define GEN2_IMR	_MMIO(0x20a8)
+#define GEN2_ISR	_MMIO(0x20ac)
 #define VLV_GUNIT_CLOCK_GATE	_MMIO(VLV_DISPLAY_BASE + 0x2060)
 #define   GINT_DIS		(1 << 22)
 #define   GCFG_DIS		(1 << 8)
@@ -2657,7 +2749,7 @@ enum i915_power_well_id {
 #define   INSTPM_FORCE_ORDERING				(1 << 7) /* GEN6+ */
 #define   INSTPM_TLB_INVALIDATE	(1 << 9)
 #define   INSTPM_SYNC_FLUSH	(1 << 5)
-#define ACTHD	        _MMIO(0x20c8)
+#define ACTHD(base)	_MMIO((base) + 0xc8)
 #define MEM_MODE	_MMIO(0x20cc)
 #define   MEM_DISPLAY_B_TRICKLE_FEED_DISABLE (1 << 3) /* 830 only */
 #define   MEM_DISPLAY_A_TRICKLE_FEED_DISABLE (1 << 2) /* 830/845 only */
@@ -3857,7 +3949,7 @@ enum i915_power_well_id {
 /*
  * Logical Context regs
  */
-#define CCID				_MMIO(0x2180)
+#define CCID(base)			_MMIO((base) + 0x180)
 #define   CCID_EN			BIT(0)
 #define   CCID_EXTENDED_STATE_RESTORE	BIT(2)
 #define   CCID_EXTENDED_STATE_SAVE	BIT(3)
@@ -3989,6 +4081,15 @@ enum {
 /* Pipe A CRC regs */
 #define _PIPE_CRC_CTL_A			0x60050
 #define   PIPE_CRC_ENABLE		(1 << 31)
+/* skl+ source selection */
+#define   PIPE_CRC_SOURCE_PLANE_1_SKL	(0 << 28)
+#define   PIPE_CRC_SOURCE_PLANE_2_SKL	(2 << 28)
+#define   PIPE_CRC_SOURCE_DMUX_SKL	(4 << 28)
+#define   PIPE_CRC_SOURCE_PLANE_3_SKL	(6 << 28)
+#define   PIPE_CRC_SOURCE_PLANE_4_SKL	(7 << 28)
+#define   PIPE_CRC_SOURCE_PLANE_5_SKL	(5 << 28)
+#define   PIPE_CRC_SOURCE_PLANE_6_SKL	(3 << 28)
+#define   PIPE_CRC_SOURCE_PLANE_7_SKL	(1 << 28)
 /* ivb+ source selection */
 #define   PIPE_CRC_SOURCE_PRIMARY_IVB	(0 << 29)
 #define   PIPE_CRC_SOURCE_SPRITE_IVB	(1 << 29)
@@ -4110,42 +4211,6 @@ enum {
 #define PIPESRC(trans)		_MMIO_TRANS2(trans, _PIPEASRC)
 #define PIPE_MULT(trans)	_MMIO_TRANS2(trans, _PIPE_MULT_A)
 
-/* VLV eDP PSR registers */
-#define _PSRCTLA				(VLV_DISPLAY_BASE + 0x60090)
-#define _PSRCTLB				(VLV_DISPLAY_BASE + 0x61090)
-#define  VLV_EDP_PSR_ENABLE			(1 << 0)
-#define  VLV_EDP_PSR_RESET			(1 << 1)
-#define  VLV_EDP_PSR_MODE_MASK			(7 << 2)
-#define  VLV_EDP_PSR_MODE_HW_TIMER		(1 << 3)
-#define  VLV_EDP_PSR_MODE_SW_TIMER		(1 << 2)
-#define  VLV_EDP_PSR_SINGLE_FRAME_UPDATE	(1 << 7)
-#define  VLV_EDP_PSR_ACTIVE_ENTRY		(1 << 8)
-#define  VLV_EDP_PSR_SRC_TRANSMITTER_STATE	(1 << 9)
-#define  VLV_EDP_PSR_DBL_FRAME			(1 << 10)
-#define  VLV_EDP_PSR_FRAME_COUNT_MASK		(0xff << 16)
-#define  VLV_EDP_PSR_IDLE_FRAME_SHIFT		16
-#define VLV_PSRCTL(pipe)	_MMIO_PIPE(pipe, _PSRCTLA, _PSRCTLB)
-
-#define _VSCSDPA			(VLV_DISPLAY_BASE + 0x600a0)
-#define _VSCSDPB			(VLV_DISPLAY_BASE + 0x610a0)
-#define  VLV_EDP_PSR_SDP_FREQ_MASK	(3 << 30)
-#define  VLV_EDP_PSR_SDP_FREQ_ONCE	(1 << 31)
-#define  VLV_EDP_PSR_SDP_FREQ_EVFRAME	(1 << 30)
-#define VLV_VSCSDP(pipe)	_MMIO_PIPE(pipe, _VSCSDPA, _VSCSDPB)
-
-#define _PSRSTATA			(VLV_DISPLAY_BASE + 0x60094)
-#define _PSRSTATB			(VLV_DISPLAY_BASE + 0x61094)
-#define  VLV_EDP_PSR_LAST_STATE_MASK	(7 << 3)
-#define  VLV_EDP_PSR_CURR_STATE_MASK	7
-#define  VLV_EDP_PSR_DISABLED		(0 << 0)
-#define  VLV_EDP_PSR_INACTIVE		(1 << 0)
-#define  VLV_EDP_PSR_IN_TRANS_TO_ACTIVE	(2 << 0)
-#define  VLV_EDP_PSR_ACTIVE_NORFB_UP	(3 << 0)
-#define  VLV_EDP_PSR_ACTIVE_SF_UPDATE	(4 << 0)
-#define  VLV_EDP_PSR_EXIT		(5 << 0)
-#define  VLV_EDP_PSR_IN_TRANS		(1 << 7)
-#define VLV_PSRSTAT(pipe)	_MMIO_PIPE(pipe, _PSRSTATA, _PSRSTATB)
-
 /* HSW+ eDP PSR registers */
 #define HSW_EDP_PSR_BASE	0x64800
 #define BDW_EDP_PSR_BASE	0x6f800
@@ -4168,6 +4233,7 @@ enum {
 #define   EDP_PSR_TP2_TP3_TIME_100us		(1 << 8)
 #define   EDP_PSR_TP2_TP3_TIME_2500us		(2 << 8)
 #define   EDP_PSR_TP2_TP3_TIME_0us		(3 << 8)
+#define   EDP_PSR_TP4_TIME_0US			(3 << 6) /* ICL+ */
 #define   EDP_PSR_TP1_TIME_500us		(0 << 4)
 #define   EDP_PSR_TP1_TIME_100us		(1 << 4)
 #define   EDP_PSR_TP1_TIME_2500us		(2 << 4)
@@ -4612,13 +4678,14 @@ enum {
 #define   VIDEO_DIP_ENABLE		(1 << 31)
 #define   VIDEO_DIP_PORT(port)		((port) << 29)
 #define   VIDEO_DIP_PORT_MASK		(3 << 29)
-#define   VIDEO_DIP_ENABLE_GCP		(1 << 25)
+#define   VIDEO_DIP_ENABLE_GCP		(1 << 25) /* ilk+ */
 #define   VIDEO_DIP_ENABLE_AVI		(1 << 21)
 #define   VIDEO_DIP_ENABLE_VENDOR	(2 << 21)
-#define   VIDEO_DIP_ENABLE_GAMUT	(4 << 21)
+#define   VIDEO_DIP_ENABLE_GAMUT	(4 << 21) /* ilk+ */
 #define   VIDEO_DIP_ENABLE_SPD		(8 << 21)
 #define   VIDEO_DIP_SELECT_AVI		(0 << 19)
 #define   VIDEO_DIP_SELECT_VENDOR	(1 << 19)
+#define   VIDEO_DIP_SELECT_GAMUT	(2 << 19)
 #define   VIDEO_DIP_SELECT_SPD		(3 << 19)
 #define   VIDEO_DIP_SELECT_MASK		(3 << 19)
 #define   VIDEO_DIP_FREQ_ONCE		(0 << 16)
@@ -4653,18 +4720,17 @@ enum {
 
 #define _PP_STATUS			0x61200
 #define PP_STATUS(pps_idx)		_MMIO_PPS(pps_idx, _PP_STATUS)
-#define   PP_ON				(1 << 31)
+#define   PP_ON				REG_BIT(31)
 
 #define _PP_CONTROL_1			0xc7204
 #define _PP_CONTROL_2			0xc7304
 #define ICP_PP_CONTROL(x)		_MMIO(((x) == 1) ? _PP_CONTROL_1 : \
 					      _PP_CONTROL_2)
-#define  POWER_CYCLE_DELAY_MASK	(0x1f << 4)
-#define  POWER_CYCLE_DELAY_SHIFT	4
-#define  VDD_OVERRIDE_FORCE		(1 << 3)
-#define  BACKLIGHT_ENABLE		(1 << 2)
-#define  PWR_DOWN_ON_RESET		(1 << 1)
-#define  PWR_STATE_TARGET		(1 << 0)
+#define  POWER_CYCLE_DELAY_MASK		REG_GENMASK(8, 4)
+#define  VDD_OVERRIDE_FORCE		REG_BIT(3)
+#define  BACKLIGHT_ENABLE		REG_BIT(2)
+#define  PWR_DOWN_ON_RESET		REG_BIT(1)
+#define  PWR_STATE_TARGET		REG_BIT(0)
 /*
  * Indicates that all dependencies of the panel are on:
  *
@@ -4672,62 +4738,53 @@ enum {
  * - pipe enabled
  * - LVDS/DVOB/DVOC on
  */
-#define   PP_READY			(1 << 30)
-#define   PP_SEQUENCE_NONE		(0 << 28)
-#define   PP_SEQUENCE_POWER_UP		(1 << 28)
-#define   PP_SEQUENCE_POWER_DOWN	(2 << 28)
-#define   PP_SEQUENCE_MASK		(3 << 28)
-#define   PP_SEQUENCE_SHIFT		28
-#define   PP_CYCLE_DELAY_ACTIVE		(1 << 27)
-#define   PP_SEQUENCE_STATE_MASK	0x0000000f
-#define   PP_SEQUENCE_STATE_OFF_IDLE	(0x0 << 0)
-#define   PP_SEQUENCE_STATE_OFF_S0_1	(0x1 << 0)
-#define   PP_SEQUENCE_STATE_OFF_S0_2	(0x2 << 0)
-#define   PP_SEQUENCE_STATE_OFF_S0_3	(0x3 << 0)
-#define   PP_SEQUENCE_STATE_ON_IDLE	(0x8 << 0)
-#define   PP_SEQUENCE_STATE_ON_S1_0	(0x9 << 0)
-#define   PP_SEQUENCE_STATE_ON_S1_2	(0xa << 0)
-#define   PP_SEQUENCE_STATE_ON_S1_3	(0xb << 0)
-#define   PP_SEQUENCE_STATE_RESET	(0xf << 0)
+#define   PP_READY			REG_BIT(30)
+#define   PP_SEQUENCE_MASK		REG_GENMASK(29, 28)
+#define   PP_SEQUENCE_NONE		REG_FIELD_PREP(PP_SEQUENCE_MASK, 0)
+#define   PP_SEQUENCE_POWER_UP		REG_FIELD_PREP(PP_SEQUENCE_MASK, 1)
+#define   PP_SEQUENCE_POWER_DOWN	REG_FIELD_PREP(PP_SEQUENCE_MASK, 2)
+#define   PP_CYCLE_DELAY_ACTIVE		REG_BIT(27)
+#define   PP_SEQUENCE_STATE_MASK	REG_GENMASK(3, 0)
+#define   PP_SEQUENCE_STATE_OFF_IDLE	REG_FIELD_PREP(PP_SEQUENCE_STATE_MASK, 0x0)
+#define   PP_SEQUENCE_STATE_OFF_S0_1	REG_FIELD_PREP(PP_SEQUENCE_STATE_MASK, 0x1)
+#define   PP_SEQUENCE_STATE_OFF_S0_2	REG_FIELD_PREP(PP_SEQUENCE_STATE_MASK, 0x2)
+#define   PP_SEQUENCE_STATE_OFF_S0_3	REG_FIELD_PREP(PP_SEQUENCE_STATE_MASK, 0x3)
+#define   PP_SEQUENCE_STATE_ON_IDLE	REG_FIELD_PREP(PP_SEQUENCE_STATE_MASK, 0x8)
+#define   PP_SEQUENCE_STATE_ON_S1_1	REG_FIELD_PREP(PP_SEQUENCE_STATE_MASK, 0x9)
+#define   PP_SEQUENCE_STATE_ON_S1_2	REG_FIELD_PREP(PP_SEQUENCE_STATE_MASK, 0xa)
+#define   PP_SEQUENCE_STATE_ON_S1_3	REG_FIELD_PREP(PP_SEQUENCE_STATE_MASK, 0xb)
+#define   PP_SEQUENCE_STATE_RESET	REG_FIELD_PREP(PP_SEQUENCE_STATE_MASK, 0xf)
 
 #define _PP_CONTROL			0x61204
 #define PP_CONTROL(pps_idx)		_MMIO_PPS(pps_idx, _PP_CONTROL)
-#define  PANEL_UNLOCK_REGS		(0xabcd << 16)
-#define  PANEL_UNLOCK_MASK		(0xffff << 16)
-#define  BXT_POWER_CYCLE_DELAY_MASK	0x1f0
-#define  BXT_POWER_CYCLE_DELAY_SHIFT	4
-#define  EDP_FORCE_VDD			(1 << 3)
-#define  EDP_BLC_ENABLE			(1 << 2)
-#define  PANEL_POWER_RESET		(1 << 1)
-#define  PANEL_POWER_ON			(1 << 0)
+#define  PANEL_UNLOCK_MASK		REG_GENMASK(31, 16)
+#define  PANEL_UNLOCK_REGS		REG_FIELD_PREP(PANEL_UNLOCK_MASK, 0xabcd)
+#define  BXT_POWER_CYCLE_DELAY_MASK	REG_GENMASK(8, 4)
+#define  EDP_FORCE_VDD			REG_BIT(3)
+#define  EDP_BLC_ENABLE			REG_BIT(2)
+#define  PANEL_POWER_RESET		REG_BIT(1)
+#define  PANEL_POWER_ON			REG_BIT(0)
 
 #define _PP_ON_DELAYS			0x61208
 #define PP_ON_DELAYS(pps_idx)		_MMIO_PPS(pps_idx, _PP_ON_DELAYS)
-#define  PANEL_PORT_SELECT_SHIFT	30
-#define  PANEL_PORT_SELECT_MASK		(3 << 30)
-#define  PANEL_PORT_SELECT_LVDS		(0 << 30)
-#define  PANEL_PORT_SELECT_DPA		(1 << 30)
-#define  PANEL_PORT_SELECT_DPC		(2 << 30)
-#define  PANEL_PORT_SELECT_DPD		(3 << 30)
-#define  PANEL_PORT_SELECT_VLV(port)	((port) << 30)
-#define  PANEL_POWER_UP_DELAY_MASK	0x1fff0000
-#define  PANEL_POWER_UP_DELAY_SHIFT	16
-#define  PANEL_LIGHT_ON_DELAY_MASK	0x1fff
-#define  PANEL_LIGHT_ON_DELAY_SHIFT	0
+#define  PANEL_PORT_SELECT_MASK		REG_GENMASK(31, 30)
+#define  PANEL_PORT_SELECT_LVDS		REG_FIELD_PREP(PANEL_PORT_SELECT_MASK, 0)
+#define  PANEL_PORT_SELECT_DPA		REG_FIELD_PREP(PANEL_PORT_SELECT_MASK, 1)
+#define  PANEL_PORT_SELECT_DPC		REG_FIELD_PREP(PANEL_PORT_SELECT_MASK, 2)
+#define  PANEL_PORT_SELECT_DPD		REG_FIELD_PREP(PANEL_PORT_SELECT_MASK, 3)
+#define  PANEL_PORT_SELECT_VLV(port)	REG_FIELD_PREP(PANEL_PORT_SELECT_MASK, port)
+#define  PANEL_POWER_UP_DELAY_MASK	REG_GENMASK(28, 16)
+#define  PANEL_LIGHT_ON_DELAY_MASK	REG_GENMASK(12, 0)
 
 #define _PP_OFF_DELAYS			0x6120C
 #define PP_OFF_DELAYS(pps_idx)		_MMIO_PPS(pps_idx, _PP_OFF_DELAYS)
-#define  PANEL_POWER_DOWN_DELAY_MASK	0x1fff0000
-#define  PANEL_POWER_DOWN_DELAY_SHIFT	16
-#define  PANEL_LIGHT_OFF_DELAY_MASK	0x1fff
-#define  PANEL_LIGHT_OFF_DELAY_SHIFT	0
+#define  PANEL_POWER_DOWN_DELAY_MASK	REG_GENMASK(28, 16)
+#define  PANEL_LIGHT_OFF_DELAY_MASK	REG_GENMASK(12, 0)
 
 #define _PP_DIVISOR			0x61210
 #define PP_DIVISOR(pps_idx)		_MMIO_PPS(pps_idx, _PP_DIVISOR)
-#define  PP_REFERENCE_DIVIDER_MASK	0xffffff00
-#define  PP_REFERENCE_DIVIDER_SHIFT	8
-#define  PANEL_POWER_CYCLE_DELAY_MASK	0x1f
-#define  PANEL_POWER_CYCLE_DELAY_SHIFT	0
+#define  PP_REFERENCE_DIVIDER_MASK	REG_GENMASK(31, 8)
+#define  PANEL_POWER_CYCLE_DELAY_MASK	REG_GENMASK(4, 0)
 
 /* Panel fitting */
 #define PFIT_CONTROL	_MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x61230)
@@ -5590,9 +5647,15 @@ enum {
 #define   PIPECONF_SINGLE_WIDE	0
 #define   PIPECONF_PIPE_UNLOCKED 0
 #define   PIPECONF_PIPE_LOCKED	(1 << 25)
-#define   PIPECONF_PALETTE	0
-#define   PIPECONF_GAMMA		(1 << 24)
 #define   PIPECONF_FORCE_BORDER	(1 << 25)
+#define   PIPECONF_GAMMA_MODE_MASK_I9XX	(1 << 24) /* gmch */
+#define   PIPECONF_GAMMA_MODE_MASK_ILK	(3 << 24) /* ilk-ivb */
+#define   PIPECONF_GAMMA_MODE_8BIT	(0 << 24) /* gmch,ilk-ivb */
+#define   PIPECONF_GAMMA_MODE_10BIT	(1 << 24) /* gmch,ilk-ivb */
+#define   PIPECONF_GAMMA_MODE_12BIT	(2 << 24) /* ilk-ivb */
+#define   PIPECONF_GAMMA_MODE_SPLIT	(3 << 24) /* ivb */
+#define   PIPECONF_GAMMA_MODE(x)	((x) << 24) /* pass in GAMMA_MODE_MODE_* */
+#define   PIPECONF_GAMMA_MODE_SHIFT	24
 #define   PIPECONF_INTERLACE_MASK	(7 << 21)
 #define   PIPECONF_INTERLACE_MASK_HSW	(3 << 21)
 /* Note that pre-gen3 does not support interlaced display directly. Panel
@@ -5698,6 +5761,10 @@ enum {
 #define PIPEFRAMEPIXEL(pipe)	_MMIO_PIPE2(pipe, _PIPEAFRAMEPIXEL)
 #define PIPESTAT(pipe)		_MMIO_PIPE2(pipe, _PIPEASTAT)
 
+#define  _PIPEAGCMAX           0x70010
+#define  _PIPEBGCMAX           0x71010
+#define PIPEGCMAX(pipe, i)     _MMIO_PIPE2(pipe, _PIPEAGCMAX + (i) * 4)
+
 #define _PIPE_MISC_A			0x70030
 #define _PIPE_MISC_B			0x71030
 #define   PIPEMISC_YUV420_ENABLE	(1 << 27)
@@ -5998,6 +6065,7 @@ enum {
 #define _CUR_WM_TRANS_A_0	0x70168
 #define _CUR_WM_TRANS_B_0	0x71168
 #define   PLANE_WM_EN		(1 << 31)
+#define   PLANE_WM_IGNORE_LINES	(1 << 30)
 #define   PLANE_WM_LINES_SHIFT	14
 #define   PLANE_WM_LINES_MASK	0x1f
 #define   PLANE_WM_BLOCKS_MASK	0x7ff /* skl+: 10 bits, icl+ 11 bits */
@@ -6124,7 +6192,7 @@ enum {
 #define   MCURSOR_PIPE_SELECT_SHIFT	28
 #define   MCURSOR_PIPE_SELECT(pipe)	((pipe) << 28)
 #define   MCURSOR_GAMMA_ENABLE  (1 << 26)
-#define   MCURSOR_PIPE_CSC_ENABLE (1 << 24)
+#define   MCURSOR_PIPE_CSC_ENABLE (1 << 24) /* ilk+ */
 #define   MCURSOR_ROTATE_180	(1 << 15)
 #define   MCURSOR_TRICKLE_FEED_DISABLE	(1 << 14)
 #define _CURABASE		0x70084
@@ -6179,7 +6247,7 @@ enum {
 #define   DISPPLANE_RGBA888			(0xf << 26)
 #define   DISPPLANE_STEREO_ENABLE		(1 << 25)
 #define   DISPPLANE_STEREO_DISABLE		0
-#define   DISPPLANE_PIPE_CSC_ENABLE		(1 << 24)
+#define   DISPPLANE_PIPE_CSC_ENABLE		(1 << 24) /* ilk+ */
 #define   DISPPLANE_SEL_PIPE_SHIFT		24
 #define   DISPPLANE_SEL_PIPE_MASK		(3 << DISPPLANE_SEL_PIPE_SHIFT)
 #define   DISPPLANE_SEL_PIPE(pipe)		((pipe) << DISPPLANE_SEL_PIPE_SHIFT)
@@ -6557,13 +6625,22 @@ enum {
 #define   PLANE_CTL_FORMAT_YUV422		(0 << 24)
 #define   PLANE_CTL_FORMAT_NV12			(1 << 24)
 #define   PLANE_CTL_FORMAT_XRGB_2101010		(2 << 24)
+#define   PLANE_CTL_FORMAT_P010			(3 << 24)
 #define   PLANE_CTL_FORMAT_XRGB_8888		(4 << 24)
+#define   PLANE_CTL_FORMAT_P012			(5 << 24)
 #define   PLANE_CTL_FORMAT_XRGB_16161616F	(6 << 24)
+#define   PLANE_CTL_FORMAT_P016			(7 << 24)
 #define   PLANE_CTL_FORMAT_AYUV			(8 << 24)
 #define   PLANE_CTL_FORMAT_INDEXED		(12 << 24)
 #define   PLANE_CTL_FORMAT_RGB_565		(14 << 24)
 #define   ICL_PLANE_CTL_FORMAT_MASK		(0x1f << 23)
 #define   PLANE_CTL_PIPE_CSC_ENABLE		(1 << 23) /* Pre-GLK */
+#define   PLANE_CTL_FORMAT_Y210                 (1 << 23)
+#define   PLANE_CTL_FORMAT_Y212                 (3 << 23)
+#define   PLANE_CTL_FORMAT_Y216                 (5 << 23)
+#define   PLANE_CTL_FORMAT_Y410                 (7 << 23)
+#define   PLANE_CTL_FORMAT_Y412                 (9 << 23)
+#define   PLANE_CTL_FORMAT_Y416                 (0xb << 23)
 #define   PLANE_CTL_KEY_ENABLE_MASK		(0x3 << 21)
 #define   PLANE_CTL_KEY_ENABLE_SOURCE		(1 << 21)
 #define   PLANE_CTL_KEY_ENABLE_DESTINATION	(2 << 21)
@@ -7102,14 +7179,25 @@ enum {
 #define _LGC_PALETTE_B           0x4a800
 #define LGC_PALETTE(pipe, i) _MMIO(_PIPE(pipe, _LGC_PALETTE_A, _LGC_PALETTE_B) + (i) * 4)
 
+/* ilk/snb precision palette */
+#define _PREC_PALETTE_A           0x4b000
+#define _PREC_PALETTE_B           0x4c000
+#define PREC_PALETTE(pipe, i) _MMIO(_PIPE(pipe, _PREC_PALETTE_A, _PREC_PALETTE_B) + (i) * 4)
+
+#define  _PREC_PIPEAGCMAX              0x4d000
+#define  _PREC_PIPEBGCMAX              0x4d010
+#define PREC_PIPEGCMAX(pipe, i)        _MMIO(_PIPE(pipe, _PIPEAGCMAX, _PIPEBGCMAX) + (i) * 4)
+
 #define _GAMMA_MODE_A		0x4a480
 #define _GAMMA_MODE_B		0x4ac80
 #define GAMMA_MODE(pipe) _MMIO_PIPE(pipe, _GAMMA_MODE_A, _GAMMA_MODE_B)
-#define GAMMA_MODE_MODE_MASK	(3 << 0)
-#define GAMMA_MODE_MODE_8BIT	(0 << 0)
-#define GAMMA_MODE_MODE_10BIT	(1 << 0)
-#define GAMMA_MODE_MODE_12BIT	(2 << 0)
-#define GAMMA_MODE_MODE_SPLIT	(3 << 0)
+#define  PRE_CSC_GAMMA_ENABLE	(1 << 31)
+#define  POST_CSC_GAMMA_ENABLE	(1 << 30)
+#define  GAMMA_MODE_MODE_MASK	(3 << 0)
+#define  GAMMA_MODE_MODE_8BIT	(0 << 0)
+#define  GAMMA_MODE_MODE_10BIT	(1 << 0)
+#define  GAMMA_MODE_MODE_12BIT	(2 << 0)
+#define  GAMMA_MODE_MODE_SPLIT	(3 << 0)
 
 /* DMC/CSR */
 #define CSR_PROGRAM(i)		_MMIO(0x80000 + (i) * 4)
@@ -7204,8 +7292,8 @@ enum {
 #define  GEN8_GT_VECS_IRQ		(1 << 6)
 #define  GEN8_GT_GUC_IRQ		(1 << 5)
 #define  GEN8_GT_PM_IRQ			(1 << 4)
-#define  GEN8_GT_VCS2_IRQ		(1 << 3)
-#define  GEN8_GT_VCS1_IRQ		(1 << 2)
+#define  GEN8_GT_VCS1_IRQ		(1 << 3) /* NB: VCS2 in bspec! */
+#define  GEN8_GT_VCS0_IRQ		(1 << 2) /* NB: VCS1 in bpsec! */
 #define  GEN8_GT_BCS_IRQ		(1 << 1)
 #define  GEN8_GT_RCS_IRQ		(1 << 0)
 
@@ -7226,8 +7314,8 @@ enum {
 
 #define GEN8_RCS_IRQ_SHIFT 0
 #define GEN8_BCS_IRQ_SHIFT 16
-#define GEN8_VCS1_IRQ_SHIFT 0
-#define GEN8_VCS2_IRQ_SHIFT 16
+#define GEN8_VCS0_IRQ_SHIFT 0  /* NB: VCS1 in bspec! */
+#define GEN8_VCS1_IRQ_SHIFT 16 /* NB: VCS2 in bpsec! */
 #define GEN8_VECS_IRQ_SHIFT 0
 #define GEN8_WD_IRQ_SHIFT 16
 
@@ -7613,13 +7701,13 @@ enum {
 #define  GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE	(1 << 2)
 
 /*GEN11 chicken */
-#define _PIPEA_CHICKEN			0x70038
-#define _PIPEB_CHICKEN			0x71038
-#define _PIPEC_CHICKEN			0x72038
-#define  PER_PIXEL_ALPHA_BYPASS_EN	(1 << 7)
-#define  PM_FILL_MAINTAIN_DBUF_FULLNESS	(1 << 0)
-#define PIPE_CHICKEN(pipe)		_MMIO_PIPE(pipe, _PIPEA_CHICKEN,\
-						   _PIPEB_CHICKEN)
+#define _PIPEA_CHICKEN				0x70038
+#define _PIPEB_CHICKEN				0x71038
+#define _PIPEC_CHICKEN				0x72038
+#define PIPE_CHICKEN(pipe)			_MMIO_PIPE(pipe, _PIPEA_CHICKEN,\
+							   _PIPEB_CHICKEN)
+#define   PIXEL_ROUNDING_TRUNC_FB_PASSTHRU 	(1 << 15)
+#define   PER_PIXEL_ALPHA_BYPASS_EN		(1 << 7)
 
 /* PCH */
 
@@ -8089,10 +8177,11 @@ enum {
 #define _ICL_VIDEO_DIP_PPS_ECC_B	0x613D4
 
 #define HSW_TVIDEO_DIP_CTL(trans)		_MMIO_TRANS2(trans, _HSW_VIDEO_DIP_CTL_A)
+#define HSW_TVIDEO_DIP_GCP(trans)		_MMIO_TRANS2(trans, _HSW_VIDEO_DIP_GCP_A)
 #define HSW_TVIDEO_DIP_AVI_DATA(trans, i)	_MMIO_TRANS2(trans, _HSW_VIDEO_DIP_AVI_DATA_A + (i) * 4)
 #define HSW_TVIDEO_DIP_VS_DATA(trans, i)	_MMIO_TRANS2(trans, _HSW_VIDEO_DIP_VS_DATA_A + (i) * 4)
 #define HSW_TVIDEO_DIP_SPD_DATA(trans, i)	_MMIO_TRANS2(trans, _HSW_VIDEO_DIP_SPD_DATA_A + (i) * 4)
-#define HSW_TVIDEO_DIP_GCP(trans)		_MMIO_TRANS2(trans, _HSW_VIDEO_DIP_GCP_A)
+#define HSW_TVIDEO_DIP_GMP_DATA(trans, i)	_MMIO_TRANS2(trans, _HSW_VIDEO_DIP_GMP_DATA_A + (i) * 4)
 #define HSW_TVIDEO_DIP_VSC_DATA(trans, i)	_MMIO_TRANS2(trans, _HSW_VIDEO_DIP_VSC_DATA_A + (i) * 4)
 #define ICL_VIDEO_DIP_PPS_DATA(trans, i)	_MMIO_TRANS2(trans, _ICL_VIDEO_DIP_PPS_DATA_A + (i) * 4)
 #define ICL_VIDEO_DIP_PPS_ECC(trans, i)		_MMIO_TRANS2(trans, _ICL_VIDEO_DIP_PPS_ECC_A + (i) * 4)
@@ -8600,8 +8689,9 @@ enum {
 #define GEN9_MEDIA_PG_IDLE_HYSTERESIS		_MMIO(0xA0C4)
 #define GEN9_RENDER_PG_IDLE_HYSTERESIS		_MMIO(0xA0C8)
 #define GEN9_PG_ENABLE				_MMIO(0xA210)
-#define GEN9_RENDER_PG_ENABLE			(1 << 0)
-#define GEN9_MEDIA_PG_ENABLE			(1 << 1)
+#define GEN9_RENDER_PG_ENABLE			REG_BIT(0)
+#define GEN9_MEDIA_PG_ENABLE			REG_BIT(1)
+#define GEN11_MEDIA_SAMPLER_PG_ENABLE		REG_BIT(2)
 #define GEN8_PUSHBUS_CONTROL			_MMIO(0xA248)
 #define GEN8_PUSHBUS_ENABLE			_MMIO(0xA250)
 #define GEN8_PUSHBUS_SHIFT			_MMIO(0xA25C)
@@ -8616,6 +8706,11 @@ enum {
 #define GEN6_PMIER				_MMIO(0x4402C)
 #define  GEN6_PM_MBOX_EVENT			(1 << 25)
 #define  GEN6_PM_THERMAL_EVENT			(1 << 24)
+
+/*
+ * For Gen11 these are in the upper word of the GPM_WGBOXPERF
+ * registers. Shifting is handled on accessing the imr and ier.
+ */
 #define  GEN6_PM_RP_DOWN_TIMEOUT		(1 << 6)
 #define  GEN6_PM_RP_UP_THRESHOLD		(1 << 5)
 #define  GEN6_PM_RP_DOWN_THRESHOLD		(1 << 4)
@@ -9741,7 +9836,7 @@ enum skl_power_gate {
 #define  DPLL_CFGCR1_KDIV(x)		((x) << 6)
 #define  DPLL_CFGCR1_KDIV_1		(1 << 6)
 #define  DPLL_CFGCR1_KDIV_2		(2 << 6)
-#define  DPLL_CFGCR1_KDIV_4		(4 << 6)
+#define  DPLL_CFGCR1_KDIV_3		(4 << 6)
 #define  DPLL_CFGCR1_PDIV_MASK		(0xf << 2)
 #define  DPLL_CFGCR1_PDIV_SHIFT		(2)
 #define  DPLL_CFGCR1_PDIV(x)		((x) << 2)
@@ -9810,16 +9905,29 @@ enum skl_power_gate {
 #define  BXT_DRAM_WIDTH_X64			(0x3 << 4)
 #define  BXT_DRAM_SIZE_MASK			(0x7 << 6)
 #define  BXT_DRAM_SIZE_SHIFT			6
-#define  BXT_DRAM_SIZE_4GB			(0x0 << 6)
-#define  BXT_DRAM_SIZE_6GB			(0x1 << 6)
-#define  BXT_DRAM_SIZE_8GB			(0x2 << 6)
-#define  BXT_DRAM_SIZE_12GB			(0x3 << 6)
-#define  BXT_DRAM_SIZE_16GB			(0x4 << 6)
+#define  BXT_DRAM_SIZE_4GBIT			(0x0 << 6)
+#define  BXT_DRAM_SIZE_6GBIT			(0x1 << 6)
+#define  BXT_DRAM_SIZE_8GBIT			(0x2 << 6)
+#define  BXT_DRAM_SIZE_12GBIT			(0x3 << 6)
+#define  BXT_DRAM_SIZE_16GBIT			(0x4 << 6)
+#define  BXT_DRAM_TYPE_MASK			(0x7 << 22)
+#define  BXT_DRAM_TYPE_SHIFT			22
+#define  BXT_DRAM_TYPE_DDR3			(0x0 << 22)
+#define  BXT_DRAM_TYPE_LPDDR3			(0x1 << 22)
+#define  BXT_DRAM_TYPE_LPDDR4			(0x2 << 22)
+#define  BXT_DRAM_TYPE_DDR4			(0x4 << 22)
 
 #define SKL_MEMORY_FREQ_MULTIPLIER_HZ		266666666
 #define SKL_MC_BIOS_DATA_0_0_0_MCHBAR_PCU	_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5E04)
 #define  SKL_REQ_DATA_MASK			(0xF << 0)
 
+#define SKL_MAD_INTER_CHANNEL_0_0_0_MCHBAR_MCMAIN _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5000)
+#define  SKL_DRAM_DDR_TYPE_MASK			(0x3 << 0)
+#define  SKL_DRAM_DDR_TYPE_DDR4			(0 << 0)
+#define  SKL_DRAM_DDR_TYPE_DDR3			(1 << 0)
+#define  SKL_DRAM_DDR_TYPE_LPDDR3		(2 << 0)
+#define  SKL_DRAM_DDR_TYPE_LPDDR4		(3 << 0)
+
 #define SKL_MAD_DIMM_CH0_0_0_0_MCHBAR_MCMAIN	_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x500C)
 #define SKL_MAD_DIMM_CH1_0_0_0_MCHBAR_MCMAIN	_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5010)
 #define  SKL_DRAM_S_SHIFT			16
@@ -9831,8 +9939,21 @@ enum skl_power_gate {
 #define  SKL_DRAM_WIDTH_X32			(0x2 << 8)
 #define  SKL_DRAM_RANK_MASK			(0x1 << 10)
 #define  SKL_DRAM_RANK_SHIFT			10
-#define  SKL_DRAM_RANK_SINGLE			(0x0 << 10)
-#define  SKL_DRAM_RANK_DUAL			(0x1 << 10)
+#define  SKL_DRAM_RANK_1			(0x0 << 10)
+#define  SKL_DRAM_RANK_2			(0x1 << 10)
+#define  SKL_DRAM_RANK_MASK			(0x1 << 10)
+#define  CNL_DRAM_SIZE_MASK			0x7F
+#define  CNL_DRAM_WIDTH_MASK			(0x3 << 7)
+#define  CNL_DRAM_WIDTH_SHIFT			7
+#define  CNL_DRAM_WIDTH_X8			(0x0 << 7)
+#define  CNL_DRAM_WIDTH_X16			(0x1 << 7)
+#define  CNL_DRAM_WIDTH_X32			(0x2 << 7)
+#define  CNL_DRAM_RANK_MASK			(0x3 << 9)
+#define  CNL_DRAM_RANK_SHIFT			9
+#define  CNL_DRAM_RANK_1			(0x0 << 9)
+#define  CNL_DRAM_RANK_2			(0x1 << 9)
+#define  CNL_DRAM_RANK_3			(0x2 << 9)
+#define  CNL_DRAM_RANK_4			(0x3 << 9)
 
 /* Please see hsw_read_dcomp() and hsw_write_dcomp() before using this register,
  * since on HSW we can't write to it using I915_WRITE. */
@@ -9877,10 +9998,14 @@ enum skl_power_gate {
 #define _PIPE_A_CSC_COEFF_BU	0x4901c
 #define _PIPE_A_CSC_COEFF_RV_GV	0x49020
 #define _PIPE_A_CSC_COEFF_BV	0x49024
+
 #define _PIPE_A_CSC_MODE	0x49028
-#define   CSC_BLACK_SCREEN_OFFSET	(1 << 2)
-#define   CSC_POSITION_BEFORE_GAMMA	(1 << 1)
-#define   CSC_MODE_YUV_TO_RGB		(1 << 0)
+#define  ICL_CSC_ENABLE			(1 << 31)
+#define  ICL_OUTPUT_CSC_ENABLE		(1 << 30)
+#define  CSC_BLACK_SCREEN_OFFSET	(1 << 2)
+#define  CSC_POSITION_BEFORE_GAMMA	(1 << 1)
+#define  CSC_MODE_YUV_TO_RGB		(1 << 0)
+
 #define _PIPE_A_CSC_PREOFF_HI	0x49030
 #define _PIPE_A_CSC_PREOFF_ME	0x49034
 #define _PIPE_A_CSC_PREOFF_LO	0x49038
@@ -9916,6 +10041,70 @@ enum skl_power_gate {
 #define PIPE_CSC_POSTOFF_ME(pipe)	_MMIO_PIPE(pipe, _PIPE_A_CSC_POSTOFF_ME, _PIPE_B_CSC_POSTOFF_ME)
 #define PIPE_CSC_POSTOFF_LO(pipe)	_MMIO_PIPE(pipe, _PIPE_A_CSC_POSTOFF_LO, _PIPE_B_CSC_POSTOFF_LO)
 
+/* Pipe Output CSC */
+#define _PIPE_A_OUTPUT_CSC_COEFF_RY_GY	0x49050
+#define _PIPE_A_OUTPUT_CSC_COEFF_BY	0x49054
+#define _PIPE_A_OUTPUT_CSC_COEFF_RU_GU	0x49058
+#define _PIPE_A_OUTPUT_CSC_COEFF_BU	0x4905c
+#define _PIPE_A_OUTPUT_CSC_COEFF_RV_GV	0x49060
+#define _PIPE_A_OUTPUT_CSC_COEFF_BV	0x49064
+#define _PIPE_A_OUTPUT_CSC_PREOFF_HI	0x49068
+#define _PIPE_A_OUTPUT_CSC_PREOFF_ME	0x4906c
+#define _PIPE_A_OUTPUT_CSC_PREOFF_LO	0x49070
+#define _PIPE_A_OUTPUT_CSC_POSTOFF_HI	0x49074
+#define _PIPE_A_OUTPUT_CSC_POSTOFF_ME	0x49078
+#define _PIPE_A_OUTPUT_CSC_POSTOFF_LO	0x4907c
+
+#define _PIPE_B_OUTPUT_CSC_COEFF_RY_GY	0x49150
+#define _PIPE_B_OUTPUT_CSC_COEFF_BY	0x49154
+#define _PIPE_B_OUTPUT_CSC_COEFF_RU_GU	0x49158
+#define _PIPE_B_OUTPUT_CSC_COEFF_BU	0x4915c
+#define _PIPE_B_OUTPUT_CSC_COEFF_RV_GV	0x49160
+#define _PIPE_B_OUTPUT_CSC_COEFF_BV	0x49164
+#define _PIPE_B_OUTPUT_CSC_PREOFF_HI	0x49168
+#define _PIPE_B_OUTPUT_CSC_PREOFF_ME	0x4916c
+#define _PIPE_B_OUTPUT_CSC_PREOFF_LO	0x49170
+#define _PIPE_B_OUTPUT_CSC_POSTOFF_HI	0x49174
+#define _PIPE_B_OUTPUT_CSC_POSTOFF_ME	0x49178
+#define _PIPE_B_OUTPUT_CSC_POSTOFF_LO	0x4917c
+
+#define PIPE_CSC_OUTPUT_COEFF_RY_GY(pipe)	_MMIO_PIPE(pipe,\
+							   _PIPE_A_OUTPUT_CSC_COEFF_RY_GY,\
+							   _PIPE_B_OUTPUT_CSC_COEFF_RY_GY)
+#define PIPE_CSC_OUTPUT_COEFF_BY(pipe)		_MMIO_PIPE(pipe, \
+							   _PIPE_A_OUTPUT_CSC_COEFF_BY, \
+							   _PIPE_B_OUTPUT_CSC_COEFF_BY)
+#define PIPE_CSC_OUTPUT_COEFF_RU_GU(pipe)	_MMIO_PIPE(pipe, \
+							   _PIPE_A_OUTPUT_CSC_COEFF_RU_GU, \
+							   _PIPE_B_OUTPUT_CSC_COEFF_RU_GU)
+#define PIPE_CSC_OUTPUT_COEFF_BU(pipe)		_MMIO_PIPE(pipe, \
+							   _PIPE_A_OUTPUT_CSC_COEFF_BU, \
+							   _PIPE_B_OUTPUT_CSC_COEFF_BU)
+#define PIPE_CSC_OUTPUT_COEFF_RV_GV(pipe)	_MMIO_PIPE(pipe, \
+							   _PIPE_A_OUTPUT_CSC_COEFF_RV_GV, \
+							   _PIPE_B_OUTPUT_CSC_COEFF_RV_GV)
+#define PIPE_CSC_OUTPUT_COEFF_BV(pipe)		_MMIO_PIPE(pipe, \
+							   _PIPE_A_OUTPUT_CSC_COEFF_BV, \
+							   _PIPE_B_OUTPUT_CSC_COEFF_BV)
+#define PIPE_CSC_OUTPUT_PREOFF_HI(pipe)		_MMIO_PIPE(pipe, \
+							   _PIPE_A_OUTPUT_CSC_PREOFF_HI, \
+							   _PIPE_B_OUTPUT_CSC_PREOFF_HI)
+#define PIPE_CSC_OUTPUT_PREOFF_ME(pipe)		_MMIO_PIPE(pipe, \
+							   _PIPE_A_OUTPUT_CSC_PREOFF_ME, \
+							   _PIPE_B_OUTPUT_CSC_PREOFF_ME)
+#define PIPE_CSC_OUTPUT_PREOFF_LO(pipe)		_MMIO_PIPE(pipe, \
+							   _PIPE_A_OUTPUT_CSC_PREOFF_LO, \
+							   _PIPE_B_OUTPUT_CSC_PREOFF_LO)
+#define PIPE_CSC_OUTPUT_POSTOFF_HI(pipe)	_MMIO_PIPE(pipe, \
+							   _PIPE_A_OUTPUT_CSC_POSTOFF_HI, \
+							   _PIPE_B_OUTPUT_CSC_POSTOFF_HI)
+#define PIPE_CSC_OUTPUT_POSTOFF_ME(pipe)	_MMIO_PIPE(pipe, \
+							   _PIPE_A_OUTPUT_CSC_POSTOFF_ME, \
+							   _PIPE_B_OUTPUT_CSC_POSTOFF_ME)
+#define PIPE_CSC_OUTPUT_POSTOFF_LO(pipe)	_MMIO_PIPE(pipe, \
+							   _PIPE_A_OUTPUT_CSC_POSTOFF_LO, \
+							   _PIPE_B_OUTPUT_CSC_POSTOFF_LO)
+
 /* pipe degamma/gamma LUTs on IVB+ */
 #define _PAL_PREC_INDEX_A	0x4A400
 #define _PAL_PREC_INDEX_B	0x4AC00
@@ -9924,6 +10113,7 @@ enum skl_power_gate {
 #define   PAL_PREC_SPLIT_MODE		(1 << 31)
 #define   PAL_PREC_AUTO_INCREMENT	(1 << 15)
 #define   PAL_PREC_INDEX_VALUE_MASK	(0x3ff << 0)
+#define   PAL_PREC_INDEX_VALUE(x)	((x) << 0)
 #define _PAL_PREC_DATA_A	0x4A404
 #define _PAL_PREC_DATA_B	0x4AC04
 #define _PAL_PREC_DATA_C	0x4B404
@@ -9941,6 +10131,7 @@ enum skl_power_gate {
 #define PREC_PAL_DATA(pipe)		_MMIO_PIPE(pipe, _PAL_PREC_DATA_A, _PAL_PREC_DATA_B)
 #define PREC_PAL_GC_MAX(pipe, i)	_MMIO(_PIPE(pipe, _PAL_PREC_GC_MAX_A, _PAL_PREC_GC_MAX_B) + (i) * 4)
 #define PREC_PAL_EXT_GC_MAX(pipe, i)	_MMIO(_PIPE(pipe, _PAL_PREC_EXT_GC_MAX_A, _PAL_PREC_EXT_GC_MAX_B) + (i) * 4)
+#define PREC_PAL_EXT2_GC_MAX(pipe, i)	_MMIO(_PIPE(pipe, _PAL_PREC_EXT2_GC_MAX_A, _PAL_PREC_EXT2_GC_MAX_B) + (i) * 4)
 
 #define _PRE_CSC_GAMC_INDEX_A	0x4A484
 #define _PRE_CSC_GAMC_INDEX_B	0x4AC84
diff --git a/drivers/gpu/drm/i915/i915_request.c b/drivers/gpu/drm/i915/i915_request.c
index c2a5c48c7541..b836721d3b13 100644
--- a/drivers/gpu/drm/i915/i915_request.c
+++ b/drivers/gpu/drm/i915/i915_request.c
@@ -22,15 +22,31 @@
  *
  */
 
-#include <linux/prefetch.h>
 #include <linux/dma-fence-array.h>
+#include <linux/irq_work.h>
+#include <linux/prefetch.h>
 #include <linux/sched.h>
 #include <linux/sched/clock.h>
 #include <linux/sched/signal.h>
 
-#include "i915_drv.h"
 #include "i915_active.h"
+#include "i915_drv.h"
+#include "i915_globals.h"
 #include "i915_reset.h"
+#include "intel_pm.h"
+
+struct execute_cb {
+	struct list_head link;
+	struct irq_work work;
+	struct i915_sw_fence *fence;
+};
+
+static struct i915_global_request {
+	struct i915_global base;
+	struct kmem_cache *slab_requests;
+	struct kmem_cache *slab_dependencies;
+	struct kmem_cache *slab_execute_cbs;
+} global;
 
 static const char *i915_fence_get_driver_name(struct dma_fence *fence)
 {
@@ -51,7 +67,7 @@ static const char *i915_fence_get_timeline_name(struct dma_fence *fence)
 	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
 		return "signaled";
 
-	return to_request(fence)->timeline->name;
+	return to_request(fence)->gem_context->name ?: "[i915]";
 }
 
 static bool i915_fence_signaled(struct dma_fence *fence)
@@ -68,7 +84,9 @@ static signed long i915_fence_wait(struct dma_fence *fence,
 				   bool interruptible,
 				   signed long timeout)
 {
-	return i915_request_wait(to_request(fence), interruptible, timeout);
+	return i915_request_wait(to_request(fence),
+				 interruptible | I915_WAIT_PRIORITY,
+				 timeout);
 }
 
 static void i915_fence_release(struct dma_fence *fence)
@@ -83,8 +101,9 @@ static void i915_fence_release(struct dma_fence *fence)
 	 * caught trying to reuse dead objects.
 	 */
 	i915_sw_fence_fini(&rq->submit);
+	i915_sw_fence_fini(&rq->semaphore);
 
-	kmem_cache_free(rq->i915->requests, rq);
+	kmem_cache_free(global.slab_requests, rq);
 }
 
 const struct dma_fence_ops i915_fence_ops = {
@@ -150,7 +169,6 @@ static void advance_ring(struct i915_request *request)
 		 * is just about to be. Either works, if we miss the last two
 		 * noops - they are safe to be replayed on a reset.
 		 */
-		GEM_TRACE("marking %s as inactive\n", ring->timeline->name);
 		tail = READ_ONCE(request->tail);
 		list_del(&ring->active_link);
 	} else {
@@ -177,12 +195,10 @@ static void free_capture_list(struct i915_request *request)
 static void __retire_engine_request(struct intel_engine_cs *engine,
 				    struct i915_request *rq)
 {
-	GEM_TRACE("%s(%s) fence %llx:%lld, global=%d, current %d:%d\n",
+	GEM_TRACE("%s(%s) fence %llx:%lld, current %d\n",
 		  __func__, engine->name,
 		  rq->fence.context, rq->fence.seqno,
-		  rq->global_seqno,
-		  hwsp_seqno(rq),
-		  intel_engine_get_seqno(engine));
+		  hwsp_seqno(rq));
 
 	GEM_BUG_ON(!i915_request_completed(rq));
 
@@ -241,12 +257,10 @@ static void i915_request_retire(struct i915_request *request)
 {
 	struct i915_active_request *active, *next;
 
-	GEM_TRACE("%s fence %llx:%lld, global=%d, current %d:%d\n",
+	GEM_TRACE("%s fence %llx:%lld, current %d\n",
 		  request->engine->name,
 		  request->fence.context, request->fence.seqno,
-		  request->global_seqno,
-		  hwsp_seqno(request),
-		  intel_engine_get_seqno(request->engine));
+		  hwsp_seqno(request));
 
 	lockdep_assert_held(&request->i915->drm.struct_mutex);
 	GEM_BUG_ON(!i915_sw_fence_signaled(&request->submit));
@@ -288,15 +302,13 @@ static void i915_request_retire(struct i915_request *request)
 
 	i915_request_remove_from_client(request);
 
-	/* Retirement decays the ban score as it is a sign of ctx progress */
-	atomic_dec_if_positive(&request->gem_context->ban_score);
 	intel_context_unpin(request->hw_context);
 
 	__retire_engine_upto(request->engine, request);
 
 	unreserve_gt(request->i915);
 
-	i915_sched_node_fini(request->i915, &request->sched);
+	i915_sched_node_fini(&request->sched);
 	i915_request_put(request);
 }
 
@@ -305,12 +317,10 @@ void i915_request_retire_upto(struct i915_request *rq)
 	struct intel_ring *ring = rq->ring;
 	struct i915_request *tmp;
 
-	GEM_TRACE("%s fence %llx:%lld, global=%d, current %d:%d\n",
+	GEM_TRACE("%s fence %llx:%lld, current %d\n",
 		  rq->engine->name,
 		  rq->fence.context, rq->fence.seqno,
-		  rq->global_seqno,
-		  hwsp_seqno(rq),
-		  intel_engine_get_seqno(rq->engine));
+		  hwsp_seqno(rq));
 
 	lockdep_assert_held(&rq->i915->drm.struct_mutex);
 	GEM_BUG_ON(!i915_request_completed(rq));
@@ -326,9 +336,67 @@ void i915_request_retire_upto(struct i915_request *rq)
 	} while (tmp != rq);
 }
 
-static u32 timeline_get_seqno(struct i915_timeline *tl)
+static void irq_execute_cb(struct irq_work *wrk)
+{
+	struct execute_cb *cb = container_of(wrk, typeof(*cb), work);
+
+	i915_sw_fence_complete(cb->fence);
+	kmem_cache_free(global.slab_execute_cbs, cb);
+}
+
+static void __notify_execute_cb(struct i915_request *rq)
+{
+	struct execute_cb *cb;
+
+	lockdep_assert_held(&rq->lock);
+
+	if (list_empty(&rq->execute_cb))
+		return;
+
+	list_for_each_entry(cb, &rq->execute_cb, link)
+		irq_work_queue(&cb->work);
+
+	/*
+	 * XXX Rollback on __i915_request_unsubmit()
+	 *
+	 * In the future, perhaps when we have an active time-slicing scheduler,
+	 * it will be interesting to unsubmit parallel execution and remove
+	 * busywaits from the GPU until their master is restarted. This is
+	 * quite hairy, we have to carefully rollback the fence and do a
+	 * preempt-to-idle cycle on the target engine, all the while the
+	 * master execute_cb may refire.
+	 */
+	INIT_LIST_HEAD(&rq->execute_cb);
+}
+
+static int
+i915_request_await_execution(struct i915_request *rq,
+			     struct i915_request *signal,
+			     gfp_t gfp)
 {
-	return tl->seqno += 1 + tl->has_initial_breadcrumb;
+	struct execute_cb *cb;
+
+	if (i915_request_is_active(signal))
+		return 0;
+
+	cb = kmem_cache_alloc(global.slab_execute_cbs, gfp);
+	if (!cb)
+		return -ENOMEM;
+
+	cb->fence = &rq->submit;
+	i915_sw_fence_await(cb->fence);
+	init_irq_work(&cb->work, irq_execute_cb);
+
+	spin_lock_irq(&signal->lock);
+	if (i915_request_is_active(signal)) {
+		i915_sw_fence_complete(cb->fence);
+		kmem_cache_free(global.slab_execute_cbs, cb);
+	} else {
+		list_add_tail(&cb->link, &signal->execute_cb);
+	}
+	spin_unlock_irq(&signal->lock);
+
+	return 0;
 }
 
 static void move_to_timeline(struct i915_request *request,
@@ -342,42 +410,33 @@ static void move_to_timeline(struct i915_request *request,
 	spin_unlock(&request->timeline->lock);
 }
 
-static u32 next_global_seqno(struct i915_timeline *tl)
-{
-	if (!++tl->seqno)
-		++tl->seqno;
-	return tl->seqno;
-}
-
 void __i915_request_submit(struct i915_request *request)
 {
 	struct intel_engine_cs *engine = request->engine;
-	u32 seqno;
 
-	GEM_TRACE("%s fence %llx:%lld -> global=%d, current %d:%d\n",
+	GEM_TRACE("%s fence %llx:%lld -> current %d\n",
 		  engine->name,
 		  request->fence.context, request->fence.seqno,
-		  engine->timeline.seqno + 1,
-		  hwsp_seqno(request),
-		  intel_engine_get_seqno(engine));
+		  hwsp_seqno(request));
 
 	GEM_BUG_ON(!irqs_disabled());
 	lockdep_assert_held(&engine->timeline.lock);
 
-	GEM_BUG_ON(request->global_seqno);
-
-	seqno = next_global_seqno(&engine->timeline);
-	GEM_BUG_ON(!seqno);
-	GEM_BUG_ON(intel_engine_signaled(engine, seqno));
+	if (i915_gem_context_is_banned(request->gem_context))
+		i915_request_skip(request, -EIO);
 
 	/* We may be recursing from the signal callback of another i915 fence */
 	spin_lock_nested(&request->lock, SINGLE_DEPTH_NESTING);
+
 	GEM_BUG_ON(test_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags));
 	set_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags);
-	request->global_seqno = seqno;
+
 	if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &request->fence.flags) &&
 	    !i915_request_enable_breadcrumb(request))
 		intel_engine_queue_breadcrumbs(engine);
+
+	__notify_execute_cb(request);
+
 	spin_unlock(&request->lock);
 
 	engine->emit_fini_breadcrumb(request,
@@ -406,12 +465,10 @@ void __i915_request_unsubmit(struct i915_request *request)
 {
 	struct intel_engine_cs *engine = request->engine;
 
-	GEM_TRACE("%s fence %llx:%lld <- global=%d, current %d:%d\n",
+	GEM_TRACE("%s fence %llx:%lld, current %d\n",
 		  engine->name,
 		  request->fence.context, request->fence.seqno,
-		  request->global_seqno,
-		  hwsp_seqno(request),
-		  intel_engine_get_seqno(engine));
+		  hwsp_seqno(request));
 
 	GEM_BUG_ON(!irqs_disabled());
 	lockdep_assert_held(&engine->timeline.lock);
@@ -420,18 +477,25 @@ void __i915_request_unsubmit(struct i915_request *request)
 	 * Only unwind in reverse order, required so that the per-context list
 	 * is kept in seqno/ring order.
 	 */
-	GEM_BUG_ON(!request->global_seqno);
-	GEM_BUG_ON(request->global_seqno != engine->timeline.seqno);
-	GEM_BUG_ON(intel_engine_has_completed(engine, request->global_seqno));
-	engine->timeline.seqno--;
 
 	/* We may be recursing from the signal callback of another i915 fence */
 	spin_lock_nested(&request->lock, SINGLE_DEPTH_NESTING);
-	request->global_seqno = 0;
+
+	/*
+	 * As we do not allow WAIT to preempt inflight requests,
+	 * once we have executed a request, along with triggering
+	 * any execution callbacks, we must preserve its ordering
+	 * within the non-preemptible FIFO.
+	 */
+	BUILD_BUG_ON(__NO_PREEMPTION & ~I915_PRIORITY_MASK); /* only internal */
+	request->sched.attr.priority |= __NO_PREEMPTION;
+
 	if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &request->fence.flags))
 		i915_request_cancel_breadcrumb(request);
+
 	GEM_BUG_ON(!test_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags));
 	clear_bit(I915_FENCE_FLAG_ACTIVE, &request->fence.flags);
+
 	spin_unlock(&request->lock);
 
 	/* Transfer back from the global per-engine timeline to per-context */
@@ -489,6 +553,36 @@ submit_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
 	return NOTIFY_DONE;
 }
 
+static int __i915_sw_fence_call
+semaphore_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
+{
+	struct i915_request *request =
+		container_of(fence, typeof(*request), semaphore);
+
+	switch (state) {
+	case FENCE_COMPLETE:
+		/*
+		 * We only check a small portion of our dependencies
+		 * and so cannot guarantee that there remains no
+		 * semaphore chain across all. Instead of opting
+		 * for the full NOSEMAPHORE boost, we go for the
+		 * smaller (but still preempting) boost of
+		 * NEWCLIENT. This will be enough to boost over
+		 * a busywaiting request (as that cannot be
+		 * NEWCLIENT) without accidentally boosting
+		 * a busywait over real work elsewhere.
+		 */
+		i915_schedule_bump_priority(request, I915_PRIORITY_NEWCLIENT);
+		break;
+
+	case FENCE_FREE:
+		i915_request_put(request);
+		break;
+	}
+
+	return NOTIFY_DONE;
+}
+
 static void ring_retire_requests(struct intel_ring *ring)
 {
 	struct i915_request *rq, *rn;
@@ -518,12 +612,7 @@ i915_request_alloc_slow(struct intel_context *ce)
 	ring_retire_requests(ring);
 
 out:
-	return kmem_cache_alloc(ce->gem_context->i915->requests, GFP_KERNEL);
-}
-
-static int add_timeline_barrier(struct i915_request *rq)
-{
-	return i915_request_await_active_request(rq, &rq->timeline->barrier);
+	return kmem_cache_alloc(global.slab_requests, GFP_KERNEL);
 }
 
 /**
@@ -539,8 +628,10 @@ struct i915_request *
 i915_request_alloc(struct intel_engine_cs *engine, struct i915_gem_context *ctx)
 {
 	struct drm_i915_private *i915 = engine->i915;
-	struct i915_request *rq;
 	struct intel_context *ce;
+	struct i915_timeline *tl;
+	struct i915_request *rq;
+	u32 seqno;
 	int ret;
 
 	lockdep_assert_held(&i915->drm.struct_mutex);
@@ -556,8 +647,9 @@ i915_request_alloc(struct intel_engine_cs *engine, struct i915_gem_context *ctx)
 	 * ABI: Before userspace accesses the GPU (e.g. execbuffer), report
 	 * EIO if the GPU is already wedged.
 	 */
-	if (i915_terminally_wedged(&i915->gpu_error))
-		return ERR_PTR(-EIO);
+	ret = i915_terminally_wedged(i915);
+	if (ret)
+		return ERR_PTR(ret);
 
 	/*
 	 * Pinning the contexts may generate requests in order to acquire
@@ -569,6 +661,7 @@ i915_request_alloc(struct intel_engine_cs *engine, struct i915_gem_context *ctx)
 		return ERR_CAST(ce);
 
 	reserve_gt(i915);
+	mutex_lock(&ce->ring->timeline->mutex);
 
 	/* Move our oldest request to the slab-cache (if not in use!) */
 	rq = list_first_entry(&ce->ring->request_list, typeof(*rq), ring_link);
@@ -605,7 +698,7 @@ i915_request_alloc(struct intel_engine_cs *engine, struct i915_gem_context *ctx)
 	 *
 	 * Do not use kmem_cache_zalloc() here!
 	 */
-	rq = kmem_cache_alloc(i915->requests,
+	rq = kmem_cache_alloc(global.slab_requests,
 			      GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
 	if (unlikely(!rq)) {
 		rq = i915_request_alloc_slow(ce);
@@ -615,32 +708,36 @@ i915_request_alloc(struct intel_engine_cs *engine, struct i915_gem_context *ctx)
 		}
 	}
 
-	rq->rcustate = get_state_synchronize_rcu();
-
 	INIT_LIST_HEAD(&rq->active_list);
+	INIT_LIST_HEAD(&rq->execute_cb);
+
+	tl = ce->ring->timeline;
+	ret = i915_timeline_get_seqno(tl, rq, &seqno);
+	if (ret)
+		goto err_free;
+
 	rq->i915 = i915;
 	rq->engine = engine;
 	rq->gem_context = ctx;
 	rq->hw_context = ce;
 	rq->ring = ce->ring;
-	rq->timeline = ce->ring->timeline;
+	rq->timeline = tl;
 	GEM_BUG_ON(rq->timeline == &engine->timeline);
-	rq->hwsp_seqno = rq->timeline->hwsp_seqno;
+	rq->hwsp_seqno = tl->hwsp_seqno;
+	rq->hwsp_cacheline = tl->hwsp_cacheline;
+	rq->rcustate = get_state_synchronize_rcu(); /* acts as smp_mb() */
 
 	spin_lock_init(&rq->lock);
-	dma_fence_init(&rq->fence,
-		       &i915_fence_ops,
-		       &rq->lock,
-		       rq->timeline->fence_context,
-		       timeline_get_seqno(rq->timeline));
+	dma_fence_init(&rq->fence, &i915_fence_ops, &rq->lock,
+		       tl->fence_context, seqno);
 
 	/* We bump the ref for the fence chain */
 	i915_sw_fence_init(&i915_request_get(rq)->submit, submit_notify);
+	i915_sw_fence_init(&i915_request_get(rq)->semaphore, semaphore_notify);
 
 	i915_sched_node_init(&rq->sched);
 
 	/* No zalloc, must clear what we need by hand */
-	rq->global_seqno = 0;
 	rq->file_priv = NULL;
 	rq->batch = NULL;
 	rq->capture_list = NULL;
@@ -668,10 +765,6 @@ i915_request_alloc(struct intel_engine_cs *engine, struct i915_gem_context *ctx)
 	 */
 	rq->head = rq->ring->emit;
 
-	ret = add_timeline_barrier(rq);
-	if (ret)
-		goto err_unwind;
-
 	ret = engine->request_alloc(rq);
 	if (ret)
 		goto err_unwind;
@@ -682,7 +775,10 @@ i915_request_alloc(struct intel_engine_cs *engine, struct i915_gem_context *ctx)
 	rq->infix = rq->ring->emit; /* end of header; start of user payload */
 
 	/* Check that we didn't interrupt ourselves with a new request */
+	lockdep_assert_held(&rq->timeline->mutex);
 	GEM_BUG_ON(rq->timeline->seqno != rq->fence.seqno);
+	rq->cookie = lockdep_pin_lock(&rq->timeline->mutex);
+
 	return rq;
 
 err_unwind:
@@ -693,14 +789,76 @@ err_unwind:
 	GEM_BUG_ON(!list_empty(&rq->sched.signalers_list));
 	GEM_BUG_ON(!list_empty(&rq->sched.waiters_list));
 
-	kmem_cache_free(i915->requests, rq);
+err_free:
+	kmem_cache_free(global.slab_requests, rq);
 err_unreserve:
+	mutex_unlock(&ce->ring->timeline->mutex);
 	unreserve_gt(i915);
 	intel_context_unpin(ce);
 	return ERR_PTR(ret);
 }
 
 static int
+emit_semaphore_wait(struct i915_request *to,
+		    struct i915_request *from,
+		    gfp_t gfp)
+{
+	u32 hwsp_offset;
+	u32 *cs;
+	int err;
+
+	GEM_BUG_ON(!from->timeline->has_initial_breadcrumb);
+	GEM_BUG_ON(INTEL_GEN(to->i915) < 8);
+
+	/* Just emit the first semaphore we see as request space is limited. */
+	if (to->sched.semaphores & from->engine->mask)
+		return i915_sw_fence_await_dma_fence(&to->submit,
+						     &from->fence, 0,
+						     I915_FENCE_GFP);
+
+	err = i915_sw_fence_await_dma_fence(&to->semaphore,
+					    &from->fence, 0,
+					    I915_FENCE_GFP);
+	if (err < 0)
+		return err;
+
+	/* We need to pin the signaler's HWSP until we are finished reading. */
+	err = i915_timeline_read_hwsp(from, to, &hwsp_offset);
+	if (err)
+		return err;
+
+	/* Only submit our spinner after the signaler is running! */
+	err = i915_request_await_execution(to, from, gfp);
+	if (err)
+		return err;
+
+	cs = intel_ring_begin(to, 4);
+	if (IS_ERR(cs))
+		return PTR_ERR(cs);
+
+	/*
+	 * Using greater-than-or-equal here means we have to worry
+	 * about seqno wraparound. To side step that issue, we swap
+	 * the timeline HWSP upon wrapping, so that everyone listening
+	 * for the old (pre-wrap) values do not see the much smaller
+	 * (post-wrap) values than they were expecting (and so wait
+	 * forever).
+	 */
+	*cs++ = MI_SEMAPHORE_WAIT |
+		MI_SEMAPHORE_GLOBAL_GTT |
+		MI_SEMAPHORE_POLL |
+		MI_SEMAPHORE_SAD_GTE_SDD;
+	*cs++ = from->fence.seqno;
+	*cs++ = hwsp_offset;
+	*cs++ = 0;
+
+	intel_ring_advance(to, cs);
+	to->sched.semaphores |= from->engine->mask;
+	to->sched.flags |= I915_SCHED_HAS_SEMAPHORE_CHAIN;
+	return 0;
+}
+
+static int
 i915_request_await_request(struct i915_request *to, struct i915_request *from)
 {
 	int ret;
@@ -712,9 +870,7 @@ i915_request_await_request(struct i915_request *to, struct i915_request *from)
 		return 0;
 
 	if (to->engine->schedule) {
-		ret = i915_sched_node_add_dependency(to->i915,
-						     &to->sched,
-						     &from->sched);
+		ret = i915_sched_node_add_dependency(&to->sched, &from->sched);
 		if (ret < 0)
 			return ret;
 	}
@@ -723,6 +879,9 @@ i915_request_await_request(struct i915_request *to, struct i915_request *from)
 		ret = i915_sw_fence_await_sw_fence_gfp(&to->submit,
 						       &from->submit,
 						       I915_FENCE_GFP);
+	} else if (intel_engine_has_semaphores(to->engine) &&
+		   to->gem_context->sched.priority >= I915_PRIORITY_NORMAL) {
+		ret = emit_semaphore_wait(to, from, I915_FENCE_GFP);
 	} else {
 		ret = i915_sw_fence_await_dma_fence(&to->submit,
 						    &from->fence, 0,
@@ -873,6 +1032,60 @@ void i915_request_skip(struct i915_request *rq, int error)
 	memset(vaddr + head, 0, rq->postfix - head);
 }
 
+static struct i915_request *
+__i915_request_add_to_timeline(struct i915_request *rq)
+{
+	struct i915_timeline *timeline = rq->timeline;
+	struct i915_request *prev;
+
+	/*
+	 * Dependency tracking and request ordering along the timeline
+	 * is special cased so that we can eliminate redundant ordering
+	 * operations while building the request (we know that the timeline
+	 * itself is ordered, and here we guarantee it).
+	 *
+	 * As we know we will need to emit tracking along the timeline,
+	 * we embed the hooks into our request struct -- at the cost of
+	 * having to have specialised no-allocation interfaces (which will
+	 * be beneficial elsewhere).
+	 *
+	 * A second benefit to open-coding i915_request_await_request is
+	 * that we can apply a slight variant of the rules specialised
+	 * for timelines that jump between engines (such as virtual engines).
+	 * If we consider the case of virtual engine, we must emit a dma-fence
+	 * to prevent scheduling of the second request until the first is
+	 * complete (to maximise our greedy late load balancing) and this
+	 * precludes optimising to use semaphores serialisation of a single
+	 * timeline across engines.
+	 */
+	prev = i915_active_request_raw(&timeline->last_request,
+				       &rq->i915->drm.struct_mutex);
+	if (prev && !i915_request_completed(prev)) {
+		if (is_power_of_2(prev->engine->mask | rq->engine->mask))
+			i915_sw_fence_await_sw_fence(&rq->submit,
+						     &prev->submit,
+						     &rq->submitq);
+		else
+			__i915_sw_fence_await_dma_fence(&rq->submit,
+							&prev->fence,
+							&rq->dmaq);
+		if (rq->engine->schedule)
+			__i915_sched_node_add_dependency(&rq->sched,
+							 &prev->sched,
+							 &rq->dep,
+							 0);
+	}
+
+	spin_lock_irq(&timeline->lock);
+	list_add_tail(&rq->link, &timeline->requests);
+	spin_unlock_irq(&timeline->lock);
+
+	GEM_BUG_ON(timeline->seqno != rq->fence.seqno);
+	__i915_active_request_set(&timeline->last_request, rq);
+
+	return prev;
+}
+
 /*
  * NB: This function is not allowed to fail. Doing so would mean the the
  * request is not being tracked for completion but the work itself is
@@ -889,7 +1102,9 @@ void i915_request_add(struct i915_request *request)
 	GEM_TRACE("%s fence %llx:%lld\n",
 		  engine->name, request->fence.context, request->fence.seqno);
 
-	lockdep_assert_held(&request->i915->drm.struct_mutex);
+	lockdep_assert_held(&request->timeline->mutex);
+	lockdep_unpin_lock(&request->timeline->mutex, request->cookie);
+
 	trace_i915_request_add(request);
 
 	/*
@@ -917,37 +1132,12 @@ void i915_request_add(struct i915_request *request)
 	GEM_BUG_ON(IS_ERR(cs));
 	request->postfix = intel_ring_offset(request, cs);
 
-	/*
-	 * Seal the request and mark it as pending execution. Note that
-	 * we may inspect this state, without holding any locks, during
-	 * hangcheck. Hence we apply the barrier to ensure that we do not
-	 * see a more recent value in the hws than we are tracking.
-	 */
-
-	prev = i915_active_request_raw(&timeline->last_request,
-				       &request->i915->drm.struct_mutex);
-	if (prev && !i915_request_completed(prev)) {
-		i915_sw_fence_await_sw_fence(&request->submit, &prev->submit,
-					     &request->submitq);
-		if (engine->schedule)
-			__i915_sched_node_add_dependency(&request->sched,
-							 &prev->sched,
-							 &request->dep,
-							 0);
-	}
-
-	spin_lock_irq(&timeline->lock);
-	list_add_tail(&request->link, &timeline->requests);
-	spin_unlock_irq(&timeline->lock);
-
-	GEM_BUG_ON(timeline->seqno != request->fence.seqno);
-	__i915_active_request_set(&timeline->last_request, request);
+	prev = __i915_request_add_to_timeline(request);
 
 	list_add_tail(&request->ring_link, &ring->request_list);
-	if (list_is_first(&request->ring_link, &ring->request_list)) {
-		GEM_TRACE("marking %s as active\n", ring->timeline->name);
+	if (list_is_first(&request->ring_link, &ring->request_list))
 		list_add(&ring->active_link, &request->i915->gt.active_rings);
-	}
+	request->i915->gt.active_engines |= request->engine->mask;
 	request->emitted_jiffies = jiffies;
 
 	/*
@@ -962,11 +1152,27 @@ void i915_request_add(struct i915_request *request)
 	 * run at the earliest possible convenience.
 	 */
 	local_bh_disable();
+	i915_sw_fence_commit(&request->semaphore);
 	rcu_read_lock(); /* RCU serialisation for set-wedged protection */
 	if (engine->schedule) {
 		struct i915_sched_attr attr = request->gem_context->sched;
 
 		/*
+		 * Boost actual workloads past semaphores!
+		 *
+		 * With semaphores we spin on one engine waiting for another,
+		 * simply to reduce the latency of starting our work when
+		 * the signaler completes. However, if there is any other
+		 * work that we could be doing on this engine instead, that
+		 * is better utilisation and will reduce the overall duration
+		 * of the current work. To avoid PI boosting a semaphore
+		 * far in the distance past over useful work, we keep a history
+		 * of any semaphore use along our dependency chain.
+		 */
+		if (!(request->sched.flags & I915_SCHED_HAS_SEMAPHORE_CHAIN))
+			attr.priority |= I915_PRIORITY_NOSEMAPHORE;
+
+		/*
 		 * Boost priorities to new clients (new request flows).
 		 *
 		 * Allow interactive/synchronous clients to jump ahead of
@@ -1000,6 +1206,8 @@ void i915_request_add(struct i915_request *request)
 	 */
 	if (prev && i915_request_completed(prev))
 		i915_request_retire_upto(prev);
+
+	mutex_unlock(&request->timeline->mutex);
 }
 
 static unsigned long local_clock_us(unsigned int *cpu)
@@ -1136,8 +1344,25 @@ long i915_request_wait(struct i915_request *rq,
 	if (__i915_spin_request(rq, state, 5))
 		goto out;
 
-	if (flags & I915_WAIT_PRIORITY)
+	/*
+	 * This client is about to stall waiting for the GPU. In many cases
+	 * this is undesirable and limits the throughput of the system, as
+	 * many clients cannot continue processing user input/output whilst
+	 * blocked. RPS autotuning may take tens of milliseconds to respond
+	 * to the GPU load and thus incurs additional latency for the client.
+	 * We can circumvent that by promoting the GPU frequency to maximum
+	 * before we sleep. This makes the GPU throttle up much more quickly
+	 * (good for benchmarks and user experience, e.g. window animations),
+	 * but at a cost of spending more power processing the workload
+	 * (bad for battery).
+	 */
+	if (flags & I915_WAIT_PRIORITY) {
+		if (!i915_request_started(rq) && INTEL_GEN(rq->i915) >= 6)
+			gen6_rps_boost(rq);
+		local_bh_disable(); /* suspend tasklets for reprioritisation */
 		i915_schedule_bump_priority(rq, I915_PRIORITY_WAIT);
+		local_bh_enable(); /* kick tasklets en masse */
+	}
 
 	wait.tsk = current;
 	if (dma_fence_add_callback(&rq->fence, &wait.cb, request_wait_wake))
@@ -1179,11 +1404,66 @@ void i915_retire_requests(struct drm_i915_private *i915)
 	if (!i915->gt.active_requests)
 		return;
 
-	list_for_each_entry_safe(ring, tmp, &i915->gt.active_rings, active_link)
+	list_for_each_entry_safe(ring, tmp,
+				 &i915->gt.active_rings, active_link) {
+		intel_ring_get(ring); /* last rq holds reference! */
 		ring_retire_requests(ring);
+		intel_ring_put(ring);
+	}
 }
 
 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 #include "selftests/mock_request.c"
 #include "selftests/i915_request.c"
 #endif
+
+static void i915_global_request_shrink(void)
+{
+	kmem_cache_shrink(global.slab_dependencies);
+	kmem_cache_shrink(global.slab_execute_cbs);
+	kmem_cache_shrink(global.slab_requests);
+}
+
+static void i915_global_request_exit(void)
+{
+	kmem_cache_destroy(global.slab_dependencies);
+	kmem_cache_destroy(global.slab_execute_cbs);
+	kmem_cache_destroy(global.slab_requests);
+}
+
+static struct i915_global_request global = { {
+	.shrink = i915_global_request_shrink,
+	.exit = i915_global_request_exit,
+} };
+
+int __init i915_global_request_init(void)
+{
+	global.slab_requests = KMEM_CACHE(i915_request,
+					  SLAB_HWCACHE_ALIGN |
+					  SLAB_RECLAIM_ACCOUNT |
+					  SLAB_TYPESAFE_BY_RCU);
+	if (!global.slab_requests)
+		return -ENOMEM;
+
+	global.slab_execute_cbs = KMEM_CACHE(execute_cb,
+					     SLAB_HWCACHE_ALIGN |
+					     SLAB_RECLAIM_ACCOUNT |
+					     SLAB_TYPESAFE_BY_RCU);
+	if (!global.slab_execute_cbs)
+		goto err_requests;
+
+	global.slab_dependencies = KMEM_CACHE(i915_dependency,
+					      SLAB_HWCACHE_ALIGN |
+					      SLAB_RECLAIM_ACCOUNT);
+	if (!global.slab_dependencies)
+		goto err_execute_cbs;
+
+	i915_global_register(&global.base);
+	return 0;
+
+err_execute_cbs:
+	kmem_cache_destroy(global.slab_execute_cbs);
+err_requests:
+	kmem_cache_destroy(global.slab_requests);
+	return -ENOMEM;
+}
diff --git a/drivers/gpu/drm/i915/i915_request.h b/drivers/gpu/drm/i915/i915_request.h
index 40f3e8dcbdd5..a982664618c2 100644
--- a/drivers/gpu/drm/i915/i915_request.h
+++ b/drivers/gpu/drm/i915/i915_request.h
@@ -26,9 +26,11 @@
 #define I915_REQUEST_H
 
 #include <linux/dma-fence.h>
+#include <linux/lockdep.h>
 
 #include "i915_gem.h"
 #include "i915_scheduler.h"
+#include "i915_selftest.h"
 #include "i915_sw_fence.h"
 
 #include <uapi/drm/i915_drm.h>
@@ -37,6 +39,7 @@ struct drm_file;
 struct drm_i915_gem_object;
 struct i915_request;
 struct i915_timeline;
+struct i915_timeline_cacheline;
 
 struct i915_capture_list {
 	struct i915_capture_list *next;
@@ -119,6 +122,15 @@ struct i915_request {
 	unsigned long rcustate;
 
 	/*
+	 * We pin the timeline->mutex while constructing the request to
+	 * ensure that no caller accidentally drops it during construction.
+	 * The timeline->mutex must be held to ensure that only this caller
+	 * can use the ring and manipulate the associated timeline during
+	 * construction.
+	 */
+	struct pin_cookie cookie;
+
+	/*
 	 * Fences for the various phases in the request's lifetime.
 	 *
 	 * The submit fence is used to await upon all of the request's
@@ -126,7 +138,12 @@ struct i915_request {
 	 * It is used by the driver to then queue the request for execution.
 	 */
 	struct i915_sw_fence submit;
-	wait_queue_entry_t submitq;
+	union {
+		wait_queue_entry_t submitq;
+		struct i915_sw_dma_fence_cb dmaq;
+	};
+	struct list_head execute_cb;
+	struct i915_sw_fence semaphore;
 
 	/*
 	 * A list of everyone we wait upon, and everyone who waits upon us.
@@ -147,13 +164,15 @@ struct i915_request {
 	 */
 	const u32 *hwsp_seqno;
 
-	/**
-	 * GEM sequence number associated with this request on the
-	 * global execution timeline. It is zero when the request is not
-	 * on the HW queue (i.e. not on the engine timeline list).
-	 * Its value is guarded by the timeline spinlock.
+	/*
+	 * If we need to access the timeline's seqno for this request in
+	 * another request, we need to keep a read reference to this associated
+	 * cacheline, so that we do not free and recycle it before the foreign
+	 * observers have completed. Hence, we keep a pointer to the cacheline
+	 * inside the timeline's HWSP vma, but it is only valid while this
+	 * request has not completed and guarded by the timeline mutex.
 	 */
-	u32 global_seqno;
+	struct i915_timeline_cacheline *hwsp_cacheline;
 
 	/** Position in the ring of the start of the request */
 	u32 head;
@@ -204,6 +223,11 @@ struct i915_request {
 	struct drm_i915_file_private *file_priv;
 	/** file_priv list entry for this request */
 	struct list_head client_link;
+
+	I915_SELFTEST_DECLARE(struct {
+		struct list_head link;
+		unsigned long delay;
+	} mock;)
 };
 
 #define I915_FENCE_GFP (GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN)
@@ -247,30 +271,6 @@ i915_request_put(struct i915_request *rq)
 	dma_fence_put(&rq->fence);
 }
 
-/**
- * i915_request_global_seqno - report the current global seqno
- * @request - the request
- *
- * A request is assigned a global seqno only when it is on the hardware
- * execution queue. The global seqno can be used to maintain a list of
- * requests on the same engine in retirement order, for example for
- * constructing a priority queue for waiting. Prior to its execution, or
- * if it is subsequently removed in the event of preemption, its global
- * seqno is zero. As both insertion and removal from the execution queue
- * may operate in IRQ context, it is not guarded by the usual struct_mutex
- * BKL. Instead those relying on the global seqno must be prepared for its
- * value to change between reads. Only when the request is complete can
- * the global seqno be stable (due to the memory barriers on submitting
- * the commands to the hardware to write the breadcrumb, if the HWS shows
- * that it has passed the global seqno and the global seqno is unchanged
- * after the read, it is indeed complete).
- */
-static inline u32
-i915_request_global_seqno(const struct i915_request *request)
-{
-	return READ_ONCE(request->global_seqno);
-}
-
 int i915_request_await_object(struct i915_request *to,
 			      struct drm_i915_gem_object *obj,
 			      bool write);
@@ -358,10 +358,27 @@ static inline bool __i915_request_has_started(const struct i915_request *rq)
  * i915_request_started - check if the request has begun being executed
  * @rq: the request
  *
- * Returns true if the request has been submitted to hardware, and the hardware
- * has advanced passed the end of the previous request and so should be either
- * currently processing the request (though it may be preempted and so
- * not necessarily the next request to complete) or have completed the request.
+ * If the timeline is not using initial breadcrumbs, a request is
+ * considered started if the previous request on its timeline (i.e.
+ * context) has been signaled.
+ *
+ * If the timeline is using semaphores, it will also be emitting an
+ * "initial breadcrumb" after the semaphores are complete and just before
+ * it began executing the user payload. A request can therefore be active
+ * on the HW and not yet started as it is still busywaiting on its
+ * dependencies (via HW semaphores).
+ *
+ * If the request has started, its dependencies will have been signaled
+ * (either by fences or by semaphores) and it will have begun processing
+ * the user payload.
+ *
+ * However, even if a request has started, it may have been preempted and
+ * so no longer active, or it may have already completed.
+ *
+ * See also i915_request_is_active().
+ *
+ * Returns true if the request has begun executing the user payload, or
+ * has completed:
  */
 static inline bool i915_request_started(const struct i915_request *rq)
 {
diff --git a/drivers/gpu/drm/i915/i915_reset.c b/drivers/gpu/drm/i915/i915_reset.c
index 0e0ddf2e6815..677d59304e78 100644
--- a/drivers/gpu/drm/i915/i915_reset.c
+++ b/drivers/gpu/drm/i915/i915_reset.c
@@ -18,28 +18,39 @@
 /* XXX How to handle concurrent GGTT updates using tiling registers? */
 #define RESET_UNDER_STOP_MACHINE 0
 
+static void rmw_set(struct intel_uncore *uncore, i915_reg_t reg, u32 set)
+{
+	intel_uncore_rmw(uncore, reg, 0, set);
+}
+
+static void rmw_clear(struct intel_uncore *uncore, i915_reg_t reg, u32 clr)
+{
+	intel_uncore_rmw(uncore, reg, clr, 0);
+}
+
+static void rmw_set_fw(struct intel_uncore *uncore, i915_reg_t reg, u32 set)
+{
+	intel_uncore_rmw_fw(uncore, reg, 0, set);
+}
+
+static void rmw_clear_fw(struct intel_uncore *uncore, i915_reg_t reg, u32 clr)
+{
+	intel_uncore_rmw_fw(uncore, reg, clr, 0);
+}
+
 static void engine_skip_context(struct i915_request *rq)
 {
 	struct intel_engine_cs *engine = rq->engine;
 	struct i915_gem_context *hung_ctx = rq->gem_context;
-	struct i915_timeline *timeline = rq->timeline;
 
 	lockdep_assert_held(&engine->timeline.lock);
-	GEM_BUG_ON(timeline == &engine->timeline);
-
-	spin_lock(&timeline->lock);
-
-	if (i915_request_is_active(rq)) {
-		list_for_each_entry_continue(rq,
-					     &engine->timeline.requests, link)
-			if (rq->gem_context == hung_ctx)
-				i915_request_skip(rq, -EIO);
-	}
 
-	list_for_each_entry(rq, &timeline->requests, link)
-		i915_request_skip(rq, -EIO);
+	if (!i915_request_is_active(rq))
+		return;
 
-	spin_unlock(&timeline->lock);
+	list_for_each_entry_continue(rq, &engine->timeline.requests, link)
+		if (rq->gem_context == hung_ctx)
+			i915_request_skip(rq, -EIO);
 }
 
 static void client_mark_guilty(struct drm_i915_file_private *file_priv,
@@ -68,23 +79,29 @@ static void client_mark_guilty(struct drm_i915_file_private *file_priv,
 
 static bool context_mark_guilty(struct i915_gem_context *ctx)
 {
-	unsigned int score;
-	bool banned, bannable;
+	unsigned long prev_hang;
+	bool banned;
+	int i;
 
 	atomic_inc(&ctx->guilty_count);
 
-	bannable = i915_gem_context_is_bannable(ctx);
-	score = atomic_add_return(CONTEXT_SCORE_GUILTY, &ctx->ban_score);
-	banned = score >= CONTEXT_SCORE_BAN_THRESHOLD;
-
-	/* Cool contexts don't accumulate client ban score */
-	if (!bannable)
+	/* Cool contexts are too cool to be banned! (Used for reset testing.) */
+	if (!i915_gem_context_is_bannable(ctx))
 		return false;
 
+	/* Record the timestamp for the last N hangs */
+	prev_hang = ctx->hang_timestamp[0];
+	for (i = 0; i < ARRAY_SIZE(ctx->hang_timestamp) - 1; i++)
+		ctx->hang_timestamp[i] = ctx->hang_timestamp[i + 1];
+	ctx->hang_timestamp[i] = jiffies;
+
+	/* If we have hung N+1 times in rapid succession, we ban the context! */
+	banned = !i915_gem_context_is_recoverable(ctx);
+	if (time_before(jiffies, prev_hang + CONTEXT_FAST_HANG_JIFFIES))
+		banned = true;
 	if (banned) {
-		DRM_DEBUG_DRIVER("context %s: guilty %d, score %u, banned\n",
-				 ctx->name, atomic_read(&ctx->guilty_count),
-				 score);
+		DRM_DEBUG_DRIVER("context %s: guilty %d, banned\n",
+				 ctx->name, atomic_read(&ctx->guilty_count));
 		i915_gem_context_set_banned(ctx);
 	}
 
@@ -101,6 +118,12 @@ static void context_mark_innocent(struct i915_gem_context *ctx)
 
 void i915_reset_request(struct i915_request *rq, bool guilty)
 {
+	GEM_TRACE("%s rq=%llx:%lld, guilty? %s\n",
+		  rq->engine->name,
+		  rq->fence.context,
+		  rq->fence.seqno,
+		  yesno(guilty));
+
 	lockdep_assert_held(&rq->engine->timeline.lock);
 	GEM_BUG_ON(i915_request_completed(rq));
 
@@ -116,38 +139,43 @@ void i915_reset_request(struct i915_request *rq, bool guilty)
 
 static void gen3_stop_engine(struct intel_engine_cs *engine)
 {
-	struct drm_i915_private *dev_priv = engine->i915;
+	struct intel_uncore *uncore = engine->uncore;
 	const u32 base = engine->mmio_base;
 
+	GEM_TRACE("%s\n", engine->name);
+
 	if (intel_engine_stop_cs(engine))
-		DRM_DEBUG_DRIVER("%s: timed out on STOP_RING\n", engine->name);
+		GEM_TRACE("%s: timed out on STOP_RING\n", engine->name);
 
-	I915_WRITE_FW(RING_HEAD(base), I915_READ_FW(RING_TAIL(base)));
-	POSTING_READ_FW(RING_HEAD(base)); /* paranoia */
+	intel_uncore_write_fw(uncore,
+			      RING_HEAD(base),
+			      intel_uncore_read_fw(uncore, RING_TAIL(base)));
+	intel_uncore_posting_read_fw(uncore, RING_HEAD(base)); /* paranoia */
 
-	I915_WRITE_FW(RING_HEAD(base), 0);
-	I915_WRITE_FW(RING_TAIL(base), 0);
-	POSTING_READ_FW(RING_TAIL(base));
+	intel_uncore_write_fw(uncore, RING_HEAD(base), 0);
+	intel_uncore_write_fw(uncore, RING_TAIL(base), 0);
+	intel_uncore_posting_read_fw(uncore, RING_TAIL(base));
 
 	/* The ring must be empty before it is disabled */
-	I915_WRITE_FW(RING_CTL(base), 0);
+	intel_uncore_write_fw(uncore, RING_CTL(base), 0);
 
 	/* Check acts as a post */
-	if (I915_READ_FW(RING_HEAD(base)) != 0)
-		DRM_DEBUG_DRIVER("%s: ring head not parked\n",
-				 engine->name);
+	if (intel_uncore_read_fw(uncore, RING_HEAD(base)))
+		GEM_TRACE("%s: ring head [%x] not parked\n",
+			  engine->name,
+			  intel_uncore_read_fw(uncore, RING_HEAD(base)));
 }
 
 static void i915_stop_engines(struct drm_i915_private *i915,
-			      unsigned int engine_mask)
+			      intel_engine_mask_t engine_mask)
 {
 	struct intel_engine_cs *engine;
-	enum intel_engine_id id;
+	intel_engine_mask_t tmp;
 
 	if (INTEL_GEN(i915) < 3)
 		return;
 
-	for_each_engine_masked(engine, i915, engine_mask, id)
+	for_each_engine_masked(engine, i915, engine_mask, tmp)
 		gen3_stop_engine(engine);
 }
 
@@ -160,7 +188,7 @@ static bool i915_in_reset(struct pci_dev *pdev)
 }
 
 static int i915_do_reset(struct drm_i915_private *i915,
-			 unsigned int engine_mask,
+			 intel_engine_mask_t engine_mask,
 			 unsigned int retry)
 {
 	struct pci_dev *pdev = i915->drm.pdev;
@@ -189,7 +217,7 @@ static bool g4x_reset_complete(struct pci_dev *pdev)
 }
 
 static int g33_do_reset(struct drm_i915_private *i915,
-			unsigned int engine_mask,
+			intel_engine_mask_t engine_mask,
 			unsigned int retry)
 {
 	struct pci_dev *pdev = i915->drm.pdev;
@@ -198,17 +226,17 @@ static int g33_do_reset(struct drm_i915_private *i915,
 	return wait_for_atomic(g4x_reset_complete(pdev), 50);
 }
 
-static int g4x_do_reset(struct drm_i915_private *dev_priv,
-			unsigned int engine_mask,
+static int g4x_do_reset(struct drm_i915_private *i915,
+			intel_engine_mask_t engine_mask,
 			unsigned int retry)
 {
-	struct pci_dev *pdev = dev_priv->drm.pdev;
+	struct pci_dev *pdev = i915->drm.pdev;
+	struct intel_uncore *uncore = &i915->uncore;
 	int ret;
 
 	/* WaVcpClkGateDisableForMediaReset:ctg,elk */
-	I915_WRITE_FW(VDECCLK_GATE_D,
-		      I915_READ(VDECCLK_GATE_D) | VCP_UNIT_CLOCK_GATE_DISABLE);
-	POSTING_READ_FW(VDECCLK_GATE_D);
+	rmw_set_fw(uncore, VDECCLK_GATE_D, VCP_UNIT_CLOCK_GATE_DISABLE);
+	intel_uncore_posting_read_fw(uncore, VDECCLK_GATE_D);
 
 	pci_write_config_byte(pdev, I915_GDRST,
 			      GRDOM_MEDIA | GRDOM_RESET_ENABLE);
@@ -229,21 +257,22 @@ static int g4x_do_reset(struct drm_i915_private *dev_priv,
 out:
 	pci_write_config_byte(pdev, I915_GDRST, 0);
 
-	I915_WRITE_FW(VDECCLK_GATE_D,
-		      I915_READ(VDECCLK_GATE_D) & ~VCP_UNIT_CLOCK_GATE_DISABLE);
-	POSTING_READ_FW(VDECCLK_GATE_D);
+	rmw_clear_fw(uncore, VDECCLK_GATE_D, VCP_UNIT_CLOCK_GATE_DISABLE);
+	intel_uncore_posting_read_fw(uncore, VDECCLK_GATE_D);
 
 	return ret;
 }
 
-static int ironlake_do_reset(struct drm_i915_private *dev_priv,
-			     unsigned int engine_mask,
+static int ironlake_do_reset(struct drm_i915_private *i915,
+			     intel_engine_mask_t engine_mask,
 			     unsigned int retry)
 {
+	struct intel_uncore *uncore = &i915->uncore;
 	int ret;
 
-	I915_WRITE_FW(ILK_GDSR, ILK_GRDOM_RENDER | ILK_GRDOM_RESET_ENABLE);
-	ret = __intel_wait_for_register_fw(dev_priv, ILK_GDSR,
+	intel_uncore_write_fw(uncore, ILK_GDSR,
+			      ILK_GRDOM_RENDER | ILK_GRDOM_RESET_ENABLE);
+	ret = __intel_wait_for_register_fw(uncore, ILK_GDSR,
 					   ILK_GRDOM_RESET_ENABLE, 0,
 					   5000, 0,
 					   NULL);
@@ -252,8 +281,9 @@ static int ironlake_do_reset(struct drm_i915_private *dev_priv,
 		goto out;
 	}
 
-	I915_WRITE_FW(ILK_GDSR, ILK_GRDOM_MEDIA | ILK_GRDOM_RESET_ENABLE);
-	ret = __intel_wait_for_register_fw(dev_priv, ILK_GDSR,
+	intel_uncore_write_fw(uncore, ILK_GDSR,
+			      ILK_GRDOM_MEDIA | ILK_GRDOM_RESET_ENABLE);
+	ret = __intel_wait_for_register_fw(uncore, ILK_GDSR,
 					   ILK_GRDOM_RESET_ENABLE, 0,
 					   5000, 0,
 					   NULL);
@@ -263,15 +293,16 @@ static int ironlake_do_reset(struct drm_i915_private *dev_priv,
 	}
 
 out:
-	I915_WRITE_FW(ILK_GDSR, 0);
-	POSTING_READ_FW(ILK_GDSR);
+	intel_uncore_write_fw(uncore, ILK_GDSR, 0);
+	intel_uncore_posting_read_fw(uncore, ILK_GDSR);
 	return ret;
 }
 
 /* Reset the hardware domains (GENX_GRDOM_*) specified by mask */
-static int gen6_hw_domain_reset(struct drm_i915_private *dev_priv,
+static int gen6_hw_domain_reset(struct drm_i915_private *i915,
 				u32 hw_domain_mask)
 {
+	struct intel_uncore *uncore = &i915->uncore;
 	int err;
 
 	/*
@@ -279,10 +310,10 @@ static int gen6_hw_domain_reset(struct drm_i915_private *dev_priv,
 	 * for fifo space for the write or forcewake the chip for
 	 * the read
 	 */
-	I915_WRITE_FW(GEN6_GDRST, hw_domain_mask);
+	intel_uncore_write_fw(uncore, GEN6_GDRST, hw_domain_mask);
 
 	/* Wait for the device to ack the reset requests */
-	err = __intel_wait_for_register_fw(dev_priv,
+	err = __intel_wait_for_register_fw(uncore,
 					   GEN6_GDRST, hw_domain_mask, 0,
 					   500, 0,
 					   NULL);
@@ -294,36 +325,38 @@ static int gen6_hw_domain_reset(struct drm_i915_private *dev_priv,
 }
 
 static int gen6_reset_engines(struct drm_i915_private *i915,
-			      unsigned int engine_mask,
+			      intel_engine_mask_t engine_mask,
 			      unsigned int retry)
 {
 	struct intel_engine_cs *engine;
-	const u32 hw_engine_mask[I915_NUM_ENGINES] = {
-		[RCS] = GEN6_GRDOM_RENDER,
-		[BCS] = GEN6_GRDOM_BLT,
-		[VCS] = GEN6_GRDOM_MEDIA,
-		[VCS2] = GEN8_GRDOM_MEDIA2,
-		[VECS] = GEN6_GRDOM_VECS,
+	const u32 hw_engine_mask[] = {
+		[RCS0]  = GEN6_GRDOM_RENDER,
+		[BCS0]  = GEN6_GRDOM_BLT,
+		[VCS0]  = GEN6_GRDOM_MEDIA,
+		[VCS1]  = GEN8_GRDOM_MEDIA2,
+		[VECS0] = GEN6_GRDOM_VECS,
 	};
 	u32 hw_mask;
 
 	if (engine_mask == ALL_ENGINES) {
 		hw_mask = GEN6_GRDOM_FULL;
 	} else {
-		unsigned int tmp;
+		intel_engine_mask_t tmp;
 
 		hw_mask = 0;
-		for_each_engine_masked(engine, i915, engine_mask, tmp)
+		for_each_engine_masked(engine, i915, engine_mask, tmp) {
+			GEM_BUG_ON(engine->id >= ARRAY_SIZE(hw_engine_mask));
 			hw_mask |= hw_engine_mask[engine->id];
+		}
 	}
 
 	return gen6_hw_domain_reset(i915, hw_mask);
 }
 
-static u32 gen11_lock_sfc(struct drm_i915_private *dev_priv,
-			  struct intel_engine_cs *engine)
+static u32 gen11_lock_sfc(struct intel_engine_cs *engine)
 {
-	u8 vdbox_sfc_access = RUNTIME_INFO(dev_priv)->vdbox_sfc_access;
+	struct intel_uncore *uncore = engine->uncore;
+	u8 vdbox_sfc_access = RUNTIME_INFO(engine->i915)->vdbox_sfc_access;
 	i915_reg_t sfc_forced_lock, sfc_forced_lock_ack;
 	u32 sfc_forced_lock_bit, sfc_forced_lock_ack_bit;
 	i915_reg_t sfc_usage;
@@ -370,10 +403,9 @@ static u32 gen11_lock_sfc(struct drm_i915_private *dev_priv,
 	 * ends up being locked to the engine we want to reset, we have to reset
 	 * it as well (we will unlock it once the reset sequence is completed).
 	 */
-	I915_WRITE_FW(sfc_forced_lock,
-		      I915_READ_FW(sfc_forced_lock) | sfc_forced_lock_bit);
+	rmw_set_fw(uncore, sfc_forced_lock, sfc_forced_lock_bit);
 
-	if (__intel_wait_for_register_fw(dev_priv,
+	if (__intel_wait_for_register_fw(uncore,
 					 sfc_forced_lock_ack,
 					 sfc_forced_lock_ack_bit,
 					 sfc_forced_lock_ack_bit,
@@ -382,16 +414,16 @@ static u32 gen11_lock_sfc(struct drm_i915_private *dev_priv,
 		return 0;
 	}
 
-	if (I915_READ_FW(sfc_usage) & sfc_usage_bit)
+	if (intel_uncore_read_fw(uncore, sfc_usage) & sfc_usage_bit)
 		return sfc_reset_bit;
 
 	return 0;
 }
 
-static void gen11_unlock_sfc(struct drm_i915_private *dev_priv,
-			     struct intel_engine_cs *engine)
+static void gen11_unlock_sfc(struct intel_engine_cs *engine)
 {
-	u8 vdbox_sfc_access = RUNTIME_INFO(dev_priv)->vdbox_sfc_access;
+	struct intel_uncore *uncore = engine->uncore;
+	u8 vdbox_sfc_access = RUNTIME_INFO(engine->i915)->vdbox_sfc_access;
 	i915_reg_t sfc_forced_lock;
 	u32 sfc_forced_lock_bit;
 
@@ -413,38 +445,36 @@ static void gen11_unlock_sfc(struct drm_i915_private *dev_priv,
 		return;
 	}
 
-	I915_WRITE_FW(sfc_forced_lock,
-		      I915_READ_FW(sfc_forced_lock) & ~sfc_forced_lock_bit);
+	rmw_clear_fw(uncore, sfc_forced_lock, sfc_forced_lock_bit);
 }
 
 static int gen11_reset_engines(struct drm_i915_private *i915,
-			       unsigned int engine_mask,
+			       intel_engine_mask_t engine_mask,
 			       unsigned int retry)
 {
-	const u32 hw_engine_mask[I915_NUM_ENGINES] = {
-		[RCS] = GEN11_GRDOM_RENDER,
-		[BCS] = GEN11_GRDOM_BLT,
-		[VCS] = GEN11_GRDOM_MEDIA,
-		[VCS2] = GEN11_GRDOM_MEDIA2,
-		[VCS3] = GEN11_GRDOM_MEDIA3,
-		[VCS4] = GEN11_GRDOM_MEDIA4,
-		[VECS] = GEN11_GRDOM_VECS,
-		[VECS2] = GEN11_GRDOM_VECS2,
+	const u32 hw_engine_mask[] = {
+		[RCS0]  = GEN11_GRDOM_RENDER,
+		[BCS0]  = GEN11_GRDOM_BLT,
+		[VCS0]  = GEN11_GRDOM_MEDIA,
+		[VCS1]  = GEN11_GRDOM_MEDIA2,
+		[VCS2]  = GEN11_GRDOM_MEDIA3,
+		[VCS3]  = GEN11_GRDOM_MEDIA4,
+		[VECS0] = GEN11_GRDOM_VECS,
+		[VECS1] = GEN11_GRDOM_VECS2,
 	};
 	struct intel_engine_cs *engine;
-	unsigned int tmp;
+	intel_engine_mask_t tmp;
 	u32 hw_mask;
 	int ret;
 
-	BUILD_BUG_ON(VECS2 + 1 != I915_NUM_ENGINES);
-
 	if (engine_mask == ALL_ENGINES) {
 		hw_mask = GEN11_GRDOM_FULL;
 	} else {
 		hw_mask = 0;
 		for_each_engine_masked(engine, i915, engine_mask, tmp) {
+			GEM_BUG_ON(engine->id >= ARRAY_SIZE(hw_engine_mask));
 			hw_mask |= hw_engine_mask[engine->id];
-			hw_mask |= gen11_lock_sfc(i915, engine);
+			hw_mask |= gen11_lock_sfc(engine);
 		}
 	}
 
@@ -452,46 +482,62 @@ static int gen11_reset_engines(struct drm_i915_private *i915,
 
 	if (engine_mask != ALL_ENGINES)
 		for_each_engine_masked(engine, i915, engine_mask, tmp)
-			gen11_unlock_sfc(i915, engine);
+			gen11_unlock_sfc(engine);
 
 	return ret;
 }
 
 static int gen8_engine_reset_prepare(struct intel_engine_cs *engine)
 {
-	struct drm_i915_private *dev_priv = engine->i915;
+	struct intel_uncore *uncore = engine->uncore;
+	const i915_reg_t reg = RING_RESET_CTL(engine->mmio_base);
+	u32 request, mask, ack;
 	int ret;
 
-	I915_WRITE_FW(RING_RESET_CTL(engine->mmio_base),
-		      _MASKED_BIT_ENABLE(RESET_CTL_REQUEST_RESET));
+	ack = intel_uncore_read_fw(uncore, reg);
+	if (ack & RESET_CTL_CAT_ERROR) {
+		/*
+		 * For catastrophic errors, ready-for-reset sequence
+		 * needs to be bypassed: HAS#396813
+		 */
+		request = RESET_CTL_CAT_ERROR;
+		mask = RESET_CTL_CAT_ERROR;
+
+		/* Catastrophic errors need to be cleared by HW */
+		ack = 0;
+	} else if (!(ack & RESET_CTL_READY_TO_RESET)) {
+		request = RESET_CTL_REQUEST_RESET;
+		mask = RESET_CTL_READY_TO_RESET;
+		ack = RESET_CTL_READY_TO_RESET;
+	} else {
+		return 0;
+	}
 
-	ret = __intel_wait_for_register_fw(dev_priv,
-					   RING_RESET_CTL(engine->mmio_base),
-					   RESET_CTL_READY_TO_RESET,
-					   RESET_CTL_READY_TO_RESET,
-					   700, 0,
-					   NULL);
+	intel_uncore_write_fw(uncore, reg, _MASKED_BIT_ENABLE(request));
+	ret = __intel_wait_for_register_fw(uncore, reg, mask, ack,
+					   700, 0, NULL);
 	if (ret)
-		DRM_ERROR("%s: reset request timeout\n", engine->name);
+		DRM_ERROR("%s reset request timed out: {request: %08x, RESET_CTL: %08x}\n",
+			  engine->name, request,
+			  intel_uncore_read_fw(uncore, reg));
 
 	return ret;
 }
 
 static void gen8_engine_reset_cancel(struct intel_engine_cs *engine)
 {
-	struct drm_i915_private *dev_priv = engine->i915;
-
-	I915_WRITE_FW(RING_RESET_CTL(engine->mmio_base),
-		      _MASKED_BIT_DISABLE(RESET_CTL_REQUEST_RESET));
+	intel_uncore_write_fw(engine->uncore,
+			      RING_RESET_CTL(engine->mmio_base),
+			      _MASKED_BIT_DISABLE(RESET_CTL_REQUEST_RESET));
 }
 
 static int gen8_reset_engines(struct drm_i915_private *i915,
-			      unsigned int engine_mask,
+			      intel_engine_mask_t engine_mask,
 			      unsigned int retry)
 {
 	struct intel_engine_cs *engine;
 	const bool reset_non_ready = retry >= 1;
-	unsigned int tmp;
+	intel_engine_mask_t tmp;
 	int ret;
 
 	for_each_engine_masked(engine, i915, engine_mask, tmp) {
@@ -527,14 +573,11 @@ skip_reset:
 }
 
 typedef int (*reset_func)(struct drm_i915_private *,
-			  unsigned int engine_mask,
+			  intel_engine_mask_t engine_mask,
 			  unsigned int retry);
 
 static reset_func intel_get_gpu_reset(struct drm_i915_private *i915)
 {
-	if (!i915_modparams.reset)
-		return NULL;
-
 	if (INTEL_GEN(i915) >= 8)
 		return gen8_reset_engines;
 	else if (INTEL_GEN(i915) >= 6)
@@ -551,7 +594,8 @@ static reset_func intel_get_gpu_reset(struct drm_i915_private *i915)
 		return NULL;
 }
 
-int intel_gpu_reset(struct drm_i915_private *i915, unsigned int engine_mask)
+int intel_gpu_reset(struct drm_i915_private *i915,
+		    intel_engine_mask_t engine_mask)
 {
 	const int retries = engine_mask == ALL_ENGINES ? RESET_MAX_RETRIES : 1;
 	reset_func reset;
@@ -566,7 +610,7 @@ int intel_gpu_reset(struct drm_i915_private *i915, unsigned int engine_mask)
 	 * If the power well sleeps during the reset, the reset
 	 * request may be dropped and never completes (causing -EIO).
 	 */
-	intel_uncore_forcewake_get(i915, FORCEWAKE_ALL);
+	intel_uncore_forcewake_get(&i915->uncore, FORCEWAKE_ALL);
 	for (retry = 0; ret == -ETIMEDOUT && retry < retries; retry++) {
 		/*
 		 * We stop engines, otherwise we might get failed reset and a
@@ -582,14 +626,15 @@ int intel_gpu_reset(struct drm_i915_private *i915, unsigned int engine_mask)
 		 *
 		 * FIXME: Wa for more modern gens needs to be validated
 		 */
-		i915_stop_engines(i915, engine_mask);
+		if (retry)
+			i915_stop_engines(i915, engine_mask);
 
 		GEM_TRACE("engine_mask=%x\n", engine_mask);
 		preempt_disable();
 		ret = reset(i915, engine_mask, retry);
 		preempt_enable();
 	}
-	intel_uncore_forcewake_put(i915, FORCEWAKE_ALL);
+	intel_uncore_forcewake_put(&i915->uncore, FORCEWAKE_ALL);
 
 	return ret;
 }
@@ -599,6 +644,9 @@ bool intel_has_gpu_reset(struct drm_i915_private *i915)
 	if (USES_GUC(i915))
 		return false;
 
+	if (!i915_modparams.reset)
+		return NULL;
+
 	return intel_get_gpu_reset(i915);
 }
 
@@ -615,9 +663,9 @@ int intel_reset_guc(struct drm_i915_private *i915)
 
 	GEM_BUG_ON(!HAS_GUC(i915));
 
-	intel_uncore_forcewake_get(i915, FORCEWAKE_ALL);
+	intel_uncore_forcewake_get(&i915->uncore, FORCEWAKE_ALL);
 	ret = gen6_hw_domain_reset(i915, guc_domain);
-	intel_uncore_forcewake_put(i915, FORCEWAKE_ALL);
+	intel_uncore_forcewake_put(&i915->uncore, FORCEWAKE_ALL);
 
 	return ret;
 }
@@ -635,10 +683,36 @@ static void reset_prepare_engine(struct intel_engine_cs *engine)
 	 * written to the powercontext is undefined and so we may lose
 	 * GPU state upon resume, i.e. fail to restart after a reset.
 	 */
-	intel_uncore_forcewake_get(engine->i915, FORCEWAKE_ALL);
+	intel_uncore_forcewake_get(engine->uncore, FORCEWAKE_ALL);
 	engine->reset.prepare(engine);
 }
 
+static void revoke_mmaps(struct drm_i915_private *i915)
+{
+	int i;
+
+	for (i = 0; i < i915->num_fence_regs; i++) {
+		struct drm_vma_offset_node *node;
+		struct i915_vma *vma;
+		u64 vma_offset;
+
+		vma = READ_ONCE(i915->fence_regs[i].vma);
+		if (!vma)
+			continue;
+
+		if (!i915_vma_has_userfault(vma))
+			continue;
+
+		GEM_BUG_ON(vma->fence != &i915->fence_regs[i]);
+		node = &vma->obj->base.vma_node;
+		vma_offset = vma->ggtt_view.partial.offset << PAGE_SHIFT;
+		unmap_mapping_range(i915->drm.anon_inode->i_mapping,
+				    drm_vma_node_offset_addr(node) + vma_offset,
+				    vma->size,
+				    1);
+	}
+}
+
 static void reset_prepare(struct drm_i915_private *i915)
 {
 	struct intel_engine_cs *engine;
@@ -647,10 +721,16 @@ static void reset_prepare(struct drm_i915_private *i915)
 	for_each_engine(engine, i915, id)
 		reset_prepare_engine(engine);
 
-	intel_uc_sanitize(i915);
+	intel_uc_reset_prepare(i915);
+}
+
+static void gt_revoke(struct drm_i915_private *i915)
+{
+	revoke_mmaps(i915);
 }
 
-static int gt_reset(struct drm_i915_private *i915, unsigned int stalled_mask)
+static int gt_reset(struct drm_i915_private *i915,
+		    intel_engine_mask_t stalled_mask)
 {
 	struct intel_engine_cs *engine;
 	enum intel_engine_id id;
@@ -665,7 +745,7 @@ static int gt_reset(struct drm_i915_private *i915, unsigned int stalled_mask)
 		return err;
 
 	for_each_engine(engine, i915, id)
-		intel_engine_reset(engine, stalled_mask & ENGINE_MASK(id));
+		intel_engine_reset(engine, stalled_mask & engine->mask);
 
 	i915_gem_restore_fences(i915);
 
@@ -675,7 +755,7 @@ static int gt_reset(struct drm_i915_private *i915, unsigned int stalled_mask)
 static void reset_finish_engine(struct intel_engine_cs *engine)
 {
 	engine->reset.finish(engine);
-	intel_uncore_forcewake_put(engine->i915, FORCEWAKE_ALL);
+	intel_uncore_forcewake_put(engine->uncore, FORCEWAKE_ALL);
 }
 
 struct i915_gpu_restart {
@@ -722,8 +802,10 @@ static void reset_finish(struct drm_i915_private *i915)
 	struct intel_engine_cs *engine;
 	enum intel_engine_id id;
 
-	for_each_engine(engine, i915, id)
+	for_each_engine(engine, i915, id) {
 		reset_finish_engine(engine);
+		intel_engine_signal_breadcrumbs(engine);
+	}
 }
 
 static void reset_restart(struct drm_i915_private *i915)
@@ -761,23 +843,19 @@ static void nop_submit_request(struct i915_request *request)
 	spin_lock_irqsave(&engine->timeline.lock, flags);
 	__i915_request_submit(request);
 	i915_request_mark_complete(request);
-	intel_engine_write_global_seqno(engine, request->global_seqno);
 	spin_unlock_irqrestore(&engine->timeline.lock, flags);
 
 	intel_engine_queue_breadcrumbs(engine);
 }
 
-void i915_gem_set_wedged(struct drm_i915_private *i915)
+static void __i915_gem_set_wedged(struct drm_i915_private *i915)
 {
 	struct i915_gpu_error *error = &i915->gpu_error;
 	struct intel_engine_cs *engine;
 	enum intel_engine_id id;
 
-	mutex_lock(&error->wedge_mutex);
-	if (test_bit(I915_WEDGED, &error->flags)) {
-		mutex_unlock(&error->wedge_mutex);
+	if (test_bit(I915_WEDGED, &error->flags))
 		return;
-	}
 
 	if (GEM_SHOW_DEBUG() && !intel_engines_are_idle(i915)) {
 		struct drm_printer p = drm_debug_printer(__func__);
@@ -793,11 +871,10 @@ void i915_gem_set_wedged(struct drm_i915_private *i915)
 	 * rolling the global seqno forward (since this would complete requests
 	 * for which we haven't set the fence error to EIO yet).
 	 */
-	for_each_engine(engine, i915, id)
-		reset_prepare_engine(engine);
+	reset_prepare(i915);
 
 	/* Even if the GPU reset fails, it should still stop the engines */
-	if (INTEL_GEN(i915) >= 5)
+	if (!INTEL_INFO(i915)->gpu_reset_clobbers_display)
 		intel_gpu_reset(i915, ALL_ENGINES);
 
 	for_each_engine(engine, i915, id) {
@@ -811,31 +888,35 @@ void i915_gem_set_wedged(struct drm_i915_private *i915)
 	 * either this call here to intel_engine_write_global_seqno, or the one
 	 * in nop_submit_request.
 	 */
-	synchronize_rcu();
+	synchronize_rcu_expedited();
 
 	/* Mark all executing requests as skipped */
 	for_each_engine(engine, i915, id)
 		engine->cancel_requests(engine);
 
-	for_each_engine(engine, i915, id) {
-		reset_finish_engine(engine);
-		intel_engine_signal_breadcrumbs(engine);
-	}
+	reset_finish(i915);
 
 	smp_mb__before_atomic();
 	set_bit(I915_WEDGED, &error->flags);
 
 	GEM_TRACE("end\n");
-	mutex_unlock(&error->wedge_mutex);
+}
 
-	wake_up_all(&error->reset_queue);
+void i915_gem_set_wedged(struct drm_i915_private *i915)
+{
+	struct i915_gpu_error *error = &i915->gpu_error;
+	intel_wakeref_t wakeref;
+
+	mutex_lock(&error->wedge_mutex);
+	with_intel_runtime_pm(i915, wakeref)
+		__i915_gem_set_wedged(i915);
+	mutex_unlock(&error->wedge_mutex);
 }
 
-bool i915_gem_unset_wedged(struct drm_i915_private *i915)
+static bool __i915_gem_unset_wedged(struct drm_i915_private *i915)
 {
 	struct i915_gpu_error *error = &i915->gpu_error;
 	struct i915_timeline *tl;
-	bool ret = false;
 
 	if (!test_bit(I915_WEDGED, &error->flags))
 		return true;
@@ -843,8 +924,6 @@ bool i915_gem_unset_wedged(struct drm_i915_private *i915)
 	if (!i915->gt.scratch) /* Never full initialised, recovery impossible */
 		return false;
 
-	mutex_lock(&error->wedge_mutex);
-
 	GEM_TRACE("start\n");
 
 	/*
@@ -860,30 +939,20 @@ bool i915_gem_unset_wedged(struct drm_i915_private *i915)
 	mutex_lock(&i915->gt.timelines.mutex);
 	list_for_each_entry(tl, &i915->gt.timelines.active_list, link) {
 		struct i915_request *rq;
-		long timeout;
 
 		rq = i915_active_request_get_unlocked(&tl->last_request);
 		if (!rq)
 			continue;
 
 		/*
-		 * We can't use our normal waiter as we want to
-		 * avoid recursively trying to handle the current
-		 * reset. The basic dma_fence_default_wait() installs
-		 * a callback for dma_fence_signal(), which is
-		 * triggered by our nop handler (indirectly, the
-		 * callback enables the signaler thread which is
-		 * woken by the nop_submit_request() advancing the seqno
-		 * and when the seqno passes the fence, the signaler
-		 * then signals the fence waking us up).
+		 * All internal dependencies (i915_requests) will have
+		 * been flushed by the set-wedge, but we may be stuck waiting
+		 * for external fences. These should all be capped to 10s
+		 * (I915_FENCE_TIMEOUT) so this wait should not be unbounded
+		 * in the worst case.
 		 */
-		timeout = dma_fence_default_wait(&rq->fence, true,
-						 MAX_SCHEDULE_TIMEOUT);
+		dma_fence_default_wait(&rq->fence, false, MAX_SCHEDULE_TIMEOUT);
 		i915_request_put(rq);
-		if (timeout < 0) {
-			mutex_unlock(&i915->gt.timelines.mutex);
-			goto unlock;
-		}
 	}
 	mutex_unlock(&i915->gt.timelines.mutex);
 
@@ -904,57 +973,38 @@ bool i915_gem_unset_wedged(struct drm_i915_private *i915)
 
 	smp_mb__before_atomic(); /* complete takeover before enabling execbuf */
 	clear_bit(I915_WEDGED, &i915->gpu_error.flags);
-	ret = true;
-unlock:
-	mutex_unlock(&i915->gpu_error.wedge_mutex);
 
-	return ret;
+	return true;
 }
 
-struct __i915_reset {
-	struct drm_i915_private *i915;
-	unsigned int stalled_mask;
-};
-
-static int __i915_reset__BKL(void *data)
+bool i915_gem_unset_wedged(struct drm_i915_private *i915)
 {
-	struct __i915_reset *arg = data;
-	int err;
+	struct i915_gpu_error *error = &i915->gpu_error;
+	bool result;
 
-	err = intel_gpu_reset(arg->i915, ALL_ENGINES);
-	if (err)
-		return err;
+	mutex_lock(&error->wedge_mutex);
+	result = __i915_gem_unset_wedged(i915);
+	mutex_unlock(&error->wedge_mutex);
 
-	return gt_reset(arg->i915, arg->stalled_mask);
+	return result;
 }
 
-#if RESET_UNDER_STOP_MACHINE
-/*
- * XXX An alternative to using stop_machine would be to park only the
- * processes that have a GGTT mmap. By remote parking the threads (SIGSTOP)
- * we should be able to prevent their memmory accesses via the lost fence
- * registers over the course of the reset without the potential recursive
- * of mutexes between the pagefault handler and reset.
- *
- * See igt/gem_mmap_gtt/hang
- */
-#define __do_reset(fn, arg) stop_machine(fn, arg, NULL)
-#else
-#define __do_reset(fn, arg) fn(arg)
-#endif
-
-static int do_reset(struct drm_i915_private *i915, unsigned int stalled_mask)
+static int do_reset(struct drm_i915_private *i915,
+		    intel_engine_mask_t stalled_mask)
 {
-	struct __i915_reset arg = { i915, stalled_mask };
 	int err, i;
 
-	err = __do_reset(__i915_reset__BKL, &arg);
+	gt_revoke(i915);
+
+	err = intel_gpu_reset(i915, ALL_ENGINES);
 	for (i = 0; err && i < RESET_MAX_RETRIES; i++) {
-		msleep(100);
-		err = __do_reset(__i915_reset__BKL, &arg);
+		msleep(10 * (i + 1));
+		err = intel_gpu_reset(i915, ALL_ENGINES);
 	}
+	if (err)
+		return err;
 
-	return err;
+	return gt_reset(i915, stalled_mask);
 }
 
 /**
@@ -966,8 +1016,6 @@ static int do_reset(struct drm_i915_private *i915, unsigned int stalled_mask)
  * Reset the chip.  Useful if a hang is detected. Marks the device as wedged
  * on failure.
  *
- * Caller must hold the struct_mutex.
- *
  * Procedure is fairly simple:
  *   - reset the chip using the reset reg
  *   - re-init context state
@@ -977,7 +1025,7 @@ static int do_reset(struct drm_i915_private *i915, unsigned int stalled_mask)
  *   - re-init display
  */
 void i915_reset(struct drm_i915_private *i915,
-		unsigned int stalled_mask,
+		intel_engine_mask_t stalled_mask,
 		const char *reason)
 {
 	struct i915_gpu_error *error = &i915->gpu_error;
@@ -990,7 +1038,7 @@ void i915_reset(struct drm_i915_private *i915,
 	GEM_BUG_ON(!test_bit(I915_RESET_BACKOFF, &error->flags));
 
 	/* Clear any previous failed attempts at recovery. Time to try again. */
-	if (!i915_gem_unset_wedged(i915))
+	if (!__i915_gem_unset_wedged(i915))
 		return;
 
 	if (reason)
@@ -1007,11 +1055,17 @@ void i915_reset(struct drm_i915_private *i915,
 		goto error;
 	}
 
+	if (INTEL_INFO(i915)->gpu_reset_clobbers_display)
+		intel_runtime_pm_disable_interrupts(i915);
+
 	if (do_reset(i915, stalled_mask)) {
 		dev_err(i915->drm.dev, "Failed to reset chip\n");
 		goto taint;
 	}
 
+	if (INTEL_INFO(i915)->gpu_reset_clobbers_display)
+		intel_runtime_pm_enable_interrupts(i915);
+
 	intel_overlay_reset(i915);
 
 	/*
@@ -1033,7 +1087,7 @@ void i915_reset(struct drm_i915_private *i915,
 
 finish:
 	reset_finish(i915);
-	if (!i915_terminally_wedged(error))
+	if (!__i915_wedged(error))
 		reset_restart(i915);
 	return;
 
@@ -1052,14 +1106,14 @@ taint:
 	 */
 	add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
 error:
-	i915_gem_set_wedged(i915);
+	__i915_gem_set_wedged(i915);
 	goto finish;
 }
 
 static inline int intel_gt_reset_engine(struct drm_i915_private *i915,
 					struct intel_engine_cs *engine)
 {
-	return intel_gpu_reset(i915, intel_engine_flag(engine));
+	return intel_gpu_reset(i915, engine->mask);
 }
 
 /**
@@ -1144,7 +1198,12 @@ static void i915_reset_device(struct drm_i915_private *i915,
 	i915_wedge_on_timeout(&w, i915, 5 * HZ) {
 		intel_prepare_reset(i915);
 
+		/* Flush everyone using a resource about to be clobbered */
+		synchronize_srcu_expedited(&error->reset_backoff_srcu);
+
+		mutex_lock(&error->wedge_mutex);
 		i915_reset(i915, engine_mask, reason);
+		mutex_unlock(&error->wedge_mutex);
 
 		intel_finish_reset(i915);
 	}
@@ -1153,44 +1212,50 @@ static void i915_reset_device(struct drm_i915_private *i915,
 		kobject_uevent_env(kobj, KOBJ_CHANGE, reset_done_event);
 }
 
-void i915_clear_error_registers(struct drm_i915_private *dev_priv)
+static void clear_register(struct intel_uncore *uncore, i915_reg_t reg)
 {
+	intel_uncore_rmw(uncore, reg, 0, 0);
+}
+
+void i915_clear_error_registers(struct drm_i915_private *i915)
+{
+	struct intel_uncore *uncore = &i915->uncore;
 	u32 eir;
 
-	if (!IS_GEN(dev_priv, 2))
-		I915_WRITE(PGTBL_ER, I915_READ(PGTBL_ER));
+	if (!IS_GEN(i915, 2))
+		clear_register(uncore, PGTBL_ER);
 
-	if (INTEL_GEN(dev_priv) < 4)
-		I915_WRITE(IPEIR, I915_READ(IPEIR));
+	if (INTEL_GEN(i915) < 4)
+		clear_register(uncore, IPEIR(RENDER_RING_BASE));
 	else
-		I915_WRITE(IPEIR_I965, I915_READ(IPEIR_I965));
+		clear_register(uncore, IPEIR_I965);
 
-	I915_WRITE(EIR, I915_READ(EIR));
-	eir = I915_READ(EIR);
+	clear_register(uncore, EIR);
+	eir = intel_uncore_read(uncore, EIR);
 	if (eir) {
 		/*
 		 * some errors might have become stuck,
 		 * mask them.
 		 */
 		DRM_DEBUG_DRIVER("EIR stuck: 0x%08x, masking\n", eir);
-		I915_WRITE(EMR, I915_READ(EMR) | eir);
-		I915_WRITE(IIR, I915_MASTER_ERROR_INTERRUPT);
+		rmw_set(uncore, EMR, eir);
+		intel_uncore_write(uncore, GEN2_IIR,
+				   I915_MASTER_ERROR_INTERRUPT);
 	}
 
-	if (INTEL_GEN(dev_priv) >= 8) {
-		I915_WRITE(GEN8_RING_FAULT_REG,
-			   I915_READ(GEN8_RING_FAULT_REG) & ~RING_FAULT_VALID);
-		POSTING_READ(GEN8_RING_FAULT_REG);
-	} else if (INTEL_GEN(dev_priv) >= 6) {
+	if (INTEL_GEN(i915) >= 8) {
+		rmw_clear(uncore, GEN8_RING_FAULT_REG, RING_FAULT_VALID);
+		intel_uncore_posting_read(uncore, GEN8_RING_FAULT_REG);
+	} else if (INTEL_GEN(i915) >= 6) {
 		struct intel_engine_cs *engine;
 		enum intel_engine_id id;
 
-		for_each_engine(engine, dev_priv, id) {
-			I915_WRITE(RING_FAULT_REG(engine),
-				   I915_READ(RING_FAULT_REG(engine)) &
-				   ~RING_FAULT_VALID);
+		for_each_engine(engine, i915, id) {
+			rmw_clear(uncore,
+				  RING_FAULT_REG(engine), RING_FAULT_VALID);
+			intel_uncore_posting_read(uncore,
+						  RING_FAULT_REG(engine));
 		}
-		POSTING_READ(RING_FAULT_REG(dev_priv->engine[RCS]));
 	}
 }
 
@@ -1208,13 +1273,14 @@ void i915_clear_error_registers(struct drm_i915_private *dev_priv)
  * of a ring dump etc.).
  */
 void i915_handle_error(struct drm_i915_private *i915,
-		       u32 engine_mask,
+		       intel_engine_mask_t engine_mask,
 		       unsigned long flags,
 		       const char *fmt, ...)
 {
+	struct i915_gpu_error *error = &i915->gpu_error;
 	struct intel_engine_cs *engine;
 	intel_wakeref_t wakeref;
-	unsigned int tmp;
+	intel_engine_mask_t tmp;
 	char error_msg[80];
 	char *msg = NULL;
 
@@ -1237,7 +1303,7 @@ void i915_handle_error(struct drm_i915_private *i915,
 	 */
 	wakeref = intel_runtime_pm_get(i915);
 
-	engine_mask &= INTEL_INFO(i915)->ring_mask;
+	engine_mask &= INTEL_INFO(i915)->engine_mask;
 
 	if (flags & I915_ERROR_CAPTURE) {
 		i915_capture_error_state(i915, engine_mask, msg);
@@ -1248,20 +1314,19 @@ void i915_handle_error(struct drm_i915_private *i915,
 	 * Try engine reset when available. We fall back to full reset if
 	 * single reset fails.
 	 */
-	if (intel_has_reset_engine(i915) &&
-	    !i915_terminally_wedged(&i915->gpu_error)) {
+	if (intel_has_reset_engine(i915) && !__i915_wedged(error)) {
 		for_each_engine_masked(engine, i915, engine_mask, tmp) {
 			BUILD_BUG_ON(I915_RESET_MODESET >= I915_RESET_ENGINE);
 			if (test_and_set_bit(I915_RESET_ENGINE + engine->id,
-					     &i915->gpu_error.flags))
+					     &error->flags))
 				continue;
 
 			if (i915_reset_engine(engine, msg) == 0)
-				engine_mask &= ~intel_engine_flag(engine);
+				engine_mask &= ~engine->mask;
 
 			clear_bit(I915_RESET_ENGINE + engine->id,
-				  &i915->gpu_error.flags);
-			wake_up_bit(&i915->gpu_error.flags,
+				  &error->flags);
+			wake_up_bit(&error->flags,
 				    I915_RESET_ENGINE + engine->id);
 		}
 	}
@@ -1270,18 +1335,20 @@ void i915_handle_error(struct drm_i915_private *i915,
 		goto out;
 
 	/* Full reset needs the mutex, stop any other user trying to do so. */
-	if (test_and_set_bit(I915_RESET_BACKOFF, &i915->gpu_error.flags)) {
-		wait_event(i915->gpu_error.reset_queue,
-			   !test_bit(I915_RESET_BACKOFF,
-				     &i915->gpu_error.flags));
-		goto out;
+	if (test_and_set_bit(I915_RESET_BACKOFF, &error->flags)) {
+		wait_event(error->reset_queue,
+			   !test_bit(I915_RESET_BACKOFF, &error->flags));
+		goto out; /* piggy-back on the other reset */
 	}
 
+	/* Make sure i915_reset_trylock() sees the I915_RESET_BACKOFF */
+	synchronize_rcu_expedited();
+
 	/* Prevent any other reset-engine attempt. */
 	for_each_engine(engine, i915, tmp) {
 		while (test_and_set_bit(I915_RESET_ENGINE + engine->id,
-					&i915->gpu_error.flags))
-			wait_on_bit(&i915->gpu_error.flags,
+					&error->flags))
+			wait_on_bit(&error->flags,
 				    I915_RESET_ENGINE + engine->id,
 				    TASK_UNINTERRUPTIBLE);
 	}
@@ -1290,16 +1357,74 @@ void i915_handle_error(struct drm_i915_private *i915,
 
 	for_each_engine(engine, i915, tmp) {
 		clear_bit(I915_RESET_ENGINE + engine->id,
-			  &i915->gpu_error.flags);
+			  &error->flags);
 	}
 
-	clear_bit(I915_RESET_BACKOFF, &i915->gpu_error.flags);
-	wake_up_all(&i915->gpu_error.reset_queue);
+	clear_bit(I915_RESET_BACKOFF, &error->flags);
+	wake_up_all(&error->reset_queue);
 
 out:
 	intel_runtime_pm_put(i915, wakeref);
 }
 
+int i915_reset_trylock(struct drm_i915_private *i915)
+{
+	struct i915_gpu_error *error = &i915->gpu_error;
+	int srcu;
+
+	might_lock(&error->reset_backoff_srcu);
+	might_sleep();
+
+	rcu_read_lock();
+	while (test_bit(I915_RESET_BACKOFF, &error->flags)) {
+		rcu_read_unlock();
+
+		if (wait_event_interruptible(error->reset_queue,
+					     !test_bit(I915_RESET_BACKOFF,
+						       &error->flags)))
+			return -EINTR;
+
+		rcu_read_lock();
+	}
+	srcu = srcu_read_lock(&error->reset_backoff_srcu);
+	rcu_read_unlock();
+
+	return srcu;
+}
+
+void i915_reset_unlock(struct drm_i915_private *i915, int tag)
+__releases(&i915->gpu_error.reset_backoff_srcu)
+{
+	struct i915_gpu_error *error = &i915->gpu_error;
+
+	srcu_read_unlock(&error->reset_backoff_srcu, tag);
+}
+
+int i915_terminally_wedged(struct drm_i915_private *i915)
+{
+	struct i915_gpu_error *error = &i915->gpu_error;
+
+	might_sleep();
+
+	if (!__i915_wedged(error))
+		return 0;
+
+	/* Reset still in progress? Maybe we will recover? */
+	if (!test_bit(I915_RESET_BACKOFF, &error->flags))
+		return -EIO;
+
+	/* XXX intel_reset_finish() still takes struct_mutex!!! */
+	if (mutex_is_locked(&i915->drm.struct_mutex))
+		return -EAGAIN;
+
+	if (wait_event_interruptible(error->reset_queue,
+				     !test_bit(I915_RESET_BACKOFF,
+					       &error->flags)))
+		return -EINTR;
+
+	return __i915_wedged(error) ? -EIO : 0;
+}
+
 bool i915_reset_flush(struct drm_i915_private *i915)
 {
 	int err;
diff --git a/drivers/gpu/drm/i915/i915_reset.h b/drivers/gpu/drm/i915/i915_reset.h
index f2d347f319df..3c0450289b8f 100644
--- a/drivers/gpu/drm/i915/i915_reset.h
+++ b/drivers/gpu/drm/i915/i915_reset.h
@@ -9,14 +9,18 @@
 
 #include <linux/compiler.h>
 #include <linux/types.h>
+#include <linux/srcu.h>
+
+#include "intel_engine_types.h"
 
 struct drm_i915_private;
+struct i915_request;
 struct intel_engine_cs;
 struct intel_guc;
 
 __printf(4, 5)
 void i915_handle_error(struct drm_i915_private *i915,
-		       u32 engine_mask,
+		       intel_engine_mask_t engine_mask,
 		       unsigned long flags,
 		       const char *fmt, ...);
 #define I915_ERROR_CAPTURE BIT(0)
@@ -24,7 +28,7 @@ void i915_handle_error(struct drm_i915_private *i915,
 void i915_clear_error_registers(struct drm_i915_private *i915);
 
 void i915_reset(struct drm_i915_private *i915,
-		unsigned int stalled_mask,
+		intel_engine_mask_t stalled_mask,
 		const char *reason);
 int i915_reset_engine(struct intel_engine_cs *engine,
 		      const char *reason);
@@ -32,10 +36,16 @@ int i915_reset_engine(struct intel_engine_cs *engine,
 void i915_reset_request(struct i915_request *rq, bool guilty);
 bool i915_reset_flush(struct drm_i915_private *i915);
 
+int __must_check i915_reset_trylock(struct drm_i915_private *i915);
+void i915_reset_unlock(struct drm_i915_private *i915, int tag);
+
+int i915_terminally_wedged(struct drm_i915_private *i915);
+
 bool intel_has_gpu_reset(struct drm_i915_private *i915);
 bool intel_has_reset_engine(struct drm_i915_private *i915);
 
-int intel_gpu_reset(struct drm_i915_private *i915, u32 engine_mask);
+int intel_gpu_reset(struct drm_i915_private *i915,
+		    intel_engine_mask_t engine_mask);
 
 int intel_reset_guc(struct drm_i915_private *i915);
 
diff --git a/drivers/gpu/drm/i915/i915_scheduler.c b/drivers/gpu/drm/i915/i915_scheduler.c
index 8bc042551692..39bc4f54e272 100644
--- a/drivers/gpu/drm/i915/i915_scheduler.c
+++ b/drivers/gpu/drm/i915/i915_scheduler.c
@@ -7,9 +7,16 @@
 #include <linux/mutex.h>
 
 #include "i915_drv.h"
+#include "i915_globals.h"
 #include "i915_request.h"
 #include "i915_scheduler.h"
 
+static struct i915_global_scheduler {
+	struct i915_global base;
+	struct kmem_cache *slab_dependencies;
+	struct kmem_cache *slab_priorities;
+} global;
+
 static DEFINE_SPINLOCK(schedule_lock);
 
 static const struct i915_request *
@@ -18,6 +25,11 @@ node_to_request(const struct i915_sched_node *node)
 	return container_of(node, const struct i915_request, sched);
 }
 
+static inline bool node_started(const struct i915_sched_node *node)
+{
+	return i915_request_started(node_to_request(node));
+}
+
 static inline bool node_signaled(const struct i915_sched_node *node)
 {
 	return i915_request_completed(node_to_request(node));
@@ -29,19 +41,20 @@ void i915_sched_node_init(struct i915_sched_node *node)
 	INIT_LIST_HEAD(&node->waiters_list);
 	INIT_LIST_HEAD(&node->link);
 	node->attr.priority = I915_PRIORITY_INVALID;
+	node->semaphores = 0;
+	node->flags = 0;
 }
 
 static struct i915_dependency *
-i915_dependency_alloc(struct drm_i915_private *i915)
+i915_dependency_alloc(void)
 {
-	return kmem_cache_alloc(i915->dependencies, GFP_KERNEL);
+	return kmem_cache_alloc(global.slab_dependencies, GFP_KERNEL);
 }
 
 static void
-i915_dependency_free(struct drm_i915_private *i915,
-		     struct i915_dependency *dep)
+i915_dependency_free(struct i915_dependency *dep)
 {
-	kmem_cache_free(i915->dependencies, dep);
+	kmem_cache_free(global.slab_dependencies, dep);
 }
 
 bool __i915_sched_node_add_dependency(struct i915_sched_node *node,
@@ -51,7 +64,7 @@ bool __i915_sched_node_add_dependency(struct i915_sched_node *node,
 {
 	bool ret = false;
 
-	spin_lock(&schedule_lock);
+	spin_lock_irq(&schedule_lock);
 
 	if (!node_signaled(signal)) {
 		INIT_LIST_HEAD(&dep->dfs_link);
@@ -60,39 +73,42 @@ bool __i915_sched_node_add_dependency(struct i915_sched_node *node,
 		dep->signaler = signal;
 		dep->flags = flags;
 
+		/* Keep track of whether anyone on this chain has a semaphore */
+		if (signal->flags & I915_SCHED_HAS_SEMAPHORE_CHAIN &&
+		    !node_started(signal))
+			node->flags |= I915_SCHED_HAS_SEMAPHORE_CHAIN;
+
 		ret = true;
 	}
 
-	spin_unlock(&schedule_lock);
+	spin_unlock_irq(&schedule_lock);
 
 	return ret;
 }
 
-int i915_sched_node_add_dependency(struct drm_i915_private *i915,
-				   struct i915_sched_node *node,
+int i915_sched_node_add_dependency(struct i915_sched_node *node,
 				   struct i915_sched_node *signal)
 {
 	struct i915_dependency *dep;
 
-	dep = i915_dependency_alloc(i915);
+	dep = i915_dependency_alloc();
 	if (!dep)
 		return -ENOMEM;
 
 	if (!__i915_sched_node_add_dependency(node, signal, dep,
 					      I915_DEPENDENCY_ALLOC))
-		i915_dependency_free(i915, dep);
+		i915_dependency_free(dep);
 
 	return 0;
 }
 
-void i915_sched_node_fini(struct drm_i915_private *i915,
-			  struct i915_sched_node *node)
+void i915_sched_node_fini(struct i915_sched_node *node)
 {
 	struct i915_dependency *dep, *tmp;
 
 	GEM_BUG_ON(!list_empty(&node->link));
 
-	spin_lock(&schedule_lock);
+	spin_lock_irq(&schedule_lock);
 
 	/*
 	 * Everyone we depended upon (the fences we wait to be signaled)
@@ -106,7 +122,7 @@ void i915_sched_node_fini(struct drm_i915_private *i915,
 
 		list_del(&dep->wait_link);
 		if (dep->flags & I915_DEPENDENCY_ALLOC)
-			i915_dependency_free(i915, dep);
+			i915_dependency_free(dep);
 	}
 
 	/* Remove ourselves from everyone who depends upon us */
@@ -116,10 +132,10 @@ void i915_sched_node_fini(struct drm_i915_private *i915,
 
 		list_del(&dep->signal_link);
 		if (dep->flags & I915_DEPENDENCY_ALLOC)
-			i915_dependency_free(i915, dep);
+			i915_dependency_free(dep);
 	}
 
-	spin_unlock(&schedule_lock);
+	spin_unlock_irq(&schedule_lock);
 }
 
 static inline struct i915_priolist *to_priolist(struct rb_node *rb)
@@ -193,7 +209,7 @@ find_priolist:
 	if (prio == I915_PRIORITY_NORMAL) {
 		p = &execlists->default_priolist;
 	} else {
-		p = kmem_cache_alloc(engine->i915->priorities, GFP_ATOMIC);
+		p = kmem_cache_alloc(global.slab_priorities, GFP_ATOMIC);
 		/* Convert an allocation failure to a priority bump */
 		if (unlikely(!p)) {
 			prio = I915_PRIORITY_NORMAL; /* recurses just once */
@@ -301,6 +317,10 @@ static void __i915_schedule(struct i915_request *rq,
 	list_for_each_entry(dep, &dfs, dfs_link) {
 		struct i915_sched_node *node = dep->signaler;
 
+		/* If we are already flying, we know we have no signalers */
+		if (node_started(node))
+			continue;
+
 		/*
 		 * Within an engine, there can be no cycle, but we may
 		 * refer to the same dependency chain multiple times
@@ -313,7 +333,6 @@ static void __i915_schedule(struct i915_request *rq,
 			if (node_signaled(p->signaler))
 				continue;
 
-			GEM_BUG_ON(p->signaler->attr.priority < node->attr.priority);
 			if (prio > READ_ONCE(p->signaler->attr.priority))
 				list_move_tail(&p->dfs_link, &dfs);
 		}
@@ -337,7 +356,7 @@ static void __i915_schedule(struct i915_request *rq,
 
 	memset(&cache, 0, sizeof(cache));
 	engine = rq->engine;
-	spin_lock_irq(&engine->timeline.lock);
+	spin_lock(&engine->timeline.lock);
 
 	/* Fifo and depth-first replacement ensure our deps execute before us */
 	list_for_each_entry_safe_reverse(dep, p, &dfs, dfs_link) {
@@ -388,30 +407,73 @@ static void __i915_schedule(struct i915_request *rq,
 		tasklet_hi_schedule(&engine->execlists.tasklet);
 	}
 
-	spin_unlock_irq(&engine->timeline.lock);
+	spin_unlock(&engine->timeline.lock);
 }
 
 void i915_schedule(struct i915_request *rq, const struct i915_sched_attr *attr)
 {
-	spin_lock(&schedule_lock);
+	spin_lock_irq(&schedule_lock);
 	__i915_schedule(rq, attr);
-	spin_unlock(&schedule_lock);
+	spin_unlock_irq(&schedule_lock);
 }
 
 void i915_schedule_bump_priority(struct i915_request *rq, unsigned int bump)
 {
 	struct i915_sched_attr attr;
+	unsigned long flags;
 
 	GEM_BUG_ON(bump & ~I915_PRIORITY_MASK);
 
 	if (READ_ONCE(rq->sched.attr.priority) == I915_PRIORITY_INVALID)
 		return;
 
-	spin_lock_bh(&schedule_lock);
+	spin_lock_irqsave(&schedule_lock, flags);
 
 	attr = rq->sched.attr;
 	attr.priority |= bump;
 	__i915_schedule(rq, &attr);
 
-	spin_unlock_bh(&schedule_lock);
+	spin_unlock_irqrestore(&schedule_lock, flags);
+}
+
+void __i915_priolist_free(struct i915_priolist *p)
+{
+	kmem_cache_free(global.slab_priorities, p);
+}
+
+static void i915_global_scheduler_shrink(void)
+{
+	kmem_cache_shrink(global.slab_dependencies);
+	kmem_cache_shrink(global.slab_priorities);
+}
+
+static void i915_global_scheduler_exit(void)
+{
+	kmem_cache_destroy(global.slab_dependencies);
+	kmem_cache_destroy(global.slab_priorities);
+}
+
+static struct i915_global_scheduler global = { {
+	.shrink = i915_global_scheduler_shrink,
+	.exit = i915_global_scheduler_exit,
+} };
+
+int __init i915_global_scheduler_init(void)
+{
+	global.slab_dependencies = KMEM_CACHE(i915_dependency,
+					      SLAB_HWCACHE_ALIGN);
+	if (!global.slab_dependencies)
+		return -ENOMEM;
+
+	global.slab_priorities = KMEM_CACHE(i915_priolist,
+					    SLAB_HWCACHE_ALIGN);
+	if (!global.slab_priorities)
+		goto err_priorities;
+
+	i915_global_register(&global.base);
+	return 0;
+
+err_priorities:
+	kmem_cache_destroy(global.slab_priorities);
+	return -ENOMEM;
 }
diff --git a/drivers/gpu/drm/i915/i915_scheduler.h b/drivers/gpu/drm/i915/i915_scheduler.h
index dbe9cb7ecd82..07d243acf553 100644
--- a/drivers/gpu/drm/i915/i915_scheduler.h
+++ b/drivers/gpu/drm/i915/i915_scheduler.h
@@ -8,80 +8,22 @@
 #define _I915_SCHEDULER_H_
 
 #include <linux/bitops.h>
+#include <linux/list.h>
 #include <linux/kernel.h>
 
-#include <uapi/drm/i915_drm.h>
+#include "i915_scheduler_types.h"
 
-struct drm_i915_private;
-struct i915_request;
-struct intel_engine_cs;
+#define priolist_for_each_request(it, plist, idx) \
+	for (idx = 0; idx < ARRAY_SIZE((plist)->requests); idx++) \
+		list_for_each_entry(it, &(plist)->requests[idx], sched.link)
 
-enum {
-	I915_PRIORITY_MIN = I915_CONTEXT_MIN_USER_PRIORITY - 1,
-	I915_PRIORITY_NORMAL = I915_CONTEXT_DEFAULT_PRIORITY,
-	I915_PRIORITY_MAX = I915_CONTEXT_MAX_USER_PRIORITY + 1,
-
-	I915_PRIORITY_INVALID = INT_MIN
-};
-
-#define I915_USER_PRIORITY_SHIFT 2
-#define I915_USER_PRIORITY(x) ((x) << I915_USER_PRIORITY_SHIFT)
-
-#define I915_PRIORITY_COUNT BIT(I915_USER_PRIORITY_SHIFT)
-#define I915_PRIORITY_MASK (I915_PRIORITY_COUNT - 1)
-
-#define I915_PRIORITY_WAIT	((u8)BIT(0))
-#define I915_PRIORITY_NEWCLIENT	((u8)BIT(1))
-
-struct i915_sched_attr {
-	/**
-	 * @priority: execution and service priority
-	 *
-	 * All clients are equal, but some are more equal than others!
-	 *
-	 * Requests from a context with a greater (more positive) value of
-	 * @priority will be executed before those with a lower @priority
-	 * value, forming a simple QoS.
-	 *
-	 * The &drm_i915_private.kernel_context is assigned the lowest priority.
-	 */
-	int priority;
-};
-
-/*
- * "People assume that time is a strict progression of cause to effect, but
- * actually, from a nonlinear, non-subjective viewpoint, it's more like a big
- * ball of wibbly-wobbly, timey-wimey ... stuff." -The Doctor, 2015
- *
- * Requests exist in a complex web of interdependencies. Each request
- * has to wait for some other request to complete before it is ready to be run
- * (e.g. we have to wait until the pixels have been rendering into a texture
- * before we can copy from it). We track the readiness of a request in terms
- * of fences, but we also need to keep the dependency tree for the lifetime
- * of the request (beyond the life of an individual fence). We use the tree
- * at various points to reorder the requests whilst keeping the requests
- * in order with respect to their various dependencies.
- *
- * There is no active component to the "scheduler". As we know the dependency
- * DAG of each request, we are able to insert it into a sorted queue when it
- * is ready, and are able to reorder its portion of the graph to accommodate
- * dynamic priority changes.
- */
-struct i915_sched_node {
-	struct list_head signalers_list; /* those before us, we depend upon */
-	struct list_head waiters_list; /* those after us, they depend upon us */
-	struct list_head link;
-	struct i915_sched_attr attr;
-};
-
-struct i915_dependency {
-	struct i915_sched_node *signaler;
-	struct list_head signal_link;
-	struct list_head wait_link;
-	struct list_head dfs_link;
-	unsigned long flags;
-#define I915_DEPENDENCY_ALLOC BIT(0)
-};
+#define priolist_for_each_request_consume(it, n, plist, idx) \
+	for (; \
+	     (plist)->used ? (idx = __ffs((plist)->used)), 1 : 0; \
+	     (plist)->used &= ~BIT(idx)) \
+		list_for_each_entry_safe(it, n, \
+					 &(plist)->requests[idx], \
+					 sched.link)
 
 void i915_sched_node_init(struct i915_sched_node *node);
 
@@ -90,12 +32,10 @@ bool __i915_sched_node_add_dependency(struct i915_sched_node *node,
 				      struct i915_dependency *dep,
 				      unsigned long flags);
 
-int i915_sched_node_add_dependency(struct drm_i915_private *i915,
-				   struct i915_sched_node *node,
+int i915_sched_node_add_dependency(struct i915_sched_node *node,
 				   struct i915_sched_node *signal);
 
-void i915_sched_node_fini(struct drm_i915_private *i915,
-			  struct i915_sched_node *node);
+void i915_sched_node_fini(struct i915_sched_node *node);
 
 void i915_schedule(struct i915_request *request,
 		   const struct i915_sched_attr *attr);
@@ -105,4 +45,11 @@ void i915_schedule_bump_priority(struct i915_request *rq, unsigned int bump);
 struct list_head *
 i915_sched_lookup_priolist(struct intel_engine_cs *engine, int prio);
 
+void __i915_priolist_free(struct i915_priolist *p);
+static inline void i915_priolist_free(struct i915_priolist *p)
+{
+	if (p->priority != I915_PRIORITY_NORMAL)
+		__i915_priolist_free(p);
+}
+
 #endif /* _I915_SCHEDULER_H_ */
diff --git a/drivers/gpu/drm/i915/i915_scheduler_types.h b/drivers/gpu/drm/i915/i915_scheduler_types.h
new file mode 100644
index 000000000000..f1af3916a808
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_scheduler_types.h
@@ -0,0 +1,72 @@
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2018 Intel Corporation
+ */
+
+#ifndef _I915_SCHEDULER_TYPES_H_
+#define _I915_SCHEDULER_TYPES_H_
+
+#include <linux/list.h>
+
+#include "i915_priolist_types.h"
+#include "intel_engine_types.h"
+
+struct drm_i915_private;
+struct i915_request;
+struct intel_engine_cs;
+
+struct i915_sched_attr {
+	/**
+	 * @priority: execution and service priority
+	 *
+	 * All clients are equal, but some are more equal than others!
+	 *
+	 * Requests from a context with a greater (more positive) value of
+	 * @priority will be executed before those with a lower @priority
+	 * value, forming a simple QoS.
+	 *
+	 * The &drm_i915_private.kernel_context is assigned the lowest priority.
+	 */
+	int priority;
+};
+
+/*
+ * "People assume that time is a strict progression of cause to effect, but
+ * actually, from a nonlinear, non-subjective viewpoint, it's more like a big
+ * ball of wibbly-wobbly, timey-wimey ... stuff." -The Doctor, 2015
+ *
+ * Requests exist in a complex web of interdependencies. Each request
+ * has to wait for some other request to complete before it is ready to be run
+ * (e.g. we have to wait until the pixels have been rendering into a texture
+ * before we can copy from it). We track the readiness of a request in terms
+ * of fences, but we also need to keep the dependency tree for the lifetime
+ * of the request (beyond the life of an individual fence). We use the tree
+ * at various points to reorder the requests whilst keeping the requests
+ * in order with respect to their various dependencies.
+ *
+ * There is no active component to the "scheduler". As we know the dependency
+ * DAG of each request, we are able to insert it into a sorted queue when it
+ * is ready, and are able to reorder its portion of the graph to accommodate
+ * dynamic priority changes.
+ */
+struct i915_sched_node {
+	struct list_head signalers_list; /* those before us, we depend upon */
+	struct list_head waiters_list; /* those after us, they depend upon us */
+	struct list_head link;
+	struct i915_sched_attr attr;
+	unsigned int flags;
+#define I915_SCHED_HAS_SEMAPHORE_CHAIN	BIT(0)
+	intel_engine_mask_t semaphores;
+};
+
+struct i915_dependency {
+	struct i915_sched_node *signaler;
+	struct list_head signal_link;
+	struct list_head wait_link;
+	struct list_head dfs_link;
+	unsigned long flags;
+#define I915_DEPENDENCY_ALLOC BIT(0)
+};
+
+#endif /* _I915_SCHEDULER_TYPES_H_ */
diff --git a/drivers/gpu/drm/i915/i915_suspend.c b/drivers/gpu/drm/i915/i915_suspend.c
index d2f2a9c2fabd..95f3dab1b229 100644
--- a/drivers/gpu/drm/i915/i915_suspend.c
+++ b/drivers/gpu/drm/i915/i915_suspend.c
@@ -25,8 +25,10 @@
  */
 
 #include <drm/i915_drm.h>
-#include "intel_drv.h"
+
 #include "i915_reg.h"
+#include "intel_drv.h"
+#include "intel_fbc.h"
 
 static void i915_save_display(struct drm_i915_private *dev_priv)
 {
diff --git a/drivers/gpu/drm/i915/i915_sw_fence.c b/drivers/gpu/drm/i915/i915_sw_fence.c
index 7c58b049ecb5..5387aafd3424 100644
--- a/drivers/gpu/drm/i915/i915_sw_fence.c
+++ b/drivers/gpu/drm/i915/i915_sw_fence.c
@@ -192,7 +192,7 @@ static void __i915_sw_fence_complete(struct i915_sw_fence *fence,
 	__i915_sw_fence_notify(fence, FENCE_FREE);
 }
 
-static void i915_sw_fence_complete(struct i915_sw_fence *fence)
+void i915_sw_fence_complete(struct i915_sw_fence *fence)
 {
 	debug_fence_assert(fence);
 
@@ -202,7 +202,7 @@ static void i915_sw_fence_complete(struct i915_sw_fence *fence)
 	__i915_sw_fence_complete(fence, NULL);
 }
 
-static void i915_sw_fence_await(struct i915_sw_fence *fence)
+void i915_sw_fence_await(struct i915_sw_fence *fence)
 {
 	debug_fence_assert(fence);
 	WARN_ON(atomic_inc_return(&fence->pending) <= 1);
@@ -359,11 +359,6 @@ int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence,
 	return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp);
 }
 
-struct i915_sw_dma_fence_cb {
-	struct dma_fence_cb base;
-	struct i915_sw_fence *fence;
-};
-
 struct i915_sw_dma_fence_cb_timer {
 	struct i915_sw_dma_fence_cb base;
 	struct dma_fence *dma;
@@ -480,6 +475,40 @@ int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
 	return ret;
 }
 
+static void __dma_i915_sw_fence_wake(struct dma_fence *dma,
+				     struct dma_fence_cb *data)
+{
+	struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base);
+
+	i915_sw_fence_complete(cb->fence);
+}
+
+int __i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
+				    struct dma_fence *dma,
+				    struct i915_sw_dma_fence_cb *cb)
+{
+	int ret;
+
+	debug_fence_assert(fence);
+
+	if (dma_fence_is_signaled(dma))
+		return 0;
+
+	cb->fence = fence;
+	i915_sw_fence_await(fence);
+
+	ret = dma_fence_add_callback(dma, &cb->base, __dma_i915_sw_fence_wake);
+	if (ret == 0) {
+		ret = 1;
+	} else {
+		i915_sw_fence_complete(fence);
+		if (ret == -ENOENT) /* fence already signaled */
+			ret = 0;
+	}
+
+	return ret;
+}
+
 int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,
 				    struct reservation_object *resv,
 				    const struct dma_fence_ops *exclude,
diff --git a/drivers/gpu/drm/i915/i915_sw_fence.h b/drivers/gpu/drm/i915/i915_sw_fence.h
index 0e055ea0179f..9cb5c3b307a6 100644
--- a/drivers/gpu/drm/i915/i915_sw_fence.h
+++ b/drivers/gpu/drm/i915/i915_sw_fence.h
@@ -9,14 +9,13 @@
 #ifndef _I915_SW_FENCE_H_
 #define _I915_SW_FENCE_H_
 
+#include <linux/dma-fence.h>
 #include <linux/gfp.h>
 #include <linux/kref.h>
 #include <linux/notifier.h> /* for NOTIFY_DONE */
 #include <linux/wait.h>
 
 struct completion;
-struct dma_fence;
-struct dma_fence_ops;
 struct reservation_object;
 
 struct i915_sw_fence {
@@ -68,10 +67,20 @@ int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
 int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence,
 				     struct i915_sw_fence *after,
 				     gfp_t gfp);
+
+struct i915_sw_dma_fence_cb {
+	struct dma_fence_cb base;
+	struct i915_sw_fence *fence;
+};
+
+int __i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
+				    struct dma_fence *dma,
+				    struct i915_sw_dma_fence_cb *cb);
 int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
 				  struct dma_fence *dma,
 				  unsigned long timeout,
 				  gfp_t gfp);
+
 int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,
 				    struct reservation_object *resv,
 				    const struct dma_fence_ops *exclude,
@@ -79,6 +88,9 @@ int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,
 				    unsigned long timeout,
 				    gfp_t gfp);
 
+void i915_sw_fence_await(struct i915_sw_fence *fence);
+void i915_sw_fence_complete(struct i915_sw_fence *fence);
+
 static inline bool i915_sw_fence_signaled(const struct i915_sw_fence *fence)
 {
 	return atomic_read(&fence->pending) <= 0;
diff --git a/drivers/gpu/drm/i915/i915_timeline.c b/drivers/gpu/drm/i915/i915_timeline.c
index b2202d2e58a2..5fbea0892f33 100644
--- a/drivers/gpu/drm/i915/i915_timeline.c
+++ b/drivers/gpu/drm/i915/i915_timeline.c
@@ -6,19 +6,32 @@
 
 #include "i915_drv.h"
 
-#include "i915_timeline.h"
+#include "i915_active.h"
 #include "i915_syncmap.h"
+#include "i915_timeline.h"
+
+#define ptr_set_bit(ptr, bit) ((typeof(ptr))((unsigned long)(ptr) | BIT(bit)))
+#define ptr_test_bit(ptr, bit) ((unsigned long)(ptr) & BIT(bit))
 
 struct i915_timeline_hwsp {
-	struct i915_vma *vma;
+	struct i915_gt_timelines *gt;
 	struct list_head free_link;
+	struct i915_vma *vma;
 	u64 free_bitmap;
 };
 
-static inline struct i915_timeline_hwsp *
-i915_timeline_hwsp(const struct i915_timeline *tl)
+struct i915_timeline_cacheline {
+	struct i915_active active;
+	struct i915_timeline_hwsp *hwsp;
+	void *vaddr;
+#define CACHELINE_BITS 6
+#define CACHELINE_FREE CACHELINE_BITS
+};
+
+static inline struct drm_i915_private *
+hwsp_to_i915(struct i915_timeline_hwsp *hwsp)
 {
-	return tl->hwsp_ggtt->private;
+	return container_of(hwsp->gt, struct drm_i915_private, gt.timelines);
 }
 
 static struct i915_vma *__hwsp_alloc(struct drm_i915_private *i915)
@@ -71,6 +84,7 @@ hwsp_alloc(struct i915_timeline *timeline, unsigned int *cacheline)
 		vma->private = hwsp;
 		hwsp->vma = vma;
 		hwsp->free_bitmap = ~0ull;
+		hwsp->gt = gt;
 
 		spin_lock(&gt->hwsp_lock);
 		list_add(&hwsp->free_link, &gt->hwsp_free_list);
@@ -88,14 +102,9 @@ hwsp_alloc(struct i915_timeline *timeline, unsigned int *cacheline)
 	return hwsp->vma;
 }
 
-static void hwsp_free(struct i915_timeline *timeline)
+static void __idle_hwsp_free(struct i915_timeline_hwsp *hwsp, int cacheline)
 {
-	struct i915_gt_timelines *gt = &timeline->i915->gt.timelines;
-	struct i915_timeline_hwsp *hwsp;
-
-	hwsp = i915_timeline_hwsp(timeline);
-	if (!hwsp) /* leave global HWSP alone! */
-		return;
+	struct i915_gt_timelines *gt = hwsp->gt;
 
 	spin_lock(&gt->hwsp_lock);
 
@@ -103,7 +112,8 @@ static void hwsp_free(struct i915_timeline *timeline)
 	if (!hwsp->free_bitmap)
 		list_add_tail(&hwsp->free_link, &gt->hwsp_free_list);
 
-	hwsp->free_bitmap |= BIT_ULL(timeline->hwsp_offset / CACHELINE_BYTES);
+	GEM_BUG_ON(cacheline >= BITS_PER_TYPE(hwsp->free_bitmap));
+	hwsp->free_bitmap |= BIT_ULL(cacheline);
 
 	/* And if no one is left using it, give the page back to the system */
 	if (hwsp->free_bitmap == ~0ull) {
@@ -115,9 +125,78 @@ static void hwsp_free(struct i915_timeline *timeline)
 	spin_unlock(&gt->hwsp_lock);
 }
 
+static void __idle_cacheline_free(struct i915_timeline_cacheline *cl)
+{
+	GEM_BUG_ON(!i915_active_is_idle(&cl->active));
+
+	i915_gem_object_unpin_map(cl->hwsp->vma->obj);
+	i915_vma_put(cl->hwsp->vma);
+	__idle_hwsp_free(cl->hwsp, ptr_unmask_bits(cl->vaddr, CACHELINE_BITS));
+
+	i915_active_fini(&cl->active);
+	kfree(cl);
+}
+
+static void __cacheline_retire(struct i915_active *active)
+{
+	struct i915_timeline_cacheline *cl =
+		container_of(active, typeof(*cl), active);
+
+	i915_vma_unpin(cl->hwsp->vma);
+	if (ptr_test_bit(cl->vaddr, CACHELINE_FREE))
+		__idle_cacheline_free(cl);
+}
+
+static struct i915_timeline_cacheline *
+cacheline_alloc(struct i915_timeline_hwsp *hwsp, unsigned int cacheline)
+{
+	struct i915_timeline_cacheline *cl;
+	void *vaddr;
+
+	GEM_BUG_ON(cacheline >= BIT(CACHELINE_BITS));
+
+	cl = kmalloc(sizeof(*cl), GFP_KERNEL);
+	if (!cl)
+		return ERR_PTR(-ENOMEM);
+
+	vaddr = i915_gem_object_pin_map(hwsp->vma->obj, I915_MAP_WB);
+	if (IS_ERR(vaddr)) {
+		kfree(cl);
+		return ERR_CAST(vaddr);
+	}
+
+	i915_vma_get(hwsp->vma);
+	cl->hwsp = hwsp;
+	cl->vaddr = page_pack_bits(vaddr, cacheline);
+
+	i915_active_init(hwsp_to_i915(hwsp), &cl->active, __cacheline_retire);
+
+	return cl;
+}
+
+static void cacheline_acquire(struct i915_timeline_cacheline *cl)
+{
+	if (cl && i915_active_acquire(&cl->active))
+		__i915_vma_pin(cl->hwsp->vma);
+}
+
+static void cacheline_release(struct i915_timeline_cacheline *cl)
+{
+	if (cl)
+		i915_active_release(&cl->active);
+}
+
+static void cacheline_free(struct i915_timeline_cacheline *cl)
+{
+	GEM_BUG_ON(ptr_test_bit(cl->vaddr, CACHELINE_FREE));
+	cl->vaddr = ptr_set_bit(cl->vaddr, CACHELINE_FREE);
+
+	if (i915_active_is_idle(&cl->active))
+		__idle_cacheline_free(cl);
+}
+
 int i915_timeline_init(struct drm_i915_private *i915,
 		       struct i915_timeline *timeline,
-		       const char *name,
 		       struct i915_vma *hwsp)
 {
 	void *vaddr;
@@ -133,37 +212,47 @@ int i915_timeline_init(struct drm_i915_private *i915,
 	BUILD_BUG_ON(KSYNCMAP < I915_NUM_ENGINES);
 
 	timeline->i915 = i915;
-	timeline->name = name;
 	timeline->pin_count = 0;
 	timeline->has_initial_breadcrumb = !hwsp;
+	timeline->hwsp_cacheline = NULL;
 
-	timeline->hwsp_offset = I915_GEM_HWS_SEQNO_ADDR;
 	if (!hwsp) {
+		struct i915_timeline_cacheline *cl;
 		unsigned int cacheline;
 
 		hwsp = hwsp_alloc(timeline, &cacheline);
 		if (IS_ERR(hwsp))
 			return PTR_ERR(hwsp);
 
+		cl = cacheline_alloc(hwsp->private, cacheline);
+		if (IS_ERR(cl)) {
+			__idle_hwsp_free(hwsp->private, cacheline);
+			return PTR_ERR(cl);
+		}
+
+		timeline->hwsp_cacheline = cl;
 		timeline->hwsp_offset = cacheline * CACHELINE_BYTES;
-	}
-	timeline->hwsp_ggtt = i915_vma_get(hwsp);
 
-	vaddr = i915_gem_object_pin_map(hwsp->obj, I915_MAP_WB);
-	if (IS_ERR(vaddr)) {
-		hwsp_free(timeline);
-		i915_vma_put(hwsp);
-		return PTR_ERR(vaddr);
+		vaddr = page_mask_bits(cl->vaddr);
+	} else {
+		timeline->hwsp_offset = I915_GEM_HWS_SEQNO_ADDR;
+
+		vaddr = i915_gem_object_pin_map(hwsp->obj, I915_MAP_WB);
+		if (IS_ERR(vaddr))
+			return PTR_ERR(vaddr);
 	}
 
 	timeline->hwsp_seqno =
 		memset(vaddr + timeline->hwsp_offset, 0, CACHELINE_BYTES);
 
+	timeline->hwsp_ggtt = i915_vma_get(hwsp);
+	GEM_BUG_ON(timeline->hwsp_offset >= hwsp->size);
+
 	timeline->fence_context = dma_fence_context_alloc(1);
 
 	spin_lock_init(&timeline->lock);
+	mutex_init(&timeline->mutex);
 
-	INIT_ACTIVE_REQUEST(&timeline->barrier);
 	INIT_ACTIVE_REQUEST(&timeline->last_request);
 	INIT_LIST_HEAD(&timeline->requests);
 
@@ -236,18 +325,19 @@ void i915_timeline_fini(struct i915_timeline *timeline)
 {
 	GEM_BUG_ON(timeline->pin_count);
 	GEM_BUG_ON(!list_empty(&timeline->requests));
-	GEM_BUG_ON(i915_active_request_isset(&timeline->barrier));
 
 	i915_syncmap_free(&timeline->sync);
-	hwsp_free(timeline);
 
-	i915_gem_object_unpin_map(timeline->hwsp_ggtt->obj);
+	if (timeline->hwsp_cacheline)
+		cacheline_free(timeline->hwsp_cacheline);
+	else
+		i915_gem_object_unpin_map(timeline->hwsp_ggtt->obj);
+
 	i915_vma_put(timeline->hwsp_ggtt);
 }
 
 struct i915_timeline *
 i915_timeline_create(struct drm_i915_private *i915,
-		     const char *name,
 		     struct i915_vma *global_hwsp)
 {
 	struct i915_timeline *timeline;
@@ -257,7 +347,7 @@ i915_timeline_create(struct drm_i915_private *i915,
 	if (!timeline)
 		return ERR_PTR(-ENOMEM);
 
-	err = i915_timeline_init(i915, timeline, name, global_hwsp);
+	err = i915_timeline_init(i915, timeline, global_hwsp);
 	if (err) {
 		kfree(timeline);
 		return ERR_PTR(err);
@@ -284,6 +374,7 @@ int i915_timeline_pin(struct i915_timeline *tl)
 		i915_ggtt_offset(tl->hwsp_ggtt) +
 		offset_in_page(tl->hwsp_offset);
 
+	cacheline_acquire(tl->hwsp_cacheline);
 	timeline_add_to_active(tl);
 
 	return 0;
@@ -293,6 +384,157 @@ unpin:
 	return err;
 }
 
+static u32 timeline_advance(struct i915_timeline *tl)
+{
+	GEM_BUG_ON(!tl->pin_count);
+	GEM_BUG_ON(tl->seqno & tl->has_initial_breadcrumb);
+
+	return tl->seqno += 1 + tl->has_initial_breadcrumb;
+}
+
+static void timeline_rollback(struct i915_timeline *tl)
+{
+	tl->seqno -= 1 + tl->has_initial_breadcrumb;
+}
+
+static noinline int
+__i915_timeline_get_seqno(struct i915_timeline *tl,
+			  struct i915_request *rq,
+			  u32 *seqno)
+{
+	struct i915_timeline_cacheline *cl;
+	unsigned int cacheline;
+	struct i915_vma *vma;
+	void *vaddr;
+	int err;
+
+	/*
+	 * If there is an outstanding GPU reference to this cacheline,
+	 * such as it being sampled by a HW semaphore on another timeline,
+	 * we cannot wraparound our seqno value (the HW semaphore does
+	 * a strict greater-than-or-equals compare, not i915_seqno_passed).
+	 * So if the cacheline is still busy, we must detach ourselves
+	 * from it and leave it inflight alongside its users.
+	 *
+	 * However, if nobody is watching and we can guarantee that nobody
+	 * will, we could simply reuse the same cacheline.
+	 *
+	 * if (i915_active_request_is_signaled(&tl->last_request) &&
+	 *     i915_active_is_signaled(&tl->hwsp_cacheline->active))
+	 *	return 0;
+	 *
+	 * That seems unlikely for a busy timeline that needed to wrap in
+	 * the first place, so just replace the cacheline.
+	 */
+
+	vma = hwsp_alloc(tl, &cacheline);
+	if (IS_ERR(vma)) {
+		err = PTR_ERR(vma);
+		goto err_rollback;
+	}
+
+	err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL | PIN_HIGH);
+	if (err) {
+		__idle_hwsp_free(vma->private, cacheline);
+		goto err_rollback;
+	}
+
+	cl = cacheline_alloc(vma->private, cacheline);
+	if (IS_ERR(cl)) {
+		err = PTR_ERR(cl);
+		__idle_hwsp_free(vma->private, cacheline);
+		goto err_unpin;
+	}
+	GEM_BUG_ON(cl->hwsp->vma != vma);
+
+	/*
+	 * Attach the old cacheline to the current request, so that we only
+	 * free it after the current request is retired, which ensures that
+	 * all writes into the cacheline from previous requests are complete.
+	 */
+	err = i915_active_ref(&tl->hwsp_cacheline->active,
+			      tl->fence_context, rq);
+	if (err)
+		goto err_cacheline;
+
+	cacheline_release(tl->hwsp_cacheline); /* ownership now xfered to rq */
+	cacheline_free(tl->hwsp_cacheline);
+
+	i915_vma_unpin(tl->hwsp_ggtt); /* binding kept alive by old cacheline */
+	i915_vma_put(tl->hwsp_ggtt);
+
+	tl->hwsp_ggtt = i915_vma_get(vma);
+
+	vaddr = page_mask_bits(cl->vaddr);
+	tl->hwsp_offset = cacheline * CACHELINE_BYTES;
+	tl->hwsp_seqno =
+		memset(vaddr + tl->hwsp_offset, 0, CACHELINE_BYTES);
+
+	tl->hwsp_offset += i915_ggtt_offset(vma);
+
+	cacheline_acquire(cl);
+	tl->hwsp_cacheline = cl;
+
+	*seqno = timeline_advance(tl);
+	GEM_BUG_ON(i915_seqno_passed(*tl->hwsp_seqno, *seqno));
+	return 0;
+
+err_cacheline:
+	cacheline_free(cl);
+err_unpin:
+	i915_vma_unpin(vma);
+err_rollback:
+	timeline_rollback(tl);
+	return err;
+}
+
+int i915_timeline_get_seqno(struct i915_timeline *tl,
+			    struct i915_request *rq,
+			    u32 *seqno)
+{
+	*seqno = timeline_advance(tl);
+
+	/* Replace the HWSP on wraparound for HW semaphores */
+	if (unlikely(!*seqno && tl->hwsp_cacheline))
+		return __i915_timeline_get_seqno(tl, rq, seqno);
+
+	return 0;
+}
+
+static int cacheline_ref(struct i915_timeline_cacheline *cl,
+			 struct i915_request *rq)
+{
+	return i915_active_ref(&cl->active, rq->fence.context, rq);
+}
+
+int i915_timeline_read_hwsp(struct i915_request *from,
+			    struct i915_request *to,
+			    u32 *hwsp)
+{
+	struct i915_timeline_cacheline *cl = from->hwsp_cacheline;
+	struct i915_timeline *tl = from->timeline;
+	int err;
+
+	GEM_BUG_ON(to->timeline == tl);
+
+	mutex_lock_nested(&tl->mutex, SINGLE_DEPTH_NESTING);
+	err = i915_request_completed(from);
+	if (!err)
+		err = cacheline_ref(cl, to);
+	if (!err) {
+		if (likely(cl == tl->hwsp_cacheline)) {
+			*hwsp = tl->hwsp_offset;
+		} else { /* across a seqno wrap, recover the original offset */
+			*hwsp = i915_ggtt_offset(cl->hwsp->vma) +
+				ptr_unmask_bits(cl->vaddr, CACHELINE_BITS) *
+				CACHELINE_BYTES;
+		}
+	}
+	mutex_unlock(&tl->mutex);
+
+	return err;
+}
+
 void i915_timeline_unpin(struct i915_timeline *tl)
 {
 	GEM_BUG_ON(!tl->pin_count);
@@ -300,6 +542,7 @@ void i915_timeline_unpin(struct i915_timeline *tl)
 		return;
 
 	timeline_remove_from_active(tl);
+	cacheline_release(tl->hwsp_cacheline);
 
 	/*
 	 * Since this timeline is idle, all bariers upon which we were waiting
diff --git a/drivers/gpu/drm/i915/i915_timeline.h b/drivers/gpu/drm/i915/i915_timeline.h
index 7bec7d2e45bf..27668a1a69a3 100644
--- a/drivers/gpu/drm/i915/i915_timeline.h
+++ b/drivers/gpu/drm/i915/i915_timeline.h
@@ -25,76 +25,14 @@
 #ifndef I915_TIMELINE_H
 #define I915_TIMELINE_H
 
-#include <linux/list.h>
-#include <linux/kref.h>
+#include <linux/lockdep.h>
 
 #include "i915_active.h"
-#include "i915_request.h"
 #include "i915_syncmap.h"
-#include "i915_utils.h"
-
-struct i915_vma;
-struct i915_timeline_hwsp;
-
-struct i915_timeline {
-	u64 fence_context;
-	u32 seqno;
-
-	spinlock_t lock;
-#define TIMELINE_CLIENT 0 /* default subclass */
-#define TIMELINE_ENGINE 1
-
-	unsigned int pin_count;
-	const u32 *hwsp_seqno;
-	struct i915_vma *hwsp_ggtt;
-	u32 hwsp_offset;
-
-	bool has_initial_breadcrumb;
-
-	/**
-	 * List of breadcrumbs associated with GPU requests currently
-	 * outstanding.
-	 */
-	struct list_head requests;
-
-	/* Contains an RCU guarded pointer to the last request. No reference is
-	 * held to the request, users must carefully acquire a reference to
-	 * the request using i915_active_request_get_request_rcu(), or hold the
-	 * struct_mutex.
-	 */
-	struct i915_active_request last_request;
-
-	/**
-	 * We track the most recent seqno that we wait on in every context so
-	 * that we only have to emit a new await and dependency on a more
-	 * recent sync point. As the contexts may be executed out-of-order, we
-	 * have to track each individually and can not rely on an absolute
-	 * global_seqno. When we know that all tracked fences are completed
-	 * (i.e. when the driver is idle), we know that the syncmap is
-	 * redundant and we can discard it without loss of generality.
-	 */
-	struct i915_syncmap *sync;
-
-	/**
-	 * Barrier provides the ability to serialize ordering between different
-	 * timelines.
-	 *
-	 * Users can call i915_timeline_set_barrier which will make all
-	 * subsequent submissions to this timeline be executed only after the
-	 * barrier has been completed.
-	 */
-	struct i915_active_request barrier;
-
-	struct list_head link;
-	const char *name;
-	struct drm_i915_private *i915;
-
-	struct kref kref;
-};
+#include "i915_timeline_types.h"
 
 int i915_timeline_init(struct drm_i915_private *i915,
 		       struct i915_timeline *tl,
-		       const char *name,
 		       struct i915_vma *hwsp);
 void i915_timeline_fini(struct i915_timeline *tl);
 
@@ -119,7 +57,6 @@ i915_timeline_set_subclass(struct i915_timeline *timeline,
 
 struct i915_timeline *
 i915_timeline_create(struct drm_i915_private *i915,
-		     const char *name,
 		     struct i915_vma *global_hwsp);
 
 static inline struct i915_timeline *
@@ -160,25 +97,17 @@ static inline bool i915_timeline_sync_is_later(struct i915_timeline *tl,
 }
 
 int i915_timeline_pin(struct i915_timeline *tl);
+int i915_timeline_get_seqno(struct i915_timeline *tl,
+			    struct i915_request *rq,
+			    u32 *seqno);
 void i915_timeline_unpin(struct i915_timeline *tl);
 
+int i915_timeline_read_hwsp(struct i915_request *from,
+			    struct i915_request *until,
+			    u32 *hwsp_offset);
+
 void i915_timelines_init(struct drm_i915_private *i915);
 void i915_timelines_park(struct drm_i915_private *i915);
 void i915_timelines_fini(struct drm_i915_private *i915);
 
-/**
- * i915_timeline_set_barrier - orders submission between different timelines
- * @timeline: timeline to set the barrier on
- * @rq: request after which new submissions can proceed
- *
- * Sets the passed in request as the serialization point for all subsequent
- * submissions on @timeline. Subsequent requests will not be submitted to GPU
- * until the barrier has been completed.
- */
-static inline int
-i915_timeline_set_barrier(struct i915_timeline *tl, struct i915_request *rq)
-{
-	return i915_active_request_set(&tl->barrier, rq);
-}
-
 #endif
diff --git a/drivers/gpu/drm/i915/i915_timeline_types.h b/drivers/gpu/drm/i915/i915_timeline_types.h
new file mode 100644
index 000000000000..5256a0b5c5f7
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_timeline_types.h
@@ -0,0 +1,70 @@
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2016 Intel Corporation
+ */
+
+#ifndef __I915_TIMELINE_TYPES_H__
+#define __I915_TIMELINE_TYPES_H__
+
+#include <linux/list.h>
+#include <linux/kref.h>
+#include <linux/mutex.h>
+#include <linux/types.h>
+
+#include "i915_active_types.h"
+
+struct drm_i915_private;
+struct i915_vma;
+struct i915_timeline_cacheline;
+struct i915_syncmap;
+
+struct i915_timeline {
+	u64 fence_context;
+	u32 seqno;
+
+	spinlock_t lock;
+#define TIMELINE_CLIENT 0 /* default subclass */
+#define TIMELINE_ENGINE 1
+	struct mutex mutex; /* protects the flow of requests */
+
+	unsigned int pin_count;
+	const u32 *hwsp_seqno;
+	struct i915_vma *hwsp_ggtt;
+	u32 hwsp_offset;
+
+	struct i915_timeline_cacheline *hwsp_cacheline;
+
+	bool has_initial_breadcrumb;
+
+	/**
+	 * List of breadcrumbs associated with GPU requests currently
+	 * outstanding.
+	 */
+	struct list_head requests;
+
+	/* Contains an RCU guarded pointer to the last request. No reference is
+	 * held to the request, users must carefully acquire a reference to
+	 * the request using i915_active_request_get_request_rcu(), or hold the
+	 * struct_mutex.
+	 */
+	struct i915_active_request last_request;
+
+	/**
+	 * We track the most recent seqno that we wait on in every context so
+	 * that we only have to emit a new await and dependency on a more
+	 * recent sync point. As the contexts may be executed out-of-order, we
+	 * have to track each individually and can not rely on an absolute
+	 * global_seqno. When we know that all tracked fences are completed
+	 * (i.e. when the driver is idle), we know that the syncmap is
+	 * redundant and we can discard it without loss of generality.
+	 */
+	struct i915_syncmap *sync;
+
+	struct list_head link;
+	struct drm_i915_private *i915;
+
+	struct kref kref;
+};
+
+#endif /* __I915_TIMELINE_TYPES_H__ */
diff --git a/drivers/gpu/drm/i915/i915_trace.h b/drivers/gpu/drm/i915/i915_trace.h
index eab313c3163c..12893304c8f8 100644
--- a/drivers/gpu/drm/i915/i915_trace.h
+++ b/drivers/gpu/drm/i915/i915_trace.h
@@ -18,6 +18,87 @@
 
 /* watermark/fifo updates */
 
+TRACE_EVENT(intel_pipe_enable,
+	    TP_PROTO(struct drm_i915_private *dev_priv, enum pipe pipe),
+	    TP_ARGS(dev_priv, pipe),
+
+	    TP_STRUCT__entry(
+			     __array(u32, frame, 3)
+			     __array(u32, scanline, 3)
+			     __field(enum pipe, pipe)
+			     ),
+
+	    TP_fast_assign(
+			   enum pipe _pipe;
+			   for_each_pipe(dev_priv, _pipe) {
+				   __entry->frame[_pipe] =
+					   dev_priv->drm.driver->get_vblank_counter(&dev_priv->drm, _pipe);
+				   __entry->scanline[_pipe] =
+					   intel_get_crtc_scanline(intel_get_crtc_for_pipe(dev_priv, _pipe));
+			   }
+			   __entry->pipe = pipe;
+			   ),
+
+	    TP_printk("pipe %c enable, pipe A: frame=%u, scanline=%u, pipe B: frame=%u, scanline=%u, pipe C: frame=%u, scanline=%u",
+		      pipe_name(__entry->pipe),
+		      __entry->frame[PIPE_A], __entry->scanline[PIPE_A],
+		      __entry->frame[PIPE_B], __entry->scanline[PIPE_B],
+		      __entry->frame[PIPE_C], __entry->scanline[PIPE_C])
+);
+
+TRACE_EVENT(intel_pipe_disable,
+	    TP_PROTO(struct drm_i915_private *dev_priv, enum pipe pipe),
+	    TP_ARGS(dev_priv, pipe),
+
+	    TP_STRUCT__entry(
+			     __array(u32, frame, 3)
+			     __array(u32, scanline, 3)
+			     __field(enum pipe, pipe)
+			     ),
+
+	    TP_fast_assign(
+			   enum pipe _pipe;
+			   for_each_pipe(dev_priv, _pipe) {
+				   __entry->frame[_pipe] =
+					   dev_priv->drm.driver->get_vblank_counter(&dev_priv->drm, _pipe);
+				   __entry->scanline[_pipe] =
+					   intel_get_crtc_scanline(intel_get_crtc_for_pipe(dev_priv, _pipe));
+			   }
+			   __entry->pipe = pipe;
+			   ),
+
+	    TP_printk("pipe %c disable, pipe A: frame=%u, scanline=%u, pipe B: frame=%u, scanline=%u, pipe C: frame=%u, scanline=%u",
+		      pipe_name(__entry->pipe),
+		      __entry->frame[PIPE_A], __entry->scanline[PIPE_A],
+		      __entry->frame[PIPE_B], __entry->scanline[PIPE_B],
+		      __entry->frame[PIPE_C], __entry->scanline[PIPE_C])
+);
+
+TRACE_EVENT(intel_pipe_crc,
+	    TP_PROTO(struct intel_crtc *crtc, const u32 *crcs),
+	    TP_ARGS(crtc, crcs),
+
+	    TP_STRUCT__entry(
+			     __field(enum pipe, pipe)
+			     __field(u32, frame)
+			     __field(u32, scanline)
+			     __array(u32, crcs, 5)
+			     ),
+
+	    TP_fast_assign(
+			   __entry->pipe = crtc->pipe;
+			   __entry->frame = crtc->base.dev->driver->get_vblank_counter(crtc->base.dev,
+										       crtc->pipe);
+			   __entry->scanline = intel_get_crtc_scanline(crtc);
+			   memcpy(__entry->crcs, crcs, sizeof(__entry->crcs));
+			   ),
+
+	    TP_printk("pipe %c, frame=%u, scanline=%u crc=%08x %08x %08x %08x %08x",
+		      pipe_name(__entry->pipe), __entry->frame, __entry->scanline,
+		      __entry->crcs[0], __entry->crcs[1], __entry->crcs[2],
+		      __entry->crcs[3], __entry->crcs[4])
+);
+
 TRACE_EVENT(intel_cpu_fifo_underrun,
 	    TP_PROTO(struct drm_i915_private *dev_priv, enum pipe pipe),
 	    TP_ARGS(dev_priv, pipe),
@@ -627,7 +708,6 @@ DECLARE_EVENT_CLASS(i915_request,
 			     __field(u16, class)
 			     __field(u16, instance)
 			     __field(u32, seqno)
-			     __field(u32, global)
 			     ),
 
 	    TP_fast_assign(
@@ -637,13 +717,11 @@ DECLARE_EVENT_CLASS(i915_request,
 			   __entry->instance = rq->engine->instance;
 			   __entry->ctx = rq->fence.context;
 			   __entry->seqno = rq->fence.seqno;
-			   __entry->global = rq->global_seqno;
 			   ),
 
-	    TP_printk("dev=%u, engine=%u:%u, hw_id=%u, ctx=%llu, seqno=%u, global=%u",
+	    TP_printk("dev=%u, engine=%u:%u, hw_id=%u, ctx=%llu, seqno=%u",
 		      __entry->dev, __entry->class, __entry->instance,
-		      __entry->hw_id, __entry->ctx, __entry->seqno,
-		      __entry->global)
+		      __entry->hw_id, __entry->ctx, __entry->seqno)
 );
 
 DEFINE_EVENT(i915_request, i915_request_add,
@@ -673,7 +751,6 @@ TRACE_EVENT(i915_request_in,
 			     __field(u16, class)
 			     __field(u16, instance)
 			     __field(u32, seqno)
-			     __field(u32, global_seqno)
 			     __field(u32, port)
 			     __field(u32, prio)
 			    ),
@@ -685,15 +762,14 @@ TRACE_EVENT(i915_request_in,
 			   __entry->instance = rq->engine->instance;
 			   __entry->ctx = rq->fence.context;
 			   __entry->seqno = rq->fence.seqno;
-			   __entry->global_seqno = rq->global_seqno;
 			   __entry->prio = rq->sched.attr.priority;
 			   __entry->port = port;
 			   ),
 
-	    TP_printk("dev=%u, engine=%u:%u, hw_id=%u, ctx=%llu, seqno=%u, prio=%u, global=%u, port=%u",
+	    TP_printk("dev=%u, engine=%u:%u, hw_id=%u, ctx=%llu, seqno=%u, prio=%u, port=%u",
 		      __entry->dev, __entry->class, __entry->instance,
 		      __entry->hw_id, __entry->ctx, __entry->seqno,
-		      __entry->prio, __entry->global_seqno, __entry->port)
+		      __entry->prio, __entry->port)
 );
 
 TRACE_EVENT(i915_request_out,
@@ -707,7 +783,6 @@ TRACE_EVENT(i915_request_out,
 			     __field(u16, class)
 			     __field(u16, instance)
 			     __field(u32, seqno)
-			     __field(u32, global_seqno)
 			     __field(u32, completed)
 			    ),
 
@@ -718,14 +793,13 @@ TRACE_EVENT(i915_request_out,
 			   __entry->instance = rq->engine->instance;
 			   __entry->ctx = rq->fence.context;
 			   __entry->seqno = rq->fence.seqno;
-			   __entry->global_seqno = rq->global_seqno;
 			   __entry->completed = i915_request_completed(rq);
 			   ),
 
-		    TP_printk("dev=%u, engine=%u:%u, hw_id=%u, ctx=%llu, seqno=%u, global=%u, completed?=%u",
+		    TP_printk("dev=%u, engine=%u:%u, hw_id=%u, ctx=%llu, seqno=%u, completed?=%u",
 			      __entry->dev, __entry->class, __entry->instance,
 			      __entry->hw_id, __entry->ctx, __entry->seqno,
-			      __entry->global_seqno, __entry->completed)
+			      __entry->completed)
 );
 
 #else
@@ -768,7 +842,6 @@ TRACE_EVENT(i915_request_wait_begin,
 			     __field(u16, class)
 			     __field(u16, instance)
 			     __field(u32, seqno)
-			     __field(u32, global)
 			     __field(unsigned int, flags)
 			     ),
 
@@ -785,14 +858,13 @@ TRACE_EVENT(i915_request_wait_begin,
 			   __entry->instance = rq->engine->instance;
 			   __entry->ctx = rq->fence.context;
 			   __entry->seqno = rq->fence.seqno;
-			   __entry->global = rq->global_seqno;
 			   __entry->flags = flags;
 			   ),
 
-	    TP_printk("dev=%u, engine=%u:%u, hw_id=%u, ctx=%llu, seqno=%u, global=%u, blocking=%u, flags=0x%x",
+	    TP_printk("dev=%u, engine=%u:%u, hw_id=%u, ctx=%llu, seqno=%u, blocking=%u, flags=0x%x",
 		      __entry->dev, __entry->class, __entry->instance,
 		      __entry->hw_id, __entry->ctx, __entry->seqno,
-		      __entry->global, !!(__entry->flags & I915_WAIT_LOCKED),
+		      !!(__entry->flags & I915_WAIT_LOCKED),
 		      __entry->flags)
 );
 
diff --git a/drivers/gpu/drm/i915/i915_user_extensions.c b/drivers/gpu/drm/i915/i915_user_extensions.c
new file mode 100644
index 000000000000..c822d0aafd2d
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_user_extensions.c
@@ -0,0 +1,61 @@
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2018 Intel Corporation
+ */
+
+#include <linux/nospec.h>
+#include <linux/sched/signal.h>
+#include <linux/uaccess.h>
+
+#include <uapi/drm/i915_drm.h>
+
+#include "i915_user_extensions.h"
+#include "i915_utils.h"
+
+int i915_user_extensions(struct i915_user_extension __user *ext,
+			 const i915_user_extension_fn *tbl,
+			 unsigned int count,
+			 void *data)
+{
+	unsigned int stackdepth = 512;
+
+	while (ext) {
+		int i, err;
+		u32 name;
+		u64 next;
+
+		if (!stackdepth--) /* recursion vs useful flexibility */
+			return -E2BIG;
+
+		err = check_user_mbz(&ext->flags);
+		if (err)
+			return err;
+
+		for (i = 0; i < ARRAY_SIZE(ext->rsvd); i++) {
+			err = check_user_mbz(&ext->rsvd[i]);
+			if (err)
+				return err;
+		}
+
+		if (get_user(name, &ext->name))
+			return -EFAULT;
+
+		err = -EINVAL;
+		if (name < count) {
+			name = array_index_nospec(name, count);
+			if (tbl[name])
+				err = tbl[name](ext, data);
+		}
+		if (err)
+			return err;
+
+		if (get_user(next, &ext->next_extension) ||
+		    overflows_type(next, ext))
+			return -EFAULT;
+
+		ext = u64_to_user_ptr(next);
+	}
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/i915/i915_user_extensions.h b/drivers/gpu/drm/i915/i915_user_extensions.h
new file mode 100644
index 000000000000..a14bf6bba9a1
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_user_extensions.h
@@ -0,0 +1,20 @@
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2018 Intel Corporation
+ */
+
+#ifndef I915_USER_EXTENSIONS_H
+#define I915_USER_EXTENSIONS_H
+
+struct i915_user_extension;
+
+typedef int (*i915_user_extension_fn)(struct i915_user_extension __user *ext,
+				      void *data);
+
+int i915_user_extensions(struct i915_user_extension __user *ext,
+			 const i915_user_extension_fn *tbl,
+			 unsigned int count,
+			 void *data);
+
+#endif /* I915_USER_EXTENSIONS_H */
diff --git a/drivers/gpu/drm/i915/i915_utils.h b/drivers/gpu/drm/i915/i915_utils.h
index 540e20eb032c..2dbe8933b50a 100644
--- a/drivers/gpu/drm/i915/i915_utils.h
+++ b/drivers/gpu/drm/i915/i915_utils.h
@@ -105,6 +105,37 @@
 	__T;								\
 })
 
+/*
+ * container_of_user: Extract the superclass from a pointer to a member.
+ *
+ * Exactly like container_of() with the exception that it plays nicely
+ * with sparse for __user @ptr.
+ */
+#define container_of_user(ptr, type, member) ({				\
+	void __user *__mptr = (void __user *)(ptr);			\
+	BUILD_BUG_ON_MSG(!__same_type(*(ptr), ((type *)0)->member) &&	\
+			 !__same_type(*(ptr), void),			\
+			 "pointer type mismatch in container_of()");	\
+	((type __user *)(__mptr - offsetof(type, member))); })
+
+/*
+ * check_user_mbz: Check that a user value exists and is zero
+ *
+ * Frequently in our uABI we reserve space for future extensions, and
+ * two ensure that userspace is prepared we enforce that space must
+ * be zero. (Then any future extension can safely assume a default value
+ * of 0.)
+ *
+ * check_user_mbz() combines checking that the user pointer is accessible
+ * and that the contained value is zero.
+ *
+ * Returns: -EFAULT if not accessible, -EINVAL if !zero, or 0 on success.
+ */
+#define check_user_mbz(U) ({						\
+	typeof(*(U)) mbz__;						\
+	get_user(mbz__, (U)) ? -EFAULT : mbz__ ? -EINVAL : 0;		\
+})
+
 static inline u64 ptr_to_u64(const void *ptr)
 {
 	return (uintptr_t)ptr;
diff --git a/drivers/gpu/drm/i915/i915_vgpu.c b/drivers/gpu/drm/i915/i915_vgpu.c
index 869cf4a3b6de..94d3992b599d 100644
--- a/drivers/gpu/drm/i915/i915_vgpu.c
+++ b/drivers/gpu/drm/i915/i915_vgpu.c
@@ -60,30 +60,31 @@
  */
 void i915_check_vgpu(struct drm_i915_private *dev_priv)
 {
+	struct intel_uncore *uncore = &dev_priv->uncore;
 	u64 magic;
 	u16 version_major;
 
 	BUILD_BUG_ON(sizeof(struct vgt_if) != VGT_PVINFO_SIZE);
 
-	magic = __raw_i915_read64(dev_priv, vgtif_reg(magic));
+	magic = __raw_uncore_read64(uncore, vgtif_reg(magic));
 	if (magic != VGT_MAGIC)
 		return;
 
-	version_major = __raw_i915_read16(dev_priv, vgtif_reg(version_major));
+	version_major = __raw_uncore_read16(uncore, vgtif_reg(version_major));
 	if (version_major < VGT_VERSION_MAJOR) {
 		DRM_INFO("VGT interface version mismatch!\n");
 		return;
 	}
 
-	dev_priv->vgpu.caps = __raw_i915_read32(dev_priv, vgtif_reg(vgt_caps));
+	dev_priv->vgpu.caps = __raw_uncore_read32(uncore, vgtif_reg(vgt_caps));
 
 	dev_priv->vgpu.active = true;
 	DRM_INFO("Virtual GPU for Intel GVT-g detected.\n");
 }
 
-bool intel_vgpu_has_full_48bit_ppgtt(struct drm_i915_private *dev_priv)
+bool intel_vgpu_has_full_ppgtt(struct drm_i915_private *dev_priv)
 {
-	return dev_priv->vgpu.caps & VGT_CAPS_FULL_48BIT_PPGTT;
+	return dev_priv->vgpu.caps & VGT_CAPS_FULL_PPGTT;
 }
 
 struct _balloon_info_ {
diff --git a/drivers/gpu/drm/i915/i915_vgpu.h b/drivers/gpu/drm/i915/i915_vgpu.h
index 551acc390046..ebe1b7bced98 100644
--- a/drivers/gpu/drm/i915/i915_vgpu.h
+++ b/drivers/gpu/drm/i915/i915_vgpu.h
@@ -28,7 +28,7 @@
 
 void i915_check_vgpu(struct drm_i915_private *dev_priv);
 
-bool intel_vgpu_has_full_48bit_ppgtt(struct drm_i915_private *dev_priv);
+bool intel_vgpu_has_full_ppgtt(struct drm_i915_private *dev_priv);
 
 static inline bool
 intel_vgpu_has_hwsp_emulation(struct drm_i915_private *dev_priv)
diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c
index 41b5bcb803cb..961268f66c63 100644
--- a/drivers/gpu/drm/i915/i915_vma.c
+++ b/drivers/gpu/drm/i915/i915_vma.c
@@ -25,11 +25,27 @@
 #include "i915_vma.h"
 
 #include "i915_drv.h"
+#include "i915_globals.h"
 #include "intel_ringbuffer.h"
 #include "intel_frontbuffer.h"
 
 #include <drm/drm_gem.h>
 
+static struct i915_global_vma {
+	struct i915_global base;
+	struct kmem_cache *slab_vmas;
+} global;
+
+struct i915_vma *i915_vma_alloc(void)
+{
+	return kmem_cache_zalloc(global.slab_vmas, GFP_KERNEL);
+}
+
+void i915_vma_free(struct i915_vma *vma)
+{
+	return kmem_cache_free(global.slab_vmas, vma);
+}
+
 #if IS_ENABLED(CONFIG_DRM_I915_ERRLOG_GEM) && IS_ENABLED(CONFIG_DRM_DEBUG_MM)
 
 #include <linux/stackdepot.h>
@@ -112,7 +128,7 @@ vma_create(struct drm_i915_gem_object *obj,
 	/* The aliasing_ppgtt should never be used directly! */
 	GEM_BUG_ON(vm == &vm->i915->mm.aliasing_ppgtt->vm);
 
-	vma = kmem_cache_zalloc(vm->i915->vmas, GFP_KERNEL);
+	vma = i915_vma_alloc();
 	if (vma == NULL)
 		return ERR_PTR(-ENOMEM);
 
@@ -187,7 +203,7 @@ vma_create(struct drm_i915_gem_object *obj,
 		cmp = i915_vma_compare(pos, vm, view);
 		if (cmp == 0) {
 			spin_unlock(&obj->vma.lock);
-			kmem_cache_free(vm->i915->vmas, vma);
+			i915_vma_free(vma);
 			return pos;
 		}
 
@@ -219,7 +235,7 @@ vma_create(struct drm_i915_gem_object *obj,
 	return vma;
 
 err_vma:
-	kmem_cache_free(vm->i915->vmas, vma);
+	i915_vma_free(vma);
 	return ERR_PTR(-E2BIG);
 }
 
@@ -800,8 +816,6 @@ void i915_vma_reopen(struct i915_vma *vma)
 
 static void __i915_vma_destroy(struct i915_vma *vma)
 {
-	struct drm_i915_private *i915 = vma->vm->i915;
-
 	GEM_BUG_ON(vma->node.allocated);
 	GEM_BUG_ON(vma->fence);
 
@@ -822,7 +836,7 @@ static void __i915_vma_destroy(struct i915_vma *vma)
 
 	i915_active_fini(&vma->active);
 
-	kmem_cache_free(i915->vmas, vma);
+	i915_vma_free(vma);
 }
 
 void i915_vma_destroy(struct i915_vma *vma)
@@ -1038,3 +1052,28 @@ unpin:
 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 #include "selftests/i915_vma.c"
 #endif
+
+static void i915_global_vma_shrink(void)
+{
+	kmem_cache_shrink(global.slab_vmas);
+}
+
+static void i915_global_vma_exit(void)
+{
+	kmem_cache_destroy(global.slab_vmas);
+}
+
+static struct i915_global_vma global = { {
+	.shrink = i915_global_vma_shrink,
+	.exit = i915_global_vma_exit,
+} };
+
+int __init i915_global_vma_init(void)
+{
+	global.slab_vmas = KMEM_CACHE(i915_vma, SLAB_HWCACHE_ALIGN);
+	if (!global.slab_vmas)
+		return -ENOMEM;
+
+	i915_global_register(&global.base);
+	return 0;
+}
diff --git a/drivers/gpu/drm/i915/i915_vma.h b/drivers/gpu/drm/i915/i915_vma.h
index 7c742027f866..6eab70953a57 100644
--- a/drivers/gpu/drm/i915/i915_vma.h
+++ b/drivers/gpu/drm/i915/i915_vma.h
@@ -440,4 +440,7 @@ void i915_vma_parked(struct drm_i915_private *i915);
 	list_for_each_entry(V, &(OBJ)->vma.list, obj_link)		\
 		for_each_until(!i915_vma_is_ggtt(V))
 
+struct i915_vma *i915_vma_alloc(void);
+void i915_vma_free(struct i915_vma *vma);
+
 #endif
diff --git a/drivers/gpu/drm/i915/icl_dsi.c b/drivers/gpu/drm/i915/icl_dsi.c
index 641e0778fa9c..9d962ea1e635 100644
--- a/drivers/gpu/drm/i915/icl_dsi.c
+++ b/drivers/gpu/drm/i915/icl_dsi.c
@@ -25,9 +25,13 @@
  *   Jani Nikula <jani.nikula@intel.com>
  */
 
-#include <drm/drm_mipi_dsi.h>
 #include <drm/drm_atomic_helper.h>
+#include <drm/drm_mipi_dsi.h>
+
+#include "intel_connector.h"
+#include "intel_ddi.h"
 #include "intel_dsi.h"
+#include "intel_panel.h"
 
 static inline int header_credits_available(struct drm_i915_private *dev_priv,
 					   enum transcoder dsi_trans)
@@ -246,13 +250,13 @@ static void dsi_program_swing_and_deemphasis(struct intel_encoder *encoder)
 
 		for (lane = 0; lane <= 3; lane++) {
 			/* Bspec: must not use GRP register for write */
-			tmp = I915_READ(ICL_PORT_TX_DW4_LN(port, lane));
+			tmp = I915_READ(ICL_PORT_TX_DW4_LN(lane, port));
 			tmp &= ~(POST_CURSOR_1_MASK | POST_CURSOR_2_MASK |
 				 CURSOR_COEFF_MASK);
 			tmp |= POST_CURSOR_1(0x0);
 			tmp |= POST_CURSOR_2(0x0);
 			tmp |= CURSOR_COEFF(0x3f);
-			I915_WRITE(ICL_PORT_TX_DW4_LN(port, lane), tmp);
+			I915_WRITE(ICL_PORT_TX_DW4_LN(lane, port), tmp);
 		}
 	}
 }
@@ -399,11 +403,11 @@ static void gen11_dsi_config_phy_lanes_sequence(struct intel_encoder *encoder)
 		tmp &= ~LOADGEN_SELECT;
 		I915_WRITE(ICL_PORT_TX_DW4_AUX(port), tmp);
 		for (lane = 0; lane <= 3; lane++) {
-			tmp = I915_READ(ICL_PORT_TX_DW4_LN(port, lane));
+			tmp = I915_READ(ICL_PORT_TX_DW4_LN(lane, port));
 			tmp &= ~LOADGEN_SELECT;
 			if (lane != 2)
 				tmp |= LOADGEN_SELECT;
-			I915_WRITE(ICL_PORT_TX_DW4_LN(port, lane), tmp);
+			I915_WRITE(ICL_PORT_TX_DW4_LN(lane, port), tmp);
 		}
 	}
 
@@ -876,7 +880,8 @@ static void gen11_dsi_enable_transcoder(struct intel_encoder *encoder)
 		I915_WRITE(PIPECONF(dsi_trans), tmp);
 
 		/* wait for transcoder to be enabled */
-		if (intel_wait_for_register(dev_priv, PIPECONF(dsi_trans),
+		if (intel_wait_for_register(&dev_priv->uncore,
+					    PIPECONF(dsi_trans),
 					    I965_PIPECONF_ACTIVE,
 					    I965_PIPECONF_ACTIVE, 10))
 			DRM_ERROR("DSI transcoder not enabled\n");
@@ -1054,7 +1059,8 @@ static void gen11_dsi_disable_transcoder(struct intel_encoder *encoder)
 		I915_WRITE(PIPECONF(dsi_trans), tmp);
 
 		/* wait for transcoder to be disabled */
-		if (intel_wait_for_register(dev_priv, PIPECONF(dsi_trans),
+		if (intel_wait_for_register(&dev_priv->uncore,
+					    PIPECONF(dsi_trans),
 					    I965_PIPECONF_ACTIVE, 0, 50))
 			DRM_ERROR("DSI trancoder not disabled\n");
 	}
@@ -1146,13 +1152,11 @@ static void gen11_dsi_disable_io_power(struct intel_encoder *encoder)
 		intel_wakeref_t wakeref;
 
 		wakeref = fetch_and_zero(&intel_dsi->io_wakeref[port]);
-		if (wakeref) {
-			intel_display_power_put(dev_priv,
-						port == PORT_A ?
-						POWER_DOMAIN_PORT_DDI_A_IO :
-						POWER_DOMAIN_PORT_DDI_B_IO,
-						wakeref);
-		}
+		intel_display_power_put(dev_priv,
+					port == PORT_A ?
+					POWER_DOMAIN_PORT_DDI_A_IO :
+					POWER_DOMAIN_PORT_DDI_B_IO,
+					wakeref);
 	}
 
 	/* set mode to DDI */
@@ -1194,11 +1198,10 @@ static void gen11_dsi_get_config(struct intel_encoder *encoder,
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
-	u32 pll_id;
 
 	/* FIXME: adapt icl_ddi_clock_get() for DSI and use that? */
-	pll_id = intel_get_shared_dpll_id(dev_priv, pipe_config->shared_dpll);
-	pipe_config->port_clock = cnl_calc_wrpll_link(dev_priv, pll_id);
+	pipe_config->port_clock =
+		cnl_calc_wrpll_link(dev_priv, &pipe_config->dpll_hw_state);
 	pipe_config->base.adjusted_mode.crtc_clock = intel_dsi->pclk;
 	pipe_config->output_types |= BIT(INTEL_OUTPUT_DSI);
 }
@@ -1367,7 +1370,7 @@ void icl_dsi_init(struct drm_i915_private *dev_priv)
 	struct intel_encoder *encoder;
 	struct intel_connector *intel_connector;
 	struct drm_connector *connector;
-	struct drm_display_mode *scan, *fixed_mode = NULL;
+	struct drm_display_mode *fixed_mode;
 	enum port port;
 
 	if (!intel_bios_is_dsi_present(dev_priv, &port))
@@ -1417,15 +1420,8 @@ void icl_dsi_init(struct drm_i915_private *dev_priv)
 	/* attach connector to encoder */
 	intel_connector_attach_encoder(intel_connector, encoder);
 
-	/* fill mode info from VBT */
 	mutex_lock(&dev->mode_config.mutex);
-	intel_dsi_vbt_get_modes(intel_dsi);
-	list_for_each_entry(scan, &connector->probed_modes, head) {
-		if (scan->type & DRM_MODE_TYPE_PREFERRED) {
-			fixed_mode = drm_mode_duplicate(dev, scan);
-			break;
-		}
-	}
+	fixed_mode = intel_panel_vbt_fixed_mode(intel_connector);
 	mutex_unlock(&dev->mode_config.mutex);
 
 	if (!fixed_mode) {
@@ -1433,12 +1429,9 @@ void icl_dsi_init(struct drm_i915_private *dev_priv)
 		goto err;
 	}
 
-	connector->display_info.width_mm = fixed_mode->width_mm;
-	connector->display_info.height_mm = fixed_mode->height_mm;
 	intel_panel_init(&intel_connector->panel, fixed_mode, NULL);
 	intel_panel_setup_backlight(connector, INVALID_PIPE);
 
-
 	if (dev_priv->vbt.dsi.config->dual_link)
 		intel_dsi->ports = BIT(PORT_A) | BIT(PORT_B);
 	else
diff --git a/drivers/gpu/drm/i915/intel_atomic.c b/drivers/gpu/drm/i915/intel_atomic.c
index 7cf9290ea34a..8c8fae32ec50 100644
--- a/drivers/gpu/drm/i915/intel_atomic.c
+++ b/drivers/gpu/drm/i915/intel_atomic.c
@@ -35,6 +35,8 @@
 #include <drm/drm_plane_helper.h>
 
 #include "intel_drv.h"
+#include "intel_hdcp.h"
+#include "intel_sprite.h"
 
 /**
  * intel_digital_connector_atomic_get_property - hook for connector->atomic_get_property.
@@ -126,6 +128,7 @@ int intel_digital_connector_atomic_check(struct drm_connector *conn,
 	 */
 	if (new_conn_state->force_audio != old_conn_state->force_audio ||
 	    new_conn_state->broadcast_rgb != old_conn_state->broadcast_rgb ||
+	    new_conn_state->base.colorspace != old_conn_state->base.colorspace ||
 	    new_conn_state->base.picture_aspect_ratio != old_conn_state->base.picture_aspect_ratio ||
 	    new_conn_state->base.content_type != old_conn_state->base.content_type ||
 	    new_conn_state->base.scaling_mode != old_conn_state->base.scaling_mode)
@@ -234,10 +237,11 @@ static void intel_atomic_setup_scaler(struct intel_crtc_scaler_state *scaler_sta
 	if (plane_state && plane_state->base.fb &&
 	    plane_state->base.fb->format->is_yuv &&
 	    plane_state->base.fb->format->num_planes > 1) {
+		struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
 		if (IS_GEN(dev_priv, 9) &&
 		    !IS_GEMINILAKE(dev_priv)) {
 			mode = SKL_PS_SCALER_MODE_NV12;
-		} else if (icl_is_hdr_plane(to_intel_plane(plane_state->base.plane))) {
+		} else if (icl_is_hdr_plane(dev_priv, plane->id)) {
 			/*
 			 * On gen11+'s HDR planes we only use the scaler for
 			 * scaling. They have a dedicated chroma upsampler, so
diff --git a/drivers/gpu/drm/i915/intel_atomic_plane.c b/drivers/gpu/drm/i915/intel_atomic_plane.c
index db0965904439..d11681d71add 100644
--- a/drivers/gpu/drm/i915/intel_atomic_plane.c
+++ b/drivers/gpu/drm/i915/intel_atomic_plane.c
@@ -35,7 +35,10 @@
 #include <drm/drm_fourcc.h>
 #include <drm/drm_plane_helper.h>
 
+#include "intel_atomic_plane.h"
 #include "intel_drv.h"
+#include "intel_pm.h"
+#include "intel_sprite.h"
 
 struct intel_plane *intel_plane_alloc(void)
 {
@@ -121,6 +124,7 @@ int intel_plane_atomic_check_with_state(const struct intel_crtc_state *old_crtc_
 
 	new_crtc_state->active_planes &= ~BIT(plane->id);
 	new_crtc_state->nv12_planes &= ~BIT(plane->id);
+	new_crtc_state->c8_planes &= ~BIT(plane->id);
 	new_plane_state->base.visible = false;
 
 	if (!new_plane_state->base.crtc && !old_plane_state->base.crtc)
@@ -135,9 +139,13 @@ int intel_plane_atomic_check_with_state(const struct intel_crtc_state *old_crtc_
 		new_crtc_state->active_planes |= BIT(plane->id);
 
 	if (new_plane_state->base.visible &&
-	    new_plane_state->base.fb->format->format == DRM_FORMAT_NV12)
+	    is_planar_yuv_format(new_plane_state->base.fb->format->format))
 		new_crtc_state->nv12_planes |= BIT(plane->id);
 
+	if (new_plane_state->base.visible &&
+	    new_plane_state->base.fb->format->format == DRM_FORMAT_C8)
+		new_crtc_state->c8_planes |= BIT(plane->id);
+
 	if (new_plane_state->base.visible || old_plane_state->base.visible)
 		new_crtc_state->update_planes |= BIT(plane->id);
 
@@ -214,6 +222,35 @@ skl_next_plane_to_commit(struct intel_atomic_state *state,
 	return NULL;
 }
 
+void intel_update_plane(struct intel_plane *plane,
+			const struct intel_crtc_state *crtc_state,
+			const struct intel_plane_state *plane_state)
+{
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+
+	trace_intel_update_plane(&plane->base, crtc);
+	plane->update_plane(plane, crtc_state, plane_state);
+}
+
+void intel_update_slave(struct intel_plane *plane,
+			const struct intel_crtc_state *crtc_state,
+			const struct intel_plane_state *plane_state)
+{
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+
+	trace_intel_update_plane(&plane->base, crtc);
+	plane->update_slave(plane, crtc_state, plane_state);
+}
+
+void intel_disable_plane(struct intel_plane *plane,
+			 const struct intel_crtc_state *crtc_state)
+{
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+
+	trace_intel_disable_plane(&plane->base, crtc);
+	plane->disable_plane(plane, crtc_state);
+}
+
 void skl_update_planes_on_crtc(struct intel_atomic_state *state,
 			       struct intel_crtc *crtc)
 {
@@ -238,8 +275,7 @@ void skl_update_planes_on_crtc(struct intel_atomic_state *state,
 			intel_atomic_get_new_plane_state(state, plane);
 
 		if (new_plane_state->base.visible) {
-			trace_intel_update_plane(&plane->base, crtc);
-			plane->update_plane(plane, new_crtc_state, new_plane_state);
+			intel_update_plane(plane, new_crtc_state, new_plane_state);
 		} else if (new_plane_state->slave) {
 			struct intel_plane *master =
 				new_plane_state->linked_plane;
@@ -256,11 +292,9 @@ void skl_update_planes_on_crtc(struct intel_atomic_state *state,
 			new_plane_state =
 				intel_atomic_get_new_plane_state(state, master);
 
-			trace_intel_update_plane(&plane->base, crtc);
-			plane->update_slave(plane, new_crtc_state, new_plane_state);
+			intel_update_slave(plane, new_crtc_state, new_plane_state);
 		} else {
-			trace_intel_disable_plane(&plane->base, crtc);
-			plane->disable_plane(plane, new_crtc_state);
+			intel_disable_plane(plane, new_crtc_state);
 		}
 	}
 }
@@ -280,13 +314,10 @@ void i9xx_update_planes_on_crtc(struct intel_atomic_state *state,
 		    !(update_mask & BIT(plane->id)))
 			continue;
 
-		if (new_plane_state->base.visible) {
-			trace_intel_update_plane(&plane->base, crtc);
-			plane->update_plane(plane, new_crtc_state, new_plane_state);
-		} else {
-			trace_intel_disable_plane(&plane->base, crtc);
-			plane->disable_plane(plane, new_crtc_state);
-		}
+		if (new_plane_state->base.visible)
+			intel_update_plane(plane, new_crtc_state, new_plane_state);
+		else
+			intel_disable_plane(plane, new_crtc_state);
 	}
 }
 
diff --git a/drivers/gpu/drm/i915/intel_atomic_plane.h b/drivers/gpu/drm/i915/intel_atomic_plane.h
new file mode 100644
index 000000000000..14678620440f
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_atomic_plane.h
@@ -0,0 +1,40 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2019 Intel Corporation
+ */
+
+#ifndef __INTEL_ATOMIC_PLANE_H__
+#define __INTEL_ATOMIC_PLANE_H__
+
+struct drm_plane;
+struct intel_atomic_state;
+struct intel_crtc;
+struct intel_crtc_state;
+struct intel_plane;
+struct intel_plane_state;
+
+extern const struct drm_plane_helper_funcs intel_plane_helper_funcs;
+
+void intel_update_plane(struct intel_plane *plane,
+			const struct intel_crtc_state *crtc_state,
+			const struct intel_plane_state *plane_state);
+void intel_update_slave(struct intel_plane *plane,
+			const struct intel_crtc_state *crtc_state,
+			const struct intel_plane_state *plane_state);
+void intel_disable_plane(struct intel_plane *plane,
+			 const struct intel_crtc_state *crtc_state);
+struct intel_plane *intel_plane_alloc(void);
+void intel_plane_free(struct intel_plane *plane);
+struct drm_plane_state *intel_plane_duplicate_state(struct drm_plane *plane);
+void intel_plane_destroy_state(struct drm_plane *plane,
+			       struct drm_plane_state *state);
+void skl_update_planes_on_crtc(struct intel_atomic_state *state,
+			       struct intel_crtc *crtc);
+void i9xx_update_planes_on_crtc(struct intel_atomic_state *state,
+				struct intel_crtc *crtc);
+int intel_plane_atomic_check_with_state(const struct intel_crtc_state *old_crtc_state,
+					struct intel_crtc_state *crtc_state,
+					const struct intel_plane_state *old_plane_state,
+					struct intel_plane_state *intel_state);
+
+#endif /* __INTEL_ATOMIC_PLANE_H__ */
diff --git a/drivers/gpu/drm/i915/intel_audio.c b/drivers/gpu/drm/i915/intel_audio.c
index 5104c6bbd66f..bca4cc025d3d 100644
--- a/drivers/gpu/drm/i915/intel_audio.c
+++ b/drivers/gpu/drm/i915/intel_audio.c
@@ -21,14 +21,16 @@
  * DEALINGS IN THE SOFTWARE.
  */
 
-#include <linux/kernel.h>
 #include <linux/component.h>
+#include <linux/kernel.h>
+
+#include <drm/drm_edid.h>
 #include <drm/i915_component.h>
 #include <drm/intel_lpe_audio.h>
-#include "intel_drv.h"
 
-#include <drm/drm_edid.h>
 #include "i915_drv.h"
+#include "intel_audio.h"
+#include "intel_drv.h"
 
 /**
  * DOC: High Definition Audio over HDMI and Display Port
@@ -741,27 +743,91 @@ void intel_init_audio_hooks(struct drm_i915_private *dev_priv)
 	}
 }
 
-static void i915_audio_component_get_power(struct device *kdev)
+static void glk_force_audio_cdclk(struct drm_i915_private *dev_priv,
+				  bool enable)
 {
-	intel_display_power_get(kdev_to_i915(kdev), POWER_DOMAIN_AUDIO);
+	struct drm_modeset_acquire_ctx ctx;
+	struct drm_atomic_state *state;
+	int ret;
+
+	drm_modeset_acquire_init(&ctx, 0);
+	state = drm_atomic_state_alloc(&dev_priv->drm);
+	if (WARN_ON(!state))
+		return;
+
+	state->acquire_ctx = &ctx;
+
+retry:
+	to_intel_atomic_state(state)->cdclk.force_min_cdclk_changed = true;
+	to_intel_atomic_state(state)->cdclk.force_min_cdclk =
+		enable ? 2 * 96000 : 0;
+
+	/*
+	 * Protects dev_priv->cdclk.force_min_cdclk
+	 * Need to lock this here in case we have no active pipes
+	 * and thus wouldn't lock it during the commit otherwise.
+	 */
+	ret = drm_modeset_lock(&dev_priv->drm.mode_config.connection_mutex,
+			       &ctx);
+	if (!ret)
+		ret = drm_atomic_commit(state);
+
+	if (ret == -EDEADLK) {
+		drm_atomic_state_clear(state);
+		drm_modeset_backoff(&ctx);
+		goto retry;
+	}
+
+	WARN_ON(ret);
+
+	drm_atomic_state_put(state);
+
+	drm_modeset_drop_locks(&ctx);
+	drm_modeset_acquire_fini(&ctx);
 }
 
-static void i915_audio_component_put_power(struct device *kdev)
+static unsigned long i915_audio_component_get_power(struct device *kdev)
 {
-	intel_display_power_put_unchecked(kdev_to_i915(kdev),
-					  POWER_DOMAIN_AUDIO);
+	struct drm_i915_private *dev_priv = kdev_to_i915(kdev);
+	intel_wakeref_t ret;
+
+	/* Catch potential impedance mismatches before they occur! */
+	BUILD_BUG_ON(sizeof(intel_wakeref_t) > sizeof(unsigned long));
+
+	ret = intel_display_power_get(dev_priv, POWER_DOMAIN_AUDIO);
+
+	/* Force CDCLK to 2*BCLK as long as we need audio to be powered. */
+	if (dev_priv->audio_power_refcount++ == 0)
+		if (IS_CANNONLAKE(dev_priv) || IS_GEMINILAKE(dev_priv))
+			glk_force_audio_cdclk(dev_priv, true);
+
+	return ret;
+}
+
+static void i915_audio_component_put_power(struct device *kdev,
+					   unsigned long cookie)
+{
+	struct drm_i915_private *dev_priv = kdev_to_i915(kdev);
+
+	/* Stop forcing CDCLK to 2*BCLK if no need for audio to be powered. */
+	if (--dev_priv->audio_power_refcount == 0)
+		if (IS_CANNONLAKE(dev_priv) || IS_GEMINILAKE(dev_priv))
+			glk_force_audio_cdclk(dev_priv, false);
+
+	intel_display_power_put(dev_priv, POWER_DOMAIN_AUDIO, cookie);
 }
 
 static void i915_audio_component_codec_wake_override(struct device *kdev,
 						     bool enable)
 {
 	struct drm_i915_private *dev_priv = kdev_to_i915(kdev);
+	unsigned long cookie;
 	u32 tmp;
 
 	if (!IS_GEN(dev_priv, 9))
 		return;
 
-	i915_audio_component_get_power(kdev);
+	cookie = i915_audio_component_get_power(kdev);
 
 	/*
 	 * Enable/disable generating the codec wake signal, overriding the
@@ -779,7 +845,7 @@ static void i915_audio_component_codec_wake_override(struct device *kdev,
 		usleep_range(1000, 1500);
 	}
 
-	i915_audio_component_put_power(kdev);
+	i915_audio_component_put_power(kdev, cookie);
 }
 
 /* Get CDCLK in kHz  */
@@ -850,12 +916,13 @@ static int i915_audio_component_sync_audio_rate(struct device *kdev, int port,
 	struct i915_audio_component *acomp = dev_priv->audio_component;
 	struct intel_encoder *encoder;
 	struct intel_crtc *crtc;
+	unsigned long cookie;
 	int err = 0;
 
 	if (!HAS_DDI(dev_priv))
 		return 0;
 
-	i915_audio_component_get_power(kdev);
+	cookie = i915_audio_component_get_power(kdev);
 	mutex_lock(&dev_priv->av_mutex);
 
 	/* 1. get the pipe */
@@ -875,7 +942,7 @@ static int i915_audio_component_sync_audio_rate(struct device *kdev, int port,
 
  unlock:
 	mutex_unlock(&dev_priv->av_mutex);
-	i915_audio_component_put_power(kdev);
+	i915_audio_component_put_power(kdev, cookie);
 	return err;
 }
 
@@ -980,7 +1047,7 @@ static const struct component_ops i915_audio_component_bind_ops = {
  * We ignore any error during registration and continue with reduced
  * functionality (i.e. without HDMI audio).
  */
-void i915_audio_component_init(struct drm_i915_private *dev_priv)
+static void i915_audio_component_init(struct drm_i915_private *dev_priv)
 {
 	int ret;
 
@@ -1003,7 +1070,7 @@ void i915_audio_component_init(struct drm_i915_private *dev_priv)
  * Deregisters the audio component, breaking any existing binding to the
  * corresponding snd_hda_intel driver's master component.
  */
-void i915_audio_component_cleanup(struct drm_i915_private *dev_priv)
+static void i915_audio_component_cleanup(struct drm_i915_private *dev_priv)
 {
 	if (!dev_priv->audio_component_registered)
 		return;
diff --git a/drivers/gpu/drm/i915/intel_audio.h b/drivers/gpu/drm/i915/intel_audio.h
new file mode 100644
index 000000000000..a3657c7a7ba2
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_audio.h
@@ -0,0 +1,24 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2019 Intel Corporation
+ */
+
+#ifndef __INTEL_AUDIO_H__
+#define __INTEL_AUDIO_H__
+
+struct drm_connector_state;
+struct drm_i915_private;
+struct intel_crtc_state;
+struct intel_encoder;
+
+void intel_init_audio_hooks(struct drm_i915_private *dev_priv);
+void intel_audio_codec_enable(struct intel_encoder *encoder,
+			      const struct intel_crtc_state *crtc_state,
+			      const struct drm_connector_state *conn_state);
+void intel_audio_codec_disable(struct intel_encoder *encoder,
+			       const struct intel_crtc_state *old_crtc_state,
+			       const struct drm_connector_state *old_conn_state);
+void intel_audio_init(struct drm_i915_private *dev_priv);
+void intel_audio_deinit(struct drm_i915_private *dev_priv);
+
+#endif /* __INTEL_AUDIO_H__ */
diff --git a/drivers/gpu/drm/i915/intel_bios.c b/drivers/gpu/drm/i915/intel_bios.c
index 4364f42cac6b..1dc8d03ff127 100644
--- a/drivers/gpu/drm/i915/intel_bios.c
+++ b/drivers/gpu/drm/i915/intel_bios.c
@@ -760,6 +760,31 @@ parse_psr(struct drm_i915_private *dev_priv, const struct bdb_header *bdb)
 		dev_priv->vbt.psr.tp1_wakeup_time_us = psr_table->tp1_wakeup_time * 100;
 		dev_priv->vbt.psr.tp2_tp3_wakeup_time_us = psr_table->tp2_tp3_wakeup_time * 100;
 	}
+
+	if (bdb->version >= 226) {
+		u32 wakeup_time = psr_table->psr2_tp2_tp3_wakeup_time;
+
+		wakeup_time = (wakeup_time >> (2 * panel_type)) & 0x3;
+		switch (wakeup_time) {
+		case 0:
+			wakeup_time = 500;
+			break;
+		case 1:
+			wakeup_time = 100;
+			break;
+		case 3:
+			wakeup_time = 50;
+			break;
+		default:
+		case 2:
+			wakeup_time = 2500;
+			break;
+		}
+		dev_priv->vbt.psr.psr2_tp2_tp3_wakeup_time_us = wakeup_time;
+	} else {
+		/* Reusing PSR1 wakeup time for PSR2 in older VBTs */
+		dev_priv->vbt.psr.psr2_tp2_tp3_wakeup_time_us = dev_priv->vbt.psr.tp2_tp3_wakeup_time_us;
+	}
 }
 
 static void parse_dsi_backlight_ports(struct drm_i915_private *dev_priv,
@@ -1222,10 +1247,11 @@ static void sanitize_ddc_pin(struct drm_i915_private *dev_priv,
 	if (!info->alternate_ddc_pin)
 		return;
 
-	for_each_port_masked(p, (1 << port) - 1) {
+	for (p = PORT_A; p < I915_MAX_PORTS; p++) {
 		struct ddi_vbt_port_info *i = &dev_priv->vbt.ddi_port_info[p];
 
-		if (info->alternate_ddc_pin != i->alternate_ddc_pin)
+		if (p == port || !i->present ||
+		    info->alternate_ddc_pin != i->alternate_ddc_pin)
 			continue;
 
 		DRM_DEBUG_KMS("port %c trying to use the same DDC pin (0x%x) as port %c, "
@@ -1239,8 +1265,8 @@ static void sanitize_ddc_pin(struct drm_i915_private *dev_priv,
 		 * port. Otherwise they share the same ddc bin and
 		 * system couldn't communicate with them separately.
 		 *
-		 * Due to parsing the ports in alphabetical order,
-		 * a higher port will always clobber a lower one.
+		 * Due to parsing the ports in child device order,
+		 * a later device will always clobber an earlier one.
 		 */
 		i->supports_dvi = false;
 		i->supports_hdmi = false;
@@ -1258,10 +1284,11 @@ static void sanitize_aux_ch(struct drm_i915_private *dev_priv,
 	if (!info->alternate_aux_channel)
 		return;
 
-	for_each_port_masked(p, (1 << port) - 1) {
+	for (p = PORT_A; p < I915_MAX_PORTS; p++) {
 		struct ddi_vbt_port_info *i = &dev_priv->vbt.ddi_port_info[p];
 
-		if (info->alternate_aux_channel != i->alternate_aux_channel)
+		if (p == port || !i->present ||
+		    info->alternate_aux_channel != i->alternate_aux_channel)
 			continue;
 
 		DRM_DEBUG_KMS("port %c trying to use the same AUX CH (0x%x) as port %c, "
@@ -1275,8 +1302,8 @@ static void sanitize_aux_ch(struct drm_i915_private *dev_priv,
 		 * port. Otherwise they share the same aux channel
 		 * and system couldn't communicate with them separately.
 		 *
-		 * Due to parsing the ports in alphabetical order,
-		 * a higher port will always clobber a lower one.
+		 * Due to parsing the ports in child device order,
+		 * a later device will always clobber an earlier one.
 		 */
 		i->supports_dp = false;
 		i->alternate_aux_channel = 0;
@@ -1324,48 +1351,57 @@ static u8 map_ddc_pin(struct drm_i915_private *dev_priv, u8 vbt_pin)
 	return 0;
 }
 
-static void parse_ddi_port(struct drm_i915_private *dev_priv, enum port port,
-			   u8 bdb_version)
+static enum port dvo_port_to_port(u8 dvo_port)
 {
-	struct child_device_config *it, *child = NULL;
-	struct ddi_vbt_port_info *info = &dev_priv->vbt.ddi_port_info[port];
-	int i, j;
-	bool is_dvi, is_hdmi, is_dp, is_edp, is_crt;
-	/* Each DDI port can have more than one value on the "DVO Port" field,
+	/*
+	 * Each DDI port can have more than one value on the "DVO Port" field,
 	 * so look for all the possible values for each port.
 	 */
-	int dvo_ports[][3] = {
-		{DVO_PORT_HDMIA, DVO_PORT_DPA, -1},
-		{DVO_PORT_HDMIB, DVO_PORT_DPB, -1},
-		{DVO_PORT_HDMIC, DVO_PORT_DPC, -1},
-		{DVO_PORT_HDMID, DVO_PORT_DPD, -1},
-		{DVO_PORT_CRT, DVO_PORT_HDMIE, DVO_PORT_DPE},
-		{DVO_PORT_HDMIF, DVO_PORT_DPF, -1},
+	static const int dvo_ports[][3] = {
+		[PORT_A] = { DVO_PORT_HDMIA, DVO_PORT_DPA, -1},
+		[PORT_B] = { DVO_PORT_HDMIB, DVO_PORT_DPB, -1},
+		[PORT_C] = { DVO_PORT_HDMIC, DVO_PORT_DPC, -1},
+		[PORT_D] = { DVO_PORT_HDMID, DVO_PORT_DPD, -1},
+		[PORT_E] = { DVO_PORT_CRT, DVO_PORT_HDMIE, DVO_PORT_DPE},
+		[PORT_F] = { DVO_PORT_HDMIF, DVO_PORT_DPF, -1},
 	};
+	enum port port;
+	int i;
 
-	/*
-	 * Find the first child device to reference the port, report if more
-	 * than one found.
-	 */
-	for (i = 0; i < dev_priv->vbt.child_dev_num; i++) {
-		it = dev_priv->vbt.child_dev + i;
-
-		for (j = 0; j < 3; j++) {
-			if (dvo_ports[port][j] == -1)
+	for (port = PORT_A; port < ARRAY_SIZE(dvo_ports); port++) {
+		for (i = 0; i < ARRAY_SIZE(dvo_ports[port]); i++) {
+			if (dvo_ports[port][i] == -1)
 				break;
 
-			if (it->dvo_port == dvo_ports[port][j]) {
-				if (child) {
-					DRM_DEBUG_KMS("More than one child device for port %c in VBT, using the first.\n",
-						      port_name(port));
-				} else {
-					child = it;
-				}
-			}
+			if (dvo_port == dvo_ports[port][i])
+				return port;
 		}
 	}
-	if (!child)
+
+	return PORT_NONE;
+}
+
+static void parse_ddi_port(struct drm_i915_private *dev_priv,
+			   const struct child_device_config *child,
+			   u8 bdb_version)
+{
+	struct ddi_vbt_port_info *info;
+	bool is_dvi, is_hdmi, is_dp, is_edp, is_crt;
+	enum port port;
+
+	port = dvo_port_to_port(child->dvo_port);
+	if (port == PORT_NONE)
+		return;
+
+	info = &dev_priv->vbt.ddi_port_info[port];
+
+	if (info->present) {
+		DRM_DEBUG_KMS("More than one child device for port %c in VBT, using the first.\n",
+			      port_name(port));
 		return;
+	}
+
+	info->present = true;
 
 	is_dvi = child->device_type & DEVICE_TYPE_TMDS_DVI_SIGNALING;
 	is_dp = child->device_type & DEVICE_TYPE_DISPLAYPORT_OUTPUT;
@@ -1498,19 +1534,20 @@ static void parse_ddi_port(struct drm_i915_private *dev_priv, enum port port,
 
 static void parse_ddi_ports(struct drm_i915_private *dev_priv, u8 bdb_version)
 {
-	enum port port;
+	const struct child_device_config *child;
+	int i;
 
 	if (!HAS_DDI(dev_priv) && !IS_CHERRYVIEW(dev_priv))
 		return;
 
-	if (!dev_priv->vbt.child_dev_num)
-		return;
-
 	if (bdb_version < 155)
 		return;
 
-	for (port = PORT_A; port < I915_MAX_PORTS; port++)
-		parse_ddi_port(dev_priv, port, bdb_version);
+	for (i = 0; i < dev_priv->vbt.child_dev_num; i++) {
+		child = dev_priv->vbt.child_dev + i;
+
+		parse_ddi_port(dev_priv, child, bdb_version);
+	}
 }
 
 static void
@@ -2094,8 +2131,8 @@ bool intel_bios_is_dsi_present(struct drm_i915_private *dev_priv,
 		dvo_port = child->dvo_port;
 
 		if (dvo_port == DVO_PORT_MIPIA ||
-		    (dvo_port == DVO_PORT_MIPIB && IS_ICELAKE(dev_priv)) ||
-		    (dvo_port == DVO_PORT_MIPIC && !IS_ICELAKE(dev_priv))) {
+		    (dvo_port == DVO_PORT_MIPIB && INTEL_GEN(dev_priv) >= 11) ||
+		    (dvo_port == DVO_PORT_MIPIC && INTEL_GEN(dev_priv) < 11)) {
 			if (port)
 				*port = dvo_port - DVO_PORT_MIPIA;
 			return true;
diff --git a/drivers/gpu/drm/i915/intel_breadcrumbs.c b/drivers/gpu/drm/i915/intel_breadcrumbs.c
index 09ed90c0ba00..3cbffd400b1b 100644
--- a/drivers/gpu/drm/i915/intel_breadcrumbs.c
+++ b/drivers/gpu/drm/i915/intel_breadcrumbs.c
@@ -27,8 +27,6 @@
 
 #include "i915_drv.h"
 
-#define task_asleep(tsk) ((tsk)->state & TASK_NORMAL && !(tsk)->on_rq)
-
 static void irq_enable(struct intel_engine_cs *engine)
 {
 	if (!engine->irq_enable)
@@ -82,7 +80,7 @@ static inline bool __request_completed(const struct i915_request *rq)
 	return i915_seqno_passed(__hwsp_seqno(rq), rq->fence.seqno);
 }
 
-bool intel_engine_breadcrumbs_irq(struct intel_engine_cs *engine)
+void intel_engine_breadcrumbs_irq(struct intel_engine_cs *engine)
 {
 	struct intel_breadcrumbs *b = &engine->breadcrumbs;
 	struct intel_context *ce, *cn;
@@ -146,19 +144,13 @@ bool intel_engine_breadcrumbs_irq(struct intel_engine_cs *engine)
 		dma_fence_signal(&rq->fence);
 		i915_request_put(rq);
 	}
-
-	return !list_empty(&signal);
 }
 
-bool intel_engine_signal_breadcrumbs(struct intel_engine_cs *engine)
+void intel_engine_signal_breadcrumbs(struct intel_engine_cs *engine)
 {
-	bool result;
-
 	local_irq_disable();
-	result = intel_engine_breadcrumbs_irq(engine);
+	intel_engine_breadcrumbs_irq(engine);
 	local_irq_enable();
-
-	return result;
 }
 
 static void signal_irq_work(struct irq_work *work)
diff --git a/drivers/gpu/drm/i915/intel_cdclk.c b/drivers/gpu/drm/i915/intel_cdclk.c
index 15ba950dee00..ae40a8679314 100644
--- a/drivers/gpu/drm/i915/intel_cdclk.c
+++ b/drivers/gpu/drm/i915/intel_cdclk.c
@@ -21,6 +21,7 @@
  * DEALINGS IN THE SOFTWARE.
  */
 
+#include "intel_cdclk.h"
 #include "intel_drv.h"
 
 /**
@@ -234,7 +235,8 @@ static unsigned int intel_hpll_vco(struct drm_i915_private *dev_priv)
 	else
 		return 0;
 
-	tmp = I915_READ(IS_MOBILE(dev_priv) ? HPLLVCO_MOBILE : HPLLVCO);
+	tmp = I915_READ(IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv) ?
+			HPLLVCO_MOBILE : HPLLVCO);
 
 	vco = vco_table[tmp & 0x7];
 	if (vco == 0)
@@ -468,7 +470,7 @@ static void vlv_get_cdclk(struct drm_i915_private *dev_priv,
 					       cdclk_state->vco);
 
 	mutex_lock(&dev_priv->pcu_lock);
-	val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
+	val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 	mutex_unlock(&dev_priv->pcu_lock);
 
 	if (IS_VALLEYVIEW(dev_priv))
@@ -516,7 +518,8 @@ static void vlv_program_pfi_credits(struct drm_i915_private *dev_priv)
 }
 
 static void vlv_set_cdclk(struct drm_i915_private *dev_priv,
-			  const struct intel_cdclk_state *cdclk_state)
+			  const struct intel_cdclk_state *cdclk_state,
+			  enum pipe pipe)
 {
 	int cdclk = cdclk_state->cdclk;
 	u32 val, cmd = cdclk_state->voltage_level;
@@ -543,11 +546,11 @@ static void vlv_set_cdclk(struct drm_i915_private *dev_priv,
 	wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_PIPE_A);
 
 	mutex_lock(&dev_priv->pcu_lock);
-	val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
+	val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 	val &= ~DSPFREQGUAR_MASK;
 	val |= (cmd << DSPFREQGUAR_SHIFT);
-	vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, val);
-	if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) &
+	vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
+	if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) &
 		      DSPFREQSTAT_MASK) == (cmd << DSPFREQSTAT_SHIFT),
 		     50)) {
 		DRM_ERROR("timed out waiting for CDclk change\n");
@@ -598,7 +601,8 @@ static void vlv_set_cdclk(struct drm_i915_private *dev_priv,
 }
 
 static void chv_set_cdclk(struct drm_i915_private *dev_priv,
-			  const struct intel_cdclk_state *cdclk_state)
+			  const struct intel_cdclk_state *cdclk_state,
+			  enum pipe pipe)
 {
 	int cdclk = cdclk_state->cdclk;
 	u32 val, cmd = cdclk_state->voltage_level;
@@ -624,11 +628,11 @@ static void chv_set_cdclk(struct drm_i915_private *dev_priv,
 	wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_PIPE_A);
 
 	mutex_lock(&dev_priv->pcu_lock);
-	val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
+	val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 	val &= ~DSPFREQGUAR_MASK_CHV;
 	val |= (cmd << DSPFREQGUAR_SHIFT_CHV);
-	vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, val);
-	if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) &
+	vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
+	if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) &
 		      DSPFREQSTAT_MASK_CHV) == (cmd << DSPFREQSTAT_SHIFT_CHV),
 		     50)) {
 		DRM_ERROR("timed out waiting for CDclk change\n");
@@ -697,7 +701,8 @@ static void bdw_get_cdclk(struct drm_i915_private *dev_priv,
 }
 
 static void bdw_set_cdclk(struct drm_i915_private *dev_priv,
-			  const struct intel_cdclk_state *cdclk_state)
+			  const struct intel_cdclk_state *cdclk_state,
+			  enum pipe pipe)
 {
 	int cdclk = cdclk_state->cdclk;
 	u32 val;
@@ -964,7 +969,7 @@ static void skl_dpll0_enable(struct drm_i915_private *dev_priv, int vco)
 
 	I915_WRITE(LCPLL1_CTL, I915_READ(LCPLL1_CTL) | LCPLL_PLL_ENABLE);
 
-	if (intel_wait_for_register(dev_priv,
+	if (intel_wait_for_register(&dev_priv->uncore,
 				    LCPLL1_CTL, LCPLL_PLL_LOCK, LCPLL_PLL_LOCK,
 				    5))
 		DRM_ERROR("DPLL0 not locked\n");
@@ -978,16 +983,17 @@ static void skl_dpll0_enable(struct drm_i915_private *dev_priv, int vco)
 static void skl_dpll0_disable(struct drm_i915_private *dev_priv)
 {
 	I915_WRITE(LCPLL1_CTL, I915_READ(LCPLL1_CTL) & ~LCPLL_PLL_ENABLE);
-	if (intel_wait_for_register(dev_priv,
-				   LCPLL1_CTL, LCPLL_PLL_LOCK, 0,
-				   1))
+	if (intel_wait_for_register(&dev_priv->uncore,
+				    LCPLL1_CTL, LCPLL_PLL_LOCK, 0,
+				    1))
 		DRM_ERROR("Couldn't disable DPLL0\n");
 
 	dev_priv->cdclk.hw.vco = 0;
 }
 
 static void skl_set_cdclk(struct drm_i915_private *dev_priv,
-			  const struct intel_cdclk_state *cdclk_state)
+			  const struct intel_cdclk_state *cdclk_state,
+			  enum pipe pipe)
 {
 	int cdclk = cdclk_state->cdclk;
 	int vco = cdclk_state->vco;
@@ -1123,16 +1129,7 @@ sanitize:
 	dev_priv->cdclk.hw.vco = -1;
 }
 
-/**
- * skl_init_cdclk - Initialize CDCLK on SKL
- * @dev_priv: i915 device
- *
- * Initialize CDCLK for SKL and derivatives. This is generally
- * done only during the display core initialization sequence,
- * after which the DMC will take care of turning CDCLK off/on
- * as needed.
- */
-void skl_init_cdclk(struct drm_i915_private *dev_priv)
+static void skl_init_cdclk(struct drm_i915_private *dev_priv)
 {
 	struct intel_cdclk_state cdclk_state;
 
@@ -1158,17 +1155,10 @@ void skl_init_cdclk(struct drm_i915_private *dev_priv)
 	cdclk_state.cdclk = skl_calc_cdclk(0, cdclk_state.vco);
 	cdclk_state.voltage_level = skl_calc_voltage_level(cdclk_state.cdclk);
 
-	skl_set_cdclk(dev_priv, &cdclk_state);
+	skl_set_cdclk(dev_priv, &cdclk_state, INVALID_PIPE);
 }
 
-/**
- * skl_uninit_cdclk - Uninitialize CDCLK on SKL
- * @dev_priv: i915 device
- *
- * Uninitialize CDCLK for SKL and derivatives. This is done only
- * during the display core uninitialization sequence.
- */
-void skl_uninit_cdclk(struct drm_i915_private *dev_priv)
+static void skl_uninit_cdclk(struct drm_i915_private *dev_priv)
 {
 	struct intel_cdclk_state cdclk_state = dev_priv->cdclk.hw;
 
@@ -1176,7 +1166,7 @@ void skl_uninit_cdclk(struct drm_i915_private *dev_priv)
 	cdclk_state.vco = 0;
 	cdclk_state.voltage_level = skl_calc_voltage_level(cdclk_state.cdclk);
 
-	skl_set_cdclk(dev_priv, &cdclk_state);
+	skl_set_cdclk(dev_priv, &cdclk_state, INVALID_PIPE);
 }
 
 static int bxt_calc_cdclk(int min_cdclk)
@@ -1323,7 +1313,7 @@ static void bxt_de_pll_disable(struct drm_i915_private *dev_priv)
 	I915_WRITE(BXT_DE_PLL_ENABLE, 0);
 
 	/* Timeout 200us */
-	if (intel_wait_for_register(dev_priv,
+	if (intel_wait_for_register(&dev_priv->uncore,
 				    BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 0,
 				    1))
 		DRM_ERROR("timeout waiting for DE PLL unlock\n");
@@ -1344,7 +1334,7 @@ static void bxt_de_pll_enable(struct drm_i915_private *dev_priv, int vco)
 	I915_WRITE(BXT_DE_PLL_ENABLE, BXT_DE_PLL_PLL_ENABLE);
 
 	/* Timeout 200us */
-	if (intel_wait_for_register(dev_priv,
+	if (intel_wait_for_register(&dev_priv->uncore,
 				    BXT_DE_PLL_ENABLE,
 				    BXT_DE_PLL_LOCK,
 				    BXT_DE_PLL_LOCK,
@@ -1355,7 +1345,8 @@ static void bxt_de_pll_enable(struct drm_i915_private *dev_priv, int vco)
 }
 
 static void bxt_set_cdclk(struct drm_i915_private *dev_priv,
-			  const struct intel_cdclk_state *cdclk_state)
+			  const struct intel_cdclk_state *cdclk_state,
+			  enum pipe pipe)
 {
 	int cdclk = cdclk_state->cdclk;
 	int vco = cdclk_state->vco;
@@ -1408,11 +1399,10 @@ static void bxt_set_cdclk(struct drm_i915_private *dev_priv,
 		bxt_de_pll_enable(dev_priv, vco);
 
 	val = divider | skl_cdclk_decimal(cdclk);
-	/*
-	 * FIXME if only the cd2x divider needs changing, it could be done
-	 * without shutting off the pipe (if only one pipe is active).
-	 */
-	val |= BXT_CDCLK_CD2X_PIPE_NONE;
+	if (pipe == INVALID_PIPE)
+		val |= BXT_CDCLK_CD2X_PIPE_NONE;
+	else
+		val |= BXT_CDCLK_CD2X_PIPE(pipe);
 	/*
 	 * Disable SSA Precharge when CD clock frequency < 500 MHz,
 	 * enable otherwise.
@@ -1421,6 +1411,9 @@ static void bxt_set_cdclk(struct drm_i915_private *dev_priv,
 		val |= BXT_CDCLK_SSA_PRECHARGE_ENABLE;
 	I915_WRITE(CDCLK_CTL, val);
 
+	if (pipe != INVALID_PIPE)
+		intel_wait_for_vblank(dev_priv, pipe);
+
 	mutex_lock(&dev_priv->pcu_lock);
 	/*
 	 * The timeout isn't specified, the 2ms used here is based on
@@ -1490,16 +1483,7 @@ sanitize:
 	dev_priv->cdclk.hw.vco = -1;
 }
 
-/**
- * bxt_init_cdclk - Initialize CDCLK on BXT
- * @dev_priv: i915 device
- *
- * Initialize CDCLK for BXT and derivatives. This is generally
- * done only during the display core initialization sequence,
- * after which the DMC will take care of turning CDCLK off/on
- * as needed.
- */
-void bxt_init_cdclk(struct drm_i915_private *dev_priv)
+static void bxt_init_cdclk(struct drm_i915_private *dev_priv)
 {
 	struct intel_cdclk_state cdclk_state;
 
@@ -1525,17 +1509,10 @@ void bxt_init_cdclk(struct drm_i915_private *dev_priv)
 	}
 	cdclk_state.voltage_level = bxt_calc_voltage_level(cdclk_state.cdclk);
 
-	bxt_set_cdclk(dev_priv, &cdclk_state);
+	bxt_set_cdclk(dev_priv, &cdclk_state, INVALID_PIPE);
 }
 
-/**
- * bxt_uninit_cdclk - Uninitialize CDCLK on BXT
- * @dev_priv: i915 device
- *
- * Uninitialize CDCLK for BXT and derivatives. This is done only
- * during the display core uninitialization sequence.
- */
-void bxt_uninit_cdclk(struct drm_i915_private *dev_priv)
+static void bxt_uninit_cdclk(struct drm_i915_private *dev_priv)
 {
 	struct intel_cdclk_state cdclk_state = dev_priv->cdclk.hw;
 
@@ -1543,7 +1520,7 @@ void bxt_uninit_cdclk(struct drm_i915_private *dev_priv)
 	cdclk_state.vco = 0;
 	cdclk_state.voltage_level = bxt_calc_voltage_level(cdclk_state.cdclk);
 
-	bxt_set_cdclk(dev_priv, &cdclk_state);
+	bxt_set_cdclk(dev_priv, &cdclk_state, INVALID_PIPE);
 }
 
 static int cnl_calc_cdclk(int min_cdclk)
@@ -1663,7 +1640,8 @@ static void cnl_cdclk_pll_enable(struct drm_i915_private *dev_priv, int vco)
 }
 
 static void cnl_set_cdclk(struct drm_i915_private *dev_priv,
-			  const struct intel_cdclk_state *cdclk_state)
+			  const struct intel_cdclk_state *cdclk_state,
+			  enum pipe pipe)
 {
 	int cdclk = cdclk_state->cdclk;
 	int vco = cdclk_state->vco;
@@ -1704,13 +1682,15 @@ static void cnl_set_cdclk(struct drm_i915_private *dev_priv,
 		cnl_cdclk_pll_enable(dev_priv, vco);
 
 	val = divider | skl_cdclk_decimal(cdclk);
-	/*
-	 * FIXME if only the cd2x divider needs changing, it could be done
-	 * without shutting off the pipe (if only one pipe is active).
-	 */
-	val |= BXT_CDCLK_CD2X_PIPE_NONE;
+	if (pipe == INVALID_PIPE)
+		val |= BXT_CDCLK_CD2X_PIPE_NONE;
+	else
+		val |= BXT_CDCLK_CD2X_PIPE(pipe);
 	I915_WRITE(CDCLK_CTL, val);
 
+	if (pipe != INVALID_PIPE)
+		intel_wait_for_vblank(dev_priv, pipe);
+
 	/* inform PCU of the change */
 	mutex_lock(&dev_priv->pcu_lock);
 	sandybridge_pcode_write(dev_priv, SKL_PCODE_CDCLK_CONTROL,
@@ -1847,7 +1827,8 @@ static int icl_calc_cdclk_pll_vco(struct drm_i915_private *dev_priv, int cdclk)
 }
 
 static void icl_set_cdclk(struct drm_i915_private *dev_priv,
-			  const struct intel_cdclk_state *cdclk_state)
+			  const struct intel_cdclk_state *cdclk_state,
+			  enum pipe pipe)
 {
 	unsigned int cdclk = cdclk_state->cdclk;
 	unsigned int vco = cdclk_state->vco;
@@ -1872,6 +1853,11 @@ static void icl_set_cdclk(struct drm_i915_private *dev_priv,
 	if (dev_priv->cdclk.hw.vco != vco)
 		cnl_cdclk_pll_enable(dev_priv, vco);
 
+	/*
+	 * On ICL CD2X_DIV can only be 1, so we'll never end up changing the
+	 * divider here synchronized to a pipe while CDCLK is on, nor will we
+	 * need the corresponding vblank wait.
+	 */
 	I915_WRITE(CDCLK_CTL, ICL_CDCLK_CD2X_PIPE_NONE |
 			      skl_cdclk_decimal(cdclk));
 
@@ -1959,16 +1945,7 @@ out:
 		icl_calc_voltage_level(cdclk_state->cdclk);
 }
 
-/**
- * icl_init_cdclk - Initialize CDCLK on ICL
- * @dev_priv: i915 device
- *
- * Initialize CDCLK for ICL. This consists mainly of initializing
- * dev_priv->cdclk.hw and sanitizing the state of the hardware if needed. This
- * is generally done only during the display core initialization sequence, after
- * which the DMC will take care of turning CDCLK off/on as needed.
- */
-void icl_init_cdclk(struct drm_i915_private *dev_priv)
+static void icl_init_cdclk(struct drm_i915_private *dev_priv)
 {
 	struct intel_cdclk_state sanitized_state;
 	u32 val;
@@ -2002,17 +1979,10 @@ sanitize:
 	sanitized_state.voltage_level =
 				icl_calc_voltage_level(sanitized_state.cdclk);
 
-	icl_set_cdclk(dev_priv, &sanitized_state);
+	icl_set_cdclk(dev_priv, &sanitized_state, INVALID_PIPE);
 }
 
-/**
- * icl_uninit_cdclk - Uninitialize CDCLK on ICL
- * @dev_priv: i915 device
- *
- * Uninitialize CDCLK for ICL. This is done only during the display core
- * uninitialization sequence.
- */
-void icl_uninit_cdclk(struct drm_i915_private *dev_priv)
+static void icl_uninit_cdclk(struct drm_i915_private *dev_priv)
 {
 	struct intel_cdclk_state cdclk_state = dev_priv->cdclk.hw;
 
@@ -2020,19 +1990,10 @@ void icl_uninit_cdclk(struct drm_i915_private *dev_priv)
 	cdclk_state.vco = 0;
 	cdclk_state.voltage_level = icl_calc_voltage_level(cdclk_state.cdclk);
 
-	icl_set_cdclk(dev_priv, &cdclk_state);
+	icl_set_cdclk(dev_priv, &cdclk_state, INVALID_PIPE);
 }
 
-/**
- * cnl_init_cdclk - Initialize CDCLK on CNL
- * @dev_priv: i915 device
- *
- * Initialize CDCLK for CNL. This is generally
- * done only during the display core initialization sequence,
- * after which the DMC will take care of turning CDCLK off/on
- * as needed.
- */
-void cnl_init_cdclk(struct drm_i915_private *dev_priv)
+static void cnl_init_cdclk(struct drm_i915_private *dev_priv)
 {
 	struct intel_cdclk_state cdclk_state;
 
@@ -2048,17 +2009,10 @@ void cnl_init_cdclk(struct drm_i915_private *dev_priv)
 	cdclk_state.vco = cnl_cdclk_pll_vco(dev_priv, cdclk_state.cdclk);
 	cdclk_state.voltage_level = cnl_calc_voltage_level(cdclk_state.cdclk);
 
-	cnl_set_cdclk(dev_priv, &cdclk_state);
+	cnl_set_cdclk(dev_priv, &cdclk_state, INVALID_PIPE);
 }
 
-/**
- * cnl_uninit_cdclk - Uninitialize CDCLK on CNL
- * @dev_priv: i915 device
- *
- * Uninitialize CDCLK for CNL. This is done only
- * during the display core uninitialization sequence.
- */
-void cnl_uninit_cdclk(struct drm_i915_private *dev_priv)
+static void cnl_uninit_cdclk(struct drm_i915_private *dev_priv)
 {
 	struct intel_cdclk_state cdclk_state = dev_priv->cdclk.hw;
 
@@ -2066,7 +2020,47 @@ void cnl_uninit_cdclk(struct drm_i915_private *dev_priv)
 	cdclk_state.vco = 0;
 	cdclk_state.voltage_level = cnl_calc_voltage_level(cdclk_state.cdclk);
 
-	cnl_set_cdclk(dev_priv, &cdclk_state);
+	cnl_set_cdclk(dev_priv, &cdclk_state, INVALID_PIPE);
+}
+
+/**
+ * intel_cdclk_init - Initialize CDCLK
+ * @i915: i915 device
+ *
+ * Initialize CDCLK. This consists mainly of initializing dev_priv->cdclk.hw and
+ * sanitizing the state of the hardware if needed. This is generally done only
+ * during the display core initialization sequence, after which the DMC will
+ * take care of turning CDCLK off/on as needed.
+ */
+void intel_cdclk_init(struct drm_i915_private *i915)
+{
+	if (INTEL_GEN(i915) >= 11)
+		icl_init_cdclk(i915);
+	else if (IS_CANNONLAKE(i915))
+		cnl_init_cdclk(i915);
+	else if (IS_GEN9_BC(i915))
+		skl_init_cdclk(i915);
+	else if (IS_GEN9_LP(i915))
+		bxt_init_cdclk(i915);
+}
+
+/**
+ * intel_cdclk_uninit - Uninitialize CDCLK
+ * @i915: i915 device
+ *
+ * Uninitialize CDCLK. This is done only during the display core
+ * uninitialization sequence.
+ */
+void intel_cdclk_uninit(struct drm_i915_private *i915)
+{
+	if (INTEL_GEN(i915) >= 11)
+		icl_uninit_cdclk(i915);
+	else if (IS_CANNONLAKE(i915))
+		cnl_uninit_cdclk(i915);
+	else if (IS_GEN9_BC(i915))
+		skl_uninit_cdclk(i915);
+	else if (IS_GEN9_LP(i915))
+		bxt_uninit_cdclk(i915);
 }
 
 /**
@@ -2086,6 +2080,28 @@ bool intel_cdclk_needs_modeset(const struct intel_cdclk_state *a,
 }
 
 /**
+ * intel_cdclk_needs_cd2x_update - Determine if two CDCLK states require a cd2x divider update
+ * @dev_priv: Not a CDCLK state, it's the drm_i915_private!
+ * @a: first CDCLK state
+ * @b: second CDCLK state
+ *
+ * Returns:
+ * True if the CDCLK states require just a cd2x divider update, false if not.
+ */
+bool intel_cdclk_needs_cd2x_update(struct drm_i915_private *dev_priv,
+				   const struct intel_cdclk_state *a,
+				   const struct intel_cdclk_state *b)
+{
+	/* Older hw doesn't have the capability */
+	if (INTEL_GEN(dev_priv) < 10 && !IS_GEN9_LP(dev_priv))
+		return false;
+
+	return a->cdclk != b->cdclk &&
+		a->vco == b->vco &&
+		a->ref == b->ref;
+}
+
+/**
  * intel_cdclk_changed - Determine if two CDCLK states are different
  * @a: first CDCLK state
  * @b: second CDCLK state
@@ -2100,6 +2116,26 @@ bool intel_cdclk_changed(const struct intel_cdclk_state *a,
 		a->voltage_level != b->voltage_level;
 }
 
+/**
+ * intel_cdclk_swap_state - make atomic CDCLK configuration effective
+ * @state: atomic state
+ *
+ * This is the CDCLK version of drm_atomic_helper_swap_state() since the
+ * helper does not handle driver-specific global state.
+ *
+ * Similarly to the atomic helpers this function does a complete swap,
+ * i.e. it also puts the old state into @state. This is used by the commit
+ * code to determine how CDCLK has changed (for instance did it increase or
+ * decrease).
+ */
+void intel_cdclk_swap_state(struct intel_atomic_state *state)
+{
+	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
+
+	swap(state->cdclk.logical, dev_priv->cdclk.logical);
+	swap(state->cdclk.actual, dev_priv->cdclk.actual);
+}
+
 void intel_dump_cdclk_state(const struct intel_cdclk_state *cdclk_state,
 			    const char *context)
 {
@@ -2113,12 +2149,14 @@ void intel_dump_cdclk_state(const struct intel_cdclk_state *cdclk_state,
  * intel_set_cdclk - Push the CDCLK state to the hardware
  * @dev_priv: i915 device
  * @cdclk_state: new CDCLK state
+ * @pipe: pipe with which to synchronize the update
  *
  * Program the hardware based on the passed in CDCLK state,
  * if necessary.
  */
-void intel_set_cdclk(struct drm_i915_private *dev_priv,
-		     const struct intel_cdclk_state *cdclk_state)
+static void intel_set_cdclk(struct drm_i915_private *dev_priv,
+			    const struct intel_cdclk_state *cdclk_state,
+			    enum pipe pipe)
 {
 	if (!intel_cdclk_changed(&dev_priv->cdclk.hw, cdclk_state))
 		return;
@@ -2128,7 +2166,7 @@ void intel_set_cdclk(struct drm_i915_private *dev_priv,
 
 	intel_dump_cdclk_state(cdclk_state, "Changing CDCLK to");
 
-	dev_priv->display.set_cdclk(dev_priv, cdclk_state);
+	dev_priv->display.set_cdclk(dev_priv, cdclk_state, pipe);
 
 	if (WARN(intel_cdclk_changed(&dev_priv->cdclk.hw, cdclk_state),
 		 "cdclk state doesn't match!\n")) {
@@ -2137,6 +2175,46 @@ void intel_set_cdclk(struct drm_i915_private *dev_priv,
 	}
 }
 
+/**
+ * intel_set_cdclk_pre_plane_update - Push the CDCLK state to the hardware
+ * @dev_priv: i915 device
+ * @old_state: old CDCLK state
+ * @new_state: new CDCLK state
+ * @pipe: pipe with which to synchronize the update
+ *
+ * Program the hardware before updating the HW plane state based on the passed
+ * in CDCLK state, if necessary.
+ */
+void
+intel_set_cdclk_pre_plane_update(struct drm_i915_private *dev_priv,
+				 const struct intel_cdclk_state *old_state,
+				 const struct intel_cdclk_state *new_state,
+				 enum pipe pipe)
+{
+	if (pipe == INVALID_PIPE || old_state->cdclk <= new_state->cdclk)
+		intel_set_cdclk(dev_priv, new_state, pipe);
+}
+
+/**
+ * intel_set_cdclk_post_plane_update - Push the CDCLK state to the hardware
+ * @dev_priv: i915 device
+ * @old_state: old CDCLK state
+ * @new_state: new CDCLK state
+ * @pipe: pipe with which to synchronize the update
+ *
+ * Program the hardware after updating the HW plane state based on the passed
+ * in CDCLK state, if necessary.
+ */
+void
+intel_set_cdclk_post_plane_update(struct drm_i915_private *dev_priv,
+				  const struct intel_cdclk_state *old_state,
+				  const struct intel_cdclk_state *new_state,
+				  enum pipe pipe)
+{
+	if (pipe != INVALID_PIPE && old_state->cdclk > new_state->cdclk)
+		intel_set_cdclk(dev_priv, new_state, pipe);
+}
+
 static int intel_pixel_rate_to_cdclk(struct drm_i915_private *dev_priv,
 				     int pixel_rate)
 {
@@ -2187,19 +2265,8 @@ int intel_crtc_compute_min_cdclk(const struct intel_crtc_state *crtc_state)
 	/*
 	 * According to BSpec, "The CD clock frequency must be at least twice
 	 * the frequency of the Azalia BCLK." and BCLK is 96 MHz by default.
-	 *
-	 * FIXME: Check the actual, not default, BCLK being used.
-	 *
-	 * FIXME: This does not depend on ->has_audio because the higher CDCLK
-	 * is required for audio probe, also when there are no audio capable
-	 * displays connected at probe time. This leads to unnecessarily high
-	 * CDCLK when audio is not required.
-	 *
-	 * FIXME: This limit is only applied when there are displays connected
-	 * at probe time. If we probe without displays, we'll still end up using
-	 * the platform minimum CDCLK, failing audio probe.
 	 */
-	if (INTEL_GEN(dev_priv) >= 9)
+	if (crtc_state->has_audio && INTEL_GEN(dev_priv) >= 9)
 		min_cdclk = max(2 * 96000, min_cdclk);
 
 	/*
@@ -2239,7 +2306,7 @@ static int intel_compute_min_cdclk(struct drm_atomic_state *state)
 		intel_state->min_cdclk[i] = min_cdclk;
 	}
 
-	min_cdclk = 0;
+	min_cdclk = intel_state->cdclk.force_min_cdclk;
 	for_each_pipe(dev_priv, pipe)
 		min_cdclk = max(intel_state->min_cdclk[pipe], min_cdclk);
 
@@ -2300,7 +2367,8 @@ static int vlv_modeset_calc_cdclk(struct drm_atomic_state *state)
 		vlv_calc_voltage_level(dev_priv, cdclk);
 
 	if (!intel_state->active_crtcs) {
-		cdclk = vlv_calc_cdclk(dev_priv, 0);
+		cdclk = vlv_calc_cdclk(dev_priv,
+				       intel_state->cdclk.force_min_cdclk);
 
 		intel_state->cdclk.actual.cdclk = cdclk;
 		intel_state->cdclk.actual.voltage_level =
@@ -2333,7 +2401,7 @@ static int bdw_modeset_calc_cdclk(struct drm_atomic_state *state)
 		bdw_calc_voltage_level(cdclk);
 
 	if (!intel_state->active_crtcs) {
-		cdclk = bdw_calc_cdclk(0);
+		cdclk = bdw_calc_cdclk(intel_state->cdclk.force_min_cdclk);
 
 		intel_state->cdclk.actual.cdclk = cdclk;
 		intel_state->cdclk.actual.voltage_level =
@@ -2405,7 +2473,7 @@ static int skl_modeset_calc_cdclk(struct drm_atomic_state *state)
 		skl_calc_voltage_level(cdclk);
 
 	if (!intel_state->active_crtcs) {
-		cdclk = skl_calc_cdclk(0, vco);
+		cdclk = skl_calc_cdclk(intel_state->cdclk.force_min_cdclk, vco);
 
 		intel_state->cdclk.actual.vco = vco;
 		intel_state->cdclk.actual.cdclk = cdclk;
@@ -2444,10 +2512,10 @@ static int bxt_modeset_calc_cdclk(struct drm_atomic_state *state)
 
 	if (!intel_state->active_crtcs) {
 		if (IS_GEMINILAKE(dev_priv)) {
-			cdclk = glk_calc_cdclk(0);
+			cdclk = glk_calc_cdclk(intel_state->cdclk.force_min_cdclk);
 			vco = glk_de_pll_vco(dev_priv, cdclk);
 		} else {
-			cdclk = bxt_calc_cdclk(0);
+			cdclk = bxt_calc_cdclk(intel_state->cdclk.force_min_cdclk);
 			vco = bxt_de_pll_vco(dev_priv, cdclk);
 		}
 
@@ -2483,7 +2551,7 @@ static int cnl_modeset_calc_cdclk(struct drm_atomic_state *state)
 		    cnl_compute_min_voltage_level(intel_state));
 
 	if (!intel_state->active_crtcs) {
-		cdclk = cnl_calc_cdclk(0);
+		cdclk = cnl_calc_cdclk(intel_state->cdclk.force_min_cdclk);
 		vco = cnl_cdclk_pll_vco(dev_priv, cdclk);
 
 		intel_state->cdclk.actual.vco = vco;
@@ -2519,7 +2587,7 @@ static int icl_modeset_calc_cdclk(struct drm_atomic_state *state)
 		    cnl_compute_min_voltage_level(intel_state));
 
 	if (!intel_state->active_crtcs) {
-		cdclk = icl_calc_cdclk(0, ref);
+		cdclk = icl_calc_cdclk(intel_state->cdclk.force_min_cdclk, ref);
 		vco = icl_calc_cdclk_pll_vco(dev_priv, cdclk);
 
 		intel_state->cdclk.actual.vco = vco;
@@ -2560,7 +2628,7 @@ static int intel_compute_max_dotclk(struct drm_i915_private *dev_priv)
  */
 void intel_update_max_cdclk(struct drm_i915_private *dev_priv)
 {
-	if (IS_ICELAKE(dev_priv)) {
+	if (INTEL_GEN(dev_priv) >= 11) {
 		if (dev_priv->cdclk.hw.ref == 24000)
 			dev_priv->max_cdclk_freq = 648000;
 		else
@@ -2668,7 +2736,7 @@ static int cnp_rawclk(struct drm_i915_private *dev_priv)
 
 		rawclk |= CNP_RAWCLK_DEN(DIV_ROUND_CLOSEST(numerator * 1000,
 							   fraction) - 1);
-		if (HAS_PCH_ICP(dev_priv))
+		if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
 			rawclk |= ICP_RAWCLK_NUM(numerator);
 	}
 
@@ -2723,7 +2791,7 @@ static int g4x_hrawclk(struct drm_i915_private *dev_priv)
  */
 void intel_update_rawclk(struct drm_i915_private *dev_priv)
 {
-	if (HAS_PCH_CNP(dev_priv) || HAS_PCH_ICP(dev_priv))
+	if (INTEL_PCH_TYPE(dev_priv) >= PCH_CNP)
 		dev_priv->rawclk_freq = cnp_rawclk(dev_priv);
 	else if (HAS_PCH_SPLIT(dev_priv))
 		dev_priv->rawclk_freq = pch_rawclk(dev_priv);
@@ -2744,18 +2812,13 @@ void intel_update_rawclk(struct drm_i915_private *dev_priv)
  */
 void intel_init_cdclk_hooks(struct drm_i915_private *dev_priv)
 {
-	if (IS_CHERRYVIEW(dev_priv)) {
-		dev_priv->display.set_cdclk = chv_set_cdclk;
-		dev_priv->display.modeset_calc_cdclk =
-			vlv_modeset_calc_cdclk;
-	} else if (IS_VALLEYVIEW(dev_priv)) {
-		dev_priv->display.set_cdclk = vlv_set_cdclk;
-		dev_priv->display.modeset_calc_cdclk =
-			vlv_modeset_calc_cdclk;
-	} else if (IS_BROADWELL(dev_priv)) {
-		dev_priv->display.set_cdclk = bdw_set_cdclk;
+	if (INTEL_GEN(dev_priv) >= 11) {
+		dev_priv->display.set_cdclk = icl_set_cdclk;
+		dev_priv->display.modeset_calc_cdclk = icl_modeset_calc_cdclk;
+	} else if (IS_CANNONLAKE(dev_priv)) {
+		dev_priv->display.set_cdclk = cnl_set_cdclk;
 		dev_priv->display.modeset_calc_cdclk =
-			bdw_modeset_calc_cdclk;
+			cnl_modeset_calc_cdclk;
 	} else if (IS_GEN9_LP(dev_priv)) {
 		dev_priv->display.set_cdclk = bxt_set_cdclk;
 		dev_priv->display.modeset_calc_cdclk =
@@ -2764,23 +2827,28 @@ void intel_init_cdclk_hooks(struct drm_i915_private *dev_priv)
 		dev_priv->display.set_cdclk = skl_set_cdclk;
 		dev_priv->display.modeset_calc_cdclk =
 			skl_modeset_calc_cdclk;
-	} else if (IS_CANNONLAKE(dev_priv)) {
-		dev_priv->display.set_cdclk = cnl_set_cdclk;
+	} else if (IS_BROADWELL(dev_priv)) {
+		dev_priv->display.set_cdclk = bdw_set_cdclk;
 		dev_priv->display.modeset_calc_cdclk =
-			cnl_modeset_calc_cdclk;
-	} else if (IS_ICELAKE(dev_priv)) {
-		dev_priv->display.set_cdclk = icl_set_cdclk;
-		dev_priv->display.modeset_calc_cdclk = icl_modeset_calc_cdclk;
+			bdw_modeset_calc_cdclk;
+	} else if (IS_CHERRYVIEW(dev_priv)) {
+		dev_priv->display.set_cdclk = chv_set_cdclk;
+		dev_priv->display.modeset_calc_cdclk =
+			vlv_modeset_calc_cdclk;
+	} else if (IS_VALLEYVIEW(dev_priv)) {
+		dev_priv->display.set_cdclk = vlv_set_cdclk;
+		dev_priv->display.modeset_calc_cdclk =
+			vlv_modeset_calc_cdclk;
 	}
 
-	if (IS_ICELAKE(dev_priv))
+	if (INTEL_GEN(dev_priv) >= 11)
 		dev_priv->display.get_cdclk = icl_get_cdclk;
 	else if (IS_CANNONLAKE(dev_priv))
 		dev_priv->display.get_cdclk = cnl_get_cdclk;
-	else if (IS_GEN9_BC(dev_priv))
-		dev_priv->display.get_cdclk = skl_get_cdclk;
 	else if (IS_GEN9_LP(dev_priv))
 		dev_priv->display.get_cdclk = bxt_get_cdclk;
+	else if (IS_GEN9_BC(dev_priv))
+		dev_priv->display.get_cdclk = skl_get_cdclk;
 	else if (IS_BROADWELL(dev_priv))
 		dev_priv->display.get_cdclk = bdw_get_cdclk;
 	else if (IS_HASWELL(dev_priv))
diff --git a/drivers/gpu/drm/i915/intel_cdclk.h b/drivers/gpu/drm/i915/intel_cdclk.h
new file mode 100644
index 000000000000..4d6f7f5f8930
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_cdclk.h
@@ -0,0 +1,46 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2019 Intel Corporation
+ */
+
+#ifndef __INTEL_CDCLK_H__
+#define __INTEL_CDCLK_H__
+
+#include <linux/types.h>
+
+#include "intel_display.h"
+
+struct drm_i915_private;
+struct intel_atomic_state;
+struct intel_cdclk_state;
+struct intel_crtc_state;
+
+int intel_crtc_compute_min_cdclk(const struct intel_crtc_state *crtc_state);
+void intel_cdclk_init(struct drm_i915_private *i915);
+void intel_cdclk_uninit(struct drm_i915_private *i915);
+void intel_init_cdclk_hooks(struct drm_i915_private *dev_priv);
+void intel_update_max_cdclk(struct drm_i915_private *dev_priv);
+void intel_update_cdclk(struct drm_i915_private *dev_priv);
+void intel_update_rawclk(struct drm_i915_private *dev_priv);
+bool intel_cdclk_needs_cd2x_update(struct drm_i915_private *dev_priv,
+				   const struct intel_cdclk_state *a,
+				   const struct intel_cdclk_state *b);
+bool intel_cdclk_needs_modeset(const struct intel_cdclk_state *a,
+			       const struct intel_cdclk_state *b);
+bool intel_cdclk_changed(const struct intel_cdclk_state *a,
+			 const struct intel_cdclk_state *b);
+void intel_cdclk_swap_state(struct intel_atomic_state *state);
+void
+intel_set_cdclk_pre_plane_update(struct drm_i915_private *dev_priv,
+				 const struct intel_cdclk_state *old_state,
+				 const struct intel_cdclk_state *new_state,
+				 enum pipe pipe);
+void
+intel_set_cdclk_post_plane_update(struct drm_i915_private *dev_priv,
+				  const struct intel_cdclk_state *old_state,
+				  const struct intel_cdclk_state *new_state,
+				  enum pipe pipe);
+void intel_dump_cdclk_state(const struct intel_cdclk_state *cdclk_state,
+			    const char *context);
+
+#endif /* __INTEL_CDCLK_H__ */
diff --git a/drivers/gpu/drm/i915/intel_color.c b/drivers/gpu/drm/i915/intel_color.c
index 71a1f12c6b2a..9093daabc290 100644
--- a/drivers/gpu/drm/i915/intel_color.c
+++ b/drivers/gpu/drm/i915/intel_color.c
@@ -22,6 +22,7 @@
  *
  */
 
+#include "intel_color.h"
 #include "intel_drv.h"
 
 #define CTM_COEFF_SIGN	(1ULL << 63)
@@ -40,23 +41,6 @@
 #define CTM_COEFF_ABS(coeff)		((coeff) & (CTM_COEFF_SIGN - 1))
 
 #define LEGACY_LUT_LENGTH		256
-
-/* Post offset values for RGB->YCBCR conversion */
-#define POSTOFF_RGB_TO_YUV_HI 0x800
-#define POSTOFF_RGB_TO_YUV_ME 0x100
-#define POSTOFF_RGB_TO_YUV_LO 0x800
-
-/*
- * These values are direct register values specified in the Bspec,
- * for RGB->YUV conversion matrix (colorspace BT709)
- */
-#define CSC_RGB_TO_YUV_RU_GU 0x2ba809d8
-#define CSC_RGB_TO_YUV_BU 0x37e80000
-#define CSC_RGB_TO_YUV_RY_GY 0x1e089cc0
-#define CSC_RGB_TO_YUV_BY 0xb5280000
-#define CSC_RGB_TO_YUV_RV_GV 0xbce89ad8
-#define CSC_RGB_TO_YUV_BV 0x1e080000
-
 /*
  * Extract the CSC coefficient from a CTM coefficient (in U32.32 fixed point
  * format). This macro takes the coefficient we want transformed and the
@@ -69,10 +53,45 @@
 #define ILK_CSC_COEFF_FP(coeff, fbits)	\
 	(clamp_val(((coeff) >> (32 - (fbits) - 3)) + 4, 0, 0xfff) & 0xff8)
 
-#define ILK_CSC_COEFF_LIMITED_RANGE	\
-	ILK_CSC_COEFF_FP(CTM_COEFF_LIMITED_RANGE, 9)
-#define ILK_CSC_COEFF_1_0		\
-	((7 << 12) | ILK_CSC_COEFF_FP(CTM_COEFF_1_0, 8))
+#define ILK_CSC_COEFF_LIMITED_RANGE 0x0dc0
+#define ILK_CSC_COEFF_1_0 0x7800
+
+#define ILK_CSC_POSTOFF_LIMITED_RANGE (16 * (1 << 12) / 255)
+
+static const u16 ilk_csc_off_zero[3] = {};
+
+static const u16 ilk_csc_coeff_identity[9] = {
+	ILK_CSC_COEFF_1_0, 0, 0,
+	0, ILK_CSC_COEFF_1_0, 0,
+	0, 0, ILK_CSC_COEFF_1_0,
+};
+
+static const u16 ilk_csc_postoff_limited_range[3] = {
+	ILK_CSC_POSTOFF_LIMITED_RANGE,
+	ILK_CSC_POSTOFF_LIMITED_RANGE,
+	ILK_CSC_POSTOFF_LIMITED_RANGE,
+};
+
+static const u16 ilk_csc_coeff_limited_range[9] = {
+	ILK_CSC_COEFF_LIMITED_RANGE, 0, 0,
+	0, ILK_CSC_COEFF_LIMITED_RANGE, 0,
+	0, 0, ILK_CSC_COEFF_LIMITED_RANGE,
+};
+
+/*
+ * These values are direct register values specified in the Bspec,
+ * for RGB->YUV conversion matrix (colorspace BT709)
+ */
+static const u16 ilk_csc_coeff_rgb_to_ycbcr[9] = {
+	0x1e08, 0x9cc0, 0xb528,
+	0x2ba8, 0x09d8, 0x37e8,
+	0xbce8, 0x9ad8, 0x1e08,
+};
+
+/* Post offset values for RGB->YCBCR conversion */
+static const u16 ilk_csc_postoff_rgb_to_ycbcr[3] = {
+	0x0800, 0x0100, 0x0800,
+};
 
 static bool lut_is_legacy(const struct drm_property_blob *lut)
 {
@@ -113,145 +132,188 @@ static u64 *ctm_mult_by_limited(u64 *result, const u64 *input)
 	return result;
 }
 
-static void ilk_load_ycbcr_conversion_matrix(struct intel_crtc *crtc)
+static void ilk_update_pipe_csc(struct intel_crtc *crtc,
+				const u16 preoff[3],
+				const u16 coeff[9],
+				const u16 postoff[3])
 {
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	enum pipe pipe = crtc->pipe;
 
-	I915_WRITE(PIPE_CSC_PREOFF_HI(pipe), 0);
-	I915_WRITE(PIPE_CSC_PREOFF_ME(pipe), 0);
-	I915_WRITE(PIPE_CSC_PREOFF_LO(pipe), 0);
+	I915_WRITE(PIPE_CSC_PREOFF_HI(pipe), preoff[0]);
+	I915_WRITE(PIPE_CSC_PREOFF_ME(pipe), preoff[1]);
+	I915_WRITE(PIPE_CSC_PREOFF_LO(pipe), preoff[2]);
 
-	I915_WRITE(PIPE_CSC_COEFF_RU_GU(pipe), CSC_RGB_TO_YUV_RU_GU);
-	I915_WRITE(PIPE_CSC_COEFF_BU(pipe), CSC_RGB_TO_YUV_BU);
+	I915_WRITE(PIPE_CSC_COEFF_RY_GY(pipe), coeff[0] << 16 | coeff[1]);
+	I915_WRITE(PIPE_CSC_COEFF_BY(pipe), coeff[2] << 16);
 
-	I915_WRITE(PIPE_CSC_COEFF_RY_GY(pipe), CSC_RGB_TO_YUV_RY_GY);
-	I915_WRITE(PIPE_CSC_COEFF_BY(pipe), CSC_RGB_TO_YUV_BY);
+	I915_WRITE(PIPE_CSC_COEFF_RU_GU(pipe), coeff[3] << 16 | coeff[4]);
+	I915_WRITE(PIPE_CSC_COEFF_BU(pipe), coeff[5] << 16);
 
-	I915_WRITE(PIPE_CSC_COEFF_RV_GV(pipe), CSC_RGB_TO_YUV_RV_GV);
-	I915_WRITE(PIPE_CSC_COEFF_BV(pipe), CSC_RGB_TO_YUV_BV);
+	I915_WRITE(PIPE_CSC_COEFF_RV_GV(pipe), coeff[6] << 16 | coeff[7]);
+	I915_WRITE(PIPE_CSC_COEFF_BV(pipe), coeff[8] << 16);
 
-	I915_WRITE(PIPE_CSC_POSTOFF_HI(pipe), POSTOFF_RGB_TO_YUV_HI);
-	I915_WRITE(PIPE_CSC_POSTOFF_ME(pipe), POSTOFF_RGB_TO_YUV_ME);
-	I915_WRITE(PIPE_CSC_POSTOFF_LO(pipe), POSTOFF_RGB_TO_YUV_LO);
-	I915_WRITE(PIPE_CSC_MODE(pipe), 0);
+	if (INTEL_GEN(dev_priv) >= 7) {
+		I915_WRITE(PIPE_CSC_POSTOFF_HI(pipe), postoff[0]);
+		I915_WRITE(PIPE_CSC_POSTOFF_ME(pipe), postoff[1]);
+		I915_WRITE(PIPE_CSC_POSTOFF_LO(pipe), postoff[2]);
+	}
 }
 
-static void ilk_load_csc_matrix(const struct intel_crtc_state *crtc_state)
+static void icl_update_output_csc(struct intel_crtc *crtc,
+				  const u16 preoff[3],
+				  const u16 coeff[9],
+				  const u16 postoff[3])
 {
-	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	bool limited_color_range = false;
 	enum pipe pipe = crtc->pipe;
-	u16 coeffs[9] = {};
-	int i;
+
+	I915_WRITE(PIPE_CSC_OUTPUT_PREOFF_HI(pipe), preoff[0]);
+	I915_WRITE(PIPE_CSC_OUTPUT_PREOFF_ME(pipe), preoff[1]);
+	I915_WRITE(PIPE_CSC_OUTPUT_PREOFF_LO(pipe), preoff[2]);
+
+	I915_WRITE(PIPE_CSC_OUTPUT_COEFF_RY_GY(pipe), coeff[0] << 16 | coeff[1]);
+	I915_WRITE(PIPE_CSC_OUTPUT_COEFF_BY(pipe), coeff[2] << 16);
+
+	I915_WRITE(PIPE_CSC_OUTPUT_COEFF_RU_GU(pipe), coeff[3] << 16 | coeff[4]);
+	I915_WRITE(PIPE_CSC_OUTPUT_COEFF_BU(pipe), coeff[5] << 16);
+
+	I915_WRITE(PIPE_CSC_OUTPUT_COEFF_RV_GV(pipe), coeff[6] << 16 | coeff[7]);
+	I915_WRITE(PIPE_CSC_OUTPUT_COEFF_BV(pipe), coeff[8] << 16);
+
+	I915_WRITE(PIPE_CSC_OUTPUT_POSTOFF_HI(pipe), postoff[0]);
+	I915_WRITE(PIPE_CSC_OUTPUT_POSTOFF_ME(pipe), postoff[1]);
+	I915_WRITE(PIPE_CSC_OUTPUT_POSTOFF_LO(pipe), postoff[2]);
+}
+
+static bool ilk_csc_limited_range(const struct intel_crtc_state *crtc_state)
+{
+	struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
 
 	/*
 	 * FIXME if there's a gamma LUT after the CSC, we should
 	 * do the range compression using the gamma LUT instead.
 	 */
-	if (INTEL_GEN(dev_priv) >= 8 || IS_HASWELL(dev_priv))
-		limited_color_range = crtc_state->limited_color_range;
+	return crtc_state->limited_color_range &&
+		(IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv) ||
+		 IS_GEN_RANGE(dev_priv, 9, 10));
+}
 
-	if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 ||
-	    crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR444) {
-		ilk_load_ycbcr_conversion_matrix(crtc);
-		return;
-	} else if (crtc_state->base.ctm) {
-		struct drm_color_ctm *ctm = crtc_state->base.ctm->data;
-		const u64 *input;
-		u64 temp[9];
+static void ilk_csc_convert_ctm(const struct intel_crtc_state *crtc_state,
+				u16 coeffs[9])
+{
+	const struct drm_color_ctm *ctm = crtc_state->base.ctm->data;
+	const u64 *input;
+	u64 temp[9];
+	int i;
 
-		if (limited_color_range)
-			input = ctm_mult_by_limited(temp, ctm->matrix);
-		else
-			input = ctm->matrix;
+	if (ilk_csc_limited_range(crtc_state))
+		input = ctm_mult_by_limited(temp, ctm->matrix);
+	else
+		input = ctm->matrix;
+
+	/*
+	 * Convert fixed point S31.32 input to format supported by the
+	 * hardware.
+	 */
+	for (i = 0; i < 9; i++) {
+		u64 abs_coeff = ((1ULL << 63) - 1) & input[i];
 
 		/*
-		 * Convert fixed point S31.32 input to format supported by the
+		 * Clamp input value to min/max supported by
 		 * hardware.
 		 */
-		for (i = 0; i < ARRAY_SIZE(coeffs); i++) {
-			u64 abs_coeff = ((1ULL << 63) - 1) & input[i];
-
-			/*
-			 * Clamp input value to min/max supported by
-			 * hardware.
-			 */
-			abs_coeff = clamp_val(abs_coeff, 0, CTM_COEFF_4_0 - 1);
-
-			/* sign bit */
-			if (CTM_COEFF_NEGATIVE(input[i]))
-				coeffs[i] |= 1 << 15;
-
-			if (abs_coeff < CTM_COEFF_0_125)
-				coeffs[i] |= (3 << 12) |
-					ILK_CSC_COEFF_FP(abs_coeff, 12);
-			else if (abs_coeff < CTM_COEFF_0_25)
-				coeffs[i] |= (2 << 12) |
-					ILK_CSC_COEFF_FP(abs_coeff, 11);
-			else if (abs_coeff < CTM_COEFF_0_5)
-				coeffs[i] |= (1 << 12) |
-					ILK_CSC_COEFF_FP(abs_coeff, 10);
-			else if (abs_coeff < CTM_COEFF_1_0)
-				coeffs[i] |= ILK_CSC_COEFF_FP(abs_coeff, 9);
-			else if (abs_coeff < CTM_COEFF_2_0)
-				coeffs[i] |= (7 << 12) |
-					ILK_CSC_COEFF_FP(abs_coeff, 8);
-			else
-				coeffs[i] |= (6 << 12) |
-					ILK_CSC_COEFF_FP(abs_coeff, 7);
-		}
-	} else {
-		/*
-		 * Load an identity matrix if no coefficients are provided.
-		 *
-		 * TODO: Check what kind of values actually come out of the
-		 * pipe with these coeff/postoff values and adjust to get the
-		 * best accuracy. Perhaps we even need to take the bpc value
-		 * into consideration.
-		 */
-		for (i = 0; i < 3; i++) {
-			if (limited_color_range)
-				coeffs[i * 3 + i] =
-					ILK_CSC_COEFF_LIMITED_RANGE;
-			else
-				coeffs[i * 3 + i] = ILK_CSC_COEFF_1_0;
-		}
+		abs_coeff = clamp_val(abs_coeff, 0, CTM_COEFF_4_0 - 1);
+
+		coeffs[i] = 0;
+
+		/* sign bit */
+		if (CTM_COEFF_NEGATIVE(input[i]))
+			coeffs[i] |= 1 << 15;
+
+		if (abs_coeff < CTM_COEFF_0_125)
+			coeffs[i] |= (3 << 12) |
+				ILK_CSC_COEFF_FP(abs_coeff, 12);
+		else if (abs_coeff < CTM_COEFF_0_25)
+			coeffs[i] |= (2 << 12) |
+				ILK_CSC_COEFF_FP(abs_coeff, 11);
+		else if (abs_coeff < CTM_COEFF_0_5)
+			coeffs[i] |= (1 << 12) |
+				ILK_CSC_COEFF_FP(abs_coeff, 10);
+		else if (abs_coeff < CTM_COEFF_1_0)
+			coeffs[i] |= ILK_CSC_COEFF_FP(abs_coeff, 9);
+		else if (abs_coeff < CTM_COEFF_2_0)
+			coeffs[i] |= (7 << 12) |
+				ILK_CSC_COEFF_FP(abs_coeff, 8);
+		else
+			coeffs[i] |= (6 << 12) |
+				ILK_CSC_COEFF_FP(abs_coeff, 7);
 	}
+}
 
-	I915_WRITE(PIPE_CSC_COEFF_RY_GY(pipe), coeffs[0] << 16 | coeffs[1]);
-	I915_WRITE(PIPE_CSC_COEFF_BY(pipe), coeffs[2] << 16);
-
-	I915_WRITE(PIPE_CSC_COEFF_RU_GU(pipe), coeffs[3] << 16 | coeffs[4]);
-	I915_WRITE(PIPE_CSC_COEFF_BU(pipe), coeffs[5] << 16);
-
-	I915_WRITE(PIPE_CSC_COEFF_RV_GV(pipe), coeffs[6] << 16 | coeffs[7]);
-	I915_WRITE(PIPE_CSC_COEFF_BV(pipe), coeffs[8] << 16);
+static void ilk_load_csc_matrix(const struct intel_crtc_state *crtc_state)
+{
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+	bool limited_color_range = ilk_csc_limited_range(crtc_state);
 
-	I915_WRITE(PIPE_CSC_PREOFF_HI(pipe), 0);
-	I915_WRITE(PIPE_CSC_PREOFF_ME(pipe), 0);
-	I915_WRITE(PIPE_CSC_PREOFF_LO(pipe), 0);
+	if (crtc_state->base.ctm) {
+		u16 coeff[9];
+
+		ilk_csc_convert_ctm(crtc_state, coeff);
+		ilk_update_pipe_csc(crtc, ilk_csc_off_zero, coeff,
+				    limited_color_range ?
+				    ilk_csc_postoff_limited_range :
+				    ilk_csc_off_zero);
+	} else if (crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB) {
+		ilk_update_pipe_csc(crtc, ilk_csc_off_zero,
+				    ilk_csc_coeff_rgb_to_ycbcr,
+				    ilk_csc_postoff_rgb_to_ycbcr);
+	} else if (limited_color_range) {
+		ilk_update_pipe_csc(crtc, ilk_csc_off_zero,
+				    ilk_csc_coeff_limited_range,
+				    ilk_csc_postoff_limited_range);
+	} else if (crtc_state->csc_enable) {
+		/*
+		 * On GLK+ both pipe CSC and degamma LUT are controlled
+		 * by csc_enable. Hence for the cases where the degama
+		 * LUT is needed but CSC is not we need to load an
+		 * identity matrix.
+		 */
+		WARN_ON(!IS_CANNONLAKE(dev_priv) && !IS_GEMINILAKE(dev_priv));
 
-	if (INTEL_GEN(dev_priv) > 6) {
-		u16 postoff = 0;
+		ilk_update_pipe_csc(crtc, ilk_csc_off_zero,
+				    ilk_csc_coeff_identity,
+				    ilk_csc_off_zero);
+	}
 
-		if (limited_color_range)
-			postoff = (16 * (1 << 12) / 255) & 0x1fff;
+	I915_WRITE(PIPE_CSC_MODE(crtc->pipe), crtc_state->csc_mode);
+}
 
-		I915_WRITE(PIPE_CSC_POSTOFF_HI(pipe), postoff);
-		I915_WRITE(PIPE_CSC_POSTOFF_ME(pipe), postoff);
-		I915_WRITE(PIPE_CSC_POSTOFF_LO(pipe), postoff);
+static void icl_load_csc_matrix(const struct intel_crtc_state *crtc_state)
+{
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 
-		I915_WRITE(PIPE_CSC_MODE(pipe), 0);
-	} else {
-		u32 mode = CSC_MODE_YUV_TO_RGB;
+	if (crtc_state->base.ctm) {
+		u16 coeff[9];
 
-		if (limited_color_range)
-			mode |= CSC_BLACK_SCREEN_OFFSET;
+		ilk_csc_convert_ctm(crtc_state, coeff);
+		ilk_update_pipe_csc(crtc, ilk_csc_off_zero,
+				    coeff, ilk_csc_off_zero);
+	}
 
-		I915_WRITE(PIPE_CSC_MODE(pipe), mode);
+	if (crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB) {
+		icl_update_output_csc(crtc, ilk_csc_off_zero,
+				      ilk_csc_coeff_rgb_to_ycbcr,
+				      ilk_csc_postoff_rgb_to_ycbcr);
+	} else if (crtc_state->limited_color_range) {
+		icl_update_output_csc(crtc, ilk_csc_off_zero,
+				      ilk_csc_coeff_limited_range,
+				      ilk_csc_postoff_limited_range);
 	}
+
+	I915_WRITE(PIPE_CSC_MODE(crtc->pipe), crtc_state->csc_mode);
 }
 
 /*
@@ -262,7 +324,6 @@ static void cherryview_load_csc_matrix(const struct intel_crtc_state *crtc_state
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	enum pipe pipe = crtc->pipe;
-	u32 mode;
 
 	if (crtc_state->base.ctm) {
 		const struct drm_color_ctm *ctm = crtc_state->base.ctm->data;
@@ -296,12 +357,30 @@ static void cherryview_load_csc_matrix(const struct intel_crtc_state *crtc_state
 		I915_WRITE(CGM_PIPE_CSC_COEFF8(pipe), coeffs[8]);
 	}
 
-	mode = (crtc_state->base.ctm ? CGM_PIPE_MODE_CSC : 0);
-	if (!crtc_state_is_legacy_gamma(crtc_state)) {
-		mode |= (crtc_state->base.degamma_lut ? CGM_PIPE_MODE_DEGAMMA : 0) |
-			(crtc_state->base.gamma_lut ? CGM_PIPE_MODE_GAMMA : 0);
-	}
-	I915_WRITE(CGM_PIPE_MODE(pipe), mode);
+	I915_WRITE(CGM_PIPE_MODE(pipe), crtc_state->cgm_mode);
+}
+
+/* i965+ "10.6" bit interpolated format "even DW" (low 8 bits) */
+static u32 i965_lut_10p6_ldw(const struct drm_color_lut *color)
+{
+	return (color->red & 0xff) << 16 |
+		(color->green & 0xff) << 8 |
+		(color->blue & 0xff);
+}
+
+/* i965+ "10.6" interpolated format "odd DW" (high 8 bits) */
+static u32 i965_lut_10p6_udw(const struct drm_color_lut *color)
+{
+	return (color->red >> 8) << 16 |
+		(color->green >> 8) << 8 |
+		(color->blue >> 8);
+}
+
+static u32 ilk_lut_10(const struct drm_color_lut *color)
+{
+	return drm_color_lut_extract(color->red, 10) << 20 |
+		drm_color_lut_extract(color->green, 10) << 10 |
+		drm_color_lut_extract(color->blue, 10);
 }
 
 /* Loads the legacy palette/gamma unit for the CRTC. */
@@ -334,15 +413,6 @@ static void i9xx_load_luts_internal(const struct intel_crtc_state *crtc_state,
 			else
 				I915_WRITE(LGC_PALETTE(pipe, i), word);
 		}
-	} else {
-		for (i = 0; i < 256; i++) {
-			u32 word = (i << 16) | (i << 8) | i;
-
-			if (HAS_GMCH(dev_priv))
-				I915_WRITE(PALETTE(pipe, i), word);
-			else
-				I915_WRITE(LGC_PALETTE(pipe, i), word);
-		}
 	}
 }
 
@@ -351,6 +421,34 @@ static void i9xx_load_luts(const struct intel_crtc_state *crtc_state)
 	i9xx_load_luts_internal(crtc_state, crtc_state->base.gamma_lut);
 }
 
+static void i9xx_color_commit(const struct intel_crtc_state *crtc_state)
+{
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+	enum pipe pipe = crtc->pipe;
+	u32 val;
+
+	val = I915_READ(PIPECONF(pipe));
+	val &= ~PIPECONF_GAMMA_MODE_MASK_I9XX;
+	val |= PIPECONF_GAMMA_MODE(crtc_state->gamma_mode);
+	I915_WRITE(PIPECONF(pipe), val);
+}
+
+static void ilk_color_commit(const struct intel_crtc_state *crtc_state)
+{
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+	enum pipe pipe = crtc->pipe;
+	u32 val;
+
+	val = I915_READ(PIPECONF(pipe));
+	val &= ~PIPECONF_GAMMA_MODE_MASK_ILK;
+	val |= PIPECONF_GAMMA_MODE(crtc_state->gamma_mode);
+	I915_WRITE(PIPECONF(pipe), val);
+
+	ilk_load_csc_matrix(crtc_state);
+}
+
 static void hsw_color_commit(const struct intel_crtc_state *crtc_state)
 {
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
@@ -361,106 +459,219 @@ static void hsw_color_commit(const struct intel_crtc_state *crtc_state)
 	ilk_load_csc_matrix(crtc_state);
 }
 
-static void bdw_load_degamma_lut(const struct intel_crtc_state *crtc_state)
+static void skl_color_commit(const struct intel_crtc_state *crtc_state)
 {
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	const struct drm_property_blob *degamma_lut = crtc_state->base.degamma_lut;
-	u32 i, lut_size = INTEL_INFO(dev_priv)->color.degamma_lut_size;
 	enum pipe pipe = crtc->pipe;
+	u32 val = 0;
 
-	I915_WRITE(PREC_PAL_INDEX(pipe),
-		   PAL_PREC_SPLIT_MODE | PAL_PREC_AUTO_INCREMENT);
+	/*
+	 * We don't (yet) allow userspace to control the pipe background color,
+	 * so force it to black, but apply pipe gamma and CSC appropriately
+	 * so that its handling will match how we program our planes.
+	 */
+	if (crtc_state->gamma_enable)
+		val |= SKL_BOTTOM_COLOR_GAMMA_ENABLE;
+	if (crtc_state->csc_enable)
+		val |= SKL_BOTTOM_COLOR_CSC_ENABLE;
+	I915_WRITE(SKL_BOTTOM_COLOR(pipe), val);
 
-	if (degamma_lut) {
-		const struct drm_color_lut *lut = degamma_lut->data;
+	I915_WRITE(GAMMA_MODE(crtc->pipe), crtc_state->gamma_mode);
 
-		for (i = 0; i < lut_size; i++) {
-			u32 word =
-			drm_color_lut_extract(lut[i].red, 10) << 20 |
-			drm_color_lut_extract(lut[i].green, 10) << 10 |
-			drm_color_lut_extract(lut[i].blue, 10);
+	if (INTEL_GEN(dev_priv) >= 11)
+		icl_load_csc_matrix(crtc_state);
+	else
+		ilk_load_csc_matrix(crtc_state);
+}
 
-			I915_WRITE(PREC_PAL_DATA(pipe), word);
-		}
-	} else {
-		for (i = 0; i < lut_size; i++) {
-			u32 v = (i * ((1 << 10) - 1)) / (lut_size - 1);
+static void i965_load_lut_10p6(struct intel_crtc *crtc,
+			       const struct drm_property_blob *blob)
+{
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+	const struct drm_color_lut *lut = blob->data;
+	int i, lut_size = drm_color_lut_size(blob);
+	enum pipe pipe = crtc->pipe;
 
-			I915_WRITE(PREC_PAL_DATA(pipe),
-				   (v << 20) | (v << 10) | v);
-		}
+	for (i = 0; i < lut_size - 1; i++) {
+		I915_WRITE(PALETTE(pipe, 2 * i + 0),
+			   i965_lut_10p6_ldw(&lut[i]));
+		I915_WRITE(PALETTE(pipe, 2 * i + 1),
+			   i965_lut_10p6_udw(&lut[i]));
 	}
+
+	I915_WRITE(PIPEGCMAX(pipe, 0), lut[i].red);
+	I915_WRITE(PIPEGCMAX(pipe, 1), lut[i].green);
+	I915_WRITE(PIPEGCMAX(pipe, 2), lut[i].blue);
 }
 
-static void bdw_load_gamma_lut(const struct intel_crtc_state *crtc_state, u32 offset)
+static void i965_load_luts(const struct intel_crtc_state *crtc_state)
 {
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
-	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	const struct drm_property_blob *gamma_lut = crtc_state->base.gamma_lut;
-	u32 i, lut_size = INTEL_INFO(dev_priv)->color.gamma_lut_size;
+
+	if (crtc_state->gamma_mode == GAMMA_MODE_MODE_8BIT)
+		i9xx_load_luts(crtc_state);
+	else
+		i965_load_lut_10p6(crtc, gamma_lut);
+}
+
+static void ilk_load_lut_10(struct intel_crtc *crtc,
+			    const struct drm_property_blob *blob)
+{
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+	const struct drm_color_lut *lut = blob->data;
+	int i, lut_size = drm_color_lut_size(blob);
 	enum pipe pipe = crtc->pipe;
 
-	WARN_ON(offset & ~PAL_PREC_INDEX_VALUE_MASK);
+	for (i = 0; i < lut_size; i++)
+		I915_WRITE(PREC_PALETTE(pipe, i), ilk_lut_10(&lut[i]));
+}
 
-	I915_WRITE(PREC_PAL_INDEX(pipe),
-		   (offset ? PAL_PREC_SPLIT_MODE : 0) |
-		   PAL_PREC_AUTO_INCREMENT |
-		   offset);
+static void ilk_load_luts(const struct intel_crtc_state *crtc_state)
+{
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+	const struct drm_property_blob *gamma_lut = crtc_state->base.gamma_lut;
 
-	if (gamma_lut) {
-		const struct drm_color_lut *lut = gamma_lut->data;
+	if (crtc_state->gamma_mode == GAMMA_MODE_MODE_8BIT)
+		i9xx_load_luts(crtc_state);
+	else
+		ilk_load_lut_10(crtc, gamma_lut);
+}
 
-		for (i = 0; i < lut_size; i++) {
-			u32 word =
-			(drm_color_lut_extract(lut[i].red, 10) << 20) |
-			(drm_color_lut_extract(lut[i].green, 10) << 10) |
-			drm_color_lut_extract(lut[i].blue, 10);
+static int ivb_lut_10_size(u32 prec_index)
+{
+	if (prec_index & PAL_PREC_SPLIT_MODE)
+		return 512;
+	else
+		return 1024;
+}
 
-			I915_WRITE(PREC_PAL_DATA(pipe), word);
-		}
+/*
+ * IVB/HSW Bspec / PAL_PREC_INDEX:
+ * "Restriction : Index auto increment mode is not
+ *  supported and must not be enabled."
+ */
+static void ivb_load_lut_10(struct intel_crtc *crtc,
+			    const struct drm_property_blob *blob,
+			    u32 prec_index)
+{
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+	int hw_lut_size = ivb_lut_10_size(prec_index);
+	const struct drm_color_lut *lut = blob->data;
+	int i, lut_size = drm_color_lut_size(blob);
+	enum pipe pipe = crtc->pipe;
 
-		/* Program the max register to clamp values > 1.0. */
-		i = lut_size - 1;
-		I915_WRITE(PREC_PAL_GC_MAX(pipe, 0),
-			   drm_color_lut_extract(lut[i].red, 16));
-		I915_WRITE(PREC_PAL_GC_MAX(pipe, 1),
-			   drm_color_lut_extract(lut[i].green, 16));
-		I915_WRITE(PREC_PAL_GC_MAX(pipe, 2),
-			   drm_color_lut_extract(lut[i].blue, 16));
-	} else {
-		for (i = 0; i < lut_size; i++) {
-			u32 v = (i * ((1 << 10) - 1)) / (lut_size - 1);
+	for (i = 0; i < hw_lut_size; i++) {
+		/* We discard half the user entries in split gamma mode */
+		const struct drm_color_lut *entry =
+			&lut[i * (lut_size - 1) / (hw_lut_size - 1)];
 
-			I915_WRITE(PREC_PAL_DATA(pipe),
-				   (v << 20) | (v << 10) | v);
-		}
+		I915_WRITE(PREC_PAL_INDEX(pipe), prec_index++);
+		I915_WRITE(PREC_PAL_DATA(pipe), ilk_lut_10(entry));
+	}
+
+	/*
+	 * Reset the index, otherwise it prevents the legacy palette to be
+	 * written properly.
+	 */
+	I915_WRITE(PREC_PAL_INDEX(pipe), 0);
+}
 
-		I915_WRITE(PREC_PAL_GC_MAX(pipe, 0), (1 << 16) - 1);
-		I915_WRITE(PREC_PAL_GC_MAX(pipe, 1), (1 << 16) - 1);
-		I915_WRITE(PREC_PAL_GC_MAX(pipe, 2), (1 << 16) - 1);
+/* On BDW+ the index auto increment mode actually works */
+static void bdw_load_lut_10(struct intel_crtc *crtc,
+			    const struct drm_property_blob *blob,
+			    u32 prec_index)
+{
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+	int hw_lut_size = ivb_lut_10_size(prec_index);
+	const struct drm_color_lut *lut = blob->data;
+	int i, lut_size = drm_color_lut_size(blob);
+	enum pipe pipe = crtc->pipe;
+
+	I915_WRITE(PREC_PAL_INDEX(pipe), prec_index |
+		   PAL_PREC_AUTO_INCREMENT);
+
+	for (i = 0; i < hw_lut_size; i++) {
+		/* We discard half the user entries in split gamma mode */
+		const struct drm_color_lut *entry =
+			&lut[i * (lut_size - 1) / (hw_lut_size - 1)];
+
+		I915_WRITE(PREC_PAL_DATA(pipe), ilk_lut_10(entry));
 	}
+
+	/*
+	 * Reset the index, otherwise it prevents the legacy palette to be
+	 * written properly.
+	 */
+	I915_WRITE(PREC_PAL_INDEX(pipe), 0);
 }
 
-/* Loads the palette/gamma unit for the CRTC on Broadwell+. */
-static void broadwell_load_luts(const struct intel_crtc_state *crtc_state)
+static void ivb_load_lut_10_max(struct intel_crtc *crtc)
 {
-	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	enum pipe pipe = crtc->pipe;
 
-	if (crtc_state_is_legacy_gamma(crtc_state)) {
+	/* Program the max register to clamp values > 1.0. */
+	I915_WRITE(PREC_PAL_EXT_GC_MAX(pipe, 0), 1 << 16);
+	I915_WRITE(PREC_PAL_EXT_GC_MAX(pipe, 1), 1 << 16);
+	I915_WRITE(PREC_PAL_EXT_GC_MAX(pipe, 2), 1 << 16);
+
+	/*
+	 * Program the gc max 2 register to clamp values > 1.0.
+	 * ToDo: Extend the ABI to be able to program values
+	 * from 3.0 to 7.0
+	 */
+	if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) {
+		I915_WRITE(PREC_PAL_EXT2_GC_MAX(pipe, 0), 1 << 16);
+		I915_WRITE(PREC_PAL_EXT2_GC_MAX(pipe, 1), 1 << 16);
+		I915_WRITE(PREC_PAL_EXT2_GC_MAX(pipe, 2), 1 << 16);
+	}
+}
+
+static void ivb_load_luts(const struct intel_crtc_state *crtc_state)
+{
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+	const struct drm_property_blob *gamma_lut = crtc_state->base.gamma_lut;
+	const struct drm_property_blob *degamma_lut = crtc_state->base.degamma_lut;
+
+	if (crtc_state->gamma_mode == GAMMA_MODE_MODE_8BIT) {
 		i9xx_load_luts(crtc_state);
+	} else if (crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT) {
+		ivb_load_lut_10(crtc, degamma_lut, PAL_PREC_SPLIT_MODE |
+				PAL_PREC_INDEX_VALUE(0));
+		ivb_load_lut_10_max(crtc);
+		ivb_load_lut_10(crtc, gamma_lut, PAL_PREC_SPLIT_MODE |
+				PAL_PREC_INDEX_VALUE(512));
 	} else {
-		bdw_load_degamma_lut(crtc_state);
-		bdw_load_gamma_lut(crtc_state,
-				   INTEL_INFO(dev_priv)->color.degamma_lut_size);
+		const struct drm_property_blob *blob = gamma_lut ?: degamma_lut;
 
-		/*
-		 * Reset the index, otherwise it prevents the legacy palette to be
-		 * written properly.
-		 */
-		I915_WRITE(PREC_PAL_INDEX(pipe), 0);
+		ivb_load_lut_10(crtc, blob,
+				PAL_PREC_INDEX_VALUE(0));
+		ivb_load_lut_10_max(crtc);
+	}
+}
+
+static void bdw_load_luts(const struct intel_crtc_state *crtc_state)
+{
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+	const struct drm_property_blob *gamma_lut = crtc_state->base.gamma_lut;
+	const struct drm_property_blob *degamma_lut = crtc_state->base.degamma_lut;
+
+	if (crtc_state->gamma_mode == GAMMA_MODE_MODE_8BIT) {
+		i9xx_load_luts(crtc_state);
+	} else if (crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT) {
+		bdw_load_lut_10(crtc, degamma_lut, PAL_PREC_SPLIT_MODE |
+				PAL_PREC_INDEX_VALUE(0));
+		ivb_load_lut_10_max(crtc);
+		bdw_load_lut_10(crtc, gamma_lut, PAL_PREC_SPLIT_MODE |
+				PAL_PREC_INDEX_VALUE(512));
+	} else {
+		const struct drm_property_blob *blob = gamma_lut ?: degamma_lut;
+
+		bdw_load_lut_10(crtc, blob,
+				PAL_PREC_INDEX_VALUE(0));
+		ivb_load_lut_10_max(crtc);
 	}
 }
 
@@ -469,7 +680,8 @@ static void glk_load_degamma_lut(const struct intel_crtc_state *crtc_state)
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	enum pipe pipe = crtc->pipe;
-	const u32 lut_size = 33;
+	const u32 lut_size = INTEL_INFO(dev_priv)->color.degamma_lut_size;
+	const struct drm_color_lut *lut = crtc_state->base.degamma_lut->data;
 	u32 i;
 
 	/*
@@ -480,39 +692,95 @@ static void glk_load_degamma_lut(const struct intel_crtc_state *crtc_state)
 	I915_WRITE(PRE_CSC_GAMC_INDEX(pipe), 0);
 	I915_WRITE(PRE_CSC_GAMC_INDEX(pipe), PRE_CSC_GAMC_AUTO_INCREMENT);
 
+	for (i = 0; i < lut_size; i++) {
+		/*
+		 * First 33 entries represent range from 0 to 1.0
+		 * 34th and 35th entry will represent extended range
+		 * inputs 3.0 and 7.0 respectively, currently clamped
+		 * at 1.0. Since the precision is 16bit, the user
+		 * value can be directly filled to register.
+		 * The pipe degamma table in GLK+ onwards doesn't
+		 * support different values per channel, so this just
+		 * programs green value which will be equal to Red and
+		 * Blue into the lut registers.
+		 * ToDo: Extend to max 7.0. Enable 32 bit input value
+		 * as compared to just 16 to achieve this.
+		 */
+		I915_WRITE(PRE_CSC_GAMC_DATA(pipe), lut[i].green);
+	}
+
+	/* Clamp values > 1.0. */
+	while (i++ < 35)
+		I915_WRITE(PRE_CSC_GAMC_DATA(pipe), 1 << 16);
+}
+
+static void glk_load_degamma_lut_linear(const struct intel_crtc_state *crtc_state)
+{
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+	enum pipe pipe = crtc->pipe;
+	const u32 lut_size = INTEL_INFO(dev_priv)->color.degamma_lut_size;
+	u32 i;
+
 	/*
-	 *  FIXME: The pipe degamma table in geminilake doesn't support
-	 *  different values per channel, so this just loads a linear table.
+	 * When setting the auto-increment bit, the hardware seems to
+	 * ignore the index bits, so we need to reset it to index 0
+	 * separately.
 	 */
+	I915_WRITE(PRE_CSC_GAMC_INDEX(pipe), 0);
+	I915_WRITE(PRE_CSC_GAMC_INDEX(pipe), PRE_CSC_GAMC_AUTO_INCREMENT);
+
 	for (i = 0; i < lut_size; i++) {
-		u32 v = (i * (1 << 16)) / (lut_size - 1);
+		u32 v = (i << 16) / (lut_size - 1);
 
 		I915_WRITE(PRE_CSC_GAMC_DATA(pipe), v);
 	}
 
 	/* Clamp values > 1.0. */
 	while (i++ < 35)
-		I915_WRITE(PRE_CSC_GAMC_DATA(pipe), (1 << 16));
+		I915_WRITE(PRE_CSC_GAMC_DATA(pipe), 1 << 16);
 }
 
 static void glk_load_luts(const struct intel_crtc_state *crtc_state)
 {
+	const struct drm_property_blob *gamma_lut = crtc_state->base.gamma_lut;
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
-	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	enum pipe pipe = crtc->pipe;
 
-	glk_load_degamma_lut(crtc_state);
+	/*
+	 * On GLK+ both pipe CSC and degamma LUT are controlled
+	 * by csc_enable. Hence for the cases where the CSC is
+	 * needed but degamma LUT is not we need to load a
+	 * linear degamma LUT. In fact we'll just always load
+	 * the degama LUT so that we don't have to reload
+	 * it every time the pipe CSC is being enabled.
+	 */
+	if (crtc_state->base.degamma_lut)
+		glk_load_degamma_lut(crtc_state);
+	else
+		glk_load_degamma_lut_linear(crtc_state);
 
-	if (crtc_state_is_legacy_gamma(crtc_state)) {
+	if (crtc_state->gamma_mode == GAMMA_MODE_MODE_8BIT) {
 		i9xx_load_luts(crtc_state);
 	} else {
-		bdw_load_gamma_lut(crtc_state, 0);
+		bdw_load_lut_10(crtc, gamma_lut, PAL_PREC_INDEX_VALUE(0));
+		ivb_load_lut_10_max(crtc);
+	}
+}
 
-		/*
-		 * Reset the index, otherwise it prevents the legacy palette to be
-		 * written properly.
-		 */
-		I915_WRITE(PREC_PAL_INDEX(pipe), 0);
+static void icl_load_luts(const struct intel_crtc_state *crtc_state)
+{
+	const struct drm_property_blob *gamma_lut = crtc_state->base.gamma_lut;
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+
+	if (crtc_state->base.degamma_lut)
+		glk_load_degamma_lut(crtc_state);
+
+	if ((crtc_state->gamma_mode & GAMMA_MODE_MODE_MASK) ==
+	    GAMMA_MODE_MODE_8BIT) {
+		i9xx_load_luts(crtc_state);
+	} else {
+		bdw_load_lut_10(crtc, gamma_lut, PAL_PREC_INDEX_VALUE(0));
+		ivb_load_lut_10_max(crtc);
 	}
 }
 
@@ -527,7 +795,7 @@ static void cherryview_load_luts(const struct intel_crtc_state *crtc_state)
 	cherryview_load_csc_matrix(crtc_state);
 
 	if (crtc_state_is_legacy_gamma(crtc_state)) {
-		i9xx_load_luts_internal(crtc_state, gamma_lut);
+		i9xx_load_luts(crtc_state);
 		return;
 	}
 
@@ -566,12 +834,6 @@ static void cherryview_load_luts(const struct intel_crtc_state *crtc_state)
 			I915_WRITE(CGM_PIPE_GAMMA(pipe, i, 1), word1);
 		}
 	}
-
-	/*
-	 * Also program a linear LUT in the legacy block (behind the
-	 * CGM block).
-	 */
-	i9xx_load_luts_internal(crtc_state, NULL);
 }
 
 void intel_color_load_luts(const struct intel_crtc_state *crtc_state)
@@ -585,8 +847,64 @@ void intel_color_commit(const struct intel_crtc_state *crtc_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
 
-	if (dev_priv->display.color_commit)
-		dev_priv->display.color_commit(crtc_state);
+	dev_priv->display.color_commit(crtc_state);
+}
+
+int intel_color_check(struct intel_crtc_state *crtc_state)
+{
+	struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
+
+	return dev_priv->display.color_check(crtc_state);
+}
+
+static bool need_plane_update(struct intel_plane *plane,
+			      const struct intel_crtc_state *crtc_state)
+{
+	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
+
+	/*
+	 * On pre-SKL the pipe gamma enable and pipe csc enable for
+	 * the pipe bottom color are configured via the primary plane.
+	 * We have to reconfigure that even if the plane is inactive.
+	 */
+	return crtc_state->active_planes & BIT(plane->id) ||
+		(INTEL_GEN(dev_priv) < 9 &&
+		 plane->id == PLANE_PRIMARY);
+}
+
+static int
+intel_color_add_affected_planes(struct intel_crtc_state *new_crtc_state)
+{
+	struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->base.crtc);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+	struct intel_atomic_state *state =
+		to_intel_atomic_state(new_crtc_state->base.state);
+	const struct intel_crtc_state *old_crtc_state =
+		intel_atomic_get_old_crtc_state(state, crtc);
+	struct intel_plane *plane;
+
+	if (!new_crtc_state->base.active ||
+	    drm_atomic_crtc_needs_modeset(&new_crtc_state->base))
+		return 0;
+
+	if (new_crtc_state->gamma_enable == old_crtc_state->gamma_enable &&
+	    new_crtc_state->csc_enable == old_crtc_state->csc_enable)
+		return 0;
+
+	for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
+		struct intel_plane_state *plane_state;
+
+		if (!need_plane_update(plane, new_crtc_state))
+			continue;
+
+		plane_state = intel_atomic_get_plane_state(state, plane);
+		if (IS_ERR(plane_state))
+			return PTR_ERR(plane_state);
+
+		new_crtc_state->update_planes |= BIT(plane->id);
+	}
+
+	return 0;
 }
 
 static int check_lut_size(const struct drm_property_blob *lut, int expected)
@@ -606,7 +924,7 @@ static int check_lut_size(const struct drm_property_blob *lut, int expected)
 	return 0;
 }
 
-int intel_color_check(struct intel_crtc_state *crtc_state)
+static int check_luts(const struct intel_crtc_state *crtc_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
 	const struct drm_property_blob *gamma_lut = crtc_state->base.gamma_lut;
@@ -614,17 +932,19 @@ int intel_color_check(struct intel_crtc_state *crtc_state)
 	int gamma_length, degamma_length;
 	u32 gamma_tests, degamma_tests;
 
+	/* Always allow legacy gamma LUT with no further checking. */
+	if (crtc_state_is_legacy_gamma(crtc_state))
+		return 0;
+
+	/* C8 relies on its palette being stored in the legacy LUT */
+	if (crtc_state->c8_planes)
+		return -EINVAL;
+
 	degamma_length = INTEL_INFO(dev_priv)->color.degamma_lut_size;
 	gamma_length = INTEL_INFO(dev_priv)->color.gamma_lut_size;
 	degamma_tests = INTEL_INFO(dev_priv)->color.degamma_lut_tests;
 	gamma_tests = INTEL_INFO(dev_priv)->color.gamma_lut_tests;
 
-	/* Always allow legacy gamma LUT with no further checking. */
-	if (crtc_state_is_legacy_gamma(crtc_state)) {
-		crtc_state->gamma_mode = GAMMA_MODE_MODE_8BIT;
-		return 0;
-	}
-
 	if (check_lut_size(degamma_lut, degamma_length) ||
 	    check_lut_size(gamma_lut, gamma_length))
 		return -EINVAL;
@@ -633,12 +953,270 @@ int intel_color_check(struct intel_crtc_state *crtc_state)
 	    drm_color_lut_check(gamma_lut, gamma_tests))
 		return -EINVAL;
 
-	if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
-		crtc_state->gamma_mode = GAMMA_MODE_MODE_10BIT;
-	else if (INTEL_GEN(dev_priv) >= 9 || IS_BROADWELL(dev_priv))
-		crtc_state->gamma_mode = GAMMA_MODE_MODE_SPLIT;
+	return 0;
+}
+
+static u32 i9xx_gamma_mode(struct intel_crtc_state *crtc_state)
+{
+	if (!crtc_state->gamma_enable ||
+	    crtc_state_is_legacy_gamma(crtc_state))
+		return GAMMA_MODE_MODE_8BIT;
 	else
-		crtc_state->gamma_mode = GAMMA_MODE_MODE_8BIT;
+		return GAMMA_MODE_MODE_10BIT; /* i965+ only */
+}
+
+static int i9xx_color_check(struct intel_crtc_state *crtc_state)
+{
+	int ret;
+
+	ret = check_luts(crtc_state);
+	if (ret)
+		return ret;
+
+	crtc_state->gamma_enable =
+		crtc_state->base.gamma_lut &&
+		!crtc_state->c8_planes;
+
+	crtc_state->gamma_mode = i9xx_gamma_mode(crtc_state);
+
+	ret = intel_color_add_affected_planes(crtc_state);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static u32 chv_cgm_mode(const struct intel_crtc_state *crtc_state)
+{
+	u32 cgm_mode = 0;
+
+	if (crtc_state_is_legacy_gamma(crtc_state))
+		return 0;
+
+	if (crtc_state->base.degamma_lut)
+		cgm_mode |= CGM_PIPE_MODE_DEGAMMA;
+	if (crtc_state->base.ctm)
+		cgm_mode |= CGM_PIPE_MODE_CSC;
+	if (crtc_state->base.gamma_lut)
+		cgm_mode |= CGM_PIPE_MODE_GAMMA;
+
+	return cgm_mode;
+}
+
+/*
+ * CHV color pipeline:
+ * u0.10 -> CGM degamma -> u0.14 -> CGM csc -> u0.14 -> CGM gamma ->
+ * u0.10 -> WGC csc -> u0.10 -> pipe gamma -> u0.10
+ *
+ * We always bypass the WGC csc and use the CGM csc
+ * instead since it has degamma and better precision.
+ */
+static int chv_color_check(struct intel_crtc_state *crtc_state)
+{
+	int ret;
+
+	ret = check_luts(crtc_state);
+	if (ret)
+		return ret;
+
+	/*
+	 * Pipe gamma will be used only for the legacy LUT.
+	 * Otherwise we bypass it and use the CGM gamma instead.
+	 */
+	crtc_state->gamma_enable =
+		crtc_state_is_legacy_gamma(crtc_state) &&
+		!crtc_state->c8_planes;
+
+	crtc_state->gamma_mode = GAMMA_MODE_MODE_8BIT;
+
+	crtc_state->cgm_mode = chv_cgm_mode(crtc_state);
+
+	ret = intel_color_add_affected_planes(crtc_state);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static u32 ilk_gamma_mode(const struct intel_crtc_state *crtc_state)
+{
+	if (!crtc_state->gamma_enable ||
+	    crtc_state_is_legacy_gamma(crtc_state))
+		return GAMMA_MODE_MODE_8BIT;
+	else
+		return GAMMA_MODE_MODE_10BIT;
+}
+
+static int ilk_color_check(struct intel_crtc_state *crtc_state)
+{
+	int ret;
+
+	ret = check_luts(crtc_state);
+	if (ret)
+		return ret;
+
+	crtc_state->gamma_enable =
+		crtc_state->base.gamma_lut &&
+		!crtc_state->c8_planes;
+
+	/*
+	 * We don't expose the ctm on ilk/snb currently,
+	 * nor do we enable YCbCr output. Also RGB limited
+	 * range output is handled by the hw automagically.
+	 */
+	crtc_state->csc_enable = false;
+
+	crtc_state->gamma_mode = ilk_gamma_mode(crtc_state);
+
+	crtc_state->csc_mode = 0;
+
+	ret = intel_color_add_affected_planes(crtc_state);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static u32 ivb_gamma_mode(const struct intel_crtc_state *crtc_state)
+{
+	if (!crtc_state->gamma_enable ||
+	    crtc_state_is_legacy_gamma(crtc_state))
+		return GAMMA_MODE_MODE_8BIT;
+	else if (crtc_state->base.gamma_lut &&
+		 crtc_state->base.degamma_lut)
+		return GAMMA_MODE_MODE_SPLIT;
+	else
+		return GAMMA_MODE_MODE_10BIT;
+}
+
+static u32 ivb_csc_mode(const struct intel_crtc_state *crtc_state)
+{
+	bool limited_color_range = ilk_csc_limited_range(crtc_state);
+
+	/*
+	 * CSC comes after the LUT in degamma, RGB->YCbCr,
+	 * and RGB full->limited range mode.
+	 */
+	if (crtc_state->base.degamma_lut ||
+	    crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB ||
+	    limited_color_range)
+		return 0;
+
+	return CSC_POSITION_BEFORE_GAMMA;
+}
+
+static int ivb_color_check(struct intel_crtc_state *crtc_state)
+{
+	bool limited_color_range = ilk_csc_limited_range(crtc_state);
+	int ret;
+
+	ret = check_luts(crtc_state);
+	if (ret)
+		return ret;
+
+	crtc_state->gamma_enable =
+		(crtc_state->base.gamma_lut ||
+		 crtc_state->base.degamma_lut) &&
+		!crtc_state->c8_planes;
+
+	crtc_state->csc_enable =
+		crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB ||
+		crtc_state->base.ctm || limited_color_range;
+
+	crtc_state->gamma_mode = ivb_gamma_mode(crtc_state);
+
+	crtc_state->csc_mode = ivb_csc_mode(crtc_state);
+
+	ret = intel_color_add_affected_planes(crtc_state);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static u32 glk_gamma_mode(const struct intel_crtc_state *crtc_state)
+{
+	if (!crtc_state->gamma_enable ||
+	    crtc_state_is_legacy_gamma(crtc_state))
+		return GAMMA_MODE_MODE_8BIT;
+	else
+		return GAMMA_MODE_MODE_10BIT;
+}
+
+static int glk_color_check(struct intel_crtc_state *crtc_state)
+{
+	int ret;
+
+	ret = check_luts(crtc_state);
+	if (ret)
+		return ret;
+
+	crtc_state->gamma_enable =
+		crtc_state->base.gamma_lut &&
+		!crtc_state->c8_planes;
+
+	/* On GLK+ degamma LUT is controlled by csc_enable */
+	crtc_state->csc_enable =
+		crtc_state->base.degamma_lut ||
+		crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB ||
+		crtc_state->base.ctm || crtc_state->limited_color_range;
+
+	crtc_state->gamma_mode = glk_gamma_mode(crtc_state);
+
+	crtc_state->csc_mode = 0;
+
+	ret = intel_color_add_affected_planes(crtc_state);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static u32 icl_gamma_mode(const struct intel_crtc_state *crtc_state)
+{
+	u32 gamma_mode = 0;
+
+	if (crtc_state->base.degamma_lut)
+		gamma_mode |= PRE_CSC_GAMMA_ENABLE;
+
+	if (crtc_state->base.gamma_lut &&
+	    !crtc_state->c8_planes)
+		gamma_mode |= POST_CSC_GAMMA_ENABLE;
+
+	if (!crtc_state->base.gamma_lut ||
+	    crtc_state_is_legacy_gamma(crtc_state))
+		gamma_mode |= GAMMA_MODE_MODE_8BIT;
+	else
+		gamma_mode |= GAMMA_MODE_MODE_10BIT;
+
+	return gamma_mode;
+}
+
+static u32 icl_csc_mode(const struct intel_crtc_state *crtc_state)
+{
+	u32 csc_mode = 0;
+
+	if (crtc_state->base.ctm)
+		csc_mode |= ICL_CSC_ENABLE;
+
+	if (crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB ||
+	    crtc_state->limited_color_range)
+		csc_mode |= ICL_OUTPUT_CSC_ENABLE;
+
+	return csc_mode;
+}
+
+static int icl_color_check(struct intel_crtc_state *crtc_state)
+{
+	int ret;
+
+	ret = check_luts(crtc_state);
+	if (ret)
+		return ret;
+
+	crtc_state->gamma_mode = icl_gamma_mode(crtc_state);
+
+	crtc_state->csc_mode = icl_csc_mode(crtc_state);
 
 	return 0;
 }
@@ -646,30 +1224,55 @@ int intel_color_check(struct intel_crtc_state *crtc_state)
 void intel_color_init(struct intel_crtc *crtc)
 {
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+	bool has_ctm = INTEL_INFO(dev_priv)->color.degamma_lut_size != 0;
 
 	drm_mode_crtc_set_gamma_size(&crtc->base, 256);
 
-	if (IS_CHERRYVIEW(dev_priv)) {
-		dev_priv->display.load_luts = cherryview_load_luts;
-	} else if (IS_HASWELL(dev_priv)) {
-		dev_priv->display.load_luts = i9xx_load_luts;
-		dev_priv->display.color_commit = hsw_color_commit;
-	} else if (IS_BROADWELL(dev_priv) || IS_GEN9_BC(dev_priv) ||
-		   IS_BROXTON(dev_priv)) {
-		dev_priv->display.load_luts = broadwell_load_luts;
-		dev_priv->display.color_commit = hsw_color_commit;
-	} else if (IS_GEMINILAKE(dev_priv) || IS_CANNONLAKE(dev_priv)) {
-		dev_priv->display.load_luts = glk_load_luts;
-		dev_priv->display.color_commit = hsw_color_commit;
+	if (HAS_GMCH(dev_priv)) {
+		if (IS_CHERRYVIEW(dev_priv)) {
+			dev_priv->display.color_check = chv_color_check;
+			dev_priv->display.color_commit = i9xx_color_commit;
+			dev_priv->display.load_luts = cherryview_load_luts;
+		} else if (INTEL_GEN(dev_priv) >= 4) {
+			dev_priv->display.color_check = i9xx_color_check;
+			dev_priv->display.color_commit = i9xx_color_commit;
+			dev_priv->display.load_luts = i965_load_luts;
+		} else {
+			dev_priv->display.color_check = i9xx_color_check;
+			dev_priv->display.color_commit = i9xx_color_commit;
+			dev_priv->display.load_luts = i9xx_load_luts;
+		}
 	} else {
-		dev_priv->display.load_luts = i9xx_load_luts;
+		if (INTEL_GEN(dev_priv) >= 11)
+			dev_priv->display.color_check = icl_color_check;
+		else if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
+			dev_priv->display.color_check = glk_color_check;
+		else if (INTEL_GEN(dev_priv) >= 7)
+			dev_priv->display.color_check = ivb_color_check;
+		else
+			dev_priv->display.color_check = ilk_color_check;
+
+		if (INTEL_GEN(dev_priv) >= 9)
+			dev_priv->display.color_commit = skl_color_commit;
+		else if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
+			dev_priv->display.color_commit = hsw_color_commit;
+		else
+			dev_priv->display.color_commit = ilk_color_commit;
+
+		if (INTEL_GEN(dev_priv) >= 11)
+			dev_priv->display.load_luts = icl_load_luts;
+		else if (IS_CANNONLAKE(dev_priv) || IS_GEMINILAKE(dev_priv))
+			dev_priv->display.load_luts = glk_load_luts;
+		else if (INTEL_GEN(dev_priv) >= 8)
+			dev_priv->display.load_luts = bdw_load_luts;
+		else if (INTEL_GEN(dev_priv) >= 7)
+			dev_priv->display.load_luts = ivb_load_luts;
+		else
+			dev_priv->display.load_luts = ilk_load_luts;
 	}
 
-	/* Enable color management support when we have degamma & gamma LUTs. */
-	if (INTEL_INFO(dev_priv)->color.degamma_lut_size != 0 &&
-	    INTEL_INFO(dev_priv)->color.gamma_lut_size != 0)
-		drm_crtc_enable_color_mgmt(&crtc->base,
-					   INTEL_INFO(dev_priv)->color.degamma_lut_size,
-					   true,
-					   INTEL_INFO(dev_priv)->color.gamma_lut_size);
+	drm_crtc_enable_color_mgmt(&crtc->base,
+				   INTEL_INFO(dev_priv)->color.degamma_lut_size,
+				   has_ctm,
+				   INTEL_INFO(dev_priv)->color.gamma_lut_size);
 }
diff --git a/drivers/gpu/drm/i915/intel_color.h b/drivers/gpu/drm/i915/intel_color.h
new file mode 100644
index 000000000000..b8a3ce609587
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_color.h
@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2019 Intel Corporation
+ */
+
+#ifndef __INTEL_COLOR_H__
+#define __INTEL_COLOR_H__
+
+struct intel_crtc_state;
+struct intel_crtc;
+
+void intel_color_init(struct intel_crtc *crtc);
+int intel_color_check(struct intel_crtc_state *crtc_state);
+void intel_color_commit(const struct intel_crtc_state *crtc_state);
+void intel_color_load_luts(const struct intel_crtc_state *crtc_state);
+
+#endif /* __INTEL_COLOR_H__ */
diff --git a/drivers/gpu/drm/i915/intel_combo_phy.c b/drivers/gpu/drm/i915/intel_combo_phy.c
index 3d0271cebf99..2bf4359d7e41 100644
--- a/drivers/gpu/drm/i915/intel_combo_phy.c
+++ b/drivers/gpu/drm/i915/intel_combo_phy.c
@@ -239,7 +239,8 @@ void icl_combo_phys_uninit(struct drm_i915_private *dev_priv)
 	for_each_combo_port_reverse(dev_priv, port) {
 		u32 val;
 
-		if (!icl_combo_phy_verify_state(dev_priv, port))
+		if (port == PORT_A &&
+		    !icl_combo_phy_verify_state(dev_priv, port))
 			DRM_WARN("Port %c combo PHY HW state changed unexpectedly\n",
 				 port_name(port));
 
diff --git a/drivers/gpu/drm/i915/intel_connector.c b/drivers/gpu/drm/i915/intel_connector.c
index ee16758747c5..073b6c3ab7cc 100644
--- a/drivers/gpu/drm/i915/intel_connector.c
+++ b/drivers/gpu/drm/i915/intel_connector.c
@@ -23,12 +23,17 @@
  * DEALINGS IN THE SOFTWARE.
  */
 
-#include <linux/slab.h>
 #include <linux/i2c.h>
+#include <linux/slab.h>
+
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_edid.h>
-#include "intel_drv.h"
+
 #include "i915_drv.h"
+#include "intel_connector.h"
+#include "intel_drv.h"
+#include "intel_hdcp.h"
+#include "intel_panel.h"
 
 int intel_connector_init(struct intel_connector *connector)
 {
@@ -88,6 +93,8 @@ void intel_connector_destroy(struct drm_connector *connector)
 
 	kfree(intel_connector->detect_edid);
 
+	intel_hdcp_cleanup(intel_connector);
+
 	if (!IS_ERR_OR_NULL(intel_connector->edid))
 		kfree(intel_connector->edid);
 
@@ -265,3 +272,11 @@ intel_attach_aspect_ratio_property(struct drm_connector *connector)
 			connector->dev->mode_config.aspect_ratio_property,
 			DRM_MODE_PICTURE_ASPECT_NONE);
 }
+
+void
+intel_attach_colorspace_property(struct drm_connector *connector)
+{
+	if (!drm_mode_create_colorspace_property(connector))
+		drm_object_attach_property(&connector->base,
+					   connector->colorspace_property, 0);
+}
diff --git a/drivers/gpu/drm/i915/intel_connector.h b/drivers/gpu/drm/i915/intel_connector.h
new file mode 100644
index 000000000000..93a7375c8196
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_connector.h
@@ -0,0 +1,35 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2019 Intel Corporation
+ */
+
+#ifndef __INTEL_CONNECTOR_H__
+#define __INTEL_CONNECTOR_H__
+
+#include "intel_display.h"
+
+struct drm_connector;
+struct edid;
+struct i2c_adapter;
+struct intel_connector;
+struct intel_encoder;
+
+int intel_connector_init(struct intel_connector *connector);
+struct intel_connector *intel_connector_alloc(void);
+void intel_connector_free(struct intel_connector *connector);
+void intel_connector_destroy(struct drm_connector *connector);
+int intel_connector_register(struct drm_connector *connector);
+void intel_connector_unregister(struct drm_connector *connector);
+void intel_connector_attach_encoder(struct intel_connector *connector,
+				    struct intel_encoder *encoder);
+bool intel_connector_get_hw_state(struct intel_connector *connector);
+enum pipe intel_connector_get_pipe(struct intel_connector *connector);
+int intel_connector_update_modes(struct drm_connector *connector,
+				 struct edid *edid);
+int intel_ddc_get_modes(struct drm_connector *c, struct i2c_adapter *adapter);
+void intel_attach_force_audio_property(struct drm_connector *connector);
+void intel_attach_broadcast_rgb_property(struct drm_connector *connector);
+void intel_attach_aspect_ratio_property(struct drm_connector *connector);
+void intel_attach_colorspace_property(struct drm_connector *connector);
+
+#endif /* __INTEL_CONNECTOR_H__ */
diff --git a/drivers/gpu/drm/i915/intel_context.c b/drivers/gpu/drm/i915/intel_context.c
new file mode 100644
index 000000000000..8931e0fee873
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_context.c
@@ -0,0 +1,269 @@
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2019 Intel Corporation
+ */
+
+#include "i915_drv.h"
+#include "i915_gem_context.h"
+#include "i915_globals.h"
+#include "intel_context.h"
+#include "intel_ringbuffer.h"
+
+static struct i915_global_context {
+	struct i915_global base;
+	struct kmem_cache *slab_ce;
+} global;
+
+struct intel_context *intel_context_alloc(void)
+{
+	return kmem_cache_zalloc(global.slab_ce, GFP_KERNEL);
+}
+
+void intel_context_free(struct intel_context *ce)
+{
+	kmem_cache_free(global.slab_ce, ce);
+}
+
+struct intel_context *
+intel_context_lookup(struct i915_gem_context *ctx,
+		     struct intel_engine_cs *engine)
+{
+	struct intel_context *ce = NULL;
+	struct rb_node *p;
+
+	spin_lock(&ctx->hw_contexts_lock);
+	p = ctx->hw_contexts.rb_node;
+	while (p) {
+		struct intel_context *this =
+			rb_entry(p, struct intel_context, node);
+
+		if (this->engine == engine) {
+			GEM_BUG_ON(this->gem_context != ctx);
+			ce = this;
+			break;
+		}
+
+		if (this->engine < engine)
+			p = p->rb_right;
+		else
+			p = p->rb_left;
+	}
+	spin_unlock(&ctx->hw_contexts_lock);
+
+	return ce;
+}
+
+struct intel_context *
+__intel_context_insert(struct i915_gem_context *ctx,
+		       struct intel_engine_cs *engine,
+		       struct intel_context *ce)
+{
+	struct rb_node **p, *parent;
+	int err = 0;
+
+	spin_lock(&ctx->hw_contexts_lock);
+
+	parent = NULL;
+	p = &ctx->hw_contexts.rb_node;
+	while (*p) {
+		struct intel_context *this;
+
+		parent = *p;
+		this = rb_entry(parent, struct intel_context, node);
+
+		if (this->engine == engine) {
+			err = -EEXIST;
+			ce = this;
+			break;
+		}
+
+		if (this->engine < engine)
+			p = &parent->rb_right;
+		else
+			p = &parent->rb_left;
+	}
+	if (!err) {
+		rb_link_node(&ce->node, parent, p);
+		rb_insert_color(&ce->node, &ctx->hw_contexts);
+	}
+
+	spin_unlock(&ctx->hw_contexts_lock);
+
+	return ce;
+}
+
+void __intel_context_remove(struct intel_context *ce)
+{
+	struct i915_gem_context *ctx = ce->gem_context;
+
+	spin_lock(&ctx->hw_contexts_lock);
+	rb_erase(&ce->node, &ctx->hw_contexts);
+	spin_unlock(&ctx->hw_contexts_lock);
+}
+
+static struct intel_context *
+intel_context_instance(struct i915_gem_context *ctx,
+		       struct intel_engine_cs *engine)
+{
+	struct intel_context *ce, *pos;
+
+	ce = intel_context_lookup(ctx, engine);
+	if (likely(ce))
+		return ce;
+
+	ce = intel_context_alloc();
+	if (!ce)
+		return ERR_PTR(-ENOMEM);
+
+	intel_context_init(ce, ctx, engine);
+
+	pos = __intel_context_insert(ctx, engine, ce);
+	if (unlikely(pos != ce)) /* Beaten! Use their HW context instead */
+		intel_context_free(ce);
+
+	GEM_BUG_ON(intel_context_lookup(ctx, engine) != pos);
+	return pos;
+}
+
+struct intel_context *
+intel_context_pin_lock(struct i915_gem_context *ctx,
+		       struct intel_engine_cs *engine)
+	__acquires(ce->pin_mutex)
+{
+	struct intel_context *ce;
+
+	ce = intel_context_instance(ctx, engine);
+	if (IS_ERR(ce))
+		return ce;
+
+	if (mutex_lock_interruptible(&ce->pin_mutex))
+		return ERR_PTR(-EINTR);
+
+	return ce;
+}
+
+struct intel_context *
+intel_context_pin(struct i915_gem_context *ctx,
+		  struct intel_engine_cs *engine)
+{
+	struct intel_context *ce;
+	int err;
+
+	ce = intel_context_instance(ctx, engine);
+	if (IS_ERR(ce))
+		return ce;
+
+	if (likely(atomic_inc_not_zero(&ce->pin_count)))
+		return ce;
+
+	if (mutex_lock_interruptible(&ce->pin_mutex))
+		return ERR_PTR(-EINTR);
+
+	if (likely(!atomic_read(&ce->pin_count))) {
+		err = ce->ops->pin(ce);
+		if (err)
+			goto err;
+
+		i915_gem_context_get(ctx);
+		GEM_BUG_ON(ce->gem_context != ctx);
+
+		mutex_lock(&ctx->mutex);
+		list_add(&ce->active_link, &ctx->active_engines);
+		mutex_unlock(&ctx->mutex);
+
+		intel_context_get(ce);
+		smp_mb__before_atomic(); /* flush pin before it is visible */
+	}
+
+	atomic_inc(&ce->pin_count);
+	GEM_BUG_ON(!intel_context_is_pinned(ce)); /* no overflow! */
+
+	mutex_unlock(&ce->pin_mutex);
+	return ce;
+
+err:
+	mutex_unlock(&ce->pin_mutex);
+	return ERR_PTR(err);
+}
+
+void intel_context_unpin(struct intel_context *ce)
+{
+	if (likely(atomic_add_unless(&ce->pin_count, -1, 1)))
+		return;
+
+	/* We may be called from inside intel_context_pin() to evict another */
+	intel_context_get(ce);
+	mutex_lock_nested(&ce->pin_mutex, SINGLE_DEPTH_NESTING);
+
+	if (likely(atomic_dec_and_test(&ce->pin_count))) {
+		ce->ops->unpin(ce);
+
+		mutex_lock(&ce->gem_context->mutex);
+		list_del(&ce->active_link);
+		mutex_unlock(&ce->gem_context->mutex);
+
+		i915_gem_context_put(ce->gem_context);
+		intel_context_put(ce);
+	}
+
+	mutex_unlock(&ce->pin_mutex);
+	intel_context_put(ce);
+}
+
+static void intel_context_retire(struct i915_active_request *active,
+				 struct i915_request *rq)
+{
+	struct intel_context *ce =
+		container_of(active, typeof(*ce), active_tracker);
+
+	intel_context_unpin(ce);
+}
+
+void
+intel_context_init(struct intel_context *ce,
+		   struct i915_gem_context *ctx,
+		   struct intel_engine_cs *engine)
+{
+	kref_init(&ce->ref);
+
+	ce->gem_context = ctx;
+	ce->engine = engine;
+	ce->ops = engine->cops;
+
+	INIT_LIST_HEAD(&ce->signal_link);
+	INIT_LIST_HEAD(&ce->signals);
+
+	mutex_init(&ce->pin_mutex);
+
+	/* Use the whole device by default */
+	ce->sseu = intel_device_default_sseu(ctx->i915);
+
+	i915_active_request_init(&ce->active_tracker,
+				 NULL, intel_context_retire);
+}
+
+static void i915_global_context_shrink(void)
+{
+	kmem_cache_shrink(global.slab_ce);
+}
+
+static void i915_global_context_exit(void)
+{
+	kmem_cache_destroy(global.slab_ce);
+}
+
+static struct i915_global_context global = { {
+	.shrink = i915_global_context_shrink,
+	.exit = i915_global_context_exit,
+} };
+
+int __init i915_global_context_init(void)
+{
+	global.slab_ce = KMEM_CACHE(intel_context, SLAB_HWCACHE_ALIGN);
+	if (!global.slab_ce)
+		return -ENOMEM;
+
+	i915_global_register(&global.base);
+	return 0;
+}
diff --git a/drivers/gpu/drm/i915/intel_context.h b/drivers/gpu/drm/i915/intel_context.h
new file mode 100644
index 000000000000..ebc861b1a49e
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_context.h
@@ -0,0 +1,87 @@
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2019 Intel Corporation
+ */
+
+#ifndef __INTEL_CONTEXT_H__
+#define __INTEL_CONTEXT_H__
+
+#include <linux/lockdep.h>
+
+#include "intel_context_types.h"
+#include "intel_engine_types.h"
+
+struct intel_context *intel_context_alloc(void);
+void intel_context_free(struct intel_context *ce);
+
+void intel_context_init(struct intel_context *ce,
+			struct i915_gem_context *ctx,
+			struct intel_engine_cs *engine);
+
+/**
+ * intel_context_lookup - Find the matching HW context for this (ctx, engine)
+ * @ctx - the parent GEM context
+ * @engine - the target HW engine
+ *
+ * May return NULL if the HW context hasn't been instantiated (i.e. unused).
+ */
+struct intel_context *
+intel_context_lookup(struct i915_gem_context *ctx,
+		     struct intel_engine_cs *engine);
+
+/**
+ * intel_context_pin_lock - Stablises the 'pinned' status of the HW context
+ * @ctx - the parent GEM context
+ * @engine - the target HW engine
+ *
+ * Acquire a lock on the pinned status of the HW context, such that the context
+ * can neither be bound to the GPU or unbound whilst the lock is held, i.e.
+ * intel_context_is_pinned() remains stable.
+ */
+struct intel_context *
+intel_context_pin_lock(struct i915_gem_context *ctx,
+		       struct intel_engine_cs *engine);
+
+static inline bool
+intel_context_is_pinned(struct intel_context *ce)
+{
+	return atomic_read(&ce->pin_count);
+}
+
+static inline void intel_context_pin_unlock(struct intel_context *ce)
+__releases(ce->pin_mutex)
+{
+	mutex_unlock(&ce->pin_mutex);
+}
+
+struct intel_context *
+__intel_context_insert(struct i915_gem_context *ctx,
+		       struct intel_engine_cs *engine,
+		       struct intel_context *ce);
+void
+__intel_context_remove(struct intel_context *ce);
+
+struct intel_context *
+intel_context_pin(struct i915_gem_context *ctx, struct intel_engine_cs *engine);
+
+static inline void __intel_context_pin(struct intel_context *ce)
+{
+	GEM_BUG_ON(!intel_context_is_pinned(ce));
+	atomic_inc(&ce->pin_count);
+}
+
+void intel_context_unpin(struct intel_context *ce);
+
+static inline struct intel_context *intel_context_get(struct intel_context *ce)
+{
+	kref_get(&ce->ref);
+	return ce;
+}
+
+static inline void intel_context_put(struct intel_context *ce)
+{
+	kref_put(&ce->ref, ce->ops->destroy);
+}
+
+#endif /* __INTEL_CONTEXT_H__ */
diff --git a/drivers/gpu/drm/i915/intel_context_types.h b/drivers/gpu/drm/i915/intel_context_types.h
new file mode 100644
index 000000000000..68b4ca1611e0
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_context_types.h
@@ -0,0 +1,74 @@
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2019 Intel Corporation
+ */
+
+#ifndef __INTEL_CONTEXT_TYPES__
+#define __INTEL_CONTEXT_TYPES__
+
+#include <linux/kref.h>
+#include <linux/list.h>
+#include <linux/mutex.h>
+#include <linux/rbtree.h>
+#include <linux/types.h>
+
+#include "i915_active_types.h"
+
+struct i915_gem_context;
+struct i915_vma;
+struct intel_context;
+struct intel_ring;
+
+struct intel_context_ops {
+	int (*pin)(struct intel_context *ce);
+	void (*unpin)(struct intel_context *ce);
+
+	void (*reset)(struct intel_context *ce);
+	void (*destroy)(struct kref *kref);
+};
+
+/*
+ * Powergating configuration for a particular (context,engine).
+ */
+struct intel_sseu {
+	u8 slice_mask;
+	u8 subslice_mask;
+	u8 min_eus_per_subslice;
+	u8 max_eus_per_subslice;
+};
+
+struct intel_context {
+	struct kref ref;
+
+	struct i915_gem_context *gem_context;
+	struct intel_engine_cs *engine;
+	struct intel_engine_cs *active;
+
+	struct list_head active_link;
+	struct list_head signal_link;
+	struct list_head signals;
+
+	struct i915_vma *state;
+	struct intel_ring *ring;
+
+	u32 *lrc_reg_state;
+	u64 lrc_desc;
+
+	atomic_t pin_count;
+	struct mutex pin_mutex; /* guards pinning and associated on-gpuing */
+
+	/**
+	 * active_tracker: Active tracker for the external rq activity
+	 * on this intel_context object.
+	 */
+	struct i915_active_request active_tracker;
+
+	const struct intel_context_ops *ops;
+	struct rb_node node;
+
+	/** sseu: Control eu/slice partitioning */
+	struct intel_sseu sseu;
+};
+
+#endif /* __INTEL_CONTEXT_TYPES__ */
diff --git a/drivers/gpu/drm/i915/intel_crt.c b/drivers/gpu/drm/i915/intel_crt.c
index 3716b2ee362f..b665c370111b 100644
--- a/drivers/gpu/drm/i915/intel_crt.c
+++ b/drivers/gpu/drm/i915/intel_crt.c
@@ -27,13 +27,18 @@
 #include <linux/dmi.h>
 #include <linux/i2c.h>
 #include <linux/slab.h>
+
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_edid.h>
 #include <drm/drm_probe_helper.h>
-#include "intel_drv.h"
 #include <drm/i915_drm.h>
+
 #include "i915_drv.h"
+#include "intel_connector.h"
+#include "intel_crt.h"
+#include "intel_ddi.h"
+#include "intel_drv.h"
 
 /* Here's the desired hotplug mode */
 #define ADPA_HOTPLUG_BITS (ADPA_CRT_HOTPLUG_PERIOD_128 |		\
@@ -435,7 +440,7 @@ static bool intel_ironlake_crt_detect_hotplug(struct drm_connector *connector)
 
 		I915_WRITE(crt->adpa_reg, adpa);
 
-		if (intel_wait_for_register(dev_priv,
+		if (intel_wait_for_register(&dev_priv->uncore,
 					    crt->adpa_reg,
 					    ADPA_CRT_HOTPLUG_FORCE_TRIGGER, 0,
 					    1000))
@@ -489,7 +494,7 @@ static bool valleyview_crt_detect_hotplug(struct drm_connector *connector)
 
 	I915_WRITE(crt->adpa_reg, adpa);
 
-	if (intel_wait_for_register(dev_priv,
+	if (intel_wait_for_register(&dev_priv->uncore,
 				    crt->adpa_reg,
 				    ADPA_CRT_HOTPLUG_FORCE_TRIGGER, 0,
 				    1000)) {
@@ -542,7 +547,7 @@ static bool intel_crt_detect_hotplug(struct drm_connector *connector)
 					      CRT_HOTPLUG_FORCE_DETECT,
 					      CRT_HOTPLUG_FORCE_DETECT);
 		/* wait for FORCE_DETECT to go off */
-		if (intel_wait_for_register(dev_priv, PORT_HOTPLUG_EN,
+		if (intel_wait_for_register(&dev_priv->uncore, PORT_HOTPLUG_EN,
 					    CRT_HOTPLUG_FORCE_DETECT, 0,
 					    1000))
 			DRM_DEBUG_KMS("timed out waiting for FORCE_DETECT to go off");
diff --git a/drivers/gpu/drm/i915/intel_crt.h b/drivers/gpu/drm/i915/intel_crt.h
new file mode 100644
index 000000000000..1b3fba359efc
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_crt.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2019 Intel Corporation
+ */
+
+#ifndef __INTEL_CRT_H__
+#define __INTEL_CRT_H__
+
+#include "i915_reg.h"
+
+enum pipe;
+struct drm_encoder;
+struct drm_i915_private;
+struct drm_i915_private;
+
+bool intel_crt_port_enabled(struct drm_i915_private *dev_priv,
+			    i915_reg_t adpa_reg, enum pipe *pipe);
+void intel_crt_init(struct drm_i915_private *dev_priv);
+void intel_crt_reset(struct drm_encoder *encoder);
+
+#endif /* __INTEL_CRT_H__ */
diff --git a/drivers/gpu/drm/i915/intel_csr.c b/drivers/gpu/drm/i915/intel_csr.c
index e8ac04c33e29..f43c2a2563a5 100644
--- a/drivers/gpu/drm/i915/intel_csr.c
+++ b/drivers/gpu/drm/i915/intel_csr.c
@@ -21,9 +21,12 @@
  * IN THE SOFTWARE.
  *
  */
+
 #include <linux/firmware.h>
+
 #include "i915_drv.h"
 #include "i915_reg.h"
+#include "intel_csr.h"
 
 /**
  * DOC: csr support for dmc
@@ -486,7 +489,7 @@ void intel_csr_ucode_init(struct drm_i915_private *dev_priv)
 	if (INTEL_GEN(dev_priv) >= 12) {
 		/* Allow to load fw via parameter using the last known size */
 		csr->max_fw_size = GEN12_CSR_MAX_FW_SIZE;
-	} else if (IS_ICELAKE(dev_priv)) {
+	} else if (IS_GEN(dev_priv, 11)) {
 		csr->fw_path = ICL_CSR_PATH;
 		csr->required_version = ICL_CSR_VERSION_REQUIRED;
 		csr->max_fw_size = ICL_CSR_MAX_FW_SIZE;
diff --git a/drivers/gpu/drm/i915/intel_csr.h b/drivers/gpu/drm/i915/intel_csr.h
new file mode 100644
index 000000000000..17a32c1e8a35
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_csr.h
@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2019 Intel Corporation
+ */
+
+#ifndef __INTEL_CSR_H__
+#define __INTEL_CSR_H__
+
+struct drm_i915_private;
+
+void intel_csr_ucode_init(struct drm_i915_private *i915);
+void intel_csr_load_program(struct drm_i915_private *i915);
+void intel_csr_ucode_fini(struct drm_i915_private *i915);
+void intel_csr_ucode_suspend(struct drm_i915_private *i915);
+void intel_csr_ucode_resume(struct drm_i915_private *i915);
+
+#endif /* __INTEL_CSR_H__ */
diff --git a/drivers/gpu/drm/i915/intel_ddi.c b/drivers/gpu/drm/i915/intel_ddi.c
index 98cea1f4b3bf..f181c26f62fd 100644
--- a/drivers/gpu/drm/i915/intel_ddi.c
+++ b/drivers/gpu/drm/i915/intel_ddi.c
@@ -26,9 +26,19 @@
  */
 
 #include <drm/drm_scdc_helper.h>
+
 #include "i915_drv.h"
+#include "intel_audio.h"
+#include "intel_connector.h"
+#include "intel_ddi.h"
+#include "intel_dp.h"
 #include "intel_drv.h"
 #include "intel_dsi.h"
+#include "intel_hdcp.h"
+#include "intel_hdmi.h"
+#include "intel_lspcon.h"
+#include "intel_panel.h"
+#include "intel_psr.h"
 
 struct ddi_buf_trans {
 	u32 trans1;	/* balance leg enable, de-emph level */
@@ -851,7 +861,7 @@ static int intel_ddi_hdmi_level(struct drm_i915_private *dev_priv, enum port por
 
 	level = dev_priv->vbt.ddi_port_info[port].hdmi_level_shift;
 
-	if (IS_ICELAKE(dev_priv)) {
+	if (INTEL_GEN(dev_priv) >= 11) {
 		if (intel_port_is_combophy(dev_priv, port))
 			icl_get_combo_buf_trans(dev_priv, port, INTEL_OUTPUT_HDMI,
 						0, &n_entries);
@@ -1240,24 +1250,15 @@ static int hsw_ddi_calc_wrpll_link(struct drm_i915_private *dev_priv,
 	return (refclk * n * 100) / (p * r);
 }
 
-static int skl_calc_wrpll_link(struct drm_i915_private *dev_priv,
-			       enum intel_dpll_id pll_id)
+static int skl_calc_wrpll_link(const struct intel_dpll_hw_state *pll_state)
 {
-	i915_reg_t cfgcr1_reg, cfgcr2_reg;
-	u32 cfgcr1_val, cfgcr2_val;
 	u32 p0, p1, p2, dco_freq;
 
-	cfgcr1_reg = DPLL_CFGCR1(pll_id);
-	cfgcr2_reg = DPLL_CFGCR2(pll_id);
-
-	cfgcr1_val = I915_READ(cfgcr1_reg);
-	cfgcr2_val = I915_READ(cfgcr2_reg);
-
-	p0 = cfgcr2_val & DPLL_CFGCR2_PDIV_MASK;
-	p2 = cfgcr2_val & DPLL_CFGCR2_KDIV_MASK;
+	p0 = pll_state->cfgcr2 & DPLL_CFGCR2_PDIV_MASK;
+	p2 = pll_state->cfgcr2 & DPLL_CFGCR2_KDIV_MASK;
 
-	if (cfgcr2_val &  DPLL_CFGCR2_QDIV_MODE(1))
-		p1 = (cfgcr2_val & DPLL_CFGCR2_QDIV_RATIO_MASK) >> 8;
+	if (pll_state->cfgcr2 &  DPLL_CFGCR2_QDIV_MODE(1))
+		p1 = (pll_state->cfgcr2 & DPLL_CFGCR2_QDIV_RATIO_MASK) >> 8;
 	else
 		p1 = 1;
 
@@ -1292,10 +1293,11 @@ static int skl_calc_wrpll_link(struct drm_i915_private *dev_priv,
 		break;
 	}
 
-	dco_freq = (cfgcr1_val & DPLL_CFGCR1_DCO_INTEGER_MASK) * 24 * 1000;
+	dco_freq = (pll_state->cfgcr1 & DPLL_CFGCR1_DCO_INTEGER_MASK)
+		* 24 * 1000;
 
-	dco_freq += (((cfgcr1_val & DPLL_CFGCR1_DCO_FRACTION_MASK) >> 9) * 24 *
-		1000) / 0x8000;
+	dco_freq += (((pll_state->cfgcr1 & DPLL_CFGCR1_DCO_FRACTION_MASK) >> 9)
+		     * 24 * 1000) / 0x8000;
 
 	if (WARN_ON(p0 == 0 || p1 == 0 || p2 == 0))
 		return 0;
@@ -1304,24 +1306,15 @@ static int skl_calc_wrpll_link(struct drm_i915_private *dev_priv,
 }
 
 int cnl_calc_wrpll_link(struct drm_i915_private *dev_priv,
-			enum intel_dpll_id pll_id)
+			struct intel_dpll_hw_state *pll_state)
 {
-	u32 cfgcr0, cfgcr1;
 	u32 p0, p1, p2, dco_freq, ref_clock;
 
-	if (INTEL_GEN(dev_priv) >= 11) {
-		cfgcr0 = I915_READ(ICL_DPLL_CFGCR0(pll_id));
-		cfgcr1 = I915_READ(ICL_DPLL_CFGCR1(pll_id));
-	} else {
-		cfgcr0 = I915_READ(CNL_DPLL_CFGCR0(pll_id));
-		cfgcr1 = I915_READ(CNL_DPLL_CFGCR1(pll_id));
-	}
-
-	p0 = cfgcr1 & DPLL_CFGCR1_PDIV_MASK;
-	p2 = cfgcr1 & DPLL_CFGCR1_KDIV_MASK;
+	p0 = pll_state->cfgcr1 & DPLL_CFGCR1_PDIV_MASK;
+	p2 = pll_state->cfgcr1 & DPLL_CFGCR1_KDIV_MASK;
 
-	if (cfgcr1 & DPLL_CFGCR1_QDIV_MODE(1))
-		p1 = (cfgcr1 & DPLL_CFGCR1_QDIV_RATIO_MASK) >>
+	if (pll_state->cfgcr1 & DPLL_CFGCR1_QDIV_MODE(1))
+		p1 = (pll_state->cfgcr1 & DPLL_CFGCR1_QDIV_RATIO_MASK) >>
 			DPLL_CFGCR1_QDIV_RATIO_SHIFT;
 	else
 		p1 = 1;
@@ -1349,16 +1342,17 @@ int cnl_calc_wrpll_link(struct drm_i915_private *dev_priv,
 	case DPLL_CFGCR1_KDIV_2:
 		p2 = 2;
 		break;
-	case DPLL_CFGCR1_KDIV_4:
-		p2 = 4;
+	case DPLL_CFGCR1_KDIV_3:
+		p2 = 3;
 		break;
 	}
 
 	ref_clock = cnl_hdmi_pll_ref_clock(dev_priv);
 
-	dco_freq = (cfgcr0 & DPLL_CFGCR0_DCO_INTEGER_MASK) * ref_clock;
+	dco_freq = (pll_state->cfgcr0 & DPLL_CFGCR0_DCO_INTEGER_MASK)
+		* ref_clock;
 
-	dco_freq += (((cfgcr0 & DPLL_CFGCR0_DCO_FRACTION_MASK) >>
+	dco_freq += (((pll_state->cfgcr0 & DPLL_CFGCR0_DCO_FRACTION_MASK) >>
 		      DPLL_CFGCR0_DCO_FRACTION_SHIFT) * ref_clock) / 0x8000;
 
 	if (WARN_ON(p0 == 0 || p1 == 0 || p2 == 0))
@@ -1390,25 +1384,21 @@ static int icl_calc_tbt_pll_link(struct drm_i915_private *dev_priv,
 }
 
 static int icl_calc_mg_pll_link(struct drm_i915_private *dev_priv,
-				enum port port)
+				const struct intel_dpll_hw_state *pll_state)
 {
-	enum tc_port tc_port = intel_port_to_tc(dev_priv, port);
-	u32 mg_pll_div0, mg_clktop_hsclkctl;
-	u32 m1, m2_int, m2_frac, div1, div2, refclk;
+	u32 m1, m2_int, m2_frac, div1, div2, ref_clock;
 	u64 tmp;
 
-	refclk = dev_priv->cdclk.hw.ref;
-
-	mg_pll_div0 = I915_READ(MG_PLL_DIV0(tc_port));
-	mg_clktop_hsclkctl = I915_READ(MG_CLKTOP2_HSCLKCTL(tc_port));
+	ref_clock = dev_priv->cdclk.hw.ref;
 
-	m1 = I915_READ(MG_PLL_DIV1(tc_port)) & MG_PLL_DIV1_FBPREDIV_MASK;
-	m2_int = mg_pll_div0 & MG_PLL_DIV0_FBDIV_INT_MASK;
-	m2_frac = (mg_pll_div0 & MG_PLL_DIV0_FRACNEN_H) ?
-		  (mg_pll_div0 & MG_PLL_DIV0_FBDIV_FRAC_MASK) >>
-		  MG_PLL_DIV0_FBDIV_FRAC_SHIFT : 0;
+	m1 = pll_state->mg_pll_div1 & MG_PLL_DIV1_FBPREDIV_MASK;
+	m2_int = pll_state->mg_pll_div0 & MG_PLL_DIV0_FBDIV_INT_MASK;
+	m2_frac = (pll_state->mg_pll_div0 & MG_PLL_DIV0_FRACNEN_H) ?
+		(pll_state->mg_pll_div0 & MG_PLL_DIV0_FBDIV_FRAC_MASK) >>
+		MG_PLL_DIV0_FBDIV_FRAC_SHIFT : 0;
 
-	switch (mg_clktop_hsclkctl & MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_MASK) {
+	switch (pll_state->mg_clktop2_hsclkctl &
+		MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_MASK) {
 	case MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_2:
 		div1 = 2;
 		break;
@@ -1422,12 +1412,14 @@ static int icl_calc_mg_pll_link(struct drm_i915_private *dev_priv,
 		div1 = 7;
 		break;
 	default:
-		MISSING_CASE(mg_clktop_hsclkctl);
+		MISSING_CASE(pll_state->mg_clktop2_hsclkctl);
 		return 0;
 	}
 
-	div2 = (mg_clktop_hsclkctl & MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO_MASK) >>
+	div2 = (pll_state->mg_clktop2_hsclkctl &
+		MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO_MASK) >>
 		MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO_SHIFT;
+
 	/* div2 value of 0 is same as 1 means no div */
 	if (div2 == 0)
 		div2 = 1;
@@ -1436,8 +1428,8 @@ static int icl_calc_mg_pll_link(struct drm_i915_private *dev_priv,
 	 * Adjust the original formula to delay the division by 2^22 in order to
 	 * minimize possible rounding errors.
 	 */
-	tmp = (u64)m1 * m2_int * refclk +
-	      (((u64)m1 * m2_frac * refclk) >> 22);
+	tmp = (u64)m1 * m2_int * ref_clock +
+	      (((u64)m1 * m2_frac * ref_clock) >> 22);
 	tmp = div_u64(tmp, 5 * div1 * div2);
 
 	return tmp;
@@ -1471,25 +1463,24 @@ static void icl_ddi_clock_get(struct intel_encoder *encoder,
 			      struct intel_crtc_state *pipe_config)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct intel_dpll_hw_state *pll_state = &pipe_config->dpll_hw_state;
 	enum port port = encoder->port;
-	int link_clock = 0;
-	u32 pll_id;
+	int link_clock;
 
-	pll_id = intel_get_shared_dpll_id(dev_priv, pipe_config->shared_dpll);
 	if (intel_port_is_combophy(dev_priv, port)) {
-		if (intel_crtc_has_type(pipe_config, INTEL_OUTPUT_HDMI))
-			link_clock = cnl_calc_wrpll_link(dev_priv, pll_id);
-		else
-			link_clock = icl_calc_dp_combo_pll_link(dev_priv,
-								pll_id);
+		link_clock = cnl_calc_wrpll_link(dev_priv, pll_state);
 	} else {
+		enum intel_dpll_id pll_id = intel_get_shared_dpll_id(dev_priv,
+						pipe_config->shared_dpll);
+
 		if (pll_id == DPLL_ID_ICL_TBTPLL)
 			link_clock = icl_calc_tbt_pll_link(dev_priv, port);
 		else
-			link_clock = icl_calc_mg_pll_link(dev_priv, port);
+			link_clock = icl_calc_mg_pll_link(dev_priv, pll_state);
 	}
 
 	pipe_config->port_clock = link_clock;
+
 	ddi_dotclock_get(pipe_config);
 }
 
@@ -1497,18 +1488,13 @@ static void cnl_ddi_clock_get(struct intel_encoder *encoder,
 			      struct intel_crtc_state *pipe_config)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	int link_clock = 0;
-	u32 cfgcr0;
-	enum intel_dpll_id pll_id;
-
-	pll_id = intel_get_shared_dpll_id(dev_priv, pipe_config->shared_dpll);
+	struct intel_dpll_hw_state *pll_state = &pipe_config->dpll_hw_state;
+	int link_clock;
 
-	cfgcr0 = I915_READ(CNL_DPLL_CFGCR0(pll_id));
-
-	if (cfgcr0 & DPLL_CFGCR0_HDMI_MODE) {
-		link_clock = cnl_calc_wrpll_link(dev_priv, pll_id);
+	if (pll_state->cfgcr0 & DPLL_CFGCR0_HDMI_MODE) {
+		link_clock = cnl_calc_wrpll_link(dev_priv, pll_state);
 	} else {
-		link_clock = cfgcr0 & DPLL_CFGCR0_LINK_RATE_MASK;
+		link_clock = pll_state->cfgcr0 & DPLL_CFGCR0_LINK_RATE_MASK;
 
 		switch (link_clock) {
 		case DPLL_CFGCR0_LINK_RATE_810:
@@ -1548,22 +1534,20 @@ static void cnl_ddi_clock_get(struct intel_encoder *encoder,
 }
 
 static void skl_ddi_clock_get(struct intel_encoder *encoder,
-				struct intel_crtc_state *pipe_config)
+			      struct intel_crtc_state *pipe_config)
 {
-	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	int link_clock = 0;
-	u32 dpll_ctl1;
-	enum intel_dpll_id pll_id;
-
-	pll_id = intel_get_shared_dpll_id(dev_priv, pipe_config->shared_dpll);
+	struct intel_dpll_hw_state *pll_state = &pipe_config->dpll_hw_state;
+	int link_clock;
 
-	dpll_ctl1 = I915_READ(DPLL_CTRL1);
-
-	if (dpll_ctl1 & DPLL_CTRL1_HDMI_MODE(pll_id)) {
-		link_clock = skl_calc_wrpll_link(dev_priv, pll_id);
+	/*
+	 * ctrl1 register is already shifted for each pll, just use 0 to get
+	 * the internal shift for each field
+	 */
+	if (pll_state->ctrl1 & DPLL_CTRL1_HDMI_MODE(0)) {
+		link_clock = skl_calc_wrpll_link(pll_state);
 	} else {
-		link_clock = dpll_ctl1 & DPLL_CTRL1_LINK_RATE_MASK(pll_id);
-		link_clock >>= DPLL_CTRL1_LINK_RATE_SHIFT(pll_id);
+		link_clock = pll_state->ctrl1 & DPLL_CTRL1_LINK_RATE_MASK(0);
+		link_clock >>= DPLL_CTRL1_LINK_RATE_SHIFT(0);
 
 		switch (link_clock) {
 		case DPLL_CTRL1_LINK_RATE_810:
@@ -1643,24 +1627,17 @@ static void hsw_ddi_clock_get(struct intel_encoder *encoder,
 	ddi_dotclock_get(pipe_config);
 }
 
-static int bxt_calc_pll_link(struct intel_crtc_state *crtc_state)
+static int bxt_calc_pll_link(const struct intel_dpll_hw_state *pll_state)
 {
-	struct intel_dpll_hw_state *state;
 	struct dpll clock;
 
-	/* For DDI ports we always use a shared PLL. */
-	if (WARN_ON(!crtc_state->shared_dpll))
-		return 0;
-
-	state = &crtc_state->dpll_hw_state;
-
 	clock.m1 = 2;
-	clock.m2 = (state->pll0 & PORT_PLL_M2_MASK) << 22;
-	if (state->pll3 & PORT_PLL_M2_FRAC_ENABLE)
-		clock.m2 |= state->pll2 & PORT_PLL_M2_FRAC_MASK;
-	clock.n = (state->pll1 & PORT_PLL_N_MASK) >> PORT_PLL_N_SHIFT;
-	clock.p1 = (state->ebb0 & PORT_PLL_P1_MASK) >> PORT_PLL_P1_SHIFT;
-	clock.p2 = (state->ebb0 & PORT_PLL_P2_MASK) >> PORT_PLL_P2_SHIFT;
+	clock.m2 = (pll_state->pll0 & PORT_PLL_M2_MASK) << 22;
+	if (pll_state->pll3 & PORT_PLL_M2_FRAC_ENABLE)
+		clock.m2 |= pll_state->pll2 & PORT_PLL_M2_FRAC_MASK;
+	clock.n = (pll_state->pll1 & PORT_PLL_N_MASK) >> PORT_PLL_N_SHIFT;
+	clock.p1 = (pll_state->ebb0 & PORT_PLL_P1_MASK) >> PORT_PLL_P1_SHIFT;
+	clock.p2 = (pll_state->ebb0 & PORT_PLL_P2_MASK) >> PORT_PLL_P2_SHIFT;
 
 	return chv_calc_dpll_params(100000, &clock);
 }
@@ -1668,7 +1645,8 @@ static int bxt_calc_pll_link(struct intel_crtc_state *crtc_state)
 static void bxt_ddi_clock_get(struct intel_encoder *encoder,
 			      struct intel_crtc_state *pipe_config)
 {
-	pipe_config->port_clock = bxt_calc_pll_link(pipe_config);
+	pipe_config->port_clock =
+		bxt_calc_pll_link(&pipe_config->dpll_hw_state);
 
 	ddi_dotclock_get(pipe_config);
 }
@@ -1678,7 +1656,7 @@ static void intel_ddi_clock_get(struct intel_encoder *encoder,
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
-	if (IS_ICELAKE(dev_priv))
+	if (INTEL_GEN(dev_priv) >= 11)
 		icl_ddi_clock_get(encoder, pipe_config);
 	else if (IS_CANNONLAKE(dev_priv))
 		cnl_ddi_clock_get(encoder, pipe_config);
@@ -1911,7 +1889,7 @@ bool intel_ddi_connector_get_hw_state(struct intel_connector *intel_connector)
 		goto out;
 	}
 
-	if (port == PORT_A)
+	if (HAS_TRANSCODER_EDP(dev_priv) && port == PORT_A)
 		cpu_transcoder = TRANSCODER_EDP;
 	else
 		cpu_transcoder = (enum transcoder) pipe;
@@ -1973,7 +1951,7 @@ static void intel_ddi_get_encoder_pipes(struct intel_encoder *encoder,
 	if (!(tmp & DDI_BUF_CTL_ENABLE))
 		goto out;
 
-	if (port == PORT_A) {
+	if (HAS_TRANSCODER_EDP(dev_priv) && port == PORT_A) {
 		tmp = I915_READ(TRANS_DDI_FUNC_CTL(TRANSCODER_EDP));
 
 		switch (tmp & TRANS_DDI_EDP_INPUT_MASK) {
@@ -2224,7 +2202,7 @@ u8 intel_ddi_dp_voltage_max(struct intel_encoder *encoder)
 	enum port port = encoder->port;
 	int n_entries;
 
-	if (IS_ICELAKE(dev_priv)) {
+	if (INTEL_GEN(dev_priv) >= 11) {
 		if (intel_port_is_combophy(dev_priv, port))
 			icl_get_combo_buf_trans(dev_priv, port, encoder->type,
 						intel_dp->link_rate, &n_entries);
@@ -2316,13 +2294,13 @@ static void cnl_ddi_vswing_program(struct intel_encoder *encoder,
 	/* Program PORT_TX_DW4 */
 	/* We cannot write to GRP. It would overrite individual loadgen */
 	for (ln = 0; ln < 4; ln++) {
-		val = I915_READ(CNL_PORT_TX_DW4_LN(port, ln));
+		val = I915_READ(CNL_PORT_TX_DW4_LN(ln, port));
 		val &= ~(POST_CURSOR_1_MASK | POST_CURSOR_2_MASK |
 			 CURSOR_COEFF_MASK);
 		val |= POST_CURSOR_1(ddi_translations[level].dw4_post_cursor_1);
 		val |= POST_CURSOR_2(ddi_translations[level].dw4_post_cursor_2);
 		val |= CURSOR_COEFF(ddi_translations[level].dw4_cursor_coeff);
-		I915_WRITE(CNL_PORT_TX_DW4_LN(port, ln), val);
+		I915_WRITE(CNL_PORT_TX_DW4_LN(ln, port), val);
 	}
 
 	/* Program PORT_TX_DW5 */
@@ -2378,14 +2356,14 @@ static void cnl_ddi_vswing_sequence(struct intel_encoder *encoder,
 	 * > 6 GHz (LN0=0, LN1=0, LN2=0, LN3=0)
 	 */
 	for (ln = 0; ln <= 3; ln++) {
-		val = I915_READ(CNL_PORT_TX_DW4_LN(port, ln));
+		val = I915_READ(CNL_PORT_TX_DW4_LN(ln, port));
 		val &= ~LOADGEN_SELECT;
 
 		if ((rate <= 600000 && width == 4 && ln >= 1)  ||
 		    (rate <= 600000 && width < 4 && (ln == 1 || ln == 2))) {
 			val |= LOADGEN_SELECT;
 		}
-		I915_WRITE(CNL_PORT_TX_DW4_LN(port, ln), val);
+		I915_WRITE(CNL_PORT_TX_DW4_LN(ln, port), val);
 	}
 
 	/* 3. Set PORT_CL_DW5 SUS Clock Config to 11b */
@@ -2447,13 +2425,13 @@ static void icl_ddi_combo_vswing_program(struct drm_i915_private *dev_priv,
 	/* Program PORT_TX_DW4 */
 	/* We cannot write to GRP. It would overwrite individual loadgen. */
 	for (ln = 0; ln <= 3; ln++) {
-		val = I915_READ(ICL_PORT_TX_DW4_LN(port, ln));
+		val = I915_READ(ICL_PORT_TX_DW4_LN(ln, port));
 		val &= ~(POST_CURSOR_1_MASK | POST_CURSOR_2_MASK |
 			 CURSOR_COEFF_MASK);
 		val |= POST_CURSOR_1(ddi_translations[level].dw4_post_cursor_1);
 		val |= POST_CURSOR_2(ddi_translations[level].dw4_post_cursor_2);
 		val |= CURSOR_COEFF(ddi_translations[level].dw4_cursor_coeff);
-		I915_WRITE(ICL_PORT_TX_DW4_LN(port, ln), val);
+		I915_WRITE(ICL_PORT_TX_DW4_LN(ln, port), val);
 	}
 
 	/* Program PORT_TX_DW7 */
@@ -2504,14 +2482,14 @@ static void icl_combo_phy_ddi_vswing_sequence(struct intel_encoder *encoder,
 	 * > 6 GHz (LN0=0, LN1=0, LN2=0, LN3=0)
 	 */
 	for (ln = 0; ln <= 3; ln++) {
-		val = I915_READ(ICL_PORT_TX_DW4_LN(port, ln));
+		val = I915_READ(ICL_PORT_TX_DW4_LN(ln, port));
 		val &= ~LOADGEN_SELECT;
 
 		if ((rate <= 600000 && width == 4 && ln >= 1) ||
 		    (rate <= 600000 && width < 4 && (ln == 1 || ln == 2))) {
 			val |= LOADGEN_SELECT;
 		}
-		I915_WRITE(ICL_PORT_TX_DW4_LN(port, ln), val);
+		I915_WRITE(ICL_PORT_TX_DW4_LN(ln, port), val);
 	}
 
 	/* 3. Set PORT_CL_DW5 SUS Clock Config to 11b */
@@ -2554,33 +2532,33 @@ static void icl_mg_phy_ddi_vswing_sequence(struct intel_encoder *encoder,
 
 	/* Set MG_TX_LINK_PARAMS cri_use_fs32 to 0. */
 	for (ln = 0; ln < 2; ln++) {
-		val = I915_READ(MG_TX1_LINK_PARAMS(port, ln));
+		val = I915_READ(MG_TX1_LINK_PARAMS(ln, port));
 		val &= ~CRI_USE_FS32;
-		I915_WRITE(MG_TX1_LINK_PARAMS(port, ln), val);
+		I915_WRITE(MG_TX1_LINK_PARAMS(ln, port), val);
 
-		val = I915_READ(MG_TX2_LINK_PARAMS(port, ln));
+		val = I915_READ(MG_TX2_LINK_PARAMS(ln, port));
 		val &= ~CRI_USE_FS32;
-		I915_WRITE(MG_TX2_LINK_PARAMS(port, ln), val);
+		I915_WRITE(MG_TX2_LINK_PARAMS(ln, port), val);
 	}
 
 	/* Program MG_TX_SWINGCTRL with values from vswing table */
 	for (ln = 0; ln < 2; ln++) {
-		val = I915_READ(MG_TX1_SWINGCTRL(port, ln));
+		val = I915_READ(MG_TX1_SWINGCTRL(ln, port));
 		val &= ~CRI_TXDEEMPH_OVERRIDE_17_12_MASK;
 		val |= CRI_TXDEEMPH_OVERRIDE_17_12(
 			ddi_translations[level].cri_txdeemph_override_17_12);
-		I915_WRITE(MG_TX1_SWINGCTRL(port, ln), val);
+		I915_WRITE(MG_TX1_SWINGCTRL(ln, port), val);
 
-		val = I915_READ(MG_TX2_SWINGCTRL(port, ln));
+		val = I915_READ(MG_TX2_SWINGCTRL(ln, port));
 		val &= ~CRI_TXDEEMPH_OVERRIDE_17_12_MASK;
 		val |= CRI_TXDEEMPH_OVERRIDE_17_12(
 			ddi_translations[level].cri_txdeemph_override_17_12);
-		I915_WRITE(MG_TX2_SWINGCTRL(port, ln), val);
+		I915_WRITE(MG_TX2_SWINGCTRL(ln, port), val);
 	}
 
 	/* Program MG_TX_DRVCTRL with values from vswing table */
 	for (ln = 0; ln < 2; ln++) {
-		val = I915_READ(MG_TX1_DRVCTRL(port, ln));
+		val = I915_READ(MG_TX1_DRVCTRL(ln, port));
 		val &= ~(CRI_TXDEEMPH_OVERRIDE_11_6_MASK |
 			 CRI_TXDEEMPH_OVERRIDE_5_0_MASK);
 		val |= CRI_TXDEEMPH_OVERRIDE_5_0(
@@ -2588,9 +2566,9 @@ static void icl_mg_phy_ddi_vswing_sequence(struct intel_encoder *encoder,
 			CRI_TXDEEMPH_OVERRIDE_11_6(
 				ddi_translations[level].cri_txdeemph_override_11_6) |
 			CRI_TXDEEMPH_OVERRIDE_EN;
-		I915_WRITE(MG_TX1_DRVCTRL(port, ln), val);
+		I915_WRITE(MG_TX1_DRVCTRL(ln, port), val);
 
-		val = I915_READ(MG_TX2_DRVCTRL(port, ln));
+		val = I915_READ(MG_TX2_DRVCTRL(ln, port));
 		val &= ~(CRI_TXDEEMPH_OVERRIDE_11_6_MASK |
 			 CRI_TXDEEMPH_OVERRIDE_5_0_MASK);
 		val |= CRI_TXDEEMPH_OVERRIDE_5_0(
@@ -2598,7 +2576,7 @@ static void icl_mg_phy_ddi_vswing_sequence(struct intel_encoder *encoder,
 			CRI_TXDEEMPH_OVERRIDE_11_6(
 				ddi_translations[level].cri_txdeemph_override_11_6) |
 			CRI_TXDEEMPH_OVERRIDE_EN;
-		I915_WRITE(MG_TX2_DRVCTRL(port, ln), val);
+		I915_WRITE(MG_TX2_DRVCTRL(ln, port), val);
 
 		/* FIXME: Program CRI_LOADGEN_SEL after the spec is updated */
 	}
@@ -2609,17 +2587,17 @@ static void icl_mg_phy_ddi_vswing_sequence(struct intel_encoder *encoder,
 	 * values from table for which TX1 and TX2 enabled.
 	 */
 	for (ln = 0; ln < 2; ln++) {
-		val = I915_READ(MG_CLKHUB(port, ln));
+		val = I915_READ(MG_CLKHUB(ln, port));
 		if (link_clock < 300000)
 			val |= CFG_LOW_RATE_LKREN_EN;
 		else
 			val &= ~CFG_LOW_RATE_LKREN_EN;
-		I915_WRITE(MG_CLKHUB(port, ln), val);
+		I915_WRITE(MG_CLKHUB(ln, port), val);
 	}
 
 	/* Program the MG_TX_DCC<LN, port being used> based on the link frequency */
 	for (ln = 0; ln < 2; ln++) {
-		val = I915_READ(MG_TX1_DCC(port, ln));
+		val = I915_READ(MG_TX1_DCC(ln, port));
 		val &= ~CFG_AMI_CK_DIV_OVERRIDE_VAL_MASK;
 		if (link_clock <= 500000) {
 			val &= ~CFG_AMI_CK_DIV_OVERRIDE_EN;
@@ -2627,9 +2605,9 @@ static void icl_mg_phy_ddi_vswing_sequence(struct intel_encoder *encoder,
 			val |= CFG_AMI_CK_DIV_OVERRIDE_EN |
 				CFG_AMI_CK_DIV_OVERRIDE_VAL(1);
 		}
-		I915_WRITE(MG_TX1_DCC(port, ln), val);
+		I915_WRITE(MG_TX1_DCC(ln, port), val);
 
-		val = I915_READ(MG_TX2_DCC(port, ln));
+		val = I915_READ(MG_TX2_DCC(ln, port));
 		val &= ~CFG_AMI_CK_DIV_OVERRIDE_VAL_MASK;
 		if (link_clock <= 500000) {
 			val &= ~CFG_AMI_CK_DIV_OVERRIDE_EN;
@@ -2637,18 +2615,18 @@ static void icl_mg_phy_ddi_vswing_sequence(struct intel_encoder *encoder,
 			val |= CFG_AMI_CK_DIV_OVERRIDE_EN |
 				CFG_AMI_CK_DIV_OVERRIDE_VAL(1);
 		}
-		I915_WRITE(MG_TX2_DCC(port, ln), val);
+		I915_WRITE(MG_TX2_DCC(ln, port), val);
 	}
 
 	/* Program MG_TX_PISO_READLOAD with values from vswing table */
 	for (ln = 0; ln < 2; ln++) {
-		val = I915_READ(MG_TX1_PISO_READLOAD(port, ln));
+		val = I915_READ(MG_TX1_PISO_READLOAD(ln, port));
 		val |= CRI_CALCINIT;
-		I915_WRITE(MG_TX1_PISO_READLOAD(port, ln), val);
+		I915_WRITE(MG_TX1_PISO_READLOAD(ln, port), val);
 
-		val = I915_READ(MG_TX2_PISO_READLOAD(port, ln));
+		val = I915_READ(MG_TX2_PISO_READLOAD(ln, port));
 		val |= CRI_CALCINIT;
-		I915_WRITE(MG_TX2_PISO_READLOAD(port, ln), val);
+		I915_WRITE(MG_TX2_PISO_READLOAD(ln, port), val);
 	}
 }
 
@@ -2697,7 +2675,7 @@ u32 bxt_signal_levels(struct intel_dp *intel_dp)
 	struct intel_encoder *encoder = &dport->base;
 	int level = intel_ddi_dp_level(intel_dp);
 
-	if (IS_ICELAKE(dev_priv))
+	if (INTEL_GEN(dev_priv) >= 11)
 		icl_ddi_vswing_sequence(encoder, intel_dp->link_rate,
 					level, encoder->type);
 	else if (IS_CANNONLAKE(dev_priv))
@@ -2866,7 +2844,7 @@ static void intel_ddi_clk_select(struct intel_encoder *encoder,
 
 	mutex_lock(&dev_priv->dpll_lock);
 
-	if (IS_ICELAKE(dev_priv)) {
+	if (INTEL_GEN(dev_priv) >= 11) {
 		if (!intel_port_is_combophy(dev_priv, port))
 			I915_WRITE(DDI_CLK_SEL(port),
 				   icl_pll_to_ddi_clk_sel(encoder, crtc_state));
@@ -2908,7 +2886,7 @@ static void intel_ddi_clk_disable(struct intel_encoder *encoder)
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	enum port port = encoder->port;
 
-	if (IS_ICELAKE(dev_priv)) {
+	if (INTEL_GEN(dev_priv) >= 11) {
 		if (!intel_port_is_combophy(dev_priv, port))
 			I915_WRITE(DDI_CLK_SEL(port), DDI_CLK_SEL_NONE);
 	} else if (IS_CANNONLAKE(dev_priv)) {
@@ -2927,21 +2905,20 @@ static void icl_enable_phy_clock_gating(struct intel_digital_port *dig_port)
 	struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
 	enum port port = dig_port->base.port;
 	enum tc_port tc_port = intel_port_to_tc(dev_priv, port);
-	i915_reg_t mg_regs[2] = { MG_DP_MODE(port, 0), MG_DP_MODE(port, 1) };
 	u32 val;
-	int i;
+	int ln;
 
 	if (tc_port == PORT_TC_NONE)
 		return;
 
-	for (i = 0; i < ARRAY_SIZE(mg_regs); i++) {
-		val = I915_READ(mg_regs[i]);
+	for (ln = 0; ln < 2; ln++) {
+		val = I915_READ(MG_DP_MODE(ln, port));
 		val |= MG_DP_MODE_CFG_TR2PWR_GATING |
 		       MG_DP_MODE_CFG_TRPWR_GATING |
 		       MG_DP_MODE_CFG_CLNPWR_GATING |
 		       MG_DP_MODE_CFG_DIGPWR_GATING |
 		       MG_DP_MODE_CFG_GAONPWR_GATING;
-		I915_WRITE(mg_regs[i], val);
+		I915_WRITE(MG_DP_MODE(ln, port), val);
 	}
 
 	val = I915_READ(MG_MISC_SUS0(tc_port));
@@ -2960,21 +2937,20 @@ static void icl_disable_phy_clock_gating(struct intel_digital_port *dig_port)
 	struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
 	enum port port = dig_port->base.port;
 	enum tc_port tc_port = intel_port_to_tc(dev_priv, port);
-	i915_reg_t mg_regs[2] = { MG_DP_MODE(port, 0), MG_DP_MODE(port, 1) };
 	u32 val;
-	int i;
+	int ln;
 
 	if (tc_port == PORT_TC_NONE)
 		return;
 
-	for (i = 0; i < ARRAY_SIZE(mg_regs); i++) {
-		val = I915_READ(mg_regs[i]);
+	for (ln = 0; ln < 2; ln++) {
+		val = I915_READ(MG_DP_MODE(ln, port));
 		val &= ~(MG_DP_MODE_CFG_TR2PWR_GATING |
 			 MG_DP_MODE_CFG_TRPWR_GATING |
 			 MG_DP_MODE_CFG_CLNPWR_GATING |
 			 MG_DP_MODE_CFG_DIGPWR_GATING |
 			 MG_DP_MODE_CFG_GAONPWR_GATING);
-		I915_WRITE(mg_regs[i], val);
+		I915_WRITE(MG_DP_MODE(ln, port), val);
 	}
 
 	val = I915_READ(MG_MISC_SUS0(tc_port));
@@ -2998,8 +2974,8 @@ static void icl_program_mg_dp_mode(struct intel_digital_port *intel_dig_port)
 	if (tc_port == PORT_TC_NONE || intel_dig_port->tc_type == TC_PORT_TBT)
 		return;
 
-	ln0 = I915_READ(MG_DP_MODE(port, 0));
-	ln1 = I915_READ(MG_DP_MODE(port, 1));
+	ln0 = I915_READ(MG_DP_MODE(0, port));
+	ln1 = I915_READ(MG_DP_MODE(1, port));
 
 	switch (intel_dig_port->tc_type) {
 	case TC_PORT_TYPEC:
@@ -3049,8 +3025,8 @@ static void icl_program_mg_dp_mode(struct intel_digital_port *intel_dig_port)
 		return;
 	}
 
-	I915_WRITE(MG_DP_MODE(port, 0), ln0);
-	I915_WRITE(MG_DP_MODE(port, 1), ln1);
+	I915_WRITE(MG_DP_MODE(0, port), ln0);
+	I915_WRITE(MG_DP_MODE(1, port), ln1);
 }
 
 static void intel_dp_sink_set_fec_ready(struct intel_dp *intel_dp,
@@ -3077,7 +3053,7 @@ static void intel_ddi_enable_fec(struct intel_encoder *encoder,
 	val |= DP_TP_CTL_FEC_ENABLE;
 	I915_WRITE(DP_TP_CTL(port), val);
 
-	if (intel_wait_for_register(dev_priv, DP_TP_STATUS(port),
+	if (intel_wait_for_register(&dev_priv->uncore, DP_TP_STATUS(port),
 				    DP_TP_STATUS_FEC_ENABLE_LIVE,
 				    DP_TP_STATUS_FEC_ENABLE_LIVE,
 				    1))
@@ -3125,7 +3101,7 @@ static void intel_ddi_pre_enable_dp(struct intel_encoder *encoder,
 	icl_program_mg_dp_mode(dig_port);
 	icl_disable_phy_clock_gating(dig_port);
 
-	if (IS_ICELAKE(dev_priv))
+	if (INTEL_GEN(dev_priv) >= 11)
 		icl_ddi_vswing_sequence(encoder, crtc_state->port_clock,
 					level, encoder->type);
 	else if (IS_CANNONLAKE(dev_priv))
@@ -3174,7 +3150,7 @@ static void intel_ddi_pre_enable_hdmi(struct intel_encoder *encoder,
 	icl_program_mg_dp_mode(dig_port);
 	icl_disable_phy_clock_gating(dig_port);
 
-	if (IS_ICELAKE(dev_priv))
+	if (INTEL_GEN(dev_priv) >= 11)
 		icl_ddi_vswing_sequence(encoder, crtc_state->port_clock,
 					level, INTEL_OUTPUT_HDMI);
 	else if (IS_CANNONLAKE(dev_priv))
@@ -3555,7 +3531,9 @@ static void intel_ddi_update_pipe_dp(struct intel_encoder *encoder,
 {
 	struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
 
-	intel_psr_enable(intel_dp, crtc_state);
+	intel_ddi_set_pipe_settings(crtc_state);
+
+	intel_psr_update(intel_dp, crtc_state);
 	intel_edp_drrs_enable(intel_dp, crtc_state);
 
 	intel_panel_update_backlight(encoder, crtc_state, conn_state);
@@ -3710,7 +3688,7 @@ static bool intel_ddi_is_audio_enabled(struct drm_i915_private *dev_priv,
 void intel_ddi_compute_min_voltage_level(struct drm_i915_private *dev_priv,
 					 struct intel_crtc_state *crtc_state)
 {
-	if (IS_ICELAKE(dev_priv) && crtc_state->port_clock > 594000)
+	if (INTEL_GEN(dev_priv) >= 11 && crtc_state->port_clock > 594000)
 		crtc_state->min_voltage_level = 1;
 	else if (IS_CANNONLAKE(dev_priv) && crtc_state->port_clock > 594000)
 		crtc_state->min_voltage_level = 2;
@@ -3763,7 +3741,10 @@ void intel_ddi_get_config(struct intel_encoder *encoder,
 		pipe_config->has_hdmi_sink = true;
 		intel_dig_port = enc_to_dig_port(&encoder->base);
 
-		if (intel_dig_port->infoframe_enabled(encoder, pipe_config))
+		pipe_config->infoframes.enable |=
+			intel_hdmi_infoframes_enabled(encoder, pipe_config);
+
+		if (pipe_config->infoframes.enable)
 			pipe_config->has_infoframe = true;
 
 		if (temp & TRANS_DDI_HDMI_SCRAMBLING)
@@ -3827,6 +3808,18 @@ void intel_ddi_get_config(struct intel_encoder *encoder,
 			bxt_ddi_phy_get_lane_lat_optim_mask(encoder);
 
 	intel_ddi_compute_min_voltage_level(dev_priv, pipe_config);
+
+	intel_hdmi_read_gcp_infoframe(encoder, pipe_config);
+
+	intel_read_infoframe(encoder, pipe_config,
+			     HDMI_INFOFRAME_TYPE_AVI,
+			     &pipe_config->infoframes.avi);
+	intel_read_infoframe(encoder, pipe_config,
+			     HDMI_INFOFRAME_TYPE_SPD,
+			     &pipe_config->infoframes.spd);
+	intel_read_infoframe(encoder, pipe_config,
+			     HDMI_INFOFRAME_TYPE_VENDOR,
+			     &pipe_config->infoframes.hdmi);
 }
 
 static enum intel_output_type
@@ -3855,7 +3848,7 @@ static int intel_ddi_compute_config(struct intel_encoder *encoder,
 	enum port port = encoder->port;
 	int ret;
 
-	if (port == PORT_A)
+	if (HAS_TRANSCODER_EDP(dev_priv) && port == PORT_A)
 		pipe_config->cpu_transcoder = TRANSCODER_EDP;
 
 	if (intel_crtc_has_type(pipe_config, INTEL_OUTPUT_HDMI))
@@ -3959,15 +3952,7 @@ static int modeset_pipe(struct drm_crtc *crtc,
 		goto out;
 	}
 
-	crtc_state->mode_changed = true;
-
-	ret = drm_atomic_add_affected_connectors(state, crtc);
-	if (ret)
-		goto out;
-
-	ret = drm_atomic_add_affected_planes(state, crtc);
-	if (ret)
-		goto out;
+	crtc_state->connectors_changed = true;
 
 	ret = drm_atomic_commit(state);
 out:
diff --git a/drivers/gpu/drm/i915/intel_ddi.h b/drivers/gpu/drm/i915/intel_ddi.h
new file mode 100644
index 000000000000..9cf69175942e
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_ddi.h
@@ -0,0 +1,53 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2019 Intel Corporation
+ */
+
+#ifndef __INTEL_DDI_H__
+#define __INTEL_DDI_H__
+
+#include <drm/i915_drm.h>
+
+#include "intel_display.h"
+
+struct drm_connector_state;
+struct drm_i915_private;
+struct intel_connector;
+struct intel_crtc;
+struct intel_crtc_state;
+struct intel_dp;
+struct intel_dpll_hw_state;
+struct intel_encoder;
+
+void intel_ddi_fdi_post_disable(struct intel_encoder *intel_encoder,
+				const struct intel_crtc_state *old_crtc_state,
+				const struct drm_connector_state *old_conn_state);
+void hsw_fdi_link_train(struct intel_crtc *crtc,
+			const struct intel_crtc_state *crtc_state);
+void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port);
+bool intel_ddi_get_hw_state(struct intel_encoder *encoder, enum pipe *pipe);
+void intel_ddi_enable_transcoder_func(const struct intel_crtc_state *crtc_state);
+void intel_ddi_disable_transcoder_func(const struct intel_crtc_state *crtc_state);
+void intel_ddi_enable_pipe_clock(const struct intel_crtc_state *crtc_state);
+void intel_ddi_disable_pipe_clock(const  struct intel_crtc_state *crtc_state);
+void intel_ddi_set_pipe_settings(const struct intel_crtc_state *crtc_state);
+void intel_ddi_prepare_link_retrain(struct intel_dp *intel_dp);
+bool intel_ddi_connector_get_hw_state(struct intel_connector *intel_connector);
+void intel_ddi_get_config(struct intel_encoder *encoder,
+			  struct intel_crtc_state *pipe_config);
+void intel_ddi_set_vc_payload_alloc(const struct intel_crtc_state *crtc_state,
+				    bool state);
+void intel_ddi_compute_min_voltage_level(struct drm_i915_private *dev_priv,
+					 struct intel_crtc_state *crtc_state);
+u32 bxt_signal_levels(struct intel_dp *intel_dp);
+u32 ddi_signal_levels(struct intel_dp *intel_dp);
+u8 intel_ddi_dp_voltage_max(struct intel_encoder *encoder);
+u8 intel_ddi_dp_pre_emphasis_max(struct intel_encoder *encoder,
+				 u8 voltage_swing);
+int intel_ddi_toggle_hdcp_signalling(struct intel_encoder *intel_encoder,
+				     bool enable);
+void icl_sanitize_encoder_pll_mapping(struct intel_encoder *encoder);
+int cnl_calc_wrpll_link(struct drm_i915_private *dev_priv,
+			struct intel_dpll_hw_state *state);
+
+#endif /* __INTEL_DDI_H__ */
diff --git a/drivers/gpu/drm/i915/intel_device_info.c b/drivers/gpu/drm/i915/intel_device_info.c
index 855a5074ad77..6af480b95bc6 100644
--- a/drivers/gpu/drm/i915/intel_device_info.c
+++ b/drivers/gpu/drm/i915/intel_device_info.c
@@ -57,6 +57,7 @@ static const char * const platform_names[] = {
 	PLATFORM_NAME(COFFEELAKE),
 	PLATFORM_NAME(CANNONLAKE),
 	PLATFORM_NAME(ICELAKE),
+	PLATFORM_NAME(ELKHARTLAKE),
 };
 #undef PLATFORM_NAME
 
@@ -155,9 +156,15 @@ static void gen11_sseu_info_init(struct drm_i915_private *dev_priv)
 	u8 eu_en;
 	int s;
 
-	sseu->max_slices = 1;
-	sseu->max_subslices = 8;
-	sseu->max_eus_per_subslice = 8;
+	if (IS_ELKHARTLAKE(dev_priv)) {
+		sseu->max_slices = 1;
+		sseu->max_subslices = 4;
+		sseu->max_eus_per_subslice = 8;
+	} else {
+		sseu->max_slices = 1;
+		sseu->max_subslices = 8;
+		sseu->max_eus_per_subslice = 8;
+	}
 
 	s_en = I915_READ(GEN11_GT_SLICE_ENABLE) & GEN11_GT_S_ENA_MASK;
 	ss_en = ~I915_READ(GEN11_GT_SUBSLICE_DISABLE);
@@ -707,6 +714,99 @@ static u32 read_timestamp_frequency(struct drm_i915_private *dev_priv)
 	return 0;
 }
 
+#undef INTEL_VGA_DEVICE
+#define INTEL_VGA_DEVICE(id, info) (id)
+
+static const u16 subplatform_ult_ids[] = {
+	INTEL_HSW_ULT_GT1_IDS(0),
+	INTEL_HSW_ULT_GT2_IDS(0),
+	INTEL_HSW_ULT_GT3_IDS(0),
+	INTEL_BDW_ULT_GT1_IDS(0),
+	INTEL_BDW_ULT_GT2_IDS(0),
+	INTEL_BDW_ULT_GT3_IDS(0),
+	INTEL_BDW_ULT_RSVD_IDS(0),
+	INTEL_SKL_ULT_GT1_IDS(0),
+	INTEL_SKL_ULT_GT2_IDS(0),
+	INTEL_SKL_ULT_GT3_IDS(0),
+	INTEL_KBL_ULT_GT1_IDS(0),
+	INTEL_KBL_ULT_GT2_IDS(0),
+	INTEL_KBL_ULT_GT3_IDS(0),
+	INTEL_CFL_U_GT2_IDS(0),
+	INTEL_CFL_U_GT3_IDS(0),
+	INTEL_WHL_U_GT1_IDS(0),
+	INTEL_WHL_U_GT2_IDS(0),
+	INTEL_WHL_U_GT3_IDS(0)
+};
+
+static const u16 subplatform_ulx_ids[] = {
+	INTEL_HSW_ULX_GT1_IDS(0),
+	INTEL_HSW_ULX_GT2_IDS(0),
+	INTEL_BDW_ULX_GT1_IDS(0),
+	INTEL_BDW_ULX_GT2_IDS(0),
+	INTEL_BDW_ULX_GT3_IDS(0),
+	INTEL_BDW_ULX_RSVD_IDS(0),
+	INTEL_SKL_ULX_GT1_IDS(0),
+	INTEL_SKL_ULX_GT2_IDS(0),
+	INTEL_KBL_ULX_GT1_IDS(0),
+	INTEL_KBL_ULX_GT2_IDS(0)
+};
+
+static const u16 subplatform_aml_ids[] = {
+	INTEL_AML_KBL_GT2_IDS(0),
+	INTEL_AML_CFL_GT2_IDS(0)
+};
+
+static const u16 subplatform_portf_ids[] = {
+	INTEL_CNL_PORT_F_IDS(0),
+	INTEL_ICL_PORT_F_IDS(0)
+};
+
+static bool find_devid(u16 id, const u16 *p, unsigned int num)
+{
+	for (; num; num--, p++) {
+		if (*p == id)
+			return true;
+	}
+
+	return false;
+}
+
+void intel_device_info_subplatform_init(struct drm_i915_private *i915)
+{
+	const struct intel_device_info *info = INTEL_INFO(i915);
+	const struct intel_runtime_info *rinfo = RUNTIME_INFO(i915);
+	const unsigned int pi = __platform_mask_index(rinfo, info->platform);
+	const unsigned int pb = __platform_mask_bit(rinfo, info->platform);
+	u16 devid = INTEL_DEVID(i915);
+	u32 mask = 0;
+
+	/* Make sure IS_<platform> checks are working. */
+	RUNTIME_INFO(i915)->platform_mask[pi] = BIT(pb);
+
+	/* Find and mark subplatform bits based on the PCI device id. */
+	if (find_devid(devid, subplatform_ult_ids,
+		       ARRAY_SIZE(subplatform_ult_ids))) {
+		mask = BIT(INTEL_SUBPLATFORM_ULT);
+	} else if (find_devid(devid, subplatform_ulx_ids,
+			      ARRAY_SIZE(subplatform_ulx_ids))) {
+		mask = BIT(INTEL_SUBPLATFORM_ULX);
+		if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
+			/* ULX machines are also considered ULT. */
+			mask |= BIT(INTEL_SUBPLATFORM_ULT);
+		}
+	} else if (find_devid(devid, subplatform_aml_ids,
+			      ARRAY_SIZE(subplatform_aml_ids))) {
+		mask = BIT(INTEL_SUBPLATFORM_AML);
+	} else if (find_devid(devid, subplatform_portf_ids,
+			      ARRAY_SIZE(subplatform_portf_ids))) {
+		mask = BIT(INTEL_SUBPLATFORM_PORTF);
+	}
+
+	GEM_BUG_ON(mask & ~INTEL_SUBPLATFORM_BITS);
+
+	RUNTIME_INFO(i915)->platform_mask[pi] |= mask;
+}
+
 /**
  * intel_device_info_runtime_init - initialize runtime info
  * @dev_priv: the i915 device
@@ -738,9 +838,9 @@ void intel_device_info_runtime_init(struct drm_i915_private *dev_priv)
 		runtime->num_scalers[PIPE_C] = 1;
 	}
 
-	BUILD_BUG_ON(I915_NUM_ENGINES > BITS_PER_TYPE(intel_ring_mask_t));
+	BUILD_BUG_ON(BITS_PER_TYPE(intel_engine_mask_t) < I915_NUM_ENGINES);
 
-	if (IS_GEN(dev_priv, 11))
+	if (INTEL_GEN(dev_priv) >= 11)
 		for_each_pipe(dev_priv, pipe)
 			runtime->num_sprites[pipe] = 6;
 	else if (IS_GEN(dev_priv, 10) || IS_GEMINILAKE(dev_priv))
@@ -844,7 +944,7 @@ void intel_device_info_runtime_init(struct drm_i915_private *dev_priv)
 
 	if (IS_GEN(dev_priv, 6) && intel_vtd_active()) {
 		DRM_INFO("Disabling ppGTT for VT-d support\n");
-		info->ppgtt = INTEL_PPGTT_NONE;
+		info->ppgtt_type = INTEL_PPGTT_NONE;
 	}
 
 	/* Initialize command stream timestamp frequency */
@@ -871,23 +971,24 @@ void intel_device_info_init_mmio(struct drm_i915_private *dev_priv)
 	unsigned int logical_vdbox = 0;
 	unsigned int i;
 	u32 media_fuse;
+	u16 vdbox_mask;
+	u16 vebox_mask;
 
 	if (INTEL_GEN(dev_priv) < 11)
 		return;
 
 	media_fuse = ~I915_READ(GEN11_GT_VEBOX_VDBOX_DISABLE);
 
-	RUNTIME_INFO(dev_priv)->vdbox_enable = media_fuse & GEN11_GT_VDBOX_DISABLE_MASK;
-	RUNTIME_INFO(dev_priv)->vebox_enable = (media_fuse & GEN11_GT_VEBOX_DISABLE_MASK) >>
-		GEN11_GT_VEBOX_DISABLE_SHIFT;
+	vdbox_mask = media_fuse & GEN11_GT_VDBOX_DISABLE_MASK;
+	vebox_mask = (media_fuse & GEN11_GT_VEBOX_DISABLE_MASK) >>
+		      GEN11_GT_VEBOX_DISABLE_SHIFT;
 
-	DRM_DEBUG_DRIVER("vdbox enable: %04x\n", RUNTIME_INFO(dev_priv)->vdbox_enable);
 	for (i = 0; i < I915_MAX_VCS; i++) {
 		if (!HAS_ENGINE(dev_priv, _VCS(i)))
 			continue;
 
-		if (!(BIT(i) & RUNTIME_INFO(dev_priv)->vdbox_enable)) {
-			info->ring_mask &= ~ENGINE_MASK(_VCS(i));
+		if (!(BIT(i) & vdbox_mask)) {
+			info->engine_mask &= ~BIT(_VCS(i));
 			DRM_DEBUG_DRIVER("vcs%u fused off\n", i);
 			continue;
 		}
@@ -899,15 +1000,20 @@ void intel_device_info_init_mmio(struct drm_i915_private *dev_priv)
 		if (logical_vdbox++ % 2 == 0)
 			RUNTIME_INFO(dev_priv)->vdbox_sfc_access |= BIT(i);
 	}
+	DRM_DEBUG_DRIVER("vdbox enable: %04x, instances: %04lx\n",
+			 vdbox_mask, VDBOX_MASK(dev_priv));
+	GEM_BUG_ON(vdbox_mask != VDBOX_MASK(dev_priv));
 
-	DRM_DEBUG_DRIVER("vebox enable: %04x\n", RUNTIME_INFO(dev_priv)->vebox_enable);
 	for (i = 0; i < I915_MAX_VECS; i++) {
 		if (!HAS_ENGINE(dev_priv, _VECS(i)))
 			continue;
 
-		if (!(BIT(i) & RUNTIME_INFO(dev_priv)->vebox_enable)) {
-			info->ring_mask &= ~ENGINE_MASK(_VECS(i));
+		if (!(BIT(i) & vebox_mask)) {
+			info->engine_mask &= ~BIT(_VECS(i));
 			DRM_DEBUG_DRIVER("vecs%u fused off\n", i);
 		}
 	}
+	DRM_DEBUG_DRIVER("vebox enable: %04x, instances: %04lx\n",
+			 vebox_mask, VEBOX_MASK(dev_priv));
+	GEM_BUG_ON(vebox_mask != VEBOX_MASK(dev_priv));
 }
diff --git a/drivers/gpu/drm/i915/intel_device_info.h b/drivers/gpu/drm/i915/intel_device_info.h
index e8b8661df746..0e579f158016 100644
--- a/drivers/gpu/drm/i915/intel_device_info.h
+++ b/drivers/gpu/drm/i915/intel_device_info.h
@@ -27,6 +27,7 @@
 
 #include <uapi/drm/i915_drm.h>
 
+#include "intel_engine_types.h"
 #include "intel_display.h"
 
 struct drm_printer;
@@ -73,14 +74,29 @@ enum intel_platform {
 	INTEL_CANNONLAKE,
 	/* gen11 */
 	INTEL_ICELAKE,
+	INTEL_ELKHARTLAKE,
 	INTEL_MAX_PLATFORMS
 };
 
-enum intel_ppgtt {
+/*
+ * Subplatform bits share the same namespace per parent platform. In other words
+ * it is fine for the same bit to be used on multiple parent platforms.
+ */
+
+#define INTEL_SUBPLATFORM_BITS (3)
+
+/* HSW/BDW/SKL/KBL/CFL */
+#define INTEL_SUBPLATFORM_ULT	(0)
+#define INTEL_SUBPLATFORM_ULX	(1)
+#define INTEL_SUBPLATFORM_AML	(2)
+
+/* CNL/ICL */
+#define INTEL_SUBPLATFORM_PORTF	(0)
+
+enum intel_ppgtt_type {
 	INTEL_PPGTT_NONE = I915_GEM_PPGTT_NONE,
 	INTEL_PPGTT_ALIASING = I915_GEM_PPGTT_ALIASING,
 	INTEL_PPGTT_FULL = I915_GEM_PPGTT_FULL,
-	INTEL_PPGTT_FULL_4LVL,
 };
 
 #define DEV_INFO_FOR_EACH_FLAG(func) \
@@ -150,19 +166,18 @@ struct sseu_dev_info {
 	u8 eu_mask[GEN_MAX_SLICES * GEN_MAX_SUBSLICES];
 };
 
-typedef u8 intel_ring_mask_t;
-
 struct intel_device_info {
 	u16 gen_mask;
 
 	u8 gen;
 	u8 gt; /* GT number, 0 if undefined */
-	intel_ring_mask_t ring_mask; /* Rings supported by the HW */
+	intel_engine_mask_t engine_mask; /* Engines supported by the HW */
 
 	enum intel_platform platform;
-	u32 platform_mask;
 
-	enum intel_ppgtt ppgtt;
+	enum intel_ppgtt_type ppgtt_type;
+	unsigned int ppgtt_size; /* log2, e.g. 31/32/48 bits */
+
 	unsigned int page_sizes; /* page sizes supported by the HW */
 
 	u32 display_mmio_offset;
@@ -195,22 +210,28 @@ struct intel_device_info {
 };
 
 struct intel_runtime_info {
+	/*
+	 * Platform mask is used for optimizing or-ed IS_PLATFORM calls into
+	 * into single runtime conditionals, and also to provide groundwork
+	 * for future per platform, or per SKU build optimizations.
+	 *
+	 * Array can be extended when necessary if the corresponding
+	 * BUILD_BUG_ON is hit.
+	 */
+	u32 platform_mask[2];
+
 	u16 device_id;
 
 	u8 num_sprites[I915_MAX_PIPES];
 	u8 num_scalers[I915_MAX_PIPES];
 
-	u8 num_rings;
+	u8 num_engines;
 
 	/* Slice/subslice/EU info */
 	struct sseu_dev_info sseu;
 
 	u32 cs_timestamp_frequency_khz;
 
-	/* Enabled (not fused off) media engine bitmasks. */
-	u8 vdbox_enable;
-	u8 vebox_enable;
-
 	/* Media engine access to SFC per instance */
 	u8 vdbox_sfc_access;
 };
@@ -269,6 +290,7 @@ static inline void sseu_set_eus(struct sseu_dev_info *sseu,
 
 const char *intel_platform_name(enum intel_platform platform);
 
+void intel_device_info_subplatform_init(struct drm_i915_private *dev_priv);
 void intel_device_info_runtime_init(struct drm_i915_private *dev_priv);
 void intel_device_info_dump_flags(const struct intel_device_info *info,
 				  struct drm_printer *p);
diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index 421aac80a838..3bd40a4a6739 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -46,19 +46,29 @@
 
 #include "i915_drv.h"
 #include "i915_gem_clflush.h"
+#include "i915_reset.h"
 #include "i915_trace.h"
+#include "intel_atomic_plane.h"
+#include "intel_color.h"
+#include "intel_cdclk.h"
+#include "intel_crt.h"
+#include "intel_ddi.h"
+#include "intel_dp.h"
 #include "intel_drv.h"
 #include "intel_dsi.h"
+#include "intel_dvo.h"
+#include "intel_fbc.h"
+#include "intel_fbdev.h"
 #include "intel_frontbuffer.h"
-
-#include "intel_drv.h"
-#include "intel_dsi.h"
-#include "intel_frontbuffer.h"
-
-#include "i915_drv.h"
-#include "i915_gem_clflush.h"
-#include "i915_reset.h"
-#include "i915_trace.h"
+#include "intel_hdcp.h"
+#include "intel_hdmi.h"
+#include "intel_lvds.h"
+#include "intel_pipe_crc.h"
+#include "intel_pm.h"
+#include "intel_psr.h"
+#include "intel_sdvo.h"
+#include "intel_sprite.h"
+#include "intel_tv.h"
 
 /* Primary plane formats for gen <= 3 */
 static const u32 i8xx_primary_formats[] = {
@@ -115,8 +125,8 @@ static void vlv_prepare_pll(struct intel_crtc *crtc,
 			    const struct intel_crtc_state *pipe_config);
 static void chv_prepare_pll(struct intel_crtc *crtc,
 			    const struct intel_crtc_state *pipe_config);
-static void intel_begin_crtc_commit(struct drm_crtc *, struct drm_crtc_state *);
-static void intel_finish_crtc_commit(struct drm_crtc *, struct drm_crtc_state *);
+static void intel_begin_crtc_commit(struct intel_atomic_state *, struct intel_crtc *);
+static void intel_finish_crtc_commit(struct intel_atomic_state *, struct intel_crtc *);
 static void intel_crtc_init_scalers(struct intel_crtc *crtc,
 				    struct intel_crtc_state *crtc_state);
 static void skylake_pfit_enable(const struct intel_crtc_state *crtc_state);
@@ -467,10 +477,11 @@ static const struct intel_limit intel_limits_bxt = {
 };
 
 static void
-skl_wa_clkgate(struct drm_i915_private *dev_priv, int pipe, bool enable)
+skl_wa_827(struct drm_i915_private *dev_priv, int pipe, bool enable)
 {
 	if (enable)
 		I915_WRITE(CLKGATE_DIS_PSL(pipe),
+			   I915_READ(CLKGATE_DIS_PSL(pipe)) |
 			   DUPS1_GATING_DIS | DUPS2_GATING_DIS);
 	else
 		I915_WRITE(CLKGATE_DIS_PSL(pipe),
@@ -595,7 +606,7 @@ i9xx_select_p2_div(const struct intel_limit *limit,
 		   const struct intel_crtc_state *crtc_state,
 		   int target)
 {
-	struct drm_device *dev = crtc_state->base.crtc->dev;
+	struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
 
 	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
 		/*
@@ -603,7 +614,7 @@ i9xx_select_p2_div(const struct intel_limit *limit,
 		 * We haven't figured out how to reliably set up different
 		 * single/dual channel state, if we even can.
 		 */
-		if (intel_is_dual_link_lvds(dev))
+		if (intel_is_dual_link_lvds(dev_priv))
 			return limit->p2.p2_fast;
 		else
 			return limit->p2.p2_slow;
@@ -951,14 +962,15 @@ chv_find_best_dpll(const struct intel_limit *limit,
 	return found;
 }
 
-bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state, int target_clock,
+bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state,
 			struct dpll *best_clock)
 {
 	int refclk = 100000;
 	const struct intel_limit *limit = &intel_limits_bxt;
 
 	return chv_find_best_dpll(limit, crtc_state,
-				  target_clock, refclk, NULL, best_clock);
+				  crtc_state->port_clock, refclk,
+				  NULL, best_clock);
 }
 
 bool intel_crtc_active(struct intel_crtc *crtc)
@@ -1039,7 +1051,7 @@ intel_wait_for_pipe_off(const struct intel_crtc_state *old_crtc_state)
 		i915_reg_t reg = PIPECONF(cpu_transcoder);
 
 		/* Wait for the Pipe State to go off */
-		if (intel_wait_for_register(dev_priv,
+		if (intel_wait_for_register(&dev_priv->uncore,
 					    reg, I965_PIPECONF_ACTIVE, 0,
 					    100))
 			WARN(1, "pipe_off wait timed out\n");
@@ -1345,7 +1357,7 @@ static void _vlv_enable_pll(struct intel_crtc *crtc,
 	POSTING_READ(DPLL(pipe));
 	udelay(150);
 
-	if (intel_wait_for_register(dev_priv,
+	if (intel_wait_for_register(&dev_priv->uncore,
 				    DPLL(pipe),
 				    DPLL_LOCK_VLV,
 				    DPLL_LOCK_VLV,
@@ -1398,7 +1410,7 @@ static void _chv_enable_pll(struct intel_crtc *crtc,
 	I915_WRITE(DPLL(pipe), pipe_config->dpll_hw_state.dpll);
 
 	/* Check PLL is locked */
-	if (intel_wait_for_register(dev_priv,
+	if (intel_wait_for_register(&dev_priv->uncore,
 				    DPLL(pipe), DPLL_LOCK_VLV, DPLL_LOCK_VLV,
 				    1))
 		DRM_ERROR("PLL %d failed to lock\n", pipe);
@@ -1441,17 +1453,12 @@ static void chv_enable_pll(struct intel_crtc *crtc,
 	}
 }
 
-static int intel_num_dvo_pipes(struct drm_i915_private *dev_priv)
+static bool i9xx_has_pps(struct drm_i915_private *dev_priv)
 {
-	struct intel_crtc *crtc;
-	int count = 0;
-
-	for_each_intel_crtc(&dev_priv->drm, crtc) {
-		count += crtc->base.state->active &&
-			intel_crtc_has_type(crtc->config, INTEL_OUTPUT_DVO);
-	}
+	if (IS_I830(dev_priv))
+		return false;
 
-	return count;
+	return IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv);
 }
 
 static void i9xx_enable_pll(struct intel_crtc *crtc,
@@ -1465,29 +1472,15 @@ static void i9xx_enable_pll(struct intel_crtc *crtc,
 	assert_pipe_disabled(dev_priv, crtc->pipe);
 
 	/* PLL is protected by panel, make sure we can write it */
-	if (IS_MOBILE(dev_priv) && !IS_I830(dev_priv))
+	if (i9xx_has_pps(dev_priv))
 		assert_panel_unlocked(dev_priv, crtc->pipe);
 
-	/* Enable DVO 2x clock on both PLLs if necessary */
-	if (IS_I830(dev_priv) && intel_num_dvo_pipes(dev_priv) > 0) {
-		/*
-		 * It appears to be important that we don't enable this
-		 * for the current pipe before otherwise configuring the
-		 * PLL. No idea how this should be handled if multiple
-		 * DVO outputs are enabled simultaneosly.
-		 */
-		dpll |= DPLL_DVO_2X_MODE;
-		I915_WRITE(DPLL(!crtc->pipe),
-			   I915_READ(DPLL(!crtc->pipe)) | DPLL_DVO_2X_MODE);
-	}
-
 	/*
 	 * Apparently we need to have VGA mode enabled prior to changing
 	 * the P1/P2 dividers. Otherwise the DPLL will keep using the old
 	 * dividers, even though the register value does change.
 	 */
-	I915_WRITE(reg, 0);
-
+	I915_WRITE(reg, dpll & ~DPLL_VGA_MODE_DIS);
 	I915_WRITE(reg, dpll);
 
 	/* Wait for the clocks to stabilize. */
@@ -1520,16 +1513,6 @@ static void i9xx_disable_pll(const struct intel_crtc_state *crtc_state)
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	enum pipe pipe = crtc->pipe;
 
-	/* Disable DVO 2x clock on both PLLs if necessary */
-	if (IS_I830(dev_priv) &&
-	    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO) &&
-	    !intel_num_dvo_pipes(dev_priv)) {
-		I915_WRITE(DPLL(PIPE_B),
-			   I915_READ(DPLL(PIPE_B)) & ~DPLL_DVO_2X_MODE);
-		I915_WRITE(DPLL(PIPE_A),
-			   I915_READ(DPLL(PIPE_A)) & ~DPLL_DVO_2X_MODE);
-	}
-
 	/* Don't disable pipe or pipe PLLs if needed */
 	if (IS_I830(dev_priv))
 		return;
@@ -1608,7 +1591,7 @@ void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
 		BUG();
 	}
 
-	if (intel_wait_for_register(dev_priv,
+	if (intel_wait_for_register(&dev_priv->uncore,
 				    dpll_reg, port_mask, expected_mask,
 				    1000))
 		WARN(1, "timed out waiting for port %c ready: got 0x%x, expected 0x%x\n",
@@ -1658,17 +1641,18 @@ static void ironlake_enable_pch_transcoder(const struct intel_crtc_state *crtc_s
 	}
 
 	val &= ~TRANS_INTERLACE_MASK;
-	if ((pipeconf_val & PIPECONF_INTERLACE_MASK) == PIPECONF_INTERLACED_ILK)
+	if ((pipeconf_val & PIPECONF_INTERLACE_MASK) == PIPECONF_INTERLACED_ILK) {
 		if (HAS_PCH_IBX(dev_priv) &&
 		    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
 			val |= TRANS_LEGACY_INTERLACED_ILK;
 		else
 			val |= TRANS_INTERLACED;
-	else
+	} else {
 		val |= TRANS_PROGRESSIVE;
+	}
 
 	I915_WRITE(reg, val | TRANS_ENABLE);
-	if (intel_wait_for_register(dev_priv,
+	if (intel_wait_for_register(&dev_priv->uncore,
 				    reg, TRANS_STATE_ENABLE, TRANS_STATE_ENABLE,
 				    100))
 		DRM_ERROR("failed to enable transcoder %c\n", pipe_name(pipe));
@@ -1698,7 +1682,7 @@ static void lpt_enable_pch_transcoder(struct drm_i915_private *dev_priv,
 		val |= TRANS_PROGRESSIVE;
 
 	I915_WRITE(LPT_TRANSCONF, val);
-	if (intel_wait_for_register(dev_priv,
+	if (intel_wait_for_register(&dev_priv->uncore,
 				    LPT_TRANSCONF,
 				    TRANS_STATE_ENABLE,
 				    TRANS_STATE_ENABLE,
@@ -1724,7 +1708,7 @@ static void ironlake_disable_pch_transcoder(struct drm_i915_private *dev_priv,
 	val &= ~TRANS_ENABLE;
 	I915_WRITE(reg, val);
 	/* wait for PCH transcoder off, transcoder state */
-	if (intel_wait_for_register(dev_priv,
+	if (intel_wait_for_register(&dev_priv->uncore,
 				    reg, TRANS_STATE_ENABLE, 0,
 				    50))
 		DRM_ERROR("failed to disable transcoder %c\n", pipe_name(pipe));
@@ -1746,7 +1730,7 @@ void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv)
 	val &= ~TRANS_ENABLE;
 	I915_WRITE(LPT_TRANSCONF, val);
 	/* wait for PCH transcoder off, transcoder state */
-	if (intel_wait_for_register(dev_priv,
+	if (intel_wait_for_register(&dev_priv->uncore,
 				    LPT_TRANSCONF, TRANS_STATE_ENABLE, 0,
 				    50))
 		DRM_ERROR("Failed to disable PCH transcoder\n");
@@ -1830,6 +1814,8 @@ static void intel_enable_pipe(const struct intel_crtc_state *new_crtc_state)
 		/* FIXME: assert CPU port conditions for SNB+ */
 	}
 
+	trace_intel_pipe_enable(dev_priv, pipe);
+
 	reg = PIPECONF(cpu_transcoder);
 	val = I915_READ(reg);
 	if (val & PIPECONF_ENABLE) {
@@ -1869,6 +1855,8 @@ static void intel_disable_pipe(const struct intel_crtc_state *old_crtc_state)
 	 */
 	assert_planes_disabled(crtc);
 
+	trace_intel_pipe_disable(dev_priv, pipe);
+
 	reg = PIPECONF(cpu_transcoder);
 	val = I915_READ(reg);
 	if ((val & PIPECONF_ENABLE) == 0)
@@ -2677,6 +2665,24 @@ int skl_format_to_fourcc(int format, bool rgb_order, bool alpha)
 		return DRM_FORMAT_RGB565;
 	case PLANE_CTL_FORMAT_NV12:
 		return DRM_FORMAT_NV12;
+	case PLANE_CTL_FORMAT_P010:
+		return DRM_FORMAT_P010;
+	case PLANE_CTL_FORMAT_P012:
+		return DRM_FORMAT_P012;
+	case PLANE_CTL_FORMAT_P016:
+		return DRM_FORMAT_P016;
+	case PLANE_CTL_FORMAT_Y210:
+		return DRM_FORMAT_Y210;
+	case PLANE_CTL_FORMAT_Y212:
+		return DRM_FORMAT_Y212;
+	case PLANE_CTL_FORMAT_Y216:
+		return DRM_FORMAT_Y216;
+	case PLANE_CTL_FORMAT_Y410:
+		return DRM_FORMAT_XVYU2101010;
+	case PLANE_CTL_FORMAT_Y412:
+		return DRM_FORMAT_XVYU12_16161616;
+	case PLANE_CTL_FORMAT_Y416:
+		return DRM_FORMAT_XVYU16161616;
 	default:
 	case PLANE_CTL_FORMAT_XRGB_8888:
 		if (rgb_order) {
@@ -2695,6 +2701,18 @@ int skl_format_to_fourcc(int format, bool rgb_order, bool alpha)
 			return DRM_FORMAT_XBGR2101010;
 		else
 			return DRM_FORMAT_XRGB2101010;
+	case PLANE_CTL_FORMAT_XRGB_16161616F:
+		if (rgb_order) {
+			if (alpha)
+				return DRM_FORMAT_ABGR16161616F;
+			else
+				return DRM_FORMAT_XBGR16161616F;
+		} else {
+			if (alpha)
+				return DRM_FORMAT_ARGB16161616F;
+			else
+				return DRM_FORMAT_XRGB16161616F;
+		}
 	}
 }
 
@@ -2825,8 +2843,7 @@ static void intel_plane_disable_noatomic(struct intel_crtc *crtc,
 	if (plane->id == PLANE_PRIMARY)
 		intel_pre_disable_primary_noatomic(&crtc->base);
 
-	trace_intel_disable_plane(&plane->base, crtc);
-	plane->disable_plane(plane, crtc_state);
+	intel_disable_plane(plane, crtc_state);
 }
 
 static void
@@ -3176,7 +3193,7 @@ int skl_check_plane_surface(struct intel_plane_state *plane_state)
 	 * Handle the AUX surface first since
 	 * the main surface setup depends on it.
 	 */
-	if (fb->format->format == DRM_FORMAT_NV12) {
+	if (is_planar_yuv_format(fb->format->format)) {
 		ret = skl_check_nv12_aux_surface(plane_state);
 		if (ret)
 			return ret;
@@ -3230,9 +3247,10 @@ static u32 i9xx_plane_ctl_crtc(const struct intel_crtc_state *crtc_state)
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	u32 dspcntr = 0;
 
-	dspcntr |= DISPPLANE_GAMMA_ENABLE;
+	if (crtc_state->gamma_enable)
+		dspcntr |= DISPPLANE_GAMMA_ENABLE;
 
-	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
+	if (crtc_state->csc_enable)
 		dspcntr |= DISPPLANE_PIPE_CSC_ENABLE;
 
 	if (INTEL_GEN(dev_priv) < 5)
@@ -3459,7 +3477,7 @@ static void i9xx_disable_plane(struct intel_plane *plane,
 	 *
 	 * On pre-g4x there is no way to gamma correct the
 	 * pipe bottom color but we'll keep on doing this
-	 * anyway.
+	 * anyway so that the crtc state readout works correctly.
 	 */
 	dspcntr = i9xx_plane_ctl_crtc(crtc_state);
 
@@ -3590,6 +3608,12 @@ static u32 skl_plane_ctl_format(u32 pixel_format)
 		return PLANE_CTL_FORMAT_XRGB_2101010;
 	case DRM_FORMAT_XBGR2101010:
 		return PLANE_CTL_ORDER_RGBX | PLANE_CTL_FORMAT_XRGB_2101010;
+	case DRM_FORMAT_XBGR16161616F:
+	case DRM_FORMAT_ABGR16161616F:
+		return PLANE_CTL_FORMAT_XRGB_16161616F | PLANE_CTL_ORDER_RGBX;
+	case DRM_FORMAT_XRGB16161616F:
+	case DRM_FORMAT_ARGB16161616F:
+		return PLANE_CTL_FORMAT_XRGB_16161616F;
 	case DRM_FORMAT_YUYV:
 		return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YUYV;
 	case DRM_FORMAT_YVYU:
@@ -3600,6 +3624,24 @@ static u32 skl_plane_ctl_format(u32 pixel_format)
 		return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_VYUY;
 	case DRM_FORMAT_NV12:
 		return PLANE_CTL_FORMAT_NV12;
+	case DRM_FORMAT_P010:
+		return PLANE_CTL_FORMAT_P010;
+	case DRM_FORMAT_P012:
+		return PLANE_CTL_FORMAT_P012;
+	case DRM_FORMAT_P016:
+		return PLANE_CTL_FORMAT_P016;
+	case DRM_FORMAT_Y210:
+		return PLANE_CTL_FORMAT_Y210;
+	case DRM_FORMAT_Y212:
+		return PLANE_CTL_FORMAT_Y212;
+	case DRM_FORMAT_Y216:
+		return PLANE_CTL_FORMAT_Y216;
+	case DRM_FORMAT_XVYU2101010:
+		return PLANE_CTL_FORMAT_Y410;
+	case DRM_FORMAT_XVYU12_16161616:
+		return PLANE_CTL_FORMAT_Y412;
+	case DRM_FORMAT_XVYU16161616:
+		return PLANE_CTL_FORMAT_Y416;
 	default:
 		MISSING_CASE(pixel_format);
 	}
@@ -3710,8 +3752,11 @@ u32 skl_plane_ctl_crtc(const struct intel_crtc_state *crtc_state)
 	if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
 		return plane_ctl;
 
-	plane_ctl |= PLANE_CTL_PIPE_GAMMA_ENABLE;
-	plane_ctl |= PLANE_CTL_PIPE_CSC_ENABLE;
+	if (crtc_state->gamma_enable)
+		plane_ctl |= PLANE_CTL_PIPE_GAMMA_ENABLE;
+
+	if (crtc_state->csc_enable)
+		plane_ctl |= PLANE_CTL_PIPE_CSC_ENABLE;
 
 	return plane_ctl;
 }
@@ -3763,8 +3808,11 @@ u32 glk_plane_color_ctl_crtc(const struct intel_crtc_state *crtc_state)
 	if (INTEL_GEN(dev_priv) >= 11)
 		return plane_color_ctl;
 
-	plane_color_ctl |= PLANE_COLOR_PIPE_GAMMA_ENABLE;
-	plane_color_ctl |= PLANE_COLOR_PIPE_CSC_ENABLE;
+	if (crtc_state->gamma_enable)
+		plane_color_ctl |= PLANE_COLOR_PIPE_GAMMA_ENABLE;
+
+	if (crtc_state->csc_enable)
+		plane_color_ctl |= PLANE_COLOR_PIPE_CSC_ENABLE;
 
 	return plane_color_ctl;
 }
@@ -3772,6 +3820,8 @@ u32 glk_plane_color_ctl_crtc(const struct intel_crtc_state *crtc_state)
 u32 glk_plane_color_ctl(const struct intel_crtc_state *crtc_state,
 			const struct intel_plane_state *plane_state)
 {
+	struct drm_i915_private *dev_priv =
+		to_i915(plane_state->base.plane->dev);
 	const struct drm_framebuffer *fb = plane_state->base.fb;
 	struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
 	u32 plane_color_ctl = 0;
@@ -3779,7 +3829,7 @@ u32 glk_plane_color_ctl(const struct intel_crtc_state *crtc_state,
 	plane_color_ctl |= PLANE_COLOR_PLANE_GAMMA_DISABLE;
 	plane_color_ctl |= glk_plane_color_ctl_alpha(plane_state);
 
-	if (fb->format->is_yuv && !icl_is_hdr_plane(plane)) {
+	if (fb->format->is_yuv && !icl_is_hdr_plane(dev_priv, plane->id)) {
 		if (plane_state->base.color_encoding == DRM_COLOR_YCBCR_BT709)
 			plane_color_ctl |= PLANE_COLOR_CSC_MODE_YUV709_TO_RGB709;
 		else
@@ -3921,9 +3971,6 @@ void intel_finish_reset(struct drm_i915_private *dev_priv)
 		 * The display has been reset as well,
 		 * so need a full re-initialization.
 		 */
-		intel_runtime_pm_disable_interrupts(dev_priv);
-		intel_runtime_pm_enable_interrupts(dev_priv);
-
 		intel_pps_unlock_regs_wa(dev_priv);
 		intel_modeset_init_hw(dev);
 		intel_init_clock_gating(dev_priv);
@@ -3963,13 +4010,13 @@ static void icl_set_pipe_chicken(struct intel_crtc *crtc)
 	 * and rounding for per-pixel values 00 and 0xff
 	 */
 	tmp |= PER_PIXEL_ALPHA_BYPASS_EN;
-
 	/*
-	 * W/A for underruns with linear/X-tiled with
-	 * WM1+ disabled.
+	 * Display WA # 1605353570: icl
+	 * Set the pixel rounding bit to 1 for allowing
+	 * passthrough of Frame buffer pixels unmodified
+	 * across pipe
 	 */
-	tmp |= PM_FILL_MAINTAIN_DBUF_FULLNESS;
-
+	tmp |= PIXEL_ROUNDING_TRUNC_FB_PASSTHRU;
 	I915_WRITE(PIPE_CHICKEN(pipe), tmp);
 }
 
@@ -4008,16 +4055,6 @@ static void intel_update_pipe_config(const struct intel_crtc_state *old_crtc_sta
 			ironlake_pfit_disable(old_crtc_state);
 	}
 
-	/*
-	 * We don't (yet) allow userspace to control the pipe background color,
-	 * so force it to black, but apply pipe gamma and CSC so that its
-	 * handling will match how we program our planes.
-	 */
-	if (INTEL_GEN(dev_priv) >= 9)
-		I915_WRITE(SKL_BOTTOM_COLOR(crtc->pipe),
-			   SKL_BOTTOM_COLOR_GAMMA_ENABLE |
-			   SKL_BOTTOM_COLOR_CSC_ENABLE);
-
 	if (INTEL_GEN(dev_priv) >= 11)
 		icl_set_pipe_chicken(crtc);
 }
@@ -5036,19 +5073,19 @@ skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach,
 		return 0;
 	}
 
-	if (format && format->format == DRM_FORMAT_NV12 &&
+	if (format && is_planar_yuv_format(format->format) &&
 	    (src_h < SKL_MIN_YUV_420_SRC_H || src_w < SKL_MIN_YUV_420_SRC_W)) {
-		DRM_DEBUG_KMS("NV12: src dimensions not met\n");
+		DRM_DEBUG_KMS("Planar YUV: src dimensions not met\n");
 		return -EINVAL;
 	}
 
 	/* range checks */
 	if (src_w < SKL_MIN_SRC_W || src_h < SKL_MIN_SRC_H ||
 	    dst_w < SKL_MIN_DST_W || dst_h < SKL_MIN_DST_H ||
-	    (IS_GEN(dev_priv, 11) &&
+	    (INTEL_GEN(dev_priv) >= 11 &&
 	     (src_w > ICL_MAX_SRC_W || src_h > ICL_MAX_SRC_H ||
 	      dst_w > ICL_MAX_DST_W || dst_h > ICL_MAX_DST_H)) ||
-	    (!IS_GEN(dev_priv, 11) &&
+	    (INTEL_GEN(dev_priv) < 11 &&
 	     (src_w > SKL_MAX_SRC_W || src_h > SKL_MAX_SRC_H ||
 	      dst_w > SKL_MAX_DST_W || dst_h > SKL_MAX_DST_H)))	{
 		DRM_DEBUG_KMS("scaler_user index %u.%u: src %ux%u dst %ux%u "
@@ -5105,14 +5142,15 @@ static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state,
 {
 	struct intel_plane *intel_plane =
 		to_intel_plane(plane_state->base.plane);
+	struct drm_i915_private *dev_priv = to_i915(intel_plane->base.dev);
 	struct drm_framebuffer *fb = plane_state->base.fb;
 	int ret;
 	bool force_detach = !fb || !plane_state->base.visible;
 	bool need_scaler = false;
 
 	/* Pre-gen11 and SDR planes always need a scaler for planar formats. */
-	if (!icl_is_hdr_plane(intel_plane) &&
-	    fb && fb->format->format == DRM_FORMAT_NV12)
+	if (!icl_is_hdr_plane(dev_priv, intel_plane->id) &&
+	    fb && is_planar_yuv_format(fb->format->format))
 		need_scaler = true;
 
 	ret = skl_update_scaler(crtc_state, force_detach,
@@ -5144,11 +5182,24 @@ static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state,
 	case DRM_FORMAT_ARGB8888:
 	case DRM_FORMAT_XRGB2101010:
 	case DRM_FORMAT_XBGR2101010:
+	case DRM_FORMAT_XBGR16161616F:
+	case DRM_FORMAT_ABGR16161616F:
+	case DRM_FORMAT_XRGB16161616F:
+	case DRM_FORMAT_ARGB16161616F:
 	case DRM_FORMAT_YUYV:
 	case DRM_FORMAT_YVYU:
 	case DRM_FORMAT_UYVY:
 	case DRM_FORMAT_VYUY:
 	case DRM_FORMAT_NV12:
+	case DRM_FORMAT_P010:
+	case DRM_FORMAT_P012:
+	case DRM_FORMAT_P016:
+	case DRM_FORMAT_Y210:
+	case DRM_FORMAT_Y212:
+	case DRM_FORMAT_Y216:
+	case DRM_FORMAT_XVYU2101010:
+	case DRM_FORMAT_XVYU12_16161616:
+	case DRM_FORMAT_XVYU16161616:
 		break;
 	default:
 		DRM_DEBUG_KMS("[PLANE:%d:%s] FB:%d unsupported scaling format 0x%x\n",
@@ -5259,7 +5310,7 @@ void hsw_enable_ips(const struct intel_crtc_state *crtc_state)
 		 * and don't wait for vblanks until the end of crtc_enable, then
 		 * the HW state readout code will complain that the expected
 		 * IPS_CTL value is not the one we read. */
-		if (intel_wait_for_register(dev_priv,
+		if (intel_wait_for_register(&dev_priv->uncore,
 					    IPS_CTL, IPS_ENABLE, IPS_ENABLE,
 					    50))
 			DRM_ERROR("Timed out waiting for IPS enable\n");
@@ -5284,7 +5335,7 @@ void hsw_disable_ips(const struct intel_crtc_state *crtc_state)
 		 * 42ms timeout value leads to occasional timeouts so use 100ms
 		 * instead.
 		 */
-		if (intel_wait_for_register(dev_priv,
+		if (intel_wait_for_register(&dev_priv->uncore,
 					    IPS_CTL, IPS_ENABLE, 0,
 					    100))
 			DRM_ERROR("Timed out waiting for IPS disable\n");
@@ -5490,7 +5541,7 @@ static void intel_post_plane_update(struct intel_crtc_state *old_crtc_state)
 	/* Display WA 827 */
 	if (needs_nv12_wa(dev_priv, old_crtc_state) &&
 	    !needs_nv12_wa(dev_priv, pipe_config)) {
-		skl_wa_clkgate(dev_priv, crtc->pipe, false);
+		skl_wa_827(dev_priv, crtc->pipe, false);
 	}
 }
 
@@ -5529,7 +5580,7 @@ static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state,
 	/* Display WA 827 */
 	if (!needs_nv12_wa(dev_priv, old_crtc_state) &&
 	    needs_nv12_wa(dev_priv, pipe_config)) {
-		skl_wa_clkgate(dev_priv, crtc->pipe, true);
+		skl_wa_827(dev_priv, crtc->pipe, true);
 	}
 
 	/*
@@ -5603,7 +5654,7 @@ static void intel_crtc_disable_planes(struct intel_atomic_state *state,
 		    !(update_mask & BIT(plane->id)))
 			continue;
 
-		plane->disable_plane(plane, new_crtc_state);
+		intel_disable_plane(plane, new_crtc_state);
 
 		if (old_plane_state->base.visible)
 			fb_bits |= plane->frontbuffer_bit;
@@ -5754,6 +5805,14 @@ static void intel_encoders_update_pipe(struct drm_crtc *crtc,
 	}
 }
 
+static void intel_disable_primary_plane(const struct intel_crtc_state *crtc_state)
+{
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+	struct intel_plane *plane = to_intel_plane(crtc->base.primary);
+
+	plane->disable_plane(plane, crtc_state);
+}
+
 static void ironlake_crtc_enable(struct intel_crtc_state *pipe_config,
 				 struct drm_atomic_state *old_state)
 {
@@ -5819,6 +5878,8 @@ static void ironlake_crtc_enable(struct intel_crtc_state *pipe_config,
 	 */
 	intel_color_load_luts(pipe_config);
 	intel_color_commit(pipe_config);
+	/* update DSPCNTR to configure gamma for pipe bottom color */
+	intel_disable_primary_plane(pipe_config);
 
 	if (dev_priv->display.initial_watermarks != NULL)
 		dev_priv->display.initial_watermarks(old_intel_state, pipe_config);
@@ -5947,6 +6008,9 @@ static void haswell_crtc_enable(struct intel_crtc_state *pipe_config,
 	 */
 	intel_color_load_luts(pipe_config);
 	intel_color_commit(pipe_config);
+	/* update DSPCNTR to configure gamma/csc for pipe bottom color */
+	if (INTEL_GEN(dev_priv) < 9)
+		intel_disable_primary_plane(pipe_config);
 
 	if (INTEL_GEN(dev_priv) >= 11)
 		icl_set_pipe_chicken(intel_crtc);
@@ -6127,7 +6191,10 @@ bool intel_port_is_combophy(struct drm_i915_private *dev_priv, enum port port)
 	if (port == PORT_NONE)
 		return false;
 
-	if (IS_ICELAKE(dev_priv))
+	if (IS_ELKHARTLAKE(dev_priv))
+		return port <= PORT_C;
+
+	if (INTEL_GEN(dev_priv) >= 11)
 		return port <= PORT_B;
 
 	return false;
@@ -6135,7 +6202,7 @@ bool intel_port_is_combophy(struct drm_i915_private *dev_priv, enum port port)
 
 bool intel_port_is_tc(struct drm_i915_private *dev_priv, enum port port)
 {
-	if (IS_ICELAKE(dev_priv))
+	if (INTEL_GEN(dev_priv) >= 11 && !IS_ELKHARTLAKE(dev_priv))
 		return port >= PORT_C && port <= PORT_F;
 
 	return false;
@@ -6304,6 +6371,8 @@ static void valleyview_crtc_enable(struct intel_crtc_state *pipe_config,
 
 	intel_color_load_luts(pipe_config);
 	intel_color_commit(pipe_config);
+	/* update DSPCNTR to configure gamma for pipe bottom color */
+	intel_disable_primary_plane(pipe_config);
 
 	dev_priv->display.initial_watermarks(old_intel_state,
 					     pipe_config);
@@ -6361,6 +6430,8 @@ static void i9xx_crtc_enable(struct intel_crtc_state *pipe_config,
 
 	intel_color_load_luts(pipe_config);
 	intel_color_commit(pipe_config);
+	/* update DSPCNTR to configure gamma for pipe bottom color */
+	intel_disable_primary_plane(pipe_config);
 
 	if (dev_priv->display.initial_watermarks != NULL)
 		dev_priv->display.initial_watermarks(old_intel_state,
@@ -6743,7 +6814,13 @@ static bool hsw_compute_ips_config(struct intel_crtc_state *crtc_state)
 	if (!hsw_crtc_state_ips_capable(crtc_state))
 		return false;
 
-	if (crtc_state->ips_force_disable)
+	/*
+	 * When IPS gets enabled, the pipe CRC changes. Since IPS gets
+	 * enabled and disabled dynamically based on package C states,
+	 * user space can't make reliable use of the CRCs, so let's just
+	 * completely disable it.
+	 */
+	if (crtc_state->crc_enabled)
 		return false;
 
 	/* IPS should be fine as long as at least one plane is enabled. */
@@ -6818,8 +6895,7 @@ static void intel_crtc_compute_pixel_rate(struct intel_crtc_state *crtc_state)
 static int intel_crtc_compute_config(struct intel_crtc *crtc,
 				     struct intel_crtc_state *pipe_config)
 {
-	struct drm_device *dev = crtc->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
 	int clock_limit = dev_priv->max_dotclk_freq;
 
@@ -6869,7 +6945,7 @@ static int intel_crtc_compute_config(struct intel_crtc *crtc,
 		}
 
 		if (intel_crtc_has_type(pipe_config, INTEL_OUTPUT_LVDS) &&
-		    intel_is_dual_link_lvds(dev)) {
+		    intel_is_dual_link_lvds(dev_priv)) {
 			DRM_DEBUG_KMS("Odd pipe source width not supported with dual link LVDS\n");
 			return -EINVAL;
 		}
@@ -7486,7 +7562,19 @@ static void i8xx_compute_dpll(struct intel_crtc *crtc,
 			dpll |= PLL_P2_DIVIDE_BY_4;
 	}
 
-	if (!IS_I830(dev_priv) &&
+	/*
+	 * Bspec:
+	 * "[Almador Errata}: For the correct operation of the muxed DVO pins
+	 *  (GDEVSELB/I2Cdata, GIRDBY/I2CClk) and (GFRAMEB/DVI_Data,
+	 *  GTRDYB/DVI_Clk): Bit 31 (DPLL VCO Enable) and Bit 30 (2X Clock
+	 *  Enable) must be set to “1� in both the DPLL A Control Register
+	 *  (06014h-06017h) and DPLL B Control Register (06018h-0601Bh)."
+	 *
+	 * For simplicity We simply keep both bits always enabled in
+	 * both DPLLS. The spec says we should disable the DVO 2X clock
+	 * when not needed, but this seems to work fine in practice.
+	 */
+	if (IS_I830(dev_priv) ||
 	    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO))
 		dpll |= DPLL_DVO_2X_MODE;
 
@@ -7694,13 +7782,16 @@ static void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state)
 			pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION;
 		else
 			pipeconf |= PIPECONF_INTERLACE_W_SYNC_SHIFT;
-	} else
+	} else {
 		pipeconf |= PIPECONF_PROGRESSIVE;
+	}
 
 	if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
 	     crtc_state->limited_color_range)
 		pipeconf |= PIPECONF_COLOR_RANGE_SELECT;
 
+	pipeconf |= PIPECONF_GAMMA_MODE(crtc_state->gamma_mode);
+
 	I915_WRITE(PIPECONF(crtc->pipe), pipeconf);
 	POSTING_READ(PIPECONF(crtc->pipe));
 }
@@ -7744,8 +7835,7 @@ static int i8xx_crtc_compute_clock(struct intel_crtc *crtc,
 static int g4x_crtc_compute_clock(struct intel_crtc *crtc,
 				  struct intel_crtc_state *crtc_state)
 {
-	struct drm_device *dev = crtc->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	const struct intel_limit *limit;
 	int refclk = 96000;
 
@@ -7758,7 +7848,7 @@ static int g4x_crtc_compute_clock(struct intel_crtc *crtc,
 			DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
 		}
 
-		if (intel_is_dual_link_lvds(dev))
+		if (intel_is_dual_link_lvds(dev_priv))
 			limit = &intel_limits_g4x_dual_channel_lvds;
 		else
 			limit = &intel_limits_g4x_single_channel_lvds;
@@ -7894,14 +7984,22 @@ static int vlv_crtc_compute_clock(struct intel_crtc *crtc,
 	return 0;
 }
 
+static bool i9xx_has_pfit(struct drm_i915_private *dev_priv)
+{
+	if (IS_I830(dev_priv))
+		return false;
+
+	return INTEL_GEN(dev_priv) >= 4 ||
+		IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv);
+}
+
 static void i9xx_get_pfit_config(struct intel_crtc *crtc,
 				 struct intel_crtc_state *pipe_config)
 {
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	u32 tmp;
 
-	if (INTEL_GEN(dev_priv) <= 3 &&
-	    (IS_I830(dev_priv) || !IS_MOBILE(dev_priv)))
+	if (!i9xx_has_pfit(dev_priv))
 		return;
 
 	tmp = I915_READ(PFIT_CONTROL);
@@ -8108,6 +8206,24 @@ static void intel_get_crtc_ycbcr_config(struct intel_crtc *crtc,
 	pipe_config->output_format = output;
 }
 
+static void i9xx_get_pipe_color_config(struct intel_crtc_state *crtc_state)
+{
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+	struct intel_plane *plane = to_intel_plane(crtc->base.primary);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+	enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
+	u32 tmp;
+
+	tmp = I915_READ(DSPCNTR(i9xx_plane));
+
+	if (tmp & DISPPLANE_GAMMA_ENABLE)
+		crtc_state->gamma_enable = true;
+
+	if (!HAS_GMCH(dev_priv) &&
+	    tmp & DISPPLANE_PIPE_CSC_ENABLE)
+		crtc_state->csc_enable = true;
+}
+
 static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
 				 struct intel_crtc_state *pipe_config)
 {
@@ -8153,6 +8269,14 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
 	    (tmp & PIPECONF_COLOR_RANGE_SELECT))
 		pipe_config->limited_color_range = true;
 
+	pipe_config->gamma_mode = (tmp & PIPECONF_GAMMA_MODE_MASK_I9XX) >>
+		PIPECONF_GAMMA_MODE_SHIFT;
+
+	if (IS_CHERRYVIEW(dev_priv))
+		pipe_config->cgm_mode = I915_READ(CGM_PIPE_MODE(crtc->pipe));
+
+	i9xx_get_pipe_color_config(pipe_config);
+
 	if (INTEL_GEN(dev_priv) < 4)
 		pipe_config->double_wide = tmp & PIPECONF_DOUBLE_WIDE;
 
@@ -8185,14 +8309,6 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
 	}
 	pipe_config->dpll_hw_state.dpll = I915_READ(DPLL(crtc->pipe));
 	if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) {
-		/*
-		 * DPLL_DVO_2X_MODE must be enabled for both DPLLs
-		 * on 830. Filter it out here so that we don't
-		 * report errors due to that.
-		 */
-		if (IS_I830(dev_priv))
-			pipe_config->dpll_hw_state.dpll &= ~DPLL_DVO_2X_MODE;
-
 		pipe_config->dpll_hw_state.fp0 = I915_READ(FP0(crtc->pipe));
 		pipe_config->dpll_hw_state.fp1 = I915_READ(FP1(crtc->pipe));
 	} else {
@@ -8692,6 +8808,8 @@ static void ironlake_set_pipeconf(const struct intel_crtc_state *crtc_state)
 	if (crtc_state->limited_color_range)
 		val |= PIPECONF_COLOR_RANGE_SELECT;
 
+	val |= PIPECONF_GAMMA_MODE(crtc_state->gamma_mode);
+
 	I915_WRITE(PIPECONF(pipe), val);
 	POSTING_READ(PIPECONF(pipe));
 }
@@ -8772,13 +8890,11 @@ static bool ironlake_needs_fb_cb_tune(struct dpll *dpll, int factor)
 	return i9xx_dpll_compute_m(dpll) < factor * dpll->n;
 }
 
-static void ironlake_compute_dpll(struct intel_crtc *intel_crtc,
+static void ironlake_compute_dpll(struct intel_crtc *crtc,
 				  struct intel_crtc_state *crtc_state,
 				  struct dpll *reduced_clock)
 {
-	struct drm_crtc *crtc = &intel_crtc->base;
-	struct drm_device *dev = crtc->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	u32 dpll, fp, fp2;
 	int factor;
 
@@ -8787,10 +8903,12 @@ static void ironlake_compute_dpll(struct intel_crtc *intel_crtc,
 	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
 		if ((intel_panel_use_ssc(dev_priv) &&
 		     dev_priv->vbt.lvds_ssc_freq == 100000) ||
-		    (HAS_PCH_IBX(dev_priv) && intel_is_dual_link_lvds(dev)))
+		    (HAS_PCH_IBX(dev_priv) &&
+		     intel_is_dual_link_lvds(dev_priv)))
 			factor = 25;
-	} else if (crtc_state->sdvo_tv_clock)
+	} else if (crtc_state->sdvo_tv_clock) {
 		factor = 20;
+	}
 
 	fp = i9xx_dpll_compute_fp(&crtc_state->dpll);
 
@@ -8877,8 +8995,7 @@ static void ironlake_compute_dpll(struct intel_crtc *intel_crtc,
 static int ironlake_crtc_compute_clock(struct intel_crtc *crtc,
 				       struct intel_crtc_state *crtc_state)
 {
-	struct drm_device *dev = crtc->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	const struct intel_limit *limit;
 	int refclk = 120000;
 
@@ -8896,7 +9013,7 @@ static int ironlake_crtc_compute_clock(struct intel_crtc *crtc,
 			refclk = dev_priv->vbt.lvds_ssc_freq;
 		}
 
-		if (intel_is_dual_link_lvds(dev)) {
+		if (intel_is_dual_link_lvds(dev_priv)) {
 			if (refclk == 100000)
 				limit = &intel_limits_ironlake_dual_lvds_100m;
 			else
@@ -8920,7 +9037,7 @@ static int ironlake_crtc_compute_clock(struct intel_crtc *crtc,
 
 	ironlake_compute_dpll(crtc, crtc_state, NULL);
 
-	if (!intel_get_shared_dpll(crtc, crtc_state, NULL)) {
+	if (!intel_get_shared_dpll(crtc_state, NULL)) {
 		DRM_DEBUG_KMS("failed to find PLL for pipe %c\n",
 			      pipe_name(crtc->pipe));
 		return -EINVAL;
@@ -9226,6 +9343,13 @@ static bool ironlake_get_pipe_config(struct intel_crtc *crtc,
 	if (tmp & PIPECONF_COLOR_RANGE_SELECT)
 		pipe_config->limited_color_range = true;
 
+	pipe_config->gamma_mode = (tmp & PIPECONF_GAMMA_MODE_MASK_ILK) >>
+		PIPECONF_GAMMA_MODE_SHIFT;
+
+	pipe_config->csc_mode = I915_READ(PIPE_CSC_MODE(crtc->pipe));
+
+	i9xx_get_pipe_color_config(pipe_config);
+
 	if (I915_READ(PCH_TRANSCONF(crtc->pipe)) & TRANS_ENABLE) {
 		struct intel_shared_dpll *pll;
 		enum intel_dpll_id pll_id;
@@ -9371,7 +9495,8 @@ static void hsw_disable_lcpll(struct drm_i915_private *dev_priv,
 	I915_WRITE(LCPLL_CTL, val);
 	POSTING_READ(LCPLL_CTL);
 
-	if (intel_wait_for_register(dev_priv, LCPLL_CTL, LCPLL_PLL_LOCK, 0, 1))
+	if (intel_wait_for_register(&dev_priv->uncore,
+				    LCPLL_CTL, LCPLL_PLL_LOCK, 0, 1))
 		DRM_ERROR("LCPLL still locked\n");
 
 	val = hsw_read_dcomp(dev_priv);
@@ -9409,7 +9534,7 @@ static void hsw_restore_lcpll(struct drm_i915_private *dev_priv)
 	 * Make sure we're not on PC8 state before disabling PC8, otherwise
 	 * we'll hang the machine. To prevent PC8 state, just enable force_wake.
 	 */
-	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
 
 	if (val & LCPLL_POWER_DOWN_ALLOW) {
 		val &= ~LCPLL_POWER_DOWN_ALLOW;
@@ -9426,7 +9551,7 @@ static void hsw_restore_lcpll(struct drm_i915_private *dev_priv)
 	val &= ~LCPLL_PLL_DISABLE;
 	I915_WRITE(LCPLL_CTL, val);
 
-	if (intel_wait_for_register(dev_priv,
+	if (intel_wait_for_register(&dev_priv->uncore,
 				    LCPLL_CTL, LCPLL_PLL_LOCK, LCPLL_PLL_LOCK,
 				    5))
 		DRM_ERROR("LCPLL not locked yet\n");
@@ -9441,7 +9566,7 @@ static void hsw_restore_lcpll(struct drm_i915_private *dev_priv)
 			DRM_ERROR("Switching back to LCPLL failed\n");
 	}
 
-	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
 
 	intel_update_cdclk(dev_priv);
 	intel_dump_cdclk_state(&dev_priv->cdclk.hw, "Current CDCLK");
@@ -9510,11 +9635,11 @@ static int haswell_crtc_compute_clock(struct intel_crtc *crtc,
 		to_intel_atomic_state(crtc_state->base.state);
 
 	if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) ||
-	    IS_ICELAKE(dev_priv)) {
+	    INTEL_GEN(dev_priv) >= 11) {
 		struct intel_encoder *encoder =
 			intel_get_crtc_new_encoder(state, crtc_state);
 
-		if (!intel_get_shared_dpll(crtc, crtc_state, encoder)) {
+		if (!intel_get_shared_dpll(crtc_state, encoder)) {
 			DRM_DEBUG_KMS("failed to find PLL for pipe %c\n",
 				      pipe_name(crtc->pipe));
 			return -EINVAL;
@@ -9552,9 +9677,6 @@ static void icelake_get_ddi_pll(struct drm_i915_private *dev_priv,
 		temp = I915_READ(DPCLKA_CFGCR0_ICL) &
 		       DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port);
 		id = temp >> DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(port);
-
-		if (WARN_ON(!intel_dpll_is_combophy(id)))
-			return;
 	} else if (intel_port_is_tc(dev_priv, port)) {
 		id = icl_tc_port_to_pll_id(intel_port_to_tc(dev_priv, port));
 	} else {
@@ -9643,20 +9765,25 @@ static void haswell_get_ddi_pll(struct drm_i915_private *dev_priv,
 
 static bool hsw_get_transcoder_state(struct intel_crtc *crtc,
 				     struct intel_crtc_state *pipe_config,
-				     u64 *power_domain_mask)
+				     u64 *power_domain_mask,
+				     intel_wakeref_t *wakerefs)
 {
 	struct drm_device *dev = crtc->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	enum intel_display_power_domain power_domain;
-	unsigned long panel_transcoder_mask = BIT(TRANSCODER_EDP);
+	unsigned long panel_transcoder_mask = 0;
 	unsigned long enabled_panel_transcoders = 0;
 	enum transcoder panel_transcoder;
+	intel_wakeref_t wf;
 	u32 tmp;
 
-	if (IS_ICELAKE(dev_priv))
+	if (INTEL_GEN(dev_priv) >= 11)
 		panel_transcoder_mask |=
 			BIT(TRANSCODER_DSI_0) | BIT(TRANSCODER_DSI_1);
 
+	if (HAS_TRANSCODER_EDP(dev_priv))
+		panel_transcoder_mask |= BIT(TRANSCODER_EDP);
+
 	/*
 	 * The pipe->transcoder mapping is fixed with the exception of the eDP
 	 * and DSI transcoders handled below.
@@ -9713,10 +9840,13 @@ static bool hsw_get_transcoder_state(struct intel_crtc *crtc,
 		enabled_panel_transcoders != BIT(TRANSCODER_EDP));
 
 	power_domain = POWER_DOMAIN_TRANSCODER(pipe_config->cpu_transcoder);
-	if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
+	WARN_ON(*power_domain_mask & BIT_ULL(power_domain));
+
+	wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
+	if (!wf)
 		return false;
 
-	WARN_ON(*power_domain_mask & BIT_ULL(power_domain));
+	wakerefs[power_domain] = wf;
 	*power_domain_mask |= BIT_ULL(power_domain);
 
 	tmp = I915_READ(PIPECONF(pipe_config->cpu_transcoder));
@@ -9726,13 +9856,15 @@ static bool hsw_get_transcoder_state(struct intel_crtc *crtc,
 
 static bool bxt_get_dsi_transcoder_state(struct intel_crtc *crtc,
 					 struct intel_crtc_state *pipe_config,
-					 u64 *power_domain_mask)
+					 u64 *power_domain_mask,
+					 intel_wakeref_t *wakerefs)
 {
 	struct drm_device *dev = crtc->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	enum intel_display_power_domain power_domain;
-	enum port port;
 	enum transcoder cpu_transcoder;
+	intel_wakeref_t wf;
+	enum port port;
 	u32 tmp;
 
 	for_each_port_masked(port, BIT(PORT_A) | BIT(PORT_C)) {
@@ -9742,10 +9874,13 @@ static bool bxt_get_dsi_transcoder_state(struct intel_crtc *crtc,
 			cpu_transcoder = TRANSCODER_DSI_C;
 
 		power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
-		if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
+		WARN_ON(*power_domain_mask & BIT_ULL(power_domain));
+
+		wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
+		if (!wf)
 			continue;
 
-		WARN_ON(*power_domain_mask & BIT_ULL(power_domain));
+		wakerefs[power_domain] = wf;
 		*power_domain_mask |= BIT_ULL(power_domain);
 
 		/*
@@ -9786,7 +9921,7 @@ static void haswell_get_ddi_port_state(struct intel_crtc *crtc,
 
 	port = (tmp & TRANS_DDI_PORT_MASK) >> TRANS_DDI_PORT_SHIFT;
 
-	if (IS_ICELAKE(dev_priv))
+	if (INTEL_GEN(dev_priv) >= 11)
 		icelake_get_ddi_pll(dev_priv, port, pipe_config);
 	else if (IS_CANNONLAKE(dev_priv))
 		cannonlake_get_ddi_pll(dev_priv, port, pipe_config);
@@ -9824,6 +9959,7 @@ static bool haswell_get_pipe_config(struct intel_crtc *crtc,
 				    struct intel_crtc_state *pipe_config)
 {
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+	intel_wakeref_t wakerefs[POWER_DOMAIN_NUM], wf;
 	enum intel_display_power_domain power_domain;
 	u64 power_domain_mask;
 	bool active;
@@ -9831,16 +9967,21 @@ static bool haswell_get_pipe_config(struct intel_crtc *crtc,
 	intel_crtc_init_scalers(crtc, pipe_config);
 
 	power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
-	if (!intel_display_power_get_if_enabled(dev_priv, power_domain))
+	wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
+	if (!wf)
 		return false;
+
+	wakerefs[power_domain] = wf;
 	power_domain_mask = BIT_ULL(power_domain);
 
 	pipe_config->shared_dpll = NULL;
 
-	active = hsw_get_transcoder_state(crtc, pipe_config, &power_domain_mask);
+	active = hsw_get_transcoder_state(crtc, pipe_config,
+					  &power_domain_mask, wakerefs);
 
 	if (IS_GEN9_LP(dev_priv) &&
-	    bxt_get_dsi_transcoder_state(crtc, pipe_config, &power_domain_mask)) {
+	    bxt_get_dsi_transcoder_state(crtc, pipe_config,
+					 &power_domain_mask, wakerefs)) {
 		WARN_ON(active);
 		active = true;
 	}
@@ -9849,7 +9990,7 @@ static bool haswell_get_pipe_config(struct intel_crtc *crtc,
 		goto out;
 
 	if (!transcoder_is_dsi(pipe_config->cpu_transcoder) ||
-	    IS_ICELAKE(dev_priv)) {
+	    INTEL_GEN(dev_priv) >= 11) {
 		haswell_get_ddi_port_state(crtc, pipe_config);
 		intel_get_pipe_timings(crtc, pipe_config);
 	}
@@ -9857,12 +9998,28 @@ static bool haswell_get_pipe_config(struct intel_crtc *crtc,
 	intel_get_pipe_src_size(crtc, pipe_config);
 	intel_get_crtc_ycbcr_config(crtc, pipe_config);
 
-	pipe_config->gamma_mode =
-		I915_READ(GAMMA_MODE(crtc->pipe)) & GAMMA_MODE_MODE_MASK;
+	pipe_config->gamma_mode = I915_READ(GAMMA_MODE(crtc->pipe));
+
+	pipe_config->csc_mode = I915_READ(PIPE_CSC_MODE(crtc->pipe));
+
+	if (INTEL_GEN(dev_priv) >= 9) {
+		u32 tmp = I915_READ(SKL_BOTTOM_COLOR(crtc->pipe));
+
+		if (tmp & SKL_BOTTOM_COLOR_GAMMA_ENABLE)
+			pipe_config->gamma_enable = true;
+
+		if (tmp & SKL_BOTTOM_COLOR_CSC_ENABLE)
+			pipe_config->csc_enable = true;
+	} else {
+		i9xx_get_pipe_color_config(pipe_config);
+	}
 
 	power_domain = POWER_DOMAIN_PIPE_PANEL_FITTER(crtc->pipe);
-	if (intel_display_power_get_if_enabled(dev_priv, power_domain)) {
-		WARN_ON(power_domain_mask & BIT_ULL(power_domain));
+	WARN_ON(power_domain_mask & BIT_ULL(power_domain));
+
+	wf = intel_display_power_get_if_enabled(dev_priv, power_domain);
+	if (wf) {
+		wakerefs[power_domain] = wf;
 		power_domain_mask |= BIT_ULL(power_domain);
 
 		if (INTEL_GEN(dev_priv) >= 9)
@@ -9894,7 +10051,8 @@ static bool haswell_get_pipe_config(struct intel_crtc *crtc,
 
 out:
 	for_each_power_domain(power_domain, power_domain_mask)
-		intel_display_power_put_unchecked(dev_priv, power_domain);
+		intel_display_power_put(dev_priv,
+					power_domain, wakerefs[power_domain]);
 
 	return active;
 }
@@ -10030,7 +10188,12 @@ i845_cursor_max_stride(struct intel_plane *plane,
 
 static u32 i845_cursor_ctl_crtc(const struct intel_crtc_state *crtc_state)
 {
-	return CURSOR_GAMMA_ENABLE;
+	u32 cntl = 0;
+
+	if (crtc_state->gamma_enable)
+		cntl |= CURSOR_GAMMA_ENABLE;
+
+	return cntl;
 }
 
 static u32 i845_cursor_ctl(const struct intel_crtc_state *crtc_state,
@@ -10184,9 +10347,10 @@ static u32 i9xx_cursor_ctl_crtc(const struct intel_crtc_state *crtc_state)
 	if (INTEL_GEN(dev_priv) >= 11)
 		return cntl;
 
-	cntl |= MCURSOR_GAMMA_ENABLE;
+	if (crtc_state->gamma_enable)
+		cntl = MCURSOR_GAMMA_ENABLE;
 
-	if (HAS_DDI(dev_priv))
+	if (crtc_state->csc_enable)
 		cntl |= MCURSOR_PIPE_CSC_ENABLE;
 
 	if (INTEL_GEN(dev_priv) < 5 && !IS_G4X(dev_priv))
@@ -11134,7 +11298,7 @@ static int icl_check_nv12_planes(struct intel_crtc_state *crtc_state)
 		}
 
 		if (!linked_state) {
-			DRM_DEBUG_KMS("Need %d free Y planes for NV12\n",
+			DRM_DEBUG_KMS("Need %d free Y planes for planar YUV\n",
 				      hweight8(crtc_state->nv12_planes));
 
 			return -EINVAL;
@@ -11175,16 +11339,11 @@ static int intel_crtc_atomic_check(struct drm_crtc *crtc,
 			return ret;
 	}
 
-	if (mode_changed || crtc_state->color_mgmt_changed) {
+	if (mode_changed || pipe_config->update_pipe ||
+	    crtc_state->color_mgmt_changed) {
 		ret = intel_color_check(pipe_config);
 		if (ret)
 			return ret;
-
-		/*
-		 * Changing color management on Intel hardware is
-		 * handled as part of planes update.
-		 */
-		crtc_state->planes_changed = true;
 	}
 
 	ret = 0;
@@ -11355,6 +11514,16 @@ intel_dump_m_n_config(struct intel_crtc_state *pipe_config, char *id,
 		      m_n->link_m, m_n->link_n, m_n->tu);
 }
 
+static void
+intel_dump_infoframe(struct drm_i915_private *dev_priv,
+		     const union hdmi_infoframe *frame)
+{
+	if ((drm_debug & DRM_UT_KMS) == 0)
+		return;
+
+	hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, frame);
+}
+
 #define OUTPUT_TYPE(x) [INTEL_OUTPUT_ ## x] = #x
 
 static const char * const output_type_str[] = {
@@ -11458,6 +11627,22 @@ static void intel_dump_pipe_config(struct intel_crtc *crtc,
 	DRM_DEBUG_KMS("audio: %i, infoframes: %i\n",
 		      pipe_config->has_audio, pipe_config->has_infoframe);
 
+	DRM_DEBUG_KMS("infoframes enabled: 0x%x\n",
+		      pipe_config->infoframes.enable);
+
+	if (pipe_config->infoframes.enable &
+	    intel_hdmi_infoframe_enable(HDMI_PACKET_TYPE_GENERAL_CONTROL))
+		DRM_DEBUG_KMS("GCP: 0x%x\n", pipe_config->infoframes.gcp);
+	if (pipe_config->infoframes.enable &
+	    intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_AVI))
+		intel_dump_infoframe(dev_priv, &pipe_config->infoframes.avi);
+	if (pipe_config->infoframes.enable &
+	    intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_SPD))
+		intel_dump_infoframe(dev_priv, &pipe_config->infoframes.spd);
+	if (pipe_config->infoframes.enable &
+	    intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_VENDOR))
+		intel_dump_infoframe(dev_priv, &pipe_config->infoframes.hdmi);
+
 	DRM_DEBUG_KMS("requested mode:\n");
 	drm_mode_debug_printmodeline(&pipe_config->base.mode);
 	DRM_DEBUG_KMS("adjusted mode:\n");
@@ -11606,7 +11791,7 @@ clear_intel_crtc_state(struct intel_crtc_state *crtc_state)
 	saved_state->shared_dpll = crtc_state->shared_dpll;
 	saved_state->dpll_hw_state = crtc_state->dpll_hw_state;
 	saved_state->pch_pfit.force_thru = crtc_state->pch_pfit.force_thru;
-	saved_state->ips_force_disable = crtc_state->ips_force_disable;
+	saved_state->crc_enabled = crtc_state->crc_enabled;
 	if (IS_G4X(dev_priv) ||
 	    IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 		saved_state->wm = crtc_state->wm;
@@ -11825,6 +12010,37 @@ intel_compare_link_m_n(const struct intel_link_m_n *m_n,
 	return false;
 }
 
+static bool
+intel_compare_infoframe(const union hdmi_infoframe *a,
+			const union hdmi_infoframe *b)
+{
+	return memcmp(a, b, sizeof(*a)) == 0;
+}
+
+static void
+pipe_config_infoframe_err(struct drm_i915_private *dev_priv,
+			  bool adjust, const char *name,
+			  const union hdmi_infoframe *a,
+			  const union hdmi_infoframe *b)
+{
+	if (adjust) {
+		if ((drm_debug & DRM_UT_KMS) == 0)
+			return;
+
+		drm_dbg(DRM_UT_KMS, "mismatch in %s infoframe", name);
+		drm_dbg(DRM_UT_KMS, "expected:");
+		hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, a);
+		drm_dbg(DRM_UT_KMS, "found");
+		hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, b);
+	} else {
+		drm_err("mismatch in %s infoframe", name);
+		drm_err("expected:");
+		hdmi_infoframe_log(KERN_ERR, dev_priv->drm.dev, a);
+		drm_err("found");
+		hdmi_infoframe_log(KERN_ERR, dev_priv->drm.dev, b);
+	}
+}
+
 static void __printf(3, 4)
 pipe_config_err(bool adjust, const char *name, const char *format, ...)
 {
@@ -12008,7 +12224,17 @@ intel_pipe_config_compare(struct drm_i915_private *dev_priv,
 	} \
 } while (0)
 
-#define PIPE_CONF_QUIRK(quirk)	\
+#define PIPE_CONF_CHECK_INFOFRAME(name) do { \
+	if (!intel_compare_infoframe(&current_config->infoframes.name, \
+				     &pipe_config->infoframes.name)) { \
+		pipe_config_infoframe_err(dev_priv, adjust, __stringify(name), \
+					  &current_config->infoframes.name, \
+					  &pipe_config->infoframes.name); \
+		ret = false; \
+	} \
+} while (0)
+
+#define PIPE_CONF_QUIRK(quirk) \
 	((current_config->quirks | pipe_config->quirks) & (quirk))
 
 	PIPE_CONF_CHECK_I(cpu_transcoder);
@@ -12089,6 +12315,14 @@ intel_pipe_config_compare(struct drm_i915_private *dev_priv,
 
 		PIPE_CONF_CHECK_I(scaler_state.scaler_id);
 		PIPE_CONF_CHECK_CLOCK_FUZZY(pixel_rate);
+
+		PIPE_CONF_CHECK_X(gamma_mode);
+		if (IS_CHERRYVIEW(dev_priv))
+			PIPE_CONF_CHECK_X(cgm_mode);
+		else
+			PIPE_CONF_CHECK_X(csc_mode);
+		PIPE_CONF_CHECK_BOOL(gamma_enable);
+		PIPE_CONF_CHECK_BOOL(csc_enable);
 	}
 
 	PIPE_CONF_CHECK_BOOL(double_wide);
@@ -12137,6 +12371,12 @@ intel_pipe_config_compare(struct drm_i915_private *dev_priv,
 
 	PIPE_CONF_CHECK_I(min_voltage_level);
 
+	PIPE_CONF_CHECK_X(infoframes.enable);
+	PIPE_CONF_CHECK_X(infoframes.gcp);
+	PIPE_CONF_CHECK_INFOFRAME(avi);
+	PIPE_CONF_CHECK_INFOFRAME(spd);
+	PIPE_CONF_CHECK_INFOFRAME(hdmi);
+
 #undef PIPE_CONF_CHECK_X
 #undef PIPE_CONF_CHECK_I
 #undef PIPE_CONF_CHECK_BOOL
@@ -12171,12 +12411,15 @@ static void verify_wm_state(struct drm_crtc *crtc,
 			    struct drm_crtc_state *new_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
-	struct skl_ddb_allocation hw_ddb, *sw_ddb;
-	struct skl_pipe_wm hw_wm, *sw_wm;
-	struct skl_plane_wm *hw_plane_wm, *sw_plane_wm;
+	struct skl_hw_state {
+		struct skl_ddb_entry ddb_y[I915_MAX_PLANES];
+		struct skl_ddb_entry ddb_uv[I915_MAX_PLANES];
+		struct skl_ddb_allocation ddb;
+		struct skl_pipe_wm wm;
+	} *hw;
+	struct skl_ddb_allocation *sw_ddb;
+	struct skl_pipe_wm *sw_wm;
 	struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry;
-	struct skl_ddb_entry hw_ddb_y[I915_MAX_PLANES];
-	struct skl_ddb_entry hw_ddb_uv[I915_MAX_PLANES];
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	const enum pipe pipe = intel_crtc->pipe;
 	int plane, level, max_level = ilk_wm_max_level(dev_priv);
@@ -12184,22 +12427,29 @@ static void verify_wm_state(struct drm_crtc *crtc,
 	if (INTEL_GEN(dev_priv) < 9 || !new_state->active)
 		return;
 
-	skl_pipe_wm_get_hw_state(intel_crtc, &hw_wm);
+	hw = kzalloc(sizeof(*hw), GFP_KERNEL);
+	if (!hw)
+		return;
+
+	skl_pipe_wm_get_hw_state(intel_crtc, &hw->wm);
 	sw_wm = &to_intel_crtc_state(new_state)->wm.skl.optimal;
 
-	skl_pipe_ddb_get_hw_state(intel_crtc, hw_ddb_y, hw_ddb_uv);
+	skl_pipe_ddb_get_hw_state(intel_crtc, hw->ddb_y, hw->ddb_uv);
 
-	skl_ddb_get_hw_state(dev_priv, &hw_ddb);
+	skl_ddb_get_hw_state(dev_priv, &hw->ddb);
 	sw_ddb = &dev_priv->wm.skl_hw.ddb;
 
-	if (INTEL_GEN(dev_priv) >= 11)
-		if (hw_ddb.enabled_slices != sw_ddb->enabled_slices)
-			DRM_ERROR("mismatch in DBUF Slices (expected %u, got %u)\n",
-				  sw_ddb->enabled_slices,
-				  hw_ddb.enabled_slices);
+	if (INTEL_GEN(dev_priv) >= 11 &&
+	    hw->ddb.enabled_slices != sw_ddb->enabled_slices)
+		DRM_ERROR("mismatch in DBUF Slices (expected %u, got %u)\n",
+			  sw_ddb->enabled_slices,
+			  hw->ddb.enabled_slices);
+
 	/* planes */
 	for_each_universal_plane(dev_priv, pipe, plane) {
-		hw_plane_wm = &hw_wm.planes[plane];
+		struct skl_plane_wm *hw_plane_wm, *sw_plane_wm;
+
+		hw_plane_wm = &hw->wm.planes[plane];
 		sw_plane_wm = &sw_wm->planes[plane];
 
 		/* Watermarks */
@@ -12231,7 +12481,7 @@ static void verify_wm_state(struct drm_crtc *crtc,
 		}
 
 		/* DDB */
-		hw_ddb_entry = &hw_ddb_y[plane];
+		hw_ddb_entry = &hw->ddb_y[plane];
 		sw_ddb_entry = &to_intel_crtc_state(new_state)->wm.skl.plane_ddb_y[plane];
 
 		if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
@@ -12249,7 +12499,9 @@ static void verify_wm_state(struct drm_crtc *crtc,
 	 * once the plane becomes visible, we can skip this check
 	 */
 	if (1) {
-		hw_plane_wm = &hw_wm.planes[PLANE_CURSOR];
+		struct skl_plane_wm *hw_plane_wm, *sw_plane_wm;
+
+		hw_plane_wm = &hw->wm.planes[PLANE_CURSOR];
 		sw_plane_wm = &sw_wm->planes[PLANE_CURSOR];
 
 		/* Watermarks */
@@ -12281,7 +12533,7 @@ static void verify_wm_state(struct drm_crtc *crtc,
 		}
 
 		/* DDB */
-		hw_ddb_entry = &hw_ddb_y[PLANE_CURSOR];
+		hw_ddb_entry = &hw->ddb_y[PLANE_CURSOR];
 		sw_ddb_entry = &to_intel_crtc_state(new_state)->wm.skl.plane_ddb_y[PLANE_CURSOR];
 
 		if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
@@ -12291,6 +12543,8 @@ static void verify_wm_state(struct drm_crtc *crtc,
 				  hw_ddb_entry->start, hw_ddb_entry->end);
 		}
 	}
+
+	kfree(hw);
 }
 
 static void
@@ -12447,7 +12701,8 @@ intel_verify_planes(struct intel_atomic_state *state)
 
 	for_each_new_intel_plane_in_state(state, plane,
 					  plane_state, i)
-		assert_plane(plane, plane_state->base.visible);
+		assert_plane(plane, plane_state->slave ||
+			     plane_state->base.visible);
 }
 
 static void
@@ -12769,10 +13024,16 @@ static int intel_modeset_checks(struct drm_atomic_state *state)
 		return -EINVAL;
 	}
 
+	/* keep the current setting */
+	if (!intel_state->cdclk.force_min_cdclk_changed)
+		intel_state->cdclk.force_min_cdclk =
+			dev_priv->cdclk.force_min_cdclk;
+
 	intel_state->modeset = true;
 	intel_state->active_crtcs = dev_priv->active_crtcs;
 	intel_state->cdclk.logical = dev_priv->cdclk.logical;
 	intel_state->cdclk.actual = dev_priv->cdclk.actual;
+	intel_state->cdclk.pipe = INVALID_PIPE;
 
 	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
 		if (new_crtc_state->active)
@@ -12792,6 +13053,8 @@ static int intel_modeset_checks(struct drm_atomic_state *state)
 	 * adjusted_mode bits in the crtc directly.
 	 */
 	if (dev_priv->display.modeset_calc_cdclk) {
+		enum pipe pipe;
+
 		ret = dev_priv->display.modeset_calc_cdclk(state);
 		if (ret < 0)
 			return ret;
@@ -12808,12 +13071,36 @@ static int intel_modeset_checks(struct drm_atomic_state *state)
 				return ret;
 		}
 
+		if (is_power_of_2(intel_state->active_crtcs)) {
+			struct drm_crtc *crtc;
+			struct drm_crtc_state *crtc_state;
+
+			pipe = ilog2(intel_state->active_crtcs);
+			crtc = &intel_get_crtc_for_pipe(dev_priv, pipe)->base;
+			crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
+			if (crtc_state && needs_modeset(crtc_state))
+				pipe = INVALID_PIPE;
+		} else {
+			pipe = INVALID_PIPE;
+		}
+
 		/* All pipes must be switched off while we change the cdclk. */
-		if (intel_cdclk_needs_modeset(&dev_priv->cdclk.actual,
-					      &intel_state->cdclk.actual)) {
+		if (pipe != INVALID_PIPE &&
+		    intel_cdclk_needs_cd2x_update(dev_priv,
+						  &dev_priv->cdclk.actual,
+						  &intel_state->cdclk.actual)) {
+			ret = intel_lock_all_pipes(state);
+			if (ret < 0)
+				return ret;
+
+			intel_state->cdclk.pipe = pipe;
+		} else if (intel_cdclk_needs_modeset(&dev_priv->cdclk.actual,
+						     &intel_state->cdclk.actual)) {
 			ret = intel_modeset_all_pipes(state);
 			if (ret < 0)
 				return ret;
+
+			intel_state->cdclk.pipe = INVALID_PIPE;
 		}
 
 		DRM_DEBUG_KMS("New cdclk calculated to be logical %u kHz, actual %u kHz\n",
@@ -12822,8 +13109,6 @@ static int intel_modeset_checks(struct drm_atomic_state *state)
 		DRM_DEBUG_KMS("New voltage level calculated to be logical %u, actual %u\n",
 			      intel_state->cdclk.logical.voltage_level,
 			      intel_state->cdclk.actual.voltage_level);
-	} else {
-		to_intel_atomic_state(state)->cdclk.logical = dev_priv->cdclk.logical;
 	}
 
 	intel_modeset_clear_plls(state);
@@ -12864,7 +13149,7 @@ static int intel_atomic_check(struct drm_device *dev,
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *old_crtc_state, *crtc_state;
 	int ret, i;
-	bool any_ms = false;
+	bool any_ms = intel_state->cdclk.force_min_cdclk_changed;
 
 	/* Catch I915_MODE_FLAG_INHERITED */
 	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state,
@@ -12989,14 +13274,14 @@ static void intel_update_crtc(struct drm_crtc *crtc,
 	else if (new_plane_state)
 		intel_fbc_enable(intel_crtc, pipe_config, new_plane_state);
 
-	intel_begin_crtc_commit(crtc, old_crtc_state);
+	intel_begin_crtc_commit(to_intel_atomic_state(state), intel_crtc);
 
 	if (INTEL_GEN(dev_priv) >= 9)
 		skl_update_planes_on_crtc(to_intel_atomic_state(state), intel_crtc);
 	else
 		i9xx_update_planes_on_crtc(to_intel_atomic_state(state), intel_crtc);
 
-	intel_finish_crtc_commit(crtc, old_crtc_state);
+	intel_finish_crtc_commit(to_intel_atomic_state(state), intel_crtc);
 }
 
 static void intel_update_crtcs(struct drm_atomic_state *state)
@@ -13224,7 +13509,10 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state)
 	if (intel_state->modeset) {
 		drm_atomic_helper_update_legacy_modeset_state(state->dev, state);
 
-		intel_set_cdclk(dev_priv, &dev_priv->cdclk.actual);
+		intel_set_cdclk_pre_plane_update(dev_priv,
+						 &intel_state->cdclk.actual,
+						 &dev_priv->cdclk.actual,
+						 intel_state->cdclk.pipe);
 
 		/*
 		 * SKL workaround: bspec recommends we disable the SAGV when we
@@ -13253,6 +13541,12 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state)
 	/* Now enable the clocks, plane, pipe, and connectors that we set up. */
 	dev_priv->display.update_crtcs(state);
 
+	if (intel_state->modeset)
+		intel_set_cdclk_post_plane_update(dev_priv,
+						  &intel_state->cdclk.actual,
+						  &dev_priv->cdclk.actual,
+						  intel_state->cdclk.pipe);
+
 	/* FIXME: We should call drm_atomic_helper_commit_hw_done() here
 	 * already, but still need the state for the delayed optimization. To
 	 * fix this:
@@ -13313,7 +13607,7 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state)
 		 * so enable debugging for the next modeset - and hope we catch
 		 * the culprit.
 		 */
-		intel_uncore_arm_unclaimed_mmio_detection(dev_priv);
+		intel_uncore_arm_unclaimed_mmio_detection(&dev_priv->uncore);
 		intel_display_power_put(dev_priv, POWER_DOMAIN_MODESET, wakeref);
 	}
 
@@ -13454,8 +13748,10 @@ static int intel_atomic_commit(struct drm_device *dev,
 		       intel_state->min_voltage_level,
 		       sizeof(intel_state->min_voltage_level));
 		dev_priv->active_crtcs = intel_state->active_crtcs;
-		dev_priv->cdclk.logical = intel_state->cdclk.logical;
-		dev_priv->cdclk.actual = intel_state->cdclk.actual;
+		dev_priv->cdclk.force_min_cdclk =
+			intel_state->cdclk.force_min_cdclk;
+
+		intel_cdclk_swap_state(intel_state);
 	}
 
 	drm_atomic_state_get(state);
@@ -13506,7 +13802,7 @@ static int do_rps_boost(struct wait_queue_entry *_wait,
 	 * vblank without our intervention, so leave RPS alone.
 	 */
 	if (!i915_request_started(rq))
-		gen6_rps_boost(rq, NULL);
+		gen6_rps_boost(rq);
 	i915_request_put(rq);
 
 	drm_crtc_vblank_put(wait->crtc);
@@ -13767,7 +14063,7 @@ skl_max_scale(const struct intel_crtc_state *crtc_state,
 	 *            or
 	 *    cdclk/crtc_clock
 	 */
-	mult = pixel_format == DRM_FORMAT_NV12 ? 2 : 3;
+	mult = is_planar_yuv_format(pixel_format) ? 2 : 3;
 	tmpclk1 = (1 << 16) * mult - 1;
 	tmpclk2 = (1 << 8) * ((max_dotclk << 8) / crtc_clock);
 	max_scale = min(tmpclk1, tmpclk2);
@@ -13775,39 +14071,35 @@ skl_max_scale(const struct intel_crtc_state *crtc_state,
 	return max_scale;
 }
 
-static void intel_begin_crtc_commit(struct drm_crtc *crtc,
-				    struct drm_crtc_state *old_crtc_state)
+static void intel_begin_crtc_commit(struct intel_atomic_state *state,
+				    struct intel_crtc *crtc)
 {
-	struct drm_device *dev = crtc->dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	struct intel_crtc_state *old_intel_cstate =
-		to_intel_crtc_state(old_crtc_state);
-	struct intel_atomic_state *old_intel_state =
-		to_intel_atomic_state(old_crtc_state->state);
-	struct intel_crtc_state *intel_cstate =
-		intel_atomic_get_new_crtc_state(old_intel_state, intel_crtc);
-	bool modeset = needs_modeset(&intel_cstate->base);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+	struct intel_crtc_state *old_crtc_state =
+		intel_atomic_get_old_crtc_state(state, crtc);
+	struct intel_crtc_state *new_crtc_state =
+		intel_atomic_get_new_crtc_state(state, crtc);
+	bool modeset = needs_modeset(&new_crtc_state->base);
 
 	/* Perform vblank evasion around commit operation */
-	intel_pipe_update_start(intel_cstate);
+	intel_pipe_update_start(new_crtc_state);
 
 	if (modeset)
 		goto out;
 
-	if (intel_cstate->base.color_mgmt_changed ||
-	    intel_cstate->update_pipe)
-		intel_color_commit(intel_cstate);
+	if (new_crtc_state->base.color_mgmt_changed ||
+	    new_crtc_state->update_pipe)
+		intel_color_commit(new_crtc_state);
 
-	if (intel_cstate->update_pipe)
-		intel_update_pipe_config(old_intel_cstate, intel_cstate);
+	if (new_crtc_state->update_pipe)
+		intel_update_pipe_config(old_crtc_state, new_crtc_state);
 	else if (INTEL_GEN(dev_priv) >= 9)
-		skl_detach_scalers(intel_cstate);
+		skl_detach_scalers(new_crtc_state);
 
 out:
 	if (dev_priv->display.atomic_update_watermarks)
-		dev_priv->display.atomic_update_watermarks(old_intel_state,
-							   intel_cstate);
+		dev_priv->display.atomic_update_watermarks(state,
+							   new_crtc_state);
 }
 
 void intel_crtc_arm_fifo_underrun(struct intel_crtc *crtc,
@@ -13826,21 +14118,20 @@ void intel_crtc_arm_fifo_underrun(struct intel_crtc *crtc,
 	}
 }
 
-static void intel_finish_crtc_commit(struct drm_crtc *crtc,
-				     struct drm_crtc_state *old_crtc_state)
+static void intel_finish_crtc_commit(struct intel_atomic_state *state,
+				     struct intel_crtc *crtc)
 {
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	struct intel_atomic_state *old_intel_state =
-		to_intel_atomic_state(old_crtc_state->state);
+	struct intel_crtc_state *old_crtc_state =
+		intel_atomic_get_old_crtc_state(state, crtc);
 	struct intel_crtc_state *new_crtc_state =
-		intel_atomic_get_new_crtc_state(old_intel_state, intel_crtc);
+		intel_atomic_get_new_crtc_state(state, crtc);
 
 	intel_pipe_update_end(new_crtc_state);
 
 	if (new_crtc_state->update_pipe &&
 	    !needs_modeset(&new_crtc_state->base) &&
-	    old_crtc_state->mode.private_flags & I915_MODE_FLAG_INHERITED)
-		intel_crtc_arm_fifo_underrun(intel_crtc, new_crtc_state);
+	    old_crtc_state->base.mode.private_flags & I915_MODE_FLAG_INHERITED)
+		intel_crtc_arm_fifo_underrun(crtc, new_crtc_state);
 }
 
 /**
@@ -14039,14 +14330,11 @@ intel_legacy_cursor_update(struct drm_plane *plane,
 	 */
 	crtc_state->active_planes = new_crtc_state->active_planes;
 
-	if (plane->state->visible) {
-		trace_intel_update_plane(plane, to_intel_crtc(crtc));
-		intel_plane->update_plane(intel_plane, crtc_state,
-					  to_intel_plane_state(plane->state));
-	} else {
-		trace_intel_disable_plane(plane, to_intel_crtc(crtc));
-		intel_plane->disable_plane(intel_plane, crtc_state);
-	}
+	if (plane->state->visible)
+		intel_update_plane(intel_plane, crtc_state,
+				   to_intel_plane_state(plane->state));
+	else
+		intel_disable_plane(intel_plane, crtc_state);
 
 	intel_plane_unpin_fb(to_intel_plane_state(old_plane_state));
 
@@ -14496,7 +14784,12 @@ static void intel_setup_outputs(struct drm_i915_private *dev_priv)
 	if (!HAS_DISPLAY(dev_priv))
 		return;
 
-	if (IS_ICELAKE(dev_priv)) {
+	if (IS_ELKHARTLAKE(dev_priv)) {
+		intel_ddi_init(dev_priv, PORT_A);
+		intel_ddi_init(dev_priv, PORT_B);
+		intel_ddi_init(dev_priv, PORT_C);
+		icl_dsi_init(dev_priv);
+	} else if (INTEL_GEN(dev_priv) >= 11) {
 		intel_ddi_init(dev_priv, PORT_A);
 		intel_ddi_init(dev_priv, PORT_B);
 		intel_ddi_init(dev_priv, PORT_C);
@@ -15397,6 +15690,8 @@ int intel_modeset_init(struct drm_device *dev)
 	intel_update_czclk(dev_priv);
 	intel_modeset_init_hw(dev);
 
+	intel_hdcp_component_init(dev_priv);
+
 	if (dev_priv->max_cdclk_freq == 0)
 		intel_update_max_cdclk(dev_priv);
 
@@ -15472,7 +15767,7 @@ void i830_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
 		      pipe_name(pipe), clock.vco, clock.dot);
 
 	fp = i9xx_dpll_compute_fp(&clock);
-	dpll = (I915_READ(DPLL(pipe)) & DPLL_DVO_2X_MODE) |
+	dpll = DPLL_DVO_2X_MODE |
 		DPLL_VGA_MODE_DIS |
 		((clock.p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT) |
 		PLL_P2_DIVIDE_BY_4 |
@@ -16254,6 +16549,8 @@ void intel_modeset_cleanup(struct drm_device *dev)
 	/* flush any delayed tasks or pending work */
 	flush_scheduled_work();
 
+	intel_hdcp_component_fini(dev_priv);
+
 	drm_mode_config_cleanup(dev);
 
 	intel_overlay_cleanup(dev_priv);
@@ -16300,8 +16597,6 @@ struct intel_display_error_state {
 
 	u32 power_well_driver;
 
-	int num_transcoders;
-
 	struct intel_cursor_error_state {
 		u32 control;
 		u32 position;
@@ -16326,6 +16621,7 @@ struct intel_display_error_state {
 	} plane[I915_MAX_PIPES];
 
 	struct intel_transcoder_error_state {
+		bool available;
 		bool power_domain_on;
 		enum transcoder cpu_transcoder;
 
@@ -16352,6 +16648,8 @@ intel_display_capture_error_state(struct drm_i915_private *dev_priv)
 	};
 	int i;
 
+	BUILD_BUG_ON(ARRAY_SIZE(transcoders) != ARRAY_SIZE(error->transcoder));
+
 	if (!HAS_DISPLAY(dev_priv))
 		return NULL;
 
@@ -16392,14 +16690,13 @@ intel_display_capture_error_state(struct drm_i915_private *dev_priv)
 			error->pipe[i].stat = I915_READ(PIPESTAT(i));
 	}
 
-	/* Note: this does not include DSI transcoders. */
-	error->num_transcoders = INTEL_INFO(dev_priv)->num_pipes;
-	if (HAS_DDI(dev_priv))
-		error->num_transcoders++; /* Account for eDP. */
-
-	for (i = 0; i < error->num_transcoders; i++) {
+	for (i = 0; i < ARRAY_SIZE(error->transcoder); i++) {
 		enum transcoder cpu_transcoder = transcoders[i];
 
+		if (!INTEL_INFO(dev_priv)->trans_offsets[cpu_transcoder])
+			continue;
+
+		error->transcoder[i].available = true;
 		error->transcoder[i].power_domain_on =
 			__intel_display_power_is_enabled(dev_priv,
 				POWER_DOMAIN_TRANSCODER(cpu_transcoder));
@@ -16463,7 +16760,10 @@ intel_display_print_error_state(struct drm_i915_error_state_buf *m,
 		err_printf(m, "  BASE: %08x\n", error->cursor[i].base);
 	}
 
-	for (i = 0; i < error->num_transcoders; i++) {
+	for (i = 0; i < ARRAY_SIZE(error->transcoder); i++) {
+		if (!error->transcoder[i].available)
+			continue;
+
 		err_printf(m, "CPU transcoder: %s\n",
 			   transcoder_name(error->transcoder[i].cpu_transcoder));
 		err_printf(m, "  Power: %s\n",
diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c
index 48da4a969a0a..560274d1c50b 100644
--- a/drivers/gpu/drm/i915/intel_dp.c
+++ b/drivers/gpu/drm/i915/intel_dp.c
@@ -25,22 +25,34 @@
  *
  */
 
-#include <linux/i2c.h>
-#include <linux/slab.h>
 #include <linux/export.h>
-#include <linux/types.h>
+#include <linux/i2c.h>
 #include <linux/notifier.h>
 #include <linux/reboot.h>
+#include <linux/slab.h>
+#include <linux/types.h>
 #include <asm/byteorder.h>
+
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_dp_helper.h>
 #include <drm/drm_edid.h>
 #include <drm/drm_hdcp.h>
 #include <drm/drm_probe_helper.h>
-#include "intel_drv.h"
 #include <drm/i915_drm.h>
+
 #include "i915_drv.h"
+#include "intel_audio.h"
+#include "intel_connector.h"
+#include "intel_ddi.h"
+#include "intel_dp.h"
+#include "intel_drv.h"
+#include "intel_hdcp.h"
+#include "intel_hdmi.h"
+#include "intel_lspcon.h"
+#include "intel_lvds.h"
+#include "intel_panel.h"
+#include "intel_psr.h"
 
 #define DP_DPRX_ESI_LEN 14
 
@@ -949,8 +961,11 @@ static void intel_pps_get_registers(struct intel_dp *intel_dp,
 	regs->pp_stat = PP_STATUS(pps_idx);
 	regs->pp_on = PP_ON_DELAYS(pps_idx);
 	regs->pp_off = PP_OFF_DELAYS(pps_idx);
-	if (!IS_GEN9_LP(dev_priv) && !HAS_PCH_CNP(dev_priv) &&
-	    !HAS_PCH_ICP(dev_priv))
+
+	/* Cycle delay moved from PP_DIVISOR to PP_CONTROL */
+	if (IS_GEN9_LP(dev_priv) || INTEL_PCH_TYPE(dev_priv) >= PCH_CNP)
+		regs->pp_div = INVALID_MMIO_REG;
+	else
 		regs->pp_div = PP_DIVISOR(pps_idx);
 }
 
@@ -1720,12 +1735,6 @@ void intel_dp_compute_rate(struct intel_dp *intel_dp, int port_clock,
 	}
 }
 
-struct link_config_limits {
-	int min_clock, max_clock;
-	int min_lane_count, max_lane_count;
-	int min_bpp, max_bpp;
-};
-
 static bool intel_dp_source_supports_fec(struct intel_dp *intel_dp,
 					 const struct intel_crtc_state *pipe_config)
 {
@@ -1788,7 +1797,7 @@ static int intel_dp_compute_bpp(struct intel_dp *intel_dp,
 }
 
 /* Adjust link config limits based on compliance test requests. */
-static void
+void
 intel_dp_adjust_compliance_config(struct intel_dp *intel_dp,
 				  struct intel_crtc_state *pipe_config,
 				  struct link_config_limits *limits)
@@ -1972,6 +1981,14 @@ static int intel_dp_dsc_compute_config(struct intel_dp *intel_dp,
 	return 0;
 }
 
+int intel_dp_min_bpp(const struct intel_crtc_state *crtc_state)
+{
+	if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_RGB)
+		return 6 * 3;
+	else
+		return 8 * 3;
+}
+
 static int
 intel_dp_compute_link_config(struct intel_encoder *encoder,
 			     struct intel_crtc_state *pipe_config,
@@ -1995,7 +2012,7 @@ intel_dp_compute_link_config(struct intel_encoder *encoder,
 	limits.min_lane_count = 1;
 	limits.max_lane_count = intel_dp_max_lane_count(intel_dp);
 
-	limits.min_bpp = 6 * 3;
+	limits.min_bpp = intel_dp_min_bpp(pipe_config);
 	limits.max_bpp = intel_dp_compute_bpp(intel_dp, pipe_config);
 
 	if (intel_dp_is_edp(intel_dp)) {
@@ -2058,6 +2075,29 @@ intel_dp_compute_link_config(struct intel_encoder *encoder,
 	return 0;
 }
 
+bool intel_dp_limited_color_range(const struct intel_crtc_state *crtc_state,
+				  const struct drm_connector_state *conn_state)
+{
+	const struct intel_digital_connector_state *intel_conn_state =
+		to_intel_digital_connector_state(conn_state);
+	const struct drm_display_mode *adjusted_mode =
+		&crtc_state->base.adjusted_mode;
+
+	if (intel_conn_state->broadcast_rgb == INTEL_BROADCAST_RGB_AUTO) {
+		/*
+		 * See:
+		 * CEA-861-E - 5.1 Default Encoding Parameters
+		 * VESA DisplayPort Ver.1.2a - 5.1.1.1 Video Colorimetry
+		 */
+		return crtc_state->pipe_bpp != 18 &&
+			drm_default_rgb_quant_range(adjusted_mode) ==
+			HDMI_QUANTIZATION_RANGE_LIMITED;
+	} else {
+		return intel_conn_state->broadcast_rgb ==
+			INTEL_BROADCAST_RGB_LIMITED;
+	}
+}
+
 int
 intel_dp_compute_config(struct intel_encoder *encoder,
 			struct intel_crtc_state *pipe_config,
@@ -2074,7 +2114,7 @@ intel_dp_compute_config(struct intel_encoder *encoder,
 		to_intel_digital_connector_state(conn_state);
 	bool constant_n = drm_dp_has_quirk(&intel_dp->desc,
 					   DP_DPCD_QUIRK_CONSTANT_N);
-	int ret;
+	int ret, output_bpp;
 
 	if (HAS_PCH_SPLIT(dev_priv) && !HAS_DDI(dev_priv) && port != PORT_A)
 		pipe_config->has_pch_encoder = true;
@@ -2123,40 +2163,25 @@ intel_dp_compute_config(struct intel_encoder *encoder,
 	if (ret < 0)
 		return ret;
 
-	if (intel_conn_state->broadcast_rgb == INTEL_BROADCAST_RGB_AUTO) {
-		/*
-		 * See:
-		 * CEA-861-E - 5.1 Default Encoding Parameters
-		 * VESA DisplayPort Ver.1.2a - 5.1.1.1 Video Colorimetry
-		 */
-		pipe_config->limited_color_range =
-			pipe_config->pipe_bpp != 18 &&
-			drm_default_rgb_quant_range(adjusted_mode) ==
-			HDMI_QUANTIZATION_RANGE_LIMITED;
-	} else {
-		pipe_config->limited_color_range =
-			intel_conn_state->broadcast_rgb == INTEL_BROADCAST_RGB_LIMITED;
-	}
+	pipe_config->limited_color_range =
+		intel_dp_limited_color_range(pipe_config, conn_state);
 
-	if (!pipe_config->dsc_params.compression_enable)
-		intel_link_compute_m_n(pipe_config->pipe_bpp,
-				       pipe_config->lane_count,
-				       adjusted_mode->crtc_clock,
-				       pipe_config->port_clock,
-				       &pipe_config->dp_m_n,
-				       constant_n);
+	if (pipe_config->dsc_params.compression_enable)
+		output_bpp = pipe_config->dsc_params.compressed_bpp;
 	else
-		intel_link_compute_m_n(pipe_config->dsc_params.compressed_bpp,
-				       pipe_config->lane_count,
-				       adjusted_mode->crtc_clock,
-				       pipe_config->port_clock,
-				       &pipe_config->dp_m_n,
-				       constant_n);
+		output_bpp = pipe_config->pipe_bpp;
+
+	intel_link_compute_m_n(output_bpp,
+			       pipe_config->lane_count,
+			       adjusted_mode->crtc_clock,
+			       pipe_config->port_clock,
+			       &pipe_config->dp_m_n,
+			       constant_n);
 
 	if (intel_connector->panel.downclock_mode != NULL &&
 		dev_priv->drrs.type == SEAMLESS_DRRS_SUPPORT) {
 			pipe_config->has_drrs = true;
-			intel_link_compute_m_n(pipe_config->pipe_bpp,
+			intel_link_compute_m_n(output_bpp,
 					       pipe_config->lane_count,
 					       intel_connector->panel.downclock_mode->clock,
 					       pipe_config->port_clock,
@@ -2296,7 +2321,7 @@ static void wait_panel_status(struct intel_dp *intel_dp,
 			I915_READ(pp_stat_reg),
 			I915_READ(pp_ctrl_reg));
 
-	if (intel_wait_for_register(dev_priv,
+	if (intel_wait_for_register(&dev_priv->uncore,
 				    pp_stat_reg, mask, value,
 				    5000))
 		DRM_ERROR("Panel status timeout: status %08x control %08x\n",
@@ -3885,7 +3910,7 @@ void intel_dp_set_idle_link_train(struct intel_dp *intel_dp)
 	if (port == PORT_A)
 		return;
 
-	if (intel_wait_for_register(dev_priv,DP_TP_STATUS(port),
+	if (intel_wait_for_register(&dev_priv->uncore, DP_TP_STATUS(port),
 				    DP_TP_STATUS_IDLE_DONE,
 				    DP_TP_STATUS_IDLE_DONE,
 				    1))
@@ -4731,7 +4756,7 @@ static void intel_dp_check_service_irq(struct intel_dp *intel_dp)
 		intel_dp_handle_test_request(intel_dp);
 
 	if (val & DP_CP_IRQ)
-		intel_hdcp_check_link(intel_dp->attached_connector);
+		intel_hdcp_handle_cp_irq(intel_dp->attached_connector);
 
 	if (val & DP_SINK_SPECIFIC_IRQ)
 		DRM_DEBUG_DRIVER("Sink specific irq unhandled\n");
@@ -5574,6 +5599,18 @@ void intel_dp_encoder_suspend(struct intel_encoder *intel_encoder)
 		edp_panel_vdd_off_sync(intel_dp);
 }
 
+static void intel_dp_hdcp_wait_for_cp_irq(struct intel_hdcp *hdcp, int timeout)
+{
+	long ret;
+
+#define C (hdcp->cp_irq_count_cached != atomic_read(&hdcp->cp_irq_count))
+	ret = wait_event_interruptible_timeout(hdcp->cp_irq_queue, C,
+					       msecs_to_jiffies(timeout));
+
+	if (!ret)
+		DRM_DEBUG_KMS("Timedout at waiting for CP_IRQ\n");
+}
+
 static
 int intel_dp_hdcp_write_an_aksv(struct intel_digital_port *intel_dig_port,
 				u8 *an)
@@ -5798,6 +5835,336 @@ int intel_dp_hdcp_capable(struct intel_digital_port *intel_dig_port,
 	return 0;
 }
 
+struct hdcp2_dp_errata_stream_type {
+	u8	msg_id;
+	u8	stream_type;
+} __packed;
+
+static struct hdcp2_dp_msg_data {
+	u8 msg_id;
+	u32 offset;
+	bool msg_detectable;
+	u32 timeout;
+	u32 timeout2; /* Added for non_paired situation */
+	} hdcp2_msg_data[] = {
+		{HDCP_2_2_AKE_INIT, DP_HDCP_2_2_AKE_INIT_OFFSET, false, 0, 0},
+		{HDCP_2_2_AKE_SEND_CERT, DP_HDCP_2_2_AKE_SEND_CERT_OFFSET,
+				false, HDCP_2_2_CERT_TIMEOUT_MS, 0},
+		{HDCP_2_2_AKE_NO_STORED_KM, DP_HDCP_2_2_AKE_NO_STORED_KM_OFFSET,
+				false, 0, 0},
+		{HDCP_2_2_AKE_STORED_KM, DP_HDCP_2_2_AKE_STORED_KM_OFFSET,
+				false, 0, 0},
+		{HDCP_2_2_AKE_SEND_HPRIME, DP_HDCP_2_2_AKE_SEND_HPRIME_OFFSET,
+				true, HDCP_2_2_HPRIME_PAIRED_TIMEOUT_MS,
+				HDCP_2_2_HPRIME_NO_PAIRED_TIMEOUT_MS},
+		{HDCP_2_2_AKE_SEND_PAIRING_INFO,
+				DP_HDCP_2_2_AKE_SEND_PAIRING_INFO_OFFSET, true,
+				HDCP_2_2_PAIRING_TIMEOUT_MS, 0},
+		{HDCP_2_2_LC_INIT, DP_HDCP_2_2_LC_INIT_OFFSET, false, 0, 0},
+		{HDCP_2_2_LC_SEND_LPRIME, DP_HDCP_2_2_LC_SEND_LPRIME_OFFSET,
+				false, HDCP_2_2_DP_LPRIME_TIMEOUT_MS, 0},
+		{HDCP_2_2_SKE_SEND_EKS, DP_HDCP_2_2_SKE_SEND_EKS_OFFSET, false,
+				0, 0},
+		{HDCP_2_2_REP_SEND_RECVID_LIST,
+				DP_HDCP_2_2_REP_SEND_RECVID_LIST_OFFSET, true,
+				HDCP_2_2_RECVID_LIST_TIMEOUT_MS, 0},
+		{HDCP_2_2_REP_SEND_ACK, DP_HDCP_2_2_REP_SEND_ACK_OFFSET, false,
+				0, 0},
+		{HDCP_2_2_REP_STREAM_MANAGE,
+				DP_HDCP_2_2_REP_STREAM_MANAGE_OFFSET, false,
+				0, 0},
+		{HDCP_2_2_REP_STREAM_READY, DP_HDCP_2_2_REP_STREAM_READY_OFFSET,
+				false, HDCP_2_2_STREAM_READY_TIMEOUT_MS, 0},
+/* local define to shovel this through the write_2_2 interface */
+#define HDCP_2_2_ERRATA_DP_STREAM_TYPE	50
+		{HDCP_2_2_ERRATA_DP_STREAM_TYPE,
+				DP_HDCP_2_2_REG_STREAM_TYPE_OFFSET, false,
+				0, 0},
+		};
+
+static inline
+int intel_dp_hdcp2_read_rx_status(struct intel_digital_port *intel_dig_port,
+				  u8 *rx_status)
+{
+	ssize_t ret;
+
+	ret = drm_dp_dpcd_read(&intel_dig_port->dp.aux,
+			       DP_HDCP_2_2_REG_RXSTATUS_OFFSET, rx_status,
+			       HDCP_2_2_DP_RXSTATUS_LEN);
+	if (ret != HDCP_2_2_DP_RXSTATUS_LEN) {
+		DRM_DEBUG_KMS("Read bstatus from DP/AUX failed (%zd)\n", ret);
+		return ret >= 0 ? -EIO : ret;
+	}
+
+	return 0;
+}
+
+static
+int hdcp2_detect_msg_availability(struct intel_digital_port *intel_dig_port,
+				  u8 msg_id, bool *msg_ready)
+{
+	u8 rx_status;
+	int ret;
+
+	*msg_ready = false;
+	ret = intel_dp_hdcp2_read_rx_status(intel_dig_port, &rx_status);
+	if (ret < 0)
+		return ret;
+
+	switch (msg_id) {
+	case HDCP_2_2_AKE_SEND_HPRIME:
+		if (HDCP_2_2_DP_RXSTATUS_H_PRIME(rx_status))
+			*msg_ready = true;
+		break;
+	case HDCP_2_2_AKE_SEND_PAIRING_INFO:
+		if (HDCP_2_2_DP_RXSTATUS_PAIRING(rx_status))
+			*msg_ready = true;
+		break;
+	case HDCP_2_2_REP_SEND_RECVID_LIST:
+		if (HDCP_2_2_DP_RXSTATUS_READY(rx_status))
+			*msg_ready = true;
+		break;
+	default:
+		DRM_ERROR("Unidentified msg_id: %d\n", msg_id);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static ssize_t
+intel_dp_hdcp2_wait_for_msg(struct intel_digital_port *intel_dig_port,
+			    struct hdcp2_dp_msg_data *hdcp2_msg_data)
+{
+	struct intel_dp *dp = &intel_dig_port->dp;
+	struct intel_hdcp *hdcp = &dp->attached_connector->hdcp;
+	u8 msg_id = hdcp2_msg_data->msg_id;
+	int ret, timeout;
+	bool msg_ready = false;
+
+	if (msg_id == HDCP_2_2_AKE_SEND_HPRIME && !hdcp->is_paired)
+		timeout = hdcp2_msg_data->timeout2;
+	else
+		timeout = hdcp2_msg_data->timeout;
+
+	/*
+	 * There is no way to detect the CERT, LPRIME and STREAM_READY
+	 * availability. So Wait for timeout and read the msg.
+	 */
+	if (!hdcp2_msg_data->msg_detectable) {
+		mdelay(timeout);
+		ret = 0;
+	} else {
+		/*
+		 * As we want to check the msg availability at timeout, Ignoring
+		 * the timeout at wait for CP_IRQ.
+		 */
+		intel_dp_hdcp_wait_for_cp_irq(hdcp, timeout);
+		ret = hdcp2_detect_msg_availability(intel_dig_port,
+						    msg_id, &msg_ready);
+		if (!msg_ready)
+			ret = -ETIMEDOUT;
+	}
+
+	if (ret)
+		DRM_DEBUG_KMS("msg_id %d, ret %d, timeout(mSec): %d\n",
+			      hdcp2_msg_data->msg_id, ret, timeout);
+
+	return ret;
+}
+
+static struct hdcp2_dp_msg_data *get_hdcp2_dp_msg_data(u8 msg_id)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(hdcp2_msg_data); i++)
+		if (hdcp2_msg_data[i].msg_id == msg_id)
+			return &hdcp2_msg_data[i];
+
+	return NULL;
+}
+
+static
+int intel_dp_hdcp2_write_msg(struct intel_digital_port *intel_dig_port,
+			     void *buf, size_t size)
+{
+	struct intel_dp *dp = &intel_dig_port->dp;
+	struct intel_hdcp *hdcp = &dp->attached_connector->hdcp;
+	unsigned int offset;
+	u8 *byte = buf;
+	ssize_t ret, bytes_to_write, len;
+	struct hdcp2_dp_msg_data *hdcp2_msg_data;
+
+	hdcp2_msg_data = get_hdcp2_dp_msg_data(*byte);
+	if (!hdcp2_msg_data)
+		return -EINVAL;
+
+	offset = hdcp2_msg_data->offset;
+
+	/* No msg_id in DP HDCP2.2 msgs */
+	bytes_to_write = size - 1;
+	byte++;
+
+	hdcp->cp_irq_count_cached = atomic_read(&hdcp->cp_irq_count);
+
+	while (bytes_to_write) {
+		len = bytes_to_write > DP_AUX_MAX_PAYLOAD_BYTES ?
+				DP_AUX_MAX_PAYLOAD_BYTES : bytes_to_write;
+
+		ret = drm_dp_dpcd_write(&intel_dig_port->dp.aux,
+					offset, (void *)byte, len);
+		if (ret < 0)
+			return ret;
+
+		bytes_to_write -= ret;
+		byte += ret;
+		offset += ret;
+	}
+
+	return size;
+}
+
+static
+ssize_t get_receiver_id_list_size(struct intel_digital_port *intel_dig_port)
+{
+	u8 rx_info[HDCP_2_2_RXINFO_LEN];
+	u32 dev_cnt;
+	ssize_t ret;
+
+	ret = drm_dp_dpcd_read(&intel_dig_port->dp.aux,
+			       DP_HDCP_2_2_REG_RXINFO_OFFSET,
+			       (void *)rx_info, HDCP_2_2_RXINFO_LEN);
+	if (ret != HDCP_2_2_RXINFO_LEN)
+		return ret >= 0 ? -EIO : ret;
+
+	dev_cnt = (HDCP_2_2_DEV_COUNT_HI(rx_info[0]) << 4 |
+		   HDCP_2_2_DEV_COUNT_LO(rx_info[1]));
+
+	if (dev_cnt > HDCP_2_2_MAX_DEVICE_COUNT)
+		dev_cnt = HDCP_2_2_MAX_DEVICE_COUNT;
+
+	ret = sizeof(struct hdcp2_rep_send_receiverid_list) -
+		HDCP_2_2_RECEIVER_IDS_MAX_LEN +
+		(dev_cnt * HDCP_2_2_RECEIVER_ID_LEN);
+
+	return ret;
+}
+
+static
+int intel_dp_hdcp2_read_msg(struct intel_digital_port *intel_dig_port,
+			    u8 msg_id, void *buf, size_t size)
+{
+	unsigned int offset;
+	u8 *byte = buf;
+	ssize_t ret, bytes_to_recv, len;
+	struct hdcp2_dp_msg_data *hdcp2_msg_data;
+
+	hdcp2_msg_data = get_hdcp2_dp_msg_data(msg_id);
+	if (!hdcp2_msg_data)
+		return -EINVAL;
+	offset = hdcp2_msg_data->offset;
+
+	ret = intel_dp_hdcp2_wait_for_msg(intel_dig_port, hdcp2_msg_data);
+	if (ret < 0)
+		return ret;
+
+	if (msg_id == HDCP_2_2_REP_SEND_RECVID_LIST) {
+		ret = get_receiver_id_list_size(intel_dig_port);
+		if (ret < 0)
+			return ret;
+
+		size = ret;
+	}
+	bytes_to_recv = size - 1;
+
+	/* DP adaptation msgs has no msg_id */
+	byte++;
+
+	while (bytes_to_recv) {
+		len = bytes_to_recv > DP_AUX_MAX_PAYLOAD_BYTES ?
+		      DP_AUX_MAX_PAYLOAD_BYTES : bytes_to_recv;
+
+		ret = drm_dp_dpcd_read(&intel_dig_port->dp.aux, offset,
+				       (void *)byte, len);
+		if (ret < 0) {
+			DRM_DEBUG_KMS("msg_id %d, ret %zd\n", msg_id, ret);
+			return ret;
+		}
+
+		bytes_to_recv -= ret;
+		byte += ret;
+		offset += ret;
+	}
+	byte = buf;
+	*byte = msg_id;
+
+	return size;
+}
+
+static
+int intel_dp_hdcp2_config_stream_type(struct intel_digital_port *intel_dig_port,
+				      bool is_repeater, u8 content_type)
+{
+	struct hdcp2_dp_errata_stream_type stream_type_msg;
+
+	if (is_repeater)
+		return 0;
+
+	/*
+	 * Errata for DP: As Stream type is used for encryption, Receiver
+	 * should be communicated with stream type for the decryption of the
+	 * content.
+	 * Repeater will be communicated with stream type as a part of it's
+	 * auth later in time.
+	 */
+	stream_type_msg.msg_id = HDCP_2_2_ERRATA_DP_STREAM_TYPE;
+	stream_type_msg.stream_type = content_type;
+
+	return intel_dp_hdcp2_write_msg(intel_dig_port, &stream_type_msg,
+					sizeof(stream_type_msg));
+}
+
+static
+int intel_dp_hdcp2_check_link(struct intel_digital_port *intel_dig_port)
+{
+	u8 rx_status;
+	int ret;
+
+	ret = intel_dp_hdcp2_read_rx_status(intel_dig_port, &rx_status);
+	if (ret)
+		return ret;
+
+	if (HDCP_2_2_DP_RXSTATUS_REAUTH_REQ(rx_status))
+		ret = HDCP_REAUTH_REQUEST;
+	else if (HDCP_2_2_DP_RXSTATUS_LINK_FAILED(rx_status))
+		ret = HDCP_LINK_INTEGRITY_FAILURE;
+	else if (HDCP_2_2_DP_RXSTATUS_READY(rx_status))
+		ret = HDCP_TOPOLOGY_CHANGE;
+
+	return ret;
+}
+
+static
+int intel_dp_hdcp2_capable(struct intel_digital_port *intel_dig_port,
+			   bool *capable)
+{
+	u8 rx_caps[3];
+	int ret;
+
+	*capable = false;
+	ret = drm_dp_dpcd_read(&intel_dig_port->dp.aux,
+			       DP_HDCP_2_2_REG_RX_CAPS_OFFSET,
+			       rx_caps, HDCP_2_2_RXCAPS_LEN);
+	if (ret != HDCP_2_2_RXCAPS_LEN)
+		return ret >= 0 ? -EIO : ret;
+
+	if (rx_caps[0] == HDCP_2_2_RX_CAPS_VERSION_VAL &&
+	    HDCP_2_2_DP_HDCP_CAPABLE(rx_caps[2]))
+		*capable = true;
+
+	return 0;
+}
+
 static const struct intel_hdcp_shim intel_dp_hdcp_shim = {
 	.write_an_aksv = intel_dp_hdcp_write_an_aksv,
 	.read_bksv = intel_dp_hdcp_read_bksv,
@@ -5810,6 +6177,12 @@ static const struct intel_hdcp_shim intel_dp_hdcp_shim = {
 	.toggle_signalling = intel_dp_hdcp_toggle_signalling,
 	.check_link = intel_dp_hdcp_check_link,
 	.hdcp_capable = intel_dp_hdcp_capable,
+	.write_2_2_msg = intel_dp_hdcp2_write_msg,
+	.read_2_2_msg = intel_dp_hdcp2_read_msg,
+	.config_stream_type = intel_dp_hdcp2_config_stream_type,
+	.check_2_2_link = intel_dp_hdcp2_check_link,
+	.hdcp_2_2_capable = intel_dp_hdcp2_capable,
+	.protocol = HDCP_PROTOCOL_DP,
 };
 
 static void intel_edp_panel_vdd_sanitize(struct intel_dp *intel_dp)
@@ -6023,43 +6396,34 @@ static void
 intel_pps_readout_hw_state(struct intel_dp *intel_dp, struct edp_power_seq *seq)
 {
 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
-	u32 pp_on, pp_off, pp_div = 0, pp_ctl = 0;
+	u32 pp_on, pp_off, pp_ctl;
 	struct pps_registers regs;
 
 	intel_pps_get_registers(intel_dp, &regs);
 
-	/* Workaround: Need to write PP_CONTROL with the unlock key as
-	 * the very first thing. */
 	pp_ctl = ironlake_get_pp_control(intel_dp);
 
+	/* Ensure PPS is unlocked */
+	if (!HAS_DDI(dev_priv))
+		I915_WRITE(regs.pp_ctrl, pp_ctl);
+
 	pp_on = I915_READ(regs.pp_on);
 	pp_off = I915_READ(regs.pp_off);
-	if (!IS_GEN9_LP(dev_priv) && !HAS_PCH_CNP(dev_priv) &&
-	    !HAS_PCH_ICP(dev_priv)) {
-		I915_WRITE(regs.pp_ctrl, pp_ctl);
-		pp_div = I915_READ(regs.pp_div);
-	}
 
 	/* Pull timing values out of registers */
-	seq->t1_t3 = (pp_on & PANEL_POWER_UP_DELAY_MASK) >>
-		     PANEL_POWER_UP_DELAY_SHIFT;
+	seq->t1_t3 = REG_FIELD_GET(PANEL_POWER_UP_DELAY_MASK, pp_on);
+	seq->t8 = REG_FIELD_GET(PANEL_LIGHT_ON_DELAY_MASK, pp_on);
+	seq->t9 = REG_FIELD_GET(PANEL_LIGHT_OFF_DELAY_MASK, pp_off);
+	seq->t10 = REG_FIELD_GET(PANEL_POWER_DOWN_DELAY_MASK, pp_off);
 
-	seq->t8 = (pp_on & PANEL_LIGHT_ON_DELAY_MASK) >>
-		  PANEL_LIGHT_ON_DELAY_SHIFT;
+	if (i915_mmio_reg_valid(regs.pp_div)) {
+		u32 pp_div;
 
-	seq->t9 = (pp_off & PANEL_LIGHT_OFF_DELAY_MASK) >>
-		  PANEL_LIGHT_OFF_DELAY_SHIFT;
-
-	seq->t10 = (pp_off & PANEL_POWER_DOWN_DELAY_MASK) >>
-		   PANEL_POWER_DOWN_DELAY_SHIFT;
+		pp_div = I915_READ(regs.pp_div);
 
-	if (IS_GEN9_LP(dev_priv) || HAS_PCH_CNP(dev_priv) ||
-	    HAS_PCH_ICP(dev_priv)) {
-		seq->t11_t12 = ((pp_ctl & BXT_POWER_CYCLE_DELAY_MASK) >>
-				BXT_POWER_CYCLE_DELAY_SHIFT) * 1000;
+		seq->t11_t12 = REG_FIELD_GET(PANEL_POWER_CYCLE_DELAY_MASK, pp_div) * 1000;
 	} else {
-		seq->t11_t12 = ((pp_div & PANEL_POWER_CYCLE_DELAY_MASK) >>
-		       PANEL_POWER_CYCLE_DELAY_SHIFT) * 1000;
+		seq->t11_t12 = REG_FIELD_GET(BXT_POWER_CYCLE_DELAY_MASK, pp_ctl) * 1000;
 	}
 }
 
@@ -6184,7 +6548,7 @@ intel_dp_init_panel_power_sequencer_registers(struct intel_dp *intel_dp,
 					      bool force_disable_vdd)
 {
 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
-	u32 pp_on, pp_off, pp_div, port_sel = 0;
+	u32 pp_on, pp_off, port_sel = 0;
 	int div = dev_priv->rawclk_freq / 1000;
 	struct pps_registers regs;
 	enum port port = dp_to_dig_port(intel_dp)->base.port;
@@ -6219,23 +6583,10 @@ intel_dp_init_panel_power_sequencer_registers(struct intel_dp *intel_dp,
 		I915_WRITE(regs.pp_ctrl, pp);
 	}
 
-	pp_on = (seq->t1_t3 << PANEL_POWER_UP_DELAY_SHIFT) |
-		(seq->t8 << PANEL_LIGHT_ON_DELAY_SHIFT);
-	pp_off = (seq->t9 << PANEL_LIGHT_OFF_DELAY_SHIFT) |
-		 (seq->t10 << PANEL_POWER_DOWN_DELAY_SHIFT);
-	/* Compute the divisor for the pp clock, simply match the Bspec
-	 * formula. */
-	if (IS_GEN9_LP(dev_priv) || HAS_PCH_CNP(dev_priv) ||
-	    HAS_PCH_ICP(dev_priv)) {
-		pp_div = I915_READ(regs.pp_ctrl);
-		pp_div &= ~BXT_POWER_CYCLE_DELAY_MASK;
-		pp_div |= (DIV_ROUND_UP(seq->t11_t12, 1000)
-				<< BXT_POWER_CYCLE_DELAY_SHIFT);
-	} else {
-		pp_div = ((100 * div)/2 - 1) << PP_REFERENCE_DIVIDER_SHIFT;
-		pp_div |= (DIV_ROUND_UP(seq->t11_t12, 1000)
-				<< PANEL_POWER_CYCLE_DELAY_SHIFT);
-	}
+	pp_on = REG_FIELD_PREP(PANEL_POWER_UP_DELAY_MASK, seq->t1_t3) |
+		REG_FIELD_PREP(PANEL_LIGHT_ON_DELAY_MASK, seq->t8);
+	pp_off = REG_FIELD_PREP(PANEL_LIGHT_OFF_DELAY_MASK, seq->t9) |
+		REG_FIELD_PREP(PANEL_POWER_DOWN_DELAY_MASK, seq->t10);
 
 	/* Haswell doesn't have any port selection bits for the panel
 	 * power sequencer any more. */
@@ -6262,19 +6613,29 @@ intel_dp_init_panel_power_sequencer_registers(struct intel_dp *intel_dp,
 
 	I915_WRITE(regs.pp_on, pp_on);
 	I915_WRITE(regs.pp_off, pp_off);
-	if (IS_GEN9_LP(dev_priv) || HAS_PCH_CNP(dev_priv) ||
-	    HAS_PCH_ICP(dev_priv))
-		I915_WRITE(regs.pp_ctrl, pp_div);
-	else
-		I915_WRITE(regs.pp_div, pp_div);
+
+	/*
+	 * Compute the divisor for the pp clock, simply match the Bspec formula.
+	 */
+	if (i915_mmio_reg_valid(regs.pp_div)) {
+		I915_WRITE(regs.pp_div,
+			   REG_FIELD_PREP(PP_REFERENCE_DIVIDER_MASK, (100 * div) / 2 - 1) |
+			   REG_FIELD_PREP(PANEL_POWER_CYCLE_DELAY_MASK, DIV_ROUND_UP(seq->t11_t12, 1000)));
+	} else {
+		u32 pp_ctl;
+
+		pp_ctl = I915_READ(regs.pp_ctrl);
+		pp_ctl &= ~BXT_POWER_CYCLE_DELAY_MASK;
+		pp_ctl |= REG_FIELD_PREP(BXT_POWER_CYCLE_DELAY_MASK, DIV_ROUND_UP(seq->t11_t12, 1000));
+		I915_WRITE(regs.pp_ctrl, pp_ctl);
+	}
 
 	DRM_DEBUG_KMS("panel power sequencer register settings: PP_ON %#x, PP_OFF %#x, PP_DIV %#x\n",
 		      I915_READ(regs.pp_on),
 		      I915_READ(regs.pp_off),
-		      (IS_GEN9_LP(dev_priv) || HAS_PCH_CNP(dev_priv)  ||
-		       HAS_PCH_ICP(dev_priv)) ?
-		      (I915_READ(regs.pp_ctrl) & BXT_POWER_CYCLE_DELAY_MASK) :
-		      I915_READ(regs.pp_div));
+		      i915_mmio_reg_valid(regs.pp_div) ?
+		      I915_READ(regs.pp_div) :
+		      (I915_READ(regs.pp_ctrl) & BXT_POWER_CYCLE_DELAY_MASK));
 }
 
 static void intel_dp_pps_init(struct intel_dp *intel_dp)
@@ -6645,9 +7006,7 @@ intel_dp_drrs_init(struct intel_connector *connector,
 		return NULL;
 	}
 
-	downclock_mode = intel_find_panel_downclock(dev_priv, fixed_mode,
-						    &connector->base);
-
+	downclock_mode = intel_panel_edid_downclock_mode(connector, fixed_mode);
 	if (!downclock_mode) {
 		DRM_DEBUG_KMS("Downclock mode is not found. DRRS not supported\n");
 		return NULL;
@@ -6669,7 +7028,6 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp,
 	struct drm_display_mode *fixed_mode = NULL;
 	struct drm_display_mode *downclock_mode = NULL;
 	bool has_dpcd;
-	struct drm_display_mode *scan;
 	enum pipe pipe = INVALID_PIPE;
 	intel_wakeref_t wakeref;
 	struct edid *edid;
@@ -6685,7 +7043,7 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp,
 	 * eDP and LVDS bail out early in this case to prevent interfering
 	 * with an already powered-on LVDS power sequencer.
 	 */
-	if (intel_get_lvds_encoder(&dev_priv->drm)) {
+	if (intel_get_lvds_encoder(dev_priv)) {
 		WARN_ON(!(HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv)));
 		DRM_INFO("LVDS was detected, not registering eDP\n");
 
@@ -6722,26 +7080,13 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp,
 	}
 	intel_connector->edid = edid;
 
-	/* prefer fixed mode from EDID if available */
-	list_for_each_entry(scan, &connector->probed_modes, head) {
-		if ((scan->type & DRM_MODE_TYPE_PREFERRED)) {
-			fixed_mode = drm_mode_duplicate(dev, scan);
-			downclock_mode = intel_dp_drrs_init(
-						intel_connector, fixed_mode);
-			break;
-		}
-	}
+	fixed_mode = intel_panel_edid_fixed_mode(intel_connector);
+	if (fixed_mode)
+		downclock_mode = intel_dp_drrs_init(intel_connector, fixed_mode);
 
 	/* fallback to VBT if available for eDP */
-	if (!fixed_mode && dev_priv->vbt.lfp_lvds_vbt_mode) {
-		fixed_mode = drm_mode_duplicate(dev,
-					dev_priv->vbt.lfp_lvds_vbt_mode);
-		if (fixed_mode) {
-			fixed_mode->type |= DRM_MODE_TYPE_PREFERRED;
-			connector->display_info.width_mm = fixed_mode->width_mm;
-			connector->display_info.height_mm = fixed_mode->height_mm;
-		}
-	}
+	if (!fixed_mode)
+		fixed_mode = intel_panel_vbt_fixed_mode(intel_connector);
 	mutex_unlock(&dev->mode_config.mutex);
 
 	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
diff --git a/drivers/gpu/drm/i915/intel_dp.h b/drivers/gpu/drm/i915/intel_dp.h
new file mode 100644
index 000000000000..5e9e8d13de6e
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_dp.h
@@ -0,0 +1,122 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2019 Intel Corporation
+ */
+
+#ifndef __INTEL_DP_H__
+#define __INTEL_DP_H__
+
+#include <linux/types.h>
+
+#include <drm/i915_drm.h>
+
+#include "i915_reg.h"
+
+enum pipe;
+struct drm_connector_state;
+struct drm_encoder;
+struct drm_i915_private;
+struct drm_modeset_acquire_ctx;
+struct intel_connector;
+struct intel_crtc_state;
+struct intel_digital_port;
+struct intel_dp;
+struct intel_encoder;
+
+struct link_config_limits {
+	int min_clock, max_clock;
+	int min_lane_count, max_lane_count;
+	int min_bpp, max_bpp;
+};
+
+void intel_dp_adjust_compliance_config(struct intel_dp *intel_dp,
+				       struct intel_crtc_state *pipe_config,
+				       struct link_config_limits *limits);
+bool intel_dp_limited_color_range(const struct intel_crtc_state *crtc_state,
+				  const struct drm_connector_state *conn_state);
+int intel_dp_min_bpp(const struct intel_crtc_state *crtc_state);
+bool intel_dp_port_enabled(struct drm_i915_private *dev_priv,
+			   i915_reg_t dp_reg, enum port port,
+			   enum pipe *pipe);
+bool intel_dp_init(struct drm_i915_private *dev_priv, i915_reg_t output_reg,
+		   enum port port);
+bool intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
+			     struct intel_connector *intel_connector);
+void intel_dp_set_link_params(struct intel_dp *intel_dp,
+			      int link_rate, u8 lane_count,
+			      bool link_mst);
+int intel_dp_get_link_train_fallback_values(struct intel_dp *intel_dp,
+					    int link_rate, u8 lane_count);
+int intel_dp_retrain_link(struct intel_encoder *encoder,
+			  struct drm_modeset_acquire_ctx *ctx);
+void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode);
+void intel_dp_sink_set_decompression_state(struct intel_dp *intel_dp,
+					   const struct intel_crtc_state *crtc_state,
+					   bool enable);
+void intel_dp_encoder_reset(struct drm_encoder *encoder);
+void intel_dp_encoder_suspend(struct intel_encoder *intel_encoder);
+void intel_dp_encoder_flush_work(struct drm_encoder *encoder);
+int intel_dp_compute_config(struct intel_encoder *encoder,
+			    struct intel_crtc_state *pipe_config,
+			    struct drm_connector_state *conn_state);
+bool intel_dp_is_edp(struct intel_dp *intel_dp);
+bool intel_dp_is_port_edp(struct drm_i915_private *dev_priv, enum port port);
+enum irqreturn intel_dp_hpd_pulse(struct intel_digital_port *intel_dig_port,
+				  bool long_hpd);
+void intel_edp_backlight_on(const struct intel_crtc_state *crtc_state,
+			    const struct drm_connector_state *conn_state);
+void intel_edp_backlight_off(const struct drm_connector_state *conn_state);
+void intel_edp_panel_vdd_on(struct intel_dp *intel_dp);
+void intel_edp_panel_on(struct intel_dp *intel_dp);
+void intel_edp_panel_off(struct intel_dp *intel_dp);
+void intel_dp_mst_suspend(struct drm_i915_private *dev_priv);
+void intel_dp_mst_resume(struct drm_i915_private *dev_priv);
+int intel_dp_max_link_rate(struct intel_dp *intel_dp);
+int intel_dp_max_lane_count(struct intel_dp *intel_dp);
+int intel_dp_rate_select(struct intel_dp *intel_dp, int rate);
+void intel_power_sequencer_reset(struct drm_i915_private *dev_priv);
+u32 intel_dp_pack_aux(const u8 *src, int src_bytes);
+
+void intel_edp_drrs_enable(struct intel_dp *intel_dp,
+			   const struct intel_crtc_state *crtc_state);
+void intel_edp_drrs_disable(struct intel_dp *intel_dp,
+			    const struct intel_crtc_state *crtc_state);
+void intel_edp_drrs_invalidate(struct drm_i915_private *dev_priv,
+			       unsigned int frontbuffer_bits);
+void intel_edp_drrs_flush(struct drm_i915_private *dev_priv,
+			  unsigned int frontbuffer_bits);
+
+void
+intel_dp_program_link_training_pattern(struct intel_dp *intel_dp,
+				       u8 dp_train_pat);
+void
+intel_dp_set_signal_levels(struct intel_dp *intel_dp);
+void intel_dp_set_idle_link_train(struct intel_dp *intel_dp);
+u8
+intel_dp_voltage_max(struct intel_dp *intel_dp);
+u8
+intel_dp_pre_emphasis_max(struct intel_dp *intel_dp, u8 voltage_swing);
+void intel_dp_compute_rate(struct intel_dp *intel_dp, int port_clock,
+			   u8 *link_bw, u8 *rate_select);
+bool intel_dp_source_supports_hbr2(struct intel_dp *intel_dp);
+bool intel_dp_source_supports_hbr3(struct intel_dp *intel_dp);
+bool
+intel_dp_get_link_status(struct intel_dp *intel_dp, u8 *link_status);
+u16 intel_dp_dsc_get_output_bpp(int link_clock, u8 lane_count,
+				int mode_clock, int mode_hdisplay);
+u8 intel_dp_dsc_get_slice_count(struct intel_dp *intel_dp, int mode_clock,
+				int mode_hdisplay);
+
+bool intel_dp_read_dpcd(struct intel_dp *intel_dp);
+int intel_dp_link_required(int pixel_clock, int bpp);
+int intel_dp_max_data_rate(int max_link_clock, int max_lanes);
+bool intel_digital_port_connected(struct intel_encoder *encoder);
+void icl_tc_phy_disconnect(struct drm_i915_private *dev_priv,
+			   struct intel_digital_port *dig_port);
+
+static inline unsigned int intel_dp_unused_lane_mask(int lane_count)
+{
+	return ~((1 << lane_count) - 1) & 0xf;
+}
+
+#endif /* __INTEL_DP_H__ */
diff --git a/drivers/gpu/drm/i915/intel_dp_link_training.c b/drivers/gpu/drm/i915/intel_dp_link_training.c
index b59c87daa4f7..54b069333e2f 100644
--- a/drivers/gpu/drm/i915/intel_dp_link_training.c
+++ b/drivers/gpu/drm/i915/intel_dp_link_training.c
@@ -21,6 +21,7 @@
  * IN THE SOFTWARE.
  */
 
+#include "intel_dp.h"
 #include "intel_drv.h"
 
 static void
diff --git a/drivers/gpu/drm/i915/intel_dp_mst.c b/drivers/gpu/drm/i915/intel_dp_mst.c
index fb67cd931117..8839eaea8371 100644
--- a/drivers/gpu/drm/i915/intel_dp_mst.c
+++ b/drivers/gpu/drm/i915/intel_dp_mst.c
@@ -23,78 +23,119 @@
  *
  */
 
-#include "i915_drv.h"
-#include "intel_drv.h"
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_edid.h>
 #include <drm/drm_probe_helper.h>
 
+#include "i915_drv.h"
+#include "intel_audio.h"
+#include "intel_connector.h"
+#include "intel_ddi.h"
+#include "intel_dp.h"
+#include "intel_drv.h"
+
+static int intel_dp_mst_compute_link_config(struct intel_encoder *encoder,
+					    struct intel_crtc_state *crtc_state,
+					    struct drm_connector_state *conn_state,
+					    struct link_config_limits *limits)
+{
+	struct drm_atomic_state *state = crtc_state->base.state;
+	struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base);
+	struct intel_dp *intel_dp = &intel_mst->primary->dp;
+	struct intel_connector *connector =
+		to_intel_connector(conn_state->connector);
+	const struct drm_display_mode *adjusted_mode =
+		&crtc_state->base.adjusted_mode;
+	void *port = connector->port;
+	bool constant_n = drm_dp_has_quirk(&intel_dp->desc,
+					   DP_DPCD_QUIRK_CONSTANT_N);
+	int bpp, slots = -EINVAL;
+
+	crtc_state->lane_count = limits->max_lane_count;
+	crtc_state->port_clock = limits->max_clock;
+
+	for (bpp = limits->max_bpp; bpp >= limits->min_bpp; bpp -= 2 * 3) {
+		crtc_state->pipe_bpp = bpp;
+
+		crtc_state->pbn = drm_dp_calc_pbn_mode(adjusted_mode->crtc_clock,
+						       crtc_state->pipe_bpp);
+
+		slots = drm_dp_atomic_find_vcpi_slots(state, &intel_dp->mst_mgr,
+						      port, crtc_state->pbn);
+		if (slots == -EDEADLK)
+			return slots;
+		if (slots >= 0)
+			break;
+	}
+
+	if (slots < 0) {
+		DRM_DEBUG_KMS("failed finding vcpi slots:%d\n", slots);
+		return slots;
+	}
+
+	intel_link_compute_m_n(crtc_state->pipe_bpp,
+			       crtc_state->lane_count,
+			       adjusted_mode->crtc_clock,
+			       crtc_state->port_clock,
+			       &crtc_state->dp_m_n,
+			       constant_n);
+	crtc_state->dp_m_n.tu = slots;
+
+	return 0;
+}
+
 static int intel_dp_mst_compute_config(struct intel_encoder *encoder,
 				       struct intel_crtc_state *pipe_config,
 				       struct drm_connector_state *conn_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base);
-	struct intel_digital_port *intel_dig_port = intel_mst->primary;
-	struct intel_dp *intel_dp = &intel_dig_port->dp;
-	struct drm_connector *connector = conn_state->connector;
-	void *port = to_intel_connector(connector)->port;
-	struct drm_atomic_state *state = pipe_config->base.state;
-	struct drm_crtc *crtc = pipe_config->base.crtc;
-	struct drm_crtc_state *old_crtc_state =
-		drm_atomic_get_old_crtc_state(state, crtc);
-	int bpp;
-	int lane_count, slots =
-		to_intel_crtc_state(old_crtc_state)->dp_m_n.tu;
-	const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
-	int mst_pbn;
-	bool constant_n = drm_dp_has_quirk(&intel_dp->desc,
-					   DP_DPCD_QUIRK_CONSTANT_N);
+	struct intel_dp *intel_dp = &intel_mst->primary->dp;
+	struct intel_connector *connector =
+		to_intel_connector(conn_state->connector);
+	struct intel_digital_connector_state *intel_conn_state =
+		to_intel_digital_connector_state(conn_state);
+	const struct drm_display_mode *adjusted_mode =
+		&pipe_config->base.adjusted_mode;
+	void *port = connector->port;
+	struct link_config_limits limits;
+	int ret;
 
 	if (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN)
 		return -EINVAL;
 
 	pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
 	pipe_config->has_pch_encoder = false;
-	bpp = 24;
-	if (intel_dp->compliance.test_data.bpc) {
-		bpp = intel_dp->compliance.test_data.bpc * 3;
-		DRM_DEBUG_KMS("Setting pipe bpp to %d\n",
-			      bpp);
-	}
+
+	if (intel_conn_state->force_audio == HDMI_AUDIO_AUTO)
+		pipe_config->has_audio =
+			drm_dp_mst_port_has_audio(&intel_dp->mst_mgr, port);
+	else
+		pipe_config->has_audio =
+			intel_conn_state->force_audio == HDMI_AUDIO_ON;
+
 	/*
 	 * for MST we always configure max link bw - the spec doesn't
 	 * seem to suggest we should do otherwise.
 	 */
-	lane_count = intel_dp_max_lane_count(intel_dp);
-
-	pipe_config->lane_count = lane_count;
-
-	pipe_config->pipe_bpp = bpp;
+	limits.min_clock =
+	limits.max_clock = intel_dp_max_link_rate(intel_dp);
 
-	pipe_config->port_clock = intel_dp_max_link_rate(intel_dp);
+	limits.min_lane_count =
+	limits.max_lane_count = intel_dp_max_lane_count(intel_dp);
 
-	if (drm_dp_mst_port_has_audio(&intel_dp->mst_mgr, port))
-		pipe_config->has_audio = true;
+	limits.min_bpp = intel_dp_min_bpp(pipe_config);
+	limits.max_bpp = pipe_config->pipe_bpp;
 
-	mst_pbn = drm_dp_calc_pbn_mode(adjusted_mode->crtc_clock, bpp);
-	pipe_config->pbn = mst_pbn;
+	intel_dp_adjust_compliance_config(intel_dp, pipe_config, &limits);
 
-	slots = drm_dp_atomic_find_vcpi_slots(state, &intel_dp->mst_mgr, port,
-					      mst_pbn);
-	if (slots < 0) {
-		DRM_DEBUG_KMS("failed finding vcpi slots:%d\n",
-			      slots);
-		return slots;
-	}
-
-	intel_link_compute_m_n(bpp, lane_count,
-			       adjusted_mode->crtc_clock,
-			       pipe_config->port_clock,
-			       &pipe_config->dp_m_n,
-			       constant_n);
+	ret = intel_dp_mst_compute_link_config(encoder, pipe_config,
+					       conn_state, &limits);
+	if (ret)
+		return ret;
 
-	pipe_config->dp_m_n.tu = slots;
+	pipe_config->limited_color_range =
+		intel_dp_limited_color_range(pipe_config, conn_state);
 
 	if (IS_GEN9_LP(dev_priv))
 		pipe_config->lane_lat_optim_mask =
@@ -117,7 +158,11 @@ intel_dp_mst_atomic_check(struct drm_connector *connector,
 	struct drm_crtc *new_crtc = new_conn_state->crtc;
 	struct drm_crtc_state *crtc_state;
 	struct drm_dp_mst_topology_mgr *mgr;
-	int ret = 0;
+	int ret;
+
+	ret = intel_digital_connector_atomic_check(connector, new_conn_state);
+	if (ret)
+		return ret;
 
 	if (!old_conn_state->crtc)
 		return 0;
@@ -289,7 +334,7 @@ static void intel_mst_enable_dp(struct intel_encoder *encoder,
 
 	DRM_DEBUG_KMS("active links %d\n", intel_dp->active_mst_links);
 
-	if (intel_wait_for_register(dev_priv,
+	if (intel_wait_for_register(&dev_priv->uncore,
 				    DP_TP_STATUS(port),
 				    DP_TP_STATUS_ACT_SENT,
 				    DP_TP_STATUS_ACT_SENT,
@@ -354,11 +399,13 @@ intel_dp_mst_detect(struct drm_connector *connector, bool force)
 static const struct drm_connector_funcs intel_dp_mst_connector_funcs = {
 	.detect = intel_dp_mst_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
+	.atomic_get_property = intel_digital_connector_atomic_get_property,
+	.atomic_set_property = intel_digital_connector_atomic_set_property,
 	.late_register = intel_connector_register,
 	.early_unregister = intel_connector_unregister,
 	.destroy = intel_connector_destroy,
 	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
-	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_duplicate_state = intel_digital_connector_duplicate_state,
 };
 
 static int intel_dp_mst_get_modes(struct drm_connector *connector)
@@ -373,7 +420,6 @@ intel_dp_mst_mode_valid(struct drm_connector *connector,
 	struct intel_connector *intel_connector = to_intel_connector(connector);
 	struct intel_dp *intel_dp = intel_connector->mst_port;
 	int max_dotclk = to_i915(connector->dev)->max_dotclk_freq;
-	int bpp = 24; /* MST uses fixed bpp */
 	int max_rate, mode_rate, max_lanes, max_link_clock;
 
 	if (drm_connector_is_unregistered(connector))
@@ -386,7 +432,7 @@ intel_dp_mst_mode_valid(struct drm_connector *connector,
 	max_lanes = intel_dp_max_lane_count(intel_dp);
 
 	max_rate = intel_dp_max_data_rate(max_link_clock, max_lanes);
-	mode_rate = intel_dp_link_required(mode->clock, bpp);
+	mode_rate = intel_dp_link_required(mode->clock, 18);
 
 	/* TODO - validate mode against available PBN for link */
 	if (mode->clock < 10000)
@@ -487,6 +533,10 @@ static struct drm_connector *intel_dp_add_mst_connector(struct drm_dp_mst_topolo
 	if (ret)
 		goto err;
 
+	intel_attach_force_audio_property(connector);
+	intel_attach_broadcast_rgb_property(connector);
+	drm_connector_attach_max_bpc_property(connector, 6, 12);
+
 	return connector;
 
 err:
diff --git a/drivers/gpu/drm/i915/intel_dpio_phy.c b/drivers/gpu/drm/i915/intel_dpio_phy.c
index 95cb8b154f87..ab4ac7158b79 100644
--- a/drivers/gpu/drm/i915/intel_dpio_phy.c
+++ b/drivers/gpu/drm/i915/intel_dpio_phy.c
@@ -21,6 +21,7 @@
  * DEALINGS IN THE SOFTWARE.
  */
 
+#include "intel_dp.h"
 #include "intel_drv.h"
 
 /**
@@ -341,7 +342,7 @@ static u32 bxt_get_grc(struct drm_i915_private *dev_priv, enum dpio_phy phy)
 static void bxt_phy_wait_grc_done(struct drm_i915_private *dev_priv,
 				  enum dpio_phy phy)
 {
-	if (intel_wait_for_register(dev_priv,
+	if (intel_wait_for_register(&dev_priv->uncore,
 				    BXT_PORT_REF_DW3(phy),
 				    GRC_DONE, GRC_DONE,
 				    10))
@@ -383,7 +384,8 @@ static void _bxt_ddi_phy_init(struct drm_i915_private *dev_priv,
 	 * The flag should get set in 100us according to the HW team, but
 	 * use 1ms due to occasional timeouts observed with that.
 	 */
-	if (intel_wait_for_register_fw(dev_priv, BXT_PORT_CL1CM_DW0(phy),
+	if (intel_wait_for_register_fw(&dev_priv->uncore,
+				       BXT_PORT_CL1CM_DW0(phy),
 				       PHY_RESERVED | PHY_POWER_GOOD,
 				       PHY_POWER_GOOD,
 				       1))
diff --git a/drivers/gpu/drm/i915/intel_dpll_mgr.c b/drivers/gpu/drm/i915/intel_dpll_mgr.c
index 0a42d11c4c33..e01c057ce50b 100644
--- a/drivers/gpu/drm/i915/intel_dpll_mgr.c
+++ b/drivers/gpu/drm/i915/intel_dpll_mgr.c
@@ -241,11 +241,11 @@ out:
 }
 
 static struct intel_shared_dpll *
-intel_find_shared_dpll(struct intel_crtc *crtc,
-		       struct intel_crtc_state *crtc_state,
+intel_find_shared_dpll(struct intel_crtc_state *crtc_state,
 		       enum intel_dpll_id range_min,
 		       enum intel_dpll_id range_max)
 {
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	struct intel_shared_dpll *pll, *unused_pll = NULL;
 	struct intel_shared_dpll_state *shared_dpll;
@@ -420,9 +420,10 @@ static void ibx_pch_dpll_disable(struct drm_i915_private *dev_priv,
 }
 
 static struct intel_shared_dpll *
-ibx_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
+ibx_get_dpll(struct intel_crtc_state *crtc_state,
 	     struct intel_encoder *encoder)
 {
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	struct intel_shared_dpll *pll;
 	enum intel_dpll_id i;
@@ -436,7 +437,7 @@ ibx_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
 			      crtc->base.base.id, crtc->base.name,
 			      pll->info->name);
 	} else {
-		pll = intel_find_shared_dpll(crtc, crtc_state,
+		pll = intel_find_shared_dpll(crtc_state,
 					     DPLL_ID_PCH_PLL_A,
 					     DPLL_ID_PCH_PLL_B);
 	}
@@ -764,15 +765,13 @@ hsw_ddi_calculate_wrpll(int clock /* in Hz */,
 	*r2_out = best.r2;
 }
 
-static struct intel_shared_dpll *hsw_ddi_hdmi_get_dpll(int clock,
-						       struct intel_crtc *crtc,
-						       struct intel_crtc_state *crtc_state)
+static struct intel_shared_dpll *hsw_ddi_hdmi_get_dpll(struct intel_crtc_state *crtc_state)
 {
 	struct intel_shared_dpll *pll;
 	u32 val;
 	unsigned int p, n2, r2;
 
-	hsw_ddi_calculate_wrpll(clock * 1000, &r2, &n2, &p);
+	hsw_ddi_calculate_wrpll(crtc_state->port_clock * 1000, &r2, &n2, &p);
 
 	val = WRPLL_PLL_ENABLE | WRPLL_PLL_LCPLL |
 	      WRPLL_DIVIDER_REFERENCE(r2) | WRPLL_DIVIDER_FEEDBACK(n2) |
@@ -780,7 +779,7 @@ static struct intel_shared_dpll *hsw_ddi_hdmi_get_dpll(int clock,
 
 	crtc_state->dpll_hw_state.wrpll = val;
 
-	pll = intel_find_shared_dpll(crtc, crtc_state,
+	pll = intel_find_shared_dpll(crtc_state,
 				     DPLL_ID_WRPLL1, DPLL_ID_WRPLL2);
 
 	if (!pll)
@@ -790,11 +789,12 @@ static struct intel_shared_dpll *hsw_ddi_hdmi_get_dpll(int clock,
 }
 
 static struct intel_shared_dpll *
-hsw_ddi_dp_get_dpll(struct intel_encoder *encoder, int clock)
+hsw_ddi_dp_get_dpll(struct intel_crtc_state *crtc_state)
 {
-	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
 	struct intel_shared_dpll *pll;
 	enum intel_dpll_id pll_id;
+	int clock = crtc_state->port_clock;
 
 	switch (clock / 2) {
 	case 81000:
@@ -820,19 +820,18 @@ hsw_ddi_dp_get_dpll(struct intel_encoder *encoder, int clock)
 }
 
 static struct intel_shared_dpll *
-hsw_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
+hsw_get_dpll(struct intel_crtc_state *crtc_state,
 	     struct intel_encoder *encoder)
 {
 	struct intel_shared_dpll *pll;
-	int clock = crtc_state->port_clock;
 
 	memset(&crtc_state->dpll_hw_state, 0,
 	       sizeof(crtc_state->dpll_hw_state));
 
 	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) {
-		pll = hsw_ddi_hdmi_get_dpll(clock, crtc, crtc_state);
+		pll = hsw_ddi_hdmi_get_dpll(crtc_state);
 	} else if (intel_crtc_has_dp_encoder(crtc_state)) {
-		pll = hsw_ddi_dp_get_dpll(encoder, clock);
+		pll = hsw_ddi_dp_get_dpll(crtc_state);
 	} else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG)) {
 		if (WARN_ON(crtc_state->port_clock / 2 != 135000))
 			return NULL;
@@ -840,7 +839,7 @@ hsw_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
 		crtc_state->dpll_hw_state.spll =
 			SPLL_PLL_ENABLE | SPLL_PLL_FREQ_1350MHz | SPLL_PLL_SSC;
 
-		pll = intel_find_shared_dpll(crtc, crtc_state,
+		pll = intel_find_shared_dpll(crtc_state,
 					     DPLL_ID_SPLL, DPLL_ID_SPLL);
 	} else {
 		return NULL;
@@ -961,7 +960,7 @@ static void skl_ddi_pll_enable(struct drm_i915_private *dev_priv,
 	I915_WRITE(regs[id].ctl,
 		   I915_READ(regs[id].ctl) | LCPLL_PLL_ENABLE);
 
-	if (intel_wait_for_register(dev_priv,
+	if (intel_wait_for_register(&dev_priv->uncore,
 				    DPLL_STATUS,
 				    DPLL_LOCK(id),
 				    DPLL_LOCK(id),
@@ -1308,9 +1307,7 @@ skip_remaining_dividers:
 	return true;
 }
 
-static bool skl_ddi_hdmi_pll_dividers(struct intel_crtc *crtc,
-				      struct intel_crtc_state *crtc_state,
-				      int clock)
+static bool skl_ddi_hdmi_pll_dividers(struct intel_crtc_state *crtc_state)
 {
 	u32 ctrl1, cfgcr1, cfgcr2;
 	struct skl_wrpll_params wrpll_params = { 0, };
@@ -1323,7 +1320,8 @@ static bool skl_ddi_hdmi_pll_dividers(struct intel_crtc *crtc,
 
 	ctrl1 |= DPLL_CTRL1_HDMI_MODE(0);
 
-	if (!skl_ddi_calculate_wrpll(clock * 1000, &wrpll_params))
+	if (!skl_ddi_calculate_wrpll(crtc_state->port_clock * 1000,
+				     &wrpll_params))
 		return false;
 
 	cfgcr1 = DPLL_CFGCR1_FREQ_ENABLE |
@@ -1346,8 +1344,7 @@ static bool skl_ddi_hdmi_pll_dividers(struct intel_crtc *crtc,
 }
 
 static bool
-skl_ddi_dp_set_dpll_hw_state(int clock,
-			     struct intel_dpll_hw_state *dpll_hw_state)
+skl_ddi_dp_set_dpll_hw_state(struct intel_crtc_state *crtc_state)
 {
 	u32 ctrl1;
 
@@ -1356,7 +1353,7 @@ skl_ddi_dp_set_dpll_hw_state(int clock,
 	 * as the DPLL id in this function.
 	 */
 	ctrl1 = DPLL_CTRL1_OVERRIDE(0);
-	switch (clock / 2) {
+	switch (crtc_state->port_clock / 2) {
 	case 81000:
 		ctrl1 |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_810, 0);
 		break;
@@ -1378,44 +1375,43 @@ skl_ddi_dp_set_dpll_hw_state(int clock,
 		break;
 	}
 
-	dpll_hw_state->ctrl1 = ctrl1;
+	memset(&crtc_state->dpll_hw_state, 0,
+	       sizeof(crtc_state->dpll_hw_state));
+
+	crtc_state->dpll_hw_state.ctrl1 = ctrl1;
+
 	return true;
 }
 
 static struct intel_shared_dpll *
-skl_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
+skl_get_dpll(struct intel_crtc_state *crtc_state,
 	     struct intel_encoder *encoder)
 {
 	struct intel_shared_dpll *pll;
-	int clock = crtc_state->port_clock;
 	bool bret;
-	struct intel_dpll_hw_state dpll_hw_state;
-
-	memset(&dpll_hw_state, 0, sizeof(dpll_hw_state));
 
 	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) {
-		bret = skl_ddi_hdmi_pll_dividers(crtc, crtc_state, clock);
+		bret = skl_ddi_hdmi_pll_dividers(crtc_state);
 		if (!bret) {
 			DRM_DEBUG_KMS("Could not get HDMI pll dividers.\n");
 			return NULL;
 		}
 	} else if (intel_crtc_has_dp_encoder(crtc_state)) {
-		bret = skl_ddi_dp_set_dpll_hw_state(clock, &dpll_hw_state);
+		bret = skl_ddi_dp_set_dpll_hw_state(crtc_state);
 		if (!bret) {
 			DRM_DEBUG_KMS("Could not set DP dpll HW state.\n");
 			return NULL;
 		}
-		crtc_state->dpll_hw_state = dpll_hw_state;
 	} else {
 		return NULL;
 	}
 
 	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP))
-		pll = intel_find_shared_dpll(crtc, crtc_state,
+		pll = intel_find_shared_dpll(crtc_state,
 					     DPLL_ID_SKL_DPLL0,
 					     DPLL_ID_SKL_DPLL0);
 	else
-		pll = intel_find_shared_dpll(crtc, crtc_state,
+		pll = intel_find_shared_dpll(crtc_state,
 					     DPLL_ID_SKL_DPLL1,
 					     DPLL_ID_SKL_DPLL3);
 	if (!pll)
@@ -1692,10 +1688,10 @@ static const struct bxt_clk_div bxt_dp_clk_val[] = {
 };
 
 static bool
-bxt_ddi_hdmi_pll_dividers(struct intel_crtc *intel_crtc,
-			  struct intel_crtc_state *crtc_state, int clock,
+bxt_ddi_hdmi_pll_dividers(struct intel_crtc_state *crtc_state,
 			  struct bxt_clk_div *clk_div)
 {
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 	struct dpll best_clock;
 
 	/* Calculate HDMI div */
@@ -1703,9 +1699,10 @@ bxt_ddi_hdmi_pll_dividers(struct intel_crtc *intel_crtc,
 	 * FIXME: tie the following calculation into
 	 * i9xx_crtc_compute_clock
 	 */
-	if (!bxt_find_best_dpll(crtc_state, clock, &best_clock)) {
+	if (!bxt_find_best_dpll(crtc_state, &best_clock)) {
 		DRM_DEBUG_DRIVER("no PLL dividers found for clock %d pipe %c\n",
-				 clock, pipe_name(intel_crtc->pipe));
+				 crtc_state->port_clock,
+				 pipe_name(crtc->pipe));
 		return false;
 	}
 
@@ -1722,8 +1719,10 @@ bxt_ddi_hdmi_pll_dividers(struct intel_crtc *intel_crtc,
 	return true;
 }
 
-static void bxt_ddi_dp_pll_dividers(int clock, struct bxt_clk_div *clk_div)
+static void bxt_ddi_dp_pll_dividers(struct intel_crtc_state *crtc_state,
+				    struct bxt_clk_div *clk_div)
 {
+	int clock = crtc_state->port_clock;
 	int i;
 
 	*clk_div = bxt_dp_clk_val[0];
@@ -1737,14 +1736,17 @@ static void bxt_ddi_dp_pll_dividers(int clock, struct bxt_clk_div *clk_div)
 	clk_div->vco = clock * 10 / 2 * clk_div->p1 * clk_div->p2;
 }
 
-static bool bxt_ddi_set_dpll_hw_state(int clock,
-			  struct bxt_clk_div *clk_div,
-			  struct intel_dpll_hw_state *dpll_hw_state)
+static bool bxt_ddi_set_dpll_hw_state(struct intel_crtc_state *crtc_state,
+				      const struct bxt_clk_div *clk_div)
 {
+	struct intel_dpll_hw_state *dpll_hw_state = &crtc_state->dpll_hw_state;
+	int clock = crtc_state->port_clock;
 	int vco = clk_div->vco;
 	u32 prop_coef, int_coef, gain_ctl, targ_cnt;
 	u32 lanestagger;
 
+	memset(dpll_hw_state, 0, sizeof(*dpll_hw_state));
+
 	if (vco >= 6200000 && vco <= 6700000) {
 		prop_coef = 4;
 		int_coef = 9;
@@ -1804,55 +1806,45 @@ static bool bxt_ddi_set_dpll_hw_state(int clock,
 }
 
 static bool
-bxt_ddi_dp_set_dpll_hw_state(int clock,
-			     struct intel_dpll_hw_state *dpll_hw_state)
+bxt_ddi_dp_set_dpll_hw_state(struct intel_crtc_state *crtc_state)
 {
-	struct bxt_clk_div clk_div = {0};
+	struct bxt_clk_div clk_div = {};
 
-	bxt_ddi_dp_pll_dividers(clock, &clk_div);
+	bxt_ddi_dp_pll_dividers(crtc_state, &clk_div);
 
-	return bxt_ddi_set_dpll_hw_state(clock, &clk_div, dpll_hw_state);
+	return bxt_ddi_set_dpll_hw_state(crtc_state, &clk_div);
 }
 
 static bool
-bxt_ddi_hdmi_set_dpll_hw_state(struct intel_crtc *intel_crtc,
-			       struct intel_crtc_state *crtc_state, int clock,
-			       struct intel_dpll_hw_state *dpll_hw_state)
+bxt_ddi_hdmi_set_dpll_hw_state(struct intel_crtc_state *crtc_state)
 {
-	struct bxt_clk_div clk_div = { };
+	struct bxt_clk_div clk_div = {};
 
-	bxt_ddi_hdmi_pll_dividers(intel_crtc, crtc_state, clock, &clk_div);
+	bxt_ddi_hdmi_pll_dividers(crtc_state, &clk_div);
 
-	return bxt_ddi_set_dpll_hw_state(clock, &clk_div, dpll_hw_state);
+	return bxt_ddi_set_dpll_hw_state(crtc_state, &clk_div);
 }
 
 static struct intel_shared_dpll *
-bxt_get_dpll(struct intel_crtc *crtc,
-		struct intel_crtc_state *crtc_state,
-		struct intel_encoder *encoder)
+bxt_get_dpll(struct intel_crtc_state *crtc_state,
+	     struct intel_encoder *encoder)
 {
-	struct intel_dpll_hw_state dpll_hw_state = { };
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	struct intel_shared_dpll *pll;
-	int i, clock = crtc_state->port_clock;
+	enum intel_dpll_id id;
 
 	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI) &&
-	    !bxt_ddi_hdmi_set_dpll_hw_state(crtc, crtc_state, clock,
-					    &dpll_hw_state))
+	    !bxt_ddi_hdmi_set_dpll_hw_state(crtc_state))
 		return NULL;
 
 	if (intel_crtc_has_dp_encoder(crtc_state) &&
-	    !bxt_ddi_dp_set_dpll_hw_state(clock, &dpll_hw_state))
+	    !bxt_ddi_dp_set_dpll_hw_state(crtc_state))
 		return NULL;
 
-	memset(&crtc_state->dpll_hw_state, 0,
-	       sizeof(crtc_state->dpll_hw_state));
-
-	crtc_state->dpll_hw_state = dpll_hw_state;
-
 	/* 1:1 mapping between ports and PLLs */
-	i = (enum intel_dpll_id) encoder->port;
-	pll = intel_get_shared_dpll_by_id(dev_priv, i);
+	id = (enum intel_dpll_id) encoder->port;
+	pll = intel_get_shared_dpll_by_id(dev_priv, id);
 
 	DRM_DEBUG_KMS("[CRTC:%d:%s] using pre-allocated %s\n",
 		      crtc->base.base.id, crtc->base.name, pll->info->name);
@@ -1911,8 +1903,7 @@ static void intel_ddi_pll_init(struct drm_device *dev)
 struct intel_dpll_mgr {
 	const struct dpll_info *dpll_info;
 
-	struct intel_shared_dpll *(*get_dpll)(struct intel_crtc *crtc,
-					      struct intel_crtc_state *crtc_state,
+	struct intel_shared_dpll *(*get_dpll)(struct intel_crtc_state *crtc_state,
 					      struct intel_encoder *encoder);
 
 	void (*dump_hw_state)(struct drm_i915_private *dev_priv,
@@ -1986,7 +1977,7 @@ static void cnl_ddi_pll_enable(struct drm_i915_private *dev_priv,
 	I915_WRITE(CNL_DPLL_ENABLE(id), val);
 
 	/* 2. Wait for DPLL power state enabled in DPLL_ENABLE. */
-	if (intel_wait_for_register(dev_priv,
+	if (intel_wait_for_register(&dev_priv->uncore,
 				    CNL_DPLL_ENABLE(id),
 				    PLL_POWER_STATE,
 				    PLL_POWER_STATE,
@@ -2027,7 +2018,7 @@ static void cnl_ddi_pll_enable(struct drm_i915_private *dev_priv,
 	I915_WRITE(CNL_DPLL_ENABLE(id), val);
 
 	/* 7. Wait for PLL lock status in DPLL_ENABLE. */
-	if (intel_wait_for_register(dev_priv,
+	if (intel_wait_for_register(&dev_priv->uncore,
 				    CNL_DPLL_ENABLE(id),
 				    PLL_LOCK,
 				    PLL_LOCK,
@@ -2075,7 +2066,7 @@ static void cnl_ddi_pll_disable(struct drm_i915_private *dev_priv,
 	I915_WRITE(CNL_DPLL_ENABLE(id), val);
 
 	/* 4. Wait for PLL not locked status in DPLL_ENABLE. */
-	if (intel_wait_for_register(dev_priv,
+	if (intel_wait_for_register(&dev_priv->uncore,
 				    CNL_DPLL_ENABLE(id),
 				    PLL_LOCK,
 				    0,
@@ -2097,7 +2088,7 @@ static void cnl_ddi_pll_disable(struct drm_i915_private *dev_priv,
 	I915_WRITE(CNL_DPLL_ENABLE(id), val);
 
 	/* 7. Wait for DPLL power state disabled in DPLL_ENABLE. */
-	if (intel_wait_for_register(dev_priv,
+	if (intel_wait_for_register(&dev_priv->uncore,
 				    CNL_DPLL_ENABLE(id),
 				    PLL_POWER_STATE,
 				    0,
@@ -2242,11 +2233,11 @@ int cnl_hdmi_pll_ref_clock(struct drm_i915_private *dev_priv)
 }
 
 static bool
-cnl_ddi_calculate_wrpll(int clock,
-			struct drm_i915_private *dev_priv,
+cnl_ddi_calculate_wrpll(struct intel_crtc_state *crtc_state,
 			struct skl_wrpll_params *wrpll_params)
 {
-	u32 afe_clock = clock * 5;
+	struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
+	u32 afe_clock = crtc_state->port_clock * 5;
 	u32 ref_clock;
 	u32 dco_min = 7998000;
 	u32 dco_max = 10000000;
@@ -2282,23 +2273,20 @@ cnl_ddi_calculate_wrpll(int clock,
 
 	ref_clock = cnl_hdmi_pll_ref_clock(dev_priv);
 
-	cnl_wrpll_params_populate(wrpll_params, best_dco, ref_clock, pdiv, qdiv,
-				  kdiv);
+	cnl_wrpll_params_populate(wrpll_params, best_dco, ref_clock,
+				  pdiv, qdiv, kdiv);
 
 	return true;
 }
 
-static bool cnl_ddi_hdmi_pll_dividers(struct intel_crtc *crtc,
-				      struct intel_crtc_state *crtc_state,
-				      int clock)
+static bool cnl_ddi_hdmi_pll_dividers(struct intel_crtc_state *crtc_state)
 {
-	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	u32 cfgcr0, cfgcr1;
 	struct skl_wrpll_params wrpll_params = { 0, };
 
 	cfgcr0 = DPLL_CFGCR0_HDMI_MODE;
 
-	if (!cnl_ddi_calculate_wrpll(clock, dev_priv, &wrpll_params))
+	if (!cnl_ddi_calculate_wrpll(crtc_state, &wrpll_params))
 		return false;
 
 	cfgcr0 |= DPLL_CFGCR0_DCO_FRACTION(wrpll_params.dco_fraction) |
@@ -2319,14 +2307,13 @@ static bool cnl_ddi_hdmi_pll_dividers(struct intel_crtc *crtc,
 }
 
 static bool
-cnl_ddi_dp_set_dpll_hw_state(int clock,
-			     struct intel_dpll_hw_state *dpll_hw_state)
+cnl_ddi_dp_set_dpll_hw_state(struct intel_crtc_state *crtc_state)
 {
 	u32 cfgcr0;
 
 	cfgcr0 = DPLL_CFGCR0_SSC_ENABLE;
 
-	switch (clock / 2) {
+	switch (crtc_state->port_clock / 2) {
 	case 81000:
 		cfgcr0 |= DPLL_CFGCR0_LINK_RATE_810;
 		break;
@@ -2356,41 +2343,40 @@ cnl_ddi_dp_set_dpll_hw_state(int clock,
 		break;
 	}
 
-	dpll_hw_state->cfgcr0 = cfgcr0;
+	memset(&crtc_state->dpll_hw_state, 0,
+	       sizeof(crtc_state->dpll_hw_state));
+
+	crtc_state->dpll_hw_state.cfgcr0 = cfgcr0;
+
 	return true;
 }
 
 static struct intel_shared_dpll *
-cnl_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
+cnl_get_dpll(struct intel_crtc_state *crtc_state,
 	     struct intel_encoder *encoder)
 {
 	struct intel_shared_dpll *pll;
-	int clock = crtc_state->port_clock;
 	bool bret;
-	struct intel_dpll_hw_state dpll_hw_state;
-
-	memset(&dpll_hw_state, 0, sizeof(dpll_hw_state));
 
 	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) {
-		bret = cnl_ddi_hdmi_pll_dividers(crtc, crtc_state, clock);
+		bret = cnl_ddi_hdmi_pll_dividers(crtc_state);
 		if (!bret) {
 			DRM_DEBUG_KMS("Could not get HDMI pll dividers.\n");
 			return NULL;
 		}
 	} else if (intel_crtc_has_dp_encoder(crtc_state)) {
-		bret = cnl_ddi_dp_set_dpll_hw_state(clock, &dpll_hw_state);
+		bret = cnl_ddi_dp_set_dpll_hw_state(crtc_state);
 		if (!bret) {
 			DRM_DEBUG_KMS("Could not set DP dpll HW state.\n");
 			return NULL;
 		}
-		crtc_state->dpll_hw_state = dpll_hw_state;
 	} else {
 		DRM_DEBUG_KMS("Skip DPLL setup for output_types 0x%x\n",
 			      crtc_state->output_types);
 		return NULL;
 	}
 
-	pll = intel_find_shared_dpll(crtc, crtc_state,
+	pll = intel_find_shared_dpll(crtc_state,
 				     DPLL_ID_SKL_DPLL0,
 				     DPLL_ID_SKL_DPLL2);
 	if (!pll) {
@@ -2431,47 +2417,69 @@ static const struct intel_dpll_mgr cnl_pll_mgr = {
 	.dump_hw_state = cnl_dump_hw_state,
 };
 
+struct icl_combo_pll_params {
+	int clock;
+	struct skl_wrpll_params wrpll;
+};
+
 /*
  * These values alrea already adjusted: they're the bits we write to the
  * registers, not the logical values.
  */
-static const struct skl_wrpll_params icl_dp_combo_pll_24MHz_values[] = {
-	{ .dco_integer = 0x151, .dco_fraction = 0x4000,		/* [0]: 5.4 */
-	  .pdiv = 0x2 /* 3 */, .kdiv = 1, .qdiv_mode = 0, .qdiv_ratio = 0},
-	{ .dco_integer = 0x151, .dco_fraction = 0x4000,		/* [1]: 2.7 */
-	  .pdiv = 0x2 /* 3 */, .kdiv = 2, .qdiv_mode = 0, .qdiv_ratio = 0},
-	{ .dco_integer = 0x151, .dco_fraction = 0x4000,		/* [2]: 1.62 */
-	  .pdiv = 0x4 /* 5 */, .kdiv = 2, .qdiv_mode = 0, .qdiv_ratio = 0},
-	{ .dco_integer = 0x151, .dco_fraction = 0x4000,		/* [3]: 3.24 */
-	  .pdiv = 0x4 /* 5 */, .kdiv = 1, .qdiv_mode = 0, .qdiv_ratio = 0},
-	{ .dco_integer = 0x168, .dco_fraction = 0x0000,		/* [4]: 2.16 */
-	  .pdiv = 0x1 /* 2 */, .kdiv = 2, .qdiv_mode = 1, .qdiv_ratio = 2},
-	{ .dco_integer = 0x168, .dco_fraction = 0x0000,		/* [5]: 4.32 */
-	  .pdiv = 0x1 /* 2 */, .kdiv = 2, .qdiv_mode = 0, .qdiv_ratio = 0},
-	{ .dco_integer = 0x195, .dco_fraction = 0x0000,		/* [6]: 6.48 */
-	  .pdiv = 0x2 /* 3 */, .kdiv = 1, .qdiv_mode = 0, .qdiv_ratio = 0},
-	{ .dco_integer = 0x151, .dco_fraction = 0x4000,		/* [7]: 8.1 */
-	  .pdiv = 0x1 /* 2 */, .kdiv = 1, .qdiv_mode = 0, .qdiv_ratio = 0},
+static const struct icl_combo_pll_params icl_dp_combo_pll_24MHz_values[] = {
+	{ 540000,
+	  { .dco_integer = 0x151, .dco_fraction = 0x4000,		/* [0]: 5.4 */
+	    .pdiv = 0x2 /* 3 */, .kdiv = 1, .qdiv_mode = 0, .qdiv_ratio = 0, }, },
+	{ 270000,
+	  { .dco_integer = 0x151, .dco_fraction = 0x4000,		/* [1]: 2.7 */
+	    .pdiv = 0x2 /* 3 */, .kdiv = 2, .qdiv_mode = 0, .qdiv_ratio = 0, }, },
+	{ 162000,
+	  { .dco_integer = 0x151, .dco_fraction = 0x4000,		/* [2]: 1.62 */
+	    .pdiv = 0x4 /* 5 */, .kdiv = 2, .qdiv_mode = 0, .qdiv_ratio = 0, }, },
+	{ 324000,
+	  { .dco_integer = 0x151, .dco_fraction = 0x4000,		/* [3]: 3.24 */
+	    .pdiv = 0x4 /* 5 */, .kdiv = 1, .qdiv_mode = 0, .qdiv_ratio = 0, }, },
+	{ 216000,
+	  { .dco_integer = 0x168, .dco_fraction = 0x0000,		/* [4]: 2.16 */
+	    .pdiv = 0x1 /* 2 */, .kdiv = 2, .qdiv_mode = 1, .qdiv_ratio = 2, }, },
+	{ 432000,
+	  { .dco_integer = 0x168, .dco_fraction = 0x0000,		/* [5]: 4.32 */
+	    .pdiv = 0x1 /* 2 */, .kdiv = 2, .qdiv_mode = 0, .qdiv_ratio = 0, }, },
+	{ 648000,
+	  { .dco_integer = 0x195, .dco_fraction = 0x0000,		/* [6]: 6.48 */
+	    .pdiv = 0x2 /* 3 */, .kdiv = 1, .qdiv_mode = 0, .qdiv_ratio = 0, }, },
+	{ 810000,
+	  { .dco_integer = 0x151, .dco_fraction = 0x4000,		/* [7]: 8.1 */
+	    .pdiv = 0x1 /* 2 */, .kdiv = 1, .qdiv_mode = 0, .qdiv_ratio = 0, }, },
 };
 
+
 /* Also used for 38.4 MHz values. */
-static const struct skl_wrpll_params icl_dp_combo_pll_19_2MHz_values[] = {
-	{ .dco_integer = 0x1A5, .dco_fraction = 0x7000,		/* [0]: 5.4 */
-	  .pdiv = 0x2 /* 3 */, .kdiv = 1, .qdiv_mode = 0, .qdiv_ratio = 0},
-	{ .dco_integer = 0x1A5, .dco_fraction = 0x7000,		/* [1]: 2.7 */
-	  .pdiv = 0x2 /* 3 */, .kdiv = 2, .qdiv_mode = 0, .qdiv_ratio = 0},
-	{ .dco_integer = 0x1A5, .dco_fraction = 0x7000,		/* [2]: 1.62 */
-	  .pdiv = 0x4 /* 5 */, .kdiv = 2, .qdiv_mode = 0, .qdiv_ratio = 0},
-	{ .dco_integer = 0x1A5, .dco_fraction = 0x7000,		/* [3]: 3.24 */
-	  .pdiv = 0x4 /* 5 */, .kdiv = 1, .qdiv_mode = 0, .qdiv_ratio = 0},
-	{ .dco_integer = 0x1C2, .dco_fraction = 0x0000,		/* [4]: 2.16 */
-	  .pdiv = 0x1 /* 2 */, .kdiv = 2, .qdiv_mode = 1, .qdiv_ratio = 2},
-	{ .dco_integer = 0x1C2, .dco_fraction = 0x0000,		/* [5]: 4.32 */
-	  .pdiv = 0x1 /* 2 */, .kdiv = 2, .qdiv_mode = 0, .qdiv_ratio = 0},
-	{ .dco_integer = 0x1FA, .dco_fraction = 0x2000,		/* [6]: 6.48 */
-	  .pdiv = 0x2 /* 3 */, .kdiv = 1, .qdiv_mode = 0, .qdiv_ratio = 0},
-	{ .dco_integer = 0x1A5, .dco_fraction = 0x7000,		/* [7]: 8.1 */
-	  .pdiv = 0x1 /* 2 */, .kdiv = 1, .qdiv_mode = 0, .qdiv_ratio = 0},
+static const struct icl_combo_pll_params icl_dp_combo_pll_19_2MHz_values[] = {
+	{ 540000,
+	  { .dco_integer = 0x1A5, .dco_fraction = 0x7000,		/* [0]: 5.4 */
+	    .pdiv = 0x2 /* 3 */, .kdiv = 1, .qdiv_mode = 0, .qdiv_ratio = 0, }, },
+	{ 270000,
+	  { .dco_integer = 0x1A5, .dco_fraction = 0x7000,		/* [1]: 2.7 */
+	    .pdiv = 0x2 /* 3 */, .kdiv = 2, .qdiv_mode = 0, .qdiv_ratio = 0, }, },
+	{ 162000,
+	  { .dco_integer = 0x1A5, .dco_fraction = 0x7000,		/* [2]: 1.62 */
+	    .pdiv = 0x4 /* 5 */, .kdiv = 2, .qdiv_mode = 0, .qdiv_ratio = 0, }, },
+	{ 324000,
+	  { .dco_integer = 0x1A5, .dco_fraction = 0x7000,		/* [3]: 3.24 */
+	    .pdiv = 0x4 /* 5 */, .kdiv = 1, .qdiv_mode = 0, .qdiv_ratio = 0, }, },
+	{ 216000,
+	  { .dco_integer = 0x1C2, .dco_fraction = 0x0000,		/* [4]: 2.16 */
+	    .pdiv = 0x1 /* 2 */, .kdiv = 2, .qdiv_mode = 1, .qdiv_ratio = 2, }, },
+	{ 432000,
+	  { .dco_integer = 0x1C2, .dco_fraction = 0x0000,		/* [5]: 4.32 */
+	    .pdiv = 0x1 /* 2 */, .kdiv = 2, .qdiv_mode = 0, .qdiv_ratio = 0, }, },
+	{ 648000,
+	  { .dco_integer = 0x1FA, .dco_fraction = 0x2000,		/* [6]: 6.48 */
+	    .pdiv = 0x2 /* 3 */, .kdiv = 1, .qdiv_mode = 0, .qdiv_ratio = 0, }, },
+	{ 810000,
+	  { .dco_integer = 0x1A5, .dco_fraction = 0x7000,		/* [7]: 8.1 */
+	    .pdiv = 0x1 /* 2 */, .kdiv = 1, .qdiv_mode = 0, .qdiv_ratio = 0, }, },
 };
 
 static const struct skl_wrpll_params icl_tbt_pll_24MHz_values = {
@@ -2484,72 +2492,53 @@ static const struct skl_wrpll_params icl_tbt_pll_19_2MHz_values = {
 	.pdiv = 0x4 /* 5 */, .kdiv = 1, .qdiv_mode = 0, .qdiv_ratio = 0,
 };
 
-static bool icl_calc_dp_combo_pll(struct drm_i915_private *dev_priv, int clock,
+static bool icl_calc_dp_combo_pll(struct intel_crtc_state *crtc_state,
 				  struct skl_wrpll_params *pll_params)
 {
-	const struct skl_wrpll_params *params;
-
-	params = dev_priv->cdclk.hw.ref == 24000 ?
-			icl_dp_combo_pll_24MHz_values :
-			icl_dp_combo_pll_19_2MHz_values;
+	struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
+	const struct icl_combo_pll_params *params =
+		dev_priv->cdclk.hw.ref == 24000 ?
+		icl_dp_combo_pll_24MHz_values :
+		icl_dp_combo_pll_19_2MHz_values;
+	int clock = crtc_state->port_clock;
+	int i;
 
-	switch (clock) {
-	case 540000:
-		*pll_params = params[0];
-		break;
-	case 270000:
-		*pll_params = params[1];
-		break;
-	case 162000:
-		*pll_params = params[2];
-		break;
-	case 324000:
-		*pll_params = params[3];
-		break;
-	case 216000:
-		*pll_params = params[4];
-		break;
-	case 432000:
-		*pll_params = params[5];
-		break;
-	case 648000:
-		*pll_params = params[6];
-		break;
-	case 810000:
-		*pll_params = params[7];
-		break;
-	default:
-		MISSING_CASE(clock);
-		return false;
+	for (i = 0; i < ARRAY_SIZE(icl_dp_combo_pll_24MHz_values); i++) {
+		if (clock == params[i].clock) {
+			*pll_params = params[i].wrpll;
+			return true;
+		}
 	}
 
-	return true;
+	MISSING_CASE(clock);
+	return false;
 }
 
-static bool icl_calc_tbt_pll(struct drm_i915_private *dev_priv, int clock,
+static bool icl_calc_tbt_pll(struct intel_crtc_state *crtc_state,
 			     struct skl_wrpll_params *pll_params)
 {
+	struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
+
 	*pll_params = dev_priv->cdclk.hw.ref == 24000 ?
 			icl_tbt_pll_24MHz_values : icl_tbt_pll_19_2MHz_values;
 	return true;
 }
 
 static bool icl_calc_dpll_state(struct intel_crtc_state *crtc_state,
-				struct intel_encoder *encoder, int clock,
-				struct intel_dpll_hw_state *pll_state)
+				struct intel_encoder *encoder)
 {
-	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
 	u32 cfgcr0, cfgcr1;
 	struct skl_wrpll_params pll_params = { 0 };
 	bool ret;
 
 	if (intel_port_is_tc(dev_priv, encoder->port))
-		ret = icl_calc_tbt_pll(dev_priv, clock, &pll_params);
+		ret = icl_calc_tbt_pll(crtc_state, &pll_params);
 	else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI) ||
 		 intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI))
-		ret = cnl_ddi_calculate_wrpll(clock, dev_priv, &pll_params);
+		ret = cnl_ddi_calculate_wrpll(crtc_state, &pll_params);
 	else
-		ret = icl_calc_dp_combo_pll(dev_priv, clock, &pll_params);
+		ret = icl_calc_dp_combo_pll(crtc_state, &pll_params);
 
 	if (!ret)
 		return false;
@@ -2563,82 +2552,16 @@ static bool icl_calc_dpll_state(struct intel_crtc_state *crtc_state,
 		 DPLL_CFGCR1_PDIV(pll_params.pdiv) |
 		 DPLL_CFGCR1_CENTRAL_FREQ_8400;
 
-	pll_state->cfgcr0 = cfgcr0;
-	pll_state->cfgcr1 = cfgcr1;
-	return true;
-}
-
-int icl_calc_dp_combo_pll_link(struct drm_i915_private *dev_priv,
-			       u32 pll_id)
-{
-	u32 cfgcr0, cfgcr1;
-	u32 pdiv, kdiv, qdiv_mode, qdiv_ratio, dco_integer, dco_fraction;
-	const struct skl_wrpll_params *params;
-	int index, n_entries, link_clock;
-
-	/* Read back values from DPLL CFGCR registers */
-	cfgcr0 = I915_READ(ICL_DPLL_CFGCR0(pll_id));
-	cfgcr1 = I915_READ(ICL_DPLL_CFGCR1(pll_id));
-
-	dco_integer = cfgcr0 & DPLL_CFGCR0_DCO_INTEGER_MASK;
-	dco_fraction = (cfgcr0 & DPLL_CFGCR0_DCO_FRACTION_MASK) >>
-		DPLL_CFGCR0_DCO_FRACTION_SHIFT;
-	pdiv = (cfgcr1 & DPLL_CFGCR1_PDIV_MASK) >> DPLL_CFGCR1_PDIV_SHIFT;
-	kdiv = (cfgcr1 & DPLL_CFGCR1_KDIV_MASK) >> DPLL_CFGCR1_KDIV_SHIFT;
-	qdiv_mode = (cfgcr1 & DPLL_CFGCR1_QDIV_MODE(1)) >>
-		DPLL_CFGCR1_QDIV_MODE_SHIFT;
-	qdiv_ratio = (cfgcr1 & DPLL_CFGCR1_QDIV_RATIO_MASK) >>
-		DPLL_CFGCR1_QDIV_RATIO_SHIFT;
-
-	params = dev_priv->cdclk.hw.ref == 24000 ?
-		icl_dp_combo_pll_24MHz_values :
-		icl_dp_combo_pll_19_2MHz_values;
-	n_entries = ARRAY_SIZE(icl_dp_combo_pll_24MHz_values);
-
-	for (index = 0; index < n_entries; index++) {
-		if (dco_integer == params[index].dco_integer &&
-		    dco_fraction == params[index].dco_fraction &&
-		    pdiv == params[index].pdiv &&
-		    kdiv == params[index].kdiv &&
-		    qdiv_mode == params[index].qdiv_mode &&
-		    qdiv_ratio == params[index].qdiv_ratio)
-			break;
-	}
+	memset(&crtc_state->dpll_hw_state, 0,
+	       sizeof(crtc_state->dpll_hw_state));
 
-	/* Map PLL Index to Link Clock */
-	switch (index) {
-	default:
-		MISSING_CASE(index);
-		/* fall through */
-	case 0:
-		link_clock = 540000;
-		break;
-	case 1:
-		link_clock = 270000;
-		break;
-	case 2:
-		link_clock = 162000;
-		break;
-	case 3:
-		link_clock = 324000;
-		break;
-	case 4:
-		link_clock = 216000;
-		break;
-	case 5:
-		link_clock = 432000;
-		break;
-	case 6:
-		link_clock = 648000;
-		break;
-	case 7:
-		link_clock = 810000;
-		break;
-	}
+	crtc_state->dpll_hw_state.cfgcr0 = cfgcr0;
+	crtc_state->dpll_hw_state.cfgcr1 = cfgcr1;
 
-	return link_clock;
+	return true;
 }
 
+
 static enum tc_port icl_pll_id_to_tc_port(enum intel_dpll_id id)
 {
 	return id - DPLL_ID_ICL_MGPLL1;
@@ -2649,11 +2572,6 @@ enum intel_dpll_id icl_tc_port_to_pll_id(enum tc_port tc_port)
 	return tc_port + DPLL_ID_ICL_MGPLL1;
 }
 
-bool intel_dpll_is_combophy(enum intel_dpll_id id)
-{
-	return id == DPLL_ID_ICL_DPLL0 || id == DPLL_ID_ICL_DPLL1;
-}
-
 static bool icl_mg_pll_find_divisors(int clock_khz, bool is_dp, bool use_ssc,
 				     u32 *target_dco_khz,
 				     struct intel_dpll_hw_state *state)
@@ -2728,12 +2646,12 @@ static bool icl_mg_pll_find_divisors(int clock_khz, bool is_dp, bool use_ssc,
  * The specification for this function uses real numbers, so the math had to be
  * adapted to integer-only calculation, that's why it looks so different.
  */
-static bool icl_calc_mg_pll_state(struct intel_crtc_state *crtc_state,
-				  struct intel_encoder *encoder, int clock,
-				  struct intel_dpll_hw_state *pll_state)
+static bool icl_calc_mg_pll_state(struct intel_crtc_state *crtc_state)
 {
-	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
+	struct intel_dpll_hw_state *pll_state = &crtc_state->dpll_hw_state;
 	int refclk_khz = dev_priv->cdclk.hw.ref;
+	int clock = crtc_state->port_clock;
 	u32 dco_khz, m1div, m2div_int, m2div_rem, m2div_frac;
 	u32 iref_ndiv, iref_trim, iref_pulse_w;
 	u32 prop_coeff, int_coeff;
@@ -2743,6 +2661,8 @@ static bool icl_calc_mg_pll_state(struct intel_crtc_state *crtc_state,
 	bool use_ssc = false;
 	bool is_dp = !intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI);
 
+	memset(pll_state, 0, sizeof(*pll_state));
+
 	if (!icl_mg_pll_find_divisors(clock, is_dp, use_ssc, &dco_khz,
 				      pll_state)) {
 		DRM_DEBUG_KMS("Failed to find divisors for clock %d\n", clock);
@@ -2892,23 +2812,20 @@ static bool icl_calc_mg_pll_state(struct intel_crtc_state *crtc_state,
 }
 
 static struct intel_shared_dpll *
-icl_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
+icl_get_dpll(struct intel_crtc_state *crtc_state,
 	     struct intel_encoder *encoder)
 {
-	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
 	struct intel_digital_port *intel_dig_port;
 	struct intel_shared_dpll *pll;
-	struct intel_dpll_hw_state pll_state = {};
 	enum port port = encoder->port;
 	enum intel_dpll_id min, max;
-	int clock = crtc_state->port_clock;
 	bool ret;
 
 	if (intel_port_is_combophy(dev_priv, port)) {
 		min = DPLL_ID_ICL_DPLL0;
 		max = DPLL_ID_ICL_DPLL1;
-		ret = icl_calc_dpll_state(crtc_state, encoder, clock,
-					  &pll_state);
+		ret = icl_calc_dpll_state(crtc_state, encoder);
 	} else if (intel_port_is_tc(dev_priv, port)) {
 		if (encoder->type == INTEL_OUTPUT_DP_MST) {
 			struct intel_dp_mst_encoder *mst_encoder;
@@ -2922,16 +2839,14 @@ icl_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
 		if (intel_dig_port->tc_type == TC_PORT_TBT) {
 			min = DPLL_ID_ICL_TBTPLL;
 			max = min;
-			ret = icl_calc_dpll_state(crtc_state, encoder, clock,
-						  &pll_state);
+			ret = icl_calc_dpll_state(crtc_state, encoder);
 		} else {
 			enum tc_port tc_port;
 
 			tc_port = intel_port_to_tc(dev_priv, port);
 			min = icl_tc_port_to_pll_id(tc_port);
 			max = min;
-			ret = icl_calc_mg_pll_state(crtc_state, encoder, clock,
-						    &pll_state);
+			ret = icl_calc_mg_pll_state(crtc_state);
 		}
 	} else {
 		MISSING_CASE(port);
@@ -2943,9 +2858,8 @@ icl_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
 		return NULL;
 	}
 
-	crtc_state->dpll_hw_state = pll_state;
 
-	pll = intel_find_shared_dpll(crtc, crtc_state, min, max);
+	pll = intel_find_shared_dpll(crtc_state, min, max);
 	if (!pll) {
 		DRM_DEBUG_KMS("No PLL selected\n");
 		return NULL;
@@ -2956,19 +2870,72 @@ icl_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
 	return pll;
 }
 
-static i915_reg_t icl_pll_id_to_enable_reg(enum intel_dpll_id id)
+static bool mg_pll_get_hw_state(struct drm_i915_private *dev_priv,
+				struct intel_shared_dpll *pll,
+				struct intel_dpll_hw_state *hw_state)
 {
-	if (intel_dpll_is_combophy(id))
-		return CNL_DPLL_ENABLE(id);
-	else if (id == DPLL_ID_ICL_TBTPLL)
-		return TBT_PLL_ENABLE;
+	const enum intel_dpll_id id = pll->info->id;
+	enum tc_port tc_port = icl_pll_id_to_tc_port(id);
+	intel_wakeref_t wakeref;
+	bool ret = false;
+	u32 val;
+
+	wakeref = intel_display_power_get_if_enabled(dev_priv,
+						     POWER_DOMAIN_PLLS);
+	if (!wakeref)
+		return false;
 
-	return MG_PLL_ENABLE(icl_pll_id_to_tc_port(id));
+	val = I915_READ(MG_PLL_ENABLE(tc_port));
+	if (!(val & PLL_ENABLE))
+		goto out;
+
+	hw_state->mg_refclkin_ctl = I915_READ(MG_REFCLKIN_CTL(tc_port));
+	hw_state->mg_refclkin_ctl &= MG_REFCLKIN_CTL_OD_2_MUX_MASK;
+
+	hw_state->mg_clktop2_coreclkctl1 =
+		I915_READ(MG_CLKTOP2_CORECLKCTL1(tc_port));
+	hw_state->mg_clktop2_coreclkctl1 &=
+		MG_CLKTOP2_CORECLKCTL1_A_DIVRATIO_MASK;
+
+	hw_state->mg_clktop2_hsclkctl =
+		I915_READ(MG_CLKTOP2_HSCLKCTL(tc_port));
+	hw_state->mg_clktop2_hsclkctl &=
+		MG_CLKTOP2_HSCLKCTL_TLINEDRV_CLKSEL_MASK |
+		MG_CLKTOP2_HSCLKCTL_CORE_INPUTSEL_MASK |
+		MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_MASK |
+		MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO_MASK;
+
+	hw_state->mg_pll_div0 = I915_READ(MG_PLL_DIV0(tc_port));
+	hw_state->mg_pll_div1 = I915_READ(MG_PLL_DIV1(tc_port));
+	hw_state->mg_pll_lf = I915_READ(MG_PLL_LF(tc_port));
+	hw_state->mg_pll_frac_lock = I915_READ(MG_PLL_FRAC_LOCK(tc_port));
+	hw_state->mg_pll_ssc = I915_READ(MG_PLL_SSC(tc_port));
+
+	hw_state->mg_pll_bias = I915_READ(MG_PLL_BIAS(tc_port));
+	hw_state->mg_pll_tdc_coldst_bias =
+		I915_READ(MG_PLL_TDC_COLDST_BIAS(tc_port));
+
+	if (dev_priv->cdclk.hw.ref == 38400) {
+		hw_state->mg_pll_tdc_coldst_bias_mask = MG_PLL_TDC_COLDST_COLDSTART;
+		hw_state->mg_pll_bias_mask = 0;
+	} else {
+		hw_state->mg_pll_tdc_coldst_bias_mask = -1U;
+		hw_state->mg_pll_bias_mask = -1U;
+	}
+
+	hw_state->mg_pll_tdc_coldst_bias &= hw_state->mg_pll_tdc_coldst_bias_mask;
+	hw_state->mg_pll_bias &= hw_state->mg_pll_bias_mask;
+
+	ret = true;
+out:
+	intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS, wakeref);
+	return ret;
 }
 
 static bool icl_pll_get_hw_state(struct drm_i915_private *dev_priv,
 				 struct intel_shared_dpll *pll,
-				 struct intel_dpll_hw_state *hw_state)
+				 struct intel_dpll_hw_state *hw_state,
+				 i915_reg_t enable_reg)
 {
 	const enum intel_dpll_id id = pll->info->id;
 	intel_wakeref_t wakeref;
@@ -2980,54 +2947,12 @@ static bool icl_pll_get_hw_state(struct drm_i915_private *dev_priv,
 	if (!wakeref)
 		return false;
 
-	val = I915_READ(icl_pll_id_to_enable_reg(id));
+	val = I915_READ(enable_reg);
 	if (!(val & PLL_ENABLE))
 		goto out;
 
-	if (intel_dpll_is_combophy(id) ||
-	    id == DPLL_ID_ICL_TBTPLL) {
-		hw_state->cfgcr0 = I915_READ(ICL_DPLL_CFGCR0(id));
-		hw_state->cfgcr1 = I915_READ(ICL_DPLL_CFGCR1(id));
-	} else {
-		enum tc_port tc_port = icl_pll_id_to_tc_port(id);
-
-		hw_state->mg_refclkin_ctl = I915_READ(MG_REFCLKIN_CTL(tc_port));
-		hw_state->mg_refclkin_ctl &= MG_REFCLKIN_CTL_OD_2_MUX_MASK;
-
-		hw_state->mg_clktop2_coreclkctl1 =
-			I915_READ(MG_CLKTOP2_CORECLKCTL1(tc_port));
-		hw_state->mg_clktop2_coreclkctl1 &=
-			MG_CLKTOP2_CORECLKCTL1_A_DIVRATIO_MASK;
-
-		hw_state->mg_clktop2_hsclkctl =
-			I915_READ(MG_CLKTOP2_HSCLKCTL(tc_port));
-		hw_state->mg_clktop2_hsclkctl &=
-			MG_CLKTOP2_HSCLKCTL_TLINEDRV_CLKSEL_MASK |
-			MG_CLKTOP2_HSCLKCTL_CORE_INPUTSEL_MASK |
-			MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_MASK |
-			MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO_MASK;
-
-		hw_state->mg_pll_div0 = I915_READ(MG_PLL_DIV0(tc_port));
-		hw_state->mg_pll_div1 = I915_READ(MG_PLL_DIV1(tc_port));
-		hw_state->mg_pll_lf = I915_READ(MG_PLL_LF(tc_port));
-		hw_state->mg_pll_frac_lock = I915_READ(MG_PLL_FRAC_LOCK(tc_port));
-		hw_state->mg_pll_ssc = I915_READ(MG_PLL_SSC(tc_port));
-
-		hw_state->mg_pll_bias = I915_READ(MG_PLL_BIAS(tc_port));
-		hw_state->mg_pll_tdc_coldst_bias =
-			I915_READ(MG_PLL_TDC_COLDST_BIAS(tc_port));
-
-		if (dev_priv->cdclk.hw.ref == 38400) {
-			hw_state->mg_pll_tdc_coldst_bias_mask = MG_PLL_TDC_COLDST_COLDSTART;
-			hw_state->mg_pll_bias_mask = 0;
-		} else {
-			hw_state->mg_pll_tdc_coldst_bias_mask = -1U;
-			hw_state->mg_pll_bias_mask = -1U;
-		}
-
-		hw_state->mg_pll_tdc_coldst_bias &= hw_state->mg_pll_tdc_coldst_bias_mask;
-		hw_state->mg_pll_bias &= hw_state->mg_pll_bias_mask;
-	}
+	hw_state->cfgcr0 = I915_READ(ICL_DPLL_CFGCR0(id));
+	hw_state->cfgcr1 = I915_READ(ICL_DPLL_CFGCR1(id));
 
 	ret = true;
 out:
@@ -3035,6 +2960,21 @@ out:
 	return ret;
 }
 
+static bool combo_pll_get_hw_state(struct drm_i915_private *dev_priv,
+				   struct intel_shared_dpll *pll,
+				   struct intel_dpll_hw_state *hw_state)
+{
+	return icl_pll_get_hw_state(dev_priv, pll, hw_state,
+				    CNL_DPLL_ENABLE(pll->info->id));
+}
+
+static bool tbt_pll_get_hw_state(struct drm_i915_private *dev_priv,
+				 struct intel_shared_dpll *pll,
+				 struct intel_dpll_hw_state *hw_state)
+{
+	return icl_pll_get_hw_state(dev_priv, pll, hw_state, TBT_PLL_ENABLE);
+}
+
 static void icl_dpll_write(struct drm_i915_private *dev_priv,
 			   struct intel_shared_dpll *pll)
 {
@@ -3096,11 +3036,10 @@ static void icl_mg_pll_write(struct drm_i915_private *dev_priv,
 	POSTING_READ(MG_PLL_TDC_COLDST_BIAS(tc_port));
 }
 
-static void icl_pll_enable(struct drm_i915_private *dev_priv,
-			   struct intel_shared_dpll *pll)
+static void icl_pll_power_enable(struct drm_i915_private *dev_priv,
+				 struct intel_shared_dpll *pll,
+				 i915_reg_t enable_reg)
 {
-	const enum intel_dpll_id id = pll->info->id;
-	i915_reg_t enable_reg = icl_pll_id_to_enable_reg(id);
 	u32 val;
 
 	val = I915_READ(enable_reg);
@@ -3111,37 +3050,90 @@ static void icl_pll_enable(struct drm_i915_private *dev_priv,
 	 * The spec says we need to "wait" but it also says it should be
 	 * immediate.
 	 */
-	if (intel_wait_for_register(dev_priv, enable_reg, PLL_POWER_STATE,
-				    PLL_POWER_STATE, 1))
-		DRM_ERROR("PLL %d Power not enabled\n", id);
+	if (intel_wait_for_register(&dev_priv->uncore, enable_reg,
+				    PLL_POWER_STATE, PLL_POWER_STATE, 1))
+		DRM_ERROR("PLL %d Power not enabled\n", pll->info->id);
+}
 
-	if (intel_dpll_is_combophy(id) || id == DPLL_ID_ICL_TBTPLL)
-		icl_dpll_write(dev_priv, pll);
-	else
-		icl_mg_pll_write(dev_priv, pll);
+static void icl_pll_enable(struct drm_i915_private *dev_priv,
+			   struct intel_shared_dpll *pll,
+			   i915_reg_t enable_reg)
+{
+	u32 val;
+
+	val = I915_READ(enable_reg);
+	val |= PLL_ENABLE;
+	I915_WRITE(enable_reg, val);
+
+	/* Timeout is actually 600us. */
+	if (intel_wait_for_register(&dev_priv->uncore, enable_reg,
+				    PLL_LOCK, PLL_LOCK, 1))
+		DRM_ERROR("PLL %d not locked\n", pll->info->id);
+}
+
+static void combo_pll_enable(struct drm_i915_private *dev_priv,
+			     struct intel_shared_dpll *pll)
+{
+	i915_reg_t enable_reg = CNL_DPLL_ENABLE(pll->info->id);
+
+	icl_pll_power_enable(dev_priv, pll, enable_reg);
+
+	icl_dpll_write(dev_priv, pll);
 
 	/*
 	 * DVFS pre sequence would be here, but in our driver the cdclk code
 	 * paths should already be setting the appropriate voltage, hence we do
-	 * nothign here.
+	 * nothing here.
 	 */
 
-	val = I915_READ(enable_reg);
-	val |= PLL_ENABLE;
-	I915_WRITE(enable_reg, val);
+	icl_pll_enable(dev_priv, pll, enable_reg);
 
-	if (intel_wait_for_register(dev_priv, enable_reg, PLL_LOCK, PLL_LOCK,
-				    1)) /* 600us actually. */
-		DRM_ERROR("PLL %d not locked\n", id);
+	/* DVFS post sequence would be here. See the comment above. */
+}
+
+static void tbt_pll_enable(struct drm_i915_private *dev_priv,
+			   struct intel_shared_dpll *pll)
+{
+	icl_pll_power_enable(dev_priv, pll, TBT_PLL_ENABLE);
+
+	icl_dpll_write(dev_priv, pll);
+
+	/*
+	 * DVFS pre sequence would be here, but in our driver the cdclk code
+	 * paths should already be setting the appropriate voltage, hence we do
+	 * nothing here.
+	 */
+
+	icl_pll_enable(dev_priv, pll, TBT_PLL_ENABLE);
+
+	/* DVFS post sequence would be here. See the comment above. */
+}
+
+static void mg_pll_enable(struct drm_i915_private *dev_priv,
+			  struct intel_shared_dpll *pll)
+{
+	i915_reg_t enable_reg =
+		MG_PLL_ENABLE(icl_pll_id_to_tc_port(pll->info->id));
+
+	icl_pll_power_enable(dev_priv, pll, enable_reg);
+
+	icl_mg_pll_write(dev_priv, pll);
+
+	/*
+	 * DVFS pre sequence would be here, but in our driver the cdclk code
+	 * paths should already be setting the appropriate voltage, hence we do
+	 * nothing here.
+	 */
+
+	icl_pll_enable(dev_priv, pll, enable_reg);
 
 	/* DVFS post sequence would be here. See the comment above. */
 }
 
 static void icl_pll_disable(struct drm_i915_private *dev_priv,
-			    struct intel_shared_dpll *pll)
+			    struct intel_shared_dpll *pll,
+			    i915_reg_t enable_reg)
 {
-	const enum intel_dpll_id id = pll->info->id;
-	i915_reg_t enable_reg = icl_pll_id_to_enable_reg(id);
 	u32 val;
 
 	/* The first steps are done by intel_ddi_post_disable(). */
@@ -3157,8 +3149,9 @@ static void icl_pll_disable(struct drm_i915_private *dev_priv,
 	I915_WRITE(enable_reg, val);
 
 	/* Timeout is actually 1us. */
-	if (intel_wait_for_register(dev_priv, enable_reg, PLL_LOCK, 0, 1))
-		DRM_ERROR("PLL %d locked\n", id);
+	if (intel_wait_for_register(&dev_priv->uncore,
+				    enable_reg, PLL_LOCK, 0, 1))
+		DRM_ERROR("PLL %d locked\n", pll->info->id);
 
 	/* DVFS post sequence would be here. See the comment above. */
 
@@ -3170,9 +3163,30 @@ static void icl_pll_disable(struct drm_i915_private *dev_priv,
 	 * The spec says we need to "wait" but it also says it should be
 	 * immediate.
 	 */
-	if (intel_wait_for_register(dev_priv, enable_reg, PLL_POWER_STATE, 0,
-				    1))
-		DRM_ERROR("PLL %d Power not disabled\n", id);
+	if (intel_wait_for_register(&dev_priv->uncore,
+				    enable_reg, PLL_POWER_STATE, 0, 1))
+		DRM_ERROR("PLL %d Power not disabled\n", pll->info->id);
+}
+
+static void combo_pll_disable(struct drm_i915_private *dev_priv,
+			      struct intel_shared_dpll *pll)
+{
+	icl_pll_disable(dev_priv, pll, CNL_DPLL_ENABLE(pll->info->id));
+}
+
+static void tbt_pll_disable(struct drm_i915_private *dev_priv,
+			    struct intel_shared_dpll *pll)
+{
+	icl_pll_disable(dev_priv, pll, TBT_PLL_ENABLE);
+}
+
+static void mg_pll_disable(struct drm_i915_private *dev_priv,
+			   struct intel_shared_dpll *pll)
+{
+	i915_reg_t enable_reg =
+		MG_PLL_ENABLE(icl_pll_id_to_tc_port(pll->info->id));
+
+	icl_pll_disable(dev_priv, pll, enable_reg);
 }
 
 static void icl_dump_hw_state(struct drm_i915_private *dev_priv,
@@ -3197,20 +3211,32 @@ static void icl_dump_hw_state(struct drm_i915_private *dev_priv,
 		      hw_state->mg_pll_tdc_coldst_bias);
 }
 
-static const struct intel_shared_dpll_funcs icl_pll_funcs = {
-	.enable = icl_pll_enable,
-	.disable = icl_pll_disable,
-	.get_hw_state = icl_pll_get_hw_state,
+static const struct intel_shared_dpll_funcs combo_pll_funcs = {
+	.enable = combo_pll_enable,
+	.disable = combo_pll_disable,
+	.get_hw_state = combo_pll_get_hw_state,
+};
+
+static const struct intel_shared_dpll_funcs tbt_pll_funcs = {
+	.enable = tbt_pll_enable,
+	.disable = tbt_pll_disable,
+	.get_hw_state = tbt_pll_get_hw_state,
+};
+
+static const struct intel_shared_dpll_funcs mg_pll_funcs = {
+	.enable = mg_pll_enable,
+	.disable = mg_pll_disable,
+	.get_hw_state = mg_pll_get_hw_state,
 };
 
 static const struct dpll_info icl_plls[] = {
-	{ "DPLL 0",   &icl_pll_funcs, DPLL_ID_ICL_DPLL0,  0 },
-	{ "DPLL 1",   &icl_pll_funcs, DPLL_ID_ICL_DPLL1,  0 },
-	{ "TBT PLL",  &icl_pll_funcs, DPLL_ID_ICL_TBTPLL, 0 },
-	{ "MG PLL 1", &icl_pll_funcs, DPLL_ID_ICL_MGPLL1, 0 },
-	{ "MG PLL 2", &icl_pll_funcs, DPLL_ID_ICL_MGPLL2, 0 },
-	{ "MG PLL 3", &icl_pll_funcs, DPLL_ID_ICL_MGPLL3, 0 },
-	{ "MG PLL 4", &icl_pll_funcs, DPLL_ID_ICL_MGPLL4, 0 },
+	{ "DPLL 0",   &combo_pll_funcs, DPLL_ID_ICL_DPLL0,  0 },
+	{ "DPLL 1",   &combo_pll_funcs, DPLL_ID_ICL_DPLL1,  0 },
+	{ "TBT PLL",  &tbt_pll_funcs, DPLL_ID_ICL_TBTPLL, 0 },
+	{ "MG PLL 1", &mg_pll_funcs, DPLL_ID_ICL_MGPLL1, 0 },
+	{ "MG PLL 2", &mg_pll_funcs, DPLL_ID_ICL_MGPLL2, 0 },
+	{ "MG PLL 3", &mg_pll_funcs, DPLL_ID_ICL_MGPLL3, 0 },
+	{ "MG PLL 4", &mg_pll_funcs, DPLL_ID_ICL_MGPLL4, 0 },
 	{ },
 };
 
@@ -3220,6 +3246,18 @@ static const struct intel_dpll_mgr icl_pll_mgr = {
 	.dump_hw_state = icl_dump_hw_state,
 };
 
+static const struct dpll_info ehl_plls[] = {
+	{ "DPLL 0", &combo_pll_funcs, DPLL_ID_ICL_DPLL0, 0 },
+	{ "DPLL 1", &combo_pll_funcs, DPLL_ID_ICL_DPLL1, 0 },
+	{ },
+};
+
+static const struct intel_dpll_mgr ehl_pll_mgr = {
+	.dpll_info = ehl_plls,
+	.get_dpll = icl_get_dpll,
+	.dump_hw_state = icl_dump_hw_state,
+};
+
 /**
  * intel_shared_dpll_init - Initialize shared DPLLs
  * @dev: drm device
@@ -3233,7 +3271,9 @@ void intel_shared_dpll_init(struct drm_device *dev)
 	const struct dpll_info *dpll_info;
 	int i;
 
-	if (IS_ICELAKE(dev_priv))
+	if (IS_ELKHARTLAKE(dev_priv))
+		dpll_mgr = &ehl_pll_mgr;
+	else if (INTEL_GEN(dev_priv) >= 11)
 		dpll_mgr = &icl_pll_mgr;
 	else if (IS_CANNONLAKE(dev_priv))
 		dpll_mgr = &cnl_pll_mgr;
@@ -3271,31 +3311,29 @@ void intel_shared_dpll_init(struct drm_device *dev)
 
 /**
  * intel_get_shared_dpll - get a shared DPLL for CRTC and encoder combination
- * @crtc: CRTC
- * @crtc_state: atomic state for @crtc
+ * @crtc_state: atomic state for the crtc
  * @encoder: encoder
  *
  * Find an appropriate DPLL for the given CRTC and encoder combination. A
- * reference from the @crtc to the returned pll is registered in the atomic
- * state. That configuration is made effective by calling
+ * reference from the @crtc_state to the returned pll is registered in the
+ * atomic state. That configuration is made effective by calling
  * intel_shared_dpll_swap_state(). The reference should be released by calling
  * intel_release_shared_dpll().
  *
  * Returns:
- * A shared DPLL to be used by @crtc and @encoder with the given @crtc_state.
+ * A shared DPLL to be used by @crtc_state and @encoder.
  */
 struct intel_shared_dpll *
-intel_get_shared_dpll(struct intel_crtc *crtc,
-		      struct intel_crtc_state *crtc_state,
+intel_get_shared_dpll(struct intel_crtc_state *crtc_state,
 		      struct intel_encoder *encoder)
 {
-	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
+	struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
 	const struct intel_dpll_mgr *dpll_mgr = dev_priv->dpll_mgr;
 
 	if (WARN_ON(!dpll_mgr))
 		return NULL;
 
-	return dpll_mgr->get_dpll(crtc, crtc_state, encoder);
+	return dpll_mgr->get_dpll(crtc_state, encoder);
 }
 
 /**
diff --git a/drivers/gpu/drm/i915/intel_dpll_mgr.h b/drivers/gpu/drm/i915/intel_dpll_mgr.h
index 40e8391a92f2..bd8124cc81ed 100644
--- a/drivers/gpu/drm/i915/intel_dpll_mgr.h
+++ b/drivers/gpu/drm/i915/intel_dpll_mgr.h
@@ -327,8 +327,7 @@ void assert_shared_dpll(struct drm_i915_private *dev_priv,
 			bool state);
 #define assert_shared_dpll_enabled(d, p) assert_shared_dpll(d, p, true)
 #define assert_shared_dpll_disabled(d, p) assert_shared_dpll(d, p, false)
-struct intel_shared_dpll *intel_get_shared_dpll(struct intel_crtc *crtc,
-						struct intel_crtc_state *state,
+struct intel_shared_dpll *intel_get_shared_dpll(struct intel_crtc_state *state,
 						struct intel_encoder *encoder);
 void intel_release_shared_dpll(struct intel_shared_dpll *dpll,
 			       struct intel_crtc *crtc,
@@ -341,8 +340,6 @@ void intel_shared_dpll_init(struct drm_device *dev);
 
 void intel_dpll_dump_hw_state(struct drm_i915_private *dev_priv,
 			      struct intel_dpll_hw_state *hw_state);
-int icl_calc_dp_combo_pll_link(struct drm_i915_private *dev_priv,
-			       u32 pll_id);
 int cnl_hdmi_pll_ref_clock(struct drm_i915_private *dev_priv);
 enum intel_dpll_id icl_tc_port_to_pll_id(enum tc_port tc_port);
 bool intel_dpll_is_combophy(enum intel_dpll_id id);
diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h
index d5660ac1b0d6..a38b9cff5cd0 100644
--- a/drivers/gpu/drm/i915/intel_drv.h
+++ b/drivers/gpu/drm/i915/intel_drv.h
@@ -27,22 +27,24 @@
 
 #include <linux/async.h>
 #include <linux/i2c.h>
-#include <linux/hdmi.h>
 #include <linux/sched/clock.h>
 #include <linux/stackdepot.h>
-#include <drm/i915_drm.h>
-#include "i915_drv.h"
+
+#include <drm/drm_atomic.h>
 #include <drm/drm_crtc.h>
-#include <drm/drm_encoder.h>
-#include <drm/drm_fb_helper.h>
 #include <drm/drm_dp_dual_mode_helper.h>
 #include <drm/drm_dp_mst_helper.h>
+#include <drm/drm_encoder.h>
+#include <drm/drm_fb_helper.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_rect.h>
 #include <drm/drm_vblank.h>
-#include <drm/drm_atomic.h>
+#include <drm/i915_drm.h>
+#include <drm/i915_mei_hdcp_interface.h>
 #include <media/cec-notifier.h>
 
+#include "i915_drv.h"
+
 struct drm_printer;
 
 /**
@@ -325,6 +327,13 @@ struct intel_panel {
 
 struct intel_digital_port;
 
+enum check_link_response {
+	HDCP_LINK_PROTECTED	= 0,
+	HDCP_TOPOLOGY_CHANGE,
+	HDCP_LINK_INTEGRITY_FAILURE,
+	HDCP_REAUTH_REQUEST
+};
+
 /*
  * This structure serves as a translation layer between the generic HDCP code
  * and the bus-specific code. What that means is that HDCP over HDMI differs
@@ -397,6 +406,32 @@ struct intel_hdcp_shim {
 	/* Detects panel's hdcp capability. This is optional for HDMI. */
 	int (*hdcp_capable)(struct intel_digital_port *intel_dig_port,
 			    bool *hdcp_capable);
+
+	/* HDCP adaptation(DP/HDMI) required on the port */
+	enum hdcp_wired_protocol protocol;
+
+	/* Detects whether sink is HDCP2.2 capable */
+	int (*hdcp_2_2_capable)(struct intel_digital_port *intel_dig_port,
+				bool *capable);
+
+	/* Write HDCP2.2 messages */
+	int (*write_2_2_msg)(struct intel_digital_port *intel_dig_port,
+			     void *buf, size_t size);
+
+	/* Read HDCP2.2 messages */
+	int (*read_2_2_msg)(struct intel_digital_port *intel_dig_port,
+			    u8 msg_id, void *buf, size_t size);
+
+	/*
+	 * Implementation of DP HDCP2.2 Errata for the communication of stream
+	 * type to Receivers. In DP HDCP2.2 Stream type is one of the input to
+	 * the HDCP2.2 Cipher for En/De-Cryption. Not applicable for HDMI.
+	 */
+	int (*config_stream_type)(struct intel_digital_port *intel_dig_port,
+				  bool is_repeater, u8 type);
+
+	/* HDCP2.2 Link Integrity Check */
+	int (*check_2_2_link)(struct intel_digital_port *intel_dig_port);
 };
 
 struct intel_hdcp {
@@ -406,6 +441,50 @@ struct intel_hdcp {
 	u64 value;
 	struct delayed_work check_work;
 	struct work_struct prop_work;
+
+	/* HDCP1.4 Encryption status */
+	bool hdcp_encrypted;
+
+	/* HDCP2.2 related definitions */
+	/* Flag indicates whether this connector supports HDCP2.2 or not. */
+	bool hdcp2_supported;
+
+	/* HDCP2.2 Encryption status */
+	bool hdcp2_encrypted;
+
+	/*
+	 * Content Stream Type defined by content owner. TYPE0(0x0) content can
+	 * flow in the link protected by HDCP2.2 or HDCP1.4, where as TYPE1(0x1)
+	 * content can flow only through a link protected by HDCP2.2.
+	 */
+	u8 content_type;
+	struct hdcp_port_data port_data;
+
+	bool is_paired;
+	bool is_repeater;
+
+	/*
+	 * Count of ReceiverID_List received. Initialized to 0 at AKE_INIT.
+	 * Incremented after processing the RepeaterAuth_Send_ReceiverID_List.
+	 * When it rolls over re-auth has to be triggered.
+	 */
+	u32 seq_num_v;
+
+	/*
+	 * Count of RepeaterAuth_Stream_Manage msg propagated.
+	 * Initialized to 0 on AKE_INIT. Incremented after every successful
+	 * transmission of RepeaterAuth_Stream_Manage message. When it rolls
+	 * over re-Auth has to be triggered.
+	 */
+	u32 seq_num_m;
+
+	/*
+	 * Work queue to signal the CP_IRQ. Used for the waiters to read the
+	 * available information from HDCP DP sink.
+	 */
+	wait_queue_head_t cp_irq_queue;
+	atomic_t cp_irq_count;
+	int cp_irq_count_cached;
 };
 
 struct intel_connector {
@@ -480,6 +559,11 @@ struct intel_atomic_state {
 		 * state only when all crtc's are DPMS off.
 		 */
 		struct intel_cdclk_state actual;
+
+		int force_min_cdclk;
+		bool force_min_cdclk_changed;
+		/* pipe to which cd2x update is synchronized */
+		enum pipe pipe;
 	} cdclk;
 
 	bool dpll_set, modeset;
@@ -923,7 +1007,8 @@ struct intel_crtc_state {
 	struct intel_link_m_n fdi_m_n;
 
 	bool ips_enabled;
-	bool ips_force_disable;
+
+	bool crc_enabled;
 
 	bool enable_fbc;
 
@@ -944,13 +1029,30 @@ struct intel_crtc_state {
 	/* Gamma mode programmed on the pipe */
 	u32 gamma_mode;
 
+	union {
+		/* CSC mode programmed on the pipe */
+		u32 csc_mode;
+
+		/* CHV CGM mode */
+		u32 cgm_mode;
+	};
+
 	/* bitmask of visible planes (enum plane_id) */
 	u8 active_planes;
 	u8 nv12_planes;
+	u8 c8_planes;
 
 	/* bitmask of planes that will be updated during the commit */
 	u8 update_planes;
 
+	struct {
+		u32 enable;
+		u32 gcp;
+		union hdmi_infoframe avi;
+		union hdmi_infoframe spd;
+		union hdmi_infoframe hdmi;
+	} infoframes;
+
 	/* HDMI scrambling status */
 	bool hdmi_scrambling;
 
@@ -963,6 +1065,12 @@ struct intel_crtc_state {
 	/* Output down scaling is done in LSPCON device */
 	bool lspcon_downsampling;
 
+	/* enable pipe gamma? */
+	bool gamma_enable;
+
+	/* enable pipe csc? */
+	bool csc_enable;
+
 	/* Display Stream compression state */
 	struct {
 		bool compression_enable;
@@ -991,9 +1099,6 @@ struct intel_crtc {
 
 	struct intel_crtc_state *config;
 
-	/* global reset count when the last flip was submitted */
-	unsigned int reset_count;
-
 	/* Access to these should be protected by dev_priv->irq_lock. */
 	bool cpu_fifo_underrun_disabled;
 	bool pch_fifo_underrun_disabled;
@@ -1262,11 +1367,15 @@ struct intel_digital_port {
 				const struct intel_crtc_state *crtc_state,
 				unsigned int type,
 				const void *frame, ssize_t len);
+	void (*read_infoframe)(struct intel_encoder *encoder,
+			       const struct intel_crtc_state *crtc_state,
+			       unsigned int type,
+			       void *frame, ssize_t len);
 	void (*set_infoframes)(struct intel_encoder *encoder,
 			       bool enable,
 			       const struct intel_crtc_state *crtc_state,
 			       const struct drm_connector_state *conn_state);
-	bool (*infoframe_enabled)(struct intel_encoder *encoder,
+	u32 (*infoframes_enabled)(struct intel_encoder *encoder,
 				  const struct intel_crtc_state *pipe_config);
 };
 
@@ -1494,6 +1603,7 @@ void gen11_reset_rps_interrupts(struct drm_i915_private *dev_priv);
 void gen6_reset_rps_interrupts(struct drm_i915_private *dev_priv);
 void gen6_enable_rps_interrupts(struct drm_i915_private *dev_priv);
 void gen6_disable_rps_interrupts(struct drm_i915_private *dev_priv);
+void gen6_rps_reset_ei(struct drm_i915_private *dev_priv);
 
 static inline u32 gen6_sanitize_rps_pm_mask(const struct drm_i915_private *i915,
 					    u32 mask)
@@ -1521,85 +1631,8 @@ void gen9_reset_guc_interrupts(struct drm_i915_private *dev_priv);
 void gen9_enable_guc_interrupts(struct drm_i915_private *dev_priv);
 void gen9_disable_guc_interrupts(struct drm_i915_private *dev_priv);
 
-/* intel_crt.c */
-bool intel_crt_port_enabled(struct drm_i915_private *dev_priv,
-			    i915_reg_t adpa_reg, enum pipe *pipe);
-void intel_crt_init(struct drm_i915_private *dev_priv);
-void intel_crt_reset(struct drm_encoder *encoder);
-
-/* intel_ddi.c */
-void intel_ddi_fdi_post_disable(struct intel_encoder *intel_encoder,
-				const struct intel_crtc_state *old_crtc_state,
-				const struct drm_connector_state *old_conn_state);
-void hsw_fdi_link_train(struct intel_crtc *crtc,
-			const struct intel_crtc_state *crtc_state);
-void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port);
-bool intel_ddi_get_hw_state(struct intel_encoder *encoder, enum pipe *pipe);
-void intel_ddi_enable_transcoder_func(const struct intel_crtc_state *crtc_state);
-void intel_ddi_disable_transcoder_func(const struct intel_crtc_state *crtc_state);
-void intel_ddi_enable_pipe_clock(const struct intel_crtc_state *crtc_state);
-void intel_ddi_disable_pipe_clock(const  struct intel_crtc_state *crtc_state);
-void intel_ddi_set_pipe_settings(const struct intel_crtc_state *crtc_state);
-void intel_ddi_prepare_link_retrain(struct intel_dp *intel_dp);
-bool intel_ddi_connector_get_hw_state(struct intel_connector *intel_connector);
-void intel_ddi_get_config(struct intel_encoder *encoder,
-			  struct intel_crtc_state *pipe_config);
-
-void intel_ddi_set_vc_payload_alloc(const struct intel_crtc_state *crtc_state,
-				    bool state);
-void intel_ddi_compute_min_voltage_level(struct drm_i915_private *dev_priv,
-					 struct intel_crtc_state *crtc_state);
-u32 bxt_signal_levels(struct intel_dp *intel_dp);
-u32 ddi_signal_levels(struct intel_dp *intel_dp);
-u8 intel_ddi_dp_voltage_max(struct intel_encoder *encoder);
-u8 intel_ddi_dp_pre_emphasis_max(struct intel_encoder *encoder,
-				 u8 voltage_swing);
-int intel_ddi_toggle_hdcp_signalling(struct intel_encoder *intel_encoder,
-				     bool enable);
-void icl_sanitize_encoder_pll_mapping(struct intel_encoder *encoder);
-int cnl_calc_wrpll_link(struct drm_i915_private *dev_priv,
-			enum intel_dpll_id pll_id);
-
-unsigned int intel_fb_align_height(const struct drm_framebuffer *fb,
-				   int color_plane, unsigned int height);
-
-/* intel_audio.c */
-void intel_init_audio_hooks(struct drm_i915_private *dev_priv);
-void intel_audio_codec_enable(struct intel_encoder *encoder,
-			      const struct intel_crtc_state *crtc_state,
-			      const struct drm_connector_state *conn_state);
-void intel_audio_codec_disable(struct intel_encoder *encoder,
-			       const struct intel_crtc_state *old_crtc_state,
-			       const struct drm_connector_state *old_conn_state);
-void i915_audio_component_init(struct drm_i915_private *dev_priv);
-void i915_audio_component_cleanup(struct drm_i915_private *dev_priv);
-void intel_audio_init(struct drm_i915_private *dev_priv);
-void intel_audio_deinit(struct drm_i915_private *dev_priv);
-
-/* intel_cdclk.c */
-int intel_crtc_compute_min_cdclk(const struct intel_crtc_state *crtc_state);
-void skl_init_cdclk(struct drm_i915_private *dev_priv);
-void skl_uninit_cdclk(struct drm_i915_private *dev_priv);
-void cnl_init_cdclk(struct drm_i915_private *dev_priv);
-void cnl_uninit_cdclk(struct drm_i915_private *dev_priv);
-void bxt_init_cdclk(struct drm_i915_private *dev_priv);
-void bxt_uninit_cdclk(struct drm_i915_private *dev_priv);
-void icl_init_cdclk(struct drm_i915_private *dev_priv);
-void icl_uninit_cdclk(struct drm_i915_private *dev_priv);
-void intel_init_cdclk_hooks(struct drm_i915_private *dev_priv);
-void intel_update_max_cdclk(struct drm_i915_private *dev_priv);
-void intel_update_cdclk(struct drm_i915_private *dev_priv);
-void intel_update_rawclk(struct drm_i915_private *dev_priv);
-bool intel_cdclk_needs_modeset(const struct intel_cdclk_state *a,
-			       const struct intel_cdclk_state *b);
-bool intel_cdclk_changed(const struct intel_cdclk_state *a,
-			 const struct intel_cdclk_state *b);
-void intel_set_cdclk(struct drm_i915_private *dev_priv,
-		     const struct intel_cdclk_state *cdclk_state);
-void intel_dump_cdclk_state(const struct intel_cdclk_state *cdclk_state,
-			    const char *context);
-
 /* intel_display.c */
+void intel_plane_destroy(struct drm_plane *plane);
 void i830_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe);
 void i830_disable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe);
 enum pipe intel_crtc_pch_transcoder(struct intel_crtc *crtc);
@@ -1614,6 +1647,8 @@ void intel_init_display_hooks(struct drm_i915_private *dev_priv);
 unsigned int intel_fb_xy_to_linear(int x, int y,
 				   const struct intel_plane_state *state,
 				   int plane);
+unsigned int intel_fb_align_height(const struct drm_framebuffer *fb,
+				   int color_plane, unsigned int height);
 void intel_add_fb_offsets(int *x, int *y,
 			  const struct intel_plane_state *state, int plane);
 unsigned int intel_rotation_info_size(const struct intel_rotation_info *rot_info);
@@ -1740,7 +1775,7 @@ void intel_dp_get_m_n(struct intel_crtc *crtc,
 void intel_dp_set_m_n(const struct intel_crtc_state *crtc_state,
 		      enum link_m_n_set m_n);
 int intel_dotclock_calculate(int link_freq, const struct intel_link_m_n *m_n);
-bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state, int target_clock,
+bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state,
 			struct dpll *best_clock);
 int chv_calc_dpll_params(int refclk, struct dpll *pll_clock);
 
@@ -1781,106 +1816,9 @@ unsigned int i9xx_plane_max_stride(struct intel_plane *plane,
 				   u32 pixel_format, u64 modifier,
 				   unsigned int rotation);
 
-/* intel_connector.c */
-int intel_connector_init(struct intel_connector *connector);
-struct intel_connector *intel_connector_alloc(void);
-void intel_connector_free(struct intel_connector *connector);
-void intel_connector_destroy(struct drm_connector *connector);
-int intel_connector_register(struct drm_connector *connector);
-void intel_connector_unregister(struct drm_connector *connector);
-void intel_connector_attach_encoder(struct intel_connector *connector,
-				    struct intel_encoder *encoder);
-bool intel_connector_get_hw_state(struct intel_connector *connector);
-enum pipe intel_connector_get_pipe(struct intel_connector *connector);
-int intel_connector_update_modes(struct drm_connector *connector,
-				 struct edid *edid);
-int intel_ddc_get_modes(struct drm_connector *c, struct i2c_adapter *adapter);
-void intel_attach_force_audio_property(struct drm_connector *connector);
-void intel_attach_broadcast_rgb_property(struct drm_connector *connector);
-void intel_attach_aspect_ratio_property(struct drm_connector *connector);
-
-/* intel_csr.c */
-void intel_csr_ucode_init(struct drm_i915_private *);
-void intel_csr_load_program(struct drm_i915_private *);
-void intel_csr_ucode_fini(struct drm_i915_private *);
-void intel_csr_ucode_suspend(struct drm_i915_private *);
-void intel_csr_ucode_resume(struct drm_i915_private *);
-
-/* intel_dp.c */
-bool intel_dp_port_enabled(struct drm_i915_private *dev_priv,
-			   i915_reg_t dp_reg, enum port port,
-			   enum pipe *pipe);
-bool intel_dp_init(struct drm_i915_private *dev_priv, i915_reg_t output_reg,
-		   enum port port);
-bool intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
-			     struct intel_connector *intel_connector);
-void intel_dp_set_link_params(struct intel_dp *intel_dp,
-			      int link_rate, u8 lane_count,
-			      bool link_mst);
-int intel_dp_get_link_train_fallback_values(struct intel_dp *intel_dp,
-					    int link_rate, u8 lane_count);
+/* intel_dp_link_training.c */
 void intel_dp_start_link_train(struct intel_dp *intel_dp);
 void intel_dp_stop_link_train(struct intel_dp *intel_dp);
-int intel_dp_retrain_link(struct intel_encoder *encoder,
-			  struct drm_modeset_acquire_ctx *ctx);
-void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode);
-void intel_dp_sink_set_decompression_state(struct intel_dp *intel_dp,
-					   const struct intel_crtc_state *crtc_state,
-					   bool enable);
-void intel_dp_encoder_reset(struct drm_encoder *encoder);
-void intel_dp_encoder_suspend(struct intel_encoder *intel_encoder);
-void intel_dp_encoder_flush_work(struct drm_encoder *encoder);
-int intel_dp_compute_config(struct intel_encoder *encoder,
-			    struct intel_crtc_state *pipe_config,
-			    struct drm_connector_state *conn_state);
-bool intel_dp_is_edp(struct intel_dp *intel_dp);
-bool intel_dp_is_port_edp(struct drm_i915_private *dev_priv, enum port port);
-enum irqreturn intel_dp_hpd_pulse(struct intel_digital_port *intel_dig_port,
-				  bool long_hpd);
-void intel_edp_backlight_on(const struct intel_crtc_state *crtc_state,
-			    const struct drm_connector_state *conn_state);
-void intel_edp_backlight_off(const struct drm_connector_state *conn_state);
-void intel_edp_panel_vdd_on(struct intel_dp *intel_dp);
-void intel_edp_panel_on(struct intel_dp *intel_dp);
-void intel_edp_panel_off(struct intel_dp *intel_dp);
-void intel_dp_mst_suspend(struct drm_i915_private *dev_priv);
-void intel_dp_mst_resume(struct drm_i915_private *dev_priv);
-int intel_dp_max_link_rate(struct intel_dp *intel_dp);
-int intel_dp_max_lane_count(struct intel_dp *intel_dp);
-int intel_dp_rate_select(struct intel_dp *intel_dp, int rate);
-void intel_dp_hot_plug(struct intel_encoder *intel_encoder);
-void intel_power_sequencer_reset(struct drm_i915_private *dev_priv);
-u32 intel_dp_pack_aux(const u8 *src, int src_bytes);
-void intel_plane_destroy(struct drm_plane *plane);
-void intel_edp_drrs_enable(struct intel_dp *intel_dp,
-			   const struct intel_crtc_state *crtc_state);
-void intel_edp_drrs_disable(struct intel_dp *intel_dp,
-			    const struct intel_crtc_state *crtc_state);
-void intel_edp_drrs_invalidate(struct drm_i915_private *dev_priv,
-			       unsigned int frontbuffer_bits);
-void intel_edp_drrs_flush(struct drm_i915_private *dev_priv,
-			  unsigned int frontbuffer_bits);
-
-void
-intel_dp_program_link_training_pattern(struct intel_dp *intel_dp,
-				       u8 dp_train_pat);
-void
-intel_dp_set_signal_levels(struct intel_dp *intel_dp);
-void intel_dp_set_idle_link_train(struct intel_dp *intel_dp);
-u8
-intel_dp_voltage_max(struct intel_dp *intel_dp);
-u8
-intel_dp_pre_emphasis_max(struct intel_dp *intel_dp, u8 voltage_swing);
-void intel_dp_compute_rate(struct intel_dp *intel_dp, int port_clock,
-			   u8 *link_bw, u8 *rate_select);
-bool intel_dp_source_supports_hbr2(struct intel_dp *intel_dp);
-bool intel_dp_source_supports_hbr3(struct intel_dp *intel_dp);
-bool
-intel_dp_get_link_status(struct intel_dp *intel_dp, u8 link_status[DP_LINK_STATUS_SIZE]);
-u16 intel_dp_dsc_get_output_bpp(int link_clock, u8 lane_count,
-				int mode_clock, int mode_hdisplay);
-u8 intel_dp_dsc_get_slice_count(struct intel_dp *intel_dp, int mode_clock,
-				int mode_hdisplay);
 
 /* intel_vdsc.c */
 int intel_dp_compute_dsc_params(struct intel_dp *intel_dp,
@@ -1888,18 +1826,6 @@ int intel_dp_compute_dsc_params(struct intel_dp *intel_dp,
 enum intel_display_power_domain
 intel_dsc_power_domain(const struct intel_crtc_state *crtc_state);
 
-static inline unsigned int intel_dp_unused_lane_mask(int lane_count)
-{
-	return ~((1 << lane_count) - 1) & 0xf;
-}
-
-bool intel_dp_read_dpcd(struct intel_dp *intel_dp);
-int intel_dp_link_required(int pixel_clock, int bpp);
-int intel_dp_max_data_rate(int max_link_clock, int max_lanes);
-bool intel_digital_port_connected(struct intel_encoder *encoder);
-void icl_tc_phy_disconnect(struct drm_i915_private *dev_priv,
-			   struct intel_digital_port *dig_port);
-
 /* intel_dp_aux_backlight.c */
 int intel_dp_aux_init_backlight_funcs(struct intel_connector *intel_connector);
 
@@ -1915,100 +1841,11 @@ void icl_dsi_init(struct drm_i915_private *dev_priv);
 /* intel_dsi_dcs_backlight.c */
 int intel_dsi_dcs_init_backlight_funcs(struct intel_connector *intel_connector);
 
-/* intel_dvo.c */
-void intel_dvo_init(struct drm_i915_private *dev_priv);
 /* intel_hotplug.c */
 void intel_hpd_poll_init(struct drm_i915_private *dev_priv);
 bool intel_encoder_hotplug(struct intel_encoder *encoder,
 			   struct intel_connector *connector);
 
-/* legacy fbdev emulation in intel_fbdev.c */
-#ifdef CONFIG_DRM_FBDEV_EMULATION
-extern int intel_fbdev_init(struct drm_device *dev);
-extern void intel_fbdev_initial_config_async(struct drm_device *dev);
-extern void intel_fbdev_unregister(struct drm_i915_private *dev_priv);
-extern void intel_fbdev_fini(struct drm_i915_private *dev_priv);
-extern void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous);
-extern void intel_fbdev_output_poll_changed(struct drm_device *dev);
-extern void intel_fbdev_restore_mode(struct drm_device *dev);
-#else
-static inline int intel_fbdev_init(struct drm_device *dev)
-{
-	return 0;
-}
-
-static inline void intel_fbdev_initial_config_async(struct drm_device *dev)
-{
-}
-
-static inline void intel_fbdev_unregister(struct drm_i915_private *dev_priv)
-{
-}
-
-static inline void intel_fbdev_fini(struct drm_i915_private *dev_priv)
-{
-}
-
-static inline void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous)
-{
-}
-
-static inline void intel_fbdev_output_poll_changed(struct drm_device *dev)
-{
-}
-
-static inline void intel_fbdev_restore_mode(struct drm_device *dev)
-{
-}
-#endif
-
-/* intel_fbc.c */
-void intel_fbc_choose_crtc(struct drm_i915_private *dev_priv,
-			   struct intel_atomic_state *state);
-bool intel_fbc_is_active(struct drm_i915_private *dev_priv);
-void intel_fbc_pre_update(struct intel_crtc *crtc,
-			  struct intel_crtc_state *crtc_state,
-			  struct intel_plane_state *plane_state);
-void intel_fbc_post_update(struct intel_crtc *crtc);
-void intel_fbc_init(struct drm_i915_private *dev_priv);
-void intel_fbc_init_pipe_state(struct drm_i915_private *dev_priv);
-void intel_fbc_enable(struct intel_crtc *crtc,
-		      struct intel_crtc_state *crtc_state,
-		      struct intel_plane_state *plane_state);
-void intel_fbc_disable(struct intel_crtc *crtc);
-void intel_fbc_global_disable(struct drm_i915_private *dev_priv);
-void intel_fbc_invalidate(struct drm_i915_private *dev_priv,
-			  unsigned int frontbuffer_bits,
-			  enum fb_op_origin origin);
-void intel_fbc_flush(struct drm_i915_private *dev_priv,
-		     unsigned int frontbuffer_bits, enum fb_op_origin origin);
-void intel_fbc_cleanup_cfb(struct drm_i915_private *dev_priv);
-void intel_fbc_handle_fifo_underrun_irq(struct drm_i915_private *dev_priv);
-int intel_fbc_reset_underrun(struct drm_i915_private *dev_priv);
-
-/* intel_hdmi.c */
-void intel_hdmi_init(struct drm_i915_private *dev_priv, i915_reg_t hdmi_reg,
-		     enum port port);
-void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port,
-			       struct intel_connector *intel_connector);
-struct intel_hdmi *enc_to_intel_hdmi(struct drm_encoder *encoder);
-int intel_hdmi_compute_config(struct intel_encoder *encoder,
-			      struct intel_crtc_state *pipe_config,
-			      struct drm_connector_state *conn_state);
-bool intel_hdmi_handle_sink_scrambling(struct intel_encoder *encoder,
-				       struct drm_connector *connector,
-				       bool high_tmds_clock_ratio,
-				       bool scrambling);
-void intel_dp_dual_mode_set_tmds_output(struct intel_hdmi *hdmi, bool enable);
-void intel_infoframe_init(struct intel_digital_port *intel_dig_port);
-
-/* intel_lvds.c */
-bool intel_lvds_port_enabled(struct drm_i915_private *dev_priv,
-			     i915_reg_t lvds_reg, enum pipe *pipe);
-void intel_lvds_init(struct drm_i915_private *dev_priv);
-struct intel_encoder *intel_get_lvds_encoder(struct drm_device *dev);
-bool intel_is_dual_link_lvds(struct drm_device *dev);
-
 /* intel_overlay.c */
 void intel_overlay_setup(struct drm_i915_private *dev_priv);
 void intel_overlay_cleanup(struct drm_i915_private *dev_priv);
@@ -2019,86 +1856,6 @@ int intel_overlay_attrs_ioctl(struct drm_device *dev, void *data,
 			      struct drm_file *file_priv);
 void intel_overlay_reset(struct drm_i915_private *dev_priv);
 
-
-/* intel_panel.c */
-int intel_panel_init(struct intel_panel *panel,
-		     struct drm_display_mode *fixed_mode,
-		     struct drm_display_mode *downclock_mode);
-void intel_panel_fini(struct intel_panel *panel);
-void intel_fixed_panel_mode(const struct drm_display_mode *fixed_mode,
-			    struct drm_display_mode *adjusted_mode);
-void intel_pch_panel_fitting(struct intel_crtc *crtc,
-			     struct intel_crtc_state *pipe_config,
-			     int fitting_mode);
-void intel_gmch_panel_fitting(struct intel_crtc *crtc,
-			      struct intel_crtc_state *pipe_config,
-			      int fitting_mode);
-void intel_panel_set_backlight_acpi(const struct drm_connector_state *conn_state,
-				    u32 level, u32 max);
-int intel_panel_setup_backlight(struct drm_connector *connector,
-				enum pipe pipe);
-void intel_panel_enable_backlight(const struct intel_crtc_state *crtc_state,
-				  const struct drm_connector_state *conn_state);
-void intel_panel_update_backlight(struct intel_encoder *encoder,
-				  const struct intel_crtc_state *crtc_state,
-				  const struct drm_connector_state *conn_state);
-void intel_panel_disable_backlight(const struct drm_connector_state *old_conn_state);
-extern struct drm_display_mode *intel_find_panel_downclock(
-				struct drm_i915_private *dev_priv,
-				struct drm_display_mode *fixed_mode,
-				struct drm_connector *connector);
-
-#if IS_ENABLED(CONFIG_BACKLIGHT_CLASS_DEVICE)
-int intel_backlight_device_register(struct intel_connector *connector);
-void intel_backlight_device_unregister(struct intel_connector *connector);
-#else /* CONFIG_BACKLIGHT_CLASS_DEVICE */
-static inline int intel_backlight_device_register(struct intel_connector *connector)
-{
-	return 0;
-}
-static inline void intel_backlight_device_unregister(struct intel_connector *connector)
-{
-}
-#endif /* CONFIG_BACKLIGHT_CLASS_DEVICE */
-
-/* intel_hdcp.c */
-void intel_hdcp_atomic_check(struct drm_connector *connector,
-			     struct drm_connector_state *old_state,
-			     struct drm_connector_state *new_state);
-int intel_hdcp_init(struct intel_connector *connector,
-		    const struct intel_hdcp_shim *hdcp_shim);
-int intel_hdcp_enable(struct intel_connector *connector);
-int intel_hdcp_disable(struct intel_connector *connector);
-int intel_hdcp_check_link(struct intel_connector *connector);
-bool is_hdcp_supported(struct drm_i915_private *dev_priv, enum port port);
-bool intel_hdcp_capable(struct intel_connector *connector);
-
-/* intel_psr.c */
-#define CAN_PSR(dev_priv) (HAS_PSR(dev_priv) && dev_priv->psr.sink_support)
-void intel_psr_init_dpcd(struct intel_dp *intel_dp);
-void intel_psr_enable(struct intel_dp *intel_dp,
-		      const struct intel_crtc_state *crtc_state);
-void intel_psr_disable(struct intel_dp *intel_dp,
-		      const struct intel_crtc_state *old_crtc_state);
-int intel_psr_set_debugfs_mode(struct drm_i915_private *dev_priv,
-			       struct drm_modeset_acquire_ctx *ctx,
-			       u64 value);
-void intel_psr_invalidate(struct drm_i915_private *dev_priv,
-			  unsigned frontbuffer_bits,
-			  enum fb_op_origin origin);
-void intel_psr_flush(struct drm_i915_private *dev_priv,
-		     unsigned frontbuffer_bits,
-		     enum fb_op_origin origin);
-void intel_psr_init(struct drm_i915_private *dev_priv);
-void intel_psr_compute_config(struct intel_dp *intel_dp,
-			      struct intel_crtc_state *crtc_state);
-void intel_psr_irq_control(struct drm_i915_private *dev_priv, u32 debug);
-void intel_psr_irq_handler(struct drm_i915_private *dev_priv, u32 psr_iir);
-void intel_psr_short_pulse(struct intel_dp *intel_dp);
-int intel_psr_wait_for_idle(const struct intel_crtc_state *new_crtc_state,
-			    u32 *out_value);
-bool intel_psr_enabled(struct intel_dp *intel_dp);
-
 /* intel_quirks.c */
 void intel_init_quirks(struct drm_i915_private *dev_priv);
 
@@ -2153,20 +1910,26 @@ void icl_dbuf_slices_update(struct drm_i915_private *dev_priv,
 			    u8 req_slices);
 
 static inline void
-assert_rpm_device_not_suspended(struct drm_i915_private *i915)
+assert_rpm_device_not_suspended(struct i915_runtime_pm *rpm)
 {
-	WARN_ONCE(i915->runtime_pm.suspended,
+	WARN_ONCE(rpm->suspended,
 		  "Device suspended during HW access\n");
 }
 
 static inline void
-assert_rpm_wakelock_held(struct drm_i915_private *i915)
+__assert_rpm_wakelock_held(struct i915_runtime_pm *rpm)
 {
-	assert_rpm_device_not_suspended(i915);
-	WARN_ONCE(!atomic_read(&i915->runtime_pm.wakeref_count),
+	assert_rpm_device_not_suspended(rpm);
+	WARN_ONCE(!atomic_read(&rpm->wakeref_count),
 		  "RPM wakelock ref not held during HW access");
 }
 
+static inline void
+assert_rpm_wakelock_held(struct drm_i915_private *i915)
+{
+	__assert_rpm_wakelock_held(&i915->runtime_pm);
+}
+
 /**
  * disable_rpm_wakeref_asserts - disable the RPM assert checks
  * @i915: i915 device instance
@@ -2242,101 +2005,6 @@ void chv_phy_powergate_lanes(struct intel_encoder *encoder,
 bool chv_phy_powergate_ch(struct drm_i915_private *dev_priv, enum dpio_phy phy,
 			  enum dpio_channel ch, bool override);
 
-
-/* intel_pm.c */
-void intel_init_clock_gating(struct drm_i915_private *dev_priv);
-void intel_suspend_hw(struct drm_i915_private *dev_priv);
-int ilk_wm_max_level(const struct drm_i915_private *dev_priv);
-void intel_update_watermarks(struct intel_crtc *crtc);
-void intel_init_pm(struct drm_i915_private *dev_priv);
-void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv);
-void intel_pm_setup(struct drm_i915_private *dev_priv);
-void intel_gpu_ips_init(struct drm_i915_private *dev_priv);
-void intel_gpu_ips_teardown(void);
-void intel_init_gt_powersave(struct drm_i915_private *dev_priv);
-void intel_cleanup_gt_powersave(struct drm_i915_private *dev_priv);
-void intel_sanitize_gt_powersave(struct drm_i915_private *dev_priv);
-void intel_enable_gt_powersave(struct drm_i915_private *dev_priv);
-void intel_disable_gt_powersave(struct drm_i915_private *dev_priv);
-void intel_suspend_gt_powersave(struct drm_i915_private *dev_priv);
-void gen6_rps_busy(struct drm_i915_private *dev_priv);
-void gen6_rps_reset_ei(struct drm_i915_private *dev_priv);
-void gen6_rps_idle(struct drm_i915_private *dev_priv);
-void gen6_rps_boost(struct i915_request *rq, struct intel_rps_client *rps);
-void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv);
-void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv);
-void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv);
-void skl_wm_get_hw_state(struct drm_i915_private *dev_priv);
-void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
-			       struct skl_ddb_entry *ddb_y,
-			       struct skl_ddb_entry *ddb_uv);
-void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
-			  struct skl_ddb_allocation *ddb /* out */);
-void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
-			      struct skl_pipe_wm *out);
-void g4x_wm_sanitize(struct drm_i915_private *dev_priv);
-void vlv_wm_sanitize(struct drm_i915_private *dev_priv);
-bool intel_can_enable_sagv(struct drm_atomic_state *state);
-int intel_enable_sagv(struct drm_i915_private *dev_priv);
-int intel_disable_sagv(struct drm_i915_private *dev_priv);
-bool skl_wm_level_equals(const struct skl_wm_level *l1,
-			 const struct skl_wm_level *l2);
-bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
-				 const struct skl_ddb_entry entries[],
-				 int num_entries, int ignore_idx);
-void skl_write_plane_wm(struct intel_plane *plane,
-			const struct intel_crtc_state *crtc_state);
-void skl_write_cursor_wm(struct intel_plane *plane,
-			 const struct intel_crtc_state *crtc_state);
-bool ilk_disable_lp_wm(struct drm_device *dev);
-int skl_check_pipe_max_pixel_rate(struct intel_crtc *intel_crtc,
-				  struct intel_crtc_state *cstate);
-void intel_init_ipc(struct drm_i915_private *dev_priv);
-void intel_enable_ipc(struct drm_i915_private *dev_priv);
-
-/* intel_sdvo.c */
-bool intel_sdvo_port_enabled(struct drm_i915_private *dev_priv,
-			     i915_reg_t sdvo_reg, enum pipe *pipe);
-bool intel_sdvo_init(struct drm_i915_private *dev_priv,
-		     i915_reg_t reg, enum port port);
-
-
-/* intel_sprite.c */
-int intel_usecs_to_scanlines(const struct drm_display_mode *adjusted_mode,
-			     int usecs);
-struct intel_plane *intel_sprite_plane_create(struct drm_i915_private *dev_priv,
-					      enum pipe pipe, int plane);
-int intel_sprite_set_colorkey_ioctl(struct drm_device *dev, void *data,
-				    struct drm_file *file_priv);
-void intel_pipe_update_start(const struct intel_crtc_state *new_crtc_state);
-void intel_pipe_update_end(struct intel_crtc_state *new_crtc_state);
-int intel_plane_check_stride(const struct intel_plane_state *plane_state);
-int intel_plane_check_src_coordinates(struct intel_plane_state *plane_state);
-int chv_plane_check_rotation(const struct intel_plane_state *plane_state);
-struct intel_plane *
-skl_universal_plane_create(struct drm_i915_private *dev_priv,
-			   enum pipe pipe, enum plane_id plane_id);
-
-static inline bool icl_is_nv12_y_plane(enum plane_id id)
-{
-	/* Don't need to do a gen check, these planes are only available on gen11 */
-	if (id == PLANE_SPRITE4 || id == PLANE_SPRITE5)
-		return true;
-
-	return false;
-}
-
-static inline bool icl_is_hdr_plane(struct intel_plane *plane)
-{
-	if (INTEL_GEN(to_i915(plane->base.dev)) < 11)
-		return false;
-
-	return plane->id < PLANE_SPRITE2;
-}
-
-/* intel_tv.c */
-void intel_tv_init(struct drm_i915_private *dev_priv);
-
 /* intel_atomic.c */
 int intel_digital_connector_atomic_get_property(struct drm_connector *connector,
 						const struct drm_connector_state *state,
@@ -2373,64 +2041,4 @@ int intel_atomic_setup_scalers(struct drm_i915_private *dev_priv,
 			       struct intel_crtc *intel_crtc,
 			       struct intel_crtc_state *crtc_state);
 
-/* intel_atomic_plane.c */
-struct intel_plane *intel_plane_alloc(void);
-void intel_plane_free(struct intel_plane *plane);
-struct drm_plane_state *intel_plane_duplicate_state(struct drm_plane *plane);
-void intel_plane_destroy_state(struct drm_plane *plane,
-			       struct drm_plane_state *state);
-extern const struct drm_plane_helper_funcs intel_plane_helper_funcs;
-void skl_update_planes_on_crtc(struct intel_atomic_state *state,
-			       struct intel_crtc *crtc);
-void i9xx_update_planes_on_crtc(struct intel_atomic_state *state,
-				struct intel_crtc *crtc);
-int intel_plane_atomic_check_with_state(const struct intel_crtc_state *old_crtc_state,
-					struct intel_crtc_state *crtc_state,
-					const struct intel_plane_state *old_plane_state,
-					struct intel_plane_state *intel_state);
-
-/* intel_color.c */
-void intel_color_init(struct intel_crtc *crtc);
-int intel_color_check(struct intel_crtc_state *crtc_state);
-void intel_color_commit(const struct intel_crtc_state *crtc_state);
-void intel_color_load_luts(const struct intel_crtc_state *crtc_state);
-
-/* intel_lspcon.c */
-bool lspcon_init(struct intel_digital_port *intel_dig_port);
-void lspcon_resume(struct intel_lspcon *lspcon);
-void lspcon_wait_pcon_mode(struct intel_lspcon *lspcon);
-void lspcon_write_infoframe(struct intel_encoder *encoder,
-			    const struct intel_crtc_state *crtc_state,
-			    unsigned int type,
-			    const void *buf, ssize_t len);
-void lspcon_set_infoframes(struct intel_encoder *encoder,
-			   bool enable,
-			   const struct intel_crtc_state *crtc_state,
-			   const struct drm_connector_state *conn_state);
-bool lspcon_infoframe_enabled(struct intel_encoder *encoder,
-			      const struct intel_crtc_state *pipe_config);
-void lspcon_ycbcr420_config(struct drm_connector *connector,
-			    struct intel_crtc_state *crtc_state);
-
-/* intel_pipe_crc.c */
-#ifdef CONFIG_DEBUG_FS
-int intel_crtc_set_crc_source(struct drm_crtc *crtc, const char *source_name);
-int intel_crtc_verify_crc_source(struct drm_crtc *crtc,
-				 const char *source_name, size_t *values_cnt);
-const char *const *intel_crtc_get_crc_sources(struct drm_crtc *crtc,
-					      size_t *count);
-void intel_crtc_disable_pipe_crc(struct intel_crtc *crtc);
-void intel_crtc_enable_pipe_crc(struct intel_crtc *crtc);
-#else
-#define intel_crtc_set_crc_source NULL
-#define intel_crtc_verify_crc_source NULL
-#define intel_crtc_get_crc_sources NULL
-static inline void intel_crtc_disable_pipe_crc(struct intel_crtc *crtc)
-{
-}
-
-static inline void intel_crtc_enable_pipe_crc(struct intel_crtc *crtc)
-{
-}
-#endif
 #endif /* __INTEL_DRV_H__ */
diff --git a/drivers/gpu/drm/i915/intel_dsi.h b/drivers/gpu/drm/i915/intel_dsi.h
index a9a19778dc7f..705a609050c0 100644
--- a/drivers/gpu/drm/i915/intel_dsi.h
+++ b/drivers/gpu/drm/i915/intel_dsi.h
@@ -189,7 +189,6 @@ void bxt_dsi_reset_clocks(struct intel_encoder *encoder, enum port port);
 
 /* intel_dsi_vbt.c */
 bool intel_dsi_vbt_init(struct intel_dsi *intel_dsi, u16 panel_id);
-int intel_dsi_vbt_get_modes(struct intel_dsi *intel_dsi);
 void intel_dsi_vbt_exec_sequence(struct intel_dsi *intel_dsi,
 				 enum mipi_seq seq_id);
 void intel_dsi_msleep(struct intel_dsi *intel_dsi, int msec);
diff --git a/drivers/gpu/drm/i915/intel_dsi_vbt.c b/drivers/gpu/drm/i915/intel_dsi_vbt.c
index 06a11c35a784..3074448446bc 100644
--- a/drivers/gpu/drm/i915/intel_dsi_vbt.c
+++ b/drivers/gpu/drm/i915/intel_dsi_vbt.c
@@ -194,7 +194,7 @@ static const u8 *mipi_exec_send_packet(struct intel_dsi *intel_dsi,
 		break;
 	}
 
-	if (!IS_ICELAKE(dev_priv))
+	if (INTEL_GEN(dev_priv) < 11)
 		vlv_dsi_wait_for_fifo_empty(intel_dsi, port);
 
 out:
@@ -365,7 +365,7 @@ static const u8 *mipi_exec_gpio(struct intel_dsi *intel_dsi, const u8 *data)
 	/* pull up/down */
 	value = *data++ & 1;
 
-	if (IS_ICELAKE(dev_priv))
+	if (INTEL_GEN(dev_priv) >= 11)
 		icl_exec_gpio(dev_priv, gpio_source, gpio_index, value);
 	else if (IS_VALLEYVIEW(dev_priv))
 		vlv_exec_gpio(dev_priv, gpio_source, gpio_number, value);
@@ -532,24 +532,6 @@ void intel_dsi_msleep(struct intel_dsi *intel_dsi, int msec)
 	msleep(msec);
 }
 
-int intel_dsi_vbt_get_modes(struct intel_dsi *intel_dsi)
-{
-	struct intel_connector *connector = intel_dsi->attached_connector;
-	struct drm_device *dev = intel_dsi->base.base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct drm_display_mode *mode;
-
-	mode = drm_mode_duplicate(dev, dev_priv->vbt.lfp_lvds_vbt_mode);
-	if (!mode)
-		return 0;
-
-	mode->type |= DRM_MODE_TYPE_PREFERRED;
-
-	drm_mode_probed_add(&connector->base, mode);
-
-	return 1;
-}
-
 #define ICL_PREPARE_CNT_MAX	0x7
 #define ICL_CLK_ZERO_CNT_MAX	0xf
 #define ICL_TRAIL_CNT_MAX	0x7
@@ -890,7 +872,7 @@ bool intel_dsi_vbt_init(struct intel_dsi *intel_dsi, u16 panel_id)
 
 	intel_dsi->burst_mode_ratio = burst_mode_ratio;
 
-	if (IS_ICELAKE(dev_priv))
+	if (INTEL_GEN(dev_priv) >= 11)
 		icl_dphy_param_init(intel_dsi);
 	else
 		vlv_dphy_param_init(intel_dsi);
diff --git a/drivers/gpu/drm/i915/intel_dvo.c b/drivers/gpu/drm/i915/intel_dvo.c
index a6c82482a841..adef81c8cccb 100644
--- a/drivers/gpu/drm/i915/intel_dvo.c
+++ b/drivers/gpu/drm/i915/intel_dvo.c
@@ -24,14 +24,20 @@
  * Authors:
  *	Eric Anholt <eric@anholt.net>
  */
+
 #include <linux/i2c.h>
 #include <linux/slab.h>
+
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc.h>
-#include "intel_drv.h"
 #include <drm/i915_drm.h>
-#include "i915_drv.h"
+
 #include "dvo.h"
+#include "i915_drv.h"
+#include "intel_connector.h"
+#include "intel_drv.h"
+#include "intel_dvo.h"
+#include "intel_panel.h"
 
 #define SIL164_ADDR	0x38
 #define CH7xxx_ADDR	0x76
diff --git a/drivers/gpu/drm/i915/intel_dvo.h b/drivers/gpu/drm/i915/intel_dvo.h
new file mode 100644
index 000000000000..3ed0fdf8efff
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_dvo.h
@@ -0,0 +1,13 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2019 Intel Corporation
+ */
+
+#ifndef __INTEL_DVO_H__
+#define __INTEL_DVO_H__
+
+struct drm_i915_private;
+
+void intel_dvo_init(struct drm_i915_private *dev_priv);
+
+#endif /* __INTEL_DVO_H__ */
diff --git a/drivers/gpu/drm/i915/intel_engine_cs.c b/drivers/gpu/drm/i915/intel_engine_cs.c
index 49fa43ff02ba..eea9bec04f1b 100644
--- a/drivers/gpu/drm/i915/intel_engine_cs.c
+++ b/drivers/gpu/drm/i915/intel_engine_cs.c
@@ -84,7 +84,6 @@ static const struct engine_class_info intel_engine_classes[] = {
 #define MAX_MMIO_BASES 3
 struct engine_info {
 	unsigned int hw_id;
-	unsigned int uabi_id;
 	u8 class;
 	u8 instance;
 	/* mmio bases table *must* be sorted in reverse gen order */
@@ -95,27 +94,24 @@ struct engine_info {
 };
 
 static const struct engine_info intel_engines[] = {
-	[RCS] = {
-		.hw_id = RCS_HW,
-		.uabi_id = I915_EXEC_RENDER,
+	[RCS0] = {
+		.hw_id = RCS0_HW,
 		.class = RENDER_CLASS,
 		.instance = 0,
 		.mmio_bases = {
 			{ .gen = 1, .base = RENDER_RING_BASE }
 		},
 	},
-	[BCS] = {
-		.hw_id = BCS_HW,
-		.uabi_id = I915_EXEC_BLT,
+	[BCS0] = {
+		.hw_id = BCS0_HW,
 		.class = COPY_ENGINE_CLASS,
 		.instance = 0,
 		.mmio_bases = {
 			{ .gen = 6, .base = BLT_RING_BASE }
 		},
 	},
-	[VCS] = {
-		.hw_id = VCS_HW,
-		.uabi_id = I915_EXEC_BSD,
+	[VCS0] = {
+		.hw_id = VCS0_HW,
 		.class = VIDEO_DECODE_CLASS,
 		.instance = 0,
 		.mmio_bases = {
@@ -124,9 +120,8 @@ static const struct engine_info intel_engines[] = {
 			{ .gen = 4, .base = BSD_RING_BASE }
 		},
 	},
-	[VCS2] = {
-		.hw_id = VCS2_HW,
-		.uabi_id = I915_EXEC_BSD,
+	[VCS1] = {
+		.hw_id = VCS1_HW,
 		.class = VIDEO_DECODE_CLASS,
 		.instance = 1,
 		.mmio_bases = {
@@ -134,27 +129,24 @@ static const struct engine_info intel_engines[] = {
 			{ .gen = 8, .base = GEN8_BSD2_RING_BASE }
 		},
 	},
-	[VCS3] = {
-		.hw_id = VCS3_HW,
-		.uabi_id = I915_EXEC_BSD,
+	[VCS2] = {
+		.hw_id = VCS2_HW,
 		.class = VIDEO_DECODE_CLASS,
 		.instance = 2,
 		.mmio_bases = {
 			{ .gen = 11, .base = GEN11_BSD3_RING_BASE }
 		},
 	},
-	[VCS4] = {
-		.hw_id = VCS4_HW,
-		.uabi_id = I915_EXEC_BSD,
+	[VCS3] = {
+		.hw_id = VCS3_HW,
 		.class = VIDEO_DECODE_CLASS,
 		.instance = 3,
 		.mmio_bases = {
 			{ .gen = 11, .base = GEN11_BSD4_RING_BASE }
 		},
 	},
-	[VECS] = {
-		.hw_id = VECS_HW,
-		.uabi_id = I915_EXEC_VEBOX,
+	[VECS0] = {
+		.hw_id = VECS0_HW,
 		.class = VIDEO_ENHANCEMENT_CLASS,
 		.instance = 0,
 		.mmio_bases = {
@@ -162,9 +154,8 @@ static const struct engine_info intel_engines[] = {
 			{ .gen = 7, .base = VEBOX_RING_BASE }
 		},
 	},
-	[VECS2] = {
-		.hw_id = VECS2_HW,
-		.uabi_id = I915_EXEC_VEBOX,
+	[VECS1] = {
+		.hw_id = VECS1_HW,
 		.class = VIDEO_ENHANCEMENT_CLASS,
 		.instance = 1,
 		.mmio_bases = {
@@ -264,21 +255,17 @@ static void __sprint_engine_name(char *name, const struct engine_info *info)
 
 void intel_engine_set_hwsp_writemask(struct intel_engine_cs *engine, u32 mask)
 {
-	struct drm_i915_private *dev_priv = engine->i915;
-	i915_reg_t hwstam;
-
 	/*
 	 * Though they added more rings on g4x/ilk, they did not add
 	 * per-engine HWSTAM until gen6.
 	 */
-	if (INTEL_GEN(dev_priv) < 6 && engine->class != RENDER_CLASS)
+	if (INTEL_GEN(engine->i915) < 6 && engine->class != RENDER_CLASS)
 		return;
 
-	hwstam = RING_HWSTAM(engine->mmio_base);
-	if (INTEL_GEN(dev_priv) >= 3)
-		I915_WRITE(hwstam, mask);
+	if (INTEL_GEN(engine->i915) >= 3)
+		ENGINE_WRITE(engine, RING_HWSTAM, mask);
 	else
-		I915_WRITE16(hwstam, mask);
+		ENGINE_WRITE16(engine, RING_HWSTAM, mask);
 }
 
 static void intel_engine_sanitize_mmio(struct intel_engine_cs *engine)
@@ -313,15 +300,18 @@ intel_engine_setup(struct drm_i915_private *dev_priv,
 	if (!engine)
 		return -ENOMEM;
 
+	BUILD_BUG_ON(BITS_PER_TYPE(engine->mask) < I915_NUM_ENGINES);
+
 	engine->id = id;
+	engine->mask = BIT(id);
 	engine->i915 = dev_priv;
+	engine->uncore = &dev_priv->uncore;
 	__sprint_engine_name(engine->name, info);
 	engine->hw_id = engine->guc_id = info->hw_id;
 	engine->mmio_base = __engine_mmio_base(dev_priv, info->mmio_bases);
 	engine->class = info->class;
 	engine->instance = info->instance;
 
-	engine->uabi_id = info->uabi_id;
 	engine->uabi_class = intel_engine_classes[info->class].uabi_class;
 
 	engine->context_size = __intel_engine_context_size(dev_priv,
@@ -355,15 +345,15 @@ intel_engine_setup(struct drm_i915_private *dev_priv,
 int intel_engines_init_mmio(struct drm_i915_private *dev_priv)
 {
 	struct intel_device_info *device_info = mkwrite_device_info(dev_priv);
-	const unsigned int ring_mask = INTEL_INFO(dev_priv)->ring_mask;
+	const unsigned int engine_mask = INTEL_INFO(dev_priv)->engine_mask;
 	struct intel_engine_cs *engine;
 	enum intel_engine_id id;
 	unsigned int mask = 0;
 	unsigned int i;
 	int err;
 
-	WARN_ON(ring_mask == 0);
-	WARN_ON(ring_mask &
+	WARN_ON(engine_mask == 0);
+	WARN_ON(engine_mask &
 		GENMASK(BITS_PER_TYPE(mask) - 1, I915_NUM_ENGINES));
 
 	if (i915_inject_load_failure())
@@ -377,7 +367,7 @@ int intel_engines_init_mmio(struct drm_i915_private *dev_priv)
 		if (err)
 			goto cleanup;
 
-		mask |= ENGINE_MASK(i);
+		mask |= BIT(i);
 	}
 
 	/*
@@ -385,16 +375,16 @@ int intel_engines_init_mmio(struct drm_i915_private *dev_priv)
 	 * are added to the driver by a warning and disabling the forgotten
 	 * engines.
 	 */
-	if (WARN_ON(mask != ring_mask))
-		device_info->ring_mask = mask;
+	if (WARN_ON(mask != engine_mask))
+		device_info->engine_mask = mask;
 
 	/* We always presume we have at least RCS available for later probing */
-	if (WARN_ON(!HAS_ENGINE(dev_priv, RCS))) {
+	if (WARN_ON(!HAS_ENGINE(dev_priv, RCS0))) {
 		err = -ENODEV;
 		goto cleanup;
 	}
 
-	RUNTIME_INFO(dev_priv)->num_rings = hweight32(mask);
+	RUNTIME_INFO(dev_priv)->num_engines = hweight32(mask);
 
 	i915_check_and_clear_faults(dev_priv);
 
@@ -455,12 +445,6 @@ cleanup:
 	return err;
 }
 
-void intel_engine_write_global_seqno(struct intel_engine_cs *engine, u32 seqno)
-{
-	intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno);
-	GEM_BUG_ON(intel_engine_get_seqno(engine) != seqno);
-}
-
 static void intel_engine_init_batch_pool(struct intel_engine_cs *engine)
 {
 	i915_gem_batch_pool_init(&engine->batch_pool, engine);
@@ -541,9 +525,7 @@ static int init_status_page(struct intel_engine_cs *engine)
 		return PTR_ERR(obj);
 	}
 
-	ret = i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
-	if (ret)
-		goto err;
+	i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
 
 	vma = i915_vma_instance(obj, &engine->i915->ggtt.vm, NULL);
 	if (IS_ERR(vma)) {
@@ -594,7 +576,6 @@ int intel_engine_setup_common(struct intel_engine_cs *engine)
 
 	err = i915_timeline_init(engine->i915,
 				 &engine->timeline,
-				 engine->name,
 				 engine->status_page.vma);
 	if (err)
 		goto err_hwsp;
@@ -614,10 +595,44 @@ err_hwsp:
 	return err;
 }
 
-static void __intel_context_unpin(struct i915_gem_context *ctx,
-				  struct intel_engine_cs *engine)
+void intel_engines_set_scheduler_caps(struct drm_i915_private *i915)
 {
-	intel_context_unpin(to_intel_context(ctx, engine));
+	static const struct {
+		u8 engine;
+		u8 sched;
+	} map[] = {
+#define MAP(x, y) { ilog2(I915_ENGINE_HAS_##x), ilog2(I915_SCHEDULER_CAP_##y) }
+		MAP(PREEMPTION, PREEMPTION),
+		MAP(SEMAPHORES, SEMAPHORES),
+#undef MAP
+	};
+	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
+	u32 enabled, disabled;
+
+	enabled = 0;
+	disabled = 0;
+	for_each_engine(engine, i915, id) { /* all engines must agree! */
+		int i;
+
+		if (engine->schedule)
+			enabled |= (I915_SCHEDULER_CAP_ENABLED |
+				    I915_SCHEDULER_CAP_PRIORITY);
+		else
+			disabled |= (I915_SCHEDULER_CAP_ENABLED |
+				     I915_SCHEDULER_CAP_PRIORITY);
+
+		for (i = 0; i < ARRAY_SIZE(map); i++) {
+			if (engine->flags & BIT(map[i].engine))
+				enabled |= BIT(map[i].sched);
+			else
+				disabled |= BIT(map[i].sched);
+		}
+	}
+
+	i915->caps.scheduler = enabled & ~disabled;
+	if (!(i915->caps.scheduler & I915_SCHEDULER_CAP_ENABLED))
+		i915->caps.scheduler = 0;
 }
 
 struct measure_breadcrumb {
@@ -639,7 +654,7 @@ static int measure_breadcrumb_dw(struct intel_engine_cs *engine)
 		return -ENOMEM;
 
 	if (i915_timeline_init(engine->i915,
-			       &frame->timeline, "measure",
+			       &frame->timeline,
 			       engine->status_page.vma))
 		goto out_frame;
 
@@ -670,6 +685,20 @@ out_frame:
 	return dw;
 }
 
+static int pin_context(struct i915_gem_context *ctx,
+		       struct intel_engine_cs *engine,
+		       struct intel_context **out)
+{
+	struct intel_context *ce;
+
+	ce = intel_context_pin(ctx, engine);
+	if (IS_ERR(ce))
+		return PTR_ERR(ce);
+
+	*out = ce;
+	return 0;
+}
+
 /**
  * intel_engines_init_common - initialize cengine state which might require hw access
  * @engine: Engine to initialize.
@@ -684,11 +713,8 @@ out_frame:
 int intel_engine_init_common(struct intel_engine_cs *engine)
 {
 	struct drm_i915_private *i915 = engine->i915;
-	struct intel_context *ce;
 	int ret;
 
-	engine->set_default_submission(engine);
-
 	/* We may need to do things with the shrinker which
 	 * require us to immediately switch back to the default
 	 * context. This can cause a problem as pinning the
@@ -696,39 +722,61 @@ int intel_engine_init_common(struct intel_engine_cs *engine)
 	 * be available. To avoid this we always pin the default
 	 * context.
 	 */
-	ce = intel_context_pin(i915->kernel_context, engine);
-	if (IS_ERR(ce))
-		return PTR_ERR(ce);
+	ret = pin_context(i915->kernel_context, engine,
+			  &engine->kernel_context);
+	if (ret)
+		return ret;
 
 	/*
 	 * Similarly the preempt context must always be available so that
-	 * we can interrupt the engine at any time.
+	 * we can interrupt the engine at any time. However, as preemption
+	 * is optional, we allow it to fail.
 	 */
-	if (i915->preempt_context) {
-		ce = intel_context_pin(i915->preempt_context, engine);
-		if (IS_ERR(ce)) {
-			ret = PTR_ERR(ce);
-			goto err_unpin_kernel;
-		}
-	}
+	if (i915->preempt_context)
+		pin_context(i915->preempt_context, engine,
+			    &engine->preempt_context);
 
 	ret = measure_breadcrumb_dw(engine);
 	if (ret < 0)
-		goto err_unpin_preempt;
+		goto err_unpin;
 
 	engine->emit_fini_breadcrumb_dw = ret;
 
-	return 0;
+	engine->set_default_submission(engine);
 
-err_unpin_preempt:
-	if (i915->preempt_context)
-		__intel_context_unpin(i915->preempt_context, engine);
+	return 0;
 
-err_unpin_kernel:
-	__intel_context_unpin(i915->kernel_context, engine);
+err_unpin:
+	if (engine->preempt_context)
+		intel_context_unpin(engine->preempt_context);
+	intel_context_unpin(engine->kernel_context);
 	return ret;
 }
 
+void intel_gt_resume(struct drm_i915_private *i915)
+{
+	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
+
+	/*
+	 * After resume, we may need to poke into the pinned kernel
+	 * contexts to paper over any damage caused by the sudden suspend.
+	 * Only the kernel contexts should remain pinned over suspend,
+	 * allowing us to fixup the user contexts on their first pin.
+	 */
+	for_each_engine(engine, i915, id) {
+		struct intel_context *ce;
+
+		ce = engine->kernel_context;
+		if (ce)
+			ce->ops->reset(ce);
+
+		ce = engine->preempt_context;
+		if (ce)
+			ce->ops->reset(ce);
+	}
+}
+
 /**
  * intel_engines_cleanup_common - cleans up the engine state created by
  *                                the common initiailizers.
@@ -738,8 +786,6 @@ err_unpin_kernel:
  */
 void intel_engine_cleanup_common(struct intel_engine_cs *engine)
 {
-	struct drm_i915_private *i915 = engine->i915;
-
 	cleanup_status_page(engine);
 
 	intel_engine_fini_breadcrumbs(engine);
@@ -749,9 +795,9 @@ void intel_engine_cleanup_common(struct intel_engine_cs *engine)
 	if (engine->default_state)
 		i915_gem_object_put(engine->default_state);
 
-	if (i915->preempt_context)
-		__intel_context_unpin(i915->preempt_context, engine);
-	__intel_context_unpin(i915->kernel_context, engine);
+	if (engine->preempt_context)
+		intel_context_unpin(engine->preempt_context);
+	intel_context_unpin(engine->kernel_context);
 
 	i915_timeline_fini(&engine->timeline);
 
@@ -762,50 +808,48 @@ void intel_engine_cleanup_common(struct intel_engine_cs *engine)
 
 u64 intel_engine_get_active_head(const struct intel_engine_cs *engine)
 {
-	struct drm_i915_private *dev_priv = engine->i915;
+	struct drm_i915_private *i915 = engine->i915;
+
 	u64 acthd;
 
-	if (INTEL_GEN(dev_priv) >= 8)
-		acthd = I915_READ64_2x32(RING_ACTHD(engine->mmio_base),
-					 RING_ACTHD_UDW(engine->mmio_base));
-	else if (INTEL_GEN(dev_priv) >= 4)
-		acthd = I915_READ(RING_ACTHD(engine->mmio_base));
+	if (INTEL_GEN(i915) >= 8)
+		acthd = ENGINE_READ64(engine, RING_ACTHD, RING_ACTHD_UDW);
+	else if (INTEL_GEN(i915) >= 4)
+		acthd = ENGINE_READ(engine, RING_ACTHD);
 	else
-		acthd = I915_READ(ACTHD);
+		acthd = ENGINE_READ(engine, ACTHD);
 
 	return acthd;
 }
 
 u64 intel_engine_get_last_batch_head(const struct intel_engine_cs *engine)
 {
-	struct drm_i915_private *dev_priv = engine->i915;
 	u64 bbaddr;
 
-	if (INTEL_GEN(dev_priv) >= 8)
-		bbaddr = I915_READ64_2x32(RING_BBADDR(engine->mmio_base),
-					  RING_BBADDR_UDW(engine->mmio_base));
+	if (INTEL_GEN(engine->i915) >= 8)
+		bbaddr = ENGINE_READ64(engine, RING_BBADDR, RING_BBADDR_UDW);
 	else
-		bbaddr = I915_READ(RING_BBADDR(engine->mmio_base));
+		bbaddr = ENGINE_READ(engine, RING_BBADDR);
 
 	return bbaddr;
 }
 
 int intel_engine_stop_cs(struct intel_engine_cs *engine)
 {
-	struct drm_i915_private *dev_priv = engine->i915;
+	struct intel_uncore *uncore = engine->uncore;
 	const u32 base = engine->mmio_base;
 	const i915_reg_t mode = RING_MI_MODE(base);
 	int err;
 
-	if (INTEL_GEN(dev_priv) < 3)
+	if (INTEL_GEN(engine->i915) < 3)
 		return -ENODEV;
 
 	GEM_TRACE("%s\n", engine->name);
 
-	I915_WRITE_FW(mode, _MASKED_BIT_ENABLE(STOP_RING));
+	intel_uncore_write_fw(uncore, mode, _MASKED_BIT_ENABLE(STOP_RING));
 
 	err = 0;
-	if (__intel_wait_for_register_fw(dev_priv,
+	if (__intel_wait_for_register_fw(uncore,
 					 mode, MODE_IDLE, MODE_IDLE,
 					 1000, 0,
 					 NULL)) {
@@ -814,19 +858,16 @@ int intel_engine_stop_cs(struct intel_engine_cs *engine)
 	}
 
 	/* A final mmio read to let GPU writes be hopefully flushed to memory */
-	POSTING_READ_FW(mode);
+	intel_uncore_posting_read_fw(uncore, mode);
 
 	return err;
 }
 
 void intel_engine_cancel_stop_cs(struct intel_engine_cs *engine)
 {
-	struct drm_i915_private *dev_priv = engine->i915;
-
 	GEM_TRACE("%s\n", engine->name);
 
-	I915_WRITE_FW(RING_MI_MODE(engine->mmio_base),
-		      _MASKED_BIT_DISABLE(STOP_RING));
+	ENGINE_WRITE_FW(engine, RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING));
 }
 
 const char *i915_cache_level_str(struct drm_i915_private *i915, int type)
@@ -863,6 +904,7 @@ static inline u32
 read_subslice_reg(struct drm_i915_private *dev_priv, int slice,
 		  int subslice, i915_reg_t reg)
 {
+	struct intel_uncore *uncore = &dev_priv->uncore;
 	u32 mcr_slice_subslice_mask;
 	u32 mcr_slice_subslice_select;
 	u32 default_mcr_s_ss_select;
@@ -884,33 +926,33 @@ read_subslice_reg(struct drm_i915_private *dev_priv, int slice,
 
 	default_mcr_s_ss_select = intel_calculate_mcr_s_ss_select(dev_priv);
 
-	fw_domains = intel_uncore_forcewake_for_reg(dev_priv, reg,
+	fw_domains = intel_uncore_forcewake_for_reg(uncore, reg,
 						    FW_REG_READ);
-	fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
+	fw_domains |= intel_uncore_forcewake_for_reg(uncore,
 						     GEN8_MCR_SELECTOR,
 						     FW_REG_READ | FW_REG_WRITE);
 
-	spin_lock_irq(&dev_priv->uncore.lock);
-	intel_uncore_forcewake_get__locked(dev_priv, fw_domains);
+	spin_lock_irq(&uncore->lock);
+	intel_uncore_forcewake_get__locked(uncore, fw_domains);
 
-	mcr = I915_READ_FW(GEN8_MCR_SELECTOR);
+	mcr = intel_uncore_read_fw(uncore, GEN8_MCR_SELECTOR);
 
 	WARN_ON_ONCE((mcr & mcr_slice_subslice_mask) !=
 		     default_mcr_s_ss_select);
 
 	mcr &= ~mcr_slice_subslice_mask;
 	mcr |= mcr_slice_subslice_select;
-	I915_WRITE_FW(GEN8_MCR_SELECTOR, mcr);
+	intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr);
 
-	ret = I915_READ_FW(reg);
+	ret = intel_uncore_read_fw(uncore, reg);
 
 	mcr &= ~mcr_slice_subslice_mask;
 	mcr |= default_mcr_s_ss_select;
 
-	I915_WRITE_FW(GEN8_MCR_SELECTOR, mcr);
+	intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr);
 
-	intel_uncore_forcewake_put__locked(dev_priv, fw_domains);
-	spin_unlock_irq(&dev_priv->uncore.lock);
+	intel_uncore_forcewake_put__locked(uncore, fw_domains);
+	spin_unlock_irq(&uncore->lock);
 
 	return ret;
 }
@@ -920,6 +962,7 @@ void intel_engine_get_instdone(struct intel_engine_cs *engine,
 			       struct intel_instdone *instdone)
 {
 	struct drm_i915_private *dev_priv = engine->i915;
+	struct intel_uncore *uncore = engine->uncore;
 	u32 mmio_base = engine->mmio_base;
 	int slice;
 	int subslice;
@@ -928,12 +971,14 @@ void intel_engine_get_instdone(struct intel_engine_cs *engine,
 
 	switch (INTEL_GEN(dev_priv)) {
 	default:
-		instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));
+		instdone->instdone =
+			intel_uncore_read(uncore, RING_INSTDONE(mmio_base));
 
-		if (engine->id != RCS)
+		if (engine->id != RCS0)
 			break;
 
-		instdone->slice_common = I915_READ(GEN7_SC_INSTDONE);
+		instdone->slice_common =
+			intel_uncore_read(uncore, GEN7_SC_INSTDONE);
 		for_each_instdone_slice_subslice(dev_priv, slice, subslice) {
 			instdone->sampler[slice][subslice] =
 				read_subslice_reg(dev_priv, slice, subslice,
@@ -944,28 +989,33 @@ void intel_engine_get_instdone(struct intel_engine_cs *engine,
 		}
 		break;
 	case 7:
-		instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));
+		instdone->instdone =
+			intel_uncore_read(uncore, RING_INSTDONE(mmio_base));
 
-		if (engine->id != RCS)
+		if (engine->id != RCS0)
 			break;
 
-		instdone->slice_common = I915_READ(GEN7_SC_INSTDONE);
-		instdone->sampler[0][0] = I915_READ(GEN7_SAMPLER_INSTDONE);
-		instdone->row[0][0] = I915_READ(GEN7_ROW_INSTDONE);
+		instdone->slice_common =
+			intel_uncore_read(uncore, GEN7_SC_INSTDONE);
+		instdone->sampler[0][0] =
+			intel_uncore_read(uncore, GEN7_SAMPLER_INSTDONE);
+		instdone->row[0][0] =
+			intel_uncore_read(uncore, GEN7_ROW_INSTDONE);
 
 		break;
 	case 6:
 	case 5:
 	case 4:
-		instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));
-
-		if (engine->id == RCS)
+		instdone->instdone =
+			intel_uncore_read(uncore, RING_INSTDONE(mmio_base));
+		if (engine->id == RCS0)
 			/* HACK: Using the wrong struct member */
-			instdone->slice_common = I915_READ(GEN4_INSTDONE1);
+			instdone->slice_common =
+				intel_uncore_read(uncore, GEN4_INSTDONE1);
 		break;
 	case 3:
 	case 2:
-		instdone->instdone = I915_READ(GEN2_INSTDONE);
+		instdone->instdone = intel_uncore_read(uncore, GEN2_INSTDONE);
 		break;
 	}
 }
@@ -985,12 +1035,13 @@ static bool ring_is_idle(struct intel_engine_cs *engine)
 		return true;
 
 	/* First check that no commands are left in the ring */
-	if ((I915_READ_HEAD(engine) & HEAD_ADDR) !=
-	    (I915_READ_TAIL(engine) & TAIL_ADDR))
+	if ((ENGINE_READ(engine, RING_HEAD) & HEAD_ADDR) !=
+	    (ENGINE_READ(engine, RING_TAIL) & TAIL_ADDR))
 		idle = false;
 
 	/* No bit for gen2, so assume the CS parser is idle */
-	if (INTEL_GEN(dev_priv) > 2 && !(I915_READ_MODE(engine) & MODE_IDLE))
+	if (INTEL_GEN(dev_priv) > 2 &&
+	    !(ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE))
 		idle = false;
 
 	intel_runtime_pm_put(dev_priv, wakeref);
@@ -1007,16 +1058,10 @@ static bool ring_is_idle(struct intel_engine_cs *engine)
  */
 bool intel_engine_is_idle(struct intel_engine_cs *engine)
 {
-	struct drm_i915_private *dev_priv = engine->i915;
-
 	/* More white lies, if wedged, hw state is inconsistent */
-	if (i915_terminally_wedged(&dev_priv->gpu_error))
+	if (i915_reset_failed(engine->i915))
 		return true;
 
-	/* Any inflight/incomplete requests? */
-	if (!intel_engine_signaled(engine, intel_engine_last_submit(engine)))
-		return false;
-
 	/* Waiting to drain ELSP? */
 	if (READ_ONCE(engine->execlists.active)) {
 		struct tasklet_struct *t = &engine->execlists.tasklet;
@@ -1045,7 +1090,7 @@ bool intel_engine_is_idle(struct intel_engine_cs *engine)
 	return ring_is_idle(engine);
 }
 
-bool intel_engines_are_idle(struct drm_i915_private *dev_priv)
+bool intel_engines_are_idle(struct drm_i915_private *i915)
 {
 	struct intel_engine_cs *engine;
 	enum intel_engine_id id;
@@ -1054,10 +1099,14 @@ bool intel_engines_are_idle(struct drm_i915_private *dev_priv)
 	 * If the driver is wedged, HW state may be very inconsistent and
 	 * report that it is still busy, even though we have stopped using it.
 	 */
-	if (i915_terminally_wedged(&dev_priv->gpu_error))
+	if (i915_reset_failed(i915))
 		return true;
 
-	for_each_engine(engine, dev_priv, id) {
+	/* Already parked (and passed an idleness test); must still be idle */
+	if (!READ_ONCE(i915->gt.awake))
+		return true;
+
+	for_each_engine(engine, i915, id) {
 		if (!intel_engine_is_idle(engine))
 			return false;
 	}
@@ -1065,34 +1114,6 @@ bool intel_engines_are_idle(struct drm_i915_private *dev_priv)
 	return true;
 }
 
-/**
- * intel_engine_has_kernel_context:
- * @engine: the engine
- *
- * Returns true if the last context to be executed on this engine, or has been
- * executed if the engine is already idle, is the kernel context
- * (#i915.kernel_context).
- */
-bool intel_engine_has_kernel_context(const struct intel_engine_cs *engine)
-{
-	const struct intel_context *kernel_context =
-		to_intel_context(engine->i915->kernel_context, engine);
-	struct i915_request *rq;
-
-	lockdep_assert_held(&engine->i915->drm.struct_mutex);
-
-	/*
-	 * Check the last context seen by the engine. If active, it will be
-	 * the last request that remains in the timeline. When idle, it is
-	 * the last executed context as tracked by retirement.
-	 */
-	rq = __i915_active_request_peek(&engine->timeline.last_request);
-	if (rq)
-		return rq->hw_context == kernel_context;
-	else
-		return engine->last_retired_context == kernel_context;
-}
-
 void intel_engines_reset_default_submission(struct drm_i915_private *i915)
 {
 	struct intel_engine_cs *engine;
@@ -1180,6 +1201,8 @@ void intel_engines_park(struct drm_i915_private *i915)
 		i915_gem_batch_pool_fini(&engine->batch_pool);
 		engine->execlists.no_priolist = false;
 	}
+
+	i915->gt.active_engines = 0;
 }
 
 /**
@@ -1283,15 +1306,14 @@ static void print_request(struct drm_printer *m,
 
 	x = print_sched_attr(rq->i915, &rq->sched.attr, buf, x, sizeof(buf));
 
-	drm_printf(m, "%s%x%s%s [%llx:%llx]%s @ %dms: %s\n",
+	drm_printf(m, "%s %llx:%llx%s%s %s @ %dms: %s\n",
 		   prefix,
-		   rq->global_seqno,
+		   rq->fence.context, rq->fence.seqno,
 		   i915_request_completed(rq) ? "!" :
 		   i915_request_started(rq) ? "*" :
 		   "",
 		   test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
 			    &rq->fence.flags) ?  "+" : "",
-		   rq->fence.context, rq->fence.seqno,
 		   buf,
 		   jiffies_to_msecs(jiffies - rq->emitted_jiffies),
 		   name);
@@ -1334,25 +1356,26 @@ static void intel_engine_print_registers(const struct intel_engine_cs *engine,
 		&engine->execlists;
 	u64 addr;
 
-	if (engine->id == RCS && IS_GEN_RANGE(dev_priv, 4, 7))
-		drm_printf(m, "\tCCID: 0x%08x\n", I915_READ(CCID));
+	if (engine->id == RCS0 && IS_GEN_RANGE(dev_priv, 4, 7))
+		drm_printf(m, "\tCCID: 0x%08x\n", ENGINE_READ(engine, CCID));
 	drm_printf(m, "\tRING_START: 0x%08x\n",
-		   I915_READ(RING_START(engine->mmio_base)));
+		   ENGINE_READ(engine, RING_START));
 	drm_printf(m, "\tRING_HEAD:  0x%08x\n",
-		   I915_READ(RING_HEAD(engine->mmio_base)) & HEAD_ADDR);
+		   ENGINE_READ(engine, RING_HEAD) & HEAD_ADDR);
 	drm_printf(m, "\tRING_TAIL:  0x%08x\n",
-		   I915_READ(RING_TAIL(engine->mmio_base)) & TAIL_ADDR);
+		   ENGINE_READ(engine, RING_TAIL) & TAIL_ADDR);
 	drm_printf(m, "\tRING_CTL:   0x%08x%s\n",
-		   I915_READ(RING_CTL(engine->mmio_base)),
-		   I915_READ(RING_CTL(engine->mmio_base)) & (RING_WAIT | RING_WAIT_SEMAPHORE) ? " [waiting]" : "");
+		   ENGINE_READ(engine, RING_CTL),
+		   ENGINE_READ(engine, RING_CTL) & (RING_WAIT | RING_WAIT_SEMAPHORE) ? " [waiting]" : "");
 	if (INTEL_GEN(engine->i915) > 2) {
 		drm_printf(m, "\tRING_MODE:  0x%08x%s\n",
-			   I915_READ(RING_MI_MODE(engine->mmio_base)),
-			   I915_READ(RING_MI_MODE(engine->mmio_base)) & (MODE_IDLE) ? " [idle]" : "");
+			   ENGINE_READ(engine, RING_MI_MODE),
+			   ENGINE_READ(engine, RING_MI_MODE) & (MODE_IDLE) ? " [idle]" : "");
 	}
 
 	if (INTEL_GEN(dev_priv) >= 6) {
-		drm_printf(m, "\tRING_IMR: %08x\n", I915_READ_IMR(engine));
+		drm_printf(m, "\tRING_IMR: %08x\n",
+			   ENGINE_READ(engine, RING_IMR));
 	}
 
 	addr = intel_engine_get_active_head(engine);
@@ -1362,57 +1385,53 @@ static void intel_engine_print_registers(const struct intel_engine_cs *engine,
 	drm_printf(m, "\tBBADDR: 0x%08x_%08x\n",
 		   upper_32_bits(addr), lower_32_bits(addr));
 	if (INTEL_GEN(dev_priv) >= 8)
-		addr = I915_READ64_2x32(RING_DMA_FADD(engine->mmio_base),
-					RING_DMA_FADD_UDW(engine->mmio_base));
+		addr = ENGINE_READ64(engine, RING_DMA_FADD, RING_DMA_FADD_UDW);
 	else if (INTEL_GEN(dev_priv) >= 4)
-		addr = I915_READ(RING_DMA_FADD(engine->mmio_base));
+		addr = ENGINE_READ(engine, RING_DMA_FADD);
 	else
-		addr = I915_READ(DMA_FADD_I8XX);
+		addr = ENGINE_READ(engine, DMA_FADD_I8XX);
 	drm_printf(m, "\tDMA_FADDR: 0x%08x_%08x\n",
 		   upper_32_bits(addr), lower_32_bits(addr));
 	if (INTEL_GEN(dev_priv) >= 4) {
 		drm_printf(m, "\tIPEIR: 0x%08x\n",
-			   I915_READ(RING_IPEIR(engine->mmio_base)));
+			   ENGINE_READ(engine, RING_IPEIR));
 		drm_printf(m, "\tIPEHR: 0x%08x\n",
-			   I915_READ(RING_IPEHR(engine->mmio_base)));
+			   ENGINE_READ(engine, RING_IPEHR));
 	} else {
-		drm_printf(m, "\tIPEIR: 0x%08x\n", I915_READ(IPEIR));
-		drm_printf(m, "\tIPEHR: 0x%08x\n", I915_READ(IPEHR));
+		drm_printf(m, "\tIPEIR: 0x%08x\n", ENGINE_READ(engine, IPEIR));
+		drm_printf(m, "\tIPEHR: 0x%08x\n", ENGINE_READ(engine, IPEHR));
 	}
 
 	if (HAS_EXECLISTS(dev_priv)) {
 		const u32 *hws =
 			&engine->status_page.addr[I915_HWS_CSB_BUF0_INDEX];
+		const u8 num_entries = execlists->csb_size;
 		unsigned int idx;
 		u8 read, write;
 
-		drm_printf(m, "\tExeclist status: 0x%08x %08x\n",
-			   I915_READ(RING_EXECLIST_STATUS_LO(engine)),
-			   I915_READ(RING_EXECLIST_STATUS_HI(engine)));
+		drm_printf(m, "\tExeclist status: 0x%08x %08x, entries %u\n",
+			   ENGINE_READ(engine, RING_EXECLIST_STATUS_LO),
+			   ENGINE_READ(engine, RING_EXECLIST_STATUS_HI),
+			   num_entries);
 
 		read = execlists->csb_head;
 		write = READ_ONCE(*execlists->csb_write);
 
-		drm_printf(m, "\tExeclist CSB read %d, write %d [mmio:%d], tasklet queued? %s (%s)\n",
+		drm_printf(m, "\tExeclist CSB read %d, write %d, tasklet queued? %s (%s)\n",
 			   read, write,
-			   GEN8_CSB_WRITE_PTR(I915_READ(RING_CONTEXT_STATUS_PTR(engine))),
 			   yesno(test_bit(TASKLET_STATE_SCHED,
 					  &engine->execlists.tasklet.state)),
 			   enableddisabled(!atomic_read(&engine->execlists.tasklet.count)));
-		if (read >= GEN8_CSB_ENTRIES)
+		if (read >= num_entries)
 			read = 0;
-		if (write >= GEN8_CSB_ENTRIES)
+		if (write >= num_entries)
 			write = 0;
 		if (read > write)
-			write += GEN8_CSB_ENTRIES;
+			write += num_entries;
 		while (read < write) {
-			idx = ++read % GEN8_CSB_ENTRIES;
-			drm_printf(m, "\tExeclist CSB[%d]: 0x%08x [mmio:0x%08x], context: %d [mmio:%d]\n",
-				   idx,
-				   hws[idx * 2],
-				   I915_READ(RING_CONTEXT_STATUS_BUF_LO(engine, idx)),
-				   hws[idx * 2 + 1],
-				   I915_READ(RING_CONTEXT_STATUS_BUF_HI(engine, idx)));
+			idx = ++read % num_entries;
+			drm_printf(m, "\tExeclist CSB[%d]: 0x%08x, context: %d\n",
+				   idx, hws[idx * 2], hws[idx * 2 + 1]);
 		}
 
 		rcu_read_lock();
@@ -1425,10 +1444,11 @@ static void intel_engine_print_registers(const struct intel_engine_cs *engine,
 				char hdr[80];
 
 				snprintf(hdr, sizeof(hdr),
-					 "\t\tELSP[%d] count=%d, ring:{start:%08x, hwsp:%08x}, rq: ",
+					 "\t\tELSP[%d] count=%d, ring:{start:%08x, hwsp:%08x, seqno:%08x}, rq: ",
 					 idx, count,
 					 i915_ggtt_offset(rq->ring->vma),
-					 rq->timeline->hwsp_offset);
+					 rq->timeline->hwsp_offset,
+					 hwsp_seqno(rq));
 				print_request(m, rq, hdr);
 			} else {
 				drm_printf(m, "\t\tELSP[%d] idle\n", idx);
@@ -1438,11 +1458,11 @@ static void intel_engine_print_registers(const struct intel_engine_cs *engine,
 		rcu_read_unlock();
 	} else if (INTEL_GEN(dev_priv) > 6) {
 		drm_printf(m, "\tPP_DIR_BASE: 0x%08x\n",
-			   I915_READ(RING_PP_DIR_BASE(engine)));
+			   ENGINE_READ(engine, RING_PP_DIR_BASE));
 		drm_printf(m, "\tPP_DIR_BASE_READ: 0x%08x\n",
-			   I915_READ(RING_PP_DIR_BASE_READ(engine)));
+			   ENGINE_READ(engine, RING_PP_DIR_BASE_READ));
 		drm_printf(m, "\tPP_DIR_DCLV: 0x%08x\n",
-			   I915_READ(RING_PP_DIR_DCLV(engine)));
+			   ENGINE_READ(engine, RING_PP_DIR_DCLV));
 	}
 }
 
@@ -1495,13 +1515,12 @@ void intel_engine_dump(struct intel_engine_cs *engine,
 		va_end(ap);
 	}
 
-	if (i915_terminally_wedged(&engine->i915->gpu_error))
+	if (i915_reset_failed(engine->i915))
 		drm_printf(m, "*** WEDGED ***\n");
 
-	drm_printf(m, "\tcurrent seqno %x, last %x, hangcheck %x [%d ms]\n",
-		   intel_engine_get_seqno(engine),
-		   intel_engine_last_submit(engine),
-		   engine->hangcheck.seqno,
+	drm_printf(m, "\tHangcheck %x:%x [%d ms]\n",
+		   engine->hangcheck.last_seqno,
+		   engine->hangcheck.next_seqno,
 		   jiffies_to_msecs(jiffies - engine->hangcheck.action_timestamp));
 	drm_printf(m, "\tReset count: %d (global %d)\n",
 		   i915_reset_engine_count(error, engine),
@@ -1521,7 +1540,7 @@ void intel_engine_dump(struct intel_engine_cs *engine,
 	if (&rq->link != &engine->timeline.requests)
 		print_request(m, rq, "\t\tlast   ");
 
-	rq = i915_gem_find_active_request(engine);
+	rq = intel_engine_find_active_request(engine);
 	if (rq) {
 		print_request(m, rq, "\t\tactive ");
 
@@ -1688,6 +1707,50 @@ void intel_disable_engine_stats(struct intel_engine_cs *engine)
 	write_sequnlock_irqrestore(&engine->stats.lock, flags);
 }
 
+static bool match_ring(struct i915_request *rq)
+{
+	u32 ring = ENGINE_READ(rq->engine, RING_START);
+
+	return ring == i915_ggtt_offset(rq->ring->vma);
+}
+
+struct i915_request *
+intel_engine_find_active_request(struct intel_engine_cs *engine)
+{
+	struct i915_request *request, *active = NULL;
+	unsigned long flags;
+
+	/*
+	 * We are called by the error capture, reset and to dump engine
+	 * state at random points in time. In particular, note that neither is
+	 * crucially ordered with an interrupt. After a hang, the GPU is dead
+	 * and we assume that no more writes can happen (we waited long enough
+	 * for all writes that were in transaction to be flushed) - adding an
+	 * extra delay for a recent interrupt is pointless. Hence, we do
+	 * not need an engine->irq_seqno_barrier() before the seqno reads.
+	 * At all other times, we must assume the GPU is still running, but
+	 * we only care about the snapshot of this moment.
+	 */
+	spin_lock_irqsave(&engine->timeline.lock, flags);
+	list_for_each_entry(request, &engine->timeline.requests, link) {
+		if (i915_request_completed(request))
+			continue;
+
+		if (!i915_request_started(request))
+			break;
+
+		/* More than one preemptible request may match! */
+		if (!match_ring(request))
+			break;
+
+		active = request;
+		break;
+	}
+	spin_unlock_irqrestore(&engine->timeline.lock, flags);
+
+	return active;
+}
+
 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 #include "selftests/mock_engine.c"
 #include "selftests/intel_engine_cs.c"
diff --git a/drivers/gpu/drm/i915/intel_engine_types.h b/drivers/gpu/drm/i915/intel_engine_types.h
new file mode 100644
index 000000000000..1f970c76b6a6
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_engine_types.h
@@ -0,0 +1,546 @@
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2019 Intel Corporation
+ */
+
+#ifndef __INTEL_ENGINE_TYPES__
+#define __INTEL_ENGINE_TYPES__
+
+#include <linux/hashtable.h>
+#include <linux/irq_work.h>
+#include <linux/kref.h>
+#include <linux/list.h>
+#include <linux/types.h>
+
+#include "i915_gem.h"
+#include "i915_priolist_types.h"
+#include "i915_selftest.h"
+#include "i915_timeline_types.h"
+#include "intel_workarounds_types.h"
+
+#include "i915_gem_batch_pool.h"
+#include "i915_pmu.h"
+
+#define I915_MAX_SLICES	3
+#define I915_MAX_SUBSLICES 8
+
+#define I915_CMD_HASH_ORDER 9
+
+struct dma_fence;
+struct drm_i915_reg_table;
+struct i915_gem_context;
+struct i915_request;
+struct i915_sched_attr;
+struct intel_uncore;
+
+typedef u8 intel_engine_mask_t;
+#define ALL_ENGINES ((intel_engine_mask_t)~0ul)
+
+struct intel_hw_status_page {
+	struct i915_vma *vma;
+	u32 *addr;
+};
+
+struct intel_instdone {
+	u32 instdone;
+	/* The following exist only in the RCS engine */
+	u32 slice_common;
+	u32 sampler[I915_MAX_SLICES][I915_MAX_SUBSLICES];
+	u32 row[I915_MAX_SLICES][I915_MAX_SUBSLICES];
+};
+
+struct intel_engine_hangcheck {
+	u64 acthd;
+	u32 last_seqno;
+	u32 next_seqno;
+	unsigned long action_timestamp;
+	struct intel_instdone instdone;
+};
+
+struct intel_ring {
+	struct kref ref;
+	struct i915_vma *vma;
+	void *vaddr;
+
+	struct i915_timeline *timeline;
+	struct list_head request_list;
+	struct list_head active_link;
+
+	u32 head;
+	u32 tail;
+	u32 emit;
+
+	u32 space;
+	u32 size;
+	u32 effective_size;
+};
+
+/*
+ * we use a single page to load ctx workarounds so all of these
+ * values are referred in terms of dwords
+ *
+ * struct i915_wa_ctx_bb:
+ *  offset: specifies batch starting position, also helpful in case
+ *    if we want to have multiple batches at different offsets based on
+ *    some criteria. It is not a requirement at the moment but provides
+ *    an option for future use.
+ *  size: size of the batch in DWORDS
+ */
+struct i915_ctx_workarounds {
+	struct i915_wa_ctx_bb {
+		u32 offset;
+		u32 size;
+	} indirect_ctx, per_ctx;
+	struct i915_vma *vma;
+};
+
+#define I915_MAX_VCS	4
+#define I915_MAX_VECS	2
+
+/*
+ * Engine IDs definitions.
+ * Keep instances of the same type engine together.
+ */
+enum intel_engine_id {
+	RCS0 = 0,
+	BCS0,
+	VCS0,
+	VCS1,
+	VCS2,
+	VCS3,
+#define _VCS(n) (VCS0 + (n))
+	VECS0,
+	VECS1,
+#define _VECS(n) (VECS0 + (n))
+	I915_NUM_ENGINES
+};
+
+struct st_preempt_hang {
+	struct completion completion;
+	unsigned int count;
+	bool inject_hang;
+};
+
+/**
+ * struct intel_engine_execlists - execlist submission queue and port state
+ *
+ * The struct intel_engine_execlists represents the combined logical state of
+ * driver and the hardware state for execlist mode of submission.
+ */
+struct intel_engine_execlists {
+	/**
+	 * @tasklet: softirq tasklet for bottom handler
+	 */
+	struct tasklet_struct tasklet;
+
+	/**
+	 * @default_priolist: priority list for I915_PRIORITY_NORMAL
+	 */
+	struct i915_priolist default_priolist;
+
+	/**
+	 * @no_priolist: priority lists disabled
+	 */
+	bool no_priolist;
+
+	/**
+	 * @submit_reg: gen-specific execlist submission register
+	 * set to the ExecList Submission Port (elsp) register pre-Gen11 and to
+	 * the ExecList Submission Queue Contents register array for Gen11+
+	 */
+	u32 __iomem *submit_reg;
+
+	/**
+	 * @ctrl_reg: the enhanced execlists control register, used to load the
+	 * submit queue on the HW and to request preemptions to idle
+	 */
+	u32 __iomem *ctrl_reg;
+
+	/**
+	 * @port: execlist port states
+	 *
+	 * For each hardware ELSP (ExecList Submission Port) we keep
+	 * track of the last request and the number of times we submitted
+	 * that port to hw. We then count the number of times the hw reports
+	 * a context completion or preemption. As only one context can
+	 * be active on hw, we limit resubmission of context to port[0]. This
+	 * is called Lite Restore, of the context.
+	 */
+	struct execlist_port {
+		/**
+		 * @request_count: combined request and submission count
+		 */
+		struct i915_request *request_count;
+#define EXECLIST_COUNT_BITS 2
+#define port_request(p) ptr_mask_bits((p)->request_count, EXECLIST_COUNT_BITS)
+#define port_count(p) ptr_unmask_bits((p)->request_count, EXECLIST_COUNT_BITS)
+#define port_pack(rq, count) ptr_pack_bits(rq, count, EXECLIST_COUNT_BITS)
+#define port_unpack(p, count) ptr_unpack_bits((p)->request_count, count, EXECLIST_COUNT_BITS)
+#define port_set(p, packed) ((p)->request_count = (packed))
+#define port_isset(p) ((p)->request_count)
+#define port_index(p, execlists) ((p) - (execlists)->port)
+
+		/**
+		 * @context_id: context ID for port
+		 */
+		GEM_DEBUG_DECL(u32 context_id);
+
+#define EXECLIST_MAX_PORTS 2
+	} port[EXECLIST_MAX_PORTS];
+
+	/**
+	 * @active: is the HW active? We consider the HW as active after
+	 * submitting any context for execution and until we have seen the
+	 * last context completion event. After that, we do not expect any
+	 * more events until we submit, and so can park the HW.
+	 *
+	 * As we have a small number of different sources from which we feed
+	 * the HW, we track the state of each inside a single bitfield.
+	 */
+	unsigned int active;
+#define EXECLISTS_ACTIVE_USER 0
+#define EXECLISTS_ACTIVE_PREEMPT 1
+#define EXECLISTS_ACTIVE_HWACK 2
+
+	/**
+	 * @port_mask: number of execlist ports - 1
+	 */
+	unsigned int port_mask;
+
+	/**
+	 * @queue_priority_hint: Highest pending priority.
+	 *
+	 * When we add requests into the queue, or adjust the priority of
+	 * executing requests, we compute the maximum priority of those
+	 * pending requests. We can then use this value to determine if
+	 * we need to preempt the executing requests to service the queue.
+	 * However, since the we may have recorded the priority of an inflight
+	 * request we wanted to preempt but since completed, at the time of
+	 * dequeuing the priority hint may no longer may match the highest
+	 * available request priority.
+	 */
+	int queue_priority_hint;
+
+	/**
+	 * @queue: queue of requests, in priority lists
+	 */
+	struct rb_root_cached queue;
+
+	/**
+	 * @csb_write: control register for Context Switch buffer
+	 *
+	 * Note this register may be either mmio or HWSP shadow.
+	 */
+	u32 *csb_write;
+
+	/**
+	 * @csb_status: status array for Context Switch buffer
+	 *
+	 * Note these register may be either mmio or HWSP shadow.
+	 */
+	u32 *csb_status;
+
+	/**
+	 * @preempt_complete_status: expected CSB upon completing preemption
+	 */
+	u32 preempt_complete_status;
+
+	/**
+	 * @csb_size: context status buffer FIFO size
+	 */
+	u8 csb_size;
+
+	/**
+	 * @csb_head: context status buffer head
+	 */
+	u8 csb_head;
+
+	I915_SELFTEST_DECLARE(struct st_preempt_hang preempt_hang;)
+};
+
+#define INTEL_ENGINE_CS_MAX_NAME 8
+
+struct intel_engine_cs {
+	struct drm_i915_private *i915;
+	struct intel_uncore *uncore;
+	char name[INTEL_ENGINE_CS_MAX_NAME];
+
+	enum intel_engine_id id;
+	unsigned int hw_id;
+	unsigned int guc_id;
+	intel_engine_mask_t mask;
+
+	u8 uabi_class;
+
+	u8 class;
+	u8 instance;
+	u32 context_size;
+	u32 mmio_base;
+
+	struct intel_ring *buffer;
+
+	struct i915_timeline timeline;
+
+	struct intel_context *kernel_context; /* pinned */
+	struct intel_context *preempt_context; /* pinned; optional */
+
+	struct drm_i915_gem_object *default_state;
+	void *pinned_default_state;
+
+	/* Rather than have every client wait upon all user interrupts,
+	 * with the herd waking after every interrupt and each doing the
+	 * heavyweight seqno dance, we delegate the task (of being the
+	 * bottom-half of the user interrupt) to the first client. After
+	 * every interrupt, we wake up one client, who does the heavyweight
+	 * coherent seqno read and either goes back to sleep (if incomplete),
+	 * or wakes up all the completed clients in parallel, before then
+	 * transferring the bottom-half status to the next client in the queue.
+	 *
+	 * Compared to walking the entire list of waiters in a single dedicated
+	 * bottom-half, we reduce the latency of the first waiter by avoiding
+	 * a context switch, but incur additional coherent seqno reads when
+	 * following the chain of request breadcrumbs. Since it is most likely
+	 * that we have a single client waiting on each seqno, then reducing
+	 * the overhead of waking that client is much preferred.
+	 */
+	struct intel_breadcrumbs {
+		spinlock_t irq_lock;
+		struct list_head signalers;
+
+		struct irq_work irq_work; /* for use from inside irq_lock */
+
+		unsigned int irq_enabled;
+
+		bool irq_armed;
+	} breadcrumbs;
+
+	struct intel_engine_pmu {
+		/**
+		 * @enable: Bitmask of enable sample events on this engine.
+		 *
+		 * Bits correspond to sample event types, for instance
+		 * I915_SAMPLE_QUEUED is bit 0 etc.
+		 */
+		u32 enable;
+		/**
+		 * @enable_count: Reference count for the enabled samplers.
+		 *
+		 * Index number corresponds to @enum drm_i915_pmu_engine_sample.
+		 */
+		unsigned int enable_count[I915_ENGINE_SAMPLE_COUNT];
+		/**
+		 * @sample: Counter values for sampling events.
+		 *
+		 * Our internal timer stores the current counters in this field.
+		 *
+		 * Index number corresponds to @enum drm_i915_pmu_engine_sample.
+		 */
+		struct i915_pmu_sample sample[I915_ENGINE_SAMPLE_COUNT];
+	} pmu;
+
+	/*
+	 * A pool of objects to use as shadow copies of client batch buffers
+	 * when the command parser is enabled. Prevents the client from
+	 * modifying the batch contents after software parsing.
+	 */
+	struct i915_gem_batch_pool batch_pool;
+
+	struct intel_hw_status_page status_page;
+	struct i915_ctx_workarounds wa_ctx;
+	struct i915_wa_list ctx_wa_list;
+	struct i915_wa_list wa_list;
+	struct i915_wa_list whitelist;
+
+	u32             irq_keep_mask; /* always keep these interrupts */
+	u32		irq_enable_mask; /* bitmask to enable ring interrupt */
+	void		(*irq_enable)(struct intel_engine_cs *engine);
+	void		(*irq_disable)(struct intel_engine_cs *engine);
+
+	int		(*init_hw)(struct intel_engine_cs *engine);
+
+	struct {
+		void (*prepare)(struct intel_engine_cs *engine);
+		void (*reset)(struct intel_engine_cs *engine, bool stalled);
+		void (*finish)(struct intel_engine_cs *engine);
+	} reset;
+
+	void		(*park)(struct intel_engine_cs *engine);
+	void		(*unpark)(struct intel_engine_cs *engine);
+
+	void		(*set_default_submission)(struct intel_engine_cs *engine);
+
+	const struct intel_context_ops *cops;
+
+	int		(*request_alloc)(struct i915_request *rq);
+	int		(*init_context)(struct i915_request *rq);
+
+	int		(*emit_flush)(struct i915_request *request, u32 mode);
+#define EMIT_INVALIDATE	BIT(0)
+#define EMIT_FLUSH	BIT(1)
+#define EMIT_BARRIER	(EMIT_INVALIDATE | EMIT_FLUSH)
+	int		(*emit_bb_start)(struct i915_request *rq,
+					 u64 offset, u32 length,
+					 unsigned int dispatch_flags);
+#define I915_DISPATCH_SECURE BIT(0)
+#define I915_DISPATCH_PINNED BIT(1)
+	int		 (*emit_init_breadcrumb)(struct i915_request *rq);
+	u32		*(*emit_fini_breadcrumb)(struct i915_request *rq,
+						 u32 *cs);
+	unsigned int	emit_fini_breadcrumb_dw;
+
+	/* Pass the request to the hardware queue (e.g. directly into
+	 * the legacy ringbuffer or to the end of an execlist).
+	 *
+	 * This is called from an atomic context with irqs disabled; must
+	 * be irq safe.
+	 */
+	void		(*submit_request)(struct i915_request *rq);
+
+	/*
+	 * Call when the priority on a request has changed and it and its
+	 * dependencies may need rescheduling. Note the request itself may
+	 * not be ready to run!
+	 */
+	void		(*schedule)(struct i915_request *request,
+				    const struct i915_sched_attr *attr);
+
+	/*
+	 * Cancel all requests on the hardware, or queued for execution.
+	 * This should only cancel the ready requests that have been
+	 * submitted to the engine (via the engine->submit_request callback).
+	 * This is called when marking the device as wedged.
+	 */
+	void		(*cancel_requests)(struct intel_engine_cs *engine);
+
+	void		(*cleanup)(struct intel_engine_cs *engine);
+
+	struct intel_engine_execlists execlists;
+
+	/* Contexts are pinned whilst they are active on the GPU. The last
+	 * context executed remains active whilst the GPU is idle - the
+	 * switch away and write to the context object only occurs on the
+	 * next execution.  Contexts are only unpinned on retirement of the
+	 * following request ensuring that we can always write to the object
+	 * on the context switch even after idling. Across suspend, we switch
+	 * to the kernel context and trash it as the save may not happen
+	 * before the hardware is powered down.
+	 */
+	struct intel_context *last_retired_context;
+
+	/* status_notifier: list of callbacks for context-switch changes */
+	struct atomic_notifier_head context_status_notifier;
+
+	struct intel_engine_hangcheck hangcheck;
+
+#define I915_ENGINE_NEEDS_CMD_PARSER BIT(0)
+#define I915_ENGINE_SUPPORTS_STATS   BIT(1)
+#define I915_ENGINE_HAS_PREEMPTION   BIT(2)
+#define I915_ENGINE_HAS_SEMAPHORES   BIT(3)
+#define I915_ENGINE_NEEDS_BREADCRUMB_TASKLET BIT(4)
+	unsigned int flags;
+
+	/*
+	 * Table of commands the command parser needs to know about
+	 * for this engine.
+	 */
+	DECLARE_HASHTABLE(cmd_hash, I915_CMD_HASH_ORDER);
+
+	/*
+	 * Table of registers allowed in commands that read/write registers.
+	 */
+	const struct drm_i915_reg_table *reg_tables;
+	int reg_table_count;
+
+	/*
+	 * Returns the bitmask for the length field of the specified command.
+	 * Return 0 for an unrecognized/invalid command.
+	 *
+	 * If the command parser finds an entry for a command in the engine's
+	 * cmd_tables, it gets the command's length based on the table entry.
+	 * If not, it calls this function to determine the per-engine length
+	 * field encoding for the command (i.e. different opcode ranges use
+	 * certain bits to encode the command length in the header).
+	 */
+	u32 (*get_cmd_length_mask)(u32 cmd_header);
+
+	struct {
+		/**
+		 * @lock: Lock protecting the below fields.
+		 */
+		seqlock_t lock;
+		/**
+		 * @enabled: Reference count indicating number of listeners.
+		 */
+		unsigned int enabled;
+		/**
+		 * @active: Number of contexts currently scheduled in.
+		 */
+		unsigned int active;
+		/**
+		 * @enabled_at: Timestamp when busy stats were enabled.
+		 */
+		ktime_t enabled_at;
+		/**
+		 * @start: Timestamp of the last idle to active transition.
+		 *
+		 * Idle is defined as active == 0, active is active > 0.
+		 */
+		ktime_t start;
+		/**
+		 * @total: Total time this engine was busy.
+		 *
+		 * Accumulated time not counting the most recent block in cases
+		 * where engine is currently busy (active > 0).
+		 */
+		ktime_t total;
+	} stats;
+};
+
+static inline bool
+intel_engine_needs_cmd_parser(const struct intel_engine_cs *engine)
+{
+	return engine->flags & I915_ENGINE_NEEDS_CMD_PARSER;
+}
+
+static inline bool
+intel_engine_supports_stats(const struct intel_engine_cs *engine)
+{
+	return engine->flags & I915_ENGINE_SUPPORTS_STATS;
+}
+
+static inline bool
+intel_engine_has_preemption(const struct intel_engine_cs *engine)
+{
+	return engine->flags & I915_ENGINE_HAS_PREEMPTION;
+}
+
+static inline bool
+intel_engine_has_semaphores(const struct intel_engine_cs *engine)
+{
+	return engine->flags & I915_ENGINE_HAS_SEMAPHORES;
+}
+
+static inline bool
+intel_engine_needs_breadcrumb_tasklet(const struct intel_engine_cs *engine)
+{
+	return engine->flags & I915_ENGINE_NEEDS_BREADCRUMB_TASKLET;
+}
+
+#define instdone_slice_mask(dev_priv__) \
+	(IS_GEN(dev_priv__, 7) ? \
+	 1 : RUNTIME_INFO(dev_priv__)->sseu.slice_mask)
+
+#define instdone_subslice_mask(dev_priv__) \
+	(IS_GEN(dev_priv__, 7) ? \
+	 1 : RUNTIME_INFO(dev_priv__)->sseu.subslice_mask[0])
+
+#define for_each_instdone_slice_subslice(dev_priv__, slice__, subslice__) \
+	for ((slice__) = 0, (subslice__) = 0; \
+	     (slice__) < I915_MAX_SLICES; \
+	     (subslice__) = ((subslice__) + 1) < I915_MAX_SUBSLICES ? (subslice__) + 1 : 0, \
+	       (slice__) += ((subslice__) == 0)) \
+		for_each_if((BIT(slice__) & instdone_slice_mask(dev_priv__)) && \
+			    (BIT(subslice__) & instdone_subslice_mask(dev_priv__)))
+
+#endif /* __INTEL_ENGINE_TYPES_H__ */
diff --git a/drivers/gpu/drm/i915/intel_fbc.c b/drivers/gpu/drm/i915/intel_fbc.c
index 656e684e7c9a..c805a0966395 100644
--- a/drivers/gpu/drm/i915/intel_fbc.c
+++ b/drivers/gpu/drm/i915/intel_fbc.c
@@ -40,8 +40,10 @@
 
 #include <drm/drm_fourcc.h>
 
-#include "intel_drv.h"
 #include "i915_drv.h"
+#include "intel_drv.h"
+#include "intel_fbc.h"
+#include "intel_frontbuffer.h"
 
 static inline bool fbc_supported(struct drm_i915_private *dev_priv)
 {
@@ -108,7 +110,7 @@ static void i8xx_fbc_deactivate(struct drm_i915_private *dev_priv)
 	I915_WRITE(FBC_CONTROL, fbc_ctl);
 
 	/* Wait for compressing bit to clear */
-	if (intel_wait_for_register(dev_priv,
+	if (intel_wait_for_register(&dev_priv->uncore,
 				    FBC_STATUS, FBC_STAT_COMPRESSING, 0,
 				    10)) {
 		DRM_DEBUG_KMS("FBC idle timed out\n");
diff --git a/drivers/gpu/drm/i915/intel_fbc.h b/drivers/gpu/drm/i915/intel_fbc.h
new file mode 100644
index 000000000000..50272eda8d43
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_fbc.h
@@ -0,0 +1,42 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2019 Intel Corporation
+ */
+
+#ifndef __INTEL_FBC_H__
+#define __INTEL_FBC_H__
+
+#include <linux/types.h>
+
+#include "intel_frontbuffer.h"
+
+struct drm_i915_private;
+struct intel_atomic_state;
+struct intel_crtc;
+struct intel_crtc_state;
+struct intel_plane_state;
+
+void intel_fbc_choose_crtc(struct drm_i915_private *dev_priv,
+			   struct intel_atomic_state *state);
+bool intel_fbc_is_active(struct drm_i915_private *dev_priv);
+void intel_fbc_pre_update(struct intel_crtc *crtc,
+			  struct intel_crtc_state *crtc_state,
+			  struct intel_plane_state *plane_state);
+void intel_fbc_post_update(struct intel_crtc *crtc);
+void intel_fbc_init(struct drm_i915_private *dev_priv);
+void intel_fbc_init_pipe_state(struct drm_i915_private *dev_priv);
+void intel_fbc_enable(struct intel_crtc *crtc,
+		      struct intel_crtc_state *crtc_state,
+		      struct intel_plane_state *plane_state);
+void intel_fbc_disable(struct intel_crtc *crtc);
+void intel_fbc_global_disable(struct drm_i915_private *dev_priv);
+void intel_fbc_invalidate(struct drm_i915_private *dev_priv,
+			  unsigned int frontbuffer_bits,
+			  enum fb_op_origin origin);
+void intel_fbc_flush(struct drm_i915_private *dev_priv,
+		     unsigned int frontbuffer_bits, enum fb_op_origin origin);
+void intel_fbc_cleanup_cfb(struct drm_i915_private *dev_priv);
+void intel_fbc_handle_fifo_underrun_irq(struct drm_i915_private *dev_priv);
+int intel_fbc_reset_underrun(struct drm_i915_private *dev_priv);
+
+#endif /* __INTEL_FBC_H__ */
diff --git a/drivers/gpu/drm/i915/intel_fbdev.c b/drivers/gpu/drm/i915/intel_fbdev.c
index 376ffe842e26..89db71996148 100644
--- a/drivers/gpu/drm/i915/intel_fbdev.c
+++ b/drivers/gpu/drm/i915/intel_fbdev.c
@@ -25,26 +25,27 @@
  */
 
 #include <linux/async.h>
-#include <linux/module.h>
-#include <linux/kernel.h>
 #include <linux/console.h>
+#include <linux/delay.h>
 #include <linux/errno.h>
-#include <linux/string.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
 #include <linux/mm.h>
-#include <linux/tty.h>
+#include <linux/module.h>
+#include <linux/string.h>
 #include <linux/sysrq.h>
-#include <linux/delay.h>
-#include <linux/init.h>
+#include <linux/tty.h>
 #include <linux/vga_switcheroo.h>
 
 #include <drm/drm_crtc.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_fourcc.h>
+#include <drm/i915_drm.h>
 
+#include "i915_drv.h"
 #include "intel_drv.h"
+#include "intel_fbdev.h"
 #include "intel_frontbuffer.h"
-#include <drm/i915_drm.h>
-#include "i915_drv.h"
 
 static void intel_fbdev_invalidate(struct intel_fbdev *ifbdev)
 {
@@ -235,12 +236,8 @@ static int intelfb_create(struct drm_fb_helper *helper,
 		goto out_unpin;
 	}
 
-	info->par = helper;
-
 	ifbdev->helper.fb = fb;
 
-	strcpy(info->fix.id, "inteldrmfb");
-
 	info->fbops = &intelfb_ops;
 
 	/* setup aperture base/size for vesafb takeover */
@@ -259,11 +256,7 @@ static int intelfb_create(struct drm_fb_helper *helper,
 	info->screen_base = vaddr;
 	info->screen_size = vma->node.size;
 
-	/* This driver doesn't need a VT switch to restore the mode on resume */
-	info->skip_vt_switch = true;
-
-	drm_fb_helper_fill_fix(info, fb->pitches[0], fb->format->depth);
-	drm_fb_helper_fill_var(info, &ifbdev->helper, sizes->fb_width, sizes->fb_height);
+	drm_fb_helper_fill_info(info, &ifbdev->helper, sizes);
 
 	/* If the object is shmemfs backed, it will have given us zeroed pages.
 	 * If the object is stolen however, it will be full of whatever
@@ -292,223 +285,7 @@ out_unlock:
 	return ret;
 }
 
-static struct drm_fb_helper_crtc *
-intel_fb_helper_crtc(struct drm_fb_helper *fb_helper, struct drm_crtc *crtc)
-{
-	int i;
-
-	for (i = 0; i < fb_helper->crtc_count; i++)
-		if (fb_helper->crtc_info[i].mode_set.crtc == crtc)
-			return &fb_helper->crtc_info[i];
-
-	return NULL;
-}
-
-/*
- * Try to read the BIOS display configuration and use it for the initial
- * fb configuration.
- *
- * The BIOS or boot loader will generally create an initial display
- * configuration for us that includes some set of active pipes and displays.
- * This routine tries to figure out which pipes and connectors are active
- * and stuffs them into the crtcs and modes array given to us by the
- * drm_fb_helper code.
- *
- * The overall sequence is:
- *   intel_fbdev_init - from driver load
- *     intel_fbdev_init_bios - initialize the intel_fbdev using BIOS data
- *     drm_fb_helper_init - build fb helper structs
- *     drm_fb_helper_single_add_all_connectors - more fb helper structs
- *   intel_fbdev_initial_config - apply the config
- *     drm_fb_helper_initial_config - call ->probe then register_framebuffer()
- *         drm_setup_crtcs - build crtc config for fbdev
- *           intel_fb_initial_config - find active connectors etc
- *         drm_fb_helper_single_fb_probe - set up fbdev
- *           intelfb_create - re-use or alloc fb, build out fbdev structs
- *
- * Note that we don't make special consideration whether we could actually
- * switch to the selected modes without a full modeset. E.g. when the display
- * is in VGA mode we need to recalculate watermarks and set a new high-res
- * framebuffer anyway.
- */
-static bool intel_fb_initial_config(struct drm_fb_helper *fb_helper,
-				    struct drm_fb_helper_crtc **crtcs,
-				    struct drm_display_mode **modes,
-				    struct drm_fb_offset *offsets,
-				    bool *enabled, int width, int height)
-{
-	struct drm_i915_private *dev_priv = to_i915(fb_helper->dev);
-	unsigned long conn_configured, conn_seq, mask;
-	unsigned int count = min(fb_helper->connector_count, BITS_PER_LONG);
-	int i, j;
-	bool *save_enabled;
-	bool fallback = true, ret = true;
-	int num_connectors_enabled = 0;
-	int num_connectors_detected = 0;
-	struct drm_modeset_acquire_ctx ctx;
-
-	save_enabled = kcalloc(count, sizeof(bool), GFP_KERNEL);
-	if (!save_enabled)
-		return false;
-
-	drm_modeset_acquire_init(&ctx, 0);
-
-	while (drm_modeset_lock_all_ctx(fb_helper->dev, &ctx) != 0)
-		drm_modeset_backoff(&ctx);
-
-	memcpy(save_enabled, enabled, count);
-	mask = GENMASK(count - 1, 0);
-	conn_configured = 0;
-retry:
-	conn_seq = conn_configured;
-	for (i = 0; i < count; i++) {
-		struct drm_fb_helper_connector *fb_conn;
-		struct drm_connector *connector;
-		struct drm_encoder *encoder;
-		struct drm_fb_helper_crtc *new_crtc;
-
-		fb_conn = fb_helper->connector_info[i];
-		connector = fb_conn->connector;
-
-		if (conn_configured & BIT(i))
-			continue;
-
-		if (conn_seq == 0 && !connector->has_tile)
-			continue;
-
-		if (connector->status == connector_status_connected)
-			num_connectors_detected++;
-
-		if (!enabled[i]) {
-			DRM_DEBUG_KMS("connector %s not enabled, skipping\n",
-				      connector->name);
-			conn_configured |= BIT(i);
-			continue;
-		}
-
-		if (connector->force == DRM_FORCE_OFF) {
-			DRM_DEBUG_KMS("connector %s is disabled by user, skipping\n",
-				      connector->name);
-			enabled[i] = false;
-			continue;
-		}
-
-		encoder = connector->state->best_encoder;
-		if (!encoder || WARN_ON(!connector->state->crtc)) {
-			if (connector->force > DRM_FORCE_OFF)
-				goto bail;
-
-			DRM_DEBUG_KMS("connector %s has no encoder or crtc, skipping\n",
-				      connector->name);
-			enabled[i] = false;
-			conn_configured |= BIT(i);
-			continue;
-		}
-
-		num_connectors_enabled++;
-
-		new_crtc = intel_fb_helper_crtc(fb_helper,
-						connector->state->crtc);
-
-		/*
-		 * Make sure we're not trying to drive multiple connectors
-		 * with a single CRTC, since our cloning support may not
-		 * match the BIOS.
-		 */
-		for (j = 0; j < count; j++) {
-			if (crtcs[j] == new_crtc) {
-				DRM_DEBUG_KMS("fallback: cloned configuration\n");
-				goto bail;
-			}
-		}
-
-		DRM_DEBUG_KMS("looking for cmdline mode on connector %s\n",
-			      connector->name);
-
-		/* go for command line mode first */
-		modes[i] = drm_pick_cmdline_mode(fb_conn);
-
-		/* try for preferred next */
-		if (!modes[i]) {
-			DRM_DEBUG_KMS("looking for preferred mode on connector %s %d\n",
-				      connector->name, connector->has_tile);
-			modes[i] = drm_has_preferred_mode(fb_conn, width,
-							  height);
-		}
-
-		/* No preferred mode marked by the EDID? Are there any modes? */
-		if (!modes[i] && !list_empty(&connector->modes)) {
-			DRM_DEBUG_KMS("using first mode listed on connector %s\n",
-				      connector->name);
-			modes[i] = list_first_entry(&connector->modes,
-						    struct drm_display_mode,
-						    head);
-		}
-
-		/* last resort: use current mode */
-		if (!modes[i]) {
-			/*
-			 * IMPORTANT: We want to use the adjusted mode (i.e.
-			 * after the panel fitter upscaling) as the initial
-			 * config, not the input mode, which is what crtc->mode
-			 * usually contains. But since our current
-			 * code puts a mode derived from the post-pfit timings
-			 * into crtc->mode this works out correctly.
-			 *
-			 * This is crtc->mode and not crtc->state->mode for the
-			 * fastboot check to work correctly. crtc_state->mode has
-			 * I915_MODE_FLAG_INHERITED, which we clear to force check
-			 * state.
-			 */
-			DRM_DEBUG_KMS("looking for current mode on connector %s\n",
-				      connector->name);
-			modes[i] = &connector->state->crtc->mode;
-		}
-		crtcs[i] = new_crtc;
-
-		DRM_DEBUG_KMS("connector %s on [CRTC:%d:%s]: %dx%d%s\n",
-			      connector->name,
-			      connector->state->crtc->base.id,
-			      connector->state->crtc->name,
-			      modes[i]->hdisplay, modes[i]->vdisplay,
-			      modes[i]->flags & DRM_MODE_FLAG_INTERLACE ? "i" :"");
-
-		fallback = false;
-		conn_configured |= BIT(i);
-	}
-
-	if ((conn_configured & mask) != mask && conn_configured != conn_seq)
-		goto retry;
-
-	/*
-	 * If the BIOS didn't enable everything it could, fall back to have the
-	 * same user experiencing of lighting up as much as possible like the
-	 * fbdev helper library.
-	 */
-	if (num_connectors_enabled != num_connectors_detected &&
-	    num_connectors_enabled < INTEL_INFO(dev_priv)->num_pipes) {
-		DRM_DEBUG_KMS("fallback: Not all outputs enabled\n");
-		DRM_DEBUG_KMS("Enabled: %i, detected: %i\n", num_connectors_enabled,
-			      num_connectors_detected);
-		fallback = true;
-	}
-
-	if (fallback) {
-bail:
-		DRM_DEBUG_KMS("Not using firmware configuration\n");
-		memcpy(enabled, save_enabled, count);
-		ret = false;
-	}
-
-	drm_modeset_drop_locks(&ctx);
-	drm_modeset_acquire_fini(&ctx);
-
-	kfree(save_enabled);
-	return ret;
-}
-
 static const struct drm_fb_helper_funcs intel_fb_helper_funcs = {
-	.initial_config = intel_fb_initial_config,
 	.fb_probe = intelfb_create,
 };
 
diff --git a/drivers/gpu/drm/i915/intel_fbdev.h b/drivers/gpu/drm/i915/intel_fbdev.h
new file mode 100644
index 000000000000..de7c84250eb5
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_fbdev.h
@@ -0,0 +1,53 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2019 Intel Corporation
+ */
+
+#ifndef __INTEL_FBDEV_H__
+#define __INTEL_FBDEV_H__
+
+#include <linux/types.h>
+
+struct drm_device;
+struct drm_i915_private;
+
+#ifdef CONFIG_DRM_FBDEV_EMULATION
+int intel_fbdev_init(struct drm_device *dev);
+void intel_fbdev_initial_config_async(struct drm_device *dev);
+void intel_fbdev_unregister(struct drm_i915_private *dev_priv);
+void intel_fbdev_fini(struct drm_i915_private *dev_priv);
+void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous);
+void intel_fbdev_output_poll_changed(struct drm_device *dev);
+void intel_fbdev_restore_mode(struct drm_device *dev);
+#else
+static inline int intel_fbdev_init(struct drm_device *dev)
+{
+	return 0;
+}
+
+static inline void intel_fbdev_initial_config_async(struct drm_device *dev)
+{
+}
+
+static inline void intel_fbdev_unregister(struct drm_i915_private *dev_priv)
+{
+}
+
+static inline void intel_fbdev_fini(struct drm_i915_private *dev_priv)
+{
+}
+
+static inline void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous)
+{
+}
+
+static inline void intel_fbdev_output_poll_changed(struct drm_device *dev)
+{
+}
+
+static inline void intel_fbdev_restore_mode(struct drm_device *dev)
+{
+}
+#endif
+
+#endif /* __INTEL_FBDEV_H__ */
diff --git a/drivers/gpu/drm/i915/intel_fifo_underrun.c b/drivers/gpu/drm/i915/intel_fifo_underrun.c
index f33de4be4b89..74c8b0528294 100644
--- a/drivers/gpu/drm/i915/intel_fifo_underrun.c
+++ b/drivers/gpu/drm/i915/intel_fifo_underrun.c
@@ -27,6 +27,7 @@
 
 #include "i915_drv.h"
 #include "intel_drv.h"
+#include "intel_fbc.h"
 
 /**
  * DOC: fifo underrun handling
diff --git a/drivers/gpu/drm/i915/intel_frontbuffer.c b/drivers/gpu/drm/i915/intel_frontbuffer.c
index 16f253deaf8d..aa34e33b6087 100644
--- a/drivers/gpu/drm/i915/intel_frontbuffer.c
+++ b/drivers/gpu/drm/i915/intel_frontbuffer.c
@@ -61,9 +61,12 @@
  */
 
 
+#include "i915_drv.h"
+#include "intel_dp.h"
 #include "intel_drv.h"
+#include "intel_fbc.h"
 #include "intel_frontbuffer.h"
-#include "i915_drv.h"
+#include "intel_psr.h"
 
 void __intel_fb_obj_invalidate(struct drm_i915_gem_object *obj,
 			       enum fb_op_origin origin,
diff --git a/drivers/gpu/drm/i915/intel_frontbuffer.h b/drivers/gpu/drm/i915/intel_frontbuffer.h
index 63cd9a753a72..d5894666f658 100644
--- a/drivers/gpu/drm/i915/intel_frontbuffer.h
+++ b/drivers/gpu/drm/i915/intel_frontbuffer.h
@@ -24,9 +24,19 @@
 #ifndef __INTEL_FRONTBUFFER_H__
 #define __INTEL_FRONTBUFFER_H__
 
+#include "i915_gem_object.h"
+
 struct drm_i915_private;
 struct drm_i915_gem_object;
 
+enum fb_op_origin {
+	ORIGIN_GTT,
+	ORIGIN_CPU,
+	ORIGIN_CS,
+	ORIGIN_FLIP,
+	ORIGIN_DIRTYFB,
+};
+
 void intel_frontbuffer_flip_prepare(struct drm_i915_private *dev_priv,
 				    unsigned frontbuffer_bits);
 void intel_frontbuffer_flip_complete(struct drm_i915_private *dev_priv,
diff --git a/drivers/gpu/drm/i915/intel_gpu_commands.h b/drivers/gpu/drm/i915/intel_gpu_commands.h
index b96a31bc1080..a34ece53a771 100644
--- a/drivers/gpu/drm/i915/intel_gpu_commands.h
+++ b/drivers/gpu/drm/i915/intel_gpu_commands.h
@@ -105,8 +105,13 @@
 #define MI_SEMAPHORE_SIGNAL	MI_INSTR(0x1b, 0) /* GEN8+ */
 #define   MI_SEMAPHORE_TARGET(engine)	((engine)<<15)
 #define MI_SEMAPHORE_WAIT	MI_INSTR(0x1c, 2) /* GEN8+ */
-#define   MI_SEMAPHORE_POLL		(1<<15)
-#define   MI_SEMAPHORE_SAD_GTE_SDD	(1<<12)
+#define   MI_SEMAPHORE_POLL		(1 << 15)
+#define   MI_SEMAPHORE_SAD_GT_SDD	(0 << 12)
+#define   MI_SEMAPHORE_SAD_GTE_SDD	(1 << 12)
+#define   MI_SEMAPHORE_SAD_LT_SDD	(2 << 12)
+#define   MI_SEMAPHORE_SAD_LTE_SDD	(3 << 12)
+#define   MI_SEMAPHORE_SAD_EQ_SDD	(4 << 12)
+#define   MI_SEMAPHORE_SAD_NEQ_SDD	(5 << 12)
 #define MI_STORE_DWORD_IMM	MI_INSTR(0x20, 1)
 #define MI_STORE_DWORD_IMM_GEN4	MI_INSTR(0x20, 2)
 #define   MI_MEM_VIRTUAL	(1 << 22) /* 945,g33,965 */
diff --git a/drivers/gpu/drm/i915/intel_guc.c b/drivers/gpu/drm/i915/intel_guc.c
index 8660af3fd755..3aabfa2d9198 100644
--- a/drivers/gpu/drm/i915/intel_guc.c
+++ b/drivers/gpu/drm/i915/intel_guc.c
@@ -54,7 +54,7 @@ void intel_guc_init_send_regs(struct intel_guc *guc)
 	BUILD_BUG_ON(GUC_MAX_MMIO_MSG_LEN > SOFT_SCRATCH_COUNT);
 
 	for (i = 0; i < guc->send_regs.count; i++) {
-		fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
+		fw_domains |= intel_uncore_forcewake_for_reg(&dev_priv->uncore,
 					guc_send_reg(guc, i),
 					FW_REG_READ | FW_REG_WRITE);
 	}
@@ -203,11 +203,19 @@ int intel_guc_init(struct intel_guc *guc)
 		goto err_log;
 	GEM_BUG_ON(!guc->ads_vma);
 
+	if (HAS_GUC_CT(dev_priv)) {
+		ret = intel_guc_ct_init(&guc->ct);
+		if (ret)
+			goto err_ads;
+	}
+
 	/* We need to notify the guc whenever we change the GGTT */
 	i915_ggtt_enable_guc(dev_priv);
 
 	return 0;
 
+err_ads:
+	intel_guc_ads_destroy(guc);
 err_log:
 	intel_guc_log_destroy(&guc->log);
 err_shared:
@@ -222,6 +230,10 @@ void intel_guc_fini(struct intel_guc *guc)
 	struct drm_i915_private *dev_priv = guc_to_i915(guc);
 
 	i915_ggtt_disable_guc(dev_priv);
+
+	if (HAS_GUC_CT(dev_priv))
+		intel_guc_ct_fini(&guc->ct);
+
 	intel_guc_ads_destroy(guc);
 	intel_guc_log_destroy(&guc->log);
 	guc_shared_data_destroy(guc);
@@ -357,14 +369,14 @@ void intel_guc_init_params(struct intel_guc *guc)
 	 * they are power context saved so it's ok to release forcewake
 	 * when we are done here and take it again at xfer time.
 	 */
-	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_BLITTER);
+	intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_BLITTER);
 
 	I915_WRITE(SOFT_SCRATCH(0), 0);
 
 	for (i = 0; i < GUC_CTL_MAX_DWORDS; i++)
 		I915_WRITE(SOFT_SCRATCH(1 + i), params[i]);
 
-	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_BLITTER);
+	intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_BLITTER);
 }
 
 int intel_guc_send_nop(struct intel_guc *guc, const u32 *action, u32 len,
@@ -386,6 +398,7 @@ int intel_guc_send_mmio(struct intel_guc *guc, const u32 *action, u32 len,
 			u32 *response_buf, u32 response_buf_size)
 {
 	struct drm_i915_private *dev_priv = guc_to_i915(guc);
+	struct intel_uncore *uncore = &dev_priv->uncore;
 	u32 status;
 	int i;
 	int ret;
@@ -402,12 +415,12 @@ int intel_guc_send_mmio(struct intel_guc *guc, const u32 *action, u32 len,
 		*action != INTEL_GUC_ACTION_DEREGISTER_COMMAND_TRANSPORT_BUFFER);
 
 	mutex_lock(&guc->send_mutex);
-	intel_uncore_forcewake_get(dev_priv, guc->send_regs.fw_domains);
+	intel_uncore_forcewake_get(uncore, guc->send_regs.fw_domains);
 
 	for (i = 0; i < len; i++)
-		I915_WRITE(guc_send_reg(guc, i), action[i]);
+		intel_uncore_write(uncore, guc_send_reg(guc, i), action[i]);
 
-	POSTING_READ(guc_send_reg(guc, i - 1));
+	intel_uncore_posting_read(uncore, guc_send_reg(guc, i - 1));
 
 	intel_guc_notify(guc);
 
@@ -415,7 +428,7 @@ int intel_guc_send_mmio(struct intel_guc *guc, const u32 *action, u32 len,
 	 * No GuC command should ever take longer than 10ms.
 	 * Fast commands should still complete in 10us.
 	 */
-	ret = __intel_wait_for_register_fw(dev_priv,
+	ret = __intel_wait_for_register_fw(uncore,
 					   guc_send_reg(guc, 0),
 					   INTEL_GUC_MSG_TYPE_MASK,
 					   INTEL_GUC_MSG_TYPE_RESPONSE <<
@@ -442,7 +455,7 @@ int intel_guc_send_mmio(struct intel_guc *guc, const u32 *action, u32 len,
 	ret = INTEL_GUC_MSG_TO_DATA(status);
 
 out:
-	intel_uncore_forcewake_put(dev_priv, guc->send_regs.fw_domains);
+	intel_uncore_forcewake_put(uncore, guc->send_regs.fw_domains);
 	mutex_unlock(&guc->send_mutex);
 
 	return ret;
@@ -472,17 +485,25 @@ void intel_guc_to_host_event_handler_mmio(struct intel_guc *guc)
 	spin_unlock(&guc->irq_lock);
 	enable_rpm_wakeref_asserts(dev_priv);
 
-	intel_guc_to_host_process_recv_msg(guc, msg);
+	intel_guc_to_host_process_recv_msg(guc, &msg, 1);
 }
 
-void intel_guc_to_host_process_recv_msg(struct intel_guc *guc, u32 msg)
+int intel_guc_to_host_process_recv_msg(struct intel_guc *guc,
+				       const u32 *payload, u32 len)
 {
+	u32 msg;
+
+	if (unlikely(!len))
+		return -EPROTO;
+
 	/* Make sure to handle only enabled messages */
-	msg &= guc->msg_enabled_mask;
+	msg = payload[0] & guc->msg_enabled_mask;
 
 	if (msg & (INTEL_GUC_RECV_MSG_FLUSH_LOG_BUFFER |
 		   INTEL_GUC_RECV_MSG_CRASH_DUMP_POSTED))
 		intel_guc_log_handle_flush_event(&guc->log);
+
+	return 0;
 }
 
 int intel_guc_sample_forcewake(struct intel_guc *guc)
@@ -544,7 +565,7 @@ static int guc_sleep_state_action(struct intel_guc *guc,
 	if (ret)
 		return ret;
 
-	ret = __intel_wait_for_register(dev_priv, SOFT_SCRATCH(14),
+	ret = __intel_wait_for_register(&dev_priv->uncore, SOFT_SCRATCH(14),
 					INTEL_GUC_SLEEP_STATE_INVALID_MASK,
 					0, 0, 10, &status);
 	if (ret)
diff --git a/drivers/gpu/drm/i915/intel_guc.h b/drivers/gpu/drm/i915/intel_guc.h
index 744220296653..2c59ff8d9f39 100644
--- a/drivers/gpu/drm/i915/intel_guc.h
+++ b/drivers/gpu/drm/i915/intel_guc.h
@@ -32,6 +32,7 @@
 #include "intel_guc_log.h"
 #include "intel_guc_reg.h"
 #include "intel_uc_fw.h"
+#include "i915_utils.h"
 #include "i915_vma.h"
 
 struct guc_preempt_work {
@@ -164,7 +165,8 @@ int intel_guc_send_mmio(struct intel_guc *guc, const u32 *action, u32 len,
 void intel_guc_to_host_event_handler(struct intel_guc *guc);
 void intel_guc_to_host_event_handler_nop(struct intel_guc *guc);
 void intel_guc_to_host_event_handler_mmio(struct intel_guc *guc);
-void intel_guc_to_host_process_recv_msg(struct intel_guc *guc, u32 msg);
+int intel_guc_to_host_process_recv_msg(struct intel_guc *guc,
+				       const u32 *payload, u32 len);
 int intel_guc_sample_forcewake(struct intel_guc *guc);
 int intel_guc_auth_huc(struct intel_guc *guc, u32 rsa_offset);
 int intel_guc_suspend(struct intel_guc *guc);
diff --git a/drivers/gpu/drm/i915/intel_guc_ads.c b/drivers/gpu/drm/i915/intel_guc_ads.c
index f0db62887f50..bec62f34b15a 100644
--- a/drivers/gpu/drm/i915/intel_guc_ads.c
+++ b/drivers/gpu/drm/i915/intel_guc_ads.c
@@ -121,8 +121,7 @@ int intel_guc_ads_create(struct intel_guc *guc)
 	 * to find it. Note that we have to skip our header (1 page),
 	 * because our GuC shared data is there.
 	 */
-	kernel_ctx_vma = to_intel_context(dev_priv->kernel_context,
-					  dev_priv->engine[RCS])->state;
+	kernel_ctx_vma = dev_priv->engine[RCS0]->kernel_context->state;
 	blob->ads.golden_context_lrca =
 		intel_guc_ggtt_offset(guc, kernel_ctx_vma) + skipped_offset;
 
diff --git a/drivers/gpu/drm/i915/intel_guc_ct.c b/drivers/gpu/drm/i915/intel_guc_ct.c
index a52883e9146f..dde1dc0d6e69 100644
--- a/drivers/gpu/drm/i915/intel_guc_ct.c
+++ b/drivers/gpu/drm/i915/intel_guc_ct.c
@@ -140,11 +140,6 @@ static int guc_action_deregister_ct_buffer(struct intel_guc *guc,
 	return err;
 }
 
-static bool ctch_is_open(struct intel_guc_ct_channel *ctch)
-{
-	return ctch->vma != NULL;
-}
-
 static int ctch_init(struct intel_guc *guc,
 		     struct intel_guc_ct_channel *ctch)
 {
@@ -214,25 +209,21 @@ err_out:
 static void ctch_fini(struct intel_guc *guc,
 		      struct intel_guc_ct_channel *ctch)
 {
+	GEM_BUG_ON(ctch->enabled);
+
 	i915_vma_unpin_and_release(&ctch->vma, I915_VMA_RELEASE_MAP);
 }
 
-static int ctch_open(struct intel_guc *guc,
-		     struct intel_guc_ct_channel *ctch)
+static int ctch_enable(struct intel_guc *guc,
+		       struct intel_guc_ct_channel *ctch)
 {
 	u32 base;
 	int err;
 	int i;
 
-	CT_DEBUG_DRIVER("CT: channel %d reopen=%s\n",
-			ctch->owner, yesno(ctch_is_open(ctch)));
+	GEM_BUG_ON(!ctch->vma);
 
-	if (!ctch->vma) {
-		err = ctch_init(guc, ctch);
-		if (unlikely(err))
-			goto err_out;
-		GEM_BUG_ON(!ctch->vma);
-	}
+	GEM_BUG_ON(ctch->enabled);
 
 	/* vma should be already allocated and map'ed */
 	base = intel_guc_ggtt_offset(guc, ctch->vma);
@@ -255,7 +246,7 @@ static int ctch_open(struct intel_guc *guc,
 					    base + PAGE_SIZE/4 * CTB_RECV,
 					    INTEL_GUC_CT_BUFFER_TYPE_RECV);
 	if (unlikely(err))
-		goto err_fini;
+		goto err_out;
 
 	err = guc_action_register_ct_buffer(guc,
 					    base + PAGE_SIZE/4 * CTB_SEND,
@@ -263,23 +254,25 @@ static int ctch_open(struct intel_guc *guc,
 	if (unlikely(err))
 		goto err_deregister;
 
+	ctch->enabled = true;
+
 	return 0;
 
 err_deregister:
 	guc_action_deregister_ct_buffer(guc,
 					ctch->owner,
 					INTEL_GUC_CT_BUFFER_TYPE_RECV);
-err_fini:
-	ctch_fini(guc, ctch);
 err_out:
 	DRM_ERROR("CT: can't open channel %d; err=%d\n", ctch->owner, err);
 	return err;
 }
 
-static void ctch_close(struct intel_guc *guc,
-		       struct intel_guc_ct_channel *ctch)
+static void ctch_disable(struct intel_guc *guc,
+			 struct intel_guc_ct_channel *ctch)
 {
-	GEM_BUG_ON(!ctch_is_open(ctch));
+	GEM_BUG_ON(!ctch->enabled);
+
+	ctch->enabled = false;
 
 	guc_action_deregister_ct_buffer(guc,
 					ctch->owner,
@@ -287,7 +280,6 @@ static void ctch_close(struct intel_guc *guc,
 	guc_action_deregister_ct_buffer(guc,
 					ctch->owner,
 					INTEL_GUC_CT_BUFFER_TYPE_RECV);
-	ctch_fini(guc, ctch);
 }
 
 static u32 ctch_get_next_fence(struct intel_guc_ct_channel *ctch)
@@ -481,7 +473,7 @@ static int ctch_send(struct intel_guc_ct *ct,
 	u32 fence;
 	int err;
 
-	GEM_BUG_ON(!ctch_is_open(ctch));
+	GEM_BUG_ON(!ctch->enabled);
 	GEM_BUG_ON(!len);
 	GEM_BUG_ON(len & ~GUC_CT_MSG_LEN_MASK);
 	GEM_BUG_ON(!response_buf && response_buf_size);
@@ -709,14 +701,15 @@ static void ct_process_request(struct intel_guc_ct *ct,
 			       u32 action, u32 len, const u32 *payload)
 {
 	struct intel_guc *guc = ct_to_guc(ct);
+	int ret;
 
 	CT_DEBUG_DRIVER("CT: request %x %*ph\n", action, 4 * len, payload);
 
 	switch (action) {
 	case INTEL_GUC_ACTION_DEFAULT:
-		if (unlikely(len < 1))
+		ret = intel_guc_to_host_process_recv_msg(guc, payload, len);
+		if (unlikely(ret))
 			goto fail_unexpected;
-		intel_guc_to_host_process_recv_msg(guc, *payload);
 		break;
 
 	default:
@@ -817,7 +810,7 @@ static void ct_process_host_channel(struct intel_guc_ct *ct)
 	u32 msg[GUC_CT_MSG_LEN_MASK + 1]; /* one extra dw for the header */
 	int err = 0;
 
-	if (!ctch_is_open(ctch))
+	if (!ctch->enabled)
 		return;
 
 	do {
@@ -849,6 +842,51 @@ static void intel_guc_to_host_event_handler_ct(struct intel_guc *guc)
 }
 
 /**
+ * intel_guc_ct_init - Init CT communication
+ * @ct: pointer to CT struct
+ *
+ * Allocate memory required for communication via
+ * the CT channel.
+ *
+ * Shall only be called for platforms with HAS_GUC_CT.
+ *
+ * Return: 0 on success, a negative errno code on failure.
+ */
+int intel_guc_ct_init(struct intel_guc_ct *ct)
+{
+	struct intel_guc *guc = ct_to_guc(ct);
+	struct intel_guc_ct_channel *ctch = &ct->host_channel;
+	int err;
+
+	err = ctch_init(guc, ctch);
+	if (unlikely(err)) {
+		DRM_ERROR("CT: can't open channel %d; err=%d\n",
+			  ctch->owner, err);
+		return err;
+	}
+
+	GEM_BUG_ON(!ctch->vma);
+	return 0;
+}
+
+/**
+ * intel_guc_ct_fini - Fini CT communication
+ * @ct: pointer to CT struct
+ *
+ * Deallocate memory required for communication via
+ * the CT channel.
+ *
+ * Shall only be called for platforms with HAS_GUC_CT.
+ */
+void intel_guc_ct_fini(struct intel_guc_ct *ct)
+{
+	struct intel_guc *guc = ct_to_guc(ct);
+	struct intel_guc_ct_channel *ctch = &ct->host_channel;
+
+	ctch_fini(guc, ctch);
+}
+
+/**
  * intel_guc_ct_enable - Enable buffer based command transport.
  * @ct: pointer to CT struct
  *
@@ -865,7 +903,10 @@ int intel_guc_ct_enable(struct intel_guc_ct *ct)
 
 	GEM_BUG_ON(!HAS_GUC_CT(i915));
 
-	err = ctch_open(guc, ctch);
+	if (ctch->enabled)
+		return 0;
+
+	err = ctch_enable(guc, ctch);
 	if (unlikely(err))
 		return err;
 
@@ -890,10 +931,10 @@ void intel_guc_ct_disable(struct intel_guc_ct *ct)
 
 	GEM_BUG_ON(!HAS_GUC_CT(i915));
 
-	if (!ctch_is_open(ctch))
+	if (!ctch->enabled)
 		return;
 
-	ctch_close(guc, ctch);
+	ctch_disable(guc, ctch);
 
 	/* Disable send */
 	guc->send = intel_guc_send_nop;
diff --git a/drivers/gpu/drm/i915/intel_guc_ct.h b/drivers/gpu/drm/i915/intel_guc_ct.h
index d774895ab143..f5e7f0663304 100644
--- a/drivers/gpu/drm/i915/intel_guc_ct.h
+++ b/drivers/gpu/drm/i915/intel_guc_ct.h
@@ -66,6 +66,7 @@ struct intel_guc_ct_channel {
 	struct intel_guc_ct_buffer ctbs[2];
 	u32 owner;
 	u32 next_fence;
+	bool enabled;
 };
 
 /** Holds all command transport channels.
@@ -90,6 +91,8 @@ struct intel_guc_ct {
 };
 
 void intel_guc_ct_init_early(struct intel_guc_ct *ct);
+int intel_guc_ct_init(struct intel_guc_ct *ct);
+void intel_guc_ct_fini(struct intel_guc_ct *ct);
 int intel_guc_ct_enable(struct intel_guc_ct *ct);
 void intel_guc_ct_disable(struct intel_guc_ct *ct);
 
diff --git a/drivers/gpu/drm/i915/intel_guc_fw.c b/drivers/gpu/drm/i915/intel_guc_fw.c
index 13ff7003c6be..792a551450c7 100644
--- a/drivers/gpu/drm/i915/intel_guc_fw.c
+++ b/drivers/gpu/drm/i915/intel_guc_fw.c
@@ -241,7 +241,7 @@ static int guc_fw_xfer(struct intel_uc_fw *guc_fw, struct i915_vma *vma)
 
 	GEM_BUG_ON(guc_fw->type != INTEL_UC_FW_TYPE_GUC);
 
-	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
 
 	guc_prepare_xfer(guc);
 
@@ -254,7 +254,7 @@ static int guc_fw_xfer(struct intel_uc_fw *guc_fw, struct i915_vma *vma)
 
 	ret = guc_xfer_ucode(guc, vma);
 
-	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
 
 	return ret;
 }
diff --git a/drivers/gpu/drm/i915/intel_guc_log.c b/drivers/gpu/drm/i915/intel_guc_log.c
index 806fdfd7c78a..7146524264dd 100644
--- a/drivers/gpu/drm/i915/intel_guc_log.c
+++ b/drivers/gpu/drm/i915/intel_guc_log.c
@@ -620,7 +620,12 @@ void intel_guc_log_relay_flush(struct intel_guc_log *log)
 
 void intel_guc_log_relay_close(struct intel_guc_log *log)
 {
+	struct intel_guc *guc = log_to_guc(log);
+	struct drm_i915_private *i915 = guc_to_i915(guc);
+
 	guc_log_disable_flush_events(log);
+	synchronize_irq(i915->drm.irq);
+
 	flush_work(&log->relay.flush_work);
 
 	mutex_lock(&log->relay.lock);
diff --git a/drivers/gpu/drm/i915/intel_guc_submission.c b/drivers/gpu/drm/i915/intel_guc_submission.c
index 8bc8aa54aa35..37f60cb8e9e1 100644
--- a/drivers/gpu/drm/i915/intel_guc_submission.c
+++ b/drivers/gpu/drm/i915/intel_guc_submission.c
@@ -382,7 +382,7 @@ static void guc_stage_desc_init(struct intel_guc_client *client)
 	desc->db_id = client->doorbell_id;
 
 	for_each_engine_masked(engine, dev_priv, client->engines, tmp) {
-		struct intel_context *ce = to_intel_context(ctx, engine);
+		struct intel_context *ce = intel_context_lookup(ctx, engine);
 		u32 guc_engine_id = engine->guc_id;
 		struct guc_execlist_context *lrc = &desc->lrc[guc_engine_id];
 
@@ -393,7 +393,7 @@ static void guc_stage_desc_init(struct intel_guc_client *client)
 		 * for now who owns a GuC client. But for future owner of GuC
 		 * client, need to make sure lrc is pinned prior to enter here.
 		 */
-		if (!ce->state)
+		if (!ce || !ce->state)
 			break;	/* XXX: continue? */
 
 		/*
@@ -535,7 +535,7 @@ static void guc_add_request(struct intel_guc *guc, struct i915_request *rq)
 	spin_lock(&client->wq_lock);
 
 	guc_wq_item_append(client, engine->guc_id, ctx_desc,
-			   ring_tail, rq->global_seqno);
+			   ring_tail, rq->fence.seqno);
 	guc_ring_doorbell(client);
 
 	client->submissions[engine->id] += 1;
@@ -567,7 +567,7 @@ static void inject_preempt_context(struct work_struct *work)
 					     preempt_work[engine->id]);
 	struct intel_guc_client *client = guc->preempt_client;
 	struct guc_stage_desc *stage_desc = __get_stage_desc(client);
-	struct intel_context *ce = to_intel_context(client->owner, engine);
+	struct intel_context *ce = engine->preempt_context;
 	u32 data[7];
 
 	if (!ce->ring->emit) { /* recreate upon load/resume */
@@ -575,7 +575,7 @@ static void inject_preempt_context(struct work_struct *work)
 		u32 *cs;
 
 		cs = ce->ring->vaddr;
-		if (engine->id == RCS) {
+		if (engine->class == RENDER_CLASS) {
 			cs = gen8_emit_ggtt_write_rcs(cs,
 						      GUC_PREEMPT_FINISHED,
 						      addr,
@@ -583,7 +583,8 @@ static void inject_preempt_context(struct work_struct *work)
 		} else {
 			cs = gen8_emit_ggtt_write(cs,
 						  GUC_PREEMPT_FINISHED,
-						  addr);
+						  addr,
+						  0);
 			*cs++ = MI_NOOP;
 			*cs++ = MI_NOOP;
 		}
@@ -649,9 +650,10 @@ static void wait_for_guc_preempt_report(struct intel_engine_cs *engine)
 	struct guc_ctx_report *report =
 		&data->preempt_ctx_report[engine->guc_id];
 
-	WARN_ON(wait_for_atomic(report->report_return_status ==
-				INTEL_GUC_REPORT_STATUS_COMPLETE,
-				GUC_PREEMPT_POSTPROCESS_DELAY_MS));
+	if (wait_for_atomic(report->report_return_status ==
+			    INTEL_GUC_REPORT_STATUS_COMPLETE,
+			    GUC_PREEMPT_POSTPROCESS_DELAY_MS))
+		DRM_ERROR("Timed out waiting for GuC preemption report\n");
 	/*
 	 * GuC is expecting that we're also going to clear the affected context
 	 * counter, let's also reset the return status to not depend on GuC
@@ -720,7 +722,7 @@ static inline int rq_prio(const struct i915_request *rq)
 
 static inline int port_prio(const struct execlist_port *port)
 {
-	return rq_prio(port_request(port));
+	return rq_prio(port_request(port)) | __NO_PREEMPTION;
 }
 
 static bool __guc_dequeue(struct intel_engine_cs *engine)
@@ -781,8 +783,7 @@ static bool __guc_dequeue(struct intel_engine_cs *engine)
 		}
 
 		rb_erase_cached(&p->node, &execlists->queue);
-		if (p->priority != I915_PRIORITY_NORMAL)
-			kmem_cache_free(engine->i915->priorities, p);
+		i915_priolist_free(p);
 	}
 done:
 	execlists->queue_priority_hint =
@@ -871,6 +872,104 @@ static void guc_reset_prepare(struct intel_engine_cs *engine)
 		flush_workqueue(engine->i915->guc.preempt_wq);
 }
 
+static void guc_reset(struct intel_engine_cs *engine, bool stalled)
+{
+	struct intel_engine_execlists * const execlists = &engine->execlists;
+	struct i915_request *rq;
+	unsigned long flags;
+
+	spin_lock_irqsave(&engine->timeline.lock, flags);
+
+	execlists_cancel_port_requests(execlists);
+
+	/* Push back any incomplete requests for replay after the reset. */
+	rq = execlists_unwind_incomplete_requests(execlists);
+	if (!rq)
+		goto out_unlock;
+
+	if (!i915_request_started(rq))
+		stalled = false;
+
+	i915_reset_request(rq, stalled);
+	intel_lr_context_reset(engine, rq->hw_context, rq->head, stalled);
+
+out_unlock:
+	spin_unlock_irqrestore(&engine->timeline.lock, flags);
+}
+
+static void guc_cancel_requests(struct intel_engine_cs *engine)
+{
+	struct intel_engine_execlists * const execlists = &engine->execlists;
+	struct i915_request *rq, *rn;
+	struct rb_node *rb;
+	unsigned long flags;
+
+	GEM_TRACE("%s\n", engine->name);
+
+	/*
+	 * Before we call engine->cancel_requests(), we should have exclusive
+	 * access to the submission state. This is arranged for us by the
+	 * caller disabling the interrupt generation, the tasklet and other
+	 * threads that may then access the same state, giving us a free hand
+	 * to reset state. However, we still need to let lockdep be aware that
+	 * we know this state may be accessed in hardirq context, so we
+	 * disable the irq around this manipulation and we want to keep
+	 * the spinlock focused on its duties and not accidentally conflate
+	 * coverage to the submission's irq state. (Similarly, although we
+	 * shouldn't need to disable irq around the manipulation of the
+	 * submission's irq state, we also wish to remind ourselves that
+	 * it is irq state.)
+	 */
+	spin_lock_irqsave(&engine->timeline.lock, flags);
+
+	/* Cancel the requests on the HW and clear the ELSP tracker. */
+	execlists_cancel_port_requests(execlists);
+
+	/* Mark all executing requests as skipped. */
+	list_for_each_entry(rq, &engine->timeline.requests, link) {
+		if (!i915_request_signaled(rq))
+			dma_fence_set_error(&rq->fence, -EIO);
+
+		i915_request_mark_complete(rq);
+	}
+
+	/* Flush the queued requests to the timeline list (for retiring). */
+	while ((rb = rb_first_cached(&execlists->queue))) {
+		struct i915_priolist *p = to_priolist(rb);
+		int i;
+
+		priolist_for_each_request_consume(rq, rn, p, i) {
+			list_del_init(&rq->sched.link);
+			__i915_request_submit(rq);
+			dma_fence_set_error(&rq->fence, -EIO);
+			i915_request_mark_complete(rq);
+		}
+
+		rb_erase_cached(&p->node, &execlists->queue);
+		i915_priolist_free(p);
+	}
+
+	/* Remaining _unready_ requests will be nop'ed when submitted */
+
+	execlists->queue_priority_hint = INT_MIN;
+	execlists->queue = RB_ROOT_CACHED;
+	GEM_BUG_ON(port_isset(execlists->port));
+
+	spin_unlock_irqrestore(&engine->timeline.lock, flags);
+}
+
+static void guc_reset_finish(struct intel_engine_cs *engine)
+{
+	struct intel_engine_execlists * const execlists = &engine->execlists;
+
+	if (__tasklet_enable(&execlists->tasklet))
+		/* And kick in case we missed a new request submission. */
+		tasklet_hi_schedule(&execlists->tasklet);
+
+	GEM_TRACE("%s: depth->%d\n", engine->name,
+		  atomic_read(&execlists->tasklet.count));
+}
+
 /*
  * Everything below here is concerned with setup & teardown, and is
  * therefore not part of the somewhat time-critical batch-submission
@@ -1031,7 +1130,7 @@ static int guc_clients_create(struct intel_guc *guc)
 	GEM_BUG_ON(guc->preempt_client);
 
 	client = guc_client_alloc(dev_priv,
-				  INTEL_INFO(dev_priv)->ring_mask,
+				  INTEL_INFO(dev_priv)->engine_mask,
 				  GUC_CLIENT_PRIORITY_KMD_NORMAL,
 				  dev_priv->kernel_context);
 	if (IS_ERR(client)) {
@@ -1042,7 +1141,7 @@ static int guc_clients_create(struct intel_guc *guc)
 
 	if (dev_priv->preempt_context) {
 		client = guc_client_alloc(dev_priv,
-					  INTEL_INFO(dev_priv)->ring_mask,
+					  INTEL_INFO(dev_priv)->engine_mask,
 					  GUC_CLIENT_PRIORITY_KMD_HIGH,
 					  dev_priv->preempt_context);
 		if (IS_ERR(client)) {
@@ -1262,10 +1361,12 @@ static void guc_interrupts_release(struct drm_i915_private *dev_priv)
 static void guc_submission_park(struct intel_engine_cs *engine)
 {
 	intel_engine_unpin_breadcrumbs_irq(engine);
+	engine->flags &= ~I915_ENGINE_NEEDS_BREADCRUMB_TASKLET;
 }
 
 static void guc_submission_unpark(struct intel_engine_cs *engine)
 {
+	engine->flags |= I915_ENGINE_NEEDS_BREADCRUMB_TASKLET;
 	intel_engine_pin_breadcrumbs_irq(engine);
 }
 
@@ -1290,6 +1391,10 @@ static void guc_set_default_submission(struct intel_engine_cs *engine)
 	engine->unpark = guc_submission_unpark;
 
 	engine->reset.prepare = guc_reset_prepare;
+	engine->reset.reset = guc_reset;
+	engine->reset.finish = guc_reset_finish;
+
+	engine->cancel_requests = guc_cancel_requests;
 
 	engine->flags &= ~I915_ENGINE_SUPPORTS_STATS;
 }
diff --git a/drivers/gpu/drm/i915/intel_guc_submission.h b/drivers/gpu/drm/i915/intel_guc_submission.h
index 169c54568340..aa5e6749c925 100644
--- a/drivers/gpu/drm/i915/intel_guc_submission.h
+++ b/drivers/gpu/drm/i915/intel_guc_submission.h
@@ -29,6 +29,7 @@
 
 #include "i915_gem.h"
 #include "i915_selftest.h"
+#include "intel_engine_types.h"
 
 struct drm_i915_private;
 
diff --git a/drivers/gpu/drm/i915/intel_hangcheck.c b/drivers/gpu/drm/i915/intel_hangcheck.c
index a219c796e56d..3d51ed1428d4 100644
--- a/drivers/gpu/drm/i915/intel_hangcheck.c
+++ b/drivers/gpu/drm/i915/intel_hangcheck.c
@@ -56,7 +56,7 @@ static bool subunits_stuck(struct intel_engine_cs *engine)
 	int slice;
 	int subslice;
 
-	if (engine->id != RCS)
+	if (engine->id != RCS0)
 		return true;
 
 	intel_engine_get_instdone(engine, &instdone);
@@ -118,11 +118,11 @@ engine_stuck(struct intel_engine_cs *engine, u64 acthd)
 	 * and break the hang. This should work on
 	 * all but the second generation chipsets.
 	 */
-	tmp = I915_READ_CTL(engine);
+	tmp = ENGINE_READ(engine, RING_CTL);
 	if (tmp & RING_WAIT) {
-		i915_handle_error(dev_priv, BIT(engine->id), 0,
+		i915_handle_error(dev_priv, engine->mask, 0,
 				  "stuck wait on %s", engine->name);
-		I915_WRITE_CTL(engine, tmp);
+		ENGINE_WRITE(engine, RING_CTL, tmp);
 		return ENGINE_WAIT_KICK;
 	}
 
@@ -133,21 +133,21 @@ static void hangcheck_load_sample(struct intel_engine_cs *engine,
 				  struct hangcheck *hc)
 {
 	hc->acthd = intel_engine_get_active_head(engine);
-	hc->seqno = intel_engine_get_seqno(engine);
+	hc->seqno = intel_engine_get_hangcheck_seqno(engine);
 }
 
 static void hangcheck_store_sample(struct intel_engine_cs *engine,
 				   const struct hangcheck *hc)
 {
 	engine->hangcheck.acthd = hc->acthd;
-	engine->hangcheck.seqno = hc->seqno;
+	engine->hangcheck.last_seqno = hc->seqno;
 }
 
 static enum intel_engine_hangcheck_action
 hangcheck_get_action(struct intel_engine_cs *engine,
 		     const struct hangcheck *hc)
 {
-	if (engine->hangcheck.seqno != hc->seqno)
+	if (engine->hangcheck.last_seqno != hc->seqno)
 		return ENGINE_ACTIVE_SEQNO;
 
 	if (intel_engine_is_idle(engine))
@@ -221,8 +221,8 @@ static void hangcheck_declare_hang(struct drm_i915_private *i915,
 				   unsigned int stuck)
 {
 	struct intel_engine_cs *engine;
+	intel_engine_mask_t tmp;
 	char msg[80];
-	unsigned int tmp;
 	int len;
 
 	/* If some rings hung but others were still busy, only
@@ -263,14 +263,14 @@ static void i915_hangcheck_elapsed(struct work_struct *work)
 	if (!READ_ONCE(dev_priv->gt.awake))
 		return;
 
-	if (i915_terminally_wedged(&dev_priv->gpu_error))
+	if (i915_terminally_wedged(dev_priv))
 		return;
 
 	/* As enabling the GPU requires fairly extensive mmio access,
 	 * periodically arm the mmio checker to see if we are triggering
 	 * any invalid access.
 	 */
-	intel_uncore_arm_unclaimed_mmio_detection(dev_priv);
+	intel_uncore_arm_unclaimed_mmio_detection(&dev_priv->uncore);
 
 	for_each_engine(engine, dev_priv, id) {
 		struct hangcheck hc;
@@ -282,13 +282,13 @@ static void i915_hangcheck_elapsed(struct work_struct *work)
 		hangcheck_store_sample(engine, &hc);
 
 		if (hc.stalled) {
-			hung |= intel_engine_flag(engine);
+			hung |= engine->mask;
 			if (hc.action != ENGINE_DEAD)
-				stuck |= intel_engine_flag(engine);
+				stuck |= engine->mask;
 		}
 
 		if (hc.wedged)
-			wedged |= intel_engine_flag(engine);
+			wedged |= engine->mask;
 	}
 
 	if (GEM_SHOW_DEBUG() && (hung | stuck)) {
diff --git a/drivers/gpu/drm/i915/intel_hdcp.c b/drivers/gpu/drm/i915/intel_hdcp.c
index ce7ba3a9c000..99b007169c49 100644
--- a/drivers/gpu/drm/i915/intel_hdcp.c
+++ b/drivers/gpu/drm/i915/intel_hdcp.c
@@ -6,15 +6,20 @@
  * Sean Paul <seanpaul@chromium.org>
  */
 
-#include <drm/drm_hdcp.h>
+#include <linux/component.h>
 #include <linux/i2c.h>
 #include <linux/random.h>
 
-#include "intel_drv.h"
+#include <drm/drm_hdcp.h>
+#include <drm/i915_component.h>
+
 #include "i915_reg.h"
+#include "intel_drv.h"
+#include "intel_hdcp.h"
 
 #define KEY_LOAD_TRIES	5
 #define ENCRYPT_STATUS_CHANGE_TIMEOUT_MS	50
+#define HDCP2_LC_RETRY_CNT			3
 
 static
 bool intel_hdcp_is_ksv_valid(u8 *ksv)
@@ -72,6 +77,52 @@ bool intel_hdcp_capable(struct intel_connector *connector)
 	return capable;
 }
 
+/* Is HDCP2.2 capable on Platform and Sink */
+static bool intel_hdcp2_capable(struct intel_connector *connector)
+{
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+	struct intel_digital_port *intel_dig_port = conn_to_dig_port(connector);
+	struct intel_hdcp *hdcp = &connector->hdcp;
+	bool capable = false;
+
+	/* I915 support for HDCP2.2 */
+	if (!hdcp->hdcp2_supported)
+		return false;
+
+	/* MEI interface is solid */
+	mutex_lock(&dev_priv->hdcp_comp_mutex);
+	if (!dev_priv->hdcp_comp_added ||  !dev_priv->hdcp_master) {
+		mutex_unlock(&dev_priv->hdcp_comp_mutex);
+		return false;
+	}
+	mutex_unlock(&dev_priv->hdcp_comp_mutex);
+
+	/* Sink's capability for HDCP2.2 */
+	hdcp->shim->hdcp_2_2_capable(intel_dig_port, &capable);
+
+	return capable;
+}
+
+static inline bool intel_hdcp_in_use(struct intel_connector *connector)
+{
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+	enum port port = connector->encoder->port;
+	u32 reg;
+
+	reg = I915_READ(PORT_HDCP_STATUS(port));
+	return reg & HDCP_STATUS_ENC;
+}
+
+static inline bool intel_hdcp2_in_use(struct intel_connector *connector)
+{
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+	enum port port = connector->encoder->port;
+	u32 reg;
+
+	reg = I915_READ(HDCP2_STATUS_DDI(port));
+	return reg & LINK_ENCRYPTION_STATUS;
+}
+
 static int intel_hdcp_poll_ksv_fifo(struct intel_digital_port *intel_dig_port,
 				    const struct intel_hdcp_shim *shim)
 {
@@ -176,7 +227,7 @@ static int intel_hdcp_load_keys(struct drm_i915_private *dev_priv)
 	}
 
 	/* Wait for the keys to load (500us) */
-	ret = __intel_wait_for_register(dev_priv, HDCP_KEY_STATUS,
+	ret = __intel_wait_for_register(&dev_priv->uncore, HDCP_KEY_STATUS,
 					HDCP_KEY_LOAD_DONE, HDCP_KEY_LOAD_DONE,
 					10, 1, &val);
 	if (ret)
@@ -194,7 +245,7 @@ static int intel_hdcp_load_keys(struct drm_i915_private *dev_priv)
 static int intel_write_sha_text(struct drm_i915_private *dev_priv, u32 sha_text)
 {
 	I915_WRITE(HDCP_SHA_TEXT, sha_text);
-	if (intel_wait_for_register(dev_priv, HDCP_REP_CTL,
+	if (intel_wait_for_register(&dev_priv->uncore, HDCP_REP_CTL,
 				    HDCP_SHA1_READY, HDCP_SHA1_READY, 1)) {
 		DRM_ERROR("Timed out waiting for SHA1 ready\n");
 		return -ETIMEDOUT;
@@ -425,7 +476,7 @@ int intel_hdcp_validate_v_prime(struct intel_digital_port *intel_dig_port,
 
 	/* Tell the HW we're done with the hash and wait for it to ACK */
 	I915_WRITE(HDCP_REP_CTL, rep_ctl | HDCP_SHA1_COMPLETE_HASH);
-	if (intel_wait_for_register(dev_priv, HDCP_REP_CTL,
+	if (intel_wait_for_register(&dev_priv->uncore, HDCP_REP_CTL,
 				    HDCP_SHA1_COMPLETE,
 				    HDCP_SHA1_COMPLETE, 1)) {
 		DRM_ERROR("Timed out waiting for SHA1 complete\n");
@@ -555,7 +606,7 @@ static int intel_hdcp_auth(struct intel_digital_port *intel_dig_port,
 	I915_WRITE(PORT_HDCP_CONF(port), HDCP_CONF_CAPTURE_AN);
 
 	/* Wait for An to be acquired */
-	if (intel_wait_for_register(dev_priv, PORT_HDCP_STATUS(port),
+	if (intel_wait_for_register(&dev_priv->uncore, PORT_HDCP_STATUS(port),
 				    HDCP_STATUS_AN_READY,
 				    HDCP_STATUS_AN_READY, 1)) {
 		DRM_ERROR("Timed out waiting for An\n");
@@ -636,7 +687,7 @@ static int intel_hdcp_auth(struct intel_digital_port *intel_dig_port,
 	}
 
 	/* Wait for encryption confirmation */
-	if (intel_wait_for_register(dev_priv, PORT_HDCP_STATUS(port),
+	if (intel_wait_for_register(&dev_priv->uncore, PORT_HDCP_STATUS(port),
 				    HDCP_STATUS_ENC, HDCP_STATUS_ENC,
 				    ENCRYPT_STATUS_CHANGE_TIMEOUT_MS)) {
 		DRM_ERROR("Timed out waiting for encryption\n");
@@ -666,8 +717,10 @@ static int _intel_hdcp_disable(struct intel_connector *connector)
 	DRM_DEBUG_KMS("[%s:%d] HDCP is being disabled...\n",
 		      connector->base.name, connector->base.base.id);
 
+	hdcp->hdcp_encrypted = false;
 	I915_WRITE(PORT_HDCP_CONF(port), 0);
-	if (intel_wait_for_register(dev_priv, PORT_HDCP_STATUS(port), ~0, 0,
+	if (intel_wait_for_register(&dev_priv->uncore,
+				    PORT_HDCP_STATUS(port), ~0, 0,
 				    ENCRYPT_STATUS_CHANGE_TIMEOUT_MS)) {
 		DRM_ERROR("Failed to disable HDCP, timeout clearing status\n");
 		return -ETIMEDOUT;
@@ -711,8 +764,10 @@ static int _intel_hdcp_enable(struct intel_connector *connector)
 	/* Incase of authentication failures, HDCP spec expects reauth. */
 	for (i = 0; i < tries; i++) {
 		ret = intel_hdcp_auth(conn_to_dig_port(connector), hdcp->shim);
-		if (!ret)
+		if (!ret) {
+			hdcp->hdcp_encrypted = true;
 			return 0;
+		}
 
 		DRM_DEBUG_KMS("HDCP Auth failure (%d)\n", ret);
 
@@ -730,16 +785,64 @@ struct intel_connector *intel_hdcp_to_connector(struct intel_hdcp *hdcp)
 	return container_of(hdcp, struct intel_connector, hdcp);
 }
 
-static void intel_hdcp_check_work(struct work_struct *work)
+/* Implements Part 3 of the HDCP authorization procedure */
+static int intel_hdcp_check_link(struct intel_connector *connector)
 {
-	struct intel_hdcp *hdcp = container_of(to_delayed_work(work),
-					       struct intel_hdcp,
-					       check_work);
-	struct intel_connector *connector = intel_hdcp_to_connector(hdcp);
+	struct intel_hdcp *hdcp = &connector->hdcp;
+	struct drm_i915_private *dev_priv = connector->base.dev->dev_private;
+	struct intel_digital_port *intel_dig_port = conn_to_dig_port(connector);
+	enum port port = intel_dig_port->base.port;
+	int ret = 0;
 
-	if (!intel_hdcp_check_link(connector))
-		schedule_delayed_work(&hdcp->check_work,
-				      DRM_HDCP_CHECK_PERIOD_MS);
+	mutex_lock(&hdcp->mutex);
+
+	/* Check_link valid only when HDCP1.4 is enabled */
+	if (hdcp->value != DRM_MODE_CONTENT_PROTECTION_ENABLED ||
+	    !hdcp->hdcp_encrypted) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	if (WARN_ON(!intel_hdcp_in_use(connector))) {
+		DRM_ERROR("%s:%d HDCP link stopped encryption,%x\n",
+			  connector->base.name, connector->base.base.id,
+			  I915_READ(PORT_HDCP_STATUS(port)));
+		ret = -ENXIO;
+		hdcp->value = DRM_MODE_CONTENT_PROTECTION_DESIRED;
+		schedule_work(&hdcp->prop_work);
+		goto out;
+	}
+
+	if (hdcp->shim->check_link(intel_dig_port)) {
+		if (hdcp->value != DRM_MODE_CONTENT_PROTECTION_UNDESIRED) {
+			hdcp->value = DRM_MODE_CONTENT_PROTECTION_ENABLED;
+			schedule_work(&hdcp->prop_work);
+		}
+		goto out;
+	}
+
+	DRM_DEBUG_KMS("[%s:%d] HDCP link failed, retrying authentication\n",
+		      connector->base.name, connector->base.base.id);
+
+	ret = _intel_hdcp_disable(connector);
+	if (ret) {
+		DRM_ERROR("Failed to disable hdcp (%d)\n", ret);
+		hdcp->value = DRM_MODE_CONTENT_PROTECTION_DESIRED;
+		schedule_work(&hdcp->prop_work);
+		goto out;
+	}
+
+	ret = _intel_hdcp_enable(connector);
+	if (ret) {
+		DRM_ERROR("Failed to enable hdcp (%d)\n", ret);
+		hdcp->value = DRM_MODE_CONTENT_PROTECTION_DESIRED;
+		schedule_work(&hdcp->prop_work);
+		goto out;
+	}
+
+out:
+	mutex_unlock(&hdcp->mutex);
+	return ret;
 }
 
 static void intel_hdcp_prop_work(struct work_struct *work)
@@ -773,14 +876,929 @@ bool is_hdcp_supported(struct drm_i915_private *dev_priv, enum port port)
 	return INTEL_GEN(dev_priv) >= 9 && port < PORT_E;
 }
 
+static int
+hdcp2_prepare_ake_init(struct intel_connector *connector,
+		       struct hdcp2_ake_init *ake_data)
+{
+	struct hdcp_port_data *data = &connector->hdcp.port_data;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+	struct i915_hdcp_comp_master *comp;
+	int ret;
+
+	mutex_lock(&dev_priv->hdcp_comp_mutex);
+	comp = dev_priv->hdcp_master;
+
+	if (!comp || !comp->ops) {
+		mutex_unlock(&dev_priv->hdcp_comp_mutex);
+		return -EINVAL;
+	}
+
+	ret = comp->ops->initiate_hdcp2_session(comp->mei_dev, data, ake_data);
+	if (ret)
+		DRM_DEBUG_KMS("Prepare_ake_init failed. %d\n", ret);
+	mutex_unlock(&dev_priv->hdcp_comp_mutex);
+
+	return ret;
+}
+
+static int
+hdcp2_verify_rx_cert_prepare_km(struct intel_connector *connector,
+				struct hdcp2_ake_send_cert *rx_cert,
+				bool *paired,
+				struct hdcp2_ake_no_stored_km *ek_pub_km,
+				size_t *msg_sz)
+{
+	struct hdcp_port_data *data = &connector->hdcp.port_data;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+	struct i915_hdcp_comp_master *comp;
+	int ret;
+
+	mutex_lock(&dev_priv->hdcp_comp_mutex);
+	comp = dev_priv->hdcp_master;
+
+	if (!comp || !comp->ops) {
+		mutex_unlock(&dev_priv->hdcp_comp_mutex);
+		return -EINVAL;
+	}
+
+	ret = comp->ops->verify_receiver_cert_prepare_km(comp->mei_dev, data,
+							 rx_cert, paired,
+							 ek_pub_km, msg_sz);
+	if (ret < 0)
+		DRM_DEBUG_KMS("Verify rx_cert failed. %d\n", ret);
+	mutex_unlock(&dev_priv->hdcp_comp_mutex);
+
+	return ret;
+}
+
+static int hdcp2_verify_hprime(struct intel_connector *connector,
+			       struct hdcp2_ake_send_hprime *rx_hprime)
+{
+	struct hdcp_port_data *data = &connector->hdcp.port_data;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+	struct i915_hdcp_comp_master *comp;
+	int ret;
+
+	mutex_lock(&dev_priv->hdcp_comp_mutex);
+	comp = dev_priv->hdcp_master;
+
+	if (!comp || !comp->ops) {
+		mutex_unlock(&dev_priv->hdcp_comp_mutex);
+		return -EINVAL;
+	}
+
+	ret = comp->ops->verify_hprime(comp->mei_dev, data, rx_hprime);
+	if (ret < 0)
+		DRM_DEBUG_KMS("Verify hprime failed. %d\n", ret);
+	mutex_unlock(&dev_priv->hdcp_comp_mutex);
+
+	return ret;
+}
+
+static int
+hdcp2_store_pairing_info(struct intel_connector *connector,
+			 struct hdcp2_ake_send_pairing_info *pairing_info)
+{
+	struct hdcp_port_data *data = &connector->hdcp.port_data;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+	struct i915_hdcp_comp_master *comp;
+	int ret;
+
+	mutex_lock(&dev_priv->hdcp_comp_mutex);
+	comp = dev_priv->hdcp_master;
+
+	if (!comp || !comp->ops) {
+		mutex_unlock(&dev_priv->hdcp_comp_mutex);
+		return -EINVAL;
+	}
+
+	ret = comp->ops->store_pairing_info(comp->mei_dev, data, pairing_info);
+	if (ret < 0)
+		DRM_DEBUG_KMS("Store pairing info failed. %d\n", ret);
+	mutex_unlock(&dev_priv->hdcp_comp_mutex);
+
+	return ret;
+}
+
+static int
+hdcp2_prepare_lc_init(struct intel_connector *connector,
+		      struct hdcp2_lc_init *lc_init)
+{
+	struct hdcp_port_data *data = &connector->hdcp.port_data;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+	struct i915_hdcp_comp_master *comp;
+	int ret;
+
+	mutex_lock(&dev_priv->hdcp_comp_mutex);
+	comp = dev_priv->hdcp_master;
+
+	if (!comp || !comp->ops) {
+		mutex_unlock(&dev_priv->hdcp_comp_mutex);
+		return -EINVAL;
+	}
+
+	ret = comp->ops->initiate_locality_check(comp->mei_dev, data, lc_init);
+	if (ret < 0)
+		DRM_DEBUG_KMS("Prepare lc_init failed. %d\n", ret);
+	mutex_unlock(&dev_priv->hdcp_comp_mutex);
+
+	return ret;
+}
+
+static int
+hdcp2_verify_lprime(struct intel_connector *connector,
+		    struct hdcp2_lc_send_lprime *rx_lprime)
+{
+	struct hdcp_port_data *data = &connector->hdcp.port_data;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+	struct i915_hdcp_comp_master *comp;
+	int ret;
+
+	mutex_lock(&dev_priv->hdcp_comp_mutex);
+	comp = dev_priv->hdcp_master;
+
+	if (!comp || !comp->ops) {
+		mutex_unlock(&dev_priv->hdcp_comp_mutex);
+		return -EINVAL;
+	}
+
+	ret = comp->ops->verify_lprime(comp->mei_dev, data, rx_lprime);
+	if (ret < 0)
+		DRM_DEBUG_KMS("Verify L_Prime failed. %d\n", ret);
+	mutex_unlock(&dev_priv->hdcp_comp_mutex);
+
+	return ret;
+}
+
+static int hdcp2_prepare_skey(struct intel_connector *connector,
+			      struct hdcp2_ske_send_eks *ske_data)
+{
+	struct hdcp_port_data *data = &connector->hdcp.port_data;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+	struct i915_hdcp_comp_master *comp;
+	int ret;
+
+	mutex_lock(&dev_priv->hdcp_comp_mutex);
+	comp = dev_priv->hdcp_master;
+
+	if (!comp || !comp->ops) {
+		mutex_unlock(&dev_priv->hdcp_comp_mutex);
+		return -EINVAL;
+	}
+
+	ret = comp->ops->get_session_key(comp->mei_dev, data, ske_data);
+	if (ret < 0)
+		DRM_DEBUG_KMS("Get session key failed. %d\n", ret);
+	mutex_unlock(&dev_priv->hdcp_comp_mutex);
+
+	return ret;
+}
+
+static int
+hdcp2_verify_rep_topology_prepare_ack(struct intel_connector *connector,
+				      struct hdcp2_rep_send_receiverid_list
+								*rep_topology,
+				      struct hdcp2_rep_send_ack *rep_send_ack)
+{
+	struct hdcp_port_data *data = &connector->hdcp.port_data;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+	struct i915_hdcp_comp_master *comp;
+	int ret;
+
+	mutex_lock(&dev_priv->hdcp_comp_mutex);
+	comp = dev_priv->hdcp_master;
+
+	if (!comp || !comp->ops) {
+		mutex_unlock(&dev_priv->hdcp_comp_mutex);
+		return -EINVAL;
+	}
+
+	ret = comp->ops->repeater_check_flow_prepare_ack(comp->mei_dev, data,
+							 rep_topology,
+							 rep_send_ack);
+	if (ret < 0)
+		DRM_DEBUG_KMS("Verify rep topology failed. %d\n", ret);
+	mutex_unlock(&dev_priv->hdcp_comp_mutex);
+
+	return ret;
+}
+
+static int
+hdcp2_verify_mprime(struct intel_connector *connector,
+		    struct hdcp2_rep_stream_ready *stream_ready)
+{
+	struct hdcp_port_data *data = &connector->hdcp.port_data;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+	struct i915_hdcp_comp_master *comp;
+	int ret;
+
+	mutex_lock(&dev_priv->hdcp_comp_mutex);
+	comp = dev_priv->hdcp_master;
+
+	if (!comp || !comp->ops) {
+		mutex_unlock(&dev_priv->hdcp_comp_mutex);
+		return -EINVAL;
+	}
+
+	ret = comp->ops->verify_mprime(comp->mei_dev, data, stream_ready);
+	if (ret < 0)
+		DRM_DEBUG_KMS("Verify mprime failed. %d\n", ret);
+	mutex_unlock(&dev_priv->hdcp_comp_mutex);
+
+	return ret;
+}
+
+static int hdcp2_authenticate_port(struct intel_connector *connector)
+{
+	struct hdcp_port_data *data = &connector->hdcp.port_data;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+	struct i915_hdcp_comp_master *comp;
+	int ret;
+
+	mutex_lock(&dev_priv->hdcp_comp_mutex);
+	comp = dev_priv->hdcp_master;
+
+	if (!comp || !comp->ops) {
+		mutex_unlock(&dev_priv->hdcp_comp_mutex);
+		return -EINVAL;
+	}
+
+	ret = comp->ops->enable_hdcp_authentication(comp->mei_dev, data);
+	if (ret < 0)
+		DRM_DEBUG_KMS("Enable hdcp auth failed. %d\n", ret);
+	mutex_unlock(&dev_priv->hdcp_comp_mutex);
+
+	return ret;
+}
+
+static int hdcp2_close_mei_session(struct intel_connector *connector)
+{
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+	struct i915_hdcp_comp_master *comp;
+	int ret;
+
+	mutex_lock(&dev_priv->hdcp_comp_mutex);
+	comp = dev_priv->hdcp_master;
+
+	if (!comp || !comp->ops) {
+		mutex_unlock(&dev_priv->hdcp_comp_mutex);
+		return -EINVAL;
+	}
+
+	ret = comp->ops->close_hdcp_session(comp->mei_dev,
+					     &connector->hdcp.port_data);
+	mutex_unlock(&dev_priv->hdcp_comp_mutex);
+
+	return ret;
+}
+
+static int hdcp2_deauthenticate_port(struct intel_connector *connector)
+{
+	return hdcp2_close_mei_session(connector);
+}
+
+/* Authentication flow starts from here */
+static int hdcp2_authentication_key_exchange(struct intel_connector *connector)
+{
+	struct intel_digital_port *intel_dig_port = conn_to_dig_port(connector);
+	struct intel_hdcp *hdcp = &connector->hdcp;
+	union {
+		struct hdcp2_ake_init ake_init;
+		struct hdcp2_ake_send_cert send_cert;
+		struct hdcp2_ake_no_stored_km no_stored_km;
+		struct hdcp2_ake_send_hprime send_hprime;
+		struct hdcp2_ake_send_pairing_info pairing_info;
+	} msgs;
+	const struct intel_hdcp_shim *shim = hdcp->shim;
+	size_t size;
+	int ret;
+
+	/* Init for seq_num */
+	hdcp->seq_num_v = 0;
+	hdcp->seq_num_m = 0;
+
+	ret = hdcp2_prepare_ake_init(connector, &msgs.ake_init);
+	if (ret < 0)
+		return ret;
+
+	ret = shim->write_2_2_msg(intel_dig_port, &msgs.ake_init,
+				  sizeof(msgs.ake_init));
+	if (ret < 0)
+		return ret;
+
+	ret = shim->read_2_2_msg(intel_dig_port, HDCP_2_2_AKE_SEND_CERT,
+				 &msgs.send_cert, sizeof(msgs.send_cert));
+	if (ret < 0)
+		return ret;
+
+	if (msgs.send_cert.rx_caps[0] != HDCP_2_2_RX_CAPS_VERSION_VAL)
+		return -EINVAL;
+
+	hdcp->is_repeater = HDCP_2_2_RX_REPEATER(msgs.send_cert.rx_caps[2]);
+
+	/*
+	 * Here msgs.no_stored_km will hold msgs corresponding to the km
+	 * stored also.
+	 */
+	ret = hdcp2_verify_rx_cert_prepare_km(connector, &msgs.send_cert,
+					      &hdcp->is_paired,
+					      &msgs.no_stored_km, &size);
+	if (ret < 0)
+		return ret;
+
+	ret = shim->write_2_2_msg(intel_dig_port, &msgs.no_stored_km, size);
+	if (ret < 0)
+		return ret;
+
+	ret = shim->read_2_2_msg(intel_dig_port, HDCP_2_2_AKE_SEND_HPRIME,
+				 &msgs.send_hprime, sizeof(msgs.send_hprime));
+	if (ret < 0)
+		return ret;
+
+	ret = hdcp2_verify_hprime(connector, &msgs.send_hprime);
+	if (ret < 0)
+		return ret;
+
+	if (!hdcp->is_paired) {
+		/* Pairing is required */
+		ret = shim->read_2_2_msg(intel_dig_port,
+					 HDCP_2_2_AKE_SEND_PAIRING_INFO,
+					 &msgs.pairing_info,
+					 sizeof(msgs.pairing_info));
+		if (ret < 0)
+			return ret;
+
+		ret = hdcp2_store_pairing_info(connector, &msgs.pairing_info);
+		if (ret < 0)
+			return ret;
+		hdcp->is_paired = true;
+	}
+
+	return 0;
+}
+
+static int hdcp2_locality_check(struct intel_connector *connector)
+{
+	struct intel_digital_port *intel_dig_port = conn_to_dig_port(connector);
+	struct intel_hdcp *hdcp = &connector->hdcp;
+	union {
+		struct hdcp2_lc_init lc_init;
+		struct hdcp2_lc_send_lprime send_lprime;
+	} msgs;
+	const struct intel_hdcp_shim *shim = hdcp->shim;
+	int tries = HDCP2_LC_RETRY_CNT, ret, i;
+
+	for (i = 0; i < tries; i++) {
+		ret = hdcp2_prepare_lc_init(connector, &msgs.lc_init);
+		if (ret < 0)
+			continue;
+
+		ret = shim->write_2_2_msg(intel_dig_port, &msgs.lc_init,
+				      sizeof(msgs.lc_init));
+		if (ret < 0)
+			continue;
+
+		ret = shim->read_2_2_msg(intel_dig_port,
+					 HDCP_2_2_LC_SEND_LPRIME,
+					 &msgs.send_lprime,
+					 sizeof(msgs.send_lprime));
+		if (ret < 0)
+			continue;
+
+		ret = hdcp2_verify_lprime(connector, &msgs.send_lprime);
+		if (!ret)
+			break;
+	}
+
+	return ret;
+}
+
+static int hdcp2_session_key_exchange(struct intel_connector *connector)
+{
+	struct intel_digital_port *intel_dig_port = conn_to_dig_port(connector);
+	struct intel_hdcp *hdcp = &connector->hdcp;
+	struct hdcp2_ske_send_eks send_eks;
+	int ret;
+
+	ret = hdcp2_prepare_skey(connector, &send_eks);
+	if (ret < 0)
+		return ret;
+
+	ret = hdcp->shim->write_2_2_msg(intel_dig_port, &send_eks,
+					sizeof(send_eks));
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static
+int hdcp2_propagate_stream_management_info(struct intel_connector *connector)
+{
+	struct intel_digital_port *intel_dig_port = conn_to_dig_port(connector);
+	struct intel_hdcp *hdcp = &connector->hdcp;
+	union {
+		struct hdcp2_rep_stream_manage stream_manage;
+		struct hdcp2_rep_stream_ready stream_ready;
+	} msgs;
+	const struct intel_hdcp_shim *shim = hdcp->shim;
+	int ret;
+
+	/* Prepare RepeaterAuth_Stream_Manage msg */
+	msgs.stream_manage.msg_id = HDCP_2_2_REP_STREAM_MANAGE;
+	drm_hdcp2_u32_to_seq_num(msgs.stream_manage.seq_num_m, hdcp->seq_num_m);
+
+	/* K no of streams is fixed as 1. Stored as big-endian. */
+	msgs.stream_manage.k = cpu_to_be16(1);
+
+	/* For HDMI this is forced to be 0x0. For DP SST also this is 0x0. */
+	msgs.stream_manage.streams[0].stream_id = 0;
+	msgs.stream_manage.streams[0].stream_type = hdcp->content_type;
+
+	/* Send it to Repeater */
+	ret = shim->write_2_2_msg(intel_dig_port, &msgs.stream_manage,
+				  sizeof(msgs.stream_manage));
+	if (ret < 0)
+		return ret;
+
+	ret = shim->read_2_2_msg(intel_dig_port, HDCP_2_2_REP_STREAM_READY,
+				 &msgs.stream_ready, sizeof(msgs.stream_ready));
+	if (ret < 0)
+		return ret;
+
+	hdcp->port_data.seq_num_m = hdcp->seq_num_m;
+	hdcp->port_data.streams[0].stream_type = hdcp->content_type;
+
+	ret = hdcp2_verify_mprime(connector, &msgs.stream_ready);
+	if (ret < 0)
+		return ret;
+
+	hdcp->seq_num_m++;
+
+	if (hdcp->seq_num_m > HDCP_2_2_SEQ_NUM_MAX) {
+		DRM_DEBUG_KMS("seq_num_m roll over.\n");
+		return -1;
+	}
+
+	return 0;
+}
+
+static
+int hdcp2_authenticate_repeater_topology(struct intel_connector *connector)
+{
+	struct intel_digital_port *intel_dig_port = conn_to_dig_port(connector);
+	struct intel_hdcp *hdcp = &connector->hdcp;
+	union {
+		struct hdcp2_rep_send_receiverid_list recvid_list;
+		struct hdcp2_rep_send_ack rep_ack;
+	} msgs;
+	const struct intel_hdcp_shim *shim = hdcp->shim;
+	u8 *rx_info;
+	u32 seq_num_v;
+	int ret;
+
+	ret = shim->read_2_2_msg(intel_dig_port, HDCP_2_2_REP_SEND_RECVID_LIST,
+				 &msgs.recvid_list, sizeof(msgs.recvid_list));
+	if (ret < 0)
+		return ret;
+
+	rx_info = msgs.recvid_list.rx_info;
+
+	if (HDCP_2_2_MAX_CASCADE_EXCEEDED(rx_info[1]) ||
+	    HDCP_2_2_MAX_DEVS_EXCEEDED(rx_info[1])) {
+		DRM_DEBUG_KMS("Topology Max Size Exceeded\n");
+		return -EINVAL;
+	}
+
+	/* Converting and Storing the seq_num_v to local variable as DWORD */
+	seq_num_v = drm_hdcp2_seq_num_to_u32(msgs.recvid_list.seq_num_v);
+
+	if (seq_num_v < hdcp->seq_num_v) {
+		/* Roll over of the seq_num_v from repeater. Reauthenticate. */
+		DRM_DEBUG_KMS("Seq_num_v roll over.\n");
+		return -EINVAL;
+	}
+
+	ret = hdcp2_verify_rep_topology_prepare_ack(connector,
+						    &msgs.recvid_list,
+						    &msgs.rep_ack);
+	if (ret < 0)
+		return ret;
+
+	hdcp->seq_num_v = seq_num_v;
+	ret = shim->write_2_2_msg(intel_dig_port, &msgs.rep_ack,
+				  sizeof(msgs.rep_ack));
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int hdcp2_authenticate_repeater(struct intel_connector *connector)
+{
+	int ret;
+
+	ret = hdcp2_authenticate_repeater_topology(connector);
+	if (ret < 0)
+		return ret;
+
+	return hdcp2_propagate_stream_management_info(connector);
+}
+
+static int hdcp2_authenticate_sink(struct intel_connector *connector)
+{
+	struct intel_digital_port *intel_dig_port = conn_to_dig_port(connector);
+	struct intel_hdcp *hdcp = &connector->hdcp;
+	const struct intel_hdcp_shim *shim = hdcp->shim;
+	int ret;
+
+	ret = hdcp2_authentication_key_exchange(connector);
+	if (ret < 0) {
+		DRM_DEBUG_KMS("AKE Failed. Err : %d\n", ret);
+		return ret;
+	}
+
+	ret = hdcp2_locality_check(connector);
+	if (ret < 0) {
+		DRM_DEBUG_KMS("Locality Check failed. Err : %d\n", ret);
+		return ret;
+	}
+
+	ret = hdcp2_session_key_exchange(connector);
+	if (ret < 0) {
+		DRM_DEBUG_KMS("SKE Failed. Err : %d\n", ret);
+		return ret;
+	}
+
+	if (shim->config_stream_type) {
+		ret = shim->config_stream_type(intel_dig_port,
+					       hdcp->is_repeater,
+					       hdcp->content_type);
+		if (ret < 0)
+			return ret;
+	}
+
+	if (hdcp->is_repeater) {
+		ret = hdcp2_authenticate_repeater(connector);
+		if (ret < 0) {
+			DRM_DEBUG_KMS("Repeater Auth Failed. Err: %d\n", ret);
+			return ret;
+		}
+	}
+
+	hdcp->port_data.streams[0].stream_type = hdcp->content_type;
+	ret = hdcp2_authenticate_port(connector);
+	if (ret < 0)
+		return ret;
+
+	return ret;
+}
+
+static int hdcp2_enable_encryption(struct intel_connector *connector)
+{
+	struct intel_digital_port *intel_dig_port = conn_to_dig_port(connector);
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+	struct intel_hdcp *hdcp = &connector->hdcp;
+	enum port port = connector->encoder->port;
+	int ret;
+
+	WARN_ON(I915_READ(HDCP2_STATUS_DDI(port)) & LINK_ENCRYPTION_STATUS);
+
+	if (hdcp->shim->toggle_signalling) {
+		ret = hdcp->shim->toggle_signalling(intel_dig_port, true);
+		if (ret) {
+			DRM_ERROR("Failed to enable HDCP signalling. %d\n",
+				  ret);
+			return ret;
+		}
+	}
+
+	if (I915_READ(HDCP2_STATUS_DDI(port)) & LINK_AUTH_STATUS) {
+		/* Link is Authenticated. Now set for Encryption */
+		I915_WRITE(HDCP2_CTL_DDI(port),
+			   I915_READ(HDCP2_CTL_DDI(port)) |
+			   CTL_LINK_ENCRYPTION_REQ);
+	}
+
+	ret = intel_wait_for_register(&dev_priv->uncore, HDCP2_STATUS_DDI(port),
+				      LINK_ENCRYPTION_STATUS,
+				      LINK_ENCRYPTION_STATUS,
+				      ENCRYPT_STATUS_CHANGE_TIMEOUT_MS);
+
+	return ret;
+}
+
+static int hdcp2_disable_encryption(struct intel_connector *connector)
+{
+	struct intel_digital_port *intel_dig_port = conn_to_dig_port(connector);
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+	struct intel_hdcp *hdcp = &connector->hdcp;
+	enum port port = connector->encoder->port;
+	int ret;
+
+	WARN_ON(!(I915_READ(HDCP2_STATUS_DDI(port)) & LINK_ENCRYPTION_STATUS));
+
+	I915_WRITE(HDCP2_CTL_DDI(port),
+		   I915_READ(HDCP2_CTL_DDI(port)) & ~CTL_LINK_ENCRYPTION_REQ);
+
+	ret = intel_wait_for_register(&dev_priv->uncore, HDCP2_STATUS_DDI(port),
+				      LINK_ENCRYPTION_STATUS, 0x0,
+				      ENCRYPT_STATUS_CHANGE_TIMEOUT_MS);
+	if (ret == -ETIMEDOUT)
+		DRM_DEBUG_KMS("Disable Encryption Timedout");
+
+	if (hdcp->shim->toggle_signalling) {
+		ret = hdcp->shim->toggle_signalling(intel_dig_port, false);
+		if (ret) {
+			DRM_ERROR("Failed to disable HDCP signalling. %d\n",
+				  ret);
+			return ret;
+		}
+	}
+
+	return ret;
+}
+
+static int hdcp2_authenticate_and_encrypt(struct intel_connector *connector)
+{
+	int ret, i, tries = 3;
+
+	for (i = 0; i < tries; i++) {
+		ret = hdcp2_authenticate_sink(connector);
+		if (!ret)
+			break;
+
+		/* Clearing the mei hdcp session */
+		DRM_DEBUG_KMS("HDCP2.2 Auth %d of %d Failed.(%d)\n",
+			      i + 1, tries, ret);
+		if (hdcp2_deauthenticate_port(connector) < 0)
+			DRM_DEBUG_KMS("Port deauth failed.\n");
+	}
+
+	if (i != tries) {
+		/*
+		 * Ensuring the required 200mSec min time interval between
+		 * Session Key Exchange and encryption.
+		 */
+		msleep(HDCP_2_2_DELAY_BEFORE_ENCRYPTION_EN);
+		ret = hdcp2_enable_encryption(connector);
+		if (ret < 0) {
+			DRM_DEBUG_KMS("Encryption Enable Failed.(%d)\n", ret);
+			if (hdcp2_deauthenticate_port(connector) < 0)
+				DRM_DEBUG_KMS("Port deauth failed.\n");
+		}
+	}
+
+	return ret;
+}
+
+static int _intel_hdcp2_enable(struct intel_connector *connector)
+{
+	struct intel_hdcp *hdcp = &connector->hdcp;
+	int ret;
+
+	DRM_DEBUG_KMS("[%s:%d] HDCP2.2 is being enabled. Type: %d\n",
+		      connector->base.name, connector->base.base.id,
+		      hdcp->content_type);
+
+	ret = hdcp2_authenticate_and_encrypt(connector);
+	if (ret) {
+		DRM_DEBUG_KMS("HDCP2 Type%d  Enabling Failed. (%d)\n",
+			      hdcp->content_type, ret);
+		return ret;
+	}
+
+	DRM_DEBUG_KMS("[%s:%d] HDCP2.2 is enabled. Type %d\n",
+		      connector->base.name, connector->base.base.id,
+		      hdcp->content_type);
+
+	hdcp->hdcp2_encrypted = true;
+	return 0;
+}
+
+static int _intel_hdcp2_disable(struct intel_connector *connector)
+{
+	int ret;
+
+	DRM_DEBUG_KMS("[%s:%d] HDCP2.2 is being Disabled\n",
+		      connector->base.name, connector->base.base.id);
+
+	ret = hdcp2_disable_encryption(connector);
+
+	if (hdcp2_deauthenticate_port(connector) < 0)
+		DRM_DEBUG_KMS("Port deauth failed.\n");
+
+	connector->hdcp.hdcp2_encrypted = false;
+
+	return ret;
+}
+
+/* Implements the Link Integrity Check for HDCP2.2 */
+static int intel_hdcp2_check_link(struct intel_connector *connector)
+{
+	struct intel_digital_port *intel_dig_port = conn_to_dig_port(connector);
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+	struct intel_hdcp *hdcp = &connector->hdcp;
+	enum port port = connector->encoder->port;
+	int ret = 0;
+
+	mutex_lock(&hdcp->mutex);
+
+	/* hdcp2_check_link is expected only when HDCP2.2 is Enabled */
+	if (hdcp->value != DRM_MODE_CONTENT_PROTECTION_ENABLED ||
+	    !hdcp->hdcp2_encrypted) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	if (WARN_ON(!intel_hdcp2_in_use(connector))) {
+		DRM_ERROR("HDCP2.2 link stopped the encryption, %x\n",
+			  I915_READ(HDCP2_STATUS_DDI(port)));
+		ret = -ENXIO;
+		hdcp->value = DRM_MODE_CONTENT_PROTECTION_DESIRED;
+		schedule_work(&hdcp->prop_work);
+		goto out;
+	}
+
+	ret = hdcp->shim->check_2_2_link(intel_dig_port);
+	if (ret == HDCP_LINK_PROTECTED) {
+		if (hdcp->value != DRM_MODE_CONTENT_PROTECTION_UNDESIRED) {
+			hdcp->value = DRM_MODE_CONTENT_PROTECTION_ENABLED;
+			schedule_work(&hdcp->prop_work);
+		}
+		goto out;
+	}
+
+	if (ret == HDCP_TOPOLOGY_CHANGE) {
+		if (hdcp->value == DRM_MODE_CONTENT_PROTECTION_UNDESIRED)
+			goto out;
+
+		DRM_DEBUG_KMS("HDCP2.2 Downstream topology change\n");
+		ret = hdcp2_authenticate_repeater_topology(connector);
+		if (!ret) {
+			hdcp->value = DRM_MODE_CONTENT_PROTECTION_ENABLED;
+			schedule_work(&hdcp->prop_work);
+			goto out;
+		}
+		DRM_DEBUG_KMS("[%s:%d] Repeater topology auth failed.(%d)\n",
+			      connector->base.name, connector->base.base.id,
+			      ret);
+	} else {
+		DRM_DEBUG_KMS("[%s:%d] HDCP2.2 link failed, retrying auth\n",
+			      connector->base.name, connector->base.base.id);
+	}
+
+	ret = _intel_hdcp2_disable(connector);
+	if (ret) {
+		DRM_ERROR("[%s:%d] Failed to disable hdcp2.2 (%d)\n",
+			  connector->base.name, connector->base.base.id, ret);
+		hdcp->value = DRM_MODE_CONTENT_PROTECTION_DESIRED;
+		schedule_work(&hdcp->prop_work);
+		goto out;
+	}
+
+	ret = _intel_hdcp2_enable(connector);
+	if (ret) {
+		DRM_DEBUG_KMS("[%s:%d] Failed to enable hdcp2.2 (%d)\n",
+			      connector->base.name, connector->base.base.id,
+			      ret);
+		hdcp->value = DRM_MODE_CONTENT_PROTECTION_DESIRED;
+		schedule_work(&hdcp->prop_work);
+		goto out;
+	}
+
+out:
+	mutex_unlock(&hdcp->mutex);
+	return ret;
+}
+
+static void intel_hdcp_check_work(struct work_struct *work)
+{
+	struct intel_hdcp *hdcp = container_of(to_delayed_work(work),
+					       struct intel_hdcp,
+					       check_work);
+	struct intel_connector *connector = intel_hdcp_to_connector(hdcp);
+
+	if (!intel_hdcp2_check_link(connector))
+		schedule_delayed_work(&hdcp->check_work,
+				      DRM_HDCP2_CHECK_PERIOD_MS);
+	else if (!intel_hdcp_check_link(connector))
+		schedule_delayed_work(&hdcp->check_work,
+				      DRM_HDCP_CHECK_PERIOD_MS);
+}
+
+static int i915_hdcp_component_bind(struct device *i915_kdev,
+				    struct device *mei_kdev, void *data)
+{
+	struct drm_i915_private *dev_priv = kdev_to_i915(i915_kdev);
+
+	DRM_DEBUG("I915 HDCP comp bind\n");
+	mutex_lock(&dev_priv->hdcp_comp_mutex);
+	dev_priv->hdcp_master = (struct i915_hdcp_comp_master *)data;
+	dev_priv->hdcp_master->mei_dev = mei_kdev;
+	mutex_unlock(&dev_priv->hdcp_comp_mutex);
+
+	return 0;
+}
+
+static void i915_hdcp_component_unbind(struct device *i915_kdev,
+				       struct device *mei_kdev, void *data)
+{
+	struct drm_i915_private *dev_priv = kdev_to_i915(i915_kdev);
+
+	DRM_DEBUG("I915 HDCP comp unbind\n");
+	mutex_lock(&dev_priv->hdcp_comp_mutex);
+	dev_priv->hdcp_master = NULL;
+	mutex_unlock(&dev_priv->hdcp_comp_mutex);
+}
+
+static const struct component_ops i915_hdcp_component_ops = {
+	.bind   = i915_hdcp_component_bind,
+	.unbind = i915_hdcp_component_unbind,
+};
+
+static inline int initialize_hdcp_port_data(struct intel_connector *connector)
+{
+	struct intel_hdcp *hdcp = &connector->hdcp;
+	struct hdcp_port_data *data = &hdcp->port_data;
+
+	data->port = connector->encoder->port;
+	data->port_type = (u8)HDCP_PORT_TYPE_INTEGRATED;
+	data->protocol = (u8)hdcp->shim->protocol;
+
+	data->k = 1;
+	if (!data->streams)
+		data->streams = kcalloc(data->k,
+					sizeof(struct hdcp2_streamid_type),
+					GFP_KERNEL);
+	if (!data->streams) {
+		DRM_ERROR("Out of Memory\n");
+		return -ENOMEM;
+	}
+
+	data->streams[0].stream_id = 0;
+	data->streams[0].stream_type = hdcp->content_type;
+
+	return 0;
+}
+
+static bool is_hdcp2_supported(struct drm_i915_private *dev_priv)
+{
+	if (!IS_ENABLED(CONFIG_INTEL_MEI_HDCP))
+		return false;
+
+	return (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv) ||
+		IS_KABYLAKE(dev_priv));
+}
+
+void intel_hdcp_component_init(struct drm_i915_private *dev_priv)
+{
+	int ret;
+
+	if (!is_hdcp2_supported(dev_priv))
+		return;
+
+	mutex_lock(&dev_priv->hdcp_comp_mutex);
+	WARN_ON(dev_priv->hdcp_comp_added);
+
+	dev_priv->hdcp_comp_added = true;
+	mutex_unlock(&dev_priv->hdcp_comp_mutex);
+	ret = component_add_typed(dev_priv->drm.dev, &i915_hdcp_component_ops,
+				  I915_COMPONENT_HDCP);
+	if (ret < 0) {
+		DRM_DEBUG_KMS("Failed at component add(%d)\n", ret);
+		mutex_lock(&dev_priv->hdcp_comp_mutex);
+		dev_priv->hdcp_comp_added = false;
+		mutex_unlock(&dev_priv->hdcp_comp_mutex);
+		return;
+	}
+}
+
+static void intel_hdcp2_init(struct intel_connector *connector)
+{
+	struct intel_hdcp *hdcp = &connector->hdcp;
+	int ret;
+
+	ret = initialize_hdcp_port_data(connector);
+	if (ret) {
+		DRM_DEBUG_KMS("Mei hdcp data init failed\n");
+		return;
+	}
+
+	hdcp->hdcp2_supported = true;
+}
+
 int intel_hdcp_init(struct intel_connector *connector,
 		    const struct intel_hdcp_shim *shim)
 {
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	struct intel_hdcp *hdcp = &connector->hdcp;
 	int ret;
 
-	ret = drm_connector_attach_content_protection_property(
-			&connector->base);
+	if (!shim)
+		return -EINVAL;
+
+	ret = drm_connector_attach_content_protection_property(&connector->base);
 	if (ret)
 		return ret;
 
@@ -788,28 +1806,47 @@ int intel_hdcp_init(struct intel_connector *connector,
 	mutex_init(&hdcp->mutex);
 	INIT_DELAYED_WORK(&hdcp->check_work, intel_hdcp_check_work);
 	INIT_WORK(&hdcp->prop_work, intel_hdcp_prop_work);
+
+	if (is_hdcp2_supported(dev_priv))
+		intel_hdcp2_init(connector);
+	init_waitqueue_head(&hdcp->cp_irq_queue);
+
 	return 0;
 }
 
 int intel_hdcp_enable(struct intel_connector *connector)
 {
 	struct intel_hdcp *hdcp = &connector->hdcp;
-	int ret;
+	unsigned long check_link_interval = DRM_HDCP_CHECK_PERIOD_MS;
+	int ret = -EINVAL;
 
 	if (!hdcp->shim)
 		return -ENOENT;
 
 	mutex_lock(&hdcp->mutex);
+	WARN_ON(hdcp->value == DRM_MODE_CONTENT_PROTECTION_ENABLED);
 
-	ret = _intel_hdcp_enable(connector);
-	if (ret)
-		goto out;
+	/*
+	 * Considering that HDCP2.2 is more secure than HDCP1.4, If the setup
+	 * is capable of HDCP2.2, it is preferred to use HDCP2.2.
+	 */
+	if (intel_hdcp2_capable(connector)) {
+		ret = _intel_hdcp2_enable(connector);
+		if (!ret)
+			check_link_interval = DRM_HDCP2_CHECK_PERIOD_MS;
+	}
+
+	/* When HDCP2.2 fails, HDCP1.4 will be attempted */
+	if (ret && intel_hdcp_capable(connector)) {
+		ret = _intel_hdcp_enable(connector);
+	}
+
+	if (!ret) {
+		schedule_delayed_work(&hdcp->check_work, check_link_interval);
+		hdcp->value = DRM_MODE_CONTENT_PROTECTION_ENABLED;
+		schedule_work(&hdcp->prop_work);
+	}
 
-	hdcp->value = DRM_MODE_CONTENT_PROTECTION_ENABLED;
-	schedule_work(&hdcp->prop_work);
-	schedule_delayed_work(&hdcp->check_work,
-			      DRM_HDCP_CHECK_PERIOD_MS);
-out:
 	mutex_unlock(&hdcp->mutex);
 	return ret;
 }
@@ -826,7 +1863,10 @@ int intel_hdcp_disable(struct intel_connector *connector)
 
 	if (hdcp->value != DRM_MODE_CONTENT_PROTECTION_UNDESIRED) {
 		hdcp->value = DRM_MODE_CONTENT_PROTECTION_UNDESIRED;
-		ret = _intel_hdcp_disable(connector);
+		if (hdcp->hdcp2_encrypted)
+			ret = _intel_hdcp2_disable(connector);
+		else if (hdcp->hdcp_encrypted)
+			ret = _intel_hdcp_disable(connector);
 	}
 
 	mutex_unlock(&hdcp->mutex);
@@ -834,6 +1874,30 @@ int intel_hdcp_disable(struct intel_connector *connector)
 	return ret;
 }
 
+void intel_hdcp_component_fini(struct drm_i915_private *dev_priv)
+{
+	mutex_lock(&dev_priv->hdcp_comp_mutex);
+	if (!dev_priv->hdcp_comp_added) {
+		mutex_unlock(&dev_priv->hdcp_comp_mutex);
+		return;
+	}
+
+	dev_priv->hdcp_comp_added = false;
+	mutex_unlock(&dev_priv->hdcp_comp_mutex);
+
+	component_del(dev_priv->drm.dev, &i915_hdcp_component_ops);
+}
+
+void intel_hdcp_cleanup(struct intel_connector *connector)
+{
+	if (!connector->hdcp.shim)
+		return;
+
+	mutex_lock(&connector->hdcp.mutex);
+	kfree(connector->hdcp.port_data.streams);
+	mutex_unlock(&connector->hdcp.mutex);
+}
+
 void intel_hdcp_atomic_check(struct drm_connector *connector,
 			     struct drm_connector_state *old_state,
 			     struct drm_connector_state *new_state)
@@ -867,61 +1931,16 @@ void intel_hdcp_atomic_check(struct drm_connector *connector,
 	crtc_state->mode_changed = true;
 }
 
-/* Implements Part 3 of the HDCP authorization procedure */
-int intel_hdcp_check_link(struct intel_connector *connector)
+/* Handles the CP_IRQ raised from the DP HDCP sink */
+void intel_hdcp_handle_cp_irq(struct intel_connector *connector)
 {
 	struct intel_hdcp *hdcp = &connector->hdcp;
-	struct drm_i915_private *dev_priv = connector->base.dev->dev_private;
-	struct intel_digital_port *intel_dig_port = conn_to_dig_port(connector);
-	enum port port = intel_dig_port->base.port;
-	int ret = 0;
 
 	if (!hdcp->shim)
-		return -ENOENT;
-
-	mutex_lock(&hdcp->mutex);
-
-	if (hdcp->value == DRM_MODE_CONTENT_PROTECTION_UNDESIRED)
-		goto out;
-
-	if (!(I915_READ(PORT_HDCP_STATUS(port)) & HDCP_STATUS_ENC)) {
-		DRM_ERROR("%s:%d HDCP check failed: link is not encrypted,%x\n",
-			  connector->base.name, connector->base.base.id,
-			  I915_READ(PORT_HDCP_STATUS(port)));
-		ret = -ENXIO;
-		hdcp->value = DRM_MODE_CONTENT_PROTECTION_DESIRED;
-		schedule_work(&hdcp->prop_work);
-		goto out;
-	}
-
-	if (hdcp->shim->check_link(intel_dig_port)) {
-		if (hdcp->value != DRM_MODE_CONTENT_PROTECTION_UNDESIRED) {
-			hdcp->value = DRM_MODE_CONTENT_PROTECTION_ENABLED;
-			schedule_work(&hdcp->prop_work);
-		}
-		goto out;
-	}
-
-	DRM_DEBUG_KMS("[%s:%d] HDCP link failed, retrying authentication\n",
-		      connector->base.name, connector->base.base.id);
-
-	ret = _intel_hdcp_disable(connector);
-	if (ret) {
-		DRM_ERROR("Failed to disable hdcp (%d)\n", ret);
-		hdcp->value = DRM_MODE_CONTENT_PROTECTION_DESIRED;
-		schedule_work(&hdcp->prop_work);
-		goto out;
-	}
+		return;
 
-	ret = _intel_hdcp_enable(connector);
-	if (ret) {
-		DRM_DEBUG_KMS("Failed to enable hdcp (%d)\n", ret);
-		hdcp->value = DRM_MODE_CONTENT_PROTECTION_DESIRED;
-		schedule_work(&hdcp->prop_work);
-		goto out;
-	}
+	atomic_inc(&connector->hdcp.cp_irq_count);
+	wake_up_all(&connector->hdcp.cp_irq_queue);
 
-out:
-	mutex_unlock(&hdcp->mutex);
-	return ret;
+	schedule_delayed_work(&hdcp->check_work, 0);
 }
diff --git a/drivers/gpu/drm/i915/intel_hdcp.h b/drivers/gpu/drm/i915/intel_hdcp.h
new file mode 100644
index 000000000000..a75f25f09d39
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_hdcp.h
@@ -0,0 +1,33 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2019 Intel Corporation
+ */
+
+#ifndef __INTEL_HDCP_H__
+#define __INTEL_HDCP_H__
+
+#include <linux/types.h>
+
+#include <drm/i915_drm.h>
+
+struct drm_connector;
+struct drm_connector_state;
+struct drm_i915_private;
+struct intel_connector;
+struct intel_hdcp_shim;
+
+void intel_hdcp_atomic_check(struct drm_connector *connector,
+			     struct drm_connector_state *old_state,
+			     struct drm_connector_state *new_state);
+int intel_hdcp_init(struct intel_connector *connector,
+		    const struct intel_hdcp_shim *hdcp_shim);
+int intel_hdcp_enable(struct intel_connector *connector);
+int intel_hdcp_disable(struct intel_connector *connector);
+bool is_hdcp_supported(struct drm_i915_private *dev_priv, enum port port);
+bool intel_hdcp_capable(struct intel_connector *connector);
+void intel_hdcp_component_init(struct drm_i915_private *dev_priv);
+void intel_hdcp_component_fini(struct drm_i915_private *dev_priv);
+void intel_hdcp_cleanup(struct intel_connector *connector);
+void intel_hdcp_handle_cp_irq(struct intel_connector *connector);
+
+#endif /* __INTEL_HDCP_H__ */
diff --git a/drivers/gpu/drm/i915/intel_hdmi.c b/drivers/gpu/drm/i915/intel_hdmi.c
index a46bffe2b288..34be2cfd0ec8 100644
--- a/drivers/gpu/drm/i915/intel_hdmi.c
+++ b/drivers/gpu/drm/i915/intel_hdmi.c
@@ -26,19 +26,30 @@
  *	Jesse Barnes <jesse.barnes@intel.com>
  */
 
-#include <linux/i2c.h>
-#include <linux/slab.h>
 #include <linux/delay.h>
 #include <linux/hdmi.h>
+#include <linux/i2c.h>
+#include <linux/slab.h>
+
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_edid.h>
 #include <drm/drm_hdcp.h>
 #include <drm/drm_scdc_helper.h>
-#include "intel_drv.h"
 #include <drm/i915_drm.h>
 #include <drm/intel_lpe_audio.h>
+
 #include "i915_drv.h"
+#include "intel_audio.h"
+#include "intel_connector.h"
+#include "intel_ddi.h"
+#include "intel_dp.h"
+#include "intel_drv.h"
+#include "intel_hdcp.h"
+#include "intel_hdmi.h"
+#include "intel_lspcon.h"
+#include "intel_sdvo.h"
+#include "intel_panel.h"
 
 static struct drm_device *intel_hdmi_to_dev(struct intel_hdmi *intel_hdmi)
 {
@@ -82,6 +93,8 @@ static struct intel_hdmi *intel_attached_hdmi(struct drm_connector *connector)
 static u32 g4x_infoframe_index(unsigned int type)
 {
 	switch (type) {
+	case HDMI_PACKET_TYPE_GAMUT_METADATA:
+		return VIDEO_DIP_SELECT_GAMUT;
 	case HDMI_INFOFRAME_TYPE_AVI:
 		return VIDEO_DIP_SELECT_AVI;
 	case HDMI_INFOFRAME_TYPE_SPD:
@@ -97,6 +110,12 @@ static u32 g4x_infoframe_index(unsigned int type)
 static u32 g4x_infoframe_enable(unsigned int type)
 {
 	switch (type) {
+	case HDMI_PACKET_TYPE_GENERAL_CONTROL:
+		return VIDEO_DIP_ENABLE_GCP;
+	case HDMI_PACKET_TYPE_GAMUT_METADATA:
+		return VIDEO_DIP_ENABLE_GAMUT;
+	case DP_SDP_VSC:
+		return 0;
 	case HDMI_INFOFRAME_TYPE_AVI:
 		return VIDEO_DIP_ENABLE_AVI;
 	case HDMI_INFOFRAME_TYPE_SPD:
@@ -112,6 +131,10 @@ static u32 g4x_infoframe_enable(unsigned int type)
 static u32 hsw_infoframe_enable(unsigned int type)
 {
 	switch (type) {
+	case HDMI_PACKET_TYPE_GENERAL_CONTROL:
+		return VIDEO_DIP_ENABLE_GCP_HSW;
+	case HDMI_PACKET_TYPE_GAMUT_METADATA:
+		return VIDEO_DIP_ENABLE_GMP_HSW;
 	case DP_SDP_VSC:
 		return VIDEO_DIP_ENABLE_VSC_HSW;
 	case DP_SDP_PPS:
@@ -135,6 +158,8 @@ hsw_dip_data_reg(struct drm_i915_private *dev_priv,
 		 int i)
 {
 	switch (type) {
+	case HDMI_PACKET_TYPE_GAMUT_METADATA:
+		return HSW_TVIDEO_DIP_GMP_DATA(cpu_transcoder, i);
 	case DP_SDP_VSC:
 		return HSW_TVIDEO_DIP_VSC_DATA(cpu_transcoder, i);
 	case DP_SDP_PPS:
@@ -198,17 +223,37 @@ static void g4x_write_infoframe(struct intel_encoder *encoder,
 	POSTING_READ(VIDEO_DIP_CTL);
 }
 
-static bool g4x_infoframe_enabled(struct intel_encoder *encoder,
+static void g4x_read_infoframe(struct intel_encoder *encoder,
+			       const struct intel_crtc_state *crtc_state,
+			       unsigned int type,
+			       void *frame, ssize_t len)
+{
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	u32 val, *data = frame;
+	int i;
+
+	val = I915_READ(VIDEO_DIP_CTL);
+
+	val &= ~(VIDEO_DIP_SELECT_MASK | 0xf); /* clear DIP data offset */
+	val |= g4x_infoframe_index(type);
+
+	I915_WRITE(VIDEO_DIP_CTL, val);
+
+	for (i = 0; i < len; i += 4)
+		*data++ = I915_READ(VIDEO_DIP_DATA);
+}
+
+static u32 g4x_infoframes_enabled(struct intel_encoder *encoder,
 				  const struct intel_crtc_state *pipe_config)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	u32 val = I915_READ(VIDEO_DIP_CTL);
 
 	if ((val & VIDEO_DIP_ENABLE) == 0)
-		return false;
+		return 0;
 
 	if ((val & VIDEO_DIP_PORT_MASK) != VIDEO_DIP_PORT(encoder->port))
-		return false;
+		return 0;
 
 	return val & (VIDEO_DIP_ENABLE_AVI |
 		      VIDEO_DIP_ENABLE_VENDOR | VIDEO_DIP_ENABLE_SPD);
@@ -251,7 +296,28 @@ static void ibx_write_infoframe(struct intel_encoder *encoder,
 	POSTING_READ(reg);
 }
 
-static bool ibx_infoframe_enabled(struct intel_encoder *encoder,
+static void ibx_read_infoframe(struct intel_encoder *encoder,
+			       const struct intel_crtc_state *crtc_state,
+			       unsigned int type,
+			       void *frame, ssize_t len)
+{
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+	u32 val, *data = frame;
+	int i;
+
+	val = I915_READ(TVIDEO_DIP_CTL(crtc->pipe));
+
+	val &= ~(VIDEO_DIP_SELECT_MASK | 0xf); /* clear DIP data offset */
+	val |= g4x_infoframe_index(type);
+
+	I915_WRITE(TVIDEO_DIP_CTL(crtc->pipe), val);
+
+	for (i = 0; i < len; i += 4)
+		*data++ = I915_READ(TVIDEO_DIP_DATA(crtc->pipe));
+}
+
+static u32 ibx_infoframes_enabled(struct intel_encoder *encoder,
 				  const struct intel_crtc_state *pipe_config)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
@@ -260,10 +326,10 @@ static bool ibx_infoframe_enabled(struct intel_encoder *encoder,
 	u32 val = I915_READ(reg);
 
 	if ((val & VIDEO_DIP_ENABLE) == 0)
-		return false;
+		return 0;
 
 	if ((val & VIDEO_DIP_PORT_MASK) != VIDEO_DIP_PORT(encoder->port))
-		return false;
+		return 0;
 
 	return val & (VIDEO_DIP_ENABLE_AVI |
 		      VIDEO_DIP_ENABLE_VENDOR | VIDEO_DIP_ENABLE_GAMUT |
@@ -310,7 +376,28 @@ static void cpt_write_infoframe(struct intel_encoder *encoder,
 	POSTING_READ(reg);
 }
 
-static bool cpt_infoframe_enabled(struct intel_encoder *encoder,
+static void cpt_read_infoframe(struct intel_encoder *encoder,
+			       const struct intel_crtc_state *crtc_state,
+			       unsigned int type,
+			       void *frame, ssize_t len)
+{
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+	u32 val, *data = frame;
+	int i;
+
+	val = I915_READ(TVIDEO_DIP_CTL(crtc->pipe));
+
+	val &= ~(VIDEO_DIP_SELECT_MASK | 0xf); /* clear DIP data offset */
+	val |= g4x_infoframe_index(type);
+
+	I915_WRITE(TVIDEO_DIP_CTL(crtc->pipe), val);
+
+	for (i = 0; i < len; i += 4)
+		*data++ = I915_READ(TVIDEO_DIP_DATA(crtc->pipe));
+}
+
+static u32 cpt_infoframes_enabled(struct intel_encoder *encoder,
 				  const struct intel_crtc_state *pipe_config)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
@@ -318,7 +405,7 @@ static bool cpt_infoframe_enabled(struct intel_encoder *encoder,
 	u32 val = I915_READ(TVIDEO_DIP_CTL(pipe));
 
 	if ((val & VIDEO_DIP_ENABLE) == 0)
-		return false;
+		return 0;
 
 	return val & (VIDEO_DIP_ENABLE_AVI |
 		      VIDEO_DIP_ENABLE_VENDOR | VIDEO_DIP_ENABLE_GAMUT |
@@ -362,7 +449,28 @@ static void vlv_write_infoframe(struct intel_encoder *encoder,
 	POSTING_READ(reg);
 }
 
-static bool vlv_infoframe_enabled(struct intel_encoder *encoder,
+static void vlv_read_infoframe(struct intel_encoder *encoder,
+			       const struct intel_crtc_state *crtc_state,
+			       unsigned int type,
+			       void *frame, ssize_t len)
+{
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+	u32 val, *data = frame;
+	int i;
+
+	val = I915_READ(VLV_TVIDEO_DIP_CTL(crtc->pipe));
+
+	val &= ~(VIDEO_DIP_SELECT_MASK | 0xf); /* clear DIP data offset */
+	val |= g4x_infoframe_index(type);
+
+	I915_WRITE(VLV_TVIDEO_DIP_CTL(crtc->pipe), val);
+
+	for (i = 0; i < len; i += 4)
+		*data++ = I915_READ(VLV_TVIDEO_DIP_DATA(crtc->pipe));
+}
+
+static u32 vlv_infoframes_enabled(struct intel_encoder *encoder,
 				  const struct intel_crtc_state *pipe_config)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
@@ -370,10 +478,10 @@ static bool vlv_infoframe_enabled(struct intel_encoder *encoder,
 	u32 val = I915_READ(VLV_TVIDEO_DIP_CTL(pipe));
 
 	if ((val & VIDEO_DIP_ENABLE) == 0)
-		return false;
+		return 0;
 
 	if ((val & VIDEO_DIP_PORT_MASK) != VIDEO_DIP_PORT(encoder->port))
-		return false;
+		return 0;
 
 	return val & (VIDEO_DIP_ENABLE_AVI |
 		      VIDEO_DIP_ENABLE_VENDOR | VIDEO_DIP_ENABLE_GAMUT |
@@ -413,7 +521,24 @@ static void hsw_write_infoframe(struct intel_encoder *encoder,
 	POSTING_READ(ctl_reg);
 }
 
-static bool hsw_infoframe_enabled(struct intel_encoder *encoder,
+static void hsw_read_infoframe(struct intel_encoder *encoder,
+			       const struct intel_crtc_state *crtc_state,
+			       unsigned int type,
+			       void *frame, ssize_t len)
+{
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
+	u32 val, *data = frame;
+	int i;
+
+	val = I915_READ(HSW_TVIDEO_DIP_CTL(cpu_transcoder));
+
+	for (i = 0; i < len; i += 4)
+		*data++ = I915_READ(hsw_dip_data_reg(dev_priv, cpu_transcoder,
+						     type, i >> 2));
+}
+
+static u32 hsw_infoframes_enabled(struct intel_encoder *encoder,
 				  const struct intel_crtc_state *pipe_config)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
@@ -424,6 +549,53 @@ static bool hsw_infoframe_enabled(struct intel_encoder *encoder,
 		      VIDEO_DIP_ENABLE_GMP_HSW | VIDEO_DIP_ENABLE_SPD_HSW);
 }
 
+static const u8 infoframe_type_to_idx[] = {
+	HDMI_PACKET_TYPE_GENERAL_CONTROL,
+	HDMI_PACKET_TYPE_GAMUT_METADATA,
+	DP_SDP_VSC,
+	HDMI_INFOFRAME_TYPE_AVI,
+	HDMI_INFOFRAME_TYPE_SPD,
+	HDMI_INFOFRAME_TYPE_VENDOR,
+};
+
+u32 intel_hdmi_infoframe_enable(unsigned int type)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(infoframe_type_to_idx); i++) {
+		if (infoframe_type_to_idx[i] == type)
+			return BIT(i);
+	}
+
+	return 0;
+}
+
+u32 intel_hdmi_infoframes_enabled(struct intel_encoder *encoder,
+				  const struct intel_crtc_state *crtc_state)
+{
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct intel_digital_port *dig_port = enc_to_dig_port(&encoder->base);
+	u32 val, ret = 0;
+	int i;
+
+	val = dig_port->infoframes_enabled(encoder, crtc_state);
+
+	/* map from hardware bits to dip idx */
+	for (i = 0; i < ARRAY_SIZE(infoframe_type_to_idx); i++) {
+		unsigned int type = infoframe_type_to_idx[i];
+
+		if (HAS_DDI(dev_priv)) {
+			if (val & hsw_infoframe_enable(type))
+				ret |= BIT(i);
+		} else {
+			if (val & g4x_infoframe_enable(type))
+				ret |= BIT(i);
+		}
+	}
+
+	return ret;
+}
+
 /*
  * The data we write to the DIP data buffer registers is 1 byte bigger than the
  * HDMI infoframe size because of an ECC/reserved byte at position 3 (starting
@@ -443,15 +615,23 @@ static bool hsw_infoframe_enabled(struct intel_encoder *encoder,
  */
 static void intel_write_infoframe(struct intel_encoder *encoder,
 				  const struct intel_crtc_state *crtc_state,
-				  union hdmi_infoframe *frame)
+				  enum hdmi_infoframe_type type,
+				  const union hdmi_infoframe *frame)
 {
 	struct intel_digital_port *intel_dig_port = enc_to_dig_port(&encoder->base);
 	u8 buffer[VIDEO_DIP_DATA_SIZE];
 	ssize_t len;
 
+	if ((crtc_state->infoframes.enable &
+	     intel_hdmi_infoframe_enable(type)) == 0)
+		return;
+
+	if (WARN_ON(frame->any.type != type))
+		return;
+
 	/* see comment above for the reason for this offset */
-	len = hdmi_infoframe_pack(frame, buffer + 1, sizeof(buffer) - 1);
-	if (len < 0)
+	len = hdmi_infoframe_pack_only(frame, buffer + 1, sizeof(buffer) - 1);
+	if (WARN_ON(len < 0))
 		return;
 
 	/* Insert the 'hole' (see big comment above) at position 3 */
@@ -459,84 +639,143 @@ static void intel_write_infoframe(struct intel_encoder *encoder,
 	buffer[3] = 0;
 	len++;
 
-	intel_dig_port->write_infoframe(encoder,
-					crtc_state,
-					frame->any.type, buffer, len);
+	intel_dig_port->write_infoframe(encoder, crtc_state, type, buffer, len);
 }
 
-static void intel_hdmi_set_avi_infoframe(struct intel_encoder *encoder,
-					 const struct intel_crtc_state *crtc_state,
-					 const struct drm_connector_state *conn_state)
+void intel_read_infoframe(struct intel_encoder *encoder,
+			  const struct intel_crtc_state *crtc_state,
+			  enum hdmi_infoframe_type type,
+			  union hdmi_infoframe *frame)
+{
+	struct intel_digital_port *intel_dig_port = enc_to_dig_port(&encoder->base);
+	u8 buffer[VIDEO_DIP_DATA_SIZE];
+	int ret;
+
+	if ((crtc_state->infoframes.enable &
+	     intel_hdmi_infoframe_enable(type)) == 0)
+		return;
+
+	intel_dig_port->read_infoframe(encoder, crtc_state,
+				       type, buffer, sizeof(buffer));
+
+	/* Fill the 'hole' (see big comment above) at position 3 */
+	memmove(&buffer[1], &buffer[0], 3);
+
+	/* see comment above for the reason for this offset */
+	ret = hdmi_infoframe_unpack(frame, buffer + 1, sizeof(buffer) - 1);
+	if (ret) {
+		DRM_DEBUG_KMS("Failed to unpack infoframe type 0x%02x\n", type);
+		return;
+	}
+
+	if (frame->any.type != type)
+		DRM_DEBUG_KMS("Found the wrong infoframe type 0x%x (expected 0x%02x)\n",
+			      frame->any.type, type);
+}
+
+static bool
+intel_hdmi_compute_avi_infoframe(struct intel_encoder *encoder,
+				 struct intel_crtc_state *crtc_state,
+				 struct drm_connector_state *conn_state)
 {
+	struct hdmi_avi_infoframe *frame = &crtc_state->infoframes.avi.avi;
 	const struct drm_display_mode *adjusted_mode =
 		&crtc_state->base.adjusted_mode;
-	union hdmi_infoframe frame;
+	struct drm_connector *connector = conn_state->connector;
 	int ret;
 
-	ret = drm_hdmi_avi_infoframe_from_display_mode(&frame.avi,
-						       conn_state->connector,
+	if (!crtc_state->has_infoframe)
+		return true;
+
+	crtc_state->infoframes.enable |=
+		intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_AVI);
+
+	ret = drm_hdmi_avi_infoframe_from_display_mode(frame, connector,
 						       adjusted_mode);
-	if (ret < 0) {
-		DRM_ERROR("couldn't fill AVI infoframe\n");
-		return;
-	}
+	if (ret)
+		return false;
 
 	if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420)
-		frame.avi.colorspace = HDMI_COLORSPACE_YUV420;
+		frame->colorspace = HDMI_COLORSPACE_YUV420;
 	else if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR444)
-		frame.avi.colorspace = HDMI_COLORSPACE_YUV444;
+		frame->colorspace = HDMI_COLORSPACE_YUV444;
 	else
-		frame.avi.colorspace = HDMI_COLORSPACE_RGB;
+		frame->colorspace = HDMI_COLORSPACE_RGB;
+
+	drm_hdmi_avi_infoframe_colorspace(frame, conn_state);
 
-	drm_hdmi_avi_infoframe_quant_range(&frame.avi,
-					   conn_state->connector,
+	drm_hdmi_avi_infoframe_quant_range(frame, connector,
 					   adjusted_mode,
 					   crtc_state->limited_color_range ?
 					   HDMI_QUANTIZATION_RANGE_LIMITED :
 					   HDMI_QUANTIZATION_RANGE_FULL);
 
-	drm_hdmi_avi_infoframe_content_type(&frame.avi,
-					    conn_state);
+	drm_hdmi_avi_infoframe_content_type(frame, conn_state);
 
 	/* TODO: handle pixel repetition for YCBCR420 outputs */
-	intel_write_infoframe(encoder, crtc_state,
-			      &frame);
+
+	ret = hdmi_avi_infoframe_check(frame);
+	if (WARN_ON(ret))
+		return false;
+
+	return true;
 }
 
-static void intel_hdmi_set_spd_infoframe(struct intel_encoder *encoder,
-					 const struct intel_crtc_state *crtc_state)
+static bool
+intel_hdmi_compute_spd_infoframe(struct intel_encoder *encoder,
+				 struct intel_crtc_state *crtc_state,
+				 struct drm_connector_state *conn_state)
 {
-	union hdmi_infoframe frame;
+	struct hdmi_spd_infoframe *frame = &crtc_state->infoframes.spd.spd;
 	int ret;
 
-	ret = hdmi_spd_infoframe_init(&frame.spd, "Intel", "Integrated gfx");
-	if (ret < 0) {
-		DRM_ERROR("couldn't fill SPD infoframe\n");
-		return;
-	}
+	if (!crtc_state->has_infoframe)
+		return true;
 
-	frame.spd.sdi = HDMI_SPD_SDI_PC;
+	crtc_state->infoframes.enable |=
+		intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_SPD);
 
-	intel_write_infoframe(encoder, crtc_state,
-			      &frame);
+	ret = hdmi_spd_infoframe_init(frame, "Intel", "Integrated gfx");
+	if (WARN_ON(ret))
+		return false;
+
+	frame->sdi = HDMI_SPD_SDI_PC;
+
+	ret = hdmi_spd_infoframe_check(frame);
+	if (WARN_ON(ret))
+		return false;
+
+	return true;
 }
 
-static void
-intel_hdmi_set_hdmi_infoframe(struct intel_encoder *encoder,
-			      const struct intel_crtc_state *crtc_state,
-			      const struct drm_connector_state *conn_state)
-{
-	union hdmi_infoframe frame;
+static bool
+intel_hdmi_compute_hdmi_infoframe(struct intel_encoder *encoder,
+				  struct intel_crtc_state *crtc_state,
+				  struct drm_connector_state *conn_state)
+{
+	struct hdmi_vendor_infoframe *frame =
+		&crtc_state->infoframes.hdmi.vendor.hdmi;
+	const struct drm_display_info *info =
+		&conn_state->connector->display_info;
 	int ret;
 
-	ret = drm_hdmi_vendor_infoframe_from_display_mode(&frame.vendor.hdmi,
+	if (!crtc_state->has_infoframe || !info->has_hdmi_infoframe)
+		return true;
+
+	crtc_state->infoframes.enable |=
+		intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_VENDOR);
+
+	ret = drm_hdmi_vendor_infoframe_from_display_mode(frame,
 							  conn_state->connector,
 							  &crtc_state->base.adjusted_mode);
-	if (ret < 0)
-		return;
+	if (WARN_ON(ret))
+		return false;
 
-	intel_write_infoframe(encoder, crtc_state,
-			      &frame);
+	ret = hdmi_vendor_infoframe_check(frame);
+	if (WARN_ON(ret))
+		return false;
+
+	return true;
 }
 
 static void g4x_set_infoframes(struct intel_encoder *encoder,
@@ -596,9 +835,15 @@ static void g4x_set_infoframes(struct intel_encoder *encoder,
 	I915_WRITE(reg, val);
 	POSTING_READ(reg);
 
-	intel_hdmi_set_avi_infoframe(encoder, crtc_state, conn_state);
-	intel_hdmi_set_spd_infoframe(encoder, crtc_state);
-	intel_hdmi_set_hdmi_infoframe(encoder, crtc_state, conn_state);
+	intel_write_infoframe(encoder, crtc_state,
+			      HDMI_INFOFRAME_TYPE_AVI,
+			      &crtc_state->infoframes.avi);
+	intel_write_infoframe(encoder, crtc_state,
+			      HDMI_INFOFRAME_TYPE_SPD,
+			      &crtc_state->infoframes.spd);
+	intel_write_infoframe(encoder, crtc_state,
+			      HDMI_INFOFRAME_TYPE_VENDOR,
+			      &crtc_state->infoframes.hdmi);
 }
 
 static bool hdmi_sink_is_deep_color(const struct drm_connector_state *conn_state)
@@ -664,7 +909,10 @@ static bool intel_hdmi_set_gcp_infoframe(struct intel_encoder *encoder,
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 	i915_reg_t reg;
-	u32 val = 0;
+
+	if ((crtc_state->infoframes.enable &
+	     intel_hdmi_infoframe_enable(HDMI_PACKET_TYPE_GENERAL_CONTROL)) == 0)
+		return false;
 
 	if (HAS_DDI(dev_priv))
 		reg = HSW_TVIDEO_DIP_GCP(crtc_state->cpu_transcoder);
@@ -675,18 +923,54 @@ static bool intel_hdmi_set_gcp_infoframe(struct intel_encoder *encoder,
 	else
 		return false;
 
+	I915_WRITE(reg, crtc_state->infoframes.gcp);
+
+	return true;
+}
+
+void intel_hdmi_read_gcp_infoframe(struct intel_encoder *encoder,
+				   struct intel_crtc_state *crtc_state)
+{
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+	i915_reg_t reg;
+
+	if ((crtc_state->infoframes.enable &
+	     intel_hdmi_infoframe_enable(HDMI_PACKET_TYPE_GENERAL_CONTROL)) == 0)
+		return;
+
+	if (HAS_DDI(dev_priv))
+		reg = HSW_TVIDEO_DIP_GCP(crtc_state->cpu_transcoder);
+	else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
+		reg = VLV_TVIDEO_DIP_GCP(crtc->pipe);
+	else if (HAS_PCH_SPLIT(dev_priv))
+		reg = TVIDEO_DIP_GCP(crtc->pipe);
+	else
+		return;
+
+	crtc_state->infoframes.gcp = I915_READ(reg);
+}
+
+static void intel_hdmi_compute_gcp_infoframe(struct intel_encoder *encoder,
+					     struct intel_crtc_state *crtc_state,
+					     struct drm_connector_state *conn_state)
+{
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+
+	if (IS_G4X(dev_priv) || !crtc_state->has_infoframe)
+		return;
+
+	crtc_state->infoframes.enable |=
+		intel_hdmi_infoframe_enable(HDMI_PACKET_TYPE_GENERAL_CONTROL);
+
 	/* Indicate color depth whenever the sink supports deep color */
 	if (hdmi_sink_is_deep_color(conn_state))
-		val |= GCP_COLOR_INDICATION;
+		crtc_state->infoframes.gcp |= GCP_COLOR_INDICATION;
 
 	/* Enable default_phase whenever the display mode is suitably aligned */
 	if (gcp_default_phase_possible(crtc_state->pipe_bpp,
 				       &crtc_state->base.adjusted_mode))
-		val |= GCP_DEFAULT_PHASE_ENABLE;
-
-	I915_WRITE(reg, val);
-
-	return val != 0;
+		crtc_state->infoframes.gcp |= GCP_DEFAULT_PHASE_ENABLE;
 }
 
 static void ibx_set_infoframes(struct intel_encoder *encoder,
@@ -737,9 +1021,15 @@ static void ibx_set_infoframes(struct intel_encoder *encoder,
 	I915_WRITE(reg, val);
 	POSTING_READ(reg);
 
-	intel_hdmi_set_avi_infoframe(encoder, crtc_state, conn_state);
-	intel_hdmi_set_spd_infoframe(encoder, crtc_state);
-	intel_hdmi_set_hdmi_infoframe(encoder, crtc_state, conn_state);
+	intel_write_infoframe(encoder, crtc_state,
+			      HDMI_INFOFRAME_TYPE_AVI,
+			      &crtc_state->infoframes.avi);
+	intel_write_infoframe(encoder, crtc_state,
+			      HDMI_INFOFRAME_TYPE_SPD,
+			      &crtc_state->infoframes.spd);
+	intel_write_infoframe(encoder, crtc_state,
+			      HDMI_INFOFRAME_TYPE_VENDOR,
+			      &crtc_state->infoframes.hdmi);
 }
 
 static void cpt_set_infoframes(struct intel_encoder *encoder,
@@ -780,9 +1070,15 @@ static void cpt_set_infoframes(struct intel_encoder *encoder,
 	I915_WRITE(reg, val);
 	POSTING_READ(reg);
 
-	intel_hdmi_set_avi_infoframe(encoder, crtc_state, conn_state);
-	intel_hdmi_set_spd_infoframe(encoder, crtc_state);
-	intel_hdmi_set_hdmi_infoframe(encoder, crtc_state, conn_state);
+	intel_write_infoframe(encoder, crtc_state,
+			      HDMI_INFOFRAME_TYPE_AVI,
+			      &crtc_state->infoframes.avi);
+	intel_write_infoframe(encoder, crtc_state,
+			      HDMI_INFOFRAME_TYPE_SPD,
+			      &crtc_state->infoframes.spd);
+	intel_write_infoframe(encoder, crtc_state,
+			      HDMI_INFOFRAME_TYPE_VENDOR,
+			      &crtc_state->infoframes.hdmi);
 }
 
 static void vlv_set_infoframes(struct intel_encoder *encoder,
@@ -832,9 +1128,15 @@ static void vlv_set_infoframes(struct intel_encoder *encoder,
 	I915_WRITE(reg, val);
 	POSTING_READ(reg);
 
-	intel_hdmi_set_avi_infoframe(encoder, crtc_state, conn_state);
-	intel_hdmi_set_spd_infoframe(encoder, crtc_state);
-	intel_hdmi_set_hdmi_infoframe(encoder, crtc_state, conn_state);
+	intel_write_infoframe(encoder, crtc_state,
+			      HDMI_INFOFRAME_TYPE_AVI,
+			      &crtc_state->infoframes.avi);
+	intel_write_infoframe(encoder, crtc_state,
+			      HDMI_INFOFRAME_TYPE_SPD,
+			      &crtc_state->infoframes.spd);
+	intel_write_infoframe(encoder, crtc_state,
+			      HDMI_INFOFRAME_TYPE_VENDOR,
+			      &crtc_state->infoframes.hdmi);
 }
 
 static void hsw_set_infoframes(struct intel_encoder *encoder,
@@ -865,9 +1167,15 @@ static void hsw_set_infoframes(struct intel_encoder *encoder,
 	I915_WRITE(reg, val);
 	POSTING_READ(reg);
 
-	intel_hdmi_set_avi_infoframe(encoder, crtc_state, conn_state);
-	intel_hdmi_set_spd_infoframe(encoder, crtc_state);
-	intel_hdmi_set_hdmi_infoframe(encoder, crtc_state, conn_state);
+	intel_write_infoframe(encoder, crtc_state,
+			      HDMI_INFOFRAME_TYPE_AVI,
+			      &crtc_state->infoframes.avi);
+	intel_write_infoframe(encoder, crtc_state,
+			      HDMI_INFOFRAME_TYPE_SPD,
+			      &crtc_state->infoframes.spd);
+	intel_write_infoframe(encoder, crtc_state,
+			      HDMI_INFOFRAME_TYPE_VENDOR,
+			      &crtc_state->infoframes.hdmi);
 }
 
 void intel_dp_dual_mode_set_tmds_output(struct intel_hdmi *hdmi, bool enable)
@@ -1073,10 +1381,44 @@ int intel_hdmi_hdcp_read_v_prime_part(struct intel_digital_port *intel_dig_port,
 	return ret;
 }
 
+static int kbl_repositioning_enc_en_signal(struct intel_connector *connector)
+{
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+	struct intel_digital_port *intel_dig_port = conn_to_dig_port(connector);
+	struct drm_crtc *crtc = connector->base.state->crtc;
+	struct intel_crtc *intel_crtc = container_of(crtc,
+						     struct intel_crtc, base);
+	u32 scanline;
+	int ret;
+
+	for (;;) {
+		scanline = I915_READ(PIPEDSL(intel_crtc->pipe));
+		if (scanline > 100 && scanline < 200)
+			break;
+		usleep_range(25, 50);
+	}
+
+	ret = intel_ddi_toggle_hdcp_signalling(&intel_dig_port->base, false);
+	if (ret) {
+		DRM_ERROR("Disable HDCP signalling failed (%d)\n", ret);
+		return ret;
+	}
+	ret = intel_ddi_toggle_hdcp_signalling(&intel_dig_port->base, true);
+	if (ret) {
+		DRM_ERROR("Enable HDCP signalling failed (%d)\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
 static
 int intel_hdmi_hdcp_toggle_signalling(struct intel_digital_port *intel_dig_port,
 				      bool enable)
 {
+	struct intel_hdmi *hdmi = &intel_dig_port->hdmi;
+	struct intel_connector *connector = hdmi->attached_connector;
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	int ret;
 
 	if (!enable)
@@ -1088,6 +1430,14 @@ int intel_hdmi_hdcp_toggle_signalling(struct intel_digital_port *intel_dig_port,
 			  enable ? "Enable" : "Disable", ret);
 		return ret;
 	}
+
+	/*
+	 * WA: To fix incorrect positioning of the window of
+	 * opportunity and enc_en signalling in KABYLAKE.
+	 */
+	if (IS_KABYLAKE(dev_priv) && enable)
+		return kbl_repositioning_enc_en_signal(connector);
+
 	return 0;
 }
 
@@ -1119,6 +1469,190 @@ bool intel_hdmi_hdcp_check_link(struct intel_digital_port *intel_dig_port)
 	return true;
 }
 
+static struct hdcp2_hdmi_msg_data {
+	u8 msg_id;
+	u32 timeout;
+	u32 timeout2;
+	} hdcp2_msg_data[] = {
+		{HDCP_2_2_AKE_INIT, 0, 0},
+		{HDCP_2_2_AKE_SEND_CERT, HDCP_2_2_CERT_TIMEOUT_MS, 0},
+		{HDCP_2_2_AKE_NO_STORED_KM, 0, 0},
+		{HDCP_2_2_AKE_STORED_KM, 0, 0},
+		{HDCP_2_2_AKE_SEND_HPRIME, HDCP_2_2_HPRIME_PAIRED_TIMEOUT_MS,
+				HDCP_2_2_HPRIME_NO_PAIRED_TIMEOUT_MS},
+		{HDCP_2_2_AKE_SEND_PAIRING_INFO, HDCP_2_2_PAIRING_TIMEOUT_MS,
+				0},
+		{HDCP_2_2_LC_INIT, 0, 0},
+		{HDCP_2_2_LC_SEND_LPRIME, HDCP_2_2_HDMI_LPRIME_TIMEOUT_MS, 0},
+		{HDCP_2_2_SKE_SEND_EKS, 0, 0},
+		{HDCP_2_2_REP_SEND_RECVID_LIST,
+				HDCP_2_2_RECVID_LIST_TIMEOUT_MS, 0},
+		{HDCP_2_2_REP_SEND_ACK, 0, 0},
+		{HDCP_2_2_REP_STREAM_MANAGE, 0, 0},
+		{HDCP_2_2_REP_STREAM_READY, HDCP_2_2_STREAM_READY_TIMEOUT_MS,
+				0},
+	};
+
+static
+int intel_hdmi_hdcp2_read_rx_status(struct intel_digital_port *intel_dig_port,
+				    u8 *rx_status)
+{
+	return intel_hdmi_hdcp_read(intel_dig_port,
+				    HDCP_2_2_HDMI_REG_RXSTATUS_OFFSET,
+				    rx_status,
+				    HDCP_2_2_HDMI_RXSTATUS_LEN);
+}
+
+static int get_hdcp2_msg_timeout(u8 msg_id, bool is_paired)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(hdcp2_msg_data); i++)
+		if (hdcp2_msg_data[i].msg_id == msg_id &&
+		    (msg_id != HDCP_2_2_AKE_SEND_HPRIME || is_paired))
+			return hdcp2_msg_data[i].timeout;
+		else if (hdcp2_msg_data[i].msg_id == msg_id)
+			return hdcp2_msg_data[i].timeout2;
+
+	return -EINVAL;
+}
+
+static inline
+int hdcp2_detect_msg_availability(struct intel_digital_port *intel_digital_port,
+				  u8 msg_id, bool *msg_ready,
+				  ssize_t *msg_sz)
+{
+	u8 rx_status[HDCP_2_2_HDMI_RXSTATUS_LEN];
+	int ret;
+
+	ret = intel_hdmi_hdcp2_read_rx_status(intel_digital_port, rx_status);
+	if (ret < 0) {
+		DRM_DEBUG_KMS("rx_status read failed. Err %d\n", ret);
+		return ret;
+	}
+
+	*msg_sz = ((HDCP_2_2_HDMI_RXSTATUS_MSG_SZ_HI(rx_status[1]) << 8) |
+		  rx_status[0]);
+
+	if (msg_id == HDCP_2_2_REP_SEND_RECVID_LIST)
+		*msg_ready = (HDCP_2_2_HDMI_RXSTATUS_READY(rx_status[1]) &&
+			     *msg_sz);
+	else
+		*msg_ready = *msg_sz;
+
+	return 0;
+}
+
+static ssize_t
+intel_hdmi_hdcp2_wait_for_msg(struct intel_digital_port *intel_dig_port,
+			      u8 msg_id, bool paired)
+{
+	bool msg_ready = false;
+	int timeout, ret;
+	ssize_t msg_sz = 0;
+
+	timeout = get_hdcp2_msg_timeout(msg_id, paired);
+	if (timeout < 0)
+		return timeout;
+
+	ret = __wait_for(ret = hdcp2_detect_msg_availability(intel_dig_port,
+							     msg_id, &msg_ready,
+							     &msg_sz),
+			 !ret && msg_ready && msg_sz, timeout * 1000,
+			 1000, 5 * 1000);
+	if (ret)
+		DRM_DEBUG_KMS("msg_id: %d, ret: %d, timeout: %d\n",
+			      msg_id, ret, timeout);
+
+	return ret ? ret : msg_sz;
+}
+
+static
+int intel_hdmi_hdcp2_write_msg(struct intel_digital_port *intel_dig_port,
+			       void *buf, size_t size)
+{
+	unsigned int offset;
+
+	offset = HDCP_2_2_HDMI_REG_WR_MSG_OFFSET;
+	return intel_hdmi_hdcp_write(intel_dig_port, offset, buf, size);
+}
+
+static
+int intel_hdmi_hdcp2_read_msg(struct intel_digital_port *intel_dig_port,
+			      u8 msg_id, void *buf, size_t size)
+{
+	struct intel_hdmi *hdmi = &intel_dig_port->hdmi;
+	struct intel_hdcp *hdcp = &hdmi->attached_connector->hdcp;
+	unsigned int offset;
+	ssize_t ret;
+
+	ret = intel_hdmi_hdcp2_wait_for_msg(intel_dig_port, msg_id,
+					    hdcp->is_paired);
+	if (ret < 0)
+		return ret;
+
+	/*
+	 * Available msg size should be equal to or lesser than the
+	 * available buffer.
+	 */
+	if (ret > size) {
+		DRM_DEBUG_KMS("msg_sz(%zd) is more than exp size(%zu)\n",
+			      ret, size);
+		return -1;
+	}
+
+	offset = HDCP_2_2_HDMI_REG_RD_MSG_OFFSET;
+	ret = intel_hdmi_hdcp_read(intel_dig_port, offset, buf, ret);
+	if (ret)
+		DRM_DEBUG_KMS("Failed to read msg_id: %d(%zd)\n", msg_id, ret);
+
+	return ret;
+}
+
+static
+int intel_hdmi_hdcp2_check_link(struct intel_digital_port *intel_dig_port)
+{
+	u8 rx_status[HDCP_2_2_HDMI_RXSTATUS_LEN];
+	int ret;
+
+	ret = intel_hdmi_hdcp2_read_rx_status(intel_dig_port, rx_status);
+	if (ret)
+		return ret;
+
+	/*
+	 * Re-auth request and Link Integrity Failures are represented by
+	 * same bit. i.e reauth_req.
+	 */
+	if (HDCP_2_2_HDMI_RXSTATUS_REAUTH_REQ(rx_status[1]))
+		ret = HDCP_REAUTH_REQUEST;
+	else if (HDCP_2_2_HDMI_RXSTATUS_READY(rx_status[1]))
+		ret = HDCP_TOPOLOGY_CHANGE;
+
+	return ret;
+}
+
+static
+int intel_hdmi_hdcp2_capable(struct intel_digital_port *intel_dig_port,
+			     bool *capable)
+{
+	u8 hdcp2_version;
+	int ret;
+
+	*capable = false;
+	ret = intel_hdmi_hdcp_read(intel_dig_port, HDCP_2_2_HDMI_REG_VER_OFFSET,
+				   &hdcp2_version, sizeof(hdcp2_version));
+	if (!ret && hdcp2_version & HDCP_2_2_HDMI_SUPPORT_MASK)
+		*capable = true;
+
+	return ret;
+}
+
+static inline
+enum hdcp_wired_protocol intel_hdmi_hdcp2_protocol(void)
+{
+	return HDCP_PROTOCOL_HDMI;
+}
+
 static const struct intel_hdcp_shim intel_hdmi_hdcp_shim = {
 	.write_an_aksv = intel_hdmi_hdcp_write_an_aksv,
 	.read_bksv = intel_hdmi_hdcp_read_bksv,
@@ -1130,6 +1664,11 @@ static const struct intel_hdcp_shim intel_hdmi_hdcp_shim = {
 	.read_v_prime_part = intel_hdmi_hdcp_read_v_prime_part,
 	.toggle_signalling = intel_hdmi_hdcp_toggle_signalling,
 	.check_link = intel_hdmi_hdcp_check_link,
+	.write_2_2_msg = intel_hdmi_hdcp2_write_msg,
+	.read_2_2_msg = intel_hdmi_hdcp2_read_msg,
+	.check_2_2_link	= intel_hdmi_hdcp2_check_link,
+	.hdcp_2_2_capable = intel_hdmi_hdcp2_capable,
+	.protocol = HDCP_PROTOCOL_HDMI,
 };
 
 static void intel_hdmi_prepare(struct intel_encoder *encoder,
@@ -1195,7 +1734,6 @@ static void intel_hdmi_get_config(struct intel_encoder *encoder,
 				  struct intel_crtc_state *pipe_config)
 {
 	struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base);
-	struct intel_digital_port *intel_dig_port = hdmi_to_dig_port(intel_hdmi);
 	struct drm_device *dev = encoder->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	u32 tmp, flags = 0;
@@ -1218,7 +1756,10 @@ static void intel_hdmi_get_config(struct intel_encoder *encoder,
 	if (tmp & HDMI_MODE_SELECT_HDMI)
 		pipe_config->has_hdmi_sink = true;
 
-	if (intel_dig_port->infoframe_enabled(encoder, pipe_config))
+	pipe_config->infoframes.enable |=
+		intel_hdmi_infoframes_enabled(encoder, pipe_config);
+
+	if (pipe_config->infoframes.enable)
 		pipe_config->has_infoframe = true;
 
 	if (tmp & SDVO_AUDIO_ENABLE)
@@ -1241,6 +1782,18 @@ static void intel_hdmi_get_config(struct intel_encoder *encoder,
 	pipe_config->base.adjusted_mode.crtc_clock = dotclock;
 
 	pipe_config->lane_count = 4;
+
+	intel_hdmi_read_gcp_infoframe(encoder, pipe_config);
+
+	intel_read_infoframe(encoder, pipe_config,
+			     HDMI_INFOFRAME_TYPE_AVI,
+			     &pipe_config->infoframes.avi);
+	intel_read_infoframe(encoder, pipe_config,
+			     HDMI_INFOFRAME_TYPE_SPD,
+			     &pipe_config->infoframes.spd);
+	intel_read_infoframe(encoder, pipe_config,
+			     HDMI_INFOFRAME_TYPE_VENDOR,
+			     &pipe_config->infoframes.hdmi);
 }
 
 static void intel_enable_hdmi_audio(struct intel_encoder *encoder,
@@ -1654,7 +2207,7 @@ static bool hdmi_deep_color_possible(const struct intel_crtc_state *crtc_state,
 
 	/* Display Wa_1405510057:icl */
 	if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 &&
-	    bpc == 10 && IS_ICELAKE(dev_priv) &&
+	    bpc == 10 && INTEL_GEN(dev_priv) >= 11 &&
 	    (adjusted_mode->crtc_hblank_end -
 	     adjusted_mode->crtc_hblank_start) % 8 == 2)
 		return false;
@@ -1812,6 +2365,23 @@ int intel_hdmi_compute_config(struct intel_encoder *encoder,
 		}
 	}
 
+	intel_hdmi_compute_gcp_infoframe(encoder, pipe_config, conn_state);
+
+	if (!intel_hdmi_compute_avi_infoframe(encoder, pipe_config, conn_state)) {
+		DRM_DEBUG_KMS("bad AVI infoframe\n");
+		return -EINVAL;
+	}
+
+	if (!intel_hdmi_compute_spd_infoframe(encoder, pipe_config, conn_state)) {
+		DRM_DEBUG_KMS("bad SPD infoframe\n");
+		return -EINVAL;
+	}
+
+	if (!intel_hdmi_compute_hdmi_infoframe(encoder, pipe_config, conn_state)) {
+		DRM_DEBUG_KMS("bad HDMI infoframe\n");
+		return -EINVAL;
+	}
+
 	return 0;
 }
 
@@ -1931,7 +2501,7 @@ intel_hdmi_detect(struct drm_connector *connector, bool force)
 
 	wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_GMBUS);
 
-	if (IS_ICELAKE(dev_priv) &&
+	if (INTEL_GEN(dev_priv) >= 11 &&
 	    !intel_digital_port_connected(encoder))
 		goto out;
 
@@ -2133,10 +2703,21 @@ static void
 intel_hdmi_add_properties(struct intel_hdmi *intel_hdmi, struct drm_connector *connector)
 {
 	struct drm_i915_private *dev_priv = to_i915(connector->dev);
+	struct intel_digital_port *intel_dig_port =
+				hdmi_to_dig_port(intel_hdmi);
 
 	intel_attach_force_audio_property(connector);
 	intel_attach_broadcast_rgb_property(connector);
 	intel_attach_aspect_ratio_property(connector);
+
+	/*
+	 * Attach Colorspace property for Non LSPCON based device
+	 * ToDo: This needs to be extended for LSPCON implementation
+	 * as well. Will be implemented separately.
+	 */
+	if (!intel_dig_port->lspcon.active)
+		intel_attach_colorspace_property(connector);
+
 	drm_connector_attach_content_type_property(connector);
 	connector->state->picture_aspect_ratio = HDMI_PICTURE_ASPECT_NONE;
 
@@ -2321,14 +2902,14 @@ static u8 intel_hdmi_ddc_pin(struct drm_i915_private *dev_priv,
 		return info->alternate_ddc_pin;
 	}
 
-	if (IS_CHERRYVIEW(dev_priv))
-		ddc_pin = chv_port_to_ddc_pin(dev_priv, port);
-	else if (IS_GEN9_LP(dev_priv))
-		ddc_pin = bxt_port_to_ddc_pin(dev_priv, port);
+	if (HAS_PCH_ICP(dev_priv))
+		ddc_pin = icl_port_to_ddc_pin(dev_priv, port);
 	else if (HAS_PCH_CNP(dev_priv))
 		ddc_pin = cnp_port_to_ddc_pin(dev_priv, port);
-	else if (HAS_PCH_ICP(dev_priv))
-		ddc_pin = icl_port_to_ddc_pin(dev_priv, port);
+	else if (IS_GEN9_LP(dev_priv))
+		ddc_pin = bxt_port_to_ddc_pin(dev_priv, port);
+	else if (IS_CHERRYVIEW(dev_priv))
+		ddc_pin = chv_port_to_ddc_pin(dev_priv, port);
 	else
 		ddc_pin = g4x_port_to_ddc_pin(dev_priv, port);
 
@@ -2345,33 +2926,36 @@ void intel_infoframe_init(struct intel_digital_port *intel_dig_port)
 
 	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 		intel_dig_port->write_infoframe = vlv_write_infoframe;
+		intel_dig_port->read_infoframe = vlv_read_infoframe;
 		intel_dig_port->set_infoframes = vlv_set_infoframes;
-		intel_dig_port->infoframe_enabled = vlv_infoframe_enabled;
+		intel_dig_port->infoframes_enabled = vlv_infoframes_enabled;
 	} else if (IS_G4X(dev_priv)) {
 		intel_dig_port->write_infoframe = g4x_write_infoframe;
+		intel_dig_port->read_infoframe = g4x_read_infoframe;
 		intel_dig_port->set_infoframes = g4x_set_infoframes;
-		intel_dig_port->infoframe_enabled = g4x_infoframe_enabled;
+		intel_dig_port->infoframes_enabled = g4x_infoframes_enabled;
 	} else if (HAS_DDI(dev_priv)) {
 		if (intel_dig_port->lspcon.active) {
-			intel_dig_port->write_infoframe =
-					lspcon_write_infoframe;
+			intel_dig_port->write_infoframe = lspcon_write_infoframe;
+			intel_dig_port->read_infoframe = lspcon_read_infoframe;
 			intel_dig_port->set_infoframes = lspcon_set_infoframes;
-			intel_dig_port->infoframe_enabled =
-						lspcon_infoframe_enabled;
+			intel_dig_port->infoframes_enabled = lspcon_infoframes_enabled;
 		} else {
-			intel_dig_port->set_infoframes = hsw_set_infoframes;
-			intel_dig_port->infoframe_enabled =
-						hsw_infoframe_enabled;
 			intel_dig_port->write_infoframe = hsw_write_infoframe;
+			intel_dig_port->read_infoframe = hsw_read_infoframe;
+			intel_dig_port->set_infoframes = hsw_set_infoframes;
+			intel_dig_port->infoframes_enabled = hsw_infoframes_enabled;
 		}
 	} else if (HAS_PCH_IBX(dev_priv)) {
 		intel_dig_port->write_infoframe = ibx_write_infoframe;
+		intel_dig_port->read_infoframe = ibx_read_infoframe;
 		intel_dig_port->set_infoframes = ibx_set_infoframes;
-		intel_dig_port->infoframe_enabled = ibx_infoframe_enabled;
+		intel_dig_port->infoframes_enabled = ibx_infoframes_enabled;
 	} else {
 		intel_dig_port->write_infoframe = cpt_write_infoframe;
+		intel_dig_port->read_infoframe = cpt_read_infoframe;
 		intel_dig_port->set_infoframes = cpt_set_infoframes;
-		intel_dig_port->infoframe_enabled = cpt_infoframe_enabled;
+		intel_dig_port->infoframes_enabled = cpt_infoframes_enabled;
 	}
 }
 
@@ -2417,6 +3001,9 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port,
 
 	intel_hdmi_add_properties(intel_hdmi, connector);
 
+	intel_connector_attach_encoder(intel_connector, intel_encoder);
+	intel_hdmi->attached_connector = intel_connector;
+
 	if (is_hdcp_supported(dev_priv, port)) {
 		int ret = intel_hdcp_init(intel_connector,
 					  &intel_hdmi_hdcp_shim);
@@ -2424,9 +3011,6 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port,
 			DRM_DEBUG_KMS("HDCP init failed, skipping.\n");
 	}
 
-	intel_connector_attach_encoder(intel_connector, intel_encoder);
-	intel_hdmi->attached_connector = intel_connector;
-
 	/* For G4X desktop chip, PEG_BAND_GAP_DATA 3:0 must first be written
 	 * 0xd.  Failure to do so will result in spurious interrupts being
 	 * generated on the port when a cable is not attached.
diff --git a/drivers/gpu/drm/i915/intel_hdmi.h b/drivers/gpu/drm/i915/intel_hdmi.h
new file mode 100644
index 000000000000..106c2e0bc3c9
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_hdmi.h
@@ -0,0 +1,51 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2019 Intel Corporation
+ */
+
+#ifndef __INTEL_HDMI_H__
+#define __INTEL_HDMI_H__
+
+#include <linux/hdmi.h>
+#include <linux/types.h>
+
+#include <drm/i915_drm.h>
+
+#include "i915_reg.h"
+
+struct drm_connector;
+struct drm_encoder;
+struct drm_i915_private;
+struct intel_connector;
+struct intel_digital_port;
+struct intel_encoder;
+struct intel_crtc_state;
+struct intel_hdmi;
+struct drm_connector_state;
+union hdmi_infoframe;
+
+void intel_hdmi_init(struct drm_i915_private *dev_priv, i915_reg_t hdmi_reg,
+		     enum port port);
+void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port,
+			       struct intel_connector *intel_connector);
+struct intel_hdmi *enc_to_intel_hdmi(struct drm_encoder *encoder);
+int intel_hdmi_compute_config(struct intel_encoder *encoder,
+			      struct intel_crtc_state *pipe_config,
+			      struct drm_connector_state *conn_state);
+bool intel_hdmi_handle_sink_scrambling(struct intel_encoder *encoder,
+				       struct drm_connector *connector,
+				       bool high_tmds_clock_ratio,
+				       bool scrambling);
+void intel_dp_dual_mode_set_tmds_output(struct intel_hdmi *hdmi, bool enable);
+void intel_infoframe_init(struct intel_digital_port *intel_dig_port);
+u32 intel_hdmi_infoframes_enabled(struct intel_encoder *encoder,
+				  const struct intel_crtc_state *crtc_state);
+u32 intel_hdmi_infoframe_enable(unsigned int type);
+void intel_hdmi_read_gcp_infoframe(struct intel_encoder *encoder,
+				   struct intel_crtc_state *crtc_state);
+void intel_read_infoframe(struct intel_encoder *encoder,
+			  const struct intel_crtc_state *crtc_state,
+			  enum hdmi_infoframe_type type,
+			  union hdmi_infoframe *frame);
+
+#endif /* __INTEL_HDMI_H__ */
diff --git a/drivers/gpu/drm/i915/intel_huc.c b/drivers/gpu/drm/i915/intel_huc.c
index 9bd1c9002c2a..94c04f16a2ad 100644
--- a/drivers/gpu/drm/i915/intel_huc.c
+++ b/drivers/gpu/drm/i915/intel_huc.c
@@ -79,7 +79,7 @@ int intel_huc_auth(struct intel_huc *huc)
 	}
 
 	/* Check authentication status, it should be done by now */
-	ret = __intel_wait_for_register(i915,
+	ret = __intel_wait_for_register(&i915->uncore,
 					HUC_STATUS2,
 					HUC_FW_VERIFIED,
 					HUC_FW_VERIFIED,
diff --git a/drivers/gpu/drm/i915/intel_huc_fw.c b/drivers/gpu/drm/i915/intel_huc_fw.c
index 7d7bfc7f7ca7..68d47c105939 100644
--- a/drivers/gpu/drm/i915/intel_huc_fw.c
+++ b/drivers/gpu/drm/i915/intel_huc_fw.c
@@ -106,41 +106,46 @@ static int huc_fw_xfer(struct intel_uc_fw *huc_fw, struct i915_vma *vma)
 {
 	struct intel_huc *huc = container_of(huc_fw, struct intel_huc, fw);
 	struct drm_i915_private *dev_priv = huc_to_i915(huc);
+	struct intel_uncore *uncore = &dev_priv->uncore;
 	unsigned long offset = 0;
 	u32 size;
 	int ret;
 
 	GEM_BUG_ON(huc_fw->type != INTEL_UC_FW_TYPE_HUC);
 
-	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
 
 	/* Set the source address for the uCode */
 	offset = intel_guc_ggtt_offset(&dev_priv->guc, vma) +
 		 huc_fw->header_offset;
-	I915_WRITE(DMA_ADDR_0_LOW, lower_32_bits(offset));
-	I915_WRITE(DMA_ADDR_0_HIGH, upper_32_bits(offset) & 0xFFFF);
+	intel_uncore_write(uncore, DMA_ADDR_0_LOW,
+			   lower_32_bits(offset));
+	intel_uncore_write(uncore, DMA_ADDR_0_HIGH,
+			   upper_32_bits(offset) & 0xFFFF);
 
-	/* Hardware doesn't look at destination address for HuC. Set it to 0,
+	/*
+	 * Hardware doesn't look at destination address for HuC. Set it to 0,
 	 * but still program the correct address space.
 	 */
-	I915_WRITE(DMA_ADDR_1_LOW, 0);
-	I915_WRITE(DMA_ADDR_1_HIGH, DMA_ADDRESS_SPACE_WOPCM);
+	intel_uncore_write(uncore, DMA_ADDR_1_LOW, 0);
+	intel_uncore_write(uncore, DMA_ADDR_1_HIGH, DMA_ADDRESS_SPACE_WOPCM);
 
 	size = huc_fw->header_size + huc_fw->ucode_size;
-	I915_WRITE(DMA_COPY_SIZE, size);
+	intel_uncore_write(uncore, DMA_COPY_SIZE, size);
 
 	/* Start the DMA */
-	I915_WRITE(DMA_CTRL, _MASKED_BIT_ENABLE(HUC_UKERNEL | START_DMA));
+	intel_uncore_write(uncore, DMA_CTRL,
+			   _MASKED_BIT_ENABLE(HUC_UKERNEL | START_DMA));
 
 	/* Wait for DMA to finish */
-	ret = intel_wait_for_register_fw(dev_priv, DMA_CTRL, START_DMA, 0, 100);
+	ret = intel_wait_for_register_fw(uncore, DMA_CTRL, START_DMA, 0, 100);
 
 	DRM_DEBUG_DRIVER("HuC DMA transfer wait over with ret %d\n", ret);
 
 	/* Disable the bits once DMA is over */
-	I915_WRITE(DMA_CTRL, _MASKED_BIT_DISABLE(HUC_UKERNEL));
+	intel_uncore_write(uncore, DMA_CTRL, _MASKED_BIT_DISABLE(HUC_UKERNEL));
 
-	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
 
 	return ret;
 }
diff --git a/drivers/gpu/drm/i915/intel_i2c.c b/drivers/gpu/drm/i915/intel_i2c.c
index 5a733e711355..422685d120e9 100644
--- a/drivers/gpu/drm/i915/intel_i2c.c
+++ b/drivers/gpu/drm/i915/intel_i2c.c
@@ -348,7 +348,7 @@ gmbus_wait_idle(struct drm_i915_private *dev_priv)
 	add_wait_queue(&dev_priv->gmbus_wait_queue, &wait);
 	I915_WRITE_FW(GMBUS4, irq_enable);
 
-	ret = intel_wait_for_register_fw(dev_priv,
+	ret = intel_wait_for_register_fw(&dev_priv->uncore,
 					 GMBUS2, GMBUS_ACTIVE, 0,
 					 10);
 
diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c
index 5e98fd79bd9d..4e0a351bfbca 100644
--- a/drivers/gpu/drm/i915/intel_lrc.c
+++ b/drivers/gpu/drm/i915/intel_lrc.c
@@ -164,20 +164,15 @@
 #define WA_TAIL_DWORDS 2
 #define WA_TAIL_BYTES (sizeof(u32) * WA_TAIL_DWORDS)
 
-static int execlists_context_deferred_alloc(struct i915_gem_context *ctx,
-					    struct intel_engine_cs *engine,
-					    struct intel_context *ce);
+#define ACTIVE_PRIORITY (I915_PRIORITY_NEWCLIENT | I915_PRIORITY_NOSEMAPHORE)
+
+static int execlists_context_deferred_alloc(struct intel_context *ce,
+					    struct intel_engine_cs *engine);
 static void execlists_init_reg_state(u32 *reg_state,
-				     struct i915_gem_context *ctx,
+				     struct intel_context *ce,
 				     struct intel_engine_cs *engine,
 				     struct intel_ring *ring);
 
-static inline u32 intel_hws_seqno_address(struct intel_engine_cs *engine)
-{
-	return (i915_ggtt_offset(engine->status_page.vma) +
-		I915_GEM_HWS_INDEX_ADDR);
-}
-
 static inline struct i915_priolist *to_priolist(struct rb_node *rb)
 {
 	return rb_entry(rb, struct i915_priolist, node);
@@ -188,6 +183,34 @@ static inline int rq_prio(const struct i915_request *rq)
 	return rq->sched.attr.priority;
 }
 
+static int effective_prio(const struct i915_request *rq)
+{
+	int prio = rq_prio(rq);
+
+	/*
+	 * On unwinding the active request, we give it a priority bump
+	 * equivalent to a freshly submitted request. This protects it from
+	 * being gazumped again, but it would be preferable if we didn't
+	 * let it be gazumped in the first place!
+	 *
+	 * See __unwind_incomplete_requests()
+	 */
+	if (~prio & ACTIVE_PRIORITY && __i915_request_has_started(rq)) {
+		/*
+		 * After preemption, we insert the active request at the
+		 * end of the new priority level. This means that we will be
+		 * _lower_ priority than the preemptee all things equal (and
+		 * so the preemption is valid), so adjust our comparison
+		 * accordingly.
+		 */
+		prio |= ACTIVE_PRIORITY;
+		prio--;
+	}
+
+	/* Restrict mere WAIT boosts from triggering preemption */
+	return prio | __NO_PREEMPTION;
+}
+
 static int queue_prio(const struct intel_engine_execlists *execlists)
 {
 	struct i915_priolist *p;
@@ -208,9 +231,9 @@ static int queue_prio(const struct intel_engine_execlists *execlists)
 static inline bool need_preempt(const struct intel_engine_cs *engine,
 				const struct i915_request *rq)
 {
-	const int last_prio = rq_prio(rq);
+	int last_prio;
 
-	if (!intel_engine_has_preemption(engine))
+	if (!engine->preempt_context)
 		return false;
 
 	if (i915_request_completed(rq))
@@ -228,6 +251,7 @@ static inline bool need_preempt(const struct intel_engine_cs *engine,
 	 * preempt. If that hint is stale or we may be trying to preempt
 	 * ourselves, ignore the request.
 	 */
+	last_prio = effective_prio(rq);
 	if (!__execlists_need_preempt(engine->execlists.queue_priority_hint,
 				      last_prio))
 		return false;
@@ -254,12 +278,11 @@ static inline bool need_preempt(const struct intel_engine_cs *engine,
 }
 
 __maybe_unused static inline bool
-assert_priority_queue(const struct intel_engine_execlists *execlists,
-		      const struct i915_request *prev,
+assert_priority_queue(const struct i915_request *prev,
 		      const struct i915_request *next)
 {
-	if (!prev)
-		return true;
+	const struct intel_engine_execlists *execlists =
+		&prev->engine->execlists;
 
 	/*
 	 * Without preemption, the prev may refer to the still active element
@@ -300,11 +323,10 @@ assert_priority_queue(const struct intel_engine_execlists *execlists,
  * engine info, SW context ID and SW counter need to form a unique number
  * (Context ID) per lrc.
  */
-static void
-intel_lr_context_descriptor_update(struct i915_gem_context *ctx,
-				   struct intel_engine_cs *engine,
-				   struct intel_context *ce)
+static u64
+lrc_descriptor(struct intel_context *ce, struct intel_engine_cs *engine)
 {
+	struct i915_gem_context *ctx = ce->gem_context;
 	u64 desc;
 
 	BUILD_BUG_ON(MAX_CONTEXT_HW_ID > (BIT(GEN8_CTX_ID_WIDTH)));
@@ -322,7 +344,7 @@ intel_lr_context_descriptor_update(struct i915_gem_context *ctx,
 	 * Consider updating oa_get_render_ctx_id in i915_perf.c when changing
 	 * anything below.
 	 */
-	if (INTEL_GEN(ctx->i915) >= 11) {
+	if (INTEL_GEN(engine->i915) >= 11) {
 		GEM_BUG_ON(ctx->hw_id >= BIT(GEN11_SW_CTX_ID_WIDTH));
 		desc |= (u64)ctx->hw_id << GEN11_SW_CTX_ID_SHIFT;
 								/* bits 37-47 */
@@ -339,7 +361,7 @@ intel_lr_context_descriptor_update(struct i915_gem_context *ctx,
 		desc |= (u64)ctx->hw_id << GEN8_CTX_ID_SHIFT;	/* bits 32-52 */
 	}
 
-	ce->lrc_desc = desc;
+	return desc;
 }
 
 static void unwind_wa_tail(struct i915_request *rq)
@@ -353,7 +375,7 @@ __unwind_incomplete_requests(struct intel_engine_cs *engine)
 {
 	struct i915_request *rq, *rn, *active = NULL;
 	struct list_head *uninitialized_var(pl);
-	int prio = I915_PRIORITY_INVALID | I915_PRIORITY_NEWCLIENT;
+	int prio = I915_PRIORITY_INVALID | ACTIVE_PRIORITY;
 
 	lockdep_assert_held(&engine->timeline.lock);
 
@@ -384,9 +406,21 @@ __unwind_incomplete_requests(struct intel_engine_cs *engine)
 	 * The active request is now effectively the start of a new client
 	 * stream, so give it the equivalent small priority bump to prevent
 	 * it being gazumped a second time by another peer.
+	 *
+	 * Note we have to be careful not to apply a priority boost to a request
+	 * still spinning on its semaphores. If the request hasn't started, that
+	 * means it is still waiting for its dependencies to be signaled, and
+	 * if we apply a priority boost to this request, we will boost it past
+	 * its signalers and so break PI.
+	 *
+	 * One consequence of this preemption boost is that we may jump
+	 * over lesser priorities (such as I915_PRIORITY_WAIT), effectively
+	 * making those priorities non-preemptible. They will be moved forward
+	 * in the priority queue, but they will not gain immediate access to
+	 * the GPU.
 	 */
-	if (!(prio & I915_PRIORITY_NEWCLIENT)) {
-		prio |= I915_PRIORITY_NEWCLIENT;
+	if (~prio & ACTIVE_PRIORITY && __i915_request_has_started(active)) {
+		prio |= ACTIVE_PRIORITY;
 		active->sched.attr.priority = prio;
 		list_move_tail(&active->sched.link,
 			       i915_sched_lookup_priolist(engine, prio));
@@ -395,13 +429,13 @@ __unwind_incomplete_requests(struct intel_engine_cs *engine)
 	return active;
 }
 
-void
+struct i915_request *
 execlists_unwind_incomplete_requests(struct intel_engine_execlists *execlists)
 {
 	struct intel_engine_cs *engine =
 		container_of(execlists, typeof(*engine), execlists);
 
-	__unwind_incomplete_requests(engine);
+	return __unwind_incomplete_requests(engine);
 }
 
 static inline void
@@ -523,13 +557,11 @@ static void execlists_submit_ports(struct intel_engine_cs *engine)
 			desc = execlists_update_context(rq);
 			GEM_DEBUG_EXEC(port[n].context_id = upper_32_bits(desc));
 
-			GEM_TRACE("%s in[%d]:  ctx=%d.%d, global=%d (fence %llx:%lld) (current %d:%d), prio=%d\n",
+			GEM_TRACE("%s in[%d]:  ctx=%d.%d, fence %llx:%lld (current %d), prio=%d\n",
 				  engine->name, n,
 				  port[n].context_id, count,
-				  rq->global_seqno,
 				  rq->fence.context, rq->fence.seqno,
 				  hwsp_seqno(rq),
-				  intel_engine_get_seqno(engine),
 				  rq_prio(rq));
 		} else {
 			GEM_BUG_ON(!n);
@@ -564,6 +596,17 @@ static bool can_merge_ctx(const struct intel_context *prev,
 	return true;
 }
 
+static bool can_merge_rq(const struct i915_request *prev,
+			 const struct i915_request *next)
+{
+	GEM_BUG_ON(!assert_priority_queue(prev, next));
+
+	if (!can_merge_ctx(prev->hw_context, next->hw_context))
+		return false;
+
+	return true;
+}
+
 static void port_assign(struct execlist_port *port, struct i915_request *rq)
 {
 	GEM_BUG_ON(rq == port_request(port));
@@ -577,8 +620,7 @@ static void port_assign(struct execlist_port *port, struct i915_request *rq)
 static void inject_preempt_context(struct intel_engine_cs *engine)
 {
 	struct intel_engine_execlists *execlists = &engine->execlists;
-	struct intel_context *ce =
-		to_intel_context(engine->i915->preempt_context, engine);
+	struct intel_context *ce = engine->preempt_context;
 	unsigned int n;
 
 	GEM_BUG_ON(execlists->preempt_complete_status !=
@@ -716,8 +758,6 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
 		int i;
 
 		priolist_for_each_request_consume(rq, rn, p, i) {
-			GEM_BUG_ON(!assert_priority_queue(execlists, last, rq));
-
 			/*
 			 * Can we combine this request with the current port?
 			 * It has to be the same context/ringbuffer and not
@@ -729,8 +769,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
 			 * second request, and so we never need to tell the
 			 * hardware about the first.
 			 */
-			if (last &&
-			    !can_merge_ctx(rq->hw_context, last->hw_context)) {
+			if (last && !can_merge_rq(last, rq)) {
 				/*
 				 * If we are on the second port and cannot
 				 * combine this request with the last, then we
@@ -740,6 +779,14 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
 					goto done;
 
 				/*
+				 * We must not populate both ELSP[] with the
+				 * same LRCA, i.e. we must submit 2 different
+				 * contexts if we submit 2 ELSP.
+				 */
+				if (last->hw_context == rq->hw_context)
+					goto done;
+
+				/*
 				 * If GVT overrides us we only ever submit
 				 * port[0], leaving port[1] empty. Note that we
 				 * also have to be careful that we don't queue
@@ -750,7 +797,6 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
 				    ctx_single_port_submission(rq->hw_context))
 					goto done;
 
-				GEM_BUG_ON(last->hw_context == rq->hw_context);
 
 				if (submit)
 					port_assign(port, last);
@@ -769,8 +815,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
 		}
 
 		rb_erase_cached(&p->node, &execlists->queue);
-		if (p->priority != I915_PRIORITY_NORMAL)
-			kmem_cache_free(engine->i915->priorities, p);
+		i915_priolist_free(p);
 	}
 
 done:
@@ -790,8 +835,7 @@ done:
 	 * request triggering preemption on the next dequeue (or subsequent
 	 * interrupt for secondary ports).
 	 */
-	execlists->queue_priority_hint =
-		port != execlists->port ? rq_prio(last) : INT_MIN;
+	execlists->queue_priority_hint = queue_prio(execlists);
 
 	if (submit) {
 		port_assign(port, last);
@@ -821,13 +865,11 @@ execlists_cancel_port_requests(struct intel_engine_execlists * const execlists)
 	while (num_ports-- && port_isset(port)) {
 		struct i915_request *rq = port_request(port);
 
-		GEM_TRACE("%s:port%u global=%d (fence %llx:%lld), (current %d:%d)\n",
+		GEM_TRACE("%s:port%u fence %llx:%lld, (current %d)\n",
 			  rq->engine->name,
 			  (unsigned int)(port - execlists->port),
-			  rq->global_seqno,
 			  rq->fence.context, rq->fence.seqno,
-			  hwsp_seqno(rq),
-			  intel_engine_get_seqno(rq->engine));
+			  hwsp_seqno(rq));
 
 		GEM_BUG_ON(!execlists->active);
 		execlists_context_schedule_out(rq,
@@ -851,104 +893,6 @@ invalidate_csb_entries(const u32 *first, const u32 *last)
 	clflush((void *)last);
 }
 
-static void reset_csb_pointers(struct intel_engine_execlists *execlists)
-{
-	const unsigned int reset_value = GEN8_CSB_ENTRIES - 1;
-
-	/*
-	 * After a reset, the HW starts writing into CSB entry [0]. We
-	 * therefore have to set our HEAD pointer back one entry so that
-	 * the *first* entry we check is entry 0. To complicate this further,
-	 * as we don't wait for the first interrupt after reset, we have to
-	 * fake the HW write to point back to the last entry so that our
-	 * inline comparison of our cached head position against the last HW
-	 * write works even before the first interrupt.
-	 */
-	execlists->csb_head = reset_value;
-	WRITE_ONCE(*execlists->csb_write, reset_value);
-
-	invalidate_csb_entries(&execlists->csb_status[0],
-			       &execlists->csb_status[GEN8_CSB_ENTRIES - 1]);
-}
-
-static void nop_submission_tasklet(unsigned long data)
-{
-	/* The driver is wedged; don't process any more events. */
-}
-
-static void execlists_cancel_requests(struct intel_engine_cs *engine)
-{
-	struct intel_engine_execlists * const execlists = &engine->execlists;
-	struct i915_request *rq, *rn;
-	struct rb_node *rb;
-	unsigned long flags;
-
-	GEM_TRACE("%s current %d\n",
-		  engine->name, intel_engine_get_seqno(engine));
-
-	/*
-	 * Before we call engine->cancel_requests(), we should have exclusive
-	 * access to the submission state. This is arranged for us by the
-	 * caller disabling the interrupt generation, the tasklet and other
-	 * threads that may then access the same state, giving us a free hand
-	 * to reset state. However, we still need to let lockdep be aware that
-	 * we know this state may be accessed in hardirq context, so we
-	 * disable the irq around this manipulation and we want to keep
-	 * the spinlock focused on its duties and not accidentally conflate
-	 * coverage to the submission's irq state. (Similarly, although we
-	 * shouldn't need to disable irq around the manipulation of the
-	 * submission's irq state, we also wish to remind ourselves that
-	 * it is irq state.)
-	 */
-	spin_lock_irqsave(&engine->timeline.lock, flags);
-
-	/* Cancel the requests on the HW and clear the ELSP tracker. */
-	execlists_cancel_port_requests(execlists);
-	execlists_user_end(execlists);
-
-	/* Mark all executing requests as skipped. */
-	list_for_each_entry(rq, &engine->timeline.requests, link) {
-		GEM_BUG_ON(!rq->global_seqno);
-
-		if (!i915_request_signaled(rq))
-			dma_fence_set_error(&rq->fence, -EIO);
-
-		i915_request_mark_complete(rq);
-	}
-
-	/* Flush the queued requests to the timeline list (for retiring). */
-	while ((rb = rb_first_cached(&execlists->queue))) {
-		struct i915_priolist *p = to_priolist(rb);
-		int i;
-
-		priolist_for_each_request_consume(rq, rn, p, i) {
-			list_del_init(&rq->sched.link);
-			__i915_request_submit(rq);
-			dma_fence_set_error(&rq->fence, -EIO);
-			i915_request_mark_complete(rq);
-		}
-
-		rb_erase_cached(&p->node, &execlists->queue);
-		if (p->priority != I915_PRIORITY_NORMAL)
-			kmem_cache_free(engine->i915->priorities, p);
-	}
-
-	intel_write_status_page(engine,
-				I915_GEM_HWS_INDEX,
-				intel_engine_last_submit(engine));
-
-	/* Remaining _unready_ requests will be nop'ed when submitted */
-
-	execlists->queue_priority_hint = INT_MIN;
-	execlists->queue = RB_ROOT_CACHED;
-	GEM_BUG_ON(port_isset(execlists->port));
-
-	GEM_BUG_ON(__tasklet_is_enabled(&execlists->tasklet));
-	execlists->tasklet.func = nop_submission_tasklet;
-
-	spin_unlock_irqrestore(&engine->timeline.lock, flags);
-}
-
 static inline bool
 reset_in_progress(const struct intel_engine_execlists *execlists)
 {
@@ -960,6 +904,7 @@ static void process_csb(struct intel_engine_cs *engine)
 	struct intel_engine_execlists * const execlists = &engine->execlists;
 	struct execlist_port *port = execlists->port;
 	const u32 * const buf = execlists->csb_status;
+	const u8 num_entries = execlists->csb_size;
 	u8 head, tail;
 
 	lockdep_assert_held(&engine->timeline.lock);
@@ -995,7 +940,7 @@ static void process_csb(struct intel_engine_cs *engine)
 		unsigned int status;
 		unsigned int count;
 
-		if (++head == GEN8_CSB_ENTRIES)
+		if (++head == num_entries)
 			head = 0;
 
 		/*
@@ -1052,14 +997,12 @@ static void process_csb(struct intel_engine_cs *engine)
 						EXECLISTS_ACTIVE_USER));
 
 		rq = port_unpack(port, &count);
-		GEM_TRACE("%s out[0]: ctx=%d.%d, global=%d (fence %llx:%lld) (current %d:%d), prio=%d\n",
+		GEM_TRACE("%s out[0]: ctx=%d.%d, fence %llx:%lld (current %d), prio=%d\n",
 			  engine->name,
 			  port->context_id, count,
-			  rq ? rq->global_seqno : 0,
 			  rq ? rq->fence.context : 0,
 			  rq ? rq->fence.seqno : 0,
 			  rq ? hwsp_seqno(rq) : 0,
-			  intel_engine_get_seqno(engine),
 			  rq ? rq_prio(rq) : 0);
 
 		/* Check the context/desc id for this event matches */
@@ -1119,7 +1062,7 @@ static void process_csb(struct intel_engine_cs *engine)
 	 * the wash as hardware, working or not, will need to do the
 	 * invalidation before.
 	 */
-	invalidate_csb_entries(&buf[0], &buf[GEN8_CSB_ENTRIES - 1]);
+	invalidate_csb_entries(&buf[0], &buf[num_entries - 1]);
 }
 
 static void __execlists_submission_tasklet(struct intel_engine_cs *const engine)
@@ -1196,19 +1139,50 @@ static void execlists_submit_request(struct i915_request *request)
 	spin_unlock_irqrestore(&engine->timeline.lock, flags);
 }
 
-static void execlists_context_destroy(struct intel_context *ce)
+static void __execlists_context_fini(struct intel_context *ce)
 {
-	GEM_BUG_ON(ce->pin_count);
-
-	if (!ce->state)
-		return;
-
-	intel_ring_free(ce->ring);
+	intel_ring_put(ce->ring);
 
 	GEM_BUG_ON(i915_gem_object_is_active(ce->state->obj));
 	i915_gem_object_put(ce->state->obj);
 }
 
+static void execlists_context_destroy(struct kref *kref)
+{
+	struct intel_context *ce = container_of(kref, typeof(*ce), ref);
+
+	GEM_BUG_ON(intel_context_is_pinned(ce));
+
+	if (ce->state)
+		__execlists_context_fini(ce);
+
+	intel_context_free(ce);
+}
+
+static int __context_pin(struct i915_vma *vma)
+{
+	unsigned int flags;
+	int err;
+
+	flags = PIN_GLOBAL | PIN_HIGH;
+	flags |= PIN_OFFSET_BIAS | i915_ggtt_pin_bias(vma);
+
+	err = i915_vma_pin(vma, 0, 0, flags);
+	if (err)
+		return err;
+
+	vma->obj->pin_global++;
+	vma->obj->mm.dirty = true;
+
+	return 0;
+}
+
+static void __context_unpin(struct i915_vma *vma)
+{
+	vma->obj->pin_global--;
+	__i915_vma_unpin(vma);
+}
+
 static void execlists_context_unpin(struct intel_context *ce)
 {
 	struct intel_engine_cs *engine;
@@ -1237,41 +1211,19 @@ static void execlists_context_unpin(struct intel_context *ce)
 
 	intel_ring_unpin(ce->ring);
 
-	ce->state->obj->pin_global--;
 	i915_gem_object_unpin_map(ce->state->obj);
-	i915_vma_unpin(ce->state);
-
-	i915_gem_context_put(ce->gem_context);
-}
-
-static int __context_pin(struct i915_gem_context *ctx, struct i915_vma *vma)
-{
-	unsigned int flags;
-	int err;
-
-	/*
-	 * Clear this page out of any CPU caches for coherent swap-in/out.
-	 * We only want to do this on the first bind so that we do not stall
-	 * on an active context (which by nature is already on the GPU).
-	 */
-	if (!(vma->flags & I915_VMA_GLOBAL_BIND)) {
-		err = i915_gem_object_set_to_wc_domain(vma->obj, true);
-		if (err)
-			return err;
-	}
-
-	flags = PIN_GLOBAL | PIN_HIGH;
-	flags |= PIN_OFFSET_BIAS | i915_ggtt_pin_bias(vma);
-
-	return i915_vma_pin(vma, 0, 0, flags);
+	__context_unpin(ce->state);
 }
 
 static void
-__execlists_update_reg_state(struct intel_engine_cs *engine,
-			     struct intel_context *ce)
+__execlists_update_reg_state(struct intel_context *ce,
+			     struct intel_engine_cs *engine)
 {
-	u32 *regs = ce->lrc_reg_state;
 	struct intel_ring *ring = ce->ring;
+	u32 *regs = ce->lrc_reg_state;
+
+	GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->head));
+	GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->tail));
 
 	regs[CTX_RING_BUFFER_START + 1] = i915_ggtt_offset(ring->vma);
 	regs[CTX_RING_HEAD + 1] = ring->head;
@@ -1279,29 +1231,30 @@ __execlists_update_reg_state(struct intel_engine_cs *engine,
 
 	/* RPCS */
 	if (engine->class == RENDER_CLASS)
-		regs[CTX_R_PWR_CLK_STATE + 1] = gen8_make_rpcs(engine->i915,
-							       &ce->sseu);
+		regs[CTX_R_PWR_CLK_STATE + 1] =
+			gen8_make_rpcs(engine->i915, &ce->sseu);
 }
 
-static struct intel_context *
-__execlists_context_pin(struct intel_engine_cs *engine,
-			struct i915_gem_context *ctx,
-			struct intel_context *ce)
+static int
+__execlists_context_pin(struct intel_context *ce,
+			struct intel_engine_cs *engine)
 {
 	void *vaddr;
 	int ret;
 
-	ret = execlists_context_deferred_alloc(ctx, engine, ce);
+	GEM_BUG_ON(!ce->gem_context->ppgtt);
+
+	ret = execlists_context_deferred_alloc(ce, engine);
 	if (ret)
 		goto err;
 	GEM_BUG_ON(!ce->state);
 
-	ret = __context_pin(ctx, ce->state);
+	ret = __context_pin(ce->state);
 	if (ret)
 		goto err;
 
 	vaddr = i915_gem_object_pin_map(ce->state->obj,
-					i915_coherent_map_type(ctx->i915) |
+					i915_coherent_map_type(engine->i915) |
 					I915_MAP_OVERRIDE);
 	if (IS_ERR(vaddr)) {
 		ret = PTR_ERR(vaddr);
@@ -1312,55 +1265,60 @@ __execlists_context_pin(struct intel_engine_cs *engine,
 	if (ret)
 		goto unpin_map;
 
-	ret = i915_gem_context_pin_hw_id(ctx);
+	ret = i915_gem_context_pin_hw_id(ce->gem_context);
 	if (ret)
 		goto unpin_ring;
 
-	intel_lr_context_descriptor_update(ctx, engine, ce);
-
-	GEM_BUG_ON(!intel_ring_offset_valid(ce->ring, ce->ring->head));
-
+	ce->lrc_desc = lrc_descriptor(ce, engine);
 	ce->lrc_reg_state = vaddr + LRC_STATE_PN * PAGE_SIZE;
+	__execlists_update_reg_state(ce, engine);
 
-	__execlists_update_reg_state(engine, ce);
-
-	ce->state->obj->pin_global++;
-	i915_gem_context_get(ctx);
-	return ce;
+	return 0;
 
 unpin_ring:
 	intel_ring_unpin(ce->ring);
 unpin_map:
 	i915_gem_object_unpin_map(ce->state->obj);
 unpin_vma:
-	__i915_vma_unpin(ce->state);
+	__context_unpin(ce->state);
 err:
-	ce->pin_count = 0;
-	return ERR_PTR(ret);
+	return ret;
 }
 
-static const struct intel_context_ops execlists_context_ops = {
-	.unpin = execlists_context_unpin,
-	.destroy = execlists_context_destroy,
-};
-
-static struct intel_context *
-execlists_context_pin(struct intel_engine_cs *engine,
-		      struct i915_gem_context *ctx)
+static int execlists_context_pin(struct intel_context *ce)
 {
-	struct intel_context *ce = to_intel_context(ctx, engine);
-
-	lockdep_assert_held(&ctx->i915->drm.struct_mutex);
-	GEM_BUG_ON(!ctx->ppgtt);
+	return __execlists_context_pin(ce, ce->engine);
+}
 
-	if (likely(ce->pin_count++))
-		return ce;
-	GEM_BUG_ON(!ce->pin_count); /* no overflow please! */
+static void execlists_context_reset(struct intel_context *ce)
+{
+	/*
+	 * Because we emit WA_TAIL_DWORDS there may be a disparity
+	 * between our bookkeeping in ce->ring->head and ce->ring->tail and
+	 * that stored in context. As we only write new commands from
+	 * ce->ring->tail onwards, everything before that is junk. If the GPU
+	 * starts reading from its RING_HEAD from the context, it may try to
+	 * execute that junk and die.
+	 *
+	 * The contexts that are stilled pinned on resume belong to the
+	 * kernel, and are local to each engine. All other contexts will
+	 * have their head/tail sanitized upon pinning before use, so they
+	 * will never see garbage,
+	 *
+	 * So to avoid that we reset the context images upon resume. For
+	 * simplicity, we just zero everything out.
+	 */
+	intel_ring_reset(ce->ring, 0);
+	__execlists_update_reg_state(ce, ce->engine);
+}
 
-	ce->ops = &execlists_context_ops;
+static const struct intel_context_ops execlists_context_ops = {
+	.pin = execlists_context_pin,
+	.unpin = execlists_context_unpin,
 
-	return __execlists_context_pin(engine, ctx, ce);
-}
+	.reset = execlists_context_reset,
+	.destroy = execlists_context_destroy,
+};
 
 static int gen8_emit_init_breadcrumb(struct i915_request *rq)
 {
@@ -1387,6 +1345,10 @@ static int gen8_emit_init_breadcrumb(struct i915_request *rq)
 	*cs++ = rq->fence.seqno - 1;
 
 	intel_ring_advance(rq, cs);
+
+	/* Record the updated position of the request's payload */
+	rq->infix = intel_ring_offset(rq, cs);
+
 	return 0;
 }
 
@@ -1424,10 +1386,11 @@ static int emit_pdps(struct i915_request *rq)
 	*cs++ = MI_LOAD_REGISTER_IMM(2 * GEN8_3LVL_PDPES) | MI_LRI_FORCE_POSTED;
 	for (i = GEN8_3LVL_PDPES; i--; ) {
 		const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i);
+		u32 base = engine->mmio_base;
 
-		*cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(engine, i));
+		*cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(base, i));
 		*cs++ = upper_32_bits(pd_daddr);
-		*cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(engine, i));
+		*cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(base, i));
 		*cs++ = lower_32_bits(pd_daddr);
 	}
 	*cs++ = MI_NOOP;
@@ -1447,7 +1410,7 @@ static int execlists_request_alloc(struct i915_request *request)
 {
 	int ret;
 
-	GEM_BUG_ON(!request->hw_context->pin_count);
+	GEM_BUG_ON(!intel_context_is_pinned(request->hw_context));
 
 	/*
 	 * Flush enough space to reduce the likelihood of waiting after
@@ -1465,7 +1428,7 @@ static int execlists_request_alloc(struct i915_request *request)
 	 */
 
 	/* Unconditionally invalidate GPU caches and TLBs. */
-	if (i915_vm_is_48bit(&request->gem_context->ppgtt->vm))
+	if (i915_vm_is_4lvl(&request->gem_context->ppgtt->vm))
 		ret = request->engine->emit_flush(request, EMIT_INVALIDATE);
 	else
 		ret = emit_pdps(request);
@@ -1732,7 +1695,7 @@ static int intel_init_workaround_bb(struct intel_engine_cs *engine)
 	unsigned int i;
 	int ret;
 
-	if (GEM_DEBUG_WARN_ON(engine->id != RCS))
+	if (GEM_DEBUG_WARN_ON(engine->id != RCS0))
 		return -EINVAL;
 
 	switch (INTEL_GEN(engine->i915)) {
@@ -1796,17 +1759,9 @@ static void enable_execlists(struct intel_engine_cs *engine)
 
 	intel_engine_set_hwsp_writemask(engine, ~0u); /* HWSTAM */
 
-	/*
-	 * Make sure we're not enabling the new 12-deep CSB
-	 * FIFO as that requires a slightly updated handling
-	 * in the ctx switch irq. Since we're currently only
-	 * using only 2 elements of the enhanced execlists the
-	 * deeper FIFO it's not needed and it's not worth adding
-	 * more statements to the irq handler to support it.
-	 */
 	if (INTEL_GEN(dev_priv) >= 11)
 		I915_WRITE(RING_MODE_GEN7(engine),
-			   _MASKED_BIT_DISABLE(GEN11_GFX_DISABLE_LEGACY_MODE));
+			   _MASKED_BIT_ENABLE(GEN11_GFX_DISABLE_LEGACY_MODE));
 	else
 		I915_WRITE(RING_MODE_GEN7(engine),
 			   _MASKED_BIT_ENABLE(GFX_RUN_LIST_ENABLE));
@@ -1872,20 +1827,72 @@ static void execlists_reset_prepare(struct intel_engine_cs *engine)
 	__tasklet_disable_sync_once(&execlists->tasklet);
 	GEM_BUG_ON(!reset_in_progress(execlists));
 
+	intel_engine_stop_cs(engine);
+
 	/* And flush any current direct submission. */
 	spin_lock_irqsave(&engine->timeline.lock, flags);
-	process_csb(engine); /* drain preemption events */
 	spin_unlock_irqrestore(&engine->timeline.lock, flags);
 }
 
-static void execlists_reset(struct intel_engine_cs *engine, bool stalled)
+static bool lrc_regs_ok(const struct i915_request *rq)
+{
+	const struct intel_ring *ring = rq->ring;
+	const u32 *regs = rq->hw_context->lrc_reg_state;
+
+	/* Quick spot check for the common signs of context corruption */
+
+	if (regs[CTX_RING_BUFFER_CONTROL + 1] !=
+	    (RING_CTL_SIZE(ring->size) | RING_VALID))
+		return false;
+
+	if (regs[CTX_RING_BUFFER_START + 1] != i915_ggtt_offset(ring->vma))
+		return false;
+
+	return true;
+}
+
+static void reset_csb_pointers(struct intel_engine_execlists *execlists)
+{
+	const unsigned int reset_value = execlists->csb_size - 1;
+
+	/*
+	 * After a reset, the HW starts writing into CSB entry [0]. We
+	 * therefore have to set our HEAD pointer back one entry so that
+	 * the *first* entry we check is entry 0. To complicate this further,
+	 * as we don't wait for the first interrupt after reset, we have to
+	 * fake the HW write to point back to the last entry so that our
+	 * inline comparison of our cached head position against the last HW
+	 * write works even before the first interrupt.
+	 */
+	execlists->csb_head = reset_value;
+	WRITE_ONCE(*execlists->csb_write, reset_value);
+	wmb(); /* Make sure this is visible to HW (paranoia?) */
+
+	invalidate_csb_entries(&execlists->csb_status[0],
+			       &execlists->csb_status[reset_value]);
+}
+
+static void __execlists_reset(struct intel_engine_cs *engine, bool stalled)
 {
 	struct intel_engine_execlists * const execlists = &engine->execlists;
+	struct intel_context *ce;
 	struct i915_request *rq;
-	unsigned long flags;
 	u32 *regs;
 
-	spin_lock_irqsave(&engine->timeline.lock, flags);
+	process_csb(engine); /* drain preemption events */
+
+	/* Following the reset, we need to reload the CSB read/write pointers */
+	reset_csb_pointers(&engine->execlists);
+
+	/*
+	 * Save the currently executing context, even if we completed
+	 * its request, it was still running at the time of the
+	 * reset and will have been clobbered.
+	 */
+	if (!port_isset(execlists->port))
+		goto out_clear;
+
+	ce = port_request(execlists->port)->hw_context;
 
 	/*
 	 * Catch up with any missed context-switch interrupts.
@@ -1900,17 +1907,28 @@ static void execlists_reset(struct intel_engine_cs *engine, bool stalled)
 
 	/* Push back any incomplete requests for replay after the reset. */
 	rq = __unwind_incomplete_requests(engine);
+	if (!rq)
+		goto out_replay;
 
-	/* Following the reset, we need to reload the CSB read/write pointers */
-	reset_csb_pointers(&engine->execlists);
+	if (rq->hw_context != ce) { /* caught just before a CS event */
+		rq = NULL;
+		goto out_replay;
+	}
 
-	GEM_TRACE("%s seqno=%d, current=%d, stalled? %s\n",
-		  engine->name,
-		  rq ? rq->global_seqno : 0,
-		  intel_engine_get_seqno(engine),
-		  yesno(stalled));
-	if (!rq)
-		goto out_unlock;
+	/*
+	 * If this request hasn't started yet, e.g. it is waiting on a
+	 * semaphore, we need to avoid skipping the request or else we
+	 * break the signaling chain. However, if the context is corrupt
+	 * the request will not restart and we will be stuck with a wedged
+	 * device. It is quite often the case that if we issue a reset
+	 * while the GPU is loading the context image, that the context
+	 * image becomes corrupt.
+	 *
+	 * Otherwise, if we have not started yet, the request should replay
+	 * perfectly and we do not need to flag the result as being erroneous.
+	 */
+	if (!i915_request_started(rq) && lrc_regs_ok(rq))
+		goto out_replay;
 
 	/*
 	 * If the request was innocent, we leave the request in the ELSP
@@ -1924,8 +1942,8 @@ static void execlists_reset(struct intel_engine_cs *engine, bool stalled)
 	 * image back to the expected values to skip over the guilty request.
 	 */
 	i915_reset_request(rq, stalled);
-	if (!stalled)
-		goto out_unlock;
+	if (!stalled && lrc_regs_ok(rq))
+		goto out_replay;
 
 	/*
 	 * We want a simple context + ring to execute the breadcrumb update.
@@ -1935,21 +1953,103 @@ static void execlists_reset(struct intel_engine_cs *engine, bool stalled)
 	 * future request will be after userspace has had the opportunity
 	 * to recreate its own state.
 	 */
-	regs = rq->hw_context->lrc_reg_state;
+	regs = ce->lrc_reg_state;
 	if (engine->pinned_default_state) {
 		memcpy(regs, /* skip restoring the vanilla PPHWSP */
 		       engine->pinned_default_state + LRC_STATE_PN * PAGE_SIZE,
 		       engine->context_size - PAGE_SIZE);
 	}
+	execlists_init_reg_state(regs, ce, engine, ce->ring);
 
-	/* Move the RING_HEAD onto the breadcrumb, past the hanging batch */
-	rq->ring->head = intel_ring_wrap(rq->ring, rq->postfix);
-	intel_ring_update_space(rq->ring);
+	/* Rerun the request; its payload has been neutered (if guilty). */
+out_replay:
+	ce->ring->head =
+		rq ? intel_ring_wrap(ce->ring, rq->head) : ce->ring->tail;
+	intel_ring_update_space(ce->ring);
+	__execlists_update_reg_state(ce, engine);
+
+out_clear:
+	execlists_clear_all_active(execlists);
+}
+
+static void execlists_reset(struct intel_engine_cs *engine, bool stalled)
+{
+	unsigned long flags;
+
+	GEM_TRACE("%s\n", engine->name);
+
+	spin_lock_irqsave(&engine->timeline.lock, flags);
 
-	execlists_init_reg_state(regs, rq->gem_context, engine, rq->ring);
-	__execlists_update_reg_state(engine, rq->hw_context);
+	__execlists_reset(engine, stalled);
+
+	spin_unlock_irqrestore(&engine->timeline.lock, flags);
+}
+
+static void nop_submission_tasklet(unsigned long data)
+{
+	/* The driver is wedged; don't process any more events. */
+}
+
+static void execlists_cancel_requests(struct intel_engine_cs *engine)
+{
+	struct intel_engine_execlists * const execlists = &engine->execlists;
+	struct i915_request *rq, *rn;
+	struct rb_node *rb;
+	unsigned long flags;
+
+	GEM_TRACE("%s\n", engine->name);
+
+	/*
+	 * Before we call engine->cancel_requests(), we should have exclusive
+	 * access to the submission state. This is arranged for us by the
+	 * caller disabling the interrupt generation, the tasklet and other
+	 * threads that may then access the same state, giving us a free hand
+	 * to reset state. However, we still need to let lockdep be aware that
+	 * we know this state may be accessed in hardirq context, so we
+	 * disable the irq around this manipulation and we want to keep
+	 * the spinlock focused on its duties and not accidentally conflate
+	 * coverage to the submission's irq state. (Similarly, although we
+	 * shouldn't need to disable irq around the manipulation of the
+	 * submission's irq state, we also wish to remind ourselves that
+	 * it is irq state.)
+	 */
+	spin_lock_irqsave(&engine->timeline.lock, flags);
+
+	__execlists_reset(engine, true);
+
+	/* Mark all executing requests as skipped. */
+	list_for_each_entry(rq, &engine->timeline.requests, link) {
+		if (!i915_request_signaled(rq))
+			dma_fence_set_error(&rq->fence, -EIO);
+
+		i915_request_mark_complete(rq);
+	}
+
+	/* Flush the queued requests to the timeline list (for retiring). */
+	while ((rb = rb_first_cached(&execlists->queue))) {
+		struct i915_priolist *p = to_priolist(rb);
+		int i;
+
+		priolist_for_each_request_consume(rq, rn, p, i) {
+			list_del_init(&rq->sched.link);
+			__i915_request_submit(rq);
+			dma_fence_set_error(&rq->fence, -EIO);
+			i915_request_mark_complete(rq);
+		}
+
+		rb_erase_cached(&p->node, &execlists->queue);
+		i915_priolist_free(p);
+	}
+
+	/* Remaining _unready_ requests will be nop'ed when submitted */
+
+	execlists->queue_priority_hint = INT_MIN;
+	execlists->queue = RB_ROOT_CACHED;
+	GEM_BUG_ON(port_isset(execlists->port));
+
+	GEM_BUG_ON(__tasklet_is_enabled(&execlists->tasklet));
+	execlists->tasklet.func = nop_submission_tasklet;
 
-out_unlock:
 	spin_unlock_irqrestore(&engine->timeline.lock, flags);
 }
 
@@ -1961,13 +2061,14 @@ static void execlists_reset_finish(struct intel_engine_cs *engine)
 	 * After a GPU reset, we may have requests to replay. Do so now while
 	 * we still have the forcewake to be sure that the GPU is not allowed
 	 * to sleep before we restart and reload a context.
-	 *
 	 */
 	GEM_BUG_ON(!reset_in_progress(execlists));
 	if (!RB_EMPTY_ROOT(&execlists->queue.rb_root))
 		execlists->tasklet.func(execlists->tasklet.data);
 
-	tasklet_enable(&execlists->tasklet);
+	if (__tasklet_enable(&execlists->tasklet))
+		/* And kick in case we missed a new request submission. */
+		tasklet_hi_schedule(&execlists->tasklet);
 	GEM_TRACE("%s: depth->%d\n", engine->name,
 		  atomic_read(&execlists->tasklet.count));
 }
@@ -1978,7 +2079,7 @@ static int gen8_emit_bb_start(struct i915_request *rq,
 {
 	u32 *cs;
 
-	cs = intel_ring_begin(rq, 6);
+	cs = intel_ring_begin(rq, 4);
 	if (IS_ERR(cs))
 		return PTR_ERR(cs);
 
@@ -1989,19 +2090,37 @@ static int gen8_emit_bb_start(struct i915_request *rq,
 	 * particular all the gen that do not need the w/a at all!), if we
 	 * took care to make sure that on every switch into this context
 	 * (both ordinary and for preemption) that arbitrartion was enabled
-	 * we would be fine. However, there doesn't seem to be a downside to
-	 * being paranoid and making sure it is set before each batch and
-	 * every context-switch.
-	 *
-	 * Note that if we fail to enable arbitration before the request
-	 * is complete, then we do not see the context-switch interrupt and
-	 * the engine hangs (with RING_HEAD == RING_TAIL).
-	 *
-	 * That satisfies both the GPGPU w/a and our heavy-handed paranoia.
+	 * we would be fine.  However, for gen8 there is another w/a that
+	 * requires us to not preempt inside GPGPU execution, so we keep
+	 * arbitration disabled for gen8 batches. Arbitration will be
+	 * re-enabled before we close the request
+	 * (engine->emit_fini_breadcrumb).
 	 */
+	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
+
+	/* FIXME(BDW+): Address space and security selectors. */
+	*cs++ = MI_BATCH_BUFFER_START_GEN8 |
+		(flags & I915_DISPATCH_SECURE ? 0 : BIT(8));
+	*cs++ = lower_32_bits(offset);
+	*cs++ = upper_32_bits(offset);
+
+	intel_ring_advance(rq, cs);
+
+	return 0;
+}
+
+static int gen9_emit_bb_start(struct i915_request *rq,
+			      u64 offset, u32 len,
+			      const unsigned int flags)
+{
+	u32 *cs;
+
+	cs = intel_ring_begin(rq, 6);
+	if (IS_ERR(cs))
+		return PTR_ERR(cs);
+
 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
 
-	/* FIXME(BDW): Address space and security selectors. */
 	*cs++ = MI_BATCH_BUFFER_START_GEN8 |
 		(flags & I915_DISPATCH_SECURE ? 0 : BIT(8));
 	*cs++ = lower_32_bits(offset);
@@ -2017,16 +2136,14 @@ static int gen8_emit_bb_start(struct i915_request *rq,
 
 static void gen8_logical_ring_enable_irq(struct intel_engine_cs *engine)
 {
-	struct drm_i915_private *dev_priv = engine->i915;
-	I915_WRITE_IMR(engine,
-		       ~(engine->irq_enable_mask | engine->irq_keep_mask));
-	POSTING_READ_FW(RING_IMR(engine->mmio_base));
+	ENGINE_WRITE(engine, RING_IMR,
+		     ~(engine->irq_enable_mask | engine->irq_keep_mask));
+	ENGINE_POSTING_READ(engine, RING_IMR);
 }
 
 static void gen8_logical_ring_disable_irq(struct intel_engine_cs *engine)
 {
-	struct drm_i915_private *dev_priv = engine->i915;
-	I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
+	ENGINE_WRITE(engine, RING_IMR, ~engine->irq_keep_mask);
 }
 
 static int gen8_emit_flush(struct i915_request *request, u32 mode)
@@ -2148,16 +2265,16 @@ static u32 *gen8_emit_wa_tail(struct i915_request *request, u32 *cs)
 
 static u32 *gen8_emit_fini_breadcrumb(struct i915_request *request, u32 *cs)
 {
-	/* w/a: bit 5 needs to be zero for MI_FLUSH_DW address. */
-	BUILD_BUG_ON(I915_GEM_HWS_INDEX_ADDR & (1 << 5));
-
 	cs = gen8_emit_ggtt_write(cs,
 				  request->fence.seqno,
-				  request->timeline->hwsp_offset);
+				  request->timeline->hwsp_offset,
+				  0);
 
 	cs = gen8_emit_ggtt_write(cs,
-				  request->global_seqno,
-				  intel_hws_seqno_address(request->engine));
+				  intel_engine_next_hangcheck_seqno(request->engine),
+				  I915_GEM_HWS_HANGCHECK_ADDR,
+				  MI_FLUSH_DW_STORE_INDEX);
+
 
 	*cs++ = MI_USER_INTERRUPT;
 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
@@ -2180,9 +2297,9 @@ static u32 *gen8_emit_fini_breadcrumb_rcs(struct i915_request *request, u32 *cs)
 				      PIPE_CONTROL_CS_STALL);
 
 	cs = gen8_emit_ggtt_write_rcs(cs,
-				      request->global_seqno,
-				      intel_hws_seqno_address(request->engine),
-				      PIPE_CONTROL_CS_STALL);
+				      intel_engine_next_hangcheck_seqno(request->engine),
+				      I915_GEM_HWS_HANGCHECK_ADDR,
+				      PIPE_CONTROL_STORE_DATA_INDEX);
 
 	*cs++ = MI_USER_INTERRUPT;
 	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
@@ -2231,7 +2348,7 @@ void intel_logical_ring_cleanup(struct intel_engine_cs *engine)
 	dev_priv = engine->i915;
 
 	if (engine->buffer) {
-		WARN_ON((I915_READ_MODE(engine) & MODE_IDLE) == 0);
+		WARN_ON((ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE) == 0);
 	}
 
 	if (engine->cleanup)
@@ -2254,19 +2371,18 @@ void intel_execlists_set_default_submission(struct intel_engine_cs *engine)
 	engine->execlists.tasklet.func = execlists_submission_tasklet;
 
 	engine->reset.prepare = execlists_reset_prepare;
+	engine->reset.reset = execlists_reset;
+	engine->reset.finish = execlists_reset_finish;
 
 	engine->park = NULL;
 	engine->unpark = NULL;
 
 	engine->flags |= I915_ENGINE_SUPPORTS_STATS;
-	if (engine->i915->preempt_context)
+	if (!intel_vgpu_active(engine->i915))
+		engine->flags |= I915_ENGINE_HAS_SEMAPHORES;
+	if (engine->preempt_context &&
+	    HAS_LOGICAL_RING_PREEMPTION(engine->i915))
 		engine->flags |= I915_ENGINE_HAS_PREEMPTION;
-
-	engine->i915->caps.scheduler =
-		I915_SCHEDULER_CAP_ENABLED |
-		I915_SCHEDULER_CAP_PRIORITY;
-	if (intel_engine_has_preemption(engine))
-		engine->i915->caps.scheduler |= I915_SCHEDULER_CAP_PREEMPTION;
 }
 
 static void
@@ -2279,7 +2395,7 @@ logical_ring_default_vfuncs(struct intel_engine_cs *engine)
 	engine->reset.reset = execlists_reset;
 	engine->reset.finish = execlists_reset_finish;
 
-	engine->context_pin = execlists_context_pin;
+	engine->cops = &execlists_context_ops;
 	engine->request_alloc = execlists_request_alloc;
 
 	engine->emit_flush = gen8_emit_flush;
@@ -2299,7 +2415,10 @@ logical_ring_default_vfuncs(struct intel_engine_cs *engine)
 		 * until a more refined solution exists.
 		 */
 	}
-	engine->emit_bb_start = gen8_emit_bb_start;
+	if (IS_GEN(engine->i915, 8))
+		engine->emit_bb_start = gen8_emit_bb_start;
+	else
+		engine->emit_bb_start = gen9_emit_bb_start;
 }
 
 static inline void
@@ -2309,11 +2428,11 @@ logical_ring_default_irqs(struct intel_engine_cs *engine)
 
 	if (INTEL_GEN(engine->i915) < 11) {
 		const u8 irq_shifts[] = {
-			[RCS]  = GEN8_RCS_IRQ_SHIFT,
-			[BCS]  = GEN8_BCS_IRQ_SHIFT,
-			[VCS]  = GEN8_VCS1_IRQ_SHIFT,
-			[VCS2] = GEN8_VCS2_IRQ_SHIFT,
-			[VECS] = GEN8_VECS_IRQ_SHIFT,
+			[RCS0]  = GEN8_RCS_IRQ_SHIFT,
+			[BCS0]  = GEN8_BCS_IRQ_SHIFT,
+			[VCS0]  = GEN8_VCS0_IRQ_SHIFT,
+			[VCS1]  = GEN8_VCS1_IRQ_SHIFT,
+			[VECS0] = GEN8_VECS_IRQ_SHIFT,
 		};
 
 		shift = irq_shifts[engine->id];
@@ -2348,6 +2467,7 @@ static int logical_ring_init(struct intel_engine_cs *engine)
 {
 	struct drm_i915_private *i915 = engine->i915;
 	struct intel_engine_execlists * const execlists = &engine->execlists;
+	u32 base = engine->mmio_base;
 	int ret;
 
 	ret = intel_engine_init_common(engine);
@@ -2357,23 +2477,19 @@ static int logical_ring_init(struct intel_engine_cs *engine)
 	intel_engine_init_workarounds(engine);
 
 	if (HAS_LOGICAL_RING_ELSQ(i915)) {
-		execlists->submit_reg = i915->regs +
-			i915_mmio_reg_offset(RING_EXECLIST_SQ_CONTENTS(engine));
-		execlists->ctrl_reg = i915->regs +
-			i915_mmio_reg_offset(RING_EXECLIST_CONTROL(engine));
+		execlists->submit_reg = i915->uncore.regs +
+			i915_mmio_reg_offset(RING_EXECLIST_SQ_CONTENTS(base));
+		execlists->ctrl_reg = i915->uncore.regs +
+			i915_mmio_reg_offset(RING_EXECLIST_CONTROL(base));
 	} else {
-		execlists->submit_reg = i915->regs +
-			i915_mmio_reg_offset(RING_ELSP(engine));
+		execlists->submit_reg = i915->uncore.regs +
+			i915_mmio_reg_offset(RING_ELSP(base));
 	}
 
 	execlists->preempt_complete_status = ~0u;
-	if (i915->preempt_context) {
-		struct intel_context *ce =
-			to_intel_context(i915->preempt_context, engine);
-
+	if (engine->preempt_context)
 		execlists->preempt_complete_status =
-			upper_32_bits(ce->lrc_desc);
-	}
+			upper_32_bits(engine->preempt_context->lrc_desc);
 
 	execlists->csb_status =
 		&engine->status_page.addr[I915_HWS_CSB_BUF0_INDEX];
@@ -2381,6 +2497,11 @@ static int logical_ring_init(struct intel_engine_cs *engine)
 	execlists->csb_write =
 		&engine->status_page.addr[intel_hws_csb_write_index(i915)];
 
+	if (INTEL_GEN(engine->i915) < 11)
+		execlists->csb_size = GEN8_CSB_ENTRIES;
+	else
+		execlists->csb_size = GEN11_CSB_ENTRIES;
+
 	reset_csb_pointers(execlists);
 
 	return 0;
@@ -2592,13 +2713,13 @@ static u32 intel_lr_indirect_ctx_offset(struct intel_engine_cs *engine)
 }
 
 static void execlists_init_reg_state(u32 *regs,
-				     struct i915_gem_context *ctx,
+				     struct intel_context *ce,
 				     struct intel_engine_cs *engine,
 				     struct intel_ring *ring)
 {
-	struct drm_i915_private *dev_priv = engine->i915;
-	u32 base = engine->mmio_base;
+	struct i915_hw_ppgtt *ppgtt = ce->gem_context->ppgtt;
 	bool rcs = engine->class == RENDER_CLASS;
+	u32 base = engine->mmio_base;
 
 	/* A context is actually a big batch buffer with several
 	 * MI_LOAD_REGISTER_IMM commands followed by (reg, value) pairs. The
@@ -2610,10 +2731,10 @@ static void execlists_init_reg_state(u32 *regs,
 	regs[CTX_LRI_HEADER_0] = MI_LOAD_REGISTER_IMM(rcs ? 14 : 11) |
 				 MI_LRI_FORCE_POSTED;
 
-	CTX_REG(regs, CTX_CONTEXT_CONTROL, RING_CONTEXT_CONTROL(engine),
+	CTX_REG(regs, CTX_CONTEXT_CONTROL, RING_CONTEXT_CONTROL(base),
 		_MASKED_BIT_DISABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT) |
 		_MASKED_BIT_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH));
-	if (INTEL_GEN(dev_priv) < 11) {
+	if (INTEL_GEN(engine->i915) < 11) {
 		regs[CTX_CONTEXT_CONTROL + 1] |=
 			_MASKED_BIT_DISABLE(CTX_CTRL_ENGINE_CTX_SAVE_INHIBIT |
 					    CTX_CTRL_RS_CTX_ENABLE);
@@ -2659,42 +2780,42 @@ static void execlists_init_reg_state(u32 *regs,
 
 	CTX_REG(regs, CTX_CTX_TIMESTAMP, RING_CTX_TIMESTAMP(base), 0);
 	/* PDP values well be assigned later if needed */
-	CTX_REG(regs, CTX_PDP3_UDW, GEN8_RING_PDP_UDW(engine, 3), 0);
-	CTX_REG(regs, CTX_PDP3_LDW, GEN8_RING_PDP_LDW(engine, 3), 0);
-	CTX_REG(regs, CTX_PDP2_UDW, GEN8_RING_PDP_UDW(engine, 2), 0);
-	CTX_REG(regs, CTX_PDP2_LDW, GEN8_RING_PDP_LDW(engine, 2), 0);
-	CTX_REG(regs, CTX_PDP1_UDW, GEN8_RING_PDP_UDW(engine, 1), 0);
-	CTX_REG(regs, CTX_PDP1_LDW, GEN8_RING_PDP_LDW(engine, 1), 0);
-	CTX_REG(regs, CTX_PDP0_UDW, GEN8_RING_PDP_UDW(engine, 0), 0);
-	CTX_REG(regs, CTX_PDP0_LDW, GEN8_RING_PDP_LDW(engine, 0), 0);
-
-	if (i915_vm_is_48bit(&ctx->ppgtt->vm)) {
+	CTX_REG(regs, CTX_PDP3_UDW, GEN8_RING_PDP_UDW(base, 3), 0);
+	CTX_REG(regs, CTX_PDP3_LDW, GEN8_RING_PDP_LDW(base, 3), 0);
+	CTX_REG(regs, CTX_PDP2_UDW, GEN8_RING_PDP_UDW(base, 2), 0);
+	CTX_REG(regs, CTX_PDP2_LDW, GEN8_RING_PDP_LDW(base, 2), 0);
+	CTX_REG(regs, CTX_PDP1_UDW, GEN8_RING_PDP_UDW(base, 1), 0);
+	CTX_REG(regs, CTX_PDP1_LDW, GEN8_RING_PDP_LDW(base, 1), 0);
+	CTX_REG(regs, CTX_PDP0_UDW, GEN8_RING_PDP_UDW(base, 0), 0);
+	CTX_REG(regs, CTX_PDP0_LDW, GEN8_RING_PDP_LDW(base, 0), 0);
+
+	if (i915_vm_is_4lvl(&ppgtt->vm)) {
 		/* 64b PPGTT (48bit canonical)
 		 * PDP0_DESCRIPTOR contains the base address to PML4 and
 		 * other PDP Descriptors are ignored.
 		 */
-		ASSIGN_CTX_PML4(ctx->ppgtt, regs);
+		ASSIGN_CTX_PML4(ppgtt, regs);
 	} else {
-		ASSIGN_CTX_PDP(ctx->ppgtt, regs, 3);
-		ASSIGN_CTX_PDP(ctx->ppgtt, regs, 2);
-		ASSIGN_CTX_PDP(ctx->ppgtt, regs, 1);
-		ASSIGN_CTX_PDP(ctx->ppgtt, regs, 0);
+		ASSIGN_CTX_PDP(ppgtt, regs, 3);
+		ASSIGN_CTX_PDP(ppgtt, regs, 2);
+		ASSIGN_CTX_PDP(ppgtt, regs, 1);
+		ASSIGN_CTX_PDP(ppgtt, regs, 0);
 	}
 
 	if (rcs) {
 		regs[CTX_LRI_HEADER_2] = MI_LOAD_REGISTER_IMM(1);
 		CTX_REG(regs, CTX_R_PWR_CLK_STATE, GEN8_R_PWR_CLK_STATE, 0);
 
-		i915_oa_init_reg_state(engine, ctx, regs);
+		i915_oa_init_reg_state(engine, ce, regs);
 	}
 
 	regs[CTX_END] = MI_BATCH_BUFFER_END;
-	if (INTEL_GEN(dev_priv) >= 10)
+	if (INTEL_GEN(engine->i915) >= 10)
 		regs[CTX_END] |= BIT(0);
 }
 
 static int
-populate_lr_context(struct i915_gem_context *ctx,
+populate_lr_context(struct intel_context *ce,
 		    struct drm_i915_gem_object *ctx_obj,
 		    struct intel_engine_cs *engine,
 		    struct intel_ring *ring)
@@ -2703,19 +2824,12 @@ populate_lr_context(struct i915_gem_context *ctx,
 	u32 *regs;
 	int ret;
 
-	ret = i915_gem_object_set_to_cpu_domain(ctx_obj, true);
-	if (ret) {
-		DRM_DEBUG_DRIVER("Could not set to CPU domain\n");
-		return ret;
-	}
-
 	vaddr = i915_gem_object_pin_map(ctx_obj, I915_MAP_WB);
 	if (IS_ERR(vaddr)) {
 		ret = PTR_ERR(vaddr);
 		DRM_DEBUG_DRIVER("Could not map object pages! (%d)\n", ret);
 		return ret;
 	}
-	ctx_obj->mm.dirty = true;
 
 	if (engine->default_state) {
 		/*
@@ -2740,23 +2854,35 @@ populate_lr_context(struct i915_gem_context *ctx,
 	/* The second page of the context object contains some fields which must
 	 * be set up prior to the first execution. */
 	regs = vaddr + LRC_STATE_PN * PAGE_SIZE;
-	execlists_init_reg_state(regs, ctx, engine, ring);
+	execlists_init_reg_state(regs, ce, engine, ring);
 	if (!engine->default_state)
 		regs[CTX_CONTEXT_CONTROL + 1] |=
 			_MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT);
-	if (ctx == ctx->i915->preempt_context && INTEL_GEN(engine->i915) < 11)
+	if (ce->gem_context == engine->i915->preempt_context &&
+	    INTEL_GEN(engine->i915) < 11)
 		regs[CTX_CONTEXT_CONTROL + 1] |=
 			_MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT |
 					   CTX_CTRL_ENGINE_CTX_SAVE_INHIBIT);
 
+	ret = 0;
 err_unpin_ctx:
+	__i915_gem_object_flush_map(ctx_obj,
+				    LRC_HEADER_PAGES * PAGE_SIZE,
+				    engine->context_size);
 	i915_gem_object_unpin_map(ctx_obj);
 	return ret;
 }
 
-static int execlists_context_deferred_alloc(struct i915_gem_context *ctx,
-					    struct intel_engine_cs *engine,
-					    struct intel_context *ce)
+static struct i915_timeline *get_timeline(struct i915_gem_context *ctx)
+{
+	if (ctx->timeline)
+		return i915_timeline_get(ctx->timeline);
+	else
+		return i915_timeline_create(ctx->i915, NULL);
+}
+
+static int execlists_context_deferred_alloc(struct intel_context *ce,
+					    struct intel_engine_cs *engine)
 {
 	struct drm_i915_gem_object *ctx_obj;
 	struct i915_vma *vma;
@@ -2776,30 +2902,32 @@ static int execlists_context_deferred_alloc(struct i915_gem_context *ctx,
 	 */
 	context_size += LRC_HEADER_PAGES * PAGE_SIZE;
 
-	ctx_obj = i915_gem_object_create(ctx->i915, context_size);
+	ctx_obj = i915_gem_object_create(engine->i915, context_size);
 	if (IS_ERR(ctx_obj))
 		return PTR_ERR(ctx_obj);
 
-	vma = i915_vma_instance(ctx_obj, &ctx->i915->ggtt.vm, NULL);
+	vma = i915_vma_instance(ctx_obj, &engine->i915->ggtt.vm, NULL);
 	if (IS_ERR(vma)) {
 		ret = PTR_ERR(vma);
 		goto error_deref_obj;
 	}
 
-	timeline = i915_timeline_create(ctx->i915, ctx->name, NULL);
+	timeline = get_timeline(ce->gem_context);
 	if (IS_ERR(timeline)) {
 		ret = PTR_ERR(timeline);
 		goto error_deref_obj;
 	}
 
-	ring = intel_engine_create_ring(engine, timeline, ctx->ring_size);
+	ring = intel_engine_create_ring(engine,
+					timeline,
+					ce->gem_context->ring_size);
 	i915_timeline_put(timeline);
 	if (IS_ERR(ring)) {
 		ret = PTR_ERR(ring);
 		goto error_deref_obj;
 	}
 
-	ret = populate_lr_context(ctx, ctx_obj, engine, ring);
+	ret = populate_lr_context(ce, ctx_obj, engine, ring);
 	if (ret) {
 		DRM_DEBUG_DRIVER("Failed to populate LRC: %d\n", ret);
 		goto error_ring_free;
@@ -2811,45 +2939,12 @@ static int execlists_context_deferred_alloc(struct i915_gem_context *ctx,
 	return 0;
 
 error_ring_free:
-	intel_ring_free(ring);
+	intel_ring_put(ring);
 error_deref_obj:
 	i915_gem_object_put(ctx_obj);
 	return ret;
 }
 
-void intel_lr_context_resume(struct drm_i915_private *i915)
-{
-	struct intel_engine_cs *engine;
-	struct i915_gem_context *ctx;
-	enum intel_engine_id id;
-
-	/*
-	 * Because we emit WA_TAIL_DWORDS there may be a disparity
-	 * between our bookkeeping in ce->ring->head and ce->ring->tail and
-	 * that stored in context. As we only write new commands from
-	 * ce->ring->tail onwards, everything before that is junk. If the GPU
-	 * starts reading from its RING_HEAD from the context, it may try to
-	 * execute that junk and die.
-	 *
-	 * So to avoid that we reset the context images upon resume. For
-	 * simplicity, we just zero everything out.
-	 */
-	list_for_each_entry(ctx, &i915->contexts.list, link) {
-		for_each_engine(engine, i915, id) {
-			struct intel_context *ce =
-				to_intel_context(ctx, engine);
-
-			if (!ce->state)
-				continue;
-
-			intel_ring_reset(ce->ring, 0);
-
-			if (ce->pin_count) /* otherwise done in context_pin */
-				__execlists_update_reg_state(engine, ce);
-		}
-	}
-}
-
 void intel_execlists_show_requests(struct intel_engine_cs *engine,
 				   struct drm_printer *m,
 				   void (*show_request)(struct drm_printer *m,
@@ -2910,6 +3005,37 @@ void intel_execlists_show_requests(struct intel_engine_cs *engine,
 	spin_unlock_irqrestore(&engine->timeline.lock, flags);
 }
 
+void intel_lr_context_reset(struct intel_engine_cs *engine,
+			    struct intel_context *ce,
+			    u32 head,
+			    bool scrub)
+{
+	/*
+	 * We want a simple context + ring to execute the breadcrumb update.
+	 * We cannot rely on the context being intact across the GPU hang,
+	 * so clear it and rebuild just what we need for the breadcrumb.
+	 * All pending requests for this context will be zapped, and any
+	 * future request will be after userspace has had the opportunity
+	 * to recreate its own state.
+	 */
+	if (scrub) {
+		u32 *regs = ce->lrc_reg_state;
+
+		if (engine->pinned_default_state) {
+			memcpy(regs, /* skip restoring the vanilla PPHWSP */
+			       engine->pinned_default_state + LRC_STATE_PN * PAGE_SIZE,
+			       engine->context_size - PAGE_SIZE);
+		}
+		execlists_init_reg_state(regs, ce, engine, ce->ring);
+	}
+
+	/* Rerun the request; its payload has been neutered (if guilty). */
+	ce->ring->head = head;
+	intel_ring_update_space(ce->ring);
+
+	__execlists_update_reg_state(ce, engine);
+}
+
 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 #include "selftests/intel_lrc.c"
 #endif
diff --git a/drivers/gpu/drm/i915/intel_lrc.h b/drivers/gpu/drm/i915/intel_lrc.h
index f1aec8a6986f..84aa230ea27b 100644
--- a/drivers/gpu/drm/i915/intel_lrc.h
+++ b/drivers/gpu/drm/i915/intel_lrc.h
@@ -28,20 +28,18 @@
 #include "i915_gem_context.h"
 
 /* Execlists regs */
-#define RING_ELSP(engine)			_MMIO((engine)->mmio_base + 0x230)
-#define RING_EXECLIST_STATUS_LO(engine)		_MMIO((engine)->mmio_base + 0x234)
-#define RING_EXECLIST_STATUS_HI(engine)		_MMIO((engine)->mmio_base + 0x234 + 4)
-#define RING_CONTEXT_CONTROL(engine)		_MMIO((engine)->mmio_base + 0x244)
+#define RING_ELSP(base)				_MMIO((base) + 0x230)
+#define RING_EXECLIST_STATUS_LO(base)		_MMIO((base) + 0x234)
+#define RING_EXECLIST_STATUS_HI(base)		_MMIO((base) + 0x234 + 4)
+#define RING_CONTEXT_CONTROL(base)		_MMIO((base) + 0x244)
 #define	  CTX_CTRL_INHIBIT_SYN_CTX_SWITCH	(1 << 3)
 #define	  CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT	(1 << 0)
-#define   CTX_CTRL_RS_CTX_ENABLE                (1 << 1)
+#define   CTX_CTRL_RS_CTX_ENABLE		(1 << 1)
 #define	  CTX_CTRL_ENGINE_CTX_SAVE_INHIBIT	(1 << 2)
-#define RING_CONTEXT_STATUS_BUF_BASE(engine)	_MMIO((engine)->mmio_base + 0x370)
-#define RING_CONTEXT_STATUS_BUF_LO(engine, i)	_MMIO((engine)->mmio_base + 0x370 + (i) * 8)
-#define RING_CONTEXT_STATUS_BUF_HI(engine, i)	_MMIO((engine)->mmio_base + 0x370 + (i) * 8 + 4)
-#define RING_CONTEXT_STATUS_PTR(engine)		_MMIO((engine)->mmio_base + 0x3a0)
-#define RING_EXECLIST_SQ_CONTENTS(engine)	_MMIO((engine)->mmio_base + 0x510)
-#define RING_EXECLIST_CONTROL(engine)		_MMIO((engine)->mmio_base + 0x550)
+#define RING_CONTEXT_STATUS_PTR(base)		_MMIO((base) + 0x3a0)
+#define RING_EXECLIST_SQ_CONTENTS(base)		_MMIO((base) + 0x510)
+#define RING_EXECLIST_CONTROL(base)		_MMIO((base) + 0x550)
+
 #define	  EL_CTRL_LOAD				(1 << 0)
 
 /* The docs specify that the write pointer wraps around after 5h, "After status
@@ -55,10 +53,11 @@
 #define GEN8_CSB_PTR_MASK 0x7
 #define GEN8_CSB_READ_PTR_MASK (GEN8_CSB_PTR_MASK << 8)
 #define GEN8_CSB_WRITE_PTR_MASK (GEN8_CSB_PTR_MASK << 0)
-#define GEN8_CSB_WRITE_PTR(csb_status) \
-	(((csb_status) & GEN8_CSB_WRITE_PTR_MASK) >> 0)
-#define GEN8_CSB_READ_PTR(csb_status) \
-	(((csb_status) & GEN8_CSB_READ_PTR_MASK) >> 8)
+
+#define GEN11_CSB_ENTRIES 12
+#define GEN11_CSB_PTR_MASK 0xf
+#define GEN11_CSB_READ_PTR_MASK (GEN11_CSB_PTR_MASK << 8)
+#define GEN11_CSB_WRITE_PTR_MASK (GEN11_CSB_PTR_MASK << 0)
 
 enum {
 	INTEL_CONTEXT_SCHEDULE_IN = 0,
@@ -102,9 +101,13 @@ struct drm_printer;
 struct drm_i915_private;
 struct i915_gem_context;
 
-void intel_lr_context_resume(struct drm_i915_private *dev_priv);
 void intel_execlists_set_default_submission(struct intel_engine_cs *engine);
 
+void intel_lr_context_reset(struct intel_engine_cs *engine,
+			    struct intel_context *ce,
+			    u32 head,
+			    bool scrub);
+
 void intel_execlists_show_requests(struct intel_engine_cs *engine,
 				   struct drm_printer *m,
 				   void (*show_request)(struct drm_printer *m,
diff --git a/drivers/gpu/drm/i915/intel_lspcon.c b/drivers/gpu/drm/i915/intel_lspcon.c
index 322bdddda164..7028d0cf3bb1 100644
--- a/drivers/gpu/drm/i915/intel_lspcon.c
+++ b/drivers/gpu/drm/i915/intel_lspcon.c
@@ -22,10 +22,14 @@
  *
  *
  */
-#include <drm/drm_edid.h>
+
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_dp_dual_mode_helper.h>
+#include <drm/drm_edid.h>
+
+#include "intel_dp.h"
 #include "intel_drv.h"
+#include "intel_lspcon.h"
 
 /* LSPCON OUI Vendor ID(signatures) */
 #define LSPCON_VENDOR_PARADE_OUI 0x001CF8
@@ -452,6 +456,14 @@ void lspcon_write_infoframe(struct intel_encoder *encoder,
 	DRM_DEBUG_DRIVER("AVI infoframes updated successfully\n");
 }
 
+void lspcon_read_infoframe(struct intel_encoder *encoder,
+			   const struct intel_crtc_state *crtc_state,
+			   unsigned int type,
+			   void *frame, ssize_t len)
+{
+	/* FIXME implement this */
+}
+
 void lspcon_set_infoframes(struct intel_encoder *encoder,
 			   bool enable,
 			   const struct intel_crtc_state *crtc_state,
@@ -470,6 +482,8 @@ void lspcon_set_infoframes(struct intel_encoder *encoder,
 		return;
 	}
 
+	/* FIXME precompute infoframes */
+
 	ret = drm_hdmi_avi_infoframe_from_display_mode(&frame.avi,
 						       conn_state->connector,
 						       adjusted_mode);
@@ -504,9 +518,10 @@ void lspcon_set_infoframes(struct intel_encoder *encoder,
 				  buf, ret);
 }
 
-bool lspcon_infoframe_enabled(struct intel_encoder *encoder,
+u32 lspcon_infoframes_enabled(struct intel_encoder *encoder,
 			      const struct intel_crtc_state *pipe_config)
 {
+	/* FIXME actually read this from the hw */
 	return enc_to_intel_lspcon(&encoder->base)->active;
 }
 
diff --git a/drivers/gpu/drm/i915/intel_lspcon.h b/drivers/gpu/drm/i915/intel_lspcon.h
new file mode 100644
index 000000000000..37cfddf8a9c5
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_lspcon.h
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2019 Intel Corporation
+ */
+
+#ifndef __INTEL_LSPCON_H__
+#define __INTEL_LSPCON_H__
+
+#include <linux/types.h>
+
+struct drm_connector;
+struct drm_connector_state;
+struct intel_crtc_state;
+struct intel_digital_port;
+struct intel_encoder;
+struct intel_lspcon;
+
+bool lspcon_init(struct intel_digital_port *intel_dig_port);
+void lspcon_resume(struct intel_lspcon *lspcon);
+void lspcon_wait_pcon_mode(struct intel_lspcon *lspcon);
+void lspcon_write_infoframe(struct intel_encoder *encoder,
+			    const struct intel_crtc_state *crtc_state,
+			    unsigned int type,
+			    const void *buf, ssize_t len);
+void lspcon_read_infoframe(struct intel_encoder *encoder,
+			   const struct intel_crtc_state *crtc_state,
+			   unsigned int type,
+			   void *frame, ssize_t len);
+void lspcon_set_infoframes(struct intel_encoder *encoder,
+			   bool enable,
+			   const struct intel_crtc_state *crtc_state,
+			   const struct drm_connector_state *conn_state);
+u32 lspcon_infoframes_enabled(struct intel_encoder *encoder,
+			      const struct intel_crtc_state *pipe_config);
+void lspcon_ycbcr420_config(struct drm_connector *connector,
+			    struct intel_crtc_state *crtc_state);
+
+#endif /* __INTEL_LSPCON_H__ */
diff --git a/drivers/gpu/drm/i915/intel_lvds.c b/drivers/gpu/drm/i915/intel_lvds.c
index b4aa49768e90..51d1d59c1619 100644
--- a/drivers/gpu/drm/i915/intel_lvds.c
+++ b/drivers/gpu/drm/i915/intel_lvds.c
@@ -28,17 +28,22 @@
  */
 
 #include <acpi/button.h>
+#include <linux/acpi.h>
 #include <linux/dmi.h>
 #include <linux/i2c.h>
 #include <linux/slab.h>
 #include <linux/vga_switcheroo.h>
+
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_edid.h>
-#include "intel_drv.h"
 #include <drm/i915_drm.h>
+
 #include "i915_drv.h"
-#include <linux/acpi.h>
+#include "intel_connector.h"
+#include "intel_drv.h"
+#include "intel_lvds.h"
+#include "intel_panel.h"
 
 /* Private structure for the integrated LVDS support */
 struct intel_lvds_pps {
@@ -152,24 +157,17 @@ static void intel_lvds_pps_get_hw_state(struct drm_i915_private *dev_priv,
 	pps->powerdown_on_reset = I915_READ(PP_CONTROL(0)) & PANEL_POWER_RESET;
 
 	val = I915_READ(PP_ON_DELAYS(0));
-	pps->port = (val & PANEL_PORT_SELECT_MASK) >>
-		    PANEL_PORT_SELECT_SHIFT;
-	pps->t1_t2 = (val & PANEL_POWER_UP_DELAY_MASK) >>
-		     PANEL_POWER_UP_DELAY_SHIFT;
-	pps->t5 = (val & PANEL_LIGHT_ON_DELAY_MASK) >>
-		  PANEL_LIGHT_ON_DELAY_SHIFT;
+	pps->port = REG_FIELD_GET(PANEL_PORT_SELECT_MASK, val);
+	pps->t1_t2 = REG_FIELD_GET(PANEL_POWER_UP_DELAY_MASK, val);
+	pps->t5 = REG_FIELD_GET(PANEL_LIGHT_ON_DELAY_MASK, val);
 
 	val = I915_READ(PP_OFF_DELAYS(0));
-	pps->t3 = (val & PANEL_POWER_DOWN_DELAY_MASK) >>
-		  PANEL_POWER_DOWN_DELAY_SHIFT;
-	pps->tx = (val & PANEL_LIGHT_OFF_DELAY_MASK) >>
-		  PANEL_LIGHT_OFF_DELAY_SHIFT;
+	pps->t3 = REG_FIELD_GET(PANEL_POWER_DOWN_DELAY_MASK, val);
+	pps->tx = REG_FIELD_GET(PANEL_LIGHT_OFF_DELAY_MASK, val);
 
 	val = I915_READ(PP_DIVISOR(0));
-	pps->divider = (val & PP_REFERENCE_DIVIDER_MASK) >>
-		       PP_REFERENCE_DIVIDER_SHIFT;
-	val = (val & PANEL_POWER_CYCLE_DELAY_MASK) >>
-	      PANEL_POWER_CYCLE_DELAY_SHIFT;
+	pps->divider = REG_FIELD_GET(PP_REFERENCE_DIVIDER_MASK, val);
+	val = REG_FIELD_GET(PANEL_POWER_CYCLE_DELAY_MASK, val);
 	/*
 	 * Remove the BSpec specified +1 (100ms) offset that accounts for a
 	 * too short power-cycle delay due to the asynchronous programming of
@@ -209,16 +207,19 @@ static void intel_lvds_pps_init_hw(struct drm_i915_private *dev_priv,
 		val |= PANEL_POWER_RESET;
 	I915_WRITE(PP_CONTROL(0), val);
 
-	I915_WRITE(PP_ON_DELAYS(0), (pps->port << PANEL_PORT_SELECT_SHIFT) |
-				    (pps->t1_t2 << PANEL_POWER_UP_DELAY_SHIFT) |
-				    (pps->t5 << PANEL_LIGHT_ON_DELAY_SHIFT));
-	I915_WRITE(PP_OFF_DELAYS(0), (pps->t3 << PANEL_POWER_DOWN_DELAY_SHIFT) |
-				     (pps->tx << PANEL_LIGHT_OFF_DELAY_SHIFT));
+	I915_WRITE(PP_ON_DELAYS(0),
+		   REG_FIELD_PREP(PANEL_PORT_SELECT_MASK, pps->port) |
+		   REG_FIELD_PREP(PANEL_POWER_UP_DELAY_MASK, pps->t1_t2) |
+		   REG_FIELD_PREP(PANEL_LIGHT_ON_DELAY_MASK, pps->t5));
+
+	I915_WRITE(PP_OFF_DELAYS(0),
+		   REG_FIELD_PREP(PANEL_POWER_DOWN_DELAY_MASK, pps->t3) |
+		   REG_FIELD_PREP(PANEL_LIGHT_OFF_DELAY_MASK, pps->tx));
 
-	val = pps->divider << PP_REFERENCE_DIVIDER_SHIFT;
-	val |= (DIV_ROUND_UP(pps->t4, 1000) + 1) <<
-	       PANEL_POWER_CYCLE_DELAY_SHIFT;
-	I915_WRITE(PP_DIVISOR(0), val);
+	I915_WRITE(PP_DIVISOR(0),
+		   REG_FIELD_PREP(PP_REFERENCE_DIVIDER_MASK, pps->divider) |
+		   REG_FIELD_PREP(PANEL_POWER_CYCLE_DELAY_MASK,
+				  DIV_ROUND_UP(pps->t4, 1000) + 1));
 }
 
 static void intel_pre_enable_lvds(struct intel_encoder *encoder,
@@ -315,7 +316,8 @@ static void intel_enable_lvds(struct intel_encoder *encoder,
 	I915_WRITE(PP_CONTROL(0), I915_READ(PP_CONTROL(0)) | PANEL_POWER_ON);
 	POSTING_READ(lvds_encoder->reg);
 
-	if (intel_wait_for_register(dev_priv, PP_STATUS(0), PP_ON, PP_ON, 5000))
+	if (intel_wait_for_register(&dev_priv->uncore,
+				    PP_STATUS(0), PP_ON, PP_ON, 5000))
 		DRM_ERROR("timed out waiting for panel to power on\n");
 
 	intel_panel_enable_backlight(pipe_config, conn_state);
@@ -329,7 +331,8 @@ static void intel_disable_lvds(struct intel_encoder *encoder,
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
 	I915_WRITE(PP_CONTROL(0), I915_READ(PP_CONTROL(0)) & ~PANEL_POWER_ON);
-	if (intel_wait_for_register(dev_priv, PP_STATUS(0), PP_ON, 0, 1000))
+	if (intel_wait_for_register(&dev_priv->uncore,
+				    PP_STATUS(0), PP_ON, 0, 1000))
 		DRM_ERROR("timed out waiting for panel to power off\n");
 
 	I915_WRITE(lvds_encoder->reg, I915_READ(lvds_encoder->reg) & ~LVDS_PORT_EN);
@@ -746,20 +749,21 @@ static const struct dmi_system_id intel_dual_link_lvds[] = {
 	{ }	/* terminating entry */
 };
 
-struct intel_encoder *intel_get_lvds_encoder(struct drm_device *dev)
+struct intel_encoder *intel_get_lvds_encoder(struct drm_i915_private *dev_priv)
 {
-	struct intel_encoder *intel_encoder;
+	struct intel_encoder *encoder;
 
-	for_each_intel_encoder(dev, intel_encoder)
-		if (intel_encoder->type == INTEL_OUTPUT_LVDS)
-			return intel_encoder;
+	for_each_intel_encoder(&dev_priv->drm, encoder) {
+		if (encoder->type == INTEL_OUTPUT_LVDS)
+			return encoder;
+	}
 
 	return NULL;
 }
 
-bool intel_is_dual_link_lvds(struct drm_device *dev)
+bool intel_is_dual_link_lvds(struct drm_i915_private *dev_priv)
 {
-	struct intel_encoder *encoder = intel_get_lvds_encoder(dev);
+	struct intel_encoder *encoder = intel_get_lvds_encoder(dev_priv);
 
 	return encoder && to_lvds_encoder(&encoder->base)->is_dual_link;
 }
@@ -813,7 +817,6 @@ void intel_lvds_init(struct drm_i915_private *dev_priv)
 	struct intel_connector *intel_connector;
 	struct drm_connector *connector;
 	struct drm_encoder *encoder;
-	struct drm_display_mode *scan; /* *modes, *bios_mode; */
 	struct drm_display_mode *fixed_mode = NULL;
 	struct drm_display_mode *downclock_mode = NULL;
 	struct edid *edid;
@@ -952,30 +955,14 @@ void intel_lvds_init(struct drm_i915_private *dev_priv)
 	}
 	intel_connector->edid = edid;
 
-	list_for_each_entry(scan, &connector->probed_modes, head) {
-		if (scan->type & DRM_MODE_TYPE_PREFERRED) {
-			DRM_DEBUG_KMS("using preferred mode from EDID: ");
-			drm_mode_debug_printmodeline(scan);
-
-			fixed_mode = drm_mode_duplicate(dev, scan);
-			if (fixed_mode)
-				goto out;
-		}
-	}
+	fixed_mode = intel_panel_edid_fixed_mode(intel_connector);
+	if (fixed_mode)
+		goto out;
 
 	/* Failed to get EDID, what about VBT? */
-	if (dev_priv->vbt.lfp_lvds_vbt_mode) {
-		DRM_DEBUG_KMS("using mode from VBT: ");
-		drm_mode_debug_printmodeline(dev_priv->vbt.lfp_lvds_vbt_mode);
-
-		fixed_mode = drm_mode_duplicate(dev, dev_priv->vbt.lfp_lvds_vbt_mode);
-		if (fixed_mode) {
-			fixed_mode->type |= DRM_MODE_TYPE_PREFERRED;
-			connector->display_info.width_mm = fixed_mode->width_mm;
-			connector->display_info.height_mm = fixed_mode->height_mm;
-			goto out;
-		}
-	}
+	fixed_mode = intel_panel_vbt_fixed_mode(intel_connector);
+	if (fixed_mode)
+		goto out;
 
 	/*
 	 * If we didn't get EDID, try checking if the panel is already turned
diff --git a/drivers/gpu/drm/i915/intel_lvds.h b/drivers/gpu/drm/i915/intel_lvds.h
new file mode 100644
index 000000000000..bc9c8b84ba2f
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_lvds.h
@@ -0,0 +1,22 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2019 Intel Corporation
+ */
+
+#ifndef __INTEL_LVDS_H__
+#define __INTEL_LVDS_H__
+
+#include <linux/types.h>
+
+#include "i915_reg.h"
+
+enum pipe;
+struct drm_i915_private;
+
+bool intel_lvds_port_enabled(struct drm_i915_private *dev_priv,
+			     i915_reg_t lvds_reg, enum pipe *pipe);
+void intel_lvds_init(struct drm_i915_private *dev_priv);
+struct intel_encoder *intel_get_lvds_encoder(struct drm_i915_private *dev_priv);
+bool intel_is_dual_link_lvds(struct drm_i915_private *dev_priv);
+
+#endif /* __INTEL_LVDS_H__ */
diff --git a/drivers/gpu/drm/i915/intel_mocs.c b/drivers/gpu/drm/i915/intel_mocs.c
index 331e7a678fb7..274ba78500c0 100644
--- a/drivers/gpu/drm/i915/intel_mocs.c
+++ b/drivers/gpu/drm/i915/intel_mocs.c
@@ -252,7 +252,7 @@ static bool get_mocs_settings(struct drm_i915_private *dev_priv,
 {
 	bool result = false;
 
-	if (IS_ICELAKE(dev_priv)) {
+	if (INTEL_GEN(dev_priv) >= 11) {
 		table->size  = ARRAY_SIZE(icelake_mocs_table);
 		table->table = icelake_mocs_table;
 		table->n_entries = GEN11_NUM_MOCS_ENTRIES;
@@ -288,17 +288,17 @@ static bool get_mocs_settings(struct drm_i915_private *dev_priv,
 static i915_reg_t mocs_register(enum intel_engine_id engine_id, int index)
 {
 	switch (engine_id) {
-	case RCS:
+	case RCS0:
 		return GEN9_GFX_MOCS(index);
-	case VCS:
+	case VCS0:
 		return GEN9_MFX0_MOCS(index);
-	case BCS:
+	case BCS0:
 		return GEN9_BLT_MOCS(index);
-	case VECS:
+	case VECS0:
 		return GEN9_VEBOX_MOCS(index);
-	case VCS2:
+	case VCS1:
 		return GEN9_MFX1_MOCS(index);
-	case VCS3:
+	case VCS2:
 		return GEN11_MFX2_MOCS(index);
 	default:
 		MISSING_CASE(engine_id);
diff --git a/drivers/gpu/drm/i915/intel_opregion.c b/drivers/gpu/drm/i915/intel_opregion.c
index 5e00ee9270b5..8fa1159d097f 100644
--- a/drivers/gpu/drm/i915/intel_opregion.c
+++ b/drivers/gpu/drm/i915/intel_opregion.c
@@ -32,9 +32,10 @@
 
 #include <drm/i915_drm.h>
 
-#include "intel_opregion.h"
 #include "i915_drv.h"
 #include "intel_drv.h"
+#include "intel_opregion.h"
+#include "intel_panel.h"
 
 #define OPREGION_HEADER_OFFSET 0
 #define OPREGION_ACPI_OFFSET   0x100
diff --git a/drivers/gpu/drm/i915/intel_overlay.c b/drivers/gpu/drm/i915/intel_overlay.c
index c0df1dbb0069..eb317759b5d3 100644
--- a/drivers/gpu/drm/i915/intel_overlay.c
+++ b/drivers/gpu/drm/i915/intel_overlay.c
@@ -236,7 +236,7 @@ static int intel_overlay_do_wait_request(struct intel_overlay *overlay,
 static struct i915_request *alloc_request(struct intel_overlay *overlay)
 {
 	struct drm_i915_private *dev_priv = overlay->i915;
-	struct intel_engine_cs *engine = dev_priv->engine[RCS];
+	struct intel_engine_cs *engine = dev_priv->engine[RCS0];
 
 	return i915_request_alloc(engine, dev_priv->kernel_context);
 }
@@ -446,7 +446,7 @@ static int intel_overlay_release_old_vid(struct intel_overlay *overlay)
 	if (!overlay->old_vma)
 		return 0;
 
-	if (I915_READ(ISR) & I915_OVERLAY_PLANE_FLIP_PENDING_INTERRUPT) {
+	if (I915_READ(GEN2_ISR) & I915_OVERLAY_PLANE_FLIP_PENDING_INTERRUPT) {
 		/* synchronous slowpath */
 		struct i915_request *rq;
 
@@ -1430,7 +1430,7 @@ intel_overlay_capture_error_state(struct drm_i915_private *dev_priv)
 		return NULL;
 
 	error->dovsta = I915_READ(DOVSTA);
-	error->isr = I915_READ(ISR);
+	error->isr = I915_READ(GEN2_ISR);
 	error->base = overlay->flip_addr;
 
 	memcpy_fromio(&error->regs, overlay->regs, sizeof(error->regs));
diff --git a/drivers/gpu/drm/i915/intel_panel.c b/drivers/gpu/drm/i915/intel_panel.c
index beca98d2b035..4ab4ce6569e7 100644
--- a/drivers/gpu/drm/i915/intel_panel.c
+++ b/drivers/gpu/drm/i915/intel_panel.c
@@ -33,7 +33,10 @@
 #include <linux/kernel.h>
 #include <linux/moduleparam.h>
 #include <linux/pwm.h>
+
+#include "intel_connector.h"
 #include "intel_drv.h"
+#include "intel_panel.h"
 
 #define CRC_PMIC_PWM_PERIOD_NS	21333
 
@@ -46,27 +49,26 @@ intel_fixed_panel_mode(const struct drm_display_mode *fixed_mode,
 	drm_mode_set_crtcinfo(adjusted_mode, 0);
 }
 
-/**
- * intel_find_panel_downclock - find the reduced downclock for LVDS in EDID
- * @dev_priv: i915 device instance
- * @fixed_mode : panel native mode
- * @connector: LVDS/eDP connector
- *
- * Return downclock_avail
- * Find the reduced downclock for LVDS/eDP in EDID.
- */
-struct drm_display_mode *
-intel_find_panel_downclock(struct drm_i915_private *dev_priv,
-			struct drm_display_mode *fixed_mode,
-			struct drm_connector *connector)
+static bool is_downclock_mode(const struct drm_display_mode *downclock_mode,
+			      const struct drm_display_mode *fixed_mode)
 {
-	struct drm_display_mode *scan, *tmp_mode;
-	int temp_downclock;
+	return drm_mode_match(downclock_mode, fixed_mode,
+			      DRM_MODE_MATCH_TIMINGS |
+			      DRM_MODE_MATCH_FLAGS |
+			      DRM_MODE_MATCH_3D_FLAGS) &&
+		downclock_mode->clock < fixed_mode->clock;
+}
 
-	temp_downclock = fixed_mode->clock;
-	tmp_mode = NULL;
+struct drm_display_mode *
+intel_panel_edid_downclock_mode(struct intel_connector *connector,
+				const struct drm_display_mode *fixed_mode)
+{
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+	const struct drm_display_mode *scan, *best_mode = NULL;
+	struct drm_display_mode *downclock_mode;
+	int best_clock = fixed_mode->clock;
 
-	list_for_each_entry(scan, &connector->probed_modes, head) {
+	list_for_each_entry(scan, &connector->base.probed_modes, head) {
 		/*
 		 * If one mode has the same resolution with the fixed_panel
 		 * mode while they have the different refresh rate, it means
@@ -74,29 +76,98 @@ intel_find_panel_downclock(struct drm_i915_private *dev_priv,
 		 * case we can set the different FPx0/1 to dynamically select
 		 * between low and high frequency.
 		 */
-		if (scan->hdisplay == fixed_mode->hdisplay &&
-		    scan->hsync_start == fixed_mode->hsync_start &&
-		    scan->hsync_end == fixed_mode->hsync_end &&
-		    scan->htotal == fixed_mode->htotal &&
-		    scan->vdisplay == fixed_mode->vdisplay &&
-		    scan->vsync_start == fixed_mode->vsync_start &&
-		    scan->vsync_end == fixed_mode->vsync_end &&
-		    scan->vtotal == fixed_mode->vtotal) {
-			if (scan->clock < temp_downclock) {
-				/*
-				 * The downclock is already found. But we
-				 * expect to find the lower downclock.
-				 */
-				temp_downclock = scan->clock;
-				tmp_mode = scan;
-			}
+		if (is_downclock_mode(scan, fixed_mode) &&
+		    scan->clock < best_clock) {
+			/*
+			 * The downclock is already found. But we
+			 * expect to find the lower downclock.
+			 */
+			best_clock = scan->clock;
+			best_mode = scan;
 		}
 	}
 
-	if (temp_downclock < fixed_mode->clock)
-		return drm_mode_duplicate(&dev_priv->drm, tmp_mode);
-	else
+	if (!best_mode)
+		return NULL;
+
+	downclock_mode = drm_mode_duplicate(&dev_priv->drm, best_mode);
+	if (!downclock_mode)
+		return NULL;
+
+	DRM_DEBUG_KMS("[CONNECTOR:%d:%s] using downclock mode from EDID: ",
+		      connector->base.base.id, connector->base.name);
+	drm_mode_debug_printmodeline(downclock_mode);
+
+	return downclock_mode;
+}
+
+struct drm_display_mode *
+intel_panel_edid_fixed_mode(struct intel_connector *connector)
+{
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+	const struct drm_display_mode *scan;
+	struct drm_display_mode *fixed_mode;
+
+	if (list_empty(&connector->base.probed_modes))
+		return NULL;
+
+	/* prefer fixed mode from EDID if available */
+	list_for_each_entry(scan, &connector->base.probed_modes, head) {
+		if ((scan->type & DRM_MODE_TYPE_PREFERRED) == 0)
+			continue;
+
+		fixed_mode = drm_mode_duplicate(&dev_priv->drm, scan);
+		if (!fixed_mode)
+			return NULL;
+
+		DRM_DEBUG_KMS("[CONNECTOR:%d:%s] using preferred mode from EDID: ",
+			      connector->base.base.id, connector->base.name);
+		drm_mode_debug_printmodeline(fixed_mode);
+
+		return fixed_mode;
+	}
+
+	scan = list_first_entry(&connector->base.probed_modes,
+				typeof(*scan), head);
+
+	fixed_mode = drm_mode_duplicate(&dev_priv->drm, scan);
+	if (!fixed_mode)
 		return NULL;
+
+	fixed_mode->type |= DRM_MODE_TYPE_PREFERRED;
+
+	DRM_DEBUG_KMS("[CONNECTOR:%d:%s] using first mode from EDID: ",
+		      connector->base.base.id, connector->base.name);
+	drm_mode_debug_printmodeline(fixed_mode);
+
+	return fixed_mode;
+}
+
+struct drm_display_mode *
+intel_panel_vbt_fixed_mode(struct intel_connector *connector)
+{
+	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+	struct drm_display_info *info = &connector->base.display_info;
+	struct drm_display_mode *fixed_mode;
+
+	if (!dev_priv->vbt.lfp_lvds_vbt_mode)
+		return NULL;
+
+	fixed_mode = drm_mode_duplicate(&dev_priv->drm,
+					dev_priv->vbt.lfp_lvds_vbt_mode);
+	if (!fixed_mode)
+		return NULL;
+
+	fixed_mode->type |= DRM_MODE_TYPE_PREFERRED;
+
+	DRM_DEBUG_KMS("[CONNECTOR:%d:%s] using mode from VBT: ",
+		      connector->base.base.id, connector->base.name);
+	drm_mode_debug_printmodeline(fixed_mode);
+
+	info->width_mm = fixed_mode->width_mm;
+	info->height_mm = fixed_mode->height_mm;
+
+	return fixed_mode;
 }
 
 /* adjusted_mode has been preset to be the panel's fixed mode */
@@ -1894,15 +1965,14 @@ intel_panel_init_backlight_funcs(struct intel_panel *panel)
 		panel->backlight.set = bxt_set_backlight;
 		panel->backlight.get = bxt_get_backlight;
 		panel->backlight.hz_to_pwm = bxt_hz_to_pwm;
-	} else if (HAS_PCH_CNP(dev_priv) || HAS_PCH_ICP(dev_priv)) {
+	} else if (INTEL_PCH_TYPE(dev_priv) >= PCH_CNP) {
 		panel->backlight.setup = cnp_setup_backlight;
 		panel->backlight.enable = cnp_enable_backlight;
 		panel->backlight.disable = cnp_disable_backlight;
 		panel->backlight.set = bxt_set_backlight;
 		panel->backlight.get = bxt_get_backlight;
 		panel->backlight.hz_to_pwm = cnp_hz_to_pwm;
-	} else if (HAS_PCH_LPT(dev_priv) || HAS_PCH_SPT(dev_priv) ||
-		   HAS_PCH_KBP(dev_priv)) {
+	} else if (INTEL_PCH_TYPE(dev_priv) >= PCH_LPT) {
 		panel->backlight.setup = lpt_setup_backlight;
 		panel->backlight.enable = lpt_enable_backlight;
 		panel->backlight.disable = lpt_disable_backlight;
diff --git a/drivers/gpu/drm/i915/intel_panel.h b/drivers/gpu/drm/i915/intel_panel.h
new file mode 100644
index 000000000000..cedeea443336
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_panel.h
@@ -0,0 +1,65 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2019 Intel Corporation
+ */
+
+#ifndef __INTEL_PANEL_H__
+#define __INTEL_PANEL_H__
+
+#include <linux/types.h>
+
+#include "intel_display.h"
+
+struct drm_connector;
+struct drm_connector_state;
+struct drm_display_mode;
+struct intel_connector;
+struct intel_crtc;
+struct intel_crtc_state;
+struct intel_encoder;
+struct intel_panel;
+
+int intel_panel_init(struct intel_panel *panel,
+		     struct drm_display_mode *fixed_mode,
+		     struct drm_display_mode *downclock_mode);
+void intel_panel_fini(struct intel_panel *panel);
+void intel_fixed_panel_mode(const struct drm_display_mode *fixed_mode,
+			    struct drm_display_mode *adjusted_mode);
+void intel_pch_panel_fitting(struct intel_crtc *crtc,
+			     struct intel_crtc_state *pipe_config,
+			     int fitting_mode);
+void intel_gmch_panel_fitting(struct intel_crtc *crtc,
+			      struct intel_crtc_state *pipe_config,
+			      int fitting_mode);
+void intel_panel_set_backlight_acpi(const struct drm_connector_state *conn_state,
+				    u32 level, u32 max);
+int intel_panel_setup_backlight(struct drm_connector *connector,
+				enum pipe pipe);
+void intel_panel_enable_backlight(const struct intel_crtc_state *crtc_state,
+				  const struct drm_connector_state *conn_state);
+void intel_panel_update_backlight(struct intel_encoder *encoder,
+				  const struct intel_crtc_state *crtc_state,
+				  const struct drm_connector_state *conn_state);
+void intel_panel_disable_backlight(const struct drm_connector_state *old_conn_state);
+struct drm_display_mode *
+intel_panel_edid_downclock_mode(struct intel_connector *connector,
+				const struct drm_display_mode *fixed_mode);
+struct drm_display_mode *
+intel_panel_edid_fixed_mode(struct intel_connector *connector);
+struct drm_display_mode *
+intel_panel_vbt_fixed_mode(struct intel_connector *connector);
+
+#if IS_ENABLED(CONFIG_BACKLIGHT_CLASS_DEVICE)
+int intel_backlight_device_register(struct intel_connector *connector);
+void intel_backlight_device_unregister(struct intel_connector *connector);
+#else /* CONFIG_BACKLIGHT_CLASS_DEVICE */
+static inline int intel_backlight_device_register(struct intel_connector *connector)
+{
+	return 0;
+}
+static inline void intel_backlight_device_unregister(struct intel_connector *connector)
+{
+}
+#endif /* CONFIG_BACKLIGHT_CLASS_DEVICE */
+
+#endif /* __INTEL_PANEL_H__ */
diff --git a/drivers/gpu/drm/i915/intel_pipe_crc.c b/drivers/gpu/drm/i915/intel_pipe_crc.c
index a8554dc4f196..e94b5b1bc1b7 100644
--- a/drivers/gpu/drm/i915/intel_pipe_crc.c
+++ b/drivers/gpu/drm/i915/intel_pipe_crc.c
@@ -24,23 +24,29 @@
  *
  */
 
-#include <linux/seq_file.h>
 #include <linux/circ_buf.h>
 #include <linux/ctype.h>
 #include <linux/debugfs.h>
+#include <linux/seq_file.h>
+
 #include "intel_drv.h"
+#include "intel_pipe_crc.h"
 
 static const char * const pipe_crc_sources[] = {
-	"none",
-	"plane1",
-	"plane2",
-	"pf",
-	"pipe",
-	"TV",
-	"DP-B",
-	"DP-C",
-	"DP-D",
-	"auto",
+	[INTEL_PIPE_CRC_SOURCE_NONE] = "none",
+	[INTEL_PIPE_CRC_SOURCE_PLANE1] = "plane1",
+	[INTEL_PIPE_CRC_SOURCE_PLANE2] = "plane2",
+	[INTEL_PIPE_CRC_SOURCE_PLANE3] = "plane3",
+	[INTEL_PIPE_CRC_SOURCE_PLANE4] = "plane4",
+	[INTEL_PIPE_CRC_SOURCE_PLANE5] = "plane5",
+	[INTEL_PIPE_CRC_SOURCE_PLANE6] = "plane6",
+	[INTEL_PIPE_CRC_SOURCE_PLANE7] = "plane7",
+	[INTEL_PIPE_CRC_SOURCE_PIPE] = "pipe",
+	[INTEL_PIPE_CRC_SOURCE_TV] = "TV",
+	[INTEL_PIPE_CRC_SOURCE_DP_B] = "DP-B",
+	[INTEL_PIPE_CRC_SOURCE_DP_C] = "DP-C",
+	[INTEL_PIPE_CRC_SOURCE_DP_D] = "DP-D",
+	[INTEL_PIPE_CRC_SOURCE_AUTO] = "auto",
 };
 
 static int i8xx_pipe_crc_ctl_reg(enum intel_pipe_crc_source *source,
@@ -192,8 +198,6 @@ static int i9xx_pipe_crc_ctl_reg(struct drm_i915_private *dev_priv,
 				 enum intel_pipe_crc_source *source,
 				 u32 *val)
 {
-	bool need_stable_symbols = false;
-
 	if (*source == INTEL_PIPE_CRC_SOURCE_AUTO) {
 		int ret = i9xx_pipe_crc_auto_source(dev_priv, pipe, source);
 		if (ret)
@@ -209,56 +213,23 @@ static int i9xx_pipe_crc_ctl_reg(struct drm_i915_private *dev_priv,
 			return -EINVAL;
 		*val = PIPE_CRC_ENABLE | PIPE_CRC_SOURCE_TV_PRE;
 		break;
-	case INTEL_PIPE_CRC_SOURCE_DP_B:
-		if (!IS_G4X(dev_priv))
-			return -EINVAL;
-		*val = PIPE_CRC_ENABLE | PIPE_CRC_SOURCE_DP_B_G4X;
-		need_stable_symbols = true;
-		break;
-	case INTEL_PIPE_CRC_SOURCE_DP_C:
-		if (!IS_G4X(dev_priv))
-			return -EINVAL;
-		*val = PIPE_CRC_ENABLE | PIPE_CRC_SOURCE_DP_C_G4X;
-		need_stable_symbols = true;
-		break;
-	case INTEL_PIPE_CRC_SOURCE_DP_D:
-		if (!IS_G4X(dev_priv))
-			return -EINVAL;
-		*val = PIPE_CRC_ENABLE | PIPE_CRC_SOURCE_DP_D_G4X;
-		need_stable_symbols = true;
-		break;
 	case INTEL_PIPE_CRC_SOURCE_NONE:
 		*val = 0;
 		break;
 	default:
+		/*
+		 * The DP CRC source doesn't work on g4x.
+		 * It can be made to work to some degree by selecting
+		 * the correct CRC source before the port is enabled,
+		 * and not touching the CRC source bits again until
+		 * the port is disabled. But even then the bits
+		 * eventually get stuck and a reboot is needed to get
+		 * working CRCs on the pipe again. Let's simply
+		 * refuse to use DP CRCs on g4x.
+		 */
 		return -EINVAL;
 	}
 
-	/*
-	 * When the pipe CRC tap point is after the transcoders we need
-	 * to tweak symbol-level features to produce a deterministic series of
-	 * symbols for a given frame. We need to reset those features only once
-	 * a frame (instead of every nth symbol):
-	 *   - DC-balance: used to ensure a better clock recovery from the data
-	 *     link (SDVO)
-	 *   - DisplayPort scrambling: used for EMI reduction
-	 */
-	if (need_stable_symbols) {
-		u32 tmp = I915_READ(PORT_DFT2_G4X);
-
-		WARN_ON(!IS_G4X(dev_priv));
-
-		I915_WRITE(PORT_DFT_I9XX,
-			   I915_READ(PORT_DFT_I9XX) | DC_BALANCE_RESET);
-
-		if (pipe == PIPE_A)
-			tmp |= PIPE_A_SCRAMBLE_RESET;
-		else
-			tmp |= PIPE_B_SCRAMBLE_RESET;
-
-		I915_WRITE(PORT_DFT2_G4X, tmp);
-	}
-
 	return 0;
 }
 
@@ -283,24 +254,6 @@ static void vlv_undo_pipe_scramble_reset(struct drm_i915_private *dev_priv,
 	if (!(tmp & PIPE_SCRAMBLE_RESET_MASK))
 		tmp &= ~DC_BALANCE_RESET_VLV;
 	I915_WRITE(PORT_DFT2_G4X, tmp);
-
-}
-
-static void g4x_undo_pipe_scramble_reset(struct drm_i915_private *dev_priv,
-					 enum pipe pipe)
-{
-	u32 tmp = I915_READ(PORT_DFT2_G4X);
-
-	if (pipe == PIPE_A)
-		tmp &= ~PIPE_A_SCRAMBLE_RESET;
-	else
-		tmp &= ~PIPE_B_SCRAMBLE_RESET;
-	I915_WRITE(PORT_DFT2_G4X, tmp);
-
-	if (!(tmp & PIPE_SCRAMBLE_RESET_MASK)) {
-		I915_WRITE(PORT_DFT_I9XX,
-			   I915_READ(PORT_DFT_I9XX) & ~DC_BALANCE_RESET);
-	}
 }
 
 static int ilk_pipe_crc_ctl_reg(enum intel_pipe_crc_source *source,
@@ -329,19 +282,18 @@ static int ilk_pipe_crc_ctl_reg(enum intel_pipe_crc_source *source,
 	return 0;
 }
 
-static void hsw_pipe_A_crc_wa(struct drm_i915_private *dev_priv,
-			      bool enable)
+static void
+intel_crtc_crc_setup_workarounds(struct intel_crtc *crtc, bool enable)
 {
-	struct drm_device *dev = &dev_priv->drm;
-	struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_A);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	struct intel_crtc_state *pipe_config;
 	struct drm_atomic_state *state;
 	struct drm_modeset_acquire_ctx ctx;
-	int ret = 0;
+	int ret;
 
 	drm_modeset_acquire_init(&ctx, 0);
 
-	state = drm_atomic_state_alloc(dev);
+	state = drm_atomic_state_alloc(&dev_priv->drm);
 	if (!state) {
 		ret = -ENOMEM;
 		goto unlock;
@@ -356,17 +308,10 @@ retry:
 		goto put_state;
 	}
 
-	if (HAS_IPS(dev_priv)) {
-		/*
-		 * When IPS gets enabled, the pipe CRC changes. Since IPS gets
-		 * enabled and disabled dynamically based on package C states,
-		 * user space can't make reliable use of the CRCs, so let's just
-		 * completely disable it.
-		 */
-		pipe_config->ips_force_disable = enable;
-	}
+	pipe_config->base.mode_changed = pipe_config->has_psr;
+	pipe_config->crc_enabled = enable;
 
-	if (IS_HASWELL(dev_priv)) {
+	if (IS_HASWELL(dev_priv) && crtc->pipe == PIPE_A) {
 		pipe_config->pch_pfit.force_thru = enable;
 		if (pipe_config->cpu_transcoder == TRANSCODER_EDP &&
 		    pipe_config->pch_pfit.enabled != enable)
@@ -392,11 +337,10 @@ unlock:
 static int ivb_pipe_crc_ctl_reg(struct drm_i915_private *dev_priv,
 				enum pipe pipe,
 				enum intel_pipe_crc_source *source,
-				u32 *val,
-				bool set_wa)
+				u32 *val)
 {
 	if (*source == INTEL_PIPE_CRC_SOURCE_AUTO)
-		*source = INTEL_PIPE_CRC_SOURCE_PF;
+		*source = INTEL_PIPE_CRC_SOURCE_PIPE;
 
 	switch (*source) {
 	case INTEL_PIPE_CRC_SOURCE_PLANE1:
@@ -405,11 +349,7 @@ static int ivb_pipe_crc_ctl_reg(struct drm_i915_private *dev_priv,
 	case INTEL_PIPE_CRC_SOURCE_PLANE2:
 		*val = PIPE_CRC_ENABLE | PIPE_CRC_SOURCE_SPRITE_IVB;
 		break;
-	case INTEL_PIPE_CRC_SOURCE_PF:
-		if (set_wa && (IS_HASWELL(dev_priv) ||
-		     IS_BROADWELL(dev_priv)) && pipe == PIPE_A)
-			hsw_pipe_A_crc_wa(dev_priv, true);
-
+	case INTEL_PIPE_CRC_SOURCE_PIPE:
 		*val = PIPE_CRC_ENABLE | PIPE_CRC_SOURCE_PF_IVB;
 		break;
 	case INTEL_PIPE_CRC_SOURCE_NONE:
@@ -422,10 +362,52 @@ static int ivb_pipe_crc_ctl_reg(struct drm_i915_private *dev_priv,
 	return 0;
 }
 
+static int skl_pipe_crc_ctl_reg(struct drm_i915_private *dev_priv,
+				enum pipe pipe,
+				enum intel_pipe_crc_source *source,
+				u32 *val)
+{
+	if (*source == INTEL_PIPE_CRC_SOURCE_AUTO)
+		*source = INTEL_PIPE_CRC_SOURCE_PIPE;
+
+	switch (*source) {
+	case INTEL_PIPE_CRC_SOURCE_PLANE1:
+		*val = PIPE_CRC_ENABLE | PIPE_CRC_SOURCE_PLANE_1_SKL;
+		break;
+	case INTEL_PIPE_CRC_SOURCE_PLANE2:
+		*val = PIPE_CRC_ENABLE | PIPE_CRC_SOURCE_PLANE_2_SKL;
+		break;
+	case INTEL_PIPE_CRC_SOURCE_PLANE3:
+		*val = PIPE_CRC_ENABLE | PIPE_CRC_SOURCE_PLANE_3_SKL;
+		break;
+	case INTEL_PIPE_CRC_SOURCE_PLANE4:
+		*val = PIPE_CRC_ENABLE | PIPE_CRC_SOURCE_PLANE_4_SKL;
+		break;
+	case INTEL_PIPE_CRC_SOURCE_PLANE5:
+		*val = PIPE_CRC_ENABLE | PIPE_CRC_SOURCE_PLANE_5_SKL;
+		break;
+	case INTEL_PIPE_CRC_SOURCE_PLANE6:
+		*val = PIPE_CRC_ENABLE | PIPE_CRC_SOURCE_PLANE_6_SKL;
+		break;
+	case INTEL_PIPE_CRC_SOURCE_PLANE7:
+		*val = PIPE_CRC_ENABLE | PIPE_CRC_SOURCE_PLANE_7_SKL;
+		break;
+	case INTEL_PIPE_CRC_SOURCE_PIPE:
+		*val = PIPE_CRC_ENABLE | PIPE_CRC_SOURCE_DMUX_SKL;
+		break;
+	case INTEL_PIPE_CRC_SOURCE_NONE:
+		*val = 0;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 static int get_new_crc_ctl_reg(struct drm_i915_private *dev_priv,
 			       enum pipe pipe,
-			       enum intel_pipe_crc_source *source, u32 *val,
-			       bool set_wa)
+			       enum intel_pipe_crc_source *source, u32 *val)
 {
 	if (IS_GEN(dev_priv, 2))
 		return i8xx_pipe_crc_ctl_reg(source, val);
@@ -435,8 +417,10 @@ static int get_new_crc_ctl_reg(struct drm_i915_private *dev_priv,
 		return vlv_pipe_crc_ctl_reg(dev_priv, pipe, source, val);
 	else if (IS_GEN_RANGE(dev_priv, 5, 6))
 		return ilk_pipe_crc_ctl_reg(source, val);
+	else if (INTEL_GEN(dev_priv) < 9)
+		return ivb_pipe_crc_ctl_reg(dev_priv, pipe, source, val);
 	else
-		return ivb_pipe_crc_ctl_reg(dev_priv, pipe, source, val, set_wa);
+		return skl_pipe_crc_ctl_reg(dev_priv, pipe, source, val);
 }
 
 static int
@@ -486,9 +470,6 @@ static int i9xx_crc_source_valid(struct drm_i915_private *dev_priv,
 	switch (source) {
 	case INTEL_PIPE_CRC_SOURCE_PIPE:
 	case INTEL_PIPE_CRC_SOURCE_TV:
-	case INTEL_PIPE_CRC_SOURCE_DP_B:
-	case INTEL_PIPE_CRC_SOURCE_DP_C:
-	case INTEL_PIPE_CRC_SOURCE_DP_D:
 	case INTEL_PIPE_CRC_SOURCE_NONE:
 		return 0;
 	default:
@@ -532,7 +513,25 @@ static int ivb_crc_source_valid(struct drm_i915_private *dev_priv,
 	case INTEL_PIPE_CRC_SOURCE_PIPE:
 	case INTEL_PIPE_CRC_SOURCE_PLANE1:
 	case INTEL_PIPE_CRC_SOURCE_PLANE2:
-	case INTEL_PIPE_CRC_SOURCE_PF:
+	case INTEL_PIPE_CRC_SOURCE_NONE:
+		return 0;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int skl_crc_source_valid(struct drm_i915_private *dev_priv,
+				const enum intel_pipe_crc_source source)
+{
+	switch (source) {
+	case INTEL_PIPE_CRC_SOURCE_PIPE:
+	case INTEL_PIPE_CRC_SOURCE_PLANE1:
+	case INTEL_PIPE_CRC_SOURCE_PLANE2:
+	case INTEL_PIPE_CRC_SOURCE_PLANE3:
+	case INTEL_PIPE_CRC_SOURCE_PLANE4:
+	case INTEL_PIPE_CRC_SOURCE_PLANE5:
+	case INTEL_PIPE_CRC_SOURCE_PLANE6:
+	case INTEL_PIPE_CRC_SOURCE_PLANE7:
 	case INTEL_PIPE_CRC_SOURCE_NONE:
 		return 0;
 	default:
@@ -552,8 +551,10 @@ intel_is_valid_crc_source(struct drm_i915_private *dev_priv,
 		return vlv_crc_source_valid(dev_priv, source);
 	else if (IS_GEN_RANGE(dev_priv, 5, 6))
 		return ilk_crc_source_valid(dev_priv, source);
-	else
+	else if (INTEL_GEN(dev_priv) < 9)
 		return ivb_crc_source_valid(dev_priv, source);
+	else
+		return skl_crc_source_valid(dev_priv, source);
 }
 
 const char *const *intel_crtc_get_crc_sources(struct drm_crtc *crtc,
@@ -592,6 +593,7 @@ int intel_crtc_set_crc_source(struct drm_crtc *crtc, const char *source_name)
 	intel_wakeref_t wakeref;
 	u32 val = 0; /* shut up gcc */
 	int ret = 0;
+	bool enable;
 
 	if (display_crc_ctl_parse_source(source_name, &source) < 0) {
 		DRM_DEBUG_DRIVER("unknown source %s\n", source_name);
@@ -605,7 +607,11 @@ int intel_crtc_set_crc_source(struct drm_crtc *crtc, const char *source_name)
 		return -EIO;
 	}
 
-	ret = get_new_crc_ctl_reg(dev_priv, crtc->index, &source, &val, true);
+	enable = source != INTEL_PIPE_CRC_SOURCE_NONE;
+	if (enable)
+		intel_crtc_crc_setup_workarounds(to_intel_crtc(crtc), true);
+
+	ret = get_new_crc_ctl_reg(dev_priv, crtc->index, &source, &val);
 	if (ret != 0)
 		goto out;
 
@@ -614,18 +620,16 @@ int intel_crtc_set_crc_source(struct drm_crtc *crtc, const char *source_name)
 	POSTING_READ(PIPE_CRC_CTL(crtc->index));
 
 	if (!source) {
-		if (IS_G4X(dev_priv))
-			g4x_undo_pipe_scramble_reset(dev_priv, crtc->index);
-		else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
+		if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 			vlv_undo_pipe_scramble_reset(dev_priv, crtc->index);
-		else if ((IS_HASWELL(dev_priv) ||
-			  IS_BROADWELL(dev_priv)) && crtc->index == PIPE_A)
-			hsw_pipe_A_crc_wa(dev_priv, false);
 	}
 
 	pipe_crc->skipped = 0;
 
 out:
+	if (!enable)
+		intel_crtc_crc_setup_workarounds(to_intel_crtc(crtc), false);
+
 	intel_display_power_put(dev_priv, power_domain, wakeref);
 
 	return ret;
@@ -641,7 +645,7 @@ void intel_crtc_enable_pipe_crc(struct intel_crtc *intel_crtc)
 	if (!crtc->crc.opened)
 		return;
 
-	if (get_new_crc_ctl_reg(dev_priv, crtc->index, &pipe_crc->source, &val, false) < 0)
+	if (get_new_crc_ctl_reg(dev_priv, crtc->index, &pipe_crc->source, &val) < 0)
 		return;
 
 	/* Don't need pipe_crc->lock here, IRQs are not generated. */
diff --git a/drivers/gpu/drm/i915/intel_pipe_crc.h b/drivers/gpu/drm/i915/intel_pipe_crc.h
new file mode 100644
index 000000000000..81eaf1854788
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_pipe_crc.h
@@ -0,0 +1,35 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2019 Intel Corporation
+ */
+
+#ifndef __INTEL_PIPE_CRC_H__
+#define __INTEL_PIPE_CRC_H__
+
+#include <linux/types.h>
+
+struct drm_crtc;
+struct intel_crtc;
+
+#ifdef CONFIG_DEBUG_FS
+int intel_crtc_set_crc_source(struct drm_crtc *crtc, const char *source_name);
+int intel_crtc_verify_crc_source(struct drm_crtc *crtc,
+				 const char *source_name, size_t *values_cnt);
+const char *const *intel_crtc_get_crc_sources(struct drm_crtc *crtc,
+					      size_t *count);
+void intel_crtc_disable_pipe_crc(struct intel_crtc *crtc);
+void intel_crtc_enable_pipe_crc(struct intel_crtc *crtc);
+#else
+#define intel_crtc_set_crc_source NULL
+#define intel_crtc_verify_crc_source NULL
+#define intel_crtc_get_crc_sources NULL
+static inline void intel_crtc_disable_pipe_crc(struct intel_crtc *crtc)
+{
+}
+
+static inline void intel_crtc_enable_pipe_crc(struct intel_crtc *crtc)
+{
+}
+#endif
+
+#endif /* __INTEL_PIPE_CRC_H__ */
diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c
index 54307f1df6cf..44be676fabd6 100644
--- a/drivers/gpu/drm/i915/intel_pm.c
+++ b/drivers/gpu/drm/i915/intel_pm.c
@@ -35,6 +35,9 @@
 
 #include "i915_drv.h"
 #include "intel_drv.h"
+#include "intel_fbc.h"
+#include "intel_pm.h"
+#include "intel_sprite.h"
 #include "../../../platform/x86/intel_ips.h"
 
 /**
@@ -338,12 +341,12 @@ static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
 
 	mutex_lock(&dev_priv->pcu_lock);
 
-	val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
+	val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 	if (enable)
 		val |= DSP_MAXFIFO_PM5_ENABLE;
 	else
 		val &= ~DSP_MAXFIFO_PM5_ENABLE;
-	vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, val);
+	vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
 
 	mutex_unlock(&dev_priv->pcu_lock);
 }
@@ -850,7 +853,7 @@ static void pineview_update_wm(struct intel_crtc *unused_crtc)
 	u32 reg;
 	unsigned int wm;
 
-	latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev_priv),
+	latency = intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
 					 dev_priv->is_ddr3,
 					 dev_priv->fsb_freq,
 					 dev_priv->mem_freq);
@@ -3624,7 +3627,12 @@ static u8 intel_enabled_dbuf_slices_num(struct drm_i915_private *dev_priv)
 	if (INTEL_GEN(dev_priv) < 11)
 		return enabled_slices;
 
-	if (I915_READ(DBUF_CTL_S2) & DBUF_POWER_STATE)
+	/*
+	 * FIXME: for now we'll only ever use 1 slice; pretend that we have
+	 * only that 1 slice enabled until we have a proper way for on-demand
+	 * toggling of the second slice.
+	 */
+	if (0 && I915_READ(DBUF_CTL_S2) & DBUF_POWER_STATE)
 		enabled_slices++;
 
 	return enabled_slices;
@@ -3919,12 +3927,43 @@ skl_ddb_get_pipe_allocation_limits(struct drm_i915_private *dev_priv,
 	alloc->end = ddb_size * (width_before_pipe + pipe_width) / total_width;
 }
 
-static unsigned int skl_cursor_allocation(int num_active)
+static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
+				 int width, const struct drm_format_info *format,
+				 u64 modifier, unsigned int rotation,
+				 u32 plane_pixel_rate, struct skl_wm_params *wp,
+				 int color_plane);
+static void skl_compute_plane_wm(const struct intel_crtc_state *cstate,
+				 int level,
+				 const struct skl_wm_params *wp,
+				 const struct skl_wm_level *result_prev,
+				 struct skl_wm_level *result /* out */);
+
+static unsigned int
+skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
+		      int num_active)
 {
-	if (num_active == 1)
-		return 32;
+	struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
+	int level, max_level = ilk_wm_max_level(dev_priv);
+	struct skl_wm_level wm = {};
+	int ret, min_ddb_alloc = 0;
+	struct skl_wm_params wp;
+
+	ret = skl_compute_wm_params(crtc_state, 256,
+				    drm_format_info(DRM_FORMAT_ARGB8888),
+				    DRM_FORMAT_MOD_LINEAR,
+				    DRM_MODE_ROTATE_0,
+				    crtc_state->pixel_rate, &wp, 0);
+	WARN_ON(ret);
 
-	return 8;
+	for (level = 0; level <= max_level; level++) {
+		skl_compute_plane_wm(crtc_state, level, &wp, &wm, &wm);
+		if (wm.min_ddb_alloc == U16_MAX)
+			break;
+
+		min_ddb_alloc = wm.min_ddb_alloc;
+	}
+
+	return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
 }
 
 static void skl_ddb_entry_init_from_hw(struct drm_i915_private *dev_priv,
@@ -3970,7 +4009,7 @@ skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv,
 		val = I915_READ(PLANE_BUF_CFG(pipe, plane_id));
 		val2 = I915_READ(PLANE_NV12_BUF_CFG(pipe, plane_id));
 
-		if (fourcc == DRM_FORMAT_NV12)
+		if (is_planar_yuv_format(fourcc))
 			swap(val, val2);
 
 		skl_ddb_entry_init_from_hw(dev_priv, ddb_y, val);
@@ -4180,7 +4219,7 @@ skl_plane_relative_data_rate(const struct intel_crtc_state *cstate,
 
 	if (intel_plane->id == PLANE_CURSOR)
 		return 0;
-	if (plane == 1 && format != DRM_FORMAT_NV12)
+	if (plane == 1 && !is_planar_yuv_format(format))
 		return 0;
 
 	/*
@@ -4192,7 +4231,7 @@ skl_plane_relative_data_rate(const struct intel_crtc_state *cstate,
 	height = drm_rect_height(&intel_pstate->base.src) >> 16;
 
 	/* UV plane does 1/2 pixel sub-sampling */
-	if (plane == 1 && format == DRM_FORMAT_NV12) {
+	if (plane == 1 && is_planar_yuv_format(format)) {
 		width /= 2;
 		height /= 2;
 	}
@@ -4308,7 +4347,6 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
 	struct drm_i915_private *dev_priv = to_i915(crtc->dev);
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	struct skl_ddb_entry *alloc = &cstate->wm.skl.ddb;
-	struct skl_plane_wm *wm;
 	u16 alloc_size, start = 0;
 	u16 total[I915_MAX_PLANES] = {};
 	u16 uv_total[I915_MAX_PLANES] = {};
@@ -4349,7 +4387,7 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
 		return 0;
 
 	/* Allocate fixed number of blocks for cursor. */
-	total[PLANE_CURSOR] = skl_cursor_allocation(num_active);
+	total[PLANE_CURSOR] = skl_cursor_allocation(cstate, num_active);
 	alloc_size -= total[PLANE_CURSOR];
 	cstate->wm.skl.plane_ddb_y[PLANE_CURSOR].start =
 		alloc->end - total[PLANE_CURSOR];
@@ -4365,15 +4403,23 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
 	for (level = ilk_wm_max_level(dev_priv); level >= 0; level--) {
 		blocks = 0;
 		for_each_plane_id_on_crtc(intel_crtc, plane_id) {
-			if (plane_id == PLANE_CURSOR)
+			const struct skl_plane_wm *wm =
+				&cstate->wm.skl.optimal.planes[plane_id];
+
+			if (plane_id == PLANE_CURSOR) {
+				if (WARN_ON(wm->wm[level].min_ddb_alloc >
+					    total[PLANE_CURSOR])) {
+					blocks = U32_MAX;
+					break;
+				}
 				continue;
+			}
 
-			wm = &cstate->wm.skl.optimal.planes[plane_id];
 			blocks += wm->wm[level].min_ddb_alloc;
 			blocks += wm->uv_wm[level].min_ddb_alloc;
 		}
 
-		if (blocks < alloc_size) {
+		if (blocks <= alloc_size) {
 			alloc_size -= blocks;
 			break;
 		}
@@ -4392,6 +4438,8 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
 	 * proportional to its relative data rate.
 	 */
 	for_each_plane_id_on_crtc(intel_crtc, plane_id) {
+		const struct skl_plane_wm *wm =
+			&cstate->wm.skl.optimal.planes[plane_id];
 		u64 rate;
 		u16 extra;
 
@@ -4405,8 +4453,6 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
 		if (total_data_rate == 0)
 			break;
 
-		wm = &cstate->wm.skl.optimal.planes[plane_id];
-
 		rate = plane_data_rate[plane_id];
 		extra = min_t(u16, alloc_size,
 			      DIV64_U64_ROUND_UP(alloc_size * rate,
@@ -4431,14 +4477,14 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
 	/* Set the actual DDB start/end points for each plane */
 	start = alloc->start;
 	for_each_plane_id_on_crtc(intel_crtc, plane_id) {
-		struct skl_ddb_entry *plane_alloc, *uv_plane_alloc;
+		struct skl_ddb_entry *plane_alloc =
+			&cstate->wm.skl.plane_ddb_y[plane_id];
+		struct skl_ddb_entry *uv_plane_alloc =
+			&cstate->wm.skl.plane_ddb_uv[plane_id];
 
 		if (plane_id == PLANE_CURSOR)
 			continue;
 
-		plane_alloc = &cstate->wm.skl.plane_ddb_y[plane_id];
-		uv_plane_alloc = &cstate->wm.skl.plane_ddb_uv[plane_id];
-
 		/* Gen11+ uses a separate plane for UV watermarks */
 		WARN_ON(INTEL_GEN(dev_priv) >= 11 && uv_total[plane_id]);
 
@@ -4464,8 +4510,35 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
 	 */
 	for (level++; level <= ilk_wm_max_level(dev_priv); level++) {
 		for_each_plane_id_on_crtc(intel_crtc, plane_id) {
-			wm = &cstate->wm.skl.optimal.planes[plane_id];
-			memset(&wm->wm[level], 0, sizeof(wm->wm[level]));
+			struct skl_plane_wm *wm =
+				&cstate->wm.skl.optimal.planes[plane_id];
+
+			/*
+			 * We only disable the watermarks for each plane if
+			 * they exceed the ddb allocation of said plane. This
+			 * is done so that we don't end up touching cursor
+			 * watermarks needlessly when some other plane reduces
+			 * our max possible watermark level.
+			 *
+			 * Bspec has this to say about the PLANE_WM enable bit:
+			 * "All the watermarks at this level for all enabled
+			 *  planes must be enabled before the level will be used."
+			 * So this is actually safe to do.
+			 */
+			if (wm->wm[level].min_ddb_alloc > total[plane_id] ||
+			    wm->uv_wm[level].min_ddb_alloc > uv_total[plane_id])
+				memset(&wm->wm[level], 0, sizeof(wm->wm[level]));
+
+			/*
+			 * Wa_1408961008:icl, ehl
+			 * Underruns with WM1+ disabled
+			 */
+			if (IS_GEN(dev_priv, 11) &&
+			    level == 1 && wm->wm[0].plane_en) {
+				wm->wm[level].plane_res_b = wm->wm[0].plane_res_b;
+				wm->wm[level].plane_res_l = wm->wm[0].plane_res_l;
+				wm->wm[level].ignore_lines = wm->wm[0].ignore_lines;
+			}
 		}
 	}
 
@@ -4474,7 +4547,9 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
 	 * don't have enough DDB blocks for it.
 	 */
 	for_each_plane_id_on_crtc(intel_crtc, plane_id) {
-		wm = &cstate->wm.skl.optimal.planes[plane_id];
+		struct skl_plane_wm *wm =
+			&cstate->wm.skl.optimal.planes[plane_id];
+
 		if (wm->trans_wm.plane_res_b >= total[plane_id])
 			memset(&wm->trans_wm, 0, sizeof(wm->trans_wm));
 	}
@@ -4568,57 +4643,45 @@ skl_adjusted_plane_pixel_rate(const struct intel_crtc_state *cstate,
 }
 
 static int
-skl_compute_plane_wm_params(const struct intel_crtc_state *cstate,
-			    const struct intel_plane_state *intel_pstate,
-			    struct skl_wm_params *wp, int color_plane)
+skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
+		      int width, const struct drm_format_info *format,
+		      u64 modifier, unsigned int rotation,
+		      u32 plane_pixel_rate, struct skl_wm_params *wp,
+		      int color_plane)
 {
-	struct intel_plane *plane = to_intel_plane(intel_pstate->base.plane);
-	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
-	const struct drm_plane_state *pstate = &intel_pstate->base;
-	const struct drm_framebuffer *fb = pstate->fb;
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 	u32 interm_pbpl;
 
-	/* only NV12 format has two planes */
-	if (color_plane == 1 && fb->format->format != DRM_FORMAT_NV12) {
-		DRM_DEBUG_KMS("Non NV12 format have single plane\n");
+	/* only planar format has two planes */
+	if (color_plane == 1 && !is_planar_yuv_format(format->format)) {
+		DRM_DEBUG_KMS("Non planar format have single plane\n");
 		return -EINVAL;
 	}
 
-	wp->y_tiled = fb->modifier == I915_FORMAT_MOD_Y_TILED ||
-		      fb->modifier == I915_FORMAT_MOD_Yf_TILED ||
-		      fb->modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
-		      fb->modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
-	wp->x_tiled = fb->modifier == I915_FORMAT_MOD_X_TILED;
-	wp->rc_surface = fb->modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
-			 fb->modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
-	wp->is_planar = fb->format->format == DRM_FORMAT_NV12;
-
-	if (plane->id == PLANE_CURSOR) {
-		wp->width = intel_pstate->base.crtc_w;
-	} else {
-		/*
-		 * Src coordinates are already rotated by 270 degrees for
-		 * the 90/270 degree plane rotation cases (to match the
-		 * GTT mapping), hence no need to account for rotation here.
-		 */
-		wp->width = drm_rect_width(&intel_pstate->base.src) >> 16;
-	}
+	wp->y_tiled = modifier == I915_FORMAT_MOD_Y_TILED ||
+		      modifier == I915_FORMAT_MOD_Yf_TILED ||
+		      modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
+		      modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
+	wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
+	wp->rc_surface = modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
+			 modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
+	wp->is_planar = is_planar_yuv_format(format->format);
 
+	wp->width = width;
 	if (color_plane == 1 && wp->is_planar)
 		wp->width /= 2;
 
-	wp->cpp = fb->format->cpp[color_plane];
-	wp->plane_pixel_rate = skl_adjusted_plane_pixel_rate(cstate,
-							     intel_pstate);
+	wp->cpp = format->cpp[color_plane];
+	wp->plane_pixel_rate = plane_pixel_rate;
 
 	if (INTEL_GEN(dev_priv) >= 11 &&
-	    fb->modifier == I915_FORMAT_MOD_Yf_TILED && wp->cpp == 1)
+	    modifier == I915_FORMAT_MOD_Yf_TILED  && wp->cpp == 1)
 		wp->dbuf_block_size = 256;
 	else
 		wp->dbuf_block_size = 512;
 
-	if (drm_rotation_90_or_270(pstate->rotation)) {
-
+	if (drm_rotation_90_or_270(rotation)) {
 		switch (wp->cpp) {
 		case 1:
 			wp->y_min_scanlines = 16;
@@ -4663,12 +4726,40 @@ skl_compute_plane_wm_params(const struct intel_crtc_state *cstate,
 
 	wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
 					     wp->plane_blocks_per_line);
+
 	wp->linetime_us = fixed16_to_u32_round_up(
-					intel_get_linetime_us(cstate));
+					intel_get_linetime_us(crtc_state));
 
 	return 0;
 }
 
+static int
+skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
+			    const struct intel_plane_state *plane_state,
+			    struct skl_wm_params *wp, int color_plane)
+{
+	struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
+	const struct drm_framebuffer *fb = plane_state->base.fb;
+	int width;
+
+	if (plane->id == PLANE_CURSOR) {
+		width = plane_state->base.crtc_w;
+	} else {
+		/*
+		 * Src coordinates are already rotated by 270 degrees for
+		 * the 90/270 degree plane rotation cases (to match the
+		 * GTT mapping), hence no need to account for rotation here.
+		 */
+		width = drm_rect_width(&plane_state->base.src) >> 16;
+	}
+
+	return skl_compute_wm_params(crtc_state, width,
+				     fb->format, fb->modifier,
+				     plane_state->base.rotation,
+				     skl_adjusted_plane_pixel_rate(crtc_state, plane_state),
+				     wp, color_plane);
+}
+
 static bool skl_wm_has_lines(struct drm_i915_private *dev_priv, int level)
 {
 	if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
@@ -4679,14 +4770,12 @@ static bool skl_wm_has_lines(struct drm_i915_private *dev_priv, int level)
 }
 
 static void skl_compute_plane_wm(const struct intel_crtc_state *cstate,
-				 const struct intel_plane_state *intel_pstate,
 				 int level,
 				 const struct skl_wm_params *wp,
 				 const struct skl_wm_level *result_prev,
 				 struct skl_wm_level *result /* out */)
 {
-	struct drm_i915_private *dev_priv =
-		to_i915(intel_pstate->base.plane->dev);
+	struct drm_i915_private *dev_priv = to_i915(cstate->base.crtc->dev);
 	u32 latency = dev_priv->wm.skl_latency[level];
 	uint_fixed_16_16_t method1, method2;
 	uint_fixed_16_16_t selected_result;
@@ -4805,19 +4894,17 @@ static void skl_compute_plane_wm(const struct intel_crtc_state *cstate,
 
 static void
 skl_compute_wm_levels(const struct intel_crtc_state *cstate,
-		      const struct intel_plane_state *intel_pstate,
 		      const struct skl_wm_params *wm_params,
 		      struct skl_wm_level *levels)
 {
-	struct drm_i915_private *dev_priv =
-		to_i915(intel_pstate->base.plane->dev);
+	struct drm_i915_private *dev_priv = to_i915(cstate->base.crtc->dev);
 	int level, max_level = ilk_wm_max_level(dev_priv);
 	struct skl_wm_level *result_prev = &levels[0];
 
 	for (level = 0; level <= max_level; level++) {
 		struct skl_wm_level *result = &levels[level];
 
-		skl_compute_plane_wm(cstate, intel_pstate, level, wm_params,
+		skl_compute_plane_wm(cstate, level, wm_params,
 				     result_prev, result);
 
 		result_prev = result;
@@ -4914,7 +5001,7 @@ static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
 	if (ret)
 		return ret;
 
-	skl_compute_wm_levels(crtc_state, plane_state, &wm_params, wm->wm);
+	skl_compute_wm_levels(crtc_state, &wm_params, wm->wm);
 	skl_compute_transition_wm(crtc_state, &wm_params, wm);
 
 	return 0;
@@ -4936,13 +5023,12 @@ static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
 	if (ret)
 		return ret;
 
-	skl_compute_wm_levels(crtc_state, plane_state, &wm_params, wm->uv_wm);
+	skl_compute_wm_levels(crtc_state, &wm_params, wm->uv_wm);
 
 	return 0;
 }
 
-static int skl_build_plane_wm(struct skl_pipe_wm *pipe_wm,
-			      struct intel_crtc_state *crtc_state,
+static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
 			      const struct intel_plane_state *plane_state)
 {
 	struct intel_plane *plane = to_intel_plane(plane_state->base.plane);
@@ -4968,8 +5054,7 @@ static int skl_build_plane_wm(struct skl_pipe_wm *pipe_wm,
 	return 0;
 }
 
-static int icl_build_plane_wm(struct skl_pipe_wm *pipe_wm,
-			      struct intel_crtc_state *crtc_state,
+static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
 			      const struct intel_plane_state *plane_state)
 {
 	enum plane_id plane_id = to_intel_plane(plane_state->base.plane)->id;
@@ -5006,10 +5091,10 @@ static int icl_build_plane_wm(struct skl_pipe_wm *pipe_wm,
 	return 0;
 }
 
-static int skl_build_pipe_wm(struct intel_crtc_state *cstate,
-			     struct skl_pipe_wm *pipe_wm)
+static int skl_build_pipe_wm(struct intel_crtc_state *cstate)
 {
 	struct drm_i915_private *dev_priv = to_i915(cstate->base.crtc->dev);
+	struct skl_pipe_wm *pipe_wm = &cstate->wm.skl.optimal;
 	struct drm_crtc_state *crtc_state = &cstate->base;
 	struct drm_plane *plane;
 	const struct drm_plane_state *pstate;
@@ -5026,11 +5111,9 @@ static int skl_build_pipe_wm(struct intel_crtc_state *cstate,
 						to_intel_plane_state(pstate);
 
 		if (INTEL_GEN(dev_priv) >= 11)
-			ret = icl_build_plane_wm(pipe_wm,
-						 cstate, intel_pstate);
+			ret = icl_build_plane_wm(cstate, intel_pstate);
 		else
-			ret = skl_build_plane_wm(pipe_wm,
-						 cstate, intel_pstate);
+			ret = skl_build_plane_wm(cstate, intel_pstate);
 		if (ret)
 			return ret;
 	}
@@ -5056,11 +5139,12 @@ static void skl_write_wm_level(struct drm_i915_private *dev_priv,
 {
 	u32 val = 0;
 
-	if (level->plane_en) {
+	if (level->plane_en)
 		val |= PLANE_WM_EN;
-		val |= level->plane_res_b;
-		val |= level->plane_res_l << PLANE_WM_LINES_SHIFT;
-	}
+	if (level->ignore_lines)
+		val |= PLANE_WM_IGNORE_LINES;
+	val |= level->plane_res_b;
+	val |= level->plane_res_l << PLANE_WM_LINES_SHIFT;
 
 	I915_WRITE_FW(reg, val);
 }
@@ -5126,6 +5210,7 @@ bool skl_wm_level_equals(const struct skl_wm_level *l1,
 			 const struct skl_wm_level *l2)
 {
 	return l1->plane_en == l2->plane_en &&
+		l1->ignore_lines == l2->ignore_lines &&
 		l1->plane_res_l == l2->plane_res_l &&
 		l1->plane_res_b == l2->plane_res_b;
 }
@@ -5169,7 +5254,7 @@ static inline bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
 }
 
 bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
-				 const struct skl_ddb_entry entries[],
+				 const struct skl_ddb_entry *entries,
 				 int num_entries, int ignore_idx)
 {
 	int i;
@@ -5183,23 +5268,6 @@ bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
 	return false;
 }
 
-static int skl_update_pipe_wm(struct intel_crtc_state *cstate,
-			      const struct skl_pipe_wm *old_pipe_wm,
-			      struct skl_pipe_wm *pipe_wm, /* out */
-			      bool *changed /* out */)
-{
-	struct intel_crtc *crtc = to_intel_crtc(cstate->base.crtc);
-	int ret;
-
-	ret = skl_build_pipe_wm(cstate, pipe_wm);
-	if (ret)
-		return ret;
-
-	*changed = !skl_pipe_wm_equals(crtc, old_pipe_wm, pipe_wm);
-
-	return 0;
-}
-
 static u32
 pipes_modified(struct intel_atomic_state *state)
 {
@@ -5269,6 +5337,11 @@ skl_compute_ddb(struct intel_atomic_state *state)
 	return 0;
 }
 
+static char enast(bool enable)
+{
+	return enable ? '*' : ' ';
+}
+
 static void
 skl_print_wm_changes(struct intel_atomic_state *state)
 {
@@ -5279,8 +5352,16 @@ skl_print_wm_changes(struct intel_atomic_state *state)
 	struct intel_crtc *crtc;
 	int i;
 
+	if ((drm_debug & DRM_UT_KMS) == 0)
+		return;
+
 	for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
 					    new_crtc_state, i) {
+		const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;
+
+		old_pipe_wm = &old_crtc_state->wm.skl.optimal;
+		new_pipe_wm = &new_crtc_state->wm.skl.optimal;
+
 		for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
 			enum plane_id plane_id = plane->id;
 			const struct skl_ddb_entry *old, *new;
@@ -5291,10 +5372,86 @@ skl_print_wm_changes(struct intel_atomic_state *state)
 			if (skl_ddb_entry_equal(old, new))
 				continue;
 
-			DRM_DEBUG_KMS("[PLANE:%d:%s] ddb (%d - %d) -> (%d - %d)\n",
+			DRM_DEBUG_KMS("[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
+				      plane->base.base.id, plane->base.name,
+				      old->start, old->end, new->start, new->end,
+				      skl_ddb_entry_size(old), skl_ddb_entry_size(new));
+		}
+
+		for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
+			enum plane_id plane_id = plane->id;
+			const struct skl_plane_wm *old_wm, *new_wm;
+
+			old_wm = &old_pipe_wm->planes[plane_id];
+			new_wm = &new_pipe_wm->planes[plane_id];
+
+			if (skl_plane_wm_equals(dev_priv, old_wm, new_wm))
+				continue;
+
+			DRM_DEBUG_KMS("[PLANE:%d:%s]   level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm"
+				      " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm\n",
+				      plane->base.base.id, plane->base.name,
+				      enast(old_wm->wm[0].plane_en), enast(old_wm->wm[1].plane_en),
+				      enast(old_wm->wm[2].plane_en), enast(old_wm->wm[3].plane_en),
+				      enast(old_wm->wm[4].plane_en), enast(old_wm->wm[5].plane_en),
+				      enast(old_wm->wm[6].plane_en), enast(old_wm->wm[7].plane_en),
+				      enast(old_wm->trans_wm.plane_en),
+				      enast(new_wm->wm[0].plane_en), enast(new_wm->wm[1].plane_en),
+				      enast(new_wm->wm[2].plane_en), enast(new_wm->wm[3].plane_en),
+				      enast(new_wm->wm[4].plane_en), enast(new_wm->wm[5].plane_en),
+				      enast(new_wm->wm[6].plane_en), enast(new_wm->wm[7].plane_en),
+				      enast(new_wm->trans_wm.plane_en));
+
+			DRM_DEBUG_KMS("[PLANE:%d:%s]   lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d"
+				      " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d\n",
 				      plane->base.base.id, plane->base.name,
-				      old->start, old->end,
-				      new->start, new->end);
+				      enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].plane_res_l,
+				      enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].plane_res_l,
+				      enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].plane_res_l,
+				      enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].plane_res_l,
+				      enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].plane_res_l,
+				      enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].plane_res_l,
+				      enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].plane_res_l,
+				      enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].plane_res_l,
+				      enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.plane_res_l,
+
+				      enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].plane_res_l,
+				      enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].plane_res_l,
+				      enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].plane_res_l,
+				      enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].plane_res_l,
+				      enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].plane_res_l,
+				      enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].plane_res_l,
+				      enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].plane_res_l,
+				      enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].plane_res_l,
+				      enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.plane_res_l);
+
+			DRM_DEBUG_KMS("[PLANE:%d:%s]  blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d"
+				      " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d\n",
+				      plane->base.base.id, plane->base.name,
+				      old_wm->wm[0].plane_res_b, old_wm->wm[1].plane_res_b,
+				      old_wm->wm[2].plane_res_b, old_wm->wm[3].plane_res_b,
+				      old_wm->wm[4].plane_res_b, old_wm->wm[5].plane_res_b,
+				      old_wm->wm[6].plane_res_b, old_wm->wm[7].plane_res_b,
+				      old_wm->trans_wm.plane_res_b,
+				      new_wm->wm[0].plane_res_b, new_wm->wm[1].plane_res_b,
+				      new_wm->wm[2].plane_res_b, new_wm->wm[3].plane_res_b,
+				      new_wm->wm[4].plane_res_b, new_wm->wm[5].plane_res_b,
+				      new_wm->wm[6].plane_res_b, new_wm->wm[7].plane_res_b,
+				      new_wm->trans_wm.plane_res_b);
+
+			DRM_DEBUG_KMS("[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d"
+				      " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d\n",
+				      plane->base.base.id, plane->base.name,
+				      old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
+				      old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
+				      old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
+				      old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
+				      old_wm->trans_wm.min_ddb_alloc,
+				      new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
+				      new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
+				      new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
+				      new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
+				      new_wm->trans_wm.min_ddb_alloc);
 		}
 	}
 }
@@ -5449,10 +5606,9 @@ static int
 skl_compute_wm(struct intel_atomic_state *state)
 {
 	struct intel_crtc *crtc;
-	struct intel_crtc_state *cstate;
+	struct intel_crtc_state *new_crtc_state;
 	struct intel_crtc_state *old_crtc_state;
 	struct skl_ddb_values *results = &state->wm_results;
-	struct skl_pipe_wm *pipe_wm;
 	bool changed = false;
 	int ret, i;
 
@@ -5470,12 +5626,8 @@ skl_compute_wm(struct intel_atomic_state *state)
 	 * pipe allocations had to change.
 	 */
 	for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
-					    cstate, i) {
-		const struct skl_pipe_wm *old_pipe_wm =
-			&old_crtc_state->wm.skl.optimal;
-
-		pipe_wm = &cstate->wm.skl.optimal;
-		ret = skl_update_pipe_wm(cstate, old_pipe_wm, pipe_wm, &changed);
+					    new_crtc_state, i) {
+		ret = skl_build_pipe_wm(new_crtc_state);
 		if (ret)
 			return ret;
 
@@ -5483,7 +5635,9 @@ skl_compute_wm(struct intel_atomic_state *state)
 		if (ret)
 			return ret;
 
-		if (changed)
+		if (!skl_pipe_wm_equals(crtc,
+					&old_crtc_state->wm.skl.optimal,
+					&new_crtc_state->wm.skl.optimal))
 			results->dirty_pipes |= drm_crtc_mask(&crtc->base);
 	}
 
@@ -5609,6 +5763,7 @@ static inline void skl_wm_level_from_reg_val(u32 val,
 					     struct skl_wm_level *level)
 {
 	level->plane_en = val & PLANE_WM_EN;
+	level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
 	level->plane_res_b = val & PLANE_WM_BLOCKS_MASK;
 	level->plane_res_l = (val >> PLANE_WM_LINES_SHIFT) &
 		PLANE_WM_LINES_MASK;
@@ -5986,7 +6141,7 @@ void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
 	if (IS_CHERRYVIEW(dev_priv)) {
 		mutex_lock(&dev_priv->pcu_lock);
 
-		val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
+		val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 		if (val & DSP_MAXFIFO_PM5_ENABLE)
 			wm->level = VLV_WM_LEVEL_PM5;
 
@@ -6451,7 +6606,7 @@ static void rps_set_power(struct drm_i915_private *dev_priv, int new_power)
 				       ei_down * threshold_down / 100));
 
 	I915_WRITE(GEN6_RP_CONTROL,
-		   GEN6_RP_MEDIA_TURBO |
+		   (INTEL_GEN(dev_priv) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) |
 		   GEN6_RP_MEDIA_HW_NORMAL_MODE |
 		   GEN6_RP_MEDIA_IS_GFX |
 		   GEN6_RP_ENABLE |
@@ -6629,9 +6784,9 @@ static void vlv_set_rps_idle(struct drm_i915_private *dev_priv)
 	 * punit into committing the voltage change) as that takes a lot less
 	 * power than the render powerwell.
 	 */
-	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_MEDIA);
+	intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_MEDIA);
 	err = valleyview_set_rps(dev_priv, val);
-	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_MEDIA);
+	intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_MEDIA);
 
 	if (err)
 		DRM_ERROR("Failed to set RPS for idle\n");
@@ -6691,8 +6846,7 @@ void gen6_rps_idle(struct drm_i915_private *dev_priv)
 	mutex_unlock(&dev_priv->pcu_lock);
 }
 
-void gen6_rps_boost(struct i915_request *rq,
-		    struct intel_rps_client *rps_client)
+void gen6_rps_boost(struct i915_request *rq)
 {
 	struct intel_rps *rps = &rq->i915->gt_pm.rps;
 	unsigned long flags;
@@ -6721,7 +6875,7 @@ void gen6_rps_boost(struct i915_request *rq,
 	if (READ_ONCE(rps->cur_freq) < rps->boost_freq)
 		schedule_work(&rps->work);
 
-	atomic_inc(rps_client ? &rps_client->boosts : &rps->boosts);
+	atomic_inc(&rps->boosts);
 }
 
 int intel_set_rps(struct drm_i915_private *dev_priv, u8 val)
@@ -6782,11 +6936,11 @@ static void valleyview_disable_rc6(struct drm_i915_private *dev_priv)
 {
 	/* We're doing forcewake before Disabling RC6,
 	 * This what the BIOS expects when going into suspend */
-	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
 
 	I915_WRITE(GEN6_RC_CONTROL, 0);
 
-	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
 }
 
 static void valleyview_disable_rps(struct drm_i915_private *dev_priv)
@@ -6945,7 +7099,7 @@ static void reset_rps(struct drm_i915_private *dev_priv,
 /* See the Gen9_GT_PM_Programming_Guide doc for the below */
 static void gen9_enable_rps(struct drm_i915_private *dev_priv)
 {
-	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
 
 	/* Program defaults and thresholds for RPS */
 	if (IS_GEN(dev_priv, 9))
@@ -6963,7 +7117,79 @@ static void gen9_enable_rps(struct drm_i915_private *dev_priv)
 	 * RP_INTERRUPT_LIMITS & RPNSWREQ registers */
 	reset_rps(dev_priv, gen6_set_rps);
 
-	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
+}
+
+static void gen11_enable_rc6(struct drm_i915_private *dev_priv)
+{
+	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
+
+	/* 1a: Software RC state - RC0 */
+	I915_WRITE(GEN6_RC_STATE, 0);
+
+	/*
+	 * 1b: Get forcewake during program sequence. Although the driver
+	 * hasn't enabled a state yet where we need forcewake, BIOS may have.
+	 */
+	intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
+
+	/* 2a: Disable RC states. */
+	I915_WRITE(GEN6_RC_CONTROL, 0);
+
+	/* 2b: Program RC6 thresholds.*/
+	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16 | 85);
+	I915_WRITE(GEN10_MEDIA_WAKE_RATE_LIMIT, 150);
+
+	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
+	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
+	for_each_engine(engine, dev_priv, id)
+		I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
+
+	if (HAS_GUC(dev_priv))
+		I915_WRITE(GUC_MAX_IDLE_COUNT, 0xA);
+
+	I915_WRITE(GEN6_RC_SLEEP, 0);
+
+	I915_WRITE(GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */
+
+	/*
+	 * 2c: Program Coarse Power Gating Policies.
+	 *
+	 * Bspec's guidance is to use 25us (really 25 * 1280ns) here. What we
+	 * use instead is a more conservative estimate for the maximum time
+	 * it takes us to service a CS interrupt and submit a new ELSP - that
+	 * is the time which the GPU is idle waiting for the CPU to select the
+	 * next request to execute. If the idle hysteresis is less than that
+	 * interrupt service latency, the hardware will automatically gate
+	 * the power well and we will then incur the wake up cost on top of
+	 * the service latency. A similar guide from intel_pstate is that we
+	 * do not want the enable hysteresis to less than the wakeup latency.
+	 *
+	 * igt/gem_exec_nop/sequential provides a rough estimate for the
+	 * service latency, and puts it around 10us for Broadwell (and other
+	 * big core) and around 40us for Broxton (and other low power cores).
+	 * [Note that for legacy ringbuffer submission, this is less than 1us!]
+	 * However, the wakeup latency on Broxton is closer to 100us. To be
+	 * conservative, we have to factor in a context switch on top (due
+	 * to ksoftirqd).
+	 */
+	I915_WRITE(GEN9_MEDIA_PG_IDLE_HYSTERESIS, 250);
+	I915_WRITE(GEN9_RENDER_PG_IDLE_HYSTERESIS, 250);
+
+	/* 3a: Enable RC6 */
+	I915_WRITE(GEN6_RC_CONTROL,
+		   GEN6_RC_CTL_HW_ENABLE |
+		   GEN6_RC_CTL_RC6_ENABLE |
+		   GEN6_RC_CTL_EI_MODE(1));
+
+	/* 3b: Enable Coarse Power Gating only when RC6 is enabled. */
+	I915_WRITE(GEN9_PG_ENABLE,
+		   GEN9_RENDER_PG_ENABLE |
+		   GEN9_MEDIA_PG_ENABLE |
+		   GEN11_MEDIA_SAMPLER_PG_ENABLE);
+
+	intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
 }
 
 static void gen9_enable_rc6(struct drm_i915_private *dev_priv)
@@ -6977,7 +7203,7 @@ static void gen9_enable_rc6(struct drm_i915_private *dev_priv)
 
 	/* 1b: Get forcewake during program sequence. Although the driver
 	 * hasn't enabled a state yet where we need forcewake, BIOS may have.*/
-	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
 
 	/* 2a: Disable RC states. */
 	I915_WRITE(GEN6_RC_CONTROL, 0);
@@ -7054,7 +7280,7 @@ static void gen9_enable_rc6(struct drm_i915_private *dev_priv)
 		I915_WRITE(GEN9_PG_ENABLE,
 			   GEN9_RENDER_PG_ENABLE | GEN9_MEDIA_PG_ENABLE);
 
-	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
 }
 
 static void gen8_enable_rc6(struct drm_i915_private *dev_priv)
@@ -7067,7 +7293,7 @@ static void gen8_enable_rc6(struct drm_i915_private *dev_priv)
 
 	/* 1b: Get forcewake during program sequence. Although the driver
 	 * hasn't enabled a state yet where we need forcewake, BIOS may have.*/
-	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
 
 	/* 2a: Disable RC states. */
 	I915_WRITE(GEN6_RC_CONTROL, 0);
@@ -7088,14 +7314,14 @@ static void gen8_enable_rc6(struct drm_i915_private *dev_priv)
 		   GEN7_RC_CTL_TO_MODE |
 		   GEN6_RC_CTL_RC6_ENABLE);
 
-	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
 }
 
 static void gen8_enable_rps(struct drm_i915_private *dev_priv)
 {
 	struct intel_rps *rps = &dev_priv->gt_pm.rps;
 
-	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
 
 	/* 1 Program defaults and thresholds for RPS*/
 	I915_WRITE(GEN6_RPNSWREQ,
@@ -7128,7 +7354,7 @@ static void gen8_enable_rps(struct drm_i915_private *dev_priv)
 
 	reset_rps(dev_priv, gen6_set_rps);
 
-	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
 }
 
 static void gen6_enable_rc6(struct drm_i915_private *dev_priv)
@@ -7148,7 +7374,7 @@ static void gen6_enable_rc6(struct drm_i915_private *dev_priv)
 		I915_WRITE(GTFIFODBG, gtfifodbg);
 	}
 
-	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
 
 	/* disable the counters and set deterministic thresholds */
 	I915_WRITE(GEN6_RC_CONTROL, 0);
@@ -7196,7 +7422,7 @@ static void gen6_enable_rc6(struct drm_i915_private *dev_priv)
 			DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
 	}
 
-	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
 }
 
 static void gen6_enable_rps(struct drm_i915_private *dev_priv)
@@ -7207,7 +7433,7 @@ static void gen6_enable_rps(struct drm_i915_private *dev_priv)
 	 * Perhaps there might be some value in exposing these to
 	 * userspace...
 	 */
-	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
 
 	/* Power down if completely idle for over 50ms */
 	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 50000);
@@ -7215,7 +7441,7 @@ static void gen6_enable_rps(struct drm_i915_private *dev_priv)
 
 	reset_rps(dev_priv, gen6_set_rps);
 
-	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
 }
 
 static void gen6_update_ring_freq(struct drm_i915_private *dev_priv)
@@ -7638,7 +7864,7 @@ static void cherryview_enable_rc6(struct drm_i915_private *dev_priv)
 
 	/* 1a & 1b: Get forcewake during program sequence. Although the driver
 	 * hasn't enabled a state yet where we need forcewake, BIOS may have.*/
-	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
 
 	/*  Disable RC states. */
 	I915_WRITE(GEN6_RC_CONTROL, 0);
@@ -7670,14 +7896,14 @@ static void cherryview_enable_rc6(struct drm_i915_private *dev_priv)
 		rc6_mode = GEN7_RC_CTL_TO_MODE;
 	I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
 
-	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
 }
 
 static void cherryview_enable_rps(struct drm_i915_private *dev_priv)
 {
 	u32 val;
 
-	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
 
 	/* 1: Program defaults and thresholds for RPS*/
 	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
@@ -7712,7 +7938,7 @@ static void cherryview_enable_rps(struct drm_i915_private *dev_priv)
 
 	reset_rps(dev_priv, valleyview_set_rps);
 
-	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
 }
 
 static void valleyview_enable_rc6(struct drm_i915_private *dev_priv)
@@ -7730,7 +7956,7 @@ static void valleyview_enable_rc6(struct drm_i915_private *dev_priv)
 		I915_WRITE(GTFIFODBG, gtfifodbg);
 	}
 
-	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
 
 	/*  Disable RC states. */
 	I915_WRITE(GEN6_RC_CONTROL, 0);
@@ -7755,14 +7981,14 @@ static void valleyview_enable_rc6(struct drm_i915_private *dev_priv)
 	I915_WRITE(GEN6_RC_CONTROL,
 		   GEN7_RC_CTL_TO_MODE | VLV_RC_CTL_CTX_RST_PARALLEL);
 
-	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
 }
 
 static void valleyview_enable_rps(struct drm_i915_private *dev_priv)
 {
 	u32 val;
 
-	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
 
 	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
 	I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
@@ -7796,7 +8022,7 @@ static void valleyview_enable_rps(struct drm_i915_private *dev_priv)
 
 	reset_rps(dev_priv, valleyview_set_rps);
 
-	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
 }
 
 static unsigned long intel_pxfreq(u32 vidfreq)
@@ -8037,14 +8263,14 @@ unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
 	return val;
 }
 
-static struct drm_i915_private *i915_mch_dev;
+static struct drm_i915_private __rcu *i915_mch_dev;
 
 static struct drm_i915_private *mchdev_get(void)
 {
 	struct drm_i915_private *i915;
 
 	rcu_read_lock();
-	i915 = i915_mch_dev;
+	i915 = rcu_dereference(i915_mch_dev);
 	if (!kref_get_unless_zero(&i915->drm.ref))
 		i915 = NULL;
 	rcu_read_unlock();
@@ -8343,22 +8569,6 @@ void intel_cleanup_gt_powersave(struct drm_i915_private *dev_priv)
 		pm_runtime_put(&dev_priv->drm.pdev->dev);
 }
 
-/**
- * intel_suspend_gt_powersave - suspend PM work and helper threads
- * @dev_priv: i915 device
- *
- * We don't want to disable RC6 or other features here, we just want
- * to make sure any work we've queued has finished and won't bother
- * us while we're suspended.
- */
-void intel_suspend_gt_powersave(struct drm_i915_private *dev_priv)
-{
-	if (INTEL_GEN(dev_priv) < 6)
-		return;
-
-	/* gen6_rps_idle() will be called later to disable interrupts */
-}
-
 void intel_sanitize_gt_powersave(struct drm_i915_private *dev_priv)
 {
 	dev_priv->gt_pm.rps.enabled = true; /* force RPS disabling */
@@ -8458,6 +8668,8 @@ static void intel_enable_rc6(struct drm_i915_private *dev_priv)
 		cherryview_enable_rc6(dev_priv);
 	else if (IS_VALLEYVIEW(dev_priv))
 		valleyview_enable_rc6(dev_priv);
+	else if (INTEL_GEN(dev_priv) >= 11)
+		gen11_enable_rc6(dev_priv);
 	else if (INTEL_GEN(dev_priv) >= 9)
 		gen9_enable_rc6(dev_priv);
 	else if (IS_BROADWELL(dev_priv))
@@ -9361,7 +9573,7 @@ static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
  */
 void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
 {
-	if (IS_ICELAKE(dev_priv))
+	if (IS_GEN(dev_priv, 11))
 		dev_priv->display.init_clock_gating = icl_init_clock_gating;
 	else if (IS_CANNONLAKE(dev_priv))
 		dev_priv->display.init_clock_gating = cnl_init_clock_gating;
@@ -9454,7 +9666,7 @@ void intel_init_pm(struct drm_i915_private *dev_priv)
 		dev_priv->display.initial_watermarks = g4x_initial_watermarks;
 		dev_priv->display.optimize_watermarks = g4x_optimize_watermarks;
 	} else if (IS_PINEVIEW(dev_priv)) {
-		if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev_priv),
+		if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
 					    dev_priv->is_ddr3,
 					    dev_priv->fsb_freq,
 					    dev_priv->mem_freq)) {
@@ -9552,7 +9764,7 @@ int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val
 	I915_WRITE_FW(GEN6_PCODE_DATA1, 0);
 	I915_WRITE_FW(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
 
-	if (__intel_wait_for_register_fw(dev_priv,
+	if (__intel_wait_for_register_fw(&dev_priv->uncore,
 					 GEN6_PCODE_MAILBOX, GEN6_PCODE_READY, 0,
 					 500, 0, NULL)) {
 		DRM_ERROR("timeout waiting for pcode read (from mbox %x) to finish for %ps\n",
@@ -9600,7 +9812,7 @@ int sandybridge_pcode_write_timeout(struct drm_i915_private *dev_priv,
 	I915_WRITE_FW(GEN6_PCODE_DATA1, 0);
 	I915_WRITE_FW(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
 
-	if (__intel_wait_for_register_fw(dev_priv,
+	if (__intel_wait_for_register_fw(&dev_priv->uncore,
 					 GEN6_PCODE_MAILBOX, GEN6_PCODE_READY, 0,
 					 fast_timeout_us, slow_timeout_ms,
 					 NULL)) {
@@ -9824,6 +10036,7 @@ static u64 vlv_residency_raw(struct drm_i915_private *dev_priv,
 u64 intel_rc6_residency_ns(struct drm_i915_private *dev_priv,
 			   const i915_reg_t reg)
 {
+	struct intel_uncore *uncore = &dev_priv->uncore;
 	u64 time_hw, prev_hw, overflow_hw;
 	unsigned int fw_domains;
 	unsigned long flags;
@@ -9845,10 +10058,10 @@ u64 intel_rc6_residency_ns(struct drm_i915_private *dev_priv,
 	if (WARN_ON_ONCE(i >= ARRAY_SIZE(dev_priv->gt_pm.rc6.cur_residency)))
 		return 0;
 
-	fw_domains = intel_uncore_forcewake_for_reg(dev_priv, reg, FW_REG_READ);
+	fw_domains = intel_uncore_forcewake_for_reg(uncore, reg, FW_REG_READ);
 
-	spin_lock_irqsave(&dev_priv->uncore.lock, flags);
-	intel_uncore_forcewake_get__locked(dev_priv, fw_domains);
+	spin_lock_irqsave(&uncore->lock, flags);
+	intel_uncore_forcewake_get__locked(uncore, fw_domains);
 
 	/* On VLV and CHV, residency time is in CZ units rather than 1.28us */
 	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
@@ -9867,7 +10080,7 @@ u64 intel_rc6_residency_ns(struct drm_i915_private *dev_priv,
 		}
 
 		overflow_hw = BIT_ULL(32);
-		time_hw = I915_READ_FW(reg);
+		time_hw = intel_uncore_read_fw(uncore, reg);
 	}
 
 	/*
@@ -9889,8 +10102,8 @@ u64 intel_rc6_residency_ns(struct drm_i915_private *dev_priv,
 	time_hw += dev_priv->gt_pm.rc6.cur_residency[i];
 	dev_priv->gt_pm.rc6.cur_residency[i] = time_hw;
 
-	intel_uncore_forcewake_put__locked(dev_priv, fw_domains);
-	spin_unlock_irqrestore(&dev_priv->uncore.lock, flags);
+	intel_uncore_forcewake_put__locked(uncore, fw_domains);
+	spin_unlock_irqrestore(&uncore->lock, flags);
 
 	return mul_u64_u32_div(time_hw, mul, div);
 }
diff --git a/drivers/gpu/drm/i915/intel_pm.h b/drivers/gpu/drm/i915/intel_pm.h
new file mode 100644
index 000000000000..674a3f0f16a7
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_pm.h
@@ -0,0 +1,71 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2019 Intel Corporation
+ */
+
+#ifndef __INTEL_PM_H__
+#define __INTEL_PM_H__
+
+#include <linux/types.h>
+
+struct drm_atomic_state;
+struct drm_device;
+struct drm_i915_private;
+struct i915_request;
+struct intel_crtc;
+struct intel_crtc_state;
+struct intel_plane;
+struct skl_ddb_allocation;
+struct skl_ddb_entry;
+struct skl_pipe_wm;
+struct skl_wm_level;
+
+void intel_init_clock_gating(struct drm_i915_private *dev_priv);
+void intel_suspend_hw(struct drm_i915_private *dev_priv);
+int ilk_wm_max_level(const struct drm_i915_private *dev_priv);
+void intel_update_watermarks(struct intel_crtc *crtc);
+void intel_init_pm(struct drm_i915_private *dev_priv);
+void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv);
+void intel_pm_setup(struct drm_i915_private *dev_priv);
+void intel_gpu_ips_init(struct drm_i915_private *dev_priv);
+void intel_gpu_ips_teardown(void);
+void intel_init_gt_powersave(struct drm_i915_private *dev_priv);
+void intel_cleanup_gt_powersave(struct drm_i915_private *dev_priv);
+void intel_sanitize_gt_powersave(struct drm_i915_private *dev_priv);
+void intel_enable_gt_powersave(struct drm_i915_private *dev_priv);
+void intel_disable_gt_powersave(struct drm_i915_private *dev_priv);
+void gen6_rps_busy(struct drm_i915_private *dev_priv);
+void gen6_rps_idle(struct drm_i915_private *dev_priv);
+void gen6_rps_boost(struct i915_request *rq);
+void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv);
+void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv);
+void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv);
+void skl_wm_get_hw_state(struct drm_i915_private *dev_priv);
+void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
+			       struct skl_ddb_entry *ddb_y,
+			       struct skl_ddb_entry *ddb_uv);
+void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
+			  struct skl_ddb_allocation *ddb /* out */);
+void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
+			      struct skl_pipe_wm *out);
+void g4x_wm_sanitize(struct drm_i915_private *dev_priv);
+void vlv_wm_sanitize(struct drm_i915_private *dev_priv);
+bool intel_can_enable_sagv(struct drm_atomic_state *state);
+int intel_enable_sagv(struct drm_i915_private *dev_priv);
+int intel_disable_sagv(struct drm_i915_private *dev_priv);
+bool skl_wm_level_equals(const struct skl_wm_level *l1,
+			 const struct skl_wm_level *l2);
+bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
+				 const struct skl_ddb_entry *entries,
+				 int num_entries, int ignore_idx);
+void skl_write_plane_wm(struct intel_plane *plane,
+			const struct intel_crtc_state *crtc_state);
+void skl_write_cursor_wm(struct intel_plane *plane,
+			 const struct intel_crtc_state *crtc_state);
+bool ilk_disable_lp_wm(struct drm_device *dev);
+int skl_check_pipe_max_pixel_rate(struct intel_crtc *intel_crtc,
+				  struct intel_crtc_state *cstate);
+void intel_init_ipc(struct drm_i915_private *dev_priv);
+void intel_enable_ipc(struct drm_i915_private *dev_priv);
+
+#endif /* __INTEL_PM_H__ */
diff --git a/drivers/gpu/drm/i915/intel_psr.c b/drivers/gpu/drm/i915/intel_psr.c
index 84a0fb981561..963663ba0edf 100644
--- a/drivers/gpu/drm/i915/intel_psr.c
+++ b/drivers/gpu/drm/i915/intel_psr.c
@@ -21,6 +21,14 @@
  * DEALINGS IN THE SOFTWARE.
  */
 
+#include <drm/drm_atomic_helper.h>
+
+#include "i915_drv.h"
+#include "intel_dp.h"
+#include "intel_drv.h"
+#include "intel_psr.h"
+#include "intel_sprite.h"
+
 /**
  * DOC: Panel Self Refresh (PSR/SRD)
  *
@@ -51,10 +59,6 @@
  * must be correctly synchronized/cancelled when shutting down the pipe."
  */
 
-
-#include "intel_drv.h"
-#include "i915_drv.h"
-
 static bool psr_global_enabled(u32 debug)
 {
 	switch (debug & I915_PSR_DEBUG_MODE_MASK) {
@@ -78,9 +82,6 @@ static bool intel_psr2_enabled(struct drm_i915_private *dev_priv,
 	case I915_PSR_DEBUG_DISABLE:
 	case I915_PSR_DEBUG_FORCE_PSR1:
 		return false;
-	case I915_PSR_DEBUG_DEFAULT:
-		if (i915_modparams.enable_psr <= 0)
-			return false;
 	default:
 		return crtc_state->has_psr2;
 	}
@@ -435,32 +436,16 @@ static void intel_psr_enable_sink(struct intel_dp *intel_dp)
 	drm_dp_dpcd_writeb(&intel_dp->aux, DP_SET_POWER, DP_SET_POWER_D0);
 }
 
-static void hsw_activate_psr1(struct intel_dp *intel_dp)
+static u32 intel_psr1_get_tp_time(struct intel_dp *intel_dp)
 {
 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
-	u32 max_sleep_time = 0x1f;
-	u32 val = EDP_PSR_ENABLE;
+	u32 val = 0;
 
-	/* Let's use 6 as the minimum to cover all known cases including the
-	 * off-by-one issue that HW has in some cases.
-	 */
-	int idle_frames = max(6, dev_priv->vbt.psr.idle_frames);
-
-	/* sink_sync_latency of 8 means source has to wait for more than 8
-	 * frames, we'll go with 9 frames for now
-	 */
-	idle_frames = max(idle_frames, dev_priv->psr.sink_sync_latency + 1);
-	val |= idle_frames << EDP_PSR_IDLE_FRAME_SHIFT;
-
-	val |= max_sleep_time << EDP_PSR_MAX_SLEEP_TIME_SHIFT;
-	if (IS_HASWELL(dev_priv))
-		val |= EDP_PSR_MIN_LINK_ENTRY_TIME_8_LINES;
-
-	if (dev_priv->psr.link_standby)
-		val |= EDP_PSR_LINK_STANDBY;
+	if (INTEL_GEN(dev_priv) >= 11)
+		val |= EDP_PSR_TP4_TIME_0US;
 
 	if (dev_priv->vbt.psr.tp1_wakeup_time_us == 0)
-		val |=  EDP_PSR_TP1_TIME_0us;
+		val |= EDP_PSR_TP1_TIME_0us;
 	else if (dev_priv->vbt.psr.tp1_wakeup_time_us <= 100)
 		val |= EDP_PSR_TP1_TIME_100us;
 	else if (dev_priv->vbt.psr.tp1_wakeup_time_us <= 500)
@@ -469,7 +454,7 @@ static void hsw_activate_psr1(struct intel_dp *intel_dp)
 		val |= EDP_PSR_TP1_TIME_2500us;
 
 	if (dev_priv->vbt.psr.tp2_tp3_wakeup_time_us == 0)
-		val |=  EDP_PSR_TP2_TP3_TIME_0us;
+		val |= EDP_PSR_TP2_TP3_TIME_0us;
 	else if (dev_priv->vbt.psr.tp2_tp3_wakeup_time_us <= 100)
 		val |= EDP_PSR_TP2_TP3_TIME_100us;
 	else if (dev_priv->vbt.psr.tp2_tp3_wakeup_time_us <= 500)
@@ -483,6 +468,35 @@ static void hsw_activate_psr1(struct intel_dp *intel_dp)
 	else
 		val |= EDP_PSR_TP1_TP2_SEL;
 
+	return val;
+}
+
+static void hsw_activate_psr1(struct intel_dp *intel_dp)
+{
+	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
+	u32 max_sleep_time = 0x1f;
+	u32 val = EDP_PSR_ENABLE;
+
+	/* Let's use 6 as the minimum to cover all known cases including the
+	 * off-by-one issue that HW has in some cases.
+	 */
+	int idle_frames = max(6, dev_priv->vbt.psr.idle_frames);
+
+	/* sink_sync_latency of 8 means source has to wait for more than 8
+	 * frames, we'll go with 9 frames for now
+	 */
+	idle_frames = max(idle_frames, dev_priv->psr.sink_sync_latency + 1);
+	val |= idle_frames << EDP_PSR_IDLE_FRAME_SHIFT;
+
+	val |= max_sleep_time << EDP_PSR_MAX_SLEEP_TIME_SHIFT;
+	if (IS_HASWELL(dev_priv))
+		val |= EDP_PSR_MIN_LINK_ENTRY_TIME_8_LINES;
+
+	if (dev_priv->psr.link_standby)
+		val |= EDP_PSR_LINK_STANDBY;
+
+	val |= intel_psr1_get_tp_time(intel_dp);
+
 	if (INTEL_GEN(dev_priv) >= 8)
 		val |= EDP_PSR_CRC_ENABLE;
 
@@ -509,16 +523,22 @@ static void hsw_activate_psr2(struct intel_dp *intel_dp)
 
 	val |= EDP_PSR2_FRAME_BEFORE_SU(dev_priv->psr.sink_sync_latency + 1);
 
-	if (dev_priv->vbt.psr.tp2_tp3_wakeup_time_us >= 0 &&
-	    dev_priv->vbt.psr.tp2_tp3_wakeup_time_us <= 50)
+	if (dev_priv->vbt.psr.psr2_tp2_tp3_wakeup_time_us >= 0 &&
+	    dev_priv->vbt.psr.psr2_tp2_tp3_wakeup_time_us <= 50)
 		val |= EDP_PSR2_TP2_TIME_50us;
-	else if (dev_priv->vbt.psr.tp2_tp3_wakeup_time_us <= 100)
+	else if (dev_priv->vbt.psr.psr2_tp2_tp3_wakeup_time_us <= 100)
 		val |= EDP_PSR2_TP2_TIME_100us;
-	else if (dev_priv->vbt.psr.tp2_tp3_wakeup_time_us <= 500)
+	else if (dev_priv->vbt.psr.psr2_tp2_tp3_wakeup_time_us <= 500)
 		val |= EDP_PSR2_TP2_TIME_500us;
 	else
 		val |= EDP_PSR2_TP2_TIME_2500us;
 
+	/*
+	 * PSR2 HW is incorrectly using EDP_PSR_TP1_TP3_SEL and BSpec is
+	 * recommending keep this bit unset while PSR2 is enabled.
+	 */
+	I915_WRITE(EDP_PSR_CTL, 0);
+
 	I915_WRITE(EDP_PSR2_CTL, val);
 }
 
@@ -530,11 +550,6 @@ static bool intel_psr2_config_valid(struct intel_dp *intel_dp,
 	int crtc_vdisplay = crtc_state->base.adjusted_mode.crtc_vdisplay;
 	int psr_max_h = 0, psr_max_v = 0;
 
-	/*
-	 * FIXME psr2_support is messed up. It's both computed
-	 * dynamically during PSR enable, and extracted from sink
-	 * caps during eDP detection.
-	 */
 	if (!dev_priv->psr.sink_psr2_support)
 		return false;
 
@@ -575,6 +590,11 @@ static bool intel_psr2_config_valid(struct intel_dp *intel_dp,
 		return false;
 	}
 
+	if (crtc_state->crc_enabled) {
+		DRM_DEBUG_KMS("PSR2 not enabled because it would inhibit pipe CRC calculation\n");
+		return false;
+	}
+
 	return true;
 }
 
@@ -610,9 +630,8 @@ void intel_psr_compute_config(struct intel_dp *intel_dp,
 		return;
 	}
 
-	if (IS_HASWELL(dev_priv) &&
-	    adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
-		DRM_DEBUG_KMS("PSR condition failed: Interlaced is Enabled\n");
+	if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
+		DRM_DEBUG_KMS("PSR condition failed: Interlaced mode enabled\n");
 		return;
 	}
 
@@ -718,8 +737,11 @@ static void intel_psr_enable_locked(struct drm_i915_private *dev_priv,
 {
 	struct intel_dp *intel_dp = dev_priv->psr.dp;
 
-	if (dev_priv->psr.enabled)
-		return;
+	WARN_ON(dev_priv->psr.enabled);
+
+	dev_priv->psr.psr2_enabled = intel_psr2_enabled(dev_priv, crtc_state);
+	dev_priv->psr.busy_frontbuffer_bits = 0;
+	dev_priv->psr.pipe = to_intel_crtc(crtc_state->base.crtc)->pipe;
 
 	DRM_DEBUG_KMS("Enabling PSR%s\n",
 		      dev_priv->psr.psr2_enabled ? "2" : "1");
@@ -752,20 +774,13 @@ void intel_psr_enable(struct intel_dp *intel_dp,
 	WARN_ON(dev_priv->drrs.dp);
 
 	mutex_lock(&dev_priv->psr.lock);
-	if (dev_priv->psr.prepared) {
-		DRM_DEBUG_KMS("PSR already in use\n");
+
+	if (!psr_global_enabled(dev_priv->psr.debug)) {
+		DRM_DEBUG_KMS("PSR disabled by flag\n");
 		goto unlock;
 	}
 
-	dev_priv->psr.psr2_enabled = intel_psr2_enabled(dev_priv, crtc_state);
-	dev_priv->psr.busy_frontbuffer_bits = 0;
-	dev_priv->psr.prepared = true;
-	dev_priv->psr.pipe = to_intel_crtc(crtc_state->base.crtc)->pipe;
-
-	if (psr_global_enabled(dev_priv->psr.debug))
-		intel_psr_enable_locked(dev_priv, crtc_state);
-	else
-		DRM_DEBUG_KMS("PSR disabled by flag\n");
+	intel_psr_enable_locked(dev_priv, crtc_state);
 
 unlock:
 	mutex_unlock(&dev_priv->psr.lock);
@@ -819,8 +834,8 @@ static void intel_psr_disable_locked(struct intel_dp *intel_dp)
 	}
 
 	/* Wait till PSR is idle */
-	if (intel_wait_for_register(dev_priv, psr_status, psr_status_mask, 0,
-				    2000))
+	if (intel_wait_for_register(&dev_priv->uncore,
+				    psr_status, psr_status_mask, 0, 2000))
 		DRM_ERROR("Timed out waiting PSR idle state\n");
 
 	/* Disable PSR on Sink */
@@ -848,18 +863,69 @@ void intel_psr_disable(struct intel_dp *intel_dp,
 		return;
 
 	mutex_lock(&dev_priv->psr.lock);
-	if (!dev_priv->psr.prepared) {
-		mutex_unlock(&dev_priv->psr.lock);
-		return;
-	}
 
 	intel_psr_disable_locked(intel_dp);
 
-	dev_priv->psr.prepared = false;
 	mutex_unlock(&dev_priv->psr.lock);
 	cancel_work_sync(&dev_priv->psr.work);
 }
 
+static void psr_force_hw_tracking_exit(struct drm_i915_private *dev_priv)
+{
+	/*
+	 * Display WA #0884: all
+	 * This documented WA for bxt can be safely applied
+	 * broadly so we can force HW tracking to exit PSR
+	 * instead of disabling and re-enabling.
+	 * Workaround tells us to write 0 to CUR_SURFLIVE_A,
+	 * but it makes more sense write to the current active
+	 * pipe.
+	 */
+	I915_WRITE(CURSURFLIVE(dev_priv->psr.pipe), 0);
+}
+
+/**
+ * intel_psr_update - Update PSR state
+ * @intel_dp: Intel DP
+ * @crtc_state: new CRTC state
+ *
+ * This functions will update PSR states, disabling, enabling or switching PSR
+ * version when executing fastsets. For full modeset, intel_psr_disable() and
+ * intel_psr_enable() should be called instead.
+ */
+void intel_psr_update(struct intel_dp *intel_dp,
+		      const struct intel_crtc_state *crtc_state)
+{
+	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
+	struct i915_psr *psr = &dev_priv->psr;
+	bool enable, psr2_enable;
+
+	if (!CAN_PSR(dev_priv) || READ_ONCE(psr->dp) != intel_dp)
+		return;
+
+	mutex_lock(&dev_priv->psr.lock);
+
+	enable = crtc_state->has_psr && psr_global_enabled(psr->debug);
+	psr2_enable = intel_psr2_enabled(dev_priv, crtc_state);
+
+	if (enable == psr->enabled && psr2_enable == psr->psr2_enabled) {
+		/* Force a PSR exit when enabling CRC to avoid CRC timeouts */
+		if (crtc_state->crc_enabled && psr->enabled)
+			psr_force_hw_tracking_exit(dev_priv);
+
+		goto unlock;
+	}
+
+	if (psr->enabled)
+		intel_psr_disable_locked(intel_dp);
+
+	if (enable)
+		intel_psr_enable_locked(dev_priv, crtc_state);
+
+unlock:
+	mutex_unlock(&dev_priv->psr.lock);
+}
+
 /**
  * intel_psr_wait_for_idle - wait for PSR1 to idle
  * @new_crtc_state: new CRTC state
@@ -890,7 +956,7 @@ int intel_psr_wait_for_idle(const struct intel_crtc_state *new_crtc_state,
 	 * defensive enough to cover everything.
 	 */
 
-	return __intel_wait_for_register(dev_priv, EDP_PSR_STATUS,
+	return __intel_wait_for_register(&dev_priv->uncore, EDP_PSR_STATUS,
 					 EDP_PSR_STATUS_STATE_MASK,
 					 EDP_PSR_STATUS_STATE_IDLE, 2, 50,
 					 out_value);
@@ -915,7 +981,7 @@ static bool __psr_wait_for_idle_locked(struct drm_i915_private *dev_priv)
 
 	mutex_unlock(&dev_priv->psr.lock);
 
-	err = intel_wait_for_register(dev_priv, reg, mask, 0, 50);
+	err = intel_wait_for_register(&dev_priv->uncore, reg, mask, 0, 50);
 	if (err)
 		DRM_ERROR("Timed out waiting for PSR Idle for re-enable\n");
 
@@ -924,36 +990,63 @@ static bool __psr_wait_for_idle_locked(struct drm_i915_private *dev_priv)
 	return err == 0 && dev_priv->psr.enabled;
 }
 
-static bool switching_psr(struct drm_i915_private *dev_priv,
-			  struct intel_crtc_state *crtc_state,
-			  u32 mode)
+static int intel_psr_fastset_force(struct drm_i915_private *dev_priv)
 {
-	/* Can't switch psr state anyway if PSR2 is not supported. */
-	if (!crtc_state || !crtc_state->has_psr2)
-		return false;
+	struct drm_device *dev = &dev_priv->drm;
+	struct drm_modeset_acquire_ctx ctx;
+	struct drm_atomic_state *state;
+	struct drm_crtc *crtc;
+	int err;
 
-	if (dev_priv->psr.psr2_enabled && mode == I915_PSR_DEBUG_FORCE_PSR1)
-		return true;
+	state = drm_atomic_state_alloc(dev);
+	if (!state)
+		return -ENOMEM;
 
-	if (!dev_priv->psr.psr2_enabled && mode != I915_PSR_DEBUG_FORCE_PSR1)
-		return true;
+	drm_modeset_acquire_init(&ctx, DRM_MODESET_ACQUIRE_INTERRUPTIBLE);
+	state->acquire_ctx = &ctx;
+
+retry:
+	drm_for_each_crtc(crtc, dev) {
+		struct drm_crtc_state *crtc_state;
+		struct intel_crtc_state *intel_crtc_state;
+
+		crtc_state = drm_atomic_get_crtc_state(state, crtc);
+		if (IS_ERR(crtc_state)) {
+			err = PTR_ERR(crtc_state);
+			goto error;
+		}
+
+		intel_crtc_state = to_intel_crtc_state(crtc_state);
 
-	return false;
+		if (crtc_state->active && intel_crtc_state->has_psr) {
+			/* Mark mode as changed to trigger a pipe->update() */
+			crtc_state->mode_changed = true;
+			break;
+		}
+	}
+
+	err = drm_atomic_commit(state);
+
+error:
+	if (err == -EDEADLK) {
+		drm_atomic_state_clear(state);
+		err = drm_modeset_backoff(&ctx);
+		if (!err)
+			goto retry;
+	}
+
+	drm_modeset_drop_locks(&ctx);
+	drm_modeset_acquire_fini(&ctx);
+	drm_atomic_state_put(state);
+
+	return err;
 }
 
-int intel_psr_set_debugfs_mode(struct drm_i915_private *dev_priv,
-			       struct drm_modeset_acquire_ctx *ctx,
-			       u64 val)
+int intel_psr_debug_set(struct drm_i915_private *dev_priv, u64 val)
 {
-	struct drm_device *dev = &dev_priv->drm;
-	struct drm_connector_state *conn_state;
-	struct intel_crtc_state *crtc_state = NULL;
-	struct drm_crtc_commit *commit;
-	struct drm_crtc *crtc;
-	struct intel_dp *dp;
+	const u32 mode = val & I915_PSR_DEBUG_MODE_MASK;
+	u32 old_mode;
 	int ret;
-	bool enable;
-	u32 mode = val & I915_PSR_DEBUG_MODE_MASK;
 
 	if (val & ~(I915_PSR_DEBUG_IRQ | I915_PSR_DEBUG_MODE_MASK) ||
 	    mode > I915_PSR_DEBUG_FORCE_PSR1) {
@@ -961,49 +1054,19 @@ int intel_psr_set_debugfs_mode(struct drm_i915_private *dev_priv,
 		return -EINVAL;
 	}
 
-	ret = drm_modeset_lock(&dev->mode_config.connection_mutex, ctx);
-	if (ret)
-		return ret;
-
-	/* dev_priv->psr.dp should be set once and then never touched again. */
-	dp = READ_ONCE(dev_priv->psr.dp);
-	conn_state = dp->attached_connector->base.state;
-	crtc = conn_state->crtc;
-	if (crtc) {
-		ret = drm_modeset_lock(&crtc->mutex, ctx);
-		if (ret)
-			return ret;
-
-		crtc_state = to_intel_crtc_state(crtc->state);
-		commit = crtc_state->base.commit;
-	} else {
-		commit = conn_state->commit;
-	}
-	if (commit) {
-		ret = wait_for_completion_interruptible(&commit->hw_done);
-		if (ret)
-			return ret;
-	}
-
 	ret = mutex_lock_interruptible(&dev_priv->psr.lock);
 	if (ret)
 		return ret;
 
-	enable = psr_global_enabled(val);
-
-	if (!enable || switching_psr(dev_priv, crtc_state, mode))
-		intel_psr_disable_locked(dev_priv->psr.dp);
-
+	old_mode = dev_priv->psr.debug & I915_PSR_DEBUG_MODE_MASK;
 	dev_priv->psr.debug = val;
-	if (crtc)
-		dev_priv->psr.psr2_enabled = intel_psr2_enabled(dev_priv, crtc_state);
-
 	intel_psr_irq_control(dev_priv, dev_priv->psr.debug);
 
-	if (dev_priv->psr.prepared && enable)
-		intel_psr_enable_locked(dev_priv, crtc_state);
-
 	mutex_unlock(&dev_priv->psr.lock);
+
+	if (old_mode != mode)
+		ret = intel_psr_fastset_force(dev_priv);
+
 	return ret;
 }
 
@@ -1121,18 +1184,8 @@ void intel_psr_flush(struct drm_i915_private *dev_priv,
 	dev_priv->psr.busy_frontbuffer_bits &= ~frontbuffer_bits;
 
 	/* By definition flush = invalidate + flush */
-	if (frontbuffer_bits) {
-		/*
-		 * Display WA #0884: all
-		 * This documented WA for bxt can be safely applied
-		 * broadly so we can force HW tracking to exit PSR
-		 * instead of disabling and re-enabling.
-		 * Workaround tells us to write 0 to CUR_SURFLIVE_A,
-		 * but it makes more sense write to the current active
-		 * pipe.
-		 */
-		I915_WRITE(CURSURFLIVE(dev_priv->psr.pipe), 0);
-	}
+	if (frontbuffer_bits)
+		psr_force_hw_tracking_exit(dev_priv);
 
 	if (!dev_priv->psr.active && !dev_priv->psr.busy_frontbuffer_bits)
 		schedule_work(&dev_priv->psr.work);
@@ -1176,7 +1229,6 @@ void intel_psr_init(struct drm_i915_private *dev_priv)
 	if (val) {
 		DRM_DEBUG_KMS("PSR interruption error set\n");
 		dev_priv->psr.sink_not_reliable = true;
-		return;
 	}
 
 	/* Set link_standby x link_off defaults */
diff --git a/drivers/gpu/drm/i915/intel_psr.h b/drivers/gpu/drm/i915/intel_psr.h
new file mode 100644
index 000000000000..dc818826f36d
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_psr.h
@@ -0,0 +1,40 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2019 Intel Corporation
+ */
+
+#ifndef __INTEL_PSR_H__
+#define __INTEL_PSR_H__
+
+#include "intel_frontbuffer.h"
+
+struct drm_i915_private;
+struct intel_crtc_state;
+struct intel_dp;
+
+#define CAN_PSR(dev_priv) (HAS_PSR(dev_priv) && dev_priv->psr.sink_support)
+void intel_psr_init_dpcd(struct intel_dp *intel_dp);
+void intel_psr_enable(struct intel_dp *intel_dp,
+		      const struct intel_crtc_state *crtc_state);
+void intel_psr_disable(struct intel_dp *intel_dp,
+		       const struct intel_crtc_state *old_crtc_state);
+void intel_psr_update(struct intel_dp *intel_dp,
+		      const struct intel_crtc_state *crtc_state);
+int intel_psr_debug_set(struct drm_i915_private *dev_priv, u64 value);
+void intel_psr_invalidate(struct drm_i915_private *dev_priv,
+			  unsigned frontbuffer_bits,
+			  enum fb_op_origin origin);
+void intel_psr_flush(struct drm_i915_private *dev_priv,
+		     unsigned frontbuffer_bits,
+		     enum fb_op_origin origin);
+void intel_psr_init(struct drm_i915_private *dev_priv);
+void intel_psr_compute_config(struct intel_dp *intel_dp,
+			      struct intel_crtc_state *crtc_state);
+void intel_psr_irq_control(struct drm_i915_private *dev_priv, u32 debug);
+void intel_psr_irq_handler(struct drm_i915_private *dev_priv, u32 psr_iir);
+void intel_psr_short_pulse(struct intel_dp *intel_dp);
+int intel_psr_wait_for_idle(const struct intel_crtc_state *new_crtc_state,
+			    u32 *out_value);
+bool intel_psr_enabled(struct intel_dp *intel_dp);
+
+#endif /* __INTEL_PSR_H__ */
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index 7f841dba87b3..029fd8ec1857 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -43,12 +43,6 @@
  */
 #define LEGACY_REQUEST_SIZE 200
 
-static inline u32 intel_hws_seqno_address(struct intel_engine_cs *engine)
-{
-	return (i915_ggtt_offset(engine->status_page.vma) +
-		I915_GEM_HWS_INDEX_ADDR);
-}
-
 unsigned int intel_ring_update_space(struct intel_ring *ring)
 {
 	unsigned int space;
@@ -317,9 +311,9 @@ static u32 *gen6_rcs_emit_breadcrumb(struct i915_request *rq, u32 *cs)
 	*cs++ = rq->fence.seqno;
 
 	*cs++ = GFX_OP_PIPE_CONTROL(4);
-	*cs++ = PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL;
-	*cs++ = intel_hws_seqno_address(rq->engine) | PIPE_CONTROL_GLOBAL_GTT;
-	*cs++ = rq->global_seqno;
+	*cs++ = PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_STORE_DATA_INDEX;
+	*cs++ = I915_GEM_HWS_HANGCHECK_ADDR | PIPE_CONTROL_GLOBAL_GTT;
+	*cs++ = intel_engine_next_hangcheck_seqno(rq->engine);
 
 	*cs++ = MI_USER_INTERRUPT;
 	*cs++ = MI_NOOP;
@@ -424,10 +418,10 @@ static u32 *gen7_rcs_emit_breadcrumb(struct i915_request *rq, u32 *cs)
 
 	*cs++ = GFX_OP_PIPE_CONTROL(4);
 	*cs++ = (PIPE_CONTROL_QW_WRITE |
-		 PIPE_CONTROL_GLOBAL_GTT_IVB |
-		 PIPE_CONTROL_CS_STALL);
-	*cs++ = intel_hws_seqno_address(rq->engine);
-	*cs++ = rq->global_seqno;
+		 PIPE_CONTROL_STORE_DATA_INDEX |
+		 PIPE_CONTROL_GLOBAL_GTT_IVB);
+	*cs++ = I915_GEM_HWS_HANGCHECK_ADDR;
+	*cs++ = intel_engine_next_hangcheck_seqno(rq->engine);
 
 	*cs++ = MI_USER_INTERRUPT;
 	*cs++ = MI_NOOP;
@@ -448,8 +442,8 @@ static u32 *gen6_xcs_emit_breadcrumb(struct i915_request *rq, u32 *cs)
 	*cs++ = rq->fence.seqno;
 
 	*cs++ = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_DW_STORE_INDEX;
-	*cs++ = I915_GEM_HWS_INDEX_ADDR | MI_FLUSH_DW_USE_GTT;
-	*cs++ = rq->global_seqno;
+	*cs++ = I915_GEM_HWS_HANGCHECK_ADDR | MI_FLUSH_DW_USE_GTT;
+	*cs++ = intel_engine_next_hangcheck_seqno(rq->engine);
 
 	*cs++ = MI_USER_INTERRUPT;
 	*cs++ = MI_NOOP;
@@ -473,8 +467,8 @@ static u32 *gen7_xcs_emit_breadcrumb(struct i915_request *rq, u32 *cs)
 	*cs++ = rq->fence.seqno;
 
 	*cs++ = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_DW_STORE_INDEX;
-	*cs++ = I915_GEM_HWS_INDEX_ADDR | MI_FLUSH_DW_USE_GTT;
-	*cs++ = rq->global_seqno;
+	*cs++ = I915_GEM_HWS_HANGCHECK_ADDR | MI_FLUSH_DW_USE_GTT;
+	*cs++ = intel_engine_next_hangcheck_seqno(rq->engine);
 
 	for (i = 0; i < GEN7_XCS_WA; i++) {
 		*cs++ = MI_STORE_DWORD_INDEX;
@@ -554,16 +548,17 @@ static void set_hwsp(struct intel_engine_cs *engine, u32 offset)
 		 */
 		default:
 			GEM_BUG_ON(engine->id);
-		case RCS:
+			/* fallthrough */
+		case RCS0:
 			hwsp = RENDER_HWS_PGA_GEN7;
 			break;
-		case BCS:
+		case BCS0:
 			hwsp = BLT_HWS_PGA_GEN7;
 			break;
-		case VCS:
+		case VCS0:
 			hwsp = BSD_HWS_PGA_GEN7;
 			break;
-		case VECS:
+		case VECS0:
 			hwsp = VEBOX_HWS_PGA_GEN7;
 			break;
 		}
@@ -580,19 +575,19 @@ static void set_hwsp(struct intel_engine_cs *engine, u32 offset)
 static void flush_cs_tlb(struct intel_engine_cs *engine)
 {
 	struct drm_i915_private *dev_priv = engine->i915;
-	i915_reg_t instpm = RING_INSTPM(engine->mmio_base);
 
 	if (!IS_GEN_RANGE(dev_priv, 6, 7))
 		return;
 
 	/* ring should be idle before issuing a sync flush*/
-	WARN_ON((I915_READ_MODE(engine) & MODE_IDLE) == 0);
-
-	I915_WRITE(instpm,
-		   _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE |
-				      INSTPM_SYNC_FLUSH));
-	if (intel_wait_for_register(dev_priv,
-				    instpm, INSTPM_SYNC_FLUSH, 0,
+	WARN_ON((ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE) == 0);
+
+	ENGINE_WRITE(engine, RING_INSTPM,
+		     _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE |
+					INSTPM_SYNC_FLUSH));
+	if (intel_wait_for_register(engine->uncore,
+				    RING_INSTPM(engine->mmio_base),
+				    INSTPM_SYNC_FLUSH, 0,
 				    1000))
 		DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n",
 			  engine->name);
@@ -611,32 +606,36 @@ static bool stop_ring(struct intel_engine_cs *engine)
 	struct drm_i915_private *dev_priv = engine->i915;
 
 	if (INTEL_GEN(dev_priv) > 2) {
-		I915_WRITE_MODE(engine, _MASKED_BIT_ENABLE(STOP_RING));
-		if (intel_wait_for_register(dev_priv,
+		ENGINE_WRITE(engine,
+			     RING_MI_MODE, _MASKED_BIT_ENABLE(STOP_RING));
+		if (intel_wait_for_register(engine->uncore,
 					    RING_MI_MODE(engine->mmio_base),
 					    MODE_IDLE,
 					    MODE_IDLE,
 					    1000)) {
 			DRM_ERROR("%s : timed out trying to stop ring\n",
 				  engine->name);
-			/* Sometimes we observe that the idle flag is not
+
+			/*
+			 * Sometimes we observe that the idle flag is not
 			 * set even though the ring is empty. So double
 			 * check before giving up.
 			 */
-			if (I915_READ_HEAD(engine) != I915_READ_TAIL(engine))
+			if (ENGINE_READ(engine, RING_HEAD) !=
+			    ENGINE_READ(engine, RING_TAIL))
 				return false;
 		}
 	}
 
-	I915_WRITE_HEAD(engine, I915_READ_TAIL(engine));
+	ENGINE_WRITE(engine, RING_HEAD, ENGINE_READ(engine, RING_TAIL));
 
-	I915_WRITE_HEAD(engine, 0);
-	I915_WRITE_TAIL(engine, 0);
+	ENGINE_WRITE(engine, RING_HEAD, 0);
+	ENGINE_WRITE(engine, RING_TAIL, 0);
 
 	/* The ring must be empty before it is disabled */
-	I915_WRITE_CTL(engine, 0);
+	ENGINE_WRITE(engine, RING_CTL, 0);
 
-	return (I915_READ_HEAD(engine) & HEAD_ADDR) == 0;
+	return (ENGINE_READ(engine, RING_HEAD) & HEAD_ADDR) == 0;
 }
 
 static int init_ring_common(struct intel_engine_cs *engine)
@@ -645,26 +644,26 @@ static int init_ring_common(struct intel_engine_cs *engine)
 	struct intel_ring *ring = engine->buffer;
 	int ret = 0;
 
-	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_get(engine->uncore, FORCEWAKE_ALL);
 
 	if (!stop_ring(engine)) {
 		/* G45 ring initialization often fails to reset head to zero */
 		DRM_DEBUG_DRIVER("%s head not reset to zero "
 				"ctl %08x head %08x tail %08x start %08x\n",
 				engine->name,
-				I915_READ_CTL(engine),
-				I915_READ_HEAD(engine),
-				I915_READ_TAIL(engine),
-				I915_READ_START(engine));
+				ENGINE_READ(engine, RING_CTL),
+				ENGINE_READ(engine, RING_HEAD),
+				ENGINE_READ(engine, RING_TAIL),
+				ENGINE_READ(engine, RING_START));
 
 		if (!stop_ring(engine)) {
 			DRM_ERROR("failed to set %s head to zero "
 				  "ctl %08x head %08x tail %08x start %08x\n",
 				  engine->name,
-				  I915_READ_CTL(engine),
-				  I915_READ_HEAD(engine),
-				  I915_READ_TAIL(engine),
-				  I915_READ_START(engine));
+				  ENGINE_READ(engine, RING_CTL),
+				  ENGINE_READ(engine, RING_HEAD),
+				  ENGINE_READ(engine, RING_TAIL),
+				  ENGINE_READ(engine, RING_START));
 			ret = -EIO;
 			goto out;
 		}
@@ -678,18 +677,18 @@ static int init_ring_common(struct intel_engine_cs *engine)
 	intel_engine_reset_breadcrumbs(engine);
 
 	/* Enforce ordering by reading HEAD register back */
-	I915_READ_HEAD(engine);
+	ENGINE_READ(engine, RING_HEAD);
 
 	/* Initialize the ring. This must happen _after_ we've cleared the ring
 	 * registers with the above sequence (the readback of the HEAD registers
 	 * also enforces ordering), otherwise the hw might lose the new ring
 	 * register values. */
-	I915_WRITE_START(engine, i915_ggtt_offset(ring->vma));
+	ENGINE_WRITE(engine, RING_START, i915_ggtt_offset(ring->vma));
 
 	/* WaClearRingBufHeadRegAtInit:ctg,elk */
-	if (I915_READ_HEAD(engine))
+	if (ENGINE_READ(engine, RING_HEAD))
 		DRM_DEBUG_DRIVER("%s initialization failed [head=%08x], fudging\n",
-				 engine->name, I915_READ_HEAD(engine));
+				 engine->name, ENGINE_READ(engine, RING_HEAD));
 
 	/* Check that the ring offsets point within the ring! */
 	GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->head));
@@ -697,42 +696,44 @@ static int init_ring_common(struct intel_engine_cs *engine)
 	intel_ring_update_space(ring);
 
 	/* First wake the ring up to an empty/idle ring */
-	I915_WRITE_HEAD(engine, ring->head);
-	I915_WRITE_TAIL(engine, ring->head);
-	(void)I915_READ_TAIL(engine);
+	ENGINE_WRITE(engine, RING_HEAD, ring->head);
+	ENGINE_WRITE(engine, RING_TAIL, ring->head);
+	ENGINE_POSTING_READ(engine, RING_TAIL);
 
-	I915_WRITE_CTL(engine, RING_CTL_SIZE(ring->size) | RING_VALID);
+	ENGINE_WRITE(engine, RING_CTL, RING_CTL_SIZE(ring->size) | RING_VALID);
 
 	/* If the head is still not zero, the ring is dead */
-	if (intel_wait_for_register(dev_priv, RING_CTL(engine->mmio_base),
+	if (intel_wait_for_register(engine->uncore,
+				    RING_CTL(engine->mmio_base),
 				    RING_VALID, RING_VALID,
 				    50)) {
 		DRM_ERROR("%s initialization failed "
 			  "ctl %08x (valid? %d) head %08x [%08x] tail %08x [%08x] start %08x [expected %08x]\n",
 			  engine->name,
-			  I915_READ_CTL(engine),
-			  I915_READ_CTL(engine) & RING_VALID,
-			  I915_READ_HEAD(engine), ring->head,
-			  I915_READ_TAIL(engine), ring->tail,
-			  I915_READ_START(engine),
+			  ENGINE_READ(engine, RING_CTL),
+			  ENGINE_READ(engine, RING_CTL) & RING_VALID,
+			  ENGINE_READ(engine, RING_HEAD), ring->head,
+			  ENGINE_READ(engine, RING_TAIL), ring->tail,
+			  ENGINE_READ(engine, RING_START),
 			  i915_ggtt_offset(ring->vma));
 		ret = -EIO;
 		goto out;
 	}
 
 	if (INTEL_GEN(dev_priv) > 2)
-		I915_WRITE_MODE(engine, _MASKED_BIT_DISABLE(STOP_RING));
+		ENGINE_WRITE(engine,
+			     RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING));
 
 	/* Now awake, let it get started */
 	if (ring->tail != ring->head) {
-		I915_WRITE_TAIL(engine, ring->tail);
-		(void)I915_READ_TAIL(engine);
+		ENGINE_WRITE(engine, RING_TAIL, ring->tail);
+		ENGINE_POSTING_READ(engine, RING_TAIL);
 	}
 
 	/* Papering over lost _interrupts_ immediately following the restart */
 	intel_engine_queue_breadcrumbs(engine);
 out:
-	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_put(engine->uncore, FORCEWAKE_ALL);
 
 	return ret;
 }
@@ -758,11 +759,6 @@ static void reset_ring(struct intel_engine_cs *engine, bool stalled)
 		}
 	}
 
-	GEM_TRACE("%s seqno=%d, current=%d, stalled? %s\n",
-		  engine->name,
-		  rq ? rq->global_seqno : 0,
-		  intel_engine_get_seqno(engine),
-		  yesno(stalled));
 	/*
 	 * The guilty request will get skipped on a hung engine.
 	 *
@@ -878,7 +874,7 @@ static int init_render_ring(struct intel_engine_cs *engine)
 		I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
 
 	if (INTEL_GEN(dev_priv) >= 6)
-		I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
+		ENGINE_WRITE(engine, RING_IMR, ~engine->irq_keep_mask);
 
 	return 0;
 }
@@ -892,18 +888,12 @@ static void cancel_requests(struct intel_engine_cs *engine)
 
 	/* Mark all submitted requests as skipped. */
 	list_for_each_entry(request, &engine->timeline.requests, link) {
-		GEM_BUG_ON(!request->global_seqno);
-
 		if (!i915_request_signaled(request))
 			dma_fence_set_error(&request->fence, -EIO);
 
 		i915_request_mark_complete(request);
 	}
 
-	intel_write_status_page(engine,
-				I915_GEM_HWS_INDEX,
-				intel_engine_last_submit(engine));
-
 	/* Remaining _unready_ requests will be nop'ed when submitted */
 
 	spin_unlock_irqrestore(&engine->timeline.lock, flags);
@@ -911,12 +901,10 @@ static void cancel_requests(struct intel_engine_cs *engine)
 
 static void i9xx_submit_request(struct i915_request *request)
 {
-	struct drm_i915_private *dev_priv = request->i915;
-
 	i915_request_submit(request);
 
-	I915_WRITE_TAIL(request->engine,
-			intel_ring_set_tail(request->ring, request->tail));
+	ENGINE_WRITE(request->engine, RING_TAIL,
+		     intel_ring_set_tail(request->ring, request->tail));
 }
 
 static u32 *i9xx_emit_breadcrumb(struct i915_request *rq, u32 *cs)
@@ -931,8 +919,8 @@ static u32 *i9xx_emit_breadcrumb(struct i915_request *rq, u32 *cs)
 	*cs++ = rq->fence.seqno;
 
 	*cs++ = MI_STORE_DWORD_INDEX;
-	*cs++ = I915_GEM_HWS_INDEX_ADDR;
-	*cs++ = rq->global_seqno;
+	*cs++ = I915_GEM_HWS_HANGCHECK_ADDR;
+	*cs++ = intel_engine_next_hangcheck_seqno(rq->engine);
 
 	*cs++ = MI_USER_INTERRUPT;
 
@@ -953,14 +941,14 @@ static u32 *gen5_emit_breadcrumb(struct i915_request *rq, u32 *cs)
 	*cs++ = MI_FLUSH;
 
 	*cs++ = MI_STORE_DWORD_INDEX;
-	*cs++ = I915_GEM_HWS_SEQNO_ADDR;
-	*cs++ = rq->fence.seqno;
+	*cs++ = I915_GEM_HWS_HANGCHECK_ADDR;
+	*cs++ = intel_engine_next_hangcheck_seqno(rq->engine);
 
 	BUILD_BUG_ON(GEN5_WA_STORES < 1);
 	for (i = 0; i < GEN5_WA_STORES; i++) {
 		*cs++ = MI_STORE_DWORD_INDEX;
-		*cs++ = I915_GEM_HWS_INDEX_ADDR;
-		*cs++ = rq->global_seqno;
+		*cs++ = I915_GEM_HWS_SEQNO_ADDR;
+		*cs++ = rq->fence.seqno;
 	}
 
 	*cs++ = MI_USER_INTERRUPT;
@@ -988,20 +976,16 @@ gen5_irq_disable(struct intel_engine_cs *engine)
 static void
 i9xx_irq_enable(struct intel_engine_cs *engine)
 {
-	struct drm_i915_private *dev_priv = engine->i915;
-
-	dev_priv->irq_mask &= ~engine->irq_enable_mask;
-	I915_WRITE(IMR, dev_priv->irq_mask);
-	POSTING_READ_FW(RING_IMR(engine->mmio_base));
+	engine->i915->irq_mask &= ~engine->irq_enable_mask;
+	intel_uncore_write(engine->uncore, GEN2_IMR, engine->i915->irq_mask);
+	intel_uncore_posting_read_fw(engine->uncore, GEN2_IMR);
 }
 
 static void
 i9xx_irq_disable(struct intel_engine_cs *engine)
 {
-	struct drm_i915_private *dev_priv = engine->i915;
-
-	dev_priv->irq_mask |= engine->irq_enable_mask;
-	I915_WRITE(IMR, dev_priv->irq_mask);
+	engine->i915->irq_mask |= engine->irq_enable_mask;
+	intel_uncore_write(engine->uncore, GEN2_IMR, engine->i915->irq_mask);
 }
 
 static void
@@ -1010,7 +994,7 @@ i8xx_irq_enable(struct intel_engine_cs *engine)
 	struct drm_i915_private *dev_priv = engine->i915;
 
 	dev_priv->irq_mask &= ~engine->irq_enable_mask;
-	I915_WRITE16(IMR, dev_priv->irq_mask);
+	I915_WRITE16(GEN2_IMR, dev_priv->irq_mask);
 	POSTING_READ16(RING_IMR(engine->mmio_base));
 }
 
@@ -1020,7 +1004,7 @@ i8xx_irq_disable(struct intel_engine_cs *engine)
 	struct drm_i915_private *dev_priv = engine->i915;
 
 	dev_priv->irq_mask |= engine->irq_enable_mask;
-	I915_WRITE16(IMR, dev_priv->irq_mask);
+	I915_WRITE16(GEN2_IMR, dev_priv->irq_mask);
 }
 
 static int
@@ -1041,47 +1025,38 @@ bsd_ring_flush(struct i915_request *rq, u32 mode)
 static void
 gen6_irq_enable(struct intel_engine_cs *engine)
 {
-	struct drm_i915_private *dev_priv = engine->i915;
-
-	I915_WRITE_IMR(engine,
-		       ~(engine->irq_enable_mask |
-			 engine->irq_keep_mask));
+	ENGINE_WRITE(engine, RING_IMR,
+		     ~(engine->irq_enable_mask | engine->irq_keep_mask));
 
 	/* Flush/delay to ensure the RING_IMR is active before the GT IMR */
-	POSTING_READ_FW(RING_IMR(engine->mmio_base));
+	ENGINE_POSTING_READ(engine, RING_IMR);
 
-	gen5_enable_gt_irq(dev_priv, engine->irq_enable_mask);
+	gen5_enable_gt_irq(engine->i915, engine->irq_enable_mask);
 }
 
 static void
 gen6_irq_disable(struct intel_engine_cs *engine)
 {
-	struct drm_i915_private *dev_priv = engine->i915;
-
-	I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
-	gen5_disable_gt_irq(dev_priv, engine->irq_enable_mask);
+	ENGINE_WRITE(engine, RING_IMR, ~engine->irq_keep_mask);
+	gen5_disable_gt_irq(engine->i915, engine->irq_enable_mask);
 }
 
 static void
 hsw_vebox_irq_enable(struct intel_engine_cs *engine)
 {
-	struct drm_i915_private *dev_priv = engine->i915;
-
-	I915_WRITE_IMR(engine, ~engine->irq_enable_mask);
+	ENGINE_WRITE(engine, RING_IMR, ~engine->irq_enable_mask);
 
 	/* Flush/delay to ensure the RING_IMR is active before the GT IMR */
-	POSTING_READ_FW(RING_IMR(engine->mmio_base));
+	ENGINE_POSTING_READ(engine, RING_IMR);
 
-	gen6_unmask_pm_irq(dev_priv, engine->irq_enable_mask);
+	gen6_unmask_pm_irq(engine->i915, engine->irq_enable_mask);
 }
 
 static void
 hsw_vebox_irq_disable(struct intel_engine_cs *engine)
 {
-	struct drm_i915_private *dev_priv = engine->i915;
-
-	I915_WRITE_IMR(engine, ~0);
-	gen6_mask_pm_irq(dev_priv, engine->irq_enable_mask);
+	ENGINE_WRITE(engine, RING_IMR, ~0);
+	gen6_mask_pm_irq(engine->i915, engine->irq_enable_mask);
 }
 
 static int
@@ -1211,15 +1186,6 @@ int intel_ring_pin(struct intel_ring *ring)
 	else
 		flags |= PIN_HIGH;
 
-	if (!(vma->flags & I915_VMA_GLOBAL_BIND)) {
-		if (flags & PIN_MAPPABLE || map == I915_MAP_WC)
-			ret = i915_gem_object_set_to_gtt_domain(vma->obj, true);
-		else
-			ret = i915_gem_object_set_to_cpu_domain(vma->obj, true);
-		if (unlikely(ret))
-			goto unpin_timeline;
-	}
-
 	ret = i915_vma_pin(vma, 0, 0, flags);
 	if (unlikely(ret))
 		goto unpin_timeline;
@@ -1323,6 +1289,7 @@ intel_engine_create_ring(struct intel_engine_cs *engine,
 	if (!ring)
 		return ERR_PTR(-ENOMEM);
 
+	kref_init(&ring->ref);
 	INIT_LIST_HEAD(&ring->request_list);
 	ring->timeline = i915_timeline_get(timeline);
 
@@ -1347,9 +1314,9 @@ intel_engine_create_ring(struct intel_engine_cs *engine,
 	return ring;
 }
 
-void
-intel_ring_free(struct intel_ring *ring)
+void intel_ring_free(struct kref *ref)
 {
+	struct intel_ring *ring = container_of(ref, typeof(*ring), ref);
 	struct drm_i915_gem_object *obj = ring->vma->obj;
 
 	i915_vma_close(ring->vma);
@@ -1359,17 +1326,24 @@ intel_ring_free(struct intel_ring *ring)
 	kfree(ring);
 }
 
-static void intel_ring_context_destroy(struct intel_context *ce)
+static void __ring_context_fini(struct intel_context *ce)
 {
-	GEM_BUG_ON(ce->pin_count);
-
-	if (!ce->state)
-		return;
-
 	GEM_BUG_ON(i915_gem_object_is_active(ce->state->obj));
 	i915_gem_object_put(ce->state->obj);
 }
 
+static void ring_context_destroy(struct kref *ref)
+{
+	struct intel_context *ce = container_of(ref, typeof(*ce), ref);
+
+	GEM_BUG_ON(intel_context_is_pinned(ce));
+
+	if (ce->state)
+		__ring_context_fini(ce);
+
+	intel_context_free(ce);
+}
+
 static int __context_pin_ppgtt(struct i915_gem_context *ctx)
 {
 	struct i915_hw_ppgtt *ppgtt;
@@ -1400,17 +1374,6 @@ static int __context_pin(struct intel_context *ce)
 	if (!vma)
 		return 0;
 
-	/*
-	 * Clear this page out of any CPU caches for coherent swap-in/out.
-	 * We only want to do this on the first bind so that we do not stall
-	 * on an active context (which by nature is already on the GPU).
-	 */
-	if (!(vma->flags & I915_VMA_GLOBAL_BIND)) {
-		err = i915_gem_object_set_to_gtt_domain(vma->obj, true);
-		if (err)
-			return err;
-	}
-
 	err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL | PIN_HIGH);
 	if (err)
 		return err;
@@ -1420,6 +1383,7 @@ static int __context_pin(struct intel_context *ce)
 	 * it cannot reclaim the object until we release it.
 	 */
 	vma->obj->pin_global++;
+	vma->obj->mm.dirty = true;
 
 	return 0;
 }
@@ -1436,12 +1400,10 @@ static void __context_unpin(struct intel_context *ce)
 	i915_vma_unpin(vma);
 }
 
-static void intel_ring_context_unpin(struct intel_context *ce)
+static void ring_context_unpin(struct intel_context *ce)
 {
 	__context_unpin_ppgtt(ce->gem_context);
 	__context_unpin(ce);
-
-	i915_gem_context_put(ce->gem_context);
 }
 
 static struct i915_vma *
@@ -1456,6 +1418,24 @@ alloc_context_vma(struct intel_engine_cs *engine)
 	if (IS_ERR(obj))
 		return ERR_CAST(obj);
 
+	/*
+	 * Try to make the context utilize L3 as well as LLC.
+	 *
+	 * On VLV we don't have L3 controls in the PTEs so we
+	 * shouldn't touch the cache level, especially as that
+	 * would make the object snooped which might have a
+	 * negative performance impact.
+	 *
+	 * Snooping is required on non-llc platforms in execlist
+	 * mode, but since all GGTT accesses use PAT entry 0 we
+	 * get snooping anyway regardless of cache_level.
+	 *
+	 * This is only applicable for Ivy Bridge devices since
+	 * later platforms don't have L3 control bits in the PTE.
+	 */
+	if (IS_IVYBRIDGE(i915))
+		i915_gem_object_set_cache_coherency(obj, I915_CACHE_L3_LLC);
+
 	if (engine->default_state) {
 		void *defaults, *vaddr;
 
@@ -1473,29 +1453,10 @@ alloc_context_vma(struct intel_engine_cs *engine)
 		}
 
 		memcpy(vaddr, defaults, engine->context_size);
-
 		i915_gem_object_unpin_map(engine->default_state);
-		i915_gem_object_unpin_map(obj);
-	}
 
-	/*
-	 * Try to make the context utilize L3 as well as LLC.
-	 *
-	 * On VLV we don't have L3 controls in the PTEs so we
-	 * shouldn't touch the cache level, especially as that
-	 * would make the object snooped which might have a
-	 * negative performance impact.
-	 *
-	 * Snooping is required on non-llc platforms in execlist
-	 * mode, but since all GGTT accesses use PAT entry 0 we
-	 * get snooping anyway regardless of cache_level.
-	 *
-	 * This is only applicable for Ivy Bridge devices since
-	 * later platforms don't have L3 control bits in the PTE.
-	 */
-	if (IS_IVYBRIDGE(i915)) {
-		/* Ignore any error, regard it as a simple optimisation */
-		i915_gem_object_set_cache_level(obj, I915_CACHE_L3_LLC);
+		i915_gem_object_flush_map(obj);
+		i915_gem_object_unpin_map(obj);
 	}
 
 	vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
@@ -1513,69 +1474,52 @@ err_obj:
 	return ERR_PTR(err);
 }
 
-static struct intel_context *
-__ring_context_pin(struct intel_engine_cs *engine,
-		   struct i915_gem_context *ctx,
-		   struct intel_context *ce)
+static int ring_context_pin(struct intel_context *ce)
 {
+	struct intel_engine_cs *engine = ce->engine;
 	int err;
 
+	/* One ringbuffer to rule them all */
+	GEM_BUG_ON(!engine->buffer);
+	ce->ring = engine->buffer;
+
 	if (!ce->state && engine->context_size) {
 		struct i915_vma *vma;
 
 		vma = alloc_context_vma(engine);
-		if (IS_ERR(vma)) {
-			err = PTR_ERR(vma);
-			goto err;
-		}
+		if (IS_ERR(vma))
+			return PTR_ERR(vma);
 
 		ce->state = vma;
 	}
 
 	err = __context_pin(ce);
 	if (err)
-		goto err;
+		return err;
 
 	err = __context_pin_ppgtt(ce->gem_context);
 	if (err)
 		goto err_unpin;
 
-	i915_gem_context_get(ctx);
-
-	/* One ringbuffer to rule them all */
-	GEM_BUG_ON(!engine->buffer);
-	ce->ring = engine->buffer;
-
-	return ce;
+	return 0;
 
 err_unpin:
 	__context_unpin(ce);
-err:
-	ce->pin_count = 0;
-	return ERR_PTR(err);
+	return err;
 }
 
-static const struct intel_context_ops ring_context_ops = {
-	.unpin = intel_ring_context_unpin,
-	.destroy = intel_ring_context_destroy,
-};
-
-static struct intel_context *
-intel_ring_context_pin(struct intel_engine_cs *engine,
-		       struct i915_gem_context *ctx)
+static void ring_context_reset(struct intel_context *ce)
 {
-	struct intel_context *ce = to_intel_context(ctx, engine);
-
-	lockdep_assert_held(&ctx->i915->drm.struct_mutex);
-
-	if (likely(ce->pin_count++))
-		return ce;
-	GEM_BUG_ON(!ce->pin_count); /* no overflow please! */
+	intel_ring_reset(ce->ring, 0);
+}
 
-	ce->ops = &ring_context_ops;
+static const struct intel_context_ops ring_context_ops = {
+	.pin = ring_context_pin,
+	.unpin = ring_context_unpin,
 
-	return __ring_context_pin(engine, ctx, ce);
-}
+	.reset = ring_context_reset,
+	.destroy = ring_context_destroy,
+};
 
 static int intel_init_ring_buffer(struct intel_engine_cs *engine)
 {
@@ -1587,9 +1531,7 @@ static int intel_init_ring_buffer(struct intel_engine_cs *engine)
 	if (err)
 		return err;
 
-	timeline = i915_timeline_create(engine->i915,
-					engine->name,
-					engine->status_page.vma);
+	timeline = i915_timeline_create(engine->i915, engine->status_page.vma);
 	if (IS_ERR(timeline)) {
 		err = PTR_ERR(timeline);
 		goto err;
@@ -1621,7 +1563,7 @@ static int intel_init_ring_buffer(struct intel_engine_cs *engine)
 err_unpin:
 	intel_ring_unpin(ring);
 err_ring:
-	intel_ring_free(ring);
+	intel_ring_put(ring);
 err:
 	intel_engine_cleanup_common(engine);
 	return err;
@@ -1632,10 +1574,10 @@ void intel_engine_cleanup(struct intel_engine_cs *engine)
 	struct drm_i915_private *dev_priv = engine->i915;
 
 	WARN_ON(INTEL_GEN(dev_priv) > 2 &&
-		(I915_READ_MODE(engine) & MODE_IDLE) == 0);
+		(ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE) == 0);
 
 	intel_ring_unpin(engine->buffer);
-	intel_ring_free(engine->buffer);
+	intel_ring_put(engine->buffer);
 
 	if (engine->cleanup)
 		engine->cleanup(engine);
@@ -1646,16 +1588,6 @@ void intel_engine_cleanup(struct intel_engine_cs *engine)
 	kfree(engine);
 }
 
-void intel_legacy_submission_resume(struct drm_i915_private *dev_priv)
-{
-	struct intel_engine_cs *engine;
-	enum intel_engine_id id;
-
-	/* Restart from the beginning of the rings for convenience */
-	for_each_engine(engine, dev_priv, id)
-		intel_ring_reset(engine->buffer, 0);
-}
-
 static int load_pd_dir(struct i915_request *rq,
 		       const struct i915_hw_ppgtt *ppgtt)
 {
@@ -1667,11 +1599,11 @@ static int load_pd_dir(struct i915_request *rq,
 		return PTR_ERR(cs);
 
 	*cs++ = MI_LOAD_REGISTER_IMM(1);
-	*cs++ = i915_mmio_reg_offset(RING_PP_DIR_DCLV(engine));
+	*cs++ = i915_mmio_reg_offset(RING_PP_DIR_DCLV(engine->mmio_base));
 	*cs++ = PP_DIR_DCLV_2G;
 
 	*cs++ = MI_LOAD_REGISTER_IMM(1);
-	*cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine));
+	*cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base));
 	*cs++ = ppgtt->pd.base.ggtt_offset << 10;
 
 	intel_ring_advance(rq, cs);
@@ -1690,7 +1622,7 @@ static int flush_pd_dir(struct i915_request *rq)
 
 	/* Stall until the page table load is complete */
 	*cs++ = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
-	*cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine));
+	*cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base));
 	*cs++ = i915_scratch_offset(rq->i915);
 	*cs++ = MI_NOOP;
 
@@ -1703,8 +1635,8 @@ static inline int mi_set_context(struct i915_request *rq, u32 flags)
 	struct drm_i915_private *i915 = rq->i915;
 	struct intel_engine_cs *engine = rq->engine;
 	enum intel_engine_id id;
-	const int num_rings =
-		IS_HSW_GT1(i915) ? RUNTIME_INFO(i915)->num_rings - 1 : 0;
+	const int num_engines =
+		IS_HSW_GT1(i915) ? RUNTIME_INFO(i915)->num_engines - 1 : 0;
 	bool force_restore = false;
 	int len;
 	u32 *cs;
@@ -1718,7 +1650,7 @@ static inline int mi_set_context(struct i915_request *rq, u32 flags)
 
 	len = 4;
 	if (IS_GEN(i915, 7))
-		len += 2 + (num_rings ? 4*num_rings + 6 : 0);
+		len += 2 + (num_engines ? 4 * num_engines + 6 : 0);
 	if (flags & MI_FORCE_RESTORE) {
 		GEM_BUG_ON(flags & MI_RESTORE_INHIBIT);
 		flags &= ~MI_FORCE_RESTORE;
@@ -1733,10 +1665,10 @@ static inline int mi_set_context(struct i915_request *rq, u32 flags)
 	/* WaProgramMiArbOnOffAroundMiSetContext:ivb,vlv,hsw,bdw,chv */
 	if (IS_GEN(i915, 7)) {
 		*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
-		if (num_rings) {
+		if (num_engines) {
 			struct intel_engine_cs *signaller;
 
-			*cs++ = MI_LOAD_REGISTER_IMM(num_rings);
+			*cs++ = MI_LOAD_REGISTER_IMM(num_engines);
 			for_each_engine(signaller, i915, id) {
 				if (signaller == engine)
 					continue;
@@ -1763,8 +1695,7 @@ static inline int mi_set_context(struct i915_request *rq, u32 flags)
 		 * placeholder we use to flush other contexts.
 		 */
 		*cs++ = MI_SET_CONTEXT;
-		*cs++ = i915_ggtt_offset(to_intel_context(i915->kernel_context,
-							  engine)->state) |
+		*cs++ = i915_ggtt_offset(engine->kernel_context->state) |
 			MI_MM_SPACE_GTT |
 			MI_RESTORE_INHIBIT;
 	}
@@ -1779,11 +1710,11 @@ static inline int mi_set_context(struct i915_request *rq, u32 flags)
 	*cs++ = MI_NOOP;
 
 	if (IS_GEN(i915, 7)) {
-		if (num_rings) {
+		if (num_engines) {
 			struct intel_engine_cs *signaller;
 			i915_reg_t last_reg = {}; /* keep gcc quiet */
 
-			*cs++ = MI_LOAD_REGISTER_IMM(num_rings);
+			*cs++ = MI_LOAD_REGISTER_IMM(num_engines);
 			for_each_engine(signaller, i915, id) {
 				if (signaller == engine)
 					continue;
@@ -1861,7 +1792,7 @@ static int switch_context(struct i915_request *rq)
 		 * explanation.
 		 */
 		loops = 1;
-		if (engine->id == BCS && IS_VALLEYVIEW(engine->i915))
+		if (engine->id == BCS0 && IS_VALLEYVIEW(engine->i915))
 			loops = 32;
 
 		do {
@@ -1870,15 +1801,15 @@ static int switch_context(struct i915_request *rq)
 				goto err;
 		} while (--loops);
 
-		if (intel_engine_flag(engine) & ppgtt->pd_dirty_rings) {
-			unwind_mm = intel_engine_flag(engine);
-			ppgtt->pd_dirty_rings &= ~unwind_mm;
+		if (ppgtt->pd_dirty_engines & engine->mask) {
+			unwind_mm = engine->mask;
+			ppgtt->pd_dirty_engines &= ~unwind_mm;
 			hw_flags = MI_FORCE_RESTORE;
 		}
 	}
 
 	if (rq->hw_context->state) {
-		GEM_BUG_ON(engine->id != RCS);
+		GEM_BUG_ON(engine->id != RCS0);
 
 		/*
 		 * The kernel context(s) is treated as pure scratch and is not
@@ -1938,7 +1869,7 @@ static int switch_context(struct i915_request *rq)
 
 err_mm:
 	if (unwind_mm)
-		ppgtt->pd_dirty_rings |= unwind_mm;
+		ppgtt->pd_dirty_engines |= unwind_mm;
 err:
 	return ret;
 }
@@ -1947,7 +1878,7 @@ static int ring_request_alloc(struct i915_request *request)
 {
 	int ret;
 
-	GEM_BUG_ON(!request->hw_context->pin_count);
+	GEM_BUG_ON(!intel_context_is_pinned(request->hw_context));
 	GEM_BUG_ON(request->timeline->has_initial_breadcrumb);
 
 	/*
@@ -2108,23 +2039,23 @@ int intel_ring_cacheline_align(struct i915_request *rq)
 
 static void gen6_bsd_submit_request(struct i915_request *request)
 {
-	struct drm_i915_private *dev_priv = request->i915;
+	struct intel_uncore *uncore = request->engine->uncore;
 
-	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
 
        /* Every tail move must follow the sequence below */
 
 	/* Disable notification that the ring is IDLE. The GT
 	 * will then assume that it is busy and bring it out of rc6.
 	 */
-	I915_WRITE_FW(GEN6_BSD_SLEEP_PSMI_CONTROL,
-		      _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
+	intel_uncore_write_fw(uncore, GEN6_BSD_SLEEP_PSMI_CONTROL,
+			      _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
 
 	/* Clear the context id. Here be magic! */
-	I915_WRITE64_FW(GEN6_BSD_RNCID, 0x0);
+	intel_uncore_write64_fw(uncore, GEN6_BSD_RNCID, 0x0);
 
 	/* Wait for the ring not to be idle, i.e. for it to wake up. */
-	if (__intel_wait_for_register_fw(dev_priv,
+	if (__intel_wait_for_register_fw(uncore,
 					 GEN6_BSD_SLEEP_PSMI_CONTROL,
 					 GEN6_BSD_SLEEP_INDICATOR,
 					 0,
@@ -2137,10 +2068,10 @@ static void gen6_bsd_submit_request(struct i915_request *request)
 	/* Let the ring send IDLE messages to the GT again,
 	 * and so let it sleep to conserve power when idle.
 	 */
-	I915_WRITE_FW(GEN6_BSD_SLEEP_PSMI_CONTROL,
-		      _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
+	intel_uncore_write_fw(uncore, GEN6_BSD_SLEEP_PSMI_CONTROL,
+			      _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
 
-	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
 }
 
 static int mi_flush_dw(struct i915_request *rq, u32 flags)
@@ -2282,7 +2213,7 @@ static void intel_ring_default_vfuncs(struct drm_i915_private *dev_priv,
 	engine->reset.reset = reset_ring;
 	engine->reset.finish = reset_finish;
 
-	engine->context_pin = intel_ring_context_pin;
+	engine->cops = &ring_context_ops;
 	engine->request_alloc = ring_request_alloc;
 
 	/*
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h
index 710ffb221775..72c7c337ace9 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.h
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.h
@@ -6,22 +6,20 @@
 
 #include <linux/hashtable.h>
 #include <linux/irq_work.h>
+#include <linux/random.h>
 #include <linux/seqlock.h>
 
 #include "i915_gem_batch_pool.h"
-
-#include "i915_reg.h"
 #include "i915_pmu.h"
+#include "i915_reg.h"
 #include "i915_request.h"
 #include "i915_selftest.h"
 #include "i915_timeline.h"
+#include "intel_engine_types.h"
 #include "intel_gpu_commands.h"
 #include "intel_workarounds.h"
 
 struct drm_printer;
-struct i915_sched_attr;
-
-#define I915_CMD_HASH_ORDER 9
 
 /* Early gen2 devices have a cacheline of just 32 bytes, using 64 is overkill,
  * but keeps the logic simple. Indeed, the whole purpose of this macro is just
@@ -31,28 +29,44 @@ struct i915_sched_attr;
 #define CACHELINE_BYTES 64
 #define CACHELINE_DWORDS (CACHELINE_BYTES / sizeof(u32))
 
-struct intel_hw_status_page {
-	struct i915_vma *vma;
-	u32 *addr;
-};
+/*
+ * The register defines to be used with the following macros need to accept a
+ * base param, e.g:
+ *
+ * REG_FOO(base) _MMIO((base) + <relative offset>)
+ * ENGINE_READ(engine, REG_FOO);
+ *
+ * register arrays are to be defined and accessed as follows:
+ *
+ * REG_BAR(base, i) _MMIO((base) + <relative offset> + (i) * <shift>)
+ * ENGINE_READ_IDX(engine, REG_BAR, i)
+ */
+
+#define __ENGINE_REG_OP(op__, engine__, ...) \
+	intel_uncore_##op__((engine__)->uncore, __VA_ARGS__)
 
-#define I915_READ_TAIL(engine) I915_READ(RING_TAIL((engine)->mmio_base))
-#define I915_WRITE_TAIL(engine, val) I915_WRITE(RING_TAIL((engine)->mmio_base), val)
+#define __ENGINE_READ_OP(op__, engine__, reg__) \
+	__ENGINE_REG_OP(op__, (engine__), reg__((engine__)->mmio_base))
 
-#define I915_READ_START(engine) I915_READ(RING_START((engine)->mmio_base))
-#define I915_WRITE_START(engine, val) I915_WRITE(RING_START((engine)->mmio_base), val)
+#define ENGINE_READ16(...)	__ENGINE_READ_OP(read16, __VA_ARGS__)
+#define ENGINE_READ(...)	__ENGINE_READ_OP(read, __VA_ARGS__)
+#define ENGINE_READ_FW(...)	__ENGINE_READ_OP(read_fw, __VA_ARGS__)
+#define ENGINE_POSTING_READ(...) __ENGINE_READ_OP(posting_read, __VA_ARGS__)
 
-#define I915_READ_HEAD(engine)  I915_READ(RING_HEAD((engine)->mmio_base))
-#define I915_WRITE_HEAD(engine, val) I915_WRITE(RING_HEAD((engine)->mmio_base), val)
+#define ENGINE_READ64(engine__, lower_reg__, upper_reg__) \
+	__ENGINE_REG_OP(read64_2x32, (engine__), \
+			lower_reg__((engine__)->mmio_base), \
+			upper_reg__((engine__)->mmio_base))
 
-#define I915_READ_CTL(engine) I915_READ(RING_CTL((engine)->mmio_base))
-#define I915_WRITE_CTL(engine, val) I915_WRITE(RING_CTL((engine)->mmio_base), val)
+#define ENGINE_READ_IDX(engine__, reg__, idx__) \
+	__ENGINE_REG_OP(read, (engine__), reg__((engine__)->mmio_base, (idx__)))
 
-#define I915_READ_IMR(engine) I915_READ(RING_IMR((engine)->mmio_base))
-#define I915_WRITE_IMR(engine, val) I915_WRITE(RING_IMR((engine)->mmio_base), val)
+#define __ENGINE_WRITE_OP(op__, engine__, reg__, val__) \
+	__ENGINE_REG_OP(op__, (engine__), reg__((engine__)->mmio_base), (val__))
 
-#define I915_READ_MODE(engine) I915_READ(RING_MI_MODE((engine)->mmio_base))
-#define I915_WRITE_MODE(engine, val) I915_WRITE(RING_MI_MODE((engine)->mmio_base), val)
+#define ENGINE_WRITE16(...)	__ENGINE_WRITE_OP(write16, __VA_ARGS__)
+#define ENGINE_WRITE(...)	__ENGINE_WRITE_OP(write, __VA_ARGS__)
+#define ENGINE_WRITE_FW(...)	__ENGINE_WRITE_OP(write_fw, __VA_ARGS__)
 
 /* seqno size is actually only a uint32, but since we plan to use MI_FLUSH_DW to
  * do the writes, and that must have qw aligned offsets, simply pretend it's 8b.
@@ -90,506 +104,7 @@ hangcheck_action_to_str(const enum intel_engine_hangcheck_action a)
 	return "unknown";
 }
 
-#define I915_MAX_SLICES	3
-#define I915_MAX_SUBSLICES 8
-
-#define instdone_slice_mask(dev_priv__) \
-	(IS_GEN(dev_priv__, 7) ? \
-	 1 : RUNTIME_INFO(dev_priv__)->sseu.slice_mask)
-
-#define instdone_subslice_mask(dev_priv__) \
-	(IS_GEN(dev_priv__, 7) ? \
-	 1 : RUNTIME_INFO(dev_priv__)->sseu.subslice_mask[0])
-
-#define for_each_instdone_slice_subslice(dev_priv__, slice__, subslice__) \
-	for ((slice__) = 0, (subslice__) = 0; \
-	     (slice__) < I915_MAX_SLICES; \
-	     (subslice__) = ((subslice__) + 1) < I915_MAX_SUBSLICES ? (subslice__) + 1 : 0, \
-	       (slice__) += ((subslice__) == 0)) \
-		for_each_if((BIT(slice__) & instdone_slice_mask(dev_priv__)) && \
-			    (BIT(subslice__) & instdone_subslice_mask(dev_priv__)))
-
-struct intel_instdone {
-	u32 instdone;
-	/* The following exist only in the RCS engine */
-	u32 slice_common;
-	u32 sampler[I915_MAX_SLICES][I915_MAX_SUBSLICES];
-	u32 row[I915_MAX_SLICES][I915_MAX_SUBSLICES];
-};
-
-struct intel_engine_hangcheck {
-	u64 acthd;
-	u32 seqno;
-	unsigned long action_timestamp;
-	struct intel_instdone instdone;
-};
-
-struct intel_ring {
-	struct i915_vma *vma;
-	void *vaddr;
-
-	struct i915_timeline *timeline;
-	struct list_head request_list;
-	struct list_head active_link;
-
-	u32 head;
-	u32 tail;
-	u32 emit;
-
-	u32 space;
-	u32 size;
-	u32 effective_size;
-};
-
-struct i915_gem_context;
-struct drm_i915_reg_table;
-
-/*
- * we use a single page to load ctx workarounds so all of these
- * values are referred in terms of dwords
- *
- * struct i915_wa_ctx_bb:
- *  offset: specifies batch starting position, also helpful in case
- *    if we want to have multiple batches at different offsets based on
- *    some criteria. It is not a requirement at the moment but provides
- *    an option for future use.
- *  size: size of the batch in DWORDS
- */
-struct i915_ctx_workarounds {
-	struct i915_wa_ctx_bb {
-		u32 offset;
-		u32 size;
-	} indirect_ctx, per_ctx;
-	struct i915_vma *vma;
-};
-
-struct i915_request;
-
-#define I915_MAX_VCS	4
-#define I915_MAX_VECS	2
-
-/*
- * Engine IDs definitions.
- * Keep instances of the same type engine together.
- */
-enum intel_engine_id {
-	RCS = 0,
-	BCS,
-	VCS,
-	VCS2,
-	VCS3,
-	VCS4,
-#define _VCS(n) (VCS + (n))
-	VECS,
-	VECS2
-#define _VECS(n) (VECS + (n))
-};
-
-struct i915_priolist {
-	struct list_head requests[I915_PRIORITY_COUNT];
-	struct rb_node node;
-	unsigned long used;
-	int priority;
-};
-
-#define priolist_for_each_request(it, plist, idx) \
-	for (idx = 0; idx < ARRAY_SIZE((plist)->requests); idx++) \
-		list_for_each_entry(it, &(plist)->requests[idx], sched.link)
-
-#define priolist_for_each_request_consume(it, n, plist, idx) \
-	for (; (idx = ffs((plist)->used)); (plist)->used &= ~BIT(idx - 1)) \
-		list_for_each_entry_safe(it, n, \
-					 &(plist)->requests[idx - 1], \
-					 sched.link)
-
-struct st_preempt_hang {
-	struct completion completion;
-	unsigned int count;
-	bool inject_hang;
-};
-
-/**
- * struct intel_engine_execlists - execlist submission queue and port state
- *
- * The struct intel_engine_execlists represents the combined logical state of
- * driver and the hardware state for execlist mode of submission.
- */
-struct intel_engine_execlists {
-	/**
-	 * @tasklet: softirq tasklet for bottom handler
-	 */
-	struct tasklet_struct tasklet;
-
-	/**
-	 * @default_priolist: priority list for I915_PRIORITY_NORMAL
-	 */
-	struct i915_priolist default_priolist;
-
-	/**
-	 * @no_priolist: priority lists disabled
-	 */
-	bool no_priolist;
-
-	/**
-	 * @submit_reg: gen-specific execlist submission register
-	 * set to the ExecList Submission Port (elsp) register pre-Gen11 and to
-	 * the ExecList Submission Queue Contents register array for Gen11+
-	 */
-	u32 __iomem *submit_reg;
-
-	/**
-	 * @ctrl_reg: the enhanced execlists control register, used to load the
-	 * submit queue on the HW and to request preemptions to idle
-	 */
-	u32 __iomem *ctrl_reg;
-
-	/**
-	 * @port: execlist port states
-	 *
-	 * For each hardware ELSP (ExecList Submission Port) we keep
-	 * track of the last request and the number of times we submitted
-	 * that port to hw. We then count the number of times the hw reports
-	 * a context completion or preemption. As only one context can
-	 * be active on hw, we limit resubmission of context to port[0]. This
-	 * is called Lite Restore, of the context.
-	 */
-	struct execlist_port {
-		/**
-		 * @request_count: combined request and submission count
-		 */
-		struct i915_request *request_count;
-#define EXECLIST_COUNT_BITS 2
-#define port_request(p) ptr_mask_bits((p)->request_count, EXECLIST_COUNT_BITS)
-#define port_count(p) ptr_unmask_bits((p)->request_count, EXECLIST_COUNT_BITS)
-#define port_pack(rq, count) ptr_pack_bits(rq, count, EXECLIST_COUNT_BITS)
-#define port_unpack(p, count) ptr_unpack_bits((p)->request_count, count, EXECLIST_COUNT_BITS)
-#define port_set(p, packed) ((p)->request_count = (packed))
-#define port_isset(p) ((p)->request_count)
-#define port_index(p, execlists) ((p) - (execlists)->port)
-
-		/**
-		 * @context_id: context ID for port
-		 */
-		GEM_DEBUG_DECL(u32 context_id);
-
-#define EXECLIST_MAX_PORTS 2
-	} port[EXECLIST_MAX_PORTS];
-
-	/**
-	 * @active: is the HW active? We consider the HW as active after
-	 * submitting any context for execution and until we have seen the
-	 * last context completion event. After that, we do not expect any
-	 * more events until we submit, and so can park the HW.
-	 *
-	 * As we have a small number of different sources from which we feed
-	 * the HW, we track the state of each inside a single bitfield.
-	 */
-	unsigned int active;
-#define EXECLISTS_ACTIVE_USER 0
-#define EXECLISTS_ACTIVE_PREEMPT 1
-#define EXECLISTS_ACTIVE_HWACK 2
-
-	/**
-	 * @port_mask: number of execlist ports - 1
-	 */
-	unsigned int port_mask;
-
-	/**
-	 * @queue_priority_hint: Highest pending priority.
-	 *
-	 * When we add requests into the queue, or adjust the priority of
-	 * executing requests, we compute the maximum priority of those
-	 * pending requests. We can then use this value to determine if
-	 * we need to preempt the executing requests to service the queue.
-	 * However, since the we may have recorded the priority of an inflight
-	 * request we wanted to preempt but since completed, at the time of
-	 * dequeuing the priority hint may no longer may match the highest
-	 * available request priority.
-	 */
-	int queue_priority_hint;
-
-	/**
-	 * @queue: queue of requests, in priority lists
-	 */
-	struct rb_root_cached queue;
-
-	/**
-	 * @csb_write: control register for Context Switch buffer
-	 *
-	 * Note this register may be either mmio or HWSP shadow.
-	 */
-	u32 *csb_write;
-
-	/**
-	 * @csb_status: status array for Context Switch buffer
-	 *
-	 * Note these register may be either mmio or HWSP shadow.
-	 */
-	u32 *csb_status;
-
-	/**
-	 * @preempt_complete_status: expected CSB upon completing preemption
-	 */
-	u32 preempt_complete_status;
-
-	/**
-	 * @csb_head: context status buffer head
-	 */
-	u8 csb_head;
-
-	I915_SELFTEST_DECLARE(struct st_preempt_hang preempt_hang;)
-};
-
-#define INTEL_ENGINE_CS_MAX_NAME 8
-
-struct intel_engine_cs {
-	struct drm_i915_private *i915;
-	char name[INTEL_ENGINE_CS_MAX_NAME];
-
-	enum intel_engine_id id;
-	unsigned int hw_id;
-	unsigned int guc_id;
-
-	u8 uabi_id;
-	u8 uabi_class;
-
-	u8 class;
-	u8 instance;
-	u32 context_size;
-	u32 mmio_base;
-
-	struct intel_ring *buffer;
-
-	struct i915_timeline timeline;
-
-	struct drm_i915_gem_object *default_state;
-	void *pinned_default_state;
-
-	/* Rather than have every client wait upon all user interrupts,
-	 * with the herd waking after every interrupt and each doing the
-	 * heavyweight seqno dance, we delegate the task (of being the
-	 * bottom-half of the user interrupt) to the first client. After
-	 * every interrupt, we wake up one client, who does the heavyweight
-	 * coherent seqno read and either goes back to sleep (if incomplete),
-	 * or wakes up all the completed clients in parallel, before then
-	 * transferring the bottom-half status to the next client in the queue.
-	 *
-	 * Compared to walking the entire list of waiters in a single dedicated
-	 * bottom-half, we reduce the latency of the first waiter by avoiding
-	 * a context switch, but incur additional coherent seqno reads when
-	 * following the chain of request breadcrumbs. Since it is most likely
-	 * that we have a single client waiting on each seqno, then reducing
-	 * the overhead of waking that client is much preferred.
-	 */
-	struct intel_breadcrumbs {
-		spinlock_t irq_lock;
-		struct list_head signalers;
-
-		struct irq_work irq_work; /* for use from inside irq_lock */
-
-		unsigned int irq_enabled;
-
-		bool irq_armed;
-	} breadcrumbs;
-
-	struct {
-		/**
-		 * @enable: Bitmask of enable sample events on this engine.
-		 *
-		 * Bits correspond to sample event types, for instance
-		 * I915_SAMPLE_QUEUED is bit 0 etc.
-		 */
-		u32 enable;
-		/**
-		 * @enable_count: Reference count for the enabled samplers.
-		 *
-		 * Index number corresponds to @enum drm_i915_pmu_engine_sample.
-		 */
-		unsigned int enable_count[I915_ENGINE_SAMPLE_COUNT];
-		/**
-		 * @sample: Counter values for sampling events.
-		 *
-		 * Our internal timer stores the current counters in this field.
-		 *
-		 * Index number corresponds to @enum drm_i915_pmu_engine_sample.
-		 */
-		struct i915_pmu_sample sample[I915_ENGINE_SAMPLE_COUNT];
-	} pmu;
-
-	/*
-	 * A pool of objects to use as shadow copies of client batch buffers
-	 * when the command parser is enabled. Prevents the client from
-	 * modifying the batch contents after software parsing.
-	 */
-	struct i915_gem_batch_pool batch_pool;
-
-	struct intel_hw_status_page status_page;
-	struct i915_ctx_workarounds wa_ctx;
-	struct i915_wa_list ctx_wa_list;
-	struct i915_wa_list wa_list;
-	struct i915_wa_list whitelist;
-
-	u32             irq_keep_mask; /* always keep these interrupts */
-	u32		irq_enable_mask; /* bitmask to enable ring interrupt */
-	void		(*irq_enable)(struct intel_engine_cs *engine);
-	void		(*irq_disable)(struct intel_engine_cs *engine);
-
-	int		(*init_hw)(struct intel_engine_cs *engine);
-
-	struct {
-		void (*prepare)(struct intel_engine_cs *engine);
-		void (*reset)(struct intel_engine_cs *engine, bool stalled);
-		void (*finish)(struct intel_engine_cs *engine);
-	} reset;
-
-	void		(*park)(struct intel_engine_cs *engine);
-	void		(*unpark)(struct intel_engine_cs *engine);
-
-	void		(*set_default_submission)(struct intel_engine_cs *engine);
-
-	struct intel_context *(*context_pin)(struct intel_engine_cs *engine,
-					     struct i915_gem_context *ctx);
-
-	int		(*request_alloc)(struct i915_request *rq);
-	int		(*init_context)(struct i915_request *rq);
-
-	int		(*emit_flush)(struct i915_request *request, u32 mode);
-#define EMIT_INVALIDATE	BIT(0)
-#define EMIT_FLUSH	BIT(1)
-#define EMIT_BARRIER	(EMIT_INVALIDATE | EMIT_FLUSH)
-	int		(*emit_bb_start)(struct i915_request *rq,
-					 u64 offset, u32 length,
-					 unsigned int dispatch_flags);
-#define I915_DISPATCH_SECURE BIT(0)
-#define I915_DISPATCH_PINNED BIT(1)
-	int		 (*emit_init_breadcrumb)(struct i915_request *rq);
-	u32		*(*emit_fini_breadcrumb)(struct i915_request *rq,
-						 u32 *cs);
-	unsigned int	emit_fini_breadcrumb_dw;
-
-	/* Pass the request to the hardware queue (e.g. directly into
-	 * the legacy ringbuffer or to the end of an execlist).
-	 *
-	 * This is called from an atomic context with irqs disabled; must
-	 * be irq safe.
-	 */
-	void		(*submit_request)(struct i915_request *rq);
-
-	/*
-	 * Call when the priority on a request has changed and it and its
-	 * dependencies may need rescheduling. Note the request itself may
-	 * not be ready to run!
-	 */
-	void		(*schedule)(struct i915_request *request,
-				    const struct i915_sched_attr *attr);
-
-	/*
-	 * Cancel all requests on the hardware, or queued for execution.
-	 * This should only cancel the ready requests that have been
-	 * submitted to the engine (via the engine->submit_request callback).
-	 * This is called when marking the device as wedged.
-	 */
-	void		(*cancel_requests)(struct intel_engine_cs *engine);
-
-	void		(*cleanup)(struct intel_engine_cs *engine);
-
-	struct intel_engine_execlists execlists;
-
-	/* Contexts are pinned whilst they are active on the GPU. The last
-	 * context executed remains active whilst the GPU is idle - the
-	 * switch away and write to the context object only occurs on the
-	 * next execution.  Contexts are only unpinned on retirement of the
-	 * following request ensuring that we can always write to the object
-	 * on the context switch even after idling. Across suspend, we switch
-	 * to the kernel context and trash it as the save may not happen
-	 * before the hardware is powered down.
-	 */
-	struct intel_context *last_retired_context;
-
-	/* status_notifier: list of callbacks for context-switch changes */
-	struct atomic_notifier_head context_status_notifier;
-
-	struct intel_engine_hangcheck hangcheck;
-
-#define I915_ENGINE_NEEDS_CMD_PARSER BIT(0)
-#define I915_ENGINE_SUPPORTS_STATS   BIT(1)
-#define I915_ENGINE_HAS_PREEMPTION   BIT(2)
-	unsigned int flags;
-
-	/*
-	 * Table of commands the command parser needs to know about
-	 * for this engine.
-	 */
-	DECLARE_HASHTABLE(cmd_hash, I915_CMD_HASH_ORDER);
-
-	/*
-	 * Table of registers allowed in commands that read/write registers.
-	 */
-	const struct drm_i915_reg_table *reg_tables;
-	int reg_table_count;
-
-	/*
-	 * Returns the bitmask for the length field of the specified command.
-	 * Return 0 for an unrecognized/invalid command.
-	 *
-	 * If the command parser finds an entry for a command in the engine's
-	 * cmd_tables, it gets the command's length based on the table entry.
-	 * If not, it calls this function to determine the per-engine length
-	 * field encoding for the command (i.e. different opcode ranges use
-	 * certain bits to encode the command length in the header).
-	 */
-	u32 (*get_cmd_length_mask)(u32 cmd_header);
-
-	struct {
-		/**
-		 * @lock: Lock protecting the below fields.
-		 */
-		seqlock_t lock;
-		/**
-		 * @enabled: Reference count indicating number of listeners.
-		 */
-		unsigned int enabled;
-		/**
-		 * @active: Number of contexts currently scheduled in.
-		 */
-		unsigned int active;
-		/**
-		 * @enabled_at: Timestamp when busy stats were enabled.
-		 */
-		ktime_t enabled_at;
-		/**
-		 * @start: Timestamp of the last idle to active transition.
-		 *
-		 * Idle is defined as active == 0, active is active > 0.
-		 */
-		ktime_t start;
-		/**
-		 * @total: Total time this engine was busy.
-		 *
-		 * Accumulated time not counting the most recent block in cases
-		 * where engine is currently busy (active > 0).
-		 */
-		ktime_t total;
-	} stats;
-};
-
-static inline bool
-intel_engine_needs_cmd_parser(const struct intel_engine_cs *engine)
-{
-	return engine->flags & I915_ENGINE_NEEDS_CMD_PARSER;
-}
-
-static inline bool
-intel_engine_supports_stats(const struct intel_engine_cs *engine)
-{
-	return engine->flags & I915_ENGINE_SUPPORTS_STATS;
-}
-
-static inline bool
-intel_engine_has_preemption(const struct intel_engine_cs *engine)
-{
-	return engine->flags & I915_ENGINE_HAS_PREEMPTION;
-}
+void intel_engines_set_scheduler_caps(struct drm_i915_private *i915);
 
 static inline bool __execlists_need_preempt(int prio, int last)
 {
@@ -650,7 +165,7 @@ void execlists_user_end(struct intel_engine_execlists *execlists);
 void
 execlists_cancel_port_requests(struct intel_engine_execlists * const execlists);
 
-void
+struct i915_request *
 execlists_unwind_incomplete_requests(struct intel_engine_execlists *execlists);
 
 static inline unsigned int
@@ -674,12 +189,6 @@ execlists_port_complete(struct intel_engine_execlists * const execlists,
 	return port;
 }
 
-static inline unsigned int
-intel_engine_flag(const struct intel_engine_cs *engine)
-{
-	return BIT(engine->id);
-}
-
 static inline u32
 intel_read_status_page(const struct intel_engine_cs *engine, int reg)
 {
@@ -722,10 +231,10 @@ intel_write_status_page(struct intel_engine_cs *engine, int reg, u32 value)
  *
  * The area from dword 0x30 to 0x3ff is available for driver usage.
  */
-#define I915_GEM_HWS_INDEX		0x30
-#define I915_GEM_HWS_INDEX_ADDR		(I915_GEM_HWS_INDEX * sizeof(u32))
 #define I915_GEM_HWS_PREEMPT		0x32
 #define I915_GEM_HWS_PREEMPT_ADDR	(I915_GEM_HWS_PREEMPT * sizeof(u32))
+#define I915_GEM_HWS_HANGCHECK		0x34
+#define I915_GEM_HWS_HANGCHECK_ADDR	(I915_GEM_HWS_HANGCHECK * sizeof(u32))
 #define I915_GEM_HWS_SEQNO		0x40
 #define I915_GEM_HWS_SEQNO_ADDR		(I915_GEM_HWS_SEQNO * sizeof(u32))
 #define I915_GEM_HWS_SCRATCH		0x80
@@ -743,13 +252,22 @@ int intel_ring_pin(struct intel_ring *ring);
 void intel_ring_reset(struct intel_ring *ring, u32 tail);
 unsigned int intel_ring_update_space(struct intel_ring *ring);
 void intel_ring_unpin(struct intel_ring *ring);
-void intel_ring_free(struct intel_ring *ring);
+void intel_ring_free(struct kref *ref);
+
+static inline struct intel_ring *intel_ring_get(struct intel_ring *ring)
+{
+	kref_get(&ring->ref);
+	return ring;
+}
+
+static inline void intel_ring_put(struct intel_ring *ring)
+{
+	kref_put(&ring->ref, intel_ring_free);
+}
 
 void intel_engine_stop(struct intel_engine_cs *engine);
 void intel_engine_cleanup(struct intel_engine_cs *engine);
 
-void intel_legacy_submission_resume(struct drm_i915_private *dev_priv);
-
 int __must_check intel_ring_cacheline_align(struct i915_request *rq);
 
 u32 __must_check *intel_ring_begin(struct i915_request *rq, unsigned int n);
@@ -844,8 +362,6 @@ __intel_ring_space(unsigned int head, unsigned int tail, unsigned int size)
 	return (head - tail - CACHELINE_BYTES) & (size - 1);
 }
 
-void intel_engine_write_global_seqno(struct intel_engine_cs *engine, u32 seqno);
-
 int intel_engine_setup_common(struct intel_engine_cs *engine);
 int intel_engine_init_common(struct intel_engine_cs *engine);
 void intel_engine_cleanup_common(struct intel_engine_cs *engine);
@@ -863,44 +379,6 @@ void intel_engine_set_hwsp_writemask(struct intel_engine_cs *engine, u32 mask);
 u64 intel_engine_get_active_head(const struct intel_engine_cs *engine);
 u64 intel_engine_get_last_batch_head(const struct intel_engine_cs *engine);
 
-static inline u32 intel_engine_last_submit(struct intel_engine_cs *engine)
-{
-	/*
-	 * We are only peeking at the tail of the submit queue (and not the
-	 * queue itself) in order to gain a hint as to the current active
-	 * state of the engine. Callers are not expected to be taking
-	 * engine->timeline->lock, nor are they expected to be concerned
-	 * wtih serialising this hint with anything, so document it as
-	 * a hint and nothing more.
-	 */
-	return READ_ONCE(engine->timeline.seqno);
-}
-
-static inline u32 intel_engine_get_seqno(struct intel_engine_cs *engine)
-{
-	return intel_read_status_page(engine, I915_GEM_HWS_INDEX);
-}
-
-static inline bool intel_engine_signaled(struct intel_engine_cs *engine,
-					 u32 seqno)
-{
-	return i915_seqno_passed(intel_engine_get_seqno(engine), seqno);
-}
-
-static inline bool intel_engine_has_completed(struct intel_engine_cs *engine,
-					      u32 seqno)
-{
-	GEM_BUG_ON(!seqno);
-	return intel_engine_signaled(engine, seqno);
-}
-
-static inline bool intel_engine_has_started(struct intel_engine_cs *engine,
-					    u32 seqno)
-{
-	GEM_BUG_ON(!seqno);
-	return intel_engine_signaled(engine, seqno - 1);
-}
-
 void intel_engine_get_instdone(struct intel_engine_cs *engine,
 			       struct intel_instdone *instdone);
 
@@ -910,7 +388,7 @@ void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine);
 void intel_engine_pin_breadcrumbs_irq(struct intel_engine_cs *engine);
 void intel_engine_unpin_breadcrumbs_irq(struct intel_engine_cs *engine);
 
-bool intel_engine_signal_breadcrumbs(struct intel_engine_cs *engine);
+void intel_engine_signal_breadcrumbs(struct intel_engine_cs *engine);
 void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine);
 
 static inline void
@@ -919,7 +397,7 @@ intel_engine_queue_breadcrumbs(struct intel_engine_cs *engine)
 	irq_work_queue(&engine->breadcrumbs.irq_work);
 }
 
-bool intel_engine_breadcrumbs_irq(struct intel_engine_cs *engine);
+void intel_engine_breadcrumbs_irq(struct intel_engine_cs *engine);
 
 void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine);
 void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine);
@@ -960,14 +438,14 @@ gen8_emit_ggtt_write_rcs(u32 *cs, u32 value, u32 gtt_offset, u32 flags)
 }
 
 static inline u32 *
-gen8_emit_ggtt_write(u32 *cs, u32 value, u32 gtt_offset)
+gen8_emit_ggtt_write(u32 *cs, u32 value, u32 gtt_offset, u32 flags)
 {
 	/* w/a: bit 5 needs to be zero for MI_FLUSH_DW address. */
 	GEM_BUG_ON(gtt_offset & (1 << 5));
 	/* Offset should be aligned to 8 bytes for both (QW/DW) write types */
 	GEM_BUG_ON(!IS_ALIGNED(gtt_offset, 8));
 
-	*cs++ = (MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW;
+	*cs++ = (MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW | flags;
 	*cs++ = gtt_offset | MI_FLUSH_DW_USE_GTT;
 	*cs++ = 0;
 	*cs++ = value;
@@ -983,11 +461,11 @@ static inline void intel_engine_reset(struct intel_engine_cs *engine,
 }
 
 void intel_engines_sanitize(struct drm_i915_private *i915, bool force);
+void intel_gt_resume(struct drm_i915_private *i915);
 
 bool intel_engine_is_idle(struct intel_engine_cs *engine);
 bool intel_engines_are_idle(struct drm_i915_private *dev_priv);
 
-bool intel_engine_has_kernel_context(const struct intel_engine_cs *engine);
 void intel_engine_lost_context(struct intel_engine_cs *engine);
 
 void intel_engines_park(struct drm_i915_private *i915);
@@ -1066,6 +544,9 @@ void intel_disable_engine_stats(struct intel_engine_cs *engine);
 
 ktime_t intel_engine_get_busy_time(struct intel_engine_cs *engine);
 
+struct i915_request *
+intel_engine_find_active_request(struct intel_engine_cs *engine);
+
 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 
 static inline bool inject_preempt_hang(struct intel_engine_execlists *execlists)
@@ -1086,4 +567,17 @@ static inline bool inject_preempt_hang(struct intel_engine_execlists *execlists)
 
 #endif
 
+static inline u32
+intel_engine_next_hangcheck_seqno(struct intel_engine_cs *engine)
+{
+	return engine->hangcheck.next_seqno =
+		next_pseudo_random32(engine->hangcheck.next_seqno);
+}
+
+static inline u32
+intel_engine_get_hangcheck_seqno(struct intel_engine_cs *engine)
+{
+	return intel_read_status_page(engine, I915_GEM_HWS_HANGCHECK);
+}
+
 #endif /* _INTEL_RINGBUFFER_H_ */
diff --git a/drivers/gpu/drm/i915/intel_runtime_pm.c b/drivers/gpu/drm/i915/intel_runtime_pm.c
index 20c4434474e3..6150e35bf7b5 100644
--- a/drivers/gpu/drm/i915/intel_runtime_pm.c
+++ b/drivers/gpu/drm/i915/intel_runtime_pm.c
@@ -32,6 +32,10 @@
 #include <drm/drm_print.h>
 
 #include "i915_drv.h"
+#include "intel_cdclk.h"
+#include "intel_crt.h"
+#include "intel_csr.h"
+#include "intel_dp.h"
 #include "intel_drv.h"
 
 /**
@@ -147,7 +151,7 @@ static void cancel_intel_runtime_pm_wakeref(struct drm_i915_private *i915,
 		 rpm->debug.count, atomic_read(&rpm->wakeref_count))) {
 		char *buf;
 
-		buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
+		buf = kmalloc(PAGE_SIZE, GFP_NOWAIT | __GFP_NOWARN);
 		if (!buf)
 			return;
 
@@ -183,7 +187,7 @@ __print_intel_runtime_pm_wakeref(struct drm_printer *p,
 	unsigned long i;
 	char *buf;
 
-	buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
+	buf = kmalloc(PAGE_SIZE, GFP_NOWAIT | __GFP_NOWARN);
 	if (!buf)
 		return;
 
@@ -267,7 +271,9 @@ void print_intel_runtime_pm_wakeref(struct drm_i915_private *i915,
 		if (dbg.count <= alloc)
 			break;
 
-		s = krealloc(dbg.owners, dbg.count * sizeof(*s), GFP_KERNEL);
+		s = krealloc(dbg.owners,
+			     dbg.count * sizeof(*s),
+			     GFP_NOWAIT | __GFP_NOWARN);
 		if (!s)
 			goto out;
 
@@ -554,7 +560,7 @@ static void hsw_wait_for_power_well_enable(struct drm_i915_private *dev_priv,
 	int pw_idx = power_well->desc->hsw.idx;
 
 	/* Timeout for PW1:10 us, AUX:not specified, other PWs:20 us. */
-	WARN_ON(intel_wait_for_register(dev_priv,
+	WARN_ON(intel_wait_for_register(&dev_priv->uncore,
 					regs->driver,
 					HSW_PWR_WELL_CTL_STATE(pw_idx),
 					HSW_PWR_WELL_CTL_STATE(pw_idx),
@@ -609,7 +615,7 @@ static void gen9_wait_for_power_well_fuses(struct drm_i915_private *dev_priv,
 					   enum skl_power_gate pg)
 {
 	/* Timeout 5us for PG#0, for other PGs 1us */
-	WARN_ON(intel_wait_for_register(dev_priv, SKL_FUSE_STATUS,
+	WARN_ON(intel_wait_for_register(&dev_priv->uncore, SKL_FUSE_STATUS,
 					SKL_FUSE_PG_DIST_STATUS(pg),
 					SKL_FUSE_PG_DIST_STATUS(pg), 1));
 }
@@ -1510,7 +1516,7 @@ static void assert_chv_phy_status(struct drm_i915_private *dev_priv)
 	 * The PHY may be busy with some initial calibration and whatnot,
 	 * so the power state can take a while to actually change.
 	 */
-	if (intel_wait_for_register(dev_priv,
+	if (intel_wait_for_register(&dev_priv->uncore,
 				    DISPLAY_PHY_STATUS,
 				    phy_status_mask,
 				    phy_status,
@@ -1545,7 +1551,7 @@ static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
 	vlv_set_power_well(dev_priv, power_well, true);
 
 	/* Poll for phypwrgood signal */
-	if (intel_wait_for_register(dev_priv,
+	if (intel_wait_for_register(&dev_priv->uncore,
 				    DISPLAY_PHY_STATUS,
 				    PHY_POWERGOOD(phy),
 				    PHY_POWERGOOD(phy),
@@ -1749,7 +1755,7 @@ static bool chv_pipe_power_well_enabled(struct drm_i915_private *dev_priv,
 
 	mutex_lock(&dev_priv->pcu_lock);
 
-	state = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSS_MASK(pipe);
+	state = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) & DP_SSS_MASK(pipe);
 	/*
 	 * We only ever set the power-on and power-gate states, anything
 	 * else is unexpected.
@@ -1761,7 +1767,7 @@ static bool chv_pipe_power_well_enabled(struct drm_i915_private *dev_priv,
 	 * A transient state at this point would mean some unexpected party
 	 * is poking at the power controls too.
 	 */
-	ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSC_MASK(pipe);
+	ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) & DP_SSC_MASK(pipe);
 	WARN_ON(ctrl << 16 != state);
 
 	mutex_unlock(&dev_priv->pcu_lock);
@@ -1782,20 +1788,20 @@ static void chv_set_pipe_power_well(struct drm_i915_private *dev_priv,
 	mutex_lock(&dev_priv->pcu_lock);
 
 #define COND \
-	((vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSS_MASK(pipe)) == state)
+	((vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) & DP_SSS_MASK(pipe)) == state)
 
 	if (COND)
 		goto out;
 
-	ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
+	ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 	ctrl &= ~DP_SSC_MASK(pipe);
 	ctrl |= enable ? DP_SSC_PWR_ON(pipe) : DP_SSC_PWR_GATE(pipe);
-	vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, ctrl);
+	vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, ctrl);
 
 	if (wait_for(COND, 100))
 		DRM_ERROR("timeout setting power well state %08x (%08x)\n",
 			  state,
-			  vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ));
+			  vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM));
 
 #undef COND
 
@@ -3431,7 +3437,7 @@ int intel_power_domains_init(struct drm_i915_private *dev_priv)
 	 * The enabling order will be from lower to higher indexed wells,
 	 * the disabling order is reversed.
 	 */
-	if (IS_ICELAKE(dev_priv)) {
+	if (IS_GEN(dev_priv, 11)) {
 		err = set_power_wells(power_domains, icl_power_wells);
 	} else if (IS_CANNONLAKE(dev_priv)) {
 		err = set_power_wells(power_domains, cnl_power_wells);
@@ -3565,7 +3571,11 @@ static void icl_dbuf_enable(struct drm_i915_private *dev_priv)
 	    !(I915_READ(DBUF_CTL_S2) & DBUF_POWER_STATE))
 		DRM_ERROR("DBuf power enable timeout\n");
 	else
-		dev_priv->wm.skl_hw.ddb.enabled_slices = 2;
+		/*
+		 * FIXME: for now pretend that we only have 1 slice, see
+		 * intel_enabled_dbuf_slices_num().
+		 */
+		dev_priv->wm.skl_hw.ddb.enabled_slices = 1;
 }
 
 static void icl_dbuf_disable(struct drm_i915_private *dev_priv)
@@ -3580,7 +3590,11 @@ static void icl_dbuf_disable(struct drm_i915_private *dev_priv)
 	    (I915_READ(DBUF_CTL_S2) & DBUF_POWER_STATE))
 		DRM_ERROR("DBuf power disable timeout!\n");
 	else
-		dev_priv->wm.skl_hw.ddb.enabled_slices = 0;
+		/*
+		 * FIXME: for now pretend that the first slice is always
+		 * enabled, see intel_enabled_dbuf_slices_num().
+		 */
+		dev_priv->wm.skl_hw.ddb.enabled_slices = 1;
 }
 
 static void icl_mbus_init(struct drm_i915_private *dev_priv)
@@ -3641,7 +3655,7 @@ static void skl_display_core_init(struct drm_i915_private *dev_priv,
 
 	mutex_unlock(&power_domains->lock);
 
-	skl_init_cdclk(dev_priv);
+	intel_cdclk_init(dev_priv);
 
 	gen9_dbuf_enable(dev_priv);
 
@@ -3658,7 +3672,7 @@ static void skl_display_core_uninit(struct drm_i915_private *dev_priv)
 
 	gen9_dbuf_disable(dev_priv);
 
-	skl_uninit_cdclk(dev_priv);
+	intel_cdclk_uninit(dev_priv);
 
 	/* The spec doesn't call for removing the reset handshake flag */
 	/* disable PG1 and Misc I/O */
@@ -3703,7 +3717,7 @@ void bxt_display_core_init(struct drm_i915_private *dev_priv,
 
 	mutex_unlock(&power_domains->lock);
 
-	bxt_init_cdclk(dev_priv);
+	intel_cdclk_init(dev_priv);
 
 	gen9_dbuf_enable(dev_priv);
 
@@ -3720,7 +3734,7 @@ void bxt_display_core_uninit(struct drm_i915_private *dev_priv)
 
 	gen9_dbuf_disable(dev_priv);
 
-	bxt_uninit_cdclk(dev_priv);
+	intel_cdclk_uninit(dev_priv);
 
 	/* The spec doesn't call for removing the reset handshake flag */
 
@@ -3762,7 +3776,7 @@ static void cnl_display_core_init(struct drm_i915_private *dev_priv, bool resume
 	mutex_unlock(&power_domains->lock);
 
 	/* 5. Enable CD clock */
-	cnl_init_cdclk(dev_priv);
+	intel_cdclk_init(dev_priv);
 
 	/* 6. Enable DBUF */
 	gen9_dbuf_enable(dev_priv);
@@ -3784,7 +3798,7 @@ static void cnl_display_core_uninit(struct drm_i915_private *dev_priv)
 	gen9_dbuf_disable(dev_priv);
 
 	/* 3. Disable CD clock */
-	cnl_uninit_cdclk(dev_priv);
+	intel_cdclk_uninit(dev_priv);
 
 	/*
 	 * 4. Disable Power Well 1 (PG1).
@@ -3826,7 +3840,7 @@ void icl_display_core_init(struct drm_i915_private *dev_priv,
 	mutex_unlock(&power_domains->lock);
 
 	/* 5. Enable CDCLK. */
-	icl_init_cdclk(dev_priv);
+	intel_cdclk_init(dev_priv);
 
 	/* 6. Enable DBUF. */
 	icl_dbuf_enable(dev_priv);
@@ -3851,7 +3865,7 @@ void icl_display_core_uninit(struct drm_i915_private *dev_priv)
 	icl_dbuf_disable(dev_priv);
 
 	/* 3. Disable CD clock */
-	icl_uninit_cdclk(dev_priv);
+	intel_cdclk_uninit(dev_priv);
 
 	/*
 	 * 4. Disable Power Well 1 (PG1).
@@ -3982,6 +3996,36 @@ static void vlv_cmnlane_wa(struct drm_i915_private *dev_priv)
 	cmn->desc->ops->disable(dev_priv, cmn);
 }
 
+static bool vlv_punit_is_power_gated(struct drm_i915_private *dev_priv, u32 reg0)
+{
+	bool ret;
+
+	mutex_lock(&dev_priv->pcu_lock);
+	ret = (vlv_punit_read(dev_priv, reg0) & SSPM0_SSC_MASK) == SSPM0_SSC_PWR_GATE;
+	mutex_unlock(&dev_priv->pcu_lock);
+
+	return ret;
+}
+
+static void assert_ved_power_gated(struct drm_i915_private *dev_priv)
+{
+	WARN(!vlv_punit_is_power_gated(dev_priv, PUNIT_REG_VEDSSPM0),
+	     "VED not power gated\n");
+}
+
+static void assert_isp_power_gated(struct drm_i915_private *dev_priv)
+{
+	static const struct pci_device_id isp_ids[] = {
+		{PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x0f38)},
+		{PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x22b8)},
+		{}
+	};
+
+	WARN(!pci_dev_present(isp_ids) &&
+	     !vlv_punit_is_power_gated(dev_priv, PUNIT_REG_ISPSSPM0),
+	     "ISP not power gated\n");
+}
+
 static void intel_power_domains_verify_state(struct drm_i915_private *dev_priv);
 
 /**
@@ -4006,7 +4050,7 @@ void intel_power_domains_init_hw(struct drm_i915_private *i915, bool resume)
 
 	power_domains->initializing = true;
 
-	if (IS_ICELAKE(i915)) {
+	if (INTEL_GEN(i915) >= 11) {
 		icl_display_core_init(i915, resume);
 	} else if (IS_CANNONLAKE(i915)) {
 		cnl_display_core_init(i915, resume);
@@ -4018,10 +4062,13 @@ void intel_power_domains_init_hw(struct drm_i915_private *i915, bool resume)
 		mutex_lock(&power_domains->lock);
 		chv_phy_control_init(i915);
 		mutex_unlock(&power_domains->lock);
+		assert_isp_power_gated(i915);
 	} else if (IS_VALLEYVIEW(i915)) {
 		mutex_lock(&power_domains->lock);
 		vlv_cmnlane_wa(i915);
 		mutex_unlock(&power_domains->lock);
+		assert_ved_power_gated(i915);
+		assert_isp_power_gated(i915);
 	} else if (IS_IVYBRIDGE(i915) || INTEL_GEN(i915) >= 7) {
 		intel_pch_reset_handshake(i915, !HAS_PCH_NOP(i915));
 	}
@@ -4151,7 +4198,7 @@ void intel_power_domains_suspend(struct drm_i915_private *i915,
 		intel_power_domains_verify_state(i915);
 	}
 
-	if (IS_ICELAKE(i915))
+	if (INTEL_GEN(i915) >= 11)
 		icl_display_core_uninit(i915);
 	else if (IS_CANNONLAKE(i915))
 		cnl_display_core_uninit(i915);
diff --git a/drivers/gpu/drm/i915/intel_sdvo.c b/drivers/gpu/drm/i915/intel_sdvo.c
index e7b0884ba5a5..0e3d91d9ef13 100644
--- a/drivers/gpu/drm/i915/intel_sdvo.c
+++ b/drivers/gpu/drm/i915/intel_sdvo.c
@@ -25,16 +25,23 @@
  * Authors:
  *	Eric Anholt <eric@anholt.net>
  */
-#include <linux/i2c.h>
-#include <linux/slab.h>
+
 #include <linux/delay.h>
 #include <linux/export.h>
+#include <linux/i2c.h>
+#include <linux/slab.h>
+
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_edid.h>
-#include "intel_drv.h"
 #include <drm/i915_drm.h>
+
 #include "i915_drv.h"
+#include "intel_connector.h"
+#include "intel_drv.h"
+#include "intel_hdmi.h"
+#include "intel_panel.h"
+#include "intel_sdvo.h"
 #include "intel_sdvo_regs.h"
 
 #define SDVO_TMDS_MASK (SDVO_OUTPUT_TMDS0 | SDVO_OUTPUT_TMDS1)
@@ -978,34 +985,109 @@ static bool intel_sdvo_write_infoframe(struct intel_sdvo *intel_sdvo,
 				    &tx_rate, 1);
 }
 
-static bool intel_sdvo_set_avi_infoframe(struct intel_sdvo *intel_sdvo,
-					 const struct intel_crtc_state *pipe_config,
-					 const struct drm_connector_state *conn_state)
+static ssize_t intel_sdvo_read_infoframe(struct intel_sdvo *intel_sdvo,
+					 unsigned int if_index,
+					 u8 *data, unsigned int length)
 {
+	u8 set_buf_index[2] = { if_index, 0 };
+	u8 hbuf_size, tx_rate, av_split;
+	int i;
+
+	if (!intel_sdvo_get_value(intel_sdvo,
+				  SDVO_CMD_GET_HBUF_AV_SPLIT,
+				  &av_split, 1))
+		return -ENXIO;
+
+	if (av_split < if_index)
+		return 0;
+
+	if (!intel_sdvo_get_value(intel_sdvo,
+				  SDVO_CMD_GET_HBUF_TXRATE,
+				  &tx_rate, 1))
+		return -ENXIO;
+
+	if (tx_rate == SDVO_HBUF_TX_DISABLED)
+		return 0;
+
+	if (!intel_sdvo_set_value(intel_sdvo,
+				  SDVO_CMD_SET_HBUF_INDEX,
+				  set_buf_index, 2))
+		return -ENXIO;
+
+	if (!intel_sdvo_get_value(intel_sdvo, SDVO_CMD_GET_HBUF_INFO,
+				  &hbuf_size, 1))
+		return -ENXIO;
+
+	/* Buffer size is 0 based, hooray! */
+	hbuf_size++;
+
+	DRM_DEBUG_KMS("reading sdvo hbuf: %i, hbuf_size %i, hbuf_size: %i\n",
+		      if_index, length, hbuf_size);
+
+	hbuf_size = min_t(unsigned int, length, hbuf_size);
+
+	for (i = 0; i < hbuf_size; i += 8) {
+		if (!intel_sdvo_write_cmd(intel_sdvo, SDVO_CMD_GET_HBUF_DATA, NULL, 0))
+			return -ENXIO;
+		if (!intel_sdvo_read_response(intel_sdvo, &data[i],
+					      min_t(unsigned int, 8, hbuf_size - i)))
+			return -ENXIO;
+	}
+
+	return hbuf_size;
+}
+
+static bool intel_sdvo_compute_avi_infoframe(struct intel_sdvo *intel_sdvo,
+					     struct intel_crtc_state *crtc_state,
+					     struct drm_connector_state *conn_state)
+{
+	struct hdmi_avi_infoframe *frame = &crtc_state->infoframes.avi.avi;
 	const struct drm_display_mode *adjusted_mode =
-		&pipe_config->base.adjusted_mode;
-	u8 sdvo_data[HDMI_INFOFRAME_SIZE(AVI)];
-	union hdmi_infoframe frame;
+		&crtc_state->base.adjusted_mode;
 	int ret;
-	ssize_t len;
 
-	ret = drm_hdmi_avi_infoframe_from_display_mode(&frame.avi,
+	if (!crtc_state->has_hdmi_sink)
+		return true;
+
+	crtc_state->infoframes.enable |=
+		intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_AVI);
+
+	ret = drm_hdmi_avi_infoframe_from_display_mode(frame,
 						       conn_state->connector,
 						       adjusted_mode);
-	if (ret < 0) {
-		DRM_ERROR("couldn't fill AVI infoframe\n");
+	if (ret)
 		return false;
-	}
 
-	drm_hdmi_avi_infoframe_quant_range(&frame.avi,
+	drm_hdmi_avi_infoframe_quant_range(frame,
 					   conn_state->connector,
 					   adjusted_mode,
-					   pipe_config->limited_color_range ?
+					   crtc_state->limited_color_range ?
 					   HDMI_QUANTIZATION_RANGE_LIMITED :
 					   HDMI_QUANTIZATION_RANGE_FULL);
 
-	len = hdmi_infoframe_pack(&frame, sdvo_data, sizeof(sdvo_data));
-	if (len < 0)
+	ret = hdmi_avi_infoframe_check(frame);
+	if (WARN_ON(ret))
+		return false;
+
+	return true;
+}
+
+static bool intel_sdvo_set_avi_infoframe(struct intel_sdvo *intel_sdvo,
+					 const struct intel_crtc_state *crtc_state)
+{
+	u8 sdvo_data[HDMI_INFOFRAME_SIZE(AVI)];
+	const union hdmi_infoframe *frame = &crtc_state->infoframes.avi;
+	ssize_t len;
+
+	if ((crtc_state->infoframes.enable &
+	     intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_AVI)) == 0)
+		return true;
+
+	if (WARN_ON(frame->any.type != HDMI_INFOFRAME_TYPE_AVI))
+		return false;
+
+	len = hdmi_infoframe_pack_only(frame, sdvo_data, sizeof(sdvo_data));
+	if (WARN_ON(len < 0))
 		return false;
 
 	return intel_sdvo_write_infoframe(intel_sdvo, SDVO_HBUF_INDEX_AVI_IF,
@@ -1013,6 +1095,40 @@ static bool intel_sdvo_set_avi_infoframe(struct intel_sdvo *intel_sdvo,
 					  sdvo_data, sizeof(sdvo_data));
 }
 
+static void intel_sdvo_get_avi_infoframe(struct intel_sdvo *intel_sdvo,
+					 struct intel_crtc_state *crtc_state)
+{
+	u8 sdvo_data[HDMI_INFOFRAME_SIZE(AVI)];
+	union hdmi_infoframe *frame = &crtc_state->infoframes.avi;
+	ssize_t len;
+	int ret;
+
+	if (!crtc_state->has_hdmi_sink)
+		return;
+
+	len = intel_sdvo_read_infoframe(intel_sdvo, SDVO_HBUF_INDEX_AVI_IF,
+					sdvo_data, sizeof(sdvo_data));
+	if (len < 0) {
+		DRM_DEBUG_KMS("failed to read AVI infoframe\n");
+		return;
+	} else if (len == 0) {
+		return;
+	}
+
+	crtc_state->infoframes.enable |=
+		intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_AVI);
+
+	ret = hdmi_infoframe_unpack(frame, sdvo_data, sizeof(sdvo_data));
+	if (ret) {
+		DRM_DEBUG_KMS("Failed to unpack AVI infoframe\n");
+		return;
+	}
+
+	if (frame->any.type != HDMI_INFOFRAME_TYPE_AVI)
+		DRM_DEBUG_KMS("Found the wrong infoframe type 0x%x (expected 0x%02x)\n",
+			      frame->any.type, HDMI_INFOFRAME_TYPE_AVI);
+}
+
 static bool intel_sdvo_set_tv_format(struct intel_sdvo *intel_sdvo,
 				     const struct drm_connector_state *conn_state)
 {
@@ -1193,6 +1309,12 @@ static int intel_sdvo_compute_config(struct intel_encoder *encoder,
 	if (intel_sdvo_connector->is_hdmi)
 		adjusted_mode->picture_aspect_ratio = conn_state->picture_aspect_ratio;
 
+	if (!intel_sdvo_compute_avi_infoframe(intel_sdvo,
+					      pipe_config, conn_state)) {
+		DRM_DEBUG_KMS("bad AVI infoframe\n");
+		return -EINVAL;
+	}
+
 	return 0;
 }
 
@@ -1315,8 +1437,7 @@ static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder,
 		intel_sdvo_set_encode(intel_sdvo, SDVO_ENCODE_HDMI);
 		intel_sdvo_set_colorimetry(intel_sdvo,
 					   SDVO_COLORIMETRY_RGB256);
-		intel_sdvo_set_avi_infoframe(intel_sdvo,
-					     crtc_state, conn_state);
+		intel_sdvo_set_avi_infoframe(intel_sdvo, crtc_state);
 	} else
 		intel_sdvo_set_encode(intel_sdvo, SDVO_ENCODE_DVI);
 
@@ -1507,6 +1628,10 @@ static void intel_sdvo_get_config(struct intel_encoder *encoder,
 		}
 	}
 
+	WARN(encoder_pixel_multiplier != pipe_config->pixel_multiplier,
+	     "SDVO pixel multiplier mismatch, port: %i, encoder: %i\n",
+	     pipe_config->pixel_multiplier, encoder_pixel_multiplier);
+
 	if (sdvox & HDMI_COLOR_RANGE_16_235)
 		pipe_config->limited_color_range = true;
 
@@ -1519,9 +1644,7 @@ static void intel_sdvo_get_config(struct intel_encoder *encoder,
 			pipe_config->has_hdmi_sink = true;
 	}
 
-	WARN(encoder_pixel_multiplier != pipe_config->pixel_multiplier,
-	     "SDVO pixel multiplier mismatch, port: %i, encoder: %i\n",
-	     pipe_config->pixel_multiplier, encoder_pixel_multiplier);
+	intel_sdvo_get_avi_infoframe(intel_sdvo, pipe_config);
 }
 
 static void intel_disable_sdvo(struct intel_encoder *encoder,
diff --git a/drivers/gpu/drm/i915/intel_sdvo.h b/drivers/gpu/drm/i915/intel_sdvo.h
new file mode 100644
index 000000000000..c9e05bcdd141
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_sdvo.h
@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2019 Intel Corporation
+ */
+
+#ifndef __INTEL_SDVO_H__
+#define __INTEL_SDVO_H__
+
+#include <linux/types.h>
+
+#include <drm/i915_drm.h>
+
+#include "i915_reg.h"
+
+struct drm_i915_private;
+enum pipe;
+
+bool intel_sdvo_port_enabled(struct drm_i915_private *dev_priv,
+			     i915_reg_t sdvo_reg, enum pipe *pipe);
+bool intel_sdvo_init(struct drm_i915_private *dev_priv,
+		     i915_reg_t reg, enum port port);
+
+#endif /* __INTEL_SDVO_H__ */
diff --git a/drivers/gpu/drm/i915/intel_sideband.c b/drivers/gpu/drm/i915/intel_sideband.c
index 75c872bb8cc9..57de41b1f989 100644
--- a/drivers/gpu/drm/i915/intel_sideband.c
+++ b/drivers/gpu/drm/i915/intel_sideband.c
@@ -51,7 +51,7 @@ static int vlv_sideband_rw(struct drm_i915_private *dev_priv, u32 devfn,
 
 	WARN_ON(!mutex_is_locked(&dev_priv->sb_lock));
 
-	if (intel_wait_for_register(dev_priv,
+	if (intel_wait_for_register(&dev_priv->uncore,
 				    VLV_IOSF_DOORBELL_REQ, IOSF_SB_BUSY, 0,
 				    5)) {
 		DRM_DEBUG_DRIVER("IOSF sideband idle wait (%s) timed out\n",
@@ -63,7 +63,7 @@ static int vlv_sideband_rw(struct drm_i915_private *dev_priv, u32 devfn,
 	I915_WRITE(VLV_IOSF_DATA, is_read ? 0 : *val);
 	I915_WRITE(VLV_IOSF_DOORBELL_REQ, cmd);
 
-	if (intel_wait_for_register(dev_priv,
+	if (intel_wait_for_register(&dev_priv->uncore,
 				    VLV_IOSF_DOORBELL_REQ, IOSF_SB_BUSY, 0,
 				    5)) {
 		DRM_DEBUG_DRIVER("IOSF sideband finish wait (%s) timed out\n",
@@ -208,7 +208,7 @@ u32 intel_sbi_read(struct drm_i915_private *dev_priv, u16 reg,
 	u32 value = 0;
 	WARN_ON(!mutex_is_locked(&dev_priv->sb_lock));
 
-	if (intel_wait_for_register(dev_priv,
+	if (intel_wait_for_register(&dev_priv->uncore,
 				    SBI_CTL_STAT, SBI_BUSY, 0,
 				    100)) {
 		DRM_ERROR("timeout waiting for SBI to become ready\n");
@@ -224,7 +224,7 @@ u32 intel_sbi_read(struct drm_i915_private *dev_priv, u16 reg,
 		value = SBI_CTL_DEST_MPHY | SBI_CTL_OP_IORD;
 	I915_WRITE(SBI_CTL_STAT, value | SBI_BUSY);
 
-	if (intel_wait_for_register(dev_priv,
+	if (intel_wait_for_register(&dev_priv->uncore,
 				    SBI_CTL_STAT,
 				    SBI_BUSY,
 				    0,
@@ -248,7 +248,7 @@ void intel_sbi_write(struct drm_i915_private *dev_priv, u16 reg, u32 value,
 
 	WARN_ON(!mutex_is_locked(&dev_priv->sb_lock));
 
-	if (intel_wait_for_register(dev_priv,
+	if (intel_wait_for_register(&dev_priv->uncore,
 				    SBI_CTL_STAT, SBI_BUSY, 0,
 				    100)) {
 		DRM_ERROR("timeout waiting for SBI to become ready\n");
@@ -264,7 +264,7 @@ void intel_sbi_write(struct drm_i915_private *dev_priv, u16 reg, u32 value,
 		tmp = SBI_CTL_DEST_MPHY | SBI_CTL_OP_IOWR;
 	I915_WRITE(SBI_CTL_STAT, SBI_BUSY | tmp);
 
-	if (intel_wait_for_register(dev_priv,
+	if (intel_wait_for_register(&dev_priv->uncore,
 				    SBI_CTL_STAT,
 				    SBI_BUSY,
 				    0,
diff --git a/drivers/gpu/drm/i915/intel_sprite.c b/drivers/gpu/drm/i915/intel_sprite.c
index b56a1a9ad01d..2913e89280d7 100644
--- a/drivers/gpu/drm/i915/intel_sprite.c
+++ b/drivers/gpu/drm/i915/intel_sprite.c
@@ -29,17 +29,36 @@
  * registers; newer ones are much simpler and we can use the new DRM plane
  * support.
  */
+
+#include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
+#include <drm/drm_color_mgmt.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_fourcc.h>
-#include <drm/drm_rect.h>
-#include <drm/drm_atomic.h>
 #include <drm/drm_plane_helper.h>
-#include "intel_drv.h"
-#include "intel_frontbuffer.h"
+#include <drm/drm_rect.h>
 #include <drm/i915_drm.h>
+
 #include "i915_drv.h"
-#include <drm/drm_color_mgmt.h>
+#include "intel_atomic_plane.h"
+#include "intel_drv.h"
+#include "intel_frontbuffer.h"
+#include "intel_pm.h"
+#include "intel_psr.h"
+#include "intel_sprite.h"
+
+bool is_planar_yuv_format(u32 pixelformat)
+{
+	switch (pixelformat) {
+	case DRM_FORMAT_NV12:
+	case DRM_FORMAT_P010:
+	case DRM_FORMAT_P012:
+	case DRM_FORMAT_P016:
+		return true;
+	default:
+		return false;
+	}
+}
 
 int intel_usecs_to_scanlines(const struct drm_display_mode *adjusted_mode,
 			     int usecs)
@@ -256,7 +275,8 @@ int intel_plane_check_src_coordinates(struct intel_plane_state *plane_state)
 {
 	const struct drm_framebuffer *fb = plane_state->base.fb;
 	struct drm_rect *src = &plane_state->base.src;
-	u32 src_x, src_y, src_w, src_h;
+	u32 src_x, src_y, src_w, src_h, hsub, vsub;
+	bool rotated = drm_rotation_90_or_270(plane_state->base.rotation);
 
 	/*
 	 * Hardware doesn't handle subpixel coordinates.
@@ -274,18 +294,26 @@ int intel_plane_check_src_coordinates(struct intel_plane_state *plane_state)
 	src->y1 = src_y << 16;
 	src->y2 = (src_y + src_h) << 16;
 
-	if (fb->format->is_yuv &&
-	    (src_x & 1 || src_w & 1)) {
-		DRM_DEBUG_KMS("src x/w (%u, %u) must be a multiple of 2 for YUV planes\n",
-			      src_x, src_w);
+	if (!fb->format->is_yuv)
+		return 0;
+
+	/* YUV specific checks */
+	if (!rotated) {
+		hsub = fb->format->hsub;
+		vsub = fb->format->vsub;
+	} else {
+		hsub = vsub = max(fb->format->hsub, fb->format->vsub);
+	}
+
+	if (src_x % hsub || src_w % hsub) {
+		DRM_DEBUG_KMS("src x/w (%u, %u) must be a multiple of %u for %sYUV planes\n",
+			      src_x, src_w, hsub, rotated ? "rotated " : "");
 		return -EINVAL;
 	}
 
-	if (fb->format->is_yuv &&
-	    fb->format->num_planes > 1 &&
-	    (src_y & 1 || src_h & 1)) {
-		DRM_DEBUG_KMS("src y/h (%u, %u) must be a multiple of 2 for planar YUV planes\n",
-			      src_y, src_h);
+	if (src_y % vsub || src_h % vsub) {
+		DRM_DEBUG_KMS("src y/h (%u, %u) must be a multiple of %u for %sYUV planes\n",
+			      src_y, src_h, vsub, rotated ? "rotated " : "");
 		return -EINVAL;
 	}
 
@@ -335,8 +363,8 @@ skl_program_scaler(struct intel_plane *plane,
 				      0, INT_MAX);
 
 	/* TODO: handle sub-pixel coordinates */
-	if (plane_state->base.fb->format->format == DRM_FORMAT_NV12 &&
-	    !icl_is_hdr_plane(plane)) {
+	if (is_planar_yuv_format(plane_state->base.fb->format->format) &&
+	    !icl_is_hdr_plane(dev_priv, plane->id)) {
 		y_hphase = skl_scaler_calc_phase(1, hscale, false);
 		y_vphase = skl_scaler_calc_phase(1, vscale, false);
 
@@ -518,7 +546,7 @@ skl_program_plane(struct intel_plane *plane,
 	I915_WRITE_FW(PLANE_AUX_DIST(pipe, plane_id),
 		      (plane_state->color_plane[1].offset - surf_addr) | aux_stride);
 
-	if (icl_is_hdr_plane(plane)) {
+	if (icl_is_hdr_plane(dev_priv, plane_id)) {
 		u32 cus_ctl = 0;
 
 		if (linked) {
@@ -542,7 +570,7 @@ skl_program_plane(struct intel_plane *plane,
 	if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
 		I915_WRITE_FW(PLANE_COLOR_CTL(pipe, plane_id), plane_color_ctl);
 
-	if (fb->format->is_yuv && icl_is_hdr_plane(plane))
+	if (fb->format->is_yuv && icl_is_hdr_plane(dev_priv, plane_id))
 		icl_program_input_csc(plane, crtc_state, plane_state);
 
 	skl_write_plane_wm(plane, crtc_state);
@@ -609,6 +637,9 @@ skl_disable_plane(struct intel_plane *plane,
 
 	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 
+	if (icl_is_hdr_plane(dev_priv, plane_id))
+		I915_WRITE_FW(PLANE_CUS_CTL(pipe, plane_id), 0);
+
 	skl_write_plane_wm(plane, crtc_state);
 
 	I915_WRITE_FW(PLANE_CTL(pipe, plane_id), 0);
@@ -741,7 +772,12 @@ vlv_update_clrc(const struct intel_plane_state *plane_state)
 
 static u32 vlv_sprite_ctl_crtc(const struct intel_crtc_state *crtc_state)
 {
-	return SP_GAMMA_ENABLE;
+	u32 sprctl = 0;
+
+	if (crtc_state->gamma_enable)
+		sprctl |= SP_GAMMA_ENABLE;
+
+	return sprctl;
 }
 
 static u32 vlv_sprite_ctl(const struct intel_crtc_state *crtc_state,
@@ -916,12 +952,12 @@ vlv_plane_get_hw_state(struct intel_plane *plane,
 
 static u32 ivb_sprite_ctl_crtc(const struct intel_crtc_state *crtc_state)
 {
-	struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev);
 	u32 sprctl = 0;
 
-	sprctl |= SPRITE_GAMMA_ENABLE;
+	if (crtc_state->gamma_enable)
+		sprctl |= SPRITE_GAMMA_ENABLE;
 
-	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
+	if (crtc_state->csc_enable)
 		sprctl |= SPRITE_PIPE_CSC_ENABLE;
 
 	return sprctl;
@@ -1107,7 +1143,15 @@ g4x_sprite_max_stride(struct intel_plane *plane,
 
 static u32 g4x_sprite_ctl_crtc(const struct intel_crtc_state *crtc_state)
 {
-	return DVS_GAMMA_ENABLE;
+	u32 dvscntr = 0;
+
+	if (crtc_state->gamma_enable)
+		dvscntr |= DVS_GAMMA_ENABLE;
+
+	if (crtc_state->csc_enable)
+		dvscntr |= DVS_PIPE_CSC_ENABLE;
+
+	return dvscntr;
 }
 
 static u32 g4x_sprite_ctl(const struct intel_crtc_state *crtc_state,
@@ -1482,8 +1526,6 @@ static int skl_plane_check_fb(const struct intel_crtc_state *crtc_state,
 		/*
 		 * 90/270 is not allowed with RGB64 16:16:16:16 and
 		 * Indexed 8-bit. RGB 16-bit 5:6:5 is allowed gen11 onwards.
-		 * TBD: Add RGB64 case once its added in supported format
-		 * list.
 		 */
 		switch (fb->format->format) {
 		case DRM_FORMAT_RGB565:
@@ -1491,6 +1533,15 @@ static int skl_plane_check_fb(const struct intel_crtc_state *crtc_state,
 				break;
 			/* fall through */
 		case DRM_FORMAT_C8:
+		case DRM_FORMAT_XRGB16161616F:
+		case DRM_FORMAT_XBGR16161616F:
+		case DRM_FORMAT_ARGB16161616F:
+		case DRM_FORMAT_ABGR16161616F:
+		case DRM_FORMAT_Y210:
+		case DRM_FORMAT_Y212:
+		case DRM_FORMAT_Y216:
+		case DRM_FORMAT_XVYU12_16161616:
+		case DRM_FORMAT_XVYU16161616:
 			DRM_DEBUG_KMS("Unsupported pixel format %s for 90/270!\n",
 				      drm_get_format_name(fb->format->format,
 							  &format_name));
@@ -1551,10 +1602,10 @@ static int skl_plane_check_nv12_rotation(const struct intel_plane_state *plane_s
 	int src_w = drm_rect_width(&plane_state->base.src) >> 16;
 
 	/* Display WA #1106 */
-	if (fb->format->format == DRM_FORMAT_NV12 && src_w & 3 &&
+	if (is_planar_yuv_format(fb->format->format) && src_w & 3 &&
 	    (rotation == DRM_MODE_ROTATE_270 ||
 	     rotation == (DRM_MODE_REFLECT_X | DRM_MODE_ROTATE_90))) {
-		DRM_DEBUG_KMS("src width must be multiple of 4 for rotated NV12\n");
+		DRM_DEBUG_KMS("src width must be multiple of 4 for rotated planar YUV\n");
 		return -EINVAL;
 	}
 
@@ -1790,6 +1841,52 @@ static const u32 skl_plane_formats[] = {
 	DRM_FORMAT_VYUY,
 };
 
+static const u32 icl_plane_formats[] = {
+	DRM_FORMAT_C8,
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_XBGR8888,
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_ABGR8888,
+	DRM_FORMAT_XRGB2101010,
+	DRM_FORMAT_XBGR2101010,
+	DRM_FORMAT_YUYV,
+	DRM_FORMAT_YVYU,
+	DRM_FORMAT_UYVY,
+	DRM_FORMAT_VYUY,
+	DRM_FORMAT_Y210,
+	DRM_FORMAT_Y212,
+	DRM_FORMAT_Y216,
+	DRM_FORMAT_XVYU2101010,
+	DRM_FORMAT_XVYU12_16161616,
+	DRM_FORMAT_XVYU16161616,
+};
+
+static const u32 icl_hdr_plane_formats[] = {
+	DRM_FORMAT_C8,
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_XBGR8888,
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_ABGR8888,
+	DRM_FORMAT_XRGB2101010,
+	DRM_FORMAT_XBGR2101010,
+	DRM_FORMAT_XRGB16161616F,
+	DRM_FORMAT_XBGR16161616F,
+	DRM_FORMAT_ARGB16161616F,
+	DRM_FORMAT_ABGR16161616F,
+	DRM_FORMAT_YUYV,
+	DRM_FORMAT_YVYU,
+	DRM_FORMAT_UYVY,
+	DRM_FORMAT_VYUY,
+	DRM_FORMAT_Y210,
+	DRM_FORMAT_Y212,
+	DRM_FORMAT_Y216,
+	DRM_FORMAT_XVYU2101010,
+	DRM_FORMAT_XVYU12_16161616,
+	DRM_FORMAT_XVYU16161616,
+};
+
 static const u32 skl_planar_formats[] = {
 	DRM_FORMAT_C8,
 	DRM_FORMAT_RGB565,
@@ -1806,6 +1903,79 @@ static const u32 skl_planar_formats[] = {
 	DRM_FORMAT_NV12,
 };
 
+static const u32 glk_planar_formats[] = {
+	DRM_FORMAT_C8,
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_XBGR8888,
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_ABGR8888,
+	DRM_FORMAT_XRGB2101010,
+	DRM_FORMAT_XBGR2101010,
+	DRM_FORMAT_YUYV,
+	DRM_FORMAT_YVYU,
+	DRM_FORMAT_UYVY,
+	DRM_FORMAT_VYUY,
+	DRM_FORMAT_NV12,
+	DRM_FORMAT_P010,
+	DRM_FORMAT_P012,
+	DRM_FORMAT_P016,
+};
+
+static const u32 icl_planar_formats[] = {
+	DRM_FORMAT_C8,
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_XBGR8888,
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_ABGR8888,
+	DRM_FORMAT_XRGB2101010,
+	DRM_FORMAT_XBGR2101010,
+	DRM_FORMAT_YUYV,
+	DRM_FORMAT_YVYU,
+	DRM_FORMAT_UYVY,
+	DRM_FORMAT_VYUY,
+	DRM_FORMAT_NV12,
+	DRM_FORMAT_P010,
+	DRM_FORMAT_P012,
+	DRM_FORMAT_P016,
+	DRM_FORMAT_Y210,
+	DRM_FORMAT_Y212,
+	DRM_FORMAT_Y216,
+	DRM_FORMAT_XVYU2101010,
+	DRM_FORMAT_XVYU12_16161616,
+	DRM_FORMAT_XVYU16161616,
+};
+
+static const u32 icl_hdr_planar_formats[] = {
+	DRM_FORMAT_C8,
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_XBGR8888,
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_ABGR8888,
+	DRM_FORMAT_XRGB2101010,
+	DRM_FORMAT_XBGR2101010,
+	DRM_FORMAT_XRGB16161616F,
+	DRM_FORMAT_XBGR16161616F,
+	DRM_FORMAT_ARGB16161616F,
+	DRM_FORMAT_ABGR16161616F,
+	DRM_FORMAT_YUYV,
+	DRM_FORMAT_YVYU,
+	DRM_FORMAT_UYVY,
+	DRM_FORMAT_VYUY,
+	DRM_FORMAT_NV12,
+	DRM_FORMAT_P010,
+	DRM_FORMAT_P012,
+	DRM_FORMAT_P016,
+	DRM_FORMAT_Y210,
+	DRM_FORMAT_Y212,
+	DRM_FORMAT_Y216,
+	DRM_FORMAT_XVYU2101010,
+	DRM_FORMAT_XVYU12_16161616,
+	DRM_FORMAT_XVYU16161616,
+};
+
 static const u64 skl_plane_format_modifiers_noccs[] = {
 	I915_FORMAT_MOD_Yf_TILED,
 	I915_FORMAT_MOD_Y_TILED,
@@ -1945,10 +2115,23 @@ static bool skl_plane_format_mod_supported(struct drm_plane *_plane,
 	case DRM_FORMAT_UYVY:
 	case DRM_FORMAT_VYUY:
 	case DRM_FORMAT_NV12:
+	case DRM_FORMAT_P010:
+	case DRM_FORMAT_P012:
+	case DRM_FORMAT_P016:
+	case DRM_FORMAT_XVYU2101010:
 		if (modifier == I915_FORMAT_MOD_Yf_TILED)
 			return true;
 		/* fall through */
 	case DRM_FORMAT_C8:
+	case DRM_FORMAT_XBGR16161616F:
+	case DRM_FORMAT_ABGR16161616F:
+	case DRM_FORMAT_XRGB16161616F:
+	case DRM_FORMAT_ARGB16161616F:
+	case DRM_FORMAT_Y210:
+	case DRM_FORMAT_Y212:
+	case DRM_FORMAT_Y216:
+	case DRM_FORMAT_XVYU12_16161616:
+	case DRM_FORMAT_XVYU16161616:
 		if (modifier == DRM_FORMAT_MOD_LINEAR ||
 		    modifier == I915_FORMAT_MOD_X_TILED ||
 		    modifier == I915_FORMAT_MOD_Y_TILED)
@@ -2085,8 +2268,25 @@ skl_universal_plane_create(struct drm_i915_private *dev_priv,
 		plane->update_slave = icl_update_slave;
 
 	if (skl_plane_has_planar(dev_priv, pipe, plane_id)) {
-		formats = skl_planar_formats;
-		num_formats = ARRAY_SIZE(skl_planar_formats);
+		if (icl_is_hdr_plane(dev_priv, plane_id)) {
+			formats = icl_hdr_planar_formats;
+			num_formats = ARRAY_SIZE(icl_hdr_planar_formats);
+		} else if (INTEL_GEN(dev_priv) >= 11) {
+			formats = icl_planar_formats;
+			num_formats = ARRAY_SIZE(icl_planar_formats);
+		} else if (INTEL_GEN(dev_priv) == 10 || IS_GEMINILAKE(dev_priv)) {
+			formats = glk_planar_formats;
+			num_formats = ARRAY_SIZE(glk_planar_formats);
+		} else {
+			formats = skl_planar_formats;
+			num_formats = ARRAY_SIZE(skl_planar_formats);
+		}
+	} else if (icl_is_hdr_plane(dev_priv, plane_id)) {
+		formats = icl_hdr_plane_formats;
+		num_formats = ARRAY_SIZE(icl_hdr_plane_formats);
+	} else if (INTEL_GEN(dev_priv) >= 11) {
+		formats = icl_plane_formats;
+		num_formats = ARRAY_SIZE(icl_plane_formats);
 	} else {
 		formats = skl_plane_formats;
 		num_formats = ARRAY_SIZE(skl_plane_formats);
diff --git a/drivers/gpu/drm/i915/intel_sprite.h b/drivers/gpu/drm/i915/intel_sprite.h
new file mode 100644
index 000000000000..84be8686be16
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_sprite.h
@@ -0,0 +1,55 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2019 Intel Corporation
+ */
+
+#ifndef __INTEL_SPRITE_H__
+#define __INTEL_SPRITE_H__
+
+#include <linux/types.h>
+
+#include "i915_drv.h"
+#include "intel_display.h"
+
+struct drm_device;
+struct drm_display_mode;
+struct drm_file;
+struct drm_i915_private;
+struct intel_crtc_state;
+struct intel_plane_state;
+
+bool is_planar_yuv_format(u32 pixelformat);
+int intel_usecs_to_scanlines(const struct drm_display_mode *adjusted_mode,
+			     int usecs);
+struct intel_plane *intel_sprite_plane_create(struct drm_i915_private *dev_priv,
+					      enum pipe pipe, int plane);
+int intel_sprite_set_colorkey_ioctl(struct drm_device *dev, void *data,
+				    struct drm_file *file_priv);
+void intel_pipe_update_start(const struct intel_crtc_state *new_crtc_state);
+void intel_pipe_update_end(struct intel_crtc_state *new_crtc_state);
+int intel_plane_check_stride(const struct intel_plane_state *plane_state);
+int intel_plane_check_src_coordinates(struct intel_plane_state *plane_state);
+int chv_plane_check_rotation(const struct intel_plane_state *plane_state);
+struct intel_plane *
+skl_universal_plane_create(struct drm_i915_private *dev_priv,
+			   enum pipe pipe, enum plane_id plane_id);
+
+static inline bool icl_is_nv12_y_plane(enum plane_id id)
+{
+	/* Don't need to do a gen check, these planes are only available on gen11 */
+	if (id == PLANE_SPRITE4 || id == PLANE_SPRITE5)
+		return true;
+
+	return false;
+}
+
+static inline bool icl_is_hdr_plane(struct drm_i915_private *dev_priv,
+				    enum plane_id plane_id)
+{
+	if (INTEL_GEN(dev_priv) < 11)
+		return false;
+
+	return plane_id < PLANE_SPRITE2;
+}
+
+#endif /* __INTEL_SPRITE_H__ */
diff --git a/drivers/gpu/drm/i915/intel_tv.c b/drivers/gpu/drm/i915/intel_tv.c
index 3924c4944e1f..5dbba33f4202 100644
--- a/drivers/gpu/drm/i915/intel_tv.c
+++ b/drivers/gpu/drm/i915/intel_tv.c
@@ -33,9 +33,12 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_edid.h>
-#include "intel_drv.h"
 #include <drm/i915_drm.h>
+
 #include "i915_drv.h"
+#include "intel_connector.h"
+#include "intel_drv.h"
+#include "intel_tv.h"
 
 enum tv_margin {
 	TV_MARGIN_LEFT, TV_MARGIN_TOP,
diff --git a/drivers/gpu/drm/i915/intel_tv.h b/drivers/gpu/drm/i915/intel_tv.h
new file mode 100644
index 000000000000..44518575ec5c
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_tv.h
@@ -0,0 +1,13 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2019 Intel Corporation
+ */
+
+#ifndef __INTEL_TV_H__
+#define __INTEL_TV_H__
+
+struct drm_i915_private;
+
+void intel_tv_init(struct drm_i915_private *dev_priv);
+
+#endif /* __INTEL_TV_H__ */
diff --git a/drivers/gpu/drm/i915/intel_uc.c b/drivers/gpu/drm/i915/intel_uc.c
index e711eb3268bc..25b80ffe71ad 100644
--- a/drivers/gpu/drm/i915/intel_uc.c
+++ b/drivers/gpu/drm/i915/intel_uc.c
@@ -332,8 +332,6 @@ void intel_uc_sanitize(struct drm_i915_private *i915)
 
 	GEM_BUG_ON(!HAS_GUC(i915));
 
-	guc_disable_communication(guc);
-
 	intel_huc_sanitize(huc);
 	intel_guc_sanitize(guc);
 
@@ -377,7 +375,7 @@ int intel_uc_init_hw(struct drm_i915_private *i915)
 
 		intel_guc_init_params(guc);
 		ret = intel_guc_fw_upload(guc);
-		if (ret == 0 || ret != -ETIMEDOUT)
+		if (ret == 0)
 			break;
 
 		DRM_DEBUG_DRIVER("GuC fw load failed: %d; will reset and "
@@ -451,6 +449,23 @@ void intel_uc_fini_hw(struct drm_i915_private *i915)
 	guc_disable_communication(guc);
 }
 
+/**
+ * intel_uc_reset_prepare - Prepare for reset
+ * @i915: device private
+ *
+ * Preparing for full gpu reset.
+ */
+void intel_uc_reset_prepare(struct drm_i915_private *i915)
+{
+	struct intel_guc *guc = &i915->guc;
+
+	if (!USES_GUC(i915))
+		return;
+
+	guc_disable_communication(guc);
+	intel_uc_sanitize(i915);
+}
+
 int intel_uc_suspend(struct drm_i915_private *i915)
 {
 	struct intel_guc *guc = &i915->guc;
@@ -468,7 +483,7 @@ int intel_uc_suspend(struct drm_i915_private *i915)
 		return err;
 	}
 
-	gen9_disable_guc_interrupts(i915);
+	guc_disable_communication(guc);
 
 	return 0;
 }
@@ -484,7 +499,7 @@ int intel_uc_resume(struct drm_i915_private *i915)
 	if (guc->fw.load_status != INTEL_UC_FIRMWARE_SUCCESS)
 		return 0;
 
-	gen9_enable_guc_interrupts(i915);
+	guc_enable_communication(guc);
 
 	err = intel_guc_resume(guc);
 	if (err) {
diff --git a/drivers/gpu/drm/i915/intel_uc.h b/drivers/gpu/drm/i915/intel_uc.h
index 870faf9011b9..c14729786652 100644
--- a/drivers/gpu/drm/i915/intel_uc.h
+++ b/drivers/gpu/drm/i915/intel_uc.h
@@ -38,6 +38,7 @@ int intel_uc_init_hw(struct drm_i915_private *dev_priv);
 void intel_uc_fini_hw(struct drm_i915_private *dev_priv);
 int intel_uc_init(struct drm_i915_private *dev_priv);
 void intel_uc_fini(struct drm_i915_private *dev_priv);
+void intel_uc_reset_prepare(struct drm_i915_private *i915);
 int intel_uc_suspend(struct drm_i915_private *dev_priv);
 int intel_uc_resume(struct drm_i915_private *dev_priv);
 
diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c
index 75646a1e0051..d1d51e1121e2 100644
--- a/drivers/gpu/drm/i915/intel_uncore.c
+++ b/drivers/gpu/drm/i915/intel_uncore.c
@@ -21,17 +21,18 @@
  * IN THE SOFTWARE.
  */
 
+#include <linux/pm_runtime.h>
+#include <asm/iosf_mbi.h>
+
 #include "i915_drv.h"
-#include "intel_drv.h"
 #include "i915_vgpu.h"
-
-#include <asm/iosf_mbi.h>
-#include <linux/pm_runtime.h>
+#include "intel_drv.h"
+#include "intel_pm.h"
 
 #define FORCEWAKE_ACK_TIMEOUT_MS 50
 #define GT_FIFO_TIMEOUT_MS	 10
 
-#define __raw_posting_read(dev_priv__, reg__) (void)__raw_i915_read32((dev_priv__), (reg__))
+#define __raw_posting_read(...) ((void)__raw_uncore_read32(__VA_ARGS__))
 
 static const char * const forcewake_domain_names[] = {
 	"render",
@@ -58,16 +59,20 @@ intel_uncore_forcewake_domain_to_str(const enum forcewake_domain_id id)
 	return "unknown";
 }
 
+#define fw_ack(d) readl((d)->reg_ack)
+#define fw_set(d, val) writel(_MASKED_BIT_ENABLE((val)), (d)->reg_set)
+#define fw_clear(d, val) writel(_MASKED_BIT_DISABLE((val)), (d)->reg_set)
+
 static inline void
-fw_domain_reset(struct drm_i915_private *i915,
-		const struct intel_uncore_forcewake_domain *d)
+fw_domain_reset(const struct intel_uncore_forcewake_domain *d)
 {
 	/*
 	 * We don't really know if the powerwell for the forcewake domain we are
 	 * trying to reset here does exist at this point (engines could be fused
 	 * off in ICL+), so no waiting for acks
 	 */
-	__raw_i915_write32(i915, d->reg_set, i915->uncore.fw_reset);
+	/* WaRsClearFWBitsAtReset:bdw,skl */
+	fw_clear(d, 0xffff);
 }
 
 static inline void
@@ -81,36 +86,32 @@ fw_domain_arm_timer(struct intel_uncore_forcewake_domain *d)
 }
 
 static inline int
-__wait_for_ack(const struct drm_i915_private *i915,
-	       const struct intel_uncore_forcewake_domain *d,
+__wait_for_ack(const struct intel_uncore_forcewake_domain *d,
 	       const u32 ack,
 	       const u32 value)
 {
-	return wait_for_atomic((__raw_i915_read32(i915, d->reg_ack) & ack) == value,
+	return wait_for_atomic((fw_ack(d) & ack) == value,
 			       FORCEWAKE_ACK_TIMEOUT_MS);
 }
 
 static inline int
-wait_ack_clear(const struct drm_i915_private *i915,
-	       const struct intel_uncore_forcewake_domain *d,
+wait_ack_clear(const struct intel_uncore_forcewake_domain *d,
 	       const u32 ack)
 {
-	return __wait_for_ack(i915, d, ack, 0);
+	return __wait_for_ack(d, ack, 0);
 }
 
 static inline int
-wait_ack_set(const struct drm_i915_private *i915,
-	     const struct intel_uncore_forcewake_domain *d,
+wait_ack_set(const struct intel_uncore_forcewake_domain *d,
 	     const u32 ack)
 {
-	return __wait_for_ack(i915, d, ack, ack);
+	return __wait_for_ack(d, ack, ack);
 }
 
 static inline void
-fw_domain_wait_ack_clear(const struct drm_i915_private *i915,
-			 const struct intel_uncore_forcewake_domain *d)
+fw_domain_wait_ack_clear(const struct intel_uncore_forcewake_domain *d)
 {
-	if (wait_ack_clear(i915, d, FORCEWAKE_KERNEL))
+	if (wait_ack_clear(d, FORCEWAKE_KERNEL))
 		DRM_ERROR("%s: timed out waiting for forcewake ack to clear.\n",
 			  intel_uncore_forcewake_domain_to_str(d->id));
 }
@@ -121,8 +122,7 @@ enum ack_type {
 };
 
 static int
-fw_domain_wait_ack_with_fallback(const struct drm_i915_private *i915,
-				 const struct intel_uncore_forcewake_domain *d,
+fw_domain_wait_ack_with_fallback(const struct intel_uncore_forcewake_domain *d,
 				 const enum ack_type type)
 {
 	const u32 ack_bit = FORCEWAKE_KERNEL;
@@ -146,129 +146,122 @@ fw_domain_wait_ack_with_fallback(const struct drm_i915_private *i915,
 
 	pass = 1;
 	do {
-		wait_ack_clear(i915, d, FORCEWAKE_KERNEL_FALLBACK);
+		wait_ack_clear(d, FORCEWAKE_KERNEL_FALLBACK);
 
-		__raw_i915_write32(i915, d->reg_set,
-				   _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL_FALLBACK));
+		fw_set(d, FORCEWAKE_KERNEL_FALLBACK);
 		/* Give gt some time to relax before the polling frenzy */
 		udelay(10 * pass);
-		wait_ack_set(i915, d, FORCEWAKE_KERNEL_FALLBACK);
+		wait_ack_set(d, FORCEWAKE_KERNEL_FALLBACK);
 
-		ack_detected = (__raw_i915_read32(i915, d->reg_ack) & ack_bit) == value;
+		ack_detected = (fw_ack(d) & ack_bit) == value;
 
-		__raw_i915_write32(i915, d->reg_set,
-				   _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL_FALLBACK));
+		fw_clear(d, FORCEWAKE_KERNEL_FALLBACK);
 	} while (!ack_detected && pass++ < 10);
 
 	DRM_DEBUG_DRIVER("%s had to use fallback to %s ack, 0x%x (passes %u)\n",
 			 intel_uncore_forcewake_domain_to_str(d->id),
 			 type == ACK_SET ? "set" : "clear",
-			 __raw_i915_read32(i915, d->reg_ack),
+			 fw_ack(d),
 			 pass);
 
 	return ack_detected ? 0 : -ETIMEDOUT;
 }
 
 static inline void
-fw_domain_wait_ack_clear_fallback(const struct drm_i915_private *i915,
-				  const struct intel_uncore_forcewake_domain *d)
+fw_domain_wait_ack_clear_fallback(const struct intel_uncore_forcewake_domain *d)
 {
-	if (likely(!wait_ack_clear(i915, d, FORCEWAKE_KERNEL)))
+	if (likely(!wait_ack_clear(d, FORCEWAKE_KERNEL)))
 		return;
 
-	if (fw_domain_wait_ack_with_fallback(i915, d, ACK_CLEAR))
-		fw_domain_wait_ack_clear(i915, d);
+	if (fw_domain_wait_ack_with_fallback(d, ACK_CLEAR))
+		fw_domain_wait_ack_clear(d);
 }
 
 static inline void
-fw_domain_get(struct drm_i915_private *i915,
-	      const struct intel_uncore_forcewake_domain *d)
+fw_domain_get(const struct intel_uncore_forcewake_domain *d)
 {
-	__raw_i915_write32(i915, d->reg_set, i915->uncore.fw_set);
+	fw_set(d, FORCEWAKE_KERNEL);
 }
 
 static inline void
-fw_domain_wait_ack_set(const struct drm_i915_private *i915,
-		       const struct intel_uncore_forcewake_domain *d)
+fw_domain_wait_ack_set(const struct intel_uncore_forcewake_domain *d)
 {
-	if (wait_ack_set(i915, d, FORCEWAKE_KERNEL))
+	if (wait_ack_set(d, FORCEWAKE_KERNEL))
 		DRM_ERROR("%s: timed out waiting for forcewake ack request.\n",
 			  intel_uncore_forcewake_domain_to_str(d->id));
 }
 
 static inline void
-fw_domain_wait_ack_set_fallback(const struct drm_i915_private *i915,
-				const struct intel_uncore_forcewake_domain *d)
+fw_domain_wait_ack_set_fallback(const struct intel_uncore_forcewake_domain *d)
 {
-	if (likely(!wait_ack_set(i915, d, FORCEWAKE_KERNEL)))
+	if (likely(!wait_ack_set(d, FORCEWAKE_KERNEL)))
 		return;
 
-	if (fw_domain_wait_ack_with_fallback(i915, d, ACK_SET))
-		fw_domain_wait_ack_set(i915, d);
+	if (fw_domain_wait_ack_with_fallback(d, ACK_SET))
+		fw_domain_wait_ack_set(d);
 }
 
 static inline void
-fw_domain_put(const struct drm_i915_private *i915,
-	      const struct intel_uncore_forcewake_domain *d)
+fw_domain_put(const struct intel_uncore_forcewake_domain *d)
 {
-	__raw_i915_write32(i915, d->reg_set, i915->uncore.fw_clear);
+	fw_clear(d, FORCEWAKE_KERNEL);
 }
 
 static void
-fw_domains_get(struct drm_i915_private *i915, enum forcewake_domains fw_domains)
+fw_domains_get(struct intel_uncore *uncore, enum forcewake_domains fw_domains)
 {
 	struct intel_uncore_forcewake_domain *d;
 	unsigned int tmp;
 
-	GEM_BUG_ON(fw_domains & ~i915->uncore.fw_domains);
+	GEM_BUG_ON(fw_domains & ~uncore->fw_domains);
 
-	for_each_fw_domain_masked(d, fw_domains, i915, tmp) {
-		fw_domain_wait_ack_clear(i915, d);
-		fw_domain_get(i915, d);
+	for_each_fw_domain_masked(d, fw_domains, uncore, tmp) {
+		fw_domain_wait_ack_clear(d);
+		fw_domain_get(d);
 	}
 
-	for_each_fw_domain_masked(d, fw_domains, i915, tmp)
-		fw_domain_wait_ack_set(i915, d);
+	for_each_fw_domain_masked(d, fw_domains, uncore, tmp)
+		fw_domain_wait_ack_set(d);
 
-	i915->uncore.fw_domains_active |= fw_domains;
+	uncore->fw_domains_active |= fw_domains;
 }
 
 static void
-fw_domains_get_with_fallback(struct drm_i915_private *i915,
+fw_domains_get_with_fallback(struct intel_uncore *uncore,
 			     enum forcewake_domains fw_domains)
 {
 	struct intel_uncore_forcewake_domain *d;
 	unsigned int tmp;
 
-	GEM_BUG_ON(fw_domains & ~i915->uncore.fw_domains);
+	GEM_BUG_ON(fw_domains & ~uncore->fw_domains);
 
-	for_each_fw_domain_masked(d, fw_domains, i915, tmp) {
-		fw_domain_wait_ack_clear_fallback(i915, d);
-		fw_domain_get(i915, d);
+	for_each_fw_domain_masked(d, fw_domains, uncore, tmp) {
+		fw_domain_wait_ack_clear_fallback(d);
+		fw_domain_get(d);
 	}
 
-	for_each_fw_domain_masked(d, fw_domains, i915, tmp)
-		fw_domain_wait_ack_set_fallback(i915, d);
+	for_each_fw_domain_masked(d, fw_domains, uncore, tmp)
+		fw_domain_wait_ack_set_fallback(d);
 
-	i915->uncore.fw_domains_active |= fw_domains;
+	uncore->fw_domains_active |= fw_domains;
 }
 
 static void
-fw_domains_put(struct drm_i915_private *i915, enum forcewake_domains fw_domains)
+fw_domains_put(struct intel_uncore *uncore, enum forcewake_domains fw_domains)
 {
 	struct intel_uncore_forcewake_domain *d;
 	unsigned int tmp;
 
-	GEM_BUG_ON(fw_domains & ~i915->uncore.fw_domains);
+	GEM_BUG_ON(fw_domains & ~uncore->fw_domains);
 
-	for_each_fw_domain_masked(d, fw_domains, i915, tmp)
-		fw_domain_put(i915, d);
+	for_each_fw_domain_masked(d, fw_domains, uncore, tmp)
+		fw_domain_put(d);
 
-	i915->uncore.fw_domains_active &= ~fw_domains;
+	uncore->fw_domains_active &= ~fw_domains;
 }
 
 static void
-fw_domains_reset(struct drm_i915_private *i915,
+fw_domains_reset(struct intel_uncore *uncore,
 		 enum forcewake_domains fw_domains)
 {
 	struct intel_uncore_forcewake_domain *d;
@@ -277,61 +270,61 @@ fw_domains_reset(struct drm_i915_private *i915,
 	if (!fw_domains)
 		return;
 
-	GEM_BUG_ON(fw_domains & ~i915->uncore.fw_domains);
+	GEM_BUG_ON(fw_domains & ~uncore->fw_domains);
 
-	for_each_fw_domain_masked(d, fw_domains, i915, tmp)
-		fw_domain_reset(i915, d);
+	for_each_fw_domain_masked(d, fw_domains, uncore, tmp)
+		fw_domain_reset(d);
 }
 
-static inline u32 gt_thread_status(struct drm_i915_private *dev_priv)
+static inline u32 gt_thread_status(struct intel_uncore *uncore)
 {
 	u32 val;
 
-	val = __raw_i915_read32(dev_priv, GEN6_GT_THREAD_STATUS_REG);
+	val = __raw_uncore_read32(uncore, GEN6_GT_THREAD_STATUS_REG);
 	val &= GEN6_GT_THREAD_STATUS_CORE_MASK;
 
 	return val;
 }
 
-static void __gen6_gt_wait_for_thread_c0(struct drm_i915_private *dev_priv)
+static void __gen6_gt_wait_for_thread_c0(struct intel_uncore *uncore)
 {
 	/*
 	 * w/a for a sporadic read returning 0 by waiting for the GT
 	 * thread to wake up.
 	 */
-	WARN_ONCE(wait_for_atomic_us(gt_thread_status(dev_priv) == 0, 5000),
+	WARN_ONCE(wait_for_atomic_us(gt_thread_status(uncore) == 0, 5000),
 		  "GT thread status wait timed out\n");
 }
 
-static void fw_domains_get_with_thread_status(struct drm_i915_private *dev_priv,
+static void fw_domains_get_with_thread_status(struct intel_uncore *uncore,
 					      enum forcewake_domains fw_domains)
 {
-	fw_domains_get(dev_priv, fw_domains);
+	fw_domains_get(uncore, fw_domains);
 
 	/* WaRsForcewakeWaitTC0:snb,ivb,hsw,bdw,vlv */
-	__gen6_gt_wait_for_thread_c0(dev_priv);
+	__gen6_gt_wait_for_thread_c0(uncore);
 }
 
-static inline u32 fifo_free_entries(struct drm_i915_private *dev_priv)
+static inline u32 fifo_free_entries(struct intel_uncore *uncore)
 {
-	u32 count = __raw_i915_read32(dev_priv, GTFIFOCTL);
+	u32 count = __raw_uncore_read32(uncore, GTFIFOCTL);
 
 	return count & GT_FIFO_FREE_ENTRIES_MASK;
 }
 
-static void __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv)
+static void __gen6_gt_wait_for_fifo(struct intel_uncore *uncore)
 {
 	u32 n;
 
 	/* On VLV, FIFO will be shared by both SW and HW.
 	 * So, we need to read the FREE_ENTRIES everytime */
-	if (IS_VALLEYVIEW(dev_priv))
-		n = fifo_free_entries(dev_priv);
+	if (IS_VALLEYVIEW(uncore_to_i915(uncore)))
+		n = fifo_free_entries(uncore);
 	else
-		n = dev_priv->uncore.fifo_count;
+		n = uncore->fifo_count;
 
 	if (n <= GT_FIFO_NUM_RESERVED_ENTRIES) {
-		if (wait_for_atomic((n = fifo_free_entries(dev_priv)) >
+		if (wait_for_atomic((n = fifo_free_entries(uncore)) >
 				    GT_FIFO_NUM_RESERVED_ENTRIES,
 				    GT_FIFO_TIMEOUT_MS)) {
 			DRM_DEBUG("GT_FIFO timeout, entries: %u\n", n);
@@ -339,7 +332,7 @@ static void __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv)
 		}
 	}
 
-	dev_priv->uncore.fifo_count = n - 1;
+	uncore->fifo_count = n - 1;
 }
 
 static enum hrtimer_restart
@@ -347,30 +340,29 @@ intel_uncore_fw_release_timer(struct hrtimer *timer)
 {
 	struct intel_uncore_forcewake_domain *domain =
 	       container_of(timer, struct intel_uncore_forcewake_domain, timer);
-	struct drm_i915_private *dev_priv =
-		container_of(domain, struct drm_i915_private, uncore.fw_domain[domain->id]);
+	struct intel_uncore *uncore = forcewake_domain_to_uncore(domain);
 	unsigned long irqflags;
 
-	assert_rpm_device_not_suspended(dev_priv);
+	assert_rpm_device_not_suspended(uncore->rpm);
 
 	if (xchg(&domain->active, false))
 		return HRTIMER_RESTART;
 
-	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
+	spin_lock_irqsave(&uncore->lock, irqflags);
 	if (WARN_ON(domain->wake_count == 0))
 		domain->wake_count++;
 
 	if (--domain->wake_count == 0)
-		dev_priv->uncore.funcs.force_wake_put(dev_priv, domain->mask);
+		uncore->funcs.force_wake_put(uncore, domain->mask);
 
-	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
+	spin_unlock_irqrestore(&uncore->lock, irqflags);
 
 	return HRTIMER_NORESTART;
 }
 
 /* Note callers must have acquired the PUNIT->PMIC bus, before calling this. */
 static unsigned int
-intel_uncore_forcewake_reset(struct drm_i915_private *dev_priv)
+intel_uncore_forcewake_reset(struct intel_uncore *uncore)
 {
 	unsigned long irqflags;
 	struct intel_uncore_forcewake_domain *domain;
@@ -388,7 +380,7 @@ intel_uncore_forcewake_reset(struct drm_i915_private *dev_priv)
 
 		active_domains = 0;
 
-		for_each_fw_domain(domain, dev_priv, tmp) {
+		for_each_fw_domain(domain, uncore, tmp) {
 			smp_store_mb(domain->active, false);
 			if (hrtimer_cancel(&domain->timer) == 0)
 				continue;
@@ -396,9 +388,9 @@ intel_uncore_forcewake_reset(struct drm_i915_private *dev_priv)
 			intel_uncore_fw_release_timer(&domain->timer);
 		}
 
-		spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
+		spin_lock_irqsave(&uncore->lock, irqflags);
 
-		for_each_fw_domain(domain, dev_priv, tmp) {
+		for_each_fw_domain(domain, uncore, tmp) {
 			if (hrtimer_active(&domain->timer))
 				active_domains |= domain->mask;
 		}
@@ -411,185 +403,134 @@ intel_uncore_forcewake_reset(struct drm_i915_private *dev_priv)
 			break;
 		}
 
-		spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
+		spin_unlock_irqrestore(&uncore->lock, irqflags);
 		cond_resched();
 	}
 
 	WARN_ON(active_domains);
 
-	fw = dev_priv->uncore.fw_domains_active;
+	fw = uncore->fw_domains_active;
 	if (fw)
-		dev_priv->uncore.funcs.force_wake_put(dev_priv, fw);
+		uncore->funcs.force_wake_put(uncore, fw);
 
-	fw_domains_reset(dev_priv, dev_priv->uncore.fw_domains);
-	assert_forcewakes_inactive(dev_priv);
+	fw_domains_reset(uncore, uncore->fw_domains);
+	assert_forcewakes_inactive(uncore);
 
-	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
+	spin_unlock_irqrestore(&uncore->lock, irqflags);
 
 	return fw; /* track the lost user forcewake domains */
 }
 
-static u64 gen9_edram_size(struct drm_i915_private *dev_priv)
-{
-	const unsigned int ways[8] = { 4, 8, 12, 16, 16, 16, 16, 16 };
-	const unsigned int sets[4] = { 1, 1, 2, 2 };
-	const u32 cap = dev_priv->edram_cap;
-
-	return EDRAM_NUM_BANKS(cap) *
-		ways[EDRAM_WAYS_IDX(cap)] *
-		sets[EDRAM_SETS_IDX(cap)] *
-		1024 * 1024;
-}
-
-u64 intel_uncore_edram_size(struct drm_i915_private *dev_priv)
-{
-	if (!HAS_EDRAM(dev_priv))
-		return 0;
-
-	/* The needed capability bits for size calculation
-	 * are not there with pre gen9 so return 128MB always.
-	 */
-	if (INTEL_GEN(dev_priv) < 9)
-		return 128 * 1024 * 1024;
-
-	return gen9_edram_size(dev_priv);
-}
-
-static void intel_uncore_edram_detect(struct drm_i915_private *dev_priv)
-{
-	if (IS_HASWELL(dev_priv) ||
-	    IS_BROADWELL(dev_priv) ||
-	    INTEL_GEN(dev_priv) >= 9) {
-		dev_priv->edram_cap = __raw_i915_read32(dev_priv,
-							HSW_EDRAM_CAP);
-
-		/* NB: We can't write IDICR yet because we do not have gt funcs
-		 * set up */
-	} else {
-		dev_priv->edram_cap = 0;
-	}
-
-	if (HAS_EDRAM(dev_priv))
-		DRM_INFO("Found %lluMB of eDRAM\n",
-			 intel_uncore_edram_size(dev_priv) / (1024 * 1024));
-}
-
 static bool
-fpga_check_for_unclaimed_mmio(struct drm_i915_private *dev_priv)
+fpga_check_for_unclaimed_mmio(struct intel_uncore *uncore)
 {
 	u32 dbg;
 
-	dbg = __raw_i915_read32(dev_priv, FPGA_DBG);
+	dbg = __raw_uncore_read32(uncore, FPGA_DBG);
 	if (likely(!(dbg & FPGA_DBG_RM_NOCLAIM)))
 		return false;
 
-	__raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
+	__raw_uncore_write32(uncore, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
 
 	return true;
 }
 
 static bool
-vlv_check_for_unclaimed_mmio(struct drm_i915_private *dev_priv)
+vlv_check_for_unclaimed_mmio(struct intel_uncore *uncore)
 {
 	u32 cer;
 
-	cer = __raw_i915_read32(dev_priv, CLAIM_ER);
+	cer = __raw_uncore_read32(uncore, CLAIM_ER);
 	if (likely(!(cer & (CLAIM_ER_OVERFLOW | CLAIM_ER_CTR_MASK))))
 		return false;
 
-	__raw_i915_write32(dev_priv, CLAIM_ER, CLAIM_ER_CLR);
+	__raw_uncore_write32(uncore, CLAIM_ER, CLAIM_ER_CLR);
 
 	return true;
 }
 
 static bool
-gen6_check_for_fifo_debug(struct drm_i915_private *dev_priv)
+gen6_check_for_fifo_debug(struct intel_uncore *uncore)
 {
 	u32 fifodbg;
 
-	fifodbg = __raw_i915_read32(dev_priv, GTFIFODBG);
+	fifodbg = __raw_uncore_read32(uncore, GTFIFODBG);
 
 	if (unlikely(fifodbg)) {
 		DRM_DEBUG_DRIVER("GTFIFODBG = 0x08%x\n", fifodbg);
-		__raw_i915_write32(dev_priv, GTFIFODBG, fifodbg);
+		__raw_uncore_write32(uncore, GTFIFODBG, fifodbg);
 	}
 
 	return fifodbg;
 }
 
 static bool
-check_for_unclaimed_mmio(struct drm_i915_private *dev_priv)
+check_for_unclaimed_mmio(struct intel_uncore *uncore)
 {
 	bool ret = false;
 
-	if (HAS_FPGA_DBG_UNCLAIMED(dev_priv))
-		ret |= fpga_check_for_unclaimed_mmio(dev_priv);
+	if (intel_uncore_has_fpga_dbg_unclaimed(uncore))
+		ret |= fpga_check_for_unclaimed_mmio(uncore);
 
-	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
-		ret |= vlv_check_for_unclaimed_mmio(dev_priv);
+	if (intel_uncore_has_dbg_unclaimed(uncore))
+		ret |= vlv_check_for_unclaimed_mmio(uncore);
 
-	if (IS_GEN_RANGE(dev_priv, 6, 7))
-		ret |= gen6_check_for_fifo_debug(dev_priv);
+	if (intel_uncore_has_fifo(uncore))
+		ret |= gen6_check_for_fifo_debug(uncore);
 
 	return ret;
 }
 
-static void __intel_uncore_early_sanitize(struct drm_i915_private *dev_priv,
+static void __intel_uncore_early_sanitize(struct intel_uncore *uncore,
 					  unsigned int restore_forcewake)
 {
 	/* clear out unclaimed reg detection bit */
-	if (check_for_unclaimed_mmio(dev_priv))
+	if (check_for_unclaimed_mmio(uncore))
 		DRM_DEBUG("unclaimed mmio detected on uncore init, clearing\n");
 
 	/* WaDisableShadowRegForCpd:chv */
-	if (IS_CHERRYVIEW(dev_priv)) {
-		__raw_i915_write32(dev_priv, GTFIFOCTL,
-				   __raw_i915_read32(dev_priv, GTFIFOCTL) |
-				   GT_FIFO_CTL_BLOCK_ALL_POLICY_STALL |
-				   GT_FIFO_CTL_RC6_POLICY_STALL);
+	if (IS_CHERRYVIEW(uncore_to_i915(uncore))) {
+		__raw_uncore_write32(uncore, GTFIFOCTL,
+				     __raw_uncore_read32(uncore, GTFIFOCTL) |
+				     GT_FIFO_CTL_BLOCK_ALL_POLICY_STALL |
+				     GT_FIFO_CTL_RC6_POLICY_STALL);
 	}
 
 	iosf_mbi_punit_acquire();
-	intel_uncore_forcewake_reset(dev_priv);
+	intel_uncore_forcewake_reset(uncore);
 	if (restore_forcewake) {
-		spin_lock_irq(&dev_priv->uncore.lock);
-		dev_priv->uncore.funcs.force_wake_get(dev_priv,
-						      restore_forcewake);
-
-		if (IS_GEN_RANGE(dev_priv, 6, 7))
-			dev_priv->uncore.fifo_count =
-				fifo_free_entries(dev_priv);
-		spin_unlock_irq(&dev_priv->uncore.lock);
+		spin_lock_irq(&uncore->lock);
+		uncore->funcs.force_wake_get(uncore, restore_forcewake);
+
+		if (intel_uncore_has_fifo(uncore))
+			uncore->fifo_count = fifo_free_entries(uncore);
+		spin_unlock_irq(&uncore->lock);
 	}
 	iosf_mbi_punit_release();
 }
 
-void intel_uncore_suspend(struct drm_i915_private *dev_priv)
+void intel_uncore_suspend(struct intel_uncore *uncore)
 {
 	iosf_mbi_punit_acquire();
 	iosf_mbi_unregister_pmic_bus_access_notifier_unlocked(
-		&dev_priv->uncore.pmic_bus_access_nb);
-	dev_priv->uncore.fw_domains_saved =
-		intel_uncore_forcewake_reset(dev_priv);
+		&uncore->pmic_bus_access_nb);
+	uncore->fw_domains_saved = intel_uncore_forcewake_reset(uncore);
 	iosf_mbi_punit_release();
 }
 
-void intel_uncore_resume_early(struct drm_i915_private *dev_priv)
+void intel_uncore_resume_early(struct intel_uncore *uncore)
 {
 	unsigned int restore_forcewake;
 
-	restore_forcewake = fetch_and_zero(&dev_priv->uncore.fw_domains_saved);
-	__intel_uncore_early_sanitize(dev_priv, restore_forcewake);
+	restore_forcewake = fetch_and_zero(&uncore->fw_domains_saved);
+	__intel_uncore_early_sanitize(uncore, restore_forcewake);
 
-	iosf_mbi_register_pmic_bus_access_notifier(
-		&dev_priv->uncore.pmic_bus_access_nb);
-	i915_check_and_clear_faults(dev_priv);
+	iosf_mbi_register_pmic_bus_access_notifier(&uncore->pmic_bus_access_nb);
 }
 
-void intel_uncore_runtime_resume(struct drm_i915_private *dev_priv)
+void intel_uncore_runtime_resume(struct intel_uncore *uncore)
 {
-	iosf_mbi_register_pmic_bus_access_notifier(
-		&dev_priv->uncore.pmic_bus_access_nb);
+	iosf_mbi_register_pmic_bus_access_notifier(&uncore->pmic_bus_access_nb);
 }
 
 void intel_uncore_sanitize(struct drm_i915_private *dev_priv)
@@ -598,15 +539,15 @@ void intel_uncore_sanitize(struct drm_i915_private *dev_priv)
 	intel_sanitize_gt_powersave(dev_priv);
 }
 
-static void __intel_uncore_forcewake_get(struct drm_i915_private *dev_priv,
+static void __intel_uncore_forcewake_get(struct intel_uncore *uncore,
 					 enum forcewake_domains fw_domains)
 {
 	struct intel_uncore_forcewake_domain *domain;
 	unsigned int tmp;
 
-	fw_domains &= dev_priv->uncore.fw_domains;
+	fw_domains &= uncore->fw_domains;
 
-	for_each_fw_domain_masked(domain, fw_domains, dev_priv, tmp) {
+	for_each_fw_domain_masked(domain, fw_domains, uncore, tmp) {
 		if (domain->wake_count++) {
 			fw_domains &= ~domain->mask;
 			domain->active = true;
@@ -614,12 +555,12 @@ static void __intel_uncore_forcewake_get(struct drm_i915_private *dev_priv,
 	}
 
 	if (fw_domains)
-		dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains);
+		uncore->funcs.force_wake_get(uncore, fw_domains);
 }
 
 /**
  * intel_uncore_forcewake_get - grab forcewake domain references
- * @dev_priv: i915 device instance
+ * @uncore: the intel_uncore structure
  * @fw_domains: forcewake domains to get reference on
  *
  * This function can be used get GT's forcewake domain references.
@@ -630,100 +571,100 @@ static void __intel_uncore_forcewake_get(struct drm_i915_private *dev_priv,
  * call to intel_unforce_forcewake_put(). Usually caller wants all the domains
  * to be kept awake so the @fw_domains would be then FORCEWAKE_ALL.
  */
-void intel_uncore_forcewake_get(struct drm_i915_private *dev_priv,
+void intel_uncore_forcewake_get(struct intel_uncore *uncore,
 				enum forcewake_domains fw_domains)
 {
 	unsigned long irqflags;
 
-	if (!dev_priv->uncore.funcs.force_wake_get)
+	if (!uncore->funcs.force_wake_get)
 		return;
 
-	assert_rpm_wakelock_held(dev_priv);
+	__assert_rpm_wakelock_held(uncore->rpm);
 
-	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
-	__intel_uncore_forcewake_get(dev_priv, fw_domains);
-	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
+	spin_lock_irqsave(&uncore->lock, irqflags);
+	__intel_uncore_forcewake_get(uncore, fw_domains);
+	spin_unlock_irqrestore(&uncore->lock, irqflags);
 }
 
 /**
  * intel_uncore_forcewake_user_get - claim forcewake on behalf of userspace
- * @dev_priv: i915 device instance
+ * @uncore: the intel_uncore structure
  *
  * This function is a wrapper around intel_uncore_forcewake_get() to acquire
  * the GT powerwell and in the process disable our debugging for the
  * duration of userspace's bypass.
  */
-void intel_uncore_forcewake_user_get(struct drm_i915_private *dev_priv)
+void intel_uncore_forcewake_user_get(struct intel_uncore *uncore)
 {
-	spin_lock_irq(&dev_priv->uncore.lock);
-	if (!dev_priv->uncore.user_forcewake.count++) {
-		intel_uncore_forcewake_get__locked(dev_priv, FORCEWAKE_ALL);
+	spin_lock_irq(&uncore->lock);
+	if (!uncore->user_forcewake.count++) {
+		intel_uncore_forcewake_get__locked(uncore, FORCEWAKE_ALL);
 
 		/* Save and disable mmio debugging for the user bypass */
-		dev_priv->uncore.user_forcewake.saved_mmio_check =
-			dev_priv->uncore.unclaimed_mmio_check;
-		dev_priv->uncore.user_forcewake.saved_mmio_debug =
+		uncore->user_forcewake.saved_mmio_check =
+			uncore->unclaimed_mmio_check;
+		uncore->user_forcewake.saved_mmio_debug =
 			i915_modparams.mmio_debug;
 
-		dev_priv->uncore.unclaimed_mmio_check = 0;
+		uncore->unclaimed_mmio_check = 0;
 		i915_modparams.mmio_debug = 0;
 	}
-	spin_unlock_irq(&dev_priv->uncore.lock);
+	spin_unlock_irq(&uncore->lock);
 }
 
 /**
  * intel_uncore_forcewake_user_put - release forcewake on behalf of userspace
- * @dev_priv: i915 device instance
+ * @uncore: the intel_uncore structure
  *
  * This function complements intel_uncore_forcewake_user_get() and releases
  * the GT powerwell taken on behalf of the userspace bypass.
  */
-void intel_uncore_forcewake_user_put(struct drm_i915_private *dev_priv)
+void intel_uncore_forcewake_user_put(struct intel_uncore *uncore)
 {
-	spin_lock_irq(&dev_priv->uncore.lock);
-	if (!--dev_priv->uncore.user_forcewake.count) {
-		if (intel_uncore_unclaimed_mmio(dev_priv))
-			dev_info(dev_priv->drm.dev,
+	spin_lock_irq(&uncore->lock);
+	if (!--uncore->user_forcewake.count) {
+		if (intel_uncore_unclaimed_mmio(uncore))
+			dev_info(uncore_to_i915(uncore)->drm.dev,
 				 "Invalid mmio detected during user access\n");
 
-		dev_priv->uncore.unclaimed_mmio_check =
-			dev_priv->uncore.user_forcewake.saved_mmio_check;
+		uncore->unclaimed_mmio_check =
+			uncore->user_forcewake.saved_mmio_check;
 		i915_modparams.mmio_debug =
-			dev_priv->uncore.user_forcewake.saved_mmio_debug;
+			uncore->user_forcewake.saved_mmio_debug;
 
-		intel_uncore_forcewake_put__locked(dev_priv, FORCEWAKE_ALL);
+		intel_uncore_forcewake_put__locked(uncore, FORCEWAKE_ALL);
 	}
-	spin_unlock_irq(&dev_priv->uncore.lock);
+	spin_unlock_irq(&uncore->lock);
 }
 
 /**
  * intel_uncore_forcewake_get__locked - grab forcewake domain references
- * @dev_priv: i915 device instance
+ * @uncore: the intel_uncore structure
  * @fw_domains: forcewake domains to get reference on
  *
  * See intel_uncore_forcewake_get(). This variant places the onus
  * on the caller to explicitly handle the dev_priv->uncore.lock spinlock.
  */
-void intel_uncore_forcewake_get__locked(struct drm_i915_private *dev_priv,
+void intel_uncore_forcewake_get__locked(struct intel_uncore *uncore,
 					enum forcewake_domains fw_domains)
 {
-	lockdep_assert_held(&dev_priv->uncore.lock);
+	lockdep_assert_held(&uncore->lock);
 
-	if (!dev_priv->uncore.funcs.force_wake_get)
+	if (!uncore->funcs.force_wake_get)
 		return;
 
-	__intel_uncore_forcewake_get(dev_priv, fw_domains);
+	__intel_uncore_forcewake_get(uncore, fw_domains);
 }
 
-static void __intel_uncore_forcewake_put(struct drm_i915_private *dev_priv,
+static void __intel_uncore_forcewake_put(struct intel_uncore *uncore,
 					 enum forcewake_domains fw_domains)
 {
 	struct intel_uncore_forcewake_domain *domain;
 	unsigned int tmp;
 
-	fw_domains &= dev_priv->uncore.fw_domains;
+	fw_domains &= uncore->fw_domains;
 
-	for_each_fw_domain_masked(domain, fw_domains, dev_priv, tmp) {
+	for_each_fw_domain_masked(domain, fw_domains, uncore, tmp) {
 		if (WARN_ON(domain->wake_count == 0))
 			continue;
 
@@ -738,66 +679,66 @@ static void __intel_uncore_forcewake_put(struct drm_i915_private *dev_priv,
 
 /**
  * intel_uncore_forcewake_put - release a forcewake domain reference
- * @dev_priv: i915 device instance
+ * @uncore: the intel_uncore structure
  * @fw_domains: forcewake domains to put references
  *
  * This function drops the device-level forcewakes for specified
  * domains obtained by intel_uncore_forcewake_get().
  */
-void intel_uncore_forcewake_put(struct drm_i915_private *dev_priv,
+void intel_uncore_forcewake_put(struct intel_uncore *uncore,
 				enum forcewake_domains fw_domains)
 {
 	unsigned long irqflags;
 
-	if (!dev_priv->uncore.funcs.force_wake_put)
+	if (!uncore->funcs.force_wake_put)
 		return;
 
-	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
-	__intel_uncore_forcewake_put(dev_priv, fw_domains);
-	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
+	spin_lock_irqsave(&uncore->lock, irqflags);
+	__intel_uncore_forcewake_put(uncore, fw_domains);
+	spin_unlock_irqrestore(&uncore->lock, irqflags);
 }
 
 /**
  * intel_uncore_forcewake_put__locked - grab forcewake domain references
- * @dev_priv: i915 device instance
+ * @uncore: the intel_uncore structure
  * @fw_domains: forcewake domains to get reference on
  *
  * See intel_uncore_forcewake_put(). This variant places the onus
  * on the caller to explicitly handle the dev_priv->uncore.lock spinlock.
  */
-void intel_uncore_forcewake_put__locked(struct drm_i915_private *dev_priv,
+void intel_uncore_forcewake_put__locked(struct intel_uncore *uncore,
 					enum forcewake_domains fw_domains)
 {
-	lockdep_assert_held(&dev_priv->uncore.lock);
+	lockdep_assert_held(&uncore->lock);
 
-	if (!dev_priv->uncore.funcs.force_wake_put)
+	if (!uncore->funcs.force_wake_put)
 		return;
 
-	__intel_uncore_forcewake_put(dev_priv, fw_domains);
+	__intel_uncore_forcewake_put(uncore, fw_domains);
 }
 
-void assert_forcewakes_inactive(struct drm_i915_private *dev_priv)
+void assert_forcewakes_inactive(struct intel_uncore *uncore)
 {
-	if (!dev_priv->uncore.funcs.force_wake_get)
+	if (!uncore->funcs.force_wake_get)
 		return;
 
-	WARN(dev_priv->uncore.fw_domains_active,
+	WARN(uncore->fw_domains_active,
 	     "Expected all fw_domains to be inactive, but %08x are still on\n",
-	     dev_priv->uncore.fw_domains_active);
+	     uncore->fw_domains_active);
 }
 
-void assert_forcewakes_active(struct drm_i915_private *dev_priv,
+void assert_forcewakes_active(struct intel_uncore *uncore,
 			      enum forcewake_domains fw_domains)
 {
-	if (!dev_priv->uncore.funcs.force_wake_get)
+	if (!uncore->funcs.force_wake_get)
 		return;
 
-	assert_rpm_wakelock_held(dev_priv);
+	__assert_rpm_wakelock_held(uncore->rpm);
 
-	fw_domains &= dev_priv->uncore.fw_domains;
-	WARN(fw_domains & ~dev_priv->uncore.fw_domains_active,
+	fw_domains &= uncore->fw_domains;
+	WARN(fw_domains & ~uncore->fw_domains_active,
 	     "Expected %08x fw_domains to be active, but %08x are off\n",
-	     fw_domains, fw_domains & ~dev_priv->uncore.fw_domains_active);
+	     fw_domains, fw_domains & ~uncore->fw_domains_active);
 }
 
 /* We give fast paths for the really cool registers */
@@ -806,7 +747,7 @@ void assert_forcewakes_active(struct drm_i915_private *dev_priv,
 #define GEN11_NEEDS_FORCE_WAKE(reg) \
 	((reg) < 0x40000 || ((reg) >= 0x1c0000 && (reg) < 0x1dc000))
 
-#define __gen6_reg_read_fw_domains(offset) \
+#define __gen6_reg_read_fw_domains(uncore, offset) \
 ({ \
 	enum forcewake_domains __fwd; \
 	if (NEEDS_FORCE_WAKE(offset)) \
@@ -846,13 +787,13 @@ static int fw_range_cmp(u32 offset, const struct intel_forcewake_range *entry)
 })
 
 static enum forcewake_domains
-find_fw_domain(struct drm_i915_private *dev_priv, u32 offset)
+find_fw_domain(struct intel_uncore *uncore, u32 offset)
 {
 	const struct intel_forcewake_range *entry;
 
 	entry = BSEARCH(offset,
-			dev_priv->uncore.fw_domains_table,
-			dev_priv->uncore.fw_domains_table_entries,
+			uncore->fw_domains_table,
+			uncore->fw_domains_table_entries,
 			fw_range_cmp);
 
 	if (!entry)
@@ -864,11 +805,11 @@ find_fw_domain(struct drm_i915_private *dev_priv, u32 offset)
 	 * translate it here to the list of available domains.
 	 */
 	if (entry->domains == FORCEWAKE_ALL)
-		return dev_priv->uncore.fw_domains;
+		return uncore->fw_domains;
 
-	WARN(entry->domains & ~dev_priv->uncore.fw_domains,
+	WARN(entry->domains & ~uncore->fw_domains,
 	     "Uninitialized forcewake domain(s) 0x%x accessed at 0x%x\n",
-	     entry->domains & ~dev_priv->uncore.fw_domains, offset);
+	     entry->domains & ~uncore->fw_domains, offset);
 
 	return entry->domains;
 }
@@ -892,19 +833,19 @@ static const struct intel_forcewake_range __vlv_fw_ranges[] = {
 	GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA),
 };
 
-#define __fwtable_reg_read_fw_domains(offset) \
+#define __fwtable_reg_read_fw_domains(uncore, offset) \
 ({ \
 	enum forcewake_domains __fwd = 0; \
 	if (NEEDS_FORCE_WAKE((offset))) \
-		__fwd = find_fw_domain(dev_priv, offset); \
+		__fwd = find_fw_domain(uncore, offset); \
 	__fwd; \
 })
 
-#define __gen11_fwtable_reg_read_fw_domains(offset) \
+#define __gen11_fwtable_reg_read_fw_domains(uncore, offset) \
 ({ \
 	enum forcewake_domains __fwd = 0; \
 	if (GEN11_NEEDS_FORCE_WAKE((offset))) \
-		__fwd = find_fw_domain(dev_priv, offset); \
+		__fwd = find_fw_domain(uncore, offset); \
 	__fwd; \
 })
 
@@ -956,7 +897,7 @@ static bool is_gen##x##_shadowed(u32 offset) \
 __is_genX_shadowed(8)
 __is_genX_shadowed(11)
 
-#define __gen8_reg_write_fw_domains(offset) \
+#define __gen8_reg_write_fw_domains(uncore, offset) \
 ({ \
 	enum forcewake_domains __fwd; \
 	if (NEEDS_FORCE_WAKE(offset) && !is_gen8_shadowed(offset)) \
@@ -986,19 +927,19 @@ static const struct intel_forcewake_range __chv_fw_ranges[] = {
 	GEN_FW_RANGE(0x30000, 0x37fff, FORCEWAKE_MEDIA),
 };
 
-#define __fwtable_reg_write_fw_domains(offset) \
+#define __fwtable_reg_write_fw_domains(uncore, offset) \
 ({ \
 	enum forcewake_domains __fwd = 0; \
 	if (NEEDS_FORCE_WAKE((offset)) && !is_gen8_shadowed(offset)) \
-		__fwd = find_fw_domain(dev_priv, offset); \
+		__fwd = find_fw_domain(uncore, offset); \
 	__fwd; \
 })
 
-#define __gen11_fwtable_reg_write_fw_domains(offset) \
+#define __gen11_fwtable_reg_write_fw_domains(uncore, offset) \
 ({ \
 	enum forcewake_domains __fwd = 0; \
 	if (GEN11_NEEDS_FORCE_WAKE((offset)) && !is_gen11_shadowed(offset)) \
-		__fwd = find_fw_domain(dev_priv, offset); \
+		__fwd = find_fw_domain(uncore, offset); \
 	__fwd; \
 })
 
@@ -1073,21 +1014,21 @@ static const struct intel_forcewake_range __gen11_fw_ranges[] = {
 };
 
 static void
-ilk_dummy_write(struct drm_i915_private *dev_priv)
+ilk_dummy_write(struct intel_uncore *uncore)
 {
 	/* WaIssueDummyWriteToWakeupFromRC6:ilk Issue a dummy write to wake up
 	 * the chip from rc6 before touching it for real. MI_MODE is masked,
 	 * hence harmless to write 0 into. */
-	__raw_i915_write32(dev_priv, MI_MODE, 0);
+	__raw_uncore_write32(uncore, MI_MODE, 0);
 }
 
 static void
-__unclaimed_reg_debug(struct drm_i915_private *dev_priv,
+__unclaimed_reg_debug(struct intel_uncore *uncore,
 		      const i915_reg_t reg,
 		      const bool read,
 		      const bool before)
 {
-	if (WARN(check_for_unclaimed_mmio(dev_priv) && !before,
+	if (WARN(check_for_unclaimed_mmio(uncore) && !before,
 		 "Unclaimed %s register 0x%x\n",
 		 read ? "read from" : "write to",
 		 i915_mmio_reg_offset(reg)))
@@ -1096,7 +1037,7 @@ __unclaimed_reg_debug(struct drm_i915_private *dev_priv,
 }
 
 static inline void
-unclaimed_reg_debug(struct drm_i915_private *dev_priv,
+unclaimed_reg_debug(struct intel_uncore *uncore,
 		    const i915_reg_t reg,
 		    const bool read,
 		    const bool before)
@@ -1104,12 +1045,12 @@ unclaimed_reg_debug(struct drm_i915_private *dev_priv,
 	if (likely(!i915_modparams.mmio_debug))
 		return;
 
-	__unclaimed_reg_debug(dev_priv, reg, read, before);
+	__unclaimed_reg_debug(uncore, reg, read, before);
 }
 
 #define GEN2_READ_HEADER(x) \
 	u##x val = 0; \
-	assert_rpm_wakelock_held(dev_priv);
+	__assert_rpm_wakelock_held(uncore->rpm);
 
 #define GEN2_READ_FOOTER \
 	trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
@@ -1117,18 +1058,18 @@ unclaimed_reg_debug(struct drm_i915_private *dev_priv,
 
 #define __gen2_read(x) \
 static u##x \
-gen2_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
+gen2_read##x(struct intel_uncore *uncore, i915_reg_t reg, bool trace) { \
 	GEN2_READ_HEADER(x); \
-	val = __raw_i915_read##x(dev_priv, reg); \
+	val = __raw_uncore_read##x(uncore, reg); \
 	GEN2_READ_FOOTER; \
 }
 
 #define __gen5_read(x) \
 static u##x \
-gen5_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
+gen5_read##x(struct intel_uncore *uncore, i915_reg_t reg, bool trace) { \
 	GEN2_READ_HEADER(x); \
-	ilk_dummy_write(dev_priv); \
-	val = __raw_i915_read##x(dev_priv, reg); \
+	ilk_dummy_write(uncore); \
+	val = __raw_uncore_read##x(uncore, reg); \
 	GEN2_READ_FOOTER; \
 }
 
@@ -1151,53 +1092,53 @@ __gen2_read(64)
 	u32 offset = i915_mmio_reg_offset(reg); \
 	unsigned long irqflags; \
 	u##x val = 0; \
-	assert_rpm_wakelock_held(dev_priv); \
-	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); \
-	unclaimed_reg_debug(dev_priv, reg, true, true)
+	__assert_rpm_wakelock_held(uncore->rpm); \
+	spin_lock_irqsave(&uncore->lock, irqflags); \
+	unclaimed_reg_debug(uncore, reg, true, true)
 
 #define GEN6_READ_FOOTER \
-	unclaimed_reg_debug(dev_priv, reg, true, false); \
-	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \
+	unclaimed_reg_debug(uncore, reg, true, false); \
+	spin_unlock_irqrestore(&uncore->lock, irqflags); \
 	trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
 	return val
 
-static noinline void ___force_wake_auto(struct drm_i915_private *dev_priv,
+static noinline void ___force_wake_auto(struct intel_uncore *uncore,
 					enum forcewake_domains fw_domains)
 {
 	struct intel_uncore_forcewake_domain *domain;
 	unsigned int tmp;
 
-	GEM_BUG_ON(fw_domains & ~dev_priv->uncore.fw_domains);
+	GEM_BUG_ON(fw_domains & ~uncore->fw_domains);
 
-	for_each_fw_domain_masked(domain, fw_domains, dev_priv, tmp)
+	for_each_fw_domain_masked(domain, fw_domains, uncore, tmp)
 		fw_domain_arm_timer(domain);
 
-	dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains);
+	uncore->funcs.force_wake_get(uncore, fw_domains);
 }
 
-static inline void __force_wake_auto(struct drm_i915_private *dev_priv,
+static inline void __force_wake_auto(struct intel_uncore *uncore,
 				     enum forcewake_domains fw_domains)
 {
 	if (WARN_ON(!fw_domains))
 		return;
 
 	/* Turn on all requested but inactive supported forcewake domains. */
-	fw_domains &= dev_priv->uncore.fw_domains;
-	fw_domains &= ~dev_priv->uncore.fw_domains_active;
+	fw_domains &= uncore->fw_domains;
+	fw_domains &= ~uncore->fw_domains_active;
 
 	if (fw_domains)
-		___force_wake_auto(dev_priv, fw_domains);
+		___force_wake_auto(uncore, fw_domains);
 }
 
 #define __gen_read(func, x) \
 static u##x \
-func##_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
+func##_read##x(struct intel_uncore *uncore, i915_reg_t reg, bool trace) { \
 	enum forcewake_domains fw_engine; \
 	GEN6_READ_HEADER(x); \
-	fw_engine = __##func##_reg_read_fw_domains(offset); \
+	fw_engine = __##func##_reg_read_fw_domains(uncore, offset); \
 	if (fw_engine) \
-		__force_wake_auto(dev_priv, fw_engine); \
-	val = __raw_i915_read##x(dev_priv, reg); \
+		__force_wake_auto(uncore, fw_engine); \
+	val = __raw_uncore_read##x(uncore, reg); \
 	GEN6_READ_FOOTER; \
 }
 #define __gen6_read(x) __gen_read(gen6, x)
@@ -1225,24 +1166,24 @@ __gen6_read(64)
 
 #define GEN2_WRITE_HEADER \
 	trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
-	assert_rpm_wakelock_held(dev_priv); \
+	__assert_rpm_wakelock_held(uncore->rpm); \
 
 #define GEN2_WRITE_FOOTER
 
 #define __gen2_write(x) \
 static void \
-gen2_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
+gen2_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { \
 	GEN2_WRITE_HEADER; \
-	__raw_i915_write##x(dev_priv, reg, val); \
+	__raw_uncore_write##x(uncore, reg, val); \
 	GEN2_WRITE_FOOTER; \
 }
 
 #define __gen5_write(x) \
 static void \
-gen5_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
+gen5_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { \
 	GEN2_WRITE_HEADER; \
-	ilk_dummy_write(dev_priv); \
-	__raw_i915_write##x(dev_priv, reg, val); \
+	ilk_dummy_write(uncore); \
+	__raw_uncore_write##x(uncore, reg, val); \
 	GEN2_WRITE_FOOTER; \
 }
 
@@ -1263,33 +1204,33 @@ __gen2_write(32)
 	u32 offset = i915_mmio_reg_offset(reg); \
 	unsigned long irqflags; \
 	trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
-	assert_rpm_wakelock_held(dev_priv); \
-	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); \
-	unclaimed_reg_debug(dev_priv, reg, false, true)
+	__assert_rpm_wakelock_held(uncore->rpm); \
+	spin_lock_irqsave(&uncore->lock, irqflags); \
+	unclaimed_reg_debug(uncore, reg, false, true)
 
 #define GEN6_WRITE_FOOTER \
-	unclaimed_reg_debug(dev_priv, reg, false, false); \
-	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags)
+	unclaimed_reg_debug(uncore, reg, false, false); \
+	spin_unlock_irqrestore(&uncore->lock, irqflags)
 
 #define __gen6_write(x) \
 static void \
-gen6_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
+gen6_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { \
 	GEN6_WRITE_HEADER; \
 	if (NEEDS_FORCE_WAKE(offset)) \
-		__gen6_gt_wait_for_fifo(dev_priv); \
-	__raw_i915_write##x(dev_priv, reg, val); \
+		__gen6_gt_wait_for_fifo(uncore); \
+	__raw_uncore_write##x(uncore, reg, val); \
 	GEN6_WRITE_FOOTER; \
 }
 
 #define __gen_write(func, x) \
 static void \
-func##_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
+func##_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { \
 	enum forcewake_domains fw_engine; \
 	GEN6_WRITE_HEADER; \
-	fw_engine = __##func##_reg_write_fw_domains(offset); \
+	fw_engine = __##func##_reg_write_fw_domains(uncore, offset); \
 	if (fw_engine) \
-		__force_wake_auto(dev_priv, fw_engine); \
-	__raw_i915_write##x(dev_priv, reg, val); \
+		__force_wake_auto(uncore, fw_engine); \
+	__raw_uncore_write##x(uncore, reg, val); \
 	GEN6_WRITE_FOOTER; \
 }
 #define __gen8_write(x) __gen_write(gen8, x)
@@ -1316,23 +1257,23 @@ __gen6_write(32)
 #undef GEN6_WRITE_FOOTER
 #undef GEN6_WRITE_HEADER
 
-#define ASSIGN_WRITE_MMIO_VFUNCS(i915, x) \
+#define ASSIGN_WRITE_MMIO_VFUNCS(uncore, x) \
 do { \
-	(i915)->uncore.funcs.mmio_writeb = x##_write8; \
-	(i915)->uncore.funcs.mmio_writew = x##_write16; \
-	(i915)->uncore.funcs.mmio_writel = x##_write32; \
+	(uncore)->funcs.mmio_writeb = x##_write8; \
+	(uncore)->funcs.mmio_writew = x##_write16; \
+	(uncore)->funcs.mmio_writel = x##_write32; \
 } while (0)
 
-#define ASSIGN_READ_MMIO_VFUNCS(i915, x) \
+#define ASSIGN_READ_MMIO_VFUNCS(uncore, x) \
 do { \
-	(i915)->uncore.funcs.mmio_readb = x##_read8; \
-	(i915)->uncore.funcs.mmio_readw = x##_read16; \
-	(i915)->uncore.funcs.mmio_readl = x##_read32; \
-	(i915)->uncore.funcs.mmio_readq = x##_read64; \
+	(uncore)->funcs.mmio_readb = x##_read8; \
+	(uncore)->funcs.mmio_readw = x##_read16; \
+	(uncore)->funcs.mmio_readl = x##_read32; \
+	(uncore)->funcs.mmio_readq = x##_read64; \
 } while (0)
 
 
-static void fw_domain_init(struct drm_i915_private *dev_priv,
+static void fw_domain_init(struct intel_uncore *uncore,
 			   enum forcewake_domain_id domain_id,
 			   i915_reg_t reg_set,
 			   i915_reg_t reg_ack)
@@ -1342,7 +1283,7 @@ static void fw_domain_init(struct drm_i915_private *dev_priv,
 	if (WARN_ON(domain_id >= FW_DOMAIN_ID_COUNT))
 		return;
 
-	d = &dev_priv->uncore.fw_domain[domain_id];
+	d = &uncore->fw_domain[domain_id];
 
 	WARN_ON(d->wake_count);
 
@@ -1350,8 +1291,8 @@ static void fw_domain_init(struct drm_i915_private *dev_priv,
 	WARN_ON(!i915_mmio_reg_valid(reg_ack));
 
 	d->wake_count = 0;
-	d->reg_set = reg_set;
-	d->reg_ack = reg_ack;
+	d->reg_set = uncore->regs + i915_mmio_reg_offset(reg_set);
+	d->reg_ack = uncore->regs + i915_mmio_reg_offset(reg_ack);
 
 	d->id = domain_id;
 
@@ -1371,12 +1312,12 @@ static void fw_domain_init(struct drm_i915_private *dev_priv,
 	hrtimer_init(&d->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 	d->timer.function = intel_uncore_fw_release_timer;
 
-	dev_priv->uncore.fw_domains |= BIT(domain_id);
+	uncore->fw_domains |= BIT(domain_id);
 
-	fw_domain_reset(dev_priv, d);
+	fw_domain_reset(d);
 }
 
-static void fw_domain_fini(struct drm_i915_private *dev_priv,
+static void fw_domain_fini(struct intel_uncore *uncore,
 			   enum forcewake_domain_id domain_id)
 {
 	struct intel_uncore_forcewake_domain *d;
@@ -1384,85 +1325,76 @@ static void fw_domain_fini(struct drm_i915_private *dev_priv,
 	if (WARN_ON(domain_id >= FW_DOMAIN_ID_COUNT))
 		return;
 
-	d = &dev_priv->uncore.fw_domain[domain_id];
+	d = &uncore->fw_domain[domain_id];
 
 	WARN_ON(d->wake_count);
 	WARN_ON(hrtimer_cancel(&d->timer));
 	memset(d, 0, sizeof(*d));
 
-	dev_priv->uncore.fw_domains &= ~BIT(domain_id);
+	uncore->fw_domains &= ~BIT(domain_id);
 }
 
-static void intel_uncore_fw_domains_init(struct drm_i915_private *dev_priv)
+static void intel_uncore_fw_domains_init(struct intel_uncore *uncore)
 {
-	if (INTEL_GEN(dev_priv) <= 5 || intel_vgpu_active(dev_priv))
-		return;
+	struct drm_i915_private *i915 = uncore_to_i915(uncore);
 
-	if (IS_GEN(dev_priv, 6)) {
-		dev_priv->uncore.fw_reset = 0;
-		dev_priv->uncore.fw_set = FORCEWAKE_KERNEL;
-		dev_priv->uncore.fw_clear = 0;
-	} else {
-		/* WaRsClearFWBitsAtReset:bdw,skl */
-		dev_priv->uncore.fw_reset = _MASKED_BIT_DISABLE(0xffff);
-		dev_priv->uncore.fw_set = _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL);
-		dev_priv->uncore.fw_clear = _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL);
-	}
+	if (!intel_uncore_has_forcewake(uncore))
+		return;
 
-	if (INTEL_GEN(dev_priv) >= 11) {
+	if (INTEL_GEN(i915) >= 11) {
 		int i;
 
-		dev_priv->uncore.funcs.force_wake_get =
+		uncore->funcs.force_wake_get =
 			fw_domains_get_with_fallback;
-		dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
-		fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
+		uncore->funcs.force_wake_put = fw_domains_put;
+		fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
 			       FORCEWAKE_RENDER_GEN9,
 			       FORCEWAKE_ACK_RENDER_GEN9);
-		fw_domain_init(dev_priv, FW_DOMAIN_ID_BLITTER,
+		fw_domain_init(uncore, FW_DOMAIN_ID_BLITTER,
 			       FORCEWAKE_BLITTER_GEN9,
 			       FORCEWAKE_ACK_BLITTER_GEN9);
 		for (i = 0; i < I915_MAX_VCS; i++) {
-			if (!HAS_ENGINE(dev_priv, _VCS(i)))
+			if (!HAS_ENGINE(i915, _VCS(i)))
 				continue;
 
-			fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA_VDBOX0 + i,
+			fw_domain_init(uncore, FW_DOMAIN_ID_MEDIA_VDBOX0 + i,
 				       FORCEWAKE_MEDIA_VDBOX_GEN11(i),
 				       FORCEWAKE_ACK_MEDIA_VDBOX_GEN11(i));
 		}
 		for (i = 0; i < I915_MAX_VECS; i++) {
-			if (!HAS_ENGINE(dev_priv, _VECS(i)))
+			if (!HAS_ENGINE(i915, _VECS(i)))
 				continue;
 
-			fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA_VEBOX0 + i,
+			fw_domain_init(uncore, FW_DOMAIN_ID_MEDIA_VEBOX0 + i,
 				       FORCEWAKE_MEDIA_VEBOX_GEN11(i),
 				       FORCEWAKE_ACK_MEDIA_VEBOX_GEN11(i));
 		}
-	} else if (IS_GEN_RANGE(dev_priv, 9, 10)) {
-		dev_priv->uncore.funcs.force_wake_get =
+	} else if (IS_GEN_RANGE(i915, 9, 10)) {
+		uncore->funcs.force_wake_get =
 			fw_domains_get_with_fallback;
-		dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
-		fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
+		uncore->funcs.force_wake_put = fw_domains_put;
+		fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
 			       FORCEWAKE_RENDER_GEN9,
 			       FORCEWAKE_ACK_RENDER_GEN9);
-		fw_domain_init(dev_priv, FW_DOMAIN_ID_BLITTER,
+		fw_domain_init(uncore, FW_DOMAIN_ID_BLITTER,
 			       FORCEWAKE_BLITTER_GEN9,
 			       FORCEWAKE_ACK_BLITTER_GEN9);
-		fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA,
+		fw_domain_init(uncore, FW_DOMAIN_ID_MEDIA,
 			       FORCEWAKE_MEDIA_GEN9, FORCEWAKE_ACK_MEDIA_GEN9);
-	} else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
-		dev_priv->uncore.funcs.force_wake_get = fw_domains_get;
-		dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
-		fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
+	} else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) {
+		uncore->funcs.force_wake_get = fw_domains_get;
+		uncore->funcs.force_wake_put = fw_domains_put;
+		fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
 			       FORCEWAKE_VLV, FORCEWAKE_ACK_VLV);
-		fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA,
+		fw_domain_init(uncore, FW_DOMAIN_ID_MEDIA,
 			       FORCEWAKE_MEDIA_VLV, FORCEWAKE_ACK_MEDIA_VLV);
-	} else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
-		dev_priv->uncore.funcs.force_wake_get =
+	} else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
+		uncore->funcs.force_wake_get =
 			fw_domains_get_with_thread_status;
-		dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
-		fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
+		uncore->funcs.force_wake_put = fw_domains_put;
+		fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
 			       FORCEWAKE_MT, FORCEWAKE_ACK_HSW);
-	} else if (IS_IVYBRIDGE(dev_priv)) {
+	} else if (IS_IVYBRIDGE(i915)) {
 		u32 ecobus;
 
 		/* IVB configs may use multi-threaded forcewake */
@@ -1474,9 +1406,9 @@ static void intel_uncore_fw_domains_init(struct drm_i915_private *dev_priv)
 		 * (correctly) interpreted by the test below as MT
 		 * forcewake being disabled.
 		 */
-		dev_priv->uncore.funcs.force_wake_get =
+		uncore->funcs.force_wake_get =
 			fw_domains_get_with_thread_status;
-		dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
+		uncore->funcs.force_wake_put = fw_domains_put;
 
 		/* We need to init first for ECOBUS access and then
 		 * determine later if we want to reinit, in case of MT access is
@@ -1485,41 +1417,41 @@ static void intel_uncore_fw_domains_init(struct drm_i915_private *dev_priv)
 		 * before the ecobus check.
 		 */
 
-		__raw_i915_write32(dev_priv, FORCEWAKE, 0);
-		__raw_posting_read(dev_priv, ECOBUS);
+		__raw_uncore_write32(uncore, FORCEWAKE, 0);
+		__raw_posting_read(uncore, ECOBUS);
 
-		fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
+		fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
 			       FORCEWAKE_MT, FORCEWAKE_MT_ACK);
 
-		spin_lock_irq(&dev_priv->uncore.lock);
-		fw_domains_get_with_thread_status(dev_priv, FORCEWAKE_RENDER);
-		ecobus = __raw_i915_read32(dev_priv, ECOBUS);
-		fw_domains_put(dev_priv, FORCEWAKE_RENDER);
-		spin_unlock_irq(&dev_priv->uncore.lock);
+		spin_lock_irq(&uncore->lock);
+		fw_domains_get_with_thread_status(uncore, FORCEWAKE_RENDER);
+		ecobus = __raw_uncore_read32(uncore, ECOBUS);
+		fw_domains_put(uncore, FORCEWAKE_RENDER);
+		spin_unlock_irq(&uncore->lock);
 
 		if (!(ecobus & FORCEWAKE_MT_ENABLE)) {
 			DRM_INFO("No MT forcewake available on Ivybridge, this can result in issues\n");
 			DRM_INFO("when using vblank-synced partial screen updates.\n");
-			fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
+			fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
 				       FORCEWAKE, FORCEWAKE_ACK);
 		}
-	} else if (IS_GEN(dev_priv, 6)) {
-		dev_priv->uncore.funcs.force_wake_get =
+	} else if (IS_GEN(i915, 6)) {
+		uncore->funcs.force_wake_get =
 			fw_domains_get_with_thread_status;
-		dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
-		fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
+		uncore->funcs.force_wake_put = fw_domains_put;
+		fw_domain_init(uncore, FW_DOMAIN_ID_RENDER,
 			       FORCEWAKE, FORCEWAKE_ACK);
 	}
 
 	/* All future platforms are expected to require complex power gating */
-	WARN_ON(dev_priv->uncore.fw_domains == 0);
+	WARN_ON(uncore->fw_domains == 0);
 }
 
-#define ASSIGN_FW_DOMAINS_TABLE(d) \
+#define ASSIGN_FW_DOMAINS_TABLE(uncore, d) \
 { \
-	dev_priv->uncore.fw_domains_table = \
+	(uncore)->fw_domains_table = \
 			(struct intel_forcewake_range *)(d); \
-	dev_priv->uncore.fw_domains_table_entries = ARRAY_SIZE((d)); \
+	(uncore)->fw_domains_table_entries = ARRAY_SIZE((d)); \
 }
 
 static int i915_pmic_bus_access_notifier(struct notifier_block *nb,
@@ -1544,66 +1476,132 @@ static int i915_pmic_bus_access_notifier(struct notifier_block *nb,
 		 * the access.
 		 */
 		disable_rpm_wakeref_asserts(dev_priv);
-		intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
+		intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
 		enable_rpm_wakeref_asserts(dev_priv);
 		break;
 	case MBI_PMIC_BUS_ACCESS_END:
-		intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+		intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
 		break;
 	}
 
 	return NOTIFY_OK;
 }
 
-void intel_uncore_init(struct drm_i915_private *dev_priv)
+static int uncore_mmio_setup(struct intel_uncore *uncore)
 {
-	i915_check_vgpu(dev_priv);
+	struct drm_i915_private *i915 = uncore_to_i915(uncore);
+	struct pci_dev *pdev = i915->drm.pdev;
+	int mmio_bar;
+	int mmio_size;
 
-	intel_uncore_edram_detect(dev_priv);
-	intel_uncore_fw_domains_init(dev_priv);
-	__intel_uncore_early_sanitize(dev_priv, 0);
+	mmio_bar = IS_GEN(i915, 2) ? 1 : 0;
+	/*
+	 * Before gen4, the registers and the GTT are behind different BARs.
+	 * However, from gen4 onwards, the registers and the GTT are shared
+	 * in the same BAR, so we want to restrict this ioremap from
+	 * clobbering the GTT which we want ioremap_wc instead. Fortunately,
+	 * the register BAR remains the same size for all the earlier
+	 * generations up to Ironlake.
+	 */
+	if (INTEL_GEN(i915) < 5)
+		mmio_size = 512 * 1024;
+	else
+		mmio_size = 2 * 1024 * 1024;
+	uncore->regs = pci_iomap(pdev, mmio_bar, mmio_size);
+	if (uncore->regs == NULL) {
+		DRM_ERROR("failed to map registers\n");
+
+		return -EIO;
+	}
+
+	return 0;
+}
+
+static void uncore_mmio_cleanup(struct intel_uncore *uncore)
+{
+	struct drm_i915_private *i915 = uncore_to_i915(uncore);
+	struct pci_dev *pdev = i915->drm.pdev;
+
+	pci_iounmap(pdev, uncore->regs);
+}
+
+void intel_uncore_init_early(struct intel_uncore *uncore)
+{
+	spin_lock_init(&uncore->lock);
+}
+
+int intel_uncore_init_mmio(struct intel_uncore *uncore)
+{
+	struct drm_i915_private *i915 = uncore_to_i915(uncore);
+	int ret;
+
+	ret = uncore_mmio_setup(uncore);
+	if (ret)
+		return ret;
+
+	i915_check_vgpu(i915);
+
+	if (INTEL_GEN(i915) > 5 && !intel_vgpu_active(i915))
+		uncore->flags |= UNCORE_HAS_FORCEWAKE;
 
-	dev_priv->uncore.unclaimed_mmio_check = 1;
-	dev_priv->uncore.pmic_bus_access_nb.notifier_call =
+	intel_uncore_fw_domains_init(uncore);
+	__intel_uncore_early_sanitize(uncore, 0);
+
+	uncore->unclaimed_mmio_check = 1;
+	uncore->pmic_bus_access_nb.notifier_call =
 		i915_pmic_bus_access_notifier;
 
-	if (IS_GEN_RANGE(dev_priv, 2, 4) || intel_vgpu_active(dev_priv)) {
-		ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen2);
-		ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen2);
-	} else if (IS_GEN(dev_priv, 5)) {
-		ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen5);
-		ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen5);
-	} else if (IS_GEN_RANGE(dev_priv, 6, 7)) {
-		ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen6);
-
-		if (IS_VALLEYVIEW(dev_priv)) {
-			ASSIGN_FW_DOMAINS_TABLE(__vlv_fw_ranges);
-			ASSIGN_READ_MMIO_VFUNCS(dev_priv, fwtable);
+	uncore->rpm = &i915->runtime_pm;
+
+	if (!intel_uncore_has_forcewake(uncore)) {
+		if (IS_GEN(i915, 5)) {
+			ASSIGN_WRITE_MMIO_VFUNCS(uncore, gen5);
+			ASSIGN_READ_MMIO_VFUNCS(uncore, gen5);
+		} else {
+			ASSIGN_WRITE_MMIO_VFUNCS(uncore, gen2);
+			ASSIGN_READ_MMIO_VFUNCS(uncore, gen2);
+		}
+	} else if (IS_GEN_RANGE(i915, 6, 7)) {
+		ASSIGN_WRITE_MMIO_VFUNCS(uncore, gen6);
+
+		if (IS_VALLEYVIEW(i915)) {
+			ASSIGN_FW_DOMAINS_TABLE(uncore, __vlv_fw_ranges);
+			ASSIGN_READ_MMIO_VFUNCS(uncore, fwtable);
 		} else {
-			ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen6);
+			ASSIGN_READ_MMIO_VFUNCS(uncore, gen6);
 		}
-	} else if (IS_GEN(dev_priv, 8)) {
-		if (IS_CHERRYVIEW(dev_priv)) {
-			ASSIGN_FW_DOMAINS_TABLE(__chv_fw_ranges);
-			ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, fwtable);
-			ASSIGN_READ_MMIO_VFUNCS(dev_priv, fwtable);
+	} else if (IS_GEN(i915, 8)) {
+		if (IS_CHERRYVIEW(i915)) {
+			ASSIGN_FW_DOMAINS_TABLE(uncore, __chv_fw_ranges);
+			ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
+			ASSIGN_READ_MMIO_VFUNCS(uncore, fwtable);
 
 		} else {
-			ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen8);
-			ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen6);
+			ASSIGN_WRITE_MMIO_VFUNCS(uncore, gen8);
+			ASSIGN_READ_MMIO_VFUNCS(uncore, gen6);
 		}
-	} else if (IS_GEN_RANGE(dev_priv, 9, 10)) {
-		ASSIGN_FW_DOMAINS_TABLE(__gen9_fw_ranges);
-		ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, fwtable);
-		ASSIGN_READ_MMIO_VFUNCS(dev_priv, fwtable);
+	} else if (IS_GEN_RANGE(i915, 9, 10)) {
+		ASSIGN_FW_DOMAINS_TABLE(uncore, __gen9_fw_ranges);
+		ASSIGN_WRITE_MMIO_VFUNCS(uncore, fwtable);
+		ASSIGN_READ_MMIO_VFUNCS(uncore, fwtable);
 	} else {
-		ASSIGN_FW_DOMAINS_TABLE(__gen11_fw_ranges);
-		ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen11_fwtable);
-		ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen11_fwtable);
+		ASSIGN_FW_DOMAINS_TABLE(uncore, __gen11_fw_ranges);
+		ASSIGN_WRITE_MMIO_VFUNCS(uncore, gen11_fwtable);
+		ASSIGN_READ_MMIO_VFUNCS(uncore, gen11_fwtable);
 	}
 
-	iosf_mbi_register_pmic_bus_access_notifier(
-		&dev_priv->uncore.pmic_bus_access_nb);
+	if (HAS_FPGA_DBG_UNCLAIMED(i915))
+		uncore->flags |= UNCORE_HAS_FPGA_DBG_UNCLAIMED;
+
+	if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
+		uncore->flags |= UNCORE_HAS_DBG_UNCLAIMED;
+
+	if (IS_GEN_RANGE(i915, 6, 7))
+		uncore->flags |= UNCORE_HAS_FIFO;
+
+	iosf_mbi_register_pmic_bus_access_notifier(&uncore->pmic_bus_access_nb);
+
+	return 0;
 }
 
 /*
@@ -1611,45 +1609,48 @@ void intel_uncore_init(struct drm_i915_private *dev_priv)
  * the forcewake domains. Prune them, to make sure they only reference existing
  * engines.
  */
-void intel_uncore_prune(struct drm_i915_private *dev_priv)
+void intel_uncore_prune_mmio_domains(struct intel_uncore *uncore)
 {
-	if (INTEL_GEN(dev_priv) >= 11) {
-		enum forcewake_domains fw_domains = dev_priv->uncore.fw_domains;
+	struct drm_i915_private *i915 = uncore_to_i915(uncore);
+
+	if (INTEL_GEN(i915) >= 11) {
+		enum forcewake_domains fw_domains = uncore->fw_domains;
 		enum forcewake_domain_id domain_id;
 		int i;
 
 		for (i = 0; i < I915_MAX_VCS; i++) {
 			domain_id = FW_DOMAIN_ID_MEDIA_VDBOX0 + i;
 
-			if (HAS_ENGINE(dev_priv, _VCS(i)))
+			if (HAS_ENGINE(i915, _VCS(i)))
 				continue;
 
 			if (fw_domains & BIT(domain_id))
-				fw_domain_fini(dev_priv, domain_id);
+				fw_domain_fini(uncore, domain_id);
 		}
 
 		for (i = 0; i < I915_MAX_VECS; i++) {
 			domain_id = FW_DOMAIN_ID_MEDIA_VEBOX0 + i;
 
-			if (HAS_ENGINE(dev_priv, _VECS(i)))
+			if (HAS_ENGINE(i915, _VECS(i)))
 				continue;
 
 			if (fw_domains & BIT(domain_id))
-				fw_domain_fini(dev_priv, domain_id);
+				fw_domain_fini(uncore, domain_id);
 		}
 	}
 }
 
-void intel_uncore_fini(struct drm_i915_private *dev_priv)
+void intel_uncore_fini_mmio(struct intel_uncore *uncore)
 {
 	/* Paranoia: make sure we have disabled everything before we exit. */
-	intel_uncore_sanitize(dev_priv);
+	intel_uncore_sanitize(uncore_to_i915(uncore));
 
 	iosf_mbi_punit_acquire();
 	iosf_mbi_unregister_pmic_bus_access_notifier_unlocked(
-		&dev_priv->uncore.pmic_bus_access_nb);
-	intel_uncore_forcewake_reset(dev_priv);
+		&uncore->pmic_bus_access_nb);
+	intel_uncore_forcewake_reset(uncore);
 	iosf_mbi_punit_release();
+	uncore_mmio_cleanup(uncore);
 }
 
 static const struct reg_whitelist {
@@ -1717,7 +1718,7 @@ int i915_reg_read_ioctl(struct drm_device *dev,
 
 /**
  * __intel_wait_for_register_fw - wait until register matches expected state
- * @dev_priv: the i915 device
+ * @uncore: the struct intel_uncore
  * @reg: the register to read
  * @mask: mask to apply to register value
  * @value: expected value
@@ -1741,7 +1742,7 @@ int i915_reg_read_ioctl(struct drm_device *dev,
  *
  * Returns 0 if the register matches the desired condition, or -ETIMEOUT.
  */
-int __intel_wait_for_register_fw(struct drm_i915_private *dev_priv,
+int __intel_wait_for_register_fw(struct intel_uncore *uncore,
 				 i915_reg_t reg,
 				 u32 mask,
 				 u32 value,
@@ -1750,7 +1751,7 @@ int __intel_wait_for_register_fw(struct drm_i915_private *dev_priv,
 				 u32 *out_value)
 {
 	u32 uninitialized_var(reg_value);
-#define done (((reg_value = I915_READ_FW(reg)) & mask) == value)
+#define done (((reg_value = intel_uncore_read_fw(uncore, reg)) & mask) == value)
 	int ret;
 
 	/* Catch any overuse of this function */
@@ -1772,7 +1773,7 @@ int __intel_wait_for_register_fw(struct drm_i915_private *dev_priv,
 
 /**
  * __intel_wait_for_register - wait until register matches expected state
- * @dev_priv: the i915 device
+ * @uncore: the struct intel_uncore
  * @reg: the register to read
  * @mask: mask to apply to register value
  * @value: expected value
@@ -1789,33 +1790,34 @@ int __intel_wait_for_register_fw(struct drm_i915_private *dev_priv,
  *
  * Returns 0 if the register matches the desired condition, or -ETIMEOUT.
  */
-int __intel_wait_for_register(struct drm_i915_private *dev_priv,
-			    i915_reg_t reg,
-			    u32 mask,
-			    u32 value,
-			    unsigned int fast_timeout_us,
-			    unsigned int slow_timeout_ms,
-			    u32 *out_value)
+int __intel_wait_for_register(struct intel_uncore *uncore,
+			      i915_reg_t reg,
+			      u32 mask,
+			      u32 value,
+			      unsigned int fast_timeout_us,
+			      unsigned int slow_timeout_ms,
+			      u32 *out_value)
 {
 	unsigned fw =
-		intel_uncore_forcewake_for_reg(dev_priv, reg, FW_REG_READ);
+		intel_uncore_forcewake_for_reg(uncore, reg, FW_REG_READ);
 	u32 reg_value;
 	int ret;
 
 	might_sleep_if(slow_timeout_ms);
 
-	spin_lock_irq(&dev_priv->uncore.lock);
-	intel_uncore_forcewake_get__locked(dev_priv, fw);
+	spin_lock_irq(&uncore->lock);
+	intel_uncore_forcewake_get__locked(uncore, fw);
 
-	ret = __intel_wait_for_register_fw(dev_priv,
+	ret = __intel_wait_for_register_fw(uncore,
 					   reg, mask, value,
 					   fast_timeout_us, 0, &reg_value);
 
-	intel_uncore_forcewake_put__locked(dev_priv, fw);
-	spin_unlock_irq(&dev_priv->uncore.lock);
+	intel_uncore_forcewake_put__locked(uncore, fw);
+	spin_unlock_irq(&uncore->lock);
 
 	if (ret && slow_timeout_ms)
-		ret = __wait_for(reg_value = I915_READ_NOTRACE(reg),
+		ret = __wait_for(reg_value = intel_uncore_read_notrace(uncore,
+								       reg),
 				 (reg_value & mask) == value,
 				 slow_timeout_ms * 1000, 10, 1000);
 
@@ -1828,82 +1830,90 @@ int __intel_wait_for_register(struct drm_i915_private *dev_priv,
 	return ret;
 }
 
-bool intel_uncore_unclaimed_mmio(struct drm_i915_private *dev_priv)
+bool intel_uncore_unclaimed_mmio(struct intel_uncore *uncore)
 {
-	return check_for_unclaimed_mmio(dev_priv);
+	return check_for_unclaimed_mmio(uncore);
 }
 
 bool
-intel_uncore_arm_unclaimed_mmio_detection(struct drm_i915_private *dev_priv)
+intel_uncore_arm_unclaimed_mmio_detection(struct intel_uncore *uncore)
 {
 	bool ret = false;
 
-	spin_lock_irq(&dev_priv->uncore.lock);
+	spin_lock_irq(&uncore->lock);
 
-	if (unlikely(dev_priv->uncore.unclaimed_mmio_check <= 0))
+	if (unlikely(uncore->unclaimed_mmio_check <= 0))
 		goto out;
 
-	if (unlikely(intel_uncore_unclaimed_mmio(dev_priv))) {
+	if (unlikely(intel_uncore_unclaimed_mmio(uncore))) {
 		if (!i915_modparams.mmio_debug) {
 			DRM_DEBUG("Unclaimed register detected, "
 				  "enabling oneshot unclaimed register reporting. "
 				  "Please use i915.mmio_debug=N for more information.\n");
 			i915_modparams.mmio_debug++;
 		}
-		dev_priv->uncore.unclaimed_mmio_check--;
+		uncore->unclaimed_mmio_check--;
 		ret = true;
 	}
 
 out:
-	spin_unlock_irq(&dev_priv->uncore.lock);
+	spin_unlock_irq(&uncore->lock);
 
 	return ret;
 }
 
 static enum forcewake_domains
-intel_uncore_forcewake_for_read(struct drm_i915_private *dev_priv,
+intel_uncore_forcewake_for_read(struct intel_uncore *uncore,
 				i915_reg_t reg)
 {
+	struct drm_i915_private *i915 = uncore_to_i915(uncore);
 	u32 offset = i915_mmio_reg_offset(reg);
 	enum forcewake_domains fw_domains;
 
-	if (INTEL_GEN(dev_priv) >= 11) {
-		fw_domains = __gen11_fwtable_reg_read_fw_domains(offset);
-	} else if (HAS_FWTABLE(dev_priv)) {
-		fw_domains = __fwtable_reg_read_fw_domains(offset);
-	} else if (INTEL_GEN(dev_priv) >= 6) {
-		fw_domains = __gen6_reg_read_fw_domains(offset);
+	if (INTEL_GEN(i915) >= 11) {
+		fw_domains = __gen11_fwtable_reg_read_fw_domains(uncore, offset);
+	} else if (HAS_FWTABLE(i915)) {
+		fw_domains = __fwtable_reg_read_fw_domains(uncore, offset);
+	} else if (INTEL_GEN(i915) >= 6) {
+		fw_domains = __gen6_reg_read_fw_domains(uncore, offset);
 	} else {
-		WARN_ON(!IS_GEN_RANGE(dev_priv, 2, 5));
+		/* on devices with FW we expect to hit one of the above cases */
+		if (intel_uncore_has_forcewake(uncore))
+			MISSING_CASE(INTEL_GEN(i915));
+
 		fw_domains = 0;
 	}
 
-	WARN_ON(fw_domains & ~dev_priv->uncore.fw_domains);
+	WARN_ON(fw_domains & ~uncore->fw_domains);
 
 	return fw_domains;
 }
 
 static enum forcewake_domains
-intel_uncore_forcewake_for_write(struct drm_i915_private *dev_priv,
+intel_uncore_forcewake_for_write(struct intel_uncore *uncore,
 				 i915_reg_t reg)
 {
+	struct drm_i915_private *i915 = uncore_to_i915(uncore);
 	u32 offset = i915_mmio_reg_offset(reg);
 	enum forcewake_domains fw_domains;
 
-	if (INTEL_GEN(dev_priv) >= 11) {
-		fw_domains = __gen11_fwtable_reg_write_fw_domains(offset);
-	} else if (HAS_FWTABLE(dev_priv) && !IS_VALLEYVIEW(dev_priv)) {
-		fw_domains = __fwtable_reg_write_fw_domains(offset);
-	} else if (IS_GEN(dev_priv, 8)) {
-		fw_domains = __gen8_reg_write_fw_domains(offset);
-	} else if (IS_GEN_RANGE(dev_priv, 6, 7)) {
+	if (INTEL_GEN(i915) >= 11) {
+		fw_domains = __gen11_fwtable_reg_write_fw_domains(uncore, offset);
+	} else if (HAS_FWTABLE(i915) && !IS_VALLEYVIEW(i915)) {
+		fw_domains = __fwtable_reg_write_fw_domains(uncore, offset);
+	} else if (IS_GEN(i915, 8)) {
+		fw_domains = __gen8_reg_write_fw_domains(uncore, offset);
+	} else if (IS_GEN_RANGE(i915, 6, 7)) {
 		fw_domains = FORCEWAKE_RENDER;
 	} else {
-		WARN_ON(!IS_GEN_RANGE(dev_priv, 2, 5));
+		/* on devices with FW we expect to hit one of the above cases */
+		if (intel_uncore_has_forcewake(uncore))
+			MISSING_CASE(INTEL_GEN(i915));
+
 		fw_domains = 0;
 	}
 
-	WARN_ON(fw_domains & ~dev_priv->uncore.fw_domains);
+	WARN_ON(fw_domains & ~uncore->fw_domains);
 
 	return fw_domains;
 }
@@ -1911,7 +1921,7 @@ intel_uncore_forcewake_for_write(struct drm_i915_private *dev_priv,
 /**
  * intel_uncore_forcewake_for_reg - which forcewake domains are needed to access
  * 				    a register
- * @dev_priv: pointer to struct drm_i915_private
+ * @uncore: pointer to struct intel_uncore
  * @reg: register in question
  * @op: operation bitmask of FW_REG_READ and/or FW_REG_WRITE
  *
@@ -1923,21 +1933,21 @@ intel_uncore_forcewake_for_write(struct drm_i915_private *dev_priv,
  * callers to do FIFO management on their own or risk losing writes.
  */
 enum forcewake_domains
-intel_uncore_forcewake_for_reg(struct drm_i915_private *dev_priv,
+intel_uncore_forcewake_for_reg(struct intel_uncore *uncore,
 			       i915_reg_t reg, unsigned int op)
 {
 	enum forcewake_domains fw_domains = 0;
 
 	WARN_ON(!op);
 
-	if (intel_vgpu_active(dev_priv))
+	if (!intel_uncore_has_forcewake(uncore))
 		return 0;
 
 	if (op & FW_REG_READ)
-		fw_domains = intel_uncore_forcewake_for_read(dev_priv, reg);
+		fw_domains = intel_uncore_forcewake_for_read(uncore, reg);
 
 	if (op & FW_REG_WRITE)
-		fw_domains |= intel_uncore_forcewake_for_write(dev_priv, reg);
+		fw_domains |= intel_uncore_forcewake_for_write(uncore, reg);
 
 	return fw_domains;
 }
diff --git a/drivers/gpu/drm/i915/intel_uncore.h b/drivers/gpu/drm/i915/intel_uncore.h
index e5e157d288de..d6af3de70121 100644
--- a/drivers/gpu/drm/i915/intel_uncore.h
+++ b/drivers/gpu/drm/i915/intel_uncore.h
@@ -28,10 +28,13 @@
 #include <linux/spinlock.h>
 #include <linux/notifier.h>
 #include <linux/hrtimer.h>
+#include <linux/io-64-nonatomic-lo-hi.h>
 
 #include "i915_reg.h"
 
 struct drm_i915_private;
+struct i915_runtime_pm;
+struct intel_uncore;
 
 enum forcewake_domain_id {
 	FW_DOMAIN_ID_RENDER = 0,
@@ -62,25 +65,25 @@ enum forcewake_domains {
 };
 
 struct intel_uncore_funcs {
-	void (*force_wake_get)(struct drm_i915_private *dev_priv,
+	void (*force_wake_get)(struct intel_uncore *uncore,
 			       enum forcewake_domains domains);
-	void (*force_wake_put)(struct drm_i915_private *dev_priv,
+	void (*force_wake_put)(struct intel_uncore *uncore,
 			       enum forcewake_domains domains);
 
-	u8 (*mmio_readb)(struct drm_i915_private *dev_priv,
+	u8 (*mmio_readb)(struct intel_uncore *uncore,
 			 i915_reg_t r, bool trace);
-	u16 (*mmio_readw)(struct drm_i915_private *dev_priv,
+	u16 (*mmio_readw)(struct intel_uncore *uncore,
 			  i915_reg_t r, bool trace);
-	u32 (*mmio_readl)(struct drm_i915_private *dev_priv,
+	u32 (*mmio_readl)(struct intel_uncore *uncore,
 			  i915_reg_t r, bool trace);
-	u64 (*mmio_readq)(struct drm_i915_private *dev_priv,
+	u64 (*mmio_readq)(struct intel_uncore *uncore,
 			  i915_reg_t r, bool trace);
 
-	void (*mmio_writeb)(struct drm_i915_private *dev_priv,
+	void (*mmio_writeb)(struct intel_uncore *uncore,
 			    i915_reg_t r, u8 val, bool trace);
-	void (*mmio_writew)(struct drm_i915_private *dev_priv,
+	void (*mmio_writew)(struct intel_uncore *uncore,
 			    i915_reg_t r, u16 val, bool trace);
-	void (*mmio_writel)(struct drm_i915_private *dev_priv,
+	void (*mmio_writel)(struct intel_uncore *uncore,
 			    i915_reg_t r, u32 val, bool trace);
 };
 
@@ -92,8 +95,18 @@ struct intel_forcewake_range {
 };
 
 struct intel_uncore {
+	void __iomem *regs;
+
+	struct i915_runtime_pm *rpm;
+
 	spinlock_t lock; /** lock is also taken in irq contexts. */
 
+	unsigned int flags;
+#define UNCORE_HAS_FORCEWAKE		BIT(0)
+#define UNCORE_HAS_FPGA_DBG_UNCLAIMED	BIT(1)
+#define UNCORE_HAS_DBG_UNCLAIMED	BIT(2)
+#define UNCORE_HAS_FIFO			BIT(3)
+
 	const struct intel_forcewake_range *fw_domains_table;
 	unsigned int fw_domains_table_entries;
 
@@ -106,18 +119,14 @@ struct intel_uncore {
 	enum forcewake_domains fw_domains_active;
 	enum forcewake_domains fw_domains_saved; /* user domains saved for S3 */
 
-	u32 fw_set;
-	u32 fw_clear;
-	u32 fw_reset;
-
 	struct intel_uncore_forcewake_domain {
 		enum forcewake_domain_id id;
 		enum forcewake_domains mask;
 		unsigned int wake_count;
 		bool active;
 		struct hrtimer timer;
-		i915_reg_t reg_set;
-		i915_reg_t reg_ack;
+		u32 __iomem *reg_set;
+		u32 __iomem *reg_ack;
 	} fw_domain[FW_DOMAIN_ID_COUNT];
 
 	struct {
@@ -131,86 +140,257 @@ struct intel_uncore {
 };
 
 /* Iterate over initialised fw domains */
-#define for_each_fw_domain_masked(domain__, mask__, dev_priv__, tmp__) \
+#define for_each_fw_domain_masked(domain__, mask__, uncore__, tmp__) \
 	for (tmp__ = (mask__); \
-	     tmp__ ? (domain__ = &(dev_priv__)->uncore.fw_domain[__mask_next_bit(tmp__)]), 1 : 0;)
+	     tmp__ ? (domain__ = &(uncore__)->fw_domain[__mask_next_bit(tmp__)]), 1 : 0;)
+
+#define for_each_fw_domain(domain__, uncore__, tmp__) \
+	for_each_fw_domain_masked(domain__, (uncore__)->fw_domains, uncore__, tmp__)
+
+static inline struct intel_uncore *
+forcewake_domain_to_uncore(const struct intel_uncore_forcewake_domain *d)
+{
+	return container_of(d, struct intel_uncore, fw_domain[d->id]);
+}
 
-#define for_each_fw_domain(domain__, dev_priv__, tmp__) \
-	for_each_fw_domain_masked(domain__, (dev_priv__)->uncore.fw_domains, dev_priv__, tmp__)
+static inline bool
+intel_uncore_has_forcewake(const struct intel_uncore *uncore)
+{
+	return uncore->flags & UNCORE_HAS_FORCEWAKE;
+}
 
+static inline bool
+intel_uncore_has_fpga_dbg_unclaimed(const struct intel_uncore *uncore)
+{
+	return uncore->flags & UNCORE_HAS_FPGA_DBG_UNCLAIMED;
+}
+
+static inline bool
+intel_uncore_has_dbg_unclaimed(const struct intel_uncore *uncore)
+{
+	return uncore->flags & UNCORE_HAS_DBG_UNCLAIMED;
+}
+
+static inline bool
+intel_uncore_has_fifo(const struct intel_uncore *uncore)
+{
+	return uncore->flags & UNCORE_HAS_FIFO;
+}
 
 void intel_uncore_sanitize(struct drm_i915_private *dev_priv);
-void intel_uncore_init(struct drm_i915_private *dev_priv);
-void intel_uncore_prune(struct drm_i915_private *dev_priv);
-bool intel_uncore_unclaimed_mmio(struct drm_i915_private *dev_priv);
-bool intel_uncore_arm_unclaimed_mmio_detection(struct drm_i915_private *dev_priv);
-void intel_uncore_fini(struct drm_i915_private *dev_priv);
-void intel_uncore_suspend(struct drm_i915_private *dev_priv);
-void intel_uncore_resume_early(struct drm_i915_private *dev_priv);
-void intel_uncore_runtime_resume(struct drm_i915_private *dev_priv);
-
-u64 intel_uncore_edram_size(struct drm_i915_private *dev_priv);
-void assert_forcewakes_inactive(struct drm_i915_private *dev_priv);
-void assert_forcewakes_active(struct drm_i915_private *dev_priv,
+void intel_uncore_init_early(struct intel_uncore *uncore);
+int intel_uncore_init_mmio(struct intel_uncore *uncore);
+void intel_uncore_prune_mmio_domains(struct intel_uncore *uncore);
+bool intel_uncore_unclaimed_mmio(struct intel_uncore *uncore);
+bool intel_uncore_arm_unclaimed_mmio_detection(struct intel_uncore *uncore);
+void intel_uncore_fini_mmio(struct intel_uncore *uncore);
+void intel_uncore_suspend(struct intel_uncore *uncore);
+void intel_uncore_resume_early(struct intel_uncore *uncore);
+void intel_uncore_runtime_resume(struct intel_uncore *uncore);
+
+void assert_forcewakes_inactive(struct intel_uncore *uncore);
+void assert_forcewakes_active(struct intel_uncore *uncore,
 			      enum forcewake_domains fw_domains);
 const char *intel_uncore_forcewake_domain_to_str(const enum forcewake_domain_id id);
 
 enum forcewake_domains
-intel_uncore_forcewake_for_reg(struct drm_i915_private *dev_priv,
+intel_uncore_forcewake_for_reg(struct intel_uncore *uncore,
 			       i915_reg_t reg, unsigned int op);
 #define FW_REG_READ  (1)
 #define FW_REG_WRITE (2)
 
-void intel_uncore_forcewake_get(struct drm_i915_private *dev_priv,
+void intel_uncore_forcewake_get(struct intel_uncore *uncore,
 				enum forcewake_domains domains);
-void intel_uncore_forcewake_put(struct drm_i915_private *dev_priv,
+void intel_uncore_forcewake_put(struct intel_uncore *uncore,
 				enum forcewake_domains domains);
 /* Like above but the caller must manage the uncore.lock itself.
  * Must be used with I915_READ_FW and friends.
  */
-void intel_uncore_forcewake_get__locked(struct drm_i915_private *dev_priv,
+void intel_uncore_forcewake_get__locked(struct intel_uncore *uncore,
 					enum forcewake_domains domains);
-void intel_uncore_forcewake_put__locked(struct drm_i915_private *dev_priv,
+void intel_uncore_forcewake_put__locked(struct intel_uncore *uncore,
 					enum forcewake_domains domains);
 
-void intel_uncore_forcewake_user_get(struct drm_i915_private *dev_priv);
-void intel_uncore_forcewake_user_put(struct drm_i915_private *dev_priv);
+void intel_uncore_forcewake_user_get(struct intel_uncore *uncore);
+void intel_uncore_forcewake_user_put(struct intel_uncore *uncore);
 
-int __intel_wait_for_register(struct drm_i915_private *dev_priv,
+int __intel_wait_for_register(struct intel_uncore *uncore,
 			      i915_reg_t reg,
 			      u32 mask,
 			      u32 value,
 			      unsigned int fast_timeout_us,
 			      unsigned int slow_timeout_ms,
 			      u32 *out_value);
-static inline
-int intel_wait_for_register(struct drm_i915_private *dev_priv,
-			    i915_reg_t reg,
-			    u32 mask,
-			    u32 value,
-			    unsigned int timeout_ms)
+static inline int
+intel_wait_for_register(struct intel_uncore *uncore,
+			i915_reg_t reg,
+			u32 mask,
+			u32 value,
+			unsigned int timeout_ms)
 {
-	return __intel_wait_for_register(dev_priv, reg, mask, value, 2,
+	return __intel_wait_for_register(uncore, reg, mask, value, 2,
 					 timeout_ms, NULL);
 }
-int __intel_wait_for_register_fw(struct drm_i915_private *dev_priv,
+
+int __intel_wait_for_register_fw(struct intel_uncore *uncore,
 				 i915_reg_t reg,
 				 u32 mask,
 				 u32 value,
 				 unsigned int fast_timeout_us,
 				 unsigned int slow_timeout_ms,
 				 u32 *out_value);
-static inline
-int intel_wait_for_register_fw(struct drm_i915_private *dev_priv,
-			       i915_reg_t reg,
-			       u32 mask,
-			       u32 value,
+static inline int
+intel_wait_for_register_fw(struct intel_uncore *uncore,
+			   i915_reg_t reg,
+			   u32 mask,
+			   u32 value,
 			       unsigned int timeout_ms)
 {
-	return __intel_wait_for_register_fw(dev_priv, reg, mask, value,
+	return __intel_wait_for_register_fw(uncore, reg, mask, value,
 					    2, timeout_ms, NULL);
 }
 
+/* register access functions */
+#define __raw_read(x__, s__) \
+static inline u##x__ __raw_uncore_read##x__(const struct intel_uncore *uncore, \
+					    i915_reg_t reg) \
+{ \
+	return read##s__(uncore->regs + i915_mmio_reg_offset(reg)); \
+}
+
+#define __raw_write(x__, s__) \
+static inline void __raw_uncore_write##x__(const struct intel_uncore *uncore, \
+					   i915_reg_t reg, u##x__ val) \
+{ \
+	write##s__(val, uncore->regs + i915_mmio_reg_offset(reg)); \
+}
+__raw_read(8, b)
+__raw_read(16, w)
+__raw_read(32, l)
+__raw_read(64, q)
+
+__raw_write(8, b)
+__raw_write(16, w)
+__raw_write(32, l)
+__raw_write(64, q)
+
+#undef __raw_read
+#undef __raw_write
+
+#define __uncore_read(name__, x__, s__, trace__) \
+static inline u##x__ intel_uncore_##name__(struct intel_uncore *uncore, \
+					   i915_reg_t reg) \
+{ \
+	return uncore->funcs.mmio_read##s__(uncore, reg, (trace__)); \
+}
+
+#define __uncore_write(name__, x__, s__, trace__) \
+static inline void intel_uncore_##name__(struct intel_uncore *uncore, \
+					 i915_reg_t reg, u##x__ val) \
+{ \
+	uncore->funcs.mmio_write##s__(uncore, reg, val, (trace__)); \
+}
+
+__uncore_read(read8, 8, b, true)
+__uncore_read(read16, 16, w, true)
+__uncore_read(read, 32, l, true)
+__uncore_read(read16_notrace, 16, w, false)
+__uncore_read(read_notrace, 32, l, false)
+
+__uncore_write(write8, 8, b, true)
+__uncore_write(write16, 16, w, true)
+__uncore_write(write, 32, l, true)
+__uncore_write(write_notrace, 32, l, false)
+
+/* Be very careful with read/write 64-bit values. On 32-bit machines, they
+ * will be implemented using 2 32-bit writes in an arbitrary order with
+ * an arbitrary delay between them. This can cause the hardware to
+ * act upon the intermediate value, possibly leading to corruption and
+ * machine death. For this reason we do not support I915_WRITE64, or
+ * uncore->funcs.mmio_writeq.
+ *
+ * When reading a 64-bit value as two 32-bit values, the delay may cause
+ * the two reads to mismatch, e.g. a timestamp overflowing. Also note that
+ * occasionally a 64-bit register does not actually support a full readq
+ * and must be read using two 32-bit reads.
+ *
+ * You have been warned.
+ */
+__uncore_read(read64, 64, q, true)
+
+static inline u64
+intel_uncore_read64_2x32(struct intel_uncore *uncore,
+			 i915_reg_t lower_reg, i915_reg_t upper_reg)
+{
+	u32 upper, lower, old_upper, loop = 0;
+	upper = intel_uncore_read(uncore, upper_reg);
+	do {
+		old_upper = upper;
+		lower = intel_uncore_read(uncore, lower_reg);
+		upper = intel_uncore_read(uncore, upper_reg);
+	} while (upper != old_upper && loop++ < 2);
+	return (u64)upper << 32 | lower;
+}
+
+#define intel_uncore_posting_read(...) ((void)intel_uncore_read_notrace(__VA_ARGS__))
+#define intel_uncore_posting_read16(...) ((void)intel_uncore_read16_notrace(__VA_ARGS__))
+
+#undef __uncore_read
+#undef __uncore_write
+
+/* These are untraced mmio-accessors that are only valid to be used inside
+ * critical sections, such as inside IRQ handlers, where forcewake is explicitly
+ * controlled.
+ *
+ * Think twice, and think again, before using these.
+ *
+ * As an example, these accessors can possibly be used between:
+ *
+ * spin_lock_irq(&uncore->lock);
+ * intel_uncore_forcewake_get__locked();
+ *
+ * and
+ *
+ * intel_uncore_forcewake_put__locked();
+ * spin_unlock_irq(&uncore->lock);
+ *
+ *
+ * Note: some registers may not need forcewake held, so
+ * intel_uncore_forcewake_{get,put} can be omitted, see
+ * intel_uncore_forcewake_for_reg().
+ *
+ * Certain architectures will die if the same cacheline is concurrently accessed
+ * by different clients (e.g. on Ivybridge). Access to registers should
+ * therefore generally be serialised, by either the dev_priv->uncore.lock or
+ * a more localised lock guarding all access to that bank of registers.
+ */
+#define intel_uncore_read_fw(...) __raw_uncore_read32(__VA_ARGS__)
+#define intel_uncore_write_fw(...) __raw_uncore_write32(__VA_ARGS__)
+#define intel_uncore_write64_fw(...) __raw_uncore_write64(__VA_ARGS__)
+#define intel_uncore_posting_read_fw(...) ((void)intel_uncore_read_fw(__VA_ARGS__))
+
+static inline void intel_uncore_rmw(struct intel_uncore *uncore,
+				    i915_reg_t reg, u32 clear, u32 set)
+{
+	u32 val;
+
+	val = intel_uncore_read(uncore, reg);
+	val &= ~clear;
+	val |= set;
+	intel_uncore_write(uncore, reg, val);
+}
+
+static inline void intel_uncore_rmw_fw(struct intel_uncore *uncore,
+				       i915_reg_t reg, u32 clear, u32 set)
+{
+	u32 val;
+
+	val = intel_uncore_read_fw(uncore, reg);
+	val &= ~clear;
+	val |= set;
+	intel_uncore_write_fw(uncore, reg, val);
+}
+
 #define raw_reg_read(base, reg) \
 	readl(base + i915_mmio_reg_offset(reg))
 #define raw_reg_write(base, reg, value) \
diff --git a/drivers/gpu/drm/i915/intel_vbt_defs.h b/drivers/gpu/drm/i915/intel_vbt_defs.h
index bf3662ad5fed..fdbbb9a53804 100644
--- a/drivers/gpu/drm/i915/intel_vbt_defs.h
+++ b/drivers/gpu/drm/i915/intel_vbt_defs.h
@@ -772,6 +772,9 @@ struct psr_table {
 	/* TP wake up time in multiple of 100 */
 	u16 tp1_wakeup_time;
 	u16 tp2_tp3_wakeup_time;
+
+	/* PSR2 TP2/TP3 wakeup time for 16 panels */
+	u32 psr2_tp2_tp3_wakeup_time;
 } __packed;
 
 struct bdb_psr {
diff --git a/drivers/gpu/drm/i915/intel_vdsc.c b/drivers/gpu/drm/i915/intel_vdsc.c
index 23abf03736e7..3f9921ba4a76 100644
--- a/drivers/gpu/drm/i915/intel_vdsc.c
+++ b/drivers/gpu/drm/i915/intel_vdsc.c
@@ -317,129 +317,6 @@ static int get_column_index_for_rc_params(u8 bits_per_component)
 	}
 }
 
-static int intel_compute_rc_parameters(struct drm_dsc_config *vdsc_cfg)
-{
-	unsigned long groups_per_line = 0;
-	unsigned long groups_total = 0;
-	unsigned long num_extra_mux_bits = 0;
-	unsigned long slice_bits = 0;
-	unsigned long hrd_delay = 0;
-	unsigned long final_scale = 0;
-	unsigned long rbs_min = 0;
-
-	/* Number of groups used to code each line of a slice */
-	groups_per_line = DIV_ROUND_UP(vdsc_cfg->slice_width,
-				       DSC_RC_PIXELS_PER_GROUP);
-
-	/* chunksize in Bytes */
-	vdsc_cfg->slice_chunk_size = DIV_ROUND_UP(vdsc_cfg->slice_width *
-						  vdsc_cfg->bits_per_pixel,
-						  (8 * 16));
-
-	if (vdsc_cfg->convert_rgb)
-		num_extra_mux_bits = 3 * (vdsc_cfg->mux_word_size +
-					  (4 * vdsc_cfg->bits_per_component + 4)
-					  - 2);
-	else
-		num_extra_mux_bits = 3 * vdsc_cfg->mux_word_size +
-			(4 * vdsc_cfg->bits_per_component + 4) +
-			2 * (4 * vdsc_cfg->bits_per_component) - 2;
-	/* Number of bits in one Slice */
-	slice_bits = 8 * vdsc_cfg->slice_chunk_size * vdsc_cfg->slice_height;
-
-	while ((num_extra_mux_bits > 0) &&
-	       ((slice_bits - num_extra_mux_bits) % vdsc_cfg->mux_word_size))
-		num_extra_mux_bits--;
-
-	if (groups_per_line < vdsc_cfg->initial_scale_value - 8)
-		vdsc_cfg->initial_scale_value = groups_per_line + 8;
-
-	/* scale_decrement_interval calculation according to DSC spec 1.11 */
-	if (vdsc_cfg->initial_scale_value > 8)
-		vdsc_cfg->scale_decrement_interval = groups_per_line /
-			(vdsc_cfg->initial_scale_value - 8);
-	else
-		vdsc_cfg->scale_decrement_interval = DSC_SCALE_DECREMENT_INTERVAL_MAX;
-
-	vdsc_cfg->final_offset = vdsc_cfg->rc_model_size -
-		(vdsc_cfg->initial_xmit_delay *
-		 vdsc_cfg->bits_per_pixel + 8) / 16 + num_extra_mux_bits;
-
-	if (vdsc_cfg->final_offset >= vdsc_cfg->rc_model_size) {
-		DRM_DEBUG_KMS("FinalOfs < RcModelSze for this InitialXmitDelay\n");
-		return -ERANGE;
-	}
-
-	final_scale = (vdsc_cfg->rc_model_size * 8) /
-		(vdsc_cfg->rc_model_size - vdsc_cfg->final_offset);
-	if (vdsc_cfg->slice_height > 1)
-		/*
-		 * NflBpgOffset is 16 bit value with 11 fractional bits
-		 * hence we multiply by 2^11 for preserving the
-		 * fractional part
-		 */
-		vdsc_cfg->nfl_bpg_offset = DIV_ROUND_UP((vdsc_cfg->first_line_bpg_offset << 11),
-							(vdsc_cfg->slice_height - 1));
-	else
-		vdsc_cfg->nfl_bpg_offset = 0;
-
-	/* 2^16 - 1 */
-	if (vdsc_cfg->nfl_bpg_offset > 65535) {
-		DRM_DEBUG_KMS("NflBpgOffset is too large for this slice height\n");
-		return -ERANGE;
-	}
-
-	/* Number of groups used to code the entire slice */
-	groups_total = groups_per_line * vdsc_cfg->slice_height;
-
-	/* slice_bpg_offset is 16 bit value with 11 fractional bits */
-	vdsc_cfg->slice_bpg_offset = DIV_ROUND_UP(((vdsc_cfg->rc_model_size -
-						    vdsc_cfg->initial_offset +
-						    num_extra_mux_bits) << 11),
-						  groups_total);
-
-	if (final_scale > 9) {
-		/*
-		 * ScaleIncrementInterval =
-		 * finaloffset/((NflBpgOffset + SliceBpgOffset)*8(finalscale - 1.125))
-		 * as (NflBpgOffset + SliceBpgOffset) has 11 bit fractional value,
-		 * we need divide by 2^11 from pstDscCfg values
-		 */
-		vdsc_cfg->scale_increment_interval =
-				(vdsc_cfg->final_offset * (1 << 11)) /
-				((vdsc_cfg->nfl_bpg_offset +
-				vdsc_cfg->slice_bpg_offset) *
-				(final_scale - 9));
-	} else {
-		/*
-		 * If finalScaleValue is less than or equal to 9, a value of 0 should
-		 * be used to disable the scale increment at the end of the slice
-		 */
-		vdsc_cfg->scale_increment_interval = 0;
-	}
-
-	if (vdsc_cfg->scale_increment_interval > 65535) {
-		DRM_DEBUG_KMS("ScaleIncrementInterval is large for slice height\n");
-		return -ERANGE;
-	}
-
-	/*
-	 * DSC spec mentions that bits_per_pixel specifies the target
-	 * bits/pixel (bpp) rate that is used by the encoder,
-	 * in steps of 1/16 of a bit per pixel
-	 */
-	rbs_min = vdsc_cfg->rc_model_size - vdsc_cfg->initial_offset +
-		DIV_ROUND_UP(vdsc_cfg->initial_xmit_delay *
-			     vdsc_cfg->bits_per_pixel, 16) +
-		groups_per_line * vdsc_cfg->first_line_bpg_offset;
-
-	hrd_delay = DIV_ROUND_UP((rbs_min * 16), vdsc_cfg->bits_per_pixel);
-	vdsc_cfg->rc_bits = (hrd_delay * vdsc_cfg->bits_per_pixel) / 16;
-	vdsc_cfg->initial_dec_delay = hrd_delay - vdsc_cfg->initial_xmit_delay;
-
-	return 0;
-}
-
 int intel_dp_compute_dsc_params(struct intel_dp *intel_dp,
 				struct intel_crtc_state *pipe_config)
 {
@@ -491,7 +368,7 @@ int intel_dp_compute_dsc_params(struct intel_dp *intel_dp,
 			DSC_1_1_MAX_LINEBUF_DEPTH_BITS : line_buf_depth;
 
 	/* Gen 11 does not support YCbCr */
-	vdsc_cfg->enable422 = false;
+	vdsc_cfg->simple_422 = false;
 	/* Gen 11 does not support VBR */
 	vdsc_cfg->vbr_enable = false;
 	vdsc_cfg->block_pred_enable =
@@ -574,7 +451,7 @@ int intel_dp_compute_dsc_params(struct intel_dp *intel_dp,
 	vdsc_cfg->initial_scale_value = (vdsc_cfg->rc_model_size << 3) /
 		(vdsc_cfg->rc_model_size - vdsc_cfg->initial_offset);
 
-	return intel_compute_rc_parameters(vdsc_cfg);
+	return drm_dsc_compute_rc_parameters(vdsc_cfg);
 }
 
 enum intel_display_power_domain
@@ -618,7 +495,7 @@ static void intel_configure_pps_for_dsc_encoder(struct intel_encoder *encoder,
 		pps_val |= DSC_BLOCK_PREDICTION;
 	if (vdsc_cfg->convert_rgb)
 		pps_val |= DSC_COLOR_SPACE_CONVERSION;
-	if (vdsc_cfg->enable422)
+	if (vdsc_cfg->simple_422)
 		pps_val |= DSC_422_ENABLE;
 	if (vdsc_cfg->vbr_enable)
 		pps_val |= DSC_VBR_ENABLE;
@@ -1004,10 +881,10 @@ static void intel_dp_write_dsc_pps_sdp(struct intel_encoder *encoder,
 	struct drm_dsc_pps_infoframe dp_dsc_pps_sdp;
 
 	/* Prepare DP SDP PPS header as per DP 1.4 spec, Table 2-123 */
-	drm_dsc_dp_pps_header_init(&dp_dsc_pps_sdp);
+	drm_dsc_dp_pps_header_init(&dp_dsc_pps_sdp.pps_header);
 
 	/* Fill the PPS payload bytes as per DSC spec 1.2 Table 4-1 */
-	drm_dsc_pps_infoframe_pack(&dp_dsc_pps_sdp, vdsc_cfg);
+	drm_dsc_pps_payload_pack(&dp_dsc_pps_sdp.pps_payload, vdsc_cfg);
 
 	intel_dig_port->write_infoframe(encoder, crtc_state,
 					DP_SDP_PPS, &dp_dsc_pps_sdp,
diff --git a/drivers/gpu/drm/i915/intel_workarounds.c b/drivers/gpu/drm/i915/intel_workarounds.c
index 15f4a6dee5aa..9682dd575152 100644
--- a/drivers/gpu/drm/i915/intel_workarounds.c
+++ b/drivers/gpu/drm/i915/intel_workarounds.c
@@ -541,10 +541,6 @@ static void icl_ctx_workarounds_init(struct intel_engine_cs *engine)
 		WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
 				  GEN11_TDL_CLOCK_GATING_FIX_DISABLE);
 
-	/* WaEnableStateCacheRedirectToCS:icl */
-	WA_SET_BIT_MASKED(GEN9_SLICE_COMMON_ECO_CHICKEN1,
-			  GEN11_STATE_CACHE_REDIRECT_TO_CS);
-
 	/* Wa_2006665173:icl (pre-prod) */
 	if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0))
 		WA_SET_BIT_MASKED(GEN11_COMMON_SLICE_CHICKEN3,
@@ -555,6 +551,11 @@ static void icl_ctx_workarounds_init(struct intel_engine_cs *engine)
 			   GEN10_CACHE_MODE_SS,
 			   0, /* write-only, so skip validation */
 			   _MASKED_BIT_ENABLE(FLOAT_BLEND_OPTIMIZATION_ENABLE));
+
+	/* WaDisableGPGPUMidThreadPreemption:icl */
+	WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1,
+			    GEN9_PREEMPT_GPGPU_LEVEL_MASK,
+			    GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL);
 }
 
 void intel_engine_init_ctx_wa(struct intel_engine_cs *engine)
@@ -564,26 +565,26 @@ void intel_engine_init_ctx_wa(struct intel_engine_cs *engine)
 
 	wa_init_start(wal, "context");
 
-	if (INTEL_GEN(i915) < 8)
-		return;
-	else if (IS_BROADWELL(i915))
-		bdw_ctx_workarounds_init(engine);
-	else if (IS_CHERRYVIEW(i915))
-		chv_ctx_workarounds_init(engine);
-	else if (IS_SKYLAKE(i915))
-		skl_ctx_workarounds_init(engine);
-	else if (IS_BROXTON(i915))
-		bxt_ctx_workarounds_init(engine);
-	else if (IS_KABYLAKE(i915))
-		kbl_ctx_workarounds_init(engine);
-	else if (IS_GEMINILAKE(i915))
-		glk_ctx_workarounds_init(engine);
-	else if (IS_COFFEELAKE(i915))
-		cfl_ctx_workarounds_init(engine);
+	if (IS_GEN(i915, 11))
+		icl_ctx_workarounds_init(engine);
 	else if (IS_CANNONLAKE(i915))
 		cnl_ctx_workarounds_init(engine);
-	else if (IS_ICELAKE(i915))
-		icl_ctx_workarounds_init(engine);
+	else if (IS_COFFEELAKE(i915))
+		cfl_ctx_workarounds_init(engine);
+	else if (IS_GEMINILAKE(i915))
+		glk_ctx_workarounds_init(engine);
+	else if (IS_KABYLAKE(i915))
+		kbl_ctx_workarounds_init(engine);
+	else if (IS_BROXTON(i915))
+		bxt_ctx_workarounds_init(engine);
+	else if (IS_SKYLAKE(i915))
+		skl_ctx_workarounds_init(engine);
+	else if (IS_CHERRYVIEW(i915))
+		chv_ctx_workarounds_init(engine);
+	else if (IS_BROADWELL(i915))
+		bdw_ctx_workarounds_init(engine);
+	else if (INTEL_GEN(i915) < 8)
+		return;
 	else
 		MISSING_CASE(INTEL_GEN(i915));
 
@@ -724,9 +725,9 @@ cfl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 }
 
 static void
-wa_init_mcr(struct drm_i915_private *dev_priv, struct i915_wa_list *wal)
+wa_init_mcr(struct drm_i915_private *i915, struct i915_wa_list *wal)
 {
-	const struct sseu_dev_info *sseu = &RUNTIME_INFO(dev_priv)->sseu;
+	const struct sseu_dev_info *sseu = &RUNTIME_INFO(i915)->sseu;
 	u32 mcr_slice_subslice_mask;
 
 	/*
@@ -742,14 +743,15 @@ wa_init_mcr(struct drm_i915_private *dev_priv, struct i915_wa_list *wal)
 	 * something more complex that requires checking the range of every
 	 * MMIO read).
 	 */
-	if (INTEL_GEN(dev_priv) >= 10 &&
+	if (INTEL_GEN(i915) >= 10 &&
 	    is_power_of_2(sseu->slice_mask)) {
 		/*
 		 * read FUSE3 for enabled L3 Bank IDs, if L3 Bank matches
 		 * enabled subslice, no need to redirect MCR packet
 		 */
 		u32 slice = fls(sseu->slice_mask);
-		u32 fuse3 = I915_READ(GEN10_MIRROR_FUSE3);
+		u32 fuse3 =
+			intel_uncore_read(&i915->uncore, GEN10_MIRROR_FUSE3);
 		u8 ss_mask = sseu->subslice_mask[slice];
 
 		u8 enabled_mask = (ss_mask | ss_mask >>
@@ -763,7 +765,7 @@ wa_init_mcr(struct drm_i915_private *dev_priv, struct i915_wa_list *wal)
 		WARN_ON((enabled_mask & disabled_mask) != enabled_mask);
 	}
 
-	if (INTEL_GEN(dev_priv) >= 11)
+	if (INTEL_GEN(i915) >= 11)
 		mcr_slice_subslice_mask = GEN11_MCR_SLICE_MASK |
 					  GEN11_MCR_SUBSLICE_MASK;
 	else
@@ -783,7 +785,7 @@ wa_init_mcr(struct drm_i915_private *dev_priv, struct i915_wa_list *wal)
 	wa_write_masked_or(wal,
 			   GEN8_MCR_SELECTOR,
 			   mcr_slice_subslice_mask,
-			   intel_calculate_mcr_s_ss_select(dev_priv));
+			   intel_calculate_mcr_s_ss_select(i915));
 }
 
 static void
@@ -862,26 +864,22 @@ icl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal)
 static void
 gt_init_workarounds(struct drm_i915_private *i915, struct i915_wa_list *wal)
 {
-	if (INTEL_GEN(i915) < 8)
-		return;
-	else if (IS_BROADWELL(i915))
-		return;
-	else if (IS_CHERRYVIEW(i915))
-		return;
-	else if (IS_SKYLAKE(i915))
-		skl_gt_workarounds_init(i915, wal);
-	else if (IS_BROXTON(i915))
-		bxt_gt_workarounds_init(i915, wal);
-	else if (IS_KABYLAKE(i915))
-		kbl_gt_workarounds_init(i915, wal);
-	else if (IS_GEMINILAKE(i915))
-		glk_gt_workarounds_init(i915, wal);
-	else if (IS_COFFEELAKE(i915))
-		cfl_gt_workarounds_init(i915, wal);
+	if (IS_GEN(i915, 11))
+		icl_gt_workarounds_init(i915, wal);
 	else if (IS_CANNONLAKE(i915))
 		cnl_gt_workarounds_init(i915, wal);
-	else if (IS_ICELAKE(i915))
-		icl_gt_workarounds_init(i915, wal);
+	else if (IS_COFFEELAKE(i915))
+		cfl_gt_workarounds_init(i915, wal);
+	else if (IS_GEMINILAKE(i915))
+		glk_gt_workarounds_init(i915, wal);
+	else if (IS_KABYLAKE(i915))
+		kbl_gt_workarounds_init(i915, wal);
+	else if (IS_BROXTON(i915))
+		bxt_gt_workarounds_init(i915, wal);
+	else if (IS_SKYLAKE(i915))
+		skl_gt_workarounds_init(i915, wal);
+	else if (INTEL_GEN(i915) <= 8)
+		return;
 	else
 		MISSING_CASE(INTEL_GEN(i915));
 }
@@ -896,15 +894,14 @@ void intel_gt_init_workarounds(struct drm_i915_private *i915)
 }
 
 static enum forcewake_domains
-wal_get_fw_for_rmw(struct drm_i915_private *dev_priv,
-		   const struct i915_wa_list *wal)
+wal_get_fw_for_rmw(struct intel_uncore *uncore, const struct i915_wa_list *wal)
 {
 	enum forcewake_domains fw = 0;
 	struct i915_wa *wa;
 	unsigned int i;
 
 	for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
-		fw |= intel_uncore_forcewake_for_reg(dev_priv,
+		fw |= intel_uncore_forcewake_for_reg(uncore,
 						     wa->reg,
 						     FW_REG_READ |
 						     FW_REG_WRITE);
@@ -913,7 +910,7 @@ wal_get_fw_for_rmw(struct drm_i915_private *dev_priv,
 }
 
 static void
-wa_list_apply(struct drm_i915_private *dev_priv, const struct i915_wa_list *wal)
+wa_list_apply(struct intel_uncore *uncore, const struct i915_wa_list *wal)
 {
 	enum forcewake_domains fw;
 	unsigned long flags;
@@ -923,27 +920,22 @@ wa_list_apply(struct drm_i915_private *dev_priv, const struct i915_wa_list *wal)
 	if (!wal->count)
 		return;
 
-	fw = wal_get_fw_for_rmw(dev_priv, wal);
+	fw = wal_get_fw_for_rmw(uncore, wal);
 
-	spin_lock_irqsave(&dev_priv->uncore.lock, flags);
-	intel_uncore_forcewake_get__locked(dev_priv, fw);
+	spin_lock_irqsave(&uncore->lock, flags);
+	intel_uncore_forcewake_get__locked(uncore, fw);
 
 	for (i = 0, wa = wal->list; i < wal->count; i++, wa++) {
-		u32 val = I915_READ_FW(wa->reg);
-
-		val &= ~wa->mask;
-		val |= wa->val;
-
-		I915_WRITE_FW(wa->reg, val);
+		intel_uncore_rmw_fw(uncore, wa->reg, wa->mask, wa->val);
 	}
 
-	intel_uncore_forcewake_put__locked(dev_priv, fw);
-	spin_unlock_irqrestore(&dev_priv->uncore.lock, flags);
+	intel_uncore_forcewake_put__locked(uncore, fw);
+	spin_unlock_irqrestore(&uncore->lock, flags);
 }
 
-void intel_gt_apply_workarounds(struct drm_i915_private *dev_priv)
+void intel_gt_apply_workarounds(struct drm_i915_private *i915)
 {
-	wa_list_apply(dev_priv, &dev_priv->gt_wa_list);
+	wa_list_apply(&i915->uncore, &i915->gt_wa_list);
 }
 
 static bool
@@ -960,7 +952,7 @@ wa_verify(const struct i915_wa *wa, u32 cur, const char *name, const char *from)
 	return true;
 }
 
-static bool wa_list_verify(struct drm_i915_private *dev_priv,
+static bool wa_list_verify(struct intel_uncore *uncore,
 			   const struct i915_wa_list *wal,
 			   const char *from)
 {
@@ -969,15 +961,17 @@ static bool wa_list_verify(struct drm_i915_private *dev_priv,
 	bool ok = true;
 
 	for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
-		ok &= wa_verify(wa, I915_READ(wa->reg), wal->name, from);
+		ok &= wa_verify(wa,
+				intel_uncore_read(uncore, wa->reg),
+				wal->name, from);
 
 	return ok;
 }
 
-bool intel_gt_verify_workarounds(struct drm_i915_private *dev_priv,
+bool intel_gt_verify_workarounds(struct drm_i915_private *i915,
 				 const char *from)
 {
-	return wa_list_verify(dev_priv, &dev_priv->gt_wa_list, from);
+	return wa_list_verify(&i915->uncore, &i915->gt_wa_list, from);
 }
 
 static void
@@ -1052,6 +1046,9 @@ static void icl_whitelist_build(struct i915_wa_list *w)
 
 	/* WaAllowUMDToModifySamplerMode:icl */
 	whitelist_reg(w, GEN10_SAMPLER_MODE);
+
+	/* WaEnableStateCacheRedirectToCS:icl */
+	whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1);
 }
 
 void intel_engine_init_whitelist(struct intel_engine_cs *engine)
@@ -1059,30 +1056,26 @@ void intel_engine_init_whitelist(struct intel_engine_cs *engine)
 	struct drm_i915_private *i915 = engine->i915;
 	struct i915_wa_list *w = &engine->whitelist;
 
-	GEM_BUG_ON(engine->id != RCS);
+	GEM_BUG_ON(engine->id != RCS0);
 
 	wa_init_start(w, "whitelist");
 
-	if (INTEL_GEN(i915) < 8)
-		return;
-	else if (IS_BROADWELL(i915))
-		return;
-	else if (IS_CHERRYVIEW(i915))
-		return;
-	else if (IS_SKYLAKE(i915))
-		skl_whitelist_build(w);
-	else if (IS_BROXTON(i915))
-		bxt_whitelist_build(w);
-	else if (IS_KABYLAKE(i915))
-		kbl_whitelist_build(w);
-	else if (IS_GEMINILAKE(i915))
-		glk_whitelist_build(w);
-	else if (IS_COFFEELAKE(i915))
-		cfl_whitelist_build(w);
+	if (IS_GEN(i915, 11))
+		icl_whitelist_build(w);
 	else if (IS_CANNONLAKE(i915))
 		cnl_whitelist_build(w);
-	else if (IS_ICELAKE(i915))
-		icl_whitelist_build(w);
+	else if (IS_COFFEELAKE(i915))
+		cfl_whitelist_build(w);
+	else if (IS_GEMINILAKE(i915))
+		glk_whitelist_build(w);
+	else if (IS_KABYLAKE(i915))
+		kbl_whitelist_build(w);
+	else if (IS_BROXTON(i915))
+		bxt_whitelist_build(w);
+	else if (IS_SKYLAKE(i915))
+		skl_whitelist_build(w);
+	else if (INTEL_GEN(i915) <= 8)
+		return;
 	else
 		MISSING_CASE(INTEL_GEN(i915));
 
@@ -1091,8 +1084,8 @@ void intel_engine_init_whitelist(struct intel_engine_cs *engine)
 
 void intel_engine_apply_whitelist(struct intel_engine_cs *engine)
 {
-	struct drm_i915_private *dev_priv = engine->i915;
 	const struct i915_wa_list *wal = &engine->whitelist;
+	struct intel_uncore *uncore = engine->uncore;
 	const u32 base = engine->mmio_base;
 	struct i915_wa *wa;
 	unsigned int i;
@@ -1101,13 +1094,15 @@ void intel_engine_apply_whitelist(struct intel_engine_cs *engine)
 		return;
 
 	for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
-		I915_WRITE(RING_FORCE_TO_NONPRIV(base, i),
-			   i915_mmio_reg_offset(wa->reg));
+		intel_uncore_write(uncore,
+				   RING_FORCE_TO_NONPRIV(base, i),
+				   i915_mmio_reg_offset(wa->reg));
 
 	/* And clear the rest just in case of garbage */
 	for (; i < RING_MAX_NONPRIV_SLOTS; i++)
-		I915_WRITE(RING_FORCE_TO_NONPRIV(base, i),
-			   i915_mmio_reg_offset(RING_NOPID(base)));
+		intel_uncore_write(uncore,
+				   RING_FORCE_TO_NONPRIV(base, i),
+				   i915_mmio_reg_offset(RING_NOPID(base)));
 }
 
 static void
@@ -1115,7 +1110,7 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
 {
 	struct drm_i915_private *i915 = engine->i915;
 
-	if (IS_ICELAKE(i915)) {
+	if (IS_GEN(i915, 11)) {
 		/* This is not an Wa. Enable for better image quality */
 		wa_masked_en(wal,
 			     _3D_CHICKEN3,
@@ -1170,8 +1165,8 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
 				    GEN7_DISABLE_SAMPLER_PREFETCH);
 	}
 
-	if (IS_GEN(i915, 9) || IS_CANNONLAKE(i915)) {
-		/* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl,cnl */
+	if (IS_GEN_RANGE(i915, 9, 11)) {
+		/* FtrPerCtxtPreemptionGranularityControl:skl,bxt,kbl,cfl,cnl,icl */
 		wa_masked_en(wal,
 			     GEN7_FF_SLICE_CS_CHICKEN1,
 			     GEN9_FFSC_PERCTX_PREEMPT_CTRL);
@@ -1236,7 +1231,7 @@ engine_init_workarounds(struct intel_engine_cs *engine, struct i915_wa_list *wal
 	if (I915_SELFTEST_ONLY(INTEL_GEN(engine->i915) < 8))
 		return;
 
-	if (engine->id == RCS)
+	if (engine->id == RCS0)
 		rcs_engine_wa_init(engine, wal);
 	else
 		xcs_engine_wa_init(engine, wal);
@@ -1256,7 +1251,7 @@ void intel_engine_init_workarounds(struct intel_engine_cs *engine)
 
 void intel_engine_apply_workarounds(struct intel_engine_cs *engine)
 {
-	wa_list_apply(engine->i915, &engine->wa_list);
+	wa_list_apply(engine->uncore, &engine->wa_list);
 }
 
 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
diff --git a/drivers/gpu/drm/i915/intel_workarounds.h b/drivers/gpu/drm/i915/intel_workarounds.h
index 7c734714b05e..34eee5ec511e 100644
--- a/drivers/gpu/drm/i915/intel_workarounds.h
+++ b/drivers/gpu/drm/i915/intel_workarounds.h
@@ -9,18 +9,7 @@
 
 #include <linux/slab.h>
 
-struct i915_wa {
-	i915_reg_t	  reg;
-	u32		  mask;
-	u32		  val;
-};
-
-struct i915_wa_list {
-	const char	*name;
-	struct i915_wa	*list;
-	unsigned int	count;
-	unsigned int	wa_count;
-};
+#include "intel_workarounds_types.h"
 
 static inline void intel_wa_list_free(struct i915_wa_list *wal)
 {
@@ -31,9 +20,9 @@ static inline void intel_wa_list_free(struct i915_wa_list *wal)
 void intel_engine_init_ctx_wa(struct intel_engine_cs *engine);
 int intel_engine_emit_ctx_wa(struct i915_request *rq);
 
-void intel_gt_init_workarounds(struct drm_i915_private *dev_priv);
-void intel_gt_apply_workarounds(struct drm_i915_private *dev_priv);
-bool intel_gt_verify_workarounds(struct drm_i915_private *dev_priv,
+void intel_gt_init_workarounds(struct drm_i915_private *i915);
+void intel_gt_apply_workarounds(struct drm_i915_private *i915);
+bool intel_gt_verify_workarounds(struct drm_i915_private *i915,
 				 const char *from);
 
 void intel_engine_init_whitelist(struct intel_engine_cs *engine);
diff --git a/drivers/gpu/drm/i915/intel_workarounds_types.h b/drivers/gpu/drm/i915/intel_workarounds_types.h
new file mode 100644
index 000000000000..30918da180ff
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_workarounds_types.h
@@ -0,0 +1,27 @@
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2014-2018 Intel Corporation
+ */
+
+#ifndef __INTEL_WORKAROUNDS_TYPES_H__
+#define __INTEL_WORKAROUNDS_TYPES_H__
+
+#include <linux/types.h>
+
+#include "i915_reg.h"
+
+struct i915_wa {
+	i915_reg_t	  reg;
+	u32		  mask;
+	u32		  val;
+};
+
+struct i915_wa_list {
+	const char	*name;
+	struct i915_wa	*list;
+	unsigned int	count;
+	unsigned int	wa_count;
+};
+
+#endif /* __INTEL_WORKAROUNDS_TYPES_H__ */
diff --git a/drivers/gpu/drm/i915/selftests/huge_gem_object.c b/drivers/gpu/drm/i915/selftests/huge_gem_object.c
index 391f3d9ffdf1..419fd4d6a8f0 100644
--- a/drivers/gpu/drm/i915/selftests/huge_gem_object.c
+++ b/drivers/gpu/drm/i915/selftests/huge_gem_object.c
@@ -122,7 +122,7 @@ huge_gem_object(struct drm_i915_private *i915,
 	if (overflows_type(dma_size, obj->base.size))
 		return ERR_PTR(-E2BIG);
 
-	obj = i915_gem_object_alloc(i915);
+	obj = i915_gem_object_alloc();
 	if (!obj)
 		return ERR_PTR(-ENOMEM);
 
diff --git a/drivers/gpu/drm/i915/selftests/huge_pages.c b/drivers/gpu/drm/i915/selftests/huge_pages.c
index a9a2fa35876f..90721b54e7ae 100644
--- a/drivers/gpu/drm/i915/selftests/huge_pages.c
+++ b/drivers/gpu/drm/i915/selftests/huge_pages.c
@@ -171,7 +171,7 @@ huge_pages_object(struct drm_i915_private *i915,
 	if (overflows_type(size, obj->base.size))
 		return ERR_PTR(-E2BIG);
 
-	obj = i915_gem_object_alloc(i915);
+	obj = i915_gem_object_alloc();
 	if (!obj)
 		return ERR_PTR(-ENOMEM);
 
@@ -320,7 +320,7 @@ fake_huge_pages_object(struct drm_i915_private *i915, u64 size, bool single)
 	if (overflows_type(size, obj->base.size))
 		return ERR_PTR(-E2BIG);
 
-	obj = i915_gem_object_alloc(i915);
+	obj = i915_gem_object_alloc();
 	if (!obj)
 		return ERR_PTR(-ENOMEM);
 
@@ -908,10 +908,6 @@ gpu_write_dw(struct i915_vma *vma, u64 offset, u32 val)
 	if (IS_ERR(obj))
 		return ERR_CAST(obj);
 
-	err = i915_gem_object_set_to_wc_domain(obj, true);
-	if (err)
-		goto err;
-
 	cmd = i915_gem_object_pin_map(obj, I915_MAP_WC);
 	if (IS_ERR(cmd)) {
 		err = PTR_ERR(cmd);
@@ -1449,7 +1445,7 @@ static int igt_ppgtt_pin_update(void *arg)
 	 * huge-gtt-pages.
 	 */
 
-	if (!ppgtt || !i915_vm_is_48bit(&ppgtt->vm)) {
+	if (!ppgtt || !i915_vm_is_4lvl(&ppgtt->vm)) {
 		pr_info("48b PPGTT not supported, skipping\n");
 		return 0;
 	}
@@ -1535,7 +1531,7 @@ static int igt_ppgtt_pin_update(void *arg)
 	 * land in the now stale 2M page.
 	 */
 
-	err = gpu_write(vma, ctx, dev_priv->engine[RCS], 0, 0xdeadbeaf);
+	err = gpu_write(vma, ctx, dev_priv->engine[RCS0], 0, 0xdeadbeaf);
 	if (err)
 		goto out_unpin;
 
@@ -1584,6 +1580,7 @@ static int igt_tmpfs_fallback(void *arg)
 	}
 	*vaddr = 0xdeadbeaf;
 
+	__i915_gem_object_flush_map(obj, 0, 64);
 	i915_gem_object_unpin_map(obj);
 
 	vma = i915_vma_instance(obj, vm, NULL);
@@ -1653,7 +1650,7 @@ static int igt_shrink_thp(void *arg)
 	if (err)
 		goto out_unpin;
 
-	err = gpu_write(vma, ctx, i915->engine[RCS], 0, 0xdeadbeaf);
+	err = gpu_write(vma, ctx, i915->engine[RCS0], 0, 0xdeadbeaf);
 	if (err)
 		goto out_unpin;
 
@@ -1709,16 +1706,17 @@ int i915_gem_huge_page_mock_selftests(void)
 		return -ENOMEM;
 
 	/* Pretend to be a device which supports the 48b PPGTT */
-	mkwrite_device_info(dev_priv)->ppgtt = INTEL_PPGTT_FULL_4LVL;
+	mkwrite_device_info(dev_priv)->ppgtt_type = INTEL_PPGTT_FULL;
+	mkwrite_device_info(dev_priv)->ppgtt_size = 48;
 
 	mutex_lock(&dev_priv->drm.struct_mutex);
-	ppgtt = i915_ppgtt_create(dev_priv, ERR_PTR(-ENODEV));
+	ppgtt = i915_ppgtt_create(dev_priv);
 	if (IS_ERR(ppgtt)) {
 		err = PTR_ERR(ppgtt);
 		goto out_unlock;
 	}
 
-	if (!i915_vm_is_48bit(&ppgtt->vm)) {
+	if (!i915_vm_is_4lvl(&ppgtt->vm)) {
 		pr_err("failed to create 48b PPGTT\n");
 		err = -EINVAL;
 		goto out_close;
@@ -1734,7 +1732,6 @@ int i915_gem_huge_page_mock_selftests(void)
 	err = i915_subtests(tests, ppgtt);
 
 out_close:
-	i915_ppgtt_close(&ppgtt->vm);
 	i915_ppgtt_put(ppgtt);
 
 out_unlock:
@@ -1764,7 +1761,7 @@ int i915_gem_huge_page_live_selftests(struct drm_i915_private *dev_priv)
 		return 0;
 	}
 
-	if (i915_terminally_wedged(&dev_priv->gpu_error))
+	if (i915_terminally_wedged(dev_priv))
 		return 0;
 
 	file = mock_file(dev_priv);
diff --git a/drivers/gpu/drm/i915/selftests/i915_active.c b/drivers/gpu/drm/i915/selftests/i915_active.c
index 337b1f98b923..27d8f853111b 100644
--- a/drivers/gpu/drm/i915/selftests/i915_active.c
+++ b/drivers/gpu/drm/i915/selftests/i915_active.c
@@ -150,7 +150,7 @@ int i915_active_live_selftests(struct drm_i915_private *i915)
 		SUBTEST(live_active_retire),
 	};
 
-	if (i915_terminally_wedged(&i915->gpu_error))
+	if (i915_terminally_wedged(i915))
 		return 0;
 
 	return i915_subtests(tests, i915);
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem.c b/drivers/gpu/drm/i915/selftests/i915_gem.c
index e77b7ed449ae..6fd70d326468 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem.c
@@ -84,14 +84,9 @@ static void simulate_hibernate(struct drm_i915_private *i915)
 
 static int pm_prepare(struct drm_i915_private *i915)
 {
-	int err = 0;
-
-	if (i915_gem_suspend(i915)) {
-		pr_err("i915_gem_suspend failed\n");
-		err = -EINVAL;
-	}
+	i915_gem_suspend(i915);
 
-	return err;
+	return 0;
 }
 
 static void pm_suspend(struct drm_i915_private *i915)
@@ -220,5 +215,8 @@ int i915_gem_live_selftests(struct drm_i915_private *i915)
 		SUBTEST(igt_gem_hibernate),
 	};
 
+	if (i915_terminally_wedged(i915))
+		return 0;
+
 	return i915_subtests(tests, i915);
 }
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c b/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c
index fd89a5a33c1a..e43630b40fce 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c
@@ -202,7 +202,7 @@ static int gpu_set(struct drm_i915_gem_object *obj,
 	if (IS_ERR(vma))
 		return PTR_ERR(vma);
 
-	rq = i915_request_alloc(i915->engine[RCS], i915->kernel_context);
+	rq = i915_request_alloc(i915->engine[RCS0], i915->kernel_context);
 	if (IS_ERR(rq)) {
 		i915_vma_unpin(vma);
 		return PTR_ERR(rq);
@@ -248,15 +248,15 @@ static bool always_valid(struct drm_i915_private *i915)
 
 static bool needs_fence_registers(struct drm_i915_private *i915)
 {
-	return !i915_terminally_wedged(&i915->gpu_error);
+	return !i915_terminally_wedged(i915);
 }
 
 static bool needs_mi_store_dword(struct drm_i915_private *i915)
 {
-	if (i915_terminally_wedged(&i915->gpu_error))
+	if (i915_terminally_wedged(i915))
 		return false;
 
-	return intel_engine_can_store_dword(i915->engine[RCS]);
+	return intel_engine_can_store_dword(i915->engine[RCS0]);
 }
 
 static const struct igt_coherency_mode {
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_context.c b/drivers/gpu/drm/i915/selftests/i915_gem_context.c
index 7eb58a9d1319..4e1b6efc6b22 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_context.c
@@ -76,7 +76,7 @@ static int live_nop_switch(void *arg)
 	}
 
 	for (n = 0; n < nctx; n++) {
-		ctx[n] = i915_gem_create_context(i915, file->driver_priv);
+		ctx[n] = live_context(i915, file);
 		if (IS_ERR(ctx[n])) {
 			err = PTR_ERR(ctx[n]);
 			goto out_unlock;
@@ -220,6 +220,7 @@ gpu_fill_dw(struct i915_vma *vma, u64 offset, unsigned long count, u32 value)
 		offset += PAGE_SIZE;
 	}
 	*cmd = MI_BATCH_BUFFER_END;
+	i915_gem_object_flush_map(obj);
 	i915_gem_object_unpin_map(obj);
 
 	err = i915_gem_object_set_to_gtt_domain(obj, false);
@@ -372,7 +373,8 @@ static int cpu_fill(struct drm_i915_gem_object *obj, u32 value)
 	return 0;
 }
 
-static int cpu_check(struct drm_i915_gem_object *obj, unsigned int max)
+static noinline int cpu_check(struct drm_i915_gem_object *obj,
+			      unsigned int idx, unsigned int max)
 {
 	unsigned int n, m, needs_flush;
 	int err;
@@ -390,8 +392,10 @@ static int cpu_check(struct drm_i915_gem_object *obj, unsigned int max)
 
 		for (m = 0; m < max; m++) {
 			if (map[m] != m) {
-				pr_err("Invalid value at page %d, offset %d: found %x expected %x\n",
-				       n, m, map[m], m);
+				pr_err("%pS: Invalid value at object %d page %d/%ld, offset %d/%d: found %x expected %x\n",
+				       __builtin_return_address(0), idx,
+				       n, real_page_count(obj), m, max,
+				       map[m], m);
 				err = -EINVAL;
 				goto out_unmap;
 			}
@@ -399,8 +403,9 @@ static int cpu_check(struct drm_i915_gem_object *obj, unsigned int max)
 
 		for (; m < DW_PER_PAGE; m++) {
 			if (map[m] != STACK_MAGIC) {
-				pr_err("Invalid value at page %d, offset %d: found %x expected %x\n",
-				       n, m, map[m], STACK_MAGIC);
+				pr_err("%pS: Invalid value at object %d page %d, offset %d: found %x expected %x (uninitialised)\n",
+				       __builtin_return_address(0), idx, n, m,
+				       map[m], STACK_MAGIC);
 				err = -EINVAL;
 				goto out_unmap;
 			}
@@ -478,12 +483,8 @@ static unsigned long max_dwords(struct drm_i915_gem_object *obj)
 static int igt_ctx_exec(void *arg)
 {
 	struct drm_i915_private *i915 = arg;
-	struct drm_i915_gem_object *obj = NULL;
-	unsigned long ncontexts, ndwords, dw;
-	struct igt_live_test t;
-	struct drm_file *file;
-	IGT_TIMEOUT(end_time);
-	LIST_HEAD(objects);
+	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
 	int err = -ENODEV;
 
 	/*
@@ -495,44 +496,167 @@ static int igt_ctx_exec(void *arg)
 	if (!DRIVER_CAPS(i915)->has_logical_contexts)
 		return 0;
 
+	for_each_engine(engine, i915, id) {
+		struct drm_i915_gem_object *obj = NULL;
+		unsigned long ncontexts, ndwords, dw;
+		struct igt_live_test t;
+		struct drm_file *file;
+		IGT_TIMEOUT(end_time);
+		LIST_HEAD(objects);
+
+		if (!intel_engine_can_store_dword(engine))
+			continue;
+
+		if (!engine->context_size)
+			continue; /* No logical context support in HW */
+
+		file = mock_file(i915);
+		if (IS_ERR(file))
+			return PTR_ERR(file);
+
+		mutex_lock(&i915->drm.struct_mutex);
+
+		err = igt_live_test_begin(&t, i915, __func__, engine->name);
+		if (err)
+			goto out_unlock;
+
+		ncontexts = 0;
+		ndwords = 0;
+		dw = 0;
+		while (!time_after(jiffies, end_time)) {
+			struct i915_gem_context *ctx;
+			intel_wakeref_t wakeref;
+
+			ctx = live_context(i915, file);
+			if (IS_ERR(ctx)) {
+				err = PTR_ERR(ctx);
+				goto out_unlock;
+			}
+
+			if (!obj) {
+				obj = create_test_object(ctx, file, &objects);
+				if (IS_ERR(obj)) {
+					err = PTR_ERR(obj);
+					goto out_unlock;
+				}
+			}
+
+			with_intel_runtime_pm(i915, wakeref)
+				err = gpu_fill(obj, ctx, engine, dw);
+			if (err) {
+				pr_err("Failed to fill dword %lu [%lu/%lu] with gpu (%s) in ctx %u [full-ppgtt? %s], err=%d\n",
+				       ndwords, dw, max_dwords(obj),
+				       engine->name, ctx->hw_id,
+				       yesno(!!ctx->ppgtt), err);
+				goto out_unlock;
+			}
+
+			if (++dw == max_dwords(obj)) {
+				obj = NULL;
+				dw = 0;
+			}
+
+			ndwords++;
+			ncontexts++;
+		}
+
+		pr_info("Submitted %lu contexts to %s, filling %lu dwords\n",
+			ncontexts, engine->name, ndwords);
+
+		ncontexts = dw = 0;
+		list_for_each_entry(obj, &objects, st_link) {
+			unsigned int rem =
+				min_t(unsigned int, ndwords - dw, max_dwords(obj));
+
+			err = cpu_check(obj, ncontexts++, rem);
+			if (err)
+				break;
+
+			dw += rem;
+		}
+
+out_unlock:
+		if (igt_live_test_end(&t))
+			err = -EIO;
+		mutex_unlock(&i915->drm.struct_mutex);
+
+		mock_file_free(i915, file);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
+
+static int igt_shared_ctx_exec(void *arg)
+{
+	struct drm_i915_private *i915 = arg;
+	struct i915_gem_context *parent;
+	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
+	struct igt_live_test t;
+	struct drm_file *file;
+	int err = 0;
+
+	/*
+	 * Create a few different contexts with the same mm and write
+	 * through each ctx using the GPU making sure those writes end
+	 * up in the expected pages of our obj.
+	 */
+	if (!DRIVER_CAPS(i915)->has_logical_contexts)
+		return 0;
+
 	file = mock_file(i915);
 	if (IS_ERR(file))
 		return PTR_ERR(file);
 
 	mutex_lock(&i915->drm.struct_mutex);
 
+	parent = live_context(i915, file);
+	if (IS_ERR(parent)) {
+		err = PTR_ERR(parent);
+		goto out_unlock;
+	}
+
+	if (!parent->ppgtt) { /* not full-ppgtt; nothing to share */
+		err = 0;
+		goto out_unlock;
+	}
+
 	err = igt_live_test_begin(&t, i915, __func__, "");
 	if (err)
 		goto out_unlock;
 
-	ncontexts = 0;
-	ndwords = 0;
-	dw = 0;
-	while (!time_after(jiffies, end_time)) {
-		struct intel_engine_cs *engine;
-		struct i915_gem_context *ctx;
-		unsigned int id;
+	for_each_engine(engine, i915, id) {
+		unsigned long ncontexts, ndwords, dw;
+		struct drm_i915_gem_object *obj = NULL;
+		IGT_TIMEOUT(end_time);
+		LIST_HEAD(objects);
 
-		ctx = i915_gem_create_context(i915, file->driver_priv);
-		if (IS_ERR(ctx)) {
-			err = PTR_ERR(ctx);
-			goto out_unlock;
-		}
+		if (!intel_engine_can_store_dword(engine))
+			continue;
 
-		for_each_engine(engine, i915, id) {
+		dw = 0;
+		ndwords = 0;
+		ncontexts = 0;
+		while (!time_after(jiffies, end_time)) {
+			struct i915_gem_context *ctx;
 			intel_wakeref_t wakeref;
 
-			if (!engine->context_size)
-				continue; /* No logical context support in HW */
+			ctx = kernel_context(i915);
+			if (IS_ERR(ctx)) {
+				err = PTR_ERR(ctx);
+				goto out_test;
+			}
 
-			if (!intel_engine_can_store_dword(engine))
-				continue;
+			__assign_ppgtt(ctx, parent->ppgtt);
 
 			if (!obj) {
-				obj = create_test_object(ctx, file, &objects);
+				obj = create_test_object(parent, file, &objects);
 				if (IS_ERR(obj)) {
 					err = PTR_ERR(obj);
-					goto out_unlock;
+					kernel_context_close(ctx);
+					goto out_test;
 				}
 			}
 
@@ -544,35 +668,39 @@ static int igt_ctx_exec(void *arg)
 				       ndwords, dw, max_dwords(obj),
 				       engine->name, ctx->hw_id,
 				       yesno(!!ctx->ppgtt), err);
-				goto out_unlock;
+				kernel_context_close(ctx);
+				goto out_test;
 			}
 
 			if (++dw == max_dwords(obj)) {
 				obj = NULL;
 				dw = 0;
 			}
+
 			ndwords++;
+			ncontexts++;
+
+			kernel_context_close(ctx);
 		}
-		ncontexts++;
-	}
-	pr_info("Submitted %lu contexts (across %u engines), filling %lu dwords\n",
-		ncontexts, RUNTIME_INFO(i915)->num_rings, ndwords);
+		pr_info("Submitted %lu contexts to %s, filling %lu dwords\n",
+			ncontexts, engine->name, ndwords);
 
-	dw = 0;
-	list_for_each_entry(obj, &objects, st_link) {
-		unsigned int rem =
-			min_t(unsigned int, ndwords - dw, max_dwords(obj));
+		ncontexts = dw = 0;
+		list_for_each_entry(obj, &objects, st_link) {
+			unsigned int rem =
+				min_t(unsigned int, ndwords - dw, max_dwords(obj));
 
-		err = cpu_check(obj, rem);
-		if (err)
-			break;
+			err = cpu_check(obj, ncontexts++, rem);
+			if (err)
+				goto out_test;
 
-		dw += rem;
+			dw += rem;
+		}
 	}
-
-out_unlock:
+out_test:
 	if (igt_live_test_end(&t))
 		err = -EIO;
+out_unlock:
 	mutex_unlock(&i915->drm.struct_mutex);
 
 	mock_file_free(i915, file);
@@ -604,12 +732,9 @@ static struct i915_vma *rpcs_query_batch(struct i915_vma *vma)
 	*cmd++ = upper_32_bits(vma->node.start);
 	*cmd = MI_BATCH_BUFFER_END;
 
+	__i915_gem_object_flush_map(obj, 0, 64);
 	i915_gem_object_unpin_map(obj);
 
-	err = i915_gem_object_set_to_gtt_domain(obj, false);
-	if (err)
-		goto err;
-
 	vma = i915_vma_instance(obj, vma->vm, NULL);
 	if (IS_ERR(vma)) {
 		err = PTR_ERR(vma);
@@ -923,7 +1048,7 @@ __igt_ctx_sseu(struct drm_i915_private *i915,
 	       unsigned int flags)
 {
 	struct intel_sseu default_sseu = intel_device_default_sseu(i915);
-	struct intel_engine_cs *engine = i915->engine[RCS];
+	struct intel_engine_cs *engine = i915->engine[RCS0];
 	struct drm_i915_gem_object *obj;
 	struct i915_gem_context *ctx;
 	struct intel_sseu pg_sseu;
@@ -962,11 +1087,12 @@ __igt_ctx_sseu(struct drm_i915_private *i915,
 
 	mutex_lock(&i915->drm.struct_mutex);
 
-	ctx = i915_gem_create_context(i915, file->driver_priv);
+	ctx = live_context(i915, file);
 	if (IS_ERR(ctx)) {
 		ret = PTR_ERR(ctx);
 		goto out_unlock;
 	}
+	i915_gem_context_clear_bannable(ctx); /* to reset and beyond! */
 
 	obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
 	if (IS_ERR(obj)) {
@@ -1047,7 +1173,7 @@ static int igt_ctx_readonly(void *arg)
 	struct drm_i915_gem_object *obj = NULL;
 	struct i915_gem_context *ctx;
 	struct i915_hw_ppgtt *ppgtt;
-	unsigned long ndwords, dw;
+	unsigned long idx, ndwords, dw;
 	struct igt_live_test t;
 	struct drm_file *file;
 	I915_RND_STATE(prng);
@@ -1071,7 +1197,7 @@ static int igt_ctx_readonly(void *arg)
 	if (err)
 		goto out_unlock;
 
-	ctx = i915_gem_create_context(i915, file->driver_priv);
+	ctx = live_context(i915, file);
 	if (IS_ERR(ctx)) {
 		err = PTR_ERR(ctx);
 		goto out_unlock;
@@ -1125,9 +1251,10 @@ static int igt_ctx_readonly(void *arg)
 		}
 	}
 	pr_info("Submitted %lu dwords (across %u engines)\n",
-		ndwords, RUNTIME_INFO(i915)->num_rings);
+		ndwords, RUNTIME_INFO(i915)->num_engines);
 
 	dw = 0;
+	idx = 0;
 	list_for_each_entry(obj, &objects, st_link) {
 		unsigned int rem =
 			min_t(unsigned int, ndwords - dw, max_dwords(obj));
@@ -1137,7 +1264,7 @@ static int igt_ctx_readonly(void *arg)
 		if (i915_gem_object_is_readonly(obj))
 			num_writes = 0;
 
-		err = cpu_check(obj, num_writes);
+		err = cpu_check(obj, idx++, num_writes);
 		if (err)
 			break;
 
@@ -1201,12 +1328,9 @@ static int write_to_scratch(struct i915_gem_context *ctx,
 	}
 	*cmd++ = value;
 	*cmd = MI_BATCH_BUFFER_END;
+	__i915_gem_object_flush_map(obj, 0, 64);
 	i915_gem_object_unpin_map(obj);
 
-	err = i915_gem_object_set_to_gtt_domain(obj, false);
-	if (err)
-		goto err;
-
 	vma = i915_vma_instance(obj, &ctx->ppgtt->vm, NULL);
 	if (IS_ERR(vma)) {
 		err = PTR_ERR(vma);
@@ -1298,11 +1422,9 @@ static int read_from_scratch(struct i915_gem_context *ctx,
 		*cmd++ = result;
 	}
 	*cmd = MI_BATCH_BUFFER_END;
-	i915_gem_object_unpin_map(obj);
 
-	err = i915_gem_object_set_to_gtt_domain(obj, false);
-	if (err)
-		goto err;
+	i915_gem_object_flush_map(obj);
+	i915_gem_object_unpin_map(obj);
 
 	vma = i915_vma_instance(obj, &ctx->ppgtt->vm, NULL);
 	if (IS_ERR(vma)) {
@@ -1396,13 +1518,13 @@ static int igt_vm_isolation(void *arg)
 	if (err)
 		goto out_unlock;
 
-	ctx_a = i915_gem_create_context(i915, file->driver_priv);
+	ctx_a = live_context(i915, file);
 	if (IS_ERR(ctx_a)) {
 		err = PTR_ERR(ctx_a);
 		goto out_unlock;
 	}
 
-	ctx_b = i915_gem_create_context(i915, file->driver_priv);
+	ctx_b = live_context(i915, file);
 	if (IS_ERR(ctx_b)) {
 		err = PTR_ERR(ctx_b);
 		goto out_unlock;
@@ -1432,7 +1554,7 @@ static int igt_vm_isolation(void *arg)
 
 			div64_u64_rem(i915_prandom_u64_state(&prng),
 				      vm_total, &offset);
-			offset &= ~sizeof(u32);
+			offset &= -sizeof(u32);
 			offset += I915_GTT_PAGE_SIZE;
 
 			err = write_to_scratch(ctx_a, engine,
@@ -1458,7 +1580,7 @@ static int igt_vm_isolation(void *arg)
 		count += this;
 	}
 	pr_info("Checked %lu scratch offsets across %d engines\n",
-		count, RUNTIME_INFO(i915)->num_rings);
+		count, RUNTIME_INFO(i915)->num_engines);
 
 out_rpm:
 	intel_runtime_pm_put(i915, wakeref);
@@ -1472,10 +1594,10 @@ out_unlock:
 }
 
 static __maybe_unused const char *
-__engine_name(struct drm_i915_private *i915, unsigned int engines)
+__engine_name(struct drm_i915_private *i915, intel_engine_mask_t engines)
 {
 	struct intel_engine_cs *engine;
-	unsigned int tmp;
+	intel_engine_mask_t tmp;
 
 	if (engines == ALL_ENGINES)
 		return "all";
@@ -1488,67 +1610,60 @@ __engine_name(struct drm_i915_private *i915, unsigned int engines)
 
 static int __igt_switch_to_kernel_context(struct drm_i915_private *i915,
 					  struct i915_gem_context *ctx,
-					  unsigned int engines)
+					  intel_engine_mask_t engines)
 {
 	struct intel_engine_cs *engine;
-	unsigned int tmp;
-	int err;
+	intel_engine_mask_t tmp;
+	int pass;
 
 	GEM_TRACE("Testing %s\n", __engine_name(i915, engines));
-	for_each_engine_masked(engine, i915, engines, tmp) {
-		struct i915_request *rq;
+	for (pass = 0; pass < 4; pass++) { /* Once busy; once idle; repeat */
+		bool from_idle = pass & 1;
+		int err;
 
-		rq = i915_request_alloc(engine, ctx);
-		if (IS_ERR(rq))
-			return PTR_ERR(rq);
+		if (!from_idle) {
+			for_each_engine_masked(engine, i915, engines, tmp) {
+				struct i915_request *rq;
 
-		i915_request_add(rq);
-	}
+				rq = i915_request_alloc(engine, ctx);
+				if (IS_ERR(rq))
+					return PTR_ERR(rq);
 
-	err = i915_gem_switch_to_kernel_context(i915);
-	if (err)
-		return err;
-
-	for_each_engine_masked(engine, i915, engines, tmp) {
-		if (!engine_has_kernel_context_barrier(engine)) {
-			pr_err("kernel context not last on engine %s!\n",
-			       engine->name);
-			return -EINVAL;
+				i915_request_add(rq);
+			}
 		}
-	}
 
-	err = i915_gem_wait_for_idle(i915,
-				     I915_WAIT_LOCKED,
-				     MAX_SCHEDULE_TIMEOUT);
-	if (err)
-		return err;
+		err = i915_gem_switch_to_kernel_context(i915,
+							i915->gt.active_engines);
+		if (err)
+			return err;
 
-	GEM_BUG_ON(i915->gt.active_requests);
-	for_each_engine_masked(engine, i915, engines, tmp) {
-		if (engine->last_retired_context->gem_context != i915->kernel_context) {
-			pr_err("engine %s not idling in kernel context!\n",
-			       engine->name);
+		if (!from_idle) {
+			err = i915_gem_wait_for_idle(i915,
+						     I915_WAIT_LOCKED,
+						     MAX_SCHEDULE_TIMEOUT);
+			if (err)
+				return err;
+		}
+
+		if (i915->gt.active_requests) {
+			pr_err("%d active requests remain after switching to kernel context, pass %d (%s) on %s engine%s\n",
+			       i915->gt.active_requests,
+			       pass, from_idle ? "idle" : "busy",
+			       __engine_name(i915, engines),
+			       is_power_of_2(engines) ? "" : "s");
 			return -EINVAL;
 		}
-	}
 
-	err = i915_gem_switch_to_kernel_context(i915);
-	if (err)
-		return err;
+		/* XXX Bonus points for proving we are the kernel context! */
 
-	if (i915->gt.active_requests) {
-		pr_err("switch-to-kernel-context emitted %d requests even though it should already be idling in the kernel context\n",
-		       i915->gt.active_requests);
-		return -EINVAL;
+		mutex_unlock(&i915->drm.struct_mutex);
+		drain_delayed_work(&i915->gt.idle_work);
+		mutex_lock(&i915->drm.struct_mutex);
 	}
 
-	for_each_engine_masked(engine, i915, engines, tmp) {
-		if (!intel_engine_has_kernel_context(engine)) {
-			pr_err("kernel context not last on engine %s!\n",
-			       engine->name);
-			return -EINVAL;
-		}
-	}
+	if (igt_flush_test(i915, I915_WAIT_LOCKED))
+		return -EIO;
 
 	return 0;
 }
@@ -1592,8 +1707,6 @@ static int igt_switch_to_kernel_context(void *arg)
 
 out_unlock:
 	GEM_TRACE_DUMP_ON(err);
-	if (igt_flush_test(i915, I915_WAIT_LOCKED))
-		err = -EIO;
 
 	intel_runtime_pm_put(i915, wakeref);
 	mutex_unlock(&i915->drm.struct_mutex);
@@ -1602,10 +1715,117 @@ out_unlock:
 	return err;
 }
 
+static void mock_barrier_task(void *data)
+{
+	unsigned int *counter = data;
+
+	++*counter;
+}
+
+static int mock_context_barrier(void *arg)
+{
+#undef pr_fmt
+#define pr_fmt(x) "context_barrier_task():" # x
+	struct drm_i915_private *i915 = arg;
+	struct i915_gem_context *ctx;
+	struct i915_request *rq;
+	intel_wakeref_t wakeref;
+	unsigned int counter;
+	int err;
+
+	/*
+	 * The context barrier provides us with a callback after it emits
+	 * a request; useful for retiring old state after loading new.
+	 */
+
+	mutex_lock(&i915->drm.struct_mutex);
+
+	ctx = mock_context(i915, "mock");
+	if (!ctx) {
+		err = -ENOMEM;
+		goto unlock;
+	}
+
+	counter = 0;
+	err = context_barrier_task(ctx, 0,
+				   NULL, mock_barrier_task, &counter);
+	if (err) {
+		pr_err("Failed at line %d, err=%d\n", __LINE__, err);
+		goto out;
+	}
+	if (counter == 0) {
+		pr_err("Did not retire immediately with 0 engines\n");
+		err = -EINVAL;
+		goto out;
+	}
+
+	counter = 0;
+	err = context_barrier_task(ctx, ALL_ENGINES,
+				   NULL, mock_barrier_task, &counter);
+	if (err) {
+		pr_err("Failed at line %d, err=%d\n", __LINE__, err);
+		goto out;
+	}
+	if (counter == 0) {
+		pr_err("Did not retire immediately for all inactive engines\n");
+		err = -EINVAL;
+		goto out;
+	}
+
+	rq = ERR_PTR(-ENODEV);
+	with_intel_runtime_pm(i915, wakeref)
+		rq = i915_request_alloc(i915->engine[RCS0], ctx);
+	if (IS_ERR(rq)) {
+		pr_err("Request allocation failed!\n");
+		goto out;
+	}
+	i915_request_add(rq);
+	GEM_BUG_ON(list_empty(&ctx->active_engines));
+
+	counter = 0;
+	context_barrier_inject_fault = BIT(RCS0);
+	err = context_barrier_task(ctx, ALL_ENGINES,
+				   NULL, mock_barrier_task, &counter);
+	context_barrier_inject_fault = 0;
+	if (err == -ENXIO)
+		err = 0;
+	else
+		pr_err("Did not hit fault injection!\n");
+	if (counter != 0) {
+		pr_err("Invoked callback on error!\n");
+		err = -EIO;
+	}
+	if (err)
+		goto out;
+
+	counter = 0;
+	err = context_barrier_task(ctx, ALL_ENGINES,
+				   NULL, mock_barrier_task, &counter);
+	if (err) {
+		pr_err("Failed at line %d, err=%d\n", __LINE__, err);
+		goto out;
+	}
+	mock_device_flush(i915);
+	if (counter == 0) {
+		pr_err("Did not retire on each active engines\n");
+		err = -EINVAL;
+		goto out;
+	}
+
+out:
+	mock_context_close(ctx);
+unlock:
+	mutex_unlock(&i915->drm.struct_mutex);
+	return err;
+#undef pr_fmt
+#define pr_fmt(x) x
+}
+
 int i915_gem_context_mock_selftests(void)
 {
 	static const struct i915_subtest tests[] = {
 		SUBTEST(igt_switch_to_kernel_context),
+		SUBTEST(mock_context_barrier),
 	};
 	struct drm_i915_private *i915;
 	int err;
@@ -1628,10 +1848,11 @@ int i915_gem_context_live_selftests(struct drm_i915_private *dev_priv)
 		SUBTEST(igt_ctx_exec),
 		SUBTEST(igt_ctx_readonly),
 		SUBTEST(igt_ctx_sseu),
+		SUBTEST(igt_shared_ctx_exec),
 		SUBTEST(igt_vm_isolation),
 	};
 
-	if (i915_terminally_wedged(&dev_priv->gpu_error))
+	if (i915_terminally_wedged(dev_priv))
 		return 0;
 
 	return i915_subtests(tests, dev_priv);
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_dmabuf.c b/drivers/gpu/drm/i915/selftests/i915_gem_dmabuf.c
index a7055b12e53c..2b943ee246c9 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_dmabuf.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_dmabuf.c
@@ -315,6 +315,7 @@ static int igt_dmabuf_export_kmap(void *arg)
 		goto err;
 	}
 	memset(ptr + PAGE_SIZE, 0xaa, PAGE_SIZE);
+	i915_gem_object_flush_map(obj);
 	i915_gem_object_unpin_map(obj);
 
 	ptr = dma_buf_kmap(dmabuf, 1);
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_evict.c b/drivers/gpu/drm/i915/selftests/i915_gem_evict.c
index b9b0ea4e2404..89766688e420 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_evict.c
@@ -274,7 +274,7 @@ static int igt_evict_for_cache_color(void *arg)
 		err = PTR_ERR(obj);
 		goto cleanup;
 	}
-	i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
+	i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
 	quirk_add(obj, &objects);
 
 	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
@@ -290,7 +290,7 @@ static int igt_evict_for_cache_color(void *arg)
 		err = PTR_ERR(obj);
 		goto cleanup;
 	}
-	i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
+	i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
 	quirk_add(obj, &objects);
 
 	/* Neighbouring; same colour - should fit */
@@ -547,7 +547,7 @@ int i915_gem_evict_live_selftests(struct drm_i915_private *i915)
 		SUBTEST(igt_evict_contexts),
 	};
 
-	if (i915_terminally_wedged(&i915->gpu_error))
+	if (i915_terminally_wedged(i915))
 		return 0;
 
 	return i915_subtests(tests, i915);
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
index 3850ef4a5ec8..9cca66e4420a 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
@@ -120,7 +120,7 @@ fake_dma_object(struct drm_i915_private *i915, u64 size)
 	if (overflows_type(size, obj->base.size))
 		return ERR_PTR(-E2BIG);
 
-	obj = i915_gem_object_alloc(i915);
+	obj = i915_gem_object_alloc();
 	if (!obj)
 		goto err;
 
@@ -1010,7 +1010,7 @@ static int exercise_ppgtt(struct drm_i915_private *dev_priv,
 		return PTR_ERR(file);
 
 	mutex_lock(&dev_priv->drm.struct_mutex);
-	ppgtt = i915_ppgtt_create(dev_priv, file->driver_priv);
+	ppgtt = i915_ppgtt_create(dev_priv);
 	if (IS_ERR(ppgtt)) {
 		err = PTR_ERR(ppgtt);
 		goto out_unlock;
@@ -1020,7 +1020,6 @@ static int exercise_ppgtt(struct drm_i915_private *dev_priv,
 
 	err = func(dev_priv, &ppgtt->vm, 0, ppgtt->vm.total, end_time);
 
-	i915_ppgtt_close(&ppgtt->vm);
 	i915_ppgtt_put(ppgtt);
 out_unlock:
 	mutex_unlock(&dev_priv->drm.struct_mutex);
@@ -1681,25 +1680,31 @@ int i915_gem_gtt_mock_selftests(void)
 		SUBTEST(igt_gtt_insert),
 	};
 	struct drm_i915_private *i915;
-	struct i915_ggtt ggtt;
+	struct i915_ggtt *ggtt;
 	int err;
 
 	i915 = mock_gem_device();
 	if (!i915)
 		return -ENOMEM;
 
-	mock_init_ggtt(i915, &ggtt);
+	ggtt = kmalloc(sizeof(*ggtt), GFP_KERNEL);
+	if (!ggtt) {
+		err = -ENOMEM;
+		goto out_put;
+	}
+	mock_init_ggtt(i915, ggtt);
 
 	mutex_lock(&i915->drm.struct_mutex);
-	err = i915_subtests(tests, &ggtt);
+	err = i915_subtests(tests, ggtt);
 	mock_device_flush(i915);
 	mutex_unlock(&i915->drm.struct_mutex);
 
 	i915_gem_drain_freed_objects(i915);
 
-	mock_fini_ggtt(&ggtt);
+	mock_fini_ggtt(ggtt);
+	kfree(ggtt);
+out_put:
 	drm_dev_put(&i915->drm);
-
 	return err;
 }
 
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_object.c b/drivers/gpu/drm/i915/selftests/i915_gem_object.c
index 395ae878e0f7..971148fbe6f5 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_object.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_object.c
@@ -468,7 +468,7 @@ static int make_obj_busy(struct drm_i915_gem_object *obj)
 	if (err)
 		return err;
 
-	rq = i915_request_alloc(i915->engine[RCS], i915->kernel_context);
+	rq = i915_request_alloc(i915->engine[RCS0], i915->kernel_context);
 	if (IS_ERR(rq)) {
 		i915_vma_unpin(vma);
 		return PTR_ERR(rq);
@@ -583,7 +583,7 @@ static int igt_mmap_offset_exhaustion(void *arg)
 	for (loop = 0; loop < 3; loop++) {
 		intel_wakeref_t wakeref;
 
-		if (i915_terminally_wedged(&i915->gpu_error))
+		if (i915_terminally_wedged(i915))
 			break;
 
 		obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
diff --git a/drivers/gpu/drm/i915/selftests/i915_request.c b/drivers/gpu/drm/i915/selftests/i915_request.c
index 6733dc5b6b4c..e6ffe2240126 100644
--- a/drivers/gpu/drm/i915/selftests/i915_request.c
+++ b/drivers/gpu/drm/i915/selftests/i915_request.c
@@ -42,7 +42,7 @@ static int igt_add_request(void *arg)
 	/* Basic preliminary test to create a request and let it loose! */
 
 	mutex_lock(&i915->drm.struct_mutex);
-	request = mock_request(i915->engine[RCS],
+	request = mock_request(i915->engine[RCS0],
 			       i915->kernel_context,
 			       HZ / 10);
 	if (!request)
@@ -66,7 +66,7 @@ static int igt_wait_request(void *arg)
 	/* Submit a request, then wait upon it */
 
 	mutex_lock(&i915->drm.struct_mutex);
-	request = mock_request(i915->engine[RCS], i915->kernel_context, T);
+	request = mock_request(i915->engine[RCS0], i915->kernel_context, T);
 	if (!request) {
 		err = -ENOMEM;
 		goto out_unlock;
@@ -136,19 +136,17 @@ static int igt_fence_wait(void *arg)
 	/* Submit a request, treat it as a fence and wait upon it */
 
 	mutex_lock(&i915->drm.struct_mutex);
-	request = mock_request(i915->engine[RCS], i915->kernel_context, T);
+	request = mock_request(i915->engine[RCS0], i915->kernel_context, T);
 	if (!request) {
 		err = -ENOMEM;
 		goto out_locked;
 	}
-	mutex_unlock(&i915->drm.struct_mutex); /* safe as we are single user */
 
 	if (dma_fence_wait_timeout(&request->fence, false, T) != -ETIME) {
 		pr_err("fence wait success before submit (expected timeout)!\n");
-		goto out_device;
+		goto out_locked;
 	}
 
-	mutex_lock(&i915->drm.struct_mutex);
 	i915_request_add(request);
 	mutex_unlock(&i915->drm.struct_mutex);
 
@@ -195,7 +193,7 @@ static int igt_request_rewind(void *arg)
 
 	mutex_lock(&i915->drm.struct_mutex);
 	ctx[0] = mock_context(i915, "A");
-	request = mock_request(i915->engine[RCS], ctx[0], 2 * HZ);
+	request = mock_request(i915->engine[RCS0], ctx[0], 2 * HZ);
 	if (!request) {
 		err = -ENOMEM;
 		goto err_context_0;
@@ -205,7 +203,7 @@ static int igt_request_rewind(void *arg)
 	i915_request_add(request);
 
 	ctx[1] = mock_context(i915, "B");
-	vip = mock_request(i915->engine[RCS], ctx[1], 0);
+	vip = mock_request(i915->engine[RCS0], ctx[1], 0);
 	if (!vip) {
 		err = -ENOMEM;
 		goto err_context_1;
@@ -226,8 +224,7 @@ static int igt_request_rewind(void *arg)
 	mutex_unlock(&i915->drm.struct_mutex);
 
 	if (i915_request_wait(vip, 0, HZ) == -ETIME) {
-		pr_err("timed out waiting for high priority request, vip.seqno=%d, current seqno=%d\n",
-		       vip->global_seqno, intel_engine_get_seqno(i915->engine[RCS]));
+		pr_err("timed out waiting for high priority request\n");
 		goto err;
 	}
 
@@ -418,7 +415,7 @@ static int mock_breadcrumbs_smoketest(void *arg)
 {
 	struct drm_i915_private *i915 = arg;
 	struct smoketest t = {
-		.engine = i915->engine[RCS],
+		.engine = i915->engine[RCS0],
 		.ncontexts = 1024,
 		.max_batch = 1024,
 		.request_alloc = __mock_request_alloc
@@ -622,13 +619,11 @@ static struct i915_vma *empty_batch(struct drm_i915_private *i915)
 	}
 
 	*cmd = MI_BATCH_BUFFER_END;
-	i915_gem_chipset_flush(i915);
 
+	__i915_gem_object_flush_map(obj, 0, 64);
 	i915_gem_object_unpin_map(obj);
 
-	err = i915_gem_object_set_to_gtt_domain(obj, false);
-	if (err)
-		goto err;
+	i915_gem_chipset_flush(i915);
 
 	vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
 	if (IS_ERR(vma)) {
@@ -780,10 +775,6 @@ static struct i915_vma *recursive_batch(struct drm_i915_private *i915)
 	if (err)
 		goto err;
 
-	err = i915_gem_object_set_to_wc_domain(obj, true);
-	if (err)
-		goto err;
-
 	cmd = i915_gem_object_pin_map(obj, I915_MAP_WC);
 	if (IS_ERR(cmd)) {
 		err = PTR_ERR(cmd);
@@ -802,10 +793,12 @@ static struct i915_vma *recursive_batch(struct drm_i915_private *i915)
 		*cmd++ = lower_32_bits(vma->node.start);
 	}
 	*cmd++ = MI_BATCH_BUFFER_END; /* terminate early in case of error */
-	i915_gem_chipset_flush(i915);
 
+	__i915_gem_object_flush_map(obj, 0, 64);
 	i915_gem_object_unpin_map(obj);
 
+	i915_gem_chipset_flush(i915);
+
 	return vma;
 
 err:
@@ -1219,7 +1212,7 @@ out_flush:
 		num_fences += atomic_long_read(&t[id].num_fences);
 	}
 	pr_info("Completed %lu waits for %lu fences across %d engines and %d cpus\n",
-		num_waits, num_fences, RUNTIME_INFO(i915)->num_rings, ncpus);
+		num_waits, num_fences, RUNTIME_INFO(i915)->num_engines, ncpus);
 
 	mutex_lock(&i915->drm.struct_mutex);
 	ret = igt_live_test_end(&live) ?: ret;
@@ -1246,7 +1239,7 @@ int i915_request_live_selftests(struct drm_i915_private *i915)
 		SUBTEST(live_breadcrumbs_smoketest),
 	};
 
-	if (i915_terminally_wedged(&i915->gpu_error))
+	if (i915_terminally_wedged(i915))
 		return 0;
 
 	return i915_subtests(tests, i915);
diff --git a/drivers/gpu/drm/i915/selftests/i915_selftest.c b/drivers/gpu/drm/i915/selftests/i915_selftest.c
index 10ef0e636a24..b18eaefef798 100644
--- a/drivers/gpu/drm/i915/selftests/i915_selftest.c
+++ b/drivers/gpu/drm/i915/selftests/i915_selftest.c
@@ -133,7 +133,7 @@ static int __run_selftests(const char *name,
 		if (signal_pending(current))
 			return -EINTR;
 
-		pr_debug(DRIVER_NAME ": Running %s\n", st->name);
+		pr_info(DRIVER_NAME ": Running %s\n", st->name);
 		if (data)
 			err = st->live(data);
 		else
@@ -255,7 +255,7 @@ int __i915_subtests(const char *caller,
 		if (!apply_subtest_filter(caller, st->name))
 			continue;
 
-		pr_debug(DRIVER_NAME ": Running %s/%s\n", caller, st->name);
+		pr_info(DRIVER_NAME ": Running %s/%s\n", caller, st->name);
 		GEM_TRACE("Running %s/%s\n", caller, st->name);
 
 		err = st->func(data);
diff --git a/drivers/gpu/drm/i915/selftests/i915_sw_fence.c b/drivers/gpu/drm/i915/selftests/i915_sw_fence.c
index cdbc8f134e5e..cbf45d85cbff 100644
--- a/drivers/gpu/drm/i915/selftests/i915_sw_fence.c
+++ b/drivers/gpu/drm/i915/selftests/i915_sw_fence.c
@@ -571,21 +571,27 @@ static int test_timer(void *arg)
 	unsigned long target, delay;
 	struct timed_fence tf;
 
+	preempt_disable();
 	timed_fence_init(&tf, target = jiffies);
 	if (!i915_sw_fence_done(&tf.fence)) {
 		pr_err("Fence with immediate expiration not signaled\n");
 		goto err;
 	}
+	preempt_enable();
 	timed_fence_fini(&tf);
 
 	for_each_prime_number(delay, i915_selftest.timeout_jiffies/2) {
+		preempt_disable();
 		timed_fence_init(&tf, target = jiffies + delay);
 		if (i915_sw_fence_done(&tf.fence)) {
 			pr_err("Fence with future expiration (%lu jiffies) already signaled\n", delay);
 			goto err;
 		}
+		preempt_enable();
 
 		i915_sw_fence_wait(&tf.fence);
+
+		preempt_disable();
 		if (!i915_sw_fence_done(&tf.fence)) {
 			pr_err("Fence not signaled after wait\n");
 			goto err;
@@ -595,13 +601,14 @@ static int test_timer(void *arg)
 			       target, jiffies);
 			goto err;
 		}
-
+		preempt_enable();
 		timed_fence_fini(&tf);
 	}
 
 	return 0;
 
 err:
+	preempt_enable();
 	timed_fence_fini(&tf);
 	return -EINVAL;
 }
diff --git a/drivers/gpu/drm/i915/selftests/i915_timeline.c b/drivers/gpu/drm/i915/selftests/i915_timeline.c
index 12ea69b1a1e5..bd96afcadfe7 100644
--- a/drivers/gpu/drm/i915/selftests/i915_timeline.c
+++ b/drivers/gpu/drm/i915/selftests/i915_timeline.c
@@ -64,7 +64,7 @@ static int __mock_hwsp_timeline(struct mock_hwsp_freelist *state,
 		unsigned long cacheline;
 		int err;
 
-		tl = i915_timeline_create(state->i915, "mock", NULL);
+		tl = i915_timeline_create(state->i915, NULL);
 		if (IS_ERR(tl))
 			return PTR_ERR(tl);
 
@@ -476,7 +476,7 @@ checked_i915_timeline_create(struct drm_i915_private *i915)
 {
 	struct i915_timeline *tl;
 
-	tl = i915_timeline_create(i915, "live", NULL);
+	tl = i915_timeline_create(i915, NULL);
 	if (IS_ERR(tl))
 		return tl;
 
@@ -641,6 +641,118 @@ out:
 #undef NUM_TIMELINES
 }
 
+static int live_hwsp_wrap(void *arg)
+{
+	struct drm_i915_private *i915 = arg;
+	struct intel_engine_cs *engine;
+	struct i915_timeline *tl;
+	enum intel_engine_id id;
+	intel_wakeref_t wakeref;
+	int err = 0;
+
+	/*
+	 * Across a seqno wrap, we need to keep the old cacheline alive for
+	 * foreign GPU references.
+	 */
+
+	mutex_lock(&i915->drm.struct_mutex);
+	wakeref = intel_runtime_pm_get(i915);
+
+	tl = i915_timeline_create(i915, NULL);
+	if (IS_ERR(tl)) {
+		err = PTR_ERR(tl);
+		goto out_rpm;
+	}
+	if (!tl->has_initial_breadcrumb || !tl->hwsp_cacheline)
+		goto out_free;
+
+	err = i915_timeline_pin(tl);
+	if (err)
+		goto out_free;
+
+	for_each_engine(engine, i915, id) {
+		const u32 *hwsp_seqno[2];
+		struct i915_request *rq;
+		u32 seqno[2];
+
+		if (!intel_engine_can_store_dword(engine))
+			continue;
+
+		rq = i915_request_alloc(engine, i915->kernel_context);
+		if (IS_ERR(rq)) {
+			err = PTR_ERR(rq);
+			goto out;
+		}
+
+		tl->seqno = -4u;
+
+		err = i915_timeline_get_seqno(tl, rq, &seqno[0]);
+		if (err) {
+			i915_request_add(rq);
+			goto out;
+		}
+		pr_debug("seqno[0]:%08x, hwsp_offset:%08x\n",
+			 seqno[0], tl->hwsp_offset);
+
+		err = emit_ggtt_store_dw(rq, tl->hwsp_offset, seqno[0]);
+		if (err) {
+			i915_request_add(rq);
+			goto out;
+		}
+		hwsp_seqno[0] = tl->hwsp_seqno;
+
+		err = i915_timeline_get_seqno(tl, rq, &seqno[1]);
+		if (err) {
+			i915_request_add(rq);
+			goto out;
+		}
+		pr_debug("seqno[1]:%08x, hwsp_offset:%08x\n",
+			 seqno[1], tl->hwsp_offset);
+
+		err = emit_ggtt_store_dw(rq, tl->hwsp_offset, seqno[1]);
+		if (err) {
+			i915_request_add(rq);
+			goto out;
+		}
+		hwsp_seqno[1] = tl->hwsp_seqno;
+
+		/* With wrap should come a new hwsp */
+		GEM_BUG_ON(seqno[1] >= seqno[0]);
+		GEM_BUG_ON(hwsp_seqno[0] == hwsp_seqno[1]);
+
+		i915_request_add(rq);
+
+		if (i915_request_wait(rq, I915_WAIT_LOCKED, HZ / 5) < 0) {
+			pr_err("Wait for timeline writes timed out!\n");
+			err = -EIO;
+			goto out;
+		}
+
+		if (*hwsp_seqno[0] != seqno[0] || *hwsp_seqno[1] != seqno[1]) {
+			pr_err("Bad timeline values: found (%x, %x), expected (%x, %x)\n",
+			       *hwsp_seqno[0], *hwsp_seqno[1],
+			       seqno[0], seqno[1]);
+			err = -EINVAL;
+			goto out;
+		}
+
+		i915_retire_requests(i915); /* recycle HWSP */
+	}
+
+out:
+	if (igt_flush_test(i915, I915_WAIT_LOCKED))
+		err = -EIO;
+
+	i915_timeline_unpin(tl);
+out_free:
+	i915_timeline_put(tl);
+out_rpm:
+	intel_runtime_pm_put(i915, wakeref);
+	mutex_unlock(&i915->drm.struct_mutex);
+
+	return err;
+}
+
 static int live_hwsp_recycle(void *arg)
 {
 	struct drm_i915_private *i915 = arg;
@@ -723,7 +835,11 @@ int i915_timeline_live_selftests(struct drm_i915_private *i915)
 		SUBTEST(live_hwsp_recycle),
 		SUBTEST(live_hwsp_engine),
 		SUBTEST(live_hwsp_alternate),
+		SUBTEST(live_hwsp_wrap),
 	};
 
+	if (i915_terminally_wedged(i915))
+		return 0;
+
 	return i915_subtests(tests, i915);
 }
diff --git a/drivers/gpu/drm/i915/selftests/i915_vma.c b/drivers/gpu/drm/i915/selftests/i915_vma.c
index cf1de82741fa..fc594b030f5a 100644
--- a/drivers/gpu/drm/i915/selftests/i915_vma.c
+++ b/drivers/gpu/drm/i915/selftests/i915_vma.c
@@ -725,24 +725,30 @@ int i915_vma_mock_selftests(void)
 		SUBTEST(igt_vma_partial),
 	};
 	struct drm_i915_private *i915;
-	struct i915_ggtt ggtt;
+	struct i915_ggtt *ggtt;
 	int err;
 
 	i915 = mock_gem_device();
 	if (!i915)
 		return -ENOMEM;
 
-	mock_init_ggtt(i915, &ggtt);
+	ggtt = kmalloc(sizeof(*ggtt), GFP_KERNEL);
+	if (!ggtt) {
+		err = -ENOMEM;
+		goto out_put;
+	}
+	mock_init_ggtt(i915, ggtt);
 
 	mutex_lock(&i915->drm.struct_mutex);
-	err = i915_subtests(tests, &ggtt);
+	err = i915_subtests(tests, ggtt);
 	mock_device_flush(i915);
 	mutex_unlock(&i915->drm.struct_mutex);
 
 	i915_gem_drain_freed_objects(i915);
 
-	mock_fini_ggtt(&ggtt);
+	mock_fini_ggtt(ggtt);
+	kfree(ggtt);
+out_put:
 	drm_dev_put(&i915->drm);
-
 	return err;
 }
diff --git a/drivers/gpu/drm/i915/selftests/igt_flush_test.c b/drivers/gpu/drm/i915/selftests/igt_flush_test.c
index af66e3d4e23a..94aee4071a66 100644
--- a/drivers/gpu/drm/i915/selftests/igt_flush_test.c
+++ b/drivers/gpu/drm/i915/selftests/igt_flush_test.c
@@ -14,7 +14,7 @@ int igt_flush_test(struct drm_i915_private *i915, unsigned int flags)
 	cond_resched();
 
 	if (flags & I915_WAIT_LOCKED &&
-	    i915_gem_switch_to_kernel_context(i915)) {
+	    i915_gem_switch_to_kernel_context(i915, i915->gt.active_engines)) {
 		pr_err("Failed to switch back to kernel context; declaring wedged\n");
 		i915_gem_set_wedged(i915);
 	}
@@ -29,5 +29,5 @@ int igt_flush_test(struct drm_i915_private *i915, unsigned int flags)
 		i915_gem_set_wedged(i915);
 	}
 
-	return i915_terminally_wedged(&i915->gpu_error) ? -EIO : 0;
+	return i915_terminally_wedged(i915);
 }
diff --git a/drivers/gpu/drm/i915/selftests/igt_spinner.c b/drivers/gpu/drm/i915/selftests/igt_spinner.c
index 9ebd9225684e..16890dfe74c0 100644
--- a/drivers/gpu/drm/i915/selftests/igt_spinner.c
+++ b/drivers/gpu/drm/i915/selftests/igt_spinner.c
@@ -29,7 +29,7 @@ int igt_spinner_init(struct igt_spinner *spin, struct drm_i915_private *i915)
 		goto err_hws;
 	}
 
-	i915_gem_object_set_cache_level(spin->hws, I915_CACHE_LLC);
+	i915_gem_object_set_cache_coherency(spin->hws, I915_CACHE_LLC);
 	vaddr = i915_gem_object_pin_map(spin->hws, I915_MAP_WB);
 	if (IS_ERR(vaddr)) {
 		err = PTR_ERR(vaddr);
@@ -144,6 +144,13 @@ igt_spinner_create_request(struct igt_spinner *spin,
 
 	i915_gem_chipset_flush(spin->i915);
 
+	if (engine->emit_init_breadcrumb &&
+	    rq->timeline->has_initial_breadcrumb) {
+		err = engine->emit_init_breadcrumb(rq);
+		if (err)
+			goto cancel_rq;
+	}
+
 	err = engine->emit_bb_start(rq, vma->node.start, PAGE_SIZE, 0);
 
 cancel_rq:
diff --git a/drivers/gpu/drm/i915/selftests/intel_guc.c b/drivers/gpu/drm/i915/selftests/intel_guc.c
index c5e0a0e98fcb..b05a21eaa8f4 100644
--- a/drivers/gpu/drm/i915/selftests/intel_guc.c
+++ b/drivers/gpu/drm/i915/selftests/intel_guc.c
@@ -111,7 +111,7 @@ static int validate_client(struct intel_guc_client *client,
 			dev_priv->preempt_context : dev_priv->kernel_context;
 
 	if (client->owner != ctx_owner ||
-	    client->engines != INTEL_INFO(dev_priv)->ring_mask ||
+	    client->engines != INTEL_INFO(dev_priv)->engine_mask ||
 	    client->priority != client_priority ||
 	    client->doorbell_id == GUC_DOORBELL_INVALID)
 		return -EINVAL;
@@ -261,7 +261,7 @@ static int igt_guc_doorbells(void *arg)
 
 	for (i = 0; i < ATTEMPTS; i++) {
 		clients[i] = guc_client_alloc(dev_priv,
-					      INTEL_INFO(dev_priv)->ring_mask,
+					      INTEL_INFO(dev_priv)->engine_mask,
 					      i % GUC_CLIENT_PRIORITY_NUM,
 					      dev_priv->kernel_context);
 
diff --git a/drivers/gpu/drm/i915/selftests/intel_hangcheck.c b/drivers/gpu/drm/i915/selftests/intel_hangcheck.c
index 7b6f3bea9ef8..050bd1e19e02 100644
--- a/drivers/gpu/drm/i915/selftests/intel_hangcheck.c
+++ b/drivers/gpu/drm/i915/selftests/intel_hangcheck.c
@@ -56,6 +56,8 @@ static int hang_init(struct hang *h, struct drm_i915_private *i915)
 	if (IS_ERR(h->ctx))
 		return PTR_ERR(h->ctx);
 
+	GEM_BUG_ON(i915_gem_context_is_bannable(h->ctx));
+
 	h->hws = i915_gem_object_create_internal(i915, PAGE_SIZE);
 	if (IS_ERR(h->hws)) {
 		err = PTR_ERR(h->hws);
@@ -68,7 +70,7 @@ static int hang_init(struct hang *h, struct drm_i915_private *i915)
 		goto err_hws;
 	}
 
-	i915_gem_object_set_cache_level(h->hws, I915_CACHE_LLC);
+	i915_gem_object_set_cache_coherency(h->hws, I915_CACHE_LLC);
 	vaddr = i915_gem_object_pin_map(h->hws, I915_MAP_WB);
 	if (IS_ERR(vaddr)) {
 		err = PTR_ERR(vaddr);
@@ -242,6 +244,12 @@ hang_create_request(struct hang *h, struct intel_engine_cs *engine)
 	*batch++ = MI_BATCH_BUFFER_END; /* not reached */
 	i915_gem_chipset_flush(h->i915);
 
+	if (rq->engine->emit_init_breadcrumb) {
+		err = rq->engine->emit_init_breadcrumb(rq);
+		if (err)
+			goto cancel_rq;
+	}
+
 	flags = 0;
 	if (INTEL_GEN(vm->i915) <= 5)
 		flags |= I915_DISPATCH_SECURE;
@@ -334,7 +342,7 @@ static int igt_hang_sanitycheck(void *arg)
 			timeout = i915_request_wait(rq,
 						    I915_WAIT_LOCKED,
 						    MAX_SCHEDULE_TIMEOUT);
-		if (i915_terminally_wedged(&i915->gpu_error))
+		if (i915_reset_failed(i915))
 			timeout = -EIO;
 
 		i915_request_put(rq);
@@ -375,7 +383,7 @@ static int igt_global_reset(void *arg)
 
 	igt_global_reset_unlock(i915);
 
-	if (i915_terminally_wedged(&i915->gpu_error))
+	if (i915_reset_failed(i915))
 		err = -EIO;
 
 	return err;
@@ -393,15 +401,13 @@ static int igt_wedged_reset(void *arg)
 
 	i915_gem_set_wedged(i915);
 
-	mutex_lock(&i915->drm.struct_mutex);
-	GEM_BUG_ON(!i915_terminally_wedged(&i915->gpu_error));
+	GEM_BUG_ON(!i915_reset_failed(i915));
 	i915_reset(i915, ALL_ENGINES, NULL);
-	mutex_unlock(&i915->drm.struct_mutex);
 
 	intel_runtime_pm_put(i915, wakeref);
 	igt_global_reset_unlock(i915);
 
-	return i915_terminally_wedged(&i915->gpu_error) ? -EIO : 0;
+	return i915_reset_failed(i915) ? -EIO : 0;
 }
 
 static bool wait_for_idle(struct intel_engine_cs *engine)
@@ -409,6 +415,222 @@ static bool wait_for_idle(struct intel_engine_cs *engine)
 	return wait_for(intel_engine_is_idle(engine), IGT_IDLE_TIMEOUT) == 0;
 }
 
+static int igt_reset_nop(void *arg)
+{
+	struct drm_i915_private *i915 = arg;
+	struct intel_engine_cs *engine;
+	struct i915_gem_context *ctx;
+	unsigned int reset_count, count;
+	enum intel_engine_id id;
+	intel_wakeref_t wakeref;
+	struct drm_file *file;
+	IGT_TIMEOUT(end_time);
+	int err = 0;
+
+	/* Check that we can reset during non-user portions of requests */
+
+	file = mock_file(i915);
+	if (IS_ERR(file))
+		return PTR_ERR(file);
+
+	mutex_lock(&i915->drm.struct_mutex);
+	ctx = live_context(i915, file);
+	mutex_unlock(&i915->drm.struct_mutex);
+	if (IS_ERR(ctx)) {
+		err = PTR_ERR(ctx);
+		goto out;
+	}
+
+	i915_gem_context_clear_bannable(ctx);
+	wakeref = intel_runtime_pm_get(i915);
+	reset_count = i915_reset_count(&i915->gpu_error);
+	count = 0;
+	do {
+		mutex_lock(&i915->drm.struct_mutex);
+		for_each_engine(engine, i915, id) {
+			int i;
+
+			for (i = 0; i < 16; i++) {
+				struct i915_request *rq;
+
+				rq = i915_request_alloc(engine, ctx);
+				if (IS_ERR(rq)) {
+					err = PTR_ERR(rq);
+					break;
+				}
+
+				i915_request_add(rq);
+			}
+		}
+		mutex_unlock(&i915->drm.struct_mutex);
+
+		igt_global_reset_lock(i915);
+		i915_reset(i915, ALL_ENGINES, NULL);
+		igt_global_reset_unlock(i915);
+		if (i915_reset_failed(i915)) {
+			err = -EIO;
+			break;
+		}
+
+		if (i915_reset_count(&i915->gpu_error) !=
+		    reset_count + ++count) {
+			pr_err("Full GPU reset not recorded!\n");
+			err = -EINVAL;
+			break;
+		}
+
+		if (!i915_reset_flush(i915)) {
+			struct drm_printer p =
+				drm_info_printer(i915->drm.dev);
+
+			pr_err("%s failed to idle after reset\n",
+			       engine->name);
+			intel_engine_dump(engine, &p,
+					  "%s\n", engine->name);
+
+			err = -EIO;
+			break;
+		}
+
+		err = igt_flush_test(i915, 0);
+		if (err)
+			break;
+	} while (time_before(jiffies, end_time));
+	pr_info("%s: %d resets\n", __func__, count);
+
+	mutex_lock(&i915->drm.struct_mutex);
+	err = igt_flush_test(i915, I915_WAIT_LOCKED);
+	mutex_unlock(&i915->drm.struct_mutex);
+
+	intel_runtime_pm_put(i915, wakeref);
+
+out:
+	mock_file_free(i915, file);
+	if (i915_reset_failed(i915))
+		err = -EIO;
+	return err;
+}
+
+static int igt_reset_nop_engine(void *arg)
+{
+	struct drm_i915_private *i915 = arg;
+	struct intel_engine_cs *engine;
+	struct i915_gem_context *ctx;
+	enum intel_engine_id id;
+	intel_wakeref_t wakeref;
+	struct drm_file *file;
+	int err = 0;
+
+	/* Check that we can engine-reset during non-user portions */
+
+	if (!intel_has_reset_engine(i915))
+		return 0;
+
+	file = mock_file(i915);
+	if (IS_ERR(file))
+		return PTR_ERR(file);
+
+	mutex_lock(&i915->drm.struct_mutex);
+	ctx = live_context(i915, file);
+	mutex_unlock(&i915->drm.struct_mutex);
+	if (IS_ERR(ctx)) {
+		err = PTR_ERR(ctx);
+		goto out;
+	}
+
+	i915_gem_context_clear_bannable(ctx);
+	wakeref = intel_runtime_pm_get(i915);
+	for_each_engine(engine, i915, id) {
+		unsigned int reset_count, reset_engine_count;
+		unsigned int count;
+		IGT_TIMEOUT(end_time);
+
+		reset_count = i915_reset_count(&i915->gpu_error);
+		reset_engine_count = i915_reset_engine_count(&i915->gpu_error,
+							     engine);
+		count = 0;
+
+		set_bit(I915_RESET_ENGINE + id, &i915->gpu_error.flags);
+		do {
+			int i;
+
+			if (!wait_for_idle(engine)) {
+				pr_err("%s failed to idle before reset\n",
+				       engine->name);
+				err = -EIO;
+				break;
+			}
+
+			mutex_lock(&i915->drm.struct_mutex);
+			for (i = 0; i < 16; i++) {
+				struct i915_request *rq;
+
+				rq = i915_request_alloc(engine, ctx);
+				if (IS_ERR(rq)) {
+					err = PTR_ERR(rq);
+					break;
+				}
+
+				i915_request_add(rq);
+			}
+			mutex_unlock(&i915->drm.struct_mutex);
+
+			err = i915_reset_engine(engine, NULL);
+			if (err) {
+				pr_err("i915_reset_engine failed\n");
+				break;
+			}
+
+			if (i915_reset_count(&i915->gpu_error) != reset_count) {
+				pr_err("Full GPU reset recorded! (engine reset expected)\n");
+				err = -EINVAL;
+				break;
+			}
+
+			if (i915_reset_engine_count(&i915->gpu_error, engine) !=
+			    reset_engine_count + ++count) {
+				pr_err("%s engine reset not recorded!\n",
+				       engine->name);
+				err = -EINVAL;
+				break;
+			}
+
+			if (!i915_reset_flush(i915)) {
+				struct drm_printer p =
+					drm_info_printer(i915->drm.dev);
+
+				pr_err("%s failed to idle after reset\n",
+				       engine->name);
+				intel_engine_dump(engine, &p,
+						  "%s\n", engine->name);
+
+				err = -EIO;
+				break;
+			}
+		} while (time_before(jiffies, end_time));
+		clear_bit(I915_RESET_ENGINE + id, &i915->gpu_error.flags);
+		pr_info("%s(%s): %d resets\n", __func__, engine->name, count);
+
+		if (err)
+			break;
+
+		err = igt_flush_test(i915, 0);
+		if (err)
+			break;
+	}
+
+	mutex_lock(&i915->drm.struct_mutex);
+	err = igt_flush_test(i915, I915_WAIT_LOCKED);
+	mutex_unlock(&i915->drm.struct_mutex);
+
+	intel_runtime_pm_put(i915, wakeref);
+out:
+	mock_file_free(i915, file);
+	if (i915_reset_failed(i915))
+		err = -EIO;
+	return err;
+}
+
 static int __igt_reset_engine(struct drm_i915_private *i915, bool active)
 {
 	struct intel_engine_cs *engine;
@@ -523,7 +745,7 @@ static int __igt_reset_engine(struct drm_i915_private *i915, bool active)
 			break;
 	}
 
-	if (i915_terminally_wedged(&i915->gpu_error))
+	if (i915_reset_failed(i915))
 		err = -EIO;
 
 	if (active) {
@@ -565,11 +787,10 @@ static int active_request_put(struct i915_request *rq)
 		return 0;
 
 	if (i915_request_wait(rq, 0, 5 * HZ) < 0) {
-		GEM_TRACE("%s timed out waiting for completion of fence %llx:%lld, seqno %d.\n",
+		GEM_TRACE("%s timed out waiting for completion of fence %llx:%lld\n",
 			  rq->engine->name,
 			  rq->fence.context,
-			  rq->fence.seqno,
-			  i915_request_global_seqno(rq));
+			  rq->fence.seqno);
 		GEM_TRACE_DUMP();
 
 		i915_gem_set_wedged(rq->i915);
@@ -762,7 +983,23 @@ static int __igt_reset_engines(struct drm_i915_private *i915,
 			count++;
 
 			if (rq) {
-				i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT);
+				if (i915_request_wait(rq, 0, HZ / 5) < 0) {
+					struct drm_printer p =
+						drm_info_printer(i915->drm.dev);
+
+					pr_err("i915_reset_engine(%s:%s):"
+					       " failed to complete request after reset\n",
+					       engine->name, test_name);
+					intel_engine_dump(engine, &p,
+							  "%s\n", engine->name);
+					i915_request_put(rq);
+
+					GEM_TRACE_DUMP();
+					i915_gem_set_wedged(i915);
+					err = -EIO;
+					break;
+				}
+
 				i915_request_put(rq);
 			}
 
@@ -837,7 +1074,7 @@ unwind:
 			break;
 	}
 
-	if (i915_terminally_wedged(&i915->gpu_error))
+	if (i915_reset_failed(i915))
 		err = -EIO;
 
 	if (flags & TEST_ACTIVE) {
@@ -887,7 +1124,8 @@ static int igt_reset_engines(void *arg)
 	return 0;
 }
 
-static u32 fake_hangcheck(struct drm_i915_private *i915, u32 mask)
+static u32 fake_hangcheck(struct drm_i915_private *i915,
+			  intel_engine_mask_t mask)
 {
 	u32 count = i915_reset_count(&i915->gpu_error);
 
@@ -905,7 +1143,7 @@ static int igt_reset_wait(void *arg)
 	long timeout;
 	int err;
 
-	if (!intel_engine_can_store_dword(i915->engine[RCS]))
+	if (!intel_engine_can_store_dword(i915->engine[RCS0]))
 		return 0;
 
 	/* Check that we detect a stuck waiter and issue a reset */
@@ -917,7 +1155,7 @@ static int igt_reset_wait(void *arg)
 	if (err)
 		goto unlock;
 
-	rq = hang_create_request(&h, i915->engine[RCS]);
+	rq = hang_create_request(&h, i915->engine[RCS0]);
 	if (IS_ERR(rq)) {
 		err = PTR_ERR(rq);
 		goto fini;
@@ -963,7 +1201,7 @@ unlock:
 	mutex_unlock(&i915->drm.struct_mutex);
 	igt_global_reset_unlock(i915);
 
-	if (i915_terminally_wedged(&i915->gpu_error))
+	if (i915_reset_failed(i915))
 		return -EIO;
 
 	return err;
@@ -1034,13 +1272,11 @@ static int __igt_reset_evict_vma(struct drm_i915_private *i915,
 	struct hang h;
 	int err;
 
-	if (!intel_engine_can_store_dword(i915->engine[RCS]))
+	if (!intel_engine_can_store_dword(i915->engine[RCS0]))
 		return 0;
 
 	/* Check that we can recover an unbind stuck on a hanging request */
 
-	igt_global_reset_lock(i915);
-
 	mutex_lock(&i915->drm.struct_mutex);
 	err = hang_init(&h, i915);
 	if (err)
@@ -1066,7 +1302,7 @@ static int __igt_reset_evict_vma(struct drm_i915_private *i915,
 		goto out_obj;
 	}
 
-	rq = hang_create_request(&h, i915->engine[RCS]);
+	rq = hang_create_request(&h, i915->engine[RCS0]);
 	if (IS_ERR(rq)) {
 		err = PTR_ERR(rq);
 		goto out_obj;
@@ -1138,7 +1374,9 @@ static int __igt_reset_evict_vma(struct drm_i915_private *i915,
 	}
 
 out_reset:
-	fake_hangcheck(rq->i915, intel_engine_flag(rq->engine));
+	igt_global_reset_lock(i915);
+	fake_hangcheck(rq->i915, rq->engine->mask);
+	igt_global_reset_unlock(i915);
 
 	if (tsk) {
 		struct igt_wedge_me w;
@@ -1159,9 +1397,8 @@ fini:
 	hang_fini(&h);
 unlock:
 	mutex_unlock(&i915->drm.struct_mutex);
-	igt_global_reset_unlock(i915);
 
-	if (i915_terminally_wedged(&i915->gpu_error))
+	if (i915_reset_failed(i915))
 		return -EIO;
 
 	return err;
@@ -1317,7 +1554,7 @@ static int igt_reset_queue(void *arg)
 				goto fini;
 			}
 
-			reset_count = fake_hangcheck(i915, ENGINE_MASK(id));
+			reset_count = fake_hangcheck(i915, BIT(id));
 
 			if (prev->fence.error != -EIO) {
 				pr_err("GPU reset not recorded on hanging request [fence.error=%d]!\n",
@@ -1367,7 +1604,7 @@ unlock:
 	mutex_unlock(&i915->drm.struct_mutex);
 	igt_global_reset_unlock(i915);
 
-	if (i915_terminally_wedged(&i915->gpu_error))
+	if (i915_reset_failed(i915))
 		return -EIO;
 
 	return err;
@@ -1376,7 +1613,7 @@ unlock:
 static int igt_handle_error(void *arg)
 {
 	struct drm_i915_private *i915 = arg;
-	struct intel_engine_cs *engine = i915->engine[RCS];
+	struct intel_engine_cs *engine = i915->engine[RCS0];
 	struct hang h;
 	struct i915_request *rq;
 	struct i915_gpu_state *error;
@@ -1423,7 +1660,7 @@ static int igt_handle_error(void *arg)
 	/* Temporarily disable error capture */
 	error = xchg(&i915->gpu_error.first_error, (void *)-1);
 
-	i915_handle_error(i915, ENGINE_MASK(engine->id), 0, NULL);
+	i915_handle_error(i915, engine->mask, 0, NULL);
 
 	xchg(&i915->gpu_error.first_error, error);
 
@@ -1547,7 +1784,7 @@ static int igt_atomic_reset_engine(struct intel_engine_cs *engine,
 			i915_request_wait(rq,
 					  I915_WAIT_LOCKED,
 					  MAX_SCHEDULE_TIMEOUT);
-		if (i915_terminally_wedged(&i915->gpu_error))
+		if (i915_reset_failed(i915))
 			err = -EIO;
 	}
 
@@ -1586,7 +1823,7 @@ static int igt_atomic_reset(void *arg)
 
 	/* Flush any requests before we get started and check basics */
 	force_reset(i915);
-	if (i915_terminally_wedged(&i915->gpu_error))
+	if (i915_reset_failed(i915))
 		goto unlock;
 
 	if (intel_has_gpu_reset(i915)) {
@@ -1642,6 +1879,8 @@ int intel_hangcheck_live_selftests(struct drm_i915_private *i915)
 		SUBTEST(igt_global_reset), /* attempt to recover GPU first */
 		SUBTEST(igt_wedged_reset),
 		SUBTEST(igt_hang_sanitycheck),
+		SUBTEST(igt_reset_nop),
+		SUBTEST(igt_reset_nop_engine),
 		SUBTEST(igt_reset_idle_engine),
 		SUBTEST(igt_reset_active_engine),
 		SUBTEST(igt_reset_engines),
@@ -1660,7 +1899,7 @@ int intel_hangcheck_live_selftests(struct drm_i915_private *i915)
 	if (!intel_has_gpu_reset(i915))
 		return 0;
 
-	if (i915_terminally_wedged(&i915->gpu_error))
+	if (i915_terminally_wedged(i915))
 		return -EIO; /* we're long past hope of a successful reset */
 
 	wakeref = intel_runtime_pm_get(i915);
diff --git a/drivers/gpu/drm/i915/selftests/intel_lrc.c b/drivers/gpu/drm/i915/selftests/intel_lrc.c
index 58144e024751..fbee030db940 100644
--- a/drivers/gpu/drm/i915/selftests/intel_lrc.c
+++ b/drivers/gpu/drm/i915/selftests/intel_lrc.c
@@ -10,6 +10,7 @@
 
 #include "../i915_selftest.h"
 #include "igt_flush_test.h"
+#include "igt_live_test.h"
 #include "igt_spinner.h"
 #include "i915_random.h"
 
@@ -75,6 +76,185 @@ err_unlock:
 	return err;
 }
 
+static int live_busywait_preempt(void *arg)
+{
+	struct drm_i915_private *i915 = arg;
+	struct i915_gem_context *ctx_hi, *ctx_lo;
+	struct intel_engine_cs *engine;
+	struct drm_i915_gem_object *obj;
+	struct i915_vma *vma;
+	enum intel_engine_id id;
+	intel_wakeref_t wakeref;
+	int err = -ENOMEM;
+	u32 *map;
+
+	/*
+	 * Verify that even without HAS_LOGICAL_RING_PREEMPTION, we can
+	 * preempt the busywaits used to synchronise between rings.
+	 */
+
+	mutex_lock(&i915->drm.struct_mutex);
+	wakeref = intel_runtime_pm_get(i915);
+
+	ctx_hi = kernel_context(i915);
+	if (!ctx_hi)
+		goto err_unlock;
+	ctx_hi->sched.priority = INT_MAX;
+
+	ctx_lo = kernel_context(i915);
+	if (!ctx_lo)
+		goto err_ctx_hi;
+	ctx_lo->sched.priority = INT_MIN;
+
+	obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
+	if (IS_ERR(obj)) {
+		err = PTR_ERR(obj);
+		goto err_ctx_lo;
+	}
+
+	map = i915_gem_object_pin_map(obj, I915_MAP_WC);
+	if (IS_ERR(map)) {
+		err = PTR_ERR(map);
+		goto err_obj;
+	}
+
+	vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
+	if (IS_ERR(vma)) {
+		err = PTR_ERR(vma);
+		goto err_map;
+	}
+
+	err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
+	if (err)
+		goto err_map;
+
+	for_each_engine(engine, i915, id) {
+		struct i915_request *lo, *hi;
+		struct igt_live_test t;
+		u32 *cs;
+
+		if (!intel_engine_can_store_dword(engine))
+			continue;
+
+		if (igt_live_test_begin(&t, i915, __func__, engine->name)) {
+			err = -EIO;
+			goto err_vma;
+		}
+
+		/*
+		 * We create two requests. The low priority request
+		 * busywaits on a semaphore (inside the ringbuffer where
+		 * is should be preemptible) and the high priority requests
+		 * uses a MI_STORE_DWORD_IMM to update the semaphore value
+		 * allowing the first request to complete. If preemption
+		 * fails, we hang instead.
+		 */
+
+		lo = i915_request_alloc(engine, ctx_lo);
+		if (IS_ERR(lo)) {
+			err = PTR_ERR(lo);
+			goto err_vma;
+		}
+
+		cs = intel_ring_begin(lo, 8);
+		if (IS_ERR(cs)) {
+			err = PTR_ERR(cs);
+			i915_request_add(lo);
+			goto err_vma;
+		}
+
+		*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
+		*cs++ = i915_ggtt_offset(vma);
+		*cs++ = 0;
+		*cs++ = 1;
+
+		/* XXX Do we need a flush + invalidate here? */
+
+		*cs++ = MI_SEMAPHORE_WAIT |
+			MI_SEMAPHORE_GLOBAL_GTT |
+			MI_SEMAPHORE_POLL |
+			MI_SEMAPHORE_SAD_EQ_SDD;
+		*cs++ = 0;
+		*cs++ = i915_ggtt_offset(vma);
+		*cs++ = 0;
+
+		intel_ring_advance(lo, cs);
+		i915_request_add(lo);
+
+		if (wait_for(READ_ONCE(*map), 10)) {
+			err = -ETIMEDOUT;
+			goto err_vma;
+		}
+
+		/* Low priority request should be busywaiting now */
+		if (i915_request_wait(lo, I915_WAIT_LOCKED, 1) != -ETIME) {
+			pr_err("%s: Busywaiting request did not!\n",
+			       engine->name);
+			err = -EIO;
+			goto err_vma;
+		}
+
+		hi = i915_request_alloc(engine, ctx_hi);
+		if (IS_ERR(hi)) {
+			err = PTR_ERR(hi);
+			goto err_vma;
+		}
+
+		cs = intel_ring_begin(hi, 4);
+		if (IS_ERR(cs)) {
+			err = PTR_ERR(cs);
+			i915_request_add(hi);
+			goto err_vma;
+		}
+
+		*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
+		*cs++ = i915_ggtt_offset(vma);
+		*cs++ = 0;
+		*cs++ = 0;
+
+		intel_ring_advance(hi, cs);
+		i915_request_add(hi);
+
+		if (i915_request_wait(lo, I915_WAIT_LOCKED, HZ / 5) < 0) {
+			struct drm_printer p = drm_info_printer(i915->drm.dev);
+
+			pr_err("%s: Failed to preempt semaphore busywait!\n",
+			       engine->name);
+
+			intel_engine_dump(engine, &p, "%s\n", engine->name);
+			GEM_TRACE_DUMP();
+
+			i915_gem_set_wedged(i915);
+			err = -EIO;
+			goto err_vma;
+		}
+		GEM_BUG_ON(READ_ONCE(*map));
+
+		if (igt_live_test_end(&t)) {
+			err = -EIO;
+			goto err_vma;
+		}
+	}
+
+	err = 0;
+err_vma:
+	i915_vma_unpin(vma);
+err_map:
+	i915_gem_object_unpin_map(obj);
+err_obj:
+	i915_gem_object_put(obj);
+err_ctx_lo:
+	kernel_context_close(ctx_lo);
+err_ctx_hi:
+	kernel_context_close(ctx_hi);
+err_unlock:
+	if (igt_flush_test(i915, I915_WAIT_LOCKED))
+		err = -EIO;
+	intel_runtime_pm_put(i915, wakeref);
+	mutex_unlock(&i915->drm.struct_mutex);
+	return err;
+}
+
 static int live_preempt(void *arg)
 {
 	struct drm_i915_private *i915 = arg;
@@ -88,6 +268,9 @@ static int live_preempt(void *arg)
 	if (!HAS_LOGICAL_RING_PREEMPTION(i915))
 		return 0;
 
+	if (!(i915->caps.scheduler & I915_SCHEDULER_CAP_PREEMPTION))
+		pr_err("Logical preemption supported, but not exposed\n");
+
 	mutex_lock(&i915->drm.struct_mutex);
 	wakeref = intel_runtime_pm_get(i915);
 
@@ -110,8 +293,17 @@ static int live_preempt(void *arg)
 		I915_USER_PRIORITY(I915_CONTEXT_MIN_USER_PRIORITY);
 
 	for_each_engine(engine, i915, id) {
+		struct igt_live_test t;
 		struct i915_request *rq;
 
+		if (!intel_engine_has_preemption(engine))
+			continue;
+
+		if (igt_live_test_begin(&t, i915, __func__, engine->name)) {
+			err = -EIO;
+			goto err_ctx_lo;
+		}
+
 		rq = igt_spinner_create_request(&spin_lo, ctx_lo, engine,
 						MI_ARB_CHECK);
 		if (IS_ERR(rq)) {
@@ -147,7 +339,8 @@ static int live_preempt(void *arg)
 
 		igt_spinner_end(&spin_hi);
 		igt_spinner_end(&spin_lo);
-		if (igt_flush_test(i915, I915_WAIT_LOCKED)) {
+
+		if (igt_live_test_end(&t)) {
 			err = -EIO;
 			goto err_ctx_lo;
 		}
@@ -201,8 +394,17 @@ static int live_late_preempt(void *arg)
 		goto err_ctx_hi;
 
 	for_each_engine(engine, i915, id) {
+		struct igt_live_test t;
 		struct i915_request *rq;
 
+		if (!intel_engine_has_preemption(engine))
+			continue;
+
+		if (igt_live_test_begin(&t, i915, __func__, engine->name)) {
+			err = -EIO;
+			goto err_ctx_lo;
+		}
+
 		rq = igt_spinner_create_request(&spin_lo, ctx_lo, engine,
 						MI_ARB_CHECK);
 		if (IS_ERR(rq)) {
@@ -241,7 +443,8 @@ static int live_late_preempt(void *arg)
 
 		igt_spinner_end(&spin_hi);
 		igt_spinner_end(&spin_lo);
-		if (igt_flush_test(i915, I915_WAIT_LOCKED)) {
+
+		if (igt_live_test_end(&t)) {
 			err = -EIO;
 			goto err_ctx_lo;
 		}
@@ -335,6 +538,9 @@ static int live_suppress_self_preempt(void *arg)
 		struct i915_request *rq_a, *rq_b;
 		int depth;
 
+		if (!intel_engine_has_preemption(engine))
+			continue;
+
 		engine->execlists.preempt_hang.count = 0;
 
 		rq_a = igt_spinner_create_request(&a.spin,
@@ -407,6 +613,171 @@ err_wedged:
 	goto err_client_b;
 }
 
+static int __i915_sw_fence_call
+dummy_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
+{
+	return NOTIFY_DONE;
+}
+
+static struct i915_request *dummy_request(struct intel_engine_cs *engine)
+{
+	struct i915_request *rq;
+
+	rq = kzalloc(sizeof(*rq), GFP_KERNEL);
+	if (!rq)
+		return NULL;
+
+	INIT_LIST_HEAD(&rq->active_list);
+	rq->engine = engine;
+
+	i915_sched_node_init(&rq->sched);
+
+	/* mark this request as permanently incomplete */
+	rq->fence.seqno = 1;
+	BUILD_BUG_ON(sizeof(rq->fence.seqno) != 8); /* upper 32b == 0 */
+	rq->hwsp_seqno = (u32 *)&rq->fence.seqno + 1;
+	GEM_BUG_ON(i915_request_completed(rq));
+
+	i915_sw_fence_init(&rq->submit, dummy_notify);
+	i915_sw_fence_commit(&rq->submit);
+
+	return rq;
+}
+
+static void dummy_request_free(struct i915_request *dummy)
+{
+	i915_request_mark_complete(dummy);
+	i915_sched_node_fini(&dummy->sched);
+	i915_sw_fence_fini(&dummy->submit);
+
+	dma_fence_free(&dummy->fence);
+}
+
+static int live_suppress_wait_preempt(void *arg)
+{
+	struct drm_i915_private *i915 = arg;
+	struct preempt_client client[4];
+	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
+	intel_wakeref_t wakeref;
+	int err = -ENOMEM;
+	int i;
+
+	/*
+	 * Waiters are given a little priority nudge, but not enough
+	 * to actually cause any preemption. Double check that we do
+	 * not needlessly generate preempt-to-idle cycles.
+	 */
+
+	if (!HAS_LOGICAL_RING_PREEMPTION(i915))
+		return 0;
+
+	mutex_lock(&i915->drm.struct_mutex);
+	wakeref = intel_runtime_pm_get(i915);
+
+	if (preempt_client_init(i915, &client[0])) /* ELSP[0] */
+		goto err_unlock;
+	if (preempt_client_init(i915, &client[1])) /* ELSP[1] */
+		goto err_client_0;
+	if (preempt_client_init(i915, &client[2])) /* head of queue */
+		goto err_client_1;
+	if (preempt_client_init(i915, &client[3])) /* bystander */
+		goto err_client_2;
+
+	for_each_engine(engine, i915, id) {
+		int depth;
+
+		if (!intel_engine_has_preemption(engine))
+			continue;
+
+		if (!engine->emit_init_breadcrumb)
+			continue;
+
+		for (depth = 0; depth < ARRAY_SIZE(client); depth++) {
+			struct i915_request *rq[ARRAY_SIZE(client)];
+			struct i915_request *dummy;
+
+			engine->execlists.preempt_hang.count = 0;
+
+			dummy = dummy_request(engine);
+			if (!dummy)
+				goto err_client_3;
+
+			for (i = 0; i < ARRAY_SIZE(client); i++) {
+				rq[i] = igt_spinner_create_request(&client[i].spin,
+								   client[i].ctx, engine,
+								   MI_NOOP);
+				if (IS_ERR(rq[i])) {
+					err = PTR_ERR(rq[i]);
+					goto err_wedged;
+				}
+
+				/* Disable NEWCLIENT promotion */
+				__i915_active_request_set(&rq[i]->timeline->last_request,
+							  dummy);
+				i915_request_add(rq[i]);
+			}
+
+			dummy_request_free(dummy);
+
+			GEM_BUG_ON(i915_request_completed(rq[0]));
+			if (!igt_wait_for_spinner(&client[0].spin, rq[0])) {
+				pr_err("%s: First client failed to start\n",
+				       engine->name);
+				goto err_wedged;
+			}
+			GEM_BUG_ON(!i915_request_started(rq[0]));
+
+			if (i915_request_wait(rq[depth],
+					      I915_WAIT_LOCKED |
+					      I915_WAIT_PRIORITY,
+					      1) != -ETIME) {
+				pr_err("%s: Waiter depth:%d completed!\n",
+				       engine->name, depth);
+				goto err_wedged;
+			}
+
+			for (i = 0; i < ARRAY_SIZE(client); i++)
+				igt_spinner_end(&client[i].spin);
+
+			if (igt_flush_test(i915, I915_WAIT_LOCKED))
+				goto err_wedged;
+
+			if (engine->execlists.preempt_hang.count) {
+				pr_err("%s: Preemption recorded x%d, depth %d; should have been suppressed!\n",
+				       engine->name,
+				       engine->execlists.preempt_hang.count,
+				       depth);
+				err = -EINVAL;
+				goto err_client_3;
+			}
+		}
+	}
+
+	err = 0;
+err_client_3:
+	preempt_client_fini(&client[3]);
+err_client_2:
+	preempt_client_fini(&client[2]);
+err_client_1:
+	preempt_client_fini(&client[1]);
+err_client_0:
+	preempt_client_fini(&client[0]);
+err_unlock:
+	if (igt_flush_test(i915, I915_WAIT_LOCKED))
+		err = -EIO;
+	intel_runtime_pm_put(i915, wakeref);
+	mutex_unlock(&i915->drm.struct_mutex);
+	return err;
+
+err_wedged:
+	for (i = 0; i < ARRAY_SIZE(client); i++)
+		igt_spinner_end(&client[i].spin);
+	i915_gem_set_wedged(i915);
+	err = -EIO;
+	goto err_client_3;
+}
+
 static int live_chain_preempt(void *arg)
 {
 	struct drm_i915_private *i915 = arg;
@@ -438,11 +809,39 @@ static int live_chain_preempt(void *arg)
 		struct i915_sched_attr attr = {
 			.priority = I915_USER_PRIORITY(I915_PRIORITY_MAX),
 		};
-		int count, i;
+		struct igt_live_test t;
+		struct i915_request *rq;
+		int ring_size, count, i;
 
-		for_each_prime_number_from(count, 1, 32) { /* must fit ring! */
-			struct i915_request *rq;
+		if (!intel_engine_has_preemption(engine))
+			continue;
 
+		rq = igt_spinner_create_request(&lo.spin,
+						lo.ctx, engine,
+						MI_ARB_CHECK);
+		if (IS_ERR(rq))
+			goto err_wedged;
+		i915_request_add(rq);
+
+		ring_size = rq->wa_tail - rq->head;
+		if (ring_size < 0)
+			ring_size += rq->ring->size;
+		ring_size = rq->ring->size / ring_size;
+		pr_debug("%s(%s): Using maximum of %d requests\n",
+			 __func__, engine->name, ring_size);
+
+		igt_spinner_end(&lo.spin);
+		if (i915_request_wait(rq, I915_WAIT_LOCKED, HZ / 2) < 0) {
+			pr_err("Timed out waiting to flush %s\n", engine->name);
+			goto err_wedged;
+		}
+
+		if (igt_live_test_begin(&t, i915, __func__, engine->name)) {
+			err = -EIO;
+			goto err_wedged;
+		}
+
+		for_each_prime_number_from(count, 1, ring_size) {
 			rq = igt_spinner_create_request(&hi.spin,
 							hi.ctx, engine,
 							MI_ARB_CHECK);
@@ -484,6 +883,26 @@ static int live_chain_preempt(void *arg)
 				goto err_wedged;
 			}
 			igt_spinner_end(&lo.spin);
+
+			rq = i915_request_alloc(engine, lo.ctx);
+			if (IS_ERR(rq))
+				goto err_wedged;
+			i915_request_add(rq);
+			if (i915_request_wait(rq, I915_WAIT_LOCKED, HZ / 5) < 0) {
+				struct drm_printer p =
+					drm_info_printer(i915->drm.dev);
+
+				pr_err("Failed to flush low priority chain of %d requests\n",
+				       count);
+				intel_engine_dump(engine, &p,
+						  "%s\n", engine->name);
+				goto err_wedged;
+			}
+		}
+
+		if (igt_live_test_end(&t)) {
+			err = -EIO;
+			goto err_wedged;
 		}
 	}
 
@@ -767,7 +1186,7 @@ static int smoke_crescendo(struct preempt_smoke *smoke, unsigned int flags)
 
 	pr_info("Submitted %lu crescendo:%x requests across %d engines and %d contexts\n",
 		count, flags,
-		RUNTIME_INFO(smoke->i915)->num_rings, smoke->ncontext);
+		RUNTIME_INFO(smoke->i915)->num_engines, smoke->ncontext);
 	return 0;
 }
 
@@ -795,7 +1214,7 @@ static int smoke_random(struct preempt_smoke *smoke, unsigned int flags)
 
 	pr_info("Submitted %lu random:%x requests across %d engines and %d contexts\n",
 		count, flags,
-		RUNTIME_INFO(smoke->i915)->num_rings, smoke->ncontext);
+		RUNTIME_INFO(smoke->i915)->num_engines, smoke->ncontext);
 	return 0;
 }
 
@@ -808,6 +1227,7 @@ static int live_preempt_smoke(void *arg)
 	};
 	const unsigned int phase[] = { 0, BATCH };
 	intel_wakeref_t wakeref;
+	struct igt_live_test t;
 	int err = -ENOMEM;
 	u32 *cs;
 	int n;
@@ -838,11 +1258,13 @@ static int live_preempt_smoke(void *arg)
 	for (n = 0; n < PAGE_SIZE / sizeof(*cs) - 1; n++)
 		cs[n] = MI_ARB_CHECK;
 	cs[n] = MI_BATCH_BUFFER_END;
+	i915_gem_object_flush_map(smoke.batch);
 	i915_gem_object_unpin_map(smoke.batch);
 
-	err = i915_gem_object_set_to_gtt_domain(smoke.batch, false);
-	if (err)
+	if (igt_live_test_begin(&t, smoke.i915, __func__, "all")) {
+		err = -EIO;
 		goto err_batch;
+	}
 
 	for (n = 0; n < smoke.ncontext; n++) {
 		smoke.contexts[n] = kernel_context(smoke.i915);
@@ -861,7 +1283,7 @@ static int live_preempt_smoke(void *arg)
 	}
 
 err_ctx:
-	if (igt_flush_test(smoke.i915, I915_WAIT_LOCKED))
+	if (igt_live_test_end(&t))
 		err = -EIO;
 
 	for (n = 0; n < smoke.ncontext; n++) {
@@ -884,9 +1306,11 @@ int intel_execlists_live_selftests(struct drm_i915_private *i915)
 {
 	static const struct i915_subtest tests[] = {
 		SUBTEST(live_sanitycheck),
+		SUBTEST(live_busywait_preempt),
 		SUBTEST(live_preempt),
 		SUBTEST(live_late_preempt),
 		SUBTEST(live_suppress_self_preempt),
+		SUBTEST(live_suppress_wait_preempt),
 		SUBTEST(live_chain_preempt),
 		SUBTEST(live_preempt_hang),
 		SUBTEST(live_preempt_smoke),
@@ -895,7 +1319,7 @@ int intel_execlists_live_selftests(struct drm_i915_private *i915)
 	if (!HAS_EXECLISTS(i915))
 		return 0;
 
-	if (i915_terminally_wedged(&i915->gpu_error))
+	if (i915_terminally_wedged(i915))
 		return 0;
 
 	return i915_subtests(tests, i915);
diff --git a/drivers/gpu/drm/i915/selftests/intel_uncore.c b/drivers/gpu/drm/i915/selftests/intel_uncore.c
index 81d9d31042a9..e0d7ebecb215 100644
--- a/drivers/gpu/drm/i915/selftests/intel_uncore.c
+++ b/drivers/gpu/drm/i915/selftests/intel_uncore.c
@@ -119,9 +119,143 @@ int intel_uncore_mock_selftests(void)
 	return 0;
 }
 
-static int intel_uncore_check_forcewake_domains(struct drm_i915_private *dev_priv)
+static int live_forcewake_ops(void *arg)
+{
+	static const struct reg {
+		const char *name;
+		unsigned long platforms;
+		unsigned int offset;
+	} registers[] = {
+		{
+			"RING_START",
+			INTEL_GEN_MASK(6, 7),
+			0x38,
+		},
+		{
+			"RING_MI_MODE",
+			INTEL_GEN_MASK(8, BITS_PER_LONG),
+			0x9c,
+		}
+	};
+	const struct reg *r;
+	struct drm_i915_private *i915 = arg;
+	struct intel_uncore_forcewake_domain *domain;
+	struct intel_uncore *uncore = &i915->uncore;
+	struct intel_engine_cs *engine;
+	enum intel_engine_id id;
+	intel_wakeref_t wakeref;
+	unsigned int tmp;
+	int err = 0;
+
+	GEM_BUG_ON(i915->gt.awake);
+
+	/* vlv/chv with their pcu behave differently wrt reads */
+	if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) {
+		pr_debug("PCU fakes forcewake badly; skipping\n");
+		return 0;
+	}
+
+	/*
+	 * Not quite as reliable across the gen as one would hope.
+	 *
+	 * Either our theory of operation is incorrect, or there remain
+	 * external parties interfering with the powerwells.
+	 *
+	 * https://bugs.freedesktop.org/show_bug.cgi?id=110210
+	 */
+	if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN))
+		return 0;
+
+	/* We have to pick carefully to get the exact behaviour we need */
+	for (r = registers; r->name; r++)
+		if (r->platforms & INTEL_INFO(i915)->gen_mask)
+			break;
+	if (!r->name) {
+		pr_debug("Forcewaked register not known for %s; skipping\n",
+			 intel_platform_name(INTEL_INFO(i915)->platform));
+		return 0;
+	}
+
+	wakeref = intel_runtime_pm_get(i915);
+
+	for_each_fw_domain(domain, uncore, tmp) {
+		smp_store_mb(domain->active, false);
+		if (!hrtimer_cancel(&domain->timer))
+			continue;
+
+		intel_uncore_fw_release_timer(&domain->timer);
+	}
+
+	for_each_engine(engine, i915, id) {
+		i915_reg_t mmio = _MMIO(engine->mmio_base + r->offset);
+		u32 __iomem *reg = uncore->regs + engine->mmio_base + r->offset;
+		enum forcewake_domains fw_domains;
+		u32 val;
+
+		if (!engine->default_state)
+			continue;
+
+		fw_domains = intel_uncore_forcewake_for_reg(uncore, mmio,
+							    FW_REG_READ);
+		if (!fw_domains)
+			continue;
+
+		for_each_fw_domain_masked(domain, fw_domains, uncore, tmp) {
+			if (!domain->wake_count)
+				continue;
+
+			pr_err("fw_domain %s still active, aborting test!\n",
+			       intel_uncore_forcewake_domain_to_str(domain->id));
+			err = -EINVAL;
+			goto out_rpm;
+		}
+
+		intel_uncore_forcewake_get(uncore, fw_domains);
+		val = readl(reg);
+		intel_uncore_forcewake_put(uncore, fw_domains);
+
+		/* Flush the forcewake release (delayed onto a timer) */
+		for_each_fw_domain_masked(domain, fw_domains, uncore, tmp) {
+			smp_store_mb(domain->active, false);
+			if (hrtimer_cancel(&domain->timer))
+				intel_uncore_fw_release_timer(&domain->timer);
+
+			preempt_disable();
+			err = wait_ack_clear(domain, FORCEWAKE_KERNEL);
+			preempt_enable();
+			if (err) {
+				pr_err("Failed to clear fw_domain %s\n",
+				       intel_uncore_forcewake_domain_to_str(domain->id));
+				goto out_rpm;
+			}
+		}
+
+		if (!val) {
+			pr_err("%s:%s was zero while fw was held!\n",
+			       engine->name, r->name);
+			err = -EINVAL;
+			goto out_rpm;
+		}
+
+		/* We then expect the read to return 0 outside of the fw */
+		if (wait_for(readl(reg) == 0, 100)) {
+			pr_err("%s:%s=%0x, fw_domains 0x%x still up after 100ms!\n",
+			       engine->name, r->name, readl(reg), fw_domains);
+			err = -ETIMEDOUT;
+			goto out_rpm;
+		}
+	}
+
+out_rpm:
+	intel_runtime_pm_put(i915, wakeref);
+	return err;
+}
+
+static int live_forcewake_domains(void *arg)
 {
 #define FW_RANGE 0x40000
+	struct drm_i915_private *dev_priv = arg;
+	struct intel_uncore *uncore = &dev_priv->uncore;
 	unsigned long *valid;
 	u32 offset;
 	int err;
@@ -137,48 +271,52 @@ static int intel_uncore_check_forcewake_domains(struct drm_i915_private *dev_pri
 	if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN))
 		return 0;
 
-	valid = kcalloc(BITS_TO_LONGS(FW_RANGE), sizeof(*valid),
-			GFP_KERNEL);
+	valid = bitmap_zalloc(FW_RANGE, GFP_KERNEL);
 	if (!valid)
 		return -ENOMEM;
 
-	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
 
-	check_for_unclaimed_mmio(dev_priv);
+	check_for_unclaimed_mmio(uncore);
 	for (offset = 0; offset < FW_RANGE; offset += 4) {
 		i915_reg_t reg = { offset };
 
 		(void)I915_READ_FW(reg);
-		if (!check_for_unclaimed_mmio(dev_priv))
+		if (!check_for_unclaimed_mmio(uncore))
 			set_bit(offset, valid);
 	}
 
-	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+	intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
 
 	err = 0;
 	for_each_set_bit(offset, valid, FW_RANGE) {
 		i915_reg_t reg = { offset };
 
 		iosf_mbi_punit_acquire();
-		intel_uncore_forcewake_reset(dev_priv);
+		intel_uncore_forcewake_reset(uncore);
 		iosf_mbi_punit_release();
 
-		check_for_unclaimed_mmio(dev_priv);
+		check_for_unclaimed_mmio(uncore);
 
 		(void)I915_READ(reg);
-		if (check_for_unclaimed_mmio(dev_priv)) {
+		if (check_for_unclaimed_mmio(uncore)) {
 			pr_err("Unclaimed mmio read to register 0x%04x\n",
 			       offset);
 			err = -EINVAL;
 		}
 	}
 
-	kfree(valid);
+	bitmap_free(valid);
 	return err;
 }
 
 int intel_uncore_live_selftests(struct drm_i915_private *i915)
 {
+	static const struct i915_subtest tests[] = {
+		SUBTEST(live_forcewake_ops),
+		SUBTEST(live_forcewake_domains),
+	};
+
 	int err;
 
 	/* Confirm the table we load is still valid */
@@ -188,9 +326,5 @@ int intel_uncore_live_selftests(struct drm_i915_private *i915)
 	if (err)
 		return err;
 
-	err = intel_uncore_check_forcewake_domains(i915);
-	if (err)
-		return err;
-
-	return 0;
+	return i915_subtests(tests, i915);
 }
diff --git a/drivers/gpu/drm/i915/selftests/intel_workarounds.c b/drivers/gpu/drm/i915/selftests/intel_workarounds.c
index b15c4f26c593..567b6f8dae86 100644
--- a/drivers/gpu/drm/i915/selftests/intel_workarounds.c
+++ b/drivers/gpu/drm/i915/selftests/intel_workarounds.c
@@ -12,6 +12,14 @@
 #include "igt_spinner.h"
 #include "igt_wedge_me.h"
 #include "mock_context.h"
+#include "mock_drm.h"
+
+static const struct wo_register {
+	enum intel_platform platform;
+	u32 reg;
+} wo_registers[] = {
+	{ INTEL_GEMINILAKE, 0x731c }
+};
 
 #define REF_NAME_MAX (INTEL_ENGINE_CS_MAX_NAME + 4)
 struct wa_lists {
@@ -74,7 +82,7 @@ read_nonprivs(struct i915_gem_context *ctx, struct intel_engine_cs *engine)
 	if (IS_ERR(result))
 		return result;
 
-	i915_gem_object_set_cache_level(result, I915_CACHE_LLC);
+	i915_gem_object_set_cache_coherency(result, I915_CACHE_LLC);
 
 	cs = i915_gem_object_pin_map(result, I915_MAP_WB);
 	if (IS_ERR(cs)) {
@@ -82,6 +90,7 @@ read_nonprivs(struct i915_gem_context *ctx, struct intel_engine_cs *engine)
 		goto err_obj;
 	}
 	memset(cs, 0xc5, PAGE_SIZE);
+	i915_gem_object_flush_map(result);
 	i915_gem_object_unpin_map(result);
 
 	vma = i915_vma_instance(result, &engine->i915->ggtt.vm, NULL);
@@ -181,7 +190,7 @@ static int check_whitelist(struct i915_gem_context *ctx,
 	err = 0;
 	igt_wedge_on_timeout(&wedge, ctx->i915, HZ / 5) /* a safety net! */
 		err = i915_gem_object_set_to_cpu_domain(results, false);
-	if (i915_terminally_wedged(&ctx->i915->gpu_error))
+	if (i915_terminally_wedged(ctx->i915))
 		err = -EIO;
 	if (err)
 		goto out_put;
@@ -214,7 +223,7 @@ out_put:
 
 static int do_device_reset(struct intel_engine_cs *engine)
 {
-	i915_reset(engine->i915, ENGINE_MASK(engine->id), "live_workarounds");
+	i915_reset(engine->i915, engine->mask, "live_workarounds");
 	return 0;
 }
 
@@ -236,15 +245,11 @@ switch_to_scratch_context(struct intel_engine_cs *engine,
 	if (IS_ERR(ctx))
 		return PTR_ERR(ctx);
 
+	GEM_BUG_ON(i915_gem_context_is_bannable(ctx));
+
 	rq = ERR_PTR(-ENODEV);
-	with_intel_runtime_pm(engine->i915, wakeref) {
-		if (spin)
-			rq = igt_spinner_create_request(spin,
-							ctx, engine,
-							MI_NOOP);
-		else
-			rq = i915_request_alloc(engine, ctx);
-	}
+	with_intel_runtime_pm(engine->i915, wakeref)
+		rq = igt_spinner_create_request(spin, ctx, engine, MI_NOOP);
 
 	kernel_context_close(ctx);
 
@@ -273,7 +278,6 @@ static int check_whitelist_across_reset(struct intel_engine_cs *engine,
 					const char *name)
 {
 	struct drm_i915_private *i915 = engine->i915;
-	bool want_spin = reset == do_engine_reset;
 	struct i915_gem_context *ctx;
 	struct igt_spinner spin;
 	intel_wakeref_t wakeref;
@@ -282,11 +286,9 @@ static int check_whitelist_across_reset(struct intel_engine_cs *engine,
 	pr_info("Checking %d whitelisted registers (RING_NONPRIV) [%s]\n",
 		engine->whitelist.count, name);
 
-	if (want_spin) {
-		err = igt_spinner_init(&spin, i915);
-		if (err)
-			return err;
-	}
+	err = igt_spinner_init(&spin, i915);
+	if (err)
+		return err;
 
 	ctx = kernel_context(i915);
 	if (IS_ERR(ctx))
@@ -298,17 +300,15 @@ static int check_whitelist_across_reset(struct intel_engine_cs *engine,
 		goto out;
 	}
 
-	err = switch_to_scratch_context(engine, want_spin ? &spin : NULL);
+	err = switch_to_scratch_context(engine, &spin);
 	if (err)
 		goto out;
 
 	with_intel_runtime_pm(i915, wakeref)
 		err = reset(engine);
 
-	if (want_spin) {
-		igt_spinner_end(&spin);
-		igt_spinner_fini(&spin);
-	}
+	igt_spinner_end(&spin);
+	igt_spinner_fini(&spin);
 
 	if (err) {
 		pr_err("%s reset failed\n", name);
@@ -340,10 +340,379 @@ out:
 	return err;
 }
 
+static struct i915_vma *create_scratch(struct i915_gem_context *ctx)
+{
+	struct drm_i915_gem_object *obj;
+	struct i915_vma *vma;
+	void *ptr;
+	int err;
+
+	obj = i915_gem_object_create_internal(ctx->i915, PAGE_SIZE);
+	if (IS_ERR(obj))
+		return ERR_CAST(obj);
+
+	i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
+
+	ptr = i915_gem_object_pin_map(obj, I915_MAP_WB);
+	if (IS_ERR(ptr)) {
+		err = PTR_ERR(ptr);
+		goto err_obj;
+	}
+	memset(ptr, 0xc5, PAGE_SIZE);
+	i915_gem_object_flush_map(obj);
+	i915_gem_object_unpin_map(obj);
+
+	vma = i915_vma_instance(obj, &ctx->ppgtt->vm, NULL);
+	if (IS_ERR(vma)) {
+		err = PTR_ERR(vma);
+		goto err_obj;
+	}
+
+	err = i915_vma_pin(vma, 0, 0, PIN_USER);
+	if (err)
+		goto err_obj;
+
+	err = i915_gem_object_set_to_cpu_domain(obj, false);
+	if (err)
+		goto err_obj;
+
+	return vma;
+
+err_obj:
+	i915_gem_object_put(obj);
+	return ERR_PTR(err);
+}
+
+static struct i915_vma *create_batch(struct i915_gem_context *ctx)
+{
+	struct drm_i915_gem_object *obj;
+	struct i915_vma *vma;
+	int err;
+
+	obj = i915_gem_object_create_internal(ctx->i915, 16 * PAGE_SIZE);
+	if (IS_ERR(obj))
+		return ERR_CAST(obj);
+
+	vma = i915_vma_instance(obj, &ctx->ppgtt->vm, NULL);
+	if (IS_ERR(vma)) {
+		err = PTR_ERR(vma);
+		goto err_obj;
+	}
+
+	err = i915_vma_pin(vma, 0, 0, PIN_USER);
+	if (err)
+		goto err_obj;
+
+	err = i915_gem_object_set_to_wc_domain(obj, true);
+	if (err)
+		goto err_obj;
+
+	return vma;
+
+err_obj:
+	i915_gem_object_put(obj);
+	return ERR_PTR(err);
+}
+
+static u32 reg_write(u32 old, u32 new, u32 rsvd)
+{
+	if (rsvd == 0x0000ffff) {
+		old &= ~(new >> 16);
+		old |= new & (new >> 16);
+	} else {
+		old &= ~rsvd;
+		old |= new & rsvd;
+	}
+
+	return old;
+}
+
+static bool wo_register(struct intel_engine_cs *engine, u32 reg)
+{
+	enum intel_platform platform = INTEL_INFO(engine->i915)->platform;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(wo_registers); i++) {
+		if (wo_registers[i].platform == platform &&
+		    wo_registers[i].reg == reg)
+			return true;
+	}
+
+	return false;
+}
+
+static int check_dirty_whitelist(struct i915_gem_context *ctx,
+				 struct intel_engine_cs *engine)
+{
+	const u32 values[] = {
+		0x00000000,
+		0x01010101,
+		0x10100101,
+		0x03030303,
+		0x30300303,
+		0x05050505,
+		0x50500505,
+		0x0f0f0f0f,
+		0xf00ff00f,
+		0x10101010,
+		0xf0f01010,
+		0x30303030,
+		0xa0a03030,
+		0x50505050,
+		0xc0c05050,
+		0xf0f0f0f0,
+		0x11111111,
+		0x33333333,
+		0x55555555,
+		0x0000ffff,
+		0x00ff00ff,
+		0xff0000ff,
+		0xffff00ff,
+		0xffffffff,
+	};
+	struct i915_vma *scratch;
+	struct i915_vma *batch;
+	int err = 0, i, v;
+	u32 *cs, *results;
+
+	scratch = create_scratch(ctx);
+	if (IS_ERR(scratch))
+		return PTR_ERR(scratch);
+
+	batch = create_batch(ctx);
+	if (IS_ERR(batch)) {
+		err = PTR_ERR(batch);
+		goto out_scratch;
+	}
+
+	for (i = 0; i < engine->whitelist.count; i++) {
+		u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
+		u64 addr = scratch->node.start;
+		struct i915_request *rq;
+		u32 srm, lrm, rsvd;
+		u32 expect;
+		int idx;
+
+		if (wo_register(engine, reg))
+			continue;
+
+		srm = MI_STORE_REGISTER_MEM;
+		lrm = MI_LOAD_REGISTER_MEM;
+		if (INTEL_GEN(ctx->i915) >= 8)
+			lrm++, srm++;
+
+		pr_debug("%s: Writing garbage to %x\n",
+			 engine->name, reg);
+
+		cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC);
+		if (IS_ERR(cs)) {
+			err = PTR_ERR(cs);
+			goto out_batch;
+		}
+
+		/* SRM original */
+		*cs++ = srm;
+		*cs++ = reg;
+		*cs++ = lower_32_bits(addr);
+		*cs++ = upper_32_bits(addr);
+
+		idx = 1;
+		for (v = 0; v < ARRAY_SIZE(values); v++) {
+			/* LRI garbage */
+			*cs++ = MI_LOAD_REGISTER_IMM(1);
+			*cs++ = reg;
+			*cs++ = values[v];
+
+			/* SRM result */
+			*cs++ = srm;
+			*cs++ = reg;
+			*cs++ = lower_32_bits(addr + sizeof(u32) * idx);
+			*cs++ = upper_32_bits(addr + sizeof(u32) * idx);
+			idx++;
+		}
+		for (v = 0; v < ARRAY_SIZE(values); v++) {
+			/* LRI garbage */
+			*cs++ = MI_LOAD_REGISTER_IMM(1);
+			*cs++ = reg;
+			*cs++ = ~values[v];
+
+			/* SRM result */
+			*cs++ = srm;
+			*cs++ = reg;
+			*cs++ = lower_32_bits(addr + sizeof(u32) * idx);
+			*cs++ = upper_32_bits(addr + sizeof(u32) * idx);
+			idx++;
+		}
+		GEM_BUG_ON(idx * sizeof(u32) > scratch->size);
+
+		/* LRM original -- don't leave garbage in the context! */
+		*cs++ = lrm;
+		*cs++ = reg;
+		*cs++ = lower_32_bits(addr);
+		*cs++ = upper_32_bits(addr);
+
+		*cs++ = MI_BATCH_BUFFER_END;
+
+		i915_gem_object_flush_map(batch->obj);
+		i915_gem_object_unpin_map(batch->obj);
+		i915_gem_chipset_flush(ctx->i915);
+
+		rq = i915_request_alloc(engine, ctx);
+		if (IS_ERR(rq)) {
+			err = PTR_ERR(rq);
+			goto out_batch;
+		}
+
+		if (engine->emit_init_breadcrumb) { /* Be nice if we hang */
+			err = engine->emit_init_breadcrumb(rq);
+			if (err)
+				goto err_request;
+		}
+
+		err = engine->emit_bb_start(rq,
+					    batch->node.start, PAGE_SIZE,
+					    0);
+		if (err)
+			goto err_request;
+
+err_request:
+		i915_request_add(rq);
+		if (err)
+			goto out_batch;
+
+		if (i915_request_wait(rq, I915_WAIT_LOCKED, HZ / 5) < 0) {
+			pr_err("%s: Futzing %x timedout; cancelling test\n",
+			       engine->name, reg);
+			i915_gem_set_wedged(ctx->i915);
+			err = -EIO;
+			goto out_batch;
+		}
+
+		results = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
+		if (IS_ERR(results)) {
+			err = PTR_ERR(results);
+			goto out_batch;
+		}
+
+		GEM_BUG_ON(values[ARRAY_SIZE(values) - 1] != 0xffffffff);
+		rsvd = results[ARRAY_SIZE(values)]; /* detect write masking */
+		if (!rsvd) {
+			pr_err("%s: Unable to write to whitelisted register %x\n",
+			       engine->name, reg);
+			err = -EINVAL;
+			goto out_unpin;
+		}
+
+		expect = results[0];
+		idx = 1;
+		for (v = 0; v < ARRAY_SIZE(values); v++) {
+			expect = reg_write(expect, values[v], rsvd);
+			if (results[idx] != expect)
+				err++;
+			idx++;
+		}
+		for (v = 0; v < ARRAY_SIZE(values); v++) {
+			expect = reg_write(expect, ~values[v], rsvd);
+			if (results[idx] != expect)
+				err++;
+			idx++;
+		}
+		if (err) {
+			pr_err("%s: %d mismatch between values written to whitelisted register [%x], and values read back!\n",
+			       engine->name, err, reg);
+
+			pr_info("%s: Whitelisted register: %x, original value %08x, rsvd %08x\n",
+				engine->name, reg, results[0], rsvd);
+
+			expect = results[0];
+			idx = 1;
+			for (v = 0; v < ARRAY_SIZE(values); v++) {
+				u32 w = values[v];
+
+				expect = reg_write(expect, w, rsvd);
+				pr_info("Wrote %08x, read %08x, expect %08x\n",
+					w, results[idx], expect);
+				idx++;
+			}
+			for (v = 0; v < ARRAY_SIZE(values); v++) {
+				u32 w = ~values[v];
+
+				expect = reg_write(expect, w, rsvd);
+				pr_info("Wrote %08x, read %08x, expect %08x\n",
+					w, results[idx], expect);
+				idx++;
+			}
+
+			err = -EINVAL;
+		}
+out_unpin:
+		i915_gem_object_unpin_map(scratch->obj);
+		if (err)
+			break;
+	}
+
+	if (igt_flush_test(ctx->i915, I915_WAIT_LOCKED))
+		err = -EIO;
+out_batch:
+	i915_vma_unpin_and_release(&batch, 0);
+out_scratch:
+	i915_vma_unpin_and_release(&scratch, 0);
+	return err;
+}
+
+static int live_dirty_whitelist(void *arg)
+{
+	struct drm_i915_private *i915 = arg;
+	struct intel_engine_cs *engine;
+	struct i915_gem_context *ctx;
+	enum intel_engine_id id;
+	intel_wakeref_t wakeref;
+	struct drm_file *file;
+	int err = 0;
+
+	/* Can the user write to the whitelisted registers? */
+
+	if (INTEL_GEN(i915) < 7) /* minimum requirement for LRI, SRM, LRM */
+		return 0;
+
+	wakeref = intel_runtime_pm_get(i915);
+
+	mutex_unlock(&i915->drm.struct_mutex);
+	file = mock_file(i915);
+	mutex_lock(&i915->drm.struct_mutex);
+	if (IS_ERR(file)) {
+		err = PTR_ERR(file);
+		goto out_rpm;
+	}
+
+	ctx = live_context(i915, file);
+	if (IS_ERR(ctx)) {
+		err = PTR_ERR(ctx);
+		goto out_file;
+	}
+
+	for_each_engine(engine, i915, id) {
+		if (engine->whitelist.count == 0)
+			continue;
+
+		err = check_dirty_whitelist(ctx, engine);
+		if (err)
+			goto out_file;
+	}
+
+out_file:
+	mutex_unlock(&i915->drm.struct_mutex);
+	mock_file_free(i915, file);
+	mutex_lock(&i915->drm.struct_mutex);
+out_rpm:
+	intel_runtime_pm_put(i915, wakeref);
+	return err;
+}
+
 static int live_reset_whitelist(void *arg)
 {
 	struct drm_i915_private *i915 = arg;
-	struct intel_engine_cs *engine = i915->engine[RCS];
+	struct intel_engine_cs *engine = i915->engine[RCS0];
 	int err = 0;
 
 	/* If we reset the gpu, we should not lose the RING_NONPRIV */
@@ -381,10 +750,11 @@ static bool verify_gt_engine_wa(struct drm_i915_private *i915,
 	enum intel_engine_id id;
 	bool ok = true;
 
-	ok &= wa_list_verify(i915, &lists->gt_wa_list, str);
+	ok &= wa_list_verify(&i915->uncore, &lists->gt_wa_list, str);
 
 	for_each_engine(engine, i915, id)
-		ok &= wa_list_verify(i915, &lists->engine[id].wa_list, str);
+		ok &= wa_list_verify(engine->uncore,
+				     &lists->engine[id].wa_list, str);
 
 	return ok;
 }
@@ -513,13 +883,14 @@ err:
 int intel_workarounds_live_selftests(struct drm_i915_private *i915)
 {
 	static const struct i915_subtest tests[] = {
+		SUBTEST(live_dirty_whitelist),
 		SUBTEST(live_reset_whitelist),
 		SUBTEST(live_gpu_reset_gt_engine_workarounds),
 		SUBTEST(live_engine_reset_gt_engine_workarounds),
 	};
 	int err;
 
-	if (i915_terminally_wedged(&i915->gpu_error))
+	if (i915_terminally_wedged(i915))
 		return 0;
 
 	mutex_lock(&i915->drm.struct_mutex);
diff --git a/drivers/gpu/drm/i915/selftests/mock_context.c b/drivers/gpu/drm/i915/selftests/mock_context.c
index b646cdcdd602..0426093bf1d9 100644
--- a/drivers/gpu/drm/i915/selftests/mock_context.c
+++ b/drivers/gpu/drm/i915/selftests/mock_context.c
@@ -30,7 +30,6 @@ mock_context(struct drm_i915_private *i915,
 	     const char *name)
 {
 	struct i915_gem_context *ctx;
-	unsigned int n;
 	int ret;
 
 	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
@@ -41,25 +40,31 @@ mock_context(struct drm_i915_private *i915,
 	INIT_LIST_HEAD(&ctx->link);
 	ctx->i915 = i915;
 
+	ctx->hw_contexts = RB_ROOT;
+	spin_lock_init(&ctx->hw_contexts_lock);
+
 	INIT_RADIX_TREE(&ctx->handles_vma, GFP_KERNEL);
 	INIT_LIST_HEAD(&ctx->handles_list);
 	INIT_LIST_HEAD(&ctx->hw_id_link);
-
-	for (n = 0; n < ARRAY_SIZE(ctx->__engine); n++)
-		intel_context_init(&ctx->__engine[n], ctx, i915->engine[n]);
+	INIT_LIST_HEAD(&ctx->active_engines);
+	mutex_init(&ctx->mutex);
 
 	ret = i915_gem_context_pin_hw_id(ctx);
 	if (ret < 0)
 		goto err_handles;
 
 	if (name) {
+		struct i915_hw_ppgtt *ppgtt;
+
 		ctx->name = kstrdup(name, GFP_KERNEL);
 		if (!ctx->name)
 			goto err_put;
 
-		ctx->ppgtt = mock_ppgtt(i915, name);
-		if (!ctx->ppgtt)
+		ppgtt = mock_ppgtt(i915, name);
+		if (!ppgtt)
 			goto err_put;
+
+		__set_ppgtt(ctx, ppgtt);
 	}
 
 	return ctx;
@@ -87,9 +92,24 @@ void mock_init_contexts(struct drm_i915_private *i915)
 struct i915_gem_context *
 live_context(struct drm_i915_private *i915, struct drm_file *file)
 {
+	struct i915_gem_context *ctx;
+	int err;
+
 	lockdep_assert_held(&i915->drm.struct_mutex);
 
-	return i915_gem_create_context(i915, file->driver_priv);
+	ctx = i915_gem_create_context(i915, 0);
+	if (IS_ERR(ctx))
+		return ctx;
+
+	err = gem_context_register(ctx, file->driver_priv);
+	if (err < 0)
+		goto err_ctx;
+
+	return ctx;
+
+err_ctx:
+	context_close(ctx);
+	return ERR_PTR(err);
 }
 
 struct i915_gem_context *
diff --git a/drivers/gpu/drm/i915/selftests/mock_engine.c b/drivers/gpu/drm/i915/selftests/mock_engine.c
index 08f0cab02e0f..61a8206ed677 100644
--- a/drivers/gpu/drm/i915/selftests/mock_engine.c
+++ b/drivers/gpu/drm/i915/selftests/mock_engine.c
@@ -50,13 +50,12 @@ static struct intel_ring *mock_ring(struct intel_engine_cs *engine)
 	if (!ring)
 		return NULL;
 
-	if (i915_timeline_init(engine->i915,
-			       &ring->timeline, engine->name,
-			       NULL)) {
+	if (i915_timeline_init(engine->i915, &ring->timeline, NULL)) {
 		kfree(ring);
 		return NULL;
 	}
 
+	kref_init(&ring->base.ref);
 	ring->base.size = sz;
 	ring->base.effective_size = sz;
 	ring->base.vaddr = (void *)(ring + 1);
@@ -76,28 +75,26 @@ static void mock_ring_free(struct intel_ring *base)
 	kfree(ring);
 }
 
-static struct mock_request *first_request(struct mock_engine *engine)
+static struct i915_request *first_request(struct mock_engine *engine)
 {
 	return list_first_entry_or_null(&engine->hw_queue,
-					struct mock_request,
-					link);
+					struct i915_request,
+					mock.link);
 }
 
-static void advance(struct mock_request *request)
+static void advance(struct i915_request *request)
 {
-	list_del_init(&request->link);
-	intel_engine_write_global_seqno(request->base.engine,
-					request->base.global_seqno);
-	i915_request_mark_complete(&request->base);
-	GEM_BUG_ON(!i915_request_completed(&request->base));
+	list_del_init(&request->mock.link);
+	i915_request_mark_complete(request);
+	GEM_BUG_ON(!i915_request_completed(request));
 
-	intel_engine_queue_breadcrumbs(request->base.engine);
+	intel_engine_queue_breadcrumbs(request->engine);
 }
 
 static void hw_delay_complete(struct timer_list *t)
 {
 	struct mock_engine *engine = from_timer(engine, t, hw_delay);
-	struct mock_request *request;
+	struct i915_request *request;
 	unsigned long flags;
 
 	spin_lock_irqsave(&engine->hw_lock, flags);
@@ -112,8 +109,9 @@ static void hw_delay_complete(struct timer_list *t)
 	 * requeue the timer for the next delayed request.
 	 */
 	while ((request = first_request(engine))) {
-		if (request->delay) {
-			mod_timer(&engine->hw_delay, jiffies + request->delay);
+		if (request->mock.delay) {
+			mod_timer(&engine->hw_delay,
+				  jiffies + request->mock.delay);
 			break;
 		}
 
@@ -126,55 +124,43 @@ static void hw_delay_complete(struct timer_list *t)
 static void mock_context_unpin(struct intel_context *ce)
 {
 	mock_timeline_unpin(ce->ring->timeline);
-	i915_gem_context_put(ce->gem_context);
 }
 
-static void mock_context_destroy(struct intel_context *ce)
+static void mock_context_destroy(struct kref *ref)
 {
-	GEM_BUG_ON(ce->pin_count);
+	struct intel_context *ce = container_of(ref, typeof(*ce), ref);
+
+	GEM_BUG_ON(intel_context_is_pinned(ce));
 
 	if (ce->ring)
 		mock_ring_free(ce->ring);
-}
 
-static const struct intel_context_ops mock_context_ops = {
-	.unpin = mock_context_unpin,
-	.destroy = mock_context_destroy,
-};
+	intel_context_free(ce);
+}
 
-static struct intel_context *
-mock_context_pin(struct intel_engine_cs *engine,
-		 struct i915_gem_context *ctx)
+static int mock_context_pin(struct intel_context *ce)
 {
-	struct intel_context *ce = to_intel_context(ctx, engine);
-	int err = -ENOMEM;
-
-	if (ce->pin_count++)
-		return ce;
-
 	if (!ce->ring) {
-		ce->ring = mock_ring(engine);
+		ce->ring = mock_ring(ce->engine);
 		if (!ce->ring)
-			goto err;
+			return -ENOMEM;
 	}
 
 	mock_timeline_pin(ce->ring->timeline);
+	return 0;
+}
 
-	ce->ops = &mock_context_ops;
-	i915_gem_context_get(ctx);
-	return ce;
+static const struct intel_context_ops mock_context_ops = {
+	.pin = mock_context_pin,
+	.unpin = mock_context_unpin,
 
-err:
-	ce->pin_count = 0;
-	return ERR_PTR(err);
-}
+	.destroy = mock_context_destroy,
+};
 
 static int mock_request_alloc(struct i915_request *request)
 {
-	struct mock_request *mock = container_of(request, typeof(*mock), base);
-
-	INIT_LIST_HEAD(&mock->link);
-	mock->delay = 0;
+	INIT_LIST_HEAD(&request->mock.link);
+	request->mock.delay = 0;
 
 	return 0;
 }
@@ -192,25 +178,55 @@ static u32 *mock_emit_breadcrumb(struct i915_request *request, u32 *cs)
 
 static void mock_submit_request(struct i915_request *request)
 {
-	struct mock_request *mock = container_of(request, typeof(*mock), base);
 	struct mock_engine *engine =
 		container_of(request->engine, typeof(*engine), base);
 	unsigned long flags;
 
 	i915_request_submit(request);
-	GEM_BUG_ON(!request->global_seqno);
 
 	spin_lock_irqsave(&engine->hw_lock, flags);
-	list_add_tail(&mock->link, &engine->hw_queue);
-	if (mock->link.prev == &engine->hw_queue) {
-		if (mock->delay)
-			mod_timer(&engine->hw_delay, jiffies + mock->delay);
+	list_add_tail(&request->mock.link, &engine->hw_queue);
+	if (list_is_first(&request->mock.link, &engine->hw_queue)) {
+		if (request->mock.delay)
+			mod_timer(&engine->hw_delay,
+				  jiffies + request->mock.delay);
 		else
-			advance(mock);
+			advance(request);
 	}
 	spin_unlock_irqrestore(&engine->hw_lock, flags);
 }
 
+static void mock_reset_prepare(struct intel_engine_cs *engine)
+{
+}
+
+static void mock_reset(struct intel_engine_cs *engine, bool stalled)
+{
+	GEM_BUG_ON(stalled);
+}
+
+static void mock_reset_finish(struct intel_engine_cs *engine)
+{
+}
+
+static void mock_cancel_requests(struct intel_engine_cs *engine)
+{
+	struct i915_request *request;
+	unsigned long flags;
+
+	spin_lock_irqsave(&engine->timeline.lock, flags);
+
+	/* Mark all submitted requests as skipped. */
+	list_for_each_entry(request, &engine->timeline.requests, sched.link) {
+		if (!i915_request_signaled(request))
+			dma_fence_set_error(&request->fence, -EIO);
+
+		i915_request_mark_complete(request);
+	}
+
+	spin_unlock_irqrestore(&engine->timeline.lock, flags);
+}
+
 struct intel_engine_cs *mock_engine(struct drm_i915_private *i915,
 				    const char *name,
 				    int id)
@@ -227,18 +243,21 @@ struct intel_engine_cs *mock_engine(struct drm_i915_private *i915,
 	engine->base.i915 = i915;
 	snprintf(engine->base.name, sizeof(engine->base.name), "%s", name);
 	engine->base.id = id;
+	engine->base.mask = BIT(id);
 	engine->base.status_page.addr = (void *)(engine + 1);
 
-	engine->base.context_pin = mock_context_pin;
+	engine->base.cops = &mock_context_ops;
 	engine->base.request_alloc = mock_request_alloc;
 	engine->base.emit_flush = mock_emit_flush;
 	engine->base.emit_fini_breadcrumb = mock_emit_breadcrumb;
 	engine->base.submit_request = mock_submit_request;
 
-	if (i915_timeline_init(i915,
-			       &engine->base.timeline,
-			       engine->base.name,
-			       NULL))
+	engine->base.reset.prepare = mock_reset_prepare;
+	engine->base.reset.reset = mock_reset;
+	engine->base.reset.finish = mock_reset_finish;
+	engine->base.cancel_requests = mock_cancel_requests;
+
+	if (i915_timeline_init(i915, &engine->base.timeline, NULL))
 		goto err_free;
 	i915_timeline_set_subclass(&engine->base.timeline, TIMELINE_ENGINE);
 
@@ -249,7 +268,8 @@ struct intel_engine_cs *mock_engine(struct drm_i915_private *i915,
 	timer_setup(&engine->hw_delay, hw_delay_complete, 0);
 	INIT_LIST_HEAD(&engine->hw_queue);
 
-	if (IS_ERR(intel_context_pin(i915->kernel_context, &engine->base)))
+	if (pin_context(i915->kernel_context, &engine->base,
+			&engine->base.kernel_context))
 		goto err_breadcrumbs;
 
 	return &engine->base;
@@ -266,19 +286,18 @@ void mock_engine_flush(struct intel_engine_cs *engine)
 {
 	struct mock_engine *mock =
 		container_of(engine, typeof(*mock), base);
-	struct mock_request *request, *rn;
+	struct i915_request *request, *rn;
 
 	del_timer_sync(&mock->hw_delay);
 
 	spin_lock_irq(&mock->hw_lock);
-	list_for_each_entry_safe(request, rn, &mock->hw_queue, link)
+	list_for_each_entry_safe(request, rn, &mock->hw_queue, mock.link)
 		advance(request);
 	spin_unlock_irq(&mock->hw_lock);
 }
 
 void mock_engine_reset(struct intel_engine_cs *engine)
 {
-	intel_engine_write_global_seqno(engine, 0);
 }
 
 void mock_engine_free(struct intel_engine_cs *engine)
@@ -293,7 +312,7 @@ void mock_engine_free(struct intel_engine_cs *engine)
 	if (ce)
 		intel_context_unpin(ce);
 
-	__intel_context_unpin(engine->i915->kernel_context, engine);
+	intel_context_unpin(engine->kernel_context);
 
 	intel_engine_fini_breadcrumbs(engine);
 	i915_timeline_fini(&engine->timeline);
diff --git a/drivers/gpu/drm/i915/selftests/mock_gem_device.c b/drivers/gpu/drm/i915/selftests/mock_gem_device.c
index 14ae46fda49f..60bbf8b4df40 100644
--- a/drivers/gpu/drm/i915/selftests/mock_gem_device.c
+++ b/drivers/gpu/drm/i915/selftests/mock_gem_device.c
@@ -79,12 +79,6 @@ static void mock_device_release(struct drm_device *dev)
 
 	destroy_workqueue(i915->wq);
 
-	kmem_cache_destroy(i915->priorities);
-	kmem_cache_destroy(i915->dependencies);
-	kmem_cache_destroy(i915->requests);
-	kmem_cache_destroy(i915->vmas);
-	kmem_cache_destroy(i915->objects);
-
 	i915_gemfs_fini(i915);
 
 	drm_mode_config_cleanup(&i915->drm);
@@ -115,6 +109,10 @@ static void mock_retire_work_handler(struct work_struct *work)
 
 static void mock_idle_work_handler(struct work_struct *work)
 {
+	struct drm_i915_private *i915 =
+		container_of(work, typeof(*i915), gt.idle_work.work);
+
+	i915->gt.active_engines = 0;
 }
 
 static int pm_domain_resume(struct device *dev)
@@ -184,11 +182,12 @@ struct drm_i915_private *mock_gem_device(void)
 		I915_GTT_PAGE_SIZE_64K |
 		I915_GTT_PAGE_SIZE_2M;
 
-	mock_uncore_init(i915);
+	mock_uncore_init(&i915->uncore);
 	i915_gem_init__mm(i915);
 
 	init_waitqueue_head(&i915->gpu_error.wait_queue);
 	init_waitqueue_head(&i915->gpu_error.reset_queue);
+	init_srcu_struct(&i915->gpu_error.reset_backoff_srcu);
 	mutex_init(&i915->gpu_error.wedge_mutex);
 
 	i915->wq = alloc_ordered_workqueue("mock", 0);
@@ -202,31 +201,6 @@ struct drm_i915_private *mock_gem_device(void)
 
 	i915->gt.awake = true;
 
-	i915->objects = KMEM_CACHE(mock_object, SLAB_HWCACHE_ALIGN);
-	if (!i915->objects)
-		goto err_wq;
-
-	i915->vmas = KMEM_CACHE(i915_vma, SLAB_HWCACHE_ALIGN);
-	if (!i915->vmas)
-		goto err_objects;
-
-	i915->requests = KMEM_CACHE(mock_request,
-				    SLAB_HWCACHE_ALIGN |
-				    SLAB_RECLAIM_ACCOUNT |
-				    SLAB_TYPESAFE_BY_RCU);
-	if (!i915->requests)
-		goto err_vmas;
-
-	i915->dependencies = KMEM_CACHE(i915_dependency,
-					SLAB_HWCACHE_ALIGN |
-					SLAB_RECLAIM_ACCOUNT);
-	if (!i915->dependencies)
-		goto err_requests;
-
-	i915->priorities = KMEM_CACHE(i915_priolist, SLAB_HWCACHE_ALIGN);
-	if (!i915->priorities)
-		goto err_dependencies;
-
 	i915_timelines_init(i915);
 
 	INIT_LIST_HEAD(&i915->gt.active_rings);
@@ -236,13 +210,13 @@ struct drm_i915_private *mock_gem_device(void)
 
 	mock_init_ggtt(i915, &i915->ggtt);
 
-	mkwrite_device_info(i915)->ring_mask = BIT(0);
+	mkwrite_device_info(i915)->engine_mask = BIT(0);
 	i915->kernel_context = mock_context(i915, NULL);
 	if (!i915->kernel_context)
 		goto err_unlock;
 
-	i915->engine[RCS] = mock_engine(i915, "mock", RCS);
-	if (!i915->engine[RCS])
+	i915->engine[RCS0] = mock_engine(i915, "mock", RCS0);
+	if (!i915->engine[RCS0])
 		goto err_context;
 
 	mutex_unlock(&i915->drm.struct_mutex);
@@ -256,16 +230,6 @@ err_context:
 err_unlock:
 	mutex_unlock(&i915->drm.struct_mutex);
 	i915_timelines_fini(i915);
-	kmem_cache_destroy(i915->priorities);
-err_dependencies:
-	kmem_cache_destroy(i915->dependencies);
-err_requests:
-	kmem_cache_destroy(i915->requests);
-err_vmas:
-	kmem_cache_destroy(i915->vmas);
-err_objects:
-	kmem_cache_destroy(i915->objects);
-err_wq:
 	destroy_workqueue(i915->wq);
 err_drv:
 	drm_mode_config_cleanup(&i915->drm);
diff --git a/drivers/gpu/drm/i915/selftests/mock_request.c b/drivers/gpu/drm/i915/selftests/mock_request.c
index 0dc29e242597..d1a7c9608712 100644
--- a/drivers/gpu/drm/i915/selftests/mock_request.c
+++ b/drivers/gpu/drm/i915/selftests/mock_request.c
@@ -31,29 +31,25 @@ mock_request(struct intel_engine_cs *engine,
 	     unsigned long delay)
 {
 	struct i915_request *request;
-	struct mock_request *mock;
 
 	/* NB the i915->requests slab cache is enlarged to fit mock_request */
 	request = i915_request_alloc(engine, context);
 	if (IS_ERR(request))
 		return NULL;
 
-	mock = container_of(request, typeof(*mock), base);
-	mock->delay = delay;
-
-	return &mock->base;
+	request->mock.delay = delay;
+	return request;
 }
 
 bool mock_cancel_request(struct i915_request *request)
 {
-	struct mock_request *mock = container_of(request, typeof(*mock), base);
 	struct mock_engine *engine =
 		container_of(request->engine, typeof(*engine), base);
 	bool was_queued;
 
 	spin_lock_irq(&engine->hw_lock);
-	was_queued = !list_empty(&mock->link);
-	list_del_init(&mock->link);
+	was_queued = !list_empty(&request->mock.link);
+	list_del_init(&request->mock.link);
 	spin_unlock_irq(&engine->hw_lock);
 
 	if (was_queued)
diff --git a/drivers/gpu/drm/i915/selftests/mock_request.h b/drivers/gpu/drm/i915/selftests/mock_request.h
index 995fb728380c..4acf0211df20 100644
--- a/drivers/gpu/drm/i915/selftests/mock_request.h
+++ b/drivers/gpu/drm/i915/selftests/mock_request.h
@@ -29,13 +29,6 @@
 
 #include "../i915_request.h"
 
-struct mock_request {
-	struct i915_request base;
-
-	struct list_head link;
-	unsigned long delay;
-};
-
 struct i915_request *
 mock_request(struct intel_engine_cs *engine,
 	     struct i915_gem_context *context,
diff --git a/drivers/gpu/drm/i915/selftests/mock_timeline.c b/drivers/gpu/drm/i915/selftests/mock_timeline.c
index d2de9ece2118..e084476469ef 100644
--- a/drivers/gpu/drm/i915/selftests/mock_timeline.c
+++ b/drivers/gpu/drm/i915/selftests/mock_timeline.c
@@ -14,8 +14,8 @@ void mock_timeline_init(struct i915_timeline *timeline, u64 context)
 	timeline->fence_context = context;
 
 	spin_lock_init(&timeline->lock);
+	mutex_init(&timeline->mutex);
 
-	INIT_ACTIVE_REQUEST(&timeline->barrier);
 	INIT_ACTIVE_REQUEST(&timeline->last_request);
 	INIT_LIST_HEAD(&timeline->requests);
 
diff --git a/drivers/gpu/drm/i915/selftests/mock_uncore.c b/drivers/gpu/drm/i915/selftests/mock_uncore.c
index 8ef14c7e5e38..ff8999c63a12 100644
--- a/drivers/gpu/drm/i915/selftests/mock_uncore.c
+++ b/drivers/gpu/drm/i915/selftests/mock_uncore.c
@@ -26,21 +26,21 @@
 
 #define __nop_write(x) \
 static void \
-nop_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { }
+nop_write##x(struct intel_uncore *uncore, i915_reg_t reg, u##x val, bool trace) { }
 __nop_write(8)
 __nop_write(16)
 __nop_write(32)
 
 #define __nop_read(x) \
 static u##x \
-nop_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { return 0; }
+nop_read##x(struct intel_uncore *uncore, i915_reg_t reg, bool trace) { return 0; }
 __nop_read(8)
 __nop_read(16)
 __nop_read(32)
 __nop_read(64)
 
-void mock_uncore_init(struct drm_i915_private *i915)
+void mock_uncore_init(struct intel_uncore *uncore)
 {
-	ASSIGN_WRITE_MMIO_VFUNCS(i915, nop);
-	ASSIGN_READ_MMIO_VFUNCS(i915, nop);
+	ASSIGN_WRITE_MMIO_VFUNCS(uncore, nop);
+	ASSIGN_READ_MMIO_VFUNCS(uncore, nop);
 }
diff --git a/drivers/gpu/drm/i915/selftests/mock_uncore.h b/drivers/gpu/drm/i915/selftests/mock_uncore.h
index d79aa3ca4d51..dacb36b5ffcd 100644
--- a/drivers/gpu/drm/i915/selftests/mock_uncore.h
+++ b/drivers/gpu/drm/i915/selftests/mock_uncore.h
@@ -25,6 +25,6 @@
 #ifndef __MOCK_UNCORE_H
 #define __MOCK_UNCORE_H
 
-void mock_uncore_init(struct drm_i915_private *i915);
+void mock_uncore_init(struct intel_uncore *uncore);
 
 #endif /* !__MOCK_UNCORE_H */
diff --git a/drivers/gpu/drm/i915/vlv_dsi.c b/drivers/gpu/drm/i915/vlv_dsi.c
index 31c93c3ccd00..e0b1ec821960 100644
--- a/drivers/gpu/drm/i915/vlv_dsi.c
+++ b/drivers/gpu/drm/i915/vlv_dsi.c
@@ -23,16 +23,20 @@
  * Author: Jani Nikula <jani.nikula@intel.com>
  */
 
+#include <linux/gpio/consumer.h>
+#include <linux/slab.h>
+
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_edid.h>
-#include <drm/i915_drm.h>
 #include <drm/drm_mipi_dsi.h>
-#include <linux/slab.h>
-#include <linux/gpio/consumer.h>
+#include <drm/i915_drm.h>
+
 #include "i915_drv.h"
+#include "intel_connector.h"
 #include "intel_drv.h"
 #include "intel_dsi.h"
+#include "intel_panel.h"
 
 /* return pixels in terms of txbyteclkhs */
 static u16 txbyteclkhs(u16 pixels, int bpp, int lane_count,
@@ -78,7 +82,7 @@ void vlv_dsi_wait_for_fifo_empty(struct intel_dsi *intel_dsi, enum port port)
 	mask = LP_CTRL_FIFO_EMPTY | HS_CTRL_FIFO_EMPTY |
 		LP_DATA_FIFO_EMPTY | HS_DATA_FIFO_EMPTY;
 
-	if (intel_wait_for_register(dev_priv,
+	if (intel_wait_for_register(&dev_priv->uncore,
 				    MIPI_GEN_FIFO_STAT(port), mask, mask,
 				    100))
 		DRM_ERROR("DPI FIFOs are not empty\n");
@@ -148,7 +152,7 @@ static ssize_t intel_dsi_host_transfer(struct mipi_dsi_host *host,
 
 	/* note: this is never true for reads */
 	if (packet.payload_length) {
-		if (intel_wait_for_register(dev_priv,
+		if (intel_wait_for_register(&dev_priv->uncore,
 					    MIPI_GEN_FIFO_STAT(port),
 					    data_mask, 0,
 					    50))
@@ -162,7 +166,7 @@ static ssize_t intel_dsi_host_transfer(struct mipi_dsi_host *host,
 		I915_WRITE(MIPI_INTR_STAT(port), GEN_READ_DATA_AVAIL);
 	}
 
-	if (intel_wait_for_register(dev_priv,
+	if (intel_wait_for_register(&dev_priv->uncore,
 				    MIPI_GEN_FIFO_STAT(port),
 				    ctrl_mask, 0,
 				    50)) {
@@ -174,7 +178,7 @@ static ssize_t intel_dsi_host_transfer(struct mipi_dsi_host *host,
 	/* ->rx_len is set only for reads */
 	if (msg->rx_len) {
 		data_mask = GEN_READ_DATA_AVAIL;
-		if (intel_wait_for_register(dev_priv,
+		if (intel_wait_for_register(&dev_priv->uncore,
 					    MIPI_INTR_STAT(port),
 					    data_mask, data_mask,
 					    50))
@@ -234,7 +238,7 @@ static int dpi_send_cmd(struct intel_dsi *intel_dsi, u32 cmd, bool hs,
 	I915_WRITE(MIPI_DPI_CONTROL(port), cmd);
 
 	mask = SPL_PKT_SENT_INTERRUPT;
-	if (intel_wait_for_register(dev_priv,
+	if (intel_wait_for_register(&dev_priv->uncore,
 				    MIPI_INTR_STAT(port), mask, mask,
 				    100))
 		DRM_ERROR("Video mode command 0x%08x send failed.\n", cmd);
@@ -375,16 +379,18 @@ static bool glk_dsi_enable_io(struct intel_encoder *encoder)
 
 	/* Wait for Pwr ACK */
 	for_each_dsi_port(port, intel_dsi->ports) {
-		if (intel_wait_for_register(dev_priv,
-				MIPI_CTRL(port), GLK_MIPIIO_PORT_POWERED,
-				GLK_MIPIIO_PORT_POWERED, 20))
+		if (intel_wait_for_register(&dev_priv->uncore,
+					    MIPI_CTRL(port),
+					    GLK_MIPIIO_PORT_POWERED,
+					    GLK_MIPIIO_PORT_POWERED,
+					    20))
 			DRM_ERROR("MIPIO port is powergated\n");
 	}
 
 	/* Check for cold boot scenario */
 	for_each_dsi_port(port, intel_dsi->ports) {
-		cold_boot |= !(I915_READ(MIPI_DEVICE_READY(port)) &
-							DEVICE_READY);
+		cold_boot |=
+			!(I915_READ(MIPI_DEVICE_READY(port)) & DEVICE_READY);
 	}
 
 	return cold_boot;
@@ -399,9 +405,11 @@ static void glk_dsi_device_ready(struct intel_encoder *encoder)
 
 	/* Wait for MIPI PHY status bit to set */
 	for_each_dsi_port(port, intel_dsi->ports) {
-		if (intel_wait_for_register(dev_priv,
-				MIPI_CTRL(port), GLK_PHY_STATUS_PORT_READY,
-				GLK_PHY_STATUS_PORT_READY, 20))
+		if (intel_wait_for_register(&dev_priv->uncore,
+					    MIPI_CTRL(port),
+					    GLK_PHY_STATUS_PORT_READY,
+					    GLK_PHY_STATUS_PORT_READY,
+					    20))
 			DRM_ERROR("PHY is not ON\n");
 	}
 
@@ -425,8 +433,11 @@ static void glk_dsi_device_ready(struct intel_encoder *encoder)
 			I915_WRITE(MIPI_DEVICE_READY(port), val);
 
 			/* Wait for ULPS active */
-			if (intel_wait_for_register(dev_priv,
-				MIPI_CTRL(port), GLK_ULPS_NOT_ACTIVE, 0, 20))
+			if (intel_wait_for_register(&dev_priv->uncore,
+						    MIPI_CTRL(port),
+						    GLK_ULPS_NOT_ACTIVE,
+						    0,
+						    20))
 				DRM_ERROR("ULPS not active\n");
 
 			/* Exit ULPS */
@@ -449,17 +460,21 @@ static void glk_dsi_device_ready(struct intel_encoder *encoder)
 
 	/* Wait for Stop state */
 	for_each_dsi_port(port, intel_dsi->ports) {
-		if (intel_wait_for_register(dev_priv,
-				MIPI_CTRL(port), GLK_DATA_LANE_STOP_STATE,
-				GLK_DATA_LANE_STOP_STATE, 20))
+		if (intel_wait_for_register(&dev_priv->uncore,
+					    MIPI_CTRL(port),
+					    GLK_DATA_LANE_STOP_STATE,
+					    GLK_DATA_LANE_STOP_STATE,
+					    20))
 			DRM_ERROR("Date lane not in STOP state\n");
 	}
 
 	/* Wait for AFE LATCH */
 	for_each_dsi_port(port, intel_dsi->ports) {
-		if (intel_wait_for_register(dev_priv,
-				BXT_MIPI_PORT_CTRL(port), AFE_LATCHOUT,
-				AFE_LATCHOUT, 20))
+		if (intel_wait_for_register(&dev_priv->uncore,
+					    BXT_MIPI_PORT_CTRL(port),
+					    AFE_LATCHOUT,
+					    AFE_LATCHOUT,
+					    20))
 			DRM_ERROR("D-PHY not entering LP-11 state\n");
 	}
 }
@@ -559,7 +574,7 @@ static void glk_dsi_enter_low_power_mode(struct intel_encoder *encoder)
 
 	/* Wait for MIPI PHY status bit to unset */
 	for_each_dsi_port(port, intel_dsi->ports) {
-		if (intel_wait_for_register(dev_priv,
+		if (intel_wait_for_register(&dev_priv->uncore,
 					    MIPI_CTRL(port),
 					    GLK_PHY_STATUS_PORT_READY, 0, 20))
 			DRM_ERROR("PHY is not turning OFF\n");
@@ -567,7 +582,7 @@ static void glk_dsi_enter_low_power_mode(struct intel_encoder *encoder)
 
 	/* Wait for Pwr ACK bit to unset */
 	for_each_dsi_port(port, intel_dsi->ports) {
-		if (intel_wait_for_register(dev_priv,
+		if (intel_wait_for_register(&dev_priv->uncore,
 					    MIPI_CTRL(port),
 					    GLK_MIPIIO_PORT_POWERED, 0, 20))
 			DRM_ERROR("MIPI IO Port is not powergated\n");
@@ -588,7 +603,7 @@ static void glk_dsi_disable_mipi_io(struct intel_encoder *encoder)
 
 	/* Wait for MIPI PHY status bit to unset */
 	for_each_dsi_port(port, intel_dsi->ports) {
-		if (intel_wait_for_register(dev_priv,
+		if (intel_wait_for_register(&dev_priv->uncore,
 					    MIPI_CTRL(port),
 					    GLK_PHY_STATUS_PORT_READY, 0, 20))
 			DRM_ERROR("PHY is not turning OFF\n");
@@ -638,7 +653,7 @@ static void vlv_dsi_clear_device_ready(struct intel_encoder *encoder)
 		 * Port A only. MIPI Port C has no similar bit for checking.
 		 */
 		if ((IS_GEN9_LP(dev_priv) || port == PORT_A) &&
-		    intel_wait_for_register(dev_priv,
+		    intel_wait_for_register(&dev_priv->uncore,
 					    port_ctrl, AFE_LATCHOUT, 0,
 					    30))
 			DRM_ERROR("DSI LP not going Low\n");
@@ -1682,7 +1697,7 @@ void vlv_dsi_init(struct drm_i915_private *dev_priv)
 	struct drm_encoder *encoder;
 	struct intel_connector *intel_connector;
 	struct drm_connector *connector;
-	struct drm_display_mode *scan, *fixed_mode = NULL;
+	struct drm_display_mode *fixed_mode;
 	enum port port;
 
 	DRM_DEBUG_KMS("\n");
@@ -1793,13 +1808,7 @@ void vlv_dsi_init(struct drm_i915_private *dev_priv)
 	intel_connector_attach_encoder(intel_connector, intel_encoder);
 
 	mutex_lock(&dev->mode_config.mutex);
-	intel_dsi_vbt_get_modes(intel_dsi);
-	list_for_each_entry(scan, &connector->probed_modes, head) {
-		if ((scan->type & DRM_MODE_TYPE_PREFERRED)) {
-			fixed_mode = drm_mode_duplicate(dev, scan);
-			break;
-		}
-	}
+	fixed_mode = intel_panel_vbt_fixed_mode(intel_connector);
 	mutex_unlock(&dev->mode_config.mutex);
 
 	if (!fixed_mode) {
@@ -1807,9 +1816,6 @@ void vlv_dsi_init(struct drm_i915_private *dev_priv)
 		goto err;
 	}
 
-	connector->display_info.width_mm = fixed_mode->width_mm;
-	connector->display_info.height_mm = fixed_mode->height_mm;
-
 	intel_panel_init(&intel_connector->panel, fixed_mode, NULL);
 	intel_panel_setup_backlight(connector, INVALID_PIPE);
 
diff --git a/drivers/gpu/drm/i915/vlv_dsi_pll.c b/drivers/gpu/drm/i915/vlv_dsi_pll.c
index 954d5a8c4fa7..5e7b1fb2db5d 100644
--- a/drivers/gpu/drm/i915/vlv_dsi_pll.c
+++ b/drivers/gpu/drm/i915/vlv_dsi_pll.c
@@ -244,7 +244,7 @@ void bxt_dsi_pll_disable(struct intel_encoder *encoder)
 	 * PLL lock should deassert within 200us.
 	 * Wait up to 1ms before timing out.
 	 */
-	if (intel_wait_for_register(dev_priv,
+	if (intel_wait_for_register(&dev_priv->uncore,
 				    BXT_DSI_PLL_ENABLE,
 				    BXT_DSI_PLL_LOCKED,
 				    0,
@@ -528,7 +528,7 @@ void bxt_dsi_pll_enable(struct intel_encoder *encoder,
 	I915_WRITE(BXT_DSI_PLL_ENABLE, val);
 
 	/* Timeout and fail if PLL not locked */
-	if (intel_wait_for_register(dev_priv,
+	if (intel_wait_for_register(&dev_priv->uncore,
 				    BXT_DSI_PLL_ENABLE,
 				    BXT_DSI_PLL_LOCKED,
 				    BXT_DSI_PLL_LOCKED,
diff --git a/drivers/gpu/drm/imx/imx-drm-core.c b/drivers/gpu/drm/imx/imx-drm-core.c
index c935cbe059a7..3e8bece620df 100644
--- a/drivers/gpu/drm/imx/imx-drm-core.c
+++ b/drivers/gpu/drm/imx/imx-drm-core.c
@@ -185,7 +185,7 @@ static int compare_of(struct device *dev, void *data)
 	}
 
 	/* Special case for LDB, one device for two channels */
-	if (of_node_cmp(np->name, "lvds-channel") == 0) {
+	if (of_node_name_eq(np, "lvds-channel")) {
 		np = of_get_parent(np);
 		of_node_put(np);
 	}
diff --git a/drivers/gpu/drm/imx/ipuv3-crtc.c b/drivers/gpu/drm/imx/ipuv3-crtc.c
index 54011df8c2e8..9cc1d678674f 100644
--- a/drivers/gpu/drm/imx/ipuv3-crtc.c
+++ b/drivers/gpu/drm/imx/ipuv3-crtc.c
@@ -295,7 +295,7 @@ static void ipu_crtc_mode_set_nofb(struct drm_crtc *crtc)
 	sig_cfg.enable_pol = !(imx_crtc_state->bus_flags & DRM_BUS_FLAG_DE_LOW);
 	/* Default to driving pixel data on negative clock edges */
 	sig_cfg.clk_pol = !!(imx_crtc_state->bus_flags &
-			     DRM_BUS_FLAG_PIXDATA_POSEDGE);
+			     DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE);
 	sig_cfg.bus_format = imx_crtc_state->bus_format;
 	sig_cfg.v_to_h_sync = 0;
 	sig_cfg.hsync_pin = imx_crtc_state->di_hsync_pin;
diff --git a/drivers/gpu/drm/lima/Kconfig b/drivers/gpu/drm/lima/Kconfig
new file mode 100644
index 000000000000..bb4ddc6bb0a6
--- /dev/null
+++ b/drivers/gpu/drm/lima/Kconfig
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-2.0 OR MIT
+# Copyright 2017-2019 Qiang Yu <yuq825@gmail.com>
+
+config DRM_LIMA
+       tristate "LIMA (DRM support for ARM Mali 400/450 GPU)"
+       depends on DRM
+       depends on ARM || ARM64 || COMPILE_TEST
+       depends on MMU
+       depends on COMMON_CLK
+       depends on OF
+       select DRM_SCHED
+       help
+         DRM driver for ARM Mali 400/450 GPUs.
diff --git a/drivers/gpu/drm/lima/Makefile b/drivers/gpu/drm/lima/Makefile
new file mode 100644
index 000000000000..38cc70281ba5
--- /dev/null
+++ b/drivers/gpu/drm/lima/Makefile
@@ -0,0 +1,21 @@
+# SPDX-License-Identifier: GPL-2.0 OR MIT
+# Copyright 2017-2019 Qiang Yu <yuq825@gmail.com>
+
+lima-y := \
+	lima_drv.o \
+	lima_device.o \
+	lima_pmu.o \
+	lima_l2_cache.o \
+	lima_mmu.o \
+	lima_gp.o \
+	lima_pp.o \
+	lima_gem.o \
+	lima_vm.o \
+	lima_sched.o \
+	lima_ctx.o \
+	lima_gem_prime.o \
+	lima_dlbu.o \
+	lima_bcast.o \
+	lima_object.o
+
+obj-$(CONFIG_DRM_LIMA) += lima.o
diff --git a/drivers/gpu/drm/lima/lima_bcast.c b/drivers/gpu/drm/lima/lima_bcast.c
new file mode 100644
index 000000000000..288398027bfa
--- /dev/null
+++ b/drivers/gpu/drm/lima/lima_bcast.c
@@ -0,0 +1,47 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/* Copyright 2018-2019 Qiang Yu <yuq825@gmail.com> */
+
+#include <linux/io.h>
+#include <linux/device.h>
+
+#include "lima_device.h"
+#include "lima_bcast.h"
+#include "lima_regs.h"
+
+#define bcast_write(reg, data) writel(data, ip->iomem + reg)
+#define bcast_read(reg) readl(ip->iomem + reg)
+
+void lima_bcast_enable(struct lima_device *dev, int num_pp)
+{
+	struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp;
+	struct lima_ip *ip = dev->ip + lima_ip_bcast;
+	int i, mask = bcast_read(LIMA_BCAST_BROADCAST_MASK) & 0xffff0000;
+
+	for (i = 0; i < num_pp; i++) {
+		struct lima_ip *pp = pipe->processor[i];
+
+		mask |= 1 << (pp->id - lima_ip_pp0);
+	}
+
+	bcast_write(LIMA_BCAST_BROADCAST_MASK, mask);
+}
+
+int lima_bcast_init(struct lima_ip *ip)
+{
+	int i, mask = 0;
+
+	for (i = lima_ip_pp0; i <= lima_ip_pp7; i++) {
+		if (ip->dev->ip[i].present)
+			mask |= 1 << (i - lima_ip_pp0);
+	}
+
+	bcast_write(LIMA_BCAST_BROADCAST_MASK, mask << 16);
+	bcast_write(LIMA_BCAST_INTERRUPT_MASK, mask);
+	return 0;
+}
+
+void lima_bcast_fini(struct lima_ip *ip)
+{
+
+}
+
diff --git a/drivers/gpu/drm/lima/lima_bcast.h b/drivers/gpu/drm/lima/lima_bcast.h
new file mode 100644
index 000000000000..c47e58563d0a
--- /dev/null
+++ b/drivers/gpu/drm/lima/lima_bcast.h
@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+/* Copyright 2018-2019 Qiang Yu <yuq825@gmail.com> */
+
+#ifndef __LIMA_BCAST_H__
+#define __LIMA_BCAST_H__
+
+struct lima_ip;
+
+int lima_bcast_init(struct lima_ip *ip);
+void lima_bcast_fini(struct lima_ip *ip);
+
+void lima_bcast_enable(struct lima_device *dev, int num_pp);
+
+#endif
diff --git a/drivers/gpu/drm/lima/lima_ctx.c b/drivers/gpu/drm/lima/lima_ctx.c
new file mode 100644
index 000000000000..22fff6caa961
--- /dev/null
+++ b/drivers/gpu/drm/lima/lima_ctx.c
@@ -0,0 +1,98 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/* Copyright 2018-2019 Qiang Yu <yuq825@gmail.com> */
+
+#include <linux/slab.h>
+
+#include "lima_device.h"
+#include "lima_ctx.h"
+
+int lima_ctx_create(struct lima_device *dev, struct lima_ctx_mgr *mgr, u32 *id)
+{
+	struct lima_ctx *ctx;
+	int i, err;
+
+	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
+	if (!ctx)
+		return -ENOMEM;
+	ctx->dev = dev;
+	kref_init(&ctx->refcnt);
+
+	for (i = 0; i < lima_pipe_num; i++) {
+		err = lima_sched_context_init(dev->pipe + i, ctx->context + i, &ctx->guilty);
+		if (err)
+			goto err_out0;
+	}
+
+	err = xa_alloc(&mgr->handles, id, ctx, xa_limit_32b, GFP_KERNEL);
+	if (err < 0)
+		goto err_out0;
+
+	return 0;
+
+err_out0:
+	for (i--; i >= 0; i--)
+		lima_sched_context_fini(dev->pipe + i, ctx->context + i);
+	kfree(ctx);
+	return err;
+}
+
+static void lima_ctx_do_release(struct kref *ref)
+{
+	struct lima_ctx *ctx = container_of(ref, struct lima_ctx, refcnt);
+	int i;
+
+	for (i = 0; i < lima_pipe_num; i++)
+		lima_sched_context_fini(ctx->dev->pipe + i, ctx->context + i);
+	kfree(ctx);
+}
+
+int lima_ctx_free(struct lima_ctx_mgr *mgr, u32 id)
+{
+	struct lima_ctx *ctx;
+	int ret = 0;
+
+	mutex_lock(&mgr->lock);
+	ctx = xa_erase(&mgr->handles, id);
+	if (ctx)
+		kref_put(&ctx->refcnt, lima_ctx_do_release);
+	else
+		ret = -EINVAL;
+	mutex_unlock(&mgr->lock);
+	return ret;
+}
+
+struct lima_ctx *lima_ctx_get(struct lima_ctx_mgr *mgr, u32 id)
+{
+	struct lima_ctx *ctx;
+
+	mutex_lock(&mgr->lock);
+	ctx = xa_load(&mgr->handles, id);
+	if (ctx)
+		kref_get(&ctx->refcnt);
+	mutex_unlock(&mgr->lock);
+	return ctx;
+}
+
+void lima_ctx_put(struct lima_ctx *ctx)
+{
+	kref_put(&ctx->refcnt, lima_ctx_do_release);
+}
+
+void lima_ctx_mgr_init(struct lima_ctx_mgr *mgr)
+{
+	mutex_init(&mgr->lock);
+	xa_init_flags(&mgr->handles, XA_FLAGS_ALLOC);
+}
+
+void lima_ctx_mgr_fini(struct lima_ctx_mgr *mgr)
+{
+	struct lima_ctx *ctx;
+	unsigned long id;
+
+	xa_for_each(&mgr->handles, id, ctx) {
+		kref_put(&ctx->refcnt, lima_ctx_do_release);
+	}
+
+	xa_destroy(&mgr->handles);
+	mutex_destroy(&mgr->lock);
+}
diff --git a/drivers/gpu/drm/lima/lima_ctx.h b/drivers/gpu/drm/lima/lima_ctx.h
new file mode 100644
index 000000000000..6154e5c9bfe4
--- /dev/null
+++ b/drivers/gpu/drm/lima/lima_ctx.h
@@ -0,0 +1,30 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+/* Copyright 2018-2019 Qiang Yu <yuq825@gmail.com> */
+
+#ifndef __LIMA_CTX_H__
+#define __LIMA_CTX_H__
+
+#include <linux/xarray.h>
+
+#include "lima_device.h"
+
+struct lima_ctx {
+	struct kref refcnt;
+	struct lima_device *dev;
+	struct lima_sched_context context[lima_pipe_num];
+	atomic_t guilty;
+};
+
+struct lima_ctx_mgr {
+	struct mutex lock;
+	struct xarray handles;
+};
+
+int lima_ctx_create(struct lima_device *dev, struct lima_ctx_mgr *mgr, u32 *id);
+int lima_ctx_free(struct lima_ctx_mgr *mgr, u32 id);
+struct lima_ctx *lima_ctx_get(struct lima_ctx_mgr *mgr, u32 id);
+void lima_ctx_put(struct lima_ctx *ctx);
+void lima_ctx_mgr_init(struct lima_ctx_mgr *mgr);
+void lima_ctx_mgr_fini(struct lima_ctx_mgr *mgr);
+
+#endif
diff --git a/drivers/gpu/drm/lima/lima_device.c b/drivers/gpu/drm/lima/lima_device.c
new file mode 100644
index 000000000000..570d0e93f9a9
--- /dev/null
+++ b/drivers/gpu/drm/lima/lima_device.c
@@ -0,0 +1,385 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
+
+#include <linux/regulator/consumer.h>
+#include <linux/reset.h>
+#include <linux/clk.h>
+#include <linux/dma-mapping.h>
+#include <linux/platform_device.h>
+
+#include "lima_device.h"
+#include "lima_gp.h"
+#include "lima_pp.h"
+#include "lima_mmu.h"
+#include "lima_pmu.h"
+#include "lima_l2_cache.h"
+#include "lima_dlbu.h"
+#include "lima_bcast.h"
+#include "lima_vm.h"
+
+struct lima_ip_desc {
+	char *name;
+	char *irq_name;
+	bool must_have[lima_gpu_num];
+	int offset[lima_gpu_num];
+
+	int (*init)(struct lima_ip *ip);
+	void (*fini)(struct lima_ip *ip);
+};
+
+#define LIMA_IP_DESC(ipname, mst0, mst1, off0, off1, func, irq) \
+	[lima_ip_##ipname] = { \
+		.name = #ipname, \
+		.irq_name = irq, \
+		.must_have = { \
+			[lima_gpu_mali400] = mst0, \
+			[lima_gpu_mali450] = mst1, \
+		}, \
+		.offset = { \
+			[lima_gpu_mali400] = off0, \
+			[lima_gpu_mali450] = off1, \
+		}, \
+		.init = lima_##func##_init, \
+		.fini = lima_##func##_fini, \
+	}
+
+static struct lima_ip_desc lima_ip_desc[lima_ip_num] = {
+	LIMA_IP_DESC(pmu,         false, false, 0x02000, 0x02000, pmu,      "pmu"),
+	LIMA_IP_DESC(l2_cache0,   true,  true,  0x01000, 0x10000, l2_cache, NULL),
+	LIMA_IP_DESC(l2_cache1,   false, true,  -1,      0x01000, l2_cache, NULL),
+	LIMA_IP_DESC(l2_cache2,   false, false, -1,      0x11000, l2_cache, NULL),
+	LIMA_IP_DESC(gp,          true,  true,  0x00000, 0x00000, gp,       "gp"),
+	LIMA_IP_DESC(pp0,         true,  true,  0x08000, 0x08000, pp,       "pp0"),
+	LIMA_IP_DESC(pp1,         false, false, 0x0A000, 0x0A000, pp,       "pp1"),
+	LIMA_IP_DESC(pp2,         false, false, 0x0C000, 0x0C000, pp,       "pp2"),
+	LIMA_IP_DESC(pp3,         false, false, 0x0E000, 0x0E000, pp,       "pp3"),
+	LIMA_IP_DESC(pp4,         false, false, -1,      0x28000, pp,       "pp4"),
+	LIMA_IP_DESC(pp5,         false, false, -1,      0x2A000, pp,       "pp5"),
+	LIMA_IP_DESC(pp6,         false, false, -1,      0x2C000, pp,       "pp6"),
+	LIMA_IP_DESC(pp7,         false, false, -1,      0x2E000, pp,       "pp7"),
+	LIMA_IP_DESC(gpmmu,       true,  true,  0x03000, 0x03000, mmu,      "gpmmu"),
+	LIMA_IP_DESC(ppmmu0,      true,  true,  0x04000, 0x04000, mmu,      "ppmmu0"),
+	LIMA_IP_DESC(ppmmu1,      false, false, 0x05000, 0x05000, mmu,      "ppmmu1"),
+	LIMA_IP_DESC(ppmmu2,      false, false, 0x06000, 0x06000, mmu,      "ppmmu2"),
+	LIMA_IP_DESC(ppmmu3,      false, false, 0x07000, 0x07000, mmu,      "ppmmu3"),
+	LIMA_IP_DESC(ppmmu4,      false, false, -1,      0x1C000, mmu,      "ppmmu4"),
+	LIMA_IP_DESC(ppmmu5,      false, false, -1,      0x1D000, mmu,      "ppmmu5"),
+	LIMA_IP_DESC(ppmmu6,      false, false, -1,      0x1E000, mmu,      "ppmmu6"),
+	LIMA_IP_DESC(ppmmu7,      false, false, -1,      0x1F000, mmu,      "ppmmu7"),
+	LIMA_IP_DESC(dlbu,        false, true,  -1,      0x14000, dlbu,     NULL),
+	LIMA_IP_DESC(bcast,       false, true,  -1,      0x13000, bcast,    NULL),
+	LIMA_IP_DESC(pp_bcast,    false, true,  -1,      0x16000, pp_bcast, "pp"),
+	LIMA_IP_DESC(ppmmu_bcast, false, true,  -1,      0x15000, mmu,      NULL),
+};
+
+const char *lima_ip_name(struct lima_ip *ip)
+{
+	return lima_ip_desc[ip->id].name;
+}
+
+static int lima_clk_init(struct lima_device *dev)
+{
+	int err;
+	unsigned long bus_rate, gpu_rate;
+
+	dev->clk_bus = devm_clk_get(dev->dev, "bus");
+	if (IS_ERR(dev->clk_bus)) {
+		dev_err(dev->dev, "get bus clk failed %ld\n", PTR_ERR(dev->clk_bus));
+		return PTR_ERR(dev->clk_bus);
+	}
+
+	dev->clk_gpu = devm_clk_get(dev->dev, "core");
+	if (IS_ERR(dev->clk_gpu)) {
+		dev_err(dev->dev, "get core clk failed %ld\n", PTR_ERR(dev->clk_gpu));
+		return PTR_ERR(dev->clk_gpu);
+	}
+
+	bus_rate = clk_get_rate(dev->clk_bus);
+	dev_info(dev->dev, "bus rate = %lu\n", bus_rate);
+
+	gpu_rate = clk_get_rate(dev->clk_gpu);
+	dev_info(dev->dev, "mod rate = %lu", gpu_rate);
+
+	err = clk_prepare_enable(dev->clk_bus);
+	if (err)
+		return err;
+
+	err = clk_prepare_enable(dev->clk_gpu);
+	if (err)
+		goto error_out0;
+
+	dev->reset = devm_reset_control_get_optional(dev->dev, NULL);
+	if (IS_ERR(dev->reset)) {
+		err = PTR_ERR(dev->reset);
+		goto error_out1;
+	} else if (dev->reset != NULL) {
+		err = reset_control_deassert(dev->reset);
+		if (err)
+			goto error_out1;
+	}
+
+	return 0;
+
+error_out1:
+	clk_disable_unprepare(dev->clk_gpu);
+error_out0:
+	clk_disable_unprepare(dev->clk_bus);
+	return err;
+}
+
+static void lima_clk_fini(struct lima_device *dev)
+{
+	if (dev->reset != NULL)
+		reset_control_assert(dev->reset);
+	clk_disable_unprepare(dev->clk_gpu);
+	clk_disable_unprepare(dev->clk_bus);
+}
+
+static int lima_regulator_init(struct lima_device *dev)
+{
+	int ret;
+
+	dev->regulator = devm_regulator_get_optional(dev->dev, "mali");
+	if (IS_ERR(dev->regulator)) {
+		ret = PTR_ERR(dev->regulator);
+		dev->regulator = NULL;
+		if (ret == -ENODEV)
+			return 0;
+		dev_err(dev->dev, "failed to get regulator: %d\n", ret);
+		return ret;
+	}
+
+	ret = regulator_enable(dev->regulator);
+	if (ret < 0) {
+		dev_err(dev->dev, "failed to enable regulator: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void lima_regulator_fini(struct lima_device *dev)
+{
+	if (dev->regulator)
+		regulator_disable(dev->regulator);
+}
+
+static int lima_init_ip(struct lima_device *dev, int index)
+{
+	struct lima_ip_desc *desc = lima_ip_desc + index;
+	struct lima_ip *ip = dev->ip + index;
+	int offset = desc->offset[dev->id];
+	bool must = desc->must_have[dev->id];
+	int err;
+
+	if (offset < 0)
+		return 0;
+
+	ip->dev = dev;
+	ip->id = index;
+	ip->iomem = dev->iomem + offset;
+	if (desc->irq_name) {
+		err = platform_get_irq_byname(dev->pdev, desc->irq_name);
+		if (err < 0)
+			goto out;
+		ip->irq = err;
+	}
+
+	err = desc->init(ip);
+	if (!err) {
+		ip->present = true;
+		return 0;
+	}
+
+out:
+	return must ? err : 0;
+}
+
+static void lima_fini_ip(struct lima_device *ldev, int index)
+{
+	struct lima_ip_desc *desc = lima_ip_desc + index;
+	struct lima_ip *ip = ldev->ip + index;
+
+	if (ip->present)
+		desc->fini(ip);
+}
+
+static int lima_init_gp_pipe(struct lima_device *dev)
+{
+	struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_gp;
+	int err;
+
+	err = lima_sched_pipe_init(pipe, "gp");
+	if (err)
+		return err;
+
+	pipe->l2_cache[pipe->num_l2_cache++] = dev->ip + lima_ip_l2_cache0;
+	pipe->mmu[pipe->num_mmu++] = dev->ip + lima_ip_gpmmu;
+	pipe->processor[pipe->num_processor++] = dev->ip + lima_ip_gp;
+
+	err = lima_gp_pipe_init(dev);
+	if (err) {
+		lima_sched_pipe_fini(pipe);
+		return err;
+	}
+
+	return 0;
+}
+
+static void lima_fini_gp_pipe(struct lima_device *dev)
+{
+	struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_gp;
+
+	lima_gp_pipe_fini(dev);
+	lima_sched_pipe_fini(pipe);
+}
+
+static int lima_init_pp_pipe(struct lima_device *dev)
+{
+	struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp;
+	int err, i;
+
+	err = lima_sched_pipe_init(pipe, "pp");
+	if (err)
+		return err;
+
+	for (i = 0; i < LIMA_SCHED_PIPE_MAX_PROCESSOR; i++) {
+		struct lima_ip *pp = dev->ip + lima_ip_pp0 + i;
+		struct lima_ip *ppmmu = dev->ip + lima_ip_ppmmu0 + i;
+		struct lima_ip *l2_cache;
+
+		if (dev->id == lima_gpu_mali400)
+			l2_cache = dev->ip + lima_ip_l2_cache0;
+		else
+			l2_cache = dev->ip + lima_ip_l2_cache1 + (i >> 2);
+
+		if (pp->present && ppmmu->present && l2_cache->present) {
+			pipe->mmu[pipe->num_mmu++] = ppmmu;
+			pipe->processor[pipe->num_processor++] = pp;
+			if (!pipe->l2_cache[i >> 2])
+				pipe->l2_cache[pipe->num_l2_cache++] = l2_cache;
+		}
+	}
+
+	if (dev->ip[lima_ip_bcast].present) {
+		pipe->bcast_processor = dev->ip + lima_ip_pp_bcast;
+		pipe->bcast_mmu = dev->ip + lima_ip_ppmmu_bcast;
+	}
+
+	err = lima_pp_pipe_init(dev);
+	if (err) {
+		lima_sched_pipe_fini(pipe);
+		return err;
+	}
+
+	return 0;
+}
+
+static void lima_fini_pp_pipe(struct lima_device *dev)
+{
+	struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp;
+
+	lima_pp_pipe_fini(dev);
+	lima_sched_pipe_fini(pipe);
+}
+
+int lima_device_init(struct lima_device *ldev)
+{
+	int err, i;
+	struct resource *res;
+
+	dma_set_coherent_mask(ldev->dev, DMA_BIT_MASK(32));
+
+	err = lima_clk_init(ldev);
+	if (err) {
+		dev_err(ldev->dev, "clk init fail %d\n", err);
+		return err;
+	}
+
+	err = lima_regulator_init(ldev);
+	if (err) {
+		dev_err(ldev->dev, "regulator init fail %d\n", err);
+		goto err_out0;
+	}
+
+	ldev->empty_vm = lima_vm_create(ldev);
+	if (!ldev->empty_vm) {
+		err = -ENOMEM;
+		goto err_out1;
+	}
+
+	ldev->va_start = 0;
+	if (ldev->id == lima_gpu_mali450) {
+		ldev->va_end = LIMA_VA_RESERVE_START;
+		ldev->dlbu_cpu = dma_alloc_wc(
+			ldev->dev, LIMA_PAGE_SIZE,
+			&ldev->dlbu_dma, GFP_KERNEL);
+		if (!ldev->dlbu_cpu) {
+			err = -ENOMEM;
+			goto err_out2;
+		}
+	} else
+		ldev->va_end = LIMA_VA_RESERVE_END;
+
+	res = platform_get_resource(ldev->pdev, IORESOURCE_MEM, 0);
+	ldev->iomem = devm_ioremap_resource(ldev->dev, res);
+	if (IS_ERR(ldev->iomem)) {
+		dev_err(ldev->dev, "fail to ioremap iomem\n");
+		err = PTR_ERR(ldev->iomem);
+		goto err_out3;
+	}
+
+	for (i = 0; i < lima_ip_num; i++) {
+		err = lima_init_ip(ldev, i);
+		if (err)
+			goto err_out4;
+	}
+
+	err = lima_init_gp_pipe(ldev);
+	if (err)
+		goto err_out4;
+
+	err = lima_init_pp_pipe(ldev);
+	if (err)
+		goto err_out5;
+
+	return 0;
+
+err_out5:
+	lima_fini_gp_pipe(ldev);
+err_out4:
+	while (--i >= 0)
+		lima_fini_ip(ldev, i);
+err_out3:
+	if (ldev->dlbu_cpu)
+		dma_free_wc(ldev->dev, LIMA_PAGE_SIZE,
+			    ldev->dlbu_cpu, ldev->dlbu_dma);
+err_out2:
+	lima_vm_put(ldev->empty_vm);
+err_out1:
+	lima_regulator_fini(ldev);
+err_out0:
+	lima_clk_fini(ldev);
+	return err;
+}
+
+void lima_device_fini(struct lima_device *ldev)
+{
+	int i;
+
+	lima_fini_pp_pipe(ldev);
+	lima_fini_gp_pipe(ldev);
+
+	for (i = lima_ip_num - 1; i >= 0; i--)
+		lima_fini_ip(ldev, i);
+
+	if (ldev->dlbu_cpu)
+		dma_free_wc(ldev->dev, LIMA_PAGE_SIZE,
+			    ldev->dlbu_cpu, ldev->dlbu_dma);
+
+	lima_vm_put(ldev->empty_vm);
+
+	lima_regulator_fini(ldev);
+
+	lima_clk_fini(ldev);
+}
diff --git a/drivers/gpu/drm/lima/lima_device.h b/drivers/gpu/drm/lima/lima_device.h
new file mode 100644
index 000000000000..31158d86271c
--- /dev/null
+++ b/drivers/gpu/drm/lima/lima_device.h
@@ -0,0 +1,131 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+/* Copyright 2018-2019 Qiang Yu <yuq825@gmail.com> */
+
+#ifndef __LIMA_DEVICE_H__
+#define __LIMA_DEVICE_H__
+
+#include <drm/drm_device.h>
+#include <linux/delay.h>
+
+#include "lima_sched.h"
+
+enum lima_gpu_id {
+	lima_gpu_mali400 = 0,
+	lima_gpu_mali450,
+	lima_gpu_num,
+};
+
+enum lima_ip_id {
+	lima_ip_pmu,
+	lima_ip_gpmmu,
+	lima_ip_ppmmu0,
+	lima_ip_ppmmu1,
+	lima_ip_ppmmu2,
+	lima_ip_ppmmu3,
+	lima_ip_ppmmu4,
+	lima_ip_ppmmu5,
+	lima_ip_ppmmu6,
+	lima_ip_ppmmu7,
+	lima_ip_gp,
+	lima_ip_pp0,
+	lima_ip_pp1,
+	lima_ip_pp2,
+	lima_ip_pp3,
+	lima_ip_pp4,
+	lima_ip_pp5,
+	lima_ip_pp6,
+	lima_ip_pp7,
+	lima_ip_l2_cache0,
+	lima_ip_l2_cache1,
+	lima_ip_l2_cache2,
+	lima_ip_dlbu,
+	lima_ip_bcast,
+	lima_ip_pp_bcast,
+	lima_ip_ppmmu_bcast,
+	lima_ip_num,
+};
+
+struct lima_device;
+
+struct lima_ip {
+	struct lima_device *dev;
+	enum lima_ip_id id;
+	bool present;
+
+	void __iomem *iomem;
+	int irq;
+
+	union {
+		/* gp/pp */
+		bool async_reset;
+		/* l2 cache */
+		spinlock_t lock;
+	} data;
+};
+
+enum lima_pipe_id {
+	lima_pipe_gp,
+	lima_pipe_pp,
+	lima_pipe_num,
+};
+
+struct lima_device {
+	struct device *dev;
+	struct drm_device *ddev;
+	struct platform_device *pdev;
+
+	enum lima_gpu_id id;
+	u32 gp_version;
+	u32 pp_version;
+	int num_pp;
+
+	void __iomem *iomem;
+	struct clk *clk_bus;
+	struct clk *clk_gpu;
+	struct reset_control *reset;
+	struct regulator *regulator;
+
+	struct lima_ip ip[lima_ip_num];
+	struct lima_sched_pipe pipe[lima_pipe_num];
+
+	struct lima_vm *empty_vm;
+	uint64_t va_start;
+	uint64_t va_end;
+
+	u32 *dlbu_cpu;
+	dma_addr_t dlbu_dma;
+};
+
+static inline struct lima_device *
+to_lima_dev(struct drm_device *dev)
+{
+	return dev->dev_private;
+}
+
+int lima_device_init(struct lima_device *ldev);
+void lima_device_fini(struct lima_device *ldev);
+
+const char *lima_ip_name(struct lima_ip *ip);
+
+typedef int (*lima_poll_func_t)(struct lima_ip *);
+
+static inline int lima_poll_timeout(struct lima_ip *ip, lima_poll_func_t func,
+				    int sleep_us, int timeout_us)
+{
+	ktime_t timeout = ktime_add_us(ktime_get(), timeout_us);
+
+	might_sleep_if(sleep_us);
+	while (1) {
+		if (func(ip))
+			return 0;
+
+		if (timeout_us && ktime_compare(ktime_get(), timeout) > 0)
+			return -ETIMEDOUT;
+
+		if (sleep_us)
+			usleep_range((sleep_us >> 2) + 1, sleep_us);
+	}
+	return 0;
+}
+
+#endif
diff --git a/drivers/gpu/drm/lima/lima_dlbu.c b/drivers/gpu/drm/lima/lima_dlbu.c
new file mode 100644
index 000000000000..8399ceffb94b
--- /dev/null
+++ b/drivers/gpu/drm/lima/lima_dlbu.c
@@ -0,0 +1,58 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/* Copyright 2018-2019 Qiang Yu <yuq825@gmail.com> */
+
+#include <linux/io.h>
+#include <linux/device.h>
+
+#include "lima_device.h"
+#include "lima_dlbu.h"
+#include "lima_vm.h"
+#include "lima_regs.h"
+
+#define dlbu_write(reg, data) writel(data, ip->iomem + reg)
+#define dlbu_read(reg) readl(ip->iomem + reg)
+
+void lima_dlbu_enable(struct lima_device *dev, int num_pp)
+{
+	struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp;
+	struct lima_ip *ip = dev->ip + lima_ip_dlbu;
+	int i, mask = 0;
+
+	for (i = 0; i < num_pp; i++) {
+		struct lima_ip *pp = pipe->processor[i];
+
+		mask |= 1 << (pp->id - lima_ip_pp0);
+	}
+
+	dlbu_write(LIMA_DLBU_PP_ENABLE_MASK, mask);
+}
+
+void lima_dlbu_disable(struct lima_device *dev)
+{
+	struct lima_ip *ip = dev->ip + lima_ip_dlbu;
+
+	dlbu_write(LIMA_DLBU_PP_ENABLE_MASK, 0);
+}
+
+void lima_dlbu_set_reg(struct lima_ip *ip, u32 *reg)
+{
+	dlbu_write(LIMA_DLBU_TLLIST_VBASEADDR, reg[0]);
+	dlbu_write(LIMA_DLBU_FB_DIM, reg[1]);
+	dlbu_write(LIMA_DLBU_TLLIST_CONF, reg[2]);
+	dlbu_write(LIMA_DLBU_START_TILE_POS, reg[3]);
+}
+
+int lima_dlbu_init(struct lima_ip *ip)
+{
+	struct lima_device *dev = ip->dev;
+
+	dlbu_write(LIMA_DLBU_MASTER_TLLIST_PHYS_ADDR, dev->dlbu_dma | 1);
+	dlbu_write(LIMA_DLBU_MASTER_TLLIST_VADDR, LIMA_VA_RESERVE_DLBU);
+
+	return 0;
+}
+
+void lima_dlbu_fini(struct lima_ip *ip)
+{
+
+}
diff --git a/drivers/gpu/drm/lima/lima_dlbu.h b/drivers/gpu/drm/lima/lima_dlbu.h
new file mode 100644
index 000000000000..16f877984466
--- /dev/null
+++ b/drivers/gpu/drm/lima/lima_dlbu.h
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+/* Copyright 2018-2019 Qiang Yu <yuq825@gmail.com> */
+
+#ifndef __LIMA_DLBU_H__
+#define __LIMA_DLBU_H__
+
+struct lima_ip;
+struct lima_device;
+
+void lima_dlbu_enable(struct lima_device *dev, int num_pp);
+void lima_dlbu_disable(struct lima_device *dev);
+
+void lima_dlbu_set_reg(struct lima_ip *ip, u32 *reg);
+
+int lima_dlbu_init(struct lima_ip *ip);
+void lima_dlbu_fini(struct lima_ip *ip);
+
+#endif
diff --git a/drivers/gpu/drm/lima/lima_drv.c b/drivers/gpu/drm/lima/lima_drv.c
new file mode 100644
index 000000000000..f9a281a62083
--- /dev/null
+++ b/drivers/gpu/drm/lima/lima_drv.c
@@ -0,0 +1,376 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
+
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <linux/uaccess.h>
+#include <linux/slab.h>
+#include <drm/drm_ioctl.h>
+#include <drm/drm_drv.h>
+#include <drm/drm_prime.h>
+#include <drm/lima_drm.h>
+
+#include "lima_drv.h"
+#include "lima_gem.h"
+#include "lima_gem_prime.h"
+#include "lima_vm.h"
+
+int lima_sched_timeout_ms;
+
+MODULE_PARM_DESC(sched_timeout_ms, "task run timeout in ms (0 = no timeout (default))");
+module_param_named(sched_timeout_ms, lima_sched_timeout_ms, int, 0444);
+
+static int lima_ioctl_get_param(struct drm_device *dev, void *data, struct drm_file *file)
+{
+	struct drm_lima_get_param *args = data;
+	struct lima_device *ldev = to_lima_dev(dev);
+
+	if (args->pad)
+		return -EINVAL;
+
+	switch (args->param) {
+	case DRM_LIMA_PARAM_GPU_ID:
+		switch (ldev->id) {
+		case lima_gpu_mali400:
+			args->value = DRM_LIMA_PARAM_GPU_ID_MALI400;
+			break;
+		case lima_gpu_mali450:
+			args->value = DRM_LIMA_PARAM_GPU_ID_MALI450;
+			break;
+		default:
+			args->value = DRM_LIMA_PARAM_GPU_ID_UNKNOWN;
+			break;
+		}
+		break;
+
+	case DRM_LIMA_PARAM_NUM_PP:
+		args->value = ldev->pipe[lima_pipe_pp].num_processor;
+		break;
+
+	case DRM_LIMA_PARAM_GP_VERSION:
+		args->value = ldev->gp_version;
+		break;
+
+	case DRM_LIMA_PARAM_PP_VERSION:
+		args->value = ldev->pp_version;
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int lima_ioctl_gem_create(struct drm_device *dev, void *data, struct drm_file *file)
+{
+	struct drm_lima_gem_create *args = data;
+
+	if (args->pad)
+		return -EINVAL;
+
+	if (args->flags)
+		return -EINVAL;
+
+	if (args->size == 0)
+		return -EINVAL;
+
+	return lima_gem_create_handle(dev, file, args->size, args->flags, &args->handle);
+}
+
+static int lima_ioctl_gem_info(struct drm_device *dev, void *data, struct drm_file *file)
+{
+	struct drm_lima_gem_info *args = data;
+
+	return lima_gem_get_info(file, args->handle, &args->va, &args->offset);
+}
+
+static int lima_ioctl_gem_submit(struct drm_device *dev, void *data, struct drm_file *file)
+{
+	struct drm_lima_gem_submit *args = data;
+	struct lima_device *ldev = to_lima_dev(dev);
+	struct lima_drm_priv *priv = file->driver_priv;
+	struct drm_lima_gem_submit_bo *bos;
+	struct lima_sched_pipe *pipe;
+	struct lima_sched_task *task;
+	struct lima_ctx *ctx;
+	struct lima_submit submit = {0};
+	size_t size;
+	int err = 0;
+
+	if (args->pipe >= lima_pipe_num || args->nr_bos == 0)
+		return -EINVAL;
+
+	if (args->flags & ~(LIMA_SUBMIT_FLAG_EXPLICIT_FENCE))
+		return -EINVAL;
+
+	pipe = ldev->pipe + args->pipe;
+	if (args->frame_size != pipe->frame_size)
+		return -EINVAL;
+
+	bos = kvcalloc(args->nr_bos, sizeof(*submit.bos) + sizeof(*submit.lbos), GFP_KERNEL);
+	if (!bos)
+		return -ENOMEM;
+
+	size = args->nr_bos * sizeof(*submit.bos);
+	if (copy_from_user(bos, u64_to_user_ptr(args->bos), size)) {
+		err = -EFAULT;
+		goto out0;
+	}
+
+	task = kmem_cache_zalloc(pipe->task_slab, GFP_KERNEL);
+	if (!task) {
+		err = -ENOMEM;
+		goto out0;
+	}
+
+	task->frame = task + 1;
+	if (copy_from_user(task->frame, u64_to_user_ptr(args->frame), args->frame_size)) {
+		err = -EFAULT;
+		goto out1;
+	}
+
+	err = pipe->task_validate(pipe, task);
+	if (err)
+		goto out1;
+
+	ctx = lima_ctx_get(&priv->ctx_mgr, args->ctx);
+	if (!ctx) {
+		err = -ENOENT;
+		goto out1;
+	}
+
+	submit.pipe = args->pipe;
+	submit.bos = bos;
+	submit.lbos = (void *)bos + size;
+	submit.nr_bos = args->nr_bos;
+	submit.task = task;
+	submit.ctx = ctx;
+	submit.flags = args->flags;
+	submit.in_sync[0] = args->in_sync[0];
+	submit.in_sync[1] = args->in_sync[1];
+	submit.out_sync = args->out_sync;
+
+	err = lima_gem_submit(file, &submit);
+
+	lima_ctx_put(ctx);
+out1:
+	if (err)
+		kmem_cache_free(pipe->task_slab, task);
+out0:
+	kvfree(bos);
+	return err;
+}
+
+static int lima_ioctl_gem_wait(struct drm_device *dev, void *data, struct drm_file *file)
+{
+	struct drm_lima_gem_wait *args = data;
+
+	if (args->op & ~(LIMA_GEM_WAIT_READ|LIMA_GEM_WAIT_WRITE))
+		return -EINVAL;
+
+	return lima_gem_wait(file, args->handle, args->op, args->timeout_ns);
+}
+
+static int lima_ioctl_ctx_create(struct drm_device *dev, void *data, struct drm_file *file)
+{
+	struct drm_lima_ctx_create *args = data;
+	struct lima_drm_priv *priv = file->driver_priv;
+	struct lima_device *ldev = to_lima_dev(dev);
+
+	if (args->_pad)
+		return -EINVAL;
+
+	return lima_ctx_create(ldev, &priv->ctx_mgr, &args->id);
+}
+
+static int lima_ioctl_ctx_free(struct drm_device *dev, void *data, struct drm_file *file)
+{
+	struct drm_lima_ctx_create *args = data;
+	struct lima_drm_priv *priv = file->driver_priv;
+
+	if (args->_pad)
+		return -EINVAL;
+
+	return lima_ctx_free(&priv->ctx_mgr, args->id);
+}
+
+static int lima_drm_driver_open(struct drm_device *dev, struct drm_file *file)
+{
+	int err;
+	struct lima_drm_priv *priv;
+	struct lima_device *ldev = to_lima_dev(dev);
+
+	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->vm = lima_vm_create(ldev);
+	if (!priv->vm) {
+		err = -ENOMEM;
+		goto err_out0;
+	}
+
+	lima_ctx_mgr_init(&priv->ctx_mgr);
+
+	file->driver_priv = priv;
+	return 0;
+
+err_out0:
+	kfree(priv);
+	return err;
+}
+
+static void lima_drm_driver_postclose(struct drm_device *dev, struct drm_file *file)
+{
+	struct lima_drm_priv *priv = file->driver_priv;
+
+	lima_ctx_mgr_fini(&priv->ctx_mgr);
+	lima_vm_put(priv->vm);
+	kfree(priv);
+}
+
+static const struct drm_ioctl_desc lima_drm_driver_ioctls[] = {
+	DRM_IOCTL_DEF_DRV(LIMA_GET_PARAM, lima_ioctl_get_param, DRM_AUTH|DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(LIMA_GEM_CREATE, lima_ioctl_gem_create, DRM_AUTH|DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(LIMA_GEM_INFO, lima_ioctl_gem_info, DRM_AUTH|DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(LIMA_GEM_SUBMIT, lima_ioctl_gem_submit, DRM_AUTH|DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(LIMA_GEM_WAIT, lima_ioctl_gem_wait, DRM_AUTH|DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(LIMA_CTX_CREATE, lima_ioctl_ctx_create, DRM_AUTH|DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(LIMA_CTX_FREE, lima_ioctl_ctx_free, DRM_AUTH|DRM_RENDER_ALLOW),
+};
+
+static const struct file_operations lima_drm_driver_fops = {
+	.owner              = THIS_MODULE,
+	.open               = drm_open,
+	.release            = drm_release,
+	.unlocked_ioctl     = drm_ioctl,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl       = drm_compat_ioctl,
+#endif
+	.mmap               = lima_gem_mmap,
+};
+
+static struct drm_driver lima_drm_driver = {
+	.driver_features    = DRIVER_RENDER | DRIVER_GEM | DRIVER_PRIME | DRIVER_SYNCOBJ,
+	.open               = lima_drm_driver_open,
+	.postclose          = lima_drm_driver_postclose,
+	.ioctls             = lima_drm_driver_ioctls,
+	.num_ioctls         = ARRAY_SIZE(lima_drm_driver_ioctls),
+	.fops               = &lima_drm_driver_fops,
+	.gem_free_object_unlocked = lima_gem_free_object,
+	.gem_open_object    = lima_gem_object_open,
+	.gem_close_object   = lima_gem_object_close,
+	.gem_vm_ops         = &lima_gem_vm_ops,
+	.name               = "lima",
+	.desc               = "lima DRM",
+	.date               = "20190217",
+	.major              = 1,
+	.minor              = 0,
+	.patchlevel         = 0,
+
+	.prime_fd_to_handle = drm_gem_prime_fd_to_handle,
+	.gem_prime_import_sg_table = lima_gem_prime_import_sg_table,
+	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
+	.gem_prime_get_sg_table = lima_gem_prime_get_sg_table,
+	.gem_prime_mmap = lima_gem_prime_mmap,
+};
+
+static int lima_pdev_probe(struct platform_device *pdev)
+{
+	struct lima_device *ldev;
+	struct drm_device *ddev;
+	int err;
+
+	err = lima_sched_slab_init();
+	if (err)
+		return err;
+
+	ldev = devm_kzalloc(&pdev->dev, sizeof(*ldev), GFP_KERNEL);
+	if (!ldev) {
+		err = -ENOMEM;
+		goto err_out0;
+	}
+
+	ldev->pdev = pdev;
+	ldev->dev = &pdev->dev;
+	ldev->id = (enum lima_gpu_id)of_device_get_match_data(&pdev->dev);
+
+	platform_set_drvdata(pdev, ldev);
+
+	/* Allocate and initialize the DRM device. */
+	ddev = drm_dev_alloc(&lima_drm_driver, &pdev->dev);
+	if (IS_ERR(ddev))
+		return PTR_ERR(ddev);
+
+	ddev->dev_private = ldev;
+	ldev->ddev = ddev;
+
+	err = lima_device_init(ldev);
+	if (err) {
+		dev_err(&pdev->dev, "Fatal error during GPU init\n");
+		goto err_out1;
+	}
+
+	/*
+	 * Register the DRM device with the core and the connectors with
+	 * sysfs.
+	 */
+	err = drm_dev_register(ddev, 0);
+	if (err < 0)
+		goto err_out2;
+
+	return 0;
+
+err_out2:
+	lima_device_fini(ldev);
+err_out1:
+	drm_dev_put(ddev);
+err_out0:
+	lima_sched_slab_fini();
+	return err;
+}
+
+static int lima_pdev_remove(struct platform_device *pdev)
+{
+	struct lima_device *ldev = platform_get_drvdata(pdev);
+	struct drm_device *ddev = ldev->ddev;
+
+	drm_dev_unregister(ddev);
+	lima_device_fini(ldev);
+	drm_dev_put(ddev);
+	lima_sched_slab_fini();
+	return 0;
+}
+
+static const struct of_device_id dt_match[] = {
+	{ .compatible = "arm,mali-400", .data = (void *)lima_gpu_mali400 },
+	{ .compatible = "arm,mali-450", .data = (void *)lima_gpu_mali450 },
+	{}
+};
+MODULE_DEVICE_TABLE(of, dt_match);
+
+static struct platform_driver lima_platform_driver = {
+	.probe      = lima_pdev_probe,
+	.remove     = lima_pdev_remove,
+	.driver     = {
+		.name   = "lima",
+		.of_match_table = dt_match,
+	},
+};
+
+static int __init lima_init(void)
+{
+	return platform_driver_register(&lima_platform_driver);
+}
+module_init(lima_init);
+
+static void __exit lima_exit(void)
+{
+	platform_driver_unregister(&lima_platform_driver);
+}
+module_exit(lima_exit);
+
+MODULE_AUTHOR("Lima Project Developers");
+MODULE_DESCRIPTION("Lima DRM Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/lima/lima_drv.h b/drivers/gpu/drm/lima/lima_drv.h
new file mode 100644
index 000000000000..69c7344715c9
--- /dev/null
+++ b/drivers/gpu/drm/lima/lima_drv.h
@@ -0,0 +1,45 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
+
+#ifndef __LIMA_DRV_H__
+#define __LIMA_DRV_H__
+
+#include <drm/drm_file.h>
+
+#include "lima_ctx.h"
+
+extern int lima_sched_timeout_ms;
+
+struct lima_vm;
+struct lima_bo;
+struct lima_sched_task;
+
+struct drm_lima_gem_submit_bo;
+
+struct lima_drm_priv {
+	struct lima_vm *vm;
+	struct lima_ctx_mgr ctx_mgr;
+};
+
+struct lima_submit {
+	struct lima_ctx *ctx;
+	int pipe;
+	u32 flags;
+
+	struct drm_lima_gem_submit_bo *bos;
+	struct lima_bo **lbos;
+	u32 nr_bos;
+
+	u32 in_sync[2];
+	u32 out_sync;
+
+	struct lima_sched_task *task;
+};
+
+static inline struct lima_drm_priv *
+to_lima_drm_priv(struct drm_file *file)
+{
+	return file->driver_priv;
+}
+
+#endif
diff --git a/drivers/gpu/drm/lima/lima_gem.c b/drivers/gpu/drm/lima/lima_gem.c
new file mode 100644
index 000000000000..477c0f766663
--- /dev/null
+++ b/drivers/gpu/drm/lima/lima_gem.c
@@ -0,0 +1,349 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
+
+#include <linux/mm.h>
+#include <linux/sync_file.h>
+#include <linux/pfn_t.h>
+
+#include <drm/drm_file.h>
+#include <drm/drm_syncobj.h>
+#include <drm/drm_utils.h>
+
+#include <drm/lima_drm.h>
+
+#include "lima_drv.h"
+#include "lima_gem.h"
+#include "lima_gem_prime.h"
+#include "lima_vm.h"
+#include "lima_object.h"
+
+int lima_gem_create_handle(struct drm_device *dev, struct drm_file *file,
+			   u32 size, u32 flags, u32 *handle)
+{
+	int err;
+	struct lima_bo *bo;
+	struct lima_device *ldev = to_lima_dev(dev);
+
+	bo = lima_bo_create(ldev, size, flags, NULL, NULL);
+	if (IS_ERR(bo))
+		return PTR_ERR(bo);
+
+	err = drm_gem_handle_create(file, &bo->gem, handle);
+
+	/* drop reference from allocate - handle holds it now */
+	drm_gem_object_put_unlocked(&bo->gem);
+
+	return err;
+}
+
+void lima_gem_free_object(struct drm_gem_object *obj)
+{
+	struct lima_bo *bo = to_lima_bo(obj);
+
+	if (!list_empty(&bo->va))
+		dev_err(obj->dev->dev, "lima gem free bo still has va\n");
+
+	lima_bo_destroy(bo);
+}
+
+int lima_gem_object_open(struct drm_gem_object *obj, struct drm_file *file)
+{
+	struct lima_bo *bo = to_lima_bo(obj);
+	struct lima_drm_priv *priv = to_lima_drm_priv(file);
+	struct lima_vm *vm = priv->vm;
+
+	return lima_vm_bo_add(vm, bo, true);
+}
+
+void lima_gem_object_close(struct drm_gem_object *obj, struct drm_file *file)
+{
+	struct lima_bo *bo = to_lima_bo(obj);
+	struct lima_drm_priv *priv = to_lima_drm_priv(file);
+	struct lima_vm *vm = priv->vm;
+
+	lima_vm_bo_del(vm, bo);
+}
+
+int lima_gem_get_info(struct drm_file *file, u32 handle, u32 *va, u64 *offset)
+{
+	struct drm_gem_object *obj;
+	struct lima_bo *bo;
+	struct lima_drm_priv *priv = to_lima_drm_priv(file);
+	struct lima_vm *vm = priv->vm;
+	int err;
+
+	obj = drm_gem_object_lookup(file, handle);
+	if (!obj)
+		return -ENOENT;
+
+	bo = to_lima_bo(obj);
+
+	*va = lima_vm_get_va(vm, bo);
+
+	err = drm_gem_create_mmap_offset(obj);
+	if (!err)
+		*offset = drm_vma_node_offset_addr(&obj->vma_node);
+
+	drm_gem_object_put_unlocked(obj);
+	return err;
+}
+
+static vm_fault_t lima_gem_fault(struct vm_fault *vmf)
+{
+	struct vm_area_struct *vma = vmf->vma;
+	struct drm_gem_object *obj = vma->vm_private_data;
+	struct lima_bo *bo = to_lima_bo(obj);
+	pfn_t pfn;
+	pgoff_t pgoff;
+
+	/* We don't use vmf->pgoff since that has the fake offset: */
+	pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
+	pfn = __pfn_to_pfn_t(page_to_pfn(bo->pages[pgoff]), PFN_DEV);
+
+	return vmf_insert_mixed(vma, vmf->address, pfn);
+}
+
+const struct vm_operations_struct lima_gem_vm_ops = {
+	.fault = lima_gem_fault,
+	.open = drm_gem_vm_open,
+	.close = drm_gem_vm_close,
+};
+
+void lima_set_vma_flags(struct vm_area_struct *vma)
+{
+	pgprot_t prot = vm_get_page_prot(vma->vm_flags);
+
+	vma->vm_flags |= VM_MIXEDMAP;
+	vma->vm_flags &= ~VM_PFNMAP;
+	vma->vm_page_prot = pgprot_writecombine(prot);
+}
+
+int lima_gem_mmap(struct file *filp, struct vm_area_struct *vma)
+{
+	int ret;
+
+	ret = drm_gem_mmap(filp, vma);
+	if (ret)
+		return ret;
+
+	lima_set_vma_flags(vma);
+	return 0;
+}
+
+static int lima_gem_sync_bo(struct lima_sched_task *task, struct lima_bo *bo,
+			    bool write, bool explicit)
+{
+	int err = 0;
+
+	if (!write) {
+		err = reservation_object_reserve_shared(bo->gem.resv, 1);
+		if (err)
+			return err;
+	}
+
+	/* explicit sync use user passed dep fence */
+	if (explicit)
+		return 0;
+
+	return drm_gem_fence_array_add_implicit(&task->deps, &bo->gem, write);
+}
+
+static int lima_gem_lock_bos(struct lima_bo **bos, u32 nr_bos,
+			     struct ww_acquire_ctx *ctx)
+{
+	int i, ret = 0, contended, slow_locked = -1;
+
+	ww_acquire_init(ctx, &reservation_ww_class);
+
+retry:
+	for (i = 0; i < nr_bos; i++) {
+		if (i == slow_locked) {
+			slow_locked = -1;
+			continue;
+		}
+
+		ret = ww_mutex_lock_interruptible(&bos[i]->gem.resv->lock, ctx);
+		if (ret < 0) {
+			contended = i;
+			goto err;
+		}
+	}
+
+	ww_acquire_done(ctx);
+	return 0;
+
+err:
+	for (i--; i >= 0; i--)
+		ww_mutex_unlock(&bos[i]->gem.resv->lock);
+
+	if (slow_locked >= 0)
+		ww_mutex_unlock(&bos[slow_locked]->gem.resv->lock);
+
+	if (ret == -EDEADLK) {
+		/* we lost out in a seqno race, lock and retry.. */
+		ret = ww_mutex_lock_slow_interruptible(
+			&bos[contended]->gem.resv->lock, ctx);
+		if (!ret) {
+			slow_locked = contended;
+			goto retry;
+		}
+	}
+	ww_acquire_fini(ctx);
+
+	return ret;
+}
+
+static void lima_gem_unlock_bos(struct lima_bo **bos, u32 nr_bos,
+				struct ww_acquire_ctx *ctx)
+{
+	int i;
+
+	for (i = 0; i < nr_bos; i++)
+		ww_mutex_unlock(&bos[i]->gem.resv->lock);
+	ww_acquire_fini(ctx);
+}
+
+static int lima_gem_add_deps(struct drm_file *file, struct lima_submit *submit)
+{
+	int i, err;
+
+	for (i = 0; i < ARRAY_SIZE(submit->in_sync); i++) {
+		struct dma_fence *fence = NULL;
+
+		if (!submit->in_sync[i])
+			continue;
+
+		err = drm_syncobj_find_fence(file, submit->in_sync[i],
+					     0, 0, &fence);
+		if (err)
+			return err;
+
+		err = drm_gem_fence_array_add(&submit->task->deps, fence);
+		if (err) {
+			dma_fence_put(fence);
+			return err;
+		}
+	}
+
+	return 0;
+}
+
+int lima_gem_submit(struct drm_file *file, struct lima_submit *submit)
+{
+	int i, err = 0;
+	struct ww_acquire_ctx ctx;
+	struct lima_drm_priv *priv = to_lima_drm_priv(file);
+	struct lima_vm *vm = priv->vm;
+	struct drm_syncobj *out_sync = NULL;
+	struct dma_fence *fence;
+	struct lima_bo **bos = submit->lbos;
+
+	if (submit->out_sync) {
+		out_sync = drm_syncobj_find(file, submit->out_sync);
+		if (!out_sync)
+			return -ENOENT;
+	}
+
+	for (i = 0; i < submit->nr_bos; i++) {
+		struct drm_gem_object *obj;
+		struct lima_bo *bo;
+
+		obj = drm_gem_object_lookup(file, submit->bos[i].handle);
+		if (!obj) {
+			err = -ENOENT;
+			goto err_out0;
+		}
+
+		bo = to_lima_bo(obj);
+
+		/* increase refcnt of gpu va map to prevent unmapped when executing,
+		 * will be decreased when task done
+		 */
+		err = lima_vm_bo_add(vm, bo, false);
+		if (err) {
+			drm_gem_object_put_unlocked(obj);
+			goto err_out0;
+		}
+
+		bos[i] = bo;
+	}
+
+	err = lima_gem_lock_bos(bos, submit->nr_bos, &ctx);
+	if (err)
+		goto err_out0;
+
+	err = lima_sched_task_init(
+		submit->task, submit->ctx->context + submit->pipe,
+		bos, submit->nr_bos, vm);
+	if (err)
+		goto err_out1;
+
+	err = lima_gem_add_deps(file, submit);
+	if (err)
+		goto err_out2;
+
+	for (i = 0; i < submit->nr_bos; i++) {
+		err = lima_gem_sync_bo(
+			submit->task, bos[i],
+			submit->bos[i].flags & LIMA_SUBMIT_BO_WRITE,
+			submit->flags & LIMA_SUBMIT_FLAG_EXPLICIT_FENCE);
+		if (err)
+			goto err_out2;
+	}
+
+	fence = lima_sched_context_queue_task(
+		submit->ctx->context + submit->pipe, submit->task);
+
+	for (i = 0; i < submit->nr_bos; i++) {
+		if (submit->bos[i].flags & LIMA_SUBMIT_BO_WRITE)
+			reservation_object_add_excl_fence(bos[i]->gem.resv, fence);
+		else
+			reservation_object_add_shared_fence(bos[i]->gem.resv, fence);
+	}
+
+	lima_gem_unlock_bos(bos, submit->nr_bos, &ctx);
+
+	for (i = 0; i < submit->nr_bos; i++)
+		drm_gem_object_put_unlocked(&bos[i]->gem);
+
+	if (out_sync) {
+		drm_syncobj_replace_fence(out_sync, fence);
+		drm_syncobj_put(out_sync);
+	}
+
+	dma_fence_put(fence);
+
+	return 0;
+
+err_out2:
+	lima_sched_task_fini(submit->task);
+err_out1:
+	lima_gem_unlock_bos(bos, submit->nr_bos, &ctx);
+err_out0:
+	for (i = 0; i < submit->nr_bos; i++) {
+		if (!bos[i])
+			break;
+		lima_vm_bo_del(vm, bos[i]);
+		drm_gem_object_put_unlocked(&bos[i]->gem);
+	}
+	if (out_sync)
+		drm_syncobj_put(out_sync);
+	return err;
+}
+
+int lima_gem_wait(struct drm_file *file, u32 handle, u32 op, s64 timeout_ns)
+{
+	bool write = op & LIMA_GEM_WAIT_WRITE;
+	long ret, timeout;
+
+	if (!op)
+		return 0;
+
+	timeout = drm_timeout_abs_to_jiffies(timeout_ns);
+
+	ret = drm_gem_reservation_object_wait(file, handle, write, timeout);
+	if (ret == 0)
+		ret = timeout ? -ETIMEDOUT : -EBUSY;
+
+	return ret;
+}
diff --git a/drivers/gpu/drm/lima/lima_gem.h b/drivers/gpu/drm/lima/lima_gem.h
new file mode 100644
index 000000000000..556111a01135
--- /dev/null
+++ b/drivers/gpu/drm/lima/lima_gem.h
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
+
+#ifndef __LIMA_GEM_H__
+#define __LIMA_GEM_H__
+
+struct lima_bo;
+struct lima_submit;
+
+extern const struct vm_operations_struct lima_gem_vm_ops;
+
+struct lima_bo *lima_gem_create_bo(struct drm_device *dev, u32 size, u32 flags);
+int lima_gem_create_handle(struct drm_device *dev, struct drm_file *file,
+			   u32 size, u32 flags, u32 *handle);
+void lima_gem_free_object(struct drm_gem_object *obj);
+int lima_gem_object_open(struct drm_gem_object *obj, struct drm_file *file);
+void lima_gem_object_close(struct drm_gem_object *obj, struct drm_file *file);
+int lima_gem_get_info(struct drm_file *file, u32 handle, u32 *va, u64 *offset);
+int lima_gem_mmap(struct file *filp, struct vm_area_struct *vma);
+int lima_gem_submit(struct drm_file *file, struct lima_submit *submit);
+int lima_gem_wait(struct drm_file *file, u32 handle, u32 op, s64 timeout_ns);
+
+void lima_set_vma_flags(struct vm_area_struct *vma);
+
+#endif
diff --git a/drivers/gpu/drm/lima/lima_gem_prime.c b/drivers/gpu/drm/lima/lima_gem_prime.c
new file mode 100644
index 000000000000..9c6d9f1dba55
--- /dev/null
+++ b/drivers/gpu/drm/lima/lima_gem_prime.c
@@ -0,0 +1,47 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/* Copyright 2018-2019 Qiang Yu <yuq825@gmail.com> */
+
+#include <linux/dma-buf.h>
+#include <drm/drm_prime.h>
+#include <drm/drm_drv.h>
+#include <drm/drm_file.h>
+
+#include "lima_device.h"
+#include "lima_object.h"
+#include "lima_gem.h"
+#include "lima_gem_prime.h"
+
+struct drm_gem_object *lima_gem_prime_import_sg_table(
+	struct drm_device *dev, struct dma_buf_attachment *attach,
+	struct sg_table *sgt)
+{
+	struct lima_device *ldev = to_lima_dev(dev);
+	struct lima_bo *bo;
+
+	bo = lima_bo_create(ldev, attach->dmabuf->size, 0, sgt,
+			    attach->dmabuf->resv);
+	if (IS_ERR(bo))
+		return ERR_CAST(bo);
+
+	return &bo->gem;
+}
+
+struct sg_table *lima_gem_prime_get_sg_table(struct drm_gem_object *obj)
+{
+	struct lima_bo *bo = to_lima_bo(obj);
+	int npages = obj->size >> PAGE_SHIFT;
+
+	return drm_prime_pages_to_sg(bo->pages, npages);
+}
+
+int lima_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
+{
+	int ret;
+
+	ret = drm_gem_mmap_obj(obj, obj->size, vma);
+	if (ret)
+		return ret;
+
+	lima_set_vma_flags(vma);
+	return 0;
+}
diff --git a/drivers/gpu/drm/lima/lima_gem_prime.h b/drivers/gpu/drm/lima/lima_gem_prime.h
new file mode 100644
index 000000000000..34b4d35c21e3
--- /dev/null
+++ b/drivers/gpu/drm/lima/lima_gem_prime.h
@@ -0,0 +1,13 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+/* Copyright 2018-2019 Qiang Yu <yuq825@gmail.com> */
+
+#ifndef __LIMA_GEM_PRIME_H__
+#define __LIMA_GEM_PRIME_H__
+
+struct drm_gem_object *lima_gem_prime_import_sg_table(
+	struct drm_device *dev, struct dma_buf_attachment *attach,
+	struct sg_table *sgt);
+struct sg_table *lima_gem_prime_get_sg_table(struct drm_gem_object *obj);
+int lima_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma);
+
+#endif
diff --git a/drivers/gpu/drm/lima/lima_gp.c b/drivers/gpu/drm/lima/lima_gp.c
new file mode 100644
index 000000000000..ccf49faedebf
--- /dev/null
+++ b/drivers/gpu/drm/lima/lima_gp.c
@@ -0,0 +1,283 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
+
+#include <linux/interrupt.h>
+#include <linux/iopoll.h>
+#include <linux/device.h>
+#include <linux/slab.h>
+
+#include <drm/lima_drm.h>
+
+#include "lima_device.h"
+#include "lima_gp.h"
+#include "lima_regs.h"
+
+#define gp_write(reg, data) writel(data, ip->iomem + reg)
+#define gp_read(reg) readl(ip->iomem + reg)
+
+static irqreturn_t lima_gp_irq_handler(int irq, void *data)
+{
+	struct lima_ip *ip = data;
+	struct lima_device *dev = ip->dev;
+	struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_gp;
+	u32 state = gp_read(LIMA_GP_INT_STAT);
+	u32 status = gp_read(LIMA_GP_STATUS);
+	bool done = false;
+
+	/* for shared irq case */
+	if (!state)
+		return IRQ_NONE;
+
+	if (state & LIMA_GP_IRQ_MASK_ERROR) {
+		dev_err(dev->dev, "gp error irq state=%x status=%x\n",
+			state, status);
+
+		/* mask all interrupts before hard reset */
+		gp_write(LIMA_GP_INT_MASK, 0);
+
+		pipe->error = true;
+		done = true;
+	} else {
+		bool valid = state & (LIMA_GP_IRQ_VS_END_CMD_LST |
+				      LIMA_GP_IRQ_PLBU_END_CMD_LST);
+		bool active = status & (LIMA_GP_STATUS_VS_ACTIVE |
+					LIMA_GP_STATUS_PLBU_ACTIVE);
+		done = valid && !active;
+	}
+
+	gp_write(LIMA_GP_INT_CLEAR, state);
+
+	if (done)
+		lima_sched_pipe_task_done(pipe);
+
+	return IRQ_HANDLED;
+}
+
+static void lima_gp_soft_reset_async(struct lima_ip *ip)
+{
+	if (ip->data.async_reset)
+		return;
+
+	gp_write(LIMA_GP_INT_MASK, 0);
+	gp_write(LIMA_GP_INT_CLEAR, LIMA_GP_IRQ_RESET_COMPLETED);
+	gp_write(LIMA_GP_CMD, LIMA_GP_CMD_SOFT_RESET);
+	ip->data.async_reset = true;
+}
+
+static int lima_gp_soft_reset_async_wait(struct lima_ip *ip)
+{
+	struct lima_device *dev = ip->dev;
+	int err;
+	u32 v;
+
+	if (!ip->data.async_reset)
+		return 0;
+
+	err = readl_poll_timeout(ip->iomem + LIMA_GP_INT_RAWSTAT, v,
+				 v & LIMA_GP_IRQ_RESET_COMPLETED,
+				 0, 100);
+	if (err) {
+		dev_err(dev->dev, "gp soft reset time out\n");
+		return err;
+	}
+
+	gp_write(LIMA_GP_INT_CLEAR, LIMA_GP_IRQ_MASK_ALL);
+	gp_write(LIMA_GP_INT_MASK, LIMA_GP_IRQ_MASK_USED);
+
+	ip->data.async_reset = false;
+	return 0;
+}
+
+static int lima_gp_task_validate(struct lima_sched_pipe *pipe,
+				 struct lima_sched_task *task)
+{
+	struct drm_lima_gp_frame *frame = task->frame;
+	u32 *f = frame->frame;
+	(void)pipe;
+
+	if (f[LIMA_GP_VSCL_START_ADDR >> 2] >
+	    f[LIMA_GP_VSCL_END_ADDR >> 2] ||
+	    f[LIMA_GP_PLBUCL_START_ADDR >> 2] >
+	    f[LIMA_GP_PLBUCL_END_ADDR >> 2] ||
+	    f[LIMA_GP_PLBU_ALLOC_START_ADDR >> 2] >
+	    f[LIMA_GP_PLBU_ALLOC_END_ADDR >> 2])
+		return -EINVAL;
+
+	if (f[LIMA_GP_VSCL_START_ADDR >> 2] ==
+	    f[LIMA_GP_VSCL_END_ADDR >> 2] &&
+	    f[LIMA_GP_PLBUCL_START_ADDR >> 2] ==
+	    f[LIMA_GP_PLBUCL_END_ADDR >> 2])
+		return -EINVAL;
+
+	return 0;
+}
+
+static void lima_gp_task_run(struct lima_sched_pipe *pipe,
+			     struct lima_sched_task *task)
+{
+	struct lima_ip *ip = pipe->processor[0];
+	struct drm_lima_gp_frame *frame = task->frame;
+	u32 *f = frame->frame;
+	u32 cmd = 0;
+	int i;
+
+	if (f[LIMA_GP_VSCL_START_ADDR >> 2] !=
+	    f[LIMA_GP_VSCL_END_ADDR >> 2])
+		cmd |= LIMA_GP_CMD_START_VS;
+	if (f[LIMA_GP_PLBUCL_START_ADDR >> 2] !=
+	    f[LIMA_GP_PLBUCL_END_ADDR >> 2])
+		cmd |= LIMA_GP_CMD_START_PLBU;
+
+	/* before any hw ops, wait last success task async soft reset */
+	lima_gp_soft_reset_async_wait(ip);
+
+	for (i = 0; i < LIMA_GP_FRAME_REG_NUM; i++)
+		writel(f[i], ip->iomem + LIMA_GP_VSCL_START_ADDR + i * 4);
+
+	gp_write(LIMA_GP_CMD, LIMA_GP_CMD_UPDATE_PLBU_ALLOC);
+	gp_write(LIMA_GP_CMD, cmd);
+}
+
+static int lima_gp_hard_reset_poll(struct lima_ip *ip)
+{
+	gp_write(LIMA_GP_PERF_CNT_0_LIMIT, 0xC01A0000);
+	return gp_read(LIMA_GP_PERF_CNT_0_LIMIT) == 0xC01A0000;
+}
+
+static int lima_gp_hard_reset(struct lima_ip *ip)
+{
+	struct lima_device *dev = ip->dev;
+	int ret;
+
+	gp_write(LIMA_GP_PERF_CNT_0_LIMIT, 0xC0FFE000);
+	gp_write(LIMA_GP_INT_MASK, 0);
+	gp_write(LIMA_GP_CMD, LIMA_GP_CMD_RESET);
+	ret = lima_poll_timeout(ip, lima_gp_hard_reset_poll, 10, 100);
+	if (ret) {
+		dev_err(dev->dev, "gp hard reset timeout\n");
+		return ret;
+	}
+
+	gp_write(LIMA_GP_PERF_CNT_0_LIMIT, 0);
+	gp_write(LIMA_GP_INT_CLEAR, LIMA_GP_IRQ_MASK_ALL);
+	gp_write(LIMA_GP_INT_MASK, LIMA_GP_IRQ_MASK_USED);
+	return 0;
+}
+
+static void lima_gp_task_fini(struct lima_sched_pipe *pipe)
+{
+	lima_gp_soft_reset_async(pipe->processor[0]);
+}
+
+static void lima_gp_task_error(struct lima_sched_pipe *pipe)
+{
+	struct lima_ip *ip = pipe->processor[0];
+
+	dev_err(ip->dev->dev, "gp task error int_state=%x status=%x\n",
+		gp_read(LIMA_GP_INT_STAT), gp_read(LIMA_GP_STATUS));
+
+	lima_gp_hard_reset(ip);
+}
+
+static void lima_gp_task_mmu_error(struct lima_sched_pipe *pipe)
+{
+	lima_sched_pipe_task_done(pipe);
+}
+
+static void lima_gp_print_version(struct lima_ip *ip)
+{
+	u32 version, major, minor;
+	char *name;
+
+	version = gp_read(LIMA_GP_VERSION);
+	major = (version >> 8) & 0xFF;
+	minor = version & 0xFF;
+	switch (version >> 16) {
+	case 0xA07:
+	    name = "mali200";
+		break;
+	case 0xC07:
+		name = "mali300";
+		break;
+	case 0xB07:
+		name = "mali400";
+		break;
+	case 0xD07:
+		name = "mali450";
+		break;
+	default:
+		name = "unknown";
+		break;
+	}
+	dev_info(ip->dev->dev, "%s - %s version major %d minor %d\n",
+		 lima_ip_name(ip), name, major, minor);
+}
+
+static struct kmem_cache *lima_gp_task_slab;
+static int lima_gp_task_slab_refcnt;
+
+int lima_gp_init(struct lima_ip *ip)
+{
+	struct lima_device *dev = ip->dev;
+	int err;
+
+	lima_gp_print_version(ip);
+
+	ip->data.async_reset = false;
+	lima_gp_soft_reset_async(ip);
+	err = lima_gp_soft_reset_async_wait(ip);
+	if (err)
+		return err;
+
+	err = devm_request_irq(dev->dev, ip->irq, lima_gp_irq_handler,
+			       IRQF_SHARED, lima_ip_name(ip), ip);
+	if (err) {
+		dev_err(dev->dev, "gp %s fail to request irq\n",
+			lima_ip_name(ip));
+		return err;
+	}
+
+	dev->gp_version = gp_read(LIMA_GP_VERSION);
+
+	return 0;
+}
+
+void lima_gp_fini(struct lima_ip *ip)
+{
+
+}
+
+int lima_gp_pipe_init(struct lima_device *dev)
+{
+	int frame_size = sizeof(struct drm_lima_gp_frame);
+	struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_gp;
+
+	if (!lima_gp_task_slab) {
+		lima_gp_task_slab = kmem_cache_create_usercopy(
+			"lima_gp_task", sizeof(struct lima_sched_task) + frame_size,
+			0, SLAB_HWCACHE_ALIGN, sizeof(struct lima_sched_task),
+			frame_size, NULL);
+		if (!lima_gp_task_slab)
+			return -ENOMEM;
+	}
+	lima_gp_task_slab_refcnt++;
+
+	pipe->frame_size = frame_size;
+	pipe->task_slab = lima_gp_task_slab;
+
+	pipe->task_validate = lima_gp_task_validate;
+	pipe->task_run = lima_gp_task_run;
+	pipe->task_fini = lima_gp_task_fini;
+	pipe->task_error = lima_gp_task_error;
+	pipe->task_mmu_error = lima_gp_task_mmu_error;
+
+	return 0;
+}
+
+void lima_gp_pipe_fini(struct lima_device *dev)
+{
+	if (!--lima_gp_task_slab_refcnt) {
+		kmem_cache_destroy(lima_gp_task_slab);
+		lima_gp_task_slab = NULL;
+	}
+}
diff --git a/drivers/gpu/drm/lima/lima_gp.h b/drivers/gpu/drm/lima/lima_gp.h
new file mode 100644
index 000000000000..516e5c1babbb
--- /dev/null
+++ b/drivers/gpu/drm/lima/lima_gp.h
@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
+
+#ifndef __LIMA_GP_H__
+#define __LIMA_GP_H__
+
+struct lima_ip;
+struct lima_device;
+
+int lima_gp_init(struct lima_ip *ip);
+void lima_gp_fini(struct lima_ip *ip);
+
+int lima_gp_pipe_init(struct lima_device *dev);
+void lima_gp_pipe_fini(struct lima_device *dev);
+
+#endif
diff --git a/drivers/gpu/drm/lima/lima_l2_cache.c b/drivers/gpu/drm/lima/lima_l2_cache.c
new file mode 100644
index 000000000000..6873a7af5a5c
--- /dev/null
+++ b/drivers/gpu/drm/lima/lima_l2_cache.c
@@ -0,0 +1,80 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
+
+#include <linux/iopoll.h>
+#include <linux/device.h>
+
+#include "lima_device.h"
+#include "lima_l2_cache.h"
+#include "lima_regs.h"
+
+#define l2_cache_write(reg, data) writel(data, ip->iomem + reg)
+#define l2_cache_read(reg) readl(ip->iomem + reg)
+
+static int lima_l2_cache_wait_idle(struct lima_ip *ip)
+{
+	struct lima_device *dev = ip->dev;
+	int err;
+	u32 v;
+
+	err = readl_poll_timeout(ip->iomem + LIMA_L2_CACHE_STATUS, v,
+				 !(v & LIMA_L2_CACHE_STATUS_COMMAND_BUSY),
+				 0, 1000);
+	if (err) {
+		dev_err(dev->dev, "l2 cache wait command timeout\n");
+		return err;
+	}
+	return 0;
+}
+
+int lima_l2_cache_flush(struct lima_ip *ip)
+{
+	int ret;
+
+	spin_lock(&ip->data.lock);
+	l2_cache_write(LIMA_L2_CACHE_COMMAND, LIMA_L2_CACHE_COMMAND_CLEAR_ALL);
+	ret = lima_l2_cache_wait_idle(ip);
+	spin_unlock(&ip->data.lock);
+	return ret;
+}
+
+int lima_l2_cache_init(struct lima_ip *ip)
+{
+	int i, err;
+	u32 size;
+	struct lima_device *dev = ip->dev;
+
+	/* l2_cache2 only exists when one of PP4-7 present */
+	if (ip->id == lima_ip_l2_cache2) {
+		for (i = lima_ip_pp4; i <= lima_ip_pp7; i++) {
+			if (dev->ip[i].present)
+				break;
+		}
+		if (i > lima_ip_pp7)
+			return -ENODEV;
+	}
+
+	spin_lock_init(&ip->data.lock);
+
+	size = l2_cache_read(LIMA_L2_CACHE_SIZE);
+	dev_info(dev->dev, "l2 cache %uK, %u-way, %ubyte cache line, %ubit external bus\n",
+		 1 << (((size >> 16) & 0xff) - 10),
+		 1 << ((size >> 8) & 0xff),
+		 1 << (size & 0xff),
+		 1 << ((size >> 24) & 0xff));
+
+	err = lima_l2_cache_flush(ip);
+	if (err)
+		return err;
+
+	l2_cache_write(LIMA_L2_CACHE_ENABLE,
+		       LIMA_L2_CACHE_ENABLE_ACCESS|LIMA_L2_CACHE_ENABLE_READ_ALLOCATE);
+	l2_cache_write(LIMA_L2_CACHE_MAX_READS, 0x1c);
+
+	return 0;
+}
+
+void lima_l2_cache_fini(struct lima_ip *ip)
+{
+
+}
diff --git a/drivers/gpu/drm/lima/lima_l2_cache.h b/drivers/gpu/drm/lima/lima_l2_cache.h
new file mode 100644
index 000000000000..c63fb676ff14
--- /dev/null
+++ b/drivers/gpu/drm/lima/lima_l2_cache.h
@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
+
+#ifndef __LIMA_L2_CACHE_H__
+#define __LIMA_L2_CACHE_H__
+
+struct lima_ip;
+
+int lima_l2_cache_init(struct lima_ip *ip);
+void lima_l2_cache_fini(struct lima_ip *ip);
+
+int lima_l2_cache_flush(struct lima_ip *ip);
+
+#endif
diff --git a/drivers/gpu/drm/lima/lima_mmu.c b/drivers/gpu/drm/lima/lima_mmu.c
new file mode 100644
index 000000000000..8e1651d6a61f
--- /dev/null
+++ b/drivers/gpu/drm/lima/lima_mmu.c
@@ -0,0 +1,142 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
+
+#include <linux/interrupt.h>
+#include <linux/iopoll.h>
+#include <linux/device.h>
+
+#include "lima_device.h"
+#include "lima_mmu.h"
+#include "lima_vm.h"
+#include "lima_object.h"
+#include "lima_regs.h"
+
+#define mmu_write(reg, data) writel(data, ip->iomem + reg)
+#define mmu_read(reg) readl(ip->iomem + reg)
+
+#define lima_mmu_send_command(cmd, addr, val, cond)	     \
+({							     \
+	int __ret;					     \
+							     \
+	mmu_write(LIMA_MMU_COMMAND, cmd);		     \
+	__ret = readl_poll_timeout(ip->iomem + (addr), val,  \
+				  cond, 0, 100);	     \
+	if (__ret)					     \
+		dev_err(dev->dev,			     \
+			"mmu command %x timeout\n", cmd);    \
+	__ret;						     \
+})
+
+static irqreturn_t lima_mmu_irq_handler(int irq, void *data)
+{
+	struct lima_ip *ip = data;
+	struct lima_device *dev = ip->dev;
+	u32 status = mmu_read(LIMA_MMU_INT_STATUS);
+	struct lima_sched_pipe *pipe;
+
+	/* for shared irq case */
+	if (!status)
+		return IRQ_NONE;
+
+	if (status & LIMA_MMU_INT_PAGE_FAULT) {
+		u32 fault = mmu_read(LIMA_MMU_PAGE_FAULT_ADDR);
+
+		dev_err(dev->dev, "mmu page fault at 0x%x from bus id %d of type %s on %s\n",
+			fault, LIMA_MMU_STATUS_BUS_ID(status),
+			status & LIMA_MMU_STATUS_PAGE_FAULT_IS_WRITE ? "write" : "read",
+			lima_ip_name(ip));
+	}
+
+	if (status & LIMA_MMU_INT_READ_BUS_ERROR)
+		dev_err(dev->dev, "mmu %s irq bus error\n", lima_ip_name(ip));
+
+	/* mask all interrupts before resume */
+	mmu_write(LIMA_MMU_INT_MASK, 0);
+	mmu_write(LIMA_MMU_INT_CLEAR, status);
+
+	pipe = dev->pipe + (ip->id == lima_ip_gpmmu ? lima_pipe_gp : lima_pipe_pp);
+	lima_sched_pipe_mmu_error(pipe);
+
+	return IRQ_HANDLED;
+}
+
+int lima_mmu_init(struct lima_ip *ip)
+{
+	struct lima_device *dev = ip->dev;
+	int err;
+	u32 v;
+
+	if (ip->id == lima_ip_ppmmu_bcast)
+		return 0;
+
+	mmu_write(LIMA_MMU_DTE_ADDR, 0xCAFEBABE);
+	if (mmu_read(LIMA_MMU_DTE_ADDR) != 0xCAFEB000) {
+		dev_err(dev->dev, "mmu %s dte write test fail\n", lima_ip_name(ip));
+		return -EIO;
+	}
+
+	mmu_write(LIMA_MMU_COMMAND, LIMA_MMU_COMMAND_HARD_RESET);
+	err = lima_mmu_send_command(LIMA_MMU_COMMAND_HARD_RESET,
+				    LIMA_MMU_DTE_ADDR, v, v == 0);
+	if (err)
+		return err;
+
+	err = devm_request_irq(dev->dev, ip->irq, lima_mmu_irq_handler,
+			       IRQF_SHARED, lima_ip_name(ip), ip);
+	if (err) {
+		dev_err(dev->dev, "mmu %s fail to request irq\n", lima_ip_name(ip));
+		return err;
+	}
+
+	mmu_write(LIMA_MMU_INT_MASK, LIMA_MMU_INT_PAGE_FAULT | LIMA_MMU_INT_READ_BUS_ERROR);
+	mmu_write(LIMA_MMU_DTE_ADDR, dev->empty_vm->pd.dma);
+	return lima_mmu_send_command(LIMA_MMU_COMMAND_ENABLE_PAGING,
+				     LIMA_MMU_STATUS, v,
+				     v & LIMA_MMU_STATUS_PAGING_ENABLED);
+}
+
+void lima_mmu_fini(struct lima_ip *ip)
+{
+
+}
+
+void lima_mmu_switch_vm(struct lima_ip *ip, struct lima_vm *vm)
+{
+	struct lima_device *dev = ip->dev;
+	u32 v;
+
+	lima_mmu_send_command(LIMA_MMU_COMMAND_ENABLE_STALL,
+			      LIMA_MMU_STATUS, v,
+			      v & LIMA_MMU_STATUS_STALL_ACTIVE);
+
+	if (vm)
+		mmu_write(LIMA_MMU_DTE_ADDR, vm->pd.dma);
+
+	/* flush the TLB */
+	mmu_write(LIMA_MMU_COMMAND, LIMA_MMU_COMMAND_ZAP_CACHE);
+
+	lima_mmu_send_command(LIMA_MMU_COMMAND_DISABLE_STALL,
+			      LIMA_MMU_STATUS, v,
+			      !(v & LIMA_MMU_STATUS_STALL_ACTIVE));
+}
+
+void lima_mmu_page_fault_resume(struct lima_ip *ip)
+{
+	struct lima_device *dev = ip->dev;
+	u32 status = mmu_read(LIMA_MMU_STATUS);
+	u32 v;
+
+	if (status & LIMA_MMU_STATUS_PAGE_FAULT_ACTIVE) {
+		dev_info(dev->dev, "mmu resume\n");
+
+		mmu_write(LIMA_MMU_INT_MASK, 0);
+		mmu_write(LIMA_MMU_DTE_ADDR, 0xCAFEBABE);
+		lima_mmu_send_command(LIMA_MMU_COMMAND_HARD_RESET,
+				      LIMA_MMU_DTE_ADDR, v, v == 0);
+		mmu_write(LIMA_MMU_INT_MASK, LIMA_MMU_INT_PAGE_FAULT | LIMA_MMU_INT_READ_BUS_ERROR);
+		mmu_write(LIMA_MMU_DTE_ADDR, dev->empty_vm->pd.dma);
+		lima_mmu_send_command(LIMA_MMU_COMMAND_ENABLE_PAGING,
+				      LIMA_MMU_STATUS, v,
+				      v & LIMA_MMU_STATUS_PAGING_ENABLED);
+	}
+}
diff --git a/drivers/gpu/drm/lima/lima_mmu.h b/drivers/gpu/drm/lima/lima_mmu.h
new file mode 100644
index 000000000000..8c78319bcc8e
--- /dev/null
+++ b/drivers/gpu/drm/lima/lima_mmu.h
@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
+
+#ifndef __LIMA_MMU_H__
+#define __LIMA_MMU_H__
+
+struct lima_ip;
+struct lima_vm;
+
+int lima_mmu_init(struct lima_ip *ip);
+void lima_mmu_fini(struct lima_ip *ip);
+
+void lima_mmu_switch_vm(struct lima_ip *ip, struct lima_vm *vm);
+void lima_mmu_page_fault_resume(struct lima_ip *ip);
+
+#endif
diff --git a/drivers/gpu/drm/lima/lima_object.c b/drivers/gpu/drm/lima/lima_object.c
new file mode 100644
index 000000000000..5c41f859a72f
--- /dev/null
+++ b/drivers/gpu/drm/lima/lima_object.c
@@ -0,0 +1,122 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/* Copyright 2018-2019 Qiang Yu <yuq825@gmail.com> */
+
+#include <drm/drm_prime.h>
+#include <linux/pagemap.h>
+#include <linux/dma-mapping.h>
+
+#include "lima_object.h"
+
+void lima_bo_destroy(struct lima_bo *bo)
+{
+	if (bo->sgt) {
+		kfree(bo->pages);
+		drm_prime_gem_destroy(&bo->gem, bo->sgt);
+	} else {
+		if (bo->pages_dma_addr) {
+			int i, npages = bo->gem.size >> PAGE_SHIFT;
+
+			for (i = 0; i < npages; i++) {
+				if (bo->pages_dma_addr[i])
+					dma_unmap_page(bo->gem.dev->dev,
+						       bo->pages_dma_addr[i],
+						       PAGE_SIZE, DMA_BIDIRECTIONAL);
+			}
+		}
+
+		if (bo->pages)
+			drm_gem_put_pages(&bo->gem, bo->pages, true, true);
+	}
+
+	kfree(bo->pages_dma_addr);
+	drm_gem_object_release(&bo->gem);
+	kfree(bo);
+}
+
+static struct lima_bo *lima_bo_create_struct(struct lima_device *dev, u32 size, u32 flags,
+					     struct reservation_object *resv)
+{
+	struct lima_bo *bo;
+	int err;
+
+	size = PAGE_ALIGN(size);
+
+	bo = kzalloc(sizeof(*bo), GFP_KERNEL);
+	if (!bo)
+		return ERR_PTR(-ENOMEM);
+
+	mutex_init(&bo->lock);
+	INIT_LIST_HEAD(&bo->va);
+	bo->gem.resv = resv;
+
+	err = drm_gem_object_init(dev->ddev, &bo->gem, size);
+	if (err) {
+		kfree(bo);
+		return ERR_PTR(err);
+	}
+
+	return bo;
+}
+
+struct lima_bo *lima_bo_create(struct lima_device *dev, u32 size,
+			       u32 flags, struct sg_table *sgt,
+			       struct reservation_object *resv)
+{
+	int i, err;
+	size_t npages;
+	struct lima_bo *bo, *ret;
+
+	bo = lima_bo_create_struct(dev, size, flags, resv);
+	if (IS_ERR(bo))
+		return bo;
+
+	npages = bo->gem.size >> PAGE_SHIFT;
+
+	bo->pages_dma_addr = kcalloc(npages, sizeof(dma_addr_t), GFP_KERNEL);
+	if (!bo->pages_dma_addr) {
+		ret = ERR_PTR(-ENOMEM);
+		goto err_out;
+	}
+
+	if (sgt) {
+		bo->sgt = sgt;
+
+		bo->pages = kcalloc(npages, sizeof(*bo->pages), GFP_KERNEL);
+		if (!bo->pages) {
+			ret = ERR_PTR(-ENOMEM);
+			goto err_out;
+		}
+
+		err = drm_prime_sg_to_page_addr_arrays(
+			sgt, bo->pages, bo->pages_dma_addr, npages);
+		if (err) {
+			ret = ERR_PTR(err);
+			goto err_out;
+		}
+	} else {
+		mapping_set_gfp_mask(bo->gem.filp->f_mapping, GFP_DMA32);
+		bo->pages = drm_gem_get_pages(&bo->gem);
+		if (IS_ERR(bo->pages)) {
+			ret = ERR_CAST(bo->pages);
+			bo->pages = NULL;
+			goto err_out;
+		}
+
+		for (i = 0; i < npages; i++) {
+			dma_addr_t addr = dma_map_page(dev->dev, bo->pages[i], 0,
+						       PAGE_SIZE, DMA_BIDIRECTIONAL);
+			if (dma_mapping_error(dev->dev, addr)) {
+				ret = ERR_PTR(-EFAULT);
+				goto err_out;
+			}
+			bo->pages_dma_addr[i] = addr;
+		}
+
+	}
+
+	return bo;
+
+err_out:
+	lima_bo_destroy(bo);
+	return ret;
+}
diff --git a/drivers/gpu/drm/lima/lima_object.h b/drivers/gpu/drm/lima/lima_object.h
new file mode 100644
index 000000000000..6738724afb7b
--- /dev/null
+++ b/drivers/gpu/drm/lima/lima_object.h
@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+/* Copyright 2018-2019 Qiang Yu <yuq825@gmail.com> */
+
+#ifndef __LIMA_OBJECT_H__
+#define __LIMA_OBJECT_H__
+
+#include <drm/drm_gem.h>
+
+#include "lima_device.h"
+
+struct lima_bo {
+	struct drm_gem_object gem;
+
+	struct page **pages;
+	dma_addr_t *pages_dma_addr;
+	struct sg_table *sgt;
+	void *vaddr;
+
+	struct mutex lock;
+	struct list_head va;
+};
+
+static inline struct lima_bo *
+to_lima_bo(struct drm_gem_object *obj)
+{
+	return container_of(obj, struct lima_bo, gem);
+}
+
+struct lima_bo *lima_bo_create(struct lima_device *dev, u32 size,
+			       u32 flags, struct sg_table *sgt,
+			       struct reservation_object *resv);
+void lima_bo_destroy(struct lima_bo *bo);
+void *lima_bo_vmap(struct lima_bo *bo);
+void lima_bo_vunmap(struct lima_bo *bo);
+
+#endif
diff --git a/drivers/gpu/drm/lima/lima_pmu.c b/drivers/gpu/drm/lima/lima_pmu.c
new file mode 100644
index 000000000000..571f6d661581
--- /dev/null
+++ b/drivers/gpu/drm/lima/lima_pmu.c
@@ -0,0 +1,60 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
+
+#include <linux/iopoll.h>
+#include <linux/device.h>
+
+#include "lima_device.h"
+#include "lima_pmu.h"
+#include "lima_regs.h"
+
+#define pmu_write(reg, data) writel(data, ip->iomem + reg)
+#define pmu_read(reg) readl(ip->iomem + reg)
+
+static int lima_pmu_wait_cmd(struct lima_ip *ip)
+{
+	struct lima_device *dev = ip->dev;
+	int err;
+	u32 v;
+
+	err = readl_poll_timeout(ip->iomem + LIMA_PMU_INT_RAWSTAT,
+				 v, v & LIMA_PMU_INT_CMD_MASK,
+				 100, 100000);
+	if (err) {
+		dev_err(dev->dev, "timeout wait pmd cmd\n");
+		return err;
+	}
+
+	pmu_write(LIMA_PMU_INT_CLEAR, LIMA_PMU_INT_CMD_MASK);
+	return 0;
+}
+
+int lima_pmu_init(struct lima_ip *ip)
+{
+	int err;
+	u32 stat;
+
+	pmu_write(LIMA_PMU_INT_MASK, 0);
+
+	/* If this value is too low, when in high GPU clk freq,
+	 * GPU will be in unstable state.
+	 */
+	pmu_write(LIMA_PMU_SW_DELAY, 0xffff);
+
+	/* status reg 1=off 0=on */
+	stat = pmu_read(LIMA_PMU_STATUS);
+
+	/* power up all ip */
+	if (stat) {
+		pmu_write(LIMA_PMU_POWER_UP, stat);
+		err = lima_pmu_wait_cmd(ip);
+		if (err)
+			return err;
+	}
+	return 0;
+}
+
+void lima_pmu_fini(struct lima_ip *ip)
+{
+
+}
diff --git a/drivers/gpu/drm/lima/lima_pmu.h b/drivers/gpu/drm/lima/lima_pmu.h
new file mode 100644
index 000000000000..a2a18775eb07
--- /dev/null
+++ b/drivers/gpu/drm/lima/lima_pmu.h
@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
+
+#ifndef __LIMA_PMU_H__
+#define __LIMA_PMU_H__
+
+struct lima_ip;
+
+int lima_pmu_init(struct lima_ip *ip);
+void lima_pmu_fini(struct lima_ip *ip);
+
+#endif
diff --git a/drivers/gpu/drm/lima/lima_pp.c b/drivers/gpu/drm/lima/lima_pp.c
new file mode 100644
index 000000000000..d29721e177bf
--- /dev/null
+++ b/drivers/gpu/drm/lima/lima_pp.c
@@ -0,0 +1,427 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
+
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/device.h>
+#include <linux/slab.h>
+
+#include <drm/lima_drm.h>
+
+#include "lima_device.h"
+#include "lima_pp.h"
+#include "lima_dlbu.h"
+#include "lima_bcast.h"
+#include "lima_vm.h"
+#include "lima_regs.h"
+
+#define pp_write(reg, data) writel(data, ip->iomem + reg)
+#define pp_read(reg) readl(ip->iomem + reg)
+
+static void lima_pp_handle_irq(struct lima_ip *ip, u32 state)
+{
+	struct lima_device *dev = ip->dev;
+	struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp;
+
+	if (state & LIMA_PP_IRQ_MASK_ERROR) {
+		u32 status = pp_read(LIMA_PP_STATUS);
+
+		dev_err(dev->dev, "pp error irq state=%x status=%x\n",
+			state, status);
+
+		pipe->error = true;
+
+		/* mask all interrupts before hard reset */
+		pp_write(LIMA_PP_INT_MASK, 0);
+	}
+
+	pp_write(LIMA_PP_INT_CLEAR, state);
+}
+
+static irqreturn_t lima_pp_irq_handler(int irq, void *data)
+{
+	struct lima_ip *ip = data;
+	struct lima_device *dev = ip->dev;
+	struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp;
+	u32 state = pp_read(LIMA_PP_INT_STATUS);
+
+	/* for shared irq case */
+	if (!state)
+		return IRQ_NONE;
+
+	lima_pp_handle_irq(ip, state);
+
+	if (atomic_dec_and_test(&pipe->task))
+		lima_sched_pipe_task_done(pipe);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t lima_pp_bcast_irq_handler(int irq, void *data)
+{
+	int i;
+	irqreturn_t ret = IRQ_NONE;
+	struct lima_ip *pp_bcast = data;
+	struct lima_device *dev = pp_bcast->dev;
+	struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp;
+	struct drm_lima_m450_pp_frame *frame = pipe->current_task->frame;
+
+	for (i = 0; i < frame->num_pp; i++) {
+		struct lima_ip *ip = pipe->processor[i];
+		u32 status, state;
+
+		if (pipe->done & (1 << i))
+			continue;
+
+		/* status read first in case int state change in the middle
+		 * which may miss the interrupt handling
+		 */
+		status = pp_read(LIMA_PP_STATUS);
+		state = pp_read(LIMA_PP_INT_STATUS);
+
+		if (state) {
+			lima_pp_handle_irq(ip, state);
+			ret = IRQ_HANDLED;
+		} else {
+			if (status & LIMA_PP_STATUS_RENDERING_ACTIVE)
+				continue;
+		}
+
+		pipe->done |= (1 << i);
+		if (atomic_dec_and_test(&pipe->task))
+			lima_sched_pipe_task_done(pipe);
+	}
+
+	return ret;
+}
+
+static void lima_pp_soft_reset_async(struct lima_ip *ip)
+{
+	if (ip->data.async_reset)
+		return;
+
+	pp_write(LIMA_PP_INT_MASK, 0);
+	pp_write(LIMA_PP_INT_RAWSTAT, LIMA_PP_IRQ_MASK_ALL);
+	pp_write(LIMA_PP_CTRL, LIMA_PP_CTRL_SOFT_RESET);
+	ip->data.async_reset = true;
+}
+
+static int lima_pp_soft_reset_poll(struct lima_ip *ip)
+{
+	return !(pp_read(LIMA_PP_STATUS) & LIMA_PP_STATUS_RENDERING_ACTIVE) &&
+		pp_read(LIMA_PP_INT_RAWSTAT) == LIMA_PP_IRQ_RESET_COMPLETED;
+}
+
+static int lima_pp_soft_reset_async_wait_one(struct lima_ip *ip)
+{
+	struct lima_device *dev = ip->dev;
+	int ret;
+
+	ret = lima_poll_timeout(ip, lima_pp_soft_reset_poll, 0, 100);
+	if (ret) {
+		dev_err(dev->dev, "pp %s reset time out\n", lima_ip_name(ip));
+		return ret;
+	}
+
+	pp_write(LIMA_PP_INT_CLEAR, LIMA_PP_IRQ_MASK_ALL);
+	pp_write(LIMA_PP_INT_MASK, LIMA_PP_IRQ_MASK_USED);
+	return 0;
+}
+
+static int lima_pp_soft_reset_async_wait(struct lima_ip *ip)
+{
+	int i, err = 0;
+
+	if (!ip->data.async_reset)
+		return 0;
+
+	if (ip->id == lima_ip_pp_bcast) {
+		struct lima_device *dev = ip->dev;
+		struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp;
+		struct drm_lima_m450_pp_frame *frame = pipe->current_task->frame;
+
+		for (i = 0; i < frame->num_pp; i++)
+			err |= lima_pp_soft_reset_async_wait_one(pipe->processor[i]);
+	} else
+		err = lima_pp_soft_reset_async_wait_one(ip);
+
+	ip->data.async_reset = false;
+	return err;
+}
+
+static void lima_pp_write_frame(struct lima_ip *ip, u32 *frame, u32 *wb)
+{
+	int i, j, n = 0;
+
+	for (i = 0; i < LIMA_PP_FRAME_REG_NUM; i++)
+		writel(frame[i], ip->iomem + LIMA_PP_FRAME + i * 4);
+
+	for (i = 0; i < 3; i++) {
+		for (j = 0; j < LIMA_PP_WB_REG_NUM; j++)
+			writel(wb[n++], ip->iomem + LIMA_PP_WB(i) + j * 4);
+	}
+}
+
+static int lima_pp_hard_reset_poll(struct lima_ip *ip)
+{
+	pp_write(LIMA_PP_PERF_CNT_0_LIMIT, 0xC01A0000);
+	return pp_read(LIMA_PP_PERF_CNT_0_LIMIT) == 0xC01A0000;
+}
+
+static int lima_pp_hard_reset(struct lima_ip *ip)
+{
+	struct lima_device *dev = ip->dev;
+	int ret;
+
+	pp_write(LIMA_PP_PERF_CNT_0_LIMIT, 0xC0FFE000);
+	pp_write(LIMA_PP_INT_MASK, 0);
+	pp_write(LIMA_PP_CTRL, LIMA_PP_CTRL_FORCE_RESET);
+	ret = lima_poll_timeout(ip, lima_pp_hard_reset_poll, 10, 100);
+	if (ret) {
+		dev_err(dev->dev, "pp hard reset timeout\n");
+		return ret;
+	}
+
+	pp_write(LIMA_PP_PERF_CNT_0_LIMIT, 0);
+	pp_write(LIMA_PP_INT_CLEAR, LIMA_PP_IRQ_MASK_ALL);
+	pp_write(LIMA_PP_INT_MASK, LIMA_PP_IRQ_MASK_USED);
+	return 0;
+}
+
+static void lima_pp_print_version(struct lima_ip *ip)
+{
+	u32 version, major, minor;
+	char *name;
+
+	version = pp_read(LIMA_PP_VERSION);
+	major = (version >> 8) & 0xFF;
+	minor = version & 0xFF;
+	switch (version >> 16) {
+	case 0xC807:
+	    name = "mali200";
+		break;
+	case 0xCE07:
+		name = "mali300";
+		break;
+	case 0xCD07:
+		name = "mali400";
+		break;
+	case 0xCF07:
+		name = "mali450";
+		break;
+	default:
+		name = "unknown";
+		break;
+	}
+	dev_info(ip->dev->dev, "%s - %s version major %d minor %d\n",
+		 lima_ip_name(ip), name, major, minor);
+}
+
+int lima_pp_init(struct lima_ip *ip)
+{
+	struct lima_device *dev = ip->dev;
+	int err;
+
+	lima_pp_print_version(ip);
+
+	ip->data.async_reset = false;
+	lima_pp_soft_reset_async(ip);
+	err = lima_pp_soft_reset_async_wait(ip);
+	if (err)
+		return err;
+
+	err = devm_request_irq(dev->dev, ip->irq, lima_pp_irq_handler,
+			       IRQF_SHARED, lima_ip_name(ip), ip);
+	if (err) {
+		dev_err(dev->dev, "pp %s fail to request irq\n",
+			lima_ip_name(ip));
+		return err;
+	}
+
+	dev->pp_version = pp_read(LIMA_PP_VERSION);
+
+	return 0;
+}
+
+void lima_pp_fini(struct lima_ip *ip)
+{
+
+}
+
+int lima_pp_bcast_init(struct lima_ip *ip)
+{
+	struct lima_device *dev = ip->dev;
+	int err;
+
+	err = devm_request_irq(dev->dev, ip->irq, lima_pp_bcast_irq_handler,
+			       IRQF_SHARED, lima_ip_name(ip), ip);
+	if (err) {
+		dev_err(dev->dev, "pp %s fail to request irq\n",
+			lima_ip_name(ip));
+		return err;
+	}
+
+	return 0;
+}
+
+void lima_pp_bcast_fini(struct lima_ip *ip)
+{
+
+}
+
+static int lima_pp_task_validate(struct lima_sched_pipe *pipe,
+				 struct lima_sched_task *task)
+{
+	u32 num_pp;
+
+	if (pipe->bcast_processor) {
+		struct drm_lima_m450_pp_frame *f = task->frame;
+
+		num_pp = f->num_pp;
+
+		if (f->_pad)
+			return -EINVAL;
+	} else {
+		struct drm_lima_m400_pp_frame *f = task->frame;
+
+		num_pp = f->num_pp;
+	}
+
+	if (num_pp == 0 || num_pp > pipe->num_processor)
+		return -EINVAL;
+
+	return 0;
+}
+
+static void lima_pp_task_run(struct lima_sched_pipe *pipe,
+			     struct lima_sched_task *task)
+{
+	if (pipe->bcast_processor) {
+		struct drm_lima_m450_pp_frame *frame = task->frame;
+		struct lima_device *dev = pipe->bcast_processor->dev;
+		struct lima_ip *ip = pipe->bcast_processor;
+		int i;
+
+		pipe->done = 0;
+		atomic_set(&pipe->task, frame->num_pp);
+
+		if (frame->use_dlbu) {
+			lima_dlbu_enable(dev, frame->num_pp);
+
+			frame->frame[LIMA_PP_FRAME >> 2] = LIMA_VA_RESERVE_DLBU;
+			lima_dlbu_set_reg(dev->ip + lima_ip_dlbu, frame->dlbu_regs);
+		} else
+			lima_dlbu_disable(dev);
+
+		lima_bcast_enable(dev, frame->num_pp);
+
+		lima_pp_soft_reset_async_wait(ip);
+
+		lima_pp_write_frame(ip, frame->frame, frame->wb);
+
+		for (i = 0; i < frame->num_pp; i++) {
+			struct lima_ip *ip = pipe->processor[i];
+
+			pp_write(LIMA_PP_STACK, frame->fragment_stack_address[i]);
+			if (!frame->use_dlbu)
+				pp_write(LIMA_PP_FRAME, frame->plbu_array_address[i]);
+		}
+
+		pp_write(LIMA_PP_CTRL, LIMA_PP_CTRL_START_RENDERING);
+	} else {
+		struct drm_lima_m400_pp_frame *frame = task->frame;
+		int i;
+
+		atomic_set(&pipe->task, frame->num_pp);
+
+		for (i = 0; i < frame->num_pp; i++) {
+			struct lima_ip *ip = pipe->processor[i];
+
+			frame->frame[LIMA_PP_FRAME >> 2] =
+				frame->plbu_array_address[i];
+			frame->frame[LIMA_PP_STACK >> 2] =
+				frame->fragment_stack_address[i];
+
+			lima_pp_soft_reset_async_wait(ip);
+
+			lima_pp_write_frame(ip, frame->frame, frame->wb);
+
+			pp_write(LIMA_PP_CTRL, LIMA_PP_CTRL_START_RENDERING);
+		}
+	}
+}
+
+static void lima_pp_task_fini(struct lima_sched_pipe *pipe)
+{
+	if (pipe->bcast_processor)
+		lima_pp_soft_reset_async(pipe->bcast_processor);
+	else {
+		int i;
+
+		for (i = 0; i < pipe->num_processor; i++)
+			lima_pp_soft_reset_async(pipe->processor[i]);
+	}
+}
+
+static void lima_pp_task_error(struct lima_sched_pipe *pipe)
+{
+	int i;
+
+	for (i = 0; i < pipe->num_processor; i++) {
+		struct lima_ip *ip = pipe->processor[i];
+
+		dev_err(ip->dev->dev, "pp task error %d int_state=%x status=%x\n",
+			i, pp_read(LIMA_PP_INT_STATUS), pp_read(LIMA_PP_STATUS));
+
+		lima_pp_hard_reset(ip);
+	}
+}
+
+static void lima_pp_task_mmu_error(struct lima_sched_pipe *pipe)
+{
+	if (atomic_dec_and_test(&pipe->task))
+		lima_sched_pipe_task_done(pipe);
+}
+
+static struct kmem_cache *lima_pp_task_slab;
+static int lima_pp_task_slab_refcnt;
+
+int lima_pp_pipe_init(struct lima_device *dev)
+{
+	int frame_size;
+	struct lima_sched_pipe *pipe = dev->pipe + lima_pipe_pp;
+
+	if (dev->id == lima_gpu_mali400)
+		frame_size = sizeof(struct drm_lima_m400_pp_frame);
+	else
+		frame_size = sizeof(struct drm_lima_m450_pp_frame);
+
+	if (!lima_pp_task_slab) {
+		lima_pp_task_slab = kmem_cache_create_usercopy(
+			"lima_pp_task", sizeof(struct lima_sched_task) + frame_size,
+			0, SLAB_HWCACHE_ALIGN, sizeof(struct lima_sched_task),
+			frame_size, NULL);
+		if (!lima_pp_task_slab)
+			return -ENOMEM;
+	}
+	lima_pp_task_slab_refcnt++;
+
+	pipe->frame_size = frame_size;
+	pipe->task_slab = lima_pp_task_slab;
+
+	pipe->task_validate = lima_pp_task_validate;
+	pipe->task_run = lima_pp_task_run;
+	pipe->task_fini = lima_pp_task_fini;
+	pipe->task_error = lima_pp_task_error;
+	pipe->task_mmu_error = lima_pp_task_mmu_error;
+
+	return 0;
+}
+
+void lima_pp_pipe_fini(struct lima_device *dev)
+{
+	if (!--lima_pp_task_slab_refcnt) {
+		kmem_cache_destroy(lima_pp_task_slab);
+		lima_pp_task_slab = NULL;
+	}
+}
diff --git a/drivers/gpu/drm/lima/lima_pp.h b/drivers/gpu/drm/lima/lima_pp.h
new file mode 100644
index 000000000000..bf60c77b2633
--- /dev/null
+++ b/drivers/gpu/drm/lima/lima_pp.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
+
+#ifndef __LIMA_PP_H__
+#define __LIMA_PP_H__
+
+struct lima_ip;
+struct lima_device;
+
+int lima_pp_init(struct lima_ip *ip);
+void lima_pp_fini(struct lima_ip *ip);
+
+int lima_pp_bcast_init(struct lima_ip *ip);
+void lima_pp_bcast_fini(struct lima_ip *ip);
+
+int lima_pp_pipe_init(struct lima_device *dev);
+void lima_pp_pipe_fini(struct lima_device *dev);
+
+#endif
diff --git a/drivers/gpu/drm/lima/lima_regs.h b/drivers/gpu/drm/lima/lima_regs.h
new file mode 100644
index 000000000000..ace8ecefbe90
--- /dev/null
+++ b/drivers/gpu/drm/lima/lima_regs.h
@@ -0,0 +1,298 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright 2010-2017 ARM Limited. All rights reserved.
+ * Copyright 2017-2019 Qiang Yu <yuq825@gmail.com>
+ */
+
+#ifndef __LIMA_REGS_H__
+#define __LIMA_REGS_H__
+
+/* This file's register definition is collected from the
+ * official ARM Mali Utgard GPU kernel driver source code
+ */
+
+/* PMU regs */
+#define LIMA_PMU_POWER_UP                  0x00
+#define LIMA_PMU_POWER_DOWN                0x04
+#define   LIMA_PMU_POWER_GP0_MASK          BIT(0)
+#define   LIMA_PMU_POWER_L2_MASK           BIT(1)
+#define   LIMA_PMU_POWER_PP_MASK(i)        BIT(2 + i)
+
+/*
+ * On Mali450 each block automatically starts up its corresponding L2
+ * and the PPs are not fully independent controllable.
+ * Instead PP0, PP1-3 and PP4-7 can be turned on or off.
+ */
+#define   LIMA450_PMU_POWER_PP0_MASK       BIT(1)
+#define   LIMA450_PMU_POWER_PP13_MASK      BIT(2)
+#define   LIMA450_PMU_POWER_PP47_MASK      BIT(3)
+
+#define LIMA_PMU_STATUS                    0x08
+#define LIMA_PMU_INT_MASK                  0x0C
+#define LIMA_PMU_INT_RAWSTAT               0x10
+#define LIMA_PMU_INT_CLEAR                 0x18
+#define   LIMA_PMU_INT_CMD_MASK            BIT(0)
+#define LIMA_PMU_SW_DELAY                  0x1C
+
+/* L2 cache regs */
+#define LIMA_L2_CACHE_SIZE                   0x0004
+#define LIMA_L2_CACHE_STATUS                 0x0008
+#define   LIMA_L2_CACHE_STATUS_COMMAND_BUSY  BIT(0)
+#define   LIMA_L2_CACHE_STATUS_DATA_BUSY     BIT(1)
+#define LIMA_L2_CACHE_COMMAND                0x0010
+#define   LIMA_L2_CACHE_COMMAND_CLEAR_ALL    BIT(0)
+#define LIMA_L2_CACHE_CLEAR_PAGE             0x0014
+#define LIMA_L2_CACHE_MAX_READS              0x0018
+#define LIMA_L2_CACHE_ENABLE                 0x001C
+#define   LIMA_L2_CACHE_ENABLE_ACCESS        BIT(0)
+#define   LIMA_L2_CACHE_ENABLE_READ_ALLOCATE BIT(1)
+#define LIMA_L2_CACHE_PERFCNT_SRC0           0x0020
+#define LIMA_L2_CACHE_PERFCNT_VAL0           0x0024
+#define LIMA_L2_CACHE_PERFCNT_SRC1           0x0028
+#define LIMA_L2_CACHE_ERFCNT_VAL1            0x002C
+
+/* GP regs */
+#define LIMA_GP_VSCL_START_ADDR                0x00
+#define LIMA_GP_VSCL_END_ADDR                  0x04
+#define LIMA_GP_PLBUCL_START_ADDR              0x08
+#define LIMA_GP_PLBUCL_END_ADDR                0x0c
+#define LIMA_GP_PLBU_ALLOC_START_ADDR          0x10
+#define LIMA_GP_PLBU_ALLOC_END_ADDR            0x14
+#define LIMA_GP_CMD                            0x20
+#define   LIMA_GP_CMD_START_VS                 BIT(0)
+#define   LIMA_GP_CMD_START_PLBU               BIT(1)
+#define   LIMA_GP_CMD_UPDATE_PLBU_ALLOC        BIT(4)
+#define   LIMA_GP_CMD_RESET                    BIT(5)
+#define   LIMA_GP_CMD_FORCE_HANG               BIT(6)
+#define   LIMA_GP_CMD_STOP_BUS                 BIT(9)
+#define   LIMA_GP_CMD_SOFT_RESET               BIT(10)
+#define LIMA_GP_INT_RAWSTAT                    0x24
+#define LIMA_GP_INT_CLEAR                      0x28
+#define LIMA_GP_INT_MASK                       0x2C
+#define LIMA_GP_INT_STAT                       0x30
+#define   LIMA_GP_IRQ_VS_END_CMD_LST           BIT(0)
+#define   LIMA_GP_IRQ_PLBU_END_CMD_LST         BIT(1)
+#define   LIMA_GP_IRQ_PLBU_OUT_OF_MEM          BIT(2)
+#define   LIMA_GP_IRQ_VS_SEM_IRQ               BIT(3)
+#define   LIMA_GP_IRQ_PLBU_SEM_IRQ             BIT(4)
+#define   LIMA_GP_IRQ_HANG                     BIT(5)
+#define   LIMA_GP_IRQ_FORCE_HANG               BIT(6)
+#define   LIMA_GP_IRQ_PERF_CNT_0_LIMIT         BIT(7)
+#define   LIMA_GP_IRQ_PERF_CNT_1_LIMIT         BIT(8)
+#define   LIMA_GP_IRQ_WRITE_BOUND_ERR          BIT(9)
+#define   LIMA_GP_IRQ_SYNC_ERROR               BIT(10)
+#define   LIMA_GP_IRQ_AXI_BUS_ERROR            BIT(11)
+#define   LIMA_GP_IRQ_AXI_BUS_STOPPED          BIT(12)
+#define   LIMA_GP_IRQ_VS_INVALID_CMD           BIT(13)
+#define   LIMA_GP_IRQ_PLB_INVALID_CMD          BIT(14)
+#define   LIMA_GP_IRQ_RESET_COMPLETED          BIT(19)
+#define   LIMA_GP_IRQ_SEMAPHORE_UNDERFLOW      BIT(20)
+#define   LIMA_GP_IRQ_SEMAPHORE_OVERFLOW       BIT(21)
+#define   LIMA_GP_IRQ_PTR_ARRAY_OUT_OF_BOUNDS  BIT(22)
+#define LIMA_GP_WRITE_BOUND_LOW                0x34
+#define LIMA_GP_PERF_CNT_0_ENABLE              0x3C
+#define LIMA_GP_PERF_CNT_1_ENABLE              0x40
+#define LIMA_GP_PERF_CNT_0_SRC                 0x44
+#define LIMA_GP_PERF_CNT_1_SRC                 0x48
+#define LIMA_GP_PERF_CNT_0_VALUE               0x4C
+#define LIMA_GP_PERF_CNT_1_VALUE               0x50
+#define LIMA_GP_PERF_CNT_0_LIMIT               0x54
+#define LIMA_GP_STATUS                         0x68
+#define   LIMA_GP_STATUS_VS_ACTIVE             BIT(1)
+#define   LIMA_GP_STATUS_BUS_STOPPED           BIT(2)
+#define   LIMA_GP_STATUS_PLBU_ACTIVE           BIT(3)
+#define   LIMA_GP_STATUS_BUS_ERROR             BIT(6)
+#define   LIMA_GP_STATUS_WRITE_BOUND_ERR       BIT(8)
+#define LIMA_GP_VERSION                        0x6C
+#define LIMA_GP_VSCL_START_ADDR_READ           0x80
+#define LIMA_GP_PLBCL_START_ADDR_READ          0x84
+#define LIMA_GP_CONTR_AXI_BUS_ERROR_STAT       0x94
+
+#define LIMA_GP_IRQ_MASK_ALL		   \
+	(				   \
+	 LIMA_GP_IRQ_VS_END_CMD_LST      | \
+	 LIMA_GP_IRQ_PLBU_END_CMD_LST    | \
+	 LIMA_GP_IRQ_PLBU_OUT_OF_MEM     | \
+	 LIMA_GP_IRQ_VS_SEM_IRQ          | \
+	 LIMA_GP_IRQ_PLBU_SEM_IRQ        | \
+	 LIMA_GP_IRQ_HANG                | \
+	 LIMA_GP_IRQ_FORCE_HANG          | \
+	 LIMA_GP_IRQ_PERF_CNT_0_LIMIT    | \
+	 LIMA_GP_IRQ_PERF_CNT_1_LIMIT    | \
+	 LIMA_GP_IRQ_WRITE_BOUND_ERR     | \
+	 LIMA_GP_IRQ_SYNC_ERROR          | \
+	 LIMA_GP_IRQ_AXI_BUS_ERROR       | \
+	 LIMA_GP_IRQ_AXI_BUS_STOPPED     | \
+	 LIMA_GP_IRQ_VS_INVALID_CMD      | \
+	 LIMA_GP_IRQ_PLB_INVALID_CMD     | \
+	 LIMA_GP_IRQ_RESET_COMPLETED     | \
+	 LIMA_GP_IRQ_SEMAPHORE_UNDERFLOW | \
+	 LIMA_GP_IRQ_SEMAPHORE_OVERFLOW  | \
+	 LIMA_GP_IRQ_PTR_ARRAY_OUT_OF_BOUNDS)
+
+#define LIMA_GP_IRQ_MASK_ERROR             \
+	(                                  \
+	 LIMA_GP_IRQ_PLBU_OUT_OF_MEM     | \
+	 LIMA_GP_IRQ_FORCE_HANG          | \
+	 LIMA_GP_IRQ_WRITE_BOUND_ERR     | \
+	 LIMA_GP_IRQ_SYNC_ERROR          | \
+	 LIMA_GP_IRQ_AXI_BUS_ERROR       | \
+	 LIMA_GP_IRQ_VS_INVALID_CMD      | \
+	 LIMA_GP_IRQ_PLB_INVALID_CMD     | \
+	 LIMA_GP_IRQ_SEMAPHORE_UNDERFLOW | \
+	 LIMA_GP_IRQ_SEMAPHORE_OVERFLOW  | \
+	 LIMA_GP_IRQ_PTR_ARRAY_OUT_OF_BOUNDS)
+
+#define LIMA_GP_IRQ_MASK_USED		   \
+	(				   \
+	 LIMA_GP_IRQ_VS_END_CMD_LST      | \
+	 LIMA_GP_IRQ_PLBU_END_CMD_LST    | \
+	 LIMA_GP_IRQ_MASK_ERROR)
+
+/* PP regs */
+#define LIMA_PP_FRAME                        0x0000
+#define LIMA_PP_RSW			     0x0004
+#define LIMA_PP_STACK			     0x0030
+#define LIMA_PP_STACK_SIZE		     0x0034
+#define LIMA_PP_ORIGIN_OFFSET_X	             0x0040
+#define LIMA_PP_WB(i)                        (0x0100 * (i + 1))
+#define   LIMA_PP_WB_SOURCE_SELECT           0x0000
+#define	  LIMA_PP_WB_SOURCE_ADDR             0x0004
+
+#define LIMA_PP_VERSION                      0x1000
+#define LIMA_PP_CURRENT_REND_LIST_ADDR       0x1004
+#define LIMA_PP_STATUS                       0x1008
+#define   LIMA_PP_STATUS_RENDERING_ACTIVE    BIT(0)
+#define   LIMA_PP_STATUS_BUS_STOPPED         BIT(4)
+#define LIMA_PP_CTRL                         0x100c
+#define   LIMA_PP_CTRL_STOP_BUS              BIT(0)
+#define   LIMA_PP_CTRL_FLUSH_CACHES          BIT(3)
+#define   LIMA_PP_CTRL_FORCE_RESET           BIT(5)
+#define   LIMA_PP_CTRL_START_RENDERING       BIT(6)
+#define   LIMA_PP_CTRL_SOFT_RESET            BIT(7)
+#define LIMA_PP_INT_RAWSTAT                  0x1020
+#define LIMA_PP_INT_CLEAR                    0x1024
+#define LIMA_PP_INT_MASK                     0x1028
+#define LIMA_PP_INT_STATUS                   0x102c
+#define   LIMA_PP_IRQ_END_OF_FRAME           BIT(0)
+#define   LIMA_PP_IRQ_END_OF_TILE            BIT(1)
+#define   LIMA_PP_IRQ_HANG                   BIT(2)
+#define   LIMA_PP_IRQ_FORCE_HANG             BIT(3)
+#define   LIMA_PP_IRQ_BUS_ERROR              BIT(4)
+#define   LIMA_PP_IRQ_BUS_STOP               BIT(5)
+#define   LIMA_PP_IRQ_CNT_0_LIMIT            BIT(6)
+#define   LIMA_PP_IRQ_CNT_1_LIMIT            BIT(7)
+#define   LIMA_PP_IRQ_WRITE_BOUNDARY_ERROR   BIT(8)
+#define   LIMA_PP_IRQ_INVALID_PLIST_COMMAND  BIT(9)
+#define   LIMA_PP_IRQ_CALL_STACK_UNDERFLOW   BIT(10)
+#define   LIMA_PP_IRQ_CALL_STACK_OVERFLOW    BIT(11)
+#define   LIMA_PP_IRQ_RESET_COMPLETED        BIT(12)
+#define LIMA_PP_WRITE_BOUNDARY_LOW           0x1044
+#define LIMA_PP_BUS_ERROR_STATUS             0x1050
+#define LIMA_PP_PERF_CNT_0_ENABLE            0x1080
+#define LIMA_PP_PERF_CNT_0_SRC               0x1084
+#define LIMA_PP_PERF_CNT_0_LIMIT             0x1088
+#define LIMA_PP_PERF_CNT_0_VALUE             0x108c
+#define LIMA_PP_PERF_CNT_1_ENABLE            0x10a0
+#define LIMA_PP_PERF_CNT_1_SRC               0x10a4
+#define LIMA_PP_PERF_CNT_1_LIMIT             0x10a8
+#define LIMA_PP_PERF_CNT_1_VALUE             0x10ac
+#define LIMA_PP_PERFMON_CONTR                0x10b0
+#define LIMA_PP_PERFMON_BASE                 0x10b4
+
+#define LIMA_PP_IRQ_MASK_ALL                 \
+	(                                    \
+	 LIMA_PP_IRQ_END_OF_FRAME          | \
+	 LIMA_PP_IRQ_END_OF_TILE           | \
+	 LIMA_PP_IRQ_HANG                  | \
+	 LIMA_PP_IRQ_FORCE_HANG            | \
+	 LIMA_PP_IRQ_BUS_ERROR             | \
+	 LIMA_PP_IRQ_BUS_STOP              | \
+	 LIMA_PP_IRQ_CNT_0_LIMIT           | \
+	 LIMA_PP_IRQ_CNT_1_LIMIT           | \
+	 LIMA_PP_IRQ_WRITE_BOUNDARY_ERROR  | \
+	 LIMA_PP_IRQ_INVALID_PLIST_COMMAND | \
+	 LIMA_PP_IRQ_CALL_STACK_UNDERFLOW  | \
+	 LIMA_PP_IRQ_CALL_STACK_OVERFLOW   | \
+	 LIMA_PP_IRQ_RESET_COMPLETED)
+
+#define LIMA_PP_IRQ_MASK_ERROR               \
+	(                                    \
+	 LIMA_PP_IRQ_FORCE_HANG            | \
+	 LIMA_PP_IRQ_BUS_ERROR             | \
+	 LIMA_PP_IRQ_WRITE_BOUNDARY_ERROR  | \
+	 LIMA_PP_IRQ_INVALID_PLIST_COMMAND | \
+	 LIMA_PP_IRQ_CALL_STACK_UNDERFLOW  | \
+	 LIMA_PP_IRQ_CALL_STACK_OVERFLOW)
+
+#define LIMA_PP_IRQ_MASK_USED                \
+	(                                    \
+	 LIMA_PP_IRQ_END_OF_FRAME          | \
+	 LIMA_PP_IRQ_MASK_ERROR)
+
+/* MMU regs */
+#define LIMA_MMU_DTE_ADDR                     0x0000
+#define LIMA_MMU_STATUS                       0x0004
+#define   LIMA_MMU_STATUS_PAGING_ENABLED      BIT(0)
+#define   LIMA_MMU_STATUS_PAGE_FAULT_ACTIVE   BIT(1)
+#define   LIMA_MMU_STATUS_STALL_ACTIVE        BIT(2)
+#define   LIMA_MMU_STATUS_IDLE                BIT(3)
+#define   LIMA_MMU_STATUS_REPLAY_BUFFER_EMPTY BIT(4)
+#define   LIMA_MMU_STATUS_PAGE_FAULT_IS_WRITE BIT(5)
+#define   LIMA_MMU_STATUS_BUS_ID(x)           ((x >> 6) & 0x1F)
+#define LIMA_MMU_COMMAND                      0x0008
+#define   LIMA_MMU_COMMAND_ENABLE_PAGING      0x00
+#define   LIMA_MMU_COMMAND_DISABLE_PAGING     0x01
+#define   LIMA_MMU_COMMAND_ENABLE_STALL       0x02
+#define   LIMA_MMU_COMMAND_DISABLE_STALL      0x03
+#define   LIMA_MMU_COMMAND_ZAP_CACHE          0x04
+#define   LIMA_MMU_COMMAND_PAGE_FAULT_DONE    0x05
+#define   LIMA_MMU_COMMAND_HARD_RESET         0x06
+#define LIMA_MMU_PAGE_FAULT_ADDR              0x000C
+#define LIMA_MMU_ZAP_ONE_LINE                 0x0010
+#define LIMA_MMU_INT_RAWSTAT                  0x0014
+#define LIMA_MMU_INT_CLEAR                    0x0018
+#define LIMA_MMU_INT_MASK                     0x001C
+#define   LIMA_MMU_INT_PAGE_FAULT             BIT(0)
+#define   LIMA_MMU_INT_READ_BUS_ERROR         BIT(1)
+#define LIMA_MMU_INT_STATUS                   0x0020
+
+#define LIMA_VM_FLAG_PRESENT          BIT(0)
+#define LIMA_VM_FLAG_READ_PERMISSION  BIT(1)
+#define LIMA_VM_FLAG_WRITE_PERMISSION BIT(2)
+#define LIMA_VM_FLAG_OVERRIDE_CACHE   BIT(3)
+#define LIMA_VM_FLAG_WRITE_CACHEABLE  BIT(4)
+#define LIMA_VM_FLAG_WRITE_ALLOCATE   BIT(5)
+#define LIMA_VM_FLAG_WRITE_BUFFERABLE BIT(6)
+#define LIMA_VM_FLAG_READ_CACHEABLE   BIT(7)
+#define LIMA_VM_FLAG_READ_ALLOCATE    BIT(8)
+#define LIMA_VM_FLAG_MASK             0x1FF
+
+#define LIMA_VM_FLAGS_CACHE (			 \
+		LIMA_VM_FLAG_PRESENT |		 \
+		LIMA_VM_FLAG_READ_PERMISSION |	 \
+		LIMA_VM_FLAG_WRITE_PERMISSION |	 \
+		LIMA_VM_FLAG_OVERRIDE_CACHE |	 \
+		LIMA_VM_FLAG_WRITE_CACHEABLE |	 \
+		LIMA_VM_FLAG_WRITE_BUFFERABLE |	 \
+		LIMA_VM_FLAG_READ_CACHEABLE |	 \
+		LIMA_VM_FLAG_READ_ALLOCATE)
+
+#define LIMA_VM_FLAGS_UNCACHE (			\
+		LIMA_VM_FLAG_PRESENT |		\
+		LIMA_VM_FLAG_READ_PERMISSION |	\
+		LIMA_VM_FLAG_WRITE_PERMISSION)
+
+/* DLBU regs */
+#define LIMA_DLBU_MASTER_TLLIST_PHYS_ADDR  0x0000
+#define	LIMA_DLBU_MASTER_TLLIST_VADDR      0x0004
+#define	LIMA_DLBU_TLLIST_VBASEADDR         0x0008
+#define	LIMA_DLBU_FB_DIM                   0x000C
+#define	LIMA_DLBU_TLLIST_CONF              0x0010
+#define	LIMA_DLBU_START_TILE_POS           0x0014
+#define	LIMA_DLBU_PP_ENABLE_MASK           0x0018
+
+/* BCAST regs */
+#define LIMA_BCAST_BROADCAST_MASK    0x0
+#define LIMA_BCAST_INTERRUPT_MASK    0x4
+
+#endif
diff --git a/drivers/gpu/drm/lima/lima_sched.c b/drivers/gpu/drm/lima/lima_sched.c
new file mode 100644
index 000000000000..d53bd45f8d96
--- /dev/null
+++ b/drivers/gpu/drm/lima/lima_sched.c
@@ -0,0 +1,362 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
+
+#include <linux/kthread.h>
+#include <linux/slab.h>
+#include <linux/xarray.h>
+
+#include "lima_drv.h"
+#include "lima_sched.h"
+#include "lima_vm.h"
+#include "lima_mmu.h"
+#include "lima_l2_cache.h"
+#include "lima_object.h"
+
+struct lima_fence {
+	struct dma_fence base;
+	struct lima_sched_pipe *pipe;
+};
+
+static struct kmem_cache *lima_fence_slab;
+static int lima_fence_slab_refcnt;
+
+int lima_sched_slab_init(void)
+{
+	if (!lima_fence_slab) {
+		lima_fence_slab = kmem_cache_create(
+			"lima_fence", sizeof(struct lima_fence), 0,
+			SLAB_HWCACHE_ALIGN, NULL);
+		if (!lima_fence_slab)
+			return -ENOMEM;
+	}
+
+	lima_fence_slab_refcnt++;
+	return 0;
+}
+
+void lima_sched_slab_fini(void)
+{
+	if (!--lima_fence_slab_refcnt) {
+		kmem_cache_destroy(lima_fence_slab);
+		lima_fence_slab = NULL;
+	}
+}
+
+static inline struct lima_fence *to_lima_fence(struct dma_fence *fence)
+{
+	return container_of(fence, struct lima_fence, base);
+}
+
+static const char *lima_fence_get_driver_name(struct dma_fence *fence)
+{
+	return "lima";
+}
+
+static const char *lima_fence_get_timeline_name(struct dma_fence *fence)
+{
+	struct lima_fence *f = to_lima_fence(fence);
+
+	return f->pipe->base.name;
+}
+
+static void lima_fence_release_rcu(struct rcu_head *rcu)
+{
+	struct dma_fence *f = container_of(rcu, struct dma_fence, rcu);
+	struct lima_fence *fence = to_lima_fence(f);
+
+	kmem_cache_free(lima_fence_slab, fence);
+}
+
+static void lima_fence_release(struct dma_fence *fence)
+{
+	struct lima_fence *f = to_lima_fence(fence);
+
+	call_rcu(&f->base.rcu, lima_fence_release_rcu);
+}
+
+static const struct dma_fence_ops lima_fence_ops = {
+	.get_driver_name = lima_fence_get_driver_name,
+	.get_timeline_name = lima_fence_get_timeline_name,
+	.release = lima_fence_release,
+};
+
+static struct lima_fence *lima_fence_create(struct lima_sched_pipe *pipe)
+{
+	struct lima_fence *fence;
+
+	fence = kmem_cache_zalloc(lima_fence_slab, GFP_KERNEL);
+	if (!fence)
+		return NULL;
+
+	fence->pipe = pipe;
+	dma_fence_init(&fence->base, &lima_fence_ops, &pipe->fence_lock,
+		       pipe->fence_context, ++pipe->fence_seqno);
+
+	return fence;
+}
+
+static inline struct lima_sched_task *to_lima_task(struct drm_sched_job *job)
+{
+	return container_of(job, struct lima_sched_task, base);
+}
+
+static inline struct lima_sched_pipe *to_lima_pipe(struct drm_gpu_scheduler *sched)
+{
+	return container_of(sched, struct lima_sched_pipe, base);
+}
+
+int lima_sched_task_init(struct lima_sched_task *task,
+			 struct lima_sched_context *context,
+			 struct lima_bo **bos, int num_bos,
+			 struct lima_vm *vm)
+{
+	int err, i;
+
+	task->bos = kmemdup(bos, sizeof(*bos) * num_bos, GFP_KERNEL);
+	if (!task->bos)
+		return -ENOMEM;
+
+	for (i = 0; i < num_bos; i++)
+		drm_gem_object_get(&bos[i]->gem);
+
+	err = drm_sched_job_init(&task->base, &context->base, vm);
+	if (err) {
+		kfree(task->bos);
+		return err;
+	}
+
+	task->num_bos = num_bos;
+	task->vm = lima_vm_get(vm);
+
+	xa_init_flags(&task->deps, XA_FLAGS_ALLOC);
+
+	return 0;
+}
+
+void lima_sched_task_fini(struct lima_sched_task *task)
+{
+	struct dma_fence *fence;
+	unsigned long index;
+	int i;
+
+	drm_sched_job_cleanup(&task->base);
+
+	xa_for_each(&task->deps, index, fence) {
+		dma_fence_put(fence);
+	}
+	xa_destroy(&task->deps);
+
+	if (task->bos) {
+		for (i = 0; i < task->num_bos; i++)
+			drm_gem_object_put_unlocked(&task->bos[i]->gem);
+		kfree(task->bos);
+	}
+
+	lima_vm_put(task->vm);
+}
+
+int lima_sched_context_init(struct lima_sched_pipe *pipe,
+			    struct lima_sched_context *context,
+			    atomic_t *guilty)
+{
+	struct drm_sched_rq *rq = pipe->base.sched_rq + DRM_SCHED_PRIORITY_NORMAL;
+
+	return drm_sched_entity_init(&context->base, &rq, 1, guilty);
+}
+
+void lima_sched_context_fini(struct lima_sched_pipe *pipe,
+			     struct lima_sched_context *context)
+{
+	drm_sched_entity_fini(&context->base);
+}
+
+struct dma_fence *lima_sched_context_queue_task(struct lima_sched_context *context,
+						struct lima_sched_task *task)
+{
+	struct dma_fence *fence = dma_fence_get(&task->base.s_fence->finished);
+
+	drm_sched_entity_push_job(&task->base, &context->base);
+	return fence;
+}
+
+static struct dma_fence *lima_sched_dependency(struct drm_sched_job *job,
+					       struct drm_sched_entity *entity)
+{
+	struct lima_sched_task *task = to_lima_task(job);
+
+	if (!xa_empty(&task->deps))
+		return xa_erase(&task->deps, task->last_dep++);
+
+	return NULL;
+}
+
+static struct dma_fence *lima_sched_run_job(struct drm_sched_job *job)
+{
+	struct lima_sched_task *task = to_lima_task(job);
+	struct lima_sched_pipe *pipe = to_lima_pipe(job->sched);
+	struct lima_fence *fence;
+	struct dma_fence *ret;
+	struct lima_vm *vm = NULL, *last_vm = NULL;
+	int i;
+
+	/* after GPU reset */
+	if (job->s_fence->finished.error < 0)
+		return NULL;
+
+	fence = lima_fence_create(pipe);
+	if (!fence)
+		return NULL;
+	task->fence = &fence->base;
+
+	/* for caller usage of the fence, otherwise irq handler
+	 * may consume the fence before caller use it
+	 */
+	ret = dma_fence_get(task->fence);
+
+	pipe->current_task = task;
+
+	/* this is needed for MMU to work correctly, otherwise GP/PP
+	 * will hang or page fault for unknown reason after running for
+	 * a while.
+	 *
+	 * Need to investigate:
+	 * 1. is it related to TLB
+	 * 2. how much performance will be affected by L2 cache flush
+	 * 3. can we reduce the calling of this function because all
+	 *    GP/PP use the same L2 cache on mali400
+	 *
+	 * TODO:
+	 * 1. move this to task fini to save some wait time?
+	 * 2. when GP/PP use different l2 cache, need PP wait GP l2
+	 *    cache flush?
+	 */
+	for (i = 0; i < pipe->num_l2_cache; i++)
+		lima_l2_cache_flush(pipe->l2_cache[i]);
+
+	if (task->vm != pipe->current_vm) {
+		vm = lima_vm_get(task->vm);
+		last_vm = pipe->current_vm;
+		pipe->current_vm = task->vm;
+	}
+
+	if (pipe->bcast_mmu)
+		lima_mmu_switch_vm(pipe->bcast_mmu, vm);
+	else {
+		for (i = 0; i < pipe->num_mmu; i++)
+			lima_mmu_switch_vm(pipe->mmu[i], vm);
+	}
+
+	if (last_vm)
+		lima_vm_put(last_vm);
+
+	pipe->error = false;
+	pipe->task_run(pipe, task);
+
+	return task->fence;
+}
+
+static void lima_sched_handle_error_task(struct lima_sched_pipe *pipe,
+					 struct lima_sched_task *task)
+{
+	drm_sched_stop(&pipe->base);
+
+	if (task)
+		drm_sched_increase_karma(&task->base);
+
+	pipe->task_error(pipe);
+
+	if (pipe->bcast_mmu)
+		lima_mmu_page_fault_resume(pipe->bcast_mmu);
+	else {
+		int i;
+
+		for (i = 0; i < pipe->num_mmu; i++)
+			lima_mmu_page_fault_resume(pipe->mmu[i]);
+	}
+
+	if (pipe->current_vm)
+		lima_vm_put(pipe->current_vm);
+
+	pipe->current_vm = NULL;
+	pipe->current_task = NULL;
+
+	drm_sched_resubmit_jobs(&pipe->base);
+	drm_sched_start(&pipe->base, true);
+}
+
+static void lima_sched_timedout_job(struct drm_sched_job *job)
+{
+	struct lima_sched_pipe *pipe = to_lima_pipe(job->sched);
+	struct lima_sched_task *task = to_lima_task(job);
+
+	DRM_ERROR("lima job timeout\n");
+
+	lima_sched_handle_error_task(pipe, task);
+}
+
+static void lima_sched_free_job(struct drm_sched_job *job)
+{
+	struct lima_sched_task *task = to_lima_task(job);
+	struct lima_sched_pipe *pipe = to_lima_pipe(job->sched);
+	struct lima_vm *vm = task->vm;
+	struct lima_bo **bos = task->bos;
+	int i;
+
+	dma_fence_put(task->fence);
+
+	for (i = 0; i < task->num_bos; i++)
+		lima_vm_bo_del(vm, bos[i]);
+
+	lima_sched_task_fini(task);
+	kmem_cache_free(pipe->task_slab, task);
+}
+
+static const struct drm_sched_backend_ops lima_sched_ops = {
+	.dependency = lima_sched_dependency,
+	.run_job = lima_sched_run_job,
+	.timedout_job = lima_sched_timedout_job,
+	.free_job = lima_sched_free_job,
+};
+
+static void lima_sched_error_work(struct work_struct *work)
+{
+	struct lima_sched_pipe *pipe =
+		container_of(work, struct lima_sched_pipe, error_work);
+	struct lima_sched_task *task = pipe->current_task;
+
+	lima_sched_handle_error_task(pipe, task);
+}
+
+int lima_sched_pipe_init(struct lima_sched_pipe *pipe, const char *name)
+{
+	long timeout;
+
+	if (lima_sched_timeout_ms <= 0)
+		timeout = MAX_SCHEDULE_TIMEOUT;
+	else
+		timeout = msecs_to_jiffies(lima_sched_timeout_ms);
+
+	pipe->fence_context = dma_fence_context_alloc(1);
+	spin_lock_init(&pipe->fence_lock);
+
+	INIT_WORK(&pipe->error_work, lima_sched_error_work);
+
+	return drm_sched_init(&pipe->base, &lima_sched_ops, 1, 0, timeout, name);
+}
+
+void lima_sched_pipe_fini(struct lima_sched_pipe *pipe)
+{
+	drm_sched_fini(&pipe->base);
+}
+
+void lima_sched_pipe_task_done(struct lima_sched_pipe *pipe)
+{
+	if (pipe->error)
+		schedule_work(&pipe->error_work);
+	else {
+		struct lima_sched_task *task = pipe->current_task;
+
+		pipe->task_fini(pipe);
+		dma_fence_signal(task->fence);
+	}
+}
diff --git a/drivers/gpu/drm/lima/lima_sched.h b/drivers/gpu/drm/lima/lima_sched.h
new file mode 100644
index 000000000000..928af91c1118
--- /dev/null
+++ b/drivers/gpu/drm/lima/lima_sched.h
@@ -0,0 +1,102 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
+
+#ifndef __LIMA_SCHED_H__
+#define __LIMA_SCHED_H__
+
+#include <drm/gpu_scheduler.h>
+
+struct lima_vm;
+
+struct lima_sched_task {
+	struct drm_sched_job base;
+
+	struct lima_vm *vm;
+	void *frame;
+
+	struct xarray deps;
+	unsigned long last_dep;
+
+	struct lima_bo **bos;
+	int num_bos;
+
+	/* pipe fence */
+	struct dma_fence *fence;
+};
+
+struct lima_sched_context {
+	struct drm_sched_entity base;
+};
+
+#define LIMA_SCHED_PIPE_MAX_MMU       8
+#define LIMA_SCHED_PIPE_MAX_L2_CACHE  2
+#define LIMA_SCHED_PIPE_MAX_PROCESSOR 8
+
+struct lima_ip;
+
+struct lima_sched_pipe {
+	struct drm_gpu_scheduler base;
+
+	u64 fence_context;
+	u32 fence_seqno;
+	spinlock_t fence_lock;
+
+	struct lima_sched_task *current_task;
+	struct lima_vm *current_vm;
+
+	struct lima_ip *mmu[LIMA_SCHED_PIPE_MAX_MMU];
+	int num_mmu;
+
+	struct lima_ip *l2_cache[LIMA_SCHED_PIPE_MAX_L2_CACHE];
+	int num_l2_cache;
+
+	struct lima_ip *processor[LIMA_SCHED_PIPE_MAX_PROCESSOR];
+	int num_processor;
+
+	struct lima_ip *bcast_processor;
+	struct lima_ip *bcast_mmu;
+
+	u32 done;
+	bool error;
+	atomic_t task;
+
+	int frame_size;
+	struct kmem_cache *task_slab;
+
+	int (*task_validate)(struct lima_sched_pipe *pipe, struct lima_sched_task *task);
+	void (*task_run)(struct lima_sched_pipe *pipe, struct lima_sched_task *task);
+	void (*task_fini)(struct lima_sched_pipe *pipe);
+	void (*task_error)(struct lima_sched_pipe *pipe);
+	void (*task_mmu_error)(struct lima_sched_pipe *pipe);
+
+	struct work_struct error_work;
+};
+
+int lima_sched_task_init(struct lima_sched_task *task,
+			 struct lima_sched_context *context,
+			 struct lima_bo **bos, int num_bos,
+			 struct lima_vm *vm);
+void lima_sched_task_fini(struct lima_sched_task *task);
+
+int lima_sched_context_init(struct lima_sched_pipe *pipe,
+			    struct lima_sched_context *context,
+			    atomic_t *guilty);
+void lima_sched_context_fini(struct lima_sched_pipe *pipe,
+			     struct lima_sched_context *context);
+struct dma_fence *lima_sched_context_queue_task(struct lima_sched_context *context,
+						struct lima_sched_task *task);
+
+int lima_sched_pipe_init(struct lima_sched_pipe *pipe, const char *name);
+void lima_sched_pipe_fini(struct lima_sched_pipe *pipe);
+void lima_sched_pipe_task_done(struct lima_sched_pipe *pipe);
+
+static inline void lima_sched_pipe_mmu_error(struct lima_sched_pipe *pipe)
+{
+	pipe->error = true;
+	pipe->task_mmu_error(pipe);
+}
+
+int lima_sched_slab_init(void);
+void lima_sched_slab_fini(void);
+
+#endif
diff --git a/drivers/gpu/drm/lima/lima_vm.c b/drivers/gpu/drm/lima/lima_vm.c
new file mode 100644
index 000000000000..19e88ca16527
--- /dev/null
+++ b/drivers/gpu/drm/lima/lima_vm.c
@@ -0,0 +1,282 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
+
+#include <linux/slab.h>
+#include <linux/dma-mapping.h>
+
+#include "lima_device.h"
+#include "lima_vm.h"
+#include "lima_object.h"
+#include "lima_regs.h"
+
+struct lima_bo_va {
+	struct list_head list;
+	unsigned int ref_count;
+
+	struct drm_mm_node node;
+
+	struct lima_vm *vm;
+};
+
+#define LIMA_VM_PD_SHIFT 22
+#define LIMA_VM_PT_SHIFT 12
+#define LIMA_VM_PB_SHIFT (LIMA_VM_PD_SHIFT + LIMA_VM_NUM_PT_PER_BT_SHIFT)
+#define LIMA_VM_BT_SHIFT LIMA_VM_PT_SHIFT
+
+#define LIMA_VM_PT_MASK ((1 << LIMA_VM_PD_SHIFT) - 1)
+#define LIMA_VM_BT_MASK ((1 << LIMA_VM_PB_SHIFT) - 1)
+
+#define LIMA_PDE(va) (va >> LIMA_VM_PD_SHIFT)
+#define LIMA_PTE(va) ((va & LIMA_VM_PT_MASK) >> LIMA_VM_PT_SHIFT)
+#define LIMA_PBE(va) (va >> LIMA_VM_PB_SHIFT)
+#define LIMA_BTE(va) ((va & LIMA_VM_BT_MASK) >> LIMA_VM_BT_SHIFT)
+
+
+static void lima_vm_unmap_page_table(struct lima_vm *vm, u32 start, u32 end)
+{
+	u32 addr;
+
+	for (addr = start; addr <= end; addr += LIMA_PAGE_SIZE) {
+		u32 pbe = LIMA_PBE(addr);
+		u32 bte = LIMA_BTE(addr);
+
+		vm->bts[pbe].cpu[bte] = 0;
+	}
+}
+
+static int lima_vm_map_page_table(struct lima_vm *vm, dma_addr_t *dma,
+				  u32 start, u32 end)
+{
+	u64 addr;
+	int i = 0;
+
+	for (addr = start; addr <= end; addr += LIMA_PAGE_SIZE) {
+		u32 pbe = LIMA_PBE(addr);
+		u32 bte = LIMA_BTE(addr);
+
+		if (!vm->bts[pbe].cpu) {
+			dma_addr_t pts;
+			u32 *pd;
+			int j;
+
+			vm->bts[pbe].cpu = dma_alloc_wc(
+				vm->dev->dev, LIMA_PAGE_SIZE << LIMA_VM_NUM_PT_PER_BT_SHIFT,
+				&vm->bts[pbe].dma, GFP_KERNEL | __GFP_ZERO);
+			if (!vm->bts[pbe].cpu) {
+				if (addr != start)
+					lima_vm_unmap_page_table(vm, start, addr - 1);
+				return -ENOMEM;
+			}
+
+			pts = vm->bts[pbe].dma;
+			pd = vm->pd.cpu + (pbe << LIMA_VM_NUM_PT_PER_BT_SHIFT);
+			for (j = 0; j < LIMA_VM_NUM_PT_PER_BT; j++) {
+				pd[j] = pts | LIMA_VM_FLAG_PRESENT;
+				pts += LIMA_PAGE_SIZE;
+			}
+		}
+
+		vm->bts[pbe].cpu[bte] = dma[i++] | LIMA_VM_FLAGS_CACHE;
+	}
+
+	return 0;
+}
+
+static struct lima_bo_va *
+lima_vm_bo_find(struct lima_vm *vm, struct lima_bo *bo)
+{
+	struct lima_bo_va *bo_va, *ret = NULL;
+
+	list_for_each_entry(bo_va, &bo->va, list) {
+		if (bo_va->vm == vm) {
+			ret = bo_va;
+			break;
+		}
+	}
+
+	return ret;
+}
+
+int lima_vm_bo_add(struct lima_vm *vm, struct lima_bo *bo, bool create)
+{
+	struct lima_bo_va *bo_va;
+	int err;
+
+	mutex_lock(&bo->lock);
+
+	bo_va = lima_vm_bo_find(vm, bo);
+	if (bo_va) {
+		bo_va->ref_count++;
+		mutex_unlock(&bo->lock);
+		return 0;
+	}
+
+	/* should not create new bo_va if not asked by caller */
+	if (!create) {
+		mutex_unlock(&bo->lock);
+		return -ENOENT;
+	}
+
+	bo_va = kzalloc(sizeof(*bo_va), GFP_KERNEL);
+	if (!bo_va) {
+		err = -ENOMEM;
+		goto err_out0;
+	}
+
+	bo_va->vm = vm;
+	bo_va->ref_count = 1;
+
+	mutex_lock(&vm->lock);
+
+	err = drm_mm_insert_node(&vm->mm, &bo_va->node, bo->gem.size);
+	if (err)
+		goto err_out1;
+
+	err = lima_vm_map_page_table(vm, bo->pages_dma_addr, bo_va->node.start,
+				     bo_va->node.start + bo_va->node.size - 1);
+	if (err)
+		goto err_out2;
+
+	mutex_unlock(&vm->lock);
+
+	list_add_tail(&bo_va->list, &bo->va);
+
+	mutex_unlock(&bo->lock);
+	return 0;
+
+err_out2:
+	drm_mm_remove_node(&bo_va->node);
+err_out1:
+	mutex_unlock(&vm->lock);
+	kfree(bo_va);
+err_out0:
+	mutex_unlock(&bo->lock);
+	return err;
+}
+
+void lima_vm_bo_del(struct lima_vm *vm, struct lima_bo *bo)
+{
+	struct lima_bo_va *bo_va;
+
+	mutex_lock(&bo->lock);
+
+	bo_va = lima_vm_bo_find(vm, bo);
+	if (--bo_va->ref_count > 0) {
+		mutex_unlock(&bo->lock);
+		return;
+	}
+
+	mutex_lock(&vm->lock);
+
+	lima_vm_unmap_page_table(vm, bo_va->node.start,
+				 bo_va->node.start + bo_va->node.size - 1);
+
+	drm_mm_remove_node(&bo_va->node);
+
+	mutex_unlock(&vm->lock);
+
+	list_del(&bo_va->list);
+
+	mutex_unlock(&bo->lock);
+
+	kfree(bo_va);
+}
+
+u32 lima_vm_get_va(struct lima_vm *vm, struct lima_bo *bo)
+{
+	struct lima_bo_va *bo_va;
+	u32 ret;
+
+	mutex_lock(&bo->lock);
+
+	bo_va = lima_vm_bo_find(vm, bo);
+	ret = bo_va->node.start;
+
+	mutex_unlock(&bo->lock);
+
+	return ret;
+}
+
+struct lima_vm *lima_vm_create(struct lima_device *dev)
+{
+	struct lima_vm *vm;
+
+	vm = kzalloc(sizeof(*vm), GFP_KERNEL);
+	if (!vm)
+		return NULL;
+
+	vm->dev = dev;
+	mutex_init(&vm->lock);
+	kref_init(&vm->refcount);
+
+	vm->pd.cpu = dma_alloc_wc(dev->dev, LIMA_PAGE_SIZE, &vm->pd.dma,
+				  GFP_KERNEL | __GFP_ZERO);
+	if (!vm->pd.cpu)
+		goto err_out0;
+
+	if (dev->dlbu_cpu) {
+		int err = lima_vm_map_page_table(
+			vm, &dev->dlbu_dma, LIMA_VA_RESERVE_DLBU,
+			LIMA_VA_RESERVE_DLBU + LIMA_PAGE_SIZE - 1);
+		if (err)
+			goto err_out1;
+	}
+
+	drm_mm_init(&vm->mm, dev->va_start, dev->va_end - dev->va_start);
+
+	return vm;
+
+err_out1:
+	dma_free_wc(dev->dev, LIMA_PAGE_SIZE, vm->pd.cpu, vm->pd.dma);
+err_out0:
+	kfree(vm);
+	return NULL;
+}
+
+void lima_vm_release(struct kref *kref)
+{
+	struct lima_vm *vm = container_of(kref, struct lima_vm, refcount);
+	int i;
+
+	drm_mm_takedown(&vm->mm);
+
+	for (i = 0; i < LIMA_VM_NUM_BT; i++) {
+		if (vm->bts[i].cpu)
+			dma_free_wc(vm->dev->dev, LIMA_PAGE_SIZE << LIMA_VM_NUM_PT_PER_BT_SHIFT,
+				    vm->bts[i].cpu, vm->bts[i].dma);
+	}
+
+	if (vm->pd.cpu)
+		dma_free_wc(vm->dev->dev, LIMA_PAGE_SIZE, vm->pd.cpu, vm->pd.dma);
+
+	kfree(vm);
+}
+
+void lima_vm_print(struct lima_vm *vm)
+{
+	int i, j, k;
+	u32 *pd, *pt;
+
+	if (!vm->pd.cpu)
+		return;
+
+	pd = vm->pd.cpu;
+	for (i = 0; i < LIMA_VM_NUM_BT; i++) {
+		if (!vm->bts[i].cpu)
+			continue;
+
+		pt = vm->bts[i].cpu;
+		for (j = 0; j < LIMA_VM_NUM_PT_PER_BT; j++) {
+			int idx = (i << LIMA_VM_NUM_PT_PER_BT_SHIFT) + j;
+
+			printk(KERN_INFO "lima vm pd %03x:%08x\n", idx, pd[idx]);
+
+			for (k = 0; k < LIMA_PAGE_ENT_NUM; k++) {
+				u32 pte = *pt++;
+
+				if (pte)
+					printk(KERN_INFO "  pt %03x:%08x\n", k, pte);
+			}
+		}
+	}
+}
diff --git a/drivers/gpu/drm/lima/lima_vm.h b/drivers/gpu/drm/lima/lima_vm.h
new file mode 100644
index 000000000000..caee2f8a29b4
--- /dev/null
+++ b/drivers/gpu/drm/lima/lima_vm.h
@@ -0,0 +1,62 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+/* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
+
+#ifndef __LIMA_VM_H__
+#define __LIMA_VM_H__
+
+#include <drm/drm_mm.h>
+#include <linux/kref.h>
+
+#define LIMA_PAGE_SIZE    4096
+#define LIMA_PAGE_MASK    (LIMA_PAGE_SIZE - 1)
+#define LIMA_PAGE_ENT_NUM (LIMA_PAGE_SIZE / sizeof(u32))
+
+#define LIMA_VM_NUM_PT_PER_BT_SHIFT 3
+#define LIMA_VM_NUM_PT_PER_BT (1 << LIMA_VM_NUM_PT_PER_BT_SHIFT)
+#define LIMA_VM_NUM_BT (LIMA_PAGE_ENT_NUM >> LIMA_VM_NUM_PT_PER_BT_SHIFT)
+
+#define LIMA_VA_RESERVE_START  0xFFF00000
+#define LIMA_VA_RESERVE_DLBU   LIMA_VA_RESERVE_START
+#define LIMA_VA_RESERVE_END    0x100000000
+
+struct lima_device;
+
+struct lima_vm_page {
+	u32 *cpu;
+	dma_addr_t dma;
+};
+
+struct lima_vm {
+	struct mutex lock;
+	struct kref refcount;
+
+	struct drm_mm mm;
+
+	struct lima_device *dev;
+
+	struct lima_vm_page pd;
+	struct lima_vm_page bts[LIMA_VM_NUM_BT];
+};
+
+int lima_vm_bo_add(struct lima_vm *vm, struct lima_bo *bo, bool create);
+void lima_vm_bo_del(struct lima_vm *vm, struct lima_bo *bo);
+
+u32 lima_vm_get_va(struct lima_vm *vm, struct lima_bo *bo);
+
+struct lima_vm *lima_vm_create(struct lima_device *dev);
+void lima_vm_release(struct kref *kref);
+
+static inline struct lima_vm *lima_vm_get(struct lima_vm *vm)
+{
+	kref_get(&vm->refcount);
+	return vm;
+}
+
+static inline void lima_vm_put(struct lima_vm *vm)
+{
+	kref_put(&vm->refcount, lima_vm_release);
+}
+
+void lima_vm_print(struct lima_vm *vm);
+
+#endif
diff --git a/drivers/gpu/drm/meson/Makefile b/drivers/gpu/drm/meson/Makefile
index 7709f2fbb9f7..d4ea82fc493b 100644
--- a/drivers/gpu/drm/meson/Makefile
+++ b/drivers/gpu/drm/meson/Makefile
@@ -1,5 +1,5 @@
 meson-drm-y := meson_drv.o meson_plane.o meson_crtc.o meson_venc_cvbs.o
-meson-drm-y += meson_viu.o meson_vpp.o meson_venc.o meson_vclk.o meson_canvas.o meson_overlay.o
+meson-drm-y += meson_viu.o meson_vpp.o meson_venc.o meson_vclk.o meson_overlay.o
 
 obj-$(CONFIG_DRM_MESON) += meson-drm.o
 obj-$(CONFIG_DRM_MESON_DW_HDMI) += meson_dw_hdmi.o
diff --git a/drivers/gpu/drm/meson/meson_canvas.c b/drivers/gpu/drm/meson/meson_canvas.c
deleted file mode 100644
index 5de11aa7c775..000000000000
--- a/drivers/gpu/drm/meson/meson_canvas.c
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Copyright (C) 2016 BayLibre, SAS
- * Author: Neil Armstrong <narmstrong@baylibre.com>
- * Copyright (C) 2015 Amlogic, Inc. All rights reserved.
- * Copyright (C) 2014 Endless Mobile
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, see <http://www.gnu.org/licenses/>.
- */
-
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include "meson_drv.h"
-#include "meson_canvas.h"
-#include "meson_registers.h"
-
-/**
- * DOC: Canvas
- *
- * CANVAS is a memory zone where physical memory frames information
- * are stored for the VIU to scanout.
- */
-
-/* DMC Registers */
-#define DMC_CAV_LUT_DATAL	0x48 /* 0x12 offset in data sheet */
-#define CANVAS_WIDTH_LBIT	29
-#define CANVAS_WIDTH_LWID       3
-#define DMC_CAV_LUT_DATAH	0x4c /* 0x13 offset in data sheet */
-#define CANVAS_WIDTH_HBIT       0
-#define CANVAS_HEIGHT_BIT       9
-#define CANVAS_BLKMODE_BIT      24
-#define CANVAS_ENDIAN_BIT	26
-#define DMC_CAV_LUT_ADDR	0x50 /* 0x14 offset in data sheet */
-#define CANVAS_LUT_WR_EN        (0x2 << 8)
-#define CANVAS_LUT_RD_EN        (0x1 << 8)
-
-void meson_canvas_setup(struct meson_drm *priv,
-			uint32_t canvas_index, uint32_t addr,
-			uint32_t stride, uint32_t height,
-			unsigned int wrap,
-			unsigned int blkmode,
-			unsigned int endian)
-{
-	unsigned int val;
-
-	regmap_write(priv->dmc, DMC_CAV_LUT_DATAL,
-		(((addr + 7) >> 3)) |
-		(((stride + 7) >> 3) << CANVAS_WIDTH_LBIT));
-
-	regmap_write(priv->dmc, DMC_CAV_LUT_DATAH,
-		((((stride + 7) >> 3) >> CANVAS_WIDTH_LWID) <<
-						CANVAS_WIDTH_HBIT) |
-		(height << CANVAS_HEIGHT_BIT) |
-		(wrap << 22) |
-		(blkmode << CANVAS_BLKMODE_BIT) |
-		(endian << CANVAS_ENDIAN_BIT));
-
-	regmap_write(priv->dmc, DMC_CAV_LUT_ADDR,
-			CANVAS_LUT_WR_EN | canvas_index);
-
-	/* Force a read-back to make sure everything is flushed. */
-	regmap_read(priv->dmc, DMC_CAV_LUT_DATAH, &val);
-}
diff --git a/drivers/gpu/drm/meson/meson_canvas.h b/drivers/gpu/drm/meson/meson_canvas.h
deleted file mode 100644
index 85dbf26e2826..000000000000
--- a/drivers/gpu/drm/meson/meson_canvas.h
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Copyright (C) 2016 BayLibre, SAS
- * Author: Neil Armstrong <narmstrong@baylibre.com>
- * Copyright (C) 2014 Endless Mobile
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, see <http://www.gnu.org/licenses/>.
- */
-
-/* Canvas LUT Memory */
-
-#ifndef __MESON_CANVAS_H
-#define __MESON_CANVAS_H
-
-#define MESON_CANVAS_ID_OSD1	0x4e
-#define MESON_CANVAS_ID_VD1_0	0x60
-#define MESON_CANVAS_ID_VD1_1	0x61
-#define MESON_CANVAS_ID_VD1_2	0x62
-
-/* Canvas configuration. */
-#define MESON_CANVAS_WRAP_NONE	0x00
-#define	MESON_CANVAS_WRAP_X	0x01
-#define	MESON_CANVAS_WRAP_Y	0x02
-
-#define	MESON_CANVAS_BLKMODE_LINEAR	0x00
-#define	MESON_CANVAS_BLKMODE_32x32	0x01
-#define	MESON_CANVAS_BLKMODE_64x64	0x02
-
-#define MESON_CANVAS_ENDIAN_SWAP16	0x1
-#define MESON_CANVAS_ENDIAN_SWAP32	0x3
-#define MESON_CANVAS_ENDIAN_SWAP64	0x7
-#define MESON_CANVAS_ENDIAN_SWAP128	0xf
-
-void meson_canvas_setup(struct meson_drm *priv,
-			uint32_t canvas_index, uint32_t addr,
-			uint32_t stride, uint32_t height,
-			unsigned int wrap,
-			unsigned int blkmode,
-			unsigned int endian);
-
-#endif /* __MESON_CANVAS_H */
diff --git a/drivers/gpu/drm/meson/meson_crtc.c b/drivers/gpu/drm/meson/meson_crtc.c
index 43e29984f8b1..5579f8ac3e3f 100644
--- a/drivers/gpu/drm/meson/meson_crtc.c
+++ b/drivers/gpu/drm/meson/meson_crtc.c
@@ -37,15 +37,19 @@
 #include "meson_venc.h"
 #include "meson_vpp.h"
 #include "meson_viu.h"
-#include "meson_canvas.h"
 #include "meson_registers.h"
 
+#define MESON_G12A_VIU_OFFSET	0x5ec0
+
 /* CRTC definition */
 
 struct meson_crtc {
 	struct drm_crtc base;
 	struct drm_pending_vblank_event *event;
 	struct meson_drm *priv;
+	void (*enable_osd1)(struct meson_drm *priv);
+	void (*enable_vd1)(struct meson_drm *priv);
+	unsigned int viu_offset;
 };
 #define to_meson_crtc(x) container_of(x, struct meson_crtc, base)
 
@@ -81,6 +85,44 @@ static const struct drm_crtc_funcs meson_crtc_funcs = {
 
 };
 
+static void meson_g12a_crtc_atomic_enable(struct drm_crtc *crtc,
+					  struct drm_crtc_state *old_state)
+{
+	struct meson_crtc *meson_crtc = to_meson_crtc(crtc);
+	struct drm_crtc_state *crtc_state = crtc->state;
+	struct meson_drm *priv = meson_crtc->priv;
+
+	DRM_DEBUG_DRIVER("\n");
+
+	if (!crtc_state) {
+		DRM_ERROR("Invalid crtc_state\n");
+		return;
+	}
+
+	/* VD1 Preblend vertical start/end */
+	writel(FIELD_PREP(GENMASK(11, 0), 2303),
+	       priv->io_base + _REG(VPP_PREBLEND_VD1_V_START_END));
+
+	/* Setup Blender */
+	writel(crtc_state->mode.hdisplay |
+	       crtc_state->mode.vdisplay << 16,
+	       priv->io_base + _REG(VPP_POSTBLEND_H_SIZE));
+
+	writel_relaxed(0 << 16 |
+			(crtc_state->mode.hdisplay - 1),
+			priv->io_base + _REG(VPP_OSD1_BLD_H_SCOPE));
+	writel_relaxed(0 << 16 |
+			(crtc_state->mode.vdisplay - 1),
+			priv->io_base + _REG(VPP_OSD1_BLD_V_SCOPE));
+	writel_relaxed(crtc_state->mode.hdisplay << 16 |
+			crtc_state->mode.vdisplay,
+			priv->io_base + _REG(VPP_OUT_H_V_SIZE));
+
+	drm_crtc_vblank_on(crtc);
+
+	priv->viu.osd1_enabled = true;
+}
+
 static void meson_crtc_atomic_enable(struct drm_crtc *crtc,
 				     struct drm_crtc_state *old_state)
 {
@@ -111,6 +153,31 @@ static void meson_crtc_atomic_enable(struct drm_crtc *crtc,
 	priv->viu.osd1_enabled = true;
 }
 
+static void meson_g12a_crtc_atomic_disable(struct drm_crtc *crtc,
+					   struct drm_crtc_state *old_state)
+{
+	struct meson_crtc *meson_crtc = to_meson_crtc(crtc);
+	struct meson_drm *priv = meson_crtc->priv;
+
+	DRM_DEBUG_DRIVER("\n");
+
+	drm_crtc_vblank_off(crtc);
+
+	priv->viu.osd1_enabled = false;
+	priv->viu.osd1_commit = false;
+
+	priv->viu.vd1_enabled = false;
+	priv->viu.vd1_commit = false;
+
+	if (crtc->state->event && !crtc->state->active) {
+		spin_lock_irq(&crtc->dev->event_lock);
+		drm_crtc_send_vblank_event(crtc, crtc->state->event);
+		spin_unlock_irq(&crtc->dev->event_lock);
+
+		crtc->state->event = NULL;
+	}
+}
+
 static void meson_crtc_atomic_disable(struct drm_crtc *crtc,
 				      struct drm_crtc_state *old_state)
 {
@@ -174,6 +241,53 @@ static const struct drm_crtc_helper_funcs meson_crtc_helper_funcs = {
 	.atomic_disable	= meson_crtc_atomic_disable,
 };
 
+static const struct drm_crtc_helper_funcs meson_g12a_crtc_helper_funcs = {
+	.atomic_begin	= meson_crtc_atomic_begin,
+	.atomic_flush	= meson_crtc_atomic_flush,
+	.atomic_enable	= meson_g12a_crtc_atomic_enable,
+	.atomic_disable	= meson_g12a_crtc_atomic_disable,
+};
+
+static void meson_crtc_enable_osd1(struct meson_drm *priv)
+{
+	writel_bits_relaxed(VPP_OSD1_POSTBLEND, VPP_OSD1_POSTBLEND,
+			    priv->io_base + _REG(VPP_MISC));
+}
+
+static void meson_g12a_crtc_enable_osd1(struct meson_drm *priv)
+{
+	writel_relaxed(priv->viu.osd_blend_din0_scope_h,
+		       priv->io_base +
+		       _REG(VIU_OSD_BLEND_DIN0_SCOPE_H));
+	writel_relaxed(priv->viu.osd_blend_din0_scope_v,
+		       priv->io_base +
+		       _REG(VIU_OSD_BLEND_DIN0_SCOPE_V));
+	writel_relaxed(priv->viu.osb_blend0_size,
+		       priv->io_base +
+		       _REG(VIU_OSD_BLEND_BLEND0_SIZE));
+	writel_relaxed(priv->viu.osb_blend1_size,
+		       priv->io_base +
+		       _REG(VIU_OSD_BLEND_BLEND1_SIZE));
+}
+
+static void meson_crtc_enable_vd1(struct meson_drm *priv)
+{
+	writel_bits_relaxed(VPP_VD1_PREBLEND | VPP_VD1_POSTBLEND |
+			    VPP_COLOR_MNG_ENABLE,
+			    VPP_VD1_PREBLEND | VPP_VD1_POSTBLEND |
+			    VPP_COLOR_MNG_ENABLE,
+			    priv->io_base + _REG(VPP_MISC));
+}
+
+static void meson_g12a_crtc_enable_vd1(struct meson_drm *priv)
+{
+	writel_relaxed(((1 << 16) | /* post bld premult*/
+			(1 << 8) | /* post src */
+			(1 << 4) | /* pre bld premult*/
+			(1 << 0)),
+			priv->io_base + _REG(VD1_BLEND_SRC_CTRL));
+}
+
 void meson_crtc_irq(struct meson_drm *priv)
 {
 	struct meson_crtc *meson_crtc = to_meson_crtc(priv->crtc);
@@ -214,20 +328,14 @@ void meson_crtc_irq(struct meson_drm *priv)
 		writel_relaxed(priv->viu.osd_sc_v_ctrl0,
 				priv->io_base + _REG(VPP_OSD_VSC_CTRL0));
 
-		if (priv->canvas)
-			meson_canvas_config(priv->canvas, priv->canvas_id_osd1,
-				priv->viu.osd1_addr, priv->viu.osd1_stride,
-				priv->viu.osd1_height, MESON_CANVAS_WRAP_NONE,
-				MESON_CANVAS_BLKMODE_LINEAR, 0);
-		else
-			meson_canvas_setup(priv, MESON_CANVAS_ID_OSD1,
+		meson_canvas_config(priv->canvas, priv->canvas_id_osd1,
 				priv->viu.osd1_addr, priv->viu.osd1_stride,
 				priv->viu.osd1_height, MESON_CANVAS_WRAP_NONE,
 				MESON_CANVAS_BLKMODE_LINEAR, 0);
 
 		/* Enable OSD1 */
-		writel_bits_relaxed(VPP_OSD1_POSTBLEND, VPP_OSD1_POSTBLEND,
-				    priv->io_base + _REG(VPP_MISC));
+		if (meson_crtc->enable_osd1)
+			meson_crtc->enable_osd1(priv);
 
 		priv->viu.osd1_commit = false;
 	}
@@ -237,147 +345,164 @@ void meson_crtc_irq(struct meson_drm *priv)
 
 		switch (priv->viu.vd1_planes) {
 		case 3:
-			if (priv->canvas)
-				meson_canvas_config(priv->canvas,
-						    priv->canvas_id_vd1_2,
-						    priv->viu.vd1_addr2,
-						    priv->viu.vd1_stride2,
-						    priv->viu.vd1_height2,
-						    MESON_CANVAS_WRAP_NONE,
-						    MESON_CANVAS_BLKMODE_LINEAR,
-						    MESON_CANVAS_ENDIAN_SWAP64);
-			else
-				meson_canvas_setup(priv, MESON_CANVAS_ID_VD1_2,
-						   priv->viu.vd1_addr2,
-						   priv->viu.vd1_stride2,
-						   priv->viu.vd1_height2,
-						   MESON_CANVAS_WRAP_NONE,
-						   MESON_CANVAS_BLKMODE_LINEAR,
-						   MESON_CANVAS_ENDIAN_SWAP64);
+			meson_canvas_config(priv->canvas,
+					    priv->canvas_id_vd1_2,
+					    priv->viu.vd1_addr2,
+					    priv->viu.vd1_stride2,
+					    priv->viu.vd1_height2,
+					    MESON_CANVAS_WRAP_NONE,
+					    MESON_CANVAS_BLKMODE_LINEAR,
+					    MESON_CANVAS_ENDIAN_SWAP64);
 		/* fallthrough */
 		case 2:
-			if (priv->canvas)
-				meson_canvas_config(priv->canvas,
-						    priv->canvas_id_vd1_1,
-						    priv->viu.vd1_addr1,
-						    priv->viu.vd1_stride1,
-						    priv->viu.vd1_height1,
-						    MESON_CANVAS_WRAP_NONE,
-						    MESON_CANVAS_BLKMODE_LINEAR,
-						    MESON_CANVAS_ENDIAN_SWAP64);
-			else
-				meson_canvas_setup(priv, MESON_CANVAS_ID_VD1_1,
-						   priv->viu.vd1_addr2,
-						   priv->viu.vd1_stride2,
-						   priv->viu.vd1_height2,
-						   MESON_CANVAS_WRAP_NONE,
-						   MESON_CANVAS_BLKMODE_LINEAR,
-						   MESON_CANVAS_ENDIAN_SWAP64);
+			meson_canvas_config(priv->canvas,
+					    priv->canvas_id_vd1_1,
+					    priv->viu.vd1_addr1,
+					    priv->viu.vd1_stride1,
+					    priv->viu.vd1_height1,
+					    MESON_CANVAS_WRAP_NONE,
+					    MESON_CANVAS_BLKMODE_LINEAR,
+					    MESON_CANVAS_ENDIAN_SWAP64);
 		/* fallthrough */
 		case 1:
-			if (priv->canvas)
-				meson_canvas_config(priv->canvas,
-						    priv->canvas_id_vd1_0,
-						    priv->viu.vd1_addr0,
-						    priv->viu.vd1_stride0,
-						    priv->viu.vd1_height0,
-						    MESON_CANVAS_WRAP_NONE,
-						    MESON_CANVAS_BLKMODE_LINEAR,
-						    MESON_CANVAS_ENDIAN_SWAP64);
-			else
-				meson_canvas_setup(priv, MESON_CANVAS_ID_VD1_0,
-						   priv->viu.vd1_addr2,
-						   priv->viu.vd1_stride2,
-						   priv->viu.vd1_height2,
-						   MESON_CANVAS_WRAP_NONE,
-						   MESON_CANVAS_BLKMODE_LINEAR,
-						   MESON_CANVAS_ENDIAN_SWAP64);
+			meson_canvas_config(priv->canvas,
+					    priv->canvas_id_vd1_0,
+					    priv->viu.vd1_addr0,
+					    priv->viu.vd1_stride0,
+					    priv->viu.vd1_height0,
+					    MESON_CANVAS_WRAP_NONE,
+					    MESON_CANVAS_BLKMODE_LINEAR,
+					    MESON_CANVAS_ENDIAN_SWAP64);
 		};
 
 		writel_relaxed(priv->viu.vd1_if0_gen_reg,
-				priv->io_base + _REG(VD1_IF0_GEN_REG));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VD1_IF0_GEN_REG));
 		writel_relaxed(priv->viu.vd1_if0_gen_reg,
-				priv->io_base + _REG(VD2_IF0_GEN_REG));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VD2_IF0_GEN_REG));
 		writel_relaxed(priv->viu.vd1_if0_gen_reg2,
-				priv->io_base + _REG(VD1_IF0_GEN_REG2));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VD1_IF0_GEN_REG2));
 		writel_relaxed(priv->viu.viu_vd1_fmt_ctrl,
-				priv->io_base + _REG(VIU_VD1_FMT_CTRL));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VIU_VD1_FMT_CTRL));
 		writel_relaxed(priv->viu.viu_vd1_fmt_ctrl,
-				priv->io_base + _REG(VIU_VD2_FMT_CTRL));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VIU_VD2_FMT_CTRL));
 		writel_relaxed(priv->viu.viu_vd1_fmt_w,
-				priv->io_base + _REG(VIU_VD1_FMT_W));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VIU_VD1_FMT_W));
 		writel_relaxed(priv->viu.viu_vd1_fmt_w,
-				priv->io_base + _REG(VIU_VD2_FMT_W));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VIU_VD2_FMT_W));
 		writel_relaxed(priv->viu.vd1_if0_canvas0,
-				priv->io_base + _REG(VD1_IF0_CANVAS0));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VD1_IF0_CANVAS0));
 		writel_relaxed(priv->viu.vd1_if0_canvas0,
-				priv->io_base + _REG(VD1_IF0_CANVAS1));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VD1_IF0_CANVAS1));
 		writel_relaxed(priv->viu.vd1_if0_canvas0,
-				priv->io_base + _REG(VD2_IF0_CANVAS0));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VD2_IF0_CANVAS0));
 		writel_relaxed(priv->viu.vd1_if0_canvas0,
-				priv->io_base + _REG(VD2_IF0_CANVAS1));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VD2_IF0_CANVAS1));
 		writel_relaxed(priv->viu.vd1_if0_luma_x0,
-				priv->io_base + _REG(VD1_IF0_LUMA_X0));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VD1_IF0_LUMA_X0));
 		writel_relaxed(priv->viu.vd1_if0_luma_x0,
-				priv->io_base + _REG(VD1_IF0_LUMA_X1));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VD1_IF0_LUMA_X1));
 		writel_relaxed(priv->viu.vd1_if0_luma_x0,
-				priv->io_base + _REG(VD2_IF0_LUMA_X0));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VD2_IF0_LUMA_X0));
 		writel_relaxed(priv->viu.vd1_if0_luma_x0,
-				priv->io_base + _REG(VD2_IF0_LUMA_X1));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VD2_IF0_LUMA_X1));
 		writel_relaxed(priv->viu.vd1_if0_luma_y0,
-				priv->io_base + _REG(VD1_IF0_LUMA_Y0));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VD1_IF0_LUMA_Y0));
 		writel_relaxed(priv->viu.vd1_if0_luma_y0,
-				priv->io_base + _REG(VD1_IF0_LUMA_Y1));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VD1_IF0_LUMA_Y1));
 		writel_relaxed(priv->viu.vd1_if0_luma_y0,
-				priv->io_base + _REG(VD2_IF0_LUMA_Y0));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VD2_IF0_LUMA_Y0));
 		writel_relaxed(priv->viu.vd1_if0_luma_y0,
-				priv->io_base + _REG(VD2_IF0_LUMA_Y1));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VD2_IF0_LUMA_Y1));
 		writel_relaxed(priv->viu.vd1_if0_chroma_x0,
-				priv->io_base + _REG(VD1_IF0_CHROMA_X0));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VD1_IF0_CHROMA_X0));
 		writel_relaxed(priv->viu.vd1_if0_chroma_x0,
-				priv->io_base + _REG(VD1_IF0_CHROMA_X1));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VD1_IF0_CHROMA_X1));
 		writel_relaxed(priv->viu.vd1_if0_chroma_x0,
-				priv->io_base + _REG(VD2_IF0_CHROMA_X0));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VD2_IF0_CHROMA_X0));
 		writel_relaxed(priv->viu.vd1_if0_chroma_x0,
-				priv->io_base + _REG(VD2_IF0_CHROMA_X1));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VD2_IF0_CHROMA_X1));
 		writel_relaxed(priv->viu.vd1_if0_chroma_y0,
-				priv->io_base + _REG(VD1_IF0_CHROMA_Y0));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VD1_IF0_CHROMA_Y0));
 		writel_relaxed(priv->viu.vd1_if0_chroma_y0,
-				priv->io_base + _REG(VD1_IF0_CHROMA_Y1));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VD1_IF0_CHROMA_Y1));
 		writel_relaxed(priv->viu.vd1_if0_chroma_y0,
-				priv->io_base + _REG(VD2_IF0_CHROMA_Y0));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VD2_IF0_CHROMA_Y0));
 		writel_relaxed(priv->viu.vd1_if0_chroma_y0,
-				priv->io_base + _REG(VD2_IF0_CHROMA_Y1));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VD2_IF0_CHROMA_Y1));
 		writel_relaxed(priv->viu.vd1_if0_repeat_loop,
-				priv->io_base + _REG(VD1_IF0_RPT_LOOP));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VD1_IF0_RPT_LOOP));
 		writel_relaxed(priv->viu.vd1_if0_repeat_loop,
-				priv->io_base + _REG(VD2_IF0_RPT_LOOP));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VD2_IF0_RPT_LOOP));
 		writel_relaxed(priv->viu.vd1_if0_luma0_rpt_pat,
-				priv->io_base + _REG(VD1_IF0_LUMA0_RPT_PAT));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VD1_IF0_LUMA0_RPT_PAT));
 		writel_relaxed(priv->viu.vd1_if0_luma0_rpt_pat,
-				priv->io_base + _REG(VD2_IF0_LUMA0_RPT_PAT));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VD2_IF0_LUMA0_RPT_PAT));
 		writel_relaxed(priv->viu.vd1_if0_luma0_rpt_pat,
-				priv->io_base + _REG(VD1_IF0_LUMA1_RPT_PAT));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VD1_IF0_LUMA1_RPT_PAT));
 		writel_relaxed(priv->viu.vd1_if0_luma0_rpt_pat,
-				priv->io_base + _REG(VD2_IF0_LUMA1_RPT_PAT));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VD2_IF0_LUMA1_RPT_PAT));
 		writel_relaxed(priv->viu.vd1_if0_chroma0_rpt_pat,
-				priv->io_base + _REG(VD1_IF0_CHROMA0_RPT_PAT));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VD1_IF0_CHROMA0_RPT_PAT));
 		writel_relaxed(priv->viu.vd1_if0_chroma0_rpt_pat,
-				priv->io_base + _REG(VD2_IF0_CHROMA0_RPT_PAT));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VD2_IF0_CHROMA0_RPT_PAT));
 		writel_relaxed(priv->viu.vd1_if0_chroma0_rpt_pat,
-				priv->io_base + _REG(VD1_IF0_CHROMA1_RPT_PAT));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VD1_IF0_CHROMA1_RPT_PAT));
 		writel_relaxed(priv->viu.vd1_if0_chroma0_rpt_pat,
-				priv->io_base + _REG(VD2_IF0_CHROMA1_RPT_PAT));
-		writel_relaxed(0, priv->io_base + _REG(VD1_IF0_LUMA_PSEL));
-		writel_relaxed(0, priv->io_base + _REG(VD1_IF0_CHROMA_PSEL));
-		writel_relaxed(0, priv->io_base + _REG(VD2_IF0_LUMA_PSEL));
-		writel_relaxed(0, priv->io_base + _REG(VD2_IF0_CHROMA_PSEL));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VD2_IF0_CHROMA1_RPT_PAT));
+		writel_relaxed(0, priv->io_base + meson_crtc->viu_offset +
+				_REG(VD1_IF0_LUMA_PSEL));
+		writel_relaxed(0, priv->io_base + meson_crtc->viu_offset +
+				_REG(VD1_IF0_CHROMA_PSEL));
+		writel_relaxed(0, priv->io_base + meson_crtc->viu_offset +
+				_REG(VD2_IF0_LUMA_PSEL));
+		writel_relaxed(0, priv->io_base + meson_crtc->viu_offset +
+				_REG(VD2_IF0_CHROMA_PSEL));
 		writel_relaxed(priv->viu.vd1_range_map_y,
-				priv->io_base + _REG(VD1_IF0_RANGE_MAP_Y));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VD1_IF0_RANGE_MAP_Y));
 		writel_relaxed(priv->viu.vd1_range_map_cb,
-				priv->io_base + _REG(VD1_IF0_RANGE_MAP_CB));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VD1_IF0_RANGE_MAP_CB));
 		writel_relaxed(priv->viu.vd1_range_map_cr,
-				priv->io_base + _REG(VD1_IF0_RANGE_MAP_CR));
+				priv->io_base + meson_crtc->viu_offset +
+				_REG(VD1_IF0_RANGE_MAP_CR));
 		writel_relaxed(0x78404,
 				priv->io_base + _REG(VPP_SC_MISC));
 		writel_relaxed(priv->viu.vpp_pic_in_height,
@@ -423,11 +548,8 @@ void meson_crtc_irq(struct meson_drm *priv)
 		writel_relaxed(0x42, priv->io_base + _REG(VPP_SCALE_COEF_IDX));
 
 		/* Enable VD1 */
-		writel_bits_relaxed(VPP_VD1_PREBLEND | VPP_VD1_POSTBLEND |
-				    VPP_COLOR_MNG_ENABLE,
-				    VPP_VD1_PREBLEND | VPP_VD1_POSTBLEND |
-				    VPP_COLOR_MNG_ENABLE,
-				    priv->io_base + _REG(VPP_MISC));
+		if (meson_crtc->enable_vd1)
+			meson_crtc->enable_vd1(priv);
 
 		priv->viu.vd1_commit = false;
 	}
@@ -464,7 +586,16 @@ int meson_crtc_create(struct meson_drm *priv)
 		return ret;
 	}
 
-	drm_crtc_helper_add(crtc, &meson_crtc_helper_funcs);
+	if (meson_vpu_is_compatible(priv, "amlogic,meson-g12a-vpu")) {
+		meson_crtc->enable_osd1 = meson_g12a_crtc_enable_osd1;
+		meson_crtc->enable_vd1 = meson_g12a_crtc_enable_vd1;
+		meson_crtc->viu_offset = MESON_G12A_VIU_OFFSET;
+		drm_crtc_helper_add(crtc, &meson_g12a_crtc_helper_funcs);
+	} else {
+		meson_crtc->enable_osd1 = meson_crtc_enable_osd1;
+		meson_crtc->enable_vd1 = meson_crtc_enable_vd1;
+		drm_crtc_helper_add(crtc, &meson_crtc_helper_funcs);
+	}
 
 	priv->crtc = crtc;
 
diff --git a/drivers/gpu/drm/meson/meson_drv.c b/drivers/gpu/drm/meson/meson_drv.c
index 8a4ebcb6405c..72b01e6be0d9 100644
--- a/drivers/gpu/drm/meson/meson_drv.c
+++ b/drivers/gpu/drm/meson/meson_drv.c
@@ -48,7 +48,6 @@
 #include "meson_vpp.h"
 #include "meson_viu.h"
 #include "meson_venc.h"
-#include "meson_canvas.h"
 #include "meson_registers.h"
 
 #define DRIVER_NAME "meson"
@@ -91,6 +90,18 @@ static irqreturn_t meson_irq(int irq, void *arg)
 	return IRQ_HANDLED;
 }
 
+static int meson_dumb_create(struct drm_file *file, struct drm_device *dev,
+			     struct drm_mode_create_dumb *args)
+{
+	/*
+	 * We need 64bytes aligned stride, and PAGE aligned size
+	 */
+	args->pitch = ALIGN(DIV_ROUND_UP(args->width * args->bpp, 8), SZ_64);
+	args->size = PAGE_ALIGN(args->pitch * args->height);
+
+	return drm_gem_cma_dumb_create_internal(file, dev, args);
+}
+
 DEFINE_DRM_GEM_CMA_FOPS(fops);
 
 static struct drm_driver meson_driver = {
@@ -113,7 +124,7 @@ static struct drm_driver meson_driver = {
 	.gem_prime_mmap		= drm_gem_cma_prime_mmap,
 
 	/* GEM Ops */
-	.dumb_create		= drm_gem_cma_dumb_create,
+	.dumb_create		= meson_dumb_create,
 	.gem_free_object_unlocked = drm_gem_cma_free_object,
 	.gem_vm_ops		= &drm_gem_cma_vm_ops,
 
@@ -231,50 +242,31 @@ static int meson_drv_bind_master(struct device *dev, bool has_components)
 	}
 
 	priv->canvas = meson_canvas_get(dev);
-	if (!IS_ERR(priv->canvas)) {
-		ret = meson_canvas_alloc(priv->canvas, &priv->canvas_id_osd1);
-		if (ret)
-			goto free_drm;
-		ret = meson_canvas_alloc(priv->canvas, &priv->canvas_id_vd1_0);
-		if (ret) {
-			meson_canvas_free(priv->canvas, priv->canvas_id_osd1);
-			goto free_drm;
-		}
-		ret = meson_canvas_alloc(priv->canvas, &priv->canvas_id_vd1_1);
-		if (ret) {
-			meson_canvas_free(priv->canvas, priv->canvas_id_osd1);
-			meson_canvas_free(priv->canvas, priv->canvas_id_vd1_0);
-			goto free_drm;
-		}
-		ret = meson_canvas_alloc(priv->canvas, &priv->canvas_id_vd1_2);
-		if (ret) {
-			meson_canvas_free(priv->canvas, priv->canvas_id_osd1);
-			meson_canvas_free(priv->canvas, priv->canvas_id_vd1_0);
-			meson_canvas_free(priv->canvas, priv->canvas_id_vd1_1);
-			goto free_drm;
-		}
-	} else {
-		priv->canvas = NULL;
-
-		res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dmc");
-		if (!res) {
-			ret = -EINVAL;
-			goto free_drm;
-		}
-		/* Simply ioremap since it may be a shared register zone */
-		regs = devm_ioremap(dev, res->start, resource_size(res));
-		if (!regs) {
-			ret = -EADDRNOTAVAIL;
-			goto free_drm;
-		}
+	if (IS_ERR(priv->canvas)) {
+		ret = PTR_ERR(priv->canvas);
+		goto free_drm;
+	}
 
-		priv->dmc = devm_regmap_init_mmio(dev, regs,
-						  &meson_regmap_config);
-		if (IS_ERR(priv->dmc)) {
-			dev_err(&pdev->dev, "Couldn't create the DMC regmap\n");
-			ret = PTR_ERR(priv->dmc);
-			goto free_drm;
-		}
+	ret = meson_canvas_alloc(priv->canvas, &priv->canvas_id_osd1);
+	if (ret)
+		goto free_drm;
+	ret = meson_canvas_alloc(priv->canvas, &priv->canvas_id_vd1_0);
+	if (ret) {
+		meson_canvas_free(priv->canvas, priv->canvas_id_osd1);
+		goto free_drm;
+	}
+	ret = meson_canvas_alloc(priv->canvas, &priv->canvas_id_vd1_1);
+	if (ret) {
+		meson_canvas_free(priv->canvas, priv->canvas_id_osd1);
+		meson_canvas_free(priv->canvas, priv->canvas_id_vd1_0);
+		goto free_drm;
+	}
+	ret = meson_canvas_alloc(priv->canvas, &priv->canvas_id_vd1_2);
+	if (ret) {
+		meson_canvas_free(priv->canvas, priv->canvas_id_osd1);
+		meson_canvas_free(priv->canvas, priv->canvas_id_vd1_0);
+		meson_canvas_free(priv->canvas, priv->canvas_id_vd1_1);
+		goto free_drm;
 	}
 
 	priv->vsync_irq = platform_get_irq(pdev, 0);
@@ -467,6 +459,7 @@ static const struct of_device_id dt_match[] = {
 	{ .compatible = "amlogic,meson-gxbb-vpu" },
 	{ .compatible = "amlogic,meson-gxl-vpu" },
 	{ .compatible = "amlogic,meson-gxm-vpu" },
+	{ .compatible = "amlogic,meson-g12a-vpu" },
 	{}
 };
 MODULE_DEVICE_TABLE(of, dt_match);
diff --git a/drivers/gpu/drm/meson/meson_drv.h b/drivers/gpu/drm/meson/meson_drv.h
index 4dccf4cd042a..9614baa836b9 100644
--- a/drivers/gpu/drm/meson/meson_drv.h
+++ b/drivers/gpu/drm/meson/meson_drv.h
@@ -29,7 +29,6 @@ struct meson_drm {
 	struct device *dev;
 	void __iomem *io_base;
 	struct regmap *hhi;
-	struct regmap *dmc;
 	int vsync_irq;
 
 	struct meson_canvas *canvas;
@@ -63,6 +62,10 @@ struct meson_drm {
 		uint32_t osd_sc_h_phase_step;
 		uint32_t osd_sc_h_ctrl0;
 		uint32_t osd_sc_v_ctrl0;
+		uint32_t osd_blend_din0_scope_h;
+		uint32_t osd_blend_din0_scope_v;
+		uint32_t osb_blend0_size;
+		uint32_t osb_blend1_size;
 
 		bool vd1_enabled;
 		bool vd1_commit;
diff --git a/drivers/gpu/drm/meson/meson_dw_hdmi.c b/drivers/gpu/drm/meson/meson_dw_hdmi.c
index 563953ec6ad0..779da21143b9 100644
--- a/drivers/gpu/drm/meson/meson_dw_hdmi.c
+++ b/drivers/gpu/drm/meson/meson_dw_hdmi.c
@@ -20,6 +20,7 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/component.h>
+#include <linux/of_device.h>
 #include <linux/of_graph.h>
 #include <linux/reset.h>
 #include <linux/clk.h>
@@ -105,6 +106,7 @@
 #define HDMITX_TOP_ADDR_REG	0x0
 #define HDMITX_TOP_DATA_REG	0x4
 #define HDMITX_TOP_CTRL_REG	0x8
+#define HDMITX_TOP_G12A_OFFSET	0x8000
 
 /* Controller Communication Channel */
 #define HDMITX_DWC_ADDR_REG	0x10
@@ -118,6 +120,8 @@
 #define HHI_HDMI_PHY_CNTL1	0x3a4 /* 0xe9 */
 #define HHI_HDMI_PHY_CNTL2	0x3a8 /* 0xea */
 #define HHI_HDMI_PHY_CNTL3	0x3ac /* 0xeb */
+#define HHI_HDMI_PHY_CNTL4	0x3b0 /* 0xec */
+#define HHI_HDMI_PHY_CNTL5	0x3b4 /* 0xed */
 
 static DEFINE_SPINLOCK(reg_lock);
 
@@ -127,12 +131,26 @@ enum meson_venc_source {
 	MESON_VENC_SOURCE_ENCP = 2,
 };
 
+struct meson_dw_hdmi;
+
+struct meson_dw_hdmi_data {
+	unsigned int	(*top_read)(struct meson_dw_hdmi *dw_hdmi,
+				    unsigned int addr);
+	void		(*top_write)(struct meson_dw_hdmi *dw_hdmi,
+				     unsigned int addr, unsigned int data);
+	unsigned int	(*dwc_read)(struct meson_dw_hdmi *dw_hdmi,
+				    unsigned int addr);
+	void		(*dwc_write)(struct meson_dw_hdmi *dw_hdmi,
+				     unsigned int addr, unsigned int data);
+};
+
 struct meson_dw_hdmi {
 	struct drm_encoder encoder;
 	struct dw_hdmi_plat_data dw_plat_data;
 	struct meson_drm *priv;
 	struct device *dev;
 	void __iomem *hdmitx;
+	const struct meson_dw_hdmi_data *data;
 	struct reset_control *hdmitx_apb;
 	struct reset_control *hdmitx_ctrl;
 	struct reset_control *hdmitx_phy;
@@ -174,6 +192,12 @@ static unsigned int dw_hdmi_top_read(struct meson_dw_hdmi *dw_hdmi,
 	return data;
 }
 
+static unsigned int dw_hdmi_g12a_top_read(struct meson_dw_hdmi *dw_hdmi,
+					  unsigned int addr)
+{
+	return readl(dw_hdmi->hdmitx + HDMITX_TOP_G12A_OFFSET + (addr << 2));
+}
+
 static inline void dw_hdmi_top_write(struct meson_dw_hdmi *dw_hdmi,
 				     unsigned int addr, unsigned int data)
 {
@@ -191,18 +215,24 @@ static inline void dw_hdmi_top_write(struct meson_dw_hdmi *dw_hdmi,
 	spin_unlock_irqrestore(&reg_lock, flags);
 }
 
+static inline void dw_hdmi_g12a_top_write(struct meson_dw_hdmi *dw_hdmi,
+					  unsigned int addr, unsigned int data)
+{
+	writel(data, dw_hdmi->hdmitx + HDMITX_TOP_G12A_OFFSET + (addr << 2));
+}
+
 /* Helper to change specific bits in PHY registers */
 static inline void dw_hdmi_top_write_bits(struct meson_dw_hdmi *dw_hdmi,
 					  unsigned int addr,
 					  unsigned int mask,
 					  unsigned int val)
 {
-	unsigned int data = dw_hdmi_top_read(dw_hdmi, addr);
+	unsigned int data = dw_hdmi->data->top_read(dw_hdmi, addr);
 
 	data &= ~mask;
 	data |= val;
 
-	dw_hdmi_top_write(dw_hdmi, addr, data);
+	dw_hdmi->data->top_write(dw_hdmi, addr, data);
 }
 
 static unsigned int dw_hdmi_dwc_read(struct meson_dw_hdmi *dw_hdmi,
@@ -226,6 +256,12 @@ static unsigned int dw_hdmi_dwc_read(struct meson_dw_hdmi *dw_hdmi,
 	return data;
 }
 
+static unsigned int dw_hdmi_g12a_dwc_read(struct meson_dw_hdmi *dw_hdmi,
+					  unsigned int addr)
+{
+	return readb(dw_hdmi->hdmitx + addr);
+}
+
 static inline void dw_hdmi_dwc_write(struct meson_dw_hdmi *dw_hdmi,
 				     unsigned int addr, unsigned int data)
 {
@@ -243,18 +279,24 @@ static inline void dw_hdmi_dwc_write(struct meson_dw_hdmi *dw_hdmi,
 	spin_unlock_irqrestore(&reg_lock, flags);
 }
 
+static inline void dw_hdmi_g12a_dwc_write(struct meson_dw_hdmi *dw_hdmi,
+					  unsigned int addr, unsigned int data)
+{
+	writeb(data, dw_hdmi->hdmitx + addr);
+}
+
 /* Helper to change specific bits in controller registers */
 static inline void dw_hdmi_dwc_write_bits(struct meson_dw_hdmi *dw_hdmi,
 					  unsigned int addr,
 					  unsigned int mask,
 					  unsigned int val)
 {
-	unsigned int data = dw_hdmi_dwc_read(dw_hdmi, addr);
+	unsigned int data = dw_hdmi->data->dwc_read(dw_hdmi, addr);
 
 	data &= ~mask;
 	data |= val;
 
-	dw_hdmi_dwc_write(dw_hdmi, addr, data);
+	dw_hdmi->data->dwc_write(dw_hdmi, addr, data);
 }
 
 /* Bridge */
@@ -300,6 +342,24 @@ static void meson_hdmi_phy_setup_mode(struct meson_dw_hdmi *dw_hdmi,
 			regmap_write(priv->hhi, HHI_HDMI_PHY_CNTL0, 0x33632122);
 			regmap_write(priv->hhi, HHI_HDMI_PHY_CNTL3, 0x2000115b);
 		}
+	} else if (dw_hdmi_is_compatible(dw_hdmi,
+					 "amlogic,meson-g12a-dw-hdmi")) {
+		if (pixel_clock >= 371250) {
+			/* 5.94Gbps, 3.7125Gbps */
+			regmap_write(priv->hhi, HHI_HDMI_PHY_CNTL0, 0x37eb65c4);
+			regmap_write(priv->hhi, HHI_HDMI_PHY_CNTL3, 0x2ab0ff3b);
+			regmap_write(priv->hhi, HHI_HDMI_PHY_CNTL5, 0x0000080b);
+		} else if (pixel_clock >= 297000) {
+			/* 2.97Gbps */
+			regmap_write(priv->hhi, HHI_HDMI_PHY_CNTL0, 0x33eb6262);
+			regmap_write(priv->hhi, HHI_HDMI_PHY_CNTL3, 0x2ab0ff3b);
+			regmap_write(priv->hhi, HHI_HDMI_PHY_CNTL5, 0x00000003);
+		} else {
+			/* 1.485Gbps, and below */
+			regmap_write(priv->hhi, HHI_HDMI_PHY_CNTL0, 0x33eb4242);
+			regmap_write(priv->hhi, HHI_HDMI_PHY_CNTL3, 0x2ab0ff3b);
+			regmap_write(priv->hhi, HHI_HDMI_PHY_CNTL5, 0x00000003);
+		}
 	}
 }
 
@@ -375,7 +435,7 @@ static int dw_hdmi_phy_init(struct dw_hdmi *hdmi, void *data,
 	regmap_update_bits(priv->hhi, HHI_MEM_PD_REG0, 0xff << 8, 0);
 
 	/* Bring out of reset */
-	dw_hdmi_top_write(dw_hdmi, HDMITX_TOP_SW_RESET,  0);
+	dw_hdmi->data->top_write(dw_hdmi, HDMITX_TOP_SW_RESET,  0);
 
 	/* Enable internal pixclk, tmds_clk, spdif_clk, i2s_clk, cecclk */
 	dw_hdmi_top_write_bits(dw_hdmi, HDMITX_TOP_CLK_CNTL,
@@ -384,24 +444,25 @@ static int dw_hdmi_phy_init(struct dw_hdmi *hdmi, void *data,
 			       0x3 << 4, 0x3 << 4);
 
 	/* Enable normal output to PHY */
-	dw_hdmi_top_write(dw_hdmi, HDMITX_TOP_BIST_CNTL, BIT(12));
+	dw_hdmi->data->top_write(dw_hdmi, HDMITX_TOP_BIST_CNTL, BIT(12));
 
 	/* TMDS pattern setup (TOFIX Handle the YUV420 case) */
 	if (mode->clock > 340000) {
-		dw_hdmi_top_write(dw_hdmi, HDMITX_TOP_TMDS_CLK_PTTN_01, 0);
-		dw_hdmi_top_write(dw_hdmi, HDMITX_TOP_TMDS_CLK_PTTN_23,
+		dw_hdmi->data->top_write(dw_hdmi, HDMITX_TOP_TMDS_CLK_PTTN_01,
+				  0);
+		dw_hdmi->data->top_write(dw_hdmi, HDMITX_TOP_TMDS_CLK_PTTN_23,
 				  0x03ff03ff);
 	} else {
-		dw_hdmi_top_write(dw_hdmi, HDMITX_TOP_TMDS_CLK_PTTN_01,
+		dw_hdmi->data->top_write(dw_hdmi, HDMITX_TOP_TMDS_CLK_PTTN_01,
 				  0x001f001f);
-		dw_hdmi_top_write(dw_hdmi, HDMITX_TOP_TMDS_CLK_PTTN_23,
+		dw_hdmi->data->top_write(dw_hdmi, HDMITX_TOP_TMDS_CLK_PTTN_23,
 				  0x001f001f);
 	}
 
 	/* Load TMDS pattern */
-	dw_hdmi_top_write(dw_hdmi, HDMITX_TOP_TMDS_CLK_PTTN_CNTL, 0x1);
+	dw_hdmi->data->top_write(dw_hdmi, HDMITX_TOP_TMDS_CLK_PTTN_CNTL, 0x1);
 	msleep(20);
-	dw_hdmi_top_write(dw_hdmi, HDMITX_TOP_TMDS_CLK_PTTN_CNTL, 0x2);
+	dw_hdmi->data->top_write(dw_hdmi, HDMITX_TOP_TMDS_CLK_PTTN_CNTL, 0x2);
 
 	/* Setup PHY parameters */
 	meson_hdmi_phy_setup_mode(dw_hdmi, mode);
@@ -412,7 +473,8 @@ static int dw_hdmi_phy_init(struct dw_hdmi *hdmi, void *data,
 
 	/* BIT_INVERT */
 	if (dw_hdmi_is_compatible(dw_hdmi, "amlogic,meson-gxl-dw-hdmi") ||
-	    dw_hdmi_is_compatible(dw_hdmi, "amlogic,meson-gxm-dw-hdmi"))
+	    dw_hdmi_is_compatible(dw_hdmi, "amlogic,meson-gxm-dw-hdmi") ||
+	    dw_hdmi_is_compatible(dw_hdmi, "amlogic,meson-g12a-dw-hdmi"))
 		regmap_update_bits(priv->hhi, HHI_HDMI_PHY_CNTL1,
 				   BIT(17), 0);
 	else
@@ -480,7 +542,7 @@ static enum drm_connector_status dw_hdmi_read_hpd(struct dw_hdmi *hdmi,
 {
 	struct meson_dw_hdmi *dw_hdmi = (struct meson_dw_hdmi *)data;
 
-	return !!dw_hdmi_top_read(dw_hdmi, HDMITX_TOP_STAT0) ?
+	return !!dw_hdmi->data->top_read(dw_hdmi, HDMITX_TOP_STAT0) ?
 		connector_status_connected : connector_status_disconnected;
 }
 
@@ -490,11 +552,11 @@ static void dw_hdmi_setup_hpd(struct dw_hdmi *hdmi,
 	struct meson_dw_hdmi *dw_hdmi = (struct meson_dw_hdmi *)data;
 
 	/* Setup HPD Filter */
-	dw_hdmi_top_write(dw_hdmi, HDMITX_TOP_HPD_FILTER,
+	dw_hdmi->data->top_write(dw_hdmi, HDMITX_TOP_HPD_FILTER,
 			  (0xa << 12) | 0xa0);
 
 	/* Clear interrupts */
-	dw_hdmi_top_write(dw_hdmi, HDMITX_TOP_INTR_STAT_CLR,
+	dw_hdmi->data->top_write(dw_hdmi, HDMITX_TOP_INTR_STAT_CLR,
 			  HDMITX_TOP_INTR_HPD_RISE | HDMITX_TOP_INTR_HPD_FALL);
 
 	/* Unmask interrupts */
@@ -515,8 +577,8 @@ static irqreturn_t dw_hdmi_top_irq(int irq, void *dev_id)
 	struct meson_dw_hdmi *dw_hdmi = dev_id;
 	u32 stat;
 
-	stat = dw_hdmi_top_read(dw_hdmi, HDMITX_TOP_INTR_STAT);
-	dw_hdmi_top_write(dw_hdmi, HDMITX_TOP_INTR_STAT_CLR, stat);
+	stat = dw_hdmi->data->top_read(dw_hdmi, HDMITX_TOP_INTR_STAT);
+	dw_hdmi->data->top_write(dw_hdmi, HDMITX_TOP_INTR_STAT_CLR, stat);
 
 	/* HPD Events, handle in the threaded interrupt handler */
 	if (stat & (HDMITX_TOP_INTR_HPD_RISE | HDMITX_TOP_INTR_HPD_FALL)) {
@@ -686,7 +748,9 @@ static const struct drm_encoder_helper_funcs
 static int meson_dw_hdmi_reg_read(void *context, unsigned int reg,
 				  unsigned int *result)
 {
-	*result = dw_hdmi_dwc_read(context, reg);
+	struct meson_dw_hdmi *dw_hdmi = context;
+
+	*result = dw_hdmi->data->dwc_read(dw_hdmi, reg);
 
 	return 0;
 
@@ -695,7 +759,9 @@ static int meson_dw_hdmi_reg_read(void *context, unsigned int reg,
 static int meson_dw_hdmi_reg_write(void *context, unsigned int reg,
 				   unsigned int val)
 {
-	dw_hdmi_dwc_write(context, reg, val);
+	struct meson_dw_hdmi *dw_hdmi = context;
+
+	dw_hdmi->data->dwc_write(dw_hdmi, reg, val);
 
 	return 0;
 }
@@ -709,6 +775,20 @@ static const struct regmap_config meson_dw_hdmi_regmap_config = {
 	.fast_io = true,
 };
 
+static const struct meson_dw_hdmi_data meson_dw_hdmi_gx_data = {
+	.top_read = dw_hdmi_top_read,
+	.top_write = dw_hdmi_top_write,
+	.dwc_read = dw_hdmi_dwc_read,
+	.dwc_write = dw_hdmi_dwc_write,
+};
+
+static const struct meson_dw_hdmi_data meson_dw_hdmi_g12a_data = {
+	.top_read = dw_hdmi_g12a_top_read,
+	.top_write = dw_hdmi_g12a_top_write,
+	.dwc_read = dw_hdmi_g12a_dwc_read,
+	.dwc_write = dw_hdmi_g12a_dwc_write,
+};
+
 static bool meson_hdmi_connector_is_available(struct device *dev)
 {
 	struct device_node *ep, *remote;
@@ -735,6 +815,7 @@ static int meson_dw_hdmi_bind(struct device *dev, struct device *master,
 				void *data)
 {
 	struct platform_device *pdev = to_platform_device(dev);
+	const struct meson_dw_hdmi_data *match;
 	struct meson_dw_hdmi *meson_dw_hdmi;
 	struct drm_device *drm = data;
 	struct meson_drm *priv = drm->dev_private;
@@ -751,6 +832,12 @@ static int meson_dw_hdmi_bind(struct device *dev, struct device *master,
 		return -ENODEV;
 	}
 
+	match = of_device_get_match_data(&pdev->dev);
+	if (!match) {
+		dev_err(&pdev->dev, "failed to get match data\n");
+		return -ENODEV;
+	}
+
 	meson_dw_hdmi = devm_kzalloc(dev, sizeof(*meson_dw_hdmi),
 				     GFP_KERNEL);
 	if (!meson_dw_hdmi)
@@ -758,6 +845,7 @@ static int meson_dw_hdmi_bind(struct device *dev, struct device *master,
 
 	meson_dw_hdmi->priv = priv;
 	meson_dw_hdmi->dev = dev;
+	meson_dw_hdmi->data = match;
 	dw_plat_data = &meson_dw_hdmi->dw_plat_data;
 	encoder = &meson_dw_hdmi->encoder;
 
@@ -858,24 +946,28 @@ static int meson_dw_hdmi_bind(struct device *dev, struct device *master,
 	reset_control_reset(meson_dw_hdmi->hdmitx_phy);
 
 	/* Enable APB3 fail on error */
-	writel_bits_relaxed(BIT(15), BIT(15),
-			    meson_dw_hdmi->hdmitx + HDMITX_TOP_CTRL_REG);
-	writel_bits_relaxed(BIT(15), BIT(15),
-			    meson_dw_hdmi->hdmitx + HDMITX_DWC_CTRL_REG);
+	if (!meson_vpu_is_compatible(priv, "amlogic,meson-g12a-vpu")) {
+		writel_bits_relaxed(BIT(15), BIT(15),
+				    meson_dw_hdmi->hdmitx + HDMITX_TOP_CTRL_REG);
+		writel_bits_relaxed(BIT(15), BIT(15),
+				    meson_dw_hdmi->hdmitx + HDMITX_DWC_CTRL_REG);
+	}
 
 	/* Bring out of reset */
-	dw_hdmi_top_write(meson_dw_hdmi, HDMITX_TOP_SW_RESET,  0);
+	meson_dw_hdmi->data->top_write(meson_dw_hdmi,
+				       HDMITX_TOP_SW_RESET,  0);
 
 	msleep(20);
 
-	dw_hdmi_top_write(meson_dw_hdmi, HDMITX_TOP_CLK_CNTL, 0xff);
+	meson_dw_hdmi->data->top_write(meson_dw_hdmi,
+				       HDMITX_TOP_CLK_CNTL, 0xff);
 
 	/* Enable HDMI-TX Interrupt */
-	dw_hdmi_top_write(meson_dw_hdmi, HDMITX_TOP_INTR_STAT_CLR,
-			  HDMITX_TOP_INTR_CORE);
+	meson_dw_hdmi->data->top_write(meson_dw_hdmi, HDMITX_TOP_INTR_STAT_CLR,
+				       HDMITX_TOP_INTR_CORE);
 
-	dw_hdmi_top_write(meson_dw_hdmi, HDMITX_TOP_INTR_MASKN,
-			  HDMITX_TOP_INTR_CORE);
+	meson_dw_hdmi->data->top_write(meson_dw_hdmi, HDMITX_TOP_INTR_MASKN,
+				       HDMITX_TOP_INTR_CORE);
 
 	/* Bridge / Connector */
 
@@ -924,9 +1016,14 @@ static int meson_dw_hdmi_remove(struct platform_device *pdev)
 }
 
 static const struct of_device_id meson_dw_hdmi_of_table[] = {
-	{ .compatible = "amlogic,meson-gxbb-dw-hdmi" },
-	{ .compatible = "amlogic,meson-gxl-dw-hdmi" },
-	{ .compatible = "amlogic,meson-gxm-dw-hdmi" },
+	{ .compatible = "amlogic,meson-gxbb-dw-hdmi",
+	  .data = &meson_dw_hdmi_gx_data },
+	{ .compatible = "amlogic,meson-gxl-dw-hdmi",
+	  .data = &meson_dw_hdmi_gx_data },
+	{ .compatible = "amlogic,meson-gxm-dw-hdmi",
+	  .data = &meson_dw_hdmi_gx_data },
+	{ .compatible = "amlogic,meson-g12a-dw-hdmi",
+	  .data = &meson_dw_hdmi_g12a_data },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, meson_dw_hdmi_of_table);
diff --git a/drivers/gpu/drm/meson/meson_dw_hdmi.h b/drivers/gpu/drm/meson/meson_dw_hdmi.h
index 0b81183125e3..03e2f0c1a2d5 100644
--- a/drivers/gpu/drm/meson/meson_dw_hdmi.h
+++ b/drivers/gpu/drm/meson/meson_dw_hdmi.h
@@ -21,9 +21,12 @@
 #define __MESON_DW_HDMI_H
 
 /*
- * Bit 7 RW Reserved. Default 1.
- * Bit 6 RW Reserved. Default 1.
- * Bit 5 RW Reserved. Default 1.
+ * Bit 15-10: RW Reserved. Default 1 starting from G12A
+ * Bit 9 RW sw_reset_i2c starting from G12A
+ * Bit 8 RW sw_reset_axiarb starting from G12A
+ * Bit 7 RW Reserved. Default 1, sw_reset_emp starting from G12A
+ * Bit 6 RW Reserved. Default 1, sw_reset_flt starting from G12A
+ * Bit 5 RW Reserved. Default 1, sw_reset_hdcp22 starting from G12A
  * Bit 4 RW sw_reset_phyif: PHY interface. 1=Apply reset; 0=Release from reset.
  *     Default 1.
  * Bit 3 RW sw_reset_intr: interrupt module. 1=Apply reset;
@@ -39,12 +42,16 @@
 #define HDMITX_TOP_SW_RESET                     (0x000)
 
 /*
+ * Bit 31 RW free_clk_en: 0=Enable clock gating for power saving; 1= Disable
  * Bit 12 RW i2s_ws_inv:1=Invert i2s_ws; 0=No invert. Default 0.
  * Bit 11 RW i2s_clk_inv: 1=Invert i2s_clk; 0=No invert. Default 0.
  * Bit 10 RW spdif_clk_inv: 1=Invert spdif_clk; 0=No invert. Default 0.
  * Bit 9 RW tmds_clk_inv: 1=Invert tmds_clk; 0=No invert. Default 0.
  * Bit 8 RW pixel_clk_inv: 1=Invert pixel_clk; 0=No invert. Default 0.
- * Bit 4 RW cec_clk_en: 1=enable cec_clk; 0=disable. Default 0.
+ * Bit 7 RW hdcp22_skpclk_en: starting from G12A, 1=enable; 0=disable
+ * Bit 6 RW hdcp22_esmclk_en: starting from G12A, 1=enable; 0=disable
+ * Bit 5 RW hdcp22_tmdsclk_en: starting from G12A, 1=enable; 0=disable
+ * Bit 4 RW cec_clk_en: 1=enable cec_clk; 0=disable. Default 0. Reserved for G12A
  * Bit 3 RW i2s_clk_en: 1=enable i2s_clk; 0=disable. Default 0.
  * Bit 2 RW spdif_clk_en: 1=enable spdif_clk; 0=disable. Default 0.
  * Bit 1 RW tmds_clk_en: 1=enable tmds_clk;  0=disable. Default 0.
@@ -53,6 +60,8 @@
 #define HDMITX_TOP_CLK_CNTL                     (0x001)
 
 /*
+ * Bit 31:28 RW rxsense_glitch_width: starting from G12A
+ * Bit 27:16 RW rxsense_valid_width: starting from G12A
  * Bit 11: 0 RW hpd_valid_width: filter out width <= M*1024.    Default 0.
  * Bit 15:12 RW hpd_glitch_width: filter out glitch <= N.       Default 0.
  */
@@ -61,6 +70,9 @@
 /*
  * intr_maskn: MASK_N, one bit per interrupt source.
  *     1=Enable interrupt source; 0=Disable interrupt source. Default 0.
+ * [  7] rxsense_fall starting from G12A
+ * [  6] rxsense_rise starting from G12A
+ * [  5] err_i2c_timeout starting from G12A
  * [  4] hdcp22_rndnum_err
  * [  3] nonce_rfrsh_rise
  * [  2] hpd_fall_intr
@@ -73,6 +85,9 @@
  * Bit 30: 0 RW intr_stat: For each bit, write 1 to manually set the interrupt
  *     bit, read back the interrupt status.
  * Bit    31 R  IP interrupt status
+ * Bit     7 RW rxsense_fall starting from G12A
+ * Bit     6 RW rxsense_rise starting from G12A
+ * Bit     5 RW err_i2c_timeout starting from G12A
  * Bit     2 RW hpd_fall
  * Bit     1 RW hpd_rise
  * Bit     0 RW IP interrupt
@@ -80,6 +95,9 @@
 #define HDMITX_TOP_INTR_STAT                    (0x004)
 
 /*
+ * [7]    rxsense_fall starting from G12A
+ * [6]    rxsense_rise starting from G12A
+ * [5]    err_i2c_timeout starting from G12A
  * [4]	  hdcp22_rndnum_err
  * [3]	  nonce_rfrsh_rise
  * [2]	  hpd_fall
@@ -91,6 +109,8 @@
 #define HDMITX_TOP_INTR_CORE		BIT(0)
 #define HDMITX_TOP_INTR_HPD_RISE	BIT(1)
 #define HDMITX_TOP_INTR_HPD_FALL	BIT(2)
+#define HDMITX_TOP_INTR_RXSENSE_RISE	BIT(6)
+#define HDMITX_TOP_INTR_RXSENSE_FALL	BIT(7)
 
 /* Bit 14:12 RW tmds_sel: 3'b000=Output zero; 3'b001=Output normal TMDS data;
  *     3'b010=Output PRBS data; 3'b100=Output shift pattern. Default 0.
@@ -140,7 +160,9 @@
  */
 #define HDMITX_TOP_REVOCMEM_STAT                (0x00D)
 
-/* Bit     0 R  filtered HPD status. */
+/* Bit	   1 R	filtered RxSense status
+ * Bit     0 R  filtered HPD status.
+ */
 #define HDMITX_TOP_STAT0                        (0x00E)
 
 #endif /* __MESON_DW_HDMI_H */
diff --git a/drivers/gpu/drm/meson/meson_overlay.c b/drivers/gpu/drm/meson/meson_overlay.c
index 691a9fd16b36..bdbf925ff3e8 100644
--- a/drivers/gpu/drm/meson/meson_overlay.c
+++ b/drivers/gpu/drm/meson/meson_overlay.c
@@ -22,7 +22,6 @@
 #include "meson_overlay.h"
 #include "meson_vpp.h"
 #include "meson_viu.h"
-#include "meson_canvas.h"
 #include "meson_registers.h"
 
 /* VD1_IF0_GEN_REG */
@@ -350,13 +349,6 @@ static void meson_overlay_atomic_update(struct drm_plane *plane,
 
 	DRM_DEBUG_DRIVER("\n");
 
-	/* Fallback is canvas provider is not available */
-	if (!priv->canvas) {
-		priv->canvas_id_vd1_0 = MESON_CANVAS_ID_VD1_0;
-		priv->canvas_id_vd1_1 = MESON_CANVAS_ID_VD1_1;
-		priv->canvas_id_vd1_2 = MESON_CANVAS_ID_VD1_2;
-	}
-
 	interlace_mode = state->crtc->mode.flags & DRM_MODE_FLAG_INTERLACE;
 
 	spin_lock_irqsave(&priv->drm->event_lock, flags);
@@ -524,8 +516,14 @@ static void meson_overlay_atomic_disable(struct drm_plane *plane,
 	priv->viu.vd1_enabled = false;
 
 	/* Disable VD1 */
-	writel_bits_relaxed(VPP_VD1_POSTBLEND | VPP_VD1_PREBLEND, 0,
-			    priv->io_base + _REG(VPP_MISC));
+	if (meson_vpu_is_compatible(priv, "amlogic,meson-g12a-vpu")) {
+		writel_relaxed(0, priv->io_base + _REG(VD1_BLEND_SRC_CTRL));
+		writel_relaxed(0, priv->io_base + _REG(VD2_BLEND_SRC_CTRL));
+		writel_relaxed(0, priv->io_base + _REG(VD1_IF0_GEN_REG + 0x17b0));
+		writel_relaxed(0, priv->io_base + _REG(VD2_IF0_GEN_REG + 0x17b0));
+	} else
+		writel_bits_relaxed(VPP_VD1_POSTBLEND | VPP_VD1_PREBLEND, 0,
+				    priv->io_base + _REG(VPP_MISC));
 
 }
 
diff --git a/drivers/gpu/drm/meson/meson_plane.c b/drivers/gpu/drm/meson/meson_plane.c
index 6119a0224278..bf8f1fab63aa 100644
--- a/drivers/gpu/drm/meson/meson_plane.c
+++ b/drivers/gpu/drm/meson/meson_plane.c
@@ -38,7 +38,6 @@
 #include "meson_plane.h"
 #include "meson_vpp.h"
 #include "meson_viu.h"
-#include "meson_canvas.h"
 #include "meson_registers.h"
 
 /* OSD_SCI_WH_M1 */
@@ -148,10 +147,7 @@ static void meson_plane_atomic_update(struct drm_plane *plane,
 				   (0xFF << OSD_GLOBAL_ALPHA_SHIFT) |
 				   OSD_BLK0_ENABLE;
 
-	if (priv->canvas)
-		canvas_id_osd1 = priv->canvas_id_osd1;
-	else
-		canvas_id_osd1 = MESON_CANVAS_ID_OSD1;
+	canvas_id_osd1 = priv->canvas_id_osd1;
 
 	/* Set up BLK0 to point to the right canvas */
 	priv->viu.osd1_blk0_cfg[0] = ((canvas_id_osd1 << OSD_CANVAS_SEL) |
@@ -298,6 +294,13 @@ static void meson_plane_atomic_update(struct drm_plane *plane,
 	priv->viu.osd1_blk0_cfg[3] = ((dest.x2 - 1) << 16) | dest.x1;
 	priv->viu.osd1_blk0_cfg[4] = ((dest.y2 - 1) << 16) | dest.y1;
 
+	if (meson_vpu_is_compatible(priv, "amlogic,meson-g12a-vpu")) {
+		priv->viu.osd_blend_din0_scope_h = ((dest.x2 - 1) << 16) | dest.x1;
+		priv->viu.osd_blend_din0_scope_v = ((dest.y2 - 1) << 16) | dest.y1;
+		priv->viu.osb_blend0_size = dst_h << 16 | dst_w;
+		priv->viu.osb_blend1_size = dst_h << 16 | dst_w;
+	}
+
 	/* Update Canvas with buffer address */
 	gem = drm_fb_cma_get_gem_obj(fb, 0);
 
@@ -324,8 +327,12 @@ static void meson_plane_atomic_disable(struct drm_plane *plane,
 	struct meson_drm *priv = meson_plane->priv;
 
 	/* Disable OSD1 */
-	writel_bits_relaxed(VPP_OSD1_POSTBLEND, 0,
-			    priv->io_base + _REG(VPP_MISC));
+	if (meson_vpu_is_compatible(priv, "amlogic,meson-g12a-vpu"))
+		writel_bits_relaxed(BIT(0) | BIT(21), 0,
+			priv->io_base + _REG(VIU_OSD1_CTRL_STAT));
+	else
+		writel_bits_relaxed(VPP_OSD1_POSTBLEND, 0,
+				    priv->io_base + _REG(VPP_MISC));
 
 	meson_plane->enabled = false;
 
diff --git a/drivers/gpu/drm/meson/meson_registers.h b/drivers/gpu/drm/meson/meson_registers.h
index 5c7e02c703bc..cfaf90501bb1 100644
--- a/drivers/gpu/drm/meson/meson_registers.h
+++ b/drivers/gpu/drm/meson/meson_registers.h
@@ -216,6 +216,29 @@
 #define VIU_OSD2_FIFO_CTRL_STAT 0x1a4b
 #define VIU_OSD2_TEST_RDDATA 0x1a4c
 #define VIU_OSD2_PROT_CTRL 0x1a4e
+#define VIU_OSD2_MALI_UNPACK_CTRL 0x1abd
+#define VIU_OSD2_DIMM_CTRL 0x1acf
+
+#define VIU_OSD3_CTRL_STAT 0x3d80
+#define VIU_OSD3_CTRL_STAT2 0x3d81
+#define VIU_OSD3_COLOR_ADDR 0x3d82
+#define VIU_OSD3_COLOR 0x3d83
+#define VIU_OSD3_TCOLOR_AG0 0x3d84
+#define VIU_OSD3_TCOLOR_AG1 0x3d85
+#define VIU_OSD3_TCOLOR_AG2 0x3d86
+#define VIU_OSD3_TCOLOR_AG3 0x3d87
+#define VIU_OSD3_BLK0_CFG_W0 0x3d88
+#define VIU_OSD3_BLK0_CFG_W1 0x3d8c
+#define VIU_OSD3_BLK0_CFG_W2 0x3d90
+#define VIU_OSD3_BLK0_CFG_W3 0x3d94
+#define VIU_OSD3_BLK0_CFG_W4 0x3d98
+#define VIU_OSD3_BLK1_CFG_W4 0x3d99
+#define VIU_OSD3_BLK2_CFG_W4 0x3d9a
+#define VIU_OSD3_FIFO_CTRL_STAT 0x3d9c
+#define VIU_OSD3_TEST_RDDATA 0x3d9d
+#define VIU_OSD3_PROT_CTRL 0x3d9e
+#define VIU_OSD3_MALI_UNPACK_CTRL 0x3d9f
+#define VIU_OSD3_DIMM_CTRL 0x3da0
 
 #define VD1_IF0_GEN_REG 0x1a50
 #define VD1_IF0_CANVAS0 0x1a51
@@ -287,6 +310,27 @@
 #define VIU_OSD1_MATRIX_COEF31_32 0x1a9e
 #define VIU_OSD1_MATRIX_COEF40_41 0x1a9f
 #define VD1_IF0_GEN_REG3 0x1aa7
+
+#define VIU_OSD_BLENDO_H_START_END 0x1aa9
+#define VIU_OSD_BLENDO_V_START_END 0x1aaa
+#define VIU_OSD_BLEND_GEN_CTRL0 0x1aab
+#define VIU_OSD_BLEND_GEN_CTRL1 0x1aac
+#define VIU_OSD_BLEND_DUMMY_DATA 0x1aad
+#define VIU_OSD_BLEND_CURRENT_XY 0x1aae
+
+#define VIU_OSD2_MATRIX_CTRL 0x1ab0
+#define VIU_OSD2_MATRIX_COEF00_01 0x1ab1
+#define VIU_OSD2_MATRIX_COEF02_10 0x1ab2
+#define VIU_OSD2_MATRIX_COEF11_12 0x1ab3
+#define VIU_OSD2_MATRIX_COEF20_21 0x1ab4
+#define VIU_OSD2_MATRIX_COEF22 0x1ab5
+#define VIU_OSD2_MATRIX_OFFSET0_1 0x1ab6
+#define VIU_OSD2_MATRIX_OFFSET2 0x1ab7
+#define VIU_OSD2_MATRIX_PRE_OFFSET0_1 0x1ab8
+#define VIU_OSD2_MATRIX_PRE_OFFSET2 0x1ab9
+#define VIU_OSD2_MATRIX_PROBE_COLOR 0x1aba
+#define VIU_OSD2_MATRIX_HL_COLOR 0x1abb
+#define VIU_OSD2_MATRIX_PROBE_POS 0x1abc
 #define VIU_OSD1_EOTF_CTL 0x1ad4
 #define VIU_OSD1_EOTF_COEF00_01 0x1ad5
 #define VIU_OSD1_EOTF_COEF02_10 0x1ad6
@@ -481,6 +525,82 @@
 #define VPP_OSD_SCALE_COEF 0x1dcd
 #define VPP_INT_LINE_NUM 0x1dce
 
+#define VPP_WRAP_OSD1_MATRIX_COEF00_01 0x3d60
+#define VPP_WRAP_OSD1_MATRIX_COEF02_10 0x3d61
+#define VPP_WRAP_OSD1_MATRIX_COEF11_12 0x3d62
+#define VPP_WRAP_OSD1_MATRIX_COEF20_21 0x3d63
+#define VPP_WRAP_OSD1_MATRIX_COEF22 0x3d64
+#define VPP_WRAP_OSD1_MATRIX_COEF13_14 0x3d65
+#define VPP_WRAP_OSD1_MATRIX_COEF23_24 0x3d66
+#define VPP_WRAP_OSD1_MATRIX_COEF15_25 0x3d67
+#define VPP_WRAP_OSD1_MATRIX_CLIP 0x3d68
+#define VPP_WRAP_OSD1_MATRIX_OFFSET0_1 0x3d69
+#define VPP_WRAP_OSD1_MATRIX_OFFSET2 0x3d6a
+#define VPP_WRAP_OSD1_MATRIX_PRE_OFFSET0_1 0x3d6b
+#define VPP_WRAP_OSD1_MATRIX_PRE_OFFSET2 0x3d6c
+#define VPP_WRAP_OSD1_MATRIX_EN_CTRL 0x3d6d
+
+#define VPP_WRAP_OSD2_MATRIX_COEF00_01 0x3d70
+#define VPP_WRAP_OSD2_MATRIX_COEF02_10 0x3d71
+#define VPP_WRAP_OSD2_MATRIX_COEF11_12 0x3d72
+#define VPP_WRAP_OSD2_MATRIX_COEF20_21 0x3d73
+#define VPP_WRAP_OSD2_MATRIX_COEF22 0x3d74
+#define VPP_WRAP_OSD2_MATRIX_COEF13_14 0x3d75
+#define VPP_WRAP_OSD2_MATRIX_COEF23_24 0x3d76
+#define VPP_WRAP_OSD2_MATRIX_COEF15_25 0x3d77
+#define VPP_WRAP_OSD2_MATRIX_CLIP 0x3d78
+#define VPP_WRAP_OSD2_MATRIX_OFFSET0_1 0x3d79
+#define VPP_WRAP_OSD2_MATRIX_OFFSET2 0x3d7a
+#define VPP_WRAP_OSD2_MATRIX_PRE_OFFSET0_1 0x3d7b
+#define VPP_WRAP_OSD2_MATRIX_PRE_OFFSET2 0x3d7c
+#define VPP_WRAP_OSD2_MATRIX_EN_CTRL 0x3d7d
+
+#define VPP_WRAP_OSD3_MATRIX_COEF00_01 0x3db0
+#define VPP_WRAP_OSD3_MATRIX_COEF02_10 0x3db1
+#define VPP_WRAP_OSD3_MATRIX_COEF11_12 0x3db2
+#define VPP_WRAP_OSD3_MATRIX_COEF20_21 0x3db3
+#define VPP_WRAP_OSD3_MATRIX_COEF22 0x3db4
+#define VPP_WRAP_OSD3_MATRIX_COEF13_14 0x3db5
+#define VPP_WRAP_OSD3_MATRIX_COEF23_24 0x3db6
+#define VPP_WRAP_OSD3_MATRIX_COEF15_25 0x3db7
+#define VPP_WRAP_OSD3_MATRIX_CLIP 0x3db8
+#define VPP_WRAP_OSD3_MATRIX_OFFSET0_1 0x3db9
+#define VPP_WRAP_OSD3_MATRIX_OFFSET2 0x3dba
+#define VPP_WRAP_OSD3_MATRIX_PRE_OFFSET0_1 0x3dbb
+#define VPP_WRAP_OSD3_MATRIX_PRE_OFFSET2 0x3dbc
+#define VPP_WRAP_OSD3_MATRIX_EN_CTRL 0x3dbd
+
+/* osd2 scaler */
+#define OSD2_VSC_PHASE_STEP 0x3d00
+#define OSD2_VSC_INI_PHASE 0x3d01
+#define OSD2_VSC_CTRL0 0x3d02
+#define OSD2_HSC_PHASE_STEP 0x3d03
+#define OSD2_HSC_INI_PHASE 0x3d04
+#define OSD2_HSC_CTRL0 0x3d05
+#define OSD2_HSC_INI_PAT_CTRL 0x3d06
+#define OSD2_SC_DUMMY_DATA 0x3d07
+#define OSD2_SC_CTRL0 0x3d08
+#define OSD2_SCI_WH_M1 0x3d09
+#define OSD2_SCO_H_START_END 0x3d0a
+#define OSD2_SCO_V_START_END 0x3d0b
+#define OSD2_SCALE_COEF_IDX 0x3d18
+#define OSD2_SCALE_COEF 0x3d19
+
+/* osd34 scaler */
+#define OSD34_SCALE_COEF_IDX 0x3d1e
+#define OSD34_SCALE_COEF 0x3d1f
+#define OSD34_VSC_PHASE_STEP 0x3d20
+#define OSD34_VSC_INI_PHASE 0x3d21
+#define OSD34_VSC_CTRL0 0x3d22
+#define OSD34_HSC_PHASE_STEP 0x3d23
+#define OSD34_HSC_INI_PHASE 0x3d24
+#define OSD34_HSC_CTRL0 0x3d25
+#define OSD34_HSC_INI_PAT_CTRL 0x3d26
+#define OSD34_SC_DUMMY_DATA 0x3d27
+#define OSD34_SC_CTRL0 0x3d28
+#define OSD34_SCI_WH_M1 0x3d29
+#define OSD34_SCO_H_START_END 0x3d2a
+#define OSD34_SCO_V_START_END 0x3d2b
 /* viu2 */
 #define VIU2_ADDR_START 0x1e00
 #define VIU2_ADDR_END 0x1eff
@@ -1400,4 +1520,131 @@
 #define OSDSR_YBIC_VCOEF0 0x3149
 #define OSDSR_CBIC_VCOEF0 0x314a
 
+/* osd afbcd on gxtvbb */
+#define OSD1_AFBCD_ENABLE 0x31a0
+#define OSD1_AFBCD_MODE 0x31a1
+#define OSD1_AFBCD_SIZE_IN 0x31a2
+#define OSD1_AFBCD_HDR_PTR 0x31a3
+#define OSD1_AFBCD_FRAME_PTR 0x31a4
+#define OSD1_AFBCD_CHROMA_PTR 0x31a5
+#define OSD1_AFBCD_CONV_CTRL 0x31a6
+#define OSD1_AFBCD_STATUS 0x31a8
+#define OSD1_AFBCD_PIXEL_HSCOPE 0x31a9
+#define OSD1_AFBCD_PIXEL_VSCOPE 0x31aa
+#define VIU_MISC_CTRL1 0x1a07
+
+/* add for gxm and 962e dv core2 */
+#define DOLBY_CORE2A_SWAP_CTRL1	0x3434
+#define DOLBY_CORE2A_SWAP_CTRL2	0x3435
+
+/* osd afbc on g12a */
+#define VPU_MAFBC_BLOCK_ID 0x3a00
+#define VPU_MAFBC_IRQ_RAW_STATUS 0x3a01
+#define VPU_MAFBC_IRQ_CLEAR 0x3a02
+#define VPU_MAFBC_IRQ_MASK 0x3a03
+#define VPU_MAFBC_IRQ_STATUS 0x3a04
+#define VPU_MAFBC_COMMAND 0x3a05
+#define VPU_MAFBC_STATUS 0x3a06
+#define VPU_MAFBC_SURFACE_CFG 0x3a07
+
+/* osd afbc on g12a */
+#define VPU_MAFBC_HEADER_BUF_ADDR_LOW_S0 0x3a10
+#define VPU_MAFBC_HEADER_BUF_ADDR_HIGH_S0 0x3a11
+#define VPU_MAFBC_FORMAT_SPECIFIER_S0 0x3a12
+#define VPU_MAFBC_BUFFER_WIDTH_S0 0x3a13
+#define VPU_MAFBC_BUFFER_HEIGHT_S0 0x3a14
+#define VPU_MAFBC_BOUNDING_BOX_X_START_S0 0x3a15
+#define VPU_MAFBC_BOUNDING_BOX_X_END_S0 0x3a16
+#define VPU_MAFBC_BOUNDING_BOX_Y_START_S0 0x3a17
+#define VPU_MAFBC_BOUNDING_BOX_Y_END_S0 0x3a18
+#define VPU_MAFBC_OUTPUT_BUF_ADDR_LOW_S0 0x3a19
+#define VPU_MAFBC_OUTPUT_BUF_ADDR_HIGH_S0 0x3a1a
+#define VPU_MAFBC_OUTPUT_BUF_STRIDE_S0 0x3a1b
+#define VPU_MAFBC_PREFETCH_CFG_S0 0x3a1c
+
+#define VPU_MAFBC_HEADER_BUF_ADDR_LOW_S1 0x3a30
+#define VPU_MAFBC_HEADER_BUF_ADDR_HIGH_S1 0x3a31
+#define VPU_MAFBC_FORMAT_SPECIFIER_S1 0x3a32
+#define VPU_MAFBC_BUFFER_WIDTH_S1 0x3a33
+#define VPU_MAFBC_BUFFER_HEIGHT_S1 0x3a34
+#define VPU_MAFBC_BOUNDING_BOX_X_START_S1 0x3a35
+#define VPU_MAFBC_BOUNDING_BOX_X_END_S1 0x3a36
+#define VPU_MAFBC_BOUNDING_BOX_Y_START_S1 0x3a37
+#define VPU_MAFBC_BOUNDING_BOX_Y_END_S1 0x3a38
+#define VPU_MAFBC_OUTPUT_BUF_ADDR_LOW_S1 0x3a39
+#define VPU_MAFBC_OUTPUT_BUF_ADDR_HIGH_S1 0x3a3a
+#define VPU_MAFBC_OUTPUT_BUF_STRIDE_S1 0x3a3b
+#define VPU_MAFBC_PREFETCH_CFG_S1 0x3a3c
+
+#define VPU_MAFBC_HEADER_BUF_ADDR_LOW_S2 0x3a50
+#define VPU_MAFBC_HEADER_BUF_ADDR_HIGH_S2 0x3a51
+#define VPU_MAFBC_FORMAT_SPECIFIER_S2 0x3a52
+#define VPU_MAFBC_BUFFER_WIDTH_S2 0x3a53
+#define VPU_MAFBC_BUFFER_HEIGHT_S2 0x3a54
+#define VPU_MAFBC_BOUNDING_BOX_X_START_S2 0x3a55
+#define VPU_MAFBC_BOUNDING_BOX_X_END_S2 0x3a56
+#define VPU_MAFBC_BOUNDING_BOX_Y_START_S2 0x3a57
+#define VPU_MAFBC_BOUNDING_BOX_Y_END_S2 0x3a58
+#define VPU_MAFBC_OUTPUT_BUF_ADDR_LOW_S2 0x3a59
+#define VPU_MAFBC_OUTPUT_BUF_ADDR_HIGH_S2 0x3a5a
+#define VPU_MAFBC_OUTPUT_BUF_STRIDE_S2 0x3a5b
+#define VPU_MAFBC_PREFETCH_CFG_S2 0x3a5c
+
+#define VPU_MAFBC_HEADER_BUF_ADDR_LOW_S3 0x3a70
+#define VPU_MAFBC_HEADER_BUF_ADDR_HIGH_S3 0x3a71
+#define VPU_MAFBC_FORMAT_SPECIFIER_S3 0x3a72
+#define VPU_MAFBC_BUFFER_WIDTH_S3 0x3a73
+#define VPU_MAFBC_BUFFER_HEIGHT_S3 0x3a74
+#define VPU_MAFBC_BOUNDING_BOX_X_START_S3 0x3a75
+#define VPU_MAFBC_BOUNDING_BOX_X_END_S3 0x3a76
+#define VPU_MAFBC_BOUNDING_BOX_Y_START_S3 0x3a77
+#define VPU_MAFBC_BOUNDING_BOX_Y_END_S3 0x3a78
+#define VPU_MAFBC_OUTPUT_BUF_ADDR_LOW_S3 0x3a79
+#define VPU_MAFBC_OUTPUT_BUF_ADDR_HIGH_S3 0x3a7a
+#define VPU_MAFBC_OUTPUT_BUF_STRIDE_S3 0x3a7b
+#define VPU_MAFBC_PREFETCH_CFG_S3 0x3a7c
+
+#define DOLBY_PATH_CTRL 0x1a0c
+#define OSD_PATH_MISC_CTRL 0x1a0e
+#define MALI_AFBCD_TOP_CTRL 0x1a0f
+
+#define VIU_OSD_BLEND_CTRL 0x39b0
+#define VIU_OSD_BLEND_CTRL1 0x39c0
+#define VIU_OSD_BLEND_DIN0_SCOPE_H 0x39b1
+#define VIU_OSD_BLEND_DIN0_SCOPE_V 0x39b2
+#define VIU_OSD_BLEND_DIN1_SCOPE_H 0x39b3
+#define VIU_OSD_BLEND_DIN1_SCOPE_V 0x39b4
+#define VIU_OSD_BLEND_DIN2_SCOPE_H 0x39b5
+#define VIU_OSD_BLEND_DIN2_SCOPE_V 0x39b6
+#define VIU_OSD_BLEND_DIN3_SCOPE_H 0x39b7
+#define VIU_OSD_BLEND_DIN3_SCOPE_V 0x39b8
+#define VIU_OSD_BLEND_DUMMY_DATA0 0x39b9
+#define VIU_OSD_BLEND_DUMMY_ALPHA 0x39ba
+#define VIU_OSD_BLEND_BLEND0_SIZE 0x39bb
+#define VIU_OSD_BLEND_BLEND1_SIZE 0x39bc
+#define VIU_OSD_BLEND_RO_CURRENT_XY 0x39bf
+
+#define VPP_OUT_H_V_SIZE 0x1da5
+
+#define VPP_VD2_HDR_IN_SIZE 0x1df0
+#define VPP_OSD1_IN_SIZE 0x1df1
+#define VPP_GCLK_CTRL2 0x1df2
+#define VD2_PPS_DUMMY_DATA 0x1df4
+#define VPP_OSD1_BLD_H_SCOPE 0x1df5
+#define VPP_OSD1_BLD_V_SCOPE 0x1df6
+#define VPP_OSD2_BLD_H_SCOPE 0x1df7
+#define VPP_OSD2_BLD_V_SCOPE 0x1df8
+#define VPP_WRBAK_CTRL 0x1df9
+#define VPP_SLEEP_CTRL 0x1dfa
+#define VD1_BLEND_SRC_CTRL 0x1dfb
+#define VD2_BLEND_SRC_CTRL 0x1dfc
+#define OSD1_BLEND_SRC_CTRL 0x1dfd
+#define OSD2_BLEND_SRC_CTRL 0x1dfe
+
+#define VPP_POST_BLEND_BLEND_DUMMY_DATA 0x3968
+#define VPP_POST_BLEND_DUMMY_ALPHA 0x3969
+#define VPP_RDARB_MODE 0x3978
+#define VPP_RDARB_REQEN_SLV 0x3979
+#define VPU_RDARB_MODE_L2C1 0x279d
+
 #endif /* __MESON_REGISTERS_H */
diff --git a/drivers/gpu/drm/meson/meson_vclk.c b/drivers/gpu/drm/meson/meson_vclk.c
index f6ba35a405f8..b39034745444 100644
--- a/drivers/gpu/drm/meson/meson_vclk.c
+++ b/drivers/gpu/drm/meson/meson_vclk.c
@@ -113,9 +113,12 @@
 #define HHI_HDMI_PLL_CNTL4	0x32C /* 0xcb offset in data sheet */
 #define HHI_HDMI_PLL_CNTL5	0x330 /* 0xcc offset in data sheet */
 #define HHI_HDMI_PLL_CNTL6	0x334 /* 0xcd offset in data sheet */
+#define HHI_HDMI_PLL_CNTL7	0x338 /* 0xce offset in data sheet */
 
 #define HDMI_PLL_RESET		BIT(28)
+#define HDMI_PLL_RESET_G12A	BIT(29)
 #define HDMI_PLL_LOCK		BIT(31)
+#define HDMI_PLL_LOCK_G12A	(3 << 30)
 
 #define FREQ_1000_1001(_freq)	DIV_ROUND_CLOSEST(_freq * 1000, 1001)
 
@@ -257,6 +260,10 @@ static void meson_venci_cvbs_clock_config(struct meson_drm *priv)
 		regmap_write(priv->hhi, HHI_HDMI_PLL_CNTL5, 0x71486980);
 		regmap_write(priv->hhi, HHI_HDMI_PLL_CNTL6, 0x00000e55);
 		regmap_write(priv->hhi, HHI_HDMI_PLL_CNTL, 0x4800023d);
+
+		/* Poll for lock bit */
+		regmap_read_poll_timeout(priv->hhi, HHI_HDMI_PLL_CNTL, val,
+					 (val & HDMI_PLL_LOCK), 10, 0);
 	} else if (meson_vpu_is_compatible(priv, "amlogic,meson-gxm-vpu") ||
 		   meson_vpu_is_compatible(priv, "amlogic,meson-gxl-vpu")) {
 		regmap_write(priv->hhi, HHI_HDMI_PLL_CNTL, 0x4000027b);
@@ -271,11 +278,26 @@ static void meson_venci_cvbs_clock_config(struct meson_drm *priv)
 					HDMI_PLL_RESET, HDMI_PLL_RESET);
 		regmap_update_bits(priv->hhi, HHI_HDMI_PLL_CNTL,
 					HDMI_PLL_RESET, 0);
-	}
 
-	/* Poll for lock bit */
-	regmap_read_poll_timeout(priv->hhi, HHI_HDMI_PLL_CNTL, val,
-				 (val & HDMI_PLL_LOCK), 10, 0);
+		/* Poll for lock bit */
+		regmap_read_poll_timeout(priv->hhi, HHI_HDMI_PLL_CNTL, val,
+					 (val & HDMI_PLL_LOCK), 10, 0);
+	} else if (meson_vpu_is_compatible(priv, "amlogic,meson-g12a-vpu")) {
+		regmap_write(priv->hhi, HHI_HDMI_PLL_CNTL, 0x1a0504f7);
+		regmap_write(priv->hhi, HHI_HDMI_PLL_CNTL2, 0x00010000);
+		regmap_write(priv->hhi, HHI_HDMI_PLL_CNTL3, 0x00000000);
+		regmap_write(priv->hhi, HHI_HDMI_PLL_CNTL4, 0x6a28dc00);
+		regmap_write(priv->hhi, HHI_HDMI_PLL_CNTL5, 0x65771290);
+		regmap_write(priv->hhi, HHI_HDMI_PLL_CNTL6, 0x39272000);
+		regmap_write(priv->hhi, HHI_HDMI_PLL_CNTL7, 0x56540000);
+		regmap_write(priv->hhi, HHI_HDMI_PLL_CNTL, 0x3a0504f7);
+		regmap_write(priv->hhi, HHI_HDMI_PLL_CNTL, 0x1a0504f7);
+
+		/* Poll for lock bit */
+		regmap_read_poll_timeout(priv->hhi, HHI_HDMI_PLL_CNTL, val,
+			((val & HDMI_PLL_LOCK_G12A) == HDMI_PLL_LOCK_G12A),
+			10, 0);
+	}
 
 	/* Disable VCLK2 */
 	regmap_update_bits(priv->hhi, HHI_VIID_CLK_CNTL, VCLK2_EN, 0);
@@ -288,8 +310,13 @@ static void meson_venci_cvbs_clock_config(struct meson_drm *priv)
 				VCLK2_DIV_MASK, (55 - 1));
 
 	/* select vid_pll for vclk2 */
-	regmap_update_bits(priv->hhi, HHI_VIID_CLK_CNTL,
-				VCLK2_SEL_MASK, (4 << VCLK2_SEL_SHIFT));
+	if (meson_vpu_is_compatible(priv, "amlogic,meson-g12a-vpu"))
+		regmap_update_bits(priv->hhi, HHI_VIID_CLK_CNTL,
+					VCLK2_SEL_MASK, (0 << VCLK2_SEL_SHIFT));
+	else
+		regmap_update_bits(priv->hhi, HHI_VIID_CLK_CNTL,
+					VCLK2_SEL_MASK, (4 << VCLK2_SEL_SHIFT));
+
 	/* enable vclk2 gate */
 	regmap_update_bits(priv->hhi, HHI_VIID_CLK_CNTL, VCLK2_EN, VCLK2_EN);
 
@@ -396,8 +423,8 @@ struct meson_vclk_params {
 	},
 	[MESON_VCLK_HDMI_297000] = {
 		.pixel_freq = 297000,
-		.pll_base_freq = 2970000,
-		.pll_od1 = 1,
+		.pll_base_freq = 5940000,
+		.pll_od1 = 2,
 		.pll_od2 = 1,
 		.pll_od3 = 1,
 		.vid_pll_div = VID_PLL_DIV_5,
@@ -476,32 +503,80 @@ void meson_hdmi_pll_set_params(struct meson_drm *priv, unsigned int m,
 		/* Poll for lock bit */
 		regmap_read_poll_timeout(priv->hhi, HHI_HDMI_PLL_CNTL, val,
 				(val & HDMI_PLL_LOCK), 10, 0);
+	} else if (meson_vpu_is_compatible(priv, "amlogic,meson-g12a-vpu")) {
+		regmap_write(priv->hhi, HHI_HDMI_PLL_CNTL, 0x0b3a0400 | m);
+
+		/* Enable and reset */
+		regmap_update_bits(priv->hhi, HHI_HDMI_PLL_CNTL,
+				   0x3 << 28, 0x3 << 28);
+
+		regmap_write(priv->hhi, HHI_HDMI_PLL_CNTL2, frac);
+		regmap_write(priv->hhi, HHI_HDMI_PLL_CNTL3, 0x00000000);
+
+		/* G12A HDMI PLL Needs specific parameters for 5.4GHz */
+		if (m >= 0xf7) {
+			regmap_write(priv->hhi, HHI_HDMI_PLL_CNTL4, 0xea68dc00);
+			regmap_write(priv->hhi, HHI_HDMI_PLL_CNTL5, 0x65771290);
+			regmap_write(priv->hhi, HHI_HDMI_PLL_CNTL6, 0x39272000);
+			regmap_write(priv->hhi, HHI_HDMI_PLL_CNTL7, 0x55540000);
+		} else {
+			regmap_write(priv->hhi, HHI_HDMI_PLL_CNTL4, 0x0a691c00);
+			regmap_write(priv->hhi, HHI_HDMI_PLL_CNTL5, 0x33771290);
+			regmap_write(priv->hhi, HHI_HDMI_PLL_CNTL6, 0x39270000);
+			regmap_write(priv->hhi, HHI_HDMI_PLL_CNTL7, 0x50540000);
+		}
+
+		do {
+			/* Reset PLL */
+			regmap_update_bits(priv->hhi, HHI_HDMI_PLL_CNTL,
+					HDMI_PLL_RESET_G12A, HDMI_PLL_RESET_G12A);
+
+			/* UN-Reset PLL */
+			regmap_update_bits(priv->hhi, HHI_HDMI_PLL_CNTL,
+					HDMI_PLL_RESET_G12A, 0);
+
+			/* Poll for lock bits */
+			if (!regmap_read_poll_timeout(priv->hhi,
+						      HHI_HDMI_PLL_CNTL, val,
+						      ((val & HDMI_PLL_LOCK_G12A)
+						        == HDMI_PLL_LOCK_G12A),
+						      10, 100))
+				break;
+		} while(1);
 	}
 
 	if (meson_vpu_is_compatible(priv, "amlogic,meson-gxbb-vpu"))
 		regmap_update_bits(priv->hhi, HHI_HDMI_PLL_CNTL2,
 				3 << 16, pll_od_to_reg(od1) << 16);
 	else if (meson_vpu_is_compatible(priv, "amlogic,meson-gxm-vpu") ||
-			meson_vpu_is_compatible(priv, "amlogic,meson-gxl-vpu"))
+		 meson_vpu_is_compatible(priv, "amlogic,meson-gxl-vpu"))
 		regmap_update_bits(priv->hhi, HHI_HDMI_PLL_CNTL3,
 				3 << 21, pll_od_to_reg(od1) << 21);
+	else if (meson_vpu_is_compatible(priv, "amlogic,meson-g12a-vpu"))
+		regmap_update_bits(priv->hhi, HHI_HDMI_PLL_CNTL,
+				3 << 16, pll_od_to_reg(od1) << 16);
 
 	if (meson_vpu_is_compatible(priv, "amlogic,meson-gxbb-vpu"))
 		regmap_update_bits(priv->hhi, HHI_HDMI_PLL_CNTL2,
 				3 << 22, pll_od_to_reg(od2) << 22);
 	else if (meson_vpu_is_compatible(priv, "amlogic,meson-gxm-vpu") ||
-			meson_vpu_is_compatible(priv, "amlogic,meson-gxl-vpu"))
+		 meson_vpu_is_compatible(priv, "amlogic,meson-gxl-vpu"))
 		regmap_update_bits(priv->hhi, HHI_HDMI_PLL_CNTL3,
 				3 << 23, pll_od_to_reg(od2) << 23);
+	else if (meson_vpu_is_compatible(priv, "amlogic,meson-g12a-vpu"))
+		regmap_update_bits(priv->hhi, HHI_HDMI_PLL_CNTL,
+				3 << 18, pll_od_to_reg(od2) << 18);
 
 	if (meson_vpu_is_compatible(priv, "amlogic,meson-gxbb-vpu"))
 		regmap_update_bits(priv->hhi, HHI_HDMI_PLL_CNTL2,
 				3 << 18, pll_od_to_reg(od3) << 18);
 	else if (meson_vpu_is_compatible(priv, "amlogic,meson-gxm-vpu") ||
-			meson_vpu_is_compatible(priv, "amlogic,meson-gxl-vpu"))
+		 meson_vpu_is_compatible(priv, "amlogic,meson-gxl-vpu"))
 		regmap_update_bits(priv->hhi, HHI_HDMI_PLL_CNTL3,
 				3 << 19, pll_od_to_reg(od3) << 19);
-
+	else if (meson_vpu_is_compatible(priv, "amlogic,meson-g12a-vpu"))
+		regmap_update_bits(priv->hhi, HHI_HDMI_PLL_CNTL,
+				3 << 20, pll_od_to_reg(od3) << 20);
 }
 
 #define XTAL_FREQ 24000
@@ -518,6 +593,7 @@ static unsigned int meson_hdmi_pll_get_m(struct meson_drm *priv,
 
 #define HDMI_FRAC_MAX_GXBB	4096
 #define HDMI_FRAC_MAX_GXL	1024
+#define HDMI_FRAC_MAX_G12A	131072
 
 static unsigned int meson_hdmi_pll_get_frac(struct meson_drm *priv,
 					    unsigned int m,
@@ -534,6 +610,9 @@ static unsigned int meson_hdmi_pll_get_frac(struct meson_drm *priv,
 		parent_freq *= 2;
 	}
 
+	if (meson_vpu_is_compatible(priv, "amlogic,meson-g12a-vpu"))
+		frac_max = HDMI_FRAC_MAX_G12A;
+
 	/* We can have a perfect match !*/
 	if (pll_freq / m == parent_freq &&
 	    pll_freq % m == 0)
@@ -559,7 +638,8 @@ static bool meson_hdmi_pll_validate_params(struct meson_drm *priv,
 		if (frac >= HDMI_FRAC_MAX_GXBB)
 			return false;
 	} else if (meson_vpu_is_compatible(priv, "amlogic,meson-gxm-vpu") ||
-		   meson_vpu_is_compatible(priv, "amlogic,meson-gxl-vpu")) {
+		   meson_vpu_is_compatible(priv, "amlogic,meson-gxl-vpu") ||
+		   meson_vpu_is_compatible(priv, "amlogic,meson-g12a-vpu")) {
 		/* Empiric supported min/max dividers */
 		if (m < 106 || m > 247)
 			return false;
@@ -714,6 +794,23 @@ static void meson_vclk_set(struct meson_drm *priv, unsigned int pll_base_freq,
 		}
 
 		meson_hdmi_pll_set_params(priv, m, frac, od1, od2, od3);
+	} else if (meson_vpu_is_compatible(priv, "amlogic,meson-g12a-vpu")) {
+		switch (pll_base_freq) {
+		case 2970000:
+			m = 0x7b;
+			frac = vic_alternate_clock ? 0x140b4 : 0x18000;
+			break;
+		case 4320000:
+			m = vic_alternate_clock ? 0xb3 : 0xb4;
+			frac = vic_alternate_clock ? 0x1a3ee : 0;
+			break;
+		case 5940000:
+			m = 0xf7;
+			frac = vic_alternate_clock ? 0x8148 : 0x10000;
+			break;
+		}
+
+		meson_hdmi_pll_set_params(priv, m, frac, od1, od2, od3);
 	}
 
 	/* Setup vid_pll divider */
diff --git a/drivers/gpu/drm/meson/meson_venc.c b/drivers/gpu/drm/meson/meson_venc.c
index 66d73a932d19..6faca7313339 100644
--- a/drivers/gpu/drm/meson/meson_venc.c
+++ b/drivers/gpu/drm/meson/meson_venc.c
@@ -73,7 +73,9 @@
 /* HHI Registers */
 #define HHI_GCLK_MPEG2		0x148 /* 0x52 offset in data sheet */
 #define HHI_VDAC_CNTL0		0x2F4 /* 0xbd offset in data sheet */
+#define HHI_VDAC_CNTL0_G12A	0x2EC /* 0xbd offset in data sheet */
 #define HHI_VDAC_CNTL1		0x2F8 /* 0xbe offset in data sheet */
+#define HHI_VDAC_CNTL1_G12A	0x2F0 /* 0xbe offset in data sheet */
 #define HHI_HDMI_PHY_CNTL0	0x3a0 /* 0xe8 offset in data sheet */
 
 struct meson_cvbs_enci_mode meson_cvbs_enci_pal = {
@@ -1675,8 +1677,13 @@ void meson_venc_disable_vsync(struct meson_drm *priv)
 void meson_venc_init(struct meson_drm *priv)
 {
 	/* Disable CVBS VDAC */
-	regmap_write(priv->hhi, HHI_VDAC_CNTL0, 0);
-	regmap_write(priv->hhi, HHI_VDAC_CNTL1, 8);
+	if (meson_vpu_is_compatible(priv, "amlogic,meson-g12a-vpu")) {
+		regmap_write(priv->hhi, HHI_VDAC_CNTL0_G12A, 0);
+		regmap_write(priv->hhi, HHI_VDAC_CNTL1_G12A, 8);
+	} else {
+		regmap_write(priv->hhi, HHI_VDAC_CNTL0, 0);
+		regmap_write(priv->hhi, HHI_VDAC_CNTL1, 8);
+	}
 
 	/* Power Down Dacs */
 	writel_relaxed(0xff, priv->io_base + _REG(VENC_VDAC_SETTING));
diff --git a/drivers/gpu/drm/meson/meson_venc_cvbs.c b/drivers/gpu/drm/meson/meson_venc_cvbs.c
index d622d817b6df..2c5341c881c4 100644
--- a/drivers/gpu/drm/meson/meson_venc_cvbs.c
+++ b/drivers/gpu/drm/meson/meson_venc_cvbs.c
@@ -37,7 +37,9 @@
 
 /* HHI VDAC Registers */
 #define HHI_VDAC_CNTL0		0x2F4 /* 0xbd offset in data sheet */
+#define HHI_VDAC_CNTL0_G12A	0x2EC /* 0xbd offset in data sheet */
 #define HHI_VDAC_CNTL1		0x2F8 /* 0xbe offset in data sheet */
+#define HHI_VDAC_CNTL1_G12A	0x2F0 /* 0xbe offset in data sheet */
 
 struct meson_venc_cvbs {
 	struct drm_encoder	encoder;
@@ -166,8 +168,13 @@ static void meson_venc_cvbs_encoder_disable(struct drm_encoder *encoder)
 	struct meson_drm *priv = meson_venc_cvbs->priv;
 
 	/* Disable CVBS VDAC */
-	regmap_write(priv->hhi, HHI_VDAC_CNTL0, 0);
-	regmap_write(priv->hhi, HHI_VDAC_CNTL1, 8);
+	if (meson_vpu_is_compatible(priv, "amlogic,meson-g12a-vpu")) {
+		regmap_write(priv->hhi, HHI_VDAC_CNTL0_G12A, 0);
+		regmap_write(priv->hhi, HHI_VDAC_CNTL1_G12A, 0);
+	} else {
+		regmap_write(priv->hhi, HHI_VDAC_CNTL0, 0);
+		regmap_write(priv->hhi, HHI_VDAC_CNTL1, 8);
+	}
 }
 
 static void meson_venc_cvbs_encoder_enable(struct drm_encoder *encoder)
@@ -179,13 +186,17 @@ static void meson_venc_cvbs_encoder_enable(struct drm_encoder *encoder)
 	/* VDAC0 source is not from ATV */
 	writel_bits_relaxed(BIT(5), 0, priv->io_base + _REG(VENC_VDAC_DACSEL0));
 
-	if (meson_vpu_is_compatible(priv, "amlogic,meson-gxbb-vpu"))
+	if (meson_vpu_is_compatible(priv, "amlogic,meson-gxbb-vpu")) {
 		regmap_write(priv->hhi, HHI_VDAC_CNTL0, 1);
-	else if (meson_vpu_is_compatible(priv, "amlogic,meson-gxm-vpu") ||
-		 meson_vpu_is_compatible(priv, "amlogic,meson-gxl-vpu"))
+		regmap_write(priv->hhi, HHI_VDAC_CNTL1, 0);
+	} else if (meson_vpu_is_compatible(priv, "amlogic,meson-gxm-vpu") ||
+		 meson_vpu_is_compatible(priv, "amlogic,meson-gxl-vpu")) {
 		regmap_write(priv->hhi, HHI_VDAC_CNTL0, 0xf0001);
-
-	regmap_write(priv->hhi, HHI_VDAC_CNTL1, 0);
+		regmap_write(priv->hhi, HHI_VDAC_CNTL1, 0);
+	} else if (meson_vpu_is_compatible(priv, "amlogic,meson-g12a-vpu")) {
+		regmap_write(priv->hhi, HHI_VDAC_CNTL0_G12A, 0x906001);
+		regmap_write(priv->hhi, HHI_VDAC_CNTL1_G12A, 0);
+	}
 }
 
 static void meson_venc_cvbs_encoder_mode_set(struct drm_encoder *encoder,
diff --git a/drivers/gpu/drm/meson/meson_viu.c b/drivers/gpu/drm/meson/meson_viu.c
index e46e05f50bad..b59072342cae 100644
--- a/drivers/gpu/drm/meson/meson_viu.c
+++ b/drivers/gpu/drm/meson/meson_viu.c
@@ -25,7 +25,6 @@
 #include "meson_viu.h"
 #include "meson_vpp.h"
 #include "meson_venc.h"
-#include "meson_canvas.h"
 #include "meson_registers.h"
 
 /**
@@ -91,8 +90,36 @@ static int eotf_bypass_coeff[EOTF_COEFF_SIZE] = {
 	EOTF_COEFF_RIGHTSHIFT /* right shift */
 };
 
-void meson_viu_set_osd_matrix(struct meson_drm *priv,
-			      enum viu_matrix_sel_e m_select,
+static void meson_viu_set_g12a_osd1_matrix(struct meson_drm *priv,
+					   int *m, bool csc_on)
+{
+	/* VPP WRAP OSD1 matrix */
+	writel(((m[0] & 0xfff) << 16) | (m[1] & 0xfff),
+		priv->io_base + _REG(VPP_WRAP_OSD1_MATRIX_PRE_OFFSET0_1));
+	writel(m[2] & 0xfff,
+		priv->io_base + _REG(VPP_WRAP_OSD1_MATRIX_PRE_OFFSET2));
+	writel(((m[3] & 0x1fff) << 16) | (m[4] & 0x1fff),
+		priv->io_base + _REG(VPP_WRAP_OSD1_MATRIX_COEF00_01));
+	writel(((m[5] & 0x1fff) << 16) | (m[6] & 0x1fff),
+		priv->io_base + _REG(VPP_WRAP_OSD1_MATRIX_COEF02_10));
+	writel(((m[7] & 0x1fff) << 16) | (m[8] & 0x1fff),
+		priv->io_base + _REG(VPP_WRAP_OSD1_MATRIX_COEF11_12));
+	writel(((m[9] & 0x1fff) << 16) | (m[10] & 0x1fff),
+		priv->io_base + _REG(VPP_WRAP_OSD1_MATRIX_COEF20_21));
+	writel((m[11] & 0x1fff) << 16,
+		priv->io_base +	_REG(VPP_WRAP_OSD1_MATRIX_COEF22));
+
+	writel(((m[18] & 0xfff) << 16) | (m[19] & 0xfff),
+		priv->io_base + _REG(VPP_WRAP_OSD1_MATRIX_OFFSET0_1));
+	writel(m[20] & 0xfff,
+		priv->io_base + _REG(VPP_WRAP_OSD1_MATRIX_OFFSET2));
+
+	writel_bits_relaxed(BIT(0), csc_on ? BIT(0) : 0,
+		priv->io_base + _REG(VPP_WRAP_OSD1_MATRIX_EN_CTRL));
+}
+
+static void meson_viu_set_osd_matrix(struct meson_drm *priv,
+				     enum viu_matrix_sel_e m_select,
 			      int *m, bool csc_on)
 {
 	if (m_select == VIU_MATRIX_OSD) {
@@ -160,10 +187,10 @@ void meson_viu_set_osd_matrix(struct meson_drm *priv,
 #define OSD_EOTF_LUT_SIZE 33
 #define OSD_OETF_LUT_SIZE 41
 
-void meson_viu_set_osd_lut(struct meson_drm *priv, enum viu_lut_sel_e lut_sel,
-			   unsigned int *r_map, unsigned int *g_map,
-			   unsigned int *b_map,
-			   bool csc_on)
+static void
+meson_viu_set_osd_lut(struct meson_drm *priv, enum viu_lut_sel_e lut_sel,
+		      unsigned int *r_map, unsigned int *g_map,
+		      unsigned int *b_map, bool csc_on)
 {
 	unsigned int addr_port;
 	unsigned int data_port;
@@ -337,14 +364,24 @@ void meson_viu_init(struct meson_drm *priv)
 	if (meson_vpu_is_compatible(priv, "amlogic,meson-gxm-vpu") ||
 	    meson_vpu_is_compatible(priv, "amlogic,meson-gxl-vpu"))
 		meson_viu_load_matrix(priv);
+	else if (meson_vpu_is_compatible(priv, "amlogic,meson-g12a-vpu"))
+		meson_viu_set_g12a_osd1_matrix(priv, RGB709_to_YUV709l_coeff,
+					       true);
 
 	/* Initialize OSD1 fifo control register */
 	reg = BIT(0) |	/* Urgent DDR request priority */
-	      (4 << 5) | /* hold_fifo_lines */
-	      (3 << 10) | /* burst length 64 */
-	      (32 << 12) | /* fifo_depth_val: 32*8=256 */
-	      (2 << 22) | /* 4 words in 1 burst */
-	      (2 << 24);
+	      (4 << 5); /* hold_fifo_lines */
+	if (meson_vpu_is_compatible(priv, "amlogic,meson-g12a-vpu"))
+		reg |= (1 << 10) | /* burst length 32 */
+		       (32 << 12) | /* fifo_depth_val: 32*8=256 */
+		       (2 << 22) | /* 4 words in 1 burst */
+		       (2 << 24) |
+		       (1 << 31);
+	else
+		reg |= (3 << 10) | /* burst length 64 */
+		       (32 << 12) | /* fifo_depth_val: 32*8=256 */
+		       (2 << 22) | /* 4 words in 1 burst */
+		       (2 << 24);
 	writel_relaxed(reg, priv->io_base + _REG(VIU_OSD1_FIFO_CTRL_STAT));
 	writel_relaxed(reg, priv->io_base + _REG(VIU_OSD2_FIFO_CTRL_STAT));
 
@@ -370,6 +407,30 @@ void meson_viu_init(struct meson_drm *priv)
 	writel_relaxed(0x00FF00C0,
 			priv->io_base + _REG(VD2_IF0_LUMA_FIFO_SIZE));
 
+	if (meson_vpu_is_compatible(priv, "amlogic,meson-g12a-vpu")) {
+		writel_relaxed(4 << 29 |
+				1 << 27 |
+				1 << 26 | /* blend_din0 input to blend0 */
+				1 << 25 | /* blend1_dout to blend2 */
+				1 << 24 | /* blend1_din3 input to blend1 */
+				1 << 20 |
+				0 << 16 |
+				1,
+				priv->io_base + _REG(VIU_OSD_BLEND_CTRL));
+		writel_relaxed(3 << 8 |
+				1 << 20,
+				priv->io_base + _REG(OSD1_BLEND_SRC_CTRL));
+		writel_relaxed(1 << 20,
+				priv->io_base + _REG(OSD2_BLEND_SRC_CTRL));
+		writel_relaxed(0, priv->io_base + _REG(VD1_BLEND_SRC_CTRL));
+		writel_relaxed(0, priv->io_base + _REG(VD2_BLEND_SRC_CTRL));
+		writel_relaxed(0,
+				priv->io_base + _REG(VIU_OSD_BLEND_DUMMY_DATA0));
+		writel_relaxed(0,
+				priv->io_base + _REG(VIU_OSD_BLEND_DUMMY_ALPHA));
+		writel_bits_relaxed(0x3 << 2, 0x3 << 2,
+				priv->io_base + _REG(DOLBY_PATH_CTRL));
+	}
 
 	priv->viu.osd1_enabled = false;
 	priv->viu.osd1_commit = false;
diff --git a/drivers/gpu/drm/meson/meson_vpp.c b/drivers/gpu/drm/meson/meson_vpp.c
index f9efb431e953..8c52a3455ef4 100644
--- a/drivers/gpu/drm/meson/meson_vpp.c
+++ b/drivers/gpu/drm/meson/meson_vpp.c
@@ -112,32 +112,39 @@ void meson_vpp_init(struct meson_drm *priv)
 		writel_relaxed(0x20000, priv->io_base + _REG(VPP_DOLBY_CTRL));
 		writel_relaxed(0x1020080,
 				priv->io_base + _REG(VPP_DUMMY_DATA1));
-	}
+	} else if (meson_vpu_is_compatible(priv, "amlogic,meson-g12a-vpu"))
+		writel_relaxed(0xf, priv->io_base + _REG(DOLBY_PATH_CTRL));
 
 	/* Initialize vpu fifo control registers */
-	writel_relaxed(readl_relaxed(priv->io_base + _REG(VPP_OFIFO_SIZE)) |
-			0x77f, priv->io_base + _REG(VPP_OFIFO_SIZE));
+	if (meson_vpu_is_compatible(priv, "amlogic,meson-g12a-vpu"))
+		writel_relaxed(0xfff << 20 | 0x1000,
+			       priv->io_base + _REG(VPP_OFIFO_SIZE));
+	else
+		writel_relaxed(readl_relaxed(priv->io_base + _REG(VPP_OFIFO_SIZE)) |
+				0x77f, priv->io_base + _REG(VPP_OFIFO_SIZE));
 	writel_relaxed(0x08080808, priv->io_base + _REG(VPP_HOLD_LINES));
 
-	/* Turn off preblend */
-	writel_bits_relaxed(VPP_PREBLEND_ENABLE, 0,
-			    priv->io_base + _REG(VPP_MISC));
-
-	/* Turn off POSTBLEND */
-	writel_bits_relaxed(VPP_POSTBLEND_ENABLE, 0,
-			    priv->io_base + _REG(VPP_MISC));
-
-	/* Force all planes off */
-	writel_bits_relaxed(VPP_OSD1_POSTBLEND | VPP_OSD2_POSTBLEND |
-			    VPP_VD1_POSTBLEND | VPP_VD2_POSTBLEND |
-			    VPP_VD1_PREBLEND | VPP_VD2_PREBLEND, 0,
-			    priv->io_base + _REG(VPP_MISC));
-
-	/* Setup default VD settings */
-	writel_relaxed(4096,
-			priv->io_base + _REG(VPP_PREBLEND_VD1_H_START_END));
-	writel_relaxed(4096,
-			priv->io_base + _REG(VPP_BLEND_VD2_H_START_END));
+	if (!meson_vpu_is_compatible(priv, "amlogic,meson-g12a-vpu")) {
+		/* Turn off preblend */
+		writel_bits_relaxed(VPP_PREBLEND_ENABLE, 0,
+				    priv->io_base + _REG(VPP_MISC));
+
+		/* Turn off POSTBLEND */
+		writel_bits_relaxed(VPP_POSTBLEND_ENABLE, 0,
+				    priv->io_base + _REG(VPP_MISC));
+
+		/* Force all planes off */
+		writel_bits_relaxed(VPP_OSD1_POSTBLEND | VPP_OSD2_POSTBLEND |
+				    VPP_VD1_POSTBLEND | VPP_VD2_POSTBLEND |
+				    VPP_VD1_PREBLEND | VPP_VD2_PREBLEND, 0,
+				    priv->io_base + _REG(VPP_MISC));
+
+		/* Setup default VD settings */
+		writel_relaxed(4096,
+				priv->io_base + _REG(VPP_PREBLEND_VD1_H_START_END));
+		writel_relaxed(4096,
+				priv->io_base + _REG(VPP_BLEND_VD2_H_START_END));
+	}
 
 	/* Disable Scalers */
 	writel_relaxed(0, priv->io_base + _REG(VPP_OSD_SC_CTRL0));
diff --git a/drivers/gpu/drm/mgag200/mgag200_drv.h b/drivers/gpu/drm/mgag200/mgag200_drv.h
index 0aaedc554879..8c31e4422cae 100644
--- a/drivers/gpu/drm/mgag200/mgag200_drv.h
+++ b/drivers/gpu/drm/mgag200/mgag200_drv.h
@@ -113,7 +113,7 @@ struct mga_framebuffer {
 };
 
 struct mga_fbdev {
-	struct drm_fb_helper helper;
+	struct drm_fb_helper helper; /* must be first */
 	struct mga_framebuffer mfb;
 	void *sysram;
 	int size;
@@ -269,7 +269,6 @@ mgag200_dumb_mmap_offset(struct drm_file *file,
 struct mga_i2c_chan *mgag200_i2c_create(struct drm_device *dev);
 void mgag200_i2c_destroy(struct mga_i2c_chan *i2c);
 
-#define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT)
 void mgag200_ttm_placement(struct mgag200_bo *bo, int domain);
 
 static inline int mgag200_bo_reserve(struct mgag200_bo *bo, bool no_wait)
diff --git a/drivers/gpu/drm/mgag200/mgag200_fb.c b/drivers/gpu/drm/mgag200/mgag200_fb.c
index 6893934b26c0..5b7e64cac004 100644
--- a/drivers/gpu/drm/mgag200/mgag200_fb.c
+++ b/drivers/gpu/drm/mgag200/mgag200_fb.c
@@ -195,8 +195,6 @@ static int mgag200fb_create(struct drm_fb_helper *helper,
 		goto err_alloc_fbi;
 	}
 
-	info->par = mfbdev;
-
 	ret = mgag200_framebuffer_init(dev, &mfbdev->mfb, &mode_cmd, gobj);
 	if (ret)
 		goto err_alloc_fbi;
@@ -209,17 +207,13 @@ static int mgag200fb_create(struct drm_fb_helper *helper,
 	/* setup helper */
 	mfbdev->helper.fb = fb;
 
-	strcpy(info->fix.id, "mgadrmfb");
-
 	info->fbops = &mgag200fb_ops;
 
 	/* setup aperture base/size for vesafb takeover */
 	info->apertures->ranges[0].base = mdev->dev->mode_config.fb_base;
 	info->apertures->ranges[0].size = mdev->mc.vram_size;
 
-	drm_fb_helper_fill_fix(info, fb->pitches[0], fb->format->depth);
-	drm_fb_helper_fill_var(info, &mfbdev->helper, sizes->fb_width,
-			       sizes->fb_height);
+	drm_fb_helper_fill_info(info, &mfbdev->helper, sizes);
 
 	info->screen_base = sysram;
 	info->screen_size = size;
diff --git a/drivers/gpu/drm/mgag200/mgag200_ttm.c b/drivers/gpu/drm/mgag200/mgag200_ttm.c
index d96a9b32455e..bd42365a8aa8 100644
--- a/drivers/gpu/drm/mgag200/mgag200_ttm.c
+++ b/drivers/gpu/drm/mgag200/mgag200_ttm.c
@@ -178,7 +178,6 @@ int mgag200_mm_init(struct mga_device *mdev)
 	ret = ttm_bo_device_init(&mdev->ttm.bdev,
 				 &mgag200_bo_driver,
 				 dev->anon_inode->i_mapping,
-				 DRM_FILE_PAGE_OFFSET,
 				 true);
 	if (ret) {
 		DRM_ERROR("Error initialising bo driver; %d\n", ret);
@@ -345,13 +344,8 @@ int mgag200_bo_push_sysram(struct mgag200_bo *bo)
 
 int mgag200_mmap(struct file *filp, struct vm_area_struct *vma)
 {
-	struct drm_file *file_priv;
-	struct mga_device *mdev;
+	struct drm_file *file_priv = filp->private_data;
+	struct mga_device *mdev = file_priv->minor->dev->dev_private;
 
-	if (unlikely(vma->vm_pgoff < DRM_FILE_PAGE_OFFSET))
-		return -EINVAL;
-
-	file_priv = filp->private_data;
-	mdev = file_priv->minor->dev->dev_private;
 	return ttm_bo_mmap(filp, vma, &mdev->ttm.bdev);
 }
diff --git a/drivers/gpu/drm/msm/Kconfig b/drivers/gpu/drm/msm/Kconfig
index 78c9e5a5e793..9f2029eca39f 100644
--- a/drivers/gpu/drm/msm/Kconfig
+++ b/drivers/gpu/drm/msm/Kconfig
@@ -21,6 +21,11 @@ config DRM_MSM
 	help
 	  DRM/KMS driver for MSM/snapdragon.
 
+config DRM_MSM_GPU_STATE
+	bool
+	depends on DRM_MSM && (DEBUG_FS || DEV_COREDUMP)
+	default y
+
 config DRM_MSM_REGISTER_LOGGING
 	bool "MSM DRM register logging"
 	depends on DRM_MSM
diff --git a/drivers/gpu/drm/msm/Makefile b/drivers/gpu/drm/msm/Makefile
index 56a70c74af4e..7a05cbf2f820 100644
--- a/drivers/gpu/drm/msm/Makefile
+++ b/drivers/gpu/drm/msm/Makefile
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0
-ccflags-y := -Idrivers/gpu/drm/msm
-ccflags-y += -Idrivers/gpu/drm/msm/disp/dpu1
-ccflags-$(CONFIG_DRM_MSM_DSI) += -Idrivers/gpu/drm/msm/dsi
+ccflags-y := -I $(srctree)/$(src)
+ccflags-y += -I $(srctree)/$(src)/disp/dpu1
+ccflags-$(CONFIG_DRM_MSM_DSI) += -I $(srctree)/$(src)/dsi
 
 msm-y := \
 	adreno/adreno_device.o \
@@ -15,7 +15,6 @@ msm-y := \
 	adreno/a6xx_gpu.o \
 	adreno/a6xx_gmu.o \
 	adreno/a6xx_hfi.o \
-	adreno/a6xx_gpu_state.o \
 	hdmi/hdmi.o \
 	hdmi/hdmi_audio.o \
 	hdmi/hdmi_bridge.o \
@@ -96,6 +95,8 @@ msm-y := \
 
 msm-$(CONFIG_DEBUG_FS) += adreno/a5xx_debugfs.o
 
+msm-$(CONFIG_DRM_MSM_GPU_STATE)	+= adreno/a6xx_gpu_state.o
+
 msm-$(CONFIG_DRM_FBDEV_EMULATION) += msm_fbdev.o
 msm-$(CONFIG_COMMON_CLK) += disp/mdp4/mdp4_lvds_pll.o
 msm-$(CONFIG_COMMON_CLK) += hdmi/hdmi_pll_8960.o
diff --git a/drivers/gpu/drm/msm/adreno/a5xx_gpu.c b/drivers/gpu/drm/msm/adreno/a5xx_gpu.c
index d5f5e56422f5..e5fcefa49f19 100644
--- a/drivers/gpu/drm/msm/adreno/a5xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a5xx_gpu.c
@@ -15,9 +15,6 @@
 #include <linux/types.h>
 #include <linux/cpumask.h>
 #include <linux/qcom_scm.h>
-#include <linux/dma-mapping.h>
-#include <linux/of_address.h>
-#include <linux/soc/qcom/mdt_loader.h>
 #include <linux/pm_opp.h>
 #include <linux/nvmem-consumer.h>
 #include <linux/slab.h>
@@ -30,94 +27,6 @@ static void a5xx_dump(struct msm_gpu *gpu);
 
 #define GPU_PAS_ID 13
 
-static int zap_shader_load_mdt(struct msm_gpu *gpu, const char *fwname)
-{
-	struct device *dev = &gpu->pdev->dev;
-	const struct firmware *fw;
-	struct device_node *np;
-	struct resource r;
-	phys_addr_t mem_phys;
-	ssize_t mem_size;
-	void *mem_region = NULL;
-	int ret;
-
-	if (!IS_ENABLED(CONFIG_ARCH_QCOM))
-		return -EINVAL;
-
-	np = of_get_child_by_name(dev->of_node, "zap-shader");
-	if (!np)
-		return -ENODEV;
-
-	np = of_parse_phandle(np, "memory-region", 0);
-	if (!np)
-		return -EINVAL;
-
-	ret = of_address_to_resource(np, 0, &r);
-	if (ret)
-		return ret;
-
-	mem_phys = r.start;
-	mem_size = resource_size(&r);
-
-	/* Request the MDT file for the firmware */
-	fw = adreno_request_fw(to_adreno_gpu(gpu), fwname);
-	if (IS_ERR(fw)) {
-		DRM_DEV_ERROR(dev, "Unable to load %s\n", fwname);
-		return PTR_ERR(fw);
-	}
-
-	/* Figure out how much memory we need */
-	mem_size = qcom_mdt_get_size(fw);
-	if (mem_size < 0) {
-		ret = mem_size;
-		goto out;
-	}
-
-	/* Allocate memory for the firmware image */
-	mem_region = memremap(mem_phys, mem_size,  MEMREMAP_WC);
-	if (!mem_region) {
-		ret = -ENOMEM;
-		goto out;
-	}
-
-	/*
-	 * Load the rest of the MDT
-	 *
-	 * Note that we could be dealing with two different paths, since
-	 * with upstream linux-firmware it would be in a qcom/ subdir..
-	 * adreno_request_fw() handles this, but qcom_mdt_load() does
-	 * not.  But since we've already gotten thru adreno_request_fw()
-	 * we know which of the two cases it is:
-	 */
-	if (to_adreno_gpu(gpu)->fwloc == FW_LOCATION_LEGACY) {
-		ret = qcom_mdt_load(dev, fw, fwname, GPU_PAS_ID,
-				mem_region, mem_phys, mem_size, NULL);
-	} else {
-		char *newname;
-
-		newname = kasprintf(GFP_KERNEL, "qcom/%s", fwname);
-
-		ret = qcom_mdt_load(dev, fw, newname, GPU_PAS_ID,
-				mem_region, mem_phys, mem_size, NULL);
-		kfree(newname);
-	}
-	if (ret)
-		goto out;
-
-	/* Send the image to the secure world */
-	ret = qcom_scm_pas_auth_and_reset(GPU_PAS_ID);
-	if (ret)
-		DRM_DEV_ERROR(dev, "Unable to authorize the image\n");
-
-out:
-	if (mem_region)
-		memunmap(mem_region);
-
-	release_firmware(fw);
-
-	return ret;
-}
-
 static void a5xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
 {
 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
@@ -563,8 +472,6 @@ static int a5xx_zap_shader_resume(struct msm_gpu *gpu)
 static int a5xx_zap_shader_init(struct msm_gpu *gpu)
 {
 	static bool loaded;
-	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
-	struct platform_device *pdev = gpu->pdev;
 	int ret;
 
 	/*
@@ -574,23 +481,9 @@ static int a5xx_zap_shader_init(struct msm_gpu *gpu)
 	if (loaded)
 		return a5xx_zap_shader_resume(gpu);
 
-	/* We need SCM to be able to load the firmware */
-	if (!qcom_scm_is_available()) {
-		DRM_DEV_ERROR(&pdev->dev, "SCM is not available\n");
-		return -EPROBE_DEFER;
-	}
-
-	/* Each GPU has a target specific zap shader firmware name to use */
-	if (!adreno_gpu->info->zapfw) {
-		DRM_DEV_ERROR(&pdev->dev,
-			"Zap shader firmware file not specified for this target\n");
-		return -ENODEV;
-	}
-
-	ret = zap_shader_load_mdt(gpu, adreno_gpu->info->zapfw);
+	ret = adreno_zap_shader_load(gpu, GPU_PAS_ID);
 
 	loaded = !ret;
-
 	return ret;
 }
 
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
index d1662a75c7ec..9155dafae2a9 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
@@ -3,12 +3,31 @@
 
 #include <linux/clk.h>
 #include <linux/interconnect.h>
+#include <linux/pm_domain.h>
 #include <linux/pm_opp.h>
 #include <soc/qcom/cmd-db.h>
 
 #include "a6xx_gpu.h"
 #include "a6xx_gmu.xml.h"
 
+static void a6xx_gmu_fault(struct a6xx_gmu *gmu)
+{
+	struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
+	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
+	struct msm_gpu *gpu = &adreno_gpu->base;
+	struct drm_device *dev = gpu->dev;
+	struct msm_drm_private *priv = dev->dev_private;
+
+	/* FIXME: add a banner here */
+	gmu->hung = true;
+
+	/* Turn off the hangcheck timer while we are resetting */
+	del_timer(&gpu->hangcheck_timer);
+
+	/* Queue the GPU handler because we need to treat this as a recovery */
+	queue_work(priv->wq, &gpu->recover_work);
+}
+
 static irqreturn_t a6xx_gmu_irq(int irq, void *data)
 {
 	struct a6xx_gmu *gmu = data;
@@ -20,8 +39,7 @@ static irqreturn_t a6xx_gmu_irq(int irq, void *data)
 	if (status & A6XX_GMU_AO_HOST_INTERRUPT_STATUS_WDOG_BITE) {
 		dev_err_ratelimited(gmu->dev, "GMU watchdog expired\n");
 
-		/* Temporary until we can recover safely */
-		BUG();
+		a6xx_gmu_fault(gmu);
 	}
 
 	if (status &  A6XX_GMU_AO_HOST_INTERRUPT_STATUS_HOST_AHB_BUS_ERROR)
@@ -45,8 +63,7 @@ static irqreturn_t a6xx_hfi_irq(int irq, void *data)
 	if (status & A6XX_GMU_GMU2HOST_INTR_INFO_CM3_FAULT) {
 		dev_err_ratelimited(gmu->dev, "GMU firmware fault\n");
 
-		/* Temporary until we can recover safely */
-		BUG();
+		a6xx_gmu_fault(gmu);
 	}
 
 	return IRQ_HANDLED;
@@ -165,10 +182,8 @@ static bool a6xx_gmu_check_idle_level(struct a6xx_gmu *gmu)
 }
 
 /* Wait for the GMU to get to its most idle state */
-int a6xx_gmu_wait_for_idle(struct a6xx_gpu *a6xx_gpu)
+int a6xx_gmu_wait_for_idle(struct a6xx_gmu *gmu)
 {
-	struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
-
 	return spin_until(a6xx_gmu_check_idle_level(gmu));
 }
 
@@ -567,7 +582,7 @@ static int a6xx_gmu_fw_start(struct a6xx_gmu *gmu, unsigned int state)
 		if (!rpmh_init) {
 			a6xx_gmu_rpmh_init(gmu);
 			rpmh_init = true;
-		} else if (state != GMU_RESET) {
+		} else {
 			ret = a6xx_rpmh_start(gmu);
 			if (ret)
 				return ret;
@@ -633,20 +648,6 @@ static int a6xx_gmu_fw_start(struct a6xx_gmu *gmu, unsigned int state)
 	 A6XX_GMU_AO_HOST_INTERRUPT_STATUS_HOST_AHB_BUS_ERROR | \
 	 A6XX_GMU_AO_HOST_INTERRUPT_STATUS_FENCE_ERR)
 
-static void a6xx_gmu_irq_enable(struct a6xx_gmu *gmu)
-{
-	gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_CLR, ~0);
-	gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_CLR, ~0);
-
-	gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_MASK,
-		~A6XX_GMU_IRQ_MASK);
-	gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_MASK,
-		~A6XX_HFI_IRQ_MASK);
-
-	enable_irq(gmu->gmu_irq);
-	enable_irq(gmu->hfi_irq);
-}
-
 static void a6xx_gmu_irq_disable(struct a6xx_gmu *gmu)
 {
 	disable_irq(gmu->gmu_irq);
@@ -656,21 +657,10 @@ static void a6xx_gmu_irq_disable(struct a6xx_gmu *gmu)
 	gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_MASK, ~0);
 }
 
-int a6xx_gmu_reset(struct a6xx_gpu *a6xx_gpu)
+static void a6xx_gmu_rpmh_off(struct a6xx_gmu *gmu)
 {
-	struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
-	int ret;
 	u32 val;
 
-	/* Flush all the queues */
-	a6xx_hfi_stop(gmu);
-
-	/* Stop the interrupts */
-	a6xx_gmu_irq_disable(gmu);
-
-	/* Force off SPTP in case the GMU is managing it */
-	a6xx_sptprac_disable(gmu);
-
 	/* Make sure there are no outstanding RPMh votes */
 	gmu_poll_timeout(gmu, REG_A6XX_RSCC_TCS0_DRV0_STATUS, val,
 		(val & 1), 100, 10000);
@@ -680,37 +670,22 @@ int a6xx_gmu_reset(struct a6xx_gpu *a6xx_gpu)
 		(val & 1), 100, 10000);
 	gmu_poll_timeout(gmu, REG_A6XX_RSCC_TCS3_DRV0_STATUS, val,
 		(val & 1), 100, 1000);
+}
 
-	/* Force off the GX GSDC */
-	regulator_force_disable(gmu->gx);
-
-	/* Disable the resources */
-	clk_bulk_disable_unprepare(gmu->nr_clocks, gmu->clocks);
-	pm_runtime_put_sync(gmu->dev);
-
-	/* Re-enable the resources */
-	pm_runtime_get_sync(gmu->dev);
-
-	/* Use a known rate to bring up the GMU */
-	clk_set_rate(gmu->core_clk, 200000000);
-	ret = clk_bulk_prepare_enable(gmu->nr_clocks, gmu->clocks);
-	if (ret)
-		goto out;
-
-	a6xx_gmu_irq_enable(gmu);
-
-	ret = a6xx_gmu_fw_start(gmu, GMU_RESET);
-	if (!ret)
-		ret = a6xx_hfi_start(gmu, GMU_COLD_BOOT);
+/* Force the GMU off in case it isn't responsive */
+static void a6xx_gmu_force_off(struct a6xx_gmu *gmu)
+{
+	/* Flush all the queues */
+	a6xx_hfi_stop(gmu);
 
-	/* Set the GPU back to the highest power frequency */
-	__a6xx_gmu_set_freq(gmu, gmu->nr_gpu_freqs - 1);
+	/* Stop the interrupts */
+	a6xx_gmu_irq_disable(gmu);
 
-out:
-	if (ret)
-		a6xx_gmu_clear_oob(gmu, GMU_OOB_BOOT_SLUMBER);
+	/* Force off SPTP in case the GMU is managing it */
+	a6xx_sptprac_disable(gmu);
 
-	return ret;
+	/* Make sure there are no outstanding RPMh votes */
+	a6xx_gmu_rpmh_off(gmu);
 }
 
 int a6xx_gmu_resume(struct a6xx_gpu *a6xx_gpu)
@@ -723,19 +698,26 @@ int a6xx_gmu_resume(struct a6xx_gpu *a6xx_gpu)
 	if (WARN(!gmu->mmio, "The GMU is not set up yet\n"))
 		return 0;
 
+	gmu->hung = false;
+
 	/* Turn on the resources */
 	pm_runtime_get_sync(gmu->dev);
 
 	/* Use a known rate to bring up the GMU */
 	clk_set_rate(gmu->core_clk, 200000000);
 	ret = clk_bulk_prepare_enable(gmu->nr_clocks, gmu->clocks);
-	if (ret)
-		goto out;
+	if (ret) {
+		pm_runtime_put(gmu->dev);
+		return ret;
+	}
 
 	/* Set the bus quota to a reasonable value for boot */
 	icc_set_bw(gpu->icc_path, 0, MBps_to_icc(3072));
 
-	a6xx_gmu_irq_enable(gmu);
+	/* Enable the GMU interrupt */
+	gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_CLR, ~0);
+	gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_MASK, ~A6XX_GMU_IRQ_MASK);
+	enable_irq(gmu->gmu_irq);
 
 	/* Check to see if we are doing a cold or warm boot */
 	status = gmu_read(gmu, REG_A6XX_GMU_GENERAL_7) == 1 ?
@@ -746,14 +728,35 @@ int a6xx_gmu_resume(struct a6xx_gpu *a6xx_gpu)
 		goto out;
 
 	ret = a6xx_hfi_start(gmu, status);
+	if (ret)
+		goto out;
+
+	/*
+	 * Turn on the GMU firmware fault interrupt after we know the boot
+	 * sequence is successful
+	 */
+	gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_CLR, ~0);
+	gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_MASK, ~A6XX_HFI_IRQ_MASK);
+	enable_irq(gmu->hfi_irq);
 
 	/* Set the GPU to the highest power frequency */
 	__a6xx_gmu_set_freq(gmu, gmu->nr_gpu_freqs - 1);
 
+	/*
+	 * "enable" the GX power domain which won't actually do anything but it
+	 * will make sure that the refcounting is correct in case we need to
+	 * bring down the GX after a GMU failure
+	 */
+	if (!IS_ERR(gmu->gxpd))
+		pm_runtime_get(gmu->gxpd);
+
 out:
-	/* Make sure to turn off the boot OOB request on error */
-	if (ret)
-		a6xx_gmu_clear_oob(gmu, GMU_OOB_BOOT_SLUMBER);
+	/* On failure, shut down the GMU to leave it in a good state */
+	if (ret) {
+		disable_irq(gmu->gmu_irq);
+		a6xx_rpmh_stop(gmu);
+		pm_runtime_put(gmu->dev);
+	}
 
 	return ret;
 }
@@ -773,11 +776,12 @@ bool a6xx_gmu_isidle(struct a6xx_gmu *gmu)
 	return true;
 }
 
-int a6xx_gmu_stop(struct a6xx_gpu *a6xx_gpu)
+/* Gracefully try to shut down the GMU and by extension the GPU */
+static void a6xx_gmu_shutdown(struct a6xx_gmu *gmu)
 {
+	struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
 	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
 	struct msm_gpu *gpu = &adreno_gpu->base;
-	struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
 	u32 val;
 
 	/*
@@ -787,10 +791,19 @@ int a6xx_gmu_stop(struct a6xx_gpu *a6xx_gpu)
 	val = gmu_read(gmu, REG_A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE);
 
 	if (val != 0xf) {
-		int ret = a6xx_gmu_wait_for_idle(a6xx_gpu);
+		int ret = a6xx_gmu_wait_for_idle(gmu);
 
-		/* Temporary until we can recover safely */
-		BUG_ON(ret);
+		/* If the GMU isn't responding assume it is hung */
+		if (ret) {
+			a6xx_gmu_force_off(gmu);
+			return;
+		}
+
+		/* Clear the VBIF pipe before shutting down */
+		gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0xf);
+		spin_until((gpu_read(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL1) & 0xf)
+			== 0xf);
+		gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0);
 
 		/* tell the GMU we want to slumber */
 		a6xx_gmu_notify_slumber(gmu);
@@ -822,10 +835,37 @@ int a6xx_gmu_stop(struct a6xx_gpu *a6xx_gpu)
 
 	/* Tell RPMh to power off the GPU */
 	a6xx_rpmh_stop(gmu);
+}
+
+
+int a6xx_gmu_stop(struct a6xx_gpu *a6xx_gpu)
+{
+	struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
+	struct msm_gpu *gpu = &a6xx_gpu->base.base;
+
+	if (!pm_runtime_active(gmu->dev))
+		return 0;
+
+	/*
+	 * Force the GMU off if we detected a hang, otherwise try to shut it
+	 * down gracefully
+	 */
+	if (gmu->hung)
+		a6xx_gmu_force_off(gmu);
+	else
+		a6xx_gmu_shutdown(gmu);
 
 	/* Remove the bus vote */
 	icc_set_bw(gpu->icc_path, 0, 0);
 
+	/*
+	 * Make sure the GX domain is off before turning off the GMU (CX)
+	 * domain. Usually the GMU does this but only if the shutdown sequence
+	 * was successful
+	 */
+	if (!IS_ERR(gmu->gxpd))
+		pm_runtime_put_sync(gmu->gxpd);
+
 	clk_bulk_disable_unprepare(gmu->nr_clocks, gmu->clocks);
 
 	pm_runtime_put_sync(gmu->dev);
@@ -948,25 +988,20 @@ static int a6xx_gmu_memory_probe(struct a6xx_gmu *gmu)
 }
 
 /* Return the 'arc-level' for the given frequency */
-static u32 a6xx_gmu_get_arc_level(struct device *dev, unsigned long freq)
+static unsigned int a6xx_gmu_get_arc_level(struct device *dev,
+					   unsigned long freq)
 {
 	struct dev_pm_opp *opp;
-	struct device_node *np;
-	u32 val = 0;
+	unsigned int val;
 
 	if (!freq)
 		return 0;
 
-	opp  = dev_pm_opp_find_freq_exact(dev, freq, true);
+	opp = dev_pm_opp_find_freq_exact(dev, freq, true);
 	if (IS_ERR(opp))
 		return 0;
 
-	np = dev_pm_opp_get_of_node(opp);
-
-	if (np) {
-		of_property_read_u32(np, "opp-level", &val);
-		of_node_put(np);
-	}
+	val = dev_pm_opp_get_level(opp);
 
 	dev_pm_opp_put(opp);
 
@@ -1002,7 +1037,7 @@ static int a6xx_gmu_rpmh_arc_votes_init(struct device *dev, u32 *votes,
 	/* Construct a vote for each frequency */
 	for (i = 0; i < freqs_count; i++) {
 		u8 pindex = 0, sindex = 0;
-		u32 level = a6xx_gmu_get_arc_level(dev, freqs[i]);
+		unsigned int level = a6xx_gmu_get_arc_level(dev, freqs[i]);
 
 		/* Get the primary index that matches the arc level */
 		for (j = 0; j < pri_count; j++) {
@@ -1195,9 +1230,15 @@ void a6xx_gmu_remove(struct a6xx_gpu *a6xx_gpu)
 	if (IS_ERR_OR_NULL(gmu->mmio))
 		return;
 
-	pm_runtime_disable(gmu->dev);
 	a6xx_gmu_stop(a6xx_gpu);
 
+	pm_runtime_disable(gmu->dev);
+
+	if (!IS_ERR(gmu->gxpd)) {
+		pm_runtime_disable(gmu->gxpd);
+		dev_pm_domain_detach(gmu->gxpd, false);
+	}
+
 	a6xx_gmu_irq_disable(gmu);
 	a6xx_gmu_memory_free(gmu, gmu->hfi);
 
@@ -1223,7 +1264,6 @@ int a6xx_gmu_probe(struct a6xx_gpu *a6xx_gpu, struct device_node *node)
 	gmu->idle_level = GMU_IDLE_STATE_ACTIVE;
 
 	pm_runtime_enable(gmu->dev);
-	gmu->gx = devm_regulator_get(gmu->dev, "vdd");
 
 	/* Get the list of clocks */
 	ret = a6xx_gmu_clocks_probe(gmu);
@@ -1257,6 +1297,12 @@ int a6xx_gmu_probe(struct a6xx_gpu *a6xx_gpu, struct device_node *node)
 	if (gmu->hfi_irq < 0 || gmu->gmu_irq < 0)
 		goto err;
 
+	/*
+	 * Get a link to the GX power domain to reset the GPU in case of GMU
+	 * crash
+	 */
+	gmu->gxpd = dev_pm_domain_attach_by_name(gmu->dev, "gx");
+
 	/* Get the power levels for the GMU and GPU */
 	a6xx_gmu_pwrlevels_probe(gmu);
 
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.h b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h
index c721d9165d8e..bedd8e6a63aa 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.h
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h
@@ -27,9 +27,6 @@ struct a6xx_gmu_bo {
 /* the GMU is coming up for the first time or back from a power collapse */
 #define GMU_COLD_BOOT 1
 
-/* The GMU is being soft reset after a fault */
-#define GMU_RESET 2
-
 /*
  * These define the level of control that the GMU has - the higher the number
  * the more things that the GMU hardware controls on its own.
@@ -52,11 +49,11 @@ struct a6xx_gmu {
 	int hfi_irq;
 	int gmu_irq;
 
-	struct regulator *gx;
-
 	struct iommu_domain *domain;
 	u64 uncached_iova_base;
 
+	struct device *gxpd;
+
 	int idle_level;
 
 	struct a6xx_gmu_bo *hfi;
@@ -78,7 +75,7 @@ struct a6xx_gmu {
 
 	struct a6xx_hfi_queue queues[2];
 
-	struct tasklet_struct hfi_tasklet;
+	bool hung;
 };
 
 static inline u32 gmu_read(struct a6xx_gmu *gmu, u32 offset)
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
index fefe773c989e..e74dce474250 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
@@ -10,6 +10,8 @@
 
 #include <linux/devfreq.h>
 
+#define GPU_PAS_ID 13
+
 static inline bool _a6xx_check_idle(struct msm_gpu *gpu)
 {
 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
@@ -343,6 +345,20 @@ static int a6xx_ucode_init(struct msm_gpu *gpu)
 	return 0;
 }
 
+static int a6xx_zap_shader_init(struct msm_gpu *gpu)
+{
+	static bool loaded;
+	int ret;
+
+	if (loaded)
+		return 0;
+
+	ret = adreno_zap_shader_load(gpu, GPU_PAS_ID);
+
+	loaded = !ret;
+	return ret;
+}
+
 #define A6XX_INT_MASK (A6XX_RBBM_INT_0_MASK_CP_AHB_ERROR | \
 	  A6XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNCFIFO_OVERFLOW | \
 	  A6XX_RBBM_INT_0_MASK_CP_HW_ERROR | \
@@ -491,7 +507,28 @@ static int a6xx_hw_init(struct msm_gpu *gpu)
 	if (ret)
 		goto out;
 
-	gpu_write(gpu, REG_A6XX_RBBM_SECVID_TRUST_CNTL, 0x0);
+	/*
+	 * Try to load a zap shader into the secure world. If successful
+	 * we can use the CP to switch out of secure mode. If not then we
+	 * have no resource but to try to switch ourselves out manually. If we
+	 * guessed wrong then access to the RBBM_SECVID_TRUST_CNTL register will
+	 * be blocked and a permissions violation will soon follow.
+	 */
+	ret = a6xx_zap_shader_init(gpu);
+	if (!ret) {
+		OUT_PKT7(gpu->rb[0], CP_SET_SECURE_MODE, 1);
+		OUT_RING(gpu->rb[0], 0x00000000);
+
+		a6xx_flush(gpu, gpu->rb[0]);
+		if (!a6xx_idle(gpu, gpu->rb[0]))
+			return -EINVAL;
+	} else {
+		/* Print a warning so if we die, we know why */
+		dev_warn_once(gpu->dev->dev,
+			"Zap shader not enabled - using SECVID_TRUST_CNTL instead\n");
+		gpu_write(gpu, REG_A6XX_RBBM_SECVID_TRUST_CNTL, 0x0);
+		ret = 0;
+	}
 
 out:
 	/*
@@ -678,13 +715,15 @@ static int a6xx_pm_resume(struct msm_gpu *gpu)
 	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
 	int ret;
 
-	ret = a6xx_gmu_resume(a6xx_gpu);
-
 	gpu->needs_hw_init = true;
 
+	ret = a6xx_gmu_resume(a6xx_gpu);
+	if (ret)
+		return ret;
+
 	msm_gpu_resume_devfreq(gpu);
 
-	return ret;
+	return 0;
 }
 
 static int a6xx_pm_suspend(struct msm_gpu *gpu)
@@ -694,18 +733,6 @@ static int a6xx_pm_suspend(struct msm_gpu *gpu)
 
 	devfreq_suspend_device(gpu->devfreq.devfreq);
 
-	/*
-	 * Make sure the GMU is idle before continuing (because some transitions
-	 * may use VBIF
-	 */
-	a6xx_gmu_wait_for_idle(a6xx_gpu);
-
-	/* Clear the VBIF pipe before shutting down */
-	/* FIXME: This accesses the GPU - do we need to make sure it is on? */
-	gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0xf);
-	spin_until((gpu_read(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL1) & 0xf) == 0xf);
-	gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0);
-
 	return a6xx_gmu_stop(a6xx_gpu);
 }
 
@@ -781,14 +808,16 @@ static const struct adreno_gpu_funcs funcs = {
 		.active_ring = a6xx_active_ring,
 		.irq = a6xx_irq,
 		.destroy = a6xx_destroy,
-#if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP)
+#if defined(CONFIG_DRM_MSM_GPU_STATE)
 		.show = a6xx_show,
 #endif
 		.gpu_busy = a6xx_gpu_busy,
 		.gpu_get_freq = a6xx_gmu_get_freq,
 		.gpu_set_freq = a6xx_gmu_set_freq,
+#if defined(CONFIG_DRM_MSM_GPU_STATE)
 		.gpu_state_get = a6xx_gpu_state_get,
 		.gpu_state_put = a6xx_gpu_state_put,
+#endif
 	},
 	.get_timestamp = a6xx_get_timestamp,
 };
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.h b/drivers/gpu/drm/msm/adreno/a6xx_gpu.h
index 528a4cfe07cd..b46279eb18c5 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.h
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.h
@@ -46,9 +46,8 @@ struct a6xx_gpu {
 int a6xx_gmu_resume(struct a6xx_gpu *gpu);
 int a6xx_gmu_stop(struct a6xx_gpu *gpu);
 
-int a6xx_gmu_wait_for_idle(struct a6xx_gpu *gpu);
+int a6xx_gmu_wait_for_idle(struct a6xx_gmu *gmu);
 
-int a6xx_gmu_reset(struct a6xx_gpu *a6xx_gpu);
 bool a6xx_gmu_isidle(struct a6xx_gmu *gmu);
 
 int a6xx_gmu_set_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state);
diff --git a/drivers/gpu/drm/msm/adreno/adreno_device.c b/drivers/gpu/drm/msm/adreno/adreno_device.c
index 714ed6505e47..b907245d3d96 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_device.c
+++ b/drivers/gpu/drm/msm/adreno/adreno_device.c
@@ -155,6 +155,7 @@ static const struct adreno_info gpulist[] = {
 		.gmem = SZ_1M,
 		.inactive_period = DRM_MSM_INACTIVE_PERIOD,
 		.init = a6xx_gpu_init,
+		.zapfw = "a630_zap.mdt",
 	},
 };
 
@@ -229,6 +230,7 @@ struct msm_gpu *adreno_load_gpu(struct drm_device *dev)
 
 	ret = pm_runtime_get_sync(&pdev->dev);
 	if (ret < 0) {
+		pm_runtime_put_sync(&pdev->dev);
 		DRM_DEV_ERROR(dev->dev, "Couldn't power up the GPU: %d\n", ret);
 		return NULL;
 	}
diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.c b/drivers/gpu/drm/msm/adreno/adreno_gpu.c
index 27898475cdf4..6f7f4114afcf 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.c
@@ -19,13 +19,148 @@
 
 #include <linux/ascii85.h>
 #include <linux/interconnect.h>
+#include <linux/qcom_scm.h>
 #include <linux/kernel.h>
+#include <linux/of_address.h>
 #include <linux/pm_opp.h>
 #include <linux/slab.h>
+#include <linux/soc/qcom/mdt_loader.h>
 #include "adreno_gpu.h"
 #include "msm_gem.h"
 #include "msm_mmu.h"
 
+static bool zap_available = true;
+
+static int zap_shader_load_mdt(struct msm_gpu *gpu, const char *fwname,
+		u32 pasid)
+{
+	struct device *dev = &gpu->pdev->dev;
+	const struct firmware *fw;
+	struct device_node *np, *mem_np;
+	struct resource r;
+	phys_addr_t mem_phys;
+	ssize_t mem_size;
+	void *mem_region = NULL;
+	int ret;
+
+	if (!IS_ENABLED(CONFIG_ARCH_QCOM)) {
+		zap_available = false;
+		return -EINVAL;
+	}
+
+	np = of_get_child_by_name(dev->of_node, "zap-shader");
+	if (!np) {
+		zap_available = false;
+		return -ENODEV;
+	}
+
+	mem_np = of_parse_phandle(np, "memory-region", 0);
+	of_node_put(np);
+	if (!mem_np) {
+		zap_available = false;
+		return -EINVAL;
+	}
+
+	ret = of_address_to_resource(mem_np, 0, &r);
+	of_node_put(mem_np);
+	if (ret)
+		return ret;
+
+	mem_phys = r.start;
+	mem_size = resource_size(&r);
+
+	/* Request the MDT file for the firmware */
+	fw = adreno_request_fw(to_adreno_gpu(gpu), fwname);
+	if (IS_ERR(fw)) {
+		DRM_DEV_ERROR(dev, "Unable to load %s\n", fwname);
+		return PTR_ERR(fw);
+	}
+
+	/* Figure out how much memory we need */
+	mem_size = qcom_mdt_get_size(fw);
+	if (mem_size < 0) {
+		ret = mem_size;
+		goto out;
+	}
+
+	/* Allocate memory for the firmware image */
+	mem_region = memremap(mem_phys, mem_size,  MEMREMAP_WC);
+	if (!mem_region) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	/*
+	 * Load the rest of the MDT
+	 *
+	 * Note that we could be dealing with two different paths, since
+	 * with upstream linux-firmware it would be in a qcom/ subdir..
+	 * adreno_request_fw() handles this, but qcom_mdt_load() does
+	 * not.  But since we've already gotten through adreno_request_fw()
+	 * we know which of the two cases it is:
+	 */
+	if (to_adreno_gpu(gpu)->fwloc == FW_LOCATION_LEGACY) {
+		ret = qcom_mdt_load(dev, fw, fwname, pasid,
+				mem_region, mem_phys, mem_size, NULL);
+	} else {
+		char *newname;
+
+		newname = kasprintf(GFP_KERNEL, "qcom/%s", fwname);
+
+		ret = qcom_mdt_load(dev, fw, newname, pasid,
+				mem_region, mem_phys, mem_size, NULL);
+		kfree(newname);
+	}
+	if (ret)
+		goto out;
+
+	/* Send the image to the secure world */
+	ret = qcom_scm_pas_auth_and_reset(pasid);
+
+	/*
+	 * If the scm call returns -EOPNOTSUPP we assume that this target
+	 * doesn't need/support the zap shader so quietly fail
+	 */
+	if (ret == -EOPNOTSUPP)
+		zap_available = false;
+	else if (ret)
+		DRM_DEV_ERROR(dev, "Unable to authorize the image\n");
+
+out:
+	if (mem_region)
+		memunmap(mem_region);
+
+	release_firmware(fw);
+
+	return ret;
+}
+
+int adreno_zap_shader_load(struct msm_gpu *gpu, u32 pasid)
+{
+	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+	struct platform_device *pdev = gpu->pdev;
+
+	/* Short cut if we determine the zap shader isn't available/needed */
+	if (!zap_available)
+		return -ENODEV;
+
+	/* We need SCM to be able to load the firmware */
+	if (!qcom_scm_is_available()) {
+		DRM_DEV_ERROR(&pdev->dev, "SCM is not available\n");
+		return -EPROBE_DEFER;
+	}
+
+	/* Each GPU has a target specific zap shader firmware name to use */
+	if (!adreno_gpu->info->zapfw) {
+		zap_available = false;
+		DRM_DEV_ERROR(&pdev->dev,
+			"Zap shader firmware file not specified for this target\n");
+		return -ENODEV;
+	}
+
+	return zap_shader_load_mdt(gpu, adreno_gpu->info->zapfw, pasid);
+}
+
 int adreno_get_param(struct msm_gpu *gpu, uint32_t param, uint64_t *value)
 {
 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
@@ -63,6 +198,12 @@ int adreno_get_param(struct msm_gpu *gpu, uint32_t param, uint64_t *value)
 	case MSM_PARAM_NR_RINGS:
 		*value = gpu->nr_rings;
 		return 0;
+	case MSM_PARAM_PP_PGTABLE:
+		*value = 0;
+		return 0;
+	case MSM_PARAM_FAULTS:
+		*value = gpu->global_faults;
+		return 0;
 	default:
 		DBG("%s: invalid param: %u", gpu->name, param);
 		return -EINVAL;
diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.h b/drivers/gpu/drm/msm/adreno/adreno_gpu.h
index 5db459bc28a7..0925606ec9b5 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_gpu.h
+++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.h
@@ -252,6 +252,12 @@ void adreno_gpu_state_destroy(struct msm_gpu_state *state);
 int adreno_gpu_state_get(struct msm_gpu *gpu, struct msm_gpu_state *state);
 int adreno_gpu_state_put(struct msm_gpu_state *state);
 
+/*
+ * For a5xx and a6xx targets load the zap shader that is used to pull the GPU
+ * out of secure mode
+ */
+int adreno_zap_shader_load(struct msm_gpu *gpu, u32 pasid);
+
 /* ringbuffer helpers (the parts that are adreno specific) */
 
 static inline void
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c
index b776fca571f3..dfdfa766da8f 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c
@@ -46,6 +46,9 @@
 #define LEFT_MIXER 0
 #define RIGHT_MIXER 1
 
+/* timeout in ms waiting for frame done */
+#define DPU_CRTC_FRAME_DONE_TIMEOUT_MS	60
+
 static struct dpu_kms *_dpu_crtc_get_kms(struct drm_crtc *crtc)
 {
 	struct msm_drm_private *priv = crtc->dev->dev_private;
@@ -425,65 +428,6 @@ void dpu_crtc_complete_commit(struct drm_crtc *crtc,
 	trace_dpu_crtc_complete_commit(DRMID(crtc));
 }
 
-static void _dpu_crtc_setup_mixer_for_encoder(
-		struct drm_crtc *crtc,
-		struct drm_encoder *enc)
-{
-	struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc->state);
-	struct dpu_kms *dpu_kms = _dpu_crtc_get_kms(crtc);
-	struct dpu_rm *rm = &dpu_kms->rm;
-	struct dpu_crtc_mixer *mixer;
-	struct dpu_hw_ctl *last_valid_ctl = NULL;
-	int i;
-	struct dpu_rm_hw_iter lm_iter, ctl_iter;
-
-	dpu_rm_init_hw_iter(&lm_iter, enc->base.id, DPU_HW_BLK_LM);
-	dpu_rm_init_hw_iter(&ctl_iter, enc->base.id, DPU_HW_BLK_CTL);
-
-	/* Set up all the mixers and ctls reserved by this encoder */
-	for (i = cstate->num_mixers; i < ARRAY_SIZE(cstate->mixers); i++) {
-		mixer = &cstate->mixers[i];
-
-		if (!dpu_rm_get_hw(rm, &lm_iter))
-			break;
-		mixer->hw_lm = (struct dpu_hw_mixer *)lm_iter.hw;
-
-		/* CTL may be <= LMs, if <, multiple LMs controlled by 1 CTL */
-		if (!dpu_rm_get_hw(rm, &ctl_iter)) {
-			DPU_DEBUG("no ctl assigned to lm %d, using previous\n",
-					mixer->hw_lm->idx - LM_0);
-			mixer->lm_ctl = last_valid_ctl;
-		} else {
-			mixer->lm_ctl = (struct dpu_hw_ctl *)ctl_iter.hw;
-			last_valid_ctl = mixer->lm_ctl;
-		}
-
-		/* Shouldn't happen, mixers are always >= ctls */
-		if (!mixer->lm_ctl) {
-			DPU_ERROR("no valid ctls found for lm %d\n",
-					mixer->hw_lm->idx - LM_0);
-			return;
-		}
-
-		cstate->num_mixers++;
-		DPU_DEBUG("setup mixer %d: lm %d\n",
-				i, mixer->hw_lm->idx - LM_0);
-		DPU_DEBUG("setup mixer %d: ctl %d\n",
-				i, mixer->lm_ctl->idx - CTL_0);
-	}
-}
-
-static void _dpu_crtc_setup_mixers(struct drm_crtc *crtc)
-{
-	struct drm_encoder *enc;
-
-	WARN_ON(!drm_modeset_is_locked(&crtc->mutex));
-
-	/* Check for mixers on all encoders attached to this crtc */
-	drm_for_each_encoder_mask(enc, crtc->dev, crtc->state->encoder_mask)
-		_dpu_crtc_setup_mixer_for_encoder(crtc, enc);
-}
-
 static void _dpu_crtc_setup_lm_bounds(struct drm_crtc *crtc,
 		struct drm_crtc_state *state)
 {
@@ -533,10 +477,7 @@ static void dpu_crtc_atomic_begin(struct drm_crtc *crtc,
 	dev = crtc->dev;
 	smmu_state = &dpu_crtc->smmu_state;
 
-	if (!cstate->num_mixers) {
-		_dpu_crtc_setup_mixers(crtc);
-		_dpu_crtc_setup_lm_bounds(crtc, crtc->state);
-	}
+	_dpu_crtc_setup_lm_bounds(crtc, crtc->state);
 
 	if (dpu_crtc->event) {
 		WARN_ON(dpu_crtc->event);
@@ -683,7 +624,7 @@ static int _dpu_crtc_wait_for_frame_done(struct drm_crtc *crtc)
 
 	DPU_ATRACE_BEGIN("frame done completion wait");
 	ret = wait_for_completion_timeout(&dpu_crtc->frame_done_comp,
-			msecs_to_jiffies(DPU_FRAME_DONE_TIMEOUT));
+			msecs_to_jiffies(DPU_CRTC_FRAME_DONE_TIMEOUT_MS));
 	if (!ret) {
 		DRM_ERROR("frame done wait timed out, ret:%d\n", ret);
 		rc = -ETIMEDOUT;
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c
index 5aa3307f3f0c..82bf16d61a45 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c
@@ -69,6 +69,9 @@
 
 #define MAX_VDISPLAY_SPLIT 1080
 
+/* timeout in frames waiting for frame done */
+#define DPU_ENCODER_FRAME_DONE_TIMEOUT_FRAMES 5
+
 /**
  * enum dpu_enc_rc_events - events for resource control state machine
  * @DPU_ENC_RC_EVENT_KICKOFF:
@@ -158,7 +161,7 @@ enum dpu_enc_rc_states {
  *				Bit0 = phys_encs[0] etc.
  * @crtc_frame_event_cb:	callback handler for frame event
  * @crtc_frame_event_cb_data:	callback handler private data
- * @frame_done_timeout:		frame done timeout in Hz
+ * @frame_done_timeout_ms:	frame done timeout in ms
  * @frame_done_timer:		watchdog timer for frame done event
  * @vsync_event_timer:		vsync timer
  * @disp_info:			local copy of msm_display_info struct
@@ -196,7 +199,7 @@ struct dpu_encoder_virt {
 	void (*crtc_frame_event_cb)(void *, u32 event);
 	void *crtc_frame_event_cb_data;
 
-	atomic_t frame_done_timeout;
+	atomic_t frame_done_timeout_ms;
 	struct timer_list frame_done_timer;
 	struct timer_list vsync_event_timer;
 
@@ -520,8 +523,8 @@ static void _dpu_encoder_adjust_mode(struct drm_connector *connector,
 
 	list_for_each_entry(cur_mode, &connector->modes, head) {
 		if (cur_mode->vdisplay == adj_mode->vdisplay &&
-			cur_mode->hdisplay == adj_mode->hdisplay &&
-			cur_mode->vrefresh == adj_mode->vrefresh) {
+		    cur_mode->hdisplay == adj_mode->hdisplay &&
+		    drm_mode_vrefresh(cur_mode) == drm_mode_vrefresh(adj_mode)) {
 			adj_mode->private = cur_mode->private;
 			adj_mode->private_flags |= cur_mode->private_flags;
 		}
@@ -959,10 +962,14 @@ static void dpu_encoder_virt_mode_set(struct drm_encoder *drm_enc,
 	struct dpu_kms *dpu_kms;
 	struct list_head *connector_list;
 	struct drm_connector *conn = NULL, *conn_iter;
-	struct dpu_rm_hw_iter pp_iter, ctl_iter;
+	struct drm_crtc *drm_crtc;
+	struct dpu_crtc_state *cstate;
+	struct dpu_rm_hw_iter hw_iter;
 	struct msm_display_topology topology;
 	struct dpu_hw_ctl *hw_ctl[MAX_CHANNELS_PER_ENC] = { NULL };
-	int i = 0, ret;
+	struct dpu_hw_mixer *hw_lm[MAX_CHANNELS_PER_ENC] = { NULL };
+	int num_lm = 0, num_ctl = 0;
+	int i, j, ret;
 
 	if (!drm_enc) {
 		DPU_ERROR("invalid encoder\n");
@@ -990,10 +997,14 @@ static void dpu_encoder_virt_mode_set(struct drm_encoder *drm_enc,
 		return;
 	}
 
+	drm_for_each_crtc(drm_crtc, drm_enc->dev)
+		if (drm_crtc->state->encoder_mask & drm_encoder_mask(drm_enc))
+			break;
+
 	topology = dpu_encoder_get_topology(dpu_enc, dpu_kms, adj_mode);
 
 	/* Reserve dynamic resources now. Indicating non-AtomicTest phase */
-	ret = dpu_rm_reserve(&dpu_kms->rm, drm_enc, drm_enc->crtc->state,
+	ret = dpu_rm_reserve(&dpu_kms->rm, drm_enc, drm_crtc->state,
 			     topology, false);
 	if (ret) {
 		DPU_ERROR_ENC(dpu_enc,
@@ -1001,21 +1012,41 @@ static void dpu_encoder_virt_mode_set(struct drm_encoder *drm_enc,
 		return;
 	}
 
-	dpu_rm_init_hw_iter(&pp_iter, drm_enc->base.id, DPU_HW_BLK_PINGPONG);
+	dpu_rm_init_hw_iter(&hw_iter, drm_enc->base.id, DPU_HW_BLK_PINGPONG);
 	for (i = 0; i < MAX_CHANNELS_PER_ENC; i++) {
 		dpu_enc->hw_pp[i] = NULL;
-		if (!dpu_rm_get_hw(&dpu_kms->rm, &pp_iter))
+		if (!dpu_rm_get_hw(&dpu_kms->rm, &hw_iter))
+			break;
+		dpu_enc->hw_pp[i] = (struct dpu_hw_pingpong *) hw_iter.hw;
+	}
+
+	dpu_rm_init_hw_iter(&hw_iter, drm_enc->base.id, DPU_HW_BLK_CTL);
+	for (i = 0; i < MAX_CHANNELS_PER_ENC; i++) {
+		if (!dpu_rm_get_hw(&dpu_kms->rm, &hw_iter))
 			break;
-		dpu_enc->hw_pp[i] = (struct dpu_hw_pingpong *) pp_iter.hw;
+		hw_ctl[i] = (struct dpu_hw_ctl *)hw_iter.hw;
+		num_ctl++;
 	}
 
-	dpu_rm_init_hw_iter(&ctl_iter, drm_enc->base.id, DPU_HW_BLK_CTL);
+	dpu_rm_init_hw_iter(&hw_iter, drm_enc->base.id, DPU_HW_BLK_LM);
 	for (i = 0; i < MAX_CHANNELS_PER_ENC; i++) {
-		if (!dpu_rm_get_hw(&dpu_kms->rm, &ctl_iter))
+		if (!dpu_rm_get_hw(&dpu_kms->rm, &hw_iter))
 			break;
-		hw_ctl[i] = (struct dpu_hw_ctl *)ctl_iter.hw;
+		hw_lm[i] = (struct dpu_hw_mixer *)hw_iter.hw;
+		num_lm++;
 	}
 
+	cstate = to_dpu_crtc_state(drm_crtc->state);
+
+	for (i = 0; i < num_lm; i++) {
+		int ctl_idx = (i < num_ctl) ? i : (num_ctl-1);
+
+		cstate->mixers[i].hw_lm = hw_lm[i];
+		cstate->mixers[i].lm_ctl = hw_ctl[ctl_idx];
+	}
+
+	cstate->num_mixers = num_lm;
+
 	for (i = 0; i < dpu_enc->num_phys_encs; i++) {
 		struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i];
 
@@ -1023,18 +1054,38 @@ static void dpu_encoder_virt_mode_set(struct drm_encoder *drm_enc,
 			if (!dpu_enc->hw_pp[i]) {
 				DPU_ERROR_ENC(dpu_enc, "no pp block assigned"
 					     "at idx: %d\n", i);
-				return;
+				goto error;
 			}
 
 			if (!hw_ctl[i]) {
 				DPU_ERROR_ENC(dpu_enc, "no ctl block assigned"
 					     "at idx: %d\n", i);
-				return;
+				goto error;
 			}
 
 			phys->hw_pp = dpu_enc->hw_pp[i];
 			phys->hw_ctl = hw_ctl[i];
 
+			dpu_rm_init_hw_iter(&hw_iter, drm_enc->base.id,
+					    DPU_HW_BLK_INTF);
+			for (j = 0; j < MAX_CHANNELS_PER_ENC; j++) {
+				struct dpu_hw_intf *hw_intf;
+
+				if (!dpu_rm_get_hw(&dpu_kms->rm, &hw_iter))
+					break;
+
+				hw_intf = (struct dpu_hw_intf *)hw_iter.hw;
+				if (hw_intf->idx == phys->intf_idx)
+					phys->hw_intf = hw_intf;
+			}
+
+			if (!phys->hw_intf) {
+				DPU_ERROR_ENC(dpu_enc,
+					      "no intf block assigned at idx: %d\n",
+					      i);
+				goto error;
+			}
+
 			phys->connector = conn->state->connector;
 			if (phys->ops.mode_set)
 				phys->ops.mode_set(phys, mode, adj_mode);
@@ -1042,6 +1093,9 @@ static void dpu_encoder_virt_mode_set(struct drm_encoder *drm_enc,
 	}
 
 	dpu_enc->mode_set_complete = true;
+
+error:
+	dpu_rm_release(&dpu_kms->rm, drm_enc);
 }
 
 static void _dpu_encoder_virt_enable_helper(struct drm_encoder *drm_enc)
@@ -1182,7 +1236,7 @@ static void dpu_encoder_virt_disable(struct drm_encoder *drm_enc)
 	}
 
 	/* after phys waits for frame-done, should be no more frames pending */
-	if (atomic_xchg(&dpu_enc->frame_done_timeout, 0)) {
+	if (atomic_xchg(&dpu_enc->frame_done_timeout_ms, 0)) {
 		DPU_ERROR("enc%d timeout pending\n", drm_enc->base.id);
 		del_timer_sync(&dpu_enc->frame_done_timer);
 	}
@@ -1339,7 +1393,7 @@ static void dpu_encoder_frame_done_callback(
 		}
 
 		if (!dpu_enc->frame_busy_mask[0]) {
-			atomic_set(&dpu_enc->frame_done_timeout, 0);
+			atomic_set(&dpu_enc->frame_done_timeout_ms, 0);
 			del_timer(&dpu_enc->frame_done_timer);
 
 			dpu_encoder_resource_control(drm_enc,
@@ -1547,8 +1601,14 @@ static void _dpu_encoder_kickoff_phys(struct dpu_encoder_virt *dpu_enc,
 		if (!ctl)
 			continue;
 
-		if (phys->split_role != ENC_ROLE_SLAVE)
+		/*
+		 * This is cleared in frame_done worker, which isn't invoked
+		 * for async commits. So don't set this for async, since it'll
+		 * roll over to the next commit.
+		 */
+		if (!async && phys->split_role != ENC_ROLE_SLAVE)
 			set_bit(i, dpu_enc->frame_busy_mask);
+
 		if (!phys->ops.needs_single_flush ||
 				!phys->ops.needs_single_flush(phys))
 			_dpu_encoder_trigger_flush(&dpu_enc->base, phys, 0x0,
@@ -1800,11 +1860,20 @@ void dpu_encoder_kickoff(struct drm_encoder *drm_enc, bool async)
 
 	trace_dpu_enc_kickoff(DRMID(drm_enc));
 
-	atomic_set(&dpu_enc->frame_done_timeout,
-			DPU_FRAME_DONE_TIMEOUT * 1000 /
-			drm_enc->crtc->state->adjusted_mode.vrefresh);
-	mod_timer(&dpu_enc->frame_done_timer, jiffies +
-		((atomic_read(&dpu_enc->frame_done_timeout) * HZ) / 1000));
+	/*
+	 * Asynchronous frames don't handle FRAME_DONE events. As such, they
+	 * shouldn't enable the frame_done watchdog since it will always time
+	 * out.
+	 */
+	if (!async) {
+		unsigned long timeout_ms;
+		timeout_ms = DPU_ENCODER_FRAME_DONE_TIMEOUT_FRAMES * 1000 /
+			drm_mode_vrefresh(&drm_enc->crtc->state->adjusted_mode);
+
+		atomic_set(&dpu_enc->frame_done_timeout_ms, timeout_ms);
+		mod_timer(&dpu_enc->frame_done_timer,
+			  jiffies + msecs_to_jiffies(timeout_ms));
+	}
 
 	/* All phys encs are ready to go, trigger the kickoff */
 	_dpu_encoder_kickoff_phys(dpu_enc, async);
@@ -2124,7 +2193,7 @@ static void dpu_encoder_frame_done_timeout(struct timer_list *t)
 		DRM_DEBUG_KMS("id:%u invalid timeout frame_busy_mask=%lu\n",
 			      DRMID(drm_enc), dpu_enc->frame_busy_mask[0]);
 		return;
-	} else if (!atomic_xchg(&dpu_enc->frame_done_timeout, 0)) {
+	} else if (!atomic_xchg(&dpu_enc->frame_done_timeout_ms, 0)) {
 		DRM_DEBUG_KMS("id:%u invalid timeout\n", DRMID(drm_enc));
 		return;
 	}
@@ -2170,7 +2239,7 @@ int dpu_encoder_setup(struct drm_device *dev, struct drm_encoder *enc,
 
 	spin_lock_init(&dpu_enc->enc_spinlock);
 
-	atomic_set(&dpu_enc->frame_done_timeout, 0);
+	atomic_set(&dpu_enc->frame_done_timeout_ms, 0);
 	timer_setup(&dpu_enc->frame_done_timer,
 			dpu_encoder_frame_done_timeout, 0);
 
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys.h
index db94f3d3bea3..97fb868a4ef6 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys.h
@@ -200,6 +200,7 @@ struct dpu_encoder_irq {
  * @hw_mdptop:		Hardware interface to the top registers
  * @hw_ctl:		Hardware interface to the ctl registers
  * @hw_pp:		Hardware interface to the ping pong registers
+ * @hw_intf:		Hardware interface to the intf registers
  * @dpu_kms:		Pointer to the dpu_kms top level
  * @cached_mode:	DRM mode cached at mode_set time, acted on in enable
  * @enabled:		Whether the encoder has enabled and running a mode
@@ -228,6 +229,7 @@ struct dpu_encoder_phys {
 	struct dpu_hw_mdp *hw_mdptop;
 	struct dpu_hw_ctl *hw_ctl;
 	struct dpu_hw_pingpong *hw_pp;
+	struct dpu_hw_intf *hw_intf;
 	struct dpu_kms *dpu_kms;
 	struct drm_display_mode cached_mode;
 	enum dpu_enc_split_role split_role;
@@ -251,19 +253,6 @@ static inline int dpu_encoder_phys_inc_pending(struct dpu_encoder_phys *phys)
 }
 
 /**
- * struct dpu_encoder_phys_vid - sub-class of dpu_encoder_phys to handle video
- *	mode specific operations
- * @base:	Baseclass physical encoder structure
- * @hw_intf:	Hardware interface to the intf registers
- * @timing_params: Current timing parameter
- */
-struct dpu_encoder_phys_vid {
-	struct dpu_encoder_phys base;
-	struct dpu_hw_intf *hw_intf;
-	struct intf_timing_params timing_params;
-};
-
-/**
  * struct dpu_encoder_phys_cmd - sub-class of dpu_encoder_phys to handle command
  *	mode specific operations
  * @base:	Baseclass physical encoder structure
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c
index a399e1edd313..973737fb5c9f 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_cmd.c
@@ -404,7 +404,8 @@ static void dpu_encoder_phys_cmd_tearcheck_config(
 		return;
 	}
 
-	tc_cfg.vsync_count = vsync_hz / (mode->vtotal * mode->vrefresh);
+	tc_cfg.vsync_count = vsync_hz /
+				(mode->vtotal * drm_mode_vrefresh(mode));
 
 	/* enable external TE after kickoff to avoid premature autorefresh */
 	tc_cfg.hw_vsync_mode = 0;
@@ -424,7 +425,7 @@ static void dpu_encoder_phys_cmd_tearcheck_config(
 	DPU_DEBUG_CMDENC(cmd_enc,
 		"tc %d vsync_clk_speed_hz %u vtotal %u vrefresh %u\n",
 		phys_enc->hw_pp->idx - PINGPONG_0, vsync_hz,
-		mode->vtotal, mode->vrefresh);
+		mode->vtotal, drm_mode_vrefresh(mode));
 	DPU_DEBUG_CMDENC(cmd_enc,
 		"tc %d enable %u start_pos %u rd_ptr_irq %u\n",
 		phys_enc->hw_pp->idx - PINGPONG_0, tc_enable, tc_cfg.start_pos,
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_vid.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_vid.c
index 3c4eb470a82c..1b7a335a6140 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_vid.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder_phys_vid.c
@@ -18,14 +18,14 @@
 #include "dpu_trace.h"
 
 #define DPU_DEBUG_VIDENC(e, fmt, ...) DPU_DEBUG("enc%d intf%d " fmt, \
-		(e) && (e)->base.parent ? \
-		(e)->base.parent->base.id : -1, \
+		(e) && (e)->parent ? \
+		(e)->parent->base.id : -1, \
 		(e) && (e)->hw_intf ? \
 		(e)->hw_intf->idx - INTF_0 : -1, ##__VA_ARGS__)
 
 #define DPU_ERROR_VIDENC(e, fmt, ...) DPU_ERROR("enc%d intf%d " fmt, \
-		(e) && (e)->base.parent ? \
-		(e)->base.parent->base.id : -1, \
+		(e) && (e)->parent ? \
+		(e)->parent->base.id : -1, \
 		(e) && (e)->hw_intf ? \
 		(e)->hw_intf->idx - INTF_0 : -1, ##__VA_ARGS__)
 
@@ -44,7 +44,7 @@ static bool dpu_encoder_phys_vid_is_master(
 }
 
 static void drm_mode_to_intf_timing_params(
-		const struct dpu_encoder_phys_vid *vid_enc,
+		const struct dpu_encoder_phys *phys_enc,
 		const struct drm_display_mode *mode,
 		struct intf_timing_params *timing)
 {
@@ -92,7 +92,7 @@ static void drm_mode_to_intf_timing_params(
 	timing->hsync_skew = mode->hskew;
 
 	/* DSI controller cannot handle active-low sync signals. */
-	if (vid_enc->hw_intf->cap->type == INTF_DSI) {
+	if (phys_enc->hw_intf->cap->type == INTF_DSI) {
 		timing->hsync_polarity = 0;
 		timing->vsync_polarity = 0;
 	}
@@ -143,11 +143,11 @@ static u32 get_vertical_total(const struct intf_timing_params *timing)
  * lines based on the chip worst case latencies.
  */
 static u32 programmable_fetch_get_num_lines(
-		struct dpu_encoder_phys_vid *vid_enc,
+		struct dpu_encoder_phys *phys_enc,
 		const struct intf_timing_params *timing)
 {
 	u32 worst_case_needed_lines =
-	    vid_enc->hw_intf->cap->prog_fetch_lines_worst_case;
+	    phys_enc->hw_intf->cap->prog_fetch_lines_worst_case;
 	u32 start_of_frame_lines =
 	    timing->v_back_porch + timing->vsync_pulse_width;
 	u32 needed_vfp_lines = worst_case_needed_lines - start_of_frame_lines;
@@ -155,26 +155,26 @@ static u32 programmable_fetch_get_num_lines(
 
 	/* Fetch must be outside active lines, otherwise undefined. */
 	if (start_of_frame_lines >= worst_case_needed_lines) {
-		DPU_DEBUG_VIDENC(vid_enc,
+		DPU_DEBUG_VIDENC(phys_enc,
 				"prog fetch is not needed, large vbp+vsw\n");
 		actual_vfp_lines = 0;
 	} else if (timing->v_front_porch < needed_vfp_lines) {
 		/* Warn fetch needed, but not enough porch in panel config */
 		pr_warn_once
 			("low vbp+vfp may lead to perf issues in some cases\n");
-		DPU_DEBUG_VIDENC(vid_enc,
+		DPU_DEBUG_VIDENC(phys_enc,
 				"less vfp than fetch req, using entire vfp\n");
 		actual_vfp_lines = timing->v_front_porch;
 	} else {
-		DPU_DEBUG_VIDENC(vid_enc, "room in vfp for needed prefetch\n");
+		DPU_DEBUG_VIDENC(phys_enc, "room in vfp for needed prefetch\n");
 		actual_vfp_lines = needed_vfp_lines;
 	}
 
-	DPU_DEBUG_VIDENC(vid_enc,
+	DPU_DEBUG_VIDENC(phys_enc,
 		"v_front_porch %u v_back_porch %u vsync_pulse_width %u\n",
 		timing->v_front_porch, timing->v_back_porch,
 		timing->vsync_pulse_width);
-	DPU_DEBUG_VIDENC(vid_enc,
+	DPU_DEBUG_VIDENC(phys_enc,
 		"wc_lines %u needed_vfp_lines %u actual_vfp_lines %u\n",
 		worst_case_needed_lines, needed_vfp_lines, actual_vfp_lines);
 
@@ -194,8 +194,6 @@ static u32 programmable_fetch_get_num_lines(
 static void programmable_fetch_config(struct dpu_encoder_phys *phys_enc,
 				      const struct intf_timing_params *timing)
 {
-	struct dpu_encoder_phys_vid *vid_enc =
-		to_dpu_encoder_phys_vid(phys_enc);
 	struct intf_prog_fetch f = { 0 };
 	u32 vfp_fetch_lines = 0;
 	u32 horiz_total = 0;
@@ -203,10 +201,10 @@ static void programmable_fetch_config(struct dpu_encoder_phys *phys_enc,
 	u32 vfp_fetch_start_vsync_counter = 0;
 	unsigned long lock_flags;
 
-	if (WARN_ON_ONCE(!vid_enc->hw_intf->ops.setup_prg_fetch))
+	if (WARN_ON_ONCE(!phys_enc->hw_intf->ops.setup_prg_fetch))
 		return;
 
-	vfp_fetch_lines = programmable_fetch_get_num_lines(vid_enc, timing);
+	vfp_fetch_lines = programmable_fetch_get_num_lines(phys_enc, timing);
 	if (vfp_fetch_lines) {
 		vert_total = get_vertical_total(timing);
 		horiz_total = get_horizontal_total(timing);
@@ -216,12 +214,12 @@ static void programmable_fetch_config(struct dpu_encoder_phys *phys_enc,
 		f.fetch_start = vfp_fetch_start_vsync_counter;
 	}
 
-	DPU_DEBUG_VIDENC(vid_enc,
+	DPU_DEBUG_VIDENC(phys_enc,
 		"vfp_fetch_lines %u vfp_fetch_start_vsync_counter %u\n",
 		vfp_fetch_lines, vfp_fetch_start_vsync_counter);
 
 	spin_lock_irqsave(phys_enc->enc_spinlock, lock_flags);
-	vid_enc->hw_intf->ops.setup_prg_fetch(vid_enc->hw_intf, &f);
+	phys_enc->hw_intf->ops.setup_prg_fetch(phys_enc->hw_intf, &f);
 	spin_unlock_irqrestore(phys_enc->enc_spinlock, lock_flags);
 }
 
@@ -231,7 +229,7 @@ static bool dpu_encoder_phys_vid_mode_fixup(
 		struct drm_display_mode *adj_mode)
 {
 	if (phys_enc)
-		DPU_DEBUG_VIDENC(to_dpu_encoder_phys_vid(phys_enc), "\n");
+		DPU_DEBUG_VIDENC(phys_enc, "\n");
 
 	/*
 	 * Modifying mode has consequences when the mode comes back to us
@@ -242,7 +240,6 @@ static bool dpu_encoder_phys_vid_mode_fixup(
 static void dpu_encoder_phys_vid_setup_timing_engine(
 		struct dpu_encoder_phys *phys_enc)
 {
-	struct dpu_encoder_phys_vid *vid_enc;
 	struct drm_display_mode mode;
 	struct intf_timing_params timing_params = { 0 };
 	const struct dpu_format *fmt = NULL;
@@ -256,13 +253,12 @@ static void dpu_encoder_phys_vid_setup_timing_engine(
 	}
 
 	mode = phys_enc->cached_mode;
-	vid_enc = to_dpu_encoder_phys_vid(phys_enc);
-	if (!vid_enc->hw_intf->ops.setup_timing_gen) {
+	if (!phys_enc->hw_intf->ops.setup_timing_gen) {
 		DPU_ERROR("timing engine setup is not supported\n");
 		return;
 	}
 
-	DPU_DEBUG_VIDENC(vid_enc, "enabling mode:\n");
+	DPU_DEBUG_VIDENC(phys_enc, "enabling mode:\n");
 	drm_mode_debug_printmodeline(&mode);
 
 	if (phys_enc->split_role != ENC_ROLE_SOLO) {
@@ -271,32 +267,30 @@ static void dpu_encoder_phys_vid_setup_timing_engine(
 		mode.hsync_start >>= 1;
 		mode.hsync_end >>= 1;
 
-		DPU_DEBUG_VIDENC(vid_enc,
+		DPU_DEBUG_VIDENC(phys_enc,
 			"split_role %d, halve horizontal %d %d %d %d\n",
 			phys_enc->split_role,
 			mode.hdisplay, mode.htotal,
 			mode.hsync_start, mode.hsync_end);
 	}
 
-	drm_mode_to_intf_timing_params(vid_enc, &mode, &timing_params);
+	drm_mode_to_intf_timing_params(phys_enc, &mode, &timing_params);
 
 	fmt = dpu_get_dpu_format(fmt_fourcc);
-	DPU_DEBUG_VIDENC(vid_enc, "fmt_fourcc 0x%X\n", fmt_fourcc);
+	DPU_DEBUG_VIDENC(phys_enc, "fmt_fourcc 0x%X\n", fmt_fourcc);
 
-	intf_cfg.intf = vid_enc->hw_intf->idx;
+	intf_cfg.intf = phys_enc->hw_intf->idx;
 	intf_cfg.intf_mode_sel = DPU_CTL_MODE_SEL_VID;
 	intf_cfg.stream_sel = 0; /* Don't care value for video mode */
 	intf_cfg.mode_3d = dpu_encoder_helper_get_3d_blend_mode(phys_enc);
 
 	spin_lock_irqsave(phys_enc->enc_spinlock, lock_flags);
-	vid_enc->hw_intf->ops.setup_timing_gen(vid_enc->hw_intf,
+	phys_enc->hw_intf->ops.setup_timing_gen(phys_enc->hw_intf,
 			&timing_params, fmt);
 	phys_enc->hw_ctl->ops.setup_intf_cfg(phys_enc->hw_ctl, &intf_cfg);
 	spin_unlock_irqrestore(phys_enc->enc_spinlock, lock_flags);
 
 	programmable_fetch_config(phys_enc, &timing_params);
-
-	vid_enc->timing_params = timing_params;
 }
 
 static void dpu_encoder_phys_vid_vblank_irq(void *arg, int irq_idx)
@@ -353,22 +347,10 @@ static void dpu_encoder_phys_vid_underrun_irq(void *arg, int irq_idx)
 			phys_enc);
 }
 
-static bool _dpu_encoder_phys_is_dual_ctl(struct dpu_encoder_phys *phys_enc)
-{
-	struct dpu_crtc_state *dpu_cstate;
-
-	if (!phys_enc)
-		return false;
-
-	dpu_cstate = to_dpu_crtc_state(phys_enc->parent->crtc->state);
-
-	return dpu_cstate->num_ctls > 1;
-}
-
 static bool dpu_encoder_phys_vid_needs_single_flush(
 		struct dpu_encoder_phys *phys_enc)
 {
-	return (phys_enc && _dpu_encoder_phys_is_dual_ctl(phys_enc));
+	return phys_enc->split_role != ENC_ROLE_SOLO;
 }
 
 static void _dpu_encoder_phys_vid_setup_irq_hw_idx(
@@ -396,19 +378,15 @@ static void dpu_encoder_phys_vid_mode_set(
 		struct drm_display_mode *mode,
 		struct drm_display_mode *adj_mode)
 {
-	struct dpu_encoder_phys_vid *vid_enc;
-
 	if (!phys_enc || !phys_enc->dpu_kms) {
 		DPU_ERROR("invalid encoder/kms\n");
 		return;
 	}
 
-	vid_enc = to_dpu_encoder_phys_vid(phys_enc);
-
 	if (adj_mode) {
 		phys_enc->cached_mode = *adj_mode;
 		drm_mode_debug_printmodeline(adj_mode);
-		DPU_DEBUG_VIDENC(vid_enc, "caching mode:\n");
+		DPU_DEBUG_VIDENC(phys_enc, "caching mode:\n");
 	}
 
 	_dpu_encoder_phys_vid_setup_irq_hw_idx(phys_enc);
@@ -419,7 +397,6 @@ static int dpu_encoder_phys_vid_control_vblank_irq(
 		bool enable)
 {
 	int ret = 0;
-	struct dpu_encoder_phys_vid *vid_enc;
 	int refcount;
 
 	if (!phys_enc) {
@@ -428,7 +405,6 @@ static int dpu_encoder_phys_vid_control_vblank_irq(
 	}
 
 	refcount = atomic_read(&phys_enc->vblank_refcount);
-	vid_enc = to_dpu_encoder_phys_vid(phys_enc);
 
 	/* Slave encoders don't report vblank */
 	if (!dpu_encoder_phys_vid_is_master(phys_enc))
@@ -453,7 +429,7 @@ end:
 	if (ret) {
 		DRM_ERROR("failed: id:%u intf:%d ret:%d enable:%d refcnt:%d\n",
 			  DRMID(phys_enc->parent),
-			  vid_enc->hw_intf->idx - INTF_0, ret, enable,
+			  phys_enc->hw_intf->idx - INTF_0, ret, enable,
 			  refcount);
 	}
 	return ret;
@@ -461,43 +437,17 @@ end:
 
 static void dpu_encoder_phys_vid_enable(struct dpu_encoder_phys *phys_enc)
 {
-	struct msm_drm_private *priv;
-	struct dpu_encoder_phys_vid *vid_enc;
-	struct dpu_rm_hw_iter iter;
 	struct dpu_hw_ctl *ctl;
 	u32 flush_mask = 0;
 
-	if (!phys_enc || !phys_enc->parent || !phys_enc->parent->dev ||
-			!phys_enc->parent->dev->dev_private) {
-		DPU_ERROR("invalid encoder/device\n");
-		return;
-	}
-	priv = phys_enc->parent->dev->dev_private;
-
-	vid_enc = to_dpu_encoder_phys_vid(phys_enc);
 	ctl = phys_enc->hw_ctl;
 
-	dpu_rm_init_hw_iter(&iter, phys_enc->parent->base.id, DPU_HW_BLK_INTF);
-	while (dpu_rm_get_hw(&phys_enc->dpu_kms->rm, &iter)) {
-		struct dpu_hw_intf *hw_intf = (struct dpu_hw_intf *)iter.hw;
-
-		if (hw_intf->idx == phys_enc->intf_idx) {
-			vid_enc->hw_intf = hw_intf;
-			break;
-		}
-	}
-
-	if (!vid_enc->hw_intf) {
-		DPU_ERROR("hw_intf not assigned\n");
-		return;
-	}
-
-	DPU_DEBUG_VIDENC(vid_enc, "\n");
+	DPU_DEBUG_VIDENC(phys_enc, "\n");
 
-	if (WARN_ON(!vid_enc->hw_intf->ops.enable_timing))
+	if (WARN_ON(!phys_enc->hw_intf->ops.enable_timing))
 		return;
 
-	dpu_encoder_helper_split_config(phys_enc, vid_enc->hw_intf->idx);
+	dpu_encoder_helper_split_config(phys_enc, phys_enc->hw_intf->idx);
 
 	dpu_encoder_phys_vid_setup_timing_engine(phys_enc);
 
@@ -510,12 +460,13 @@ static void dpu_encoder_phys_vid_enable(struct dpu_encoder_phys *phys_enc)
 		!dpu_encoder_phys_vid_is_master(phys_enc))
 		goto skip_flush;
 
-	ctl->ops.get_bitmask_intf(ctl, &flush_mask, vid_enc->hw_intf->idx);
+	ctl->ops.get_bitmask_intf(ctl, &flush_mask, phys_enc->hw_intf->idx);
 	ctl->ops.update_pending_flush(ctl, flush_mask);
 
 skip_flush:
-	DPU_DEBUG_VIDENC(vid_enc, "update pending flush ctl %d flush_mask %x\n",
-		ctl->idx - CTL_0, flush_mask);
+	DPU_DEBUG_VIDENC(phys_enc,
+			 "update pending flush ctl %d flush_mask %x\n",
+			 ctl->idx - CTL_0, flush_mask);
 
 	/* ctl_flush & timing engine enable will be triggered by framework */
 	if (phys_enc->enable_state == DPU_ENC_DISABLED)
@@ -524,16 +475,13 @@ skip_flush:
 
 static void dpu_encoder_phys_vid_destroy(struct dpu_encoder_phys *phys_enc)
 {
-	struct dpu_encoder_phys_vid *vid_enc;
-
 	if (!phys_enc) {
 		DPU_ERROR("invalid encoder\n");
 		return;
 	}
 
-	vid_enc = to_dpu_encoder_phys_vid(phys_enc);
-	DPU_DEBUG_VIDENC(vid_enc, "\n");
-	kfree(vid_enc);
+	DPU_DEBUG_VIDENC(phys_enc, "\n");
+	kfree(phys_enc);
 }
 
 static void dpu_encoder_phys_vid_get_hw_resources(
@@ -589,7 +537,6 @@ static int dpu_encoder_phys_vid_wait_for_vblank(
 static void dpu_encoder_phys_vid_prepare_for_kickoff(
 		struct dpu_encoder_phys *phys_enc)
 {
-	struct dpu_encoder_phys_vid *vid_enc;
 	struct dpu_hw_ctl *ctl;
 	int rc;
 
@@ -597,7 +544,6 @@ static void dpu_encoder_phys_vid_prepare_for_kickoff(
 		DPU_ERROR("invalid encoder/parameters\n");
 		return;
 	}
-	vid_enc = to_dpu_encoder_phys_vid(phys_enc);
 
 	ctl = phys_enc->hw_ctl;
 	if (!ctl || !ctl->ops.wait_reset_status)
@@ -609,7 +555,7 @@ static void dpu_encoder_phys_vid_prepare_for_kickoff(
 	 */
 	rc = ctl->ops.wait_reset_status(ctl);
 	if (rc) {
-		DPU_ERROR_VIDENC(vid_enc, "ctl %d reset failure: %d\n",
+		DPU_ERROR_VIDENC(phys_enc, "ctl %d reset failure: %d\n",
 				ctl->idx, rc);
 		dpu_encoder_helper_unregister_irq(phys_enc, INTR_IDX_VSYNC);
 	}
@@ -618,7 +564,6 @@ static void dpu_encoder_phys_vid_prepare_for_kickoff(
 static void dpu_encoder_phys_vid_disable(struct dpu_encoder_phys *phys_enc)
 {
 	struct msm_drm_private *priv;
-	struct dpu_encoder_phys_vid *vid_enc;
 	unsigned long lock_flags;
 	int ret;
 
@@ -629,16 +574,13 @@ static void dpu_encoder_phys_vid_disable(struct dpu_encoder_phys *phys_enc)
 	}
 	priv = phys_enc->parent->dev->dev_private;
 
-	vid_enc = to_dpu_encoder_phys_vid(phys_enc);
-	if (!vid_enc->hw_intf || !phys_enc->hw_ctl) {
+	if (!phys_enc->hw_intf || !phys_enc->hw_ctl) {
 		DPU_ERROR("invalid hw_intf %d hw_ctl %d\n",
-				vid_enc->hw_intf != 0, phys_enc->hw_ctl != 0);
+				phys_enc->hw_intf != 0, phys_enc->hw_ctl != 0);
 		return;
 	}
 
-	DPU_DEBUG_VIDENC(vid_enc, "\n");
-
-	if (WARN_ON(!vid_enc->hw_intf->ops.enable_timing))
+	if (WARN_ON(!phys_enc->hw_intf->ops.enable_timing))
 		return;
 
 	if (phys_enc->enable_state == DPU_ENC_DISABLED) {
@@ -647,7 +589,7 @@ static void dpu_encoder_phys_vid_disable(struct dpu_encoder_phys *phys_enc)
 	}
 
 	spin_lock_irqsave(phys_enc->enc_spinlock, lock_flags);
-	vid_enc->hw_intf->ops.enable_timing(vid_enc->hw_intf, 0);
+	phys_enc->hw_intf->ops.enable_timing(phys_enc->hw_intf, 0);
 	if (dpu_encoder_phys_vid_is_master(phys_enc))
 		dpu_encoder_phys_inc_pending(phys_enc);
 	spin_unlock_irqrestore(phys_enc->enc_spinlock, lock_flags);
@@ -666,7 +608,7 @@ static void dpu_encoder_phys_vid_disable(struct dpu_encoder_phys *phys_enc)
 			atomic_set(&phys_enc->pending_kickoff_cnt, 0);
 			DRM_ERROR("wait disable failed: id:%u intf:%d ret:%d\n",
 				  DRMID(phys_enc->parent),
-				  vid_enc->hw_intf->idx - INTF_0, ret);
+				  phys_enc->hw_intf->idx - INTF_0, ret);
 		}
 	}
 
@@ -677,25 +619,21 @@ static void dpu_encoder_phys_vid_handle_post_kickoff(
 		struct dpu_encoder_phys *phys_enc)
 {
 	unsigned long lock_flags;
-	struct dpu_encoder_phys_vid *vid_enc;
 
 	if (!phys_enc) {
 		DPU_ERROR("invalid encoder\n");
 		return;
 	}
 
-	vid_enc = to_dpu_encoder_phys_vid(phys_enc);
-	DPU_DEBUG_VIDENC(vid_enc, "enable_state %d\n", phys_enc->enable_state);
-
 	/*
 	 * Video mode must flush CTL before enabling timing engine
 	 * Video encoders need to turn on their interfaces now
 	 */
 	if (phys_enc->enable_state == DPU_ENC_ENABLING) {
 		trace_dpu_enc_phys_vid_post_kickoff(DRMID(phys_enc->parent),
-				    vid_enc->hw_intf->idx - INTF_0);
+				    phys_enc->hw_intf->idx - INTF_0);
 		spin_lock_irqsave(phys_enc->enc_spinlock, lock_flags);
-		vid_enc->hw_intf->ops.enable_timing(vid_enc->hw_intf, 1);
+		phys_enc->hw_intf->ops.enable_timing(phys_enc->hw_intf, 1);
 		spin_unlock_irqrestore(phys_enc->enc_spinlock, lock_flags);
 		phys_enc->enable_state = DPU_ENC_ENABLED;
 	}
@@ -704,16 +642,13 @@ static void dpu_encoder_phys_vid_handle_post_kickoff(
 static void dpu_encoder_phys_vid_irq_control(struct dpu_encoder_phys *phys_enc,
 		bool enable)
 {
-	struct dpu_encoder_phys_vid *vid_enc;
 	int ret;
 
 	if (!phys_enc)
 		return;
 
-	vid_enc = to_dpu_encoder_phys_vid(phys_enc);
-
 	trace_dpu_enc_phys_vid_irq_ctrl(DRMID(phys_enc->parent),
-			    vid_enc->hw_intf->idx - INTF_0,
+			    phys_enc->hw_intf->idx - INTF_0,
 			    enable,
 			    atomic_read(&phys_enc->vblank_refcount));
 
@@ -732,19 +667,16 @@ static void dpu_encoder_phys_vid_irq_control(struct dpu_encoder_phys *phys_enc,
 static int dpu_encoder_phys_vid_get_line_count(
 		struct dpu_encoder_phys *phys_enc)
 {
-	struct dpu_encoder_phys_vid *vid_enc;
-
 	if (!phys_enc)
 		return -EINVAL;
 
 	if (!dpu_encoder_phys_vid_is_master(phys_enc))
 		return -EINVAL;
 
-	vid_enc = to_dpu_encoder_phys_vid(phys_enc);
-	if (!vid_enc->hw_intf || !vid_enc->hw_intf->ops.get_line_count)
+	if (!phys_enc->hw_intf || !phys_enc->hw_intf->ops.get_line_count)
 		return -EINVAL;
 
-	return vid_enc->hw_intf->ops.get_line_count(vid_enc->hw_intf);
+	return phys_enc->hw_intf->ops.get_line_count(phys_enc->hw_intf);
 }
 
 static void dpu_encoder_phys_vid_init_ops(struct dpu_encoder_phys_ops *ops)
@@ -771,7 +703,6 @@ struct dpu_encoder_phys *dpu_encoder_phys_vid_init(
 		struct dpu_enc_phys_init_params *p)
 {
 	struct dpu_encoder_phys *phys_enc = NULL;
-	struct dpu_encoder_phys_vid *vid_enc = NULL;
 	struct dpu_encoder_irq *irq;
 	int i, ret = 0;
 
@@ -780,18 +711,16 @@ struct dpu_encoder_phys *dpu_encoder_phys_vid_init(
 		goto fail;
 	}
 
-	vid_enc = kzalloc(sizeof(*vid_enc), GFP_KERNEL);
-	if (!vid_enc) {
+	phys_enc = kzalloc(sizeof(*phys_enc), GFP_KERNEL);
+	if (!phys_enc) {
 		ret = -ENOMEM;
 		goto fail;
 	}
 
-	phys_enc = &vid_enc->base;
-
 	phys_enc->hw_mdptop = p->dpu_kms->hw_mdp;
 	phys_enc->intf_idx = p->intf_idx;
 
-	DPU_DEBUG_VIDENC(vid_enc, "\n");
+	DPU_DEBUG_VIDENC(phys_enc, "\n");
 
 	dpu_encoder_phys_vid_init_ops(&phys_enc->ops);
 	phys_enc->parent = p->parent;
@@ -825,13 +754,13 @@ struct dpu_encoder_phys *dpu_encoder_phys_vid_init(
 	init_waitqueue_head(&phys_enc->pending_kickoff_wq);
 	phys_enc->enable_state = DPU_ENC_DISABLED;
 
-	DPU_DEBUG_VIDENC(vid_enc, "created intf idx:%d\n", p->intf_idx);
+	DPU_DEBUG_VIDENC(phys_enc, "created intf idx:%d\n", p->intf_idx);
 
 	return phys_enc;
 
 fail:
 	DPU_ERROR("failed to create encoder\n");
-	if (vid_enc)
+	if (phys_enc)
 		dpu_encoder_phys_vid_destroy(phys_enc);
 
 	return ERR_PTR(ret);
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.h
index ac75cfc267f4..31e9ef96ca5d 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.h
@@ -73,9 +73,6 @@
 
 #define DPU_NAME_SIZE  12
 
-/* timeout in frames waiting for frame done */
-#define DPU_FRAME_DONE_TIMEOUT	60
-
 /*
  * struct dpu_irq_callback - IRQ callback handlers
  * @list: list to callback
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c
index b01183b309b9..da1f727d7495 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c
@@ -387,7 +387,7 @@ static void _dpu_plane_set_ot_limit(struct drm_plane *plane,
 	ot_params.width = drm_rect_width(&pdpu->pipe_cfg.src_rect);
 	ot_params.height = drm_rect_height(&pdpu->pipe_cfg.src_rect);
 	ot_params.is_wfd = !pdpu->is_rt_pipe;
-	ot_params.frame_rate = crtc->mode.vrefresh;
+	ot_params.frame_rate = drm_mode_vrefresh(&crtc->mode);
 	ot_params.vbif_idx = VBIF_RT;
 	ot_params.clk_ctrl = pdpu->pipe_hw->cap->clk_ctrl;
 	ot_params.rd = true;
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_cmd_encoder.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_cmd_encoder.c
index 9bf9d6065c55..7b9edc21bc2c 100644
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_cmd_encoder.c
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_cmd_encoder.c
@@ -59,10 +59,10 @@ static int pingpong_tearcheck_setup(struct drm_encoder *encoder,
 		return -EINVAL;
 	}
 
-	total_lines_x100 = mode->vtotal * mode->vrefresh;
+	total_lines_x100 = mode->vtotal * drm_mode_vrefresh(mode);
 	if (!total_lines_x100) {
 		DRM_DEV_ERROR(dev, "%s: vtotal(%d) or vrefresh(%d) is 0\n",
-				__func__, mode->vtotal, mode->vrefresh);
+			      __func__, mode->vtotal, drm_mode_vrefresh(mode));
 		return -EINVAL;
 	}
 
diff --git a/drivers/gpu/drm/msm/msm_debugfs.c b/drivers/gpu/drm/msm/msm_debugfs.c
index fb423d309e91..67ef300559cf 100644
--- a/drivers/gpu/drm/msm/msm_debugfs.c
+++ b/drivers/gpu/drm/msm/msm_debugfs.c
@@ -75,7 +75,7 @@ static int msm_gpu_open(struct inode *inode, struct file *file)
 	struct msm_gpu_show_priv *show_priv;
 	int ret;
 
-	if (!gpu)
+	if (!gpu || !gpu->funcs->gpu_state_get)
 		return -ENODEV;
 
 	show_priv = kmalloc(sizeof(*show_priv), GFP_KERNEL);
diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c
index 0bdd93648761..31deb87abfc6 100644
--- a/drivers/gpu/drm/msm/msm_drv.c
+++ b/drivers/gpu/drm/msm/msm_drv.c
@@ -39,9 +39,10 @@
  *           MSM_GEM_INFO ioctl.
  * - 1.4.0 - softpin, MSM_RELOC_BO_DUMP, and GEM_INFO support to set/get
  *           GEM object's debug name
+ * - 1.5.0 - Add SUBMITQUERY_QUERY ioctl
  */
 #define MSM_VERSION_MAJOR	1
-#define MSM_VERSION_MINOR	4
+#define MSM_VERSION_MINOR	5
 #define MSM_VERSION_PATCHLEVEL	0
 
 static const struct drm_mode_config_funcs mode_config_funcs = {
@@ -457,6 +458,9 @@ static int msm_drm_init(struct device *dev, struct drm_driver *drv)
 
 	priv->wq = alloc_ordered_workqueue("msm", 0);
 
+	INIT_WORK(&priv->free_work, msm_gem_free_work);
+	init_llist_head(&priv->free_list);
+
 	INIT_LIST_HEAD(&priv->inactive_list);
 
 	drm_mode_config_init(ddev);
@@ -964,6 +968,11 @@ static int msm_ioctl_submitqueue_new(struct drm_device *dev, void *data,
 		args->flags, &args->id);
 }
 
+static int msm_ioctl_submitqueue_query(struct drm_device *dev, void *data,
+		struct drm_file *file)
+{
+	return msm_submitqueue_query(dev, file->driver_priv, data);
+}
 
 static int msm_ioctl_submitqueue_close(struct drm_device *dev, void *data,
 		struct drm_file *file)
@@ -984,6 +993,7 @@ static const struct drm_ioctl_desc msm_ioctls[] = {
 	DRM_IOCTL_DEF_DRV(MSM_GEM_MADVISE,  msm_ioctl_gem_madvise,  DRM_AUTH|DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF_DRV(MSM_SUBMITQUEUE_NEW,   msm_ioctl_submitqueue_new,   DRM_AUTH|DRM_RENDER_ALLOW),
 	DRM_IOCTL_DEF_DRV(MSM_SUBMITQUEUE_CLOSE, msm_ioctl_submitqueue_close, DRM_AUTH|DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(MSM_SUBMITQUEUE_QUERY, msm_ioctl_submitqueue_query, DRM_AUTH|DRM_RENDER_ALLOW),
 };
 
 static const struct vm_operations_struct vm_ops = {
@@ -1019,7 +1029,7 @@ static struct drm_driver msm_driver = {
 	.irq_uninstall      = msm_irq_uninstall,
 	.enable_vblank      = msm_enable_vblank,
 	.disable_vblank     = msm_disable_vblank,
-	.gem_free_object    = msm_gem_free_object,
+	.gem_free_object_unlocked = msm_gem_free_object,
 	.gem_vm_ops         = &vm_ops,
 	.dumb_create        = msm_gem_dumb_create,
 	.dumb_map_offset    = msm_gem_dumb_map_offset,
@@ -1027,7 +1037,6 @@ static struct drm_driver msm_driver = {
 	.prime_fd_to_handle = drm_gem_prime_fd_to_handle,
 	.gem_prime_export   = drm_gem_prime_export,
 	.gem_prime_import   = drm_gem_prime_import,
-	.gem_prime_res_obj  = msm_gem_prime_res_obj,
 	.gem_prime_pin      = msm_gem_prime_pin,
 	.gem_prime_unpin    = msm_gem_prime_unpin,
 	.gem_prime_get_sg_table = msm_gem_prime_get_sg_table,
diff --git a/drivers/gpu/drm/msm/msm_drv.h b/drivers/gpu/drm/msm/msm_drv.h
index c56dade2c1dc..eb33d2d00d77 100644
--- a/drivers/gpu/drm/msm/msm_drv.h
+++ b/drivers/gpu/drm/msm/msm_drv.h
@@ -185,6 +185,10 @@ struct msm_drm_private {
 	/* list of GEM objects: */
 	struct list_head inactive_list;
 
+	/* worker for delayed free of objects: */
+	struct work_struct free_work;
+	struct llist_head free_list;
+
 	struct workqueue_struct *wq;
 
 	unsigned int num_planes;
@@ -292,7 +296,6 @@ struct sg_table *msm_gem_prime_get_sg_table(struct drm_gem_object *obj);
 void *msm_gem_prime_vmap(struct drm_gem_object *obj);
 void msm_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr);
 int msm_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma);
-struct reservation_object *msm_gem_prime_res_obj(struct drm_gem_object *obj);
 struct drm_gem_object *msm_gem_prime_import_sg_table(struct drm_device *dev,
 		struct dma_buf_attachment *attach, struct sg_table *sg);
 int msm_gem_prime_pin(struct drm_gem_object *obj);
@@ -325,6 +328,7 @@ void msm_gem_kernel_put(struct drm_gem_object *bo,
 		struct msm_gem_address_space *aspace, bool locked);
 struct drm_gem_object *msm_gem_import(struct drm_device *dev,
 		struct dma_buf *dmabuf, struct sg_table *sgt);
+void msm_gem_free_work(struct work_struct *work);
 
 __printf(2, 3)
 void msm_gem_object_set_name(struct drm_gem_object *bo, const char *fmt, ...);
@@ -420,6 +424,8 @@ struct msm_gpu_submitqueue *msm_submitqueue_get(struct msm_file_private *ctx,
 		u32 id);
 int msm_submitqueue_create(struct drm_device *drm, struct msm_file_private *ctx,
 		u32 prio, u32 flags, u32 *id);
+int msm_submitqueue_query(struct drm_device *drm, struct msm_file_private *ctx,
+		struct drm_msm_submitqueue_query *args);
 int msm_submitqueue_remove(struct msm_file_private *ctx, u32 id);
 void msm_submitqueue_close(struct msm_file_private *ctx);
 
diff --git a/drivers/gpu/drm/msm/msm_fbdev.c b/drivers/gpu/drm/msm/msm_fbdev.c
index c03e860ba737..d088299babf3 100644
--- a/drivers/gpu/drm/msm/msm_fbdev.c
+++ b/drivers/gpu/drm/msm/msm_fbdev.c
@@ -122,13 +122,9 @@ static int msm_fbdev_create(struct drm_fb_helper *helper,
 	fbdev->fb = fb;
 	helper->fb = fb;
 
-	fbi->par = helper;
 	fbi->fbops = &msm_fb_ops;
 
-	strcpy(fbi->fix.id, "msm");
-
-	drm_fb_helper_fill_fix(fbi, fb->pitches[0], fb->format->depth);
-	drm_fb_helper_fill_var(fbi, helper, sizes->fb_width, sizes->fb_height);
+	drm_fb_helper_fill_info(fbi, helper, sizes);
 
 	dev->mode_config.fb_base = paddr;
 
diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c
index 18ca651ab942..31d5a744d84f 100644
--- a/drivers/gpu/drm/msm/msm_gem.c
+++ b/drivers/gpu/drm/msm/msm_gem.c
@@ -672,14 +672,13 @@ void msm_gem_vunmap(struct drm_gem_object *obj, enum msm_gem_lock subclass)
 int msm_gem_sync_object(struct drm_gem_object *obj,
 		struct msm_fence_context *fctx, bool exclusive)
 {
-	struct msm_gem_object *msm_obj = to_msm_bo(obj);
 	struct reservation_object_list *fobj;
 	struct dma_fence *fence;
 	int i, ret;
 
-	fobj = reservation_object_get_list(msm_obj->resv);
+	fobj = reservation_object_get_list(obj->resv);
 	if (!fobj || (fobj->shared_count == 0)) {
-		fence = reservation_object_get_excl(msm_obj->resv);
+		fence = reservation_object_get_excl(obj->resv);
 		/* don't need to wait on our own fences, since ring is fifo */
 		if (fence && (fence->context != fctx->context)) {
 			ret = dma_fence_wait(fence, true);
@@ -693,7 +692,7 @@ int msm_gem_sync_object(struct drm_gem_object *obj,
 
 	for (i = 0; i < fobj->shared_count; i++) {
 		fence = rcu_dereference_protected(fobj->shared[i],
-						reservation_object_held(msm_obj->resv));
+						reservation_object_held(obj->resv));
 		if (fence->context != fctx->context) {
 			ret = dma_fence_wait(fence, true);
 			if (ret)
@@ -711,9 +710,9 @@ void msm_gem_move_to_active(struct drm_gem_object *obj,
 	WARN_ON(msm_obj->madv != MSM_MADV_WILLNEED);
 	msm_obj->gpu = gpu;
 	if (exclusive)
-		reservation_object_add_excl_fence(msm_obj->resv, fence);
+		reservation_object_add_excl_fence(obj->resv, fence);
 	else
-		reservation_object_add_shared_fence(msm_obj->resv, fence);
+		reservation_object_add_shared_fence(obj->resv, fence);
 	list_del_init(&msm_obj->mm_list);
 	list_add_tail(&msm_obj->mm_list, &gpu->active_list);
 }
@@ -733,13 +732,12 @@ void msm_gem_move_to_inactive(struct drm_gem_object *obj)
 
 int msm_gem_cpu_prep(struct drm_gem_object *obj, uint32_t op, ktime_t *timeout)
 {
-	struct msm_gem_object *msm_obj = to_msm_bo(obj);
 	bool write = !!(op & MSM_PREP_WRITE);
 	unsigned long remain =
 		op & MSM_PREP_NOSYNC ? 0 : timeout_to_jiffies(timeout);
 	long ret;
 
-	ret = reservation_object_wait_timeout_rcu(msm_obj->resv, write,
+	ret = reservation_object_wait_timeout_rcu(obj->resv, write,
 						  true,  remain);
 	if (ret == 0)
 		return remain == 0 ? -EBUSY : -ETIMEDOUT;
@@ -771,7 +769,7 @@ static void describe_fence(struct dma_fence *fence, const char *type,
 void msm_gem_describe(struct drm_gem_object *obj, struct seq_file *m)
 {
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
-	struct reservation_object *robj = msm_obj->resv;
+	struct reservation_object *robj = obj->resv;
 	struct reservation_object_list *fobj;
 	struct dma_fence *fence;
 	struct msm_gem_vma *vma;
@@ -853,8 +851,18 @@ void msm_gem_describe_objects(struct list_head *list, struct seq_file *m)
 /* don't call directly!  Use drm_gem_object_put() and friends */
 void msm_gem_free_object(struct drm_gem_object *obj)
 {
-	struct drm_device *dev = obj->dev;
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
+	struct drm_device *dev = obj->dev;
+	struct msm_drm_private *priv = dev->dev_private;
+
+	if (llist_add(&msm_obj->freed, &priv->free_list))
+		queue_work(priv->wq, &priv->free_work);
+}
+
+static void free_object(struct msm_gem_object *msm_obj)
+{
+	struct drm_gem_object *obj = &msm_obj->base;
+	struct drm_device *dev = obj->dev;
 
 	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
 
@@ -883,15 +891,35 @@ void msm_gem_free_object(struct drm_gem_object *obj)
 		put_pages(obj);
 	}
 
-	if (msm_obj->resv == &msm_obj->_resv)
-		reservation_object_fini(msm_obj->resv);
-
 	drm_gem_object_release(obj);
 
 	mutex_unlock(&msm_obj->lock);
 	kfree(msm_obj);
 }
 
+void msm_gem_free_work(struct work_struct *work)
+{
+	struct msm_drm_private *priv =
+		container_of(work, struct msm_drm_private, free_work);
+	struct drm_device *dev = priv->dev;
+	struct llist_node *freed;
+	struct msm_gem_object *msm_obj, *next;
+
+	while ((freed = llist_del_all(&priv->free_list))) {
+
+		mutex_lock(&dev->struct_mutex);
+
+		llist_for_each_entry_safe(msm_obj, next,
+					  freed, freed)
+			free_object(msm_obj);
+
+		mutex_unlock(&dev->struct_mutex);
+
+		if (need_resched())
+			break;
+	}
+}
+
 /* convenience method to construct a GEM buffer object, and userspace handle */
 int msm_gem_new_handle(struct drm_device *dev, struct drm_file *file,
 		uint32_t size, uint32_t flags, uint32_t *handle,
@@ -945,12 +973,8 @@ static int msm_gem_new_impl(struct drm_device *dev,
 	msm_obj->flags = flags;
 	msm_obj->madv = MSM_MADV_WILLNEED;
 
-	if (resv) {
-		msm_obj->resv = resv;
-	} else {
-		msm_obj->resv = &msm_obj->_resv;
-		reservation_object_init(msm_obj->resv);
-	}
+	if (resv)
+		msm_obj->base.resv = resv;
 
 	INIT_LIST_HEAD(&msm_obj->submit_entry);
 	INIT_LIST_HEAD(&msm_obj->vmas);
@@ -1026,6 +1050,13 @@ static struct drm_gem_object *_msm_gem_new(struct drm_device *dev,
 		ret = drm_gem_object_init(dev, obj, size);
 		if (ret)
 			goto fail;
+		/*
+		 * Our buffers are kept pinned, so allocating them from the
+		 * MOVABLE zone is a really bad idea, and conflicts with CMA.
+		 * See comments above new_inode() why this is required _and_
+		 * expected if you're going to pin these pages.
+		 */
+		mapping_set_gfp_mask(obj->filp->f_mapping, GFP_HIGHUSER);
 	}
 
 	return obj;
diff --git a/drivers/gpu/drm/msm/msm_gem.h b/drivers/gpu/drm/msm/msm_gem.h
index 2064fac871b8..c5ac781dffee 100644
--- a/drivers/gpu/drm/msm/msm_gem.h
+++ b/drivers/gpu/drm/msm/msm_gem.h
@@ -84,6 +84,8 @@ struct msm_gem_object {
 
 	struct list_head vmas;    /* list of msm_gem_vma */
 
+	struct llist_node freed;
+
 	/* normally (resv == &_resv) except for imported bo's */
 	struct reservation_object *resv;
 	struct reservation_object _resv;
@@ -133,6 +135,7 @@ enum msm_gem_lock {
 
 void msm_gem_purge(struct drm_gem_object *obj, enum msm_gem_lock subclass);
 void msm_gem_vunmap(struct drm_gem_object *obj, enum msm_gem_lock subclass);
+void msm_gem_free_work(struct work_struct *work);
 
 /* Created per submit-ioctl, to track bo's and cmdstream bufs, etc,
  * associated with the cmdstream submission for synchronization (and
@@ -163,7 +166,10 @@ struct msm_gem_submit {
 	} *cmd;  /* array of size nr_cmds */
 	struct {
 		uint32_t flags;
-		struct msm_gem_object *obj;
+		union {
+			struct msm_gem_object *obj;
+			uint32_t handle;
+		};
 		uint64_t iova;
 	} bos[0];
 };
diff --git a/drivers/gpu/drm/msm/msm_gem_prime.c b/drivers/gpu/drm/msm/msm_gem_prime.c
index 13403c6da6c7..60bb290700ce 100644
--- a/drivers/gpu/drm/msm/msm_gem_prime.c
+++ b/drivers/gpu/drm/msm/msm_gem_prime.c
@@ -70,10 +70,3 @@ void msm_gem_prime_unpin(struct drm_gem_object *obj)
 	if (!obj->import_attach)
 		msm_gem_put_pages(obj);
 }
-
-struct reservation_object *msm_gem_prime_res_obj(struct drm_gem_object *obj)
-{
-	struct msm_gem_object *msm_obj = to_msm_bo(obj);
-
-	return msm_obj->resv;
-}
diff --git a/drivers/gpu/drm/msm/msm_gem_submit.c b/drivers/gpu/drm/msm/msm_gem_submit.c
index 12b983fc0b56..1b681306aca3 100644
--- a/drivers/gpu/drm/msm/msm_gem_submit.c
+++ b/drivers/gpu/drm/msm/msm_gem_submit.c
@@ -74,27 +74,14 @@ void msm_gem_submit_free(struct msm_gem_submit *submit)
 	kfree(submit);
 }
 
-static inline unsigned long __must_check
-copy_from_user_inatomic(void *to, const void __user *from, unsigned long n)
-{
-	if (access_ok(from, n))
-		return __copy_from_user_inatomic(to, from, n);
-	return -EFAULT;
-}
-
 static int submit_lookup_objects(struct msm_gem_submit *submit,
 		struct drm_msm_gem_submit *args, struct drm_file *file)
 {
 	unsigned i;
 	int ret = 0;
 
-	spin_lock(&file->table_lock);
-	pagefault_disable();
-
 	for (i = 0; i < args->nr_bos; i++) {
 		struct drm_msm_gem_submit_bo submit_bo;
-		struct drm_gem_object *obj;
-		struct msm_gem_object *msm_obj;
 		void __user *userptr =
 			u64_to_user_ptr(args->bos + (i * sizeof(submit_bo)));
 
@@ -103,15 +90,10 @@ static int submit_lookup_objects(struct msm_gem_submit *submit,
 		 */
 		submit->bos[i].flags = 0;
 
-		if (copy_from_user_inatomic(&submit_bo, userptr, sizeof(submit_bo))) {
-			pagefault_enable();
-			spin_unlock(&file->table_lock);
-			if (copy_from_user(&submit_bo, userptr, sizeof(submit_bo))) {
-				ret = -EFAULT;
-				goto out;
-			}
-			spin_lock(&file->table_lock);
-			pagefault_disable();
+		if (copy_from_user(&submit_bo, userptr, sizeof(submit_bo))) {
+			ret = -EFAULT;
+			i = 0;
+			goto out;
 		}
 
 /* at least one of READ and/or WRITE flags should be set: */
@@ -121,19 +103,28 @@ static int submit_lookup_objects(struct msm_gem_submit *submit,
 			!(submit_bo.flags & MANDATORY_FLAGS)) {
 			DRM_ERROR("invalid flags: %x\n", submit_bo.flags);
 			ret = -EINVAL;
-			goto out_unlock;
+			i = 0;
+			goto out;
 		}
 
+		submit->bos[i].handle = submit_bo.handle;
 		submit->bos[i].flags = submit_bo.flags;
 		/* in validate_objects() we figure out if this is true: */
 		submit->bos[i].iova  = submit_bo.presumed;
+	}
+
+	spin_lock(&file->table_lock);
+
+	for (i = 0; i < args->nr_bos; i++) {
+		struct drm_gem_object *obj;
+		struct msm_gem_object *msm_obj;
 
 		/* normally use drm_gem_object_lookup(), but for bulk lookup
 		 * all under single table_lock just hit object_idr directly:
 		 */
-		obj = idr_find(&file->object_idr, submit_bo.handle);
+		obj = idr_find(&file->object_idr, submit->bos[i].handle);
 		if (!obj) {
-			DRM_ERROR("invalid handle %u at index %u\n", submit_bo.handle, i);
+			DRM_ERROR("invalid handle %u at index %u\n", submit->bos[i].handle, i);
 			ret = -EINVAL;
 			goto out_unlock;
 		}
@@ -142,7 +133,7 @@ static int submit_lookup_objects(struct msm_gem_submit *submit,
 
 		if (!list_empty(&msm_obj->submit_entry)) {
 			DRM_ERROR("handle %u at index %u already on submit list\n",
-					submit_bo.handle, i);
+					submit->bos[i].handle, i);
 			ret = -EINVAL;
 			goto out_unlock;
 		}
@@ -155,7 +146,6 @@ static int submit_lookup_objects(struct msm_gem_submit *submit,
 	}
 
 out_unlock:
-	pagefault_enable();
 	spin_unlock(&file->table_lock);
 
 out:
@@ -173,7 +163,7 @@ static void submit_unlock_unpin_bo(struct msm_gem_submit *submit,
 		msm_gem_unpin_iova(&msm_obj->base, submit->gpu->aspace);
 
 	if (submit->bos[i].flags & BO_LOCKED)
-		ww_mutex_unlock(&msm_obj->resv->lock);
+		ww_mutex_unlock(&msm_obj->base.resv->lock);
 
 	if (backoff && !(submit->bos[i].flags & BO_VALID))
 		submit->bos[i].iova = 0;
@@ -196,7 +186,7 @@ retry:
 		contended = i;
 
 		if (!(submit->bos[i].flags & BO_LOCKED)) {
-			ret = ww_mutex_lock_interruptible(&msm_obj->resv->lock,
+			ret = ww_mutex_lock_interruptible(&msm_obj->base.resv->lock,
 					&submit->ticket);
 			if (ret)
 				goto fail;
@@ -218,7 +208,7 @@ fail:
 	if (ret == -EDEADLK) {
 		struct msm_gem_object *msm_obj = submit->bos[contended].obj;
 		/* we lost out in a seqno race, lock and retry.. */
-		ret = ww_mutex_lock_slow_interruptible(&msm_obj->resv->lock,
+		ret = ww_mutex_lock_slow_interruptible(&msm_obj->base.resv->lock,
 				&submit->ticket);
 		if (!ret) {
 			submit->bos[contended].flags |= BO_LOCKED;
@@ -244,7 +234,7 @@ static int submit_fence_sync(struct msm_gem_submit *submit, bool no_implicit)
 			 * strange place to call it.  OTOH this is a
 			 * convenient can-fail point to hook it in.
 			 */
-			ret = reservation_object_reserve_shared(msm_obj->resv,
+			ret = reservation_object_reserve_shared(msm_obj->base.resv,
 								1);
 			if (ret)
 				return ret;
diff --git a/drivers/gpu/drm/msm/msm_gem_vma.c b/drivers/gpu/drm/msm/msm_gem_vma.c
index 49c04829cf34..fcf7a83f0e6f 100644
--- a/drivers/gpu/drm/msm/msm_gem_vma.c
+++ b/drivers/gpu/drm/msm/msm_gem_vma.c
@@ -85,7 +85,7 @@ msm_gem_map_vma(struct msm_gem_address_space *aspace,
 
 	vma->mapped = true;
 
-	if (aspace->mmu)
+	if (aspace && aspace->mmu)
 		ret = aspace->mmu->funcs->map(aspace->mmu, vma->iova, sgt,
 				size, prot);
 
diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c
index 10babd18e286..bf4ee2766431 100644
--- a/drivers/gpu/drm/msm/msm_gpu.c
+++ b/drivers/gpu/drm/msm/msm_gpu.c
@@ -443,24 +443,15 @@ static void recover_worker(struct work_struct *work)
 	if (submit) {
 		struct task_struct *task;
 
+		/* Increment the fault counts */
+		gpu->global_faults++;
+		submit->queue->faults++;
+
 		task = get_pid_task(submit->pid, PIDTYPE_PID);
 		if (task) {
 			comm = kstrdup(task->comm, GFP_KERNEL);
-
-			/*
-			 * So slightly annoying, in other paths like
-			 * mmap'ing gem buffers, mmap_sem is acquired
-			 * before struct_mutex, which means we can't
-			 * hold struct_mutex across the call to
-			 * get_cmdline().  But submits are retired
-			 * from the same in-order workqueue, so we can
-			 * safely drop the lock here without worrying
-			 * about the submit going away.
-			 */
-			mutex_unlock(&dev->struct_mutex);
 			cmd = kstrdup_quotable_cmdline(task, GFP_KERNEL);
 			put_task_struct(task);
-			mutex_lock(&dev->struct_mutex);
 		}
 
 		if (comm && cmd) {
diff --git a/drivers/gpu/drm/msm/msm_gpu.h b/drivers/gpu/drm/msm/msm_gpu.h
index 6241986bab51..f2739cd97cea 100644
--- a/drivers/gpu/drm/msm/msm_gpu.h
+++ b/drivers/gpu/drm/msm/msm_gpu.h
@@ -104,6 +104,9 @@ struct msm_gpu {
 	/* does gpu need hw_init? */
 	bool needs_hw_init;
 
+	/* number of GPU hangs (for all contexts) */
+	int global_faults;
+
 	/* worker for handling active-list retiring: */
 	struct work_struct retire_work;
 
diff --git a/drivers/gpu/drm/msm/msm_iommu.c b/drivers/gpu/drm/msm/msm_iommu.c
index 4d62790cd425..12bb54cefd46 100644
--- a/drivers/gpu/drm/msm/msm_iommu.c
+++ b/drivers/gpu/drm/msm/msm_iommu.c
@@ -38,13 +38,8 @@ static int msm_iommu_attach(struct msm_mmu *mmu, const char * const *names,
 			    int cnt)
 {
 	struct msm_iommu *iommu = to_msm_iommu(mmu);
-	int ret;
 
-	pm_runtime_get_sync(mmu->dev);
-	ret = iommu_attach_device(iommu->domain, mmu->dev);
-	pm_runtime_put_sync(mmu->dev);
-
-	return ret;
+	return iommu_attach_device(iommu->domain, mmu->dev);
 }
 
 static void msm_iommu_detach(struct msm_mmu *mmu, const char * const *names,
@@ -52,9 +47,7 @@ static void msm_iommu_detach(struct msm_mmu *mmu, const char * const *names,
 {
 	struct msm_iommu *iommu = to_msm_iommu(mmu);
 
-	pm_runtime_get_sync(mmu->dev);
 	iommu_detach_device(iommu->domain, mmu->dev);
-	pm_runtime_put_sync(mmu->dev);
 }
 
 static int msm_iommu_map(struct msm_mmu *mmu, uint64_t iova,
@@ -63,9 +56,7 @@ static int msm_iommu_map(struct msm_mmu *mmu, uint64_t iova,
 	struct msm_iommu *iommu = to_msm_iommu(mmu);
 	size_t ret;
 
-//	pm_runtime_get_sync(mmu->dev);
 	ret = iommu_map_sg(iommu->domain, iova, sgt->sgl, sgt->nents, prot);
-//	pm_runtime_put_sync(mmu->dev);
 	WARN_ON(!ret);
 
 	return (ret == len) ? 0 : -EINVAL;
@@ -75,9 +66,7 @@ static int msm_iommu_unmap(struct msm_mmu *mmu, uint64_t iova, unsigned len)
 {
 	struct msm_iommu *iommu = to_msm_iommu(mmu);
 
-	pm_runtime_get_sync(mmu->dev);
 	iommu_unmap(iommu->domain, iova, len);
-	pm_runtime_put_sync(mmu->dev);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/msm/msm_submitqueue.c b/drivers/gpu/drm/msm/msm_submitqueue.c
index 5115f75b5b7f..f160ec40a39b 100644
--- a/drivers/gpu/drm/msm/msm_submitqueue.c
+++ b/drivers/gpu/drm/msm/msm_submitqueue.c
@@ -120,6 +120,47 @@ int msm_submitqueue_init(struct drm_device *drm, struct msm_file_private *ctx)
 	return msm_submitqueue_create(drm, ctx, default_prio, 0, NULL);
 }
 
+static int msm_submitqueue_query_faults(struct msm_gpu_submitqueue *queue,
+		struct drm_msm_submitqueue_query *args)
+{
+	size_t size = min_t(size_t, args->len, sizeof(queue->faults));
+	int ret;
+
+	/* If a zero length was passed in, return the data size we expect */
+	if (!args->len) {
+		args->len = sizeof(queue->faults);
+		return 0;
+	}
+
+	/* Set the length to the actual size of the data */
+	args->len = size;
+
+	ret = copy_to_user(u64_to_user_ptr(args->data), &queue->faults, size);
+
+	return ret ? -EFAULT : 0;
+}
+
+int msm_submitqueue_query(struct drm_device *drm, struct msm_file_private *ctx,
+		struct drm_msm_submitqueue_query *args)
+{
+	struct msm_gpu_submitqueue *queue;
+	int ret = -EINVAL;
+
+	if (args->pad)
+		return -EINVAL;
+
+	queue = msm_submitqueue_get(ctx, args->id);
+	if (!queue)
+		return -ENOENT;
+
+	if (args->param == MSM_SUBMITQUEUE_PARAM_FAULTS)
+		ret = msm_submitqueue_query_faults(queue, args);
+
+	msm_submitqueue_put(queue);
+
+	return ret;
+}
+
 int msm_submitqueue_remove(struct msm_file_private *ctx, u32 id)
 {
 	struct msm_gpu_submitqueue *entry;
diff --git a/drivers/gpu/drm/mxsfb/mxsfb_crtc.c b/drivers/gpu/drm/mxsfb/mxsfb_crtc.c
index 0ee1ca8a316a..98e9bda91e80 100644
--- a/drivers/gpu/drm/mxsfb/mxsfb_crtc.c
+++ b/drivers/gpu/drm/mxsfb/mxsfb_crtc.c
@@ -253,12 +253,12 @@ static void mxsfb_crtc_mode_set_nofb(struct mxsfb_drm_private *mxsfb)
 	if (!(bus_flags & DRM_BUS_FLAG_DE_LOW))
 		vdctrl0 |= VDCTRL0_ENABLE_ACT_HIGH;
 	/*
-	 * DRM_BUS_FLAG_PIXDATA_ defines are controller centric,
+	 * DRM_BUS_FLAG_PIXDATA_DRIVE_ defines are controller centric,
 	 * controllers VDCTRL0_DOTCLK is display centric.
 	 * Drive on positive edge       -> display samples on falling edge
-	 * DRM_BUS_FLAG_PIXDATA_POSEDGE -> VDCTRL0_DOTCLK_ACT_FALLING
+	 * DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE -> VDCTRL0_DOTCLK_ACT_FALLING
 	 */
-	if (bus_flags & DRM_BUS_FLAG_PIXDATA_POSEDGE)
+	if (bus_flags & DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE)
 		vdctrl0 |= VDCTRL0_DOTCLK_ACT_FALLING;
 
 	writel(vdctrl0, mxsfb->base + LCDC_VDCTRL0);
diff --git a/drivers/gpu/drm/nouveau/Kbuild b/drivers/gpu/drm/nouveau/Kbuild
index 581404e6544d..378c5dd692b0 100644
--- a/drivers/gpu/drm/nouveau/Kbuild
+++ b/drivers/gpu/drm/nouveau/Kbuild
@@ -1,7 +1,7 @@
-ccflags-y += -I$(src)/include
-ccflags-y += -I$(src)/include/nvkm
-ccflags-y += -I$(src)/nvkm
-ccflags-y += -I$(src)
+ccflags-y += -I $(srctree)/$(src)/include
+ccflags-y += -I $(srctree)/$(src)/include/nvkm
+ccflags-y += -I $(srctree)/$(src)/nvkm
+ccflags-y += -I $(srctree)/$(src)
 
 # NVKM - HW resource manager
 #- code also used by various userspace tools/tests
diff --git a/drivers/gpu/drm/nouveau/Kconfig b/drivers/gpu/drm/nouveau/Kconfig
index 00cd9ab8948d..553c7da5e8e0 100644
--- a/drivers/gpu/drm/nouveau/Kconfig
+++ b/drivers/gpu/drm/nouveau/Kconfig
@@ -17,10 +17,21 @@ config DRM_NOUVEAU
 	select INPUT if ACPI && X86
 	select THERMAL if ACPI && X86
 	select ACPI_VIDEO if ACPI && X86
-	select DRM_VM
 	help
 	  Choose this option for open-source NVIDIA support.
 
+config NOUVEAU_LEGACY_CTX_SUPPORT
+	bool "Nouveau legacy context support"
+	depends on DRM_NOUVEAU
+	select DRM_LEGACY
+	default y
+	help
+	  There was a version of the nouveau DDX that relied on legacy
+	  ctx ioctls not erroring out. But that was back in time a long
+	  ways, so offer a way to disable it now. For uapi compat with
+	  old nouveau ddx this should be on by default, but modern distros
+	  should consider turning it off.
+
 config NOUVEAU_PLATFORM_DRIVER
 	bool "Nouveau (NVIDIA) SoC GPUs"
 	depends on DRM_NOUVEAU && ARCH_TEGRA
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/i2c.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/i2c.h
index eef54e9b5d77..7957eafa5f0e 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/i2c.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/i2c.h
@@ -38,6 +38,7 @@ struct nvkm_i2c_bus {
 	struct mutex mutex;
 	struct list_head head;
 	struct i2c_adapter i2c;
+	u8 enabled;
 };
 
 int nvkm_i2c_bus_acquire(struct nvkm_i2c_bus *);
@@ -57,6 +58,7 @@ struct nvkm_i2c_aux {
 	struct mutex mutex;
 	struct list_head head;
 	struct i2c_adapter i2c;
+	u8 enabled;
 
 	u32 intr;
 };
diff --git a/drivers/gpu/drm/nouveau/nouveau_display.c b/drivers/gpu/drm/nouveau/nouveau_display.c
index 55c0fa451163..832da8e0020d 100644
--- a/drivers/gpu/drm/nouveau/nouveau_display.c
+++ b/drivers/gpu/drm/nouveau/nouveau_display.c
@@ -358,15 +358,6 @@ nouveau_display_hpd_work(struct work_struct *work)
 
 #ifdef CONFIG_ACPI
 
-/*
- * Hans de Goede: This define belongs in acpi/video.h, I've submitted a patch
- * to the acpi subsys to move it there from drivers/acpi/acpi_video.c .
- * This should be dropped once that is merged.
- */
-#ifndef ACPI_VIDEO_NOTIFY_PROBE
-#define ACPI_VIDEO_NOTIFY_PROBE			0x81
-#endif
-
 static int
 nouveau_display_acpi_ntfy(struct notifier_block *nb, unsigned long val,
 			  void *data)
diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c
index 5020265bfbd9..22cd45845e07 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_drm.c
@@ -802,10 +802,15 @@ fail_display:
 static int
 nouveau_do_resume(struct drm_device *dev, bool runtime)
 {
+	int ret = 0;
 	struct nouveau_drm *drm = nouveau_drm(dev);
 
 	NV_DEBUG(drm, "resuming object tree...\n");
-	nvif_client_resume(&drm->master.base);
+	ret = nvif_client_resume(&drm->master.base);
+	if (ret) {
+		NV_ERROR(drm, "Client resume failed with error: %d\n", ret);
+		return ret;
+	}
 
 	NV_DEBUG(drm, "resuming fence...\n");
 	if (drm->fence && nouveau_fence(drm)->resume)
@@ -925,6 +930,7 @@ nouveau_pmops_runtime_resume(struct device *dev)
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
 	struct drm_device *drm_dev = pci_get_drvdata(pdev);
+	struct nouveau_drm *drm = nouveau_drm(drm_dev);
 	struct nvif_device *device = &nouveau_drm(drm_dev)->client.device;
 	int ret;
 
@@ -941,6 +947,10 @@ nouveau_pmops_runtime_resume(struct device *dev)
 	pci_set_master(pdev);
 
 	ret = nouveau_do_resume(drm_dev, true);
+	if (ret) {
+		NV_ERROR(drm, "resume failed with: %d\n", ret);
+		return ret;
+	}
 
 	/* do magic */
 	nvif_mask(&device->object, 0x088488, (1 << 25), (1 << 25));
@@ -1094,8 +1104,11 @@ nouveau_driver_fops = {
 static struct drm_driver
 driver_stub = {
 	.driver_features =
-		DRIVER_GEM | DRIVER_MODESET | DRIVER_PRIME | DRIVER_RENDER |
-		DRIVER_KMS_LEGACY_CONTEXT,
+		DRIVER_GEM | DRIVER_MODESET | DRIVER_PRIME | DRIVER_RENDER
+#if defined(CONFIG_NOUVEAU_LEGACY_CTX_SUPPORT)
+		| DRIVER_KMS_LEGACY_CONTEXT
+#endif
+		,
 
 	.open = nouveau_drm_open,
 	.postclose = nouveau_drm_postclose,
diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.h b/drivers/gpu/drm/nouveau/nouveau_drv.h
index da847244479d..35ff0ca01a3b 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drv.h
+++ b/drivers/gpu/drm/nouveau/nouveau_drv.h
@@ -60,8 +60,6 @@
 struct nouveau_channel;
 struct platform_device;
 
-#define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT)
-
 #include "nouveau_fence.h"
 #include "nouveau_bios.h"
 #include "nouveau_vmm.h"
diff --git a/drivers/gpu/drm/nouveau/nouveau_fbcon.c b/drivers/gpu/drm/nouveau/nouveau_fbcon.c
index 0d3cd4e05728..73cc3217068a 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nouveau_fbcon.c
@@ -365,14 +365,10 @@ nouveau_fbcon_create(struct drm_fb_helper *helper,
 		ret = PTR_ERR(info);
 		goto out_unlock;
 	}
-	info->skip_vt_switch = 1;
-
-	info->par = fbcon;
 
 	/* setup helper */
 	fbcon->helper.fb = &fb->base;
 
-	strcpy(info->fix.id, "nouveaufb");
 	if (!chan)
 		info->flags = FBINFO_HWACCEL_DISABLED;
 	else
@@ -387,9 +383,7 @@ nouveau_fbcon_create(struct drm_fb_helper *helper,
 	info->screen_base = nvbo_kmap_obj_iovirtual(fb->nvbo);
 	info->screen_size = fb->nvbo->bo.mem.num_pages << PAGE_SHIFT;
 
-	drm_fb_helper_fill_fix(info, fb->base.pitches[0],
-			       fb->base.format->depth);
-	drm_fb_helper_fill_var(info, &fbcon->helper, sizes->fb_width, sizes->fb_height);
+	drm_fb_helper_fill_info(info, &fbcon->helper, sizes);
 
 	/* Use default scratch pixmap (info->pixmap.flags = FB_PIXMAP_SYSTEM) */
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_fbcon.h b/drivers/gpu/drm/nouveau/nouveau_fbcon.h
index db9d52047ef8..73a7eeba3973 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fbcon.h
+++ b/drivers/gpu/drm/nouveau/nouveau_fbcon.h
@@ -32,7 +32,7 @@
 #include "nouveau_display.h"
 
 struct nouveau_fbdev {
-	struct drm_fb_helper helper;
+	struct drm_fb_helper helper; /* must be first */
 	unsigned int saved_flags;
 	struct nvif_object surf2d;
 	struct nvif_object clip;
diff --git a/drivers/gpu/drm/nouveau/nouveau_ttm.c b/drivers/gpu/drm/nouveau/nouveau_ttm.c
index 1543c2f8d3d3..f0daf958e03a 100644
--- a/drivers/gpu/drm/nouveau/nouveau_ttm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_ttm.c
@@ -168,9 +168,6 @@ nouveau_ttm_mmap(struct file *filp, struct vm_area_struct *vma)
 	struct drm_file *file_priv = filp->private_data;
 	struct nouveau_drm *drm = nouveau_drm(file_priv->minor->dev);
 
-	if (unlikely(vma->vm_pgoff < DRM_FILE_PAGE_OFFSET))
-		return drm_legacy_mmap(filp, vma);
-
 	return ttm_bo_mmap(filp, vma, &drm->ttm.bdev);
 }
 
@@ -239,7 +236,6 @@ nouveau_ttm_init(struct nouveau_drm *drm)
 	ret = ttm_bo_device_init(&drm->ttm.bdev,
 				  &nouveau_bo_driver,
 				  dev->anon_inode->i_mapping,
-				  DRM_FILE_PAGE_OFFSET,
 				  drm->client.mmu.dmabits <= 32 ? true : false);
 	if (ret) {
 		NV_ERROR(drm, "error initialising bo driver, %d\n", ret);
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c
index a3ba7f50198b..a3dcb09a40ee 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/gf100.c
@@ -94,6 +94,8 @@ gf100_bar_oneinit_bar(struct gf100_bar *bar, struct gf100_barN *bar_vm,
 		return ret;
 
 	bar_len = device->func->resource_size(device, bar_nr);
+	if (!bar_len)
+		return -ENOMEM;
 	if (bar_nr == 3 && bar->bar2_halve)
 		bar_len >>= 1;
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c
index 157b076a1272..f23a0ccc2bec 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bar/nv50.c
@@ -109,7 +109,7 @@ nv50_bar_oneinit(struct nvkm_bar *base)
 	struct nvkm_device *device = bar->base.subdev.device;
 	static struct lock_class_key bar1_lock;
 	static struct lock_class_key bar2_lock;
-	u64 start, limit;
+	u64 start, limit, size;
 	int ret;
 
 	ret = nvkm_gpuobj_new(device, 0x20000, 0, false, NULL, &bar->mem);
@@ -127,7 +127,10 @@ nv50_bar_oneinit(struct nvkm_bar *base)
 
 	/* BAR2 */
 	start = 0x0100000000ULL;
-	limit = start + device->func->resource_size(device, 3);
+	size = device->func->resource_size(device, 3);
+	if (!size)
+		return -ENOMEM;
+	limit = start + size;
 
 	ret = nvkm_vmm_new(device, start, limit-- - start, NULL, 0,
 			   &bar2_lock, "bar2", &bar->bar2_vmm);
@@ -164,10 +167,15 @@ nv50_bar_oneinit(struct nvkm_bar *base)
 
 	/* BAR1 */
 	start = 0x0000000000ULL;
-	limit = start + device->func->resource_size(device, 1);
+	size = device->func->resource_size(device, 1);
+	if (!size)
+		return -ENOMEM;
+	limit = start + size;
 
 	ret = nvkm_vmm_new(device, start, limit-- - start, NULL, 0,
 			   &bar1_lock, "bar1", &bar->bar1_vmm);
+	if (ret)
+		return ret;
 
 	atomic_inc(&bar->bar1_vmm->engref[NVKM_SUBDEV_BAR]);
 	bar->bar1_vmm->debug = bar->base.subdev.debug;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgk104.c
index 8bcb7e79a0cb..456aed1f2a02 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgk104.c
@@ -1070,7 +1070,7 @@ gk104_ram_calc_xits(struct gk104_ram *ram, struct nvkm_ram_data *next)
 			nvkm_error(subdev, "unable to calc plls\n");
 			return -EINVAL;
 		}
-		nvkm_debug(subdev, "sucessfully calced PLLs for clock %i kHz"
+		nvkm_debug(subdev, "successfully calced PLLs for clock %i kHz"
 				" (refclock: %i kHz)\n", next->freq, ret);
 	} else {
 		/* calculate refpll coefficients */
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/aux.c b/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/aux.c
index 4c1f547da463..b4e7404fe660 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/aux.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/aux.c
@@ -105,9 +105,15 @@ nvkm_i2c_aux_acquire(struct nvkm_i2c_aux *aux)
 {
 	struct nvkm_i2c_pad *pad = aux->pad;
 	int ret;
+
 	AUX_TRACE(aux, "acquire");
 	mutex_lock(&aux->mutex);
-	ret = nvkm_i2c_pad_acquire(pad, NVKM_I2C_PAD_AUX);
+
+	if (aux->enabled)
+		ret = nvkm_i2c_pad_acquire(pad, NVKM_I2C_PAD_AUX);
+	else
+		ret = -EIO;
+
 	if (ret)
 		mutex_unlock(&aux->mutex);
 	return ret;
@@ -145,6 +151,24 @@ nvkm_i2c_aux_del(struct nvkm_i2c_aux **paux)
 	}
 }
 
+void
+nvkm_i2c_aux_init(struct nvkm_i2c_aux *aux)
+{
+	AUX_TRACE(aux, "init");
+	mutex_lock(&aux->mutex);
+	aux->enabled = true;
+	mutex_unlock(&aux->mutex);
+}
+
+void
+nvkm_i2c_aux_fini(struct nvkm_i2c_aux *aux)
+{
+	AUX_TRACE(aux, "fini");
+	mutex_lock(&aux->mutex);
+	aux->enabled = false;
+	mutex_unlock(&aux->mutex);
+}
+
 int
 nvkm_i2c_aux_ctor(const struct nvkm_i2c_aux_func *func,
 		  struct nvkm_i2c_pad *pad, int id,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/aux.h b/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/aux.h
index 7d56c4ba693c..08f6b2ee64ab 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/aux.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/aux.h
@@ -16,6 +16,8 @@ int nvkm_i2c_aux_ctor(const struct nvkm_i2c_aux_func *, struct nvkm_i2c_pad *,
 int nvkm_i2c_aux_new_(const struct nvkm_i2c_aux_func *, struct nvkm_i2c_pad *,
 		      int id, struct nvkm_i2c_aux **);
 void nvkm_i2c_aux_del(struct nvkm_i2c_aux **);
+void nvkm_i2c_aux_init(struct nvkm_i2c_aux *);
+void nvkm_i2c_aux_fini(struct nvkm_i2c_aux *);
 int nvkm_i2c_aux_xfer(struct nvkm_i2c_aux *, bool retry, u8 type,
 		      u32 addr, u8 *data, u8 *size);
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/base.c
index 4f197b15acf6..ecacb22834d7 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/base.c
@@ -160,8 +160,18 @@ nvkm_i2c_fini(struct nvkm_subdev *subdev, bool suspend)
 {
 	struct nvkm_i2c *i2c = nvkm_i2c(subdev);
 	struct nvkm_i2c_pad *pad;
+	struct nvkm_i2c_bus *bus;
+	struct nvkm_i2c_aux *aux;
 	u32 mask;
 
+	list_for_each_entry(aux, &i2c->aux, head) {
+		nvkm_i2c_aux_fini(aux);
+	}
+
+	list_for_each_entry(bus, &i2c->bus, head) {
+		nvkm_i2c_bus_fini(bus);
+	}
+
 	if ((mask = (1 << i2c->func->aux) - 1), i2c->func->aux_stat) {
 		i2c->func->aux_mask(i2c, NVKM_I2C_ANY, mask, 0);
 		i2c->func->aux_stat(i2c, &mask, &mask, &mask, &mask);
@@ -180,6 +190,7 @@ nvkm_i2c_init(struct nvkm_subdev *subdev)
 	struct nvkm_i2c *i2c = nvkm_i2c(subdev);
 	struct nvkm_i2c_bus *bus;
 	struct nvkm_i2c_pad *pad;
+	struct nvkm_i2c_aux *aux;
 
 	list_for_each_entry(pad, &i2c->pad, head) {
 		nvkm_i2c_pad_init(pad);
@@ -189,6 +200,10 @@ nvkm_i2c_init(struct nvkm_subdev *subdev)
 		nvkm_i2c_bus_init(bus);
 	}
 
+	list_for_each_entry(aux, &i2c->aux, head) {
+		nvkm_i2c_aux_init(aux);
+	}
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/bus.c b/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/bus.c
index 807a2b67bd64..ed50cc3736b9 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/bus.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/bus.c
@@ -110,6 +110,19 @@ nvkm_i2c_bus_init(struct nvkm_i2c_bus *bus)
 	BUS_TRACE(bus, "init");
 	if (bus->func->init)
 		bus->func->init(bus);
+
+	mutex_lock(&bus->mutex);
+	bus->enabled = true;
+	mutex_unlock(&bus->mutex);
+}
+
+void
+nvkm_i2c_bus_fini(struct nvkm_i2c_bus *bus)
+{
+	BUS_TRACE(bus, "fini");
+	mutex_lock(&bus->mutex);
+	bus->enabled = false;
+	mutex_unlock(&bus->mutex);
 }
 
 void
@@ -126,9 +139,15 @@ nvkm_i2c_bus_acquire(struct nvkm_i2c_bus *bus)
 {
 	struct nvkm_i2c_pad *pad = bus->pad;
 	int ret;
+
 	BUS_TRACE(bus, "acquire");
 	mutex_lock(&bus->mutex);
-	ret = nvkm_i2c_pad_acquire(pad, NVKM_I2C_PAD_I2C);
+
+	if (bus->enabled)
+		ret = nvkm_i2c_pad_acquire(pad, NVKM_I2C_PAD_I2C);
+	else
+		ret = -EIO;
+
 	if (ret)
 		mutex_unlock(&bus->mutex);
 	return ret;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/bus.h b/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/bus.h
index bea0dd33961e..465464bba58b 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/bus.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/bus.h
@@ -18,6 +18,7 @@ int nvkm_i2c_bus_new_(const struct nvkm_i2c_bus_func *, struct nvkm_i2c_pad *,
 		      int id, struct nvkm_i2c_bus **);
 void nvkm_i2c_bus_del(struct nvkm_i2c_bus **);
 void nvkm_i2c_bus_init(struct nvkm_i2c_bus *);
+void nvkm_i2c_bus_fini(struct nvkm_i2c_bus *);
 
 int nvkm_i2c_bit_xfer(struct nvkm_i2c_bus *, struct i2c_msg *, int);
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c
index fa93f964e6a4..41640e0584ac 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c
@@ -1783,7 +1783,7 @@ nvkm_vmm_get(struct nvkm_vmm *vmm, u8 page, u64 size, struct nvkm_vma **pvma)
 void
 nvkm_vmm_part(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
 {
-	if (inst && vmm->func->part) {
+	if (inst && vmm && vmm->func->part) {
 		mutex_lock(&vmm->mutex);
 		vmm->func->part(vmm, inst);
 		mutex_unlock(&vmm->mutex);
diff --git a/drivers/gpu/drm/omapdrm/displays/Kconfig b/drivers/gpu/drm/omapdrm/displays/Kconfig
index a349cb61961e..7b0bcb494b5c 100644
--- a/drivers/gpu/drm/omapdrm/displays/Kconfig
+++ b/drivers/gpu/drm/omapdrm/displays/Kconfig
@@ -6,23 +6,12 @@ config DRM_OMAP_ENCODER_OPA362
 	  Driver for OPA362 external analog TV amplifier controlled
 	  through a GPIO.
 
-config DRM_OMAP_ENCODER_TFP410
-        tristate "TFP410 DPI to DVI Encoder"
-	help
-	  Driver for TFP410 DPI to DVI encoder.
-
 config DRM_OMAP_ENCODER_TPD12S015
         tristate "TPD12S015 HDMI ESD protection and level shifter"
 	help
 	  Driver for TPD12S015, which offers HDMI ESD protection and level
 	  shifting.
 
-config DRM_OMAP_CONNECTOR_DVI
-        tristate "DVI Connector"
-	depends on I2C
-	help
-	  Driver for a generic DVI connector.
-
 config DRM_OMAP_CONNECTOR_HDMI
         tristate "HDMI Connector"
 	help
@@ -33,12 +22,6 @@ config DRM_OMAP_CONNECTOR_ANALOG_TV
 	help
 	  Driver for a generic analog TV connector.
 
-config DRM_OMAP_PANEL_DPI
-	tristate "Generic DPI panel"
-	depends on BACKLIGHT_CLASS_DEVICE
-	help
-	  Driver for generic DPI panels.
-
 config DRM_OMAP_PANEL_DSI_CM
 	tristate "Generic DSI Command Mode Panel"
 	depends on BACKLIGHT_CLASS_DEVICE
diff --git a/drivers/gpu/drm/omapdrm/displays/Makefile b/drivers/gpu/drm/omapdrm/displays/Makefile
index d99659e1381b..1db34d4fed64 100644
--- a/drivers/gpu/drm/omapdrm/displays/Makefile
+++ b/drivers/gpu/drm/omapdrm/displays/Makefile
@@ -1,11 +1,8 @@
 # SPDX-License-Identifier: GPL-2.0
 obj-$(CONFIG_DRM_OMAP_ENCODER_OPA362) += encoder-opa362.o
-obj-$(CONFIG_DRM_OMAP_ENCODER_TFP410) += encoder-tfp410.o
 obj-$(CONFIG_DRM_OMAP_ENCODER_TPD12S015) += encoder-tpd12s015.o
-obj-$(CONFIG_DRM_OMAP_CONNECTOR_DVI) += connector-dvi.o
 obj-$(CONFIG_DRM_OMAP_CONNECTOR_HDMI) += connector-hdmi.o
 obj-$(CONFIG_DRM_OMAP_CONNECTOR_ANALOG_TV) += connector-analog-tv.o
-obj-$(CONFIG_DRM_OMAP_PANEL_DPI) += panel-dpi.o
 obj-$(CONFIG_DRM_OMAP_PANEL_DSI_CM) += panel-dsi-cm.o
 obj-$(CONFIG_DRM_OMAP_PANEL_SONY_ACX565AKM) += panel-sony-acx565akm.o
 obj-$(CONFIG_DRM_OMAP_PANEL_LGPHILIPS_LB035Q02) += panel-lgphilips-lb035q02.o
diff --git a/drivers/gpu/drm/omapdrm/displays/connector-analog-tv.c b/drivers/gpu/drm/omapdrm/displays/connector-analog-tv.c
index 28a3ce8f88d2..6c0561101874 100644
--- a/drivers/gpu/drm/omapdrm/displays/connector-analog-tv.c
+++ b/drivers/gpu/drm/omapdrm/displays/connector-analog-tv.c
@@ -35,50 +35,9 @@ static void tvc_disconnect(struct omap_dss_device *src,
 {
 }
 
-static int tvc_enable(struct omap_dss_device *dssdev)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *src = dssdev->src;
-	int r;
-
-	dev_dbg(ddata->dev, "enable\n");
-
-	if (!omapdss_device_is_connected(dssdev))
-		return -ENODEV;
-
-	if (omapdss_device_is_enabled(dssdev))
-		return 0;
-
-	r = src->ops->enable(src);
-	if (r)
-		return r;
-
-	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
-
-	return r;
-}
-
-static void tvc_disable(struct omap_dss_device *dssdev)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *src = dssdev->src;
-
-	dev_dbg(ddata->dev, "disable\n");
-
-	if (!omapdss_device_is_enabled(dssdev))
-		return;
-
-	src->ops->disable(src);
-
-	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
-}
-
 static const struct omap_dss_device_ops tvc_ops = {
 	.connect		= tvc_connect,
 	.disconnect		= tvc_disconnect,
-
-	.enable			= tvc_enable,
-	.disable		= tvc_disable,
 };
 
 static int tvc_probe(struct platform_device *pdev)
@@ -97,6 +56,7 @@ static int tvc_probe(struct platform_device *pdev)
 	dssdev->ops = &tvc_ops;
 	dssdev->dev = &pdev->dev;
 	dssdev->type = OMAP_DISPLAY_TYPE_VENC;
+	dssdev->display = true;
 	dssdev->owner = THIS_MODULE;
 	dssdev->of_ports = BIT(0);
 
@@ -109,12 +69,9 @@ static int tvc_probe(struct platform_device *pdev)
 static int __exit tvc_remove(struct platform_device *pdev)
 {
 	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
-	struct omap_dss_device *dssdev = &ddata->dssdev;
 
 	omapdss_device_unregister(&ddata->dssdev);
 
-	tvc_disable(dssdev);
-
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/omapdrm/displays/connector-dvi.c b/drivers/gpu/drm/omapdrm/displays/connector-dvi.c
deleted file mode 100644
index 24b14f44248e..000000000000
--- a/drivers/gpu/drm/omapdrm/displays/connector-dvi.c
+++ /dev/null
@@ -1,330 +0,0 @@
-/*
- * Generic DVI Connector driver
- *
- * Copyright (C) 2013 Texas Instruments Incorporated - http://www.ti.com/
- * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- */
-
-#include <linux/gpio/consumer.h>
-#include <linux/i2c.h>
-#include <linux/module.h>
-#include <linux/platform_device.h>
-#include <linux/slab.h>
-
-#include <drm/drm_edid.h>
-
-#include "../dss/omapdss.h"
-
-struct panel_drv_data {
-	struct omap_dss_device dssdev;
-
-	struct i2c_adapter *i2c_adapter;
-
-	struct gpio_desc *hpd_gpio;
-
-	void (*hpd_cb)(void *cb_data, enum drm_connector_status status);
-	void *hpd_cb_data;
-	bool hpd_enabled;
-	/* mutex for hpd fields above */
-	struct mutex hpd_lock;
-};
-
-#define to_panel_data(x) container_of(x, struct panel_drv_data, dssdev)
-
-static int dvic_connect(struct omap_dss_device *src,
-			struct omap_dss_device *dst)
-{
-	return 0;
-}
-
-static void dvic_disconnect(struct omap_dss_device *src,
-			    struct omap_dss_device *dst)
-{
-}
-
-static int dvic_enable(struct omap_dss_device *dssdev)
-{
-	struct omap_dss_device *src = dssdev->src;
-	int r;
-
-	if (!omapdss_device_is_connected(dssdev))
-		return -ENODEV;
-
-	if (omapdss_device_is_enabled(dssdev))
-		return 0;
-
-	r = src->ops->enable(src);
-	if (r)
-		return r;
-
-	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
-
-	return 0;
-}
-
-static void dvic_disable(struct omap_dss_device *dssdev)
-{
-	struct omap_dss_device *src = dssdev->src;
-
-	if (!omapdss_device_is_enabled(dssdev))
-		return;
-
-	src->ops->disable(src);
-
-	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
-}
-
-static int dvic_ddc_read(struct i2c_adapter *adapter,
-		unsigned char *buf, u16 count, u8 offset)
-{
-	int r, retries;
-
-	for (retries = 3; retries > 0; retries--) {
-		struct i2c_msg msgs[] = {
-			{
-				.addr   = DDC_ADDR,
-				.flags  = 0,
-				.len    = 1,
-				.buf    = &offset,
-			}, {
-				.addr   = DDC_ADDR,
-				.flags  = I2C_M_RD,
-				.len    = count,
-				.buf    = buf,
-			}
-		};
-
-		r = i2c_transfer(adapter, msgs, 2);
-		if (r == 2)
-			return 0;
-
-		if (r != -EAGAIN)
-			break;
-	}
-
-	return r < 0 ? r : -EIO;
-}
-
-static int dvic_read_edid(struct omap_dss_device *dssdev,
-		u8 *edid, int len)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	int r, l, bytes_read;
-
-	l = min(EDID_LENGTH, len);
-	r = dvic_ddc_read(ddata->i2c_adapter, edid, l, 0);
-	if (r)
-		return r;
-
-	bytes_read = l;
-
-	/* if there are extensions, read second block */
-	if (len > EDID_LENGTH && edid[0x7e] > 0) {
-		l = min(EDID_LENGTH, len - EDID_LENGTH);
-
-		r = dvic_ddc_read(ddata->i2c_adapter, edid + EDID_LENGTH,
-				l, EDID_LENGTH);
-		if (r)
-			return r;
-
-		bytes_read += l;
-	}
-
-	return bytes_read;
-}
-
-static bool dvic_detect(struct omap_dss_device *dssdev)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	unsigned char out;
-	int r;
-
-	if (ddata->hpd_gpio)
-		return gpiod_get_value_cansleep(ddata->hpd_gpio);
-
-	if (!ddata->i2c_adapter)
-		return true;
-
-	r = dvic_ddc_read(ddata->i2c_adapter, &out, 1, 0);
-
-	return r == 0;
-}
-
-static void dvic_register_hpd_cb(struct omap_dss_device *dssdev,
-				 void (*cb)(void *cb_data,
-					    enum drm_connector_status status),
-				 void *cb_data)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-
-	mutex_lock(&ddata->hpd_lock);
-	ddata->hpd_cb = cb;
-	ddata->hpd_cb_data = cb_data;
-	mutex_unlock(&ddata->hpd_lock);
-}
-
-static void dvic_unregister_hpd_cb(struct omap_dss_device *dssdev)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-
-	mutex_lock(&ddata->hpd_lock);
-	ddata->hpd_cb = NULL;
-	ddata->hpd_cb_data = NULL;
-	mutex_unlock(&ddata->hpd_lock);
-}
-
-static const struct omap_dss_device_ops dvic_ops = {
-	.connect	= dvic_connect,
-	.disconnect	= dvic_disconnect,
-
-	.enable		= dvic_enable,
-	.disable	= dvic_disable,
-
-	.read_edid	= dvic_read_edid,
-	.detect		= dvic_detect,
-
-	.register_hpd_cb	= dvic_register_hpd_cb,
-	.unregister_hpd_cb	= dvic_unregister_hpd_cb,
-};
-
-static irqreturn_t dvic_hpd_isr(int irq, void *data)
-{
-	struct panel_drv_data *ddata = data;
-
-	mutex_lock(&ddata->hpd_lock);
-	if (ddata->hpd_enabled && ddata->hpd_cb) {
-		enum drm_connector_status status;
-
-		if (dvic_detect(&ddata->dssdev))
-			status = connector_status_connected;
-		else
-			status = connector_status_disconnected;
-
-		ddata->hpd_cb(ddata->hpd_cb_data, status);
-	}
-	mutex_unlock(&ddata->hpd_lock);
-
-	return IRQ_HANDLED;
-}
-
-static int dvic_probe_of(struct platform_device *pdev)
-{
-	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
-	struct device_node *node = pdev->dev.of_node;
-	struct device_node *adapter_node;
-	struct i2c_adapter *adapter;
-	struct gpio_desc *gpio;
-	int r;
-
-	gpio = devm_gpiod_get_optional(&pdev->dev, "hpd", GPIOD_IN);
-	if (IS_ERR(gpio)) {
-		dev_err(&pdev->dev, "failed to parse HPD gpio\n");
-		return PTR_ERR(gpio);
-	}
-
-	ddata->hpd_gpio = gpio;
-
-	mutex_init(&ddata->hpd_lock);
-
-	if (ddata->hpd_gpio) {
-		r = devm_request_threaded_irq(&pdev->dev,
-			gpiod_to_irq(ddata->hpd_gpio), NULL, dvic_hpd_isr,
-			IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
-			"DVI HPD", ddata);
-		if (r)
-			return r;
-	}
-
-	adapter_node = of_parse_phandle(node, "ddc-i2c-bus", 0);
-	if (adapter_node) {
-		adapter = of_get_i2c_adapter_by_node(adapter_node);
-		of_node_put(adapter_node);
-		if (adapter == NULL) {
-			dev_err(&pdev->dev, "failed to parse ddc-i2c-bus\n");
-			return -EPROBE_DEFER;
-		}
-
-		ddata->i2c_adapter = adapter;
-	}
-
-	return 0;
-}
-
-static int dvic_probe(struct platform_device *pdev)
-{
-	struct panel_drv_data *ddata;
-	struct omap_dss_device *dssdev;
-	int r;
-
-	ddata = devm_kzalloc(&pdev->dev, sizeof(*ddata), GFP_KERNEL);
-	if (!ddata)
-		return -ENOMEM;
-
-	platform_set_drvdata(pdev, ddata);
-
-	r = dvic_probe_of(pdev);
-	if (r)
-		return r;
-
-	dssdev = &ddata->dssdev;
-	dssdev->ops = &dvic_ops;
-	dssdev->dev = &pdev->dev;
-	dssdev->type = OMAP_DISPLAY_TYPE_DVI;
-	dssdev->owner = THIS_MODULE;
-	dssdev->of_ports = BIT(0);
-
-	if (ddata->hpd_gpio)
-		dssdev->ops_flags |= OMAP_DSS_DEVICE_OP_DETECT
-				  |  OMAP_DSS_DEVICE_OP_HPD;
-	if (ddata->i2c_adapter)
-		dssdev->ops_flags |= OMAP_DSS_DEVICE_OP_DETECT
-				  |  OMAP_DSS_DEVICE_OP_EDID;
-
-	omapdss_display_init(dssdev);
-	omapdss_device_register(dssdev);
-
-	return 0;
-}
-
-static int __exit dvic_remove(struct platform_device *pdev)
-{
-	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
-	struct omap_dss_device *dssdev = &ddata->dssdev;
-
-	omapdss_device_unregister(&ddata->dssdev);
-
-	dvic_disable(dssdev);
-
-	i2c_put_adapter(ddata->i2c_adapter);
-
-	mutex_destroy(&ddata->hpd_lock);
-
-	return 0;
-}
-
-static const struct of_device_id dvic_of_match[] = {
-	{ .compatible = "omapdss,dvi-connector", },
-	{},
-};
-
-MODULE_DEVICE_TABLE(of, dvic_of_match);
-
-static struct platform_driver dvi_connector_driver = {
-	.probe	= dvic_probe,
-	.remove	= __exit_p(dvic_remove),
-	.driver	= {
-		.name	= "connector-dvi",
-		.of_match_table = dvic_of_match,
-		.suppress_bind_attrs = true,
-	},
-};
-
-module_platform_driver(dvi_connector_driver);
-
-MODULE_AUTHOR("Tomi Valkeinen <tomi.valkeinen@ti.com>");
-MODULE_DESCRIPTION("Generic DVI Connector driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/omapdrm/displays/connector-hdmi.c b/drivers/gpu/drm/omapdrm/displays/connector-hdmi.c
index e602fa4a50a4..68d6f6e44b03 100644
--- a/drivers/gpu/drm/omapdrm/displays/connector-hdmi.c
+++ b/drivers/gpu/drm/omapdrm/displays/connector-hdmi.c
@@ -41,44 +41,6 @@ static void hdmic_disconnect(struct omap_dss_device *src,
 {
 }
 
-static int hdmic_enable(struct omap_dss_device *dssdev)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *src = dssdev->src;
-	int r;
-
-	dev_dbg(ddata->dev, "enable\n");
-
-	if (!omapdss_device_is_connected(dssdev))
-		return -ENODEV;
-
-	if (omapdss_device_is_enabled(dssdev))
-		return 0;
-
-	r = src->ops->enable(src);
-	if (r)
-		return r;
-
-	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
-
-	return r;
-}
-
-static void hdmic_disable(struct omap_dss_device *dssdev)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *src = dssdev->src;
-
-	dev_dbg(ddata->dev, "disable\n");
-
-	if (!omapdss_device_is_enabled(dssdev))
-		return;
-
-	src->ops->disable(src);
-
-	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
-}
-
 static bool hdmic_detect(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
@@ -113,9 +75,6 @@ static const struct omap_dss_device_ops hdmic_ops = {
 	.connect		= hdmic_connect,
 	.disconnect		= hdmic_disconnect,
 
-	.enable			= hdmic_enable,
-	.disable		= hdmic_disable,
-
 	.detect			= hdmic_detect,
 	.register_hpd_cb	= hdmic_register_hpd_cb,
 	.unregister_hpd_cb	= hdmic_unregister_hpd_cb,
@@ -181,6 +140,7 @@ static int hdmic_probe(struct platform_device *pdev)
 	dssdev->ops = &hdmic_ops;
 	dssdev->dev = &pdev->dev;
 	dssdev->type = OMAP_DISPLAY_TYPE_HDMI;
+	dssdev->display = true;
 	dssdev->owner = THIS_MODULE;
 	dssdev->of_ports = BIT(0);
 	dssdev->ops_flags = ddata->hpd_gpio
@@ -196,12 +156,9 @@ static int hdmic_probe(struct platform_device *pdev)
 static int __exit hdmic_remove(struct platform_device *pdev)
 {
 	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
-	struct omap_dss_device *dssdev = &ddata->dssdev;
 
 	omapdss_device_unregister(&ddata->dssdev);
 
-	hdmic_disable(dssdev);
-
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/omapdrm/displays/encoder-opa362.c b/drivers/gpu/drm/omapdrm/displays/encoder-opa362.c
index 4fefd80f53bb..29a5a130ebd1 100644
--- a/drivers/gpu/drm/omapdrm/displays/encoder-opa362.c
+++ b/drivers/gpu/drm/omapdrm/displays/encoder-opa362.c
@@ -41,48 +41,20 @@ static void opa362_disconnect(struct omap_dss_device *src,
 	omapdss_device_disconnect(dst, dst->next);
 }
 
-static int opa362_enable(struct omap_dss_device *dssdev)
+static void opa362_enable(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *src = dssdev->src;
-	int r;
-
-	dev_dbg(dssdev->dev, "enable\n");
-
-	if (!omapdss_device_is_connected(dssdev))
-		return -ENODEV;
-
-	if (omapdss_device_is_enabled(dssdev))
-		return 0;
-
-	r = src->ops->enable(src);
-	if (r)
-		return r;
 
 	if (ddata->enable_gpio)
 		gpiod_set_value_cansleep(ddata->enable_gpio, 1);
-
-	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
-
-	return 0;
 }
 
 static void opa362_disable(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *src = dssdev->src;
-
-	dev_dbg(dssdev->dev, "disable\n");
-
-	if (!omapdss_device_is_enabled(dssdev))
-		return;
 
 	if (ddata->enable_gpio)
 		gpiod_set_value_cansleep(ddata->enable_gpio, 0);
-
-	src->ops->disable(src);
-
-	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
 }
 
 static const struct omap_dss_device_ops opa362_ops = {
@@ -116,7 +88,6 @@ static int opa362_probe(struct platform_device *pdev)
 	dssdev->ops = &opa362_ops;
 	dssdev->dev = &pdev->dev;
 	dssdev->type = OMAP_DISPLAY_TYPE_VENC;
-	dssdev->output_type = OMAP_DISPLAY_TYPE_VENC;
 	dssdev->owner = THIS_MODULE;
 	dssdev->of_ports = BIT(1) | BIT(0);
 
@@ -141,13 +112,7 @@ static int __exit opa362_remove(struct platform_device *pdev)
 		omapdss_device_put(dssdev->next);
 	omapdss_device_unregister(&ddata->dssdev);
 
-	WARN_ON(omapdss_device_is_enabled(dssdev));
-	if (omapdss_device_is_enabled(dssdev))
-		opa362_disable(dssdev);
-
-	WARN_ON(omapdss_device_is_connected(dssdev));
-	if (omapdss_device_is_connected(dssdev))
-		omapdss_device_disconnect(NULL, dssdev);
+	opa362_disable(dssdev);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/omapdrm/displays/encoder-tfp410.c b/drivers/gpu/drm/omapdrm/displays/encoder-tfp410.c
deleted file mode 100644
index f1a748353279..000000000000
--- a/drivers/gpu/drm/omapdrm/displays/encoder-tfp410.c
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * TFP410 DPI-to-DVI encoder driver
- *
- * Copyright (C) 2013 Texas Instruments Incorporated - http://www.ti.com/
- * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- */
-
-#include <linux/gpio/consumer.h>
-#include <linux/module.h>
-#include <linux/platform_device.h>
-#include <linux/slab.h>
-
-#include "../dss/omapdss.h"
-
-struct panel_drv_data {
-	struct omap_dss_device dssdev;
-
-	struct gpio_desc *pd_gpio;
-};
-
-#define to_panel_data(x) container_of(x, struct panel_drv_data, dssdev)
-
-static int tfp410_connect(struct omap_dss_device *src,
-			  struct omap_dss_device *dst)
-{
-	return omapdss_device_connect(dst->dss, dst, dst->next);
-}
-
-static void tfp410_disconnect(struct omap_dss_device *src,
-			      struct omap_dss_device *dst)
-{
-	omapdss_device_disconnect(dst, dst->next);
-}
-
-static int tfp410_enable(struct omap_dss_device *dssdev)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *src = dssdev->src;
-	int r;
-
-	if (!omapdss_device_is_connected(dssdev))
-		return -ENODEV;
-
-	if (omapdss_device_is_enabled(dssdev))
-		return 0;
-
-	r = src->ops->enable(src);
-	if (r)
-		return r;
-
-	if (ddata->pd_gpio)
-		gpiod_set_value_cansleep(ddata->pd_gpio, 0);
-
-	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
-
-	return 0;
-}
-
-static void tfp410_disable(struct omap_dss_device *dssdev)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *src = dssdev->src;
-
-	if (!omapdss_device_is_enabled(dssdev))
-		return;
-
-	if (ddata->pd_gpio)
-		gpiod_set_value_cansleep(ddata->pd_gpio, 0);
-
-	src->ops->disable(src);
-
-	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
-}
-
-static const struct omap_dss_device_ops tfp410_ops = {
-	.connect	= tfp410_connect,
-	.disconnect	= tfp410_disconnect,
-	.enable		= tfp410_enable,
-	.disable	= tfp410_disable,
-};
-
-static int tfp410_probe(struct platform_device *pdev)
-{
-	struct panel_drv_data *ddata;
-	struct omap_dss_device *dssdev;
-	struct gpio_desc *gpio;
-
-	ddata = devm_kzalloc(&pdev->dev, sizeof(*ddata), GFP_KERNEL);
-	if (!ddata)
-		return -ENOMEM;
-
-	platform_set_drvdata(pdev, ddata);
-
-	/* Powerdown GPIO */
-	gpio = devm_gpiod_get_optional(&pdev->dev, "powerdown", GPIOD_OUT_HIGH);
-	if (IS_ERR(gpio)) {
-		dev_err(&pdev->dev, "failed to parse powerdown gpio\n");
-		return PTR_ERR(gpio);
-	}
-
-	ddata->pd_gpio = gpio;
-
-	dssdev = &ddata->dssdev;
-	dssdev->ops = &tfp410_ops;
-	dssdev->dev = &pdev->dev;
-	dssdev->type = OMAP_DISPLAY_TYPE_DPI;
-	dssdev->output_type = OMAP_DISPLAY_TYPE_DVI;
-	dssdev->owner = THIS_MODULE;
-	dssdev->of_ports = BIT(1) | BIT(0);
-	dssdev->bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_SYNC_POSEDGE
-			  | DRM_BUS_FLAG_PIXDATA_POSEDGE;
-
-	dssdev->next = omapdss_of_find_connected_device(pdev->dev.of_node, 1);
-	if (IS_ERR(dssdev->next)) {
-		if (PTR_ERR(dssdev->next) != -EPROBE_DEFER)
-			dev_err(&pdev->dev, "failed to find video sink\n");
-		return PTR_ERR(dssdev->next);
-	}
-
-	omapdss_device_register(dssdev);
-
-	return 0;
-}
-
-static int __exit tfp410_remove(struct platform_device *pdev)
-{
-	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
-	struct omap_dss_device *dssdev = &ddata->dssdev;
-
-	if (dssdev->next)
-		omapdss_device_put(dssdev->next);
-	omapdss_device_unregister(&ddata->dssdev);
-
-	WARN_ON(omapdss_device_is_enabled(dssdev));
-	if (omapdss_device_is_enabled(dssdev))
-		tfp410_disable(dssdev);
-
-	WARN_ON(omapdss_device_is_connected(dssdev));
-	if (omapdss_device_is_connected(dssdev))
-		omapdss_device_disconnect(NULL, dssdev);
-
-	return 0;
-}
-
-static const struct of_device_id tfp410_of_match[] = {
-	{ .compatible = "omapdss,ti,tfp410", },
-	{},
-};
-
-MODULE_DEVICE_TABLE(of, tfp410_of_match);
-
-static struct platform_driver tfp410_driver = {
-	.probe	= tfp410_probe,
-	.remove	= __exit_p(tfp410_remove),
-	.driver	= {
-		.name	= "tfp410",
-		.of_match_table = tfp410_of_match,
-		.suppress_bind_attrs = true,
-	},
-};
-
-module_platform_driver(tfp410_driver);
-
-MODULE_AUTHOR("Tomi Valkeinen <tomi.valkeinen@ti.com>");
-MODULE_DESCRIPTION("TFP410 DPI to DVI encoder driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/omapdrm/displays/encoder-tpd12s015.c b/drivers/gpu/drm/omapdrm/displays/encoder-tpd12s015.c
index 94de55fd8884..bc03752d2762 100644
--- a/drivers/gpu/drm/omapdrm/displays/encoder-tpd12s015.c
+++ b/drivers/gpu/drm/omapdrm/displays/encoder-tpd12s015.c
@@ -62,35 +62,6 @@ static void tpd_disconnect(struct omap_dss_device *src,
 	omapdss_device_disconnect(dst, dst->next);
 }
 
-static int tpd_enable(struct omap_dss_device *dssdev)
-{
-	struct omap_dss_device *src = dssdev->src;
-	int r;
-
-	if (dssdev->state == OMAP_DSS_DISPLAY_ACTIVE)
-		return 0;
-
-	r = src->ops->enable(src);
-	if (r)
-		return r;
-
-	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
-
-	return r;
-}
-
-static void tpd_disable(struct omap_dss_device *dssdev)
-{
-	struct omap_dss_device *src = dssdev->src;
-
-	if (dssdev->state != OMAP_DSS_DISPLAY_ACTIVE)
-		return;
-
-	src->ops->disable(src);
-
-	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
-}
-
 static bool tpd_detect(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
@@ -124,8 +95,6 @@ static void tpd_unregister_hpd_cb(struct omap_dss_device *dssdev)
 static const struct omap_dss_device_ops tpd_ops = {
 	.connect		= tpd_connect,
 	.disconnect		= tpd_disconnect,
-	.enable			= tpd_enable,
-	.disable		= tpd_disable,
 	.detect			= tpd_detect,
 	.register_hpd_cb	= tpd_register_hpd_cb,
 	.unregister_hpd_cb	= tpd_unregister_hpd_cb,
@@ -198,7 +167,6 @@ static int tpd_probe(struct platform_device *pdev)
 	dssdev->ops = &tpd_ops;
 	dssdev->dev = &pdev->dev;
 	dssdev->type = OMAP_DISPLAY_TYPE_HDMI;
-	dssdev->output_type = OMAP_DISPLAY_TYPE_HDMI;
 	dssdev->owner = THIS_MODULE;
 	dssdev->of_ports = BIT(1) | BIT(0);
 	dssdev->ops_flags = OMAP_DSS_DEVICE_OP_DETECT
@@ -225,14 +193,6 @@ static int __exit tpd_remove(struct platform_device *pdev)
 		omapdss_device_put(dssdev->next);
 	omapdss_device_unregister(&ddata->dssdev);
 
-	WARN_ON(omapdss_device_is_enabled(dssdev));
-	if (omapdss_device_is_enabled(dssdev))
-		tpd_disable(dssdev);
-
-	WARN_ON(omapdss_device_is_connected(dssdev));
-	if (omapdss_device_is_connected(dssdev))
-		omapdss_device_disconnect(NULL, dssdev);
-
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-dpi.c b/drivers/gpu/drm/omapdrm/displays/panel-dpi.c
deleted file mode 100644
index 465120809eb3..000000000000
--- a/drivers/gpu/drm/omapdrm/displays/panel-dpi.c
+++ /dev/null
@@ -1,221 +0,0 @@
-/*
- * Generic MIPI DPI Panel Driver
- *
- * Copyright (C) 2013 Texas Instruments Incorporated - http://www.ti.com/
- * Author: Tomi Valkeinen <tomi.valkeinen@ti.com>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- */
-
-#include <linux/gpio/consumer.h>
-#include <linux/module.h>
-#include <linux/platform_device.h>
-#include <linux/slab.h>
-#include <linux/of.h>
-#include <linux/regulator/consumer.h>
-#include <linux/backlight.h>
-
-#include <video/of_display_timing.h>
-
-#include "../dss/omapdss.h"
-
-struct panel_drv_data {
-	struct omap_dss_device dssdev;
-
-	struct videomode vm;
-
-	struct backlight_device *backlight;
-
-	struct gpio_desc *enable_gpio;
-	struct regulator *vcc_supply;
-};
-
-#define to_panel_data(p) container_of(p, struct panel_drv_data, dssdev)
-
-static int panel_dpi_connect(struct omap_dss_device *src,
-			     struct omap_dss_device *dst)
-{
-	return 0;
-}
-
-static void panel_dpi_disconnect(struct omap_dss_device *src,
-				 struct omap_dss_device *dst)
-{
-}
-
-static int panel_dpi_enable(struct omap_dss_device *dssdev)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *src = dssdev->src;
-	int r;
-
-	if (!omapdss_device_is_connected(dssdev))
-		return -ENODEV;
-
-	if (omapdss_device_is_enabled(dssdev))
-		return 0;
-
-	r = src->ops->enable(src);
-	if (r)
-		return r;
-
-	r = regulator_enable(ddata->vcc_supply);
-	if (r) {
-		src->ops->disable(src);
-		return r;
-	}
-
-	gpiod_set_value_cansleep(ddata->enable_gpio, 1);
-	backlight_enable(ddata->backlight);
-
-	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
-
-	return 0;
-}
-
-static void panel_dpi_disable(struct omap_dss_device *dssdev)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *src = dssdev->src;
-
-	if (!omapdss_device_is_enabled(dssdev))
-		return;
-
-	backlight_disable(ddata->backlight);
-
-	gpiod_set_value_cansleep(ddata->enable_gpio, 0);
-	regulator_disable(ddata->vcc_supply);
-
-	src->ops->disable(src);
-
-	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
-}
-
-static void panel_dpi_get_timings(struct omap_dss_device *dssdev,
-				  struct videomode *vm)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-
-	*vm = ddata->vm;
-}
-
-static const struct omap_dss_device_ops panel_dpi_ops = {
-	.connect	= panel_dpi_connect,
-	.disconnect	= panel_dpi_disconnect,
-
-	.enable		= panel_dpi_enable,
-	.disable	= panel_dpi_disable,
-
-	.get_timings	= panel_dpi_get_timings,
-};
-
-static int panel_dpi_probe_of(struct platform_device *pdev)
-{
-	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
-	struct device_node *node = pdev->dev.of_node;
-	int r;
-	struct display_timing timing;
-	struct gpio_desc *gpio;
-
-	gpio = devm_gpiod_get_optional(&pdev->dev, "enable", GPIOD_OUT_LOW);
-	if (IS_ERR(gpio))
-		return PTR_ERR(gpio);
-
-	ddata->enable_gpio = gpio;
-
-	/*
-	 * Many different panels are supported by this driver and there are
-	 * probably very different needs for their reset pins in regards to
-	 * timing and order relative to the enable gpio. So for now it's just
-	 * ensured that the reset line isn't active.
-	 */
-	gpio = devm_gpiod_get_optional(&pdev->dev, "reset", GPIOD_OUT_LOW);
-	if (IS_ERR(gpio))
-		return PTR_ERR(gpio);
-
-	ddata->vcc_supply = devm_regulator_get(&pdev->dev, "vcc");
-	if (IS_ERR(ddata->vcc_supply))
-		return PTR_ERR(ddata->vcc_supply);
-
-	ddata->backlight = devm_of_find_backlight(&pdev->dev);
-
-	if (IS_ERR(ddata->backlight))
-		return PTR_ERR(ddata->backlight);
-
-	r = of_get_display_timing(node, "panel-timing", &timing);
-	if (r) {
-		dev_err(&pdev->dev, "failed to get video timing\n");
-		return r;
-	}
-
-	videomode_from_timing(&timing, &ddata->vm);
-
-	return 0;
-}
-
-static int panel_dpi_probe(struct platform_device *pdev)
-{
-	struct panel_drv_data *ddata;
-	struct omap_dss_device *dssdev;
-	int r;
-
-	ddata = devm_kzalloc(&pdev->dev, sizeof(*ddata), GFP_KERNEL);
-	if (ddata == NULL)
-		return -ENOMEM;
-
-	platform_set_drvdata(pdev, ddata);
-
-	r = panel_dpi_probe_of(pdev);
-	if (r)
-		return r;
-
-	dssdev = &ddata->dssdev;
-	dssdev->dev = &pdev->dev;
-	dssdev->ops = &panel_dpi_ops;
-	dssdev->type = OMAP_DISPLAY_TYPE_DPI;
-	dssdev->owner = THIS_MODULE;
-	dssdev->of_ports = BIT(0);
-	drm_bus_flags_from_videomode(&ddata->vm, &dssdev->bus_flags);
-
-	omapdss_display_init(dssdev);
-	omapdss_device_register(dssdev);
-
-	return 0;
-}
-
-static int __exit panel_dpi_remove(struct platform_device *pdev)
-{
-	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
-	struct omap_dss_device *dssdev = &ddata->dssdev;
-
-	omapdss_device_unregister(dssdev);
-
-	panel_dpi_disable(dssdev);
-
-	return 0;
-}
-
-static const struct of_device_id panel_dpi_of_match[] = {
-	{ .compatible = "omapdss,panel-dpi", },
-	{},
-};
-
-MODULE_DEVICE_TABLE(of, panel_dpi_of_match);
-
-static struct platform_driver panel_dpi_driver = {
-	.probe = panel_dpi_probe,
-	.remove = __exit_p(panel_dpi_remove),
-	.driver = {
-		.name = "panel-dpi",
-		.of_match_table = panel_dpi_of_match,
-		.suppress_bind_attrs = true,
-	},
-};
-
-module_platform_driver(panel_dpi_driver);
-
-MODULE_AUTHOR("Tomi Valkeinen <tomi.valkeinen@ti.com>");
-MODULE_DESCRIPTION("Generic MIPI DPI Panel Driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-dsi-cm.c b/drivers/gpu/drm/omapdrm/displays/panel-dsi-cm.c
index 29692a5217c5..741a5e324767 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-dsi-cm.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-dsi-cm.c
@@ -24,6 +24,8 @@
 #include <linux/of_device.h>
 #include <linux/regulator/consumer.h>
 
+#include <drm/drm_connector.h>
+
 #include <video/mipi_display.h>
 #include <video/of_display_timing.h>
 
@@ -41,6 +43,7 @@
 
 struct panel_drv_data {
 	struct omap_dss_device dssdev;
+	struct omap_dss_device *src;
 
 	struct videomode vm;
 
@@ -141,7 +144,7 @@ static void hw_guard_wait(struct panel_drv_data *ddata)
 
 static int dsicm_dcs_read_1(struct panel_drv_data *ddata, u8 dcs_cmd, u8 *data)
 {
-	struct omap_dss_device *src = ddata->dssdev.src;
+	struct omap_dss_device *src = ddata->src;
 	int r;
 	u8 buf[1];
 
@@ -157,14 +160,14 @@ static int dsicm_dcs_read_1(struct panel_drv_data *ddata, u8 dcs_cmd, u8 *data)
 
 static int dsicm_dcs_write_0(struct panel_drv_data *ddata, u8 dcs_cmd)
 {
-	struct omap_dss_device *src = ddata->dssdev.src;
+	struct omap_dss_device *src = ddata->src;
 
 	return src->ops->dsi.dcs_write(src, ddata->channel, &dcs_cmd, 1);
 }
 
 static int dsicm_dcs_write_1(struct panel_drv_data *ddata, u8 dcs_cmd, u8 param)
 {
-	struct omap_dss_device *src = ddata->dssdev.src;
+	struct omap_dss_device *src = ddata->src;
 	u8 buf[2] = { dcs_cmd, param };
 
 	return src->ops->dsi.dcs_write(src, ddata->channel, buf, 2);
@@ -173,7 +176,7 @@ static int dsicm_dcs_write_1(struct panel_drv_data *ddata, u8 dcs_cmd, u8 param)
 static int dsicm_sleep_in(struct panel_drv_data *ddata)
 
 {
-	struct omap_dss_device *src = ddata->dssdev.src;
+	struct omap_dss_device *src = ddata->src;
 	u8 cmd;
 	int r;
 
@@ -228,7 +231,7 @@ static int dsicm_get_id(struct panel_drv_data *ddata, u8 *id1, u8 *id2, u8 *id3)
 static int dsicm_set_update_window(struct panel_drv_data *ddata,
 		u16 x, u16 y, u16 w, u16 h)
 {
-	struct omap_dss_device *src = ddata->dssdev.src;
+	struct omap_dss_device *src = ddata->src;
 	int r;
 	u16 x1 = x;
 	u16 x2 = x + w - 1;
@@ -275,7 +278,7 @@ static void dsicm_cancel_ulps_work(struct panel_drv_data *ddata)
 
 static int dsicm_enter_ulps(struct panel_drv_data *ddata)
 {
-	struct omap_dss_device *src = ddata->dssdev.src;
+	struct omap_dss_device *src = ddata->src;
 	int r;
 
 	if (ddata->ulps_enabled)
@@ -309,18 +312,13 @@ err:
 
 static int dsicm_exit_ulps(struct panel_drv_data *ddata)
 {
-	struct omap_dss_device *src = ddata->dssdev.src;
+	struct omap_dss_device *src = ddata->src;
 	int r;
 
 	if (!ddata->ulps_enabled)
 		return 0;
 
-	r = src->ops->enable(src);
-	if (r) {
-		dev_err(&ddata->pdev->dev, "failed to enable DSI\n");
-		goto err1;
-	}
-
+	src->ops->enable(src);
 	src->ops->dsi.enable_hs(src, ddata->channel, true);
 
 	r = _dsicm_enable_te(ddata, true);
@@ -347,7 +345,7 @@ err2:
 			enable_irq(gpiod_to_irq(ddata->ext_te_gpio));
 		ddata->ulps_enabled = false;
 	}
-err1:
+
 	dsicm_queue_ulps_work(ddata);
 
 	return r;
@@ -366,7 +364,7 @@ static int dsicm_wake_up(struct panel_drv_data *ddata)
 static int dsicm_bl_update_status(struct backlight_device *dev)
 {
 	struct panel_drv_data *ddata = dev_get_drvdata(&dev->dev);
-	struct omap_dss_device *src = ddata->dssdev.src;
+	struct omap_dss_device *src = ddata->src;
 	int r = 0;
 	int level;
 
@@ -414,7 +412,7 @@ static ssize_t dsicm_num_errors_show(struct device *dev,
 {
 	struct platform_device *pdev = to_platform_device(dev);
 	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
-	struct omap_dss_device *src = ddata->dssdev.src;
+	struct omap_dss_device *src = ddata->src;
 	u8 errors = 0;
 	int r;
 
@@ -446,7 +444,7 @@ static ssize_t dsicm_hw_revision_show(struct device *dev,
 {
 	struct platform_device *pdev = to_platform_device(dev);
 	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
-	struct omap_dss_device *src = ddata->dssdev.src;
+	struct omap_dss_device *src = ddata->src;
 	u8 id1, id2, id3;
 	int r;
 
@@ -478,7 +476,7 @@ static ssize_t dsicm_store_ulps(struct device *dev,
 {
 	struct platform_device *pdev = to_platform_device(dev);
 	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
-	struct omap_dss_device *src = ddata->dssdev.src;
+	struct omap_dss_device *src = ddata->src;
 	unsigned long t;
 	int r;
 
@@ -528,7 +526,7 @@ static ssize_t dsicm_store_ulps_timeout(struct device *dev,
 {
 	struct platform_device *pdev = to_platform_device(dev);
 	struct panel_drv_data *ddata = platform_get_drvdata(pdev);
-	struct omap_dss_device *src = ddata->dssdev.src;
+	struct omap_dss_device *src = ddata->src;
 	unsigned long t;
 	int r;
 
@@ -603,7 +601,7 @@ static void dsicm_hw_reset(struct panel_drv_data *ddata)
 
 static int dsicm_power_on(struct panel_drv_data *ddata)
 {
-	struct omap_dss_device *src = ddata->dssdev.src;
+	struct omap_dss_device *src = ddata->src;
 	u8 id1, id2, id3;
 	int r;
 	struct omap_dss_dsi_config dsi_config = {
@@ -649,11 +647,7 @@ static int dsicm_power_on(struct panel_drv_data *ddata)
 		goto err_vddi;
 	}
 
-	r = src->ops->enable(src);
-	if (r) {
-		dev_err(&ddata->pdev->dev, "failed to enable DSI\n");
-		goto err_vddi;
-	}
+	src->ops->enable(src);
 
 	dsicm_hw_reset(ddata);
 
@@ -722,7 +716,7 @@ err_vpnl:
 
 static void dsicm_power_off(struct panel_drv_data *ddata)
 {
-	struct omap_dss_device *src = ddata->dssdev.src;
+	struct omap_dss_device *src = ddata->src;
 	int r;
 
 	src->ops->dsi.disable_video_output(src, ddata->channel);
@@ -776,6 +770,7 @@ static int dsicm_connect(struct omap_dss_device *src,
 		return r;
 	}
 
+	ddata->src = src;
 	return 0;
 }
 
@@ -785,28 +780,17 @@ static void dsicm_disconnect(struct omap_dss_device *src,
 	struct panel_drv_data *ddata = to_panel_data(dst);
 
 	src->ops->dsi.release_vc(src, ddata->channel);
+	ddata->src = NULL;
 }
 
-static int dsicm_enable(struct omap_dss_device *dssdev)
+static void dsicm_enable(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *src = dssdev->src;
+	struct omap_dss_device *src = ddata->src;
 	int r;
 
-	dev_dbg(&ddata->pdev->dev, "enable\n");
-
 	mutex_lock(&ddata->lock);
 
-	if (!omapdss_device_is_connected(dssdev)) {
-		r = -ENODEV;
-		goto err;
-	}
-
-	if (omapdss_device_is_enabled(dssdev)) {
-		r = 0;
-		goto err;
-	}
-
 	src->ops->dsi.bus_lock(src);
 
 	r = dsicm_power_on(ddata);
@@ -816,27 +800,22 @@ static int dsicm_enable(struct omap_dss_device *dssdev)
 	if (r)
 		goto err;
 
-	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
-
 	mutex_unlock(&ddata->lock);
 
 	dsicm_bl_power(ddata, true);
 
-	return 0;
+	return;
 err:
-	dev_dbg(&ddata->pdev->dev, "enable failed\n");
+	dev_dbg(&ddata->pdev->dev, "enable failed (%d)\n", r);
 	mutex_unlock(&ddata->lock);
-	return r;
 }
 
 static void dsicm_disable(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *src = dssdev->src;
+	struct omap_dss_device *src = ddata->src;
 	int r;
 
-	dev_dbg(&ddata->pdev->dev, "disable\n");
-
 	dsicm_bl_power(ddata, false);
 
 	mutex_lock(&ddata->lock);
@@ -845,23 +824,19 @@ static void dsicm_disable(struct omap_dss_device *dssdev)
 
 	src->ops->dsi.bus_lock(src);
 
-	if (omapdss_device_is_enabled(dssdev)) {
-		r = dsicm_wake_up(ddata);
-		if (!r)
-			dsicm_power_off(ddata);
-	}
+	r = dsicm_wake_up(ddata);
+	if (!r)
+		dsicm_power_off(ddata);
 
 	src->ops->dsi.bus_unlock(src);
 
-	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
-
 	mutex_unlock(&ddata->lock);
 }
 
 static void dsicm_framedone_cb(int err, void *data)
 {
 	struct panel_drv_data *ddata = data;
-	struct omap_dss_device *src = ddata->dssdev.src;
+	struct omap_dss_device *src = ddata->src;
 
 	dev_dbg(&ddata->pdev->dev, "framedone, err %d\n", err);
 	src->ops->dsi.bus_unlock(src);
@@ -870,7 +845,7 @@ static void dsicm_framedone_cb(int err, void *data)
 static irqreturn_t dsicm_te_isr(int irq, void *data)
 {
 	struct panel_drv_data *ddata = data;
-	struct omap_dss_device *src = ddata->dssdev.src;
+	struct omap_dss_device *src = ddata->src;
 	int old;
 	int r;
 
@@ -896,7 +871,7 @@ static void dsicm_te_timeout_work_callback(struct work_struct *work)
 {
 	struct panel_drv_data *ddata = container_of(work, struct panel_drv_data,
 					te_timeout_work.work);
-	struct omap_dss_device *src = ddata->dssdev.src;
+	struct omap_dss_device *src = ddata->src;
 
 	dev_err(&ddata->pdev->dev, "TE not received for 250ms!\n");
 
@@ -908,7 +883,7 @@ static int dsicm_update(struct omap_dss_device *dssdev,
 				    u16 x, u16 y, u16 w, u16 h)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *src = dssdev->src;
+	struct omap_dss_device *src = ddata->src;
 	int r;
 
 	dev_dbg(&ddata->pdev->dev, "update %d, %d, %d x %d\n", x, y, w, h);
@@ -954,7 +929,7 @@ err:
 static int dsicm_sync(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *src = dssdev->src;
+	struct omap_dss_device *src = ddata->src;
 
 	dev_dbg(&ddata->pdev->dev, "sync\n");
 
@@ -970,7 +945,7 @@ static int dsicm_sync(struct omap_dss_device *dssdev)
 
 static int _dsicm_enable_te(struct panel_drv_data *ddata, bool enable)
 {
-	struct omap_dss_device *src = ddata->dssdev.src;
+	struct omap_dss_device *src = ddata->src;
 	int r;
 
 	if (enable)
@@ -990,7 +965,7 @@ static int _dsicm_enable_te(struct panel_drv_data *ddata, bool enable)
 static int dsicm_enable_te(struct omap_dss_device *dssdev, bool enable)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *src = dssdev->src;
+	struct omap_dss_device *src = ddata->src;
 	int r;
 
 	mutex_lock(&ddata->lock);
@@ -1041,7 +1016,7 @@ static int dsicm_memory_read(struct omap_dss_device *dssdev,
 		u16 x, u16 y, u16 w, u16 h)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *src = dssdev->src;
+	struct omap_dss_device *src = ddata->src;
 	int r;
 	int first = 1;
 	int plen;
@@ -1123,7 +1098,7 @@ static void dsicm_ulps_work(struct work_struct *work)
 	struct panel_drv_data *ddata = container_of(work, struct panel_drv_data,
 			ulps_work.work);
 	struct omap_dss_device *dssdev = &ddata->dssdev;
-	struct omap_dss_device *src = dssdev->src;
+	struct omap_dss_device *src = ddata->src;
 
 	mutex_lock(&ddata->lock);
 
@@ -1140,29 +1115,32 @@ static void dsicm_ulps_work(struct work_struct *work)
 	mutex_unlock(&ddata->lock);
 }
 
-static void dsicm_get_timings(struct omap_dss_device *dssdev,
-			      struct videomode *vm)
+static int dsicm_get_modes(struct omap_dss_device *dssdev,
+			   struct drm_connector *connector)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 
-	*vm = ddata->vm;
+	connector->display_info.width_mm = ddata->width_mm;
+	connector->display_info.height_mm = ddata->height_mm;
+
+	return omapdss_display_get_modes(connector, &ddata->vm);
 }
 
 static int dsicm_check_timings(struct omap_dss_device *dssdev,
-			       struct videomode *vm)
+			       struct drm_display_mode *mode)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 	int ret = 0;
 
-	if (vm->hactive != ddata->vm.hactive)
+	if (mode->hdisplay != ddata->vm.hactive)
 		ret = -EINVAL;
 
-	if (vm->vactive != ddata->vm.vactive)
+	if (mode->vdisplay != ddata->vm.vactive)
 		ret = -EINVAL;
 
 	if (ret) {
 		dev_warn(dssdev->dev, "wrong resolution: %d x %d",
-			 vm->hactive, vm->vactive);
+			 mode->hdisplay, mode->vdisplay);
 		dev_warn(dssdev->dev, "panel resolution: %d x %d",
 			 ddata->vm.hactive, ddata->vm.vactive);
 	}
@@ -1170,15 +1148,6 @@ static int dsicm_check_timings(struct omap_dss_device *dssdev,
 	return ret;
 }
 
-static void dsicm_get_size(struct omap_dss_device *dssdev,
-			  unsigned int *width, unsigned int *height)
-{
-	struct panel_drv_data *ddata = to_panel_data(dssdev);
-
-	*width = ddata->width_mm;
-	*height = ddata->height_mm;
-}
-
 static const struct omap_dss_device_ops dsicm_ops = {
 	.connect	= dsicm_connect,
 	.disconnect	= dsicm_disconnect,
@@ -1186,7 +1155,7 @@ static const struct omap_dss_device_ops dsicm_ops = {
 	.enable		= dsicm_enable,
 	.disable	= dsicm_disable,
 
-	.get_timings	= dsicm_get_timings,
+	.get_modes	= dsicm_get_modes,
 	.check_timings	= dsicm_check_timings,
 };
 
@@ -1194,8 +1163,6 @@ static const struct omap_dss_driver dsicm_dss_driver = {
 	.update		= dsicm_update,
 	.sync		= dsicm_sync,
 
-	.get_size	= dsicm_get_size,
-
 	.enable_te	= dsicm_enable_te,
 	.get_te		= dsicm_get_te,
 
@@ -1305,8 +1272,10 @@ static int dsicm_probe(struct platform_device *pdev)
 	dssdev->ops = &dsicm_ops;
 	dssdev->driver = &dsicm_dss_driver;
 	dssdev->type = OMAP_DISPLAY_TYPE_DSI;
+	dssdev->display = true;
 	dssdev->owner = THIS_MODULE;
 	dssdev->of_ports = BIT(0);
+	dssdev->ops_flags = OMAP_DSS_DEVICE_OP_MODES;
 
 	dssdev->caps = OMAP_DSS_DISPLAY_CAP_MANUAL_UPDATE |
 		OMAP_DSS_DISPLAY_CAP_TEAR_ELIM;
@@ -1385,8 +1354,9 @@ static int __exit dsicm_remove(struct platform_device *pdev)
 
 	omapdss_device_unregister(dssdev);
 
-	dsicm_disable(dssdev);
-	omapdss_device_disconnect(dssdev->src, dssdev);
+	if (omapdss_device_is_enabled(dssdev))
+		dsicm_disable(dssdev);
+	omapdss_device_disconnect(ddata->src, dssdev);
 
 	sysfs_remove_group(&pdev->dev.kobj, &dsicm_attr_group);
 
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-lgphilips-lb035q02.c b/drivers/gpu/drm/omapdrm/displays/panel-lgphilips-lb035q02.c
index f6ef8ff964dd..99f2350d462c 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-lgphilips-lb035q02.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-lgphilips-lb035q02.c
@@ -123,52 +123,28 @@ static void lb035q02_disconnect(struct omap_dss_device *src,
 {
 }
 
-static int lb035q02_enable(struct omap_dss_device *dssdev)
+static void lb035q02_enable(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *src = dssdev->src;
-	int r;
-
-	if (!omapdss_device_is_connected(dssdev))
-		return -ENODEV;
-
-	if (omapdss_device_is_enabled(dssdev))
-		return 0;
-
-	r = src->ops->enable(src);
-	if (r)
-		return r;
 
 	if (ddata->enable_gpio)
 		gpiod_set_value_cansleep(ddata->enable_gpio, 1);
-
-	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
-
-	return 0;
 }
 
 static void lb035q02_disable(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *src = dssdev->src;
-
-	if (!omapdss_device_is_enabled(dssdev))
-		return;
 
 	if (ddata->enable_gpio)
 		gpiod_set_value_cansleep(ddata->enable_gpio, 0);
-
-	src->ops->disable(src);
-
-	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
 }
 
-static void lb035q02_get_timings(struct omap_dss_device *dssdev,
-				 struct videomode *vm)
+static int lb035q02_get_modes(struct omap_dss_device *dssdev,
+			      struct drm_connector *connector)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 
-	*vm = ddata->vm;
+	return omapdss_display_get_modes(connector, &ddata->vm);
 }
 
 static const struct omap_dss_device_ops lb035q02_ops = {
@@ -178,7 +154,7 @@ static const struct omap_dss_device_ops lb035q02_ops = {
 	.enable		= lb035q02_enable,
 	.disable	= lb035q02_disable,
 
-	.get_timings	= lb035q02_get_timings,
+	.get_modes	= lb035q02_get_modes,
 };
 
 static int lb035q02_probe_of(struct spi_device *spi)
@@ -221,16 +197,19 @@ static int lb035q02_panel_spi_probe(struct spi_device *spi)
 	dssdev->dev = &spi->dev;
 	dssdev->ops = &lb035q02_ops;
 	dssdev->type = OMAP_DISPLAY_TYPE_DPI;
+	dssdev->display = true;
 	dssdev->owner = THIS_MODULE;
 	dssdev->of_ports = BIT(0);
+	dssdev->ops_flags = OMAP_DSS_DEVICE_OP_MODES;
 
 	/*
 	 * Note: According to the panel documentation:
 	 * DE is active LOW
 	 * DATA needs to be driven on the FALLING edge
 	 */
-	dssdev->bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_SYNC_NEGEDGE
-			  | DRM_BUS_FLAG_PIXDATA_POSEDGE;
+	dssdev->bus_flags = DRM_BUS_FLAG_DE_HIGH
+			  | DRM_BUS_FLAG_SYNC_DRIVE_NEGEDGE
+			  | DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE;
 
 	omapdss_display_init(dssdev);
 	omapdss_device_register(dssdev);
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-nec-nl8048hl11.c b/drivers/gpu/drm/omapdrm/displays/panel-nec-nl8048hl11.c
index f445de6369f7..c2409815a204 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-nec-nl8048hl11.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-nec-nl8048hl11.c
@@ -118,50 +118,26 @@ static void nec_8048_disconnect(struct omap_dss_device *src,
 {
 }
 
-static int nec_8048_enable(struct omap_dss_device *dssdev)
+static void nec_8048_enable(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *src = dssdev->src;
-	int r;
-
-	if (!omapdss_device_is_connected(dssdev))
-		return -ENODEV;
-
-	if (omapdss_device_is_enabled(dssdev))
-		return 0;
-
-	r = src->ops->enable(src);
-	if (r)
-		return r;
 
 	gpiod_set_value_cansleep(ddata->res_gpio, 1);
-
-	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
-
-	return 0;
 }
 
 static void nec_8048_disable(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *src = dssdev->src;
-
-	if (!omapdss_device_is_enabled(dssdev))
-		return;
 
 	gpiod_set_value_cansleep(ddata->res_gpio, 0);
-
-	src->ops->disable(src);
-
-	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
 }
 
-static void nec_8048_get_timings(struct omap_dss_device *dssdev,
-				 struct videomode *vm)
+static int nec_8048_get_modes(struct omap_dss_device *dssdev,
+			      struct drm_connector *connector)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 
-	*vm = ddata->vm;
+	return omapdss_display_get_modes(connector, &ddata->vm);
 }
 
 static const struct omap_dss_device_ops nec_8048_ops = {
@@ -171,7 +147,7 @@ static const struct omap_dss_device_ops nec_8048_ops = {
 	.enable		= nec_8048_enable,
 	.disable	= nec_8048_disable,
 
-	.get_timings	= nec_8048_get_timings,
+	.get_modes	= nec_8048_get_modes,
 };
 
 static int nec_8048_probe(struct spi_device *spi)
@@ -216,10 +192,13 @@ static int nec_8048_probe(struct spi_device *spi)
 	dssdev->dev = &spi->dev;
 	dssdev->ops = &nec_8048_ops;
 	dssdev->type = OMAP_DISPLAY_TYPE_DPI;
+	dssdev->display = true;
 	dssdev->owner = THIS_MODULE;
 	dssdev->of_ports = BIT(0);
-	dssdev->bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_SYNC_POSEDGE
-			  | DRM_BUS_FLAG_PIXDATA_POSEDGE;
+	dssdev->ops_flags = OMAP_DSS_DEVICE_OP_MODES;
+	dssdev->bus_flags = DRM_BUS_FLAG_DE_HIGH
+			  | DRM_BUS_FLAG_SYNC_DRIVE_POSEDGE
+			  | DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE;
 
 	omapdss_display_init(dssdev);
 	omapdss_device_register(dssdev);
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-sharp-ls037v7dw01.c b/drivers/gpu/drm/omapdrm/displays/panel-sharp-ls037v7dw01.c
index 64b1369cb274..9c545de430f6 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-sharp-ls037v7dw01.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-sharp-ls037v7dw01.c
@@ -62,29 +62,22 @@ static void sharp_ls_disconnect(struct omap_dss_device *src,
 {
 }
 
-static int sharp_ls_enable(struct omap_dss_device *dssdev)
+static void sharp_ls_pre_enable(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *src = dssdev->src;
 	int r;
 
-	if (!omapdss_device_is_connected(dssdev))
-		return -ENODEV;
-
-	if (omapdss_device_is_enabled(dssdev))
-		return 0;
-
 	if (ddata->vcc) {
 		r = regulator_enable(ddata->vcc);
-		if (r != 0)
-			return r;
+		if (r)
+			dev_err(dssdev->dev, "%s: failed to enable regulator\n",
+				__func__);
 	}
+}
 
-	r = src->ops->enable(src);
-	if (r) {
-		regulator_disable(ddata->vcc);
-		return r;
-	}
+static void sharp_ls_enable(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
 
 	/* wait couple of vsyncs until enabling the LCD */
 	msleep(50);
@@ -94,19 +87,11 @@ static int sharp_ls_enable(struct omap_dss_device *dssdev)
 
 	if (ddata->ini_gpio)
 		gpiod_set_value_cansleep(ddata->ini_gpio, 1);
-
-	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
-
-	return 0;
 }
 
 static void sharp_ls_disable(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *src = dssdev->src;
-
-	if (!omapdss_device_is_enabled(dssdev))
-		return;
 
 	if (ddata->ini_gpio)
 		gpiod_set_value_cansleep(ddata->ini_gpio, 0);
@@ -115,33 +100,35 @@ static void sharp_ls_disable(struct omap_dss_device *dssdev)
 		gpiod_set_value_cansleep(ddata->resb_gpio, 0);
 
 	/* wait at least 5 vsyncs after disabling the LCD */
-
 	msleep(100);
+}
 
-	src->ops->disable(src);
+static void sharp_ls_post_disable(struct omap_dss_device *dssdev)
+{
+	struct panel_drv_data *ddata = to_panel_data(dssdev);
 
 	if (ddata->vcc)
 		regulator_disable(ddata->vcc);
-
-	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
 }
 
-static void sharp_ls_get_timings(struct omap_dss_device *dssdev,
-				 struct videomode *vm)
+static int sharp_ls_get_modes(struct omap_dss_device *dssdev,
+			      struct drm_connector *connector)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 
-	*vm = ddata->vm;
+	return omapdss_display_get_modes(connector, &ddata->vm);
 }
 
 static const struct omap_dss_device_ops sharp_ls_ops = {
 	.connect	= sharp_ls_connect,
 	.disconnect	= sharp_ls_disconnect,
 
+	.pre_enable	= sharp_ls_pre_enable,
 	.enable		= sharp_ls_enable,
 	.disable	= sharp_ls_disable,
+	.post_disable	= sharp_ls_post_disable,
 
-	.get_timings	= sharp_ls_get_timings,
+	.get_modes	= sharp_ls_get_modes,
 };
 
 static  int sharp_ls_get_gpio_of(struct device *dev, int index, int val,
@@ -220,15 +207,18 @@ static int sharp_ls_probe(struct platform_device *pdev)
 	dssdev->dev = &pdev->dev;
 	dssdev->ops = &sharp_ls_ops;
 	dssdev->type = OMAP_DISPLAY_TYPE_DPI;
+	dssdev->display = true;
 	dssdev->owner = THIS_MODULE;
 	dssdev->of_ports = BIT(0);
+	dssdev->ops_flags = OMAP_DSS_DEVICE_OP_MODES;
 
 	/*
 	 * Note: According to the panel documentation:
 	 * DATA needs to be driven on the FALLING edge
 	 */
-	dssdev->bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_SYNC_NEGEDGE
-			  | DRM_BUS_FLAG_PIXDATA_POSEDGE;
+	dssdev->bus_flags = DRM_BUS_FLAG_DE_HIGH
+			  | DRM_BUS_FLAG_SYNC_DRIVE_NEGEDGE
+			  | DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE;
 
 	omapdss_display_init(dssdev);
 	omapdss_device_register(dssdev);
@@ -243,7 +233,10 @@ static int __exit sharp_ls_remove(struct platform_device *pdev)
 
 	omapdss_device_unregister(dssdev);
 
-	sharp_ls_disable(dssdev);
+	if (omapdss_device_is_enabled(dssdev)) {
+		sharp_ls_disable(dssdev);
+		sharp_ls_post_disable(dssdev);
+	}
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-sony-acx565akm.c b/drivers/gpu/drm/omapdrm/displays/panel-sony-acx565akm.c
index e04663856b31..2038def14ba1 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-sony-acx565akm.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-sony-acx565akm.c
@@ -516,17 +516,9 @@ static void acx565akm_disconnect(struct omap_dss_device *src,
 static int acx565akm_panel_power_on(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *src = dssdev->src;
-	int r;
 
 	dev_dbg(&ddata->spi->dev, "%s\n", __func__);
 
-	r = src->ops->enable(src);
-	if (r) {
-		pr_err("%s sdi enable failed\n", __func__);
-		return r;
-	}
-
 	/*FIXME tweak me */
 	msleep(50);
 
@@ -562,7 +554,6 @@ static int acx565akm_panel_power_on(struct omap_dss_device *dssdev)
 static void acx565akm_panel_power_off(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *src = dssdev->src;
 
 	dev_dbg(dssdev->dev, "%s\n", __func__);
 
@@ -585,56 +576,32 @@ static void acx565akm_panel_power_off(struct omap_dss_device *dssdev)
 
 	/* FIXME need to tweak this delay */
 	msleep(100);
-
-	src->ops->disable(src);
 }
 
-static int acx565akm_enable(struct omap_dss_device *dssdev)
+static void acx565akm_enable(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	int r;
-
-	dev_dbg(dssdev->dev, "%s\n", __func__);
-
-	if (!omapdss_device_is_connected(dssdev))
-		return -ENODEV;
-
-	if (omapdss_device_is_enabled(dssdev))
-		return 0;
 
 	mutex_lock(&ddata->mutex);
-	r = acx565akm_panel_power_on(dssdev);
+	acx565akm_panel_power_on(dssdev);
 	mutex_unlock(&ddata->mutex);
-	if (r)
-		return r;
-
-	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
-
-	return 0;
 }
 
 static void acx565akm_disable(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 
-	dev_dbg(dssdev->dev, "%s\n", __func__);
-
-	if (!omapdss_device_is_enabled(dssdev))
-		return;
-
 	mutex_lock(&ddata->mutex);
 	acx565akm_panel_power_off(dssdev);
 	mutex_unlock(&ddata->mutex);
-
-	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
 }
 
-static void acx565akm_get_timings(struct omap_dss_device *dssdev,
-				  struct videomode *vm)
+static int acx565akm_get_modes(struct omap_dss_device *dssdev,
+			       struct drm_connector *connector)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 
-	*vm = ddata->vm;
+	return omapdss_display_get_modes(connector, &ddata->vm);
 }
 
 static const struct omap_dss_device_ops acx565akm_ops = {
@@ -644,7 +611,7 @@ static const struct omap_dss_device_ops acx565akm_ops = {
 	.enable		= acx565akm_enable,
 	.disable	= acx565akm_disable,
 
-	.get_timings	= acx565akm_get_timings,
+	.get_modes	= acx565akm_get_modes,
 };
 
 static int acx565akm_probe(struct spi_device *spi)
@@ -739,10 +706,13 @@ static int acx565akm_probe(struct spi_device *spi)
 	dssdev->dev = &spi->dev;
 	dssdev->ops = &acx565akm_ops;
 	dssdev->type = OMAP_DISPLAY_TYPE_SDI;
+	dssdev->display = true;
 	dssdev->owner = THIS_MODULE;
 	dssdev->of_ports = BIT(0);
-	dssdev->bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_SYNC_NEGEDGE
-			  | DRM_BUS_FLAG_PIXDATA_POSEDGE;
+	dssdev->ops_flags = OMAP_DSS_DEVICE_OP_MODES;
+	dssdev->bus_flags = DRM_BUS_FLAG_DE_HIGH
+			  | DRM_BUS_FLAG_SYNC_DRIVE_NEGEDGE
+			  | DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE;
 
 	omapdss_display_init(dssdev);
 	omapdss_device_register(dssdev);
@@ -766,7 +736,8 @@ static int acx565akm_remove(struct spi_device *spi)
 
 	omapdss_device_unregister(dssdev);
 
-	acx565akm_disable(dssdev);
+	if (omapdss_device_is_enabled(dssdev))
+		acx565akm_disable(dssdev);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-tpo-td028ttec1.c b/drivers/gpu/drm/omapdrm/displays/panel-tpo-td028ttec1.c
index 7ddc8c574a61..2ad161e33106 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-tpo-td028ttec1.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-tpo-td028ttec1.c
@@ -35,6 +35,8 @@ struct panel_drv_data {
 
 	struct videomode vm;
 
+	struct backlight_device *backlight;
+
 	struct spi_device *spi_dev;
 };
 
@@ -169,24 +171,12 @@ static void td028ttec1_panel_disconnect(struct omap_dss_device *src,
 {
 }
 
-static int td028ttec1_panel_enable(struct omap_dss_device *dssdev)
+static void td028ttec1_panel_enable(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *src = dssdev->src;
-	int r;
-
-	if (!omapdss_device_is_connected(dssdev))
-		return -ENODEV;
-
-	if (omapdss_device_is_enabled(dssdev))
-		return 0;
-
-	r = src->ops->enable(src);
-	if (r)
-		return r;
+	int r = 0;
 
-	dev_dbg(dssdev->dev, "td028ttec1_panel_enable() - state %d\n",
-		dssdev->state);
+	dev_dbg(dssdev->dev, "%s: state %d\n", __func__, dssdev->state);
 
 	/* three times command zero */
 	r |= jbt_ret_write_0(ddata, 0x00);
@@ -197,8 +187,8 @@ static int td028ttec1_panel_enable(struct omap_dss_device *dssdev)
 	usleep_range(1000, 2000);
 
 	if (r) {
-		dev_warn(dssdev->dev, "transfer error\n");
-		goto transfer_err;
+		dev_warn(dssdev->dev, "%s: transfer error\n", __func__);
+		return;
 	}
 
 	/* deep standby out */
@@ -268,20 +258,17 @@ static int td028ttec1_panel_enable(struct omap_dss_device *dssdev)
 
 	r |= jbt_ret_write_0(ddata, JBT_REG_DISPLAY_ON);
 
-	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
-
-transfer_err:
+	if (r)
+		dev_err(dssdev->dev, "%s: write error\n", __func__);
 
-	return r ? -EIO : 0;
+	backlight_enable(ddata->backlight);
 }
 
 static void td028ttec1_panel_disable(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *src = dssdev->src;
 
-	if (!omapdss_device_is_enabled(dssdev))
-		return;
+	backlight_disable(ddata->backlight);
 
 	dev_dbg(dssdev->dev, "td028ttec1_panel_disable()\n");
 
@@ -289,18 +276,14 @@ static void td028ttec1_panel_disable(struct omap_dss_device *dssdev)
 	jbt_reg_write_2(ddata, JBT_REG_OUTPUT_CONTROL, 0x8002);
 	jbt_ret_write_0(ddata, JBT_REG_SLEEP_IN);
 	jbt_reg_write_1(ddata, JBT_REG_POWER_ON_OFF, 0x00);
-
-	src->ops->disable(src);
-
-	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
 }
 
-static void td028ttec1_panel_get_timings(struct omap_dss_device *dssdev,
-					 struct videomode *vm)
+static int td028ttec1_panel_get_modes(struct omap_dss_device *dssdev,
+				      struct drm_connector *connector)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 
-	*vm = ddata->vm;
+	return omapdss_display_get_modes(connector, &ddata->vm);
 }
 
 static const struct omap_dss_device_ops td028ttec1_ops = {
@@ -310,7 +293,7 @@ static const struct omap_dss_device_ops td028ttec1_ops = {
 	.enable		= td028ttec1_panel_enable,
 	.disable	= td028ttec1_panel_disable,
 
-	.get_timings	= td028ttec1_panel_get_timings,
+	.get_modes	= td028ttec1_panel_get_modes,
 };
 
 static int td028ttec1_panel_probe(struct spi_device *spi)
@@ -334,6 +317,10 @@ static int td028ttec1_panel_probe(struct spi_device *spi)
 	if (ddata == NULL)
 		return -ENOMEM;
 
+	ddata->backlight = devm_of_find_backlight(&spi->dev);
+	if (IS_ERR(ddata->backlight))
+		return PTR_ERR(ddata->backlight);
+
 	dev_set_drvdata(&spi->dev, ddata);
 
 	ddata->spi_dev = spi;
@@ -344,15 +331,18 @@ static int td028ttec1_panel_probe(struct spi_device *spi)
 	dssdev->dev = &spi->dev;
 	dssdev->ops = &td028ttec1_ops;
 	dssdev->type = OMAP_DISPLAY_TYPE_DPI;
+	dssdev->display = true;
 	dssdev->owner = THIS_MODULE;
 	dssdev->of_ports = BIT(0);
+	dssdev->ops_flags = OMAP_DSS_DEVICE_OP_MODES;
 
 	/*
 	 * Note: According to the panel documentation:
 	 * SYNC needs to be driven on the FALLING edge
 	 */
-	dssdev->bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_SYNC_POSEDGE
-			  | DRM_BUS_FLAG_PIXDATA_NEGEDGE;
+	dssdev->bus_flags = DRM_BUS_FLAG_DE_HIGH
+			  | DRM_BUS_FLAG_SYNC_DRIVE_POSEDGE
+			  | DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE;
 
 	omapdss_display_init(dssdev);
 	omapdss_device_register(dssdev);
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-tpo-td043mtea1.c b/drivers/gpu/drm/omapdrm/displays/panel-tpo-td043mtea1.c
index 8440fcb744d9..0b692fc7e5ea 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-tpo-td043mtea1.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-tpo-td043mtea1.c
@@ -320,22 +320,11 @@ static void tpo_td043_disconnect(struct omap_dss_device *src,
 {
 }
 
-static int tpo_td043_enable(struct omap_dss_device *dssdev)
+static void tpo_td043_enable(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *src = dssdev->src;
 	int r;
 
-	if (!omapdss_device_is_connected(dssdev))
-		return -ENODEV;
-
-	if (omapdss_device_is_enabled(dssdev))
-		return 0;
-
-	r = src->ops->enable(src);
-	if (r)
-		return r;
-
 	/*
 	 * If we are resuming from system suspend, SPI clocks might not be
 	 * enabled yet, so we'll program the LCD from SPI PM resume callback.
@@ -343,38 +332,27 @@ static int tpo_td043_enable(struct omap_dss_device *dssdev)
 	if (!ddata->spi_suspended) {
 		r = tpo_td043_power_on(ddata);
 		if (r) {
-			src->ops->disable(src);
-			return r;
+			dev_err(&ddata->spi->dev, "%s: power on failed (%d)\n",
+				__func__, r);
+			return;
 		}
 	}
-
-	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
-
-	return 0;
 }
 
 static void tpo_td043_disable(struct omap_dss_device *dssdev)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
-	struct omap_dss_device *src = dssdev->src;
-
-	if (!omapdss_device_is_enabled(dssdev))
-		return;
-
-	src->ops->disable(src);
 
 	if (!ddata->spi_suspended)
 		tpo_td043_power_off(ddata);
-
-	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
 }
 
-static void tpo_td043_get_timings(struct omap_dss_device *dssdev,
-				  struct videomode *vm)
+static int tpo_td043_get_modes(struct omap_dss_device *dssdev,
+			       struct drm_connector *connector)
 {
 	struct panel_drv_data *ddata = to_panel_data(dssdev);
 
-	*vm = ddata->vm;
+	return omapdss_display_get_modes(connector, &ddata->vm);
 }
 
 static const struct omap_dss_device_ops tpo_td043_ops = {
@@ -384,7 +362,7 @@ static const struct omap_dss_device_ops tpo_td043_ops = {
 	.enable		= tpo_td043_enable,
 	.disable	= tpo_td043_disable,
 
-	.get_timings	= tpo_td043_get_timings,
+	.get_modes	= tpo_td043_get_modes,
 };
 
 static int tpo_td043_probe(struct spi_device *spi)
@@ -442,15 +420,18 @@ static int tpo_td043_probe(struct spi_device *spi)
 	dssdev->dev = &spi->dev;
 	dssdev->ops = &tpo_td043_ops;
 	dssdev->type = OMAP_DISPLAY_TYPE_DPI;
+	dssdev->display = true;
 	dssdev->owner = THIS_MODULE;
 	dssdev->of_ports = BIT(0);
+	dssdev->ops_flags = OMAP_DSS_DEVICE_OP_MODES;
 
 	/*
 	 * Note: According to the panel documentation:
 	 * SYNC needs to be driven on the FALLING edge
 	 */
-	dssdev->bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_SYNC_POSEDGE
-			  | DRM_BUS_FLAG_PIXDATA_NEGEDGE;
+	dssdev->bus_flags = DRM_BUS_FLAG_DE_HIGH
+			  | DRM_BUS_FLAG_SYNC_DRIVE_POSEDGE
+			  | DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE;
 
 	omapdss_display_init(dssdev);
 	omapdss_device_register(dssdev);
@@ -467,7 +448,8 @@ static int tpo_td043_remove(struct spi_device *spi)
 
 	omapdss_device_unregister(dssdev);
 
-	tpo_td043_disable(dssdev);
+	if (omapdss_device_is_enabled(dssdev))
+		tpo_td043_disable(dssdev);
 
 	sysfs_remove_group(&spi->dev.kobj, &tpo_td043_attr_group);
 
diff --git a/drivers/gpu/drm/omapdrm/dss/base.c b/drivers/gpu/drm/omapdrm/dss/base.c
index 472f56e3de70..f8dad99013e8 100644
--- a/drivers/gpu/drm/omapdrm/dss/base.c
+++ b/drivers/gpu/drm/omapdrm/dss/base.c
@@ -19,6 +19,7 @@
 #include <linux/mutex.h>
 #include <linux/of.h>
 #include <linux/of_graph.h>
+#include <linux/platform_device.h>
 
 #include "dss.h"
 #include "omapdss.h"
@@ -112,13 +113,12 @@ void omapdss_device_put(struct omap_dss_device *dssdev)
 }
 EXPORT_SYMBOL(omapdss_device_put);
 
-struct omap_dss_device *omapdss_find_device_by_port(struct device_node *src,
-						    unsigned int port)
+struct omap_dss_device *omapdss_find_device_by_node(struct device_node *node)
 {
 	struct omap_dss_device *dssdev;
 
 	list_for_each_entry(dssdev, &omapdss_devices_list, list) {
-		if (dssdev->dev->of_node == src && dssdev->of_ports & BIT(port))
+		if (dssdev->dev->of_node == node)
 			return omapdss_device_get(dssdev);
 	}
 
@@ -126,13 +126,10 @@ struct omap_dss_device *omapdss_find_device_by_port(struct device_node *src,
 }
 
 /*
- * Search for the next device starting at @from. The type argument specfies
- * which device types to consider when searching. Searching for multiple types
- * is supported by and'ing their type flags. Release the reference to the @from
- * device, and acquire a reference to the returned device if found.
+ * Search for the next output device starting at @from. Release the reference to
+ * the @from device, and acquire a reference to the returned device if found.
  */
-struct omap_dss_device *omapdss_device_get_next(struct omap_dss_device *from,
-						enum omap_dss_device_type type)
+struct omap_dss_device *omapdss_device_next_output(struct omap_dss_device *from)
 {
 	struct omap_dss_device *dssdev;
 	struct list_head *list;
@@ -160,15 +157,8 @@ struct omap_dss_device *omapdss_device_get_next(struct omap_dss_device *from,
 			goto done;
 		}
 
-		/*
-		 * Accept display entities if the display type is requested,
-		 * and output entities if the output type is requested.
-		 */
-		if ((type & OMAP_DSS_DEVICE_TYPE_DISPLAY) &&
-		    !dssdev->output_type)
-			goto done;
-		if ((type & OMAP_DSS_DEVICE_TYPE_OUTPUT) && dssdev->id &&
-		    dssdev->next)
+		if (dssdev->id &&
+		    (dssdev->next || dssdev->bridge || dssdev->panel))
 			goto done;
 	}
 
@@ -183,7 +173,12 @@ done:
 	mutex_unlock(&omapdss_devices_lock);
 	return dssdev;
 }
-EXPORT_SYMBOL(omapdss_device_get_next);
+EXPORT_SYMBOL(omapdss_device_next_output);
+
+static bool omapdss_device_is_connected(struct omap_dss_device *dssdev)
+{
+	return dssdev->dss;
+}
 
 int omapdss_device_connect(struct dss_device *dss,
 			   struct omap_dss_device *src,
@@ -191,7 +186,19 @@ int omapdss_device_connect(struct dss_device *dss,
 {
 	int ret;
 
-	dev_dbg(dst->dev, "connect\n");
+	dev_dbg(&dss->pdev->dev, "connect(%s, %s)\n",
+		src ? dev_name(src->dev) : "NULL",
+		dst ? dev_name(dst->dev) : "NULL");
+
+	if (!dst) {
+		/*
+		 * The destination is NULL when the source is connected to a
+		 * bridge or panel instead of a DSS device. Stop here, we will
+		 * attach the bridge or panel later when we will have a DRM
+		 * encoder.
+		 */
+		return src && (src->bridge || src->panel) ? 0 : -EINVAL;
+	}
 
 	if (omapdss_device_is_connected(dst))
 		return -EBUSY;
@@ -204,12 +211,6 @@ int omapdss_device_connect(struct dss_device *dss,
 		return ret;
 	}
 
-	if (src) {
-		WARN_ON(src->dst);
-		dst->src = src;
-		src->dst = dst;
-	}
-
 	return 0;
 }
 EXPORT_SYMBOL_GPL(omapdss_device_connect);
@@ -217,19 +218,20 @@ EXPORT_SYMBOL_GPL(omapdss_device_connect);
 void omapdss_device_disconnect(struct omap_dss_device *src,
 			       struct omap_dss_device *dst)
 {
-	dev_dbg(dst->dev, "disconnect\n");
+	struct dss_device *dss = src ? src->dss : dst->dss;
 
-	if (!dst->id && !omapdss_device_is_connected(dst)) {
-		WARN_ON(dst->output_type);
+	dev_dbg(&dss->pdev->dev, "disconnect(%s, %s)\n",
+		src ? dev_name(src->dev) : "NULL",
+		dst ? dev_name(dst->dev) : "NULL");
+
+	if (!dst) {
+		WARN_ON(!src->bridge && !src->panel);
 		return;
 	}
 
-	if (src) {
-		if (WARN_ON(dst != src->dst))
-			return;
-
-		dst->src = NULL;
-		src->dst = NULL;
+	if (!dst->id && !omapdss_device_is_connected(dst)) {
+		WARN_ON(!dst->display);
+		return;
 	}
 
 	WARN_ON(dst->state != OMAP_DSS_DISPLAY_DISABLED);
@@ -239,6 +241,58 @@ void omapdss_device_disconnect(struct omap_dss_device *src,
 }
 EXPORT_SYMBOL_GPL(omapdss_device_disconnect);
 
+void omapdss_device_pre_enable(struct omap_dss_device *dssdev)
+{
+	if (!dssdev)
+		return;
+
+	omapdss_device_pre_enable(dssdev->next);
+
+	if (dssdev->ops->pre_enable)
+		dssdev->ops->pre_enable(dssdev);
+}
+EXPORT_SYMBOL_GPL(omapdss_device_pre_enable);
+
+void omapdss_device_enable(struct omap_dss_device *dssdev)
+{
+	if (!dssdev)
+		return;
+
+	if (dssdev->ops->enable)
+		dssdev->ops->enable(dssdev);
+
+	omapdss_device_enable(dssdev->next);
+
+	dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
+}
+EXPORT_SYMBOL_GPL(omapdss_device_enable);
+
+void omapdss_device_disable(struct omap_dss_device *dssdev)
+{
+	if (!dssdev)
+		return;
+
+	omapdss_device_disable(dssdev->next);
+
+	if (dssdev->ops->disable)
+		dssdev->ops->disable(dssdev);
+}
+EXPORT_SYMBOL_GPL(omapdss_device_disable);
+
+void omapdss_device_post_disable(struct omap_dss_device *dssdev)
+{
+	if (!dssdev)
+		return;
+
+	if (dssdev->ops->post_disable)
+		dssdev->ops->post_disable(dssdev);
+
+	omapdss_device_post_disable(dssdev->next);
+
+	dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
+}
+EXPORT_SYMBOL_GPL(omapdss_device_post_disable);
+
 /* -----------------------------------------------------------------------------
  * Components Handling
  */
@@ -249,6 +303,7 @@ struct omapdss_comp_node {
 	struct list_head list;
 	struct device_node *node;
 	bool dss_core_component;
+	const char *compat;
 };
 
 static bool omapdss_list_contains(const struct device_node *node)
@@ -266,13 +321,20 @@ static bool omapdss_list_contains(const struct device_node *node)
 static void omapdss_walk_device(struct device *dev, struct device_node *node,
 				bool dss_core)
 {
+	struct omapdss_comp_node *comp;
 	struct device_node *n;
-	struct omapdss_comp_node *comp = devm_kzalloc(dev, sizeof(*comp),
-						      GFP_KERNEL);
+	const char *compat;
+	int ret;
 
+	ret = of_property_read_string(node, "compatible", &compat);
+	if (ret < 0)
+		return;
+
+	comp = devm_kzalloc(dev, sizeof(*comp), GFP_KERNEL);
 	if (comp) {
 		comp->node = node;
 		comp->dss_core_component = dss_core;
+		comp->compat = compat;
 		list_add(&comp->list, &omapdss_comp_list);
 	}
 
@@ -312,12 +374,8 @@ void omapdss_gather_components(struct device *dev)
 
 	omapdss_walk_device(dev, dev->of_node, true);
 
-	for_each_available_child_of_node(dev->of_node, child) {
-		if (!of_find_property(child, "compatible", NULL))
-			continue;
-
+	for_each_available_child_of_node(dev->of_node, child)
 		omapdss_walk_device(dev, child, true);
-	}
 }
 EXPORT_SYMBOL(omapdss_gather_components);
 
@@ -325,6 +383,8 @@ static bool omapdss_component_is_loaded(struct omapdss_comp_node *comp)
 {
 	if (comp->dss_core_component)
 		return true;
+	if (!strstarts(comp->compat, "omapdss,"))
+		return true;
 	if (omapdss_device_is_registered(comp->node))
 		return true;
 
diff --git a/drivers/gpu/drm/omapdrm/dss/display.c b/drivers/gpu/drm/omapdrm/dss/display.c
index 34b2a4ef63a4..e93f61a567a8 100644
--- a/drivers/gpu/drm/omapdrm/dss/display.c
+++ b/drivers/gpu/drm/omapdrm/dss/display.c
@@ -23,6 +23,9 @@
 #include <linux/kernel.h>
 #include <linux/of.h>
 
+#include <drm/drm_connector.h>
+#include <drm/drm_modes.h>
+
 #include "omapdss.h"
 
 static int disp_num_counter;
@@ -39,8 +42,6 @@ void omapdss_display_init(struct omap_dss_device *dssdev)
 	if (id < 0)
 		id = disp_num_counter++;
 
-	dssdev->alias_id = id;
-
 	/* Use 'label' property for name, if it exists */
 	of_property_read_string(dssdev->dev->of_node, "label", &dssdev->name);
 
@@ -58,3 +59,22 @@ struct omap_dss_device *omapdss_display_get(struct omap_dss_device *output)
 	return omapdss_device_get(output);
 }
 EXPORT_SYMBOL_GPL(omapdss_display_get);
+
+int omapdss_display_get_modes(struct drm_connector *connector,
+			      const struct videomode *vm)
+{
+	struct drm_display_mode *mode;
+
+	mode = drm_mode_create(connector->dev);
+	if (!mode)
+		return 0;
+
+	drm_display_mode_from_videomode(vm, mode);
+
+	mode->type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED;
+	drm_mode_set_name(mode);
+	drm_mode_probed_add(connector, mode);
+
+	return 1;
+}
+EXPORT_SYMBOL_GPL(omapdss_display_get_modes);
diff --git a/drivers/gpu/drm/omapdrm/dss/dpi.c b/drivers/gpu/drm/omapdrm/dss/dpi.c
index ca4f3c4c6318..cc78dfa07f04 100644
--- a/drivers/gpu/drm/omapdrm/dss/dpi.c
+++ b/drivers/gpu/drm/omapdrm/dss/dpi.c
@@ -47,8 +47,8 @@ struct dpi_data {
 
 	struct mutex lock;
 
-	struct videomode vm;
 	struct dss_lcd_mgr_config mgr_config;
+	unsigned long pixelclock;
 	int data_lines;
 
 	struct omap_dss_device output;
@@ -347,16 +347,15 @@ static int dpi_set_dispc_clk(struct dpi_data *dpi, unsigned long pck_req,
 
 static int dpi_set_mode(struct dpi_data *dpi)
 {
-	const struct videomode *vm = &dpi->vm;
 	int lck_div = 0, pck_div = 0;
 	unsigned long fck = 0;
 	int r = 0;
 
 	if (dpi->pll)
 		r = dpi_set_pll_clk(dpi, dpi->output.dispc_channel,
-				    vm->pixelclock, &fck, &lck_div, &pck_div);
+				    dpi->pixelclock, &fck, &lck_div, &pck_div);
 	else
-		r = dpi_set_dispc_clk(dpi, vm->pixelclock, &fck,
+		r = dpi_set_dispc_clk(dpi, dpi->pixelclock, &fck,
 				&lck_div, &pck_div);
 	if (r)
 		return r;
@@ -378,7 +377,7 @@ static void dpi_config_lcd_manager(struct dpi_data *dpi)
 	dss_mgr_set_lcd_config(&dpi->output, &dpi->mgr_config);
 }
 
-static int dpi_display_enable(struct omap_dss_device *dssdev)
+static void dpi_display_enable(struct omap_dss_device *dssdev)
 {
 	struct dpi_data *dpi = dpi_get_data_from_dssdev(dssdev);
 	struct omap_dss_device *out = &dpi->output;
@@ -386,12 +385,6 @@ static int dpi_display_enable(struct omap_dss_device *dssdev)
 
 	mutex_lock(&dpi->lock);
 
-	if (!out->dispc_channel_connected) {
-		DSSERR("failed to enable display: no output/manager\n");
-		r = -ENODEV;
-		goto err_no_out_mgr;
-	}
-
 	if (dpi->vdds_dsi_reg) {
 		r = regulator_enable(dpi->vdds_dsi_reg);
 		if (r)
@@ -426,7 +419,7 @@ static int dpi_display_enable(struct omap_dss_device *dssdev)
 
 	mutex_unlock(&dpi->lock);
 
-	return 0;
+	return;
 
 err_mgr_enable:
 err_set_mode:
@@ -439,9 +432,7 @@ err_get_dispc:
 	if (dpi->vdds_dsi_reg)
 		regulator_disable(dpi->vdds_dsi_reg);
 err_reg_enable:
-err_no_out_mgr:
 	mutex_unlock(&dpi->lock);
-	return r;
 }
 
 static void dpi_display_disable(struct omap_dss_device *dssdev)
@@ -467,7 +458,7 @@ static void dpi_display_disable(struct omap_dss_device *dssdev)
 }
 
 static void dpi_set_timings(struct omap_dss_device *dssdev,
-			    const struct videomode *vm)
+			    const struct drm_display_mode *mode)
 {
 	struct dpi_data *dpi = dpi_get_data_from_dssdev(dssdev);
 
@@ -475,13 +466,13 @@ static void dpi_set_timings(struct omap_dss_device *dssdev,
 
 	mutex_lock(&dpi->lock);
 
-	dpi->vm = *vm;
+	dpi->pixelclock = mode->clock * 1000;
 
 	mutex_unlock(&dpi->lock);
 }
 
 static int dpi_check_timings(struct omap_dss_device *dssdev,
-			     struct videomode *vm)
+			     struct drm_display_mode *mode)
 {
 	struct dpi_data *dpi = dpi_get_data_from_dssdev(dssdev);
 	int lck_div, pck_div;
@@ -490,20 +481,20 @@ static int dpi_check_timings(struct omap_dss_device *dssdev,
 	struct dpi_clk_calc_ctx ctx;
 	bool ok;
 
-	if (vm->hactive % 8 != 0)
+	if (mode->hdisplay % 8 != 0)
 		return -EINVAL;
 
-	if (vm->pixelclock == 0)
+	if (mode->clock == 0)
 		return -EINVAL;
 
 	if (dpi->pll) {
-		ok = dpi_pll_clk_calc(dpi, vm->pixelclock, &ctx);
+		ok = dpi_pll_clk_calc(dpi, mode->clock * 1000, &ctx);
 		if (!ok)
 			return -EINVAL;
 
 		fck = ctx.pll_cinfo.clkout[ctx.clkout_idx];
 	} else {
-		ok = dpi_dss_clk_calc(dpi, vm->pixelclock, &ctx);
+		ok = dpi_dss_clk_calc(dpi, mode->clock * 1000, &ctx);
 		if (!ok)
 			return -EINVAL;
 
@@ -515,7 +506,7 @@ static int dpi_check_timings(struct omap_dss_device *dssdev,
 
 	pck = fck / lck_div / pck_div;
 
-	vm->pixelclock = pck;
+	mode->clock = pck / 1000;
 
 	return 0;
 }
@@ -596,23 +587,15 @@ static int dpi_connect(struct omap_dss_device *src,
 		       struct omap_dss_device *dst)
 {
 	struct dpi_data *dpi = dpi_get_data_from_dssdev(dst);
-	int r;
 
 	dpi_init_pll(dpi);
 
-	r = omapdss_device_connect(dst->dss, dst, dst->next);
-	if (r)
-		return r;
-
-	dst->dispc_channel_connected = true;
-	return 0;
+	return omapdss_device_connect(dst->dss, dst, dst->next);
 }
 
 static void dpi_disconnect(struct omap_dss_device *src,
 			   struct omap_dss_device *dst)
 {
-	dst->dispc_channel_connected = false;
-
 	omapdss_device_disconnect(dst, dst->next);
 }
 
@@ -651,25 +634,15 @@ static int dpi_init_output_port(struct dpi_data *dpi, struct device_node *port)
 
 	out->dev = &dpi->pdev->dev;
 	out->id = OMAP_DSS_OUTPUT_DPI;
-	out->output_type = OMAP_DISPLAY_TYPE_DPI;
+	out->type = OMAP_DISPLAY_TYPE_DPI;
 	out->dispc_channel = dpi_get_channel(dpi);
 	out->of_ports = BIT(port_num);
 	out->ops = &dpi_ops;
 	out->owner = THIS_MODULE;
 
-	out->next = omapdss_of_find_connected_device(out->dev->of_node, 0);
-	if (IS_ERR(out->next)) {
-		if (PTR_ERR(out->next) != -EPROBE_DEFER)
-			dev_err(out->dev, "failed to find video sink\n");
-		return PTR_ERR(out->next);
-	}
-
-	r = omapdss_output_validate(out);
-	if (r) {
-		omapdss_device_put(out->next);
-		out->next = NULL;
+	r = omapdss_device_init_output(out);
+	if (r < 0)
 		return r;
-	}
 
 	omapdss_device_register(out);
 
@@ -681,9 +654,8 @@ static void dpi_uninit_output_port(struct device_node *port)
 	struct dpi_data *dpi = port->data;
 	struct omap_dss_device *out = &dpi->output;
 
-	if (out->next)
-		omapdss_device_put(out->next);
 	omapdss_device_unregister(out);
+	omapdss_device_cleanup_output(out);
 }
 
 static const struct soc_device_attribute dpi_soc_devices[] = {
diff --git a/drivers/gpu/drm/omapdrm/dss/dsi.c b/drivers/gpu/drm/omapdrm/dss/dsi.c
index 64fb788b6647..5202862d89b5 100644
--- a/drivers/gpu/drm/omapdrm/dss/dsi.c
+++ b/drivers/gpu/drm/omapdrm/dss/dsi.c
@@ -1342,12 +1342,9 @@ static int dsi_pll_enable(struct dss_pll *pll)
 	 */
 	dsi_enable_scp_clk(dsi);
 
-	if (!dsi->vdds_dsi_enabled) {
-		r = regulator_enable(dsi->vdds_dsi_reg);
-		if (r)
-			goto err0;
-		dsi->vdds_dsi_enabled = true;
-	}
+	r = regulator_enable(dsi->vdds_dsi_reg);
+	if (r)
+		goto err0;
 
 	/* XXX PLL does not come out of reset without this... */
 	dispc_pck_free_enable(dsi->dss->dispc, 1);
@@ -1372,36 +1369,25 @@ static int dsi_pll_enable(struct dss_pll *pll)
 
 	return 0;
 err1:
-	if (dsi->vdds_dsi_enabled) {
-		regulator_disable(dsi->vdds_dsi_reg);
-		dsi->vdds_dsi_enabled = false;
-	}
+	regulator_disable(dsi->vdds_dsi_reg);
 err0:
 	dsi_disable_scp_clk(dsi);
 	dsi_runtime_put(dsi);
 	return r;
 }
 
-static void dsi_pll_uninit(struct dsi_data *dsi, bool disconnect_lanes)
+static void dsi_pll_disable(struct dss_pll *pll)
 {
+	struct dsi_data *dsi = container_of(pll, struct dsi_data, pll);
+
 	dsi_pll_power(dsi, DSI_PLL_POWER_OFF);
-	if (disconnect_lanes) {
-		WARN_ON(!dsi->vdds_dsi_enabled);
-		regulator_disable(dsi->vdds_dsi_reg);
-		dsi->vdds_dsi_enabled = false;
-	}
+
+	regulator_disable(dsi->vdds_dsi_reg);
 
 	dsi_disable_scp_clk(dsi);
 	dsi_runtime_put(dsi);
 
-	DSSDBG("PLL uninit done\n");
-}
-
-static void dsi_pll_disable(struct dss_pll *pll)
-{
-	struct dsi_data *dsi = container_of(pll, struct dsi_data, pll);
-
-	dsi_pll_uninit(dsi, true);
+	DSSDBG("PLL disable done\n");
 }
 
 static int dsi_dump_dsi_clocks(struct seq_file *s, void *p)
@@ -3753,19 +3739,13 @@ static int dsi_enable_video_output(struct omap_dss_device *dssdev, int channel)
 {
 	struct dsi_data *dsi = to_dsi_data(dssdev);
 	int bpp = dsi_get_pixel_size(dsi->pix_fmt);
-	struct omap_dss_device *out = &dsi->output;
 	u8 data_type;
 	u16 word_count;
 	int r;
 
-	if (!out->dispc_channel_connected) {
-		DSSERR("failed to enable display: no output/manager\n");
-		return -ENODEV;
-	}
-
 	r = dsi_display_init_dispc(dsi);
 	if (r)
-		goto err_init_dispc;
+		return r;
 
 	if (dsi->mode == OMAP_DSS_DSI_VIDEO_MODE) {
 		switch (dsi->pix_fmt) {
@@ -3814,7 +3794,6 @@ err_mgr_enable:
 	}
 err_pix_fmt:
 	dsi_display_uninit_dispc(dsi);
-err_init_dispc:
 	return r;
 }
 
@@ -4096,11 +4075,11 @@ static int dsi_display_init_dsi(struct dsi_data *dsi)
 
 	r = dss_pll_enable(&dsi->pll);
 	if (r)
-		goto err0;
+		return r;
 
 	r = dsi_configure_dsi_clocks(dsi);
 	if (r)
-		goto err1;
+		goto err0;
 
 	dss_select_dsi_clk_source(dsi->dss, dsi->module_id,
 				  dsi->module_id == 0 ?
@@ -4108,6 +4087,14 @@ static int dsi_display_init_dsi(struct dsi_data *dsi)
 
 	DSSDBG("PLL OK\n");
 
+	if (!dsi->vdds_dsi_enabled) {
+		r = regulator_enable(dsi->vdds_dsi_reg);
+		if (r)
+			goto err1;
+
+		dsi->vdds_dsi_enabled = true;
+	}
+
 	r = dsi_cio_init(dsi);
 	if (r)
 		goto err2;
@@ -4136,10 +4123,13 @@ static int dsi_display_init_dsi(struct dsi_data *dsi)
 err3:
 	dsi_cio_uninit(dsi);
 err2:
-	dss_select_dsi_clk_source(dsi->dss, dsi->module_id, DSS_CLK_SRC_FCK);
+	regulator_disable(dsi->vdds_dsi_reg);
+	dsi->vdds_dsi_enabled = false;
 err1:
-	dss_pll_disable(&dsi->pll);
+	dss_select_dsi_clk_source(dsi->dss, dsi->module_id, DSS_CLK_SRC_FCK);
 err0:
+	dss_pll_disable(&dsi->pll);
+
 	return r;
 }
 
@@ -4158,13 +4148,18 @@ static void dsi_display_uninit_dsi(struct dsi_data *dsi, bool disconnect_lanes,
 
 	dss_select_dsi_clk_source(dsi->dss, dsi->module_id, DSS_CLK_SRC_FCK);
 	dsi_cio_uninit(dsi);
-	dsi_pll_uninit(dsi, disconnect_lanes);
+	dss_pll_disable(&dsi->pll);
+
+	if (disconnect_lanes) {
+		regulator_disable(dsi->vdds_dsi_reg);
+		dsi->vdds_dsi_enabled = false;
+	}
 }
 
-static int dsi_display_enable(struct omap_dss_device *dssdev)
+static void dsi_display_enable(struct omap_dss_device *dssdev)
 {
 	struct dsi_data *dsi = to_dsi_data(dssdev);
-	int r = 0;
+	int r;
 
 	DSSDBG("dsi_display_enable\n");
 
@@ -4184,14 +4179,13 @@ static int dsi_display_enable(struct omap_dss_device *dssdev)
 
 	mutex_unlock(&dsi->lock);
 
-	return 0;
+	return;
 
 err_init_dsi:
 	dsi_runtime_put(dsi);
 err_get_dsi:
 	mutex_unlock(&dsi->lock);
 	DSSDBG("dsi_display_enable FAILED\n");
-	return r;
 }
 
 static void dsi_display_disable(struct omap_dss_device *dssdev,
@@ -4888,21 +4882,12 @@ static int dsi_get_clocks(struct dsi_data *dsi)
 static int dsi_connect(struct omap_dss_device *src,
 		       struct omap_dss_device *dst)
 {
-	int r;
-
-	r = omapdss_device_connect(dst->dss, dst, dst->next);
-	if (r)
-		return r;
-
-	dst->dispc_channel_connected = true;
-	return 0;
+	return omapdss_device_connect(dst->dss, dst, dst->next);
 }
 
 static void dsi_disconnect(struct omap_dss_device *src,
 			   struct omap_dss_device *dst)
 {
-	dst->dispc_channel_connected = false;
-
 	omapdss_device_disconnect(dst, dst->next);
 }
 
@@ -5138,29 +5123,19 @@ static int dsi_init_output(struct dsi_data *dsi)
 	out->id = dsi->module_id == 0 ?
 			OMAP_DSS_OUTPUT_DSI1 : OMAP_DSS_OUTPUT_DSI2;
 
-	out->output_type = OMAP_DISPLAY_TYPE_DSI;
+	out->type = OMAP_DISPLAY_TYPE_DSI;
 	out->name = dsi->module_id == 0 ? "dsi.0" : "dsi.1";
 	out->dispc_channel = dsi_get_channel(dsi);
 	out->ops = &dsi_ops;
 	out->owner = THIS_MODULE;
 	out->of_ports = BIT(0);
-	out->bus_flags = DRM_BUS_FLAG_PIXDATA_POSEDGE
+	out->bus_flags = DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE
 		       | DRM_BUS_FLAG_DE_HIGH
-		       | DRM_BUS_FLAG_SYNC_NEGEDGE;
-
-	out->next = omapdss_of_find_connected_device(out->dev->of_node, 0);
-	if (IS_ERR(out->next)) {
-		if (PTR_ERR(out->next) != -EPROBE_DEFER)
-			dev_err(out->dev, "failed to find video sink\n");
-		return PTR_ERR(out->next);
-	}
+		       | DRM_BUS_FLAG_SYNC_DRIVE_NEGEDGE;
 
-	r = omapdss_output_validate(out);
-	if (r) {
-		omapdss_device_put(out->next);
-		out->next = NULL;
+	r = omapdss_device_init_output(out);
+	if (r < 0)
 		return r;
-	}
 
 	omapdss_device_register(out);
 
@@ -5171,9 +5146,8 @@ static void dsi_uninit_output(struct dsi_data *dsi)
 {
 	struct omap_dss_device *out = &dsi->output;
 
-	if (out->next)
-		omapdss_device_put(out->next);
 	omapdss_device_unregister(out);
+	omapdss_device_cleanup_output(out);
 }
 
 static int dsi_probe_of(struct dsi_data *dsi)
diff --git a/drivers/gpu/drm/omapdrm/dss/dss-of.c b/drivers/gpu/drm/omapdrm/dss/dss-of.c
index 0422597ac6b0..b2094055c5fc 100644
--- a/drivers/gpu/drm/omapdrm/dss/dss-of.c
+++ b/drivers/gpu/drm/omapdrm/dss/dss-of.c
@@ -12,71 +12,25 @@
  * more details.
  */
 
-#include <linux/device.h>
 #include <linux/err.h>
-#include <linux/module.h>
 #include <linux/of.h>
 #include <linux/of_graph.h>
-#include <linux/seq_file.h>
 
 #include "omapdss.h"
 
-static struct device_node *
-dss_of_port_get_parent_device(struct device_node *port)
-{
-	struct device_node *np;
-	int i;
-
-	if (!port)
-		return NULL;
-
-	np = of_get_parent(port);
-
-	for (i = 0; i < 2 && np; ++i) {
-		struct property *prop;
-
-		prop = of_find_property(np, "compatible", NULL);
-
-		if (prop)
-			return np;
-
-		np = of_get_next_parent(np);
-	}
-
-	return NULL;
-}
-
 struct omap_dss_device *
 omapdss_of_find_connected_device(struct device_node *node, unsigned int port)
 {
-	struct device_node *src_node;
-	struct device_node *src_port;
-	struct device_node *ep;
-	struct omap_dss_device *src;
-	u32 port_number = 0;
+	struct device_node *remote_node;
+	struct omap_dss_device *dssdev;
 
-	/* Get the endpoint... */
-	ep = of_graph_get_endpoint_by_regs(node, port, 0);
-	if (!ep)
+	remote_node = of_graph_get_remote_node(node, port, 0);
+	if (!remote_node)
 		return NULL;
 
-	/* ... and its remote port... */
-	src_port = of_graph_get_remote_port(ep);
-	of_node_put(ep);
-	if (!src_port)
-		return NULL;
-
-	/* ... and the remote port's number and parent... */
-	of_property_read_u32(src_port, "reg", &port_number);
-	src_node = dss_of_port_get_parent_device(src_port);
-	of_node_put(src_port);
-	if (!src_node)
-		return ERR_PTR(-EINVAL);
-
-	/* ... and finally the connected device. */
-	src = omapdss_find_device_by_port(src_node, port_number);
-	of_node_put(src_node);
+	dssdev = omapdss_find_device_by_node(remote_node);
+	of_node_put(remote_node);
 
-	return src ? src : ERR_PTR(-EPROBE_DEFER);
+	return dssdev ? dssdev : ERR_PTR(-EPROBE_DEFER);
 }
 EXPORT_SYMBOL_GPL(omapdss_of_find_connected_device);
diff --git a/drivers/gpu/drm/omapdrm/dss/dss.c b/drivers/gpu/drm/omapdrm/dss/dss.c
index 7553c7fc1c45..55e68863ef15 100644
--- a/drivers/gpu/drm/omapdrm/dss/dss.c
+++ b/drivers/gpu/drm/omapdrm/dss/dss.c
@@ -1560,7 +1560,7 @@ static void dss_shutdown(struct platform_device *pdev)
 
 	DSSDBG("shutdown\n");
 
-	for_each_dss_display(dssdev) {
+	for_each_dss_output(dssdev) {
 		if (dssdev->state == OMAP_DSS_DISPLAY_ACTIVE)
 			dssdev->ops->disable(dssdev);
 	}
diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi4.c b/drivers/gpu/drm/omapdrm/dss/hdmi4.c
index aabdda394c9c..6339e2756b34 100644
--- a/drivers/gpu/drm/omapdrm/dss/hdmi4.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi4.c
@@ -249,15 +249,15 @@ static void hdmi_power_off_full(struct omap_hdmi *hdmi)
 }
 
 static void hdmi_display_set_timings(struct omap_dss_device *dssdev,
-				     const struct videomode *vm)
+				     const struct drm_display_mode *mode)
 {
 	struct omap_hdmi *hdmi = dssdev_to_hdmi(dssdev);
 
 	mutex_lock(&hdmi->lock);
 
-	hdmi->cfg.vm = *vm;
+	drm_display_mode_to_videomode(mode, &hdmi->cfg.vm);
 
-	dispc_set_tv_pclk(hdmi->dss->dispc, vm->pixelclock);
+	dispc_set_tv_pclk(hdmi->dss->dispc, mode->clock * 1000);
 
 	mutex_unlock(&hdmi->lock);
 }
@@ -312,26 +312,20 @@ static void hdmi_stop_audio_stream(struct omap_hdmi *hd)
 	hdmi_wp_audio_enable(&hd->wp, false);
 }
 
-static int hdmi_display_enable(struct omap_dss_device *dssdev)
+static void hdmi_display_enable(struct omap_dss_device *dssdev)
 {
 	struct omap_hdmi *hdmi = dssdev_to_hdmi(dssdev);
 	unsigned long flags;
-	int r = 0;
+	int r;
 
 	DSSDBG("ENTER hdmi_display_enable\n");
 
 	mutex_lock(&hdmi->lock);
 
-	if (!dssdev->dispc_channel_connected) {
-		DSSERR("failed to enable display: no output/manager\n");
-		r = -ENODEV;
-		goto err0;
-	}
-
 	r = hdmi_power_on_full(hdmi);
 	if (r) {
 		DSSERR("failed to power on device\n");
-		goto err0;
+		goto done;
 	}
 
 	if (hdmi->audio_configured) {
@@ -351,12 +345,8 @@ static int hdmi_display_enable(struct omap_dss_device *dssdev)
 	hdmi->display_enabled = true;
 	spin_unlock_irqrestore(&hdmi->audio_playing_lock, flags);
 
+done:
 	mutex_unlock(&hdmi->lock);
-	return 0;
-
-err0:
-	mutex_unlock(&hdmi->lock);
-	return r;
 }
 
 static void hdmi_display_disable(struct omap_dss_device *dssdev)
@@ -417,21 +407,12 @@ void hdmi4_core_disable(struct hdmi_core_data *core)
 static int hdmi_connect(struct omap_dss_device *src,
 			struct omap_dss_device *dst)
 {
-	int r;
-
-	r = omapdss_device_connect(dst->dss, dst, dst->next);
-	if (r)
-		return r;
-
-	dst->dispc_channel_connected = true;
-	return 0;
+	return omapdss_device_connect(dst->dss, dst, dst->next);
 }
 
 static void hdmi_disconnect(struct omap_dss_device *src,
 			    struct omap_dss_device *dst)
 {
-	dst->dispc_channel_connected = false;
-
 	omapdss_device_disconnect(dst, dst->next);
 }
 
@@ -698,7 +679,7 @@ static int hdmi4_init_output(struct omap_hdmi *hdmi)
 
 	out->dev = &hdmi->pdev->dev;
 	out->id = OMAP_DSS_OUTPUT_HDMI;
-	out->output_type = OMAP_DISPLAY_TYPE_HDMI;
+	out->type = OMAP_DISPLAY_TYPE_HDMI;
 	out->name = "hdmi.0";
 	out->dispc_channel = OMAP_DSS_CHANNEL_DIGIT;
 	out->ops = &hdmi_ops;
@@ -706,19 +687,9 @@ static int hdmi4_init_output(struct omap_hdmi *hdmi)
 	out->of_ports = BIT(0);
 	out->ops_flags = OMAP_DSS_DEVICE_OP_EDID;
 
-	out->next = omapdss_of_find_connected_device(out->dev->of_node, 0);
-	if (IS_ERR(out->next)) {
-		if (PTR_ERR(out->next) != -EPROBE_DEFER)
-			dev_err(out->dev, "failed to find video sink\n");
-		return PTR_ERR(out->next);
-	}
-
-	r = omapdss_output_validate(out);
-	if (r) {
-		omapdss_device_put(out->next);
-		out->next = NULL;
+	r = omapdss_device_init_output(out);
+	if (r < 0)
 		return r;
-	}
 
 	omapdss_device_register(out);
 
@@ -729,9 +700,8 @@ static void hdmi4_uninit_output(struct omap_hdmi *hdmi)
 {
 	struct omap_dss_device *out = &hdmi->output;
 
-	if (out->next)
-		omapdss_device_put(out->next);
 	omapdss_device_unregister(out);
+	omapdss_device_cleanup_output(out);
 }
 
 static int hdmi4_probe_of(struct omap_hdmi *hdmi)
diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi5.c b/drivers/gpu/drm/omapdrm/dss/hdmi5.c
index 9e8556f67a29..2955bbad13bb 100644
--- a/drivers/gpu/drm/omapdrm/dss/hdmi5.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi5.c
@@ -248,15 +248,15 @@ static void hdmi_power_off_full(struct omap_hdmi *hdmi)
 }
 
 static void hdmi_display_set_timings(struct omap_dss_device *dssdev,
-				     const struct videomode *vm)
+				     const struct drm_display_mode *mode)
 {
 	struct omap_hdmi *hdmi = dssdev_to_hdmi(dssdev);
 
 	mutex_lock(&hdmi->lock);
 
-	hdmi->cfg.vm = *vm;
+	drm_display_mode_to_videomode(mode, &hdmi->cfg.vm);
 
-	dispc_set_tv_pclk(hdmi->dss->dispc, vm->pixelclock);
+	dispc_set_tv_pclk(hdmi->dss->dispc, mode->clock * 1000);
 
 	mutex_unlock(&hdmi->lock);
 }
@@ -320,26 +320,20 @@ static void hdmi_stop_audio_stream(struct omap_hdmi *hd)
 	REG_FLD_MOD(hd->wp.base, HDMI_WP_SYSCONFIG, hd->wp_idlemode, 3, 2);
 }
 
-static int hdmi_display_enable(struct omap_dss_device *dssdev)
+static void hdmi_display_enable(struct omap_dss_device *dssdev)
 {
 	struct omap_hdmi *hdmi = dssdev_to_hdmi(dssdev);
 	unsigned long flags;
-	int r = 0;
+	int r;
 
 	DSSDBG("ENTER hdmi_display_enable\n");
 
 	mutex_lock(&hdmi->lock);
 
-	if (!dssdev->dispc_channel_connected) {
-		DSSERR("failed to enable display: no output/manager\n");
-		r = -ENODEV;
-		goto err0;
-	}
-
 	r = hdmi_power_on_full(hdmi);
 	if (r) {
 		DSSERR("failed to power on device\n");
-		goto err0;
+		goto done;
 	}
 
 	if (hdmi->audio_configured) {
@@ -359,12 +353,8 @@ static int hdmi_display_enable(struct omap_dss_device *dssdev)
 	hdmi->display_enabled = true;
 	spin_unlock_irqrestore(&hdmi->audio_playing_lock, flags);
 
+done:
 	mutex_unlock(&hdmi->lock);
-	return 0;
-
-err0:
-	mutex_unlock(&hdmi->lock);
-	return r;
 }
 
 static void hdmi_display_disable(struct omap_dss_device *dssdev)
@@ -422,21 +412,12 @@ static void hdmi_core_disable(struct omap_hdmi *hdmi)
 static int hdmi_connect(struct omap_dss_device *src,
 			struct omap_dss_device *dst)
 {
-	int r;
-
-	r = omapdss_device_connect(dst->dss, dst, dst->next);
-	if (r)
-		return r;
-
-	dst->dispc_channel_connected = true;
-	return 0;
+	return omapdss_device_connect(dst->dss, dst, dst->next);
 }
 
 static void hdmi_disconnect(struct omap_dss_device *src,
 			    struct omap_dss_device *dst)
 {
-	dst->dispc_channel_connected = false;
-
 	omapdss_device_disconnect(dst, dst->next);
 }
 
@@ -682,7 +663,7 @@ static int hdmi5_init_output(struct omap_hdmi *hdmi)
 
 	out->dev = &hdmi->pdev->dev;
 	out->id = OMAP_DSS_OUTPUT_HDMI;
-	out->output_type = OMAP_DISPLAY_TYPE_HDMI;
+	out->type = OMAP_DISPLAY_TYPE_HDMI;
 	out->name = "hdmi.0";
 	out->dispc_channel = OMAP_DSS_CHANNEL_DIGIT;
 	out->ops = &hdmi_ops;
@@ -690,19 +671,9 @@ static int hdmi5_init_output(struct omap_hdmi *hdmi)
 	out->of_ports = BIT(0);
 	out->ops_flags = OMAP_DSS_DEVICE_OP_EDID;
 
-	out->next = omapdss_of_find_connected_device(out->dev->of_node, 0);
-	if (IS_ERR(out->next)) {
-		if (PTR_ERR(out->next) != -EPROBE_DEFER)
-			dev_err(out->dev, "failed to find video sink\n");
-		return PTR_ERR(out->next);
-	}
-
-	r = omapdss_output_validate(out);
-	if (r) {
-		omapdss_device_put(out->next);
-		out->next = NULL;
+	r = omapdss_device_init_output(out);
+	if (r < 0)
 		return r;
-	}
 
 	omapdss_device_register(out);
 
@@ -713,9 +684,8 @@ static void hdmi5_uninit_output(struct omap_hdmi *hdmi)
 {
 	struct omap_dss_device *out = &hdmi->output;
 
-	if (out->next)
-		omapdss_device_put(out->next);
 	omapdss_device_unregister(out);
+	omapdss_device_cleanup_output(out);
 }
 
 static int hdmi5_probe_of(struct omap_hdmi *hdmi)
diff --git a/drivers/gpu/drm/omapdrm/dss/omapdss-boot-init.c b/drivers/gpu/drm/omapdrm/dss/omapdss-boot-init.c
index 3bfb95d230e0..2b41c75ce988 100644
--- a/drivers/gpu/drm/omapdrm/dss/omapdss-boot-init.c
+++ b/drivers/gpu/drm/omapdrm/dss/omapdss-boot-init.c
@@ -184,6 +184,22 @@ static const struct of_device_id omapdss_of_match[] __initconst = {
 	{},
 };
 
+static const struct of_device_id omapdss_of_fixups_whitelist[] __initconst = {
+	{ .compatible = "composite-video-connector" },
+	{ .compatible = "hdmi-connector" },
+	{ .compatible = "lgphilips,lb035q02" },
+	{ .compatible = "nec,nl8048hl11" },
+	{ .compatible = "panel-dsi-cm" },
+	{ .compatible = "sharp,ls037v7dw01" },
+	{ .compatible = "sony,acx565akm" },
+	{ .compatible = "svideo-connector" },
+	{ .compatible = "ti,opa362" },
+	{ .compatible = "ti,tpd12s015" },
+	{ .compatible = "toppoly,td028ttec1" },
+	{ .compatible = "tpo,td028ttec1" },
+	{ .compatible = "tpo,td043mtea1" },
+};
+
 static int __init omapdss_boot_init(void)
 {
 	struct device_node *dss, *child;
@@ -210,7 +226,7 @@ static int __init omapdss_boot_init(void)
 		n = list_first_entry(&dss_conv_list, struct dss_conv_node,
 			list);
 
-		if (!n->root)
+		if (of_match_node(omapdss_of_fixups_whitelist, n->node))
 			omapdss_omapify_node(n->node);
 
 		list_del(&n->list);
diff --git a/drivers/gpu/drm/omapdrm/dss/omapdss.h b/drivers/gpu/drm/omapdrm/dss/omapdss.h
index 33e15cb77efa..0c734d1f89e1 100644
--- a/drivers/gpu/drm/omapdrm/dss/omapdss.h
+++ b/drivers/gpu/drm/omapdrm/dss/omapdss.h
@@ -19,7 +19,6 @@
 #define __OMAP_DRM_DSS_H
 
 #include <linux/list.h>
-#include <linux/kobject.h>
 #include <linux/device.h>
 #include <linux/interrupt.h>
 #include <video/videomode.h>
@@ -68,6 +67,7 @@ struct dss_lcd_mgr_config;
 struct snd_aes_iec958;
 struct snd_cea_861_aud_if;
 struct hdmi_avi_infoframe;
+struct drm_connector;
 
 enum omap_display_type {
 	OMAP_DISPLAY_TYPE_NONE		= 0,
@@ -360,15 +360,15 @@ struct omap_dss_device_ops {
 	void (*disconnect)(struct omap_dss_device *dssdev,
 			struct omap_dss_device *dst);
 
-	int (*enable)(struct omap_dss_device *dssdev);
+	void (*pre_enable)(struct omap_dss_device *dssdev);
+	void (*enable)(struct omap_dss_device *dssdev);
 	void (*disable)(struct omap_dss_device *dssdev);
+	void (*post_disable)(struct omap_dss_device *dssdev);
 
 	int (*check_timings)(struct omap_dss_device *dssdev,
-			     struct videomode *vm);
-	void (*get_timings)(struct omap_dss_device *dssdev,
-			    struct videomode *vm);
+			     struct drm_display_mode *mode);
 	void (*set_timings)(struct omap_dss_device *dssdev,
-			    const struct videomode *vm);
+			    const struct drm_display_mode *mode);
 
 	bool (*detect)(struct omap_dss_device *dssdev);
 
@@ -380,6 +380,9 @@ struct omap_dss_device_ops {
 
 	int (*read_edid)(struct omap_dss_device *dssdev, u8 *buf, int len);
 
+	int (*get_modes)(struct omap_dss_device *dssdev,
+			 struct drm_connector *connector);
+
 	union {
 		const struct omapdss_hdmi_ops hdmi;
 		const struct omapdss_dsi_ops dsi;
@@ -390,42 +393,40 @@ struct omap_dss_device_ops {
  * enum omap_dss_device_ops_flag - Indicates which device ops are supported
  * @OMAP_DSS_DEVICE_OP_DETECT: The device supports output connection detection
  * @OMAP_DSS_DEVICE_OP_HPD: The device supports all hot-plug-related operations
- * @OMAP_DSS_DEVICE_OP_EDID: The device supports readind EDID
+ * @OMAP_DSS_DEVICE_OP_EDID: The device supports reading EDID
+ * @OMAP_DSS_DEVICE_OP_MODES: The device supports reading modes
  */
 enum omap_dss_device_ops_flag {
 	OMAP_DSS_DEVICE_OP_DETECT = BIT(0),
 	OMAP_DSS_DEVICE_OP_HPD = BIT(1),
 	OMAP_DSS_DEVICE_OP_EDID = BIT(2),
-};
-
-enum omap_dss_device_type {
-	OMAP_DSS_DEVICE_TYPE_OUTPUT = (1 << 0),
-	OMAP_DSS_DEVICE_TYPE_DISPLAY = (1 << 1),
+	OMAP_DSS_DEVICE_OP_MODES = BIT(3),
 };
 
 struct omap_dss_device {
-	struct kobject kobj;
 	struct device *dev;
 
 	struct module *owner;
 
 	struct dss_device *dss;
-	struct omap_dss_device *src;
-	struct omap_dss_device *dst;
 	struct omap_dss_device *next;
+	struct drm_bridge *bridge;
+	struct drm_panel *panel;
 
 	struct list_head list;
 
-	unsigned int alias_id;
-
+	/*
+	 * DSS type that this device generates (for DSS internal devices) or
+	 * requires (for external encoders, connectors and panels). Must be a
+	 * non-zero (different than OMAP_DISPLAY_TYPE_NONE) value.
+	 */
 	enum omap_display_type type;
+
 	/*
-	 * DSS output type that this device generates (for DSS internal devices)
-	 * or requires (for external encoders). Must be OMAP_DISPLAY_TYPE_NONE
-	 * for display devices (connectors and panels) and to non-zero value for
-	 * all other devices.
+	 * True if the device is a display (panel or connector) at the end of
+	 * the pipeline, false otherwise.
 	 */
-	enum omap_display_type output_type;
+	bool display;
 
 	const char *name;
 
@@ -434,9 +435,6 @@ struct omap_dss_device {
 	unsigned long ops_flags;
 	u32 bus_flags;
 
-	/* helper variable for driver suspend/resume */
-	bool activate_after_resume;
-
 	enum omap_display_caps caps;
 
 	enum omap_dss_display_state state;
@@ -445,7 +443,6 @@ struct omap_dss_device {
 
 	/* DISPC channel for this output */
 	enum omap_channel dispc_channel;
-	bool dispc_channel_connected;
 
 	/* output instance */
 	enum omap_dss_output_id id;
@@ -465,9 +462,6 @@ struct omap_dss_driver {
 	int (*memory_read)(struct omap_dss_device *dssdev,
 			void *buf, size_t size,
 			u16 x, u16 y, u16 w, u16 h);
-
-	void (*get_size)(struct omap_dss_device *dssdev,
-			 unsigned int *width, unsigned int *height);
 };
 
 struct dss_device *omapdss_get_dss(void);
@@ -477,32 +471,35 @@ static inline bool omapdss_is_initialized(void)
 	return !!omapdss_get_dss();
 }
 
-#define for_each_dss_display(d) \
-	while ((d = omapdss_device_get_next(d, OMAP_DSS_DEVICE_TYPE_DISPLAY)) != NULL)
 void omapdss_display_init(struct omap_dss_device *dssdev);
 struct omap_dss_device *omapdss_display_get(struct omap_dss_device *output);
+int omapdss_display_get_modes(struct drm_connector *connector,
+			      const struct videomode *vm);
 
 void omapdss_device_register(struct omap_dss_device *dssdev);
 void omapdss_device_unregister(struct omap_dss_device *dssdev);
 struct omap_dss_device *omapdss_device_get(struct omap_dss_device *dssdev);
 void omapdss_device_put(struct omap_dss_device *dssdev);
-struct omap_dss_device *omapdss_find_device_by_port(struct device_node *src,
-						    unsigned int port);
-struct omap_dss_device *omapdss_device_get_next(struct omap_dss_device *from,
-						enum omap_dss_device_type type);
+struct omap_dss_device *omapdss_find_device_by_node(struct device_node *node);
 int omapdss_device_connect(struct dss_device *dss,
 			   struct omap_dss_device *src,
 			   struct omap_dss_device *dst);
 void omapdss_device_disconnect(struct omap_dss_device *src,
 			       struct omap_dss_device *dst);
+void omapdss_device_pre_enable(struct omap_dss_device *dssdev);
+void omapdss_device_enable(struct omap_dss_device *dssdev);
+void omapdss_device_disable(struct omap_dss_device *dssdev);
+void omapdss_device_post_disable(struct omap_dss_device *dssdev);
 
 int omap_dss_get_num_overlay_managers(void);
 
 int omap_dss_get_num_overlays(void);
 
 #define for_each_dss_output(d) \
-	while ((d = omapdss_device_get_next(d, OMAP_DSS_DEVICE_TYPE_OUTPUT)) != NULL)
-int omapdss_output_validate(struct omap_dss_device *out);
+	while ((d = omapdss_device_next_output(d)) != NULL)
+struct omap_dss_device *omapdss_device_next_output(struct omap_dss_device *from);
+int omapdss_device_init_output(struct omap_dss_device *out);
+void omapdss_device_cleanup_output(struct omap_dss_device *out);
 
 typedef void (*omap_dispc_isr_t) (void *arg, u32 mask);
 int omap_dispc_register_isr(omap_dispc_isr_t isr, void *arg, u32 mask);
@@ -511,11 +508,6 @@ int omap_dispc_unregister_isr(omap_dispc_isr_t isr, void *arg, u32 mask);
 int omapdss_compat_init(void);
 void omapdss_compat_uninit(void);
 
-static inline bool omapdss_device_is_connected(struct omap_dss_device *dssdev)
-{
-	return dssdev->src;
-}
-
 static inline bool omapdss_device_is_enabled(struct omap_dss_device *dssdev)
 {
 	return dssdev->state == OMAP_DSS_DISPLAY_ACTIVE;
diff --git a/drivers/gpu/drm/omapdrm/dss/output.c b/drivers/gpu/drm/omapdrm/dss/output.c
index 18505bc70f7e..10a9ee5cdc61 100644
--- a/drivers/gpu/drm/omapdrm/dss/output.c
+++ b/drivers/gpu/drm/omapdrm/dss/output.c
@@ -20,20 +20,48 @@
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 #include <linux/of.h>
+#include <linux/of_graph.h>
+
+#include <drm/drm_panel.h>
 
 #include "dss.h"
 #include "omapdss.h"
 
-int omapdss_output_validate(struct omap_dss_device *out)
+int omapdss_device_init_output(struct omap_dss_device *out)
 {
-	if (out->next && out->output_type != out->next->type) {
+	struct device_node *remote_node;
+
+	remote_node = of_graph_get_remote_node(out->dev->of_node, 0, 0);
+	if (!remote_node) {
+		dev_dbg(out->dev, "failed to find video sink\n");
+		return 0;
+	}
+
+	out->next = omapdss_find_device_by_node(remote_node);
+	out->bridge = of_drm_find_bridge(remote_node);
+	out->panel = of_drm_find_panel(remote_node);
+	if (IS_ERR(out->panel))
+		out->panel = NULL;
+
+	of_node_put(remote_node);
+
+	if (out->next && out->type != out->next->type) {
 		dev_err(out->dev, "output type and display type don't match\n");
+		omapdss_device_put(out->next);
+		out->next = NULL;
 		return -EINVAL;
 	}
 
-	return 0;
+	return out->next || out->bridge || out->panel ? 0 : -EPROBE_DEFER;
+}
+EXPORT_SYMBOL(omapdss_device_init_output);
+
+void omapdss_device_cleanup_output(struct omap_dss_device *out)
+{
+	if (out->next)
+		omapdss_device_put(out->next);
 }
-EXPORT_SYMBOL(omapdss_output_validate);
+EXPORT_SYMBOL(omapdss_device_cleanup_output);
 
 int dss_install_mgr_ops(struct dss_device *dss,
 			const struct dss_mgr_ops *mgr_ops,
diff --git a/drivers/gpu/drm/omapdrm/dss/sdi.c b/drivers/gpu/drm/omapdrm/dss/sdi.c
index b2fe2387037a..7aae52984fed 100644
--- a/drivers/gpu/drm/omapdrm/dss/sdi.c
+++ b/drivers/gpu/drm/omapdrm/dss/sdi.c
@@ -37,7 +37,7 @@ struct sdi_device {
 	struct regulator *vdds_sdi_reg;
 
 	struct dss_lcd_mgr_config mgr_config;
-	struct videomode vm;
+	unsigned long pixelclock;
 	int datapairs;
 
 	struct omap_dss_device output;
@@ -129,27 +129,22 @@ static void sdi_config_lcd_manager(struct sdi_device *sdi)
 	dss_mgr_set_lcd_config(&sdi->output, &sdi->mgr_config);
 }
 
-static int sdi_display_enable(struct omap_dss_device *dssdev)
+static void sdi_display_enable(struct omap_dss_device *dssdev)
 {
 	struct sdi_device *sdi = dssdev_to_sdi(dssdev);
 	struct dispc_clock_info dispc_cinfo;
 	unsigned long fck;
 	int r;
 
-	if (!sdi->output.dispc_channel_connected) {
-		DSSERR("failed to enable display: no output/manager\n");
-		return -ENODEV;
-	}
-
 	r = regulator_enable(sdi->vdds_sdi_reg);
 	if (r)
-		goto err_reg_enable;
+		return;
 
 	r = dispc_runtime_get(sdi->dss->dispc);
 	if (r)
 		goto err_get_dispc;
 
-	r = sdi_calc_clock_div(sdi, sdi->vm.pixelclock, &fck, &dispc_cinfo);
+	r = sdi_calc_clock_div(sdi, sdi->pixelclock, &fck, &dispc_cinfo);
 	if (r)
 		goto err_calc_clock_div;
 
@@ -185,7 +180,7 @@ static int sdi_display_enable(struct omap_dss_device *dssdev)
 	if (r)
 		goto err_mgr_enable;
 
-	return 0;
+	return;
 
 err_mgr_enable:
 	dss_sdi_disable(sdi->dss);
@@ -195,8 +190,6 @@ err_calc_clock_div:
 	dispc_runtime_put(sdi->dss->dispc);
 err_get_dispc:
 	regulator_disable(sdi->vdds_sdi_reg);
-err_reg_enable:
-	return r;
 }
 
 static void sdi_display_disable(struct omap_dss_device *dssdev)
@@ -213,36 +206,37 @@ static void sdi_display_disable(struct omap_dss_device *dssdev)
 }
 
 static void sdi_set_timings(struct omap_dss_device *dssdev,
-			    const struct videomode *vm)
+			    const struct drm_display_mode *mode)
 {
 	struct sdi_device *sdi = dssdev_to_sdi(dssdev);
 
-	sdi->vm = *vm;
+	sdi->pixelclock = mode->clock * 1000;
 }
 
 static int sdi_check_timings(struct omap_dss_device *dssdev,
-			     struct videomode *vm)
+			     struct drm_display_mode *mode)
 {
 	struct sdi_device *sdi = dssdev_to_sdi(dssdev);
 	struct dispc_clock_info dispc_cinfo;
+	unsigned long pixelclock = mode->clock * 1000;
 	unsigned long fck;
 	unsigned long pck;
 	int r;
 
-	if (vm->pixelclock == 0)
+	if (pixelclock == 0)
 		return -EINVAL;
 
-	r = sdi_calc_clock_div(sdi, vm->pixelclock, &fck, &dispc_cinfo);
+	r = sdi_calc_clock_div(sdi, pixelclock, &fck, &dispc_cinfo);
 	if (r)
 		return r;
 
 	pck = fck / dispc_cinfo.lck_div / dispc_cinfo.pck_div;
 
-	if (pck != vm->pixelclock) {
+	if (pck != pixelclock) {
 		DSSWARN("Pixel clock adjusted from %lu Hz to %lu Hz\n",
-			vm->pixelclock, pck);
+			pixelclock, pck);
 
-		vm->pixelclock = pck;
+		mode->clock = pck / 1000;
 	}
 
 	return 0;
@@ -251,21 +245,12 @@ static int sdi_check_timings(struct omap_dss_device *dssdev,
 static int sdi_connect(struct omap_dss_device *src,
 		       struct omap_dss_device *dst)
 {
-	int r;
-
-	r = omapdss_device_connect(dst->dss, dst, dst->next);
-	if (r)
-		return r;
-
-	dst->dispc_channel_connected = true;
-	return 0;
+	return omapdss_device_connect(dst->dss, dst, dst->next);
 }
 
 static void sdi_disconnect(struct omap_dss_device *src,
 			   struct omap_dss_device *dst)
 {
-	dst->dispc_channel_connected = false;
-
 	omapdss_device_disconnect(dst, dst->next);
 }
 
@@ -287,29 +272,19 @@ static int sdi_init_output(struct sdi_device *sdi)
 
 	out->dev = &sdi->pdev->dev;
 	out->id = OMAP_DSS_OUTPUT_SDI;
-	out->output_type = OMAP_DISPLAY_TYPE_SDI;
+	out->type = OMAP_DISPLAY_TYPE_SDI;
 	out->name = "sdi.0";
 	out->dispc_channel = OMAP_DSS_CHANNEL_LCD;
 	/* We have SDI only on OMAP3, where it's on port 1 */
 	out->of_ports = BIT(1);
 	out->ops = &sdi_ops;
 	out->owner = THIS_MODULE;
-	out->bus_flags = DRM_BUS_FLAG_PIXDATA_POSEDGE	/* 15.5.9.1.2 */
-		       | DRM_BUS_FLAG_SYNC_POSEDGE;
-
-	out->next = omapdss_of_find_connected_device(out->dev->of_node, 1);
-	if (IS_ERR(out->next)) {
-		if (PTR_ERR(out->next) != -EPROBE_DEFER)
-			dev_err(out->dev, "failed to find video sink\n");
-		return PTR_ERR(out->next);
-	}
+	out->bus_flags = DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE	/* 15.5.9.1.2 */
+		       | DRM_BUS_FLAG_SYNC_DRIVE_POSEDGE;
 
-	r = omapdss_output_validate(out);
-	if (r) {
-		omapdss_device_put(out->next);
-		out->next = NULL;
+	r = omapdss_device_init_output(out);
+	if (r < 0)
 		return r;
-	}
 
 	omapdss_device_register(out);
 
@@ -318,9 +293,8 @@ static int sdi_init_output(struct sdi_device *sdi)
 
 static void sdi_uninit_output(struct sdi_device *sdi)
 {
-	if (sdi->output.next)
-		omapdss_device_put(sdi->output.next);
 	omapdss_device_unregister(&sdi->output);
+	omapdss_device_cleanup_output(&sdi->output);
 }
 
 int sdi_init_port(struct dss_device *dss, struct platform_device *pdev,
diff --git a/drivers/gpu/drm/omapdrm/dss/venc.c b/drivers/gpu/drm/omapdrm/dss/venc.c
index b5f52727f8b1..da43b865d973 100644
--- a/drivers/gpu/drm/omapdrm/dss/venc.c
+++ b/drivers/gpu/drm/omapdrm/dss/venc.c
@@ -267,63 +267,40 @@ enum venc_videomode {
 	VENC_MODE_NTSC,
 };
 
-static const struct videomode omap_dss_pal_vm = {
-	.hactive	= 720,
-	.vactive	= 574,
-	.pixelclock	= 13500000,
-	.hsync_len	= 64,
-	.hfront_porch	= 12,
-	.hback_porch	= 68,
-	.vsync_len	= 5,
-	.vfront_porch	= 5,
-	.vback_porch	= 41,
-
-	.flags		= DISPLAY_FLAGS_INTERLACED | DISPLAY_FLAGS_HSYNC_LOW |
-			  DISPLAY_FLAGS_VSYNC_LOW | DISPLAY_FLAGS_DE_HIGH |
-			  DISPLAY_FLAGS_PIXDATA_POSEDGE |
-			  DISPLAY_FLAGS_SYNC_NEGEDGE,
+static const struct drm_display_mode omap_dss_pal_mode = {
+	.hdisplay	= 720,
+	.hsync_start	= 732,
+	.hsync_end	= 796,
+	.htotal		= 864,
+	.vdisplay	= 574,
+	.vsync_start	= 579,
+	.vsync_end	= 584,
+	.vtotal		= 625,
+	.clock		= 13500,
+
+	.flags		= DRM_MODE_FLAG_INTERLACE | DRM_MODE_FLAG_NHSYNC |
+			  DRM_MODE_FLAG_NVSYNC,
 };
 
-static const struct videomode omap_dss_ntsc_vm = {
-	.hactive	= 720,
-	.vactive	= 482,
-	.pixelclock	= 13500000,
-	.hsync_len	= 64,
-	.hfront_porch	= 16,
-	.hback_porch	= 58,
-	.vsync_len	= 6,
-	.vfront_porch	= 6,
-	.vback_porch	= 31,
-
-	.flags		= DISPLAY_FLAGS_INTERLACED | DISPLAY_FLAGS_HSYNC_LOW |
-			  DISPLAY_FLAGS_VSYNC_LOW | DISPLAY_FLAGS_DE_HIGH |
-			  DISPLAY_FLAGS_PIXDATA_POSEDGE |
-			  DISPLAY_FLAGS_SYNC_NEGEDGE,
+static const struct drm_display_mode omap_dss_ntsc_mode = {
+	.hdisplay	= 720,
+	.hsync_start	= 736,
+	.hsync_end	= 800,
+	.htotal		= 858,
+	.vdisplay	= 482,
+	.vsync_start	= 488,
+	.vsync_end	= 494,
+	.vtotal		= 525,
+	.clock		= 13500,
+
+	.flags		= DRM_MODE_FLAG_INTERLACE | DRM_MODE_FLAG_NHSYNC |
+			  DRM_MODE_FLAG_NVSYNC,
 };
 
-static enum venc_videomode venc_get_videomode(const struct videomode *vm)
-{
-	if (!(vm->flags & DISPLAY_FLAGS_INTERLACED))
-		return VENC_MODE_UNKNOWN;
-
-	if (vm->pixelclock == omap_dss_pal_vm.pixelclock &&
-	    vm->hactive == omap_dss_pal_vm.hactive &&
-	    vm->vactive == omap_dss_pal_vm.vactive)
-		return VENC_MODE_PAL;
-
-	if (vm->pixelclock == omap_dss_ntsc_vm.pixelclock &&
-	    vm->hactive == omap_dss_ntsc_vm.hactive &&
-	    vm->vactive == omap_dss_ntsc_vm.vactive)
-		return VENC_MODE_NTSC;
-
-	return VENC_MODE_UNKNOWN;
-}
-
 struct venc_device {
 	struct platform_device *pdev;
 	void __iomem *base;
 	struct mutex venc_lock;
-	u32 wss_data;
 	struct regulator *vdda_dac_reg;
 	struct dss_device *dss;
 
@@ -331,7 +308,7 @@ struct venc_device {
 
 	struct clk	*tv_dac_clk;
 
-	struct videomode vm;
+	const struct venc_config *config;
 	enum omap_dss_venc_type type;
 	bool invert_polarity;
 	bool requires_tv_dac_clk;
@@ -367,8 +344,7 @@ static void venc_write_config(struct venc_device *venc,
 	venc_write_reg(venc, VENC_BLACK_LEVEL, config->black_level);
 	venc_write_reg(venc, VENC_BLANK_LEVEL, config->blank_level);
 	venc_write_reg(venc, VENC_M_CONTROL, config->m_control);
-	venc_write_reg(venc, VENC_BSTAMP_WSS_DATA, config->bstamp_wss_data |
-		       venc->wss_data);
+	venc_write_reg(venc, VENC_BSTAMP_WSS_DATA, config->bstamp_wss_data);
 	venc_write_reg(venc, VENC_S_CARR, config->s_carr);
 	venc_write_reg(venc, VENC_L21__WC_CTL, config->l21__wc_ctl);
 	venc_write_reg(venc, VENC_SAVID__EAVID, config->savid__eavid);
@@ -452,18 +428,6 @@ static void venc_runtime_put(struct venc_device *venc)
 	WARN_ON(r < 0 && r != -ENOSYS);
 }
 
-static const struct venc_config *venc_timings_to_config(const struct videomode *vm)
-{
-	switch (venc_get_videomode(vm)) {
-	default:
-		WARN_ON_ONCE(1);
-	case VENC_MODE_PAL:
-		return &venc_config_pal_trm;
-	case VENC_MODE_NTSC:
-		return &venc_config_ntsc_trm;
-	}
-}
-
 static int venc_power_on(struct venc_device *venc)
 {
 	u32 l;
@@ -474,7 +438,7 @@ static int venc_power_on(struct venc_device *venc)
 		goto err0;
 
 	venc_reset(venc);
-	venc_write_config(venc, venc_timings_to_config(&venc->vm));
+	venc_write_config(venc, venc->config);
 
 	dss_set_venc_output(venc->dss, venc->type);
 	dss_set_dac_pwrdn_bgz(venc->dss, 1);
@@ -524,33 +488,17 @@ static void venc_power_off(struct venc_device *venc)
 	venc_runtime_put(venc);
 }
 
-static int venc_display_enable(struct omap_dss_device *dssdev)
+static void venc_display_enable(struct omap_dss_device *dssdev)
 {
 	struct venc_device *venc = dssdev_to_venc(dssdev);
-	int r;
 
 	DSSDBG("venc_display_enable\n");
 
 	mutex_lock(&venc->venc_lock);
 
-	if (!dssdev->dispc_channel_connected) {
-		DSSERR("Failed to enable display: no output/manager\n");
-		r = -ENODEV;
-		goto err0;
-	}
-
-	r = venc_power_on(venc);
-	if (r)
-		goto err0;
-
-	venc->wss_data = 0;
+	venc_power_on(venc);
 
 	mutex_unlock(&venc->venc_lock);
-
-	return 0;
-err0:
-	mutex_unlock(&venc->venc_lock);
-	return r;
 }
 
 static void venc_display_disable(struct omap_dss_device *dssdev)
@@ -566,30 +514,70 @@ static void venc_display_disable(struct omap_dss_device *dssdev)
 	mutex_unlock(&venc->venc_lock);
 }
 
-static void venc_get_timings(struct omap_dss_device *dssdev,
-			     struct videomode *vm)
+static int venc_get_modes(struct omap_dss_device *dssdev,
+			  struct drm_connector *connector)
 {
-	struct venc_device *venc = dssdev_to_venc(dssdev);
+	static const struct drm_display_mode *modes[] = {
+		&omap_dss_pal_mode,
+		&omap_dss_ntsc_mode,
+	};
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(modes); ++i) {
+		struct drm_display_mode *mode;
+
+		mode = drm_mode_duplicate(connector->dev, modes[i]);
+		if (!mode)
+			return i;
+
+		mode->type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED;
+		drm_mode_set_name(mode);
+		drm_mode_probed_add(connector, mode);
+	}
 
-	mutex_lock(&venc->venc_lock);
-	*vm = venc->vm;
-	mutex_unlock(&venc->venc_lock);
+	return ARRAY_SIZE(modes);
+}
+
+static enum venc_videomode venc_get_videomode(const struct drm_display_mode *mode)
+{
+	if (!(mode->flags & DRM_MODE_FLAG_INTERLACE))
+		return VENC_MODE_UNKNOWN;
+
+	if (mode->clock == omap_dss_pal_mode.clock &&
+	    mode->hdisplay == omap_dss_pal_mode.hdisplay &&
+	    mode->vdisplay == omap_dss_pal_mode.vdisplay)
+		return VENC_MODE_PAL;
+
+	if (mode->clock == omap_dss_ntsc_mode.clock &&
+	    mode->hdisplay == omap_dss_ntsc_mode.hdisplay &&
+	    mode->vdisplay == omap_dss_ntsc_mode.vdisplay)
+		return VENC_MODE_NTSC;
+
+	return VENC_MODE_UNKNOWN;
 }
 
 static void venc_set_timings(struct omap_dss_device *dssdev,
-			     const struct videomode *vm)
+			     const struct drm_display_mode *mode)
 {
 	struct venc_device *venc = dssdev_to_venc(dssdev);
+	enum venc_videomode venc_mode = venc_get_videomode(mode);
 
 	DSSDBG("venc_set_timings\n");
 
 	mutex_lock(&venc->venc_lock);
 
-	/* Reset WSS data when the TV standard changes. */
-	if (memcmp(&venc->vm, vm, sizeof(*vm)))
-		venc->wss_data = 0;
+	switch (venc_mode) {
+	default:
+		WARN_ON_ONCE(1);
+		/* Fall-through */
+	case VENC_MODE_PAL:
+		venc->config = &venc_config_pal_trm;
+		break;
 
-	venc->vm = *vm;
+	case VENC_MODE_NTSC:
+		venc->config = &venc_config_ntsc_trm;
+		break;
+	}
 
 	dispc_set_tv_pclk(venc->dss->dispc, 13500000);
 
@@ -597,22 +585,26 @@ static void venc_set_timings(struct omap_dss_device *dssdev,
 }
 
 static int venc_check_timings(struct omap_dss_device *dssdev,
-			      struct videomode *vm)
+			      struct drm_display_mode *mode)
 {
 	DSSDBG("venc_check_timings\n");
 
-	switch (venc_get_videomode(vm)) {
+	switch (venc_get_videomode(mode)) {
 	case VENC_MODE_PAL:
-		*vm = omap_dss_pal_vm;
-		return 0;
+		drm_mode_copy(mode, &omap_dss_pal_mode);
+		break;
 
 	case VENC_MODE_NTSC:
-		*vm = omap_dss_ntsc_vm;
-		return 0;
+		drm_mode_copy(mode, &omap_dss_ntsc_mode);
+		break;
 
 	default:
 		return -EINVAL;
 	}
+
+	drm_mode_set_crtcinfo(mode, CRTC_INTERLACE_HALVE_V);
+	drm_mode_set_name(mode);
+	return 0;
 }
 
 static int venc_dump_regs(struct seq_file *s, void *p)
@@ -695,21 +687,12 @@ static int venc_get_clocks(struct venc_device *venc)
 static int venc_connect(struct omap_dss_device *src,
 			struct omap_dss_device *dst)
 {
-	int r;
-
-	r = omapdss_device_connect(dst->dss, dst, dst->next);
-	if (r)
-		return r;
-
-	dst->dispc_channel_connected = true;
-	return 0;
+	return omapdss_device_connect(dst->dss, dst, dst->next);
 }
 
 static void venc_disconnect(struct omap_dss_device *src,
 			    struct omap_dss_device *dst)
 {
-	dst->dispc_channel_connected = false;
-
 	omapdss_device_disconnect(dst, dst->next);
 }
 
@@ -721,8 +704,9 @@ static const struct omap_dss_device_ops venc_ops = {
 	.disable = venc_display_disable,
 
 	.check_timings = venc_check_timings,
-	.get_timings = venc_get_timings,
 	.set_timings = venc_set_timings,
+
+	.get_modes = venc_get_modes,
 };
 
 /* -----------------------------------------------------------------------------
@@ -776,26 +760,17 @@ static int venc_init_output(struct venc_device *venc)
 
 	out->dev = &venc->pdev->dev;
 	out->id = OMAP_DSS_OUTPUT_VENC;
-	out->output_type = OMAP_DISPLAY_TYPE_VENC;
+	out->type = OMAP_DISPLAY_TYPE_VENC;
 	out->name = "venc.0";
 	out->dispc_channel = OMAP_DSS_CHANNEL_DIGIT;
 	out->ops = &venc_ops;
 	out->owner = THIS_MODULE;
 	out->of_ports = BIT(0);
+	out->ops_flags = OMAP_DSS_DEVICE_OP_MODES;
 
-	out->next = omapdss_of_find_connected_device(out->dev->of_node, 0);
-	if (IS_ERR(out->next)) {
-		if (PTR_ERR(out->next) != -EPROBE_DEFER)
-			dev_err(out->dev, "failed to find video sink\n");
-		return PTR_ERR(out->next);
-	}
-
-	r = omapdss_output_validate(out);
-	if (r) {
-		omapdss_device_put(out->next);
-		out->next = NULL;
+	r = omapdss_device_init_output(out);
+	if (r < 0)
 		return r;
-	}
 
 	omapdss_device_register(out);
 
@@ -804,9 +779,8 @@ static int venc_init_output(struct venc_device *venc)
 
 static void venc_uninit_output(struct venc_device *venc)
 {
-	if (venc->output.next)
-		omapdss_device_put(venc->output.next);
 	omapdss_device_unregister(&venc->output);
+	omapdss_device_cleanup_output(&venc->output);
 }
 
 static int venc_probe_of(struct venc_device *venc)
@@ -878,8 +852,7 @@ static int venc_probe(struct platform_device *pdev)
 
 	mutex_init(&venc->venc_lock);
 
-	venc->wss_data = 0;
-	venc->vm = omap_dss_pal_vm;
+	venc->config = &venc_config_pal_trm;
 
 	venc_mem = platform_get_resource(venc->pdev, IORESOURCE_MEM, 0);
 	venc->base = devm_ioremap_resource(&pdev->dev, venc_mem);
diff --git a/drivers/gpu/drm/omapdrm/omap_connector.c b/drivers/gpu/drm/omapdrm/omap_connector.c
index 9da94d10782a..5967283934e1 100644
--- a/drivers/gpu/drm/omapdrm/omap_connector.c
+++ b/drivers/gpu/drm/omapdrm/omap_connector.c
@@ -17,6 +17,7 @@
 
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc.h>
+#include <drm/drm_panel.h>
 #include <drm/drm_probe_helper.h>
 
 #include "omap_drv.h"
@@ -30,24 +31,27 @@
 struct omap_connector {
 	struct drm_connector base;
 	struct omap_dss_device *output;
-	struct omap_dss_device *display;
 	struct omap_dss_device *hpd;
 	bool hdmi_mode;
 };
 
 static void omap_connector_hpd_notify(struct drm_connector *connector,
-				      struct omap_dss_device *src,
 				      enum drm_connector_status status)
 {
-	if (status == connector_status_disconnected) {
-		/*
-		 * If the source is an HDMI encoder, notify it of disconnection.
-		 * This is required to let the HDMI encoder reset any internal
-		 * state related to connection status, such as the CEC address.
-		 */
-		if (src && src->type == OMAP_DISPLAY_TYPE_HDMI &&
-		    src->ops->hdmi.lost_hotplug)
-			src->ops->hdmi.lost_hotplug(src);
+	struct omap_connector *omap_connector = to_omap_connector(connector);
+	struct omap_dss_device *dssdev;
+
+	if (status != connector_status_disconnected)
+		return;
+
+	/*
+	 * Notify all devics in the pipeline of disconnection. This is required
+	 * to let the HDMI encoders reset their internal state related to
+	 * connection status, such as the CEC address.
+	 */
+	for (dssdev = omap_connector->output; dssdev; dssdev = dssdev->next) {
+		if (dssdev->ops && dssdev->ops->hdmi.lost_hotplug)
+			dssdev->ops->hdmi.lost_hotplug(dssdev);
 	}
 }
 
@@ -67,7 +71,7 @@ static void omap_connector_hpd_cb(void *cb_data,
 	if (old_status == status)
 		return;
 
-	omap_connector_hpd_notify(connector, omap_connector->hpd, status);
+	omap_connector_hpd_notify(connector, status);
 
 	drm_kms_helper_hotplug_event(dev);
 }
@@ -103,20 +107,20 @@ omap_connector_find_device(struct drm_connector *connector,
 			   enum omap_dss_device_ops_flag op)
 {
 	struct omap_connector *omap_connector = to_omap_connector(connector);
-	struct omap_dss_device *dssdev;
+	struct omap_dss_device *dssdev = NULL;
+	struct omap_dss_device *d;
 
-	for (dssdev = omap_connector->display; dssdev; dssdev = dssdev->src) {
-		if (dssdev->ops_flags & op)
-			return dssdev;
+	for (d = omap_connector->output; d; d = d->next) {
+		if (d->ops_flags & op)
+			dssdev = d;
 	}
 
-	return NULL;
+	return dssdev;
 }
 
 static enum drm_connector_status omap_connector_detect(
 		struct drm_connector *connector, bool force)
 {
-	struct omap_connector *omap_connector = to_omap_connector(connector);
 	struct omap_dss_device *dssdev;
 	enum drm_connector_status status;
 
@@ -128,13 +132,12 @@ static enum drm_connector_status omap_connector_detect(
 		       ? connector_status_connected
 		       : connector_status_disconnected;
 
-		omap_connector_hpd_notify(connector, dssdev->src, status);
+		omap_connector_hpd_notify(connector, status);
 	} else {
-		switch (omap_connector->display->type) {
-		case OMAP_DISPLAY_TYPE_DPI:
-		case OMAP_DISPLAY_TYPE_DBI:
-		case OMAP_DISPLAY_TYPE_SDI:
-		case OMAP_DISPLAY_TYPE_DSI:
+		switch (connector->connector_type) {
+		case DRM_MODE_CONNECTOR_DPI:
+		case DRM_MODE_CONNECTOR_LVDS:
+		case DRM_MODE_CONNECTOR_DSI:
 			status = connector_status_connected;
 			break;
 		default:
@@ -143,7 +146,7 @@ static enum drm_connector_status omap_connector_detect(
 		}
 	}
 
-	VERB("%s: %d (force=%d)", omap_connector->display->name, status, force);
+	VERB("%s: %d (force=%d)", connector->name, status, force);
 
 	return status;
 }
@@ -152,7 +155,7 @@ static void omap_connector_destroy(struct drm_connector *connector)
 {
 	struct omap_connector *omap_connector = to_omap_connector(connector);
 
-	DBG("%s", omap_connector->display->name);
+	DBG("%s", connector->name);
 
 	if (omap_connector->hpd) {
 		struct omap_dss_device *hpd = omap_connector->hpd;
@@ -166,7 +169,6 @@ static void omap_connector_destroy(struct drm_connector *connector)
 	drm_connector_cleanup(connector);
 
 	omapdss_device_put(omap_connector->output);
-	omapdss_device_put(omap_connector->display);
 
 	kfree(omap_connector);
 }
@@ -212,10 +214,8 @@ static int omap_connector_get_modes(struct drm_connector *connector)
 {
 	struct omap_connector *omap_connector = to_omap_connector(connector);
 	struct omap_dss_device *dssdev;
-	struct drm_display_mode *mode;
-	struct videomode vm = {0};
 
-	DBG("%s", omap_connector->display->name);
+	DBG("%s", connector->name);
 
 	/*
 	 * If display exposes EDID, then we parse that in the normal way to
@@ -227,89 +227,71 @@ static int omap_connector_get_modes(struct drm_connector *connector)
 		return omap_connector_get_modes_edid(connector, dssdev);
 
 	/*
-	 * Otherwise we have either a fixed resolution panel or an output that
-	 * doesn't support modes discovery (e.g. DVI or VGA with the DDC bus
-	 * unconnected, or analog TV). Start by querying the size.
+	 * Otherwise if the display pipeline reports modes (e.g. with a fixed
+	 * resolution panel or an analog TV output), query it.
 	 */
-	dssdev = omap_connector->display;
-	if (dssdev->driver && dssdev->driver->get_size)
-		dssdev->driver->get_size(dssdev,
-					 &connector->display_info.width_mm,
-					 &connector->display_info.height_mm);
+	dssdev = omap_connector_find_device(connector,
+					    OMAP_DSS_DEVICE_OP_MODES);
+	if (dssdev)
+		return dssdev->ops->get_modes(dssdev, connector);
 
 	/*
-	 * Iterate over the pipeline to find the first device that can provide
-	 * timing information. If we can't find any, we just let the KMS core
-	 * add the default modes.
+	 * Otherwise if the display pipeline uses a drm_panel, we delegate the
+	 * operation to the panel API.
 	 */
-	for (dssdev = omap_connector->display; dssdev; dssdev = dssdev->src) {
-		if (dssdev->ops->get_timings)
-			break;
-	}
-	if (!dssdev)
-		return 0;
+	if (omap_connector->output->panel)
+		return drm_panel_get_modes(omap_connector->output->panel);
 
-	/* Add a single mode corresponding to the fixed panel timings. */
-	mode = drm_mode_create(connector->dev);
-	if (!mode)
-		return 0;
+	/*
+	 * We can't retrieve modes, which can happen for instance for a DVI or
+	 * VGA output with the DDC bus unconnected. The KMS core will add the
+	 * default modes.
+	 */
+	return 0;
+}
 
-	dssdev->ops->get_timings(dssdev, &vm);
+enum drm_mode_status omap_connector_mode_fixup(struct omap_dss_device *dssdev,
+					const struct drm_display_mode *mode,
+					struct drm_display_mode *adjusted_mode)
+{
+	int ret;
 
-	drm_display_mode_from_videomode(&vm, mode);
+	drm_mode_copy(adjusted_mode, mode);
 
-	mode->type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED;
-	drm_mode_set_name(mode);
-	drm_mode_probed_add(connector, mode);
+	for (; dssdev; dssdev = dssdev->next) {
+		if (!dssdev->ops->check_timings)
+			continue;
+
+		ret = dssdev->ops->check_timings(dssdev, adjusted_mode);
+		if (ret)
+			return MODE_BAD;
+	}
 
-	return 1;
+	return MODE_OK;
 }
 
-static int omap_connector_mode_valid(struct drm_connector *connector,
+static enum drm_mode_status omap_connector_mode_valid(struct drm_connector *connector,
 				 struct drm_display_mode *mode)
 {
 	struct omap_connector *omap_connector = to_omap_connector(connector);
-	enum omap_channel channel = omap_connector->output->dispc_channel;
-	struct omap_drm_private *priv = connector->dev->dev_private;
-	struct omap_dss_device *dssdev;
-	struct videomode vm = {0};
-	struct drm_device *dev = connector->dev;
-	struct drm_display_mode *new_mode;
-	int r, ret = MODE_BAD;
-
-	drm_display_mode_to_videomode(mode, &vm);
-	mode->vrefresh = drm_mode_vrefresh(mode);
+	struct drm_display_mode new_mode = { { 0 } };
+	enum drm_mode_status status;
 
-	r = priv->dispc_ops->mgr_check_timings(priv->dispc, channel, &vm);
-	if (r)
+	status = omap_connector_mode_fixup(omap_connector->output, mode,
+					   &new_mode);
+	if (status != MODE_OK)
 		goto done;
 
-	for (dssdev = omap_connector->output; dssdev; dssdev = dssdev->next) {
-		if (!dssdev->ops->check_timings)
-			continue;
-
-		r = dssdev->ops->check_timings(dssdev, &vm);
-		if (r)
-			goto done;
-	}
-
-	/* check if vrefresh is still valid */
-	new_mode = drm_mode_duplicate(dev, mode);
-	if (!new_mode)
-		return MODE_BAD;
-
-	new_mode->clock = vm.pixelclock / 1000;
-	new_mode->vrefresh = 0;
-	if (mode->vrefresh == drm_mode_vrefresh(new_mode))
-		ret = MODE_OK;
-	drm_mode_destroy(dev, new_mode);
+	/* Check if vrefresh is still valid. */
+	if (drm_mode_vrefresh(mode) != drm_mode_vrefresh(&new_mode))
+		status = MODE_NOCLOCK;
 
 done:
 	DBG("connector: mode %s: " DRM_MODE_FMT,
-			(ret == MODE_OK) ? "valid" : "invalid",
+			(status == MODE_OK) ? "valid" : "invalid",
 			DRM_MODE_ARG(mode));
 
-	return ret;
+	return status;
 }
 
 static const struct drm_connector_funcs omap_connector_funcs = {
@@ -326,9 +308,16 @@ static const struct drm_connector_helper_funcs omap_connector_helper_funcs = {
 	.mode_valid = omap_connector_mode_valid,
 };
 
-static int omap_connector_get_type(struct omap_dss_device *display)
+static int omap_connector_get_type(struct omap_dss_device *output)
 {
-	switch (display->type) {
+	struct omap_dss_device *display;
+	enum omap_display_type type;
+
+	display = omapdss_display_get(output);
+	type = display->type;
+	omapdss_device_put(display);
+
+	switch (type) {
 	case OMAP_DISPLAY_TYPE_HDMI:
 		return DRM_MODE_CONNECTOR_HDMIA;
 	case OMAP_DISPLAY_TYPE_DVI:
@@ -351,28 +340,26 @@ static int omap_connector_get_type(struct omap_dss_device *display)
 /* initialize connector */
 struct drm_connector *omap_connector_init(struct drm_device *dev,
 					  struct omap_dss_device *output,
-					  struct omap_dss_device *display,
 					  struct drm_encoder *encoder)
 {
 	struct drm_connector *connector = NULL;
 	struct omap_connector *omap_connector;
 	struct omap_dss_device *dssdev;
 
-	DBG("%s", display->name);
+	DBG("%s", output->name);
 
 	omap_connector = kzalloc(sizeof(*omap_connector), GFP_KERNEL);
 	if (!omap_connector)
 		goto fail;
 
 	omap_connector->output = omapdss_device_get(output);
-	omap_connector->display = omapdss_device_get(display);
 
 	connector = &omap_connector->base;
 	connector->interlace_allowed = 1;
 	connector->doublescan_allowed = 0;
 
 	drm_connector_init(dev, connector, &omap_connector_funcs,
-			   omap_connector_get_type(display));
+			   omap_connector_get_type(output));
 	drm_connector_helper_add(connector, &omap_connector_helper_funcs);
 
 	/*
diff --git a/drivers/gpu/drm/omapdrm/omap_connector.h b/drivers/gpu/drm/omapdrm/omap_connector.h
index 854099801649..608085219336 100644
--- a/drivers/gpu/drm/omapdrm/omap_connector.h
+++ b/drivers/gpu/drm/omapdrm/omap_connector.h
@@ -22,6 +22,8 @@
 
 #include <linux/types.h>
 
+enum drm_mode_status;
+
 struct drm_connector;
 struct drm_device;
 struct drm_encoder;
@@ -29,12 +31,12 @@ struct omap_dss_device;
 
 struct drm_connector *omap_connector_init(struct drm_device *dev,
 					  struct omap_dss_device *output,
-					  struct omap_dss_device *display,
 					  struct drm_encoder *encoder);
-struct drm_encoder *omap_connector_attached_encoder(
-		struct drm_connector *connector);
 bool omap_connector_get_hdmi_mode(struct drm_connector *connector);
 void omap_connector_enable_hpd(struct drm_connector *connector);
 void omap_connector_disable_hpd(struct drm_connector *connector);
+enum drm_mode_status omap_connector_mode_fixup(struct omap_dss_device *dssdev,
+					const struct drm_display_mode *mode,
+					struct drm_display_mode *adjusted_mode);
 
 #endif /* __OMAPDRM_CONNECTOR_H__ */
diff --git a/drivers/gpu/drm/omapdrm/omap_crtc.c b/drivers/gpu/drm/omapdrm/omap_crtc.c
index d99e24dcc0bf..5a29bf01c0e8 100644
--- a/drivers/gpu/drm/omapdrm/omap_crtc.c
+++ b/drivers/gpu/drm/omapdrm/omap_crtc.c
@@ -128,7 +128,7 @@ static void omap_crtc_set_enabled(struct drm_crtc *crtc, bool enable)
 	if (WARN_ON(omap_crtc->enabled == enable))
 		return;
 
-	if (omap_crtc->pipe->output->output_type == OMAP_DISPLAY_TYPE_HDMI) {
+	if (omap_crtc->pipe->output->type == OMAP_DISPLAY_TYPE_HDMI) {
 		priv->dispc_ops->mgr_enable(priv->dispc, channel, enable);
 		omap_crtc->enabled = enable;
 		return;
@@ -390,6 +390,15 @@ static enum drm_mode_status omap_crtc_mode_valid(struct drm_crtc *crtc,
 					const struct drm_display_mode *mode)
 {
 	struct omap_drm_private *priv = crtc->dev->dev_private;
+	struct omap_crtc *omap_crtc = to_omap_crtc(crtc);
+	struct videomode vm = {0};
+	int r;
+
+	drm_display_mode_to_videomode(mode, &vm);
+	r = priv->dispc_ops->mgr_check_timings(priv->dispc, omap_crtc->channel,
+					       &vm);
+	if (r)
+		return r;
 
 	/* Check for bandwidth limit */
 	if (priv->max_bandwidth) {
@@ -657,7 +666,7 @@ struct drm_crtc *omap_crtc_init(struct drm_device *dev,
 					&omap_crtc_funcs, NULL);
 	if (ret < 0) {
 		dev_err(dev->dev, "%s(): could not init crtc for: %s\n",
-			__func__, pipe->display->name);
+			__func__, pipe->output->name);
 		kfree(omap_crtc);
 		return ERR_PTR(ret);
 	}
diff --git a/drivers/gpu/drm/omapdrm/omap_drv.c b/drivers/gpu/drm/omapdrm/omap_drv.c
index f8292278f57d..1b9b6f5e48e1 100644
--- a/drivers/gpu/drm/omapdrm/omap_drv.c
+++ b/drivers/gpu/drm/omapdrm/omap_drv.c
@@ -23,6 +23,7 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_probe_helper.h>
+#include <drm/drm_panel.h>
 
 #include "omap_dmm_tiler.h"
 #include "omap_drv.h"
@@ -137,12 +138,13 @@ static void omap_disconnect_pipelines(struct drm_device *ddev)
 	for (i = 0; i < priv->num_pipes; i++) {
 		struct omap_drm_pipeline *pipe = &priv->pipes[i];
 
+		if (pipe->output->panel)
+			drm_panel_detach(pipe->output->panel);
+
 		omapdss_device_disconnect(NULL, pipe->output);
 
 		omapdss_device_put(pipe->output);
-		omapdss_device_put(pipe->display);
 		pipe->output = NULL;
-		pipe->display = NULL;
 	}
 
 	memset(&priv->channels, 0, sizeof(priv->channels));
@@ -150,33 +152,17 @@ static void omap_disconnect_pipelines(struct drm_device *ddev)
 	priv->num_pipes = 0;
 }
 
-static int omap_compare_pipes(const void *a, const void *b)
-{
-	const struct omap_drm_pipeline *pipe1 = a;
-	const struct omap_drm_pipeline *pipe2 = b;
-
-	if (pipe1->display->alias_id > pipe2->display->alias_id)
-		return 1;
-	else if (pipe1->display->alias_id < pipe2->display->alias_id)
-		return -1;
-	return 0;
-}
-
 static int omap_connect_pipelines(struct drm_device *ddev)
 {
 	struct omap_drm_private *priv = ddev->dev_private;
 	struct omap_dss_device *output = NULL;
-	unsigned int i;
 	int r;
 
-	if (!omapdss_stack_is_ready())
-		return -EPROBE_DEFER;
-
 	for_each_dss_output(output) {
 		r = omapdss_device_connect(priv->dss, NULL, output);
 		if (r == -EPROBE_DEFER) {
 			omapdss_device_put(output);
-			goto cleanup;
+			return r;
 		} else if (r) {
 			dev_warn(output->dev, "could not connect output %s\n",
 				 output->name);
@@ -185,7 +171,6 @@ static int omap_connect_pipelines(struct drm_device *ddev)
 
 			pipe = &priv->pipes[priv->num_pipes++];
 			pipe->output = omapdss_device_get(output);
-			pipe->display = omapdss_display_get(output);
 
 			if (priv->num_pipes == ARRAY_SIZE(priv->pipes)) {
 				/* To balance the 'for_each_dss_output' loop */
@@ -195,36 +180,19 @@ static int omap_connect_pipelines(struct drm_device *ddev)
 		}
 	}
 
-	/* Sort the list by DT aliases */
-	sort(priv->pipes, priv->num_pipes, sizeof(priv->pipes[0]),
-	     omap_compare_pipes, NULL);
-
-	/*
-	 * Populate the pipeline lookup table by DISPC channel. Only one display
-	 * is allowed per channel.
-	 */
-	for (i = 0; i < priv->num_pipes; ++i) {
-		struct omap_drm_pipeline *pipe = &priv->pipes[i];
-		enum omap_channel channel = pipe->output->dispc_channel;
-
-		if (WARN_ON(priv->channels[channel] != NULL)) {
-			r = -EINVAL;
-			goto cleanup;
-		}
-
-		priv->channels[channel] = pipe;
-	}
-
 	return 0;
+}
 
-cleanup:
-	/*
-	 * if we are deferring probe, we disconnect the devices we previously
-	 * connected
-	 */
-	omap_disconnect_pipelines(ddev);
+static int omap_compare_pipelines(const void *a, const void *b)
+{
+	const struct omap_drm_pipeline *pipe1 = a;
+	const struct omap_drm_pipeline *pipe2 = b;
 
-	return r;
+	if (pipe1->alias_id > pipe2->alias_id)
+		return 1;
+	else if (pipe1->alias_id < pipe2->alias_id)
+		return -1;
+	return 0;
 }
 
 static int omap_modeset_init_properties(struct drm_device *dev)
@@ -240,6 +208,30 @@ static int omap_modeset_init_properties(struct drm_device *dev)
 	return 0;
 }
 
+static int omap_display_id(struct omap_dss_device *output)
+{
+	struct device_node *node = NULL;
+
+	if (output->next) {
+		struct omap_dss_device *display;
+
+		display = omapdss_display_get(output);
+		node = display->dev->of_node;
+		omapdss_device_put(display);
+	} else if (output->bridge) {
+		struct drm_bridge *bridge = output->bridge;
+
+		while (bridge->next)
+			bridge = bridge->next;
+
+		node = bridge->of_node;
+	} else if (output->panel) {
+		node = output->panel->dev->of_node;
+	}
+
+	return node ? of_alias_get_id(node, "display") : -ENODEV;
+}
+
 static int omap_modeset_init(struct drm_device *dev)
 {
 	struct omap_drm_private *priv = dev->dev_private;
@@ -249,6 +241,9 @@ static int omap_modeset_init(struct drm_device *dev)
 	int ret;
 	u32 plane_crtc_mask;
 
+	if (!omapdss_stack_is_ready())
+		return -EPROBE_DEFER;
+
 	drm_mode_config_init(dev);
 
 	ret = omap_modeset_init_properties(dev);
@@ -263,6 +258,10 @@ static int omap_modeset_init(struct drm_device *dev)
 	 * configuration does not match the expectations or exceeds
 	 * the available resources, the configuration is rejected.
 	 */
+	ret = omap_connect_pipelines(dev);
+	if (ret < 0)
+		return ret;
+
 	if (priv->num_pipes > num_mgrs || priv->num_pipes > num_ovls) {
 		dev_err(dev->dev, "%s(): Too many connected displays\n",
 			__func__);
@@ -288,33 +287,75 @@ static int omap_modeset_init(struct drm_device *dev)
 		priv->planes[priv->num_planes++] = plane;
 	}
 
-	/* Create the CRTCs, encoders and connectors. */
+	/*
+	 * Create the encoders, attach the bridges and get the pipeline alias
+	 * IDs.
+	 */
 	for (i = 0; i < priv->num_pipes; i++) {
 		struct omap_drm_pipeline *pipe = &priv->pipes[i];
-		struct omap_dss_device *display = pipe->display;
-		struct drm_connector *connector;
-		struct drm_encoder *encoder;
-		struct drm_crtc *crtc;
+		int id;
 
-		encoder = omap_encoder_init(dev, pipe->output, display);
-		if (!encoder)
+		pipe->encoder = omap_encoder_init(dev, pipe->output);
+		if (!pipe->encoder)
 			return -ENOMEM;
 
-		connector = omap_connector_init(dev, pipe->output, display,
-						encoder);
-		if (!connector)
-			return -ENOMEM;
+		if (pipe->output->bridge) {
+			ret = drm_bridge_attach(pipe->encoder,
+						pipe->output->bridge, NULL);
+			if (ret < 0)
+				return ret;
+		}
+
+		id = omap_display_id(pipe->output);
+		pipe->alias_id = id >= 0 ? id : i;
+	}
+
+	/* Sort the pipelines by DT aliases. */
+	sort(priv->pipes, priv->num_pipes, sizeof(priv->pipes[0]),
+	     omap_compare_pipelines, NULL);
+
+	/*
+	 * Populate the pipeline lookup table by DISPC channel. Only one display
+	 * is allowed per channel.
+	 */
+	for (i = 0; i < priv->num_pipes; ++i) {
+		struct omap_drm_pipeline *pipe = &priv->pipes[i];
+		enum omap_channel channel = pipe->output->dispc_channel;
+
+		if (WARN_ON(priv->channels[channel] != NULL))
+			return -EINVAL;
+
+		priv->channels[channel] = pipe;
+	}
+
+	/* Create the connectors and CRTCs. */
+	for (i = 0; i < priv->num_pipes; i++) {
+		struct omap_drm_pipeline *pipe = &priv->pipes[i];
+		struct drm_encoder *encoder = pipe->encoder;
+		struct drm_crtc *crtc;
+
+		if (!pipe->output->bridge) {
+			pipe->connector = omap_connector_init(dev, pipe->output,
+							      encoder);
+			if (!pipe->connector)
+				return -ENOMEM;
+
+			drm_connector_attach_encoder(pipe->connector, encoder);
+
+			if (pipe->output->panel) {
+				ret = drm_panel_attach(pipe->output->panel,
+						       pipe->connector);
+				if (ret < 0)
+					return ret;
+			}
+		}
 
 		crtc = omap_crtc_init(dev, pipe, priv->planes[i]);
 		if (IS_ERR(crtc))
 			return PTR_ERR(crtc);
 
-		drm_connector_attach_encoder(connector, encoder);
 		encoder->possible_crtcs = 1 << i;
-
 		pipe->crtc = crtc;
-		pipe->encoder = encoder;
-		pipe->connector = connector;
 	}
 
 	DBG("registered %u planes, %u crtcs/encoders/connectors\n",
@@ -351,10 +392,12 @@ static int omap_modeset_init(struct drm_device *dev)
 static void omap_modeset_enable_external_hpd(struct drm_device *ddev)
 {
 	struct omap_drm_private *priv = ddev->dev_private;
-	int i;
+	unsigned int i;
 
-	for (i = 0; i < priv->num_pipes; i++)
-		omap_connector_enable_hpd(priv->pipes[i].connector);
+	for (i = 0; i < priv->num_pipes; i++) {
+		if (priv->pipes[i].connector)
+			omap_connector_enable_hpd(priv->pipes[i].connector);
+	}
 }
 
 /*
@@ -363,10 +406,12 @@ static void omap_modeset_enable_external_hpd(struct drm_device *ddev)
 static void omap_modeset_disable_external_hpd(struct drm_device *ddev)
 {
 	struct omap_drm_private *priv = ddev->dev_private;
-	int i;
+	unsigned int i;
 
-	for (i = 0; i < priv->num_pipes; i++)
-		omap_connector_disable_hpd(priv->pipes[i].connector);
+	for (i = 0; i < priv->num_pipes; i++) {
+		if (priv->pipes[i].connector)
+			omap_connector_disable_hpd(priv->pipes[i].connector);
+	}
 }
 
 /*
@@ -551,10 +596,6 @@ static int omapdrm_init(struct omap_drm_private *priv, struct device *dev)
 
 	omap_crtc_pre_init(priv);
 
-	ret = omap_connect_pipelines(ddev);
-	if (ret)
-		goto err_crtc_uninit;
-
 	soc = soc_device_match(omapdrm_soc_devices);
 	priv->omaprev = soc ? (unsigned int)soc->data : 0;
 	priv->wq = alloc_ordered_workqueue("omapdrm", 0);
@@ -612,7 +653,6 @@ err_gem_deinit:
 	omap_gem_deinit(ddev);
 	destroy_workqueue(priv->wq);
 	omap_disconnect_pipelines(ddev);
-err_crtc_uninit:
 	omap_crtc_pre_uninit(priv);
 	drm_dev_put(ddev);
 	return ret;
@@ -685,54 +725,12 @@ static int pdev_remove(struct platform_device *pdev)
 }
 
 #ifdef CONFIG_PM_SLEEP
-static int omap_drm_suspend_all_displays(struct drm_device *ddev)
-{
-	struct omap_drm_private *priv = ddev->dev_private;
-	int i;
-
-	for (i = 0; i < priv->num_pipes; i++) {
-		struct omap_dss_device *display = priv->pipes[i].display;
-
-		if (display->state == OMAP_DSS_DISPLAY_ACTIVE) {
-			display->ops->disable(display);
-			display->activate_after_resume = true;
-		} else {
-			display->activate_after_resume = false;
-		}
-	}
-
-	return 0;
-}
-
-static int omap_drm_resume_all_displays(struct drm_device *ddev)
-{
-	struct omap_drm_private *priv = ddev->dev_private;
-	int i;
-
-	for (i = 0; i < priv->num_pipes; i++) {
-		struct omap_dss_device *display = priv->pipes[i].display;
-
-		if (display->activate_after_resume) {
-			display->ops->enable(display);
-			display->activate_after_resume = false;
-		}
-	}
-
-	return 0;
-}
-
 static int omap_drm_suspend(struct device *dev)
 {
 	struct omap_drm_private *priv = dev_get_drvdata(dev);
 	struct drm_device *drm_dev = priv->ddev;
 
-	drm_kms_helper_poll_disable(drm_dev);
-
-	drm_modeset_lock_all(drm_dev);
-	omap_drm_suspend_all_displays(drm_dev);
-	drm_modeset_unlock_all(drm_dev);
-
-	return 0;
+	return drm_mode_config_helper_suspend(drm_dev);
 }
 
 static int omap_drm_resume(struct device *dev)
@@ -740,11 +738,7 @@ static int omap_drm_resume(struct device *dev)
 	struct omap_drm_private *priv = dev_get_drvdata(dev);
 	struct drm_device *drm_dev = priv->ddev;
 
-	drm_modeset_lock_all(drm_dev);
-	omap_drm_resume_all_displays(drm_dev);
-	drm_modeset_unlock_all(drm_dev);
-
-	drm_kms_helper_poll_enable(drm_dev);
+	drm_mode_config_helper_resume(drm_dev);
 
 	return omap_gem_resume(drm_dev);
 }
diff --git a/drivers/gpu/drm/omapdrm/omap_drv.h b/drivers/gpu/drm/omapdrm/omap_drv.h
index 0c57d2814c51..3cca45cb25f3 100644
--- a/drivers/gpu/drm/omapdrm/omap_drv.h
+++ b/drivers/gpu/drm/omapdrm/omap_drv.h
@@ -49,7 +49,7 @@ struct omap_drm_pipeline {
 	struct drm_encoder *encoder;
 	struct drm_connector *connector;
 	struct omap_dss_device *output;
-	struct omap_dss_device *display;
+	unsigned int alias_id;
 };
 
 struct omap_drm_private {
diff --git a/drivers/gpu/drm/omapdrm/omap_encoder.c b/drivers/gpu/drm/omapdrm/omap_encoder.c
index 0d85b3a35767..40512419642b 100644
--- a/drivers/gpu/drm/omapdrm/omap_encoder.c
+++ b/drivers/gpu/drm/omapdrm/omap_encoder.c
@@ -20,6 +20,7 @@
 #include <drm/drm_crtc.h>
 #include <drm/drm_modeset_helper_vtables.h>
 #include <drm/drm_edid.h>
+#include <drm/drm_panel.h>
 
 #include "omap_drv.h"
 
@@ -37,7 +38,6 @@
 struct omap_encoder {
 	struct drm_encoder base;
 	struct omap_dss_device *output;
-	struct omap_dss_device *display;
 };
 
 static void omap_encoder_destroy(struct drm_encoder *encoder)
@@ -52,22 +52,43 @@ static const struct drm_encoder_funcs omap_encoder_funcs = {
 	.destroy = omap_encoder_destroy,
 };
 
-static void omap_encoder_hdmi_mode_set(struct drm_encoder *encoder,
+static void omap_encoder_update_videomode_flags(struct videomode *vm,
+						u32 bus_flags)
+{
+	if (!(vm->flags & (DISPLAY_FLAGS_DE_LOW |
+			   DISPLAY_FLAGS_DE_HIGH))) {
+		if (bus_flags & DRM_BUS_FLAG_DE_LOW)
+			vm->flags |= DISPLAY_FLAGS_DE_LOW;
+		else if (bus_flags & DRM_BUS_FLAG_DE_HIGH)
+			vm->flags |= DISPLAY_FLAGS_DE_HIGH;
+	}
+
+	if (!(vm->flags & (DISPLAY_FLAGS_PIXDATA_POSEDGE |
+			   DISPLAY_FLAGS_PIXDATA_NEGEDGE))) {
+		if (bus_flags & DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE)
+			vm->flags |= DISPLAY_FLAGS_PIXDATA_POSEDGE;
+		else if (bus_flags & DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE)
+			vm->flags |= DISPLAY_FLAGS_PIXDATA_NEGEDGE;
+	}
+
+	if (!(vm->flags & (DISPLAY_FLAGS_SYNC_POSEDGE |
+			   DISPLAY_FLAGS_SYNC_NEGEDGE))) {
+		if (bus_flags & DRM_BUS_FLAG_SYNC_DRIVE_POSEDGE)
+			vm->flags |= DISPLAY_FLAGS_SYNC_POSEDGE;
+		else if (bus_flags & DRM_BUS_FLAG_SYNC_DRIVE_NEGEDGE)
+			vm->flags |= DISPLAY_FLAGS_SYNC_NEGEDGE;
+	}
+}
+
+static void omap_encoder_hdmi_mode_set(struct drm_connector *connector,
+				       struct drm_encoder *encoder,
 				       struct drm_display_mode *adjusted_mode)
 {
-	struct drm_device *dev = encoder->dev;
 	struct omap_encoder *omap_encoder = to_omap_encoder(encoder);
 	struct omap_dss_device *dssdev = omap_encoder->output;
-	struct drm_connector *connector;
 	bool hdmi_mode;
 
-	hdmi_mode = false;
-	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
-		if (connector->encoder == encoder) {
-			hdmi_mode = omap_connector_get_hdmi_mode(connector);
-			break;
-		}
-	}
+	hdmi_mode = omap_connector_get_hdmi_mode(connector);
 
 	if (dssdev->ops->hdmi.set_hdmi_mode)
 		dssdev->ops->hdmi.set_hdmi_mode(dssdev, hdmi_mode);
@@ -88,8 +109,18 @@ static void omap_encoder_mode_set(struct drm_encoder *encoder,
 				  struct drm_display_mode *adjusted_mode)
 {
 	struct omap_encoder *omap_encoder = to_omap_encoder(encoder);
+	struct omap_dss_device *output = omap_encoder->output;
 	struct omap_dss_device *dssdev;
+	struct drm_device *dev = encoder->dev;
+	struct drm_connector *connector;
+	struct drm_bridge *bridge;
 	struct videomode vm = { 0 };
+	u32 bus_flags;
+
+	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
+		if (connector->encoder == encoder)
+			break;
+	}
 
 	drm_display_mode_to_videomode(adjusted_mode, &vm);
 
@@ -102,66 +133,102 @@ static void omap_encoder_mode_set(struct drm_encoder *encoder,
 	 *
 	 * A better solution is to use DRM's bus-flags through the whole driver.
 	 */
-	for (dssdev = omap_encoder->output; dssdev; dssdev = dssdev->next) {
-		unsigned long bus_flags = dssdev->bus_flags;
-
-		if (!(vm.flags & (DISPLAY_FLAGS_DE_LOW |
-				  DISPLAY_FLAGS_DE_HIGH))) {
-			if (bus_flags & DRM_BUS_FLAG_DE_LOW)
-				vm.flags |= DISPLAY_FLAGS_DE_LOW;
-			else if (bus_flags & DRM_BUS_FLAG_DE_HIGH)
-				vm.flags |= DISPLAY_FLAGS_DE_HIGH;
-		}
-
-		if (!(vm.flags & (DISPLAY_FLAGS_PIXDATA_POSEDGE |
-				  DISPLAY_FLAGS_PIXDATA_NEGEDGE))) {
-			if (bus_flags & DRM_BUS_FLAG_PIXDATA_POSEDGE)
-				vm.flags |= DISPLAY_FLAGS_PIXDATA_POSEDGE;
-			else if (bus_flags & DRM_BUS_FLAG_PIXDATA_NEGEDGE)
-				vm.flags |= DISPLAY_FLAGS_PIXDATA_NEGEDGE;
-		}
-
-		if (!(vm.flags & (DISPLAY_FLAGS_SYNC_POSEDGE |
-				  DISPLAY_FLAGS_SYNC_NEGEDGE))) {
-			if (bus_flags & DRM_BUS_FLAG_SYNC_POSEDGE)
-				vm.flags |= DISPLAY_FLAGS_SYNC_POSEDGE;
-			else if (bus_flags & DRM_BUS_FLAG_SYNC_NEGEDGE)
-				vm.flags |= DISPLAY_FLAGS_SYNC_NEGEDGE;
-		}
+	for (dssdev = output; dssdev; dssdev = dssdev->next)
+		omap_encoder_update_videomode_flags(&vm, dssdev->bus_flags);
+
+	for (bridge = output->bridge; bridge; bridge = bridge->next) {
+		if (!bridge->timings)
+			continue;
+
+		bus_flags = bridge->timings->input_bus_flags;
+		omap_encoder_update_videomode_flags(&vm, bus_flags);
 	}
 
+	bus_flags = connector->display_info.bus_flags;
+	omap_encoder_update_videomode_flags(&vm, bus_flags);
+
 	/* Set timings for all devices in the display pipeline. */
-	dss_mgr_set_timings(omap_encoder->output, &vm);
+	dss_mgr_set_timings(output, &vm);
 
-	for (dssdev = omap_encoder->output; dssdev; dssdev = dssdev->next) {
+	for (dssdev = output; dssdev; dssdev = dssdev->next) {
 		if (dssdev->ops->set_timings)
-			dssdev->ops->set_timings(dssdev, &vm);
+			dssdev->ops->set_timings(dssdev, adjusted_mode);
 	}
 
 	/* Set the HDMI mode and HDMI infoframe if applicable. */
-	if (omap_encoder->output->output_type == OMAP_DISPLAY_TYPE_HDMI)
-		omap_encoder_hdmi_mode_set(encoder, adjusted_mode);
+	if (output->type == OMAP_DISPLAY_TYPE_HDMI)
+		omap_encoder_hdmi_mode_set(connector, encoder, adjusted_mode);
 }
 
 static void omap_encoder_disable(struct drm_encoder *encoder)
 {
 	struct omap_encoder *omap_encoder = to_omap_encoder(encoder);
-	struct omap_dss_device *dssdev = omap_encoder->display;
+	struct omap_dss_device *dssdev = omap_encoder->output;
+	struct drm_device *dev = encoder->dev;
+
+	dev_dbg(dev->dev, "disable(%s)\n", dssdev->name);
+
+	/* Disable the panel if present. */
+	if (dssdev->panel) {
+		drm_panel_disable(dssdev->panel);
+		drm_panel_unprepare(dssdev->panel);
+	}
+
+	/*
+	 * Disable the chain of external devices, starting at the one at the
+	 * internal encoder's output.
+	 */
+	omapdss_device_disable(dssdev->next);
+
+	/*
+	 * Disable the internal encoder. This will disable the DSS output. The
+	 * DSI is treated as an exception as DSI pipelines still use the legacy
+	 * flow where the pipeline output controls the encoder.
+	 */
+	if (dssdev->type != OMAP_DISPLAY_TYPE_DSI) {
+		dssdev->ops->disable(dssdev);
+		dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
+	}
 
-	dssdev->ops->disable(dssdev);
+	/*
+	 * Perform the post-disable operations on the chain of external devices
+	 * to complete the display pipeline disable.
+	 */
+	omapdss_device_post_disable(dssdev->next);
 }
 
 static void omap_encoder_enable(struct drm_encoder *encoder)
 {
 	struct omap_encoder *omap_encoder = to_omap_encoder(encoder);
-	struct omap_dss_device *dssdev = omap_encoder->display;
-	int r;
-
-	r = dssdev->ops->enable(dssdev);
-	if (r)
-		dev_err(encoder->dev->dev,
-			"Failed to enable display '%s': %d\n",
-			dssdev->name, r);
+	struct omap_dss_device *dssdev = omap_encoder->output;
+	struct drm_device *dev = encoder->dev;
+
+	dev_dbg(dev->dev, "enable(%s)\n", dssdev->name);
+
+	/* Prepare the chain of external devices for pipeline enable. */
+	omapdss_device_pre_enable(dssdev->next);
+
+	/*
+	 * Enable the internal encoder. This will enable the DSS output. The
+	 * DSI is treated as an exception as DSI pipelines still use the legacy
+	 * flow where the pipeline output controls the encoder.
+	 */
+	if (dssdev->type != OMAP_DISPLAY_TYPE_DSI) {
+		dssdev->ops->enable(dssdev);
+		dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
+	}
+
+	/*
+	 * Enable the chain of external devices, starting at the one at the
+	 * internal encoder's output.
+	 */
+	omapdss_device_enable(dssdev->next);
+
+	/* Enable the panel if present. */
+	if (dssdev->panel) {
+		drm_panel_prepare(dssdev->panel);
+		drm_panel_enable(dssdev->panel);
+	}
 }
 
 static int omap_encoder_atomic_check(struct drm_encoder *encoder,
@@ -169,35 +236,17 @@ static int omap_encoder_atomic_check(struct drm_encoder *encoder,
 				     struct drm_connector_state *conn_state)
 {
 	struct omap_encoder *omap_encoder = to_omap_encoder(encoder);
-	enum omap_channel channel = omap_encoder->output->dispc_channel;
-	struct drm_device *dev = encoder->dev;
-	struct omap_drm_private *priv = dev->dev_private;
-	struct omap_dss_device *dssdev;
-	struct videomode vm = { 0 };
-	int ret;
-
-	drm_display_mode_to_videomode(&crtc_state->mode, &vm);
-
-	ret = priv->dispc_ops->mgr_check_timings(priv->dispc, channel, &vm);
-	if (ret)
-		goto done;
-
-	for (dssdev = omap_encoder->output; dssdev; dssdev = dssdev->next) {
-		if (!dssdev->ops->check_timings)
-			continue;
-
-		ret = dssdev->ops->check_timings(dssdev, &vm);
-		if (ret)
-			goto done;
+	enum drm_mode_status status;
+
+	status = omap_connector_mode_fixup(omap_encoder->output,
+					   &crtc_state->mode,
+					   &crtc_state->adjusted_mode);
+	if (status != MODE_OK) {
+		dev_err(encoder->dev->dev, "invalid timings: %d\n", status);
+		return -EINVAL;
 	}
 
-	drm_display_mode_from_videomode(&vm, &crtc_state->adjusted_mode);
-
-done:
-	if (ret)
-		dev_err(dev->dev, "invalid timings: %d\n", ret);
-
-	return ret;
+	return 0;
 }
 
 static const struct drm_encoder_helper_funcs omap_encoder_helper_funcs = {
@@ -209,8 +258,7 @@ static const struct drm_encoder_helper_funcs omap_encoder_helper_funcs = {
 
 /* initialize encoder */
 struct drm_encoder *omap_encoder_init(struct drm_device *dev,
-				      struct omap_dss_device *output,
-				      struct omap_dss_device *display)
+				      struct omap_dss_device *output)
 {
 	struct drm_encoder *encoder = NULL;
 	struct omap_encoder *omap_encoder;
@@ -220,7 +268,6 @@ struct drm_encoder *omap_encoder_init(struct drm_device *dev,
 		goto fail;
 
 	omap_encoder->output = output;
-	omap_encoder->display = display;
 
 	encoder = &omap_encoder->base;
 
diff --git a/drivers/gpu/drm/omapdrm/omap_encoder.h b/drivers/gpu/drm/omapdrm/omap_encoder.h
index a7b5dde63ecb..4aefb3142886 100644
--- a/drivers/gpu/drm/omapdrm/omap_encoder.h
+++ b/drivers/gpu/drm/omapdrm/omap_encoder.h
@@ -25,7 +25,6 @@ struct drm_encoder;
 struct omap_dss_device;
 
 struct drm_encoder *omap_encoder_init(struct drm_device *dev,
-				      struct omap_dss_device *output,
-				      struct omap_dss_device *display);
+				      struct omap_dss_device *output);
 
 #endif /* __OMAPDRM_ENCODER_H__ */
diff --git a/drivers/gpu/drm/omapdrm/omap_fbdev.c b/drivers/gpu/drm/omapdrm/omap_fbdev.c
index 851c59f07eb1..50aabd854f4d 100644
--- a/drivers/gpu/drm/omapdrm/omap_fbdev.c
+++ b/drivers/gpu/drm/omapdrm/omap_fbdev.c
@@ -183,13 +183,9 @@ static int omap_fbdev_create(struct drm_fb_helper *helper,
 	fbdev->fb = fb;
 	helper->fb = fb;
 
-	fbi->par = helper;
 	fbi->fbops = &omap_fb_ops;
 
-	strcpy(fbi->fix.id, MODULE_NAME);
-
-	drm_fb_helper_fill_fix(fbi, fb->pitches[0], fb->format->depth);
-	drm_fb_helper_fill_var(fbi, helper, sizes->fb_width, sizes->fb_height);
+	drm_fb_helper_fill_info(fbi, helper, sizes);
 
 	dev->mode_config.fb_base = dma_addr;
 
diff --git a/drivers/gpu/drm/panel/Kconfig b/drivers/gpu/drm/panel/Kconfig
index 3e070153ef21..e36dbb4df867 100644
--- a/drivers/gpu/drm/panel/Kconfig
+++ b/drivers/gpu/drm/panel/Kconfig
@@ -38,6 +38,15 @@ config DRM_PANEL_SIMPLE
 	  that it can be automatically turned off when the panel goes into a
 	  low power state.
 
+config DRM_PANEL_FEIYANG_FY07024DI26A30D
+	tristate "Feiyang FY07024DI26A30-D MIPI-DSI LCD panel"
+	depends on OF
+	depends on DRM_MIPI_DSI
+	depends on BACKLIGHT_CLASS_DEVICE
+	help
+	  Say Y if you want to enable support for panels based on the
+	  Feiyang FY07024DI26A30-D MIPI-DSI interface.
+
 config DRM_PANEL_ILITEK_IL9322
 	tristate "Ilitek ILI9322 320x240 QVGA panels"
 	depends on OF && SPI
@@ -149,6 +158,28 @@ config DRM_PANEL_RAYDIUM_RM68200
 	  Say Y here if you want to enable support for Raydium RM68200
 	  720x1280 DSI video mode panel.
 
+config DRM_PANEL_ROCKTECH_JH057N00900
+	tristate "Rocktech JH057N00900 MIPI touchscreen panel"
+	depends on OF
+	depends on DRM_MIPI_DSI
+	depends on BACKLIGHT_CLASS_DEVICE
+	help
+	  Say Y here if you want to enable support for Rocktech JH057N00900
+	  MIPI DSI panel as e.g. used in the Librem 5 devkit. It has a
+	  resolution of 720x1440 pixels, a built in backlight and touch
+	  controller.
+	  Touch input support is provided by the goodix driver and needs to be
+	  selected separately.
+
+config DRM_PANEL_RONBO_RB070D30
+	tristate "Ronbo Electronics RB070D30 panel"
+	depends on OF
+	depends on DRM_MIPI_DSI
+	depends on BACKLIGHT_CLASS_DEVICE
+	help
+	  Say Y here if you want to enable support for Ronbo Electronics
+	  RB070D30 1024x600 DSI panel.
+
 config DRM_PANEL_SAMSUNG_S6D16D0
 	tristate "Samsung S6D16D0 DSI video mode panel"
 	depends on OF
diff --git a/drivers/gpu/drm/panel/Makefile b/drivers/gpu/drm/panel/Makefile
index e7ab71968bbf..78e3dc376bdd 100644
--- a/drivers/gpu/drm/panel/Makefile
+++ b/drivers/gpu/drm/panel/Makefile
@@ -2,6 +2,7 @@
 obj-$(CONFIG_DRM_PANEL_ARM_VERSATILE) += panel-arm-versatile.o
 obj-$(CONFIG_DRM_PANEL_LVDS) += panel-lvds.o
 obj-$(CONFIG_DRM_PANEL_SIMPLE) += panel-simple.o
+obj-$(CONFIG_DRM_PANEL_FEIYANG_FY07024DI26A30D) += panel-feiyang-fy07024di26a30d.o
 obj-$(CONFIG_DRM_PANEL_ILITEK_IL9322) += panel-ilitek-ili9322.o
 obj-$(CONFIG_DRM_PANEL_ILITEK_ILI9881C) += panel-ilitek-ili9881c.o
 obj-$(CONFIG_DRM_PANEL_INNOLUX_P079ZCA) += panel-innolux-p079zca.o
@@ -13,6 +14,8 @@ obj-$(CONFIG_DRM_PANEL_ORISETECH_OTM8009A) += panel-orisetech-otm8009a.o
 obj-$(CONFIG_DRM_PANEL_PANASONIC_VVX10F034N00) += panel-panasonic-vvx10f034n00.o
 obj-$(CONFIG_DRM_PANEL_RASPBERRYPI_TOUCHSCREEN) += panel-raspberrypi-touchscreen.o
 obj-$(CONFIG_DRM_PANEL_RAYDIUM_RM68200) += panel-raydium-rm68200.o
+obj-$(CONFIG_DRM_PANEL_ROCKTECH_JH057N00900) += panel-rocktech-jh057n00900.o
+obj-$(CONFIG_DRM_PANEL_RONBO_RB070D30) += panel-ronbo-rb070d30.o
 obj-$(CONFIG_DRM_PANEL_SAMSUNG_LD9040) += panel-samsung-ld9040.o
 obj-$(CONFIG_DRM_PANEL_SAMSUNG_S6D16D0) += panel-samsung-s6d16d0.o
 obj-$(CONFIG_DRM_PANEL_SAMSUNG_S6E3HA2) += panel-samsung-s6e3ha2.o
diff --git a/drivers/gpu/drm/panel/panel-arm-versatile.c b/drivers/gpu/drm/panel/panel-arm-versatile.c
index b428c4678106..a79908dfa3c8 100644
--- a/drivers/gpu/drm/panel/panel-arm-versatile.c
+++ b/drivers/gpu/drm/panel/panel-arm-versatile.c
@@ -191,7 +191,7 @@ static const struct versatile_panel_type versatile_panels[] = {
 			.vrefresh = 390,
 			.flags = DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC,
 		},
-		.bus_flags = DRM_BUS_FLAG_PIXDATA_NEGEDGE,
+		.bus_flags = DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE,
 	},
 	/*
 	 * Sanyo ALR252RGT 240x320 portrait display found on the
@@ -215,7 +215,7 @@ static const struct versatile_panel_type versatile_panels[] = {
 			.vrefresh = 116,
 			.flags = DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC,
 		},
-		.bus_flags = DRM_BUS_FLAG_PIXDATA_NEGEDGE,
+		.bus_flags = DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE,
 		.ib2 = true,
 	},
 };
@@ -264,8 +264,6 @@ static int versatile_panel_get_modes(struct drm_panel *panel)
 	struct versatile_panel *vpanel = to_versatile_panel(panel);
 	struct drm_display_mode *mode;
 
-	strncpy(connector->display_info.name, vpanel->panel_type->name,
-		DRM_DISPLAY_INFO_LEN);
 	connector->display_info.width_mm = vpanel->panel_type->width_mm;
 	connector->display_info.height_mm = vpanel->panel_type->height_mm;
 	connector->display_info.bus_flags = vpanel->panel_type->bus_flags;
diff --git a/drivers/gpu/drm/panel/panel-feiyang-fy07024di26a30d.c b/drivers/gpu/drm/panel/panel-feiyang-fy07024di26a30d.c
new file mode 100644
index 000000000000..dabf59e0f56f
--- /dev/null
+++ b/drivers/gpu/drm/panel/panel-feiyang-fy07024di26a30d.c
@@ -0,0 +1,272 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (C) 2018 Amarula Solutions
+ * Author: Jagan Teki <jagan@amarulasolutions.com>
+ */
+
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_modes.h>
+#include <drm/drm_panel.h>
+#include <drm/drm_print.h>
+
+#include <linux/backlight.h>
+#include <linux/gpio/consumer.h>
+#include <linux/delay.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/regulator/consumer.h>
+
+#define FEIYANG_INIT_CMD_LEN	2
+
+struct feiyang {
+	struct drm_panel	panel;
+	struct mipi_dsi_device	*dsi;
+
+	struct backlight_device	*backlight;
+	struct regulator	*dvdd;
+	struct regulator	*avdd;
+	struct gpio_desc	*reset;
+};
+
+static inline struct feiyang *panel_to_feiyang(struct drm_panel *panel)
+{
+	return container_of(panel, struct feiyang, panel);
+}
+
+struct feiyang_init_cmd {
+	u8 data[FEIYANG_INIT_CMD_LEN];
+};
+
+static const struct feiyang_init_cmd feiyang_init_cmds[] = {
+	{ .data = { 0x80, 0x58 } },
+	{ .data = { 0x81, 0x47 } },
+	{ .data = { 0x82, 0xD4 } },
+	{ .data = { 0x83, 0x88 } },
+	{ .data = { 0x84, 0xA9 } },
+	{ .data = { 0x85, 0xC3 } },
+	{ .data = { 0x86, 0x82 } },
+};
+
+static int feiyang_prepare(struct drm_panel *panel)
+{
+	struct feiyang *ctx = panel_to_feiyang(panel);
+	struct mipi_dsi_device *dsi = ctx->dsi;
+	unsigned int i;
+	int ret;
+
+	ret = regulator_enable(ctx->dvdd);
+	if (ret)
+		return ret;
+
+	/* T1 (dvdd start + dvdd rise) 0 < T1 <= 10ms */
+	msleep(10);
+
+	ret = regulator_enable(ctx->avdd);
+	if (ret)
+		return ret;
+
+	/* T3 (dvdd rise + avdd start + avdd rise) T3 >= 20ms */
+	msleep(20);
+
+	gpiod_set_value(ctx->reset, 0);
+
+	/*
+	 * T5 + T6 (avdd rise + video & logic signal rise)
+	 * T5 >= 10ms, 0 < T6 <= 10ms
+	 */
+	msleep(20);
+
+	gpiod_set_value(ctx->reset, 1);
+
+	/* T12 (video & logic signal rise + backlight rise) T12 >= 200ms */
+	msleep(200);
+
+	for (i = 0; i < ARRAY_SIZE(feiyang_init_cmds); i++) {
+		const struct feiyang_init_cmd *cmd =
+						&feiyang_init_cmds[i];
+
+		ret = mipi_dsi_dcs_write_buffer(dsi, cmd->data,
+						FEIYANG_INIT_CMD_LEN);
+		if (ret < 0)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int feiyang_enable(struct drm_panel *panel)
+{
+	struct feiyang *ctx = panel_to_feiyang(panel);
+
+	/* T12 (video & logic signal rise + backlight rise) T12 >= 200ms */
+	msleep(200);
+
+	mipi_dsi_dcs_set_display_on(ctx->dsi);
+	backlight_enable(ctx->backlight);
+
+	return 0;
+}
+
+static int feiyang_disable(struct drm_panel *panel)
+{
+	struct feiyang *ctx = panel_to_feiyang(panel);
+
+	backlight_disable(ctx->backlight);
+	return mipi_dsi_dcs_set_display_off(ctx->dsi);
+}
+
+static int feiyang_unprepare(struct drm_panel *panel)
+{
+	struct feiyang *ctx = panel_to_feiyang(panel);
+	int ret;
+
+	ret = mipi_dsi_dcs_set_display_off(ctx->dsi);
+	if (ret < 0)
+		DRM_DEV_ERROR(panel->dev, "failed to set display off: %d\n",
+			      ret);
+
+	ret = mipi_dsi_dcs_enter_sleep_mode(ctx->dsi);
+	if (ret < 0)
+		DRM_DEV_ERROR(panel->dev, "failed to enter sleep mode: %d\n",
+			      ret);
+
+	/* T13 (backlight fall + video & logic signal fall) T13 >= 200ms */
+	msleep(200);
+
+	gpiod_set_value(ctx->reset, 0);
+
+	regulator_disable(ctx->avdd);
+
+	/* T11 (dvdd rise to fall) 0 < T11 <= 10ms  */
+	msleep(10);
+
+	regulator_disable(ctx->dvdd);
+
+	return 0;
+}
+
+static const struct drm_display_mode feiyang_default_mode = {
+	.clock		= 55000,
+
+	.hdisplay	= 1024,
+	.hsync_start	= 1024 + 310,
+	.hsync_end	= 1024 + 310 + 20,
+	.htotal		= 1024 + 310 + 20 + 90,
+
+	.vdisplay	= 600,
+	.vsync_start	= 600 + 12,
+	.vsync_end	= 600 + 12 + 2,
+	.vtotal		= 600 + 12 + 2 + 21,
+	.vrefresh	= 60,
+
+	.type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED,
+};
+
+static int feiyang_get_modes(struct drm_panel *panel)
+{
+	struct drm_connector *connector = panel->connector;
+	struct feiyang *ctx = panel_to_feiyang(panel);
+	struct drm_display_mode *mode;
+
+	mode = drm_mode_duplicate(panel->drm, &feiyang_default_mode);
+	if (!mode) {
+		DRM_DEV_ERROR(&ctx->dsi->dev, "failed to add mode %ux%ux@%u\n",
+			      feiyang_default_mode.hdisplay,
+			      feiyang_default_mode.vdisplay,
+			      feiyang_default_mode.vrefresh);
+		return -ENOMEM;
+	}
+
+	drm_mode_set_name(mode);
+
+	drm_mode_probed_add(connector, mode);
+
+	return 1;
+}
+
+static const struct drm_panel_funcs feiyang_funcs = {
+	.disable = feiyang_disable,
+	.unprepare = feiyang_unprepare,
+	.prepare = feiyang_prepare,
+	.enable = feiyang_enable,
+	.get_modes = feiyang_get_modes,
+};
+
+static int feiyang_dsi_probe(struct mipi_dsi_device *dsi)
+{
+	struct feiyang *ctx;
+	int ret;
+
+	ctx = devm_kzalloc(&dsi->dev, sizeof(*ctx), GFP_KERNEL);
+	if (!ctx)
+		return -ENOMEM;
+
+	mipi_dsi_set_drvdata(dsi, ctx);
+	ctx->dsi = dsi;
+
+	drm_panel_init(&ctx->panel);
+	ctx->panel.dev = &dsi->dev;
+	ctx->panel.funcs = &feiyang_funcs;
+
+	ctx->dvdd = devm_regulator_get(&dsi->dev, "dvdd");
+	if (IS_ERR(ctx->dvdd)) {
+		DRM_DEV_ERROR(&dsi->dev, "Couldn't get dvdd regulator\n");
+		return PTR_ERR(ctx->dvdd);
+	}
+
+	ctx->avdd = devm_regulator_get(&dsi->dev, "avdd");
+	if (IS_ERR(ctx->avdd)) {
+		DRM_DEV_ERROR(&dsi->dev, "Couldn't get avdd regulator\n");
+		return PTR_ERR(ctx->avdd);
+	}
+
+	ctx->reset = devm_gpiod_get(&dsi->dev, "reset", GPIOD_OUT_LOW);
+	if (IS_ERR(ctx->reset)) {
+		DRM_DEV_ERROR(&dsi->dev, "Couldn't get our reset GPIO\n");
+		return PTR_ERR(ctx->reset);
+	}
+
+	ctx->backlight = devm_of_find_backlight(&dsi->dev);
+	if (IS_ERR(ctx->backlight))
+		return PTR_ERR(ctx->backlight);
+
+	ret = drm_panel_add(&ctx->panel);
+	if (ret < 0)
+		return ret;
+
+	dsi->mode_flags = MIPI_DSI_MODE_VIDEO_BURST;
+	dsi->format = MIPI_DSI_FMT_RGB888;
+	dsi->lanes = 4;
+
+	return mipi_dsi_attach(dsi);
+}
+
+static int feiyang_dsi_remove(struct mipi_dsi_device *dsi)
+{
+	struct feiyang *ctx = mipi_dsi_get_drvdata(dsi);
+
+	mipi_dsi_detach(dsi);
+	drm_panel_remove(&ctx->panel);
+
+	return 0;
+}
+
+static const struct of_device_id feiyang_of_match[] = {
+	{ .compatible = "feiyang,fy07024di26a30d", },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, feiyang_of_match);
+
+static struct mipi_dsi_driver feiyang_driver = {
+	.probe = feiyang_dsi_probe,
+	.remove = feiyang_dsi_remove,
+	.driver = {
+		.name = "feiyang-fy07024di26a30d",
+		.of_match_table = feiyang_of_match,
+	},
+};
+module_mipi_dsi_driver(feiyang_driver);
+
+MODULE_AUTHOR("Jagan Teki <jagan@amarulasolutions.com>");
+MODULE_DESCRIPTION("Feiyang FY07024DI26A30-D MIPI-DSI LCD panel");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/panel/panel-ilitek-ili9322.c b/drivers/gpu/drm/panel/panel-ilitek-ili9322.c
index bd38bf4f1ba6..a1c4cd2940fb 100644
--- a/drivers/gpu/drm/panel/panel-ilitek-ili9322.c
+++ b/drivers/gpu/drm/panel/panel-ilitek-ili9322.c
@@ -412,11 +412,11 @@ static int ili9322_init(struct drm_panel *panel, struct ili9322 *ili)
 	if (ili->conf->dclk_active_high) {
 		reg = ILI9322_POL_DCLK;
 		connector->display_info.bus_flags |=
-			DRM_BUS_FLAG_PIXDATA_POSEDGE;
+			DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE;
 	} else {
 		reg = 0;
 		connector->display_info.bus_flags |=
-			DRM_BUS_FLAG_PIXDATA_NEGEDGE;
+			DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE;
 	}
 	if (ili->conf->de_active_high) {
 		reg |= ILI9322_POL_DE;
@@ -662,8 +662,6 @@ static int ili9322_get_modes(struct drm_panel *panel)
 	struct ili9322 *ili = panel_to_ili9322(panel);
 	struct drm_display_mode *mode;
 
-	strncpy(connector->display_info.name, "ILI9322 TFT LCD driver\0",
-		DRM_DISPLAY_INFO_LEN);
 	connector->display_info.width_mm = ili->conf->width_mm;
 	connector->display_info.height_mm = ili->conf->height_mm;
 
diff --git a/drivers/gpu/drm/panel/panel-olimex-lcd-olinuxino.c b/drivers/gpu/drm/panel/panel-olimex-lcd-olinuxino.c
index 5e8d4523e9ed..a1d8d92fac2b 100644
--- a/drivers/gpu/drm/panel/panel-olimex-lcd-olinuxino.c
+++ b/drivers/gpu/drm/panel/panel-olimex-lcd-olinuxino.c
@@ -190,7 +190,6 @@ static int lcd_olinuxino_get_modes(struct drm_panel *panel)
 		num++;
 	}
 
-	memcpy(connector->display_info.name, lcd_info->name, 32);
 	connector->display_info.width_mm = lcd_info->width_mm;
 	connector->display_info.height_mm = lcd_info->height_mm;
 	connector->display_info.bpc = lcd_info->bpc;
diff --git a/drivers/gpu/drm/panel/panel-orisetech-otm8009a.c b/drivers/gpu/drm/panel/panel-orisetech-otm8009a.c
index 87fa316e1d7b..f27a7e426574 100644
--- a/drivers/gpu/drm/panel/panel-orisetech-otm8009a.c
+++ b/drivers/gpu/drm/panel/panel-orisetech-otm8009a.c
@@ -67,15 +67,15 @@ struct otm8009a {
 };
 
 static const struct drm_display_mode default_mode = {
-	.clock = 32729,
+	.clock = 29700,
 	.hdisplay = 480,
-	.hsync_start = 480 + 120,
-	.hsync_end = 480 + 120 + 63,
-	.htotal = 480 + 120 + 63 + 120,
+	.hsync_start = 480 + 98,
+	.hsync_end = 480 + 98 + 32,
+	.htotal = 480 + 98 + 32 + 98,
 	.vdisplay = 800,
-	.vsync_start = 800 + 12,
-	.vsync_end = 800 + 12 + 12,
-	.vtotal = 800 + 12 + 12 + 12,
+	.vsync_start = 800 + 15,
+	.vsync_end = 800 + 15 + 10,
+	.vtotal = 800 + 15 + 10 + 14,
 	.vrefresh = 50,
 	.flags = 0,
 	.width_mm = 52,
@@ -248,6 +248,9 @@ static int otm8009a_init_sequence(struct otm8009a *ctx)
 	/* Send Command GRAM memory write (no parameters) */
 	dcs_write_seq(ctx, MIPI_DCS_WRITE_MEMORY_START);
 
+	/* Wait a short while to let the panel be ready before the 1st frame */
+	mdelay(10);
+
 	return 0;
 }
 
@@ -433,7 +436,8 @@ static int otm8009a_probe(struct mipi_dsi_device *dsi)
 	ctx->supply = devm_regulator_get(dev, "power");
 	if (IS_ERR(ctx->supply)) {
 		ret = PTR_ERR(ctx->supply);
-		dev_err(dev, "failed to request regulator: %d\n", ret);
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "failed to request regulator: %d\n", ret);
 		return ret;
 	}
 
diff --git a/drivers/gpu/drm/panel/panel-raydium-rm68200.c b/drivers/gpu/drm/panel/panel-raydium-rm68200.c
index 77593533abcd..14186827e591 100644
--- a/drivers/gpu/drm/panel/panel-raydium-rm68200.c
+++ b/drivers/gpu/drm/panel/panel-raydium-rm68200.c
@@ -383,7 +383,8 @@ static int rm68200_probe(struct mipi_dsi_device *dsi)
 	ctx->supply = devm_regulator_get(dev, "power");
 	if (IS_ERR(ctx->supply)) {
 		ret = PTR_ERR(ctx->supply);
-		dev_err(dev, "cannot get regulator: %d\n", ret);
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "cannot get regulator: %d\n", ret);
 		return ret;
 	}
 
diff --git a/drivers/gpu/drm/panel/panel-rocktech-jh057n00900.c b/drivers/gpu/drm/panel/panel-rocktech-jh057n00900.c
new file mode 100644
index 000000000000..d88ea8da2ec2
--- /dev/null
+++ b/drivers/gpu/drm/panel/panel-rocktech-jh057n00900.c
@@ -0,0 +1,387 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Rockteck jh057n00900 5.5" MIPI-DSI panel driver
+ *
+ * Copyright (C) Purism SPC 2019
+ */
+
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_modes.h>
+#include <drm/drm_panel.h>
+#include <drm/drm_print.h>
+#include <linux/backlight.h>
+#include <linux/debugfs.h>
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/media-bus-format.h>
+#include <linux/module.h>
+#include <video/display_timing.h>
+#include <video/mipi_display.h>
+
+#define DRV_NAME "panel-rocktech-jh057n00900"
+
+/* Manufacturer specific Commands send via DSI */
+#define ST7703_CMD_ALL_PIXEL_OFF 0x22
+#define ST7703_CMD_ALL_PIXEL_ON	 0x23
+#define ST7703_CMD_SETDISP	 0xB2
+#define ST7703_CMD_SETRGBIF	 0xB3
+#define ST7703_CMD_SETCYC	 0xB4
+#define ST7703_CMD_SETBGP	 0xB5
+#define ST7703_CMD_SETVCOM	 0xB6
+#define ST7703_CMD_SETOTP	 0xB7
+#define ST7703_CMD_SETPOWER_EXT	 0xB8
+#define ST7703_CMD_SETEXTC	 0xB9
+#define ST7703_CMD_SETMIPI	 0xBA
+#define ST7703_CMD_SETVDC	 0xBC
+#define ST7703_CMD_SETSCR	 0xC0
+#define ST7703_CMD_SETPOWER	 0xC1
+#define ST7703_CMD_SETPANEL	 0xCC
+#define ST7703_CMD_SETGAMMA	 0xE0
+#define ST7703_CMD_SETEQ	 0xE3
+#define ST7703_CMD_SETGIP1	 0xE9
+#define ST7703_CMD_SETGIP2	 0xEA
+
+struct jh057n {
+	struct device *dev;
+	struct drm_panel panel;
+	struct gpio_desc *reset_gpio;
+	struct backlight_device *backlight;
+	bool prepared;
+
+	struct dentry *debugfs;
+};
+
+static inline struct jh057n *panel_to_jh057n(struct drm_panel *panel)
+{
+	return container_of(panel, struct jh057n, panel);
+}
+
+#define dsi_generic_write_seq(dsi, seq...) do {				\
+		static const u8 d[] = { seq };				\
+		int ret;						\
+		ret = mipi_dsi_generic_write(dsi, d, ARRAY_SIZE(d));	\
+		if (ret < 0)						\
+			return ret;					\
+	} while (0)
+
+static int jh057n_init_sequence(struct jh057n *ctx)
+{
+	struct mipi_dsi_device *dsi = to_mipi_dsi_device(ctx->dev);
+	struct device *dev = ctx->dev;
+	int ret;
+
+	/*
+	 * Init sequence was supplied by the panel vendor. Most of the commands
+	 * resemble the ST7703 but the number of parameters often don't match
+	 * so it's likely a clone.
+	 */
+	dsi_generic_write_seq(dsi, ST7703_CMD_SETEXTC,
+			      0xF1, 0x12, 0x83);
+	dsi_generic_write_seq(dsi, ST7703_CMD_SETRGBIF,
+			      0x10, 0x10, 0x05, 0x05, 0x03, 0xFF, 0x00, 0x00,
+			      0x00, 0x00);
+	dsi_generic_write_seq(dsi, ST7703_CMD_SETSCR,
+			      0x73, 0x73, 0x50, 0x50, 0x00, 0x00, 0x08, 0x70,
+			      0x00);
+	dsi_generic_write_seq(dsi, ST7703_CMD_SETVDC, 0x4E);
+	dsi_generic_write_seq(dsi, ST7703_CMD_SETPANEL, 0x0B);
+	dsi_generic_write_seq(dsi, ST7703_CMD_SETCYC, 0x80);
+	dsi_generic_write_seq(dsi, ST7703_CMD_SETDISP, 0xF0, 0x12, 0x30);
+	dsi_generic_write_seq(dsi, ST7703_CMD_SETEQ,
+			      0x07, 0x07, 0x0B, 0x0B, 0x03, 0x0B, 0x00, 0x00,
+			      0x00, 0x00, 0xFF, 0x00, 0xC0, 0x10);
+	dsi_generic_write_seq(dsi, ST7703_CMD_SETBGP, 0x08, 0x08);
+	msleep(20);
+
+	dsi_generic_write_seq(dsi, ST7703_CMD_SETVCOM, 0x3F, 0x3F);
+	dsi_generic_write_seq(dsi, 0xBF, 0x02, 0x11, 0x00);
+	dsi_generic_write_seq(dsi, ST7703_CMD_SETGIP1,
+			      0x82, 0x10, 0x06, 0x05, 0x9E, 0x0A, 0xA5, 0x12,
+			      0x31, 0x23, 0x37, 0x83, 0x04, 0xBC, 0x27, 0x38,
+			      0x0C, 0x00, 0x03, 0x00, 0x00, 0x00, 0x0C, 0x00,
+			      0x03, 0x00, 0x00, 0x00, 0x75, 0x75, 0x31, 0x88,
+			      0x88, 0x88, 0x88, 0x88, 0x88, 0x13, 0x88, 0x64,
+			      0x64, 0x20, 0x88, 0x88, 0x88, 0x88, 0x88, 0x88,
+			      0x02, 0x88, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			      0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00);
+	dsi_generic_write_seq(dsi, ST7703_CMD_SETGIP2,
+			      0x02, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			      0x00, 0x00, 0x00, 0x00, 0x02, 0x46, 0x02, 0x88,
+			      0x88, 0x88, 0x88, 0x88, 0x88, 0x64, 0x88, 0x13,
+			      0x57, 0x13, 0x88, 0x88, 0x88, 0x88, 0x88, 0x88,
+			      0x75, 0x88, 0x23, 0x14, 0x00, 0x00, 0x02, 0x00,
+			      0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			      0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x0A,
+			      0xA5, 0x00, 0x00, 0x00, 0x00);
+	dsi_generic_write_seq(dsi, ST7703_CMD_SETGAMMA,
+			      0x00, 0x09, 0x0E, 0x29, 0x2D, 0x3C, 0x41, 0x37,
+			      0x07, 0x0B, 0x0D, 0x10, 0x11, 0x0F, 0x10, 0x11,
+			      0x18, 0x00, 0x09, 0x0E, 0x29, 0x2D, 0x3C, 0x41,
+			      0x37, 0x07, 0x0B, 0x0D, 0x10, 0x11, 0x0F, 0x10,
+			      0x11, 0x18);
+	msleep(20);
+
+	ret = mipi_dsi_dcs_exit_sleep_mode(dsi);
+	if (ret < 0) {
+		DRM_DEV_ERROR(dev, "Failed to exit sleep mode\n");
+		return ret;
+	}
+	/* Panel is operational 120 msec after reset */
+	msleep(60);
+	ret = mipi_dsi_dcs_set_display_on(dsi);
+	if (ret)
+		return ret;
+
+	DRM_DEV_DEBUG_DRIVER(dev, "Panel init sequence done\n");
+	return 0;
+}
+
+static int jh057n_enable(struct drm_panel *panel)
+{
+	struct jh057n *ctx = panel_to_jh057n(panel);
+
+	return backlight_enable(ctx->backlight);
+}
+
+static int jh057n_disable(struct drm_panel *panel)
+{
+	struct jh057n *ctx = panel_to_jh057n(panel);
+
+	return backlight_disable(ctx->backlight);
+}
+
+static int jh057n_unprepare(struct drm_panel *panel)
+{
+	struct jh057n *ctx = panel_to_jh057n(panel);
+	struct mipi_dsi_device *dsi = to_mipi_dsi_device(ctx->dev);
+
+	if (!ctx->prepared)
+		return 0;
+
+	mipi_dsi_dcs_set_display_off(dsi);
+	ctx->prepared = false;
+
+	return 0;
+}
+
+static int jh057n_prepare(struct drm_panel *panel)
+{
+	struct jh057n *ctx = panel_to_jh057n(panel);
+	int ret;
+
+	if (ctx->prepared)
+		return 0;
+
+	DRM_DEV_DEBUG_DRIVER(ctx->dev, "Resetting the panel\n");
+	gpiod_set_value_cansleep(ctx->reset_gpio, 1);
+	usleep_range(20, 40);
+	gpiod_set_value_cansleep(ctx->reset_gpio, 0);
+	msleep(20);
+
+	ret = jh057n_init_sequence(ctx);
+	if (ret < 0) {
+		DRM_DEV_ERROR(ctx->dev, "Panel init sequence failed: %d\n",
+			      ret);
+		return ret;
+	}
+
+	ctx->prepared = true;
+
+	return 0;
+}
+
+static const struct drm_display_mode default_mode = {
+	.hdisplay    = 720,
+	.hsync_start = 720 + 90,
+	.hsync_end   = 720 + 90 + 20,
+	.htotal	     = 720 + 90 + 20 + 20,
+	.vdisplay    = 1440,
+	.vsync_start = 1440 + 20,
+	.vsync_end   = 1440 + 20 + 4,
+	.vtotal	     = 1440 + 20 + 4 + 12,
+	.vrefresh    = 60,
+	.clock	     = 75276,
+	.flags	     = DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC,
+	.width_mm    = 65,
+	.height_mm   = 130,
+};
+
+static int jh057n_get_modes(struct drm_panel *panel)
+{
+	struct jh057n *ctx = panel_to_jh057n(panel);
+	struct drm_display_mode *mode;
+
+	mode = drm_mode_duplicate(panel->drm, &default_mode);
+	if (!mode) {
+		DRM_DEV_ERROR(ctx->dev, "Failed to add mode %ux%u@%u\n",
+			      default_mode.hdisplay, default_mode.vdisplay,
+			      default_mode.vrefresh);
+		return -ENOMEM;
+	}
+
+	drm_mode_set_name(mode);
+
+	mode->type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED;
+	panel->connector->display_info.width_mm = mode->width_mm;
+	panel->connector->display_info.height_mm = mode->height_mm;
+	drm_mode_probed_add(panel->connector, mode);
+
+	return 1;
+}
+
+static const struct drm_panel_funcs jh057n_drm_funcs = {
+	.disable   = jh057n_disable,
+	.unprepare = jh057n_unprepare,
+	.prepare   = jh057n_prepare,
+	.enable	   = jh057n_enable,
+	.get_modes = jh057n_get_modes,
+};
+
+static int allpixelson_set(void *data, u64 val)
+{
+	struct jh057n *ctx = data;
+	struct mipi_dsi_device *dsi = to_mipi_dsi_device(ctx->dev);
+
+	DRM_DEV_DEBUG_DRIVER(ctx->dev, "Setting all pixels on\n");
+	dsi_generic_write_seq(dsi, ST7703_CMD_ALL_PIXEL_ON);
+	msleep(val * 1000);
+	/* Reset the panel to get video back */
+	drm_panel_disable(&ctx->panel);
+	drm_panel_unprepare(&ctx->panel);
+	drm_panel_prepare(&ctx->panel);
+	drm_panel_enable(&ctx->panel);
+
+	return 0;
+}
+
+DEFINE_SIMPLE_ATTRIBUTE(allpixelson_fops, NULL,
+			allpixelson_set, "%llu\n");
+
+static int jh057n_debugfs_init(struct jh057n *ctx)
+{
+	struct dentry *f;
+
+	ctx->debugfs = debugfs_create_dir(DRV_NAME, NULL);
+	if (!ctx->debugfs)
+		return -ENOMEM;
+
+	f = debugfs_create_file("allpixelson", 0600,
+				ctx->debugfs, ctx, &allpixelson_fops);
+	if (!f)
+		return -ENOMEM;
+
+	return 0;
+}
+
+static void jh057n_debugfs_remove(struct jh057n *ctx)
+{
+	debugfs_remove_recursive(ctx->debugfs);
+	ctx->debugfs = NULL;
+}
+
+static int jh057n_probe(struct mipi_dsi_device *dsi)
+{
+	struct device *dev = &dsi->dev;
+	struct jh057n *ctx;
+	int ret;
+
+	ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
+	if (!ctx)
+		return -ENOMEM;
+
+	ctx->reset_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_LOW);
+	if (IS_ERR(ctx->reset_gpio)) {
+		DRM_DEV_ERROR(dev, "cannot get reset gpio\n");
+		return PTR_ERR(ctx->reset_gpio);
+	}
+
+	mipi_dsi_set_drvdata(dsi, ctx);
+
+	ctx->dev = dev;
+
+	dsi->lanes = 4;
+	dsi->format = MIPI_DSI_FMT_RGB888;
+	dsi->mode_flags = MIPI_DSI_MODE_VIDEO |
+		MIPI_DSI_MODE_VIDEO_BURST | MIPI_DSI_MODE_VIDEO_SYNC_PULSE;
+
+	ctx->backlight = devm_of_find_backlight(dev);
+	if (IS_ERR(ctx->backlight))
+		return PTR_ERR(ctx->backlight);
+
+	drm_panel_init(&ctx->panel);
+	ctx->panel.dev = dev;
+	ctx->panel.funcs = &jh057n_drm_funcs;
+
+	drm_panel_add(&ctx->panel);
+
+	ret = mipi_dsi_attach(dsi);
+	if (ret < 0) {
+		DRM_DEV_ERROR(dev, "mipi_dsi_attach failed. Is host ready?\n");
+		drm_panel_remove(&ctx->panel);
+		return ret;
+	}
+
+	DRM_DEV_INFO(dev, "%ux%u@%u %ubpp dsi %udl - ready\n",
+		     default_mode.hdisplay, default_mode.vdisplay,
+		     default_mode.vrefresh,
+		     mipi_dsi_pixel_format_to_bpp(dsi->format), dsi->lanes);
+
+	jh057n_debugfs_init(ctx);
+	return 0;
+}
+
+static void jh057n_shutdown(struct mipi_dsi_device *dsi)
+{
+	struct jh057n *ctx = mipi_dsi_get_drvdata(dsi);
+	int ret;
+
+	ret = jh057n_unprepare(&ctx->panel);
+	if (ret < 0)
+		DRM_DEV_ERROR(&dsi->dev, "Failed to unprepare panel: %d\n",
+			      ret);
+
+	ret = jh057n_disable(&ctx->panel);
+	if (ret < 0)
+		DRM_DEV_ERROR(&dsi->dev, "Failed to disable panel: %d\n",
+			      ret);
+}
+
+static int jh057n_remove(struct mipi_dsi_device *dsi)
+{
+	struct jh057n *ctx = mipi_dsi_get_drvdata(dsi);
+	int ret;
+
+	jh057n_shutdown(dsi);
+
+	ret = mipi_dsi_detach(dsi);
+	if (ret < 0)
+		DRM_DEV_ERROR(&dsi->dev, "Failed to detach from DSI host: %d\n",
+			      ret);
+
+	drm_panel_remove(&ctx->panel);
+
+	jh057n_debugfs_remove(ctx);
+
+	return 0;
+}
+
+static const struct of_device_id jh057n_of_match[] = {
+	{ .compatible = "rocktech,jh057n00900" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, jh057n_of_match);
+
+static struct mipi_dsi_driver jh057n_driver = {
+	.probe	= jh057n_probe,
+	.remove = jh057n_remove,
+	.shutdown = jh057n_shutdown,
+	.driver = {
+		.name = DRV_NAME,
+		.of_match_table = jh057n_of_match,
+	},
+};
+module_mipi_dsi_driver(jh057n_driver);
+
+MODULE_AUTHOR("Guido Günther <agx@sigxcpu.org>");
+MODULE_DESCRIPTION("DRM driver for Rocktech JH057N00900 MIPI DSI panel");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/panel/panel-ronbo-rb070d30.c b/drivers/gpu/drm/panel/panel-ronbo-rb070d30.c
new file mode 100644
index 000000000000..3c15764f0c03
--- /dev/null
+++ b/drivers/gpu/drm/panel/panel-ronbo-rb070d30.c
@@ -0,0 +1,258 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (C) 2018-2019, Bridge Systems BV
+ * Copyright (C) 2018-2019, Bootlin
+ * Copyright (C) 2017, Free Electrons
+ *
+ * This file based on panel-ilitek-ili9881c.c
+ */
+
+#include <linux/backlight.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/errno.h>
+#include <linux/fb.h>
+#include <linux/kernel.h>
+#include <linux/media-bus-format.h>
+#include <linux/module.h>
+
+#include <linux/gpio/consumer.h>
+#include <linux/regulator/consumer.h>
+
+#include <drm/drm_connector.h>
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_modes.h>
+#include <drm/drm_panel.h>
+#include <drm/drm_print.h>
+
+struct rb070d30_panel {
+	struct drm_panel panel;
+	struct mipi_dsi_device *dsi;
+	struct backlight_device *backlight;
+	struct regulator *supply;
+
+	struct {
+		struct gpio_desc *power;
+		struct gpio_desc *reset;
+		struct gpio_desc *updn;
+		struct gpio_desc *shlr;
+	} gpios;
+};
+
+static inline struct rb070d30_panel *panel_to_rb070d30_panel(struct drm_panel *panel)
+{
+	return container_of(panel, struct rb070d30_panel, panel);
+}
+
+static int rb070d30_panel_prepare(struct drm_panel *panel)
+{
+	struct rb070d30_panel *ctx = panel_to_rb070d30_panel(panel);
+	int ret;
+
+	ret = regulator_enable(ctx->supply);
+	if (ret < 0) {
+		DRM_DEV_ERROR(&ctx->dsi->dev, "Failed to enable supply: %d\n", ret);
+		return ret;
+	}
+
+	msleep(20);
+	gpiod_set_value(ctx->gpios.power, 1);
+	msleep(20);
+	gpiod_set_value(ctx->gpios.reset, 1);
+	msleep(20);
+	return 0;
+}
+
+static int rb070d30_panel_unprepare(struct drm_panel *panel)
+{
+	struct rb070d30_panel *ctx = panel_to_rb070d30_panel(panel);
+
+	gpiod_set_value(ctx->gpios.reset, 0);
+	gpiod_set_value(ctx->gpios.power, 0);
+	regulator_disable(ctx->supply);
+
+	return 0;
+}
+
+static int rb070d30_panel_enable(struct drm_panel *panel)
+{
+	struct rb070d30_panel *ctx = panel_to_rb070d30_panel(panel);
+	int ret;
+
+	ret = mipi_dsi_dcs_exit_sleep_mode(ctx->dsi);
+	if (ret)
+		return ret;
+
+	ret = backlight_enable(ctx->backlight);
+	if (ret)
+		goto out;
+
+	return 0;
+
+out:
+	mipi_dsi_dcs_enter_sleep_mode(ctx->dsi);
+	return ret;
+}
+
+static int rb070d30_panel_disable(struct drm_panel *panel)
+{
+	struct rb070d30_panel *ctx = panel_to_rb070d30_panel(panel);
+
+	backlight_disable(ctx->backlight);
+	return mipi_dsi_dcs_enter_sleep_mode(ctx->dsi);
+}
+
+/* Default timings */
+static const struct drm_display_mode default_mode = {
+	.clock		= 51206,
+	.hdisplay	= 1024,
+	.hsync_start	= 1024 + 160,
+	.hsync_end	= 1024 + 160 + 80,
+	.htotal		= 1024 + 160 + 80 + 80,
+	.vdisplay	= 600,
+	.vsync_start	= 600 + 12,
+	.vsync_end	= 600 + 12 + 10,
+	.vtotal		= 600 + 12 + 10 + 13,
+	.vrefresh	= 60,
+
+	.width_mm	= 154,
+	.height_mm	= 85,
+};
+
+static int rb070d30_panel_get_modes(struct drm_panel *panel)
+{
+	struct drm_connector *connector = panel->connector;
+	struct rb070d30_panel *ctx = panel_to_rb070d30_panel(panel);
+	struct drm_display_mode *mode;
+	static const u32 bus_format = MEDIA_BUS_FMT_RGB888_1X24;
+
+	mode = drm_mode_duplicate(panel->drm, &default_mode);
+	if (!mode) {
+		DRM_DEV_ERROR(&ctx->dsi->dev,
+			      "Failed to add mode " DRM_MODE_FMT "\n",
+			      DRM_MODE_ARG(&default_mode));
+		return -EINVAL;
+	}
+
+	drm_mode_set_name(mode);
+
+	mode->type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED;
+	drm_mode_probed_add(connector, mode);
+
+	panel->connector->display_info.bpc = 8;
+	panel->connector->display_info.width_mm = mode->width_mm;
+	panel->connector->display_info.height_mm = mode->height_mm;
+	drm_display_info_set_bus_formats(&connector->display_info,
+					 &bus_format, 1);
+
+	return 1;
+}
+
+static const struct drm_panel_funcs rb070d30_panel_funcs = {
+	.get_modes	= rb070d30_panel_get_modes,
+	.prepare	= rb070d30_panel_prepare,
+	.enable		= rb070d30_panel_enable,
+	.disable	= rb070d30_panel_disable,
+	.unprepare	= rb070d30_panel_unprepare,
+};
+
+static int rb070d30_panel_dsi_probe(struct mipi_dsi_device *dsi)
+{
+	struct rb070d30_panel *ctx;
+	int ret;
+
+	ctx = devm_kzalloc(&dsi->dev, sizeof(*ctx), GFP_KERNEL);
+	if (!ctx)
+		return -ENOMEM;
+
+	ctx->supply = devm_regulator_get(&dsi->dev, "vcc-lcd");
+	if (IS_ERR(ctx->supply))
+		return PTR_ERR(ctx->supply);
+
+	mipi_dsi_set_drvdata(dsi, ctx);
+	ctx->dsi = dsi;
+
+	drm_panel_init(&ctx->panel);
+	ctx->panel.dev = &dsi->dev;
+	ctx->panel.funcs = &rb070d30_panel_funcs;
+
+	ctx->gpios.reset = devm_gpiod_get(&dsi->dev, "reset", GPIOD_OUT_LOW);
+	if (IS_ERR(ctx->gpios.reset)) {
+		DRM_DEV_ERROR(&dsi->dev, "Couldn't get our reset GPIO\n");
+		return PTR_ERR(ctx->gpios.reset);
+	}
+
+	ctx->gpios.power = devm_gpiod_get(&dsi->dev, "power", GPIOD_OUT_LOW);
+	if (IS_ERR(ctx->gpios.power)) {
+		DRM_DEV_ERROR(&dsi->dev, "Couldn't get our power GPIO\n");
+		return PTR_ERR(ctx->gpios.power);
+	}
+
+	/*
+	 * We don't change the state of that GPIO later on but we need
+	 * to force it into a low state.
+	 */
+	ctx->gpios.updn = devm_gpiod_get(&dsi->dev, "updn", GPIOD_OUT_LOW);
+	if (IS_ERR(ctx->gpios.updn)) {
+		DRM_DEV_ERROR(&dsi->dev, "Couldn't get our updn GPIO\n");
+		return PTR_ERR(ctx->gpios.updn);
+	}
+
+	/*
+	 * We don't change the state of that GPIO later on but we need
+	 * to force it into a low state.
+	 */
+	ctx->gpios.shlr = devm_gpiod_get(&dsi->dev, "shlr", GPIOD_OUT_LOW);
+	if (IS_ERR(ctx->gpios.shlr)) {
+		DRM_DEV_ERROR(&dsi->dev, "Couldn't get our shlr GPIO\n");
+		return PTR_ERR(ctx->gpios.shlr);
+	}
+
+	ctx->backlight = devm_of_find_backlight(&dsi->dev);
+	if (IS_ERR(ctx->backlight)) {
+		DRM_DEV_ERROR(&dsi->dev, "Couldn't get our backlight\n");
+		return PTR_ERR(ctx->backlight);
+	}
+
+	ret = drm_panel_add(&ctx->panel);
+	if (ret < 0)
+		return ret;
+
+	dsi->mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | MIPI_DSI_MODE_LPM;
+	dsi->format = MIPI_DSI_FMT_RGB888;
+	dsi->lanes = 4;
+
+	return mipi_dsi_attach(dsi);
+}
+
+static int rb070d30_panel_dsi_remove(struct mipi_dsi_device *dsi)
+{
+	struct rb070d30_panel *ctx = mipi_dsi_get_drvdata(dsi);
+
+	mipi_dsi_detach(dsi);
+	drm_panel_remove(&ctx->panel);
+
+	return 0;
+}
+
+static const struct of_device_id rb070d30_panel_of_match[] = {
+	{ .compatible = "ronbo,rb070d30" },
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, rb070d30_panel_of_match);
+
+static struct mipi_dsi_driver rb070d30_panel_driver = {
+	.probe = rb070d30_panel_dsi_probe,
+	.remove = rb070d30_panel_dsi_remove,
+	.driver = {
+		.name = "panel-ronbo-rb070d30",
+		.of_match_table	= rb070d30_panel_of_match,
+	},
+};
+module_mipi_dsi_driver(rb070d30_panel_driver);
+
+MODULE_AUTHOR("Boris Brezillon <boris.brezillon@bootlin.com>");
+MODULE_AUTHOR("Konstantin Sudakov <k.sudakov@integrasources.com>");
+MODULE_DESCRIPTION("Ronbo RB070D30 Panel Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/panel/panel-samsung-s6d16d0.c b/drivers/gpu/drm/panel/panel-samsung-s6d16d0.c
index 33c22ee036f8..f75bef24e050 100644
--- a/drivers/gpu/drm/panel/panel-samsung-s6d16d0.c
+++ b/drivers/gpu/drm/panel/panel-samsung-s6d16d0.c
@@ -148,9 +148,6 @@ static int s6d16d0_get_modes(struct drm_panel *panel)
 	struct drm_connector *connector = panel->connector;
 	struct drm_display_mode *mode;
 
-	strncpy(connector->display_info.name, "Samsung S6D16D0\0",
-		DRM_DISPLAY_INFO_LEN);
-
 	mode = drm_mode_duplicate(panel->drm, &samsung_s6d16d0_mode);
 	if (!mode) {
 		DRM_ERROR("bad mode or failed to add mode\n");
diff --git a/drivers/gpu/drm/panel/panel-seiko-43wvf1g.c b/drivers/gpu/drm/panel/panel-seiko-43wvf1g.c
index 2d99e28ff117..bdcc5d80823d 100644
--- a/drivers/gpu/drm/panel/panel-seiko-43wvf1g.c
+++ b/drivers/gpu/drm/panel/panel-seiko-43wvf1g.c
@@ -328,7 +328,7 @@ static const struct seiko_panel_desc seiko_43wvf1g = {
 		.height = 57,
 	},
 	.bus_format = MEDIA_BUS_FMT_RGB888_1X24,
-	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_NEGEDGE,
+	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE,
 };
 
 static const struct of_device_id platform_of_match[] = {
diff --git a/drivers/gpu/drm/panel/panel-simple.c b/drivers/gpu/drm/panel/panel-simple.c
index 9e8218f6a3f2..569be4efd8d1 100644
--- a/drivers/gpu/drm/panel/panel-simple.c
+++ b/drivers/gpu/drm/panel/panel-simple.c
@@ -914,7 +914,7 @@ static const struct panel_desc cdtech_s043wq26h_ct7 = {
 		.width = 95,
 		.height = 54,
 	},
-	.bus_flags = DRM_BUS_FLAG_PIXDATA_POSEDGE,
+	.bus_flags = DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE,
 };
 
 static const struct drm_display_mode cdtech_s070wv95_ct16_mode = {
@@ -1034,7 +1034,7 @@ static const struct panel_desc dataimage_scf0700c48ggu18 = {
 		.height = 91,
 	},
 	.bus_format = MEDIA_BUS_FMT_RGB888_1X24,
-	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_POSEDGE,
+	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE,
 };
 
 static const struct display_timing dlc_dlc0700yzg_1_timing = {
@@ -1119,7 +1119,7 @@ static const struct panel_desc edt_et057090dhu = {
 		.height = 86,
 	},
 	.bus_format = MEDIA_BUS_FMT_RGB666_1X18,
-	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_NEGEDGE,
+	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE,
 };
 
 static const struct drm_display_mode edt_etm0700g0dh6_mode = {
@@ -1145,7 +1145,7 @@ static const struct panel_desc edt_etm0700g0dh6 = {
 		.height = 91,
 	},
 	.bus_format = MEDIA_BUS_FMT_RGB666_1X18,
-	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_NEGEDGE,
+	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE,
 };
 
 static const struct panel_desc edt_etm0700g0bdh6 = {
@@ -1157,7 +1157,7 @@ static const struct panel_desc edt_etm0700g0bdh6 = {
 		.height = 91,
 	},
 	.bus_format = MEDIA_BUS_FMT_RGB666_1X18,
-	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_POSEDGE,
+	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE,
 };
 
 static const struct drm_display_mode foxlink_fl500wvr00_a0t_mode = {
@@ -1311,7 +1311,7 @@ static const struct panel_desc innolux_at043tn24 = {
 		.height = 54,
 	},
 	.bus_format = MEDIA_BUS_FMT_RGB888_1X24,
-	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_POSEDGE,
+	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE,
 };
 
 static const struct drm_display_mode innolux_at070tn92_mode = {
@@ -1818,7 +1818,7 @@ static const struct panel_desc nec_nl4827hc19_05b = {
 		.height = 54,
 	},
 	.bus_format = MEDIA_BUS_FMT_RGB888_1X24,
-	.bus_flags = DRM_BUS_FLAG_PIXDATA_POSEDGE,
+	.bus_flags = DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE,
 };
 
 static const struct drm_display_mode netron_dy_e231732_mode = {
@@ -1867,8 +1867,8 @@ static const struct panel_desc newhaven_nhd_43_480272ef_atxl = {
 		.height = 54,
 	},
 	.bus_format = MEDIA_BUS_FMT_RGB888_1X24,
-	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_POSEDGE |
-		     DRM_BUS_FLAG_SYNC_POSEDGE,
+	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE |
+		     DRM_BUS_FLAG_SYNC_DRIVE_POSEDGE,
 };
 
 static const struct display_timing nlt_nl192108ac18_02d_timing = {
@@ -2029,7 +2029,33 @@ static const struct panel_desc ortustech_com43h4m85ulc = {
 		.height = 93,
 	},
 	.bus_format = MEDIA_BUS_FMT_RGB888_1X24,
-	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_POSEDGE,
+	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE,
+};
+
+static const struct drm_display_mode osddisplays_osd070t1718_19ts_mode  = {
+	.clock = 33000,
+	.hdisplay = 800,
+	.hsync_start = 800 + 210,
+	.hsync_end = 800 + 210 + 30,
+	.htotal = 800 + 210 + 30 + 16,
+	.vdisplay = 480,
+	.vsync_start = 480 + 22,
+	.vsync_end = 480 + 22 + 13,
+	.vtotal = 480 + 22 + 13 + 10,
+	.vrefresh = 60,
+	.flags = DRM_MODE_FLAG_NVSYNC | DRM_MODE_FLAG_NHSYNC,
+};
+
+static const struct panel_desc osddisplays_osd070t1718_19ts = {
+	.modes = &osddisplays_osd070t1718_19ts_mode,
+	.num_modes = 1,
+	.bpc = 8,
+	.size = {
+		.width = 152,
+		.height = 91,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X24,
+	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE,
 };
 
 static const struct drm_display_mode pda_91_00156_a0_mode = {
@@ -2398,7 +2424,7 @@ static const struct panel_desc toshiba_lt089ac29000 = {
 		.height = 116,
 	},
 	.bus_format = MEDIA_BUS_FMT_RGB888_1X24,
-	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_POSEDGE,
+	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE,
 };
 
 static const struct drm_display_mode tpk_f07a_0102_mode = {
@@ -2421,7 +2447,7 @@ static const struct panel_desc tpk_f07a_0102 = {
 		.width = 152,
 		.height = 91,
 	},
-	.bus_flags = DRM_BUS_FLAG_PIXDATA_POSEDGE,
+	.bus_flags = DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE,
 };
 
 static const struct drm_display_mode tpk_f10a_0102_mode = {
@@ -2737,6 +2763,9 @@ static const struct of_device_id platform_of_match[] = {
 		.compatible = "ortustech,com43h4m85ulc",
 		.data = &ortustech_com43h4m85ulc,
 	}, {
+		.compatible = "osddisplays,osd070t1718-19ts",
+		.data = &osddisplays_osd070t1718_19ts,
+	}, {
 		.compatible = "pda,91-00156-a0",
 		.data = &pda_91_00156_a0,
 	}, {
@@ -2996,6 +3025,34 @@ static const struct panel_desc_dsi panasonic_vvx10f004b00 = {
 	.lanes = 4,
 };
 
+static const struct drm_display_mode lg_acx467akm_7_mode = {
+	.clock = 150000,
+	.hdisplay = 1080,
+	.hsync_start = 1080 + 2,
+	.hsync_end = 1080 + 2 + 2,
+	.htotal = 1080 + 2 + 2 + 2,
+	.vdisplay = 1920,
+	.vsync_start = 1920 + 2,
+	.vsync_end = 1920 + 2 + 2,
+	.vtotal = 1920 + 2 + 2 + 2,
+	.vrefresh = 60,
+};
+
+static const struct panel_desc_dsi lg_acx467akm_7 = {
+	.desc = {
+		.modes = &lg_acx467akm_7_mode,
+		.num_modes = 1,
+		.bpc = 8,
+		.size = {
+			.width = 62,
+			.height = 110,
+		},
+	},
+	.flags = 0,
+	.format = MIPI_DSI_FMT_RGB888,
+	.lanes = 4,
+};
+
 static const struct of_device_id dsi_of_match[] = {
 	{
 		.compatible = "auo,b080uan01",
@@ -3013,6 +3070,9 @@ static const struct of_device_id dsi_of_match[] = {
 		.compatible = "panasonic,vvx10f004b00",
 		.data = &panasonic_vvx10f004b00
 	}, {
+		.compatible = "lg,acx467akm-7",
+		.data = &lg_acx467akm_7
+	}, {
 		/* sentinel */
 	}
 };
diff --git a/drivers/gpu/drm/panel/panel-tpo-tpg110.c b/drivers/gpu/drm/panel/panel-tpo-tpg110.c
index 5a9f8f4d5d24..71591e5f5938 100644
--- a/drivers/gpu/drm/panel/panel-tpo-tpg110.c
+++ b/drivers/gpu/drm/panel/panel-tpo-tpg110.c
@@ -118,7 +118,7 @@ static const struct tpg110_panel_mode tpg110_modes[] = {
 			.vtotal = 480 + 10 + 1 + 35,
 			.vrefresh = 60,
 		},
-		.bus_flags = DRM_BUS_FLAG_PIXDATA_POSEDGE,
+		.bus_flags = DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE,
 	},
 	{
 		.name = "640x480 RGB",
@@ -135,7 +135,7 @@ static const struct tpg110_panel_mode tpg110_modes[] = {
 			.vtotal = 480 + 18 + 1 + 27,
 			.vrefresh = 60,
 		},
-		.bus_flags = DRM_BUS_FLAG_PIXDATA_POSEDGE,
+		.bus_flags = DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE,
 	},
 	{
 		.name = "480x272 RGB",
@@ -152,7 +152,7 @@ static const struct tpg110_panel_mode tpg110_modes[] = {
 			.vtotal = 272 + 2 + 1 + 12,
 			.vrefresh = 60,
 		},
-		.bus_flags = DRM_BUS_FLAG_PIXDATA_POSEDGE,
+		.bus_flags = DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE,
 	},
 	{
 		.name = "480x640 RGB",
@@ -169,7 +169,7 @@ static const struct tpg110_panel_mode tpg110_modes[] = {
 			.vtotal = 640 + 4 + 1 + 8,
 			.vrefresh = 60,
 		},
-		.bus_flags = DRM_BUS_FLAG_PIXDATA_POSEDGE,
+		.bus_flags = DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE,
 	},
 	{
 		.name = "400x240 RGB",
@@ -186,7 +186,7 @@ static const struct tpg110_panel_mode tpg110_modes[] = {
 			.vtotal = 240 + 2 + 1 + 20,
 			.vrefresh = 60,
 		},
-		.bus_flags = DRM_BUS_FLAG_PIXDATA_POSEDGE,
+		.bus_flags = DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE,
 	},
 };
 
@@ -390,8 +390,6 @@ static int tpg110_get_modes(struct drm_panel *panel)
 	struct tpg110 *tpg = to_tpg110(panel);
 	struct drm_display_mode *mode;
 
-	strncpy(connector->display_info.name, tpg->panel_mode->name,
-		DRM_DISPLAY_INFO_LEN);
 	connector->display_info.width_mm = tpg->width;
 	connector->display_info.height_mm = tpg->height;
 	connector->display_info.bus_flags = tpg->panel_mode->bus_flags;
diff --git a/drivers/gpu/drm/panfrost/Kconfig b/drivers/gpu/drm/panfrost/Kconfig
new file mode 100644
index 000000000000..591611dc4e34
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/Kconfig
@@ -0,0 +1,14 @@
+# SPDX-License-Identifier: GPL-2.0
+
+config DRM_PANFROST
+	tristate "Panfrost (DRM support for ARM Mali Midgard/Bifrost GPUs)"
+	depends on DRM
+	depends on ARM || ARM64 || (COMPILE_TEST && !GENERIC_ATOMIC64)
+	depends on MMU
+	select DRM_SCHED
+	select IOMMU_SUPPORT
+	select IOMMU_IO_PGTABLE_LPAE
+	select DRM_GEM_SHMEM_HELPER
+	help
+	  DRM driver for ARM Mali Midgard (T6xx, T7xx, T8xx) and
+	  Bifrost (G3x, G5x, G7x) GPUs.
diff --git a/drivers/gpu/drm/panfrost/Makefile b/drivers/gpu/drm/panfrost/Makefile
new file mode 100644
index 000000000000..6de72d13c58f
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/Makefile
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-2.0
+
+panfrost-y := \
+	panfrost_drv.o \
+	panfrost_device.o \
+	panfrost_devfreq.o \
+	panfrost_gem.o \
+	panfrost_gpu.o \
+	panfrost_job.o \
+	panfrost_mmu.o
+
+obj-$(CONFIG_DRM_PANFROST) += panfrost.o
diff --git a/drivers/gpu/drm/panfrost/TODO b/drivers/gpu/drm/panfrost/TODO
new file mode 100644
index 000000000000..c2e44add37d8
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/TODO
@@ -0,0 +1,27 @@
+- Thermal support.
+
+- Bifrost support:
+  - DT bindings (Neil, WIP)
+  - MMU page table format and address space setup
+  - Bifrost specific feature and issue handling
+  - Coherent DMA support
+
+- Support for 2MB pages. The io-pgtable code already supports this. Finishing
+  support involves either copying or adapting the iommu API to handle passing
+  aligned addresses and sizes to the io-pgtable code.
+
+- Per FD address space support. The h/w supports multiple addresses spaces.
+  The hard part is handling when more address spaces are needed than what
+  the h/w provides.
+
+- Support pinning pages on demand (GPU page faults).
+
+- Support userspace controlled GPU virtual addresses. Needed for Vulkan. (Tomeu)
+
+- Support for madvise and a shrinker.
+
+- Compute job support. So called 'compute only' jobs need to be plumbed up to
+  userspace.
+
+- Performance counter support. (Boris)
+
diff --git a/drivers/gpu/drm/panfrost/panfrost_devfreq.c b/drivers/gpu/drm/panfrost/panfrost_devfreq.c
new file mode 100644
index 000000000000..238bd1d89d43
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/panfrost_devfreq.c
@@ -0,0 +1,219 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright 2019 Collabora ltd. */
+#include <linux/devfreq.h>
+#include <linux/platform_device.h>
+#include <linux/pm_opp.h>
+#include <linux/clk.h>
+#include <linux/regulator/consumer.h>
+
+#include "panfrost_device.h"
+#include "panfrost_devfreq.h"
+#include "panfrost_features.h"
+#include "panfrost_issues.h"
+#include "panfrost_gpu.h"
+#include "panfrost_regs.h"
+
+static void panfrost_devfreq_update_utilization(struct panfrost_device *pfdev, int slot);
+
+static int panfrost_devfreq_target(struct device *dev, unsigned long *freq,
+				   u32 flags)
+{
+	struct panfrost_device *pfdev = platform_get_drvdata(to_platform_device(dev));
+	struct dev_pm_opp *opp;
+	unsigned long old_clk_rate = pfdev->devfreq.cur_freq;
+	unsigned long target_volt, target_rate;
+	int err;
+
+	opp = devfreq_recommended_opp(dev, freq, flags);
+	if (IS_ERR(opp))
+		return PTR_ERR(opp);
+
+	target_rate = dev_pm_opp_get_freq(opp);
+	target_volt = dev_pm_opp_get_voltage(opp);
+	dev_pm_opp_put(opp);
+
+	if (old_clk_rate == target_rate)
+		return 0;
+
+	/*
+	 * If frequency scaling from low to high, adjust voltage first.
+	 * If frequency scaling from high to low, adjust frequency first.
+	 */
+	if (old_clk_rate < target_rate) {
+		err = regulator_set_voltage(pfdev->regulator, target_volt,
+					    target_volt);
+		if (err) {
+			dev_err(dev, "Cannot set voltage %lu uV\n",
+				target_volt);
+			return err;
+		}
+	}
+
+	err = clk_set_rate(pfdev->clock, target_rate);
+	if (err) {
+		dev_err(dev, "Cannot set frequency %lu (%d)\n", target_rate,
+			err);
+		regulator_set_voltage(pfdev->regulator, pfdev->devfreq.cur_volt,
+				      pfdev->devfreq.cur_volt);
+		return err;
+	}
+
+	if (old_clk_rate > target_rate) {
+		err = regulator_set_voltage(pfdev->regulator, target_volt,
+					    target_volt);
+		if (err)
+			dev_err(dev, "Cannot set voltage %lu uV\n", target_volt);
+	}
+
+	pfdev->devfreq.cur_freq = target_rate;
+	pfdev->devfreq.cur_volt = target_volt;
+
+	return 0;
+}
+
+static void panfrost_devfreq_reset(struct panfrost_device *pfdev)
+{
+	ktime_t now = ktime_get();
+	int i;
+
+	for (i = 0; i < NUM_JOB_SLOTS; i++) {
+		pfdev->devfreq.slot[i].busy_time = 0;
+		pfdev->devfreq.slot[i].idle_time = 0;
+		pfdev->devfreq.slot[i].time_last_update = now;
+	}
+}
+
+static int panfrost_devfreq_get_dev_status(struct device *dev,
+					   struct devfreq_dev_status *status)
+{
+	struct panfrost_device *pfdev = platform_get_drvdata(to_platform_device(dev));
+	int i;
+
+	for (i = 0; i < NUM_JOB_SLOTS; i++) {
+		panfrost_devfreq_update_utilization(pfdev, i);
+	}
+
+	status->current_frequency = clk_get_rate(pfdev->clock);
+	status->total_time = ktime_to_ns(ktime_add(pfdev->devfreq.slot[0].busy_time,
+						   pfdev->devfreq.slot[0].idle_time));
+
+	status->busy_time = 0;
+	for (i = 0; i < NUM_JOB_SLOTS; i++) {
+		status->busy_time += ktime_to_ns(pfdev->devfreq.slot[i].busy_time);
+	}
+
+	/* We're scheduling only to one core atm, so don't divide for now */
+	/* status->busy_time /= NUM_JOB_SLOTS; */
+
+	panfrost_devfreq_reset(pfdev);
+
+	dev_dbg(pfdev->dev, "busy %lu total %lu %lu %% freq %lu MHz\n", status->busy_time,
+		status->total_time,
+		status->busy_time / (status->total_time / 100),
+		status->current_frequency / 1000 / 1000);
+
+	return 0;
+}
+
+static int panfrost_devfreq_get_cur_freq(struct device *dev, unsigned long *freq)
+{
+	struct panfrost_device *pfdev = platform_get_drvdata(to_platform_device(dev));
+
+	*freq = pfdev->devfreq.cur_freq;
+
+	return 0;
+}
+
+static struct devfreq_dev_profile panfrost_devfreq_profile = {
+	.polling_ms = 50, /* ~3 frames */
+	.target = panfrost_devfreq_target,
+	.get_dev_status = panfrost_devfreq_get_dev_status,
+	.get_cur_freq = panfrost_devfreq_get_cur_freq,
+};
+
+int panfrost_devfreq_init(struct panfrost_device *pfdev)
+{
+	int ret;
+	struct dev_pm_opp *opp;
+
+	if (!pfdev->regulator)
+		return 0;
+
+	ret = dev_pm_opp_of_add_table(&pfdev->pdev->dev);
+	if (ret == -ENODEV) /* Optional, continue without devfreq */
+		return 0;
+
+	panfrost_devfreq_reset(pfdev);
+
+	pfdev->devfreq.cur_freq = clk_get_rate(pfdev->clock);
+
+	opp = devfreq_recommended_opp(&pfdev->pdev->dev, &pfdev->devfreq.cur_freq, 0);
+	if (IS_ERR(opp))
+		return PTR_ERR(opp);
+
+	panfrost_devfreq_profile.initial_freq = pfdev->devfreq.cur_freq;
+	dev_pm_opp_put(opp);
+
+	pfdev->devfreq.devfreq = devm_devfreq_add_device(&pfdev->pdev->dev,
+			&panfrost_devfreq_profile, "simple_ondemand", NULL);
+	if (IS_ERR(pfdev->devfreq.devfreq)) {
+		DRM_DEV_ERROR(&pfdev->pdev->dev, "Couldn't initialize GPU devfreq\n");
+		ret = PTR_ERR(pfdev->devfreq.devfreq);
+		pfdev->devfreq.devfreq = NULL;
+		return ret;
+	}
+
+	return 0;
+}
+
+void panfrost_devfreq_resume(struct panfrost_device *pfdev)
+{
+	int i;
+
+	if (!pfdev->devfreq.devfreq)
+		return;
+
+	panfrost_devfreq_reset(pfdev);
+	for (i = 0; i < NUM_JOB_SLOTS; i++)
+		pfdev->devfreq.slot[i].busy = false;
+
+	devfreq_resume_device(pfdev->devfreq.devfreq);
+}
+
+void panfrost_devfreq_suspend(struct panfrost_device *pfdev)
+{
+	if (!pfdev->devfreq.devfreq)
+		return;
+
+	devfreq_suspend_device(pfdev->devfreq.devfreq);
+}
+
+static void panfrost_devfreq_update_utilization(struct panfrost_device *pfdev, int slot)
+{
+	struct panfrost_devfreq_slot *devfreq_slot = &pfdev->devfreq.slot[slot];
+	ktime_t now;
+	ktime_t last;
+
+	if (!pfdev->devfreq.devfreq)
+		return;
+
+	now = ktime_get();
+	last = pfdev->devfreq.slot[slot].time_last_update;
+
+	/* If we last recorded a transition to busy, we have been idle since */
+	if (devfreq_slot->busy)
+		pfdev->devfreq.slot[slot].busy_time += ktime_sub(now, last);
+	else
+		pfdev->devfreq.slot[slot].idle_time += ktime_sub(now, last);
+
+	pfdev->devfreq.slot[slot].time_last_update = now;
+}
+
+/* The job scheduler is expected to call this at every transition busy <-> idle */
+void panfrost_devfreq_record_transition(struct panfrost_device *pfdev, int slot)
+{
+	struct panfrost_devfreq_slot *devfreq_slot = &pfdev->devfreq.slot[slot];
+
+	panfrost_devfreq_update_utilization(pfdev, slot);
+	devfreq_slot->busy = !devfreq_slot->busy;
+}
diff --git a/drivers/gpu/drm/panfrost/panfrost_devfreq.h b/drivers/gpu/drm/panfrost/panfrost_devfreq.h
new file mode 100644
index 000000000000..eb999531ed90
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/panfrost_devfreq.h
@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright 2019 Collabora ltd. */
+
+#ifndef __PANFROST_DEVFREQ_H__
+#define __PANFROST_DEVFREQ_H__
+
+int panfrost_devfreq_init(struct panfrost_device *pfdev);
+
+void panfrost_devfreq_resume(struct panfrost_device *pfdev);
+void panfrost_devfreq_suspend(struct panfrost_device *pfdev);
+
+void panfrost_devfreq_record_transition(struct panfrost_device *pfdev, int slot);
+
+#endif /* __PANFROST_DEVFREQ_H__ */
diff --git a/drivers/gpu/drm/panfrost/panfrost_device.c b/drivers/gpu/drm/panfrost/panfrost_device.c
new file mode 100644
index 000000000000..970f669c6d29
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/panfrost_device.c
@@ -0,0 +1,253 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */
+/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
+
+#include <linux/clk.h>
+#include <linux/reset.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/regulator/consumer.h>
+
+#include "panfrost_device.h"
+#include "panfrost_devfreq.h"
+#include "panfrost_features.h"
+#include "panfrost_gpu.h"
+#include "panfrost_job.h"
+#include "panfrost_mmu.h"
+
+static int panfrost_reset_init(struct panfrost_device *pfdev)
+{
+	int err;
+
+	pfdev->rstc = devm_reset_control_array_get(pfdev->dev, false, true);
+	if (IS_ERR(pfdev->rstc)) {
+		dev_err(pfdev->dev, "get reset failed %ld\n", PTR_ERR(pfdev->rstc));
+		return PTR_ERR(pfdev->rstc);
+	}
+
+	err = reset_control_deassert(pfdev->rstc);
+	if (err)
+		return err;
+
+	return 0;
+}
+
+static void panfrost_reset_fini(struct panfrost_device *pfdev)
+{
+	reset_control_assert(pfdev->rstc);
+}
+
+static int panfrost_clk_init(struct panfrost_device *pfdev)
+{
+	int err;
+	unsigned long rate;
+
+	pfdev->clock = devm_clk_get(pfdev->dev, NULL);
+	if (IS_ERR(pfdev->clock)) {
+		dev_err(pfdev->dev, "get clock failed %ld\n", PTR_ERR(pfdev->clock));
+		return PTR_ERR(pfdev->clock);
+	}
+
+	rate = clk_get_rate(pfdev->clock);
+	dev_info(pfdev->dev, "clock rate = %lu\n", rate);
+
+	err = clk_prepare_enable(pfdev->clock);
+	if (err)
+		return err;
+
+	return 0;
+}
+
+static void panfrost_clk_fini(struct panfrost_device *pfdev)
+{
+	clk_disable_unprepare(pfdev->clock);
+}
+
+static int panfrost_regulator_init(struct panfrost_device *pfdev)
+{
+	int ret;
+
+	pfdev->regulator = devm_regulator_get_optional(pfdev->dev, "mali");
+	if (IS_ERR(pfdev->regulator)) {
+		ret = PTR_ERR(pfdev->regulator);
+		pfdev->regulator = NULL;
+		if (ret == -ENODEV)
+			return 0;
+		dev_err(pfdev->dev, "failed to get regulator: %d\n", ret);
+		return ret;
+	}
+
+	ret = regulator_enable(pfdev->regulator);
+	if (ret < 0) {
+		dev_err(pfdev->dev, "failed to enable regulator: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void panfrost_regulator_fini(struct panfrost_device *pfdev)
+{
+	if (pfdev->regulator)
+		regulator_disable(pfdev->regulator);
+}
+
+int panfrost_device_init(struct panfrost_device *pfdev)
+{
+	int err;
+	struct resource *res;
+
+	mutex_init(&pfdev->sched_lock);
+	mutex_init(&pfdev->reset_lock);
+	INIT_LIST_HEAD(&pfdev->scheduled_jobs);
+
+	spin_lock_init(&pfdev->hwaccess_lock);
+
+	err = panfrost_clk_init(pfdev);
+	if (err) {
+		dev_err(pfdev->dev, "clk init failed %d\n", err);
+		return err;
+	}
+
+	err = panfrost_regulator_init(pfdev);
+	if (err) {
+		dev_err(pfdev->dev, "regulator init failed %d\n", err);
+		goto err_out0;
+	}
+
+	err = panfrost_reset_init(pfdev);
+	if (err) {
+		dev_err(pfdev->dev, "reset init failed %d\n", err);
+		goto err_out1;
+	}
+
+	res = platform_get_resource(pfdev->pdev, IORESOURCE_MEM, 0);
+	pfdev->iomem = devm_ioremap_resource(pfdev->dev, res);
+	if (IS_ERR(pfdev->iomem)) {
+		dev_err(pfdev->dev, "failed to ioremap iomem\n");
+		err = PTR_ERR(pfdev->iomem);
+		goto err_out2;
+	}
+
+	err = panfrost_gpu_init(pfdev);
+	if (err)
+		goto err_out2;
+
+	err = panfrost_mmu_init(pfdev);
+	if (err)
+		goto err_out3;
+
+	err = panfrost_job_init(pfdev);
+	if (err)
+		goto err_out4;
+
+	/* runtime PM will wake us up later */
+	panfrost_gpu_power_off(pfdev);
+
+	pm_runtime_set_active(pfdev->dev);
+	pm_runtime_get_sync(pfdev->dev);
+	pm_runtime_mark_last_busy(pfdev->dev);
+	pm_runtime_put_autosuspend(pfdev->dev);
+
+	return 0;
+err_out4:
+	panfrost_mmu_fini(pfdev);
+err_out3:
+	panfrost_gpu_fini(pfdev);
+err_out2:
+	panfrost_reset_fini(pfdev);
+err_out1:
+	panfrost_regulator_fini(pfdev);
+err_out0:
+	panfrost_clk_fini(pfdev);
+	return err;
+}
+
+void panfrost_device_fini(struct panfrost_device *pfdev)
+{
+	panfrost_regulator_fini(pfdev);
+	panfrost_clk_fini(pfdev);
+}
+
+const char *panfrost_exception_name(struct panfrost_device *pfdev, u32 exception_code)
+{
+	switch (exception_code) {
+		/* Non-Fault Status code */
+	case 0x00: return "NOT_STARTED/IDLE/OK";
+	case 0x01: return "DONE";
+	case 0x02: return "INTERRUPTED";
+	case 0x03: return "STOPPED";
+	case 0x04: return "TERMINATED";
+	case 0x08: return "ACTIVE";
+		/* Job exceptions */
+	case 0x40: return "JOB_CONFIG_FAULT";
+	case 0x41: return "JOB_POWER_FAULT";
+	case 0x42: return "JOB_READ_FAULT";
+	case 0x43: return "JOB_WRITE_FAULT";
+	case 0x44: return "JOB_AFFINITY_FAULT";
+	case 0x48: return "JOB_BUS_FAULT";
+	case 0x50: return "INSTR_INVALID_PC";
+	case 0x51: return "INSTR_INVALID_ENC";
+	case 0x52: return "INSTR_TYPE_MISMATCH";
+	case 0x53: return "INSTR_OPERAND_FAULT";
+	case 0x54: return "INSTR_TLS_FAULT";
+	case 0x55: return "INSTR_BARRIER_FAULT";
+	case 0x56: return "INSTR_ALIGN_FAULT";
+	case 0x58: return "DATA_INVALID_FAULT";
+	case 0x59: return "TILE_RANGE_FAULT";
+	case 0x5A: return "ADDR_RANGE_FAULT";
+	case 0x60: return "OUT_OF_MEMORY";
+		/* GPU exceptions */
+	case 0x80: return "DELAYED_BUS_FAULT";
+	case 0x88: return "SHAREABILITY_FAULT";
+		/* MMU exceptions */
+	case 0xC1: return "TRANSLATION_FAULT_LEVEL1";
+	case 0xC2: return "TRANSLATION_FAULT_LEVEL2";
+	case 0xC3: return "TRANSLATION_FAULT_LEVEL3";
+	case 0xC4: return "TRANSLATION_FAULT_LEVEL4";
+	case 0xC8: return "PERMISSION_FAULT";
+	case 0xC9 ... 0xCF: return "PERMISSION_FAULT";
+	case 0xD1: return "TRANSTAB_BUS_FAULT_LEVEL1";
+	case 0xD2: return "TRANSTAB_BUS_FAULT_LEVEL2";
+	case 0xD3: return "TRANSTAB_BUS_FAULT_LEVEL3";
+	case 0xD4: return "TRANSTAB_BUS_FAULT_LEVEL4";
+	case 0xD8: return "ACCESS_FLAG";
+	case 0xD9 ... 0xDF: return "ACCESS_FLAG";
+	case 0xE0 ... 0xE7: return "ADDRESS_SIZE_FAULT";
+	case 0xE8 ... 0xEF: return "MEMORY_ATTRIBUTES_FAULT";
+	}
+
+	return "UNKNOWN";
+}
+
+#ifdef CONFIG_PM
+int panfrost_device_resume(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct panfrost_device *pfdev = platform_get_drvdata(pdev);
+
+	panfrost_gpu_soft_reset(pfdev);
+
+	/* TODO: Re-enable all other address spaces */
+	panfrost_gpu_power_on(pfdev);
+	panfrost_mmu_enable(pfdev, 0);
+	panfrost_job_enable_interrupts(pfdev);
+	panfrost_devfreq_resume(pfdev);
+
+	return 0;
+}
+
+int panfrost_device_suspend(struct device *dev)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct panfrost_device *pfdev = platform_get_drvdata(pdev);
+
+	if (!panfrost_job_is_idle(pfdev))
+		return -EBUSY;
+
+	panfrost_devfreq_suspend(pfdev);
+	panfrost_gpu_power_off(pfdev);
+
+	return 0;
+}
+#endif
diff --git a/drivers/gpu/drm/panfrost/panfrost_device.h b/drivers/gpu/drm/panfrost/panfrost_device.h
new file mode 100644
index 000000000000..56f452dfb490
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/panfrost_device.h
@@ -0,0 +1,125 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */
+/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
+
+#ifndef __PANFROST_DEVICE_H__
+#define __PANFROST_DEVICE_H__
+
+#include <linux/spinlock.h>
+#include <drm/drm_device.h>
+#include <drm/drm_mm.h>
+#include <drm/gpu_scheduler.h>
+
+struct panfrost_device;
+struct panfrost_mmu;
+struct panfrost_job_slot;
+struct panfrost_job;
+
+#define NUM_JOB_SLOTS 3
+
+struct panfrost_features {
+	u16 id;
+	u16 revision;
+
+	u64 shader_present;
+	u64 tiler_present;
+	u64 l2_present;
+	u64 stack_present;
+	u32 as_present;
+	u32 js_present;
+
+	u32 l2_features;
+	u32 core_features;
+	u32 tiler_features;
+	u32 mem_features;
+	u32 mmu_features;
+	u32 thread_features;
+	u32 max_threads;
+	u32 thread_max_workgroup_sz;
+	u32 thread_max_barrier_sz;
+	u32 coherency_features;
+	u32 texture_features[4];
+	u32 js_features[16];
+
+	u32 nr_core_groups;
+
+	unsigned long hw_features[64 / BITS_PER_LONG];
+	unsigned long hw_issues[64 / BITS_PER_LONG];
+};
+
+struct panfrost_devfreq_slot {
+	ktime_t busy_time;
+	ktime_t idle_time;
+	ktime_t time_last_update;
+	bool busy;
+};
+
+struct panfrost_device {
+	struct device *dev;
+	struct drm_device *ddev;
+	struct platform_device *pdev;
+
+	spinlock_t hwaccess_lock;
+
+	struct drm_mm mm;
+	spinlock_t mm_lock;
+
+	void __iomem *iomem;
+	struct clk *clock;
+	struct regulator *regulator;
+	struct reset_control *rstc;
+
+	struct panfrost_features features;
+
+	struct panfrost_mmu *mmu;
+	struct panfrost_job_slot *js;
+
+	struct panfrost_job *jobs[NUM_JOB_SLOTS];
+	struct list_head scheduled_jobs;
+
+	struct mutex sched_lock;
+	struct mutex reset_lock;
+
+	struct {
+		struct devfreq *devfreq;
+		struct thermal_cooling_device *cooling;
+		unsigned long cur_freq;
+		unsigned long cur_volt;
+		struct panfrost_devfreq_slot slot[NUM_JOB_SLOTS];
+	} devfreq;
+};
+
+struct panfrost_file_priv {
+	struct panfrost_device *pfdev;
+
+	struct drm_sched_entity sched_entity[NUM_JOB_SLOTS];
+};
+
+static inline struct panfrost_device *to_panfrost_device(struct drm_device *ddev)
+{
+	return ddev->dev_private;
+}
+
+static inline int panfrost_model_cmp(struct panfrost_device *pfdev, s32 id)
+{
+	s32 match_id = pfdev->features.id;
+
+	if (match_id & 0xf000)
+		match_id &= 0xf00f;
+	return match_id - id;
+}
+
+static inline bool panfrost_model_eq(struct panfrost_device *pfdev, s32 id)
+{
+	return !panfrost_model_cmp(pfdev, id);
+}
+
+int panfrost_device_init(struct panfrost_device *pfdev);
+void panfrost_device_fini(struct panfrost_device *pfdev);
+
+int panfrost_device_resume(struct device *dev);
+int panfrost_device_suspend(struct device *dev);
+
+const char *panfrost_exception_name(struct panfrost_device *pfdev, u32 exception_code);
+
+#endif
diff --git a/drivers/gpu/drm/panfrost/panfrost_drv.c b/drivers/gpu/drm/panfrost/panfrost_drv.c
new file mode 100644
index 000000000000..94b0819ad50b
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/panfrost_drv.c
@@ -0,0 +1,474 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */
+/* Copyright 2019 Linaro, Ltd., Rob Herring <robh@kernel.org> */
+/* Copyright 2019 Collabora ltd. */
+
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <linux/pagemap.h>
+#include <linux/pm_runtime.h>
+#include <drm/panfrost_drm.h>
+#include <drm/drm_drv.h>
+#include <drm/drm_ioctl.h>
+#include <drm/drm_syncobj.h>
+#include <drm/drm_utils.h>
+
+#include "panfrost_device.h"
+#include "panfrost_devfreq.h"
+#include "panfrost_gem.h"
+#include "panfrost_mmu.h"
+#include "panfrost_job.h"
+#include "panfrost_gpu.h"
+
+static int panfrost_ioctl_get_param(struct drm_device *ddev, void *data, struct drm_file *file)
+{
+	struct drm_panfrost_get_param *param = data;
+	struct panfrost_device *pfdev = ddev->dev_private;
+
+	if (param->pad != 0)
+		return -EINVAL;
+
+	switch (param->param) {
+	case DRM_PANFROST_PARAM_GPU_PROD_ID:
+		param->value = pfdev->features.id;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int panfrost_ioctl_create_bo(struct drm_device *dev, void *data,
+		struct drm_file *file)
+{
+	int ret;
+	struct drm_gem_shmem_object *shmem;
+	struct drm_panfrost_create_bo *args = data;
+
+	if (!args->size || args->flags || args->pad)
+		return -EINVAL;
+
+	shmem = drm_gem_shmem_create_with_handle(file, dev, args->size,
+						 &args->handle);
+	if (IS_ERR(shmem))
+		return PTR_ERR(shmem);
+
+	ret = panfrost_mmu_map(to_panfrost_bo(&shmem->base));
+	if (ret)
+		goto err_free;
+
+	args->offset = to_panfrost_bo(&shmem->base)->node.start << PAGE_SHIFT;
+
+	return 0;
+
+err_free:
+	drm_gem_object_put_unlocked(&shmem->base);
+	return ret;
+}
+
+/**
+ * panfrost_lookup_bos() - Sets up job->bo[] with the GEM objects
+ * referenced by the job.
+ * @dev: DRM device
+ * @file_priv: DRM file for this fd
+ * @args: IOCTL args
+ * @job: job being set up
+ *
+ * Resolve handles from userspace to BOs and attach them to job.
+ *
+ * Note that this function doesn't need to unreference the BOs on
+ * failure, because that will happen at panfrost_job_cleanup() time.
+ */
+static int
+panfrost_lookup_bos(struct drm_device *dev,
+		  struct drm_file *file_priv,
+		  struct drm_panfrost_submit *args,
+		  struct panfrost_job *job)
+{
+	job->bo_count = args->bo_handle_count;
+
+	if (!job->bo_count)
+		return 0;
+
+	job->implicit_fences = kvmalloc_array(job->bo_count,
+				  sizeof(struct dma_fence *),
+				  GFP_KERNEL | __GFP_ZERO);
+	if (!job->implicit_fences)
+		return -ENOMEM;
+
+	return drm_gem_objects_lookup(file_priv,
+				      (void __user *)(uintptr_t)args->bo_handles,
+				      job->bo_count, &job->bos);
+}
+
+/**
+ * panfrost_copy_in_sync() - Sets up job->in_fences[] with the sync objects
+ * referenced by the job.
+ * @dev: DRM device
+ * @file_priv: DRM file for this fd
+ * @args: IOCTL args
+ * @job: job being set up
+ *
+ * Resolve syncobjs from userspace to fences and attach them to job.
+ *
+ * Note that this function doesn't need to unreference the fences on
+ * failure, because that will happen at panfrost_job_cleanup() time.
+ */
+static int
+panfrost_copy_in_sync(struct drm_device *dev,
+		  struct drm_file *file_priv,
+		  struct drm_panfrost_submit *args,
+		  struct panfrost_job *job)
+{
+	u32 *handles;
+	int ret = 0;
+	int i;
+
+	job->in_fence_count = args->in_sync_count;
+
+	if (!job->in_fence_count)
+		return 0;
+
+	job->in_fences = kvmalloc_array(job->in_fence_count,
+					sizeof(struct dma_fence *),
+					GFP_KERNEL | __GFP_ZERO);
+	if (!job->in_fences) {
+		DRM_DEBUG("Failed to allocate job in fences\n");
+		return -ENOMEM;
+	}
+
+	handles = kvmalloc_array(job->in_fence_count, sizeof(u32), GFP_KERNEL);
+	if (!handles) {
+		ret = -ENOMEM;
+		DRM_DEBUG("Failed to allocate incoming syncobj handles\n");
+		goto fail;
+	}
+
+	if (copy_from_user(handles,
+			   (void __user *)(uintptr_t)args->in_syncs,
+			   job->in_fence_count * sizeof(u32))) {
+		ret = -EFAULT;
+		DRM_DEBUG("Failed to copy in syncobj handles\n");
+		goto fail;
+	}
+
+	for (i = 0; i < job->in_fence_count; i++) {
+		ret = drm_syncobj_find_fence(file_priv, handles[i], 0, 0,
+					     &job->in_fences[i]);
+		if (ret == -EINVAL)
+			goto fail;
+	}
+
+fail:
+	kvfree(handles);
+	return ret;
+}
+
+static int panfrost_ioctl_submit(struct drm_device *dev, void *data,
+		struct drm_file *file)
+{
+	struct panfrost_device *pfdev = dev->dev_private;
+	struct drm_panfrost_submit *args = data;
+	struct drm_syncobj *sync_out = NULL;
+	struct panfrost_job *job;
+	int ret = 0;
+
+	if (!args->jc)
+		return -EINVAL;
+
+	if (args->requirements && args->requirements != PANFROST_JD_REQ_FS)
+		return -EINVAL;
+
+	if (args->out_sync > 0) {
+		sync_out = drm_syncobj_find(file, args->out_sync);
+		if (!sync_out)
+			return -ENODEV;
+	}
+
+	job = kzalloc(sizeof(*job), GFP_KERNEL);
+	if (!job) {
+		ret = -ENOMEM;
+		goto fail_out_sync;
+	}
+
+	kref_init(&job->refcount);
+
+	job->pfdev = pfdev;
+	job->jc = args->jc;
+	job->requirements = args->requirements;
+	job->flush_id = panfrost_gpu_get_latest_flush_id(pfdev);
+	job->file_priv = file->driver_priv;
+
+	ret = panfrost_copy_in_sync(dev, file, args, job);
+	if (ret)
+		goto fail_job;
+
+	ret = panfrost_lookup_bos(dev, file, args, job);
+	if (ret)
+		goto fail_job;
+
+	ret = panfrost_job_push(job);
+	if (ret)
+		goto fail_job;
+
+	/* Update the return sync object for the job */
+	if (sync_out)
+		drm_syncobj_replace_fence(sync_out, job->render_done_fence);
+
+fail_job:
+	panfrost_job_put(job);
+fail_out_sync:
+	drm_syncobj_put(sync_out);
+
+	return ret;
+}
+
+static int
+panfrost_ioctl_wait_bo(struct drm_device *dev, void *data,
+		       struct drm_file *file_priv)
+{
+	long ret;
+	struct drm_panfrost_wait_bo *args = data;
+	struct drm_gem_object *gem_obj;
+	unsigned long timeout = drm_timeout_abs_to_jiffies(args->timeout_ns);
+
+	if (args->pad)
+		return -EINVAL;
+
+	gem_obj = drm_gem_object_lookup(file_priv, args->handle);
+	if (!gem_obj)
+		return -ENOENT;
+
+	ret = reservation_object_wait_timeout_rcu(gem_obj->resv, true,
+						  true, timeout);
+	if (!ret)
+		ret = timeout ? -ETIMEDOUT : -EBUSY;
+
+	drm_gem_object_put_unlocked(gem_obj);
+
+	return ret;
+}
+
+static int panfrost_ioctl_mmap_bo(struct drm_device *dev, void *data,
+		      struct drm_file *file_priv)
+{
+	struct drm_panfrost_mmap_bo *args = data;
+	struct drm_gem_object *gem_obj;
+	int ret;
+
+	if (args->flags != 0) {
+		DRM_INFO("unknown mmap_bo flags: %d\n", args->flags);
+		return -EINVAL;
+	}
+
+	gem_obj = drm_gem_object_lookup(file_priv, args->handle);
+	if (!gem_obj) {
+		DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle);
+		return -ENOENT;
+	}
+
+	ret = drm_gem_create_mmap_offset(gem_obj);
+	if (ret == 0)
+		args->offset = drm_vma_node_offset_addr(&gem_obj->vma_node);
+	drm_gem_object_put_unlocked(gem_obj);
+
+	return ret;
+}
+
+static int panfrost_ioctl_get_bo_offset(struct drm_device *dev, void *data,
+			    struct drm_file *file_priv)
+{
+	struct drm_panfrost_get_bo_offset *args = data;
+	struct drm_gem_object *gem_obj;
+	struct panfrost_gem_object *bo;
+
+	gem_obj = drm_gem_object_lookup(file_priv, args->handle);
+	if (!gem_obj) {
+		DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle);
+		return -ENOENT;
+	}
+	bo = to_panfrost_bo(gem_obj);
+
+	args->offset = bo->node.start << PAGE_SHIFT;
+
+	drm_gem_object_put_unlocked(gem_obj);
+	return 0;
+}
+
+static int
+panfrost_open(struct drm_device *dev, struct drm_file *file)
+{
+	struct panfrost_device *pfdev = dev->dev_private;
+	struct panfrost_file_priv *panfrost_priv;
+
+	panfrost_priv = kzalloc(sizeof(*panfrost_priv), GFP_KERNEL);
+	if (!panfrost_priv)
+		return -ENOMEM;
+
+	panfrost_priv->pfdev = pfdev;
+	file->driver_priv = panfrost_priv;
+
+	return panfrost_job_open(panfrost_priv);
+}
+
+static void
+panfrost_postclose(struct drm_device *dev, struct drm_file *file)
+{
+	struct panfrost_file_priv *panfrost_priv = file->driver_priv;
+
+	panfrost_job_close(panfrost_priv);
+
+	kfree(panfrost_priv);
+}
+
+/* DRM_AUTH is required on SUBMIT for now, while all clients share a single
+ * address space.  Note that render nodes would be able to submit jobs that
+ * could access BOs from clients authenticated with the master node.
+ */
+static const struct drm_ioctl_desc panfrost_drm_driver_ioctls[] = {
+#define PANFROST_IOCTL(n, func, flags) \
+	DRM_IOCTL_DEF_DRV(PANFROST_##n, panfrost_ioctl_##func, flags)
+
+	PANFROST_IOCTL(SUBMIT,		submit,		DRM_RENDER_ALLOW | DRM_AUTH),
+	PANFROST_IOCTL(WAIT_BO,		wait_bo,	DRM_RENDER_ALLOW),
+	PANFROST_IOCTL(CREATE_BO,	create_bo,	DRM_RENDER_ALLOW),
+	PANFROST_IOCTL(MMAP_BO,		mmap_bo,	DRM_RENDER_ALLOW),
+	PANFROST_IOCTL(GET_PARAM,	get_param,	DRM_RENDER_ALLOW),
+	PANFROST_IOCTL(GET_BO_OFFSET,	get_bo_offset,	DRM_RENDER_ALLOW),
+};
+
+DEFINE_DRM_GEM_SHMEM_FOPS(panfrost_drm_driver_fops);
+
+static struct drm_driver panfrost_drm_driver = {
+	.driver_features	= DRIVER_RENDER | DRIVER_GEM | DRIVER_PRIME |
+				  DRIVER_SYNCOBJ,
+	.open			= panfrost_open,
+	.postclose		= panfrost_postclose,
+	.ioctls			= panfrost_drm_driver_ioctls,
+	.num_ioctls		= ARRAY_SIZE(panfrost_drm_driver_ioctls),
+	.fops			= &panfrost_drm_driver_fops,
+	.name			= "panfrost",
+	.desc			= "panfrost DRM",
+	.date			= "20180908",
+	.major			= 1,
+	.minor			= 0,
+
+	.gem_create_object	= panfrost_gem_create_object,
+	.prime_handle_to_fd	= drm_gem_prime_handle_to_fd,
+	.prime_fd_to_handle	= drm_gem_prime_fd_to_handle,
+	.gem_prime_import_sg_table = panfrost_gem_prime_import_sg_table,
+	.gem_prime_mmap		= drm_gem_prime_mmap,
+};
+
+static int panfrost_probe(struct platform_device *pdev)
+{
+	struct panfrost_device *pfdev;
+	struct drm_device *ddev;
+	int err;
+
+	pfdev = devm_kzalloc(&pdev->dev, sizeof(*pfdev), GFP_KERNEL);
+	if (!pfdev)
+		return -ENOMEM;
+
+	pfdev->pdev = pdev;
+	pfdev->dev = &pdev->dev;
+
+	platform_set_drvdata(pdev, pfdev);
+
+	/* Allocate and initialze the DRM device. */
+	ddev = drm_dev_alloc(&panfrost_drm_driver, &pdev->dev);
+	if (IS_ERR(ddev))
+		return PTR_ERR(ddev);
+
+	ddev->dev_private = pfdev;
+	pfdev->ddev = ddev;
+
+	spin_lock_init(&pfdev->mm_lock);
+
+	/* 4G enough for now. can be 48-bit */
+	drm_mm_init(&pfdev->mm, SZ_32M >> PAGE_SHIFT, (SZ_4G - SZ_32M) >> PAGE_SHIFT);
+
+	pm_runtime_use_autosuspend(pfdev->dev);
+	pm_runtime_set_autosuspend_delay(pfdev->dev, 50); /* ~3 frames */
+	pm_runtime_enable(pfdev->dev);
+
+	err = panfrost_device_init(pfdev);
+	if (err) {
+		if (err != -EPROBE_DEFER)
+			dev_err(&pdev->dev, "Fatal error during GPU init\n");
+		goto err_out0;
+	}
+
+	err = panfrost_devfreq_init(pfdev);
+	if (err) {
+		if (err != -EPROBE_DEFER)
+			dev_err(&pdev->dev, "Fatal error during devfreq init\n");
+		goto err_out1;
+	}
+
+	/*
+	 * Register the DRM device with the core and the connectors with
+	 * sysfs
+	 */
+	err = drm_dev_register(ddev, 0);
+	if (err < 0)
+		goto err_out1;
+
+	return 0;
+
+err_out1:
+	panfrost_device_fini(pfdev);
+err_out0:
+	pm_runtime_disable(pfdev->dev);
+	drm_dev_put(ddev);
+	return err;
+}
+
+static int panfrost_remove(struct platform_device *pdev)
+{
+	struct panfrost_device *pfdev = platform_get_drvdata(pdev);
+	struct drm_device *ddev = pfdev->ddev;
+
+	drm_dev_unregister(ddev);
+	pm_runtime_get_sync(pfdev->dev);
+	pm_runtime_put_sync_autosuspend(pfdev->dev);
+	pm_runtime_disable(pfdev->dev);
+	panfrost_device_fini(pfdev);
+	drm_dev_put(ddev);
+	return 0;
+}
+
+static const struct of_device_id dt_match[] = {
+	{ .compatible = "arm,mali-t604" },
+	{ .compatible = "arm,mali-t624" },
+	{ .compatible = "arm,mali-t628" },
+	{ .compatible = "arm,mali-t720" },
+	{ .compatible = "arm,mali-t760" },
+	{ .compatible = "arm,mali-t820" },
+	{ .compatible = "arm,mali-t830" },
+	{ .compatible = "arm,mali-t860" },
+	{ .compatible = "arm,mali-t880" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, dt_match);
+
+static const struct dev_pm_ops panfrost_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume)
+	SET_RUNTIME_PM_OPS(panfrost_device_suspend, panfrost_device_resume, NULL)
+};
+
+static struct platform_driver panfrost_driver = {
+	.probe		= panfrost_probe,
+	.remove		= panfrost_remove,
+	.driver		= {
+		.name	= "panfrost",
+		.pm	= &panfrost_pm_ops,
+		.of_match_table = dt_match,
+	},
+};
+module_platform_driver(panfrost_driver);
+
+MODULE_AUTHOR("Panfrost Project Developers");
+MODULE_DESCRIPTION("Panfrost DRM Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/panfrost/panfrost_features.h b/drivers/gpu/drm/panfrost/panfrost_features.h
new file mode 100644
index 000000000000..5056777c7744
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/panfrost_features.h
@@ -0,0 +1,309 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* (C) COPYRIGHT 2014-2018 ARM Limited. All rights reserved. */
+/* Copyright 2019 Linaro, Ltd., Rob Herring <robh@kernel.org> */
+#ifndef __PANFROST_FEATURES_H__
+#define __PANFROST_FEATURES_H__
+
+#include <linux/bitops.h>
+
+#include "panfrost_device.h"
+
+enum panfrost_hw_feature {
+	HW_FEATURE_JOBCHAIN_DISAMBIGUATION,
+	HW_FEATURE_PWRON_DURING_PWROFF_TRANS,
+	HW_FEATURE_XAFFINITY,
+	HW_FEATURE_OUT_OF_ORDER_EXEC,
+	HW_FEATURE_MRT,
+	HW_FEATURE_BRNDOUT_CC,
+	HW_FEATURE_INTERPIPE_REG_ALIASING,
+	HW_FEATURE_LD_ST_TILEBUFFER,
+	HW_FEATURE_MSAA_16X,
+	HW_FEATURE_32_BIT_UNIFORM_ADDRESS,
+	HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL,
+	HW_FEATURE_OPTIMIZED_COVERAGE_MASK,
+	HW_FEATURE_T7XX_PAIRING_RULES,
+	HW_FEATURE_LD_ST_LEA_TEX,
+	HW_FEATURE_LINEAR_FILTER_FLOAT,
+	HW_FEATURE_WORKGROUP_ROUND_MULTIPLE_OF_4,
+	HW_FEATURE_IMAGES_IN_FRAGMENT_SHADERS,
+	HW_FEATURE_TEST4_DATUM_MODE,
+	HW_FEATURE_NEXT_INSTRUCTION_TYPE,
+	HW_FEATURE_BRNDOUT_KILL,
+	HW_FEATURE_WARPING,
+	HW_FEATURE_V4,
+	HW_FEATURE_FLUSH_REDUCTION,
+	HW_FEATURE_PROTECTED_MODE,
+	HW_FEATURE_COHERENCY_REG,
+	HW_FEATURE_PROTECTED_DEBUG_MODE,
+	HW_FEATURE_AARCH64_MMU,
+	HW_FEATURE_TLS_HASHING,
+	HW_FEATURE_THREAD_GROUP_SPLIT,
+	HW_FEATURE_3BIT_EXT_RW_L2_MMU_CONFIG,
+};
+
+#define hw_features_t600 (\
+	BIT_ULL(HW_FEATURE_LD_ST_LEA_TEX) | \
+	BIT_ULL(HW_FEATURE_LINEAR_FILTER_FLOAT) | \
+	BIT_ULL(HW_FEATURE_THREAD_GROUP_SPLIT) | \
+	BIT_ULL(HW_FEATURE_V4))
+
+#define hw_features_t620 (\
+	BIT_ULL(HW_FEATURE_LD_ST_LEA_TEX) | \
+	BIT_ULL(HW_FEATURE_LINEAR_FILTER_FLOAT) | \
+	BIT_ULL(HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL) | \
+	BIT_ULL(HW_FEATURE_THREAD_GROUP_SPLIT) | \
+	BIT_ULL(HW_FEATURE_V4))
+
+#define hw_features_t720 (\
+	BIT_ULL(HW_FEATURE_32_BIT_UNIFORM_ADDRESS) | \
+	BIT_ULL(HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL) | \
+	BIT_ULL(HW_FEATURE_INTERPIPE_REG_ALIASING) | \
+	BIT_ULL(HW_FEATURE_OPTIMIZED_COVERAGE_MASK) | \
+	BIT_ULL(HW_FEATURE_T7XX_PAIRING_RULES) | \
+	BIT_ULL(HW_FEATURE_THREAD_GROUP_SPLIT) | \
+	BIT_ULL(HW_FEATURE_WORKGROUP_ROUND_MULTIPLE_OF_4) | \
+	BIT_ULL(HW_FEATURE_WARPING) | \
+	BIT_ULL(HW_FEATURE_V4))
+
+
+#define hw_features_t760 (\
+	BIT_ULL(HW_FEATURE_JOBCHAIN_DISAMBIGUATION) | \
+	BIT_ULL(HW_FEATURE_PWRON_DURING_PWROFF_TRANS) | \
+	BIT_ULL(HW_FEATURE_XAFFINITY) | \
+	BIT_ULL(HW_FEATURE_32_BIT_UNIFORM_ADDRESS) | \
+	BIT_ULL(HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL) | \
+	BIT_ULL(HW_FEATURE_BRNDOUT_CC) | \
+	BIT_ULL(HW_FEATURE_LD_ST_LEA_TEX) | \
+	BIT_ULL(HW_FEATURE_LD_ST_TILEBUFFER) | \
+	BIT_ULL(HW_FEATURE_LINEAR_FILTER_FLOAT) | \
+	BIT_ULL(HW_FEATURE_MRT) | \
+	BIT_ULL(HW_FEATURE_MSAA_16X) | \
+	BIT_ULL(HW_FEATURE_OUT_OF_ORDER_EXEC) | \
+	BIT_ULL(HW_FEATURE_T7XX_PAIRING_RULES) | \
+	BIT_ULL(HW_FEATURE_TEST4_DATUM_MODE) | \
+	BIT_ULL(HW_FEATURE_THREAD_GROUP_SPLIT))
+
+// T860
+#define hw_features_t860 (\
+	BIT_ULL(HW_FEATURE_JOBCHAIN_DISAMBIGUATION) | \
+	BIT_ULL(HW_FEATURE_PWRON_DURING_PWROFF_TRANS) | \
+	BIT_ULL(HW_FEATURE_XAFFINITY) | \
+	BIT_ULL(HW_FEATURE_32_BIT_UNIFORM_ADDRESS) | \
+	BIT_ULL(HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL) | \
+	BIT_ULL(HW_FEATURE_BRNDOUT_CC) | \
+	BIT_ULL(HW_FEATURE_BRNDOUT_KILL) | \
+	BIT_ULL(HW_FEATURE_LD_ST_LEA_TEX) | \
+	BIT_ULL(HW_FEATURE_LD_ST_TILEBUFFER) | \
+	BIT_ULL(HW_FEATURE_LINEAR_FILTER_FLOAT) | \
+	BIT_ULL(HW_FEATURE_MRT) | \
+	BIT_ULL(HW_FEATURE_MSAA_16X) | \
+	BIT_ULL(HW_FEATURE_NEXT_INSTRUCTION_TYPE) | \
+	BIT_ULL(HW_FEATURE_OUT_OF_ORDER_EXEC) | \
+	BIT_ULL(HW_FEATURE_T7XX_PAIRING_RULES) | \
+	BIT_ULL(HW_FEATURE_TEST4_DATUM_MODE) | \
+	BIT_ULL(HW_FEATURE_THREAD_GROUP_SPLIT))
+
+#define hw_features_t880 hw_features_t860
+
+#define hw_features_t830 (\
+	BIT_ULL(HW_FEATURE_JOBCHAIN_DISAMBIGUATION) | \
+	BIT_ULL(HW_FEATURE_PWRON_DURING_PWROFF_TRANS) | \
+	BIT_ULL(HW_FEATURE_XAFFINITY) | \
+	BIT_ULL(HW_FEATURE_WARPING) | \
+	BIT_ULL(HW_FEATURE_INTERPIPE_REG_ALIASING) | \
+	BIT_ULL(HW_FEATURE_32_BIT_UNIFORM_ADDRESS) | \
+	BIT_ULL(HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL) | \
+	BIT_ULL(HW_FEATURE_BRNDOUT_CC) | \
+	BIT_ULL(HW_FEATURE_BRNDOUT_KILL) | \
+	BIT_ULL(HW_FEATURE_LD_ST_LEA_TEX) | \
+	BIT_ULL(HW_FEATURE_LD_ST_TILEBUFFER) | \
+	BIT_ULL(HW_FEATURE_LINEAR_FILTER_FLOAT) | \
+	BIT_ULL(HW_FEATURE_MRT) | \
+	BIT_ULL(HW_FEATURE_NEXT_INSTRUCTION_TYPE) | \
+	BIT_ULL(HW_FEATURE_OUT_OF_ORDER_EXEC) | \
+	BIT_ULL(HW_FEATURE_T7XX_PAIRING_RULES) | \
+	BIT_ULL(HW_FEATURE_TEST4_DATUM_MODE) | \
+	BIT_ULL(HW_FEATURE_THREAD_GROUP_SPLIT))
+
+#define hw_features_t820 (\
+	BIT_ULL(HW_FEATURE_JOBCHAIN_DISAMBIGUATION) | \
+	BIT_ULL(HW_FEATURE_PWRON_DURING_PWROFF_TRANS) | \
+	BIT_ULL(HW_FEATURE_XAFFINITY) | \
+	BIT_ULL(HW_FEATURE_WARPING) | \
+	BIT_ULL(HW_FEATURE_INTERPIPE_REG_ALIASING) | \
+	BIT_ULL(HW_FEATURE_32_BIT_UNIFORM_ADDRESS) | \
+	BIT_ULL(HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL) | \
+	BIT_ULL(HW_FEATURE_BRNDOUT_CC) | \
+	BIT_ULL(HW_FEATURE_BRNDOUT_KILL) | \
+	BIT_ULL(HW_FEATURE_LD_ST_LEA_TEX) | \
+	BIT_ULL(HW_FEATURE_LD_ST_TILEBUFFER) | \
+	BIT_ULL(HW_FEATURE_LINEAR_FILTER_FLOAT) | \
+	BIT_ULL(HW_FEATURE_MRT) | \
+	BIT_ULL(HW_FEATURE_NEXT_INSTRUCTION_TYPE) | \
+	BIT_ULL(HW_FEATURE_OUT_OF_ORDER_EXEC) | \
+	BIT_ULL(HW_FEATURE_T7XX_PAIRING_RULES) | \
+	BIT_ULL(HW_FEATURE_TEST4_DATUM_MODE) | \
+	BIT_ULL(HW_FEATURE_THREAD_GROUP_SPLIT))
+
+#define hw_features_g71 (\
+	BIT_ULL(HW_FEATURE_JOBCHAIN_DISAMBIGUATION) | \
+	BIT_ULL(HW_FEATURE_PWRON_DURING_PWROFF_TRANS) | \
+	BIT_ULL(HW_FEATURE_XAFFINITY) | \
+	BIT_ULL(HW_FEATURE_WARPING) | \
+	BIT_ULL(HW_FEATURE_INTERPIPE_REG_ALIASING) | \
+	BIT_ULL(HW_FEATURE_32_BIT_UNIFORM_ADDRESS) | \
+	BIT_ULL(HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL) | \
+	BIT_ULL(HW_FEATURE_BRNDOUT_CC) | \
+	BIT_ULL(HW_FEATURE_BRNDOUT_KILL) | \
+	BIT_ULL(HW_FEATURE_LD_ST_LEA_TEX) | \
+	BIT_ULL(HW_FEATURE_LD_ST_TILEBUFFER) | \
+	BIT_ULL(HW_FEATURE_LINEAR_FILTER_FLOAT) | \
+	BIT_ULL(HW_FEATURE_MRT) | \
+	BIT_ULL(HW_FEATURE_MSAA_16X) | \
+	BIT_ULL(HW_FEATURE_NEXT_INSTRUCTION_TYPE) | \
+	BIT_ULL(HW_FEATURE_OUT_OF_ORDER_EXEC) | \
+	BIT_ULL(HW_FEATURE_T7XX_PAIRING_RULES) | \
+	BIT_ULL(HW_FEATURE_TEST4_DATUM_MODE) | \
+	BIT_ULL(HW_FEATURE_THREAD_GROUP_SPLIT) | \
+	BIT_ULL(HW_FEATURE_FLUSH_REDUCTION) | \
+	BIT_ULL(HW_FEATURE_PROTECTED_MODE) | \
+	BIT_ULL(HW_FEATURE_COHERENCY_REG))
+
+#define hw_features_g72 (\
+	BIT_ULL(HW_FEATURE_JOBCHAIN_DISAMBIGUATION) | \
+	BIT_ULL(HW_FEATURE_PWRON_DURING_PWROFF_TRANS) | \
+	BIT_ULL(HW_FEATURE_XAFFINITY) | \
+	BIT_ULL(HW_FEATURE_WARPING) | \
+	BIT_ULL(HW_FEATURE_INTERPIPE_REG_ALIASING) | \
+	BIT_ULL(HW_FEATURE_32_BIT_UNIFORM_ADDRESS) | \
+	BIT_ULL(HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL) | \
+	BIT_ULL(HW_FEATURE_BRNDOUT_CC) | \
+	BIT_ULL(HW_FEATURE_BRNDOUT_KILL) | \
+	BIT_ULL(HW_FEATURE_LD_ST_LEA_TEX) | \
+	BIT_ULL(HW_FEATURE_LD_ST_TILEBUFFER) | \
+	BIT_ULL(HW_FEATURE_LINEAR_FILTER_FLOAT) | \
+	BIT_ULL(HW_FEATURE_MRT) | \
+	BIT_ULL(HW_FEATURE_MSAA_16X) | \
+	BIT_ULL(HW_FEATURE_NEXT_INSTRUCTION_TYPE) | \
+	BIT_ULL(HW_FEATURE_OUT_OF_ORDER_EXEC) | \
+	BIT_ULL(HW_FEATURE_T7XX_PAIRING_RULES) | \
+	BIT_ULL(HW_FEATURE_TEST4_DATUM_MODE) | \
+	BIT_ULL(HW_FEATURE_THREAD_GROUP_SPLIT) | \
+	BIT_ULL(HW_FEATURE_FLUSH_REDUCTION) | \
+	BIT_ULL(HW_FEATURE_PROTECTED_MODE) | \
+	BIT_ULL(HW_FEATURE_PROTECTED_DEBUG_MODE) | \
+	BIT_ULL(HW_FEATURE_COHERENCY_REG))
+
+#define hw_features_g51 (\
+	BIT_ULL(HW_FEATURE_JOBCHAIN_DISAMBIGUATION) | \
+	BIT_ULL(HW_FEATURE_PWRON_DURING_PWROFF_TRANS) | \
+	BIT_ULL(HW_FEATURE_XAFFINITY) | \
+	BIT_ULL(HW_FEATURE_WARPING) | \
+	BIT_ULL(HW_FEATURE_INTERPIPE_REG_ALIASING) | \
+	BIT_ULL(HW_FEATURE_32_BIT_UNIFORM_ADDRESS) | \
+	BIT_ULL(HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL) | \
+	BIT_ULL(HW_FEATURE_BRNDOUT_CC) | \
+	BIT_ULL(HW_FEATURE_BRNDOUT_KILL) | \
+	BIT_ULL(HW_FEATURE_LD_ST_LEA_TEX) | \
+	BIT_ULL(HW_FEATURE_LD_ST_TILEBUFFER) | \
+	BIT_ULL(HW_FEATURE_LINEAR_FILTER_FLOAT) | \
+	BIT_ULL(HW_FEATURE_MRT) | \
+	BIT_ULL(HW_FEATURE_MSAA_16X) | \
+	BIT_ULL(HW_FEATURE_NEXT_INSTRUCTION_TYPE) | \
+	BIT_ULL(HW_FEATURE_OUT_OF_ORDER_EXEC) | \
+	BIT_ULL(HW_FEATURE_T7XX_PAIRING_RULES) | \
+	BIT_ULL(HW_FEATURE_TEST4_DATUM_MODE) | \
+	BIT_ULL(HW_FEATURE_THREAD_GROUP_SPLIT) | \
+	BIT_ULL(HW_FEATURE_FLUSH_REDUCTION) | \
+	BIT_ULL(HW_FEATURE_PROTECTED_MODE) | \
+	BIT_ULL(HW_FEATURE_PROTECTED_DEBUG_MODE) | \
+	BIT_ULL(HW_FEATURE_COHERENCY_REG))
+
+#define hw_features_g52 (\
+	BIT_ULL(HW_FEATURE_JOBCHAIN_DISAMBIGUATION) | \
+	BIT_ULL(HW_FEATURE_PWRON_DURING_PWROFF_TRANS) | \
+	BIT_ULL(HW_FEATURE_XAFFINITY) | \
+	BIT_ULL(HW_FEATURE_WARPING) | \
+	BIT_ULL(HW_FEATURE_INTERPIPE_REG_ALIASING) | \
+	BIT_ULL(HW_FEATURE_32_BIT_UNIFORM_ADDRESS) | \
+	BIT_ULL(HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL) | \
+	BIT_ULL(HW_FEATURE_BRNDOUT_CC) | \
+	BIT_ULL(HW_FEATURE_BRNDOUT_KILL) | \
+	BIT_ULL(HW_FEATURE_LD_ST_LEA_TEX) | \
+	BIT_ULL(HW_FEATURE_LD_ST_TILEBUFFER) | \
+	BIT_ULL(HW_FEATURE_LINEAR_FILTER_FLOAT) | \
+	BIT_ULL(HW_FEATURE_MRT) | \
+	BIT_ULL(HW_FEATURE_MSAA_16X) | \
+	BIT_ULL(HW_FEATURE_NEXT_INSTRUCTION_TYPE) | \
+	BIT_ULL(HW_FEATURE_OUT_OF_ORDER_EXEC) | \
+	BIT_ULL(HW_FEATURE_T7XX_PAIRING_RULES) | \
+	BIT_ULL(HW_FEATURE_TEST4_DATUM_MODE) | \
+	BIT_ULL(HW_FEATURE_THREAD_GROUP_SPLIT) | \
+	BIT_ULL(HW_FEATURE_FLUSH_REDUCTION) | \
+	BIT_ULL(HW_FEATURE_PROTECTED_MODE) | \
+	BIT_ULL(HW_FEATURE_PROTECTED_DEBUG_MODE) | \
+	BIT_ULL(HW_FEATURE_COHERENCY_REG))
+
+#define hw_features_g76 (\
+	BIT_ULL(HW_FEATURE_JOBCHAIN_DISAMBIGUATION) | \
+	BIT_ULL(HW_FEATURE_PWRON_DURING_PWROFF_TRANS) | \
+	BIT_ULL(HW_FEATURE_XAFFINITY) | \
+	BIT_ULL(HW_FEATURE_WARPING) | \
+	BIT_ULL(HW_FEATURE_INTERPIPE_REG_ALIASING) | \
+	BIT_ULL(HW_FEATURE_32_BIT_UNIFORM_ADDRESS) | \
+	BIT_ULL(HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL) | \
+	BIT_ULL(HW_FEATURE_BRNDOUT_CC) | \
+	BIT_ULL(HW_FEATURE_BRNDOUT_KILL) | \
+	BIT_ULL(HW_FEATURE_LD_ST_LEA_TEX) | \
+	BIT_ULL(HW_FEATURE_LD_ST_TILEBUFFER) | \
+	BIT_ULL(HW_FEATURE_LINEAR_FILTER_FLOAT) | \
+	BIT_ULL(HW_FEATURE_MRT) | \
+	BIT_ULL(HW_FEATURE_MSAA_16X) | \
+	BIT_ULL(HW_FEATURE_NEXT_INSTRUCTION_TYPE) | \
+	BIT_ULL(HW_FEATURE_OUT_OF_ORDER_EXEC) | \
+	BIT_ULL(HW_FEATURE_T7XX_PAIRING_RULES) | \
+	BIT_ULL(HW_FEATURE_TEST4_DATUM_MODE) | \
+	BIT_ULL(HW_FEATURE_THREAD_GROUP_SPLIT) | \
+	BIT_ULL(HW_FEATURE_FLUSH_REDUCTION) | \
+	BIT_ULL(HW_FEATURE_PROTECTED_MODE) | \
+	BIT_ULL(HW_FEATURE_PROTECTED_DEBUG_MODE) | \
+	BIT_ULL(HW_FEATURE_COHERENCY_REG) | \
+	BIT_ULL(HW_FEATURE_AARCH64_MMU) | \
+	BIT_ULL(HW_FEATURE_TLS_HASHING) | \
+	BIT_ULL(HW_FEATURE_3BIT_EXT_RW_L2_MMU_CONFIG))
+
+#define hw_features_g31 (\
+	BIT_ULL(HW_FEATURE_JOBCHAIN_DISAMBIGUATION) | \
+	BIT_ULL(HW_FEATURE_PWRON_DURING_PWROFF_TRANS) | \
+	BIT_ULL(HW_FEATURE_XAFFINITY) | \
+	BIT_ULL(HW_FEATURE_WARPING) | \
+	BIT_ULL(HW_FEATURE_INTERPIPE_REG_ALIASING) | \
+	BIT_ULL(HW_FEATURE_32_BIT_UNIFORM_ADDRESS) | \
+	BIT_ULL(HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL) | \
+	BIT_ULL(HW_FEATURE_BRNDOUT_CC) | \
+	BIT_ULL(HW_FEATURE_BRNDOUT_KILL) | \
+	BIT_ULL(HW_FEATURE_LD_ST_LEA_TEX) | \
+	BIT_ULL(HW_FEATURE_LD_ST_TILEBUFFER) | \
+	BIT_ULL(HW_FEATURE_LINEAR_FILTER_FLOAT) | \
+	BIT_ULL(HW_FEATURE_MRT) | \
+	BIT_ULL(HW_FEATURE_MSAA_16X) | \
+	BIT_ULL(HW_FEATURE_NEXT_INSTRUCTION_TYPE) | \
+	BIT_ULL(HW_FEATURE_OUT_OF_ORDER_EXEC) | \
+	BIT_ULL(HW_FEATURE_T7XX_PAIRING_RULES) | \
+	BIT_ULL(HW_FEATURE_TEST4_DATUM_MODE) | \
+	BIT_ULL(HW_FEATURE_THREAD_GROUP_SPLIT) | \
+	BIT_ULL(HW_FEATURE_FLUSH_REDUCTION) | \
+	BIT_ULL(HW_FEATURE_PROTECTED_MODE) | \
+	BIT_ULL(HW_FEATURE_PROTECTED_DEBUG_MODE) | \
+	BIT_ULL(HW_FEATURE_COHERENCY_REG) | \
+	BIT_ULL(HW_FEATURE_AARCH64_MMU) | \
+	BIT_ULL(HW_FEATURE_TLS_HASHING) | \
+	BIT_ULL(HW_FEATURE_3BIT_EXT_RW_L2_MMU_CONFIG))
+
+static inline bool panfrost_has_hw_feature(struct panfrost_device *pfdev,
+					   enum panfrost_hw_feature feat)
+{
+	return test_bit(feat, pfdev->features.hw_features);
+}
+
+#endif
diff --git a/drivers/gpu/drm/panfrost/panfrost_gem.c b/drivers/gpu/drm/panfrost/panfrost_gem.c
new file mode 100644
index 000000000000..a5528a360ef4
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/panfrost_gem.c
@@ -0,0 +1,95 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
+
+#include <linux/err.h>
+#include <linux/slab.h>
+#include <linux/dma-buf.h>
+#include <linux/dma-mapping.h>
+
+#include <drm/panfrost_drm.h>
+#include "panfrost_device.h"
+#include "panfrost_gem.h"
+#include "panfrost_mmu.h"
+
+/* Called DRM core on the last userspace/kernel unreference of the
+ * BO.
+ */
+static void panfrost_gem_free_object(struct drm_gem_object *obj)
+{
+	struct panfrost_gem_object *bo = to_panfrost_bo(obj);
+	struct panfrost_device *pfdev = obj->dev->dev_private;
+
+	panfrost_mmu_unmap(bo);
+
+	spin_lock(&pfdev->mm_lock);
+	drm_mm_remove_node(&bo->node);
+	spin_unlock(&pfdev->mm_lock);
+
+	drm_gem_shmem_free_object(obj);
+}
+
+static const struct drm_gem_object_funcs panfrost_gem_funcs = {
+	.free = panfrost_gem_free_object,
+	.print_info = drm_gem_shmem_print_info,
+	.pin = drm_gem_shmem_pin,
+	.unpin = drm_gem_shmem_unpin,
+	.get_sg_table = drm_gem_shmem_get_sg_table,
+	.vmap = drm_gem_shmem_vmap,
+	.vunmap = drm_gem_shmem_vunmap,
+	.vm_ops = &drm_gem_shmem_vm_ops,
+};
+
+/**
+ * panfrost_gem_create_object - Implementation of driver->gem_create_object.
+ * @dev: DRM device
+ * @size: Size in bytes of the memory the object will reference
+ *
+ * This lets the GEM helpers allocate object structs for us, and keep
+ * our BO stats correct.
+ */
+struct drm_gem_object *panfrost_gem_create_object(struct drm_device *dev, size_t size)
+{
+	int ret;
+	struct panfrost_device *pfdev = dev->dev_private;
+	struct panfrost_gem_object *obj;
+
+	obj = kzalloc(sizeof(*obj), GFP_KERNEL);
+	if (!obj)
+		return NULL;
+
+	obj->base.base.funcs = &panfrost_gem_funcs;
+
+	spin_lock(&pfdev->mm_lock);
+	ret = drm_mm_insert_node(&pfdev->mm, &obj->node,
+				 roundup(size, PAGE_SIZE) >> PAGE_SHIFT);
+	spin_unlock(&pfdev->mm_lock);
+	if (ret)
+		goto free_obj;
+
+	return &obj->base.base;
+
+free_obj:
+	kfree(obj);
+	return ERR_PTR(ret);
+}
+
+struct drm_gem_object *
+panfrost_gem_prime_import_sg_table(struct drm_device *dev,
+				   struct dma_buf_attachment *attach,
+				   struct sg_table *sgt)
+{
+	struct drm_gem_object *obj;
+	struct panfrost_gem_object *pobj;
+
+	obj = drm_gem_shmem_prime_import_sg_table(dev, attach, sgt);
+	if (IS_ERR(obj))
+		return ERR_CAST(obj);
+
+	pobj = to_panfrost_bo(obj);
+
+	obj->resv = attach->dmabuf->resv;
+
+	panfrost_mmu_map(pobj);
+
+	return obj;
+}
diff --git a/drivers/gpu/drm/panfrost/panfrost_gem.h b/drivers/gpu/drm/panfrost/panfrost_gem.h
new file mode 100644
index 000000000000..045000eb5fcf
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/panfrost_gem.h
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
+
+#ifndef __PANFROST_GEM_H__
+#define __PANFROST_GEM_H__
+
+#include <drm/drm_gem_shmem_helper.h>
+#include <drm/drm_mm.h>
+
+struct panfrost_gem_object {
+	struct drm_gem_shmem_object base;
+
+	struct drm_mm_node node;
+};
+
+static inline
+struct  panfrost_gem_object *to_panfrost_bo(struct drm_gem_object *obj)
+{
+	return container_of(to_drm_gem_shmem_obj(obj), struct panfrost_gem_object, base);
+}
+
+struct drm_gem_object *panfrost_gem_create_object(struct drm_device *dev, size_t size);
+
+struct drm_gem_object *
+panfrost_gem_prime_import_sg_table(struct drm_device *dev,
+				   struct dma_buf_attachment *attach,
+				   struct sg_table *sgt);
+
+#endif /* __PANFROST_GEM_H__ */
diff --git a/drivers/gpu/drm/panfrost/panfrost_gpu.c b/drivers/gpu/drm/panfrost/panfrost_gpu.c
new file mode 100644
index 000000000000..58ef25573cda
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/panfrost_gpu.c
@@ -0,0 +1,367 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */
+/* Copyright 2019 Linaro, Ltd., Rob Herring <robh@kernel.org> */
+/* Copyright 2019 Collabora ltd. */
+#include <linux/bitfield.h>
+#include <linux/bitmap.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/platform_device.h>
+
+#include "panfrost_device.h"
+#include "panfrost_features.h"
+#include "panfrost_issues.h"
+#include "panfrost_gpu.h"
+#include "panfrost_regs.h"
+
+#define gpu_write(dev, reg, data) writel(data, dev->iomem + reg)
+#define gpu_read(dev, reg) readl(dev->iomem + reg)
+
+static irqreturn_t panfrost_gpu_irq_handler(int irq, void *data)
+{
+	struct panfrost_device *pfdev = data;
+	u32 state = gpu_read(pfdev, GPU_INT_STAT);
+	u32 fault_status = gpu_read(pfdev, GPU_FAULT_STATUS);
+
+	if (!state)
+		return IRQ_NONE;
+
+	if (state & GPU_IRQ_MASK_ERROR) {
+		u64 address = (u64) gpu_read(pfdev, GPU_FAULT_ADDRESS_HI) << 32;
+		address |= gpu_read(pfdev, GPU_FAULT_ADDRESS_LO);
+
+		dev_warn(pfdev->dev, "GPU Fault 0x%08x (%s) at 0x%016llx\n",
+			 fault_status & 0xFF, panfrost_exception_name(pfdev, fault_status),
+			 address);
+
+		if (state & GPU_IRQ_MULTIPLE_FAULT)
+			dev_warn(pfdev->dev, "There were multiple GPU faults - some have not been reported\n");
+
+		gpu_write(pfdev, GPU_INT_MASK, 0);
+	}
+
+	gpu_write(pfdev, GPU_INT_CLEAR, state);
+
+	return IRQ_HANDLED;
+}
+
+int panfrost_gpu_soft_reset(struct panfrost_device *pfdev)
+{
+	int ret;
+	u32 val;
+
+	gpu_write(pfdev, GPU_INT_MASK, 0);
+	gpu_write(pfdev, GPU_INT_CLEAR, GPU_IRQ_RESET_COMPLETED);
+	gpu_write(pfdev, GPU_CMD, GPU_CMD_SOFT_RESET);
+
+	ret = readl_relaxed_poll_timeout(pfdev->iomem + GPU_INT_RAWSTAT,
+		val, val & GPU_IRQ_RESET_COMPLETED, 100, 10000);
+
+	if (ret) {
+		dev_err(pfdev->dev, "gpu soft reset timed out\n");
+		return ret;
+	}
+
+	gpu_write(pfdev, GPU_INT_CLEAR, GPU_IRQ_MASK_ALL);
+	gpu_write(pfdev, GPU_INT_MASK, GPU_IRQ_MASK_ALL);
+
+	return 0;
+}
+
+static void panfrost_gpu_init_quirks(struct panfrost_device *pfdev)
+{
+	u32 quirks = 0;
+
+	if (panfrost_has_hw_issue(pfdev, HW_ISSUE_8443) ||
+	    panfrost_has_hw_issue(pfdev, HW_ISSUE_11035))
+		quirks |= SC_LS_PAUSEBUFFER_DISABLE;
+
+	if (panfrost_has_hw_issue(pfdev, HW_ISSUE_10327))
+		quirks |= SC_SDC_DISABLE_OQ_DISCARD;
+
+	if (panfrost_has_hw_issue(pfdev, HW_ISSUE_10797))
+		quirks |= SC_ENABLE_TEXGRD_FLAGS;
+
+	if (!panfrost_has_hw_issue(pfdev, GPUCORE_1619)) {
+		if (panfrost_model_cmp(pfdev, 0x750) < 0) /* T60x, T62x, T72x */
+			quirks |= SC_LS_ATTR_CHECK_DISABLE;
+		else if (panfrost_model_cmp(pfdev, 0x880) <= 0) /* T76x, T8xx */
+			quirks |= SC_LS_ALLOW_ATTR_TYPES;
+	}
+
+	if (panfrost_has_hw_feature(pfdev, HW_FEATURE_TLS_HASHING))
+		quirks |= SC_TLS_HASH_ENABLE;
+
+	if (quirks)
+		gpu_write(pfdev, GPU_SHADER_CONFIG, quirks);
+
+
+	quirks = gpu_read(pfdev, GPU_TILER_CONFIG);
+
+	/* Set tiler clock gate override if required */
+	if (panfrost_has_hw_issue(pfdev, HW_ISSUE_T76X_3953))
+		quirks |= TC_CLOCK_GATE_OVERRIDE;
+
+	gpu_write(pfdev, GPU_TILER_CONFIG, quirks);
+
+
+	quirks = gpu_read(pfdev, GPU_L2_MMU_CONFIG);
+
+	/* Limit read & write ID width for AXI */
+	if (panfrost_has_hw_feature(pfdev, HW_FEATURE_3BIT_EXT_RW_L2_MMU_CONFIG))
+		quirks &= ~(L2_MMU_CONFIG_3BIT_LIMIT_EXTERNAL_READS |
+			    L2_MMU_CONFIG_3BIT_LIMIT_EXTERNAL_WRITES);
+	else
+		quirks &= ~(L2_MMU_CONFIG_LIMIT_EXTERNAL_READS |
+			    L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES);
+
+	gpu_write(pfdev, GPU_L2_MMU_CONFIG, quirks);
+
+	quirks = 0;
+	if ((panfrost_model_eq(pfdev, 0x860) || panfrost_model_eq(pfdev, 0x880)) &&
+	    pfdev->features.revision >= 0x2000)
+		quirks |= JM_MAX_JOB_THROTTLE_LIMIT << JM_JOB_THROTTLE_LIMIT_SHIFT;
+	else if (panfrost_model_eq(pfdev, 0x6000) &&
+		 pfdev->features.coherency_features == COHERENCY_ACE)
+		quirks |= (COHERENCY_ACE_LITE | COHERENCY_ACE) <<
+			   JM_FORCE_COHERENCY_FEATURES_SHIFT;
+
+	if (quirks)
+		gpu_write(pfdev, GPU_JM_CONFIG, quirks);
+}
+
+#define MAX_HW_REVS 6
+
+struct panfrost_model {
+	const char *name;
+	u32 id;
+	u32 id_mask;
+	u64 features;
+	u64 issues;
+	struct {
+		u32 revision;
+		u64 issues;
+	} revs[MAX_HW_REVS];
+};
+
+#define GPU_MODEL(_name, _id, ...) \
+{\
+	.name = __stringify(_name),				\
+	.id = _id,						\
+	.features = hw_features_##_name,			\
+	.issues = hw_issues_##_name,				\
+	.revs = { __VA_ARGS__ },				\
+}
+
+#define GPU_REV_EXT(name, _rev, _p, _s, stat) \
+{\
+	.revision = (_rev) << 12 | (_p) << 4 | (_s),		\
+	.issues = hw_issues_##name##_r##_rev##p##_p##stat,	\
+}
+#define GPU_REV(name, r, p) GPU_REV_EXT(name, r, p, 0, )
+
+static const struct panfrost_model gpu_models[] = {
+	/* T60x has an oddball version */
+	GPU_MODEL(t600, 0x600,
+		GPU_REV_EXT(t600, 0, 0, 1, _15dev0)),
+	GPU_MODEL(t620, 0x620,
+		GPU_REV(t620, 0, 1), GPU_REV(t620, 1, 0)),
+	GPU_MODEL(t720, 0x720),
+	GPU_MODEL(t760, 0x750,
+		GPU_REV(t760, 0, 0), GPU_REV(t760, 0, 1),
+		GPU_REV_EXT(t760, 0, 1, 0, _50rel0),
+		GPU_REV(t760, 0, 2), GPU_REV(t760, 0, 3)),
+	GPU_MODEL(t820, 0x820),
+	GPU_MODEL(t830, 0x830),
+	GPU_MODEL(t860, 0x860),
+	GPU_MODEL(t880, 0x880),
+
+	GPU_MODEL(g71, 0x6000,
+		GPU_REV_EXT(g71, 0, 0, 1, _05dev0)),
+	GPU_MODEL(g72, 0x6001),
+	GPU_MODEL(g51, 0x7000),
+	GPU_MODEL(g76, 0x7001),
+	GPU_MODEL(g52, 0x7002),
+	GPU_MODEL(g31, 0x7003,
+		GPU_REV(g31, 1, 0)),
+};
+
+static void panfrost_gpu_init_features(struct panfrost_device *pfdev)
+{
+	u32 gpu_id, num_js, major, minor, status, rev;
+	const char *name = "unknown";
+	u64 hw_feat = 0;
+	u64 hw_issues = hw_issues_all;
+	const struct panfrost_model *model;
+	int i;
+
+	pfdev->features.l2_features = gpu_read(pfdev, GPU_L2_FEATURES);
+	pfdev->features.core_features = gpu_read(pfdev, GPU_CORE_FEATURES);
+	pfdev->features.tiler_features = gpu_read(pfdev, GPU_TILER_FEATURES);
+	pfdev->features.mem_features = gpu_read(pfdev, GPU_MEM_FEATURES);
+	pfdev->features.mmu_features = gpu_read(pfdev, GPU_MMU_FEATURES);
+	pfdev->features.thread_features = gpu_read(pfdev, GPU_THREAD_FEATURES);
+	pfdev->features.coherency_features = gpu_read(pfdev, GPU_COHERENCY_FEATURES);
+	for (i = 0; i < 4; i++)
+		pfdev->features.texture_features[i] = gpu_read(pfdev, GPU_TEXTURE_FEATURES(i));
+
+	pfdev->features.as_present = gpu_read(pfdev, GPU_AS_PRESENT);
+
+	pfdev->features.js_present = gpu_read(pfdev, GPU_JS_PRESENT);
+	num_js = hweight32(pfdev->features.js_present);
+	for (i = 0; i < num_js; i++)
+		pfdev->features.js_features[i] = gpu_read(pfdev, GPU_JS_FEATURES(i));
+
+	pfdev->features.shader_present = gpu_read(pfdev, GPU_SHADER_PRESENT_LO);
+	pfdev->features.shader_present |= (u64)gpu_read(pfdev, GPU_SHADER_PRESENT_HI) << 32;
+
+	pfdev->features.tiler_present = gpu_read(pfdev, GPU_TILER_PRESENT_LO);
+	pfdev->features.tiler_present |= (u64)gpu_read(pfdev, GPU_TILER_PRESENT_HI) << 32;
+
+	pfdev->features.l2_present = gpu_read(pfdev, GPU_L2_PRESENT_LO);
+	pfdev->features.l2_present |= (u64)gpu_read(pfdev, GPU_L2_PRESENT_HI) << 32;
+	pfdev->features.nr_core_groups = hweight64(pfdev->features.l2_present);
+
+	pfdev->features.stack_present = gpu_read(pfdev, GPU_STACK_PRESENT_LO);
+	pfdev->features.stack_present |= (u64)gpu_read(pfdev, GPU_STACK_PRESENT_HI) << 32;
+
+	gpu_id = gpu_read(pfdev, GPU_ID);
+	pfdev->features.revision = gpu_id & 0xffff;
+	pfdev->features.id = gpu_id >> 16;
+
+	/* The T60x has an oddball ID value. Fix it up to the standard Midgard
+	 * format so we (and userspace) don't have to special case it.
+	 */
+	if (pfdev->features.id == 0x6956)
+		pfdev->features.id = 0x0600;
+
+	major = (pfdev->features.revision >> 12) & 0xf;
+	minor = (pfdev->features.revision >> 4) & 0xff;
+	status = pfdev->features.revision & 0xf;
+	rev = pfdev->features.revision;
+
+	gpu_id = pfdev->features.id;
+
+	for (model = gpu_models; model->name; model++) {
+		int best = -1;
+
+		if (!panfrost_model_eq(pfdev, model->id))
+			continue;
+
+		name = model->name;
+		hw_feat = model->features;
+		hw_issues |= model->issues;
+		for (i = 0; i < MAX_HW_REVS; i++) {
+			if (model->revs[i].revision == rev) {
+				best = i;
+				break;
+			} else if (model->revs[i].revision == (rev & ~0xf))
+				best = i;
+		}
+
+		if (best >= 0)
+			hw_issues |= model->revs[best].issues;
+
+		break;
+	}
+
+	bitmap_from_u64(pfdev->features.hw_features, hw_feat);
+	bitmap_from_u64(pfdev->features.hw_issues, hw_issues);
+
+	dev_info(pfdev->dev, "mali-%s id 0x%x major 0x%x minor 0x%x status 0x%x",
+		 name, gpu_id, major, minor, status);
+	dev_info(pfdev->dev, "features: %64pb, issues: %64pb",
+		 pfdev->features.hw_features,
+		 pfdev->features.hw_issues);
+
+	dev_info(pfdev->dev, "Features: L2:0x%08x Shader:0x%08x Tiler:0x%08x Mem:0x%0x MMU:0x%08x AS:0x%x JS:0x%x",
+		 pfdev->features.l2_features,
+		 pfdev->features.core_features,
+		 pfdev->features.tiler_features,
+		 pfdev->features.mem_features,
+		 pfdev->features.mmu_features,
+		 pfdev->features.as_present,
+		 pfdev->features.js_present);
+
+	dev_info(pfdev->dev, "shader_present=0x%0llx l2_present=0x%0llx",
+		 pfdev->features.shader_present, pfdev->features.l2_present);
+}
+
+void panfrost_gpu_power_on(struct panfrost_device *pfdev)
+{
+	int ret;
+	u32 val;
+
+	/* Just turn on everything for now */
+	gpu_write(pfdev, L2_PWRON_LO, pfdev->features.l2_present);
+	ret = readl_relaxed_poll_timeout(pfdev->iomem + L2_READY_LO,
+		val, val == pfdev->features.l2_present, 100, 1000);
+
+	gpu_write(pfdev, STACK_PWRON_LO, pfdev->features.stack_present);
+	ret |= readl_relaxed_poll_timeout(pfdev->iomem + STACK_READY_LO,
+		val, val == pfdev->features.stack_present, 100, 1000);
+
+	gpu_write(pfdev, SHADER_PWRON_LO, pfdev->features.shader_present);
+	ret |= readl_relaxed_poll_timeout(pfdev->iomem + SHADER_READY_LO,
+		val, val == pfdev->features.shader_present, 100, 1000);
+
+	gpu_write(pfdev, TILER_PWRON_LO, pfdev->features.tiler_present);
+	ret |= readl_relaxed_poll_timeout(pfdev->iomem + TILER_READY_LO,
+		val, val == pfdev->features.tiler_present, 100, 1000);
+
+	if (ret)
+		dev_err(pfdev->dev, "error powering up gpu");
+}
+
+void panfrost_gpu_power_off(struct panfrost_device *pfdev)
+{
+	gpu_write(pfdev, TILER_PWROFF_LO, 0);
+	gpu_write(pfdev, SHADER_PWROFF_LO, 0);
+	gpu_write(pfdev, STACK_PWROFF_LO, 0);
+	gpu_write(pfdev, L2_PWROFF_LO, 0);
+}
+
+int panfrost_gpu_init(struct panfrost_device *pfdev)
+{
+	int err, irq;
+
+	err = panfrost_gpu_soft_reset(pfdev);
+	if (err)
+		return err;
+
+	panfrost_gpu_init_features(pfdev);
+
+	dma_set_mask_and_coherent(pfdev->dev,
+		DMA_BIT_MASK(FIELD_GET(0xff00, pfdev->features.mmu_features)));
+
+	irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "gpu");
+	if (irq <= 0)
+		return -ENODEV;
+
+	err = devm_request_irq(pfdev->dev, irq, panfrost_gpu_irq_handler,
+			       IRQF_SHARED, "gpu", pfdev);
+	if (err) {
+		dev_err(pfdev->dev, "failed to request gpu irq");
+		return err;
+	}
+
+	panfrost_gpu_init_quirks(pfdev);
+	panfrost_gpu_power_on(pfdev);
+
+	return 0;
+}
+
+void panfrost_gpu_fini(struct panfrost_device *pfdev)
+{
+	panfrost_gpu_power_off(pfdev);
+}
+
+u32 panfrost_gpu_get_latest_flush_id(struct panfrost_device *pfdev)
+{
+	if (panfrost_has_hw_feature(pfdev, HW_FEATURE_FLUSH_REDUCTION))
+		return gpu_read(pfdev, GPU_LATEST_FLUSH_ID);
+	return 0;
+}
diff --git a/drivers/gpu/drm/panfrost/panfrost_gpu.h b/drivers/gpu/drm/panfrost/panfrost_gpu.h
new file mode 100644
index 000000000000..4112412087b2
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/panfrost_gpu.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */
+/* Copyright 2019 Collabora ltd. */
+
+#ifndef __PANFROST_GPU_H__
+#define __PANFROST_GPU_H__
+
+struct panfrost_device;
+
+int panfrost_gpu_init(struct panfrost_device *pfdev);
+void panfrost_gpu_fini(struct panfrost_device *pfdev);
+
+u32 panfrost_gpu_get_latest_flush_id(struct panfrost_device *pfdev);
+
+int panfrost_gpu_soft_reset(struct panfrost_device *pfdev);
+void panfrost_gpu_power_on(struct panfrost_device *pfdev);
+void panfrost_gpu_power_off(struct panfrost_device *pfdev);
+
+#endif
diff --git a/drivers/gpu/drm/panfrost/panfrost_issues.h b/drivers/gpu/drm/panfrost/panfrost_issues.h
new file mode 100644
index 000000000000..cec6dcdadb5c
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/panfrost_issues.h
@@ -0,0 +1,176 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* (C) COPYRIGHT 2014-2018 ARM Limited. All rights reserved. */
+/* Copyright 2019 Linaro, Ltd., Rob Herring <robh@kernel.org> */
+#ifndef __PANFROST_ISSUES_H__
+#define __PANFROST_ISSUES_H__
+
+#include <linux/bitops.h>
+
+#include "panfrost_device.h"
+
+/*
+ * This is not a complete list of issues, but only the ones the driver needs
+ * to care about.
+ */
+enum panfrost_hw_issue {
+	HW_ISSUE_6367,
+	HW_ISSUE_6787,
+	HW_ISSUE_8186,
+	HW_ISSUE_8245,
+	HW_ISSUE_8316,
+	HW_ISSUE_8394,
+	HW_ISSUE_8401,
+	HW_ISSUE_8408,
+	HW_ISSUE_8443,
+	HW_ISSUE_8987,
+	HW_ISSUE_9435,
+	HW_ISSUE_9510,
+	HW_ISSUE_9630,
+	HW_ISSUE_10327,
+	HW_ISSUE_10649,
+	HW_ISSUE_10676,
+	HW_ISSUE_10797,
+	HW_ISSUE_10817,
+	HW_ISSUE_10883,
+	HW_ISSUE_10959,
+	HW_ISSUE_10969,
+	HW_ISSUE_11020,
+	HW_ISSUE_11024,
+	HW_ISSUE_11035,
+	HW_ISSUE_11056,
+	HW_ISSUE_T76X_3542,
+	HW_ISSUE_T76X_3953,
+	HW_ISSUE_TMIX_8463,
+	GPUCORE_1619,
+	HW_ISSUE_TMIX_8438,
+	HW_ISSUE_TGOX_R1_1234,
+	HW_ISSUE_END
+};
+
+#define hw_issues_all (\
+	BIT_ULL(HW_ISSUE_9435))
+
+#define hw_issues_t600 (\
+	BIT_ULL(HW_ISSUE_6367) | \
+	BIT_ULL(HW_ISSUE_6787) | \
+	BIT_ULL(HW_ISSUE_8408) | \
+	BIT_ULL(HW_ISSUE_9510) | \
+	BIT_ULL(HW_ISSUE_10649) | \
+	BIT_ULL(HW_ISSUE_10676) | \
+	BIT_ULL(HW_ISSUE_10883) | \
+	BIT_ULL(HW_ISSUE_11020) | \
+	BIT_ULL(HW_ISSUE_11035) | \
+	BIT_ULL(HW_ISSUE_11056) | \
+	BIT_ULL(HW_ISSUE_TMIX_8438))
+
+#define hw_issues_t600_r0p0_15dev0 (\
+	BIT_ULL(HW_ISSUE_8186) | \
+	BIT_ULL(HW_ISSUE_8245) | \
+	BIT_ULL(HW_ISSUE_8316) | \
+	BIT_ULL(HW_ISSUE_8394) | \
+	BIT_ULL(HW_ISSUE_8401) | \
+	BIT_ULL(HW_ISSUE_8443) | \
+	BIT_ULL(HW_ISSUE_8987) | \
+	BIT_ULL(HW_ISSUE_9630) | \
+	BIT_ULL(HW_ISSUE_10969) | \
+	BIT_ULL(GPUCORE_1619))
+
+#define hw_issues_t620 (\
+	BIT_ULL(HW_ISSUE_10649) | \
+	BIT_ULL(HW_ISSUE_10883) | \
+	BIT_ULL(HW_ISSUE_10959) | \
+	BIT_ULL(HW_ISSUE_11056) | \
+	BIT_ULL(HW_ISSUE_TMIX_8438))
+
+#define hw_issues_t620_r0p1 (\
+	BIT_ULL(HW_ISSUE_10327) | \
+	BIT_ULL(HW_ISSUE_10676) | \
+	BIT_ULL(HW_ISSUE_10817) | \
+	BIT_ULL(HW_ISSUE_11020) | \
+	BIT_ULL(HW_ISSUE_11024) | \
+	BIT_ULL(HW_ISSUE_11035))
+
+#define hw_issues_t620_r1p0 (\
+	BIT_ULL(HW_ISSUE_11020) | \
+	BIT_ULL(HW_ISSUE_11024))
+
+#define hw_issues_t720 (\
+	BIT_ULL(HW_ISSUE_10649) | \
+	BIT_ULL(HW_ISSUE_10797) | \
+	BIT_ULL(HW_ISSUE_10883) | \
+	BIT_ULL(HW_ISSUE_11056) | \
+	BIT_ULL(HW_ISSUE_TMIX_8438))
+
+#define hw_issues_t760 (\
+	BIT_ULL(HW_ISSUE_10883) | \
+	BIT_ULL(HW_ISSUE_T76X_3953) | \
+	BIT_ULL(HW_ISSUE_TMIX_8438))
+
+#define hw_issues_t760_r0p0 (\
+	BIT_ULL(HW_ISSUE_11020) | \
+	BIT_ULL(HW_ISSUE_11024) | \
+	BIT_ULL(HW_ISSUE_T76X_3542))
+
+#define hw_issues_t760_r0p1 (\
+	BIT_ULL(HW_ISSUE_11020) | \
+	BIT_ULL(HW_ISSUE_11024) | \
+	BIT_ULL(HW_ISSUE_T76X_3542))
+
+#define hw_issues_t760_r0p1_50rel0 (\
+	BIT_ULL(HW_ISSUE_T76X_3542))
+
+#define hw_issues_t760_r0p2 (\
+	BIT_ULL(HW_ISSUE_11020) | \
+	BIT_ULL(HW_ISSUE_11024) | \
+	BIT_ULL(HW_ISSUE_T76X_3542))
+
+#define hw_issues_t760_r0p3 (\
+	BIT_ULL(HW_ISSUE_T76X_3542))
+
+#define hw_issues_t820 (\
+	BIT_ULL(HW_ISSUE_10883) | \
+	BIT_ULL(HW_ISSUE_T76X_3953) | \
+	BIT_ULL(HW_ISSUE_TMIX_8438))
+
+#define hw_issues_t830 (\
+	BIT_ULL(HW_ISSUE_10883) | \
+	BIT_ULL(HW_ISSUE_T76X_3953) | \
+	BIT_ULL(HW_ISSUE_TMIX_8438))
+
+#define hw_issues_t860 (\
+	BIT_ULL(HW_ISSUE_10883) | \
+	BIT_ULL(HW_ISSUE_T76X_3953) | \
+	BIT_ULL(HW_ISSUE_TMIX_8438))
+
+#define hw_issues_t880 (\
+	BIT_ULL(HW_ISSUE_10883) | \
+	BIT_ULL(HW_ISSUE_T76X_3953) | \
+	BIT_ULL(HW_ISSUE_TMIX_8438))
+
+#define hw_issues_g31 0
+
+#define hw_issues_g31_r1p0 (\
+	BIT_ULL(HW_ISSUE_TGOX_R1_1234))
+
+#define hw_issues_g51 0
+
+#define hw_issues_g52 0
+
+#define hw_issues_g71 (\
+	BIT_ULL(HW_ISSUE_TMIX_8463) | \
+	BIT_ULL(HW_ISSUE_TMIX_8438))
+
+#define hw_issues_g71_r0p0_05dev0 (\
+	BIT_ULL(HW_ISSUE_T76X_3953))
+
+#define hw_issues_g72 0
+
+#define hw_issues_g76 0
+
+static inline bool panfrost_has_hw_issue(struct panfrost_device *pfdev,
+					 enum panfrost_hw_issue issue)
+{
+	return test_bit(issue, pfdev->features.hw_issues);
+}
+
+#endif /* __PANFROST_ISSUES_H__ */
diff --git a/drivers/gpu/drm/panfrost/panfrost_job.c b/drivers/gpu/drm/panfrost/panfrost_job.c
new file mode 100644
index 000000000000..a5716c8fe8b3
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/panfrost_job.c
@@ -0,0 +1,564 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
+/* Copyright 2019 Collabora ltd. */
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/reservation.h>
+#include <drm/gpu_scheduler.h>
+#include <drm/panfrost_drm.h>
+
+#include "panfrost_device.h"
+#include "panfrost_devfreq.h"
+#include "panfrost_job.h"
+#include "panfrost_features.h"
+#include "panfrost_issues.h"
+#include "panfrost_gem.h"
+#include "panfrost_regs.h"
+#include "panfrost_gpu.h"
+#include "panfrost_mmu.h"
+
+#define job_write(dev, reg, data) writel(data, dev->iomem + (reg))
+#define job_read(dev, reg) readl(dev->iomem + (reg))
+
+struct panfrost_queue_state {
+	struct drm_gpu_scheduler sched;
+
+	u64 fence_context;
+	u64 emit_seqno;
+};
+
+struct panfrost_job_slot {
+	struct panfrost_queue_state queue[NUM_JOB_SLOTS];
+	spinlock_t job_lock;
+};
+
+static struct panfrost_job *
+to_panfrost_job(struct drm_sched_job *sched_job)
+{
+	return container_of(sched_job, struct panfrost_job, base);
+}
+
+struct panfrost_fence {
+	struct dma_fence base;
+	struct drm_device *dev;
+	/* panfrost seqno for signaled() test */
+	u64 seqno;
+	int queue;
+};
+
+static inline struct panfrost_fence *
+to_panfrost_fence(struct dma_fence *fence)
+{
+	return (struct panfrost_fence *)fence;
+}
+
+static const char *panfrost_fence_get_driver_name(struct dma_fence *fence)
+{
+	return "panfrost";
+}
+
+static const char *panfrost_fence_get_timeline_name(struct dma_fence *fence)
+{
+	struct panfrost_fence *f = to_panfrost_fence(fence);
+
+	switch (f->queue) {
+	case 0:
+		return "panfrost-js-0";
+	case 1:
+		return "panfrost-js-1";
+	case 2:
+		return "panfrost-js-2";
+	default:
+		return NULL;
+	}
+}
+
+static const struct dma_fence_ops panfrost_fence_ops = {
+	.get_driver_name = panfrost_fence_get_driver_name,
+	.get_timeline_name = panfrost_fence_get_timeline_name,
+};
+
+static struct dma_fence *panfrost_fence_create(struct panfrost_device *pfdev, int js_num)
+{
+	struct panfrost_fence *fence;
+	struct panfrost_job_slot *js = pfdev->js;
+
+	fence = kzalloc(sizeof(*fence), GFP_KERNEL);
+	if (!fence)
+		return ERR_PTR(-ENOMEM);
+
+	fence->dev = pfdev->ddev;
+	fence->queue = js_num;
+	fence->seqno = ++js->queue[js_num].emit_seqno;
+	dma_fence_init(&fence->base, &panfrost_fence_ops, &js->job_lock,
+		       js->queue[js_num].fence_context, fence->seqno);
+
+	return &fence->base;
+}
+
+static int panfrost_job_get_slot(struct panfrost_job *job)
+{
+	/* JS0: fragment jobs.
+	 * JS1: vertex/tiler jobs
+	 * JS2: compute jobs
+	 */
+	if (job->requirements & PANFROST_JD_REQ_FS)
+		return 0;
+
+/* Not exposed to userspace yet */
+#if 0
+	if (job->requirements & PANFROST_JD_REQ_ONLY_COMPUTE) {
+		if ((job->requirements & PANFROST_JD_REQ_CORE_GRP_MASK) &&
+		    (job->pfdev->features.nr_core_groups == 2))
+			return 2;
+		if (panfrost_has_hw_issue(job->pfdev, HW_ISSUE_8987))
+			return 2;
+	}
+#endif
+	return 1;
+}
+
+static void panfrost_job_write_affinity(struct panfrost_device *pfdev,
+					u32 requirements,
+					int js)
+{
+	u64 affinity;
+
+	/*
+	 * Use all cores for now.
+	 * Eventually we may need to support tiler only jobs and h/w with
+	 * multiple (2) coherent core groups
+	 */
+	affinity = pfdev->features.shader_present;
+
+	job_write(pfdev, JS_AFFINITY_NEXT_LO(js), affinity & 0xFFFFFFFF);
+	job_write(pfdev, JS_AFFINITY_NEXT_HI(js), affinity >> 32);
+}
+
+static void panfrost_job_hw_submit(struct panfrost_job *job, int js)
+{
+	struct panfrost_device *pfdev = job->pfdev;
+	unsigned long flags;
+	u32 cfg;
+	u64 jc_head = job->jc;
+	int ret;
+
+	ret = pm_runtime_get_sync(pfdev->dev);
+	if (ret < 0)
+		return;
+
+	if (WARN_ON(job_read(pfdev, JS_COMMAND_NEXT(js))))
+		goto end;
+
+	panfrost_devfreq_record_transition(pfdev, js);
+	spin_lock_irqsave(&pfdev->hwaccess_lock, flags);
+
+	job_write(pfdev, JS_HEAD_NEXT_LO(js), jc_head & 0xFFFFFFFF);
+	job_write(pfdev, JS_HEAD_NEXT_HI(js), jc_head >> 32);
+
+	panfrost_job_write_affinity(pfdev, job->requirements, js);
+
+	/* start MMU, medium priority, cache clean/flush on end, clean/flush on
+	 * start */
+	/* TODO: different address spaces */
+	cfg = JS_CONFIG_THREAD_PRI(8) |
+		JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE |
+		JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE;
+
+	if (panfrost_has_hw_feature(pfdev, HW_FEATURE_FLUSH_REDUCTION))
+		cfg |= JS_CONFIG_ENABLE_FLUSH_REDUCTION;
+
+	if (panfrost_has_hw_issue(pfdev, HW_ISSUE_10649))
+		cfg |= JS_CONFIG_START_MMU;
+
+	job_write(pfdev, JS_CONFIG_NEXT(js), cfg);
+
+	if (panfrost_has_hw_feature(pfdev, HW_FEATURE_FLUSH_REDUCTION))
+		job_write(pfdev, JS_FLUSH_ID_NEXT(js), job->flush_id);
+
+	/* GO ! */
+	dev_dbg(pfdev->dev, "JS: Submitting atom %p to js[%d] with head=0x%llx",
+				job, js, jc_head);
+
+	job_write(pfdev, JS_COMMAND_NEXT(js), JS_COMMAND_START);
+
+	spin_unlock_irqrestore(&pfdev->hwaccess_lock, flags);
+
+end:
+	pm_runtime_mark_last_busy(pfdev->dev);
+	pm_runtime_put_autosuspend(pfdev->dev);
+}
+
+static void panfrost_acquire_object_fences(struct drm_gem_object **bos,
+					   int bo_count,
+					   struct dma_fence **implicit_fences)
+{
+	int i;
+
+	for (i = 0; i < bo_count; i++)
+		implicit_fences[i] = reservation_object_get_excl_rcu(bos[i]->resv);
+}
+
+static void panfrost_attach_object_fences(struct drm_gem_object **bos,
+					  int bo_count,
+					  struct dma_fence *fence)
+{
+	int i;
+
+	for (i = 0; i < bo_count; i++)
+		reservation_object_add_excl_fence(bos[i]->resv, fence);
+}
+
+int panfrost_job_push(struct panfrost_job *job)
+{
+	struct panfrost_device *pfdev = job->pfdev;
+	int slot = panfrost_job_get_slot(job);
+	struct drm_sched_entity *entity = &job->file_priv->sched_entity[slot];
+	struct ww_acquire_ctx acquire_ctx;
+	int ret = 0;
+
+	mutex_lock(&pfdev->sched_lock);
+
+	ret = drm_gem_lock_reservations(job->bos, job->bo_count,
+					    &acquire_ctx);
+	if (ret) {
+		mutex_unlock(&pfdev->sched_lock);
+		return ret;
+	}
+
+	ret = drm_sched_job_init(&job->base, entity, NULL);
+	if (ret) {
+		mutex_unlock(&pfdev->sched_lock);
+		goto unlock;
+	}
+
+	job->render_done_fence = dma_fence_get(&job->base.s_fence->finished);
+
+	kref_get(&job->refcount); /* put by scheduler job completion */
+
+	panfrost_acquire_object_fences(job->bos, job->bo_count,
+				       job->implicit_fences);
+
+	drm_sched_entity_push_job(&job->base, entity);
+
+	mutex_unlock(&pfdev->sched_lock);
+
+	panfrost_attach_object_fences(job->bos, job->bo_count,
+				      job->render_done_fence);
+
+unlock:
+	drm_gem_unlock_reservations(job->bos, job->bo_count, &acquire_ctx);
+
+	return ret;
+}
+
+static void panfrost_job_cleanup(struct kref *ref)
+{
+	struct panfrost_job *job = container_of(ref, struct panfrost_job,
+						refcount);
+	unsigned int i;
+
+	if (job->in_fences) {
+		for (i = 0; i < job->in_fence_count; i++)
+			dma_fence_put(job->in_fences[i]);
+		kvfree(job->in_fences);
+	}
+	if (job->implicit_fences) {
+		for (i = 0; i < job->bo_count; i++)
+			dma_fence_put(job->implicit_fences[i]);
+		kvfree(job->implicit_fences);
+	}
+	dma_fence_put(job->done_fence);
+	dma_fence_put(job->render_done_fence);
+
+	if (job->bos) {
+		for (i = 0; i < job->bo_count; i++)
+			drm_gem_object_put_unlocked(job->bos[i]);
+		kvfree(job->bos);
+	}
+
+	kfree(job);
+}
+
+void panfrost_job_put(struct panfrost_job *job)
+{
+	kref_put(&job->refcount, panfrost_job_cleanup);
+}
+
+static void panfrost_job_free(struct drm_sched_job *sched_job)
+{
+	struct panfrost_job *job = to_panfrost_job(sched_job);
+
+	drm_sched_job_cleanup(sched_job);
+
+	panfrost_job_put(job);
+}
+
+static struct dma_fence *panfrost_job_dependency(struct drm_sched_job *sched_job,
+						 struct drm_sched_entity *s_entity)
+{
+	struct panfrost_job *job = to_panfrost_job(sched_job);
+	struct dma_fence *fence;
+	unsigned int i;
+
+	/* Explicit fences */
+	for (i = 0; i < job->in_fence_count; i++) {
+		if (job->in_fences[i]) {
+			fence = job->in_fences[i];
+			job->in_fences[i] = NULL;
+			return fence;
+		}
+	}
+
+	/* Implicit fences, max. one per BO */
+	for (i = 0; i < job->bo_count; i++) {
+		if (job->implicit_fences[i]) {
+			fence = job->implicit_fences[i];
+			job->implicit_fences[i] = NULL;
+			return fence;
+		}
+	}
+
+	return NULL;
+}
+
+static struct dma_fence *panfrost_job_run(struct drm_sched_job *sched_job)
+{
+	struct panfrost_job *job = to_panfrost_job(sched_job);
+	struct panfrost_device *pfdev = job->pfdev;
+	int slot = panfrost_job_get_slot(job);
+	struct dma_fence *fence = NULL;
+
+	if (unlikely(job->base.s_fence->finished.error))
+		return NULL;
+
+	pfdev->jobs[slot] = job;
+
+	fence = panfrost_fence_create(pfdev, slot);
+	if (IS_ERR(fence))
+		return NULL;
+
+	if (job->done_fence)
+		dma_fence_put(job->done_fence);
+	job->done_fence = dma_fence_get(fence);
+
+	panfrost_job_hw_submit(job, slot);
+
+	return fence;
+}
+
+void panfrost_job_enable_interrupts(struct panfrost_device *pfdev)
+{
+	int j;
+	u32 irq_mask = 0;
+
+	for (j = 0; j < NUM_JOB_SLOTS; j++) {
+		irq_mask |= MK_JS_MASK(j);
+	}
+
+	job_write(pfdev, JOB_INT_CLEAR, irq_mask);
+	job_write(pfdev, JOB_INT_MASK, irq_mask);
+}
+
+static void panfrost_job_timedout(struct drm_sched_job *sched_job)
+{
+	struct panfrost_job *job = to_panfrost_job(sched_job);
+	struct panfrost_device *pfdev = job->pfdev;
+	int js = panfrost_job_get_slot(job);
+	int i;
+
+	/*
+	 * If the GPU managed to complete this jobs fence, the timeout is
+	 * spurious. Bail out.
+	 */
+	if (dma_fence_is_signaled(job->done_fence))
+		return;
+
+	dev_err(pfdev->dev, "gpu sched timeout, js=%d, status=0x%x, head=0x%x, tail=0x%x, sched_job=%p",
+		js,
+		job_read(pfdev, JS_STATUS(js)),
+		job_read(pfdev, JS_HEAD_LO(js)),
+		job_read(pfdev, JS_TAIL_LO(js)),
+		sched_job);
+
+	mutex_lock(&pfdev->reset_lock);
+
+	for (i = 0; i < NUM_JOB_SLOTS; i++)
+		drm_sched_stop(&pfdev->js->queue[i].sched);
+
+	if (sched_job)
+		drm_sched_increase_karma(sched_job);
+
+	/* panfrost_core_dump(pfdev); */
+
+	panfrost_devfreq_record_transition(pfdev, js);
+	panfrost_gpu_soft_reset(pfdev);
+
+	/* TODO: Re-enable all other address spaces */
+	panfrost_mmu_enable(pfdev, 0);
+	panfrost_gpu_power_on(pfdev);
+	panfrost_job_enable_interrupts(pfdev);
+
+	for (i = 0; i < NUM_JOB_SLOTS; i++)
+		drm_sched_resubmit_jobs(&pfdev->js->queue[i].sched);
+
+	/* restart scheduler after GPU is usable again */
+	for (i = 0; i < NUM_JOB_SLOTS; i++)
+		drm_sched_start(&pfdev->js->queue[i].sched, true);
+
+	mutex_unlock(&pfdev->reset_lock);
+}
+
+static const struct drm_sched_backend_ops panfrost_sched_ops = {
+	.dependency = panfrost_job_dependency,
+	.run_job = panfrost_job_run,
+	.timedout_job = panfrost_job_timedout,
+	.free_job = panfrost_job_free
+};
+
+static irqreturn_t panfrost_job_irq_handler(int irq, void *data)
+{
+	struct panfrost_device *pfdev = data;
+	u32 status = job_read(pfdev, JOB_INT_STAT);
+	int j;
+
+	dev_dbg(pfdev->dev, "jobslot irq status=%x\n", status);
+
+	if (!status)
+		return IRQ_NONE;
+
+	pm_runtime_mark_last_busy(pfdev->dev);
+
+	for (j = 0; status; j++) {
+		u32 mask = MK_JS_MASK(j);
+
+		if (!(status & mask))
+			continue;
+
+		job_write(pfdev, JOB_INT_CLEAR, mask);
+
+		if (status & JOB_INT_MASK_ERR(j)) {
+			job_write(pfdev, JS_COMMAND_NEXT(j), JS_COMMAND_NOP);
+
+			dev_err(pfdev->dev, "js fault, js=%d, status=%s, head=0x%x, tail=0x%x",
+				j,
+				panfrost_exception_name(pfdev, job_read(pfdev, JS_STATUS(j))),
+				job_read(pfdev, JS_HEAD_LO(j)),
+				job_read(pfdev, JS_TAIL_LO(j)));
+
+			drm_sched_fault(&pfdev->js->queue[j].sched);
+		}
+
+		if (status & JOB_INT_MASK_DONE(j)) {
+			panfrost_devfreq_record_transition(pfdev, j);
+			dma_fence_signal(pfdev->jobs[j]->done_fence);
+		}
+
+		status &= ~mask;
+	}
+
+	return IRQ_HANDLED;
+}
+
+int panfrost_job_init(struct panfrost_device *pfdev)
+{
+	struct panfrost_job_slot *js;
+	int ret, j, irq;
+
+	pfdev->js = js = devm_kzalloc(pfdev->dev, sizeof(*js), GFP_KERNEL);
+	if (!js)
+		return -ENOMEM;
+
+	spin_lock_init(&js->job_lock);
+
+	irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "job");
+	if (irq <= 0)
+		return -ENODEV;
+
+	ret = devm_request_irq(pfdev->dev, irq, panfrost_job_irq_handler,
+			       IRQF_SHARED, "job", pfdev);
+	if (ret) {
+		dev_err(pfdev->dev, "failed to request job irq");
+		return ret;
+	}
+
+	for (j = 0; j < NUM_JOB_SLOTS; j++) {
+		js->queue[j].fence_context = dma_fence_context_alloc(1);
+
+		ret = drm_sched_init(&js->queue[j].sched,
+				     &panfrost_sched_ops,
+				     1, 0, msecs_to_jiffies(500),
+				     "pan_js");
+		if (ret) {
+			dev_err(pfdev->dev, "Failed to create scheduler: %d.", ret);
+			goto err_sched;
+		}
+	}
+
+	panfrost_job_enable_interrupts(pfdev);
+
+	return 0;
+
+err_sched:
+	for (j--; j >= 0; j--)
+		drm_sched_fini(&js->queue[j].sched);
+
+	return ret;
+}
+
+void panfrost_job_fini(struct panfrost_device *pfdev)
+{
+	struct panfrost_job_slot *js = pfdev->js;
+	int j;
+
+	job_write(pfdev, JOB_INT_MASK, 0);
+
+	for (j = 0; j < NUM_JOB_SLOTS; j++)
+		drm_sched_fini(&js->queue[j].sched);
+
+}
+
+int panfrost_job_open(struct panfrost_file_priv *panfrost_priv)
+{
+	struct panfrost_device *pfdev = panfrost_priv->pfdev;
+	struct panfrost_job_slot *js = pfdev->js;
+	struct drm_sched_rq *rq;
+	int ret, i;
+
+	for (i = 0; i < NUM_JOB_SLOTS; i++) {
+		rq = &js->queue[i].sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL];
+		ret = drm_sched_entity_init(&panfrost_priv->sched_entity[i], &rq, 1, NULL);
+		if (WARN_ON(ret))
+			return ret;
+	}
+	return 0;
+}
+
+void panfrost_job_close(struct panfrost_file_priv *panfrost_priv)
+{
+	int i;
+
+	for (i = 0; i < NUM_JOB_SLOTS; i++)
+		drm_sched_entity_destroy(&panfrost_priv->sched_entity[i]);
+}
+
+int panfrost_job_is_idle(struct panfrost_device *pfdev)
+{
+	struct panfrost_job_slot *js = pfdev->js;
+	int i;
+
+	for (i = 0; i < NUM_JOB_SLOTS; i++) {
+		/* If there are any jobs in the HW queue, we're not idle */
+		if (atomic_read(&js->queue[i].sched.hw_rq_count))
+			return false;
+
+		/* Check whether the hardware is idle */
+		if (pfdev->devfreq.slot[i].busy)
+			return false;
+	}
+
+	return true;
+}
diff --git a/drivers/gpu/drm/panfrost/panfrost_job.h b/drivers/gpu/drm/panfrost/panfrost_job.h
new file mode 100644
index 000000000000..62454128a792
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/panfrost_job.h
@@ -0,0 +1,51 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright 2019 Collabora ltd. */
+
+#ifndef __PANFROST_JOB_H__
+#define __PANFROST_JOB_H__
+
+#include <uapi/drm/panfrost_drm.h>
+#include <drm/gpu_scheduler.h>
+
+struct panfrost_device;
+struct panfrost_gem_object;
+struct panfrost_file_priv;
+
+struct panfrost_job {
+	struct drm_sched_job base;
+
+	struct kref refcount;
+
+	struct panfrost_device *pfdev;
+	struct panfrost_file_priv *file_priv;
+
+	/* Optional fences userspace can pass in for the job to depend on. */
+	struct dma_fence **in_fences;
+	u32 in_fence_count;
+
+	/* Fence to be signaled by IRQ handler when the job is complete. */
+	struct dma_fence *done_fence;
+
+	__u64 jc;
+	__u32 requirements;
+	__u32 flush_id;
+
+	/* Exclusive fences we have taken from the BOs to wait for */
+	struct dma_fence **implicit_fences;
+	struct drm_gem_object **bos;
+	u32 bo_count;
+
+	/* Fence to be signaled by drm-sched once its done with the job */
+	struct dma_fence *render_done_fence;
+};
+
+int panfrost_job_init(struct panfrost_device *pfdev);
+void panfrost_job_fini(struct panfrost_device *pfdev);
+int panfrost_job_open(struct panfrost_file_priv *panfrost_priv);
+void panfrost_job_close(struct panfrost_file_priv *panfrost_priv);
+int panfrost_job_push(struct panfrost_job *job);
+void panfrost_job_put(struct panfrost_job *job);
+void panfrost_job_enable_interrupts(struct panfrost_device *pfdev);
+int panfrost_job_is_idle(struct panfrost_device *pfdev);
+
+#endif
diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.c b/drivers/gpu/drm/panfrost/panfrost_mmu.c
new file mode 100644
index 000000000000..762b1bd2a8c2
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/panfrost_mmu.c
@@ -0,0 +1,386 @@
+// SPDX-License-Identifier:	GPL-2.0
+/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
+#include <linux/bitfield.h>
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/io-pgtable.h>
+#include <linux/iommu.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/sizes.h>
+
+#include "panfrost_device.h"
+#include "panfrost_mmu.h"
+#include "panfrost_gem.h"
+#include "panfrost_features.h"
+#include "panfrost_regs.h"
+
+#define mmu_write(dev, reg, data) writel(data, dev->iomem + reg)
+#define mmu_read(dev, reg) readl(dev->iomem + reg)
+
+struct panfrost_mmu {
+	struct io_pgtable_cfg pgtbl_cfg;
+	struct io_pgtable_ops *pgtbl_ops;
+	struct mutex lock;
+};
+
+static int wait_ready(struct panfrost_device *pfdev, u32 as_nr)
+{
+	int ret;
+	u32 val;
+
+	/* Wait for the MMU status to indicate there is no active command, in
+	 * case one is pending. */
+	ret = readl_relaxed_poll_timeout_atomic(pfdev->iomem + AS_STATUS(as_nr),
+		val, !(val & AS_STATUS_AS_ACTIVE), 10, 1000);
+
+	if (ret)
+		dev_err(pfdev->dev, "AS_ACTIVE bit stuck\n");
+
+	return ret;
+}
+
+static int write_cmd(struct panfrost_device *pfdev, u32 as_nr, u32 cmd)
+{
+	int status;
+
+	/* write AS_COMMAND when MMU is ready to accept another command */
+	status = wait_ready(pfdev, as_nr);
+	if (!status)
+		mmu_write(pfdev, AS_COMMAND(as_nr), cmd);
+
+	return status;
+}
+
+static void lock_region(struct panfrost_device *pfdev, u32 as_nr,
+			u64 iova, size_t size)
+{
+	u8 region_width;
+	u64 region = iova & PAGE_MASK;
+	/*
+	 * fls returns:
+	 * 1 .. 32
+	 *
+	 * 10 + fls(num_pages)
+	 * results in the range (11 .. 42)
+	 */
+
+	size = round_up(size, PAGE_SIZE);
+
+	region_width = 10 + fls(size >> PAGE_SHIFT);
+	if ((size >> PAGE_SHIFT) != (1ul << (region_width - 11))) {
+		/* not pow2, so must go up to the next pow2 */
+		region_width += 1;
+	}
+	region |= region_width;
+
+	/* Lock the region that needs to be updated */
+	mmu_write(pfdev, AS_LOCKADDR_LO(as_nr), region & 0xFFFFFFFFUL);
+	mmu_write(pfdev, AS_LOCKADDR_HI(as_nr), (region >> 32) & 0xFFFFFFFFUL);
+	write_cmd(pfdev, as_nr, AS_COMMAND_LOCK);
+}
+
+
+static int mmu_hw_do_operation(struct panfrost_device *pfdev, u32 as_nr,
+		u64 iova, size_t size, u32 op)
+{
+	unsigned long flags;
+	int ret;
+
+	spin_lock_irqsave(&pfdev->hwaccess_lock, flags);
+
+	if (op != AS_COMMAND_UNLOCK)
+		lock_region(pfdev, as_nr, iova, size);
+
+	/* Run the MMU operation */
+	write_cmd(pfdev, as_nr, op);
+
+	/* Wait for the flush to complete */
+	ret = wait_ready(pfdev, as_nr);
+
+	spin_unlock_irqrestore(&pfdev->hwaccess_lock, flags);
+
+	return ret;
+}
+
+void panfrost_mmu_enable(struct panfrost_device *pfdev, u32 as_nr)
+{
+	struct io_pgtable_cfg *cfg = &pfdev->mmu->pgtbl_cfg;
+	u64 transtab = cfg->arm_mali_lpae_cfg.transtab;
+	u64 memattr = cfg->arm_mali_lpae_cfg.memattr;
+
+	mmu_write(pfdev, MMU_INT_CLEAR, ~0);
+	mmu_write(pfdev, MMU_INT_MASK, ~0);
+
+	mmu_write(pfdev, AS_TRANSTAB_LO(as_nr), transtab & 0xffffffffUL);
+	mmu_write(pfdev, AS_TRANSTAB_HI(as_nr), transtab >> 32);
+
+	/* Need to revisit mem attrs.
+	 * NC is the default, Mali driver is inner WT.
+	 */
+	mmu_write(pfdev, AS_MEMATTR_LO(as_nr), memattr & 0xffffffffUL);
+	mmu_write(pfdev, AS_MEMATTR_HI(as_nr), memattr >> 32);
+
+	write_cmd(pfdev, as_nr, AS_COMMAND_UPDATE);
+}
+
+static void mmu_disable(struct panfrost_device *pfdev, u32 as_nr)
+{
+	mmu_write(pfdev, AS_TRANSTAB_LO(as_nr), 0);
+	mmu_write(pfdev, AS_TRANSTAB_HI(as_nr), 0);
+
+	mmu_write(pfdev, AS_MEMATTR_LO(as_nr), 0);
+	mmu_write(pfdev, AS_MEMATTR_HI(as_nr), 0);
+
+	write_cmd(pfdev, as_nr, AS_COMMAND_UPDATE);
+}
+
+static size_t get_pgsize(u64 addr, size_t size)
+{
+	if (addr & (SZ_2M - 1) || size < SZ_2M)
+		return SZ_4K;
+
+	return SZ_2M;
+}
+
+int panfrost_mmu_map(struct panfrost_gem_object *bo)
+{
+	struct drm_gem_object *obj = &bo->base.base;
+	struct panfrost_device *pfdev = to_panfrost_device(obj->dev);
+	struct io_pgtable_ops *ops = pfdev->mmu->pgtbl_ops;
+	u64 iova = bo->node.start << PAGE_SHIFT;
+	unsigned int count;
+	struct scatterlist *sgl;
+	struct sg_table *sgt;
+	int ret;
+
+	sgt = drm_gem_shmem_get_pages_sgt(obj);
+	if (WARN_ON(IS_ERR(sgt)))
+		return PTR_ERR(sgt);
+
+	ret = pm_runtime_get_sync(pfdev->dev);
+	if (ret < 0)
+		return ret;
+
+	mutex_lock(&pfdev->mmu->lock);
+
+	for_each_sg(sgt->sgl, sgl, sgt->nents, count) {
+		unsigned long paddr = sg_dma_address(sgl);
+		size_t len = sg_dma_len(sgl);
+
+		dev_dbg(pfdev->dev, "map: iova=%llx, paddr=%lx, len=%zx", iova, paddr, len);
+
+		while (len) {
+			size_t pgsize = get_pgsize(iova | paddr, len);
+
+			ops->map(ops, iova, paddr, pgsize, IOMMU_WRITE | IOMMU_READ);
+			iova += pgsize;
+			paddr += pgsize;
+			len -= pgsize;
+		}
+	}
+
+	mmu_hw_do_operation(pfdev, 0, bo->node.start << PAGE_SHIFT,
+			    bo->node.size << PAGE_SHIFT, AS_COMMAND_FLUSH_PT);
+
+	mutex_unlock(&pfdev->mmu->lock);
+
+	pm_runtime_mark_last_busy(pfdev->dev);
+	pm_runtime_put_autosuspend(pfdev->dev);
+
+	return 0;
+}
+
+void panfrost_mmu_unmap(struct panfrost_gem_object *bo)
+{
+	struct drm_gem_object *obj = &bo->base.base;
+	struct panfrost_device *pfdev = to_panfrost_device(obj->dev);
+	struct io_pgtable_ops *ops = pfdev->mmu->pgtbl_ops;
+	u64 iova = bo->node.start << PAGE_SHIFT;
+	size_t len = bo->node.size << PAGE_SHIFT;
+	size_t unmapped_len = 0;
+	int ret;
+
+	dev_dbg(pfdev->dev, "unmap: iova=%llx, len=%zx", iova, len);
+
+	ret = pm_runtime_get_sync(pfdev->dev);
+	if (ret < 0)
+		return;
+
+	mutex_lock(&pfdev->mmu->lock);
+
+	while (unmapped_len < len) {
+		size_t unmapped_page;
+		size_t pgsize = get_pgsize(iova, len - unmapped_len);
+
+		unmapped_page = ops->unmap(ops, iova, pgsize);
+		if (!unmapped_page)
+			break;
+
+		iova += unmapped_page;
+		unmapped_len += unmapped_page;
+	}
+
+	mmu_hw_do_operation(pfdev, 0, bo->node.start << PAGE_SHIFT,
+			    bo->node.size << PAGE_SHIFT, AS_COMMAND_FLUSH_PT);
+
+	mutex_unlock(&pfdev->mmu->lock);
+
+	pm_runtime_mark_last_busy(pfdev->dev);
+	pm_runtime_put_autosuspend(pfdev->dev);
+}
+
+static void mmu_tlb_inv_context_s1(void *cookie)
+{
+	struct panfrost_device *pfdev = cookie;
+
+	mmu_hw_do_operation(pfdev, 0, 0, ~0UL, AS_COMMAND_FLUSH_MEM);
+}
+
+static void mmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
+				     size_t granule, bool leaf, void *cookie)
+{}
+
+static void mmu_tlb_sync_context(void *cookie)
+{
+	//struct panfrost_device *pfdev = cookie;
+	// TODO: Wait 1000 GPU cycles for HW_ISSUE_6367/T60X
+}
+
+static const struct iommu_gather_ops mmu_tlb_ops = {
+	.tlb_flush_all	= mmu_tlb_inv_context_s1,
+	.tlb_add_flush	= mmu_tlb_inv_range_nosync,
+	.tlb_sync	= mmu_tlb_sync_context,
+};
+
+static const char *access_type_name(struct panfrost_device *pfdev,
+		u32 fault_status)
+{
+	switch (fault_status & AS_FAULTSTATUS_ACCESS_TYPE_MASK) {
+	case AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC:
+		if (panfrost_has_hw_feature(pfdev, HW_FEATURE_AARCH64_MMU))
+			return "ATOMIC";
+		else
+			return "UNKNOWN";
+	case AS_FAULTSTATUS_ACCESS_TYPE_READ:
+		return "READ";
+	case AS_FAULTSTATUS_ACCESS_TYPE_WRITE:
+		return "WRITE";
+	case AS_FAULTSTATUS_ACCESS_TYPE_EX:
+		return "EXECUTE";
+	default:
+		WARN_ON(1);
+		return NULL;
+	}
+}
+
+static irqreturn_t panfrost_mmu_irq_handler(int irq, void *data)
+{
+	struct panfrost_device *pfdev = data;
+	u32 status = mmu_read(pfdev, MMU_INT_STAT);
+	int i;
+
+	if (!status)
+		return IRQ_NONE;
+
+	dev_err(pfdev->dev, "mmu irq status=%x\n", status);
+
+	for (i = 0; status; i++) {
+		u32 mask = BIT(i) | BIT(i + 16);
+		u64 addr;
+		u32 fault_status;
+		u32 exception_type;
+		u32 access_type;
+		u32 source_id;
+
+		if (!(status & mask))
+			continue;
+
+		fault_status = mmu_read(pfdev, AS_FAULTSTATUS(i));
+		addr = mmu_read(pfdev, AS_FAULTADDRESS_LO(i));
+		addr |= (u64)mmu_read(pfdev, AS_FAULTADDRESS_HI(i)) << 32;
+
+		/* decode the fault status */
+		exception_type = fault_status & 0xFF;
+		access_type = (fault_status >> 8) & 0x3;
+		source_id = (fault_status >> 16);
+
+		/* terminal fault, print info about the fault */
+		dev_err(pfdev->dev,
+			"Unhandled Page fault in AS%d at VA 0x%016llX\n"
+			"Reason: %s\n"
+			"raw fault status: 0x%X\n"
+			"decoded fault status: %s\n"
+			"exception type 0x%X: %s\n"
+			"access type 0x%X: %s\n"
+			"source id 0x%X\n",
+			i, addr,
+			"TODO",
+			fault_status,
+			(fault_status & (1 << 10) ? "DECODER FAULT" : "SLAVE FAULT"),
+			exception_type, panfrost_exception_name(pfdev, exception_type),
+			access_type, access_type_name(pfdev, fault_status),
+			source_id);
+
+		mmu_write(pfdev, MMU_INT_CLEAR, mask);
+
+		status &= ~mask;
+	}
+
+	return IRQ_HANDLED;
+};
+
+int panfrost_mmu_init(struct panfrost_device *pfdev)
+{
+	struct io_pgtable_ops *pgtbl_ops;
+	int err, irq;
+
+	pfdev->mmu = devm_kzalloc(pfdev->dev, sizeof(*pfdev->mmu), GFP_KERNEL);
+	if (!pfdev->mmu)
+		return -ENOMEM;
+
+	mutex_init(&pfdev->mmu->lock);
+
+	irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "mmu");
+	if (irq <= 0)
+		return -ENODEV;
+
+	err = devm_request_irq(pfdev->dev, irq, panfrost_mmu_irq_handler,
+			       IRQF_SHARED, "mmu", pfdev);
+
+	if (err) {
+		dev_err(pfdev->dev, "failed to request mmu irq");
+		return err;
+	}
+	mmu_write(pfdev, MMU_INT_CLEAR, ~0);
+	mmu_write(pfdev, MMU_INT_MASK, ~0);
+
+	pfdev->mmu->pgtbl_cfg = (struct io_pgtable_cfg) {
+		.pgsize_bitmap	= SZ_4K | SZ_2M,
+		.ias		= FIELD_GET(0xff, pfdev->features.mmu_features),
+		.oas		= FIELD_GET(0xff00, pfdev->features.mmu_features),
+		.tlb		= &mmu_tlb_ops,
+		.iommu_dev	= pfdev->dev,
+	};
+
+	pgtbl_ops = alloc_io_pgtable_ops(ARM_MALI_LPAE, &pfdev->mmu->pgtbl_cfg,
+					 pfdev);
+	if (!pgtbl_ops)
+		return -ENOMEM;
+
+	pfdev->mmu->pgtbl_ops = pgtbl_ops;
+
+	panfrost_mmu_enable(pfdev, 0);
+
+	return 0;
+}
+
+void panfrost_mmu_fini(struct panfrost_device *pfdev)
+{
+	mmu_write(pfdev, MMU_INT_MASK, 0);
+	mmu_disable(pfdev, 0);
+
+	free_io_pgtable_ops(pfdev->mmu->pgtbl_ops);
+}
diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.h b/drivers/gpu/drm/panfrost/panfrost_mmu.h
new file mode 100644
index 000000000000..f5878d86a5ce
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/panfrost_mmu.h
@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
+
+#ifndef __PANFROST_MMU_H__
+#define __PANFROST_MMU_H__
+
+struct panfrost_gem_object;
+
+int panfrost_mmu_map(struct panfrost_gem_object *bo);
+void panfrost_mmu_unmap(struct panfrost_gem_object *bo);
+
+int panfrost_mmu_init(struct panfrost_device *pfdev);
+void panfrost_mmu_fini(struct panfrost_device *pfdev);
+
+void panfrost_mmu_enable(struct panfrost_device *pfdev, u32 as_nr);
+
+#endif
diff --git a/drivers/gpu/drm/panfrost/panfrost_regs.h b/drivers/gpu/drm/panfrost/panfrost_regs.h
new file mode 100644
index 000000000000..578c5fc2188b
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/panfrost_regs.h
@@ -0,0 +1,298 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */
+/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
+/*
+ * Register definitions based on mali_midg_regmap.h
+ * (C) COPYRIGHT 2010-2018 ARM Limited. All rights reserved.
+ */
+#ifndef __PANFROST_REGS_H__
+#define __PANFROST_REGS_H__
+
+#define GPU_ID				0x00
+#define GPU_L2_FEATURES			0x004	/* (RO) Level 2 cache features */
+#define GPU_CORE_FEATURES		0x008	/* (RO) Shader Core Features */
+#define GPU_TILER_FEATURES		0x00C	/* (RO) Tiler Features */
+#define GPU_MEM_FEATURES		0x010	/* (RO) Memory system features */
+#define   GROUPS_L2_COHERENT		BIT(0)	/* Cores groups are l2 coherent */
+
+#define GPU_MMU_FEATURES		0x014	/* (RO) MMU features */
+#define GPU_AS_PRESENT			0x018	/* (RO) Address space slots present */
+#define GPU_JS_PRESENT			0x01C	/* (RO) Job slots present */
+
+#define GPU_INT_RAWSTAT			0x20
+#define GPU_INT_CLEAR			0x24
+#define GPU_INT_MASK			0x28
+#define GPU_INT_STAT			0x2c
+#define   GPU_IRQ_FAULT			BIT(0)
+#define   GPU_IRQ_MULTIPLE_FAULT	BIT(7)
+#define   GPU_IRQ_RESET_COMPLETED	BIT(8)
+#define   GPU_IRQ_POWER_CHANGED		BIT(9)
+#define   GPU_IRQ_POWER_CHANGED_ALL	BIT(10)
+#define   GPU_IRQ_PERFCNT_SAMPLE_COMPLETED BIT(16)
+#define   GPU_IRQ_CLEAN_CACHES_COMPLETED BIT(17)
+#define   GPU_IRQ_MASK_ALL			 \
+	  (GPU_IRQ_FAULT			|\
+	   GPU_IRQ_MULTIPLE_FAULT		|\
+	   GPU_IRQ_RESET_COMPLETED		|\
+	   GPU_IRQ_POWER_CHANGED		|\
+	   GPU_IRQ_POWER_CHANGED_ALL		|\
+	   GPU_IRQ_PERFCNT_SAMPLE_COMPLETED	|\
+	   GPU_IRQ_CLEAN_CACHES_COMPLETED)
+#define GPU_IRQ_MASK_ERROR	   		\
+	(					\
+	 GPU_IRQ_FAULT				|\
+	 GPU_IRQ_MULTIPLE_FAULT)
+#define GPU_CMD				0x30
+#define   GPU_CMD_SOFT_RESET		0x01
+#define GPU_STATUS			0x34
+#define GPU_LATEST_FLUSH_ID		0x38
+#define GPU_FAULT_STATUS		0x3C
+#define GPU_FAULT_ADDRESS_LO		0x40
+#define GPU_FAULT_ADDRESS_HI		0x44
+
+#define GPU_THREAD_MAX_THREADS		0x0A0	/* (RO) Maximum number of threads per core */
+#define GPU_THREAD_MAX_WORKGROUP_SIZE	0x0A4	/* (RO) Maximum workgroup size */
+#define GPU_THREAD_MAX_BARRIER_SIZE	0x0A8	/* (RO) Maximum threads waiting at a barrier */
+#define GPU_THREAD_FEATURES		0x0AC	/* (RO) Thread features */
+#define GPU_THREAD_TLS_ALLOC		0x310   /* (RO) Number of threads per core that
+						 * TLS must be allocated for */
+
+#define GPU_TEXTURE_FEATURES(n)		(0x0B0 + ((n) * 4))
+#define GPU_JS_FEATURES(n)		(0x0C0 + ((n) * 4))
+
+#define GPU_SHADER_PRESENT_LO		0x100	/* (RO) Shader core present bitmap, low word */
+#define GPU_SHADER_PRESENT_HI		0x104	/* (RO) Shader core present bitmap, high word */
+#define GPU_TILER_PRESENT_LO		0x110	/* (RO) Tiler core present bitmap, low word */
+#define GPU_TILER_PRESENT_HI		0x114	/* (RO) Tiler core present bitmap, high word */
+
+#define GPU_L2_PRESENT_LO		0x120	/* (RO) Level 2 cache present bitmap, low word */
+#define GPU_L2_PRESENT_HI		0x124	/* (RO) Level 2 cache present bitmap, high word */
+
+#define GPU_COHERENCY_FEATURES		0x300	/* (RO) Coherency features present */
+#define   COHERENCY_ACE_LITE		BIT(0)
+#define   COHERENCY_ACE			BIT(1)
+
+#define GPU_STACK_PRESENT_LO		0xE00   /* (RO) Core stack present bitmap, low word */
+#define GPU_STACK_PRESENT_HI		0xE04   /* (RO) Core stack present bitmap, high word */
+
+#define SHADER_READY_LO			0x140	/* (RO) Shader core ready bitmap, low word */
+#define SHADER_READY_HI			0x144	/* (RO) Shader core ready bitmap, high word */
+
+#define TILER_READY_LO			0x150	/* (RO) Tiler core ready bitmap, low word */
+#define TILER_READY_HI			0x154	/* (RO) Tiler core ready bitmap, high word */
+
+#define L2_READY_LO			0x160	/* (RO) Level 2 cache ready bitmap, low word */
+#define L2_READY_HI			0x164	/* (RO) Level 2 cache ready bitmap, high word */
+
+#define STACK_READY_LO			0xE10   /* (RO) Core stack ready bitmap, low word */
+#define STACK_READY_HI			0xE14   /* (RO) Core stack ready bitmap, high word */
+
+
+#define SHADER_PWRON_LO			0x180	/* (WO) Shader core power on bitmap, low word */
+#define SHADER_PWRON_HI			0x184	/* (WO) Shader core power on bitmap, high word */
+
+#define TILER_PWRON_LO			0x190	/* (WO) Tiler core power on bitmap, low word */
+#define TILER_PWRON_HI			0x194	/* (WO) Tiler core power on bitmap, high word */
+
+#define L2_PWRON_LO			0x1A0	/* (WO) Level 2 cache power on bitmap, low word */
+#define L2_PWRON_HI			0x1A4	/* (WO) Level 2 cache power on bitmap, high word */
+
+#define STACK_PWRON_LO			0xE20   /* (RO) Core stack power on bitmap, low word */
+#define STACK_PWRON_HI			0xE24   /* (RO) Core stack power on bitmap, high word */
+
+
+#define SHADER_PWROFF_LO		0x1C0	/* (WO) Shader core power off bitmap, low word */
+#define SHADER_PWROFF_HI		0x1C4	/* (WO) Shader core power off bitmap, high word */
+
+#define TILER_PWROFF_LO			0x1D0	/* (WO) Tiler core power off bitmap, low word */
+#define TILER_PWROFF_HI			0x1D4	/* (WO) Tiler core power off bitmap, high word */
+
+#define L2_PWROFF_LO			0x1E0	/* (WO) Level 2 cache power off bitmap, low word */
+#define L2_PWROFF_HI			0x1E4	/* (WO) Level 2 cache power off bitmap, high word */
+
+#define STACK_PWROFF_LO			0xE30   /* (RO) Core stack power off bitmap, low word */
+#define STACK_PWROFF_HI			0xE34   /* (RO) Core stack power off bitmap, high word */
+
+
+#define SHADER_PWRTRANS_LO		0x200	/* (RO) Shader core power transition bitmap, low word */
+#define SHADER_PWRTRANS_HI		0x204	/* (RO) Shader core power transition bitmap, high word */
+
+#define TILER_PWRTRANS_LO		0x210	/* (RO) Tiler core power transition bitmap, low word */
+#define TILER_PWRTRANS_HI		0x214	/* (RO) Tiler core power transition bitmap, high word */
+
+#define L2_PWRTRANS_LO			0x220	/* (RO) Level 2 cache power transition bitmap, low word */
+#define L2_PWRTRANS_HI			0x224	/* (RO) Level 2 cache power transition bitmap, high word */
+
+#define STACK_PWRTRANS_LO		0xE40   /* (RO) Core stack power transition bitmap, low word */
+#define STACK_PWRTRANS_HI		0xE44   /* (RO) Core stack power transition bitmap, high word */
+
+
+#define SHADER_PWRACTIVE_LO		0x240	/* (RO) Shader core active bitmap, low word */
+#define SHADER_PWRACTIVE_HI		0x244	/* (RO) Shader core active bitmap, high word */
+
+#define TILER_PWRACTIVE_LO		0x250	/* (RO) Tiler core active bitmap, low word */
+#define TILER_PWRACTIVE_HI		0x254	/* (RO) Tiler core active bitmap, high word */
+
+#define L2_PWRACTIVE_LO			0x260	/* (RO) Level 2 cache active bitmap, low word */
+#define L2_PWRACTIVE_HI			0x264	/* (RO) Level 2 cache active bitmap, high word */
+
+#define GPU_JM_CONFIG			0xF00   /* (RW) Job Manager configuration register (Implementation specific register) */
+#define GPU_SHADER_CONFIG		0xF04	/* (RW) Shader core configuration settings (Implementation specific register) */
+#define GPU_TILER_CONFIG		0xF08   /* (RW) Tiler core configuration settings (Implementation specific register) */
+#define GPU_L2_MMU_CONFIG		0xF0C	/* (RW) Configuration of the L2 cache and MMU (Implementation specific register) */
+
+/* L2_MMU_CONFIG register */
+#define L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY_SHIFT	23
+#define L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY		(0x1 << L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY_SHIFT)
+#define L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT	24
+#define L2_MMU_CONFIG_LIMIT_EXTERNAL_READS		(0x3 << L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT)
+#define L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_OCTANT	(0x1 << L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT)
+#define L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_QUARTER	(0x2 << L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT)
+#define L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_HALF		(0x3 << L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT)
+
+#define L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT	26
+#define L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES		(0x3 << L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT)
+#define L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_OCTANT	(0x1 << L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT)
+#define L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_QUARTER	(0x2 << L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT)
+#define L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_HALF	(0x3 << L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT)
+
+#define L2_MMU_CONFIG_3BIT_LIMIT_EXTERNAL_READS_SHIFT	12
+#define L2_MMU_CONFIG_3BIT_LIMIT_EXTERNAL_READS		(0x7 << L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT)
+
+#define L2_MMU_CONFIG_3BIT_LIMIT_EXTERNAL_WRITES_SHIFT	15
+#define L2_MMU_CONFIG_3BIT_LIMIT_EXTERNAL_WRITES	(0x7 << L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT)
+
+/* SHADER_CONFIG register */
+#define SC_ALT_COUNTERS			BIT(3)
+#define SC_OVERRIDE_FWD_PIXEL_KILL	BIT(4)
+#define SC_SDC_DISABLE_OQ_DISCARD	BIT(6)
+#define SC_LS_ALLOW_ATTR_TYPES		BIT(16)
+#define SC_LS_PAUSEBUFFER_DISABLE	BIT(16)
+#define SC_TLS_HASH_ENABLE		BIT(17)
+#define SC_LS_ATTR_CHECK_DISABLE	BIT(18)
+#define SC_ENABLE_TEXGRD_FLAGS		BIT(25)
+/* End SHADER_CONFIG register */
+
+/* TILER_CONFIG register */
+#define TC_CLOCK_GATE_OVERRIDE		BIT(0)
+
+/* JM_CONFIG register */
+#define JM_TIMESTAMP_OVERRIDE		BIT(0)
+#define JM_CLOCK_GATE_OVERRIDE		BIT(1)
+#define JM_JOB_THROTTLE_ENABLE		BIT(2)
+#define JM_JOB_THROTTLE_LIMIT_SHIFT	3
+#define JM_MAX_JOB_THROTTLE_LIMIT	0x3F
+#define JM_FORCE_COHERENCY_FEATURES_SHIFT 2
+#define JM_IDVS_GROUP_SIZE_SHIFT	16
+#define JM_MAX_IDVS_GROUP_SIZE		0x3F
+
+
+/* Job Control regs */
+#define JOB_INT_RAWSTAT			0x1000
+#define JOB_INT_CLEAR			0x1004
+#define JOB_INT_MASK			0x1008
+#define JOB_INT_STAT			0x100c
+#define JOB_INT_JS_STATE		0x1010
+#define JOB_INT_THROTTLE		0x1014
+
+#define MK_JS_MASK(j)			(0x10001 << (j))
+#define JOB_INT_MASK_ERR(j)		BIT((j) + 16)
+#define JOB_INT_MASK_DONE(j)		BIT(j)
+
+#define JS_BASE				0x1800
+#define JS_HEAD_LO(n)			(JS_BASE + ((n) * 0x80) + 0x00)
+#define JS_HEAD_HI(n)			(JS_BASE + ((n) * 0x80) + 0x04)
+#define JS_TAIL_LO(n)			(JS_BASE + ((n) * 0x80) + 0x08)
+#define JS_TAIL_HI(n)			(JS_BASE + ((n) * 0x80) + 0x0c)
+#define JS_AFFINITY_LO(n)		(JS_BASE + ((n) * 0x80) + 0x10)
+#define JS_AFFINITY_HI(n)		(JS_BASE + ((n) * 0x80) + 0x14)
+#define JS_CONFIG(n)			(JS_BASE + ((n) * 0x80) + 0x18)
+#define JS_XAFFINITY(n)			(JS_BASE + ((n) * 0x80) + 0x1c)
+#define JS_COMMAND(n)			(JS_BASE + ((n) * 0x80) + 0x20)
+#define JS_STATUS(n)			(JS_BASE + ((n) * 0x80) + 0x24)
+#define JS_HEAD_NEXT_LO(n)		(JS_BASE + ((n) * 0x80) + 0x40)
+#define JS_HEAD_NEXT_HI(n)		(JS_BASE + ((n) * 0x80) + 0x44)
+#define JS_AFFINITY_NEXT_LO(n)		(JS_BASE + ((n) * 0x80) + 0x50)
+#define JS_AFFINITY_NEXT_HI(n)		(JS_BASE + ((n) * 0x80) + 0x54)
+#define JS_CONFIG_NEXT(n)		(JS_BASE + ((n) * 0x80) + 0x58)
+#define JS_COMMAND_NEXT(n)		(JS_BASE + ((n) * 0x80) + 0x60)
+#define JS_FLUSH_ID_NEXT(n)		(JS_BASE + ((n) * 0x80) + 0x70)
+
+/* Possible values of JS_CONFIG and JS_CONFIG_NEXT registers */
+#define JS_CONFIG_START_FLUSH_CLEAN		BIT(8)
+#define JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE	(3u << 8)
+#define JS_CONFIG_START_MMU			BIT(10)
+#define JS_CONFIG_JOB_CHAIN_FLAG		BIT(11)
+#define JS_CONFIG_END_FLUSH_CLEAN		BIT(12)
+#define JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE	(3u << 12)
+#define JS_CONFIG_ENABLE_FLUSH_REDUCTION	BIT(14)
+#define JS_CONFIG_DISABLE_DESCRIPTOR_WR_BK	BIT(15)
+#define JS_CONFIG_THREAD_PRI(n)			((n) << 16)
+
+#define JS_COMMAND_NOP			0x00
+#define JS_COMMAND_START		0x01
+#define JS_COMMAND_SOFT_STOP		0x02	/* Gently stop processing a job chain */
+#define JS_COMMAND_HARD_STOP		0x03	/* Rudely stop processing a job chain */
+#define JS_COMMAND_SOFT_STOP_0		0x04	/* Execute SOFT_STOP if JOB_CHAIN_FLAG is 0 */
+#define JS_COMMAND_HARD_STOP_0		0x05	/* Execute HARD_STOP if JOB_CHAIN_FLAG is 0 */
+#define JS_COMMAND_SOFT_STOP_1		0x06	/* Execute SOFT_STOP if JOB_CHAIN_FLAG is 1 */
+#define JS_COMMAND_HARD_STOP_1		0x07	/* Execute HARD_STOP if JOB_CHAIN_FLAG is 1 */
+
+#define JS_STATUS_EVENT_ACTIVE		0x08
+
+
+/* MMU regs */
+#define MMU_INT_RAWSTAT			0x2000
+#define MMU_INT_CLEAR			0x2004
+#define MMU_INT_MASK			0x2008
+#define MMU_INT_STAT			0x200c
+
+/* AS_COMMAND register commands */
+#define AS_COMMAND_NOP			0x00	/* NOP Operation */
+#define AS_COMMAND_UPDATE		0x01	/* Broadcasts the values in AS_TRANSTAB and ASn_MEMATTR to all MMUs */
+#define AS_COMMAND_LOCK			0x02	/* Issue a lock region command to all MMUs */
+#define AS_COMMAND_UNLOCK		0x03	/* Issue a flush region command to all MMUs */
+#define AS_COMMAND_FLUSH		0x04	/* Flush all L2 caches then issue a flush region command to all MMUs
+						   (deprecated - only for use with T60x) */
+#define AS_COMMAND_FLUSH_PT		0x04	/* Flush all L2 caches then issue a flush region command to all MMUs */
+#define AS_COMMAND_FLUSH_MEM		0x05	/* Wait for memory accesses to complete, flush all the L1s cache then
+						   flush all L2 caches then issue a flush region command to all MMUs */
+
+#define MMU_AS(as)			(0x2400 + ((as) << 6))
+
+#define AS_TRANSTAB_LO(as)		(MMU_AS(as) + 0x00) /* (RW) Translation Table Base Address for address space n, low word */
+#define AS_TRANSTAB_HI(as)		(MMU_AS(as) + 0x04) /* (RW) Translation Table Base Address for address space n, high word */
+#define AS_MEMATTR_LO(as)		(MMU_AS(as) + 0x08) /* (RW) Memory attributes for address space n, low word. */
+#define AS_MEMATTR_HI(as)		(MMU_AS(as) + 0x0C) /* (RW) Memory attributes for address space n, high word. */
+#define AS_LOCKADDR_LO(as)		(MMU_AS(as) + 0x10) /* (RW) Lock region address for address space n, low word */
+#define AS_LOCKADDR_HI(as)		(MMU_AS(as) + 0x14) /* (RW) Lock region address for address space n, high word */
+#define AS_COMMAND(as)			(MMU_AS(as) + 0x18) /* (WO) MMU command register for address space n */
+#define AS_FAULTSTATUS(as)		(MMU_AS(as) + 0x1C) /* (RO) MMU fault status register for address space n */
+#define AS_FAULTADDRESS_LO(as)		(MMU_AS(as) + 0x20) /* (RO) Fault Address for address space n, low word */
+#define AS_FAULTADDRESS_HI(as)		(MMU_AS(as) + 0x24) /* (RO) Fault Address for address space n, high word */
+#define AS_STATUS(as)			(MMU_AS(as) + 0x28) /* (RO) Status flags for address space n */
+/* Additional Bifrost AS regsiters */
+#define AS_TRANSCFG_LO(as)		(MMU_AS(as) + 0x30) /* (RW) Translation table configuration for address space n, low word */
+#define AS_TRANSCFG_HI(as)		(MMU_AS(as) + 0x34) /* (RW) Translation table configuration for address space n, high word */
+#define AS_FAULTEXTRA_LO(as)		(MMU_AS(as) + 0x38) /* (RO) Secondary fault address for address space n, low word */
+#define AS_FAULTEXTRA_HI(as)		(MMU_AS(as) + 0x3C) /* (RO) Secondary fault address for address space n, high word */
+
+/*
+ * Begin LPAE MMU TRANSTAB register values
+ */
+#define AS_TRANSTAB_LPAE_ADDR_SPACE_MASK	0xfffffffffffff000
+#define AS_TRANSTAB_LPAE_ADRMODE_IDENTITY	0x2
+#define AS_TRANSTAB_LPAE_ADRMODE_TABLE		0x3
+#define AS_TRANSTAB_LPAE_ADRMODE_MASK		0x3
+#define AS_TRANSTAB_LPAE_READ_INNER		BIT(2)
+#define AS_TRANSTAB_LPAE_SHARE_OUTER		BIT(4)
+
+#define AS_STATUS_AS_ACTIVE			0x01
+
+#define AS_FAULTSTATUS_ACCESS_TYPE_MASK		(0x3 << 8)
+#define AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC	(0x0 << 8)
+#define AS_FAULTSTATUS_ACCESS_TYPE_EX		(0x1 << 8)
+#define AS_FAULTSTATUS_ACCESS_TYPE_READ		(0x2 << 8)
+#define AS_FAULTSTATUS_ACCESS_TYPE_WRITE	(0x3 << 8)
+
+#endif
diff --git a/drivers/gpu/drm/pl111/pl111_display.c b/drivers/gpu/drm/pl111/pl111_display.c
index 754f6b25f265..0c5d391f0a8f 100644
--- a/drivers/gpu/drm/pl111/pl111_display.c
+++ b/drivers/gpu/drm/pl111/pl111_display.c
@@ -188,7 +188,7 @@ static void pl111_display_enable(struct drm_simple_display_pipe *pipe,
 			tim2 |= TIM2_IOE;
 
 		if (connector->display_info.bus_flags &
-		    DRM_BUS_FLAG_PIXDATA_NEGEDGE)
+		    DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE)
 			tim2 |= TIM2_IPC;
 	}
 
diff --git a/drivers/gpu/drm/pl111/pl111_versatile.c b/drivers/gpu/drm/pl111/pl111_versatile.c
index b9baefdba38a..1c318ad32a8c 100644
--- a/drivers/gpu/drm/pl111/pl111_versatile.c
+++ b/drivers/gpu/drm/pl111/pl111_versatile.c
@@ -330,6 +330,7 @@ int pl111_versatile_init(struct device *dev, struct pl111_drm_dev_private *priv)
 		ret = vexpress_muxfpga_init();
 		if (ret) {
 			dev_err(dev, "unable to initialize muxfpga driver\n");
+			of_node_put(np);
 			return ret;
 		}
 
@@ -337,17 +338,20 @@ int pl111_versatile_init(struct device *dev, struct pl111_drm_dev_private *priv)
 		pdev = of_find_device_by_node(np);
 		if (!pdev) {
 			dev_err(dev, "can't find the sysreg device, deferring\n");
+			of_node_put(np);
 			return -EPROBE_DEFER;
 		}
 		map = dev_get_drvdata(&pdev->dev);
 		if (!map) {
 			dev_err(dev, "sysreg has not yet probed\n");
 			platform_device_put(pdev);
+			of_node_put(np);
 			return -EPROBE_DEFER;
 		}
 	} else {
 		map = syscon_node_to_regmap(np);
 	}
+	of_node_put(np);
 
 	if (IS_ERR(map)) {
 		dev_err(dev, "no Versatile syscon regmap\n");
diff --git a/drivers/gpu/drm/qxl/qxl_display.c b/drivers/gpu/drm/qxl/qxl_display.c
index 08c725544a2f..8b319ebbb0fb 100644
--- a/drivers/gpu/drm/qxl/qxl_display.c
+++ b/drivers/gpu/drm/qxl/qxl_display.c
@@ -535,7 +535,7 @@ static void qxl_primary_atomic_update(struct drm_plane *plane,
 {
 	struct qxl_device *qdev = plane->dev->dev_private;
 	struct qxl_bo *bo = gem_to_qxl_bo(plane->state->fb->obj[0]);
-	struct qxl_bo *bo_old, *primary;
+	struct qxl_bo *primary;
 	struct drm_clip_rect norect = {
 	    .x1 = 0,
 	    .y1 = 0,
@@ -544,12 +544,6 @@ static void qxl_primary_atomic_update(struct drm_plane *plane,
 	};
 	uint32_t dumb_shadow_offset = 0;
 
-	if (old_state->fb) {
-		bo_old = gem_to_qxl_bo(old_state->fb->obj[0]);
-	} else {
-		bo_old = NULL;
-	}
-
 	primary = bo->shadow ? bo->shadow : bo;
 
 	if (!primary->is_primary) {
diff --git a/drivers/gpu/drm/qxl/qxl_drv.h b/drivers/gpu/drm/qxl/qxl_drv.h
index 4a0331b3ff7d..2896bb6fdbf4 100644
--- a/drivers/gpu/drm/qxl/qxl_drv.h
+++ b/drivers/gpu/drm/qxl/qxl_drv.h
@@ -65,9 +65,6 @@
 extern int qxl_num_crtc;
 extern int qxl_max_ioctls;
 
-#define DRM_FILE_OFFSET 0x100000000ULL
-#define DRM_FILE_PAGE_OFFSET (DRM_FILE_OFFSET >> PAGE_SHIFT)
-
 #define QXL_INTERRUPT_MASK (\
 	QXL_INTERRUPT_DISPLAY |\
 	QXL_INTERRUPT_CURSOR |\
diff --git a/drivers/gpu/drm/qxl/qxl_ttm.c b/drivers/gpu/drm/qxl/qxl_ttm.c
index 92f5db5b296f..0234f8556ada 100644
--- a/drivers/gpu/drm/qxl/qxl_ttm.c
+++ b/drivers/gpu/drm/qxl/qxl_ttm.c
@@ -63,15 +63,10 @@ static vm_fault_t qxl_ttm_fault(struct vm_fault *vmf)
 
 int qxl_mmap(struct file *filp, struct vm_area_struct *vma)
 {
-	struct drm_file *file_priv;
-	struct qxl_device *qdev;
 	int r;
+	struct drm_file *file_priv = filp->private_data;
+	struct qxl_device *qdev = file_priv->minor->dev->dev_private;
 
-	if (unlikely(vma->vm_pgoff < DRM_FILE_PAGE_OFFSET))
-		return -EINVAL;
-
-	file_priv = filp->private_data;
-	qdev = file_priv->minor->dev->dev_private;
 	if (qdev == NULL) {
 		DRM_ERROR(
 		 "filp->private_data->minor->dev->dev_private == NULL\n");
@@ -328,7 +323,7 @@ int qxl_ttm_init(struct qxl_device *qdev)
 	r = ttm_bo_device_init(&qdev->mman.bdev,
 			       &qxl_bo_driver,
 			       qdev->ddev.anon_inode->i_mapping,
-			       DRM_FILE_PAGE_OFFSET, 0);
+			       false);
 	if (r) {
 		DRM_ERROR("failed initializing buffer object driver(%d).\n", r);
 		return r;
diff --git a/drivers/gpu/drm/radeon/radeon_device.c b/drivers/gpu/drm/radeon/radeon_device.c
index 53f29a115104..0a9312ea250a 100644
--- a/drivers/gpu/drm/radeon/radeon_device.c
+++ b/drivers/gpu/drm/radeon/radeon_device.c
@@ -1388,7 +1388,7 @@ int radeon_device_init(struct radeon_device *rdev,
 		pci_set_consistent_dma_mask(rdev->pdev, DMA_BIT_MASK(32));
 		pr_warn("radeon: No coherent DMA available\n");
 	}
-	rdev->need_swiotlb = drm_get_max_iomem() > ((u64)1 << dma_bits);
+	rdev->need_swiotlb = drm_need_swiotlb(dma_bits);
 
 	/* Registers mapping */
 	/* TODO: block userspace mapping of io register */
diff --git a/drivers/gpu/drm/radeon/radeon_drv.h b/drivers/gpu/drm/radeon/radeon_drv.h
index afef2d9fccd8..173deb463414 100644
--- a/drivers/gpu/drm/radeon/radeon_drv.h
+++ b/drivers/gpu/drm/radeon/radeon_drv.h
@@ -35,7 +35,6 @@
 #include <linux/platform_device.h>
 #include <drm/drm_legacy.h>
 
-#include <drm/ati_pcigart.h>
 #include "radeon_family.h"
 
 /* General customization:
diff --git a/drivers/gpu/drm/radeon/radeon_fb.c b/drivers/gpu/drm/radeon/radeon_fb.c
index 1179034024ae..1298b84cb1c7 100644
--- a/drivers/gpu/drm/radeon/radeon_fb.c
+++ b/drivers/gpu/drm/radeon/radeon_fb.c
@@ -42,7 +42,7 @@
  * the helper contains a pointer to radeon framebuffer baseclass.
  */
 struct radeon_fbdev {
-	struct drm_fb_helper helper;
+	struct drm_fb_helper helper; /* must be first */
 	struct drm_framebuffer fb;
 	struct radeon_device *rdev;
 };
@@ -244,7 +244,8 @@ static int radeonfb_create(struct drm_fb_helper *helper,
 		goto out;
 	}
 
-	info->par = rfbdev;
+	/* radeon resume is fragile and needs a vt switch to help it along */
+	info->skip_vt_switch = false;
 
 	ret = radeon_framebuffer_init(rdev->ddev, &rfbdev->fb, &mode_cmd, gobj);
 	if (ret) {
@@ -259,10 +260,6 @@ static int radeonfb_create(struct drm_fb_helper *helper,
 
 	memset_io(rbo->kptr, 0x0, radeon_bo_size(rbo));
 
-	strcpy(info->fix.id, "radeondrmfb");
-
-	drm_fb_helper_fill_fix(info, fb->pitches[0], fb->format->depth);
-
 	info->fbops = &radeonfb_ops;
 
 	tmp = radeon_bo_gpu_offset(rbo) - rdev->mc.vram_start;
@@ -271,7 +268,7 @@ static int radeonfb_create(struct drm_fb_helper *helper,
 	info->screen_base = rbo->kptr;
 	info->screen_size = radeon_bo_size(rbo);
 
-	drm_fb_helper_fill_var(info, &rfbdev->helper, sizes->fb_width, sizes->fb_height);
+	drm_fb_helper_fill_info(info, &rfbdev->helper, sizes);
 
 	/* setup aperture base/size for vesafb takeover */
 	info->apertures->ranges[0].base = rdev->ddev->mode_config.fb_base;
diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c
index 9920a6fc11bf..5d42f8d8e68d 100644
--- a/drivers/gpu/drm/radeon/radeon_ttm.c
+++ b/drivers/gpu/drm/radeon/radeon_ttm.c
@@ -45,8 +45,6 @@
 #include "radeon_reg.h"
 #include "radeon.h"
 
-#define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT)
-
 static int radeon_ttm_debugfs_init(struct radeon_device *rdev);
 static void radeon_ttm_debugfs_fini(struct radeon_device *rdev);
 
@@ -253,14 +251,12 @@ static int radeon_move_vram_ram(struct ttm_buffer_object *bo,
 				struct ttm_mem_reg *new_mem)
 {
 	struct ttm_operation_ctx ctx = { interruptible, no_wait_gpu };
-	struct radeon_device *rdev;
 	struct ttm_mem_reg *old_mem = &bo->mem;
 	struct ttm_mem_reg tmp_mem;
 	struct ttm_place placements;
 	struct ttm_placement placement;
 	int r;
 
-	rdev = radeon_get_rdev(bo->bdev);
 	tmp_mem = *new_mem;
 	tmp_mem.mm_node = NULL;
 	placement.num_placement = 1;
@@ -300,14 +296,12 @@ static int radeon_move_ram_vram(struct ttm_buffer_object *bo,
 				struct ttm_mem_reg *new_mem)
 {
 	struct ttm_operation_ctx ctx = { interruptible, no_wait_gpu };
-	struct radeon_device *rdev;
 	struct ttm_mem_reg *old_mem = &bo->mem;
 	struct ttm_mem_reg tmp_mem;
 	struct ttm_placement placement;
 	struct ttm_place placements;
 	int r;
 
-	rdev = radeon_get_rdev(bo->bdev);
 	tmp_mem = *new_mem;
 	tmp_mem.mm_node = NULL;
 	placement.num_placement = 1;
@@ -792,7 +786,6 @@ int radeon_ttm_init(struct radeon_device *rdev)
 	r = ttm_bo_device_init(&rdev->mman.bdev,
 			       &radeon_bo_driver,
 			       rdev->ddev->anon_inode->i_mapping,
-			       DRM_FILE_PAGE_OFFSET,
 			       rdev->need_dma32);
 	if (r) {
 		DRM_ERROR("failed initializing buffer object driver(%d).\n", r);
@@ -901,16 +894,10 @@ static vm_fault_t radeon_ttm_fault(struct vm_fault *vmf)
 
 int radeon_mmap(struct file *filp, struct vm_area_struct *vma)
 {
-	struct drm_file *file_priv;
-	struct radeon_device *rdev;
 	int r;
+	struct drm_file *file_priv = filp->private_data;
+	struct radeon_device *rdev = file_priv->minor->dev->dev_private;
 
-	if (unlikely(vma->vm_pgoff < DRM_FILE_PAGE_OFFSET)) {
-		return -EINVAL;
-	}
-
-	file_priv = filp->private_data;
-	rdev = file_priv->minor->dev->dev_private;
 	if (rdev == NULL) {
 		return -EINVAL;
 	}
diff --git a/drivers/gpu/drm/rcar-du/Kconfig b/drivers/gpu/drm/rcar-du/Kconfig
index 7c36e2777a15..1529849e217e 100644
--- a/drivers/gpu/drm/rcar-du/Kconfig
+++ b/drivers/gpu/drm/rcar-du/Kconfig
@@ -36,3 +36,7 @@ config DRM_RCAR_VSP
 	depends on VIDEO_RENESAS_VSP1=y || (VIDEO_RENESAS_VSP1 && DRM_RCAR_DU=m)
 	help
 	  Enable support to expose the R-Car VSP Compositor as KMS planes.
+
+config DRM_RCAR_WRITEBACK
+	bool
+	default y if ARM64
diff --git a/drivers/gpu/drm/rcar-du/Makefile b/drivers/gpu/drm/rcar-du/Makefile
index 2a3b8d7972b5..6c2ed9c46467 100644
--- a/drivers/gpu/drm/rcar-du/Makefile
+++ b/drivers/gpu/drm/rcar-du/Makefile
@@ -4,7 +4,7 @@ rcar-du-drm-y := rcar_du_crtc.o \
 		 rcar_du_encoder.o \
 		 rcar_du_group.o \
 		 rcar_du_kms.o \
-		 rcar_du_plane.o
+		 rcar_du_plane.o \
 
 rcar-du-drm-$(CONFIG_DRM_RCAR_LVDS)	+= rcar_du_of.o \
 					   rcar_du_of_lvds_r8a7790.dtb.o \
@@ -13,6 +13,7 @@ rcar-du-drm-$(CONFIG_DRM_RCAR_LVDS)	+= rcar_du_of.o \
 					   rcar_du_of_lvds_r8a7795.dtb.o \
 					   rcar_du_of_lvds_r8a7796.dtb.o
 rcar-du-drm-$(CONFIG_DRM_RCAR_VSP)	+= rcar_du_vsp.o
+rcar-du-drm-$(CONFIG_DRM_RCAR_WRITEBACK) += rcar_du_writeback.o
 
 obj-$(CONFIG_DRM_RCAR_DU)		+= rcar-du-drm.o
 obj-$(CONFIG_DRM_RCAR_DW_HDMI)		+= rcar_dw_hdmi.o
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_crtc.c b/drivers/gpu/drm/rcar-du/rcar_du_crtc.c
index 4cdea14d552f..2da46e3dc4ae 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_crtc.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_crtc.c
@@ -32,21 +32,21 @@
 
 static u32 rcar_du_crtc_read(struct rcar_du_crtc *rcrtc, u32 reg)
 {
-	struct rcar_du_device *rcdu = rcrtc->group->dev;
+	struct rcar_du_device *rcdu = rcrtc->dev;
 
 	return rcar_du_read(rcdu, rcrtc->mmio_offset + reg);
 }
 
 static void rcar_du_crtc_write(struct rcar_du_crtc *rcrtc, u32 reg, u32 data)
 {
-	struct rcar_du_device *rcdu = rcrtc->group->dev;
+	struct rcar_du_device *rcdu = rcrtc->dev;
 
 	rcar_du_write(rcdu, rcrtc->mmio_offset + reg, data);
 }
 
 static void rcar_du_crtc_clr(struct rcar_du_crtc *rcrtc, u32 reg, u32 clr)
 {
-	struct rcar_du_device *rcdu = rcrtc->group->dev;
+	struct rcar_du_device *rcdu = rcrtc->dev;
 
 	rcar_du_write(rcdu, rcrtc->mmio_offset + reg,
 		      rcar_du_read(rcdu, rcrtc->mmio_offset + reg) & ~clr);
@@ -54,7 +54,7 @@ static void rcar_du_crtc_clr(struct rcar_du_crtc *rcrtc, u32 reg, u32 clr)
 
 static void rcar_du_crtc_set(struct rcar_du_crtc *rcrtc, u32 reg, u32 set)
 {
-	struct rcar_du_device *rcdu = rcrtc->group->dev;
+	struct rcar_du_device *rcdu = rcrtc->dev;
 
 	rcar_du_write(rcdu, rcrtc->mmio_offset + reg,
 		      rcar_du_read(rcdu, rcrtc->mmio_offset + reg) | set);
@@ -62,7 +62,7 @@ static void rcar_du_crtc_set(struct rcar_du_crtc *rcrtc, u32 reg, u32 set)
 
 void rcar_du_crtc_dsysr_clr_set(struct rcar_du_crtc *rcrtc, u32 clr, u32 set)
 {
-	struct rcar_du_device *rcdu = rcrtc->group->dev;
+	struct rcar_du_device *rcdu = rcrtc->dev;
 
 	rcrtc->dsysr = (rcrtc->dsysr & ~clr) | set;
 	rcar_du_write(rcdu, rcrtc->mmio_offset + DSYSR, rcrtc->dsysr);
@@ -157,10 +157,9 @@ static void rcar_du_dpll_divider(struct rcar_du_crtc *rcrtc,
 	}
 
 done:
-	dev_dbg(rcrtc->group->dev->dev,
+	dev_dbg(rcrtc->dev->dev,
 		"output:%u, fdpll:%u, n:%u, m:%u, diff:%lu\n",
-		 dpll->output, dpll->fdpll, dpll->n, dpll->m,
-		 best_diff);
+		 dpll->output, dpll->fdpll, dpll->n, dpll->m, best_diff);
 }
 
 struct du_clk_params {
@@ -212,7 +211,7 @@ static const struct soc_device_attribute rcar_du_r8a7795_es1[] = {
 static void rcar_du_crtc_set_display_timing(struct rcar_du_crtc *rcrtc)
 {
 	const struct drm_display_mode *mode = &rcrtc->crtc.state->adjusted_mode;
-	struct rcar_du_device *rcdu = rcrtc->group->dev;
+	struct rcar_du_device *rcdu = rcrtc->dev;
 	unsigned long mode_clock = mode->clock * 1000;
 	u32 dsmr;
 	u32 escr;
@@ -277,7 +276,7 @@ static void rcar_du_crtc_set_display_timing(struct rcar_du_crtc *rcrtc)
 			rcar_du_escr_divider(rcrtc->extclock, mode_clock,
 					     ESCR_DCLKSEL_DCLKIN, &params);
 
-		dev_dbg(rcrtc->group->dev->dev,	"mode clock %lu %s rate %lu\n",
+		dev_dbg(rcrtc->dev->dev, "mode clock %lu %s rate %lu\n",
 			mode_clock, params.clk == rcrtc->clock ? "cpg" : "ext",
 			params.rate);
 
@@ -285,7 +284,7 @@ static void rcar_du_crtc_set_display_timing(struct rcar_du_crtc *rcrtc)
 		escr = params.escr;
 	}
 
-	dev_dbg(rcrtc->group->dev->dev, "%s: ESCR 0x%08x\n", __func__, escr);
+	dev_dbg(rcrtc->dev->dev, "%s: ESCR 0x%08x\n", __func__, escr);
 
 	rcar_du_crtc_write(rcrtc, rcrtc->index % 2 ? ESCR13 : ESCR02, escr);
 	rcar_du_crtc_write(rcrtc, rcrtc->index % 2 ? OTAR13 : OTAR02, 0);
@@ -333,7 +332,7 @@ plane_format(struct rcar_du_plane *plane)
 static void rcar_du_crtc_update_planes(struct rcar_du_crtc *rcrtc)
 {
 	struct rcar_du_plane *planes[RCAR_DU_NUM_HW_PLANES];
-	struct rcar_du_device *rcdu = rcrtc->group->dev;
+	struct rcar_du_device *rcdu = rcrtc->dev;
 	unsigned int num_planes = 0;
 	unsigned int dptsr_planes;
 	unsigned int hwplanes = 0;
@@ -463,7 +462,7 @@ static bool rcar_du_crtc_page_flip_pending(struct rcar_du_crtc *rcrtc)
 
 static void rcar_du_crtc_wait_page_flip(struct rcar_du_crtc *rcrtc)
 {
-	struct rcar_du_device *rcdu = rcrtc->group->dev;
+	struct rcar_du_device *rcdu = rcrtc->dev;
 
 	if (wait_event_timeout(rcrtc->flip_wait,
 			       !rcar_du_crtc_page_flip_pending(rcrtc),
@@ -493,7 +492,7 @@ static void rcar_du_crtc_setup(struct rcar_du_crtc *rcrtc)
 	rcar_du_group_write(rcrtc->group, rcrtc->index % 2 ? DS2PR : DS1PR, 0);
 
 	/* Enable the VSP compositor. */
-	if (rcar_du_has(rcrtc->group->dev, RCAR_DU_FEATURE_VSP1_SOURCE))
+	if (rcar_du_has(rcrtc->dev, RCAR_DU_FEATURE_VSP1_SOURCE))
 		rcar_du_vsp_enable(rcrtc);
 
 	/* Turn vertical blanking interrupt reporting on. */
@@ -564,7 +563,7 @@ static void rcar_du_crtc_start(struct rcar_du_crtc *rcrtc)
 
 static void rcar_du_crtc_disable_planes(struct rcar_du_crtc *rcrtc)
 {
-	struct rcar_du_device *rcdu = rcrtc->group->dev;
+	struct rcar_du_device *rcdu = rcrtc->dev;
 	struct drm_crtc *crtc = &rcrtc->crtc;
 	u32 status;
 
@@ -617,7 +616,7 @@ static void rcar_du_crtc_stop(struct rcar_du_crtc *rcrtc)
 	drm_crtc_vblank_off(crtc);
 
 	/* Disable the VSP compositor. */
-	if (rcar_du_has(rcrtc->group->dev, RCAR_DU_FEATURE_VSP1_SOURCE))
+	if (rcar_du_has(rcrtc->dev, RCAR_DU_FEATURE_VSP1_SOURCE))
 		rcar_du_vsp_disable(rcrtc);
 
 	/*
@@ -627,7 +626,7 @@ static void rcar_du_crtc_stop(struct rcar_du_crtc *rcrtc)
 	 * TODO: Find another way to stop the display for DUs that don't support
 	 * TVM sync.
 	 */
-	if (rcar_du_has(rcrtc->group->dev, RCAR_DU_FEATURE_TVM_SYNC))
+	if (rcar_du_has(rcrtc->dev, RCAR_DU_FEATURE_TVM_SYNC))
 		rcar_du_crtc_dsysr_clr_set(rcrtc, DSYSR_TVM_MASK,
 					   DSYSR_TVM_SWITCH);
 
@@ -648,8 +647,13 @@ static int rcar_du_crtc_atomic_check(struct drm_crtc *crtc,
 	rstate->outputs = 0;
 
 	drm_for_each_encoder_mask(encoder, crtc->dev, state->encoder_mask) {
-		struct rcar_du_encoder *renc = to_rcar_encoder(encoder);
+		struct rcar_du_encoder *renc;
 
+		/* Skip the writeback encoder. */
+		if (encoder->encoder_type == DRM_MODE_ENCODER_VIRTUAL)
+			continue;
+
+		renc = to_rcar_encoder(encoder);
 		rstate->outputs |= BIT(renc->output);
 	}
 
@@ -661,7 +665,7 @@ static void rcar_du_crtc_atomic_enable(struct drm_crtc *crtc,
 {
 	struct rcar_du_crtc *rcrtc = to_rcar_crtc(crtc);
 	struct rcar_du_crtc_state *rstate = to_rcar_crtc_state(crtc->state);
-	struct rcar_du_device *rcdu = rcrtc->group->dev;
+	struct rcar_du_device *rcdu = rcrtc->dev;
 
 	rcar_du_crtc_get(rcrtc);
 
@@ -689,7 +693,7 @@ static void rcar_du_crtc_atomic_disable(struct drm_crtc *crtc,
 {
 	struct rcar_du_crtc *rcrtc = to_rcar_crtc(crtc);
 	struct rcar_du_crtc_state *rstate = to_rcar_crtc_state(old_state);
-	struct rcar_du_device *rcdu = rcrtc->group->dev;
+	struct rcar_du_device *rcdu = rcrtc->dev;
 
 	rcar_du_crtc_stop(rcrtc);
 	rcar_du_crtc_put(rcrtc);
@@ -735,7 +739,7 @@ static void rcar_du_crtc_atomic_begin(struct drm_crtc *crtc,
 	 */
 	rcar_du_crtc_get(rcrtc);
 
-	if (rcar_du_has(rcrtc->group->dev, RCAR_DU_FEATURE_VSP1_SOURCE))
+	if (rcar_du_has(rcrtc->dev, RCAR_DU_FEATURE_VSP1_SOURCE))
 		rcar_du_vsp_atomic_begin(rcrtc);
 }
 
@@ -757,15 +761,16 @@ static void rcar_du_crtc_atomic_flush(struct drm_crtc *crtc,
 		spin_unlock_irqrestore(&dev->event_lock, flags);
 	}
 
-	if (rcar_du_has(rcrtc->group->dev, RCAR_DU_FEATURE_VSP1_SOURCE))
+	if (rcar_du_has(rcrtc->dev, RCAR_DU_FEATURE_VSP1_SOURCE))
 		rcar_du_vsp_atomic_flush(rcrtc);
 }
 
-enum drm_mode_status rcar_du_crtc_mode_valid(struct drm_crtc *crtc,
-				   const struct drm_display_mode *mode)
+static enum drm_mode_status
+rcar_du_crtc_mode_valid(struct drm_crtc *crtc,
+			const struct drm_display_mode *mode)
 {
 	struct rcar_du_crtc *rcrtc = to_rcar_crtc(crtc);
-	struct rcar_du_device *rcdu = rcrtc->group->dev;
+	struct rcar_du_device *rcdu = rcrtc->dev;
 	bool interlaced = mode->flags & DRM_MODE_FLAG_INTERLACE;
 	unsigned int vbp;
 
@@ -797,7 +802,7 @@ static const struct drm_crtc_helper_funcs crtc_helper_funcs = {
 
 static void rcar_du_crtc_crc_init(struct rcar_du_crtc *rcrtc)
 {
-	struct rcar_du_device *rcdu = rcrtc->group->dev;
+	struct rcar_du_device *rcdu = rcrtc->dev;
 	const char **sources;
 	unsigned int count;
 	int i = -1;
@@ -981,8 +986,8 @@ static int rcar_du_crtc_verify_crc_source(struct drm_crtc *crtc,
 	return 0;
 }
 
-const char *const *rcar_du_crtc_get_crc_sources(struct drm_crtc *crtc,
-						size_t *count)
+static const char *const *
+rcar_du_crtc_get_crc_sources(struct drm_crtc *crtc, size_t *count)
 {
 	struct rcar_du_crtc *rcrtc = to_rcar_crtc(crtc);
 
@@ -1079,7 +1084,7 @@ static const struct drm_crtc_funcs crtc_funcs_gen3 = {
 static irqreturn_t rcar_du_crtc_irq(int irq, void *arg)
 {
 	struct rcar_du_crtc *rcrtc = arg;
-	struct rcar_du_device *rcdu = rcrtc->group->dev;
+	struct rcar_du_device *rcdu = rcrtc->dev;
 	irqreturn_t ret = IRQ_NONE;
 	u32 status;
 
@@ -1171,6 +1176,7 @@ int rcar_du_crtc_create(struct rcar_du_group *rgrp, unsigned int swindex,
 	init_waitqueue_head(&rcrtc->vblank_wait);
 	spin_lock_init(&rcrtc->vblank_lock);
 
+	rcrtc->dev = rcdu;
 	rcrtc->group = rgrp;
 	rcrtc->mmio_offset = mmio_offsets[hwindex];
 	rcrtc->index = hwindex;
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_crtc.h b/drivers/gpu/drm/rcar-du/rcar_du_crtc.h
index bcb35b0b7612..3b7fc668996f 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_crtc.h
+++ b/drivers/gpu/drm/rcar-du/rcar_du_crtc.h
@@ -15,6 +15,7 @@
 #include <linux/wait.h>
 
 #include <drm/drm_crtc.h>
+#include <drm/drm_writeback.h>
 
 #include <media/vsp1.h>
 
@@ -24,10 +25,11 @@ struct rcar_du_vsp;
 /**
  * struct rcar_du_crtc - the CRTC, representing a DU superposition processor
  * @crtc: base DRM CRTC
+ * @dev: the DU device
  * @clock: the CRTC functional clock
  * @extclock: external pixel dot clock (optional)
  * @mmio_offset: offset of the CRTC registers in the DU MMIO block
- * @index: CRTC software and hardware index
+ * @index: CRTC hardware index
  * @initialized: whether the CRTC has been initialized and clocks enabled
  * @dsysr: cached value of the DSYSR register
  * @vblank_enable: whether vblank events are enabled on this CRTC
@@ -39,10 +41,12 @@ struct rcar_du_vsp;
  * @group: CRTC group this CRTC belongs to
  * @vsp: VSP feeding video to this CRTC
  * @vsp_pipe: index of the VSP pipeline feeding video to this CRTC
+ * @writeback: the writeback connector
  */
 struct rcar_du_crtc {
 	struct drm_crtc crtc;
 
+	struct rcar_du_device *dev;
 	struct clk *clock;
 	struct clk *extclock;
 	unsigned int mmio_offset;
@@ -65,9 +69,12 @@ struct rcar_du_crtc {
 
 	const char *const *sources;
 	unsigned int sources_count;
+
+	struct drm_writeback_connector writeback;
 };
 
-#define to_rcar_crtc(c)	container_of(c, struct rcar_du_crtc, crtc)
+#define to_rcar_crtc(c)		container_of(c, struct rcar_du_crtc, crtc)
+#define wb_to_rcar_crtc(c)	container_of(c, struct rcar_du_crtc, writeback)
 
 /**
  * struct rcar_du_crtc_state - Driver-specific CRTC state
@@ -97,8 +104,6 @@ enum rcar_du_output {
 
 int rcar_du_crtc_create(struct rcar_du_group *rgrp, unsigned int swindex,
 			unsigned int hwindex);
-void rcar_du_crtc_suspend(struct rcar_du_crtc *rcrtc);
-void rcar_du_crtc_resume(struct rcar_du_crtc *rcrtc);
 
 void rcar_du_crtc_finish_page_flip(struct rcar_du_crtc *rcrtc);
 
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_encoder.c b/drivers/gpu/drm/rcar-du/rcar_du_encoder.c
index 8ee4e762f4e5..6c91753af7bc 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_encoder.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_encoder.c
@@ -28,13 +28,33 @@ static const struct drm_encoder_funcs encoder_funcs = {
 	.destroy = drm_encoder_cleanup,
 };
 
+static unsigned int rcar_du_encoder_count_ports(struct device_node *node)
+{
+	struct device_node *ports;
+	struct device_node *port;
+	unsigned int num_ports = 0;
+
+	ports = of_get_child_by_name(node, "ports");
+	if (!ports)
+		ports = of_node_get(node);
+
+	for_each_child_of_node(ports, port) {
+		if (of_node_name_eq(port, "port"))
+			num_ports++;
+	}
+
+	of_node_put(ports);
+
+	return num_ports;
+}
+
 int rcar_du_encoder_init(struct rcar_du_device *rcdu,
 			 enum rcar_du_output output,
 			 struct device_node *enc_node)
 {
 	struct rcar_du_encoder *renc;
 	struct drm_encoder *encoder;
-	struct drm_bridge *bridge = NULL;
+	struct drm_bridge *bridge;
 	int ret;
 
 	renc = devm_kzalloc(rcdu->dev, sizeof(*renc), GFP_KERNEL);
@@ -48,11 +68,33 @@ int rcar_du_encoder_init(struct rcar_du_device *rcdu,
 	dev_dbg(rcdu->dev, "initializing encoder %pOF for output %u\n",
 		enc_node, output);
 
-	/* Locate the DRM bridge from the encoder DT node. */
-	bridge = of_drm_find_bridge(enc_node);
-	if (!bridge) {
-		ret = -EPROBE_DEFER;
-		goto done;
+	/*
+	 * Locate the DRM bridge from the DT node. For the DPAD outputs, if the
+	 * DT node has a single port, assume that it describes a panel and
+	 * create a panel bridge.
+	 */
+	if ((output == RCAR_DU_OUTPUT_DPAD0 ||
+	     output == RCAR_DU_OUTPUT_DPAD1) &&
+	    rcar_du_encoder_count_ports(enc_node) == 1) {
+		struct drm_panel *panel = of_drm_find_panel(enc_node);
+
+		if (IS_ERR(panel)) {
+			ret = PTR_ERR(panel);
+			goto done;
+		}
+
+		bridge = devm_drm_panel_bridge_add(rcdu->dev, panel,
+						   DRM_MODE_CONNECTOR_DPI);
+		if (IS_ERR(bridge)) {
+			ret = PTR_ERR(bridge);
+			goto done;
+		}
+	} else {
+		bridge = of_drm_find_bridge(enc_node);
+		if (!bridge) {
+			ret = -EPROBE_DEFER;
+			goto done;
+		}
 	}
 
 	ret = drm_encoder_init(rcdu->ddev, encoder, &encoder_funcs,
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_kms.c b/drivers/gpu/drm/rcar-du/rcar_du_kms.c
index 3b7d50a8fb9b..f8f7fff34dff 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_kms.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_kms.c
@@ -26,6 +26,7 @@
 #include "rcar_du_kms.h"
 #include "rcar_du_regs.h"
 #include "rcar_du_vsp.h"
+#include "rcar_du_writeback.h"
 
 /* -----------------------------------------------------------------------------
  * Format helpers
@@ -34,60 +35,70 @@
 static const struct rcar_du_format_info rcar_du_format_infos[] = {
 	{
 		.fourcc = DRM_FORMAT_RGB565,
+		.v4l2 = V4L2_PIX_FMT_RGB565,
 		.bpp = 16,
 		.planes = 1,
 		.pnmr = PnMR_SPIM_TP | PnMR_DDDF_16BPP,
 		.edf = PnDDCR4_EDF_NONE,
 	}, {
 		.fourcc = DRM_FORMAT_ARGB1555,
+		.v4l2 = V4L2_PIX_FMT_ARGB555,
 		.bpp = 16,
 		.planes = 1,
 		.pnmr = PnMR_SPIM_ALP | PnMR_DDDF_ARGB,
 		.edf = PnDDCR4_EDF_NONE,
 	}, {
 		.fourcc = DRM_FORMAT_XRGB1555,
+		.v4l2 = V4L2_PIX_FMT_XRGB555,
 		.bpp = 16,
 		.planes = 1,
 		.pnmr = PnMR_SPIM_ALP | PnMR_DDDF_ARGB,
 		.edf = PnDDCR4_EDF_NONE,
 	}, {
 		.fourcc = DRM_FORMAT_XRGB8888,
+		.v4l2 = V4L2_PIX_FMT_XBGR32,
 		.bpp = 32,
 		.planes = 1,
 		.pnmr = PnMR_SPIM_TP | PnMR_DDDF_16BPP,
 		.edf = PnDDCR4_EDF_RGB888,
 	}, {
 		.fourcc = DRM_FORMAT_ARGB8888,
+		.v4l2 = V4L2_PIX_FMT_ABGR32,
 		.bpp = 32,
 		.planes = 1,
 		.pnmr = PnMR_SPIM_ALP | PnMR_DDDF_16BPP,
 		.edf = PnDDCR4_EDF_ARGB8888,
 	}, {
 		.fourcc = DRM_FORMAT_UYVY,
+		.v4l2 = V4L2_PIX_FMT_UYVY,
 		.bpp = 16,
 		.planes = 1,
 		.pnmr = PnMR_SPIM_TP_OFF | PnMR_DDDF_YC,
 		.edf = PnDDCR4_EDF_NONE,
 	}, {
 		.fourcc = DRM_FORMAT_YUYV,
+		.v4l2 = V4L2_PIX_FMT_YUYV,
 		.bpp = 16,
 		.planes = 1,
 		.pnmr = PnMR_SPIM_TP_OFF | PnMR_DDDF_YC,
 		.edf = PnDDCR4_EDF_NONE,
 	}, {
 		.fourcc = DRM_FORMAT_NV12,
+		.v4l2 = V4L2_PIX_FMT_NV12M,
 		.bpp = 12,
 		.planes = 2,
 		.pnmr = PnMR_SPIM_TP_OFF | PnMR_DDDF_YC,
 		.edf = PnDDCR4_EDF_NONE,
 	}, {
 		.fourcc = DRM_FORMAT_NV21,
+		.v4l2 = V4L2_PIX_FMT_NV21M,
 		.bpp = 12,
 		.planes = 2,
 		.pnmr = PnMR_SPIM_TP_OFF | PnMR_DDDF_YC,
 		.edf = PnDDCR4_EDF_NONE,
 	}, {
 		.fourcc = DRM_FORMAT_NV16,
+		.v4l2 = V4L2_PIX_FMT_NV16M,
 		.bpp = 16,
 		.planes = 2,
 		.pnmr = PnMR_SPIM_TP_OFF | PnMR_DDDF_YC,
@@ -99,62 +110,77 @@ static const struct rcar_du_format_info rcar_du_format_infos[] = {
 	 */
 	{
 		.fourcc = DRM_FORMAT_RGB332,
+		.v4l2 = V4L2_PIX_FMT_RGB332,
 		.bpp = 8,
 		.planes = 1,
 	}, {
 		.fourcc = DRM_FORMAT_ARGB4444,
+		.v4l2 = V4L2_PIX_FMT_ARGB444,
 		.bpp = 16,
 		.planes = 1,
 	}, {
 		.fourcc = DRM_FORMAT_XRGB4444,
+		.v4l2 = V4L2_PIX_FMT_XRGB444,
 		.bpp = 16,
 		.planes = 1,
 	}, {
 		.fourcc = DRM_FORMAT_BGR888,
+		.v4l2 = V4L2_PIX_FMT_RGB24,
 		.bpp = 24,
 		.planes = 1,
 	}, {
 		.fourcc = DRM_FORMAT_RGB888,
+		.v4l2 = V4L2_PIX_FMT_BGR24,
 		.bpp = 24,
 		.planes = 1,
 	}, {
 		.fourcc = DRM_FORMAT_BGRA8888,
+		.v4l2 = V4L2_PIX_FMT_ARGB32,
 		.bpp = 32,
 		.planes = 1,
 	}, {
 		.fourcc = DRM_FORMAT_BGRX8888,
+		.v4l2 = V4L2_PIX_FMT_XRGB32,
 		.bpp = 32,
 		.planes = 1,
 	}, {
 		.fourcc = DRM_FORMAT_YVYU,
+		.v4l2 = V4L2_PIX_FMT_YVYU,
 		.bpp = 16,
 		.planes = 1,
 	}, {
 		.fourcc = DRM_FORMAT_NV61,
+		.v4l2 = V4L2_PIX_FMT_NV61M,
 		.bpp = 16,
 		.planes = 2,
 	}, {
 		.fourcc = DRM_FORMAT_YUV420,
+		.v4l2 = V4L2_PIX_FMT_YUV420M,
 		.bpp = 12,
 		.planes = 3,
 	}, {
 		.fourcc = DRM_FORMAT_YVU420,
+		.v4l2 = V4L2_PIX_FMT_YVU420M,
 		.bpp = 12,
 		.planes = 3,
 	}, {
 		.fourcc = DRM_FORMAT_YUV422,
+		.v4l2 = V4L2_PIX_FMT_YUV422M,
 		.bpp = 16,
 		.planes = 3,
 	}, {
 		.fourcc = DRM_FORMAT_YVU422,
+		.v4l2 = V4L2_PIX_FMT_YVU422M,
 		.bpp = 16,
 		.planes = 3,
 	}, {
 		.fourcc = DRM_FORMAT_YUV444,
+		.v4l2 = V4L2_PIX_FMT_YUV444M,
 		.bpp = 24,
 		.planes = 3,
 	}, {
 		.fourcc = DRM_FORMAT_YVU444,
+		.v4l2 = V4L2_PIX_FMT_YVU444M,
 		.bpp = 24,
 		.planes = 3,
 	},
@@ -639,6 +665,17 @@ int rcar_du_modeset_init(struct rcar_du_device *rcdu)
 		encoder->possible_clones = (1 << num_encoders) - 1;
 	}
 
+	/* Create the writeback connectors. */
+	if (rcdu->info->gen >= 3) {
+		for (i = 0; i < rcdu->num_crtcs; ++i) {
+			struct rcar_du_crtc *rcrtc = &rcdu->crtcs[i];
+
+			ret = rcar_du_writeback_init(rcdu, rcrtc);
+			if (ret < 0)
+				return ret;
+		}
+	}
+
 	/*
 	 * Initialize the default DPAD0 source to the index of the first DU
 	 * channel that can be connected to DPAD0. The exact value doesn't
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_kms.h b/drivers/gpu/drm/rcar-du/rcar_du_kms.h
index e171527abdaa..0346504d8c59 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_kms.h
+++ b/drivers/gpu/drm/rcar-du/rcar_du_kms.h
@@ -19,6 +19,7 @@ struct rcar_du_device;
 
 struct rcar_du_format_info {
 	u32 fourcc;
+	u32 v4l2;
 	unsigned int bpp;
 	unsigned int planes;
 	unsigned int pnmr;
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_vsp.c b/drivers/gpu/drm/rcar-du/rcar_du_vsp.c
index 0878accbd134..5e4faf258c31 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_vsp.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_vsp.c
@@ -10,6 +10,7 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fourcc.h>
 #include <drm/drm_gem_cma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_plane_helper.h>
@@ -26,16 +27,19 @@
 #include "rcar_du_drv.h"
 #include "rcar_du_kms.h"
 #include "rcar_du_vsp.h"
+#include "rcar_du_writeback.h"
 
-static void rcar_du_vsp_complete(void *private, bool completed, u32 crc)
+static void rcar_du_vsp_complete(void *private, unsigned int status, u32 crc)
 {
 	struct rcar_du_crtc *crtc = private;
 
 	if (crtc->vblank_enable)
 		drm_crtc_handle_vblank(&crtc->crtc);
 
-	if (completed)
+	if (status & VSP1_DU_STATUS_COMPLETE)
 		rcar_du_crtc_finish_page_flip(crtc);
+	if (status & VSP1_DU_STATUS_WRITEBACK)
+		rcar_du_writeback_complete(crtc);
 
 	drm_crtc_add_crc_entry(&crtc->crtc, false, 0, &crc);
 }
@@ -43,7 +47,7 @@ static void rcar_du_vsp_complete(void *private, bool completed, u32 crc)
 void rcar_du_vsp_enable(struct rcar_du_crtc *crtc)
 {
 	const struct drm_display_mode *mode = &crtc->crtc.state->adjusted_mode;
-	struct rcar_du_device *rcdu = crtc->group->dev;
+	struct rcar_du_device *rcdu = crtc->dev;
 	struct vsp1_du_lif_config cfg = {
 		.width = mode->hdisplay,
 		.height = mode->vdisplay,
@@ -107,11 +111,12 @@ void rcar_du_vsp_atomic_flush(struct rcar_du_crtc *crtc)
 	state = to_rcar_crtc_state(crtc->crtc.state);
 	cfg.crc = state->crc;
 
+	rcar_du_writeback_setup(crtc, &cfg.writeback);
+
 	vsp1_du_atomic_flush(crtc->vsp->vsp, crtc->vsp_pipe, &cfg);
 }
 
-/* Keep the two tables in sync. */
-static const u32 formats_kms[] = {
+static const u32 rcar_du_vsp_formats[] = {
 	DRM_FORMAT_RGB332,
 	DRM_FORMAT_ARGB4444,
 	DRM_FORMAT_XRGB4444,
@@ -139,40 +144,13 @@ static const u32 formats_kms[] = {
 	DRM_FORMAT_YVU444,
 };
 
-static const u32 formats_v4l2[] = {
-	V4L2_PIX_FMT_RGB332,
-	V4L2_PIX_FMT_ARGB444,
-	V4L2_PIX_FMT_XRGB444,
-	V4L2_PIX_FMT_ARGB555,
-	V4L2_PIX_FMT_XRGB555,
-	V4L2_PIX_FMT_RGB565,
-	V4L2_PIX_FMT_RGB24,
-	V4L2_PIX_FMT_BGR24,
-	V4L2_PIX_FMT_ARGB32,
-	V4L2_PIX_FMT_XRGB32,
-	V4L2_PIX_FMT_ABGR32,
-	V4L2_PIX_FMT_XBGR32,
-	V4L2_PIX_FMT_UYVY,
-	V4L2_PIX_FMT_YUYV,
-	V4L2_PIX_FMT_YVYU,
-	V4L2_PIX_FMT_NV12M,
-	V4L2_PIX_FMT_NV21M,
-	V4L2_PIX_FMT_NV16M,
-	V4L2_PIX_FMT_NV61M,
-	V4L2_PIX_FMT_YUV420M,
-	V4L2_PIX_FMT_YVU420M,
-	V4L2_PIX_FMT_YUV422M,
-	V4L2_PIX_FMT_YVU422M,
-	V4L2_PIX_FMT_YUV444M,
-	V4L2_PIX_FMT_YVU444M,
-};
-
 static void rcar_du_vsp_plane_setup(struct rcar_du_vsp_plane *plane)
 {
 	struct rcar_du_vsp_plane_state *state =
 		to_rcar_vsp_plane_state(plane->plane.state);
 	struct rcar_du_crtc *crtc = to_rcar_crtc(state->state.crtc);
 	struct drm_framebuffer *fb = plane->plane.state->fb;
+	const struct rcar_du_format_info *format;
 	struct vsp1_du_atomic_config cfg = {
 		.pixelformat = 0,
 		.pitch = fb->pitches[0],
@@ -195,37 +173,23 @@ static void rcar_du_vsp_plane_setup(struct rcar_du_vsp_plane *plane)
 		cfg.mem[i] = sg_dma_address(state->sg_tables[i].sgl)
 			   + fb->offsets[i];
 
-	for (i = 0; i < ARRAY_SIZE(formats_kms); ++i) {
-		if (formats_kms[i] == state->format->fourcc) {
-			cfg.pixelformat = formats_v4l2[i];
-			break;
-		}
-	}
+	format = rcar_du_format_info(state->format->fourcc);
+	cfg.pixelformat = format->v4l2;
 
 	vsp1_du_atomic_update(plane->vsp->vsp, crtc->vsp_pipe,
 			      plane->index, &cfg);
 }
 
-static int rcar_du_vsp_plane_prepare_fb(struct drm_plane *plane,
-					struct drm_plane_state *state)
+int rcar_du_vsp_map_fb(struct rcar_du_vsp *vsp, struct drm_framebuffer *fb,
+		       struct sg_table sg_tables[3])
 {
-	struct rcar_du_vsp_plane_state *rstate = to_rcar_vsp_plane_state(state);
-	struct rcar_du_vsp *vsp = to_rcar_vsp_plane(plane)->vsp;
 	struct rcar_du_device *rcdu = vsp->dev;
 	unsigned int i;
 	int ret;
 
-	/*
-	 * There's no need to prepare (and unprepare) the framebuffer when the
-	 * plane is not visible, as it will not be displayed.
-	 */
-	if (!state->visible)
-		return 0;
-
-	for (i = 0; i < rstate->format->planes; ++i) {
-		struct drm_gem_cma_object *gem =
-			drm_fb_cma_get_gem_obj(state->fb, i);
-		struct sg_table *sgt = &rstate->sg_tables[i];
+	for (i = 0; i < fb->format->num_planes; ++i) {
+		struct drm_gem_cma_object *gem = drm_fb_cma_get_gem_obj(fb, i);
+		struct sg_table *sgt = &sg_tables[i];
 
 		ret = dma_get_sgtable(rcdu->dev, sgt, gem->vaddr, gem->paddr,
 				      gem->base.size);
@@ -240,15 +204,11 @@ static int rcar_du_vsp_plane_prepare_fb(struct drm_plane *plane,
 		}
 	}
 
-	ret = drm_gem_fb_prepare_fb(plane, state);
-	if (ret)
-		goto fail;
-
 	return 0;
 
 fail:
 	while (i--) {
-		struct sg_table *sgt = &rstate->sg_tables[i];
+		struct sg_table *sgt = &sg_tables[i];
 
 		vsp1_du_unmap_sg(vsp->vsp, sgt);
 		sg_free_table(sgt);
@@ -257,24 +217,52 @@ fail:
 	return ret;
 }
 
-static void rcar_du_vsp_plane_cleanup_fb(struct drm_plane *plane,
-					 struct drm_plane_state *state)
+static int rcar_du_vsp_plane_prepare_fb(struct drm_plane *plane,
+					struct drm_plane_state *state)
 {
 	struct rcar_du_vsp_plane_state *rstate = to_rcar_vsp_plane_state(state);
 	struct rcar_du_vsp *vsp = to_rcar_vsp_plane(plane)->vsp;
-	unsigned int i;
+	int ret;
 
+	/*
+	 * There's no need to prepare (and unprepare) the framebuffer when the
+	 * plane is not visible, as it will not be displayed.
+	 */
 	if (!state->visible)
-		return;
+		return 0;
+
+	ret = rcar_du_vsp_map_fb(vsp, state->fb, rstate->sg_tables);
+	if (ret < 0)
+		return ret;
+
+	return drm_gem_fb_prepare_fb(plane, state);
+}
+
+void rcar_du_vsp_unmap_fb(struct rcar_du_vsp *vsp, struct drm_framebuffer *fb,
+			  struct sg_table sg_tables[3])
+{
+	unsigned int i;
 
-	for (i = 0; i < rstate->format->planes; ++i) {
-		struct sg_table *sgt = &rstate->sg_tables[i];
+	for (i = 0; i < fb->format->num_planes; ++i) {
+		struct sg_table *sgt = &sg_tables[i];
 
 		vsp1_du_unmap_sg(vsp->vsp, sgt);
 		sg_free_table(sgt);
 	}
 }
 
+static void rcar_du_vsp_plane_cleanup_fb(struct drm_plane *plane,
+					 struct drm_plane_state *state)
+{
+	struct rcar_du_vsp_plane_state *rstate = to_rcar_vsp_plane_state(state);
+	struct rcar_du_vsp *vsp = to_rcar_vsp_plane(plane)->vsp;
+
+	if (!state->visible)
+		return;
+
+	rcar_du_vsp_unmap_fb(vsp, state->fb, rstate->sg_tables);
+}
+
 static int rcar_du_vsp_plane_atomic_check(struct drm_plane *plane,
 					  struct drm_plane_state *state)
 {
@@ -395,8 +383,8 @@ int rcar_du_vsp_init(struct rcar_du_vsp *vsp, struct device_node *np,
 
 		ret = drm_universal_plane_init(rcdu->ddev, &plane->plane, crtcs,
 					       &rcar_du_vsp_plane_funcs,
-					       formats_kms,
-					       ARRAY_SIZE(formats_kms),
+					       rcar_du_vsp_formats,
+					       ARRAY_SIZE(rcar_du_vsp_formats),
 					       NULL, type, NULL);
 		if (ret < 0)
 			return ret;
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_vsp.h b/drivers/gpu/drm/rcar-du/rcar_du_vsp.h
index db232037f24a..9b4724159378 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_vsp.h
+++ b/drivers/gpu/drm/rcar-du/rcar_du_vsp.h
@@ -12,8 +12,10 @@
 
 #include <drm/drm_plane.h>
 
+struct drm_framebuffer;
 struct rcar_du_format_info;
 struct rcar_du_vsp;
+struct sg_table;
 
 struct rcar_du_vsp_plane {
 	struct drm_plane plane;
@@ -60,6 +62,10 @@ void rcar_du_vsp_enable(struct rcar_du_crtc *crtc);
 void rcar_du_vsp_disable(struct rcar_du_crtc *crtc);
 void rcar_du_vsp_atomic_begin(struct rcar_du_crtc *crtc);
 void rcar_du_vsp_atomic_flush(struct rcar_du_crtc *crtc);
+int rcar_du_vsp_map_fb(struct rcar_du_vsp *vsp, struct drm_framebuffer *fb,
+		       struct sg_table sg_tables[3]);
+void rcar_du_vsp_unmap_fb(struct rcar_du_vsp *vsp, struct drm_framebuffer *fb,
+			  struct sg_table sg_tables[3]);
 #else
 static inline int rcar_du_vsp_init(struct rcar_du_vsp *vsp,
 				   struct device_node *np,
@@ -71,6 +77,17 @@ static inline void rcar_du_vsp_enable(struct rcar_du_crtc *crtc) { };
 static inline void rcar_du_vsp_disable(struct rcar_du_crtc *crtc) { };
 static inline void rcar_du_vsp_atomic_begin(struct rcar_du_crtc *crtc) { };
 static inline void rcar_du_vsp_atomic_flush(struct rcar_du_crtc *crtc) { };
+static inline int rcar_du_vsp_map_fb(struct rcar_du_vsp *vsp,
+				     struct drm_framebuffer *fb,
+				     struct sg_table sg_tables[3])
+{
+	return -ENXIO;
+}
+static inline void rcar_du_vsp_unmap_fb(struct rcar_du_vsp *vsp,
+					struct drm_framebuffer *fb,
+					struct sg_table sg_tables[3])
+{
+}
 #endif
 
 #endif /* __RCAR_DU_VSP_H__ */
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_writeback.c b/drivers/gpu/drm/rcar-du/rcar_du_writeback.c
new file mode 100644
index 000000000000..989a0be94131
--- /dev/null
+++ b/drivers/gpu/drm/rcar-du/rcar_du_writeback.c
@@ -0,0 +1,243 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * rcar_du_writeback.c  --  R-Car Display Unit Writeback Support
+ *
+ * Copyright (C) 2019 Laurent Pinchart <laurent.pinchart@ideasonboard.com>
+ */
+
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_device.h>
+#include <drm/drm_fourcc.h>
+#include <drm/drm_probe_helper.h>
+#include <drm/drm_writeback.h>
+
+#include "rcar_du_crtc.h"
+#include "rcar_du_drv.h"
+#include "rcar_du_kms.h"
+
+/**
+ * struct rcar_du_wb_conn_state - Driver-specific writeback connector state
+ * @state: base DRM connector state
+ * @format: format of the writeback framebuffer
+ */
+struct rcar_du_wb_conn_state {
+	struct drm_connector_state state;
+	const struct rcar_du_format_info *format;
+};
+
+#define to_rcar_wb_conn_state(s) \
+	container_of(s, struct rcar_du_wb_conn_state, state)
+
+/**
+ * struct rcar_du_wb_job - Driver-private data for writeback jobs
+ * @sg_tables: scatter-gather tables for the framebuffer memory
+ */
+struct rcar_du_wb_job {
+	struct sg_table sg_tables[3];
+};
+
+static int rcar_du_wb_conn_get_modes(struct drm_connector *connector)
+{
+	struct drm_device *dev = connector->dev;
+
+	return drm_add_modes_noedid(connector, dev->mode_config.max_width,
+				    dev->mode_config.max_height);
+}
+
+static int rcar_du_wb_prepare_job(struct drm_writeback_connector *connector,
+				  struct drm_writeback_job *job)
+{
+	struct rcar_du_crtc *rcrtc = wb_to_rcar_crtc(connector);
+	struct rcar_du_wb_job *rjob;
+	int ret;
+
+	if (!job->fb)
+		return 0;
+
+	rjob = kzalloc(sizeof(*rjob), GFP_KERNEL);
+	if (!rjob)
+		return -ENOMEM;
+
+	/* Map the framebuffer to the VSP. */
+	ret = rcar_du_vsp_map_fb(rcrtc->vsp, job->fb, rjob->sg_tables);
+	if (ret < 0) {
+		kfree(rjob);
+		return ret;
+	}
+
+	job->priv = rjob;
+	return 0;
+}
+
+static void rcar_du_wb_cleanup_job(struct drm_writeback_connector *connector,
+				   struct drm_writeback_job *job)
+{
+	struct rcar_du_crtc *rcrtc = wb_to_rcar_crtc(connector);
+	struct rcar_du_wb_job *rjob = job->priv;
+
+	if (!job->fb)
+		return;
+
+	rcar_du_vsp_unmap_fb(rcrtc->vsp, job->fb, rjob->sg_tables);
+	kfree(rjob);
+}
+
+static const struct drm_connector_helper_funcs rcar_du_wb_conn_helper_funcs = {
+	.get_modes = rcar_du_wb_conn_get_modes,
+	.prepare_writeback_job = rcar_du_wb_prepare_job,
+	.cleanup_writeback_job = rcar_du_wb_cleanup_job,
+};
+
+static struct drm_connector_state *
+rcar_du_wb_conn_duplicate_state(struct drm_connector *connector)
+{
+	struct rcar_du_wb_conn_state *copy;
+
+	if (WARN_ON(!connector->state))
+		return NULL;
+
+	copy = kzalloc(sizeof(*copy), GFP_KERNEL);
+	if (!copy)
+		return NULL;
+
+	__drm_atomic_helper_connector_duplicate_state(connector, &copy->state);
+
+	return &copy->state;
+}
+
+static void rcar_du_wb_conn_destroy_state(struct drm_connector *connector,
+					  struct drm_connector_state *state)
+{
+	__drm_atomic_helper_connector_destroy_state(state);
+	kfree(to_rcar_wb_conn_state(state));
+}
+
+static void rcar_du_wb_conn_reset(struct drm_connector *connector)
+{
+	struct rcar_du_wb_conn_state *state;
+
+	if (connector->state) {
+		rcar_du_wb_conn_destroy_state(connector, connector->state);
+		connector->state = NULL;
+	}
+
+	state = kzalloc(sizeof(*state), GFP_KERNEL);
+	if (state == NULL)
+		return;
+
+	__drm_atomic_helper_connector_reset(connector, &state->state);
+}
+
+static const struct drm_connector_funcs rcar_du_wb_conn_funcs = {
+	.reset = rcar_du_wb_conn_reset,
+	.fill_modes = drm_helper_probe_single_connector_modes,
+	.destroy = drm_connector_cleanup,
+	.atomic_duplicate_state = rcar_du_wb_conn_duplicate_state,
+	.atomic_destroy_state = rcar_du_wb_conn_destroy_state,
+};
+
+static int rcar_du_wb_enc_atomic_check(struct drm_encoder *encoder,
+				       struct drm_crtc_state *crtc_state,
+				       struct drm_connector_state *conn_state)
+{
+	struct rcar_du_wb_conn_state *wb_state =
+		to_rcar_wb_conn_state(conn_state);
+	const struct drm_display_mode *mode = &crtc_state->mode;
+	struct drm_device *dev = encoder->dev;
+	struct drm_framebuffer *fb;
+
+	if (!conn_state->writeback_job || !conn_state->writeback_job->fb)
+		return 0;
+
+	fb = conn_state->writeback_job->fb;
+
+	/*
+	 * Verify that the framebuffer format is supported and that its size
+	 * matches the current mode.
+	 */
+	if (fb->width != mode->hdisplay || fb->height != mode->vdisplay) {
+		dev_dbg(dev->dev, "%s: invalid framebuffer size %ux%u\n",
+			__func__, fb->width, fb->height);
+		return -EINVAL;
+	}
+
+	wb_state->format = rcar_du_format_info(fb->format->format);
+	if (wb_state->format == NULL) {
+		dev_dbg(dev->dev, "%s: unsupported format %08x\n", __func__,
+			fb->format->format);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static const struct drm_encoder_helper_funcs rcar_du_wb_enc_helper_funcs = {
+	.atomic_check = rcar_du_wb_enc_atomic_check,
+};
+
+/*
+ * Only RGB formats are currently supported as the VSP outputs RGB to the DU
+ * and can't convert to YUV separately for writeback.
+ */
+static const u32 writeback_formats[] = {
+	DRM_FORMAT_RGB332,
+	DRM_FORMAT_ARGB4444,
+	DRM_FORMAT_XRGB4444,
+	DRM_FORMAT_ARGB1555,
+	DRM_FORMAT_XRGB1555,
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_BGR888,
+	DRM_FORMAT_RGB888,
+	DRM_FORMAT_BGRA8888,
+	DRM_FORMAT_BGRX8888,
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_XRGB8888,
+};
+
+int rcar_du_writeback_init(struct rcar_du_device *rcdu,
+			   struct rcar_du_crtc *rcrtc)
+{
+	struct drm_writeback_connector *wb_conn = &rcrtc->writeback;
+
+	wb_conn->encoder.possible_crtcs = 1 << drm_crtc_index(&rcrtc->crtc);
+	drm_connector_helper_add(&wb_conn->base,
+				 &rcar_du_wb_conn_helper_funcs);
+
+	return drm_writeback_connector_init(rcdu->ddev, wb_conn,
+					    &rcar_du_wb_conn_funcs,
+					    &rcar_du_wb_enc_helper_funcs,
+					    writeback_formats,
+					    ARRAY_SIZE(writeback_formats));
+}
+
+void rcar_du_writeback_setup(struct rcar_du_crtc *rcrtc,
+			     struct vsp1_du_writeback_config *cfg)
+{
+	struct rcar_du_wb_conn_state *wb_state;
+	struct drm_connector_state *state;
+	struct rcar_du_wb_job *rjob;
+	struct drm_framebuffer *fb;
+	unsigned int i;
+
+	state = rcrtc->writeback.base.state;
+	if (!state || !state->writeback_job || !state->writeback_job->fb)
+		return;
+
+	fb = state->writeback_job->fb;
+	rjob = state->writeback_job->priv;
+	wb_state = to_rcar_wb_conn_state(state);
+
+	cfg->pixelformat = wb_state->format->v4l2;
+	cfg->pitch = fb->pitches[0];
+
+	for (i = 0; i < wb_state->format->planes; ++i)
+		cfg->mem[i] = sg_dma_address(rjob->sg_tables[i].sgl)
+			    + fb->offsets[i];
+
+	drm_writeback_queue_job(&rcrtc->writeback, state);
+}
+
+void rcar_du_writeback_complete(struct rcar_du_crtc *rcrtc)
+{
+	drm_writeback_signal_completion(&rcrtc->writeback, 0);
+}
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_writeback.h b/drivers/gpu/drm/rcar-du/rcar_du_writeback.h
new file mode 100644
index 000000000000..fa87ebf8d21f
--- /dev/null
+++ b/drivers/gpu/drm/rcar-du/rcar_du_writeback.h
@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * rcar_du_writeback.h  --  R-Car Display Unit Writeback Support
+ *
+ * Copyright (C) 2019 Laurent Pinchart <laurent.pinchart@ideasonboard.com>
+ */
+
+#ifndef __RCAR_DU_WRITEBACK_H__
+#define __RCAR_DU_WRITEBACK_H__
+
+#include <drm/drm_plane.h>
+
+struct rcar_du_crtc;
+struct rcar_du_device;
+struct vsp1_du_atomic_pipe_config;
+
+#ifdef CONFIG_DRM_RCAR_WRITEBACK
+int rcar_du_writeback_init(struct rcar_du_device *rcdu,
+			   struct rcar_du_crtc *rcrtc);
+void rcar_du_writeback_setup(struct rcar_du_crtc *rcrtc,
+			     struct vsp1_du_writeback_config *cfg);
+void rcar_du_writeback_complete(struct rcar_du_crtc *rcrtc);
+#else
+static inline int rcar_du_writeback_init(struct rcar_du_device *rcdu,
+					 struct rcar_du_crtc *rcrtc)
+{
+	return -ENXIO;
+}
+static inline void
+rcar_du_writeback_setup(struct rcar_du_crtc *rcrtc,
+			struct vsp1_du_writeback_config *cfg)
+{
+}
+static inline void rcar_du_writeback_complete(struct rcar_du_crtc *rcrtc)
+{
+}
+#endif
+
+#endif /* __RCAR_DU_WRITEBACK_H__ */
diff --git a/drivers/gpu/drm/rcar-du/rcar_lvds.c b/drivers/gpu/drm/rcar-du/rcar_lvds.c
index 7ef97b2a6eda..620b51aab291 100644
--- a/drivers/gpu/drm/rcar-du/rcar_lvds.c
+++ b/drivers/gpu/drm/rcar-du/rcar_lvds.c
@@ -283,7 +283,7 @@ static void rcar_lvds_d3_e3_pll_calc(struct rcar_lvds *lvds, struct clk *clk,
 				 * divider.
 				 */
 				fout = fvco / (1 << e) / div7;
-				div = DIV_ROUND_CLOSEST(fout, target);
+				div = max(1UL, DIV_ROUND_CLOSEST(fout, target));
 				diff = abs(fout / div - target);
 
 				if (diff < pll->diff) {
@@ -485,9 +485,13 @@ static void rcar_lvds_enable(struct drm_bridge *bridge)
 	}
 
 	if (lvds->info->quirks & RCAR_LVDS_QUIRK_GEN3_LVEN) {
-		/* Turn on the LVDS PHY. */
+		/*
+		 * Turn on the LVDS PHY. On D3, the LVEN and LVRES bit must be
+		 * set at the same time, so don't write the register yet.
+		 */
 		lvdcr0 |= LVDCR0_LVEN;
-		rcar_lvds_write(lvds, LVDCR0, lvdcr0);
+		if (!(lvds->info->quirks & RCAR_LVDS_QUIRK_PWD))
+			rcar_lvds_write(lvds, LVDCR0, lvdcr0);
 	}
 
 	if (!(lvds->info->quirks & RCAR_LVDS_QUIRK_EXT_PLL)) {
@@ -531,11 +535,16 @@ static bool rcar_lvds_mode_fixup(struct drm_bridge *bridge,
 				 const struct drm_display_mode *mode,
 				 struct drm_display_mode *adjusted_mode)
 {
+	struct rcar_lvds *lvds = bridge_to_rcar_lvds(bridge);
+	int min_freq;
+
 	/*
 	 * The internal LVDS encoder has a restricted clock frequency operating
-	 * range (31MHz to 148.5MHz). Clamp the clock accordingly.
+	 * range, from 5MHz to 148.5MHz on D3 and E3, and from 31MHz to
+	 * 148.5MHz on all other platforms. Clamp the clock accordingly.
 	 */
-	adjusted_mode->clock = clamp(adjusted_mode->clock, 31000, 148500);
+	min_freq = lvds->info->quirks & RCAR_LVDS_QUIRK_EXT_PLL ? 5000 : 31000;
+	adjusted_mode->clock = clamp(adjusted_mode->clock, min_freq, 148500);
 
 	return true;
 }
diff --git a/drivers/gpu/drm/rockchip/Kconfig b/drivers/gpu/drm/rockchip/Kconfig
index 1e75196f9659..2cdf3b62d559 100644
--- a/drivers/gpu/drm/rockchip/Kconfig
+++ b/drivers/gpu/drm/rockchip/Kconfig
@@ -77,4 +77,12 @@ config ROCKCHIP_RGB
 	  Some Rockchip CRTCs, like rv1108, can directly output parallel
 	  and serial RGB format to panel or connect to a conversion chip.
 	  say Y to enable its driver.
+
+config ROCKCHIP_RK3066_HDMI
+	bool "Rockchip specific extensions for RK3066 HDMI"
+	depends on DRM_ROCKCHIP
+	help
+	  This selects support for Rockchip SoC specific extensions
+	  for the RK3066 HDMI driver. If you want to enable
+	  HDMI on RK3066 based SoC, you should select this option.
 endif
diff --git a/drivers/gpu/drm/rockchip/Makefile b/drivers/gpu/drm/rockchip/Makefile
index f6fc9d5dd0ad..524684ba7f6a 100644
--- a/drivers/gpu/drm/rockchip/Makefile
+++ b/drivers/gpu/drm/rockchip/Makefile
@@ -15,5 +15,6 @@ rockchipdrm-$(CONFIG_ROCKCHIP_DW_MIPI_DSI) += dw-mipi-dsi-rockchip.o
 rockchipdrm-$(CONFIG_ROCKCHIP_INNO_HDMI) += inno_hdmi.o
 rockchipdrm-$(CONFIG_ROCKCHIP_LVDS) += rockchip_lvds.o
 rockchipdrm-$(CONFIG_ROCKCHIP_RGB) += rockchip_rgb.o
+rockchipdrm-$(CONFIG_ROCKCHIP_RK3066_HDMI) += rk3066_hdmi.o
 
 obj-$(CONFIG_DRM_ROCKCHIP) += rockchipdrm.o
diff --git a/drivers/gpu/drm/rockchip/rk3066_hdmi.c b/drivers/gpu/drm/rockchip/rk3066_hdmi.c
new file mode 100644
index 000000000000..85fc5f01f761
--- /dev/null
+++ b/drivers/gpu/drm/rockchip/rk3066_hdmi.c
@@ -0,0 +1,876 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd
+ *    Zheng Yang <zhengyang@rock-chips.com>
+ */
+
+#include <drm/drm_of.h>
+#include <drm/drm_probe_helper.h>
+
+#include <linux/clk.h>
+#include <linux/mfd/syscon.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#include "rk3066_hdmi.h"
+
+#include "rockchip_drm_drv.h"
+#include "rockchip_drm_vop.h"
+
+#define DEFAULT_PLLA_RATE 30000000
+
+struct hdmi_data_info {
+	int vic; /* The CEA Video ID (VIC) of the current drm display mode. */
+	bool sink_is_hdmi;
+	unsigned int enc_out_format;
+	unsigned int colorimetry;
+};
+
+struct rk3066_hdmi_i2c {
+	struct i2c_adapter adap;
+
+	u8 ddc_addr;
+	u8 segment_addr;
+	u8 stat;
+
+	struct mutex i2c_lock; /* For i2c operation. */
+	struct completion cmpltn;
+};
+
+struct rk3066_hdmi {
+	struct device *dev;
+	struct drm_device *drm_dev;
+	struct regmap *grf_regmap;
+	int irq;
+	struct clk *hclk;
+	void __iomem *regs;
+
+	struct drm_connector connector;
+	struct drm_encoder encoder;
+
+	struct rk3066_hdmi_i2c *i2c;
+	struct i2c_adapter *ddc;
+
+	unsigned int tmdsclk;
+
+	struct hdmi_data_info hdmi_data;
+	struct drm_display_mode previous_mode;
+};
+
+#define to_rk3066_hdmi(x) container_of(x, struct rk3066_hdmi, x)
+
+static inline u8 hdmi_readb(struct rk3066_hdmi *hdmi, u16 offset)
+{
+	return readl_relaxed(hdmi->regs + offset);
+}
+
+static inline void hdmi_writeb(struct rk3066_hdmi *hdmi, u16 offset, u32 val)
+{
+	writel_relaxed(val, hdmi->regs + offset);
+}
+
+static inline void hdmi_modb(struct rk3066_hdmi *hdmi, u16 offset,
+			     u32 msk, u32 val)
+{
+	u8 temp = hdmi_readb(hdmi, offset) & ~msk;
+
+	temp |= val & msk;
+	hdmi_writeb(hdmi, offset, temp);
+}
+
+static void rk3066_hdmi_i2c_init(struct rk3066_hdmi *hdmi)
+{
+	int ddc_bus_freq;
+
+	ddc_bus_freq = (hdmi->tmdsclk >> 2) / HDMI_SCL_RATE;
+
+	hdmi_writeb(hdmi, HDMI_DDC_BUS_FREQ_L, ddc_bus_freq & 0xFF);
+	hdmi_writeb(hdmi, HDMI_DDC_BUS_FREQ_H, (ddc_bus_freq >> 8) & 0xFF);
+
+	/* Clear the EDID interrupt flag and mute the interrupt. */
+	hdmi_modb(hdmi, HDMI_INTR_MASK1, HDMI_INTR_EDID_MASK, 0);
+	hdmi_writeb(hdmi, HDMI_INTR_STATUS1, HDMI_INTR_EDID_MASK);
+}
+
+static inline u8 rk3066_hdmi_get_power_mode(struct rk3066_hdmi *hdmi)
+{
+	return hdmi_readb(hdmi, HDMI_SYS_CTRL) & HDMI_SYS_POWER_MODE_MASK;
+}
+
+static void rk3066_hdmi_set_power_mode(struct rk3066_hdmi *hdmi, int mode)
+{
+	u8 current_mode, next_mode;
+	u8 i = 0;
+
+	current_mode = rk3066_hdmi_get_power_mode(hdmi);
+
+	DRM_DEV_DEBUG(hdmi->dev, "mode         :%d\n", mode);
+	DRM_DEV_DEBUG(hdmi->dev, "current_mode :%d\n", current_mode);
+
+	if (current_mode == mode)
+		return;
+
+	do {
+		if (current_mode > mode) {
+			next_mode = current_mode / 2;
+		} else {
+			if (current_mode < HDMI_SYS_POWER_MODE_A)
+				next_mode = HDMI_SYS_POWER_MODE_A;
+			else
+				next_mode = current_mode * 2;
+		}
+
+		DRM_DEV_DEBUG(hdmi->dev, "%d: next_mode :%d\n", i, next_mode);
+
+		if (next_mode != HDMI_SYS_POWER_MODE_D) {
+			hdmi_modb(hdmi, HDMI_SYS_CTRL,
+				  HDMI_SYS_POWER_MODE_MASK, next_mode);
+		} else {
+			hdmi_writeb(hdmi, HDMI_SYS_CTRL,
+				    HDMI_SYS_POWER_MODE_D |
+				    HDMI_SYS_PLL_RESET_MASK);
+			usleep_range(90, 100);
+			hdmi_writeb(hdmi, HDMI_SYS_CTRL,
+				    HDMI_SYS_POWER_MODE_D |
+				    HDMI_SYS_PLLB_RESET);
+			usleep_range(90, 100);
+			hdmi_writeb(hdmi, HDMI_SYS_CTRL,
+				    HDMI_SYS_POWER_MODE_D);
+		}
+		current_mode = next_mode;
+		i = i + 1;
+	} while ((next_mode != mode) && (i < 5));
+
+	/*
+	 * When the IP controller isn't configured with accurate video timing,
+	 * DDC_CLK should be equal to the PLLA frequency, which is 30MHz,
+	 * so we need to init the TMDS rate to the PCLK rate and reconfigure
+	 * the DDC clock.
+	 */
+	if (mode < HDMI_SYS_POWER_MODE_D)
+		hdmi->tmdsclk = DEFAULT_PLLA_RATE;
+}
+
+static int
+rk3066_hdmi_upload_frame(struct rk3066_hdmi *hdmi, int setup_rc,
+			 union hdmi_infoframe *frame, u32 frame_index,
+			 u32 mask, u32 disable, u32 enable)
+{
+	if (mask)
+		hdmi_modb(hdmi, HDMI_CP_AUTO_SEND_CTRL, mask, disable);
+
+	hdmi_writeb(hdmi, HDMI_CP_BUF_INDEX, frame_index);
+
+	if (setup_rc >= 0) {
+		u8 packed_frame[HDMI_MAXIMUM_INFO_FRAME_SIZE];
+		ssize_t rc, i;
+
+		rc = hdmi_infoframe_pack(frame, packed_frame,
+					 sizeof(packed_frame));
+		if (rc < 0)
+			return rc;
+
+		for (i = 0; i < rc; i++)
+			hdmi_writeb(hdmi, HDMI_CP_BUF_ACC_HB0 + i * 4,
+				    packed_frame[i]);
+
+		if (mask)
+			hdmi_modb(hdmi, HDMI_CP_AUTO_SEND_CTRL, mask, enable);
+	}
+
+	return setup_rc;
+}
+
+static int rk3066_hdmi_config_avi(struct rk3066_hdmi *hdmi,
+				  struct drm_display_mode *mode)
+{
+	union hdmi_infoframe frame;
+	int rc;
+
+	rc = drm_hdmi_avi_infoframe_from_display_mode(&frame.avi,
+						      &hdmi->connector, mode);
+
+	if (hdmi->hdmi_data.enc_out_format == HDMI_COLORSPACE_YUV444)
+		frame.avi.colorspace = HDMI_COLORSPACE_YUV444;
+	else if (hdmi->hdmi_data.enc_out_format == HDMI_COLORSPACE_YUV422)
+		frame.avi.colorspace = HDMI_COLORSPACE_YUV422;
+	else
+		frame.avi.colorspace = HDMI_COLORSPACE_RGB;
+
+	frame.avi.colorimetry = hdmi->hdmi_data.colorimetry;
+	frame.avi.scan_mode = HDMI_SCAN_MODE_NONE;
+
+	return rk3066_hdmi_upload_frame(hdmi, rc, &frame,
+					HDMI_INFOFRAME_AVI, 0, 0, 0);
+}
+
+static int rk3066_hdmi_config_video_timing(struct rk3066_hdmi *hdmi,
+					   struct drm_display_mode *mode)
+{
+	int value, vsync_offset;
+
+	/* Set the details for the external polarity and interlace mode. */
+	value = HDMI_EXT_VIDEO_SET_EN;
+	value |= mode->flags & DRM_MODE_FLAG_PHSYNC ?
+		 HDMI_VIDEO_HSYNC_ACTIVE_HIGH : HDMI_VIDEO_HSYNC_ACTIVE_LOW;
+	value |= mode->flags & DRM_MODE_FLAG_PVSYNC ?
+		 HDMI_VIDEO_VSYNC_ACTIVE_HIGH : HDMI_VIDEO_VSYNC_ACTIVE_LOW;
+	value |= mode->flags & DRM_MODE_FLAG_INTERLACE ?
+		 HDMI_VIDEO_MODE_INTERLACE : HDMI_VIDEO_MODE_PROGRESSIVE;
+
+	if (hdmi->hdmi_data.vic == 2 || hdmi->hdmi_data.vic == 3)
+		vsync_offset = 6;
+	else
+		vsync_offset = 0;
+
+	value |= vsync_offset << HDMI_VIDEO_VSYNC_OFFSET_SHIFT;
+	hdmi_writeb(hdmi, HDMI_EXT_VIDEO_PARA, value);
+
+	/* Set the details for the external video timing. */
+	value = mode->htotal;
+	hdmi_writeb(hdmi, HDMI_EXT_HTOTAL_L, value & 0xFF);
+	hdmi_writeb(hdmi, HDMI_EXT_HTOTAL_H, (value >> 8) & 0xFF);
+
+	value = mode->htotal - mode->hdisplay;
+	hdmi_writeb(hdmi, HDMI_EXT_HBLANK_L, value & 0xFF);
+	hdmi_writeb(hdmi, HDMI_EXT_HBLANK_H, (value >> 8) & 0xFF);
+
+	value = mode->htotal - mode->hsync_start;
+	hdmi_writeb(hdmi, HDMI_EXT_HDELAY_L, value & 0xFF);
+	hdmi_writeb(hdmi, HDMI_EXT_HDELAY_H, (value >> 8) & 0xFF);
+
+	value = mode->hsync_end - mode->hsync_start;
+	hdmi_writeb(hdmi, HDMI_EXT_HDURATION_L, value & 0xFF);
+	hdmi_writeb(hdmi, HDMI_EXT_HDURATION_H, (value >> 8) & 0xFF);
+
+	value = mode->vtotal;
+	hdmi_writeb(hdmi, HDMI_EXT_VTOTAL_L, value & 0xFF);
+	hdmi_writeb(hdmi, HDMI_EXT_VTOTAL_H, (value >> 8) & 0xFF);
+
+	value = mode->vtotal - mode->vdisplay;
+	hdmi_writeb(hdmi, HDMI_EXT_VBLANK_L, value & 0xFF);
+
+	value = mode->vtotal - mode->vsync_start + vsync_offset;
+	hdmi_writeb(hdmi, HDMI_EXT_VDELAY, value & 0xFF);
+
+	value = mode->vsync_end - mode->vsync_start;
+	hdmi_writeb(hdmi, HDMI_EXT_VDURATION, value & 0xFF);
+
+	return 0;
+}
+
+static void
+rk3066_hdmi_phy_write(struct rk3066_hdmi *hdmi, u16 offset, u8 value)
+{
+	hdmi_writeb(hdmi, offset, value);
+	hdmi_modb(hdmi, HDMI_SYS_CTRL,
+		  HDMI_SYS_PLL_RESET_MASK, HDMI_SYS_PLL_RESET);
+	usleep_range(90, 100);
+	hdmi_modb(hdmi, HDMI_SYS_CTRL, HDMI_SYS_PLL_RESET_MASK, 0);
+	usleep_range(900, 1000);
+}
+
+static void rk3066_hdmi_config_phy(struct rk3066_hdmi *hdmi)
+{
+	/* TMDS uses the same frequency as dclk. */
+	hdmi_writeb(hdmi, HDMI_DEEP_COLOR_MODE, 0x22);
+
+	/*
+	 * The semi-public documentation does not describe the hdmi registers
+	 * used by the function rk3066_hdmi_phy_write(), so we keep using
+	 * these magic values for now.
+	 */
+	if (hdmi->tmdsclk > 100000000) {
+		rk3066_hdmi_phy_write(hdmi, 0x158, 0x0E);
+		rk3066_hdmi_phy_write(hdmi, 0x15c, 0x00);
+		rk3066_hdmi_phy_write(hdmi, 0x160, 0x60);
+		rk3066_hdmi_phy_write(hdmi, 0x164, 0x00);
+		rk3066_hdmi_phy_write(hdmi, 0x168, 0xDA);
+		rk3066_hdmi_phy_write(hdmi, 0x16c, 0xA1);
+		rk3066_hdmi_phy_write(hdmi, 0x170, 0x0e);
+		rk3066_hdmi_phy_write(hdmi, 0x174, 0x22);
+		rk3066_hdmi_phy_write(hdmi, 0x178, 0x00);
+	} else if (hdmi->tmdsclk > 50000000) {
+		rk3066_hdmi_phy_write(hdmi, 0x158, 0x06);
+		rk3066_hdmi_phy_write(hdmi, 0x15c, 0x00);
+		rk3066_hdmi_phy_write(hdmi, 0x160, 0x60);
+		rk3066_hdmi_phy_write(hdmi, 0x164, 0x00);
+		rk3066_hdmi_phy_write(hdmi, 0x168, 0xCA);
+		rk3066_hdmi_phy_write(hdmi, 0x16c, 0xA3);
+		rk3066_hdmi_phy_write(hdmi, 0x170, 0x0e);
+		rk3066_hdmi_phy_write(hdmi, 0x174, 0x20);
+		rk3066_hdmi_phy_write(hdmi, 0x178, 0x00);
+	} else {
+		rk3066_hdmi_phy_write(hdmi, 0x158, 0x02);
+		rk3066_hdmi_phy_write(hdmi, 0x15c, 0x00);
+		rk3066_hdmi_phy_write(hdmi, 0x160, 0x60);
+		rk3066_hdmi_phy_write(hdmi, 0x164, 0x00);
+		rk3066_hdmi_phy_write(hdmi, 0x168, 0xC2);
+		rk3066_hdmi_phy_write(hdmi, 0x16c, 0xA2);
+		rk3066_hdmi_phy_write(hdmi, 0x170, 0x0e);
+		rk3066_hdmi_phy_write(hdmi, 0x174, 0x20);
+		rk3066_hdmi_phy_write(hdmi, 0x178, 0x00);
+	}
+}
+
+static int rk3066_hdmi_setup(struct rk3066_hdmi *hdmi,
+			     struct drm_display_mode *mode)
+{
+	hdmi->hdmi_data.vic = drm_match_cea_mode(mode);
+	hdmi->hdmi_data.enc_out_format = HDMI_COLORSPACE_RGB;
+
+	if (hdmi->hdmi_data.vic == 6 || hdmi->hdmi_data.vic == 7 ||
+	    hdmi->hdmi_data.vic == 21 || hdmi->hdmi_data.vic == 22 ||
+	    hdmi->hdmi_data.vic == 2 || hdmi->hdmi_data.vic == 3 ||
+	    hdmi->hdmi_data.vic == 17 || hdmi->hdmi_data.vic == 18)
+		hdmi->hdmi_data.colorimetry = HDMI_COLORIMETRY_ITU_601;
+	else
+		hdmi->hdmi_data.colorimetry = HDMI_COLORIMETRY_ITU_709;
+
+	hdmi->tmdsclk = mode->clock * 1000;
+
+	/* Mute video and audio output. */
+	hdmi_modb(hdmi, HDMI_VIDEO_CTRL2, HDMI_VIDEO_AUDIO_DISABLE_MASK,
+		  HDMI_AUDIO_DISABLE | HDMI_VIDEO_DISABLE);
+
+	/* Set power state to mode B. */
+	if (rk3066_hdmi_get_power_mode(hdmi) != HDMI_SYS_POWER_MODE_B)
+		rk3066_hdmi_set_power_mode(hdmi, HDMI_SYS_POWER_MODE_B);
+
+	/* Input video mode is RGB 24 bit. Use external data enable signal. */
+	hdmi_modb(hdmi, HDMI_AV_CTRL1,
+		  HDMI_VIDEO_DE_MASK, HDMI_VIDEO_EXTERNAL_DE);
+	hdmi_writeb(hdmi, HDMI_VIDEO_CTRL1,
+		    HDMI_VIDEO_OUTPUT_RGB444 |
+		    HDMI_VIDEO_INPUT_DATA_DEPTH_8BIT |
+		    HDMI_VIDEO_INPUT_COLOR_RGB);
+	hdmi_writeb(hdmi, HDMI_DEEP_COLOR_MODE, 0x20);
+
+	rk3066_hdmi_config_video_timing(hdmi, mode);
+
+	if (hdmi->hdmi_data.sink_is_hdmi) {
+		hdmi_modb(hdmi, HDMI_HDCP_CTRL, HDMI_VIDEO_MODE_MASK,
+			  HDMI_VIDEO_MODE_HDMI);
+		rk3066_hdmi_config_avi(hdmi, mode);
+	} else {
+		hdmi_modb(hdmi, HDMI_HDCP_CTRL, HDMI_VIDEO_MODE_MASK, 0);
+	}
+
+	rk3066_hdmi_config_phy(hdmi);
+
+	rk3066_hdmi_set_power_mode(hdmi, HDMI_SYS_POWER_MODE_E);
+
+	/*
+	 * When the IP controller is configured with accurate video
+	 * timing, the TMDS clock source should be switched to
+	 * DCLK_LCDC, so we need to init the TMDS rate to the pixel mode
+	 * clock rate and reconfigure the DDC clock.
+	 */
+	rk3066_hdmi_i2c_init(hdmi);
+
+	/* Unmute video output. */
+	hdmi_modb(hdmi, HDMI_VIDEO_CTRL2,
+		  HDMI_VIDEO_AUDIO_DISABLE_MASK, HDMI_AUDIO_DISABLE);
+	return 0;
+}
+
+static void
+rk3066_hdmi_encoder_mode_set(struct drm_encoder *encoder,
+			     struct drm_display_mode *mode,
+			     struct drm_display_mode *adj_mode)
+{
+	struct rk3066_hdmi *hdmi = to_rk3066_hdmi(encoder);
+
+	/* Store the display mode for plugin/DPMS poweron events. */
+	memcpy(&hdmi->previous_mode, adj_mode, sizeof(hdmi->previous_mode));
+}
+
+static void rk3066_hdmi_encoder_enable(struct drm_encoder *encoder)
+{
+	struct rk3066_hdmi *hdmi = to_rk3066_hdmi(encoder);
+	int mux, val;
+
+	mux = drm_of_encoder_active_endpoint_id(hdmi->dev->of_node, encoder);
+	if (mux)
+		val = (HDMI_VIDEO_SEL << 16) | HDMI_VIDEO_SEL;
+	else
+		val = HDMI_VIDEO_SEL << 16;
+
+	regmap_write(hdmi->grf_regmap, GRF_SOC_CON0, val);
+
+	DRM_DEV_DEBUG(hdmi->dev, "hdmi encoder enable select: vop%s\n",
+		      (mux) ? "1" : "0");
+
+	rk3066_hdmi_setup(hdmi, &hdmi->previous_mode);
+}
+
+static void rk3066_hdmi_encoder_disable(struct drm_encoder *encoder)
+{
+	struct rk3066_hdmi *hdmi = to_rk3066_hdmi(encoder);
+
+	DRM_DEV_DEBUG(hdmi->dev, "hdmi encoder disable\n");
+
+	if (rk3066_hdmi_get_power_mode(hdmi) == HDMI_SYS_POWER_MODE_E) {
+		hdmi_writeb(hdmi, HDMI_VIDEO_CTRL2,
+			    HDMI_VIDEO_AUDIO_DISABLE_MASK);
+		hdmi_modb(hdmi, HDMI_VIDEO_CTRL2,
+			  HDMI_AUDIO_CP_LOGIC_RESET_MASK,
+			  HDMI_AUDIO_CP_LOGIC_RESET);
+		usleep_range(500, 510);
+	}
+	rk3066_hdmi_set_power_mode(hdmi, HDMI_SYS_POWER_MODE_A);
+}
+
+static bool
+rk3066_hdmi_encoder_mode_fixup(struct drm_encoder *encoder,
+			       const struct drm_display_mode *mode,
+			       struct drm_display_mode *adj_mode)
+{
+	return true;
+}
+
+static int
+rk3066_hdmi_encoder_atomic_check(struct drm_encoder *encoder,
+				 struct drm_crtc_state *crtc_state,
+				 struct drm_connector_state *conn_state)
+{
+	struct rockchip_crtc_state *s = to_rockchip_crtc_state(crtc_state);
+
+	s->output_mode = ROCKCHIP_OUT_MODE_P888;
+	s->output_type = DRM_MODE_CONNECTOR_HDMIA;
+
+	return 0;
+}
+
+static const
+struct drm_encoder_helper_funcs rk3066_hdmi_encoder_helper_funcs = {
+	.enable       = rk3066_hdmi_encoder_enable,
+	.disable      = rk3066_hdmi_encoder_disable,
+	.mode_fixup   = rk3066_hdmi_encoder_mode_fixup,
+	.mode_set     = rk3066_hdmi_encoder_mode_set,
+	.atomic_check = rk3066_hdmi_encoder_atomic_check,
+};
+
+static const struct drm_encoder_funcs rk3066_hdmi_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
+static enum drm_connector_status
+rk3066_hdmi_connector_detect(struct drm_connector *connector, bool force)
+{
+	struct rk3066_hdmi *hdmi = to_rk3066_hdmi(connector);
+
+	return (hdmi_readb(hdmi, HDMI_HPG_MENS_STA) & HDMI_HPG_IN_STATUS_HIGH) ?
+		connector_status_connected : connector_status_disconnected;
+}
+
+static int rk3066_hdmi_connector_get_modes(struct drm_connector *connector)
+{
+	struct rk3066_hdmi *hdmi = to_rk3066_hdmi(connector);
+	struct edid *edid;
+	int ret = 0;
+
+	if (!hdmi->ddc)
+		return 0;
+
+	edid = drm_get_edid(connector, hdmi->ddc);
+	if (edid) {
+		hdmi->hdmi_data.sink_is_hdmi = drm_detect_hdmi_monitor(edid);
+		drm_connector_update_edid_property(connector, edid);
+		ret = drm_add_edid_modes(connector, edid);
+		kfree(edid);
+	}
+
+	return ret;
+}
+
+static enum drm_mode_status
+rk3066_hdmi_connector_mode_valid(struct drm_connector *connector,
+				 struct drm_display_mode *mode)
+{
+	u32 vic = drm_match_cea_mode(mode);
+
+	if (vic > 1)
+		return MODE_OK;
+	else
+		return MODE_BAD;
+}
+
+static struct drm_encoder *
+rk3066_hdmi_connector_best_encoder(struct drm_connector *connector)
+{
+	struct rk3066_hdmi *hdmi = to_rk3066_hdmi(connector);
+
+	return &hdmi->encoder;
+}
+
+static int
+rk3066_hdmi_probe_single_connector_modes(struct drm_connector *connector,
+					 uint32_t maxX, uint32_t maxY)
+{
+	if (maxX > 1920)
+		maxX = 1920;
+	if (maxY > 1080)
+		maxY = 1080;
+
+	return drm_helper_probe_single_connector_modes(connector, maxX, maxY);
+}
+
+static void rk3066_hdmi_connector_destroy(struct drm_connector *connector)
+{
+	drm_connector_unregister(connector);
+	drm_connector_cleanup(connector);
+}
+
+static const struct drm_connector_funcs rk3066_hdmi_connector_funcs = {
+	.fill_modes = rk3066_hdmi_probe_single_connector_modes,
+	.detect = rk3066_hdmi_connector_detect,
+	.destroy = rk3066_hdmi_connector_destroy,
+	.reset = drm_atomic_helper_connector_reset,
+	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+};
+
+static const
+struct drm_connector_helper_funcs rk3066_hdmi_connector_helper_funcs = {
+	.get_modes = rk3066_hdmi_connector_get_modes,
+	.mode_valid = rk3066_hdmi_connector_mode_valid,
+	.best_encoder = rk3066_hdmi_connector_best_encoder,
+};
+
+static int
+rk3066_hdmi_register(struct drm_device *drm, struct rk3066_hdmi *hdmi)
+{
+	struct drm_encoder *encoder = &hdmi->encoder;
+	struct device *dev = hdmi->dev;
+
+	encoder->possible_crtcs =
+		drm_of_find_possible_crtcs(drm, dev->of_node);
+
+	/*
+	 * If we failed to find the CRTC(s) which this encoder is
+	 * supposed to be connected to, it's because the CRTC has
+	 * not been registered yet.  Defer probing, and hope that
+	 * the required CRTC is added later.
+	 */
+	if (encoder->possible_crtcs == 0)
+		return -EPROBE_DEFER;
+
+	drm_encoder_helper_add(encoder, &rk3066_hdmi_encoder_helper_funcs);
+	drm_encoder_init(drm, encoder, &rk3066_hdmi_encoder_funcs,
+			 DRM_MODE_ENCODER_TMDS, NULL);
+
+	hdmi->connector.polled = DRM_CONNECTOR_POLL_HPD;
+
+	drm_connector_helper_add(&hdmi->connector,
+				 &rk3066_hdmi_connector_helper_funcs);
+	drm_connector_init(drm, &hdmi->connector,
+			   &rk3066_hdmi_connector_funcs,
+			   DRM_MODE_CONNECTOR_HDMIA);
+
+	drm_connector_attach_encoder(&hdmi->connector, encoder);
+
+	return 0;
+}
+
+static irqreturn_t rk3066_hdmi_hardirq(int irq, void *dev_id)
+{
+	struct rk3066_hdmi *hdmi = dev_id;
+	irqreturn_t ret = IRQ_NONE;
+	u8 interrupt;
+
+	if (rk3066_hdmi_get_power_mode(hdmi) == HDMI_SYS_POWER_MODE_A)
+		hdmi_writeb(hdmi, HDMI_SYS_CTRL, HDMI_SYS_POWER_MODE_B);
+
+	interrupt = hdmi_readb(hdmi, HDMI_INTR_STATUS1);
+	if (interrupt)
+		hdmi_writeb(hdmi, HDMI_INTR_STATUS1, interrupt);
+
+	if (interrupt & HDMI_INTR_EDID_MASK) {
+		hdmi->i2c->stat = interrupt;
+		complete(&hdmi->i2c->cmpltn);
+	}
+
+	if (interrupt & (HDMI_INTR_HOTPLUG | HDMI_INTR_MSENS))
+		ret = IRQ_WAKE_THREAD;
+
+	return ret;
+}
+
+static irqreturn_t rk3066_hdmi_irq(int irq, void *dev_id)
+{
+	struct rk3066_hdmi *hdmi = dev_id;
+
+	drm_helper_hpd_irq_event(hdmi->connector.dev);
+
+	return IRQ_HANDLED;
+}
+
+static int rk3066_hdmi_i2c_read(struct rk3066_hdmi *hdmi, struct i2c_msg *msgs)
+{
+	int length = msgs->len;
+	u8 *buf = msgs->buf;
+	int ret;
+
+	ret = wait_for_completion_timeout(&hdmi->i2c->cmpltn, HZ / 10);
+	if (!ret || hdmi->i2c->stat & HDMI_INTR_EDID_ERR)
+		return -EAGAIN;
+
+	while (length--)
+		*buf++ = hdmi_readb(hdmi, HDMI_DDC_READ_FIFO_ADDR);
+
+	return 0;
+}
+
+static int rk3066_hdmi_i2c_write(struct rk3066_hdmi *hdmi, struct i2c_msg *msgs)
+{
+	/*
+	 * The DDC module only supports read EDID message, so
+	 * we assume that each word write to this i2c adapter
+	 * should be the offset of the EDID word address.
+	 */
+	if (msgs->len != 1 ||
+	    (msgs->addr != DDC_ADDR && msgs->addr != DDC_SEGMENT_ADDR))
+		return -EINVAL;
+
+	reinit_completion(&hdmi->i2c->cmpltn);
+
+	if (msgs->addr == DDC_SEGMENT_ADDR)
+		hdmi->i2c->segment_addr = msgs->buf[0];
+	if (msgs->addr == DDC_ADDR)
+		hdmi->i2c->ddc_addr = msgs->buf[0];
+
+	/* Set edid word address 0x00/0x80. */
+	hdmi_writeb(hdmi, HDMI_EDID_WORD_ADDR, hdmi->i2c->ddc_addr);
+
+	/* Set edid segment pointer. */
+	hdmi_writeb(hdmi, HDMI_EDID_SEGMENT_POINTER, hdmi->i2c->segment_addr);
+
+	return 0;
+}
+
+static int rk3066_hdmi_i2c_xfer(struct i2c_adapter *adap,
+				struct i2c_msg *msgs, int num)
+{
+	struct rk3066_hdmi *hdmi = i2c_get_adapdata(adap);
+	struct rk3066_hdmi_i2c *i2c = hdmi->i2c;
+	int i, ret = 0;
+
+	mutex_lock(&i2c->i2c_lock);
+
+	rk3066_hdmi_i2c_init(hdmi);
+
+	/* Unmute HDMI EDID interrupt. */
+	hdmi_modb(hdmi, HDMI_INTR_MASK1,
+		  HDMI_INTR_EDID_MASK, HDMI_INTR_EDID_MASK);
+	i2c->stat = 0;
+
+	for (i = 0; i < num; i++) {
+		DRM_DEV_DEBUG(hdmi->dev,
+			      "xfer: num: %d/%d, len: %d, flags: %#x\n",
+			      i + 1, num, msgs[i].len, msgs[i].flags);
+
+		if (msgs[i].flags & I2C_M_RD)
+			ret = rk3066_hdmi_i2c_read(hdmi, &msgs[i]);
+		else
+			ret = rk3066_hdmi_i2c_write(hdmi, &msgs[i]);
+
+		if (ret < 0)
+			break;
+	}
+
+	if (!ret)
+		ret = num;
+
+	/* Mute HDMI EDID interrupt. */
+	hdmi_modb(hdmi, HDMI_INTR_MASK1, HDMI_INTR_EDID_MASK, 0);
+
+	mutex_unlock(&i2c->i2c_lock);
+
+	return ret;
+}
+
+static u32 rk3066_hdmi_i2c_func(struct i2c_adapter *adapter)
+{
+	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL;
+}
+
+static const struct i2c_algorithm rk3066_hdmi_algorithm = {
+	.master_xfer   = rk3066_hdmi_i2c_xfer,
+	.functionality = rk3066_hdmi_i2c_func,
+};
+
+static struct i2c_adapter *rk3066_hdmi_i2c_adapter(struct rk3066_hdmi *hdmi)
+{
+	struct i2c_adapter *adap;
+	struct rk3066_hdmi_i2c *i2c;
+	int ret;
+
+	i2c = devm_kzalloc(hdmi->dev, sizeof(*i2c), GFP_KERNEL);
+	if (!i2c)
+		return ERR_PTR(-ENOMEM);
+
+	mutex_init(&i2c->i2c_lock);
+	init_completion(&i2c->cmpltn);
+
+	adap = &i2c->adap;
+	adap->class = I2C_CLASS_DDC;
+	adap->owner = THIS_MODULE;
+	adap->dev.parent = hdmi->dev;
+	adap->dev.of_node = hdmi->dev->of_node;
+	adap->algo = &rk3066_hdmi_algorithm;
+	strlcpy(adap->name, "RK3066 HDMI", sizeof(adap->name));
+	i2c_set_adapdata(adap, hdmi);
+
+	ret = i2c_add_adapter(adap);
+	if (ret) {
+		DRM_DEV_ERROR(hdmi->dev, "cannot add %s I2C adapter\n",
+			      adap->name);
+		devm_kfree(hdmi->dev, i2c);
+		return ERR_PTR(ret);
+	}
+
+	hdmi->i2c = i2c;
+
+	DRM_DEV_DEBUG(hdmi->dev, "registered %s I2C bus driver\n", adap->name);
+
+	return adap;
+}
+
+static int rk3066_hdmi_bind(struct device *dev, struct device *master,
+			    void *data)
+{
+	struct platform_device *pdev = to_platform_device(dev);
+	struct drm_device *drm = data;
+	struct rk3066_hdmi *hdmi;
+	struct resource *iores;
+	int irq;
+	int ret;
+
+	hdmi = devm_kzalloc(dev, sizeof(*hdmi), GFP_KERNEL);
+	if (!hdmi)
+		return -ENOMEM;
+
+	hdmi->dev = dev;
+	hdmi->drm_dev = drm;
+
+	iores = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!iores)
+		return -ENXIO;
+
+	hdmi->regs = devm_ioremap_resource(dev, iores);
+	if (IS_ERR(hdmi->regs))
+		return PTR_ERR(hdmi->regs);
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0)
+		return irq;
+
+	hdmi->hclk = devm_clk_get(dev, "hclk");
+	if (IS_ERR(hdmi->hclk)) {
+		DRM_DEV_ERROR(dev, "unable to get HDMI hclk clock\n");
+		return PTR_ERR(hdmi->hclk);
+	}
+
+	ret = clk_prepare_enable(hdmi->hclk);
+	if (ret) {
+		DRM_DEV_ERROR(dev, "cannot enable HDMI hclk clock: %d\n", ret);
+		return ret;
+	}
+
+	hdmi->grf_regmap = syscon_regmap_lookup_by_phandle(dev->of_node,
+							   "rockchip,grf");
+	if (IS_ERR(hdmi->grf_regmap)) {
+		DRM_DEV_ERROR(dev, "unable to get rockchip,grf\n");
+		ret = PTR_ERR(hdmi->grf_regmap);
+		goto err_disable_hclk;
+	}
+
+	/* internal hclk = hdmi_hclk / 25 */
+	hdmi_writeb(hdmi, HDMI_INTERNAL_CLK_DIVIDER, 25);
+
+	hdmi->ddc = rk3066_hdmi_i2c_adapter(hdmi);
+	if (IS_ERR(hdmi->ddc)) {
+		ret = PTR_ERR(hdmi->ddc);
+		hdmi->ddc = NULL;
+		goto err_disable_hclk;
+	}
+
+	rk3066_hdmi_set_power_mode(hdmi, HDMI_SYS_POWER_MODE_B);
+	usleep_range(999, 1000);
+	hdmi_writeb(hdmi, HDMI_INTR_MASK1, HDMI_INTR_HOTPLUG);
+	hdmi_writeb(hdmi, HDMI_INTR_MASK2, 0);
+	hdmi_writeb(hdmi, HDMI_INTR_MASK3, 0);
+	hdmi_writeb(hdmi, HDMI_INTR_MASK4, 0);
+	rk3066_hdmi_set_power_mode(hdmi, HDMI_SYS_POWER_MODE_A);
+
+	ret = rk3066_hdmi_register(drm, hdmi);
+	if (ret)
+		goto err_disable_i2c;
+
+	dev_set_drvdata(dev, hdmi);
+
+	ret = devm_request_threaded_irq(dev, irq, rk3066_hdmi_hardirq,
+					rk3066_hdmi_irq, IRQF_SHARED,
+					dev_name(dev), hdmi);
+	if (ret) {
+		DRM_DEV_ERROR(dev, "failed to request hdmi irq: %d\n", ret);
+		goto err_cleanup_hdmi;
+	}
+
+	return 0;
+
+err_cleanup_hdmi:
+	hdmi->connector.funcs->destroy(&hdmi->connector);
+	hdmi->encoder.funcs->destroy(&hdmi->encoder);
+err_disable_i2c:
+	i2c_put_adapter(hdmi->ddc);
+err_disable_hclk:
+	clk_disable_unprepare(hdmi->hclk);
+
+	return ret;
+}
+
+static void rk3066_hdmi_unbind(struct device *dev, struct device *master,
+			       void *data)
+{
+	struct rk3066_hdmi *hdmi = dev_get_drvdata(dev);
+
+	hdmi->connector.funcs->destroy(&hdmi->connector);
+	hdmi->encoder.funcs->destroy(&hdmi->encoder);
+
+	i2c_put_adapter(hdmi->ddc);
+	clk_disable_unprepare(hdmi->hclk);
+}
+
+static const struct component_ops rk3066_hdmi_ops = {
+	.bind   = rk3066_hdmi_bind,
+	.unbind = rk3066_hdmi_unbind,
+};
+
+static int rk3066_hdmi_probe(struct platform_device *pdev)
+{
+	return component_add(&pdev->dev, &rk3066_hdmi_ops);
+}
+
+static int rk3066_hdmi_remove(struct platform_device *pdev)
+{
+	component_del(&pdev->dev, &rk3066_hdmi_ops);
+
+	return 0;
+}
+
+static const struct of_device_id rk3066_hdmi_dt_ids[] = {
+	{ .compatible = "rockchip,rk3066-hdmi" },
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, rk3066_hdmi_dt_ids);
+
+struct platform_driver rk3066_hdmi_driver = {
+	.probe  = rk3066_hdmi_probe,
+	.remove = rk3066_hdmi_remove,
+	.driver = {
+		.name = "rockchip-rk3066-hdmi",
+		.of_match_table = rk3066_hdmi_dt_ids,
+	},
+};
diff --git a/drivers/gpu/drm/rockchip/rk3066_hdmi.h b/drivers/gpu/drm/rockchip/rk3066_hdmi.h
new file mode 100644
index 000000000000..39a31c62a428
--- /dev/null
+++ b/drivers/gpu/drm/rockchip/rk3066_hdmi.h
@@ -0,0 +1,229 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd
+ *    Zheng Yang <zhengyang@rock-chips.com>
+ */
+
+#ifndef __RK3066_HDMI_H__
+#define __RK3066_HDMI_H__
+
+#define GRF_SOC_CON0				0x150
+#define HDMI_VIDEO_SEL				BIT(14)
+
+#define DDC_SEGMENT_ADDR			0x30
+#define HDMI_SCL_RATE				(50 * 1000)
+#define HDMI_MAXIMUM_INFO_FRAME_SIZE		0x11
+
+#define N_32K					0x1000
+#define N_441K					0x1880
+#define N_882K					0x3100
+#define N_1764K					0x6200
+#define N_48K					0x1800
+#define N_96K					0x3000
+#define N_192K					0x6000
+
+#define HDMI_SYS_CTRL				0x000
+#define HDMI_LR_SWAP_N3				0x004
+#define HDMI_N2					0x008
+#define HDMI_N1					0x00c
+#define HDMI_SPDIF_FS_CTS_INT3			0x010
+#define HDMI_CTS_INT2				0x014
+#define HDMI_CTS_INT1				0x018
+#define HDMI_CTS_EXT3				0x01c
+#define HDMI_CTS_EXT2				0x020
+#define HDMI_CTS_EXT1				0x024
+#define HDMI_AUDIO_CTRL1			0x028
+#define HDMI_AUDIO_CTRL2			0x02c
+#define HDMI_I2S_AUDIO_CTRL			0x030
+#define HDMI_I2S_SWAP				0x040
+#define HDMI_AUDIO_STA_BIT_CTRL1		0x044
+#define HDMI_AUDIO_STA_BIT_CTRL2		0x048
+#define HDMI_AUDIO_SRC_NUM_AND_LENGTH		0x050
+#define HDMI_AV_CTRL1				0x054
+#define HDMI_VIDEO_CTRL1			0x058
+#define HDMI_DEEP_COLOR_MODE			0x05c
+
+#define HDMI_EXT_VIDEO_PARA			0x0c0
+#define HDMI_EXT_HTOTAL_L			0x0c4
+#define HDMI_EXT_HTOTAL_H			0x0c8
+#define HDMI_EXT_HBLANK_L			0x0cc
+#define HDMI_EXT_HBLANK_H			0x0d0
+#define HDMI_EXT_HDELAY_L			0x0d4
+#define HDMI_EXT_HDELAY_H			0x0d8
+#define HDMI_EXT_HDURATION_L			0x0dc
+#define HDMI_EXT_HDURATION_H			0x0e0
+#define HDMI_EXT_VTOTAL_L			0x0e4
+#define HDMI_EXT_VTOTAL_H			0x0e8
+#define HDMI_AV_CTRL2				0x0ec
+#define HDMI_EXT_VBLANK_L			0x0f4
+#define HDMI_EXT_VBLANK_H			0x10c
+#define HDMI_EXT_VDELAY				0x0f8
+#define HDMI_EXT_VDURATION			0x0fc
+
+#define HDMI_CP_MANU_SEND_CTRL			0x100
+#define HDMI_CP_AUTO_SEND_CTRL			0x104
+#define HDMI_AUTO_CHECKSUM_OPT			0x108
+
+#define HDMI_VIDEO_CTRL2			0x114
+
+#define HDMI_PHY_OPTION				0x144
+
+#define HDMI_CP_BUF_INDEX			0x17c
+#define HDMI_CP_BUF_ACC_HB0			0x180
+#define HDMI_CP_BUF_ACC_HB1			0x184
+#define HDMI_CP_BUF_ACC_HB2			0x188
+#define HDMI_CP_BUF_ACC_PB0			0x18c
+
+#define HDMI_DDC_READ_FIFO_ADDR			0x200
+#define HDMI_DDC_BUS_FREQ_L			0x204
+#define HDMI_DDC_BUS_FREQ_H			0x208
+#define HDMI_DDC_BUS_CTRL			0x2dc
+#define HDMI_DDC_I2C_LEN			0x278
+#define HDMI_DDC_I2C_OFFSET			0x280
+#define HDMI_DDC_I2C_CTRL			0x284
+#define HDMI_DDC_I2C_READ_BUF0			0x288
+#define HDMI_DDC_I2C_READ_BUF1			0x28c
+#define HDMI_DDC_I2C_READ_BUF2			0x290
+#define HDMI_DDC_I2C_READ_BUF3			0x294
+#define HDMI_DDC_I2C_WRITE_BUF0			0x298
+#define HDMI_DDC_I2C_WRITE_BUF1			0x29c
+#define HDMI_DDC_I2C_WRITE_BUF2			0x2a0
+#define HDMI_DDC_I2C_WRITE_BUF3			0x2a4
+#define HDMI_DDC_I2C_WRITE_BUF4			0x2ac
+#define HDMI_DDC_I2C_WRITE_BUF5			0x2b0
+#define HDMI_DDC_I2C_WRITE_BUF6			0x2b4
+
+#define HDMI_INTR_MASK1				0x248
+#define HDMI_INTR_MASK2				0x24c
+#define HDMI_INTR_STATUS1			0x250
+#define HDMI_INTR_STATUS2			0x254
+#define HDMI_INTR_MASK3				0x258
+#define HDMI_INTR_MASK4				0x25c
+#define HDMI_INTR_STATUS3			0x260
+#define HDMI_INTR_STATUS4			0x264
+
+#define HDMI_HDCP_CTRL				0x2bc
+
+#define HDMI_EDID_SEGMENT_POINTER		0x310
+#define HDMI_EDID_WORD_ADDR			0x314
+#define HDMI_EDID_FIFO_ADDR			0x318
+
+#define HDMI_HPG_MENS_STA			0x37c
+
+#define HDMI_INTERNAL_CLK_DIVIDER		0x800
+
+enum {
+	/* HDMI_SYS_CTRL */
+	HDMI_SYS_POWER_MODE_MASK = 0xf0,
+	HDMI_SYS_POWER_MODE_A = 0x10,
+	HDMI_SYS_POWER_MODE_B = 0x20,
+	HDMI_SYS_POWER_MODE_D = 0x40,
+	HDMI_SYS_POWER_MODE_E = 0x80,
+	HDMI_SYS_PLL_RESET_MASK = 0x0c,
+	HDMI_SYS_PLL_RESET = 0x0c,
+	HDMI_SYS_PLLB_RESET = 0x08,
+
+	/* HDMI_LR_SWAP_N3 */
+	HDMI_AUDIO_LR_SWAP_MASK = 0xf0,
+	HDMI_AUDIO_LR_SWAP_SUBPACKET0 = 0x10,
+	HDMI_AUDIO_LR_SWAP_SUBPACKET1 = 0x20,
+	HDMI_AUDIO_LR_SWAP_SUBPACKET2 = 0x40,
+	HDMI_AUDIO_LR_SWAP_SUBPACKET3 = 0x80,
+	HDMI_AUDIO_N_19_16_MASK = 0x0f,
+
+	/* HDMI_AUDIO_CTRL1 */
+	HDMI_AUDIO_EXTERNAL_CTS = BIT(7),
+	HDMI_AUDIO_INPUT_IIS = 0,
+	HDMI_AUDIO_INPUT_SPDIF = 0x08,
+	HDMI_AUDIO_INPUT_MCLK_ACTIVE = 0x04,
+	HDMI_AUDIO_INPUT_MCLK_DEACTIVE = 0,
+	HDMI_AUDIO_INPUT_MCLK_RATE_128X = 0,
+	HDMI_AUDIO_INPUT_MCLK_RATE_256X = 1,
+	HDMI_AUDIO_INPUT_MCLK_RATE_384X = 2,
+	HDMI_AUDIO_INPUT_MCLK_RATE_512X = 3,
+
+	/* HDMI_I2S_AUDIO_CTRL */
+	HDMI_AUDIO_I2S_FORMAT_STANDARD = 0,
+	HDMI_AUDIO_I2S_CHANNEL_1_2 = 0x04,
+	HDMI_AUDIO_I2S_CHANNEL_3_4 = 0x0c,
+	HDMI_AUDIO_I2S_CHANNEL_5_6 = 0x1c,
+	HDMI_AUDIO_I2S_CHANNEL_7_8 = 0x3c,
+
+	/* HDMI_AV_CTRL1 */
+	HDMI_AUDIO_SAMPLE_FRE_MASK = 0xf0,
+	HDMI_AUDIO_SAMPLE_FRE_32000 = 0x30,
+	HDMI_AUDIO_SAMPLE_FRE_44100 = 0,
+	HDMI_AUDIO_SAMPLE_FRE_48000 = 0x20,
+	HDMI_AUDIO_SAMPLE_FRE_88200 = 0x80,
+	HDMI_AUDIO_SAMPLE_FRE_96000 = 0xa0,
+	HDMI_AUDIO_SAMPLE_FRE_176400 = 0xc0,
+	HDMI_AUDIO_SAMPLE_FRE_192000 = 0xe0,
+	HDMI_AUDIO_SAMPLE_FRE_768000 = 0x90,
+
+	HDMI_VIDEO_INPUT_FORMAT_MASK = 0x0e,
+	HDMI_VIDEO_INPUT_RGB_YCBCR444 = 0,
+	HDMI_VIDEO_INPUT_YCBCR422 = 0x02,
+	HDMI_VIDEO_DE_MASK = 0x1,
+	HDMI_VIDEO_INTERNAL_DE = 0,
+	HDMI_VIDEO_EXTERNAL_DE = 0x01,
+
+	/* HDMI_VIDEO_CTRL1 */
+	HDMI_VIDEO_OUTPUT_FORMAT_MASK = 0xc0,
+	HDMI_VIDEO_OUTPUT_RGB444 = 0,
+	HDMI_VIDEO_OUTPUT_YCBCR444 = 0x40,
+	HDMI_VIDEO_OUTPUT_YCBCR422 = 0x80,
+	HDMI_VIDEO_INPUT_DATA_DEPTH_MASK = 0x30,
+	HDMI_VIDEO_INPUT_DATA_DEPTH_12BIT = 0,
+	HDMI_VIDEO_INPUT_DATA_DEPTH_10BIT = 0x10,
+	HDMI_VIDEO_INPUT_DATA_DEPTH_8BIT = 0x30,
+	HDMI_VIDEO_INPUT_COLOR_MASK = 1,
+	HDMI_VIDEO_INPUT_COLOR_RGB = 0,
+	HDMI_VIDEO_INPUT_COLOR_YCBCR = 1,
+
+	/* HDMI_EXT_VIDEO_PARA */
+	HDMI_VIDEO_VSYNC_OFFSET_SHIFT = 4,
+	HDMI_VIDEO_VSYNC_ACTIVE_HIGH = BIT(3),
+	HDMI_VIDEO_VSYNC_ACTIVE_LOW = 0,
+	HDMI_VIDEO_HSYNC_ACTIVE_HIGH = BIT(2),
+	HDMI_VIDEO_HSYNC_ACTIVE_LOW = 0,
+	HDMI_VIDEO_MODE_INTERLACE = BIT(1),
+	HDMI_VIDEO_MODE_PROGRESSIVE = 0,
+	HDMI_EXT_VIDEO_SET_EN = BIT(0),
+
+	/* HDMI_CP_AUTO_SEND_CTRL */
+
+	/* HDMI_VIDEO_CTRL2 */
+	HDMI_VIDEO_AV_MUTE_MASK = 0xc0,
+	HDMI_VIDEO_CLR_AV_MUTE = BIT(7),
+	HDMI_VIDEO_SET_AV_MUTE = BIT(6),
+	HDMI_AUDIO_CP_LOGIC_RESET_MASK = BIT(2),
+	HDMI_AUDIO_CP_LOGIC_RESET = BIT(2),
+	HDMI_VIDEO_AUDIO_DISABLE_MASK = 0x3,
+	HDMI_AUDIO_DISABLE = BIT(1),
+	HDMI_VIDEO_DISABLE = BIT(0),
+
+	/* HDMI_CP_BUF_INDEX */
+	HDMI_INFOFRAME_VSI = 0x05,
+	HDMI_INFOFRAME_AVI = 0x06,
+	HDMI_INFOFRAME_AAI = 0x08,
+
+	/* HDMI_INTR_MASK1 */
+	/* HDMI_INTR_STATUS1 */
+	HDMI_INTR_HOTPLUG = BIT(7),
+	HDMI_INTR_MSENS = BIT(6),
+	HDMI_INTR_VSYNC = BIT(5),
+	HDMI_INTR_AUDIO_FIFO_FULL = BIT(4),
+	HDMI_INTR_EDID_MASK = 0x6,
+	HDMI_INTR_EDID_READY = BIT(2),
+	HDMI_INTR_EDID_ERR = BIT(1),
+
+	/* HDMI_HDCP_CTRL */
+	HDMI_VIDEO_MODE_MASK = BIT(1),
+	HDMI_VIDEO_MODE_HDMI = BIT(1),
+
+	/* HDMI_HPG_MENS_STA */
+	HDMI_HPG_IN_STATUS_HIGH = BIT(7),
+	HDMI_MSENS_IN_STATUS_HIGH = BIT(6),
+};
+
+#endif /* __RK3066_HDMI_H__ */
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_drv.c b/drivers/gpu/drm/rockchip/rockchip_drm_drv.c
index d7fa17f12769..cb938d3cd3c2 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_drv.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_drv.c
@@ -448,6 +448,14 @@ static int rockchip_drm_platform_remove(struct platform_device *pdev)
 	return 0;
 }
 
+static void rockchip_drm_platform_shutdown(struct platform_device *pdev)
+{
+	struct drm_device *drm = platform_get_drvdata(pdev);
+
+	if (drm)
+		drm_atomic_helper_shutdown(drm);
+}
+
 static const struct of_device_id rockchip_drm_dt_ids[] = {
 	{ .compatible = "rockchip,display-subsystem", },
 	{ /* sentinel */ },
@@ -457,6 +465,7 @@ MODULE_DEVICE_TABLE(of, rockchip_drm_dt_ids);
 static struct platform_driver rockchip_drm_platform_driver = {
 	.probe = rockchip_drm_platform_probe,
 	.remove = rockchip_drm_platform_remove,
+	.shutdown = rockchip_drm_platform_shutdown,
 	.driver = {
 		.name = "rockchip-drm",
 		.of_match_table = rockchip_drm_dt_ids,
@@ -486,6 +495,8 @@ static int __init rockchip_drm_init(void)
 	ADD_ROCKCHIP_SUB_DRIVER(dw_mipi_dsi_rockchip_driver,
 				CONFIG_ROCKCHIP_DW_MIPI_DSI);
 	ADD_ROCKCHIP_SUB_DRIVER(inno_hdmi_driver, CONFIG_ROCKCHIP_INNO_HDMI);
+	ADD_ROCKCHIP_SUB_DRIVER(rk3066_hdmi_driver,
+				CONFIG_ROCKCHIP_RK3066_HDMI);
 
 	ret = platform_register_drivers(rockchip_sub_drivers,
 					num_rockchip_sub_drivers);
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_drv.h b/drivers/gpu/drm/rockchip/rockchip_drm_drv.h
index ce48568ec8a0..e4bc4322bc3f 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_drv.h
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_drv.h
@@ -73,4 +73,5 @@ extern struct platform_driver inno_hdmi_driver;
 extern struct platform_driver rockchip_dp_driver;
 extern struct platform_driver rockchip_lvds_driver;
 extern struct platform_driver vop_platform_driver;
+extern struct platform_driver rk3066_hdmi_driver;
 #endif /* _ROCKCHIP_DRM_DRV_H_ */
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_fbdev.c b/drivers/gpu/drm/rockchip/rockchip_drm_fbdev.c
index 8ce68bd508be..30459de66b67 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_fbdev.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_fbdev.c
@@ -90,12 +90,10 @@ static int rockchip_drm_fbdev_create(struct drm_fb_helper *helper,
 		goto out;
 	}
 
-	fbi->par = helper;
 	fbi->fbops = &rockchip_drm_fbdev_ops;
 
 	fb = helper->fb;
-	drm_fb_helper_fill_fix(fbi, fb->pitches[0], fb->format->depth);
-	drm_fb_helper_fill_var(fbi, helper, sizes->fb_width, sizes->fb_height);
+	drm_fb_helper_fill_info(fbi, helper, sizes);
 
 	offset = fbi->var.xoffset * bytes_per_pixel;
 	offset += fbi->var.yoffset * fb->pitches[0];
@@ -110,8 +108,6 @@ static int rockchip_drm_fbdev_create(struct drm_fb_helper *helper,
 		      rk_obj->kvaddr,
 		      offset, size);
 
-	fbi->skip_vt_switch = true;
-
 	return 0;
 
 out:
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop.c b/drivers/gpu/drm/rockchip/rockchip_drm_vop.c
index 0d4ade9d4722..20a9c296d027 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_vop.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop.c
@@ -1041,6 +1041,7 @@ static void vop_crtc_atomic_enable(struct drm_crtc *crtc,
 	u16 vact_st = adjusted_mode->vtotal - adjusted_mode->vsync_start;
 	u16 vact_end = vact_st + vdisplay;
 	uint32_t pin_pol, val;
+	int dither_bpc = s->output_bpc ? s->output_bpc : 10;
 	int ret;
 
 	mutex_lock(&vop->vop_lock);
@@ -1098,11 +1099,19 @@ static void vop_crtc_atomic_enable(struct drm_crtc *crtc,
 	    !(vop_data->feature & VOP_FEATURE_OUTPUT_RGB10))
 		s->output_mode = ROCKCHIP_OUT_MODE_P888;
 
-	if (s->output_mode == ROCKCHIP_OUT_MODE_AAAA && s->output_bpc == 8)
+	if (s->output_mode == ROCKCHIP_OUT_MODE_AAAA && dither_bpc <= 8)
 		VOP_REG_SET(vop, common, pre_dither_down, 1);
 	else
 		VOP_REG_SET(vop, common, pre_dither_down, 0);
 
+	if (dither_bpc == 6) {
+		VOP_REG_SET(vop, common, dither_down_sel, DITHER_DOWN_ALLEGRO);
+		VOP_REG_SET(vop, common, dither_down_mode, RGB888_TO_RGB666);
+		VOP_REG_SET(vop, common, dither_down_en, 1);
+	} else {
+		VOP_REG_SET(vop, common, dither_down_en, 0);
+	}
+
 	VOP_REG_SET(vop, common, out_mode, s->output_mode);
 
 	VOP_REG_SET(vop, modeset, htotal_pw, (htotal << 16) | hsync_len);
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop.h b/drivers/gpu/drm/rockchip/rockchip_drm_vop.h
index 04ed401d2325..e64351dab610 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_vop.h
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop.h
@@ -71,7 +71,9 @@ struct vop_common {
 	struct vop_reg dsp_blank;
 	struct vop_reg data_blank;
 	struct vop_reg pre_dither_down;
-	struct vop_reg dither_down;
+	struct vop_reg dither_down_sel;
+	struct vop_reg dither_down_mode;
+	struct vop_reg dither_down_en;
 	struct vop_reg dither_up;
 	struct vop_reg gate_en;
 	struct vop_reg mmu_en;
@@ -287,6 +289,16 @@ enum scale_down_mode {
 	SCALE_DOWN_AVG = 0x1
 };
 
+enum dither_down_mode {
+	RGB888_TO_RGB565 = 0x0,
+	RGB888_TO_RGB666 = 0x1
+};
+
+enum dither_down_mode_sel {
+	DITHER_DOWN_ALLEGRO = 0x0,
+	DITHER_DOWN_FRC = 0x1
+};
+
 enum vop_pol {
 	HSYNC_POSITIVE = 0,
 	VSYNC_POSITIVE = 1,
diff --git a/drivers/gpu/drm/rockchip/rockchip_vop_reg.c b/drivers/gpu/drm/rockchip/rockchip_vop_reg.c
index bd76328c0fdb..e732b73033c8 100644
--- a/drivers/gpu/drm/rockchip/rockchip_vop_reg.c
+++ b/drivers/gpu/drm/rockchip/rockchip_vop_reg.c
@@ -137,6 +137,9 @@ static const struct vop_common rk3036_common = {
 	.standby = VOP_REG_SYNC(RK3036_SYS_CTRL, 0x1, 30),
 	.out_mode = VOP_REG(RK3036_DSP_CTRL0, 0xf, 0),
 	.dsp_blank = VOP_REG(RK3036_DSP_CTRL1, 0x1, 24),
+	.dither_down_sel = VOP_REG(RK3036_DSP_CTRL0, 0x1, 27),
+	.dither_down_en = VOP_REG(RK3036_DSP_CTRL0, 0x1, 11),
+	.dither_down_mode = VOP_REG(RK3036_DSP_CTRL0, 0x1, 10),
 	.cfg_done = VOP_REG_SYNC(RK3036_REG_CFG_DONE, 0x1, 0),
 };
 
@@ -200,6 +203,9 @@ static const struct vop_common px30_common = {
 	.standby = VOP_REG_SYNC(PX30_SYS_CTRL2, 0x1, 1),
 	.out_mode = VOP_REG(PX30_DSP_CTRL2, 0xf, 16),
 	.dsp_blank = VOP_REG(PX30_DSP_CTRL2, 0x1, 14),
+	.dither_down_en = VOP_REG(PX30_DSP_CTRL2, 0x1, 8),
+	.dither_down_sel = VOP_REG(PX30_DSP_CTRL2, 0x1, 7),
+	.dither_down_mode = VOP_REG(PX30_DSP_CTRL2, 0x1, 6),
 	.cfg_done = VOP_REG_SYNC(PX30_REG_CFG_DONE, 0x1, 0),
 };
 
@@ -365,6 +371,8 @@ static const struct vop_common rk3066_common = {
 	.standby = VOP_REG(RK3066_SYS_CTRL0, 0x1, 1),
 	.out_mode = VOP_REG(RK3066_DSP_CTRL0, 0xf, 0),
 	.cfg_done = VOP_REG(RK3066_REG_CFG_DONE, 0x1, 0),
+	.dither_down_en = VOP_REG(RK3066_DSP_CTRL0, 0x1, 11),
+	.dither_down_mode = VOP_REG(RK3066_DSP_CTRL0, 0x1, 10),
 	.dsp_blank = VOP_REG(RK3066_DSP_CTRL1, 0x1, 24),
 };
 
@@ -458,6 +466,9 @@ static const struct vop_common rk3188_common = {
 	.standby = VOP_REG(RK3188_SYS_CTRL, 0x1, 30),
 	.out_mode = VOP_REG(RK3188_DSP_CTRL0, 0xf, 0),
 	.cfg_done = VOP_REG(RK3188_REG_CFG_DONE, 0x1, 0),
+	.dither_down_sel = VOP_REG(RK3188_DSP_CTRL0, 0x1, 27),
+	.dither_down_en = VOP_REG(RK3188_DSP_CTRL0, 0x1, 11),
+	.dither_down_mode = VOP_REG(RK3188_DSP_CTRL0, 0x1, 10),
 	.dsp_blank = VOP_REG(RK3188_DSP_CTRL1, 0x3, 24),
 };
 
@@ -585,8 +596,10 @@ static const struct vop_common rk3288_common = {
 	.standby = VOP_REG_SYNC(RK3288_SYS_CTRL, 0x1, 22),
 	.gate_en = VOP_REG(RK3288_SYS_CTRL, 0x1, 23),
 	.mmu_en = VOP_REG(RK3288_SYS_CTRL, 0x1, 20),
+	.dither_down_sel = VOP_REG(RK3288_DSP_CTRL1, 0x1, 4),
+	.dither_down_mode = VOP_REG(RK3288_DSP_CTRL1, 0x1, 3),
+	.dither_down_en = VOP_REG(RK3288_DSP_CTRL1, 0x1, 2),
 	.pre_dither_down = VOP_REG(RK3288_DSP_CTRL1, 0x1, 1),
-	.dither_down = VOP_REG(RK3288_DSP_CTRL1, 0xf, 1),
 	.dither_up = VOP_REG(RK3288_DSP_CTRL1, 0x1, 6),
 	.data_blank = VOP_REG(RK3288_DSP_CTRL0, 0x1, 19),
 	.dsp_blank = VOP_REG(RK3288_DSP_CTRL0, 0x3, 18),
@@ -878,7 +891,10 @@ static const struct vop_misc rk3328_misc = {
 
 static const struct vop_common rk3328_common = {
 	.standby = VOP_REG_SYNC(RK3328_SYS_CTRL, 0x1, 22),
-	.dither_down = VOP_REG(RK3328_DSP_CTRL1, 0xf, 1),
+	.dither_down_sel = VOP_REG(RK3328_DSP_CTRL1, 0x1, 4),
+	.dither_down_mode = VOP_REG(RK3328_DSP_CTRL1, 0x1, 3),
+	.dither_down_en = VOP_REG(RK3328_DSP_CTRL1, 0x1, 2),
+	.pre_dither_down = VOP_REG(RK3328_DSP_CTRL1, 0x1, 1),
 	.dither_up = VOP_REG(RK3328_DSP_CTRL1, 0x1, 6),
 	.dsp_blank = VOP_REG(RK3328_DSP_CTRL0, 0x3, 18),
 	.out_mode = VOP_REG(RK3328_DSP_CTRL0, 0xf, 0),
diff --git a/drivers/gpu/drm/selftests/test-drm_mm.c b/drivers/gpu/drm/selftests/test-drm_mm.c
index fbed2c90fd51..286a0eeefcb6 100644
--- a/drivers/gpu/drm/selftests/test-drm_mm.c
+++ b/drivers/gpu/drm/selftests/test-drm_mm.c
@@ -1615,7 +1615,7 @@ static int igt_topdown(void *ignored)
 	DRM_RND_STATE(prng, random_seed);
 	const unsigned int count = 8192;
 	unsigned int size;
-	unsigned long *bitmap = NULL;
+	unsigned long *bitmap;
 	struct drm_mm mm;
 	struct drm_mm_node *nodes, *node, *next;
 	unsigned int *order, n, m, o = 0;
@@ -1631,8 +1631,7 @@ static int igt_topdown(void *ignored)
 	if (!nodes)
 		goto err;
 
-	bitmap = kcalloc(count / BITS_PER_LONG, sizeof(unsigned long),
-			 GFP_KERNEL);
+	bitmap = bitmap_zalloc(count, GFP_KERNEL);
 	if (!bitmap)
 		goto err_nodes;
 
@@ -1717,7 +1716,7 @@ out:
 	drm_mm_takedown(&mm);
 	kfree(order);
 err_bitmap:
-	kfree(bitmap);
+	bitmap_free(bitmap);
 err_nodes:
 	vfree(nodes);
 err:
@@ -1745,8 +1744,7 @@ static int igt_bottomup(void *ignored)
 	if (!nodes)
 		goto err;
 
-	bitmap = kcalloc(count / BITS_PER_LONG, sizeof(unsigned long),
-			 GFP_KERNEL);
+	bitmap = bitmap_zalloc(count, GFP_KERNEL);
 	if (!bitmap)
 		goto err_nodes;
 
@@ -1818,7 +1816,7 @@ out:
 	drm_mm_takedown(&mm);
 	kfree(order);
 err_bitmap:
-	kfree(bitmap);
+	bitmap_free(bitmap);
 err_nodes:
 	vfree(nodes);
 err:
diff --git a/drivers/gpu/drm/stm/Kconfig b/drivers/gpu/drm/stm/Kconfig
index 35367ada3bc1..d15b10de1da6 100644
--- a/drivers/gpu/drm/stm/Kconfig
+++ b/drivers/gpu/drm/stm/Kconfig
@@ -6,7 +6,7 @@ config DRM_STM
 	select DRM_KMS_CMA_HELPER
 	select DRM_PANEL_BRIDGE
 	select VIDEOMODE_HELPERS
-	select FB_PROVIDE_GET_FB_UNMAPPED_AREA
+	select FB_PROVIDE_GET_FB_UNMAPPED_AREA if FB
 
 	help
 	  Enable support for the on-chip display controller on
diff --git a/drivers/gpu/drm/stm/drv.c b/drivers/gpu/drm/stm/drv.c
index 0a7f933ab007..5834ef56fbaa 100644
--- a/drivers/gpu/drm/stm/drv.c
+++ b/drivers/gpu/drm/stm/drv.c
@@ -129,6 +129,40 @@ static void drv_unload(struct drm_device *ddev)
 	drm_mode_config_cleanup(ddev);
 }
 
+static __maybe_unused int drv_suspend(struct device *dev)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct ltdc_device *ldev = ddev->dev_private;
+	struct drm_atomic_state *state;
+
+	drm_kms_helper_poll_disable(ddev);
+	state = drm_atomic_helper_suspend(ddev);
+	if (IS_ERR(state)) {
+		drm_kms_helper_poll_enable(ddev);
+		return PTR_ERR(state);
+	}
+	ldev->suspend_state = state;
+	ltdc_suspend(ddev);
+
+	return 0;
+}
+
+static __maybe_unused int drv_resume(struct device *dev)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct ltdc_device *ldev = ddev->dev_private;
+
+	ltdc_resume(ddev);
+	drm_atomic_helper_resume(ddev, ldev->suspend_state);
+	drm_kms_helper_poll_enable(ddev);
+
+	return 0;
+}
+
+static const struct dev_pm_ops drv_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(drv_suspend, drv_resume)
+};
+
 static int stm_drm_platform_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
@@ -186,6 +220,7 @@ static struct platform_driver stm_drm_platform_driver = {
 	.driver = {
 		.name = "stm32-display",
 		.of_match_table = drv_dt_ids,
+		.pm = &drv_pm_ops,
 	},
 };
 
diff --git a/drivers/gpu/drm/stm/dw_mipi_dsi-stm.c b/drivers/gpu/drm/stm/dw_mipi_dsi-stm.c
index a672b59a2226..1bef73e8c8fe 100644
--- a/drivers/gpu/drm/stm/dw_mipi_dsi-stm.c
+++ b/drivers/gpu/drm/stm/dw_mipi_dsi-stm.c
@@ -356,12 +356,40 @@ static int dw_mipi_dsi_stm_remove(struct platform_device *pdev)
 	return 0;
 }
 
+static int __maybe_unused dw_mipi_dsi_stm_suspend(struct device *dev)
+{
+	struct dw_mipi_dsi_stm *dsi = dw_mipi_dsi_stm_plat_data.priv_data;
+
+	DRM_DEBUG_DRIVER("\n");
+
+	clk_disable_unprepare(dsi->pllref_clk);
+
+	return 0;
+}
+
+static int __maybe_unused dw_mipi_dsi_stm_resume(struct device *dev)
+{
+	struct dw_mipi_dsi_stm *dsi = dw_mipi_dsi_stm_plat_data.priv_data;
+
+	DRM_DEBUG_DRIVER("\n");
+
+	clk_prepare_enable(dsi->pllref_clk);
+
+	return 0;
+}
+
+static const struct dev_pm_ops dw_mipi_dsi_stm_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(dw_mipi_dsi_stm_suspend,
+				dw_mipi_dsi_stm_resume)
+};
+
 static struct platform_driver dw_mipi_dsi_stm_driver = {
 	.probe		= dw_mipi_dsi_stm_probe,
 	.remove		= dw_mipi_dsi_stm_remove,
 	.driver		= {
 		.of_match_table = dw_mipi_dsi_stm_dt_ids,
 		.name	= "stm32-display-dsi",
+		.pm = &dw_mipi_dsi_stm_pm_ops,
 	},
 };
 
diff --git a/drivers/gpu/drm/stm/ltdc.c b/drivers/gpu/drm/stm/ltdc.c
index b1741a9d5be2..32fd6a3b37fb 100644
--- a/drivers/gpu/drm/stm/ltdc.c
+++ b/drivers/gpu/drm/stm/ltdc.c
@@ -1062,6 +1062,30 @@ static int ltdc_get_caps(struct drm_device *ddev)
 	return 0;
 }
 
+void ltdc_suspend(struct drm_device *ddev)
+{
+	struct ltdc_device *ldev = ddev->dev_private;
+
+	DRM_DEBUG_DRIVER("\n");
+	clk_disable_unprepare(ldev->pixel_clk);
+}
+
+int ltdc_resume(struct drm_device *ddev)
+{
+	struct ltdc_device *ldev = ddev->dev_private;
+	int ret;
+
+	DRM_DEBUG_DRIVER("\n");
+
+	ret = clk_prepare_enable(ldev->pixel_clk);
+	if (ret) {
+		DRM_ERROR("failed to enable pixel clock (%d)\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
 int ltdc_load(struct drm_device *ddev)
 {
 	struct platform_device *pdev = to_platform_device(ddev->dev);
diff --git a/drivers/gpu/drm/stm/ltdc.h b/drivers/gpu/drm/stm/ltdc.h
index e46f477a8494..a1ad0ae3b006 100644
--- a/drivers/gpu/drm/stm/ltdc.h
+++ b/drivers/gpu/drm/stm/ltdc.h
@@ -36,6 +36,7 @@ struct ltdc_device {
 	u32 error_status;
 	u32 irq_status;
 	struct fps_info plane_fpsi[LTDC_MAX_LAYER];
+	struct drm_atomic_state *suspend_state;
 };
 
 bool ltdc_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
@@ -45,5 +46,7 @@ bool ltdc_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
 
 int ltdc_load(struct drm_device *ddev);
 void ltdc_unload(struct drm_device *ddev);
+void ltdc_suspend(struct drm_device *ddev);
+int ltdc_resume(struct drm_device *ddev);
 
 #endif
diff --git a/drivers/gpu/drm/sun4i/sun4i_backend.c b/drivers/gpu/drm/sun4i/sun4i_backend.c
index 4c0d51f73237..4e5922c89d7b 100644
--- a/drivers/gpu/drm/sun4i/sun4i_backend.c
+++ b/drivers/gpu/drm/sun4i/sun4i_backend.c
@@ -361,13 +361,6 @@ int sun4i_backend_update_layer_buffer(struct sun4i_backend *backend,
 	paddr = drm_fb_cma_get_gem_addr(fb, state, 0);
 	DRM_DEBUG_DRIVER("Setting buffer address to %pad\n", &paddr);
 
-	/*
-	 * backend DMA accesses DRAM directly, bypassing the system
-	 * bus. As such, the address range is different and the buffer
-	 * address needs to be corrected.
-	 */
-	paddr -= PHYS_OFFSET;
-
 	if (fb->format->is_yuv)
 		return sun4i_backend_update_yuv_buffer(backend, fb, paddr);
 
@@ -720,33 +713,22 @@ static int sun4i_backend_free_sat(struct device *dev) {
  */
 static int sun4i_backend_of_get_id(struct device_node *node)
 {
-	struct device_node *port, *ep;
-	int ret = -EINVAL;
+	struct device_node *ep, *remote;
+	struct of_endpoint of_ep;
 
-	/* input is port 0 */
-	port = of_graph_get_port_by_id(node, 0);
-	if (!port)
+	/* Input port is 0, and we want the first endpoint. */
+	ep = of_graph_get_endpoint_by_regs(node, 0, -1);
+	if (!ep)
 		return -EINVAL;
 
-	/* try finding an upstream endpoint */
-	for_each_available_child_of_node(port, ep) {
-		struct device_node *remote;
-		u32 reg;
-
-		remote = of_graph_get_remote_endpoint(ep);
-		if (!remote)
-			continue;
-
-		ret = of_property_read_u32(remote, "reg", &reg);
-		if (ret)
-			continue;
-
-		ret = reg;
-	}
-
-	of_node_put(port);
+	remote = of_graph_get_remote_endpoint(ep);
+	of_node_put(ep);
+	if (!remote)
+		return -EINVAL;
 
-	return ret;
+	of_graph_parse_endpoint(remote, &of_ep);
+	of_node_put(remote);
+	return of_ep.id;
 }
 
 /* TODO: This needs to take multiple pipelines into account */
@@ -814,6 +796,27 @@ static int sun4i_backend_bind(struct device *dev, struct device *master,
 	dev_set_drvdata(dev, backend);
 	spin_lock_init(&backend->frontend_lock);
 
+	if (of_find_property(dev->of_node, "interconnects", NULL)) {
+		/*
+		 * This assume we have the same DMA constraints for all our the
+		 * devices in our pipeline (all the backends, but also the
+		 * frontends). This sounds bad, but it has always been the case
+		 * for us, and DRM doesn't do per-device allocation either, so
+		 * we would need to fix DRM first...
+		 */
+		ret = of_dma_configure(drm->dev, dev->of_node, true);
+		if (ret)
+			return ret;
+	} else {
+		/*
+		 * If we don't have the interconnect property, most likely
+		 * because of an old DT, we need to set the DMA offset by hand
+		 * on our device since the RAM mapping is at 0 for the DMA bus,
+		 * unlike the CPU.
+		 */
+		drm->dev->dma_pfn_offset = PHYS_PFN_OFFSET;
+	}
+
 	backend->engine.node = dev->of_node;
 	backend->engine.ops = &sun4i_backend_engine_ops;
 	backend->engine.id = sun4i_backend_of_get_id(dev->of_node);
diff --git a/drivers/gpu/drm/sun4i/sun4i_hdmi.h b/drivers/gpu/drm/sun4i/sun4i_hdmi.h
index b685ee11623d..b08c4453d47c 100644
--- a/drivers/gpu/drm/sun4i/sun4i_hdmi.h
+++ b/drivers/gpu/drm/sun4i/sun4i_hdmi.h
@@ -269,6 +269,7 @@ struct sun4i_hdmi {
 	struct clk		*tmds_clk;
 
 	struct i2c_adapter	*i2c;
+	struct i2c_adapter	*ddc_i2c;
 
 	/* Regmap fields for I2C adapter */
 	struct regmap_field	*field_ddc_en;
diff --git a/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c b/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c
index d18862629301..8c122e637697 100644
--- a/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c
+++ b/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c
@@ -217,7 +217,7 @@ static int sun4i_hdmi_get_modes(struct drm_connector *connector)
 	struct edid *edid;
 	int ret;
 
-	edid = drm_get_edid(connector, hdmi->i2c);
+	edid = drm_get_edid(connector, hdmi->ddc_i2c ?: hdmi->i2c);
 	if (!edid)
 		return 0;
 
@@ -233,6 +233,28 @@ static int sun4i_hdmi_get_modes(struct drm_connector *connector)
 	return ret;
 }
 
+static struct i2c_adapter *sun4i_hdmi_get_ddc(struct device *dev)
+{
+	struct device_node *phandle, *remote;
+	struct i2c_adapter *ddc;
+
+	remote = of_graph_get_remote_node(dev->of_node, 1, -1);
+	if (!remote)
+		return ERR_PTR(-EINVAL);
+
+	phandle = of_parse_phandle(remote, "ddc-i2c-bus", 0);
+	of_node_put(remote);
+	if (!phandle)
+		return ERR_PTR(-ENODEV);
+
+	ddc = of_get_i2c_adapter_by_node(phandle);
+	of_node_put(phandle);
+	if (!ddc)
+		return ERR_PTR(-EPROBE_DEFER);
+
+	return ddc;
+}
+
 static const struct drm_connector_helper_funcs sun4i_hdmi_connector_helper_funcs = {
 	.get_modes	= sun4i_hdmi_get_modes,
 };
@@ -580,6 +602,15 @@ static int sun4i_hdmi_bind(struct device *dev, struct device *master,
 		goto err_disable_mod_clk;
 	}
 
+	hdmi->ddc_i2c = sun4i_hdmi_get_ddc(dev);
+	if (IS_ERR(hdmi->ddc_i2c)) {
+		ret = PTR_ERR(hdmi->ddc_i2c);
+		if (ret == -ENODEV)
+			hdmi->ddc_i2c = NULL;
+		else
+			goto err_del_i2c_adapter;
+	}
+
 	drm_encoder_helper_add(&hdmi->encoder,
 			       &sun4i_hdmi_helper_funcs);
 	ret = drm_encoder_init(drm,
@@ -589,14 +620,14 @@ static int sun4i_hdmi_bind(struct device *dev, struct device *master,
 			       NULL);
 	if (ret) {
 		dev_err(dev, "Couldn't initialise the HDMI encoder\n");
-		goto err_del_i2c_adapter;
+		goto err_put_ddc_i2c;
 	}
 
 	hdmi->encoder.possible_crtcs = drm_of_find_possible_crtcs(drm,
 								  dev->of_node);
 	if (!hdmi->encoder.possible_crtcs) {
 		ret = -EPROBE_DEFER;
-		goto err_del_i2c_adapter;
+		goto err_put_ddc_i2c;
 	}
 
 #ifdef CONFIG_DRM_SUN4I_HDMI_CEC
@@ -635,6 +666,8 @@ static int sun4i_hdmi_bind(struct device *dev, struct device *master,
 err_cleanup_connector:
 	cec_delete_adapter(hdmi->cec_adap);
 	drm_encoder_cleanup(&hdmi->encoder);
+err_put_ddc_i2c:
+	i2c_put_adapter(hdmi->ddc_i2c);
 err_del_i2c_adapter:
 	i2c_del_adapter(hdmi->i2c);
 err_disable_mod_clk:
@@ -655,6 +688,7 @@ static void sun4i_hdmi_unbind(struct device *dev, struct device *master,
 	drm_connector_cleanup(&hdmi->connector);
 	drm_encoder_cleanup(&hdmi->encoder);
 	i2c_del_adapter(hdmi->i2c);
+	i2c_put_adapter(hdmi->ddc_i2c);
 	clk_disable_unprepare(hdmi->mod_clk);
 	clk_disable_unprepare(hdmi->bus_clk);
 }
diff --git a/drivers/gpu/drm/sun4i/sun4i_lvds.c b/drivers/gpu/drm/sun4i/sun4i_lvds.c
index 147b97ed1a09..3a3ba99fed22 100644
--- a/drivers/gpu/drm/sun4i/sun4i_lvds.c
+++ b/drivers/gpu/drm/sun4i/sun4i_lvds.c
@@ -20,7 +20,7 @@ struct sun4i_lvds {
 	struct drm_connector	connector;
 	struct drm_encoder	encoder;
 
-	struct sun4i_tcon	*tcon;
+	struct drm_panel	*panel;
 };
 
 static inline struct sun4i_lvds *
@@ -41,9 +41,8 @@ static int sun4i_lvds_get_modes(struct drm_connector *connector)
 {
 	struct sun4i_lvds *lvds =
 		drm_connector_to_sun4i_lvds(connector);
-	struct sun4i_tcon *tcon = lvds->tcon;
 
-	return drm_panel_get_modes(tcon->panel);
+	return drm_panel_get_modes(lvds->panel);
 }
 
 static struct drm_connector_helper_funcs sun4i_lvds_con_helper_funcs = {
@@ -54,9 +53,8 @@ static void
 sun4i_lvds_connector_destroy(struct drm_connector *connector)
 {
 	struct sun4i_lvds *lvds = drm_connector_to_sun4i_lvds(connector);
-	struct sun4i_tcon *tcon = lvds->tcon;
 
-	drm_panel_detach(tcon->panel);
+	drm_panel_detach(lvds->panel);
 	drm_connector_cleanup(connector);
 }
 
@@ -71,26 +69,24 @@ static const struct drm_connector_funcs sun4i_lvds_con_funcs = {
 static void sun4i_lvds_encoder_enable(struct drm_encoder *encoder)
 {
 	struct sun4i_lvds *lvds = drm_encoder_to_sun4i_lvds(encoder);
-	struct sun4i_tcon *tcon = lvds->tcon;
 
 	DRM_DEBUG_DRIVER("Enabling LVDS output\n");
 
-	if (tcon->panel) {
-		drm_panel_prepare(tcon->panel);
-		drm_panel_enable(tcon->panel);
+	if (lvds->panel) {
+		drm_panel_prepare(lvds->panel);
+		drm_panel_enable(lvds->panel);
 	}
 }
 
 static void sun4i_lvds_encoder_disable(struct drm_encoder *encoder)
 {
 	struct sun4i_lvds *lvds = drm_encoder_to_sun4i_lvds(encoder);
-	struct sun4i_tcon *tcon = lvds->tcon;
 
 	DRM_DEBUG_DRIVER("Disabling LVDS output\n");
 
-	if (tcon->panel) {
-		drm_panel_disable(tcon->panel);
-		drm_panel_unprepare(tcon->panel);
+	if (lvds->panel) {
+		drm_panel_disable(lvds->panel);
+		drm_panel_unprepare(lvds->panel);
 	}
 }
 
@@ -113,11 +109,10 @@ int sun4i_lvds_init(struct drm_device *drm, struct sun4i_tcon *tcon)
 	lvds = devm_kzalloc(drm->dev, sizeof(*lvds), GFP_KERNEL);
 	if (!lvds)
 		return -ENOMEM;
-	lvds->tcon = tcon;
 	encoder = &lvds->encoder;
 
 	ret = drm_of_find_panel_or_bridge(tcon->dev->of_node, 1, 0,
-					  &tcon->panel, &bridge);
+					  &lvds->panel, &bridge);
 	if (ret) {
 		dev_info(drm->dev, "No panel or bridge found... LVDS output disabled\n");
 		return 0;
@@ -138,7 +133,7 @@ int sun4i_lvds_init(struct drm_device *drm, struct sun4i_tcon *tcon)
 	/* The LVDS encoder can only work with the TCON channel 0 */
 	lvds->encoder.possible_crtcs = drm_crtc_mask(&tcon->crtc->crtc);
 
-	if (tcon->panel) {
+	if (lvds->panel) {
 		drm_connector_helper_add(&lvds->connector,
 					 &sun4i_lvds_con_helper_funcs);
 		ret = drm_connector_init(drm, &lvds->connector,
@@ -152,7 +147,7 @@ int sun4i_lvds_init(struct drm_device *drm, struct sun4i_tcon *tcon)
 		drm_connector_attach_encoder(&lvds->connector,
 						  &lvds->encoder);
 
-		ret = drm_panel_attach(tcon->panel, &lvds->connector);
+		ret = drm_panel_attach(lvds->panel, &lvds->connector);
 		if (ret) {
 			dev_err(drm->dev, "Couldn't attach our panel\n");
 			goto err_cleanup_connector;
diff --git a/drivers/gpu/drm/sun4i/sun4i_rgb.c b/drivers/gpu/drm/sun4i/sun4i_rgb.c
index cae19e7bbeaa..d9e2502b49fa 100644
--- a/drivers/gpu/drm/sun4i/sun4i_rgb.c
+++ b/drivers/gpu/drm/sun4i/sun4i_rgb.c
@@ -27,6 +27,8 @@ struct sun4i_rgb {
 	struct drm_encoder	encoder;
 
 	struct sun4i_tcon	*tcon;
+	struct drm_panel	*panel;
+	struct drm_bridge	*bridge;
 };
 
 static inline struct sun4i_rgb *
@@ -47,11 +49,18 @@ static int sun4i_rgb_get_modes(struct drm_connector *connector)
 {
 	struct sun4i_rgb *rgb =
 		drm_connector_to_sun4i_rgb(connector);
-	struct sun4i_tcon *tcon = rgb->tcon;
 
-	return drm_panel_get_modes(tcon->panel);
+	return drm_panel_get_modes(rgb->panel);
 }
 
+/*
+ * VESA DMT defines a tolerance of 0.5% on the pixel clock, while the
+ * CVT spec reuses that tolerance in its examples, so it looks to be a
+ * good default tolerance for the EDID-based modes. Define it to 5 per
+ * mille to avoid floating point operations.
+ */
+#define SUN4I_RGB_DOTCLOCK_TOLERANCE_PER_MILLE	5
+
 static enum drm_mode_status sun4i_rgb_mode_valid(struct drm_encoder *crtc,
 						 const struct drm_display_mode *mode)
 {
@@ -59,8 +68,9 @@ static enum drm_mode_status sun4i_rgb_mode_valid(struct drm_encoder *crtc,
 	struct sun4i_tcon *tcon = rgb->tcon;
 	u32 hsync = mode->hsync_end - mode->hsync_start;
 	u32 vsync = mode->vsync_end - mode->vsync_start;
-	unsigned long rate = mode->clock * 1000;
-	long rounded_rate;
+	unsigned long long rate = mode->clock * 1000;
+	unsigned long long lowest, highest;
+	unsigned long long rounded_rate;
 
 	DRM_DEBUG_DRIVER("Validating modes...\n");
 
@@ -92,15 +102,39 @@ static enum drm_mode_status sun4i_rgb_mode_valid(struct drm_encoder *crtc,
 
 	DRM_DEBUG_DRIVER("Vertical parameters OK\n");
 
+	/*
+	 * TODO: We should use the struct display_timing if available
+	 * and / or trying to stretch the timings within that
+	 * tolerancy to take care of panels that we wouldn't be able
+	 * to have a exact match for.
+	 */
+	if (rgb->panel) {
+		DRM_DEBUG_DRIVER("RGB panel used, skipping clock rate checks");
+		goto out;
+	}
+
+	/*
+	 * That shouldn't ever happen unless something is really wrong, but it
+	 * doesn't harm to check.
+	 */
+	if (!rgb->bridge)
+		goto out;
+
 	tcon->dclk_min_div = 6;
 	tcon->dclk_max_div = 127;
 	rounded_rate = clk_round_rate(tcon->dclk, rate);
-	if (rounded_rate < rate)
+
+	lowest = rate * (1000 - SUN4I_RGB_DOTCLOCK_TOLERANCE_PER_MILLE);
+	do_div(lowest, 1000);
+	if (rounded_rate < lowest)
 		return MODE_CLOCK_LOW;
 
-	if (rounded_rate > rate)
+	highest = rate * (1000 + SUN4I_RGB_DOTCLOCK_TOLERANCE_PER_MILLE);
+	do_div(highest, 1000);
+	if (rounded_rate > highest)
 		return MODE_CLOCK_HIGH;
 
+out:
 	DRM_DEBUG_DRIVER("Clock rate OK\n");
 
 	return MODE_OK;
@@ -114,9 +148,8 @@ static void
 sun4i_rgb_connector_destroy(struct drm_connector *connector)
 {
 	struct sun4i_rgb *rgb = drm_connector_to_sun4i_rgb(connector);
-	struct sun4i_tcon *tcon = rgb->tcon;
 
-	drm_panel_detach(tcon->panel);
+	drm_panel_detach(rgb->panel);
 	drm_connector_cleanup(connector);
 }
 
@@ -131,26 +164,24 @@ static const struct drm_connector_funcs sun4i_rgb_con_funcs = {
 static void sun4i_rgb_encoder_enable(struct drm_encoder *encoder)
 {
 	struct sun4i_rgb *rgb = drm_encoder_to_sun4i_rgb(encoder);
-	struct sun4i_tcon *tcon = rgb->tcon;
 
 	DRM_DEBUG_DRIVER("Enabling RGB output\n");
 
-	if (tcon->panel) {
-		drm_panel_prepare(tcon->panel);
-		drm_panel_enable(tcon->panel);
+	if (rgb->panel) {
+		drm_panel_prepare(rgb->panel);
+		drm_panel_enable(rgb->panel);
 	}
 }
 
 static void sun4i_rgb_encoder_disable(struct drm_encoder *encoder)
 {
 	struct sun4i_rgb *rgb = drm_encoder_to_sun4i_rgb(encoder);
-	struct sun4i_tcon *tcon = rgb->tcon;
 
 	DRM_DEBUG_DRIVER("Disabling RGB output\n");
 
-	if (tcon->panel) {
-		drm_panel_disable(tcon->panel);
-		drm_panel_unprepare(tcon->panel);
+	if (rgb->panel) {
+		drm_panel_disable(rgb->panel);
+		drm_panel_unprepare(rgb->panel);
 	}
 }
 
@@ -172,7 +203,6 @@ static struct drm_encoder_funcs sun4i_rgb_enc_funcs = {
 int sun4i_rgb_init(struct drm_device *drm, struct sun4i_tcon *tcon)
 {
 	struct drm_encoder *encoder;
-	struct drm_bridge *bridge;
 	struct sun4i_rgb *rgb;
 	int ret;
 
@@ -183,7 +213,7 @@ int sun4i_rgb_init(struct drm_device *drm, struct sun4i_tcon *tcon)
 	encoder = &rgb->encoder;
 
 	ret = drm_of_find_panel_or_bridge(tcon->dev->of_node, 1, 0,
-					  &tcon->panel, &bridge);
+					  &rgb->panel, &rgb->bridge);
 	if (ret) {
 		dev_info(drm->dev, "No panel or bridge found... RGB output disabled\n");
 		return 0;
@@ -204,7 +234,7 @@ int sun4i_rgb_init(struct drm_device *drm, struct sun4i_tcon *tcon)
 	/* The RGB encoder can only work with the TCON channel 0 */
 	rgb->encoder.possible_crtcs = drm_crtc_mask(&tcon->crtc->crtc);
 
-	if (tcon->panel) {
+	if (rgb->panel) {
 		drm_connector_helper_add(&rgb->connector,
 					 &sun4i_rgb_con_helper_funcs);
 		ret = drm_connector_init(drm, &rgb->connector,
@@ -218,15 +248,15 @@ int sun4i_rgb_init(struct drm_device *drm, struct sun4i_tcon *tcon)
 		drm_connector_attach_encoder(&rgb->connector,
 						  &rgb->encoder);
 
-		ret = drm_panel_attach(tcon->panel, &rgb->connector);
+		ret = drm_panel_attach(rgb->panel, &rgb->connector);
 		if (ret) {
 			dev_err(drm->dev, "Couldn't attach our panel\n");
 			goto err_cleanup_connector;
 		}
 	}
 
-	if (bridge) {
-		ret = drm_bridge_attach(encoder, bridge, NULL);
+	if (rgb->bridge) {
+		ret = drm_bridge_attach(encoder, rgb->bridge, NULL);
 		if (ret) {
 			dev_err(drm->dev, "Couldn't attach our bridge\n");
 			goto err_cleanup_connector;
diff --git a/drivers/gpu/drm/sun4i/sun4i_tcon.c b/drivers/gpu/drm/sun4i/sun4i_tcon.c
index 7136fc91c603..9d8d8124b1f6 100644
--- a/drivers/gpu/drm/sun4i/sun4i_tcon.c
+++ b/drivers/gpu/drm/sun4i/sun4i_tcon.c
@@ -236,8 +236,8 @@ static struct sun4i_tcon *sun4i_get_tcon0(struct drm_device *drm)
 	return NULL;
 }
 
-void sun4i_tcon_set_mux(struct sun4i_tcon *tcon, int channel,
-			const struct drm_encoder *encoder)
+static void sun4i_tcon_set_mux(struct sun4i_tcon *tcon, int channel,
+			       const struct drm_encoder *encoder)
 {
 	int ret = -ENOTSUPP;
 
@@ -341,8 +341,8 @@ static void sun4i_tcon0_mode_set_cpu(struct sun4i_tcon *tcon,
 	u32 block_space, start_delay;
 	u32 tcon_div;
 
-	tcon->dclk_min_div = 4;
-	tcon->dclk_max_div = 127;
+	tcon->dclk_min_div = SUN6I_DSI_TCON_DIV;
+	tcon->dclk_max_div = SUN6I_DSI_TCON_DIV;
 
 	sun4i_tcon0_mode_set_common(tcon, mode);
 
@@ -561,10 +561,10 @@ static void sun4i_tcon0_mode_set_rgb(struct sun4i_tcon *tcon,
 	 * Following code is a way to avoid quirks all around TCON
 	 * and DOTCLOCK drivers.
 	 */
-	if (display_info.bus_flags & DRM_BUS_FLAG_PIXDATA_POSEDGE)
+	if (display_info.bus_flags & DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE)
 		clk_set_phase(tcon->dclk, 240);
 
-	if (display_info.bus_flags & DRM_BUS_FLAG_PIXDATA_NEGEDGE)
+	if (display_info.bus_flags & DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE)
 		clk_set_phase(tcon->dclk, 0);
 
 	regmap_update_bits(tcon->regs, SUN4I_TCON0_IO_POL_REG,
diff --git a/drivers/gpu/drm/sun4i/sun4i_tcon.h b/drivers/gpu/drm/sun4i/sun4i_tcon.h
index b5214d71610f..84cfb1952ff7 100644
--- a/drivers/gpu/drm/sun4i/sun4i_tcon.h
+++ b/drivers/gpu/drm/sun4i/sun4i_tcon.h
@@ -257,8 +257,6 @@ struct sun4i_tcon {
 	struct reset_control		*lcd_rst;
 	struct reset_control		*lvds_rst;
 
-	struct drm_panel		*panel;
-
 	/* Platform adjustments */
 	const struct sun4i_tcon_quirks	*quirks;
 
diff --git a/drivers/gpu/drm/sun4i/sun6i_mipi_dsi.c b/drivers/gpu/drm/sun4i/sun6i_mipi_dsi.c
index 318994cd1b85..6ff585055a07 100644
--- a/drivers/gpu/drm/sun4i/sun6i_mipi_dsi.c
+++ b/drivers/gpu/drm/sun4i/sun6i_mipi_dsi.c
@@ -24,7 +24,9 @@
 #include <drm/drm_panel.h>
 #include <drm/drm_probe_helper.h>
 
+#include "sun4i_crtc.h"
 #include "sun4i_drv.h"
+#include "sun4i_tcon.h"
 #include "sun6i_mipi_dsi.h"
 
 #include <video/mipi_display.h>
@@ -33,6 +35,8 @@
 #define SUN6I_DSI_CTL_EN			BIT(0)
 
 #define SUN6I_DSI_BASIC_CTL_REG		0x00c
+#define SUN6I_DSI_BASIC_CTL_TRAIL_INV(n)		(((n) & 0xf) << 4)
+#define SUN6I_DSI_BASIC_CTL_TRAIL_FILL		BIT(3)
 #define SUN6I_DSI_BASIC_CTL_HBP_DIS		BIT(2)
 #define SUN6I_DSI_BASIC_CTL_HSA_HSE_DIS		BIT(1)
 #define SUN6I_DSI_BASIC_CTL_VIDEO_BURST		BIT(0)
@@ -153,6 +157,8 @@
 
 #define SUN6I_DSI_CMD_TX_REG(n)		(0x300 + (n) * 0x04)
 
+#define SUN6I_DSI_SYNC_POINT		40
+
 enum sun6i_dsi_start_inst {
 	DSI_START_LPRX,
 	DSI_START_LPTX,
@@ -358,7 +364,54 @@ static void sun6i_dsi_inst_init(struct sun6i_dsi *dsi,
 static u16 sun6i_dsi_get_video_start_delay(struct sun6i_dsi *dsi,
 					   struct drm_display_mode *mode)
 {
-	return mode->vtotal - (mode->vsync_end - mode->vdisplay) + 1;
+	u16 start = clamp(mode->vtotal - mode->vdisplay - 10, 8, 100);
+	u16 delay = mode->vtotal - (mode->vsync_end - mode->vdisplay) + start;
+
+	if (delay > mode->vtotal)
+		delay = delay % mode->vtotal;
+
+	return max_t(u16, delay, 1);
+}
+
+static u16 sun6i_dsi_get_line_num(struct sun6i_dsi *dsi,
+				  struct drm_display_mode *mode)
+{
+	struct mipi_dsi_device *device = dsi->device;
+	unsigned int Bpp = mipi_dsi_pixel_format_to_bpp(device->format) / 8;
+
+	return mode->htotal * Bpp / device->lanes;
+}
+
+static u16 sun6i_dsi_get_drq_edge0(struct sun6i_dsi *dsi,
+				   struct drm_display_mode *mode,
+				   u16 line_num, u16 edge1)
+{
+	u16 edge0 = edge1;
+
+	edge0 += (mode->hdisplay + 40) * SUN6I_DSI_TCON_DIV / 8;
+
+	if (edge0 > line_num)
+		return edge0 - line_num;
+
+	return 1;
+}
+
+static u16 sun6i_dsi_get_drq_edge1(struct sun6i_dsi *dsi,
+				   struct drm_display_mode *mode,
+				   u16 line_num)
+{
+	struct mipi_dsi_device *device = dsi->device;
+	unsigned int Bpp = mipi_dsi_pixel_format_to_bpp(device->format) / 8;
+	unsigned int hbp = mode->htotal - mode->hsync_end;
+	u16 edge1;
+
+	edge1 = SUN6I_DSI_SYNC_POINT;
+	edge1 += (mode->hdisplay + hbp + 20) * Bpp / device->lanes;
+
+	if (edge1 > line_num)
+		return line_num;
+
+	return edge1;
 }
 
 static void sun6i_dsi_setup_burst(struct sun6i_dsi *dsi,
@@ -367,7 +420,23 @@ static void sun6i_dsi_setup_burst(struct sun6i_dsi *dsi,
 	struct mipi_dsi_device *device = dsi->device;
 	u32 val = 0;
 
-	if ((mode->hsync_end - mode->hdisplay) > 20) {
+	if (device->mode_flags & MIPI_DSI_MODE_VIDEO_BURST) {
+		u16 line_num = sun6i_dsi_get_line_num(dsi, mode);
+		u16 edge0, edge1;
+
+		edge1 = sun6i_dsi_get_drq_edge1(dsi, mode, line_num);
+		edge0 = sun6i_dsi_get_drq_edge0(dsi, mode, line_num, edge1);
+
+		regmap_write(dsi->regs, SUN6I_DSI_BURST_DRQ_REG,
+			     SUN6I_DSI_BURST_DRQ_EDGE0(edge0) |
+			     SUN6I_DSI_BURST_DRQ_EDGE1(edge1));
+
+		regmap_write(dsi->regs, SUN6I_DSI_BURST_LINE_REG,
+			     SUN6I_DSI_BURST_LINE_NUM(line_num) |
+			     SUN6I_DSI_BURST_LINE_SYNC_POINT(SUN6I_DSI_SYNC_POINT));
+
+		val = SUN6I_DSI_TCON_DRQ_ENABLE_MODE;
+	} else if ((mode->hsync_end - mode->hdisplay) > 20) {
 		/* Maaaaaagic */
 		u16 drq = (mode->hsync_end - mode->hdisplay) - 20;
 
@@ -384,8 +453,19 @@ static void sun6i_dsi_setup_burst(struct sun6i_dsi *dsi,
 static void sun6i_dsi_setup_inst_loop(struct sun6i_dsi *dsi,
 				      struct drm_display_mode *mode)
 {
+	struct mipi_dsi_device *device = dsi->device;
 	u16 delay = 50 - 1;
 
+	if (device->mode_flags & MIPI_DSI_MODE_VIDEO_BURST) {
+		delay = (mode->htotal - mode->hdisplay) * 150;
+		delay /= (mode->clock / 1000) * 8;
+		delay -= 50;
+	}
+
+	regmap_write(dsi->regs, SUN6I_DSI_INST_LOOP_SEL_REG,
+		     2 << (4 * DSI_INST_ID_LP11) |
+		     3 << (4 * DSI_INST_ID_DLY));
+
 	regmap_write(dsi->regs, SUN6I_DSI_INST_LOOP_NUM_REG(0),
 		     SUN6I_DSI_INST_LOOP_NUM_N0(50 - 1) |
 		     SUN6I_DSI_INST_LOOP_NUM_N1(delay));
@@ -451,49 +531,68 @@ static void sun6i_dsi_setup_timings(struct sun6i_dsi *dsi,
 {
 	struct mipi_dsi_device *device = dsi->device;
 	unsigned int Bpp = mipi_dsi_pixel_format_to_bpp(device->format) / 8;
-	u16 hbp, hfp, hsa, hblk, vblk;
+	u16 hbp = 0, hfp = 0, hsa = 0, hblk = 0, vblk = 0;
+	u32 basic_ctl = 0;
 	size_t bytes;
 	u8 *buffer;
 
 	/* Do all timing calculations up front to allocate buffer space */
 
-	/*
-	 * A sync period is composed of a blanking packet (4 bytes +
-	 * payload + 2 bytes) and a sync event packet (4 bytes). Its
-	 * minimal size is therefore 10 bytes
-	 */
-#define HSA_PACKET_OVERHEAD	10
-	hsa = max((unsigned int)HSA_PACKET_OVERHEAD,
-		  (mode->hsync_end - mode->hsync_start) * Bpp - HSA_PACKET_OVERHEAD);
+	if (device->mode_flags & MIPI_DSI_MODE_VIDEO_BURST) {
+		hblk = mode->hdisplay * Bpp;
+		basic_ctl = SUN6I_DSI_BASIC_CTL_VIDEO_BURST |
+			    SUN6I_DSI_BASIC_CTL_HSA_HSE_DIS |
+			    SUN6I_DSI_BASIC_CTL_HBP_DIS;
 
-	/*
-	 * The backporch is set using a blanking packet (4 bytes +
-	 * payload + 2 bytes). Its minimal size is therefore 6 bytes
-	 */
+		if (device->lanes == 4)
+			basic_ctl |= SUN6I_DSI_BASIC_CTL_TRAIL_FILL |
+				     SUN6I_DSI_BASIC_CTL_TRAIL_INV(0xc);
+	} else {
+		/*
+		 * A sync period is composed of a blanking packet (4
+		 * bytes + payload + 2 bytes) and a sync event packet
+		 * (4 bytes). Its minimal size is therefore 10 bytes
+		 */
+#define HSA_PACKET_OVERHEAD	10
+		hsa = max((unsigned int)HSA_PACKET_OVERHEAD,
+			  (mode->hsync_end - mode->hsync_start) * Bpp - HSA_PACKET_OVERHEAD);
+
+		/*
+		 * The backporch is set using a blanking packet (4
+		 * bytes + payload + 2 bytes). Its minimal size is
+		 * therefore 6 bytes
+		 */
 #define HBP_PACKET_OVERHEAD	6
-	hbp = max((unsigned int)HBP_PACKET_OVERHEAD,
-		  (mode->hsync_start - mode->hdisplay) * Bpp - HBP_PACKET_OVERHEAD);
-
-	/*
-	 * The frontporch is set using a blanking packet (4 bytes +
-	 * payload + 2 bytes). Its minimal size is therefore 6 bytes
-	 */
+		hbp = max((unsigned int)HBP_PACKET_OVERHEAD,
+			  (mode->htotal - mode->hsync_end) * Bpp - HBP_PACKET_OVERHEAD);
+
+		/*
+		 * The frontporch is set using a blanking packet (4
+		 * bytes + payload + 2 bytes). Its minimal size is
+		 * therefore 6 bytes
+		 */
 #define HFP_PACKET_OVERHEAD	6
-	hfp = max((unsigned int)HFP_PACKET_OVERHEAD,
-		  (mode->htotal - mode->hsync_end) * Bpp - HFP_PACKET_OVERHEAD);
-
-	/*
-	 * hblk seems to be the line + porches length.
-	 */
-	hblk = mode->htotal * Bpp - hsa;
-
-	/*
-	 * And I'm not entirely sure what vblk is about. The driver in
-	 * Allwinner BSP is using a rather convoluted calculation
-	 * there only for 4 lanes. However, using 0 (the !4 lanes
-	 * case) even with a 4 lanes screen seems to work...
-	 */
-	vblk = 0;
+		hfp = max((unsigned int)HFP_PACKET_OVERHEAD,
+			  (mode->hsync_start - mode->hdisplay) * Bpp - HFP_PACKET_OVERHEAD);
+
+		/*
+		 * The blanking is set using a sync event (4 bytes)
+		 * and a blanking packet (4 bytes + payload + 2
+		 * bytes). Its minimal size is therefore 10 bytes.
+		 */
+#define HBLK_PACKET_OVERHEAD	10
+		hblk = max((unsigned int)HBLK_PACKET_OVERHEAD,
+			   (mode->htotal - (mode->hsync_end - mode->hsync_start)) * Bpp -
+			   HBLK_PACKET_OVERHEAD);
+
+		/*
+		 * And I'm not entirely sure what vblk is about. The driver in
+		 * Allwinner BSP is using a rather convoluted calculation
+		 * there only for 4 lanes. However, using 0 (the !4 lanes
+		 * case) even with a 4 lanes screen seems to work...
+		 */
+		vblk = 0;
+	}
 
 	/* How many bytes do we need to send all payloads? */
 	bytes = max_t(size_t, max(max(hfp, hblk), max(hsa, hbp)), vblk);
@@ -501,7 +600,7 @@ static void sun6i_dsi_setup_timings(struct sun6i_dsi *dsi,
 	if (WARN_ON(!buffer))
 		return;
 
-	regmap_write(dsi->regs, SUN6I_DSI_BASIC_CTL_REG, 0);
+	regmap_write(dsi->regs, SUN6I_DSI_BASIC_CTL_REG, basic_ctl);
 
 	regmap_write(dsi->regs, SUN6I_DSI_SYNC_HSS_REG,
 		     sun6i_dsi_build_sync_pkt(MIPI_DSI_H_SYNC_START,
@@ -526,8 +625,8 @@ static void sun6i_dsi_setup_timings(struct sun6i_dsi *dsi,
 	regmap_write(dsi->regs, SUN6I_DSI_BASIC_SIZE0_REG,
 		     SUN6I_DSI_BASIC_SIZE0_VSA(mode->vsync_end -
 					       mode->vsync_start) |
-		     SUN6I_DSI_BASIC_SIZE0_VBP(mode->vsync_start -
-					       mode->vdisplay));
+		     SUN6I_DSI_BASIC_SIZE0_VBP(mode->vtotal -
+					       mode->vsync_end));
 
 	regmap_write(dsi->regs, SUN6I_DSI_BASIC_SIZE1_REG,
 		     SUN6I_DSI_BASIC_SIZE1_VACT(mode->vdisplay) |
diff --git a/drivers/gpu/drm/sun4i/sun6i_mipi_dsi.h b/drivers/gpu/drm/sun4i/sun6i_mipi_dsi.h
index a07090579f84..5c3ad5be0690 100644
--- a/drivers/gpu/drm/sun4i/sun6i_mipi_dsi.h
+++ b/drivers/gpu/drm/sun4i/sun6i_mipi_dsi.h
@@ -13,6 +13,8 @@
 #include <drm/drm_encoder.h>
 #include <drm/drm_mipi_dsi.h>
 
+#define SUN6I_DSI_TCON_DIV	4
+
 struct sun6i_dsi {
 	struct drm_connector	connector;
 	struct drm_encoder	encoder;
diff --git a/drivers/gpu/drm/sun4i/sun8i_mixer.c b/drivers/gpu/drm/sun4i/sun8i_mixer.c
index 30a2eff55687..fd20a928cf4d 100644
--- a/drivers/gpu/drm/sun4i/sun8i_mixer.c
+++ b/drivers/gpu/drm/sun4i/sun8i_mixer.c
@@ -325,38 +325,22 @@ static struct regmap_config sun8i_mixer_regmap_config = {
 
 static int sun8i_mixer_of_get_id(struct device_node *node)
 {
-	struct device_node *port, *ep;
-	int ret = -EINVAL;
+	struct device_node *ep, *remote;
+	struct of_endpoint of_ep;
 
-	/* output is port 1 */
-	port = of_graph_get_port_by_id(node, 1);
-	if (!port)
+	/* Output port is 1, and we want the first endpoint. */
+	ep = of_graph_get_endpoint_by_regs(node, 1, -1);
+	if (!ep)
 		return -EINVAL;
 
-	/* try to find downstream endpoint */
-	for_each_available_child_of_node(port, ep) {
-		struct device_node *remote;
-		u32 reg;
-
-		remote = of_graph_get_remote_endpoint(ep);
-		if (!remote)
-			continue;
-
-		ret = of_property_read_u32(remote, "reg", &reg);
-		if (!ret) {
-			of_node_put(remote);
-			of_node_put(ep);
-			of_node_put(port);
-
-			return reg;
-		}
-
-		of_node_put(remote);
-	}
-
-	of_node_put(port);
+	remote = of_graph_get_remote_endpoint(ep);
+	of_node_put(ep);
+	if (!remote)
+		return -EINVAL;
 
-	return ret;
+	of_graph_parse_endpoint(remote, &of_ep);
+	of_node_put(remote);
+	return of_ep.id;
 }
 
 static int sun8i_mixer_bind(struct device *dev, struct device *master,
@@ -554,6 +538,7 @@ static int sun8i_mixer_remove(struct platform_device *pdev)
 static const struct sun8i_mixer_cfg sun8i_a83t_mixer0_cfg = {
 	.ccsc		= 0,
 	.scaler_mask	= 0xf,
+	.scanline_yuv	= 2048,
 	.ui_num		= 3,
 	.vi_num		= 1,
 };
@@ -561,6 +546,7 @@ static const struct sun8i_mixer_cfg sun8i_a83t_mixer0_cfg = {
 static const struct sun8i_mixer_cfg sun8i_a83t_mixer1_cfg = {
 	.ccsc		= 1,
 	.scaler_mask	= 0x3,
+	.scanline_yuv	= 2048,
 	.ui_num		= 1,
 	.vi_num		= 1,
 };
@@ -569,6 +555,7 @@ static const struct sun8i_mixer_cfg sun8i_h3_mixer0_cfg = {
 	.ccsc		= 0,
 	.mod_rate	= 432000000,
 	.scaler_mask	= 0xf,
+	.scanline_yuv	= 2048,
 	.ui_num		= 3,
 	.vi_num		= 1,
 };
@@ -577,6 +564,7 @@ static const struct sun8i_mixer_cfg sun8i_r40_mixer0_cfg = {
 	.ccsc		= 0,
 	.mod_rate	= 297000000,
 	.scaler_mask	= 0xf,
+	.scanline_yuv	= 2048,
 	.ui_num		= 3,
 	.vi_num		= 1,
 };
@@ -585,6 +573,7 @@ static const struct sun8i_mixer_cfg sun8i_r40_mixer1_cfg = {
 	.ccsc		= 1,
 	.mod_rate	= 297000000,
 	.scaler_mask	= 0x3,
+	.scanline_yuv	= 2048,
 	.ui_num		= 1,
 	.vi_num		= 1,
 };
@@ -593,6 +582,7 @@ static const struct sun8i_mixer_cfg sun8i_v3s_mixer_cfg = {
 	.vi_num = 2,
 	.ui_num = 1,
 	.scaler_mask = 0x3,
+	.scanline_yuv = 2048,
 	.ccsc = 0,
 	.mod_rate = 150000000,
 };
@@ -601,6 +591,7 @@ static const struct sun8i_mixer_cfg sun50i_a64_mixer0_cfg = {
 	.ccsc		= 0,
 	.mod_rate	= 297000000,
 	.scaler_mask	= 0xf,
+	.scanline_yuv	= 4096,
 	.ui_num		= 3,
 	.vi_num		= 1,
 };
@@ -609,6 +600,7 @@ static const struct sun8i_mixer_cfg sun50i_a64_mixer1_cfg = {
 	.ccsc		= 1,
 	.mod_rate	= 297000000,
 	.scaler_mask	= 0x3,
+	.scanline_yuv	= 2048,
 	.ui_num		= 1,
 	.vi_num		= 1,
 };
@@ -618,6 +610,7 @@ static const struct sun8i_mixer_cfg sun50i_h6_mixer0_cfg = {
 	.is_de3		= true,
 	.mod_rate	= 600000000,
 	.scaler_mask	= 0xf,
+	.scanline_yuv	= 4096,
 	.ui_num		= 3,
 	.vi_num		= 1,
 };
diff --git a/drivers/gpu/drm/sun4i/sun8i_mixer.h b/drivers/gpu/drm/sun4i/sun8i_mixer.h
index 913d14ce68b0..80e084caa084 100644
--- a/drivers/gpu/drm/sun4i/sun8i_mixer.h
+++ b/drivers/gpu/drm/sun4i/sun8i_mixer.h
@@ -159,6 +159,7 @@ struct de2_fmt_info {
  * @mod_rate: module clock rate that needs to be set in order to have
  *	a functional block.
  * @is_de3: true, if this is next gen display engine 3.0, false otherwise.
+ * @scaline_yuv: size of a scanline for VI scaler for YUV formats.
  */
 struct sun8i_mixer_cfg {
 	int		vi_num;
@@ -167,6 +168,7 @@ struct sun8i_mixer_cfg {
 	int		ccsc;
 	unsigned long	mod_rate;
 	unsigned int	is_de3 : 1;
+	unsigned int	scanline_yuv;
 };
 
 struct sun8i_mixer {
diff --git a/drivers/gpu/drm/sun4i/sun8i_tcon_top.c b/drivers/gpu/drm/sun4i/sun8i_tcon_top.c
index b1e7c76e9c17..3267d0f9b9b2 100644
--- a/drivers/gpu/drm/sun4i/sun8i_tcon_top.c
+++ b/drivers/gpu/drm/sun4i/sun8i_tcon_top.c
@@ -269,12 +269,12 @@ static int sun8i_tcon_top_remove(struct platform_device *pdev)
 	return 0;
 }
 
-const struct sun8i_tcon_top_quirks sun8i_r40_tcon_top_quirks = {
+static const struct sun8i_tcon_top_quirks sun8i_r40_tcon_top_quirks = {
 	.has_tcon_tv1	= true,
 	.has_dsi	= true,
 };
 
-const struct sun8i_tcon_top_quirks sun50i_h6_tcon_top_quirks = {
+static const struct sun8i_tcon_top_quirks sun50i_h6_tcon_top_quirks = {
 	/* Nothing special */
 };
 
diff --git a/drivers/gpu/drm/sun4i/sun8i_vi_layer.c b/drivers/gpu/drm/sun4i/sun8i_vi_layer.c
index 8a0616238467..bb8e026d6405 100644
--- a/drivers/gpu/drm/sun4i/sun8i_vi_layer.c
+++ b/drivers/gpu/drm/sun4i/sun8i_vi_layer.c
@@ -80,6 +80,8 @@ static int sun8i_vi_layer_update_coord(struct sun8i_mixer *mixer, int channel,
 	u32 bld_base, ch_base;
 	u32 outsize, insize;
 	u32 hphase, vphase;
+	u32 hn = 0, hm = 0;
+	u32 vn = 0, vm = 0;
 	bool subsampled;
 
 	DRM_DEBUG_DRIVER("Updating VI channel %d overlay %d\n",
@@ -137,12 +139,41 @@ static int sun8i_vi_layer_update_coord(struct sun8i_mixer *mixer, int channel,
 	subsampled = format->hsub > 1 || format->vsub > 1;
 
 	if (insize != outsize || subsampled || hphase || vphase) {
-		u32 hscale, vscale;
+		unsigned int scanline, required;
+		struct drm_display_mode *mode;
+		u32 hscale, vscale, fps;
+		u64 ability;
 
 		DRM_DEBUG_DRIVER("HW scaling is enabled\n");
 
-		hscale = state->src_w / state->crtc_w;
-		vscale = state->src_h / state->crtc_h;
+		mode = &plane->state->crtc->state->mode;
+		fps = (mode->clock * 1000) / (mode->vtotal * mode->htotal);
+		ability = clk_get_rate(mixer->mod_clk);
+		/* BSP algorithm assumes 80% efficiency of VI scaler unit */
+		ability *= 80;
+		do_div(ability, mode->vdisplay * fps * max(src_w, dst_w));
+
+		required = src_h * 100 / dst_h;
+
+		if (ability < required) {
+			DRM_DEBUG_DRIVER("Using vertical coarse scaling\n");
+			vm = src_h;
+			vn = (u32)ability * dst_h / 100;
+			src_h = vn;
+		}
+
+		/* it seems that every RGB scaler has buffer for 2048 pixels */
+		scanline = subsampled ? mixer->cfg->scanline_yuv : 2048;
+
+		if (src_w > scanline) {
+			DRM_DEBUG_DRIVER("Using horizontal coarse scaling\n");
+			hm = src_w;
+			hn = scanline;
+			src_w = hn;
+		}
+
+		hscale = (src_w << 16) / dst_w;
+		vscale = (src_h << 16) / dst_h;
 
 		sun8i_vi_scaler_setup(mixer, channel, src_w, src_h, dst_w,
 				      dst_h, hscale, vscale, hphase, vphase,
@@ -153,6 +184,23 @@ static int sun8i_vi_layer_update_coord(struct sun8i_mixer *mixer, int channel,
 		sun8i_vi_scaler_enable(mixer, channel, false);
 	}
 
+	regmap_write(mixer->engine.regs,
+		     SUN8I_MIXER_CHAN_VI_HDS_Y(ch_base),
+		     SUN8I_MIXER_CHAN_VI_DS_N(hn) |
+		     SUN8I_MIXER_CHAN_VI_DS_M(hm));
+	regmap_write(mixer->engine.regs,
+		     SUN8I_MIXER_CHAN_VI_HDS_UV(ch_base),
+		     SUN8I_MIXER_CHAN_VI_DS_N(hn) |
+		     SUN8I_MIXER_CHAN_VI_DS_M(hm));
+	regmap_write(mixer->engine.regs,
+		     SUN8I_MIXER_CHAN_VI_VDS_Y(ch_base),
+		     SUN8I_MIXER_CHAN_VI_DS_N(vn) |
+		     SUN8I_MIXER_CHAN_VI_DS_M(vm));
+	regmap_write(mixer->engine.regs,
+		     SUN8I_MIXER_CHAN_VI_VDS_UV(ch_base),
+		     SUN8I_MIXER_CHAN_VI_DS_N(vn) |
+		     SUN8I_MIXER_CHAN_VI_DS_M(vm));
+
 	/* Set base coordinates */
 	DRM_DEBUG_DRIVER("Layer destination coordinates X: %d Y: %d\n",
 			 state->dst.x1, state->dst.y1);
diff --git a/drivers/gpu/drm/sun4i/sun8i_vi_layer.h b/drivers/gpu/drm/sun4i/sun8i_vi_layer.h
index 8a5e6d01c85d..a223a4839f45 100644
--- a/drivers/gpu/drm/sun4i/sun8i_vi_layer.h
+++ b/drivers/gpu/drm/sun4i/sun8i_vi_layer.h
@@ -24,6 +24,14 @@
 		((base) + 0x30 * (layer) + 0x18 + 4 * (plane))
 #define SUN8I_MIXER_CHAN_VI_OVL_SIZE(base) \
 		((base) + 0xe8)
+#define SUN8I_MIXER_CHAN_VI_HDS_Y(base) \
+		((base) + 0xf0)
+#define SUN8I_MIXER_CHAN_VI_HDS_UV(base) \
+		((base) + 0xf4)
+#define SUN8I_MIXER_CHAN_VI_VDS_Y(base) \
+		((base) + 0xf8)
+#define SUN8I_MIXER_CHAN_VI_VDS_UV(base) \
+		((base) + 0xfc)
 
 #define SUN8I_MIXER_CHAN_VI_LAYER_ATTR_EN		BIT(0)
 /* RGB mode should be set for RGB formats and cleared for YCbCr */
@@ -33,6 +41,9 @@
 #define SUN50I_MIXER_CHAN_VI_LAYER_ATTR_ALPHA_MASK	GENMASK(31, 24)
 #define SUN50I_MIXER_CHAN_VI_LAYER_ATTR_ALPHA(x)	((x) << 24)
 
+#define SUN8I_MIXER_CHAN_VI_DS_N(x)			((x) << 16)
+#define SUN8I_MIXER_CHAN_VI_DS_M(x)			((x) << 0)
+
 struct sun8i_mixer;
 
 struct sun8i_vi_layer {
diff --git a/drivers/gpu/drm/tegra/fb.c b/drivers/gpu/drm/tegra/fb.c
index 0a4ce05e00ab..1dd83a757dba 100644
--- a/drivers/gpu/drm/tegra/fb.c
+++ b/drivers/gpu/drm/tegra/fb.c
@@ -255,11 +255,9 @@ static int tegra_fbdev_probe(struct drm_fb_helper *helper,
 	helper->fb = fb;
 	helper->fbdev = info;
 
-	info->par = helper;
 	info->fbops = &tegra_fb_ops;
 
-	drm_fb_helper_fill_fix(info, fb->pitches[0], fb->format->depth);
-	drm_fb_helper_fill_var(info, helper, fb->width, fb->height);
+	drm_fb_helper_fill_info(info, helper, sizes);
 
 	offset = info->var.xoffset * bytes_per_pixel +
 		 info->var.yoffset * fb->pitches[0];
diff --git a/drivers/gpu/drm/tegra/gem.c b/drivers/gpu/drm/tegra/gem.c
index 4f80100ff5f3..4cce11fd8836 100644
--- a/drivers/gpu/drm/tegra/gem.c
+++ b/drivers/gpu/drm/tegra/gem.c
@@ -204,7 +204,7 @@ static void tegra_bo_free(struct drm_device *drm, struct tegra_bo *bo)
 {
 	if (bo->pages) {
 		dma_unmap_sg(drm->dev, bo->sgt->sgl, bo->sgt->nents,
-			     DMA_BIDIRECTIONAL);
+			     DMA_FROM_DEVICE);
 		drm_gem_put_pages(&bo->gem, bo->pages, true, true);
 		sg_free_table(bo->sgt);
 		kfree(bo->sgt);
@@ -230,7 +230,7 @@ static int tegra_bo_get_pages(struct drm_device *drm, struct tegra_bo *bo)
 	}
 
 	err = dma_map_sg(drm->dev, bo->sgt->sgl, bo->sgt->nents,
-			 DMA_BIDIRECTIONAL);
+			 DMA_FROM_DEVICE);
 	if (err == 0) {
 		err = -EFAULT;
 		goto free_sgt;
diff --git a/drivers/gpu/drm/tegra/sor.c b/drivers/gpu/drm/tegra/sor.c
index 40057106f5f3..5be5a0817dfe 100644
--- a/drivers/gpu/drm/tegra/sor.c
+++ b/drivers/gpu/drm/tegra/sor.c
@@ -2871,6 +2871,13 @@ static int tegra_sor_init(struct host1x_client *client)
 	 * kernel is possible.
 	 */
 	if (sor->rst) {
+		err = reset_control_acquire(sor->rst);
+		if (err < 0) {
+			dev_err(sor->dev, "failed to acquire SOR reset: %d\n",
+				err);
+			return err;
+		}
+
 		err = reset_control_assert(sor->rst);
 		if (err < 0) {
 			dev_err(sor->dev, "failed to assert SOR reset: %d\n",
@@ -2894,6 +2901,8 @@ static int tegra_sor_init(struct host1x_client *client)
 				err);
 			return err;
 		}
+
+		reset_control_release(sor->rst);
 	}
 
 	err = clk_prepare_enable(sor->clk_safe);
@@ -3331,7 +3340,7 @@ static int tegra_sor_probe(struct platform_device *pdev)
 		goto remove;
 	}
 
-	sor->rst = devm_reset_control_get(&pdev->dev, "sor");
+	sor->rst = devm_reset_control_get_exclusive_released(&pdev->dev, "sor");
 	if (IS_ERR(sor->rst)) {
 		err = PTR_ERR(sor->rst);
 
@@ -3519,6 +3528,8 @@ static int tegra_sor_suspend(struct device *dev)
 			dev_err(dev, "failed to assert reset: %d\n", err);
 			return err;
 		}
+
+		reset_control_release(sor->rst);
 	}
 
 	usleep_range(1000, 2000);
@@ -3542,9 +3553,17 @@ static int tegra_sor_resume(struct device *dev)
 	usleep_range(1000, 2000);
 
 	if (sor->rst) {
+		err = reset_control_acquire(sor->rst);
+		if (err < 0) {
+			dev_err(dev, "failed to acquire reset: %d\n", err);
+			clk_disable_unprepare(sor->clk);
+			return err;
+		}
+
 		err = reset_control_deassert(sor->rst);
 		if (err < 0) {
 			dev_err(dev, "failed to deassert reset: %d\n", err);
+			reset_control_release(sor->rst);
 			clk_disable_unprepare(sor->clk);
 			return err;
 		}
diff --git a/drivers/gpu/drm/tinydrm/core/Makefile b/drivers/gpu/drm/tinydrm/core/Makefile
index fb221e6f8885..6f8f764560e0 100644
--- a/drivers/gpu/drm/tinydrm/core/Makefile
+++ b/drivers/gpu/drm/tinydrm/core/Makefile
@@ -1,3 +1,3 @@
-tinydrm-y := tinydrm-core.o tinydrm-pipe.o tinydrm-helpers.o
+tinydrm-y := tinydrm-pipe.o tinydrm-helpers.o
 
 obj-$(CONFIG_DRM_TINYDRM) += tinydrm.o
diff --git a/drivers/gpu/drm/tinydrm/core/tinydrm-core.c b/drivers/gpu/drm/tinydrm/core/tinydrm-core.c
deleted file mode 100644
index 554abd5d3b53..000000000000
--- a/drivers/gpu/drm/tinydrm/core/tinydrm-core.c
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
- * Copyright (C) 2016 Noralf Trønnes
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
-
-#include <drm/drm_atomic.h>
-#include <drm/drm_atomic_helper.h>
-#include <drm/drm_drv.h>
-#include <drm/drm_fb_helper.h>
-#include <drm/drm_gem_framebuffer_helper.h>
-#include <drm/drm_probe_helper.h>
-#include <drm/drm_print.h>
-#include <drm/tinydrm/tinydrm.h>
-#include <linux/device.h>
-#include <linux/dma-buf.h>
-#include <linux/module.h>
-
-/**
- * DOC: overview
- *
- * This library provides driver helpers for very simple display hardware.
- *
- * It is based on &drm_simple_display_pipe coupled with a &drm_connector which
- * has only one fixed &drm_display_mode. The framebuffers are backed by the
- * cma helper and have support for framebuffer flushing (dirty).
- * fbdev support is also included.
- *
- */
-
-/**
- * DOC: core
- *
- * The driver allocates &tinydrm_device, initializes it using
- * devm_tinydrm_init(), sets up the pipeline using tinydrm_display_pipe_init()
- * and registers the DRM device using devm_tinydrm_register().
- */
-
-static const struct drm_mode_config_funcs tinydrm_mode_config_funcs = {
-	.fb_create = drm_gem_fb_create_with_dirty,
-	.atomic_check = drm_atomic_helper_check,
-	.atomic_commit = drm_atomic_helper_commit,
-};
-
-static int tinydrm_init(struct device *parent, struct tinydrm_device *tdev,
-			struct drm_driver *driver)
-{
-	struct drm_device *drm;
-
-	/*
-	 * We don't embed drm_device, because that prevent us from using
-	 * devm_kzalloc() to allocate tinydrm_device in the driver since
-	 * drm_dev_put() frees the structure. The devm_ functions provide
-	 * for easy error handling.
-	 */
-	drm = drm_dev_alloc(driver, parent);
-	if (IS_ERR(drm))
-		return PTR_ERR(drm);
-
-	tdev->drm = drm;
-	drm->dev_private = tdev;
-	drm_mode_config_init(drm);
-	drm->mode_config.funcs = &tinydrm_mode_config_funcs;
-	drm->mode_config.allow_fb_modifiers = true;
-
-	return 0;
-}
-
-static void tinydrm_fini(struct tinydrm_device *tdev)
-{
-	drm_mode_config_cleanup(tdev->drm);
-	tdev->drm->dev_private = NULL;
-	drm_dev_put(tdev->drm);
-}
-
-static void devm_tinydrm_release(void *data)
-{
-	tinydrm_fini(data);
-}
-
-/**
- * devm_tinydrm_init - Initialize tinydrm device
- * @parent: Parent device object
- * @tdev: tinydrm device
- * @driver: DRM driver
- *
- * This function initializes @tdev, the underlying DRM device and it's
- * mode_config. Resources will be automatically freed on driver detach (devres)
- * using drm_mode_config_cleanup() and drm_dev_put().
- *
- * Returns:
- * Zero on success, negative error code on failure.
- */
-int devm_tinydrm_init(struct device *parent, struct tinydrm_device *tdev,
-		      struct drm_driver *driver)
-{
-	int ret;
-
-	ret = tinydrm_init(parent, tdev, driver);
-	if (ret)
-		return ret;
-
-	ret = devm_add_action(parent, devm_tinydrm_release, tdev);
-	if (ret)
-		tinydrm_fini(tdev);
-
-	return ret;
-}
-EXPORT_SYMBOL(devm_tinydrm_init);
-
-static int tinydrm_register(struct tinydrm_device *tdev)
-{
-	struct drm_device *drm = tdev->drm;
-	int ret;
-
-	ret = drm_dev_register(tdev->drm, 0);
-	if (ret)
-		return ret;
-
-	ret = drm_fbdev_generic_setup(drm, 0);
-	if (ret)
-		DRM_ERROR("Failed to initialize fbdev: %d\n", ret);
-
-	return 0;
-}
-
-static void tinydrm_unregister(struct tinydrm_device *tdev)
-{
-	drm_atomic_helper_shutdown(tdev->drm);
-	drm_dev_unregister(tdev->drm);
-}
-
-static void devm_tinydrm_register_release(void *data)
-{
-	tinydrm_unregister(data);
-}
-
-/**
- * devm_tinydrm_register - Register tinydrm device
- * @tdev: tinydrm device
- *
- * This function registers the underlying DRM device and fbdev.
- * These resources will be automatically unregistered on driver detach (devres)
- * and the display pipeline will be disabled.
- *
- * Returns:
- * Zero on success, negative error code on failure.
- */
-int devm_tinydrm_register(struct tinydrm_device *tdev)
-{
-	struct device *dev = tdev->drm->dev;
-	int ret;
-
-	ret = tinydrm_register(tdev);
-	if (ret)
-		return ret;
-
-	ret = devm_add_action(dev, devm_tinydrm_register_release, tdev);
-	if (ret)
-		tinydrm_unregister(tdev);
-
-	return ret;
-}
-EXPORT_SYMBOL(devm_tinydrm_register);
-
-/**
- * tinydrm_shutdown - Shutdown tinydrm
- * @tdev: tinydrm device
- *
- * This function makes sure that the display pipeline is disabled.
- * Used by drivers in their shutdown callback to turn off the display
- * on machine shutdown and reboot.
- */
-void tinydrm_shutdown(struct tinydrm_device *tdev)
-{
-	drm_atomic_helper_shutdown(tdev->drm);
-}
-EXPORT_SYMBOL(tinydrm_shutdown);
-
-MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/tinydrm/core/tinydrm-helpers.c b/drivers/gpu/drm/tinydrm/core/tinydrm-helpers.c
index 2737b6fdadc8..6d540d93758f 100644
--- a/drivers/gpu/drm/tinydrm/core/tinydrm-helpers.c
+++ b/drivers/gpu/drm/tinydrm/core/tinydrm-helpers.c
@@ -26,164 +26,6 @@ static unsigned int spi_max;
 module_param(spi_max, uint, 0400);
 MODULE_PARM_DESC(spi_max, "Set a lower SPI max transfer size");
 
-/**
- * tinydrm_memcpy - Copy clip buffer
- * @dst: Destination buffer
- * @vaddr: Source buffer
- * @fb: DRM framebuffer
- * @clip: Clip rectangle area to copy
- */
-void tinydrm_memcpy(void *dst, void *vaddr, struct drm_framebuffer *fb,
-		    struct drm_rect *clip)
-{
-	unsigned int cpp = drm_format_plane_cpp(fb->format->format, 0);
-	unsigned int pitch = fb->pitches[0];
-	void *src = vaddr + (clip->y1 * pitch) + (clip->x1 * cpp);
-	size_t len = (clip->x2 - clip->x1) * cpp;
-	unsigned int y;
-
-	for (y = clip->y1; y < clip->y2; y++) {
-		memcpy(dst, src, len);
-		src += pitch;
-		dst += len;
-	}
-}
-EXPORT_SYMBOL(tinydrm_memcpy);
-
-/**
- * tinydrm_swab16 - Swap bytes into clip buffer
- * @dst: RGB565 destination buffer
- * @vaddr: RGB565 source buffer
- * @fb: DRM framebuffer
- * @clip: Clip rectangle area to copy
- */
-void tinydrm_swab16(u16 *dst, void *vaddr, struct drm_framebuffer *fb,
-		    struct drm_rect *clip)
-{
-	size_t len = (clip->x2 - clip->x1) * sizeof(u16);
-	unsigned int x, y;
-	u16 *src, *buf;
-
-	/*
-	 * The cma memory is write-combined so reads are uncached.
-	 * Speed up by fetching one line at a time.
-	 */
-	buf = kmalloc(len, GFP_KERNEL);
-	if (!buf)
-		return;
-
-	for (y = clip->y1; y < clip->y2; y++) {
-		src = vaddr + (y * fb->pitches[0]);
-		src += clip->x1;
-		memcpy(buf, src, len);
-		src = buf;
-		for (x = clip->x1; x < clip->x2; x++)
-			*dst++ = swab16(*src++);
-	}
-
-	kfree(buf);
-}
-EXPORT_SYMBOL(tinydrm_swab16);
-
-/**
- * tinydrm_xrgb8888_to_rgb565 - Convert XRGB8888 to RGB565 clip buffer
- * @dst: RGB565 destination buffer
- * @vaddr: XRGB8888 source buffer
- * @fb: DRM framebuffer
- * @clip: Clip rectangle area to copy
- * @swap: Swap bytes
- *
- * Drivers can use this function for RGB565 devices that don't natively
- * support XRGB8888.
- */
-void tinydrm_xrgb8888_to_rgb565(u16 *dst, void *vaddr,
-				struct drm_framebuffer *fb,
-				struct drm_rect *clip, bool swap)
-{
-	size_t len = (clip->x2 - clip->x1) * sizeof(u32);
-	unsigned int x, y;
-	u32 *src, *buf;
-	u16 val16;
-
-	buf = kmalloc(len, GFP_KERNEL);
-	if (!buf)
-		return;
-
-	for (y = clip->y1; y < clip->y2; y++) {
-		src = vaddr + (y * fb->pitches[0]);
-		src += clip->x1;
-		memcpy(buf, src, len);
-		src = buf;
-		for (x = clip->x1; x < clip->x2; x++) {
-			val16 = ((*src & 0x00F80000) >> 8) |
-				((*src & 0x0000FC00) >> 5) |
-				((*src & 0x000000F8) >> 3);
-			src++;
-			if (swap)
-				*dst++ = swab16(val16);
-			else
-				*dst++ = val16;
-		}
-	}
-
-	kfree(buf);
-}
-EXPORT_SYMBOL(tinydrm_xrgb8888_to_rgb565);
-
-/**
- * tinydrm_xrgb8888_to_gray8 - Convert XRGB8888 to grayscale
- * @dst: 8-bit grayscale destination buffer
- * @vaddr: XRGB8888 source buffer
- * @fb: DRM framebuffer
- * @clip: Clip rectangle area to copy
- *
- * Drm doesn't have native monochrome or grayscale support.
- * Such drivers can announce the commonly supported XR24 format to userspace
- * and use this function to convert to the native format.
- *
- * Monochrome drivers will use the most significant bit,
- * where 1 means foreground color and 0 background color.
- *
- * ITU BT.601 is used for the RGB -> luma (brightness) conversion.
- */
-void tinydrm_xrgb8888_to_gray8(u8 *dst, void *vaddr, struct drm_framebuffer *fb,
-			       struct drm_rect *clip)
-{
-	unsigned int len = (clip->x2 - clip->x1) * sizeof(u32);
-	unsigned int x, y;
-	void *buf;
-	u32 *src;
-
-	if (WARN_ON(fb->format->format != DRM_FORMAT_XRGB8888))
-		return;
-	/*
-	 * The cma memory is write-combined so reads are uncached.
-	 * Speed up by fetching one line at a time.
-	 */
-	buf = kmalloc(len, GFP_KERNEL);
-	if (!buf)
-		return;
-
-	for (y = clip->y1; y < clip->y2; y++) {
-		src = vaddr + (y * fb->pitches[0]);
-		src += clip->x1;
-		memcpy(buf, src, len);
-		src = buf;
-		for (x = clip->x1; x < clip->x2; x++) {
-			u8 r = (*src & 0x00ff0000) >> 16;
-			u8 g = (*src & 0x0000ff00) >> 8;
-			u8 b =  *src & 0x000000ff;
-
-			/* ITU BT.601: Y = 0.299 R + 0.587 G + 0.114 B */
-			*dst++ = (3 * r + 6 * g + b) / 10;
-			src++;
-		}
-	}
-
-	kfree(buf);
-}
-EXPORT_SYMBOL(tinydrm_xrgb8888_to_gray8);
-
 #if IS_ENABLED(CONFIG_SPI)
 
 /**
@@ -365,3 +207,5 @@ int tinydrm_spi_transfer(struct spi_device *spi, u32 speed_hz,
 EXPORT_SYMBOL(tinydrm_spi_transfer);
 
 #endif /* CONFIG_SPI */
+
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/tinydrm/core/tinydrm-pipe.c b/drivers/gpu/drm/tinydrm/core/tinydrm-pipe.c
index bb5b1c1e21ba..bb8a7ed8ddf6 100644
--- a/drivers/gpu/drm/tinydrm/core/tinydrm-pipe.c
+++ b/drivers/gpu/drm/tinydrm/core/tinydrm-pipe.c
@@ -13,7 +13,7 @@
 #include <drm/drm_modes.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_print.h>
-#include <drm/tinydrm/tinydrm.h>
+#include <drm/drm_simple_kms_helper.h>
 
 struct tinydrm_connector {
 	struct drm_connector base;
@@ -129,7 +129,8 @@ static int tinydrm_rotate_mode(struct drm_display_mode *mode,
 
 /**
  * tinydrm_display_pipe_init - Initialize display pipe
- * @tdev: tinydrm device
+ * @drm: DRM device
+ * @pipe: Display pipe
  * @funcs: Display pipe functions
  * @connector_type: Connector type
  * @formats: Array of supported formats (DRM_FORMAT\_\*)
@@ -143,16 +144,15 @@ static int tinydrm_rotate_mode(struct drm_display_mode *mode,
  * Returns:
  * Zero on success, negative error code on failure.
  */
-int
-tinydrm_display_pipe_init(struct tinydrm_device *tdev,
-			  const struct drm_simple_display_pipe_funcs *funcs,
-			  int connector_type,
-			  const uint32_t *formats,
-			  unsigned int format_count,
-			  const struct drm_display_mode *mode,
-			  unsigned int rotation)
+int tinydrm_display_pipe_init(struct drm_device *drm,
+			      struct drm_simple_display_pipe *pipe,
+			      const struct drm_simple_display_pipe_funcs *funcs,
+			      int connector_type,
+			      const uint32_t *formats,
+			      unsigned int format_count,
+			      const struct drm_display_mode *mode,
+			      unsigned int rotation)
 {
-	struct drm_device *drm = tdev->drm;
 	struct drm_display_mode mode_copy;
 	struct drm_connector *connector;
 	int ret;
@@ -177,7 +177,7 @@ tinydrm_display_pipe_init(struct tinydrm_device *tdev,
 	if (IS_ERR(connector))
 		return PTR_ERR(connector);
 
-	return drm_simple_display_pipe_init(drm, &tdev->pipe, funcs, formats,
+	return drm_simple_display_pipe_init(drm, pipe, funcs, formats,
 					    format_count, modifiers, connector);
 }
 EXPORT_SYMBOL(tinydrm_display_pipe_init);
diff --git a/drivers/gpu/drm/tinydrm/hx8357d.c b/drivers/gpu/drm/tinydrm/hx8357d.c
index 8bbd0beafc6a..5773d0fb6ca1 100644
--- a/drivers/gpu/drm/tinydrm/hx8357d.c
+++ b/drivers/gpu/drm/tinydrm/hx8357d.c
@@ -16,7 +16,9 @@
 #include <linux/property.h>
 #include <linux/spi/spi.h>
 
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_drv.h>
+#include <drm/drm_fb_helper.h>
 #include <drm/drm_gem_cma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_modeset_helper.h>
@@ -46,16 +48,18 @@ static void yx240qv29_enable(struct drm_simple_display_pipe *pipe,
 			     struct drm_crtc_state *crtc_state,
 			     struct drm_plane_state *plane_state)
 {
-	struct tinydrm_device *tdev = pipe_to_tinydrm(pipe);
-	struct mipi_dbi *mipi = mipi_dbi_from_tinydrm(tdev);
+	struct mipi_dbi *mipi = drm_to_mipi_dbi(pipe->crtc.dev);
 	u8 addr_mode;
-	int ret;
+	int ret, idx;
+
+	if (!drm_dev_enter(pipe->crtc.dev, &idx))
+		return;
 
 	DRM_DEBUG_KMS("\n");
 
 	ret = mipi_dbi_poweron_conditional_reset(mipi);
 	if (ret < 0)
-		return;
+		goto out_exit;
 	if (ret == 1)
 		goto out_enable;
 
@@ -171,6 +175,8 @@ out_enable:
 	}
 	mipi_dbi_command(mipi, MIPI_DCS_SET_ADDRESS_MODE, addr_mode);
 	mipi_dbi_enable_flush(mipi, crtc_state, plane_state);
+out_exit:
+	drm_dev_exit(idx);
 }
 
 static const struct drm_simple_display_pipe_funcs hx8357d_pipe_funcs = {
@@ -181,7 +187,7 @@ static const struct drm_simple_display_pipe_funcs hx8357d_pipe_funcs = {
 };
 
 static const struct drm_display_mode yx350hv15_mode = {
-	TINYDRM_MODE(320, 480, 60, 75),
+	DRM_SIMPLE_MODE(320, 480, 60, 75),
 };
 
 DEFINE_DRM_GEM_CMA_FOPS(hx8357d_fops);
@@ -189,6 +195,7 @@ DEFINE_DRM_GEM_CMA_FOPS(hx8357d_fops);
 static struct drm_driver hx8357d_driver = {
 	.driver_features	= DRIVER_GEM | DRIVER_MODESET | DRIVER_PRIME | DRIVER_ATOMIC,
 	.fops			= &hx8357d_fops,
+	.release		= mipi_dbi_release,
 	DRM_GEM_CMA_VMAP_DRIVER_OPS,
 	.debugfs_init		= mipi_dbi_debugfs_init,
 	.name			= "hx8357d",
@@ -213,15 +220,25 @@ MODULE_DEVICE_TABLE(spi, hx8357d_id);
 static int hx8357d_probe(struct spi_device *spi)
 {
 	struct device *dev = &spi->dev;
+	struct drm_device *drm;
 	struct mipi_dbi *mipi;
 	struct gpio_desc *dc;
 	u32 rotation = 0;
 	int ret;
 
-	mipi = devm_kzalloc(dev, sizeof(*mipi), GFP_KERNEL);
+	mipi = kzalloc(sizeof(*mipi), GFP_KERNEL);
 	if (!mipi)
 		return -ENOMEM;
 
+	drm = &mipi->drm;
+	ret = devm_drm_dev_init(dev, drm, &hx8357d_driver);
+	if (ret) {
+		kfree(mipi);
+		return ret;
+	}
+
+	drm_mode_config_init(drm);
+
 	dc = devm_gpiod_get(dev, "dc", GPIOD_OUT_LOW);
 	if (IS_ERR(dc)) {
 		DRM_DEV_ERROR(dev, "Failed to get gpio 'dc'\n");
@@ -238,21 +255,36 @@ static int hx8357d_probe(struct spi_device *spi)
 	if (ret)
 		return ret;
 
-	ret = mipi_dbi_init(&spi->dev, mipi, &hx8357d_pipe_funcs,
-			    &hx8357d_driver, &yx350hv15_mode, rotation);
+	ret = mipi_dbi_init(mipi, &hx8357d_pipe_funcs, &yx350hv15_mode, rotation);
 	if (ret)
 		return ret;
 
-	spi_set_drvdata(spi, mipi);
+	drm_mode_config_reset(drm);
 
-	return devm_tinydrm_register(&mipi->tinydrm);
+	ret = drm_dev_register(drm, 0);
+	if (ret)
+		return ret;
+
+	spi_set_drvdata(spi, drm);
+
+	drm_fbdev_generic_setup(drm, 0);
+
+	return 0;
 }
 
-static void hx8357d_shutdown(struct spi_device *spi)
+static int hx8357d_remove(struct spi_device *spi)
 {
-	struct mipi_dbi *mipi = spi_get_drvdata(spi);
+	struct drm_device *drm = spi_get_drvdata(spi);
+
+	drm_dev_unplug(drm);
+	drm_atomic_helper_shutdown(drm);
 
-	tinydrm_shutdown(&mipi->tinydrm);
+	return 0;
+}
+
+static void hx8357d_shutdown(struct spi_device *spi)
+{
+	drm_atomic_helper_shutdown(spi_get_drvdata(spi));
 }
 
 static struct spi_driver hx8357d_spi_driver = {
@@ -262,6 +294,7 @@ static struct spi_driver hx8357d_spi_driver = {
 	},
 	.id_table = hx8357d_id,
 	.probe = hx8357d_probe,
+	.remove = hx8357d_remove,
 	.shutdown = hx8357d_shutdown,
 };
 module_spi_driver(hx8357d_spi_driver);
diff --git a/drivers/gpu/drm/tinydrm/ili9225.c b/drivers/gpu/drm/tinydrm/ili9225.c
index 43a3b68d90a2..4b1a587c0134 100644
--- a/drivers/gpu/drm/tinydrm/ili9225.c
+++ b/drivers/gpu/drm/tinydrm/ili9225.c
@@ -20,9 +20,11 @@
 #include <linux/spi/spi.h>
 #include <video/mipi_display.h>
 
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_damage_helper.h>
 #include <drm/drm_drv.h>
 #include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_gem_cma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
@@ -81,20 +83,22 @@ static inline int ili9225_command(struct mipi_dbi *mipi, u8 cmd, u16 data)
 static void ili9225_fb_dirty(struct drm_framebuffer *fb, struct drm_rect *rect)
 {
 	struct drm_gem_cma_object *cma_obj = drm_fb_cma_get_gem_obj(fb, 0);
-	struct tinydrm_device *tdev = fb->dev->dev_private;
-	struct mipi_dbi *mipi = mipi_dbi_from_tinydrm(tdev);
+	struct mipi_dbi *mipi = drm_to_mipi_dbi(fb->dev);
 	unsigned int height = rect->y2 - rect->y1;
 	unsigned int width = rect->x2 - rect->x1;
 	bool swap = mipi->swap_bytes;
 	u16 x_start, y_start;
 	u16 x1, x2, y1, y2;
-	int ret = 0;
+	int idx, ret = 0;
 	bool full;
 	void *tr;
 
 	if (!mipi->enabled)
 		return;
 
+	if (!drm_dev_enter(fb->dev, &idx))
+		return;
+
 	full = width == fb->width && height == fb->height;
 
 	DRM_DEBUG_KMS("Flushing [FB:%d] " DRM_RECT_FMT "\n", fb->base.id, DRM_RECT_ARG(rect));
@@ -157,6 +161,8 @@ static void ili9225_fb_dirty(struct drm_framebuffer *fb, struct drm_rect *rect)
 err_msg:
 	if (ret)
 		dev_err_once(fb->dev->dev, "Failed to update display %d\n", ret);
+
+	drm_dev_exit(idx);
 }
 
 static void ili9225_pipe_update(struct drm_simple_display_pipe *pipe,
@@ -181,19 +187,21 @@ static void ili9225_pipe_enable(struct drm_simple_display_pipe *pipe,
 				struct drm_crtc_state *crtc_state,
 				struct drm_plane_state *plane_state)
 {
-	struct tinydrm_device *tdev = pipe_to_tinydrm(pipe);
-	struct mipi_dbi *mipi = mipi_dbi_from_tinydrm(tdev);
+	struct mipi_dbi *mipi = drm_to_mipi_dbi(pipe->crtc.dev);
 	struct drm_framebuffer *fb = plane_state->fb;
-	struct device *dev = tdev->drm->dev;
+	struct device *dev = pipe->crtc.dev->dev;
 	struct drm_rect rect = {
 		.x1 = 0,
 		.x2 = fb->width,
 		.y1 = 0,
 		.y2 = fb->height,
 	};
-	int ret;
+	int ret, idx;
 	u8 am_id;
 
+	if (!drm_dev_enter(pipe->crtc.dev, &idx))
+		return;
+
 	DRM_DEBUG_KMS("\n");
 
 	mipi_dbi_hw_reset(mipi);
@@ -207,7 +215,7 @@ static void ili9225_pipe_enable(struct drm_simple_display_pipe *pipe,
 	ret = ili9225_command(mipi, ILI9225_POWER_CONTROL_1, 0x0000);
 	if (ret) {
 		DRM_DEV_ERROR(dev, "Error sending command %d\n", ret);
-		return;
+		goto out_exit;
 	}
 	ili9225_command(mipi, ILI9225_POWER_CONTROL_2, 0x0000);
 	ili9225_command(mipi, ILI9225_POWER_CONTROL_3, 0x0000);
@@ -280,15 +288,23 @@ static void ili9225_pipe_enable(struct drm_simple_display_pipe *pipe,
 
 	mipi->enabled = true;
 	ili9225_fb_dirty(fb, &rect);
+out_exit:
+	drm_dev_exit(idx);
 }
 
 static void ili9225_pipe_disable(struct drm_simple_display_pipe *pipe)
 {
-	struct tinydrm_device *tdev = pipe_to_tinydrm(pipe);
-	struct mipi_dbi *mipi = mipi_dbi_from_tinydrm(tdev);
+	struct mipi_dbi *mipi = drm_to_mipi_dbi(pipe->crtc.dev);
 
 	DRM_DEBUG_KMS("\n");
 
+	/*
+	 * This callback is not protected by drm_dev_enter/exit since we want to
+	 * turn off the display on regular driver unload. It's highly unlikely
+	 * that the underlying SPI controller is gone should this be called after
+	 * unplug.
+	 */
+
 	if (!mipi->enabled)
 		return;
 
@@ -301,7 +317,7 @@ static void ili9225_pipe_disable(struct drm_simple_display_pipe *pipe)
 	mipi->enabled = false;
 }
 
-static int ili9225_dbi_command(struct mipi_dbi *mipi, u8 cmd, u8 *par,
+static int ili9225_dbi_command(struct mipi_dbi *mipi, u8 *cmd, u8 *par,
 			       size_t num)
 {
 	struct spi_device *spi = mipi->spi;
@@ -311,11 +327,11 @@ static int ili9225_dbi_command(struct mipi_dbi *mipi, u8 cmd, u8 *par,
 
 	gpiod_set_value_cansleep(mipi->dc, 0);
 	speed_hz = mipi_dbi_spi_cmd_max_speed(spi, 1);
-	ret = tinydrm_spi_transfer(spi, speed_hz, NULL, 8, &cmd, 1);
+	ret = tinydrm_spi_transfer(spi, speed_hz, NULL, 8, cmd, 1);
 	if (ret || !num)
 		return ret;
 
-	if (cmd == ILI9225_WRITE_DATA_TO_GRAM && !mipi->swap_bytes)
+	if (*cmd == ILI9225_WRITE_DATA_TO_GRAM && !mipi->swap_bytes)
 		bpw = 16;
 
 	gpiod_set_value_cansleep(mipi->dc, 1);
@@ -332,7 +348,7 @@ static const struct drm_simple_display_pipe_funcs ili9225_pipe_funcs = {
 };
 
 static const struct drm_display_mode ili9225_mode = {
-	TINYDRM_MODE(176, 220, 35, 44),
+	DRM_SIMPLE_MODE(176, 220, 35, 44),
 };
 
 DEFINE_DRM_GEM_CMA_FOPS(ili9225_fops);
@@ -341,6 +357,7 @@ static struct drm_driver ili9225_driver = {
 	.driver_features	= DRIVER_GEM | DRIVER_MODESET | DRIVER_PRIME |
 				  DRIVER_ATOMIC,
 	.fops			= &ili9225_fops,
+	.release		= mipi_dbi_release,
 	DRM_GEM_CMA_VMAP_DRIVER_OPS,
 	.name			= "ili9225",
 	.desc			= "Ilitek ILI9225",
@@ -364,15 +381,25 @@ MODULE_DEVICE_TABLE(spi, ili9225_id);
 static int ili9225_probe(struct spi_device *spi)
 {
 	struct device *dev = &spi->dev;
+	struct drm_device *drm;
 	struct mipi_dbi *mipi;
 	struct gpio_desc *rs;
 	u32 rotation = 0;
 	int ret;
 
-	mipi = devm_kzalloc(dev, sizeof(*mipi), GFP_KERNEL);
+	mipi = kzalloc(sizeof(*mipi), GFP_KERNEL);
 	if (!mipi)
 		return -ENOMEM;
 
+	drm = &mipi->drm;
+	ret = devm_drm_dev_init(dev, drm, &ili9225_driver);
+	if (ret) {
+		kfree(mipi);
+		return ret;
+	}
+
+	drm_mode_config_init(drm);
+
 	mipi->reset = devm_gpiod_get(dev, "reset", GPIOD_OUT_HIGH);
 	if (IS_ERR(mipi->reset)) {
 		DRM_DEV_ERROR(dev, "Failed to get gpio 'reset'\n");
@@ -394,21 +421,36 @@ static int ili9225_probe(struct spi_device *spi)
 	/* override the command function set in  mipi_dbi_spi_init() */
 	mipi->command = ili9225_dbi_command;
 
-	ret = mipi_dbi_init(&spi->dev, mipi, &ili9225_pipe_funcs,
-			    &ili9225_driver, &ili9225_mode, rotation);
+	ret = mipi_dbi_init(mipi, &ili9225_pipe_funcs, &ili9225_mode, rotation);
 	if (ret)
 		return ret;
 
-	spi_set_drvdata(spi, mipi);
+	drm_mode_config_reset(drm);
+
+	ret = drm_dev_register(drm, 0);
+	if (ret)
+		return ret;
+
+	spi_set_drvdata(spi, drm);
+
+	drm_fbdev_generic_setup(drm, 0);
 
-	return devm_tinydrm_register(&mipi->tinydrm);
+	return 0;
 }
 
-static void ili9225_shutdown(struct spi_device *spi)
+static int ili9225_remove(struct spi_device *spi)
 {
-	struct mipi_dbi *mipi = spi_get_drvdata(spi);
+	struct drm_device *drm = spi_get_drvdata(spi);
+
+	drm_dev_unplug(drm);
+	drm_atomic_helper_shutdown(drm);
+
+	return 0;
+}
 
-	tinydrm_shutdown(&mipi->tinydrm);
+static void ili9225_shutdown(struct spi_device *spi)
+{
+	drm_atomic_helper_shutdown(spi_get_drvdata(spi));
 }
 
 static struct spi_driver ili9225_spi_driver = {
@@ -419,6 +461,7 @@ static struct spi_driver ili9225_spi_driver = {
 	},
 	.id_table = ili9225_id,
 	.probe = ili9225_probe,
+	.remove = ili9225_remove,
 	.shutdown = ili9225_shutdown,
 };
 module_spi_driver(ili9225_spi_driver);
diff --git a/drivers/gpu/drm/tinydrm/ili9341.c b/drivers/gpu/drm/tinydrm/ili9341.c
index 713bb2dd7e04..4ade9e4b924f 100644
--- a/drivers/gpu/drm/tinydrm/ili9341.c
+++ b/drivers/gpu/drm/tinydrm/ili9341.c
@@ -15,7 +15,9 @@
 #include <linux/property.h>
 #include <linux/spi/spi.h>
 
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_drv.h>
+#include <drm/drm_fb_helper.h>
 #include <drm/drm_gem_cma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_modeset_helper.h>
@@ -52,16 +54,18 @@ static void yx240qv29_enable(struct drm_simple_display_pipe *pipe,
 			     struct drm_crtc_state *crtc_state,
 			     struct drm_plane_state *plane_state)
 {
-	struct tinydrm_device *tdev = pipe_to_tinydrm(pipe);
-	struct mipi_dbi *mipi = mipi_dbi_from_tinydrm(tdev);
+	struct mipi_dbi *mipi = drm_to_mipi_dbi(pipe->crtc.dev);
 	u8 addr_mode;
-	int ret;
+	int ret, idx;
+
+	if (!drm_dev_enter(pipe->crtc.dev, &idx))
+		return;
 
 	DRM_DEBUG_KMS("\n");
 
 	ret = mipi_dbi_poweron_conditional_reset(mipi);
 	if (ret < 0)
-		return;
+		goto out_exit;
 	if (ret == 1)
 		goto out_enable;
 
@@ -127,6 +131,8 @@ out_enable:
 	addr_mode |= ILI9341_MADCTL_BGR;
 	mipi_dbi_command(mipi, MIPI_DCS_SET_ADDRESS_MODE, addr_mode);
 	mipi_dbi_enable_flush(mipi, crtc_state, plane_state);
+out_exit:
+	drm_dev_exit(idx);
 }
 
 static const struct drm_simple_display_pipe_funcs ili9341_pipe_funcs = {
@@ -137,7 +143,7 @@ static const struct drm_simple_display_pipe_funcs ili9341_pipe_funcs = {
 };
 
 static const struct drm_display_mode yx240qv29_mode = {
-	TINYDRM_MODE(240, 320, 37, 49),
+	DRM_SIMPLE_MODE(240, 320, 37, 49),
 };
 
 DEFINE_DRM_GEM_CMA_FOPS(ili9341_fops);
@@ -145,6 +151,7 @@ DEFINE_DRM_GEM_CMA_FOPS(ili9341_fops);
 static struct drm_driver ili9341_driver = {
 	.driver_features	= DRIVER_GEM | DRIVER_MODESET | DRIVER_PRIME | DRIVER_ATOMIC,
 	.fops			= &ili9341_fops,
+	.release		= mipi_dbi_release,
 	DRM_GEM_CMA_VMAP_DRIVER_OPS,
 	.debugfs_init		= mipi_dbi_debugfs_init,
 	.name			= "ili9341",
@@ -169,15 +176,25 @@ MODULE_DEVICE_TABLE(spi, ili9341_id);
 static int ili9341_probe(struct spi_device *spi)
 {
 	struct device *dev = &spi->dev;
+	struct drm_device *drm;
 	struct mipi_dbi *mipi;
 	struct gpio_desc *dc;
 	u32 rotation = 0;
 	int ret;
 
-	mipi = devm_kzalloc(dev, sizeof(*mipi), GFP_KERNEL);
+	mipi = kzalloc(sizeof(*mipi), GFP_KERNEL);
 	if (!mipi)
 		return -ENOMEM;
 
+	drm = &mipi->drm;
+	ret = devm_drm_dev_init(dev, drm, &ili9341_driver);
+	if (ret) {
+		kfree(mipi);
+		return ret;
+	}
+
+	drm_mode_config_init(drm);
+
 	mipi->reset = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_HIGH);
 	if (IS_ERR(mipi->reset)) {
 		DRM_DEV_ERROR(dev, "Failed to get gpio 'reset'\n");
@@ -200,21 +217,36 @@ static int ili9341_probe(struct spi_device *spi)
 	if (ret)
 		return ret;
 
-	ret = mipi_dbi_init(&spi->dev, mipi, &ili9341_pipe_funcs,
-			    &ili9341_driver, &yx240qv29_mode, rotation);
+	ret = mipi_dbi_init(mipi, &ili9341_pipe_funcs, &yx240qv29_mode, rotation);
 	if (ret)
 		return ret;
 
-	spi_set_drvdata(spi, mipi);
+	drm_mode_config_reset(drm);
 
-	return devm_tinydrm_register(&mipi->tinydrm);
+	ret = drm_dev_register(drm, 0);
+	if (ret)
+		return ret;
+
+	spi_set_drvdata(spi, drm);
+
+	drm_fbdev_generic_setup(drm, 0);
+
+	return 0;
 }
 
-static void ili9341_shutdown(struct spi_device *spi)
+static int ili9341_remove(struct spi_device *spi)
 {
-	struct mipi_dbi *mipi = spi_get_drvdata(spi);
+	struct drm_device *drm = spi_get_drvdata(spi);
+
+	drm_dev_unplug(drm);
+	drm_atomic_helper_shutdown(drm);
 
-	tinydrm_shutdown(&mipi->tinydrm);
+	return 0;
+}
+
+static void ili9341_shutdown(struct spi_device *spi)
+{
+	drm_atomic_helper_shutdown(spi_get_drvdata(spi));
 }
 
 static struct spi_driver ili9341_spi_driver = {
@@ -224,6 +256,7 @@ static struct spi_driver ili9341_spi_driver = {
 	},
 	.id_table = ili9341_id,
 	.probe = ili9341_probe,
+	.remove = ili9341_remove,
 	.shutdown = ili9341_shutdown,
 };
 module_spi_driver(ili9341_spi_driver);
diff --git a/drivers/gpu/drm/tinydrm/mi0283qt.c b/drivers/gpu/drm/tinydrm/mi0283qt.c
index 82a92ec9ae3c..8e169846fbd8 100644
--- a/drivers/gpu/drm/tinydrm/mi0283qt.c
+++ b/drivers/gpu/drm/tinydrm/mi0283qt.c
@@ -17,7 +17,9 @@
 #include <linux/regulator/consumer.h>
 #include <linux/spi/spi.h>
 
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_drv.h>
+#include <drm/drm_fb_helper.h>
 #include <drm/drm_gem_cma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_modeset_helper.h>
@@ -54,16 +56,18 @@ static void mi0283qt_enable(struct drm_simple_display_pipe *pipe,
 			    struct drm_crtc_state *crtc_state,
 			    struct drm_plane_state *plane_state)
 {
-	struct tinydrm_device *tdev = pipe_to_tinydrm(pipe);
-	struct mipi_dbi *mipi = mipi_dbi_from_tinydrm(tdev);
+	struct mipi_dbi *mipi = drm_to_mipi_dbi(pipe->crtc.dev);
 	u8 addr_mode;
-	int ret;
+	int ret, idx;
+
+	if (!drm_dev_enter(pipe->crtc.dev, &idx))
+		return;
 
 	DRM_DEBUG_KMS("\n");
 
 	ret = mipi_dbi_poweron_conditional_reset(mipi);
 	if (ret < 0)
-		return;
+		goto out_exit;
 	if (ret == 1)
 		goto out_enable;
 
@@ -135,6 +139,8 @@ out_enable:
 	addr_mode |= ILI9341_MADCTL_BGR;
 	mipi_dbi_command(mipi, MIPI_DCS_SET_ADDRESS_MODE, addr_mode);
 	mipi_dbi_enable_flush(mipi, crtc_state, plane_state);
+out_exit:
+	drm_dev_exit(idx);
 }
 
 static const struct drm_simple_display_pipe_funcs mi0283qt_pipe_funcs = {
@@ -145,7 +151,7 @@ static const struct drm_simple_display_pipe_funcs mi0283qt_pipe_funcs = {
 };
 
 static const struct drm_display_mode mi0283qt_mode = {
-	TINYDRM_MODE(320, 240, 58, 43),
+	DRM_SIMPLE_MODE(320, 240, 58, 43),
 };
 
 DEFINE_DRM_GEM_CMA_FOPS(mi0283qt_fops);
@@ -154,6 +160,7 @@ static struct drm_driver mi0283qt_driver = {
 	.driver_features	= DRIVER_GEM | DRIVER_MODESET | DRIVER_PRIME |
 				  DRIVER_ATOMIC,
 	.fops			= &mi0283qt_fops,
+	.release		= mipi_dbi_release,
 	DRM_GEM_CMA_VMAP_DRIVER_OPS,
 	.debugfs_init		= mipi_dbi_debugfs_init,
 	.name			= "mi0283qt",
@@ -178,15 +185,25 @@ MODULE_DEVICE_TABLE(spi, mi0283qt_id);
 static int mi0283qt_probe(struct spi_device *spi)
 {
 	struct device *dev = &spi->dev;
+	struct drm_device *drm;
 	struct mipi_dbi *mipi;
 	struct gpio_desc *dc;
 	u32 rotation = 0;
 	int ret;
 
-	mipi = devm_kzalloc(dev, sizeof(*mipi), GFP_KERNEL);
+	mipi = kzalloc(sizeof(*mipi), GFP_KERNEL);
 	if (!mipi)
 		return -ENOMEM;
 
+	drm = &mipi->drm;
+	ret = devm_drm_dev_init(dev, drm, &mi0283qt_driver);
+	if (ret) {
+		kfree(mipi);
+		return ret;
+	}
+
+	drm_mode_config_init(drm);
+
 	mipi->reset = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_HIGH);
 	if (IS_ERR(mipi->reset)) {
 		DRM_DEV_ERROR(dev, "Failed to get gpio 'reset'\n");
@@ -213,35 +230,46 @@ static int mi0283qt_probe(struct spi_device *spi)
 	if (ret)
 		return ret;
 
-	ret = mipi_dbi_init(&spi->dev, mipi, &mi0283qt_pipe_funcs,
-			    &mi0283qt_driver, &mi0283qt_mode, rotation);
+	ret = mipi_dbi_init(mipi, &mi0283qt_pipe_funcs, &mi0283qt_mode, rotation);
+	if (ret)
+		return ret;
+
+	drm_mode_config_reset(drm);
+
+	ret = drm_dev_register(drm, 0);
 	if (ret)
 		return ret;
 
-	spi_set_drvdata(spi, mipi);
+	spi_set_drvdata(spi, drm);
 
-	return devm_tinydrm_register(&mipi->tinydrm);
+	drm_fbdev_generic_setup(drm, 0);
+
+	return 0;
 }
 
-static void mi0283qt_shutdown(struct spi_device *spi)
+static int mi0283qt_remove(struct spi_device *spi)
 {
-	struct mipi_dbi *mipi = spi_get_drvdata(spi);
+	struct drm_device *drm = spi_get_drvdata(spi);
 
-	tinydrm_shutdown(&mipi->tinydrm);
+	drm_dev_unplug(drm);
+	drm_atomic_helper_shutdown(drm);
+
+	return 0;
 }
 
-static int __maybe_unused mi0283qt_pm_suspend(struct device *dev)
+static void mi0283qt_shutdown(struct spi_device *spi)
 {
-	struct mipi_dbi *mipi = dev_get_drvdata(dev);
+	drm_atomic_helper_shutdown(spi_get_drvdata(spi));
+}
 
-	return drm_mode_config_helper_suspend(mipi->tinydrm.drm);
+static int __maybe_unused mi0283qt_pm_suspend(struct device *dev)
+{
+	return drm_mode_config_helper_suspend(dev_get_drvdata(dev));
 }
 
 static int __maybe_unused mi0283qt_pm_resume(struct device *dev)
 {
-	struct mipi_dbi *mipi = dev_get_drvdata(dev);
-
-	drm_mode_config_helper_resume(mipi->tinydrm.drm);
+	drm_mode_config_helper_resume(dev_get_drvdata(dev));
 
 	return 0;
 }
@@ -259,6 +287,7 @@ static struct spi_driver mi0283qt_spi_driver = {
 	},
 	.id_table = mi0283qt_id,
 	.probe = mi0283qt_probe,
+	.remove = mi0283qt_remove,
 	.shutdown = mi0283qt_shutdown,
 };
 module_spi_driver(mi0283qt_spi_driver);
diff --git a/drivers/gpu/drm/tinydrm/mipi-dbi.c b/drivers/gpu/drm/tinydrm/mipi-dbi.c
index 918f77c7de34..85761b4abb83 100644
--- a/drivers/gpu/drm/tinydrm/mipi-dbi.c
+++ b/drivers/gpu/drm/tinydrm/mipi-dbi.c
@@ -21,6 +21,7 @@
 #include <drm/drm_drv.h>
 #include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_format_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_vblank.h>
@@ -153,16 +154,42 @@ EXPORT_SYMBOL(mipi_dbi_command_read);
  */
 int mipi_dbi_command_buf(struct mipi_dbi *mipi, u8 cmd, u8 *data, size_t len)
 {
+	u8 *cmdbuf;
 	int ret;
 
+	/* SPI requires dma-safe buffers */
+	cmdbuf = kmemdup(&cmd, 1, GFP_KERNEL);
+	if (!cmdbuf)
+		return -ENOMEM;
+
 	mutex_lock(&mipi->cmdlock);
-	ret = mipi->command(mipi, cmd, data, len);
+	ret = mipi->command(mipi, cmdbuf, data, len);
 	mutex_unlock(&mipi->cmdlock);
 
+	kfree(cmdbuf);
+
 	return ret;
 }
 EXPORT_SYMBOL(mipi_dbi_command_buf);
 
+/* This should only be used by mipi_dbi_command() */
+int mipi_dbi_command_stackbuf(struct mipi_dbi *mipi, u8 cmd, u8 *data, size_t len)
+{
+	u8 *buf;
+	int ret;
+
+	buf = kmemdup(data, len, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	ret = mipi_dbi_command_buf(mipi, cmd, buf, len);
+
+	kfree(buf);
+
+	return ret;
+}
+EXPORT_SYMBOL(mipi_dbi_command_stackbuf);
+
 /**
  * mipi_dbi_buf_copy - Copy a framebuffer, transforming it if necessary
  * @dst: The destination buffer
@@ -192,12 +219,12 @@ int mipi_dbi_buf_copy(void *dst, struct drm_framebuffer *fb,
 	switch (fb->format->format) {
 	case DRM_FORMAT_RGB565:
 		if (swap)
-			tinydrm_swab16(dst, src, fb, clip);
+			drm_fb_swab16(dst, src, fb, clip);
 		else
-			tinydrm_memcpy(dst, src, fb, clip);
+			drm_fb_memcpy(dst, src, fb, clip);
 		break;
 	case DRM_FORMAT_XRGB8888:
-		tinydrm_xrgb8888_to_rgb565(dst, src, fb, clip, swap);
+		drm_fb_xrgb8888_to_rgb565(dst, src, fb, clip, swap);
 		break;
 	default:
 		dev_err_once(fb->dev->dev, "Format is not supported: %s\n",
@@ -216,18 +243,20 @@ EXPORT_SYMBOL(mipi_dbi_buf_copy);
 static void mipi_dbi_fb_dirty(struct drm_framebuffer *fb, struct drm_rect *rect)
 {
 	struct drm_gem_cma_object *cma_obj = drm_fb_cma_get_gem_obj(fb, 0);
-	struct tinydrm_device *tdev = fb->dev->dev_private;
-	struct mipi_dbi *mipi = mipi_dbi_from_tinydrm(tdev);
+	struct mipi_dbi *mipi = drm_to_mipi_dbi(fb->dev);
 	unsigned int height = rect->y2 - rect->y1;
 	unsigned int width = rect->x2 - rect->x1;
 	bool swap = mipi->swap_bytes;
-	int ret = 0;
+	int idx, ret = 0;
 	bool full;
 	void *tr;
 
 	if (!mipi->enabled)
 		return;
 
+	if (!drm_dev_enter(fb->dev, &idx))
+		return;
+
 	full = width == fb->width && height == fb->height;
 
 	DRM_DEBUG_KMS("Flushing [FB:%d] " DRM_RECT_FMT "\n", fb->base.id, DRM_RECT_ARG(rect));
@@ -254,6 +283,8 @@ static void mipi_dbi_fb_dirty(struct drm_framebuffer *fb, struct drm_rect *rect)
 err_msg:
 	if (ret)
 		dev_err_once(fb->dev->dev, "Failed to update display %d\n", ret);
+
+	drm_dev_exit(idx);
 }
 
 /**
@@ -308,19 +339,29 @@ void mipi_dbi_enable_flush(struct mipi_dbi *mipi,
 		.y1 = 0,
 		.y2 = fb->height,
 	};
+	int idx;
+
+	if (!drm_dev_enter(&mipi->drm, &idx))
+		return;
 
 	mipi->enabled = true;
 	mipi_dbi_fb_dirty(fb, &rect);
 	backlight_enable(mipi->backlight);
+
+	drm_dev_exit(idx);
 }
 EXPORT_SYMBOL(mipi_dbi_enable_flush);
 
 static void mipi_dbi_blank(struct mipi_dbi *mipi)
 {
-	struct drm_device *drm = mipi->tinydrm.drm;
+	struct drm_device *drm = &mipi->drm;
 	u16 height = drm->mode_config.min_height;
 	u16 width = drm->mode_config.min_width;
 	size_t len = width * height * 2;
+	int idx;
+
+	if (!drm_dev_enter(drm, &idx))
+		return;
 
 	memset(mipi->tx_buf, 0, len);
 
@@ -330,6 +371,8 @@ static void mipi_dbi_blank(struct mipi_dbi *mipi)
 			 (height >> 8) & 0xFF, (height - 1) & 0xFF);
 	mipi_dbi_command_buf(mipi, MIPI_DCS_WRITE_MEMORY_START,
 			     (u8 *)mipi->tx_buf, len);
+
+	drm_dev_exit(idx);
 }
 
 /**
@@ -342,8 +385,10 @@ static void mipi_dbi_blank(struct mipi_dbi *mipi)
  */
 void mipi_dbi_pipe_disable(struct drm_simple_display_pipe *pipe)
 {
-	struct tinydrm_device *tdev = pipe_to_tinydrm(pipe);
-	struct mipi_dbi *mipi = mipi_dbi_from_tinydrm(tdev);
+	struct mipi_dbi *mipi = drm_to_mipi_dbi(pipe->crtc.dev);
+
+	if (!mipi->enabled)
+		return;
 
 	DRM_DEBUG_KMS("\n");
 
@@ -359,6 +404,12 @@ void mipi_dbi_pipe_disable(struct drm_simple_display_pipe *pipe)
 }
 EXPORT_SYMBOL(mipi_dbi_pipe_disable);
 
+static const struct drm_mode_config_funcs mipi_dbi_mode_config_funcs = {
+	.fb_create = drm_gem_fb_create_with_dirty,
+	.atomic_check = drm_atomic_helper_check,
+	.atomic_commit = drm_atomic_helper_commit,
+};
+
 static const uint32_t mipi_dbi_formats[] = {
 	DRM_FORMAT_RGB565,
 	DRM_FORMAT_XRGB8888,
@@ -366,31 +417,27 @@ static const uint32_t mipi_dbi_formats[] = {
 
 /**
  * mipi_dbi_init - MIPI DBI initialization
- * @dev: Parent device
  * @mipi: &mipi_dbi structure to initialize
- * @pipe_funcs: Display pipe functions
- * @driver: DRM driver
+ * @funcs: Display pipe functions
  * @mode: Display mode
  * @rotation: Initial rotation in degrees Counter Clock Wise
  *
- * This function initializes a &mipi_dbi structure and it's underlying
- * @tinydrm_device. It also sets up the display pipeline.
+ * This function sets up a &drm_simple_display_pipe with a &drm_connector that
+ * has one fixed &drm_display_mode which is rotated according to @rotation.
+ * This mode is used to set the mode config min/max width/height properties.
+ * Additionally &mipi_dbi.tx_buf is allocated.
  *
  * Supported formats: Native RGB565 and emulated XRGB8888.
  *
- * Objects created by this function will be automatically freed on driver
- * detach (devres).
- *
  * Returns:
  * Zero on success, negative error code on failure.
  */
-int mipi_dbi_init(struct device *dev, struct mipi_dbi *mipi,
-		  const struct drm_simple_display_pipe_funcs *pipe_funcs,
-		  struct drm_driver *driver,
+int mipi_dbi_init(struct mipi_dbi *mipi,
+		  const struct drm_simple_display_pipe_funcs *funcs,
 		  const struct drm_display_mode *mode, unsigned int rotation)
 {
 	size_t bufsize = mode->vdisplay * mode->hdisplay * sizeof(u16);
-	struct tinydrm_device *tdev = &mipi->tinydrm;
+	struct drm_device *drm = &mipi->drm;
 	int ret;
 
 	if (!mipi->command)
@@ -398,16 +445,12 @@ int mipi_dbi_init(struct device *dev, struct mipi_dbi *mipi,
 
 	mutex_init(&mipi->cmdlock);
 
-	mipi->tx_buf = devm_kmalloc(dev, bufsize, GFP_KERNEL);
+	mipi->tx_buf = devm_kmalloc(drm->dev, bufsize, GFP_KERNEL);
 	if (!mipi->tx_buf)
 		return -ENOMEM;
 
-	ret = devm_tinydrm_init(dev, tdev, driver);
-	if (ret)
-		return ret;
-
 	/* TODO: Maybe add DRM_MODE_CONNECTOR_SPI */
-	ret = tinydrm_display_pipe_init(tdev, pipe_funcs,
+	ret = tinydrm_display_pipe_init(drm, &mipi->pipe, funcs,
 					DRM_MODE_CONNECTOR_VIRTUAL,
 					mipi_dbi_formats,
 					ARRAY_SIZE(mipi_dbi_formats), mode,
@@ -415,21 +458,40 @@ int mipi_dbi_init(struct device *dev, struct mipi_dbi *mipi,
 	if (ret)
 		return ret;
 
-	drm_plane_enable_fb_damage_clips(&tdev->pipe.plane);
+	drm_plane_enable_fb_damage_clips(&mipi->pipe.plane);
 
-	tdev->drm->mode_config.preferred_depth = 16;
+	drm->mode_config.funcs = &mipi_dbi_mode_config_funcs;
+	drm->mode_config.preferred_depth = 16;
 	mipi->rotation = rotation;
 
-	drm_mode_config_reset(tdev->drm);
-
 	DRM_DEBUG_KMS("preferred_depth=%u, rotation = %u\n",
-		      tdev->drm->mode_config.preferred_depth, rotation);
+		      drm->mode_config.preferred_depth, rotation);
 
 	return 0;
 }
 EXPORT_SYMBOL(mipi_dbi_init);
 
 /**
+ * mipi_dbi_release - DRM driver release helper
+ * @drm: DRM device
+ *
+ * This function finalizes and frees &mipi_dbi.
+ *
+ * Drivers can use this as their &drm_driver->release callback.
+ */
+void mipi_dbi_release(struct drm_device *drm)
+{
+	struct mipi_dbi *dbi = drm_to_mipi_dbi(drm);
+
+	DRM_DEBUG_DRIVER("\n");
+
+	drm_mode_config_cleanup(drm);
+	drm_dev_fini(drm);
+	kfree(dbi);
+}
+EXPORT_SYMBOL(mipi_dbi_release);
+
+/**
  * mipi_dbi_hw_reset - Hardware reset of controller
  * @mipi: MIPI DBI structure
  *
@@ -481,7 +543,7 @@ EXPORT_SYMBOL(mipi_dbi_display_is_on);
 
 static int mipi_dbi_poweron_reset_conditional(struct mipi_dbi *mipi, bool cond)
 {
-	struct device *dev = mipi->tinydrm.drm->dev;
+	struct device *dev = mipi->drm.dev;
 	int ret;
 
 	if (mipi->regulator) {
@@ -774,18 +836,18 @@ static int mipi_dbi_spi1_transfer(struct mipi_dbi *mipi, int dc,
 	return 0;
 }
 
-static int mipi_dbi_typec1_command(struct mipi_dbi *mipi, u8 cmd,
+static int mipi_dbi_typec1_command(struct mipi_dbi *mipi, u8 *cmd,
 				   u8 *parameters, size_t num)
 {
-	unsigned int bpw = (cmd == MIPI_DCS_WRITE_MEMORY_START) ? 16 : 8;
+	unsigned int bpw = (*cmd == MIPI_DCS_WRITE_MEMORY_START) ? 16 : 8;
 	int ret;
 
-	if (mipi_dbi_command_is_read(mipi, cmd))
+	if (mipi_dbi_command_is_read(mipi, *cmd))
 		return -ENOTSUPP;
 
-	MIPI_DBI_DEBUG_COMMAND(cmd, parameters, num);
+	MIPI_DBI_DEBUG_COMMAND(*cmd, parameters, num);
 
-	ret = mipi_dbi_spi1_transfer(mipi, 0, &cmd, 1, 8);
+	ret = mipi_dbi_spi1_transfer(mipi, 0, cmd, 1, 8);
 	if (ret || !num)
 		return ret;
 
@@ -794,7 +856,7 @@ static int mipi_dbi_typec1_command(struct mipi_dbi *mipi, u8 cmd,
 
 /* MIPI DBI Type C Option 3 */
 
-static int mipi_dbi_typec3_command_read(struct mipi_dbi *mipi, u8 cmd,
+static int mipi_dbi_typec3_command_read(struct mipi_dbi *mipi, u8 *cmd,
 					u8 *data, size_t len)
 {
 	struct spi_device *spi = mipi->spi;
@@ -803,7 +865,7 @@ static int mipi_dbi_typec3_command_read(struct mipi_dbi *mipi, u8 cmd,
 	struct spi_transfer tr[2] = {
 		{
 			.speed_hz = speed_hz,
-			.tx_buf = &cmd,
+			.tx_buf = cmd,
 			.len = 1,
 		}, {
 			.speed_hz = speed_hz,
@@ -821,8 +883,8 @@ static int mipi_dbi_typec3_command_read(struct mipi_dbi *mipi, u8 cmd,
 	 * Support non-standard 24-bit and 32-bit Nokia read commands which
 	 * start with a dummy clock, so we need to read an extra byte.
 	 */
-	if (cmd == MIPI_DCS_GET_DISPLAY_ID ||
-	    cmd == MIPI_DCS_GET_DISPLAY_STATUS) {
+	if (*cmd == MIPI_DCS_GET_DISPLAY_ID ||
+	    *cmd == MIPI_DCS_GET_DISPLAY_STATUS) {
 		if (!(len == 3 || len == 4))
 			return -EINVAL;
 
@@ -852,7 +914,7 @@ static int mipi_dbi_typec3_command_read(struct mipi_dbi *mipi, u8 cmd,
 			data[i] = (buf[i] << 1) | !!(buf[i + 1] & BIT(7));
 	}
 
-	MIPI_DBI_DEBUG_COMMAND(cmd, data, len);
+	MIPI_DBI_DEBUG_COMMAND(*cmd, data, len);
 
 err_free:
 	kfree(buf);
@@ -860,7 +922,7 @@ err_free:
 	return ret;
 }
 
-static int mipi_dbi_typec3_command(struct mipi_dbi *mipi, u8 cmd,
+static int mipi_dbi_typec3_command(struct mipi_dbi *mipi, u8 *cmd,
 				   u8 *par, size_t num)
 {
 	struct spi_device *spi = mipi->spi;
@@ -868,18 +930,18 @@ static int mipi_dbi_typec3_command(struct mipi_dbi *mipi, u8 cmd,
 	u32 speed_hz;
 	int ret;
 
-	if (mipi_dbi_command_is_read(mipi, cmd))
+	if (mipi_dbi_command_is_read(mipi, *cmd))
 		return mipi_dbi_typec3_command_read(mipi, cmd, par, num);
 
-	MIPI_DBI_DEBUG_COMMAND(cmd, par, num);
+	MIPI_DBI_DEBUG_COMMAND(*cmd, par, num);
 
 	gpiod_set_value_cansleep(mipi->dc, 0);
 	speed_hz = mipi_dbi_spi_cmd_max_speed(spi, 1);
-	ret = tinydrm_spi_transfer(spi, speed_hz, NULL, 8, &cmd, 1);
+	ret = tinydrm_spi_transfer(spi, speed_hz, NULL, 8, cmd, 1);
 	if (ret || !num)
 		return ret;
 
-	if (cmd == MIPI_DCS_WRITE_MEMORY_START && !mipi->swap_bytes)
+	if (*cmd == MIPI_DCS_WRITE_MEMORY_START && !mipi->swap_bytes)
 		bpw = 16;
 
 	gpiod_set_value_cansleep(mipi->dc, 1);
@@ -926,7 +988,7 @@ int mipi_dbi_spi_init(struct spi_device *spi, struct mipi_dbi *mipi,
 	 * Even though it's not the SPI device that does DMA (the master does),
 	 * the dma mask is necessary for the dma_alloc_wc() in
 	 * drm_gem_cma_create(). The dma_addr returned will be a physical
-	 * adddress which might be different from the bus address, but this is
+	 * address which might be different from the bus address, but this is
 	 * not a problem since the address will not be used.
 	 * The virtual address is used in the transfer and the SPI core
 	 * re-maps it on the SPI master device using the DMA streaming API
@@ -976,11 +1038,16 @@ static ssize_t mipi_dbi_debugfs_command_write(struct file *file,
 	u8 val, cmd = 0, parameters[64];
 	char *buf, *pos, *token;
 	unsigned int i;
-	int ret;
+	int ret, idx;
+
+	if (!drm_dev_enter(&mipi->drm, &idx))
+		return -ENODEV;
 
 	buf = memdup_user_nul(ubuf, count);
-	if (IS_ERR(buf))
-		return PTR_ERR(buf);
+	if (IS_ERR(buf)) {
+		ret = PTR_ERR(buf);
+		goto err_exit;
+	}
 
 	/* strip trailing whitespace */
 	for (i = count - 1; i > 0; i--)
@@ -1016,6 +1083,8 @@ static ssize_t mipi_dbi_debugfs_command_write(struct file *file,
 
 err_free:
 	kfree(buf);
+err_exit:
+	drm_dev_exit(idx);
 
 	return ret < 0 ? ret : count;
 }
@@ -1024,8 +1093,11 @@ static int mipi_dbi_debugfs_command_show(struct seq_file *m, void *unused)
 {
 	struct mipi_dbi *mipi = m->private;
 	u8 cmd, val[4];
+	int ret, idx;
 	size_t len;
-	int ret;
+
+	if (!drm_dev_enter(&mipi->drm, &idx))
+		return -ENODEV;
 
 	for (cmd = 0; cmd < 255; cmd++) {
 		if (!mipi_dbi_command_is_read(mipi, cmd))
@@ -1056,6 +1128,8 @@ static int mipi_dbi_debugfs_command_show(struct seq_file *m, void *unused)
 		seq_printf(m, "%*phN\n", (int)len, val);
 	}
 
+	drm_dev_exit(idx);
+
 	return 0;
 }
 
@@ -1088,8 +1162,7 @@ static const struct file_operations mipi_dbi_debugfs_command_fops = {
  */
 int mipi_dbi_debugfs_init(struct drm_minor *minor)
 {
-	struct tinydrm_device *tdev = minor->dev->dev_private;
-	struct mipi_dbi *mipi = mipi_dbi_from_tinydrm(tdev);
+	struct mipi_dbi *mipi = drm_to_mipi_dbi(minor->dev);
 	umode_t mode = S_IFREG | S_IWUSR;
 
 	if (mipi->read_commands)
diff --git a/drivers/gpu/drm/tinydrm/repaper.c b/drivers/gpu/drm/tinydrm/repaper.c
index b037c6540cf3..370629e2de94 100644
--- a/drivers/gpu/drm/tinydrm/repaper.c
+++ b/drivers/gpu/drm/tinydrm/repaper.c
@@ -26,14 +26,17 @@
 #include <linux/spi/spi.h>
 #include <linux/thermal.h>
 
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_damage_helper.h>
 #include <drm/drm_drv.h>
 #include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_helper.h>
+#include <drm/drm_format_helper.h>
 #include <drm/drm_gem_cma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_rect.h>
 #include <drm/drm_vblank.h>
-#include <drm/tinydrm/tinydrm.h>
+#include <drm/drm_simple_kms_helper.h>
 #include <drm/tinydrm/tinydrm-helpers.h>
 
 #define REPAPER_RID_G2_COG_ID	0x12
@@ -59,7 +62,8 @@ enum repaper_epd_border_byte {
 };
 
 struct repaper_epd {
-	struct tinydrm_device tinydrm;
+	struct drm_device drm;
+	struct drm_simple_display_pipe pipe;
 	struct spi_device *spi;
 
 	struct gpio_desc *panel_on;
@@ -88,10 +92,9 @@ struct repaper_epd {
 	bool partial;
 };
 
-static inline struct repaper_epd *
-epd_from_tinydrm(struct tinydrm_device *tdev)
+static inline struct repaper_epd *drm_to_epd(struct drm_device *drm)
 {
-	return container_of(tdev, struct repaper_epd, tinydrm);
+	return container_of(drm, struct repaper_epd, drm);
 }
 
 static int repaper_spi_transfer(struct spi_device *spi, u8 header,
@@ -529,11 +532,16 @@ static int repaper_fb_dirty(struct drm_framebuffer *fb)
 {
 	struct drm_gem_cma_object *cma_obj = drm_fb_cma_get_gem_obj(fb, 0);
 	struct dma_buf_attachment *import_attach = cma_obj->base.import_attach;
-	struct tinydrm_device *tdev = fb->dev->dev_private;
-	struct repaper_epd *epd = epd_from_tinydrm(tdev);
+	struct repaper_epd *epd = drm_to_epd(fb->dev);
 	struct drm_rect clip;
+	int idx, ret = 0;
 	u8 *buf = NULL;
-	int ret = 0;
+
+	if (!epd->enabled)
+		return 0;
+
+	if (!drm_dev_enter(fb->dev, &idx))
+		return -ENODEV;
 
 	/* repaper can't do partial updates */
 	clip.x1 = 0;
@@ -541,17 +549,16 @@ static int repaper_fb_dirty(struct drm_framebuffer *fb)
 	clip.y1 = 0;
 	clip.y2 = fb->height;
 
-	if (!epd->enabled)
-		return 0;
-
 	repaper_get_temperature(epd);
 
 	DRM_DEBUG("Flushing [FB:%d] st=%ums\n", fb->base.id,
 		  epd->factored_stage_time);
 
 	buf = kmalloc_array(fb->width, fb->height, GFP_KERNEL);
-	if (!buf)
-		return -ENOMEM;
+	if (!buf) {
+		ret = -ENOMEM;
+		goto out_exit;
+	}
 
 	if (import_attach) {
 		ret = dma_buf_begin_cpu_access(import_attach->dmabuf,
@@ -560,7 +567,7 @@ static int repaper_fb_dirty(struct drm_framebuffer *fb)
 			goto out_free;
 	}
 
-	tinydrm_xrgb8888_to_gray8(buf, cma_obj->vaddr, fb, &clip);
+	drm_fb_xrgb8888_to_gray8(buf, cma_obj->vaddr, fb, &clip);
 
 	if (import_attach) {
 		ret = dma_buf_end_cpu_access(import_attach->dmabuf,
@@ -620,6 +627,8 @@ static int repaper_fb_dirty(struct drm_framebuffer *fb)
 
 out_free:
 	kfree(buf);
+out_exit:
+	drm_dev_exit(idx);
 
 	return ret;
 }
@@ -645,12 +654,14 @@ static void repaper_pipe_enable(struct drm_simple_display_pipe *pipe,
 				struct drm_crtc_state *crtc_state,
 				struct drm_plane_state *plane_state)
 {
-	struct tinydrm_device *tdev = pipe_to_tinydrm(pipe);
-	struct repaper_epd *epd = epd_from_tinydrm(tdev);
+	struct repaper_epd *epd = drm_to_epd(pipe->crtc.dev);
 	struct spi_device *spi = epd->spi;
 	struct device *dev = &spi->dev;
 	bool dc_ok = false;
-	int i, ret;
+	int i, ret, idx;
+
+	if (!drm_dev_enter(pipe->crtc.dev, &idx))
+		return;
 
 	DRM_DEBUG_DRIVER("\n");
 
@@ -689,7 +700,7 @@ static void repaper_pipe_enable(struct drm_simple_display_pipe *pipe,
 	if (!i) {
 		DRM_DEV_ERROR(dev, "timeout waiting for panel to become ready.\n");
 		power_off(epd);
-		return;
+		goto out_exit;
 	}
 
 	repaper_read_id(spi);
@@ -700,7 +711,7 @@ static void repaper_pipe_enable(struct drm_simple_display_pipe *pipe,
 		else
 			dev_err(dev, "wrong COG ID 0x%02x\n", ret);
 		power_off(epd);
-		return;
+		goto out_exit;
 	}
 
 	/* Disable OE */
@@ -713,7 +724,7 @@ static void repaper_pipe_enable(struct drm_simple_display_pipe *pipe,
 		else
 			DRM_DEV_ERROR(dev, "panel is reported broken\n");
 		power_off(epd);
-		return;
+		goto out_exit;
 	}
 
 	/* Power saving mode */
@@ -753,7 +764,7 @@ static void repaper_pipe_enable(struct drm_simple_display_pipe *pipe,
 		if (ret < 0) {
 			DRM_DEV_ERROR(dev, "failed to read chip (%d)\n", ret);
 			power_off(epd);
-			return;
+			goto out_exit;
 		}
 
 		if (ret & 0x40) {
@@ -765,7 +776,7 @@ static void repaper_pipe_enable(struct drm_simple_display_pipe *pipe,
 	if (!dc_ok) {
 		DRM_DEV_ERROR(dev, "dc/dc failed\n");
 		power_off(epd);
-		return;
+		goto out_exit;
 	}
 
 	/*
@@ -776,15 +787,26 @@ static void repaper_pipe_enable(struct drm_simple_display_pipe *pipe,
 
 	epd->enabled = true;
 	epd->partial = false;
+out_exit:
+	drm_dev_exit(idx);
 }
 
 static void repaper_pipe_disable(struct drm_simple_display_pipe *pipe)
 {
-	struct tinydrm_device *tdev = pipe_to_tinydrm(pipe);
-	struct repaper_epd *epd = epd_from_tinydrm(tdev);
+	struct repaper_epd *epd = drm_to_epd(pipe->crtc.dev);
 	struct spi_device *spi = epd->spi;
 	unsigned int line;
 
+	/*
+	 * This callback is not protected by drm_dev_enter/exit since we want to
+	 * turn off the display on regular driver unload. It's highly unlikely
+	 * that the underlying SPI controller is gone should this be called after
+	 * unplug.
+	 */
+
+	if (!epd->enabled)
+		return;
+
 	DRM_DEBUG_DRIVER("\n");
 
 	epd->enabled = false;
@@ -855,33 +877,50 @@ static const struct drm_simple_display_pipe_funcs repaper_pipe_funcs = {
 	.prepare_fb = drm_gem_fb_simple_display_pipe_prepare_fb,
 };
 
+static const struct drm_mode_config_funcs repaper_mode_config_funcs = {
+	.fb_create = drm_gem_fb_create_with_dirty,
+	.atomic_check = drm_atomic_helper_check,
+	.atomic_commit = drm_atomic_helper_commit,
+};
+
+static void repaper_release(struct drm_device *drm)
+{
+	struct repaper_epd *epd = drm_to_epd(drm);
+
+	DRM_DEBUG_DRIVER("\n");
+
+	drm_mode_config_cleanup(drm);
+	drm_dev_fini(drm);
+	kfree(epd);
+}
+
 static const uint32_t repaper_formats[] = {
 	DRM_FORMAT_XRGB8888,
 };
 
 static const struct drm_display_mode repaper_e1144cs021_mode = {
-	TINYDRM_MODE(128, 96, 29, 22),
+	DRM_SIMPLE_MODE(128, 96, 29, 22),
 };
 
 static const u8 repaper_e1144cs021_cs[] = { 0x00, 0x00, 0x00, 0x00,
 					    0x00, 0x0f, 0xff, 0x00 };
 
 static const struct drm_display_mode repaper_e1190cs021_mode = {
-	TINYDRM_MODE(144, 128, 36, 32),
+	DRM_SIMPLE_MODE(144, 128, 36, 32),
 };
 
 static const u8 repaper_e1190cs021_cs[] = { 0x00, 0x00, 0x00, 0x03,
 					    0xfc, 0x00, 0x00, 0xff };
 
 static const struct drm_display_mode repaper_e2200cs021_mode = {
-	TINYDRM_MODE(200, 96, 46, 22),
+	DRM_SIMPLE_MODE(200, 96, 46, 22),
 };
 
 static const u8 repaper_e2200cs021_cs[] = { 0x00, 0x00, 0x00, 0x00,
 					    0x01, 0xff, 0xe0, 0x00 };
 
 static const struct drm_display_mode repaper_e2271cs021_mode = {
-	TINYDRM_MODE(264, 176, 57, 38),
+	DRM_SIMPLE_MODE(264, 176, 57, 38),
 };
 
 static const u8 repaper_e2271cs021_cs[] = { 0x00, 0x00, 0x00, 0x7f,
@@ -893,6 +932,7 @@ static struct drm_driver repaper_driver = {
 	.driver_features	= DRIVER_GEM | DRIVER_MODESET | DRIVER_PRIME |
 				  DRIVER_ATOMIC,
 	.fops			= &repaper_fops,
+	.release		= repaper_release,
 	DRM_GEM_CMA_VMAP_DRIVER_OPS,
 	.name			= "repaper",
 	.desc			= "Pervasive Displays RePaper e-ink panels",
@@ -925,11 +965,11 @@ static int repaper_probe(struct spi_device *spi)
 	const struct spi_device_id *spi_id;
 	const struct of_device_id *match;
 	struct device *dev = &spi->dev;
-	struct tinydrm_device *tdev;
 	enum repaper_model model;
 	const char *thermal_zone;
 	struct repaper_epd *epd;
 	size_t line_buffer_size;
+	struct drm_device *drm;
 	int ret;
 
 	match = of_match_device(repaper_of_match, dev);
@@ -949,10 +989,21 @@ static int repaper_probe(struct spi_device *spi)
 		}
 	}
 
-	epd = devm_kzalloc(dev, sizeof(*epd), GFP_KERNEL);
+	epd = kzalloc(sizeof(*epd), GFP_KERNEL);
 	if (!epd)
 		return -ENOMEM;
 
+	drm = &epd->drm;
+
+	ret = devm_drm_dev_init(dev, drm, &repaper_driver);
+	if (ret) {
+		kfree(epd);
+		return ret;
+	}
+
+	drm_mode_config_init(drm);
+	drm->mode_config.funcs = &repaper_mode_config_funcs;
+
 	epd->spi = spi;
 
 	epd->panel_on = devm_gpiod_get(dev, "panel-on", GPIOD_OUT_LOW);
@@ -1063,32 +1114,41 @@ static int repaper_probe(struct spi_device *spi)
 	if (!epd->current_frame)
 		return -ENOMEM;
 
-	tdev = &epd->tinydrm;
-
-	ret = devm_tinydrm_init(dev, tdev, &repaper_driver);
-	if (ret)
-		return ret;
-
-	ret = tinydrm_display_pipe_init(tdev, &repaper_pipe_funcs,
+	ret = tinydrm_display_pipe_init(drm, &epd->pipe, &repaper_pipe_funcs,
 					DRM_MODE_CONNECTOR_VIRTUAL,
 					repaper_formats,
 					ARRAY_SIZE(repaper_formats), mode, 0);
 	if (ret)
 		return ret;
 
-	drm_mode_config_reset(tdev->drm);
-	spi_set_drvdata(spi, tdev);
+	drm_mode_config_reset(drm);
+
+	ret = drm_dev_register(drm, 0);
+	if (ret)
+		return ret;
+
+	spi_set_drvdata(spi, drm);
 
 	DRM_DEBUG_DRIVER("SPI speed: %uMHz\n", spi->max_speed_hz / 1000000);
 
-	return devm_tinydrm_register(tdev);
+	drm_fbdev_generic_setup(drm, 0);
+
+	return 0;
 }
 
-static void repaper_shutdown(struct spi_device *spi)
+static int repaper_remove(struct spi_device *spi)
 {
-	struct tinydrm_device *tdev = spi_get_drvdata(spi);
+	struct drm_device *drm = spi_get_drvdata(spi);
+
+	drm_dev_unplug(drm);
+	drm_atomic_helper_shutdown(drm);
+
+	return 0;
+}
 
-	tinydrm_shutdown(tdev);
+static void repaper_shutdown(struct spi_device *spi)
+{
+	drm_atomic_helper_shutdown(spi_get_drvdata(spi));
 }
 
 static struct spi_driver repaper_spi_driver = {
@@ -1099,6 +1159,7 @@ static struct spi_driver repaper_spi_driver = {
 	},
 	.id_table = repaper_id,
 	.probe = repaper_probe,
+	.remove = repaper_remove,
 	.shutdown = repaper_shutdown,
 };
 module_spi_driver(repaper_spi_driver);
diff --git a/drivers/gpu/drm/tinydrm/st7586.c b/drivers/gpu/drm/tinydrm/st7586.c
index 01a8077954b3..36bb16a15f7e 100644
--- a/drivers/gpu/drm/tinydrm/st7586.c
+++ b/drivers/gpu/drm/tinydrm/st7586.c
@@ -17,9 +17,12 @@
 #include <linux/spi/spi.h>
 #include <video/mipi_display.h>
 
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_damage_helper.h>
 #include <drm/drm_drv.h>
 #include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_helper.h>
+#include <drm/drm_format_helper.h>
 #include <drm/drm_gem_cma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_rect.h>
@@ -75,7 +78,7 @@ static void st7586_xrgb8888_to_gray332(u8 *dst, void *vaddr,
 	if (!buf)
 		return;
 
-	tinydrm_xrgb8888_to_gray8(buf, vaddr, fb, clip);
+	drm_fb_xrgb8888_to_gray8(buf, vaddr, fb, clip);
 	src = buf;
 
 	for (y = clip->y1; y < clip->y2; y++) {
@@ -116,14 +119,15 @@ static int st7586_buf_copy(void *dst, struct drm_framebuffer *fb,
 
 static void st7586_fb_dirty(struct drm_framebuffer *fb, struct drm_rect *rect)
 {
-	struct tinydrm_device *tdev = fb->dev->dev_private;
-	struct mipi_dbi *mipi = mipi_dbi_from_tinydrm(tdev);
-	int start, end;
-	int ret = 0;
+	struct mipi_dbi *mipi = drm_to_mipi_dbi(fb->dev);
+	int start, end, idx, ret = 0;
 
 	if (!mipi->enabled)
 		return;
 
+	if (!drm_dev_enter(fb->dev, &idx))
+		return;
+
 	/* 3 pixels per byte, so grow clip to nearest multiple of 3 */
 	rect->x1 = rounddown(rect->x1, 3);
 	rect->x2 = roundup(rect->x2, 3);
@@ -151,6 +155,8 @@ static void st7586_fb_dirty(struct drm_framebuffer *fb, struct drm_rect *rect)
 err_msg:
 	if (ret)
 		dev_err_once(fb->dev->dev, "Failed to update display %d\n", ret);
+
+	drm_dev_exit(idx);
 }
 
 static void st7586_pipe_update(struct drm_simple_display_pipe *pipe,
@@ -175,8 +181,7 @@ static void st7586_pipe_enable(struct drm_simple_display_pipe *pipe,
 			       struct drm_crtc_state *crtc_state,
 			       struct drm_plane_state *plane_state)
 {
-	struct tinydrm_device *tdev = pipe_to_tinydrm(pipe);
-	struct mipi_dbi *mipi = mipi_dbi_from_tinydrm(tdev);
+	struct mipi_dbi *mipi = drm_to_mipi_dbi(pipe->crtc.dev);
 	struct drm_framebuffer *fb = plane_state->fb;
 	struct drm_rect rect = {
 		.x1 = 0,
@@ -184,14 +189,17 @@ static void st7586_pipe_enable(struct drm_simple_display_pipe *pipe,
 		.y1 = 0,
 		.y2 = fb->height,
 	};
-	int ret;
+	int idx, ret;
 	u8 addr_mode;
 
+	if (!drm_dev_enter(pipe->crtc.dev, &idx))
+		return;
+
 	DRM_DEBUG_KMS("\n");
 
 	ret = mipi_dbi_poweron_reset(mipi);
 	if (ret)
-		return;
+		goto out_exit;
 
 	mipi_dbi_command(mipi, ST7586_AUTO_READ_CTRL, 0x9f);
 	mipi_dbi_command(mipi, ST7586_OTP_RW_CTRL, 0x00);
@@ -244,12 +252,20 @@ static void st7586_pipe_enable(struct drm_simple_display_pipe *pipe,
 	st7586_fb_dirty(fb, &rect);
 
 	mipi_dbi_command(mipi, MIPI_DCS_SET_DISPLAY_ON);
+out_exit:
+	drm_dev_exit(idx);
 }
 
 static void st7586_pipe_disable(struct drm_simple_display_pipe *pipe)
 {
-	struct tinydrm_device *tdev = pipe_to_tinydrm(pipe);
-	struct mipi_dbi *mipi = mipi_dbi_from_tinydrm(tdev);
+	struct mipi_dbi *mipi = drm_to_mipi_dbi(pipe->crtc.dev);
+
+	/*
+	 * This callback is not protected by drm_dev_enter/exit since we want to
+	 * turn off the display on regular driver unload. It's highly unlikely
+	 * that the underlying SPI controller is gone should this be called after
+	 * unplug.
+	 */
 
 	DRM_DEBUG_KMS("\n");
 
@@ -264,46 +280,6 @@ static const u32 st7586_formats[] = {
 	DRM_FORMAT_XRGB8888,
 };
 
-static int st7586_init(struct device *dev, struct mipi_dbi *mipi,
-		const struct drm_simple_display_pipe_funcs *pipe_funcs,
-		struct drm_driver *driver, const struct drm_display_mode *mode,
-		unsigned int rotation)
-{
-	size_t bufsize = (mode->vdisplay + 2) / 3 * mode->hdisplay;
-	struct tinydrm_device *tdev = &mipi->tinydrm;
-	int ret;
-
-	mutex_init(&mipi->cmdlock);
-
-	mipi->tx_buf = devm_kmalloc(dev, bufsize, GFP_KERNEL);
-	if (!mipi->tx_buf)
-		return -ENOMEM;
-
-	ret = devm_tinydrm_init(dev, tdev, driver);
-	if (ret)
-		return ret;
-
-	ret = tinydrm_display_pipe_init(tdev, pipe_funcs,
-					DRM_MODE_CONNECTOR_VIRTUAL,
-					st7586_formats,
-					ARRAY_SIZE(st7586_formats),
-					mode, rotation);
-	if (ret)
-		return ret;
-
-	drm_plane_enable_fb_damage_clips(&tdev->pipe.plane);
-
-	tdev->drm->mode_config.preferred_depth = 32;
-	mipi->rotation = rotation;
-
-	drm_mode_config_reset(tdev->drm);
-
-	DRM_DEBUG_KMS("preferred_depth=%u, rotation = %u\n",
-		      tdev->drm->mode_config.preferred_depth, rotation);
-
-	return 0;
-}
-
 static const struct drm_simple_display_pipe_funcs st7586_pipe_funcs = {
 	.enable		= st7586_pipe_enable,
 	.disable	= st7586_pipe_disable,
@@ -311,8 +287,14 @@ static const struct drm_simple_display_pipe_funcs st7586_pipe_funcs = {
 	.prepare_fb	= drm_gem_fb_simple_display_pipe_prepare_fb,
 };
 
+static const struct drm_mode_config_funcs st7586_mode_config_funcs = {
+	.fb_create = drm_gem_fb_create_with_dirty,
+	.atomic_check = drm_atomic_helper_check,
+	.atomic_commit = drm_atomic_helper_commit,
+};
+
 static const struct drm_display_mode st7586_mode = {
-	TINYDRM_MODE(178, 128, 37, 27),
+	DRM_SIMPLE_MODE(178, 128, 37, 27),
 };
 
 DEFINE_DRM_GEM_CMA_FOPS(st7586_fops);
@@ -321,6 +303,7 @@ static struct drm_driver st7586_driver = {
 	.driver_features	= DRIVER_GEM | DRIVER_MODESET | DRIVER_PRIME |
 				  DRIVER_ATOMIC,
 	.fops			= &st7586_fops,
+	.release		= mipi_dbi_release,
 	DRM_GEM_CMA_VMAP_DRIVER_OPS,
 	.debugfs_init		= mipi_dbi_debugfs_init,
 	.name			= "st7586",
@@ -345,15 +328,35 @@ MODULE_DEVICE_TABLE(spi, st7586_id);
 static int st7586_probe(struct spi_device *spi)
 {
 	struct device *dev = &spi->dev;
+	struct drm_device *drm;
 	struct mipi_dbi *mipi;
 	struct gpio_desc *a0;
 	u32 rotation = 0;
+	size_t bufsize;
 	int ret;
 
-	mipi = devm_kzalloc(dev, sizeof(*mipi), GFP_KERNEL);
+	mipi = kzalloc(sizeof(*mipi), GFP_KERNEL);
 	if (!mipi)
 		return -ENOMEM;
 
+	drm = &mipi->drm;
+	ret = devm_drm_dev_init(dev, drm, &st7586_driver);
+	if (ret) {
+		kfree(mipi);
+		return ret;
+	}
+
+	drm_mode_config_init(drm);
+	drm->mode_config.preferred_depth = 32;
+	drm->mode_config.funcs = &st7586_mode_config_funcs;
+
+	mutex_init(&mipi->cmdlock);
+
+	bufsize = (st7586_mode.vdisplay + 2) / 3 * st7586_mode.hdisplay;
+	mipi->tx_buf = devm_kmalloc(dev, bufsize, GFP_KERNEL);
+	if (!mipi->tx_buf)
+		return -ENOMEM;
+
 	mipi->reset = devm_gpiod_get(dev, "reset", GPIOD_OUT_HIGH);
 	if (IS_ERR(mipi->reset)) {
 		DRM_DEV_ERROR(dev, "Failed to get gpio 'reset'\n");
@@ -367,6 +370,7 @@ static int st7586_probe(struct spi_device *spi)
 	}
 
 	device_property_read_u32(dev, "rotation", &rotation);
+	mipi->rotation = rotation;
 
 	ret = mipi_dbi_spi_init(spi, mipi, a0);
 	if (ret)
@@ -384,21 +388,44 @@ static int st7586_probe(struct spi_device *spi)
 	 */
 	mipi->swap_bytes = true;
 
-	ret = st7586_init(&spi->dev, mipi, &st7586_pipe_funcs, &st7586_driver,
-			  &st7586_mode, rotation);
+	ret = tinydrm_display_pipe_init(drm, &mipi->pipe, &st7586_pipe_funcs,
+					DRM_MODE_CONNECTOR_VIRTUAL,
+					st7586_formats, ARRAY_SIZE(st7586_formats),
+					&st7586_mode, rotation);
 	if (ret)
 		return ret;
 
-	spi_set_drvdata(spi, mipi);
+	drm_plane_enable_fb_damage_clips(&mipi->pipe.plane);
 
-	return devm_tinydrm_register(&mipi->tinydrm);
+	drm_mode_config_reset(drm);
+
+	ret = drm_dev_register(drm, 0);
+	if (ret)
+		return ret;
+
+	spi_set_drvdata(spi, drm);
+
+	DRM_DEBUG_KMS("preferred_depth=%u, rotation = %u\n",
+		      drm->mode_config.preferred_depth, rotation);
+
+	drm_fbdev_generic_setup(drm, 0);
+
+	return 0;
 }
 
-static void st7586_shutdown(struct spi_device *spi)
+static int st7586_remove(struct spi_device *spi)
 {
-	struct mipi_dbi *mipi = spi_get_drvdata(spi);
+	struct drm_device *drm = spi_get_drvdata(spi);
+
+	drm_dev_unplug(drm);
+	drm_atomic_helper_shutdown(drm);
+
+	return 0;
+}
 
-	tinydrm_shutdown(&mipi->tinydrm);
+static void st7586_shutdown(struct spi_device *spi)
+{
+	drm_atomic_helper_shutdown(spi_get_drvdata(spi));
 }
 
 static struct spi_driver st7586_spi_driver = {
@@ -409,6 +436,7 @@ static struct spi_driver st7586_spi_driver = {
 	},
 	.id_table = st7586_id,
 	.probe = st7586_probe,
+	.remove = st7586_remove,
 	.shutdown = st7586_shutdown,
 };
 module_spi_driver(st7586_spi_driver);
diff --git a/drivers/gpu/drm/tinydrm/st7735r.c b/drivers/gpu/drm/tinydrm/st7735r.c
index 3bab9a9569a6..ce9109e613e0 100644
--- a/drivers/gpu/drm/tinydrm/st7735r.c
+++ b/drivers/gpu/drm/tinydrm/st7735r.c
@@ -14,7 +14,9 @@
 #include <linux/spi/spi.h>
 #include <video/mipi_display.h>
 
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_drv.h>
+#include <drm/drm_fb_helper.h>
 #include <drm/drm_gem_cma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/tinydrm/mipi-dbi.h>
@@ -41,16 +43,18 @@ static void jd_t18003_t01_pipe_enable(struct drm_simple_display_pipe *pipe,
 				      struct drm_crtc_state *crtc_state,
 				      struct drm_plane_state *plane_state)
 {
-	struct tinydrm_device *tdev = pipe_to_tinydrm(pipe);
-	struct mipi_dbi *mipi = mipi_dbi_from_tinydrm(tdev);
-	int ret;
+	struct mipi_dbi *mipi = drm_to_mipi_dbi(pipe->crtc.dev);
+	int ret, idx;
 	u8 addr_mode;
 
+	if (!drm_dev_enter(pipe->crtc.dev, &idx))
+		return;
+
 	DRM_DEBUG_KMS("\n");
 
 	ret = mipi_dbi_poweron_reset(mipi);
 	if (ret)
-		return;
+		goto out_exit;
 
 	msleep(150);
 
@@ -101,6 +105,8 @@ static void jd_t18003_t01_pipe_enable(struct drm_simple_display_pipe *pipe,
 	msleep(20);
 
 	mipi_dbi_enable_flush(mipi, crtc_state, plane_state);
+out_exit:
+	drm_dev_exit(idx);
 }
 
 static const struct drm_simple_display_pipe_funcs jd_t18003_t01_pipe_funcs = {
@@ -111,7 +117,7 @@ static const struct drm_simple_display_pipe_funcs jd_t18003_t01_pipe_funcs = {
 };
 
 static const struct drm_display_mode jd_t18003_t01_mode = {
-	TINYDRM_MODE(128, 160, 28, 35),
+	DRM_SIMPLE_MODE(128, 160, 28, 35),
 };
 
 DEFINE_DRM_GEM_CMA_FOPS(st7735r_fops);
@@ -120,6 +126,7 @@ static struct drm_driver st7735r_driver = {
 	.driver_features	= DRIVER_GEM | DRIVER_MODESET | DRIVER_PRIME |
 				  DRIVER_ATOMIC,
 	.fops			= &st7735r_fops,
+	.release		= mipi_dbi_release,
 	DRM_GEM_CMA_VMAP_DRIVER_OPS,
 	.debugfs_init		= mipi_dbi_debugfs_init,
 	.name			= "st7735r",
@@ -144,15 +151,25 @@ MODULE_DEVICE_TABLE(spi, st7735r_id);
 static int st7735r_probe(struct spi_device *spi)
 {
 	struct device *dev = &spi->dev;
+	struct drm_device *drm;
 	struct mipi_dbi *mipi;
 	struct gpio_desc *dc;
 	u32 rotation = 0;
 	int ret;
 
-	mipi = devm_kzalloc(dev, sizeof(*mipi), GFP_KERNEL);
+	mipi = kzalloc(sizeof(*mipi), GFP_KERNEL);
 	if (!mipi)
 		return -ENOMEM;
 
+	drm = &mipi->drm;
+	ret = devm_drm_dev_init(dev, drm, &st7735r_driver);
+	if (ret) {
+		kfree(mipi);
+		return ret;
+	}
+
+	drm_mode_config_init(drm);
+
 	mipi->reset = devm_gpiod_get(dev, "reset", GPIOD_OUT_HIGH);
 	if (IS_ERR(mipi->reset)) {
 		DRM_DEV_ERROR(dev, "Failed to get gpio 'reset'\n");
@@ -178,21 +195,36 @@ static int st7735r_probe(struct spi_device *spi)
 	/* Cannot read from Adafruit 1.8" display via SPI */
 	mipi->read_commands = NULL;
 
-	ret = mipi_dbi_init(&spi->dev, mipi, &jd_t18003_t01_pipe_funcs,
-			    &st7735r_driver, &jd_t18003_t01_mode, rotation);
+	ret = mipi_dbi_init(mipi, &jd_t18003_t01_pipe_funcs, &jd_t18003_t01_mode, rotation);
 	if (ret)
 		return ret;
 
-	spi_set_drvdata(spi, mipi);
+	drm_mode_config_reset(drm);
 
-	return devm_tinydrm_register(&mipi->tinydrm);
+	ret = drm_dev_register(drm, 0);
+	if (ret)
+		return ret;
+
+	spi_set_drvdata(spi, drm);
+
+	drm_fbdev_generic_setup(drm, 0);
+
+	return 0;
 }
 
-static void st7735r_shutdown(struct spi_device *spi)
+static int st7735r_remove(struct spi_device *spi)
 {
-	struct mipi_dbi *mipi = spi_get_drvdata(spi);
+	struct drm_device *drm = spi_get_drvdata(spi);
+
+	drm_dev_unplug(drm);
+	drm_atomic_helper_shutdown(drm);
 
-	tinydrm_shutdown(&mipi->tinydrm);
+	return 0;
+}
+
+static void st7735r_shutdown(struct spi_device *spi)
+{
+	drm_atomic_helper_shutdown(spi_get_drvdata(spi));
 }
 
 static struct spi_driver st7735r_spi_driver = {
@@ -203,6 +235,7 @@ static struct spi_driver st7735r_spi_driver = {
 	},
 	.id_table = st7735r_id,
 	.probe = st7735r_probe,
+	.remove = st7735r_remove,
 	.shutdown = st7735r_shutdown,
 };
 module_spi_driver(st7735r_spi_driver);
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index 1a01669b159a..2845fceb2fbd 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -1626,7 +1626,6 @@ EXPORT_SYMBOL(ttm_bo_device_release);
 int ttm_bo_device_init(struct ttm_bo_device *bdev,
 		       struct ttm_bo_driver *driver,
 		       struct address_space *mapping,
-		       uint64_t file_page_offset,
 		       bool need_dma32)
 {
 	struct ttm_bo_global *glob = &ttm_bo_glob;
@@ -1648,8 +1647,9 @@ int ttm_bo_device_init(struct ttm_bo_device *bdev,
 	if (unlikely(ret != 0))
 		goto out_no_sys;
 
-	drm_vma_offset_manager_init(&bdev->vma_manager, file_page_offset,
-				    0x10000000);
+	drm_vma_offset_manager_init(&bdev->vma_manager,
+				    DRM_FILE_PAGE_OFFSET_START,
+				    DRM_FILE_PAGE_OFFSET_SIZE);
 	INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue);
 	INIT_LIST_HEAD(&bdev->ddestroy);
 	bdev->dev_mapping = mapping;
diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c
index e86a29a1e51f..6dacff49c1cc 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_vm.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c
@@ -432,6 +432,9 @@ int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma,
 	struct ttm_buffer_object *bo;
 	int ret;
 
+	if (unlikely(vma->vm_pgoff < DRM_FILE_PAGE_OFFSET_START))
+		return -EINVAL;
+
 	bo = ttm_bo_vm_lookup(bdev, vma->vm_pgoff, vma_pages(vma));
 	if (unlikely(!bo))
 		return -EINVAL;
diff --git a/drivers/gpu/drm/ttm/ttm_execbuf_util.c b/drivers/gpu/drm/ttm/ttm_execbuf_util.c
index 93860346c426..0075eb9a0b52 100644
--- a/drivers/gpu/drm/ttm/ttm_execbuf_util.c
+++ b/drivers/gpu/drm/ttm/ttm_execbuf_util.c
@@ -188,13 +188,11 @@ void ttm_eu_fence_buffer_objects(struct ww_acquire_ctx *ticket,
 	struct ttm_validate_buffer *entry;
 	struct ttm_buffer_object *bo;
 	struct ttm_bo_global *glob;
-	struct ttm_bo_device *bdev;
 
 	if (list_empty(list))
 		return;
 
 	bo = list_first_entry(list, struct ttm_validate_buffer, head)->bo;
-	bdev = bo->bdev;
 	glob = bo->bdev->glob;
 
 	spin_lock(&glob->lru_lock);
diff --git a/drivers/gpu/drm/ttm/ttm_memory.c b/drivers/gpu/drm/ttm/ttm_memory.c
index 9a0909decb36..8617958b7ae6 100644
--- a/drivers/gpu/drm/ttm/ttm_memory.c
+++ b/drivers/gpu/drm/ttm/ttm_memory.c
@@ -81,7 +81,7 @@ static void ttm_mem_zone_kobj_release(struct kobject *kobj)
 	struct ttm_mem_zone *zone =
 		container_of(kobj, struct ttm_mem_zone, kobj);
 
-	pr_info("Zone %7s: Used memory at exit: %llu kiB\n",
+	pr_info("Zone %7s: Used memory at exit: %llu KiB\n",
 		zone->name, (unsigned long long)zone->used_mem >> 10);
 	kfree(zone);
 }
@@ -448,7 +448,7 @@ int ttm_mem_global_init(struct ttm_mem_global *glob)
 #endif
 	for (i = 0; i < glob->num_zones; ++i) {
 		zone = glob->zones[i];
-		pr_info("Zone %7s: Available graphics memory: %llu kiB\n",
+		pr_info("Zone %7s: Available graphics memory: %llu KiB\n",
 			zone->name, (unsigned long long)zone->max_mem >> 10);
 	}
 	ttm_page_alloc_init(glob, glob->zone_kernel->max_mem/(2*PAGE_SIZE));
@@ -523,7 +523,7 @@ static void ttm_mem_global_free_zone(struct ttm_mem_global *glob,
 void ttm_mem_global_free(struct ttm_mem_global *glob,
 			 uint64_t amount)
 {
-	return ttm_mem_global_free_zone(glob, NULL, amount);
+	return ttm_mem_global_free_zone(glob, glob->zone_kernel, amount);
 }
 EXPORT_SYMBOL(ttm_mem_global_free);
 
@@ -622,10 +622,10 @@ int ttm_mem_global_alloc(struct ttm_mem_global *glob, uint64_t memory,
 {
 	/**
 	 * Normal allocations of kernel memory are registered in
-	 * all zones.
+	 * the kernel zone.
 	 */
 
-	return ttm_mem_global_alloc_zone(glob, NULL, memory, ctx);
+	return ttm_mem_global_alloc_zone(glob, glob->zone_kernel, memory, ctx);
 }
 EXPORT_SYMBOL(ttm_mem_global_alloc);
 
diff --git a/drivers/gpu/drm/tve200/tve200_display.c b/drivers/gpu/drm/tve200/tve200_display.c
index e8723a2412a6..d775d10dbe6a 100644
--- a/drivers/gpu/drm/tve200/tve200_display.c
+++ b/drivers/gpu/drm/tve200/tve200_display.c
@@ -149,7 +149,8 @@ static void tve200_display_enable(struct drm_simple_display_pipe *pipe,
 	/* Vsync IRQ at start of Vsync at first */
 	ctrl1 |= TVE200_VSTSTYPE_VSYNC;
 
-	if (connector->display_info.bus_flags & DRM_BUS_FLAG_PIXDATA_NEGEDGE)
+	if (connector->display_info.bus_flags &
+	    DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE)
 		ctrl1 |= TVE200_CTRL_TVCLKP;
 
 	if ((mode->hdisplay == 352 && mode->vdisplay == 240) || /* SIF(525) */
diff --git a/drivers/gpu/drm/udl/udl_drv.c b/drivers/gpu/drm/udl/udl_drv.c
index ff47f890e6ad..312bf324841a 100644
--- a/drivers/gpu/drm/udl/udl_drv.c
+++ b/drivers/gpu/drm/udl/udl_drv.c
@@ -48,10 +48,16 @@ static const struct file_operations udl_driver_fops = {
 	.llseek = noop_llseek,
 };
 
+static void udl_driver_release(struct drm_device *dev)
+{
+	udl_fini(dev);
+	udl_modeset_cleanup(dev);
+	drm_dev_fini(dev);
+	kfree(dev);
+}
+
 static struct drm_driver driver = {
 	.driver_features = DRIVER_MODESET | DRIVER_GEM | DRIVER_PRIME,
-	.load = udl_driver_load,
-	.unload = udl_driver_unload,
 	.release = udl_driver_release,
 
 	/* gem hooks */
@@ -75,28 +81,56 @@ static struct drm_driver driver = {
 	.patchlevel = DRIVER_PATCHLEVEL,
 };
 
+static struct udl_device *udl_driver_create(struct usb_interface *interface)
+{
+	struct usb_device *udev = interface_to_usbdev(interface);
+	struct udl_device *udl;
+	int r;
+
+	udl = kzalloc(sizeof(*udl), GFP_KERNEL);
+	if (!udl)
+		return ERR_PTR(-ENOMEM);
+
+	r = drm_dev_init(&udl->drm, &driver, &interface->dev);
+	if (r) {
+		kfree(udl);
+		return ERR_PTR(r);
+	}
+
+	udl->udev = udev;
+	udl->drm.dev_private = udl;
+
+	r = udl_init(udl);
+	if (r) {
+		drm_dev_fini(&udl->drm);
+		kfree(udl);
+		return ERR_PTR(r);
+	}
+
+	usb_set_intfdata(interface, udl);
+	return udl;
+}
+
 static int udl_usb_probe(struct usb_interface *interface,
 			 const struct usb_device_id *id)
 {
-	struct usb_device *udev = interface_to_usbdev(interface);
-	struct drm_device *dev;
 	int r;
+	struct udl_device *udl;
 
-	dev = drm_dev_alloc(&driver, &interface->dev);
-	if (IS_ERR(dev))
-		return PTR_ERR(dev);
+	udl = udl_driver_create(interface);
+	if (IS_ERR(udl))
+		return PTR_ERR(udl);
 
-	r = drm_dev_register(dev, (unsigned long)udev);
+	r = drm_dev_register(&udl->drm, 0);
 	if (r)
 		goto err_free;
 
-	usb_set_intfdata(interface, dev);
-	DRM_INFO("Initialized udl on minor %d\n", dev->primary->index);
+	DRM_INFO("Initialized udl on minor %d\n", udl->drm.primary->index);
 
 	return 0;
 
 err_free:
-	drm_dev_put(dev);
+	drm_dev_put(&udl->drm);
 	return r;
 }
 
@@ -108,6 +142,7 @@ static void udl_usb_disconnect(struct usb_interface *interface)
 	udl_fbdev_unplug(dev);
 	udl_drop_usb(dev);
 	drm_dev_unplug(dev);
+	drm_dev_put(dev);
 }
 
 /*
diff --git a/drivers/gpu/drm/udl/udl_drv.h b/drivers/gpu/drm/udl/udl_drv.h
index 4ae67d882eae..35c1f33fbc1a 100644
--- a/drivers/gpu/drm/udl/udl_drv.h
+++ b/drivers/gpu/drm/udl/udl_drv.h
@@ -50,8 +50,8 @@ struct urb_list {
 struct udl_fbdev;
 
 struct udl_device {
+	struct drm_device drm;
 	struct device *dev;
-	struct drm_device *ddev;
 	struct usb_device *udev;
 	struct drm_crtc *crtc;
 
@@ -71,6 +71,8 @@ struct udl_device {
 	atomic_t cpu_kcycles_used; /* transpired during pixel processing */
 };
 
+#define to_udl(x) container_of(x, struct udl_device, drm)
+
 struct udl_gem_object {
 	struct drm_gem_object base;
 	struct page **pages;
@@ -102,9 +104,8 @@ struct urb *udl_get_urb(struct drm_device *dev);
 int udl_submit_urb(struct drm_device *dev, struct urb *urb, size_t len);
 void udl_urb_completion(struct urb *urb);
 
-int udl_driver_load(struct drm_device *dev, unsigned long flags);
-void udl_driver_unload(struct drm_device *dev);
-void udl_driver_release(struct drm_device *dev);
+int udl_init(struct udl_device *udl);
+void udl_fini(struct drm_device *dev);
 
 int udl_fbdev_init(struct drm_device *dev);
 void udl_fbdev_cleanup(struct drm_device *dev);
diff --git a/drivers/gpu/drm/udl/udl_fb.c b/drivers/gpu/drm/udl/udl_fb.c
index dd9ffded223b..b9b67a546d4c 100644
--- a/drivers/gpu/drm/udl/udl_fb.c
+++ b/drivers/gpu/drm/udl/udl_fb.c
@@ -32,7 +32,7 @@ module_param(fb_bpp, int, S_IWUSR | S_IRUSR | S_IWGRP | S_IRGRP);
 module_param(fb_defio, int, S_IWUSR | S_IRUSR | S_IWGRP | S_IRGRP);
 
 struct udl_fbdev {
-	struct drm_fb_helper helper;
+	struct drm_fb_helper helper; /* must be first */
 	struct udl_framebuffer ufb;
 	int fb_count;
 };
@@ -82,7 +82,7 @@ int udl_handle_damage(struct udl_framebuffer *fb, int x, int y,
 		      int width, int height)
 {
 	struct drm_device *dev = fb->base.dev;
-	struct udl_device *udl = dev->dev_private;
+	struct udl_device *udl = to_udl(dev);
 	int i, ret;
 	char *cmd;
 	cycles_t start_cycles, end_cycles;
@@ -210,10 +210,10 @@ static int udl_fb_open(struct fb_info *info, int user)
 {
 	struct udl_fbdev *ufbdev = info->par;
 	struct drm_device *dev = ufbdev->ufb.base.dev;
-	struct udl_device *udl = dev->dev_private;
+	struct udl_device *udl = to_udl(dev);
 
 	/* If the USB device is gone, we don't accept new opens */
-	if (drm_dev_is_unplugged(udl->ddev))
+	if (drm_dev_is_unplugged(&udl->drm))
 		return -ENODEV;
 
 	ufbdev->fb_count++;
@@ -392,7 +392,6 @@ static int udlfb_create(struct drm_fb_helper *helper,
 		ret = PTR_ERR(info);
 		goto out_gfree;
 	}
-	info->par = ufbdev;
 
 	ret = udl_framebuffer_init(dev, &ufbdev->ufb, &mode_cmd, obj);
 	if (ret)
@@ -402,15 +401,12 @@ static int udlfb_create(struct drm_fb_helper *helper,
 
 	ufbdev->helper.fb = fb;
 
-	strcpy(info->fix.id, "udldrmfb");
-
 	info->screen_base = ufbdev->ufb.obj->vmapping;
 	info->fix.smem_len = size;
 	info->fix.smem_start = (unsigned long)ufbdev->ufb.obj->vmapping;
 
 	info->fbops = &udlfb_ops;
-	drm_fb_helper_fill_fix(info, fb->pitches[0], fb->format->depth);
-	drm_fb_helper_fill_var(info, &ufbdev->helper, sizes->fb_width, sizes->fb_height);
+	drm_fb_helper_fill_info(info, &ufbdev->helper, sizes);
 
 	DRM_DEBUG_KMS("allocated %dx%d vmal %p\n",
 		      fb->width, fb->height,
@@ -441,7 +437,7 @@ static void udl_fbdev_destroy(struct drm_device *dev,
 
 int udl_fbdev_init(struct drm_device *dev)
 {
-	struct udl_device *udl = dev->dev_private;
+	struct udl_device *udl = to_udl(dev);
 	int bpp_sel = fb_bpp;
 	struct udl_fbdev *ufbdev;
 	int ret;
@@ -480,7 +476,7 @@ free:
 
 void udl_fbdev_cleanup(struct drm_device *dev)
 {
-	struct udl_device *udl = dev->dev_private;
+	struct udl_device *udl = to_udl(dev);
 	if (!udl->fbdev)
 		return;
 
@@ -491,7 +487,7 @@ void udl_fbdev_cleanup(struct drm_device *dev)
 
 void udl_fbdev_unplug(struct drm_device *dev)
 {
-	struct udl_device *udl = dev->dev_private;
+	struct udl_device *udl = to_udl(dev);
 	struct udl_fbdev *ufbdev;
 	if (!udl->fbdev)
 		return;
diff --git a/drivers/gpu/drm/udl/udl_gem.c b/drivers/gpu/drm/udl/udl_gem.c
index bb7b58407039..3b3e17652bb2 100644
--- a/drivers/gpu/drm/udl/udl_gem.c
+++ b/drivers/gpu/drm/udl/udl_gem.c
@@ -203,7 +203,7 @@ int udl_gem_mmap(struct drm_file *file, struct drm_device *dev,
 {
 	struct udl_gem_object *gobj;
 	struct drm_gem_object *obj;
-	struct udl_device *udl = dev->dev_private;
+	struct udl_device *udl = to_udl(dev);
 	int ret = 0;
 
 	mutex_lock(&udl->gem_lock);
diff --git a/drivers/gpu/drm/udl/udl_main.c b/drivers/gpu/drm/udl/udl_main.c
index 1f8ef34ade24..6743eaef4594 100644
--- a/drivers/gpu/drm/udl/udl_main.c
+++ b/drivers/gpu/drm/udl/udl_main.c
@@ -30,7 +30,7 @@
 static int udl_parse_vendor_descriptor(struct drm_device *dev,
 				       struct usb_device *usbdev)
 {
-	struct udl_device *udl = dev->dev_private;
+	struct udl_device *udl = to_udl(dev);
 	char *desc;
 	char *buf;
 	char *desc_end;
@@ -166,7 +166,7 @@ void udl_urb_completion(struct urb *urb)
 
 static void udl_free_urb_list(struct drm_device *dev)
 {
-	struct udl_device *udl = dev->dev_private;
+	struct udl_device *udl = to_udl(dev);
 	int count = udl->urbs.count;
 	struct list_head *node;
 	struct urb_node *unode;
@@ -199,7 +199,7 @@ static void udl_free_urb_list(struct drm_device *dev)
 
 static int udl_alloc_urb_list(struct drm_device *dev, int count, size_t size)
 {
-	struct udl_device *udl = dev->dev_private;
+	struct udl_device *udl = to_udl(dev);
 	struct urb *urb;
 	struct urb_node *unode;
 	char *buf;
@@ -263,7 +263,7 @@ retry:
 
 struct urb *udl_get_urb(struct drm_device *dev)
 {
-	struct udl_device *udl = dev->dev_private;
+	struct udl_device *udl = to_udl(dev);
 	int ret = 0;
 	struct list_head *entry;
 	struct urb_node *unode;
@@ -296,7 +296,7 @@ error:
 
 int udl_submit_urb(struct drm_device *dev, struct urb *urb, size_t len)
 {
-	struct udl_device *udl = dev->dev_private;
+	struct udl_device *udl = to_udl(dev);
 	int ret;
 
 	BUG_ON(len > udl->urbs.size);
@@ -311,20 +311,12 @@ int udl_submit_urb(struct drm_device *dev, struct urb *urb, size_t len)
 	return ret;
 }
 
-int udl_driver_load(struct drm_device *dev, unsigned long flags)
+int udl_init(struct udl_device *udl)
 {
-	struct usb_device *udev = (void*)flags;
-	struct udl_device *udl;
+	struct drm_device *dev = &udl->drm;
 	int ret = -ENOMEM;
 
 	DRM_DEBUG("\n");
-	udl = kzalloc(sizeof(struct udl_device), GFP_KERNEL);
-	if (!udl)
-		return -ENOMEM;
-
-	udl->udev = udev;
-	udl->ddev = dev;
-	dev->dev_private = udl;
 
 	mutex_init(&udl->gem_lock);
 
@@ -358,7 +350,6 @@ int udl_driver_load(struct drm_device *dev, unsigned long flags)
 err:
 	if (udl->urbs.count)
 		udl_free_urb_list(dev);
-	kfree(udl);
 	DRM_ERROR("%d\n", ret);
 	return ret;
 }
@@ -369,9 +360,9 @@ int udl_drop_usb(struct drm_device *dev)
 	return 0;
 }
 
-void udl_driver_unload(struct drm_device *dev)
+void udl_fini(struct drm_device *dev)
 {
-	struct udl_device *udl = dev->dev_private;
+	struct udl_device *udl = to_udl(dev);
 
 	drm_kms_helper_poll_fini(dev);
 
@@ -379,12 +370,4 @@ void udl_driver_unload(struct drm_device *dev)
 		udl_free_urb_list(dev);
 
 	udl_fbdev_cleanup(dev);
-	kfree(udl);
-}
-
-void udl_driver_release(struct drm_device *dev)
-{
-	udl_modeset_cleanup(dev);
-	drm_dev_fini(dev);
-	kfree(dev);
 }
diff --git a/drivers/gpu/drm/v3d/Kconfig b/drivers/gpu/drm/v3d/Kconfig
index 1552bf552c94..75a74c45f109 100644
--- a/drivers/gpu/drm/v3d/Kconfig
+++ b/drivers/gpu/drm/v3d/Kconfig
@@ -5,6 +5,7 @@ config DRM_V3D
 	depends on COMMON_CLK
 	depends on MMU
 	select DRM_SCHED
+	select DRM_GEM_SHMEM_HELPER
 	help
 	  Choose this option if you have a system that has a Broadcom
 	  V3D 3.x or newer GPU, such as BCM7268.
diff --git a/drivers/gpu/drm/v3d/v3d_bo.c b/drivers/gpu/drm/v3d/v3d_bo.c
index a08766d39eab..a22b75a3a533 100644
--- a/drivers/gpu/drm/v3d/v3d_bo.c
+++ b/drivers/gpu/drm/v3d/v3d_bo.c
@@ -25,162 +25,6 @@
 #include "v3d_drv.h"
 #include "uapi/drm/v3d_drm.h"
 
-/* Pins the shmem pages, fills in the .pages and .sgt fields of the BO, and maps
- * it for DMA.
- */
-static int
-v3d_bo_get_pages(struct v3d_bo *bo)
-{
-	struct drm_gem_object *obj = &bo->base;
-	struct drm_device *dev = obj->dev;
-	int npages = obj->size >> PAGE_SHIFT;
-	int ret = 0;
-
-	mutex_lock(&bo->lock);
-	if (bo->pages_refcount++ != 0)
-		goto unlock;
-
-	if (!obj->import_attach) {
-		bo->pages = drm_gem_get_pages(obj);
-		if (IS_ERR(bo->pages)) {
-			ret = PTR_ERR(bo->pages);
-			goto unlock;
-		}
-
-		bo->sgt = drm_prime_pages_to_sg(bo->pages, npages);
-		if (IS_ERR(bo->sgt)) {
-			ret = PTR_ERR(bo->sgt);
-			goto put_pages;
-		}
-
-		/* Map the pages for use by the GPU. */
-		dma_map_sg(dev->dev, bo->sgt->sgl,
-			   bo->sgt->nents, DMA_BIDIRECTIONAL);
-	} else {
-		bo->pages = kcalloc(npages, sizeof(*bo->pages), GFP_KERNEL);
-		if (!bo->pages)
-			goto put_pages;
-
-		drm_prime_sg_to_page_addr_arrays(bo->sgt, bo->pages,
-						 NULL, npages);
-
-		/* Note that dma-bufs come in mapped. */
-	}
-
-	mutex_unlock(&bo->lock);
-
-	return 0;
-
-put_pages:
-	drm_gem_put_pages(obj, bo->pages, true, true);
-	bo->pages = NULL;
-unlock:
-	bo->pages_refcount--;
-	mutex_unlock(&bo->lock);
-	return ret;
-}
-
-static void
-v3d_bo_put_pages(struct v3d_bo *bo)
-{
-	struct drm_gem_object *obj = &bo->base;
-
-	mutex_lock(&bo->lock);
-	if (--bo->pages_refcount == 0) {
-		if (!obj->import_attach) {
-			dma_unmap_sg(obj->dev->dev, bo->sgt->sgl,
-				     bo->sgt->nents, DMA_BIDIRECTIONAL);
-			sg_free_table(bo->sgt);
-			kfree(bo->sgt);
-			drm_gem_put_pages(obj, bo->pages, true, true);
-		} else {
-			kfree(bo->pages);
-		}
-	}
-	mutex_unlock(&bo->lock);
-}
-
-static struct v3d_bo *v3d_bo_create_struct(struct drm_device *dev,
-					   size_t unaligned_size)
-{
-	struct v3d_dev *v3d = to_v3d_dev(dev);
-	struct drm_gem_object *obj;
-	struct v3d_bo *bo;
-	size_t size = roundup(unaligned_size, PAGE_SIZE);
-	int ret;
-
-	if (size == 0)
-		return ERR_PTR(-EINVAL);
-
-	bo = kzalloc(sizeof(*bo), GFP_KERNEL);
-	if (!bo)
-		return ERR_PTR(-ENOMEM);
-	obj = &bo->base;
-
-	INIT_LIST_HEAD(&bo->vmas);
-	INIT_LIST_HEAD(&bo->unref_head);
-	mutex_init(&bo->lock);
-
-	ret = drm_gem_object_init(dev, obj, size);
-	if (ret)
-		goto free_bo;
-
-	spin_lock(&v3d->mm_lock);
-	ret = drm_mm_insert_node_generic(&v3d->mm, &bo->node,
-					 obj->size >> PAGE_SHIFT,
-					 GMP_GRANULARITY >> PAGE_SHIFT, 0, 0);
-	spin_unlock(&v3d->mm_lock);
-	if (ret)
-		goto free_obj;
-
-	return bo;
-
-free_obj:
-	drm_gem_object_release(obj);
-free_bo:
-	kfree(bo);
-	return ERR_PTR(ret);
-}
-
-struct v3d_bo *v3d_bo_create(struct drm_device *dev, struct drm_file *file_priv,
-			     size_t unaligned_size)
-{
-	struct v3d_dev *v3d = to_v3d_dev(dev);
-	struct drm_gem_object *obj;
-	struct v3d_bo *bo;
-	int ret;
-
-	bo = v3d_bo_create_struct(dev, unaligned_size);
-	if (IS_ERR(bo))
-		return bo;
-	obj = &bo->base;
-
-	bo->resv = &bo->_resv;
-	reservation_object_init(bo->resv);
-
-	ret = v3d_bo_get_pages(bo);
-	if (ret)
-		goto free_mm;
-
-	v3d_mmu_insert_ptes(bo);
-
-	mutex_lock(&v3d->bo_lock);
-	v3d->bo_stats.num_allocated++;
-	v3d->bo_stats.pages_allocated += obj->size >> PAGE_SHIFT;
-	mutex_unlock(&v3d->bo_lock);
-
-	return bo;
-
-free_mm:
-	spin_lock(&v3d->mm_lock);
-	drm_mm_remove_node(&bo->node);
-	spin_unlock(&v3d->mm_lock);
-
-	drm_gem_object_release(obj);
-	kfree(bo);
-	return ERR_PTR(ret);
-}
-
 /* Called DRM core on the last userspace/kernel unreference of the
  * BO.
  */
@@ -189,92 +33,116 @@ void v3d_free_object(struct drm_gem_object *obj)
 	struct v3d_dev *v3d = to_v3d_dev(obj->dev);
 	struct v3d_bo *bo = to_v3d_bo(obj);
 
+	v3d_mmu_remove_ptes(bo);
+
 	mutex_lock(&v3d->bo_lock);
 	v3d->bo_stats.num_allocated--;
 	v3d->bo_stats.pages_allocated -= obj->size >> PAGE_SHIFT;
 	mutex_unlock(&v3d->bo_lock);
 
-	reservation_object_fini(&bo->_resv);
-
-	v3d_bo_put_pages(bo);
-
-	if (obj->import_attach)
-		drm_prime_gem_destroy(obj, bo->sgt);
-
-	v3d_mmu_remove_ptes(bo);
 	spin_lock(&v3d->mm_lock);
 	drm_mm_remove_node(&bo->node);
 	spin_unlock(&v3d->mm_lock);
 
-	mutex_destroy(&bo->lock);
+	/* GPU execution may have dirtied any pages in the BO. */
+	bo->base.pages_mark_dirty_on_put = true;
 
-	drm_gem_object_release(obj);
-	kfree(bo);
+	drm_gem_shmem_free_object(obj);
 }
 
-struct reservation_object *v3d_prime_res_obj(struct drm_gem_object *obj)
+static const struct drm_gem_object_funcs v3d_gem_funcs = {
+	.free = v3d_free_object,
+	.print_info = drm_gem_shmem_print_info,
+	.pin = drm_gem_shmem_pin,
+	.unpin = drm_gem_shmem_unpin,
+	.get_sg_table = drm_gem_shmem_get_sg_table,
+	.vmap = drm_gem_shmem_vmap,
+	.vunmap = drm_gem_shmem_vunmap,
+	.vm_ops = &drm_gem_shmem_vm_ops,
+};
+
+/* gem_create_object function for allocating a BO struct and doing
+ * early setup.
+ */
+struct drm_gem_object *v3d_create_object(struct drm_device *dev, size_t size)
 {
-	struct v3d_bo *bo = to_v3d_bo(obj);
+	struct v3d_bo *bo;
+	struct drm_gem_object *obj;
 
-	return bo->resv;
-}
+	if (size == 0)
+		return NULL;
 
-static void
-v3d_set_mmap_vma_flags(struct vm_area_struct *vma)
-{
-	vma->vm_flags &= ~VM_PFNMAP;
-	vma->vm_flags |= VM_MIXEDMAP;
-	vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
-}
+	bo = kzalloc(sizeof(*bo), GFP_KERNEL);
+	if (!bo)
+		return NULL;
+	obj = &bo->base.base;
 
-vm_fault_t v3d_gem_fault(struct vm_fault *vmf)
-{
-	struct vm_area_struct *vma = vmf->vma;
-	struct drm_gem_object *obj = vma->vm_private_data;
-	struct v3d_bo *bo = to_v3d_bo(obj);
-	pfn_t pfn;
-	pgoff_t pgoff;
+	obj->funcs = &v3d_gem_funcs;
 
-	/* We don't use vmf->pgoff since that has the fake offset: */
-	pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
-	pfn = __pfn_to_pfn_t(page_to_pfn(bo->pages[pgoff]), PFN_DEV);
+	INIT_LIST_HEAD(&bo->unref_head);
 
-	return vmf_insert_mixed(vma, vmf->address, pfn);
+	return &bo->base.base;
 }
 
-int v3d_mmap(struct file *filp, struct vm_area_struct *vma)
+static int
+v3d_bo_create_finish(struct drm_gem_object *obj)
 {
+	struct v3d_dev *v3d = to_v3d_dev(obj->dev);
+	struct v3d_bo *bo = to_v3d_bo(obj);
+	struct sg_table *sgt;
 	int ret;
 
-	ret = drm_gem_mmap(filp, vma);
+	/* So far we pin the BO in the MMU for its lifetime, so use
+	 * shmem's helper for getting a lifetime sgt.
+	 */
+	sgt = drm_gem_shmem_get_pages_sgt(&bo->base.base);
+	if (IS_ERR(sgt))
+		return PTR_ERR(sgt);
+
+	spin_lock(&v3d->mm_lock);
+	/* Allocate the object's space in the GPU's page tables.
+	 * Inserting PTEs will happen later, but the offset is for the
+	 * lifetime of the BO.
+	 */
+	ret = drm_mm_insert_node_generic(&v3d->mm, &bo->node,
+					 obj->size >> PAGE_SHIFT,
+					 GMP_GRANULARITY >> PAGE_SHIFT, 0, 0);
+	spin_unlock(&v3d->mm_lock);
 	if (ret)
 		return ret;
 
-	v3d_set_mmap_vma_flags(vma);
+	/* Track stats for /debug/dri/n/bo_stats. */
+	mutex_lock(&v3d->bo_lock);
+	v3d->bo_stats.num_allocated++;
+	v3d->bo_stats.pages_allocated += obj->size >> PAGE_SHIFT;
+	mutex_unlock(&v3d->bo_lock);
 
-	return ret;
+	v3d_mmu_insert_ptes(bo);
+
+	return 0;
 }
 
-int v3d_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
+struct v3d_bo *v3d_bo_create(struct drm_device *dev, struct drm_file *file_priv,
+			     size_t unaligned_size)
 {
+	struct drm_gem_shmem_object *shmem_obj;
+	struct v3d_bo *bo;
 	int ret;
 
-	ret = drm_gem_mmap_obj(obj, obj->size, vma);
-	if (ret < 0)
-		return ret;
-
-	v3d_set_mmap_vma_flags(vma);
+	shmem_obj = drm_gem_shmem_create(dev, unaligned_size);
+	if (IS_ERR(shmem_obj))
+		return ERR_CAST(shmem_obj);
+	bo = to_v3d_bo(&shmem_obj->base);
 
-	return 0;
-}
+	ret = v3d_bo_create_finish(&shmem_obj->base);
+	if (ret)
+		goto free_obj;
 
-struct sg_table *
-v3d_prime_get_sg_table(struct drm_gem_object *obj)
-{
-	struct v3d_bo *bo = to_v3d_bo(obj);
-	int npages = obj->size >> PAGE_SHIFT;
+	return bo;
 
-	return drm_prime_pages_to_sg(bo->pages, npages);
+free_obj:
+	drm_gem_shmem_free_object(&shmem_obj->base);
+	return ERR_PTR(ret);
 }
 
 struct drm_gem_object *
@@ -283,20 +151,17 @@ v3d_prime_import_sg_table(struct drm_device *dev,
 			  struct sg_table *sgt)
 {
 	struct drm_gem_object *obj;
-	struct v3d_bo *bo;
-
-	bo = v3d_bo_create_struct(dev, attach->dmabuf->size);
-	if (IS_ERR(bo))
-		return ERR_CAST(bo);
-	obj = &bo->base;
-
-	bo->resv = attach->dmabuf->resv;
+	int ret;
 
-	bo->sgt = sgt;
-	obj->import_attach = attach;
-	v3d_bo_get_pages(bo);
+	obj = drm_gem_shmem_prime_import_sg_table(dev, attach, sgt);
+	if (IS_ERR(obj))
+		return obj;
 
-	v3d_mmu_insert_ptes(bo);
+	ret = v3d_bo_create_finish(obj);
+	if (ret) {
+		drm_gem_shmem_free_object(obj);
+		return ERR_PTR(ret);
+	}
 
 	return obj;
 }
@@ -319,8 +184,8 @@ int v3d_create_bo_ioctl(struct drm_device *dev, void *data,
 
 	args->offset = bo->node.start << PAGE_SHIFT;
 
-	ret = drm_gem_handle_create(file_priv, &bo->base, &args->handle);
-	drm_gem_object_put_unlocked(&bo->base);
+	ret = drm_gem_handle_create(file_priv, &bo->base.base, &args->handle);
+	drm_gem_object_put_unlocked(&bo->base.base);
 
 	return ret;
 }
@@ -330,7 +195,6 @@ int v3d_mmap_bo_ioctl(struct drm_device *dev, void *data,
 {
 	struct drm_v3d_mmap_bo *args = data;
 	struct drm_gem_object *gem_obj;
-	int ret;
 
 	if (args->flags != 0) {
 		DRM_INFO("unknown mmap_bo flags: %d\n", args->flags);
@@ -343,12 +207,10 @@ int v3d_mmap_bo_ioctl(struct drm_device *dev, void *data,
 		return -ENOENT;
 	}
 
-	ret = drm_gem_create_mmap_offset(gem_obj);
-	if (ret == 0)
-		args->offset = drm_vma_node_offset_addr(&gem_obj->vma_node);
+	args->offset = drm_vma_node_offset_addr(&gem_obj->vma_node);
 	drm_gem_object_put_unlocked(gem_obj);
 
-	return ret;
+	return 0;
 }
 
 int v3d_get_bo_offset_ioctl(struct drm_device *dev, void *data,
diff --git a/drivers/gpu/drm/v3d/v3d_debugfs.c b/drivers/gpu/drm/v3d/v3d_debugfs.c
index eb2b2d2f8553..a24af2d2f574 100644
--- a/drivers/gpu/drm/v3d/v3d_debugfs.c
+++ b/drivers/gpu/drm/v3d/v3d_debugfs.c
@@ -187,6 +187,11 @@ static int v3d_measure_clock(struct seq_file *m, void *unused)
 	uint32_t cycles;
 	int core = 0;
 	int measure_ms = 1000;
+	int ret;
+
+	ret = pm_runtime_get_sync(v3d->dev);
+	if (ret < 0)
+		return ret;
 
 	if (v3d->ver >= 40) {
 		V3D_CORE_WRITE(core, V3D_V4_PCTR_0_SRC_0_3,
@@ -210,6 +215,9 @@ static int v3d_measure_clock(struct seq_file *m, void *unused)
 		   cycles / (measure_ms * 1000),
 		   (cycles / (measure_ms * 100)) % 10);
 
+	pm_runtime_mark_last_busy(v3d->dev);
+	pm_runtime_put_autosuspend(v3d->dev);
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/v3d/v3d_drv.c b/drivers/gpu/drm/v3d/v3d_drv.c
index f0afcec72c34..a06b05f714a5 100644
--- a/drivers/gpu/drm/v3d/v3d_drv.c
+++ b/drivers/gpu/drm/v3d/v3d_drv.c
@@ -7,9 +7,9 @@
  * This driver supports the Broadcom V3D 3.3 and 4.1 OpenGL ES GPUs.
  * For V3D 2.x support, see the VC4 driver.
  *
- * Currently only single-core rendering using the binner and renderer
- * is supported.  The TFU (texture formatting unit) and V3D 4.x's CSD
- * (compute shader dispatch) are not yet supported.
+ * Currently only single-core rendering using the binner and renderer,
+ * along with TFU (texture formatting unit) rendering is supported.
+ * V3D 4.x's CSD (compute shader dispatch) is not yet supported.
  */
 
 #include <linux/clk.h>
@@ -19,6 +19,7 @@
 #include <linux/of_platform.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
+#include <linux/reset.h>
 #include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_fb_helper.h>
 
@@ -101,6 +102,8 @@ static int v3d_get_param_ioctl(struct drm_device *dev, void *data,
 			return -EINVAL;
 
 		ret = pm_runtime_get_sync(v3d->dev);
+		if (ret < 0)
+			return ret;
 		if (args->param >= DRM_V3D_PARAM_V3D_CORE0_IDENT0 &&
 		    args->param <= DRM_V3D_PARAM_V3D_CORE0_IDENT2) {
 			args->value = V3D_CORE_READ(0, offset);
@@ -160,17 +163,7 @@ v3d_postclose(struct drm_device *dev, struct drm_file *file)
 	kfree(v3d_priv);
 }
 
-static const struct file_operations v3d_drm_fops = {
-	.owner = THIS_MODULE,
-	.open = drm_open,
-	.release = drm_release,
-	.unlocked_ioctl = drm_ioctl,
-	.mmap = v3d_mmap,
-	.poll = drm_poll,
-	.read = drm_read,
-	.compat_ioctl = drm_compat_ioctl,
-	.llseek = noop_llseek,
-};
+DEFINE_DRM_GEM_SHMEM_FOPS(v3d_drm_fops);
 
 /* DRM_AUTH is required on SUBMIT_CL for now, while we don't have GMP
  * protection between clients.  Note that render nodes would be be
@@ -188,12 +181,6 @@ static const struct drm_ioctl_desc v3d_drm_ioctls[] = {
 	DRM_IOCTL_DEF_DRV(V3D_SUBMIT_TFU, v3d_submit_tfu_ioctl, DRM_RENDER_ALLOW | DRM_AUTH),
 };
 
-static const struct vm_operations_struct v3d_vm_ops = {
-	.fault = v3d_gem_fault,
-	.open = drm_gem_vm_open,
-	.close = drm_gem_vm_close,
-};
-
 static struct drm_driver v3d_drm_driver = {
 	.driver_features = (DRIVER_GEM |
 			    DRIVER_RENDER |
@@ -207,17 +194,11 @@ static struct drm_driver v3d_drm_driver = {
 	.debugfs_init = v3d_debugfs_init,
 #endif
 
-	.gem_free_object_unlocked = v3d_free_object,
-	.gem_vm_ops = &v3d_vm_ops,
-
+	.gem_create_object = v3d_create_object,
 	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
 	.prime_fd_to_handle = drm_gem_prime_fd_to_handle,
-	.gem_prime_import = drm_gem_prime_import,
-	.gem_prime_export = drm_gem_prime_export,
-	.gem_prime_res_obj = v3d_prime_res_obj,
-	.gem_prime_get_sg_table	= v3d_prime_get_sg_table,
 	.gem_prime_import_sg_table = v3d_prime_import_sg_table,
-	.gem_prime_mmap = v3d_prime_mmap,
+	.gem_prime_mmap = drm_gem_prime_mmap,
 
 	.ioctls = v3d_drm_ioctls,
 	.num_ioctls = ARRAY_SIZE(v3d_drm_ioctls),
@@ -265,10 +246,6 @@ static int v3d_platform_drm_probe(struct platform_device *pdev)
 	v3d->pdev = pdev;
 	drm = &v3d->drm;
 
-	ret = map_regs(v3d, &v3d->bridge_regs, "bridge");
-	if (ret)
-		goto dev_free;
-
 	ret = map_regs(v3d, &v3d->hub_regs, "hub");
 	if (ret)
 		goto dev_free;
@@ -283,6 +260,22 @@ static int v3d_platform_drm_probe(struct platform_device *pdev)
 	v3d->cores = V3D_GET_FIELD(ident1, V3D_HUB_IDENT1_NCORES);
 	WARN_ON(v3d->cores > 1); /* multicore not yet implemented */
 
+	v3d->reset = devm_reset_control_get_exclusive(dev, NULL);
+	if (IS_ERR(v3d->reset)) {
+		ret = PTR_ERR(v3d->reset);
+
+		if (ret == -EPROBE_DEFER)
+			goto dev_free;
+
+		v3d->reset = NULL;
+		ret = map_regs(v3d, &v3d->bridge_regs, "bridge");
+		if (ret) {
+			dev_err(dev,
+				"Failed to get reset control or bridge regs\n");
+			goto dev_free;
+		}
+	}
+
 	if (v3d->ver < 41) {
 		ret = map_regs(v3d, &v3d->gca_regs, "gca");
 		if (ret)
@@ -312,14 +305,18 @@ static int v3d_platform_drm_probe(struct platform_device *pdev)
 	if (ret)
 		goto dev_destroy;
 
-	v3d_irq_init(v3d);
+	ret = v3d_irq_init(v3d);
+	if (ret)
+		goto gem_destroy;
 
 	ret = drm_dev_register(drm, 0);
 	if (ret)
-		goto gem_destroy;
+		goto irq_disable;
 
 	return 0;
 
+irq_disable:
+	v3d_irq_disable(v3d);
 gem_destroy:
 	v3d_gem_destroy(drm);
 dev_destroy:
diff --git a/drivers/gpu/drm/v3d/v3d_drv.h b/drivers/gpu/drm/v3d/v3d_drv.h
index fdda3037f7af..e9d4a2fdcf44 100644
--- a/drivers/gpu/drm/v3d/v3d_drv.h
+++ b/drivers/gpu/drm/v3d/v3d_drv.h
@@ -1,11 +1,11 @@
 // SPDX-License-Identifier: GPL-2.0+
 /* Copyright (C) 2015-2018 Broadcom */
 
-#include <linux/reservation.h>
 #include <linux/mm_types.h>
 #include <drm/drmP.h>
 #include <drm/drm_encoder.h>
 #include <drm/drm_gem.h>
+#include <drm/drm_gem_shmem_helper.h>
 #include <drm/gpu_scheduler.h>
 #include "uapi/drm/v3d_drm.h"
 
@@ -34,6 +34,7 @@ struct v3d_dev {
 	 * and revision.
 	 */
 	int ver;
+	bool single_irq_line;
 
 	struct device *dev;
 	struct platform_device *pdev;
@@ -42,6 +43,7 @@ struct v3d_dev {
 	void __iomem *bridge_regs;
 	void __iomem *gca_regs;
 	struct clk *clk;
+	struct reset_control *reset;
 
 	/* Virtual and DMA addresses of the single shared page table. */
 	volatile u32 *pt;
@@ -109,34 +111,15 @@ struct v3d_file_priv {
 	struct drm_sched_entity sched_entity[V3D_MAX_QUEUES];
 };
 
-/* Tracks a mapping of a BO into a per-fd address space */
-struct v3d_vma {
-	struct v3d_page_table *pt;
-	struct list_head list; /* entry in v3d_bo.vmas */
-};
-
 struct v3d_bo {
-	struct drm_gem_object base;
-
-	struct mutex lock;
+	struct drm_gem_shmem_object base;
 
 	struct drm_mm_node node;
 
-	u32 pages_refcount;
-	struct page **pages;
-	struct sg_table *sgt;
-	void *vaddr;
-
-	struct list_head vmas;    /* list of v3d_vma */
-
 	/* List entry for the BO's position in
 	 * v3d_exec_info->unref_list
 	 */
 	struct list_head unref_head;
-
-	/* normally (resv == &_resv) except for imported bo's */
-	struct reservation_object *resv;
-	struct reservation_object _resv;
 };
 
 static inline struct v3d_bo *
@@ -180,7 +163,7 @@ struct v3d_job {
 	struct dma_fence *in_fence;
 
 	/* v3d fence to be signaled by IRQ handler when the job is complete. */
-	struct dma_fence *done_fence;
+	struct dma_fence *irq_fence;
 
 	/* GPU virtual addresses of the start/end of the CL job. */
 	u32 start, end;
@@ -227,7 +210,7 @@ struct v3d_tfu_job {
 	struct dma_fence *in_fence;
 
 	/* v3d fence to be signaled by IRQ handler when the job is complete. */
-	struct dma_fence *done_fence;
+	struct dma_fence *irq_fence;
 
 	struct v3d_dev *v3d;
 
@@ -270,6 +253,7 @@ static inline unsigned long nsecs_to_jiffies_timeout(const u64 n)
 }
 
 /* v3d_bo.c */
+struct drm_gem_object *v3d_create_object(struct drm_device *dev, size_t size);
 void v3d_free_object(struct drm_gem_object *gem_obj);
 struct v3d_bo *v3d_bo_create(struct drm_device *dev, struct drm_file *file_priv,
 			     size_t size);
@@ -279,11 +263,6 @@ int v3d_mmap_bo_ioctl(struct drm_device *dev, void *data,
 		      struct drm_file *file_priv);
 int v3d_get_bo_offset_ioctl(struct drm_device *dev, void *data,
 			    struct drm_file *file_priv);
-vm_fault_t v3d_gem_fault(struct vm_fault *vmf);
-int v3d_mmap(struct file *filp, struct vm_area_struct *vma);
-struct reservation_object *v3d_prime_res_obj(struct drm_gem_object *obj);
-int v3d_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma);
-struct sg_table *v3d_prime_get_sg_table(struct drm_gem_object *obj);
 struct drm_gem_object *v3d_prime_import_sg_table(struct drm_device *dev,
 						 struct dma_buf_attachment *attach,
 						 struct sg_table *sgt);
@@ -310,7 +289,7 @@ void v3d_reset(struct v3d_dev *v3d);
 void v3d_invalidate_caches(struct v3d_dev *v3d);
 
 /* v3d_irq.c */
-void v3d_irq_init(struct v3d_dev *v3d);
+int v3d_irq_init(struct v3d_dev *v3d);
 void v3d_irq_enable(struct v3d_dev *v3d);
 void v3d_irq_disable(struct v3d_dev *v3d);
 void v3d_irq_reset(struct v3d_dev *v3d);
diff --git a/drivers/gpu/drm/v3d/v3d_gem.c b/drivers/gpu/drm/v3d/v3d_gem.c
index 803f31467ec1..93ff8fcbe475 100644
--- a/drivers/gpu/drm/v3d/v3d_gem.c
+++ b/drivers/gpu/drm/v3d/v3d_gem.c
@@ -6,6 +6,7 @@
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
+#include <linux/reset.h>
 #include <linux/device.h>
 #include <linux/io.h>
 #include <linux/sched/signal.h>
@@ -24,7 +25,8 @@ v3d_init_core(struct v3d_dev *v3d, int core)
 	 * type.  If you want the default behavior, you can still put
 	 * "2" in the indirect texture state's output_type field.
 	 */
-	V3D_CORE_WRITE(core, V3D_CTL_MISCCFG, V3D_MISCCFG_OVRTMUOUT);
+	if (v3d->ver < 40)
+		V3D_CORE_WRITE(core, V3D_CTL_MISCCFG, V3D_MISCCFG_OVRTMUOUT);
 
 	/* Whenever we flush the L2T cache, we always want to flush
 	 * the whole thing.
@@ -69,7 +71,7 @@ v3d_idle_gca(struct v3d_dev *v3d)
 }
 
 static void
-v3d_reset_v3d(struct v3d_dev *v3d)
+v3d_reset_by_bridge(struct v3d_dev *v3d)
 {
 	int version = V3D_BRIDGE_READ(V3D_TOP_GR_BRIDGE_REVISION);
 
@@ -89,6 +91,15 @@ v3d_reset_v3d(struct v3d_dev *v3d)
 				 V3D_TOP_GR_BRIDGE_SW_INIT_1_V3D_CLK_108_SW_INIT);
 		V3D_BRIDGE_WRITE(V3D_TOP_GR_BRIDGE_SW_INIT_1, 0);
 	}
+}
+
+static void
+v3d_reset_v3d(struct v3d_dev *v3d)
+{
+	if (v3d->reset)
+		reset_control_reset(v3d->reset);
+	else
+		v3d_reset_by_bridge(v3d);
 
 	v3d_init_hw_state(v3d);
 }
@@ -190,7 +201,8 @@ v3d_attach_object_fences(struct v3d_bo **bos, int bo_count,
 
 	for (i = 0; i < bo_count; i++) {
 		/* XXX: Use shared fences for read-only objects. */
-		reservation_object_add_excl_fence(bos[i]->resv, fence);
+		reservation_object_add_excl_fence(bos[i]->base.base.resv,
+						  fence);
 	}
 }
 
@@ -199,12 +211,8 @@ v3d_unlock_bo_reservations(struct v3d_bo **bos,
 			   int bo_count,
 			   struct ww_acquire_ctx *acquire_ctx)
 {
-	int i;
-
-	for (i = 0; i < bo_count; i++)
-		ww_mutex_unlock(&bos[i]->resv->lock);
-
-	ww_acquire_fini(acquire_ctx);
+	drm_gem_unlock_reservations((struct drm_gem_object **)bos, bo_count,
+				    acquire_ctx);
 }
 
 /* Takes the reservation lock on all the BOs being referenced, so that
@@ -219,58 +227,19 @@ v3d_lock_bo_reservations(struct v3d_bo **bos,
 			 int bo_count,
 			 struct ww_acquire_ctx *acquire_ctx)
 {
-	int contended_lock = -1;
 	int i, ret;
 
-	ww_acquire_init(acquire_ctx, &reservation_ww_class);
-
-retry:
-	if (contended_lock != -1) {
-		struct v3d_bo *bo = bos[contended_lock];
-
-		ret = ww_mutex_lock_slow_interruptible(&bo->resv->lock,
-						       acquire_ctx);
-		if (ret) {
-			ww_acquire_done(acquire_ctx);
-			return ret;
-		}
-	}
-
-	for (i = 0; i < bo_count; i++) {
-		if (i == contended_lock)
-			continue;
-
-		ret = ww_mutex_lock_interruptible(&bos[i]->resv->lock,
-						  acquire_ctx);
-		if (ret) {
-			int j;
-
-			for (j = 0; j < i; j++)
-				ww_mutex_unlock(&bos[j]->resv->lock);
-
-			if (contended_lock != -1 && contended_lock >= i) {
-				struct v3d_bo *bo = bos[contended_lock];
-
-				ww_mutex_unlock(&bo->resv->lock);
-			}
-
-			if (ret == -EDEADLK) {
-				contended_lock = i;
-				goto retry;
-			}
-
-			ww_acquire_done(acquire_ctx);
-			return ret;
-		}
-	}
-
-	ww_acquire_done(acquire_ctx);
+	ret = drm_gem_lock_reservations((struct drm_gem_object **)bos,
+					bo_count, acquire_ctx);
+	if (ret)
+		return ret;
 
 	/* Reserve space for our shared (read-only) fence references,
 	 * before we commit the CL to the hardware.
 	 */
 	for (i = 0; i < bo_count; i++) {
-		ret = reservation_object_reserve_shared(bos[i]->resv, 1);
+		ret = reservation_object_reserve_shared(bos[i]->base.base.resv,
+							1);
 		if (ret) {
 			v3d_unlock_bo_reservations(bos, bo_count,
 						   acquire_ctx);
@@ -371,18 +340,18 @@ v3d_exec_cleanup(struct kref *ref)
 	dma_fence_put(exec->bin.in_fence);
 	dma_fence_put(exec->render.in_fence);
 
-	dma_fence_put(exec->bin.done_fence);
-	dma_fence_put(exec->render.done_fence);
+	dma_fence_put(exec->bin.irq_fence);
+	dma_fence_put(exec->render.irq_fence);
 
 	dma_fence_put(exec->bin_done_fence);
 	dma_fence_put(exec->render_done_fence);
 
 	for (i = 0; i < exec->bo_count; i++)
-		drm_gem_object_put_unlocked(&exec->bo[i]->base);
+		drm_gem_object_put_unlocked(&exec->bo[i]->base.base);
 	kvfree(exec->bo);
 
 	list_for_each_entry_safe(bo, save, &exec->unref_list, unref_head) {
-		drm_gem_object_put_unlocked(&bo->base);
+		drm_gem_object_put_unlocked(&bo->base.base);
 	}
 
 	pm_runtime_mark_last_busy(v3d->dev);
@@ -405,11 +374,11 @@ v3d_tfu_job_cleanup(struct kref *ref)
 	unsigned int i;
 
 	dma_fence_put(job->in_fence);
-	dma_fence_put(job->done_fence);
+	dma_fence_put(job->irq_fence);
 
 	for (i = 0; i < ARRAY_SIZE(job->bo); i++) {
 		if (job->bo[i])
-			drm_gem_object_put_unlocked(&job->bo[i]->base);
+			drm_gem_object_put_unlocked(&job->bo[i]->base.base);
 	}
 
 	pm_runtime_mark_last_busy(v3d->dev);
@@ -429,8 +398,6 @@ v3d_wait_bo_ioctl(struct drm_device *dev, void *data,
 {
 	int ret;
 	struct drm_v3d_wait_bo *args = data;
-	struct drm_gem_object *gem_obj;
-	struct v3d_bo *bo;
 	ktime_t start = ktime_get();
 	u64 delta_ns;
 	unsigned long timeout_jiffies =
@@ -439,21 +406,8 @@ v3d_wait_bo_ioctl(struct drm_device *dev, void *data,
 	if (args->pad != 0)
 		return -EINVAL;
 
-	gem_obj = drm_gem_object_lookup(file_priv, args->handle);
-	if (!gem_obj) {
-		DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle);
-		return -EINVAL;
-	}
-	bo = to_v3d_bo(gem_obj);
-
-	ret = reservation_object_wait_timeout_rcu(bo->resv,
-						  true, true,
-						  timeout_jiffies);
-
-	if (ret == 0)
-		ret = -ETIME;
-	else if (ret > 0)
-		ret = 0;
+	ret = drm_gem_reservation_object_wait(file_priv, args->handle,
+					      true, timeout_jiffies);
 
 	/* Decrement the user's timeout, in case we got interrupted
 	 * such that the ioctl will be restarted.
@@ -468,8 +422,6 @@ v3d_wait_bo_ioctl(struct drm_device *dev, void *data,
 	if (ret == -ETIME && args->timeout_ns)
 		ret = -EAGAIN;
 
-	drm_gem_object_put_unlocked(gem_obj);
-
 	return ret;
 }
 
diff --git a/drivers/gpu/drm/v3d/v3d_irq.c b/drivers/gpu/drm/v3d/v3d_irq.c
index 69338da70ddc..aa0a180ae700 100644
--- a/drivers/gpu/drm/v3d/v3d_irq.c
+++ b/drivers/gpu/drm/v3d/v3d_irq.c
@@ -27,6 +27,9 @@
 			    V3D_HUB_INT_MMU_CAP |	\
 			    V3D_HUB_INT_TFUC))
 
+static irqreturn_t
+v3d_hub_irq(int irq, void *arg);
+
 static void
 v3d_overflow_mem_work(struct work_struct *work)
 {
@@ -34,12 +37,14 @@ v3d_overflow_mem_work(struct work_struct *work)
 		container_of(work, struct v3d_dev, overflow_mem_work);
 	struct drm_device *dev = &v3d->drm;
 	struct v3d_bo *bo = v3d_bo_create(dev, NULL /* XXX: GMP */, 256 * 1024);
+	struct drm_gem_object *obj;
 	unsigned long irqflags;
 
 	if (IS_ERR(bo)) {
 		DRM_ERROR("Couldn't allocate binner overflow mem\n");
 		return;
 	}
+	obj = &bo->base.base;
 
 	/* We lost a race, and our work task came in after the bin job
 	 * completed and exited.  This can happen because the HW
@@ -56,15 +61,15 @@ v3d_overflow_mem_work(struct work_struct *work)
 		goto out;
 	}
 
-	drm_gem_object_get(&bo->base);
+	drm_gem_object_get(obj);
 	list_add_tail(&bo->unref_head, &v3d->bin_job->unref_list);
 	spin_unlock_irqrestore(&v3d->job_lock, irqflags);
 
 	V3D_CORE_WRITE(0, V3D_PTB_BPOA, bo->node.start << PAGE_SHIFT);
-	V3D_CORE_WRITE(0, V3D_PTB_BPOS, bo->base.size);
+	V3D_CORE_WRITE(0, V3D_PTB_BPOS, obj->size);
 
 out:
-	drm_gem_object_put_unlocked(&bo->base);
+	drm_gem_object_put_unlocked(obj);
 }
 
 static irqreturn_t
@@ -82,7 +87,8 @@ v3d_irq(int irq, void *arg)
 	if (intsts & V3D_INT_OUTOMEM) {
 		/* Note that the OOM status is edge signaled, so the
 		 * interrupt won't happen again until the we actually
-		 * add more memory.
+		 * add more memory.  Also, as of V3D 4.1, FLDONE won't
+		 * be reported until any OOM state has been cleared.
 		 */
 		schedule_work(&v3d->overflow_mem_work);
 		status = IRQ_HANDLED;
@@ -90,7 +96,7 @@ v3d_irq(int irq, void *arg)
 
 	if (intsts & V3D_INT_FLDONE) {
 		struct v3d_fence *fence =
-			to_v3d_fence(v3d->bin_job->bin.done_fence);
+			to_v3d_fence(v3d->bin_job->bin.irq_fence);
 
 		trace_v3d_bcl_irq(&v3d->drm, fence->seqno);
 		dma_fence_signal(&fence->base);
@@ -99,7 +105,7 @@ v3d_irq(int irq, void *arg)
 
 	if (intsts & V3D_INT_FRDONE) {
 		struct v3d_fence *fence =
-			to_v3d_fence(v3d->render_job->render.done_fence);
+			to_v3d_fence(v3d->render_job->render.irq_fence);
 
 		trace_v3d_rcl_irq(&v3d->drm, fence->seqno);
 		dma_fence_signal(&fence->base);
@@ -112,6 +118,12 @@ v3d_irq(int irq, void *arg)
 	if (intsts & V3D_INT_GMPV)
 		dev_err(v3d->dev, "GMP violation\n");
 
+	/* V3D 4.2 wires the hub and core IRQs together, so if we &
+	 * didn't see the common one then check hub for MMU IRQs.
+	 */
+	if (v3d->single_irq_line && status == IRQ_NONE)
+		return v3d_hub_irq(irq, arg);
+
 	return status;
 }
 
@@ -129,7 +141,7 @@ v3d_hub_irq(int irq, void *arg)
 
 	if (intsts & V3D_HUB_INT_TFUC) {
 		struct v3d_fence *fence =
-			to_v3d_fence(v3d->tfu_job->done_fence);
+			to_v3d_fence(v3d->tfu_job->irq_fence);
 
 		trace_v3d_tfu_irq(&v3d->drm, fence->seqno);
 		dma_fence_signal(&fence->base);
@@ -156,10 +168,10 @@ v3d_hub_irq(int irq, void *arg)
 	return status;
 }
 
-void
+int
 v3d_irq_init(struct v3d_dev *v3d)
 {
-	int ret, core;
+	int irq1, ret, core;
 
 	INIT_WORK(&v3d->overflow_mem_work, v3d_overflow_mem_work);
 
@@ -170,16 +182,37 @@ v3d_irq_init(struct v3d_dev *v3d)
 		V3D_CORE_WRITE(core, V3D_CTL_INT_CLR, V3D_CORE_IRQS);
 	V3D_WRITE(V3D_HUB_INT_CLR, V3D_HUB_IRQS);
 
-	ret = devm_request_irq(v3d->dev, platform_get_irq(v3d->pdev, 0),
-			       v3d_hub_irq, IRQF_SHARED,
-			       "v3d_hub", v3d);
-	ret = devm_request_irq(v3d->dev, platform_get_irq(v3d->pdev, 1),
-			       v3d_irq, IRQF_SHARED,
-			       "v3d_core0", v3d);
-	if (ret)
-		dev_err(v3d->dev, "IRQ setup failed: %d\n", ret);
+	irq1 = platform_get_irq(v3d->pdev, 1);
+	if (irq1 == -EPROBE_DEFER)
+		return irq1;
+	if (irq1 > 0) {
+		ret = devm_request_irq(v3d->dev, irq1,
+				       v3d_irq, IRQF_SHARED,
+				       "v3d_core0", v3d);
+		if (ret)
+			goto fail;
+		ret = devm_request_irq(v3d->dev, platform_get_irq(v3d->pdev, 0),
+				       v3d_hub_irq, IRQF_SHARED,
+				       "v3d_hub", v3d);
+		if (ret)
+			goto fail;
+	} else {
+		v3d->single_irq_line = true;
+
+		ret = devm_request_irq(v3d->dev, platform_get_irq(v3d->pdev, 0),
+				       v3d_irq, IRQF_SHARED,
+				       "v3d", v3d);
+		if (ret)
+			goto fail;
+	}
 
 	v3d_irq_enable(v3d);
+	return 0;
+
+fail:
+	if (ret != -EPROBE_DEFER)
+		dev_err(v3d->dev, "IRQ setup failed: %d\n", ret);
+	return ret;
 }
 
 void
diff --git a/drivers/gpu/drm/v3d/v3d_mmu.c b/drivers/gpu/drm/v3d/v3d_mmu.c
index b00f97c31b70..7a21f1787ab1 100644
--- a/drivers/gpu/drm/v3d/v3d_mmu.c
+++ b/drivers/gpu/drm/v3d/v3d_mmu.c
@@ -83,13 +83,14 @@ int v3d_mmu_set_page_table(struct v3d_dev *v3d)
 
 void v3d_mmu_insert_ptes(struct v3d_bo *bo)
 {
-	struct v3d_dev *v3d = to_v3d_dev(bo->base.dev);
+	struct drm_gem_shmem_object *shmem_obj = &bo->base;
+	struct v3d_dev *v3d = to_v3d_dev(shmem_obj->base.dev);
 	u32 page = bo->node.start;
 	u32 page_prot = V3D_PTE_WRITEABLE | V3D_PTE_VALID;
 	unsigned int count;
 	struct scatterlist *sgl;
 
-	for_each_sg(bo->sgt->sgl, sgl, bo->sgt->nents, count) {
+	for_each_sg(shmem_obj->sgt->sgl, sgl, shmem_obj->sgt->nents, count) {
 		u32 page_address = sg_dma_address(sgl) >> V3D_MMU_PAGE_SHIFT;
 		u32 pte = page_prot | page_address;
 		u32 i;
@@ -102,7 +103,7 @@ void v3d_mmu_insert_ptes(struct v3d_bo *bo)
 	}
 
 	WARN_ON_ONCE(page - bo->node.start !=
-		     bo->base.size >> V3D_MMU_PAGE_SHIFT);
+		     shmem_obj->base.size >> V3D_MMU_PAGE_SHIFT);
 
 	if (v3d_mmu_flush_all(v3d))
 		dev_err(v3d->dev, "MMU flush timeout\n");
@@ -110,8 +111,8 @@ void v3d_mmu_insert_ptes(struct v3d_bo *bo)
 
 void v3d_mmu_remove_ptes(struct v3d_bo *bo)
 {
-	struct v3d_dev *v3d = to_v3d_dev(bo->base.dev);
-	u32 npages = bo->base.size >> V3D_MMU_PAGE_SHIFT;
+	struct v3d_dev *v3d = to_v3d_dev(bo->base.base.dev);
+	u32 npages = bo->base.base.size >> V3D_MMU_PAGE_SHIFT;
 	u32 page;
 
 	for (page = bo->node.start; page < bo->node.start + npages; page++)
diff --git a/drivers/gpu/drm/v3d/v3d_regs.h b/drivers/gpu/drm/v3d/v3d_regs.h
index 6ccdee9d47bd..8e88af237610 100644
--- a/drivers/gpu/drm/v3d/v3d_regs.h
+++ b/drivers/gpu/drm/v3d/v3d_regs.h
@@ -216,6 +216,8 @@
 # define V3D_IDENT2_BCG_INT                            BIT(28)
 
 #define V3D_CTL_MISCCFG                                0x00018
+# define V3D_CTL_MISCCFG_QRMAXCNT_MASK                 V3D_MASK(3, 1)
+# define V3D_CTL_MISCCFG_QRMAXCNT_SHIFT                1
 # define V3D_MISCCFG_OVRTMUOUT                         BIT(0)
 
 #define V3D_CTL_L2CACTL                                0x00020
diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c
index 4704b2df3688..e740f3b99aa5 100644
--- a/drivers/gpu/drm/v3d/v3d_sched.c
+++ b/drivers/gpu/drm/v3d/v3d_sched.c
@@ -156,9 +156,9 @@ static struct dma_fence *v3d_job_run(struct drm_sched_job *sched_job)
 	if (IS_ERR(fence))
 		return NULL;
 
-	if (job->done_fence)
-		dma_fence_put(job->done_fence);
-	job->done_fence = dma_fence_get(fence);
+	if (job->irq_fence)
+		dma_fence_put(job->irq_fence);
+	job->irq_fence = dma_fence_get(fence);
 
 	trace_v3d_submit_cl(dev, q == V3D_RENDER, to_v3d_fence(fence)->seqno,
 			    job->start, job->end);
@@ -199,9 +199,9 @@ v3d_tfu_job_run(struct drm_sched_job *sched_job)
 		return NULL;
 
 	v3d->tfu_job = job;
-	if (job->done_fence)
-		dma_fence_put(job->done_fence);
-	job->done_fence = dma_fence_get(fence);
+	if (job->irq_fence)
+		dma_fence_put(job->irq_fence);
+	job->irq_fence = dma_fence_get(fence);
 
 	trace_v3d_submit_tfu(dev, to_v3d_fence(fence)->seqno);
 
@@ -231,20 +231,17 @@ v3d_gpu_reset_for_timeout(struct v3d_dev *v3d, struct drm_sched_job *sched_job)
 	mutex_lock(&v3d->reset_lock);
 
 	/* block scheduler */
-	for (q = 0; q < V3D_MAX_QUEUES; q++) {
-		struct drm_gpu_scheduler *sched = &v3d->queue[q].sched;
-
-		drm_sched_stop(sched);
+	for (q = 0; q < V3D_MAX_QUEUES; q++)
+		drm_sched_stop(&v3d->queue[q].sched);
 
-		if(sched_job)
-			drm_sched_increase_karma(sched_job);
-	}
+	if (sched_job)
+		drm_sched_increase_karma(sched_job);
 
 	/* get the GPU back into the init state */
 	v3d_reset(v3d);
 
 	for (q = 0; q < V3D_MAX_QUEUES; q++)
-		drm_sched_resubmit_jobs(sched_job->sched);
+		drm_sched_resubmit_jobs(&v3d->queue[q].sched);
 
 	/* Unblock schedulers and restart their jobs. */
 	for (q = 0; q < V3D_MAX_QUEUES; q++) {
diff --git a/drivers/staging/vboxvideo/Kconfig b/drivers/gpu/drm/vboxvideo/Kconfig
index d6ab955c0768..d6ab955c0768 100644
--- a/drivers/staging/vboxvideo/Kconfig
+++ b/drivers/gpu/drm/vboxvideo/Kconfig
diff --git a/drivers/staging/vboxvideo/Makefile b/drivers/gpu/drm/vboxvideo/Makefile
index 1224f313af0c..1224f313af0c 100644
--- a/drivers/staging/vboxvideo/Makefile
+++ b/drivers/gpu/drm/vboxvideo/Makefile
diff --git a/drivers/staging/vboxvideo/hgsmi_base.c b/drivers/gpu/drm/vboxvideo/hgsmi_base.c
index 361d3193258e..361d3193258e 100644
--- a/drivers/staging/vboxvideo/hgsmi_base.c
+++ b/drivers/gpu/drm/vboxvideo/hgsmi_base.c
diff --git a/drivers/staging/vboxvideo/hgsmi_ch_setup.h b/drivers/gpu/drm/vboxvideo/hgsmi_ch_setup.h
index 4e93418d6a13..4e93418d6a13 100644
--- a/drivers/staging/vboxvideo/hgsmi_ch_setup.h
+++ b/drivers/gpu/drm/vboxvideo/hgsmi_ch_setup.h
diff --git a/drivers/staging/vboxvideo/hgsmi_channels.h b/drivers/gpu/drm/vboxvideo/hgsmi_channels.h
index 9b83f4ff3faf..9b83f4ff3faf 100644
--- a/drivers/staging/vboxvideo/hgsmi_channels.h
+++ b/drivers/gpu/drm/vboxvideo/hgsmi_channels.h
diff --git a/drivers/staging/vboxvideo/hgsmi_defs.h b/drivers/gpu/drm/vboxvideo/hgsmi_defs.h
index 6c8df1cdb087..6c8df1cdb087 100644
--- a/drivers/staging/vboxvideo/hgsmi_defs.h
+++ b/drivers/gpu/drm/vboxvideo/hgsmi_defs.h
diff --git a/drivers/staging/vboxvideo/modesetting.c b/drivers/gpu/drm/vboxvideo/modesetting.c
index 7580b9002379..7580b9002379 100644
--- a/drivers/staging/vboxvideo/modesetting.c
+++ b/drivers/gpu/drm/vboxvideo/modesetting.c
diff --git a/drivers/staging/vboxvideo/vbox_drv.c b/drivers/gpu/drm/vboxvideo/vbox_drv.c
index e7755a179850..fb6a0f0b8167 100644
--- a/drivers/staging/vboxvideo/vbox_drv.c
+++ b/drivers/gpu/drm/vboxvideo/vbox_drv.c
@@ -200,36 +200,11 @@ static const struct file_operations vbox_fops = {
 	.read = drm_read,
 };
 
-static int vbox_master_set(struct drm_device *dev,
-			   struct drm_file *file_priv, bool from_open)
-{
-	struct vbox_private *vbox = dev->dev_private;
-
-	/*
-	 * We do not yet know whether the new owner can handle hotplug, so we
-	 * do not advertise dynamic modes on the first query and send a
-	 * tentative hotplug notification after that to see if they query again.
-	 */
-	vbox->initial_mode_queried = false;
-
-	return 0;
-}
-
-static void vbox_master_drop(struct drm_device *dev, struct drm_file *file_priv)
-{
-	struct vbox_private *vbox = dev->dev_private;
-
-	/* See vbox_master_set() */
-	vbox->initial_mode_queried = false;
-}
-
 static struct drm_driver driver = {
 	.driver_features =
 	    DRIVER_MODESET | DRIVER_GEM | DRIVER_PRIME | DRIVER_ATOMIC,
 
 	.lastclose = drm_fb_helper_lastclose,
-	.master_set = vbox_master_set,
-	.master_drop = vbox_master_drop,
 
 	.fops = &vbox_fops,
 	.irq_handler = vbox_irq_handler,
diff --git a/drivers/staging/vboxvideo/vbox_drv.h b/drivers/gpu/drm/vboxvideo/vbox_drv.h
index aa40e5cc2861..ece31f395540 100644
--- a/drivers/staging/vboxvideo/vbox_drv.h
+++ b/drivers/gpu/drm/vboxvideo/vbox_drv.h
@@ -14,7 +14,6 @@
 #include <linux/io.h>
 #include <linux/irqreturn.h>
 #include <linux/string.h>
-#include <linux/version.h>
 
 #include <drm/drm_encoder.h>
 #include <drm/drm_fb_helper.h>
@@ -83,12 +82,6 @@ struct vbox_private {
 	} ttm;
 
 	struct mutex hw_mutex; /* protects modeset and accel/vbva accesses */
-	/*
-	 * We decide whether or not user-space supports display hot-plug
-	 * depending on whether they react to a hot-plug event after the initial
-	 * mode query.
-	 */
-	bool initial_mode_queried;
 	struct work_struct hotplug_work;
 	u32 input_mapping_width;
 	u32 input_mapping_height;
@@ -209,8 +202,6 @@ int vbox_dumb_mmap_offset(struct drm_file *file,
 			  struct drm_device *dev,
 			  u32 handle, u64 *offset);
 
-#define DRM_FILE_PAGE_OFFSET (0x10000000ULL >> PAGE_SHIFT)
-
 int vbox_mm_init(struct vbox_private *vbox);
 void vbox_mm_fini(struct vbox_private *vbox);
 
diff --git a/drivers/staging/vboxvideo/vbox_fb.c b/drivers/gpu/drm/vboxvideo/vbox_fb.c
index 83a04afd1766..b724fe7c0c30 100644
--- a/drivers/staging/vboxvideo/vbox_fb.c
+++ b/drivers/gpu/drm/vboxvideo/vbox_fb.c
@@ -90,13 +90,9 @@ int vboxfb_create(struct drm_fb_helper *helper,
 	if (IS_ERR(info->screen_base))
 		return PTR_ERR(info->screen_base);
 
-	info->par = helper;
-
 	fb = &vbox->afb.base;
 	helper->fb = fb;
 
-	strcpy(info->fix.id, "vboxdrmfb");
-
 	info->fbops = &vboxfb_ops;
 
 	/*
@@ -106,9 +102,7 @@ int vboxfb_create(struct drm_fb_helper *helper,
 	info->apertures->ranges[0].base = pci_resource_start(pdev, 0);
 	info->apertures->ranges[0].size = pci_resource_len(pdev, 0);
 
-	drm_fb_helper_fill_fix(info, fb->pitches[0], fb->format->depth);
-	drm_fb_helper_fill_var(info, helper, sizes->fb_width,
-			       sizes->fb_height);
+	drm_fb_helper_fill_info(info, helper, sizes);
 
 	gpu_addr = vbox_bo_gpu_offset(bo);
 	info->fix.smem_start = info->apertures->ranges[0].base + gpu_addr;
diff --git a/drivers/staging/vboxvideo/vbox_hgsmi.c b/drivers/gpu/drm/vboxvideo/vbox_hgsmi.c
index 94b60654a012..94b60654a012 100644
--- a/drivers/staging/vboxvideo/vbox_hgsmi.c
+++ b/drivers/gpu/drm/vboxvideo/vbox_hgsmi.c
diff --git a/drivers/staging/vboxvideo/vbox_irq.c b/drivers/gpu/drm/vboxvideo/vbox_irq.c
index 195484713365..16a1e29f5292 100644
--- a/drivers/staging/vboxvideo/vbox_irq.c
+++ b/drivers/gpu/drm/vboxvideo/vbox_irq.c
@@ -105,6 +105,7 @@ static void validate_or_set_position_hints(struct vbox_private *vbox)
 /* Query the host for the most recent video mode hints. */
 static void vbox_update_mode_hints(struct vbox_private *vbox)
 {
+	struct drm_connector_list_iter conn_iter;
 	struct drm_device *dev = &vbox->ddev;
 	struct drm_connector *connector;
 	struct vbox_connector *vbox_conn;
@@ -122,8 +123,10 @@ static void vbox_update_mode_hints(struct vbox_private *vbox)
 	}
 
 	validate_or_set_position_hints(vbox);
-	drm_modeset_lock_all(dev);
-	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
+
+	drm_modeset_lock(&dev->mode_config.connection_mutex, NULL);
+	drm_connector_list_iter_begin(dev, &conn_iter);
+	drm_for_each_connector_iter(connector, &conn_iter) {
 		vbox_conn = to_vbox_connector(connector);
 
 		hints = &vbox->last_mode_hints[vbox_conn->vbox_crtc->crtc_id];
@@ -152,7 +155,8 @@ static void vbox_update_mode_hints(struct vbox_private *vbox)
 
 		vbox_conn->vbox_crtc->disconnected = disconnected;
 	}
-	drm_modeset_unlock_all(dev);
+	drm_connector_list_iter_end(&conn_iter);
+	drm_modeset_unlock(&dev->mode_config.connection_mutex);
 }
 
 static void vbox_hotplug_worker(struct work_struct *work)
diff --git a/drivers/staging/vboxvideo/vbox_main.c b/drivers/gpu/drm/vboxvideo/vbox_main.c
index e1fb70a42d32..f4d02de5518a 100644
--- a/drivers/staging/vboxvideo/vbox_main.c
+++ b/drivers/gpu/drm/vboxvideo/vbox_main.c
@@ -32,9 +32,9 @@ void vbox_report_caps(struct vbox_private *vbox)
 	u32 caps = VBVACAPS_DISABLE_CURSOR_INTEGRATION |
 		   VBVACAPS_IRQ | VBVACAPS_USE_VBVA_ONLY;
 
-	if (vbox->initial_mode_queried)
-		caps |= VBVACAPS_VIDEO_MODE_HINTS;
-
+	/* The host only accepts VIDEO_MODE_HINTS if it is send separately. */
+	hgsmi_send_caps_info(vbox->guest_pool, caps);
+	caps |= VBVACAPS_VIDEO_MODE_HINTS;
 	hgsmi_send_caps_info(vbox->guest_pool, caps);
 }
 
diff --git a/drivers/staging/vboxvideo/vbox_mode.c b/drivers/gpu/drm/vboxvideo/vbox_mode.c
index f105360a574e..58cea131470e 100644
--- a/drivers/staging/vboxvideo/vbox_mode.c
+++ b/drivers/gpu/drm/vboxvideo/vbox_mode.c
@@ -735,29 +735,12 @@ static int vbox_get_modes(struct drm_connector *connector)
 
 	vbox_connector = to_vbox_connector(connector);
 	vbox = connector->dev->dev_private;
-	/*
-	 * Heuristic: we do not want to tell the host that we support dynamic
-	 * resizing unless we feel confident that the user space client using
-	 * the video driver can handle hot-plug events.  So the first time modes
-	 * are queried after a "master" switch we tell the host that we do not,
-	 * and immediately after we send the client a hot-plug notification as
-	 * a test to see if they will respond and query again.
-	 * That is also the reason why capabilities are reported to the host at
-	 * this place in the code rather than elsewhere.
-	 * We need to report the flags location before reporting the IRQ
-	 * capability.
-	 */
+
 	hgsmi_report_flags_location(vbox->guest_pool, GUEST_HEAP_OFFSET(vbox) +
 				    HOST_FLAGS_OFFSET);
 	if (vbox_connector->vbox_crtc->crtc_id == 0)
 		vbox_report_caps(vbox);
-	if (!vbox->initial_mode_queried) {
-		if (vbox_connector->vbox_crtc->crtc_id == 0) {
-			vbox->initial_mode_queried = true;
-			vbox_report_hotplug(vbox);
-		}
-		return drm_add_modes_noedid(connector, 800, 600);
-	}
+
 	num_modes = drm_add_modes_noedid(connector, 2560, 1600);
 	preferred_width = vbox_connector->mode_hint.width ?
 			  vbox_connector->mode_hint.width : 1024;
diff --git a/drivers/staging/vboxvideo/vbox_prime.c b/drivers/gpu/drm/vboxvideo/vbox_prime.c
index 702b1aa53494..702b1aa53494 100644
--- a/drivers/staging/vboxvideo/vbox_prime.c
+++ b/drivers/gpu/drm/vboxvideo/vbox_prime.c
diff --git a/drivers/staging/vboxvideo/vbox_ttm.c b/drivers/gpu/drm/vboxvideo/vbox_ttm.c
index 30f270027acf..9d78438c2877 100644
--- a/drivers/staging/vboxvideo/vbox_ttm.c
+++ b/drivers/gpu/drm/vboxvideo/vbox_ttm.c
@@ -156,7 +156,7 @@ int vbox_mm_init(struct vbox_private *vbox)
 	ret = ttm_bo_device_init(&vbox->ttm.bdev,
 				 &vbox_bo_driver,
 				 dev->anon_inode->i_mapping,
-				 DRM_FILE_PAGE_OFFSET, true);
+				 true);
 	if (ret) {
 		DRM_ERROR("Error initialising bo driver; %d\n", ret);
 		return ret;
@@ -357,14 +357,8 @@ int vbox_bo_push_sysram(struct vbox_bo *bo)
 
 int vbox_mmap(struct file *filp, struct vm_area_struct *vma)
 {
-	struct drm_file *file_priv;
-	struct vbox_private *vbox;
-
-	if (unlikely(vma->vm_pgoff < DRM_FILE_PAGE_OFFSET))
-		return -EINVAL;
-
-	file_priv = filp->private_data;
-	vbox = file_priv->minor->dev->dev_private;
+	struct drm_file *file_priv = filp->private_data;
+	struct vbox_private *vbox = file_priv->minor->dev->dev_private;
 
 	return ttm_bo_mmap(filp, vma, &vbox->ttm.bdev);
 }
diff --git a/drivers/staging/vboxvideo/vboxvideo.h b/drivers/gpu/drm/vboxvideo/vboxvideo.h
index 0592004f71aa..0592004f71aa 100644
--- a/drivers/staging/vboxvideo/vboxvideo.h
+++ b/drivers/gpu/drm/vboxvideo/vboxvideo.h
diff --git a/drivers/staging/vboxvideo/vboxvideo_guest.h b/drivers/gpu/drm/vboxvideo/vboxvideo_guest.h
index 55fcee3a6470..55fcee3a6470 100644
--- a/drivers/staging/vboxvideo/vboxvideo_guest.h
+++ b/drivers/gpu/drm/vboxvideo/vboxvideo_guest.h
diff --git a/drivers/staging/vboxvideo/vboxvideo_vbe.h b/drivers/gpu/drm/vboxvideo/vboxvideo_vbe.h
index 427235869297..427235869297 100644
--- a/drivers/staging/vboxvideo/vboxvideo_vbe.h
+++ b/drivers/gpu/drm/vboxvideo/vboxvideo_vbe.h
diff --git a/drivers/staging/vboxvideo/vbva_base.c b/drivers/gpu/drm/vboxvideo/vbva_base.c
index 36bc9824ec3f..36bc9824ec3f 100644
--- a/drivers/staging/vboxvideo/vbva_base.c
+++ b/drivers/gpu/drm/vboxvideo/vbva_base.c
diff --git a/drivers/gpu/drm/vc4/vc4_bo.c b/drivers/gpu/drm/vc4/vc4_bo.c
index 8dcce7182bb7..88ebd681d7eb 100644
--- a/drivers/gpu/drm/vc4/vc4_bo.c
+++ b/drivers/gpu/drm/vc4/vc4_bo.c
@@ -40,7 +40,7 @@ static bool is_user_label(int label)
 	return label >= VC4_BO_TYPE_COUNT;
 }
 
-static void vc4_bo_stats_dump(struct vc4_dev *vc4)
+static void vc4_bo_stats_print(struct drm_printer *p, struct vc4_dev *vc4)
 {
 	int i;
 
@@ -48,58 +48,35 @@ static void vc4_bo_stats_dump(struct vc4_dev *vc4)
 		if (!vc4->bo_labels[i].num_allocated)
 			continue;
 
-		DRM_INFO("%30s: %6dkb BOs (%d)\n",
-			 vc4->bo_labels[i].name,
-			 vc4->bo_labels[i].size_allocated / 1024,
-			 vc4->bo_labels[i].num_allocated);
+		drm_printf(p, "%30s: %6dkb BOs (%d)\n",
+			   vc4->bo_labels[i].name,
+			   vc4->bo_labels[i].size_allocated / 1024,
+			   vc4->bo_labels[i].num_allocated);
 	}
 
 	mutex_lock(&vc4->purgeable.lock);
 	if (vc4->purgeable.num)
-		DRM_INFO("%30s: %6zdkb BOs (%d)\n", "userspace BO cache",
-			 vc4->purgeable.size / 1024, vc4->purgeable.num);
+		drm_printf(p, "%30s: %6zdkb BOs (%d)\n", "userspace BO cache",
+			   vc4->purgeable.size / 1024, vc4->purgeable.num);
 
 	if (vc4->purgeable.purged_num)
-		DRM_INFO("%30s: %6zdkb BOs (%d)\n", "total purged BO",
-			 vc4->purgeable.purged_size / 1024,
-			 vc4->purgeable.purged_num);
+		drm_printf(p, "%30s: %6zdkb BOs (%d)\n", "total purged BO",
+			   vc4->purgeable.purged_size / 1024,
+			   vc4->purgeable.purged_num);
 	mutex_unlock(&vc4->purgeable.lock);
 }
 
-#ifdef CONFIG_DEBUG_FS
-int vc4_bo_stats_debugfs(struct seq_file *m, void *unused)
+static int vc4_bo_stats_debugfs(struct seq_file *m, void *unused)
 {
 	struct drm_info_node *node = (struct drm_info_node *)m->private;
 	struct drm_device *dev = node->minor->dev;
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
-	int i;
-
-	mutex_lock(&vc4->bo_lock);
-	for (i = 0; i < vc4->num_labels; i++) {
-		if (!vc4->bo_labels[i].num_allocated)
-			continue;
-
-		seq_printf(m, "%30s: %6dkb BOs (%d)\n",
-			   vc4->bo_labels[i].name,
-			   vc4->bo_labels[i].size_allocated / 1024,
-			   vc4->bo_labels[i].num_allocated);
-	}
-	mutex_unlock(&vc4->bo_lock);
-
-	mutex_lock(&vc4->purgeable.lock);
-	if (vc4->purgeable.num)
-		seq_printf(m, "%30s: %6zdkb BOs (%d)\n", "userspace BO cache",
-			   vc4->purgeable.size / 1024, vc4->purgeable.num);
+	struct drm_printer p = drm_seq_file_printer(m);
 
-	if (vc4->purgeable.purged_num)
-		seq_printf(m, "%30s: %6zdkb BOs (%d)\n", "total purged BO",
-			   vc4->purgeable.purged_size / 1024,
-			   vc4->purgeable.purged_num);
-	mutex_unlock(&vc4->purgeable.lock);
+	vc4_bo_stats_print(&p, vc4);
 
 	return 0;
 }
-#endif
 
 /* Takes ownership of *name and returns the appropriate slot for it in
  * the bo_labels[] array, extending it as necessary.
@@ -201,8 +178,6 @@ static void vc4_bo_destroy(struct vc4_bo *bo)
 		bo->validated_shader = NULL;
 	}
 
-	reservation_object_fini(&bo->_resv);
-
 	drm_gem_cma_free_object(obj);
 }
 
@@ -427,8 +402,6 @@ struct drm_gem_object *vc4_create_object(struct drm_device *dev, size_t size)
 	vc4->bo_labels[VC4_BO_TYPE_KERNEL].num_allocated++;
 	vc4->bo_labels[VC4_BO_TYPE_KERNEL].size_allocated += size;
 	mutex_unlock(&vc4->bo_lock);
-	bo->resv = &bo->_resv;
-	reservation_object_init(bo->resv);
 
 	return &bo->base.base;
 }
@@ -479,8 +452,9 @@ struct vc4_bo *vc4_bo_create(struct drm_device *dev, size_t unaligned_size,
 	}
 
 	if (IS_ERR(cma_obj)) {
+		struct drm_printer p = drm_info_printer(vc4->dev->dev);
 		DRM_ERROR("Failed to allocate from CMA:\n");
-		vc4_bo_stats_dump(vc4);
+		vc4_bo_stats_print(&p, vc4);
 		return ERR_PTR(-ENOMEM);
 	}
 	bo = to_vc4_bo(&cma_obj->base);
@@ -684,13 +658,6 @@ static void vc4_bo_cache_time_timer(struct timer_list *t)
 	schedule_work(&vc4->bo_cache.time_work);
 }
 
-struct reservation_object *vc4_prime_res_obj(struct drm_gem_object *obj)
-{
-	struct vc4_bo *bo = to_vc4_bo(obj);
-
-	return bo->resv;
-}
-
 struct dma_buf *
 vc4_prime_export(struct drm_device *dev, struct drm_gem_object *obj, int flags)
 {
@@ -822,14 +789,12 @@ vc4_prime_import_sg_table(struct drm_device *dev,
 			  struct sg_table *sgt)
 {
 	struct drm_gem_object *obj;
-	struct vc4_bo *bo;
 
 	obj = drm_gem_cma_prime_import_sg_table(dev, attach, sgt);
 	if (IS_ERR(obj))
 		return obj;
 
-	bo = to_vc4_bo(obj);
-	bo->resv = attach->dmabuf->resv;
+	obj->resv = attach->dmabuf->resv;
 
 	return obj;
 }
@@ -1038,6 +1003,8 @@ int vc4_bo_cache_init(struct drm_device *dev)
 
 	mutex_init(&vc4->bo_lock);
 
+	vc4_debugfs_add_file(dev, "bo_stats", vc4_bo_stats_debugfs, NULL);
+
 	INIT_LIST_HEAD(&vc4->bo_cache.time_list);
 
 	INIT_WORK(&vc4->bo_cache.time_work, vc4_bo_cache_time_work);
diff --git a/drivers/gpu/drm/vc4/vc4_crtc.c b/drivers/gpu/drm/vc4/vc4_crtc.c
index 1baa10e94484..5e09389e1514 100644
--- a/drivers/gpu/drm/vc4/vc4_crtc.c
+++ b/drivers/gpu/drm/vc4/vc4_crtc.c
@@ -35,6 +35,7 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_atomic_uapi.h>
+#include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
 #include <linux/clk.h>
 #include <drm/drm_fb_cma_helper.h>
@@ -67,67 +68,22 @@ to_vc4_crtc_state(struct drm_crtc_state *crtc_state)
 #define CRTC_WRITE(offset, val) writel(val, vc4_crtc->regs + (offset))
 #define CRTC_READ(offset) readl(vc4_crtc->regs + (offset))
 
-#define CRTC_REG(reg) { reg, #reg }
-static const struct {
-	u32 reg;
-	const char *name;
-} crtc_regs[] = {
-	CRTC_REG(PV_CONTROL),
-	CRTC_REG(PV_V_CONTROL),
-	CRTC_REG(PV_VSYNCD_EVEN),
-	CRTC_REG(PV_HORZA),
-	CRTC_REG(PV_HORZB),
-	CRTC_REG(PV_VERTA),
-	CRTC_REG(PV_VERTB),
-	CRTC_REG(PV_VERTA_EVEN),
-	CRTC_REG(PV_VERTB_EVEN),
-	CRTC_REG(PV_INTEN),
-	CRTC_REG(PV_INTSTAT),
-	CRTC_REG(PV_STAT),
-	CRTC_REG(PV_HACT_ACT),
+static const struct debugfs_reg32 crtc_regs[] = {
+	VC4_REG32(PV_CONTROL),
+	VC4_REG32(PV_V_CONTROL),
+	VC4_REG32(PV_VSYNCD_EVEN),
+	VC4_REG32(PV_HORZA),
+	VC4_REG32(PV_HORZB),
+	VC4_REG32(PV_VERTA),
+	VC4_REG32(PV_VERTB),
+	VC4_REG32(PV_VERTA_EVEN),
+	VC4_REG32(PV_VERTB_EVEN),
+	VC4_REG32(PV_INTEN),
+	VC4_REG32(PV_INTSTAT),
+	VC4_REG32(PV_STAT),
+	VC4_REG32(PV_HACT_ACT),
 };
 
-static void vc4_crtc_dump_regs(struct vc4_crtc *vc4_crtc)
-{
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(crtc_regs); i++) {
-		DRM_INFO("0x%04x (%s): 0x%08x\n",
-			 crtc_regs[i].reg, crtc_regs[i].name,
-			 CRTC_READ(crtc_regs[i].reg));
-	}
-}
-
-#ifdef CONFIG_DEBUG_FS
-int vc4_crtc_debugfs_regs(struct seq_file *m, void *unused)
-{
-	struct drm_info_node *node = (struct drm_info_node *)m->private;
-	struct drm_device *dev = node->minor->dev;
-	int crtc_index = (uintptr_t)node->info_ent->data;
-	struct drm_crtc *crtc;
-	struct vc4_crtc *vc4_crtc;
-	int i;
-
-	i = 0;
-	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
-		if (i == crtc_index)
-			break;
-		i++;
-	}
-	if (!crtc)
-		return 0;
-	vc4_crtc = to_vc4_crtc(crtc);
-
-	for (i = 0; i < ARRAY_SIZE(crtc_regs); i++) {
-		seq_printf(m, "%s (0x%04x): 0x%08x\n",
-			   crtc_regs[i].name, crtc_regs[i].reg,
-			   CRTC_READ(crtc_regs[i].reg));
-	}
-
-	return 0;
-}
-#endif
-
 bool vc4_crtc_get_scanoutpos(struct drm_device *dev, unsigned int crtc_id,
 			     bool in_vblank_irq, int *vpos, int *hpos,
 			     ktime_t *stime, ktime_t *etime,
@@ -434,8 +390,10 @@ static void vc4_crtc_mode_set_nofb(struct drm_crtc *crtc)
 	bool debug_dump_regs = false;
 
 	if (debug_dump_regs) {
-		DRM_INFO("CRTC %d regs before:\n", drm_crtc_index(crtc));
-		vc4_crtc_dump_regs(vc4_crtc);
+		struct drm_printer p = drm_info_printer(&vc4_crtc->pdev->dev);
+		dev_info(&vc4_crtc->pdev->dev, "CRTC %d regs before:\n",
+			 drm_crtc_index(crtc));
+		drm_print_regset32(&p, &vc4_crtc->regset);
 	}
 
 	if (vc4_crtc->channel == 2) {
@@ -476,8 +434,10 @@ static void vc4_crtc_mode_set_nofb(struct drm_crtc *crtc)
 	vc4_crtc_lut_load(crtc);
 
 	if (debug_dump_regs) {
-		DRM_INFO("CRTC %d regs after:\n", drm_crtc_index(crtc));
-		vc4_crtc_dump_regs(vc4_crtc);
+		struct drm_printer p = drm_info_printer(&vc4_crtc->pdev->dev);
+		dev_info(&vc4_crtc->pdev->dev, "CRTC %d regs after:\n",
+			 drm_crtc_index(crtc));
+		drm_print_regset32(&p, &vc4_crtc->regset);
 	}
 }
 
@@ -834,6 +794,14 @@ static void vc4_crtc_handle_page_flip(struct vc4_crtc *vc4_crtc)
 		drm_crtc_send_vblank_event(crtc, vc4_crtc->event);
 		vc4_crtc->event = NULL;
 		drm_crtc_vblank_put(crtc);
+
+		/* Wait for the page flip to unmask the underrun to ensure that
+		 * the display list was updated by the hardware. Before that
+		 * happens, the HVS will be using the previous display list with
+		 * the CRTC and encoder already reconfigured, leading to
+		 * underruns. This can be seen when reconfiguring the CRTC.
+		 */
+		vc4_hvs_unmask_underrun(dev, vc4_crtc->channel);
 	}
 	spin_unlock_irqrestore(&dev->event_lock, flags);
 }
@@ -1075,6 +1043,7 @@ static const struct drm_crtc_helper_funcs vc4_crtc_helper_funcs = {
 
 static const struct vc4_crtc_data pv0_data = {
 	.hvs_channel = 0,
+	.debugfs_name = "crtc0_regs",
 	.encoder_types = {
 		[PV_CONTROL_CLK_SELECT_DSI] = VC4_ENCODER_TYPE_DSI0,
 		[PV_CONTROL_CLK_SELECT_DPI_SMI_HDMI] = VC4_ENCODER_TYPE_DPI,
@@ -1083,6 +1052,7 @@ static const struct vc4_crtc_data pv0_data = {
 
 static const struct vc4_crtc_data pv1_data = {
 	.hvs_channel = 2,
+	.debugfs_name = "crtc1_regs",
 	.encoder_types = {
 		[PV_CONTROL_CLK_SELECT_DSI] = VC4_ENCODER_TYPE_DSI1,
 		[PV_CONTROL_CLK_SELECT_DPI_SMI_HDMI] = VC4_ENCODER_TYPE_SMI,
@@ -1091,6 +1061,7 @@ static const struct vc4_crtc_data pv1_data = {
 
 static const struct vc4_crtc_data pv2_data = {
 	.hvs_channel = 1,
+	.debugfs_name = "crtc2_regs",
 	.encoder_types = {
 		[PV_CONTROL_CLK_SELECT_DPI_SMI_HDMI] = VC4_ENCODER_TYPE_HDMI,
 		[PV_CONTROL_CLK_SELECT_VEC] = VC4_ENCODER_TYPE_VEC,
@@ -1169,11 +1140,16 @@ static int vc4_crtc_bind(struct device *dev, struct device *master, void *data)
 	if (!match)
 		return -ENODEV;
 	vc4_crtc->data = match->data;
+	vc4_crtc->pdev = pdev;
 
 	vc4_crtc->regs = vc4_ioremap_regs(pdev, 0);
 	if (IS_ERR(vc4_crtc->regs))
 		return PTR_ERR(vc4_crtc->regs);
 
+	vc4_crtc->regset.base = vc4_crtc->regs;
+	vc4_crtc->regset.regs = crtc_regs;
+	vc4_crtc->regset.nregs = ARRAY_SIZE(crtc_regs);
+
 	/* For now, we create just the primary and the legacy cursor
 	 * planes.  We should be able to stack more planes on easily,
 	 * but to do that we would need to compute the bandwidth
@@ -1247,6 +1223,9 @@ static int vc4_crtc_bind(struct device *dev, struct device *master, void *data)
 
 	platform_set_drvdata(pdev, vc4_crtc);
 
+	vc4_debugfs_add_regset32(drm, vc4_crtc->data->debugfs_name,
+				 &vc4_crtc->regset);
+
 	return 0;
 
 err_destroy_planes:
diff --git a/drivers/gpu/drm/vc4/vc4_debugfs.c b/drivers/gpu/drm/vc4/vc4_debugfs.c
index 7a0003de71ab..f9dec08267dc 100644
--- a/drivers/gpu/drm/vc4/vc4_debugfs.c
+++ b/drivers/gpu/drm/vc4/vc4_debugfs.c
@@ -15,26 +15,82 @@
 #include "vc4_drv.h"
 #include "vc4_regs.h"
 
-static const struct drm_info_list vc4_debugfs_list[] = {
-	{"bo_stats", vc4_bo_stats_debugfs, 0},
-	{"dpi_regs", vc4_dpi_debugfs_regs, 0},
-	{"dsi1_regs", vc4_dsi_debugfs_regs, 0, (void *)(uintptr_t)1},
-	{"hdmi_regs", vc4_hdmi_debugfs_regs, 0},
-	{"vec_regs", vc4_vec_debugfs_regs, 0},
-	{"txp_regs", vc4_txp_debugfs_regs, 0},
-	{"hvs_regs", vc4_hvs_debugfs_regs, 0},
-	{"crtc0_regs", vc4_crtc_debugfs_regs, 0, (void *)(uintptr_t)0},
-	{"crtc1_regs", vc4_crtc_debugfs_regs, 0, (void *)(uintptr_t)1},
-	{"crtc2_regs", vc4_crtc_debugfs_regs, 0, (void *)(uintptr_t)2},
-	{"v3d_ident", vc4_v3d_debugfs_ident, 0},
-	{"v3d_regs", vc4_v3d_debugfs_regs, 0},
+struct vc4_debugfs_info_entry {
+	struct list_head link;
+	struct drm_info_list info;
 };
 
-#define VC4_DEBUGFS_ENTRIES ARRAY_SIZE(vc4_debugfs_list)
-
+/**
+ * Called at drm_dev_register() time on each of the minors registered
+ * by the DRM device, to attach the debugfs files.
+ */
 int
 vc4_debugfs_init(struct drm_minor *minor)
 {
-	return drm_debugfs_create_files(vc4_debugfs_list, VC4_DEBUGFS_ENTRIES,
-					minor->debugfs_root, minor);
+	struct vc4_dev *vc4 = to_vc4_dev(minor->dev);
+	struct vc4_debugfs_info_entry *entry;
+	struct dentry *dentry;
+
+	dentry = debugfs_create_bool("hvs_load_tracker", S_IRUGO | S_IWUSR,
+				     minor->debugfs_root,
+				     &vc4->load_tracker_enabled);
+	if (!dentry)
+		return -ENOMEM;
+
+	list_for_each_entry(entry, &vc4->debugfs_list, link) {
+		int ret = drm_debugfs_create_files(&entry->info, 1,
+						   minor->debugfs_root, minor);
+
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int vc4_debugfs_regset32(struct seq_file *m, void *unused)
+{
+	struct drm_info_node *node = (struct drm_info_node *)m->private;
+	struct debugfs_regset32 *regset = node->info_ent->data;
+	struct drm_printer p = drm_seq_file_printer(m);
+
+	drm_print_regset32(&p, regset);
+
+	return 0;
+}
+
+/**
+ * Registers a debugfs file with a callback function for a vc4 component.
+ *
+ * This is like drm_debugfs_create_files(), but that can only be
+ * called a given DRM minor, while the various VC4 components want to
+ * register their debugfs files during the component bind process.  We
+ * track the request and delay it to be called on each minor during
+ * vc4_debugfs_init().
+ */
+void vc4_debugfs_add_file(struct drm_device *dev,
+			  const char *name,
+			  int (*show)(struct seq_file*, void*),
+			  void *data)
+{
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+
+	struct vc4_debugfs_info_entry *entry =
+		devm_kzalloc(dev->dev, sizeof(*entry), GFP_KERNEL);
+
+	if (!entry)
+		return;
+
+	entry->info.name = name;
+	entry->info.show = show;
+	entry->info.data = data;
+
+	list_add(&entry->link, &vc4->debugfs_list);
+}
+
+void vc4_debugfs_add_regset32(struct drm_device *drm,
+			      const char *name,
+			      struct debugfs_regset32 *regset)
+{
+	vc4_debugfs_add_file(drm, name, vc4_debugfs_regset32, regset);
 }
diff --git a/drivers/gpu/drm/vc4/vc4_dpi.c b/drivers/gpu/drm/vc4/vc4_dpi.c
index 169521e547ba..34f90ca8f479 100644
--- a/drivers/gpu/drm/vc4/vc4_dpi.c
+++ b/drivers/gpu/drm/vc4/vc4_dpi.c
@@ -101,6 +101,8 @@ struct vc4_dpi {
 
 	struct clk *pixel_clock;
 	struct clk *core_clock;
+
+	struct debugfs_regset32 regset;
 };
 
 #define DPI_READ(offset) readl(dpi->regs + (offset))
@@ -118,37 +120,11 @@ to_vc4_dpi_encoder(struct drm_encoder *encoder)
 	return container_of(encoder, struct vc4_dpi_encoder, base.base);
 }
 
-#define DPI_REG(reg) { reg, #reg }
-static const struct {
-	u32 reg;
-	const char *name;
-} dpi_regs[] = {
-	DPI_REG(DPI_C),
-	DPI_REG(DPI_ID),
+static const struct debugfs_reg32 dpi_regs[] = {
+	VC4_REG32(DPI_C),
+	VC4_REG32(DPI_ID),
 };
 
-#ifdef CONFIG_DEBUG_FS
-int vc4_dpi_debugfs_regs(struct seq_file *m, void *unused)
-{
-	struct drm_info_node *node = (struct drm_info_node *)m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct vc4_dev *vc4 = to_vc4_dev(dev);
-	struct vc4_dpi *dpi = vc4->dpi;
-	int i;
-
-	if (!dpi)
-		return 0;
-
-	for (i = 0; i < ARRAY_SIZE(dpi_regs); i++) {
-		seq_printf(m, "%s (0x%04x): 0x%08x\n",
-			   dpi_regs[i].name, dpi_regs[i].reg,
-			   DPI_READ(dpi_regs[i].reg));
-	}
-
-	return 0;
-}
-#endif
-
 static const struct drm_encoder_funcs vc4_dpi_encoder_funcs = {
 	.destroy = drm_encoder_cleanup,
 };
@@ -314,6 +290,9 @@ static int vc4_dpi_bind(struct device *dev, struct device *master, void *data)
 	dpi->regs = vc4_ioremap_regs(pdev, 0);
 	if (IS_ERR(dpi->regs))
 		return PTR_ERR(dpi->regs);
+	dpi->regset.base = dpi->regs;
+	dpi->regset.regs = dpi_regs;
+	dpi->regset.nregs = ARRAY_SIZE(dpi_regs);
 
 	if (DPI_READ(DPI_ID) != DPI_ID_VALUE) {
 		dev_err(dev, "Port returned 0x%08x for ID instead of 0x%08x\n",
@@ -352,6 +331,8 @@ static int vc4_dpi_bind(struct device *dev, struct device *master, void *data)
 
 	vc4->dpi = dpi;
 
+	vc4_debugfs_add_regset32(drm, "dpi_regs", &dpi->regset);
+
 	return 0;
 
 err_destroy_encoder:
diff --git a/drivers/gpu/drm/vc4/vc4_drv.c b/drivers/gpu/drm/vc4/vc4_drv.c
index 5fcd2f0da7f7..6d9be20a32be 100644
--- a/drivers/gpu/drm/vc4/vc4_drv.c
+++ b/drivers/gpu/drm/vc4/vc4_drv.c
@@ -72,30 +72,30 @@ static int vc4_get_param_ioctl(struct drm_device *dev, void *data,
 	if (args->pad != 0)
 		return -EINVAL;
 
+	if (!vc4->v3d)
+		return -ENODEV;
+
 	switch (args->param) {
 	case DRM_VC4_PARAM_V3D_IDENT0:
-		ret = pm_runtime_get_sync(&vc4->v3d->pdev->dev);
-		if (ret < 0)
+		ret = vc4_v3d_pm_get(vc4);
+		if (ret)
 			return ret;
 		args->value = V3D_READ(V3D_IDENT0);
-		pm_runtime_mark_last_busy(&vc4->v3d->pdev->dev);
-		pm_runtime_put_autosuspend(&vc4->v3d->pdev->dev);
+		vc4_v3d_pm_put(vc4);
 		break;
 	case DRM_VC4_PARAM_V3D_IDENT1:
-		ret = pm_runtime_get_sync(&vc4->v3d->pdev->dev);
-		if (ret < 0)
+		ret = vc4_v3d_pm_get(vc4);
+		if (ret)
 			return ret;
 		args->value = V3D_READ(V3D_IDENT1);
-		pm_runtime_mark_last_busy(&vc4->v3d->pdev->dev);
-		pm_runtime_put_autosuspend(&vc4->v3d->pdev->dev);
+		vc4_v3d_pm_put(vc4);
 		break;
 	case DRM_VC4_PARAM_V3D_IDENT2:
-		ret = pm_runtime_get_sync(&vc4->v3d->pdev->dev);
-		if (ret < 0)
+		ret = vc4_v3d_pm_get(vc4);
+		if (ret)
 			return ret;
 		args->value = V3D_READ(V3D_IDENT2);
-		pm_runtime_mark_last_busy(&vc4->v3d->pdev->dev);
-		pm_runtime_put_autosuspend(&vc4->v3d->pdev->dev);
+		vc4_v3d_pm_put(vc4);
 		break;
 	case DRM_VC4_PARAM_SUPPORTS_BRANCHES:
 	case DRM_VC4_PARAM_SUPPORTS_ETC1:
@@ -200,7 +200,6 @@ static struct drm_driver vc4_drm_driver = {
 	.prime_fd_to_handle = drm_gem_prime_fd_to_handle,
 	.gem_prime_import = drm_gem_prime_import,
 	.gem_prime_export = vc4_prime_export,
-	.gem_prime_res_obj = vc4_prime_res_obj,
 	.gem_prime_get_sg_table	= drm_gem_cma_prime_get_sg_table,
 	.gem_prime_import_sg_table = vc4_prime_import_sg_table,
 	.gem_prime_vmap = vc4_prime_vmap,
@@ -252,6 +251,7 @@ static int vc4_drm_bind(struct device *dev)
 	struct platform_device *pdev = to_platform_device(dev);
 	struct drm_device *drm;
 	struct vc4_dev *vc4;
+	struct device_node *node;
 	int ret = 0;
 
 	dev->coherent_dma_mask = DMA_BIT_MASK(32);
@@ -260,12 +260,19 @@ static int vc4_drm_bind(struct device *dev)
 	if (!vc4)
 		return -ENOMEM;
 
+	/* If VC4 V3D is missing, don't advertise render nodes. */
+	node = of_find_matching_node_and_match(NULL, vc4_v3d_dt_match, NULL);
+	if (!node || !of_device_is_available(node))
+		vc4_drm_driver.driver_features &= ~DRIVER_RENDER;
+	of_node_put(node);
+
 	drm = drm_dev_alloc(&vc4_drm_driver, dev);
 	if (IS_ERR(drm))
 		return PTR_ERR(drm);
 	platform_set_drvdata(pdev, drm);
 	vc4->dev = drm;
 	drm->dev_private = vc4;
+	INIT_LIST_HEAD(&vc4->debugfs_list);
 
 	ret = vc4_bo_cache_init(drm);
 	if (ret)
@@ -281,13 +288,15 @@ static int vc4_drm_bind(struct device *dev)
 
 	drm_fb_helper_remove_conflicting_framebuffers(NULL, "vc4drmfb", false);
 
-	ret = drm_dev_register(drm, 0);
+	ret = vc4_kms_load(drm);
 	if (ret < 0)
 		goto unbind_all;
 
-	vc4_kms_load(drm);
+	ret = drm_dev_register(drm, 0);
+	if (ret < 0)
+		goto unbind_all;
 
-	drm_fbdev_generic_setup(drm, 32);
+	drm_fbdev_generic_setup(drm, 16);
 
 	return 0;
 
@@ -312,6 +321,7 @@ static void vc4_drm_unbind(struct device *dev)
 
 	drm_mode_config_cleanup(drm);
 
+	drm_atomic_private_obj_fini(&vc4->load_tracker);
 	drm_atomic_private_obj_fini(&vc4->ctm_manager);
 
 	drm_dev_put(drm);
diff --git a/drivers/gpu/drm/vc4/vc4_drv.h b/drivers/gpu/drm/vc4/vc4_drv.h
index 2c635f001c71..4f13f6262491 100644
--- a/drivers/gpu/drm/vc4/vc4_drv.h
+++ b/drivers/gpu/drm/vc4/vc4_drv.h
@@ -7,7 +7,6 @@
  */
 
 #include <linux/mm_types.h>
-#include <linux/reservation.h>
 #include <drm/drmP.h>
 #include <drm/drm_util.h>
 #include <drm/drm_encoder.h>
@@ -185,10 +184,20 @@ struct vc4_dev {
 	/* Bitmask of the current bin_alloc used for overflow memory. */
 	uint32_t bin_alloc_overflow;
 
+	/* Incremented when an underrun error happened after an atomic commit.
+	 * This is particularly useful to detect when a specific modeset is too
+	 * demanding in term of memory or HVS bandwidth which is hard to guess
+	 * at atomic check time.
+	 */
+	atomic_t underrun;
+
 	struct work_struct overflow_mem_work;
 
 	int power_refcount;
 
+	/* Set to true when the load tracker is active. */
+	bool load_tracker_enabled;
+
 	/* Mutex controlling the power refcount. */
 	struct mutex power_lock;
 
@@ -201,6 +210,12 @@ struct vc4_dev {
 
 	struct drm_modeset_lock ctm_state_lock;
 	struct drm_private_obj ctm_manager;
+	struct drm_private_obj load_tracker;
+
+	/* List of vc4_debugfs_info_entry for adding to debugfs once
+	 * the minor is available (after drm_dev_register()).
+	 */
+	struct list_head debugfs_list;
 };
 
 static inline struct vc4_dev *
@@ -240,10 +255,6 @@ struct vc4_bo {
 	 */
 	struct vc4_validated_shader_info *validated_shader;
 
-	/* normally (resv == &_resv) except for imported bo's */
-	struct reservation_object *resv;
-	struct reservation_object _resv;
-
 	/* One of enum vc4_kernel_bo_type, or VC4_BO_TYPE_COUNT + i
 	 * for user-allocated labels.
 	 */
@@ -290,6 +301,7 @@ struct vc4_v3d {
 	struct platform_device *pdev;
 	void __iomem *regs;
 	struct clk *clk;
+	struct debugfs_regset32 regset;
 };
 
 struct vc4_hvs {
@@ -306,6 +318,7 @@ struct vc4_hvs {
 	spinlock_t mm_lock;
 
 	struct drm_mm_node mitchell_netravali_filter;
+	struct debugfs_regset32 regset;
 };
 
 struct vc4_plane {
@@ -376,6 +389,16 @@ struct vc4_plane_state {
 	 * when async update is not possible.
 	 */
 	bool dlist_initialized;
+
+	/* Load of this plane on the HVS block. The load is expressed in HVS
+	 * cycles/sec.
+	 */
+	u64 hvs_load;
+
+	/* Memory bandwidth needed for this plane. This is expressed in
+	 * bytes/sec.
+	 */
+	u64 membus_load;
 };
 
 static inline struct vc4_plane_state *
@@ -411,10 +434,12 @@ struct vc4_crtc_data {
 	int hvs_channel;
 
 	enum vc4_encoder_type encoder_types[4];
+	const char *debugfs_name;
 };
 
 struct vc4_crtc {
 	struct drm_crtc base;
+	struct platform_device *pdev;
 	const struct vc4_crtc_data *data;
 	void __iomem *regs;
 
@@ -431,6 +456,8 @@ struct vc4_crtc {
 	u32 cob_size;
 
 	struct drm_pending_vblank_event *event;
+
+	struct debugfs_regset32 regset;
 };
 
 static inline struct vc4_crtc *
@@ -444,6 +471,8 @@ to_vc4_crtc(struct drm_crtc *crtc)
 #define HVS_READ(offset) readl(vc4->hvs->regs + offset)
 #define HVS_WRITE(offset, val) writel(val, vc4->hvs->regs + offset)
 
+#define VC4_REG32(reg) { .name = #reg, .offset = reg }
+
 struct vc4_exec_info {
 	/* Sequence number for this bin/render job. */
 	uint64_t seqno;
@@ -685,7 +714,6 @@ int vc4_label_bo_ioctl(struct drm_device *dev, void *data,
 		       struct drm_file *file_priv);
 vm_fault_t vc4_fault(struct vm_fault *vmf);
 int vc4_mmap(struct file *filp, struct vm_area_struct *vma);
-struct reservation_object *vc4_prime_res_obj(struct drm_gem_object *obj);
 int vc4_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma);
 struct drm_gem_object *vc4_prime_import_sg_table(struct drm_device *dev,
 						 struct dma_buf_attachment *attach,
@@ -693,7 +721,6 @@ struct drm_gem_object *vc4_prime_import_sg_table(struct drm_device *dev,
 void *vc4_prime_vmap(struct drm_gem_object *obj);
 int vc4_bo_cache_init(struct drm_device *dev);
 void vc4_bo_cache_destroy(struct drm_device *dev);
-int vc4_bo_stats_debugfs(struct seq_file *m, void *arg);
 int vc4_bo_inc_usecnt(struct vc4_bo *bo);
 void vc4_bo_dec_usecnt(struct vc4_bo *bo);
 void vc4_bo_add_to_purgeable_pool(struct vc4_bo *bo);
@@ -701,7 +728,6 @@ void vc4_bo_remove_from_purgeable_pool(struct vc4_bo *bo);
 
 /* vc4_crtc.c */
 extern struct platform_driver vc4_crtc_driver;
-int vc4_crtc_debugfs_regs(struct seq_file *m, void *arg);
 bool vc4_crtc_get_scanoutpos(struct drm_device *dev, unsigned int crtc_id,
 			     bool in_vblank_irq, int *vpos, int *hpos,
 			     ktime_t *stime, ktime_t *etime,
@@ -714,17 +740,37 @@ void vc4_crtc_get_margins(struct drm_crtc_state *state,
 
 /* vc4_debugfs.c */
 int vc4_debugfs_init(struct drm_minor *minor);
+#ifdef CONFIG_DEBUG_FS
+void vc4_debugfs_add_file(struct drm_device *drm,
+			  const char *filename,
+			  int (*show)(struct seq_file*, void*),
+			  void *data);
+void vc4_debugfs_add_regset32(struct drm_device *drm,
+			      const char *filename,
+			      struct debugfs_regset32 *regset);
+#else
+static inline void vc4_debugfs_add_file(struct drm_device *drm,
+					const char *filename,
+					int (*show)(struct seq_file*, void*),
+					void *data)
+{
+}
+
+static inline void vc4_debugfs_add_regset32(struct drm_device *drm,
+					    const char *filename,
+					    struct debugfs_regset32 *regset)
+{
+}
+#endif
 
 /* vc4_drv.c */
 void __iomem *vc4_ioremap_regs(struct platform_device *dev, int index);
 
 /* vc4_dpi.c */
 extern struct platform_driver vc4_dpi_driver;
-int vc4_dpi_debugfs_regs(struct seq_file *m, void *unused);
 
 /* vc4_dsi.c */
 extern struct platform_driver vc4_dsi_driver;
-int vc4_dsi_debugfs_regs(struct seq_file *m, void *unused);
 
 /* vc4_fence.c */
 extern const struct dma_fence_ops vc4_fence_ops;
@@ -752,15 +798,12 @@ int vc4_gem_madvise_ioctl(struct drm_device *dev, void *data,
 
 /* vc4_hdmi.c */
 extern struct platform_driver vc4_hdmi_driver;
-int vc4_hdmi_debugfs_regs(struct seq_file *m, void *unused);
 
 /* vc4_vec.c */
 extern struct platform_driver vc4_vec_driver;
-int vc4_vec_debugfs_regs(struct seq_file *m, void *unused);
 
 /* vc4_txp.c */
 extern struct platform_driver vc4_txp_driver;
-int vc4_txp_debugfs_regs(struct seq_file *m, void *unused);
 
 /* vc4_irq.c */
 irqreturn_t vc4_irq(int irq, void *arg);
@@ -772,7 +815,8 @@ void vc4_irq_reset(struct drm_device *dev);
 /* vc4_hvs.c */
 extern struct platform_driver vc4_hvs_driver;
 void vc4_hvs_dump_state(struct drm_device *dev);
-int vc4_hvs_debugfs_regs(struct seq_file *m, void *unused);
+void vc4_hvs_unmask_underrun(struct drm_device *dev, int channel);
+void vc4_hvs_mask_underrun(struct drm_device *dev, int channel);
 
 /* vc4_kms.c */
 int vc4_kms_load(struct drm_device *dev);
@@ -787,9 +831,10 @@ void vc4_plane_async_set_fb(struct drm_plane *plane,
 
 /* vc4_v3d.c */
 extern struct platform_driver vc4_v3d_driver;
-int vc4_v3d_debugfs_ident(struct seq_file *m, void *unused);
-int vc4_v3d_debugfs_regs(struct seq_file *m, void *unused);
+extern const struct of_device_id vc4_v3d_dt_match[];
 int vc4_v3d_get_bin_slot(struct vc4_dev *vc4);
+int vc4_v3d_pm_get(struct vc4_dev *vc4);
+void vc4_v3d_pm_put(struct vc4_dev *vc4);
 
 /* vc4_validate.c */
 int
diff --git a/drivers/gpu/drm/vc4/vc4_dsi.c b/drivers/gpu/drm/vc4/vc4_dsi.c
index 11702e1d9011..9412709067f5 100644
--- a/drivers/gpu/drm/vc4/vc4_dsi.c
+++ b/drivers/gpu/drm/vc4/vc4_dsi.c
@@ -545,6 +545,8 @@ struct vc4_dsi {
 
 	struct completion xfer_completion;
 	int xfer_result;
+
+	struct debugfs_regset32 regset;
 };
 
 #define host_to_dsi(host) container_of(host, struct vc4_dsi, dsi_host)
@@ -605,113 +607,56 @@ to_vc4_dsi_encoder(struct drm_encoder *encoder)
 	return container_of(encoder, struct vc4_dsi_encoder, base.base);
 }
 
-#define DSI_REG(reg) { reg, #reg }
-static const struct {
-	u32 reg;
-	const char *name;
-} dsi0_regs[] = {
-	DSI_REG(DSI0_CTRL),
-	DSI_REG(DSI0_STAT),
-	DSI_REG(DSI0_HSTX_TO_CNT),
-	DSI_REG(DSI0_LPRX_TO_CNT),
-	DSI_REG(DSI0_TA_TO_CNT),
-	DSI_REG(DSI0_PR_TO_CNT),
-	DSI_REG(DSI0_DISP0_CTRL),
-	DSI_REG(DSI0_DISP1_CTRL),
-	DSI_REG(DSI0_INT_STAT),
-	DSI_REG(DSI0_INT_EN),
-	DSI_REG(DSI0_PHYC),
-	DSI_REG(DSI0_HS_CLT0),
-	DSI_REG(DSI0_HS_CLT1),
-	DSI_REG(DSI0_HS_CLT2),
-	DSI_REG(DSI0_HS_DLT3),
-	DSI_REG(DSI0_HS_DLT4),
-	DSI_REG(DSI0_HS_DLT5),
-	DSI_REG(DSI0_HS_DLT6),
-	DSI_REG(DSI0_HS_DLT7),
-	DSI_REG(DSI0_PHY_AFEC0),
-	DSI_REG(DSI0_PHY_AFEC1),
-	DSI_REG(DSI0_ID),
+static const struct debugfs_reg32 dsi0_regs[] = {
+	VC4_REG32(DSI0_CTRL),
+	VC4_REG32(DSI0_STAT),
+	VC4_REG32(DSI0_HSTX_TO_CNT),
+	VC4_REG32(DSI0_LPRX_TO_CNT),
+	VC4_REG32(DSI0_TA_TO_CNT),
+	VC4_REG32(DSI0_PR_TO_CNT),
+	VC4_REG32(DSI0_DISP0_CTRL),
+	VC4_REG32(DSI0_DISP1_CTRL),
+	VC4_REG32(DSI0_INT_STAT),
+	VC4_REG32(DSI0_INT_EN),
+	VC4_REG32(DSI0_PHYC),
+	VC4_REG32(DSI0_HS_CLT0),
+	VC4_REG32(DSI0_HS_CLT1),
+	VC4_REG32(DSI0_HS_CLT2),
+	VC4_REG32(DSI0_HS_DLT3),
+	VC4_REG32(DSI0_HS_DLT4),
+	VC4_REG32(DSI0_HS_DLT5),
+	VC4_REG32(DSI0_HS_DLT6),
+	VC4_REG32(DSI0_HS_DLT7),
+	VC4_REG32(DSI0_PHY_AFEC0),
+	VC4_REG32(DSI0_PHY_AFEC1),
+	VC4_REG32(DSI0_ID),
 };
 
-static const struct {
-	u32 reg;
-	const char *name;
-} dsi1_regs[] = {
-	DSI_REG(DSI1_CTRL),
-	DSI_REG(DSI1_STAT),
-	DSI_REG(DSI1_HSTX_TO_CNT),
-	DSI_REG(DSI1_LPRX_TO_CNT),
-	DSI_REG(DSI1_TA_TO_CNT),
-	DSI_REG(DSI1_PR_TO_CNT),
-	DSI_REG(DSI1_DISP0_CTRL),
-	DSI_REG(DSI1_DISP1_CTRL),
-	DSI_REG(DSI1_INT_STAT),
-	DSI_REG(DSI1_INT_EN),
-	DSI_REG(DSI1_PHYC),
-	DSI_REG(DSI1_HS_CLT0),
-	DSI_REG(DSI1_HS_CLT1),
-	DSI_REG(DSI1_HS_CLT2),
-	DSI_REG(DSI1_HS_DLT3),
-	DSI_REG(DSI1_HS_DLT4),
-	DSI_REG(DSI1_HS_DLT5),
-	DSI_REG(DSI1_HS_DLT6),
-	DSI_REG(DSI1_HS_DLT7),
-	DSI_REG(DSI1_PHY_AFEC0),
-	DSI_REG(DSI1_PHY_AFEC1),
-	DSI_REG(DSI1_ID),
+static const struct debugfs_reg32 dsi1_regs[] = {
+	VC4_REG32(DSI1_CTRL),
+	VC4_REG32(DSI1_STAT),
+	VC4_REG32(DSI1_HSTX_TO_CNT),
+	VC4_REG32(DSI1_LPRX_TO_CNT),
+	VC4_REG32(DSI1_TA_TO_CNT),
+	VC4_REG32(DSI1_PR_TO_CNT),
+	VC4_REG32(DSI1_DISP0_CTRL),
+	VC4_REG32(DSI1_DISP1_CTRL),
+	VC4_REG32(DSI1_INT_STAT),
+	VC4_REG32(DSI1_INT_EN),
+	VC4_REG32(DSI1_PHYC),
+	VC4_REG32(DSI1_HS_CLT0),
+	VC4_REG32(DSI1_HS_CLT1),
+	VC4_REG32(DSI1_HS_CLT2),
+	VC4_REG32(DSI1_HS_DLT3),
+	VC4_REG32(DSI1_HS_DLT4),
+	VC4_REG32(DSI1_HS_DLT5),
+	VC4_REG32(DSI1_HS_DLT6),
+	VC4_REG32(DSI1_HS_DLT7),
+	VC4_REG32(DSI1_PHY_AFEC0),
+	VC4_REG32(DSI1_PHY_AFEC1),
+	VC4_REG32(DSI1_ID),
 };
 
-static void vc4_dsi_dump_regs(struct vc4_dsi *dsi)
-{
-	int i;
-
-	if (dsi->port == 0) {
-		for (i = 0; i < ARRAY_SIZE(dsi0_regs); i++) {
-			DRM_INFO("0x%04x (%s): 0x%08x\n",
-				 dsi0_regs[i].reg, dsi0_regs[i].name,
-				 DSI_READ(dsi0_regs[i].reg));
-		}
-	} else {
-		for (i = 0; i < ARRAY_SIZE(dsi1_regs); i++) {
-			DRM_INFO("0x%04x (%s): 0x%08x\n",
-				 dsi1_regs[i].reg, dsi1_regs[i].name,
-				 DSI_READ(dsi1_regs[i].reg));
-		}
-	}
-}
-
-#ifdef CONFIG_DEBUG_FS
-int vc4_dsi_debugfs_regs(struct seq_file *m, void *unused)
-{
-	struct drm_info_node *node = (struct drm_info_node *)m->private;
-	struct drm_device *drm = node->minor->dev;
-	struct vc4_dev *vc4 = to_vc4_dev(drm);
-	int dsi_index = (uintptr_t)node->info_ent->data;
-	struct vc4_dsi *dsi = (dsi_index == 1 ? vc4->dsi1 : NULL);
-	int i;
-
-	if (!dsi)
-		return 0;
-
-	if (dsi->port == 0) {
-		for (i = 0; i < ARRAY_SIZE(dsi0_regs); i++) {
-			seq_printf(m, "0x%04x (%s): 0x%08x\n",
-				   dsi0_regs[i].reg, dsi0_regs[i].name,
-				   DSI_READ(dsi0_regs[i].reg));
-		}
-	} else {
-		for (i = 0; i < ARRAY_SIZE(dsi1_regs); i++) {
-			seq_printf(m, "0x%04x (%s): 0x%08x\n",
-				   dsi1_regs[i].reg, dsi1_regs[i].name,
-				   DSI_READ(dsi1_regs[i].reg));
-		}
-	}
-
-	return 0;
-}
-#endif
-
 static void vc4_dsi_encoder_destroy(struct drm_encoder *encoder)
 {
 	drm_encoder_cleanup(encoder);
@@ -900,8 +845,9 @@ static void vc4_dsi_encoder_enable(struct drm_encoder *encoder)
 	}
 
 	if (debug_dump_regs) {
-		DRM_INFO("DSI regs before:\n");
-		vc4_dsi_dump_regs(dsi);
+		struct drm_printer p = drm_info_printer(&dsi->pdev->dev);
+		dev_info(&dsi->pdev->dev, "DSI regs before:\n");
+		drm_print_regset32(&p, &dsi->regset);
 	}
 
 	/* Round up the clk_set_rate() request slightly, since
@@ -1135,8 +1081,9 @@ static void vc4_dsi_encoder_enable(struct drm_encoder *encoder)
 	drm_bridge_enable(dsi->bridge);
 
 	if (debug_dump_regs) {
-		DRM_INFO("DSI regs after:\n");
-		vc4_dsi_dump_regs(dsi);
+		struct drm_printer p = drm_info_printer(&dsi->pdev->dev);
+		dev_info(&dsi->pdev->dev, "DSI regs after:\n");
+		drm_print_regset32(&p, &dsi->regset);
 	}
 }
 
@@ -1527,6 +1474,15 @@ static int vc4_dsi_bind(struct device *dev, struct device *master, void *data)
 	if (IS_ERR(dsi->regs))
 		return PTR_ERR(dsi->regs);
 
+	dsi->regset.base = dsi->regs;
+	if (dsi->port == 0) {
+		dsi->regset.regs = dsi0_regs;
+		dsi->regset.nregs = ARRAY_SIZE(dsi0_regs);
+	} else {
+		dsi->regset.regs = dsi1_regs;
+		dsi->regset.nregs = ARRAY_SIZE(dsi1_regs);
+	}
+
 	if (DSI_PORT_READ(ID) != DSI_ID_VALUE) {
 		dev_err(dev, "Port returned 0x%08x for ID instead of 0x%08x\n",
 			DSI_PORT_READ(ID), DSI_ID_VALUE);
@@ -1662,6 +1618,11 @@ static int vc4_dsi_bind(struct device *dev, struct device *master, void *data)
 	 */
 	dsi->encoder->bridge = NULL;
 
+	if (dsi->port == 0)
+		vc4_debugfs_add_regset32(drm, "dsi0_regs", &dsi->regset);
+	else
+		vc4_debugfs_add_regset32(drm, "dsi1_regs", &dsi->regset);
+
 	pm_runtime_enable(dev);
 
 	return 0;
diff --git a/drivers/gpu/drm/vc4/vc4_gem.c b/drivers/gpu/drm/vc4/vc4_gem.c
index aea2b8dfec17..d9311be32a4f 100644
--- a/drivers/gpu/drm/vc4/vc4_gem.c
+++ b/drivers/gpu/drm/vc4/vc4_gem.c
@@ -74,6 +74,11 @@ vc4_get_hang_state_ioctl(struct drm_device *dev, void *data,
 	u32 i;
 	int ret = 0;
 
+	if (!vc4->v3d) {
+		DRM_DEBUG("VC4_GET_HANG_STATE with no VC4 V3D probed\n");
+		return -ENODEV;
+	}
+
 	spin_lock_irqsave(&vc4->job_lock, irqflags);
 	kernel_state = vc4->hang_state;
 	if (!kernel_state) {
@@ -536,7 +541,7 @@ vc4_update_bo_seqnos(struct vc4_exec_info *exec, uint64_t seqno)
 		bo = to_vc4_bo(&exec->bo[i]->base);
 		bo->seqno = seqno;
 
-		reservation_object_add_shared_fence(bo->resv, exec->fence);
+		reservation_object_add_shared_fence(bo->base.base.resv, exec->fence);
 	}
 
 	list_for_each_entry(bo, &exec->unref_list, unref_head) {
@@ -547,7 +552,7 @@ vc4_update_bo_seqnos(struct vc4_exec_info *exec, uint64_t seqno)
 		bo = to_vc4_bo(&exec->rcl_write_bo[i]->base);
 		bo->write_seqno = seqno;
 
-		reservation_object_add_excl_fence(bo->resv, exec->fence);
+		reservation_object_add_excl_fence(bo->base.base.resv, exec->fence);
 	}
 }
 
@@ -559,7 +564,7 @@ vc4_unlock_bo_reservations(struct drm_device *dev,
 	int i;
 
 	for (i = 0; i < exec->bo_count; i++) {
-		struct vc4_bo *bo = to_vc4_bo(&exec->bo[i]->base);
+		struct drm_gem_object *bo = &exec->bo[i]->base;
 
 		ww_mutex_unlock(&bo->resv->lock);
 	}
@@ -581,13 +586,13 @@ vc4_lock_bo_reservations(struct drm_device *dev,
 {
 	int contended_lock = -1;
 	int i, ret;
-	struct vc4_bo *bo;
+	struct drm_gem_object *bo;
 
 	ww_acquire_init(acquire_ctx, &reservation_ww_class);
 
 retry:
 	if (contended_lock != -1) {
-		bo = to_vc4_bo(&exec->bo[contended_lock]->base);
+		bo = &exec->bo[contended_lock]->base;
 		ret = ww_mutex_lock_slow_interruptible(&bo->resv->lock,
 						       acquire_ctx);
 		if (ret) {
@@ -600,19 +605,19 @@ retry:
 		if (i == contended_lock)
 			continue;
 
-		bo = to_vc4_bo(&exec->bo[i]->base);
+		bo = &exec->bo[i]->base;
 
 		ret = ww_mutex_lock_interruptible(&bo->resv->lock, acquire_ctx);
 		if (ret) {
 			int j;
 
 			for (j = 0; j < i; j++) {
-				bo = to_vc4_bo(&exec->bo[j]->base);
+				bo = &exec->bo[j]->base;
 				ww_mutex_unlock(&bo->resv->lock);
 			}
 
 			if (contended_lock != -1 && contended_lock >= i) {
-				bo = to_vc4_bo(&exec->bo[contended_lock]->base);
+				bo = &exec->bo[contended_lock]->base;
 
 				ww_mutex_unlock(&bo->resv->lock);
 			}
@@ -633,7 +638,7 @@ retry:
 	 * before we commit the CL to the hardware.
 	 */
 	for (i = 0; i < exec->bo_count; i++) {
-		bo = to_vc4_bo(&exec->bo[i]->base);
+		bo = &exec->bo[i]->base;
 
 		ret = reservation_object_reserve_shared(bo->resv, 1);
 		if (ret) {
@@ -964,12 +969,7 @@ vc4_complete_exec(struct drm_device *dev, struct vc4_exec_info *exec)
 	/* Release the reference we had on the perf monitor. */
 	vc4_perfmon_put(exec->perfmon);
 
-	mutex_lock(&vc4->power_lock);
-	if (--vc4->power_refcount == 0) {
-		pm_runtime_mark_last_busy(&vc4->v3d->pdev->dev);
-		pm_runtime_put_autosuspend(&vc4->v3d->pdev->dev);
-	}
-	mutex_unlock(&vc4->power_lock);
+	vc4_v3d_pm_put(vc4);
 
 	kfree(exec);
 }
@@ -1124,6 +1124,11 @@ vc4_submit_cl_ioctl(struct drm_device *dev, void *data,
 	struct dma_fence *in_fence;
 	int ret = 0;
 
+	if (!vc4->v3d) {
+		DRM_DEBUG("VC4_SUBMIT_CL with no VC4 V3D probed\n");
+		return -ENODEV;
+	}
+
 	if ((args->flags & ~(VC4_SUBMIT_CL_USE_CLEAR_COLOR |
 			     VC4_SUBMIT_CL_FIXED_RCL_ORDER |
 			     VC4_SUBMIT_CL_RCL_ORDER_INCREASING_X |
@@ -1143,17 +1148,11 @@ vc4_submit_cl_ioctl(struct drm_device *dev, void *data,
 		return -ENOMEM;
 	}
 
-	mutex_lock(&vc4->power_lock);
-	if (vc4->power_refcount++ == 0) {
-		ret = pm_runtime_get_sync(&vc4->v3d->pdev->dev);
-		if (ret < 0) {
-			mutex_unlock(&vc4->power_lock);
-			vc4->power_refcount--;
-			kfree(exec);
-			return ret;
-		}
+	ret = vc4_v3d_pm_get(vc4);
+	if (ret) {
+		kfree(exec);
+		return ret;
 	}
-	mutex_unlock(&vc4->power_lock);
 
 	exec->args = args;
 	INIT_LIST_HEAD(&exec->unref_list);
diff --git a/drivers/gpu/drm/vc4/vc4_hdmi.c b/drivers/gpu/drm/vc4/vc4_hdmi.c
index 88fd5df7e7dc..99fc8569e0f5 100644
--- a/drivers/gpu/drm/vc4/vc4_hdmi.c
+++ b/drivers/gpu/drm/vc4/vc4_hdmi.c
@@ -97,6 +97,9 @@ struct vc4_hdmi {
 
 	struct clk *pixel_clock;
 	struct clk *hsm_clock;
+
+	struct debugfs_regset32 hdmi_regset;
+	struct debugfs_regset32 hd_regset;
 };
 
 #define HDMI_READ(offset) readl(vc4->hdmi->hdmicore_regs + offset)
@@ -134,103 +137,69 @@ to_vc4_hdmi_connector(struct drm_connector *connector)
 	return container_of(connector, struct vc4_hdmi_connector, base);
 }
 
-#define HDMI_REG(reg) { reg, #reg }
-static const struct {
-	u32 reg;
-	const char *name;
-} hdmi_regs[] = {
-	HDMI_REG(VC4_HDMI_CORE_REV),
-	HDMI_REG(VC4_HDMI_SW_RESET_CONTROL),
-	HDMI_REG(VC4_HDMI_HOTPLUG_INT),
-	HDMI_REG(VC4_HDMI_HOTPLUG),
-	HDMI_REG(VC4_HDMI_MAI_CHANNEL_MAP),
-	HDMI_REG(VC4_HDMI_MAI_CONFIG),
-	HDMI_REG(VC4_HDMI_MAI_FORMAT),
-	HDMI_REG(VC4_HDMI_AUDIO_PACKET_CONFIG),
-	HDMI_REG(VC4_HDMI_RAM_PACKET_CONFIG),
-	HDMI_REG(VC4_HDMI_HORZA),
-	HDMI_REG(VC4_HDMI_HORZB),
-	HDMI_REG(VC4_HDMI_FIFO_CTL),
-	HDMI_REG(VC4_HDMI_SCHEDULER_CONTROL),
-	HDMI_REG(VC4_HDMI_VERTA0),
-	HDMI_REG(VC4_HDMI_VERTA1),
-	HDMI_REG(VC4_HDMI_VERTB0),
-	HDMI_REG(VC4_HDMI_VERTB1),
-	HDMI_REG(VC4_HDMI_TX_PHY_RESET_CTL),
-	HDMI_REG(VC4_HDMI_TX_PHY_CTL0),
-
-	HDMI_REG(VC4_HDMI_CEC_CNTRL_1),
-	HDMI_REG(VC4_HDMI_CEC_CNTRL_2),
-	HDMI_REG(VC4_HDMI_CEC_CNTRL_3),
-	HDMI_REG(VC4_HDMI_CEC_CNTRL_4),
-	HDMI_REG(VC4_HDMI_CEC_CNTRL_5),
-	HDMI_REG(VC4_HDMI_CPU_STATUS),
-	HDMI_REG(VC4_HDMI_CPU_MASK_STATUS),
-
-	HDMI_REG(VC4_HDMI_CEC_RX_DATA_1),
-	HDMI_REG(VC4_HDMI_CEC_RX_DATA_2),
-	HDMI_REG(VC4_HDMI_CEC_RX_DATA_3),
-	HDMI_REG(VC4_HDMI_CEC_RX_DATA_4),
-	HDMI_REG(VC4_HDMI_CEC_TX_DATA_1),
-	HDMI_REG(VC4_HDMI_CEC_TX_DATA_2),
-	HDMI_REG(VC4_HDMI_CEC_TX_DATA_3),
-	HDMI_REG(VC4_HDMI_CEC_TX_DATA_4),
+static const struct debugfs_reg32 hdmi_regs[] = {
+	VC4_REG32(VC4_HDMI_CORE_REV),
+	VC4_REG32(VC4_HDMI_SW_RESET_CONTROL),
+	VC4_REG32(VC4_HDMI_HOTPLUG_INT),
+	VC4_REG32(VC4_HDMI_HOTPLUG),
+	VC4_REG32(VC4_HDMI_MAI_CHANNEL_MAP),
+	VC4_REG32(VC4_HDMI_MAI_CONFIG),
+	VC4_REG32(VC4_HDMI_MAI_FORMAT),
+	VC4_REG32(VC4_HDMI_AUDIO_PACKET_CONFIG),
+	VC4_REG32(VC4_HDMI_RAM_PACKET_CONFIG),
+	VC4_REG32(VC4_HDMI_HORZA),
+	VC4_REG32(VC4_HDMI_HORZB),
+	VC4_REG32(VC4_HDMI_FIFO_CTL),
+	VC4_REG32(VC4_HDMI_SCHEDULER_CONTROL),
+	VC4_REG32(VC4_HDMI_VERTA0),
+	VC4_REG32(VC4_HDMI_VERTA1),
+	VC4_REG32(VC4_HDMI_VERTB0),
+	VC4_REG32(VC4_HDMI_VERTB1),
+	VC4_REG32(VC4_HDMI_TX_PHY_RESET_CTL),
+	VC4_REG32(VC4_HDMI_TX_PHY_CTL0),
+
+	VC4_REG32(VC4_HDMI_CEC_CNTRL_1),
+	VC4_REG32(VC4_HDMI_CEC_CNTRL_2),
+	VC4_REG32(VC4_HDMI_CEC_CNTRL_3),
+	VC4_REG32(VC4_HDMI_CEC_CNTRL_4),
+	VC4_REG32(VC4_HDMI_CEC_CNTRL_5),
+	VC4_REG32(VC4_HDMI_CPU_STATUS),
+	VC4_REG32(VC4_HDMI_CPU_MASK_STATUS),
+
+	VC4_REG32(VC4_HDMI_CEC_RX_DATA_1),
+	VC4_REG32(VC4_HDMI_CEC_RX_DATA_2),
+	VC4_REG32(VC4_HDMI_CEC_RX_DATA_3),
+	VC4_REG32(VC4_HDMI_CEC_RX_DATA_4),
+	VC4_REG32(VC4_HDMI_CEC_TX_DATA_1),
+	VC4_REG32(VC4_HDMI_CEC_TX_DATA_2),
+	VC4_REG32(VC4_HDMI_CEC_TX_DATA_3),
+	VC4_REG32(VC4_HDMI_CEC_TX_DATA_4),
 };
 
-static const struct {
-	u32 reg;
-	const char *name;
-} hd_regs[] = {
-	HDMI_REG(VC4_HD_M_CTL),
-	HDMI_REG(VC4_HD_MAI_CTL),
-	HDMI_REG(VC4_HD_MAI_THR),
-	HDMI_REG(VC4_HD_MAI_FMT),
-	HDMI_REG(VC4_HD_MAI_SMP),
-	HDMI_REG(VC4_HD_VID_CTL),
-	HDMI_REG(VC4_HD_CSC_CTL),
-	HDMI_REG(VC4_HD_FRAME_COUNT),
+static const struct debugfs_reg32 hd_regs[] = {
+	VC4_REG32(VC4_HD_M_CTL),
+	VC4_REG32(VC4_HD_MAI_CTL),
+	VC4_REG32(VC4_HD_MAI_THR),
+	VC4_REG32(VC4_HD_MAI_FMT),
+	VC4_REG32(VC4_HD_MAI_SMP),
+	VC4_REG32(VC4_HD_VID_CTL),
+	VC4_REG32(VC4_HD_CSC_CTL),
+	VC4_REG32(VC4_HD_FRAME_COUNT),
 };
 
-#ifdef CONFIG_DEBUG_FS
-int vc4_hdmi_debugfs_regs(struct seq_file *m, void *unused)
+static int vc4_hdmi_debugfs_regs(struct seq_file *m, void *unused)
 {
 	struct drm_info_node *node = (struct drm_info_node *)m->private;
 	struct drm_device *dev = node->minor->dev;
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(hdmi_regs); i++) {
-		seq_printf(m, "%s (0x%04x): 0x%08x\n",
-			   hdmi_regs[i].name, hdmi_regs[i].reg,
-			   HDMI_READ(hdmi_regs[i].reg));
-	}
+	struct vc4_hdmi *hdmi = vc4->hdmi;
+	struct drm_printer p = drm_seq_file_printer(m);
 
-	for (i = 0; i < ARRAY_SIZE(hd_regs); i++) {
-		seq_printf(m, "%s (0x%04x): 0x%08x\n",
-			   hd_regs[i].name, hd_regs[i].reg,
-			   HD_READ(hd_regs[i].reg));
-	}
+	drm_print_regset32(&p, &hdmi->hdmi_regset);
+	drm_print_regset32(&p, &hdmi->hd_regset);
 
 	return 0;
 }
-#endif /* CONFIG_DEBUG_FS */
-
-static void vc4_hdmi_dump_regs(struct drm_device *dev)
-{
-	struct vc4_dev *vc4 = to_vc4_dev(dev);
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(hdmi_regs); i++) {
-		DRM_INFO("0x%04x (%s): 0x%08x\n",
-			 hdmi_regs[i].reg, hdmi_regs[i].name,
-			 HDMI_READ(hdmi_regs[i].reg));
-	}
-	for (i = 0; i < ARRAY_SIZE(hd_regs); i++) {
-		DRM_INFO("0x%04x (%s): 0x%08x\n",
-			 hd_regs[i].reg, hd_regs[i].name,
-			 HD_READ(hd_regs[i].reg));
-	}
-}
 
 static enum drm_connector_status
 vc4_hdmi_connector_detect(struct drm_connector *connector, bool force)
@@ -561,8 +530,11 @@ static void vc4_hdmi_encoder_enable(struct drm_encoder *encoder)
 	HDMI_WRITE(VC4_HDMI_TX_PHY_RESET_CTL, 0);
 
 	if (debug_dump_regs) {
-		DRM_INFO("HDMI regs before:\n");
-		vc4_hdmi_dump_regs(dev);
+		struct drm_printer p = drm_info_printer(&hdmi->pdev->dev);
+
+		dev_info(&hdmi->pdev->dev, "HDMI regs before:\n");
+		drm_print_regset32(&p, &hdmi->hdmi_regset);
+		drm_print_regset32(&p, &hdmi->hd_regset);
 	}
 
 	HD_WRITE(VC4_HD_VID_CTL, 0);
@@ -637,8 +609,11 @@ static void vc4_hdmi_encoder_enable(struct drm_encoder *encoder)
 	HDMI_WRITE(VC4_HDMI_FIFO_CTL, VC4_HDMI_FIFO_CTL_MASTER_SLAVE_N);
 
 	if (debug_dump_regs) {
-		DRM_INFO("HDMI regs after:\n");
-		vc4_hdmi_dump_regs(dev);
+		struct drm_printer p = drm_info_printer(&hdmi->pdev->dev);
+
+		dev_info(&hdmi->pdev->dev, "HDMI regs after:\n");
+		drm_print_regset32(&p, &hdmi->hdmi_regset);
+		drm_print_regset32(&p, &hdmi->hd_regset);
 	}
 
 	HD_WRITE(VC4_HD_VID_CTL,
@@ -1333,6 +1308,13 @@ static int vc4_hdmi_bind(struct device *dev, struct device *master, void *data)
 	if (IS_ERR(hdmi->hd_regs))
 		return PTR_ERR(hdmi->hd_regs);
 
+	hdmi->hdmi_regset.base = hdmi->hdmicore_regs;
+	hdmi->hdmi_regset.regs = hdmi_regs;
+	hdmi->hdmi_regset.nregs = ARRAY_SIZE(hdmi_regs);
+	hdmi->hd_regset.base = hdmi->hd_regs;
+	hdmi->hd_regset.regs = hd_regs;
+	hdmi->hd_regset.nregs = ARRAY_SIZE(hd_regs);
+
 	hdmi->pixel_clock = devm_clk_get(dev, "pixel");
 	if (IS_ERR(hdmi->pixel_clock)) {
 		DRM_ERROR("Failed to get pixel clock\n");
@@ -1448,6 +1430,8 @@ static int vc4_hdmi_bind(struct device *dev, struct device *master, void *data)
 	if (ret)
 		goto err_destroy_encoder;
 
+	vc4_debugfs_add_file(drm, "hdmi_regs", vc4_hdmi_debugfs_regs, hdmi);
+
 	return 0;
 
 #ifdef CONFIG_DRM_VC4_HDMI_CEC
diff --git a/drivers/gpu/drm/vc4/vc4_hvs.c b/drivers/gpu/drm/vc4/vc4_hvs.c
index 5d8c749c9749..f746e9a7a88c 100644
--- a/drivers/gpu/drm/vc4/vc4_hvs.c
+++ b/drivers/gpu/drm/vc4/vc4_hvs.c
@@ -22,58 +22,52 @@
  * each CRTC.
  */
 
+#include <drm/drm_atomic_helper.h>
 #include <linux/component.h>
 #include "vc4_drv.h"
 #include "vc4_regs.h"
 
-#define HVS_REG(reg) { reg, #reg }
-static const struct {
-	u32 reg;
-	const char *name;
-} hvs_regs[] = {
-	HVS_REG(SCALER_DISPCTRL),
-	HVS_REG(SCALER_DISPSTAT),
-	HVS_REG(SCALER_DISPID),
-	HVS_REG(SCALER_DISPECTRL),
-	HVS_REG(SCALER_DISPPROF),
-	HVS_REG(SCALER_DISPDITHER),
-	HVS_REG(SCALER_DISPEOLN),
-	HVS_REG(SCALER_DISPLIST0),
-	HVS_REG(SCALER_DISPLIST1),
-	HVS_REG(SCALER_DISPLIST2),
-	HVS_REG(SCALER_DISPLSTAT),
-	HVS_REG(SCALER_DISPLACT0),
-	HVS_REG(SCALER_DISPLACT1),
-	HVS_REG(SCALER_DISPLACT2),
-	HVS_REG(SCALER_DISPCTRL0),
-	HVS_REG(SCALER_DISPBKGND0),
-	HVS_REG(SCALER_DISPSTAT0),
-	HVS_REG(SCALER_DISPBASE0),
-	HVS_REG(SCALER_DISPCTRL1),
-	HVS_REG(SCALER_DISPBKGND1),
-	HVS_REG(SCALER_DISPSTAT1),
-	HVS_REG(SCALER_DISPBASE1),
-	HVS_REG(SCALER_DISPCTRL2),
-	HVS_REG(SCALER_DISPBKGND2),
-	HVS_REG(SCALER_DISPSTAT2),
-	HVS_REG(SCALER_DISPBASE2),
-	HVS_REG(SCALER_DISPALPHA2),
-	HVS_REG(SCALER_OLEDOFFS),
-	HVS_REG(SCALER_OLEDCOEF0),
-	HVS_REG(SCALER_OLEDCOEF1),
-	HVS_REG(SCALER_OLEDCOEF2),
+static const struct debugfs_reg32 hvs_regs[] = {
+	VC4_REG32(SCALER_DISPCTRL),
+	VC4_REG32(SCALER_DISPSTAT),
+	VC4_REG32(SCALER_DISPID),
+	VC4_REG32(SCALER_DISPECTRL),
+	VC4_REG32(SCALER_DISPPROF),
+	VC4_REG32(SCALER_DISPDITHER),
+	VC4_REG32(SCALER_DISPEOLN),
+	VC4_REG32(SCALER_DISPLIST0),
+	VC4_REG32(SCALER_DISPLIST1),
+	VC4_REG32(SCALER_DISPLIST2),
+	VC4_REG32(SCALER_DISPLSTAT),
+	VC4_REG32(SCALER_DISPLACT0),
+	VC4_REG32(SCALER_DISPLACT1),
+	VC4_REG32(SCALER_DISPLACT2),
+	VC4_REG32(SCALER_DISPCTRL0),
+	VC4_REG32(SCALER_DISPBKGND0),
+	VC4_REG32(SCALER_DISPSTAT0),
+	VC4_REG32(SCALER_DISPBASE0),
+	VC4_REG32(SCALER_DISPCTRL1),
+	VC4_REG32(SCALER_DISPBKGND1),
+	VC4_REG32(SCALER_DISPSTAT1),
+	VC4_REG32(SCALER_DISPBASE1),
+	VC4_REG32(SCALER_DISPCTRL2),
+	VC4_REG32(SCALER_DISPBKGND2),
+	VC4_REG32(SCALER_DISPSTAT2),
+	VC4_REG32(SCALER_DISPBASE2),
+	VC4_REG32(SCALER_DISPALPHA2),
+	VC4_REG32(SCALER_OLEDOFFS),
+	VC4_REG32(SCALER_OLEDCOEF0),
+	VC4_REG32(SCALER_OLEDCOEF1),
+	VC4_REG32(SCALER_OLEDCOEF2),
 };
 
 void vc4_hvs_dump_state(struct drm_device *dev)
 {
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	struct drm_printer p = drm_info_printer(&vc4->hvs->pdev->dev);
 	int i;
 
-	for (i = 0; i < ARRAY_SIZE(hvs_regs); i++) {
-		DRM_INFO("0x%04x (%s): 0x%08x\n",
-			 hvs_regs[i].reg, hvs_regs[i].name,
-			 HVS_READ(hvs_regs[i].reg));
-	}
+	drm_print_regset32(&p, &vc4->hvs->regset);
 
 	DRM_INFO("HVS ctx:\n");
 	for (i = 0; i < 64; i += 4) {
@@ -86,23 +80,17 @@ void vc4_hvs_dump_state(struct drm_device *dev)
 	}
 }
 
-#ifdef CONFIG_DEBUG_FS
-int vc4_hvs_debugfs_regs(struct seq_file *m, void *unused)
+static int vc4_hvs_debugfs_underrun(struct seq_file *m, void *data)
 {
-	struct drm_info_node *node = (struct drm_info_node *)m->private;
+	struct drm_info_node *node = m->private;
 	struct drm_device *dev = node->minor->dev;
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
-	int i;
+	struct drm_printer p = drm_seq_file_printer(m);
 
-	for (i = 0; i < ARRAY_SIZE(hvs_regs); i++) {
-		seq_printf(m, "%s (0x%04x): 0x%08x\n",
-			   hvs_regs[i].name, hvs_regs[i].reg,
-			   HVS_READ(hvs_regs[i].reg));
-	}
+	drm_printf(&p, "%d\n", atomic_read(&vc4->underrun));
 
 	return 0;
 }
-#endif
 
 /* The filter kernel is composed of dwords each containing 3 9-bit
  * signed integers packed next to each other.
@@ -166,6 +154,67 @@ static int vc4_hvs_upload_linear_kernel(struct vc4_hvs *hvs,
 	return 0;
 }
 
+void vc4_hvs_mask_underrun(struct drm_device *dev, int channel)
+{
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	u32 dispctrl = HVS_READ(SCALER_DISPCTRL);
+
+	dispctrl &= ~SCALER_DISPCTRL_DSPEISLUR(channel);
+
+	HVS_WRITE(SCALER_DISPCTRL, dispctrl);
+}
+
+void vc4_hvs_unmask_underrun(struct drm_device *dev, int channel)
+{
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	u32 dispctrl = HVS_READ(SCALER_DISPCTRL);
+
+	dispctrl |= SCALER_DISPCTRL_DSPEISLUR(channel);
+
+	HVS_WRITE(SCALER_DISPSTAT,
+		  SCALER_DISPSTAT_EUFLOW(channel));
+	HVS_WRITE(SCALER_DISPCTRL, dispctrl);
+}
+
+static void vc4_hvs_report_underrun(struct drm_device *dev)
+{
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+
+	atomic_inc(&vc4->underrun);
+	DRM_DEV_ERROR(dev->dev, "HVS underrun\n");
+}
+
+static irqreturn_t vc4_hvs_irq_handler(int irq, void *data)
+{
+	struct drm_device *dev = data;
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	irqreturn_t irqret = IRQ_NONE;
+	int channel;
+	u32 control;
+	u32 status;
+
+	status = HVS_READ(SCALER_DISPSTAT);
+	control = HVS_READ(SCALER_DISPCTRL);
+
+	for (channel = 0; channel < SCALER_CHANNELS_COUNT; channel++) {
+		/* Interrupt masking is not always honored, so check it here. */
+		if (status & SCALER_DISPSTAT_EUFLOW(channel) &&
+		    control & SCALER_DISPCTRL_DSPEISLUR(channel)) {
+			vc4_hvs_mask_underrun(dev, channel);
+			vc4_hvs_report_underrun(dev);
+
+			irqret = IRQ_HANDLED;
+		}
+	}
+
+	/* Clear every per-channel interrupt flag. */
+	HVS_WRITE(SCALER_DISPSTAT, SCALER_DISPSTAT_IRQMASK(0) |
+				   SCALER_DISPSTAT_IRQMASK(1) |
+				   SCALER_DISPSTAT_IRQMASK(2));
+
+	return irqret;
+}
+
 static int vc4_hvs_bind(struct device *dev, struct device *master, void *data)
 {
 	struct platform_device *pdev = to_platform_device(dev);
@@ -185,6 +234,10 @@ static int vc4_hvs_bind(struct device *dev, struct device *master, void *data)
 	if (IS_ERR(hvs->regs))
 		return PTR_ERR(hvs->regs);
 
+	hvs->regset.base = hvs->regs;
+	hvs->regset.regs = hvs_regs;
+	hvs->regset.nregs = ARRAY_SIZE(hvs_regs);
+
 	hvs->dlist = hvs->regs + SCALER_DLIST_START;
 
 	spin_lock_init(&hvs->mm_lock);
@@ -219,15 +272,40 @@ static int vc4_hvs_bind(struct device *dev, struct device *master, void *data)
 	dispctrl = HVS_READ(SCALER_DISPCTRL);
 
 	dispctrl |= SCALER_DISPCTRL_ENABLE;
+	dispctrl |= SCALER_DISPCTRL_DISPEIRQ(0) |
+		    SCALER_DISPCTRL_DISPEIRQ(1) |
+		    SCALER_DISPCTRL_DISPEIRQ(2);
 
 	/* Set DSP3 (PV1) to use HVS channel 2, which would otherwise
 	 * be unused.
 	 */
 	dispctrl &= ~SCALER_DISPCTRL_DSP3_MUX_MASK;
+	dispctrl &= ~(SCALER_DISPCTRL_DMAEIRQ |
+		      SCALER_DISPCTRL_SLVWREIRQ |
+		      SCALER_DISPCTRL_SLVRDEIRQ |
+		      SCALER_DISPCTRL_DSPEIEOF(0) |
+		      SCALER_DISPCTRL_DSPEIEOF(1) |
+		      SCALER_DISPCTRL_DSPEIEOF(2) |
+		      SCALER_DISPCTRL_DSPEIEOLN(0) |
+		      SCALER_DISPCTRL_DSPEIEOLN(1) |
+		      SCALER_DISPCTRL_DSPEIEOLN(2) |
+		      SCALER_DISPCTRL_DSPEISLUR(0) |
+		      SCALER_DISPCTRL_DSPEISLUR(1) |
+		      SCALER_DISPCTRL_DSPEISLUR(2) |
+		      SCALER_DISPCTRL_SCLEIRQ);
 	dispctrl |= VC4_SET_FIELD(2, SCALER_DISPCTRL_DSP3_MUX);
 
 	HVS_WRITE(SCALER_DISPCTRL, dispctrl);
 
+	ret = devm_request_irq(dev, platform_get_irq(pdev, 0),
+			       vc4_hvs_irq_handler, 0, "vc4 hvs", drm);
+	if (ret)
+		return ret;
+
+	vc4_debugfs_add_regset32(drm, "hvs_regs", &hvs->regset);
+	vc4_debugfs_add_file(drm, "hvs_underrun", vc4_hvs_debugfs_underrun,
+			     NULL);
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/vc4/vc4_irq.c b/drivers/gpu/drm/vc4/vc4_irq.c
index 4cd2ccfe15f4..ffd0a4388752 100644
--- a/drivers/gpu/drm/vc4/vc4_irq.c
+++ b/drivers/gpu/drm/vc4/vc4_irq.c
@@ -229,6 +229,9 @@ vc4_irq_preinstall(struct drm_device *dev)
 {
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
 
+	if (!vc4->v3d)
+		return;
+
 	init_waitqueue_head(&vc4->job_wait_queue);
 	INIT_WORK(&vc4->overflow_mem_work, vc4_overflow_mem_work);
 
@@ -243,6 +246,9 @@ vc4_irq_postinstall(struct drm_device *dev)
 {
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
 
+	if (!vc4->v3d)
+		return 0;
+
 	/* Enable both the render done and out of memory interrupts. */
 	V3D_WRITE(V3D_INTENA, V3D_DRIVER_IRQS);
 
@@ -254,6 +260,9 @@ vc4_irq_uninstall(struct drm_device *dev)
 {
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
 
+	if (!vc4->v3d)
+		return;
+
 	/* Disable sending interrupts for our driver's IRQs. */
 	V3D_WRITE(V3D_INTDIS, V3D_DRIVER_IRQS);
 
diff --git a/drivers/gpu/drm/vc4/vc4_kms.c b/drivers/gpu/drm/vc4/vc4_kms.c
index 91b8c72ff361..295dacc8bcb9 100644
--- a/drivers/gpu/drm/vc4/vc4_kms.c
+++ b/drivers/gpu/drm/vc4/vc4_kms.c
@@ -34,6 +34,18 @@ static struct vc4_ctm_state *to_vc4_ctm_state(struct drm_private_state *priv)
 	return container_of(priv, struct vc4_ctm_state, base);
 }
 
+struct vc4_load_tracker_state {
+	struct drm_private_state base;
+	u64 hvs_load;
+	u64 membus_load;
+};
+
+static struct vc4_load_tracker_state *
+to_vc4_load_tracker_state(struct drm_private_state *priv)
+{
+	return container_of(priv, struct vc4_load_tracker_state, base);
+}
+
 static struct vc4_ctm_state *vc4_get_ctm_state(struct drm_atomic_state *state,
 					       struct drm_private_obj *manager)
 {
@@ -138,6 +150,16 @@ vc4_atomic_complete_commit(struct drm_atomic_state *state)
 {
 	struct drm_device *dev = state->dev;
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	struct vc4_crtc *vc4_crtc;
+	int i;
+
+	for (i = 0; i < dev->mode_config.num_crtc; i++) {
+		if (!state->crtcs[i].ptr || !state->crtcs[i].commit)
+			continue;
+
+		vc4_crtc = to_vc4_crtc(state->crtcs[i].ptr);
+		vc4_hvs_mask_underrun(dev, vc4_crtc->channel);
+	}
 
 	drm_atomic_helper_wait_for_fences(dev, state, false);
 
@@ -385,6 +407,85 @@ vc4_ctm_atomic_check(struct drm_device *dev, struct drm_atomic_state *state)
 	return 0;
 }
 
+static int vc4_load_tracker_atomic_check(struct drm_atomic_state *state)
+{
+	struct drm_plane_state *old_plane_state, *new_plane_state;
+	struct vc4_dev *vc4 = to_vc4_dev(state->dev);
+	struct vc4_load_tracker_state *load_state;
+	struct drm_private_state *priv_state;
+	struct drm_plane *plane;
+	int i;
+
+	priv_state = drm_atomic_get_private_obj_state(state,
+						      &vc4->load_tracker);
+	if (IS_ERR(priv_state))
+		return PTR_ERR(priv_state);
+
+	load_state = to_vc4_load_tracker_state(priv_state);
+	for_each_oldnew_plane_in_state(state, plane, old_plane_state,
+				       new_plane_state, i) {
+		struct vc4_plane_state *vc4_plane_state;
+
+		if (old_plane_state->fb && old_plane_state->crtc) {
+			vc4_plane_state = to_vc4_plane_state(old_plane_state);
+			load_state->membus_load -= vc4_plane_state->membus_load;
+			load_state->hvs_load -= vc4_plane_state->hvs_load;
+		}
+
+		if (new_plane_state->fb && new_plane_state->crtc) {
+			vc4_plane_state = to_vc4_plane_state(new_plane_state);
+			load_state->membus_load += vc4_plane_state->membus_load;
+			load_state->hvs_load += vc4_plane_state->hvs_load;
+		}
+	}
+
+	/* Don't check the load when the tracker is disabled. */
+	if (!vc4->load_tracker_enabled)
+		return 0;
+
+	/* The absolute limit is 2Gbyte/sec, but let's take a margin to let
+	 * the system work when other blocks are accessing the memory.
+	 */
+	if (load_state->membus_load > SZ_1G + SZ_512M)
+		return -ENOSPC;
+
+	/* HVS clock is supposed to run @ 250Mhz, let's take a margin and
+	 * consider the maximum number of cycles is 240M.
+	 */
+	if (load_state->hvs_load > 240000000ULL)
+		return -ENOSPC;
+
+	return 0;
+}
+
+static struct drm_private_state *
+vc4_load_tracker_duplicate_state(struct drm_private_obj *obj)
+{
+	struct vc4_load_tracker_state *state;
+
+	state = kmemdup(obj->state, sizeof(*state), GFP_KERNEL);
+	if (!state)
+		return NULL;
+
+	__drm_atomic_helper_private_obj_duplicate_state(obj, &state->base);
+
+	return &state->base;
+}
+
+static void vc4_load_tracker_destroy_state(struct drm_private_obj *obj,
+					   struct drm_private_state *state)
+{
+	struct vc4_load_tracker_state *load_state;
+
+	load_state = to_vc4_load_tracker_state(state);
+	kfree(load_state);
+}
+
+static const struct drm_private_state_funcs vc4_load_tracker_state_funcs = {
+	.atomic_duplicate_state = vc4_load_tracker_duplicate_state,
+	.atomic_destroy_state = vc4_load_tracker_destroy_state,
+};
+
 static int
 vc4_atomic_check(struct drm_device *dev, struct drm_atomic_state *state)
 {
@@ -394,7 +495,11 @@ vc4_atomic_check(struct drm_device *dev, struct drm_atomic_state *state)
 	if (ret < 0)
 		return ret;
 
-	return drm_atomic_helper_check(dev, state);
+	ret = drm_atomic_helper_check(dev, state);
+	if (ret)
+		return ret;
+
+	return vc4_load_tracker_atomic_check(state);
 }
 
 static const struct drm_mode_config_funcs vc4_mode_funcs = {
@@ -407,13 +512,20 @@ int vc4_kms_load(struct drm_device *dev)
 {
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
 	struct vc4_ctm_state *ctm_state;
+	struct vc4_load_tracker_state *load_state;
 	int ret;
 
+	/* Start with the load tracker enabled. Can be disabled through the
+	 * debugfs load_tracker file.
+	 */
+	vc4->load_tracker_enabled = true;
+
 	sema_init(&vc4->async_modeset, 1);
 
 	/* Set support for vblank irq fast disable, before drm_vblank_init() */
 	dev->vblank_disable_immediate = true;
 
+	dev->irq_enabled = true;
 	ret = drm_vblank_init(dev, dev->mode_config.num_crtc);
 	if (ret < 0) {
 		dev_err(dev->dev, "failed to initialize vblank\n");
@@ -436,6 +548,15 @@ int vc4_kms_load(struct drm_device *dev)
 	drm_atomic_private_obj_init(dev, &vc4->ctm_manager, &ctm_state->base,
 				    &vc4_ctm_state_funcs);
 
+	load_state = kzalloc(sizeof(*load_state), GFP_KERNEL);
+	if (!load_state) {
+		drm_atomic_private_obj_fini(&vc4->ctm_manager);
+		return -ENOMEM;
+	}
+
+	drm_atomic_private_obj_init(dev, &vc4->load_tracker, &load_state->base,
+				    &vc4_load_tracker_state_funcs);
+
 	drm_mode_config_reset(dev);
 
 	drm_kms_helper_poll_init(dev);
diff --git a/drivers/gpu/drm/vc4/vc4_perfmon.c b/drivers/gpu/drm/vc4/vc4_perfmon.c
index 495150415020..f4aa75efd16b 100644
--- a/drivers/gpu/drm/vc4/vc4_perfmon.c
+++ b/drivers/gpu/drm/vc4/vc4_perfmon.c
@@ -100,12 +100,18 @@ void vc4_perfmon_close_file(struct vc4_file *vc4file)
 int vc4_perfmon_create_ioctl(struct drm_device *dev, void *data,
 			     struct drm_file *file_priv)
 {
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
 	struct vc4_file *vc4file = file_priv->driver_priv;
 	struct drm_vc4_perfmon_create *req = data;
 	struct vc4_perfmon *perfmon;
 	unsigned int i;
 	int ret;
 
+	if (!vc4->v3d) {
+		DRM_DEBUG("Creating perfmon no VC4 V3D probed\n");
+		return -ENODEV;
+	}
+
 	/* Number of monitored counters cannot exceed HW limits. */
 	if (req->ncounters > DRM_VC4_MAX_PERF_COUNTERS ||
 	    !req->ncounters)
@@ -146,10 +152,16 @@ int vc4_perfmon_create_ioctl(struct drm_device *dev, void *data,
 int vc4_perfmon_destroy_ioctl(struct drm_device *dev, void *data,
 			      struct drm_file *file_priv)
 {
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
 	struct vc4_file *vc4file = file_priv->driver_priv;
 	struct drm_vc4_perfmon_destroy *req = data;
 	struct vc4_perfmon *perfmon;
 
+	if (!vc4->v3d) {
+		DRM_DEBUG("Destroying perfmon no VC4 V3D probed\n");
+		return -ENODEV;
+	}
+
 	mutex_lock(&vc4file->perfmon.lock);
 	perfmon = idr_remove(&vc4file->perfmon.idr, req->id);
 	mutex_unlock(&vc4file->perfmon.lock);
@@ -164,11 +176,17 @@ int vc4_perfmon_destroy_ioctl(struct drm_device *dev, void *data,
 int vc4_perfmon_get_values_ioctl(struct drm_device *dev, void *data,
 				 struct drm_file *file_priv)
 {
+	struct vc4_dev *vc4 = to_vc4_dev(dev);
 	struct vc4_file *vc4file = file_priv->driver_priv;
 	struct drm_vc4_perfmon_get_values *req = data;
 	struct vc4_perfmon *perfmon;
 	int ret;
 
+	if (!vc4->v3d) {
+		DRM_DEBUG("Getting perfmon no VC4 V3D probed\n");
+		return -ENODEV;
+	}
+
 	mutex_lock(&vc4file->perfmon.lock);
 	perfmon = idr_find(&vc4file->perfmon.idr, req->id);
 	vc4_perfmon_get(perfmon);
diff --git a/drivers/gpu/drm/vc4/vc4_plane.c b/drivers/gpu/drm/vc4/vc4_plane.c
index d098337c10e9..4d918d3e4858 100644
--- a/drivers/gpu/drm/vc4/vc4_plane.c
+++ b/drivers/gpu/drm/vc4/vc4_plane.c
@@ -488,6 +488,61 @@ static void vc4_write_scaling_parameters(struct drm_plane_state *state,
 	}
 }
 
+static void vc4_plane_calc_load(struct drm_plane_state *state)
+{
+	unsigned int hvs_load_shift, vrefresh, i;
+	struct drm_framebuffer *fb = state->fb;
+	struct vc4_plane_state *vc4_state;
+	struct drm_crtc_state *crtc_state;
+	unsigned int vscale_factor;
+
+	vc4_state = to_vc4_plane_state(state);
+	crtc_state = drm_atomic_get_existing_crtc_state(state->state,
+							state->crtc);
+	vrefresh = drm_mode_vrefresh(&crtc_state->adjusted_mode);
+
+	/* The HVS is able to process 2 pixels/cycle when scaling the source,
+	 * 4 pixels/cycle otherwise.
+	 * Alpha blending step seems to be pipelined and it's always operating
+	 * at 4 pixels/cycle, so the limiting aspect here seems to be the
+	 * scaler block.
+	 * HVS load is expressed in clk-cycles/sec (AKA Hz).
+	 */
+	if (vc4_state->x_scaling[0] != VC4_SCALING_NONE ||
+	    vc4_state->x_scaling[1] != VC4_SCALING_NONE ||
+	    vc4_state->y_scaling[0] != VC4_SCALING_NONE ||
+	    vc4_state->y_scaling[1] != VC4_SCALING_NONE)
+		hvs_load_shift = 1;
+	else
+		hvs_load_shift = 2;
+
+	vc4_state->membus_load = 0;
+	vc4_state->hvs_load = 0;
+	for (i = 0; i < fb->format->num_planes; i++) {
+		/* Even if the bandwidth/plane required for a single frame is
+		 *
+		 * vc4_state->src_w[i] * vc4_state->src_h[i] * cpp * vrefresh
+		 *
+		 * when downscaling, we have to read more pixels per line in
+		 * the time frame reserved for a single line, so the bandwidth
+		 * demand can be punctually higher. To account for that, we
+		 * calculate the down-scaling factor and multiply the plane
+		 * load by this number. We're likely over-estimating the read
+		 * demand, but that's better than under-estimating it.
+		 */
+		vscale_factor = DIV_ROUND_UP(vc4_state->src_h[i],
+					     vc4_state->crtc_h);
+		vc4_state->membus_load += vc4_state->src_w[i] *
+					  vc4_state->src_h[i] * vscale_factor *
+					  fb->format->cpp[i];
+		vc4_state->hvs_load += vc4_state->crtc_h * vc4_state->crtc_w;
+	}
+
+	vc4_state->hvs_load *= vrefresh;
+	vc4_state->hvs_load >>= hvs_load_shift;
+	vc4_state->membus_load *= vrefresh;
+}
+
 static int vc4_plane_allocate_lbm(struct drm_plane_state *state)
 {
 	struct vc4_dev *vc4 = to_vc4_dev(state->plane->dev);
@@ -875,6 +930,8 @@ static int vc4_plane_mode_set(struct drm_plane *plane,
 	 */
 	vc4_state->dlist_initialized = 1;
 
+	vc4_plane_calc_load(state);
+
 	return 0;
 }
 
@@ -1082,7 +1139,7 @@ static int vc4_prepare_fb(struct drm_plane *plane,
 
 	bo = to_vc4_bo(&drm_fb_cma_get_gem_obj(state->fb, 0)->base);
 
-	fence = reservation_object_get_excl_rcu(bo->resv);
+	fence = reservation_object_get_excl_rcu(bo->base.base.resv);
 	drm_atomic_set_fence_for_plane(state, fence);
 
 	if (plane->state->fb == state->fb)
diff --git a/drivers/gpu/drm/vc4/vc4_regs.h b/drivers/gpu/drm/vc4/vc4_regs.h
index 931088014272..c0c5fadaf7e3 100644
--- a/drivers/gpu/drm/vc4/vc4_regs.h
+++ b/drivers/gpu/drm/vc4/vc4_regs.h
@@ -212,11 +212,11 @@
 
 #define PV_HACT_ACT				0x30
 
+#define SCALER_CHANNELS_COUNT			3
+
 #define SCALER_DISPCTRL                         0x00000000
 /* Global register for clock gating the HVS */
 # define SCALER_DISPCTRL_ENABLE			BIT(31)
-# define SCALER_DISPCTRL_DSP2EISLUR		BIT(15)
-# define SCALER_DISPCTRL_DSP1EISLUR		BIT(14)
 # define SCALER_DISPCTRL_DSP3_MUX_MASK		VC4_MASK(19, 18)
 # define SCALER_DISPCTRL_DSP3_MUX_SHIFT		18
 
@@ -224,45 +224,25 @@
  * SCALER_DISPSTAT_IRQDISP0.  Note that short frame contributions are
  * always enabled.
  */
-# define SCALER_DISPCTRL_DSP0EISLUR		BIT(13)
-# define SCALER_DISPCTRL_DSP2EIEOLN		BIT(12)
-# define SCALER_DISPCTRL_DSP2EIEOF		BIT(11)
-# define SCALER_DISPCTRL_DSP1EIEOLN		BIT(10)
-# define SCALER_DISPCTRL_DSP1EIEOF		BIT(9)
+# define SCALER_DISPCTRL_DSPEISLUR(x)		BIT(13 + (x))
 /* Enables Display 0 end-of-line-N contribution to
  * SCALER_DISPSTAT_IRQDISP0
  */
-# define SCALER_DISPCTRL_DSP0EIEOLN		BIT(8)
+# define SCALER_DISPCTRL_DSPEIEOLN(x)		BIT(8 + ((x) * 2))
 /* Enables Display 0 EOF contribution to SCALER_DISPSTAT_IRQDISP0 */
-# define SCALER_DISPCTRL_DSP0EIEOF		BIT(7)
+# define SCALER_DISPCTRL_DSPEIEOF(x)		BIT(7 + ((x) * 2))
 
 # define SCALER_DISPCTRL_SLVRDEIRQ		BIT(6)
 # define SCALER_DISPCTRL_SLVWREIRQ		BIT(5)
 # define SCALER_DISPCTRL_DMAEIRQ		BIT(4)
-# define SCALER_DISPCTRL_DISP2EIRQ		BIT(3)
-# define SCALER_DISPCTRL_DISP1EIRQ		BIT(2)
 /* Enables interrupt generation on the enabled EOF/EOLN/EISLUR
  * bits and short frames..
  */
-# define SCALER_DISPCTRL_DISP0EIRQ		BIT(1)
+# define SCALER_DISPCTRL_DISPEIRQ(x)		BIT(1 + (x))
 /* Enables interrupt generation on scaler profiler interrupt. */
 # define SCALER_DISPCTRL_SCLEIRQ		BIT(0)
 
 #define SCALER_DISPSTAT                         0x00000004
-# define SCALER_DISPSTAT_COBLOW2		BIT(29)
-# define SCALER_DISPSTAT_EOLN2			BIT(28)
-# define SCALER_DISPSTAT_ESFRAME2		BIT(27)
-# define SCALER_DISPSTAT_ESLINE2		BIT(26)
-# define SCALER_DISPSTAT_EUFLOW2		BIT(25)
-# define SCALER_DISPSTAT_EOF2			BIT(24)
-
-# define SCALER_DISPSTAT_COBLOW1		BIT(21)
-# define SCALER_DISPSTAT_EOLN1			BIT(20)
-# define SCALER_DISPSTAT_ESFRAME1		BIT(19)
-# define SCALER_DISPSTAT_ESLINE1		BIT(18)
-# define SCALER_DISPSTAT_EUFLOW1		BIT(17)
-# define SCALER_DISPSTAT_EOF1			BIT(16)
-
 # define SCALER_DISPSTAT_RESP_MASK		VC4_MASK(15, 14)
 # define SCALER_DISPSTAT_RESP_SHIFT		14
 # define SCALER_DISPSTAT_RESP_OKAY		0
@@ -270,23 +250,26 @@
 # define SCALER_DISPSTAT_RESP_SLVERR		2
 # define SCALER_DISPSTAT_RESP_DECERR		3
 
-# define SCALER_DISPSTAT_COBLOW0		BIT(13)
+# define SCALER_DISPSTAT_COBLOW(x)		BIT(13 + ((x) * 8))
 /* Set when the DISPEOLN line is done compositing. */
-# define SCALER_DISPSTAT_EOLN0			BIT(12)
+# define SCALER_DISPSTAT_EOLN(x)		BIT(12 + ((x) * 8))
 /* Set when VSTART is seen but there are still pixels in the current
  * output line.
  */
-# define SCALER_DISPSTAT_ESFRAME0		BIT(11)
+# define SCALER_DISPSTAT_ESFRAME(x)		BIT(11 + ((x) * 8))
 /* Set when HSTART is seen but there are still pixels in the current
  * output line.
  */
-# define SCALER_DISPSTAT_ESLINE0		BIT(10)
+# define SCALER_DISPSTAT_ESLINE(x)		BIT(10 + ((x) * 8))
 /* Set when the the downstream tries to read from the display FIFO
  * while it's empty.
  */
-# define SCALER_DISPSTAT_EUFLOW0		BIT(9)
+# define SCALER_DISPSTAT_EUFLOW(x)		BIT(9 + ((x) * 8))
 /* Set when the display mode changes from RUN to EOF */
-# define SCALER_DISPSTAT_EOF0			BIT(8)
+# define SCALER_DISPSTAT_EOF(x)			BIT(8 + ((x) * 8))
+
+# define SCALER_DISPSTAT_IRQMASK(x)		VC4_MASK(13 + ((x) * 8), \
+							 8 + ((x) * 8))
 
 /* Set on AXI invalid DMA ID error. */
 # define SCALER_DISPSTAT_DMA_ERROR		BIT(7)
@@ -298,12 +281,10 @@
  * SCALER_DISPSTAT_RESP_ERROR is not SCALER_DISPSTAT_RESP_OKAY.
  */
 # define SCALER_DISPSTAT_IRQDMA			BIT(4)
-# define SCALER_DISPSTAT_IRQDISP2		BIT(3)
-# define SCALER_DISPSTAT_IRQDISP1		BIT(2)
 /* Set when any of the EOF/EOLN/ESFRAME/ESLINE bits are set and their
  * corresponding interrupt bit is enabled in DISPCTRL.
  */
-# define SCALER_DISPSTAT_IRQDISP0		BIT(1)
+# define SCALER_DISPSTAT_IRQDISP(x)		BIT(1 + (x))
 /* On read, the profiler interrupt.  On write, clear *all* interrupt bits. */
 # define SCALER_DISPSTAT_IRQSCL			BIT(0)
 
diff --git a/drivers/gpu/drm/vc4/vc4_render_cl.c b/drivers/gpu/drm/vc4/vc4_render_cl.c
index 273984f71ae2..3c918eeaf56e 100644
--- a/drivers/gpu/drm/vc4/vc4_render_cl.c
+++ b/drivers/gpu/drm/vc4/vc4_render_cl.c
@@ -148,6 +148,12 @@ static void emit_tile(struct vc4_exec_info *exec,
 	}
 
 	if (setup->zs_read) {
+		if (setup->color_read) {
+			/* Exec previous load. */
+			vc4_tile_coordinates(setup, x, y);
+			vc4_store_before_load(setup);
+		}
+
 		if (args->zs_read.flags &
 		    VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES) {
 			rcl_u8(setup, VC4_PACKET_LOAD_FULL_RES_TILE_BUFFER);
@@ -156,12 +162,6 @@ static void emit_tile(struct vc4_exec_info *exec,
 						    &args->zs_read, x, y) |
 				VC4_LOADSTORE_FULL_RES_DISABLE_COLOR);
 		} else {
-			if (setup->color_read) {
-				/* Exec previous load. */
-				vc4_tile_coordinates(setup, x, y);
-				vc4_store_before_load(setup);
-			}
-
 			rcl_u8(setup, VC4_PACKET_LOAD_TILE_BUFFER_GENERAL);
 			rcl_u16(setup, args->zs_read.bits);
 			rcl_u32(setup, setup->zs_read->paddr +
@@ -291,16 +291,15 @@ static int vc4_create_rcl_bo(struct drm_device *dev, struct vc4_exec_info *exec,
 		}
 	}
 	if (setup->zs_read) {
+		if (setup->color_read) {
+			loop_body_size += VC4_PACKET_TILE_COORDINATES_SIZE;
+			loop_body_size += VC4_PACKET_STORE_TILE_BUFFER_GENERAL_SIZE;
+		}
+
 		if (args->zs_read.flags &
 		    VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES) {
 			loop_body_size += VC4_PACKET_LOAD_FULL_RES_TILE_BUFFER_SIZE;
 		} else {
-			if (setup->color_read &&
-			    !(args->color_read.flags &
-			      VC4_SUBMIT_RCL_SURFACE_READ_IS_FULL_RES)) {
-				loop_body_size += VC4_PACKET_TILE_COORDINATES_SIZE;
-				loop_body_size += VC4_PACKET_STORE_TILE_BUFFER_GENERAL_SIZE;
-			}
 			loop_body_size += VC4_PACKET_LOAD_TILE_BUFFER_GENERAL_SIZE;
 		}
 	}
diff --git a/drivers/gpu/drm/vc4/vc4_txp.c b/drivers/gpu/drm/vc4/vc4_txp.c
index aa279b5b0de7..c8b89a78f9f4 100644
--- a/drivers/gpu/drm/vc4/vc4_txp.c
+++ b/drivers/gpu/drm/vc4/vc4_txp.c
@@ -148,6 +148,7 @@ struct vc4_txp {
 	struct drm_writeback_connector connector;
 
 	void __iomem *regs;
+	struct debugfs_regset32 regset;
 };
 
 static inline struct vc4_txp *encoder_to_vc4_txp(struct drm_encoder *encoder)
@@ -160,40 +161,14 @@ static inline struct vc4_txp *connector_to_vc4_txp(struct drm_connector *conn)
 	return container_of(conn, struct vc4_txp, connector.base);
 }
 
-#define TXP_REG(reg) { reg, #reg }
-static const struct {
-	u32 reg;
-	const char *name;
-} txp_regs[] = {
-	TXP_REG(TXP_DST_PTR),
-	TXP_REG(TXP_DST_PITCH),
-	TXP_REG(TXP_DIM),
-	TXP_REG(TXP_DST_CTRL),
-	TXP_REG(TXP_PROGRESS),
+static const struct debugfs_reg32 txp_regs[] = {
+	VC4_REG32(TXP_DST_PTR),
+	VC4_REG32(TXP_DST_PITCH),
+	VC4_REG32(TXP_DIM),
+	VC4_REG32(TXP_DST_CTRL),
+	VC4_REG32(TXP_PROGRESS),
 };
 
-#ifdef CONFIG_DEBUG_FS
-int vc4_txp_debugfs_regs(struct seq_file *m, void *unused)
-{
-	struct drm_info_node *node = (struct drm_info_node *)m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct vc4_dev *vc4 = to_vc4_dev(dev);
-	struct vc4_txp *txp = vc4->txp;
-	int i;
-
-	if (!txp)
-		return 0;
-
-	for (i = 0; i < ARRAY_SIZE(txp_regs); i++) {
-		seq_printf(m, "%s (0x%04x): 0x%08x\n",
-			   txp_regs[i].name, txp_regs[i].reg,
-			   TXP_READ(txp_regs[i].reg));
-	}
-
-	return 0;
-}
-#endif
-
 static int vc4_txp_connector_get_modes(struct drm_connector *connector)
 {
 	struct drm_device *dev = connector->dev;
@@ -249,7 +224,6 @@ static int vc4_txp_connector_atomic_check(struct drm_connector *conn,
 					struct drm_connector_state *conn_state)
 {
 	struct drm_crtc_state *crtc_state;
-	struct drm_gem_cma_object *gem;
 	struct drm_framebuffer *fb;
 	int i;
 
@@ -275,8 +249,6 @@ static int vc4_txp_connector_atomic_check(struct drm_connector *conn,
 	if (i == ARRAY_SIZE(drm_fmts))
 		return -EINVAL;
 
-	gem = drm_fb_cma_get_gem_obj(fb, 0);
-
 	/* Pitch must be aligned on 16 bytes. */
 	if (fb->pitches[0] & GENMASK(3, 0))
 		return -EINVAL;
@@ -327,7 +299,7 @@ static void vc4_txp_connector_atomic_commit(struct drm_connector *conn,
 
 	TXP_WRITE(TXP_DST_CTRL, ctrl);
 
-	drm_writeback_queue_job(&txp->connector, conn_state->writeback_job);
+	drm_writeback_queue_job(&txp->connector, conn_state);
 }
 
 static const struct drm_connector_helper_funcs vc4_txp_connector_helper_funcs = {
@@ -413,6 +385,9 @@ static int vc4_txp_bind(struct device *dev, struct device *master, void *data)
 	txp->regs = vc4_ioremap_regs(pdev, 0);
 	if (IS_ERR(txp->regs))
 		return PTR_ERR(txp->regs);
+	txp->regset.base = txp->regs;
+	txp->regset.regs = txp_regs;
+	txp->regset.nregs = ARRAY_SIZE(txp_regs);
 
 	drm_connector_helper_add(&txp->connector.base,
 				 &vc4_txp_connector_helper_funcs);
@@ -431,6 +406,8 @@ static int vc4_txp_bind(struct device *dev, struct device *master, void *data)
 	dev_set_drvdata(dev, txp);
 	vc4->txp = txp;
 
+	vc4_debugfs_add_regset32(drm, "txp_regs", &txp->regset);
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/vc4/vc4_v3d.c b/drivers/gpu/drm/vc4/vc4_v3d.c
index e47e29426078..a4b6859e3af6 100644
--- a/drivers/gpu/drm/vc4/vc4_v3d.c
+++ b/drivers/gpu/drm/vc4/vc4_v3d.c
@@ -22,129 +22,145 @@
 #include "vc4_drv.h"
 #include "vc4_regs.h"
 
-#ifdef CONFIG_DEBUG_FS
-#define REGDEF(reg) { reg, #reg }
-static const struct {
-	uint32_t reg;
-	const char *name;
-} vc4_reg_defs[] = {
-	REGDEF(V3D_IDENT0),
-	REGDEF(V3D_IDENT1),
-	REGDEF(V3D_IDENT2),
-	REGDEF(V3D_SCRATCH),
-	REGDEF(V3D_L2CACTL),
-	REGDEF(V3D_SLCACTL),
-	REGDEF(V3D_INTCTL),
-	REGDEF(V3D_INTENA),
-	REGDEF(V3D_INTDIS),
-	REGDEF(V3D_CT0CS),
-	REGDEF(V3D_CT1CS),
-	REGDEF(V3D_CT0EA),
-	REGDEF(V3D_CT1EA),
-	REGDEF(V3D_CT0CA),
-	REGDEF(V3D_CT1CA),
-	REGDEF(V3D_CT00RA0),
-	REGDEF(V3D_CT01RA0),
-	REGDEF(V3D_CT0LC),
-	REGDEF(V3D_CT1LC),
-	REGDEF(V3D_CT0PC),
-	REGDEF(V3D_CT1PC),
-	REGDEF(V3D_PCS),
-	REGDEF(V3D_BFC),
-	REGDEF(V3D_RFC),
-	REGDEF(V3D_BPCA),
-	REGDEF(V3D_BPCS),
-	REGDEF(V3D_BPOA),
-	REGDEF(V3D_BPOS),
-	REGDEF(V3D_BXCF),
-	REGDEF(V3D_SQRSV0),
-	REGDEF(V3D_SQRSV1),
-	REGDEF(V3D_SQCNTL),
-	REGDEF(V3D_SRQPC),
-	REGDEF(V3D_SRQUA),
-	REGDEF(V3D_SRQUL),
-	REGDEF(V3D_SRQCS),
-	REGDEF(V3D_VPACNTL),
-	REGDEF(V3D_VPMBASE),
-	REGDEF(V3D_PCTRC),
-	REGDEF(V3D_PCTRE),
-	REGDEF(V3D_PCTR(0)),
-	REGDEF(V3D_PCTRS(0)),
-	REGDEF(V3D_PCTR(1)),
-	REGDEF(V3D_PCTRS(1)),
-	REGDEF(V3D_PCTR(2)),
-	REGDEF(V3D_PCTRS(2)),
-	REGDEF(V3D_PCTR(3)),
-	REGDEF(V3D_PCTRS(3)),
-	REGDEF(V3D_PCTR(4)),
-	REGDEF(V3D_PCTRS(4)),
-	REGDEF(V3D_PCTR(5)),
-	REGDEF(V3D_PCTRS(5)),
-	REGDEF(V3D_PCTR(6)),
-	REGDEF(V3D_PCTRS(6)),
-	REGDEF(V3D_PCTR(7)),
-	REGDEF(V3D_PCTRS(7)),
-	REGDEF(V3D_PCTR(8)),
-	REGDEF(V3D_PCTRS(8)),
-	REGDEF(V3D_PCTR(9)),
-	REGDEF(V3D_PCTRS(9)),
-	REGDEF(V3D_PCTR(10)),
-	REGDEF(V3D_PCTRS(10)),
-	REGDEF(V3D_PCTR(11)),
-	REGDEF(V3D_PCTRS(11)),
-	REGDEF(V3D_PCTR(12)),
-	REGDEF(V3D_PCTRS(12)),
-	REGDEF(V3D_PCTR(13)),
-	REGDEF(V3D_PCTRS(13)),
-	REGDEF(V3D_PCTR(14)),
-	REGDEF(V3D_PCTRS(14)),
-	REGDEF(V3D_PCTR(15)),
-	REGDEF(V3D_PCTRS(15)),
-	REGDEF(V3D_DBGE),
-	REGDEF(V3D_FDBGO),
-	REGDEF(V3D_FDBGB),
-	REGDEF(V3D_FDBGR),
-	REGDEF(V3D_FDBGS),
-	REGDEF(V3D_ERRSTAT),
+static const struct debugfs_reg32 v3d_regs[] = {
+	VC4_REG32(V3D_IDENT0),
+	VC4_REG32(V3D_IDENT1),
+	VC4_REG32(V3D_IDENT2),
+	VC4_REG32(V3D_SCRATCH),
+	VC4_REG32(V3D_L2CACTL),
+	VC4_REG32(V3D_SLCACTL),
+	VC4_REG32(V3D_INTCTL),
+	VC4_REG32(V3D_INTENA),
+	VC4_REG32(V3D_INTDIS),
+	VC4_REG32(V3D_CT0CS),
+	VC4_REG32(V3D_CT1CS),
+	VC4_REG32(V3D_CT0EA),
+	VC4_REG32(V3D_CT1EA),
+	VC4_REG32(V3D_CT0CA),
+	VC4_REG32(V3D_CT1CA),
+	VC4_REG32(V3D_CT00RA0),
+	VC4_REG32(V3D_CT01RA0),
+	VC4_REG32(V3D_CT0LC),
+	VC4_REG32(V3D_CT1LC),
+	VC4_REG32(V3D_CT0PC),
+	VC4_REG32(V3D_CT1PC),
+	VC4_REG32(V3D_PCS),
+	VC4_REG32(V3D_BFC),
+	VC4_REG32(V3D_RFC),
+	VC4_REG32(V3D_BPCA),
+	VC4_REG32(V3D_BPCS),
+	VC4_REG32(V3D_BPOA),
+	VC4_REG32(V3D_BPOS),
+	VC4_REG32(V3D_BXCF),
+	VC4_REG32(V3D_SQRSV0),
+	VC4_REG32(V3D_SQRSV1),
+	VC4_REG32(V3D_SQCNTL),
+	VC4_REG32(V3D_SRQPC),
+	VC4_REG32(V3D_SRQUA),
+	VC4_REG32(V3D_SRQUL),
+	VC4_REG32(V3D_SRQCS),
+	VC4_REG32(V3D_VPACNTL),
+	VC4_REG32(V3D_VPMBASE),
+	VC4_REG32(V3D_PCTRC),
+	VC4_REG32(V3D_PCTRE),
+	VC4_REG32(V3D_PCTR(0)),
+	VC4_REG32(V3D_PCTRS(0)),
+	VC4_REG32(V3D_PCTR(1)),
+	VC4_REG32(V3D_PCTRS(1)),
+	VC4_REG32(V3D_PCTR(2)),
+	VC4_REG32(V3D_PCTRS(2)),
+	VC4_REG32(V3D_PCTR(3)),
+	VC4_REG32(V3D_PCTRS(3)),
+	VC4_REG32(V3D_PCTR(4)),
+	VC4_REG32(V3D_PCTRS(4)),
+	VC4_REG32(V3D_PCTR(5)),
+	VC4_REG32(V3D_PCTRS(5)),
+	VC4_REG32(V3D_PCTR(6)),
+	VC4_REG32(V3D_PCTRS(6)),
+	VC4_REG32(V3D_PCTR(7)),
+	VC4_REG32(V3D_PCTRS(7)),
+	VC4_REG32(V3D_PCTR(8)),
+	VC4_REG32(V3D_PCTRS(8)),
+	VC4_REG32(V3D_PCTR(9)),
+	VC4_REG32(V3D_PCTRS(9)),
+	VC4_REG32(V3D_PCTR(10)),
+	VC4_REG32(V3D_PCTRS(10)),
+	VC4_REG32(V3D_PCTR(11)),
+	VC4_REG32(V3D_PCTRS(11)),
+	VC4_REG32(V3D_PCTR(12)),
+	VC4_REG32(V3D_PCTRS(12)),
+	VC4_REG32(V3D_PCTR(13)),
+	VC4_REG32(V3D_PCTRS(13)),
+	VC4_REG32(V3D_PCTR(14)),
+	VC4_REG32(V3D_PCTRS(14)),
+	VC4_REG32(V3D_PCTR(15)),
+	VC4_REG32(V3D_PCTRS(15)),
+	VC4_REG32(V3D_DBGE),
+	VC4_REG32(V3D_FDBGO),
+	VC4_REG32(V3D_FDBGB),
+	VC4_REG32(V3D_FDBGR),
+	VC4_REG32(V3D_FDBGS),
+	VC4_REG32(V3D_ERRSTAT),
 };
 
-int vc4_v3d_debugfs_regs(struct seq_file *m, void *unused)
+static int vc4_v3d_debugfs_ident(struct seq_file *m, void *unused)
 {
 	struct drm_info_node *node = (struct drm_info_node *)m->private;
 	struct drm_device *dev = node->minor->dev;
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(vc4_reg_defs); i++) {
-		seq_printf(m, "%s (0x%04x): 0x%08x\n",
-			   vc4_reg_defs[i].name, vc4_reg_defs[i].reg,
-			   V3D_READ(vc4_reg_defs[i].reg));
+	int ret = vc4_v3d_pm_get(vc4);
+
+	if (ret == 0) {
+		uint32_t ident1 = V3D_READ(V3D_IDENT1);
+		uint32_t nslc = VC4_GET_FIELD(ident1, V3D_IDENT1_NSLC);
+		uint32_t tups = VC4_GET_FIELD(ident1, V3D_IDENT1_TUPS);
+		uint32_t qups = VC4_GET_FIELD(ident1, V3D_IDENT1_QUPS);
+
+		seq_printf(m, "Revision:   %d\n",
+			   VC4_GET_FIELD(ident1, V3D_IDENT1_REV));
+		seq_printf(m, "Slices:     %d\n", nslc);
+		seq_printf(m, "TMUs:       %d\n", nslc * tups);
+		seq_printf(m, "QPUs:       %d\n", nslc * qups);
+		seq_printf(m, "Semaphores: %d\n",
+			   VC4_GET_FIELD(ident1, V3D_IDENT1_NSEM));
+		vc4_v3d_pm_put(vc4);
 	}
 
 	return 0;
 }
 
-int vc4_v3d_debugfs_ident(struct seq_file *m, void *unused)
+/**
+ * Wraps pm_runtime_get_sync() in a refcount, so that we can reliably
+ * get the pm_runtime refcount to 0 in vc4_reset().
+ */
+int
+vc4_v3d_pm_get(struct vc4_dev *vc4)
 {
-	struct drm_info_node *node = (struct drm_info_node *)m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct vc4_dev *vc4 = to_vc4_dev(dev);
-	uint32_t ident1 = V3D_READ(V3D_IDENT1);
-	uint32_t nslc = VC4_GET_FIELD(ident1, V3D_IDENT1_NSLC);
-	uint32_t tups = VC4_GET_FIELD(ident1, V3D_IDENT1_TUPS);
-	uint32_t qups = VC4_GET_FIELD(ident1, V3D_IDENT1_QUPS);
-
-	seq_printf(m, "Revision:   %d\n",
-		   VC4_GET_FIELD(ident1, V3D_IDENT1_REV));
-	seq_printf(m, "Slices:     %d\n", nslc);
-	seq_printf(m, "TMUs:       %d\n", nslc * tups);
-	seq_printf(m, "QPUs:       %d\n", nslc * qups);
-	seq_printf(m, "Semaphores: %d\n",
-		   VC4_GET_FIELD(ident1, V3D_IDENT1_NSEM));
+	mutex_lock(&vc4->power_lock);
+	if (vc4->power_refcount++ == 0) {
+		int ret = pm_runtime_get_sync(&vc4->v3d->pdev->dev);
+
+		if (ret < 0) {
+			vc4->power_refcount--;
+			mutex_unlock(&vc4->power_lock);
+			return ret;
+		}
+	}
+	mutex_unlock(&vc4->power_lock);
 
 	return 0;
 }
-#endif /* CONFIG_DEBUG_FS */
+
+void
+vc4_v3d_pm_put(struct vc4_dev *vc4)
+{
+	mutex_lock(&vc4->power_lock);
+	if (--vc4->power_refcount == 0) {
+		pm_runtime_mark_last_busy(&vc4->v3d->pdev->dev);
+		pm_runtime_put_autosuspend(&vc4->v3d->pdev->dev);
+	}
+	mutex_unlock(&vc4->power_lock);
+}
 
 static void vc4_v3d_init_hw(struct drm_device *dev)
 {
@@ -354,6 +370,9 @@ static int vc4_v3d_bind(struct device *dev, struct device *master, void *data)
 	v3d->regs = vc4_ioremap_regs(pdev, 0);
 	if (IS_ERR(v3d->regs))
 		return PTR_ERR(v3d->regs);
+	v3d->regset.base = v3d->regs;
+	v3d->regset.regs = v3d_regs;
+	v3d->regset.nregs = ARRAY_SIZE(v3d_regs);
 
 	vc4->v3d = v3d;
 	v3d->vc4 = vc4;
@@ -409,6 +428,9 @@ static int vc4_v3d_bind(struct device *dev, struct device *master, void *data)
 	pm_runtime_set_autosuspend_delay(dev, 40); /* a little over 2 frames. */
 	pm_runtime_enable(dev);
 
+	vc4_debugfs_add_file(drm, "v3d_ident", vc4_v3d_debugfs_ident, NULL);
+	vc4_debugfs_add_regset32(drm, "v3d_regs", &v3d->regset);
+
 	return 0;
 }
 
@@ -452,7 +474,7 @@ static int vc4_v3d_dev_remove(struct platform_device *pdev)
 	return 0;
 }
 
-static const struct of_device_id vc4_v3d_dt_match[] = {
+const struct of_device_id vc4_v3d_dt_match[] = {
 	{ .compatible = "brcm,bcm2835-v3d" },
 	{ .compatible = "brcm,cygnus-v3d" },
 	{ .compatible = "brcm,vc4-v3d" },
diff --git a/drivers/gpu/drm/vc4/vc4_vec.c b/drivers/gpu/drm/vc4/vc4_vec.c
index 858c3a483229..0a27e48fab31 100644
--- a/drivers/gpu/drm/vc4/vc4_vec.c
+++ b/drivers/gpu/drm/vc4/vc4_vec.c
@@ -176,6 +176,8 @@ struct vc4_vec {
 	struct clk *clock;
 
 	const struct vc4_vec_tv_mode *tv_mode;
+
+	struct debugfs_regset32 regset;
 };
 
 #define VEC_READ(offset) readl(vec->regs + (offset))
@@ -223,59 +225,33 @@ struct vc4_vec_tv_mode {
 	void (*mode_set)(struct vc4_vec *vec);
 };
 
-#define VEC_REG(reg) { reg, #reg }
-static const struct {
-	u32 reg;
-	const char *name;
-} vec_regs[] = {
-	VEC_REG(VEC_WSE_CONTROL),
-	VEC_REG(VEC_WSE_WSS_DATA),
-	VEC_REG(VEC_WSE_VPS_DATA1),
-	VEC_REG(VEC_WSE_VPS_CONTROL),
-	VEC_REG(VEC_REVID),
-	VEC_REG(VEC_CONFIG0),
-	VEC_REG(VEC_SCHPH),
-	VEC_REG(VEC_CLMP0_START),
-	VEC_REG(VEC_CLMP0_END),
-	VEC_REG(VEC_FREQ3_2),
-	VEC_REG(VEC_FREQ1_0),
-	VEC_REG(VEC_CONFIG1),
-	VEC_REG(VEC_CONFIG2),
-	VEC_REG(VEC_INTERRUPT_CONTROL),
-	VEC_REG(VEC_INTERRUPT_STATUS),
-	VEC_REG(VEC_FCW_SECAM_B),
-	VEC_REG(VEC_SECAM_GAIN_VAL),
-	VEC_REG(VEC_CONFIG3),
-	VEC_REG(VEC_STATUS0),
-	VEC_REG(VEC_MASK0),
-	VEC_REG(VEC_CFG),
-	VEC_REG(VEC_DAC_TEST),
-	VEC_REG(VEC_DAC_CONFIG),
-	VEC_REG(VEC_DAC_MISC),
+static const struct debugfs_reg32 vec_regs[] = {
+	VC4_REG32(VEC_WSE_CONTROL),
+	VC4_REG32(VEC_WSE_WSS_DATA),
+	VC4_REG32(VEC_WSE_VPS_DATA1),
+	VC4_REG32(VEC_WSE_VPS_CONTROL),
+	VC4_REG32(VEC_REVID),
+	VC4_REG32(VEC_CONFIG0),
+	VC4_REG32(VEC_SCHPH),
+	VC4_REG32(VEC_CLMP0_START),
+	VC4_REG32(VEC_CLMP0_END),
+	VC4_REG32(VEC_FREQ3_2),
+	VC4_REG32(VEC_FREQ1_0),
+	VC4_REG32(VEC_CONFIG1),
+	VC4_REG32(VEC_CONFIG2),
+	VC4_REG32(VEC_INTERRUPT_CONTROL),
+	VC4_REG32(VEC_INTERRUPT_STATUS),
+	VC4_REG32(VEC_FCW_SECAM_B),
+	VC4_REG32(VEC_SECAM_GAIN_VAL),
+	VC4_REG32(VEC_CONFIG3),
+	VC4_REG32(VEC_STATUS0),
+	VC4_REG32(VEC_MASK0),
+	VC4_REG32(VEC_CFG),
+	VC4_REG32(VEC_DAC_TEST),
+	VC4_REG32(VEC_DAC_CONFIG),
+	VC4_REG32(VEC_DAC_MISC),
 };
 
-#ifdef CONFIG_DEBUG_FS
-int vc4_vec_debugfs_regs(struct seq_file *m, void *unused)
-{
-	struct drm_info_node *node = (struct drm_info_node *)m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct vc4_dev *vc4 = to_vc4_dev(dev);
-	struct vc4_vec *vec = vc4->vec;
-	int i;
-
-	if (!vec)
-		return 0;
-
-	for (i = 0; i < ARRAY_SIZE(vec_regs); i++) {
-		seq_printf(m, "%s (0x%04x): 0x%08x\n",
-			   vec_regs[i].name, vec_regs[i].reg,
-			   VEC_READ(vec_regs[i].reg));
-	}
-
-	return 0;
-}
-#endif
-
 static void vc4_vec_ntsc_mode_set(struct vc4_vec *vec)
 {
 	VEC_WRITE(VEC_CONFIG0, VEC_CONFIG0_NTSC_STD | VEC_CONFIG0_PDEN);
@@ -587,6 +563,9 @@ static int vc4_vec_bind(struct device *dev, struct device *master, void *data)
 	vec->regs = vc4_ioremap_regs(pdev, 0);
 	if (IS_ERR(vec->regs))
 		return PTR_ERR(vec->regs);
+	vec->regset.base = vec->regs;
+	vec->regset.regs = vec_regs;
+	vec->regset.nregs = ARRAY_SIZE(vec_regs);
 
 	vec->clock = devm_clk_get(dev, NULL);
 	if (IS_ERR(vec->clock)) {
@@ -612,6 +591,8 @@ static int vc4_vec_bind(struct device *dev, struct device *master, void *data)
 
 	vc4->vec = vec;
 
+	vc4_debugfs_add_regset32(drm, "vec_regs", &vec->regset);
+
 	return 0;
 
 err_destroy_encoder:
diff --git a/drivers/gpu/drm/virtio/virtgpu_debugfs.c b/drivers/gpu/drm/virtio/virtgpu_debugfs.c
index 73dc99046c43..ed0fcda713c3 100644
--- a/drivers/gpu/drm/virtio/virtgpu_debugfs.c
+++ b/drivers/gpu/drm/virtio/virtgpu_debugfs.c
@@ -28,6 +28,30 @@
 
 #include "virtgpu_drv.h"
 
+static void virtio_add_bool(struct seq_file *m, const char *name,
+				    bool value)
+{
+	seq_printf(m, "%-16s : %s\n", name, value ? "yes" : "no");
+}
+
+static void virtio_add_int(struct seq_file *m, const char *name,
+				   int value)
+{
+	seq_printf(m, "%-16s : %d\n", name, value);
+}
+
+static int virtio_gpu_features(struct seq_file *m, void *data)
+{
+	struct drm_info_node *node = (struct drm_info_node *) m->private;
+	struct virtio_gpu_device *vgdev = node->minor->dev->dev_private;
+
+	virtio_add_bool(m, "virgl", vgdev->has_virgl_3d);
+	virtio_add_bool(m, "edid", vgdev->has_edid);
+	virtio_add_int(m, "cap sets", vgdev->num_capsets);
+	virtio_add_int(m, "scanouts", vgdev->num_scanouts);
+	return 0;
+}
+
 static int
 virtio_gpu_debugfs_irq_info(struct seq_file *m, void *data)
 {
@@ -41,7 +65,8 @@ virtio_gpu_debugfs_irq_info(struct seq_file *m, void *data)
 }
 
 static struct drm_info_list virtio_gpu_debugfs_list[] = {
-	{ "irq_fence", virtio_gpu_debugfs_irq_info, 0, NULL },
+	{ "virtio-gpu-features", virtio_gpu_features },
+	{ "virtio-gpu-irq-fence", virtio_gpu_debugfs_irq_info, 0, NULL },
 };
 
 #define VIRTIO_GPU_DEBUGFS_ENTRIES ARRAY_SIZE(virtio_gpu_debugfs_list)
diff --git a/drivers/gpu/drm/virtio/virtgpu_display.c b/drivers/gpu/drm/virtio/virtgpu_display.c
index 653ec7d0bf4d..86843a4d6102 100644
--- a/drivers/gpu/drm/virtio/virtgpu_display.c
+++ b/drivers/gpu/drm/virtio/virtgpu_display.c
@@ -385,5 +385,6 @@ void virtio_gpu_modeset_fini(struct virtio_gpu_device *vgdev)
 
 	for (i = 0 ; i < vgdev->num_scanouts; ++i)
 		kfree(vgdev->outputs[i].edid);
+	drm_atomic_helper_shutdown(vgdev->ddev);
 	drm_mode_config_cleanup(vgdev->ddev);
 }
diff --git a/drivers/gpu/drm/virtio/virtgpu_drv.c b/drivers/gpu/drm/virtio/virtgpu_drv.c
index af92964b6889..c50868753132 100644
--- a/drivers/gpu/drm/virtio/virtgpu_drv.c
+++ b/drivers/gpu/drm/virtio/virtgpu_drv.c
@@ -209,8 +209,6 @@ static struct drm_driver driver = {
 	.prime_fd_to_handle = drm_gem_prime_fd_to_handle,
 	.gem_prime_export = drm_gem_prime_export,
 	.gem_prime_import = drm_gem_prime_import,
-	.gem_prime_pin = virtgpu_gem_prime_pin,
-	.gem_prime_unpin = virtgpu_gem_prime_unpin,
 	.gem_prime_get_sg_table = virtgpu_gem_prime_get_sg_table,
 	.gem_prime_import_sg_table = virtgpu_gem_prime_import_sg_table,
 	.gem_prime_vmap = virtgpu_gem_prime_vmap,
diff --git a/drivers/gpu/drm/virtio/virtgpu_drv.h b/drivers/gpu/drm/virtio/virtgpu_drv.h
index d577cb76f5ad..b69ae10ca238 100644
--- a/drivers/gpu/drm/virtio/virtgpu_drv.h
+++ b/drivers/gpu/drm/virtio/virtgpu_drv.h
@@ -50,6 +50,23 @@
 #define DRIVER_MINOR 1
 #define DRIVER_PATCHLEVEL 0
 
+struct virtio_gpu_object_params {
+	uint32_t format;
+	uint32_t width;
+	uint32_t height;
+	unsigned long size;
+	bool dumb;
+	/* 3d */
+	bool virgl;
+	uint32_t target;
+	uint32_t bind;
+	uint32_t depth;
+	uint32_t array_size;
+	uint32_t last_level;
+	uint32_t nr_samples;
+	uint32_t flags;
+};
+
 struct virtio_gpu_object {
 	struct drm_gem_object gem_base;
 	uint32_t hw_res_handle;
@@ -204,6 +221,9 @@ struct virtio_gpu_fpriv {
 /* virtio_ioctl.c */
 #define DRM_VIRTIO_NUM_IOCTLS 10
 extern struct drm_ioctl_desc virtio_gpu_ioctls[DRM_VIRTIO_NUM_IOCTLS];
+int virtio_gpu_object_list_validate(struct ww_acquire_ctx *ticket,
+				    struct list_head *head);
+void virtio_gpu_unref_list(struct list_head *head);
 
 /* virtio_kms.c */
 int virtio_gpu_init(struct drm_device *dev);
@@ -217,16 +237,17 @@ int virtio_gpu_gem_init(struct virtio_gpu_device *vgdev);
 void virtio_gpu_gem_fini(struct virtio_gpu_device *vgdev);
 int virtio_gpu_gem_create(struct drm_file *file,
 			  struct drm_device *dev,
-			  uint64_t size,
+			  struct virtio_gpu_object_params *params,
 			  struct drm_gem_object **obj_p,
 			  uint32_t *handle_p);
 int virtio_gpu_gem_object_open(struct drm_gem_object *obj,
 			       struct drm_file *file);
 void virtio_gpu_gem_object_close(struct drm_gem_object *obj,
 				 struct drm_file *file);
-struct virtio_gpu_object *virtio_gpu_alloc_object(struct drm_device *dev,
-						  size_t size, bool kernel,
-						  bool pinned);
+struct virtio_gpu_object*
+virtio_gpu_alloc_object(struct drm_device *dev,
+			struct virtio_gpu_object_params *params,
+			struct virtio_gpu_fence *fence);
 int virtio_gpu_mode_dumb_create(struct drm_file *file_priv,
 				struct drm_device *dev,
 				struct drm_mode_create_dumb *args);
@@ -243,9 +264,8 @@ int virtio_gpu_alloc_vbufs(struct virtio_gpu_device *vgdev);
 void virtio_gpu_free_vbufs(struct virtio_gpu_device *vgdev);
 void virtio_gpu_cmd_create_resource(struct virtio_gpu_device *vgdev,
 				    struct virtio_gpu_object *bo,
-				    uint32_t format,
-				    uint32_t width,
-				    uint32_t height);
+				    struct virtio_gpu_object_params *params,
+				    struct virtio_gpu_fence *fence);
 void virtio_gpu_cmd_unref_resource(struct virtio_gpu_device *vgdev,
 				   uint32_t resource_id);
 void virtio_gpu_cmd_transfer_to_host_2d(struct virtio_gpu_device *vgdev,
@@ -304,7 +324,8 @@ void virtio_gpu_cmd_transfer_to_host_3d(struct virtio_gpu_device *vgdev,
 void
 virtio_gpu_cmd_resource_create_3d(struct virtio_gpu_device *vgdev,
 				  struct virtio_gpu_object *bo,
-				  struct virtio_gpu_resource_create_3d *rc_3d);
+				  struct virtio_gpu_object_params *params,
+				  struct virtio_gpu_fence *fence);
 void virtio_gpu_ctrl_ack(struct virtqueue *vq);
 void virtio_gpu_cursor_ack(struct virtqueue *vq);
 void virtio_gpu_fence_ack(struct virtqueue *vq);
@@ -332,6 +353,7 @@ void virtio_gpu_ttm_fini(struct virtio_gpu_device *vgdev);
 int virtio_gpu_mmap(struct file *filp, struct vm_area_struct *vma);
 
 /* virtio_gpu_fence.c */
+bool virtio_fence_signaled(struct dma_fence *f);
 struct virtio_gpu_fence *virtio_gpu_fence_alloc(
 	struct virtio_gpu_device *vgdev);
 int virtio_gpu_fence_emit(struct virtio_gpu_device *vgdev,
@@ -342,8 +364,9 @@ void virtio_gpu_fence_event_process(struct virtio_gpu_device *vdev,
 
 /* virtio_gpu_object */
 int virtio_gpu_object_create(struct virtio_gpu_device *vgdev,
-			     unsigned long size, bool kernel, bool pinned,
-			     struct virtio_gpu_object **bo_ptr);
+			     struct virtio_gpu_object_params *params,
+			     struct virtio_gpu_object **bo_ptr,
+			     struct virtio_gpu_fence *fence);
 void virtio_gpu_object_kunmap(struct virtio_gpu_object *bo);
 int virtio_gpu_object_kmap(struct virtio_gpu_object *bo);
 int virtio_gpu_object_get_sg_table(struct virtio_gpu_device *qdev,
@@ -352,8 +375,6 @@ void virtio_gpu_object_free_sg_table(struct virtio_gpu_object *bo);
 int virtio_gpu_object_wait(struct virtio_gpu_object *bo, bool no_wait);
 
 /* virtgpu_prime.c */
-int virtgpu_gem_prime_pin(struct drm_gem_object *obj);
-void virtgpu_gem_prime_unpin(struct drm_gem_object *obj);
 struct sg_table *virtgpu_gem_prime_get_sg_table(struct drm_gem_object *obj);
 struct drm_gem_object *virtgpu_gem_prime_import_sg_table(
 	struct drm_device *dev, struct dma_buf_attachment *attach,
diff --git a/drivers/gpu/drm/virtio/virtgpu_fence.c b/drivers/gpu/drm/virtio/virtgpu_fence.c
index 21bd4c4a32d1..87d1966192f4 100644
--- a/drivers/gpu/drm/virtio/virtgpu_fence.c
+++ b/drivers/gpu/drm/virtio/virtgpu_fence.c
@@ -36,7 +36,7 @@ static const char *virtio_get_timeline_name(struct dma_fence *f)
 	return "controlq";
 }
 
-static bool virtio_signaled(struct dma_fence *f)
+bool virtio_fence_signaled(struct dma_fence *f)
 {
 	struct virtio_gpu_fence *fence = to_virtio_fence(f);
 
@@ -62,7 +62,7 @@ static void virtio_timeline_value_str(struct dma_fence *f, char *str, int size)
 static const struct dma_fence_ops virtio_fence_ops = {
 	.get_driver_name     = virtio_get_driver_name,
 	.get_timeline_name   = virtio_get_timeline_name,
-	.signaled            = virtio_signaled,
+	.signaled            = virtio_fence_signaled,
 	.fence_value_str     = virtio_fence_value_str,
 	.timeline_value_str  = virtio_timeline_value_str,
 };
diff --git a/drivers/gpu/drm/virtio/virtgpu_gem.c b/drivers/gpu/drm/virtio/virtgpu_gem.c
index f06586393974..1e49e08dd545 100644
--- a/drivers/gpu/drm/virtio/virtgpu_gem.c
+++ b/drivers/gpu/drm/virtio/virtgpu_gem.c
@@ -34,15 +34,16 @@ void virtio_gpu_gem_free_object(struct drm_gem_object *gem_obj)
 		virtio_gpu_object_unref(&obj);
 }
 
-struct virtio_gpu_object *virtio_gpu_alloc_object(struct drm_device *dev,
-						  size_t size, bool kernel,
-						  bool pinned)
+struct virtio_gpu_object*
+virtio_gpu_alloc_object(struct drm_device *dev,
+			struct virtio_gpu_object_params *params,
+			struct virtio_gpu_fence *fence)
 {
 	struct virtio_gpu_device *vgdev = dev->dev_private;
 	struct virtio_gpu_object *obj;
 	int ret;
 
-	ret = virtio_gpu_object_create(vgdev, size, kernel, pinned, &obj);
+	ret = virtio_gpu_object_create(vgdev, params, &obj, fence);
 	if (ret)
 		return ERR_PTR(ret);
 
@@ -51,7 +52,7 @@ struct virtio_gpu_object *virtio_gpu_alloc_object(struct drm_device *dev,
 
 int virtio_gpu_gem_create(struct drm_file *file,
 			  struct drm_device *dev,
-			  uint64_t size,
+			  struct virtio_gpu_object_params *params,
 			  struct drm_gem_object **obj_p,
 			  uint32_t *handle_p)
 {
@@ -59,7 +60,7 @@ int virtio_gpu_gem_create(struct drm_file *file,
 	int ret;
 	u32 handle;
 
-	obj = virtio_gpu_alloc_object(dev, size, false, false);
+	obj = virtio_gpu_alloc_object(dev, params, NULL);
 	if (IS_ERR(obj))
 		return PTR_ERR(obj);
 
@@ -82,12 +83,10 @@ int virtio_gpu_mode_dumb_create(struct drm_file *file_priv,
 				struct drm_device *dev,
 				struct drm_mode_create_dumb *args)
 {
-	struct virtio_gpu_device *vgdev = dev->dev_private;
 	struct drm_gem_object *gobj;
-	struct virtio_gpu_object *obj;
+	struct virtio_gpu_object_params params = { 0 };
 	int ret;
 	uint32_t pitch;
-	uint32_t format;
 
 	if (args->bpp != 32)
 		return -EINVAL;
@@ -96,22 +95,16 @@ int virtio_gpu_mode_dumb_create(struct drm_file *file_priv,
 	args->size = pitch * args->height;
 	args->size = ALIGN(args->size, PAGE_SIZE);
 
-	ret = virtio_gpu_gem_create(file_priv, dev, args->size, &gobj,
+	params.format = virtio_gpu_translate_format(DRM_FORMAT_HOST_XRGB8888);
+	params.width = args->width;
+	params.height = args->height;
+	params.size = args->size;
+	params.dumb = true;
+	ret = virtio_gpu_gem_create(file_priv, dev, &params, &gobj,
 				    &args->handle);
 	if (ret)
 		goto fail;
 
-	format = virtio_gpu_translate_format(DRM_FORMAT_HOST_XRGB8888);
-	obj = gem_to_virtio_gpu_obj(gobj);
-	virtio_gpu_cmd_create_resource(vgdev, obj, format,
-				       args->width, args->height);
-
-	/* attach the object to the resource */
-	ret = virtio_gpu_object_attach(vgdev, obj, NULL);
-	if (ret)
-		goto fail;
-
-	obj->dumb = true;
 	args->pitch = pitch;
 	return ret;
 
diff --git a/drivers/gpu/drm/virtio/virtgpu_ioctl.c b/drivers/gpu/drm/virtio/virtgpu_ioctl.c
index 14ce8188c052..949a264985fc 100644
--- a/drivers/gpu/drm/virtio/virtgpu_ioctl.c
+++ b/drivers/gpu/drm/virtio/virtgpu_ioctl.c
@@ -54,8 +54,8 @@ static int virtio_gpu_map_ioctl(struct drm_device *dev, void *data,
 					 &virtio_gpu_map->offset);
 }
 
-static int virtio_gpu_object_list_validate(struct ww_acquire_ctx *ticket,
-					   struct list_head *head)
+int virtio_gpu_object_list_validate(struct ww_acquire_ctx *ticket,
+				    struct list_head *head)
 {
 	struct ttm_operation_ctx ctx = { false, false };
 	struct ttm_validate_buffer *buf;
@@ -79,7 +79,7 @@ static int virtio_gpu_object_list_validate(struct ww_acquire_ctx *ticket,
 	return 0;
 }
 
-static void virtio_gpu_unref_list(struct list_head *head)
+void virtio_gpu_unref_list(struct list_head *head)
 {
 	struct ttm_validate_buffer *buf;
 	struct ttm_buffer_object *bo;
@@ -275,16 +275,12 @@ static int virtio_gpu_resource_create_ioctl(struct drm_device *dev, void *data,
 {
 	struct virtio_gpu_device *vgdev = dev->dev_private;
 	struct drm_virtgpu_resource_create *rc = data;
+	struct virtio_gpu_fence *fence;
 	int ret;
 	struct virtio_gpu_object *qobj;
 	struct drm_gem_object *obj;
 	uint32_t handle = 0;
-	uint32_t size;
-	struct list_head validate_list;
-	struct ttm_validate_buffer mainbuf;
-	struct virtio_gpu_fence *fence = NULL;
-	struct ww_acquire_ctx ticket;
-	struct virtio_gpu_resource_create_3d rc_3d;
+	struct virtio_gpu_object_params params = { 0 };
 
 	if (vgdev->has_virgl_3d == false) {
 		if (rc->depth > 1)
@@ -299,94 +295,43 @@ static int virtio_gpu_resource_create_ioctl(struct drm_device *dev, void *data,
 			return -EINVAL;
 	}
 
-	INIT_LIST_HEAD(&validate_list);
-	memset(&mainbuf, 0, sizeof(struct ttm_validate_buffer));
-
-	size = rc->size;
-
+	params.format = rc->format;
+	params.width = rc->width;
+	params.height = rc->height;
+	params.size = rc->size;
+	if (vgdev->has_virgl_3d) {
+		params.virgl = true;
+		params.target = rc->target;
+		params.bind = rc->bind;
+		params.depth = rc->depth;
+		params.array_size = rc->array_size;
+		params.last_level = rc->last_level;
+		params.nr_samples = rc->nr_samples;
+		params.flags = rc->flags;
+	}
 	/* allocate a single page size object */
-	if (size == 0)
-		size = PAGE_SIZE;
+	if (params.size == 0)
+		params.size = PAGE_SIZE;
 
-	qobj = virtio_gpu_alloc_object(dev, size, false, false);
+	fence = virtio_gpu_fence_alloc(vgdev);
+	if (!fence)
+		return -ENOMEM;
+	qobj = virtio_gpu_alloc_object(dev, &params, fence);
+	dma_fence_put(&fence->f);
 	if (IS_ERR(qobj))
 		return PTR_ERR(qobj);
 	obj = &qobj->gem_base;
 
-	if (!vgdev->has_virgl_3d) {
-		virtio_gpu_cmd_create_resource(vgdev, qobj, rc->format,
-					       rc->width, rc->height);
-
-		ret = virtio_gpu_object_attach(vgdev, qobj, NULL);
-	} else {
-		/* use a gem reference since unref list undoes them */
-		drm_gem_object_get(&qobj->gem_base);
-		mainbuf.bo = &qobj->tbo;
-		list_add(&mainbuf.head, &validate_list);
-
-		ret = virtio_gpu_object_list_validate(&ticket, &validate_list);
-		if (ret) {
-			DRM_DEBUG("failed to validate\n");
-			goto fail_unref;
-		}
-
-		rc_3d.resource_id = cpu_to_le32(qobj->hw_res_handle);
-		rc_3d.target = cpu_to_le32(rc->target);
-		rc_3d.format = cpu_to_le32(rc->format);
-		rc_3d.bind = cpu_to_le32(rc->bind);
-		rc_3d.width = cpu_to_le32(rc->width);
-		rc_3d.height = cpu_to_le32(rc->height);
-		rc_3d.depth = cpu_to_le32(rc->depth);
-		rc_3d.array_size = cpu_to_le32(rc->array_size);
-		rc_3d.last_level = cpu_to_le32(rc->last_level);
-		rc_3d.nr_samples = cpu_to_le32(rc->nr_samples);
-		rc_3d.flags = cpu_to_le32(rc->flags);
-
-		fence = virtio_gpu_fence_alloc(vgdev);
-		if (!fence) {
-			ret = -ENOMEM;
-			goto fail_backoff;
-		}
-
-		virtio_gpu_cmd_resource_create_3d(vgdev, qobj, &rc_3d);
-		ret = virtio_gpu_object_attach(vgdev, qobj, fence);
-		if (ret) {
-			dma_fence_put(&fence->f);
-			goto fail_backoff;
-		}
-		ttm_eu_fence_buffer_objects(&ticket, &validate_list, &fence->f);
-	}
-
 	ret = drm_gem_handle_create(file_priv, obj, &handle);
 	if (ret) {
-
 		drm_gem_object_release(obj);
-		if (vgdev->has_virgl_3d) {
-			virtio_gpu_unref_list(&validate_list);
-			dma_fence_put(&fence->f);
-		}
 		return ret;
 	}
 	drm_gem_object_put_unlocked(obj);
 
 	rc->res_handle = qobj->hw_res_handle; /* similiar to a VM address */
 	rc->bo_handle = handle;
-
-	if (vgdev->has_virgl_3d) {
-		virtio_gpu_unref_list(&validate_list);
-		dma_fence_put(&fence->f);
-	}
 	return 0;
-fail_backoff:
-	ttm_eu_backoff_reservation(&ticket, &validate_list);
-fail_unref:
-	if (vgdev->has_virgl_3d) {
-		virtio_gpu_unref_list(&validate_list);
-		dma_fence_put(&fence->f);
-	}
-//fail_obj:
-//	drm_gem_object_handle_unreference_unlocked(obj);
-	return ret;
 }
 
 static int virtio_gpu_resource_info_ioctl(struct drm_device *dev, void *data,
diff --git a/drivers/gpu/drm/virtio/virtgpu_object.c b/drivers/gpu/drm/virtio/virtgpu_object.c
index e7e946035027..b2da31310d24 100644
--- a/drivers/gpu/drm/virtio/virtgpu_object.c
+++ b/drivers/gpu/drm/virtio/virtgpu_object.c
@@ -23,6 +23,8 @@
  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
+#include <drm/ttm/ttm_execbuf_util.h>
+
 #include "virtgpu_drv.h"
 
 static int virtio_gpu_resource_id_get(struct virtio_gpu_device *vgdev,
@@ -74,39 +76,34 @@ static void virtio_gpu_ttm_bo_destroy(struct ttm_buffer_object *tbo)
 	kfree(bo);
 }
 
-static void virtio_gpu_init_ttm_placement(struct virtio_gpu_object *vgbo,
-					  bool pinned)
+static void virtio_gpu_init_ttm_placement(struct virtio_gpu_object *vgbo)
 {
 	u32 c = 1;
-	u32 pflag = pinned ? TTM_PL_FLAG_NO_EVICT : 0;
 
 	vgbo->placement.placement = &vgbo->placement_code;
 	vgbo->placement.busy_placement = &vgbo->placement_code;
 	vgbo->placement_code.fpfn = 0;
 	vgbo->placement_code.lpfn = 0;
 	vgbo->placement_code.flags =
-		TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT | pflag;
+		TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT |
+		TTM_PL_FLAG_NO_EVICT;
 	vgbo->placement.num_placement = c;
 	vgbo->placement.num_busy_placement = c;
 
 }
 
 int virtio_gpu_object_create(struct virtio_gpu_device *vgdev,
-			     unsigned long size, bool kernel, bool pinned,
-			     struct virtio_gpu_object **bo_ptr)
+			     struct virtio_gpu_object_params *params,
+			     struct virtio_gpu_object **bo_ptr,
+			     struct virtio_gpu_fence *fence)
 {
 	struct virtio_gpu_object *bo;
-	enum ttm_bo_type type;
 	size_t acc_size;
 	int ret;
 
-	if (kernel)
-		type = ttm_bo_type_kernel;
-	else
-		type = ttm_bo_type_device;
 	*bo_ptr = NULL;
 
-	acc_size = ttm_bo_dma_acc_size(&vgdev->mman.bdev, size,
+	acc_size = ttm_bo_dma_acc_size(&vgdev->mman.bdev, params->size,
 				       sizeof(struct virtio_gpu_object));
 
 	bo = kzalloc(sizeof(struct virtio_gpu_object), GFP_KERNEL);
@@ -117,23 +114,62 @@ int virtio_gpu_object_create(struct virtio_gpu_device *vgdev,
 		kfree(bo);
 		return ret;
 	}
-	size = roundup(size, PAGE_SIZE);
-	ret = drm_gem_object_init(vgdev->ddev, &bo->gem_base, size);
+	params->size = roundup(params->size, PAGE_SIZE);
+	ret = drm_gem_object_init(vgdev->ddev, &bo->gem_base, params->size);
 	if (ret != 0) {
 		virtio_gpu_resource_id_put(vgdev, bo->hw_res_handle);
 		kfree(bo);
 		return ret;
 	}
-	bo->dumb = false;
-	virtio_gpu_init_ttm_placement(bo, pinned);
+	bo->dumb = params->dumb;
+
+	if (params->virgl) {
+		virtio_gpu_cmd_resource_create_3d(vgdev, bo, params, fence);
+	} else {
+		virtio_gpu_cmd_create_resource(vgdev, bo, params, fence);
+	}
 
-	ret = ttm_bo_init(&vgdev->mman.bdev, &bo->tbo, size, type,
-			  &bo->placement, 0, !kernel, acc_size,
-			  NULL, NULL, &virtio_gpu_ttm_bo_destroy);
+	virtio_gpu_init_ttm_placement(bo);
+	ret = ttm_bo_init(&vgdev->mman.bdev, &bo->tbo, params->size,
+			  ttm_bo_type_device, &bo->placement, 0,
+			  true, acc_size, NULL, NULL,
+			  &virtio_gpu_ttm_bo_destroy);
 	/* ttm_bo_init failure will call the destroy */
 	if (ret != 0)
 		return ret;
 
+	if (fence) {
+		struct virtio_gpu_fence_driver *drv = &vgdev->fence_drv;
+		struct list_head validate_list;
+		struct ttm_validate_buffer mainbuf;
+		struct ww_acquire_ctx ticket;
+		unsigned long irq_flags;
+		bool signaled;
+
+		INIT_LIST_HEAD(&validate_list);
+		memset(&mainbuf, 0, sizeof(struct ttm_validate_buffer));
+
+		/* use a gem reference since unref list undoes them */
+		drm_gem_object_get(&bo->gem_base);
+		mainbuf.bo = &bo->tbo;
+		list_add(&mainbuf.head, &validate_list);
+
+		ret = virtio_gpu_object_list_validate(&ticket, &validate_list);
+		if (ret == 0) {
+			spin_lock_irqsave(&drv->lock, irq_flags);
+			signaled = virtio_fence_signaled(&fence->f);
+			if (!signaled)
+				/* virtio create command still in flight */
+				ttm_eu_fence_buffer_objects(&ticket, &validate_list,
+							    &fence->f);
+			spin_unlock_irqrestore(&drv->lock, irq_flags);
+			if (signaled)
+				/* virtio create command finished */
+				ttm_eu_backoff_reservation(&ticket, &validate_list);
+		}
+		virtio_gpu_unref_list(&validate_list);
+	}
+
 	*bo_ptr = bo;
 	return 0;
 }
diff --git a/drivers/gpu/drm/virtio/virtgpu_prime.c b/drivers/gpu/drm/virtio/virtgpu_prime.c
index eb51a78e1199..8fbf71bd0c5e 100644
--- a/drivers/gpu/drm/virtio/virtgpu_prime.c
+++ b/drivers/gpu/drm/virtio/virtgpu_prime.c
@@ -28,20 +28,16 @@
  * device that might share buffers with virtgpu
  */
 
-int virtgpu_gem_prime_pin(struct drm_gem_object *obj)
+struct sg_table *virtgpu_gem_prime_get_sg_table(struct drm_gem_object *obj)
 {
-	WARN_ONCE(1, "not implemented");
-	return -ENODEV;
-}
+	struct virtio_gpu_object *bo = gem_to_virtio_gpu_obj(obj);
 
-void virtgpu_gem_prime_unpin(struct drm_gem_object *obj)
-{
-	WARN_ONCE(1, "not implemented");
-}
+	if (!bo->tbo.ttm->pages || !bo->tbo.ttm->num_pages)
+		/* should not happen */
+		return ERR_PTR(-EINVAL);
 
-struct sg_table *virtgpu_gem_prime_get_sg_table(struct drm_gem_object *obj)
-{
-	return ERR_PTR(-ENODEV);
+	return drm_prime_pages_to_sg(bo->tbo.ttm->pages,
+				     bo->tbo.ttm->num_pages);
 }
 
 struct drm_gem_object *virtgpu_gem_prime_import_sg_table(
@@ -68,7 +64,10 @@ void virtgpu_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr)
 }
 
 int virtgpu_gem_prime_mmap(struct drm_gem_object *obj,
-		       struct vm_area_struct *area)
+			   struct vm_area_struct *vma)
 {
-	return -ENODEV;
+	struct virtio_gpu_object *bo = gem_to_virtio_gpu_obj(obj);
+
+	bo->gem_base.vma_node.vm_node.start = bo->tbo.vma_node.vm_node.start;
+	return drm_gem_prime_mmap(obj, vma);
 }
diff --git a/drivers/gpu/drm/virtio/virtgpu_ttm.c b/drivers/gpu/drm/virtio/virtgpu_ttm.c
index 4bfbf25fabff..300ef3a83538 100644
--- a/drivers/gpu/drm/virtio/virtgpu_ttm.c
+++ b/drivers/gpu/drm/virtio/virtgpu_ttm.c
@@ -37,8 +37,6 @@
 
 #include <linux/delay.h>
 
-#define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT)
-
 static struct
 virtio_gpu_device *virtio_gpu_get_vgdev(struct ttm_bo_device *bdev)
 {
@@ -116,10 +114,6 @@ static const struct ttm_mem_type_manager_func virtio_gpu_bo_manager_func = {
 static int virtio_gpu_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
 				    struct ttm_mem_type_manager *man)
 {
-	struct virtio_gpu_device *vgdev;
-
-	vgdev = virtio_gpu_get_vgdev(bdev);
-
 	switch (type) {
 	case TTM_PL_SYSTEM:
 		/* System memory */
@@ -194,42 +188,45 @@ static void virtio_gpu_ttm_io_mem_free(struct ttm_bo_device *bdev,
  */
 struct virtio_gpu_ttm_tt {
 	struct ttm_dma_tt		ttm;
-	struct virtio_gpu_device	*vgdev;
-	u64				offset;
+	struct virtio_gpu_object        *obj;
 };
 
-static int virtio_gpu_ttm_backend_bind(struct ttm_tt *ttm,
-				       struct ttm_mem_reg *bo_mem)
+static int virtio_gpu_ttm_tt_bind(struct ttm_tt *ttm,
+				  struct ttm_mem_reg *bo_mem)
 {
-	struct virtio_gpu_ttm_tt *gtt = (void *)ttm;
+	struct virtio_gpu_ttm_tt *gtt =
+		container_of(ttm, struct virtio_gpu_ttm_tt, ttm.ttm);
+	struct virtio_gpu_device *vgdev =
+		virtio_gpu_get_vgdev(gtt->obj->tbo.bdev);
 
-	gtt->offset = (unsigned long)(bo_mem->start << PAGE_SHIFT);
-	if (!ttm->num_pages)
-		WARN(1, "nothing to bind %lu pages for mreg %p back %p!\n",
-		     ttm->num_pages, bo_mem, ttm);
-
-	/* Not implemented */
+	virtio_gpu_object_attach(vgdev, gtt->obj, NULL);
 	return 0;
 }
 
-static int virtio_gpu_ttm_backend_unbind(struct ttm_tt *ttm)
+static int virtio_gpu_ttm_tt_unbind(struct ttm_tt *ttm)
 {
-	/* Not implemented */
+	struct virtio_gpu_ttm_tt *gtt =
+		container_of(ttm, struct virtio_gpu_ttm_tt, ttm.ttm);
+	struct virtio_gpu_device *vgdev =
+		virtio_gpu_get_vgdev(gtt->obj->tbo.bdev);
+
+	virtio_gpu_object_detach(vgdev, gtt->obj);
 	return 0;
 }
 
-static void virtio_gpu_ttm_backend_destroy(struct ttm_tt *ttm)
+static void virtio_gpu_ttm_tt_destroy(struct ttm_tt *ttm)
 {
-	struct virtio_gpu_ttm_tt *gtt = (void *)ttm;
+	struct virtio_gpu_ttm_tt *gtt =
+		container_of(ttm, struct virtio_gpu_ttm_tt, ttm.ttm);
 
 	ttm_dma_tt_fini(&gtt->ttm);
 	kfree(gtt);
 }
 
-static struct ttm_backend_func virtio_gpu_backend_func = {
-	.bind = &virtio_gpu_ttm_backend_bind,
-	.unbind = &virtio_gpu_ttm_backend_unbind,
-	.destroy = &virtio_gpu_ttm_backend_destroy,
+static struct ttm_backend_func virtio_gpu_tt_func = {
+	.bind = &virtio_gpu_ttm_tt_bind,
+	.unbind = &virtio_gpu_ttm_tt_unbind,
+	.destroy = &virtio_gpu_ttm_tt_destroy,
 };
 
 static struct ttm_tt *virtio_gpu_ttm_tt_create(struct ttm_buffer_object *bo,
@@ -242,8 +239,8 @@ static struct ttm_tt *virtio_gpu_ttm_tt_create(struct ttm_buffer_object *bo,
 	gtt = kzalloc(sizeof(struct virtio_gpu_ttm_tt), GFP_KERNEL);
 	if (gtt == NULL)
 		return NULL;
-	gtt->ttm.ttm.func = &virtio_gpu_backend_func;
-	gtt->vgdev = vgdev;
+	gtt->ttm.ttm.func = &virtio_gpu_tt_func;
+	gtt->obj = container_of(bo, struct virtio_gpu_object, tbo);
 	if (ttm_dma_tt_init(&gtt->ttm, bo, page_flags)) {
 		kfree(gtt);
 		return NULL;
@@ -251,58 +248,11 @@ static struct ttm_tt *virtio_gpu_ttm_tt_create(struct ttm_buffer_object *bo,
 	return &gtt->ttm.ttm;
 }
 
-static void virtio_gpu_move_null(struct ttm_buffer_object *bo,
-				 struct ttm_mem_reg *new_mem)
-{
-	struct ttm_mem_reg *old_mem = &bo->mem;
-
-	BUG_ON(old_mem->mm_node != NULL);
-	*old_mem = *new_mem;
-	new_mem->mm_node = NULL;
-}
-
-static int virtio_gpu_bo_move(struct ttm_buffer_object *bo, bool evict,
-			      struct ttm_operation_ctx *ctx,
-			      struct ttm_mem_reg *new_mem)
-{
-	int ret;
-
-	ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu);
-	if (ret)
-		return ret;
-
-	virtio_gpu_move_null(bo, new_mem);
-	return 0;
-}
-
-static void virtio_gpu_bo_move_notify(struct ttm_buffer_object *tbo,
-				      bool evict,
-				      struct ttm_mem_reg *new_mem)
-{
-	struct virtio_gpu_object *bo;
-	struct virtio_gpu_device *vgdev;
-
-	bo = container_of(tbo, struct virtio_gpu_object, tbo);
-	vgdev = (struct virtio_gpu_device *)bo->gem_base.dev->dev_private;
-
-	if (!new_mem || (new_mem->placement & TTM_PL_FLAG_SYSTEM)) {
-		if (bo->hw_res_handle)
-			virtio_gpu_object_detach(vgdev, bo);
-
-	} else if (new_mem->placement & TTM_PL_FLAG_TT) {
-		if (bo->hw_res_handle) {
-			virtio_gpu_object_attach(vgdev, bo, NULL);
-		}
-	}
-}
-
 static void virtio_gpu_bo_swap_notify(struct ttm_buffer_object *tbo)
 {
 	struct virtio_gpu_object *bo;
-	struct virtio_gpu_device *vgdev;
 
 	bo = container_of(tbo, struct virtio_gpu_object, tbo);
-	vgdev = (struct virtio_gpu_device *)bo->gem_base.dev->dev_private;
 
 	if (bo->pages)
 		virtio_gpu_object_free_sg_table(bo);
@@ -314,11 +264,9 @@ static struct ttm_bo_driver virtio_gpu_bo_driver = {
 	.init_mem_type = &virtio_gpu_init_mem_type,
 	.eviction_valuable = ttm_bo_eviction_valuable,
 	.evict_flags = &virtio_gpu_evict_flags,
-	.move = &virtio_gpu_bo_move,
 	.verify_access = &virtio_gpu_verify_access,
 	.io_mem_reserve = &virtio_gpu_ttm_io_mem_reserve,
 	.io_mem_free = &virtio_gpu_ttm_io_mem_free,
-	.move_notify = &virtio_gpu_bo_move_notify,
 	.swap_notify = &virtio_gpu_bo_swap_notify,
 };
 
@@ -330,7 +278,7 @@ int virtio_gpu_ttm_init(struct virtio_gpu_device *vgdev)
 	r = ttm_bo_device_init(&vgdev->mman.bdev,
 			       &virtio_gpu_bo_driver,
 			       vgdev->ddev->anon_inode->i_mapping,
-			       DRM_FILE_PAGE_OFFSET, 0);
+			       false);
 	if (r) {
 		DRM_ERROR("failed initializing buffer object driver(%d).\n", r);
 		goto err_dev_init;
diff --git a/drivers/gpu/drm/virtio/virtgpu_vq.c b/drivers/gpu/drm/virtio/virtgpu_vq.c
index 6bc2008b0d0d..e62fe24b1a2e 100644
--- a/drivers/gpu/drm/virtio/virtgpu_vq.c
+++ b/drivers/gpu/drm/virtio/virtgpu_vq.c
@@ -376,9 +376,8 @@ retry:
 /* create a basic resource */
 void virtio_gpu_cmd_create_resource(struct virtio_gpu_device *vgdev,
 				    struct virtio_gpu_object *bo,
-				    uint32_t format,
-				    uint32_t width,
-				    uint32_t height)
+				    struct virtio_gpu_object_params *params,
+				    struct virtio_gpu_fence *fence)
 {
 	struct virtio_gpu_resource_create_2d *cmd_p;
 	struct virtio_gpu_vbuffer *vbuf;
@@ -388,11 +387,11 @@ void virtio_gpu_cmd_create_resource(struct virtio_gpu_device *vgdev,
 
 	cmd_p->hdr.type = cpu_to_le32(VIRTIO_GPU_CMD_RESOURCE_CREATE_2D);
 	cmd_p->resource_id = cpu_to_le32(bo->hw_res_handle);
-	cmd_p->format = cpu_to_le32(format);
-	cmd_p->width = cpu_to_le32(width);
-	cmd_p->height = cpu_to_le32(height);
+	cmd_p->format = cpu_to_le32(params->format);
+	cmd_p->width = cpu_to_le32(params->width);
+	cmd_p->height = cpu_to_le32(params->height);
 
-	virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);
+	virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, &cmd_p->hdr, fence);
 	bo->created = true;
 }
 
@@ -828,7 +827,8 @@ void virtio_gpu_cmd_context_detach_resource(struct virtio_gpu_device *vgdev,
 void
 virtio_gpu_cmd_resource_create_3d(struct virtio_gpu_device *vgdev,
 				  struct virtio_gpu_object *bo,
-				  struct virtio_gpu_resource_create_3d *rc_3d)
+				  struct virtio_gpu_object_params *params,
+				  struct virtio_gpu_fence *fence)
 {
 	struct virtio_gpu_resource_create_3d *cmd_p;
 	struct virtio_gpu_vbuffer *vbuf;
@@ -836,11 +836,21 @@ virtio_gpu_cmd_resource_create_3d(struct virtio_gpu_device *vgdev,
 	cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));
 	memset(cmd_p, 0, sizeof(*cmd_p));
 
-	*cmd_p = *rc_3d;
 	cmd_p->hdr.type = cpu_to_le32(VIRTIO_GPU_CMD_RESOURCE_CREATE_3D);
-	cmd_p->hdr.flags = 0;
+	cmd_p->resource_id = cpu_to_le32(bo->hw_res_handle);
+	cmd_p->format = cpu_to_le32(params->format);
+	cmd_p->width = cpu_to_le32(params->width);
+	cmd_p->height = cpu_to_le32(params->height);
+
+	cmd_p->target = cpu_to_le32(params->target);
+	cmd_p->bind = cpu_to_le32(params->bind);
+	cmd_p->depth = cpu_to_le32(params->depth);
+	cmd_p->array_size = cpu_to_le32(params->array_size);
+	cmd_p->last_level = cpu_to_le32(params->last_level);
+	cmd_p->nr_samples = cpu_to_le32(params->nr_samples);
+	cmd_p->flags = cpu_to_le32(params->flags);
 
-	virtio_gpu_queue_ctrl_buffer(vgdev, vbuf);
+	virtio_gpu_queue_fenced_ctrl_buffer(vgdev, vbuf, &cmd_p->hdr, fence);
 	bo->created = true;
 }
 
@@ -924,8 +934,8 @@ int virtio_gpu_object_attach(struct virtio_gpu_device *vgdev,
 	struct scatterlist *sg;
 	int si, nents;
 
-	if (!obj->created)
-		return 0;
+	if (WARN_ON_ONCE(!obj->created))
+		return -EINVAL;
 
 	if (!obj->pages) {
 		int ret;
diff --git a/drivers/gpu/drm/vkms/vkms_crtc.c b/drivers/gpu/drm/vkms/vkms_crtc.c
index 8a9aeb0a9ea8..bb66dbcd5e3f 100644
--- a/drivers/gpu/drm/vkms/vkms_crtc.c
+++ b/drivers/gpu/drm/vkms/vkms_crtc.c
@@ -219,6 +219,8 @@ int vkms_crtc_init(struct drm_device *dev, struct drm_crtc *crtc,
 	spin_lock_init(&vkms_out->state_lock);
 
 	vkms_out->crc_workq = alloc_ordered_workqueue("vkms_crc_workq", 0);
+	if (!vkms_out->crc_workq)
+		return -ENOMEM;
 
 	return ret;
 }
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_binding.c b/drivers/gpu/drm/vmwgfx/vmwgfx_binding.c
index 0b9ee7fb45d6..66e14e38d5e8 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_binding.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_binding.c
@@ -499,12 +499,9 @@ static int vmw_binding_scrub_shader(struct vmw_ctx_bindinfo *bi, bool rebind)
 		SVGA3dCmdSetShader body;
 	} *cmd;
 
-	cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
-	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Failed reserving FIFO space for shader "
-			  "unbinding.\n");
+	cmd = VMW_FIFO_RESERVE(dev_priv, sizeof(*cmd));
+	if (unlikely(cmd == NULL))
 		return -ENOMEM;
-	}
 
 	cmd->header.id = SVGA_3D_CMD_SET_SHADER;
 	cmd->header.size = sizeof(cmd->body);
@@ -534,12 +531,9 @@ static int vmw_binding_scrub_render_target(struct vmw_ctx_bindinfo *bi,
 		SVGA3dCmdSetRenderTarget body;
 	} *cmd;
 
-	cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
-	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Failed reserving FIFO space for render target "
-			  "unbinding.\n");
+	cmd = VMW_FIFO_RESERVE(dev_priv, sizeof(*cmd));
+	if (unlikely(cmd == NULL))
 		return -ENOMEM;
-	}
 
 	cmd->header.id = SVGA_3D_CMD_SETRENDERTARGET;
 	cmd->header.size = sizeof(cmd->body);
@@ -576,12 +570,9 @@ static int vmw_binding_scrub_texture(struct vmw_ctx_bindinfo *bi,
 		} body;
 	} *cmd;
 
-	cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
-	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Failed reserving FIFO space for texture "
-			  "unbinding.\n");
+	cmd = VMW_FIFO_RESERVE(dev_priv, sizeof(*cmd));
+	if (unlikely(cmd == NULL))
 		return -ENOMEM;
-	}
 
 	cmd->header.id = SVGA_3D_CMD_SETTEXTURESTATE;
 	cmd->header.size = sizeof(cmd->body);
@@ -610,12 +601,10 @@ static int vmw_binding_scrub_dx_shader(struct vmw_ctx_bindinfo *bi, bool rebind)
 		SVGA3dCmdDXSetShader body;
 	} *cmd;
 
-	cmd = vmw_fifo_reserve_dx(dev_priv, sizeof(*cmd), bi->ctx->id);
-	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Failed reserving FIFO space for DX shader "
-			  "unbinding.\n");
+	cmd = VMW_FIFO_RESERVE_DX(dev_priv, sizeof(*cmd), bi->ctx->id);
+	if (unlikely(cmd == NULL))
 		return -ENOMEM;
-	}
+
 	cmd->header.id = SVGA_3D_CMD_DX_SET_SHADER;
 	cmd->header.size = sizeof(cmd->body);
 	cmd->body.type = binding->shader_slot + SVGA3D_SHADERTYPE_MIN;
@@ -641,12 +630,9 @@ static int vmw_binding_scrub_cb(struct vmw_ctx_bindinfo *bi, bool rebind)
 		SVGA3dCmdDXSetSingleConstantBuffer body;
 	} *cmd;
 
-	cmd = vmw_fifo_reserve_dx(dev_priv, sizeof(*cmd), bi->ctx->id);
-	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Failed reserving FIFO space for DX shader "
-			  "unbinding.\n");
+	cmd = VMW_FIFO_RESERVE_DX(dev_priv, sizeof(*cmd), bi->ctx->id);
+	if (unlikely(cmd == NULL))
 		return -ENOMEM;
-	}
 
 	cmd->header.id = SVGA_3D_CMD_DX_SET_SINGLE_CONSTANT_BUFFER;
 	cmd->header.size = sizeof(cmd->body);
@@ -768,12 +754,9 @@ static int vmw_emit_set_sr(struct vmw_ctx_binding_state *cbs,
 
 	view_id_size = cbs->bind_cmd_count*sizeof(uint32);
 	cmd_size = sizeof(*cmd) + view_id_size;
-	cmd = vmw_fifo_reserve_dx(ctx->dev_priv, cmd_size, ctx->id);
-	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Failed reserving FIFO space for DX shader"
-			  " resource binding.\n");
+	cmd = VMW_FIFO_RESERVE_DX(ctx->dev_priv, cmd_size, ctx->id);
+	if (unlikely(cmd == NULL))
 		return -ENOMEM;
-	}
 
 	cmd->header.id = SVGA_3D_CMD_DX_SET_SHADER_RESOURCES;
 	cmd->header.size = sizeof(cmd->body) + view_id_size;
@@ -807,12 +790,9 @@ static int vmw_emit_set_rt(struct vmw_ctx_binding_state *cbs)
 	vmw_collect_view_ids(cbs, loc, SVGA3D_MAX_SIMULTANEOUS_RENDER_TARGETS);
 	view_id_size = cbs->bind_cmd_count*sizeof(uint32);
 	cmd_size = sizeof(*cmd) + view_id_size;
-	cmd = vmw_fifo_reserve_dx(ctx->dev_priv, cmd_size, ctx->id);
-	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Failed reserving FIFO space for DX render-target"
-			  " binding.\n");
+	cmd = VMW_FIFO_RESERVE_DX(ctx->dev_priv, cmd_size, ctx->id);
+	if (unlikely(cmd == NULL))
 		return -ENOMEM;
-	}
 
 	cmd->header.id = SVGA_3D_CMD_DX_SET_RENDERTARGETS;
 	cmd->header.size = sizeof(cmd->body) + view_id_size;
@@ -894,12 +874,9 @@ static int vmw_emit_set_so(struct vmw_ctx_binding_state *cbs)
 
 	so_target_size = cbs->bind_cmd_count*sizeof(SVGA3dSoTarget);
 	cmd_size = sizeof(*cmd) + so_target_size;
-	cmd = vmw_fifo_reserve_dx(ctx->dev_priv, cmd_size, ctx->id);
-	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Failed reserving FIFO space for DX SO target"
-			  " binding.\n");
+	cmd = VMW_FIFO_RESERVE_DX(ctx->dev_priv, cmd_size, ctx->id);
+	if (unlikely(cmd == NULL))
 		return -ENOMEM;
-	}
 
 	cmd->header.id = SVGA_3D_CMD_DX_SET_SOTARGETS;
 	cmd->header.size = sizeof(cmd->body) + so_target_size;
@@ -1011,12 +988,9 @@ static int vmw_emit_set_vb(struct vmw_ctx_binding_state *cbs)
 
 	set_vb_size = cbs->bind_cmd_count*sizeof(SVGA3dVertexBuffer);
 	cmd_size = sizeof(*cmd) + set_vb_size;
-	cmd = vmw_fifo_reserve_dx(ctx->dev_priv, cmd_size, ctx->id);
-	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Failed reserving FIFO space for DX vertex buffer"
-			  " binding.\n");
+	cmd = VMW_FIFO_RESERVE_DX(ctx->dev_priv, cmd_size, ctx->id);
+	if (unlikely(cmd == NULL))
 		return -ENOMEM;
-	}
 
 	cmd->header.id = SVGA_3D_CMD_DX_SET_VERTEX_BUFFERS;
 	cmd->header.size = sizeof(cmd->body) + set_vb_size;
@@ -1167,12 +1141,10 @@ static int vmw_binding_scrub_ib(struct vmw_ctx_bindinfo *bi, bool rebind)
 		SVGA3dCmdDXSetIndexBuffer body;
 	} *cmd;
 
-	cmd = vmw_fifo_reserve_dx(dev_priv, sizeof(*cmd), bi->ctx->id);
-	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Failed reserving FIFO space for DX index buffer "
-			  "binding.\n");
+	cmd = VMW_FIFO_RESERVE_DX(dev_priv, sizeof(*cmd), bi->ctx->id);
+	if (unlikely(cmd == NULL))
 		return -ENOMEM;
-	}
+
 	cmd->header.id = SVGA_3D_CMD_DX_SET_INDEX_BUFFER;
 	cmd->header.size = sizeof(cmd->body);
 	if (rebind) {
@@ -1269,6 +1241,32 @@ void vmw_binding_state_reset(struct vmw_ctx_binding_state *cbs)
 		vmw_binding_drop(entry);
 }
 
+/**
+ * vmw_binding_dirtying - Return whether a binding type is dirtying its resource
+ * @binding_type: The binding type
+ *
+ * Each time a resource is put on the validation list as the result of a
+ * context binding referencing it, we need to determine whether that resource
+ * will be dirtied (written to by the GPU) as a result of the corresponding
+ * GPU operation. Currently rendertarget-, depth-stencil-, and
+ * stream-output-target bindings are capable of dirtying its resource.
+ *
+ * Return: Whether the binding type dirties the resource its binding points to.
+ */
+u32 vmw_binding_dirtying(enum vmw_ctx_binding_type binding_type)
+{
+	static u32 is_binding_dirtying[vmw_ctx_binding_max] = {
+		[vmw_ctx_binding_rt] = VMW_RES_DIRTY_SET,
+		[vmw_ctx_binding_dx_rt] = VMW_RES_DIRTY_SET,
+		[vmw_ctx_binding_ds] = VMW_RES_DIRTY_SET,
+		[vmw_ctx_binding_so] = VMW_RES_DIRTY_SET,
+	};
+
+	/* Review this function as new bindings are added. */
+	BUILD_BUG_ON(vmw_ctx_binding_max != 11);
+	return is_binding_dirtying[binding_type];
+}
+
 /*
  * This function is unused at run-time, and only used to hold various build
  * asserts important for code optimization assumptions.
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_binding.h b/drivers/gpu/drm/vmwgfx/vmwgfx_binding.h
index 6a2a9d69043b..f6ab79d23923 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_binding.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_binding.h
@@ -205,5 +205,7 @@ extern void vmw_binding_state_free(struct vmw_ctx_binding_state *cbs);
 extern struct list_head *
 vmw_binding_state_list(struct vmw_ctx_binding_state *cbs);
 extern void vmw_binding_state_reset(struct vmw_ctx_binding_state *cbs);
+extern u32 vmw_binding_dirtying(enum vmw_ctx_binding_type binding_type);
+
 
 #endif
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_cmdbuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_cmdbuf.c
index 70dab55e7888..56979e412ca8 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_cmdbuf.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_cmdbuf.c
@@ -393,6 +393,7 @@ static void vmw_cmdbuf_ctx_process(struct vmw_cmdbuf_man *man,
 			__vmw_cmdbuf_header_free(entry);
 			break;
 		case SVGA_CB_STATUS_COMMAND_ERROR:
+			WARN_ONCE(true, "Command buffer error.\n");
 			entry->cb_header->status = SVGA_CB_STATUS_NONE;
 			list_add_tail(&entry->list, &man->error);
 			schedule_work(&man->work);
@@ -511,17 +512,14 @@ static void vmw_cmdbuf_work_func(struct work_struct *work)
 		container_of(work, struct vmw_cmdbuf_man, work);
 	struct vmw_cmdbuf_header *entry, *next;
 	uint32_t dummy;
-	bool restart[SVGA_CB_CONTEXT_MAX];
 	bool send_fence = false;
 	struct list_head restart_head[SVGA_CB_CONTEXT_MAX];
 	int i;
 	struct vmw_cmdbuf_context *ctx;
 	bool global_block = false;
 
-	for_each_cmdbuf_ctx(man, i, ctx) {
+	for_each_cmdbuf_ctx(man, i, ctx)
 		INIT_LIST_HEAD(&restart_head[i]);
-		restart[i] = false;
-	}
 
 	mutex_lock(&man->error_mutex);
 	spin_lock(&man->lock);
@@ -533,23 +531,23 @@ static void vmw_cmdbuf_work_func(struct work_struct *work)
 		const char *cmd_name;
 
 		list_del_init(&entry->list);
-		restart[entry->cb_context] = true;
 		global_block = true;
 
 		if (!vmw_cmd_describe(header, &error_cmd_size, &cmd_name)) {
-			DRM_ERROR("Unknown command causing device error.\n");
-			DRM_ERROR("Command buffer offset is %lu\n",
-				  (unsigned long) cb_hdr->errorOffset);
+			VMW_DEBUG_USER("Unknown command causing device error.\n");
+			VMW_DEBUG_USER("Command buffer offset is %lu\n",
+				       (unsigned long) cb_hdr->errorOffset);
 			__vmw_cmdbuf_header_free(entry);
 			send_fence = true;
 			continue;
 		}
 
-		DRM_ERROR("Command \"%s\" causing device error.\n", cmd_name);
-		DRM_ERROR("Command buffer offset is %lu\n",
-			  (unsigned long) cb_hdr->errorOffset);
-		DRM_ERROR("Command size is %lu\n",
-			  (unsigned long) error_cmd_size);
+		VMW_DEBUG_USER("Command \"%s\" causing device error.\n",
+			       cmd_name);
+		VMW_DEBUG_USER("Command buffer offset is %lu\n",
+			       (unsigned long) cb_hdr->errorOffset);
+		VMW_DEBUG_USER("Command size is %lu\n",
+			       (unsigned long) error_cmd_size);
 
 		new_start_offset = cb_hdr->errorOffset + error_cmd_size;
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_context.c b/drivers/gpu/drm/vmwgfx/vmwgfx_context.c
index 14bd760a62fd..63f111068a44 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_context.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_context.c
@@ -156,12 +156,9 @@ static void vmw_hw_context_destroy(struct vmw_resource *res)
 	}
 
 	vmw_execbuf_release_pinned_bo(dev_priv);
-	cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
-	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Failed reserving FIFO space for surface "
-			  "destruction.\n");
+	cmd = VMW_FIFO_RESERVE(dev_priv, sizeof(*cmd));
+	if (unlikely(cmd == NULL))
 		return;
-	}
 
 	cmd->header.id = SVGA_3D_CMD_CONTEXT_DESTROY;
 	cmd->header.size = sizeof(cmd->body);
@@ -210,7 +207,7 @@ static int vmw_gb_context_init(struct vmw_private *dev_priv,
 		for (i = 0; i < SVGA_COTABLE_DX10_MAX; ++i) {
 			uctx->cotables[i] = vmw_cotable_alloc(dev_priv,
 							      &uctx->res, i);
-			if (unlikely(IS_ERR(uctx->cotables[i]))) {
+			if (IS_ERR(uctx->cotables[i])) {
 				ret = PTR_ERR(uctx->cotables[i]);
 				goto out_cotables;
 			}
@@ -259,9 +256,8 @@ static int vmw_context_init(struct vmw_private *dev_priv,
 		return -ENOMEM;
 	}
 
-	cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
+	cmd = VMW_FIFO_RESERVE(dev_priv, sizeof(*cmd));
 	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Fifo reserve failed.\n");
 		vmw_resource_unreference(&res);
 		return -ENOMEM;
 	}
@@ -311,10 +307,8 @@ static int vmw_gb_context_create(struct vmw_resource *res)
 		goto out_no_fifo;
 	}
 
-	cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
+	cmd = VMW_FIFO_RESERVE(dev_priv, sizeof(*cmd));
 	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Failed reserving FIFO space for context "
-			  "creation.\n");
 		ret = -ENOMEM;
 		goto out_no_fifo;
 	}
@@ -345,12 +339,10 @@ static int vmw_gb_context_bind(struct vmw_resource *res,
 
 	BUG_ON(bo->mem.mem_type != VMW_PL_MOB);
 
-	cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
-	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Failed reserving FIFO space for context "
-			  "binding.\n");
+	cmd = VMW_FIFO_RESERVE(dev_priv, sizeof(*cmd));
+	if (unlikely(cmd == NULL))
 		return -ENOMEM;
-	}
+
 	cmd->header.id = SVGA_3D_CMD_BIND_GB_CONTEXT;
 	cmd->header.size = sizeof(cmd->body);
 	cmd->body.cid = res->id;
@@ -391,10 +383,8 @@ static int vmw_gb_context_unbind(struct vmw_resource *res,
 
 	submit_size = sizeof(*cmd2) + (readback ? sizeof(*cmd1) : 0);
 
-	cmd = vmw_fifo_reserve(dev_priv, submit_size);
+	cmd = VMW_FIFO_RESERVE(dev_priv, submit_size);
 	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Failed reserving FIFO space for context "
-			  "unbinding.\n");
 		mutex_unlock(&dev_priv->binding_mutex);
 		return -ENOMEM;
 	}
@@ -441,12 +431,9 @@ static int vmw_gb_context_destroy(struct vmw_resource *res)
 	if (likely(res->id == -1))
 		return 0;
 
-	cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
-	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Failed reserving FIFO space for context "
-			  "destruction.\n");
+	cmd = VMW_FIFO_RESERVE(dev_priv, sizeof(*cmd));
+	if (unlikely(cmd == NULL))
 		return -ENOMEM;
-	}
 
 	cmd->header.id = SVGA_3D_CMD_DESTROY_GB_CONTEXT;
 	cmd->header.size = sizeof(cmd->body);
@@ -487,10 +474,8 @@ static int vmw_dx_context_create(struct vmw_resource *res)
 		goto out_no_fifo;
 	}
 
-	cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
+	cmd = VMW_FIFO_RESERVE(dev_priv, sizeof(*cmd));
 	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Failed reserving FIFO space for context "
-			  "creation.\n");
 		ret = -ENOMEM;
 		goto out_no_fifo;
 	}
@@ -521,12 +506,9 @@ static int vmw_dx_context_bind(struct vmw_resource *res,
 
 	BUG_ON(bo->mem.mem_type != VMW_PL_MOB);
 
-	cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
-	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Failed reserving FIFO space for context "
-			  "binding.\n");
+	cmd = VMW_FIFO_RESERVE(dev_priv, sizeof(*cmd));
+	if (unlikely(cmd == NULL))
 		return -ENOMEM;
-	}
 
 	cmd->header.id = SVGA_3D_CMD_DX_BIND_CONTEXT;
 	cmd->header.size = sizeof(cmd->body);
@@ -615,10 +597,8 @@ static int vmw_dx_context_unbind(struct vmw_resource *res,
 
 	submit_size = sizeof(*cmd2) + (readback ? sizeof(*cmd1) : 0);
 
-	cmd = vmw_fifo_reserve(dev_priv, submit_size);
+	cmd = VMW_FIFO_RESERVE(dev_priv, submit_size);
 	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Failed reserving FIFO space for context "
-			  "unbinding.\n");
 		mutex_unlock(&dev_priv->binding_mutex);
 		return -ENOMEM;
 	}
@@ -665,12 +645,9 @@ static int vmw_dx_context_destroy(struct vmw_resource *res)
 	if (likely(res->id == -1))
 		return 0;
 
-	cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
-	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Failed reserving FIFO space for context "
-			  "destruction.\n");
+	cmd = VMW_FIFO_RESERVE(dev_priv, sizeof(*cmd));
+	if (unlikely(cmd == NULL))
 		return -ENOMEM;
-	}
 
 	cmd->header.id = SVGA_3D_CMD_DX_DESTROY_CONTEXT;
 	cmd->header.size = sizeof(cmd->body);
@@ -751,7 +728,7 @@ static int vmw_context_define(struct drm_device *dev, void *data,
 	int ret;
 
 	if (!dev_priv->has_dx && dx) {
-		DRM_ERROR("DX contexts not supported by device.\n");
+		VMW_DEBUG_USER("DX contexts not supported by device.\n");
 		return -EINVAL;
 	}
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_cotable.c b/drivers/gpu/drm/vmwgfx/vmwgfx_cotable.c
index 44f3f6f107d3..b4f6e1217c9d 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_cotable.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_cotable.c
@@ -171,12 +171,9 @@ static int vmw_cotable_unscrub(struct vmw_resource *res)
 	WARN_ON_ONCE(bo->mem.mem_type != VMW_PL_MOB);
 	lockdep_assert_held(&bo->resv->lock.base);
 
-	cmd = vmw_fifo_reserve_dx(dev_priv, sizeof(*cmd), SVGA3D_INVALID_ID);
-	if (!cmd) {
-		DRM_ERROR("Failed reserving FIFO space for cotable "
-			  "binding.\n");
+	cmd = VMW_FIFO_RESERVE(dev_priv, sizeof(*cmd));
+	if (!cmd)
 		return -ENOMEM;
-	}
 
 	WARN_ON(vcotbl->ctx->id == SVGA3D_INVALID_ID);
 	WARN_ON(bo->mem.mem_type != VMW_PL_MOB);
@@ -262,12 +259,9 @@ int vmw_cotable_scrub(struct vmw_resource *res, bool readback)
 	if (readback)
 		submit_size += sizeof(*cmd0);
 
-	cmd1 = vmw_fifo_reserve_dx(dev_priv, submit_size, SVGA3D_INVALID_ID);
-	if (!cmd1) {
-		DRM_ERROR("Failed reserving FIFO space for cotable "
-			  "unbinding.\n");
+	cmd1 = VMW_FIFO_RESERVE(dev_priv, submit_size);
+	if (!cmd1)
 		return -ENOMEM;
-	}
 
 	vcotbl->size_read_back = 0;
 	if (readback) {
@@ -351,13 +345,10 @@ static int vmw_cotable_readback(struct vmw_resource *res)
 	struct vmw_fence_obj *fence;
 
 	if (!vcotbl->scrubbed) {
-		cmd = vmw_fifo_reserve_dx(dev_priv, sizeof(*cmd),
-					  SVGA3D_INVALID_ID);
-		if (!cmd) {
-			DRM_ERROR("Failed reserving FIFO space for cotable "
-				  "readback.\n");
+		cmd = VMW_FIFO_RESERVE(dev_priv, sizeof(*cmd));
+		if (!cmd)
 			return -ENOMEM;
-		}
+
 		cmd->header.id = SVGA_3D_CMD_DX_READBACK_COTABLE;
 		cmd->header.size = sizeof(cmd->body);
 		cmd->body.cid = vcotbl->ctx->id;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
index 1bfa353d995c..bf6c3500d363 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
@@ -828,7 +828,6 @@ static int vmw_driver_load(struct drm_device *dev, unsigned long chipset)
 	ret = ttm_bo_device_init(&dev_priv->bdev,
 				 &vmw_bo_driver,
 				 dev->anon_inode->i_mapping,
-				 VMWGFX_FILE_PAGE_OFFSET,
 				 false);
 	if (unlikely(ret != 0)) {
 		DRM_ERROR("Failed initializing TTM buffer object driver.\n");
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
index accb2fafe2f1..96983c47fb40 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
@@ -48,7 +48,6 @@
 #define VMWGFX_DRIVER_MAJOR 2
 #define VMWGFX_DRIVER_MINOR 15
 #define VMWGFX_DRIVER_PATCHLEVEL 0
-#define VMWGFX_FILE_PAGE_OFFSET 0x00100000
 #define VMWGFX_FIFO_STATIC_SIZE (1024*1024)
 #define VMWGFX_MAX_RELOCATIONS 2048
 #define VMWGFX_MAX_VALIDATIONS 2048
@@ -700,6 +699,8 @@ extern int vmw_user_stream_lookup(struct vmw_private *dev_priv,
 				  uint32_t *inout_id,
 				  struct vmw_resource **out);
 extern void vmw_resource_unreserve(struct vmw_resource *res,
+				   bool dirty_set,
+				   bool dirty,
 				   bool switch_backup,
 				   struct vmw_buffer_object *new_backup,
 				   unsigned long new_backup_offset);
@@ -812,7 +813,6 @@ extern int vmw_fifo_init(struct vmw_private *dev_priv,
 			 struct vmw_fifo_state *fifo);
 extern void vmw_fifo_release(struct vmw_private *dev_priv,
 			     struct vmw_fifo_state *fifo);
-extern void *vmw_fifo_reserve(struct vmw_private *dev_priv, uint32_t bytes);
 extern void *
 vmw_fifo_reserve_dx(struct vmw_private *dev_priv, uint32_t bytes, int ctx_id);
 extern void vmw_fifo_commit(struct vmw_private *dev_priv, uint32_t bytes);
@@ -828,6 +828,18 @@ extern int vmw_fifo_emit_dummy_query(struct vmw_private *dev_priv,
 extern int vmw_fifo_flush(struct vmw_private *dev_priv,
 			  bool interruptible);
 
+#define VMW_FIFO_RESERVE_DX(__priv, __bytes, __ctx_id)                        \
+({                                                                            \
+	vmw_fifo_reserve_dx(__priv, __bytes, __ctx_id) ? : ({                 \
+		DRM_ERROR("FIFO reserve failed at %s for %u bytes\n",         \
+			  __func__, (unsigned int) __bytes);                  \
+		NULL;                                                         \
+	});                                                                   \
+})
+
+#define VMW_FIFO_RESERVE(__priv, __bytes)                                     \
+	VMW_FIFO_RESERVE_DX(__priv, __bytes, SVGA3D_INVALID_ID)
+
 /**
  * TTM glue - vmwgfx_ttm_glue.c
  */
@@ -1312,6 +1324,20 @@ int vmw_host_get_guestinfo(const char *guest_info_param,
 			   char *buffer, size_t *length);
 int vmw_host_log(const char *log);
 
+/* VMW logging */
+
+/**
+ * VMW_DEBUG_USER - Debug output for user-space debugging.
+ *
+ * @fmt: printf() like format string.
+ *
+ * This macro is for logging user-space error and debugging messages for e.g.
+ * command buffer execution errors due to malformed commands, invalid context,
+ * etc.
+ */
+#define VMW_DEBUG_USER(fmt, ...)                                              \
+	DRM_DEBUG_DRIVER(fmt, ##__VA_ARGS__)
+
 /**
  * Inline helper functions
  */
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
index 88b8178d4687..2ff7ba04d8c8 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
@@ -36,6 +36,25 @@
 #define VMW_RES_HT_ORDER 12
 
 /*
+ * Helper macro to get dx_ctx_node if available otherwise print an error
+ * message. This is for use in command verifier function where if dx_ctx_node
+ * is not set then command is invalid.
+ */
+#define VMW_GET_CTX_NODE(__sw_context)                                        \
+({                                                                            \
+	__sw_context->dx_ctx_node ? __sw_context->dx_ctx_node : ({            \
+		VMW_DEBUG_USER("SM context is not set at %s\n", __func__);    \
+		__sw_context->dx_ctx_node;                                    \
+	});                                                                   \
+})
+
+#define VMW_DECLARE_CMD_VAR(__var, __type)                                    \
+	struct {                                                              \
+		SVGA3dCmdHeader header;                                       \
+		__type body;                                                  \
+	} __var
+
+/**
  * struct vmw_relocation - Buffer object relocation
  *
  * @head: List head for the command submission context's relocation list
@@ -59,9 +78,8 @@ struct vmw_relocation {
  * command stream is replaced with the actual id after validation.
  * @vmw_res_rel_nop: NOP relocation. The command is unconditionally replaced
  * with a NOP.
- * @vmw_res_rel_cond_nop: Conditional NOP relocation. If the resource id
- * after validation is -1, the command is replaced with a NOP. Otherwise no
- * action.
+ * @vmw_res_rel_cond_nop: Conditional NOP relocation. If the resource id after
+ * validation is -1, the command is replaced with a NOP. Otherwise no action.
  */
 enum vmw_resource_relocation_type {
 	vmw_res_rel_normal,
@@ -75,8 +93,8 @@ enum vmw_resource_relocation_type {
  *
  * @head: List head for the software context's relocation list.
  * @res: Non-ref-counted pointer to the resource.
- * @offset: Offset of single byte entries into the command buffer where the
- * id that needs fixup is located.
+ * @offset: Offset of single byte entries into the command buffer where the id
+ * that needs fixup is located.
  * @rel_type: Type of relocation.
  */
 struct vmw_resource_relocation {
@@ -86,8 +104,9 @@ struct vmw_resource_relocation {
 	enum vmw_resource_relocation_type rel_type:3;
 };
 
-/*
+/**
  * struct vmw_ctx_validation_info - Extra validation metadata for contexts
+ *
  * @head: List head of context list
  * @ctx: The context resource
  * @cur: The context's persistent binding state
@@ -142,9 +161,10 @@ static size_t vmw_ptr_diff(void *a, void *b)
 
 /**
  * vmw_execbuf_bindings_commit - Commit modified binding state
+ *
  * @sw_context: The command submission context
- * @backoff: Whether this is part of the error path and binding state
- * changes should be ignored
+ * @backoff: Whether this is part of the error path and binding state changes
+ * should be ignored
  */
 static void vmw_execbuf_bindings_commit(struct vmw_sw_context *sw_context,
 					bool backoff)
@@ -154,6 +174,7 @@ static void vmw_execbuf_bindings_commit(struct vmw_sw_context *sw_context,
 	list_for_each_entry(entry, &sw_context->ctx_list, head) {
 		if (!backoff)
 			vmw_binding_state_commit(entry->cur, entry->staged);
+
 		if (entry->staged != sw_context->staged_bindings)
 			vmw_binding_state_free(entry->staged);
 		else
@@ -166,6 +187,7 @@ static void vmw_execbuf_bindings_commit(struct vmw_sw_context *sw_context,
 
 /**
  * vmw_bind_dx_query_mob - Bind the DX query MOB if referenced
+ *
  * @sw_context: The command submission context
  */
 static void vmw_bind_dx_query_mob(struct vmw_sw_context *sw_context)
@@ -176,8 +198,8 @@ static void vmw_bind_dx_query_mob(struct vmw_sw_context *sw_context)
 }
 
 /**
- * vmw_cmd_ctx_first_setup - Perform the setup needed when a context is
- * added to the validate list.
+ * vmw_cmd_ctx_first_setup - Perform the setup needed when a context is added to
+ * the validate list.
  *
  * @dev_priv: Pointer to the device private:
  * @sw_context: The command submission context
@@ -195,11 +217,8 @@ static int vmw_cmd_ctx_first_setup(struct vmw_private *dev_priv,
 		goto out_err;
 
 	if (!sw_context->staged_bindings) {
-		sw_context->staged_bindings =
-			vmw_binding_state_alloc(dev_priv);
+		sw_context->staged_bindings = vmw_binding_state_alloc(dev_priv);
 		if (IS_ERR(sw_context->staged_bindings)) {
-			DRM_ERROR("Failed to allocate context binding "
-				  "information.\n");
 			ret = PTR_ERR(sw_context->staged_bindings);
 			sw_context->staged_bindings = NULL;
 			goto out_err;
@@ -209,8 +228,6 @@ static int vmw_cmd_ctx_first_setup(struct vmw_private *dev_priv,
 	if (sw_context->staged_bindings_inuse) {
 		node->staged = vmw_binding_state_alloc(dev_priv);
 		if (IS_ERR(node->staged)) {
-			DRM_ERROR("Failed to allocate context binding "
-				  "information.\n");
 			ret = PTR_ERR(node->staged);
 			node->staged = NULL;
 			goto out_err;
@@ -225,19 +242,20 @@ static int vmw_cmd_ctx_first_setup(struct vmw_private *dev_priv,
 	list_add_tail(&node->head, &sw_context->ctx_list);
 
 	return 0;
+
 out_err:
 	return ret;
 }
 
 /**
- * vmw_execbuf_res_size - calculate extra size fore the resource validation
- * node
+ * vmw_execbuf_res_size - calculate extra size fore the resource validation node
+ *
  * @dev_priv: Pointer to the device private struct.
  * @res_type: The resource type.
  *
- * Guest-backed contexts and DX contexts require extra size to store
- * execbuf private information in the validation node. Typically the
- * binding manager associated data structures.
+ * Guest-backed contexts and DX contexts require extra size to store execbuf
+ * private information in the validation node. Typically the binding manager
+ * associated data structures.
  *
  * Returns: The extra size requirement based on resource type.
  */
@@ -254,8 +272,8 @@ static unsigned int vmw_execbuf_res_size(struct vmw_private *dev_priv,
  *
  * @rcache: Pointer to the entry to update.
  * @res: Pointer to the resource.
- * @private: Pointer to the execbuf-private space in the resource
- * validation node.
+ * @private: Pointer to the execbuf-private space in the resource validation
+ * node.
  */
 static void vmw_execbuf_rcache_update(struct vmw_res_cache_entry *rcache,
 				      struct vmw_resource *res,
@@ -268,17 +286,19 @@ static void vmw_execbuf_rcache_update(struct vmw_res_cache_entry *rcache,
 }
 
 /**
- * vmw_execbuf_res_noref_val_add - Add a resource described by an
- * unreferenced rcu-protected pointer to the validation list.
+ * vmw_execbuf_res_noref_val_add - Add a resource described by an unreferenced
+ * rcu-protected pointer to the validation list.
+ *
  * @sw_context: Pointer to the software context.
  * @res: Unreferenced rcu-protected pointer to the resource.
+ * @dirty: Whether to change dirty status.
  *
- * Returns: 0 on success. Negative error code on failure. Typical error
- * codes are %-EINVAL on inconsistency and %-ESRCH if the resource was
- * doomed.
+ * Returns: 0 on success. Negative error code on failure. Typical error codes
+ * are %-EINVAL on inconsistency and %-ESRCH if the resource was doomed.
  */
 static int vmw_execbuf_res_noref_val_add(struct vmw_sw_context *sw_context,
-					 struct vmw_resource *res)
+					 struct vmw_resource *res,
+					 u32 dirty)
 {
 	struct vmw_private *dev_priv = res->dev_priv;
 	int ret;
@@ -290,13 +310,17 @@ static int vmw_execbuf_res_noref_val_add(struct vmw_sw_context *sw_context,
 
 	rcache = &sw_context->res_cache[res_type];
 	if (likely(rcache->valid && rcache->res == res)) {
+		if (dirty)
+			vmw_validation_res_set_dirty(sw_context->ctx,
+						     rcache->private, dirty);
 		vmw_user_resource_noref_release();
 		return 0;
 	}
 
 	priv_size = vmw_execbuf_res_size(dev_priv, res_type);
 	ret = vmw_validation_add_resource(sw_context->ctx, res, priv_size,
-					  (void **)&ctx_info, &first_usage);
+					  dirty, (void **)&ctx_info,
+					  &first_usage);
 	vmw_user_resource_noref_release();
 	if (ret)
 		return ret;
@@ -304,8 +328,10 @@ static int vmw_execbuf_res_noref_val_add(struct vmw_sw_context *sw_context,
 	if (priv_size && first_usage) {
 		ret = vmw_cmd_ctx_first_setup(dev_priv, sw_context, res,
 					      ctx_info);
-		if (ret)
+		if (ret) {
+			VMW_DEBUG_USER("Failed first usage context setup.\n");
 			return ret;
+		}
 	}
 
 	vmw_execbuf_rcache_update(rcache, res, ctx_info);
@@ -315,13 +341,16 @@ static int vmw_execbuf_res_noref_val_add(struct vmw_sw_context *sw_context,
 /**
  * vmw_execbuf_res_noctx_val_add - Add a non-context resource to the resource
  * validation list if it's not already on it
+ *
  * @sw_context: Pointer to the software context.
  * @res: Pointer to the resource.
+ * @dirty: Whether to change dirty status.
  *
  * Returns: Zero on success. Negative error code on failure.
  */
 static int vmw_execbuf_res_noctx_val_add(struct vmw_sw_context *sw_context,
-					 struct vmw_resource *res)
+					 struct vmw_resource *res,
+					 u32 dirty)
 {
 	struct vmw_res_cache_entry *rcache;
 	enum vmw_res_type res_type = vmw_res_type(res);
@@ -329,10 +358,15 @@ static int vmw_execbuf_res_noctx_val_add(struct vmw_sw_context *sw_context,
 	int ret;
 
 	rcache = &sw_context->res_cache[res_type];
-	if (likely(rcache->valid && rcache->res == res))
+	if (likely(rcache->valid && rcache->res == res)) {
+		if (dirty)
+			vmw_validation_res_set_dirty(sw_context->ctx,
+						     rcache->private, dirty);
 		return 0;
+	}
 
-	ret = vmw_validation_add_resource(sw_context->ctx, res, 0, &ptr, NULL);
+	ret = vmw_validation_add_resource(sw_context->ctx, res, 0, dirty,
+					  &ptr, NULL);
 	if (ret)
 		return ret;
 
@@ -342,8 +376,8 @@ static int vmw_execbuf_res_noctx_val_add(struct vmw_sw_context *sw_context,
 }
 
 /**
- * vmw_view_res_val_add - Add a view and the surface it's pointing to
- * to the validation list
+ * vmw_view_res_val_add - Add a view and the surface it's pointing to to the
+ * validation list
  *
  * @sw_context: The software context holding the validation list.
  * @view: Pointer to the view resource.
@@ -356,27 +390,29 @@ static int vmw_view_res_val_add(struct vmw_sw_context *sw_context,
 	int ret;
 
 	/*
-	 * First add the resource the view is pointing to, otherwise
-	 * it may be swapped out when the view is validated.
+	 * First add the resource the view is pointing to, otherwise it may be
+	 * swapped out when the view is validated.
 	 */
-	ret = vmw_execbuf_res_noctx_val_add(sw_context, vmw_view_srf(view));
+	ret = vmw_execbuf_res_noctx_val_add(sw_context, vmw_view_srf(view),
+					    vmw_view_dirtying(view));
 	if (ret)
 		return ret;
 
-	return vmw_execbuf_res_noctx_val_add(sw_context, view);
+	return vmw_execbuf_res_noctx_val_add(sw_context, view,
+					     VMW_RES_DIRTY_NONE);
 }
 
 /**
- * vmw_view_id_val_add - Look up a view and add it and the surface it's
- * pointing to to the validation list.
+ * vmw_view_id_val_add - Look up a view and add it and the surface it's pointing
+ * to to the validation list.
  *
  * @sw_context: The software context holding the validation list.
  * @view_type: The view type to look up.
  * @id: view id of the view.
  *
- * The view is represented by a view id and the DX context it's created on,
- * or scheduled for creation on. If there is no DX context set, the function
- * will return an -EINVAL error pointer.
+ * The view is represented by a view id and the DX context it's created on, or
+ * scheduled for creation on. If there is no DX context set, the function will
+ * return an -EINVAL error pointer.
  *
  * Returns: Unreferenced pointer to the resource on success, negative error
  * pointer on failure.
@@ -389,10 +425,8 @@ vmw_view_id_val_add(struct vmw_sw_context *sw_context,
 	struct vmw_resource *view;
 	int ret;
 
-	if (!ctx_node) {
-		DRM_ERROR("DX Context not set.\n");
+	if (!ctx_node)
 		return ERR_PTR(-EINVAL);
-	}
 
 	view = vmw_view_lookup(sw_context->man, view_type, id);
 	if (IS_ERR(view))
@@ -413,8 +447,8 @@ vmw_view_id_val_add(struct vmw_sw_context *sw_context,
  * @sw_context: Pointer to a software context used for this command submission
  * @ctx: Pointer to the context resource
  *
- * This function puts all resources that were previously bound to @ctx on
- * the resource validation list. This is part of the context state reemission
+ * This function puts all resources that were previously bound to @ctx on the
+ * resource validation list. This is part of the context state reemission
  */
 static int vmw_resource_context_res_add(struct vmw_private *dev_priv,
 					struct vmw_sw_context *sw_context,
@@ -433,13 +467,13 @@ static int vmw_resource_context_res_add(struct vmw_private *dev_priv,
 			if (IS_ERR(res))
 				continue;
 
-			ret = vmw_execbuf_res_noctx_val_add(sw_context, res);
+			ret = vmw_execbuf_res_noctx_val_add(sw_context, res,
+							    VMW_RES_DIRTY_SET);
 			if (unlikely(ret != 0))
 				return ret;
 		}
 	}
 
-
 	/* Add all resources bound to the context to the validation list */
 	mutex_lock(&dev_priv->binding_mutex);
 	binding_list = vmw_context_binding_list(ctx);
@@ -448,8 +482,9 @@ static int vmw_resource_context_res_add(struct vmw_private *dev_priv,
 		if (vmw_res_type(entry->res) == vmw_res_view)
 			ret = vmw_view_res_val_add(sw_context, entry->res);
 		else
-			ret = vmw_execbuf_res_noctx_val_add(sw_context,
-							    entry->res);
+			ret = vmw_execbuf_res_noctx_val_add
+				(sw_context, entry->res,
+				 vmw_binding_dirtying(entry->bt));
 		if (unlikely(ret != 0))
 			break;
 	}
@@ -472,8 +507,8 @@ static int vmw_resource_context_res_add(struct vmw_private *dev_priv,
  *
  * @list: Pointer to head of relocation list.
  * @res: The resource.
- * @offset: Offset into the command buffer currently being parsed where the
- * id that needs fixup is located. Granularity is one byte.
+ * @offset: Offset into the command buffer currently being parsed where the id
+ * that needs fixup is located. Granularity is one byte.
  * @rel_type: Relocation type.
  */
 static int vmw_resource_relocation_add(struct vmw_sw_context *sw_context,
@@ -486,7 +521,7 @@ static int vmw_resource_relocation_add(struct vmw_sw_context *sw_context,
 
 	rel = vmw_validation_mem_alloc(sw_context->ctx, sizeof(*rel));
 	if (unlikely(!rel)) {
-		DRM_ERROR("Failed to allocate a resource relocation.\n");
+		VMW_DEBUG_USER("Failed to allocate a resource relocation.\n");
 		return -ENOMEM;
 	}
 
@@ -506,17 +541,15 @@ static int vmw_resource_relocation_add(struct vmw_sw_context *sw_context,
 static void vmw_resource_relocations_free(struct list_head *list)
 {
 	/* Memory is validation context memory, so no need to free it */
-
 	INIT_LIST_HEAD(list);
 }
 
 /**
  * vmw_resource_relocations_apply - Apply all relocations on a list
  *
- * @cb: Pointer to the start of the command buffer bein patch. This need
- * not be the same buffer as the one being parsed when the relocation
- * list was built, but the contents must be the same modulo the
- * resource ids.
+ * @cb: Pointer to the start of the command buffer bein patch. This need not be
+ * the same buffer as the one being parsed when the relocation list was built,
+ * but the contents must be the same modulo the resource ids.
  * @list: Pointer to the head of the relocation list.
  */
 static void vmw_resource_relocations_apply(uint32_t *cb,
@@ -560,14 +593,14 @@ static int vmw_cmd_ok(struct vmw_private *dev_priv,
 }
 
 /**
- * vmw_resources_reserve - Reserve all resources on the sw_context's
- * resource list.
+ * vmw_resources_reserve - Reserve all resources on the sw_context's resource
+ * list.
  *
  * @sw_context: Pointer to the software context.
  *
- * Note that since vmware's command submission currently is protected by
- * the cmdbuf mutex, no fancy deadlock avoidance is required for resources,
- * since only a single thread at once will attempt this.
+ * Note that since vmware's command submission currently is protected by the
+ * cmdbuf mutex, no fancy deadlock avoidance is required for resources, since
+ * only a single thread at once will attempt this.
  */
 static int vmw_resources_reserve(struct vmw_sw_context *sw_context)
 {
@@ -592,22 +625,24 @@ static int vmw_resources_reserve(struct vmw_sw_context *sw_context)
 }
 
 /**
- * vmw_cmd_res_check - Check that a resource is present and if so, put it
- * on the resource validate list unless it's already there.
+ * vmw_cmd_res_check - Check that a resource is present and if so, put it on the
+ * resource validate list unless it's already there.
  *
  * @dev_priv: Pointer to a device private structure.
  * @sw_context: Pointer to the software context.
  * @res_type: Resource type.
+ * @dirty: Whether to change dirty status.
  * @converter: User-space visisble type specific information.
- * @id_loc: Pointer to the location in the command buffer currently being
- * parsed from where the user-space resource id handle is located.
- * @p_val: Pointer to pointer to resource validalidation node. Populated
- * on exit.
+ * @id_loc: Pointer to the location in the command buffer currently being parsed
+ * from where the user-space resource id handle is located.
+ * @p_val: Pointer to pointer to resource validalidation node. Populated on
+ * exit.
  */
 static int
 vmw_cmd_res_check(struct vmw_private *dev_priv,
 		  struct vmw_sw_context *sw_context,
 		  enum vmw_res_type res_type,
+		  u32 dirty,
 		  const struct vmw_user_resource_conv *converter,
 		  uint32_t *id_loc,
 		  struct vmw_resource **p_res)
@@ -621,7 +656,7 @@ vmw_cmd_res_check(struct vmw_private *dev_priv,
 
 	if (*id_loc == SVGA3D_INVALID_ID) {
 		if (res_type == vmw_res_context) {
-			DRM_ERROR("Illegal context invalid id.\n");
+			VMW_DEBUG_USER("Illegal context invalid id.\n");
 			return -EINVAL;
 		}
 		return 0;
@@ -629,6 +664,9 @@ vmw_cmd_res_check(struct vmw_private *dev_priv,
 
 	if (likely(rcache->valid_handle && *id_loc == rcache->handle)) {
 		res = rcache->res;
+		if (dirty)
+			vmw_validation_res_set_dirty(sw_context->ctx,
+						     rcache->private, dirty);
 	} else {
 		unsigned int size = vmw_execbuf_res_size(dev_priv, res_type);
 
@@ -638,13 +676,13 @@ vmw_cmd_res_check(struct vmw_private *dev_priv,
 
 		res = vmw_user_resource_noref_lookup_handle
 			(dev_priv, sw_context->fp->tfile, *id_loc, converter);
-		if (unlikely(IS_ERR(res))) {
-			DRM_ERROR("Could not find or use resource 0x%08x.\n",
-				  (unsigned int) *id_loc);
+		if (IS_ERR(res)) {
+			VMW_DEBUG_USER("Could not find/use resource 0x%08x.\n",
+				       (unsigned int) *id_loc);
 			return PTR_ERR(res);
 		}
 
-		ret = vmw_execbuf_res_noref_val_add(sw_context, res);
+		ret = vmw_execbuf_res_noref_val_add(sw_context, res, dirty);
 		if (unlikely(ret != 0))
 			return ret;
 
@@ -675,23 +713,16 @@ static int vmw_rebind_all_dx_query(struct vmw_resource *ctx_res)
 {
 	struct vmw_private *dev_priv = ctx_res->dev_priv;
 	struct vmw_buffer_object *dx_query_mob;
-	struct {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdDXBindAllQuery body;
-	} *cmd;
-
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdDXBindAllQuery);
 
 	dx_query_mob = vmw_context_get_dx_query_mob(ctx_res);
 
 	if (!dx_query_mob || dx_query_mob->dx_query_ctx)
 		return 0;
 
-	cmd = vmw_fifo_reserve_dx(dev_priv, sizeof(*cmd), ctx_res->id);
-
-	if (cmd == NULL) {
-		DRM_ERROR("Failed to rebind queries.\n");
+	cmd = VMW_FIFO_RESERVE_DX(dev_priv, sizeof(*cmd), ctx_res->id);
+	if (cmd == NULL)
 		return -ENOMEM;
-	}
 
 	cmd->header.id = SVGA_3D_CMD_DX_BIND_ALL_QUERY;
 	cmd->header.size = sizeof(cmd->body);
@@ -705,8 +736,8 @@ static int vmw_rebind_all_dx_query(struct vmw_resource *ctx_res)
 }
 
 /**
- * vmw_rebind_contexts - Rebind all resources previously bound to
- * referenced contexts.
+ * vmw_rebind_contexts - Rebind all resources previously bound to referenced
+ * contexts.
  *
  * @sw_context: Pointer to the software context.
  *
@@ -721,21 +752,23 @@ static int vmw_rebind_contexts(struct vmw_sw_context *sw_context)
 		ret = vmw_binding_rebind_all(val->cur);
 		if (unlikely(ret != 0)) {
 			if (ret != -ERESTARTSYS)
-				DRM_ERROR("Failed to rebind context.\n");
+				VMW_DEBUG_USER("Failed to rebind context.\n");
 			return ret;
 		}
 
 		ret = vmw_rebind_all_dx_query(val->ctx);
-		if (ret != 0)
+		if (ret != 0) {
+			VMW_DEBUG_USER("Failed to rebind queries.\n");
 			return ret;
+		}
 	}
 
 	return 0;
 }
 
 /**
- * vmw_view_bindings_add - Add an array of view bindings to a context
- * binding state tracker.
+ * vmw_view_bindings_add - Add an array of view bindings to a context binding
+ * state tracker.
  *
  * @sw_context: The execbuf state used for this command.
  * @view_type: View type for the bindings.
@@ -752,13 +785,11 @@ static int vmw_view_bindings_add(struct vmw_sw_context *sw_context,
 				 uint32 view_ids[], u32 num_views,
 				 u32 first_slot)
 {
-	struct vmw_ctx_validation_info *ctx_node = sw_context->dx_ctx_node;
+	struct vmw_ctx_validation_info *ctx_node = VMW_GET_CTX_NODE(sw_context);
 	u32 i;
 
-	if (!ctx_node) {
-		DRM_ERROR("DX Context not set.\n");
+	if (!ctx_node)
 		return -EINVAL;
-	}
 
 	for (i = 0; i < num_views; ++i) {
 		struct vmw_ctx_bindinfo_view binding;
@@ -768,7 +799,7 @@ static int vmw_view_bindings_add(struct vmw_sw_context *sw_context,
 			view = vmw_view_id_val_add(sw_context, view_type,
 						   view_ids[i]);
 			if (IS_ERR(view)) {
-				DRM_ERROR("View not found.\n");
+				VMW_DEBUG_USER("View not found.\n");
 				return PTR_ERR(view);
 			}
 		}
@@ -798,19 +829,18 @@ static int vmw_cmd_cid_check(struct vmw_private *dev_priv,
 			     struct vmw_sw_context *sw_context,
 			     SVGA3dCmdHeader *header)
 {
-	struct vmw_cid_cmd {
-		SVGA3dCmdHeader header;
-		uint32_t cid;
-	} *cmd;
+	VMW_DECLARE_CMD_VAR(*cmd, uint32_t) =
+		container_of(header, typeof(*cmd), header);
 
-	cmd = container_of(header, struct vmw_cid_cmd, header);
 	return vmw_cmd_res_check(dev_priv, sw_context, vmw_res_context,
-				 user_context_converter, &cmd->cid, NULL);
+				 VMW_RES_DIRTY_SET, user_context_converter,
+				 &cmd->body, NULL);
 }
 
 /**
  * vmw_execbuf_info_from_res - Get the private validation metadata for a
  * recently validated resource
+ *
  * @sw_context: Pointer to the command submission context
  * @res: The resource
  *
@@ -818,8 +848,8 @@ static int vmw_cmd_cid_check(struct vmw_private *dev_priv,
  * context's resource cache and hence the last resource of that type to be
  * processed by the validation code.
  *
- * Return: a pointer to the private metadata of the resource, or NULL
- * if it wasn't found
+ * Return: a pointer to the private metadata of the resource, or NULL if it
+ * wasn't found
  */
 static struct vmw_ctx_validation_info *
 vmw_execbuf_info_from_res(struct vmw_sw_context *sw_context,
@@ -835,36 +865,32 @@ vmw_execbuf_info_from_res(struct vmw_sw_context *sw_context,
 	return NULL;
 }
 
-
 static int vmw_cmd_set_render_target_check(struct vmw_private *dev_priv,
 					   struct vmw_sw_context *sw_context,
 					   SVGA3dCmdHeader *header)
 {
-	struct vmw_sid_cmd {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdSetRenderTarget body;
-	} *cmd;
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdSetRenderTarget);
 	struct vmw_resource *ctx;
 	struct vmw_resource *res;
 	int ret;
 
-	cmd = container_of(header, struct vmw_sid_cmd, header);
+	cmd = container_of(header, typeof(*cmd), header);
 
 	if (cmd->body.type >= SVGA3D_RT_MAX) {
-		DRM_ERROR("Illegal render target type %u.\n",
-			  (unsigned) cmd->body.type);
+		VMW_DEBUG_USER("Illegal render target type %u.\n",
+			       (unsigned int) cmd->body.type);
 		return -EINVAL;
 	}
 
 	ret = vmw_cmd_res_check(dev_priv, sw_context, vmw_res_context,
-				user_context_converter, &cmd->body.cid,
-				&ctx);
+				VMW_RES_DIRTY_SET, user_context_converter,
+				&cmd->body.cid, &ctx);
 	if (unlikely(ret != 0))
 		return ret;
 
 	ret = vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface,
-				user_surface_converter, &cmd->body.target.sid,
-				&res);
+				VMW_RES_DIRTY_SET, user_surface_converter,
+				&cmd->body.target.sid, &res);
 	if (unlikely(ret))
 		return ret;
 
@@ -890,44 +916,38 @@ static int vmw_cmd_surface_copy_check(struct vmw_private *dev_priv,
 				      struct vmw_sw_context *sw_context,
 				      SVGA3dCmdHeader *header)
 {
-	struct vmw_sid_cmd {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdSurfaceCopy body;
-	} *cmd;
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdSurfaceCopy);
 	int ret;
 
-	cmd = container_of(header, struct vmw_sid_cmd, header);
+	cmd = container_of(header, typeof(*cmd), header);
 
 	ret = vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface,
-					  user_surface_converter,
-					  &cmd->body.src.sid, NULL);
+				VMW_RES_DIRTY_NONE, user_surface_converter,
+				&cmd->body.src.sid, NULL);
 	if (ret)
 		return ret;
 
 	return vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface,
-				 user_surface_converter,
+				 VMW_RES_DIRTY_SET, user_surface_converter,
 				 &cmd->body.dest.sid, NULL);
 }
 
 static int vmw_cmd_buffer_copy_check(struct vmw_private *dev_priv,
-				      struct vmw_sw_context *sw_context,
-				      SVGA3dCmdHeader *header)
+				     struct vmw_sw_context *sw_context,
+				     SVGA3dCmdHeader *header)
 {
-	struct {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdDXBufferCopy body;
-	} *cmd;
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdDXBufferCopy);
 	int ret;
 
 	cmd = container_of(header, typeof(*cmd), header);
 	ret = vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface,
-				user_surface_converter,
+				VMW_RES_DIRTY_NONE, user_surface_converter,
 				&cmd->body.src, NULL);
 	if (ret != 0)
 		return ret;
 
 	return vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface,
-				 user_surface_converter,
+				 VMW_RES_DIRTY_SET, user_surface_converter,
 				 &cmd->body.dest, NULL);
 }
 
@@ -935,21 +955,18 @@ static int vmw_cmd_pred_copy_check(struct vmw_private *dev_priv,
 				   struct vmw_sw_context *sw_context,
 				   SVGA3dCmdHeader *header)
 {
-	struct {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdDXPredCopyRegion body;
-	} *cmd;
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdDXPredCopyRegion);
 	int ret;
 
 	cmd = container_of(header, typeof(*cmd), header);
 	ret = vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface,
-				user_surface_converter,
+				VMW_RES_DIRTY_NONE, user_surface_converter,
 				&cmd->body.srcSid, NULL);
 	if (ret != 0)
 		return ret;
 
 	return vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface,
-				 user_surface_converter,
+				 VMW_RES_DIRTY_SET, user_surface_converter,
 				 &cmd->body.dstSid, NULL);
 }
 
@@ -957,20 +974,18 @@ static int vmw_cmd_stretch_blt_check(struct vmw_private *dev_priv,
 				     struct vmw_sw_context *sw_context,
 				     SVGA3dCmdHeader *header)
 {
-	struct vmw_sid_cmd {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdSurfaceStretchBlt body;
-	} *cmd;
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdSurfaceStretchBlt);
 	int ret;
 
-	cmd = container_of(header, struct vmw_sid_cmd, header);
+	cmd = container_of(header, typeof(*cmd), header);
 	ret = vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface,
-				user_surface_converter,
+				VMW_RES_DIRTY_NONE, user_surface_converter,
 				&cmd->body.src.sid, NULL);
 	if (unlikely(ret != 0))
 		return ret;
+
 	return vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface,
-				 user_surface_converter,
+				 VMW_RES_DIRTY_SET, user_surface_converter,
 				 &cmd->body.dest.sid, NULL);
 }
 
@@ -978,15 +993,11 @@ static int vmw_cmd_blt_surf_screen_check(struct vmw_private *dev_priv,
 					 struct vmw_sw_context *sw_context,
 					 SVGA3dCmdHeader *header)
 {
-	struct vmw_sid_cmd {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdBlitSurfaceToScreen body;
-	} *cmd;
-
-	cmd = container_of(header, struct vmw_sid_cmd, header);
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdBlitSurfaceToScreen) =
+		container_of(header, typeof(*cmd), header);
 
 	return vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface,
-				 user_surface_converter,
+				 VMW_RES_DIRTY_NONE, user_surface_converter,
 				 &cmd->body.srcImage.sid, NULL);
 }
 
@@ -994,17 +1005,12 @@ static int vmw_cmd_present_check(struct vmw_private *dev_priv,
 				 struct vmw_sw_context *sw_context,
 				 SVGA3dCmdHeader *header)
 {
-	struct vmw_sid_cmd {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdPresent body;
-	} *cmd;
-
-
-	cmd = container_of(header, struct vmw_sid_cmd, header);
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdPresent) =
+		container_of(header, typeof(*cmd), header);
 
 	return vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface,
-				 user_surface_converter, &cmd->body.sid,
-				 NULL);
+				 VMW_RES_DIRTY_NONE, user_surface_converter,
+				 &cmd->body.sid, NULL);
 }
 
 /**
@@ -1014,11 +1020,10 @@ static int vmw_cmd_present_check(struct vmw_private *dev_priv,
  * @new_query_bo: The new buffer holding query results.
  * @sw_context: The software context used for this command submission.
  *
- * This function checks whether @new_query_bo is suitable for holding
- * query results, and if another buffer currently is pinned for query
- * results. If so, the function prepares the state of @sw_context for
- * switching pinned buffers after successful submission of the current
- * command batch.
+ * This function checks whether @new_query_bo is suitable for holding query
+ * results, and if another buffer currently is pinned for query results. If so,
+ * the function prepares the state of @sw_context for switching pinned buffers
+ * after successful submission of the current command batch.
  */
 static int vmw_query_bo_switch_prepare(struct vmw_private *dev_priv,
 				       struct vmw_buffer_object *new_query_bo,
@@ -1034,7 +1039,7 @@ static int vmw_query_bo_switch_prepare(struct vmw_private *dev_priv,
 	if (unlikely(new_query_bo != sw_context->cur_query_bo)) {
 
 		if (unlikely(new_query_bo->base.num_pages > 4)) {
-			DRM_ERROR("Query buffer too large.\n");
+			VMW_DEBUG_USER("Query buffer too large.\n");
 			return -EINVAL;
 		}
 
@@ -1053,13 +1058,11 @@ static int vmw_query_bo_switch_prepare(struct vmw_private *dev_priv,
 					    dev_priv->has_mob, false);
 		if (unlikely(ret != 0))
 			return ret;
-
 	}
 
 	return 0;
 }
 
-
 /**
  * vmw_query_bo_switch_commit - Finalize switching pinned query buffer
  *
@@ -1068,11 +1071,11 @@ static int vmw_query_bo_switch_prepare(struct vmw_private *dev_priv,
  *
  * This function will check if we're switching query buffers, and will then,
  * issue a dummy occlusion query wait used as a query barrier. When the fence
- * object following that query wait has signaled, we are sure that all
- * preceding queries have finished, and the old query buffer can be unpinned.
- * However, since both the new query buffer and the old one are fenced with
- * that fence, we can do an asynchronus unpin now, and be sure that the
- * old query buffer won't be moved until the fence has signaled.
+ * object following that query wait has signaled, we are sure that all preceding
+ * queries have finished, and the old query buffer can be unpinned. However,
+ * since both the new query buffer and the old one are fenced with that fence,
+ * we can do an asynchronus unpin now, and be sure that the old query buffer
+ * won't be moved until the fence has signaled.
  *
  * As mentioned above, both the new - and old query buffers need to be fenced
  * using a sequence emitted *after* calling this function.
@@ -1084,7 +1087,6 @@ static void vmw_query_bo_switch_commit(struct vmw_private *dev_priv,
 	 * The validate list should still hold references to all
 	 * contexts here.
 	 */
-
 	if (sw_context->needs_post_query_barrier) {
 		struct vmw_res_cache_entry *ctx_entry =
 			&sw_context->res_cache[vmw_res_context];
@@ -1097,7 +1099,7 @@ static void vmw_query_bo_switch_commit(struct vmw_private *dev_priv,
 		ret = vmw_fifo_emit_dummy_query(dev_priv, ctx->id);
 
 		if (unlikely(ret != 0))
-			DRM_ERROR("Out of fifo space for dummy query.\n");
+			VMW_DEBUG_USER("Out of fifo space for dummy query.\n");
 	}
 
 	if (dev_priv->pinned_bo != sw_context->cur_query_bo) {
@@ -1111,10 +1113,9 @@ static void vmw_query_bo_switch_commit(struct vmw_private *dev_priv,
 
 			/*
 			 * We pin also the dummy_query_bo buffer so that we
-			 * don't need to validate it when emitting
-			 * dummy queries in context destroy paths.
+			 * don't need to validate it when emitting dummy queries
+			 * in context destroy paths.
 			 */
-
 			if (!dev_priv->dummy_query_bo_pinned) {
 				vmw_bo_pin_reserved(dev_priv->dummy_query_bo,
 						    true);
@@ -1131,22 +1132,24 @@ static void vmw_query_bo_switch_commit(struct vmw_private *dev_priv,
 }
 
 /**
- * vmw_translate_mob_pointer - Prepare to translate a user-space buffer
- * handle to a MOB id.
+ * vmw_translate_mob_pointer - Prepare to translate a user-space buffer handle
+ * to a MOB id.
  *
  * @dev_priv: Pointer to a device private structure.
  * @sw_context: The software context used for this command batch validation.
  * @id: Pointer to the user-space handle to be translated.
- * @vmw_bo_p: Points to a location that, on successful return will carry
- * a non-reference-counted pointer to the buffer object identified by the
+ * @vmw_bo_p: Points to a location that, on successful return will carry a
+ * non-reference-counted pointer to the buffer object identified by the
  * user-space handle in @id.
  *
  * This function saves information needed to translate a user-space buffer
  * handle to a MOB id. The translation does not take place immediately, but
- * during a call to vmw_apply_relocations(). This function builds a relocation
- * list and a list of buffers to validate. The former needs to be freed using
- * either vmw_apply_relocations() or vmw_free_relocations(). The latter
- * needs to be freed using vmw_clear_validations.
+ * during a call to vmw_apply_relocations().
+ *
+ * This function builds a relocation list and a list of buffers to validate. The
+ * former needs to be freed using either vmw_apply_relocations() or
+ * vmw_free_relocations(). The latter needs to be freed using
+ * vmw_clear_validations.
  */
 static int vmw_translate_mob_ptr(struct vmw_private *dev_priv,
 				 struct vmw_sw_context *sw_context,
@@ -1161,7 +1164,7 @@ static int vmw_translate_mob_ptr(struct vmw_private *dev_priv,
 	vmw_validation_preload_bo(sw_context->ctx);
 	vmw_bo = vmw_user_bo_noref_lookup(sw_context->fp->tfile, handle);
 	if (IS_ERR(vmw_bo)) {
-		DRM_ERROR("Could not find or use MOB buffer.\n");
+		VMW_DEBUG_USER("Could not find or use MOB buffer.\n");
 		return PTR_ERR(vmw_bo);
 	}
 
@@ -1184,19 +1187,20 @@ static int vmw_translate_mob_ptr(struct vmw_private *dev_priv,
 }
 
 /**
- * vmw_translate_guest_pointer - Prepare to translate a user-space buffer
- * handle to a valid SVGAGuestPtr
+ * vmw_translate_guest_pointer - Prepare to translate a user-space buffer handle
+ * to a valid SVGAGuestPtr
  *
  * @dev_priv: Pointer to a device private structure.
  * @sw_context: The software context used for this command batch validation.
  * @ptr: Pointer to the user-space handle to be translated.
- * @vmw_bo_p: Points to a location that, on successful return will carry
- * a non-reference-counted pointer to the DMA buffer identified by the
- * user-space handle in @id.
+ * @vmw_bo_p: Points to a location that, on successful return will carry a
+ * non-reference-counted pointer to the DMA buffer identified by the user-space
+ * handle in @id.
  *
  * This function saves information needed to translate a user-space buffer
  * handle to a valid SVGAGuestPtr. The translation does not take place
  * immediately, but during a call to vmw_apply_relocations().
+ *
  * This function builds a relocation list and a list of buffers to validate.
  * The former needs to be freed using either vmw_apply_relocations() or
  * vmw_free_relocations(). The latter needs to be freed using
@@ -1215,7 +1219,7 @@ static int vmw_translate_guest_ptr(struct vmw_private *dev_priv,
 	vmw_validation_preload_bo(sw_context->ctx);
 	vmw_bo = vmw_user_bo_noref_lookup(sw_context->fp->tfile, handle);
 	if (IS_ERR(vmw_bo)) {
-		DRM_ERROR("Could not find or use GMR region.\n");
+		VMW_DEBUG_USER("Could not find or use GMR region.\n");
 		return PTR_ERR(vmw_bo);
 	}
 
@@ -1236,10 +1240,8 @@ static int vmw_translate_guest_ptr(struct vmw_private *dev_priv,
 	return 0;
 }
 
-
-
 /**
- * vmw_cmd_dx_define_query - validate a SVGA_3D_CMD_DX_DEFINE_QUERY command.
+ * vmw_cmd_dx_define_query - validate SVGA_3D_CMD_DX_DEFINE_QUERY command.
  *
  * @dev_priv: Pointer to a device private struct.
  * @sw_context: The software context used for this command submission.
@@ -1251,67 +1253,52 @@ static int vmw_cmd_dx_define_query(struct vmw_private *dev_priv,
 				   struct vmw_sw_context *sw_context,
 				   SVGA3dCmdHeader *header)
 {
-	struct vmw_dx_define_query_cmd {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdDXDefineQuery q;
-	} *cmd;
-
-	int    ret;
-	struct vmw_ctx_validation_info *ctx_node = sw_context->dx_ctx_node;
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdDXDefineQuery);
+	struct vmw_ctx_validation_info *ctx_node = VMW_GET_CTX_NODE(sw_context);
 	struct vmw_resource *cotable_res;
+	int ret;
 
-
-	if (ctx_node == NULL) {
-		DRM_ERROR("DX Context not set for query.\n");
+	if (!ctx_node)
 		return -EINVAL;
-	}
 
-	cmd = container_of(header, struct vmw_dx_define_query_cmd, header);
+	cmd = container_of(header, typeof(*cmd), header);
 
-	if (cmd->q.type <  SVGA3D_QUERYTYPE_MIN ||
-	    cmd->q.type >= SVGA3D_QUERYTYPE_MAX)
+	if (cmd->body.type <  SVGA3D_QUERYTYPE_MIN ||
+	    cmd->body.type >= SVGA3D_QUERYTYPE_MAX)
 		return -EINVAL;
 
 	cotable_res = vmw_context_cotable(ctx_node->ctx, SVGA_COTABLE_DXQUERY);
-	ret = vmw_cotable_notify(cotable_res, cmd->q.queryId);
+	ret = vmw_cotable_notify(cotable_res, cmd->body.queryId);
 
 	return ret;
 }
 
-
-
 /**
- * vmw_cmd_dx_bind_query - validate a SVGA_3D_CMD_DX_BIND_QUERY command.
+ * vmw_cmd_dx_bind_query - validate SVGA_3D_CMD_DX_BIND_QUERY command.
  *
  * @dev_priv: Pointer to a device private struct.
  * @sw_context: The software context used for this command submission.
  * @header: Pointer to the command header in the command stream.
  *
- * The query bind operation will eventually associate the query ID
- * with its backing MOB.  In this function, we take the user mode
- * MOB ID and use vmw_translate_mob_ptr() to translate it to its
- * kernel mode equivalent.
+ * The query bind operation will eventually associate the query ID with its
+ * backing MOB.  In this function, we take the user mode MOB ID and use
+ * vmw_translate_mob_ptr() to translate it to its kernel mode equivalent.
  */
 static int vmw_cmd_dx_bind_query(struct vmw_private *dev_priv,
 				 struct vmw_sw_context *sw_context,
 				 SVGA3dCmdHeader *header)
 {
-	struct vmw_dx_bind_query_cmd {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdDXBindQuery q;
-	} *cmd;
-
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdDXBindQuery);
 	struct vmw_buffer_object *vmw_bo;
-	int    ret;
-
+	int ret;
 
-	cmd = container_of(header, struct vmw_dx_bind_query_cmd, header);
+	cmd = container_of(header, typeof(*cmd), header);
 
 	/*
 	 * Look up the buffer pointed to by q.mobid, put it on the relocation
 	 * list so its kernel mode MOB ID can be filled in later
 	 */
-	ret = vmw_translate_mob_ptr(dev_priv, sw_context, &cmd->q.mobid,
+	ret = vmw_translate_mob_ptr(dev_priv, sw_context, &cmd->body.mobid,
 				    &vmw_bo);
 
 	if (ret != 0)
@@ -1322,10 +1309,8 @@ static int vmw_cmd_dx_bind_query(struct vmw_private *dev_priv,
 	return 0;
 }
 
-
-
 /**
- * vmw_cmd_begin_gb_query - validate a  SVGA_3D_CMD_BEGIN_GB_QUERY command.
+ * vmw_cmd_begin_gb_query - validate SVGA_3D_CMD_BEGIN_GB_QUERY command.
  *
  * @dev_priv: Pointer to a device private struct.
  * @sw_context: The software context used for this command submission.
@@ -1335,21 +1320,16 @@ static int vmw_cmd_begin_gb_query(struct vmw_private *dev_priv,
 				  struct vmw_sw_context *sw_context,
 				  SVGA3dCmdHeader *header)
 {
-	struct vmw_begin_gb_query_cmd {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdBeginGBQuery q;
-	} *cmd;
-
-	cmd = container_of(header, struct vmw_begin_gb_query_cmd,
-			   header);
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdBeginGBQuery) =
+		container_of(header, typeof(*cmd), header);
 
 	return vmw_cmd_res_check(dev_priv, sw_context, vmw_res_context,
-				 user_context_converter, &cmd->q.cid,
-				 NULL);
+				 VMW_RES_DIRTY_SET, user_context_converter,
+				 &cmd->body.cid, NULL);
 }
 
 /**
- * vmw_cmd_begin_query - validate a  SVGA_3D_CMD_BEGIN_QUERY command.
+ * vmw_cmd_begin_query - validate SVGA_3D_CMD_BEGIN_QUERY command.
  *
  * @dev_priv: Pointer to a device private struct.
  * @sw_context: The software context used for this command submission.
@@ -1359,38 +1339,30 @@ static int vmw_cmd_begin_query(struct vmw_private *dev_priv,
 			       struct vmw_sw_context *sw_context,
 			       SVGA3dCmdHeader *header)
 {
-	struct vmw_begin_query_cmd {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdBeginQuery q;
-	} *cmd;
-
-	cmd = container_of(header, struct vmw_begin_query_cmd,
-			   header);
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdBeginQuery) =
+		container_of(header, typeof(*cmd), header);
 
 	if (unlikely(dev_priv->has_mob)) {
-		struct {
-			SVGA3dCmdHeader header;
-			SVGA3dCmdBeginGBQuery q;
-		} gb_cmd;
+		VMW_DECLARE_CMD_VAR(gb_cmd, SVGA3dCmdBeginGBQuery);
 
 		BUG_ON(sizeof(gb_cmd) != sizeof(*cmd));
 
 		gb_cmd.header.id = SVGA_3D_CMD_BEGIN_GB_QUERY;
 		gb_cmd.header.size = cmd->header.size;
-		gb_cmd.q.cid = cmd->q.cid;
-		gb_cmd.q.type = cmd->q.type;
+		gb_cmd.body.cid = cmd->body.cid;
+		gb_cmd.body.type = cmd->body.type;
 
 		memcpy(cmd, &gb_cmd, sizeof(*cmd));
 		return vmw_cmd_begin_gb_query(dev_priv, sw_context, header);
 	}
 
 	return vmw_cmd_res_check(dev_priv, sw_context, vmw_res_context,
-				 user_context_converter, &cmd->q.cid,
-				 NULL);
+				 VMW_RES_DIRTY_SET, user_context_converter,
+				 &cmd->body.cid, NULL);
 }
 
 /**
- * vmw_cmd_end_gb_query - validate a  SVGA_3D_CMD_END_GB_QUERY command.
+ * vmw_cmd_end_gb_query - validate SVGA_3D_CMD_END_GB_QUERY command.
  *
  * @dev_priv: Pointer to a device private struct.
  * @sw_context: The software context used for this command submission.
@@ -1401,19 +1373,15 @@ static int vmw_cmd_end_gb_query(struct vmw_private *dev_priv,
 				SVGA3dCmdHeader *header)
 {
 	struct vmw_buffer_object *vmw_bo;
-	struct vmw_query_cmd {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdEndGBQuery q;
-	} *cmd;
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdEndGBQuery);
 	int ret;
 
-	cmd = container_of(header, struct vmw_query_cmd, header);
+	cmd = container_of(header, typeof(*cmd), header);
 	ret = vmw_cmd_cid_check(dev_priv, sw_context, header);
 	if (unlikely(ret != 0))
 		return ret;
 
-	ret = vmw_translate_mob_ptr(dev_priv, sw_context,
-				    &cmd->q.mobid,
+	ret = vmw_translate_mob_ptr(dev_priv, sw_context, &cmd->body.mobid,
 				    &vmw_bo);
 	if (unlikely(ret != 0))
 		return ret;
@@ -1424,7 +1392,7 @@ static int vmw_cmd_end_gb_query(struct vmw_private *dev_priv,
 }
 
 /**
- * vmw_cmd_end_query - validate a  SVGA_3D_CMD_END_QUERY command.
+ * vmw_cmd_end_query - validate SVGA_3D_CMD_END_QUERY command.
  *
  * @dev_priv: Pointer to a device private struct.
  * @sw_context: The software context used for this command submission.
@@ -1435,27 +1403,21 @@ static int vmw_cmd_end_query(struct vmw_private *dev_priv,
 			     SVGA3dCmdHeader *header)
 {
 	struct vmw_buffer_object *vmw_bo;
-	struct vmw_query_cmd {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdEndQuery q;
-	} *cmd;
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdEndQuery);
 	int ret;
 
-	cmd = container_of(header, struct vmw_query_cmd, header);
+	cmd = container_of(header, typeof(*cmd), header);
 	if (dev_priv->has_mob) {
-		struct {
-			SVGA3dCmdHeader header;
-			SVGA3dCmdEndGBQuery q;
-		} gb_cmd;
+		VMW_DECLARE_CMD_VAR(gb_cmd, SVGA3dCmdEndGBQuery);
 
 		BUG_ON(sizeof(gb_cmd) != sizeof(*cmd));
 
 		gb_cmd.header.id = SVGA_3D_CMD_END_GB_QUERY;
 		gb_cmd.header.size = cmd->header.size;
-		gb_cmd.q.cid = cmd->q.cid;
-		gb_cmd.q.type = cmd->q.type;
-		gb_cmd.q.mobid = cmd->q.guestResult.gmrId;
-		gb_cmd.q.offset = cmd->q.guestResult.offset;
+		gb_cmd.body.cid = cmd->body.cid;
+		gb_cmd.body.type = cmd->body.type;
+		gb_cmd.body.mobid = cmd->body.guestResult.gmrId;
+		gb_cmd.body.offset = cmd->body.guestResult.offset;
 
 		memcpy(cmd, &gb_cmd, sizeof(*cmd));
 		return vmw_cmd_end_gb_query(dev_priv, sw_context, header);
@@ -1466,8 +1428,7 @@ static int vmw_cmd_end_query(struct vmw_private *dev_priv,
 		return ret;
 
 	ret = vmw_translate_guest_ptr(dev_priv, sw_context,
-				      &cmd->q.guestResult,
-				      &vmw_bo);
+				      &cmd->body.guestResult, &vmw_bo);
 	if (unlikely(ret != 0))
 		return ret;
 
@@ -1477,7 +1438,7 @@ static int vmw_cmd_end_query(struct vmw_private *dev_priv,
 }
 
 /**
- * vmw_cmd_wait_gb_query - validate a  SVGA_3D_CMD_WAIT_GB_QUERY command.
+ * vmw_cmd_wait_gb_query - validate SVGA_3D_CMD_WAIT_GB_QUERY command.
  *
  * @dev_priv: Pointer to a device private struct.
  * @sw_context: The software context used for this command submission.
@@ -1488,19 +1449,15 @@ static int vmw_cmd_wait_gb_query(struct vmw_private *dev_priv,
 				 SVGA3dCmdHeader *header)
 {
 	struct vmw_buffer_object *vmw_bo;
-	struct vmw_query_cmd {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdWaitForGBQuery q;
-	} *cmd;
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdWaitForGBQuery);
 	int ret;
 
-	cmd = container_of(header, struct vmw_query_cmd, header);
+	cmd = container_of(header, typeof(*cmd), header);
 	ret = vmw_cmd_cid_check(dev_priv, sw_context, header);
 	if (unlikely(ret != 0))
 		return ret;
 
-	ret = vmw_translate_mob_ptr(dev_priv, sw_context,
-				    &cmd->q.mobid,
+	ret = vmw_translate_mob_ptr(dev_priv, sw_context, &cmd->body.mobid,
 				    &vmw_bo);
 	if (unlikely(ret != 0))
 		return ret;
@@ -1509,7 +1466,7 @@ static int vmw_cmd_wait_gb_query(struct vmw_private *dev_priv,
 }
 
 /**
- * vmw_cmd_wait_query - validate a  SVGA_3D_CMD_WAIT_QUERY command.
+ * vmw_cmd_wait_query - validate SVGA_3D_CMD_WAIT_QUERY command.
  *
  * @dev_priv: Pointer to a device private struct.
  * @sw_context: The software context used for this command submission.
@@ -1520,27 +1477,21 @@ static int vmw_cmd_wait_query(struct vmw_private *dev_priv,
 			      SVGA3dCmdHeader *header)
 {
 	struct vmw_buffer_object *vmw_bo;
-	struct vmw_query_cmd {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdWaitForQuery q;
-	} *cmd;
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdWaitForQuery);
 	int ret;
 
-	cmd = container_of(header, struct vmw_query_cmd, header);
+	cmd = container_of(header, typeof(*cmd), header);
 	if (dev_priv->has_mob) {
-		struct {
-			SVGA3dCmdHeader header;
-			SVGA3dCmdWaitForGBQuery q;
-		} gb_cmd;
+		VMW_DECLARE_CMD_VAR(gb_cmd, SVGA3dCmdWaitForGBQuery);
 
 		BUG_ON(sizeof(gb_cmd) != sizeof(*cmd));
 
 		gb_cmd.header.id = SVGA_3D_CMD_WAIT_FOR_GB_QUERY;
 		gb_cmd.header.size = cmd->header.size;
-		gb_cmd.q.cid = cmd->q.cid;
-		gb_cmd.q.type = cmd->q.type;
-		gb_cmd.q.mobid = cmd->q.guestResult.gmrId;
-		gb_cmd.q.offset = cmd->q.guestResult.offset;
+		gb_cmd.body.cid = cmd->body.cid;
+		gb_cmd.body.type = cmd->body.type;
+		gb_cmd.body.mobid = cmd->body.guestResult.gmrId;
+		gb_cmd.body.offset = cmd->body.guestResult.offset;
 
 		memcpy(cmd, &gb_cmd, sizeof(*cmd));
 		return vmw_cmd_wait_gb_query(dev_priv, sw_context, header);
@@ -1551,8 +1502,7 @@ static int vmw_cmd_wait_query(struct vmw_private *dev_priv,
 		return ret;
 
 	ret = vmw_translate_guest_ptr(dev_priv, sw_context,
-				      &cmd->q.guestResult,
-				      &vmw_bo);
+				      &cmd->body.guestResult, &vmw_bo);
 	if (unlikely(ret != 0))
 		return ret;
 
@@ -1565,54 +1515,52 @@ static int vmw_cmd_dma(struct vmw_private *dev_priv,
 {
 	struct vmw_buffer_object *vmw_bo = NULL;
 	struct vmw_surface *srf = NULL;
-	struct vmw_dma_cmd {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdSurfaceDMA dma;
-	} *cmd;
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdSurfaceDMA);
 	int ret;
 	SVGA3dCmdSurfaceDMASuffix *suffix;
 	uint32_t bo_size;
+	bool dirty;
 
-	cmd = container_of(header, struct vmw_dma_cmd, header);
-	suffix = (SVGA3dCmdSurfaceDMASuffix *)((unsigned long) &cmd->dma +
+	cmd = container_of(header, typeof(*cmd), header);
+	suffix = (SVGA3dCmdSurfaceDMASuffix *)((unsigned long) &cmd->body +
 					       header->size - sizeof(*suffix));
 
 	/* Make sure device and verifier stays in sync. */
 	if (unlikely(suffix->suffixSize != sizeof(*suffix))) {
-		DRM_ERROR("Invalid DMA suffix size.\n");
+		VMW_DEBUG_USER("Invalid DMA suffix size.\n");
 		return -EINVAL;
 	}
 
 	ret = vmw_translate_guest_ptr(dev_priv, sw_context,
-				      &cmd->dma.guest.ptr,
-				      &vmw_bo);
+				      &cmd->body.guest.ptr, &vmw_bo);
 	if (unlikely(ret != 0))
 		return ret;
 
 	/* Make sure DMA doesn't cross BO boundaries. */
 	bo_size = vmw_bo->base.num_pages * PAGE_SIZE;
-	if (unlikely(cmd->dma.guest.ptr.offset > bo_size)) {
-		DRM_ERROR("Invalid DMA offset.\n");
+	if (unlikely(cmd->body.guest.ptr.offset > bo_size)) {
+		VMW_DEBUG_USER("Invalid DMA offset.\n");
 		return -EINVAL;
 	}
 
-	bo_size -= cmd->dma.guest.ptr.offset;
+	bo_size -= cmd->body.guest.ptr.offset;
 	if (unlikely(suffix->maximumOffset > bo_size))
 		suffix->maximumOffset = bo_size;
 
+	dirty = (cmd->body.transfer == SVGA3D_WRITE_HOST_VRAM) ?
+		VMW_RES_DIRTY_SET : 0;
 	ret = vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface,
-				user_surface_converter, &cmd->dma.host.sid,
-				NULL);
+				dirty, user_surface_converter,
+				&cmd->body.host.sid, NULL);
 	if (unlikely(ret != 0)) {
 		if (unlikely(ret != -ERESTARTSYS))
-			DRM_ERROR("could not find surface for DMA.\n");
+			VMW_DEBUG_USER("could not find surface for DMA.\n");
 		return ret;
 	}
 
 	srf = vmw_res_to_srf(sw_context->res_cache[vmw_res_surface].res);
 
-	vmw_kms_cursor_snoop(srf, sw_context->fp->tfile, &vmw_bo->base,
-			     header);
+	vmw_kms_cursor_snoop(srf, sw_context->fp->tfile, &vmw_bo->base, header);
 
 	return 0;
 }
@@ -1621,10 +1569,7 @@ static int vmw_cmd_draw(struct vmw_private *dev_priv,
 			struct vmw_sw_context *sw_context,
 			SVGA3dCmdHeader *header)
 {
-	struct vmw_draw_cmd {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdDrawPrimitives body;
-	} *cmd;
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdDrawPrimitives);
 	SVGA3dVertexDecl *decl = (SVGA3dVertexDecl *)(
 		(unsigned long)header + sizeof(*cmd));
 	SVGA3dPrimitiveRange *range;
@@ -1636,16 +1581,17 @@ static int vmw_cmd_draw(struct vmw_private *dev_priv,
 	if (unlikely(ret != 0))
 		return ret;
 
-	cmd = container_of(header, struct vmw_draw_cmd, header);
+	cmd = container_of(header, typeof(*cmd), header);
 	maxnum = (header->size - sizeof(cmd->body)) / sizeof(*decl);
 
 	if (unlikely(cmd->body.numVertexDecls > maxnum)) {
-		DRM_ERROR("Illegal number of vertex declarations.\n");
+		VMW_DEBUG_USER("Illegal number of vertex declarations.\n");
 		return -EINVAL;
 	}
 
 	for (i = 0; i < cmd->body.numVertexDecls; ++i, ++decl) {
 		ret = vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface,
+					VMW_RES_DIRTY_NONE,
 					user_surface_converter,
 					&decl->array.surfaceId, NULL);
 		if (unlikely(ret != 0))
@@ -1655,13 +1601,14 @@ static int vmw_cmd_draw(struct vmw_private *dev_priv,
 	maxnum = (header->size - sizeof(cmd->body) -
 		  cmd->body.numVertexDecls * sizeof(*decl)) / sizeof(*range);
 	if (unlikely(cmd->body.numRanges > maxnum)) {
-		DRM_ERROR("Illegal number of index ranges.\n");
+		VMW_DEBUG_USER("Illegal number of index ranges.\n");
 		return -EINVAL;
 	}
 
 	range = (SVGA3dPrimitiveRange *) decl;
 	for (i = 0; i < cmd->body.numRanges; ++i, ++range) {
 		ret = vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface,
+					VMW_RES_DIRTY_NONE,
 					user_surface_converter,
 					&range->indexArray.surfaceId, NULL);
 		if (unlikely(ret != 0))
@@ -1670,30 +1617,24 @@ static int vmw_cmd_draw(struct vmw_private *dev_priv,
 	return 0;
 }
 
-
 static int vmw_cmd_tex_state(struct vmw_private *dev_priv,
 			     struct vmw_sw_context *sw_context,
 			     SVGA3dCmdHeader *header)
 {
-	struct vmw_tex_state_cmd {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdSetTextureState state;
-	} *cmd;
-
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdSetTextureState);
 	SVGA3dTextureState *last_state = (SVGA3dTextureState *)
 	  ((unsigned long) header + header->size + sizeof(header));
 	SVGA3dTextureState *cur_state = (SVGA3dTextureState *)
-		((unsigned long) header + sizeof(struct vmw_tex_state_cmd));
+		((unsigned long) header + sizeof(*cmd));
 	struct vmw_resource *ctx;
 	struct vmw_resource *res;
 	int ret;
 
-	cmd = container_of(header, struct vmw_tex_state_cmd,
-			   header);
+	cmd = container_of(header, typeof(*cmd), header);
 
 	ret = vmw_cmd_res_check(dev_priv, sw_context, vmw_res_context,
-				user_context_converter, &cmd->state.cid,
-				&ctx);
+				VMW_RES_DIRTY_SET, user_context_converter,
+				&cmd->body.cid, &ctx);
 	if (unlikely(ret != 0))
 		return ret;
 
@@ -1702,12 +1643,13 @@ static int vmw_cmd_tex_state(struct vmw_private *dev_priv,
 			continue;
 
 		if (cur_state->stage >= SVGA3D_NUM_TEXTURE_UNITS) {
-			DRM_ERROR("Illegal texture/sampler unit %u.\n",
-				  (unsigned) cur_state->stage);
+			VMW_DEBUG_USER("Illegal texture/sampler unit %u.\n",
+				       (unsigned int) cur_state->stage);
 			return -EINVAL;
 		}
 
 		ret = vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface,
+					VMW_RES_DIRTY_NONE,
 					user_surface_converter,
 					&cur_state->value, &res);
 		if (unlikely(ret != 0))
@@ -1744,12 +1686,10 @@ static int vmw_cmd_check_define_gmrfb(struct vmw_private *dev_priv,
 		SVGAFifoCmdDefineGMRFB body;
 	} *cmd = buf;
 
-	return vmw_translate_guest_ptr(dev_priv, sw_context,
-				       &cmd->body.ptr,
+	return vmw_translate_guest_ptr(dev_priv, sw_context, &cmd->body.ptr,
 				       &vmw_bo);
 }
 
-
 /**
  * vmw_cmd_res_switch_backup - Utility function to handle backup buffer
  * switching
@@ -1761,14 +1701,13 @@ static int vmw_cmd_check_define_gmrfb(struct vmw_private *dev_priv,
  * stream.
  * @backup_offset: Offset of backup into MOB.
  *
- * This function prepares for registering a switch of backup buffers
- * in the resource metadata just prior to unreserving. It's basically a wrapper
- * around vmw_cmd_res_switch_backup with a different interface.
+ * This function prepares for registering a switch of backup buffers in the
+ * resource metadata just prior to unreserving. It's basically a wrapper around
+ * vmw_cmd_res_switch_backup with a different interface.
  */
 static int vmw_cmd_res_switch_backup(struct vmw_private *dev_priv,
 				     struct vmw_sw_context *sw_context,
-				     struct vmw_resource *res,
-				     uint32_t *buf_id,
+				     struct vmw_resource *res, uint32_t *buf_id,
 				     unsigned long backup_offset)
 {
 	struct vmw_buffer_object *vbo;
@@ -1788,7 +1727,6 @@ static int vmw_cmd_res_switch_backup(struct vmw_private *dev_priv,
 	return 0;
 }
 
-
 /**
  * vmw_cmd_switch_backup - Utility function to handle backup buffer switching
  *
@@ -1801,34 +1739,31 @@ static int vmw_cmd_res_switch_backup(struct vmw_private *dev_priv,
  * stream.
  * @backup_offset: Offset of backup into MOB.
  *
- * This function prepares for registering a switch of backup buffers
- * in the resource metadata just prior to unreserving. It's basically a wrapper
- * around vmw_cmd_res_switch_backup with a different interface.
+ * This function prepares for registering a switch of backup buffers in the
+ * resource metadata just prior to unreserving. It's basically a wrapper around
+ * vmw_cmd_res_switch_backup with a different interface.
  */
 static int vmw_cmd_switch_backup(struct vmw_private *dev_priv,
 				 struct vmw_sw_context *sw_context,
 				 enum vmw_res_type res_type,
 				 const struct vmw_user_resource_conv
-				 *converter,
-				 uint32_t *res_id,
-				 uint32_t *buf_id,
+				 *converter, uint32_t *res_id, uint32_t *buf_id,
 				 unsigned long backup_offset)
 {
 	struct vmw_resource *res;
 	int ret;
 
 	ret = vmw_cmd_res_check(dev_priv, sw_context, res_type,
-				converter, res_id, &res);
+				VMW_RES_DIRTY_NONE, converter, res_id, &res);
 	if (ret)
 		return ret;
 
-	return vmw_cmd_res_switch_backup(dev_priv, sw_context, res,
-					 buf_id, backup_offset);
+	return vmw_cmd_res_switch_backup(dev_priv, sw_context, res, buf_id,
+					 backup_offset);
 }
 
 /**
- * vmw_cmd_bind_gb_surface - Validate an SVGA_3D_CMD_BIND_GB_SURFACE
- * command
+ * vmw_cmd_bind_gb_surface - Validate SVGA_3D_CMD_BIND_GB_SURFACE command
  *
  * @dev_priv: Pointer to a device private struct.
  * @sw_context: The software context being used for this batch.
@@ -1838,22 +1773,16 @@ static int vmw_cmd_bind_gb_surface(struct vmw_private *dev_priv,
 				   struct vmw_sw_context *sw_context,
 				   SVGA3dCmdHeader *header)
 {
-	struct vmw_bind_gb_surface_cmd {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdBindGBSurface body;
-	} *cmd;
-
-	cmd = container_of(header, struct vmw_bind_gb_surface_cmd, header);
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdBindGBSurface) =
+		container_of(header, typeof(*cmd), header);
 
 	return vmw_cmd_switch_backup(dev_priv, sw_context, vmw_res_surface,
-				     user_surface_converter,
-				     &cmd->body.sid, &cmd->body.mobid,
-				     0);
+				     user_surface_converter, &cmd->body.sid,
+				     &cmd->body.mobid, 0);
 }
 
 /**
- * vmw_cmd_update_gb_image - Validate an SVGA_3D_CMD_UPDATE_GB_IMAGE
- * command
+ * vmw_cmd_update_gb_image - Validate SVGA_3D_CMD_UPDATE_GB_IMAGE command
  *
  * @dev_priv: Pointer to a device private struct.
  * @sw_context: The software context being used for this batch.
@@ -1863,21 +1792,16 @@ static int vmw_cmd_update_gb_image(struct vmw_private *dev_priv,
 				   struct vmw_sw_context *sw_context,
 				   SVGA3dCmdHeader *header)
 {
-	struct vmw_gb_surface_cmd {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdUpdateGBImage body;
-	} *cmd;
-
-	cmd = container_of(header, struct vmw_gb_surface_cmd, header);
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdUpdateGBImage) =
+		container_of(header, typeof(*cmd), header);
 
 	return vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface,
-				 user_surface_converter,
+				 VMW_RES_DIRTY_NONE, user_surface_converter,
 				 &cmd->body.image.sid, NULL);
 }
 
 /**
- * vmw_cmd_update_gb_surface - Validate an SVGA_3D_CMD_UPDATE_GB_SURFACE
- * command
+ * vmw_cmd_update_gb_surface - Validate SVGA_3D_CMD_UPDATE_GB_SURFACE command
  *
  * @dev_priv: Pointer to a device private struct.
  * @sw_context: The software context being used for this batch.
@@ -1887,21 +1811,16 @@ static int vmw_cmd_update_gb_surface(struct vmw_private *dev_priv,
 				     struct vmw_sw_context *sw_context,
 				     SVGA3dCmdHeader *header)
 {
-	struct vmw_gb_surface_cmd {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdUpdateGBSurface body;
-	} *cmd;
-
-	cmd = container_of(header, struct vmw_gb_surface_cmd, header);
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdUpdateGBSurface) =
+		container_of(header, typeof(*cmd), header);
 
 	return vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface,
-				 user_surface_converter,
+				 VMW_RES_DIRTY_CLEAR, user_surface_converter,
 				 &cmd->body.sid, NULL);
 }
 
 /**
- * vmw_cmd_readback_gb_image - Validate an SVGA_3D_CMD_READBACK_GB_IMAGE
- * command
+ * vmw_cmd_readback_gb_image - Validate SVGA_3D_CMD_READBACK_GB_IMAGE command
  *
  * @dev_priv: Pointer to a device private struct.
  * @sw_context: The software context being used for this batch.
@@ -1911,20 +1830,16 @@ static int vmw_cmd_readback_gb_image(struct vmw_private *dev_priv,
 				     struct vmw_sw_context *sw_context,
 				     SVGA3dCmdHeader *header)
 {
-	struct vmw_gb_surface_cmd {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdReadbackGBImage body;
-	} *cmd;
-
-	cmd = container_of(header, struct vmw_gb_surface_cmd, header);
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdReadbackGBImage) =
+		container_of(header, typeof(*cmd), header);
 
 	return vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface,
-				 user_surface_converter,
+				 VMW_RES_DIRTY_NONE, user_surface_converter,
 				 &cmd->body.image.sid, NULL);
 }
 
 /**
- * vmw_cmd_readback_gb_surface - Validate an SVGA_3D_CMD_READBACK_GB_SURFACE
+ * vmw_cmd_readback_gb_surface - Validate SVGA_3D_CMD_READBACK_GB_SURFACE
  * command
  *
  * @dev_priv: Pointer to a device private struct.
@@ -1935,20 +1850,16 @@ static int vmw_cmd_readback_gb_surface(struct vmw_private *dev_priv,
 				       struct vmw_sw_context *sw_context,
 				       SVGA3dCmdHeader *header)
 {
-	struct vmw_gb_surface_cmd {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdReadbackGBSurface body;
-	} *cmd;
-
-	cmd = container_of(header, struct vmw_gb_surface_cmd, header);
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdReadbackGBSurface) =
+		container_of(header, typeof(*cmd), header);
 
 	return vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface,
-				 user_surface_converter,
+				 VMW_RES_DIRTY_CLEAR, user_surface_converter,
 				 &cmd->body.sid, NULL);
 }
 
 /**
- * vmw_cmd_invalidate_gb_image - Validate an SVGA_3D_CMD_INVALIDATE_GB_IMAGE
+ * vmw_cmd_invalidate_gb_image - Validate SVGA_3D_CMD_INVALIDATE_GB_IMAGE
  * command
  *
  * @dev_priv: Pointer to a device private struct.
@@ -1959,21 +1870,17 @@ static int vmw_cmd_invalidate_gb_image(struct vmw_private *dev_priv,
 				       struct vmw_sw_context *sw_context,
 				       SVGA3dCmdHeader *header)
 {
-	struct vmw_gb_surface_cmd {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdInvalidateGBImage body;
-	} *cmd;
-
-	cmd = container_of(header, struct vmw_gb_surface_cmd, header);
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdInvalidateGBImage) =
+		container_of(header, typeof(*cmd), header);
 
 	return vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface,
-				 user_surface_converter,
+				 VMW_RES_DIRTY_NONE, user_surface_converter,
 				 &cmd->body.image.sid, NULL);
 }
 
 /**
- * vmw_cmd_invalidate_gb_surface - Validate an
- * SVGA_3D_CMD_INVALIDATE_GB_SURFACE command
+ * vmw_cmd_invalidate_gb_surface - Validate SVGA_3D_CMD_INVALIDATE_GB_SURFACE
+ * command
  *
  * @dev_priv: Pointer to a device private struct.
  * @sw_context: The software context being used for this batch.
@@ -1983,22 +1890,16 @@ static int vmw_cmd_invalidate_gb_surface(struct vmw_private *dev_priv,
 					 struct vmw_sw_context *sw_context,
 					 SVGA3dCmdHeader *header)
 {
-	struct vmw_gb_surface_cmd {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdInvalidateGBSurface body;
-	} *cmd;
-
-	cmd = container_of(header, struct vmw_gb_surface_cmd, header);
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdInvalidateGBSurface) =
+		container_of(header, typeof(*cmd), header);
 
 	return vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface,
-				 user_surface_converter,
+				 VMW_RES_DIRTY_CLEAR, user_surface_converter,
 				 &cmd->body.sid, NULL);
 }
 
-
 /**
- * vmw_cmd_shader_define - Validate an SVGA_3D_CMD_SHADER_DEFINE
- * command
+ * vmw_cmd_shader_define - Validate SVGA_3D_CMD_SHADER_DEFINE command
  *
  * @dev_priv: Pointer to a device private struct.
  * @sw_context: The software context being used for this batch.
@@ -2008,20 +1909,16 @@ static int vmw_cmd_shader_define(struct vmw_private *dev_priv,
 				 struct vmw_sw_context *sw_context,
 				 SVGA3dCmdHeader *header)
 {
-	struct vmw_shader_define_cmd {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdDefineShader body;
-	} *cmd;
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdDefineShader);
 	int ret;
 	size_t size;
 	struct vmw_resource *ctx;
 
-	cmd = container_of(header, struct vmw_shader_define_cmd,
-			   header);
+	cmd = container_of(header, typeof(*cmd), header);
 
 	ret = vmw_cmd_res_check(dev_priv, sw_context, vmw_res_context,
-				user_context_converter, &cmd->body.cid,
-				&ctx);
+				VMW_RES_DIRTY_SET, user_context_converter,
+				&cmd->body.cid, &ctx);
 	if (unlikely(ret != 0))
 		return ret;
 
@@ -2029,24 +1926,20 @@ static int vmw_cmd_shader_define(struct vmw_private *dev_priv,
 		return 0;
 
 	size = cmd->header.size - sizeof(cmd->body);
-	ret = vmw_compat_shader_add(dev_priv,
-				    vmw_context_res_man(ctx),
-				    cmd->body.shid, cmd + 1,
-				    cmd->body.type, size,
-				    &sw_context->staged_cmd_res);
+	ret = vmw_compat_shader_add(dev_priv, vmw_context_res_man(ctx),
+				    cmd->body.shid, cmd + 1, cmd->body.type,
+				    size, &sw_context->staged_cmd_res);
 	if (unlikely(ret != 0))
 		return ret;
 
-	return vmw_resource_relocation_add(sw_context,
-					   NULL,
+	return vmw_resource_relocation_add(sw_context, NULL,
 					   vmw_ptr_diff(sw_context->buf_start,
 							&cmd->header.id),
 					   vmw_res_rel_nop);
 }
 
 /**
- * vmw_cmd_shader_destroy - Validate an SVGA_3D_CMD_SHADER_DESTROY
- * command
+ * vmw_cmd_shader_destroy - Validate SVGA_3D_CMD_SHADER_DESTROY command
  *
  * @dev_priv: Pointer to a device private struct.
  * @sw_context: The software context being used for this batch.
@@ -2056,42 +1949,34 @@ static int vmw_cmd_shader_destroy(struct vmw_private *dev_priv,
 				  struct vmw_sw_context *sw_context,
 				  SVGA3dCmdHeader *header)
 {
-	struct vmw_shader_destroy_cmd {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdDestroyShader body;
-	} *cmd;
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdDestroyShader);
 	int ret;
 	struct vmw_resource *ctx;
 
-	cmd = container_of(header, struct vmw_shader_destroy_cmd,
-			   header);
+	cmd = container_of(header, typeof(*cmd), header);
 
 	ret = vmw_cmd_res_check(dev_priv, sw_context, vmw_res_context,
-				user_context_converter, &cmd->body.cid,
-				&ctx);
+				VMW_RES_DIRTY_SET, user_context_converter,
+				&cmd->body.cid, &ctx);
 	if (unlikely(ret != 0))
 		return ret;
 
 	if (unlikely(!dev_priv->has_mob))
 		return 0;
 
-	ret = vmw_shader_remove(vmw_context_res_man(ctx),
-				cmd->body.shid,
-				cmd->body.type,
-				&sw_context->staged_cmd_res);
+	ret = vmw_shader_remove(vmw_context_res_man(ctx), cmd->body.shid,
+				cmd->body.type, &sw_context->staged_cmd_res);
 	if (unlikely(ret != 0))
 		return ret;
 
-	return vmw_resource_relocation_add(sw_context,
-					   NULL,
+	return vmw_resource_relocation_add(sw_context, NULL,
 					   vmw_ptr_diff(sw_context->buf_start,
 							&cmd->header.id),
 					   vmw_res_rel_nop);
 }
 
 /**
- * vmw_cmd_set_shader - Validate an SVGA_3D_CMD_SET_SHADER
- * command
+ * vmw_cmd_set_shader - Validate SVGA_3D_CMD_SET_SHADER command
  *
  * @dev_priv: Pointer to a device private struct.
  * @sw_context: The software context being used for this batch.
@@ -2101,27 +1986,23 @@ static int vmw_cmd_set_shader(struct vmw_private *dev_priv,
 			      struct vmw_sw_context *sw_context,
 			      SVGA3dCmdHeader *header)
 {
-	struct vmw_set_shader_cmd {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdSetShader body;
-	} *cmd;
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdSetShader);
 	struct vmw_ctx_bindinfo_shader binding;
 	struct vmw_resource *ctx, *res = NULL;
 	struct vmw_ctx_validation_info *ctx_info;
 	int ret;
 
-	cmd = container_of(header, struct vmw_set_shader_cmd,
-			   header);
+	cmd = container_of(header, typeof(*cmd), header);
 
 	if (cmd->body.type >= SVGA3D_SHADERTYPE_PREDX_MAX) {
-		DRM_ERROR("Illegal shader type %u.\n",
-			  (unsigned) cmd->body.type);
+		VMW_DEBUG_USER("Illegal shader type %u.\n",
+			       (unsigned int) cmd->body.type);
 		return -EINVAL;
 	}
 
 	ret = vmw_cmd_res_check(dev_priv, sw_context, vmw_res_context,
-				user_context_converter, &cmd->body.cid,
-				&ctx);
+				VMW_RES_DIRTY_SET, user_context_converter,
+				&cmd->body.cid, &ctx);
 	if (unlikely(ret != 0))
 		return ret;
 
@@ -2130,21 +2011,20 @@ static int vmw_cmd_set_shader(struct vmw_private *dev_priv,
 
 	if (cmd->body.shid != SVGA3D_INVALID_ID) {
 		res = vmw_shader_lookup(vmw_context_res_man(ctx),
-					cmd->body.shid,
-					cmd->body.type);
-
+					cmd->body.shid, cmd->body.type);
 		if (!IS_ERR(res)) {
-			ret = vmw_execbuf_res_noctx_val_add(sw_context, res);
+			ret = vmw_execbuf_res_noctx_val_add(sw_context, res,
+							    VMW_RES_DIRTY_NONE);
 			if (unlikely(ret != 0))
 				return ret;
 		}
 	}
 
 	if (IS_ERR_OR_NULL(res)) {
-		ret = vmw_cmd_res_check(dev_priv, sw_context,
-					vmw_res_shader,
-					user_shader_converter,
-					&cmd->body.shid, &res);
+		ret = vmw_cmd_res_check(dev_priv, sw_context, vmw_res_shader,
+					VMW_RES_DIRTY_NONE,
+					user_shader_converter, &cmd->body.shid,
+					&res);
 		if (unlikely(ret != 0))
 			return ret;
 	}
@@ -2157,14 +2037,13 @@ static int vmw_cmd_set_shader(struct vmw_private *dev_priv,
 	binding.bi.res = res;
 	binding.bi.bt = vmw_ctx_binding_shader;
 	binding.shader_slot = cmd->body.type - SVGA3D_SHADERTYPE_MIN;
-	vmw_binding_add(ctx_info->staged, &binding.bi,
-			binding.shader_slot, 0);
+	vmw_binding_add(ctx_info->staged, &binding.bi, binding.shader_slot, 0);
+
 	return 0;
 }
 
 /**
- * vmw_cmd_set_shader_const - Validate an SVGA_3D_CMD_SET_SHADER_CONST
- * command
+ * vmw_cmd_set_shader_const - Validate SVGA_3D_CMD_SET_SHADER_CONST command
  *
  * @dev_priv: Pointer to a device private struct.
  * @sw_context: The software context being used for this batch.
@@ -2174,18 +2053,14 @@ static int vmw_cmd_set_shader_const(struct vmw_private *dev_priv,
 				    struct vmw_sw_context *sw_context,
 				    SVGA3dCmdHeader *header)
 {
-	struct vmw_set_shader_const_cmd {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdSetShaderConst body;
-	} *cmd;
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdSetShaderConst);
 	int ret;
 
-	cmd = container_of(header, struct vmw_set_shader_const_cmd,
-			   header);
+	cmd = container_of(header, typeof(*cmd), header);
 
 	ret = vmw_cmd_res_check(dev_priv, sw_context, vmw_res_context,
-				user_context_converter, &cmd->body.cid,
-				NULL);
+				VMW_RES_DIRTY_SET, user_context_converter,
+				&cmd->body.cid, NULL);
 	if (unlikely(ret != 0))
 		return ret;
 
@@ -2196,8 +2071,7 @@ static int vmw_cmd_set_shader_const(struct vmw_private *dev_priv,
 }
 
 /**
- * vmw_cmd_bind_gb_shader - Validate an SVGA_3D_CMD_BIND_GB_SHADER
- * command
+ * vmw_cmd_bind_gb_shader - Validate SVGA_3D_CMD_BIND_GB_SHADER command
  *
  * @dev_priv: Pointer to a device private struct.
  * @sw_context: The software context being used for this batch.
@@ -2207,22 +2081,16 @@ static int vmw_cmd_bind_gb_shader(struct vmw_private *dev_priv,
 				  struct vmw_sw_context *sw_context,
 				  SVGA3dCmdHeader *header)
 {
-	struct vmw_bind_gb_shader_cmd {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdBindGBShader body;
-	} *cmd;
-
-	cmd = container_of(header, struct vmw_bind_gb_shader_cmd,
-			   header);
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdBindGBShader) =
+		container_of(header, typeof(*cmd), header);
 
 	return vmw_cmd_switch_backup(dev_priv, sw_context, vmw_res_shader,
-				     user_shader_converter,
-				     &cmd->body.shid, &cmd->body.mobid,
-				     cmd->body.offsetInBytes);
+				     user_shader_converter, &cmd->body.shid,
+				     &cmd->body.mobid, cmd->body.offsetInBytes);
 }
 
 /**
- * vmw_cmd_dx_set_single_constant_buffer - Validate an
+ * vmw_cmd_dx_set_single_constant_buffer - Validate
  * SVGA_3D_CMD_DX_SET_SINGLE_CONSTANT_BUFFER command.
  *
  * @dev_priv: Pointer to a device private struct.
@@ -2234,23 +2102,18 @@ vmw_cmd_dx_set_single_constant_buffer(struct vmw_private *dev_priv,
 				      struct vmw_sw_context *sw_context,
 				      SVGA3dCmdHeader *header)
 {
-	struct {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdDXSetSingleConstantBuffer body;
-	} *cmd;
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdDXSetSingleConstantBuffer);
 	struct vmw_resource *res = NULL;
-	struct vmw_ctx_validation_info *ctx_node = sw_context->dx_ctx_node;
+	struct vmw_ctx_validation_info *ctx_node = VMW_GET_CTX_NODE(sw_context);
 	struct vmw_ctx_bindinfo_cb binding;
 	int ret;
 
-	if (unlikely(ctx_node == NULL)) {
-		DRM_ERROR("DX Context not set.\n");
+	if (!ctx_node)
 		return -EINVAL;
-	}
 
 	cmd = container_of(header, typeof(*cmd), header);
 	ret = vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface,
-				user_surface_converter,
+				VMW_RES_DIRTY_NONE, user_surface_converter,
 				&cmd->body.sid, &res);
 	if (unlikely(ret != 0))
 		return ret;
@@ -2265,21 +2128,21 @@ vmw_cmd_dx_set_single_constant_buffer(struct vmw_private *dev_priv,
 
 	if (binding.shader_slot >= SVGA3D_NUM_SHADERTYPE_DX10 ||
 	    binding.slot >= SVGA3D_DX_MAX_CONSTBUFFERS) {
-		DRM_ERROR("Illegal const buffer shader %u slot %u.\n",
-			  (unsigned) cmd->body.type,
-			  (unsigned) binding.slot);
+		VMW_DEBUG_USER("Illegal const buffer shader %u slot %u.\n",
+			       (unsigned int) cmd->body.type,
+			       (unsigned int) binding.slot);
 		return -EINVAL;
 	}
 
-	vmw_binding_add(ctx_node->staged, &binding.bi,
-			binding.shader_slot, binding.slot);
+	vmw_binding_add(ctx_node->staged, &binding.bi, binding.shader_slot,
+			binding.slot);
 
 	return 0;
 }
 
 /**
- * vmw_cmd_dx_set_shader_res - Validate an
- * SVGA_3D_CMD_DX_SET_SHADER_RESOURCES command
+ * vmw_cmd_dx_set_shader_res - Validate SVGA_3D_CMD_DX_SET_SHADER_RESOURCES
+ * command
  *
  * @dev_priv: Pointer to a device private struct.
  * @sw_context: The software context being used for this batch.
@@ -2289,17 +2152,15 @@ static int vmw_cmd_dx_set_shader_res(struct vmw_private *dev_priv,
 				     struct vmw_sw_context *sw_context,
 				     SVGA3dCmdHeader *header)
 {
-	struct {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdDXSetShaderResources body;
-	} *cmd = container_of(header, typeof(*cmd), header);
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdDXSetShaderResources) =
+		container_of(header, typeof(*cmd), header);
 	u32 num_sr_view = (cmd->header.size - sizeof(cmd->body)) /
 		sizeof(SVGA3dShaderResourceViewId);
 
 	if ((u64) cmd->body.startView + (u64) num_sr_view >
 	    (u64) SVGA3D_DX_MAX_SRVIEWS ||
 	    cmd->body.type >= SVGA3D_SHADERTYPE_DX10_MAX) {
-		DRM_ERROR("Invalid shader binding.\n");
+		VMW_DEBUG_USER("Invalid shader binding.\n");
 		return -EINVAL;
 	}
 
@@ -2311,8 +2172,7 @@ static int vmw_cmd_dx_set_shader_res(struct vmw_private *dev_priv,
 }
 
 /**
- * vmw_cmd_dx_set_shader - Validate an SVGA_3D_CMD_DX_SET_SHADER
- * command
+ * vmw_cmd_dx_set_shader - Validate SVGA_3D_CMD_DX_SET_SHADER command
  *
  * @dev_priv: Pointer to a device private struct.
  * @sw_context: The software context being used for this batch.
@@ -2322,36 +2182,32 @@ static int vmw_cmd_dx_set_shader(struct vmw_private *dev_priv,
 				 struct vmw_sw_context *sw_context,
 				 SVGA3dCmdHeader *header)
 {
-	struct {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdDXSetShader body;
-	} *cmd;
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdDXSetShader);
 	struct vmw_resource *res = NULL;
-	struct vmw_ctx_validation_info *ctx_node = sw_context->dx_ctx_node;
+	struct vmw_ctx_validation_info *ctx_node = VMW_GET_CTX_NODE(sw_context);
 	struct vmw_ctx_bindinfo_shader binding;
 	int ret = 0;
 
-	if (unlikely(ctx_node == NULL)) {
-		DRM_ERROR("DX Context not set.\n");
+	if (!ctx_node)
 		return -EINVAL;
-	}
 
 	cmd = container_of(header, typeof(*cmd), header);
 
 	if (cmd->body.type >= SVGA3D_SHADERTYPE_DX10_MAX) {
-		DRM_ERROR("Illegal shader type %u.\n",
-			  (unsigned) cmd->body.type);
+		VMW_DEBUG_USER("Illegal shader type %u.\n",
+			       (unsigned int) cmd->body.type);
 		return -EINVAL;
 	}
 
 	if (cmd->body.shaderId != SVGA3D_INVALID_ID) {
 		res = vmw_shader_lookup(sw_context->man, cmd->body.shaderId, 0);
 		if (IS_ERR(res)) {
-			DRM_ERROR("Could not find shader for binding.\n");
+			VMW_DEBUG_USER("Could not find shader for binding.\n");
 			return PTR_ERR(res);
 		}
 
-		ret = vmw_execbuf_res_noctx_val_add(sw_context, res);
+		ret = vmw_execbuf_res_noctx_val_add(sw_context, res,
+						    VMW_RES_DIRTY_NONE);
 		if (ret)
 			return ret;
 	}
@@ -2361,15 +2217,14 @@ static int vmw_cmd_dx_set_shader(struct vmw_private *dev_priv,
 	binding.bi.bt = vmw_ctx_binding_dx_shader;
 	binding.shader_slot = cmd->body.type - SVGA3D_SHADERTYPE_MIN;
 
-	vmw_binding_add(ctx_node->staged, &binding.bi,
-			binding.shader_slot, 0);
+	vmw_binding_add(ctx_node->staged, &binding.bi, binding.shader_slot, 0);
 
 	return 0;
 }
 
 /**
- * vmw_cmd_dx_set_vertex_buffers - Validates an
- * SVGA_3D_CMD_DX_SET_VERTEX_BUFFERS command
+ * vmw_cmd_dx_set_vertex_buffers - Validates SVGA_3D_CMD_DX_SET_VERTEX_BUFFERS
+ * command
  *
  * @dev_priv: Pointer to a device private struct.
  * @sw_context: The software context being used for this batch.
@@ -2379,7 +2234,7 @@ static int vmw_cmd_dx_set_vertex_buffers(struct vmw_private *dev_priv,
 					 struct vmw_sw_context *sw_context,
 					 SVGA3dCmdHeader *header)
 {
-	struct vmw_ctx_validation_info *ctx_node = sw_context->dx_ctx_node;
+	struct vmw_ctx_validation_info *ctx_node = VMW_GET_CTX_NODE(sw_context);
 	struct vmw_ctx_bindinfo_vb binding;
 	struct vmw_resource *res;
 	struct {
@@ -2389,22 +2244,21 @@ static int vmw_cmd_dx_set_vertex_buffers(struct vmw_private *dev_priv,
 	} *cmd;
 	int i, ret, num;
 
-	if (unlikely(ctx_node == NULL)) {
-		DRM_ERROR("DX Context not set.\n");
+	if (!ctx_node)
 		return -EINVAL;
-	}
 
 	cmd = container_of(header, typeof(*cmd), header);
 	num = (cmd->header.size - sizeof(cmd->body)) /
 		sizeof(SVGA3dVertexBuffer);
 	if ((u64)num + (u64)cmd->body.startBuffer >
 	    (u64)SVGA3D_DX_MAX_VERTEXBUFFERS) {
-		DRM_ERROR("Invalid number of vertex buffers.\n");
+		VMW_DEBUG_USER("Invalid number of vertex buffers.\n");
 		return -EINVAL;
 	}
 
 	for (i = 0; i < num; i++) {
 		ret = vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface,
+					VMW_RES_DIRTY_NONE,
 					user_surface_converter,
 					&cmd->buf[i].sid, &res);
 		if (unlikely(ret != 0))
@@ -2417,15 +2271,14 @@ static int vmw_cmd_dx_set_vertex_buffers(struct vmw_private *dev_priv,
 		binding.stride = cmd->buf[i].stride;
 		binding.slot = i + cmd->body.startBuffer;
 
-		vmw_binding_add(ctx_node->staged, &binding.bi,
-				0, binding.slot);
+		vmw_binding_add(ctx_node->staged, &binding.bi, 0, binding.slot);
 	}
 
 	return 0;
 }
 
 /**
- * vmw_cmd_dx_ia_set_vertex_buffers - Validate an
+ * vmw_cmd_dx_ia_set_vertex_buffers - Validate
  * SVGA_3D_CMD_DX_IA_SET_INDEX_BUFFER command.
  *
  * @dev_priv: Pointer to a device private struct.
@@ -2436,23 +2289,18 @@ static int vmw_cmd_dx_set_index_buffer(struct vmw_private *dev_priv,
 				       struct vmw_sw_context *sw_context,
 				       SVGA3dCmdHeader *header)
 {
-	struct vmw_ctx_validation_info *ctx_node = sw_context->dx_ctx_node;
+	struct vmw_ctx_validation_info *ctx_node = VMW_GET_CTX_NODE(sw_context);
 	struct vmw_ctx_bindinfo_ib binding;
 	struct vmw_resource *res;
-	struct {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdDXSetIndexBuffer body;
-	} *cmd;
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdDXSetIndexBuffer);
 	int ret;
 
-	if (unlikely(ctx_node == NULL)) {
-		DRM_ERROR("DX Context not set.\n");
+	if (!ctx_node)
 		return -EINVAL;
-	}
 
 	cmd = container_of(header, typeof(*cmd), header);
 	ret = vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface,
-				user_surface_converter,
+				VMW_RES_DIRTY_NONE, user_surface_converter,
 				&cmd->body.sid, &res);
 	if (unlikely(ret != 0))
 		return ret;
@@ -2469,8 +2317,8 @@ static int vmw_cmd_dx_set_index_buffer(struct vmw_private *dev_priv,
 }
 
 /**
- * vmw_cmd_dx_set_rendertarget - Validate an
- * SVGA_3D_CMD_DX_SET_RENDERTARGETS command
+ * vmw_cmd_dx_set_rendertarget - Validate SVGA_3D_CMD_DX_SET_RENDERTARGETS
+ * command
  *
  * @dev_priv: Pointer to a device private struct.
  * @sw_context: The software context being used for this batch.
@@ -2480,32 +2328,29 @@ static int vmw_cmd_dx_set_rendertargets(struct vmw_private *dev_priv,
 					struct vmw_sw_context *sw_context,
 					SVGA3dCmdHeader *header)
 {
-	struct {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdDXSetRenderTargets body;
-	} *cmd = container_of(header, typeof(*cmd), header);
-	int ret;
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdDXSetRenderTargets) =
+		container_of(header, typeof(*cmd), header);
 	u32 num_rt_view = (cmd->header.size - sizeof(cmd->body)) /
 		sizeof(SVGA3dRenderTargetViewId);
+	int ret;
 
 	if (num_rt_view > SVGA3D_MAX_SIMULTANEOUS_RENDER_TARGETS) {
-		DRM_ERROR("Invalid DX Rendertarget binding.\n");
+		VMW_DEBUG_USER("Invalid DX Rendertarget binding.\n");
 		return -EINVAL;
 	}
 
-	ret = vmw_view_bindings_add(sw_context, vmw_view_ds,
-				    vmw_ctx_binding_ds, 0,
-				    &cmd->body.depthStencilViewId, 1, 0);
+	ret = vmw_view_bindings_add(sw_context, vmw_view_ds, vmw_ctx_binding_ds,
+				    0, &cmd->body.depthStencilViewId, 1, 0);
 	if (ret)
 		return ret;
 
 	return vmw_view_bindings_add(sw_context, vmw_view_rt,
-				     vmw_ctx_binding_dx_rt, 0,
-				     (void *)&cmd[1], num_rt_view, 0);
+				     vmw_ctx_binding_dx_rt, 0, (void *)&cmd[1],
+				     num_rt_view, 0);
 }
 
 /**
- * vmw_cmd_dx_clear_rendertarget_view - Validate an
+ * vmw_cmd_dx_clear_rendertarget_view - Validate
  * SVGA_3D_CMD_DX_CLEAR_RENDERTARGET_VIEW command
  *
  * @dev_priv: Pointer to a device private struct.
@@ -2516,17 +2361,15 @@ static int vmw_cmd_dx_clear_rendertarget_view(struct vmw_private *dev_priv,
 					      struct vmw_sw_context *sw_context,
 					      SVGA3dCmdHeader *header)
 {
-	struct {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdDXClearRenderTargetView body;
-	} *cmd = container_of(header, typeof(*cmd), header);
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdDXClearRenderTargetView) =
+		container_of(header, typeof(*cmd), header);
 
 	return PTR_RET(vmw_view_id_val_add(sw_context, vmw_view_rt,
 					   cmd->body.renderTargetViewId));
 }
 
 /**
- * vmw_cmd_dx_clear_rendertarget_view - Validate an
+ * vmw_cmd_dx_clear_rendertarget_view - Validate
  * SVGA_3D_CMD_DX_CLEAR_DEPTHSTENCIL_VIEW command
  *
  * @dev_priv: Pointer to a device private struct.
@@ -2537,10 +2380,8 @@ static int vmw_cmd_dx_clear_depthstencil_view(struct vmw_private *dev_priv,
 					      struct vmw_sw_context *sw_context,
 					      SVGA3dCmdHeader *header)
 {
-	struct {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdDXClearDepthStencilView body;
-	} *cmd = container_of(header, typeof(*cmd), header);
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdDXClearDepthStencilView) =
+		container_of(header, typeof(*cmd), header);
 
 	return PTR_RET(vmw_view_id_val_add(sw_context, vmw_view_ds,
 					   cmd->body.depthStencilViewId));
@@ -2550,14 +2391,14 @@ static int vmw_cmd_dx_view_define(struct vmw_private *dev_priv,
 				  struct vmw_sw_context *sw_context,
 				  SVGA3dCmdHeader *header)
 {
-	struct vmw_ctx_validation_info *ctx_node = sw_context->dx_ctx_node;
+	struct vmw_ctx_validation_info *ctx_node = VMW_GET_CTX_NODE(sw_context);
 	struct vmw_resource *srf;
 	struct vmw_resource *res;
 	enum vmw_view_type view_type;
 	int ret;
 	/*
-	 * This is based on the fact that all affected define commands have
-	 * the same initial command body layout.
+	 * This is based on the fact that all affected define commands have the
+	 * same initial command body layout.
 	 */
 	struct {
 		SVGA3dCmdHeader header;
@@ -2565,17 +2406,16 @@ static int vmw_cmd_dx_view_define(struct vmw_private *dev_priv,
 		uint32 sid;
 	} *cmd;
 
-	if (unlikely(ctx_node == NULL)) {
-		DRM_ERROR("DX Context not set.\n");
+	if (!ctx_node)
 		return -EINVAL;
-	}
 
 	view_type = vmw_view_cmd_to_type(header->id);
 	if (view_type == vmw_view_max)
 		return -EINVAL;
+
 	cmd = container_of(header, typeof(*cmd), header);
 	ret = vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface,
-				user_surface_converter,
+				VMW_RES_DIRTY_NONE, user_surface_converter,
 				&cmd->sid, &srf);
 	if (unlikely(ret != 0))
 		return ret;
@@ -2585,19 +2425,14 @@ static int vmw_cmd_dx_view_define(struct vmw_private *dev_priv,
 	if (unlikely(ret != 0))
 		return ret;
 
-	return vmw_view_add(sw_context->man,
-			    ctx_node->ctx,
-			    srf,
-			    view_type,
-			    cmd->defined_id,
-			    header,
+	return vmw_view_add(sw_context->man, ctx_node->ctx, srf, view_type,
+			    cmd->defined_id, header,
 			    header->size + sizeof(*header),
 			    &sw_context->staged_cmd_res);
 }
 
 /**
- * vmw_cmd_dx_set_so_targets - Validate an
- * SVGA_3D_CMD_DX_SET_SOTARGETS command.
+ * vmw_cmd_dx_set_so_targets - Validate SVGA_3D_CMD_DX_SET_SOTARGETS command.
  *
  * @dev_priv: Pointer to a device private struct.
  * @sw_context: The software context being used for this batch.
@@ -2607,7 +2442,7 @@ static int vmw_cmd_dx_set_so_targets(struct vmw_private *dev_priv,
 				     struct vmw_sw_context *sw_context,
 				     SVGA3dCmdHeader *header)
 {
-	struct vmw_ctx_validation_info *ctx_node = sw_context->dx_ctx_node;
+	struct vmw_ctx_validation_info *ctx_node = VMW_GET_CTX_NODE(sw_context);
 	struct vmw_ctx_bindinfo_so binding;
 	struct vmw_resource *res;
 	struct {
@@ -2617,22 +2452,20 @@ static int vmw_cmd_dx_set_so_targets(struct vmw_private *dev_priv,
 	} *cmd;
 	int i, ret, num;
 
-	if (unlikely(ctx_node == NULL)) {
-		DRM_ERROR("DX Context not set.\n");
+	if (!ctx_node)
 		return -EINVAL;
-	}
 
 	cmd = container_of(header, typeof(*cmd), header);
-	num = (cmd->header.size - sizeof(cmd->body)) /
-		sizeof(SVGA3dSoTarget);
+	num = (cmd->header.size - sizeof(cmd->body)) / sizeof(SVGA3dSoTarget);
 
 	if (num > SVGA3D_DX_MAX_SOTARGETS) {
-		DRM_ERROR("Invalid DX SO binding.\n");
+		VMW_DEBUG_USER("Invalid DX SO binding.\n");
 		return -EINVAL;
 	}
 
 	for (i = 0; i < num; i++) {
 		ret = vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface,
+					VMW_RES_DIRTY_SET,
 					user_surface_converter,
 					&cmd->targets[i].sid, &res);
 		if (unlikely(ret != 0))
@@ -2645,8 +2478,7 @@ static int vmw_cmd_dx_set_so_targets(struct vmw_private *dev_priv,
 		binding.size = cmd->targets[i].sizeInBytes;
 		binding.slot = i;
 
-		vmw_binding_add(ctx_node->staged, &binding.bi,
-				0, binding.slot);
+		vmw_binding_add(ctx_node->staged, &binding.bi, 0, binding.slot);
 	}
 
 	return 0;
@@ -2656,7 +2488,7 @@ static int vmw_cmd_dx_so_define(struct vmw_private *dev_priv,
 				struct vmw_sw_context *sw_context,
 				SVGA3dCmdHeader *header)
 {
-	struct vmw_ctx_validation_info *ctx_node = sw_context->dx_ctx_node;
+	struct vmw_ctx_validation_info *ctx_node = VMW_GET_CTX_NODE(sw_context);
 	struct vmw_resource *res;
 	/*
 	 * This is based on the fact that all affected define commands have
@@ -2669,10 +2501,8 @@ static int vmw_cmd_dx_so_define(struct vmw_private *dev_priv,
 	enum vmw_so_type so_type;
 	int ret;
 
-	if (unlikely(ctx_node == NULL)) {
-		DRM_ERROR("DX Context not set.\n");
+	if (!ctx_node)
 		return -EINVAL;
-	}
 
 	so_type = vmw_so_cmd_to_type(header->id);
 	res = vmw_context_cotable(ctx_node->ctx, vmw_so_cotables[so_type]);
@@ -2683,8 +2513,8 @@ static int vmw_cmd_dx_so_define(struct vmw_private *dev_priv,
 }
 
 /**
- * vmw_cmd_dx_check_subresource - Validate an
- * SVGA_3D_CMD_DX_[X]_SUBRESOURCE command
+ * vmw_cmd_dx_check_subresource - Validate SVGA_3D_CMD_DX_[X]_SUBRESOURCE
+ * command
  *
  * @dev_priv: Pointer to a device private struct.
  * @sw_context: The software context being used for this batch.
@@ -2714,7 +2544,7 @@ static int vmw_cmd_dx_check_subresource(struct vmw_private *dev_priv,
 	cmd = container_of(header, typeof(*cmd), header);
 
 	return vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface,
-				 user_surface_converter,
+				 VMW_RES_DIRTY_NONE, user_surface_converter,
 				 &cmd->sid, NULL);
 }
 
@@ -2722,32 +2552,30 @@ static int vmw_cmd_dx_cid_check(struct vmw_private *dev_priv,
 				struct vmw_sw_context *sw_context,
 				SVGA3dCmdHeader *header)
 {
-	struct vmw_ctx_validation_info *ctx_node = sw_context->dx_ctx_node;
+	struct vmw_ctx_validation_info *ctx_node = VMW_GET_CTX_NODE(sw_context);
 
-	if (unlikely(ctx_node == NULL)) {
-		DRM_ERROR("DX Context not set.\n");
+	if (!ctx_node)
 		return -EINVAL;
-	}
 
 	return 0;
 }
 
 /**
- * vmw_cmd_dx_view_remove - validate a view remove command and
- * schedule the view resource for removal.
+ * vmw_cmd_dx_view_remove - validate a view remove command and schedule the view
+ * resource for removal.
  *
  * @dev_priv: Pointer to a device private struct.
  * @sw_context: The software context being used for this batch.
  * @header: Pointer to the command header in the command stream.
  *
- * Check that the view exists, and if it was not created using this
- * command batch, conditionally make this command a NOP.
+ * Check that the view exists, and if it was not created using this command
+ * batch, conditionally make this command a NOP.
  */
 static int vmw_cmd_dx_view_remove(struct vmw_private *dev_priv,
 				  struct vmw_sw_context *sw_context,
 				  SVGA3dCmdHeader *header)
 {
-	struct vmw_ctx_validation_info *ctx_node = sw_context->dx_ctx_node;
+	struct vmw_ctx_validation_info *ctx_node = VMW_GET_CTX_NODE(sw_context);
 	struct {
 		SVGA3dCmdHeader header;
 		union vmw_view_destroy body;
@@ -2756,15 +2584,11 @@ static int vmw_cmd_dx_view_remove(struct vmw_private *dev_priv,
 	struct vmw_resource *view;
 	int ret;
 
-	if (!ctx_node) {
-		DRM_ERROR("DX Context not set.\n");
+	if (!ctx_node)
 		return -EINVAL;
-	}
 
-	ret = vmw_view_remove(sw_context->man,
-			      cmd->body.view_id, view_type,
-			      &sw_context->staged_cmd_res,
-			      &view);
+	ret = vmw_view_remove(sw_context->man, cmd->body.view_id, view_type,
+			      &sw_context->staged_cmd_res, &view);
 	if (ret || !view)
 		return ret;
 
@@ -2774,16 +2598,14 @@ static int vmw_cmd_dx_view_remove(struct vmw_private *dev_priv,
 	 * relocation to conditionally make this command a NOP to avoid
 	 * device errors.
 	 */
-	return vmw_resource_relocation_add(sw_context,
-					   view,
+	return vmw_resource_relocation_add(sw_context, view,
 					   vmw_ptr_diff(sw_context->buf_start,
 							&cmd->header.id),
 					   vmw_res_rel_cond_nop);
 }
 
 /**
- * vmw_cmd_dx_define_shader - Validate an SVGA_3D_CMD_DX_DEFINE_SHADER
- * command
+ * vmw_cmd_dx_define_shader - Validate SVGA_3D_CMD_DX_DEFINE_SHADER command
  *
  * @dev_priv: Pointer to a device private struct.
  * @sw_context: The software context being used for this batch.
@@ -2793,18 +2615,14 @@ static int vmw_cmd_dx_define_shader(struct vmw_private *dev_priv,
 				    struct vmw_sw_context *sw_context,
 				    SVGA3dCmdHeader *header)
 {
-	struct vmw_ctx_validation_info *ctx_node = sw_context->dx_ctx_node;
+	struct vmw_ctx_validation_info *ctx_node = VMW_GET_CTX_NODE(sw_context);
 	struct vmw_resource *res;
-	struct {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdDXDefineShader body;
-	} *cmd = container_of(header, typeof(*cmd), header);
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdDXDefineShader) =
+		container_of(header, typeof(*cmd), header);
 	int ret;
 
-	if (!ctx_node) {
-		DRM_ERROR("DX Context not set.\n");
+	if (!ctx_node)
 		return -EINVAL;
-	}
 
 	res = vmw_context_cotable(ctx_node->ctx, SVGA_COTABLE_DXSHADER);
 	ret = vmw_cotable_notify(res, cmd->body.shaderId);
@@ -2817,8 +2635,7 @@ static int vmw_cmd_dx_define_shader(struct vmw_private *dev_priv,
 }
 
 /**
- * vmw_cmd_dx_destroy_shader - Validate an SVGA_3D_CMD_DX_DESTROY_SHADER
- * command
+ * vmw_cmd_dx_destroy_shader - Validate SVGA_3D_CMD_DX_DESTROY_SHADER command
  *
  * @dev_priv: Pointer to a device private struct.
  * @sw_context: The software context being used for this batch.
@@ -2828,29 +2645,22 @@ static int vmw_cmd_dx_destroy_shader(struct vmw_private *dev_priv,
 				     struct vmw_sw_context *sw_context,
 				     SVGA3dCmdHeader *header)
 {
-	struct vmw_ctx_validation_info *ctx_node = sw_context->dx_ctx_node;
-	struct {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdDXDestroyShader body;
-	} *cmd = container_of(header, typeof(*cmd), header);
+	struct vmw_ctx_validation_info *ctx_node = VMW_GET_CTX_NODE(sw_context);
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdDXDestroyShader) =
+		container_of(header, typeof(*cmd), header);
 	int ret;
 
-	if (!ctx_node) {
-		DRM_ERROR("DX Context not set.\n");
+	if (!ctx_node)
 		return -EINVAL;
-	}
 
 	ret = vmw_shader_remove(sw_context->man, cmd->body.shaderId, 0,
 				&sw_context->staged_cmd_res);
-	if (ret)
-		DRM_ERROR("Could not find shader to remove.\n");
 
 	return ret;
 }
 
 /**
- * vmw_cmd_dx_bind_shader - Validate an SVGA_3D_CMD_DX_BIND_SHADER
- * command
+ * vmw_cmd_dx_bind_shader - Validate SVGA_3D_CMD_DX_BIND_SHADER command
  *
  * @dev_priv: Pointer to a device private struct.
  * @sw_context: The software context being used for this batch.
@@ -2862,36 +2672,37 @@ static int vmw_cmd_dx_bind_shader(struct vmw_private *dev_priv,
 {
 	struct vmw_resource *ctx;
 	struct vmw_resource *res;
-	struct {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdDXBindShader body;
-	} *cmd = container_of(header, typeof(*cmd), header);
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdDXBindShader) =
+		container_of(header, typeof(*cmd), header);
 	int ret;
 
 	if (cmd->body.cid != SVGA3D_INVALID_ID) {
 		ret = vmw_cmd_res_check(dev_priv, sw_context, vmw_res_context,
-					user_context_converter,
-					&cmd->body.cid, &ctx);
+					VMW_RES_DIRTY_SET,
+					user_context_converter, &cmd->body.cid,
+					&ctx);
 		if (ret)
 			return ret;
 	} else {
-		if (!sw_context->dx_ctx_node) {
-			DRM_ERROR("DX Context not set.\n");
+		struct vmw_ctx_validation_info *ctx_node =
+			VMW_GET_CTX_NODE(sw_context);
+
+		if (!ctx_node)
 			return -EINVAL;
-		}
-		ctx = sw_context->dx_ctx_node->ctx;
+
+		ctx = ctx_node->ctx;
 	}
 
-	res = vmw_shader_lookup(vmw_context_res_man(ctx),
-				cmd->body.shid, 0);
+	res = vmw_shader_lookup(vmw_context_res_man(ctx), cmd->body.shid, 0);
 	if (IS_ERR(res)) {
-		DRM_ERROR("Could not find shader to bind.\n");
+		VMW_DEBUG_USER("Could not find shader to bind.\n");
 		return PTR_ERR(res);
 	}
 
-	ret = vmw_execbuf_res_noctx_val_add(sw_context, res);
+	ret = vmw_execbuf_res_noctx_val_add(sw_context, res,
+					    VMW_RES_DIRTY_NONE);
 	if (ret) {
-		DRM_ERROR("Error creating resource validation node.\n");
+		VMW_DEBUG_USER("Error creating resource validation node.\n");
 		return ret;
 	}
 
@@ -2901,7 +2712,7 @@ static int vmw_cmd_dx_bind_shader(struct vmw_private *dev_priv,
 }
 
 /**
- * vmw_cmd_dx_genmips - Validate an SVGA_3D_CMD_DX_GENMIPS command
+ * vmw_cmd_dx_genmips - Validate SVGA_3D_CMD_DX_GENMIPS command
  *
  * @dev_priv: Pointer to a device private struct.
  * @sw_context: The software context being used for this batch.
@@ -2911,18 +2722,16 @@ static int vmw_cmd_dx_genmips(struct vmw_private *dev_priv,
 			      struct vmw_sw_context *sw_context,
 			      SVGA3dCmdHeader *header)
 {
-	struct {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdDXGenMips body;
-	} *cmd = container_of(header, typeof(*cmd), header);
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdDXGenMips) =
+		container_of(header, typeof(*cmd), header);
 
 	return PTR_RET(vmw_view_id_val_add(sw_context, vmw_view_sr,
 					   cmd->body.shaderResourceViewId));
 }
 
 /**
- * vmw_cmd_dx_transfer_from_buffer -
- * Validate an SVGA_3D_CMD_DX_TRANSFER_FROM_BUFFER command
+ * vmw_cmd_dx_transfer_from_buffer - Validate
+ * SVGA_3D_CMD_DX_TRANSFER_FROM_BUFFER command
  *
  * @dev_priv: Pointer to a device private struct.
  * @sw_context: The software context being used for this batch.
@@ -2932,26 +2741,23 @@ static int vmw_cmd_dx_transfer_from_buffer(struct vmw_private *dev_priv,
 					   struct vmw_sw_context *sw_context,
 					   SVGA3dCmdHeader *header)
 {
-	struct {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdDXTransferFromBuffer body;
-	} *cmd = container_of(header, typeof(*cmd), header);
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdDXTransferFromBuffer) =
+		container_of(header, typeof(*cmd), header);
 	int ret;
 
 	ret = vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface,
-				user_surface_converter,
+				VMW_RES_DIRTY_NONE, user_surface_converter,
 				&cmd->body.srcSid, NULL);
 	if (ret != 0)
 		return ret;
 
 	return vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface,
-				 user_surface_converter,
+				 VMW_RES_DIRTY_SET, user_surface_converter,
 				 &cmd->body.destSid, NULL);
 }
 
 /**
- * vmw_cmd_intra_surface_copy -
- * Validate an SVGA_3D_CMD_INTRA_SURFACE_COPY command
+ * vmw_cmd_intra_surface_copy - Validate SVGA_3D_CMD_INTRA_SURFACE_COPY command
  *
  * @dev_priv: Pointer to a device private struct.
  * @sw_context: The software context being used for this batch.
@@ -2961,20 +2767,17 @@ static int vmw_cmd_intra_surface_copy(struct vmw_private *dev_priv,
 					   struct vmw_sw_context *sw_context,
 					   SVGA3dCmdHeader *header)
 {
-	struct {
-		SVGA3dCmdHeader header;
-		SVGA3dCmdIntraSurfaceCopy body;
-	} *cmd = container_of(header, typeof(*cmd), header);
+	VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdIntraSurfaceCopy) =
+		container_of(header, typeof(*cmd), header);
 
 	if (!(dev_priv->capabilities2 & SVGA_CAP2_INTRA_SURFACE_COPY))
 		return -EINVAL;
 
 	return vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface,
-				user_surface_converter,
-				&cmd->body.surface.sid, NULL);
+				 VMW_RES_DIRTY_SET, user_surface_converter,
+				 &cmd->body.surface.sid, NULL);
 }
 
-
 static int vmw_cmd_check_not_3d(struct vmw_private *dev_priv,
 				struct vmw_sw_context *sw_context,
 				void *buf, uint32_t *size)
@@ -2997,18 +2800,18 @@ static int vmw_cmd_check_not_3d(struct vmw_private *dev_priv,
 		*size = sizeof(uint32_t) + sizeof(SVGAFifoCmdBlitGMRFBToScreen);
 		break;
 	default:
-		DRM_ERROR("Unsupported SVGA command: %u.\n", cmd_id);
+		VMW_DEBUG_USER("Unsupported SVGA command: %u.\n", cmd_id);
 		return -EINVAL;
 	}
 
 	if (*size > size_remaining) {
-		DRM_ERROR("Invalid SVGA command (size mismatch):"
-			  " %u.\n", cmd_id);
+		VMW_DEBUG_USER("Invalid SVGA command (size mismatch): %u.\n",
+			       cmd_id);
 		return -EINVAL;
 	}
 
 	if (unlikely(!sw_context->kernel)) {
-		DRM_ERROR("Kernel only SVGA command: %u.\n", cmd_id);
+		VMW_DEBUG_USER("Kernel only SVGA command: %u.\n", cmd_id);
 		return -EPERM;
 	}
 
@@ -3196,9 +2999,7 @@ static const struct vmw_cmd_entry vmw_cmd_entries[SVGA_3D_CMD_MAX] = {
 	VMW_CMD_DEF(SVGA_3D_CMD_DEFINE_GB_SURFACE_V2, &vmw_cmd_invalid,
 		    false, false, true),
 
-	/*
-	 * DX commands
-	 */
+	/* SM commands */
 	VMW_CMD_DEF(SVGA_3D_CMD_DX_DEFINE_CONTEXT, &vmw_cmd_invalid,
 		    false, false, true),
 	VMW_CMD_DEF(SVGA_3D_CMD_DX_DESTROY_CONTEXT, &vmw_cmd_invalid,
@@ -3380,8 +3181,8 @@ bool vmw_cmd_describe(const void *buf, u32 *size, char const **cmd)
 }
 
 static int vmw_cmd_check(struct vmw_private *dev_priv,
-			 struct vmw_sw_context *sw_context,
-			 void *buf, uint32_t *size)
+			 struct vmw_sw_context *sw_context, void *buf,
+			 uint32_t *size)
 {
 	uint32_t cmd_id;
 	uint32_t size_remaining = *size;
@@ -3420,31 +3221,33 @@ static int vmw_cmd_check(struct vmw_private *dev_priv,
 		goto out_new;
 
 	ret = entry->func(dev_priv, sw_context, header);
-	if (unlikely(ret != 0))
-		goto out_invalid;
+	if (unlikely(ret != 0)) {
+		VMW_DEBUG_USER("SVGA3D command: %d failed with error %d\n",
+			       cmd_id + SVGA_3D_CMD_BASE, ret);
+		return ret;
+	}
 
 	return 0;
 out_invalid:
-	DRM_ERROR("Invalid SVGA3D command: %d\n",
-		  cmd_id + SVGA_3D_CMD_BASE);
+	VMW_DEBUG_USER("Invalid SVGA3D command: %d\n",
+		       cmd_id + SVGA_3D_CMD_BASE);
 	return -EINVAL;
 out_privileged:
-	DRM_ERROR("Privileged SVGA3D command: %d\n",
-		  cmd_id + SVGA_3D_CMD_BASE);
+	VMW_DEBUG_USER("Privileged SVGA3D command: %d\n",
+		       cmd_id + SVGA_3D_CMD_BASE);
 	return -EPERM;
 out_old:
-	DRM_ERROR("Deprecated (disallowed) SVGA3D command: %d\n",
-		  cmd_id + SVGA_3D_CMD_BASE);
+	VMW_DEBUG_USER("Deprecated (disallowed) SVGA3D command: %d\n",
+		       cmd_id + SVGA_3D_CMD_BASE);
 	return -EINVAL;
 out_new:
-	DRM_ERROR("SVGA3D command: %d not supported by virtual hardware.\n",
-		  cmd_id + SVGA_3D_CMD_BASE);
+	VMW_DEBUG_USER("SVGA3D command: %d not supported by virtual device.\n",
+		       cmd_id + SVGA_3D_CMD_BASE);
 	return -EINVAL;
 }
 
 static int vmw_cmd_check_all(struct vmw_private *dev_priv,
-			     struct vmw_sw_context *sw_context,
-			     void *buf,
+			     struct vmw_sw_context *sw_context, void *buf,
 			     uint32_t size)
 {
 	int32_t cur_size = size;
@@ -3462,7 +3265,7 @@ static int vmw_cmd_check_all(struct vmw_private *dev_priv,
 	}
 
 	if (unlikely(cur_size != 0)) {
-		DRM_ERROR("Command verifier out of sync.\n");
+		VMW_DEBUG_USER("Command verifier out of sync.\n");
 		return -EINVAL;
 	}
 
@@ -3472,7 +3275,6 @@ static int vmw_cmd_check_all(struct vmw_private *dev_priv,
 static void vmw_free_relocations(struct vmw_sw_context *sw_context)
 {
 	/* Memory is validation context memory, so no need to free it */
-
 	INIT_LIST_HEAD(&sw_context->bo_relocations);
 }
 
@@ -3520,7 +3322,7 @@ static int vmw_resize_cmd_bounce(struct vmw_sw_context *sw_context,
 	sw_context->cmd_bounce = vmalloc(sw_context->cmd_bounce_size);
 
 	if (sw_context->cmd_bounce == NULL) {
-		DRM_ERROR("Failed to allocate command bounce buffer.\n");
+		VMW_DEBUG_USER("Failed to allocate command bounce buffer.\n");
 		sw_context->cmd_bounce_size = 0;
 		return -ENOMEM;
 	}
@@ -3535,8 +3337,8 @@ static int vmw_resize_cmd_bounce(struct vmw_sw_context *sw_context,
  * If this fails for some reason, We sync the fifo and return NULL.
  * It is then safe to fence buffers with a NULL pointer.
  *
- * If @p_handle is not NULL @file_priv must also not be NULL. Creates
- * a userspace handle if @p_handle is not NULL, otherwise not.
+ * If @p_handle is not NULL @file_priv must also not be NULL. Creates a
+ * userspace handle if @p_handle is not NULL, otherwise not.
  */
 
 int vmw_execbuf_fence_commands(struct drm_file *file_priv,
@@ -3553,7 +3355,7 @@ int vmw_execbuf_fence_commands(struct drm_file *file_priv,
 
 	ret = vmw_fifo_send_fence(dev_priv, &sequence);
 	if (unlikely(ret != 0)) {
-		DRM_ERROR("Fence submission error. Syncing.\n");
+		VMW_DEBUG_USER("Fence submission error. Syncing.\n");
 		synced = true;
 	}
 
@@ -3564,9 +3366,8 @@ int vmw_execbuf_fence_commands(struct drm_file *file_priv,
 		ret = vmw_fence_create(dev_priv->fman, sequence, p_fence);
 
 	if (unlikely(ret != 0 && !synced)) {
-		(void) vmw_fallback_wait(dev_priv, false, false,
-					 sequence, false,
-					 VMW_FENCE_WAIT_TIMEOUT);
+		(void) vmw_fallback_wait(dev_priv, false, false, sequence,
+					 false, VMW_FENCE_WAIT_TIMEOUT);
 		*p_fence = NULL;
 	}
 
@@ -3574,36 +3375,32 @@ int vmw_execbuf_fence_commands(struct drm_file *file_priv,
 }
 
 /**
- * vmw_execbuf_copy_fence_user - copy fence object information to
- * user-space.
+ * vmw_execbuf_copy_fence_user - copy fence object information to user-space.
  *
  * @dev_priv: Pointer to a vmw_private struct.
  * @vmw_fp: Pointer to the struct vmw_fpriv representing the calling file.
  * @ret: Return value from fence object creation.
- * @user_fence_rep: User space address of a struct drm_vmw_fence_rep to
- * which the information should be copied.
+ * @user_fence_rep: User space address of a struct drm_vmw_fence_rep to which
+ * the information should be copied.
  * @fence: Pointer to the fenc object.
  * @fence_handle: User-space fence handle.
  * @out_fence_fd: exported file descriptor for the fence.  -1 if not used
  * @sync_file:  Only used to clean up in case of an error in this function.
  *
- * This function copies fence information to user-space. If copying fails,
- * The user-space struct drm_vmw_fence_rep::error member is hopefully
- * left untouched, and if it's preloaded with an -EFAULT by user-space,
- * the error will hopefully be detected.
- * Also if copying fails, user-space will be unable to signal the fence
- * object so we wait for it immediately, and then unreference the
- * user-space reference.
+ * This function copies fence information to user-space. If copying fails, the
+ * user-space struct drm_vmw_fence_rep::error member is hopefully left
+ * untouched, and if it's preloaded with an -EFAULT by user-space, the error
+ * will hopefully be detected.
+ *
+ * Also if copying fails, user-space will be unable to signal the fence object
+ * so we wait for it immediately, and then unreference the user-space reference.
  */
 void
 vmw_execbuf_copy_fence_user(struct vmw_private *dev_priv,
-			    struct vmw_fpriv *vmw_fp,
-			    int ret,
+			    struct vmw_fpriv *vmw_fp, int ret,
 			    struct drm_vmw_fence_rep __user *user_fence_rep,
-			    struct vmw_fence_obj *fence,
-			    uint32_t fence_handle,
-			    int32_t out_fence_fd,
-			    struct sync_file *sync_file)
+			    struct vmw_fence_obj *fence, uint32_t fence_handle,
+			    int32_t out_fence_fd, struct sync_file *sync_file)
 {
 	struct drm_vmw_fence_rep fence_rep;
 
@@ -3624,16 +3421,16 @@ vmw_execbuf_copy_fence_user(struct vmw_private *dev_priv,
 	}
 
 	/*
-	 * copy_to_user errors will be detected by user space not
-	 * seeing fence_rep::error filled in. Typically
-	 * user-space would have pre-set that member to -EFAULT.
+	 * copy_to_user errors will be detected by user space not seeing
+	 * fence_rep::error filled in. Typically user-space would have pre-set
+	 * that member to -EFAULT.
 	 */
 	ret = copy_to_user(user_fence_rep, &fence_rep,
 			   sizeof(fence_rep));
 
 	/*
-	 * User-space lost the fence object. We need to sync
-	 * and unreference the handle.
+	 * User-space lost the fence object. We need to sync and unreference the
+	 * handle.
 	 */
 	if (unlikely(ret != 0) && (fence_rep.error == 0)) {
 		if (sync_file)
@@ -3644,42 +3441,39 @@ vmw_execbuf_copy_fence_user(struct vmw_private *dev_priv,
 			fence_rep.fd = -1;
 		}
 
-		ttm_ref_object_base_unref(vmw_fp->tfile,
-					  fence_handle, TTM_REF_USAGE);
-		DRM_ERROR("Fence copy error. Syncing.\n");
+		ttm_ref_object_base_unref(vmw_fp->tfile, fence_handle,
+					  TTM_REF_USAGE);
+		VMW_DEBUG_USER("Fence copy error. Syncing.\n");
 		(void) vmw_fence_obj_wait(fence, false, false,
 					  VMW_FENCE_WAIT_TIMEOUT);
 	}
 }
 
 /**
- * vmw_execbuf_submit_fifo - Patch a command batch and submit it using
- * the fifo.
+ * vmw_execbuf_submit_fifo - Patch a command batch and submit it using the fifo.
  *
  * @dev_priv: Pointer to a device private structure.
  * @kernel_commands: Pointer to the unpatched command batch.
  * @command_size: Size of the unpatched command batch.
  * @sw_context: Structure holding the relocation lists.
  *
- * Side effects: If this function returns 0, then the command batch
- * pointed to by @kernel_commands will have been modified.
+ * Side effects: If this function returns 0, then the command batch pointed to
+ * by @kernel_commands will have been modified.
  */
 static int vmw_execbuf_submit_fifo(struct vmw_private *dev_priv,
-				   void *kernel_commands,
-				   u32 command_size,
+				   void *kernel_commands, u32 command_size,
 				   struct vmw_sw_context *sw_context)
 {
 	void *cmd;
 
 	if (sw_context->dx_ctx_node)
-		cmd = vmw_fifo_reserve_dx(dev_priv, command_size,
+		cmd = VMW_FIFO_RESERVE_DX(dev_priv, command_size,
 					  sw_context->dx_ctx_node->ctx->id);
 	else
-		cmd = vmw_fifo_reserve(dev_priv, command_size);
-	if (!cmd) {
-		DRM_ERROR("Failed reserving fifo space for commands.\n");
+		cmd = VMW_FIFO_RESERVE(dev_priv, command_size);
+
+	if (!cmd)
 		return -ENOMEM;
-	}
 
 	vmw_apply_relocations(sw_context);
 	memcpy(cmd, kernel_commands, command_size);
@@ -3691,16 +3485,16 @@ static int vmw_execbuf_submit_fifo(struct vmw_private *dev_priv,
 }
 
 /**
- * vmw_execbuf_submit_cmdbuf - Patch a command batch and submit it using
- * the command buffer manager.
+ * vmw_execbuf_submit_cmdbuf - Patch a command batch and submit it using the
+ * command buffer manager.
  *
  * @dev_priv: Pointer to a device private structure.
  * @header: Opaque handle to the command buffer allocation.
  * @command_size: Size of the unpatched command batch.
  * @sw_context: Structure holding the relocation lists.
  *
- * Side effects: If this function returns 0, then the command buffer
- * represented by @header will have been modified.
+ * Side effects: If this function returns 0, then the command buffer represented
+ * by @header will have been modified.
  */
 static int vmw_execbuf_submit_cmdbuf(struct vmw_private *dev_priv,
 				     struct vmw_cmdbuf_header *header,
@@ -3709,8 +3503,8 @@ static int vmw_execbuf_submit_cmdbuf(struct vmw_private *dev_priv,
 {
 	u32 id = ((sw_context->dx_ctx_node) ? sw_context->dx_ctx_node->ctx->id :
 		  SVGA3D_INVALID_ID);
-	void *cmd = vmw_cmdbuf_reserve(dev_priv->cman, command_size,
-				       id, false, header);
+	void *cmd = vmw_cmdbuf_reserve(dev_priv->cman, command_size, id, false,
+				       header);
 
 	vmw_apply_relocations(sw_context);
 	vmw_resource_relocations_apply(cmd, &sw_context->res_relocations);
@@ -3730,22 +3524,23 @@ static int vmw_execbuf_submit_cmdbuf(struct vmw_private *dev_priv,
  * @header: Out parameter returning the opaque pointer to the command buffer.
  *
  * This function checks whether we can use the command buffer manager for
- * submission and if so, creates a command buffer of suitable size and
- * copies the user data into that buffer.
+ * submission and if so, creates a command buffer of suitable size and copies
+ * the user data into that buffer.
  *
  * On successful return, the function returns a pointer to the data in the
  * command buffer and *@header is set to non-NULL.
- * If command buffers could not be used, the function will return the value
- * of @kernel_commands on function call. That value may be NULL. In that case,
- * the value of *@header will be set to NULL.
+ *
+ * If command buffers could not be used, the function will return the value of
+ * @kernel_commands on function call. That value may be NULL. In that case, the
+ * value of *@header will be set to NULL.
+ *
  * If an error is encountered, the function will return a pointer error value.
  * If the function is interrupted by a signal while sleeping, it will return
  * -ERESTARTSYS casted to a pointer error value.
  */
 static void *vmw_execbuf_cmdbuf(struct vmw_private *dev_priv,
 				void __user *user_commands,
-				void *kernel_commands,
-				u32 command_size,
+				void *kernel_commands, u32 command_size,
 				struct vmw_cmdbuf_header **header)
 {
 	size_t cmdbuf_size;
@@ -3753,7 +3548,7 @@ static void *vmw_execbuf_cmdbuf(struct vmw_private *dev_priv,
 
 	*header = NULL;
 	if (command_size > SVGA_CB_MAX_SIZE) {
-		DRM_ERROR("Command buffer is too large.\n");
+		VMW_DEBUG_USER("Command buffer is too large.\n");
 		return ERR_PTR(-EINVAL);
 	}
 
@@ -3763,15 +3558,14 @@ static void *vmw_execbuf_cmdbuf(struct vmw_private *dev_priv,
 	/* If possible, add a little space for fencing. */
 	cmdbuf_size = command_size + 512;
 	cmdbuf_size = min_t(size_t, cmdbuf_size, SVGA_CB_MAX_SIZE);
-	kernel_commands = vmw_cmdbuf_alloc(dev_priv->cman, cmdbuf_size,
-					   true, header);
+	kernel_commands = vmw_cmdbuf_alloc(dev_priv->cman, cmdbuf_size, true,
+					   header);
 	if (IS_ERR(kernel_commands))
 		return kernel_commands;
 
-	ret = copy_from_user(kernel_commands, user_commands,
-			     command_size);
+	ret = copy_from_user(kernel_commands, user_commands, command_size);
 	if (ret) {
-		DRM_ERROR("Failed copying commands.\n");
+		VMW_DEBUG_USER("Failed copying commands.\n");
 		vmw_cmdbuf_header_free(*header);
 		*header = NULL;
 		return ERR_PTR(-EFAULT);
@@ -3799,13 +3593,13 @@ static int vmw_execbuf_tie_context(struct vmw_private *dev_priv,
 	res = vmw_user_resource_noref_lookup_handle
 		(dev_priv, sw_context->fp->tfile, handle,
 		 user_context_converter);
-	if (unlikely(IS_ERR(res))) {
-		DRM_ERROR("Could not find or user DX context 0x%08x.\n",
-			  (unsigned) handle);
+	if (IS_ERR(res)) {
+		VMW_DEBUG_USER("Could not find or user DX context 0x%08x.\n",
+			       (unsigned int) handle);
 		return PTR_ERR(res);
 	}
 
-	ret = vmw_execbuf_res_noref_val_add(sw_context, res);
+	ret = vmw_execbuf_res_noref_val_add(sw_context, res, VMW_RES_DIRTY_SET);
 	if (unlikely(ret != 0))
 		return ret;
 
@@ -3817,19 +3611,16 @@ static int vmw_execbuf_tie_context(struct vmw_private *dev_priv,
 
 int vmw_execbuf_process(struct drm_file *file_priv,
 			struct vmw_private *dev_priv,
-			void __user *user_commands,
-			void *kernel_commands,
-			uint32_t command_size,
-			uint64_t throttle_us,
+			void __user *user_commands, void *kernel_commands,
+			uint32_t command_size, uint64_t throttle_us,
 			uint32_t dx_context_handle,
 			struct drm_vmw_fence_rep __user *user_fence_rep,
-			struct vmw_fence_obj **out_fence,
-			uint32_t flags)
+			struct vmw_fence_obj **out_fence, uint32_t flags)
 {
 	struct vmw_sw_context *sw_context = &dev_priv->ctx;
 	struct vmw_fence_obj *fence = NULL;
 	struct vmw_cmdbuf_header *header;
-	uint32_t handle;
+	uint32_t handle = 0;
 	int ret;
 	int32_t out_fence_fd = -1;
 	struct sync_file *sync_file = NULL;
@@ -3840,7 +3631,7 @@ int vmw_execbuf_process(struct drm_file *file_priv,
 	if (flags & DRM_VMW_EXECBUF_FLAG_EXPORT_FENCE_FD) {
 		out_fence_fd = get_unused_fd_flags(O_CLOEXEC);
 		if (out_fence_fd < 0) {
-			DRM_ERROR("Failed to get a fence file descriptor.\n");
+			VMW_DEBUG_USER("Failed to get a fence fd.\n");
 			return out_fence_fd;
 		}
 	}
@@ -3873,18 +3664,18 @@ int vmw_execbuf_process(struct drm_file *file_priv,
 		if (unlikely(ret != 0))
 			goto out_unlock;
 
-
-		ret = copy_from_user(sw_context->cmd_bounce,
-				     user_commands, command_size);
-
+		ret = copy_from_user(sw_context->cmd_bounce, user_commands,
+				     command_size);
 		if (unlikely(ret != 0)) {
 			ret = -EFAULT;
-			DRM_ERROR("Failed copying commands.\n");
+			VMW_DEBUG_USER("Failed copying commands.\n");
 			goto out_unlock;
 		}
+
 		kernel_commands = sw_context->cmd_bounce;
-	} else if (!header)
+	} else if (!header) {
 		sw_context->kernel = true;
+	}
 
 	sw_context->fp = vmw_fpriv(file_priv);
 	INIT_LIST_HEAD(&sw_context->ctx_list);
@@ -3897,6 +3688,7 @@ int vmw_execbuf_process(struct drm_file *file_priv,
 	memset(sw_context->res_cache, 0, sizeof(sw_context->res_cache));
 	INIT_LIST_HEAD(&sw_context->res_relocations);
 	INIT_LIST_HEAD(&sw_context->bo_relocations);
+
 	if (sw_context->staged_bindings)
 		vmw_binding_state_reset(sw_context->staged_bindings);
 
@@ -3904,8 +3696,10 @@ int vmw_execbuf_process(struct drm_file *file_priv,
 		ret = drm_ht_create(&sw_context->res_ht, VMW_RES_HT_ORDER);
 		if (unlikely(ret != 0))
 			goto out_unlock;
+
 		sw_context->res_ht_initialized = true;
 	}
+
 	INIT_LIST_HEAD(&sw_context->staged_cmd_res);
 	sw_context->ctx = &val_ctx;
 	ret = vmw_execbuf_tie_context(dev_priv, sw_context, dx_context_handle);
@@ -3932,6 +3726,7 @@ int vmw_execbuf_process(struct drm_file *file_priv,
 	ret = vmw_validation_res_validate(&val_ctx, true);
 	if (unlikely(ret != 0))
 		goto out_err;
+
 	vmw_validation_drop_ht(&val_ctx);
 
 	ret = mutex_lock_interruptible(&dev_priv->binding_mutex);
@@ -3959,17 +3754,15 @@ int vmw_execbuf_process(struct drm_file *file_priv,
 		goto out_err;
 
 	vmw_query_bo_switch_commit(dev_priv, sw_context);
-	ret = vmw_execbuf_fence_commands(file_priv, dev_priv,
-					 &fence,
+	ret = vmw_execbuf_fence_commands(file_priv, dev_priv, &fence,
 					 (user_fence_rep) ? &handle : NULL);
 	/*
 	 * This error is harmless, because if fence submission fails,
 	 * vmw_fifo_send_fence will sync. The error will be propagated to
 	 * user-space in @fence_rep
 	 */
-
 	if (ret != 0)
-		DRM_ERROR("Fence submission error. Syncing.\n");
+		VMW_DEBUG_USER("Fence submission error. Syncing.\n");
 
 	vmw_execbuf_bindings_commit(sw_context, false);
 	vmw_bind_dx_query_mob(sw_context);
@@ -3977,21 +3770,19 @@ int vmw_execbuf_process(struct drm_file *file_priv,
 
 	vmw_validation_bo_fence(sw_context->ctx, fence);
 
-	if (unlikely(dev_priv->pinned_bo != NULL &&
-		     !dev_priv->query_cid_valid))
+	if (unlikely(dev_priv->pinned_bo != NULL && !dev_priv->query_cid_valid))
 		__vmw_execbuf_release_pinned_bo(dev_priv, fence);
 
 	/*
-	 * If anything fails here, give up trying to export the fence
-	 * and do a sync since the user mode will not be able to sync
-	 * the fence itself.  This ensures we are still functionally
-	 * correct.
+	 * If anything fails here, give up trying to export the fence and do a
+	 * sync since the user mode will not be able to sync the fence itself.
+	 * This ensures we are still functionally correct.
 	 */
 	if (flags & DRM_VMW_EXECBUF_FLAG_EXPORT_FENCE_FD) {
 
 		sync_file = sync_file_create(&fence->base);
 		if (!sync_file) {
-			DRM_ERROR("Unable to create sync file for fence\n");
+			VMW_DEBUG_USER("Sync file create failed for fence\n");
 			put_unused_fd(out_fence_fd);
 			out_fence_fd = -1;
 
@@ -4004,8 +3795,8 @@ int vmw_execbuf_process(struct drm_file *file_priv,
 	}
 
 	vmw_execbuf_copy_fence_user(dev_priv, vmw_fpriv(file_priv), ret,
-				    user_fence_rep, fence, handle,
-				    out_fence_fd, sync_file);
+				    user_fence_rep, fence, handle, out_fence_fd,
+				    sync_file);
 
 	/* Don't unreference when handing fence out */
 	if (unlikely(out_fence != NULL)) {
@@ -4019,8 +3810,8 @@ int vmw_execbuf_process(struct drm_file *file_priv,
 	mutex_unlock(&dev_priv->cmdbuf_mutex);
 
 	/*
-	 * Unreference resources outside of the cmdbuf_mutex to
-	 * avoid deadlocks in resource destruction paths.
+	 * Unreference resources outside of the cmdbuf_mutex to avoid deadlocks
+	 * in resource destruction paths.
 	 */
 	vmw_validation_unref_lists(&val_ctx);
 
@@ -4035,8 +3826,7 @@ out_err_nores:
 	vmw_validation_res_unreserve(&val_ctx, true);
 	vmw_resource_relocations_free(&sw_context->res_relocations);
 	vmw_free_relocations(sw_context);
-	if (unlikely(dev_priv->pinned_bo != NULL &&
-		     !dev_priv->query_cid_valid))
+	if (unlikely(dev_priv->pinned_bo != NULL && !dev_priv->query_cid_valid))
 		__vmw_execbuf_release_pinned_bo(dev_priv, NULL);
 out_unlock:
 	vmw_cmdbuf_res_revert(&sw_context->staged_cmd_res);
@@ -4045,8 +3835,8 @@ out_unlock:
 	mutex_unlock(&dev_priv->cmdbuf_mutex);
 
 	/*
-	 * Unreference resources outside of the cmdbuf_mutex to
-	 * avoid deadlocks in resource destruction paths.
+	 * Unreference resources outside of the cmdbuf_mutex to avoid deadlocks
+	 * in resource destruction paths.
 	 */
 	vmw_validation_unref_lists(&val_ctx);
 out_free_header:
@@ -4064,13 +3854,13 @@ out_free_fence_fd:
  *
  * @dev_priv: The device private structure.
  *
- * This function is called to idle the fifo and unpin the query buffer
- * if the normal way to do this hits an error, which should typically be
- * extremely rare.
+ * This function is called to idle the fifo and unpin the query buffer if the
+ * normal way to do this hits an error, which should typically be extremely
+ * rare.
  */
 static void vmw_execbuf_unpin_panic(struct vmw_private *dev_priv)
 {
-	DRM_ERROR("Can't unpin query buffer. Trying to recover.\n");
+	VMW_DEBUG_USER("Can't unpin query buffer. Trying to recover.\n");
 
 	(void) vmw_fallback_wait(dev_priv, false, true, 0, false, 10*HZ);
 	vmw_bo_pin_reserved(dev_priv->pinned_bo, false);
@@ -4082,28 +3872,27 @@ static void vmw_execbuf_unpin_panic(struct vmw_private *dev_priv)
 
 
 /**
- * __vmw_execbuf_release_pinned_bo - Flush queries and unpin the pinned
- * query bo.
+ * __vmw_execbuf_release_pinned_bo - Flush queries and unpin the pinned query
+ * bo.
  *
  * @dev_priv: The device private structure.
- * @fence: If non-NULL should point to a struct vmw_fence_obj issued
- * _after_ a query barrier that flushes all queries touching the current
- * buffer pointed to by @dev_priv->pinned_bo
+ * @fence: If non-NULL should point to a struct vmw_fence_obj issued _after_ a
+ * query barrier that flushes all queries touching the current buffer pointed to
+ * by @dev_priv->pinned_bo
  *
- * This function should be used to unpin the pinned query bo, or
- * as a query barrier when we need to make sure that all queries have
- * finished before the next fifo command. (For example on hardware
- * context destructions where the hardware may otherwise leak unfinished
- * queries).
+ * This function should be used to unpin the pinned query bo, or as a query
+ * barrier when we need to make sure that all queries have finished before the
+ * next fifo command. (For example on hardware context destructions where the
+ * hardware may otherwise leak unfinished queries).
  *
- * This function does not return any failure codes, but make attempts
- * to do safe unpinning in case of errors.
+ * This function does not return any failure codes, but make attempts to do safe
+ * unpinning in case of errors.
  *
- * The function will synchronize on the previous query barrier, and will
- * thus not finish until that barrier has executed.
+ * The function will synchronize on the previous query barrier, and will thus
+ * not finish until that barrier has executed.
  *
- * the @dev_priv->cmdbuf_mutex needs to be held by the current thread
- * before calling this function.
+ * the @dev_priv->cmdbuf_mutex needs to be held by the current thread before
+ * calling this function.
  */
 void __vmw_execbuf_release_pinned_bo(struct vmw_private *dev_priv,
 				     struct vmw_fence_obj *fence)
@@ -4153,35 +3942,32 @@ void __vmw_execbuf_release_pinned_bo(struct vmw_private *dev_priv,
 
 	vmw_validation_unref_lists(&val_ctx);
 	vmw_bo_unreference(&dev_priv->pinned_bo);
+
 out_unlock:
 	return;
-
 out_no_emit:
 	vmw_validation_bo_backoff(&val_ctx);
 out_no_reserve:
 	vmw_validation_unref_lists(&val_ctx);
 	vmw_execbuf_unpin_panic(dev_priv);
 	vmw_bo_unreference(&dev_priv->pinned_bo);
-
 }
 
 /**
- * vmw_execbuf_release_pinned_bo - Flush queries and unpin the pinned
- * query bo.
+ * vmw_execbuf_release_pinned_bo - Flush queries and unpin the pinned query bo.
  *
  * @dev_priv: The device private structure.
  *
- * This function should be used to unpin the pinned query bo, or
- * as a query barrier when we need to make sure that all queries have
- * finished before the next fifo command. (For example on hardware
- * context destructions where the hardware may otherwise leak unfinished
- * queries).
+ * This function should be used to unpin the pinned query bo, or as a query
+ * barrier when we need to make sure that all queries have finished before the
+ * next fifo command. (For example on hardware context destructions where the
+ * hardware may otherwise leak unfinished queries).
  *
- * This function does not return any failure codes, but make attempts
- * to do safe unpinning in case of errors.
+ * This function does not return any failure codes, but make attempts to do safe
+ * unpinning in case of errors.
  *
- * The function will synchronize on the previous query barrier, and will
- * thus not finish until that barrier has executed.
+ * The function will synchronize on the previous query barrier, and will thus
+ * not finish until that barrier has executed.
  */
 void vmw_execbuf_release_pinned_bo(struct vmw_private *dev_priv)
 {
@@ -4203,8 +3989,8 @@ int vmw_execbuf_ioctl(struct drm_device *dev, unsigned long data,
 	struct dma_fence *in_fence = NULL;
 
 	if (unlikely(size < copy_offset[0])) {
-		DRM_ERROR("Invalid command size, ioctl %d\n",
-			  DRM_VMW_EXECBUF);
+		VMW_DEBUG_USER("Invalid command size, ioctl %d\n",
+			       DRM_VMW_EXECBUF);
 		return -EINVAL;
 	}
 
@@ -4212,23 +3998,19 @@ int vmw_execbuf_ioctl(struct drm_device *dev, unsigned long data,
 		return -EFAULT;
 
 	/*
-	 * Extend the ioctl argument while
-	 * maintaining backwards compatibility:
-	 * We take different code paths depending on the value of
-	 * arg.version.
+	 * Extend the ioctl argument while maintaining backwards compatibility:
+	 * We take different code paths depending on the value of arg.version.
 	 */
-
 	if (unlikely(arg.version > DRM_VMW_EXECBUF_VERSION ||
 		     arg.version == 0)) {
-		DRM_ERROR("Incorrect execbuf version.\n");
+		VMW_DEBUG_USER("Incorrect execbuf version.\n");
 		return -EINVAL;
 	}
 
 	if (arg.version > 1 &&
 	    copy_from_user(&arg.context_handle,
 			   (void __user *) (data + copy_offset[0]),
-			   copy_offset[arg.version - 1] -
-			   copy_offset[0]) != 0)
+			   copy_offset[arg.version - 1] - copy_offset[0]) != 0)
 		return -EFAULT;
 
 	switch (arg.version) {
@@ -4240,13 +4022,12 @@ int vmw_execbuf_ioctl(struct drm_device *dev, unsigned long data,
 		break;
 	}
 
-
 	/* If imported a fence FD from elsewhere, then wait on it */
 	if (arg.flags & DRM_VMW_EXECBUF_FLAG_IMPORT_FENCE_FD) {
 		in_fence = sync_file_get_fence(arg.imported_fence_fd);
 
 		if (!in_fence) {
-			DRM_ERROR("Cannot get imported fence\n");
+			VMW_DEBUG_USER("Cannot get imported fence\n");
 			return -EINVAL;
 		}
 
@@ -4264,8 +4045,8 @@ int vmw_execbuf_ioctl(struct drm_device *dev, unsigned long data,
 				  NULL, arg.command_size, arg.throttle_us,
 				  arg.context_handle,
 				  (void __user *)(unsigned long)arg.fence_rep,
-				  NULL,
-				  arg.flags);
+				  NULL, arg.flags);
+
 	ttm_read_unlock(&dev_priv->reservation_sem);
 	if (unlikely(ret != 0))
 		goto out;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c b/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c
index 2a9112515f46..972e8fda6d35 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c
@@ -642,12 +642,11 @@ int vmw_fb_init(struct vmw_private *vmw_priv)
 	struct vmw_fb_par *par;
 	struct fb_info *info;
 	unsigned fb_width, fb_height;
-	unsigned fb_bpp, fb_depth, fb_offset, fb_pitch, fb_size;
+	unsigned int fb_bpp, fb_pitch, fb_size;
 	struct drm_display_mode *init_mode;
 	int ret;
 
 	fb_bpp = 32;
-	fb_depth = 24;
 
 	/* XXX As shouldn't these be as well. */
 	fb_width = min(vmw_priv->fb_max_width, (unsigned)2048);
@@ -655,7 +654,6 @@ int vmw_fb_init(struct vmw_private *vmw_priv)
 
 	fb_pitch = fb_width * fb_bpp / 8;
 	fb_size = fb_pitch * fb_height;
-	fb_offset = vmw_read(vmw_priv, SVGA_REG_FB_OFFSET);
 
 	info = framebuffer_alloc(sizeof(*par), device);
 	if (!info)
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_fifo.c b/drivers/gpu/drm/vmwgfx/vmwgfx_fifo.c
index d0fd147ef75f..ff3586cb6851 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_fifo.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_fifo.c
@@ -395,12 +395,8 @@ void *vmw_fifo_reserve_dx(struct vmw_private *dev_priv, uint32_t bytes,
 		WARN(1, "Command buffer has not been allocated.\n");
 		ret = NULL;
 	}
-	if (IS_ERR_OR_NULL(ret)) {
-		DRM_ERROR("Fifo reserve failure of %u bytes.\n",
-			  (unsigned) bytes);
-		dump_stack();
+	if (IS_ERR_OR_NULL(ret))
 		return NULL;
-	}
 
 	return ret;
 }
@@ -544,7 +540,7 @@ int vmw_fifo_send_fence(struct vmw_private *dev_priv, uint32_t *seqno)
 	int ret = 0;
 	uint32_t bytes = sizeof(u32) + sizeof(*cmd_fence);
 
-	fm = vmw_fifo_reserve(dev_priv, bytes);
+	fm = VMW_FIFO_RESERVE(dev_priv, bytes);
 	if (unlikely(fm == NULL)) {
 		*seqno = atomic_read(&dev_priv->marker_seq);
 		ret = -ENOMEM;
@@ -603,12 +599,9 @@ static int vmw_fifo_emit_dummy_legacy_query(struct vmw_private *dev_priv,
 		SVGA3dCmdWaitForQuery body;
 	} *cmd;
 
-	cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
-
-	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Out of fifo space for dummy query.\n");
+	cmd = VMW_FIFO_RESERVE(dev_priv, sizeof(*cmd));
+	if (unlikely(cmd == NULL))
 		return -ENOMEM;
-	}
 
 	cmd->header.id = SVGA_3D_CMD_WAIT_FOR_QUERY;
 	cmd->header.size = sizeof(cmd->body);
@@ -652,12 +645,9 @@ static int vmw_fifo_emit_dummy_gb_query(struct vmw_private *dev_priv,
 		SVGA3dCmdWaitForGBQuery body;
 	} *cmd;
 
-	cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
-
-	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Out of fifo space for dummy query.\n");
+	cmd = VMW_FIFO_RESERVE(dev_priv, sizeof(*cmd));
+	if (unlikely(cmd == NULL))
 		return -ENOMEM;
-	}
 
 	cmd->header.id = SVGA_3D_CMD_WAIT_FOR_GB_QUERY;
 	cmd->header.size = sizeof(cmd->body);
@@ -699,8 +689,3 @@ int vmw_fifo_emit_dummy_query(struct vmw_private *dev_priv,
 
 	return vmw_fifo_emit_dummy_legacy_query(dev_priv, cid);
 }
-
-void *vmw_fifo_reserve(struct vmw_private *dev_priv, uint32_t bytes)
-{
-	return vmw_fifo_reserve_dx(dev_priv, bytes, SVGA3D_INVALID_ID);
-}
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_gmr.c b/drivers/gpu/drm/vmwgfx/vmwgfx_gmr.c
index 007a0cc7f232..ae7acc6f3dda 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_gmr.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_gmr.c
@@ -51,7 +51,7 @@ static int vmw_gmr2_bind(struct vmw_private *dev_priv,
 	uint32_t cmd_size = define_size + remap_size;
 	uint32_t i;
 
-	cmd_orig = cmd = vmw_fifo_reserve(dev_priv, cmd_size);
+	cmd_orig = cmd = VMW_FIFO_RESERVE(dev_priv, cmd_size);
 	if (unlikely(cmd == NULL))
 		return -ENOMEM;
 
@@ -110,11 +110,10 @@ static void vmw_gmr2_unbind(struct vmw_private *dev_priv,
 	uint32_t define_size = sizeof(define_cmd) + 4;
 	uint32_t *cmd;
 
-	cmd = vmw_fifo_reserve(dev_priv, define_size);
-	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("GMR2 unbind failed.\n");
+	cmd = VMW_FIFO_RESERVE(dev_priv, define_size);
+	if (unlikely(cmd == NULL))
 		return;
-	}
+
 	define_cmd.gmrId = gmr_id;
 	define_cmd.numPages = 0;
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c
index 172a6ba6539c..a15375eb476e 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c
@@ -188,7 +188,7 @@ int vmw_get_cap_3d_ioctl(struct drm_device *dev, void *data,
 	struct vmw_fpriv *vmw_fp = vmw_fpriv(file_priv);
 
 	if (unlikely(arg->pad64 != 0 || arg->max_size == 0)) {
-		DRM_ERROR("Illegal GET_3D_CAP argument.\n");
+		VMW_DEBUG_USER("Illegal GET_3D_CAP argument.\n");
 		return -EINVAL;
 	}
 
@@ -268,7 +268,7 @@ int vmw_present_ioctl(struct drm_device *dev, void *data,
 		return 0;
 
 	if (clips_ptr == NULL) {
-		DRM_ERROR("Variable clips_ptr must be specified.\n");
+		VMW_DEBUG_USER("Variable clips_ptr must be specified.\n");
 		ret = -EINVAL;
 		goto out_clips;
 	}
@@ -291,7 +291,7 @@ int vmw_present_ioctl(struct drm_device *dev, void *data,
 
 	fb = drm_framebuffer_lookup(dev, file_priv, arg->fb_id);
 	if (!fb) {
-		DRM_ERROR("Invalid framebuffer id.\n");
+		VMW_DEBUG_USER("Invalid framebuffer id.\n");
 		ret = -ENOENT;
 		goto out_no_fb;
 	}
@@ -351,7 +351,7 @@ int vmw_present_readback_ioctl(struct drm_device *dev, void *data,
 		return 0;
 
 	if (clips_ptr == NULL) {
-		DRM_ERROR("Argument clips_ptr must be specified.\n");
+		VMW_DEBUG_USER("Argument clips_ptr must be specified.\n");
 		ret = -EINVAL;
 		goto out_clips;
 	}
@@ -374,14 +374,14 @@ int vmw_present_readback_ioctl(struct drm_device *dev, void *data,
 
 	fb = drm_framebuffer_lookup(dev, file_priv, arg->fb_id);
 	if (!fb) {
-		DRM_ERROR("Invalid framebuffer id.\n");
+		VMW_DEBUG_USER("Invalid framebuffer id.\n");
 		ret = -ENOENT;
 		goto out_no_fb;
 	}
 
 	vfb = vmw_framebuffer_to_vfb(fb);
 	if (!vfb->bo) {
-		DRM_ERROR("Framebuffer not buffer backed.\n");
+		VMW_DEBUG_USER("Framebuffer not buffer backed.\n");
 		ret = -EINVAL;
 		goto out_no_ttm_lock;
 	}
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
index ed2f67822f45..b97bc8e5944b 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
@@ -64,11 +64,9 @@ static int vmw_cursor_update_image(struct vmw_private *dev_priv,
 	if (!image)
 		return -EINVAL;
 
-	cmd = vmw_fifo_reserve(dev_priv, cmd_size);
-	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Fifo reserve failed.\n");
+	cmd = VMW_FIFO_RESERVE(dev_priv, cmd_size);
+	if (unlikely(cmd == NULL))
 		return -ENOMEM;
-	}
 
 	memset(cmd, 0, sizeof(*cmd));
 
@@ -1202,7 +1200,7 @@ static int vmw_create_bo_proxy(struct drm_device *dev,
 	vmw_bo_unreference(&res->backup);
 	res->backup = vmw_bo_reference(bo_mob);
 	res->backup_offset = 0;
-	vmw_resource_unreserve(res, false, NULL, 0);
+	vmw_resource_unreserve(res, false, false, false, NULL, 0);
 	mutex_unlock(&res->dev_priv->cmdbuf_mutex);
 
 	return 0;
@@ -2468,13 +2466,11 @@ int vmw_kms_helper_dirty(struct vmw_private *dev_priv,
 
 		dirty->unit = unit;
 		if (dirty->fifo_reserve_size > 0) {
-			dirty->cmd = vmw_fifo_reserve(dev_priv,
+			dirty->cmd = VMW_FIFO_RESERVE(dev_priv,
 						      dirty->fifo_reserve_size);
-			if (!dirty->cmd) {
-				DRM_ERROR("Couldn't reserve fifo space "
-					  "for dirty blits.\n");
+			if (!dirty->cmd)
 				return -ENOMEM;
-			}
+
 			memset(dirty->cmd, 0, dirty->fifo_reserve_size);
 		}
 		dirty->num_hits = 0;
@@ -2604,12 +2600,9 @@ int vmw_kms_update_proxy(struct vmw_resource *res,
 	if (!clips)
 		return 0;
 
-	cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd) * num_clips);
-	if (!cmd) {
-		DRM_ERROR("Couldn't reserve fifo space for proxy surface "
-			  "update.\n");
+	cmd = VMW_FIFO_RESERVE(dev_priv, sizeof(*cmd) * num_clips);
+	if (!cmd)
 		return -ENOMEM;
-	}
 
 	for (i = 0; i < num_clips; ++i, clips += increment, ++cmd) {
 		box = &cmd->body.box;
@@ -2827,7 +2820,8 @@ int vmw_du_helper_plane_update(struct vmw_du_update_plane *update)
 			container_of(update->vfb, typeof(*vfbs), base);
 
 		ret = vmw_validation_add_resource(&val_ctx, &vfbs->surface->res,
-						  0, NULL, NULL);
+						  0, VMW_RES_DIRTY_NONE, NULL,
+						  NULL);
 	}
 
 	if (ret)
@@ -2838,7 +2832,7 @@ int vmw_du_helper_plane_update(struct vmw_du_update_plane *update)
 		goto out_unref;
 
 	reserved_size = update->calc_fifo_size(update, num_hits);
-	cmd_start = vmw_fifo_reserve(update->dev_priv, reserved_size);
+	cmd_start = VMW_FIFO_RESERVE(update->dev_priv, reserved_size);
 	if (!cmd_start) {
 		ret = -ENOMEM;
 		goto out_revert;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
index 16be515c4c0f..25e6343bcf21 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
@@ -554,11 +554,9 @@ int vmw_kms_ldu_do_bo_dirty(struct vmw_private *dev_priv,
 	} *cmd;
 
 	fifo_size = sizeof(*cmd) * num_clips;
-	cmd = vmw_fifo_reserve(dev_priv, fifo_size);
-	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Fifo reserve failed.\n");
+	cmd = VMW_FIFO_RESERVE(dev_priv, fifo_size);
+	if (unlikely(cmd == NULL))
 		return -ENOMEM;
-	}
 
 	memset(cmd, 0, fifo_size);
 	for (i = 0; i < num_clips; i++, clips += increment) {
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_mob.c b/drivers/gpu/drm/vmwgfx/vmwgfx_mob.c
index d83cc66e1210..406edc8cef35 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_mob.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_mob.c
@@ -146,9 +146,8 @@ static int vmw_setup_otable_base(struct vmw_private *dev_priv,
 		mob->pt_level += VMW_MOBFMT_PTDEPTH_1 - SVGA3D_MOBFMT_PTDEPTH_1;
 	}
 
-	cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
+	cmd = VMW_FIFO_RESERVE(dev_priv, sizeof(*cmd));
 	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Failed reserving FIFO space for OTable setup.\n");
 		ret = -ENOMEM;
 		goto out_no_fifo;
 	}
@@ -202,12 +201,9 @@ static void vmw_takedown_otable_base(struct vmw_private *dev_priv,
 		return;
 
 	bo = otable->page_table->pt_bo;
-	cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
-	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Failed reserving FIFO space for OTable "
-			  "takedown.\n");
+	cmd = VMW_FIFO_RESERVE(dev_priv, sizeof(*cmd));
+	if (unlikely(cmd == NULL))
 		return;
-	}
 
 	memset(cmd, 0, sizeof(*cmd));
 	cmd->header.id = SVGA_3D_CMD_SET_OTABLE_BASE;
@@ -614,16 +610,14 @@ void vmw_mob_unbind(struct vmw_private *dev_priv,
 		BUG_ON(ret != 0);
 	}
 
-	cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
-	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Failed reserving FIFO space for Memory "
-			  "Object unbinding.\n");
-	} else {
+	cmd = VMW_FIFO_RESERVE(dev_priv, sizeof(*cmd));
+	if (cmd) {
 		cmd->header.id = SVGA_3D_CMD_DESTROY_GB_MOB;
 		cmd->header.size = sizeof(cmd->body);
 		cmd->body.mobid = mob->id;
 		vmw_fifo_commit(dev_priv, sizeof(*cmd));
 	}
+
 	if (bo) {
 		vmw_bo_fence_single(bo, NULL);
 		ttm_bo_unreserve(bo);
@@ -683,12 +677,9 @@ int vmw_mob_bind(struct vmw_private *dev_priv,
 
 	vmw_fifo_resource_inc(dev_priv);
 
-	cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
-	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Failed reserving FIFO space for Memory "
-			  "Object binding.\n");
+	cmd = VMW_FIFO_RESERVE(dev_priv, sizeof(*cmd));
+	if (unlikely(cmd == NULL))
 		goto out_no_cmd_space;
-	}
 
 	cmd->header.id = SVGA_3D_CMD_DEFINE_GB_MOB64;
 	cmd->header.size = sizeof(cmd->body);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_overlay.c b/drivers/gpu/drm/vmwgfx/vmwgfx_overlay.c
index 9f1b9d289bec..d5ef8cf802de 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_overlay.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_overlay.c
@@ -124,7 +124,7 @@ static int vmw_overlay_send_put(struct vmw_private *dev_priv,
 
 	fifo_size = sizeof(*cmds) + sizeof(*flush) + sizeof(*items) * num_items;
 
-	cmds = vmw_fifo_reserve(dev_priv, fifo_size);
+	cmds = VMW_FIFO_RESERVE(dev_priv, fifo_size);
 	/* hardware has hung, can't do anything here */
 	if (!cmds)
 		return -ENOMEM;
@@ -194,7 +194,7 @@ static int vmw_overlay_send_stop(struct vmw_private *dev_priv,
 	int ret;
 
 	for (;;) {
-		cmds = vmw_fifo_reserve(dev_priv, sizeof(*cmds));
+		cmds = VMW_FIFO_RESERVE(dev_priv, sizeof(*cmds));
 		if (cmds)
 			break;
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c b/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
index a7c30e567f09..711f8fd0dd45 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
@@ -365,14 +365,6 @@ static int vmw_resource_do_validate(struct vmw_resource *res,
 			list_add_tail(&res->mob_head, &res->backup->res_list);
 	}
 
-	/*
-	 * Only do this on write operations, and move to
-	 * vmw_resource_unreserve if it can be called after
-	 * backup buffers have been unreserved. Otherwise
-	 * sort out locking.
-	 */
-	res->res_dirty = true;
-
 	return 0;
 
 out_bind_failed:
@@ -386,6 +378,8 @@ out_bind_failed:
  * command submission.
  *
  * @res:               Pointer to the struct vmw_resource to unreserve.
+ * @dirty_set:         Change dirty status of the resource.
+ * @dirty:             When changing dirty status indicates the new status.
  * @switch_backup:     Backup buffer has been switched.
  * @new_backup:        Pointer to new backup buffer if command submission
  *                     switched. May be NULL.
@@ -395,6 +389,8 @@ out_bind_failed:
  * resource lru list, so that it can be evicted if necessary.
  */
 void vmw_resource_unreserve(struct vmw_resource *res,
+			    bool dirty_set,
+			    bool dirty,
 			    bool switch_backup,
 			    struct vmw_buffer_object *new_backup,
 			    unsigned long new_backup_offset)
@@ -422,6 +418,9 @@ void vmw_resource_unreserve(struct vmw_resource *res,
 	if (switch_backup)
 		res->backup_offset = new_backup_offset;
 
+	if (dirty_set)
+		res->res_dirty = dirty;
+
 	if (!res->func->may_evict || res->id == -1 || res->pin_count)
 		return;
 
@@ -696,7 +695,7 @@ void vmw_resource_unbind_list(struct vmw_buffer_object *vbo)
 		if (!res->func->unbind)
 			continue;
 
-		(void) res->func->unbind(res, true, &val_buf);
+		(void) res->func->unbind(res, res->res_dirty, &val_buf);
 		res->backup_dirty = true;
 		res->res_dirty = false;
 		list_del_init(&res->mob_head);
@@ -731,12 +730,9 @@ int vmw_query_readback_all(struct vmw_buffer_object *dx_query_mob)
 	dx_query_ctx = dx_query_mob->dx_query_ctx;
 	dev_priv     = dx_query_ctx->dev_priv;
 
-	cmd = vmw_fifo_reserve_dx(dev_priv, sizeof(*cmd), dx_query_ctx->id);
-	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Failed reserving FIFO space for "
-			  "query MOB read back.\n");
+	cmd = VMW_FIFO_RESERVE_DX(dev_priv, sizeof(*cmd), dx_query_ctx->id);
+	if (unlikely(cmd == NULL))
 		return -ENOMEM;
-	}
 
 	cmd->header.id   = SVGA_3D_CMD_DX_READBACK_ALL_QUERY;
 	cmd->header.size = sizeof(cmd->body);
@@ -932,7 +928,7 @@ int vmw_resource_pin(struct vmw_resource *res, bool interruptible)
 	res->pin_count++;
 
 out_no_validate:
-	vmw_resource_unreserve(res, false, NULL, 0UL);
+	vmw_resource_unreserve(res, false, false, false, NULL, 0UL);
 out_no_reserve:
 	mutex_unlock(&dev_priv->cmdbuf_mutex);
 	ttm_write_unlock(&dev_priv->reservation_sem);
@@ -968,7 +964,7 @@ void vmw_resource_unpin(struct vmw_resource *res)
 		ttm_bo_unreserve(&vbo->base);
 	}
 
-	vmw_resource_unreserve(res, false, NULL, 0UL);
+	vmw_resource_unreserve(res, false, false, false, NULL, 0UL);
 
 	mutex_unlock(&dev_priv->cmdbuf_mutex);
 	ttm_read_unlock(&dev_priv->reservation_sem);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c b/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c
index cd586c52af7e..9a2a3836d89a 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c
@@ -130,12 +130,9 @@ static int vmw_sou_fifo_create(struct vmw_private *dev_priv,
 	BUG_ON(!sou->buffer);
 
 	fifo_size = sizeof(*cmd);
-	cmd = vmw_fifo_reserve(dev_priv, fifo_size);
-	/* The hardware has hung, nothing we can do about it here. */
-	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Fifo reserve failed.\n");
+	cmd = VMW_FIFO_RESERVE(dev_priv, fifo_size);
+	if (unlikely(cmd == NULL))
 		return -ENOMEM;
-	}
 
 	memset(cmd, 0, fifo_size);
 	cmd->header.cmdType = SVGA_CMD_DEFINE_SCREEN;
@@ -182,12 +179,9 @@ static int vmw_sou_fifo_destroy(struct vmw_private *dev_priv,
 		return 0;
 
 	fifo_size = sizeof(*cmd);
-	cmd = vmw_fifo_reserve(dev_priv, fifo_size);
-	/* the hardware has hung, nothing we can do about it here */
-	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Fifo reserve failed.\n");
+	cmd = VMW_FIFO_RESERVE(dev_priv, fifo_size);
+	if (unlikely(cmd == NULL))
 		return -ENOMEM;
-	}
 
 	memset(cmd, 0, fifo_size);
 	cmd->header.cmdType = SVGA_CMD_DESTROY_SCREEN;
@@ -998,11 +992,9 @@ static int do_bo_define_gmrfb(struct vmw_private *dev_priv,
 	if (depth == 32)
 		depth = 24;
 
-	cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
-	if (!cmd) {
-		DRM_ERROR("Out of fifo space for dirty framebuffer command.\n");
+	cmd = VMW_FIFO_RESERVE(dev_priv, sizeof(*cmd));
+	if (!cmd)
 		return -ENOMEM;
-	}
 
 	cmd->header = SVGA_CMD_DEFINE_GMRFB;
 	cmd->body.format.bitsPerPixel = framebuffer->base.format->cpp[0] * 8;
@@ -1148,7 +1140,8 @@ int vmw_kms_sou_do_surface_dirty(struct vmw_private *dev_priv,
 	if (!srf)
 		srf = &vfbs->surface->res;
 
-	ret = vmw_validation_add_resource(&val_ctx, srf, 0, NULL, NULL);
+	ret = vmw_validation_add_resource(&val_ctx, srf, 0, VMW_RES_DIRTY_NONE,
+					  NULL, NULL);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_shader.c b/drivers/gpu/drm/vmwgfx/vmwgfx_shader.c
index bf32fe446219..d310d21f0d54 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_shader.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_shader.c
@@ -218,10 +218,8 @@ static int vmw_gb_shader_create(struct vmw_resource *res)
 		goto out_no_fifo;
 	}
 
-	cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
+	cmd = VMW_FIFO_RESERVE(dev_priv, sizeof(*cmd));
 	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Failed reserving FIFO space for shader "
-			  "creation.\n");
 		ret = -ENOMEM;
 		goto out_no_fifo;
 	}
@@ -254,12 +252,9 @@ static int vmw_gb_shader_bind(struct vmw_resource *res,
 
 	BUG_ON(bo->mem.mem_type != VMW_PL_MOB);
 
-	cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
-	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Failed reserving FIFO space for shader "
-			  "binding.\n");
+	cmd = VMW_FIFO_RESERVE(dev_priv, sizeof(*cmd));
+	if (unlikely(cmd == NULL))
 		return -ENOMEM;
-	}
 
 	cmd->header.id = SVGA_3D_CMD_BIND_GB_SHADER;
 	cmd->header.size = sizeof(cmd->body);
@@ -285,12 +280,9 @@ static int vmw_gb_shader_unbind(struct vmw_resource *res,
 
 	BUG_ON(res->backup->base.mem.mem_type != VMW_PL_MOB);
 
-	cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
-	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Failed reserving FIFO space for shader "
-			  "unbinding.\n");
+	cmd = VMW_FIFO_RESERVE(dev_priv, sizeof(*cmd));
+	if (unlikely(cmd == NULL))
 		return -ENOMEM;
-	}
 
 	cmd->header.id = SVGA_3D_CMD_BIND_GB_SHADER;
 	cmd->header.size = sizeof(cmd->body);
@@ -328,10 +320,8 @@ static int vmw_gb_shader_destroy(struct vmw_resource *res)
 	mutex_lock(&dev_priv->binding_mutex);
 	vmw_binding_res_list_scrub(&res->binding_head);
 
-	cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
+	cmd = VMW_FIFO_RESERVE(dev_priv, sizeof(*cmd));
 	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Failed reserving FIFO space for shader "
-			  "destruction.\n");
 		mutex_unlock(&dev_priv->binding_mutex);
 		return -ENOMEM;
 	}
@@ -400,13 +390,9 @@ static int vmw_dx_shader_unscrub(struct vmw_resource *res)
 	if (!list_empty(&shader->cotable_head) || !shader->committed)
 		return 0;
 
-	cmd = vmw_fifo_reserve_dx(dev_priv, sizeof(*cmd),
-				  shader->ctx->id);
-	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Failed reserving FIFO space for shader "
-			  "scrubbing.\n");
+	cmd = VMW_FIFO_RESERVE_DX(dev_priv, sizeof(*cmd), shader->ctx->id);
+	if (unlikely(cmd == NULL))
 		return -ENOMEM;
-	}
 
 	cmd->header.id = SVGA_3D_CMD_DX_BIND_SHADER;
 	cmd->header.size = sizeof(cmd->body);
@@ -491,12 +477,9 @@ static int vmw_dx_shader_scrub(struct vmw_resource *res)
 		return 0;
 
 	WARN_ON_ONCE(!shader->committed);
-	cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
-	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Failed reserving FIFO space for shader "
-			  "scrubbing.\n");
+	cmd = VMW_FIFO_RESERVE(dev_priv, sizeof(*cmd));
+	if (unlikely(cmd == NULL))
 		return -ENOMEM;
-	}
 
 	cmd->header.id = SVGA_3D_CMD_DX_BIND_SHADER;
 	cmd->header.size = sizeof(cmd->body);
@@ -865,14 +848,13 @@ static int vmw_shader_define(struct drm_device *dev, struct drm_file *file_priv,
 		ret = vmw_user_bo_lookup(tfile, buffer_handle,
 					     &buffer, NULL);
 		if (unlikely(ret != 0)) {
-			DRM_ERROR("Could not find buffer for shader "
-				  "creation.\n");
+			VMW_DEBUG_USER("Couldn't find buffer for shader creation.\n");
 			return ret;
 		}
 
 		if ((u64)buffer->base.num_pages * PAGE_SIZE <
 		    (u64)size + (u64)offset) {
-			DRM_ERROR("Illegal buffer- or shader size.\n");
+			VMW_DEBUG_USER("Illegal buffer- or shader size.\n");
 			ret = -EINVAL;
 			goto out_bad_arg;
 		}
@@ -886,7 +868,7 @@ static int vmw_shader_define(struct drm_device *dev, struct drm_file *file_priv,
 		shader_type = SVGA3D_SHADERTYPE_PS;
 		break;
 	default:
-		DRM_ERROR("Illegal shader type.\n");
+		VMW_DEBUG_USER("Illegal shader type.\n");
 		ret = -EINVAL;
 		goto out_bad_arg;
 	}
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_simple_resource.c b/drivers/gpu/drm/vmwgfx/vmwgfx_simple_resource.c
index 6a6865384e91..73e9a487e659 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_simple_resource.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_simple_resource.c
@@ -239,17 +239,17 @@ vmw_simple_resource_lookup(struct ttm_object_file *tfile,
 
 	base = ttm_base_object_lookup(tfile, handle);
 	if (!base) {
-		DRM_ERROR("Invalid %s handle 0x%08lx.\n",
-			  func->res_func.type_name,
-			  (unsigned long) handle);
+		VMW_DEBUG_USER("Invalid %s handle 0x%08lx.\n",
+			       func->res_func.type_name,
+			       (unsigned long) handle);
 		return ERR_PTR(-ESRCH);
 	}
 
 	if (ttm_base_object_type(base) != func->ttm_res_type) {
 		ttm_base_object_unref(&base);
-		DRM_ERROR("Invalid type of %s handle 0x%08lx.\n",
-			  func->res_func.type_name,
-			  (unsigned long) handle);
+		VMW_DEBUG_USER("Invalid type of %s handle 0x%08lx.\n",
+			       func->res_func.type_name,
+			       (unsigned long) handle);
 		return ERR_PTR(-EINVAL);
 	}
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_so.c b/drivers/gpu/drm/vmwgfx/vmwgfx_so.c
index bc8bb690f1ea..63807361e16f 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_so.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_so.c
@@ -170,13 +170,12 @@ static int vmw_view_create(struct vmw_resource *res)
 		return 0;
 	}
 
-	cmd = vmw_fifo_reserve_dx(res->dev_priv, view->cmd_size,
-				  view->ctx->id);
+	cmd = VMW_FIFO_RESERVE_DX(res->dev_priv, view->cmd_size, view->ctx->id);
 	if (!cmd) {
-		DRM_ERROR("Failed reserving FIFO space for view creation.\n");
 		mutex_unlock(&dev_priv->binding_mutex);
 		return -ENOMEM;
 	}
+
 	memcpy(cmd, &view->cmd, view->cmd_size);
 	WARN_ON(cmd->body.view_id != view->view_id);
 	/* Sid may have changed due to surface eviction. */
@@ -214,12 +213,9 @@ static int vmw_view_destroy(struct vmw_resource *res)
 	if (!view->committed || res->id == -1)
 		return 0;
 
-	cmd = vmw_fifo_reserve_dx(dev_priv, sizeof(*cmd), view->ctx->id);
-	if (!cmd) {
-		DRM_ERROR("Failed reserving FIFO space for view "
-			  "destruction.\n");
+	cmd = VMW_FIFO_RESERVE_DX(dev_priv, sizeof(*cmd), view->ctx->id);
+	if (!cmd)
 		return -ENOMEM;
-	}
 
 	cmd->header.id = vmw_view_destroy_cmds[view->view_type];
 	cmd->header.size = sizeof(cmd->body);
@@ -338,12 +334,12 @@ int vmw_view_add(struct vmw_cmdbuf_res_manager *man,
 
 	if (cmd_size != vmw_view_define_sizes[view_type] +
 	    sizeof(SVGA3dCmdHeader)) {
-		DRM_ERROR("Illegal view create command size.\n");
+		VMW_DEBUG_USER("Illegal view create command size.\n");
 		return -EINVAL;
 	}
 
 	if (!vmw_view_id_ok(user_key, view_type)) {
-		DRM_ERROR("Illegal view add view id.\n");
+		VMW_DEBUG_USER("Illegal view add view id.\n");
 		return -EINVAL;
 	}
 
@@ -352,8 +348,7 @@ int vmw_view_add(struct vmw_cmdbuf_res_manager *man,
 	ret = ttm_mem_global_alloc(vmw_mem_glob(dev_priv), size, &ttm_opt_ctx);
 	if (ret) {
 		if (ret != -ERESTARTSYS)
-			DRM_ERROR("Out of graphics memory for view"
-				  " creation.\n");
+			DRM_ERROR("Out of graphics memory for view creation\n");
 		return ret;
 	}
 
@@ -413,7 +408,7 @@ int vmw_view_remove(struct vmw_cmdbuf_res_manager *man,
 		    struct vmw_resource **res_p)
 {
 	if (!vmw_view_id_ok(user_key, view_type)) {
-		DRM_ERROR("Illegal view remove view id.\n");
+		VMW_DEBUG_USER("Illegal view remove view id.\n");
 		return -EINVAL;
 	}
 
@@ -497,6 +492,30 @@ struct vmw_resource *vmw_view_lookup(struct vmw_cmdbuf_res_manager *man,
 				     vmw_view_key(user_key, view_type));
 }
 
+/**
+ * vmw_view_dirtying - Return whether a view type is dirtying its resource
+ * @res: Pointer to the view
+ *
+ * Each time a resource is put on the validation list as the result of a
+ * view pointing to it, we need to determine whether that resource will
+ * be dirtied (written to by the GPU) as a result of the corresponding
+ * GPU operation. Currently only rendertarget- and depth-stencil views are
+ * capable of dirtying its resource.
+ *
+ * Return: Whether the view type of @res dirties the resource it points to.
+ */
+u32 vmw_view_dirtying(struct vmw_resource *res)
+{
+	static u32 view_is_dirtying[vmw_view_max] = {
+		[vmw_view_rt] = VMW_RES_DIRTY_SET,
+		[vmw_view_ds] = VMW_RES_DIRTY_SET,
+	};
+
+	/* Update this function as we add more view types */
+	BUILD_BUG_ON(vmw_view_max != 3);
+	return view_is_dirtying[vmw_view(res)->view_type];
+}
+
 const u32 vmw_view_destroy_cmds[] = {
 	[vmw_view_sr] = SVGA_3D_CMD_DX_DESTROY_SHADERRESOURCE_VIEW,
 	[vmw_view_rt] = SVGA_3D_CMD_DX_DESTROY_RENDERTARGET_VIEW,
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_so.h b/drivers/gpu/drm/vmwgfx/vmwgfx_so.h
index b80c7252f2fd..12565047bc55 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_so.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_so.h
@@ -157,4 +157,5 @@ extern struct vmw_resource *vmw_view_srf(struct vmw_resource *res);
 extern struct vmw_resource *vmw_view_lookup(struct vmw_cmdbuf_res_manager *man,
 					    enum vmw_view_type view_type,
 					    u32 user_key);
+extern u32 vmw_view_dirtying(struct vmw_resource *res);
 #endif
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c b/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
index 096c2941a8e4..f803bb5e782b 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
@@ -111,7 +111,7 @@ struct vmw_stdu_update_gb_image {
  */
 struct vmw_screen_target_display_unit {
 	struct vmw_display_unit base;
-	const struct vmw_surface *display_srf;
+	struct vmw_surface *display_srf;
 	enum stdu_content_type content_fb_type;
 	s32 display_width, display_height;
 
@@ -167,12 +167,9 @@ static int vmw_stdu_define_st(struct vmw_private *dev_priv,
 		SVGA3dCmdDefineGBScreenTarget body;
 	} *cmd;
 
-	cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
-
-	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Out of FIFO space defining Screen Target\n");
+	cmd = VMW_FIFO_RESERVE(dev_priv, sizeof(*cmd));
+	if (unlikely(cmd == NULL))
 		return -ENOMEM;
-	}
 
 	cmd->header.id   = SVGA_3D_CMD_DEFINE_GB_SCREENTARGET;
 	cmd->header.size = sizeof(cmd->body);
@@ -229,12 +226,9 @@ static int vmw_stdu_bind_st(struct vmw_private *dev_priv,
 	memset(&image, 0, sizeof(image));
 	image.sid = res ? res->id : SVGA3D_INVALID_ID;
 
-	cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
-
-	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Out of FIFO space binding a screen target\n");
+	cmd = VMW_FIFO_RESERVE(dev_priv, sizeof(*cmd));
+	if (unlikely(cmd == NULL))
 		return -ENOMEM;
-	}
 
 	cmd->header.id   = SVGA_3D_CMD_BIND_GB_SCREENTARGET;
 	cmd->header.size = sizeof(cmd->body);
@@ -296,12 +290,9 @@ static int vmw_stdu_update_st(struct vmw_private *dev_priv,
 		return -EINVAL;
 	}
 
-	cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
-
-	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Out of FIFO space updating a Screen Target\n");
+	cmd = VMW_FIFO_RESERVE(dev_priv, sizeof(*cmd));
+	if (unlikely(cmd == NULL))
 		return -ENOMEM;
-	}
 
 	vmw_stdu_populate_update(cmd, stdu->base.unit,
 				 0, stdu->display_width,
@@ -335,12 +326,9 @@ static int vmw_stdu_destroy_st(struct vmw_private *dev_priv,
 	if (unlikely(!stdu->defined))
 		return 0;
 
-	cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
-
-	if (unlikely(cmd == NULL)) {
-		DRM_ERROR("Out of FIFO space, screen target not destroyed\n");
+	cmd = VMW_FIFO_RESERVE(dev_priv, sizeof(*cmd));
+	if (unlikely(cmd == NULL))
 		return -ENOMEM;
-	}
 
 	cmd->header.id   = SVGA_3D_CMD_DESTROY_GB_SCREENTARGET;
 	cmd->header.size = sizeof(cmd->body);
@@ -533,6 +521,7 @@ static void vmw_stdu_bo_fifo_commit(struct vmw_kms_dirty *dirty)
 
 	vmw_fifo_commit(dirty->dev_priv, sizeof(*cmd) + blit_size);
 
+	stdu->display_srf->res.res_dirty = true;
 	ddirty->left = ddirty->top = S32_MAX;
 	ddirty->right = ddirty->bottom = S32_MIN;
 }
@@ -629,20 +618,16 @@ static void vmw_stdu_bo_cpu_commit(struct vmw_kms_dirty *dirty)
 		region.x2 = diff.rect.x2;
 		region.y1 = diff.rect.y1;
 		region.y2 = diff.rect.y2;
-		ret = vmw_kms_update_proxy(
-			(struct vmw_resource *) &stdu->display_srf->res,
-			(const struct drm_clip_rect *) &region, 1, 1);
+		ret = vmw_kms_update_proxy(&stdu->display_srf->res, &region,
+					   1, 1);
 		if (ret)
 			goto out_cleanup;
 
 
 		dev_priv = vmw_priv(stdu->base.crtc.dev);
-		cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
-
-		if (!cmd) {
-			DRM_ERROR("Cannot reserve FIFO space to update STDU");
+		cmd = VMW_FIFO_RESERVE(dev_priv, sizeof(*cmd));
+		if (!cmd)
 			goto out_cleanup;
-		}
 
 		vmw_stdu_populate_update(cmd, stdu->base.unit,
 					 region.x1, region.x2,
@@ -820,6 +805,7 @@ static void vmw_kms_stdu_surface_fifo_commit(struct vmw_kms_dirty *dirty)
 		cmd->body.dest.sid = stdu->display_srf->res.id;
 		update = (struct vmw_stdu_update *) &blit[dirty->num_hits];
 		commit_size = sizeof(*cmd) + blit_size + sizeof(*update);
+		stdu->display_srf->res.res_dirty = true;
 	} else {
 		update = dirty->cmd;
 		commit_size = sizeof(*update);
@@ -876,7 +862,8 @@ int vmw_kms_stdu_surface_dirty(struct vmw_private *dev_priv,
 	if (!srf)
 		srf = &vfbs->surface->res;
 
-	ret = vmw_validation_add_resource(&val_ctx, srf, 0, NULL, NULL);
+	ret = vmw_validation_add_resource(&val_ctx, srf, 0, VMW_RES_DIRTY_NONE,
+					  NULL, NULL);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c b/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c
index ef09f7edf931..219471903bc1 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c
@@ -342,12 +342,9 @@ static void vmw_hw_surface_destroy(struct vmw_resource *res)
 
 	if (res->id != -1) {
 
-		cmd = vmw_fifo_reserve(dev_priv, vmw_surface_destroy_size());
-		if (unlikely(!cmd)) {
-			DRM_ERROR("Failed reserving FIFO space for surface "
-				  "destruction.\n");
+		cmd = VMW_FIFO_RESERVE(dev_priv, vmw_surface_destroy_size());
+		if (unlikely(!cmd))
 			return;
-		}
 
 		vmw_surface_destroy_encode(res->id, cmd);
 		vmw_fifo_commit(dev_priv, vmw_surface_destroy_size());
@@ -414,10 +411,8 @@ static int vmw_legacy_srf_create(struct vmw_resource *res)
 	 */
 
 	submit_size = vmw_surface_define_size(srf);
-	cmd = vmw_fifo_reserve(dev_priv, submit_size);
+	cmd = VMW_FIFO_RESERVE(dev_priv, submit_size);
 	if (unlikely(!cmd)) {
-		DRM_ERROR("Failed reserving FIFO space for surface "
-			  "creation.\n");
 		ret = -ENOMEM;
 		goto out_no_fifo;
 	}
@@ -468,12 +463,10 @@ static int vmw_legacy_srf_dma(struct vmw_resource *res,
 
 	BUG_ON(!val_buf->bo);
 	submit_size = vmw_surface_dma_size(srf);
-	cmd = vmw_fifo_reserve(dev_priv, submit_size);
-	if (unlikely(!cmd)) {
-		DRM_ERROR("Failed reserving FIFO space for surface "
-			  "DMA.\n");
+	cmd = VMW_FIFO_RESERVE(dev_priv, submit_size);
+	if (unlikely(!cmd))
 		return -ENOMEM;
-	}
+
 	vmw_bo_get_guest_ptr(val_buf->bo, &ptr);
 	vmw_surface_dma_encode(srf, cmd, &ptr, bind);
 
@@ -556,12 +549,9 @@ static int vmw_legacy_srf_destroy(struct vmw_resource *res)
 	 */
 
 	submit_size = vmw_surface_destroy_size();
-	cmd = vmw_fifo_reserve(dev_priv, submit_size);
-	if (unlikely(!cmd)) {
-		DRM_ERROR("Failed reserving FIFO space for surface "
-			  "eviction.\n");
+	cmd = VMW_FIFO_RESERVE(dev_priv, submit_size);
+	if (unlikely(!cmd))
 		return -ENOMEM;
-	}
 
 	vmw_surface_destroy_encode(res->id, cmd);
 	vmw_fifo_commit(dev_priv, submit_size);
@@ -748,11 +738,10 @@ int vmw_surface_define_ioctl(struct drm_device *dev, void *data,
 		ttm_round_pot(num_sizes * sizeof(struct drm_vmw_size)) +
 		ttm_round_pot(num_sizes * sizeof(struct vmw_surface_offset));
 
-
 	desc = svga3dsurface_get_desc(req->format);
 	if (unlikely(desc->block_desc == SVGA3DBLOCKDESC_NONE)) {
-		DRM_ERROR("Invalid surface format for surface creation.\n");
-		DRM_ERROR("Format requested is: %d\n", req->format);
+		VMW_DEBUG_USER("Invalid format %d for surface creation.\n",
+			       req->format);
 		return -EINVAL;
 	}
 
@@ -764,8 +753,7 @@ int vmw_surface_define_ioctl(struct drm_device *dev, void *data,
 				   size, &ctx);
 	if (unlikely(ret != 0)) {
 		if (ret != -ERESTARTSYS)
-			DRM_ERROR("Out of graphics memory for surface"
-				  " creation.\n");
+			DRM_ERROR("Out of graphics memory for surface.\n");
 		goto out_unlock;
 	}
 
@@ -939,12 +927,12 @@ vmw_surface_handle_reference(struct vmw_private *dev_priv,
 	ret = -EINVAL;
 	base = ttm_base_object_lookup_for_ref(dev_priv->tdev, handle);
 	if (unlikely(!base)) {
-		DRM_ERROR("Could not find surface to reference.\n");
+		VMW_DEBUG_USER("Could not find surface to reference.\n");
 		goto out_no_lookup;
 	}
 
 	if (unlikely(ttm_base_object_type(base) != VMW_RES_SURFACE)) {
-		DRM_ERROR("Referenced object is not a surface.\n");
+		VMW_DEBUG_USER("Referenced object is not a surface.\n");
 		goto out_bad_resource;
 	}
 
@@ -1022,8 +1010,8 @@ int vmw_surface_reference_ioctl(struct drm_device *dev, void *data,
 		ret = copy_to_user(user_sizes, &srf->base_size,
 				   sizeof(srf->base_size));
 	if (unlikely(ret != 0)) {
-		DRM_ERROR("copy_to_user failed %p %u\n",
-			  user_sizes, srf->num_sizes);
+		VMW_DEBUG_USER("copy_to_user failed %p %u\n", user_sizes,
+			       srf->num_sizes);
 		ttm_ref_object_base_unref(tfile, base->handle, TTM_REF_USAGE);
 		ret = -EFAULT;
 	}
@@ -1088,12 +1076,10 @@ static int vmw_gb_surface_create(struct vmw_resource *res)
 		submit_len = sizeof(*cmd);
 	}
 
-	cmd = vmw_fifo_reserve(dev_priv, submit_len);
+	cmd = VMW_FIFO_RESERVE(dev_priv, submit_len);
 	cmd2 = (typeof(cmd2))cmd;
 	cmd3 = (typeof(cmd3))cmd;
 	if (unlikely(!cmd)) {
-		DRM_ERROR("Failed reserving FIFO space for surface "
-			  "creation.\n");
 		ret = -ENOMEM;
 		goto out_no_fifo;
 	}
@@ -1171,12 +1157,9 @@ static int vmw_gb_surface_bind(struct vmw_resource *res,
 
 	submit_size = sizeof(*cmd1) + (res->backup_dirty ? sizeof(*cmd2) : 0);
 
-	cmd1 = vmw_fifo_reserve(dev_priv, submit_size);
-	if (unlikely(!cmd1)) {
-		DRM_ERROR("Failed reserving FIFO space for surface "
-			  "binding.\n");
+	cmd1 = VMW_FIFO_RESERVE(dev_priv, submit_size);
+	if (unlikely(!cmd1))
 		return -ENOMEM;
-	}
 
 	cmd1->header.id = SVGA_3D_CMD_BIND_GB_SURFACE;
 	cmd1->header.size = sizeof(cmd1->body);
@@ -1221,12 +1204,9 @@ static int vmw_gb_surface_unbind(struct vmw_resource *res,
 	BUG_ON(bo->mem.mem_type != VMW_PL_MOB);
 
 	submit_size = sizeof(*cmd3) + (readback ? sizeof(*cmd1) : sizeof(*cmd2));
-	cmd = vmw_fifo_reserve(dev_priv, submit_size);
-	if (unlikely(!cmd)) {
-		DRM_ERROR("Failed reserving FIFO space for surface "
-			  "unbinding.\n");
+	cmd = VMW_FIFO_RESERVE(dev_priv, submit_size);
+	if (unlikely(!cmd))
 		return -ENOMEM;
-	}
 
 	if (readback) {
 		cmd1 = (void *) cmd;
@@ -1280,10 +1260,8 @@ static int vmw_gb_surface_destroy(struct vmw_resource *res)
 	vmw_view_surface_list_destroy(dev_priv, &srf->view_list);
 	vmw_binding_res_list_scrub(&res->binding_head);
 
-	cmd = vmw_fifo_reserve(dev_priv, sizeof(*cmd));
+	cmd = VMW_FIFO_RESERVE(dev_priv, sizeof(*cmd));
 	if (unlikely(!cmd)) {
-		DRM_ERROR("Failed reserving FIFO space for surface "
-			  "destruction.\n");
 		mutex_unlock(&dev_priv->binding_mutex);
 		return -ENOMEM;
 	}
@@ -1405,16 +1383,16 @@ int vmw_surface_gb_priv_define(struct drm_device *dev,
 
 	if (for_scanout) {
 		if (!svga3dsurface_is_screen_target_format(format)) {
-			DRM_ERROR("Invalid Screen Target surface format.");
+			VMW_DEBUG_USER("Invalid Screen Target surface format.");
 			return -EINVAL;
 		}
 
 		if (size.width > dev_priv->texture_max_width ||
 		    size.height > dev_priv->texture_max_height) {
-			DRM_ERROR("%ux%u\n, exceeds max surface size %ux%u",
-				  size.width, size.height,
-				  dev_priv->texture_max_width,
-				  dev_priv->texture_max_height);
+			VMW_DEBUG_USER("%ux%u\n, exceeds max surface size %ux%u",
+				       size.width, size.height,
+				       dev_priv->texture_max_width,
+				       dev_priv->texture_max_height);
 			return -EINVAL;
 		}
 	} else {
@@ -1422,14 +1400,14 @@ int vmw_surface_gb_priv_define(struct drm_device *dev,
 
 		desc = svga3dsurface_get_desc(format);
 		if (unlikely(desc->block_desc == SVGA3DBLOCKDESC_NONE)) {
-			DRM_ERROR("Invalid surface format.\n");
+			VMW_DEBUG_USER("Invalid surface format.\n");
 			return -EINVAL;
 		}
 	}
 
 	/* array_size must be null for non-GL3 host. */
 	if (array_size > 0 && !dev_priv->has_dx) {
-		DRM_ERROR("Tried to create DX surface on non-DX host.\n");
+		VMW_DEBUG_USER("Tried to create DX surface on non-DX host.\n");
 		return -EINVAL;
 	}
 
@@ -1651,7 +1629,7 @@ vmw_gb_surface_define_internal(struct drm_device *dev,
 		if (ret == 0) {
 			if (res->backup->base.num_pages * PAGE_SIZE <
 			    res->backup_size) {
-				DRM_ERROR("Surface backup buffer too small.\n");
+				VMW_DEBUG_USER("Surface backup buffer too small.\n");
 				vmw_bo_unreference(&res->backup);
 				ret = -EINVAL;
 				goto out_unlock;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_glue.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_glue.c
index e6d75e377dd8..8bafa6eac5a8 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_glue.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ttm_glue.c
@@ -30,16 +30,9 @@
 
 int vmw_mmap(struct file *filp, struct vm_area_struct *vma)
 {
-	struct drm_file *file_priv;
-	struct vmw_private *dev_priv;
+	struct drm_file *file_priv = filp->private_data;
+	struct vmw_private *dev_priv = vmw_priv(file_priv->minor->dev);
 
-	if (unlikely(vma->vm_pgoff < VMWGFX_FILE_PAGE_OFFSET)) {
-		DRM_ERROR("Illegal attempt to mmap old fifo space.\n");
-		return -EINVAL;
-	}
-
-	file_priv = filp->private_data;
-	dev_priv = vmw_priv(file_priv->minor->dev);
 	return ttm_bo_mmap(filp, vma, &dev_priv->bdev);
 }
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_validation.c b/drivers/gpu/drm/vmwgfx/vmwgfx_validation.c
index e9944ac2e057..f611b2290a1b 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_validation.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_validation.c
@@ -76,6 +76,8 @@ struct vmw_validation_res_node {
 	u32 switching_backup : 1;
 	u32 first_usage : 1;
 	u32 reserved : 1;
+	u32 dirty : 1;
+	u32 dirty_set : 1;
 	unsigned long private[0];
 };
 
@@ -299,6 +301,7 @@ int vmw_validation_add_bo(struct vmw_validation_context *ctx,
  * @ctx: The validation context.
  * @res: The resource.
  * @priv_size: Size of private, additional metadata.
+ * @dirty: Whether to change dirty status.
  * @p_node: Output pointer of additional metadata address.
  * @first_usage: Whether this was the first time this resource was seen.
  *
@@ -307,6 +310,7 @@ int vmw_validation_add_bo(struct vmw_validation_context *ctx,
 int vmw_validation_add_resource(struct vmw_validation_context *ctx,
 				struct vmw_resource *res,
 				size_t priv_size,
+				u32 dirty,
 				void **p_node,
 				bool *first_usage)
 {
@@ -321,8 +325,7 @@ int vmw_validation_add_resource(struct vmw_validation_context *ctx,
 
 	node = vmw_validation_mem_alloc(ctx, sizeof(*node) + priv_size);
 	if (!node) {
-		DRM_ERROR("Failed to allocate a resource validation "
-			  "entry.\n");
+		VMW_DEBUG_USER("Failed to allocate a resource validation entry.\n");
 		return -ENOMEM;
 	}
 
@@ -358,6 +361,11 @@ int vmw_validation_add_resource(struct vmw_validation_context *ctx,
 	}
 
 out_fill:
+	if (dirty) {
+		node->dirty_set = 1;
+		/* Overwriting previous information here is intentional! */
+		node->dirty = (dirty & VMW_RES_DIRTY_SET) ? 1 : 0;
+	}
 	if (first_usage)
 		*first_usage = node->first_usage;
 	if (p_node)
@@ -367,6 +375,29 @@ out_fill:
 }
 
 /**
+ * vmw_validation_res_set_dirty - Register a resource dirty set or clear during
+ * validation.
+ * @ctx: The validation context.
+ * @val_private: The additional meta-data pointer returned when the
+ * resource was registered with the validation context. Used to identify
+ * the resource.
+ * @dirty: Dirty information VMW_RES_DIRTY_XX
+ */
+void vmw_validation_res_set_dirty(struct vmw_validation_context *ctx,
+				  void *val_private, u32 dirty)
+{
+	struct vmw_validation_res_node *val;
+
+	if (!dirty)
+		return;
+
+	val = container_of(val_private, typeof(*val), private);
+	val->dirty_set = 1;
+	/* Overwriting previous information here is intentional! */
+	val->dirty = (dirty & VMW_RES_DIRTY_SET) ? 1 : 0;
+}
+
+/**
  * vmw_validation_res_switch_backup - Register a backup MOB switch during
  * validation.
  * @ctx: The validation context.
@@ -450,15 +481,23 @@ void vmw_validation_res_unreserve(struct vmw_validation_context *ctx,
 	struct vmw_validation_res_node *val;
 
 	list_splice_init(&ctx->resource_ctx_list, &ctx->resource_list);
-
-	list_for_each_entry(val, &ctx->resource_list, head) {
-		if (val->reserved)
-			vmw_resource_unreserve(val->res,
-					       !backoff &&
-					       val->switching_backup,
-					       val->new_backup,
-					       val->new_backup_offset);
-	}
+	if (backoff)
+		list_for_each_entry(val, &ctx->resource_list, head) {
+			if (val->reserved)
+				vmw_resource_unreserve(val->res,
+						       false, false, false,
+						       NULL, 0);
+		}
+	else
+		list_for_each_entry(val, &ctx->resource_list, head) {
+			if (val->reserved)
+				vmw_resource_unreserve(val->res,
+						       val->dirty_set,
+						       val->dirty,
+						       val->switching_backup,
+						       val->new_backup,
+						       val->new_backup_offset);
+		}
 }
 
 /**
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_validation.h b/drivers/gpu/drm/vmwgfx/vmwgfx_validation.h
index 3b396fea40d7..523f6ac5c335 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_validation.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_validation.h
@@ -33,6 +33,10 @@
 #include <linux/ww_mutex.h>
 #include <drm/ttm/ttm_execbuf_util.h>
 
+#define VMW_RES_DIRTY_NONE 0
+#define VMW_RES_DIRTY_SET BIT(0)
+#define VMW_RES_DIRTY_CLEAR BIT(1)
+
 /**
  * struct vmw_validation_mem - Custom interface to provide memory reservations
  * for the validation code.
@@ -237,6 +241,7 @@ void vmw_validation_unref_lists(struct vmw_validation_context *ctx);
 int vmw_validation_add_resource(struct vmw_validation_context *ctx,
 				struct vmw_resource *res,
 				size_t priv_size,
+				u32 dirty,
 				void **p_node,
 				bool *first_usage);
 void vmw_validation_drop_ht(struct vmw_validation_context *ctx);
@@ -261,4 +266,6 @@ void *vmw_validation_mem_alloc(struct vmw_validation_context *ctx,
 int vmw_validation_preload_bo(struct vmw_validation_context *ctx);
 int vmw_validation_preload_res(struct vmw_validation_context *ctx,
 			       unsigned int size);
+void vmw_validation_res_set_dirty(struct vmw_validation_context *ctx,
+				  void *val_private, u32 dirty);
 #endif
diff --git a/drivers/gpu/drm/xen/xen_drm_front.c b/drivers/gpu/drm/xen/xen_drm_front.c
index 3e78a832d7f9..84aa4d61dc42 100644
--- a/drivers/gpu/drm/xen/xen_drm_front.c
+++ b/drivers/gpu/drm/xen/xen_drm_front.c
@@ -582,6 +582,7 @@ static void xen_drm_drv_fini(struct xen_drm_front_info *front_info)
 
 	drm_kms_helper_poll_fini(dev);
 	drm_dev_unplug(dev);
+	drm_dev_put(dev);
 
 	front_info->drm_info = NULL;
 
diff --git a/drivers/i2c/algos/i2c-algo-bit.c b/drivers/i2c/algos/i2c-algo-bit.c
index 5e5990a83da5..913db013fe90 100644
--- a/drivers/i2c/algos/i2c-algo-bit.c
+++ b/drivers/i2c/algos/i2c-algo-bit.c
@@ -603,6 +603,23 @@ bailout:
 	return ret;
 }
 
+/*
+ * We print a warning when we are not flagged to support atomic transfers but
+ * will try anyhow. That's what the I2C core would do as well. Sadly, we can't
+ * modify the algorithm struct at probe time because this struct is exported
+ * 'const'.
+ */
+static int bit_xfer_atomic(struct i2c_adapter *i2c_adap, struct i2c_msg msgs[],
+			   int num)
+{
+	struct i2c_algo_bit_data *adap = i2c_adap->algo_data;
+
+	if (!adap->can_do_atomic)
+		dev_warn(&i2c_adap->dev, "not flagged for atomic transfers\n");
+
+	return bit_xfer(i2c_adap, msgs, num);
+}
+
 static u32 bit_func(struct i2c_adapter *adap)
 {
 	return I2C_FUNC_I2C | I2C_FUNC_NOSTART | I2C_FUNC_SMBUS_EMUL |
@@ -615,8 +632,9 @@ static u32 bit_func(struct i2c_adapter *adap)
 /* -----exported algorithm data: -------------------------------------	*/
 
 const struct i2c_algorithm i2c_bit_algo = {
-	.master_xfer	= bit_xfer,
-	.functionality	= bit_func,
+	.master_xfer = bit_xfer,
+	.master_xfer_atomic = bit_xfer_atomic,
+	.functionality = bit_func,
 };
 EXPORT_SYMBOL(i2c_bit_algo);
 
diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig
index f8979abb9a19..26186439db6b 100644
--- a/drivers/i2c/busses/Kconfig
+++ b/drivers/i2c/busses/Kconfig
@@ -77,6 +77,16 @@ config I2C_AMD8111
 	  This driver can also be built as a module.  If so, the module
 	  will be called i2c-amd8111.
 
+config I2C_AMD_MP2
+	tristate "AMD MP2 PCIe"
+	depends on PCI && ACPI
+	help
+	  If you say yes to this option, support will be included for the AMD
+	  MP2 PCIe I2C adapter.
+
+	  This driver can also be built as modules.  If so, the modules will
+	  be called i2c-amd-mp2-pci and i2c-amd-mp2-plat.
+
 config I2C_HIX5HD2
 	tristate "Hix5hd2 high-speed I2C driver"
 	depends on ARCH_HISI || ARCH_HIX5HD2 || COMPILE_TEST
@@ -176,6 +186,7 @@ config I2C_PIIX4
 	    AMD Hudson-2
 	    AMD ML
 	    AMD CZ
+	    Hygon CZ
 	    Serverworks OSB4
 	    Serverworks CSB5
 	    Serverworks CSB6
@@ -388,6 +399,19 @@ config I2C_AT91
 	  the latency to fill the transmission register is too long. If you
 	  are facing this situation, use the i2c-gpio driver.
 
+config I2C_AT91_SLAVE_EXPERIMENTAL
+	tristate "Microchip AT91 I2C experimental slave mode"
+	depends on I2C_AT91
+	select I2C_SLAVE
+	help
+	  If you say yes to this option, support for the slave mode will be
+	  added. Caution: do not use it for production. This feature has not
+	  been tested in a heavy way, help wanted.
+	  There are known bugs:
+	    - It can hang, on a SAMA5D4, after several transfers.
+	    - There are some mismtaches with a SAMA5D4 as slave and a SAMA5D2 as
+	    master.
+
 config I2C_AU1550
 	tristate "Au1550/Au1200/Au1300 SMBus interface"
 	depends on MIPS_ALCHEMY
@@ -425,6 +449,7 @@ config I2C_BCM_IPROC
 	tristate "Broadcom iProc I2C controller"
 	depends on ARCH_BCM_IPROC || COMPILE_TEST
 	default ARCH_BCM_IPROC
+	select I2C_SLAVE
 	help
 	  If you say yes to this option, support will be included for the
 	  Broadcom iProc I2C controller.
diff --git a/drivers/i2c/busses/Makefile b/drivers/i2c/busses/Makefile
index 5f0cb6915969..a3245231b0b7 100644
--- a/drivers/i2c/busses/Makefile
+++ b/drivers/i2c/busses/Makefile
@@ -33,8 +33,13 @@ obj-$(CONFIG_I2C_POWERMAC)	+= i2c-powermac.o
 
 # Embedded system I2C/SMBus host controller drivers
 obj-$(CONFIG_I2C_ALTERA)	+= i2c-altera.o
+obj-$(CONFIG_I2C_AMD_MP2)	+= i2c-amd-mp2-pci.o i2c-amd-mp2-plat.o
 obj-$(CONFIG_I2C_ASPEED)	+= i2c-aspeed.o
 obj-$(CONFIG_I2C_AT91)		+= i2c-at91.o
+i2c-at91-objs			:= i2c-at91-core.o i2c-at91-master.o
+ifeq ($(CONFIG_I2C_AT91_SLAVE_EXPERIMENTAL),y)
+	i2c-at91-objs		+= i2c-at91-slave.o
+endif
 obj-$(CONFIG_I2C_AU1550)	+= i2c-au1550.o
 obj-$(CONFIG_I2C_AXXIA)		+= i2c-axxia.o
 obj-$(CONFIG_I2C_BCM2835)	+= i2c-bcm2835.o
diff --git a/drivers/i2c/busses/i2c-amd-mp2-pci.c b/drivers/i2c/busses/i2c-amd-mp2-pci.c
new file mode 100644
index 000000000000..455e1f36a2a3
--- /dev/null
+++ b/drivers/i2c/busses/i2c-amd-mp2-pci.c
@@ -0,0 +1,483 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * AMD MP2 PCIe communication driver
+ *
+ * Authors: Shyam Sundar S K <Shyam-sundar.S-k@amd.com>
+ *          Elie Morisse <syniurge@gmail.com>
+ */
+
+#include <linux/dma-mapping.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/slab.h>
+
+#include "i2c-amd-mp2.h"
+
+#include <linux/io-64-nonatomic-lo-hi.h>
+
+static void amd_mp2_c2p_mutex_lock(struct amd_i2c_common *i2c_common)
+{
+	struct amd_mp2_dev *privdata = i2c_common->mp2_dev;
+
+	/* there is only one data mailbox for two i2c adapters */
+	mutex_lock(&privdata->c2p_lock);
+	privdata->c2p_lock_busid = i2c_common->bus_id;
+}
+
+static void amd_mp2_c2p_mutex_unlock(struct amd_i2c_common *i2c_common)
+{
+	struct amd_mp2_dev *privdata = i2c_common->mp2_dev;
+
+	if (unlikely(privdata->c2p_lock_busid != i2c_common->bus_id)) {
+		dev_warn(ndev_dev(privdata),
+			 "bus %d attempting to unlock C2P locked by bus %d\n",
+			 i2c_common->bus_id, privdata->c2p_lock_busid);
+		return;
+	}
+
+	mutex_unlock(&privdata->c2p_lock);
+}
+
+static int amd_mp2_cmd(struct amd_i2c_common *i2c_common,
+		       union i2c_cmd_base i2c_cmd_base)
+{
+	struct amd_mp2_dev *privdata = i2c_common->mp2_dev;
+	void __iomem *reg;
+
+	i2c_common->reqcmd = i2c_cmd_base.s.i2c_cmd;
+
+	reg = privdata->mmio + ((i2c_cmd_base.s.bus_id == 1) ?
+				AMD_C2P_MSG1 : AMD_C2P_MSG0);
+	writel(i2c_cmd_base.ul, reg);
+
+	return 0;
+}
+
+int amd_mp2_bus_enable_set(struct amd_i2c_common *i2c_common, bool enable)
+{
+	struct amd_mp2_dev *privdata = i2c_common->mp2_dev;
+	union i2c_cmd_base i2c_cmd_base;
+
+	dev_dbg(ndev_dev(privdata), "%s id: %d\n", __func__,
+		i2c_common->bus_id);
+
+	i2c_cmd_base.ul = 0;
+	i2c_cmd_base.s.i2c_cmd = enable ? i2c_enable : i2c_disable;
+	i2c_cmd_base.s.bus_id = i2c_common->bus_id;
+	i2c_cmd_base.s.i2c_speed = i2c_common->i2c_speed;
+
+	amd_mp2_c2p_mutex_lock(i2c_common);
+
+	return amd_mp2_cmd(i2c_common, i2c_cmd_base);
+}
+EXPORT_SYMBOL_GPL(amd_mp2_bus_enable_set);
+
+static void amd_mp2_cmd_rw_fill(struct amd_i2c_common *i2c_common,
+				union i2c_cmd_base *i2c_cmd_base,
+				enum i2c_cmd reqcmd)
+{
+	i2c_cmd_base->s.i2c_cmd = reqcmd;
+	i2c_cmd_base->s.bus_id = i2c_common->bus_id;
+	i2c_cmd_base->s.i2c_speed = i2c_common->i2c_speed;
+	i2c_cmd_base->s.slave_addr = i2c_common->msg->addr;
+	i2c_cmd_base->s.length = i2c_common->msg->len;
+}
+
+int amd_mp2_rw(struct amd_i2c_common *i2c_common, enum i2c_cmd reqcmd)
+{
+	struct amd_mp2_dev *privdata = i2c_common->mp2_dev;
+	union i2c_cmd_base i2c_cmd_base;
+
+	amd_mp2_cmd_rw_fill(i2c_common, &i2c_cmd_base, reqcmd);
+	amd_mp2_c2p_mutex_lock(i2c_common);
+
+	if (i2c_common->msg->len <= 32) {
+		i2c_cmd_base.s.mem_type = use_c2pmsg;
+		if (reqcmd == i2c_write)
+			memcpy_toio(privdata->mmio + AMD_C2P_MSG2,
+				    i2c_common->msg->buf,
+				    i2c_common->msg->len);
+	} else {
+		i2c_cmd_base.s.mem_type = use_dram;
+		writeq((u64)i2c_common->dma_addr,
+		       privdata->mmio + AMD_C2P_MSG2);
+	}
+
+	return amd_mp2_cmd(i2c_common, i2c_cmd_base);
+}
+EXPORT_SYMBOL_GPL(amd_mp2_rw);
+
+static void amd_mp2_pci_check_rw_event(struct amd_i2c_common *i2c_common)
+{
+	struct amd_mp2_dev *privdata = i2c_common->mp2_dev;
+	int len = i2c_common->eventval.r.length;
+	u32 slave_addr = i2c_common->eventval.r.slave_addr;
+	bool err = false;
+
+	if (unlikely(len != i2c_common->msg->len)) {
+		dev_err(ndev_dev(privdata),
+			"length %d in event doesn't match buffer length %d!\n",
+			len, i2c_common->msg->len);
+		err = true;
+	}
+
+	if (unlikely(slave_addr != i2c_common->msg->addr)) {
+		dev_err(ndev_dev(privdata),
+			"unexpected slave address %x (expected: %x)!\n",
+			slave_addr, i2c_common->msg->addr);
+		err = true;
+	}
+
+	if (!err)
+		i2c_common->cmd_success = true;
+}
+
+static void __amd_mp2_process_event(struct amd_i2c_common *i2c_common)
+{
+	struct amd_mp2_dev *privdata = i2c_common->mp2_dev;
+	enum status_type sts = i2c_common->eventval.r.status;
+	enum response_type res = i2c_common->eventval.r.response;
+	int len = i2c_common->eventval.r.length;
+
+	if (res != command_success) {
+		if (res != command_failed)
+			dev_err(ndev_dev(privdata), "invalid response to i2c command!\n");
+		return;
+	}
+
+	switch (i2c_common->reqcmd) {
+	case i2c_read:
+		if (sts == i2c_readcomplete_event) {
+			amd_mp2_pci_check_rw_event(i2c_common);
+			if (len <= 32)
+				memcpy_fromio(i2c_common->msg->buf,
+					      privdata->mmio + AMD_C2P_MSG2,
+					      len);
+		} else if (sts != i2c_readfail_event) {
+			dev_err(ndev_dev(privdata),
+				"invalid i2c status after read (%d)!\n", sts);
+		}
+		break;
+	case i2c_write:
+		if (sts == i2c_writecomplete_event)
+			amd_mp2_pci_check_rw_event(i2c_common);
+		else if (sts != i2c_writefail_event)
+			dev_err(ndev_dev(privdata),
+				"invalid i2c status after write (%d)!\n", sts);
+		break;
+	case i2c_enable:
+		if (sts == i2c_busenable_complete)
+			i2c_common->cmd_success = true;
+		else if (sts != i2c_busenable_failed)
+			dev_err(ndev_dev(privdata),
+				"invalid i2c status after bus enable (%d)!\n",
+				sts);
+		break;
+	case i2c_disable:
+		if (sts == i2c_busdisable_complete)
+			i2c_common->cmd_success = true;
+		else if (sts != i2c_busdisable_failed)
+			dev_err(ndev_dev(privdata),
+				"invalid i2c status after bus disable (%d)!\n",
+				sts);
+		break;
+	default:
+		break;
+	}
+}
+
+void amd_mp2_process_event(struct amd_i2c_common *i2c_common)
+{
+	struct amd_mp2_dev *privdata = i2c_common->mp2_dev;
+
+	if (unlikely(i2c_common->reqcmd == i2c_none)) {
+		dev_warn(ndev_dev(privdata),
+			 "received msg but no cmd was sent (bus = %d)!\n",
+			 i2c_common->bus_id);
+		return;
+	}
+
+	__amd_mp2_process_event(i2c_common);
+
+	i2c_common->reqcmd = i2c_none;
+	amd_mp2_c2p_mutex_unlock(i2c_common);
+}
+EXPORT_SYMBOL_GPL(amd_mp2_process_event);
+
+static irqreturn_t amd_mp2_irq_isr(int irq, void *dev)
+{
+	struct amd_mp2_dev *privdata = dev;
+	struct amd_i2c_common *i2c_common;
+	u32 val;
+	unsigned int bus_id;
+	void __iomem *reg;
+	enum irqreturn ret = IRQ_NONE;
+
+	for (bus_id = 0; bus_id < 2; bus_id++) {
+		i2c_common = privdata->busses[bus_id];
+		if (!i2c_common)
+			continue;
+
+		reg = privdata->mmio + ((bus_id == 0) ?
+					AMD_P2C_MSG1 : AMD_P2C_MSG2);
+		val = readl(reg);
+		if (val != 0) {
+			writel(0, reg);
+			writel(0, privdata->mmio + AMD_P2C_MSG_INTEN);
+			i2c_common->eventval.ul = val;
+			i2c_common->cmd_completion(i2c_common);
+
+			ret = IRQ_HANDLED;
+		}
+	}
+
+	if (ret != IRQ_HANDLED) {
+		val = readl(privdata->mmio + AMD_P2C_MSG_INTEN);
+		if (val != 0) {
+			writel(0, privdata->mmio + AMD_P2C_MSG_INTEN);
+			dev_warn(ndev_dev(privdata),
+				 "received irq without message\n");
+			ret = IRQ_HANDLED;
+		}
+	}
+
+	return ret;
+}
+
+void amd_mp2_rw_timeout(struct amd_i2c_common *i2c_common)
+{
+	i2c_common->reqcmd = i2c_none;
+	amd_mp2_c2p_mutex_unlock(i2c_common);
+}
+EXPORT_SYMBOL_GPL(amd_mp2_rw_timeout);
+
+int amd_mp2_register_cb(struct amd_i2c_common *i2c_common)
+{
+	struct amd_mp2_dev *privdata = i2c_common->mp2_dev;
+
+	if (i2c_common->bus_id > 1)
+		return -EINVAL;
+
+	if (privdata->busses[i2c_common->bus_id]) {
+		dev_err(ndev_dev(privdata),
+			"Bus %d already taken!\n", i2c_common->bus_id);
+		return -EINVAL;
+	}
+
+	privdata->busses[i2c_common->bus_id] = i2c_common;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(amd_mp2_register_cb);
+
+int amd_mp2_unregister_cb(struct amd_i2c_common *i2c_common)
+{
+	struct amd_mp2_dev *privdata = i2c_common->mp2_dev;
+
+	privdata->busses[i2c_common->bus_id] = NULL;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(amd_mp2_unregister_cb);
+
+static void amd_mp2_clear_reg(struct amd_mp2_dev *privdata)
+{
+	int reg;
+
+	for (reg = AMD_C2P_MSG0; reg <= AMD_C2P_MSG9; reg += 4)
+		writel(0, privdata->mmio + reg);
+
+	for (reg = AMD_P2C_MSG1; reg <= AMD_P2C_MSG2; reg += 4)
+		writel(0, privdata->mmio + reg);
+}
+
+static int amd_mp2_pci_init(struct amd_mp2_dev *privdata,
+			    struct pci_dev *pci_dev)
+{
+	int rc;
+
+	pci_set_drvdata(pci_dev, privdata);
+
+	rc = pcim_enable_device(pci_dev);
+	if (rc) {
+		dev_err(ndev_dev(privdata), "Failed to enable MP2 PCI device\n");
+		goto err_pci_enable;
+	}
+
+	rc = pcim_iomap_regions(pci_dev, 1 << 2, pci_name(pci_dev));
+	if (rc) {
+		dev_err(ndev_dev(privdata), "I/O memory remapping failed\n");
+		goto err_pci_enable;
+	}
+	privdata->mmio = pcim_iomap_table(pci_dev)[2];
+
+	pci_set_master(pci_dev);
+
+	rc = pci_set_dma_mask(pci_dev, DMA_BIT_MASK(64));
+	if (rc) {
+		rc = pci_set_dma_mask(pci_dev, DMA_BIT_MASK(32));
+		if (rc)
+			goto err_dma_mask;
+	}
+
+	/* Set up intx irq */
+	writel(0, privdata->mmio + AMD_P2C_MSG_INTEN);
+	pci_intx(pci_dev, 1);
+	rc = devm_request_irq(&pci_dev->dev, pci_dev->irq, amd_mp2_irq_isr,
+			      IRQF_SHARED, dev_name(&pci_dev->dev), privdata);
+	if (rc)
+		dev_err(&pci_dev->dev, "Failure requesting irq %i: %d\n",
+			pci_dev->irq, rc);
+
+	return rc;
+
+err_dma_mask:
+	pci_clear_master(pci_dev);
+err_pci_enable:
+	pci_set_drvdata(pci_dev, NULL);
+	return rc;
+}
+
+static int amd_mp2_pci_probe(struct pci_dev *pci_dev,
+			     const struct pci_device_id *id)
+{
+	struct amd_mp2_dev *privdata;
+	int rc;
+
+	privdata = devm_kzalloc(&pci_dev->dev, sizeof(*privdata), GFP_KERNEL);
+	if (!privdata)
+		return -ENOMEM;
+
+	rc = amd_mp2_pci_init(privdata, pci_dev);
+	if (rc)
+		return rc;
+
+	mutex_init(&privdata->c2p_lock);
+	privdata->pci_dev = pci_dev;
+
+	pm_runtime_set_autosuspend_delay(&pci_dev->dev, 1000);
+	pm_runtime_use_autosuspend(&pci_dev->dev);
+	pm_runtime_put_autosuspend(&pci_dev->dev);
+	pm_runtime_allow(&pci_dev->dev);
+
+	privdata->probed = true;
+
+	dev_info(&pci_dev->dev, "MP2 device registered.\n");
+	return 0;
+}
+
+static void amd_mp2_pci_remove(struct pci_dev *pci_dev)
+{
+	struct amd_mp2_dev *privdata = pci_get_drvdata(pci_dev);
+
+	pm_runtime_forbid(&pci_dev->dev);
+	pm_runtime_get_noresume(&pci_dev->dev);
+
+	pci_intx(pci_dev, 0);
+	pci_clear_master(pci_dev);
+
+	amd_mp2_clear_reg(privdata);
+}
+
+#ifdef CONFIG_PM
+static int amd_mp2_pci_suspend(struct device *dev)
+{
+	struct pci_dev *pci_dev = to_pci_dev(dev);
+	struct amd_mp2_dev *privdata = pci_get_drvdata(pci_dev);
+	struct amd_i2c_common *i2c_common;
+	unsigned int bus_id;
+	int ret = 0;
+
+	for (bus_id = 0; bus_id < 2; bus_id++) {
+		i2c_common = privdata->busses[bus_id];
+		if (i2c_common)
+			i2c_common->suspend(i2c_common);
+	}
+
+	ret = pci_save_state(pci_dev);
+	if (ret) {
+		dev_err(ndev_dev(privdata),
+			"pci_save_state failed = %d\n", ret);
+		return ret;
+	}
+
+	pci_disable_device(pci_dev);
+	return ret;
+}
+
+static int amd_mp2_pci_resume(struct device *dev)
+{
+	struct pci_dev *pci_dev = to_pci_dev(dev);
+	struct amd_mp2_dev *privdata = pci_get_drvdata(pci_dev);
+	struct amd_i2c_common *i2c_common;
+	unsigned int bus_id;
+	int ret = 0;
+
+	pci_restore_state(pci_dev);
+	ret = pci_enable_device(pci_dev);
+	if (ret < 0) {
+		dev_err(ndev_dev(privdata),
+			"pci_enable_device failed = %d\n", ret);
+		return ret;
+	}
+
+	for (bus_id = 0; bus_id < 2; bus_id++) {
+		i2c_common = privdata->busses[bus_id];
+		if (i2c_common) {
+			ret = i2c_common->resume(i2c_common);
+			if (ret < 0)
+				return ret;
+		}
+	}
+
+	return ret;
+}
+
+static UNIVERSAL_DEV_PM_OPS(amd_mp2_pci_pm_ops, amd_mp2_pci_suspend,
+			    amd_mp2_pci_resume, NULL);
+#endif /* CONFIG_PM */
+
+static const struct pci_device_id amd_mp2_pci_tbl[] = {
+	{PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_MP2)},
+	{0}
+};
+MODULE_DEVICE_TABLE(pci, amd_mp2_pci_tbl);
+
+static struct pci_driver amd_mp2_pci_driver = {
+	.name		= "i2c_amd_mp2",
+	.id_table	= amd_mp2_pci_tbl,
+	.probe		= amd_mp2_pci_probe,
+	.remove		= amd_mp2_pci_remove,
+#ifdef CONFIG_PM
+	.driver = {
+		.pm	= &amd_mp2_pci_pm_ops,
+	},
+#endif
+};
+module_pci_driver(amd_mp2_pci_driver);
+
+static int amd_mp2_device_match(struct device *dev, void *data)
+{
+	return 1;
+}
+
+struct amd_mp2_dev *amd_mp2_find_device(void)
+{
+	struct device *dev;
+	struct pci_dev *pci_dev;
+
+	dev = driver_find_device(&amd_mp2_pci_driver.driver, NULL, NULL,
+				 amd_mp2_device_match);
+	if (!dev)
+		return NULL;
+
+	pci_dev = to_pci_dev(dev);
+	return (struct amd_mp2_dev *)pci_get_drvdata(pci_dev);
+}
+EXPORT_SYMBOL_GPL(amd_mp2_find_device);
+
+MODULE_DESCRIPTION("AMD(R) PCI-E MP2 I2C Controller Driver");
+MODULE_AUTHOR("Shyam Sundar S K <Shyam-sundar.S-k@amd.com>");
+MODULE_AUTHOR("Elie Morisse <syniurge@gmail.com>");
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/i2c/busses/i2c-amd-mp2-plat.c b/drivers/i2c/busses/i2c-amd-mp2-plat.c
new file mode 100644
index 000000000000..f5b3f00c6559
--- /dev/null
+++ b/drivers/i2c/busses/i2c-amd-mp2-plat.c
@@ -0,0 +1,367 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * AMD MP2 platform driver
+ *
+ * Setup the I2C adapters enumerated in the ACPI namespace.
+ * MP2 controllers have 2 separate busses, up to 2 I2C adapters may be listed.
+ *
+ * Authors: Nehal Bakulchandra Shah <Nehal-bakulchandra.shah@amd.com>
+ *          Elie Morisse <syniurge@gmail.com>
+ */
+
+#include <linux/acpi.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+
+#include "i2c-amd-mp2.h"
+
+#define AMD_MP2_I2C_MAX_RW_LENGTH ((1 << 12) - 1)
+#define AMD_I2C_TIMEOUT (msecs_to_jiffies(250))
+
+/**
+ * struct amd_i2c_dev - MP2 bus/i2c adapter context
+ * @common: shared context with the MP2 PCI driver
+ * @pdev: platform driver node
+ * @adap: i2c adapter
+ * @cmd_complete: xfer completion object
+ */
+struct amd_i2c_dev {
+	struct amd_i2c_common common;
+	struct platform_device *pdev;
+	struct i2c_adapter adap;
+	struct completion cmd_complete;
+};
+
+#define amd_i2c_dev_common(__common) \
+	container_of(__common, struct amd_i2c_dev, common)
+
+static int i2c_amd_dma_map(struct amd_i2c_common *i2c_common)
+{
+	struct device *dev_pci = &i2c_common->mp2_dev->pci_dev->dev;
+	struct amd_i2c_dev *i2c_dev = amd_i2c_dev_common(i2c_common);
+	enum dma_data_direction dma_direction =
+			i2c_common->msg->flags & I2C_M_RD ?
+			DMA_FROM_DEVICE : DMA_TO_DEVICE;
+
+	i2c_common->dma_buf = i2c_get_dma_safe_msg_buf(i2c_common->msg, 0);
+	i2c_common->dma_addr = dma_map_single(dev_pci, i2c_common->dma_buf,
+					      i2c_common->msg->len,
+					      dma_direction);
+
+	if (unlikely(dma_mapping_error(dev_pci, i2c_common->dma_addr))) {
+		dev_err(&i2c_dev->pdev->dev,
+			"Error while mapping dma buffer %p\n",
+			i2c_common->dma_buf);
+		return -EIO;
+	}
+
+	return 0;
+}
+
+static void i2c_amd_dma_unmap(struct amd_i2c_common *i2c_common)
+{
+	struct device *dev_pci = &i2c_common->mp2_dev->pci_dev->dev;
+	enum dma_data_direction dma_direction =
+			i2c_common->msg->flags & I2C_M_RD ?
+			DMA_FROM_DEVICE : DMA_TO_DEVICE;
+
+	dma_unmap_single(dev_pci, i2c_common->dma_addr,
+			 i2c_common->msg->len, dma_direction);
+
+	i2c_put_dma_safe_msg_buf(i2c_common->dma_buf, i2c_common->msg, true);
+}
+
+static void i2c_amd_start_cmd(struct amd_i2c_dev *i2c_dev)
+{
+	struct amd_i2c_common *i2c_common = &i2c_dev->common;
+
+	reinit_completion(&i2c_dev->cmd_complete);
+	i2c_common->cmd_success = false;
+}
+
+static void i2c_amd_cmd_completion(struct amd_i2c_common *i2c_common)
+{
+	struct amd_i2c_dev *i2c_dev = amd_i2c_dev_common(i2c_common);
+	union i2c_event *event = &i2c_common->eventval;
+
+	if (event->r.status == i2c_readcomplete_event)
+		dev_dbg(&i2c_dev->pdev->dev, "%s readdata:%*ph\n",
+			__func__, event->r.length,
+			i2c_common->msg->buf);
+
+	complete(&i2c_dev->cmd_complete);
+}
+
+static int i2c_amd_check_cmd_completion(struct amd_i2c_dev *i2c_dev)
+{
+	struct amd_i2c_common *i2c_common = &i2c_dev->common;
+	unsigned long timeout;
+
+	timeout = wait_for_completion_timeout(&i2c_dev->cmd_complete,
+					      i2c_dev->adap.timeout);
+
+	if ((i2c_common->reqcmd == i2c_read ||
+	     i2c_common->reqcmd == i2c_write) &&
+	    i2c_common->msg->len > 32)
+		i2c_amd_dma_unmap(i2c_common);
+
+	if (timeout == 0) {
+		amd_mp2_rw_timeout(i2c_common);
+		return -ETIMEDOUT;
+	}
+
+	amd_mp2_process_event(i2c_common);
+
+	if (!i2c_common->cmd_success)
+		return -EIO;
+
+	return 0;
+}
+
+static int i2c_amd_enable_set(struct amd_i2c_dev *i2c_dev, bool enable)
+{
+	struct amd_i2c_common *i2c_common = &i2c_dev->common;
+
+	i2c_amd_start_cmd(i2c_dev);
+	amd_mp2_bus_enable_set(i2c_common, enable);
+
+	return i2c_amd_check_cmd_completion(i2c_dev);
+}
+
+static int i2c_amd_xfer_msg(struct amd_i2c_dev *i2c_dev, struct i2c_msg *pmsg)
+{
+	struct amd_i2c_common *i2c_common = &i2c_dev->common;
+
+	i2c_amd_start_cmd(i2c_dev);
+	i2c_common->msg = pmsg;
+
+	if (pmsg->len > 32)
+		if (i2c_amd_dma_map(i2c_common))
+			return -EIO;
+
+	if (pmsg->flags & I2C_M_RD)
+		amd_mp2_rw(i2c_common, i2c_read);
+	else
+		amd_mp2_rw(i2c_common, i2c_write);
+
+	return i2c_amd_check_cmd_completion(i2c_dev);
+}
+
+static int i2c_amd_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs, int num)
+{
+	struct amd_i2c_dev *i2c_dev = i2c_get_adapdata(adap);
+	int i;
+	struct i2c_msg *pmsg;
+	int err;
+
+	/* the adapter might have been deleted while waiting for the bus lock */
+	if (unlikely(!i2c_dev->common.mp2_dev))
+		return -EINVAL;
+
+	amd_mp2_pm_runtime_get(i2c_dev->common.mp2_dev);
+
+	for (i = 0; i < num; i++) {
+		pmsg = &msgs[i];
+		err = i2c_amd_xfer_msg(i2c_dev, pmsg);
+		if (err)
+			break;
+	}
+
+	amd_mp2_pm_runtime_put(i2c_dev->common.mp2_dev);
+	return err ? err : num;
+}
+
+static u32 i2c_amd_func(struct i2c_adapter *a)
+{
+	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL;
+}
+
+static const struct i2c_algorithm i2c_amd_algorithm = {
+	.master_xfer = i2c_amd_xfer,
+	.functionality = i2c_amd_func,
+};
+
+#ifdef CONFIG_PM
+static int i2c_amd_suspend(struct amd_i2c_common *i2c_common)
+{
+	struct amd_i2c_dev *i2c_dev = amd_i2c_dev_common(i2c_common);
+
+	i2c_amd_enable_set(i2c_dev, false);
+	return 0;
+}
+
+static int i2c_amd_resume(struct amd_i2c_common *i2c_common)
+{
+	struct amd_i2c_dev *i2c_dev = amd_i2c_dev_common(i2c_common);
+
+	return i2c_amd_enable_set(i2c_dev, true);
+}
+#endif
+
+static enum speed_enum i2c_amd_get_bus_speed(struct platform_device *pdev)
+{
+	u32 acpi_speed;
+	int i;
+	static const u32 supported_speeds[] = {
+		0, 100000, 400000, 1000000, 1400000, 3400000
+	};
+
+	acpi_speed = i2c_acpi_find_bus_speed(&pdev->dev);
+	/* round down to the lowest standard speed */
+	for (i = 1; i < ARRAY_SIZE(supported_speeds); i++) {
+		if (acpi_speed < supported_speeds[i])
+			break;
+	}
+	acpi_speed = supported_speeds[i - 1];
+
+	switch (acpi_speed) {
+	case 100000:
+		return speed100k;
+	case 400000:
+		return speed400k;
+	case 1000000:
+		return speed1000k;
+	case 1400000:
+		return speed1400k;
+	case 3400000:
+		return speed3400k;
+	default:
+		return speed400k;
+	}
+}
+
+static const struct i2c_adapter_quirks amd_i2c_dev_quirks = {
+	.max_read_len = AMD_MP2_I2C_MAX_RW_LENGTH,
+	.max_write_len = AMD_MP2_I2C_MAX_RW_LENGTH,
+};
+
+static int i2c_amd_probe(struct platform_device *pdev)
+{
+	int ret;
+	struct amd_i2c_dev *i2c_dev;
+	acpi_handle handle = ACPI_HANDLE(&pdev->dev);
+	struct acpi_device *adev;
+	struct amd_mp2_dev *mp2_dev;
+	const char *uid;
+
+	if (acpi_bus_get_device(handle, &adev))
+		return -ENODEV;
+
+	/* The ACPI namespace doesn't contain information about which MP2 PCI
+	 * device an AMDI0011 ACPI device is related to, so assume that there's
+	 * only one MP2 PCI device per system.
+	 */
+	mp2_dev = amd_mp2_find_device();
+	if (!mp2_dev || !mp2_dev->probed)
+		/* The MP2 PCI device should get probed later */
+		return -EPROBE_DEFER;
+
+	i2c_dev = devm_kzalloc(&pdev->dev, sizeof(*i2c_dev), GFP_KERNEL);
+	if (!i2c_dev)
+		return -ENOMEM;
+
+	i2c_dev->common.mp2_dev = mp2_dev;
+	i2c_dev->pdev = pdev;
+	platform_set_drvdata(pdev, i2c_dev);
+
+	i2c_dev->common.cmd_completion = &i2c_amd_cmd_completion;
+#ifdef CONFIG_PM
+	i2c_dev->common.suspend = &i2c_amd_suspend;
+	i2c_dev->common.resume = &i2c_amd_resume;
+#endif
+
+	uid = adev->pnp.unique_id;
+	if (!uid) {
+		dev_err(&pdev->dev, "missing UID/bus id!\n");
+		return -EINVAL;
+	} else if (strcmp(uid, "0") == 0) {
+		i2c_dev->common.bus_id = 0;
+	} else if (strcmp(uid, "1") == 0) {
+		i2c_dev->common.bus_id = 1;
+	} else {
+		dev_err(&pdev->dev, "incorrect UID/bus id \"%s\"!\n", uid);
+		return -EINVAL;
+	}
+	dev_dbg(&pdev->dev, "bus id is %u\n", i2c_dev->common.bus_id);
+
+	/* Register the adapter */
+	amd_mp2_pm_runtime_get(mp2_dev);
+
+	i2c_dev->common.reqcmd = i2c_none;
+	if (amd_mp2_register_cb(&i2c_dev->common))
+		return -EINVAL;
+	device_link_add(&i2c_dev->pdev->dev, &mp2_dev->pci_dev->dev,
+			DL_FLAG_AUTOREMOVE_CONSUMER);
+
+	i2c_dev->common.i2c_speed = i2c_amd_get_bus_speed(pdev);
+
+	/* Setup i2c adapter description */
+	i2c_dev->adap.owner = THIS_MODULE;
+	i2c_dev->adap.algo = &i2c_amd_algorithm;
+	i2c_dev->adap.quirks = &amd_i2c_dev_quirks;
+	i2c_dev->adap.dev.parent = &pdev->dev;
+	i2c_dev->adap.algo_data = i2c_dev;
+	i2c_dev->adap.timeout = AMD_I2C_TIMEOUT;
+	ACPI_COMPANION_SET(&i2c_dev->adap.dev, ACPI_COMPANION(&pdev->dev));
+	i2c_dev->adap.dev.of_node = pdev->dev.of_node;
+	snprintf(i2c_dev->adap.name, sizeof(i2c_dev->adap.name),
+		 "AMD MP2 i2c bus %u", i2c_dev->common.bus_id);
+	i2c_set_adapdata(&i2c_dev->adap, i2c_dev);
+
+	init_completion(&i2c_dev->cmd_complete);
+
+	/* Enable the bus */
+	if (i2c_amd_enable_set(i2c_dev, true))
+		dev_err(&pdev->dev, "initial bus enable failed\n");
+
+	/* Attach to the i2c layer */
+	ret = i2c_add_adapter(&i2c_dev->adap);
+
+	amd_mp2_pm_runtime_put(mp2_dev);
+
+	if (ret < 0)
+		dev_err(&pdev->dev, "i2c add adapter failed = %d\n", ret);
+
+	return ret;
+}
+
+static int i2c_amd_remove(struct platform_device *pdev)
+{
+	struct amd_i2c_dev *i2c_dev = platform_get_drvdata(pdev);
+	struct amd_i2c_common *i2c_common = &i2c_dev->common;
+
+	i2c_lock_bus(&i2c_dev->adap, I2C_LOCK_ROOT_ADAPTER);
+
+	i2c_amd_enable_set(i2c_dev, false);
+	amd_mp2_unregister_cb(i2c_common);
+	i2c_common->mp2_dev = NULL;
+
+	i2c_unlock_bus(&i2c_dev->adap, I2C_LOCK_ROOT_ADAPTER);
+
+	i2c_del_adapter(&i2c_dev->adap);
+	return 0;
+}
+
+static const struct acpi_device_id i2c_amd_acpi_match[] = {
+	{ "AMDI0011" },
+	{ },
+};
+MODULE_DEVICE_TABLE(acpi, i2c_amd_acpi_match);
+
+static struct platform_driver i2c_amd_plat_driver = {
+	.probe = i2c_amd_probe,
+	.remove = i2c_amd_remove,
+	.driver = {
+		.name = "i2c_amd_mp2",
+		.acpi_match_table = ACPI_PTR(i2c_amd_acpi_match),
+	},
+};
+module_platform_driver(i2c_amd_plat_driver);
+
+MODULE_DESCRIPTION("AMD(R) MP2 I2C Platform Driver");
+MODULE_AUTHOR("Nehal Shah <nehal-bakulchandra.shah@amd.com>");
+MODULE_AUTHOR("Elie Morisse <syniurge@gmail.com>");
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/i2c/busses/i2c-amd-mp2.h b/drivers/i2c/busses/i2c-amd-mp2.h
new file mode 100644
index 000000000000..058362edebaa
--- /dev/null
+++ b/drivers/i2c/busses/i2c-amd-mp2.h
@@ -0,0 +1,219 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
+/*
+ * AMD MP2 I2C adapter driver
+ *
+ * Authors: Shyam Sundar S K <Shyam-sundar.S-k@amd.com>
+ *          Elie Morisse <syniurge@gmail.com>
+ */
+
+#ifndef I2C_AMD_PCI_MP2_H
+#define I2C_AMD_PCI_MP2_H
+
+#include <linux/i2c.h>
+#include <linux/pci.h>
+#include <linux/pm_runtime.h>
+
+#define PCI_DEVICE_ID_AMD_MP2	0x15E6
+
+struct amd_i2c_common;
+struct amd_mp2_dev;
+
+enum {
+	/* MP2 C2P Message Registers */
+	AMD_C2P_MSG0 = 0x10500,			/* MP2 Message for I2C0 */
+	AMD_C2P_MSG1 = 0x10504,			/* MP2 Message for I2C1 */
+	AMD_C2P_MSG2 = 0x10508,			/* DRAM Address Lo / Data 0 */
+	AMD_C2P_MSG3 = 0x1050c,			/* DRAM Address HI / Data 1 */
+	AMD_C2P_MSG4 = 0x10510,			/* Data 2 */
+	AMD_C2P_MSG5 = 0x10514,			/* Data 3 */
+	AMD_C2P_MSG6 = 0x10518,			/* Data 4 */
+	AMD_C2P_MSG7 = 0x1051c,			/* Data 5 */
+	AMD_C2P_MSG8 = 0x10520,			/* Data 6 */
+	AMD_C2P_MSG9 = 0x10524,			/* Data 7 */
+
+	/* MP2 P2C Message Registers */
+	AMD_P2C_MSG0 = 0x10680,			/* Do not use */
+	AMD_P2C_MSG1 = 0x10684,			/* I2C0 interrupt register */
+	AMD_P2C_MSG2 = 0x10688,			/* I2C1 interrupt register */
+	AMD_P2C_MSG3 = 0x1068C,			/* MP2 debug info */
+	AMD_P2C_MSG_INTEN = 0x10690,		/* MP2 interrupt gen register */
+	AMD_P2C_MSG_INTSTS = 0x10694,		/* Interrupt status */
+};
+
+/* Command register data structures */
+
+#define i2c_none (-1)
+enum i2c_cmd {
+	i2c_read = 0,
+	i2c_write,
+	i2c_enable,
+	i2c_disable,
+	number_of_sensor_discovered,
+	is_mp2_active,
+	invalid_cmd = 0xF,
+};
+
+enum speed_enum {
+	speed100k = 0,
+	speed400k = 1,
+	speed1000k = 2,
+	speed1400k = 3,
+	speed3400k = 4
+};
+
+enum mem_type {
+	use_dram = 0,
+	use_c2pmsg = 1,
+};
+
+/**
+ * union i2c_cmd_base : bit access of C2P commands
+ * @i2c_cmd: bit 0..3 i2c R/W command
+ * @bus_id: bit 4..7 i2c bus index
+ * @slave_addr: bit 8..15 slave address
+ * @length: bit 16..27 read/write length
+ * @i2c_speed: bit 28..30 bus speed
+ * @mem_type: bit 31 0-DRAM; 1-C2P msg o/p
+ */
+union i2c_cmd_base {
+	u32 ul;
+	struct {
+		enum i2c_cmd i2c_cmd : 4;
+		u8 bus_id : 4;
+		u32 slave_addr : 8;
+		u32 length : 12;
+		enum speed_enum i2c_speed : 3;
+		enum mem_type mem_type : 1;
+	} s;
+};
+
+enum response_type {
+	invalid_response = 0,
+	command_success = 1,
+	command_failed = 2,
+};
+
+enum status_type {
+	i2c_readcomplete_event = 0,
+	i2c_readfail_event = 1,
+	i2c_writecomplete_event = 2,
+	i2c_writefail_event = 3,
+	i2c_busenable_complete = 4,
+	i2c_busenable_failed = 5,
+	i2c_busdisable_complete = 6,
+	i2c_busdisable_failed = 7,
+	invalid_data_length = 8,
+	invalid_slave_address = 9,
+	invalid_i2cbus_id = 10,
+	invalid_dram_addr = 11,
+	invalid_command = 12,
+	mp2_active = 13,
+	numberof_sensors_discovered_resp = 14,
+	i2c_bus_notinitialized
+};
+
+/**
+ * union i2c_event : bit access of P2C events
+ * @response: bit 0..1 i2c response type
+ * @status: bit 2..6 status_type
+ * @mem_type: bit 7 0-DRAM; 1-C2P msg o/p
+ * @bus_id: bit 8..11 i2c bus id
+ * @length: bit 12..23 message length
+ * @slave_addr: bit 24-31 slave address
+ */
+union i2c_event {
+	u32 ul;
+	struct {
+		enum response_type response : 2;
+		enum status_type status : 5;
+		enum mem_type mem_type : 1;
+		u8 bus_id : 4;
+		u32 length : 12;
+		u32 slave_addr : 8;
+	} r;
+};
+
+/**
+ * struct amd_i2c_common - per bus/i2c adapter context, shared
+ *	between the pci and the platform driver
+ * @eventval: MP2 event value set by the IRQ handler
+ * @mp2_dev: MP2 pci device this adapter is part of
+ * @msg: i2c message
+ * @cmd_completion: function called by the IRQ handler to signal
+ *		    the platform driver
+ * @reqcmd: requested i2c command type
+ * @cmd_success: set to true if the MP2 responded to a command with
+ *		 the expected status and response type
+ * @bus_id: bus index
+ * @i2c_speed: i2c bus speed determined by the slowest slave
+ * @dma_buf: if msg length > 32, holds the DMA buffer virtual address
+ * @dma_addr: if msg length > 32, holds the DMA buffer address
+ */
+struct amd_i2c_common {
+	union i2c_event eventval;
+	struct amd_mp2_dev *mp2_dev;
+	struct i2c_msg *msg;
+	void (*cmd_completion)(struct amd_i2c_common *i2c_common);
+	enum i2c_cmd reqcmd;
+	u8 cmd_success;
+	u8 bus_id;
+	enum speed_enum i2c_speed;
+	u8 *dma_buf;
+	dma_addr_t dma_addr;
+#ifdef CONFIG_PM
+	int (*suspend)(struct amd_i2c_common *i2c_common);
+	int (*resume)(struct amd_i2c_common *i2c_common);
+#endif /* CONFIG_PM */
+};
+
+/**
+ * struct amd_mp2_dev - per PCI device context
+ * @pci_dev: PCI driver node
+ * @busses: MP2 devices may have up to two busses,
+ *	    each bus corresponding to an i2c adapter
+ * @mmio: iommapped registers
+ * @c2p_lock: controls access to the C2P mailbox shared between
+ *	      the two adapters
+ * @c2p_lock_busid: id of the adapter which locked c2p_lock
+ */
+struct amd_mp2_dev {
+	struct pci_dev *pci_dev;
+	struct amd_i2c_common *busses[2];
+	void __iomem *mmio;
+	struct mutex c2p_lock;
+	u8 c2p_lock_busid;
+	unsigned int probed;
+};
+
+#define ndev_pdev(ndev) ((ndev)->pci_dev)
+#define ndev_name(ndev) pci_name(ndev_pdev(ndev))
+#define ndev_dev(ndev) (&ndev_pdev(ndev)->dev)
+#define work_amd_i2c_common(__work) \
+	container_of(__work, struct amd_i2c_common, work.work)
+
+/* PCIe communication driver */
+
+int amd_mp2_rw(struct amd_i2c_common *i2c_common, enum i2c_cmd reqcmd);
+int amd_mp2_bus_enable_set(struct amd_i2c_common *i2c_common, bool enable);
+
+void amd_mp2_process_event(struct amd_i2c_common *i2c_common);
+
+void amd_mp2_rw_timeout(struct amd_i2c_common *i2c_common);
+
+int amd_mp2_register_cb(struct amd_i2c_common *i2c_common);
+int amd_mp2_unregister_cb(struct amd_i2c_common *i2c_common);
+
+struct amd_mp2_dev *amd_mp2_find_device(void);
+
+static inline void amd_mp2_pm_runtime_get(struct amd_mp2_dev *mp2_dev)
+{
+	pm_runtime_get_sync(&mp2_dev->pci_dev->dev);
+}
+
+static inline void amd_mp2_pm_runtime_put(struct amd_mp2_dev *mp2_dev)
+{
+	pm_runtime_mark_last_busy(&mp2_dev->pci_dev->dev);
+	pm_runtime_put_autosuspend(&mp2_dev->pci_dev->dev);
+}
+
+#endif
diff --git a/drivers/i2c/busses/i2c-at91-core.c b/drivers/i2c/busses/i2c-at91-core.c
new file mode 100644
index 000000000000..8d55cdd69ff4
--- /dev/null
+++ b/drivers/i2c/busses/i2c-at91-core.c
@@ -0,0 +1,376 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  i2c Support for Atmel's AT91 Two-Wire Interface (TWI)
+ *
+ *  Copyright (C) 2011 Weinmann Medical GmbH
+ *  Author: Nikolaus Voss <n.voss@weinmann.de>
+ *
+ *  Evolved from original work by:
+ *  Copyright (C) 2004 Rick Bronson
+ *  Converted to 2.6 by Andrew Victor <andrew@sanpeople.com>
+ *
+ *  Borrowed heavily from original work by:
+ *  Copyright (C) 2000 Philip Edelbrock <phil@stimpy.netroedge.com>
+ */
+
+#include <linux/clk.h>
+#include <linux/err.h>
+#include <linux/i2c.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/pinctrl/consumer.h>
+
+#include "i2c-at91.h"
+
+unsigned at91_twi_read(struct at91_twi_dev *dev, unsigned reg)
+{
+	return readl_relaxed(dev->base + reg);
+}
+
+void at91_twi_write(struct at91_twi_dev *dev, unsigned reg, unsigned val)
+{
+	writel_relaxed(val, dev->base + reg);
+}
+
+void at91_disable_twi_interrupts(struct at91_twi_dev *dev)
+{
+	at91_twi_write(dev, AT91_TWI_IDR, AT91_TWI_INT_MASK);
+}
+
+void at91_twi_irq_save(struct at91_twi_dev *dev)
+{
+	dev->imr = at91_twi_read(dev, AT91_TWI_IMR) & AT91_TWI_INT_MASK;
+	at91_disable_twi_interrupts(dev);
+}
+
+void at91_twi_irq_restore(struct at91_twi_dev *dev)
+{
+	at91_twi_write(dev, AT91_TWI_IER, dev->imr);
+}
+
+void at91_init_twi_bus(struct at91_twi_dev *dev)
+{
+	at91_disable_twi_interrupts(dev);
+	at91_twi_write(dev, AT91_TWI_CR, AT91_TWI_SWRST);
+	if (dev->slave_detected)
+		at91_init_twi_bus_slave(dev);
+	else
+		at91_init_twi_bus_master(dev);
+}
+
+static struct at91_twi_pdata at91rm9200_config = {
+	.clk_max_div = 5,
+	.clk_offset = 3,
+	.has_unre_flag = true,
+	.has_alt_cmd = false,
+	.has_hold_field = false,
+};
+
+static struct at91_twi_pdata at91sam9261_config = {
+	.clk_max_div = 5,
+	.clk_offset = 4,
+	.has_unre_flag = false,
+	.has_alt_cmd = false,
+	.has_hold_field = false,
+};
+
+static struct at91_twi_pdata at91sam9260_config = {
+	.clk_max_div = 7,
+	.clk_offset = 4,
+	.has_unre_flag = false,
+	.has_alt_cmd = false,
+	.has_hold_field = false,
+};
+
+static struct at91_twi_pdata at91sam9g20_config = {
+	.clk_max_div = 7,
+	.clk_offset = 4,
+	.has_unre_flag = false,
+	.has_alt_cmd = false,
+	.has_hold_field = false,
+};
+
+static struct at91_twi_pdata at91sam9g10_config = {
+	.clk_max_div = 7,
+	.clk_offset = 4,
+	.has_unre_flag = false,
+	.has_alt_cmd = false,
+	.has_hold_field = false,
+};
+
+static const struct platform_device_id at91_twi_devtypes[] = {
+	{
+		.name = "i2c-at91rm9200",
+		.driver_data = (unsigned long) &at91rm9200_config,
+	}, {
+		.name = "i2c-at91sam9261",
+		.driver_data = (unsigned long) &at91sam9261_config,
+	}, {
+		.name = "i2c-at91sam9260",
+		.driver_data = (unsigned long) &at91sam9260_config,
+	}, {
+		.name = "i2c-at91sam9g20",
+		.driver_data = (unsigned long) &at91sam9g20_config,
+	}, {
+		.name = "i2c-at91sam9g10",
+		.driver_data = (unsigned long) &at91sam9g10_config,
+	}, {
+		/* sentinel */
+	}
+};
+
+#if defined(CONFIG_OF)
+static struct at91_twi_pdata at91sam9x5_config = {
+	.clk_max_div = 7,
+	.clk_offset = 4,
+	.has_unre_flag = false,
+	.has_alt_cmd = false,
+	.has_hold_field = false,
+};
+
+static struct at91_twi_pdata sama5d4_config = {
+	.clk_max_div = 7,
+	.clk_offset = 4,
+	.has_unre_flag = false,
+	.has_alt_cmd = false,
+	.has_hold_field = true,
+};
+
+static struct at91_twi_pdata sama5d2_config = {
+	.clk_max_div = 7,
+	.clk_offset = 4,
+	.has_unre_flag = true,
+	.has_alt_cmd = true,
+	.has_hold_field = true,
+};
+
+static const struct of_device_id atmel_twi_dt_ids[] = {
+	{
+		.compatible = "atmel,at91rm9200-i2c",
+		.data = &at91rm9200_config,
+	}, {
+		.compatible = "atmel,at91sam9260-i2c",
+		.data = &at91sam9260_config,
+	}, {
+		.compatible = "atmel,at91sam9261-i2c",
+		.data = &at91sam9261_config,
+	}, {
+		.compatible = "atmel,at91sam9g20-i2c",
+		.data = &at91sam9g20_config,
+	}, {
+		.compatible = "atmel,at91sam9g10-i2c",
+		.data = &at91sam9g10_config,
+	}, {
+		.compatible = "atmel,at91sam9x5-i2c",
+		.data = &at91sam9x5_config,
+	}, {
+		.compatible = "atmel,sama5d4-i2c",
+		.data = &sama5d4_config,
+	}, {
+		.compatible = "atmel,sama5d2-i2c",
+		.data = &sama5d2_config,
+	}, {
+		/* sentinel */
+	}
+};
+MODULE_DEVICE_TABLE(of, atmel_twi_dt_ids);
+#endif
+
+static struct at91_twi_pdata *at91_twi_get_driver_data(
+					struct platform_device *pdev)
+{
+	if (pdev->dev.of_node) {
+		const struct of_device_id *match;
+		match = of_match_node(atmel_twi_dt_ids, pdev->dev.of_node);
+		if (!match)
+			return NULL;
+		return (struct at91_twi_pdata *)match->data;
+	}
+	return (struct at91_twi_pdata *) platform_get_device_id(pdev)->driver_data;
+}
+
+static int at91_twi_probe(struct platform_device *pdev)
+{
+	struct at91_twi_dev *dev;
+	struct resource *mem;
+	int rc;
+	u32 phy_addr;
+
+	dev = devm_kzalloc(&pdev->dev, sizeof(*dev), GFP_KERNEL);
+	if (!dev)
+		return -ENOMEM;
+
+	dev->dev = &pdev->dev;
+
+	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!mem)
+		return -ENODEV;
+	phy_addr = mem->start;
+
+	dev->pdata = at91_twi_get_driver_data(pdev);
+	if (!dev->pdata)
+		return -ENODEV;
+
+	dev->base = devm_ioremap_resource(&pdev->dev, mem);
+	if (IS_ERR(dev->base))
+		return PTR_ERR(dev->base);
+
+	dev->irq = platform_get_irq(pdev, 0);
+	if (dev->irq < 0)
+		return dev->irq;
+
+	platform_set_drvdata(pdev, dev);
+
+	dev->clk = devm_clk_get(dev->dev, NULL);
+	if (IS_ERR(dev->clk)) {
+		dev_err(dev->dev, "no clock defined\n");
+		return -ENODEV;
+	}
+	clk_prepare_enable(dev->clk);
+
+	snprintf(dev->adapter.name, sizeof(dev->adapter.name), "AT91");
+	i2c_set_adapdata(&dev->adapter, dev);
+	dev->adapter.owner = THIS_MODULE;
+	dev->adapter.class = I2C_CLASS_DEPRECATED;
+	dev->adapter.dev.parent = dev->dev;
+	dev->adapter.nr = pdev->id;
+	dev->adapter.timeout = AT91_I2C_TIMEOUT;
+	dev->adapter.dev.of_node = pdev->dev.of_node;
+
+	dev->slave_detected = i2c_detect_slave_mode(&pdev->dev);
+
+	if (dev->slave_detected)
+		rc = at91_twi_probe_slave(pdev, phy_addr, dev);
+	else
+		rc = at91_twi_probe_master(pdev, phy_addr, dev);
+	if (rc)
+		return rc;
+
+	at91_init_twi_bus(dev);
+
+	pm_runtime_set_autosuspend_delay(dev->dev, AUTOSUSPEND_TIMEOUT);
+	pm_runtime_use_autosuspend(dev->dev);
+	pm_runtime_set_active(dev->dev);
+	pm_runtime_enable(dev->dev);
+
+	rc = i2c_add_numbered_adapter(&dev->adapter);
+	if (rc) {
+		clk_disable_unprepare(dev->clk);
+
+		pm_runtime_disable(dev->dev);
+		pm_runtime_set_suspended(dev->dev);
+
+		return rc;
+	}
+
+	dev_info(dev->dev, "AT91 i2c bus driver (hw version: %#x).\n",
+		 at91_twi_read(dev, AT91_TWI_VER));
+	return 0;
+}
+
+static int at91_twi_remove(struct platform_device *pdev)
+{
+	struct at91_twi_dev *dev = platform_get_drvdata(pdev);
+
+	i2c_del_adapter(&dev->adapter);
+	clk_disable_unprepare(dev->clk);
+
+	pm_runtime_disable(dev->dev);
+	pm_runtime_set_suspended(dev->dev);
+
+	return 0;
+}
+
+#ifdef CONFIG_PM
+
+static int at91_twi_runtime_suspend(struct device *dev)
+{
+	struct at91_twi_dev *twi_dev = dev_get_drvdata(dev);
+
+	clk_disable_unprepare(twi_dev->clk);
+
+	pinctrl_pm_select_sleep_state(dev);
+
+	return 0;
+}
+
+static int at91_twi_runtime_resume(struct device *dev)
+{
+	struct at91_twi_dev *twi_dev = dev_get_drvdata(dev);
+
+	pinctrl_pm_select_default_state(dev);
+
+	return clk_prepare_enable(twi_dev->clk);
+}
+
+static int at91_twi_suspend_noirq(struct device *dev)
+{
+	if (!pm_runtime_status_suspended(dev))
+		at91_twi_runtime_suspend(dev);
+
+	return 0;
+}
+
+static int at91_twi_resume_noirq(struct device *dev)
+{
+	struct at91_twi_dev *twi_dev = dev_get_drvdata(dev);
+	int ret;
+
+	if (!pm_runtime_status_suspended(dev)) {
+		ret = at91_twi_runtime_resume(dev);
+		if (ret)
+			return ret;
+	}
+
+	pm_runtime_mark_last_busy(dev);
+	pm_request_autosuspend(dev);
+
+	at91_init_twi_bus(twi_dev);
+
+	return 0;
+}
+
+static const struct dev_pm_ops at91_twi_pm = {
+	.suspend_noirq	= at91_twi_suspend_noirq,
+	.resume_noirq	= at91_twi_resume_noirq,
+	.runtime_suspend	= at91_twi_runtime_suspend,
+	.runtime_resume		= at91_twi_runtime_resume,
+};
+
+#define at91_twi_pm_ops (&at91_twi_pm)
+#else
+#define at91_twi_pm_ops NULL
+#endif
+
+static struct platform_driver at91_twi_driver = {
+	.probe		= at91_twi_probe,
+	.remove		= at91_twi_remove,
+	.id_table	= at91_twi_devtypes,
+	.driver		= {
+		.name	= "at91_i2c",
+		.of_match_table = of_match_ptr(atmel_twi_dt_ids),
+		.pm	= at91_twi_pm_ops,
+	},
+};
+
+static int __init at91_twi_init(void)
+{
+	return platform_driver_register(&at91_twi_driver);
+}
+
+static void __exit at91_twi_exit(void)
+{
+	platform_driver_unregister(&at91_twi_driver);
+}
+
+subsys_initcall(at91_twi_init);
+module_exit(at91_twi_exit);
+
+MODULE_AUTHOR("Nikolaus Voss <n.voss@weinmann.de>");
+MODULE_DESCRIPTION("I2C (TWI) driver for Atmel AT91");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:at91_i2c");
diff --git a/drivers/i2c/busses/i2c-at91.c b/drivers/i2c/busses/i2c-at91-master.c
index 3f3e8b3bf5ff..e87232f2e708 100644
--- a/drivers/i2c/busses/i2c-at91.c
+++ b/drivers/i2c/busses/i2c-at91-master.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  *  i2c Support for Atmel's AT91 Two-Wire Interface (TWI)
  *
@@ -10,11 +11,6 @@
  *
  *  Borrowed heavily from original work by:
  *  Copyright (C) 2000 Philip Edelbrock <phil@stimpy.netroedge.com>
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
  */
 
 #include <linux/clk.h>
@@ -25,159 +21,16 @@
 #include <linux/i2c.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
-#include <linux/module.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
 #include <linux/platform_device.h>
-#include <linux/slab.h>
 #include <linux/platform_data/dma-atmel.h>
 #include <linux/pm_runtime.h>
-#include <linux/pinctrl/consumer.h>
-
-#define DEFAULT_TWI_CLK_HZ		100000		/* max 400 Kbits/s */
-#define AT91_I2C_TIMEOUT	msecs_to_jiffies(100)	/* transfer timeout */
-#define AT91_I2C_DMA_THRESHOLD	8			/* enable DMA if transfer size is bigger than this threshold */
-#define AUTOSUSPEND_TIMEOUT		2000
-#define AT91_I2C_MAX_ALT_CMD_DATA_SIZE	256
-
-/* AT91 TWI register definitions */
-#define	AT91_TWI_CR		0x0000	/* Control Register */
-#define	AT91_TWI_START		BIT(0)	/* Send a Start Condition */
-#define	AT91_TWI_STOP		BIT(1)	/* Send a Stop Condition */
-#define	AT91_TWI_MSEN		BIT(2)	/* Master Transfer Enable */
-#define	AT91_TWI_MSDIS		BIT(3)	/* Master Transfer Disable */
-#define	AT91_TWI_SVEN		BIT(4)	/* Slave Transfer Enable */
-#define	AT91_TWI_SVDIS		BIT(5)	/* Slave Transfer Disable */
-#define	AT91_TWI_QUICK		BIT(6)	/* SMBus quick command */
-#define	AT91_TWI_SWRST		BIT(7)	/* Software Reset */
-#define	AT91_TWI_ACMEN		BIT(16) /* Alternative Command Mode Enable */
-#define	AT91_TWI_ACMDIS		BIT(17) /* Alternative Command Mode Disable */
-#define	AT91_TWI_THRCLR		BIT(24) /* Transmit Holding Register Clear */
-#define	AT91_TWI_RHRCLR		BIT(25) /* Receive Holding Register Clear */
-#define	AT91_TWI_LOCKCLR	BIT(26) /* Lock Clear */
-#define	AT91_TWI_FIFOEN		BIT(28) /* FIFO Enable */
-#define	AT91_TWI_FIFODIS	BIT(29) /* FIFO Disable */
-
-#define	AT91_TWI_MMR		0x0004	/* Master Mode Register */
-#define	AT91_TWI_IADRSZ_1	0x0100	/* Internal Device Address Size */
-#define	AT91_TWI_MREAD		BIT(12)	/* Master Read Direction */
-
-#define	AT91_TWI_IADR		0x000c	/* Internal Address Register */
-
-#define	AT91_TWI_CWGR		0x0010	/* Clock Waveform Generator Reg */
-#define	AT91_TWI_CWGR_HOLD_MAX	0x1f
-#define	AT91_TWI_CWGR_HOLD(x)	(((x) & AT91_TWI_CWGR_HOLD_MAX) << 24)
-
-#define	AT91_TWI_SR		0x0020	/* Status Register */
-#define	AT91_TWI_TXCOMP		BIT(0)	/* Transmission Complete */
-#define	AT91_TWI_RXRDY		BIT(1)	/* Receive Holding Register Ready */
-#define	AT91_TWI_TXRDY		BIT(2)	/* Transmit Holding Register Ready */
-#define	AT91_TWI_OVRE		BIT(6)	/* Overrun Error */
-#define	AT91_TWI_UNRE		BIT(7)	/* Underrun Error */
-#define	AT91_TWI_NACK		BIT(8)	/* Not Acknowledged */
-#define	AT91_TWI_LOCK		BIT(23) /* TWI Lock due to Frame Errors */
-
-#define	AT91_TWI_INT_MASK \
-	(AT91_TWI_TXCOMP | AT91_TWI_RXRDY | AT91_TWI_TXRDY | AT91_TWI_NACK)
-
-#define	AT91_TWI_IER		0x0024	/* Interrupt Enable Register */
-#define	AT91_TWI_IDR		0x0028	/* Interrupt Disable Register */
-#define	AT91_TWI_IMR		0x002c	/* Interrupt Mask Register */
-#define	AT91_TWI_RHR		0x0030	/* Receive Holding Register */
-#define	AT91_TWI_THR		0x0034	/* Transmit Holding Register */
-
-#define	AT91_TWI_ACR		0x0040	/* Alternative Command Register */
-#define	AT91_TWI_ACR_DATAL(len)	((len) & 0xff)
-#define	AT91_TWI_ACR_DIR	BIT(8)
-
-#define	AT91_TWI_FMR		0x0050	/* FIFO Mode Register */
-#define	AT91_TWI_FMR_TXRDYM(mode)	(((mode) & 0x3) << 0)
-#define	AT91_TWI_FMR_TXRDYM_MASK	(0x3 << 0)
-#define	AT91_TWI_FMR_RXRDYM(mode)	(((mode) & 0x3) << 4)
-#define	AT91_TWI_FMR_RXRDYM_MASK	(0x3 << 4)
-#define	AT91_TWI_ONE_DATA	0x0
-#define	AT91_TWI_TWO_DATA	0x1
-#define	AT91_TWI_FOUR_DATA	0x2
-
-#define	AT91_TWI_FLR		0x0054	/* FIFO Level Register */
-
-#define	AT91_TWI_FSR		0x0060	/* FIFO Status Register */
-#define	AT91_TWI_FIER		0x0064	/* FIFO Interrupt Enable Register */
-#define	AT91_TWI_FIDR		0x0068	/* FIFO Interrupt Disable Register */
-#define	AT91_TWI_FIMR		0x006c	/* FIFO Interrupt Mask Register */
-
-#define	AT91_TWI_VER		0x00fc	/* Version Register */
-
-struct at91_twi_pdata {
-	unsigned clk_max_div;
-	unsigned clk_offset;
-	bool has_unre_flag;
-	bool has_alt_cmd;
-	bool has_hold_field;
-	struct at_dma_slave dma_slave;
-};
-
-struct at91_twi_dma {
-	struct dma_chan *chan_rx;
-	struct dma_chan *chan_tx;
-	struct scatterlist sg[2];
-	struct dma_async_tx_descriptor *data_desc;
-	enum dma_data_direction direction;
-	bool buf_mapped;
-	bool xfer_in_progress;
-};
 
-struct at91_twi_dev {
-	struct device *dev;
-	void __iomem *base;
-	struct completion cmd_complete;
-	struct clk *clk;
-	u8 *buf;
-	size_t buf_len;
-	struct i2c_msg *msg;
-	int irq;
-	unsigned imr;
-	unsigned transfer_status;
-	struct i2c_adapter adapter;
-	unsigned twi_cwgr_reg;
-	struct at91_twi_pdata *pdata;
-	bool use_dma;
-	bool use_alt_cmd;
-	bool recv_len_abort;
-	u32 fifo_size;
-	struct at91_twi_dma dma;
-};
-
-static unsigned at91_twi_read(struct at91_twi_dev *dev, unsigned reg)
-{
-	return readl_relaxed(dev->base + reg);
-}
-
-static void at91_twi_write(struct at91_twi_dev *dev, unsigned reg, unsigned val)
-{
-	writel_relaxed(val, dev->base + reg);
-}
-
-static void at91_disable_twi_interrupts(struct at91_twi_dev *dev)
-{
-	at91_twi_write(dev, AT91_TWI_IDR, AT91_TWI_INT_MASK);
-}
+#include "i2c-at91.h"
 
-static void at91_twi_irq_save(struct at91_twi_dev *dev)
+void at91_init_twi_bus_master(struct at91_twi_dev *dev)
 {
-	dev->imr = at91_twi_read(dev, AT91_TWI_IMR) & AT91_TWI_INT_MASK;
-	at91_disable_twi_interrupts(dev);
-}
-
-static void at91_twi_irq_restore(struct at91_twi_dev *dev)
-{
-	at91_twi_write(dev, AT91_TWI_IER, dev->imr);
-}
-
-static void at91_init_twi_bus(struct at91_twi_dev *dev)
-{
-	at91_disable_twi_interrupts(dev);
-	at91_twi_write(dev, AT91_TWI_CR, AT91_TWI_SWRST);
 	/* FIFO should be enabled immediately after the software reset */
 	if (dev->fifo_size)
 		at91_twi_write(dev, AT91_TWI_CR, AT91_TWI_FIFOEN);
@@ -190,16 +43,18 @@ static void at91_init_twi_bus(struct at91_twi_dev *dev)
  * Calculate symmetric clock as stated in datasheet:
  * twi_clk = F_MAIN / (2 * (cdiv * (1 << ckdiv) + offset))
  */
-static void at91_calc_twi_clock(struct at91_twi_dev *dev, int twi_clk)
+static void at91_calc_twi_clock(struct at91_twi_dev *dev)
 {
 	int ckdiv, cdiv, div, hold = 0;
 	struct at91_twi_pdata *pdata = dev->pdata;
 	int offset = pdata->clk_offset;
 	int max_ckdiv = pdata->clk_max_div;
-	u32 twd_hold_time_ns = 0;
+	struct i2c_timings timings, *t = &timings;
+
+	i2c_parse_fw_timings(dev->dev, t, true);
 
 	div = max(0, (int)DIV_ROUND_UP(clk_get_rate(dev->clk),
-				       2 * twi_clk) - offset);
+				       2 * t->bus_freq_hz) - offset);
 	ckdiv = fls(div >> 8);
 	cdiv = div >> ckdiv;
 
@@ -211,15 +66,12 @@ static void at91_calc_twi_clock(struct at91_twi_dev *dev, int twi_clk)
 	}
 
 	if (pdata->has_hold_field) {
-		of_property_read_u32(dev->dev->of_node, "i2c-sda-hold-time-ns",
-				     &twd_hold_time_ns);
-
 		/*
 		 * hold time = HOLD + 3 x T_peripheral_clock
 		 * Use clk rate in kHz to prevent overflows when computing
 		 * hold.
 		 */
-		hold = DIV_ROUND_UP(twd_hold_time_ns
+		hold = DIV_ROUND_UP(t->sda_hold_ns
 				    * (clk_get_rate(dev->clk) / 1000), 1000000);
 		hold -= 3;
 		if (hold < 0)
@@ -236,7 +88,7 @@ static void at91_calc_twi_clock(struct at91_twi_dev *dev, int twi_clk)
 			    | AT91_TWI_CWGR_HOLD(hold);
 
 	dev_dbg(dev->dev, "cdiv %d ckdiv %d hold %d (%d ns)\n",
-		cdiv, ckdiv, hold, twd_hold_time_ns);
+		cdiv, ckdiv, hold, t->sda_hold_ns);
 }
 
 static void at91_twi_dma_cleanup(struct at91_twi_dev *dev)
@@ -833,124 +685,6 @@ static const struct i2c_algorithm at91_twi_algorithm = {
 	.functionality	= at91_twi_func,
 };
 
-static struct at91_twi_pdata at91rm9200_config = {
-	.clk_max_div = 5,
-	.clk_offset = 3,
-	.has_unre_flag = true,
-	.has_alt_cmd = false,
-	.has_hold_field = false,
-};
-
-static struct at91_twi_pdata at91sam9261_config = {
-	.clk_max_div = 5,
-	.clk_offset = 4,
-	.has_unre_flag = false,
-	.has_alt_cmd = false,
-	.has_hold_field = false,
-};
-
-static struct at91_twi_pdata at91sam9260_config = {
-	.clk_max_div = 7,
-	.clk_offset = 4,
-	.has_unre_flag = false,
-	.has_alt_cmd = false,
-	.has_hold_field = false,
-};
-
-static struct at91_twi_pdata at91sam9g20_config = {
-	.clk_max_div = 7,
-	.clk_offset = 4,
-	.has_unre_flag = false,
-	.has_alt_cmd = false,
-	.has_hold_field = false,
-};
-
-static struct at91_twi_pdata at91sam9g10_config = {
-	.clk_max_div = 7,
-	.clk_offset = 4,
-	.has_unre_flag = false,
-	.has_alt_cmd = false,
-	.has_hold_field = false,
-};
-
-static const struct platform_device_id at91_twi_devtypes[] = {
-	{
-		.name = "i2c-at91rm9200",
-		.driver_data = (unsigned long) &at91rm9200_config,
-	}, {
-		.name = "i2c-at91sam9261",
-		.driver_data = (unsigned long) &at91sam9261_config,
-	}, {
-		.name = "i2c-at91sam9260",
-		.driver_data = (unsigned long) &at91sam9260_config,
-	}, {
-		.name = "i2c-at91sam9g20",
-		.driver_data = (unsigned long) &at91sam9g20_config,
-	}, {
-		.name = "i2c-at91sam9g10",
-		.driver_data = (unsigned long) &at91sam9g10_config,
-	}, {
-		/* sentinel */
-	}
-};
-
-#if defined(CONFIG_OF)
-static struct at91_twi_pdata at91sam9x5_config = {
-	.clk_max_div = 7,
-	.clk_offset = 4,
-	.has_unre_flag = false,
-	.has_alt_cmd = false,
-	.has_hold_field = false,
-};
-
-static struct at91_twi_pdata sama5d4_config = {
-	.clk_max_div = 7,
-	.clk_offset = 4,
-	.has_unre_flag = false,
-	.has_alt_cmd = false,
-	.has_hold_field = true,
-};
-
-static struct at91_twi_pdata sama5d2_config = {
-	.clk_max_div = 7,
-	.clk_offset = 4,
-	.has_unre_flag = true,
-	.has_alt_cmd = true,
-	.has_hold_field = true,
-};
-
-static const struct of_device_id atmel_twi_dt_ids[] = {
-	{
-		.compatible = "atmel,at91rm9200-i2c",
-		.data = &at91rm9200_config,
-	} , {
-		.compatible = "atmel,at91sam9260-i2c",
-		.data = &at91sam9260_config,
-	} , {
-		.compatible = "atmel,at91sam9261-i2c",
-		.data = &at91sam9261_config,
-	} , {
-		.compatible = "atmel,at91sam9g20-i2c",
-		.data = &at91sam9g20_config,
-	} , {
-		.compatible = "atmel,at91sam9g10-i2c",
-		.data = &at91sam9g10_config,
-	}, {
-		.compatible = "atmel,at91sam9x5-i2c",
-		.data = &at91sam9x5_config,
-	}, {
-		.compatible = "atmel,sama5d4-i2c",
-		.data = &sama5d4_config,
-	}, {
-		.compatible = "atmel,sama5d2-i2c",
-		.data = &sama5d2_config,
-	}, {
-		/* sentinel */
-	}
-};
-MODULE_DEVICE_TABLE(of, atmel_twi_dt_ids);
-#endif
-
 static int at91_twi_configure_dma(struct at91_twi_dev *dev, u32 phy_addr)
 {
 	int ret = 0;
@@ -1033,74 +767,24 @@ error:
 	return ret;
 }
 
-static struct at91_twi_pdata *at91_twi_get_driver_data(
-					struct platform_device *pdev)
+int at91_twi_probe_master(struct platform_device *pdev,
+			  u32 phy_addr, struct at91_twi_dev *dev)
 {
-	if (pdev->dev.of_node) {
-		const struct of_device_id *match;
-		match = of_match_node(atmel_twi_dt_ids, pdev->dev.of_node);
-		if (!match)
-			return NULL;
-		return (struct at91_twi_pdata *)match->data;
-	}
-	return (struct at91_twi_pdata *) platform_get_device_id(pdev)->driver_data;
-}
-
-static int at91_twi_probe(struct platform_device *pdev)
-{
-	struct at91_twi_dev *dev;
-	struct resource *mem;
 	int rc;
-	u32 phy_addr;
-	u32 bus_clk_rate;
 
-	dev = devm_kzalloc(&pdev->dev, sizeof(*dev), GFP_KERNEL);
-	if (!dev)
-		return -ENOMEM;
 	init_completion(&dev->cmd_complete);
-	dev->dev = &pdev->dev;
-
-	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!mem)
-		return -ENODEV;
-	phy_addr = mem->start;
-
-	dev->pdata = at91_twi_get_driver_data(pdev);
-	if (!dev->pdata)
-		return -ENODEV;
-
-	dev->base = devm_ioremap_resource(&pdev->dev, mem);
-	if (IS_ERR(dev->base))
-		return PTR_ERR(dev->base);
-
-	dev->irq = platform_get_irq(pdev, 0);
-	if (dev->irq < 0)
-		return dev->irq;
 
 	rc = devm_request_irq(&pdev->dev, dev->irq, atmel_twi_interrupt, 0,
-			 dev_name(dev->dev), dev);
+			      dev_name(dev->dev), dev);
 	if (rc) {
 		dev_err(dev->dev, "Cannot get irq %d: %d\n", dev->irq, rc);
 		return rc;
 	}
 
-	platform_set_drvdata(pdev, dev);
-
-	dev->clk = devm_clk_get(dev->dev, NULL);
-	if (IS_ERR(dev->clk)) {
-		dev_err(dev->dev, "no clock defined\n");
-		return -ENODEV;
-	}
-	rc = clk_prepare_enable(dev->clk);
-	if (rc)
-		return rc;
-
 	if (dev->dev->of_node) {
 		rc = at91_twi_configure_dma(dev, phy_addr);
-		if (rc == -EPROBE_DEFER) {
-			clk_disable_unprepare(dev->clk);
+		if (rc == -EPROBE_DEFER)
 			return rc;
-		}
 	}
 
 	if (!of_property_read_u32(pdev->dev.of_node, "atmel,fifo-size",
@@ -1108,144 +792,10 @@ static int at91_twi_probe(struct platform_device *pdev)
 		dev_info(dev->dev, "Using FIFO (%u data)\n", dev->fifo_size);
 	}
 
-	rc = of_property_read_u32(dev->dev->of_node, "clock-frequency",
-			&bus_clk_rate);
-	if (rc)
-		bus_clk_rate = DEFAULT_TWI_CLK_HZ;
-
-	at91_calc_twi_clock(dev, bus_clk_rate);
-	at91_init_twi_bus(dev);
+	at91_calc_twi_clock(dev);
 
-	snprintf(dev->adapter.name, sizeof(dev->adapter.name), "AT91");
-	i2c_set_adapdata(&dev->adapter, dev);
-	dev->adapter.owner = THIS_MODULE;
-	dev->adapter.class = I2C_CLASS_DEPRECATED;
 	dev->adapter.algo = &at91_twi_algorithm;
 	dev->adapter.quirks = &at91_twi_quirks;
-	dev->adapter.dev.parent = dev->dev;
-	dev->adapter.nr = pdev->id;
-	dev->adapter.timeout = AT91_I2C_TIMEOUT;
-	dev->adapter.dev.of_node = pdev->dev.of_node;
-
-	pm_runtime_set_autosuspend_delay(dev->dev, AUTOSUSPEND_TIMEOUT);
-	pm_runtime_use_autosuspend(dev->dev);
-	pm_runtime_set_active(dev->dev);
-	pm_runtime_enable(dev->dev);
-
-	rc = i2c_add_numbered_adapter(&dev->adapter);
-	if (rc) {
-		clk_disable_unprepare(dev->clk);
-
-		pm_runtime_disable(dev->dev);
-		pm_runtime_set_suspended(dev->dev);
-
-		return rc;
-	}
-
-	dev_info(dev->dev, "AT91 i2c bus driver (hw version: %#x).\n",
-		 at91_twi_read(dev, AT91_TWI_VER));
-	return 0;
-}
-
-static int at91_twi_remove(struct platform_device *pdev)
-{
-	struct at91_twi_dev *dev = platform_get_drvdata(pdev);
-
-	i2c_del_adapter(&dev->adapter);
-	clk_disable_unprepare(dev->clk);
-
-	pm_runtime_disable(dev->dev);
-	pm_runtime_set_suspended(dev->dev);
-
-	return 0;
-}
-
-#ifdef CONFIG_PM
-
-static int at91_twi_runtime_suspend(struct device *dev)
-{
-	struct at91_twi_dev *twi_dev = dev_get_drvdata(dev);
-
-	clk_disable_unprepare(twi_dev->clk);
-
-	pinctrl_pm_select_sleep_state(dev);
-
-	return 0;
-}
-
-static int at91_twi_runtime_resume(struct device *dev)
-{
-	struct at91_twi_dev *twi_dev = dev_get_drvdata(dev);
-
-	pinctrl_pm_select_default_state(dev);
-
-	return clk_prepare_enable(twi_dev->clk);
-}
-
-static int at91_twi_suspend_noirq(struct device *dev)
-{
-	if (!pm_runtime_status_suspended(dev))
-		at91_twi_runtime_suspend(dev);
-
-	return 0;
-}
-
-static int at91_twi_resume_noirq(struct device *dev)
-{
-	struct at91_twi_dev *twi_dev = dev_get_drvdata(dev);
-	int ret;
-
-	if (!pm_runtime_status_suspended(dev)) {
-		ret = at91_twi_runtime_resume(dev);
-		if (ret)
-			return ret;
-	}
-
-	pm_runtime_mark_last_busy(dev);
-	pm_request_autosuspend(dev);
-
-	at91_init_twi_bus(twi_dev);
 
 	return 0;
 }
-
-static const struct dev_pm_ops at91_twi_pm = {
-	.suspend_noirq	= at91_twi_suspend_noirq,
-	.resume_noirq	= at91_twi_resume_noirq,
-	.runtime_suspend	= at91_twi_runtime_suspend,
-	.runtime_resume		= at91_twi_runtime_resume,
-};
-
-#define at91_twi_pm_ops (&at91_twi_pm)
-#else
-#define at91_twi_pm_ops NULL
-#endif
-
-static struct platform_driver at91_twi_driver = {
-	.probe		= at91_twi_probe,
-	.remove		= at91_twi_remove,
-	.id_table	= at91_twi_devtypes,
-	.driver		= {
-		.name	= "at91_i2c",
-		.of_match_table = of_match_ptr(atmel_twi_dt_ids),
-		.pm	= at91_twi_pm_ops,
-	},
-};
-
-static int __init at91_twi_init(void)
-{
-	return platform_driver_register(&at91_twi_driver);
-}
-
-static void __exit at91_twi_exit(void)
-{
-	platform_driver_unregister(&at91_twi_driver);
-}
-
-subsys_initcall(at91_twi_init);
-module_exit(at91_twi_exit);
-
-MODULE_AUTHOR("Nikolaus Voss <n.voss@weinmann.de>");
-MODULE_DESCRIPTION("I2C (TWI) driver for Atmel AT91");
-MODULE_LICENSE("GPL");
-MODULE_ALIAS("platform:at91_i2c");
diff --git a/drivers/i2c/busses/i2c-at91-slave.c b/drivers/i2c/busses/i2c-at91-slave.c
new file mode 100644
index 000000000000..d6eeea5166c0
--- /dev/null
+++ b/drivers/i2c/busses/i2c-at91-slave.c
@@ -0,0 +1,143 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  i2c slave support for Atmel's AT91 Two-Wire Interface (TWI)
+ *
+ *  Copyright (C) 2017 Juergen Fitschen <me@jue.yt>
+ */
+
+#include <linux/err.h>
+#include <linux/i2c.h>
+#include <linux/interrupt.h>
+#include <linux/pm_runtime.h>
+
+#include "i2c-at91.h"
+
+static irqreturn_t atmel_twi_interrupt_slave(int irq, void *dev_id)
+{
+	struct at91_twi_dev *dev = dev_id;
+	const unsigned status = at91_twi_read(dev, AT91_TWI_SR);
+	const unsigned irqstatus = status & at91_twi_read(dev, AT91_TWI_IMR);
+	u8 value;
+
+	if (!irqstatus)
+		return IRQ_NONE;
+
+	/* slave address has been detected on I2C bus */
+	if (irqstatus & AT91_TWI_SVACC) {
+		if (status & AT91_TWI_SVREAD) {
+			i2c_slave_event(dev->slave,
+					I2C_SLAVE_READ_REQUESTED, &value);
+			writeb_relaxed(value, dev->base + AT91_TWI_THR);
+			at91_twi_write(dev, AT91_TWI_IER,
+				       AT91_TWI_TXRDY | AT91_TWI_EOSACC);
+		} else {
+			i2c_slave_event(dev->slave,
+					I2C_SLAVE_WRITE_REQUESTED, &value);
+			at91_twi_write(dev, AT91_TWI_IER,
+				       AT91_TWI_RXRDY | AT91_TWI_EOSACC);
+		}
+		at91_twi_write(dev, AT91_TWI_IDR, AT91_TWI_SVACC);
+	}
+
+	/* byte transmitted to remote master */
+	if (irqstatus & AT91_TWI_TXRDY) {
+		i2c_slave_event(dev->slave, I2C_SLAVE_READ_PROCESSED, &value);
+		writeb_relaxed(value, dev->base + AT91_TWI_THR);
+	}
+
+	/* byte received from remote master */
+	if (irqstatus & AT91_TWI_RXRDY) {
+		value = readb_relaxed(dev->base + AT91_TWI_RHR);
+		i2c_slave_event(dev->slave, I2C_SLAVE_WRITE_RECEIVED, &value);
+	}
+
+	/* master sent stop */
+	if (irqstatus & AT91_TWI_EOSACC) {
+		at91_twi_write(dev, AT91_TWI_IDR,
+			       AT91_TWI_TXRDY | AT91_TWI_RXRDY | AT91_TWI_EOSACC);
+		at91_twi_write(dev, AT91_TWI_IER, AT91_TWI_SVACC);
+		i2c_slave_event(dev->slave, I2C_SLAVE_STOP, &value);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static int at91_reg_slave(struct i2c_client *slave)
+{
+	struct at91_twi_dev *dev = i2c_get_adapdata(slave->adapter);
+
+	if (dev->slave)
+		return -EBUSY;
+
+	if (slave->flags & I2C_CLIENT_TEN)
+		return -EAFNOSUPPORT;
+
+	/* Make sure twi_clk doesn't get turned off! */
+	pm_runtime_get_sync(dev->dev);
+
+	dev->slave = slave;
+	dev->smr = AT91_TWI_SMR_SADR(slave->addr);
+
+	at91_init_twi_bus(dev);
+	at91_twi_write(dev, AT91_TWI_IER, AT91_TWI_SVACC);
+
+	dev_info(dev->dev, "entered slave mode (ADR=%d)\n", slave->addr);
+
+	return 0;
+}
+
+static int at91_unreg_slave(struct i2c_client *slave)
+{
+	struct at91_twi_dev *dev = i2c_get_adapdata(slave->adapter);
+
+	WARN_ON(!dev->slave);
+
+	dev_info(dev->dev, "leaving slave mode\n");
+
+	dev->slave = NULL;
+	dev->smr = 0;
+
+	at91_init_twi_bus(dev);
+
+	pm_runtime_put(dev->dev);
+
+	return 0;
+}
+
+static u32 at91_twi_func(struct i2c_adapter *adapter)
+{
+	return I2C_FUNC_SLAVE | I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL
+		| I2C_FUNC_SMBUS_READ_BLOCK_DATA;
+}
+
+static const struct i2c_algorithm at91_twi_algorithm_slave = {
+	.reg_slave	= at91_reg_slave,
+	.unreg_slave	= at91_unreg_slave,
+	.functionality	= at91_twi_func,
+};
+
+int at91_twi_probe_slave(struct platform_device *pdev,
+			 u32 phy_addr, struct at91_twi_dev *dev)
+{
+	int rc;
+
+	rc = devm_request_irq(&pdev->dev, dev->irq, atmel_twi_interrupt_slave,
+			      0, dev_name(dev->dev), dev);
+	if (rc) {
+		dev_err(dev->dev, "Cannot get irq %d: %d\n", dev->irq, rc);
+		return rc;
+	}
+
+	dev->adapter.algo = &at91_twi_algorithm_slave;
+
+	return 0;
+}
+
+void at91_init_twi_bus_slave(struct at91_twi_dev *dev)
+{
+	at91_twi_write(dev, AT91_TWI_CR, AT91_TWI_MSDIS);
+	if (dev->slave_detected && dev->smr) {
+		at91_twi_write(dev, AT91_TWI_SMR, dev->smr);
+		at91_twi_write(dev, AT91_TWI_CR, AT91_TWI_SVEN);
+	}
+}
diff --git a/drivers/i2c/busses/i2c-at91.h b/drivers/i2c/busses/i2c-at91.h
new file mode 100644
index 000000000000..499b506f6128
--- /dev/null
+++ b/drivers/i2c/busses/i2c-at91.h
@@ -0,0 +1,174 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ *  i2c Support for Atmel's AT91 Two-Wire Interface (TWI)
+ *
+ *  Copyright (C) 2011 Weinmann Medical GmbH
+ *  Author: Nikolaus Voss <n.voss@weinmann.de>
+ *
+ *  Evolved from original work by:
+ *  Copyright (C) 2004 Rick Bronson
+ *  Converted to 2.6 by Andrew Victor <andrew@sanpeople.com>
+ *
+ *  Borrowed heavily from original work by:
+ *  Copyright (C) 2000 Philip Edelbrock <phil@stimpy.netroedge.com>
+ */
+
+#include <linux/clk.h>
+#include <linux/completion.h>
+#include <linux/dma-mapping.h>
+#include <linux/dmaengine.h>
+#include <linux/i2c.h>
+#include <linux/platform_data/dma-atmel.h>
+#include <linux/platform_device.h>
+
+#define AT91_I2C_TIMEOUT	msecs_to_jiffies(100)	/* transfer timeout */
+#define AT91_I2C_DMA_THRESHOLD	8			/* enable DMA if transfer size is bigger than this threshold */
+#define AUTOSUSPEND_TIMEOUT		2000
+#define AT91_I2C_MAX_ALT_CMD_DATA_SIZE	256
+
+/* AT91 TWI register definitions */
+#define	AT91_TWI_CR		0x0000	/* Control Register */
+#define	AT91_TWI_START		BIT(0)	/* Send a Start Condition */
+#define	AT91_TWI_STOP		BIT(1)	/* Send a Stop Condition */
+#define	AT91_TWI_MSEN		BIT(2)	/* Master Transfer Enable */
+#define	AT91_TWI_MSDIS		BIT(3)	/* Master Transfer Disable */
+#define	AT91_TWI_SVEN		BIT(4)	/* Slave Transfer Enable */
+#define	AT91_TWI_SVDIS		BIT(5)	/* Slave Transfer Disable */
+#define	AT91_TWI_QUICK		BIT(6)	/* SMBus quick command */
+#define	AT91_TWI_SWRST		BIT(7)	/* Software Reset */
+#define	AT91_TWI_ACMEN		BIT(16) /* Alternative Command Mode Enable */
+#define	AT91_TWI_ACMDIS		BIT(17) /* Alternative Command Mode Disable */
+#define	AT91_TWI_THRCLR		BIT(24) /* Transmit Holding Register Clear */
+#define	AT91_TWI_RHRCLR		BIT(25) /* Receive Holding Register Clear */
+#define	AT91_TWI_LOCKCLR	BIT(26) /* Lock Clear */
+#define	AT91_TWI_FIFOEN		BIT(28) /* FIFO Enable */
+#define	AT91_TWI_FIFODIS	BIT(29) /* FIFO Disable */
+
+#define	AT91_TWI_MMR		0x0004	/* Master Mode Register */
+#define	AT91_TWI_IADRSZ_1	0x0100	/* Internal Device Address Size */
+#define	AT91_TWI_MREAD		BIT(12)	/* Master Read Direction */
+
+#define	AT91_TWI_SMR		0x0008	/* Slave Mode Register */
+#define	AT91_TWI_SMR_SADR_MAX	0x007f
+#define	AT91_TWI_SMR_SADR(x)	(((x) & AT91_TWI_SMR_SADR_MAX) << 16)
+
+#define	AT91_TWI_IADR		0x000c	/* Internal Address Register */
+
+#define	AT91_TWI_CWGR		0x0010	/* Clock Waveform Generator Reg */
+#define	AT91_TWI_CWGR_HOLD_MAX	0x1f
+#define	AT91_TWI_CWGR_HOLD(x)	(((x) & AT91_TWI_CWGR_HOLD_MAX) << 24)
+
+#define	AT91_TWI_SR		0x0020	/* Status Register */
+#define	AT91_TWI_TXCOMP		BIT(0)	/* Transmission Complete */
+#define	AT91_TWI_RXRDY		BIT(1)	/* Receive Holding Register Ready */
+#define	AT91_TWI_TXRDY		BIT(2)	/* Transmit Holding Register Ready */
+#define	AT91_TWI_SVREAD		BIT(3)	/* Slave Read */
+#define	AT91_TWI_SVACC		BIT(4)	/* Slave Access */
+#define	AT91_TWI_OVRE		BIT(6)	/* Overrun Error */
+#define	AT91_TWI_UNRE		BIT(7)	/* Underrun Error */
+#define	AT91_TWI_NACK		BIT(8)	/* Not Acknowledged */
+#define	AT91_TWI_EOSACC		BIT(11)	/* End Of Slave Access */
+#define	AT91_TWI_LOCK		BIT(23) /* TWI Lock due to Frame Errors */
+
+#define	AT91_TWI_INT_MASK \
+	(AT91_TWI_TXCOMP | AT91_TWI_RXRDY | AT91_TWI_TXRDY | AT91_TWI_NACK \
+	| AT91_TWI_SVACC | AT91_TWI_EOSACC)
+
+#define	AT91_TWI_IER		0x0024	/* Interrupt Enable Register */
+#define	AT91_TWI_IDR		0x0028	/* Interrupt Disable Register */
+#define	AT91_TWI_IMR		0x002c	/* Interrupt Mask Register */
+#define	AT91_TWI_RHR		0x0030	/* Receive Holding Register */
+#define	AT91_TWI_THR		0x0034	/* Transmit Holding Register */
+
+#define	AT91_TWI_ACR		0x0040	/* Alternative Command Register */
+#define	AT91_TWI_ACR_DATAL(len)	((len) & 0xff)
+#define	AT91_TWI_ACR_DIR	BIT(8)
+
+#define	AT91_TWI_FMR		0x0050	/* FIFO Mode Register */
+#define	AT91_TWI_FMR_TXRDYM(mode)	(((mode) & 0x3) << 0)
+#define	AT91_TWI_FMR_TXRDYM_MASK	(0x3 << 0)
+#define	AT91_TWI_FMR_RXRDYM(mode)	(((mode) & 0x3) << 4)
+#define	AT91_TWI_FMR_RXRDYM_MASK	(0x3 << 4)
+#define	AT91_TWI_ONE_DATA	0x0
+#define	AT91_TWI_TWO_DATA	0x1
+#define	AT91_TWI_FOUR_DATA	0x2
+
+#define	AT91_TWI_FLR		0x0054	/* FIFO Level Register */
+
+#define	AT91_TWI_FSR		0x0060	/* FIFO Status Register */
+#define	AT91_TWI_FIER		0x0064	/* FIFO Interrupt Enable Register */
+#define	AT91_TWI_FIDR		0x0068	/* FIFO Interrupt Disable Register */
+#define	AT91_TWI_FIMR		0x006c	/* FIFO Interrupt Mask Register */
+
+#define	AT91_TWI_VER		0x00fc	/* Version Register */
+
+struct at91_twi_pdata {
+	unsigned clk_max_div;
+	unsigned clk_offset;
+	bool has_unre_flag;
+	bool has_alt_cmd;
+	bool has_hold_field;
+	struct at_dma_slave dma_slave;
+};
+
+struct at91_twi_dma {
+	struct dma_chan *chan_rx;
+	struct dma_chan *chan_tx;
+	struct scatterlist sg[2];
+	struct dma_async_tx_descriptor *data_desc;
+	enum dma_data_direction direction;
+	bool buf_mapped;
+	bool xfer_in_progress;
+};
+
+struct at91_twi_dev {
+	struct device *dev;
+	void __iomem *base;
+	struct completion cmd_complete;
+	struct clk *clk;
+	u8 *buf;
+	size_t buf_len;
+	struct i2c_msg *msg;
+	int irq;
+	unsigned imr;
+	unsigned transfer_status;
+	struct i2c_adapter adapter;
+	unsigned twi_cwgr_reg;
+	struct at91_twi_pdata *pdata;
+	bool use_dma;
+	bool use_alt_cmd;
+	bool recv_len_abort;
+	u32 fifo_size;
+	struct at91_twi_dma dma;
+	bool slave_detected;
+#ifdef CONFIG_I2C_AT91_SLAVE_EXPERIMENTAL
+	unsigned smr;
+	struct i2c_client *slave;
+#endif
+};
+
+unsigned at91_twi_read(struct at91_twi_dev *dev, unsigned reg);
+void at91_twi_write(struct at91_twi_dev *dev, unsigned reg, unsigned val);
+void at91_disable_twi_interrupts(struct at91_twi_dev *dev);
+void at91_twi_irq_save(struct at91_twi_dev *dev);
+void at91_twi_irq_restore(struct at91_twi_dev *dev);
+void at91_init_twi_bus(struct at91_twi_dev *dev);
+
+void at91_init_twi_bus_master(struct at91_twi_dev *dev);
+int at91_twi_probe_master(struct platform_device *pdev, u32 phy_addr,
+			  struct at91_twi_dev *dev);
+
+#ifdef CONFIG_I2C_AT91_SLAVE_EXPERIMENTAL
+void at91_init_twi_bus_slave(struct at91_twi_dev *dev);
+int at91_twi_probe_slave(struct platform_device *pdev, u32 phy_addr,
+			 struct at91_twi_dev *dev);
+
+#else
+static inline void at91_init_twi_bus_slave(struct at91_twi_dev *dev) {}
+static inline int at91_twi_probe_slave(struct platform_device *pdev,
+				       u32 phy_addr, struct at91_twi_dev *dev)
+{
+	return -EINVAL;
+}
+
+#endif
diff --git a/drivers/i2c/busses/i2c-axxia.c b/drivers/i2c/busses/i2c-axxia.c
index bf564391091f..1c7b41f45c83 100644
--- a/drivers/i2c/busses/i2c-axxia.c
+++ b/drivers/i2c/busses/i2c-axxia.c
@@ -99,6 +99,7 @@
  * @adapter: core i2c abstraction
  * @i2c_clk: clock reference for i2c input clock
  * @bus_clk_rate: current i2c bus clock rate
+ * @last: a flag indicating is this is last message in transfer
  */
 struct axxia_i2c_dev {
 	void __iomem *base;
@@ -112,6 +113,7 @@ struct axxia_i2c_dev {
 	struct i2c_adapter adapter;
 	struct clk *i2c_clk;
 	u32 bus_clk_rate;
+	bool last;
 };
 
 static void i2c_int_disable(struct axxia_i2c_dev *idev, u32 mask)
@@ -324,15 +326,14 @@ static irqreturn_t axxia_i2c_isr(int irq, void *_dev)
 		/* Stop completed */
 		i2c_int_disable(idev, ~MST_STATUS_TSS);
 		complete(&idev->msg_complete);
-	} else if (status & MST_STATUS_SNS) {
+	} else if (status & (MST_STATUS_SNS | MST_STATUS_SS)) {
 		/* Transfer done */
-		i2c_int_disable(idev, ~MST_STATUS_TSS);
+		int mask = idev->last ? ~0 : ~MST_STATUS_TSS;
+
+		i2c_int_disable(idev, mask);
 		if (i2c_m_rd(idev->msg_r) && idev->msg_xfrd_r < idev->msg_r->len)
 			axxia_i2c_empty_rx_fifo(idev);
 		complete(&idev->msg_complete);
-	} else if (status & MST_STATUS_SS) {
-		/* Auto/Sequence transfer done */
-		complete(&idev->msg_complete);
 	} else if (status & MST_STATUS_TSS) {
 		/* Transfer timeout */
 		idev->msg_err = -ETIMEDOUT;
@@ -405,6 +406,7 @@ static int axxia_i2c_xfer_seq(struct axxia_i2c_dev *idev, struct i2c_msg msgs[])
 	idev->msg_r = &msgs[1];
 	idev->msg_xfrd = 0;
 	idev->msg_xfrd_r = 0;
+	idev->last = true;
 	axxia_i2c_fill_tx_fifo(idev);
 
 	writel(CMD_SEQUENCE, idev->base + MST_COMMAND);
@@ -415,10 +417,6 @@ static int axxia_i2c_xfer_seq(struct axxia_i2c_dev *idev, struct i2c_msg msgs[])
 	time_left = wait_for_completion_timeout(&idev->msg_complete,
 						I2C_XFER_TIMEOUT);
 
-	i2c_int_disable(idev, int_mask);
-
-	axxia_i2c_empty_rx_fifo(idev);
-
 	if (idev->msg_err == -ENXIO) {
 		if (axxia_i2c_handle_seq_nak(idev))
 			axxia_i2c_init(idev);
@@ -438,9 +436,10 @@ static int axxia_i2c_xfer_seq(struct axxia_i2c_dev *idev, struct i2c_msg msgs[])
 	return idev->msg_err;
 }
 
-static int axxia_i2c_xfer_msg(struct axxia_i2c_dev *idev, struct i2c_msg *msg)
+static int axxia_i2c_xfer_msg(struct axxia_i2c_dev *idev, struct i2c_msg *msg,
+			      bool last)
 {
-	u32 int_mask = MST_STATUS_ERR | MST_STATUS_SNS;
+	u32 int_mask = MST_STATUS_ERR;
 	u32 rx_xfer, tx_xfer;
 	unsigned long time_left;
 	unsigned int wt_value;
@@ -449,6 +448,7 @@ static int axxia_i2c_xfer_msg(struct axxia_i2c_dev *idev, struct i2c_msg *msg)
 	idev->msg_r = msg;
 	idev->msg_xfrd = 0;
 	idev->msg_xfrd_r = 0;
+	idev->last = last;
 	reinit_completion(&idev->msg_complete);
 
 	axxia_i2c_set_addr(idev, msg);
@@ -478,8 +478,13 @@ static int axxia_i2c_xfer_msg(struct axxia_i2c_dev *idev, struct i2c_msg *msg)
 	if (idev->msg_err)
 		goto out;
 
-	/* Start manual mode */
-	writel(CMD_MANUAL, idev->base + MST_COMMAND);
+	if (!last) {
+		writel(CMD_MANUAL, idev->base + MST_COMMAND);
+		int_mask |= MST_STATUS_SNS;
+	} else {
+		writel(CMD_AUTO, idev->base + MST_COMMAND);
+		int_mask |= MST_STATUS_SS;
+	}
 
 	writel(WT_EN | wt_value, idev->base + WAIT_TIMER_CONTROL);
 
@@ -507,28 +512,6 @@ out:
 	return idev->msg_err;
 }
 
-static int axxia_i2c_stop(struct axxia_i2c_dev *idev)
-{
-	u32 int_mask = MST_STATUS_ERR | MST_STATUS_SCC | MST_STATUS_TSS;
-	unsigned long time_left;
-
-	reinit_completion(&idev->msg_complete);
-
-	/* Issue stop */
-	writel(0xb, idev->base + MST_COMMAND);
-	i2c_int_enable(idev, int_mask);
-	time_left = wait_for_completion_timeout(&idev->msg_complete,
-					      I2C_STOP_TIMEOUT);
-	i2c_int_disable(idev, int_mask);
-	if (time_left == 0)
-		return -ETIMEDOUT;
-
-	if (readl(idev->base + MST_COMMAND) & CMD_BUSY)
-		dev_warn(idev->dev, "busy after stop\n");
-
-	return 0;
-}
-
 /* This function checks if the msgs[] array contains messages compatible with
  * Sequence mode of operation. This mode assumes there will be exactly one
  * write of non-zero length followed by exactly one read of non-zero length,
@@ -558,9 +541,7 @@ axxia_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num)
 	i2c_int_enable(idev, MST_STATUS_TSS);
 
 	for (i = 0; ret == 0 && i < num; ++i)
-		ret = axxia_i2c_xfer_msg(idev, &msgs[i]);
-
-	axxia_i2c_stop(idev);
+		ret = axxia_i2c_xfer_msg(idev, &msgs[i], i == (num - 1));
 
 	return ret ? : i;
 }
diff --git a/drivers/i2c/busses/i2c-bcm-iproc.c b/drivers/i2c/busses/i2c-bcm-iproc.c
index 4c8c3bc4669c..a845b8decac8 100644
--- a/drivers/i2c/busses/i2c-bcm-iproc.c
+++ b/drivers/i2c/busses/i2c-bcm-iproc.c
@@ -17,17 +17,38 @@
 #include <linux/io.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/of_device.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 
+#define IDM_CTRL_DIRECT_OFFSET       0x00
 #define CFG_OFFSET                   0x00
 #define CFG_RESET_SHIFT              31
 #define CFG_EN_SHIFT                 30
+#define CFG_SLAVE_ADDR_0_SHIFT       28
 #define CFG_M_RETRY_CNT_SHIFT        16
 #define CFG_M_RETRY_CNT_MASK         0x0f
 
 #define TIM_CFG_OFFSET               0x04
 #define TIM_CFG_MODE_400_SHIFT       31
+#define TIM_RAND_SLAVE_STRETCH_SHIFT      24
+#define TIM_RAND_SLAVE_STRETCH_MASK       0x7f
+#define TIM_PERIODIC_SLAVE_STRETCH_SHIFT  16
+#define TIM_PERIODIC_SLAVE_STRETCH_MASK   0x7f
+
+#define S_CFG_SMBUS_ADDR_OFFSET           0x08
+#define S_CFG_EN_NIC_SMB_ADDR3_SHIFT      31
+#define S_CFG_NIC_SMB_ADDR3_SHIFT         24
+#define S_CFG_NIC_SMB_ADDR3_MASK          0x7f
+#define S_CFG_EN_NIC_SMB_ADDR2_SHIFT      23
+#define S_CFG_NIC_SMB_ADDR2_SHIFT         16
+#define S_CFG_NIC_SMB_ADDR2_MASK          0x7f
+#define S_CFG_EN_NIC_SMB_ADDR1_SHIFT      15
+#define S_CFG_NIC_SMB_ADDR1_SHIFT         8
+#define S_CFG_NIC_SMB_ADDR1_MASK          0x7f
+#define S_CFG_EN_NIC_SMB_ADDR0_SHIFT      7
+#define S_CFG_NIC_SMB_ADDR0_SHIFT         0
+#define S_CFG_NIC_SMB_ADDR0_MASK          0x7f
 
 #define M_FIFO_CTRL_OFFSET           0x0c
 #define M_FIFO_RX_FLUSH_SHIFT        31
@@ -37,6 +58,14 @@
 #define M_FIFO_RX_THLD_SHIFT         8
 #define M_FIFO_RX_THLD_MASK          0x3f
 
+#define S_FIFO_CTRL_OFFSET           0x10
+#define S_FIFO_RX_FLUSH_SHIFT        31
+#define S_FIFO_TX_FLUSH_SHIFT        30
+#define S_FIFO_RX_CNT_SHIFT          16
+#define S_FIFO_RX_CNT_MASK           0x7f
+#define S_FIFO_RX_THLD_SHIFT         8
+#define S_FIFO_RX_THLD_MASK          0x3f
+
 #define M_CMD_OFFSET                 0x30
 #define M_CMD_START_BUSY_SHIFT       31
 #define M_CMD_STATUS_SHIFT           25
@@ -46,6 +75,8 @@
 #define M_CMD_STATUS_NACK_ADDR       0x2
 #define M_CMD_STATUS_NACK_DATA       0x3
 #define M_CMD_STATUS_TIMEOUT         0x4
+#define M_CMD_STATUS_FIFO_UNDERRUN   0x5
+#define M_CMD_STATUS_RX_FIFO_FULL    0x6
 #define M_CMD_PROTOCOL_SHIFT         9
 #define M_CMD_PROTOCOL_MASK          0xf
 #define M_CMD_PROTOCOL_BLK_WR        0x7
@@ -54,17 +85,36 @@
 #define M_CMD_RD_CNT_SHIFT           0
 #define M_CMD_RD_CNT_MASK            0xff
 
+#define S_CMD_OFFSET                 0x34
+#define S_CMD_START_BUSY_SHIFT       31
+#define S_CMD_STATUS_SHIFT           23
+#define S_CMD_STATUS_MASK            0x07
+#define S_CMD_STATUS_SUCCESS         0x0
+#define S_CMD_STATUS_TIMEOUT         0x5
+
 #define IE_OFFSET                    0x38
 #define IE_M_RX_FIFO_FULL_SHIFT      31
 #define IE_M_RX_THLD_SHIFT           30
 #define IE_M_START_BUSY_SHIFT        28
 #define IE_M_TX_UNDERRUN_SHIFT       27
+#define IE_S_RX_FIFO_FULL_SHIFT      26
+#define IE_S_RX_THLD_SHIFT           25
+#define IE_S_RX_EVENT_SHIFT          24
+#define IE_S_START_BUSY_SHIFT        23
+#define IE_S_TX_UNDERRUN_SHIFT       22
+#define IE_S_RD_EVENT_SHIFT          21
 
 #define IS_OFFSET                    0x3c
 #define IS_M_RX_FIFO_FULL_SHIFT      31
 #define IS_M_RX_THLD_SHIFT           30
 #define IS_M_START_BUSY_SHIFT        28
 #define IS_M_TX_UNDERRUN_SHIFT       27
+#define IS_S_RX_FIFO_FULL_SHIFT      26
+#define IS_S_RX_THLD_SHIFT           25
+#define IS_S_RX_EVENT_SHIFT          24
+#define IS_S_START_BUSY_SHIFT        23
+#define IS_S_TX_UNDERRUN_SHIFT       22
+#define IS_S_RD_EVENT_SHIFT          21
 
 #define M_TX_OFFSET                  0x40
 #define M_TX_WR_STATUS_SHIFT         31
@@ -78,19 +128,71 @@
 #define M_RX_DATA_SHIFT              0
 #define M_RX_DATA_MASK               0xff
 
+#define S_TX_OFFSET                  0x48
+#define S_TX_WR_STATUS_SHIFT         31
+#define S_TX_DATA_SHIFT              0
+#define S_TX_DATA_MASK               0xff
+
+#define S_RX_OFFSET                  0x4c
+#define S_RX_STATUS_SHIFT            30
+#define S_RX_STATUS_MASK             0x03
+#define S_RX_PEC_ERR_SHIFT           29
+#define S_RX_DATA_SHIFT              0
+#define S_RX_DATA_MASK               0xff
+
 #define I2C_TIMEOUT_MSEC             50000
 #define M_TX_RX_FIFO_SIZE            64
+#define M_RX_FIFO_MAX_THLD_VALUE     (M_TX_RX_FIFO_SIZE - 1)
+
+#define M_RX_MAX_READ_LEN            255
+#define M_RX_FIFO_THLD_VALUE         50
+
+#define IE_M_ALL_INTERRUPT_SHIFT     27
+#define IE_M_ALL_INTERRUPT_MASK      0x1e
+
+#define SLAVE_READ_WRITE_BIT_MASK    0x1
+#define SLAVE_READ_WRITE_BIT_SHIFT   0x1
+#define SLAVE_MAX_SIZE_TRANSACTION   64
+#define SLAVE_CLOCK_STRETCH_TIME     25
+
+#define IE_S_ALL_INTERRUPT_SHIFT     21
+#define IE_S_ALL_INTERRUPT_MASK      0x3f
+
+enum i2c_slave_read_status {
+	I2C_SLAVE_RX_FIFO_EMPTY = 0,
+	I2C_SLAVE_RX_START,
+	I2C_SLAVE_RX_DATA,
+	I2C_SLAVE_RX_END,
+};
+
+enum i2c_slave_xfer_dir {
+	I2C_SLAVE_DIR_READ = 0,
+	I2C_SLAVE_DIR_WRITE,
+	I2C_SLAVE_DIR_NONE,
+};
 
 enum bus_speed_index {
 	I2C_SPD_100K = 0,
 	I2C_SPD_400K,
 };
 
+enum bcm_iproc_i2c_type {
+	IPROC_I2C,
+	IPROC_I2C_NIC
+};
+
 struct bcm_iproc_i2c_dev {
 	struct device *device;
+	enum bcm_iproc_i2c_type type;
 	int irq;
 
 	void __iomem *base;
+	void __iomem *idm_base;
+
+	u32 ape_addr_mask;
+
+	/* lock for indirect access through IDM */
+	spinlock_t idm_lock;
 
 	struct i2c_adapter adapter;
 	unsigned int bus_speed;
@@ -100,68 +202,332 @@ struct bcm_iproc_i2c_dev {
 
 	struct i2c_msg *msg;
 
+	struct i2c_client *slave;
+	enum i2c_slave_xfer_dir xfer_dir;
+
 	/* bytes that have been transferred */
 	unsigned int tx_bytes;
+	/* bytes that have been read */
+	unsigned int rx_bytes;
+	unsigned int thld_bytes;
 };
 
 /*
  * Can be expanded in the future if more interrupt status bits are utilized
  */
-#define ISR_MASK (BIT(IS_M_START_BUSY_SHIFT) | BIT(IS_M_TX_UNDERRUN_SHIFT))
+#define ISR_MASK (BIT(IS_M_START_BUSY_SHIFT) | BIT(IS_M_TX_UNDERRUN_SHIFT)\
+		| BIT(IS_M_RX_THLD_SHIFT))
 
-static irqreturn_t bcm_iproc_i2c_isr(int irq, void *data)
+#define ISR_MASK_SLAVE (BIT(IS_S_START_BUSY_SHIFT)\
+		| BIT(IS_S_RX_EVENT_SHIFT) | BIT(IS_S_RD_EVENT_SHIFT))
+
+static int bcm_iproc_i2c_reg_slave(struct i2c_client *slave);
+static int bcm_iproc_i2c_unreg_slave(struct i2c_client *slave);
+static void bcm_iproc_i2c_enable_disable(struct bcm_iproc_i2c_dev *iproc_i2c,
+					 bool enable);
+
+static inline u32 iproc_i2c_rd_reg(struct bcm_iproc_i2c_dev *iproc_i2c,
+				   u32 offset)
 {
-	struct bcm_iproc_i2c_dev *iproc_i2c = data;
-	u32 status = readl(iproc_i2c->base + IS_OFFSET);
+	u32 val;
 
-	status &= ISR_MASK;
+	if (iproc_i2c->idm_base) {
+		spin_lock(&iproc_i2c->idm_lock);
+		writel(iproc_i2c->ape_addr_mask,
+		       iproc_i2c->idm_base + IDM_CTRL_DIRECT_OFFSET);
+		val = readl(iproc_i2c->base + offset);
+		spin_unlock(&iproc_i2c->idm_lock);
+	} else {
+		val = readl(iproc_i2c->base + offset);
+	}
 
-	if (!status)
-		return IRQ_NONE;
+	return val;
+}
 
-	/* TX FIFO is empty and we have more data to send */
-	if (status & BIT(IS_M_TX_UNDERRUN_SHIFT)) {
-		struct i2c_msg *msg = iproc_i2c->msg;
-		unsigned int tx_bytes = msg->len - iproc_i2c->tx_bytes;
-		unsigned int i;
-		u32 val;
-
-		/* can only fill up to the FIFO size */
-		tx_bytes = min_t(unsigned int, tx_bytes, M_TX_RX_FIFO_SIZE);
-		for (i = 0; i < tx_bytes; i++) {
-			/* start from where we left over */
-			unsigned int idx = iproc_i2c->tx_bytes + i;
+static inline void iproc_i2c_wr_reg(struct bcm_iproc_i2c_dev *iproc_i2c,
+				    u32 offset, u32 val)
+{
+	if (iproc_i2c->idm_base) {
+		spin_lock(&iproc_i2c->idm_lock);
+		writel(iproc_i2c->ape_addr_mask,
+		       iproc_i2c->idm_base + IDM_CTRL_DIRECT_OFFSET);
+		writel(val, iproc_i2c->base + offset);
+		spin_unlock(&iproc_i2c->idm_lock);
+	} else {
+		writel(val, iproc_i2c->base + offset);
+	}
+}
 
-			val = msg->buf[idx];
+static void bcm_iproc_i2c_slave_init(
+	struct bcm_iproc_i2c_dev *iproc_i2c, bool need_reset)
+{
+	u32 val;
 
-			/* mark the last byte */
-			if (idx == msg->len - 1) {
-				u32 tmp;
+	if (need_reset) {
+		/* put controller in reset */
+		val = iproc_i2c_rd_reg(iproc_i2c, CFG_OFFSET);
+		val |= BIT(CFG_RESET_SHIFT);
+		iproc_i2c_wr_reg(iproc_i2c, CFG_OFFSET, val);
 
-				val |= BIT(M_TX_WR_STATUS_SHIFT);
+		/* wait 100 usec per spec */
+		udelay(100);
+
+		/* bring controller out of reset */
+		val &= ~(BIT(CFG_RESET_SHIFT));
+		iproc_i2c_wr_reg(iproc_i2c, CFG_OFFSET, val);
+	}
+
+	/* flush TX/RX FIFOs */
+	val = (BIT(S_FIFO_RX_FLUSH_SHIFT) | BIT(S_FIFO_TX_FLUSH_SHIFT));
+	iproc_i2c_wr_reg(iproc_i2c, S_FIFO_CTRL_OFFSET, val);
+
+	/* Maximum slave stretch time */
+	val = iproc_i2c_rd_reg(iproc_i2c, TIM_CFG_OFFSET);
+	val &= ~(TIM_RAND_SLAVE_STRETCH_MASK << TIM_RAND_SLAVE_STRETCH_SHIFT);
+	val |= (SLAVE_CLOCK_STRETCH_TIME << TIM_RAND_SLAVE_STRETCH_SHIFT);
+	iproc_i2c_wr_reg(iproc_i2c, TIM_CFG_OFFSET, val);
+
+	/* Configure the slave address */
+	val = iproc_i2c_rd_reg(iproc_i2c, S_CFG_SMBUS_ADDR_OFFSET);
+	val |= BIT(S_CFG_EN_NIC_SMB_ADDR3_SHIFT);
+	val &= ~(S_CFG_NIC_SMB_ADDR3_MASK << S_CFG_NIC_SMB_ADDR3_SHIFT);
+	val |= (iproc_i2c->slave->addr << S_CFG_NIC_SMB_ADDR3_SHIFT);
+	iproc_i2c_wr_reg(iproc_i2c, S_CFG_SMBUS_ADDR_OFFSET, val);
+
+	/* clear all pending slave interrupts */
+	iproc_i2c_wr_reg(iproc_i2c, IS_OFFSET, ISR_MASK_SLAVE);
+
+	/* Enable interrupt register for any READ event */
+	val = BIT(IE_S_RD_EVENT_SHIFT);
+	/* Enable interrupt register to indicate a valid byte in receive fifo */
+	val |= BIT(IE_S_RX_EVENT_SHIFT);
+	/* Enable interrupt register for the Slave BUSY command */
+	val |= BIT(IE_S_START_BUSY_SHIFT);
+	iproc_i2c_wr_reg(iproc_i2c, IE_OFFSET, val);
+
+	iproc_i2c->xfer_dir = I2C_SLAVE_DIR_NONE;
+}
+
+static void bcm_iproc_i2c_check_slave_status(
+	struct bcm_iproc_i2c_dev *iproc_i2c)
+{
+	u32 val;
+
+	val = iproc_i2c_rd_reg(iproc_i2c, S_CMD_OFFSET);
+	val = (val >> S_CMD_STATUS_SHIFT) & S_CMD_STATUS_MASK;
+
+	if (val == S_CMD_STATUS_TIMEOUT) {
+		dev_err(iproc_i2c->device, "slave random stretch time timeout\n");
+
+		/* re-initialize i2c for recovery */
+		bcm_iproc_i2c_enable_disable(iproc_i2c, false);
+		bcm_iproc_i2c_slave_init(iproc_i2c, true);
+		bcm_iproc_i2c_enable_disable(iproc_i2c, true);
+	}
+}
+
+static bool bcm_iproc_i2c_slave_isr(struct bcm_iproc_i2c_dev *iproc_i2c,
+				u32 status)
+{
+	u8 value;
+	u32 val;
+	u32 rd_status;
+	u32 tmp;
+
+	/* Start of transaction. check address and populate the direction */
+	if (iproc_i2c->xfer_dir == I2C_SLAVE_DIR_NONE) {
+		tmp = iproc_i2c_rd_reg(iproc_i2c, S_RX_OFFSET);
+		rd_status = (tmp >> S_RX_STATUS_SHIFT) & S_RX_STATUS_MASK;
+		/* This condition checks whether the request is a new request */
+		if (((rd_status == I2C_SLAVE_RX_START) &&
+			(status & BIT(IS_S_RX_EVENT_SHIFT))) ||
+			((rd_status == I2C_SLAVE_RX_END) &&
+			(status & BIT(IS_S_RD_EVENT_SHIFT)))) {
+
+			/* Last bit is W/R bit.
+			 * If 1 then its a read request(by master).
+			 */
+			iproc_i2c->xfer_dir = tmp & SLAVE_READ_WRITE_BIT_MASK;
+			if (iproc_i2c->xfer_dir == I2C_SLAVE_DIR_WRITE)
+				i2c_slave_event(iproc_i2c->slave,
+					I2C_SLAVE_READ_REQUESTED, &value);
+			else
+				i2c_slave_event(iproc_i2c->slave,
+					I2C_SLAVE_WRITE_REQUESTED, &value);
+		}
+	}
+
+	/* read request from master */
+	if ((status & BIT(IS_S_RD_EVENT_SHIFT)) &&
+		(iproc_i2c->xfer_dir == I2C_SLAVE_DIR_WRITE)) {
+		i2c_slave_event(iproc_i2c->slave,
+			I2C_SLAVE_READ_PROCESSED, &value);
+		iproc_i2c_wr_reg(iproc_i2c, S_TX_OFFSET, value);
+
+		val = BIT(S_CMD_START_BUSY_SHIFT);
+		iproc_i2c_wr_reg(iproc_i2c, S_CMD_OFFSET, val);
+	}
+
+	/* write request from master */
+	if ((status & BIT(IS_S_RX_EVENT_SHIFT)) &&
+		(iproc_i2c->xfer_dir == I2C_SLAVE_DIR_READ)) {
+		val = iproc_i2c_rd_reg(iproc_i2c, S_RX_OFFSET);
+		/* Its a write request by Master to Slave.
+		 * We read data present in receive FIFO
+		 */
+		value = (u8)((val >> S_RX_DATA_SHIFT) & S_RX_DATA_MASK);
+		i2c_slave_event(iproc_i2c->slave,
+			I2C_SLAVE_WRITE_RECEIVED, &value);
+
+		/* check the status for the last byte of the transaction */
+		rd_status = (val >> S_RX_STATUS_SHIFT) & S_RX_STATUS_MASK;
+		if (rd_status == I2C_SLAVE_RX_END)
+			iproc_i2c->xfer_dir = I2C_SLAVE_DIR_NONE;
+
+		dev_dbg(iproc_i2c->device, "\nread value = 0x%x\n", value);
+	}
+
+	/* Stop */
+	if (status & BIT(IS_S_START_BUSY_SHIFT)) {
+		i2c_slave_event(iproc_i2c->slave, I2C_SLAVE_STOP, &value);
+		iproc_i2c->xfer_dir = I2C_SLAVE_DIR_NONE;
+	}
+
+	/* clear interrupt status */
+	iproc_i2c_wr_reg(iproc_i2c, IS_OFFSET, status);
+
+	bcm_iproc_i2c_check_slave_status(iproc_i2c);
+	return true;
+}
+
+static void bcm_iproc_i2c_read_valid_bytes(struct bcm_iproc_i2c_dev *iproc_i2c)
+{
+	struct i2c_msg *msg = iproc_i2c->msg;
+
+	/* Read valid data from RX FIFO */
+	while (iproc_i2c->rx_bytes < msg->len) {
+		if (!((iproc_i2c_rd_reg(iproc_i2c, M_FIFO_CTRL_OFFSET) >> M_FIFO_RX_CNT_SHIFT)
+		      & M_FIFO_RX_CNT_MASK))
+			break;
+
+		msg->buf[iproc_i2c->rx_bytes] =
+			(iproc_i2c_rd_reg(iproc_i2c, M_RX_OFFSET) >>
+			M_RX_DATA_SHIFT) & M_RX_DATA_MASK;
+		iproc_i2c->rx_bytes++;
+	}
+}
+
+static void bcm_iproc_i2c_send(struct bcm_iproc_i2c_dev *iproc_i2c)
+{
+	struct i2c_msg *msg = iproc_i2c->msg;
+	unsigned int tx_bytes = msg->len - iproc_i2c->tx_bytes;
+	unsigned int i;
+	u32 val;
+
+	/* can only fill up to the FIFO size */
+	tx_bytes = min_t(unsigned int, tx_bytes, M_TX_RX_FIFO_SIZE);
+	for (i = 0; i < tx_bytes; i++) {
+		/* start from where we left over */
+		unsigned int idx = iproc_i2c->tx_bytes + i;
+
+		val = msg->buf[idx];
+
+		/* mark the last byte */
+		if (idx == msg->len - 1) {
+			val |= BIT(M_TX_WR_STATUS_SHIFT);
+
+			if (iproc_i2c->irq) {
+				u32 tmp;
 
 				/*
-				 * Since this is the last byte, we should
-				 * now disable TX FIFO underrun interrupt
+				 * Since this is the last byte, we should now
+				 * disable TX FIFO underrun interrupt
 				 */
-				tmp = readl(iproc_i2c->base + IE_OFFSET);
+				tmp = iproc_i2c_rd_reg(iproc_i2c, IE_OFFSET);
 				tmp &= ~BIT(IE_M_TX_UNDERRUN_SHIFT);
-				writel(tmp, iproc_i2c->base + IE_OFFSET);
+				iproc_i2c_wr_reg(iproc_i2c, IE_OFFSET,
+						 tmp);
 			}
+		}
+
+		/* load data into TX FIFO */
+		iproc_i2c_wr_reg(iproc_i2c, M_TX_OFFSET, val);
+	}
 
-			/* load data into TX FIFO */
-			writel(val, iproc_i2c->base + M_TX_OFFSET);
+	/* update number of transferred bytes */
+	iproc_i2c->tx_bytes += tx_bytes;
+}
+
+static void bcm_iproc_i2c_read(struct bcm_iproc_i2c_dev *iproc_i2c)
+{
+	struct i2c_msg *msg = iproc_i2c->msg;
+	u32 bytes_left, val;
+
+	bcm_iproc_i2c_read_valid_bytes(iproc_i2c);
+	bytes_left = msg->len - iproc_i2c->rx_bytes;
+	if (bytes_left == 0) {
+		if (iproc_i2c->irq) {
+			/* finished reading all data, disable rx thld event */
+			val = iproc_i2c_rd_reg(iproc_i2c, IE_OFFSET);
+			val &= ~BIT(IS_M_RX_THLD_SHIFT);
+			iproc_i2c_wr_reg(iproc_i2c, IE_OFFSET, val);
 		}
-		/* update number of transferred bytes */
-		iproc_i2c->tx_bytes += tx_bytes;
+	} else if (bytes_left < iproc_i2c->thld_bytes) {
+		/* set bytes left as threshold */
+		val = iproc_i2c_rd_reg(iproc_i2c, M_FIFO_CTRL_OFFSET);
+		val &= ~(M_FIFO_RX_THLD_MASK << M_FIFO_RX_THLD_SHIFT);
+		val |= (bytes_left << M_FIFO_RX_THLD_SHIFT);
+		iproc_i2c_wr_reg(iproc_i2c, M_FIFO_CTRL_OFFSET, val);
+		iproc_i2c->thld_bytes = bytes_left;
 	}
+	/*
+	 * bytes_left >= iproc_i2c->thld_bytes,
+	 * hence no need to change the THRESHOLD SET.
+	 * It will remain as iproc_i2c->thld_bytes itself
+	 */
+}
+
+static void bcm_iproc_i2c_process_m_event(struct bcm_iproc_i2c_dev *iproc_i2c,
+					  u32 status)
+{
+	/* TX FIFO is empty and we have more data to send */
+	if (status & BIT(IS_M_TX_UNDERRUN_SHIFT))
+		bcm_iproc_i2c_send(iproc_i2c);
 
+	/* RX FIFO threshold is reached and data needs to be read out */
+	if (status & BIT(IS_M_RX_THLD_SHIFT))
+		bcm_iproc_i2c_read(iproc_i2c);
+
+	/* transfer is done */
 	if (status & BIT(IS_M_START_BUSY_SHIFT)) {
 		iproc_i2c->xfer_is_done = 1;
-		complete(&iproc_i2c->done);
+		if (iproc_i2c->irq)
+			complete(&iproc_i2c->done);
 	}
+}
 
-	writel(status, iproc_i2c->base + IS_OFFSET);
+static irqreturn_t bcm_iproc_i2c_isr(int irq, void *data)
+{
+	struct bcm_iproc_i2c_dev *iproc_i2c = data;
+	u32 status = iproc_i2c_rd_reg(iproc_i2c, IS_OFFSET);
+	bool ret;
+	u32 sl_status = status & ISR_MASK_SLAVE;
+
+	if (sl_status) {
+		ret = bcm_iproc_i2c_slave_isr(iproc_i2c, sl_status);
+		if (ret)
+			return IRQ_HANDLED;
+		else
+			return IRQ_NONE;
+	}
+
+	status &= ISR_MASK;
+	if (!status)
+		return IRQ_NONE;
+
+	/* process all master based events */
+	bcm_iproc_i2c_process_m_event(iproc_i2c, status);
+	iproc_i2c_wr_reg(iproc_i2c, IS_OFFSET, status);
 
 	return IRQ_HANDLED;
 }
@@ -171,26 +537,29 @@ static int bcm_iproc_i2c_init(struct bcm_iproc_i2c_dev *iproc_i2c)
 	u32 val;
 
 	/* put controller in reset */
-	val = readl(iproc_i2c->base + CFG_OFFSET);
-	val |= 1 << CFG_RESET_SHIFT;
-	val &= ~(1 << CFG_EN_SHIFT);
-	writel(val, iproc_i2c->base + CFG_OFFSET);
+	val = iproc_i2c_rd_reg(iproc_i2c, CFG_OFFSET);
+	val |= BIT(CFG_RESET_SHIFT);
+	val &= ~(BIT(CFG_EN_SHIFT));
+	iproc_i2c_wr_reg(iproc_i2c, CFG_OFFSET, val);
 
 	/* wait 100 usec per spec */
 	udelay(100);
 
 	/* bring controller out of reset */
-	val &= ~(1 << CFG_RESET_SHIFT);
-	writel(val, iproc_i2c->base + CFG_OFFSET);
+	val &= ~(BIT(CFG_RESET_SHIFT));
+	iproc_i2c_wr_reg(iproc_i2c, CFG_OFFSET, val);
 
 	/* flush TX/RX FIFOs and set RX FIFO threshold to zero */
-	val = (1 << M_FIFO_RX_FLUSH_SHIFT) | (1 << M_FIFO_TX_FLUSH_SHIFT);
-	writel(val, iproc_i2c->base + M_FIFO_CTRL_OFFSET);
+	val = (BIT(M_FIFO_RX_FLUSH_SHIFT) | BIT(M_FIFO_TX_FLUSH_SHIFT));
+	iproc_i2c_wr_reg(iproc_i2c, M_FIFO_CTRL_OFFSET, val);
 	/* disable all interrupts */
-	writel(0, iproc_i2c->base + IE_OFFSET);
+	val = iproc_i2c_rd_reg(iproc_i2c, IE_OFFSET);
+	val &= ~(IE_M_ALL_INTERRUPT_MASK <<
+			IE_M_ALL_INTERRUPT_SHIFT);
+	iproc_i2c_wr_reg(iproc_i2c, IE_OFFSET, val);
 
 	/* clear all pending interrupts */
-	writel(0xffffffff, iproc_i2c->base + IS_OFFSET);
+	iproc_i2c_wr_reg(iproc_i2c, IS_OFFSET, 0xffffffff);
 
 	return 0;
 }
@@ -200,12 +569,12 @@ static void bcm_iproc_i2c_enable_disable(struct bcm_iproc_i2c_dev *iproc_i2c,
 {
 	u32 val;
 
-	val = readl(iproc_i2c->base + CFG_OFFSET);
+	val = iproc_i2c_rd_reg(iproc_i2c, CFG_OFFSET);
 	if (enable)
 		val |= BIT(CFG_EN_SHIFT);
 	else
 		val &= ~BIT(CFG_EN_SHIFT);
-	writel(val, iproc_i2c->base + CFG_OFFSET);
+	iproc_i2c_wr_reg(iproc_i2c, CFG_OFFSET, val);
 }
 
 static int bcm_iproc_i2c_check_status(struct bcm_iproc_i2c_dev *iproc_i2c,
@@ -213,7 +582,7 @@ static int bcm_iproc_i2c_check_status(struct bcm_iproc_i2c_dev *iproc_i2c,
 {
 	u32 val;
 
-	val = readl(iproc_i2c->base + M_CMD_OFFSET);
+	val = iproc_i2c_rd_reg(iproc_i2c, M_CMD_OFFSET);
 	val = (val >> M_CMD_STATUS_SHIFT) & M_CMD_STATUS_MASK;
 
 	switch (val) {
@@ -236,6 +605,14 @@ static int bcm_iproc_i2c_check_status(struct bcm_iproc_i2c_dev *iproc_i2c,
 		dev_dbg(iproc_i2c->device, "bus timeout\n");
 		return -ETIMEDOUT;
 
+	case M_CMD_STATUS_FIFO_UNDERRUN:
+		dev_dbg(iproc_i2c->device, "FIFO under-run\n");
+		return -ENXIO;
+
+	case M_CMD_STATUS_RX_FIFO_FULL:
+		dev_dbg(iproc_i2c->device, "RX FIFO full\n");
+		return -ETIMEDOUT;
+
 	default:
 		dev_dbg(iproc_i2c->device, "unknown error code=%d\n", val);
 
@@ -248,18 +625,76 @@ static int bcm_iproc_i2c_check_status(struct bcm_iproc_i2c_dev *iproc_i2c,
 	}
 }
 
+static int bcm_iproc_i2c_xfer_wait(struct bcm_iproc_i2c_dev *iproc_i2c,
+				   struct i2c_msg *msg,
+				   u32 cmd)
+{
+	unsigned long time_left = msecs_to_jiffies(I2C_TIMEOUT_MSEC);
+	u32 val, status;
+	int ret;
+
+	iproc_i2c_wr_reg(iproc_i2c, M_CMD_OFFSET, cmd);
+
+	if (iproc_i2c->irq) {
+		time_left = wait_for_completion_timeout(&iproc_i2c->done,
+							time_left);
+		/* disable all interrupts */
+		iproc_i2c_wr_reg(iproc_i2c, IE_OFFSET, 0);
+		/* read it back to flush the write */
+		iproc_i2c_rd_reg(iproc_i2c, IE_OFFSET);
+		/* make sure the interrupt handler isn't running */
+		synchronize_irq(iproc_i2c->irq);
+
+	} else { /* polling mode */
+		unsigned long timeout = jiffies + time_left;
+
+		do {
+			status = iproc_i2c_rd_reg(iproc_i2c,
+						  IS_OFFSET) & ISR_MASK;
+			bcm_iproc_i2c_process_m_event(iproc_i2c, status);
+			iproc_i2c_wr_reg(iproc_i2c, IS_OFFSET, status);
+
+			if (time_after(jiffies, timeout)) {
+				time_left = 0;
+				break;
+			}
+
+			cpu_relax();
+			cond_resched();
+		} while (!iproc_i2c->xfer_is_done);
+	}
+
+	if (!time_left && !iproc_i2c->xfer_is_done) {
+		dev_err(iproc_i2c->device, "transaction timed out\n");
+
+		/* flush both TX/RX FIFOs */
+		val = BIT(M_FIFO_RX_FLUSH_SHIFT) | BIT(M_FIFO_TX_FLUSH_SHIFT);
+		iproc_i2c_wr_reg(iproc_i2c, M_FIFO_CTRL_OFFSET, val);
+		return -ETIMEDOUT;
+	}
+
+	ret = bcm_iproc_i2c_check_status(iproc_i2c, msg);
+	if (ret) {
+		/* flush both TX/RX FIFOs */
+		val = BIT(M_FIFO_RX_FLUSH_SHIFT) | BIT(M_FIFO_TX_FLUSH_SHIFT);
+		iproc_i2c_wr_reg(iproc_i2c, M_FIFO_CTRL_OFFSET, val);
+		return ret;
+	}
+
+	return 0;
+}
+
 static int bcm_iproc_i2c_xfer_single_msg(struct bcm_iproc_i2c_dev *iproc_i2c,
 					 struct i2c_msg *msg)
 {
-	int ret, i;
+	int i;
 	u8 addr;
-	u32 val;
+	u32 val, tmp, val_intr_en;
 	unsigned int tx_bytes;
-	unsigned long time_left = msecs_to_jiffies(I2C_TIMEOUT_MSEC);
 
 	/* check if bus is busy */
-	if (!!(readl(iproc_i2c->base + M_CMD_OFFSET) &
-	       BIT(M_CMD_START_BUSY_SHIFT))) {
+	if (!!(iproc_i2c_rd_reg(iproc_i2c,
+				M_CMD_OFFSET) & BIT(M_CMD_START_BUSY_SHIFT))) {
 		dev_warn(iproc_i2c->device, "bus is busy\n");
 		return -EBUSY;
 	}
@@ -268,7 +703,7 @@ static int bcm_iproc_i2c_xfer_single_msg(struct bcm_iproc_i2c_dev *iproc_i2c,
 
 	/* format and load slave address into the TX FIFO */
 	addr = i2c_8bit_addr_from_msg(msg);
-	writel(addr, iproc_i2c->base + M_TX_OFFSET);
+	iproc_i2c_wr_reg(iproc_i2c, M_TX_OFFSET, addr);
 
 	/*
 	 * For a write transaction, load data into the TX FIFO. Only allow
@@ -282,15 +717,17 @@ static int bcm_iproc_i2c_xfer_single_msg(struct bcm_iproc_i2c_dev *iproc_i2c,
 
 			/* mark the last byte */
 			if (i == msg->len - 1)
-				val |= 1 << M_TX_WR_STATUS_SHIFT;
+				val |= BIT(M_TX_WR_STATUS_SHIFT);
 
-			writel(val, iproc_i2c->base + M_TX_OFFSET);
+			iproc_i2c_wr_reg(iproc_i2c, M_TX_OFFSET, val);
 		}
 		iproc_i2c->tx_bytes = tx_bytes;
 	}
 
 	/* mark as incomplete before starting the transaction */
-	reinit_completion(&iproc_i2c->done);
+	if (iproc_i2c->irq)
+		reinit_completion(&iproc_i2c->done);
+
 	iproc_i2c->xfer_is_done = 0;
 
 	/*
@@ -298,7 +735,7 @@ static int bcm_iproc_i2c_xfer_single_msg(struct bcm_iproc_i2c_dev *iproc_i2c,
 	 * transaction is done, i.e., the internal start_busy bit, transitions
 	 * from 1 to 0.
 	 */
-	val = BIT(IE_M_START_BUSY_SHIFT);
+	val_intr_en = BIT(IE_M_START_BUSY_SHIFT);
 
 	/*
 	 * If TX data size is larger than the TX FIFO, need to enable TX
@@ -307,9 +744,7 @@ static int bcm_iproc_i2c_xfer_single_msg(struct bcm_iproc_i2c_dev *iproc_i2c,
 	 */
 	if (!(msg->flags & I2C_M_RD) &&
 	    msg->len > iproc_i2c->tx_bytes)
-		val |= BIT(IE_M_TX_UNDERRUN_SHIFT);
-
-	writel(val, iproc_i2c->base + IE_OFFSET);
+		val_intr_en |= BIT(IE_M_TX_UNDERRUN_SHIFT);
 
 	/*
 	 * Now we can activate the transfer. For a read operation, specify the
@@ -317,54 +752,31 @@ static int bcm_iproc_i2c_xfer_single_msg(struct bcm_iproc_i2c_dev *iproc_i2c,
 	 */
 	val = BIT(M_CMD_START_BUSY_SHIFT);
 	if (msg->flags & I2C_M_RD) {
+		iproc_i2c->rx_bytes = 0;
+		if (msg->len > M_RX_FIFO_MAX_THLD_VALUE)
+			iproc_i2c->thld_bytes = M_RX_FIFO_THLD_VALUE;
+		else
+			iproc_i2c->thld_bytes = msg->len;
+
+		/* set threshold value */
+		tmp = iproc_i2c_rd_reg(iproc_i2c, M_FIFO_CTRL_OFFSET);
+		tmp &= ~(M_FIFO_RX_THLD_MASK << M_FIFO_RX_THLD_SHIFT);
+		tmp |= iproc_i2c->thld_bytes << M_FIFO_RX_THLD_SHIFT;
+		iproc_i2c_wr_reg(iproc_i2c, M_FIFO_CTRL_OFFSET, tmp);
+
+		/* enable the RX threshold interrupt */
+		val_intr_en |= BIT(IE_M_RX_THLD_SHIFT);
+
 		val |= (M_CMD_PROTOCOL_BLK_RD << M_CMD_PROTOCOL_SHIFT) |
 		       (msg->len << M_CMD_RD_CNT_SHIFT);
 	} else {
 		val |= (M_CMD_PROTOCOL_BLK_WR << M_CMD_PROTOCOL_SHIFT);
 	}
-	writel(val, iproc_i2c->base + M_CMD_OFFSET);
-
-	time_left = wait_for_completion_timeout(&iproc_i2c->done, time_left);
-
-	/* disable all interrupts */
-	writel(0, iproc_i2c->base + IE_OFFSET);
-	/* read it back to flush the write */
-	readl(iproc_i2c->base + IE_OFFSET);
-
-	/* make sure the interrupt handler isn't running */
-	synchronize_irq(iproc_i2c->irq);
 
-	if (!time_left && !iproc_i2c->xfer_is_done) {
-		dev_err(iproc_i2c->device, "transaction timed out\n");
+	if (iproc_i2c->irq)
+		iproc_i2c_wr_reg(iproc_i2c, IE_OFFSET, val_intr_en);
 
-		/* flush FIFOs */
-		val = (1 << M_FIFO_RX_FLUSH_SHIFT) |
-		      (1 << M_FIFO_TX_FLUSH_SHIFT);
-		writel(val, iproc_i2c->base + M_FIFO_CTRL_OFFSET);
-		return -ETIMEDOUT;
-	}
-
-	ret = bcm_iproc_i2c_check_status(iproc_i2c, msg);
-	if (ret) {
-		/* flush both TX/RX FIFOs */
-		val = (1 << M_FIFO_RX_FLUSH_SHIFT) |
-		      (1 << M_FIFO_TX_FLUSH_SHIFT);
-		writel(val, iproc_i2c->base + M_FIFO_CTRL_OFFSET);
-		return ret;
-	}
-
-	/*
-	 * For a read operation, we now need to load the data from FIFO
-	 * into the memory buffer
-	 */
-	if (msg->flags & I2C_M_RD) {
-		for (i = 0; i < msg->len; i++) {
-			msg->buf[i] = (readl(iproc_i2c->base + M_RX_OFFSET) >>
-				      M_RX_DATA_SHIFT) & M_RX_DATA_MASK;
-		}
-	}
-
-	return 0;
+	return bcm_iproc_i2c_xfer_wait(iproc_i2c, msg, val);
 }
 
 static int bcm_iproc_i2c_xfer(struct i2c_adapter *adapter,
@@ -387,17 +799,23 @@ static int bcm_iproc_i2c_xfer(struct i2c_adapter *adapter,
 
 static uint32_t bcm_iproc_i2c_functionality(struct i2c_adapter *adap)
 {
-	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL;
+	u32 val = I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL;
+
+	if (adap->algo->reg_slave)
+		val |= I2C_FUNC_SLAVE;
+
+	return val;
 }
 
-static const struct i2c_algorithm bcm_iproc_algo = {
+static struct i2c_algorithm bcm_iproc_algo = {
 	.master_xfer = bcm_iproc_i2c_xfer,
 	.functionality = bcm_iproc_i2c_functionality,
+	.reg_slave = bcm_iproc_i2c_reg_slave,
+	.unreg_slave = bcm_iproc_i2c_unreg_slave,
 };
 
-static const struct i2c_adapter_quirks bcm_iproc_i2c_quirks = {
-	/* need to reserve one byte in the FIFO for the slave address */
-	.max_read_len = M_TX_RX_FIFO_SIZE - 1,
+static struct i2c_adapter_quirks bcm_iproc_i2c_quirks = {
+	.max_read_len = M_RX_MAX_READ_LEN,
 };
 
 static int bcm_iproc_i2c_cfg_speed(struct bcm_iproc_i2c_dev *iproc_i2c)
@@ -425,10 +843,10 @@ static int bcm_iproc_i2c_cfg_speed(struct bcm_iproc_i2c_dev *iproc_i2c)
 	}
 
 	iproc_i2c->bus_speed = bus_speed;
-	val = readl(iproc_i2c->base + TIM_CFG_OFFSET);
-	val &= ~(1 << TIM_CFG_MODE_400_SHIFT);
+	val = iproc_i2c_rd_reg(iproc_i2c, TIM_CFG_OFFSET);
+	val &= ~BIT(TIM_CFG_MODE_400_SHIFT);
 	val |= (bus_speed == 400000) << TIM_CFG_MODE_400_SHIFT;
-	writel(val, iproc_i2c->base + TIM_CFG_OFFSET);
+	iproc_i2c_wr_reg(iproc_i2c, TIM_CFG_OFFSET, val);
 
 	dev_info(iproc_i2c->device, "bus set to %u Hz\n", bus_speed);
 
@@ -449,6 +867,8 @@ static int bcm_iproc_i2c_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, iproc_i2c);
 	iproc_i2c->device = &pdev->dev;
+	iproc_i2c->type =
+		(enum bcm_iproc_i2c_type)of_device_get_match_data(&pdev->dev);
 	init_completion(&iproc_i2c->done);
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
@@ -456,6 +876,29 @@ static int bcm_iproc_i2c_probe(struct platform_device *pdev)
 	if (IS_ERR(iproc_i2c->base))
 		return PTR_ERR(iproc_i2c->base);
 
+	if (iproc_i2c->type == IPROC_I2C_NIC) {
+		res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+		iproc_i2c->idm_base = devm_ioremap_resource(iproc_i2c->device,
+							    res);
+		if (IS_ERR(iproc_i2c->idm_base))
+			return PTR_ERR(iproc_i2c->idm_base);
+
+		ret = of_property_read_u32(iproc_i2c->device->of_node,
+					   "brcm,ape-hsls-addr-mask",
+					   &iproc_i2c->ape_addr_mask);
+		if (ret < 0) {
+			dev_err(iproc_i2c->device,
+				"'brcm,ape-hsls-addr-mask' missing\n");
+			return -EINVAL;
+		}
+
+		spin_lock_init(&iproc_i2c->idm_lock);
+
+		/* no slave support */
+		bcm_iproc_algo.reg_slave = NULL;
+		bcm_iproc_algo.unreg_slave = NULL;
+	}
+
 	ret = bcm_iproc_i2c_init(iproc_i2c);
 	if (ret)
 		return ret;
@@ -465,17 +908,20 @@ static int bcm_iproc_i2c_probe(struct platform_device *pdev)
 		return ret;
 
 	irq = platform_get_irq(pdev, 0);
-	if (irq <= 0) {
-		dev_err(iproc_i2c->device, "no irq resource\n");
-		return irq;
-	}
-	iproc_i2c->irq = irq;
+	if (irq > 0) {
+		ret = devm_request_irq(iproc_i2c->device, irq,
+				       bcm_iproc_i2c_isr, 0, pdev->name,
+				       iproc_i2c);
+		if (ret < 0) {
+			dev_err(iproc_i2c->device,
+				"unable to request irq %i\n", irq);
+			return ret;
+		}
 
-	ret = devm_request_irq(iproc_i2c->device, irq, bcm_iproc_i2c_isr, 0,
-			       pdev->name, iproc_i2c);
-	if (ret < 0) {
-		dev_err(iproc_i2c->device, "unable to request irq %i\n", irq);
-		return ret;
+		iproc_i2c->irq = irq;
+	} else {
+		dev_warn(iproc_i2c->device,
+			 "no irq resource, falling back to poll mode\n");
 	}
 
 	bcm_iproc_i2c_enable_disable(iproc_i2c, true);
@@ -495,10 +941,15 @@ static int bcm_iproc_i2c_remove(struct platform_device *pdev)
 {
 	struct bcm_iproc_i2c_dev *iproc_i2c = platform_get_drvdata(pdev);
 
-	/* make sure there's no pending interrupt when we remove the adapter */
-	writel(0, iproc_i2c->base + IE_OFFSET);
-	readl(iproc_i2c->base + IE_OFFSET);
-	synchronize_irq(iproc_i2c->irq);
+	if (iproc_i2c->irq) {
+		/*
+		 * Make sure there's no pending interrupt when we remove the
+		 * adapter
+		 */
+		iproc_i2c_wr_reg(iproc_i2c, IE_OFFSET, 0);
+		iproc_i2c_rd_reg(iproc_i2c, IE_OFFSET);
+		synchronize_irq(iproc_i2c->irq);
+	}
 
 	i2c_del_adapter(&iproc_i2c->adapter);
 	bcm_iproc_i2c_enable_disable(iproc_i2c, false);
@@ -512,10 +963,15 @@ static int bcm_iproc_i2c_suspend(struct device *dev)
 {
 	struct bcm_iproc_i2c_dev *iproc_i2c = dev_get_drvdata(dev);
 
-	/* make sure there's no pending interrupt when we go into suspend */
-	writel(0, iproc_i2c->base + IE_OFFSET);
-	readl(iproc_i2c->base + IE_OFFSET);
-	synchronize_irq(iproc_i2c->irq);
+	if (iproc_i2c->irq) {
+		/*
+		 * Make sure there's no pending interrupt when we go into
+		 * suspend
+		 */
+		iproc_i2c_wr_reg(iproc_i2c, IE_OFFSET, 0);
+		iproc_i2c_rd_reg(iproc_i2c, IE_OFFSET);
+		synchronize_irq(iproc_i2c->irq);
+	}
 
 	/* now disable the controller */
 	bcm_iproc_i2c_enable_disable(iproc_i2c, false);
@@ -538,10 +994,10 @@ static int bcm_iproc_i2c_resume(struct device *dev)
 		return ret;
 
 	/* configure to the desired bus speed */
-	val = readl(iproc_i2c->base + TIM_CFG_OFFSET);
-	val &= ~(1 << TIM_CFG_MODE_400_SHIFT);
+	val = iproc_i2c_rd_reg(iproc_i2c, TIM_CFG_OFFSET);
+	val &= ~BIT(TIM_CFG_MODE_400_SHIFT);
 	val |= (iproc_i2c->bus_speed == 400000) << TIM_CFG_MODE_400_SHIFT;
-	writel(val, iproc_i2c->base + TIM_CFG_OFFSET);
+	iproc_i2c_wr_reg(iproc_i2c, TIM_CFG_OFFSET, val);
 
 	bcm_iproc_i2c_enable_disable(iproc_i2c, true);
 
@@ -558,8 +1014,54 @@ static const struct dev_pm_ops bcm_iproc_i2c_pm_ops = {
 #define BCM_IPROC_I2C_PM_OPS NULL
 #endif /* CONFIG_PM_SLEEP */
 
+
+static int bcm_iproc_i2c_reg_slave(struct i2c_client *slave)
+{
+	struct bcm_iproc_i2c_dev *iproc_i2c = i2c_get_adapdata(slave->adapter);
+
+	if (iproc_i2c->slave)
+		return -EBUSY;
+
+	if (slave->flags & I2C_CLIENT_TEN)
+		return -EAFNOSUPPORT;
+
+	iproc_i2c->slave = slave;
+	bcm_iproc_i2c_slave_init(iproc_i2c, false);
+	return 0;
+}
+
+static int bcm_iproc_i2c_unreg_slave(struct i2c_client *slave)
+{
+	u32 tmp;
+	struct bcm_iproc_i2c_dev *iproc_i2c = i2c_get_adapdata(slave->adapter);
+
+	if (!iproc_i2c->slave)
+		return -EINVAL;
+
+	iproc_i2c->slave = NULL;
+
+	/* disable all slave interrupts */
+	tmp = iproc_i2c_rd_reg(iproc_i2c, IE_OFFSET);
+	tmp &= ~(IE_S_ALL_INTERRUPT_MASK <<
+			IE_S_ALL_INTERRUPT_SHIFT);
+	iproc_i2c_wr_reg(iproc_i2c, IE_OFFSET, tmp);
+
+	/* Erase the slave address programmed */
+	tmp = iproc_i2c_rd_reg(iproc_i2c, S_CFG_SMBUS_ADDR_OFFSET);
+	tmp &= ~BIT(S_CFG_EN_NIC_SMB_ADDR3_SHIFT);
+	iproc_i2c_wr_reg(iproc_i2c, S_CFG_SMBUS_ADDR_OFFSET, tmp);
+
+	return 0;
+}
+
 static const struct of_device_id bcm_iproc_i2c_of_match[] = {
-	{ .compatible = "brcm,iproc-i2c" },
+	{
+		.compatible = "brcm,iproc-i2c",
+		.data = (int *)IPROC_I2C,
+	}, {
+		.compatible = "brcm,iproc-nic-i2c",
+		.data = (int *)IPROC_I2C_NIC,
+	},
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, bcm_iproc_i2c_of_match);
diff --git a/drivers/i2c/busses/i2c-brcmstb.c b/drivers/i2c/busses/i2c-brcmstb.c
index f4d862234980..506991596b68 100644
--- a/drivers/i2c/busses/i2c-brcmstb.c
+++ b/drivers/i2c/busses/i2c-brcmstb.c
@@ -165,7 +165,6 @@ static const struct bsc_clk_param bsc_clk[] = {
 struct brcmstb_i2c_dev {
 	struct device *device;
 	void __iomem *base;
-	void __iomem *irq_base;
 	int irq;
 	struct bsc_regs *bsc_regmap;
 	struct i2c_adapter adapter;
diff --git a/drivers/i2c/busses/i2c-designware-common.c b/drivers/i2c/busses/i2c-designware-common.c
index a4730111d290..2de7452fcd6d 100644
--- a/drivers/i2c/busses/i2c-designware-common.c
+++ b/drivers/i2c/busses/i2c-designware-common.c
@@ -251,13 +251,27 @@ unsigned long i2c_dw_clk_rate(struct dw_i2c_dev *dev)
 
 int i2c_dw_prepare_clk(struct dw_i2c_dev *dev, bool prepare)
 {
+	int ret;
+
 	if (IS_ERR(dev->clk))
 		return PTR_ERR(dev->clk);
 
-	if (prepare)
-		return clk_prepare_enable(dev->clk);
+	if (prepare) {
+		/* Optional interface clock */
+		ret = clk_prepare_enable(dev->pclk);
+		if (ret)
+			return ret;
+
+		ret = clk_prepare_enable(dev->clk);
+		if (ret)
+			clk_disable_unprepare(dev->pclk);
+
+		return ret;
+	}
 
 	clk_disable_unprepare(dev->clk);
+	clk_disable_unprepare(dev->pclk);
+
 	return 0;
 }
 EXPORT_SYMBOL_GPL(i2c_dw_prepare_clk);
diff --git a/drivers/i2c/busses/i2c-designware-core.h b/drivers/i2c/busses/i2c-designware-core.h
index 6b4ef1d38fb2..67edbbde1070 100644
--- a/drivers/i2c/busses/i2c-designware-core.h
+++ b/drivers/i2c/busses/i2c-designware-core.h
@@ -177,6 +177,7 @@
  * @base: IO registers pointer
  * @cmd_complete: tx completion indicator
  * @clk: input reference clock
+ * @pclk: clock required to access the registers
  * @slave: represent an I2C slave device
  * @cmd_err: run time hadware error code
  * @msgs: points to an array of messages currently being transferred
@@ -227,6 +228,7 @@ struct dw_i2c_dev {
 	void __iomem		*ext;
 	struct completion	cmd_complete;
 	struct clk		*clk;
+	struct clk		*pclk;
 	struct reset_control	*rst;
 	struct i2c_client		*slave;
 	u32			(*get_clk_rate_khz) (struct dw_i2c_dev *dev);
diff --git a/drivers/i2c/busses/i2c-designware-platdrv.c b/drivers/i2c/busses/i2c-designware-platdrv.c
index 416f89b8f881..ddfb81872906 100644
--- a/drivers/i2c/busses/i2c-designware-platdrv.c
+++ b/drivers/i2c/busses/i2c-designware-platdrv.c
@@ -344,6 +344,11 @@ static int dw_i2c_plat_probe(struct platform_device *pdev)
 	else
 		i2c_dw_configure_master(dev);
 
+	/* Optional interface clock */
+	dev->pclk = devm_clk_get_optional(&pdev->dev, "pclk");
+	if (IS_ERR(dev->pclk))
+		return PTR_ERR(dev->pclk);
+
 	dev->clk = devm_clk_get(&pdev->dev, NULL);
 	if (!i2c_dw_prepare_clk(dev, true)) {
 		u64 clk_khz;
diff --git a/drivers/i2c/busses/i2c-gpio.c b/drivers/i2c/busses/i2c-gpio.c
index bba5c4627de3..9684a0ac2a6d 100644
--- a/drivers/i2c/busses/i2c-gpio.c
+++ b/drivers/i2c/busses/i2c-gpio.c
@@ -413,6 +413,8 @@ static int i2c_gpio_probe(struct platform_device *pdev)
 
 	if (gpiod_cansleep(priv->sda) || gpiod_cansleep(priv->scl))
 		dev_warn(dev, "Slow GPIO pins might wreak havoc into I2C/SMBus bus timing");
+	else
+		bit_data->can_do_atomic = true;
 
 	bit_data->setsda = i2c_gpio_setsda_val;
 	bit_data->setscl = i2c_gpio_setscl_val;
diff --git a/drivers/i2c/busses/i2c-imx-lpi2c.c b/drivers/i2c/busses/i2c-imx-lpi2c.c
index 06c4c767af32..dc00fabc919a 100644
--- a/drivers/i2c/busses/i2c-imx-lpi2c.c
+++ b/drivers/i2c/busses/i2c-imx-lpi2c.c
@@ -639,8 +639,7 @@ static int lpi2c_imx_remove(struct platform_device *pdev)
 	return 0;
 }
 
-#ifdef CONFIG_PM_SLEEP
-static int lpi2c_runtime_suspend(struct device *dev)
+static int __maybe_unused lpi2c_runtime_suspend(struct device *dev)
 {
 	struct lpi2c_imx_struct *lpi2c_imx = dev_get_drvdata(dev);
 
@@ -650,7 +649,7 @@ static int lpi2c_runtime_suspend(struct device *dev)
 	return 0;
 }
 
-static int lpi2c_runtime_resume(struct device *dev)
+static int __maybe_unused lpi2c_runtime_resume(struct device *dev)
 {
 	struct lpi2c_imx_struct *lpi2c_imx = dev_get_drvdata(dev);
 	int ret;
@@ -671,10 +670,6 @@ static const struct dev_pm_ops lpi2c_pm_ops = {
 	SET_RUNTIME_PM_OPS(lpi2c_runtime_suspend,
 			   lpi2c_runtime_resume, NULL)
 };
-#define IMX_LPI2C_PM      (&lpi2c_pm_ops)
-#else
-#define IMX_LPI2C_PM      NULL
-#endif
 
 static struct platform_driver lpi2c_imx_driver = {
 	.probe = lpi2c_imx_probe,
@@ -682,7 +677,7 @@ static struct platform_driver lpi2c_imx_driver = {
 	.driver = {
 		.name = DRIVER_NAME,
 		.of_match_table = lpi2c_imx_of_match,
-		.pm = IMX_LPI2C_PM,
+		.pm = &lpi2c_pm_ops,
 	},
 };
 
diff --git a/drivers/i2c/busses/i2c-isch.c b/drivers/i2c/busses/i2c-isch.c
index 5c754bf659e2..f64c1d72f73f 100644
--- a/drivers/i2c/busses/i2c-isch.c
+++ b/drivers/i2c/busses/i2c-isch.c
@@ -30,7 +30,6 @@
 #include <linux/ioport.h>
 #include <linux/i2c.h>
 #include <linux/io.h>
-#include <linux/acpi.h>
 
 /* SCH SMBus address offsets */
 #define SMBHSTCNT	(0 + sch_smba)
diff --git a/drivers/i2c/busses/i2c-mt65xx.c b/drivers/i2c/busses/i2c-mt65xx.c
index 684d651612b3..745b0d03e883 100644
--- a/drivers/i2c/busses/i2c-mt65xx.c
+++ b/drivers/i2c/busses/i2c-mt65xx.c
@@ -35,6 +35,7 @@
 #include <linux/slab.h>
 
 #define I2C_RS_TRANSFER			(1 << 4)
+#define I2C_ARB_LOST			(1 << 3)
 #define I2C_HS_NACKERR			(1 << 2)
 #define I2C_ACKERR			(1 << 1)
 #define I2C_TRANSAC_COMP		(1 << 0)
@@ -76,6 +77,8 @@
 #define I2C_CONTROL_DIR_CHANGE          (0x1 << 4)
 #define I2C_CONTROL_ACKERR_DET_EN       (0x1 << 5)
 #define I2C_CONTROL_TRANSFER_LEN_CHANGE (0x1 << 6)
+#define I2C_CONTROL_DMAACK_EN           (0x1 << 8)
+#define I2C_CONTROL_ASYNC_MODE          (0x1 << 9)
 #define I2C_CONTROL_WRAPPER             (0x1 << 0)
 
 #define I2C_DRV_NAME		"i2c-mt65xx"
@@ -106,40 +109,97 @@ enum mtk_trans_op {
 };
 
 enum I2C_REGS_OFFSET {
-	OFFSET_DATA_PORT = 0x0,
-	OFFSET_SLAVE_ADDR = 0x04,
-	OFFSET_INTR_MASK = 0x08,
-	OFFSET_INTR_STAT = 0x0c,
-	OFFSET_CONTROL = 0x10,
-	OFFSET_TRANSFER_LEN = 0x14,
-	OFFSET_TRANSAC_LEN = 0x18,
-	OFFSET_DELAY_LEN = 0x1c,
-	OFFSET_TIMING = 0x20,
-	OFFSET_START = 0x24,
-	OFFSET_EXT_CONF = 0x28,
-	OFFSET_FIFO_STAT = 0x30,
-	OFFSET_FIFO_THRESH = 0x34,
-	OFFSET_FIFO_ADDR_CLR = 0x38,
-	OFFSET_IO_CONFIG = 0x40,
-	OFFSET_RSV_DEBUG = 0x44,
-	OFFSET_HS = 0x48,
-	OFFSET_SOFTRESET = 0x50,
-	OFFSET_DCM_EN = 0x54,
-	OFFSET_PATH_DIR = 0x60,
-	OFFSET_DEBUGSTAT = 0x64,
-	OFFSET_DEBUGCTRL = 0x68,
-	OFFSET_TRANSFER_LEN_AUX = 0x6c,
-	OFFSET_CLOCK_DIV = 0x70,
+	OFFSET_DATA_PORT,
+	OFFSET_SLAVE_ADDR,
+	OFFSET_INTR_MASK,
+	OFFSET_INTR_STAT,
+	OFFSET_CONTROL,
+	OFFSET_TRANSFER_LEN,
+	OFFSET_TRANSAC_LEN,
+	OFFSET_DELAY_LEN,
+	OFFSET_TIMING,
+	OFFSET_START,
+	OFFSET_EXT_CONF,
+	OFFSET_FIFO_STAT,
+	OFFSET_FIFO_THRESH,
+	OFFSET_FIFO_ADDR_CLR,
+	OFFSET_IO_CONFIG,
+	OFFSET_RSV_DEBUG,
+	OFFSET_HS,
+	OFFSET_SOFTRESET,
+	OFFSET_DCM_EN,
+	OFFSET_PATH_DIR,
+	OFFSET_DEBUGSTAT,
+	OFFSET_DEBUGCTRL,
+	OFFSET_TRANSFER_LEN_AUX,
+	OFFSET_CLOCK_DIV,
+	OFFSET_LTIMING,
+};
+
+static const u16 mt_i2c_regs_v1[] = {
+	[OFFSET_DATA_PORT] = 0x0,
+	[OFFSET_SLAVE_ADDR] = 0x4,
+	[OFFSET_INTR_MASK] = 0x8,
+	[OFFSET_INTR_STAT] = 0xc,
+	[OFFSET_CONTROL] = 0x10,
+	[OFFSET_TRANSFER_LEN] = 0x14,
+	[OFFSET_TRANSAC_LEN] = 0x18,
+	[OFFSET_DELAY_LEN] = 0x1c,
+	[OFFSET_TIMING] = 0x20,
+	[OFFSET_START] = 0x24,
+	[OFFSET_EXT_CONF] = 0x28,
+	[OFFSET_FIFO_STAT] = 0x30,
+	[OFFSET_FIFO_THRESH] = 0x34,
+	[OFFSET_FIFO_ADDR_CLR] = 0x38,
+	[OFFSET_IO_CONFIG] = 0x40,
+	[OFFSET_RSV_DEBUG] = 0x44,
+	[OFFSET_HS] = 0x48,
+	[OFFSET_SOFTRESET] = 0x50,
+	[OFFSET_DCM_EN] = 0x54,
+	[OFFSET_PATH_DIR] = 0x60,
+	[OFFSET_DEBUGSTAT] = 0x64,
+	[OFFSET_DEBUGCTRL] = 0x68,
+	[OFFSET_TRANSFER_LEN_AUX] = 0x6c,
+	[OFFSET_CLOCK_DIV] = 0x70,
+};
+
+static const u16 mt_i2c_regs_v2[] = {
+	[OFFSET_DATA_PORT] = 0x0,
+	[OFFSET_SLAVE_ADDR] = 0x4,
+	[OFFSET_INTR_MASK] = 0x8,
+	[OFFSET_INTR_STAT] = 0xc,
+	[OFFSET_CONTROL] = 0x10,
+	[OFFSET_TRANSFER_LEN] = 0x14,
+	[OFFSET_TRANSAC_LEN] = 0x18,
+	[OFFSET_DELAY_LEN] = 0x1c,
+	[OFFSET_TIMING] = 0x20,
+	[OFFSET_START] = 0x24,
+	[OFFSET_EXT_CONF] = 0x28,
+	[OFFSET_LTIMING] = 0x2c,
+	[OFFSET_HS] = 0x30,
+	[OFFSET_IO_CONFIG] = 0x34,
+	[OFFSET_FIFO_ADDR_CLR] = 0x38,
+	[OFFSET_TRANSFER_LEN_AUX] = 0x44,
+	[OFFSET_CLOCK_DIV] = 0x48,
+	[OFFSET_SOFTRESET] = 0x50,
+	[OFFSET_DEBUGSTAT] = 0xe0,
+	[OFFSET_DEBUGCTRL] = 0xe8,
+	[OFFSET_FIFO_STAT] = 0xf4,
+	[OFFSET_FIFO_THRESH] = 0xf8,
+	[OFFSET_DCM_EN] = 0xf88,
 };
 
 struct mtk_i2c_compatible {
 	const struct i2c_adapter_quirks *quirks;
+	const u16 *regs;
 	unsigned char pmic_i2c: 1;
 	unsigned char dcm: 1;
 	unsigned char auto_restart: 1;
 	unsigned char aux_len_reg: 1;
 	unsigned char support_33bits: 1;
 	unsigned char timing_adjust: 1;
+	unsigned char dma_sync: 1;
+	unsigned char ltiming_adjust: 1;
 };
 
 struct mtk_i2c {
@@ -153,6 +213,7 @@ struct mtk_i2c {
 	struct clk *clk_main;		/* main clock for i2c bus */
 	struct clk *clk_dma;		/* DMA clock for i2c via DMA */
 	struct clk *clk_pmic;		/* PMIC clock for i2c from PMIC */
+	struct clk *clk_arb;		/* Arbitrator clock for i2c */
 	bool have_pmic;			/* can use i2c pins from PMIC */
 	bool use_push_pull;		/* IO config push-pull mode */
 
@@ -162,6 +223,7 @@ struct mtk_i2c {
 	enum mtk_trans_op op;
 	u16 timing_reg;
 	u16 high_speed_reg;
+	u16 ltiming_reg;
 	unsigned char auto_restart;
 	bool ignore_restart_irq;
 	const struct mtk_i2c_compatible *dev_comp;
@@ -181,51 +243,78 @@ static const struct i2c_adapter_quirks mt7622_i2c_quirks = {
 };
 
 static const struct mtk_i2c_compatible mt2712_compat = {
+	.regs = mt_i2c_regs_v1,
 	.pmic_i2c = 0,
 	.dcm = 1,
 	.auto_restart = 1,
 	.aux_len_reg = 1,
 	.support_33bits = 1,
 	.timing_adjust = 1,
+	.dma_sync = 0,
+	.ltiming_adjust = 0,
 };
 
 static const struct mtk_i2c_compatible mt6577_compat = {
 	.quirks = &mt6577_i2c_quirks,
+	.regs = mt_i2c_regs_v1,
 	.pmic_i2c = 0,
 	.dcm = 1,
 	.auto_restart = 0,
 	.aux_len_reg = 0,
 	.support_33bits = 0,
 	.timing_adjust = 0,
+	.dma_sync = 0,
+	.ltiming_adjust = 0,
 };
 
 static const struct mtk_i2c_compatible mt6589_compat = {
 	.quirks = &mt6577_i2c_quirks,
+	.regs = mt_i2c_regs_v1,
 	.pmic_i2c = 1,
 	.dcm = 0,
 	.auto_restart = 0,
 	.aux_len_reg = 0,
 	.support_33bits = 0,
 	.timing_adjust = 0,
+	.dma_sync = 0,
+	.ltiming_adjust = 0,
 };
 
 static const struct mtk_i2c_compatible mt7622_compat = {
 	.quirks = &mt7622_i2c_quirks,
+	.regs = mt_i2c_regs_v1,
 	.pmic_i2c = 0,
 	.dcm = 1,
 	.auto_restart = 1,
 	.aux_len_reg = 1,
 	.support_33bits = 0,
 	.timing_adjust = 0,
+	.dma_sync = 0,
+	.ltiming_adjust = 0,
 };
 
 static const struct mtk_i2c_compatible mt8173_compat = {
+	.regs = mt_i2c_regs_v1,
 	.pmic_i2c = 0,
 	.dcm = 1,
 	.auto_restart = 1,
 	.aux_len_reg = 1,
 	.support_33bits = 1,
 	.timing_adjust = 0,
+	.dma_sync = 0,
+	.ltiming_adjust = 0,
+};
+
+static const struct mtk_i2c_compatible mt8183_compat = {
+	.regs = mt_i2c_regs_v2,
+	.pmic_i2c = 0,
+	.dcm = 0,
+	.auto_restart = 1,
+	.aux_len_reg = 1,
+	.support_33bits = 1,
+	.timing_adjust = 1,
+	.dma_sync = 1,
+	.ltiming_adjust = 1,
 };
 
 static const struct of_device_id mtk_i2c_of_match[] = {
@@ -234,10 +323,22 @@ static const struct of_device_id mtk_i2c_of_match[] = {
 	{ .compatible = "mediatek,mt6589-i2c", .data = &mt6589_compat },
 	{ .compatible = "mediatek,mt7622-i2c", .data = &mt7622_compat },
 	{ .compatible = "mediatek,mt8173-i2c", .data = &mt8173_compat },
+	{ .compatible = "mediatek,mt8183-i2c", .data = &mt8183_compat },
 	{}
 };
 MODULE_DEVICE_TABLE(of, mtk_i2c_of_match);
 
+static u16 mtk_i2c_readw(struct mtk_i2c *i2c, enum I2C_REGS_OFFSET reg)
+{
+	return readw(i2c->base + i2c->dev_comp->regs[reg]);
+}
+
+static void mtk_i2c_writew(struct mtk_i2c *i2c, u16 val,
+			   enum I2C_REGS_OFFSET reg)
+{
+	writew(val, i2c->base + i2c->dev_comp->regs[reg]);
+}
+
 static int mtk_i2c_clock_enable(struct mtk_i2c *i2c)
 {
 	int ret;
@@ -255,8 +356,18 @@ static int mtk_i2c_clock_enable(struct mtk_i2c *i2c)
 		if (ret)
 			goto err_pmic;
 	}
+
+	if (i2c->clk_arb) {
+		ret = clk_prepare_enable(i2c->clk_arb);
+		if (ret)
+			goto err_arb;
+	}
+
 	return 0;
 
+err_arb:
+	if (i2c->have_pmic)
+		clk_disable_unprepare(i2c->clk_pmic);
 err_pmic:
 	clk_disable_unprepare(i2c->clk_main);
 err_main:
@@ -267,6 +378,9 @@ err_main:
 
 static void mtk_i2c_clock_disable(struct mtk_i2c *i2c)
 {
+	if (i2c->clk_arb)
+		clk_disable_unprepare(i2c->clk_arb);
+
 	if (i2c->have_pmic)
 		clk_disable_unprepare(i2c->clk_pmic);
 
@@ -278,31 +392,36 @@ static void mtk_i2c_init_hw(struct mtk_i2c *i2c)
 {
 	u16 control_reg;
 
-	writew(I2C_SOFT_RST, i2c->base + OFFSET_SOFTRESET);
+	mtk_i2c_writew(i2c, I2C_SOFT_RST, OFFSET_SOFTRESET);
 
 	/* Set ioconfig */
 	if (i2c->use_push_pull)
-		writew(I2C_IO_CONFIG_PUSH_PULL, i2c->base + OFFSET_IO_CONFIG);
+		mtk_i2c_writew(i2c, I2C_IO_CONFIG_PUSH_PULL, OFFSET_IO_CONFIG);
 	else
-		writew(I2C_IO_CONFIG_OPEN_DRAIN, i2c->base + OFFSET_IO_CONFIG);
+		mtk_i2c_writew(i2c, I2C_IO_CONFIG_OPEN_DRAIN, OFFSET_IO_CONFIG);
 
 	if (i2c->dev_comp->dcm)
-		writew(I2C_DCM_DISABLE, i2c->base + OFFSET_DCM_EN);
+		mtk_i2c_writew(i2c, I2C_DCM_DISABLE, OFFSET_DCM_EN);
 
 	if (i2c->dev_comp->timing_adjust)
-		writew(I2C_DEFAULT_CLK_DIV - 1, i2c->base + OFFSET_CLOCK_DIV);
+		mtk_i2c_writew(i2c, I2C_DEFAULT_CLK_DIV - 1, OFFSET_CLOCK_DIV);
 
-	writew(i2c->timing_reg, i2c->base + OFFSET_TIMING);
-	writew(i2c->high_speed_reg, i2c->base + OFFSET_HS);
+	mtk_i2c_writew(i2c, i2c->timing_reg, OFFSET_TIMING);
+	mtk_i2c_writew(i2c, i2c->high_speed_reg, OFFSET_HS);
+	if (i2c->dev_comp->ltiming_adjust)
+		mtk_i2c_writew(i2c, i2c->ltiming_reg, OFFSET_LTIMING);
 
 	/* If use i2c pin from PMIC mt6397 side, need set PATH_DIR first */
 	if (i2c->have_pmic)
-		writew(I2C_CONTROL_WRAPPER, i2c->base + OFFSET_PATH_DIR);
+		mtk_i2c_writew(i2c, I2C_CONTROL_WRAPPER, OFFSET_PATH_DIR);
 
 	control_reg = I2C_CONTROL_ACKERR_DET_EN |
 		      I2C_CONTROL_CLK_EXT_EN | I2C_CONTROL_DMA_EN;
-	writew(control_reg, i2c->base + OFFSET_CONTROL);
-	writew(I2C_DELAY_LEN, i2c->base + OFFSET_DELAY_LEN);
+	if (i2c->dev_comp->dma_sync)
+		control_reg |= I2C_CONTROL_DMAACK_EN | I2C_CONTROL_ASYNC_MODE;
+
+	mtk_i2c_writew(i2c, control_reg, OFFSET_CONTROL);
+	mtk_i2c_writew(i2c, I2C_DELAY_LEN, OFFSET_DELAY_LEN);
 
 	writel(I2C_DMA_HARD_RST, i2c->pdmabase + OFFSET_RST);
 	udelay(50);
@@ -390,6 +509,8 @@ static int mtk_i2c_set_speed(struct mtk_i2c *i2c, unsigned int parent_clk)
 	unsigned int clk_src;
 	unsigned int step_cnt;
 	unsigned int sample_cnt;
+	unsigned int l_step_cnt;
+	unsigned int l_sample_cnt;
 	unsigned int target_speed;
 	int ret;
 
@@ -399,11 +520,11 @@ static int mtk_i2c_set_speed(struct mtk_i2c *i2c, unsigned int parent_clk)
 	if (target_speed > MAX_FS_MODE_SPEED) {
 		/* Set master code speed register */
 		ret = mtk_i2c_calculate_speed(i2c, clk_src, MAX_FS_MODE_SPEED,
-					      &step_cnt, &sample_cnt);
+					      &l_step_cnt, &l_sample_cnt);
 		if (ret < 0)
 			return ret;
 
-		i2c->timing_reg = (sample_cnt << 8) | step_cnt;
+		i2c->timing_reg = (l_sample_cnt << 8) | l_step_cnt;
 
 		/* Set the high speed mode register */
 		ret = mtk_i2c_calculate_speed(i2c, clk_src, target_speed,
@@ -413,6 +534,10 @@ static int mtk_i2c_set_speed(struct mtk_i2c *i2c, unsigned int parent_clk)
 
 		i2c->high_speed_reg = I2C_TIME_DEFAULT_VALUE |
 			(sample_cnt << 12) | (step_cnt << 8);
+
+		if (i2c->dev_comp->ltiming_adjust)
+			i2c->ltiming_reg = (l_sample_cnt << 6) | l_step_cnt |
+					   (sample_cnt << 12) | (step_cnt << 9);
 	} else {
 		ret = mtk_i2c_calculate_speed(i2c, clk_src, target_speed,
 					      &step_cnt, &sample_cnt);
@@ -423,6 +548,9 @@ static int mtk_i2c_set_speed(struct mtk_i2c *i2c, unsigned int parent_clk)
 
 		/* Disable the high speed transaction */
 		i2c->high_speed_reg = I2C_TIME_CLR_VALUE;
+
+		if (i2c->dev_comp->ltiming_adjust)
+			i2c->ltiming_reg = (sample_cnt << 6) | step_cnt;
 	}
 
 	return 0;
@@ -454,7 +582,7 @@ static int mtk_i2c_do_transfer(struct mtk_i2c *i2c, struct i2c_msg *msgs,
 
 	reinit_completion(&i2c->msg_complete);
 
-	control_reg = readw(i2c->base + OFFSET_CONTROL) &
+	control_reg = mtk_i2c_readw(i2c, OFFSET_CONTROL) &
 			~(I2C_CONTROL_DIR_CHANGE | I2C_CONTROL_RS);
 	if ((i2c->speed_hz > MAX_FS_MODE_SPEED) || (left_num >= 1))
 		control_reg |= I2C_CONTROL_RS;
@@ -462,40 +590,41 @@ static int mtk_i2c_do_transfer(struct mtk_i2c *i2c, struct i2c_msg *msgs,
 	if (i2c->op == I2C_MASTER_WRRD)
 		control_reg |= I2C_CONTROL_DIR_CHANGE | I2C_CONTROL_RS;
 
-	writew(control_reg, i2c->base + OFFSET_CONTROL);
+	mtk_i2c_writew(i2c, control_reg, OFFSET_CONTROL);
 
 	/* set start condition */
 	if (i2c->speed_hz <= I2C_DEFAULT_SPEED)
-		writew(I2C_ST_START_CON, i2c->base + OFFSET_EXT_CONF);
+		mtk_i2c_writew(i2c, I2C_ST_START_CON, OFFSET_EXT_CONF);
 	else
-		writew(I2C_FS_START_CON, i2c->base + OFFSET_EXT_CONF);
+		mtk_i2c_writew(i2c, I2C_FS_START_CON, OFFSET_EXT_CONF);
 
 	addr_reg = i2c_8bit_addr_from_msg(msgs);
-	writew(addr_reg, i2c->base + OFFSET_SLAVE_ADDR);
+	mtk_i2c_writew(i2c, addr_reg, OFFSET_SLAVE_ADDR);
 
 	/* Clear interrupt status */
-	writew(restart_flag | I2C_HS_NACKERR | I2C_ACKERR |
-	       I2C_TRANSAC_COMP, i2c->base + OFFSET_INTR_STAT);
-	writew(I2C_FIFO_ADDR_CLR, i2c->base + OFFSET_FIFO_ADDR_CLR);
+	mtk_i2c_writew(i2c, restart_flag | I2C_HS_NACKERR | I2C_ACKERR |
+			    I2C_ARB_LOST | I2C_TRANSAC_COMP, OFFSET_INTR_STAT);
+
+	mtk_i2c_writew(i2c, I2C_FIFO_ADDR_CLR, OFFSET_FIFO_ADDR_CLR);
 
 	/* Enable interrupt */
-	writew(restart_flag | I2C_HS_NACKERR | I2C_ACKERR |
-	       I2C_TRANSAC_COMP, i2c->base + OFFSET_INTR_MASK);
+	mtk_i2c_writew(i2c, restart_flag | I2C_HS_NACKERR | I2C_ACKERR |
+			    I2C_ARB_LOST | I2C_TRANSAC_COMP, OFFSET_INTR_MASK);
 
 	/* Set transfer and transaction len */
 	if (i2c->op == I2C_MASTER_WRRD) {
 		if (i2c->dev_comp->aux_len_reg) {
-			writew(msgs->len, i2c->base + OFFSET_TRANSFER_LEN);
-			writew((msgs + 1)->len, i2c->base +
-			       OFFSET_TRANSFER_LEN_AUX);
+			mtk_i2c_writew(i2c, msgs->len, OFFSET_TRANSFER_LEN);
+			mtk_i2c_writew(i2c, (msgs + 1)->len,
+					    OFFSET_TRANSFER_LEN_AUX);
 		} else {
-			writew(msgs->len | ((msgs + 1)->len) << 8,
-			       i2c->base + OFFSET_TRANSFER_LEN);
+			mtk_i2c_writew(i2c, msgs->len | ((msgs + 1)->len) << 8,
+					    OFFSET_TRANSFER_LEN);
 		}
-		writew(I2C_WRRD_TRANAC_VALUE, i2c->base + OFFSET_TRANSAC_LEN);
+		mtk_i2c_writew(i2c, I2C_WRRD_TRANAC_VALUE, OFFSET_TRANSAC_LEN);
 	} else {
-		writew(msgs->len, i2c->base + OFFSET_TRANSFER_LEN);
-		writew(num, i2c->base + OFFSET_TRANSAC_LEN);
+		mtk_i2c_writew(i2c, msgs->len, OFFSET_TRANSFER_LEN);
+		mtk_i2c_writew(i2c, num, OFFSET_TRANSAC_LEN);
 	}
 
 	/* Prepare buffer data to start transfer */
@@ -607,14 +736,14 @@ static int mtk_i2c_do_transfer(struct mtk_i2c *i2c, struct i2c_msg *msgs,
 		if (left_num >= 1)
 			start_reg |= I2C_RS_MUL_CNFG;
 	}
-	writew(start_reg, i2c->base + OFFSET_START);
+	mtk_i2c_writew(i2c, start_reg, OFFSET_START);
 
 	ret = wait_for_completion_timeout(&i2c->msg_complete,
 					  i2c->adap.timeout);
 
 	/* Clear interrupt mask */
-	writew(~(restart_flag | I2C_HS_NACKERR | I2C_ACKERR |
-	       I2C_TRANSAC_COMP), i2c->base + OFFSET_INTR_MASK);
+	mtk_i2c_writew(i2c, ~(restart_flag | I2C_HS_NACKERR | I2C_ACKERR |
+			    I2C_ARB_LOST | I2C_TRANSAC_COMP), OFFSET_INTR_MASK);
 
 	if (i2c->op == I2C_MASTER_WR) {
 		dma_unmap_single(i2c->dev, wpaddr,
@@ -724,8 +853,8 @@ static irqreturn_t mtk_i2c_irq(int irqno, void *dev_id)
 	if (i2c->auto_restart)
 		restart_flag = I2C_RS_TRANSFER;
 
-	intr_stat = readw(i2c->base + OFFSET_INTR_STAT);
-	writew(intr_stat, i2c->base + OFFSET_INTR_STAT);
+	intr_stat = mtk_i2c_readw(i2c, OFFSET_INTR_STAT);
+	mtk_i2c_writew(i2c, intr_stat, OFFSET_INTR_STAT);
 
 	/*
 	 * when occurs ack error, i2c controller generate two interrupts
@@ -737,8 +866,8 @@ static irqreturn_t mtk_i2c_irq(int irqno, void *dev_id)
 	if (i2c->ignore_restart_irq && (i2c->irq_stat & restart_flag)) {
 		i2c->ignore_restart_irq = false;
 		i2c->irq_stat = 0;
-		writew(I2C_RS_MUL_CNFG | I2C_RS_MUL_TRIG | I2C_TRANSAC_START,
-		       i2c->base + OFFSET_START);
+		mtk_i2c_writew(i2c, I2C_RS_MUL_CNFG | I2C_RS_MUL_TRIG |
+				    I2C_TRANSAC_START, OFFSET_START);
 	} else {
 		if (i2c->irq_stat & (I2C_TRANSAC_COMP | restart_flag))
 			complete(&i2c->msg_complete);
@@ -839,6 +968,10 @@ static int mtk_i2c_probe(struct platform_device *pdev)
 		return PTR_ERR(i2c->clk_dma);
 	}
 
+	i2c->clk_arb = devm_clk_get(&pdev->dev, "arb");
+	if (IS_ERR(i2c->clk_arb))
+		i2c->clk_arb = NULL;
+
 	clk = i2c->clk_main;
 	if (i2c->have_pmic) {
 		i2c->clk_pmic = devm_clk_get(&pdev->dev, "pmic");
diff --git a/drivers/i2c/busses/i2c-nomadik.c b/drivers/i2c/busses/i2c-nomadik.c
index 0ed5a41804dc..4f30a43b63da 100644
--- a/drivers/i2c/busses/i2c-nomadik.c
+++ b/drivers/i2c/busses/i2c-nomadik.c
@@ -1070,8 +1070,7 @@ static int nmk_i2c_remove(struct amba_device *adev)
 	/* disable the controller */
 	i2c_clr_bit(dev->virtbase + I2C_CR, I2C_CR_PE);
 	clk_disable_unprepare(dev->clk);
-	if (res)
-		release_mem_region(res->start, resource_size(res));
+	release_mem_region(res->start, resource_size(res));
 
 	return 0;
 }
diff --git a/drivers/i2c/busses/i2c-nvidia-gpu.c b/drivers/i2c/busses/i2c-nvidia-gpu.c
index 4e67d5ed480e..1c8f708f212b 100644
--- a/drivers/i2c/busses/i2c-nvidia-gpu.c
+++ b/drivers/i2c/busses/i2c-nvidia-gpu.c
@@ -253,6 +253,12 @@ static const struct pci_device_id gpu_i2c_ids[] = {
 };
 MODULE_DEVICE_TABLE(pci, gpu_i2c_ids);
 
+static const struct property_entry ccgx_props[] = {
+	/* Use FW built for NVIDIA (nv) only */
+	PROPERTY_ENTRY_U16("ccgx,firmware-build", ('n' << 8) | 'v'),
+	{ }
+};
+
 static int gpu_populate_client(struct gpu_i2c_dev *i2cd, int irq)
 {
 	struct i2c_client *ccgx_client;
@@ -267,6 +273,7 @@ static int gpu_populate_client(struct gpu_i2c_dev *i2cd, int irq)
 		sizeof(i2cd->gpu_ccgx_ucsi->type));
 	i2cd->gpu_ccgx_ucsi->addr = 0x8;
 	i2cd->gpu_ccgx_ucsi->irq = irq;
+	i2cd->gpu_ccgx_ucsi->properties = ccgx_props;
 	ccgx_client = i2c_new_device(&i2cd->adapter, i2cd->gpu_ccgx_ucsi);
 	if (!ccgx_client)
 		return -ENODEV;
diff --git a/drivers/i2c/busses/i2c-ocores.c b/drivers/i2c/busses/i2c-ocores.c
index 4e1a077fb688..c3dabee0aa35 100644
--- a/drivers/i2c/busses/i2c-ocores.c
+++ b/drivers/i2c/busses/i2c-ocores.c
@@ -26,8 +26,6 @@
 #include <linux/spinlock.h>
 #include <linux/jiffies.h>
 
-#define OCORES_FLAG_POLL BIT(0)
-
 /*
  * 'process_lock' exists because ocores_process() and ocores_process_timeout()
  * can't run in parallel.
@@ -37,7 +35,6 @@ struct ocores_i2c {
 	int iobase;
 	u32 reg_shift;
 	u32 reg_io_width;
-	unsigned long flags;
 	wait_queue_head_t wait;
 	struct i2c_adapter adap;
 	struct i2c_msg *msg;
@@ -403,11 +400,7 @@ static int ocores_xfer_polling(struct i2c_adapter *adap,
 static int ocores_xfer(struct i2c_adapter *adap,
 		       struct i2c_msg *msgs, int num)
 {
-	struct ocores_i2c *i2c = i2c_get_adapdata(adap);
-
-	if (i2c->flags & OCORES_FLAG_POLL)
-		return ocores_xfer_polling(adap, msgs, num);
-	return ocores_xfer_core(i2c, msgs, num, false);
+	return ocores_xfer_core(i2c_get_adapdata(adap), msgs, num, false);
 }
 
 static int ocores_init(struct device *dev, struct ocores_i2c *i2c)
@@ -447,8 +440,9 @@ static u32 ocores_func(struct i2c_adapter *adap)
 	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL;
 }
 
-static const struct i2c_algorithm ocores_algorithm = {
+static struct i2c_algorithm ocores_algorithm = {
 	.master_xfer = ocores_xfer,
+	.master_xfer_atomic = ocores_xfer_polling,
 	.functionality = ocores_func,
 };
 
@@ -673,13 +667,13 @@ static int ocores_i2c_probe(struct platform_device *pdev)
 
 	irq = platform_get_irq(pdev, 0);
 	if (irq == -ENXIO) {
-		i2c->flags |= OCORES_FLAG_POLL;
+		ocores_algorithm.master_xfer = ocores_xfer_polling;
 	} else {
 		if (irq < 0)
 			return irq;
 	}
 
-	if (!(i2c->flags & OCORES_FLAG_POLL)) {
+	if (ocores_algorithm.master_xfer != ocores_xfer_polling) {
 		ret = devm_request_irq(&pdev->dev, irq, ocores_isr, 0,
 				       pdev->name, i2c);
 		if (ret) {
diff --git a/drivers/i2c/busses/i2c-omap.c b/drivers/i2c/busses/i2c-omap.c
index cd9c65f3d404..faa0394048a0 100644
--- a/drivers/i2c/busses/i2c-omap.c
+++ b/drivers/i2c/busses/i2c-omap.c
@@ -269,6 +269,8 @@ static const u8 reg_map_ip_v2[] = {
 	[OMAP_I2C_IP_V2_IRQENABLE_CLR] = 0x30,
 };
 
+static int omap_i2c_xfer_data(struct omap_i2c_dev *omap);
+
 static inline void omap_i2c_write_reg(struct omap_i2c_dev *omap,
 				      int reg, u16 val)
 {
@@ -648,15 +650,28 @@ static void omap_i2c_resize_fifo(struct omap_i2c_dev *omap, u8 size, bool is_rx)
 			(1000 * omap->speed / 8);
 }
 
+static void omap_i2c_wait(struct omap_i2c_dev *omap)
+{
+	u16 stat;
+	u16 mask = omap_i2c_read_reg(omap, OMAP_I2C_IE_REG);
+	int count = 0;
+
+	do {
+		stat = omap_i2c_read_reg(omap, OMAP_I2C_STAT_REG);
+		count++;
+	} while (!(stat & mask) && count < 5);
+}
+
 /*
  * Low level master read/write transaction.
  */
 static int omap_i2c_xfer_msg(struct i2c_adapter *adap,
-			     struct i2c_msg *msg, int stop)
+			     struct i2c_msg *msg, int stop, bool polling)
 {
 	struct omap_i2c_dev *omap = i2c_get_adapdata(adap);
 	unsigned long timeout;
 	u16 w;
+	int ret;
 
 	dev_dbg(omap->dev, "addr: 0x%04x, len: %d, flags: 0x%x, stop: %d\n",
 		msg->addr, msg->len, msg->flags, stop);
@@ -680,7 +695,8 @@ static int omap_i2c_xfer_msg(struct i2c_adapter *adap,
 	w |= OMAP_I2C_BUF_RXFIF_CLR | OMAP_I2C_BUF_TXFIF_CLR;
 	omap_i2c_write_reg(omap, OMAP_I2C_BUF_REG, w);
 
-	reinit_completion(&omap->cmd_complete);
+	if (!polling)
+		reinit_completion(&omap->cmd_complete);
 	omap->cmd_err = 0;
 
 	w = OMAP_I2C_CON_EN | OMAP_I2C_CON_MST | OMAP_I2C_CON_STT;
@@ -732,8 +748,18 @@ static int omap_i2c_xfer_msg(struct i2c_adapter *adap,
 	 * REVISIT: We should abort the transfer on signals, but the bus goes
 	 * into arbitration and we're currently unable to recover from it.
 	 */
-	timeout = wait_for_completion_timeout(&omap->cmd_complete,
-						OMAP_I2C_TIMEOUT);
+	if (!polling) {
+		timeout = wait_for_completion_timeout(&omap->cmd_complete,
+						      OMAP_I2C_TIMEOUT);
+	} else {
+		do {
+			omap_i2c_wait(omap);
+			ret = omap_i2c_xfer_data(omap);
+		} while (ret == -EAGAIN);
+
+		timeout = !ret;
+	}
+
 	if (timeout == 0) {
 		dev_err(omap->dev, "controller timed out\n");
 		omap_i2c_reset(omap);
@@ -772,7 +798,8 @@ static int omap_i2c_xfer_msg(struct i2c_adapter *adap,
  * to do the work during IRQ processing.
  */
 static int
-omap_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num)
+omap_i2c_xfer_common(struct i2c_adapter *adap, struct i2c_msg msgs[], int num,
+		     bool polling)
 {
 	struct omap_i2c_dev *omap = i2c_get_adapdata(adap);
 	int i;
@@ -794,7 +821,8 @@ omap_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num)
 		omap->set_mpu_wkup_lat(omap->dev, omap->latency);
 
 	for (i = 0; i < num; i++) {
-		r = omap_i2c_xfer_msg(adap, &msgs[i], (i == (num - 1)));
+		r = omap_i2c_xfer_msg(adap, &msgs[i], (i == (num - 1)),
+				      polling);
 		if (r != 0)
 			break;
 	}
@@ -813,6 +841,18 @@ out:
 	return r;
 }
 
+static int
+omap_i2c_xfer_irq(struct i2c_adapter *adap, struct i2c_msg msgs[], int num)
+{
+	return omap_i2c_xfer_common(adap, msgs, num, false);
+}
+
+static int
+omap_i2c_xfer_polling(struct i2c_adapter *adap, struct i2c_msg msgs[], int num)
+{
+	return omap_i2c_xfer_common(adap, msgs, num, true);
+}
+
 static u32
 omap_i2c_func(struct i2c_adapter *adap)
 {
@@ -1035,10 +1075,8 @@ omap_i2c_isr(int irq, void *dev_id)
 	return ret;
 }
 
-static irqreturn_t
-omap_i2c_isr_thread(int this_irq, void *dev_id)
+static int omap_i2c_xfer_data(struct omap_i2c_dev *omap)
 {
-	struct omap_i2c_dev *omap = dev_id;
 	u16 bits;
 	u16 stat;
 	int err = 0, count = 0;
@@ -1056,7 +1094,8 @@ omap_i2c_isr_thread(int this_irq, void *dev_id)
 
 		if (!stat) {
 			/* my work here is done */
-			goto out;
+			err = -EAGAIN;
+			break;
 		}
 
 		dev_dbg(omap->dev, "IRQ (ISR = 0x%04x)\n", stat);
@@ -1165,14 +1204,25 @@ omap_i2c_isr_thread(int this_irq, void *dev_id)
 		}
 	} while (stat);
 
-	omap_i2c_complete_cmd(omap, err);
+	return err;
+}
+
+static irqreturn_t
+omap_i2c_isr_thread(int this_irq, void *dev_id)
+{
+	int ret;
+	struct omap_i2c_dev *omap = dev_id;
+
+	ret = omap_i2c_xfer_data(omap);
+	if (ret != -EAGAIN)
+		omap_i2c_complete_cmd(omap, ret);
 
-out:
 	return IRQ_HANDLED;
 }
 
 static const struct i2c_algorithm omap_i2c_algo = {
-	.master_xfer	= omap_i2c_xfer,
+	.master_xfer	= omap_i2c_xfer_irq,
+	.master_xfer_atomic	= omap_i2c_xfer_polling,
 	.functionality	= omap_i2c_func,
 };
 
diff --git a/drivers/i2c/busses/i2c-piix4.c b/drivers/i2c/busses/i2c-piix4.c
index 90946a8b9a75..e9a0514ae166 100644
--- a/drivers/i2c/busses/i2c-piix4.c
+++ b/drivers/i2c/busses/i2c-piix4.c
@@ -19,6 +19,7 @@
 	Serverworks OSB4, CSB5, CSB6, HT-1000, HT-1100
 	ATI IXP200, IXP300, IXP400, SB600, SB700/SP5100, SB800
 	AMD Hudson-2, ML, CZ
+	Hygon CZ
 	SMSC Victory66
 
    Note: we assume there can only be one device, with one or more
@@ -289,7 +290,9 @@ static int piix4_setup_sb800(struct pci_dev *PIIX4_dev,
 	     PIIX4_dev->revision >= 0x41) ||
 	    (PIIX4_dev->vendor == PCI_VENDOR_ID_AMD &&
 	     PIIX4_dev->device == PCI_DEVICE_ID_AMD_KERNCZ_SMBUS &&
-	     PIIX4_dev->revision >= 0x49))
+	     PIIX4_dev->revision >= 0x49) ||
+	    (PIIX4_dev->vendor == PCI_VENDOR_ID_HYGON &&
+	     PIIX4_dev->device == PCI_DEVICE_ID_AMD_KERNCZ_SMBUS))
 		smb_en = 0x00;
 	else
 		smb_en = (aux) ? 0x28 : 0x2c;
@@ -361,7 +364,8 @@ static int piix4_setup_sb800(struct pci_dev *PIIX4_dev,
 		 piix4_smba, i2ccfg >> 4);
 
 	/* Find which register is used for port selection */
-	if (PIIX4_dev->vendor == PCI_VENDOR_ID_AMD) {
+	if (PIIX4_dev->vendor == PCI_VENDOR_ID_AMD ||
+	    PIIX4_dev->vendor == PCI_VENDOR_ID_HYGON) {
 		switch (PIIX4_dev->device) {
 		case PCI_DEVICE_ID_AMD_KERNCZ_SMBUS:
 			piix4_port_sel_sb800 = SB800_PIIX4_PORT_IDX_KERNCZ;
@@ -794,6 +798,7 @@ static const struct pci_device_id piix4_ids[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_SBX00_SMBUS) },
 	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_HUDSON2_SMBUS) },
 	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_KERNCZ_SMBUS) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_HYGON, PCI_DEVICE_ID_AMD_KERNCZ_SMBUS) },
 	{ PCI_DEVICE(PCI_VENDOR_ID_SERVERWORKS,
 		     PCI_DEVICE_ID_SERVERWORKS_OSB4) },
 	{ PCI_DEVICE(PCI_VENDOR_ID_SERVERWORKS,
@@ -904,11 +909,13 @@ static int piix4_probe(struct pci_dev *dev, const struct pci_device_id *id)
 	if ((dev->vendor == PCI_VENDOR_ID_ATI &&
 	     dev->device == PCI_DEVICE_ID_ATI_SBX00_SMBUS &&
 	     dev->revision >= 0x40) ||
-	    dev->vendor == PCI_VENDOR_ID_AMD) {
+	    dev->vendor == PCI_VENDOR_ID_AMD ||
+	    dev->vendor == PCI_VENDOR_ID_HYGON) {
 		bool notify_imc = false;
 		is_sb800 = true;
 
-		if (dev->vendor == PCI_VENDOR_ID_AMD &&
+		if ((dev->vendor == PCI_VENDOR_ID_AMD ||
+		     dev->vendor == PCI_VENDOR_ID_HYGON) &&
 		    dev->device == PCI_DEVICE_ID_AMD_KERNCZ_SMBUS) {
 			u8 imc;
 
diff --git a/drivers/i2c/busses/i2c-rcar.c b/drivers/i2c/busses/i2c-rcar.c
index a7578f6da979..d39a4606f72d 100644
--- a/drivers/i2c/busses/i2c-rcar.c
+++ b/drivers/i2c/busses/i2c-rcar.c
@@ -85,6 +85,7 @@
 /* ICFBSCR */
 #define TCYC17	0x0f		/* 17*Tcyc delay 1st bit between SDA and SCL */
 
+#define RCAR_MIN_DMA_LEN	8
 
 #define RCAR_BUS_PHASE_START	(MDBS | MIE | ESG)
 #define RCAR_BUS_PHASE_DATA	(MDBS | MIE)
@@ -398,7 +399,7 @@ static void rcar_i2c_dma_callback(void *data)
 	rcar_i2c_dma_unmap(priv);
 }
 
-static void rcar_i2c_dma(struct rcar_i2c_priv *priv)
+static bool rcar_i2c_dma(struct rcar_i2c_priv *priv)
 {
 	struct device *dev = rcar_i2c_priv_to_dev(priv);
 	struct i2c_msg *msg = priv->msg;
@@ -412,9 +413,9 @@ static void rcar_i2c_dma(struct rcar_i2c_priv *priv)
 	int len;
 
 	/* Do various checks to see if DMA is feasible at all */
-	if (IS_ERR(chan) || msg->len < 8 || !(msg->flags & I2C_M_DMA_SAFE) ||
-	    (read && priv->flags & ID_P_NO_RXDMA))
-		return;
+	if (IS_ERR(chan) || msg->len < RCAR_MIN_DMA_LEN ||
+	    !(msg->flags & I2C_M_DMA_SAFE) || (read && priv->flags & ID_P_NO_RXDMA))
+		return false;
 
 	if (read) {
 		/*
@@ -434,7 +435,7 @@ static void rcar_i2c_dma(struct rcar_i2c_priv *priv)
 	dma_addr = dma_map_single(chan->device->dev, buf, len, dir);
 	if (dma_mapping_error(chan->device->dev, dma_addr)) {
 		dev_dbg(dev, "dma map failed, using PIO\n");
-		return;
+		return false;
 	}
 
 	sg_dma_len(&priv->sg) = len;
@@ -448,7 +449,7 @@ static void rcar_i2c_dma(struct rcar_i2c_priv *priv)
 	if (!txdesc) {
 		dev_dbg(dev, "dma prep slave sg failed, using PIO\n");
 		rcar_i2c_cleanup_dma(priv);
-		return;
+		return false;
 	}
 
 	txdesc->callback = rcar_i2c_dma_callback;
@@ -458,7 +459,7 @@ static void rcar_i2c_dma(struct rcar_i2c_priv *priv)
 	if (dma_submit_error(cookie)) {
 		dev_dbg(dev, "submitting dma failed, using PIO\n");
 		rcar_i2c_cleanup_dma(priv);
-		return;
+		return false;
 	}
 
 	/* Enable DMA Master Received/Transmitted */
@@ -468,6 +469,7 @@ static void rcar_i2c_dma(struct rcar_i2c_priv *priv)
 		rcar_i2c_write(priv, ICDMAER, TMDMAE);
 
 	dma_async_issue_pending(chan);
+	return true;
 }
 
 static void rcar_i2c_irq_send(struct rcar_i2c_priv *priv, u32 msr)
@@ -478,6 +480,10 @@ static void rcar_i2c_irq_send(struct rcar_i2c_priv *priv, u32 msr)
 	if (!(msr & MDE))
 		return;
 
+	/* Check if DMA can be enabled and take over */
+	if (priv->pos == 1 && rcar_i2c_dma(priv))
+		return;
+
 	if (priv->pos < msg->len) {
 		/*
 		 * Prepare next data to ICRXTX register.
@@ -488,13 +494,6 @@ static void rcar_i2c_irq_send(struct rcar_i2c_priv *priv, u32 msr)
 		 */
 		rcar_i2c_write(priv, ICRXTX, msg->buf[priv->pos]);
 		priv->pos++;
-
-		/*
-		 * Try to use DMA to transmit the rest of the data if
-		 * address transfer phase just finished.
-		 */
-		if (msr & MAT)
-			rcar_i2c_dma(priv);
 	} else {
 		/*
 		 * The last data was pushed to ICRXTX on _PREV_ empty irq.
@@ -921,6 +920,9 @@ static int rcar_i2c_probe(struct platform_device *pdev)
 	struct i2c_timings i2c_t;
 	int irq, ret;
 
+	/* Otherwise logic will break because some bytes must always use PIO */
+	BUILD_BUG_ON_MSG(RCAR_MIN_DMA_LEN < 3, "Invalid min DMA length");
+
 	priv = devm_kzalloc(dev, sizeof(struct rcar_i2c_priv), GFP_KERNEL);
 	if (!priv)
 		return -ENOMEM;
diff --git a/drivers/i2c/busses/i2c-riic.c b/drivers/i2c/busses/i2c-riic.c
index b75ff144b570..f31413fd9521 100644
--- a/drivers/i2c/busses/i2c-riic.c
+++ b/drivers/i2c/busses/i2c-riic.c
@@ -43,6 +43,7 @@
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
 
 #define RIIC_ICCR1	0x00
 #define RIIC_ICCR2	0x04
@@ -112,12 +113,10 @@ static int riic_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num)
 {
 	struct riic_dev *riic = i2c_get_adapdata(adap);
 	unsigned long time_left;
-	int i, ret;
+	int i;
 	u8 start_bit;
 
-	ret = clk_prepare_enable(riic->clk);
-	if (ret)
-		return ret;
+	pm_runtime_get_sync(adap->dev.parent);
 
 	if (readb(riic->base + RIIC_ICCR2) & ICCR2_BBSY) {
 		riic->err = -EBUSY;
@@ -150,7 +149,7 @@ static int riic_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num)
 	}
 
  out:
-	clk_disable_unprepare(riic->clk);
+	pm_runtime_put(adap->dev.parent);
 
 	return riic->err ?: num;
 }
@@ -281,20 +280,18 @@ static const struct i2c_algorithm riic_algo = {
 
 static int riic_init_hw(struct riic_dev *riic, struct i2c_timings *t)
 {
-	int ret;
+	int ret = 0;
 	unsigned long rate;
 	int total_ticks, cks, brl, brh;
 
-	ret = clk_prepare_enable(riic->clk);
-	if (ret)
-		return ret;
+	pm_runtime_get_sync(riic->adapter.dev.parent);
 
 	if (t->bus_freq_hz > 400000) {
 		dev_err(&riic->adapter.dev,
 			"unsupported bus speed (%dHz). 400000 max\n",
 			t->bus_freq_hz);
-		clk_disable_unprepare(riic->clk);
-		return -EINVAL;
+		ret = -EINVAL;
+		goto out;
 	}
 
 	rate = clk_get_rate(riic->clk);
@@ -332,8 +329,8 @@ static int riic_init_hw(struct riic_dev *riic, struct i2c_timings *t)
 	if (brl > (0x1F + 3)) {
 		dev_err(&riic->adapter.dev, "invalid speed (%lu). Too slow.\n",
 			(unsigned long)t->bus_freq_hz);
-		clk_disable_unprepare(riic->clk);
-		return -EINVAL;
+		ret = -EINVAL;
+		goto out;
 	}
 
 	brh = total_ticks - brl;
@@ -378,9 +375,9 @@ static int riic_init_hw(struct riic_dev *riic, struct i2c_timings *t)
 
 	riic_clear_set_bit(riic, ICCR1_IICRST, 0, RIIC_ICCR1);
 
-	clk_disable_unprepare(riic->clk);
-
-	return 0;
+out:
+	pm_runtime_put(riic->adapter.dev.parent);
+	return ret;
 }
 
 static struct riic_irq_desc riic_irqs[] = {
@@ -439,28 +436,36 @@ static int riic_i2c_probe(struct platform_device *pdev)
 
 	i2c_parse_fw_timings(&pdev->dev, &i2c_t, true);
 
+	pm_runtime_enable(&pdev->dev);
+
 	ret = riic_init_hw(riic, &i2c_t);
 	if (ret)
-		return ret;
-
+		goto out;
 
 	ret = i2c_add_adapter(adap);
 	if (ret)
-		return ret;
+		goto out;
 
 	platform_set_drvdata(pdev, riic);
 
 	dev_info(&pdev->dev, "registered with %dHz bus speed\n",
 		 i2c_t.bus_freq_hz);
 	return 0;
+
+out:
+	pm_runtime_disable(&pdev->dev);
+	return ret;
 }
 
 static int riic_i2c_remove(struct platform_device *pdev)
 {
 	struct riic_dev *riic = platform_get_drvdata(pdev);
 
+	pm_runtime_get_sync(&pdev->dev);
 	writeb(0, riic->base + RIIC_ICIER);
+	pm_runtime_put(&pdev->dev);
 	i2c_del_adapter(&riic->adapter);
+	pm_runtime_disable(&pdev->dev);
 
 	return 0;
 }
diff --git a/drivers/i2c/busses/i2c-stm32f7.c b/drivers/i2c/busses/i2c-stm32f7.c
index 4284fc991cfd..48337bef5b87 100644
--- a/drivers/i2c/busses/i2c-stm32f7.c
+++ b/drivers/i2c/busses/i2c-stm32f7.c
@@ -476,8 +476,12 @@ static int stm32f7_i2c_compute_timing(struct stm32f7_i2c_dev *i2c_dev,
 
 					list_add_tail(&v->node,
 						      &solutions);
+					break;
 				}
 			}
+
+			if (p_prev == p)
+				break;
 		}
 	}
 
diff --git a/drivers/i2c/busses/i2c-stu300.c b/drivers/i2c/busses/i2c-stu300.c
index 5503fa171df0..743c161b22c5 100644
--- a/drivers/i2c/busses/i2c-stu300.c
+++ b/drivers/i2c/busses/i2c-stu300.c
@@ -328,12 +328,6 @@ static int stu300_start_and_await_event(struct stu300_dev *dev,
 {
 	int ret;
 
-	if (unlikely(irqs_disabled())) {
-		/* TODO: implement polling for this case if need be. */
-		WARN(1, "irqs are disabled, cannot poll for event\n");
-		return -EIO;
-	}
-
 	/* Lock command issue, fill in an event we wait for */
 	spin_lock_irq(&dev->cmd_issue_lock);
 	init_completion(&dev->cmd_complete);
@@ -380,13 +374,6 @@ static int stu300_await_event(struct stu300_dev *dev,
 {
 	int ret;
 
-	if (unlikely(irqs_disabled())) {
-		/* TODO: implement polling for this case if need be. */
-		dev_err(&dev->pdev->dev, "irqs are disabled on this "
-			"system!\n");
-		return -EIO;
-	}
-
 	/* Is it already here? */
 	spin_lock_irq(&dev->cmd_issue_lock);
 	dev->cmd_err = STU300_ERROR_NONE;
@@ -846,6 +833,13 @@ static int stu300_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs,
 	return num;
 }
 
+static int stu300_xfer_todo(struct i2c_adapter *adap, struct i2c_msg *msgs, int num)
+{
+	/* TODO: implement polling for this case if need be. */
+	WARN(1, "%s: atomic transfers not implemented\n", dev_name(&adap->dev));
+	return -EOPNOTSUPP;
+}
+
 static u32 stu300_func(struct i2c_adapter *adap)
 {
 	/* This is the simplest thing you can think of... */
@@ -853,8 +847,9 @@ static u32 stu300_func(struct i2c_adapter *adap)
 }
 
 static const struct i2c_algorithm stu300_algo = {
-	.master_xfer	= stu300_xfer,
-	.functionality	= stu300_func,
+	.master_xfer = stu300_xfer,
+	.master_xfer_atomic = stu300_xfer_todo,
+	.functionality = stu300_func,
 };
 
 static const struct i2c_adapter_quirks stu300_quirks = {
diff --git a/drivers/i2c/busses/i2c-tegra-bpmp.c b/drivers/i2c/busses/i2c-tegra-bpmp.c
index f6cd35d0a2ac..9bb085793a0c 100644
--- a/drivers/i2c/busses/i2c-tegra-bpmp.c
+++ b/drivers/i2c/busses/i2c-tegra-bpmp.c
@@ -207,7 +207,8 @@ static int tegra_bpmp_i2c_msg_len_check(struct i2c_msg *msgs, unsigned int num)
 
 static int tegra_bpmp_i2c_msg_xfer(struct tegra_bpmp_i2c *i2c,
 				   struct mrq_i2c_request *request,
-				   struct mrq_i2c_response *response)
+				   struct mrq_i2c_response *response,
+				   bool atomic)
 {
 	struct tegra_bpmp_message msg;
 	int err;
@@ -222,7 +223,7 @@ static int tegra_bpmp_i2c_msg_xfer(struct tegra_bpmp_i2c *i2c,
 	msg.rx.data = response;
 	msg.rx.size = sizeof(*response);
 
-	if (irqs_disabled())
+	if (atomic)
 		err = tegra_bpmp_transfer_atomic(i2c->bpmp, &msg);
 	else
 		err = tegra_bpmp_transfer(i2c->bpmp, &msg);
@@ -230,8 +231,9 @@ static int tegra_bpmp_i2c_msg_xfer(struct tegra_bpmp_i2c *i2c,
 	return err;
 }
 
-static int tegra_bpmp_i2c_xfer(struct i2c_adapter *adapter,
-			       struct i2c_msg *msgs, int num)
+static int tegra_bpmp_i2c_xfer_common(struct i2c_adapter *adapter,
+				      struct i2c_msg *msgs, int num,
+				      bool atomic)
 {
 	struct tegra_bpmp_i2c *i2c = i2c_get_adapdata(adapter);
 	struct mrq_i2c_response response;
@@ -253,7 +255,7 @@ static int tegra_bpmp_i2c_xfer(struct i2c_adapter *adapter,
 		return err;
 	}
 
-	err = tegra_bpmp_i2c_msg_xfer(i2c, &request, &response);
+	err = tegra_bpmp_i2c_msg_xfer(i2c, &request, &response, atomic);
 	if (err < 0) {
 		dev_err(i2c->dev, "failed to transfer message: %d\n", err);
 		return err;
@@ -268,6 +270,18 @@ static int tegra_bpmp_i2c_xfer(struct i2c_adapter *adapter,
 	return num;
 }
 
+static int tegra_bpmp_i2c_xfer(struct i2c_adapter *adapter,
+			       struct i2c_msg *msgs, int num)
+{
+	return tegra_bpmp_i2c_xfer_common(adapter, msgs, num, false);
+}
+
+static int tegra_bpmp_i2c_xfer_atomic(struct i2c_adapter *adapter,
+				      struct i2c_msg *msgs, int num)
+{
+	return tegra_bpmp_i2c_xfer_common(adapter, msgs, num, true);
+}
+
 static u32 tegra_bpmp_i2c_func(struct i2c_adapter *adapter)
 {
 	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL | I2C_FUNC_10BIT_ADDR |
@@ -276,6 +290,7 @@ static u32 tegra_bpmp_i2c_func(struct i2c_adapter *adapter)
 
 static const struct i2c_algorithm tegra_bpmp_i2c_algo = {
 	.master_xfer = tegra_bpmp_i2c_xfer,
+	.master_xfer_atomic = tegra_bpmp_i2c_xfer_atomic,
 	.functionality = tegra_bpmp_i2c_func,
 };
 
diff --git a/drivers/i2c/i2c-core-base.c b/drivers/i2c/i2c-core-base.c
index 688aa3b5f3ac..9732a81bb7dd 100644
--- a/drivers/i2c/i2c-core-base.c
+++ b/drivers/i2c/i2c-core-base.c
@@ -1871,8 +1871,10 @@ int __i2c_transfer(struct i2c_adapter *adap, struct i2c_msg *msgs, int num)
 
 	if (WARN_ON(!msgs || num < 1))
 		return -EINVAL;
-	if (WARN_ON(test_bit(I2C_ALF_IS_SUSPENDED, &adap->locked_flags)))
-		return -ESHUTDOWN;
+
+	ret = __i2c_check_suspended(adap);
+	if (ret)
+		return ret;
 
 	if (adap->quirks && i2c_check_for_quirks(adap, msgs, num))
 		return -EOPNOTSUPP;
@@ -1894,7 +1896,11 @@ int __i2c_transfer(struct i2c_adapter *adap, struct i2c_msg *msgs, int num)
 	/* Retry automatically on arbitration loss */
 	orig_jiffies = jiffies;
 	for (ret = 0, try = 0; try <= adap->retries; try++) {
-		ret = adap->algo->master_xfer(adap, msgs, num);
+		if (i2c_in_atomic_xfer_mode() && adap->algo->master_xfer_atomic)
+			ret = adap->algo->master_xfer_atomic(adap, msgs, num);
+		else
+			ret = adap->algo->master_xfer(adap, msgs, num);
+
 		if (ret != -EAGAIN)
 			break;
 		if (time_after(jiffies, orig_jiffies + adap->timeout))
@@ -1950,14 +1956,9 @@ int i2c_transfer(struct i2c_adapter *adap, struct i2c_msg *msgs, int num)
 	 *    one (discarding status on the second message) or errno
 	 *    (discarding status on the first one).
 	 */
-	if (in_atomic() || irqs_disabled()) {
-		ret = i2c_trylock_bus(adap, I2C_LOCK_SEGMENT);
-		if (!ret)
-			/* I2C activity is ongoing. */
-			return -EAGAIN;
-	} else {
-		i2c_lock_bus(adap, I2C_LOCK_SEGMENT);
-	}
+	ret = __i2c_lock_bus_helper(adap);
+	if (ret)
+		return ret;
 
 	ret = __i2c_transfer(adap, msgs, num);
 	i2c_unlock_bus(adap, I2C_LOCK_SEGMENT);
diff --git a/drivers/i2c/i2c-core-smbus.c b/drivers/i2c/i2c-core-smbus.c
index 132119112596..788d42f2aad9 100644
--- a/drivers/i2c/i2c-core-smbus.c
+++ b/drivers/i2c/i2c-core-smbus.c
@@ -20,6 +20,8 @@
 #include <linux/i2c-smbus.h>
 #include <linux/slab.h>
 
+#include "i2c-core.h"
+
 #define CREATE_TRACE_POINTS
 #include <trace/events/smbus.h>
 
@@ -530,7 +532,10 @@ s32 i2c_smbus_xfer(struct i2c_adapter *adapter, u16 addr,
 {
 	s32 res;
 
-	i2c_lock_bus(adapter, I2C_LOCK_SEGMENT);
+	res = __i2c_lock_bus_helper(adapter);
+	if (res)
+		return res;
+
 	res = __i2c_smbus_xfer(adapter, addr, flags, read_write,
 			       command, protocol, data);
 	i2c_unlock_bus(adapter, I2C_LOCK_SEGMENT);
@@ -543,10 +548,17 @@ s32 __i2c_smbus_xfer(struct i2c_adapter *adapter, u16 addr,
 		     unsigned short flags, char read_write,
 		     u8 command, int protocol, union i2c_smbus_data *data)
 {
+	int (*xfer_func)(struct i2c_adapter *adap, u16 addr,
+			 unsigned short flags, char read_write,
+			 u8 command, int size, union i2c_smbus_data *data);
 	unsigned long orig_jiffies;
 	int try;
 	s32 res;
 
+	res = __i2c_check_suspended(adapter);
+	if (res)
+		return res;
+
 	/* If enabled, the following two tracepoints are conditional on
 	 * read_write and protocol.
 	 */
@@ -557,13 +569,20 @@ s32 __i2c_smbus_xfer(struct i2c_adapter *adapter, u16 addr,
 
 	flags &= I2C_M_TEN | I2C_CLIENT_PEC | I2C_CLIENT_SCCB;
 
-	if (adapter->algo->smbus_xfer) {
+	xfer_func = adapter->algo->smbus_xfer;
+	if (i2c_in_atomic_xfer_mode()) {
+		if (adapter->algo->smbus_xfer_atomic)
+			xfer_func = adapter->algo->smbus_xfer_atomic;
+		else if (adapter->algo->master_xfer_atomic)
+			xfer_func = NULL; /* fallback to I2C emulation */
+	}
+
+	if (xfer_func) {
 		/* Retry automatically on arbitration loss */
 		orig_jiffies = jiffies;
 		for (res = 0, try = 0; try <= adapter->retries; try++) {
-			res = adapter->algo->smbus_xfer(adapter, addr, flags,
-							read_write, command,
-							protocol, data);
+			res = xfer_func(adapter, addr, flags, read_write,
+					command, protocol, data);
 			if (res != -EAGAIN)
 				break;
 			if (time_after(jiffies,
diff --git a/drivers/i2c/i2c-core.h b/drivers/i2c/i2c-core.h
index 37576f50fe20..c88cfef81343 100644
--- a/drivers/i2c/i2c-core.h
+++ b/drivers/i2c/i2c-core.h
@@ -29,6 +29,42 @@ extern int		__i2c_first_dynamic_bus_num;
 
 int i2c_check_7bit_addr_validity_strict(unsigned short addr);
 
+/*
+ * We only allow atomic transfers for very late communication, e.g. to send
+ * the powerdown command to a PMIC. Atomic transfers are a corner case and not
+ * for generic use!
+ */
+static inline bool i2c_in_atomic_xfer_mode(void)
+{
+	return system_state > SYSTEM_RUNNING && irqs_disabled();
+}
+
+static inline int __i2c_lock_bus_helper(struct i2c_adapter *adap)
+{
+	int ret = 0;
+
+	if (i2c_in_atomic_xfer_mode()) {
+		WARN(!adap->algo->master_xfer_atomic && !adap->algo->smbus_xfer_atomic,
+		     "No atomic I2C transfer handler for '%s'\n", dev_name(&adap->dev));
+		ret = i2c_trylock_bus(adap, I2C_LOCK_SEGMENT) ? 0 : -EAGAIN;
+	} else {
+		i2c_lock_bus(adap, I2C_LOCK_SEGMENT);
+	}
+
+	return ret;
+}
+
+static inline int __i2c_check_suspended(struct i2c_adapter *adap)
+{
+	if (test_bit(I2C_ALF_IS_SUSPENDED, &adap->locked_flags)) {
+		if (!test_and_set_bit(I2C_ALF_SUSPEND_REPORTED, &adap->locked_flags))
+			dev_WARN(&adap->dev, "Transfer while suspended\n");
+		return -ESHUTDOWN;
+	}
+
+	return 0;
+}
+
 #ifdef CONFIG_ACPI
 const struct acpi_device_id *
 i2c_acpi_match_device(const struct acpi_device_id *matches,
diff --git a/drivers/i2c/i2c-mux.c b/drivers/i2c/i2c-mux.c
index f330690b4125..603252fa1284 100644
--- a/drivers/i2c/i2c-mux.c
+++ b/drivers/i2c/i2c-mux.c
@@ -310,12 +310,18 @@ int i2c_mux_add_adapter(struct i2c_mux_core *muxc,
 		else
 			priv->algo.master_xfer = __i2c_mux_master_xfer;
 	}
+	if (parent->algo->master_xfer_atomic)
+		priv->algo.master_xfer_atomic = priv->algo.master_xfer;
+
 	if (parent->algo->smbus_xfer) {
 		if (muxc->mux_locked)
 			priv->algo.smbus_xfer = i2c_mux_smbus_xfer;
 		else
 			priv->algo.smbus_xfer = __i2c_mux_smbus_xfer;
 	}
+	if (parent->algo->smbus_xfer_atomic)
+		priv->algo.smbus_xfer_atomic = priv->algo.smbus_xfer;
+
 	priv->algo.functionality = i2c_mux_functionality;
 
 	/* Now fill out new adapter structure */
diff --git a/drivers/i2c/muxes/i2c-demux-pinctrl.c b/drivers/i2c/muxes/i2c-demux-pinctrl.c
index 035032e20327..4eecffc26527 100644
--- a/drivers/i2c/muxes/i2c-demux-pinctrl.c
+++ b/drivers/i2c/muxes/i2c-demux-pinctrl.c
@@ -99,6 +99,8 @@ static int i2c_demux_activate_master(struct i2c_demux_pinctrl_priv *priv, u32 ne
 
 	/* Now fill out current adapter structure. cur_chan must be up to date */
 	priv->algo.master_xfer = i2c_demux_master_xfer;
+	if (adap->algo->master_xfer_atomic)
+		priv->algo.master_xfer_atomic = i2c_demux_master_xfer;
 	priv->algo.functionality = i2c_demux_functionality;
 
 	snprintf(priv->cur_adap.name, sizeof(priv->cur_adap.name),
@@ -219,8 +221,8 @@ static int i2c_demux_pinctrl_probe(struct platform_device *pdev)
 		return -EINVAL;
 	}
 
-	priv = devm_kzalloc(&pdev->dev, sizeof(*priv)
-			   + num_chan * sizeof(struct i2c_demux_pinctrl_chan), GFP_KERNEL);
+	priv = devm_kzalloc(&pdev->dev, struct_size(priv, chan, num_chan),
+			    GFP_KERNEL);
 
 	props = devm_kcalloc(&pdev->dev, num_chan, sizeof(*props), GFP_KERNEL);
 
diff --git a/drivers/i2c/muxes/i2c-mux-pca9541.c b/drivers/i2c/muxes/i2c-mux-pca9541.c
index 9e75d6b9140b..50e1fb4aedf5 100644
--- a/drivers/i2c/muxes/i2c-mux-pca9541.c
+++ b/drivers/i2c/muxes/i2c-mux-pca9541.c
@@ -22,7 +22,6 @@
 #include <linux/i2c-mux.h>
 #include <linux/jiffies.h>
 #include <linux/module.h>
-#include <linux/platform_data/pca954x.h>
 #include <linux/slab.h>
 
 /*
@@ -287,10 +286,8 @@ static int pca9541_probe(struct i2c_client *client,
 			 const struct i2c_device_id *id)
 {
 	struct i2c_adapter *adap = client->adapter;
-	struct pca954x_platform_data *pdata = dev_get_platdata(&client->dev);
 	struct i2c_mux_core *muxc;
 	struct pca9541 *data;
-	int force;
 	int ret;
 
 	if (!i2c_check_functionality(adap, I2C_FUNC_SMBUS_BYTE_DATA))
@@ -306,9 +303,6 @@ static int pca9541_probe(struct i2c_client *client,
 
 	/* Create mux adapter */
 
-	force = 0;
-	if (pdata)
-		force = pdata->modes[0].adap_id;
 	muxc = i2c_mux_alloc(adap, &client->dev, 1, sizeof(*data),
 			     I2C_MUX_ARBITRATOR,
 			     pca9541_select_chan, pca9541_release_chan);
@@ -320,7 +314,7 @@ static int pca9541_probe(struct i2c_client *client,
 
 	i2c_set_clientdata(client, muxc);
 
-	ret = i2c_mux_add_adapter(muxc, force, 0, 0);
+	ret = i2c_mux_add_adapter(muxc, 0, 0, 0);
 	if (ret)
 		return ret;
 
diff --git a/drivers/i2c/muxes/i2c-mux-pca954x.c b/drivers/i2c/muxes/i2c-mux-pca954x.c
index bfabf985e830..923aa3a5a3dc 100644
--- a/drivers/i2c/muxes/i2c-mux-pca954x.c
+++ b/drivers/i2c/muxes/i2c-mux-pca954x.c
@@ -46,10 +46,10 @@
 #include <linux/of.h>
 #include <linux/of_device.h>
 #include <linux/of_irq.h>
-#include <linux/platform_data/pca954x.h>
 #include <linux/pm.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
+#include <dt-bindings/mux/mux.h>
 
 #define PCA954X_MAX_NCHANS 8
 
@@ -85,7 +85,9 @@ struct pca954x {
 	const struct chip_desc *chip;
 
 	u8 last_chan;		/* last register value */
-	u8 deselect;
+	/* MUX_IDLE_AS_IS, MUX_IDLE_DISCONNECT or >= 0 for channel */
+	s8 idle_state;
+
 	struct i2c_client *client;
 
 	struct irq_domain *irq;
@@ -254,15 +256,71 @@ static int pca954x_deselect_mux(struct i2c_mux_core *muxc, u32 chan)
 {
 	struct pca954x *data = i2c_mux_priv(muxc);
 	struct i2c_client *client = data->client;
+	s8 idle_state;
+
+	idle_state = READ_ONCE(data->idle_state);
+	if (idle_state >= 0)
+		/* Set the mux back to a predetermined channel */
+		return pca954x_select_chan(muxc, idle_state);
+
+	if (idle_state == MUX_IDLE_DISCONNECT) {
+		/* Deselect active channel */
+		data->last_chan = 0;
+		return pca954x_reg_write(muxc->parent, client,
+					 data->last_chan);
+	}
 
-	if (!(data->deselect & (1 << chan)))
-		return 0;
+	/* otherwise leave as-is */
 
-	/* Deselect active channel */
-	data->last_chan = 0;
-	return pca954x_reg_write(muxc->parent, client, data->last_chan);
+	return 0;
+}
+
+static ssize_t idle_state_show(struct device *dev,
+				    struct device_attribute *attr,
+				    char *buf)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+	struct i2c_mux_core *muxc = i2c_get_clientdata(client);
+	struct pca954x *data = i2c_mux_priv(muxc);
+
+	return sprintf(buf, "%d\n", READ_ONCE(data->idle_state));
 }
 
+static ssize_t idle_state_store(struct device *dev,
+				struct device_attribute *attr,
+				const char *buf, size_t count)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+	struct i2c_mux_core *muxc = i2c_get_clientdata(client);
+	struct pca954x *data = i2c_mux_priv(muxc);
+	int val;
+	int ret;
+
+	ret = kstrtoint(buf, 0, &val);
+	if (ret < 0)
+		return ret;
+
+	if (val != MUX_IDLE_AS_IS && val != MUX_IDLE_DISCONNECT &&
+	    (val < 0 || val >= data->chip->nchans))
+		return -EINVAL;
+
+	i2c_lock_bus(muxc->parent, I2C_LOCK_SEGMENT);
+
+	WRITE_ONCE(data->idle_state, val);
+	/*
+	 * Set the mux into a state consistent with the new
+	 * idle_state.
+	 */
+	if (data->last_chan || val != MUX_IDLE_DISCONNECT)
+		ret = pca954x_deselect_mux(muxc, 0);
+
+	i2c_unlock_bus(muxc->parent, I2C_LOCK_SEGMENT);
+
+	return ret < 0 ? ret : count;
+}
+
+static DEVICE_ATTR_RW(idle_state);
+
 static irqreturn_t pca954x_irq_handler(int irq, void *dev_id)
 {
 	struct pca954x *data = dev_id;
@@ -329,8 +387,11 @@ static int pca954x_irq_setup(struct i2c_mux_core *muxc)
 static void pca954x_cleanup(struct i2c_mux_core *muxc)
 {
 	struct pca954x *data = i2c_mux_priv(muxc);
+	struct i2c_client *client = data->client;
 	int c, irq;
 
+	device_remove_file(&client->dev, &dev_attr_idle_state);
+
 	if (data->irq) {
 		for (c = 0; c < data->chip->nchans; c++) {
 			irq = irq_find_mapping(data->irq, c);
@@ -348,14 +409,13 @@ static int pca954x_probe(struct i2c_client *client,
 			 const struct i2c_device_id *id)
 {
 	struct i2c_adapter *adap = client->adapter;
-	struct pca954x_platform_data *pdata = dev_get_platdata(&client->dev);
 	struct device *dev = &client->dev;
 	struct device_node *np = dev->of_node;
 	bool idle_disconnect_dt;
 	struct gpio_desc *gpio;
-	int num, force, class;
 	struct i2c_mux_core *muxc;
 	struct pca954x *data;
+	int num;
 	int ret;
 
 	if (!i2c_check_functionality(adap, I2C_FUNC_SMBUS_BYTE))
@@ -412,9 +472,12 @@ static int pca954x_probe(struct i2c_client *client,
 	}
 
 	data->last_chan = 0;		   /* force the first selection */
+	data->idle_state = MUX_IDLE_AS_IS;
 
 	idle_disconnect_dt = np &&
 		of_property_read_bool(np, "i2c-mux-idle-disconnect");
+	if (idle_disconnect_dt)
+		data->idle_state = MUX_IDLE_DISCONNECT;
 
 	ret = pca954x_irq_setup(muxc);
 	if (ret)
@@ -422,24 +485,7 @@ static int pca954x_probe(struct i2c_client *client,
 
 	/* Now create an adapter for each channel */
 	for (num = 0; num < data->chip->nchans; num++) {
-		bool idle_disconnect_pd = false;
-
-		force = 0;			  /* dynamic adap number */
-		class = 0;			  /* no class by default */
-		if (pdata) {
-			if (num < pdata->num_modes) {
-				/* force static number */
-				force = pdata->modes[num].adap_id;
-				class = pdata->modes[num].class;
-			} else
-				/* discard unconfigured channels */
-				break;
-			idle_disconnect_pd = pdata->modes[num].deselect_on_exit;
-		}
-		data->deselect |= (idle_disconnect_pd ||
-				   idle_disconnect_dt) << num;
-
-		ret = i2c_mux_add_adapter(muxc, force, num, class);
+		ret = i2c_mux_add_adapter(muxc, 0, num, 0);
 		if (ret)
 			goto fail_cleanup;
 	}
@@ -453,6 +499,12 @@ static int pca954x_probe(struct i2c_client *client,
 			goto fail_cleanup;
 	}
 
+	/*
+	 * The attr probably isn't going to be needed in most cases,
+	 * so don't fail completely on error.
+	 */
+	device_create_file(dev, &dev_attr_idle_state);
+
 	dev_info(dev, "registered %d multiplexed busses for I2C %s %s\n",
 		 num, data->chip->muxtype == pca954x_ismux
 				? "mux" : "switch", client->name);
diff --git a/drivers/ide/ide-probe.c b/drivers/ide/ide-probe.c
index 5aeaca24a28f..4ad824984581 100644
--- a/drivers/ide/ide-probe.c
+++ b/drivers/ide/ide-probe.c
@@ -1437,6 +1437,9 @@ int ide_host_register(struct ide_host *host, const struct ide_port_info *d,
 	ide_hwif_t *hwif, *mate = NULL;
 	int i, j = 0;
 
+	pr_warn("legacy IDE will be removed in 2021, please switch to libata\n"
+		"Report any missing HW support to linux-ide@vger.kernel.org\n");
+
 	ide_host_for_each_port(i, hwif, host) {
 		if (hwif == NULL) {
 			mate = NULL;
diff --git a/drivers/infiniband/Kconfig b/drivers/infiniband/Kconfig
index d318bab25860..cbfbea49f126 100644
--- a/drivers/infiniband/Kconfig
+++ b/drivers/infiniband/Kconfig
@@ -93,6 +93,7 @@ source "drivers/infiniband/hw/mthca/Kconfig"
 source "drivers/infiniband/hw/qib/Kconfig"
 source "drivers/infiniband/hw/cxgb3/Kconfig"
 source "drivers/infiniband/hw/cxgb4/Kconfig"
+source "drivers/infiniband/hw/efa/Kconfig"
 source "drivers/infiniband/hw/i40iw/Kconfig"
 source "drivers/infiniband/hw/mlx4/Kconfig"
 source "drivers/infiniband/hw/mlx5/Kconfig"
diff --git a/drivers/infiniband/core/addr.c b/drivers/infiniband/core/addr.c
index 744b6ec0acb0..ba01b90c04e7 100644
--- a/drivers/infiniband/core/addr.c
+++ b/drivers/infiniband/core/addr.c
@@ -45,6 +45,7 @@
 #include <net/ipv6_stubs.h>
 #include <net/ip6_route.h>
 #include <rdma/ib_addr.h>
+#include <rdma/ib_cache.h>
 #include <rdma/ib_sa.h>
 #include <rdma/ib.h>
 #include <rdma/rdma_netlink.h>
diff --git a/drivers/infiniband/core/cache.c b/drivers/infiniband/core/cache.c
index 43c67e5f43c6..18e476b3ced0 100644
--- a/drivers/infiniband/core/cache.c
+++ b/drivers/infiniband/core/cache.c
@@ -78,11 +78,22 @@ enum gid_table_entry_state {
 	GID_TABLE_ENTRY_PENDING_DEL	= 3,
 };
 
+struct roce_gid_ndev_storage {
+	struct rcu_head rcu_head;
+	struct net_device *ndev;
+};
+
 struct ib_gid_table_entry {
 	struct kref			kref;
 	struct work_struct		del_work;
 	struct ib_gid_attr		attr;
 	void				*context;
+	/* Store the ndev pointer to release reference later on in
+	 * call_rcu context because by that time gid_table_entry
+	 * and attr might be already freed. So keep a copy of it.
+	 * ndev_storage is freed by rcu callback.
+	 */
+	struct roce_gid_ndev_storage	*ndev_storage;
 	enum gid_table_entry_state	state;
 };
 
@@ -206,6 +217,20 @@ static void schedule_free_gid(struct kref *kref)
 	queue_work(ib_wq, &entry->del_work);
 }
 
+static void put_gid_ndev(struct rcu_head *head)
+{
+	struct roce_gid_ndev_storage *storage =
+		container_of(head, struct roce_gid_ndev_storage, rcu_head);
+
+	WARN_ON(!storage->ndev);
+	/* At this point its safe to release netdev reference,
+	 * as all callers working on gid_attr->ndev are done
+	 * using this netdev.
+	 */
+	dev_put(storage->ndev);
+	kfree(storage);
+}
+
 static void free_gid_entry_locked(struct ib_gid_table_entry *entry)
 {
 	struct ib_device *device = entry->attr.device;
@@ -228,8 +253,8 @@ static void free_gid_entry_locked(struct ib_gid_table_entry *entry)
 	/* Now this index is ready to be allocated */
 	write_unlock_irq(&table->rwlock);
 
-	if (entry->attr.ndev)
-		dev_put(entry->attr.ndev);
+	if (entry->ndev_storage)
+		call_rcu(&entry->ndev_storage->rcu_head, put_gid_ndev);
 	kfree(entry);
 }
 
@@ -266,14 +291,25 @@ static struct ib_gid_table_entry *
 alloc_gid_entry(const struct ib_gid_attr *attr)
 {
 	struct ib_gid_table_entry *entry;
+	struct net_device *ndev;
 
 	entry = kzalloc(sizeof(*entry), GFP_KERNEL);
 	if (!entry)
 		return NULL;
+
+	ndev = rcu_dereference_protected(attr->ndev, 1);
+	if (ndev) {
+		entry->ndev_storage = kzalloc(sizeof(*entry->ndev_storage),
+					      GFP_KERNEL);
+		if (!entry->ndev_storage) {
+			kfree(entry);
+			return NULL;
+		}
+		dev_hold(ndev);
+		entry->ndev_storage->ndev = ndev;
+	}
 	kref_init(&entry->kref);
 	memcpy(&entry->attr, attr, sizeof(*attr));
-	if (entry->attr.ndev)
-		dev_hold(entry->attr.ndev);
 	INIT_WORK(&entry->del_work, free_gid_work);
 	entry->state = GID_TABLE_ENTRY_INVALID;
 	return entry;
@@ -343,6 +379,7 @@ static int add_roce_gid(struct ib_gid_table_entry *entry)
 static void del_gid(struct ib_device *ib_dev, u8 port,
 		    struct ib_gid_table *table, int ix)
 {
+	struct roce_gid_ndev_storage *ndev_storage;
 	struct ib_gid_table_entry *entry;
 
 	lockdep_assert_held(&table->lock);
@@ -360,6 +397,13 @@ static void del_gid(struct ib_device *ib_dev, u8 port,
 		table->data_vec[ix] = NULL;
 	write_unlock_irq(&table->rwlock);
 
+	ndev_storage = entry->ndev_storage;
+	if (ndev_storage) {
+		entry->ndev_storage = NULL;
+		rcu_assign_pointer(entry->attr.ndev, NULL);
+		call_rcu(&ndev_storage->rcu_head, put_gid_ndev);
+	}
+
 	if (rdma_cap_roce_gid_table(ib_dev, port))
 		ib_dev->ops.del_gid(&entry->attr, &entry->context);
 
@@ -543,30 +587,11 @@ out_unlock:
 int ib_cache_gid_add(struct ib_device *ib_dev, u8 port,
 		     union ib_gid *gid, struct ib_gid_attr *attr)
 {
-	struct net_device *idev;
-	unsigned long mask;
-	int ret;
-
-	idev = ib_device_get_netdev(ib_dev, port);
-	if (idev && attr->ndev != idev) {
-		union ib_gid default_gid;
-
-		/* Adding default GIDs is not permitted */
-		make_default_gid(idev, &default_gid);
-		if (!memcmp(gid, &default_gid, sizeof(*gid))) {
-			dev_put(idev);
-			return -EPERM;
-		}
-	}
-	if (idev)
-		dev_put(idev);
-
-	mask = GID_ATTR_FIND_MASK_GID |
-	       GID_ATTR_FIND_MASK_GID_TYPE |
-	       GID_ATTR_FIND_MASK_NETDEV;
+	unsigned long mask = GID_ATTR_FIND_MASK_GID |
+			     GID_ATTR_FIND_MASK_GID_TYPE |
+			     GID_ATTR_FIND_MASK_NETDEV;
 
-	ret = __ib_cache_gid_add(ib_dev, port, gid, attr, mask, false);
-	return ret;
+	return __ib_cache_gid_add(ib_dev, port, gid, attr, mask, false);
 }
 
 static int
@@ -1263,11 +1288,72 @@ struct net_device *rdma_read_gid_attr_ndev_rcu(const struct ib_gid_attr *attr)
 
 	read_lock_irqsave(&table->rwlock, flags);
 	valid = is_gid_entry_valid(table->data_vec[attr->index]);
-	if (valid && attr->ndev && (READ_ONCE(attr->ndev->flags) & IFF_UP))
-		ndev = attr->ndev;
+	if (valid) {
+		ndev = rcu_dereference(attr->ndev);
+		if (!ndev ||
+		    (ndev && ((READ_ONCE(ndev->flags) & IFF_UP) == 0)))
+			ndev = ERR_PTR(-ENODEV);
+	}
 	read_unlock_irqrestore(&table->rwlock, flags);
 	return ndev;
 }
+EXPORT_SYMBOL(rdma_read_gid_attr_ndev_rcu);
+
+static int get_lower_dev_vlan(struct net_device *lower_dev, void *data)
+{
+	u16 *vlan_id = data;
+
+	if (is_vlan_dev(lower_dev))
+		*vlan_id = vlan_dev_vlan_id(lower_dev);
+
+	/* We are interested only in first level vlan device, so
+	 * always return 1 to stop iterating over next level devices.
+	 */
+	return 1;
+}
+
+/**
+ * rdma_read_gid_l2_fields - Read the vlan ID and source MAC address
+ *			     of a GID entry.
+ *
+ * @attr:	GID attribute pointer whose L2 fields to be read
+ * @vlan_id:	Pointer to vlan id to fill up if the GID entry has
+ *		vlan id. It is optional.
+ * @smac:	Pointer to smac to fill up for a GID entry. It is optional.
+ *
+ * rdma_read_gid_l2_fields() returns 0 on success and returns vlan id
+ * (if gid entry has vlan) and source MAC, or returns error.
+ */
+int rdma_read_gid_l2_fields(const struct ib_gid_attr *attr,
+			    u16 *vlan_id, u8 *smac)
+{
+	struct net_device *ndev;
+
+	rcu_read_lock();
+	ndev = rcu_dereference(attr->ndev);
+	if (!ndev) {
+		rcu_read_unlock();
+		return -ENODEV;
+	}
+	if (smac)
+		ether_addr_copy(smac, ndev->dev_addr);
+	if (vlan_id) {
+		*vlan_id = 0xffff;
+		if (is_vlan_dev(ndev)) {
+			*vlan_id = vlan_dev_vlan_id(ndev);
+		} else {
+			/* If the netdev is upper device and if it's lower
+			 * device is vlan device, consider vlan id of the
+			 * the lower vlan device for this gid entry.
+			 */
+			netdev_walk_all_lower_dev_rcu(attr->ndev,
+					get_lower_dev_vlan, vlan_id);
+		}
+	}
+	rcu_read_unlock();
+	return 0;
+}
+EXPORT_SYMBOL(rdma_read_gid_l2_fields);
 
 static int config_non_roce_gid_cache(struct ib_device *device,
 				     u8 port, int gid_tbl_len)
@@ -1392,7 +1478,6 @@ static void ib_cache_event(struct ib_event_handler *handler,
 	    event->event == IB_EVENT_PORT_ACTIVE ||
 	    event->event == IB_EVENT_LID_CHANGE  ||
 	    event->event == IB_EVENT_PKEY_CHANGE ||
-	    event->event == IB_EVENT_SM_CHANGE   ||
 	    event->event == IB_EVENT_CLIENT_REREGISTER ||
 	    event->event == IB_EVENT_GID_CHANGE) {
 		work = kmalloc(sizeof *work, GFP_ATOMIC);
diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c
index b9416a6fca36..da10e6ccb43c 100644
--- a/drivers/infiniband/core/cm.c
+++ b/drivers/infiniband/core/cm.c
@@ -52,6 +52,7 @@
 #include <rdma/ib_cache.h>
 #include <rdma/ib_cm.h>
 #include "cm_msgs.h"
+#include "core_priv.h"
 
 MODULE_AUTHOR("Sean Hefty");
 MODULE_DESCRIPTION("InfiniBand CM");
@@ -124,7 +125,8 @@ static struct ib_cm {
 	struct rb_root remote_qp_table;
 	struct rb_root remote_id_table;
 	struct rb_root remote_sidr_table;
-	struct idr local_id_table;
+	struct xarray local_id_table;
+	u32 local_id_next;
 	__be32 random_id_operand;
 	struct list_head timewait_list;
 	struct workqueue_struct *wq;
@@ -219,7 +221,6 @@ struct cm_port {
 struct cm_device {
 	struct list_head list;
 	struct ib_device *ib_device;
-	struct device *device;
 	u8 ack_delay;
 	int going_down;
 	struct cm_port *port[0];
@@ -598,35 +599,31 @@ static int cm_init_av_by_path(struct sa_path_rec *path,
 
 static int cm_alloc_id(struct cm_id_private *cm_id_priv)
 {
-	unsigned long flags;
-	int id;
-
-	idr_preload(GFP_KERNEL);
-	spin_lock_irqsave(&cm.lock, flags);
+	int err;
+	u32 id;
 
-	id = idr_alloc_cyclic(&cm.local_id_table, cm_id_priv, 0, 0, GFP_NOWAIT);
-
-	spin_unlock_irqrestore(&cm.lock, flags);
-	idr_preload_end();
+	err = xa_alloc_cyclic_irq(&cm.local_id_table, &id, cm_id_priv,
+			xa_limit_32b, &cm.local_id_next, GFP_KERNEL);
 
 	cm_id_priv->id.local_id = (__force __be32)id ^ cm.random_id_operand;
-	return id < 0 ? id : 0;
+	return err;
+}
+
+static u32 cm_local_id(__be32 local_id)
+{
+	return (__force u32) (local_id ^ cm.random_id_operand);
 }
 
 static void cm_free_id(__be32 local_id)
 {
-	spin_lock_irq(&cm.lock);
-	idr_remove(&cm.local_id_table,
-		   (__force int) (local_id ^ cm.random_id_operand));
-	spin_unlock_irq(&cm.lock);
+	xa_erase_irq(&cm.local_id_table, cm_local_id(local_id));
 }
 
 static struct cm_id_private * cm_get_id(__be32 local_id, __be32 remote_id)
 {
 	struct cm_id_private *cm_id_priv;
 
-	cm_id_priv = idr_find(&cm.local_id_table,
-			      (__force int) (local_id ^ cm.random_id_operand));
+	cm_id_priv = xa_load(&cm.local_id_table, cm_local_id(local_id));
 	if (cm_id_priv) {
 		if (cm_id_priv->id.remote_id == remote_id)
 			atomic_inc(&cm_id_priv->refcount);
@@ -1988,11 +1985,12 @@ static int cm_req_handler(struct cm_work *work)
 	grh = rdma_ah_read_grh(&cm_id_priv->av.ah_attr);
 	gid_attr = grh->sgid_attr;
 
-	if (gid_attr && gid_attr->ndev) {
+	if (gid_attr &&
+	    rdma_protocol_roce(work->port->cm_dev->ib_device,
+			       work->port->port_num)) {
 		work->path[0].rec_type =
 			sa_conv_gid_to_pathrec_type(gid_attr->gid_type);
 	} else {
-		/* If no GID attribute or ndev is null, it is not RoCE. */
 		cm_path_set_rec_type(work->port->cm_dev->ib_device,
 				     work->port->port_num,
 				     &work->path[0],
@@ -2824,9 +2822,8 @@ static struct cm_id_private * cm_acquire_rejected_id(struct cm_rej_msg *rej_msg)
 			spin_unlock_irq(&cm.lock);
 			return NULL;
 		}
-		cm_id_priv = idr_find(&cm.local_id_table, (__force int)
-				      (timewait_info->work.local_id ^
-				       cm.random_id_operand));
+		cm_id_priv = xa_load(&cm.local_id_table,
+				cm_local_id(timewait_info->work.local_id));
 		if (cm_id_priv) {
 			if (cm_id_priv->id.remote_id == remote_id)
 				atomic_inc(&cm_id_priv->refcount);
@@ -4276,18 +4273,6 @@ static struct kobj_type cm_counter_obj_type = {
 	.default_attrs = cm_counter_default_attrs
 };
 
-static void cm_release_port_obj(struct kobject *obj)
-{
-	struct cm_port *cm_port;
-
-	cm_port = container_of(obj, struct cm_port, port_obj);
-	kfree(cm_port);
-}
-
-static struct kobj_type cm_port_obj_type = {
-	.release = cm_release_port_obj
-};
-
 static char *cm_devnode(struct device *dev, umode_t *mode)
 {
 	if (mode)
@@ -4306,19 +4291,12 @@ static int cm_create_port_fs(struct cm_port *port)
 {
 	int i, ret;
 
-	ret = kobject_init_and_add(&port->port_obj, &cm_port_obj_type,
-				   &port->cm_dev->device->kobj,
-				   "%d", port->port_num);
-	if (ret) {
-		kfree(port);
-		return ret;
-	}
-
 	for (i = 0; i < CM_COUNTER_GROUPS; i++) {
-		ret = kobject_init_and_add(&port->counter_group[i].obj,
-					   &cm_counter_obj_type,
-					   &port->port_obj,
-					   "%s", counter_group_names[i]);
+		ret = ib_port_register_module_stat(port->cm_dev->ib_device,
+						   port->port_num,
+						   &port->counter_group[i].obj,
+						   &cm_counter_obj_type,
+						   counter_group_names[i]);
 		if (ret)
 			goto error;
 	}
@@ -4327,8 +4305,7 @@ static int cm_create_port_fs(struct cm_port *port)
 
 error:
 	while (i--)
-		kobject_put(&port->counter_group[i].obj);
-	kobject_put(&port->port_obj);
+		ib_port_unregister_module_stat(&port->counter_group[i].obj);
 	return ret;
 
 }
@@ -4338,9 +4315,8 @@ static void cm_remove_port_fs(struct cm_port *port)
 	int i;
 
 	for (i = 0; i < CM_COUNTER_GROUPS; i++)
-		kobject_put(&port->counter_group[i].obj);
+		ib_port_unregister_module_stat(&port->counter_group[i].obj);
 
-	kobject_put(&port->port_obj);
 }
 
 static void cm_add_one(struct ib_device *ib_device)
@@ -4367,13 +4343,6 @@ static void cm_add_one(struct ib_device *ib_device)
 	cm_dev->ib_device = ib_device;
 	cm_dev->ack_delay = ib_device->attrs.local_ca_ack_delay;
 	cm_dev->going_down = 0;
-	cm_dev->device = device_create(&cm_class, &ib_device->dev,
-				       MKDEV(0, 0), NULL,
-				       "%s", dev_name(&ib_device->dev));
-	if (IS_ERR(cm_dev->device)) {
-		kfree(cm_dev);
-		return;
-	}
 
 	set_bit(IB_MGMT_METHOD_SEND, reg_req.method_mask);
 	for (i = 1; i <= ib_device->phys_port_cnt; i++) {
@@ -4440,7 +4409,6 @@ error1:
 		cm_remove_port_fs(port);
 	}
 free:
-	device_unregister(cm_dev->device);
 	kfree(cm_dev);
 }
 
@@ -4494,7 +4462,6 @@ static void cm_remove_one(struct ib_device *ib_device, void *client_data)
 		cm_remove_port_fs(port);
 	}
 
-	device_unregister(cm_dev->device);
 	kfree(cm_dev);
 }
 
@@ -4502,7 +4469,6 @@ static int __init ib_cm_init(void)
 {
 	int ret;
 
-	memset(&cm, 0, sizeof cm);
 	INIT_LIST_HEAD(&cm.device_list);
 	rwlock_init(&cm.device_lock);
 	spin_lock_init(&cm.lock);
@@ -4512,7 +4478,7 @@ static int __init ib_cm_init(void)
 	cm.remote_id_table = RB_ROOT;
 	cm.remote_qp_table = RB_ROOT;
 	cm.remote_sidr_table = RB_ROOT;
-	idr_init(&cm.local_id_table);
+	xa_init_flags(&cm.local_id_table, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_IRQ);
 	get_random_bytes(&cm.random_id_operand, sizeof cm.random_id_operand);
 	INIT_LIST_HEAD(&cm.timewait_list);
 
@@ -4538,7 +4504,6 @@ error3:
 error2:
 	class_unregister(&cm_class);
 error1:
-	idr_destroy(&cm.local_id_table);
 	return ret;
 }
 
@@ -4560,9 +4525,8 @@ static void __exit ib_cm_cleanup(void)
 	}
 
 	class_unregister(&cm_class);
-	idr_destroy(&cm.local_id_table);
+	WARN_ON(!xa_empty(&cm.local_id_table));
 }
 
 module_init(ib_cm_init);
 module_exit(ib_cm_cleanup);
-
diff --git a/drivers/infiniband/core/cm_msgs.h b/drivers/infiniband/core/cm_msgs.h
index 476d4309576d..3d16d614aff6 100644
--- a/drivers/infiniband/core/cm_msgs.h
+++ b/drivers/infiniband/core/cm_msgs.h
@@ -98,7 +98,7 @@ struct cm_req_msg {
 
 	u32 private_data[IB_CM_REQ_PRIVATE_DATA_SIZE / sizeof(u32)];
 
-} __attribute__ ((packed));
+} __packed;
 
 static inline __be32 cm_req_get_local_qpn(struct cm_req_msg *req_msg)
 {
@@ -423,7 +423,7 @@ enum cm_msg_response {
 
 	u8 private_data[IB_CM_MRA_PRIVATE_DATA_SIZE];
 
-} __attribute__ ((packed));
+} __packed;
 
 static inline u8 cm_mra_get_msg_mraed(struct cm_mra_msg *mra_msg)
 {
@@ -461,7 +461,7 @@ struct cm_rej_msg {
 
 	u8 private_data[IB_CM_REJ_PRIVATE_DATA_SIZE];
 
-} __attribute__ ((packed));
+} __packed;
 
 static inline u8 cm_rej_get_msg_rejected(struct cm_rej_msg *rej_msg)
 {
@@ -506,7 +506,7 @@ struct cm_rep_msg {
 
 	u8 private_data[IB_CM_REP_PRIVATE_DATA_SIZE];
 
-} __attribute__ ((packed));
+} __packed;
 
 static inline __be32 cm_rep_get_local_qpn(struct cm_rep_msg *rep_msg)
 {
@@ -614,7 +614,7 @@ struct cm_rtu_msg {
 
 	u8 private_data[IB_CM_RTU_PRIVATE_DATA_SIZE];
 
-} __attribute__ ((packed));
+} __packed;
 
 struct cm_dreq_msg {
 	struct ib_mad_hdr hdr;
@@ -626,7 +626,7 @@ struct cm_dreq_msg {
 
 	u8 private_data[IB_CM_DREQ_PRIVATE_DATA_SIZE];
 
-} __attribute__ ((packed));
+} __packed;
 
 static inline __be32 cm_dreq_get_remote_qpn(struct cm_dreq_msg *dreq_msg)
 {
@@ -647,7 +647,7 @@ struct cm_drep_msg {
 
 	u8 private_data[IB_CM_DREP_PRIVATE_DATA_SIZE];
 
-} __attribute__ ((packed));
+} __packed;
 
 struct cm_lap_msg {
 	struct ib_mad_hdr hdr;
@@ -675,7 +675,7 @@ struct cm_lap_msg {
 	u8 offset63;
 
 	u8 private_data[IB_CM_LAP_PRIVATE_DATA_SIZE];
-} __attribute__  ((packed));
+} __packed;
 
 static inline __be32 cm_lap_get_remote_qpn(struct cm_lap_msg *lap_msg)
 {
@@ -784,7 +784,7 @@ struct cm_apr_msg {
 	u8 info[IB_CM_APR_INFO_LENGTH];
 
 	u8 private_data[IB_CM_APR_PRIVATE_DATA_SIZE];
-} __attribute__ ((packed));
+} __packed;
 
 struct cm_sidr_req_msg {
 	struct ib_mad_hdr hdr;
@@ -795,7 +795,7 @@ struct cm_sidr_req_msg {
 	__be64 service_id;
 
 	u32 private_data[IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE / sizeof(u32)];
-} __attribute__ ((packed));
+} __packed;
 
 struct cm_sidr_rep_msg {
 	struct ib_mad_hdr hdr;
@@ -811,7 +811,7 @@ struct cm_sidr_rep_msg {
 	u8 info[IB_CM_SIDR_REP_INFO_LENGTH];
 
 	u8 private_data[IB_CM_SIDR_REP_PRIVATE_DATA_SIZE];
-} __attribute__ ((packed));
+} __packed;
 
 static inline __be32 cm_sidr_rep_get_qpn(struct cm_sidr_rep_msg *sidr_rep_msg)
 {
diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c
index 68c997be2429..19f1730a4f24 100644
--- a/drivers/infiniband/core/cma.c
+++ b/drivers/infiniband/core/cma.c
@@ -39,7 +39,7 @@
 #include <linux/mutex.h>
 #include <linux/random.h>
 #include <linux/igmp.h>
-#include <linux/idr.h>
+#include <linux/xarray.h>
 #include <linux/inetdevice.h>
 #include <linux/slab.h>
 #include <linux/module.h>
@@ -191,10 +191,10 @@ static struct workqueue_struct *cma_wq;
 static unsigned int cma_pernet_id;
 
 struct cma_pernet {
-	struct idr tcp_ps;
-	struct idr udp_ps;
-	struct idr ipoib_ps;
-	struct idr ib_ps;
+	struct xarray tcp_ps;
+	struct xarray udp_ps;
+	struct xarray ipoib_ps;
+	struct xarray ib_ps;
 };
 
 static struct cma_pernet *cma_pernet(struct net *net)
@@ -202,7 +202,8 @@ static struct cma_pernet *cma_pernet(struct net *net)
 	return net_generic(net, cma_pernet_id);
 }
 
-static struct idr *cma_pernet_idr(struct net *net, enum rdma_ucm_port_space ps)
+static
+struct xarray *cma_pernet_xa(struct net *net, enum rdma_ucm_port_space ps)
 {
 	struct cma_pernet *pernet = cma_pernet(net);
 
@@ -247,25 +248,25 @@ struct class_port_info_context {
 static int cma_ps_alloc(struct net *net, enum rdma_ucm_port_space ps,
 			struct rdma_bind_list *bind_list, int snum)
 {
-	struct idr *idr = cma_pernet_idr(net, ps);
+	struct xarray *xa = cma_pernet_xa(net, ps);
 
-	return idr_alloc(idr, bind_list, snum, snum + 1, GFP_KERNEL);
+	return xa_insert(xa, snum, bind_list, GFP_KERNEL);
 }
 
 static struct rdma_bind_list *cma_ps_find(struct net *net,
 					  enum rdma_ucm_port_space ps, int snum)
 {
-	struct idr *idr = cma_pernet_idr(net, ps);
+	struct xarray *xa = cma_pernet_xa(net, ps);
 
-	return idr_find(idr, snum);
+	return xa_load(xa, snum);
 }
 
 static void cma_ps_remove(struct net *net, enum rdma_ucm_port_space ps,
 			  int snum)
 {
-	struct idr *idr = cma_pernet_idr(net, ps);
+	struct xarray *xa = cma_pernet_xa(net, ps);
 
-	idr_remove(idr, snum);
+	xa_erase(xa, snum);
 }
 
 enum {
@@ -615,6 +616,9 @@ cma_validate_port(struct ib_device *device, u8 port,
 	int dev_type = dev_addr->dev_type;
 	struct net_device *ndev = NULL;
 
+	if (!rdma_dev_access_netns(device, id_priv->id.route.addr.dev_addr.net))
+		return ERR_PTR(-ENODEV);
+
 	if ((dev_type == ARPHRD_INFINIBAND) && !rdma_protocol_ib(device, port))
 		return ERR_PTR(-ENODEV);
 
@@ -1173,18 +1177,31 @@ static inline bool cma_any_addr(const struct sockaddr *addr)
 	return cma_zero_addr(addr) || cma_loopback_addr(addr);
 }
 
-static int cma_addr_cmp(struct sockaddr *src, struct sockaddr *dst)
+static int cma_addr_cmp(const struct sockaddr *src, const struct sockaddr *dst)
 {
 	if (src->sa_family != dst->sa_family)
 		return -1;
 
 	switch (src->sa_family) {
 	case AF_INET:
-		return ((struct sockaddr_in *) src)->sin_addr.s_addr !=
-		       ((struct sockaddr_in *) dst)->sin_addr.s_addr;
-	case AF_INET6:
-		return ipv6_addr_cmp(&((struct sockaddr_in6 *) src)->sin6_addr,
-				     &((struct sockaddr_in6 *) dst)->sin6_addr);
+		return ((struct sockaddr_in *)src)->sin_addr.s_addr !=
+		       ((struct sockaddr_in *)dst)->sin_addr.s_addr;
+	case AF_INET6: {
+		struct sockaddr_in6 *src_addr6 = (struct sockaddr_in6 *)src;
+		struct sockaddr_in6 *dst_addr6 = (struct sockaddr_in6 *)dst;
+		bool link_local;
+
+		if (ipv6_addr_cmp(&src_addr6->sin6_addr,
+					  &dst_addr6->sin6_addr))
+			return 1;
+		link_local = ipv6_addr_type(&dst_addr6->sin6_addr) &
+			     IPV6_ADDR_LINKLOCAL;
+		/* Link local must match their scope_ids */
+		return link_local ? (src_addr6->sin6_scope_id !=
+				     dst_addr6->sin6_scope_id) :
+				    0;
+	}
+
 	default:
 		return ib_addr_cmp(&((struct sockaddr_ib *) src)->sib_addr,
 				   &((struct sockaddr_ib *) dst)->sib_addr);
@@ -1469,6 +1486,7 @@ static struct net_device *
 roce_get_net_dev_by_cm_event(const struct ib_cm_event *ib_event)
 {
 	const struct ib_gid_attr *sgid_attr = NULL;
+	struct net_device *ndev;
 
 	if (ib_event->event == IB_CM_REQ_RECEIVED)
 		sgid_attr = ib_event->param.req_rcvd.ppath_sgid_attr;
@@ -1477,8 +1495,15 @@ roce_get_net_dev_by_cm_event(const struct ib_cm_event *ib_event)
 
 	if (!sgid_attr)
 		return NULL;
-	dev_hold(sgid_attr->ndev);
-	return sgid_attr->ndev;
+
+	rcu_read_lock();
+	ndev = rdma_read_gid_attr_ndev_rcu(sgid_attr);
+	if (IS_ERR(ndev))
+		ndev = NULL;
+	else
+		dev_hold(ndev);
+	rcu_read_unlock();
+	return ndev;
 }
 
 static struct net_device *cma_get_net_dev(const struct ib_cm_event *ib_event,
@@ -3247,7 +3272,7 @@ static int cma_alloc_port(enum rdma_ucm_port_space ps,
 		goto err;
 
 	bind_list->ps = ps;
-	bind_list->port = (unsigned short)ret;
+	bind_list->port = snum;
 	cma_bind_port(bind_list, id_priv);
 	return 0;
 err:
@@ -4655,10 +4680,10 @@ static int cma_init_net(struct net *net)
 {
 	struct cma_pernet *pernet = cma_pernet(net);
 
-	idr_init(&pernet->tcp_ps);
-	idr_init(&pernet->udp_ps);
-	idr_init(&pernet->ipoib_ps);
-	idr_init(&pernet->ib_ps);
+	xa_init(&pernet->tcp_ps);
+	xa_init(&pernet->udp_ps);
+	xa_init(&pernet->ipoib_ps);
+	xa_init(&pernet->ib_ps);
 
 	return 0;
 }
@@ -4667,10 +4692,10 @@ static void cma_exit_net(struct net *net)
 {
 	struct cma_pernet *pernet = cma_pernet(net);
 
-	idr_destroy(&pernet->tcp_ps);
-	idr_destroy(&pernet->udp_ps);
-	idr_destroy(&pernet->ipoib_ps);
-	idr_destroy(&pernet->ib_ps);
+	WARN_ON(!xa_empty(&pernet->tcp_ps));
+	WARN_ON(!xa_empty(&pernet->udp_ps));
+	WARN_ON(!xa_empty(&pernet->ipoib_ps));
+	WARN_ON(!xa_empty(&pernet->ib_ps));
 }
 
 static struct pernet_operations cma_pernet_operations = {
diff --git a/drivers/infiniband/core/core_priv.h b/drivers/infiniband/core/core_priv.h
index 08c690249594..ff40a450b5d2 100644
--- a/drivers/infiniband/core/core_priv.h
+++ b/drivers/infiniband/core/core_priv.h
@@ -55,6 +55,7 @@ struct pkey_index_qp_list {
 };
 
 extern const struct attribute_group ib_dev_attr_group;
+extern bool ib_devices_shared_netns;
 
 int ib_device_register_sysfs(struct ib_device *device);
 void ib_device_unregister_sysfs(struct ib_device *device);
@@ -279,7 +280,8 @@ static inline void ib_mad_agent_security_change(void)
 }
 #endif
 
-struct ib_device *ib_device_get_by_index(u32 ifindex);
+struct ib_device *ib_device_get_by_index(const struct net *net, u32 index);
+
 /* RDMA device netlink */
 void nldev_init(void);
 void nldev_exit(void);
@@ -302,6 +304,7 @@ static inline struct ib_qp *_ib_create_qp(struct ib_device *dev,
 	qp->device = dev;
 	qp->pd = pd;
 	qp->uobject = uobj;
+	qp->real_qp = qp;
 	/*
 	 * We don't track XRC QPs for now, because they don't have PD
 	 * and more importantly they are created internaly by driver,
@@ -336,4 +339,17 @@ int roce_resolve_route_from_path(struct sa_path_rec *rec,
 				 const struct ib_gid_attr *attr);
 
 struct net_device *rdma_read_gid_attr_ndev_rcu(const struct ib_gid_attr *attr);
+
+void ib_free_port_attrs(struct ib_core_device *coredev);
+int ib_setup_port_attrs(struct ib_core_device *coredev);
+
+int rdma_compatdev_set(u8 enable);
+
+int ib_port_register_module_stat(struct ib_device *device, u8 port_num,
+				 struct kobject *kobj, struct kobj_type *ktype,
+				 const char *name);
+void ib_port_unregister_module_stat(struct kobject *kobj);
+
+int ib_device_set_netns_put(struct sk_buff *skb,
+			    struct ib_device *dev, u32 ns_fd);
 #endif /* _CORE_PRIV_H */
diff --git a/drivers/infiniband/core/cq.c b/drivers/infiniband/core/cq.c
index d61e5e1427c2..a4c81992267c 100644
--- a/drivers/infiniband/core/cq.c
+++ b/drivers/infiniband/core/cq.c
@@ -128,15 +128,17 @@ static void ib_cq_completion_workqueue(struct ib_cq *cq, void *private)
  * @comp_vector:	HCA completion vectors for this CQ
  * @poll_ctx:		context to poll the CQ from.
  * @caller:		module owner name.
+ * @udata:		Valid user data or NULL for kernel object
  *
  * This is the proper interface to allocate a CQ for in-kernel users. A
  * CQ allocated with this interface will automatically be polled from the
  * specified context. The ULP must use wr->wr_cqe instead of wr->wr_id
  * to use this CQ abstraction.
  */
-struct ib_cq *__ib_alloc_cq(struct ib_device *dev, void *private,
-			    int nr_cqe, int comp_vector,
-			    enum ib_poll_context poll_ctx, const char *caller)
+struct ib_cq *__ib_alloc_cq_user(struct ib_device *dev, void *private,
+				 int nr_cqe, int comp_vector,
+				 enum ib_poll_context poll_ctx,
+				 const char *caller, struct ib_udata *udata)
 {
 	struct ib_cq_init_attr cq_attr = {
 		.cqe		= nr_cqe,
@@ -145,7 +147,7 @@ struct ib_cq *__ib_alloc_cq(struct ib_device *dev, void *private,
 	struct ib_cq *cq;
 	int ret = -ENOMEM;
 
-	cq = dev->ops.create_cq(dev, &cq_attr, NULL, NULL);
+	cq = dev->ops.create_cq(dev, &cq_attr, NULL);
 	if (IS_ERR(cq))
 		return cq;
 
@@ -193,16 +195,17 @@ out_free_wc:
 	kfree(cq->wc);
 	rdma_restrack_del(&cq->res);
 out_destroy_cq:
-	cq->device->ops.destroy_cq(cq);
+	cq->device->ops.destroy_cq(cq, udata);
 	return ERR_PTR(ret);
 }
-EXPORT_SYMBOL(__ib_alloc_cq);
+EXPORT_SYMBOL(__ib_alloc_cq_user);
 
 /**
  * ib_free_cq - free a completion queue
  * @cq:		completion queue to free.
+ * @udata:	User data or NULL for kernel object
  */
-void ib_free_cq(struct ib_cq *cq)
+void ib_free_cq_user(struct ib_cq *cq, struct ib_udata *udata)
 {
 	int ret;
 
@@ -225,7 +228,7 @@ void ib_free_cq(struct ib_cq *cq)
 
 	kfree(cq->wc);
 	rdma_restrack_del(&cq->res);
-	ret = cq->device->ops.destroy_cq(cq);
+	ret = cq->device->ops.destroy_cq(cq, udata);
 	WARN_ON_ONCE(ret);
 }
-EXPORT_SYMBOL(ib_free_cq);
+EXPORT_SYMBOL(ib_free_cq_user);
diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c
index 7421ec4883fb..78dc07c6ac4b 100644
--- a/drivers/infiniband/core/device.c
+++ b/drivers/infiniband/core/device.c
@@ -38,6 +38,8 @@
 #include <linux/slab.h>
 #include <linux/init.h>
 #include <linux/netdevice.h>
+#include <net/net_namespace.h>
+#include <net/netns/generic.h>
 #include <linux/security.h>
 #include <linux/notifier.h>
 #include <linux/hashtable.h>
@@ -101,6 +103,54 @@ static DECLARE_RWSEM(clients_rwsem);
  * be registered.
  */
 #define CLIENT_DATA_REGISTERED XA_MARK_1
+
+/**
+ * struct rdma_dev_net - rdma net namespace metadata for a net
+ * @net:	Pointer to owner net namespace
+ * @id:		xarray id to identify the net namespace.
+ */
+struct rdma_dev_net {
+	possible_net_t net;
+	u32 id;
+};
+
+static unsigned int rdma_dev_net_id;
+
+/*
+ * A list of net namespaces is maintained in an xarray. This is necessary
+ * because we can't get the locking right using the existing net ns list. We
+ * would require a init_net callback after the list is updated.
+ */
+static DEFINE_XARRAY_FLAGS(rdma_nets, XA_FLAGS_ALLOC);
+/*
+ * rwsem to protect accessing the rdma_nets xarray entries.
+ */
+static DECLARE_RWSEM(rdma_nets_rwsem);
+
+bool ib_devices_shared_netns = true;
+module_param_named(netns_mode, ib_devices_shared_netns, bool, 0444);
+MODULE_PARM_DESC(netns_mode,
+		 "Share device among net namespaces; default=1 (shared)");
+/**
+ * rdma_dev_access_netns() - Return whether a rdma device can be accessed
+ *			     from a specified net namespace or not.
+ * @device:	Pointer to rdma device which needs to be checked
+ * @net:	Pointer to net namesapce for which access to be checked
+ *
+ * rdma_dev_access_netns() - Return whether a rdma device can be accessed
+ *			     from a specified net namespace or not. When
+ *			     rdma device is in shared mode, it ignores the
+ *			     net namespace. When rdma device is exclusive
+ *			     to a net namespace, rdma device net namespace is
+ *			     checked against the specified one.
+ */
+bool rdma_dev_access_netns(const struct ib_device *dev, const struct net *net)
+{
+	return (ib_devices_shared_netns ||
+		net_eq(read_pnet(&dev->coredev.rdma_net), net));
+}
+EXPORT_SYMBOL(rdma_dev_access_netns);
+
 /*
  * xarray has this behavior where it won't iterate over NULL values stored in
  * allocated arrays.  So we need our own iterator to see all values stored in
@@ -147,10 +197,73 @@ static int ib_security_change(struct notifier_block *nb, unsigned long event,
 static void ib_policy_change_task(struct work_struct *work);
 static DECLARE_WORK(ib_policy_change_work, ib_policy_change_task);
 
+static void __ibdev_printk(const char *level, const struct ib_device *ibdev,
+			   struct va_format *vaf)
+{
+	if (ibdev && ibdev->dev.parent)
+		dev_printk_emit(level[1] - '0',
+				ibdev->dev.parent,
+				"%s %s %s: %pV",
+				dev_driver_string(ibdev->dev.parent),
+				dev_name(ibdev->dev.parent),
+				dev_name(&ibdev->dev),
+				vaf);
+	else if (ibdev)
+		printk("%s%s: %pV",
+		       level, dev_name(&ibdev->dev), vaf);
+	else
+		printk("%s(NULL ib_device): %pV", level, vaf);
+}
+
+void ibdev_printk(const char *level, const struct ib_device *ibdev,
+		  const char *format, ...)
+{
+	struct va_format vaf;
+	va_list args;
+
+	va_start(args, format);
+
+	vaf.fmt = format;
+	vaf.va = &args;
+
+	__ibdev_printk(level, ibdev, &vaf);
+
+	va_end(args);
+}
+EXPORT_SYMBOL(ibdev_printk);
+
+#define define_ibdev_printk_level(func, level)                  \
+void func(const struct ib_device *ibdev, const char *fmt, ...)  \
+{                                                               \
+	struct va_format vaf;                                   \
+	va_list args;                                           \
+								\
+	va_start(args, fmt);                                    \
+								\
+	vaf.fmt = fmt;                                          \
+	vaf.va = &args;                                         \
+								\
+	__ibdev_printk(level, ibdev, &vaf);                     \
+								\
+	va_end(args);                                           \
+}                                                               \
+EXPORT_SYMBOL(func);
+
+define_ibdev_printk_level(ibdev_emerg, KERN_EMERG);
+define_ibdev_printk_level(ibdev_alert, KERN_ALERT);
+define_ibdev_printk_level(ibdev_crit, KERN_CRIT);
+define_ibdev_printk_level(ibdev_err, KERN_ERR);
+define_ibdev_printk_level(ibdev_warn, KERN_WARNING);
+define_ibdev_printk_level(ibdev_notice, KERN_NOTICE);
+define_ibdev_printk_level(ibdev_info, KERN_INFO);
+
 static struct notifier_block ibdev_lsm_nb = {
 	.notifier_call = ib_security_change,
 };
 
+static int rdma_dev_change_netns(struct ib_device *device, struct net *cur_net,
+				 struct net *net);
+
 /* Pointer to the RCU head at the start of the ib_port_data array */
 struct ib_port_data_rcu {
 	struct rcu_head rcu_head;
@@ -200,16 +313,22 @@ static int ib_device_check_mandatory(struct ib_device *device)
  * Caller must perform ib_device_put() to return the device reference count
  * when ib_device_get_by_index() returns valid device pointer.
  */
-struct ib_device *ib_device_get_by_index(u32 index)
+struct ib_device *ib_device_get_by_index(const struct net *net, u32 index)
 {
 	struct ib_device *device;
 
 	down_read(&devices_rwsem);
 	device = xa_load(&devices, index);
 	if (device) {
+		if (!rdma_dev_access_netns(device, net)) {
+			device = NULL;
+			goto out;
+		}
+
 		if (!ib_device_try_get(device))
 			device = NULL;
 	}
+out:
 	up_read(&devices_rwsem);
 	return device;
 }
@@ -268,6 +387,26 @@ struct ib_device *ib_device_get_by_name(const char *name,
 }
 EXPORT_SYMBOL(ib_device_get_by_name);
 
+static int rename_compat_devs(struct ib_device *device)
+{
+	struct ib_core_device *cdev;
+	unsigned long index;
+	int ret = 0;
+
+	mutex_lock(&device->compat_devs_mutex);
+	xa_for_each (&device->compat_devs, index, cdev) {
+		ret = device_rename(&cdev->dev, dev_name(&device->dev));
+		if (ret) {
+			dev_warn(&cdev->dev,
+				 "Fail to rename compatdev to new name %s\n",
+				 dev_name(&device->dev));
+			break;
+		}
+	}
+	mutex_unlock(&device->compat_devs_mutex);
+	return ret;
+}
+
 int ib_device_rename(struct ib_device *ibdev, const char *name)
 {
 	int ret;
@@ -287,6 +426,7 @@ int ib_device_rename(struct ib_device *ibdev, const char *name)
 	if (ret)
 		goto out;
 	strlcpy(ibdev->name, name, IB_DEVICE_NAME_MAX);
+	ret = rename_compat_devs(ibdev);
 out:
 	up_write(&devices_rwsem);
 	return ret;
@@ -336,6 +476,7 @@ static void ib_device_release(struct device *device)
 	WARN_ON(refcount_read(&dev->refcount));
 	ib_cache_release_one(dev);
 	ib_security_release_port_pkey_list(dev);
+	xa_destroy(&dev->compat_devs);
 	xa_destroy(&dev->client_data);
 	if (dev->port_data)
 		kfree_rcu(container_of(dev->port_data, struct ib_port_data_rcu,
@@ -357,12 +498,42 @@ static int ib_device_uevent(struct device *device,
 	return 0;
 }
 
+static const void *net_namespace(struct device *d)
+{
+	struct ib_core_device *coredev =
+			container_of(d, struct ib_core_device, dev);
+
+	return read_pnet(&coredev->rdma_net);
+}
+
 static struct class ib_class = {
 	.name    = "infiniband",
 	.dev_release = ib_device_release,
 	.dev_uevent = ib_device_uevent,
+	.ns_type = &net_ns_type_operations,
+	.namespace = net_namespace,
 };
 
+static void rdma_init_coredev(struct ib_core_device *coredev,
+			      struct ib_device *dev, struct net *net)
+{
+	/* This BUILD_BUG_ON is intended to catch layout change
+	 * of union of ib_core_device and device.
+	 * dev must be the first element as ib_core and providers
+	 * driver uses it. Adding anything in ib_core_device before
+	 * device will break this assumption.
+	 */
+	BUILD_BUG_ON(offsetof(struct ib_device, coredev.dev) !=
+		     offsetof(struct ib_device, dev));
+
+	coredev->dev.class = &ib_class;
+	coredev->dev.groups = dev->groups;
+	device_initialize(&coredev->dev);
+	coredev->owner = dev;
+	INIT_LIST_HEAD(&coredev->port_list);
+	write_pnet(&coredev->rdma_net, net);
+}
+
 /**
  * _ib_alloc_device - allocate an IB device struct
  * @size:size of structure to allocate
@@ -389,10 +560,8 @@ struct ib_device *_ib_alloc_device(size_t size)
 		return NULL;
 	}
 
-	device->dev.class = &ib_class;
 	device->groups[0] = &ib_dev_attr_group;
-	device->dev.groups = device->groups;
-	device_initialize(&device->dev);
+	rdma_init_coredev(&device->coredev, device, &init_net);
 
 	INIT_LIST_HEAD(&device->event_handler_list);
 	spin_lock_init(&device->event_handler_lock);
@@ -403,7 +572,8 @@ struct ib_device *_ib_alloc_device(size_t size)
 	 */
 	xa_init_flags(&device->client_data, XA_FLAGS_ALLOC);
 	init_rwsem(&device->client_data_rwsem);
-	INIT_LIST_HEAD(&device->port_list);
+	xa_init_flags(&device->compat_devs, XA_FLAGS_ALLOC);
+	mutex_init(&device->compat_devs_mutex);
 	init_completion(&device->unreg_completion);
 	INIT_WORK(&device->unregistration_work, ib_unregister_work);
 
@@ -436,6 +606,7 @@ void ib_dealloc_device(struct ib_device *device)
 	/* Expedite releasing netdev references */
 	free_netdevs(device);
 
+	WARN_ON(!xa_empty(&device->compat_devs));
 	WARN_ON(!xa_empty(&device->client_data));
 	WARN_ON(refcount_read(&device->refcount));
 	rdma_restrack_clean(device);
@@ -644,6 +815,283 @@ static int ib_security_change(struct notifier_block *nb, unsigned long event,
 	return NOTIFY_OK;
 }
 
+static void compatdev_release(struct device *dev)
+{
+	struct ib_core_device *cdev =
+		container_of(dev, struct ib_core_device, dev);
+
+	kfree(cdev);
+}
+
+static int add_one_compat_dev(struct ib_device *device,
+			      struct rdma_dev_net *rnet)
+{
+	struct ib_core_device *cdev;
+	int ret;
+
+	lockdep_assert_held(&rdma_nets_rwsem);
+	if (!ib_devices_shared_netns)
+		return 0;
+
+	/*
+	 * Create and add compat device in all namespaces other than where it
+	 * is currently bound to.
+	 */
+	if (net_eq(read_pnet(&rnet->net),
+		   read_pnet(&device->coredev.rdma_net)))
+		return 0;
+
+	/*
+	 * The first of init_net() or ib_register_device() to take the
+	 * compat_devs_mutex wins and gets to add the device. Others will wait
+	 * for completion here.
+	 */
+	mutex_lock(&device->compat_devs_mutex);
+	cdev = xa_load(&device->compat_devs, rnet->id);
+	if (cdev) {
+		ret = 0;
+		goto done;
+	}
+	ret = xa_reserve(&device->compat_devs, rnet->id, GFP_KERNEL);
+	if (ret)
+		goto done;
+
+	cdev = kzalloc(sizeof(*cdev), GFP_KERNEL);
+	if (!cdev) {
+		ret = -ENOMEM;
+		goto cdev_err;
+	}
+
+	cdev->dev.parent = device->dev.parent;
+	rdma_init_coredev(cdev, device, read_pnet(&rnet->net));
+	cdev->dev.release = compatdev_release;
+	dev_set_name(&cdev->dev, "%s", dev_name(&device->dev));
+
+	ret = device_add(&cdev->dev);
+	if (ret)
+		goto add_err;
+	ret = ib_setup_port_attrs(cdev);
+	if (ret)
+		goto port_err;
+
+	ret = xa_err(xa_store(&device->compat_devs, rnet->id,
+			      cdev, GFP_KERNEL));
+	if (ret)
+		goto insert_err;
+
+	mutex_unlock(&device->compat_devs_mutex);
+	return 0;
+
+insert_err:
+	ib_free_port_attrs(cdev);
+port_err:
+	device_del(&cdev->dev);
+add_err:
+	put_device(&cdev->dev);
+cdev_err:
+	xa_release(&device->compat_devs, rnet->id);
+done:
+	mutex_unlock(&device->compat_devs_mutex);
+	return ret;
+}
+
+static void remove_one_compat_dev(struct ib_device *device, u32 id)
+{
+	struct ib_core_device *cdev;
+
+	mutex_lock(&device->compat_devs_mutex);
+	cdev = xa_erase(&device->compat_devs, id);
+	mutex_unlock(&device->compat_devs_mutex);
+	if (cdev) {
+		ib_free_port_attrs(cdev);
+		device_del(&cdev->dev);
+		put_device(&cdev->dev);
+	}
+}
+
+static void remove_compat_devs(struct ib_device *device)
+{
+	struct ib_core_device *cdev;
+	unsigned long index;
+
+	xa_for_each (&device->compat_devs, index, cdev)
+		remove_one_compat_dev(device, index);
+}
+
+static int add_compat_devs(struct ib_device *device)
+{
+	struct rdma_dev_net *rnet;
+	unsigned long index;
+	int ret = 0;
+
+	lockdep_assert_held(&devices_rwsem);
+
+	down_read(&rdma_nets_rwsem);
+	xa_for_each (&rdma_nets, index, rnet) {
+		ret = add_one_compat_dev(device, rnet);
+		if (ret)
+			break;
+	}
+	up_read(&rdma_nets_rwsem);
+	return ret;
+}
+
+static void remove_all_compat_devs(void)
+{
+	struct ib_compat_device *cdev;
+	struct ib_device *dev;
+	unsigned long index;
+
+	down_read(&devices_rwsem);
+	xa_for_each (&devices, index, dev) {
+		unsigned long c_index = 0;
+
+		/* Hold nets_rwsem so that any other thread modifying this
+		 * system param can sync with this thread.
+		 */
+		down_read(&rdma_nets_rwsem);
+		xa_for_each (&dev->compat_devs, c_index, cdev)
+			remove_one_compat_dev(dev, c_index);
+		up_read(&rdma_nets_rwsem);
+	}
+	up_read(&devices_rwsem);
+}
+
+static int add_all_compat_devs(void)
+{
+	struct rdma_dev_net *rnet;
+	struct ib_device *dev;
+	unsigned long index;
+	int ret = 0;
+
+	down_read(&devices_rwsem);
+	xa_for_each_marked (&devices, index, dev, DEVICE_REGISTERED) {
+		unsigned long net_index = 0;
+
+		/* Hold nets_rwsem so that any other thread modifying this
+		 * system param can sync with this thread.
+		 */
+		down_read(&rdma_nets_rwsem);
+		xa_for_each (&rdma_nets, net_index, rnet) {
+			ret = add_one_compat_dev(dev, rnet);
+			if (ret)
+				break;
+		}
+		up_read(&rdma_nets_rwsem);
+	}
+	up_read(&devices_rwsem);
+	if (ret)
+		remove_all_compat_devs();
+	return ret;
+}
+
+int rdma_compatdev_set(u8 enable)
+{
+	struct rdma_dev_net *rnet;
+	unsigned long index;
+	int ret = 0;
+
+	down_write(&rdma_nets_rwsem);
+	if (ib_devices_shared_netns == enable) {
+		up_write(&rdma_nets_rwsem);
+		return 0;
+	}
+
+	/* enable/disable of compat devices is not supported
+	 * when more than default init_net exists.
+	 */
+	xa_for_each (&rdma_nets, index, rnet) {
+		ret++;
+		break;
+	}
+	if (!ret)
+		ib_devices_shared_netns = enable;
+	up_write(&rdma_nets_rwsem);
+	if (ret)
+		return -EBUSY;
+
+	if (enable)
+		ret = add_all_compat_devs();
+	else
+		remove_all_compat_devs();
+	return ret;
+}
+
+static void rdma_dev_exit_net(struct net *net)
+{
+	struct rdma_dev_net *rnet = net_generic(net, rdma_dev_net_id);
+	struct ib_device *dev;
+	unsigned long index;
+	int ret;
+
+	down_write(&rdma_nets_rwsem);
+	/*
+	 * Prevent the ID from being re-used and hide the id from xa_for_each.
+	 */
+	ret = xa_err(xa_store(&rdma_nets, rnet->id, NULL, GFP_KERNEL));
+	WARN_ON(ret);
+	up_write(&rdma_nets_rwsem);
+
+	down_read(&devices_rwsem);
+	xa_for_each (&devices, index, dev) {
+		get_device(&dev->dev);
+		/*
+		 * Release the devices_rwsem so that pontentially blocking
+		 * device_del, doesn't hold the devices_rwsem for too long.
+		 */
+		up_read(&devices_rwsem);
+
+		remove_one_compat_dev(dev, rnet->id);
+
+		/*
+		 * If the real device is in the NS then move it back to init.
+		 */
+		rdma_dev_change_netns(dev, net, &init_net);
+
+		put_device(&dev->dev);
+		down_read(&devices_rwsem);
+	}
+	up_read(&devices_rwsem);
+
+	xa_erase(&rdma_nets, rnet->id);
+}
+
+static __net_init int rdma_dev_init_net(struct net *net)
+{
+	struct rdma_dev_net *rnet = net_generic(net, rdma_dev_net_id);
+	unsigned long index;
+	struct ib_device *dev;
+	int ret;
+
+	/* No need to create any compat devices in default init_net. */
+	if (net_eq(net, &init_net))
+		return 0;
+
+	write_pnet(&rnet->net, net);
+
+	ret = xa_alloc(&rdma_nets, &rnet->id, rnet, xa_limit_32b, GFP_KERNEL);
+	if (ret)
+		return ret;
+
+	down_read(&devices_rwsem);
+	xa_for_each_marked (&devices, index, dev, DEVICE_REGISTERED) {
+		/* Hold nets_rwsem so that netlink command cannot change
+		 * system configuration for device sharing mode.
+		 */
+		down_read(&rdma_nets_rwsem);
+		ret = add_one_compat_dev(dev, rnet);
+		up_read(&rdma_nets_rwsem);
+		if (ret)
+			break;
+	}
+	up_read(&devices_rwsem);
+
+	if (ret)
+		rdma_dev_exit_net(net);
+
+	return ret;
+}
+
 /*
  * Assign the unique string device name and the unique device index. This is
  * undone by ib_dealloc_device.
@@ -711,6 +1159,9 @@ static void setup_dma_device(struct ib_device *device)
 		WARN_ON_ONCE(!parent);
 		device->dma_device = parent;
 	}
+	/* Setup default max segment size for all IB devices */
+	dma_set_max_seg_size(device->dma_device, SZ_2G);
+
 }
 
 /*
@@ -765,8 +1216,12 @@ static void disable_device(struct ib_device *device)
 	ib_device_put(device);
 	wait_for_completion(&device->unreg_completion);
 
-	/* Expedite removing unregistered pointers from the hash table */
-	free_netdevs(device);
+	/*
+	 * compat devices must be removed after device refcount drops to zero.
+	 * Otherwise init_net() may add more compatdevs after removing compat
+	 * devices and before device is disabled.
+	 */
+	remove_compat_devs(device);
 }
 
 /*
@@ -807,7 +1262,8 @@ static int enable_device_and_get(struct ib_device *device)
 			break;
 	}
 	up_read(&clients_rwsem);
-
+	if (!ret)
+		ret = add_compat_devs(device);
 out:
 	up_read(&devices_rwsem);
 	return ret;
@@ -847,6 +1303,11 @@ int ib_register_device(struct ib_device *device, const char *name)
 
 	ib_device_register_rdmacg(device);
 
+	/*
+	 * Ensure that ADD uevent is not fired because it
+	 * is too early amd device is not initialized yet.
+	 */
+	dev_set_uevent_suppress(&device->dev, true);
 	ret = device_add(&device->dev);
 	if (ret)
 		goto cg_cleanup;
@@ -859,6 +1320,9 @@ int ib_register_device(struct ib_device *device, const char *name)
 	}
 
 	ret = enable_device_and_get(device);
+	dev_set_uevent_suppress(&device->dev, false);
+	/* Mark for userspace that device is ready */
+	kobject_uevent(&device->dev.kobj, KOBJ_ADD);
 	if (ret) {
 		void (*dealloc_fn)(struct ib_device *);
 
@@ -887,6 +1351,7 @@ int ib_register_device(struct ib_device *device, const char *name)
 dev_cleanup:
 	device_del(&device->dev);
 cg_cleanup:
+	dev_set_uevent_suppress(&device->dev, false);
 	ib_device_unregister_rdmacg(device);
 	ib_cache_cleanup_one(device);
 	return ret;
@@ -908,6 +1373,10 @@ static void __ib_unregister_device(struct ib_device *ib_dev)
 		goto out;
 
 	disable_device(ib_dev);
+
+	/* Expedite removing unregistered pointers from the hash table */
+	free_netdevs(ib_dev);
+
 	ib_device_unregister_sysfs(ib_dev);
 	device_del(&ib_dev->dev);
 	ib_device_unregister_rdmacg(ib_dev);
@@ -1038,6 +1507,126 @@ void ib_unregister_device_queued(struct ib_device *ib_dev)
 }
 EXPORT_SYMBOL(ib_unregister_device_queued);
 
+/*
+ * The caller must pass in a device that has the kref held and the refcount
+ * released. If the device is in cur_net and still registered then it is moved
+ * into net.
+ */
+static int rdma_dev_change_netns(struct ib_device *device, struct net *cur_net,
+				 struct net *net)
+{
+	int ret2 = -EINVAL;
+	int ret;
+
+	mutex_lock(&device->unregistration_lock);
+
+	/*
+	 * If a device not under ib_device_get() or if the unregistration_lock
+	 * is not held, the namespace can be changed, or it can be unregistered.
+	 * Check again under the lock.
+	 */
+	if (refcount_read(&device->refcount) == 0 ||
+	    !net_eq(cur_net, read_pnet(&device->coredev.rdma_net))) {
+		ret = -ENODEV;
+		goto out;
+	}
+
+	kobject_uevent(&device->dev.kobj, KOBJ_REMOVE);
+	disable_device(device);
+
+	/*
+	 * At this point no one can be using the device, so it is safe to
+	 * change the namespace.
+	 */
+	write_pnet(&device->coredev.rdma_net, net);
+
+	down_read(&devices_rwsem);
+	/*
+	 * Currently rdma devices are system wide unique. So the device name
+	 * is guaranteed free in the new namespace. Publish the new namespace
+	 * at the sysfs level.
+	 */
+	ret = device_rename(&device->dev, dev_name(&device->dev));
+	up_read(&devices_rwsem);
+	if (ret) {
+		dev_warn(&device->dev,
+			 "%s: Couldn't rename device after namespace change\n",
+			 __func__);
+		/* Try and put things back and re-enable the device */
+		write_pnet(&device->coredev.rdma_net, cur_net);
+	}
+
+	ret2 = enable_device_and_get(device);
+	if (ret2) {
+		/*
+		 * This shouldn't really happen, but if it does, let the user
+		 * retry at later point. So don't disable the device.
+		 */
+		dev_warn(&device->dev,
+			 "%s: Couldn't re-enable device after namespace change\n",
+			 __func__);
+	}
+	kobject_uevent(&device->dev.kobj, KOBJ_ADD);
+
+	ib_device_put(device);
+out:
+	mutex_unlock(&device->unregistration_lock);
+	if (ret)
+		return ret;
+	return ret2;
+}
+
+int ib_device_set_netns_put(struct sk_buff *skb,
+			    struct ib_device *dev, u32 ns_fd)
+{
+	struct net *net;
+	int ret;
+
+	net = get_net_ns_by_fd(ns_fd);
+	if (IS_ERR(net)) {
+		ret = PTR_ERR(net);
+		goto net_err;
+	}
+
+	if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN)) {
+		ret = -EPERM;
+		goto ns_err;
+	}
+
+	/*
+	 * Currently supported only for those providers which support
+	 * disassociation and don't do port specific sysfs init. Once a
+	 * port_cleanup infrastructure is implemented, this limitation will be
+	 * removed.
+	 */
+	if (!dev->ops.disassociate_ucontext || dev->ops.init_port ||
+	    ib_devices_shared_netns) {
+		ret = -EOPNOTSUPP;
+		goto ns_err;
+	}
+
+	get_device(&dev->dev);
+	ib_device_put(dev);
+	ret = rdma_dev_change_netns(dev, current->nsproxy->net_ns, net);
+	put_device(&dev->dev);
+
+	put_net(net);
+	return ret;
+
+ns_err:
+	put_net(net);
+net_err:
+	ib_device_put(dev);
+	return ret;
+}
+
+static struct pernet_operations rdma_dev_net_ops = {
+	.init = rdma_dev_init_net,
+	.exit = rdma_dev_exit_net,
+	.id = &rdma_dev_net_id,
+	.size = sizeof(struct rdma_dev_net),
+};
+
 static int assign_client_id(struct ib_client *client)
 {
 	int ret;
@@ -1515,6 +2104,9 @@ int ib_enum_all_devs(nldev_callback nldev_cb, struct sk_buff *skb,
 
 	down_read(&devices_rwsem);
 	xa_for_each_marked (&devices, index, dev, DEVICE_REGISTERED) {
+		if (!rdma_dev_access_netns(dev, sock_net(skb->sk)))
+			continue;
+
 		ret = nldev_cb(dev, skb, cb, idx);
 		if (ret)
 			break;
@@ -1787,6 +2379,14 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops)
 	SET_DEVICE_OP(dev_ops, get_vf_config);
 	SET_DEVICE_OP(dev_ops, get_vf_stats);
 	SET_DEVICE_OP(dev_ops, init_port);
+	SET_DEVICE_OP(dev_ops, iw_accept);
+	SET_DEVICE_OP(dev_ops, iw_add_ref);
+	SET_DEVICE_OP(dev_ops, iw_connect);
+	SET_DEVICE_OP(dev_ops, iw_create_listen);
+	SET_DEVICE_OP(dev_ops, iw_destroy_listen);
+	SET_DEVICE_OP(dev_ops, iw_get_qp);
+	SET_DEVICE_OP(dev_ops, iw_reject);
+	SET_DEVICE_OP(dev_ops, iw_rem_ref);
 	SET_DEVICE_OP(dev_ops, map_mr_sg);
 	SET_DEVICE_OP(dev_ops, map_phys_fmr);
 	SET_DEVICE_OP(dev_ops, mmap);
@@ -1823,7 +2423,9 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops)
 	SET_DEVICE_OP(dev_ops, set_vf_link_state);
 	SET_DEVICE_OP(dev_ops, unmap_fmr);
 
+	SET_OBJ_SIZE(dev_ops, ib_ah);
 	SET_OBJ_SIZE(dev_ops, ib_pd);
+	SET_OBJ_SIZE(dev_ops, ib_srq);
 	SET_OBJ_SIZE(dev_ops, ib_ucontext);
 }
 EXPORT_SYMBOL(ib_set_device_ops);
@@ -1903,12 +2505,20 @@ static int __init ib_core_init(void)
 		goto err_sa;
 	}
 
+	ret = register_pernet_device(&rdma_dev_net_ops);
+	if (ret) {
+		pr_warn("Couldn't init compat dev. ret %d\n", ret);
+		goto err_compat;
+	}
+
 	nldev_init();
 	rdma_nl_register(RDMA_NL_LS, ibnl_ls_cb_table);
 	roce_gid_mgmt_init();
 
 	return 0;
 
+err_compat:
+	unregister_lsm_notifier(&ibdev_lsm_nb);
 err_sa:
 	ib_sa_cleanup();
 err_mad:
@@ -1933,6 +2543,7 @@ static void __exit ib_core_cleanup(void)
 	roce_gid_mgmt_cleanup();
 	nldev_exit();
 	rdma_nl_unregister(RDMA_NL_LS);
+	unregister_pernet_device(&rdma_dev_net_ops);
 	unregister_lsm_notifier(&ibdev_lsm_nb);
 	ib_sa_cleanup();
 	ib_mad_cleanup();
@@ -1950,5 +2561,8 @@ static void __exit ib_core_cleanup(void)
 
 MODULE_ALIAS_RDMA_NETLINK(RDMA_NL_LS, 4);
 
-subsys_initcall(ib_core_init);
+/* ib core relies on netdev stack to first register net_ns_type_operations
+ * ns kobject type before ib_core initialization.
+ */
+fs_initcall(ib_core_init);
 module_exit(ib_core_cleanup);
diff --git a/drivers/infiniband/core/iwcm.c b/drivers/infiniband/core/iwcm.c
index 732637c913d9..72141c5b7c95 100644
--- a/drivers/infiniband/core/iwcm.c
+++ b/drivers/infiniband/core/iwcm.c
@@ -394,7 +394,7 @@ static void destroy_cm_id(struct iw_cm_id *cm_id)
 		cm_id_priv->state = IW_CM_STATE_DESTROYING;
 		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
 		/* destroy the listening endpoint */
-		cm_id->device->iwcm->destroy_listen(cm_id);
+		cm_id->device->ops.iw_destroy_listen(cm_id);
 		spin_lock_irqsave(&cm_id_priv->lock, flags);
 		break;
 	case IW_CM_STATE_ESTABLISHED:
@@ -417,7 +417,7 @@ static void destroy_cm_id(struct iw_cm_id *cm_id)
 		 */
 		cm_id_priv->state = IW_CM_STATE_DESTROYING;
 		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
-		cm_id->device->iwcm->reject(cm_id, NULL, 0);
+		cm_id->device->ops.iw_reject(cm_id, NULL, 0);
 		spin_lock_irqsave(&cm_id_priv->lock, flags);
 		break;
 	case IW_CM_STATE_CONN_SENT:
@@ -427,7 +427,7 @@ static void destroy_cm_id(struct iw_cm_id *cm_id)
 		break;
 	}
 	if (cm_id_priv->qp) {
-		cm_id_priv->id.device->iwcm->rem_ref(cm_id_priv->qp);
+		cm_id_priv->id.device->ops.iw_rem_ref(cm_id_priv->qp);
 		cm_id_priv->qp = NULL;
 	}
 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
@@ -504,7 +504,7 @@ static void iw_cm_check_wildcard(struct sockaddr_storage *pm_addr,
 static int iw_cm_map(struct iw_cm_id *cm_id, bool active)
 {
 	const char *devname = dev_name(&cm_id->device->dev);
-	const char *ifname = cm_id->device->iwcm->ifname;
+	const char *ifname = cm_id->device->iw_ifname;
 	struct iwpm_dev_data pm_reg_msg = {};
 	struct iwpm_sa_data pm_msg;
 	int status;
@@ -526,7 +526,7 @@ static int iw_cm_map(struct iw_cm_id *cm_id, bool active)
 	cm_id->mapped = true;
 	pm_msg.loc_addr = cm_id->local_addr;
 	pm_msg.rem_addr = cm_id->remote_addr;
-	pm_msg.flags = (cm_id->device->iwcm->driver_flags & IW_F_NO_PORT_MAP) ?
+	pm_msg.flags = (cm_id->device->iw_driver_flags & IW_F_NO_PORT_MAP) ?
 		       IWPM_FLAGS_NO_PORT_MAP : 0;
 	if (active)
 		status = iwpm_add_and_query_mapping(&pm_msg,
@@ -577,7 +577,8 @@ int iw_cm_listen(struct iw_cm_id *cm_id, int backlog)
 		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
 		ret = iw_cm_map(cm_id, false);
 		if (!ret)
-			ret = cm_id->device->iwcm->create_listen(cm_id, backlog);
+			ret = cm_id->device->ops.iw_create_listen(cm_id,
+								  backlog);
 		if (ret)
 			cm_id_priv->state = IW_CM_STATE_IDLE;
 		spin_lock_irqsave(&cm_id_priv->lock, flags);
@@ -617,7 +618,7 @@ int iw_cm_reject(struct iw_cm_id *cm_id,
 	cm_id_priv->state = IW_CM_STATE_IDLE;
 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
 
-	ret = cm_id->device->iwcm->reject(cm_id, private_data,
+	ret = cm_id->device->ops.iw_reject(cm_id, private_data,
 					  private_data_len);
 
 	clear_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags);
@@ -653,25 +654,25 @@ int iw_cm_accept(struct iw_cm_id *cm_id,
 		return -EINVAL;
 	}
 	/* Get the ib_qp given the QPN */
-	qp = cm_id->device->iwcm->get_qp(cm_id->device, iw_param->qpn);
+	qp = cm_id->device->ops.iw_get_qp(cm_id->device, iw_param->qpn);
 	if (!qp) {
 		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
 		clear_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags);
 		wake_up_all(&cm_id_priv->connect_wait);
 		return -EINVAL;
 	}
-	cm_id->device->iwcm->add_ref(qp);
+	cm_id->device->ops.iw_add_ref(qp);
 	cm_id_priv->qp = qp;
 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
 
-	ret = cm_id->device->iwcm->accept(cm_id, iw_param);
+	ret = cm_id->device->ops.iw_accept(cm_id, iw_param);
 	if (ret) {
 		/* An error on accept precludes provider events */
 		BUG_ON(cm_id_priv->state != IW_CM_STATE_CONN_RECV);
 		cm_id_priv->state = IW_CM_STATE_IDLE;
 		spin_lock_irqsave(&cm_id_priv->lock, flags);
 		if (cm_id_priv->qp) {
-			cm_id->device->iwcm->rem_ref(qp);
+			cm_id->device->ops.iw_rem_ref(qp);
 			cm_id_priv->qp = NULL;
 		}
 		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
@@ -712,25 +713,25 @@ int iw_cm_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *iw_param)
 	}
 
 	/* Get the ib_qp given the QPN */
-	qp = cm_id->device->iwcm->get_qp(cm_id->device, iw_param->qpn);
+	qp = cm_id->device->ops.iw_get_qp(cm_id->device, iw_param->qpn);
 	if (!qp) {
 		ret = -EINVAL;
 		goto err;
 	}
-	cm_id->device->iwcm->add_ref(qp);
+	cm_id->device->ops.iw_add_ref(qp);
 	cm_id_priv->qp = qp;
 	cm_id_priv->state = IW_CM_STATE_CONN_SENT;
 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
 
 	ret = iw_cm_map(cm_id, true);
 	if (!ret)
-		ret = cm_id->device->iwcm->connect(cm_id, iw_param);
+		ret = cm_id->device->ops.iw_connect(cm_id, iw_param);
 	if (!ret)
 		return 0;	/* success */
 
 	spin_lock_irqsave(&cm_id_priv->lock, flags);
 	if (cm_id_priv->qp) {
-		cm_id->device->iwcm->rem_ref(qp);
+		cm_id->device->ops.iw_rem_ref(qp);
 		cm_id_priv->qp = NULL;
 	}
 	cm_id_priv->state = IW_CM_STATE_IDLE;
@@ -895,7 +896,7 @@ static int cm_conn_rep_handler(struct iwcm_id_private *cm_id_priv,
 		cm_id_priv->state = IW_CM_STATE_ESTABLISHED;
 	} else {
 		/* REJECTED or RESET */
-		cm_id_priv->id.device->iwcm->rem_ref(cm_id_priv->qp);
+		cm_id_priv->id.device->ops.iw_rem_ref(cm_id_priv->qp);
 		cm_id_priv->qp = NULL;
 		cm_id_priv->state = IW_CM_STATE_IDLE;
 	}
@@ -946,7 +947,7 @@ static int cm_close_handler(struct iwcm_id_private *cm_id_priv,
 	spin_lock_irqsave(&cm_id_priv->lock, flags);
 
 	if (cm_id_priv->qp) {
-		cm_id_priv->id.device->iwcm->rem_ref(cm_id_priv->qp);
+		cm_id_priv->id.device->ops.iw_rem_ref(cm_id_priv->qp);
 		cm_id_priv->qp = NULL;
 	}
 	switch (cm_id_priv->state) {
diff --git a/drivers/infiniband/core/mad.c b/drivers/infiniband/core/mad.c
index e742a6a2c138..cc99479b2c09 100644
--- a/drivers/infiniband/core/mad.c
+++ b/drivers/infiniband/core/mad.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2005 Intel Corporation.  All rights reserved.
  * Copyright (c) 2005 Mellanox Technologies Ltd.  All rights reserved.
  * Copyright (c) 2009 HNR Consulting. All rights reserved.
- * Copyright (c) 2014 Intel Corporation.  All rights reserved.
+ * Copyright (c) 2014,2018 Intel Corporation.  All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -38,10 +38,10 @@
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
 #include <linux/dma-mapping.h>
-#include <linux/idr.h>
 #include <linux/slab.h>
 #include <linux/module.h>
 #include <linux/security.h>
+#include <linux/xarray.h>
 #include <rdma/ib_cache.h>
 
 #include "mad_priv.h"
@@ -51,6 +51,32 @@
 #include "opa_smi.h"
 #include "agent.h"
 
+#define CREATE_TRACE_POINTS
+#include <trace/events/ib_mad.h>
+
+#ifdef CONFIG_TRACEPOINTS
+static void create_mad_addr_info(struct ib_mad_send_wr_private *mad_send_wr,
+			  struct ib_mad_qp_info *qp_info,
+			  struct trace_event_raw_ib_mad_send_template *entry)
+{
+	u16 pkey;
+	struct ib_device *dev = qp_info->port_priv->device;
+	u8 pnum = qp_info->port_priv->port_num;
+	struct ib_ud_wr *wr = &mad_send_wr->send_wr;
+	struct rdma_ah_attr attr = {};
+
+	rdma_query_ah(wr->ah, &attr);
+
+	/* These are common */
+	entry->sl = attr.sl;
+	ib_query_pkey(dev, pnum, wr->pkey_index, &pkey);
+	entry->pkey = pkey;
+	entry->rqpn = wr->remote_qpn;
+	entry->rqkey = wr->remote_qkey;
+	entry->dlid = rdma_ah_get_dlid(&attr);
+}
+#endif
+
 static int mad_sendq_size = IB_MAD_QP_SEND_SIZE;
 static int mad_recvq_size = IB_MAD_QP_RECV_SIZE;
 
@@ -59,12 +85,9 @@ MODULE_PARM_DESC(send_queue_size, "Size of send queue in number of work requests
 module_param_named(recv_queue_size, mad_recvq_size, int, 0444);
 MODULE_PARM_DESC(recv_queue_size, "Size of receive queue in number of work requests");
 
-/*
- * The mlx4 driver uses the top byte to distinguish which virtual function
- * generated the MAD, so we must avoid using it.
- */
-#define AGENT_ID_LIMIT		(1 << 24)
-static DEFINE_IDR(ib_mad_clients);
+/* Client ID 0 is used for snoop-only clients */
+static DEFINE_XARRAY_ALLOC1(ib_mad_clients);
+static u32 ib_mad_client_next;
 static struct list_head ib_mad_port_list;
 
 /* Port list lock */
@@ -389,18 +412,17 @@ struct ib_mad_agent *ib_register_mad_agent(struct ib_device *device,
 		goto error4;
 	}
 
-	idr_preload(GFP_KERNEL);
-	idr_lock(&ib_mad_clients);
-	ret2 = idr_alloc_cyclic(&ib_mad_clients, mad_agent_priv, 0,
-			AGENT_ID_LIMIT, GFP_ATOMIC);
-	idr_unlock(&ib_mad_clients);
-	idr_preload_end();
-
+	/*
+	 * The mlx4 driver uses the top byte to distinguish which virtual
+	 * function generated the MAD, so we must avoid using it.
+	 */
+	ret2 = xa_alloc_cyclic(&ib_mad_clients, &mad_agent_priv->agent.hi_tid,
+			mad_agent_priv, XA_LIMIT(0, (1 << 24) - 1),
+			&ib_mad_client_next, GFP_KERNEL);
 	if (ret2 < 0) {
 		ret = ERR_PTR(ret2);
 		goto error5;
 	}
-	mad_agent_priv->agent.hi_tid = ret2;
 
 	/*
 	 * Make sure MAD registration (if supplied)
@@ -445,12 +467,11 @@ struct ib_mad_agent *ib_register_mad_agent(struct ib_device *device,
 	}
 	spin_unlock_irq(&port_priv->reg_lock);
 
+	trace_ib_mad_create_agent(mad_agent_priv);
 	return &mad_agent_priv->agent;
 error6:
 	spin_unlock_irq(&port_priv->reg_lock);
-	idr_lock(&ib_mad_clients);
-	idr_remove(&ib_mad_clients, mad_agent_priv->agent.hi_tid);
-	idr_unlock(&ib_mad_clients);
+	xa_erase(&ib_mad_clients, mad_agent_priv->agent.hi_tid);
 error5:
 	ib_mad_agent_security_cleanup(&mad_agent_priv->agent);
 error4:
@@ -602,6 +623,7 @@ static void unregister_mad_agent(struct ib_mad_agent_private *mad_agent_priv)
 	struct ib_mad_port_private *port_priv;
 
 	/* Note that we could still be handling received MADs */
+	trace_ib_mad_unregister_agent(mad_agent_priv);
 
 	/*
 	 * Canceling all sends results in dropping received response
@@ -614,9 +636,7 @@ static void unregister_mad_agent(struct ib_mad_agent_private *mad_agent_priv)
 	spin_lock_irq(&port_priv->reg_lock);
 	remove_mad_reg_req(mad_agent_priv);
 	spin_unlock_irq(&port_priv->reg_lock);
-	idr_lock(&ib_mad_clients);
-	idr_remove(&ib_mad_clients, mad_agent_priv->agent.hi_tid);
-	idr_unlock(&ib_mad_clients);
+	xa_erase(&ib_mad_clients, mad_agent_priv->agent.hi_tid);
 
 	flush_workqueue(port_priv->wq);
 	ib_cancel_rmpp_recvs(mad_agent_priv);
@@ -821,6 +841,8 @@ static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv,
 	if (opa && smp->class_version == OPA_SM_CLASS_VERSION) {
 		u32 opa_drslid;
 
+		trace_ib_mad_handle_out_opa_smi(opa_smp);
+
 		if ((opa_get_smp_direction(opa_smp)
 		     ? opa_smp->route.dr.dr_dlid : opa_smp->route.dr.dr_slid) ==
 		     OPA_LID_PERMISSIVE &&
@@ -846,6 +868,8 @@ static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv,
 		    opa_smi_check_local_returning_smp(opa_smp, device) == IB_SMI_DISCARD)
 			goto out;
 	} else {
+		trace_ib_mad_handle_out_ib_smi(smp);
+
 		if ((ib_get_smp_direction(smp) ? smp->dr_dlid : smp->dr_slid) ==
 		     IB_LID_PERMISSIVE &&
 		     smi_handle_dr_smp_send(smp, rdma_cap_ib_switch(device), port_num) ==
@@ -1223,6 +1247,7 @@ int ib_send_mad(struct ib_mad_send_wr_private *mad_send_wr)
 
 	spin_lock_irqsave(&qp_info->send_queue.lock, flags);
 	if (qp_info->send_queue.count < qp_info->send_queue.max_active) {
+		trace_ib_mad_ib_send_mad(mad_send_wr, qp_info);
 		ret = ib_post_send(mad_agent->qp, &mad_send_wr->send_wr.wr,
 				   NULL);
 		list = &qp_info->send_queue.list;
@@ -1756,7 +1781,7 @@ find_mad_agent(struct ib_mad_port_private *port_priv,
 		 */
 		hi_tid = be64_to_cpu(mad_hdr->tid) >> 32;
 		rcu_read_lock();
-		mad_agent = idr_find(&ib_mad_clients, hi_tid);
+		mad_agent = xa_load(&ib_mad_clients, hi_tid);
 		if (mad_agent && !atomic_inc_not_zero(&mad_agent->refcount))
 			mad_agent = NULL;
 		rcu_read_unlock();
@@ -2077,6 +2102,8 @@ static enum smi_action handle_ib_smi(const struct ib_mad_port_private *port_priv
 	enum smi_forward_action retsmi;
 	struct ib_smp *smp = (struct ib_smp *)recv->mad;
 
+	trace_ib_mad_handle_ib_smi(smp);
+
 	if (smi_handle_dr_smp_recv(smp,
 				   rdma_cap_ib_switch(port_priv->device),
 				   port_num,
@@ -2162,6 +2189,8 @@ handle_opa_smi(struct ib_mad_port_private *port_priv,
 	enum smi_forward_action retsmi;
 	struct opa_smp *smp = (struct opa_smp *)recv->mad;
 
+	trace_ib_mad_handle_opa_smi(smp);
+
 	if (opa_smi_handle_dr_smp_recv(smp,
 				   rdma_cap_ib_switch(port_priv->device),
 				   port_num,
@@ -2286,6 +2315,9 @@ static void ib_mad_recv_done(struct ib_cq *cq, struct ib_wc *wc)
 	if (!validate_mad((const struct ib_mad_hdr *)recv->mad, qp_info, opa))
 		goto out;
 
+	trace_ib_mad_recv_done_handler(qp_info, wc,
+				       (struct ib_mad_hdr *)recv->mad);
+
 	mad_size = recv->mad_size;
 	response = alloc_mad_private(mad_size, GFP_KERNEL);
 	if (!response)
@@ -2332,6 +2364,7 @@ static void ib_mad_recv_done(struct ib_cq *cq, struct ib_wc *wc)
 
 	mad_agent = find_mad_agent(port_priv, (const struct ib_mad_hdr *)recv->mad);
 	if (mad_agent) {
+		trace_ib_mad_recv_done_agent(mad_agent);
 		ib_mad_complete_recv(mad_agent, &recv->header.recv_wc);
 		/*
 		 * recv is freed up in error cases in ib_mad_complete_recv
@@ -2496,6 +2529,9 @@ static void ib_mad_send_done(struct ib_cq *cq, struct ib_wc *wc)
 	send_queue = mad_list->mad_queue;
 	qp_info = send_queue->qp_info;
 
+	trace_ib_mad_send_done_agent(mad_send_wr->mad_agent_priv);
+	trace_ib_mad_send_done_handler(mad_send_wr, wc);
+
 retry:
 	ib_dma_unmap_single(mad_send_wr->send_buf.mad_agent->device,
 			    mad_send_wr->header_mapping,
@@ -2527,6 +2563,7 @@ retry:
 	ib_mad_complete_send_wr(mad_send_wr, &mad_send_wc);
 
 	if (queued_send_wr) {
+		trace_ib_mad_send_done_resend(queued_send_wr, qp_info);
 		ret = ib_post_send(qp_info->qp, &queued_send_wr->send_wr.wr,
 				   NULL);
 		if (ret) {
@@ -2574,6 +2611,7 @@ static bool ib_mad_send_error(struct ib_mad_port_private *port_priv,
 		if (mad_send_wr->retry) {
 			/* Repost send */
 			mad_send_wr->retry = 0;
+			trace_ib_mad_error_handler(mad_send_wr, qp_info);
 			ret = ib_post_send(qp_info->qp, &mad_send_wr->send_wr.wr,
 					   NULL);
 			if (!ret)
@@ -3356,9 +3394,6 @@ int ib_mad_init(void)
 
 	INIT_LIST_HEAD(&ib_mad_port_list);
 
-	/* Client ID 0 is used for snoop-only clients */
-	idr_alloc(&ib_mad_clients, NULL, 0, 0, GFP_KERNEL);
-
 	if (ib_register_client(&mad_client)) {
 		pr_err("Couldn't register ib_mad client\n");
 		return -EINVAL;
diff --git a/drivers/infiniband/core/mad_priv.h b/drivers/infiniband/core/mad_priv.h
index 216509036aa8..956b3a7dfed7 100644
--- a/drivers/infiniband/core/mad_priv.h
+++ b/drivers/infiniband/core/mad_priv.h
@@ -73,14 +73,14 @@ struct ib_mad_private_header {
 	struct ib_mad_recv_wc recv_wc;
 	struct ib_wc wc;
 	u64 mapping;
-} __attribute__ ((packed));
+} __packed;
 
 struct ib_mad_private {
 	struct ib_mad_private_header header;
 	size_t mad_size;
 	struct ib_grh grh;
 	u8 mad[0];
-} __attribute__ ((packed));
+} __packed;
 
 struct ib_rmpp_segment {
 	struct list_head list;
diff --git a/drivers/infiniband/core/multicast.c b/drivers/infiniband/core/multicast.c
index d50ff70bb24b..cd338ddc4a39 100644
--- a/drivers/infiniband/core/multicast.c
+++ b/drivers/infiniband/core/multicast.c
@@ -804,7 +804,6 @@ static void mcast_event_handler(struct ib_event_handler *handler,
 	switch (event->event) {
 	case IB_EVENT_PORT_ERR:
 	case IB_EVENT_LID_CHANGE:
-	case IB_EVENT_SM_CHANGE:
 	case IB_EVENT_CLIENT_REREGISTER:
 		mcast_groups_event(&dev->port[index], MCAST_GROUP_ERROR);
 		break;
diff --git a/drivers/infiniband/core/nldev.c b/drivers/infiniband/core/nldev.c
index 85324012bf07..98eadd3089ce 100644
--- a/drivers/infiniband/core/nldev.c
+++ b/drivers/infiniband/core/nldev.c
@@ -116,6 +116,10 @@ static const struct nla_policy nldev_policy[RDMA_NLDEV_ATTR_MAX] = {
 	[RDMA_NLDEV_ATTR_RES_CTXN]              = { .type = NLA_U32 },
 	[RDMA_NLDEV_ATTR_LINK_TYPE]		= { .type = NLA_NUL_STRING,
 				    .len = RDMA_NLDEV_ATTR_ENTRY_STRLEN },
+	[RDMA_NLDEV_SYS_ATTR_NETNS_MODE]	= { .type = NLA_U8 },
+	[RDMA_NLDEV_ATTR_DEV_PROTOCOL]		= { .type = NLA_NUL_STRING,
+				    .len = RDMA_NLDEV_ATTR_ENTRY_STRLEN },
+	[RDMA_NLDEV_NET_NS_FD]			= { .type = NLA_U32 },
 };
 
 static int put_driver_name_print_type(struct sk_buff *msg, const char *name,
@@ -198,6 +202,8 @@ static int fill_nldev_handle(struct sk_buff *msg, struct ib_device *device)
 static int fill_dev_info(struct sk_buff *msg, struct ib_device *device)
 {
 	char fw[IB_FW_VERSION_NAME_MAX];
+	int ret = 0;
+	u8 port;
 
 	if (fill_nldev_handle(msg, device))
 		return -EMSGSIZE;
@@ -226,7 +232,25 @@ static int fill_dev_info(struct sk_buff *msg, struct ib_device *device)
 		return -EMSGSIZE;
 	if (nla_put_u8(msg, RDMA_NLDEV_ATTR_DEV_NODE_TYPE, device->node_type))
 		return -EMSGSIZE;
-	return 0;
+
+	/*
+	 * Link type is determined on first port and mlx4 device
+	 * which can potentially have two different link type for the same
+	 * IB device is considered as better to be avoided in the future,
+	 */
+	port = rdma_start_port(device);
+	if (rdma_cap_opa_mad(device, port))
+		ret = nla_put_string(msg, RDMA_NLDEV_ATTR_DEV_PROTOCOL, "opa");
+	else if (rdma_protocol_ib(device, port))
+		ret = nla_put_string(msg, RDMA_NLDEV_ATTR_DEV_PROTOCOL, "ib");
+	else if (rdma_protocol_iwarp(device, port))
+		ret = nla_put_string(msg, RDMA_NLDEV_ATTR_DEV_PROTOCOL, "iw");
+	else if (rdma_protocol_roce(device, port))
+		ret = nla_put_string(msg, RDMA_NLDEV_ATTR_DEV_PROTOCOL, "roce");
+	else if (rdma_protocol_usnic(device, port))
+		ret = nla_put_string(msg, RDMA_NLDEV_ATTR_DEV_PROTOCOL,
+				     "usnic");
+	return ret;
 }
 
 static int fill_port_info(struct sk_buff *msg,
@@ -615,7 +639,7 @@ static int nldev_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
 
 	index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
 
-	device = ib_device_get_by_index(index);
+	device = ib_device_get_by_index(sock_net(skb->sk), index);
 	if (!device)
 		return -EINVAL;
 
@@ -659,7 +683,7 @@ static int nldev_set_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
 		return -EINVAL;
 
 	index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
-	device = ib_device_get_by_index(index);
+	device = ib_device_get_by_index(sock_net(skb->sk), index);
 	if (!device)
 		return -EINVAL;
 
@@ -669,9 +693,20 @@ static int nldev_set_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
 		nla_strlcpy(name, tb[RDMA_NLDEV_ATTR_DEV_NAME],
 			    IB_DEVICE_NAME_MAX);
 		err = ib_device_rename(device, name);
+		goto done;
 	}
 
+	if (tb[RDMA_NLDEV_NET_NS_FD]) {
+		u32 ns_fd;
+
+		ns_fd = nla_get_u32(tb[RDMA_NLDEV_NET_NS_FD]);
+		err = ib_device_set_netns_put(skb, device, ns_fd);
+		goto put_done;
+	}
+
+done:
 	ib_device_put(device);
+put_done:
 	return err;
 }
 
@@ -707,7 +742,7 @@ static int nldev_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
 {
 	/*
 	 * There is no need to take lock, because
-	 * we are relying on ib_core's lists_rwsem
+	 * we are relying on ib_core's locking.
 	 */
 	return ib_enum_all_devs(_nldev_get_dumpit, skb, cb);
 }
@@ -730,7 +765,7 @@ static int nldev_port_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
 		return -EINVAL;
 
 	index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
-	device = ib_device_get_by_index(index);
+	device = ib_device_get_by_index(sock_net(skb->sk), index);
 	if (!device)
 		return -EINVAL;
 
@@ -784,7 +819,7 @@ static int nldev_port_get_dumpit(struct sk_buff *skb,
 		return -EINVAL;
 
 	ifindex = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
-	device = ib_device_get_by_index(ifindex);
+	device = ib_device_get_by_index(sock_net(skb->sk), ifindex);
 	if (!device)
 		return -EINVAL;
 
@@ -839,7 +874,7 @@ static int nldev_res_get_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
 		return -EINVAL;
 
 	index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
-	device = ib_device_get_by_index(index);
+	device = ib_device_get_by_index(sock_net(skb->sk), index);
 	if (!device)
 		return -EINVAL;
 
@@ -887,7 +922,6 @@ static int _nldev_res_get_dumpit(struct ib_device *device,
 		nlmsg_cancel(skb, nlh);
 		goto out;
 	}
-
 	nlmsg_end(skb, nlh);
 
 	idx++;
@@ -988,7 +1022,7 @@ static int res_get_common_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
 		return -EINVAL;
 
 	index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
-	device = ib_device_get_by_index(index);
+	device = ib_device_get_by_index(sock_net(skb->sk), index);
 	if (!device)
 		return -EINVAL;
 
@@ -1085,7 +1119,7 @@ static int res_get_common_dumpit(struct sk_buff *skb,
 		return -EINVAL;
 
 	index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
-	device = ib_device_get_by_index(index);
+	device = ib_device_get_by_index(sock_net(skb->sk), index);
 	if (!device)
 		return -EINVAL;
 
@@ -1300,7 +1334,7 @@ static int nldev_dellink(struct sk_buff *skb, struct nlmsghdr *nlh,
 		return -EINVAL;
 
 	index = nla_get_u32(tb[RDMA_NLDEV_ATTR_DEV_INDEX]);
-	device = ib_device_get_by_index(index);
+	device = ib_device_get_by_index(sock_net(skb->sk), index);
 	if (!device)
 		return -EINVAL;
 
@@ -1313,6 +1347,55 @@ static int nldev_dellink(struct sk_buff *skb, struct nlmsghdr *nlh,
 	return 0;
 }
 
+static int nldev_get_sys_get_dumpit(struct sk_buff *skb,
+				    struct netlink_callback *cb)
+{
+	struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
+	struct nlmsghdr *nlh;
+	int err;
+
+	err = nlmsg_parse(cb->nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
+			  nldev_policy, NULL);
+	if (err)
+		return err;
+
+	nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
+			RDMA_NL_GET_TYPE(RDMA_NL_NLDEV,
+					 RDMA_NLDEV_CMD_SYS_GET),
+			0, 0);
+
+	err = nla_put_u8(skb, RDMA_NLDEV_SYS_ATTR_NETNS_MODE,
+			 (u8)ib_devices_shared_netns);
+	if (err) {
+		nlmsg_cancel(skb, nlh);
+		return err;
+	}
+
+	nlmsg_end(skb, nlh);
+	return skb->len;
+}
+
+static int nldev_set_sys_set_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
+				  struct netlink_ext_ack *extack)
+{
+	struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
+	u8 enable;
+	int err;
+
+	err = nlmsg_parse(nlh, 0, tb, RDMA_NLDEV_ATTR_MAX - 1,
+			  nldev_policy, extack);
+	if (err || !tb[RDMA_NLDEV_SYS_ATTR_NETNS_MODE])
+		return -EINVAL;
+
+	enable = nla_get_u8(tb[RDMA_NLDEV_SYS_ATTR_NETNS_MODE]);
+	/* Only 0 and 1 are supported */
+	if (enable > 1)
+		return -EINVAL;
+
+	err = rdma_compatdev_set(enable);
+	return err;
+}
+
 static const struct rdma_nl_cbs nldev_cb_table[RDMA_NLDEV_NUM_OPS] = {
 	[RDMA_NLDEV_CMD_GET] = {
 		.doit = nldev_get_doit,
@@ -1358,6 +1441,13 @@ static const struct rdma_nl_cbs nldev_cb_table[RDMA_NLDEV_NUM_OPS] = {
 		.doit = nldev_res_get_pd_doit,
 		.dump = nldev_res_get_pd_dumpit,
 	},
+	[RDMA_NLDEV_CMD_SYS_GET] = {
+		.dump = nldev_get_sys_get_dumpit,
+	},
+	[RDMA_NLDEV_CMD_SYS_SET] = {
+		.doit = nldev_set_sys_set_doit,
+		.flags = RDMA_NL_ADMIN_PERM,
+	},
 };
 
 void __init nldev_init(void)
diff --git a/drivers/infiniband/core/rdma_core.c b/drivers/infiniband/core/rdma_core.c
index 778375ff664e..ccf4d069c25c 100644
--- a/drivers/infiniband/core/rdma_core.c
+++ b/drivers/infiniband/core/rdma_core.c
@@ -125,9 +125,10 @@ static void assert_uverbs_usecnt(struct ib_uobject *uobj,
  * and consumes the kref on the uobj.
  */
 static int uverbs_destroy_uobject(struct ib_uobject *uobj,
-				  enum rdma_remove_reason reason)
+				  enum rdma_remove_reason reason,
+				  struct uverbs_attr_bundle *attrs)
 {
-	struct ib_uverbs_file *ufile = uobj->ufile;
+	struct ib_uverbs_file *ufile = attrs->ufile;
 	unsigned long flags;
 	int ret;
 
@@ -135,7 +136,8 @@ static int uverbs_destroy_uobject(struct ib_uobject *uobj,
 	assert_uverbs_usecnt(uobj, UVERBS_LOOKUP_WRITE);
 
 	if (uobj->object) {
-		ret = uobj->uapi_object->type_class->destroy_hw(uobj, reason);
+		ret = uobj->uapi_object->type_class->destroy_hw(uobj, reason,
+								attrs);
 		if (ret) {
 			if (ib_is_destroy_retryable(ret, reason, uobj))
 				return ret;
@@ -196,9 +198,9 @@ static int uverbs_destroy_uobject(struct ib_uobject *uobj,
  * version requires the caller to have already obtained an
  * LOOKUP_DESTROY uobject kref.
  */
-int uobj_destroy(struct ib_uobject *uobj)
+int uobj_destroy(struct ib_uobject *uobj, struct uverbs_attr_bundle *attrs)
 {
-	struct ib_uverbs_file *ufile = uobj->ufile;
+	struct ib_uverbs_file *ufile = attrs->ufile;
 	int ret;
 
 	down_read(&ufile->hw_destroy_rwsem);
@@ -207,7 +209,7 @@ int uobj_destroy(struct ib_uobject *uobj)
 	if (ret)
 		goto out_unlock;
 
-	ret = uverbs_destroy_uobject(uobj, RDMA_REMOVE_DESTROY);
+	ret = uverbs_destroy_uobject(uobj, RDMA_REMOVE_DESTROY, attrs);
 	if (ret) {
 		atomic_set(&uobj->usecnt, 0);
 		goto out_unlock;
@@ -224,18 +226,17 @@ out_unlock:
  * uverbs_put_destroy.
  */
 struct ib_uobject *__uobj_get_destroy(const struct uverbs_api_object *obj,
-				      u32 id,
-				      const struct uverbs_attr_bundle *attrs)
+				      u32 id, struct uverbs_attr_bundle *attrs)
 {
 	struct ib_uobject *uobj;
 	int ret;
 
 	uobj = rdma_lookup_get_uobject(obj, attrs->ufile, id,
-				       UVERBS_LOOKUP_DESTROY);
+				       UVERBS_LOOKUP_DESTROY, attrs);
 	if (IS_ERR(uobj))
 		return uobj;
 
-	ret = uobj_destroy(uobj);
+	ret = uobj_destroy(uobj, attrs);
 	if (ret) {
 		rdma_lookup_put_uobject(uobj, UVERBS_LOOKUP_DESTROY);
 		return ERR_PTR(ret);
@@ -249,7 +250,7 @@ struct ib_uobject *__uobj_get_destroy(const struct uverbs_api_object *obj,
  * (negative errno on failure). For use by callers that do not need the uobj.
  */
 int __uobj_perform_destroy(const struct uverbs_api_object *obj, u32 id,
-			   const struct uverbs_attr_bundle *attrs)
+			   struct uverbs_attr_bundle *attrs)
 {
 	struct ib_uobject *uobj;
 
@@ -296,25 +297,13 @@ static struct ib_uobject *alloc_uobj(struct ib_uverbs_file *ufile,
 
 static int idr_add_uobj(struct ib_uobject *uobj)
 {
-	int ret;
-
-	idr_preload(GFP_KERNEL);
-	spin_lock(&uobj->ufile->idr_lock);
-
-	/*
-	 * We start with allocating an idr pointing to NULL. This represents an
-	 * object which isn't initialized yet. We'll replace it later on with
-	 * the real object once we commit.
-	 */
-	ret = idr_alloc(&uobj->ufile->idr, NULL, 0,
-			min_t(unsigned long, U32_MAX - 1, INT_MAX), GFP_NOWAIT);
-	if (ret >= 0)
-		uobj->id = ret;
-
-	spin_unlock(&uobj->ufile->idr_lock);
-	idr_preload_end();
-
-	return ret < 0 ? ret : 0;
+       /*
+        * We start with allocating an idr pointing to NULL. This represents an
+        * object which isn't initialized yet. We'll replace it later on with
+        * the real object once we commit.
+        */
+	return xa_alloc(&uobj->ufile->idr, &uobj->id, NULL, xa_limit_32b,
+			GFP_KERNEL);
 }
 
 /* Returns the ib_uobject or an error. The caller should check for IS_ERR. */
@@ -324,29 +313,20 @@ lookup_get_idr_uobject(const struct uverbs_api_object *obj,
 		       enum rdma_lookup_mode mode)
 {
 	struct ib_uobject *uobj;
-	unsigned long idrno = id;
 
 	if (id < 0 || id > ULONG_MAX)
 		return ERR_PTR(-EINVAL);
 
 	rcu_read_lock();
-	/* object won't be released as we're protected in rcu */
-	uobj = idr_find(&ufile->idr, idrno);
-	if (!uobj) {
-		uobj = ERR_PTR(-ENOENT);
-		goto free;
-	}
-
 	/*
 	 * The idr_find is guaranteed to return a pointer to something that
 	 * isn't freed yet, or NULL, as the free after idr_remove goes through
 	 * kfree_rcu(). However the object may still have been released and
 	 * kfree() could be called at any time.
 	 */
-	if (!kref_get_unless_zero(&uobj->ref))
+	uobj = xa_load(&ufile->idr, id);
+	if (!uobj || !kref_get_unless_zero(&uobj->ref))
 		uobj = ERR_PTR(-ENOENT);
-
-free:
 	rcu_read_unlock();
 	return uobj;
 }
@@ -393,12 +373,13 @@ lookup_get_fd_uobject(const struct uverbs_api_object *obj,
 
 struct ib_uobject *rdma_lookup_get_uobject(const struct uverbs_api_object *obj,
 					   struct ib_uverbs_file *ufile, s64 id,
-					   enum rdma_lookup_mode mode)
+					   enum rdma_lookup_mode mode,
+					   struct uverbs_attr_bundle *attrs)
 {
 	struct ib_uobject *uobj;
 	int ret;
 
-	if (IS_ERR(obj) && PTR_ERR(obj) == -ENOMSG) {
+	if (obj == ERR_PTR(-ENOMSG)) {
 		/* must be UVERBS_IDR_ANY_OBJECT, see uapi_get_object() */
 		uobj = lookup_get_idr_uobject(NULL, ufile, id, mode);
 		if (IS_ERR(uobj))
@@ -431,6 +412,8 @@ struct ib_uobject *rdma_lookup_get_uobject(const struct uverbs_api_object *obj,
 	ret = uverbs_try_lock_object(uobj, mode);
 	if (ret)
 		goto free;
+	if (attrs)
+		attrs->context = uobj->context;
 
 	return uobj;
 free:
@@ -438,38 +421,6 @@ free:
 	uverbs_uobject_put(uobj);
 	return ERR_PTR(ret);
 }
-struct ib_uobject *_uobj_get_read(enum uverbs_default_objects type,
-				  u32 object_id,
-				  struct uverbs_attr_bundle *attrs)
-{
-	struct ib_uobject *uobj;
-
-	uobj = rdma_lookup_get_uobject(uobj_get_type(attrs, type), attrs->ufile,
-				       object_id, UVERBS_LOOKUP_READ);
-	if (IS_ERR(uobj))
-		return uobj;
-
-	attrs->context = uobj->context;
-
-	return uobj;
-}
-
-struct ib_uobject *_uobj_get_write(enum uverbs_default_objects type,
-				   u32 object_id,
-				   struct uverbs_attr_bundle *attrs)
-{
-	struct ib_uobject *uobj;
-
-	uobj = rdma_lookup_get_uobject(uobj_get_type(attrs, type), attrs->ufile,
-				       object_id, UVERBS_LOOKUP_WRITE);
-
-	if (IS_ERR(uobj))
-		return uobj;
-
-	attrs->context = uobj->context;
-
-	return uobj;
-}
 
 static struct ib_uobject *
 alloc_begin_idr_uobject(const struct uverbs_api_object *obj,
@@ -489,14 +440,12 @@ alloc_begin_idr_uobject(const struct uverbs_api_object *obj,
 	ret = ib_rdmacg_try_charge(&uobj->cg_obj, uobj->context->device,
 				   RDMACG_RESOURCE_HCA_OBJECT);
 	if (ret)
-		goto idr_remove;
+		goto remove;
 
 	return uobj;
 
-idr_remove:
-	spin_lock(&ufile->idr_lock);
-	idr_remove(&ufile->idr, uobj->id);
-	spin_unlock(&ufile->idr_lock);
+remove:
+	xa_erase(&ufile->idr, uobj->id);
 uobj_put:
 	uverbs_uobject_put(uobj);
 	return ERR_PTR(ret);
@@ -526,7 +475,8 @@ alloc_begin_fd_uobject(const struct uverbs_api_object *obj,
 }
 
 struct ib_uobject *rdma_alloc_begin_uobject(const struct uverbs_api_object *obj,
-					    struct ib_uverbs_file *ufile)
+					    struct ib_uverbs_file *ufile,
+					    struct uverbs_attr_bundle *attrs)
 {
 	struct ib_uobject *ret;
 
@@ -546,6 +496,8 @@ struct ib_uobject *rdma_alloc_begin_uobject(const struct uverbs_api_object *obj,
 		up_read(&ufile->hw_destroy_rwsem);
 		return ret;
 	}
+	if (attrs)
+		attrs->context = ret->context;
 	return ret;
 }
 
@@ -554,18 +506,17 @@ static void alloc_abort_idr_uobject(struct ib_uobject *uobj)
 	ib_rdmacg_uncharge(&uobj->cg_obj, uobj->context->device,
 			   RDMACG_RESOURCE_HCA_OBJECT);
 
-	spin_lock(&uobj->ufile->idr_lock);
-	idr_remove(&uobj->ufile->idr, uobj->id);
-	spin_unlock(&uobj->ufile->idr_lock);
+	xa_erase(&uobj->ufile->idr, uobj->id);
 }
 
 static int __must_check destroy_hw_idr_uobject(struct ib_uobject *uobj,
-					       enum rdma_remove_reason why)
+					       enum rdma_remove_reason why,
+					       struct uverbs_attr_bundle *attrs)
 {
 	const struct uverbs_obj_idr_type *idr_type =
 		container_of(uobj->uapi_object->type_attrs,
 			     struct uverbs_obj_idr_type, type);
-	int ret = idr_type->destroy_object(uobj, why);
+	int ret = idr_type->destroy_object(uobj, why, attrs);
 
 	/*
 	 * We can only fail gracefully if the user requested to destroy the
@@ -586,9 +537,7 @@ static int __must_check destroy_hw_idr_uobject(struct ib_uobject *uobj,
 
 static void remove_handle_idr_uobject(struct ib_uobject *uobj)
 {
-	spin_lock(&uobj->ufile->idr_lock);
-	idr_remove(&uobj->ufile->idr, uobj->id);
-	spin_unlock(&uobj->ufile->idr_lock);
+	xa_erase(&uobj->ufile->idr, uobj->id);
 	/* Matches the kref in alloc_commit_idr_uobject */
 	uverbs_uobject_put(uobj);
 }
@@ -599,7 +548,8 @@ static void alloc_abort_fd_uobject(struct ib_uobject *uobj)
 }
 
 static int __must_check destroy_hw_fd_uobject(struct ib_uobject *uobj,
-					      enum rdma_remove_reason why)
+					      enum rdma_remove_reason why,
+					      struct uverbs_attr_bundle *attrs)
 {
 	const struct uverbs_obj_fd_type *fd_type = container_of(
 		uobj->uapi_object->type_attrs, struct uverbs_obj_fd_type, type);
@@ -618,17 +568,17 @@ static void remove_handle_fd_uobject(struct ib_uobject *uobj)
 static int alloc_commit_idr_uobject(struct ib_uobject *uobj)
 {
 	struct ib_uverbs_file *ufile = uobj->ufile;
+	void *old;
 
-	spin_lock(&ufile->idr_lock);
 	/*
 	 * We already allocated this IDR with a NULL object, so
 	 * this shouldn't fail.
 	 *
-	 * NOTE: Once we set the IDR we loose ownership of our kref on uobj.
+	 * NOTE: Storing the uobj transfers our kref on uobj to the XArray.
 	 * It will be put by remove_commit_idr_uobject()
 	 */
-	WARN_ON(idr_replace(&ufile->idr, uobj, uobj->id));
-	spin_unlock(&ufile->idr_lock);
+	old = xa_store(&ufile->idr, uobj->id, uobj, GFP_KERNEL);
+	WARN_ON(old != NULL);
 
 	return 0;
 }
@@ -675,15 +625,16 @@ static int alloc_commit_fd_uobject(struct ib_uobject *uobj)
  * caller can no longer assume uobj is valid. If this function fails it
  * destroys the uboject, including the attached HW object.
  */
-int __must_check rdma_alloc_commit_uobject(struct ib_uobject *uobj)
+int __must_check rdma_alloc_commit_uobject(struct ib_uobject *uobj,
+					   struct uverbs_attr_bundle *attrs)
 {
-	struct ib_uverbs_file *ufile = uobj->ufile;
+	struct ib_uverbs_file *ufile = attrs->ufile;
 	int ret;
 
 	/* alloc_commit consumes the uobj kref */
 	ret = uobj->uapi_object->type_class->alloc_commit(uobj);
 	if (ret) {
-		uverbs_destroy_uobject(uobj, RDMA_REMOVE_ABORT);
+		uverbs_destroy_uobject(uobj, RDMA_REMOVE_ABORT, attrs);
 		up_read(&ufile->hw_destroy_rwsem);
 		return ret;
 	}
@@ -707,12 +658,13 @@ int __must_check rdma_alloc_commit_uobject(struct ib_uobject *uobj)
  * This consumes the kref for uobj. It is up to the caller to unwind the HW
  * object and anything else connected to uobj before calling this.
  */
-void rdma_alloc_abort_uobject(struct ib_uobject *uobj)
+void rdma_alloc_abort_uobject(struct ib_uobject *uobj,
+			      struct uverbs_attr_bundle *attrs)
 {
 	struct ib_uverbs_file *ufile = uobj->ufile;
 
 	uobj->object = NULL;
-	uverbs_destroy_uobject(uobj, RDMA_REMOVE_ABORT);
+	uverbs_destroy_uobject(uobj, RDMA_REMOVE_ABORT, attrs);
 
 	/* Matches the down_read in rdma_alloc_begin_uobject */
 	up_read(&ufile->hw_destroy_rwsem);
@@ -760,29 +712,28 @@ void rdma_lookup_put_uobject(struct ib_uobject *uobj,
 
 void setup_ufile_idr_uobject(struct ib_uverbs_file *ufile)
 {
-	spin_lock_init(&ufile->idr_lock);
-	idr_init(&ufile->idr);
+	xa_init_flags(&ufile->idr, XA_FLAGS_ALLOC);
 }
 
 void release_ufile_idr_uobject(struct ib_uverbs_file *ufile)
 {
 	struct ib_uobject *entry;
-	int id;
+	unsigned long id;
 
 	/*
 	 * At this point uverbs_cleanup_ufile() is guaranteed to have run, and
-	 * there are no HW objects left, however the IDR is still populated
+	 * there are no HW objects left, however the xarray is still populated
 	 * with anything that has not been cleaned up by userspace. Since the
 	 * kref on ufile is 0, nothing is allowed to call lookup_get.
 	 *
 	 * This is an optimized equivalent to remove_handle_idr_uobject
 	 */
-	idr_for_each_entry(&ufile->idr, entry, id) {
+	xa_for_each(&ufile->idr, id, entry) {
 		WARN_ON(entry->object);
 		uverbs_uobject_put(entry);
 	}
 
-	idr_destroy(&ufile->idr);
+	xa_destroy(&ufile->idr);
 }
 
 const struct uverbs_obj_type_class uverbs_idr_class = {
@@ -814,6 +765,10 @@ void uverbs_close_fd(struct file *f)
 {
 	struct ib_uobject *uobj = f->private_data;
 	struct ib_uverbs_file *ufile = uobj->ufile;
+	struct uverbs_attr_bundle attrs = {
+		.context = uobj->context,
+		.ufile = ufile,
+	};
 
 	if (down_read_trylock(&ufile->hw_destroy_rwsem)) {
 		/*
@@ -823,7 +778,7 @@ void uverbs_close_fd(struct file *f)
 		 * write lock here, or we have a kernel bug.
 		 */
 		WARN_ON(uverbs_try_lock_object(uobj, UVERBS_LOOKUP_WRITE));
-		uverbs_destroy_uobject(uobj, RDMA_REMOVE_CLOSE);
+		uverbs_destroy_uobject(uobj, RDMA_REMOVE_CLOSE, &attrs);
 		up_read(&ufile->hw_destroy_rwsem);
 	}
 
@@ -872,6 +827,7 @@ static int __uverbs_cleanup_ufile(struct ib_uverbs_file *ufile,
 {
 	struct ib_uobject *obj, *next_obj;
 	int ret = -EINVAL;
+	struct uverbs_attr_bundle attrs = { .ufile = ufile };
 
 	/*
 	 * This shouldn't run while executing other commands on this
@@ -883,12 +839,13 @@ static int __uverbs_cleanup_ufile(struct ib_uverbs_file *ufile,
 	 * other threads (which might still use the FDs) chance to run.
 	 */
 	list_for_each_entry_safe(obj, next_obj, &ufile->uobjects, list) {
+		attrs.context = obj->context;
 		/*
 		 * if we hit this WARN_ON, that means we are
 		 * racing with a lookup_get.
 		 */
 		WARN_ON(uverbs_try_lock_object(obj, UVERBS_LOOKUP_WRITE));
-		if (!uverbs_destroy_uobject(obj, reason))
+		if (!uverbs_destroy_uobject(obj, reason, &attrs))
 			ret = 0;
 		else
 			atomic_set(&obj->usecnt, 0);
@@ -967,26 +924,25 @@ const struct uverbs_obj_type_class uverbs_fd_class = {
 EXPORT_SYMBOL(uverbs_fd_class);
 
 struct ib_uobject *
-uverbs_get_uobject_from_file(u16 object_id,
-			     struct ib_uverbs_file *ufile,
-			     enum uverbs_obj_access access, s64 id)
+uverbs_get_uobject_from_file(u16 object_id, enum uverbs_obj_access access,
+			     s64 id, struct uverbs_attr_bundle *attrs)
 {
 	const struct uverbs_api_object *obj =
-		uapi_get_object(ufile->device->uapi, object_id);
+		uapi_get_object(attrs->ufile->device->uapi, object_id);
 
 	switch (access) {
 	case UVERBS_ACCESS_READ:
-		return rdma_lookup_get_uobject(obj, ufile, id,
-					       UVERBS_LOOKUP_READ);
+		return rdma_lookup_get_uobject(obj, attrs->ufile, id,
+					       UVERBS_LOOKUP_READ, attrs);
 	case UVERBS_ACCESS_DESTROY:
 		/* Actual destruction is done inside uverbs_handle_method */
-		return rdma_lookup_get_uobject(obj, ufile, id,
-					       UVERBS_LOOKUP_DESTROY);
+		return rdma_lookup_get_uobject(obj, attrs->ufile, id,
+					       UVERBS_LOOKUP_DESTROY, attrs);
 	case UVERBS_ACCESS_WRITE:
-		return rdma_lookup_get_uobject(obj, ufile, id,
-					       UVERBS_LOOKUP_WRITE);
+		return rdma_lookup_get_uobject(obj, attrs->ufile, id,
+					       UVERBS_LOOKUP_WRITE, attrs);
 	case UVERBS_ACCESS_NEW:
-		return rdma_alloc_begin_uobject(obj, ufile);
+		return rdma_alloc_begin_uobject(obj, attrs->ufile, attrs);
 	default:
 		WARN_ON(true);
 		return ERR_PTR(-EOPNOTSUPP);
@@ -994,8 +950,8 @@ uverbs_get_uobject_from_file(u16 object_id,
 }
 
 int uverbs_finalize_object(struct ib_uobject *uobj,
-			   enum uverbs_obj_access access,
-			   bool commit)
+			   enum uverbs_obj_access access, bool commit,
+			   struct uverbs_attr_bundle *attrs)
 {
 	int ret = 0;
 
@@ -1018,9 +974,9 @@ int uverbs_finalize_object(struct ib_uobject *uobj,
 		break;
 	case UVERBS_ACCESS_NEW:
 		if (commit)
-			ret = rdma_alloc_commit_uobject(uobj);
+			ret = rdma_alloc_commit_uobject(uobj, attrs);
 		else
-			rdma_alloc_abort_uobject(uobj);
+			rdma_alloc_abort_uobject(uobj, attrs);
 		break;
 	default:
 		WARN_ON(true);
diff --git a/drivers/infiniband/core/rdma_core.h b/drivers/infiniband/core/rdma_core.h
index 69f8db66925e..5445323629b5 100644
--- a/drivers/infiniband/core/rdma_core.h
+++ b/drivers/infiniband/core/rdma_core.h
@@ -48,7 +48,7 @@ struct ib_uverbs_device;
 void uverbs_destroy_ufile_hw(struct ib_uverbs_file *ufile,
 			     enum rdma_remove_reason reason);
 
-int uobj_destroy(struct ib_uobject *uobj);
+int uobj_destroy(struct ib_uobject *uobj, struct uverbs_attr_bundle *attrs);
 
 /*
  * uverbs_uobject_get is called in order to increase the reference count on
@@ -83,9 +83,8 @@ void uverbs_close_fd(struct file *f);
  * uverbs_finalize_objects are called.
  */
 struct ib_uobject *
-uverbs_get_uobject_from_file(u16 object_id,
-			     struct ib_uverbs_file *ufile,
-			     enum uverbs_obj_access access, s64 id);
+uverbs_get_uobject_from_file(u16 object_id, enum uverbs_obj_access access,
+			     s64 id, struct uverbs_attr_bundle *attrs);
 
 /*
  * Note that certain finalize stages could return a status:
@@ -103,8 +102,8 @@ uverbs_get_uobject_from_file(u16 object_id,
  * object.
  */
 int uverbs_finalize_object(struct ib_uobject *uobj,
-			   enum uverbs_obj_access access,
-			   bool commit);
+			   enum uverbs_obj_access access, bool commit,
+			   struct uverbs_attr_bundle *attrs);
 
 int uverbs_output_written(const struct uverbs_attr_bundle *bundle, size_t idx);
 
diff --git a/drivers/infiniband/core/sa_query.c b/drivers/infiniband/core/sa_query.c
index bb534959abf0..7d8071c7e564 100644
--- a/drivers/infiniband/core/sa_query.c
+++ b/drivers/infiniband/core/sa_query.c
@@ -40,7 +40,7 @@
 #include <linux/slab.h>
 #include <linux/dma-mapping.h>
 #include <linux/kref.h>
-#include <linux/idr.h>
+#include <linux/xarray.h>
 #include <linux/workqueue.h>
 #include <uapi/linux/if_ether.h>
 #include <rdma/ib_pack.h>
@@ -183,8 +183,7 @@ static struct ib_client sa_client = {
 	.remove = ib_sa_remove_one
 };
 
-static DEFINE_SPINLOCK(idr_lock);
-static DEFINE_IDR(query_idr);
+static DEFINE_XARRAY_FLAGS(queries, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_IRQ);
 
 static DEFINE_SPINLOCK(tid_lock);
 static u32 tid;
@@ -1180,14 +1179,14 @@ void ib_sa_cancel_query(int id, struct ib_sa_query *query)
 	struct ib_mad_agent *agent;
 	struct ib_mad_send_buf *mad_buf;
 
-	spin_lock_irqsave(&idr_lock, flags);
-	if (idr_find(&query_idr, id) != query) {
-		spin_unlock_irqrestore(&idr_lock, flags);
+	xa_lock_irqsave(&queries, flags);
+	if (xa_load(&queries, id) != query) {
+		xa_unlock_irqrestore(&queries, flags);
 		return;
 	}
 	agent = query->port->agent;
 	mad_buf = query->mad_buf;
-	spin_unlock_irqrestore(&idr_lock, flags);
+	xa_unlock_irqrestore(&queries, flags);
 
 	/*
 	 * If the query is still on the netlink request list, schedule
@@ -1363,21 +1362,14 @@ static void init_mad(struct ib_sa_query *query, struct ib_mad_agent *agent)
 static int send_mad(struct ib_sa_query *query, unsigned long timeout_ms,
 		    gfp_t gfp_mask)
 {
-	bool preload = gfpflags_allow_blocking(gfp_mask);
 	unsigned long flags;
 	int ret, id;
 
-	if (preload)
-		idr_preload(gfp_mask);
-	spin_lock_irqsave(&idr_lock, flags);
-
-	id = idr_alloc(&query_idr, query, 0, 0, GFP_NOWAIT);
-
-	spin_unlock_irqrestore(&idr_lock, flags);
-	if (preload)
-		idr_preload_end();
-	if (id < 0)
-		return id;
+	xa_lock_irqsave(&queries, flags);
+	ret = __xa_alloc(&queries, &id, query, xa_limit_32b, gfp_mask);
+	xa_unlock_irqrestore(&queries, flags);
+	if (ret < 0)
+		return ret;
 
 	query->mad_buf->timeout_ms  = timeout_ms;
 	query->mad_buf->context[0] = query;
@@ -1394,9 +1386,9 @@ static int send_mad(struct ib_sa_query *query, unsigned long timeout_ms,
 
 	ret = ib_post_send_mad(query->mad_buf, NULL);
 	if (ret) {
-		spin_lock_irqsave(&idr_lock, flags);
-		idr_remove(&query_idr, id);
-		spin_unlock_irqrestore(&idr_lock, flags);
+		xa_lock_irqsave(&queries, flags);
+		__xa_erase(&queries, id);
+		xa_unlock_irqrestore(&queries, flags);
 	}
 
 	/*
@@ -2188,9 +2180,9 @@ static void send_handler(struct ib_mad_agent *agent,
 			break;
 		}
 
-	spin_lock_irqsave(&idr_lock, flags);
-	idr_remove(&query_idr, query->id);
-	spin_unlock_irqrestore(&idr_lock, flags);
+	xa_lock_irqsave(&queries, flags);
+	__xa_erase(&queries, query->id);
+	xa_unlock_irqrestore(&queries, flags);
 
 	free_mad(query);
 	if (query->client)
@@ -2475,5 +2467,5 @@ void ib_sa_cleanup(void)
 	destroy_workqueue(ib_nl_wq);
 	mcast_cleanup();
 	ib_unregister_client(&sa_client);
-	idr_destroy(&query_idr);
+	WARN_ON(!xa_empty(&queries));
 }
diff --git a/drivers/infiniband/core/sysfs.c b/drivers/infiniband/core/sysfs.c
index 9b6a065bdfa5..c78d0c9646ae 100644
--- a/drivers/infiniband/core/sysfs.c
+++ b/drivers/infiniband/core/sysfs.c
@@ -349,10 +349,15 @@ static struct attribute *port_default_attrs[] = {
 
 static size_t print_ndev(const struct ib_gid_attr *gid_attr, char *buf)
 {
-	if (!gid_attr->ndev)
-		return -EINVAL;
-
-	return sprintf(buf, "%s\n", gid_attr->ndev->name);
+	struct net_device *ndev;
+	size_t ret = -EINVAL;
+
+	rcu_read_lock();
+	ndev = rcu_dereference(gid_attr->ndev);
+	if (ndev)
+		ret = sprintf(buf, "%s\n", ndev->name);
+	rcu_read_unlock();
+	return ret;
 }
 
 static size_t print_gid_type(const struct ib_gid_attr *gid_attr, char *buf)
@@ -1015,8 +1020,10 @@ err_free_stats:
 	return;
 }
 
-static int add_port(struct ib_device *device, int port_num)
+static int add_port(struct ib_core_device *coredev, int port_num)
 {
+	struct ib_device *device = rdma_device_to_ibdev(&coredev->dev);
+	bool is_full_dev = &device->coredev == coredev;
 	struct ib_port *p;
 	struct ib_port_attr attr;
 	int i;
@@ -1034,7 +1041,7 @@ static int add_port(struct ib_device *device, int port_num)
 	p->port_num   = port_num;
 
 	ret = kobject_init_and_add(&p->kobj, &port_type,
-				   device->ports_kobj,
+				   coredev->ports_kobj,
 				   "%d", port_num);
 	if (ret) {
 		kfree(p);
@@ -1055,7 +1062,7 @@ static int add_port(struct ib_device *device, int port_num)
 		goto err_put;
 	}
 
-	if (device->ops.process_mad) {
+	if (device->ops.process_mad && is_full_dev) {
 		p->pma_table = get_counter_table(device, port_num);
 		ret = sysfs_create_group(&p->kobj, p->pma_table);
 		if (ret)
@@ -1111,7 +1118,7 @@ static int add_port(struct ib_device *device, int port_num)
 	if (ret)
 		goto err_free_pkey;
 
-	if (device->ops.init_port) {
+	if (device->ops.init_port && is_full_dev) {
 		ret = device->ops.init_port(device, port_num, &p->kobj);
 		if (ret)
 			goto err_remove_pkey;
@@ -1122,10 +1129,10 @@ static int add_port(struct ib_device *device, int port_num)
 	 * port, so holder should be device. Therefore skip per port conunter
 	 * initialization.
 	 */
-	if (device->ops.alloc_hw_stats && port_num)
+	if (device->ops.alloc_hw_stats && port_num && is_full_dev)
 		setup_hw_stats(device, p, port_num);
 
-	list_add_tail(&p->kobj.entry, &device->port_list);
+	list_add_tail(&p->kobj.entry, &coredev->port_list);
 
 	kobject_uevent(&p->kobj, KOBJ_ADD);
 	return 0;
@@ -1194,6 +1201,7 @@ static ssize_t node_type_show(struct device *device,
 	case RDMA_NODE_RNIC:	  return sprintf(buf, "%d: RNIC\n", dev->node_type);
 	case RDMA_NODE_USNIC:	  return sprintf(buf, "%d: usNIC\n", dev->node_type);
 	case RDMA_NODE_USNIC_UDP: return sprintf(buf, "%d: usNIC UDP\n", dev->node_type);
+	case RDMA_NODE_UNSPECIFIED: return sprintf(buf, "%d: unspecified\n", dev->node_type);
 	case RDMA_NODE_IB_SWITCH: return sprintf(buf, "%d: switch\n", dev->node_type);
 	case RDMA_NODE_IB_ROUTER: return sprintf(buf, "%d: router\n", dev->node_type);
 	default:		  return sprintf(buf, "%d: <unknown>\n", dev->node_type);
@@ -1279,11 +1287,11 @@ const struct attribute_group ib_dev_attr_group = {
 	.attrs = ib_dev_attrs,
 };
 
-static void ib_free_port_attrs(struct ib_device *device)
+void ib_free_port_attrs(struct ib_core_device *coredev)
 {
 	struct kobject *p, *t;
 
-	list_for_each_entry_safe(p, t, &device->port_list, entry) {
+	list_for_each_entry_safe(p, t, &coredev->port_list, entry) {
 		struct ib_port *port = container_of(p, struct ib_port, kobj);
 
 		list_del(&p->entry);
@@ -1303,20 +1311,22 @@ static void ib_free_port_attrs(struct ib_device *device)
 		kobject_put(p);
 	}
 
-	kobject_put(device->ports_kobj);
+	kobject_put(coredev->ports_kobj);
 }
 
-static int ib_setup_port_attrs(struct ib_device *device)
+int ib_setup_port_attrs(struct ib_core_device *coredev)
 {
+	struct ib_device *device = rdma_device_to_ibdev(&coredev->dev);
 	unsigned int port;
 	int ret;
 
-	device->ports_kobj = kobject_create_and_add("ports", &device->dev.kobj);
-	if (!device->ports_kobj)
+	coredev->ports_kobj = kobject_create_and_add("ports",
+						     &coredev->dev.kobj);
+	if (!coredev->ports_kobj)
 		return -ENOMEM;
 
 	rdma_for_each_port (device, port) {
-		ret = add_port(device, port);
+		ret = add_port(coredev, port);
 		if (ret)
 			goto err_put;
 	}
@@ -1324,7 +1334,7 @@ static int ib_setup_port_attrs(struct ib_device *device)
 	return 0;
 
 err_put:
-	ib_free_port_attrs(device);
+	ib_free_port_attrs(coredev);
 	return ret;
 }
 
@@ -1332,7 +1342,7 @@ int ib_device_register_sysfs(struct ib_device *device)
 {
 	int ret;
 
-	ret = ib_setup_port_attrs(device);
+	ret = ib_setup_port_attrs(&device->coredev);
 	if (ret)
 		return ret;
 
@@ -1348,5 +1358,48 @@ void ib_device_unregister_sysfs(struct ib_device *device)
 		free_hsag(&device->dev.kobj, device->hw_stats_ag);
 	kfree(device->hw_stats);
 
-	ib_free_port_attrs(device);
+	ib_free_port_attrs(&device->coredev);
+}
+
+/**
+ * ib_port_register_module_stat - add module counters under relevant port
+ *  of IB device.
+ *
+ * @device: IB device to add counters
+ * @port_num: valid port number
+ * @kobj: pointer to the kobject to initialize
+ * @ktype: pointer to the ktype for this kobject.
+ * @name: the name of the kobject
+ */
+int ib_port_register_module_stat(struct ib_device *device, u8 port_num,
+				 struct kobject *kobj, struct kobj_type *ktype,
+				 const char *name)
+{
+	struct kobject *p, *t;
+	int ret;
+
+	list_for_each_entry_safe(p, t, &device->coredev.port_list, entry) {
+		struct ib_port *port = container_of(p, struct ib_port, kobj);
+
+		if (port->port_num != port_num)
+			continue;
+
+		ret = kobject_init_and_add(kobj, ktype, &port->kobj, "%s",
+					   name);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL(ib_port_register_module_stat);
+
+/**
+ * ib_port_unregister_module_stat - release module counters
+ * @kobj: pointer to the kobject to release
+ */
+void ib_port_unregister_module_stat(struct kobject *kobj)
+{
+	kobject_put(kobj);
 }
+EXPORT_SYMBOL(ib_port_unregister_module_stat);
diff --git a/drivers/infiniband/core/ucm.c b/drivers/infiniband/core/ucm.c
index 65c3230f5663..8e7da2d41fd8 100644
--- a/drivers/infiniband/core/ucm.c
+++ b/drivers/infiniband/core/ucm.c
@@ -42,7 +42,7 @@
 #include <linux/file.h>
 #include <linux/mount.h>
 #include <linux/cdev.h>
-#include <linux/idr.h>
+#include <linux/xarray.h>
 #include <linux/mutex.h>
 #include <linux/slab.h>
 
@@ -125,23 +125,22 @@ static struct ib_client ucm_client = {
 	.remove = ib_ucm_remove_one
 };
 
-static DEFINE_MUTEX(ctx_id_mutex);
-static DEFINE_IDR(ctx_id_table);
+static DEFINE_XARRAY_ALLOC(ctx_id_table);
 static DECLARE_BITMAP(dev_map, IB_UCM_MAX_DEVICES);
 
 static struct ib_ucm_context *ib_ucm_ctx_get(struct ib_ucm_file *file, int id)
 {
 	struct ib_ucm_context *ctx;
 
-	mutex_lock(&ctx_id_mutex);
-	ctx = idr_find(&ctx_id_table, id);
+	xa_lock(&ctx_id_table);
+	ctx = xa_load(&ctx_id_table, id);
 	if (!ctx)
 		ctx = ERR_PTR(-ENOENT);
 	else if (ctx->file != file)
 		ctx = ERR_PTR(-EINVAL);
 	else
 		atomic_inc(&ctx->ref);
-	mutex_unlock(&ctx_id_mutex);
+	xa_unlock(&ctx_id_table);
 
 	return ctx;
 }
@@ -194,10 +193,7 @@ static struct ib_ucm_context *ib_ucm_ctx_alloc(struct ib_ucm_file *file)
 	ctx->file = file;
 	INIT_LIST_HEAD(&ctx->events);
 
-	mutex_lock(&ctx_id_mutex);
-	ctx->id = idr_alloc(&ctx_id_table, ctx, 0, 0, GFP_KERNEL);
-	mutex_unlock(&ctx_id_mutex);
-	if (ctx->id < 0)
+	if (xa_alloc(&ctx_id_table, &ctx->id, ctx, xa_limit_32b, GFP_KERNEL))
 		goto error;
 
 	list_add_tail(&ctx->file_list, &file->ctxs);
@@ -514,9 +510,7 @@ static ssize_t ib_ucm_create_id(struct ib_ucm_file *file,
 err2:
 	ib_destroy_cm_id(ctx->cm_id);
 err1:
-	mutex_lock(&ctx_id_mutex);
-	idr_remove(&ctx_id_table, ctx->id);
-	mutex_unlock(&ctx_id_mutex);
+	xa_erase(&ctx_id_table, ctx->id);
 	kfree(ctx);
 	return result;
 }
@@ -536,15 +530,15 @@ static ssize_t ib_ucm_destroy_id(struct ib_ucm_file *file,
 	if (copy_from_user(&cmd, inbuf, sizeof(cmd)))
 		return -EFAULT;
 
-	mutex_lock(&ctx_id_mutex);
-	ctx = idr_find(&ctx_id_table, cmd.id);
+	xa_lock(&ctx_id_table);
+	ctx = xa_load(&ctx_id_table, cmd.id);
 	if (!ctx)
 		ctx = ERR_PTR(-ENOENT);
 	else if (ctx->file != file)
 		ctx = ERR_PTR(-EINVAL);
 	else
-		idr_remove(&ctx_id_table, ctx->id);
-	mutex_unlock(&ctx_id_mutex);
+		__xa_erase(&ctx_id_table, ctx->id);
+	xa_unlock(&ctx_id_table);
 
 	if (IS_ERR(ctx))
 		return PTR_ERR(ctx);
@@ -1189,10 +1183,7 @@ static int ib_ucm_close(struct inode *inode, struct file *filp)
 				 struct ib_ucm_context, file_list);
 		mutex_unlock(&file->file_mutex);
 
-		mutex_lock(&ctx_id_mutex);
-		idr_remove(&ctx_id_table, ctx->id);
-		mutex_unlock(&ctx_id_mutex);
-
+		xa_erase(&ctx_id_table, ctx->id);
 		ib_destroy_cm_id(ctx->cm_id);
 		ib_ucm_cleanup_events(ctx);
 		kfree(ctx);
@@ -1352,7 +1343,7 @@ static void __exit ib_ucm_cleanup(void)
 	class_remove_file(&cm_class, &class_attr_abi_version.attr);
 	unregister_chrdev_region(IB_UCM_BASE_DEV, IB_UCM_NUM_FIXED_MINOR);
 	unregister_chrdev_region(dynamic_ucm_dev, IB_UCM_NUM_DYNAMIC_MINOR);
-	idr_destroy(&ctx_id_table);
+	WARN_ON(!xa_empty(&ctx_id_table));
 }
 
 module_init(ib_ucm_init);
diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c
index fe5551562dbc..0a23048db523 100644
--- a/drivers/infiniband/core/umem.c
+++ b/drivers/infiniband/core/umem.c
@@ -37,27 +37,23 @@
 #include <linux/sched/signal.h>
 #include <linux/sched/mm.h>
 #include <linux/export.h>
-#include <linux/hugetlb.h>
 #include <linux/slab.h>
+#include <linux/pagemap.h>
 #include <rdma/ib_umem_odp.h>
 
 #include "uverbs.h"
 
-
 static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int dirty)
 {
-	struct scatterlist *sg;
+	struct sg_page_iter sg_iter;
 	struct page *page;
-	int i;
 
 	if (umem->nmap > 0)
-		ib_dma_unmap_sg(dev, umem->sg_head.sgl,
-				umem->npages,
+		ib_dma_unmap_sg(dev, umem->sg_head.sgl, umem->sg_nents,
 				DMA_BIDIRECTIONAL);
 
-	for_each_sg(umem->sg_head.sgl, sg, umem->npages, i) {
-
-		page = sg_page(sg);
+	for_each_sg_page(umem->sg_head.sgl, &sg_iter, umem->sg_nents, 0) {
+		page = sg_page_iter_page(&sg_iter);
 		if (!PageDirty(page) && umem->writable && dirty)
 			set_page_dirty_lock(page);
 		put_page(page);
@@ -66,6 +62,124 @@ static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int d
 	sg_free_table(&umem->sg_head);
 }
 
+/* ib_umem_add_sg_table - Add N contiguous pages to scatter table
+ *
+ * sg: current scatterlist entry
+ * page_list: array of npage struct page pointers
+ * npages: number of pages in page_list
+ * max_seg_sz: maximum segment size in bytes
+ * nents: [out] number of entries in the scatterlist
+ *
+ * Return new end of scatterlist
+ */
+static struct scatterlist *ib_umem_add_sg_table(struct scatterlist *sg,
+						struct page **page_list,
+						unsigned long npages,
+						unsigned int max_seg_sz,
+						int *nents)
+{
+	unsigned long first_pfn;
+	unsigned long i = 0;
+	bool update_cur_sg = false;
+	bool first = !sg_page(sg);
+
+	/* Check if new page_list is contiguous with end of previous page_list.
+	 * sg->length here is a multiple of PAGE_SIZE and sg->offset is 0.
+	 */
+	if (!first && (page_to_pfn(sg_page(sg)) + (sg->length >> PAGE_SHIFT) ==
+		       page_to_pfn(page_list[0])))
+		update_cur_sg = true;
+
+	while (i != npages) {
+		unsigned long len;
+		struct page *first_page = page_list[i];
+
+		first_pfn = page_to_pfn(first_page);
+
+		/* Compute the number of contiguous pages we have starting
+		 * at i
+		 */
+		for (len = 0; i != npages &&
+			      first_pfn + len == page_to_pfn(page_list[i]) &&
+			      len < (max_seg_sz >> PAGE_SHIFT);
+		     len++)
+			i++;
+
+		/* Squash N contiguous pages from page_list into current sge */
+		if (update_cur_sg) {
+			if ((max_seg_sz - sg->length) >= (len << PAGE_SHIFT)) {
+				sg_set_page(sg, sg_page(sg),
+					    sg->length + (len << PAGE_SHIFT),
+					    0);
+				update_cur_sg = false;
+				continue;
+			}
+			update_cur_sg = false;
+		}
+
+		/* Squash N contiguous pages into next sge or first sge */
+		if (!first)
+			sg = sg_next(sg);
+
+		(*nents)++;
+		sg_set_page(sg, first_page, len << PAGE_SHIFT, 0);
+		first = false;
+	}
+
+	return sg;
+}
+
+/**
+ * ib_umem_find_best_pgsz - Find best HW page size to use for this MR
+ *
+ * @umem: umem struct
+ * @pgsz_bitmap: bitmap of HW supported page sizes
+ * @virt: IOVA
+ *
+ * This helper is intended for HW that support multiple page
+ * sizes but can do only a single page size in an MR.
+ *
+ * Returns 0 if the umem requires page sizes not supported by
+ * the driver to be mapped. Drivers always supporting PAGE_SIZE
+ * or smaller will never see a 0 result.
+ */
+unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem,
+				     unsigned long pgsz_bitmap,
+				     unsigned long virt)
+{
+	struct scatterlist *sg;
+	unsigned int best_pg_bit;
+	unsigned long va, pgoff;
+	dma_addr_t mask;
+	int i;
+
+	/* At minimum, drivers must support PAGE_SIZE or smaller */
+	if (WARN_ON(!(pgsz_bitmap & GENMASK(PAGE_SHIFT, 0))))
+		return 0;
+
+	va = virt;
+	/* max page size not to exceed MR length */
+	mask = roundup_pow_of_two(umem->length);
+	/* offset into first SGL */
+	pgoff = umem->address & ~PAGE_MASK;
+
+	for_each_sg(umem->sg_head.sgl, sg, umem->nmap, i) {
+		/* Walk SGL and reduce max page size if VA/PA bits differ
+		 * for any address.
+		 */
+		mask |= (sg_dma_address(sg) + pgoff) ^ va;
+		if (i && i != (umem->nmap - 1))
+			/* restrict by length as well for interior SGEs */
+			mask |= sg_dma_len(sg);
+		va += sg_dma_len(sg) - pgoff;
+		pgoff = 0;
+	}
+	best_pg_bit = rdma_find_pg_bit(mask, pgsz_bitmap);
+
+	return BIT_ULL(best_pg_bit);
+}
+EXPORT_SYMBOL(ib_umem_find_best_pgsz);
+
 /**
  * ib_umem_get - Pin and DMA map userspace memory.
  *
@@ -84,16 +198,14 @@ struct ib_umem *ib_umem_get(struct ib_udata *udata, unsigned long addr,
 	struct ib_ucontext *context;
 	struct ib_umem *umem;
 	struct page **page_list;
-	struct vm_area_struct **vma_list;
 	unsigned long lock_limit;
 	unsigned long new_pinned;
 	unsigned long cur_base;
 	struct mm_struct *mm;
 	unsigned long npages;
 	int ret;
-	int i;
 	unsigned long dma_attrs = 0;
-	struct scatterlist *sg, *sg_list_start;
+	struct scatterlist *sg;
 	unsigned int gup_flags = FOLL_WRITE;
 
 	if (!udata)
@@ -138,29 +250,23 @@ struct ib_umem *ib_umem_get(struct ib_udata *udata, unsigned long addr,
 	mmgrab(mm);
 
 	if (access & IB_ACCESS_ON_DEMAND) {
+		if (WARN_ON_ONCE(!context->invalidate_range)) {
+			ret = -EINVAL;
+			goto umem_kfree;
+		}
+
 		ret = ib_umem_odp_get(to_ib_umem_odp(umem), access);
 		if (ret)
 			goto umem_kfree;
 		return umem;
 	}
 
-	/* We assume the memory is from hugetlb until proved otherwise */
-	umem->hugetlb   = 1;
-
 	page_list = (struct page **) __get_free_page(GFP_KERNEL);
 	if (!page_list) {
 		ret = -ENOMEM;
 		goto umem_kfree;
 	}
 
-	/*
-	 * if we can't alloc the vma_list, it's not so bad;
-	 * just assume the memory is not hugetlb memory
-	 */
-	vma_list = (struct vm_area_struct **) __get_free_page(GFP_KERNEL);
-	if (!vma_list)
-		umem->hugetlb = 0;
-
 	npages = ib_umem_num_pages(umem);
 	if (npages == 0 || npages > UINT_MAX) {
 		ret = -EINVAL;
@@ -185,41 +291,34 @@ struct ib_umem *ib_umem_get(struct ib_udata *udata, unsigned long addr,
 	if (!umem->writable)
 		gup_flags |= FOLL_FORCE;
 
-	sg_list_start = umem->sg_head.sgl;
+	sg = umem->sg_head.sgl;
 
 	while (npages) {
 		down_read(&mm->mmap_sem);
 		ret = get_user_pages_longterm(cur_base,
 				     min_t(unsigned long, npages,
 					   PAGE_SIZE / sizeof (struct page *)),
-				     gup_flags, page_list, vma_list);
+				     gup_flags, page_list, NULL);
 		if (ret < 0) {
 			up_read(&mm->mmap_sem);
 			goto umem_release;
 		}
 
-		umem->npages += ret;
 		cur_base += ret * PAGE_SIZE;
 		npages   -= ret;
 
-		/* Continue to hold the mmap_sem as vma_list access
-		 * needs to be protected.
-		 */
-		for_each_sg(sg_list_start, sg, ret, i) {
-			if (vma_list && !is_vm_hugetlb_page(vma_list[i]))
-				umem->hugetlb = 0;
+		sg = ib_umem_add_sg_table(sg, page_list, ret,
+			dma_get_max_seg_size(context->device->dma_device),
+			&umem->sg_nents);
 
-			sg_set_page(sg, page_list[i], PAGE_SIZE, 0);
-		}
 		up_read(&mm->mmap_sem);
-
-		/* preparing for next loop */
-		sg_list_start = sg;
 	}
 
+	sg_mark_end(sg);
+
 	umem->nmap = ib_dma_map_sg_attrs(context->device,
 				  umem->sg_head.sgl,
-				  umem->npages,
+				  umem->sg_nents,
 				  DMA_BIDIRECTIONAL,
 				  dma_attrs);
 
@@ -236,8 +335,6 @@ umem_release:
 vma:
 	atomic64_sub(ib_umem_num_pages(umem), &mm->pinned_vm);
 out:
-	if (vma_list)
-		free_page((unsigned long) vma_list);
 	free_page((unsigned long) page_list);
 umem_kfree:
 	if (ret) {
@@ -315,7 +412,7 @@ int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset,
 		return -EINVAL;
 	}
 
-	ret = sg_pcopy_to_buffer(umem->sg_head.sgl, umem->npages, dst, length,
+	ret = sg_pcopy_to_buffer(umem->sg_head.sgl, umem->sg_nents, dst, length,
 				 offset + ib_umem_offset(umem));
 
 	if (ret < 0)
diff --git a/drivers/infiniband/core/umem_odp.c b/drivers/infiniband/core/umem_odp.c
index e6ec79ad9cc8..c7226cf52acc 100644
--- a/drivers/infiniband/core/umem_odp.c
+++ b/drivers/infiniband/core/umem_odp.c
@@ -241,7 +241,7 @@ static struct ib_ucontext_per_mm *alloc_per_mm(struct ib_ucontext *ctx,
 	per_mm->mm = mm;
 	per_mm->umem_tree = RB_ROOT_CACHED;
 	init_rwsem(&per_mm->umem_rwsem);
-	per_mm->active = ctx->invalidate_range;
+	per_mm->active = true;
 
 	rcu_read_lock();
 	per_mm->tgid = get_task_pid(current->group_leader, PIDTYPE_PID);
@@ -417,9 +417,6 @@ int ib_umem_odp_get(struct ib_umem_odp *umem_odp, int access)
 		h = hstate_vma(vma);
 		umem->page_shift = huge_page_shift(h);
 		up_read(&mm->mmap_sem);
-		umem->hugetlb = 1;
-	} else {
-		umem->hugetlb = 0;
 	}
 
 	mutex_init(&umem_odp->umem_mutex);
@@ -503,7 +500,6 @@ static int ib_umem_odp_map_dma_single_page(
 	struct ib_umem *umem = &umem_odp->umem;
 	struct ib_device *dev = umem->context->device;
 	dma_addr_t dma_addr;
-	int stored_page = 0;
 	int remove_existing_mapping = 0;
 	int ret = 0;
 
@@ -527,8 +523,7 @@ static int ib_umem_odp_map_dma_single_page(
 		}
 		umem_odp->dma_list[page_index] = dma_addr | access_mask;
 		umem_odp->page_list[page_index] = page;
-		umem->npages++;
-		stored_page = 1;
+		umem_odp->npages++;
 	} else if (umem_odp->page_list[page_index] == page) {
 		umem_odp->dma_list[page_index] |= access_mask;
 	} else {
@@ -540,11 +535,9 @@ static int ib_umem_odp_map_dma_single_page(
 	}
 
 out:
-	/* On Demand Paging - avoid pinning the page */
-	if (umem->context->invalidate_range || !stored_page)
-		put_page(page);
+	put_page(page);
 
-	if (remove_existing_mapping && umem->context->invalidate_range) {
+	if (remove_existing_mapping) {
 		ib_umem_notifier_start_account(umem_odp);
 		umem->context->invalidate_range(
 			umem_odp,
@@ -754,12 +747,9 @@ void ib_umem_odp_unmap_dma_pages(struct ib_umem_odp *umem_odp, u64 virt,
 				 */
 				set_page_dirty(head_page);
 			}
-			/* on demand pinning support */
-			if (!umem->context->invalidate_range)
-				put_page(page);
 			umem_odp->page_list[idx] = NULL;
 			umem_odp->dma_list[idx] = 0;
-			umem->npages--;
+			umem_odp->npages--;
 		}
 	}
 	mutex_unlock(&umem_odp->umem_mutex);
diff --git a/drivers/infiniband/core/user_mad.c b/drivers/infiniband/core/user_mad.c
index b58b07c03cfb..671f07ba1fad 100644
--- a/drivers/infiniband/core/user_mad.c
+++ b/drivers/infiniband/core/user_mad.c
@@ -129,6 +129,9 @@ struct ib_umad_packet {
 	struct ib_user_mad mad;
 };
 
+#define CREATE_TRACE_POINTS
+#include <trace/events/ib_umad.h>
+
 static const dev_t base_umad_dev = MKDEV(IB_UMAD_MAJOR, IB_UMAD_MINOR_BASE);
 static const dev_t base_issm_dev = MKDEV(IB_UMAD_MAJOR, IB_UMAD_MINOR_BASE) +
 				   IB_UMAD_NUM_FIXED_MINOR;
@@ -334,6 +337,9 @@ static ssize_t copy_recv_mad(struct ib_umad_file *file, char __user *buf,
 				return -EFAULT;
 		}
 	}
+
+	trace_ib_umad_read_recv(file, &packet->mad.hdr, &recv_buf->mad->mad_hdr);
+
 	return hdr_size(file) + packet->length;
 }
 
@@ -353,6 +359,9 @@ static ssize_t copy_send_mad(struct ib_umad_file *file, char __user *buf,
 	if (copy_to_user(buf, packet->mad.data, packet->length))
 		return -EFAULT;
 
+	trace_ib_umad_read_send(file, &packet->mad.hdr,
+				(struct ib_mad_hdr *)&packet->mad.data);
+
 	return size;
 }
 
@@ -508,6 +517,9 @@ static ssize_t ib_umad_write(struct file *filp, const char __user *buf,
 
 	mutex_lock(&file->mutex);
 
+	trace_ib_umad_write(file, &packet->mad.hdr,
+			    (struct ib_mad_hdr *)&packet->mad.data);
+
 	agent = __get_agent(file, packet->mad.hdr.id);
 	if (!agent) {
 		ret = -EINVAL;
@@ -968,6 +980,11 @@ static int ib_umad_open(struct inode *inode, struct file *filp)
 		goto out;
 	}
 
+	if (!rdma_dev_access_netns(port->ib_dev, current->nsproxy->net_ns)) {
+		ret = -EPERM;
+		goto out;
+	}
+
 	file = kzalloc(sizeof(*file), GFP_KERNEL);
 	if (!file) {
 		ret = -ENOMEM;
@@ -1061,6 +1078,11 @@ static int ib_umad_sm_open(struct inode *inode, struct file *filp)
 		}
 	}
 
+	if (!rdma_dev_access_netns(port->ib_dev, current->nsproxy->net_ns)) {
+		ret = -EPERM;
+		goto err_up_sem;
+	}
+
 	ret = ib_modify_port(port->ib_dev, port->port_num, 0, &props);
 	if (ret)
 		goto err_up_sem;
diff --git a/drivers/infiniband/core/uverbs.h b/drivers/infiniband/core/uverbs.h
index 32cc8fe7902f..1e5aeb39f774 100644
--- a/drivers/infiniband/core/uverbs.h
+++ b/drivers/infiniband/core/uverbs.h
@@ -162,9 +162,7 @@ struct ib_uverbs_file {
 	struct list_head umaps;
 	struct page *disassociate_page;
 
-	struct idr		idr;
-	/* spinlock protects write access to idr */
-	spinlock_t		idr_lock;
+	struct xarray		idr;
 };
 
 struct ib_uverbs_event {
@@ -241,7 +239,8 @@ void ib_uverbs_srq_event_handler(struct ib_event *event, void *context_ptr);
 void ib_uverbs_event_handler(struct ib_event_handler *handler,
 			     struct ib_event *event);
 int ib_uverbs_dealloc_xrcd(struct ib_uobject *uobject, struct ib_xrcd *xrcd,
-			   enum rdma_remove_reason why);
+			   enum rdma_remove_reason why,
+			   struct uverbs_attr_bundle *attrs);
 
 int uverbs_dealloc_mw(struct ib_mw *mw);
 void ib_uverbs_detach_umcast(struct ib_qp *qp,
diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c
index 062a86c04123..5a3a1780ceea 100644
--- a/drivers/infiniband/core/uverbs_cmd.c
+++ b/drivers/infiniband/core/uverbs_cmd.c
@@ -162,7 +162,7 @@ static const void __user *uverbs_request_next_ptr(struct uverbs_req_iter *iter,
 	const void __user *res = iter->cur;
 
 	if (iter->cur + len > iter->end)
-		return ERR_PTR(-ENOSPC);
+		return (void __force __user *)ERR_PTR(-ENOSPC);
 	iter->cur += len;
 	return res;
 }
@@ -175,7 +175,7 @@ static int uverbs_request_finish(struct uverbs_req_iter *iter)
 }
 
 static struct ib_uverbs_completion_event_file *
-_ib_uverbs_lookup_comp_file(s32 fd, const struct uverbs_attr_bundle *attrs)
+_ib_uverbs_lookup_comp_file(s32 fd, struct uverbs_attr_bundle *attrs)
 {
 	struct ib_uobject *uobj = ufd_get_read(UVERBS_OBJECT_COMP_CHANNEL,
 					       fd, attrs);
@@ -230,6 +230,8 @@ static int ib_uverbs_get_context(struct uverbs_attr_bundle *attrs)
 		goto err_alloc;
 	}
 
+	attrs->context = ucontext;
+
 	ucontext->res.type = RDMA_RESTRACK_CTX;
 	ucontext->device = ib_dev;
 	ucontext->cg_obj = cg_obj;
@@ -423,7 +425,7 @@ static int ib_uverbs_alloc_pd(struct uverbs_attr_bundle *attrs)
 	atomic_set(&pd->usecnt, 0);
 	pd->res.type = RDMA_RESTRACK_PD;
 
-	ret = ib_dev->ops.alloc_pd(pd, uobj->context, &attrs->driver_udata);
+	ret = ib_dev->ops.alloc_pd(pd, &attrs->driver_udata);
 	if (ret)
 		goto err_alloc;
 
@@ -436,15 +438,15 @@ static int ib_uverbs_alloc_pd(struct uverbs_attr_bundle *attrs)
 	if (ret)
 		goto err_copy;
 
-	return uobj_alloc_commit(uobj);
+	return uobj_alloc_commit(uobj, attrs);
 
 err_copy:
-	ib_dealloc_pd(pd);
+	ib_dealloc_pd_user(pd, &attrs->driver_udata);
 	pd = NULL;
 err_alloc:
 	kfree(pd);
 err:
-	uobj_alloc_abort(uobj);
+	uobj_alloc_abort(uobj, attrs);
 	return ret;
 }
 
@@ -594,8 +596,7 @@ static int ib_uverbs_open_xrcd(struct uverbs_attr_bundle *attrs)
 	}
 
 	if (!xrcd) {
-		xrcd = ib_dev->ops.alloc_xrcd(ib_dev, obj->uobject.context,
-					      &attrs->driver_udata);
+		xrcd = ib_dev->ops.alloc_xrcd(ib_dev, &attrs->driver_udata);
 		if (IS_ERR(xrcd)) {
 			ret = PTR_ERR(xrcd);
 			goto err;
@@ -633,7 +634,7 @@ static int ib_uverbs_open_xrcd(struct uverbs_attr_bundle *attrs)
 
 	mutex_unlock(&ibudev->xrcd_tree_mutex);
 
-	return uobj_alloc_commit(&obj->uobject);
+	return uobj_alloc_commit(&obj->uobject, attrs);
 
 err_copy:
 	if (inode) {
@@ -643,10 +644,10 @@ err_copy:
 	}
 
 err_dealloc_xrcd:
-	ib_dealloc_xrcd(xrcd);
+	ib_dealloc_xrcd(xrcd, &attrs->driver_udata);
 
 err:
-	uobj_alloc_abort(&obj->uobject);
+	uobj_alloc_abort(&obj->uobject, attrs);
 
 err_tree_mutex_unlock:
 	if (f.file)
@@ -669,19 +670,19 @@ static int ib_uverbs_close_xrcd(struct uverbs_attr_bundle *attrs)
 	return uobj_perform_destroy(UVERBS_OBJECT_XRCD, cmd.xrcd_handle, attrs);
 }
 
-int ib_uverbs_dealloc_xrcd(struct ib_uobject *uobject,
-			   struct ib_xrcd *xrcd,
-			   enum rdma_remove_reason why)
+int ib_uverbs_dealloc_xrcd(struct ib_uobject *uobject, struct ib_xrcd *xrcd,
+			   enum rdma_remove_reason why,
+			   struct uverbs_attr_bundle *attrs)
 {
 	struct inode *inode;
 	int ret;
-	struct ib_uverbs_device *dev = uobject->context->ufile->device;
+	struct ib_uverbs_device *dev = attrs->ufile->device;
 
 	inode = xrcd->inode;
 	if (inode && !atomic_dec_and_test(&xrcd->usecnt))
 		return 0;
 
-	ret = ib_dealloc_xrcd(xrcd);
+	ret = ib_dealloc_xrcd(xrcd, &attrs->driver_udata);
 
 	if (ib_is_destroy_retryable(ret, why, uobject)) {
 		atomic_inc(&xrcd->usecnt);
@@ -763,16 +764,16 @@ static int ib_uverbs_reg_mr(struct uverbs_attr_bundle *attrs)
 
 	uobj_put_obj_read(pd);
 
-	return uobj_alloc_commit(uobj);
+	return uobj_alloc_commit(uobj, attrs);
 
 err_copy:
-	ib_dereg_mr(mr);
+	ib_dereg_mr_user(mr, &attrs->driver_udata);
 
 err_put:
 	uobj_put_obj_read(pd);
 
 err_free:
-	uobj_alloc_abort(uobj);
+	uobj_alloc_abort(uobj, attrs);
 	return ret;
 }
 
@@ -917,14 +918,14 @@ static int ib_uverbs_alloc_mw(struct uverbs_attr_bundle *attrs)
 		goto err_copy;
 
 	uobj_put_obj_read(pd);
-	return uobj_alloc_commit(uobj);
+	return uobj_alloc_commit(uobj, attrs);
 
 err_copy:
 	uverbs_dealloc_mw(mw);
 err_put:
 	uobj_put_obj_read(pd);
 err_free:
-	uobj_alloc_abort(uobj);
+	uobj_alloc_abort(uobj, attrs);
 	return ret;
 }
 
@@ -965,11 +966,11 @@ static int ib_uverbs_create_comp_channel(struct uverbs_attr_bundle *attrs)
 
 	ret = uverbs_response(attrs, &resp, sizeof(resp));
 	if (ret) {
-		uobj_alloc_abort(uobj);
+		uobj_alloc_abort(uobj, attrs);
 		return ret;
 	}
 
-	return uobj_alloc_commit(uobj);
+	return uobj_alloc_commit(uobj, attrs);
 }
 
 static struct ib_ucq_object *create_cq(struct uverbs_attr_bundle *attrs,
@@ -1009,8 +1010,7 @@ static struct ib_ucq_object *create_cq(struct uverbs_attr_bundle *attrs,
 	attr.comp_vector = cmd->comp_vector;
 	attr.flags = cmd->flags;
 
-	cq = ib_dev->ops.create_cq(ib_dev, &attr, obj->uobject.context,
-				   &attrs->driver_udata);
+	cq = ib_dev->ops.create_cq(ib_dev, &attr, &attrs->driver_udata);
 	if (IS_ERR(cq)) {
 		ret = PTR_ERR(cq);
 		goto err_file;
@@ -1036,7 +1036,7 @@ static struct ib_ucq_object *create_cq(struct uverbs_attr_bundle *attrs,
 	if (ret)
 		goto err_cb;
 
-	ret = uobj_alloc_commit(&obj->uobject);
+	ret = uobj_alloc_commit(&obj->uobject, attrs);
 	if (ret)
 		return ERR_PTR(ret);
 	return obj;
@@ -1049,7 +1049,7 @@ err_file:
 		ib_uverbs_release_ucq(attrs->ufile, ev_file, obj);
 
 err:
-	uobj_alloc_abort(&obj->uobject);
+	uobj_alloc_abort(&obj->uobject, attrs);
 
 	return ERR_PTR(ret);
 }
@@ -1418,7 +1418,6 @@ static int create_qp(struct uverbs_attr_bundle *attrs,
 		if (ret)
 			goto err_cb;
 
-		qp->real_qp	  = qp;
 		qp->pd		  = pd;
 		qp->send_cq	  = attr.send_cq;
 		qp->recv_cq	  = attr.recv_cq;
@@ -1477,7 +1476,7 @@ static int create_qp(struct uverbs_attr_bundle *attrs,
 	if (ind_tbl)
 		uobj_put_obj_read(ind_tbl);
 
-	return uobj_alloc_commit(&obj->uevent.uobject);
+	return uobj_alloc_commit(&obj->uevent.uobject, attrs);
 err_cb:
 	ib_destroy_qp(qp);
 
@@ -1495,7 +1494,7 @@ err_put:
 	if (ind_tbl)
 		uobj_put_obj_read(ind_tbl);
 
-	uobj_alloc_abort(&obj->uevent.uobject);
+	uobj_alloc_abort(&obj->uevent.uobject, attrs);
 	return ret;
 }
 
@@ -1609,14 +1608,14 @@ static int ib_uverbs_open_qp(struct uverbs_attr_bundle *attrs)
 	qp->uobject = &obj->uevent.uobject;
 	uobj_put_read(xrcd_uobj);
 
-	return uobj_alloc_commit(&obj->uevent.uobject);
+	return uobj_alloc_commit(&obj->uevent.uobject, attrs);
 
 err_destroy:
 	ib_destroy_qp(qp);
 err_xrcd:
 	uobj_put_read(xrcd_uobj);
 err_put:
-	uobj_alloc_abort(&obj->uevent.uobject);
+	uobj_alloc_abort(&obj->uevent.uobject, attrs);
 	return ret;
 }
 
@@ -2451,7 +2450,7 @@ static int ib_uverbs_create_ah(struct uverbs_attr_bundle *attrs)
 		goto err_copy;
 
 	uobj_put_obj_read(pd);
-	return uobj_alloc_commit(uobj);
+	return uobj_alloc_commit(uobj, attrs);
 
 err_copy:
 	rdma_destroy_ah(ah, RDMA_DESTROY_AH_SLEEPABLE);
@@ -2460,7 +2459,7 @@ err_put:
 	uobj_put_obj_read(pd);
 
 err:
-	uobj_alloc_abort(uobj);
+	uobj_alloc_abort(uobj, attrs);
 	return ret;
 }
 
@@ -2962,16 +2961,16 @@ static int ib_uverbs_ex_create_wq(struct uverbs_attr_bundle *attrs)
 
 	uobj_put_obj_read(pd);
 	uobj_put_obj_read(cq);
-	return uobj_alloc_commit(&obj->uevent.uobject);
+	return uobj_alloc_commit(&obj->uevent.uobject, attrs);
 
 err_copy:
-	ib_destroy_wq(wq);
+	ib_destroy_wq(wq, &attrs->driver_udata);
 err_put_cq:
 	uobj_put_obj_read(cq);
 err_put_pd:
 	uobj_put_obj_read(pd);
 err_uobj:
-	uobj_alloc_abort(&obj->uevent.uobject);
+	uobj_alloc_abort(&obj->uevent.uobject, attrs);
 
 	return err;
 }
@@ -3136,12 +3135,12 @@ static int ib_uverbs_ex_create_rwq_ind_table(struct uverbs_attr_bundle *attrs)
 	for (j = 0; j < num_read_wqs; j++)
 		uobj_put_obj_read(wqs[j]);
 
-	return uobj_alloc_commit(uobj);
+	return uobj_alloc_commit(uobj, attrs);
 
 err_copy:
 	ib_destroy_rwq_ind_table(rwq_ind_tbl);
 err_uobj:
-	uobj_alloc_abort(uobj);
+	uobj_alloc_abort(uobj, attrs);
 put_wqs:
 	for (j = 0; j < num_read_wqs; j++)
 		uobj_put_obj_read(wqs[j]);
@@ -3314,7 +3313,7 @@ static int ib_uverbs_ex_create_flow(struct uverbs_attr_bundle *attrs)
 	kfree(flow_attr);
 	if (cmd.flow_attr.num_of_specs)
 		kfree(kern_flow_attr);
-	return uobj_alloc_commit(uobj);
+	return uobj_alloc_commit(uobj, attrs);
 err_copy:
 	if (!qp->device->ops.destroy_flow(flow_id))
 		atomic_dec(&qp->usecnt);
@@ -3325,7 +3324,7 @@ err_free_flow_attr:
 err_put:
 	uobj_put_obj_read(qp);
 err_uobj:
-	uobj_alloc_abort(uobj);
+	uobj_alloc_abort(uobj, attrs);
 err_free_attr:
 	if (cmd.flow_attr.num_of_specs)
 		kfree(kern_flow_attr);
@@ -3411,9 +3410,9 @@ static int __uverbs_create_xsrq(struct uverbs_attr_bundle *attrs,
 	obj->uevent.events_reported = 0;
 	INIT_LIST_HEAD(&obj->uevent.event_list);
 
-	srq = pd->device->ops.create_srq(pd, &attr, udata);
-	if (IS_ERR(srq)) {
-		ret = PTR_ERR(srq);
+	srq = rdma_zalloc_drv_obj(ib_dev, ib_srq);
+	if (!srq) {
+		ret = -ENOMEM;
 		goto err_put;
 	}
 
@@ -3424,6 +3423,10 @@ static int __uverbs_create_xsrq(struct uverbs_attr_bundle *attrs,
 	srq->event_handler = attr.event_handler;
 	srq->srq_context   = attr.srq_context;
 
+	ret = pd->device->ops.create_srq(srq, &attr, udata);
+	if (ret)
+		goto err_free;
+
 	if (ib_srq_has_cq(cmd->srq_type)) {
 		srq->ext.cq       = attr.ext.cq;
 		atomic_inc(&attr.ext.cq->usecnt);
@@ -3458,11 +3461,13 @@ static int __uverbs_create_xsrq(struct uverbs_attr_bundle *attrs,
 		uobj_put_obj_read(attr.ext.cq);
 
 	uobj_put_obj_read(pd);
-	return uobj_alloc_commit(&obj->uevent.uobject);
+	return uobj_alloc_commit(&obj->uevent.uobject, attrs);
 
 err_copy:
-	ib_destroy_srq(srq);
+	ib_destroy_srq_user(srq, &attrs->driver_udata);
 
+err_free:
+	kfree(srq);
 err_put:
 	uobj_put_obj_read(pd);
 
@@ -3477,7 +3482,7 @@ err_put_xrcd:
 	}
 
 err:
-	uobj_alloc_abort(&obj->uevent.uobject);
+	uobj_alloc_abort(&obj->uevent.uobject, attrs);
 	return ret;
 }
 
diff --git a/drivers/infiniband/core/uverbs_ioctl.c b/drivers/infiniband/core/uverbs_ioctl.c
index e1379949e663..829b0c6944d8 100644
--- a/drivers/infiniband/core/uverbs_ioctl.c
+++ b/drivers/infiniband/core/uverbs_ioctl.c
@@ -207,13 +207,12 @@ static int uverbs_process_idrs_array(struct bundle_priv *pbundle,
 
 	for (i = 0; i != array_len; i++) {
 		attr->uobjects[i] = uverbs_get_uobject_from_file(
-			spec->u2.objs_arr.obj_type, pbundle->bundle.ufile,
-			spec->u2.objs_arr.access, idr_vals[i]);
+			spec->u2.objs_arr.obj_type, spec->u2.objs_arr.access,
+			idr_vals[i], &pbundle->bundle);
 		if (IS_ERR(attr->uobjects[i])) {
 			ret = PTR_ERR(attr->uobjects[i]);
 			break;
 		}
-		pbundle->bundle.context = attr->uobjects[i]->context;
 	}
 
 	attr->len = i;
@@ -223,7 +222,7 @@ static int uverbs_process_idrs_array(struct bundle_priv *pbundle,
 
 static int uverbs_free_idrs_array(const struct uverbs_api_attr *attr_uapi,
 				  struct uverbs_objs_arr_attr *attr,
-				  bool commit)
+				  bool commit, struct uverbs_attr_bundle *attrs)
 {
 	const struct uverbs_attr_spec *spec = &attr_uapi->spec;
 	int current_ret;
@@ -231,8 +230,9 @@ static int uverbs_free_idrs_array(const struct uverbs_api_attr *attr_uapi,
 	size_t i;
 
 	for (i = 0; i != attr->len; i++) {
-		current_ret = uverbs_finalize_object(
-			attr->uobjects[i], spec->u2.objs_arr.access, commit);
+		current_ret = uverbs_finalize_object(attr->uobjects[i],
+						     spec->u2.objs_arr.access,
+						     commit, attrs);
 		if (!ret)
 			ret = current_ret;
 	}
@@ -325,13 +325,10 @@ static int uverbs_process_attr(struct bundle_priv *pbundle,
 		 * IDR implementation today rejects negative IDs
 		 */
 		o_attr->uobject = uverbs_get_uobject_from_file(
-					spec->u.obj.obj_type,
-					pbundle->bundle.ufile,
-					spec->u.obj.access,
-					uattr->data_s64);
+			spec->u.obj.obj_type, spec->u.obj.access,
+			uattr->data_s64, &pbundle->bundle);
 		if (IS_ERR(o_attr->uobject))
 			return PTR_ERR(o_attr->uobject);
-		pbundle->bundle.context = o_attr->uobject->context;
 		__set_bit(attr_bkey, pbundle->uobj_finalize);
 
 		if (spec->u.obj.access == UVERBS_ACCESS_NEW) {
@@ -456,12 +453,14 @@ static int ib_uverbs_run_method(struct bundle_priv *pbundle,
 		uverbs_fill_udata(&pbundle->bundle,
 				  &pbundle->bundle.driver_udata,
 				  UVERBS_ATTR_UHW_IN, UVERBS_ATTR_UHW_OUT);
+	else
+		pbundle->bundle.driver_udata = (struct ib_udata){};
 
 	if (destroy_bkey != UVERBS_API_ATTR_BKEY_LEN) {
 		struct uverbs_obj_attr *destroy_attr =
 			&pbundle->bundle.attrs[destroy_bkey].obj_attr;
 
-		ret = uobj_destroy(destroy_attr->uobject);
+		ret = uobj_destroy(destroy_attr->uobject, &pbundle->bundle);
 		if (ret)
 			return ret;
 		__clear_bit(destroy_bkey, pbundle->uobj_finalize);
@@ -512,7 +511,8 @@ static int bundle_destroy(struct bundle_priv *pbundle, bool commit)
 
 		current_ret = uverbs_finalize_object(
 			attr->obj_attr.uobject,
-			attr->obj_attr.attr_elm->spec.u.obj.access, commit);
+			attr->obj_attr.attr_elm->spec.u.obj.access, commit,
+			&pbundle->bundle);
 		if (!ret)
 			ret = current_ret;
 	}
@@ -535,7 +535,8 @@ static int bundle_destroy(struct bundle_priv *pbundle, bool commit)
 
 		if (attr_uapi->spec.type == UVERBS_ATTR_TYPE_IDRS_ARRAY) {
 			current_ret = uverbs_free_idrs_array(
-				attr_uapi, &attr->objs_arr_attr, commit);
+				attr_uapi, &attr->objs_arr_attr, commit,
+				&pbundle->bundle);
 			if (!ret)
 				ret = current_ret;
 		}
diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c
index 8b43dd96d3b2..84a5e9a6d483 100644
--- a/drivers/infiniband/core/uverbs_main.c
+++ b/drivers/infiniband/core/uverbs_main.c
@@ -723,7 +723,7 @@ static ssize_t ib_uverbs_write(struct file *filp, const char __user *buf,
 			 * then the command request structure starts
 			 * with a '__aligned u64 response' member.
 			 */
-			ret = get_user(response, (const u64 *)buf);
+			ret = get_user(response, (const u64 __user *)buf);
 			if (ret)
 				goto out_unlock;
 
@@ -926,43 +926,32 @@ static const struct vm_operations_struct rdma_umap_ops = {
 	.fault = rdma_umap_fault,
 };
 
-static struct rdma_umap_priv *rdma_user_mmap_pre(struct ib_ucontext *ucontext,
-						 struct vm_area_struct *vma,
-						 unsigned long size)
+/*
+ * Map IO memory into a process. This is to be called by drivers as part of
+ * their mmap() functions if they wish to send something like PCI-E BAR memory
+ * to userspace.
+ */
+int rdma_user_mmap_io(struct ib_ucontext *ucontext, struct vm_area_struct *vma,
+		      unsigned long pfn, unsigned long size, pgprot_t prot)
 {
 	struct ib_uverbs_file *ufile = ucontext->ufile;
 	struct rdma_umap_priv *priv;
 
 	if (!(vma->vm_flags & VM_SHARED))
-		return ERR_PTR(-EINVAL);
+		return -EINVAL;
 
 	if (vma->vm_end - vma->vm_start != size)
-		return ERR_PTR(-EINVAL);
+		return -EINVAL;
 
 	/* Driver is using this wrong, must be called by ib_uverbs_mmap */
 	if (WARN_ON(!vma->vm_file ||
 		    vma->vm_file->private_data != ufile))
-		return ERR_PTR(-EINVAL);
+		return -EINVAL;
 	lockdep_assert_held(&ufile->device->disassociate_srcu);
 
 	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
 	if (!priv)
-		return ERR_PTR(-ENOMEM);
-	return priv;
-}
-
-/*
- * Map IO memory into a process. This is to be called by drivers as part of
- * their mmap() functions if they wish to send something like PCI-E BAR memory
- * to userspace.
- */
-int rdma_user_mmap_io(struct ib_ucontext *ucontext, struct vm_area_struct *vma,
-		      unsigned long pfn, unsigned long size, pgprot_t prot)
-{
-	struct rdma_umap_priv *priv = rdma_user_mmap_pre(ucontext, vma, size);
-
-	if (IS_ERR(priv))
-		return PTR_ERR(priv);
+		return -ENOMEM;
 
 	vma->vm_page_prot = prot;
 	if (io_remap_pfn_range(vma, vma->vm_start, pfn, size, prot)) {
@@ -975,35 +964,6 @@ int rdma_user_mmap_io(struct ib_ucontext *ucontext, struct vm_area_struct *vma,
 }
 EXPORT_SYMBOL(rdma_user_mmap_io);
 
-/*
- * The page case is here for a slightly different reason, the driver expects
- * to be able to free the page it is sharing to user space when it destroys
- * its ucontext, which means we need to zap the user space references.
- *
- * We could handle this differently by providing an API to allocate a shared
- * page and then only freeing the shared page when the last ufile is
- * destroyed.
- */
-int rdma_user_mmap_page(struct ib_ucontext *ucontext,
-			struct vm_area_struct *vma, struct page *page,
-			unsigned long size)
-{
-	struct rdma_umap_priv *priv = rdma_user_mmap_pre(ucontext, vma, size);
-
-	if (IS_ERR(priv))
-		return PTR_ERR(priv);
-
-	if (remap_pfn_range(vma, vma->vm_start, page_to_pfn(page), size,
-			    vma->vm_page_prot)) {
-		kfree(priv);
-		return -EAGAIN;
-	}
-
-	rdma_umap_priv_init(priv, vma);
-	return 0;
-}
-EXPORT_SYMBOL(rdma_user_mmap_page);
-
 void uverbs_user_mmap_disassociate(struct ib_uverbs_file *ufile)
 {
 	struct rdma_umap_priv *priv, *next_priv;
@@ -1094,6 +1054,11 @@ static int ib_uverbs_open(struct inode *inode, struct file *filp)
 		goto err;
 	}
 
+	if (!rdma_dev_access_netns(ib_dev, current->nsproxy->net_ns)) {
+		ret = -EPERM;
+		goto err;
+	}
+
 	/* In case IB device supports disassociate ucontext, there is no hard
 	 * dependency between uverbs device and its low level device.
 	 */
diff --git a/drivers/infiniband/core/uverbs_std_types.c b/drivers/infiniband/core/uverbs_std_types.c
index f224cb727224..35b2e2c640cc 100644
--- a/drivers/infiniband/core/uverbs_std_types.c
+++ b/drivers/infiniband/core/uverbs_std_types.c
@@ -40,14 +40,17 @@
 #include "uverbs.h"
 
 static int uverbs_free_ah(struct ib_uobject *uobject,
-			  enum rdma_remove_reason why)
+			  enum rdma_remove_reason why,
+			  struct uverbs_attr_bundle *attrs)
 {
-	return rdma_destroy_ah((struct ib_ah *)uobject->object,
-			       RDMA_DESTROY_AH_SLEEPABLE);
+	return rdma_destroy_ah_user((struct ib_ah *)uobject->object,
+				    RDMA_DESTROY_AH_SLEEPABLE,
+				    &attrs->driver_udata);
 }
 
 static int uverbs_free_flow(struct ib_uobject *uobject,
-			    enum rdma_remove_reason why)
+			    enum rdma_remove_reason why,
+			    struct uverbs_attr_bundle *attrs)
 {
 	struct ib_flow *flow = (struct ib_flow *)uobject->object;
 	struct ib_uflow_object *uflow =
@@ -66,13 +69,15 @@ static int uverbs_free_flow(struct ib_uobject *uobject,
 }
 
 static int uverbs_free_mw(struct ib_uobject *uobject,
-			  enum rdma_remove_reason why)
+			  enum rdma_remove_reason why,
+			  struct uverbs_attr_bundle *attrs)
 {
 	return uverbs_dealloc_mw((struct ib_mw *)uobject->object);
 }
 
 static int uverbs_free_qp(struct ib_uobject *uobject,
-			  enum rdma_remove_reason why)
+			  enum rdma_remove_reason why,
+			  struct uverbs_attr_bundle *attrs)
 {
 	struct ib_qp *qp = uobject->object;
 	struct ib_uqp_object *uqp =
@@ -93,19 +98,20 @@ static int uverbs_free_qp(struct ib_uobject *uobject,
 		ib_uverbs_detach_umcast(qp, uqp);
 	}
 
-	ret = ib_destroy_qp(qp);
+	ret = ib_destroy_qp_user(qp, &attrs->driver_udata);
 	if (ib_is_destroy_retryable(ret, why, uobject))
 		return ret;
 
 	if (uqp->uxrcd)
 		atomic_dec(&uqp->uxrcd->refcnt);
 
-	ib_uverbs_release_uevent(uobject->context->ufile, &uqp->uevent);
+	ib_uverbs_release_uevent(attrs->ufile, &uqp->uevent);
 	return ret;
 }
 
 static int uverbs_free_rwq_ind_tbl(struct ib_uobject *uobject,
-				   enum rdma_remove_reason why)
+				   enum rdma_remove_reason why,
+				   struct uverbs_attr_bundle *attrs)
 {
 	struct ib_rwq_ind_table *rwq_ind_tbl = uobject->object;
 	struct ib_wq **ind_tbl = rwq_ind_tbl->ind_tbl;
@@ -120,23 +126,25 @@ static int uverbs_free_rwq_ind_tbl(struct ib_uobject *uobject,
 }
 
 static int uverbs_free_wq(struct ib_uobject *uobject,
-			  enum rdma_remove_reason why)
+			  enum rdma_remove_reason why,
+			  struct uverbs_attr_bundle *attrs)
 {
 	struct ib_wq *wq = uobject->object;
 	struct ib_uwq_object *uwq =
 		container_of(uobject, struct ib_uwq_object, uevent.uobject);
 	int ret;
 
-	ret = ib_destroy_wq(wq);
+	ret = ib_destroy_wq(wq, &attrs->driver_udata);
 	if (ib_is_destroy_retryable(ret, why, uobject))
 		return ret;
 
-	ib_uverbs_release_uevent(uobject->context->ufile, &uwq->uevent);
+	ib_uverbs_release_uevent(attrs->ufile, &uwq->uevent);
 	return ret;
 }
 
 static int uverbs_free_srq(struct ib_uobject *uobject,
-			   enum rdma_remove_reason why)
+			   enum rdma_remove_reason why,
+			   struct uverbs_attr_bundle *attrs)
 {
 	struct ib_srq *srq = uobject->object;
 	struct ib_uevent_object *uevent =
@@ -144,7 +152,7 @@ static int uverbs_free_srq(struct ib_uobject *uobject,
 	enum ib_srq_type  srq_type = srq->srq_type;
 	int ret;
 
-	ret = ib_destroy_srq(srq);
+	ret = ib_destroy_srq_user(srq, &attrs->driver_udata);
 	if (ib_is_destroy_retryable(ret, why, uobject))
 		return ret;
 
@@ -155,12 +163,13 @@ static int uverbs_free_srq(struct ib_uobject *uobject,
 		atomic_dec(&us->uxrcd->refcnt);
 	}
 
-	ib_uverbs_release_uevent(uobject->context->ufile, uevent);
+	ib_uverbs_release_uevent(attrs->ufile, uevent);
 	return ret;
 }
 
 static int uverbs_free_xrcd(struct ib_uobject *uobject,
-			    enum rdma_remove_reason why)
+			    enum rdma_remove_reason why,
+			    struct uverbs_attr_bundle *attrs)
 {
 	struct ib_xrcd *xrcd = uobject->object;
 	struct ib_uxrcd_object *uxrcd =
@@ -171,15 +180,16 @@ static int uverbs_free_xrcd(struct ib_uobject *uobject,
 	if (ret)
 		return ret;
 
-	mutex_lock(&uobject->context->ufile->device->xrcd_tree_mutex);
-	ret = ib_uverbs_dealloc_xrcd(uobject, xrcd, why);
-	mutex_unlock(&uobject->context->ufile->device->xrcd_tree_mutex);
+	mutex_lock(&attrs->ufile->device->xrcd_tree_mutex);
+	ret = ib_uverbs_dealloc_xrcd(uobject, xrcd, why, attrs);
+	mutex_unlock(&attrs->ufile->device->xrcd_tree_mutex);
 
 	return ret;
 }
 
 static int uverbs_free_pd(struct ib_uobject *uobject,
-			  enum rdma_remove_reason why)
+			  enum rdma_remove_reason why,
+			  struct uverbs_attr_bundle *attrs)
 {
 	struct ib_pd *pd = uobject->object;
 	int ret;
@@ -188,7 +198,7 @@ static int uverbs_free_pd(struct ib_uobject *uobject,
 	if (ret)
 		return ret;
 
-	ib_dealloc_pd(pd);
+	ib_dealloc_pd_user(pd, &attrs->driver_udata);
 	return 0;
 }
 
diff --git a/drivers/infiniband/core/uverbs_std_types_counters.c b/drivers/infiniband/core/uverbs_std_types_counters.c
index 309c5e80988d..9f013304e677 100644
--- a/drivers/infiniband/core/uverbs_std_types_counters.c
+++ b/drivers/infiniband/core/uverbs_std_types_counters.c
@@ -31,11 +31,13 @@
  * SOFTWARE.
  */
 
+#include "rdma_core.h"
 #include "uverbs.h"
 #include <rdma/uverbs_std_types.h>
 
 static int uverbs_free_counters(struct ib_uobject *uobject,
-				enum rdma_remove_reason why)
+				enum rdma_remove_reason why,
+				struct uverbs_attr_bundle *attrs)
 {
 	struct ib_counters *counters = uobject->object;
 	int ret;
@@ -52,7 +54,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_COUNTERS_CREATE)(
 {
 	struct ib_uobject *uobj = uverbs_attr_get_uobject(
 		attrs, UVERBS_ATTR_CREATE_COUNTERS_HANDLE);
-	struct ib_device *ib_dev = uobj->context->device;
+	struct ib_device *ib_dev = attrs->context->device;
 	struct ib_counters *counters;
 	int ret;
 
diff --git a/drivers/infiniband/core/uverbs_std_types_cq.c b/drivers/infiniband/core/uverbs_std_types_cq.c
index a59ea89e3f2b..db5c46a1bb2d 100644
--- a/drivers/infiniband/core/uverbs_std_types_cq.c
+++ b/drivers/infiniband/core/uverbs_std_types_cq.c
@@ -35,7 +35,8 @@
 #include "uverbs.h"
 
 static int uverbs_free_cq(struct ib_uobject *uobject,
-			  enum rdma_remove_reason why)
+			  enum rdma_remove_reason why,
+			  struct uverbs_attr_bundle *attrs)
 {
 	struct ib_cq *cq = uobject->object;
 	struct ib_uverbs_event_queue *ev_queue = cq->cq_context;
@@ -43,12 +44,12 @@ static int uverbs_free_cq(struct ib_uobject *uobject,
 		container_of(uobject, struct ib_ucq_object, uobject);
 	int ret;
 
-	ret = ib_destroy_cq(cq);
+	ret = ib_destroy_cq_user(cq, &attrs->driver_udata);
 	if (ib_is_destroy_retryable(ret, why, uobject))
 		return ret;
 
 	ib_uverbs_release_ucq(
-		uobject->context->ufile,
+		attrs->ufile,
 		ev_queue ? container_of(ev_queue,
 					struct ib_uverbs_completion_event_file,
 					ev_queue) :
@@ -63,7 +64,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)(
 	struct ib_ucq_object *obj = container_of(
 		uverbs_attr_get_uobject(attrs, UVERBS_ATTR_CREATE_CQ_HANDLE),
 		typeof(*obj), uobject);
-	struct ib_device *ib_dev = obj->uobject.context->device;
+	struct ib_device *ib_dev = attrs->context->device;
 	int ret;
 	u64 user_handle;
 	struct ib_cq_init_attr attr = {};
@@ -110,8 +111,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)(
 	INIT_LIST_HEAD(&obj->comp_list);
 	INIT_LIST_HEAD(&obj->async_list);
 
-	cq = ib_dev->ops.create_cq(ib_dev, &attr, obj->uobject.context,
-				   &attrs->driver_udata);
+	cq = ib_dev->ops.create_cq(ib_dev, &attr, &attrs->driver_udata);
 	if (IS_ERR(cq)) {
 		ret = PTR_ERR(cq);
 		goto err_event_file;
diff --git a/drivers/infiniband/core/uverbs_std_types_dm.c b/drivers/infiniband/core/uverbs_std_types_dm.c
index 2ef70637bee1..d5a1de33c2c9 100644
--- a/drivers/infiniband/core/uverbs_std_types_dm.c
+++ b/drivers/infiniband/core/uverbs_std_types_dm.c
@@ -30,11 +30,13 @@
  * SOFTWARE.
  */
 
+#include "rdma_core.h"
 #include "uverbs.h"
 #include <rdma/uverbs_std_types.h>
 
 static int uverbs_free_dm(struct ib_uobject *uobject,
-			  enum rdma_remove_reason why)
+			  enum rdma_remove_reason why,
+			  struct uverbs_attr_bundle *attrs)
 {
 	struct ib_dm *dm = uobject->object;
 	int ret;
@@ -43,7 +45,7 @@ static int uverbs_free_dm(struct ib_uobject *uobject,
 	if (ret)
 		return ret;
 
-	return dm->device->ops.dealloc_dm(dm);
+	return dm->device->ops.dealloc_dm(dm, attrs);
 }
 
 static int UVERBS_HANDLER(UVERBS_METHOD_DM_ALLOC)(
@@ -53,7 +55,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_DM_ALLOC)(
 	struct ib_uobject *uobj =
 		uverbs_attr_get(attrs, UVERBS_ATTR_ALLOC_DM_HANDLE)
 			->obj_attr.uobject;
-	struct ib_device *ib_dev = uobj->context->device;
+	struct ib_device *ib_dev = attrs->context->device;
 	struct ib_dm *dm;
 	int ret;
 
@@ -70,7 +72,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_DM_ALLOC)(
 	if (ret)
 		return ret;
 
-	dm = ib_dev->ops.alloc_dm(ib_dev, uobj->context, &attr, attrs);
+	dm = ib_dev->ops.alloc_dm(ib_dev, attrs->context, &attr, attrs);
 	if (IS_ERR(dm))
 		return PTR_ERR(dm);
 
diff --git a/drivers/infiniband/core/uverbs_std_types_flow_action.c b/drivers/infiniband/core/uverbs_std_types_flow_action.c
index 4962b87fa600..459cf165b231 100644
--- a/drivers/infiniband/core/uverbs_std_types_flow_action.c
+++ b/drivers/infiniband/core/uverbs_std_types_flow_action.c
@@ -30,11 +30,13 @@
  * SOFTWARE.
  */
 
+#include "rdma_core.h"
 #include "uverbs.h"
 #include <rdma/uverbs_std_types.h>
 
 static int uverbs_free_flow_action(struct ib_uobject *uobject,
-				   enum rdma_remove_reason why)
+				   enum rdma_remove_reason why,
+				   struct uverbs_attr_bundle *attrs)
 {
 	struct ib_flow_action *action = uobject->object;
 	int ret;
@@ -308,7 +310,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_FLOW_ACTION_ESP_CREATE)(
 {
 	struct ib_uobject *uobj = uverbs_attr_get_uobject(
 		attrs, UVERBS_ATTR_CREATE_FLOW_ACTION_ESP_HANDLE);
-	struct ib_device *ib_dev = uobj->context->device;
+	struct ib_device *ib_dev = attrs->context->device;
 	int				  ret;
 	struct ib_flow_action		  *action;
 	struct ib_flow_action_esp_attr	  esp_attr = {};
diff --git a/drivers/infiniband/core/uverbs_std_types_mr.c b/drivers/infiniband/core/uverbs_std_types_mr.c
index 4d4be0c2b752..610d3b9f7654 100644
--- a/drivers/infiniband/core/uverbs_std_types_mr.c
+++ b/drivers/infiniband/core/uverbs_std_types_mr.c
@@ -30,13 +30,16 @@
  * SOFTWARE.
  */
 
+#include "rdma_core.h"
 #include "uverbs.h"
 #include <rdma/uverbs_std_types.h>
 
 static int uverbs_free_mr(struct ib_uobject *uobject,
-			  enum rdma_remove_reason why)
+			  enum rdma_remove_reason why,
+			  struct uverbs_attr_bundle *attrs)
 {
-	return ib_dereg_mr((struct ib_mr *)uobject->object);
+	return ib_dereg_mr_user((struct ib_mr *)uobject->object,
+				&attrs->driver_udata);
 }
 
 static int UVERBS_HANDLER(UVERBS_METHOD_ADVISE_MR)(
@@ -145,7 +148,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_DM_MR_REG)(
 	return 0;
 
 err_dereg:
-	ib_dereg_mr(mr);
+	ib_dereg_mr_user(mr, &attrs->driver_udata);
 
 	return ret;
 }
diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
index 5a5e83f5f0fc..e666a1f7608d 100644
--- a/drivers/infiniband/core/verbs.c
+++ b/drivers/infiniband/core/verbs.c
@@ -218,6 +218,8 @@ rdma_node_get_transport(enum rdma_node_type node_type)
 		return RDMA_TRANSPORT_USNIC_UDP;
 	if (node_type == RDMA_NODE_RNIC)
 		return RDMA_TRANSPORT_IWARP;
+	if (node_type == RDMA_NODE_UNSPECIFIED)
+		return RDMA_TRANSPORT_UNSPECIFIED;
 
 	return RDMA_TRANSPORT_IB;
 }
@@ -269,7 +271,7 @@ struct ib_pd *__ib_alloc_pd(struct ib_device *device, unsigned int flags,
 	pd->res.type = RDMA_RESTRACK_PD;
 	rdma_restrack_set_task(&pd->res, caller);
 
-	ret = device->ops.alloc_pd(pd, NULL, NULL);
+	ret = device->ops.alloc_pd(pd, NULL);
 	if (ret) {
 		kfree(pd);
 		return ERR_PTR(ret);
@@ -316,17 +318,18 @@ EXPORT_SYMBOL(__ib_alloc_pd);
 /**
  * ib_dealloc_pd - Deallocates a protection domain.
  * @pd: The protection domain to deallocate.
+ * @udata: Valid user data or NULL for kernel object
  *
  * It is an error to call this function while any resources in the pd still
  * exist.  The caller is responsible to synchronously destroy them and
  * guarantee no new allocations will happen.
  */
-void ib_dealloc_pd(struct ib_pd *pd)
+void ib_dealloc_pd_user(struct ib_pd *pd, struct ib_udata *udata)
 {
 	int ret;
 
 	if (pd->__internal_mr) {
-		ret = pd->device->ops.dereg_mr(pd->__internal_mr);
+		ret = pd->device->ops.dereg_mr(pd->__internal_mr, NULL);
 		WARN_ON(ret);
 		pd->__internal_mr = NULL;
 	}
@@ -336,10 +339,10 @@ void ib_dealloc_pd(struct ib_pd *pd)
 	WARN_ON(atomic_read(&pd->usecnt));
 
 	rdma_restrack_del(&pd->res);
-	pd->device->ops.dealloc_pd(pd);
+	pd->device->ops.dealloc_pd(pd, udata);
 	kfree(pd);
 }
-EXPORT_SYMBOL(ib_dealloc_pd);
+EXPORT_SYMBOL(ib_dealloc_pd_user);
 
 /* Address handles */
 
@@ -495,25 +498,33 @@ static struct ib_ah *_rdma_create_ah(struct ib_pd *pd,
 				     u32 flags,
 				     struct ib_udata *udata)
 {
+	struct ib_device *device = pd->device;
 	struct ib_ah *ah;
+	int ret;
 
 	might_sleep_if(flags & RDMA_CREATE_AH_SLEEPABLE);
 
-	if (!pd->device->ops.create_ah)
+	if (!device->ops.create_ah)
 		return ERR_PTR(-EOPNOTSUPP);
 
-	ah = pd->device->ops.create_ah(pd, ah_attr, flags, udata);
+	ah = rdma_zalloc_drv_obj_gfp(
+		device, ib_ah,
+		(flags & RDMA_CREATE_AH_SLEEPABLE) ? GFP_KERNEL : GFP_ATOMIC);
+	if (!ah)
+		return ERR_PTR(-ENOMEM);
 
-	if (!IS_ERR(ah)) {
-		ah->device  = pd->device;
-		ah->pd      = pd;
-		ah->uobject = NULL;
-		ah->type    = ah_attr->type;
-		ah->sgid_attr = rdma_update_sgid_attr(ah_attr, NULL);
+	ah->device = device;
+	ah->pd = pd;
+	ah->type = ah_attr->type;
+	ah->sgid_attr = rdma_update_sgid_attr(ah_attr, NULL);
 
-		atomic_inc(&pd->usecnt);
+	ret = device->ops.create_ah(ah, ah_attr, flags, udata);
+	if (ret) {
+		kfree(ah);
+		return ERR_PTR(ret);
 	}
 
+	atomic_inc(&pd->usecnt);
 	return ah;
 }
 
@@ -930,25 +941,24 @@ int rdma_query_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr)
 }
 EXPORT_SYMBOL(rdma_query_ah);
 
-int rdma_destroy_ah(struct ib_ah *ah, u32 flags)
+int rdma_destroy_ah_user(struct ib_ah *ah, u32 flags, struct ib_udata *udata)
 {
 	const struct ib_gid_attr *sgid_attr = ah->sgid_attr;
 	struct ib_pd *pd;
-	int ret;
 
 	might_sleep_if(flags & RDMA_DESTROY_AH_SLEEPABLE);
 
 	pd = ah->pd;
-	ret = ah->device->ops.destroy_ah(ah, flags);
-	if (!ret) {
-		atomic_dec(&pd->usecnt);
-		if (sgid_attr)
-			rdma_put_gid_attr(sgid_attr);
-	}
 
-	return ret;
+	ah->device->ops.destroy_ah(ah, flags);
+	atomic_dec(&pd->usecnt);
+	if (sgid_attr)
+		rdma_put_gid_attr(sgid_attr);
+
+	kfree(ah);
+	return 0;
 }
-EXPORT_SYMBOL(rdma_destroy_ah);
+EXPORT_SYMBOL(rdma_destroy_ah_user);
 
 /* Shared receive queues */
 
@@ -956,29 +966,40 @@ struct ib_srq *ib_create_srq(struct ib_pd *pd,
 			     struct ib_srq_init_attr *srq_init_attr)
 {
 	struct ib_srq *srq;
+	int ret;
 
 	if (!pd->device->ops.create_srq)
 		return ERR_PTR(-EOPNOTSUPP);
 
-	srq = pd->device->ops.create_srq(pd, srq_init_attr, NULL);
-
-	if (!IS_ERR(srq)) {
-		srq->device    	   = pd->device;
-		srq->pd        	   = pd;
-		srq->uobject       = NULL;
-		srq->event_handler = srq_init_attr->event_handler;
-		srq->srq_context   = srq_init_attr->srq_context;
-		srq->srq_type      = srq_init_attr->srq_type;
-		if (ib_srq_has_cq(srq->srq_type)) {
-			srq->ext.cq   = srq_init_attr->ext.cq;
-			atomic_inc(&srq->ext.cq->usecnt);
-		}
-		if (srq->srq_type == IB_SRQT_XRC) {
-			srq->ext.xrc.xrcd = srq_init_attr->ext.xrc.xrcd;
-			atomic_inc(&srq->ext.xrc.xrcd->usecnt);
-		}
-		atomic_inc(&pd->usecnt);
-		atomic_set(&srq->usecnt, 0);
+	srq = rdma_zalloc_drv_obj(pd->device, ib_srq);
+	if (!srq)
+		return ERR_PTR(-ENOMEM);
+
+	srq->device = pd->device;
+	srq->pd = pd;
+	srq->event_handler = srq_init_attr->event_handler;
+	srq->srq_context = srq_init_attr->srq_context;
+	srq->srq_type = srq_init_attr->srq_type;
+
+	if (ib_srq_has_cq(srq->srq_type)) {
+		srq->ext.cq = srq_init_attr->ext.cq;
+		atomic_inc(&srq->ext.cq->usecnt);
+	}
+	if (srq->srq_type == IB_SRQT_XRC) {
+		srq->ext.xrc.xrcd = srq_init_attr->ext.xrc.xrcd;
+		atomic_inc(&srq->ext.xrc.xrcd->usecnt);
+	}
+	atomic_inc(&pd->usecnt);
+
+	ret = pd->device->ops.create_srq(srq, srq_init_attr, NULL);
+	if (ret) {
+		atomic_dec(&srq->pd->usecnt);
+		if (srq->srq_type == IB_SRQT_XRC)
+			atomic_dec(&srq->ext.xrc.xrcd->usecnt);
+		if (ib_srq_has_cq(srq->srq_type))
+			atomic_dec(&srq->ext.cq->usecnt);
+		kfree(srq);
+		return ERR_PTR(ret);
 	}
 
 	return srq;
@@ -1003,36 +1024,23 @@ int ib_query_srq(struct ib_srq *srq,
 }
 EXPORT_SYMBOL(ib_query_srq);
 
-int ib_destroy_srq(struct ib_srq *srq)
+int ib_destroy_srq_user(struct ib_srq *srq, struct ib_udata *udata)
 {
-	struct ib_pd *pd;
-	enum ib_srq_type srq_type;
-	struct ib_xrcd *uninitialized_var(xrcd);
-	struct ib_cq *uninitialized_var(cq);
-	int ret;
-
 	if (atomic_read(&srq->usecnt))
 		return -EBUSY;
 
-	pd = srq->pd;
-	srq_type = srq->srq_type;
-	if (ib_srq_has_cq(srq_type))
-		cq = srq->ext.cq;
-	if (srq_type == IB_SRQT_XRC)
-		xrcd = srq->ext.xrc.xrcd;
+	srq->device->ops.destroy_srq(srq, udata);
 
-	ret = srq->device->ops.destroy_srq(srq);
-	if (!ret) {
-		atomic_dec(&pd->usecnt);
-		if (srq_type == IB_SRQT_XRC)
-			atomic_dec(&xrcd->usecnt);
-		if (ib_srq_has_cq(srq_type))
-			atomic_dec(&cq->usecnt);
-	}
+	atomic_dec(&srq->pd->usecnt);
+	if (srq->srq_type == IB_SRQT_XRC)
+		atomic_dec(&srq->ext.xrc.xrcd->usecnt);
+	if (ib_srq_has_cq(srq->srq_type))
+		atomic_dec(&srq->ext.cq->usecnt);
+	kfree(srq);
 
-	return ret;
+	return 0;
 }
-EXPORT_SYMBOL(ib_destroy_srq);
+EXPORT_SYMBOL(ib_destroy_srq_user);
 
 /* Queue pairs */
 
@@ -1111,8 +1119,9 @@ struct ib_qp *ib_open_qp(struct ib_xrcd *xrcd,
 }
 EXPORT_SYMBOL(ib_open_qp);
 
-static struct ib_qp *create_xrc_qp(struct ib_qp *qp,
-				   struct ib_qp_init_attr *qp_init_attr)
+static struct ib_qp *create_xrc_qp_user(struct ib_qp *qp,
+					struct ib_qp_init_attr *qp_init_attr,
+					struct ib_udata *udata)
 {
 	struct ib_qp *real_qp = qp;
 
@@ -1134,8 +1143,9 @@ static struct ib_qp *create_xrc_qp(struct ib_qp *qp,
 	return qp;
 }
 
-struct ib_qp *ib_create_qp(struct ib_pd *pd,
-			   struct ib_qp_init_attr *qp_init_attr)
+struct ib_qp *ib_create_qp_user(struct ib_pd *pd,
+				struct ib_qp_init_attr *qp_init_attr,
+				struct ib_udata *udata)
 {
 	struct ib_device *device = pd ? pd->device : qp_init_attr->xrcd->device;
 	struct ib_qp *qp;
@@ -1164,7 +1174,6 @@ struct ib_qp *ib_create_qp(struct ib_pd *pd,
 	if (ret)
 		goto err;
 
-	qp->real_qp    = qp;
 	qp->qp_type    = qp_init_attr->qp_type;
 	qp->rwq_ind_tbl = qp_init_attr->rwq_ind_tbl;
 
@@ -1176,7 +1185,8 @@ struct ib_qp *ib_create_qp(struct ib_pd *pd,
 	qp->port = 0;
 
 	if (qp_init_attr->qp_type == IB_QPT_XRC_TGT) {
-		struct ib_qp *xrc_qp = create_xrc_qp(qp, qp_init_attr);
+		struct ib_qp *xrc_qp =
+			create_xrc_qp_user(qp, qp_init_attr, udata);
 
 		if (IS_ERR(xrc_qp)) {
 			ret = PTR_ERR(xrc_qp);
@@ -1230,7 +1240,7 @@ err:
 	return ERR_PTR(ret);
 
 }
-EXPORT_SYMBOL(ib_create_qp);
+EXPORT_SYMBOL(ib_create_qp_user);
 
 static const struct {
 	int			valid;
@@ -1837,7 +1847,7 @@ static int __ib_destroy_shared_qp(struct ib_qp *qp)
 	return 0;
 }
 
-int ib_destroy_qp(struct ib_qp *qp)
+int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata)
 {
 	const struct ib_gid_attr *alt_path_sgid_attr = qp->alt_path_sgid_attr;
 	const struct ib_gid_attr *av_sgid_attr = qp->av_sgid_attr;
@@ -1869,7 +1879,7 @@ int ib_destroy_qp(struct ib_qp *qp)
 		rdma_rw_cleanup_mrs(qp);
 
 	rdma_restrack_del(&qp->res);
-	ret = qp->device->ops.destroy_qp(qp);
+	ret = qp->device->ops.destroy_qp(qp, udata);
 	if (!ret) {
 		if (alt_path_sgid_attr)
 			rdma_put_gid_attr(alt_path_sgid_attr);
@@ -1894,7 +1904,7 @@ int ib_destroy_qp(struct ib_qp *qp)
 
 	return ret;
 }
-EXPORT_SYMBOL(ib_destroy_qp);
+EXPORT_SYMBOL(ib_destroy_qp_user);
 
 /* Completion queues */
 
@@ -1907,7 +1917,7 @@ struct ib_cq *__ib_create_cq(struct ib_device *device,
 {
 	struct ib_cq *cq;
 
-	cq = device->ops.create_cq(device, cq_attr, NULL, NULL);
+	cq = device->ops.create_cq(device, cq_attr, NULL);
 
 	if (!IS_ERR(cq)) {
 		cq->device        = device;
@@ -1933,15 +1943,15 @@ int rdma_set_cq_moderation(struct ib_cq *cq, u16 cq_count, u16 cq_period)
 }
 EXPORT_SYMBOL(rdma_set_cq_moderation);
 
-int ib_destroy_cq(struct ib_cq *cq)
+int ib_destroy_cq_user(struct ib_cq *cq, struct ib_udata *udata)
 {
 	if (atomic_read(&cq->usecnt))
 		return -EBUSY;
 
 	rdma_restrack_del(&cq->res);
-	return cq->device->ops.destroy_cq(cq);
+	return cq->device->ops.destroy_cq(cq, udata);
 }
-EXPORT_SYMBOL(ib_destroy_cq);
+EXPORT_SYMBOL(ib_destroy_cq_user);
 
 int ib_resize_cq(struct ib_cq *cq, int cqe)
 {
@@ -1952,14 +1962,14 @@ EXPORT_SYMBOL(ib_resize_cq);
 
 /* Memory regions */
 
-int ib_dereg_mr(struct ib_mr *mr)
+int ib_dereg_mr_user(struct ib_mr *mr, struct ib_udata *udata)
 {
 	struct ib_pd *pd = mr->pd;
 	struct ib_dm *dm = mr->dm;
 	int ret;
 
 	rdma_restrack_del(&mr->res);
-	ret = mr->device->ops.dereg_mr(mr);
+	ret = mr->device->ops.dereg_mr(mr, udata);
 	if (!ret) {
 		atomic_dec(&pd->usecnt);
 		if (dm)
@@ -1968,13 +1978,14 @@ int ib_dereg_mr(struct ib_mr *mr)
 
 	return ret;
 }
-EXPORT_SYMBOL(ib_dereg_mr);
+EXPORT_SYMBOL(ib_dereg_mr_user);
 
 /**
  * ib_alloc_mr() - Allocates a memory region
  * @pd:            protection domain associated with the region
  * @mr_type:       memory region type
  * @max_num_sg:    maximum sg entries available for registration.
+ * @udata:	   user data or null for kernel objects
  *
  * Notes:
  * Memory registeration page/sg lists must not exceed max_num_sg.
@@ -1982,16 +1993,15 @@ EXPORT_SYMBOL(ib_dereg_mr);
  * max_num_sg * used_page_size.
  *
  */
-struct ib_mr *ib_alloc_mr(struct ib_pd *pd,
-			  enum ib_mr_type mr_type,
-			  u32 max_num_sg)
+struct ib_mr *ib_alloc_mr_user(struct ib_pd *pd, enum ib_mr_type mr_type,
+			       u32 max_num_sg, struct ib_udata *udata)
 {
 	struct ib_mr *mr;
 
 	if (!pd->device->ops.alloc_mr)
 		return ERR_PTR(-EOPNOTSUPP);
 
-	mr = pd->device->ops.alloc_mr(pd, mr_type, max_num_sg);
+	mr = pd->device->ops.alloc_mr(pd, mr_type, max_num_sg, udata);
 	if (!IS_ERR(mr)) {
 		mr->device  = pd->device;
 		mr->pd      = pd;
@@ -2005,7 +2015,7 @@ struct ib_mr *ib_alloc_mr(struct ib_pd *pd,
 
 	return mr;
 }
-EXPORT_SYMBOL(ib_alloc_mr);
+EXPORT_SYMBOL(ib_alloc_mr_user);
 
 /* "Fast" memory regions */
 
@@ -2138,7 +2148,7 @@ struct ib_xrcd *__ib_alloc_xrcd(struct ib_device *device, const char *caller)
 	if (!device->ops.alloc_xrcd)
 		return ERR_PTR(-EOPNOTSUPP);
 
-	xrcd = device->ops.alloc_xrcd(device, NULL, NULL);
+	xrcd = device->ops.alloc_xrcd(device, NULL);
 	if (!IS_ERR(xrcd)) {
 		xrcd->device = device;
 		xrcd->inode = NULL;
@@ -2151,7 +2161,7 @@ struct ib_xrcd *__ib_alloc_xrcd(struct ib_device *device, const char *caller)
 }
 EXPORT_SYMBOL(__ib_alloc_xrcd);
 
-int ib_dealloc_xrcd(struct ib_xrcd *xrcd)
+int ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata)
 {
 	struct ib_qp *qp;
 	int ret;
@@ -2166,7 +2176,7 @@ int ib_dealloc_xrcd(struct ib_xrcd *xrcd)
 			return ret;
 	}
 
-	return xrcd->device->ops.dealloc_xrcd(xrcd);
+	return xrcd->device->ops.dealloc_xrcd(xrcd, udata);
 }
 EXPORT_SYMBOL(ib_dealloc_xrcd);
 
@@ -2210,10 +2220,11 @@ struct ib_wq *ib_create_wq(struct ib_pd *pd,
 EXPORT_SYMBOL(ib_create_wq);
 
 /**
- * ib_destroy_wq - Destroys the specified WQ.
+ * ib_destroy_wq - Destroys the specified user WQ.
  * @wq: The WQ to destroy.
+ * @udata: Valid user data
  */
-int ib_destroy_wq(struct ib_wq *wq)
+int ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata)
 {
 	int err;
 	struct ib_cq *cq = wq->cq;
@@ -2222,7 +2233,7 @@ int ib_destroy_wq(struct ib_wq *wq)
 	if (atomic_read(&wq->usecnt))
 		return -EBUSY;
 
-	err = wq->device->ops.destroy_wq(wq);
+	err = wq->device->ops.destroy_wq(wq, udata);
 	if (!err) {
 		atomic_dec(&pd->usecnt);
 		atomic_dec(&cq->usecnt);
@@ -2701,3 +2712,37 @@ int rdma_init_netdev(struct ib_device *device, u8 port_num,
 					     netdev, params.param);
 }
 EXPORT_SYMBOL(rdma_init_netdev);
+
+void __rdma_block_iter_start(struct ib_block_iter *biter,
+			     struct scatterlist *sglist, unsigned int nents,
+			     unsigned long pgsz)
+{
+	memset(biter, 0, sizeof(struct ib_block_iter));
+	biter->__sg = sglist;
+	biter->__sg_nents = nents;
+
+	/* Driver provides best block size to use */
+	biter->__pg_bit = __fls(pgsz);
+}
+EXPORT_SYMBOL(__rdma_block_iter_start);
+
+bool __rdma_block_iter_next(struct ib_block_iter *biter)
+{
+	unsigned int block_offset;
+
+	if (!biter->__sg_nents || !biter->__sg)
+		return false;
+
+	biter->__dma_addr = sg_dma_address(biter->__sg) + biter->__sg_advance;
+	block_offset = biter->__dma_addr & (BIT_ULL(biter->__pg_bit) - 1);
+	biter->__sg_advance += BIT_ULL(biter->__pg_bit) - block_offset;
+
+	if (biter->__sg_advance >= sg_dma_len(biter->__sg)) {
+		biter->__sg_advance = 0;
+		biter->__sg = sg_next(biter->__sg);
+		biter->__sg_nents--;
+	}
+
+	return true;
+}
+EXPORT_SYMBOL(__rdma_block_iter_next);
diff --git a/drivers/infiniband/hw/Makefile b/drivers/infiniband/hw/Makefile
index e4f31c1be8f7..77094be1b262 100644
--- a/drivers/infiniband/hw/Makefile
+++ b/drivers/infiniband/hw/Makefile
@@ -3,6 +3,7 @@ obj-$(CONFIG_INFINIBAND_MTHCA)		+= mthca/
 obj-$(CONFIG_INFINIBAND_QIB)		+= qib/
 obj-$(CONFIG_INFINIBAND_CXGB3)		+= cxgb3/
 obj-$(CONFIG_INFINIBAND_CXGB4)		+= cxgb4/
+obj-$(CONFIG_INFINIBAND_EFA)		+= efa/
 obj-$(CONFIG_INFINIBAND_I40IW)		+= i40iw/
 obj-$(CONFIG_MLX4_INFINIBAND)		+= mlx4/
 obj-$(CONFIG_MLX5_INFINIBAND)		+= mlx5/
diff --git a/drivers/infiniband/hw/bnxt_re/Kconfig b/drivers/infiniband/hw/bnxt_re/Kconfig
index d25439c305f7..51e8234520a9 100644
--- a/drivers/infiniband/hw/bnxt_re/Kconfig
+++ b/drivers/infiniband/hw/bnxt_re/Kconfig
@@ -1,10 +1,10 @@
 config INFINIBAND_BNXT_RE
-    tristate "Broadcom Netxtreme HCA support"
-    depends on 64BIT
-    depends on ETHERNET && NETDEVICES && PCI && INET && DCB
-    select NET_VENDOR_BROADCOM
-    select BNXT
-    ---help---
+        tristate "Broadcom Netxtreme HCA support"
+        depends on 64BIT
+        depends on ETHERNET && NETDEVICES && PCI && INET && DCB
+        select NET_VENDOR_BROADCOM
+        select BNXT
+        ---help---
 	  This driver supports Broadcom NetXtreme-E 10/25/40/50 gigabit
 	  RoCE HCAs.  To compile this driver as a module, choose M here:
 	  the module will be called bnxt_re.
diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
index 071b2fc38b0b..2c3685faa57a 100644
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
@@ -119,21 +119,6 @@ static int bnxt_re_build_sgl(struct ib_sge *ib_sg_list,
 }
 
 /* Device */
-struct net_device *bnxt_re_get_netdev(struct ib_device *ibdev, u8 port_num)
-{
-	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
-	struct net_device *netdev = NULL;
-
-	rcu_read_lock();
-	if (rdev)
-		netdev = rdev->netdev;
-	if (netdev)
-		dev_hold(netdev);
-
-	rcu_read_unlock();
-	return netdev;
-}
-
 int bnxt_re_query_device(struct ib_device *ibdev,
 			 struct ib_device_attr *ib_attr,
 			 struct ib_udata *udata)
@@ -375,8 +360,9 @@ int bnxt_re_add_gid(const struct ib_gid_attr *attr, void **context)
 	struct bnxt_re_dev *rdev = to_bnxt_re_dev(attr->device, ibdev);
 	struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
 
-	if ((attr->ndev) && is_vlan_dev(attr->ndev))
-		vlan_id = vlan_dev_vlan_id(attr->ndev);
+	rc = rdma_read_gid_l2_fields(attr, &vlan_id, NULL);
+	if (rc)
+		return rc;
 
 	rc = bnxt_qplib_add_sgid(sgid_tbl, (struct bnxt_qplib_gid *)&attr->gid,
 				 rdev->qplib_res.netdev->dev_addr,
@@ -564,7 +550,7 @@ fail:
 }
 
 /* Protection Domains */
-void bnxt_re_dealloc_pd(struct ib_pd *ib_pd)
+void bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata)
 {
 	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
 	struct bnxt_re_dev *rdev = pd->rdev;
@@ -576,14 +562,12 @@ void bnxt_re_dealloc_pd(struct ib_pd *ib_pd)
 				      &pd->qplib_pd);
 }
 
-int bnxt_re_alloc_pd(struct ib_pd *ibpd, struct ib_ucontext *ucontext,
-		     struct ib_udata *udata)
+int bnxt_re_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 {
 	struct ib_device *ibdev = ibpd->device;
 	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
-	struct bnxt_re_ucontext *ucntx = container_of(ucontext,
-						      struct bnxt_re_ucontext,
-						      ib_uctx);
+	struct bnxt_re_ucontext *ucntx = rdma_udata_to_drv_context(
+		udata, struct bnxt_re_ucontext, ib_uctx);
 	struct bnxt_re_pd *pd = container_of(ibpd, struct bnxt_re_pd, ib_pd);
 	int rc;
 
@@ -635,20 +619,13 @@ fail:
 }
 
 /* Address Handles */
-int bnxt_re_destroy_ah(struct ib_ah *ib_ah, u32 flags)
+void bnxt_re_destroy_ah(struct ib_ah *ib_ah, u32 flags)
 {
 	struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
 	struct bnxt_re_dev *rdev = ah->rdev;
-	int rc;
 
-	rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, &ah->qplib_ah,
-				   !(flags & RDMA_DESTROY_AH_SLEEPABLE));
-	if (rc) {
-		dev_err(rdev_to_dev(rdev), "Failed to destroy HW AH");
-		return rc;
-	}
-	kfree(ah);
-	return 0;
+	bnxt_qplib_destroy_ah(&rdev->qplib_res, &ah->qplib_ah,
+			      !(flags & RDMA_DESTROY_AH_SLEEPABLE));
 }
 
 static u8 bnxt_re_stack_to_dev_nw_type(enum rdma_network_type ntype)
@@ -669,26 +646,22 @@ static u8 bnxt_re_stack_to_dev_nw_type(enum rdma_network_type ntype)
 	return nw_type;
 }
 
-struct ib_ah *bnxt_re_create_ah(struct ib_pd *ib_pd,
-				struct rdma_ah_attr *ah_attr,
-				u32 flags,
-				struct ib_udata *udata)
+int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr,
+		      u32 flags, struct ib_udata *udata)
 {
+	struct ib_pd *ib_pd = ib_ah->pd;
 	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
 	const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr);
 	struct bnxt_re_dev *rdev = pd->rdev;
 	const struct ib_gid_attr *sgid_attr;
-	struct bnxt_re_ah *ah;
+	struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
 	u8 nw_type;
 	int rc;
 
 	if (!(rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH)) {
 		dev_err(rdev_to_dev(rdev), "Failed to alloc AH: GRH not set");
-		return ERR_PTR(-EINVAL);
+		return -EINVAL;
 	}
-	ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
-	if (!ah)
-		return ERR_PTR(-ENOMEM);
 
 	ah->rdev = rdev;
 	ah->qplib_ah.pd = &pd->qplib_pd;
@@ -718,7 +691,7 @@ struct ib_ah *bnxt_re_create_ah(struct ib_pd *ib_pd,
 				  !(flags & RDMA_CREATE_AH_SLEEPABLE));
 	if (rc) {
 		dev_err(rdev_to_dev(rdev), "Failed to allocate HW AH");
-		goto fail;
+		return rc;
 	}
 
 	/* Write AVID to shared page. */
@@ -735,11 +708,7 @@ struct ib_ah *bnxt_re_create_ah(struct ib_pd *ib_pd,
 		spin_unlock_irqrestore(&uctx->sh_lock, flag);
 	}
 
-	return &ah->ib_ah;
-
-fail:
-	kfree(ah);
-	return ERR_PTR(rc);
+	return 0;
 }
 
 int bnxt_re_modify_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr)
@@ -789,7 +758,7 @@ void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp,
 }
 
 /* Queue Pairs */
-int bnxt_re_destroy_qp(struct ib_qp *ib_qp)
+int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata)
 {
 	struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
 	struct bnxt_re_dev *rdev = qp->rdev;
@@ -812,13 +781,8 @@ int bnxt_re_destroy_qp(struct ib_qp *ib_qp)
 	bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp);
 
 	if (ib_qp->qp_type == IB_QPT_GSI && rdev->qp1_sqp) {
-		rc = bnxt_qplib_destroy_ah(&rdev->qplib_res,
-					   &rdev->sqp_ah->qplib_ah, false);
-		if (rc) {
-			dev_err(rdev_to_dev(rdev),
-				"Failed to destroy HW AH for shadow QP");
-			return rc;
-		}
+		bnxt_qplib_destroy_ah(&rdev->qplib_res, &rdev->sqp_ah->qplib_ah,
+				      false);
 
 		bnxt_qplib_clean_qp(&qp->qplib_qp);
 		rc = bnxt_qplib_destroy_qp(&rdev->qplib_res,
@@ -895,8 +859,9 @@ static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd,
 		return PTR_ERR(umem);
 
 	qp->sumem = umem;
-	qplib_qp->sq.sglist = umem->sg_head.sgl;
-	qplib_qp->sq.nmap = umem->nmap;
+	qplib_qp->sq.sg_info.sglist = umem->sg_head.sgl;
+	qplib_qp->sq.sg_info.npages = ib_umem_num_pages(umem);
+	qplib_qp->sq.sg_info.nmap = umem->nmap;
 	qplib_qp->qp_handle = ureq.qp_handle;
 
 	if (!qp->qplib_qp.srq) {
@@ -907,8 +872,9 @@ static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd,
 		if (IS_ERR(umem))
 			goto rqfail;
 		qp->rumem = umem;
-		qplib_qp->rq.sglist = umem->sg_head.sgl;
-		qplib_qp->rq.nmap = umem->nmap;
+		qplib_qp->rq.sg_info.sglist = umem->sg_head.sgl;
+		qplib_qp->rq.sg_info.npages = ib_umem_num_pages(umem);
+		qplib_qp->rq.sg_info.nmap = umem->nmap;
 	}
 
 	qplib_qp->dpi = &cntx->dpi;
@@ -916,8 +882,7 @@ static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd,
 rqfail:
 	ib_umem_release(qp->sumem);
 	qp->sumem = NULL;
-	qplib_qp->sq.sglist = NULL;
-	qplib_qp->sq.nmap = 0;
+	memset(&qplib_qp->sq.sg_info, 0, sizeof(qplib_qp->sq.sg_info));
 
 	return PTR_ERR(umem);
 }
@@ -1326,30 +1291,22 @@ static enum ib_mtu __to_ib_mtu(u32 mtu)
 }
 
 /* Shared Receive Queues */
-int bnxt_re_destroy_srq(struct ib_srq *ib_srq)
+void bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata)
 {
 	struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
 					       ib_srq);
 	struct bnxt_re_dev *rdev = srq->rdev;
 	struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq;
 	struct bnxt_qplib_nq *nq = NULL;
-	int rc;
 
 	if (qplib_srq->cq)
 		nq = qplib_srq->cq->nq;
-	rc = bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq);
-	if (rc) {
-		dev_err(rdev_to_dev(rdev), "Destroy HW SRQ failed!");
-		return rc;
-	}
-
+	bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq);
 	if (srq->umem)
 		ib_umem_release(srq->umem);
-	kfree(srq);
 	atomic_dec(&rdev->srq_count);
 	if (nq)
 		nq->budget--;
-	return 0;
 }
 
 static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev,
@@ -1374,22 +1331,25 @@ static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev,
 		return PTR_ERR(umem);
 
 	srq->umem = umem;
-	qplib_srq->nmap = umem->nmap;
-	qplib_srq->sglist = umem->sg_head.sgl;
+	qplib_srq->sg_info.sglist = umem->sg_head.sgl;
+	qplib_srq->sg_info.npages = ib_umem_num_pages(umem);
+	qplib_srq->sg_info.nmap = umem->nmap;
 	qplib_srq->srq_handle = ureq.srq_handle;
 	qplib_srq->dpi = &cntx->dpi;
 
 	return 0;
 }
 
-struct ib_srq *bnxt_re_create_srq(struct ib_pd *ib_pd,
-				  struct ib_srq_init_attr *srq_init_attr,
-				  struct ib_udata *udata)
+int bnxt_re_create_srq(struct ib_srq *ib_srq,
+		       struct ib_srq_init_attr *srq_init_attr,
+		       struct ib_udata *udata)
 {
+	struct ib_pd *ib_pd = ib_srq->pd;
 	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
 	struct bnxt_re_dev *rdev = pd->rdev;
 	struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
-	struct bnxt_re_srq *srq;
+	struct bnxt_re_srq *srq =
+		container_of(ib_srq, struct bnxt_re_srq, ib_srq);
 	struct bnxt_qplib_nq *nq = NULL;
 	int rc, entries;
 
@@ -1404,11 +1364,6 @@ struct ib_srq *bnxt_re_create_srq(struct ib_pd *ib_pd,
 		goto exit;
 	}
 
-	srq = kzalloc(sizeof(*srq), GFP_KERNEL);
-	if (!srq) {
-		rc = -ENOMEM;
-		goto exit;
-	}
 	srq->rdev = rdev;
 	srq->qplib_srq.pd = &pd->qplib_pd;
 	srq->qplib_srq.dpi = &rdev->dpi_privileged;
@@ -1454,14 +1409,13 @@ struct ib_srq *bnxt_re_create_srq(struct ib_pd *ib_pd,
 		nq->budget++;
 	atomic_inc(&rdev->srq_count);
 
-	return &srq->ib_srq;
+	return 0;
 
 fail:
 	if (srq->umem)
 		ib_umem_release(srq->umem);
-	kfree(srq);
 exit:
-	return ERR_PTR(rc);
+	return rc;
 }
 
 int bnxt_re_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr,
@@ -1684,8 +1638,11 @@ int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
 				qp_attr->ah_attr.roce.dmac);
 
 		sgid_attr = qp_attr->ah_attr.grh.sgid_attr;
-		memcpy(qp->qplib_qp.smac, sgid_attr->ndev->dev_addr,
-		       ETH_ALEN);
+		rc = rdma_read_gid_l2_fields(sgid_attr, NULL,
+					     &qp->qplib_qp.smac[0]);
+		if (rc)
+			return rc;
+
 		nw_type = rdma_gid_attr_network_type(sgid_attr);
 		switch (nw_type) {
 		case RDMA_NETWORK_IPV4:
@@ -1904,8 +1861,10 @@ static int bnxt_re_build_qp1_send_v2(struct bnxt_re_qp *qp,
 
 	memset(&qp->qp1_hdr, 0, sizeof(qp->qp1_hdr));
 
-	if (is_vlan_dev(sgid_attr->ndev))
-		vlan_id = vlan_dev_vlan_id(sgid_attr->ndev);
+	rc = rdma_read_gid_l2_fields(sgid_attr, &vlan_id, NULL);
+	if (rc)
+		return rc;
+
 	/* Get network header type for this GID */
 	nw_type = rdma_gid_attr_network_type(sgid_attr);
 	switch (nw_type) {
@@ -2558,7 +2517,7 @@ int bnxt_re_post_recv(struct ib_qp *ib_qp, const struct ib_recv_wr *wr,
 }
 
 /* Completion Queues */
-int bnxt_re_destroy_cq(struct ib_cq *ib_cq)
+int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
 {
 	int rc;
 	struct bnxt_re_cq *cq;
@@ -2587,7 +2546,6 @@ int bnxt_re_destroy_cq(struct ib_cq *ib_cq)
 
 struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev,
 				const struct ib_cq_init_attr *attr,
-				struct ib_ucontext *context,
 				struct ib_udata *udata)
 {
 	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
@@ -2614,12 +2572,10 @@ struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev,
 	if (entries > dev_attr->max_cq_wqes + 1)
 		entries = dev_attr->max_cq_wqes + 1;
 
-	if (context) {
+	if (udata) {
 		struct bnxt_re_cq_req req;
-		struct bnxt_re_ucontext *uctx = container_of
-						(context,
-						 struct bnxt_re_ucontext,
-						 ib_uctx);
+		struct bnxt_re_ucontext *uctx = rdma_udata_to_drv_context(
+			udata, struct bnxt_re_ucontext, ib_uctx);
 		if (ib_copy_from_udata(&req, udata, sizeof(req))) {
 			rc = -EFAULT;
 			goto fail;
@@ -2632,8 +2588,9 @@ struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev,
 			rc = PTR_ERR(cq->umem);
 			goto fail;
 		}
-		cq->qplib_cq.sghead = cq->umem->sg_head.sgl;
-		cq->qplib_cq.nmap = cq->umem->nmap;
+		cq->qplib_cq.sg_info.sglist = cq->umem->sg_head.sgl;
+		cq->qplib_cq.sg_info.npages = ib_umem_num_pages(cq->umem);
+		cq->qplib_cq.sg_info.nmap = cq->umem->nmap;
 		cq->qplib_cq.dpi = &uctx->dpi;
 	} else {
 		cq->max_cql = min_t(u32, entries, MAX_CQL_PER_POLL);
@@ -2645,8 +2602,6 @@ struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev,
 		}
 
 		cq->qplib_cq.dpi = &rdev->dpi_privileged;
-		cq->qplib_cq.sghead = NULL;
-		cq->qplib_cq.nmap = 0;
 	}
 	/*
 	 * Allocating the NQ in a round robin fashion. nq_alloc_cnt is a
@@ -2671,7 +2626,7 @@ struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev,
 	atomic_inc(&rdev->cq_count);
 	spin_lock_init(&cq->cq_lock);
 
-	if (context) {
+	if (udata) {
 		struct bnxt_re_cq_resp resp;
 
 		resp.cqid = cq->qplib_cq.id;
@@ -2689,7 +2644,7 @@ struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev,
 	return &cq->ib_cq;
 
 c2fail:
-	if (context)
+	if (udata)
 		ib_umem_release(cq->umem);
 fail:
 	kfree(cq->cql);
@@ -3381,7 +3336,7 @@ fail:
 	return ERR_PTR(rc);
 }
 
-int bnxt_re_dereg_mr(struct ib_mr *ib_mr)
+int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
 {
 	struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr);
 	struct bnxt_re_dev *rdev = mr->rdev;
@@ -3427,7 +3382,7 @@ int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, int sg_nents,
 }
 
 struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type,
-			       u32 max_num_sg)
+			       u32 max_num_sg, struct ib_udata *udata)
 {
 	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
 	struct bnxt_re_dev *rdev = pd->rdev;
@@ -3552,17 +3507,12 @@ static int fill_umem_pbl_tbl(struct ib_umem *umem, u64 *pbl_tbl_orig,
 			     int page_shift)
 {
 	u64 *pbl_tbl = pbl_tbl_orig;
-	u64 paddr;
-	u64 page_mask = (1ULL << page_shift) - 1;
-	struct sg_dma_page_iter sg_iter;
+	u64 page_size =  BIT_ULL(page_shift);
+	struct ib_block_iter biter;
+
+	rdma_for_each_block(umem->sg_head.sgl, &biter, umem->nmap, page_size)
+		*pbl_tbl++ = rdma_block_iter_dma_address(&biter);
 
-	for_each_sg_dma_page (umem->sg_head.sgl, &sg_iter, umem->nmap, 0) {
-		paddr = sg_page_iter_dma_address(&sg_iter);
-		if (pbl_tbl == pbl_tbl_orig)
-			*pbl_tbl++ = paddr & ~page_mask;
-		else if ((paddr & page_mask) == 0)
-			*pbl_tbl++ = paddr;
-	}
 	return pbl_tbl - pbl_tbl_orig;
 }
 
@@ -3624,7 +3574,9 @@ struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length,
 		goto free_umem;
 	}
 
-	page_shift = PAGE_SHIFT;
+	page_shift = __ffs(ib_umem_find_best_pgsz(umem,
+				BNXT_RE_PAGE_SIZE_4K | BNXT_RE_PAGE_SIZE_2M,
+				virt_addr));
 
 	if (!bnxt_re_page_size_ok(page_shift)) {
 		dev_err(rdev_to_dev(rdev), "umem page size unsupported!");
@@ -3632,17 +3584,13 @@ struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length,
 		goto fail;
 	}
 
-	if (!umem->hugetlb && length > BNXT_RE_MAX_MR_SIZE_LOW) {
+	if (page_shift == BNXT_RE_PAGE_SHIFT_4K &&
+	    length > BNXT_RE_MAX_MR_SIZE_LOW) {
 		dev_err(rdev_to_dev(rdev), "Requested MR Sz:%llu Max sup:%llu",
 			length,	(u64)BNXT_RE_MAX_MR_SIZE_LOW);
 		rc = -EINVAL;
 		goto fail;
 	}
-	if (umem->hugetlb && length > BNXT_RE_PAGE_SIZE_2M) {
-		page_shift = BNXT_RE_PAGE_SHIFT_2M;
-		dev_warn(rdev_to_dev(rdev), "umem hugetlb set page_size %x",
-			 1 << page_shift);
-	}
 
 	/* Map umem buf ptrs to the PBL */
 	umem_pgs = fill_umem_pbl_tbl(umem, pbl_tbl, page_shift);
@@ -3709,7 +3657,7 @@ int bnxt_re_alloc_ucontext(struct ib_ucontext *ctx, struct ib_udata *udata)
 	resp.chip_id0 = chip_met_rev_num;
 	/* Future extension of chip info */
 	resp.chip_id1 = 0;
-	/*Temp, Use idr_alloc instead */
+	/*Temp, Use xa_alloc instead */
 	resp.dev_id = rdev->en_dev->pdev->devfn;
 	resp.max_qp = rdev->qplib_ctx.qpc_count;
 	resp.pg_size = PAGE_SIZE;
diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.h b/drivers/infiniband/hw/bnxt_re/ib_verbs.h
index e45465ed4eee..09a33049e42f 100644
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.h
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.h
@@ -63,15 +63,15 @@ struct bnxt_re_pd {
 };
 
 struct bnxt_re_ah {
-	struct bnxt_re_dev	*rdev;
 	struct ib_ah		ib_ah;
+	struct bnxt_re_dev	*rdev;
 	struct bnxt_qplib_ah	qplib_ah;
 };
 
 struct bnxt_re_srq {
+	struct ib_srq		ib_srq;
 	struct bnxt_re_dev	*rdev;
 	u32			srq_limit;
-	struct ib_srq		ib_srq;
 	struct bnxt_qplib_srq	qplib_srq;
 	struct ib_umem		*umem;
 	spinlock_t		lock;		/* protect srq */
@@ -142,8 +142,6 @@ struct bnxt_re_ucontext {
 	spinlock_t		sh_lock;	/* protect shpg */
 };
 
-struct net_device *bnxt_re_get_netdev(struct ib_device *ibdev, u8 port_num);
-
 int bnxt_re_query_device(struct ib_device *ibdev,
 			 struct ib_device_attr *ib_attr,
 			 struct ib_udata *udata);
@@ -163,24 +161,21 @@ int bnxt_re_query_gid(struct ib_device *ibdev, u8 port_num,
 		      int index, union ib_gid *gid);
 enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev,
 					    u8 port_num);
-int bnxt_re_alloc_pd(struct ib_pd *pd, struct ib_ucontext *context,
-		     struct ib_udata *udata);
-void bnxt_re_dealloc_pd(struct ib_pd *pd);
-struct ib_ah *bnxt_re_create_ah(struct ib_pd *pd,
-				struct rdma_ah_attr *ah_attr,
-				u32 flags,
-				struct ib_udata *udata);
+int bnxt_re_alloc_pd(struct ib_pd *pd, struct ib_udata *udata);
+void bnxt_re_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata);
+int bnxt_re_create_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr, u32 flags,
+		      struct ib_udata *udata);
 int bnxt_re_modify_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);
 int bnxt_re_query_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);
-int bnxt_re_destroy_ah(struct ib_ah *ah, u32 flags);
-struct ib_srq *bnxt_re_create_srq(struct ib_pd *pd,
-				  struct ib_srq_init_attr *srq_init_attr,
-				  struct ib_udata *udata);
+void bnxt_re_destroy_ah(struct ib_ah *ah, u32 flags);
+int bnxt_re_create_srq(struct ib_srq *srq,
+		       struct ib_srq_init_attr *srq_init_attr,
+		       struct ib_udata *udata);
 int bnxt_re_modify_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr,
 		       enum ib_srq_attr_mask srq_attr_mask,
 		       struct ib_udata *udata);
 int bnxt_re_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr);
-int bnxt_re_destroy_srq(struct ib_srq *srq);
+void bnxt_re_destroy_srq(struct ib_srq *srq, struct ib_udata *udata);
 int bnxt_re_post_srq_recv(struct ib_srq *srq, const struct ib_recv_wr *recv_wr,
 			  const struct ib_recv_wr **bad_recv_wr);
 struct ib_qp *bnxt_re_create_qp(struct ib_pd *pd,
@@ -190,16 +185,15 @@ int bnxt_re_modify_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
 		      int qp_attr_mask, struct ib_udata *udata);
 int bnxt_re_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
 		     int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr);
-int bnxt_re_destroy_qp(struct ib_qp *qp);
+int bnxt_re_destroy_qp(struct ib_qp *qp, struct ib_udata *udata);
 int bnxt_re_post_send(struct ib_qp *qp, const struct ib_send_wr *send_wr,
 		      const struct ib_send_wr **bad_send_wr);
 int bnxt_re_post_recv(struct ib_qp *qp, const struct ib_recv_wr *recv_wr,
 		      const struct ib_recv_wr **bad_recv_wr);
 struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev,
 				const struct ib_cq_init_attr *attr,
-				struct ib_ucontext *context,
 				struct ib_udata *udata);
-int bnxt_re_destroy_cq(struct ib_cq *cq);
+int bnxt_re_destroy_cq(struct ib_cq *cq, struct ib_udata *udata);
 int bnxt_re_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc);
 int bnxt_re_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags);
 struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *pd, int mr_access_flags);
@@ -207,8 +201,8 @@ struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *pd, int mr_access_flags);
 int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, int sg_nents,
 		      unsigned int *sg_offset);
 struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type mr_type,
-			       u32 max_num_sg);
-int bnxt_re_dereg_mr(struct ib_mr *mr);
+			       u32 max_num_sg, struct ib_udata *udata);
+int bnxt_re_dereg_mr(struct ib_mr *mr, struct ib_udata *udata);
 struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type,
 			       struct ib_udata *udata);
 int bnxt_re_dealloc_mw(struct ib_mw *mw);
diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c
index 2bd24ac45ee4..814f959c7db9 100644
--- a/drivers/infiniband/hw/bnxt_re/main.c
+++ b/drivers/infiniband/hw/bnxt_re/main.c
@@ -617,7 +617,6 @@ static const struct ib_device_ops bnxt_re_dev_ops = {
 	.get_dma_mr = bnxt_re_get_dma_mr,
 	.get_hw_stats = bnxt_re_ib_get_hw_stats,
 	.get_link_layer = bnxt_re_get_link_layer,
-	.get_netdev = bnxt_re_get_netdev,
 	.get_port_immutable = bnxt_re_get_port_immutable,
 	.map_mr_sg = bnxt_re_map_mr_sg,
 	.mmap = bnxt_re_mmap,
@@ -637,13 +636,16 @@ static const struct ib_device_ops bnxt_re_dev_ops = {
 	.query_srq = bnxt_re_query_srq,
 	.reg_user_mr = bnxt_re_reg_user_mr,
 	.req_notify_cq = bnxt_re_req_notify_cq,
+	INIT_RDMA_OBJ_SIZE(ib_ah, bnxt_re_ah, ib_ah),
 	INIT_RDMA_OBJ_SIZE(ib_pd, bnxt_re_pd, ib_pd),
+	INIT_RDMA_OBJ_SIZE(ib_srq, bnxt_re_srq, ib_srq),
 	INIT_RDMA_OBJ_SIZE(ib_ucontext, bnxt_re_ucontext, ib_uctx),
 };
 
 static int bnxt_re_register_ib(struct bnxt_re_dev *rdev)
 {
 	struct ib_device *ibdev = &rdev->ibdev;
+	int ret;
 
 	/* ib device init */
 	ibdev->owner = THIS_MODULE;
@@ -691,6 +693,10 @@ static int bnxt_re_register_ib(struct bnxt_re_dev *rdev)
 	rdma_set_device_sysfs_group(ibdev, &bnxt_re_dev_attr_group);
 	ibdev->driver_id = RDMA_DRIVER_BNXT_RE;
 	ib_set_device_ops(ibdev, &bnxt_re_dev_ops);
+	ret = ib_device_set_netdev(&rdev->ibdev, rdev->netdev, 1);
+	if (ret)
+		return ret;
+
 	return ib_register_device(ibdev, "bnxt_re%d");
 }
 
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
index 71c34d5b0ac0..958c1ff9c515 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c
@@ -478,7 +478,7 @@ int bnxt_qplib_alloc_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq)
 	    nq->hwq.max_elements > BNXT_QPLIB_NQE_MAX_CNT)
 		nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT;
 	hwq_type = bnxt_qplib_get_hwq_type(nq->res);
-	if (bnxt_qplib_alloc_init_hwq(nq->pdev, &nq->hwq, NULL, 0,
+	if (bnxt_qplib_alloc_init_hwq(nq->pdev, &nq->hwq, NULL,
 				      &nq->hwq.max_elements,
 				      BNXT_QPLIB_MAX_NQE_ENTRY_SIZE, 0,
 				      PAGE_SIZE, hwq_type))
@@ -507,7 +507,7 @@ static void bnxt_qplib_arm_srq(struct bnxt_qplib_srq *srq, u32 arm_type)
 	writeq(val, db);
 }
 
-int bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res,
+void bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res,
 			   struct bnxt_qplib_srq *srq)
 {
 	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
@@ -521,14 +521,12 @@ int bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res,
 	/* Configure the request */
 	req.srq_cid = cpu_to_le32(srq->id);
 
-	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
-					  (void *)&resp, NULL, 0);
+	rc = bnxt_qplib_rcfw_send_message(rcfw, (struct cmdq_base *)&req,
+					  (struct creq_base *)&resp, NULL, 0);
+	kfree(srq->swq);
 	if (rc)
-		return rc;
-
+		return;
 	bnxt_qplib_free_hwq(res->pdev, &srq->hwq);
-	kfree(srq->swq);
-	return 0;
 }
 
 int bnxt_qplib_create_srq(struct bnxt_qplib_res *res,
@@ -542,8 +540,8 @@ int bnxt_qplib_create_srq(struct bnxt_qplib_res *res,
 	int rc, idx;
 
 	srq->hwq.max_elements = srq->max_wqe;
-	rc = bnxt_qplib_alloc_init_hwq(res->pdev, &srq->hwq, srq->sglist,
-				       srq->nmap, &srq->hwq.max_elements,
+	rc = bnxt_qplib_alloc_init_hwq(res->pdev, &srq->hwq, &srq->sg_info,
+				       &srq->hwq.max_elements,
 				       BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0,
 				       PAGE_SIZE, HWQ_TYPE_QUEUE);
 	if (rc)
@@ -742,7 +740,7 @@ int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 
 	/* SQ */
 	sq->hwq.max_elements = sq->max_wqe;
-	rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, NULL, 0,
+	rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, NULL,
 				       &sq->hwq.max_elements,
 				       BNXT_QPLIB_MAX_SQE_ENTRY_SIZE, 0,
 				       PAGE_SIZE, HWQ_TYPE_QUEUE);
@@ -781,7 +779,7 @@ int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 	/* RQ */
 	if (rq->max_wqe) {
 		rq->hwq.max_elements = qp->rq.max_wqe;
-		rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, NULL, 0,
+		rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, NULL,
 					       &rq->hwq.max_elements,
 					       BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0,
 					       PAGE_SIZE, HWQ_TYPE_QUEUE);
@@ -890,8 +888,8 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 			 sizeof(struct sq_psn_search);
 	}
 	sq->hwq.max_elements = sq->max_wqe;
-	rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, sq->sglist,
-				       sq->nmap, &sq->hwq.max_elements,
+	rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, &sq->sg_info,
+				       &sq->hwq.max_elements,
 				       BNXT_QPLIB_MAX_SQE_ENTRY_SIZE,
 				       psn_sz,
 				       PAGE_SIZE, HWQ_TYPE_QUEUE);
@@ -959,8 +957,9 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 	/* RQ */
 	if (rq->max_wqe) {
 		rq->hwq.max_elements = rq->max_wqe;
-		rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, rq->sglist,
-					       rq->nmap, &rq->hwq.max_elements,
+		rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq,
+					       &rq->sg_info,
+					       &rq->hwq.max_elements,
 					       BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0,
 					       PAGE_SIZE, HWQ_TYPE_QUEUE);
 		if (rc)
@@ -1030,7 +1029,7 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 		req_size = xrrq->max_elements *
 			   BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE + PAGE_SIZE - 1;
 		req_size &= ~(PAGE_SIZE - 1);
-		rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL, 0,
+		rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL,
 					       &xrrq->max_elements,
 					       BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE,
 					       0, req_size, HWQ_TYPE_CTX);
@@ -1046,7 +1045,7 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
 			   BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE + PAGE_SIZE - 1;
 		req_size &= ~(PAGE_SIZE - 1);
 
-		rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL, 0,
+		rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL,
 					       &xrrq->max_elements,
 					       BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE,
 					       0, req_size, HWQ_TYPE_CTX);
@@ -1935,8 +1934,8 @@ int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
 	int rc;
 
 	cq->hwq.max_elements = cq->max_wqe;
-	rc = bnxt_qplib_alloc_init_hwq(res->pdev, &cq->hwq, cq->sghead,
-				       cq->nmap, &cq->hwq.max_elements,
+	rc = bnxt_qplib_alloc_init_hwq(res->pdev, &cq->hwq, &cq->sg_info,
+				       &cq->hwq.max_elements,
 				       BNXT_QPLIB_MAX_CQE_ENTRY_SIZE, 0,
 				       PAGE_SIZE, HWQ_TYPE_QUEUE);
 	if (rc)
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.h b/drivers/infiniband/hw/bnxt_re/qplib_fp.h
index 3f618b5f1f06..99e0a13cbefa 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_fp.h
+++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.h
@@ -52,10 +52,9 @@ struct bnxt_qplib_srq {
 	struct bnxt_qplib_cq		*cq;
 	struct bnxt_qplib_hwq		hwq;
 	struct bnxt_qplib_swq		*swq;
-	struct scatterlist		*sglist;
 	int				start_idx;
 	int				last_idx;
-	u32				nmap;
+	struct bnxt_qplib_sg_info	sg_info;
 	u16				eventq_hw_ring_id;
 	spinlock_t			lock; /* protect SRQE link list */
 };
@@ -237,8 +236,7 @@ struct bnxt_qplib_swqe {
 struct bnxt_qplib_q {
 	struct bnxt_qplib_hwq		hwq;
 	struct bnxt_qplib_swq		*swq;
-	struct scatterlist		*sglist;
-	u32				nmap;
+	struct bnxt_qplib_sg_info	sg_info;
 	u32				max_wqe;
 	u16				q_full_delta;
 	u16				max_sge;
@@ -381,8 +379,7 @@ struct bnxt_qplib_cq {
 	u32				cnq_hw_ring_id;
 	struct bnxt_qplib_nq		*nq;
 	bool				resize_in_progress;
-	struct scatterlist		*sghead;
-	u32				nmap;
+	struct bnxt_qplib_sg_info	sg_info;
 	u64				cq_handle;
 
 #define CQ_RESIZE_WAIT_TIME_MS		500
@@ -521,8 +518,8 @@ int bnxt_qplib_modify_srq(struct bnxt_qplib_res *res,
 			  struct bnxt_qplib_srq *srq);
 int bnxt_qplib_query_srq(struct bnxt_qplib_res *res,
 			 struct bnxt_qplib_srq *srq);
-int bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res,
-			   struct bnxt_qplib_srq *srq);
+void bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res,
+			    struct bnxt_qplib_srq *srq);
 int bnxt_qplib_post_srq_recv(struct bnxt_qplib_srq *srq,
 			     struct bnxt_qplib_swqe *wqe);
 int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp);
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c
index c6461e957078..48b04d2f175f 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c
@@ -569,7 +569,7 @@ int bnxt_qplib_alloc_rcfw_channel(struct pci_dev *pdev,
 	rcfw->pdev = pdev;
 	rcfw->creq.max_elements = BNXT_QPLIB_CREQE_MAX_CNT;
 	hwq_type = bnxt_qplib_get_hwq_type(rcfw->res);
-	if (bnxt_qplib_alloc_init_hwq(rcfw->pdev, &rcfw->creq, NULL, 0,
+	if (bnxt_qplib_alloc_init_hwq(rcfw->pdev, &rcfw->creq, NULL,
 				      &rcfw->creq.max_elements,
 				      BNXT_QPLIB_CREQE_UNITS,
 				      0, PAGE_SIZE, hwq_type)) {
@@ -584,7 +584,7 @@ int bnxt_qplib_alloc_rcfw_channel(struct pci_dev *pdev,
 
 	rcfw->cmdq.max_elements = rcfw->cmdq_depth;
 	if (bnxt_qplib_alloc_init_hwq
-			(rcfw->pdev, &rcfw->cmdq, NULL, 0,
+			(rcfw->pdev, &rcfw->cmdq, NULL,
 			 &rcfw->cmdq.max_elements,
 			 BNXT_QPLIB_CMDQE_UNITS, 0,
 			 bnxt_qplib_cmdqe_page_size(rcfw->cmdq_depth),
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.c b/drivers/infiniband/hw/bnxt_re/qplib_res.c
index 0bc24f934829..37928b1111df 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_res.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_res.c
@@ -83,7 +83,8 @@ static void __free_pbl(struct pci_dev *pdev, struct bnxt_qplib_pbl *pbl,
 }
 
 static int __alloc_pbl(struct pci_dev *pdev, struct bnxt_qplib_pbl *pbl,
-		       struct scatterlist *sghead, u32 pages, u32 pg_size)
+		       struct scatterlist *sghead, u32 pages,
+		       u32 nmaps, u32 pg_size)
 {
 	struct sg_dma_page_iter sg_iter;
 	bool is_umem = false;
@@ -116,7 +117,7 @@ static int __alloc_pbl(struct pci_dev *pdev, struct bnxt_qplib_pbl *pbl,
 	} else {
 		i = 0;
 		is_umem = true;
-		for_each_sg_dma_page (sghead, &sg_iter, pages, 0) {
+		for_each_sg_dma_page(sghead, &sg_iter, nmaps, 0) {
 			pbl->pg_map_arr[i] = sg_page_iter_dma_address(&sg_iter);
 			pbl->pg_arr[i] = NULL;
 			pbl->pg_count++;
@@ -158,12 +159,13 @@ void bnxt_qplib_free_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq)
 
 /* All HWQs are power of 2 in size */
 int bnxt_qplib_alloc_init_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq,
-			      struct scatterlist *sghead, int nmap,
+			      struct bnxt_qplib_sg_info *sg_info,
 			      u32 *elements, u32 element_size, u32 aux,
 			      u32 pg_size, enum bnxt_qplib_hwq_type hwq_type)
 {
-	u32 pages, slots, size, aux_pages = 0, aux_size = 0;
+	u32 pages, maps, slots, size, aux_pages = 0, aux_size = 0;
 	dma_addr_t *src_phys_ptr, **dst_virt_ptr;
+	struct scatterlist *sghead = NULL;
 	int i, rc;
 
 	hwq->level = PBL_LVL_MAX;
@@ -177,6 +179,9 @@ int bnxt_qplib_alloc_init_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq,
 	}
 	size = roundup_pow_of_two(element_size);
 
+	if (sg_info)
+		sghead = sg_info->sglist;
+
 	if (!sghead) {
 		hwq->is_user = false;
 		pages = (slots * size) / pg_size + aux_pages;
@@ -184,17 +189,20 @@ int bnxt_qplib_alloc_init_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq,
 			pages++;
 		if (!pages)
 			return -EINVAL;
+		maps = 0;
 	} else {
 		hwq->is_user = true;
-		pages = nmap;
+		pages = sg_info->npages;
+		maps = sg_info->nmap;
 	}
 
 	/* Alloc the 1st memory block; can be a PDL/PTL/PBL */
 	if (sghead && (pages == MAX_PBL_LVL_0_PGS))
 		rc = __alloc_pbl(pdev, &hwq->pbl[PBL_LVL_0], sghead,
-				 pages, pg_size);
+				 pages, maps, pg_size);
 	else
-		rc = __alloc_pbl(pdev, &hwq->pbl[PBL_LVL_0], NULL, 1, pg_size);
+		rc = __alloc_pbl(pdev, &hwq->pbl[PBL_LVL_0], NULL,
+				 1, 0, pg_size);
 	if (rc)
 		goto fail;
 
@@ -204,7 +212,8 @@ int bnxt_qplib_alloc_init_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq,
 		if (pages > MAX_PBL_LVL_1_PGS) {
 			/* 2 levels of indirection */
 			rc = __alloc_pbl(pdev, &hwq->pbl[PBL_LVL_1], NULL,
-					 MAX_PBL_LVL_1_PGS_FOR_LVL_2, pg_size);
+					 MAX_PBL_LVL_1_PGS_FOR_LVL_2,
+					 0, pg_size);
 			if (rc)
 				goto fail;
 			/* Fill in lvl0 PBL */
@@ -217,7 +226,7 @@ int bnxt_qplib_alloc_init_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq,
 			hwq->level = PBL_LVL_1;
 
 			rc = __alloc_pbl(pdev, &hwq->pbl[PBL_LVL_2], sghead,
-					 pages, pg_size);
+					 pages, maps, pg_size);
 			if (rc)
 				goto fail;
 
@@ -246,7 +255,7 @@ int bnxt_qplib_alloc_init_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq,
 
 			/* 1 level of indirection */
 			rc = __alloc_pbl(pdev, &hwq->pbl[PBL_LVL_1], sghead,
-					 pages, pg_size);
+					 pages, maps, pg_size);
 			if (rc)
 				goto fail;
 			/* Fill in lvl0 PBL */
@@ -339,7 +348,7 @@ int bnxt_qplib_alloc_ctx(struct pci_dev *pdev,
 
 	/* QPC Tables */
 	ctx->qpc_tbl.max_elements = ctx->qpc_count;
-	rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->qpc_tbl, NULL, 0,
+	rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->qpc_tbl, NULL,
 				       &ctx->qpc_tbl.max_elements,
 				       BNXT_QPLIB_MAX_QP_CTX_ENTRY_SIZE, 0,
 				       PAGE_SIZE, HWQ_TYPE_CTX);
@@ -348,7 +357,7 @@ int bnxt_qplib_alloc_ctx(struct pci_dev *pdev,
 
 	/* MRW Tables */
 	ctx->mrw_tbl.max_elements = ctx->mrw_count;
-	rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->mrw_tbl, NULL, 0,
+	rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->mrw_tbl, NULL,
 				       &ctx->mrw_tbl.max_elements,
 				       BNXT_QPLIB_MAX_MRW_CTX_ENTRY_SIZE, 0,
 				       PAGE_SIZE, HWQ_TYPE_CTX);
@@ -357,7 +366,7 @@ int bnxt_qplib_alloc_ctx(struct pci_dev *pdev,
 
 	/* SRQ Tables */
 	ctx->srqc_tbl.max_elements = ctx->srqc_count;
-	rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->srqc_tbl, NULL, 0,
+	rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->srqc_tbl, NULL,
 				       &ctx->srqc_tbl.max_elements,
 				       BNXT_QPLIB_MAX_SRQ_CTX_ENTRY_SIZE, 0,
 				       PAGE_SIZE, HWQ_TYPE_CTX);
@@ -366,7 +375,7 @@ int bnxt_qplib_alloc_ctx(struct pci_dev *pdev,
 
 	/* CQ Tables */
 	ctx->cq_tbl.max_elements = ctx->cq_count;
-	rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->cq_tbl, NULL, 0,
+	rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->cq_tbl, NULL,
 				       &ctx->cq_tbl.max_elements,
 				       BNXT_QPLIB_MAX_CQ_CTX_ENTRY_SIZE, 0,
 				       PAGE_SIZE, HWQ_TYPE_CTX);
@@ -375,7 +384,7 @@ int bnxt_qplib_alloc_ctx(struct pci_dev *pdev,
 
 	/* TQM Buffer */
 	ctx->tqm_pde.max_elements = 512;
-	rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->tqm_pde, NULL, 0,
+	rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->tqm_pde, NULL,
 				       &ctx->tqm_pde.max_elements, sizeof(u64),
 				       0, PAGE_SIZE, HWQ_TYPE_CTX);
 	if (rc)
@@ -386,7 +395,7 @@ int bnxt_qplib_alloc_ctx(struct pci_dev *pdev,
 			continue;
 		ctx->tqm_tbl[i].max_elements = ctx->qpc_count *
 					       ctx->tqm_count[i];
-		rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->tqm_tbl[i], NULL, 0,
+		rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->tqm_tbl[i], NULL,
 					       &ctx->tqm_tbl[i].max_elements, 1,
 					       0, PAGE_SIZE, HWQ_TYPE_CTX);
 		if (rc)
@@ -424,7 +433,7 @@ int bnxt_qplib_alloc_ctx(struct pci_dev *pdev,
 
 	/* TIM Buffer */
 	ctx->tim_tbl.max_elements = ctx->qpc_count * 16;
-	rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->tim_tbl, NULL, 0,
+	rc = bnxt_qplib_alloc_init_hwq(pdev, &ctx->tim_tbl, NULL,
 				       &ctx->tim_tbl.max_elements, 1,
 				       0, PAGE_SIZE, HWQ_TYPE_CTX);
 	if (rc)
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.h b/drivers/infiniband/hw/bnxt_re/qplib_res.h
index 32cebd0f1436..30c42c92fac7 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_res.h
+++ b/drivers/infiniband/hw/bnxt_re/qplib_res.h
@@ -219,6 +219,12 @@ static inline u8 bnxt_qplib_get_ring_type(struct bnxt_qplib_chip_ctx *cctx)
 	       RING_ALLOC_REQ_RING_TYPE_ROCE_CMPL;
 }
 
+struct bnxt_qplib_sg_info {
+	struct scatterlist		*sglist;
+	u32				nmap;
+	u32				npages;
+};
+
 #define to_bnxt_qplib(ptr, type, member)	\
 	container_of(ptr, type, member)
 
@@ -227,7 +233,7 @@ struct bnxt_qplib_dev_attr;
 
 void bnxt_qplib_free_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq);
 int bnxt_qplib_alloc_init_hwq(struct pci_dev *pdev, struct bnxt_qplib_hwq *hwq,
-			      struct scatterlist *sl, int nmap, u32 *elements,
+			      struct bnxt_qplib_sg_info *sg_info, u32 *elements,
 			      u32 elements_per_page, u32 aux, u32 pg_size,
 			      enum bnxt_qplib_hwq_type hwq_type);
 void bnxt_qplib_get_guid(u8 *dev_addr, u8 *guid);
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.c b/drivers/infiniband/hw/bnxt_re/qplib_sp.c
index e9c53e406404..48793d3512ac 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_sp.c
+++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.c
@@ -532,25 +532,21 @@ int bnxt_qplib_create_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah,
 	return 0;
 }
 
-int bnxt_qplib_destroy_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah,
-			  bool block)
+void bnxt_qplib_destroy_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah,
+			   bool block)
 {
 	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
 	struct cmdq_destroy_ah req;
 	struct creq_destroy_ah_resp resp;
 	u16 cmd_flags = 0;
-	int rc;
 
 	/* Clean up the AH table in the device */
 	RCFW_CMD_PREP(req, DESTROY_AH, cmd_flags);
 
 	req.ah_cid = cpu_to_le32(ah->id);
 
-	rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp,
-					  NULL, block);
-	if (rc)
-		return rc;
-	return 0;
+	bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp, NULL,
+				     block);
 }
 
 /* MRW */
@@ -684,7 +680,7 @@ int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr,
 
 		mr->hwq.max_elements = pages;
 		/* Use system PAGE_SIZE */
-		rc = bnxt_qplib_alloc_init_hwq(res->pdev, &mr->hwq, NULL, 0,
+		rc = bnxt_qplib_alloc_init_hwq(res->pdev, &mr->hwq, NULL,
 					       &mr->hwq.max_elements,
 					       PAGE_SIZE, 0, PAGE_SIZE,
 					       HWQ_TYPE_CTX);
@@ -754,7 +750,7 @@ int bnxt_qplib_alloc_fast_reg_page_list(struct bnxt_qplib_res *res,
 		return -ENOMEM;
 
 	frpl->hwq.max_elements = pages;
-	rc = bnxt_qplib_alloc_init_hwq(res->pdev, &frpl->hwq, NULL, 0,
+	rc = bnxt_qplib_alloc_init_hwq(res->pdev, &frpl->hwq, NULL,
 				       &frpl->hwq.max_elements, PAGE_SIZE, 0,
 				       PAGE_SIZE, HWQ_TYPE_CTX);
 	if (!rc)
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.h b/drivers/infiniband/hw/bnxt_re/qplib_sp.h
index 39454b3f738d..0ec3b12b0bcd 100644
--- a/drivers/infiniband/hw/bnxt_re/qplib_sp.h
+++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.h
@@ -243,8 +243,8 @@ int bnxt_qplib_set_func_resources(struct bnxt_qplib_res *res,
 				  struct bnxt_qplib_ctx *ctx);
 int bnxt_qplib_create_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah,
 			 bool block);
-int bnxt_qplib_destroy_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah,
-			  bool block);
+void bnxt_qplib_destroy_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah,
+			   bool block);
 int bnxt_qplib_alloc_mrw(struct bnxt_qplib_res *res,
 			 struct bnxt_qplib_mrw *mrw);
 int bnxt_qplib_dereg_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw,
diff --git a/drivers/infiniband/hw/cxgb3/cxio_wr.h b/drivers/infiniband/hw/cxgb3/cxio_wr.h
index 83d2e19d31ae..53aa5c36247a 100644
--- a/drivers/infiniband/hw/cxgb3/cxio_wr.h
+++ b/drivers/infiniband/hw/cxgb3/cxio_wr.h
@@ -64,7 +64,7 @@ enum t3_wr_flags {
 	T3_SOLICITED_EVENT_FLAG = 0x04,
 	T3_READ_FENCE_FLAG = 0x08,
 	T3_LOCAL_FENCE_FLAG = 0x10
-} __attribute__ ((packed));
+} __packed;
 
 enum t3_wr_opcode {
 	T3_WR_BP = FW_WROPCODE_RI_BYPASS,
@@ -77,7 +77,7 @@ enum t3_wr_opcode {
 	T3_WR_INIT = FW_WROPCODE_RI_RDMA_INIT,
 	T3_WR_QP_MOD = FW_WROPCODE_RI_MODIFY_QP,
 	T3_WR_FASTREG = FW_WROPCODE_RI_FASTREGISTER_MR
-} __attribute__ ((packed));
+} __packed;
 
 enum t3_rdma_opcode {
 	T3_RDMA_WRITE,		/* IETF RDMAP v1.0 ... */
@@ -95,7 +95,7 @@ enum t3_rdma_opcode {
 	T3_QP_MOD,
 	T3_BYPASS,
 	T3_RDMA_READ_REQ_WITH_INV,
-} __attribute__ ((packed));
+} __packed;
 
 static inline enum t3_rdma_opcode wr2opcode(enum t3_wr_opcode wrop)
 {
@@ -306,7 +306,7 @@ enum t3_mpa_attrs {
 	uP_RI_MPA_TX_MARKER_ENABLE = 0x2,
 	uP_RI_MPA_CRC_ENABLE = 0x4,
 	uP_RI_MPA_IETF_ENABLE = 0x8
-} __attribute__ ((packed));
+} __packed;
 
 enum t3_qp_caps {
 	uP_RI_QP_RDMA_READ_ENABLE = 0x01,
@@ -314,7 +314,7 @@ enum t3_qp_caps {
 	uP_RI_QP_BIND_ENABLE = 0x04,
 	uP_RI_QP_FAST_REGISTER_ENABLE = 0x08,
 	uP_RI_QP_STAG0_ENABLE = 0x10
-} __attribute__ ((packed));
+} __packed;
 
 enum rdma_init_rtr_types {
 	RTR_READ = 1,
diff --git a/drivers/infiniband/hw/cxgb3/iwch.c b/drivers/infiniband/hw/cxgb3/iwch.c
index fb03bc492ef7..56a8ab6210cf 100644
--- a/drivers/infiniband/hw/cxgb3/iwch.c
+++ b/drivers/infiniband/hw/cxgb3/iwch.c
@@ -62,37 +62,30 @@ struct cxgb3_client t3c_client = {
 static LIST_HEAD(dev_list);
 static DEFINE_MUTEX(dev_mutex);
 
-static int disable_qp_db(int id, void *p, void *data)
-{
-	struct iwch_qp *qhp = p;
-
-	cxio_disable_wq_db(&qhp->wq);
-	return 0;
-}
-
-static int enable_qp_db(int id, void *p, void *data)
-{
-	struct iwch_qp *qhp = p;
-
-	if (data)
-		ring_doorbell(qhp->rhp->rdev.ctrl_qp.doorbell, qhp->wq.qpid);
-	cxio_enable_wq_db(&qhp->wq);
-	return 0;
-}
-
 static void disable_dbs(struct iwch_dev *rnicp)
 {
-	spin_lock_irq(&rnicp->lock);
-	idr_for_each(&rnicp->qpidr, disable_qp_db, NULL);
-	spin_unlock_irq(&rnicp->lock);
+	unsigned long index;
+	struct iwch_qp *qhp;
+
+	xa_lock_irq(&rnicp->qps);
+	xa_for_each(&rnicp->qps, index, qhp)
+		cxio_disable_wq_db(&qhp->wq);
+	xa_unlock_irq(&rnicp->qps);
 }
 
 static void enable_dbs(struct iwch_dev *rnicp, int ring_db)
 {
-	spin_lock_irq(&rnicp->lock);
-	idr_for_each(&rnicp->qpidr, enable_qp_db,
-		     (void *)(unsigned long)ring_db);
-	spin_unlock_irq(&rnicp->lock);
+	unsigned long index;
+	struct iwch_qp *qhp;
+
+	xa_lock_irq(&rnicp->qps);
+	xa_for_each(&rnicp->qps, index, qhp) {
+		if (ring_db)
+			ring_doorbell(qhp->rhp->rdev.ctrl_qp.doorbell,
+					qhp->wq.qpid);
+		cxio_enable_wq_db(&qhp->wq);
+	}
+	xa_unlock_irq(&rnicp->qps);
 }
 
 static void iwch_db_drop_task(struct work_struct *work)
@@ -105,10 +98,9 @@ static void iwch_db_drop_task(struct work_struct *work)
 static void rnic_init(struct iwch_dev *rnicp)
 {
 	pr_debug("%s iwch_dev %p\n", __func__,  rnicp);
-	idr_init(&rnicp->cqidr);
-	idr_init(&rnicp->qpidr);
-	idr_init(&rnicp->mmidr);
-	spin_lock_init(&rnicp->lock);
+	xa_init_flags(&rnicp->cqs, XA_FLAGS_LOCK_IRQ);
+	xa_init_flags(&rnicp->qps, XA_FLAGS_LOCK_IRQ);
+	xa_init_flags(&rnicp->mrs, XA_FLAGS_LOCK_IRQ);
 	INIT_DELAYED_WORK(&rnicp->db_drop_task, iwch_db_drop_task);
 
 	rnicp->attr.max_qps = T3_MAX_NUM_QP - 32;
@@ -190,9 +182,9 @@ static void close_rnic_dev(struct t3cdev *tdev)
 			list_del(&dev->entry);
 			iwch_unregister_device(dev);
 			cxio_rdev_close(&dev->rdev);
-			idr_destroy(&dev->cqidr);
-			idr_destroy(&dev->qpidr);
-			idr_destroy(&dev->mmidr);
+			WARN_ON(!xa_empty(&dev->cqs));
+			WARN_ON(!xa_empty(&dev->qps));
+			WARN_ON(!xa_empty(&dev->mrs));
 			ib_dealloc_device(&dev->ibdev);
 			break;
 		}
diff --git a/drivers/infiniband/hw/cxgb3/iwch.h b/drivers/infiniband/hw/cxgb3/iwch.h
index c69bc4f52049..310a937bffcf 100644
--- a/drivers/infiniband/hw/cxgb3/iwch.h
+++ b/drivers/infiniband/hw/cxgb3/iwch.h
@@ -35,7 +35,7 @@
 #include <linux/mutex.h>
 #include <linux/list.h>
 #include <linux/spinlock.h>
-#include <linux/idr.h>
+#include <linux/xarray.h>
 #include <linux/workqueue.h>
 
 #include <rdma/ib_verbs.h>
@@ -106,10 +106,9 @@ struct iwch_dev {
 	struct cxio_rdev rdev;
 	u32 device_cap_flags;
 	struct iwch_rnic_attributes attr;
-	struct idr cqidr;
-	struct idr qpidr;
-	struct idr mmidr;
-	spinlock_t lock;
+	struct xarray cqs;
+	struct xarray qps;
+	struct xarray mrs;
 	struct list_head entry;
 	struct delayed_work db_drop_task;
 };
@@ -136,40 +135,17 @@ static inline int t3a_device(const struct iwch_dev *rhp)
 
 static inline struct iwch_cq *get_chp(struct iwch_dev *rhp, u32 cqid)
 {
-	return idr_find(&rhp->cqidr, cqid);
+	return xa_load(&rhp->cqs, cqid);
 }
 
 static inline struct iwch_qp *get_qhp(struct iwch_dev *rhp, u32 qpid)
 {
-	return idr_find(&rhp->qpidr, qpid);
+	return xa_load(&rhp->qps, qpid);
 }
 
 static inline struct iwch_mr *get_mhp(struct iwch_dev *rhp, u32 mmid)
 {
-	return idr_find(&rhp->mmidr, mmid);
-}
-
-static inline int insert_handle(struct iwch_dev *rhp, struct idr *idr,
-				void *handle, u32 id)
-{
-	int ret;
-
-	idr_preload(GFP_KERNEL);
-	spin_lock_irq(&rhp->lock);
-
-	ret = idr_alloc(idr, handle, id, id + 1, GFP_NOWAIT);
-
-	spin_unlock_irq(&rhp->lock);
-	idr_preload_end();
-
-	return ret < 0 ? ret : 0;
-}
-
-static inline void remove_handle(struct iwch_dev *rhp, struct idr *idr, u32 id)
-{
-	spin_lock_irq(&rhp->lock);
-	idr_remove(idr, id);
-	spin_unlock_irq(&rhp->lock);
+	return xa_load(&rhp->mrs, mmid);
 }
 
 extern struct cxgb3_client t3c_client;
diff --git a/drivers/infiniband/hw/cxgb3/iwch_ev.c b/drivers/infiniband/hw/cxgb3/iwch_ev.c
index 4a0c82a8fb60..9d356c1301c7 100644
--- a/drivers/infiniband/hw/cxgb3/iwch_ev.c
+++ b/drivers/infiniband/hw/cxgb3/iwch_ev.c
@@ -48,14 +48,14 @@ static void post_qp_event(struct iwch_dev *rnicp, struct iwch_cq *chp,
 	struct iwch_qp *qhp;
 	unsigned long flag;
 
-	spin_lock(&rnicp->lock);
-	qhp = get_qhp(rnicp, CQE_QPID(rsp_msg->cqe));
+	xa_lock(&rnicp->qps);
+	qhp = xa_load(&rnicp->qps, CQE_QPID(rsp_msg->cqe));
 
 	if (!qhp) {
 		pr_err("%s unaffiliated error 0x%x qpid 0x%x\n",
 		       __func__, CQE_STATUS(rsp_msg->cqe),
 		       CQE_QPID(rsp_msg->cqe));
-		spin_unlock(&rnicp->lock);
+		xa_unlock(&rnicp->qps);
 		return;
 	}
 
@@ -65,7 +65,7 @@ static void post_qp_event(struct iwch_dev *rnicp, struct iwch_cq *chp,
 			 __func__,
 			 qhp->attr.state, qhp->wq.qpid,
 			 CQE_STATUS(rsp_msg->cqe));
-		spin_unlock(&rnicp->lock);
+		xa_unlock(&rnicp->qps);
 		return;
 	}
 
@@ -76,7 +76,7 @@ static void post_qp_event(struct iwch_dev *rnicp, struct iwch_cq *chp,
 	       CQE_WRID_HI(rsp_msg->cqe), CQE_WRID_LOW(rsp_msg->cqe));
 
 	atomic_inc(&qhp->refcnt);
-	spin_unlock(&rnicp->lock);
+	xa_unlock(&rnicp->qps);
 
 	if (qhp->attr.state == IWCH_QP_STATE_RTS) {
 		attrs.next_state = IWCH_QP_STATE_TERMINATE;
@@ -114,21 +114,21 @@ void iwch_ev_dispatch(struct cxio_rdev *rdev_p, struct sk_buff *skb)
 	unsigned long flag;
 
 	rnicp = (struct iwch_dev *) rdev_p->ulp;
-	spin_lock(&rnicp->lock);
+	xa_lock(&rnicp->qps);
 	chp = get_chp(rnicp, cqid);
-	qhp = get_qhp(rnicp, CQE_QPID(rsp_msg->cqe));
+	qhp = xa_load(&rnicp->qps, CQE_QPID(rsp_msg->cqe));
 	if (!chp || !qhp) {
 		pr_err("BAD AE cqid 0x%x qpid 0x%x opcode %d status 0x%x type %d wrid.hi 0x%x wrid.lo 0x%x\n",
 		       cqid, CQE_QPID(rsp_msg->cqe),
 		       CQE_OPCODE(rsp_msg->cqe), CQE_STATUS(rsp_msg->cqe),
 		       CQE_TYPE(rsp_msg->cqe), CQE_WRID_HI(rsp_msg->cqe),
 		       CQE_WRID_LOW(rsp_msg->cqe));
-		spin_unlock(&rnicp->lock);
+		xa_unlock(&rnicp->qps);
 		goto out;
 	}
 	iwch_qp_add_ref(&qhp->ibqp);
 	atomic_inc(&chp->refcnt);
-	spin_unlock(&rnicp->lock);
+	xa_unlock(&rnicp->qps);
 
 	/*
 	 * 1) completion of our sending a TERMINATE.
diff --git a/drivers/infiniband/hw/cxgb3/iwch_mem.c b/drivers/infiniband/hw/cxgb3/iwch_mem.c
index 12886b1b4b10..ce0f2741821d 100644
--- a/drivers/infiniband/hw/cxgb3/iwch_mem.c
+++ b/drivers/infiniband/hw/cxgb3/iwch_mem.c
@@ -49,7 +49,7 @@ static int iwch_finish_mem_reg(struct iwch_mr *mhp, u32 stag)
 	mmid = stag >> 8;
 	mhp->ibmr.rkey = mhp->ibmr.lkey = stag;
 	pr_debug("%s mmid 0x%x mhp %p\n", __func__, mmid, mhp);
-	return insert_handle(mhp->rhp, &mhp->rhp->mmidr, mhp, mmid);
+	return xa_insert_irq(&mhp->rhp->mrs, mmid, mhp, GFP_KERNEL);
 }
 
 int iwch_register_mem(struct iwch_dev *rhp, struct iwch_pd *php,
diff --git a/drivers/infiniband/hw/cxgb3/iwch_provider.c b/drivers/infiniband/hw/cxgb3/iwch_provider.c
index 4accf7b3dcf2..3a481dfb1607 100644
--- a/drivers/infiniband/hw/cxgb3/iwch_provider.c
+++ b/drivers/infiniband/hw/cxgb3/iwch_provider.c
@@ -88,14 +88,14 @@ static int iwch_alloc_ucontext(struct ib_ucontext *ucontext,
 	return 0;
 }
 
-static int iwch_destroy_cq(struct ib_cq *ib_cq)
+static int iwch_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
 {
 	struct iwch_cq *chp;
 
 	pr_debug("%s ib_cq %p\n", __func__, ib_cq);
 	chp = to_iwch_cq(ib_cq);
 
-	remove_handle(chp->rhp, &chp->rhp->cqidr, chp->cq.cqid);
+	xa_erase_irq(&chp->rhp->cqs, chp->cq.cqid);
 	atomic_dec(&chp->refcnt);
 	wait_event(chp->wait, !atomic_read(&chp->refcnt));
 
@@ -106,7 +106,6 @@ static int iwch_destroy_cq(struct ib_cq *ib_cq)
 
 static struct ib_cq *iwch_create_cq(struct ib_device *ibdev,
 				    const struct ib_cq_init_attr *attr,
-				    struct ib_ucontext *ib_context,
 				    struct ib_udata *udata)
 {
 	int entries = attr->cqe;
@@ -114,7 +113,6 @@ static struct ib_cq *iwch_create_cq(struct ib_device *ibdev,
 	struct iwch_cq *chp;
 	struct iwch_create_cq_resp uresp;
 	struct iwch_create_cq_req ureq;
-	struct iwch_ucontext *ucontext = NULL;
 	static int warned;
 	size_t resplen;
 
@@ -127,8 +125,7 @@ static struct ib_cq *iwch_create_cq(struct ib_device *ibdev,
 	if (!chp)
 		return ERR_PTR(-ENOMEM);
 
-	if (ib_context) {
-		ucontext = to_iwch_ucontext(ib_context);
+	if (udata) {
 		if (!t3a_device(rhp)) {
 			if (ib_copy_from_udata(&ureq, udata, sizeof (ureq))) {
 				kfree(chp);
@@ -154,7 +151,7 @@ static struct ib_cq *iwch_create_cq(struct ib_device *ibdev,
 	entries = roundup_pow_of_two(entries);
 	chp->cq.size_log2 = ilog2(entries);
 
-	if (cxio_create_cq(&rhp->rdev, &chp->cq, !ucontext)) {
+	if (cxio_create_cq(&rhp->rdev, &chp->cq, !udata)) {
 		kfree(chp);
 		return ERR_PTR(-ENOMEM);
 	}
@@ -164,18 +161,20 @@ static struct ib_cq *iwch_create_cq(struct ib_device *ibdev,
 	spin_lock_init(&chp->comp_handler_lock);
 	atomic_set(&chp->refcnt, 1);
 	init_waitqueue_head(&chp->wait);
-	if (insert_handle(rhp, &rhp->cqidr, chp, chp->cq.cqid)) {
+	if (xa_store_irq(&rhp->cqs, chp->cq.cqid, chp, GFP_KERNEL)) {
 		cxio_destroy_cq(&chp->rhp->rdev, &chp->cq);
 		kfree(chp);
 		return ERR_PTR(-ENOMEM);
 	}
 
-	if (ucontext) {
+	if (udata) {
 		struct iwch_mm_entry *mm;
+		struct iwch_ucontext *ucontext = rdma_udata_to_drv_context(
+			udata, struct iwch_ucontext, ibucontext);
 
 		mm = kmalloc(sizeof *mm, GFP_KERNEL);
 		if (!mm) {
-			iwch_destroy_cq(&chp->ibcq);
+			iwch_destroy_cq(&chp->ibcq, udata);
 			return ERR_PTR(-ENOMEM);
 		}
 		uresp.cqid = chp->cq.cqid;
@@ -201,7 +200,7 @@ static struct ib_cq *iwch_create_cq(struct ib_device *ibdev,
 		}
 		if (ib_copy_to_udata(udata, &uresp, resplen)) {
 			kfree(mm);
-			iwch_destroy_cq(&chp->ibcq);
+			iwch_destroy_cq(&chp->ibcq, udata);
 			return ERR_PTR(-EFAULT);
 		}
 		insert_mmap(ucontext, mm);
@@ -367,7 +366,7 @@ static int iwch_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
 	return ret;
 }
 
-static void iwch_deallocate_pd(struct ib_pd *pd)
+static void iwch_deallocate_pd(struct ib_pd *pd, struct ib_udata *udata)
 {
 	struct iwch_dev *rhp;
 	struct iwch_pd *php;
@@ -378,8 +377,7 @@ static void iwch_deallocate_pd(struct ib_pd *pd)
 	cxio_hal_put_pdid(rhp->rdev.rscp, php->pdid);
 }
 
-static int iwch_allocate_pd(struct ib_pd *pd, struct ib_ucontext *context,
-			    struct ib_udata *udata)
+static int iwch_allocate_pd(struct ib_pd *pd, struct ib_udata *udata)
 {
 	struct iwch_pd *php = to_iwch_pd(pd);
 	struct ib_device *ibdev = pd->device;
@@ -394,11 +392,11 @@ static int iwch_allocate_pd(struct ib_pd *pd, struct ib_ucontext *context,
 
 	php->pdid = pdid;
 	php->rhp = rhp;
-	if (context) {
+	if (udata) {
 		struct iwch_alloc_pd_resp resp = {.pdid = php->pdid};
 
 		if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
-			iwch_deallocate_pd(&php->ibpd);
+			iwch_deallocate_pd(&php->ibpd, udata);
 			return -EFAULT;
 		}
 	}
@@ -406,7 +404,7 @@ static int iwch_allocate_pd(struct ib_pd *pd, struct ib_ucontext *context,
 	return 0;
 }
 
-static int iwch_dereg_mr(struct ib_mr *ib_mr)
+static int iwch_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
 {
 	struct iwch_dev *rhp;
 	struct iwch_mr *mhp;
@@ -421,7 +419,7 @@ static int iwch_dereg_mr(struct ib_mr *ib_mr)
 	cxio_dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size,
 		       mhp->attr.pbl_addr);
 	iwch_free_pbl(mhp);
-	remove_handle(rhp, &rhp->mmidr, mmid);
+	xa_erase_irq(&rhp->mrs, mmid);
 	if (mhp->kva)
 		kfree((void *) (unsigned long) mhp->kva);
 	if (mhp->umem)
@@ -539,7 +537,7 @@ static struct ib_mr *iwch_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 
 	shift = PAGE_SHIFT;
 
-	n = mhp->umem->nmap;
+	n = ib_umem_num_pages(mhp->umem);
 
 	err = iwch_alloc_pbl(mhp, n);
 	if (err)
@@ -590,7 +588,7 @@ pbl_done:
 			 uresp.pbl_addr);
 
 		if (ib_copy_to_udata(udata, &uresp, sizeof (uresp))) {
-			iwch_dereg_mr(&mhp->ibmr);
+			iwch_dereg_mr(&mhp->ibmr, udata);
 			err = -EFAULT;
 			goto err;
 		}
@@ -636,7 +634,7 @@ static struct ib_mw *iwch_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
 	mhp->attr.stag = stag;
 	mmid = (stag) >> 8;
 	mhp->ibmw.rkey = stag;
-	if (insert_handle(rhp, &rhp->mmidr, mhp, mmid)) {
+	if (xa_insert_irq(&rhp->mrs, mmid, mhp, GFP_KERNEL)) {
 		cxio_deallocate_window(&rhp->rdev, mhp->attr.stag);
 		kfree(mhp);
 		return ERR_PTR(-ENOMEM);
@@ -655,15 +653,14 @@ static int iwch_dealloc_mw(struct ib_mw *mw)
 	rhp = mhp->rhp;
 	mmid = (mw->rkey) >> 8;
 	cxio_deallocate_window(&rhp->rdev, mhp->attr.stag);
-	remove_handle(rhp, &rhp->mmidr, mmid);
+	xa_erase_irq(&rhp->mrs, mmid);
 	pr_debug("%s ib_mw %p mmid 0x%x ptr %p\n", __func__, mw, mmid, mhp);
 	kfree(mhp);
 	return 0;
 }
 
-static struct ib_mr *iwch_alloc_mr(struct ib_pd *pd,
-				   enum ib_mr_type mr_type,
-				   u32 max_num_sg)
+static struct ib_mr *iwch_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
+				   u32 max_num_sg, struct ib_udata *udata)
 {
 	struct iwch_dev *rhp;
 	struct iwch_pd *php;
@@ -701,7 +698,7 @@ static struct ib_mr *iwch_alloc_mr(struct ib_pd *pd,
 	mhp->attr.state = 1;
 	mmid = (stag) >> 8;
 	mhp->ibmr.rkey = mhp->ibmr.lkey = stag;
-	ret = insert_handle(rhp, &rhp->mmidr, mhp, mmid);
+	ret = xa_insert_irq(&rhp->mrs, mmid, mhp, GFP_KERNEL);
 	if (ret)
 		goto err3;
 
@@ -742,7 +739,7 @@ static int iwch_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
 	return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, iwch_set_page);
 }
 
-static int iwch_destroy_qp(struct ib_qp *ib_qp)
+static int iwch_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata)
 {
 	struct iwch_dev *rhp;
 	struct iwch_qp *qhp;
@@ -756,13 +753,13 @@ static int iwch_destroy_qp(struct ib_qp *ib_qp)
 	iwch_modify_qp(rhp, qhp, IWCH_QP_ATTR_NEXT_STATE, &attrs, 0);
 	wait_event(qhp->wait, !qhp->ep);
 
-	remove_handle(rhp, &rhp->qpidr, qhp->wq.qpid);
+	xa_erase_irq(&rhp->qps, qhp->wq.qpid);
 
 	atomic_dec(&qhp->refcnt);
 	wait_event(qhp->wait, !atomic_read(&qhp->refcnt));
 
-	ucontext = ib_qp->uobject ? to_iwch_ucontext(ib_qp->uobject->context)
-				  : NULL;
+	ucontext = rdma_udata_to_drv_context(udata, struct iwch_ucontext,
+					     ibucontext);
 	cxio_destroy_qp(&rhp->rdev, &qhp->wq,
 			ucontext ? &ucontext->uctx : &rhp->rdev.uctx);
 
@@ -872,7 +869,7 @@ static struct ib_qp *iwch_create_qp(struct ib_pd *pd,
 	init_waitqueue_head(&qhp->wait);
 	atomic_set(&qhp->refcnt, 1);
 
-	if (insert_handle(rhp, &rhp->qpidr, qhp, qhp->wq.qpid)) {
+	if (xa_store_irq(&rhp->qps, qhp->wq.qpid, qhp, GFP_KERNEL)) {
 		cxio_destroy_qp(&rhp->rdev, &qhp->wq,
 			ucontext ? &ucontext->uctx : &rhp->rdev.uctx);
 		kfree(qhp);
@@ -885,14 +882,14 @@ static struct ib_qp *iwch_create_qp(struct ib_pd *pd,
 
 		mm1 = kmalloc(sizeof *mm1, GFP_KERNEL);
 		if (!mm1) {
-			iwch_destroy_qp(&qhp->ibqp);
+			iwch_destroy_qp(&qhp->ibqp, udata);
 			return ERR_PTR(-ENOMEM);
 		}
 
 		mm2 = kmalloc(sizeof *mm2, GFP_KERNEL);
 		if (!mm2) {
 			kfree(mm1);
-			iwch_destroy_qp(&qhp->ibqp);
+			iwch_destroy_qp(&qhp->ibqp, udata);
 			return ERR_PTR(-ENOMEM);
 		}
 
@@ -909,7 +906,7 @@ static struct ib_qp *iwch_create_qp(struct ib_pd *pd,
 		if (ib_copy_to_udata(udata, &uresp, sizeof (uresp))) {
 			kfree(mm1);
 			kfree(mm2);
-			iwch_destroy_qp(&qhp->ibqp);
+			iwch_destroy_qp(&qhp->ibqp, udata);
 			return ERR_PTR(-EFAULT);
 		}
 		mm1->key = uresp.key;
@@ -1324,6 +1321,14 @@ static const struct ib_device_ops iwch_dev_ops = {
 	.get_dma_mr = iwch_get_dma_mr,
 	.get_hw_stats = iwch_get_mib,
 	.get_port_immutable = iwch_port_immutable,
+	.iw_accept = iwch_accept_cr,
+	.iw_add_ref = iwch_qp_add_ref,
+	.iw_connect = iwch_connect,
+	.iw_create_listen = iwch_create_listen,
+	.iw_destroy_listen = iwch_destroy_listen,
+	.iw_get_qp = iwch_get_qp,
+	.iw_reject = iwch_reject_cr,
+	.iw_rem_ref = iwch_qp_rem_ref,
 	.map_mr_sg = iwch_map_mr_sg,
 	.mmap = iwch_mmap,
 	.modify_qp = iwch_ib_modify_qp,
@@ -1343,8 +1348,6 @@ static const struct ib_device_ops iwch_dev_ops = {
 
 int iwch_register_device(struct iwch_dev *dev)
 {
-	int ret;
-
 	pr_debug("%s iwch_dev %p\n", __func__, dev);
 	memset(&dev->ibdev.node_guid, 0, sizeof(dev->ibdev.node_guid));
 	memcpy(&dev->ibdev.node_guid, dev->rdev.t3cdev_p->lldev->dev_addr, 6);
@@ -1382,34 +1385,18 @@ int iwch_register_device(struct iwch_dev *dev)
 	dev->ibdev.dev.parent = &dev->rdev.rnic_info.pdev->dev;
 	dev->ibdev.uverbs_abi_ver = IWCH_UVERBS_ABI_VERSION;
 
-	dev->ibdev.iwcm = kzalloc(sizeof(struct iw_cm_verbs), GFP_KERNEL);
-	if (!dev->ibdev.iwcm)
-		return -ENOMEM;
-
-	dev->ibdev.iwcm->connect = iwch_connect;
-	dev->ibdev.iwcm->accept = iwch_accept_cr;
-	dev->ibdev.iwcm->reject = iwch_reject_cr;
-	dev->ibdev.iwcm->create_listen = iwch_create_listen;
-	dev->ibdev.iwcm->destroy_listen = iwch_destroy_listen;
-	dev->ibdev.iwcm->add_ref = iwch_qp_add_ref;
-	dev->ibdev.iwcm->rem_ref = iwch_qp_rem_ref;
-	dev->ibdev.iwcm->get_qp = iwch_get_qp;
-	memcpy(dev->ibdev.iwcm->ifname, dev->rdev.t3cdev_p->lldev->name,
-	       sizeof(dev->ibdev.iwcm->ifname));
+	memcpy(dev->ibdev.iw_ifname, dev->rdev.t3cdev_p->lldev->name,
+	       sizeof(dev->ibdev.iw_ifname));
 
 	dev->ibdev.driver_id = RDMA_DRIVER_CXGB3;
 	rdma_set_device_sysfs_group(&dev->ibdev, &iwch_attr_group);
 	ib_set_device_ops(&dev->ibdev, &iwch_dev_ops);
-	ret = ib_register_device(&dev->ibdev, "cxgb3_%d");
-	if (ret)
-		kfree(dev->ibdev.iwcm);
-	return ret;
+	return ib_register_device(&dev->ibdev, "cxgb3_%d");
 }
 
 void iwch_unregister_device(struct iwch_dev *dev)
 {
 	pr_debug("%s iwch_dev %p\n", __func__, dev);
 	ib_unregister_device(&dev->ibdev);
-	kfree(dev->ibdev.iwcm);
 	return;
 }
diff --git a/drivers/infiniband/hw/cxgb4/cm.c b/drivers/infiniband/hw/cxgb4/cm.c
index 4d232bdf9e97..0f3b1193d5f8 100644
--- a/drivers/infiniband/hw/cxgb4/cm.c
+++ b/drivers/infiniband/hw/cxgb4/cm.c
@@ -331,20 +331,23 @@ static void remove_ep_tid(struct c4iw_ep *ep)
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(&ep->com.dev->lock, flags);
-	_remove_handle(ep->com.dev, &ep->com.dev->hwtid_idr, ep->hwtid, 0);
-	if (idr_is_empty(&ep->com.dev->hwtid_idr))
+	xa_lock_irqsave(&ep->com.dev->hwtids, flags);
+	__xa_erase(&ep->com.dev->hwtids, ep->hwtid);
+	if (xa_empty(&ep->com.dev->hwtids))
 		wake_up(&ep->com.dev->wait);
-	spin_unlock_irqrestore(&ep->com.dev->lock, flags);
+	xa_unlock_irqrestore(&ep->com.dev->hwtids, flags);
 }
 
-static void insert_ep_tid(struct c4iw_ep *ep)
+static int insert_ep_tid(struct c4iw_ep *ep)
 {
 	unsigned long flags;
+	int err;
+
+	xa_lock_irqsave(&ep->com.dev->hwtids, flags);
+	err = __xa_insert(&ep->com.dev->hwtids, ep->hwtid, ep, GFP_KERNEL);
+	xa_unlock_irqrestore(&ep->com.dev->hwtids, flags);
 
-	spin_lock_irqsave(&ep->com.dev->lock, flags);
-	_insert_handle(ep->com.dev, &ep->com.dev->hwtid_idr, ep, ep->hwtid, 0);
-	spin_unlock_irqrestore(&ep->com.dev->lock, flags);
+	return err;
 }
 
 /*
@@ -355,11 +358,11 @@ static struct c4iw_ep *get_ep_from_tid(struct c4iw_dev *dev, unsigned int tid)
 	struct c4iw_ep *ep;
 	unsigned long flags;
 
-	spin_lock_irqsave(&dev->lock, flags);
-	ep = idr_find(&dev->hwtid_idr, tid);
+	xa_lock_irqsave(&dev->hwtids, flags);
+	ep = xa_load(&dev->hwtids, tid);
 	if (ep)
 		c4iw_get_ep(&ep->com);
-	spin_unlock_irqrestore(&dev->lock, flags);
+	xa_unlock_irqrestore(&dev->hwtids, flags);
 	return ep;
 }
 
@@ -372,11 +375,11 @@ static struct c4iw_listen_ep *get_ep_from_stid(struct c4iw_dev *dev,
 	struct c4iw_listen_ep *ep;
 	unsigned long flags;
 
-	spin_lock_irqsave(&dev->lock, flags);
-	ep = idr_find(&dev->stid_idr, stid);
+	xa_lock_irqsave(&dev->stids, flags);
+	ep = xa_load(&dev->stids, stid);
 	if (ep)
 		c4iw_get_ep(&ep->com);
-	spin_unlock_irqrestore(&dev->lock, flags);
+	xa_unlock_irqrestore(&dev->stids, flags);
 	return ep;
 }
 
@@ -457,6 +460,8 @@ static struct sk_buff *get_skb(struct sk_buff *skb, int len, gfp_t gfp)
 		skb_reset_transport_header(skb);
 	} else {
 		skb = alloc_skb(len, gfp);
+		if (!skb)
+			return NULL;
 	}
 	t4_set_arp_err_handler(skb, NULL, NULL);
 	return skb;
@@ -555,7 +560,7 @@ static void act_open_req_arp_failure(void *handle, struct sk_buff *skb)
 		cxgb4_clip_release(ep->com.dev->rdev.lldi.ports[0],
 				   (const u32 *)&sin6->sin6_addr.s6_addr, 1);
 	}
-	remove_handle(ep->com.dev, &ep->com.dev->atid_idr, ep->atid);
+	xa_erase_irq(&ep->com.dev->atids, ep->atid);
 	cxgb4_free_atid(ep->com.dev->rdev.lldi.tids, ep->atid);
 	queue_arp_failure_cpl(ep, skb, FAKE_CPL_PUT_EP_SAFE);
 }
@@ -1235,7 +1240,7 @@ static int act_establish(struct c4iw_dev *dev, struct sk_buff *skb)
 	set_emss(ep, tcp_opt);
 
 	/* dealloc the atid */
-	remove_handle(ep->com.dev, &ep->com.dev->atid_idr, atid);
+	xa_erase_irq(&ep->com.dev->atids, atid);
 	cxgb4_free_atid(t, atid);
 	set_bit(ACT_ESTAB, &ep->com.history);
 
@@ -2184,7 +2189,9 @@ static int c4iw_reconnect(struct c4iw_ep *ep)
 		err = -ENOMEM;
 		goto fail2;
 	}
-	insert_handle(ep->com.dev, &ep->com.dev->atid_idr, ep, ep->atid);
+	err = xa_insert_irq(&ep->com.dev->atids, ep->atid, ep, GFP_KERNEL);
+	if (err)
+		goto fail2a;
 
 	/* find a route */
 	if (ep->com.cm_id->m_local_addr.ss_family == AF_INET) {
@@ -2236,7 +2243,8 @@ static int c4iw_reconnect(struct c4iw_ep *ep)
 fail4:
 	dst_release(ep->dst);
 fail3:
-	remove_handle(ep->com.dev, &ep->com.dev->atid_idr, ep->atid);
+	xa_erase_irq(&ep->com.dev->atids, ep->atid);
+fail2a:
 	cxgb4_free_atid(ep->com.dev->rdev.lldi.tids, ep->atid);
 fail2:
 	/*
@@ -2319,8 +2327,7 @@ static int act_open_rpl(struct c4iw_dev *dev, struct sk_buff *skb)
 						(const u32 *)
 						&sin6->sin6_addr.s6_addr, 1);
 			}
-			remove_handle(ep->com.dev, &ep->com.dev->atid_idr,
-					atid);
+			xa_erase_irq(&ep->com.dev->atids, atid);
 			cxgb4_free_atid(t, atid);
 			dst_release(ep->dst);
 			cxgb4_l2t_release(ep->l2t);
@@ -2357,7 +2364,7 @@ fail:
 		cxgb4_remove_tid(ep->com.dev->rdev.lldi.tids, 0, GET_TID(rpl),
 				 ep->com.local_addr.ss_family);
 
-	remove_handle(ep->com.dev, &ep->com.dev->atid_idr, atid);
+	xa_erase_irq(&ep->com.dev->atids, atid);
 	cxgb4_free_atid(t, atid);
 	dst_release(ep->dst);
 	cxgb4_l2t_release(ep->l2t);
@@ -2947,7 +2954,7 @@ out:
 					(const u32 *)&sin6->sin6_addr.s6_addr,
 					1);
 		}
-		remove_handle(ep->com.dev, &ep->com.dev->hwtid_idr, ep->hwtid);
+		xa_erase_irq(&ep->com.dev->hwtids, ep->hwtid);
 		cxgb4_remove_tid(ep->com.dev->rdev.lldi.tids, 0, ep->hwtid,
 				 ep->com.local_addr.ss_family);
 		dst_release(ep->dst);
@@ -3342,7 +3349,9 @@ int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
 		err = -ENOMEM;
 		goto fail2;
 	}
-	insert_handle(dev, &dev->atid_idr, ep, ep->atid);
+	err = xa_insert_irq(&dev->atids, ep->atid, ep, GFP_KERNEL);
+	if (err)
+		goto fail5;
 
 	memcpy(&ep->com.local_addr, &cm_id->m_local_addr,
 	       sizeof(ep->com.local_addr));
@@ -3430,7 +3439,8 @@ int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
 fail4:
 	dst_release(ep->dst);
 fail3:
-	remove_handle(ep->com.dev, &ep->com.dev->atid_idr, ep->atid);
+	xa_erase_irq(&ep->com.dev->atids, ep->atid);
+fail5:
 	cxgb4_free_atid(ep->com.dev->rdev.lldi.tids, ep->atid);
 fail2:
 	skb_queue_purge(&ep->com.ep_skb_list);
@@ -3553,7 +3563,9 @@ int c4iw_create_listen(struct iw_cm_id *cm_id, int backlog)
 		err = -ENOMEM;
 		goto fail2;
 	}
-	insert_handle(dev, &dev->stid_idr, ep, ep->stid);
+	err = xa_insert_irq(&dev->stids, ep->stid, ep, GFP_KERNEL);
+	if (err)
+		goto fail3;
 
 	state_set(&ep->com, LISTEN);
 	if (ep->com.local_addr.ss_family == AF_INET)
@@ -3564,7 +3576,8 @@ int c4iw_create_listen(struct iw_cm_id *cm_id, int backlog)
 		cm_id->provider_data = ep;
 		goto out;
 	}
-	remove_handle(ep->com.dev, &ep->com.dev->stid_idr, ep->stid);
+	xa_erase_irq(&ep->com.dev->stids, ep->stid);
+fail3:
 	cxgb4_free_stid(ep->com.dev->rdev.lldi.tids, ep->stid,
 			ep->com.local_addr.ss_family);
 fail2:
@@ -3603,7 +3616,7 @@ int c4iw_destroy_listen(struct iw_cm_id *cm_id)
 		cxgb4_clip_release(ep->com.dev->rdev.lldi.ports[0],
 				   (const u32 *)&sin6->sin6_addr.s6_addr, 1);
 	}
-	remove_handle(ep->com.dev, &ep->com.dev->stid_idr, ep->stid);
+	xa_erase_irq(&ep->com.dev->stids, ep->stid);
 	cxgb4_free_stid(ep->com.dev->rdev.lldi.tids, ep->stid,
 			ep->com.local_addr.ss_family);
 done:
@@ -3763,7 +3776,7 @@ static void active_ofld_conn_reply(struct c4iw_dev *dev, struct sk_buff *skb,
 		cxgb4_clip_release(ep->com.dev->rdev.lldi.ports[0],
 				   (const u32 *)&sin6->sin6_addr.s6_addr, 1);
 	}
-	remove_handle(dev, &dev->atid_idr, atid);
+	xa_erase_irq(&dev->atids, atid);
 	cxgb4_free_atid(dev->rdev.lldi.tids, atid);
 	dst_release(ep->dst);
 	cxgb4_l2t_release(ep->l2t);
diff --git a/drivers/infiniband/hw/cxgb4/cq.c b/drivers/infiniband/hw/cxgb4/cq.c
index 1fd8798d91a7..52ce586621c6 100644
--- a/drivers/infiniband/hw/cxgb4/cq.c
+++ b/drivers/infiniband/hw/cxgb4/cq.c
@@ -30,6 +30,8 @@
  * SOFTWARE.
  */
 
+#include <rdma/uverbs_ioctl.h>
+
 #include "iw_cxgb4.h"
 
 static int destroy_cq(struct c4iw_rdev *rdev, struct t4_cq *cq,
@@ -968,7 +970,7 @@ int c4iw_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
 	return !err || err == -ENODATA ? npolled : err;
 }
 
-int c4iw_destroy_cq(struct ib_cq *ib_cq)
+int c4iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
 {
 	struct c4iw_cq *chp;
 	struct c4iw_ucontext *ucontext;
@@ -976,12 +978,12 @@ int c4iw_destroy_cq(struct ib_cq *ib_cq)
 	pr_debug("ib_cq %p\n", ib_cq);
 	chp = to_c4iw_cq(ib_cq);
 
-	remove_handle(chp->rhp, &chp->rhp->cqidr, chp->cq.cqid);
+	xa_erase_irq(&chp->rhp->cqs, chp->cq.cqid);
 	atomic_dec(&chp->refcnt);
 	wait_event(chp->wait, !atomic_read(&chp->refcnt));
 
-	ucontext = ib_cq->uobject ? to_c4iw_ucontext(ib_cq->uobject->context)
-				  : NULL;
+	ucontext = rdma_udata_to_drv_context(udata, struct c4iw_ucontext,
+					     ibucontext);
 	destroy_cq(&chp->rhp->rdev, &chp->cq,
 		   ucontext ? &ucontext->uctx : &chp->cq.rdev->uctx,
 		   chp->destroy_skb, chp->wr_waitp);
@@ -992,7 +994,6 @@ int c4iw_destroy_cq(struct ib_cq *ib_cq)
 
 struct ib_cq *c4iw_create_cq(struct ib_device *ibdev,
 			     const struct ib_cq_init_attr *attr,
-			     struct ib_ucontext *ib_context,
 			     struct ib_udata *udata)
 {
 	int entries = attr->cqe;
@@ -1001,10 +1002,11 @@ struct ib_cq *c4iw_create_cq(struct ib_device *ibdev,
 	struct c4iw_cq *chp;
 	struct c4iw_create_cq ucmd;
 	struct c4iw_create_cq_resp uresp;
-	struct c4iw_ucontext *ucontext = NULL;
 	int ret, wr_len;
 	size_t memsize, hwentries;
 	struct c4iw_mm_entry *mm, *mm2;
+	struct c4iw_ucontext *ucontext = rdma_udata_to_drv_context(
+		udata, struct c4iw_ucontext, ibucontext);
 
 	pr_debug("ib_dev %p entries %d\n", ibdev, entries);
 	if (attr->flags)
@@ -1015,8 +1017,7 @@ struct ib_cq *c4iw_create_cq(struct ib_device *ibdev,
 	if (vector >= rhp->rdev.lldi.nciq)
 		return ERR_PTR(-EINVAL);
 
-	if (ib_context) {
-		ucontext = to_c4iw_ucontext(ib_context);
+	if (udata) {
 		if (udata->inlen < sizeof(ucmd))
 			ucontext->is_32b_cqe = 1;
 	}
@@ -1068,7 +1069,7 @@ struct ib_cq *c4iw_create_cq(struct ib_device *ibdev,
 	/*
 	 * memsize must be a multiple of the page size if its a user cq.
 	 */
-	if (ucontext)
+	if (udata)
 		memsize = roundup(memsize, PAGE_SIZE);
 
 	chp->cq.size = hwentries;
@@ -1088,7 +1089,7 @@ struct ib_cq *c4iw_create_cq(struct ib_device *ibdev,
 	spin_lock_init(&chp->comp_handler_lock);
 	atomic_set(&chp->refcnt, 1);
 	init_waitqueue_head(&chp->wait);
-	ret = insert_handle(rhp, &rhp->cqidr, chp, chp->cq.cqid);
+	ret = xa_insert_irq(&rhp->cqs, chp->cq.cqid, chp, GFP_KERNEL);
 	if (ret)
 		goto err_destroy_cq;
 
@@ -1143,7 +1144,7 @@ err_free_mm2:
 err_free_mm:
 	kfree(mm);
 err_remove_handle:
-	remove_handle(rhp, &rhp->cqidr, chp->cq.cqid);
+	xa_erase_irq(&rhp->cqs, chp->cq.cqid);
 err_destroy_cq:
 	destroy_cq(&chp->rhp->rdev, &chp->cq,
 		   ucontext ? &ucontext->uctx : &rhp->rdev.uctx,
diff --git a/drivers/infiniband/hw/cxgb4/device.c b/drivers/infiniband/hw/cxgb4/device.c
index c79cf63fb0bb..4c0d925c5ff5 100644
--- a/drivers/infiniband/hw/cxgb4/device.c
+++ b/drivers/infiniband/hw/cxgb4/device.c
@@ -81,14 +81,6 @@ struct c4iw_debugfs_data {
 	int pos;
 };
 
-static int count_idrs(int id, void *p, void *data)
-{
-	int *countp = data;
-
-	*countp = *countp + 1;
-	return 0;
-}
-
 static ssize_t debugfs_read(struct file *file, char __user *buf, size_t count,
 			    loff_t *ppos)
 {
@@ -250,16 +242,11 @@ static void set_ep_sin6_addrs(struct c4iw_ep *ep,
 	}
 }
 
-static int dump_qp(int id, void *p, void *data)
+static int dump_qp(struct c4iw_qp *qp, struct c4iw_debugfs_data *qpd)
 {
-	struct c4iw_qp *qp = p;
-	struct c4iw_debugfs_data *qpd = data;
 	int space;
 	int cc;
 
-	if (id != qp->wq.sq.qid)
-		return 0;
-
 	space = qpd->bufsize - qpd->pos - 1;
 	if (space == 0)
 		return 1;
@@ -335,7 +322,9 @@ static int qp_release(struct inode *inode, struct file *file)
 
 static int qp_open(struct inode *inode, struct file *file)
 {
+	struct c4iw_qp *qp;
 	struct c4iw_debugfs_data *qpd;
+	unsigned long index;
 	int count = 1;
 
 	qpd = kmalloc(sizeof *qpd, GFP_KERNEL);
@@ -345,9 +334,12 @@ static int qp_open(struct inode *inode, struct file *file)
 	qpd->devp = inode->i_private;
 	qpd->pos = 0;
 
-	spin_lock_irq(&qpd->devp->lock);
-	idr_for_each(&qpd->devp->qpidr, count_idrs, &count);
-	spin_unlock_irq(&qpd->devp->lock);
+	/*
+	 * No need to lock; we drop the lock to call vmalloc so it's racy
+	 * anyway.  Someone who cares should switch this over to seq_file
+	 */
+	xa_for_each(&qpd->devp->qps, index, qp)
+		count++;
 
 	qpd->bufsize = count * 180;
 	qpd->buf = vmalloc(qpd->bufsize);
@@ -356,9 +348,10 @@ static int qp_open(struct inode *inode, struct file *file)
 		return -ENOMEM;
 	}
 
-	spin_lock_irq(&qpd->devp->lock);
-	idr_for_each(&qpd->devp->qpidr, dump_qp, qpd);
-	spin_unlock_irq(&qpd->devp->lock);
+	xa_lock_irq(&qpd->devp->qps);
+	xa_for_each(&qpd->devp->qps, index, qp)
+		dump_qp(qp, qpd);
+	xa_unlock_irq(&qpd->devp->qps);
 
 	qpd->buf[qpd->pos++] = 0;
 	file->private_data = qpd;
@@ -373,9 +366,8 @@ static const struct file_operations qp_debugfs_fops = {
 	.llseek  = default_llseek,
 };
 
-static int dump_stag(int id, void *p, void *data)
+static int dump_stag(unsigned long id, struct c4iw_debugfs_data *stagd)
 {
-	struct c4iw_debugfs_data *stagd = data;
 	int space;
 	int cc;
 	struct fw_ri_tpte tpte;
@@ -424,6 +416,8 @@ static int stag_release(struct inode *inode, struct file *file)
 static int stag_open(struct inode *inode, struct file *file)
 {
 	struct c4iw_debugfs_data *stagd;
+	void *p;
+	unsigned long index;
 	int ret = 0;
 	int count = 1;
 
@@ -435,9 +429,8 @@ static int stag_open(struct inode *inode, struct file *file)
 	stagd->devp = inode->i_private;
 	stagd->pos = 0;
 
-	spin_lock_irq(&stagd->devp->lock);
-	idr_for_each(&stagd->devp->mmidr, count_idrs, &count);
-	spin_unlock_irq(&stagd->devp->lock);
+	xa_for_each(&stagd->devp->mrs, index, p)
+		count++;
 
 	stagd->bufsize = count * 256;
 	stagd->buf = vmalloc(stagd->bufsize);
@@ -446,9 +439,10 @@ static int stag_open(struct inode *inode, struct file *file)
 		goto err1;
 	}
 
-	spin_lock_irq(&stagd->devp->lock);
-	idr_for_each(&stagd->devp->mmidr, dump_stag, stagd);
-	spin_unlock_irq(&stagd->devp->lock);
+	xa_lock_irq(&stagd->devp->mrs);
+	xa_for_each(&stagd->devp->mrs, index, p)
+		dump_stag(index, stagd);
+	xa_unlock_irq(&stagd->devp->mrs);
 
 	stagd->buf[stagd->pos++] = 0;
 	file->private_data = stagd;
@@ -558,10 +552,8 @@ static const struct file_operations stats_debugfs_fops = {
 	.write   = stats_clear,
 };
 
-static int dump_ep(int id, void *p, void *data)
+static int dump_ep(struct c4iw_ep *ep, struct c4iw_debugfs_data *epd)
 {
-	struct c4iw_ep *ep = p;
-	struct c4iw_debugfs_data *epd = data;
 	int space;
 	int cc;
 
@@ -617,10 +609,9 @@ static int dump_ep(int id, void *p, void *data)
 	return 0;
 }
 
-static int dump_listen_ep(int id, void *p, void *data)
+static
+int dump_listen_ep(struct c4iw_listen_ep *ep, struct c4iw_debugfs_data *epd)
 {
-	struct c4iw_listen_ep *ep = p;
-	struct c4iw_debugfs_data *epd = data;
 	int space;
 	int cc;
 
@@ -674,6 +665,9 @@ static int ep_release(struct inode *inode, struct file *file)
 
 static int ep_open(struct inode *inode, struct file *file)
 {
+	struct c4iw_ep *ep;
+	struct c4iw_listen_ep *lep;
+	unsigned long index;
 	struct c4iw_debugfs_data *epd;
 	int ret = 0;
 	int count = 1;
@@ -686,11 +680,12 @@ static int ep_open(struct inode *inode, struct file *file)
 	epd->devp = inode->i_private;
 	epd->pos = 0;
 
-	spin_lock_irq(&epd->devp->lock);
-	idr_for_each(&epd->devp->hwtid_idr, count_idrs, &count);
-	idr_for_each(&epd->devp->atid_idr, count_idrs, &count);
-	idr_for_each(&epd->devp->stid_idr, count_idrs, &count);
-	spin_unlock_irq(&epd->devp->lock);
+	xa_for_each(&epd->devp->hwtids, index, ep)
+		count++;
+	xa_for_each(&epd->devp->atids, index, ep)
+		count++;
+	xa_for_each(&epd->devp->stids, index, lep)
+		count++;
 
 	epd->bufsize = count * 240;
 	epd->buf = vmalloc(epd->bufsize);
@@ -699,11 +694,18 @@ static int ep_open(struct inode *inode, struct file *file)
 		goto err1;
 	}
 
-	spin_lock_irq(&epd->devp->lock);
-	idr_for_each(&epd->devp->hwtid_idr, dump_ep, epd);
-	idr_for_each(&epd->devp->atid_idr, dump_ep, epd);
-	idr_for_each(&epd->devp->stid_idr, dump_listen_ep, epd);
-	spin_unlock_irq(&epd->devp->lock);
+	xa_lock_irq(&epd->devp->hwtids);
+	xa_for_each(&epd->devp->hwtids, index, ep)
+		dump_ep(ep, epd);
+	xa_unlock_irq(&epd->devp->hwtids);
+	xa_lock_irq(&epd->devp->atids);
+	xa_for_each(&epd->devp->atids, index, ep)
+		dump_ep(ep, epd);
+	xa_unlock_irq(&epd->devp->atids);
+	xa_lock_irq(&epd->devp->stids);
+	xa_for_each(&epd->devp->stids, index, lep)
+		dump_listen_ep(lep, epd);
+	xa_unlock_irq(&epd->devp->stids);
 
 	file->private_data = epd;
 	goto out;
@@ -931,16 +933,12 @@ static void c4iw_rdev_close(struct c4iw_rdev *rdev)
 void c4iw_dealloc(struct uld_ctx *ctx)
 {
 	c4iw_rdev_close(&ctx->dev->rdev);
-	WARN_ON_ONCE(!idr_is_empty(&ctx->dev->cqidr));
-	idr_destroy(&ctx->dev->cqidr);
-	WARN_ON_ONCE(!idr_is_empty(&ctx->dev->qpidr));
-	idr_destroy(&ctx->dev->qpidr);
-	WARN_ON_ONCE(!idr_is_empty(&ctx->dev->mmidr));
-	idr_destroy(&ctx->dev->mmidr);
-	wait_event(ctx->dev->wait, idr_is_empty(&ctx->dev->hwtid_idr));
-	idr_destroy(&ctx->dev->hwtid_idr);
-	idr_destroy(&ctx->dev->stid_idr);
-	idr_destroy(&ctx->dev->atid_idr);
+	WARN_ON(!xa_empty(&ctx->dev->cqs));
+	WARN_ON(!xa_empty(&ctx->dev->qps));
+	WARN_ON(!xa_empty(&ctx->dev->mrs));
+	wait_event(ctx->dev->wait, xa_empty(&ctx->dev->hwtids));
+	WARN_ON(!xa_empty(&ctx->dev->stids));
+	WARN_ON(!xa_empty(&ctx->dev->atids));
 	if (ctx->dev->rdev.bar2_kva)
 		iounmap(ctx->dev->rdev.bar2_kva);
 	if (ctx->dev->rdev.oc_mw_kva)
@@ -1044,13 +1042,12 @@ static struct c4iw_dev *c4iw_alloc(const struct cxgb4_lld_info *infop)
 		return ERR_PTR(ret);
 	}
 
-	idr_init(&devp->cqidr);
-	idr_init(&devp->qpidr);
-	idr_init(&devp->mmidr);
-	idr_init(&devp->hwtid_idr);
-	idr_init(&devp->stid_idr);
-	idr_init(&devp->atid_idr);
-	spin_lock_init(&devp->lock);
+	xa_init_flags(&devp->cqs, XA_FLAGS_LOCK_IRQ);
+	xa_init_flags(&devp->qps, XA_FLAGS_LOCK_IRQ);
+	xa_init_flags(&devp->mrs, XA_FLAGS_LOCK_IRQ);
+	xa_init_flags(&devp->hwtids, XA_FLAGS_LOCK_IRQ);
+	xa_init_flags(&devp->atids, XA_FLAGS_LOCK_IRQ);
+	xa_init_flags(&devp->stids, XA_FLAGS_LOCK_IRQ);
 	mutex_init(&devp->rdev.stats.lock);
 	mutex_init(&devp->db_mutex);
 	INIT_LIST_HEAD(&devp->db_fc_list);
@@ -1265,34 +1262,21 @@ static int c4iw_uld_state_change(void *handle, enum cxgb4_state new_state)
 	return 0;
 }
 
-static int disable_qp_db(int id, void *p, void *data)
-{
-	struct c4iw_qp *qp = p;
-
-	t4_disable_wq_db(&qp->wq);
-	return 0;
-}
-
 static void stop_queues(struct uld_ctx *ctx)
 {
-	unsigned long flags;
+	struct c4iw_qp *qp;
+	unsigned long index, flags;
 
-	spin_lock_irqsave(&ctx->dev->lock, flags);
+	xa_lock_irqsave(&ctx->dev->qps, flags);
 	ctx->dev->rdev.stats.db_state_transitions++;
 	ctx->dev->db_state = STOPPED;
-	if (ctx->dev->rdev.flags & T4_STATUS_PAGE_DISABLED)
-		idr_for_each(&ctx->dev->qpidr, disable_qp_db, NULL);
-	else
+	if (ctx->dev->rdev.flags & T4_STATUS_PAGE_DISABLED) {
+		xa_for_each(&ctx->dev->qps, index, qp)
+			t4_disable_wq_db(&qp->wq);
+	} else {
 		ctx->dev->rdev.status_page->db_off = 1;
-	spin_unlock_irqrestore(&ctx->dev->lock, flags);
-}
-
-static int enable_qp_db(int id, void *p, void *data)
-{
-	struct c4iw_qp *qp = p;
-
-	t4_enable_wq_db(&qp->wq);
-	return 0;
+	}
+	xa_unlock_irqrestore(&ctx->dev->qps, flags);
 }
 
 static void resume_rc_qp(struct c4iw_qp *qp)
@@ -1322,18 +1306,21 @@ static void resume_a_chunk(struct uld_ctx *ctx)
 
 static void resume_queues(struct uld_ctx *ctx)
 {
-	spin_lock_irq(&ctx->dev->lock);
+	xa_lock_irq(&ctx->dev->qps);
 	if (ctx->dev->db_state != STOPPED)
 		goto out;
 	ctx->dev->db_state = FLOW_CONTROL;
 	while (1) {
 		if (list_empty(&ctx->dev->db_fc_list)) {
+			struct c4iw_qp *qp;
+			unsigned long index;
+
 			WARN_ON(ctx->dev->db_state != FLOW_CONTROL);
 			ctx->dev->db_state = NORMAL;
 			ctx->dev->rdev.stats.db_state_transitions++;
 			if (ctx->dev->rdev.flags & T4_STATUS_PAGE_DISABLED) {
-				idr_for_each(&ctx->dev->qpidr, enable_qp_db,
-					     NULL);
+				xa_for_each(&ctx->dev->qps, index, qp)
+					t4_enable_wq_db(&qp->wq);
 			} else {
 				ctx->dev->rdev.status_page->db_off = 0;
 			}
@@ -1345,12 +1332,12 @@ static void resume_queues(struct uld_ctx *ctx)
 				resume_a_chunk(ctx);
 			}
 			if (!list_empty(&ctx->dev->db_fc_list)) {
-				spin_unlock_irq(&ctx->dev->lock);
+				xa_unlock_irq(&ctx->dev->qps);
 				if (DB_FC_RESUME_DELAY) {
 					set_current_state(TASK_UNINTERRUPTIBLE);
 					schedule_timeout(DB_FC_RESUME_DELAY);
 				}
-				spin_lock_irq(&ctx->dev->lock);
+				xa_lock_irq(&ctx->dev->qps);
 				if (ctx->dev->db_state != FLOW_CONTROL)
 					break;
 			}
@@ -1359,7 +1346,7 @@ static void resume_queues(struct uld_ctx *ctx)
 out:
 	if (ctx->dev->db_state != NORMAL)
 		ctx->dev->rdev.stats.db_fc_interruptions++;
-	spin_unlock_irq(&ctx->dev->lock);
+	xa_unlock_irq(&ctx->dev->qps);
 }
 
 struct qp_list {
@@ -1367,23 +1354,6 @@ struct qp_list {
 	struct c4iw_qp **qps;
 };
 
-static int add_and_ref_qp(int id, void *p, void *data)
-{
-	struct qp_list *qp_listp = data;
-	struct c4iw_qp *qp = p;
-
-	c4iw_qp_add_ref(&qp->ibqp);
-	qp_listp->qps[qp_listp->idx++] = qp;
-	return 0;
-}
-
-static int count_qps(int id, void *p, void *data)
-{
-	unsigned *countp = data;
-	(*countp)++;
-	return 0;
-}
-
 static void deref_qps(struct qp_list *qp_list)
 {
 	int idx;
@@ -1400,7 +1370,7 @@ static void recover_lost_dbs(struct uld_ctx *ctx, struct qp_list *qp_list)
 	for (idx = 0; idx < qp_list->idx; idx++) {
 		struct c4iw_qp *qp = qp_list->qps[idx];
 
-		spin_lock_irq(&qp->rhp->lock);
+		xa_lock_irq(&qp->rhp->qps);
 		spin_lock(&qp->lock);
 		ret = cxgb4_sync_txq_pidx(qp->rhp->rdev.lldi.ports[0],
 					  qp->wq.sq.qid,
@@ -1410,7 +1380,7 @@ static void recover_lost_dbs(struct uld_ctx *ctx, struct qp_list *qp_list)
 			pr_err("%s: Fatal error - DB overflow recovery failed - error syncing SQ qid %u\n",
 			       pci_name(ctx->lldi.pdev), qp->wq.sq.qid);
 			spin_unlock(&qp->lock);
-			spin_unlock_irq(&qp->rhp->lock);
+			xa_unlock_irq(&qp->rhp->qps);
 			return;
 		}
 		qp->wq.sq.wq_pidx_inc = 0;
@@ -1424,12 +1394,12 @@ static void recover_lost_dbs(struct uld_ctx *ctx, struct qp_list *qp_list)
 			pr_err("%s: Fatal error - DB overflow recovery failed - error syncing RQ qid %u\n",
 			       pci_name(ctx->lldi.pdev), qp->wq.rq.qid);
 			spin_unlock(&qp->lock);
-			spin_unlock_irq(&qp->rhp->lock);
+			xa_unlock_irq(&qp->rhp->qps);
 			return;
 		}
 		qp->wq.rq.wq_pidx_inc = 0;
 		spin_unlock(&qp->lock);
-		spin_unlock_irq(&qp->rhp->lock);
+		xa_unlock_irq(&qp->rhp->qps);
 
 		/* Wait for the dbfifo to drain */
 		while (cxgb4_dbfifo_count(qp->rhp->rdev.lldi.ports[0], 1) > 0) {
@@ -1441,6 +1411,8 @@ static void recover_lost_dbs(struct uld_ctx *ctx, struct qp_list *qp_list)
 
 static void recover_queues(struct uld_ctx *ctx)
 {
+	struct c4iw_qp *qp;
+	unsigned long index;
 	int count = 0;
 	struct qp_list qp_list;
 	int ret;
@@ -1458,22 +1430,26 @@ static void recover_queues(struct uld_ctx *ctx)
 	}
 
 	/* Count active queues so we can build a list of queues to recover */
-	spin_lock_irq(&ctx->dev->lock);
+	xa_lock_irq(&ctx->dev->qps);
 	WARN_ON(ctx->dev->db_state != STOPPED);
 	ctx->dev->db_state = RECOVERY;
-	idr_for_each(&ctx->dev->qpidr, count_qps, &count);
+	xa_for_each(&ctx->dev->qps, index, qp)
+		count++;
 
 	qp_list.qps = kcalloc(count, sizeof(*qp_list.qps), GFP_ATOMIC);
 	if (!qp_list.qps) {
-		spin_unlock_irq(&ctx->dev->lock);
+		xa_unlock_irq(&ctx->dev->qps);
 		return;
 	}
 	qp_list.idx = 0;
 
 	/* add and ref each qp so it doesn't get freed */
-	idr_for_each(&ctx->dev->qpidr, add_and_ref_qp, &qp_list);
+	xa_for_each(&ctx->dev->qps, index, qp) {
+		c4iw_qp_add_ref(&qp->ibqp);
+		qp_list.qps[qp_list.idx++] = qp;
+	}
 
-	spin_unlock_irq(&ctx->dev->lock);
+	xa_unlock_irq(&ctx->dev->qps);
 
 	/* now traverse the list in a safe context to recover the db state*/
 	recover_lost_dbs(ctx, &qp_list);
@@ -1482,10 +1458,10 @@ static void recover_queues(struct uld_ctx *ctx)
 	deref_qps(&qp_list);
 	kfree(qp_list.qps);
 
-	spin_lock_irq(&ctx->dev->lock);
+	xa_lock_irq(&ctx->dev->qps);
 	WARN_ON(ctx->dev->db_state != RECOVERY);
 	ctx->dev->db_state = STOPPED;
-	spin_unlock_irq(&ctx->dev->lock);
+	xa_unlock_irq(&ctx->dev->qps);
 }
 
 static int c4iw_uld_control(void *handle, enum cxgb4_control control, ...)
diff --git a/drivers/infiniband/hw/cxgb4/ev.c b/drivers/infiniband/hw/cxgb4/ev.c
index 8741d23168f3..4cd877bd2f56 100644
--- a/drivers/infiniband/hw/cxgb4/ev.c
+++ b/drivers/infiniband/hw/cxgb4/ev.c
@@ -123,15 +123,15 @@ void c4iw_ev_dispatch(struct c4iw_dev *dev, struct t4_cqe *err_cqe)
 	struct c4iw_qp *qhp;
 	u32 cqid;
 
-	spin_lock_irq(&dev->lock);
-	qhp = get_qhp(dev, CQE_QPID(err_cqe));
+	xa_lock_irq(&dev->qps);
+	qhp = xa_load(&dev->qps, CQE_QPID(err_cqe));
 	if (!qhp) {
 		pr_err("BAD AE qpid 0x%x opcode %d status 0x%x type %d wrid.hi 0x%x wrid.lo 0x%x\n",
 		       CQE_QPID(err_cqe),
 		       CQE_OPCODE(err_cqe), CQE_STATUS(err_cqe),
 		       CQE_TYPE(err_cqe), CQE_WRID_HI(err_cqe),
 		       CQE_WRID_LOW(err_cqe));
-		spin_unlock_irq(&dev->lock);
+		xa_unlock_irq(&dev->qps);
 		goto out;
 	}
 
@@ -146,13 +146,13 @@ void c4iw_ev_dispatch(struct c4iw_dev *dev, struct t4_cqe *err_cqe)
 		       CQE_OPCODE(err_cqe), CQE_STATUS(err_cqe),
 		       CQE_TYPE(err_cqe), CQE_WRID_HI(err_cqe),
 		       CQE_WRID_LOW(err_cqe));
-		spin_unlock_irq(&dev->lock);
+		xa_unlock_irq(&dev->qps);
 		goto out;
 	}
 
 	c4iw_qp_add_ref(&qhp->ibqp);
 	atomic_inc(&chp->refcnt);
-	spin_unlock_irq(&dev->lock);
+	xa_unlock_irq(&dev->qps);
 
 	/* Bad incoming write */
 	if (RQ_TYPE(err_cqe) &&
@@ -225,11 +225,11 @@ int c4iw_ev_handler(struct c4iw_dev *dev, u32 qid)
 	struct c4iw_cq *chp;
 	unsigned long flag;
 
-	spin_lock_irqsave(&dev->lock, flag);
-	chp = get_chp(dev, qid);
+	xa_lock_irqsave(&dev->cqs, flag);
+	chp = xa_load(&dev->cqs, qid);
 	if (chp) {
 		atomic_inc(&chp->refcnt);
-		spin_unlock_irqrestore(&dev->lock, flag);
+		xa_unlock_irqrestore(&dev->cqs, flag);
 		t4_clear_cq_armed(&chp->cq);
 		spin_lock_irqsave(&chp->comp_handler_lock, flag);
 		(*chp->ibcq.comp_handler)(&chp->ibcq, chp->ibcq.cq_context);
@@ -238,7 +238,7 @@ int c4iw_ev_handler(struct c4iw_dev *dev, u32 qid)
 			wake_up(&chp->wait);
 	} else {
 		pr_debug("unknown cqid 0x%x\n", qid);
-		spin_unlock_irqrestore(&dev->lock, flag);
+		xa_unlock_irqrestore(&dev->cqs, flag);
 	}
 	return 0;
 }
diff --git a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
index 5a5da41faef6..916ef982172e 100644
--- a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
+++ b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h
@@ -34,7 +34,7 @@
 #include <linux/mutex.h>
 #include <linux/list.h>
 #include <linux/spinlock.h>
-#include <linux/idr.h>
+#include <linux/xarray.h>
 #include <linux/completion.h>
 #include <linux/netdevice.h>
 #include <linux/sched/mm.h>
@@ -315,16 +315,15 @@ struct c4iw_dev {
 	struct ib_device ibdev;
 	struct c4iw_rdev rdev;
 	u32 device_cap_flags;
-	struct idr cqidr;
-	struct idr qpidr;
-	struct idr mmidr;
-	spinlock_t lock;
+	struct xarray cqs;
+	struct xarray qps;
+	struct xarray mrs;
 	struct mutex db_mutex;
 	struct dentry *debugfs_root;
 	enum db_state db_state;
-	struct idr hwtid_idr;
-	struct idr atid_idr;
-	struct idr stid_idr;
+	struct xarray hwtids;
+	struct xarray atids;
+	struct xarray stids;
 	struct list_head db_fc_list;
 	u32 avail_ird;
 	wait_queue_head_t wait;
@@ -349,70 +348,12 @@ static inline struct c4iw_dev *rdev_to_c4iw_dev(struct c4iw_rdev *rdev)
 
 static inline struct c4iw_cq *get_chp(struct c4iw_dev *rhp, u32 cqid)
 {
-	return idr_find(&rhp->cqidr, cqid);
+	return xa_load(&rhp->cqs, cqid);
 }
 
 static inline struct c4iw_qp *get_qhp(struct c4iw_dev *rhp, u32 qpid)
 {
-	return idr_find(&rhp->qpidr, qpid);
-}
-
-static inline struct c4iw_mr *get_mhp(struct c4iw_dev *rhp, u32 mmid)
-{
-	return idr_find(&rhp->mmidr, mmid);
-}
-
-static inline int _insert_handle(struct c4iw_dev *rhp, struct idr *idr,
-				 void *handle, u32 id, int lock)
-{
-	int ret;
-
-	if (lock) {
-		idr_preload(GFP_KERNEL);
-		spin_lock_irq(&rhp->lock);
-	}
-
-	ret = idr_alloc(idr, handle, id, id + 1, GFP_ATOMIC);
-
-	if (lock) {
-		spin_unlock_irq(&rhp->lock);
-		idr_preload_end();
-	}
-
-	return ret < 0 ? ret : 0;
-}
-
-static inline int insert_handle(struct c4iw_dev *rhp, struct idr *idr,
-				void *handle, u32 id)
-{
-	return _insert_handle(rhp, idr, handle, id, 1);
-}
-
-static inline int insert_handle_nolock(struct c4iw_dev *rhp, struct idr *idr,
-				       void *handle, u32 id)
-{
-	return _insert_handle(rhp, idr, handle, id, 0);
-}
-
-static inline void _remove_handle(struct c4iw_dev *rhp, struct idr *idr,
-				   u32 id, int lock)
-{
-	if (lock)
-		spin_lock_irq(&rhp->lock);
-	idr_remove(idr, id);
-	if (lock)
-		spin_unlock_irq(&rhp->lock);
-}
-
-static inline void remove_handle(struct c4iw_dev *rhp, struct idr *idr, u32 id)
-{
-	_remove_handle(rhp, idr, id, 1);
-}
-
-static inline void remove_handle_nolock(struct c4iw_dev *rhp,
-					 struct idr *idr, u32 id)
-{
-	_remove_handle(rhp, idr, id, 0);
+	return xa_load(&rhp->qps, qpid);
 }
 
 extern uint c4iw_max_read_depth;
@@ -1038,9 +979,8 @@ int c4iw_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param);
 int c4iw_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len);
 void c4iw_qp_add_ref(struct ib_qp *qp);
 void c4iw_qp_rem_ref(struct ib_qp *qp);
-struct ib_mr *c4iw_alloc_mr(struct ib_pd *pd,
-			    enum ib_mr_type mr_type,
-			    u32 max_num_sg);
+struct ib_mr *c4iw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
+			    u32 max_num_sg, struct ib_udata *udata);
 int c4iw_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
 		   unsigned int *sg_offset);
 int c4iw_dealloc_mw(struct ib_mw *mw);
@@ -1051,21 +991,19 @@ struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start,
 					   u64 length, u64 virt, int acc,
 					   struct ib_udata *udata);
 struct ib_mr *c4iw_get_dma_mr(struct ib_pd *pd, int acc);
-int c4iw_dereg_mr(struct ib_mr *ib_mr);
-int c4iw_destroy_cq(struct ib_cq *ib_cq);
+int c4iw_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata);
+int c4iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata);
 struct ib_cq *c4iw_create_cq(struct ib_device *ibdev,
 			     const struct ib_cq_init_attr *attr,
-			     struct ib_ucontext *ib_context,
 			     struct ib_udata *udata);
 int c4iw_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
 int c4iw_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *attr,
 		    enum ib_srq_attr_mask srq_attr_mask,
 		    struct ib_udata *udata);
-int c4iw_destroy_srq(struct ib_srq *ib_srq);
-struct ib_srq *c4iw_create_srq(struct ib_pd *pd,
-			       struct ib_srq_init_attr *attrs,
-			       struct ib_udata *udata);
-int c4iw_destroy_qp(struct ib_qp *ib_qp);
+void c4iw_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata);
+int c4iw_create_srq(struct ib_srq *srq, struct ib_srq_init_attr *attrs,
+		    struct ib_udata *udata);
+int c4iw_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata);
 struct ib_qp *c4iw_create_qp(struct ib_pd *pd,
 			     struct ib_qp_init_attr *attrs,
 			     struct ib_udata *udata);
diff --git a/drivers/infiniband/hw/cxgb4/mem.c b/drivers/infiniband/hw/cxgb4/mem.c
index 5baa31ab6366..811c0c8c5b16 100644
--- a/drivers/infiniband/hw/cxgb4/mem.c
+++ b/drivers/infiniband/hw/cxgb4/mem.c
@@ -395,7 +395,7 @@ static int finish_mem_reg(struct c4iw_mr *mhp, u32 stag)
 	mhp->ibmr.iova = mhp->attr.va_fbo;
 	mhp->ibmr.page_size = 1U << (mhp->attr.page_size + 12);
 	pr_debug("mmid 0x%x mhp %p\n", mmid, mhp);
-	return insert_handle(mhp->rhp, &mhp->rhp->mmidr, mhp, mmid);
+	return xa_insert_irq(&mhp->rhp->mrs, mmid, mhp, GFP_KERNEL);
 }
 
 static int register_mem(struct c4iw_dev *rhp, struct c4iw_pd *php,
@@ -542,7 +542,7 @@ struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 
 	shift = PAGE_SHIFT;
 
-	n = mhp->umem->nmap;
+	n = ib_umem_num_pages(mhp->umem);
 	err = alloc_pbl(mhp, n);
 	if (err)
 		goto err_umem_release;
@@ -645,7 +645,7 @@ struct ib_mw *c4iw_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
 	mhp->attr.stag = stag;
 	mmid = (stag) >> 8;
 	mhp->ibmw.rkey = stag;
-	if (insert_handle(rhp, &rhp->mmidr, mhp, mmid)) {
+	if (xa_insert_irq(&rhp->mrs, mmid, mhp, GFP_KERNEL)) {
 		ret = -ENOMEM;
 		goto dealloc_win;
 	}
@@ -673,7 +673,7 @@ int c4iw_dealloc_mw(struct ib_mw *mw)
 	mhp = to_c4iw_mw(mw);
 	rhp = mhp->rhp;
 	mmid = (mw->rkey) >> 8;
-	remove_handle(rhp, &rhp->mmidr, mmid);
+	xa_erase_irq(&rhp->mrs, mmid);
 	deallocate_window(&rhp->rdev, mhp->attr.stag, mhp->dereg_skb,
 			  mhp->wr_waitp);
 	kfree_skb(mhp->dereg_skb);
@@ -683,9 +683,8 @@ int c4iw_dealloc_mw(struct ib_mw *mw)
 	return 0;
 }
 
-struct ib_mr *c4iw_alloc_mr(struct ib_pd *pd,
-			    enum ib_mr_type mr_type,
-			    u32 max_num_sg)
+struct ib_mr *c4iw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
+			    u32 max_num_sg, struct ib_udata *udata)
 {
 	struct c4iw_dev *rhp;
 	struct c4iw_pd *php;
@@ -740,7 +739,7 @@ struct ib_mr *c4iw_alloc_mr(struct ib_pd *pd,
 	mhp->attr.state = 0;
 	mmid = (stag) >> 8;
 	mhp->ibmr.rkey = mhp->ibmr.lkey = stag;
-	if (insert_handle(rhp, &rhp->mmidr, mhp, mmid)) {
+	if (xa_insert_irq(&rhp->mrs, mmid, mhp, GFP_KERNEL)) {
 		ret = -ENOMEM;
 		goto err_dereg;
 	}
@@ -786,7 +785,7 @@ int c4iw_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
 	return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, c4iw_set_page);
 }
 
-int c4iw_dereg_mr(struct ib_mr *ib_mr)
+int c4iw_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
 {
 	struct c4iw_dev *rhp;
 	struct c4iw_mr *mhp;
@@ -797,7 +796,7 @@ int c4iw_dereg_mr(struct ib_mr *ib_mr)
 	mhp = to_c4iw_mr(ib_mr);
 	rhp = mhp->rhp;
 	mmid = mhp->attr.stag >> 8;
-	remove_handle(rhp, &rhp->mmidr, mmid);
+	xa_erase_irq(&rhp->mrs, mmid);
 	if (mhp->mpl)
 		dma_free_coherent(&mhp->rhp->rdev.lldi.pdev->dev,
 				  mhp->max_mpl_len, mhp->mpl, mhp->mpl_addr);
@@ -821,9 +820,9 @@ void c4iw_invalidate_mr(struct c4iw_dev *rhp, u32 rkey)
 	struct c4iw_mr *mhp;
 	unsigned long flags;
 
-	spin_lock_irqsave(&rhp->lock, flags);
-	mhp = get_mhp(rhp, rkey >> 8);
+	xa_lock_irqsave(&rhp->mrs, flags);
+	mhp = xa_load(&rhp->mrs, rkey >> 8);
 	if (mhp)
 		mhp->attr.state = 0;
-	spin_unlock_irqrestore(&rhp->lock, flags);
+	xa_unlock_irqrestore(&rhp->mrs, flags);
 }
diff --git a/drivers/infiniband/hw/cxgb4/provider.c b/drivers/infiniband/hw/cxgb4/provider.c
index 507c54572cc9..74b795642fca 100644
--- a/drivers/infiniband/hw/cxgb4/provider.c
+++ b/drivers/infiniband/hw/cxgb4/provider.c
@@ -190,7 +190,7 @@ static int c4iw_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
 	return ret;
 }
 
-static void c4iw_deallocate_pd(struct ib_pd *pd)
+static void c4iw_deallocate_pd(struct ib_pd *pd, struct ib_udata *udata)
 {
 	struct c4iw_dev *rhp;
 	struct c4iw_pd *php;
@@ -204,8 +204,7 @@ static void c4iw_deallocate_pd(struct ib_pd *pd)
 	mutex_unlock(&rhp->rdev.stats.lock);
 }
 
-static int c4iw_allocate_pd(struct ib_pd *pd, struct ib_ucontext *context,
-			    struct ib_udata *udata)
+static int c4iw_allocate_pd(struct ib_pd *pd, struct ib_udata *udata)
 {
 	struct c4iw_pd *php = to_c4iw_pd(pd);
 	struct ib_device *ibdev = pd->device;
@@ -220,11 +219,11 @@ static int c4iw_allocate_pd(struct ib_pd *pd, struct ib_ucontext *context,
 
 	php->pdid = pdid;
 	php->rhp = rhp;
-	if (context) {
+	if (udata) {
 		struct c4iw_alloc_pd_resp uresp = {.pdid = php->pdid};
 
 		if (ib_copy_to_udata(udata, &uresp, sizeof(uresp))) {
-			c4iw_deallocate_pd(&php->ibpd);
+			c4iw_deallocate_pd(&php->ibpd, udata);
 			return -EFAULT;
 		}
 	}
@@ -483,24 +482,6 @@ static void get_dev_fw_str(struct ib_device *dev, char *str)
 		 FW_HDR_FW_VER_BUILD_G(c4iw_dev->rdev.lldi.fw_vers));
 }
 
-static struct net_device *get_netdev(struct ib_device *dev, u8 port)
-{
-	struct c4iw_dev *c4iw_dev = container_of(dev, struct c4iw_dev, ibdev);
-	struct c4iw_rdev *rdev = &c4iw_dev->rdev;
-	struct net_device *ndev;
-
-	if (!port || port > rdev->lldi.nports)
-		return NULL;
-
-	rcu_read_lock();
-	ndev = rdev->lldi.ports[port - 1];
-	if (ndev)
-		dev_hold(ndev);
-	rcu_read_unlock();
-
-	return ndev;
-}
-
 static int fill_res_entry(struct sk_buff *msg, struct rdma_restrack_entry *res)
 {
 	return (res->type < ARRAY_SIZE(c4iw_restrack_funcs) &&
@@ -528,8 +509,15 @@ static const struct ib_device_ops c4iw_dev_ops = {
 	.get_dev_fw_str = get_dev_fw_str,
 	.get_dma_mr = c4iw_get_dma_mr,
 	.get_hw_stats = c4iw_get_mib,
-	.get_netdev = get_netdev,
 	.get_port_immutable = c4iw_port_immutable,
+	.iw_accept = c4iw_accept_cr,
+	.iw_add_ref = c4iw_qp_add_ref,
+	.iw_connect = c4iw_connect,
+	.iw_create_listen = c4iw_create_listen,
+	.iw_destroy_listen = c4iw_destroy_listen,
+	.iw_get_qp = c4iw_get_qp,
+	.iw_reject = c4iw_reject_cr,
+	.iw_rem_ref = c4iw_qp_rem_ref,
 	.map_mr_sg = c4iw_map_mr_sg,
 	.mmap = c4iw_mmap,
 	.modify_qp = c4iw_ib_modify_qp,
@@ -546,9 +534,24 @@ static const struct ib_device_ops c4iw_dev_ops = {
 	.reg_user_mr = c4iw_reg_user_mr,
 	.req_notify_cq = c4iw_arm_cq,
 	INIT_RDMA_OBJ_SIZE(ib_pd, c4iw_pd, ibpd),
+	INIT_RDMA_OBJ_SIZE(ib_srq, c4iw_srq, ibsrq),
 	INIT_RDMA_OBJ_SIZE(ib_ucontext, c4iw_ucontext, ibucontext),
 };
 
+static int set_netdevs(struct ib_device *ib_dev, struct c4iw_rdev *rdev)
+{
+	int ret;
+	int i;
+
+	for (i = 0; i < rdev->lldi.nports; i++) {
+		ret = ib_device_set_netdev(ib_dev, rdev->lldi.ports[i],
+					   i + 1);
+		if (ret)
+			return ret;
+	}
+	return 0;
+}
+
 void c4iw_register_device(struct work_struct *work)
 {
 	int ret;
@@ -593,33 +596,20 @@ void c4iw_register_device(struct work_struct *work)
 	dev->ibdev.dev.parent = &dev->rdev.lldi.pdev->dev;
 	dev->ibdev.uverbs_abi_ver = C4IW_UVERBS_ABI_VERSION;
 
-	dev->ibdev.iwcm = kzalloc(sizeof(struct iw_cm_verbs), GFP_KERNEL);
-	if (!dev->ibdev.iwcm) {
-		ret = -ENOMEM;
-		goto err_dealloc_ctx;
-	}
-
-	dev->ibdev.iwcm->connect = c4iw_connect;
-	dev->ibdev.iwcm->accept = c4iw_accept_cr;
-	dev->ibdev.iwcm->reject = c4iw_reject_cr;
-	dev->ibdev.iwcm->create_listen = c4iw_create_listen;
-	dev->ibdev.iwcm->destroy_listen = c4iw_destroy_listen;
-	dev->ibdev.iwcm->add_ref = c4iw_qp_add_ref;
-	dev->ibdev.iwcm->rem_ref = c4iw_qp_rem_ref;
-	dev->ibdev.iwcm->get_qp = c4iw_get_qp;
-	memcpy(dev->ibdev.iwcm->ifname, dev->rdev.lldi.ports[0]->name,
-	       sizeof(dev->ibdev.iwcm->ifname));
+	memcpy(dev->ibdev.iw_ifname, dev->rdev.lldi.ports[0]->name,
+	       sizeof(dev->ibdev.iw_ifname));
 
 	rdma_set_device_sysfs_group(&dev->ibdev, &c4iw_attr_group);
 	dev->ibdev.driver_id = RDMA_DRIVER_CXGB4;
 	ib_set_device_ops(&dev->ibdev, &c4iw_dev_ops);
+	ret = set_netdevs(&dev->ibdev, &dev->rdev);
+	if (ret)
+		goto err_dealloc_ctx;
 	ret = ib_register_device(&dev->ibdev, "cxgb4_%d");
 	if (ret)
-		goto err_kfree_iwcm;
+		goto err_dealloc_ctx;
 	return;
 
-err_kfree_iwcm:
-	kfree(dev->ibdev.iwcm);
 err_dealloc_ctx:
 	pr_err("%s - Failed registering iwarp device: %d\n",
 	       pci_name(ctx->lldi.pdev), ret);
@@ -631,6 +621,5 @@ void c4iw_unregister_device(struct c4iw_dev *dev)
 {
 	pr_debug("c4iw_dev %p\n", dev);
 	ib_unregister_device(&dev->ibdev);
-	kfree(dev->ibdev.iwcm);
 	return;
 }
diff --git a/drivers/infiniband/hw/cxgb4/qp.c b/drivers/infiniband/hw/cxgb4/qp.c
index d3a82839f5ea..e92b9544357a 100644
--- a/drivers/infiniband/hw/cxgb4/qp.c
+++ b/drivers/infiniband/hw/cxgb4/qp.c
@@ -57,18 +57,18 @@ MODULE_PARM_DESC(db_coalescing_threshold,
 
 static int max_fr_immd = T4_MAX_FR_IMMD;
 module_param(max_fr_immd, int, 0644);
-MODULE_PARM_DESC(max_fr_immd, "fastreg threshold for using DSGL instead of immedate");
+MODULE_PARM_DESC(max_fr_immd, "fastreg threshold for using DSGL instead of immediate");
 
 static int alloc_ird(struct c4iw_dev *dev, u32 ird)
 {
 	int ret = 0;
 
-	spin_lock_irq(&dev->lock);
+	xa_lock_irq(&dev->qps);
 	if (ird <= dev->avail_ird)
 		dev->avail_ird -= ird;
 	else
 		ret = -ENOMEM;
-	spin_unlock_irq(&dev->lock);
+	xa_unlock_irq(&dev->qps);
 
 	if (ret)
 		dev_warn(&dev->rdev.lldi.pdev->dev,
@@ -79,9 +79,9 @@ static int alloc_ird(struct c4iw_dev *dev, u32 ird)
 
 static void free_ird(struct c4iw_dev *dev, int ird)
 {
-	spin_lock_irq(&dev->lock);
+	xa_lock_irq(&dev->qps);
 	dev->avail_ird += ird;
-	spin_unlock_irq(&dev->lock);
+	xa_unlock_irq(&dev->qps);
 }
 
 static void set_state(struct c4iw_qp *qhp, enum c4iw_qp_state state)
@@ -939,7 +939,7 @@ static int ring_kernel_sq_db(struct c4iw_qp *qhp, u16 inc)
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(&qhp->rhp->lock, flags);
+	xa_lock_irqsave(&qhp->rhp->qps, flags);
 	spin_lock(&qhp->lock);
 	if (qhp->rhp->db_state == NORMAL)
 		t4_ring_sq_db(&qhp->wq, inc, NULL);
@@ -948,7 +948,7 @@ static int ring_kernel_sq_db(struct c4iw_qp *qhp, u16 inc)
 		qhp->wq.sq.wq_pidx_inc += inc;
 	}
 	spin_unlock(&qhp->lock);
-	spin_unlock_irqrestore(&qhp->rhp->lock, flags);
+	xa_unlock_irqrestore(&qhp->rhp->qps, flags);
 	return 0;
 }
 
@@ -956,7 +956,7 @@ static int ring_kernel_rq_db(struct c4iw_qp *qhp, u16 inc)
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(&qhp->rhp->lock, flags);
+	xa_lock_irqsave(&qhp->rhp->qps, flags);
 	spin_lock(&qhp->lock);
 	if (qhp->rhp->db_state == NORMAL)
 		t4_ring_rq_db(&qhp->wq, inc, NULL);
@@ -965,7 +965,7 @@ static int ring_kernel_rq_db(struct c4iw_qp *qhp, u16 inc)
 		qhp->wq.rq.wq_pidx_inc += inc;
 	}
 	spin_unlock(&qhp->lock);
-	spin_unlock_irqrestore(&qhp->rhp->lock, flags);
+	xa_unlock_irqrestore(&qhp->rhp->qps, flags);
 	return 0;
 }
 
@@ -1976,10 +1976,10 @@ int c4iw_modify_qp(struct c4iw_dev *rhp, struct c4iw_qp *qhp,
 			qhp->attr.layer_etype = attrs->layer_etype;
 			qhp->attr.ecode = attrs->ecode;
 			ep = qhp->ep;
+			c4iw_get_ep(&ep->com);
+			disconnect = 1;
 			if (!internal) {
-				c4iw_get_ep(&qhp->ep->com);
 				terminate = 1;
-				disconnect = 1;
 			} else {
 				terminate = qhp->attr.send_term;
 				ret = rdma_fini(rhp, qhp, ep);
@@ -2095,7 +2095,7 @@ out:
 	return ret;
 }
 
-int c4iw_destroy_qp(struct ib_qp *ib_qp)
+int c4iw_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata)
 {
 	struct c4iw_dev *rhp;
 	struct c4iw_qp *qhp;
@@ -2111,12 +2111,11 @@ int c4iw_destroy_qp(struct ib_qp *ib_qp)
 		c4iw_modify_qp(rhp, qhp, C4IW_QP_ATTR_NEXT_STATE, &attrs, 0);
 	wait_event(qhp->wait, !qhp->ep);
 
-	remove_handle(rhp, &rhp->qpidr, qhp->wq.sq.qid);
-
-	spin_lock_irq(&rhp->lock);
+	xa_lock_irq(&rhp->qps);
+	__xa_erase(&rhp->qps, qhp->wq.sq.qid);
 	if (!list_empty(&qhp->db_fc_entry))
 		list_del_init(&qhp->db_fc_entry);
-	spin_unlock_irq(&rhp->lock);
+	xa_unlock_irq(&rhp->qps);
 	free_ird(rhp, qhp->attr.max_ird);
 
 	c4iw_qp_rem_ref(ib_qp);
@@ -2234,7 +2233,7 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs,
 	kref_init(&qhp->kref);
 	INIT_WORK(&qhp->free_work, free_qp_work);
 
-	ret = insert_handle(rhp, &rhp->qpidr, qhp, qhp->wq.sq.qid);
+	ret = xa_insert_irq(&rhp->qps, qhp->wq.sq.qid, qhp, GFP_KERNEL);
 	if (ret)
 		goto err_destroy_qp;
 
@@ -2370,7 +2369,7 @@ err_free_rq_key:
 err_free_sq_key:
 	kfree(sq_key_mm);
 err_remove_handle:
-	remove_handle(rhp, &rhp->qpidr, qhp->wq.sq.qid);
+	xa_erase_irq(&rhp->qps, qhp->wq.sq.qid);
 err_destroy_qp:
 	destroy_qp(&rhp->rdev, &qhp->wq,
 		   ucontext ? &ucontext->uctx : &rhp->rdev.uctx, !attrs->srq);
@@ -2684,11 +2683,12 @@ void c4iw_copy_wr_to_srq(struct t4_srq *srq, union t4_recv_wr *wqe, u8 len16)
 	}
 }
 
-struct ib_srq *c4iw_create_srq(struct ib_pd *pd, struct ib_srq_init_attr *attrs,
+int c4iw_create_srq(struct ib_srq *ib_srq, struct ib_srq_init_attr *attrs,
 			       struct ib_udata *udata)
 {
+	struct ib_pd *pd = ib_srq->pd;
 	struct c4iw_dev *rhp;
-	struct c4iw_srq *srq;
+	struct c4iw_srq *srq = to_c4iw_srq(ib_srq);
 	struct c4iw_pd *php;
 	struct c4iw_create_srq_resp uresp;
 	struct c4iw_ucontext *ucontext;
@@ -2703,11 +2703,11 @@ struct ib_srq *c4iw_create_srq(struct ib_pd *pd, struct ib_srq_init_attr *attrs,
 	rhp = php->rhp;
 
 	if (!rhp->rdev.lldi.vr->srq.size)
-		return ERR_PTR(-EINVAL);
+		return -EINVAL;
 	if (attrs->attr.max_wr > rhp->rdev.hw_queue.t4_max_rq_size)
-		return ERR_PTR(-E2BIG);
+		return -E2BIG;
 	if (attrs->attr.max_sge > T4_MAX_RECV_SGE)
-		return ERR_PTR(-E2BIG);
+		return -E2BIG;
 
 	/*
 	 * SRQ RQT and RQ must be a power of 2 and at least 16 deep.
@@ -2718,15 +2718,9 @@ struct ib_srq *c4iw_create_srq(struct ib_pd *pd, struct ib_srq_init_attr *attrs,
 	ucontext = rdma_udata_to_drv_context(udata, struct c4iw_ucontext,
 					     ibucontext);
 
-	srq = kzalloc(sizeof(*srq), GFP_KERNEL);
-	if (!srq)
-		return ERR_PTR(-ENOMEM);
-
 	srq->wr_waitp = c4iw_alloc_wr_wait(GFP_KERNEL);
-	if (!srq->wr_waitp) {
-		ret = -ENOMEM;
-		goto err_free_srq;
-	}
+	if (!srq->wr_waitp)
+		return -ENOMEM;
 
 	srq->idx = c4iw_alloc_srq_idx(&rhp->rdev);
 	if (srq->idx < 0) {
@@ -2760,7 +2754,7 @@ struct ib_srq *c4iw_create_srq(struct ib_pd *pd, struct ib_srq_init_attr *attrs,
 	if (CHELSIO_CHIP_VERSION(rhp->rdev.lldi.adapter_type) > CHELSIO_T6)
 		srq->flags = T4_SRQ_LIMIT_SUPPORT;
 
-	ret = insert_handle(rhp, &rhp->qpidr, srq, srq->wq.qid);
+	ret = xa_insert_irq(&rhp->qps, srq->wq.qid, srq, GFP_KERNEL);
 	if (ret)
 		goto err_free_queue;
 
@@ -2806,13 +2800,14 @@ struct ib_srq *c4iw_create_srq(struct ib_pd *pd, struct ib_srq_init_attr *attrs,
 			(unsigned long)srq->wq.memsize, attrs->attr.max_wr);
 
 	spin_lock_init(&srq->lock);
-	return &srq->ibsrq;
+	return 0;
+
 err_free_srq_db_key_mm:
 	kfree(srq_db_key_mm);
 err_free_srq_key_mm:
 	kfree(srq_key_mm);
 err_remove_handle:
-	remove_handle(rhp, &rhp->qpidr, srq->wq.qid);
+	xa_erase_irq(&rhp->qps, srq->wq.qid);
 err_free_queue:
 	free_srq_queue(srq, ucontext ? &ucontext->uctx : &rhp->rdev.uctx,
 		       srq->wr_waitp);
@@ -2822,12 +2817,10 @@ err_free_srq_idx:
 	c4iw_free_srq_idx(&rhp->rdev, srq->idx);
 err_free_wr_wait:
 	c4iw_put_wr_wait(srq->wr_waitp);
-err_free_srq:
-	kfree(srq);
-	return ERR_PTR(ret);
+	return ret;
 }
 
-int c4iw_destroy_srq(struct ib_srq *ibsrq)
+void c4iw_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
 {
 	struct c4iw_dev *rhp;
 	struct c4iw_srq *srq;
@@ -2838,13 +2831,11 @@ int c4iw_destroy_srq(struct ib_srq *ibsrq)
 
 	pr_debug("%s id %d\n", __func__, srq->wq.qid);
 
-	remove_handle(rhp, &rhp->qpidr, srq->wq.qid);
-	ucontext = ibsrq->uobject ?
-		to_c4iw_ucontext(ibsrq->uobject->context) : NULL;
+	xa_erase_irq(&rhp->qps, srq->wq.qid);
+	ucontext = rdma_udata_to_drv_context(udata, struct c4iw_ucontext,
+					     ibucontext);
 	free_srq_queue(srq, ucontext ? &ucontext->uctx : &rhp->rdev.uctx,
 		       srq->wr_waitp);
 	c4iw_free_srq_idx(&rhp->rdev, srq->idx);
 	c4iw_put_wr_wait(srq->wr_waitp);
-	kfree(srq);
-	return 0;
 }
diff --git a/drivers/infiniband/hw/efa/Kconfig b/drivers/infiniband/hw/efa/Kconfig
new file mode 100644
index 000000000000..457e18ba1d57
--- /dev/null
+++ b/drivers/infiniband/hw/efa/Kconfig
@@ -0,0 +1,15 @@
+# SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
+# Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
+#
+# Amazon fabric device configuration
+#
+
+config INFINIBAND_EFA
+	tristate "Amazon Elastic Fabric Adapter (EFA) support"
+	depends on PCI_MSI && 64BIT && !CPU_BIG_ENDIAN
+	depends on INFINIBAND_USER_ACCESS
+	help
+	  This driver supports Amazon Elastic Fabric Adapter (EFA).
+
+	  To compile this driver as a module, choose M here.
+	  The module will be called efa.
diff --git a/drivers/infiniband/hw/efa/Makefile b/drivers/infiniband/hw/efa/Makefile
new file mode 100644
index 000000000000..6e83083af0bc
--- /dev/null
+++ b/drivers/infiniband/hw/efa/Makefile
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
+# Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
+#
+# Makefile for Amazon Elastic Fabric Adapter (EFA) device driver.
+#
+
+obj-$(CONFIG_INFINIBAND_EFA) += efa.o
+
+efa-y := efa_com_cmd.o efa_com.o efa_main.o efa_verbs.o
diff --git a/drivers/infiniband/hw/efa/efa.h b/drivers/infiniband/hw/efa/efa.h
new file mode 100644
index 000000000000..9e3cc3239c13
--- /dev/null
+++ b/drivers/infiniband/hw/efa/efa.h
@@ -0,0 +1,163 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
+/*
+ * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
+ */
+
+#ifndef _EFA_H_
+#define _EFA_H_
+
+#include <linux/bitops.h>
+#include <linux/idr.h>
+#include <linux/interrupt.h>
+#include <linux/pci.h>
+#include <linux/sched.h>
+
+#include <rdma/efa-abi.h>
+#include <rdma/ib_verbs.h>
+
+#include "efa_com_cmd.h"
+
+#define DRV_MODULE_NAME         "efa"
+#define DEVICE_NAME             "Elastic Fabric Adapter (EFA)"
+
+#define EFA_IRQNAME_SIZE        40
+
+/* 1 for AENQ + ADMIN */
+#define EFA_NUM_MSIX_VEC                  1
+#define EFA_MGMNT_MSIX_VEC_IDX            0
+
+struct efa_irq {
+	irq_handler_t handler;
+	void *data;
+	int cpu;
+	u32 vector;
+	cpumask_t affinity_hint_mask;
+	char name[EFA_IRQNAME_SIZE];
+};
+
+struct efa_sw_stats {
+	atomic64_t alloc_pd_err;
+	atomic64_t create_qp_err;
+	atomic64_t create_cq_err;
+	atomic64_t reg_mr_err;
+	atomic64_t alloc_ucontext_err;
+	atomic64_t create_ah_err;
+};
+
+/* Don't use anything other than atomic64 */
+struct efa_stats {
+	struct efa_sw_stats sw_stats;
+	atomic64_t keep_alive_rcvd;
+};
+
+struct efa_dev {
+	struct ib_device ibdev;
+	struct efa_com_dev edev;
+	struct pci_dev *pdev;
+	struct efa_com_get_device_attr_result dev_attr;
+
+	u64 reg_bar_addr;
+	u64 reg_bar_len;
+	u64 mem_bar_addr;
+	u64 mem_bar_len;
+	u64 db_bar_addr;
+	u64 db_bar_len;
+	u8 addr[EFA_GID_SIZE];
+	u32 mtu;
+
+	int admin_msix_vector_idx;
+	struct efa_irq admin_irq;
+
+	struct efa_stats stats;
+};
+
+struct efa_ucontext {
+	struct ib_ucontext ibucontext;
+	struct xarray mmap_xa;
+	u32 mmap_xa_page;
+	u16 uarn;
+};
+
+struct efa_pd {
+	struct ib_pd ibpd;
+	u16 pdn;
+};
+
+struct efa_mr {
+	struct ib_mr ibmr;
+	struct ib_umem *umem;
+};
+
+struct efa_cq {
+	struct ib_cq ibcq;
+	struct efa_ucontext *ucontext;
+	dma_addr_t dma_addr;
+	void *cpu_addr;
+	size_t size;
+	u16 cq_idx;
+};
+
+struct efa_qp {
+	struct ib_qp ibqp;
+	dma_addr_t rq_dma_addr;
+	void *rq_cpu_addr;
+	size_t rq_size;
+	enum ib_qp_state state;
+	u32 qp_handle;
+	u32 max_send_wr;
+	u32 max_recv_wr;
+	u32 max_send_sge;
+	u32 max_recv_sge;
+	u32 max_inline_data;
+};
+
+struct efa_ah {
+	struct ib_ah ibah;
+	u16 ah;
+	/* dest_addr */
+	u8 id[EFA_GID_SIZE];
+};
+
+int efa_query_device(struct ib_device *ibdev,
+		     struct ib_device_attr *props,
+		     struct ib_udata *udata);
+int efa_query_port(struct ib_device *ibdev, u8 port,
+		   struct ib_port_attr *props);
+int efa_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
+		 int qp_attr_mask,
+		 struct ib_qp_init_attr *qp_init_attr);
+int efa_query_gid(struct ib_device *ibdev, u8 port, int index,
+		  union ib_gid *gid);
+int efa_query_pkey(struct ib_device *ibdev, u8 port, u16 index,
+		   u16 *pkey);
+int efa_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
+void efa_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
+int efa_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);
+struct ib_qp *efa_create_qp(struct ib_pd *ibpd,
+			    struct ib_qp_init_attr *init_attr,
+			    struct ib_udata *udata);
+int efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
+struct ib_cq *efa_create_cq(struct ib_device *ibdev,
+			    const struct ib_cq_init_attr *attr,
+			    struct ib_udata *udata);
+struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length,
+			 u64 virt_addr, int access_flags,
+			 struct ib_udata *udata);
+int efa_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
+int efa_get_port_immutable(struct ib_device *ibdev, u8 port_num,
+			   struct ib_port_immutable *immutable);
+int efa_alloc_ucontext(struct ib_ucontext *ibucontext, struct ib_udata *udata);
+void efa_dealloc_ucontext(struct ib_ucontext *ibucontext);
+int efa_mmap(struct ib_ucontext *ibucontext,
+	     struct vm_area_struct *vma);
+int efa_create_ah(struct ib_ah *ibah,
+		  struct rdma_ah_attr *ah_attr,
+		  u32 flags,
+		  struct ib_udata *udata);
+void efa_destroy_ah(struct ib_ah *ibah, u32 flags);
+int efa_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
+		  int qp_attr_mask, struct ib_udata *udata);
+enum rdma_link_layer efa_port_link_layer(struct ib_device *ibdev,
+					 u8 port_num);
+
+#endif /* _EFA_H_ */
diff --git a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h
new file mode 100644
index 000000000000..2be0469d545f
--- /dev/null
+++ b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h
@@ -0,0 +1,794 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
+/*
+ * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
+ */
+
+#ifndef _EFA_ADMIN_CMDS_H_
+#define _EFA_ADMIN_CMDS_H_
+
+#define EFA_ADMIN_API_VERSION_MAJOR          0
+#define EFA_ADMIN_API_VERSION_MINOR          1
+
+/* EFA admin queue opcodes */
+enum efa_admin_aq_opcode {
+	EFA_ADMIN_CREATE_QP                         = 1,
+	EFA_ADMIN_MODIFY_QP                         = 2,
+	EFA_ADMIN_QUERY_QP                          = 3,
+	EFA_ADMIN_DESTROY_QP                        = 4,
+	EFA_ADMIN_CREATE_AH                         = 5,
+	EFA_ADMIN_DESTROY_AH                        = 6,
+	EFA_ADMIN_REG_MR                            = 7,
+	EFA_ADMIN_DEREG_MR                          = 8,
+	EFA_ADMIN_CREATE_CQ                         = 9,
+	EFA_ADMIN_DESTROY_CQ                        = 10,
+	EFA_ADMIN_GET_FEATURE                       = 11,
+	EFA_ADMIN_SET_FEATURE                       = 12,
+	EFA_ADMIN_GET_STATS                         = 13,
+	EFA_ADMIN_ALLOC_PD                          = 14,
+	EFA_ADMIN_DEALLOC_PD                        = 15,
+	EFA_ADMIN_ALLOC_UAR                         = 16,
+	EFA_ADMIN_DEALLOC_UAR                       = 17,
+	EFA_ADMIN_MAX_OPCODE                        = 17,
+};
+
+enum efa_admin_aq_feature_id {
+	EFA_ADMIN_DEVICE_ATTR                       = 1,
+	EFA_ADMIN_AENQ_CONFIG                       = 2,
+	EFA_ADMIN_NETWORK_ATTR                      = 3,
+	EFA_ADMIN_QUEUE_ATTR                        = 4,
+	EFA_ADMIN_HW_HINTS                          = 5,
+	EFA_ADMIN_FEATURES_OPCODE_NUM               = 8,
+};
+
+/* QP transport type */
+enum efa_admin_qp_type {
+	/* Unreliable Datagram */
+	EFA_ADMIN_QP_TYPE_UD                        = 1,
+	/* Scalable Reliable Datagram */
+	EFA_ADMIN_QP_TYPE_SRD                       = 2,
+};
+
+/* QP state */
+enum efa_admin_qp_state {
+	EFA_ADMIN_QP_STATE_RESET                    = 0,
+	EFA_ADMIN_QP_STATE_INIT                     = 1,
+	EFA_ADMIN_QP_STATE_RTR                      = 2,
+	EFA_ADMIN_QP_STATE_RTS                      = 3,
+	EFA_ADMIN_QP_STATE_SQD                      = 4,
+	EFA_ADMIN_QP_STATE_SQE                      = 5,
+	EFA_ADMIN_QP_STATE_ERR                      = 6,
+};
+
+enum efa_admin_get_stats_type {
+	EFA_ADMIN_GET_STATS_TYPE_BASIC              = 0,
+};
+
+enum efa_admin_get_stats_scope {
+	EFA_ADMIN_GET_STATS_SCOPE_ALL               = 0,
+	EFA_ADMIN_GET_STATS_SCOPE_QUEUE             = 1,
+};
+
+enum efa_admin_modify_qp_mask_bits {
+	EFA_ADMIN_QP_STATE_BIT                      = 0,
+	EFA_ADMIN_CUR_QP_STATE_BIT                  = 1,
+	EFA_ADMIN_QKEY_BIT                          = 2,
+	EFA_ADMIN_SQ_PSN_BIT                        = 3,
+	EFA_ADMIN_SQ_DRAINED_ASYNC_NOTIFY_BIT       = 4,
+};
+
+/*
+ * QP allocation sizes, converted by fabric QueuePair (QP) create command
+ * from QP capabilities.
+ */
+struct efa_admin_qp_alloc_size {
+	/* Send descriptor ring size in bytes */
+	u32 send_queue_ring_size;
+
+	/* Max number of WQEs that can be outstanding on send queue. */
+	u32 send_queue_depth;
+
+	/*
+	 * Recv descriptor ring size in bytes, sufficient for user-provided
+	 * number of WQEs
+	 */
+	u32 recv_queue_ring_size;
+
+	/* Max number of WQEs that can be outstanding on recv queue */
+	u32 recv_queue_depth;
+};
+
+struct efa_admin_create_qp_cmd {
+	/* Common Admin Queue descriptor */
+	struct efa_admin_aq_common_desc aq_common_desc;
+
+	/* Protection Domain associated with this QP */
+	u16 pd;
+
+	/* QP type */
+	u8 qp_type;
+
+	/*
+	 * 0 : sq_virt - If set, SQ ring base address is
+	 *    virtual (IOVA returned by MR registration)
+	 * 1 : rq_virt - If set, RQ ring base address is
+	 *    virtual (IOVA returned by MR registration)
+	 * 7:2 : reserved - MBZ
+	 */
+	u8 flags;
+
+	/*
+	 * Send queue (SQ) ring base physical address. This field is not
+	 * used if this is a Low Latency Queue(LLQ).
+	 */
+	u64 sq_base_addr;
+
+	/* Receive queue (RQ) ring base address. */
+	u64 rq_base_addr;
+
+	/* Index of CQ to be associated with Send Queue completions */
+	u32 send_cq_idx;
+
+	/* Index of CQ to be associated with Recv Queue completions */
+	u32 recv_cq_idx;
+
+	/*
+	 * Memory registration key for the SQ ring, used only when not in
+	 * LLQ mode and base address is virtual
+	 */
+	u32 sq_l_key;
+
+	/*
+	 * Memory registration key for the RQ ring, used only when base
+	 * address is virtual
+	 */
+	u32 rq_l_key;
+
+	/* Requested QP allocation sizes */
+	struct efa_admin_qp_alloc_size qp_alloc_size;
+
+	/* UAR number */
+	u16 uar;
+
+	/* MBZ */
+	u16 reserved;
+
+	/* MBZ */
+	u32 reserved2;
+};
+
+struct efa_admin_create_qp_resp {
+	/* Common Admin Queue completion descriptor */
+	struct efa_admin_acq_common_desc acq_common_desc;
+
+	/* Opaque handle to be used for consequent operations on the QP */
+	u32 qp_handle;
+
+	/* QP number in the given EFA virtual device */
+	u16 qp_num;
+
+	/* MBZ */
+	u16 reserved;
+
+	/* Index of sub-CQ for Send Queue completions */
+	u16 send_sub_cq_idx;
+
+	/* Index of sub-CQ for Receive Queue completions */
+	u16 recv_sub_cq_idx;
+
+	/* SQ doorbell address, as offset to PCIe DB BAR */
+	u32 sq_db_offset;
+
+	/* RQ doorbell address, as offset to PCIe DB BAR */
+	u32 rq_db_offset;
+
+	/*
+	 * low latency send queue ring base address as an offset to PCIe
+	 * MMIO LLQ_MEM BAR
+	 */
+	u32 llq_descriptors_offset;
+};
+
+struct efa_admin_modify_qp_cmd {
+	/* Common Admin Queue descriptor */
+	struct efa_admin_aq_common_desc aq_common_desc;
+
+	/*
+	 * Mask indicating which fields should be updated see enum
+	 * efa_admin_modify_qp_mask_bits
+	 */
+	u32 modify_mask;
+
+	/* QP handle returned by create_qp command */
+	u32 qp_handle;
+
+	/* QP state */
+	u32 qp_state;
+
+	/* Override current QP state (before applying the transition) */
+	u32 cur_qp_state;
+
+	/* QKey */
+	u32 qkey;
+
+	/* SQ PSN */
+	u32 sq_psn;
+
+	/* Enable async notification when SQ is drained */
+	u8 sq_drained_async_notify;
+
+	/* MBZ */
+	u8 reserved1;
+
+	/* MBZ */
+	u16 reserved2;
+};
+
+struct efa_admin_modify_qp_resp {
+	/* Common Admin Queue completion descriptor */
+	struct efa_admin_acq_common_desc acq_common_desc;
+};
+
+struct efa_admin_query_qp_cmd {
+	/* Common Admin Queue descriptor */
+	struct efa_admin_aq_common_desc aq_common_desc;
+
+	/* QP handle returned by create_qp command */
+	u32 qp_handle;
+};
+
+struct efa_admin_query_qp_resp {
+	/* Common Admin Queue completion descriptor */
+	struct efa_admin_acq_common_desc acq_common_desc;
+
+	/* QP state */
+	u32 qp_state;
+
+	/* QKey */
+	u32 qkey;
+
+	/* SQ PSN */
+	u32 sq_psn;
+
+	/* Indicates that draining is in progress */
+	u8 sq_draining;
+
+	/* MBZ */
+	u8 reserved1;
+
+	/* MBZ */
+	u16 reserved2;
+};
+
+struct efa_admin_destroy_qp_cmd {
+	/* Common Admin Queue descriptor */
+	struct efa_admin_aq_common_desc aq_common_desc;
+
+	/* QP handle returned by create_qp command */
+	u32 qp_handle;
+};
+
+struct efa_admin_destroy_qp_resp {
+	/* Common Admin Queue completion descriptor */
+	struct efa_admin_acq_common_desc acq_common_desc;
+};
+
+/*
+ * Create Address Handle command parameters. Must not be called more than
+ * once for the same destination
+ */
+struct efa_admin_create_ah_cmd {
+	/* Common Admin Queue descriptor */
+	struct efa_admin_aq_common_desc aq_common_desc;
+
+	/* Destination address in network byte order */
+	u8 dest_addr[16];
+
+	/* PD number */
+	u16 pd;
+
+	u16 reserved;
+};
+
+struct efa_admin_create_ah_resp {
+	/* Common Admin Queue completion descriptor */
+	struct efa_admin_acq_common_desc acq_common_desc;
+
+	/* Target interface address handle (opaque) */
+	u16 ah;
+
+	u16 reserved;
+};
+
+struct efa_admin_destroy_ah_cmd {
+	/* Common Admin Queue descriptor */
+	struct efa_admin_aq_common_desc aq_common_desc;
+
+	/* Target interface address handle (opaque) */
+	u16 ah;
+
+	/* PD number */
+	u16 pd;
+};
+
+struct efa_admin_destroy_ah_resp {
+	/* Common Admin Queue completion descriptor */
+	struct efa_admin_acq_common_desc acq_common_desc;
+};
+
+/*
+ * Registration of MemoryRegion, required for QP working with Virtual
+ * Addresses. In standard verbs semantics, region length is limited to 2GB
+ * space, but EFA offers larger MR support for large memory space, to ease
+ * on users working with very large datasets (i.e. full GPU memory mapping).
+ */
+struct efa_admin_reg_mr_cmd {
+	/* Common Admin Queue descriptor */
+	struct efa_admin_aq_common_desc aq_common_desc;
+
+	/* Protection Domain */
+	u16 pd;
+
+	/* MBZ */
+	u16 reserved16_w1;
+
+	/* Physical Buffer List, each element is page-aligned. */
+	union {
+		/*
+		 * Inline array of guest-physical page addresses of user
+		 * memory pages (optimization for short region
+		 * registrations)
+		 */
+		u64 inline_pbl_array[4];
+
+		/* points to PBL (direct or indirect, chained if needed) */
+		struct efa_admin_ctrl_buff_info pbl;
+	} pbl;
+
+	/* Memory region length, in bytes. */
+	u64 mr_length;
+
+	/*
+	 * flags and page size
+	 * 4:0 : phys_page_size_shift - page size is (1 <<
+	 *    phys_page_size_shift). Page size is used for
+	 *    building the Virtual to Physical address mapping
+	 * 6:5 : reserved - MBZ
+	 * 7 : mem_addr_phy_mode_en - Enable bit for physical
+	 *    memory registration (no translation), can be used
+	 *    only by privileged clients. If set, PBL must
+	 *    contain a single entry.
+	 */
+	u8 flags;
+
+	/*
+	 * permissions
+	 * 0 : local_write_enable - Write permissions: value
+	 *    of 1 needed for RQ buffers and for RDMA write
+	 * 7:1 : reserved1 - remote access flags, etc
+	 */
+	u8 permissions;
+
+	u16 reserved16_w5;
+
+	/* number of pages in PBL (redundant, could be calculated) */
+	u32 page_num;
+
+	/*
+	 * IO Virtual Address associated with this MR. If
+	 * mem_addr_phy_mode_en is set, contains the physical address of
+	 * the region.
+	 */
+	u64 iova;
+};
+
+struct efa_admin_reg_mr_resp {
+	/* Common Admin Queue completion descriptor */
+	struct efa_admin_acq_common_desc acq_common_desc;
+
+	/*
+	 * L_Key, to be used in conjunction with local buffer references in
+	 * SQ and RQ WQE, or with virtual RQ/CQ rings
+	 */
+	u32 l_key;
+
+	/*
+	 * R_Key, to be used in RDMA messages to refer to remotely accessed
+	 * memory region
+	 */
+	u32 r_key;
+};
+
+struct efa_admin_dereg_mr_cmd {
+	/* Common Admin Queue descriptor */
+	struct efa_admin_aq_common_desc aq_common_desc;
+
+	/* L_Key, memory region's l_key */
+	u32 l_key;
+};
+
+struct efa_admin_dereg_mr_resp {
+	/* Common Admin Queue completion descriptor */
+	struct efa_admin_acq_common_desc acq_common_desc;
+};
+
+struct efa_admin_create_cq_cmd {
+	struct efa_admin_aq_common_desc aq_common_desc;
+
+	/*
+	 * 4:0 : reserved5
+	 * 5 : interrupt_mode_enabled - if set, cq operates
+	 *    in interrupt mode (i.e. CQ events and MSI-X are
+	 *    generated), otherwise - polling
+	 * 6 : virt - If set, ring base address is virtual
+	 *    (IOVA returned by MR registration)
+	 * 7 : reserved6
+	 */
+	u8 cq_caps_1;
+
+	/*
+	 * 4:0 : cq_entry_size_words - size of CQ entry in
+	 *    32-bit words, valid values: 4, 8.
+	 * 7:5 : reserved7
+	 */
+	u8 cq_caps_2;
+
+	/* completion queue depth in # of entries. must be power of 2 */
+	u16 cq_depth;
+
+	/* msix vector assigned to this cq */
+	u32 msix_vector_idx;
+
+	/*
+	 * CQ ring base address, virtual or physical depending on 'virt'
+	 * flag
+	 */
+	struct efa_common_mem_addr cq_ba;
+
+	/*
+	 * Memory registration key for the ring, used only when base
+	 * address is virtual
+	 */
+	u32 l_key;
+
+	/*
+	 * number of sub cqs - must be equal to sub_cqs_per_cq of queue
+	 *    attributes.
+	 */
+	u16 num_sub_cqs;
+
+	/* UAR number */
+	u16 uar;
+};
+
+struct efa_admin_create_cq_resp {
+	struct efa_admin_acq_common_desc acq_common_desc;
+
+	u16 cq_idx;
+
+	/* actual cq depth in number of entries */
+	u16 cq_actual_depth;
+};
+
+struct efa_admin_destroy_cq_cmd {
+	struct efa_admin_aq_common_desc aq_common_desc;
+
+	u16 cq_idx;
+
+	u16 reserved1;
+};
+
+struct efa_admin_destroy_cq_resp {
+	struct efa_admin_acq_common_desc acq_common_desc;
+};
+
+/*
+ * EFA AQ Get Statistics command. Extended statistics are placed in control
+ * buffer pointed by AQ entry
+ */
+struct efa_admin_aq_get_stats_cmd {
+	struct efa_admin_aq_common_desc aq_common_descriptor;
+
+	union {
+		/* command specific inline data */
+		u32 inline_data_w1[3];
+
+		struct efa_admin_ctrl_buff_info control_buffer;
+	} u;
+
+	/* stats type as defined in enum efa_admin_get_stats_type */
+	u8 type;
+
+	/* stats scope defined in enum efa_admin_get_stats_scope */
+	u8 scope;
+
+	u16 scope_modifier;
+};
+
+struct efa_admin_basic_stats {
+	u64 tx_bytes;
+
+	u64 tx_pkts;
+
+	u64 rx_bytes;
+
+	u64 rx_pkts;
+
+	u64 rx_drops;
+};
+
+struct efa_admin_acq_get_stats_resp {
+	struct efa_admin_acq_common_desc acq_common_desc;
+
+	struct efa_admin_basic_stats basic_stats;
+};
+
+struct efa_admin_get_set_feature_common_desc {
+	/*
+	 * 1:0 : select - 0x1 - current value; 0x3 - default
+	 *    value
+	 * 7:3 : reserved3
+	 */
+	u8 flags;
+
+	/* as appears in efa_admin_aq_feature_id */
+	u8 feature_id;
+
+	/* MBZ */
+	u16 reserved16;
+};
+
+struct efa_admin_feature_device_attr_desc {
+	/* Bitmap of efa_admin_aq_feature_id */
+	u64 supported_features;
+
+	/* Bitmap of supported page sizes in MR registrations */
+	u64 page_size_cap;
+
+	u32 fw_version;
+
+	u32 admin_api_version;
+
+	u32 device_version;
+
+	/* Bar used for SQ and RQ doorbells */
+	u16 db_bar;
+
+	/* Indicates how many bits are used physical address access */
+	u8 phys_addr_width;
+
+	/* Indicates how many bits are used virtual address access */
+	u8 virt_addr_width;
+};
+
+struct efa_admin_feature_queue_attr_desc {
+	/* The maximum number of queue pairs supported */
+	u32 max_qp;
+
+	u32 max_sq_depth;
+
+	/* max send wr used in inline-buf */
+	u32 inline_buf_size;
+
+	u32 max_rq_depth;
+
+	/* The maximum number of completion queues supported per VF */
+	u32 max_cq;
+
+	u32 max_cq_depth;
+
+	/* Number of sub-CQs to be created for each CQ */
+	u16 sub_cqs_per_cq;
+
+	u16 reserved;
+
+	/*
+	 * Maximum number of SGEs (buffs) allowed for a single send work
+	 *    queue element (WQE)
+	 */
+	u16 max_wr_send_sges;
+
+	/* Maximum number of SGEs allowed for a single recv WQE */
+	u16 max_wr_recv_sges;
+
+	/* The maximum number of memory regions supported */
+	u32 max_mr;
+
+	/* The maximum number of pages can be registered */
+	u32 max_mr_pages;
+
+	/* The maximum number of protection domains supported */
+	u32 max_pd;
+
+	/* The maximum number of address handles supported */
+	u32 max_ah;
+
+	/* The maximum size of LLQ in bytes */
+	u32 max_llq_size;
+};
+
+struct efa_admin_feature_aenq_desc {
+	/* bitmask for AENQ groups the device can report */
+	u32 supported_groups;
+
+	/* bitmask for AENQ groups to report */
+	u32 enabled_groups;
+};
+
+struct efa_admin_feature_network_attr_desc {
+	/* Raw address data in network byte order */
+	u8 addr[16];
+
+	u32 mtu;
+};
+
+/*
+ * When hint value is 0, hints capabilities are not supported or driver
+ * should use its own predefined value
+ */
+struct efa_admin_hw_hints {
+	/* value in ms */
+	u16 mmio_read_timeout;
+
+	/* value in ms */
+	u16 driver_watchdog_timeout;
+
+	/* value in ms */
+	u16 admin_completion_timeout;
+
+	/* poll interval in ms */
+	u16 poll_interval;
+};
+
+struct efa_admin_get_feature_cmd {
+	struct efa_admin_aq_common_desc aq_common_descriptor;
+
+	struct efa_admin_ctrl_buff_info control_buffer;
+
+	struct efa_admin_get_set_feature_common_desc feature_common;
+
+	u32 raw[11];
+};
+
+struct efa_admin_get_feature_resp {
+	struct efa_admin_acq_common_desc acq_common_desc;
+
+	union {
+		u32 raw[14];
+
+		struct efa_admin_feature_device_attr_desc device_attr;
+
+		struct efa_admin_feature_aenq_desc aenq;
+
+		struct efa_admin_feature_network_attr_desc network_attr;
+
+		struct efa_admin_feature_queue_attr_desc queue_attr;
+
+		struct efa_admin_hw_hints hw_hints;
+	} u;
+};
+
+struct efa_admin_set_feature_cmd {
+	struct efa_admin_aq_common_desc aq_common_descriptor;
+
+	struct efa_admin_ctrl_buff_info control_buffer;
+
+	struct efa_admin_get_set_feature_common_desc feature_common;
+
+	union {
+		u32 raw[11];
+
+		/* AENQ configuration */
+		struct efa_admin_feature_aenq_desc aenq;
+	} u;
+};
+
+struct efa_admin_set_feature_resp {
+	struct efa_admin_acq_common_desc acq_common_desc;
+
+	union {
+		u32 raw[14];
+	} u;
+};
+
+struct efa_admin_alloc_pd_cmd {
+	struct efa_admin_aq_common_desc aq_common_descriptor;
+};
+
+struct efa_admin_alloc_pd_resp {
+	struct efa_admin_acq_common_desc acq_common_desc;
+
+	/* PD number */
+	u16 pd;
+
+	/* MBZ */
+	u16 reserved;
+};
+
+struct efa_admin_dealloc_pd_cmd {
+	struct efa_admin_aq_common_desc aq_common_descriptor;
+
+	/* PD number */
+	u16 pd;
+
+	/* MBZ */
+	u16 reserved;
+};
+
+struct efa_admin_dealloc_pd_resp {
+	struct efa_admin_acq_common_desc acq_common_desc;
+};
+
+struct efa_admin_alloc_uar_cmd {
+	struct efa_admin_aq_common_desc aq_common_descriptor;
+};
+
+struct efa_admin_alloc_uar_resp {
+	struct efa_admin_acq_common_desc acq_common_desc;
+
+	/* UAR number */
+	u16 uar;
+
+	/* MBZ */
+	u16 reserved;
+};
+
+struct efa_admin_dealloc_uar_cmd {
+	struct efa_admin_aq_common_desc aq_common_descriptor;
+
+	/* UAR number */
+	u16 uar;
+
+	/* MBZ */
+	u16 reserved;
+};
+
+struct efa_admin_dealloc_uar_resp {
+	struct efa_admin_acq_common_desc acq_common_desc;
+};
+
+/* asynchronous event notification groups */
+enum efa_admin_aenq_group {
+	EFA_ADMIN_FATAL_ERROR                       = 1,
+	EFA_ADMIN_WARNING                           = 2,
+	EFA_ADMIN_NOTIFICATION                      = 3,
+	EFA_ADMIN_KEEP_ALIVE                        = 4,
+	EFA_ADMIN_AENQ_GROUPS_NUM                   = 5,
+};
+
+enum efa_admin_aenq_notification_syndrom {
+	EFA_ADMIN_SUSPEND                           = 0,
+	EFA_ADMIN_RESUME                            = 1,
+	EFA_ADMIN_UPDATE_HINTS                      = 2,
+};
+
+struct efa_admin_mmio_req_read_less_resp {
+	u16 req_id;
+
+	u16 reg_off;
+
+	/* value is valid when poll is cleared */
+	u32 reg_val;
+};
+
+/* create_qp_cmd */
+#define EFA_ADMIN_CREATE_QP_CMD_SQ_VIRT_MASK                BIT(0)
+#define EFA_ADMIN_CREATE_QP_CMD_RQ_VIRT_SHIFT               1
+#define EFA_ADMIN_CREATE_QP_CMD_RQ_VIRT_MASK                BIT(1)
+
+/* reg_mr_cmd */
+#define EFA_ADMIN_REG_MR_CMD_PHYS_PAGE_SIZE_SHIFT_MASK      GENMASK(4, 0)
+#define EFA_ADMIN_REG_MR_CMD_MEM_ADDR_PHY_MODE_EN_SHIFT     7
+#define EFA_ADMIN_REG_MR_CMD_MEM_ADDR_PHY_MODE_EN_MASK      BIT(7)
+#define EFA_ADMIN_REG_MR_CMD_LOCAL_WRITE_ENABLE_MASK        BIT(0)
+
+/* create_cq_cmd */
+#define EFA_ADMIN_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_SHIFT 5
+#define EFA_ADMIN_CREATE_CQ_CMD_INTERRUPT_MODE_ENABLED_MASK BIT(5)
+#define EFA_ADMIN_CREATE_CQ_CMD_VIRT_SHIFT                  6
+#define EFA_ADMIN_CREATE_CQ_CMD_VIRT_MASK                   BIT(6)
+#define EFA_ADMIN_CREATE_CQ_CMD_CQ_ENTRY_SIZE_WORDS_MASK    GENMASK(4, 0)
+
+/* get_set_feature_common_desc */
+#define EFA_ADMIN_GET_SET_FEATURE_COMMON_DESC_SELECT_MASK   GENMASK(1, 0)
+
+#endif /* _EFA_ADMIN_CMDS_H_ */
diff --git a/drivers/infiniband/hw/efa/efa_admin_defs.h b/drivers/infiniband/hw/efa/efa_admin_defs.h
new file mode 100644
index 000000000000..c8e0c8b905be
--- /dev/null
+++ b/drivers/infiniband/hw/efa/efa_admin_defs.h
@@ -0,0 +1,136 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
+/*
+ * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
+ */
+
+#ifndef _EFA_ADMIN_H_
+#define _EFA_ADMIN_H_
+
+enum efa_admin_aq_completion_status {
+	EFA_ADMIN_SUCCESS                           = 0,
+	EFA_ADMIN_RESOURCE_ALLOCATION_FAILURE       = 1,
+	EFA_ADMIN_BAD_OPCODE                        = 2,
+	EFA_ADMIN_UNSUPPORTED_OPCODE                = 3,
+	EFA_ADMIN_MALFORMED_REQUEST                 = 4,
+	/* Additional status is provided in ACQ entry extended_status */
+	EFA_ADMIN_ILLEGAL_PARAMETER                 = 5,
+	EFA_ADMIN_UNKNOWN_ERROR                     = 6,
+	EFA_ADMIN_RESOURCE_BUSY                     = 7,
+};
+
+struct efa_admin_aq_common_desc {
+	/*
+	 * 11:0 : command_id
+	 * 15:12 : reserved12
+	 */
+	u16 command_id;
+
+	/* as appears in efa_admin_aq_opcode */
+	u8 opcode;
+
+	/*
+	 * 0 : phase
+	 * 1 : ctrl_data - control buffer address valid
+	 * 2 : ctrl_data_indirect - control buffer address
+	 *    points to list of pages with addresses of control
+	 *    buffers
+	 * 7:3 : reserved3
+	 */
+	u8 flags;
+};
+
+/*
+ * used in efa_admin_aq_entry. Can point directly to control data, or to a
+ * page list chunk. Used also at the end of indirect mode page list chunks,
+ * for chaining.
+ */
+struct efa_admin_ctrl_buff_info {
+	u32 length;
+
+	struct efa_common_mem_addr address;
+};
+
+struct efa_admin_aq_entry {
+	struct efa_admin_aq_common_desc aq_common_descriptor;
+
+	union {
+		u32 inline_data_w1[3];
+
+		struct efa_admin_ctrl_buff_info control_buffer;
+	} u;
+
+	u32 inline_data_w4[12];
+};
+
+struct efa_admin_acq_common_desc {
+	/*
+	 * command identifier to associate it with the aq descriptor
+	 * 11:0 : command_id
+	 * 15:12 : reserved12
+	 */
+	u16 command;
+
+	u8 status;
+
+	/*
+	 * 0 : phase
+	 * 7:1 : reserved1
+	 */
+	u8 flags;
+
+	u16 extended_status;
+
+	/*
+	 * indicates to the driver which AQ entry has been consumed by the
+	 *    device and could be reused
+	 */
+	u16 sq_head_indx;
+};
+
+struct efa_admin_acq_entry {
+	struct efa_admin_acq_common_desc acq_common_descriptor;
+
+	u32 response_specific_data[14];
+};
+
+struct efa_admin_aenq_common_desc {
+	u16 group;
+
+	u16 syndrom;
+
+	/*
+	 * 0 : phase
+	 * 7:1 : reserved - MBZ
+	 */
+	u8 flags;
+
+	u8 reserved1[3];
+
+	u32 timestamp_low;
+
+	u32 timestamp_high;
+};
+
+struct efa_admin_aenq_entry {
+	struct efa_admin_aenq_common_desc aenq_common_desc;
+
+	/* command specific inline data */
+	u32 inline_data_w4[12];
+};
+
+/* aq_common_desc */
+#define EFA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK            GENMASK(11, 0)
+#define EFA_ADMIN_AQ_COMMON_DESC_PHASE_MASK                 BIT(0)
+#define EFA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_SHIFT            1
+#define EFA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_MASK             BIT(1)
+#define EFA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_SHIFT   2
+#define EFA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK    BIT(2)
+
+/* acq_common_desc */
+#define EFA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID_MASK           GENMASK(11, 0)
+#define EFA_ADMIN_ACQ_COMMON_DESC_PHASE_MASK                BIT(0)
+
+/* aenq_common_desc */
+#define EFA_ADMIN_AENQ_COMMON_DESC_PHASE_MASK               BIT(0)
+
+#endif /* _EFA_ADMIN_H_ */
diff --git a/drivers/infiniband/hw/efa/efa_com.c b/drivers/infiniband/hw/efa/efa_com.c
new file mode 100644
index 000000000000..a5c788741a04
--- /dev/null
+++ b/drivers/infiniband/hw/efa/efa_com.c
@@ -0,0 +1,1160 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
+/*
+ * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
+ */
+
+#include "efa_com.h"
+#include "efa_regs_defs.h"
+
+#define ADMIN_CMD_TIMEOUT_US 30000000 /* usecs */
+
+#define EFA_REG_READ_TIMEOUT_US 50000 /* usecs */
+#define EFA_MMIO_READ_INVALID 0xffffffff
+
+#define EFA_POLL_INTERVAL_MS 100 /* msecs */
+
+#define EFA_ASYNC_QUEUE_DEPTH 16
+#define EFA_ADMIN_QUEUE_DEPTH 32
+
+#define MIN_EFA_VER\
+	((EFA_ADMIN_API_VERSION_MAJOR << EFA_REGS_VERSION_MAJOR_VERSION_SHIFT) | \
+	 (EFA_ADMIN_API_VERSION_MINOR & EFA_REGS_VERSION_MINOR_VERSION_MASK))
+
+#define EFA_CTRL_MAJOR          0
+#define EFA_CTRL_MINOR          0
+#define EFA_CTRL_SUB_MINOR      1
+
+#define MIN_EFA_CTRL_VER \
+	(((EFA_CTRL_MAJOR) << \
+	(EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_SHIFT)) | \
+	((EFA_CTRL_MINOR) << \
+	(EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION_SHIFT)) | \
+	(EFA_CTRL_SUB_MINOR))
+
+#define EFA_DMA_ADDR_TO_UINT32_LOW(x)   ((u32)((u64)(x)))
+#define EFA_DMA_ADDR_TO_UINT32_HIGH(x)  ((u32)(((u64)(x)) >> 32))
+
+#define EFA_REGS_ADMIN_INTR_MASK 1
+
+enum efa_cmd_status {
+	EFA_CMD_SUBMITTED,
+	EFA_CMD_COMPLETED,
+	/* Abort - canceled by the driver */
+	EFA_CMD_ABORTED,
+};
+
+struct efa_comp_ctx {
+	struct completion wait_event;
+	struct efa_admin_acq_entry *user_cqe;
+	u32 comp_size;
+	enum efa_cmd_status status;
+	/* status from the device */
+	u8 comp_status;
+	u8 cmd_opcode;
+	u8 occupied;
+};
+
+static const char *efa_com_cmd_str(u8 cmd)
+{
+#define EFA_CMD_STR_CASE(_cmd) case EFA_ADMIN_##_cmd: return #_cmd
+
+	switch (cmd) {
+	EFA_CMD_STR_CASE(CREATE_QP);
+	EFA_CMD_STR_CASE(MODIFY_QP);
+	EFA_CMD_STR_CASE(QUERY_QP);
+	EFA_CMD_STR_CASE(DESTROY_QP);
+	EFA_CMD_STR_CASE(CREATE_AH);
+	EFA_CMD_STR_CASE(DESTROY_AH);
+	EFA_CMD_STR_CASE(REG_MR);
+	EFA_CMD_STR_CASE(DEREG_MR);
+	EFA_CMD_STR_CASE(CREATE_CQ);
+	EFA_CMD_STR_CASE(DESTROY_CQ);
+	EFA_CMD_STR_CASE(GET_FEATURE);
+	EFA_CMD_STR_CASE(SET_FEATURE);
+	EFA_CMD_STR_CASE(GET_STATS);
+	EFA_CMD_STR_CASE(ALLOC_PD);
+	EFA_CMD_STR_CASE(DEALLOC_PD);
+	EFA_CMD_STR_CASE(ALLOC_UAR);
+	EFA_CMD_STR_CASE(DEALLOC_UAR);
+	default: return "unknown command opcode";
+	}
+#undef EFA_CMD_STR_CASE
+}
+
+static u32 efa_com_reg_read32(struct efa_com_dev *edev, u16 offset)
+{
+	struct efa_com_mmio_read *mmio_read = &edev->mmio_read;
+	struct efa_admin_mmio_req_read_less_resp *read_resp;
+	unsigned long exp_time;
+	u32 mmio_read_reg;
+	u32 err;
+
+	read_resp = mmio_read->read_resp;
+
+	spin_lock(&mmio_read->lock);
+	mmio_read->seq_num++;
+
+	/* trash DMA req_id to identify when hardware is done */
+	read_resp->req_id = mmio_read->seq_num + 0x9aL;
+	mmio_read_reg = (offset << EFA_REGS_MMIO_REG_READ_REG_OFF_SHIFT) &
+			EFA_REGS_MMIO_REG_READ_REG_OFF_MASK;
+	mmio_read_reg |= mmio_read->seq_num &
+			 EFA_REGS_MMIO_REG_READ_REQ_ID_MASK;
+
+	writel(mmio_read_reg, edev->reg_bar + EFA_REGS_MMIO_REG_READ_OFF);
+
+	exp_time = jiffies + usecs_to_jiffies(mmio_read->mmio_read_timeout);
+	do {
+		if (READ_ONCE(read_resp->req_id) == mmio_read->seq_num)
+			break;
+		udelay(1);
+	} while (time_is_after_jiffies(exp_time));
+
+	if (read_resp->req_id != mmio_read->seq_num) {
+		ibdev_err(edev->efa_dev,
+			  "Reading register timed out. expected: req id[%u] offset[%#x] actual: req id[%u] offset[%#x]\n",
+			  mmio_read->seq_num, offset, read_resp->req_id,
+			  read_resp->reg_off);
+		err = EFA_MMIO_READ_INVALID;
+		goto out;
+	}
+
+	if (read_resp->reg_off != offset) {
+		ibdev_err(edev->efa_dev,
+			  "Reading register failed: wrong offset provided\n");
+		err = EFA_MMIO_READ_INVALID;
+		goto out;
+	}
+
+	err = read_resp->reg_val;
+out:
+	spin_unlock(&mmio_read->lock);
+	return err;
+}
+
+static int efa_com_admin_init_sq(struct efa_com_dev *edev)
+{
+	struct efa_com_admin_queue *aq = &edev->aq;
+	struct efa_com_admin_sq *sq = &aq->sq;
+	u16 size = aq->depth * sizeof(*sq->entries);
+	u32 addr_high;
+	u32 addr_low;
+	u32 aq_caps;
+
+	sq->entries =
+		dma_alloc_coherent(aq->dmadev, size, &sq->dma_addr, GFP_KERNEL);
+	if (!sq->entries)
+		return -ENOMEM;
+
+	spin_lock_init(&sq->lock);
+
+	sq->cc = 0;
+	sq->pc = 0;
+	sq->phase = 1;
+
+	sq->db_addr = (u32 __iomem *)(edev->reg_bar + EFA_REGS_AQ_PROD_DB_OFF);
+
+	addr_high = EFA_DMA_ADDR_TO_UINT32_HIGH(sq->dma_addr);
+	addr_low = EFA_DMA_ADDR_TO_UINT32_LOW(sq->dma_addr);
+
+	writel(addr_low, edev->reg_bar + EFA_REGS_AQ_BASE_LO_OFF);
+	writel(addr_high, edev->reg_bar + EFA_REGS_AQ_BASE_HI_OFF);
+
+	aq_caps = aq->depth & EFA_REGS_AQ_CAPS_AQ_DEPTH_MASK;
+	aq_caps |= (sizeof(struct efa_admin_aq_entry) <<
+			EFA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_SHIFT) &
+			EFA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_MASK;
+
+	writel(aq_caps, edev->reg_bar + EFA_REGS_AQ_CAPS_OFF);
+
+	return 0;
+}
+
+static int efa_com_admin_init_cq(struct efa_com_dev *edev)
+{
+	struct efa_com_admin_queue *aq = &edev->aq;
+	struct efa_com_admin_cq *cq = &aq->cq;
+	u16 size = aq->depth * sizeof(*cq->entries);
+	u32 addr_high;
+	u32 addr_low;
+	u32 acq_caps;
+
+	cq->entries =
+		dma_alloc_coherent(aq->dmadev, size, &cq->dma_addr, GFP_KERNEL);
+	if (!cq->entries)
+		return -ENOMEM;
+
+	spin_lock_init(&cq->lock);
+
+	cq->cc = 0;
+	cq->phase = 1;
+
+	addr_high = EFA_DMA_ADDR_TO_UINT32_HIGH(cq->dma_addr);
+	addr_low = EFA_DMA_ADDR_TO_UINT32_LOW(cq->dma_addr);
+
+	writel(addr_low, edev->reg_bar + EFA_REGS_ACQ_BASE_LO_OFF);
+	writel(addr_high, edev->reg_bar + EFA_REGS_ACQ_BASE_HI_OFF);
+
+	acq_caps = aq->depth & EFA_REGS_ACQ_CAPS_ACQ_DEPTH_MASK;
+	acq_caps |= (sizeof(struct efa_admin_acq_entry) <<
+			EFA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_SHIFT) &
+			EFA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_MASK;
+	acq_caps |= (aq->msix_vector_idx <<
+			EFA_REGS_ACQ_CAPS_ACQ_MSIX_VECTOR_SHIFT) &
+			EFA_REGS_ACQ_CAPS_ACQ_MSIX_VECTOR_MASK;
+
+	writel(acq_caps, edev->reg_bar + EFA_REGS_ACQ_CAPS_OFF);
+
+	return 0;
+}
+
+static int efa_com_admin_init_aenq(struct efa_com_dev *edev,
+				   struct efa_aenq_handlers *aenq_handlers)
+{
+	struct efa_com_aenq *aenq = &edev->aenq;
+	u32 addr_low, addr_high, aenq_caps;
+	u16 size;
+
+	if (!aenq_handlers) {
+		ibdev_err(edev->efa_dev, "aenq handlers pointer is NULL\n");
+		return -EINVAL;
+	}
+
+	size = EFA_ASYNC_QUEUE_DEPTH * sizeof(*aenq->entries);
+	aenq->entries = dma_alloc_coherent(edev->dmadev, size, &aenq->dma_addr,
+					   GFP_KERNEL);
+	if (!aenq->entries)
+		return -ENOMEM;
+
+	aenq->aenq_handlers = aenq_handlers;
+	aenq->depth = EFA_ASYNC_QUEUE_DEPTH;
+	aenq->cc = 0;
+	aenq->phase = 1;
+
+	addr_low = EFA_DMA_ADDR_TO_UINT32_LOW(aenq->dma_addr);
+	addr_high = EFA_DMA_ADDR_TO_UINT32_HIGH(aenq->dma_addr);
+
+	writel(addr_low, edev->reg_bar + EFA_REGS_AENQ_BASE_LO_OFF);
+	writel(addr_high, edev->reg_bar + EFA_REGS_AENQ_BASE_HI_OFF);
+
+	aenq_caps = aenq->depth & EFA_REGS_AENQ_CAPS_AENQ_DEPTH_MASK;
+	aenq_caps |= (sizeof(struct efa_admin_aenq_entry) <<
+		EFA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_SHIFT) &
+		EFA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_MASK;
+	aenq_caps |= (aenq->msix_vector_idx
+		      << EFA_REGS_AENQ_CAPS_AENQ_MSIX_VECTOR_SHIFT) &
+		     EFA_REGS_AENQ_CAPS_AENQ_MSIX_VECTOR_MASK;
+	writel(aenq_caps, edev->reg_bar + EFA_REGS_AENQ_CAPS_OFF);
+
+	/*
+	 * Init cons_db to mark that all entries in the queue
+	 * are initially available
+	 */
+	writel(edev->aenq.cc, edev->reg_bar + EFA_REGS_AENQ_CONS_DB_OFF);
+
+	return 0;
+}
+
+/* ID to be used with efa_com_get_comp_ctx */
+static u16 efa_com_alloc_ctx_id(struct efa_com_admin_queue *aq)
+{
+	u16 ctx_id;
+
+	spin_lock(&aq->comp_ctx_lock);
+	ctx_id = aq->comp_ctx_pool[aq->comp_ctx_pool_next];
+	aq->comp_ctx_pool_next++;
+	spin_unlock(&aq->comp_ctx_lock);
+
+	return ctx_id;
+}
+
+static void efa_com_dealloc_ctx_id(struct efa_com_admin_queue *aq,
+				   u16 ctx_id)
+{
+	spin_lock(&aq->comp_ctx_lock);
+	aq->comp_ctx_pool_next--;
+	aq->comp_ctx_pool[aq->comp_ctx_pool_next] = ctx_id;
+	spin_unlock(&aq->comp_ctx_lock);
+}
+
+static inline void efa_com_put_comp_ctx(struct efa_com_admin_queue *aq,
+					struct efa_comp_ctx *comp_ctx)
+{
+	u16 comp_id = comp_ctx->user_cqe->acq_common_descriptor.command &
+		      EFA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID_MASK;
+
+	ibdev_dbg(aq->efa_dev, "Putting completion command_id %d\n", comp_id);
+	comp_ctx->occupied = 0;
+	efa_com_dealloc_ctx_id(aq, comp_id);
+}
+
+static struct efa_comp_ctx *efa_com_get_comp_ctx(struct efa_com_admin_queue *aq,
+						 u16 command_id, bool capture)
+{
+	if (command_id >= aq->depth) {
+		ibdev_err(aq->efa_dev,
+			  "command id is larger than the queue size. cmd_id: %u queue size %d\n",
+			  command_id, aq->depth);
+		return NULL;
+	}
+
+	if (aq->comp_ctx[command_id].occupied && capture) {
+		ibdev_err(aq->efa_dev, "Completion context is occupied\n");
+		return NULL;
+	}
+
+	if (capture) {
+		aq->comp_ctx[command_id].occupied = 1;
+		ibdev_dbg(aq->efa_dev, "Taking completion ctxt command_id %d\n",
+			  command_id);
+	}
+
+	return &aq->comp_ctx[command_id];
+}
+
+static struct efa_comp_ctx *__efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq,
+						       struct efa_admin_aq_entry *cmd,
+						       size_t cmd_size_in_bytes,
+						       struct efa_admin_acq_entry *comp,
+						       size_t comp_size_in_bytes)
+{
+	struct efa_comp_ctx *comp_ctx;
+	u16 queue_size_mask;
+	u16 ctx_id;
+	u16 pi;
+
+	queue_size_mask = aq->depth - 1;
+	pi = aq->sq.pc & queue_size_mask;
+
+	ctx_id = efa_com_alloc_ctx_id(aq);
+
+	cmd->aq_common_descriptor.flags |= aq->sq.phase &
+		EFA_ADMIN_AQ_COMMON_DESC_PHASE_MASK;
+
+	cmd->aq_common_descriptor.command_id |= ctx_id &
+		EFA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK;
+
+	comp_ctx = efa_com_get_comp_ctx(aq, ctx_id, true);
+	if (!comp_ctx) {
+		efa_com_dealloc_ctx_id(aq, ctx_id);
+		return ERR_PTR(-EINVAL);
+	}
+
+	comp_ctx->status = EFA_CMD_SUBMITTED;
+	comp_ctx->comp_size = comp_size_in_bytes;
+	comp_ctx->user_cqe = comp;
+	comp_ctx->cmd_opcode = cmd->aq_common_descriptor.opcode;
+
+	reinit_completion(&comp_ctx->wait_event);
+
+	memcpy(&aq->sq.entries[pi], cmd, cmd_size_in_bytes);
+
+	aq->sq.pc++;
+	atomic64_inc(&aq->stats.submitted_cmd);
+
+	if ((aq->sq.pc & queue_size_mask) == 0)
+		aq->sq.phase = !aq->sq.phase;
+
+	/* barrier not needed in case of writel */
+	writel(aq->sq.pc, aq->sq.db_addr);
+
+	return comp_ctx;
+}
+
+static inline int efa_com_init_comp_ctxt(struct efa_com_admin_queue *aq)
+{
+	size_t pool_size = aq->depth * sizeof(*aq->comp_ctx_pool);
+	size_t size = aq->depth * sizeof(struct efa_comp_ctx);
+	struct efa_comp_ctx *comp_ctx;
+	u16 i;
+
+	aq->comp_ctx = devm_kzalloc(aq->dmadev, size, GFP_KERNEL);
+	aq->comp_ctx_pool = devm_kzalloc(aq->dmadev, pool_size, GFP_KERNEL);
+	if (!aq->comp_ctx || !aq->comp_ctx_pool) {
+		devm_kfree(aq->dmadev, aq->comp_ctx_pool);
+		devm_kfree(aq->dmadev, aq->comp_ctx);
+		return -ENOMEM;
+	}
+
+	for (i = 0; i < aq->depth; i++) {
+		comp_ctx = efa_com_get_comp_ctx(aq, i, false);
+		if (comp_ctx)
+			init_completion(&comp_ctx->wait_event);
+
+		aq->comp_ctx_pool[i] = i;
+	}
+
+	spin_lock_init(&aq->comp_ctx_lock);
+
+	aq->comp_ctx_pool_next = 0;
+
+	return 0;
+}
+
+static struct efa_comp_ctx *efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq,
+						     struct efa_admin_aq_entry *cmd,
+						     size_t cmd_size_in_bytes,
+						     struct efa_admin_acq_entry *comp,
+						     size_t comp_size_in_bytes)
+{
+	struct efa_comp_ctx *comp_ctx;
+
+	spin_lock(&aq->sq.lock);
+	if (!test_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state)) {
+		ibdev_err(aq->efa_dev, "Admin queue is closed\n");
+		spin_unlock(&aq->sq.lock);
+		return ERR_PTR(-ENODEV);
+	}
+
+	comp_ctx = __efa_com_submit_admin_cmd(aq, cmd, cmd_size_in_bytes, comp,
+					      comp_size_in_bytes);
+	spin_unlock(&aq->sq.lock);
+	if (IS_ERR(comp_ctx))
+		clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state);
+
+	return comp_ctx;
+}
+
+static void efa_com_handle_single_admin_completion(struct efa_com_admin_queue *aq,
+						   struct efa_admin_acq_entry *cqe)
+{
+	struct efa_comp_ctx *comp_ctx;
+	u16 cmd_id;
+
+	cmd_id = cqe->acq_common_descriptor.command &
+		 EFA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID_MASK;
+
+	comp_ctx = efa_com_get_comp_ctx(aq, cmd_id, false);
+	if (!comp_ctx) {
+		ibdev_err(aq->efa_dev,
+			  "comp_ctx is NULL. Changing the admin queue running state\n");
+		clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state);
+		return;
+	}
+
+	comp_ctx->status = EFA_CMD_COMPLETED;
+	comp_ctx->comp_status = cqe->acq_common_descriptor.status;
+	if (comp_ctx->user_cqe)
+		memcpy(comp_ctx->user_cqe, cqe, comp_ctx->comp_size);
+
+	if (!test_bit(EFA_AQ_STATE_POLLING_BIT, &aq->state))
+		complete(&comp_ctx->wait_event);
+}
+
+static void efa_com_handle_admin_completion(struct efa_com_admin_queue *aq)
+{
+	struct efa_admin_acq_entry *cqe;
+	u16 queue_size_mask;
+	u16 comp_num = 0;
+	u8 phase;
+	u16 ci;
+
+	queue_size_mask = aq->depth - 1;
+
+	ci = aq->cq.cc & queue_size_mask;
+	phase = aq->cq.phase;
+
+	cqe = &aq->cq.entries[ci];
+
+	/* Go over all the completions */
+	while ((READ_ONCE(cqe->acq_common_descriptor.flags) &
+		EFA_ADMIN_ACQ_COMMON_DESC_PHASE_MASK) == phase) {
+		/*
+		 * Do not read the rest of the completion entry before the
+		 * phase bit was validated
+		 */
+		dma_rmb();
+		efa_com_handle_single_admin_completion(aq, cqe);
+
+		ci++;
+		comp_num++;
+		if (ci == aq->depth) {
+			ci = 0;
+			phase = !phase;
+		}
+
+		cqe = &aq->cq.entries[ci];
+	}
+
+	aq->cq.cc += comp_num;
+	aq->cq.phase = phase;
+	aq->sq.cc += comp_num;
+	atomic64_add(comp_num, &aq->stats.completed_cmd);
+}
+
+static int efa_com_comp_status_to_errno(u8 comp_status)
+{
+	switch (comp_status) {
+	case EFA_ADMIN_SUCCESS:
+		return 0;
+	case EFA_ADMIN_RESOURCE_ALLOCATION_FAILURE:
+		return -ENOMEM;
+	case EFA_ADMIN_UNSUPPORTED_OPCODE:
+		return -EOPNOTSUPP;
+	case EFA_ADMIN_BAD_OPCODE:
+	case EFA_ADMIN_MALFORMED_REQUEST:
+	case EFA_ADMIN_ILLEGAL_PARAMETER:
+	case EFA_ADMIN_UNKNOWN_ERROR:
+		return -EINVAL;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int efa_com_wait_and_process_admin_cq_polling(struct efa_comp_ctx *comp_ctx,
+						     struct efa_com_admin_queue *aq)
+{
+	unsigned long timeout;
+	unsigned long flags;
+	int err;
+
+	timeout = jiffies + usecs_to_jiffies(aq->completion_timeout);
+
+	while (1) {
+		spin_lock_irqsave(&aq->cq.lock, flags);
+		efa_com_handle_admin_completion(aq);
+		spin_unlock_irqrestore(&aq->cq.lock, flags);
+
+		if (comp_ctx->status != EFA_CMD_SUBMITTED)
+			break;
+
+		if (time_is_before_jiffies(timeout)) {
+			ibdev_err(aq->efa_dev,
+				  "Wait for completion (polling) timeout\n");
+			/* EFA didn't have any completion */
+			atomic64_inc(&aq->stats.no_completion);
+
+			clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state);
+			err = -ETIME;
+			goto out;
+		}
+
+		msleep(aq->poll_interval);
+	}
+
+	if (comp_ctx->status == EFA_CMD_ABORTED) {
+		ibdev_err(aq->efa_dev, "Command was aborted\n");
+		atomic64_inc(&aq->stats.aborted_cmd);
+		err = -ENODEV;
+		goto out;
+	}
+
+	WARN_ONCE(comp_ctx->status != EFA_CMD_COMPLETED,
+		  "Invalid completion status %d\n", comp_ctx->status);
+
+	err = efa_com_comp_status_to_errno(comp_ctx->comp_status);
+out:
+	efa_com_put_comp_ctx(aq, comp_ctx);
+	return err;
+}
+
+static int efa_com_wait_and_process_admin_cq_interrupts(struct efa_comp_ctx *comp_ctx,
+							struct efa_com_admin_queue *aq)
+{
+	unsigned long flags;
+	int err;
+
+	wait_for_completion_timeout(&comp_ctx->wait_event,
+				    usecs_to_jiffies(aq->completion_timeout));
+
+	/*
+	 * In case the command wasn't completed find out the root cause.
+	 * There might be 2 kinds of errors
+	 * 1) No completion (timeout reached)
+	 * 2) There is completion but the device didn't get any msi-x interrupt.
+	 */
+	if (comp_ctx->status == EFA_CMD_SUBMITTED) {
+		spin_lock_irqsave(&aq->cq.lock, flags);
+		efa_com_handle_admin_completion(aq);
+		spin_unlock_irqrestore(&aq->cq.lock, flags);
+
+		atomic64_inc(&aq->stats.no_completion);
+
+		if (comp_ctx->status == EFA_CMD_COMPLETED)
+			ibdev_err(aq->efa_dev,
+				  "The device sent a completion but the driver didn't receive any MSI-X interrupt for admin cmd %s(%d) status %d (ctx: 0x%p, sq producer: %d, sq consumer: %d, cq consumer: %d)\n",
+				  efa_com_cmd_str(comp_ctx->cmd_opcode),
+				  comp_ctx->cmd_opcode, comp_ctx->status,
+				  comp_ctx, aq->sq.pc, aq->sq.cc, aq->cq.cc);
+		else
+			ibdev_err(aq->efa_dev,
+				  "The device didn't send any completion for admin cmd %s(%d) status %d (ctx 0x%p, sq producer: %d, sq consumer: %d, cq consumer: %d)\n",
+				  efa_com_cmd_str(comp_ctx->cmd_opcode),
+				  comp_ctx->cmd_opcode, comp_ctx->status,
+				  comp_ctx, aq->sq.pc, aq->sq.cc, aq->cq.cc);
+
+		clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state);
+		err = -ETIME;
+		goto out;
+	}
+
+	err = efa_com_comp_status_to_errno(comp_ctx->comp_status);
+out:
+	efa_com_put_comp_ctx(aq, comp_ctx);
+	return err;
+}
+
+/*
+ * There are two types to wait for completion.
+ * Polling mode - wait until the completion is available.
+ * Async mode - wait on wait queue until the completion is ready
+ * (or the timeout expired).
+ * It is expected that the IRQ called efa_com_handle_admin_completion
+ * to mark the completions.
+ */
+static int efa_com_wait_and_process_admin_cq(struct efa_comp_ctx *comp_ctx,
+					     struct efa_com_admin_queue *aq)
+{
+	if (test_bit(EFA_AQ_STATE_POLLING_BIT, &aq->state))
+		return efa_com_wait_and_process_admin_cq_polling(comp_ctx, aq);
+
+	return efa_com_wait_and_process_admin_cq_interrupts(comp_ctx, aq);
+}
+
+/**
+ * efa_com_cmd_exec - Execute admin command
+ * @aq: admin queue.
+ * @cmd: the admin command to execute.
+ * @cmd_size: the command size.
+ * @comp: command completion return entry.
+ * @comp_size: command completion size.
+ * Submit an admin command and then wait until the device will return a
+ * completion.
+ * The completion will be copied into comp.
+ *
+ * @return - 0 on success, negative value on failure.
+ */
+int efa_com_cmd_exec(struct efa_com_admin_queue *aq,
+		     struct efa_admin_aq_entry *cmd,
+		     size_t cmd_size,
+		     struct efa_admin_acq_entry *comp,
+		     size_t comp_size)
+{
+	struct efa_comp_ctx *comp_ctx;
+	int err;
+
+	might_sleep();
+
+	/* In case of queue FULL */
+	down(&aq->avail_cmds);
+
+	ibdev_dbg(aq->efa_dev, "%s (opcode %d)\n",
+		  efa_com_cmd_str(cmd->aq_common_descriptor.opcode),
+		  cmd->aq_common_descriptor.opcode);
+	comp_ctx = efa_com_submit_admin_cmd(aq, cmd, cmd_size, comp, comp_size);
+	if (IS_ERR(comp_ctx)) {
+		ibdev_err(aq->efa_dev,
+			  "Failed to submit command %s (opcode %u) err %ld\n",
+			  efa_com_cmd_str(cmd->aq_common_descriptor.opcode),
+			  cmd->aq_common_descriptor.opcode, PTR_ERR(comp_ctx));
+
+		up(&aq->avail_cmds);
+		return PTR_ERR(comp_ctx);
+	}
+
+	err = efa_com_wait_and_process_admin_cq(comp_ctx, aq);
+	if (err)
+		ibdev_err(aq->efa_dev,
+			  "Failed to process command %s (opcode %u) comp_status %d err %d\n",
+			  efa_com_cmd_str(cmd->aq_common_descriptor.opcode),
+			  cmd->aq_common_descriptor.opcode,
+			  comp_ctx->comp_status, err);
+
+	up(&aq->avail_cmds);
+
+	return err;
+}
+
+/**
+ * efa_com_abort_admin_commands - Abort all the outstanding admin commands.
+ * @edev: EFA communication layer struct
+ *
+ * This method aborts all the outstanding admin commands.
+ * The caller should then call efa_com_wait_for_abort_completion to make sure
+ * all the commands were completed.
+ */
+static void efa_com_abort_admin_commands(struct efa_com_dev *edev)
+{
+	struct efa_com_admin_queue *aq = &edev->aq;
+	struct efa_comp_ctx *comp_ctx;
+	unsigned long flags;
+	u16 i;
+
+	spin_lock(&aq->sq.lock);
+	spin_lock_irqsave(&aq->cq.lock, flags);
+	for (i = 0; i < aq->depth; i++) {
+		comp_ctx = efa_com_get_comp_ctx(aq, i, false);
+		if (!comp_ctx)
+			break;
+
+		comp_ctx->status = EFA_CMD_ABORTED;
+
+		complete(&comp_ctx->wait_event);
+	}
+	spin_unlock_irqrestore(&aq->cq.lock, flags);
+	spin_unlock(&aq->sq.lock);
+}
+
+/**
+ * efa_com_wait_for_abort_completion - Wait for admin commands abort.
+ * @edev: EFA communication layer struct
+ *
+ * This method wait until all the outstanding admin commands will be completed.
+ */
+static void efa_com_wait_for_abort_completion(struct efa_com_dev *edev)
+{
+	struct efa_com_admin_queue *aq = &edev->aq;
+	int i;
+
+	/* all mine */
+	for (i = 0; i < aq->depth; i++)
+		down(&aq->avail_cmds);
+
+	/* let it go */
+	for (i = 0; i < aq->depth; i++)
+		up(&aq->avail_cmds);
+}
+
+static void efa_com_admin_flush(struct efa_com_dev *edev)
+{
+	struct efa_com_admin_queue *aq = &edev->aq;
+
+	clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state);
+
+	efa_com_abort_admin_commands(edev);
+	efa_com_wait_for_abort_completion(edev);
+}
+
+/**
+ * efa_com_admin_destroy - Destroy the admin and the async events queues.
+ * @edev: EFA communication layer struct
+ */
+void efa_com_admin_destroy(struct efa_com_dev *edev)
+{
+	struct efa_com_admin_queue *aq = &edev->aq;
+	struct efa_com_aenq *aenq = &edev->aenq;
+	struct efa_com_admin_cq *cq = &aq->cq;
+	struct efa_com_admin_sq *sq = &aq->sq;
+	u16 size;
+
+	efa_com_admin_flush(edev);
+
+	devm_kfree(edev->dmadev, aq->comp_ctx_pool);
+	devm_kfree(edev->dmadev, aq->comp_ctx);
+
+	size = aq->depth * sizeof(*sq->entries);
+	dma_free_coherent(edev->dmadev, size, sq->entries, sq->dma_addr);
+
+	size = aq->depth * sizeof(*cq->entries);
+	dma_free_coherent(edev->dmadev, size, cq->entries, cq->dma_addr);
+
+	size = aenq->depth * sizeof(*aenq->entries);
+	dma_free_coherent(edev->dmadev, size, aenq->entries, aenq->dma_addr);
+}
+
+/**
+ * efa_com_set_admin_polling_mode - Set the admin completion queue polling mode
+ * @edev: EFA communication layer struct
+ * @polling: Enable/Disable polling mode
+ *
+ * Set the admin completion mode.
+ */
+void efa_com_set_admin_polling_mode(struct efa_com_dev *edev, bool polling)
+{
+	u32 mask_value = 0;
+
+	if (polling)
+		mask_value = EFA_REGS_ADMIN_INTR_MASK;
+
+	writel(mask_value, edev->reg_bar + EFA_REGS_INTR_MASK_OFF);
+	if (polling)
+		set_bit(EFA_AQ_STATE_POLLING_BIT, &edev->aq.state);
+	else
+		clear_bit(EFA_AQ_STATE_POLLING_BIT, &edev->aq.state);
+}
+
+static void efa_com_stats_init(struct efa_com_dev *edev)
+{
+	atomic64_t *s = (atomic64_t *)&edev->aq.stats;
+	int i;
+
+	for (i = 0; i < sizeof(edev->aq.stats) / sizeof(*s); i++, s++)
+		atomic64_set(s, 0);
+}
+
+/**
+ * efa_com_admin_init - Init the admin and the async queues
+ * @edev: EFA communication layer struct
+ * @aenq_handlers: Those handlers to be called upon event.
+ *
+ * Initialize the admin submission and completion queues.
+ * Initialize the asynchronous events notification queues.
+ *
+ * @return - 0 on success, negative value on failure.
+ */
+int efa_com_admin_init(struct efa_com_dev *edev,
+		       struct efa_aenq_handlers *aenq_handlers)
+{
+	struct efa_com_admin_queue *aq = &edev->aq;
+	u32 timeout;
+	u32 dev_sts;
+	u32 cap;
+	int err;
+
+	dev_sts = efa_com_reg_read32(edev, EFA_REGS_DEV_STS_OFF);
+	if (!(dev_sts & EFA_REGS_DEV_STS_READY_MASK)) {
+		ibdev_err(edev->efa_dev,
+			  "Device isn't ready, abort com init %#x\n", dev_sts);
+		return -ENODEV;
+	}
+
+	aq->depth = EFA_ADMIN_QUEUE_DEPTH;
+
+	aq->dmadev = edev->dmadev;
+	aq->efa_dev = edev->efa_dev;
+	set_bit(EFA_AQ_STATE_POLLING_BIT, &aq->state);
+
+	sema_init(&aq->avail_cmds, aq->depth);
+
+	efa_com_stats_init(edev);
+
+	err = efa_com_init_comp_ctxt(aq);
+	if (err)
+		return err;
+
+	err = efa_com_admin_init_sq(edev);
+	if (err)
+		goto err_destroy_comp_ctxt;
+
+	err = efa_com_admin_init_cq(edev);
+	if (err)
+		goto err_destroy_sq;
+
+	efa_com_set_admin_polling_mode(edev, false);
+
+	err = efa_com_admin_init_aenq(edev, aenq_handlers);
+	if (err)
+		goto err_destroy_cq;
+
+	cap = efa_com_reg_read32(edev, EFA_REGS_CAPS_OFF);
+	timeout = (cap & EFA_REGS_CAPS_ADMIN_CMD_TO_MASK) >>
+		  EFA_REGS_CAPS_ADMIN_CMD_TO_SHIFT;
+	if (timeout)
+		/* the resolution of timeout reg is 100ms */
+		aq->completion_timeout = timeout * 100000;
+	else
+		aq->completion_timeout = ADMIN_CMD_TIMEOUT_US;
+
+	aq->poll_interval = EFA_POLL_INTERVAL_MS;
+
+	set_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state);
+
+	return 0;
+
+err_destroy_cq:
+	dma_free_coherent(edev->dmadev, aq->depth * sizeof(*aq->cq.entries),
+			  aq->cq.entries, aq->cq.dma_addr);
+err_destroy_sq:
+	dma_free_coherent(edev->dmadev, aq->depth * sizeof(*aq->sq.entries),
+			  aq->sq.entries, aq->sq.dma_addr);
+err_destroy_comp_ctxt:
+	devm_kfree(edev->dmadev, aq->comp_ctx);
+
+	return err;
+}
+
+/**
+ * efa_com_admin_q_comp_intr_handler - admin queue interrupt handler
+ * @edev: EFA communication layer struct
+ *
+ * This method goes over the admin completion queue and wakes up
+ * all the pending threads that wait on the commands wait event.
+ *
+ * @note: Should be called after MSI-X interrupt.
+ */
+void efa_com_admin_q_comp_intr_handler(struct efa_com_dev *edev)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&edev->aq.cq.lock, flags);
+	efa_com_handle_admin_completion(&edev->aq);
+	spin_unlock_irqrestore(&edev->aq.cq.lock, flags);
+}
+
+/*
+ * efa_handle_specific_aenq_event:
+ * return the handler that is relevant to the specific event group
+ */
+static efa_aenq_handler efa_com_get_specific_aenq_cb(struct efa_com_dev *edev,
+						     u16 group)
+{
+	struct efa_aenq_handlers *aenq_handlers = edev->aenq.aenq_handlers;
+
+	if (group < EFA_MAX_HANDLERS && aenq_handlers->handlers[group])
+		return aenq_handlers->handlers[group];
+
+	return aenq_handlers->unimplemented_handler;
+}
+
+/**
+ * efa_com_aenq_intr_handler - AENQ interrupt handler
+ * @edev: EFA communication layer struct
+ * @data: Data of interrupt handler.
+ *
+ * Go over the async event notification queue and call the proper aenq handler.
+ */
+void efa_com_aenq_intr_handler(struct efa_com_dev *edev, void *data)
+{
+	struct efa_admin_aenq_common_desc *aenq_common;
+	struct efa_com_aenq *aenq = &edev->aenq;
+	struct efa_admin_aenq_entry *aenq_e;
+	efa_aenq_handler handler_cb;
+	u32 processed = 0;
+	u8 phase;
+	u32 ci;
+
+	ci = aenq->cc & (aenq->depth - 1);
+	phase = aenq->phase;
+	aenq_e = &aenq->entries[ci]; /* Get first entry */
+	aenq_common = &aenq_e->aenq_common_desc;
+
+	/* Go over all the events */
+	while ((READ_ONCE(aenq_common->flags) &
+		EFA_ADMIN_AENQ_COMMON_DESC_PHASE_MASK) == phase) {
+		/*
+		 * Do not read the rest of the completion entry before the
+		 * phase bit was validated
+		 */
+		dma_rmb();
+
+		/* Handle specific event*/
+		handler_cb = efa_com_get_specific_aenq_cb(edev,
+							  aenq_common->group);
+		handler_cb(data, aenq_e); /* call the actual event handler*/
+
+		/* Get next event entry */
+		ci++;
+		processed++;
+
+		if (ci == aenq->depth) {
+			ci = 0;
+			phase = !phase;
+		}
+		aenq_e = &aenq->entries[ci];
+		aenq_common = &aenq_e->aenq_common_desc;
+	}
+
+	aenq->cc += processed;
+	aenq->phase = phase;
+
+	/* Don't update aenq doorbell if there weren't any processed events */
+	if (!processed)
+		return;
+
+	/* barrier not needed in case of writel */
+	writel(aenq->cc, edev->reg_bar + EFA_REGS_AENQ_CONS_DB_OFF);
+}
+
+static void efa_com_mmio_reg_read_resp_addr_init(struct efa_com_dev *edev)
+{
+	struct efa_com_mmio_read *mmio_read = &edev->mmio_read;
+	u32 addr_high;
+	u32 addr_low;
+
+	/* dma_addr_bits is unknown at this point */
+	addr_high = (mmio_read->read_resp_dma_addr >> 32) & GENMASK(31, 0);
+	addr_low = mmio_read->read_resp_dma_addr & GENMASK(31, 0);
+
+	writel(addr_high, edev->reg_bar + EFA_REGS_MMIO_RESP_HI_OFF);
+	writel(addr_low, edev->reg_bar + EFA_REGS_MMIO_RESP_LO_OFF);
+}
+
+int efa_com_mmio_reg_read_init(struct efa_com_dev *edev)
+{
+	struct efa_com_mmio_read *mmio_read = &edev->mmio_read;
+
+	spin_lock_init(&mmio_read->lock);
+	mmio_read->read_resp =
+		dma_alloc_coherent(edev->dmadev, sizeof(*mmio_read->read_resp),
+				   &mmio_read->read_resp_dma_addr, GFP_KERNEL);
+	if (!mmio_read->read_resp)
+		return -ENOMEM;
+
+	efa_com_mmio_reg_read_resp_addr_init(edev);
+
+	mmio_read->read_resp->req_id = 0;
+	mmio_read->seq_num = 0;
+	mmio_read->mmio_read_timeout = EFA_REG_READ_TIMEOUT_US;
+
+	return 0;
+}
+
+void efa_com_mmio_reg_read_destroy(struct efa_com_dev *edev)
+{
+	struct efa_com_mmio_read *mmio_read = &edev->mmio_read;
+
+	dma_free_coherent(edev->dmadev, sizeof(*mmio_read->read_resp),
+			  mmio_read->read_resp, mmio_read->read_resp_dma_addr);
+}
+
+int efa_com_validate_version(struct efa_com_dev *edev)
+{
+	u32 ctrl_ver_masked;
+	u32 ctrl_ver;
+	u32 ver;
+
+	/*
+	 * Make sure the EFA version and the controller version are at least
+	 * as the driver expects
+	 */
+	ver = efa_com_reg_read32(edev, EFA_REGS_VERSION_OFF);
+	ctrl_ver = efa_com_reg_read32(edev,
+				      EFA_REGS_CONTROLLER_VERSION_OFF);
+
+	ibdev_dbg(edev->efa_dev, "efa device version: %d.%d\n",
+		  (ver & EFA_REGS_VERSION_MAJOR_VERSION_MASK) >>
+			  EFA_REGS_VERSION_MAJOR_VERSION_SHIFT,
+		  ver & EFA_REGS_VERSION_MINOR_VERSION_MASK);
+
+	if (ver < MIN_EFA_VER) {
+		ibdev_err(edev->efa_dev,
+			  "EFA version is lower than the minimal version the driver supports\n");
+		return -EOPNOTSUPP;
+	}
+
+	ibdev_dbg(edev->efa_dev,
+		  "efa controller version: %d.%d.%d implementation version %d\n",
+		  (ctrl_ver & EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_MASK) >>
+			  EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_SHIFT,
+		  (ctrl_ver & EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION_MASK) >>
+			  EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION_SHIFT,
+		  (ctrl_ver & EFA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION_MASK),
+		  (ctrl_ver & EFA_REGS_CONTROLLER_VERSION_IMPL_ID_MASK) >>
+			  EFA_REGS_CONTROLLER_VERSION_IMPL_ID_SHIFT);
+
+	ctrl_ver_masked =
+		(ctrl_ver & EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_MASK) |
+		(ctrl_ver & EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION_MASK) |
+		(ctrl_ver & EFA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION_MASK);
+
+	/* Validate the ctrl version without the implementation ID */
+	if (ctrl_ver_masked < MIN_EFA_CTRL_VER) {
+		ibdev_err(edev->efa_dev,
+			  "EFA ctrl version is lower than the minimal ctrl version the driver supports\n");
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+/**
+ * efa_com_get_dma_width - Retrieve physical dma address width the device
+ * supports.
+ * @edev: EFA communication layer struct
+ *
+ * Retrieve the maximum physical address bits the device can handle.
+ *
+ * @return: > 0 on Success and negative value otherwise.
+ */
+int efa_com_get_dma_width(struct efa_com_dev *edev)
+{
+	u32 caps = efa_com_reg_read32(edev, EFA_REGS_CAPS_OFF);
+	int width;
+
+	width = (caps & EFA_REGS_CAPS_DMA_ADDR_WIDTH_MASK) >>
+		EFA_REGS_CAPS_DMA_ADDR_WIDTH_SHIFT;
+
+	ibdev_dbg(edev->efa_dev, "DMA width: %d\n", width);
+
+	if (width < 32 || width > 64) {
+		ibdev_err(edev->efa_dev, "DMA width illegal value: %d\n", width);
+		return -EINVAL;
+	}
+
+	edev->dma_addr_bits = width;
+
+	return width;
+}
+
+static int wait_for_reset_state(struct efa_com_dev *edev, u32 timeout,
+				u16 exp_state)
+{
+	u32 val, i;
+
+	for (i = 0; i < timeout; i++) {
+		val = efa_com_reg_read32(edev, EFA_REGS_DEV_STS_OFF);
+
+		if ((val & EFA_REGS_DEV_STS_RESET_IN_PROGRESS_MASK) ==
+		    exp_state)
+			return 0;
+
+		ibdev_dbg(edev->efa_dev, "Reset indication val %d\n", val);
+		msleep(EFA_POLL_INTERVAL_MS);
+	}
+
+	return -ETIME;
+}
+
+/**
+ * efa_com_dev_reset - Perform device FLR to the device.
+ * @edev: EFA communication layer struct
+ * @reset_reason: Specify what is the trigger for the reset in case of an error.
+ *
+ * @return - 0 on success, negative value on failure.
+ */
+int efa_com_dev_reset(struct efa_com_dev *edev,
+		      enum efa_regs_reset_reason_types reset_reason)
+{
+	u32 stat, timeout, cap, reset_val;
+	int err;
+
+	stat = efa_com_reg_read32(edev, EFA_REGS_DEV_STS_OFF);
+	cap = efa_com_reg_read32(edev, EFA_REGS_CAPS_OFF);
+
+	if (!(stat & EFA_REGS_DEV_STS_READY_MASK)) {
+		ibdev_err(edev->efa_dev,
+			  "Device isn't ready, can't reset device\n");
+		return -EINVAL;
+	}
+
+	timeout = (cap & EFA_REGS_CAPS_RESET_TIMEOUT_MASK) >>
+		  EFA_REGS_CAPS_RESET_TIMEOUT_SHIFT;
+	if (!timeout) {
+		ibdev_err(edev->efa_dev, "Invalid timeout value\n");
+		return -EINVAL;
+	}
+
+	/* start reset */
+	reset_val = EFA_REGS_DEV_CTL_DEV_RESET_MASK;
+	reset_val |= (reset_reason << EFA_REGS_DEV_CTL_RESET_REASON_SHIFT) &
+		     EFA_REGS_DEV_CTL_RESET_REASON_MASK;
+	writel(reset_val, edev->reg_bar + EFA_REGS_DEV_CTL_OFF);
+
+	/* reset clears the mmio readless address, restore it */
+	efa_com_mmio_reg_read_resp_addr_init(edev);
+
+	err = wait_for_reset_state(edev, timeout,
+				   EFA_REGS_DEV_STS_RESET_IN_PROGRESS_MASK);
+	if (err) {
+		ibdev_err(edev->efa_dev, "Reset indication didn't turn on\n");
+		return err;
+	}
+
+	/* reset done */
+	writel(0, edev->reg_bar + EFA_REGS_DEV_CTL_OFF);
+	err = wait_for_reset_state(edev, timeout, 0);
+	if (err) {
+		ibdev_err(edev->efa_dev, "Reset indication didn't turn off\n");
+		return err;
+	}
+
+	timeout = (cap & EFA_REGS_CAPS_ADMIN_CMD_TO_MASK) >>
+		  EFA_REGS_CAPS_ADMIN_CMD_TO_SHIFT;
+	if (timeout)
+		/* the resolution of timeout reg is 100ms */
+		edev->aq.completion_timeout = timeout * 100000;
+	else
+		edev->aq.completion_timeout = ADMIN_CMD_TIMEOUT_US;
+
+	return 0;
+}
diff --git a/drivers/infiniband/hw/efa/efa_com.h b/drivers/infiniband/hw/efa/efa_com.h
new file mode 100644
index 000000000000..84d96724a74b
--- /dev/null
+++ b/drivers/infiniband/hw/efa/efa_com.h
@@ -0,0 +1,144 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
+/*
+ * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
+ */
+
+#ifndef _EFA_COM_H_
+#define _EFA_COM_H_
+
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+#include <linux/semaphore.h>
+#include <linux/sched.h>
+
+#include <rdma/ib_verbs.h>
+
+#include "efa_common_defs.h"
+#include "efa_admin_defs.h"
+#include "efa_admin_cmds_defs.h"
+#include "efa_regs_defs.h"
+
+#define EFA_MAX_HANDLERS 256
+
+struct efa_com_admin_cq {
+	struct efa_admin_acq_entry *entries;
+	dma_addr_t dma_addr;
+	spinlock_t lock; /* Protects ACQ */
+
+	u16 cc; /* consumer counter */
+	u8 phase;
+};
+
+struct efa_com_admin_sq {
+	struct efa_admin_aq_entry *entries;
+	dma_addr_t dma_addr;
+	spinlock_t lock; /* Protects ASQ */
+
+	u32 __iomem *db_addr;
+
+	u16 cc; /* consumer counter */
+	u16 pc; /* producer counter */
+	u8 phase;
+
+};
+
+/* Don't use anything other than atomic64 */
+struct efa_com_stats_admin {
+	atomic64_t aborted_cmd;
+	atomic64_t submitted_cmd;
+	atomic64_t completed_cmd;
+	atomic64_t no_completion;
+};
+
+enum {
+	EFA_AQ_STATE_RUNNING_BIT = 0,
+	EFA_AQ_STATE_POLLING_BIT = 1,
+};
+
+struct efa_com_admin_queue {
+	void *dmadev;
+	void *efa_dev;
+	struct efa_comp_ctx *comp_ctx;
+	u32 completion_timeout; /* usecs */
+	u16 poll_interval; /* msecs */
+	u16 depth;
+	struct efa_com_admin_cq cq;
+	struct efa_com_admin_sq sq;
+	u16 msix_vector_idx;
+
+	unsigned long state;
+
+	/* Count the number of available admin commands */
+	struct semaphore avail_cmds;
+
+	struct efa_com_stats_admin stats;
+
+	spinlock_t comp_ctx_lock; /* Protects completion context pool */
+	u32 *comp_ctx_pool;
+	u16 comp_ctx_pool_next;
+};
+
+struct efa_aenq_handlers;
+
+struct efa_com_aenq {
+	struct efa_admin_aenq_entry *entries;
+	struct efa_aenq_handlers *aenq_handlers;
+	dma_addr_t dma_addr;
+	u32 cc; /* consumer counter */
+	u16 msix_vector_idx;
+	u16 depth;
+	u8 phase;
+};
+
+struct efa_com_mmio_read {
+	struct efa_admin_mmio_req_read_less_resp *read_resp;
+	dma_addr_t read_resp_dma_addr;
+	u16 seq_num;
+	u16 mmio_read_timeout; /* usecs */
+	/* serializes mmio reads */
+	spinlock_t lock;
+};
+
+struct efa_com_dev {
+	struct efa_com_admin_queue aq;
+	struct efa_com_aenq aenq;
+	u8 __iomem *reg_bar;
+	void *dmadev;
+	void *efa_dev;
+	u32 supported_features;
+	u32 dma_addr_bits;
+
+	struct efa_com_mmio_read mmio_read;
+};
+
+typedef void (*efa_aenq_handler)(void *data,
+	      struct efa_admin_aenq_entry *aenq_e);
+
+/* Holds aenq handlers. Indexed by AENQ event group */
+struct efa_aenq_handlers {
+	efa_aenq_handler handlers[EFA_MAX_HANDLERS];
+	efa_aenq_handler unimplemented_handler;
+};
+
+int efa_com_admin_init(struct efa_com_dev *edev,
+		       struct efa_aenq_handlers *aenq_handlers);
+void efa_com_admin_destroy(struct efa_com_dev *edev);
+int efa_com_dev_reset(struct efa_com_dev *edev,
+		      enum efa_regs_reset_reason_types reset_reason);
+void efa_com_set_admin_polling_mode(struct efa_com_dev *edev, bool polling);
+void efa_com_admin_q_comp_intr_handler(struct efa_com_dev *edev);
+int efa_com_mmio_reg_read_init(struct efa_com_dev *edev);
+void efa_com_mmio_reg_read_destroy(struct efa_com_dev *edev);
+
+int efa_com_validate_version(struct efa_com_dev *edev);
+int efa_com_get_dma_width(struct efa_com_dev *edev);
+
+int efa_com_cmd_exec(struct efa_com_admin_queue *aq,
+		     struct efa_admin_aq_entry *cmd,
+		     size_t cmd_size,
+		     struct efa_admin_acq_entry *comp,
+		     size_t comp_size);
+void efa_com_aenq_intr_handler(struct efa_com_dev *edev, void *data);
+
+#endif /* _EFA_COM_H_ */
diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.c b/drivers/infiniband/hw/efa/efa_com_cmd.c
new file mode 100644
index 000000000000..14227725521c
--- /dev/null
+++ b/drivers/infiniband/hw/efa/efa_com_cmd.c
@@ -0,0 +1,692 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
+/*
+ * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
+ */
+
+#include "efa.h"
+#include "efa_com.h"
+#include "efa_com_cmd.h"
+
+void efa_com_set_dma_addr(dma_addr_t addr, u32 *addr_high, u32 *addr_low)
+{
+	*addr_low = lower_32_bits(addr);
+	*addr_high = upper_32_bits(addr);
+}
+
+int efa_com_create_qp(struct efa_com_dev *edev,
+		      struct efa_com_create_qp_params *params,
+		      struct efa_com_create_qp_result *res)
+{
+	struct efa_admin_create_qp_cmd create_qp_cmd = {};
+	struct efa_admin_create_qp_resp cmd_completion;
+	struct efa_com_admin_queue *aq = &edev->aq;
+	int err;
+
+	create_qp_cmd.aq_common_desc.opcode = EFA_ADMIN_CREATE_QP;
+
+	create_qp_cmd.pd = params->pd;
+	create_qp_cmd.qp_type = params->qp_type;
+	create_qp_cmd.rq_base_addr = params->rq_base_addr;
+	create_qp_cmd.send_cq_idx = params->send_cq_idx;
+	create_qp_cmd.recv_cq_idx = params->recv_cq_idx;
+	create_qp_cmd.qp_alloc_size.send_queue_ring_size =
+		params->sq_ring_size_in_bytes;
+	create_qp_cmd.qp_alloc_size.send_queue_depth =
+			params->sq_depth;
+	create_qp_cmd.qp_alloc_size.recv_queue_ring_size =
+			params->rq_ring_size_in_bytes;
+	create_qp_cmd.qp_alloc_size.recv_queue_depth =
+			params->rq_depth;
+	create_qp_cmd.uar = params->uarn;
+
+	err = efa_com_cmd_exec(aq,
+			       (struct efa_admin_aq_entry *)&create_qp_cmd,
+			       sizeof(create_qp_cmd),
+			       (struct efa_admin_acq_entry *)&cmd_completion,
+			       sizeof(cmd_completion));
+	if (err) {
+		ibdev_err(edev->efa_dev, "Failed to create qp [%d]\n", err);
+		return err;
+	}
+
+	res->qp_handle = cmd_completion.qp_handle;
+	res->qp_num = cmd_completion.qp_num;
+	res->sq_db_offset = cmd_completion.sq_db_offset;
+	res->rq_db_offset = cmd_completion.rq_db_offset;
+	res->llq_descriptors_offset = cmd_completion.llq_descriptors_offset;
+	res->send_sub_cq_idx = cmd_completion.send_sub_cq_idx;
+	res->recv_sub_cq_idx = cmd_completion.recv_sub_cq_idx;
+
+	return err;
+}
+
+int efa_com_modify_qp(struct efa_com_dev *edev,
+		      struct efa_com_modify_qp_params *params)
+{
+	struct efa_com_admin_queue *aq = &edev->aq;
+	struct efa_admin_modify_qp_cmd cmd = {};
+	struct efa_admin_modify_qp_resp resp;
+	int err;
+
+	cmd.aq_common_desc.opcode = EFA_ADMIN_MODIFY_QP;
+	cmd.modify_mask = params->modify_mask;
+	cmd.qp_handle = params->qp_handle;
+	cmd.qp_state = params->qp_state;
+	cmd.cur_qp_state = params->cur_qp_state;
+	cmd.qkey = params->qkey;
+	cmd.sq_psn = params->sq_psn;
+	cmd.sq_drained_async_notify = params->sq_drained_async_notify;
+
+	err = efa_com_cmd_exec(aq,
+			       (struct efa_admin_aq_entry *)&cmd,
+			       sizeof(cmd),
+			       (struct efa_admin_acq_entry *)&resp,
+			       sizeof(resp));
+	if (err) {
+		ibdev_err(edev->efa_dev,
+			  "Failed to modify qp-%u modify_mask[%#x] [%d]\n",
+			  cmd.qp_handle, cmd.modify_mask, err);
+		return err;
+	}
+
+	return 0;
+}
+
+int efa_com_query_qp(struct efa_com_dev *edev,
+		     struct efa_com_query_qp_params *params,
+		     struct efa_com_query_qp_result *result)
+{
+	struct efa_com_admin_queue *aq = &edev->aq;
+	struct efa_admin_query_qp_cmd cmd = {};
+	struct efa_admin_query_qp_resp resp;
+	int err;
+
+	cmd.aq_common_desc.opcode = EFA_ADMIN_QUERY_QP;
+	cmd.qp_handle = params->qp_handle;
+
+	err = efa_com_cmd_exec(aq,
+			       (struct efa_admin_aq_entry *)&cmd,
+			       sizeof(cmd),
+			       (struct efa_admin_acq_entry *)&resp,
+			       sizeof(resp));
+	if (err) {
+		ibdev_err(edev->efa_dev, "Failed to query qp-%u [%d]\n",
+			  cmd.qp_handle, err);
+		return err;
+	}
+
+	result->qp_state = resp.qp_state;
+	result->qkey = resp.qkey;
+	result->sq_draining = resp.sq_draining;
+	result->sq_psn = resp.sq_psn;
+
+	return 0;
+}
+
+int efa_com_destroy_qp(struct efa_com_dev *edev,
+		       struct efa_com_destroy_qp_params *params)
+{
+	struct efa_admin_destroy_qp_resp cmd_completion;
+	struct efa_admin_destroy_qp_cmd qp_cmd = {};
+	struct efa_com_admin_queue *aq = &edev->aq;
+	int err;
+
+	qp_cmd.aq_common_desc.opcode = EFA_ADMIN_DESTROY_QP;
+	qp_cmd.qp_handle = params->qp_handle;
+
+	err = efa_com_cmd_exec(aq,
+			       (struct efa_admin_aq_entry *)&qp_cmd,
+			       sizeof(qp_cmd),
+			       (struct efa_admin_acq_entry *)&cmd_completion,
+			       sizeof(cmd_completion));
+	if (err)
+		ibdev_err(edev->efa_dev, "Failed to destroy qp-%u [%d]\n",
+			  qp_cmd.qp_handle, err);
+
+	return 0;
+}
+
+int efa_com_create_cq(struct efa_com_dev *edev,
+		      struct efa_com_create_cq_params *params,
+		      struct efa_com_create_cq_result *result)
+{
+	struct efa_admin_create_cq_resp cmd_completion;
+	struct efa_admin_create_cq_cmd create_cmd = {};
+	struct efa_com_admin_queue *aq = &edev->aq;
+	int err;
+
+	create_cmd.aq_common_desc.opcode = EFA_ADMIN_CREATE_CQ;
+	create_cmd.cq_caps_2 = (params->entry_size_in_bytes / 4) &
+				EFA_ADMIN_CREATE_CQ_CMD_CQ_ENTRY_SIZE_WORDS_MASK;
+	create_cmd.cq_depth = params->cq_depth;
+	create_cmd.num_sub_cqs = params->num_sub_cqs;
+	create_cmd.uar = params->uarn;
+
+	efa_com_set_dma_addr(params->dma_addr,
+			     &create_cmd.cq_ba.mem_addr_high,
+			     &create_cmd.cq_ba.mem_addr_low);
+
+	err = efa_com_cmd_exec(aq,
+			       (struct efa_admin_aq_entry *)&create_cmd,
+			       sizeof(create_cmd),
+			       (struct efa_admin_acq_entry *)&cmd_completion,
+			       sizeof(cmd_completion));
+	if (err) {
+		ibdev_err(edev->efa_dev, "Failed to create cq[%d]\n", err);
+		return err;
+	}
+
+	result->cq_idx = cmd_completion.cq_idx;
+	result->actual_depth = params->cq_depth;
+
+	return err;
+}
+
+int efa_com_destroy_cq(struct efa_com_dev *edev,
+		       struct efa_com_destroy_cq_params *params)
+{
+	struct efa_admin_destroy_cq_cmd destroy_cmd = {};
+	struct efa_admin_destroy_cq_resp destroy_resp;
+	struct efa_com_admin_queue *aq = &edev->aq;
+	int err;
+
+	destroy_cmd.cq_idx = params->cq_idx;
+	destroy_cmd.aq_common_desc.opcode = EFA_ADMIN_DESTROY_CQ;
+
+	err = efa_com_cmd_exec(aq,
+			       (struct efa_admin_aq_entry *)&destroy_cmd,
+			       sizeof(destroy_cmd),
+			       (struct efa_admin_acq_entry *)&destroy_resp,
+			       sizeof(destroy_resp));
+
+	if (err)
+		ibdev_err(edev->efa_dev, "Failed to destroy CQ-%u [%d]\n",
+			  params->cq_idx, err);
+
+	return 0;
+}
+
+int efa_com_register_mr(struct efa_com_dev *edev,
+			struct efa_com_reg_mr_params *params,
+			struct efa_com_reg_mr_result *result)
+{
+	struct efa_admin_reg_mr_resp cmd_completion;
+	struct efa_com_admin_queue *aq = &edev->aq;
+	struct efa_admin_reg_mr_cmd mr_cmd = {};
+	int err;
+
+	mr_cmd.aq_common_desc.opcode = EFA_ADMIN_REG_MR;
+	mr_cmd.pd = params->pd;
+	mr_cmd.mr_length = params->mr_length_in_bytes;
+	mr_cmd.flags |= params->page_shift &
+		EFA_ADMIN_REG_MR_CMD_PHYS_PAGE_SIZE_SHIFT_MASK;
+	mr_cmd.iova = params->iova;
+	mr_cmd.permissions |= params->permissions &
+			      EFA_ADMIN_REG_MR_CMD_LOCAL_WRITE_ENABLE_MASK;
+
+	if (params->inline_pbl) {
+		memcpy(mr_cmd.pbl.inline_pbl_array,
+		       params->pbl.inline_pbl_array,
+		       sizeof(mr_cmd.pbl.inline_pbl_array));
+	} else {
+		mr_cmd.pbl.pbl.length = params->pbl.pbl.length;
+		mr_cmd.pbl.pbl.address.mem_addr_low =
+			params->pbl.pbl.address.mem_addr_low;
+		mr_cmd.pbl.pbl.address.mem_addr_high =
+			params->pbl.pbl.address.mem_addr_high;
+		mr_cmd.aq_common_desc.flags |=
+			EFA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_MASK;
+		if (params->indirect)
+			mr_cmd.aq_common_desc.flags |=
+				EFA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
+	}
+
+	err = efa_com_cmd_exec(aq,
+			       (struct efa_admin_aq_entry *)&mr_cmd,
+			       sizeof(mr_cmd),
+			       (struct efa_admin_acq_entry *)&cmd_completion,
+			       sizeof(cmd_completion));
+	if (err) {
+		ibdev_err(edev->efa_dev, "Failed to register mr [%d]\n", err);
+		return err;
+	}
+
+	result->l_key = cmd_completion.l_key;
+	result->r_key = cmd_completion.r_key;
+
+	return 0;
+}
+
+int efa_com_dereg_mr(struct efa_com_dev *edev,
+		     struct efa_com_dereg_mr_params *params)
+{
+	struct efa_admin_dereg_mr_resp cmd_completion;
+	struct efa_com_admin_queue *aq = &edev->aq;
+	struct efa_admin_dereg_mr_cmd mr_cmd = {};
+	int err;
+
+	mr_cmd.aq_common_desc.opcode = EFA_ADMIN_DEREG_MR;
+	mr_cmd.l_key = params->l_key;
+
+	err = efa_com_cmd_exec(aq,
+			       (struct efa_admin_aq_entry *)&mr_cmd,
+			       sizeof(mr_cmd),
+			       (struct efa_admin_acq_entry *)&cmd_completion,
+			       sizeof(cmd_completion));
+	if (err)
+		ibdev_err(edev->efa_dev,
+			  "Failed to de-register mr(lkey-%u) [%d]\n",
+			  mr_cmd.l_key, err);
+
+	return 0;
+}
+
+int efa_com_create_ah(struct efa_com_dev *edev,
+		      struct efa_com_create_ah_params *params,
+		      struct efa_com_create_ah_result *result)
+{
+	struct efa_admin_create_ah_resp cmd_completion;
+	struct efa_com_admin_queue *aq = &edev->aq;
+	struct efa_admin_create_ah_cmd ah_cmd = {};
+	int err;
+
+	ah_cmd.aq_common_desc.opcode = EFA_ADMIN_CREATE_AH;
+
+	memcpy(ah_cmd.dest_addr, params->dest_addr, sizeof(ah_cmd.dest_addr));
+	ah_cmd.pd = params->pdn;
+
+	err = efa_com_cmd_exec(aq,
+			       (struct efa_admin_aq_entry *)&ah_cmd,
+			       sizeof(ah_cmd),
+			       (struct efa_admin_acq_entry *)&cmd_completion,
+			       sizeof(cmd_completion));
+	if (err) {
+		ibdev_err(edev->efa_dev, "Failed to create ah [%d]\n", err);
+		return err;
+	}
+
+	result->ah = cmd_completion.ah;
+
+	return 0;
+}
+
+int efa_com_destroy_ah(struct efa_com_dev *edev,
+		       struct efa_com_destroy_ah_params *params)
+{
+	struct efa_admin_destroy_ah_resp cmd_completion;
+	struct efa_admin_destroy_ah_cmd ah_cmd = {};
+	struct efa_com_admin_queue *aq = &edev->aq;
+	int err;
+
+	ah_cmd.aq_common_desc.opcode = EFA_ADMIN_DESTROY_AH;
+	ah_cmd.ah = params->ah;
+	ah_cmd.pd = params->pdn;
+
+	err = efa_com_cmd_exec(aq,
+			       (struct efa_admin_aq_entry *)&ah_cmd,
+			       sizeof(ah_cmd),
+			       (struct efa_admin_acq_entry *)&cmd_completion,
+			       sizeof(cmd_completion));
+	if (err)
+		ibdev_err(edev->efa_dev, "Failed to destroy ah-%d pd-%d [%d]\n",
+			  ah_cmd.ah, ah_cmd.pd, err);
+
+	return 0;
+}
+
+static bool
+efa_com_check_supported_feature_id(struct efa_com_dev *edev,
+				   enum efa_admin_aq_feature_id feature_id)
+{
+	u32 feature_mask = 1 << feature_id;
+
+	/* Device attributes is always supported */
+	if (feature_id != EFA_ADMIN_DEVICE_ATTR &&
+	    !(edev->supported_features & feature_mask))
+		return false;
+
+	return true;
+}
+
+static int efa_com_get_feature_ex(struct efa_com_dev *edev,
+				  struct efa_admin_get_feature_resp *get_resp,
+				  enum efa_admin_aq_feature_id feature_id,
+				  dma_addr_t control_buf_dma_addr,
+				  u32 control_buff_size)
+{
+	struct efa_admin_get_feature_cmd get_cmd = {};
+	struct efa_com_admin_queue *aq;
+	int err;
+
+	if (!efa_com_check_supported_feature_id(edev, feature_id)) {
+		ibdev_err(edev->efa_dev, "Feature %d isn't supported\n",
+			  feature_id);
+		return -EOPNOTSUPP;
+	}
+
+	aq = &edev->aq;
+
+	get_cmd.aq_common_descriptor.opcode = EFA_ADMIN_GET_FEATURE;
+
+	if (control_buff_size)
+		get_cmd.aq_common_descriptor.flags =
+			EFA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
+
+
+	efa_com_set_dma_addr(control_buf_dma_addr,
+			     &get_cmd.control_buffer.address.mem_addr_high,
+			     &get_cmd.control_buffer.address.mem_addr_low);
+
+	get_cmd.control_buffer.length = control_buff_size;
+	get_cmd.feature_common.feature_id = feature_id;
+	err = efa_com_cmd_exec(aq,
+			       (struct efa_admin_aq_entry *)
+			       &get_cmd,
+			       sizeof(get_cmd),
+			       (struct efa_admin_acq_entry *)
+			       get_resp,
+			       sizeof(*get_resp));
+
+	if (err)
+		ibdev_err(edev->efa_dev,
+			  "Failed to submit get_feature command %d [%d]\n",
+			  feature_id, err);
+
+	return 0;
+}
+
+static int efa_com_get_feature(struct efa_com_dev *edev,
+			       struct efa_admin_get_feature_resp *get_resp,
+			       enum efa_admin_aq_feature_id feature_id)
+{
+	return efa_com_get_feature_ex(edev, get_resp, feature_id, 0, 0);
+}
+
+int efa_com_get_network_attr(struct efa_com_dev *edev,
+			     struct efa_com_get_network_attr_result *result)
+{
+	struct efa_admin_get_feature_resp resp;
+	int err;
+
+	err = efa_com_get_feature(edev, &resp,
+				  EFA_ADMIN_NETWORK_ATTR);
+	if (err) {
+		ibdev_err(edev->efa_dev,
+			  "Failed to get network attributes %d\n", err);
+		return err;
+	}
+
+	memcpy(result->addr, resp.u.network_attr.addr,
+	       sizeof(resp.u.network_attr.addr));
+	result->mtu = resp.u.network_attr.mtu;
+
+	return 0;
+}
+
+int efa_com_get_device_attr(struct efa_com_dev *edev,
+			    struct efa_com_get_device_attr_result *result)
+{
+	struct efa_admin_get_feature_resp resp;
+	int err;
+
+	err = efa_com_get_feature(edev, &resp, EFA_ADMIN_DEVICE_ATTR);
+	if (err) {
+		ibdev_err(edev->efa_dev, "Failed to get device attributes %d\n",
+			  err);
+		return err;
+	}
+
+	result->page_size_cap = resp.u.device_attr.page_size_cap;
+	result->fw_version = resp.u.device_attr.fw_version;
+	result->admin_api_version = resp.u.device_attr.admin_api_version;
+	result->device_version = resp.u.device_attr.device_version;
+	result->supported_features = resp.u.device_attr.supported_features;
+	result->phys_addr_width = resp.u.device_attr.phys_addr_width;
+	result->virt_addr_width = resp.u.device_attr.virt_addr_width;
+	result->db_bar = resp.u.device_attr.db_bar;
+
+	if (result->admin_api_version < 1) {
+		ibdev_err(edev->efa_dev,
+			  "Failed to get device attr api version [%u < 1]\n",
+			  result->admin_api_version);
+		return -EINVAL;
+	}
+
+	edev->supported_features = resp.u.device_attr.supported_features;
+	err = efa_com_get_feature(edev, &resp,
+				  EFA_ADMIN_QUEUE_ATTR);
+	if (err) {
+		ibdev_err(edev->efa_dev,
+			  "Failed to get network attributes %d\n", err);
+		return err;
+	}
+
+	result->max_qp = resp.u.queue_attr.max_qp;
+	result->max_sq_depth = resp.u.queue_attr.max_sq_depth;
+	result->max_rq_depth = resp.u.queue_attr.max_rq_depth;
+	result->max_cq = resp.u.queue_attr.max_cq;
+	result->max_cq_depth = resp.u.queue_attr.max_cq_depth;
+	result->inline_buf_size = resp.u.queue_attr.inline_buf_size;
+	result->max_sq_sge = resp.u.queue_attr.max_wr_send_sges;
+	result->max_rq_sge = resp.u.queue_attr.max_wr_recv_sges;
+	result->max_mr = resp.u.queue_attr.max_mr;
+	result->max_mr_pages = resp.u.queue_attr.max_mr_pages;
+	result->max_pd = resp.u.queue_attr.max_pd;
+	result->max_ah = resp.u.queue_attr.max_ah;
+	result->max_llq_size = resp.u.queue_attr.max_llq_size;
+	result->sub_cqs_per_cq = resp.u.queue_attr.sub_cqs_per_cq;
+
+	return 0;
+}
+
+int efa_com_get_hw_hints(struct efa_com_dev *edev,
+			 struct efa_com_get_hw_hints_result *result)
+{
+	struct efa_admin_get_feature_resp resp;
+	int err;
+
+	err = efa_com_get_feature(edev, &resp, EFA_ADMIN_HW_HINTS);
+	if (err) {
+		ibdev_err(edev->efa_dev, "Failed to get hw hints %d\n", err);
+		return err;
+	}
+
+	result->admin_completion_timeout = resp.u.hw_hints.admin_completion_timeout;
+	result->driver_watchdog_timeout = resp.u.hw_hints.driver_watchdog_timeout;
+	result->mmio_read_timeout = resp.u.hw_hints.mmio_read_timeout;
+	result->poll_interval = resp.u.hw_hints.poll_interval;
+
+	return 0;
+}
+
+static int efa_com_set_feature_ex(struct efa_com_dev *edev,
+				  struct efa_admin_set_feature_resp *set_resp,
+				  struct efa_admin_set_feature_cmd *set_cmd,
+				  enum efa_admin_aq_feature_id feature_id,
+				  dma_addr_t control_buf_dma_addr,
+				  u32 control_buff_size)
+{
+	struct efa_com_admin_queue *aq;
+	int err;
+
+	if (!efa_com_check_supported_feature_id(edev, feature_id)) {
+		ibdev_err(edev->efa_dev, "Feature %d isn't supported\n",
+			  feature_id);
+		return -EOPNOTSUPP;
+	}
+
+	aq = &edev->aq;
+
+	set_cmd->aq_common_descriptor.opcode = EFA_ADMIN_SET_FEATURE;
+	if (control_buff_size) {
+		set_cmd->aq_common_descriptor.flags =
+			EFA_ADMIN_AQ_COMMON_DESC_CTRL_DATA_INDIRECT_MASK;
+		efa_com_set_dma_addr(control_buf_dma_addr,
+				     &set_cmd->control_buffer.address.mem_addr_high,
+				     &set_cmd->control_buffer.address.mem_addr_low);
+	}
+
+	set_cmd->control_buffer.length = control_buff_size;
+	set_cmd->feature_common.feature_id = feature_id;
+	err = efa_com_cmd_exec(aq,
+			       (struct efa_admin_aq_entry *)set_cmd,
+			       sizeof(*set_cmd),
+			       (struct efa_admin_acq_entry *)set_resp,
+			       sizeof(*set_resp));
+
+	if (err)
+		ibdev_err(edev->efa_dev,
+			  "Failed to submit set_feature command %d error: %d\n",
+			  feature_id, err);
+
+	return 0;
+}
+
+static int efa_com_set_feature(struct efa_com_dev *edev,
+			       struct efa_admin_set_feature_resp *set_resp,
+			       struct efa_admin_set_feature_cmd *set_cmd,
+			       enum efa_admin_aq_feature_id feature_id)
+{
+	return efa_com_set_feature_ex(edev, set_resp, set_cmd, feature_id,
+				      0, 0);
+}
+
+int efa_com_set_aenq_config(struct efa_com_dev *edev, u32 groups)
+{
+	struct efa_admin_get_feature_resp get_resp;
+	struct efa_admin_set_feature_resp set_resp;
+	struct efa_admin_set_feature_cmd cmd = {};
+	int err;
+
+	ibdev_dbg(edev->efa_dev, "Configuring aenq with groups[%#x]\n", groups);
+
+	err = efa_com_get_feature(edev, &get_resp, EFA_ADMIN_AENQ_CONFIG);
+	if (err) {
+		ibdev_err(edev->efa_dev, "Failed to get aenq attributes: %d\n",
+			  err);
+		return err;
+	}
+
+	ibdev_dbg(edev->efa_dev,
+		  "Get aenq groups: supported[%#x] enabled[%#x]\n",
+		  get_resp.u.aenq.supported_groups,
+		  get_resp.u.aenq.enabled_groups);
+
+	if ((get_resp.u.aenq.supported_groups & groups) != groups) {
+		ibdev_err(edev->efa_dev,
+			  "Trying to set unsupported aenq groups[%#x] supported[%#x]\n",
+			  groups, get_resp.u.aenq.supported_groups);
+		return -EOPNOTSUPP;
+	}
+
+	cmd.u.aenq.enabled_groups = groups;
+	err = efa_com_set_feature(edev, &set_resp, &cmd,
+				  EFA_ADMIN_AENQ_CONFIG);
+	if (err) {
+		ibdev_err(edev->efa_dev, "Failed to set aenq attributes: %d\n",
+			  err);
+		return err;
+	}
+
+	return 0;
+}
+
+int efa_com_alloc_pd(struct efa_com_dev *edev,
+		     struct efa_com_alloc_pd_result *result)
+{
+	struct efa_com_admin_queue *aq = &edev->aq;
+	struct efa_admin_alloc_pd_cmd cmd = {};
+	struct efa_admin_alloc_pd_resp resp;
+	int err;
+
+	cmd.aq_common_descriptor.opcode = EFA_ADMIN_ALLOC_PD;
+
+	err = efa_com_cmd_exec(aq,
+			       (struct efa_admin_aq_entry *)&cmd,
+			       sizeof(cmd),
+			       (struct efa_admin_acq_entry *)&resp,
+			       sizeof(resp));
+	if (err) {
+		ibdev_err(edev->efa_dev, "Failed to allocate pd[%d]\n", err);
+		return err;
+	}
+
+	result->pdn = resp.pd;
+
+	return 0;
+}
+
+int efa_com_dealloc_pd(struct efa_com_dev *edev,
+		       struct efa_com_dealloc_pd_params *params)
+{
+	struct efa_com_admin_queue *aq = &edev->aq;
+	struct efa_admin_dealloc_pd_cmd cmd = {};
+	struct efa_admin_dealloc_pd_resp resp;
+	int err;
+
+	cmd.aq_common_descriptor.opcode = EFA_ADMIN_DEALLOC_PD;
+	cmd.pd = params->pdn;
+
+	err = efa_com_cmd_exec(aq,
+			       (struct efa_admin_aq_entry *)&cmd,
+			       sizeof(cmd),
+			       (struct efa_admin_acq_entry *)&resp,
+			       sizeof(resp));
+	if (err) {
+		ibdev_err(edev->efa_dev, "Failed to deallocate pd-%u [%d]\n",
+			  cmd.pd, err);
+		return err;
+	}
+
+	return 0;
+}
+
+int efa_com_alloc_uar(struct efa_com_dev *edev,
+		      struct efa_com_alloc_uar_result *result)
+{
+	struct efa_com_admin_queue *aq = &edev->aq;
+	struct efa_admin_alloc_uar_cmd cmd = {};
+	struct efa_admin_alloc_uar_resp resp;
+	int err;
+
+	cmd.aq_common_descriptor.opcode = EFA_ADMIN_ALLOC_UAR;
+
+	err = efa_com_cmd_exec(aq,
+			       (struct efa_admin_aq_entry *)&cmd,
+			       sizeof(cmd),
+			       (struct efa_admin_acq_entry *)&resp,
+			       sizeof(resp));
+	if (err) {
+		ibdev_err(edev->efa_dev, "Failed to allocate uar[%d]\n", err);
+		return err;
+	}
+
+	result->uarn = resp.uar;
+
+	return 0;
+}
+
+int efa_com_dealloc_uar(struct efa_com_dev *edev,
+			struct efa_com_dealloc_uar_params *params)
+{
+	struct efa_com_admin_queue *aq = &edev->aq;
+	struct efa_admin_dealloc_uar_cmd cmd = {};
+	struct efa_admin_dealloc_uar_resp resp;
+	int err;
+
+	cmd.aq_common_descriptor.opcode = EFA_ADMIN_DEALLOC_UAR;
+	cmd.uar = params->uarn;
+
+	err = efa_com_cmd_exec(aq,
+			       (struct efa_admin_aq_entry *)&cmd,
+			       sizeof(cmd),
+			       (struct efa_admin_acq_entry *)&resp,
+			       sizeof(resp));
+	if (err) {
+		ibdev_err(edev->efa_dev, "Failed to deallocate uar-%u [%d]\n",
+			  cmd.uar, err);
+		return err;
+	}
+
+	return 0;
+}
diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.h b/drivers/infiniband/hw/efa/efa_com_cmd.h
new file mode 100644
index 000000000000..a1174380462c
--- /dev/null
+++ b/drivers/infiniband/hw/efa/efa_com_cmd.h
@@ -0,0 +1,270 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
+/*
+ * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
+ */
+
+#ifndef _EFA_COM_CMD_H_
+#define _EFA_COM_CMD_H_
+
+#include "efa_com.h"
+
+#define EFA_GID_SIZE 16
+
+struct efa_com_create_qp_params {
+	u64 rq_base_addr;
+	u32 send_cq_idx;
+	u32 recv_cq_idx;
+	/*
+	 * Send descriptor ring size in bytes,
+	 * sufficient for user-provided number of WQEs and SGL size
+	 */
+	u32 sq_ring_size_in_bytes;
+	/* Max number of WQEs that will be posted on send queue */
+	u32 sq_depth;
+	/* Recv descriptor ring size in bytes */
+	u32 rq_ring_size_in_bytes;
+	u32 rq_depth;
+	u16 pd;
+	u16 uarn;
+	u8 qp_type;
+};
+
+struct efa_com_create_qp_result {
+	u32 qp_handle;
+	u32 qp_num;
+	u32 sq_db_offset;
+	u32 rq_db_offset;
+	u32 llq_descriptors_offset;
+	u16 send_sub_cq_idx;
+	u16 recv_sub_cq_idx;
+};
+
+struct efa_com_modify_qp_params {
+	u32 modify_mask;
+	u32 qp_handle;
+	u32 qp_state;
+	u32 cur_qp_state;
+	u32 qkey;
+	u32 sq_psn;
+	u8 sq_drained_async_notify;
+};
+
+struct efa_com_query_qp_params {
+	u32 qp_handle;
+};
+
+struct efa_com_query_qp_result {
+	u32 qp_state;
+	u32 qkey;
+	u32 sq_draining;
+	u32 sq_psn;
+};
+
+struct efa_com_destroy_qp_params {
+	u32 qp_handle;
+};
+
+struct efa_com_create_cq_params {
+	/* cq physical base address in OS memory */
+	dma_addr_t dma_addr;
+	/* completion queue depth in # of entries */
+	u16 cq_depth;
+	u16 num_sub_cqs;
+	u16 uarn;
+	u8 entry_size_in_bytes;
+};
+
+struct efa_com_create_cq_result {
+	/* cq identifier */
+	u16 cq_idx;
+	/* actual cq depth in # of entries */
+	u16 actual_depth;
+};
+
+struct efa_com_destroy_cq_params {
+	u16 cq_idx;
+};
+
+struct efa_com_create_ah_params {
+	u16 pdn;
+	/* Destination address in network byte order */
+	u8 dest_addr[EFA_GID_SIZE];
+};
+
+struct efa_com_create_ah_result {
+	u16 ah;
+};
+
+struct efa_com_destroy_ah_params {
+	u16 ah;
+	u16 pdn;
+};
+
+struct efa_com_get_network_attr_result {
+	u8 addr[EFA_GID_SIZE];
+	u32 mtu;
+};
+
+struct efa_com_get_device_attr_result {
+	u64 page_size_cap;
+	u64 max_mr_pages;
+	u32 fw_version;
+	u32 admin_api_version;
+	u32 device_version;
+	u32 supported_features;
+	u32 phys_addr_width;
+	u32 virt_addr_width;
+	u32 max_qp;
+	u32 max_sq_depth; /* wqes */
+	u32 max_rq_depth; /* wqes */
+	u32 max_cq;
+	u32 max_cq_depth; /* cqes */
+	u32 inline_buf_size;
+	u32 max_mr;
+	u32 max_pd;
+	u32 max_ah;
+	u32 max_llq_size;
+	u16 sub_cqs_per_cq;
+	u16 max_sq_sge;
+	u16 max_rq_sge;
+	u8 db_bar;
+};
+
+struct efa_com_get_hw_hints_result {
+	u16 mmio_read_timeout;
+	u16 driver_watchdog_timeout;
+	u16 admin_completion_timeout;
+	u16 poll_interval;
+	u32 reserved[4];
+};
+
+struct efa_com_mem_addr {
+	u32 mem_addr_low;
+	u32 mem_addr_high;
+};
+
+/* Used at indirect mode page list chunks for chaining */
+struct efa_com_ctrl_buff_info {
+	/* indicates length of the buffer pointed by control_buffer_address. */
+	u32 length;
+	/* points to control buffer (direct or indirect) */
+	struct efa_com_mem_addr address;
+};
+
+struct efa_com_reg_mr_params {
+	/* Memory region length, in bytes. */
+	u64 mr_length_in_bytes;
+	/* IO Virtual Address associated with this MR. */
+	u64 iova;
+	/* words 8:15: Physical Buffer List, each element is page-aligned. */
+	union {
+		/*
+		 * Inline array of physical addresses of app pages
+		 * (optimization for short region reservations)
+		 */
+		u64 inline_pbl_array[4];
+		/*
+		 * Describes the next physically contiguous chunk of indirect
+		 * page list. A page list contains physical addresses of command
+		 * data pages. Data pages are 4KB; page list chunks are
+		 * variable-sized.
+		 */
+		struct efa_com_ctrl_buff_info pbl;
+	} pbl;
+	/* number of pages in PBL (redundant, could be calculated) */
+	u32 page_num;
+	/* Protection Domain */
+	u16 pd;
+	/*
+	 * phys_page_size_shift - page size is (1 << phys_page_size_shift)
+	 * Page size is used for building the Virtual to Physical
+	 * address mapping
+	 */
+	u8 page_shift;
+	/*
+	 * permissions
+	 * 0: local_write_enable - Write permissions: value of 1 needed
+	 * for RQ buffers and for RDMA write:1: reserved1 - remote
+	 * access flags, etc
+	 */
+	u8 permissions;
+	u8 inline_pbl;
+	u8 indirect;
+};
+
+struct efa_com_reg_mr_result {
+	/*
+	 * To be used in conjunction with local buffers references in SQ and
+	 * RQ WQE
+	 */
+	u32 l_key;
+	/*
+	 * To be used in incoming RDMA semantics messages to refer to remotely
+	 * accessed memory region
+	 */
+	u32 r_key;
+};
+
+struct efa_com_dereg_mr_params {
+	u32 l_key;
+};
+
+struct efa_com_alloc_pd_result {
+	u16 pdn;
+};
+
+struct efa_com_dealloc_pd_params {
+	u16 pdn;
+};
+
+struct efa_com_alloc_uar_result {
+	u16 uarn;
+};
+
+struct efa_com_dealloc_uar_params {
+	u16 uarn;
+};
+
+void efa_com_set_dma_addr(dma_addr_t addr, u32 *addr_high, u32 *addr_low);
+int efa_com_create_qp(struct efa_com_dev *edev,
+		      struct efa_com_create_qp_params *params,
+		      struct efa_com_create_qp_result *res);
+int efa_com_modify_qp(struct efa_com_dev *edev,
+		      struct efa_com_modify_qp_params *params);
+int efa_com_query_qp(struct efa_com_dev *edev,
+		     struct efa_com_query_qp_params *params,
+		     struct efa_com_query_qp_result *result);
+int efa_com_destroy_qp(struct efa_com_dev *edev,
+		       struct efa_com_destroy_qp_params *params);
+int efa_com_create_cq(struct efa_com_dev *edev,
+		      struct efa_com_create_cq_params *params,
+		      struct efa_com_create_cq_result *result);
+int efa_com_destroy_cq(struct efa_com_dev *edev,
+		       struct efa_com_destroy_cq_params *params);
+int efa_com_register_mr(struct efa_com_dev *edev,
+			struct efa_com_reg_mr_params *params,
+			struct efa_com_reg_mr_result *result);
+int efa_com_dereg_mr(struct efa_com_dev *edev,
+		     struct efa_com_dereg_mr_params *params);
+int efa_com_create_ah(struct efa_com_dev *edev,
+		      struct efa_com_create_ah_params *params,
+		      struct efa_com_create_ah_result *result);
+int efa_com_destroy_ah(struct efa_com_dev *edev,
+		       struct efa_com_destroy_ah_params *params);
+int efa_com_get_network_attr(struct efa_com_dev *edev,
+			     struct efa_com_get_network_attr_result *result);
+int efa_com_get_device_attr(struct efa_com_dev *edev,
+			    struct efa_com_get_device_attr_result *result);
+int efa_com_get_hw_hints(struct efa_com_dev *edev,
+			 struct efa_com_get_hw_hints_result *result);
+int efa_com_set_aenq_config(struct efa_com_dev *edev, u32 groups);
+int efa_com_alloc_pd(struct efa_com_dev *edev,
+		     struct efa_com_alloc_pd_result *result);
+int efa_com_dealloc_pd(struct efa_com_dev *edev,
+		       struct efa_com_dealloc_pd_params *params);
+int efa_com_alloc_uar(struct efa_com_dev *edev,
+		      struct efa_com_alloc_uar_result *result);
+int efa_com_dealloc_uar(struct efa_com_dev *edev,
+			struct efa_com_dealloc_uar_params *params);
+
+#endif /* _EFA_COM_CMD_H_ */
diff --git a/drivers/infiniband/hw/efa/efa_common_defs.h b/drivers/infiniband/hw/efa/efa_common_defs.h
new file mode 100644
index 000000000000..c559ec08898e
--- /dev/null
+++ b/drivers/infiniband/hw/efa/efa_common_defs.h
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
+/*
+ * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
+ */
+
+#ifndef _EFA_COMMON_H_
+#define _EFA_COMMON_H_
+
+#define EFA_COMMON_SPEC_VERSION_MAJOR        2
+#define EFA_COMMON_SPEC_VERSION_MINOR        0
+
+struct efa_common_mem_addr {
+	u32 mem_addr_low;
+
+	u32 mem_addr_high;
+};
+
+#endif /* _EFA_COMMON_H_ */
diff --git a/drivers/infiniband/hw/efa/efa_main.c b/drivers/infiniband/hw/efa/efa_main.c
new file mode 100644
index 000000000000..db974caf1eb1
--- /dev/null
+++ b/drivers/infiniband/hw/efa/efa_main.c
@@ -0,0 +1,533 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
+/*
+ * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
+ */
+
+#include <linux/module.h>
+#include <linux/pci.h>
+
+#include <rdma/ib_user_verbs.h>
+
+#include "efa.h"
+
+#define PCI_DEV_ID_EFA_VF 0xefa0
+
+static const struct pci_device_id efa_pci_tbl[] = {
+	{ PCI_VDEVICE(AMAZON, PCI_DEV_ID_EFA_VF) },
+	{ }
+};
+
+MODULE_AUTHOR("Amazon.com, Inc. or its affiliates");
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_DESCRIPTION(DEVICE_NAME);
+MODULE_DEVICE_TABLE(pci, efa_pci_tbl);
+
+#define EFA_REG_BAR 0
+#define EFA_MEM_BAR 2
+#define EFA_BASE_BAR_MASK (BIT(EFA_REG_BAR) | BIT(EFA_MEM_BAR))
+
+#define EFA_AENQ_ENABLED_GROUPS \
+	(BIT(EFA_ADMIN_FATAL_ERROR) | BIT(EFA_ADMIN_WARNING) | \
+	 BIT(EFA_ADMIN_NOTIFICATION) | BIT(EFA_ADMIN_KEEP_ALIVE))
+
+static void efa_update_network_attr(struct efa_dev *dev,
+				    struct efa_com_get_network_attr_result *network_attr)
+{
+	memcpy(dev->addr, network_attr->addr, sizeof(network_attr->addr));
+	dev->mtu = network_attr->mtu;
+
+	dev_dbg(&dev->pdev->dev, "Full address %pI6\n", dev->addr);
+}
+
+/* This handler will called for unknown event group or unimplemented handlers */
+static void unimplemented_aenq_handler(void *data,
+				       struct efa_admin_aenq_entry *aenq_e)
+{
+	struct efa_dev *dev = (struct efa_dev *)data;
+
+	ibdev_err(&dev->ibdev,
+		  "Unknown event was received or event with unimplemented handler\n");
+}
+
+static void efa_keep_alive(void *data, struct efa_admin_aenq_entry *aenq_e)
+{
+	struct efa_dev *dev = (struct efa_dev *)data;
+
+	atomic64_inc(&dev->stats.keep_alive_rcvd);
+}
+
+static struct efa_aenq_handlers aenq_handlers = {
+	.handlers = {
+		[EFA_ADMIN_KEEP_ALIVE] = efa_keep_alive,
+	},
+	.unimplemented_handler = unimplemented_aenq_handler
+};
+
+static void efa_release_bars(struct efa_dev *dev, int bars_mask)
+{
+	struct pci_dev *pdev = dev->pdev;
+	int release_bars;
+
+	release_bars = pci_select_bars(pdev, IORESOURCE_MEM) & bars_mask;
+	pci_release_selected_regions(pdev, release_bars);
+}
+
+static irqreturn_t efa_intr_msix_mgmnt(int irq, void *data)
+{
+	struct efa_dev *dev = data;
+
+	efa_com_admin_q_comp_intr_handler(&dev->edev);
+	efa_com_aenq_intr_handler(&dev->edev, data);
+
+	return IRQ_HANDLED;
+}
+
+static int efa_request_mgmnt_irq(struct efa_dev *dev)
+{
+	struct efa_irq *irq;
+	int err;
+
+	irq = &dev->admin_irq;
+	err = request_irq(irq->vector, irq->handler, 0, irq->name,
+			  irq->data);
+	if (err) {
+		dev_err(&dev->pdev->dev, "Failed to request admin irq (%d)\n",
+			err);
+		return err;
+	}
+
+	dev_dbg(&dev->pdev->dev, "Set affinity hint of mgmnt irq to %*pbl (irq vector: %d)\n",
+		nr_cpumask_bits, &irq->affinity_hint_mask, irq->vector);
+	irq_set_affinity_hint(irq->vector, &irq->affinity_hint_mask);
+
+	return err;
+}
+
+static void efa_setup_mgmnt_irq(struct efa_dev *dev)
+{
+	u32 cpu;
+
+	snprintf(dev->admin_irq.name, EFA_IRQNAME_SIZE,
+		 "efa-mgmnt@pci:%s", pci_name(dev->pdev));
+	dev->admin_irq.handler = efa_intr_msix_mgmnt;
+	dev->admin_irq.data = dev;
+	dev->admin_irq.vector =
+		pci_irq_vector(dev->pdev, dev->admin_msix_vector_idx);
+	cpu = cpumask_first(cpu_online_mask);
+	dev->admin_irq.cpu = cpu;
+	cpumask_set_cpu(cpu,
+			&dev->admin_irq.affinity_hint_mask);
+	dev_info(&dev->pdev->dev, "Setup irq:0x%p vector:%d name:%s\n",
+		 &dev->admin_irq,
+		 dev->admin_irq.vector,
+		 dev->admin_irq.name);
+}
+
+static void efa_free_mgmnt_irq(struct efa_dev *dev)
+{
+	struct efa_irq *irq;
+
+	irq = &dev->admin_irq;
+	irq_set_affinity_hint(irq->vector, NULL);
+	free_irq(irq->vector, irq->data);
+}
+
+static int efa_set_mgmnt_irq(struct efa_dev *dev)
+{
+	efa_setup_mgmnt_irq(dev);
+
+	return efa_request_mgmnt_irq(dev);
+}
+
+static int efa_request_doorbell_bar(struct efa_dev *dev)
+{
+	u8 db_bar_idx = dev->dev_attr.db_bar;
+	struct pci_dev *pdev = dev->pdev;
+	int bars;
+	int err;
+
+	if (!(BIT(db_bar_idx) & EFA_BASE_BAR_MASK)) {
+		bars = pci_select_bars(pdev, IORESOURCE_MEM) & BIT(db_bar_idx);
+
+		err = pci_request_selected_regions(pdev, bars, DRV_MODULE_NAME);
+		if (err) {
+			dev_err(&dev->pdev->dev,
+				"pci_request_selected_regions for bar %d failed %d\n",
+				db_bar_idx, err);
+			return err;
+		}
+	}
+
+	dev->db_bar_addr = pci_resource_start(dev->pdev, db_bar_idx);
+	dev->db_bar_len = pci_resource_len(dev->pdev, db_bar_idx);
+
+	return 0;
+}
+
+static void efa_release_doorbell_bar(struct efa_dev *dev)
+{
+	if (!(BIT(dev->dev_attr.db_bar) & EFA_BASE_BAR_MASK))
+		efa_release_bars(dev, BIT(dev->dev_attr.db_bar));
+}
+
+static void efa_update_hw_hints(struct efa_dev *dev,
+				struct efa_com_get_hw_hints_result *hw_hints)
+{
+	struct efa_com_dev *edev = &dev->edev;
+
+	if (hw_hints->mmio_read_timeout)
+		edev->mmio_read.mmio_read_timeout =
+			hw_hints->mmio_read_timeout * 1000;
+
+	if (hw_hints->poll_interval)
+		edev->aq.poll_interval = hw_hints->poll_interval;
+
+	if (hw_hints->admin_completion_timeout)
+		edev->aq.completion_timeout =
+			hw_hints->admin_completion_timeout;
+}
+
+static void efa_stats_init(struct efa_dev *dev)
+{
+	atomic64_t *s = (atomic64_t *)&dev->stats;
+	int i;
+
+	for (i = 0; i < sizeof(dev->stats) / sizeof(*s); i++, s++)
+		atomic64_set(s, 0);
+}
+
+static const struct ib_device_ops efa_dev_ops = {
+	.alloc_pd = efa_alloc_pd,
+	.alloc_ucontext = efa_alloc_ucontext,
+	.create_ah = efa_create_ah,
+	.create_cq = efa_create_cq,
+	.create_qp = efa_create_qp,
+	.dealloc_pd = efa_dealloc_pd,
+	.dealloc_ucontext = efa_dealloc_ucontext,
+	.dereg_mr = efa_dereg_mr,
+	.destroy_ah = efa_destroy_ah,
+	.destroy_cq = efa_destroy_cq,
+	.destroy_qp = efa_destroy_qp,
+	.get_link_layer = efa_port_link_layer,
+	.get_port_immutable = efa_get_port_immutable,
+	.mmap = efa_mmap,
+	.modify_qp = efa_modify_qp,
+	.query_device = efa_query_device,
+	.query_gid = efa_query_gid,
+	.query_pkey = efa_query_pkey,
+	.query_port = efa_query_port,
+	.query_qp = efa_query_qp,
+	.reg_user_mr = efa_reg_mr,
+
+	INIT_RDMA_OBJ_SIZE(ib_ah, efa_ah, ibah),
+	INIT_RDMA_OBJ_SIZE(ib_pd, efa_pd, ibpd),
+	INIT_RDMA_OBJ_SIZE(ib_ucontext, efa_ucontext, ibucontext),
+};
+
+static int efa_ib_device_add(struct efa_dev *dev)
+{
+	struct efa_com_get_network_attr_result network_attr;
+	struct efa_com_get_hw_hints_result hw_hints;
+	struct pci_dev *pdev = dev->pdev;
+	int err;
+
+	efa_stats_init(dev);
+
+	err = efa_com_get_device_attr(&dev->edev, &dev->dev_attr);
+	if (err)
+		return err;
+
+	dev_dbg(&dev->pdev->dev, "Doorbells bar (%d)\n", dev->dev_attr.db_bar);
+	err = efa_request_doorbell_bar(dev);
+	if (err)
+		return err;
+
+	err = efa_com_get_network_attr(&dev->edev, &network_attr);
+	if (err)
+		goto err_release_doorbell_bar;
+
+	efa_update_network_attr(dev, &network_attr);
+
+	err = efa_com_get_hw_hints(&dev->edev, &hw_hints);
+	if (err)
+		goto err_release_doorbell_bar;
+
+	efa_update_hw_hints(dev, &hw_hints);
+
+	/* Try to enable all the available aenq groups */
+	err = efa_com_set_aenq_config(&dev->edev, EFA_AENQ_ENABLED_GROUPS);
+	if (err)
+		goto err_release_doorbell_bar;
+
+	dev->ibdev.owner = THIS_MODULE;
+	dev->ibdev.node_type = RDMA_NODE_UNSPECIFIED;
+	dev->ibdev.phys_port_cnt = 1;
+	dev->ibdev.num_comp_vectors = 1;
+	dev->ibdev.dev.parent = &pdev->dev;
+	dev->ibdev.uverbs_abi_ver = EFA_UVERBS_ABI_VERSION;
+
+	dev->ibdev.uverbs_cmd_mask =
+		(1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
+		(1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
+		(1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
+		(1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
+		(1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
+		(1ull << IB_USER_VERBS_CMD_REG_MR) |
+		(1ull << IB_USER_VERBS_CMD_DEREG_MR) |
+		(1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
+		(1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
+		(1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
+		(1ull << IB_USER_VERBS_CMD_CREATE_QP) |
+		(1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
+		(1ull << IB_USER_VERBS_CMD_QUERY_QP) |
+		(1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
+		(1ull << IB_USER_VERBS_CMD_CREATE_AH) |
+		(1ull << IB_USER_VERBS_CMD_DESTROY_AH);
+
+	dev->ibdev.uverbs_ex_cmd_mask =
+		(1ull << IB_USER_VERBS_EX_CMD_QUERY_DEVICE);
+
+	dev->ibdev.driver_id = RDMA_DRIVER_EFA;
+	ib_set_device_ops(&dev->ibdev, &efa_dev_ops);
+
+	err = ib_register_device(&dev->ibdev, "efa_%d");
+	if (err)
+		goto err_release_doorbell_bar;
+
+	ibdev_info(&dev->ibdev, "IB device registered\n");
+
+	return 0;
+
+err_release_doorbell_bar:
+	efa_release_doorbell_bar(dev);
+	return err;
+}
+
+static void efa_ib_device_remove(struct efa_dev *dev)
+{
+	efa_com_dev_reset(&dev->edev, EFA_REGS_RESET_NORMAL);
+	ibdev_info(&dev->ibdev, "Unregister ib device\n");
+	ib_unregister_device(&dev->ibdev);
+	efa_release_doorbell_bar(dev);
+}
+
+static void efa_disable_msix(struct efa_dev *dev)
+{
+	pci_free_irq_vectors(dev->pdev);
+}
+
+static int efa_enable_msix(struct efa_dev *dev)
+{
+	int msix_vecs, irq_num;
+
+	/* Reserve the max msix vectors we might need */
+	msix_vecs = EFA_NUM_MSIX_VEC;
+	dev_dbg(&dev->pdev->dev, "Trying to enable MSI-X, vectors %d\n",
+		msix_vecs);
+
+	dev->admin_msix_vector_idx = EFA_MGMNT_MSIX_VEC_IDX;
+	irq_num = pci_alloc_irq_vectors(dev->pdev, msix_vecs,
+					msix_vecs, PCI_IRQ_MSIX);
+
+	if (irq_num < 0) {
+		dev_err(&dev->pdev->dev, "Failed to enable MSI-X. irq_num %d\n",
+			irq_num);
+		return -ENOSPC;
+	}
+
+	if (irq_num != msix_vecs) {
+		dev_err(&dev->pdev->dev,
+			"Allocated %d MSI-X (out of %d requested)\n",
+			irq_num, msix_vecs);
+		return -ENOSPC;
+	}
+
+	return 0;
+}
+
+static int efa_device_init(struct efa_com_dev *edev, struct pci_dev *pdev)
+{
+	int dma_width;
+	int err;
+
+	err = efa_com_dev_reset(edev, EFA_REGS_RESET_NORMAL);
+	if (err)
+		return err;
+
+	err = efa_com_validate_version(edev);
+	if (err)
+		return err;
+
+	dma_width = efa_com_get_dma_width(edev);
+	if (dma_width < 0) {
+		err = dma_width;
+		return err;
+	}
+
+	err = pci_set_dma_mask(pdev, DMA_BIT_MASK(dma_width));
+	if (err) {
+		dev_err(&pdev->dev, "pci_set_dma_mask failed %d\n", err);
+		return err;
+	}
+
+	err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(dma_width));
+	if (err) {
+		dev_err(&pdev->dev,
+			"err_pci_set_consistent_dma_mask failed %d\n",
+			err);
+		return err;
+	}
+
+	return 0;
+}
+
+static struct efa_dev *efa_probe_device(struct pci_dev *pdev)
+{
+	struct efa_com_dev *edev;
+	struct efa_dev *dev;
+	int bars;
+	int err;
+
+	err = pci_enable_device_mem(pdev);
+	if (err) {
+		dev_err(&pdev->dev, "pci_enable_device_mem() failed!\n");
+		return ERR_PTR(err);
+	}
+
+	pci_set_master(pdev);
+
+	dev = ib_alloc_device(efa_dev, ibdev);
+	if (!dev) {
+		dev_err(&pdev->dev, "Device alloc failed\n");
+		err = -ENOMEM;
+		goto err_disable_device;
+	}
+
+	pci_set_drvdata(pdev, dev);
+	edev = &dev->edev;
+	edev->efa_dev = dev;
+	edev->dmadev = &pdev->dev;
+	dev->pdev = pdev;
+
+	bars = pci_select_bars(pdev, IORESOURCE_MEM) & EFA_BASE_BAR_MASK;
+	err = pci_request_selected_regions(pdev, bars, DRV_MODULE_NAME);
+	if (err) {
+		dev_err(&pdev->dev, "pci_request_selected_regions failed %d\n",
+			err);
+		goto err_ibdev_destroy;
+	}
+
+	dev->reg_bar_addr = pci_resource_start(pdev, EFA_REG_BAR);
+	dev->reg_bar_len = pci_resource_len(pdev, EFA_REG_BAR);
+	dev->mem_bar_addr = pci_resource_start(pdev, EFA_MEM_BAR);
+	dev->mem_bar_len = pci_resource_len(pdev, EFA_MEM_BAR);
+
+	edev->reg_bar = devm_ioremap(&pdev->dev,
+				     dev->reg_bar_addr,
+				     dev->reg_bar_len);
+	if (!edev->reg_bar) {
+		dev_err(&pdev->dev, "Failed to remap register bar\n");
+		err = -EFAULT;
+		goto err_release_bars;
+	}
+
+	err = efa_com_mmio_reg_read_init(edev);
+	if (err) {
+		dev_err(&pdev->dev, "Failed to init readless MMIO\n");
+		goto err_iounmap;
+	}
+
+	err = efa_device_init(edev, pdev);
+	if (err) {
+		dev_err(&pdev->dev, "EFA device init failed\n");
+		if (err == -ETIME)
+			err = -EPROBE_DEFER;
+		goto err_reg_read_destroy;
+	}
+
+	err = efa_enable_msix(dev);
+	if (err)
+		goto err_reg_read_destroy;
+
+	edev->aq.msix_vector_idx = dev->admin_msix_vector_idx;
+	edev->aenq.msix_vector_idx = dev->admin_msix_vector_idx;
+
+	err = efa_set_mgmnt_irq(dev);
+	if (err)
+		goto err_disable_msix;
+
+	err = efa_com_admin_init(edev, &aenq_handlers);
+	if (err)
+		goto err_free_mgmnt_irq;
+
+	return dev;
+
+err_free_mgmnt_irq:
+	efa_free_mgmnt_irq(dev);
+err_disable_msix:
+	efa_disable_msix(dev);
+err_reg_read_destroy:
+	efa_com_mmio_reg_read_destroy(edev);
+err_iounmap:
+	devm_iounmap(&pdev->dev, edev->reg_bar);
+err_release_bars:
+	efa_release_bars(dev, EFA_BASE_BAR_MASK);
+err_ibdev_destroy:
+	ib_dealloc_device(&dev->ibdev);
+err_disable_device:
+	pci_disable_device(pdev);
+	return ERR_PTR(err);
+}
+
+static void efa_remove_device(struct pci_dev *pdev)
+{
+	struct efa_dev *dev = pci_get_drvdata(pdev);
+	struct efa_com_dev *edev;
+
+	edev = &dev->edev;
+	efa_com_admin_destroy(edev);
+	efa_free_mgmnt_irq(dev);
+	efa_disable_msix(dev);
+	efa_com_mmio_reg_read_destroy(edev);
+	devm_iounmap(&pdev->dev, edev->reg_bar);
+	efa_release_bars(dev, EFA_BASE_BAR_MASK);
+	ib_dealloc_device(&dev->ibdev);
+	pci_disable_device(pdev);
+}
+
+static int efa_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
+{
+	struct efa_dev *dev;
+	int err;
+
+	dev = efa_probe_device(pdev);
+	if (IS_ERR(dev))
+		return PTR_ERR(dev);
+
+	err = efa_ib_device_add(dev);
+	if (err)
+		goto err_remove_device;
+
+	return 0;
+
+err_remove_device:
+	efa_remove_device(pdev);
+	return err;
+}
+
+static void efa_remove(struct pci_dev *pdev)
+{
+	struct efa_dev *dev = pci_get_drvdata(pdev);
+
+	efa_ib_device_remove(dev);
+	efa_remove_device(pdev);
+}
+
+static struct pci_driver efa_pci_driver = {
+	.name           = DRV_MODULE_NAME,
+	.id_table       = efa_pci_tbl,
+	.probe          = efa_probe,
+	.remove         = efa_remove,
+};
+
+module_pci_driver(efa_pci_driver);
diff --git a/drivers/infiniband/hw/efa/efa_regs_defs.h b/drivers/infiniband/hw/efa/efa_regs_defs.h
new file mode 100644
index 000000000000..bb9cad3d6a15
--- /dev/null
+++ b/drivers/infiniband/hw/efa/efa_regs_defs.h
@@ -0,0 +1,113 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
+/*
+ * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
+ */
+
+#ifndef _EFA_REGS_H_
+#define _EFA_REGS_H_
+
+enum efa_regs_reset_reason_types {
+	EFA_REGS_RESET_NORMAL                       = 0,
+	/* Keep alive timeout */
+	EFA_REGS_RESET_KEEP_ALIVE_TO                = 1,
+	EFA_REGS_RESET_ADMIN_TO                     = 2,
+	EFA_REGS_RESET_INIT_ERR                     = 3,
+	EFA_REGS_RESET_DRIVER_INVALID_STATE         = 4,
+	EFA_REGS_RESET_OS_TRIGGER                   = 5,
+	EFA_REGS_RESET_SHUTDOWN                     = 6,
+	EFA_REGS_RESET_USER_TRIGGER                 = 7,
+	EFA_REGS_RESET_GENERIC                      = 8,
+};
+
+/* efa_registers offsets */
+
+/* 0 base */
+#define EFA_REGS_VERSION_OFF                                0x0
+#define EFA_REGS_CONTROLLER_VERSION_OFF                     0x4
+#define EFA_REGS_CAPS_OFF                                   0x8
+#define EFA_REGS_AQ_BASE_LO_OFF                             0x10
+#define EFA_REGS_AQ_BASE_HI_OFF                             0x14
+#define EFA_REGS_AQ_CAPS_OFF                                0x18
+#define EFA_REGS_ACQ_BASE_LO_OFF                            0x20
+#define EFA_REGS_ACQ_BASE_HI_OFF                            0x24
+#define EFA_REGS_ACQ_CAPS_OFF                               0x28
+#define EFA_REGS_AQ_PROD_DB_OFF                             0x2c
+#define EFA_REGS_AENQ_CAPS_OFF                              0x34
+#define EFA_REGS_AENQ_BASE_LO_OFF                           0x38
+#define EFA_REGS_AENQ_BASE_HI_OFF                           0x3c
+#define EFA_REGS_AENQ_CONS_DB_OFF                           0x40
+#define EFA_REGS_INTR_MASK_OFF                              0x4c
+#define EFA_REGS_DEV_CTL_OFF                                0x54
+#define EFA_REGS_DEV_STS_OFF                                0x58
+#define EFA_REGS_MMIO_REG_READ_OFF                          0x5c
+#define EFA_REGS_MMIO_RESP_LO_OFF                           0x60
+#define EFA_REGS_MMIO_RESP_HI_OFF                           0x64
+
+/* version register */
+#define EFA_REGS_VERSION_MINOR_VERSION_MASK                 0xff
+#define EFA_REGS_VERSION_MAJOR_VERSION_SHIFT                8
+#define EFA_REGS_VERSION_MAJOR_VERSION_MASK                 0xff00
+
+/* controller_version register */
+#define EFA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION_MASK   0xff
+#define EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION_SHIFT     8
+#define EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION_MASK      0xff00
+#define EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_SHIFT     16
+#define EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_MASK      0xff0000
+#define EFA_REGS_CONTROLLER_VERSION_IMPL_ID_SHIFT           24
+#define EFA_REGS_CONTROLLER_VERSION_IMPL_ID_MASK            0xff000000
+
+/* caps register */
+#define EFA_REGS_CAPS_CONTIGUOUS_QUEUE_REQUIRED_MASK        0x1
+#define EFA_REGS_CAPS_RESET_TIMEOUT_SHIFT                   1
+#define EFA_REGS_CAPS_RESET_TIMEOUT_MASK                    0x3e
+#define EFA_REGS_CAPS_DMA_ADDR_WIDTH_SHIFT                  8
+#define EFA_REGS_CAPS_DMA_ADDR_WIDTH_MASK                   0xff00
+#define EFA_REGS_CAPS_ADMIN_CMD_TO_SHIFT                    16
+#define EFA_REGS_CAPS_ADMIN_CMD_TO_MASK                     0xf0000
+
+/* aq_caps register */
+#define EFA_REGS_AQ_CAPS_AQ_DEPTH_MASK                      0xffff
+#define EFA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_SHIFT                16
+#define EFA_REGS_AQ_CAPS_AQ_ENTRY_SIZE_MASK                 0xffff0000
+
+/* acq_caps register */
+#define EFA_REGS_ACQ_CAPS_ACQ_DEPTH_MASK                    0xffff
+#define EFA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_SHIFT              16
+#define EFA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE_MASK               0xff0000
+#define EFA_REGS_ACQ_CAPS_ACQ_MSIX_VECTOR_SHIFT             24
+#define EFA_REGS_ACQ_CAPS_ACQ_MSIX_VECTOR_MASK              0xff000000
+
+/* aenq_caps register */
+#define EFA_REGS_AENQ_CAPS_AENQ_DEPTH_MASK                  0xffff
+#define EFA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_SHIFT            16
+#define EFA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_MASK             0xff0000
+#define EFA_REGS_AENQ_CAPS_AENQ_MSIX_VECTOR_SHIFT           24
+#define EFA_REGS_AENQ_CAPS_AENQ_MSIX_VECTOR_MASK            0xff000000
+
+/* dev_ctl register */
+#define EFA_REGS_DEV_CTL_DEV_RESET_MASK                     0x1
+#define EFA_REGS_DEV_CTL_AQ_RESTART_SHIFT                   1
+#define EFA_REGS_DEV_CTL_AQ_RESTART_MASK                    0x2
+#define EFA_REGS_DEV_CTL_RESET_REASON_SHIFT                 28
+#define EFA_REGS_DEV_CTL_RESET_REASON_MASK                  0xf0000000
+
+/* dev_sts register */
+#define EFA_REGS_DEV_STS_READY_MASK                         0x1
+#define EFA_REGS_DEV_STS_AQ_RESTART_IN_PROGRESS_SHIFT       1
+#define EFA_REGS_DEV_STS_AQ_RESTART_IN_PROGRESS_MASK        0x2
+#define EFA_REGS_DEV_STS_AQ_RESTART_FINISHED_SHIFT          2
+#define EFA_REGS_DEV_STS_AQ_RESTART_FINISHED_MASK           0x4
+#define EFA_REGS_DEV_STS_RESET_IN_PROGRESS_SHIFT            3
+#define EFA_REGS_DEV_STS_RESET_IN_PROGRESS_MASK             0x8
+#define EFA_REGS_DEV_STS_RESET_FINISHED_SHIFT               4
+#define EFA_REGS_DEV_STS_RESET_FINISHED_MASK                0x10
+#define EFA_REGS_DEV_STS_FATAL_ERROR_SHIFT                  5
+#define EFA_REGS_DEV_STS_FATAL_ERROR_MASK                   0x20
+
+/* mmio_reg_read register */
+#define EFA_REGS_MMIO_REG_READ_REQ_ID_MASK                  0xffff
+#define EFA_REGS_MMIO_REG_READ_REG_OFF_SHIFT                16
+#define EFA_REGS_MMIO_REG_READ_REG_OFF_MASK                 0xffff0000
+
+#endif /* _EFA_REGS_H_ */
diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c
new file mode 100644
index 000000000000..6d6886c9009f
--- /dev/null
+++ b/drivers/infiniband/hw/efa/efa_verbs.c
@@ -0,0 +1,1825 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/*
+ * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
+ */
+
+#include <linux/vmalloc.h>
+
+#include <rdma/ib_addr.h>
+#include <rdma/ib_umem.h>
+#include <rdma/ib_user_verbs.h>
+#include <rdma/ib_verbs.h>
+#include <rdma/uverbs_ioctl.h>
+
+#include "efa.h"
+
+#define EFA_MMAP_FLAG_SHIFT 56
+#define EFA_MMAP_PAGE_MASK GENMASK(EFA_MMAP_FLAG_SHIFT - 1, 0)
+#define EFA_MMAP_INVALID U64_MAX
+
+enum {
+	EFA_MMAP_DMA_PAGE = 0,
+	EFA_MMAP_IO_WC,
+	EFA_MMAP_IO_NC,
+};
+
+#define EFA_AENQ_ENABLED_GROUPS \
+	(BIT(EFA_ADMIN_FATAL_ERROR) | BIT(EFA_ADMIN_WARNING) | \
+	 BIT(EFA_ADMIN_NOTIFICATION) | BIT(EFA_ADMIN_KEEP_ALIVE))
+
+struct efa_mmap_entry {
+	void  *obj;
+	u64 address;
+	u64 length;
+	u32 mmap_page;
+	u8 mmap_flag;
+};
+
+static inline u64 get_mmap_key(const struct efa_mmap_entry *efa)
+{
+	return ((u64)efa->mmap_flag << EFA_MMAP_FLAG_SHIFT) |
+	       ((u64)efa->mmap_page << PAGE_SHIFT);
+}
+
+#define EFA_CHUNK_PAYLOAD_SHIFT       12
+#define EFA_CHUNK_PAYLOAD_SIZE        BIT(EFA_CHUNK_PAYLOAD_SHIFT)
+#define EFA_CHUNK_PAYLOAD_PTR_SIZE    8
+
+#define EFA_CHUNK_SHIFT               12
+#define EFA_CHUNK_SIZE                BIT(EFA_CHUNK_SHIFT)
+#define EFA_CHUNK_PTR_SIZE            sizeof(struct efa_com_ctrl_buff_info)
+
+#define EFA_PTRS_PER_CHUNK \
+	((EFA_CHUNK_SIZE - EFA_CHUNK_PTR_SIZE) / EFA_CHUNK_PAYLOAD_PTR_SIZE)
+
+#define EFA_CHUNK_USED_SIZE \
+	((EFA_PTRS_PER_CHUNK * EFA_CHUNK_PAYLOAD_PTR_SIZE) + EFA_CHUNK_PTR_SIZE)
+
+#define EFA_SUPPORTED_ACCESS_FLAGS IB_ACCESS_LOCAL_WRITE
+
+struct pbl_chunk {
+	dma_addr_t dma_addr;
+	u64 *buf;
+	u32 length;
+};
+
+struct pbl_chunk_list {
+	struct pbl_chunk *chunks;
+	unsigned int size;
+};
+
+struct pbl_context {
+	union {
+		struct {
+			dma_addr_t dma_addr;
+		} continuous;
+		struct {
+			u32 pbl_buf_size_in_pages;
+			struct scatterlist *sgl;
+			int sg_dma_cnt;
+			struct pbl_chunk_list chunk_list;
+		} indirect;
+	} phys;
+	u64 *pbl_buf;
+	u32 pbl_buf_size_in_bytes;
+	u8 physically_continuous;
+};
+
+static inline struct efa_dev *to_edev(struct ib_device *ibdev)
+{
+	return container_of(ibdev, struct efa_dev, ibdev);
+}
+
+static inline struct efa_ucontext *to_eucontext(struct ib_ucontext *ibucontext)
+{
+	return container_of(ibucontext, struct efa_ucontext, ibucontext);
+}
+
+static inline struct efa_pd *to_epd(struct ib_pd *ibpd)
+{
+	return container_of(ibpd, struct efa_pd, ibpd);
+}
+
+static inline struct efa_mr *to_emr(struct ib_mr *ibmr)
+{
+	return container_of(ibmr, struct efa_mr, ibmr);
+}
+
+static inline struct efa_qp *to_eqp(struct ib_qp *ibqp)
+{
+	return container_of(ibqp, struct efa_qp, ibqp);
+}
+
+static inline struct efa_cq *to_ecq(struct ib_cq *ibcq)
+{
+	return container_of(ibcq, struct efa_cq, ibcq);
+}
+
+static inline struct efa_ah *to_eah(struct ib_ah *ibah)
+{
+	return container_of(ibah, struct efa_ah, ibah);
+}
+
+#define field_avail(x, fld, sz) (offsetof(typeof(x), fld) + \
+				 sizeof(((typeof(x) *)0)->fld) <= (sz))
+
+#define is_reserved_cleared(reserved) \
+	!memchr_inv(reserved, 0, sizeof(reserved))
+
+static void *efa_zalloc_mapped(struct efa_dev *dev, dma_addr_t *dma_addr,
+			       size_t size, enum dma_data_direction dir)
+{
+	void *addr;
+
+	addr = alloc_pages_exact(size, GFP_KERNEL | __GFP_ZERO);
+	if (!addr)
+		return NULL;
+
+	*dma_addr = dma_map_single(&dev->pdev->dev, addr, size, dir);
+	if (dma_mapping_error(&dev->pdev->dev, *dma_addr)) {
+		ibdev_err(&dev->ibdev, "Failed to map DMA address\n");
+		free_pages_exact(addr, size);
+		return NULL;
+	}
+
+	return addr;
+}
+
+/*
+ * This is only called when the ucontext is destroyed and there can be no
+ * concurrent query via mmap or allocate on the xarray, thus we can be sure no
+ * other thread is using the entry pointer. We also know that all the BAR
+ * pages have either been zap'd or munmaped at this point.  Normal pages are
+ * refcounted and will be freed at the proper time.
+ */
+static void mmap_entries_remove_free(struct efa_dev *dev,
+				     struct efa_ucontext *ucontext)
+{
+	struct efa_mmap_entry *entry;
+	unsigned long mmap_page;
+
+	xa_for_each(&ucontext->mmap_xa, mmap_page, entry) {
+		xa_erase(&ucontext->mmap_xa, mmap_page);
+
+		ibdev_dbg(
+			&dev->ibdev,
+			"mmap: obj[0x%p] key[%#llx] addr[%#llx] len[%#llx] removed\n",
+			entry->obj, get_mmap_key(entry), entry->address,
+			entry->length);
+		if (entry->mmap_flag == EFA_MMAP_DMA_PAGE)
+			/* DMA mapping is already gone, now free the pages */
+			free_pages_exact(phys_to_virt(entry->address),
+					 entry->length);
+		kfree(entry);
+	}
+}
+
+static struct efa_mmap_entry *mmap_entry_get(struct efa_dev *dev,
+					     struct efa_ucontext *ucontext,
+					     u64 key, u64 len)
+{
+	struct efa_mmap_entry *entry;
+	u64 mmap_page;
+
+	mmap_page = (key & EFA_MMAP_PAGE_MASK) >> PAGE_SHIFT;
+	if (mmap_page > U32_MAX)
+		return NULL;
+
+	entry = xa_load(&ucontext->mmap_xa, mmap_page);
+	if (!entry || get_mmap_key(entry) != key || entry->length != len)
+		return NULL;
+
+	ibdev_dbg(&dev->ibdev,
+		  "mmap: obj[0x%p] key[%#llx] addr[%#llx] len[%#llx] removed\n",
+		  entry->obj, key, entry->address, entry->length);
+
+	return entry;
+}
+
+/*
+ * Note this locking scheme cannot support removal of entries, except during
+ * ucontext destruction when the core code guarentees no concurrency.
+ */
+static u64 mmap_entry_insert(struct efa_dev *dev, struct efa_ucontext *ucontext,
+			     void *obj, u64 address, u64 length, u8 mmap_flag)
+{
+	struct efa_mmap_entry *entry;
+	int err;
+
+	entry = kmalloc(sizeof(*entry), GFP_KERNEL);
+	if (!entry)
+		return EFA_MMAP_INVALID;
+
+	entry->obj = obj;
+	entry->address = address;
+	entry->length = length;
+	entry->mmap_flag = mmap_flag;
+
+	xa_lock(&ucontext->mmap_xa);
+	entry->mmap_page = ucontext->mmap_xa_page;
+	ucontext->mmap_xa_page += DIV_ROUND_UP(length, PAGE_SIZE);
+	err = __xa_insert(&ucontext->mmap_xa, entry->mmap_page, entry,
+			  GFP_KERNEL);
+	xa_unlock(&ucontext->mmap_xa);
+	if (err){
+		kfree(entry);
+		return EFA_MMAP_INVALID;
+	}
+
+	ibdev_dbg(
+		&dev->ibdev,
+		"mmap: obj[0x%p] addr[%#llx], len[%#llx], key[%#llx] inserted\n",
+		entry->obj, entry->address, entry->length, get_mmap_key(entry));
+
+	return get_mmap_key(entry);
+}
+
+int efa_query_device(struct ib_device *ibdev,
+		     struct ib_device_attr *props,
+		     struct ib_udata *udata)
+{
+	struct efa_com_get_device_attr_result *dev_attr;
+	struct efa_ibv_ex_query_device_resp resp = {};
+	struct efa_dev *dev = to_edev(ibdev);
+	int err;
+
+	if (udata && udata->inlen &&
+	    !ib_is_udata_cleared(udata, 0, udata->inlen)) {
+		ibdev_dbg(ibdev,
+			  "Incompatible ABI params, udata not cleared\n");
+		return -EINVAL;
+	}
+
+	dev_attr = &dev->dev_attr;
+
+	memset(props, 0, sizeof(*props));
+	props->max_mr_size = dev_attr->max_mr_pages * PAGE_SIZE;
+	props->page_size_cap = dev_attr->page_size_cap;
+	props->vendor_id = dev->pdev->vendor;
+	props->vendor_part_id = dev->pdev->device;
+	props->hw_ver = dev->pdev->subsystem_device;
+	props->max_qp = dev_attr->max_qp;
+	props->max_cq = dev_attr->max_cq;
+	props->max_pd = dev_attr->max_pd;
+	props->max_mr = dev_attr->max_mr;
+	props->max_ah = dev_attr->max_ah;
+	props->max_cqe = dev_attr->max_cq_depth;
+	props->max_qp_wr = min_t(u32, dev_attr->max_sq_depth,
+				 dev_attr->max_rq_depth);
+	props->max_send_sge = dev_attr->max_sq_sge;
+	props->max_recv_sge = dev_attr->max_rq_sge;
+
+	if (udata && udata->outlen) {
+		resp.max_sq_sge = dev_attr->max_sq_sge;
+		resp.max_rq_sge = dev_attr->max_rq_sge;
+		resp.max_sq_wr = dev_attr->max_sq_depth;
+		resp.max_rq_wr = dev_attr->max_rq_depth;
+
+		err = ib_copy_to_udata(udata, &resp,
+				       min(sizeof(resp), udata->outlen));
+		if (err) {
+			ibdev_dbg(ibdev,
+				  "Failed to copy udata for query_device\n");
+			return err;
+		}
+	}
+
+	return 0;
+}
+
+int efa_query_port(struct ib_device *ibdev, u8 port,
+		   struct ib_port_attr *props)
+{
+	struct efa_dev *dev = to_edev(ibdev);
+
+	props->lmc = 1;
+
+	props->state = IB_PORT_ACTIVE;
+	props->phys_state = 5;
+	props->gid_tbl_len = 1;
+	props->pkey_tbl_len = 1;
+	props->active_speed = IB_SPEED_EDR;
+	props->active_width = IB_WIDTH_4X;
+	props->max_mtu = ib_mtu_int_to_enum(dev->mtu);
+	props->active_mtu = ib_mtu_int_to_enum(dev->mtu);
+	props->max_msg_sz = dev->mtu;
+	props->max_vl_num = 1;
+
+	return 0;
+}
+
+int efa_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
+		 int qp_attr_mask,
+		 struct ib_qp_init_attr *qp_init_attr)
+{
+	struct efa_dev *dev = to_edev(ibqp->device);
+	struct efa_com_query_qp_params params = {};
+	struct efa_com_query_qp_result result;
+	struct efa_qp *qp = to_eqp(ibqp);
+	int err;
+
+#define EFA_QUERY_QP_SUPP_MASK \
+	(IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT | \
+	 IB_QP_QKEY | IB_QP_SQ_PSN | IB_QP_CAP)
+
+	if (qp_attr_mask & ~EFA_QUERY_QP_SUPP_MASK) {
+		ibdev_dbg(&dev->ibdev,
+			  "Unsupported qp_attr_mask[%#x] supported[%#x]\n",
+			  qp_attr_mask, EFA_QUERY_QP_SUPP_MASK);
+		return -EOPNOTSUPP;
+	}
+
+	memset(qp_attr, 0, sizeof(*qp_attr));
+	memset(qp_init_attr, 0, sizeof(*qp_init_attr));
+
+	params.qp_handle = qp->qp_handle;
+	err = efa_com_query_qp(&dev->edev, &params, &result);
+	if (err)
+		return err;
+
+	qp_attr->qp_state = result.qp_state;
+	qp_attr->qkey = result.qkey;
+	qp_attr->sq_psn = result.sq_psn;
+	qp_attr->sq_draining = result.sq_draining;
+	qp_attr->port_num = 1;
+
+	qp_attr->cap.max_send_wr = qp->max_send_wr;
+	qp_attr->cap.max_recv_wr = qp->max_recv_wr;
+	qp_attr->cap.max_send_sge = qp->max_send_sge;
+	qp_attr->cap.max_recv_sge = qp->max_recv_sge;
+	qp_attr->cap.max_inline_data = qp->max_inline_data;
+
+	qp_init_attr->qp_type = ibqp->qp_type;
+	qp_init_attr->recv_cq = ibqp->recv_cq;
+	qp_init_attr->send_cq = ibqp->send_cq;
+	qp_init_attr->qp_context = ibqp->qp_context;
+	qp_init_attr->cap = qp_attr->cap;
+
+	return 0;
+}
+
+int efa_query_gid(struct ib_device *ibdev, u8 port, int index,
+		  union ib_gid *gid)
+{
+	struct efa_dev *dev = to_edev(ibdev);
+
+	memcpy(gid->raw, dev->addr, sizeof(dev->addr));
+
+	return 0;
+}
+
+int efa_query_pkey(struct ib_device *ibdev, u8 port, u16 index,
+		   u16 *pkey)
+{
+	if (index > 0)
+		return -EINVAL;
+
+	*pkey = 0xffff;
+	return 0;
+}
+
+static int efa_pd_dealloc(struct efa_dev *dev, u16 pdn)
+{
+	struct efa_com_dealloc_pd_params params = {
+		.pdn = pdn,
+	};
+
+	return efa_com_dealloc_pd(&dev->edev, &params);
+}
+
+int efa_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
+{
+	struct efa_dev *dev = to_edev(ibpd->device);
+	struct efa_ibv_alloc_pd_resp resp = {};
+	struct efa_com_alloc_pd_result result;
+	struct efa_pd *pd = to_epd(ibpd);
+	int err;
+
+	if (udata->inlen &&
+	    !ib_is_udata_cleared(udata, 0, udata->inlen)) {
+		ibdev_dbg(&dev->ibdev,
+			  "Incompatible ABI params, udata not cleared\n");
+		err = -EINVAL;
+		goto err_out;
+	}
+
+	err = efa_com_alloc_pd(&dev->edev, &result);
+	if (err)
+		goto err_out;
+
+	pd->pdn = result.pdn;
+	resp.pdn = result.pdn;
+
+	if (udata->outlen) {
+		err = ib_copy_to_udata(udata, &resp,
+				       min(sizeof(resp), udata->outlen));
+		if (err) {
+			ibdev_dbg(&dev->ibdev,
+				  "Failed to copy udata for alloc_pd\n");
+			goto err_dealloc_pd;
+		}
+	}
+
+	ibdev_dbg(&dev->ibdev, "Allocated pd[%d]\n", pd->pdn);
+
+	return 0;
+
+err_dealloc_pd:
+	efa_pd_dealloc(dev, result.pdn);
+err_out:
+	atomic64_inc(&dev->stats.sw_stats.alloc_pd_err);
+	return err;
+}
+
+void efa_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
+{
+	struct efa_dev *dev = to_edev(ibpd->device);
+	struct efa_pd *pd = to_epd(ibpd);
+
+	if (udata->inlen &&
+	    !ib_is_udata_cleared(udata, 0, udata->inlen)) {
+		ibdev_dbg(&dev->ibdev, "Incompatible ABI params\n");
+		return;
+	}
+
+	ibdev_dbg(&dev->ibdev, "Dealloc pd[%d]\n", pd->pdn);
+	efa_pd_dealloc(dev, pd->pdn);
+}
+
+static int efa_destroy_qp_handle(struct efa_dev *dev, u32 qp_handle)
+{
+	struct efa_com_destroy_qp_params params = { .qp_handle = qp_handle };
+
+	return efa_com_destroy_qp(&dev->edev, &params);
+}
+
+int efa_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
+{
+	struct efa_dev *dev = to_edev(ibqp->pd->device);
+	struct efa_qp *qp = to_eqp(ibqp);
+	int err;
+
+	if (udata->inlen &&
+	    !ib_is_udata_cleared(udata, 0, udata->inlen)) {
+		ibdev_dbg(&dev->ibdev, "Incompatible ABI params\n");
+		return -EINVAL;
+	}
+
+	ibdev_dbg(&dev->ibdev, "Destroy qp[%u]\n", ibqp->qp_num);
+	err = efa_destroy_qp_handle(dev, qp->qp_handle);
+	if (err)
+		return err;
+
+	if (qp->rq_cpu_addr) {
+		ibdev_dbg(&dev->ibdev,
+			  "qp->cpu_addr[0x%p] freed: size[%lu], dma[%pad]\n",
+			  qp->rq_cpu_addr, qp->rq_size,
+			  &qp->rq_dma_addr);
+		dma_unmap_single(&dev->pdev->dev, qp->rq_dma_addr, qp->rq_size,
+				 DMA_TO_DEVICE);
+	}
+
+	kfree(qp);
+	return 0;
+}
+
+static int qp_mmap_entries_setup(struct efa_qp *qp,
+				 struct efa_dev *dev,
+				 struct efa_ucontext *ucontext,
+				 struct efa_com_create_qp_params *params,
+				 struct efa_ibv_create_qp_resp *resp)
+{
+	/*
+	 * Once an entry is inserted it might be mmapped, hence cannot be
+	 * cleaned up until dealloc_ucontext.
+	 */
+	resp->sq_db_mmap_key =
+		mmap_entry_insert(dev, ucontext, qp,
+				  dev->db_bar_addr + resp->sq_db_offset,
+				  PAGE_SIZE, EFA_MMAP_IO_NC);
+	if (resp->sq_db_mmap_key == EFA_MMAP_INVALID)
+		return -ENOMEM;
+
+	resp->sq_db_offset &= ~PAGE_MASK;
+
+	resp->llq_desc_mmap_key =
+		mmap_entry_insert(dev, ucontext, qp,
+				  dev->mem_bar_addr + resp->llq_desc_offset,
+				  PAGE_ALIGN(params->sq_ring_size_in_bytes +
+					     (resp->llq_desc_offset & ~PAGE_MASK)),
+				  EFA_MMAP_IO_WC);
+	if (resp->llq_desc_mmap_key == EFA_MMAP_INVALID)
+		return -ENOMEM;
+
+	resp->llq_desc_offset &= ~PAGE_MASK;
+
+	if (qp->rq_size) {
+		resp->rq_db_mmap_key =
+			mmap_entry_insert(dev, ucontext, qp,
+					  dev->db_bar_addr + resp->rq_db_offset,
+					  PAGE_SIZE, EFA_MMAP_IO_NC);
+		if (resp->rq_db_mmap_key == EFA_MMAP_INVALID)
+			return -ENOMEM;
+
+		resp->rq_db_offset &= ~PAGE_MASK;
+
+		resp->rq_mmap_key =
+			mmap_entry_insert(dev, ucontext, qp,
+					  virt_to_phys(qp->rq_cpu_addr),
+					  qp->rq_size, EFA_MMAP_DMA_PAGE);
+		if (resp->rq_mmap_key == EFA_MMAP_INVALID)
+			return -ENOMEM;
+
+		resp->rq_mmap_size = qp->rq_size;
+	}
+
+	return 0;
+}
+
+static int efa_qp_validate_cap(struct efa_dev *dev,
+			       struct ib_qp_init_attr *init_attr)
+{
+	if (init_attr->cap.max_send_wr > dev->dev_attr.max_sq_depth) {
+		ibdev_dbg(&dev->ibdev,
+			  "qp: requested send wr[%u] exceeds the max[%u]\n",
+			  init_attr->cap.max_send_wr,
+			  dev->dev_attr.max_sq_depth);
+		return -EINVAL;
+	}
+	if (init_attr->cap.max_recv_wr > dev->dev_attr.max_rq_depth) {
+		ibdev_dbg(&dev->ibdev,
+			  "qp: requested receive wr[%u] exceeds the max[%u]\n",
+			  init_attr->cap.max_recv_wr,
+			  dev->dev_attr.max_rq_depth);
+		return -EINVAL;
+	}
+	if (init_attr->cap.max_send_sge > dev->dev_attr.max_sq_sge) {
+		ibdev_dbg(&dev->ibdev,
+			  "qp: requested sge send[%u] exceeds the max[%u]\n",
+			  init_attr->cap.max_send_sge, dev->dev_attr.max_sq_sge);
+		return -EINVAL;
+	}
+	if (init_attr->cap.max_recv_sge > dev->dev_attr.max_rq_sge) {
+		ibdev_dbg(&dev->ibdev,
+			  "qp: requested sge recv[%u] exceeds the max[%u]\n",
+			  init_attr->cap.max_recv_sge, dev->dev_attr.max_rq_sge);
+		return -EINVAL;
+	}
+	if (init_attr->cap.max_inline_data > dev->dev_attr.inline_buf_size) {
+		ibdev_dbg(&dev->ibdev,
+			  "qp: requested inline data[%u] exceeds the max[%u]\n",
+			  init_attr->cap.max_inline_data,
+			  dev->dev_attr.inline_buf_size);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int efa_qp_validate_attr(struct efa_dev *dev,
+				struct ib_qp_init_attr *init_attr)
+{
+	if (init_attr->qp_type != IB_QPT_DRIVER &&
+	    init_attr->qp_type != IB_QPT_UD) {
+		ibdev_dbg(&dev->ibdev,
+			  "Unsupported qp type %d\n", init_attr->qp_type);
+		return -EOPNOTSUPP;
+	}
+
+	if (init_attr->srq) {
+		ibdev_dbg(&dev->ibdev, "SRQ is not supported\n");
+		return -EOPNOTSUPP;
+	}
+
+	if (init_attr->create_flags) {
+		ibdev_dbg(&dev->ibdev, "Unsupported create flags\n");
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+struct ib_qp *efa_create_qp(struct ib_pd *ibpd,
+			    struct ib_qp_init_attr *init_attr,
+			    struct ib_udata *udata)
+{
+	struct efa_com_create_qp_params create_qp_params = {};
+	struct efa_com_create_qp_result create_qp_resp;
+	struct efa_dev *dev = to_edev(ibpd->device);
+	struct efa_ibv_create_qp_resp resp = {};
+	struct efa_ibv_create_qp cmd = {};
+	bool rq_entry_inserted = false;
+	struct efa_ucontext *ucontext;
+	struct efa_qp *qp;
+	int err;
+
+	ucontext = rdma_udata_to_drv_context(udata, struct efa_ucontext,
+					     ibucontext);
+
+	err = efa_qp_validate_cap(dev, init_attr);
+	if (err)
+		goto err_out;
+
+	err = efa_qp_validate_attr(dev, init_attr);
+	if (err)
+		goto err_out;
+
+	if (!field_avail(cmd, driver_qp_type, udata->inlen)) {
+		ibdev_dbg(&dev->ibdev,
+			  "Incompatible ABI params, no input udata\n");
+		err = -EINVAL;
+		goto err_out;
+	}
+
+	if (udata->inlen > sizeof(cmd) &&
+	    !ib_is_udata_cleared(udata, sizeof(cmd),
+				 udata->inlen - sizeof(cmd))) {
+		ibdev_dbg(&dev->ibdev,
+			  "Incompatible ABI params, unknown fields in udata\n");
+		err = -EINVAL;
+		goto err_out;
+	}
+
+	err = ib_copy_from_udata(&cmd, udata,
+				 min(sizeof(cmd), udata->inlen));
+	if (err) {
+		ibdev_dbg(&dev->ibdev,
+			  "Cannot copy udata for create_qp\n");
+		goto err_out;
+	}
+
+	if (cmd.comp_mask) {
+		ibdev_dbg(&dev->ibdev,
+			  "Incompatible ABI params, unknown fields in udata\n");
+		err = -EINVAL;
+		goto err_out;
+	}
+
+	qp = kzalloc(sizeof(*qp), GFP_KERNEL);
+	if (!qp) {
+		err = -ENOMEM;
+		goto err_out;
+	}
+
+	create_qp_params.uarn = ucontext->uarn;
+	create_qp_params.pd = to_epd(ibpd)->pdn;
+
+	if (init_attr->qp_type == IB_QPT_UD) {
+		create_qp_params.qp_type = EFA_ADMIN_QP_TYPE_UD;
+	} else if (cmd.driver_qp_type == EFA_QP_DRIVER_TYPE_SRD) {
+		create_qp_params.qp_type = EFA_ADMIN_QP_TYPE_SRD;
+	} else {
+		ibdev_dbg(&dev->ibdev,
+			  "Unsupported qp type %d driver qp type %d\n",
+			  init_attr->qp_type, cmd.driver_qp_type);
+		err = -EOPNOTSUPP;
+		goto err_free_qp;
+	}
+
+	ibdev_dbg(&dev->ibdev, "Create QP: qp type %d driver qp type %#x\n",
+		  init_attr->qp_type, cmd.driver_qp_type);
+	create_qp_params.send_cq_idx = to_ecq(init_attr->send_cq)->cq_idx;
+	create_qp_params.recv_cq_idx = to_ecq(init_attr->recv_cq)->cq_idx;
+	create_qp_params.sq_depth = init_attr->cap.max_send_wr;
+	create_qp_params.sq_ring_size_in_bytes = cmd.sq_ring_size;
+
+	create_qp_params.rq_depth = init_attr->cap.max_recv_wr;
+	create_qp_params.rq_ring_size_in_bytes = cmd.rq_ring_size;
+	qp->rq_size = PAGE_ALIGN(create_qp_params.rq_ring_size_in_bytes);
+	if (qp->rq_size) {
+		qp->rq_cpu_addr = efa_zalloc_mapped(dev, &qp->rq_dma_addr,
+						    qp->rq_size, DMA_TO_DEVICE);
+		if (!qp->rq_cpu_addr) {
+			err = -ENOMEM;
+			goto err_free_qp;
+		}
+
+		ibdev_dbg(&dev->ibdev,
+			  "qp->cpu_addr[0x%p] allocated: size[%lu], dma[%pad]\n",
+			  qp->rq_cpu_addr, qp->rq_size, &qp->rq_dma_addr);
+		create_qp_params.rq_base_addr = qp->rq_dma_addr;
+	}
+
+	err = efa_com_create_qp(&dev->edev, &create_qp_params,
+				&create_qp_resp);
+	if (err)
+		goto err_free_mapped;
+
+	resp.sq_db_offset = create_qp_resp.sq_db_offset;
+	resp.rq_db_offset = create_qp_resp.rq_db_offset;
+	resp.llq_desc_offset = create_qp_resp.llq_descriptors_offset;
+	resp.send_sub_cq_idx = create_qp_resp.send_sub_cq_idx;
+	resp.recv_sub_cq_idx = create_qp_resp.recv_sub_cq_idx;
+
+	err = qp_mmap_entries_setup(qp, dev, ucontext, &create_qp_params,
+				    &resp);
+	if (err)
+		goto err_destroy_qp;
+
+	rq_entry_inserted = true;
+	qp->qp_handle = create_qp_resp.qp_handle;
+	qp->ibqp.qp_num = create_qp_resp.qp_num;
+	qp->ibqp.qp_type = init_attr->qp_type;
+	qp->max_send_wr = init_attr->cap.max_send_wr;
+	qp->max_recv_wr = init_attr->cap.max_recv_wr;
+	qp->max_send_sge = init_attr->cap.max_send_sge;
+	qp->max_recv_sge = init_attr->cap.max_recv_sge;
+	qp->max_inline_data = init_attr->cap.max_inline_data;
+
+	if (udata->outlen) {
+		err = ib_copy_to_udata(udata, &resp,
+				       min(sizeof(resp), udata->outlen));
+		if (err) {
+			ibdev_dbg(&dev->ibdev,
+				  "Failed to copy udata for qp[%u]\n",
+				  create_qp_resp.qp_num);
+			goto err_destroy_qp;
+		}
+	}
+
+	ibdev_dbg(&dev->ibdev, "Created qp[%d]\n", qp->ibqp.qp_num);
+
+	return &qp->ibqp;
+
+err_destroy_qp:
+	efa_destroy_qp_handle(dev, create_qp_resp.qp_handle);
+err_free_mapped:
+	if (qp->rq_size) {
+		dma_unmap_single(&dev->pdev->dev, qp->rq_dma_addr, qp->rq_size,
+				 DMA_TO_DEVICE);
+		if (!rq_entry_inserted)
+			free_pages_exact(qp->rq_cpu_addr, qp->rq_size);
+	}
+err_free_qp:
+	kfree(qp);
+err_out:
+	atomic64_inc(&dev->stats.sw_stats.create_qp_err);
+	return ERR_PTR(err);
+}
+
+static int efa_modify_qp_validate(struct efa_dev *dev, struct efa_qp *qp,
+				  struct ib_qp_attr *qp_attr, int qp_attr_mask,
+				  enum ib_qp_state cur_state,
+				  enum ib_qp_state new_state)
+{
+#define EFA_MODIFY_QP_SUPP_MASK \
+	(IB_QP_STATE | IB_QP_CUR_STATE | IB_QP_EN_SQD_ASYNC_NOTIFY | \
+	 IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_QKEY | IB_QP_SQ_PSN)
+
+	if (qp_attr_mask & ~EFA_MODIFY_QP_SUPP_MASK) {
+		ibdev_dbg(&dev->ibdev,
+			  "Unsupported qp_attr_mask[%#x] supported[%#x]\n",
+			  qp_attr_mask, EFA_MODIFY_QP_SUPP_MASK);
+		return -EOPNOTSUPP;
+	}
+
+	if (!ib_modify_qp_is_ok(cur_state, new_state, IB_QPT_UD,
+				qp_attr_mask)) {
+		ibdev_dbg(&dev->ibdev, "Invalid modify QP parameters\n");
+		return -EINVAL;
+	}
+
+	if ((qp_attr_mask & IB_QP_PORT) && qp_attr->port_num != 1) {
+		ibdev_dbg(&dev->ibdev, "Can't change port num\n");
+		return -EOPNOTSUPP;
+	}
+
+	if ((qp_attr_mask & IB_QP_PKEY_INDEX) && qp_attr->pkey_index) {
+		ibdev_dbg(&dev->ibdev, "Can't change pkey index\n");
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+int efa_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
+		  int qp_attr_mask, struct ib_udata *udata)
+{
+	struct efa_dev *dev = to_edev(ibqp->device);
+	struct efa_com_modify_qp_params params = {};
+	struct efa_qp *qp = to_eqp(ibqp);
+	enum ib_qp_state cur_state;
+	enum ib_qp_state new_state;
+	int err;
+
+	if (udata->inlen &&
+	    !ib_is_udata_cleared(udata, 0, udata->inlen)) {
+		ibdev_dbg(&dev->ibdev,
+			  "Incompatible ABI params, udata not cleared\n");
+		return -EINVAL;
+	}
+
+	cur_state = qp_attr_mask & IB_QP_CUR_STATE ? qp_attr->cur_qp_state :
+						     qp->state;
+	new_state = qp_attr_mask & IB_QP_STATE ? qp_attr->qp_state : cur_state;
+
+	err = efa_modify_qp_validate(dev, qp, qp_attr, qp_attr_mask, cur_state,
+				     new_state);
+	if (err)
+		return err;
+
+	params.qp_handle = qp->qp_handle;
+
+	if (qp_attr_mask & IB_QP_STATE) {
+		params.modify_mask |= BIT(EFA_ADMIN_QP_STATE_BIT) |
+				      BIT(EFA_ADMIN_CUR_QP_STATE_BIT);
+		params.cur_qp_state = qp_attr->cur_qp_state;
+		params.qp_state = qp_attr->qp_state;
+	}
+
+	if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) {
+		params.modify_mask |=
+			BIT(EFA_ADMIN_SQ_DRAINED_ASYNC_NOTIFY_BIT);
+		params.sq_drained_async_notify = qp_attr->en_sqd_async_notify;
+	}
+
+	if (qp_attr_mask & IB_QP_QKEY) {
+		params.modify_mask |= BIT(EFA_ADMIN_QKEY_BIT);
+		params.qkey = qp_attr->qkey;
+	}
+
+	if (qp_attr_mask & IB_QP_SQ_PSN) {
+		params.modify_mask |= BIT(EFA_ADMIN_SQ_PSN_BIT);
+		params.sq_psn = qp_attr->sq_psn;
+	}
+
+	err = efa_com_modify_qp(&dev->edev, &params);
+	if (err)
+		return err;
+
+	qp->state = new_state;
+
+	return 0;
+}
+
+static int efa_destroy_cq_idx(struct efa_dev *dev, int cq_idx)
+{
+	struct efa_com_destroy_cq_params params = { .cq_idx = cq_idx };
+
+	return efa_com_destroy_cq(&dev->edev, &params);
+}
+
+int efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
+{
+	struct efa_dev *dev = to_edev(ibcq->device);
+	struct efa_cq *cq = to_ecq(ibcq);
+	int err;
+
+	if (udata->inlen &&
+	    !ib_is_udata_cleared(udata, 0, udata->inlen)) {
+		ibdev_dbg(&dev->ibdev, "Incompatible ABI params\n");
+		return -EINVAL;
+	}
+
+	ibdev_dbg(&dev->ibdev,
+		  "Destroy cq[%d] virt[0x%p] freed: size[%lu], dma[%pad]\n",
+		  cq->cq_idx, cq->cpu_addr, cq->size, &cq->dma_addr);
+
+	err = efa_destroy_cq_idx(dev, cq->cq_idx);
+	if (err)
+		return err;
+
+	dma_unmap_single(&dev->pdev->dev, cq->dma_addr, cq->size,
+			 DMA_FROM_DEVICE);
+
+	kfree(cq);
+	return 0;
+}
+
+static int cq_mmap_entries_setup(struct efa_dev *dev, struct efa_cq *cq,
+				 struct efa_ibv_create_cq_resp *resp)
+{
+	resp->q_mmap_size = cq->size;
+	resp->q_mmap_key = mmap_entry_insert(dev, cq->ucontext, cq,
+					     virt_to_phys(cq->cpu_addr),
+					     cq->size, EFA_MMAP_DMA_PAGE);
+	if (resp->q_mmap_key == EFA_MMAP_INVALID)
+		return -ENOMEM;
+
+	return 0;
+}
+
+static struct ib_cq *do_create_cq(struct ib_device *ibdev, int entries,
+				  int vector, struct ib_ucontext *ibucontext,
+				  struct ib_udata *udata)
+{
+	struct efa_ibv_create_cq_resp resp = {};
+	struct efa_com_create_cq_params params;
+	struct efa_com_create_cq_result result;
+	struct efa_dev *dev = to_edev(ibdev);
+	struct efa_ibv_create_cq cmd = {};
+	bool cq_entry_inserted = false;
+	struct efa_cq *cq;
+	int err;
+
+	ibdev_dbg(ibdev, "create_cq entries %d\n", entries);
+
+	if (entries < 1 || entries > dev->dev_attr.max_cq_depth) {
+		ibdev_dbg(ibdev,
+			  "cq: requested entries[%u] non-positive or greater than max[%u]\n",
+			  entries, dev->dev_attr.max_cq_depth);
+		err = -EINVAL;
+		goto err_out;
+	}
+
+	if (!field_avail(cmd, num_sub_cqs, udata->inlen)) {
+		ibdev_dbg(ibdev,
+			  "Incompatible ABI params, no input udata\n");
+		err = -EINVAL;
+		goto err_out;
+	}
+
+	if (udata->inlen > sizeof(cmd) &&
+	    !ib_is_udata_cleared(udata, sizeof(cmd),
+				 udata->inlen - sizeof(cmd))) {
+		ibdev_dbg(ibdev,
+			  "Incompatible ABI params, unknown fields in udata\n");
+		err = -EINVAL;
+		goto err_out;
+	}
+
+	err = ib_copy_from_udata(&cmd, udata,
+				 min(sizeof(cmd), udata->inlen));
+	if (err) {
+		ibdev_dbg(ibdev, "Cannot copy udata for create_cq\n");
+		goto err_out;
+	}
+
+	if (cmd.comp_mask || !is_reserved_cleared(cmd.reserved_50)) {
+		ibdev_dbg(ibdev,
+			  "Incompatible ABI params, unknown fields in udata\n");
+		err = -EINVAL;
+		goto err_out;
+	}
+
+	if (!cmd.cq_entry_size) {
+		ibdev_dbg(ibdev,
+			  "Invalid entry size [%u]\n", cmd.cq_entry_size);
+		err = -EINVAL;
+		goto err_out;
+	}
+
+	if (cmd.num_sub_cqs != dev->dev_attr.sub_cqs_per_cq) {
+		ibdev_dbg(ibdev,
+			  "Invalid number of sub cqs[%u] expected[%u]\n",
+			  cmd.num_sub_cqs, dev->dev_attr.sub_cqs_per_cq);
+		err = -EINVAL;
+		goto err_out;
+	}
+
+	cq = kzalloc(sizeof(*cq), GFP_KERNEL);
+	if (!cq) {
+		err = -ENOMEM;
+		goto err_out;
+	}
+
+	cq->ucontext = to_eucontext(ibucontext);
+	cq->size = PAGE_ALIGN(cmd.cq_entry_size * entries * cmd.num_sub_cqs);
+	cq->cpu_addr = efa_zalloc_mapped(dev, &cq->dma_addr, cq->size,
+					 DMA_FROM_DEVICE);
+	if (!cq->cpu_addr) {
+		err = -ENOMEM;
+		goto err_free_cq;
+	}
+
+	params.uarn = cq->ucontext->uarn;
+	params.cq_depth = entries;
+	params.dma_addr = cq->dma_addr;
+	params.entry_size_in_bytes = cmd.cq_entry_size;
+	params.num_sub_cqs = cmd.num_sub_cqs;
+	err = efa_com_create_cq(&dev->edev, &params, &result);
+	if (err)
+		goto err_free_mapped;
+
+	resp.cq_idx = result.cq_idx;
+	cq->cq_idx = result.cq_idx;
+	cq->ibcq.cqe = result.actual_depth;
+	WARN_ON_ONCE(entries != result.actual_depth);
+
+	err = cq_mmap_entries_setup(dev, cq, &resp);
+	if (err) {
+		ibdev_dbg(ibdev,
+			  "Could not setup cq[%u] mmap entries\n", cq->cq_idx);
+		goto err_destroy_cq;
+	}
+
+	cq_entry_inserted = true;
+
+	if (udata->outlen) {
+		err = ib_copy_to_udata(udata, &resp,
+				       min(sizeof(resp), udata->outlen));
+		if (err) {
+			ibdev_dbg(ibdev,
+				  "Failed to copy udata for create_cq\n");
+			goto err_destroy_cq;
+		}
+	}
+
+	ibdev_dbg(ibdev,
+		  "Created cq[%d], cq depth[%u]. dma[%pad] virt[0x%p]\n",
+		  cq->cq_idx, result.actual_depth, &cq->dma_addr, cq->cpu_addr);
+
+	return &cq->ibcq;
+
+err_destroy_cq:
+	efa_destroy_cq_idx(dev, cq->cq_idx);
+err_free_mapped:
+	dma_unmap_single(&dev->pdev->dev, cq->dma_addr, cq->size,
+			 DMA_FROM_DEVICE);
+	if (!cq_entry_inserted)
+		free_pages_exact(cq->cpu_addr, cq->size);
+err_free_cq:
+	kfree(cq);
+err_out:
+	atomic64_inc(&dev->stats.sw_stats.create_cq_err);
+	return ERR_PTR(err);
+}
+
+struct ib_cq *efa_create_cq(struct ib_device *ibdev,
+			    const struct ib_cq_init_attr *attr,
+			    struct ib_udata *udata)
+{
+	struct efa_ucontext *ucontext = rdma_udata_to_drv_context(udata,
+								  struct efa_ucontext,
+								  ibucontext);
+
+	return do_create_cq(ibdev, attr->cqe, attr->comp_vector,
+			    &ucontext->ibucontext, udata);
+}
+
+static int umem_to_page_list(struct efa_dev *dev,
+			     struct ib_umem *umem,
+			     u64 *page_list,
+			     u32 hp_cnt,
+			     u8 hp_shift)
+{
+	u32 pages_in_hp = BIT(hp_shift - PAGE_SHIFT);
+	struct sg_dma_page_iter sg_iter;
+	unsigned int page_idx = 0;
+	unsigned int hp_idx = 0;
+
+	ibdev_dbg(&dev->ibdev, "hp_cnt[%u], pages_in_hp[%u]\n",
+		  hp_cnt, pages_in_hp);
+
+	for_each_sg_dma_page(umem->sg_head.sgl, &sg_iter, umem->nmap, 0) {
+		if (page_idx % pages_in_hp == 0) {
+			page_list[hp_idx] = sg_page_iter_dma_address(&sg_iter);
+			hp_idx++;
+		}
+
+		page_idx++;
+	}
+
+	return 0;
+}
+
+static struct scatterlist *efa_vmalloc_buf_to_sg(u64 *buf, int page_cnt)
+{
+	struct scatterlist *sglist;
+	struct page *pg;
+	int i;
+
+	sglist = kcalloc(page_cnt, sizeof(*sglist), GFP_KERNEL);
+	if (!sglist)
+		return NULL;
+	sg_init_table(sglist, page_cnt);
+	for (i = 0; i < page_cnt; i++) {
+		pg = vmalloc_to_page(buf);
+		if (!pg)
+			goto err;
+		sg_set_page(&sglist[i], pg, PAGE_SIZE, 0);
+		buf += PAGE_SIZE / sizeof(*buf);
+	}
+	return sglist;
+
+err:
+	kfree(sglist);
+	return NULL;
+}
+
+/*
+ * create a chunk list of physical pages dma addresses from the supplied
+ * scatter gather list
+ */
+static int pbl_chunk_list_create(struct efa_dev *dev, struct pbl_context *pbl)
+{
+	unsigned int entry, payloads_in_sg, chunk_list_size, chunk_idx, payload_idx;
+	struct pbl_chunk_list *chunk_list = &pbl->phys.indirect.chunk_list;
+	int page_cnt = pbl->phys.indirect.pbl_buf_size_in_pages;
+	struct scatterlist *pages_sgl = pbl->phys.indirect.sgl;
+	int sg_dma_cnt = pbl->phys.indirect.sg_dma_cnt;
+	struct efa_com_ctrl_buff_info *ctrl_buf;
+	u64 *cur_chunk_buf, *prev_chunk_buf;
+	struct scatterlist *sg;
+	dma_addr_t dma_addr;
+	int i;
+
+	/* allocate a chunk list that consists of 4KB chunks */
+	chunk_list_size = DIV_ROUND_UP(page_cnt, EFA_PTRS_PER_CHUNK);
+
+	chunk_list->size = chunk_list_size;
+	chunk_list->chunks = kcalloc(chunk_list_size,
+				     sizeof(*chunk_list->chunks),
+				     GFP_KERNEL);
+	if (!chunk_list->chunks)
+		return -ENOMEM;
+
+	ibdev_dbg(&dev->ibdev,
+		  "chunk_list_size[%u] - pages[%u]\n", chunk_list_size,
+		  page_cnt);
+
+	/* allocate chunk buffers: */
+	for (i = 0; i < chunk_list_size; i++) {
+		chunk_list->chunks[i].buf = kzalloc(EFA_CHUNK_SIZE, GFP_KERNEL);
+		if (!chunk_list->chunks[i].buf)
+			goto chunk_list_dealloc;
+
+		chunk_list->chunks[i].length = EFA_CHUNK_USED_SIZE;
+	}
+	chunk_list->chunks[chunk_list_size - 1].length =
+		((page_cnt % EFA_PTRS_PER_CHUNK) * EFA_CHUNK_PAYLOAD_PTR_SIZE) +
+			EFA_CHUNK_PTR_SIZE;
+
+	/* fill the dma addresses of sg list pages to chunks: */
+	chunk_idx = 0;
+	payload_idx = 0;
+	cur_chunk_buf = chunk_list->chunks[0].buf;
+	for_each_sg(pages_sgl, sg, sg_dma_cnt, entry) {
+		payloads_in_sg = sg_dma_len(sg) >> EFA_CHUNK_PAYLOAD_SHIFT;
+		for (i = 0; i < payloads_in_sg; i++) {
+			cur_chunk_buf[payload_idx++] =
+				(sg_dma_address(sg) & ~(EFA_CHUNK_PAYLOAD_SIZE - 1)) +
+				(EFA_CHUNK_PAYLOAD_SIZE * i);
+
+			if (payload_idx == EFA_PTRS_PER_CHUNK) {
+				chunk_idx++;
+				cur_chunk_buf = chunk_list->chunks[chunk_idx].buf;
+				payload_idx = 0;
+			}
+		}
+	}
+
+	/* map chunks to dma and fill chunks next ptrs */
+	for (i = chunk_list_size - 1; i >= 0; i--) {
+		dma_addr = dma_map_single(&dev->pdev->dev,
+					  chunk_list->chunks[i].buf,
+					  chunk_list->chunks[i].length,
+					  DMA_TO_DEVICE);
+		if (dma_mapping_error(&dev->pdev->dev, dma_addr)) {
+			ibdev_err(&dev->ibdev,
+				  "chunk[%u] dma_map_failed\n", i);
+			goto chunk_list_unmap;
+		}
+
+		chunk_list->chunks[i].dma_addr = dma_addr;
+		ibdev_dbg(&dev->ibdev,
+			  "chunk[%u] mapped at [%pad]\n", i, &dma_addr);
+
+		if (!i)
+			break;
+
+		prev_chunk_buf = chunk_list->chunks[i - 1].buf;
+
+		ctrl_buf = (struct efa_com_ctrl_buff_info *)
+				&prev_chunk_buf[EFA_PTRS_PER_CHUNK];
+		ctrl_buf->length = chunk_list->chunks[i].length;
+
+		efa_com_set_dma_addr(dma_addr,
+				     &ctrl_buf->address.mem_addr_high,
+				     &ctrl_buf->address.mem_addr_low);
+	}
+
+	return 0;
+
+chunk_list_unmap:
+	for (; i < chunk_list_size; i++) {
+		dma_unmap_single(&dev->pdev->dev, chunk_list->chunks[i].dma_addr,
+				 chunk_list->chunks[i].length, DMA_TO_DEVICE);
+	}
+chunk_list_dealloc:
+	for (i = 0; i < chunk_list_size; i++)
+		kfree(chunk_list->chunks[i].buf);
+
+	kfree(chunk_list->chunks);
+	return -ENOMEM;
+}
+
+static void pbl_chunk_list_destroy(struct efa_dev *dev, struct pbl_context *pbl)
+{
+	struct pbl_chunk_list *chunk_list = &pbl->phys.indirect.chunk_list;
+	int i;
+
+	for (i = 0; i < chunk_list->size; i++) {
+		dma_unmap_single(&dev->pdev->dev, chunk_list->chunks[i].dma_addr,
+				 chunk_list->chunks[i].length, DMA_TO_DEVICE);
+		kfree(chunk_list->chunks[i].buf);
+	}
+
+	kfree(chunk_list->chunks);
+}
+
+/* initialize pbl continuous mode: map pbl buffer to a dma address. */
+static int pbl_continuous_initialize(struct efa_dev *dev,
+				     struct pbl_context *pbl)
+{
+	dma_addr_t dma_addr;
+
+	dma_addr = dma_map_single(&dev->pdev->dev, pbl->pbl_buf,
+				  pbl->pbl_buf_size_in_bytes, DMA_TO_DEVICE);
+	if (dma_mapping_error(&dev->pdev->dev, dma_addr)) {
+		ibdev_err(&dev->ibdev, "Unable to map pbl to DMA address\n");
+		return -ENOMEM;
+	}
+
+	pbl->phys.continuous.dma_addr = dma_addr;
+	ibdev_dbg(&dev->ibdev,
+		  "pbl continuous - dma_addr = %pad, size[%u]\n",
+		  &dma_addr, pbl->pbl_buf_size_in_bytes);
+
+	return 0;
+}
+
+/*
+ * initialize pbl indirect mode:
+ * create a chunk list out of the dma addresses of the physical pages of
+ * pbl buffer.
+ */
+static int pbl_indirect_initialize(struct efa_dev *dev, struct pbl_context *pbl)
+{
+	u32 size_in_pages = DIV_ROUND_UP(pbl->pbl_buf_size_in_bytes, PAGE_SIZE);
+	struct scatterlist *sgl;
+	int sg_dma_cnt, err;
+
+	BUILD_BUG_ON(EFA_CHUNK_PAYLOAD_SIZE > PAGE_SIZE);
+	sgl = efa_vmalloc_buf_to_sg(pbl->pbl_buf, size_in_pages);
+	if (!sgl)
+		return -ENOMEM;
+
+	sg_dma_cnt = dma_map_sg(&dev->pdev->dev, sgl, size_in_pages, DMA_TO_DEVICE);
+	if (!sg_dma_cnt) {
+		err = -EINVAL;
+		goto err_map;
+	}
+
+	pbl->phys.indirect.pbl_buf_size_in_pages = size_in_pages;
+	pbl->phys.indirect.sgl = sgl;
+	pbl->phys.indirect.sg_dma_cnt = sg_dma_cnt;
+	err = pbl_chunk_list_create(dev, pbl);
+	if (err) {
+		ibdev_dbg(&dev->ibdev,
+			  "chunk_list creation failed[%d]\n", err);
+		goto err_chunk;
+	}
+
+	ibdev_dbg(&dev->ibdev,
+		  "pbl indirect - size[%u], chunks[%u]\n",
+		  pbl->pbl_buf_size_in_bytes,
+		  pbl->phys.indirect.chunk_list.size);
+
+	return 0;
+
+err_chunk:
+	dma_unmap_sg(&dev->pdev->dev, sgl, size_in_pages, DMA_TO_DEVICE);
+err_map:
+	kfree(sgl);
+	return err;
+}
+
+static void pbl_indirect_terminate(struct efa_dev *dev, struct pbl_context *pbl)
+{
+	pbl_chunk_list_destroy(dev, pbl);
+	dma_unmap_sg(&dev->pdev->dev, pbl->phys.indirect.sgl,
+		     pbl->phys.indirect.pbl_buf_size_in_pages, DMA_TO_DEVICE);
+	kfree(pbl->phys.indirect.sgl);
+}
+
+/* create a page buffer list from a mapped user memory region */
+static int pbl_create(struct efa_dev *dev,
+		      struct pbl_context *pbl,
+		      struct ib_umem *umem,
+		      int hp_cnt,
+		      u8 hp_shift)
+{
+	int err;
+
+	pbl->pbl_buf_size_in_bytes = hp_cnt * EFA_CHUNK_PAYLOAD_PTR_SIZE;
+	pbl->pbl_buf = kzalloc(pbl->pbl_buf_size_in_bytes,
+			       GFP_KERNEL | __GFP_NOWARN);
+	if (pbl->pbl_buf) {
+		pbl->physically_continuous = 1;
+		err = umem_to_page_list(dev, umem, pbl->pbl_buf, hp_cnt,
+					hp_shift);
+		if (err)
+			goto err_continuous;
+		err = pbl_continuous_initialize(dev, pbl);
+		if (err)
+			goto err_continuous;
+	} else {
+		pbl->physically_continuous = 0;
+		pbl->pbl_buf = vzalloc(pbl->pbl_buf_size_in_bytes);
+		if (!pbl->pbl_buf)
+			return -ENOMEM;
+
+		err = umem_to_page_list(dev, umem, pbl->pbl_buf, hp_cnt,
+					hp_shift);
+		if (err)
+			goto err_indirect;
+		err = pbl_indirect_initialize(dev, pbl);
+		if (err)
+			goto err_indirect;
+	}
+
+	ibdev_dbg(&dev->ibdev,
+		  "user_pbl_created: user_pages[%u], continuous[%u]\n",
+		  hp_cnt, pbl->physically_continuous);
+
+	return 0;
+
+err_continuous:
+	kfree(pbl->pbl_buf);
+	return err;
+err_indirect:
+	vfree(pbl->pbl_buf);
+	return err;
+}
+
+static void pbl_destroy(struct efa_dev *dev, struct pbl_context *pbl)
+{
+	if (pbl->physically_continuous) {
+		dma_unmap_single(&dev->pdev->dev, pbl->phys.continuous.dma_addr,
+				 pbl->pbl_buf_size_in_bytes, DMA_TO_DEVICE);
+		kfree(pbl->pbl_buf);
+	} else {
+		pbl_indirect_terminate(dev, pbl);
+		vfree(pbl->pbl_buf);
+	}
+}
+
+static int efa_create_inline_pbl(struct efa_dev *dev, struct efa_mr *mr,
+				 struct efa_com_reg_mr_params *params)
+{
+	int err;
+
+	params->inline_pbl = 1;
+	err = umem_to_page_list(dev, mr->umem, params->pbl.inline_pbl_array,
+				params->page_num, params->page_shift);
+	if (err)
+		return err;
+
+	ibdev_dbg(&dev->ibdev,
+		  "inline_pbl_array - pages[%u]\n", params->page_num);
+
+	return 0;
+}
+
+static int efa_create_pbl(struct efa_dev *dev,
+			  struct pbl_context *pbl,
+			  struct efa_mr *mr,
+			  struct efa_com_reg_mr_params *params)
+{
+	int err;
+
+	err = pbl_create(dev, pbl, mr->umem, params->page_num,
+			 params->page_shift);
+	if (err) {
+		ibdev_dbg(&dev->ibdev, "Failed to create pbl[%d]\n", err);
+		return err;
+	}
+
+	params->inline_pbl = 0;
+	params->indirect = !pbl->physically_continuous;
+	if (pbl->physically_continuous) {
+		params->pbl.pbl.length = pbl->pbl_buf_size_in_bytes;
+
+		efa_com_set_dma_addr(pbl->phys.continuous.dma_addr,
+				     &params->pbl.pbl.address.mem_addr_high,
+				     &params->pbl.pbl.address.mem_addr_low);
+	} else {
+		params->pbl.pbl.length =
+			pbl->phys.indirect.chunk_list.chunks[0].length;
+
+		efa_com_set_dma_addr(pbl->phys.indirect.chunk_list.chunks[0].dma_addr,
+				     &params->pbl.pbl.address.mem_addr_high,
+				     &params->pbl.pbl.address.mem_addr_low);
+	}
+
+	return 0;
+}
+
+static void efa_cont_pages(struct ib_umem *umem, u64 addr,
+			   unsigned long max_page_shift,
+			   int *count, u8 *shift, u32 *ncont)
+{
+	struct scatterlist *sg;
+	u64 base = ~0, p = 0;
+	unsigned long tmp;
+	unsigned long m;
+	u64 len, pfn;
+	int i = 0;
+	int entry;
+
+	addr = addr >> PAGE_SHIFT;
+	tmp = (unsigned long)addr;
+	m = find_first_bit(&tmp, BITS_PER_LONG);
+	if (max_page_shift)
+		m = min_t(unsigned long, max_page_shift - PAGE_SHIFT, m);
+
+	for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
+		len = DIV_ROUND_UP(sg_dma_len(sg), PAGE_SIZE);
+		pfn = sg_dma_address(sg) >> PAGE_SHIFT;
+		if (base + p != pfn) {
+			/*
+			 * If either the offset or the new
+			 * base are unaligned update m
+			 */
+			tmp = (unsigned long)(pfn | p);
+			if (!IS_ALIGNED(tmp, 1 << m))
+				m = find_first_bit(&tmp, BITS_PER_LONG);
+
+			base = pfn;
+			p = 0;
+		}
+
+		p += len;
+		i += len;
+	}
+
+	if (i) {
+		m = min_t(unsigned long, ilog2(roundup_pow_of_two(i)), m);
+		*ncont = DIV_ROUND_UP(i, (1 << m));
+	} else {
+		m = 0;
+		*ncont = 0;
+	}
+
+	*shift = PAGE_SHIFT + m;
+	*count = i;
+}
+
+struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length,
+			 u64 virt_addr, int access_flags,
+			 struct ib_udata *udata)
+{
+	struct efa_dev *dev = to_edev(ibpd->device);
+	struct efa_com_reg_mr_params params = {};
+	struct efa_com_reg_mr_result result = {};
+	unsigned long max_page_shift;
+	struct pbl_context pbl;
+	struct efa_mr *mr;
+	int inline_size;
+	int npages;
+	int err;
+
+	if (udata->inlen &&
+	    !ib_is_udata_cleared(udata, 0, sizeof(udata->inlen))) {
+		ibdev_dbg(&dev->ibdev,
+			  "Incompatible ABI params, udata not cleared\n");
+		err = -EINVAL;
+		goto err_out;
+	}
+
+	if (access_flags & ~EFA_SUPPORTED_ACCESS_FLAGS) {
+		ibdev_dbg(&dev->ibdev,
+			  "Unsupported access flags[%#x], supported[%#x]\n",
+			  access_flags, EFA_SUPPORTED_ACCESS_FLAGS);
+		err = -EOPNOTSUPP;
+		goto err_out;
+	}
+
+	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
+	if (!mr) {
+		err = -ENOMEM;
+		goto err_out;
+	}
+
+	mr->umem = ib_umem_get(udata, start, length, access_flags, 0);
+	if (IS_ERR(mr->umem)) {
+		err = PTR_ERR(mr->umem);
+		ibdev_dbg(&dev->ibdev,
+			  "Failed to pin and map user space memory[%d]\n", err);
+		goto err_free;
+	}
+
+	params.pd = to_epd(ibpd)->pdn;
+	params.iova = virt_addr;
+	params.mr_length_in_bytes = length;
+	params.permissions = access_flags & 0x1;
+	max_page_shift = fls64(dev->dev_attr.page_size_cap);
+
+	efa_cont_pages(mr->umem, start, max_page_shift, &npages,
+		       &params.page_shift, &params.page_num);
+	ibdev_dbg(&dev->ibdev,
+		  "start %#llx length %#llx npages %d params.page_shift %u params.page_num %u\n",
+		  start, length, npages, params.page_shift, params.page_num);
+
+	inline_size = ARRAY_SIZE(params.pbl.inline_pbl_array);
+	if (params.page_num <= inline_size) {
+		err = efa_create_inline_pbl(dev, mr, &params);
+		if (err)
+			goto err_unmap;
+
+		err = efa_com_register_mr(&dev->edev, &params, &result);
+		if (err)
+			goto err_unmap;
+	} else {
+		err = efa_create_pbl(dev, &pbl, mr, &params);
+		if (err)
+			goto err_unmap;
+
+		err = efa_com_register_mr(&dev->edev, &params, &result);
+		pbl_destroy(dev, &pbl);
+
+		if (err)
+			goto err_unmap;
+	}
+
+	mr->ibmr.lkey = result.l_key;
+	mr->ibmr.rkey = result.r_key;
+	mr->ibmr.length = length;
+	ibdev_dbg(&dev->ibdev, "Registered mr[%d]\n", mr->ibmr.lkey);
+
+	return &mr->ibmr;
+
+err_unmap:
+	ib_umem_release(mr->umem);
+err_free:
+	kfree(mr);
+err_out:
+	atomic64_inc(&dev->stats.sw_stats.reg_mr_err);
+	return ERR_PTR(err);
+}
+
+int efa_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
+{
+	struct efa_dev *dev = to_edev(ibmr->device);
+	struct efa_com_dereg_mr_params params;
+	struct efa_mr *mr = to_emr(ibmr);
+	int err;
+
+	if (udata->inlen &&
+	    !ib_is_udata_cleared(udata, 0, udata->inlen)) {
+		ibdev_dbg(&dev->ibdev, "Incompatible ABI params\n");
+		return -EINVAL;
+	}
+
+	ibdev_dbg(&dev->ibdev, "Deregister mr[%d]\n", ibmr->lkey);
+
+	if (mr->umem) {
+		params.l_key = mr->ibmr.lkey;
+		err = efa_com_dereg_mr(&dev->edev, &params);
+		if (err)
+			return err;
+		ib_umem_release(mr->umem);
+	}
+
+	kfree(mr);
+
+	return 0;
+}
+
+int efa_get_port_immutable(struct ib_device *ibdev, u8 port_num,
+			   struct ib_port_immutable *immutable)
+{
+	struct ib_port_attr attr;
+	int err;
+
+	err = ib_query_port(ibdev, port_num, &attr);
+	if (err) {
+		ibdev_dbg(ibdev, "Couldn't query port err[%d]\n", err);
+		return err;
+	}
+
+	immutable->pkey_tbl_len = attr.pkey_tbl_len;
+	immutable->gid_tbl_len = attr.gid_tbl_len;
+
+	return 0;
+}
+
+static int efa_dealloc_uar(struct efa_dev *dev, u16 uarn)
+{
+	struct efa_com_dealloc_uar_params params = {
+		.uarn = uarn,
+	};
+
+	return efa_com_dealloc_uar(&dev->edev, &params);
+}
+
+int efa_alloc_ucontext(struct ib_ucontext *ibucontext, struct ib_udata *udata)
+{
+	struct efa_ucontext *ucontext = to_eucontext(ibucontext);
+	struct efa_dev *dev = to_edev(ibucontext->device);
+	struct efa_ibv_alloc_ucontext_resp resp = {};
+	struct efa_com_alloc_uar_result result;
+	int err;
+
+	/*
+	 * it's fine if the driver does not know all request fields,
+	 * we will ack input fields in our response.
+	 */
+
+	err = efa_com_alloc_uar(&dev->edev, &result);
+	if (err)
+		goto err_out;
+
+	ucontext->uarn = result.uarn;
+	xa_init(&ucontext->mmap_xa);
+
+	resp.cmds_supp_udata_mask |= EFA_USER_CMDS_SUPP_UDATA_QUERY_DEVICE;
+	resp.cmds_supp_udata_mask |= EFA_USER_CMDS_SUPP_UDATA_CREATE_AH;
+	resp.sub_cqs_per_cq = dev->dev_attr.sub_cqs_per_cq;
+	resp.inline_buf_size = dev->dev_attr.inline_buf_size;
+	resp.max_llq_size = dev->dev_attr.max_llq_size;
+
+	if (udata && udata->outlen) {
+		err = ib_copy_to_udata(udata, &resp,
+				       min(sizeof(resp), udata->outlen));
+		if (err)
+			goto err_dealloc_uar;
+	}
+
+	return 0;
+
+err_dealloc_uar:
+	efa_dealloc_uar(dev, result.uarn);
+err_out:
+	atomic64_inc(&dev->stats.sw_stats.alloc_ucontext_err);
+	return err;
+}
+
+void efa_dealloc_ucontext(struct ib_ucontext *ibucontext)
+{
+	struct efa_ucontext *ucontext = to_eucontext(ibucontext);
+	struct efa_dev *dev = to_edev(ibucontext->device);
+
+	mmap_entries_remove_free(dev, ucontext);
+	efa_dealloc_uar(dev, ucontext->uarn);
+}
+
+static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext,
+		      struct vm_area_struct *vma, u64 key, u64 length)
+{
+	struct efa_mmap_entry *entry;
+	unsigned long va;
+	u64 pfn;
+	int err;
+
+	entry = mmap_entry_get(dev, ucontext, key, length);
+	if (!entry) {
+		ibdev_dbg(&dev->ibdev, "key[%#llx] does not have valid entry\n",
+			  key);
+		return -EINVAL;
+	}
+
+	ibdev_dbg(&dev->ibdev,
+		  "Mapping address[%#llx], length[%#llx], mmap_flag[%d]\n",
+		  entry->address, length, entry->mmap_flag);
+
+	pfn = entry->address >> PAGE_SHIFT;
+	switch (entry->mmap_flag) {
+	case EFA_MMAP_IO_NC:
+		err = rdma_user_mmap_io(&ucontext->ibucontext, vma, pfn, length,
+					pgprot_noncached(vma->vm_page_prot));
+		break;
+	case EFA_MMAP_IO_WC:
+		err = rdma_user_mmap_io(&ucontext->ibucontext, vma, pfn, length,
+					pgprot_writecombine(vma->vm_page_prot));
+		break;
+	case EFA_MMAP_DMA_PAGE:
+		for (va = vma->vm_start; va < vma->vm_end;
+		     va += PAGE_SIZE, pfn++) {
+			err = vm_insert_page(vma, va, pfn_to_page(pfn));
+			if (err)
+				break;
+		}
+		break;
+	default:
+		err = -EINVAL;
+	}
+
+	if (err)
+		ibdev_dbg(
+			&dev->ibdev,
+			"Couldn't mmap address[%#llx] length[%#llx] mmap_flag[%d] err[%d]\n",
+			entry->address, length, entry->mmap_flag, err);
+
+	return err;
+}
+
+int efa_mmap(struct ib_ucontext *ibucontext,
+	     struct vm_area_struct *vma)
+{
+	struct efa_ucontext *ucontext = to_eucontext(ibucontext);
+	struct efa_dev *dev = to_edev(ibucontext->device);
+	u64 length = vma->vm_end - vma->vm_start;
+	u64 key = vma->vm_pgoff << PAGE_SHIFT;
+
+	ibdev_dbg(&dev->ibdev,
+		  "start %#lx, end %#lx, length = %#llx, key = %#llx\n",
+		  vma->vm_start, vma->vm_end, length, key);
+
+	if (length % PAGE_SIZE != 0 || !(vma->vm_flags & VM_SHARED)) {
+		ibdev_dbg(&dev->ibdev,
+			  "length[%#llx] is not page size aligned[%#lx] or VM_SHARED is not set [%#lx]\n",
+			  length, PAGE_SIZE, vma->vm_flags);
+		return -EINVAL;
+	}
+
+	if (vma->vm_flags & VM_EXEC) {
+		ibdev_dbg(&dev->ibdev, "Mapping executable pages is not permitted\n");
+		return -EPERM;
+	}
+	vma->vm_flags &= ~VM_MAYEXEC;
+
+	return __efa_mmap(dev, ucontext, vma, key, length);
+}
+
+static int efa_ah_destroy(struct efa_dev *dev, struct efa_ah *ah)
+{
+	struct efa_com_destroy_ah_params params = {
+		.ah = ah->ah,
+		.pdn = to_epd(ah->ibah.pd)->pdn,
+	};
+
+	return efa_com_destroy_ah(&dev->edev, &params);
+}
+
+int efa_create_ah(struct ib_ah *ibah,
+		  struct rdma_ah_attr *ah_attr,
+		  u32 flags,
+		  struct ib_udata *udata)
+{
+	struct efa_dev *dev = to_edev(ibah->device);
+	struct efa_com_create_ah_params params = {};
+	struct efa_ibv_create_ah_resp resp = {};
+	struct efa_com_create_ah_result result;
+	struct efa_ah *ah = to_eah(ibah);
+	int err;
+
+	if (!(flags & RDMA_CREATE_AH_SLEEPABLE)) {
+		ibdev_dbg(&dev->ibdev,
+			  "Create address handle is not supported in atomic context\n");
+		err = -EOPNOTSUPP;
+		goto err_out;
+	}
+
+	if (udata->inlen &&
+	    !ib_is_udata_cleared(udata, 0, udata->inlen)) {
+		ibdev_dbg(&dev->ibdev, "Incompatible ABI params\n");
+		err = -EINVAL;
+		goto err_out;
+	}
+
+	memcpy(params.dest_addr, ah_attr->grh.dgid.raw,
+	       sizeof(params.dest_addr));
+	params.pdn = to_epd(ibah->pd)->pdn;
+	err = efa_com_create_ah(&dev->edev, &params, &result);
+	if (err)
+		goto err_out;
+
+	memcpy(ah->id, ah_attr->grh.dgid.raw, sizeof(ah->id));
+	ah->ah = result.ah;
+
+	resp.efa_address_handle = result.ah;
+
+	if (udata->outlen) {
+		err = ib_copy_to_udata(udata, &resp,
+				       min(sizeof(resp), udata->outlen));
+		if (err) {
+			ibdev_dbg(&dev->ibdev,
+				  "Failed to copy udata for create_ah response\n");
+			goto err_destroy_ah;
+		}
+	}
+	ibdev_dbg(&dev->ibdev, "Created ah[%d]\n", ah->ah);
+
+	return 0;
+
+err_destroy_ah:
+	efa_ah_destroy(dev, ah);
+err_out:
+	atomic64_inc(&dev->stats.sw_stats.create_ah_err);
+	return err;
+}
+
+void efa_destroy_ah(struct ib_ah *ibah, u32 flags)
+{
+	struct efa_dev *dev = to_edev(ibah->pd->device);
+	struct efa_ah *ah = to_eah(ibah);
+
+	ibdev_dbg(&dev->ibdev, "Destroy ah[%d]\n", ah->ah);
+
+	if (!(flags & RDMA_DESTROY_AH_SLEEPABLE)) {
+		ibdev_dbg(&dev->ibdev,
+			  "Destroy address handle is not supported in atomic context\n");
+		return;
+	}
+
+	efa_ah_destroy(dev, ah);
+}
+
+enum rdma_link_layer efa_port_link_layer(struct ib_device *ibdev,
+					 u8 port_num)
+{
+	return IB_LINK_LAYER_UNSPECIFIED;
+}
+
diff --git a/drivers/infiniband/hw/hfi1/chip.c b/drivers/infiniband/hw/hfi1/chip.c
index addefae16c9c..310105d4e3de 100644
--- a/drivers/infiniband/hw/hfi1/chip.c
+++ b/drivers/infiniband/hw/hfi1/chip.c
@@ -4104,6 +4104,9 @@ def_access_ibp_counter(seq_naks);
 
 static struct cntr_entry dev_cntrs[DEV_CNTR_LAST] = {
 [C_RCV_OVF] = RXE32_DEV_CNTR_ELEM(RcvOverflow, RCV_BUF_OVFL_CNT, CNTR_SYNTH),
+[C_RX_LEN_ERR] = RXE32_DEV_CNTR_ELEM(RxLenErr, RCV_LENGTH_ERR_CNT, CNTR_SYNTH),
+[C_RX_ICRC_ERR] = RXE32_DEV_CNTR_ELEM(RxICrcErr, RCV_ICRC_ERR_CNT, CNTR_SYNTH),
+[C_RX_EBP] = RXE32_DEV_CNTR_ELEM(RxEbpCnt, RCV_EBP_CNT, CNTR_SYNTH),
 [C_RX_TID_FULL] = RXE32_DEV_CNTR_ELEM(RxTIDFullEr, RCV_TID_FULL_ERR_CNT,
 			CNTR_NORMAL),
 [C_RX_TID_INVALID] = RXE32_DEV_CNTR_ELEM(RxTIDInvalid, RCV_TID_VALID_ERR_CNT,
@@ -13294,15 +13297,18 @@ static int set_up_context_variables(struct hfi1_devdata *dd)
 	/*
 	 * The RMT entries are currently allocated as shown below:
 	 * 1. QOS (0 to 128 entries);
-	 * 2. FECN for PSM (num_user_contexts + num_vnic_contexts);
+	 * 2. FECN (num_kernel_context - 1 + num_user_contexts +
+	 *    num_vnic_contexts);
 	 * 3. VNIC (num_vnic_contexts).
-	 * It should be noted that PSM FECN oversubscribe num_vnic_contexts
+	 * It should be noted that FECN oversubscribe num_vnic_contexts
 	 * entries of RMT because both VNIC and PSM could allocate any receive
 	 * context between dd->first_dyn_alloc_text and dd->num_rcv_contexts,
 	 * and PSM FECN must reserve an RMT entry for each possible PSM receive
 	 * context.
 	 */
 	rmt_count = qos_rmt_entries(dd, NULL, NULL) + (num_vnic_contexts * 2);
+	if (HFI1_CAP_IS_KSET(TID_RDMA))
+		rmt_count += num_kernel_contexts - 1;
 	if (rmt_count + n_usr_ctxts > NUM_MAP_ENTRIES) {
 		user_rmt_reduced = NUM_MAP_ENTRIES - rmt_count;
 		dd_dev_err(dd,
@@ -14285,37 +14291,43 @@ bail:
 	init_qpmap_table(dd, FIRST_KERNEL_KCTXT, dd->n_krcv_queues - 1);
 }
 
-static void init_user_fecn_handling(struct hfi1_devdata *dd,
-				    struct rsm_map_table *rmt)
+static void init_fecn_handling(struct hfi1_devdata *dd,
+			       struct rsm_map_table *rmt)
 {
 	struct rsm_rule_data rrd;
 	u64 reg;
-	int i, idx, regoff, regidx;
+	int i, idx, regoff, regidx, start;
 	u8 offset;
 	u32 total_cnt;
 
+	if (HFI1_CAP_IS_KSET(TID_RDMA))
+		/* Exclude context 0 */
+		start = 1;
+	else
+		start = dd->first_dyn_alloc_ctxt;
+
+	total_cnt = dd->num_rcv_contexts - start;
+
 	/* there needs to be enough room in the map table */
-	total_cnt = dd->num_rcv_contexts - dd->first_dyn_alloc_ctxt;
 	if (rmt->used + total_cnt >= NUM_MAP_ENTRIES) {
-		dd_dev_err(dd, "User FECN handling disabled - too many user contexts allocated\n");
+		dd_dev_err(dd, "FECN handling disabled - too many contexts allocated\n");
 		return;
 	}
 
 	/*
 	 * RSM will extract the destination context as an index into the
 	 * map table.  The destination contexts are a sequential block
-	 * in the range first_dyn_alloc_ctxt...num_rcv_contexts-1 (inclusive).
+	 * in the range start...num_rcv_contexts-1 (inclusive).
 	 * Map entries are accessed as offset + extracted value.  Adjust
 	 * the added offset so this sequence can be placed anywhere in
 	 * the table - as long as the entries themselves do not wrap.
 	 * There are only enough bits in offset for the table size, so
 	 * start with that to allow for a "negative" offset.
 	 */
-	offset = (u8)(NUM_MAP_ENTRIES + (int)rmt->used -
-						(int)dd->first_dyn_alloc_ctxt);
+	offset = (u8)(NUM_MAP_ENTRIES + rmt->used - start);
 
-	for (i = dd->first_dyn_alloc_ctxt, idx = rmt->used;
-				i < dd->num_rcv_contexts; i++, idx++) {
+	for (i = start, idx = rmt->used; i < dd->num_rcv_contexts;
+	     i++, idx++) {
 		/* replace with identity mapping */
 		regoff = (idx % 8) * 8;
 		regidx = idx / 8;
@@ -14437,7 +14449,7 @@ static void init_rxe(struct hfi1_devdata *dd)
 	rmt = alloc_rsm_map_table(dd);
 	/* set up QOS, including the QPN map table */
 	init_qos(dd, rmt);
-	init_user_fecn_handling(dd, rmt);
+	init_fecn_handling(dd, rmt);
 	complete_rsm_map_table(dd, rmt);
 	/* record number of used rsm map entries for vnic */
 	dd->vnic.rmt_start = rmt->used;
@@ -14663,8 +14675,8 @@ void hfi1_start_cleanup(struct hfi1_devdata *dd)
  */
 static int init_asic_data(struct hfi1_devdata *dd)
 {
-	unsigned long flags;
-	struct hfi1_devdata *tmp, *peer = NULL;
+	unsigned long index;
+	struct hfi1_devdata *peer;
 	struct hfi1_asic_data *asic_data;
 	int ret = 0;
 
@@ -14673,14 +14685,12 @@ static int init_asic_data(struct hfi1_devdata *dd)
 	if (!asic_data)
 		return -ENOMEM;
 
-	spin_lock_irqsave(&hfi1_devs_lock, flags);
+	xa_lock_irq(&hfi1_dev_table);
 	/* Find our peer device */
-	list_for_each_entry(tmp, &hfi1_dev_list, list) {
-		if ((HFI_BASE_GUID(dd) == HFI_BASE_GUID(tmp)) &&
-		    dd->unit != tmp->unit) {
-			peer = tmp;
+	xa_for_each(&hfi1_dev_table, index, peer) {
+		if ((HFI_BASE_GUID(dd) == HFI_BASE_GUID(peer)) &&
+		    dd->unit != peer->unit)
 			break;
-		}
 	}
 
 	if (peer) {
@@ -14692,7 +14702,7 @@ static int init_asic_data(struct hfi1_devdata *dd)
 		mutex_init(&dd->asic_data->asic_resource_mutex);
 	}
 	dd->asic_data->dds[dd->hfi1_id] = dd; /* self back-pointer */
-	spin_unlock_irqrestore(&hfi1_devs_lock, flags);
+	xa_unlock_irq(&hfi1_dev_table);
 
 	/* first one through - set up i2c devices */
 	if (!peer)
diff --git a/drivers/infiniband/hw/hfi1/chip.h b/drivers/infiniband/hw/hfi1/chip.h
index 6c27c1c6a868..4e6c3556ec48 100644
--- a/drivers/infiniband/hw/hfi1/chip.h
+++ b/drivers/infiniband/hw/hfi1/chip.h
@@ -858,6 +858,9 @@ static inline int idx_from_vl(int vl)
 /* Per device counter indexes */
 enum {
 	C_RCV_OVF = 0,
+	C_RX_LEN_ERR,
+	C_RX_ICRC_ERR,
+	C_RX_EBP,
 	C_RX_TID_FULL,
 	C_RX_TID_INVALID,
 	C_RX_TID_FLGMS,
diff --git a/drivers/infiniband/hw/hfi1/chip_registers.h b/drivers/infiniband/hw/hfi1/chip_registers.h
index c0800ea5a3f8..ab3589d17aee 100644
--- a/drivers/infiniband/hw/hfi1/chip_registers.h
+++ b/drivers/infiniband/hw/hfi1/chip_registers.h
@@ -380,6 +380,9 @@
 #define DC_LCB_PRF_TX_FLIT_CNT (DC_LCB_CSRS + 0x000000000418)
 #define DC_LCB_STS_LINK_TRANSFER_ACTIVE (DC_LCB_CSRS + 0x000000000468)
 #define DC_LCB_STS_ROUND_TRIP_LTP_CNT (DC_LCB_CSRS + 0x0000000004B0)
+#define RCV_LENGTH_ERR_CNT 0
+#define RCV_ICRC_ERR_CNT 6
+#define RCV_EBP_CNT 9
 #define RCV_BUF_OVFL_CNT 10
 #define RCV_CONTEXT_EGR_STALL 22
 #define RCV_DATA_PKT_CNT 0
diff --git a/drivers/infiniband/hw/hfi1/common.h b/drivers/infiniband/hw/hfi1/common.h
index 7310a5dba420..d47da7b0438f 100644
--- a/drivers/infiniband/hw/hfi1/common.h
+++ b/drivers/infiniband/hw/hfi1/common.h
@@ -286,7 +286,7 @@ struct diag_pkt {
 #define RHF_TID_ERR		(0x1ull << 59)
 #define RHF_LEN_ERR		(0x1ull << 60)
 #define RHF_ECC_ERR		(0x1ull << 61)
-#define RHF_VCRC_ERR		(0x1ull << 62)
+#define RHF_RESERVED		(0x1ull << 62)
 #define RHF_ICRC_ERR		(0x1ull << 63)
 
 #define RHF_ERROR_SMASK 0xffe0000000000000ull		/* bits 63:53 */
diff --git a/drivers/infiniband/hw/hfi1/debugfs.c b/drivers/infiniband/hw/hfi1/debugfs.c
index 427ba0ce74a5..15efb4a380b2 100644
--- a/drivers/infiniband/hw/hfi1/debugfs.c
+++ b/drivers/infiniband/hw/hfi1/debugfs.c
@@ -1080,6 +1080,77 @@ static int qsfp2_debugfs_release(struct inode *in, struct file *fp)
 	return __qsfp_debugfs_release(in, fp, 1);
 }
 
+#define EXPROM_WRITE_ENABLE BIT_ULL(14)
+
+static bool exprom_wp_disabled;
+
+static int exprom_wp_set(struct hfi1_devdata *dd, bool disable)
+{
+	u64 gpio_val = 0;
+
+	if (disable) {
+		gpio_val = EXPROM_WRITE_ENABLE;
+		exprom_wp_disabled = true;
+		dd_dev_info(dd, "Disable Expansion ROM Write Protection\n");
+	} else {
+		exprom_wp_disabled = false;
+		dd_dev_info(dd, "Enable Expansion ROM Write Protection\n");
+	}
+
+	write_csr(dd, ASIC_GPIO_OUT, gpio_val);
+	write_csr(dd, ASIC_GPIO_OE, gpio_val);
+
+	return 0;
+}
+
+static ssize_t exprom_wp_debugfs_read(struct file *file, char __user *buf,
+				      size_t count, loff_t *ppos)
+{
+	return 0;
+}
+
+static ssize_t exprom_wp_debugfs_write(struct file *file,
+				       const char __user *buf, size_t count,
+				       loff_t *ppos)
+{
+	struct hfi1_pportdata *ppd = private2ppd(file);
+	char cdata;
+
+	if (count != 1)
+		return -EINVAL;
+	if (get_user(cdata, buf))
+		return -EFAULT;
+	if (cdata == '0')
+		exprom_wp_set(ppd->dd, false);
+	else if (cdata == '1')
+		exprom_wp_set(ppd->dd, true);
+	else
+		return -EINVAL;
+
+	return 1;
+}
+
+static unsigned long exprom_in_use;
+
+static int exprom_wp_debugfs_open(struct inode *in, struct file *fp)
+{
+	if (test_and_set_bit(0, &exprom_in_use))
+		return -EBUSY;
+
+	return 0;
+}
+
+static int exprom_wp_debugfs_release(struct inode *in, struct file *fp)
+{
+	struct hfi1_pportdata *ppd = private2ppd(fp);
+
+	if (exprom_wp_disabled)
+		exprom_wp_set(ppd->dd, false);
+	clear_bit(0, &exprom_in_use);
+
+	return 0;
+}
+
 #define DEBUGFS_OPS(nm, readroutine, writeroutine)	\
 { \
 	.name = nm, \
@@ -1119,6 +1190,9 @@ static const struct counter_info port_cntr_ops[] = {
 		     qsfp1_debugfs_open, qsfp1_debugfs_release),
 	DEBUGFS_XOPS("qsfp2", qsfp2_debugfs_read, qsfp2_debugfs_write,
 		     qsfp2_debugfs_open, qsfp2_debugfs_release),
+	DEBUGFS_XOPS("exprom_wp", exprom_wp_debugfs_read,
+		     exprom_wp_debugfs_write, exprom_wp_debugfs_open,
+		     exprom_wp_debugfs_release),
 	DEBUGFS_OPS("asic_flags", asic_flags_read, asic_flags_write),
 	DEBUGFS_OPS("dc8051_memory", dc8051_memory_read, NULL),
 	DEBUGFS_OPS("lcb", debugfs_lcb_read, debugfs_lcb_write),
@@ -1302,15 +1376,15 @@ static void _driver_stats_seq_stop(struct seq_file *s, void *v)
 
 static u64 hfi1_sps_ints(void)
 {
-	unsigned long flags;
+	unsigned long index, flags;
 	struct hfi1_devdata *dd;
 	u64 sps_ints = 0;
 
-	spin_lock_irqsave(&hfi1_devs_lock, flags);
-	list_for_each_entry(dd, &hfi1_dev_list, list) {
+	xa_lock_irqsave(&hfi1_dev_table, flags);
+	xa_for_each(&hfi1_dev_table, index, dd) {
 		sps_ints += get_all_cpu_total(dd->int_counter);
 	}
-	spin_unlock_irqrestore(&hfi1_devs_lock, flags);
+	xa_unlock_irqrestore(&hfi1_dev_table, flags);
 	return sps_ints;
 }
 
diff --git a/drivers/infiniband/hw/hfi1/driver.c b/drivers/infiniband/hw/hfi1/driver.c
index 2a9d2912f5db..01aa1f132f55 100644
--- a/drivers/infiniband/hw/hfi1/driver.c
+++ b/drivers/infiniband/hw/hfi1/driver.c
@@ -72,8 +72,6 @@
  */
 const char ib_hfi1_version[] = HFI1_DRIVER_VERSION "\n";
 
-DEFINE_SPINLOCK(hfi1_devs_lock);
-LIST_HEAD(hfi1_dev_list);
 DEFINE_MUTEX(hfi1_mutex);	/* general driver use */
 
 unsigned int hfi1_max_mtu = HFI1_DEFAULT_MAX_MTU;
@@ -175,11 +173,11 @@ int hfi1_count_active_units(void)
 {
 	struct hfi1_devdata *dd;
 	struct hfi1_pportdata *ppd;
-	unsigned long flags;
+	unsigned long index, flags;
 	int pidx, nunits_active = 0;
 
-	spin_lock_irqsave(&hfi1_devs_lock, flags);
-	list_for_each_entry(dd, &hfi1_dev_list, list) {
+	xa_lock_irqsave(&hfi1_dev_table, flags);
+	xa_for_each(&hfi1_dev_table, index, dd) {
 		if (!(dd->flags & HFI1_PRESENT) || !dd->kregbase1)
 			continue;
 		for (pidx = 0; pidx < dd->num_pports; ++pidx) {
@@ -190,7 +188,7 @@ int hfi1_count_active_units(void)
 			}
 		}
 	}
-	spin_unlock_irqrestore(&hfi1_devs_lock, flags);
+	xa_unlock_irqrestore(&hfi1_dev_table, flags);
 	return nunits_active;
 }
 
@@ -264,7 +262,7 @@ static void rcv_hdrerr(struct hfi1_ctxtdata *rcd, struct hfi1_pportdata *ppd,
 	    hfi1_dbg_fault_suppress_err(verbs_dev))
 		return;
 
-	if (packet->rhf & (RHF_VCRC_ERR | RHF_ICRC_ERR))
+	if (packet->rhf & RHF_ICRC_ERR)
 		return;
 
 	if (packet->etype == RHF_RCV_TYPE_BYPASS) {
@@ -516,7 +514,9 @@ bool hfi1_process_ecn_slowpath(struct rvt_qp *qp, struct hfi1_packet *pkt,
 	 */
 	do_cnp = prescan ||
 		(opcode >= IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST &&
-		 opcode <= IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE);
+		 opcode <= IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE) ||
+		opcode == TID_OP(READ_RESP) ||
+		opcode == TID_OP(ACK);
 
 	/* Call appropriate CNP handler */
 	if (!ignore_fecn && do_cnp && fecn)
@@ -1581,7 +1581,7 @@ static void show_eflags_errs(struct hfi1_packet *packet)
 	u32 rte = rhf_rcv_type_err(packet->rhf);
 
 	dd_dev_err(rcd->dd,
-		   "receive context %d: rhf 0x%016llx, errs [ %s%s%s%s%s%s%s%s] rte 0x%x\n",
+		   "receive context %d: rhf 0x%016llx, errs [ %s%s%s%s%s%s%s] rte 0x%x\n",
 		   rcd->ctxt, packet->rhf,
 		   packet->rhf & RHF_K_HDR_LEN_ERR ? "k_hdr_len " : "",
 		   packet->rhf & RHF_DC_UNC_ERR ? "dc_unc " : "",
@@ -1589,7 +1589,6 @@ static void show_eflags_errs(struct hfi1_packet *packet)
 		   packet->rhf & RHF_TID_ERR ? "tid " : "",
 		   packet->rhf & RHF_LEN_ERR ? "len " : "",
 		   packet->rhf & RHF_ECC_ERR ? "ecc " : "",
-		   packet->rhf & RHF_VCRC_ERR ? "vcrc " : "",
 		   packet->rhf & RHF_ICRC_ERR ? "icrc " : "",
 		   rte);
 }
diff --git a/drivers/infiniband/hw/hfi1/exp_rcv.c b/drivers/infiniband/hw/hfi1/exp_rcv.c
index 1be49a0d9c11..e9d5cc8b771a 100644
--- a/drivers/infiniband/hw/hfi1/exp_rcv.c
+++ b/drivers/infiniband/hw/hfi1/exp_rcv.c
@@ -112,9 +112,6 @@ int hfi1_alloc_ctxt_rcv_groups(struct hfi1_ctxtdata *rcd)
  */
 void hfi1_free_ctxt_rcv_groups(struct hfi1_ctxtdata *rcd)
 {
-	WARN_ON(!EXP_TID_SET_EMPTY(rcd->tid_full_list));
-	WARN_ON(!EXP_TID_SET_EMPTY(rcd->tid_used_list));
-
 	kfree(rcd->groups);
 	rcd->groups = NULL;
 	hfi1_exp_tid_group_init(rcd);
diff --git a/drivers/infiniband/hw/hfi1/hfi.h b/drivers/infiniband/hw/hfi1/hfi.h
index 048b5d73ba39..b458c218842b 100644
--- a/drivers/infiniband/hw/hfi1/hfi.h
+++ b/drivers/infiniband/hw/hfi1/hfi.h
@@ -54,7 +54,6 @@
 #include <linux/list.h>
 #include <linux/scatterlist.h>
 #include <linux/slab.h>
-#include <linux/idr.h>
 #include <linux/io.h>
 #include <linux/fs.h>
 #include <linux/completion.h>
@@ -65,6 +64,7 @@
 #include <linux/kthread.h>
 #include <linux/i2c.h>
 #include <linux/i2c-algo-bit.h>
+#include <linux/xarray.h>
 #include <rdma/ib_hdrs.h>
 #include <rdma/opa_addr.h>
 #include <linux/rhashtable.h>
@@ -1021,8 +1021,8 @@ struct hfi1_asic_data {
 struct hfi1_vnic_data {
 	struct hfi1_ctxtdata *ctxt[HFI1_NUM_VNIC_CTXT];
 	struct kmem_cache *txreq_cache;
+	struct xarray vesws;
 	u8 num_vports;
-	struct idr vesw_idr;
 	u8 rmt_start;
 	u8 num_ctxt;
 };
@@ -1041,7 +1041,6 @@ struct sdma_vl_map;
 typedef int (*send_routine)(struct rvt_qp *, struct hfi1_pkt_state *, u64);
 struct hfi1_devdata {
 	struct hfi1_ibdev verbs_dev;     /* must be first */
-	struct list_head list;
 	/* pointers to related structs for this device */
 	/* pci access data structure */
 	struct pci_dev *pcidev;
@@ -1426,8 +1425,7 @@ struct hfi1_filedata {
 	struct mm_struct *mm;
 };
 
-extern struct list_head hfi1_dev_list;
-extern spinlock_t hfi1_devs_lock;
+extern struct xarray hfi1_dev_table;
 struct hfi1_devdata *hfi1_lookup(int unit);
 
 static inline unsigned long uctxt_offset(struct hfi1_ctxtdata *uctxt)
diff --git a/drivers/infiniband/hw/hfi1/init.c b/drivers/infiniband/hw/hfi1/init.c
index faaaac8fbc55..71cb9525c074 100644
--- a/drivers/infiniband/hw/hfi1/init.c
+++ b/drivers/infiniband/hw/hfi1/init.c
@@ -49,7 +49,7 @@
 #include <linux/netdevice.h>
 #include <linux/vmalloc.h>
 #include <linux/delay.h>
-#include <linux/idr.h>
+#include <linux/xarray.h>
 #include <linux/module.h>
 #include <linux/printk.h>
 #include <linux/hrtimer.h>
@@ -124,7 +124,7 @@ MODULE_PARM_DESC(user_credit_return_threshold, "Credit return threshold for user
 
 static inline u64 encode_rcv_header_entry_size(u16 size);
 
-static struct idr hfi1_unit_table;
+DEFINE_XARRAY_FLAGS(hfi1_dev_table, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_IRQ);
 
 static int hfi1_create_kctxt(struct hfi1_devdata *dd,
 			     struct hfi1_pportdata *ppd)
@@ -469,7 +469,7 @@ int hfi1_create_ctxtdata(struct hfi1_pportdata *ppd, int numa,
 		if (rcd->egrbufs.size < hfi1_max_mtu) {
 			rcd->egrbufs.size = __roundup_pow_of_two(hfi1_max_mtu);
 			hfi1_cdbg(PROC,
-				  "ctxt%u: eager bufs size too small. Adjusting to %zu\n",
+				  "ctxt%u: eager bufs size too small. Adjusting to %u\n",
 				    rcd->ctxt, rcd->egrbufs.size);
 		}
 		rcd->egrbufs.rcvtid_size = HFI1_MAX_EAGER_BUFFER_SIZE;
@@ -805,7 +805,8 @@ static int create_workqueues(struct hfi1_devdata *dd)
 			ppd->hfi1_wq =
 				alloc_workqueue(
 				    "hfi%d_%d",
-				    WQ_SYSFS | WQ_HIGHPRI | WQ_CPU_INTENSIVE,
+				    WQ_SYSFS | WQ_HIGHPRI | WQ_CPU_INTENSIVE |
+				    WQ_MEM_RECLAIM,
 				    HFI1_MAX_ACTIVE_WORKQUEUE_ENTRIES,
 				    dd->unit, pidx);
 			if (!ppd->hfi1_wq)
@@ -1018,21 +1019,9 @@ done:
 	return ret;
 }
 
-static inline struct hfi1_devdata *__hfi1_lookup(int unit)
-{
-	return idr_find(&hfi1_unit_table, unit);
-}
-
 struct hfi1_devdata *hfi1_lookup(int unit)
 {
-	struct hfi1_devdata *dd;
-	unsigned long flags;
-
-	spin_lock_irqsave(&hfi1_devs_lock, flags);
-	dd = __hfi1_lookup(unit);
-	spin_unlock_irqrestore(&hfi1_devs_lock, flags);
-
-	return dd;
+	return xa_load(&hfi1_dev_table, unit);
 }
 
 /*
@@ -1200,7 +1189,7 @@ void hfi1_free_ctxtdata(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd)
 /*
  * Release our hold on the shared asic data.  If we are the last one,
  * return the structure to be finalized outside the lock.  Must be
- * holding hfi1_devs_lock.
+ * holding hfi1_dev_table lock.
  */
 static struct hfi1_asic_data *release_asic_data(struct hfi1_devdata *dd)
 {
@@ -1236,13 +1225,10 @@ static void hfi1_clean_devdata(struct hfi1_devdata *dd)
 	struct hfi1_asic_data *ad;
 	unsigned long flags;
 
-	spin_lock_irqsave(&hfi1_devs_lock, flags);
-	if (!list_empty(&dd->list)) {
-		idr_remove(&hfi1_unit_table, dd->unit);
-		list_del_init(&dd->list);
-	}
+	xa_lock_irqsave(&hfi1_dev_table, flags);
+	__xa_erase(&hfi1_dev_table, dd->unit);
 	ad = release_asic_data(dd);
-	spin_unlock_irqrestore(&hfi1_devs_lock, flags);
+	xa_unlock_irqrestore(&hfi1_dev_table, flags);
 
 	finalize_asic_data(dd, ad);
 	free_platform_config(dd);
@@ -1286,13 +1272,10 @@ void hfi1_free_devdata(struct hfi1_devdata *dd)
  * Must be done via verbs allocator, because the verbs cleanup process
  * both does cleanup and free of the data structure.
  * "extra" is for chip-specific data.
- *
- * Use the idr mechanism to get a unit number for this unit.
  */
 static struct hfi1_devdata *hfi1_alloc_devdata(struct pci_dev *pdev,
 					       size_t extra)
 {
-	unsigned long flags;
 	struct hfi1_devdata *dd;
 	int ret, nports;
 
@@ -1307,21 +1290,10 @@ static struct hfi1_devdata *hfi1_alloc_devdata(struct pci_dev *pdev,
 	dd->pport = (struct hfi1_pportdata *)(dd + 1);
 	dd->pcidev = pdev;
 	pci_set_drvdata(pdev, dd);
-
-	INIT_LIST_HEAD(&dd->list);
-	idr_preload(GFP_KERNEL);
-	spin_lock_irqsave(&hfi1_devs_lock, flags);
-
-	ret = idr_alloc(&hfi1_unit_table, dd, 0, 0, GFP_NOWAIT);
-	if (ret >= 0) {
-		dd->unit = ret;
-		list_add(&dd->list, &hfi1_dev_list);
-	}
 	dd->node = NUMA_NO_NODE;
 
-	spin_unlock_irqrestore(&hfi1_devs_lock, flags);
-	idr_preload_end();
-
+	ret = xa_alloc_irq(&hfi1_dev_table, &dd->unit, dd, xa_limit_32b,
+			GFP_KERNEL);
 	if (ret < 0) {
 		dev_err(&pdev->dev,
 			"Could not allocate unit ID: error %d\n", -ret);
@@ -1522,8 +1494,6 @@ static int __init hfi1_mod_init(void)
 	 * These must be called before the driver is registered with
 	 * the PCI subsystem.
 	 */
-	idr_init(&hfi1_unit_table);
-
 	hfi1_dbg_init();
 	ret = pci_register_driver(&hfi1_pci_driver);
 	if (ret < 0) {
@@ -1534,7 +1504,6 @@ static int __init hfi1_mod_init(void)
 
 bail_dev:
 	hfi1_dbg_exit();
-	idr_destroy(&hfi1_unit_table);
 	dev_cleanup();
 bail:
 	return ret;
@@ -1552,7 +1521,7 @@ static void __exit hfi1_mod_cleanup(void)
 	node_affinity_destroy_all();
 	hfi1_dbg_exit();
 
-	idr_destroy(&hfi1_unit_table);
+	WARN_ON(!xa_empty(&hfi1_dev_table));
 	dispose_firmware();	/* asymmetric with obtain_firmware() */
 	dev_cleanup();
 }
@@ -2071,7 +2040,7 @@ int hfi1_setup_eagerbufs(struct hfi1_ctxtdata *rcd)
 	rcd->egrbufs.size = alloced_bytes;
 
 	hfi1_cdbg(PROC,
-		  "ctxt%u: Alloced %u rcv tid entries @ %uKB, total %zuKB\n",
+		  "ctxt%u: Alloced %u rcv tid entries @ %uKB, total %uKB\n",
 		  rcd->ctxt, rcd->egrbufs.alloced,
 		  rcd->egrbufs.rcvtid_size / 1024, rcd->egrbufs.size / 1024);
 
diff --git a/drivers/infiniband/hw/hfi1/opfn.h b/drivers/infiniband/hw/hfi1/opfn.h
index 5f2011cabc25..62f93c1dc082 100644
--- a/drivers/infiniband/hw/hfi1/opfn.h
+++ b/drivers/infiniband/hw/hfi1/opfn.h
@@ -47,12 +47,14 @@
  * for future transactions
  */
 
+#include <linux/workqueue.h>
+#include <rdma/ib_verbs.h>
+#include <rdma/rdmavt_qp.h>
+
 /* STL Verbs Extended */
 #define IB_BTHE_E_SHIFT           24
 #define HFI1_VERBS_E_ATOMIC_VADDR U64_MAX
 
-struct ib_atomic_eth;
-
 enum hfi1_opfn_codes {
 	STL_VERBS_EXTD_NONE = 0,
 	STL_VERBS_EXTD_TID_RDMA,
diff --git a/drivers/infiniband/hw/hfi1/qp.c b/drivers/infiniband/hw/hfi1/qp.c
index eba300330a02..4e0e9fc0a777 100644
--- a/drivers/infiniband/hw/hfi1/qp.c
+++ b/drivers/infiniband/hw/hfi1/qp.c
@@ -742,6 +742,8 @@ void *qp_priv_alloc(struct rvt_dev_info *rdi, struct rvt_qp *qp)
 		iowait_wakeup,
 		iowait_sdma_drained,
 		hfi1_init_priority);
+	/* Init to a value to start the running average correctly */
+	priv->s_running_pkt_size = piothreshold / 2;
 	return priv;
 }
 
diff --git a/drivers/infiniband/hw/hfi1/rc.c b/drivers/infiniband/hw/hfi1/rc.c
index 5991211d72bd..a922edcf23d6 100644
--- a/drivers/infiniband/hw/hfi1/rc.c
+++ b/drivers/infiniband/hw/hfi1/rc.c
@@ -140,10 +140,7 @@ static int make_rc_ack(struct hfi1_ibdev *dev, struct rvt_qp *qp,
 	case OP(RDMA_READ_RESPONSE_LAST):
 	case OP(RDMA_READ_RESPONSE_ONLY):
 		e = &qp->s_ack_queue[qp->s_tail_ack_queue];
-		if (e->rdma_sge.mr) {
-			rvt_put_mr(e->rdma_sge.mr);
-			e->rdma_sge.mr = NULL;
-		}
+		release_rdma_sge_mr(e);
 		/* FALLTHROUGH */
 	case OP(ATOMIC_ACKNOWLEDGE):
 		/*
@@ -343,7 +340,8 @@ write_resp:
 			break;
 
 		e->sent = 1;
-		qp->s_ack_state = OP(RDMA_READ_RESPONSE_LAST);
+		/* Do not free e->rdma_sge until all data are received */
+		qp->s_ack_state = OP(ATOMIC_ACKNOWLEDGE);
 		break;
 
 	case TID_OP(READ_RESP):
@@ -1836,7 +1834,7 @@ void hfi1_rc_send_complete(struct rvt_qp *qp, struct hfi1_opa_header *opah)
 		qp->s_last = s_last;
 		/* see post_send() */
 		barrier();
-		rvt_put_swqe(wqe);
+		rvt_put_qp_swqe(qp, wqe);
 		rvt_qp_swqe_complete(qp,
 				     wqe,
 				     ib_hfi1_wc_opcode[wqe->wr.opcode],
@@ -1884,7 +1882,7 @@ struct rvt_swqe *do_rc_completion(struct rvt_qp *qp,
 		u32 s_last;
 
 		trdma_clean_swqe(qp, wqe);
-		rvt_put_swqe(wqe);
+		rvt_put_qp_swqe(qp, wqe);
 		rvt_qp_wqe_unreserve(qp, wqe);
 		s_last = qp->s_last;
 		trace_hfi1_qp_send_completion(qp, wqe, s_last);
@@ -2643,10 +2641,7 @@ static noinline int rc_rcv_error(struct ib_other_headers *ohdr, void *data,
 		len = be32_to_cpu(reth->length);
 		if (unlikely(offset + len != e->rdma_sge.sge_length))
 			goto unlock_done;
-		if (e->rdma_sge.mr) {
-			rvt_put_mr(e->rdma_sge.mr);
-			e->rdma_sge.mr = NULL;
-		}
+		release_rdma_sge_mr(e);
 		if (len != 0) {
 			u32 rkey = be32_to_cpu(reth->rkey);
 			u64 vaddr = get_ib_reth_vaddr(reth);
@@ -3088,10 +3083,7 @@ send_last:
 			update_ack_queue(qp, next);
 		}
 		e = &qp->s_ack_queue[qp->r_head_ack_queue];
-		if (e->rdma_sge.mr) {
-			rvt_put_mr(e->rdma_sge.mr);
-			e->rdma_sge.mr = NULL;
-		}
+		release_rdma_sge_mr(e);
 		reth = &ohdr->u.rc.reth;
 		len = be32_to_cpu(reth->length);
 		if (len) {
@@ -3166,10 +3158,7 @@ send_last:
 			update_ack_queue(qp, next);
 		}
 		e = &qp->s_ack_queue[qp->r_head_ack_queue];
-		if (e->rdma_sge.mr) {
-			rvt_put_mr(e->rdma_sge.mr);
-			e->rdma_sge.mr = NULL;
-		}
+		release_rdma_sge_mr(e);
 		/* Process OPFN special virtual address */
 		if (opfn) {
 			opfn_conn_response(qp, e, ateth);
diff --git a/drivers/infiniband/hw/hfi1/rc.h b/drivers/infiniband/hw/hfi1/rc.h
index 8e0935b9bf2a..5ed5e85d5841 100644
--- a/drivers/infiniband/hw/hfi1/rc.h
+++ b/drivers/infiniband/hw/hfi1/rc.h
@@ -41,6 +41,14 @@ static inline u32 restart_sge(struct rvt_sge_state *ss, struct rvt_swqe *wqe,
 	return rvt_restart_sge(ss, wqe, len);
 }
 
+static inline void release_rdma_sge_mr(struct rvt_ack_entry *e)
+{
+	if (e->rdma_sge.mr) {
+		rvt_put_mr(e->rdma_sge.mr);
+		e->rdma_sge.mr = NULL;
+	}
+}
+
 struct rvt_ack_entry *find_prev_entry(struct rvt_qp *qp, u32 psn, u8 *prev,
 				      u8 *prev_ack, bool *scheduled);
 int do_rc_ack(struct rvt_qp *qp, u32 aeth, u32 psn, int opcode, u64 val,
diff --git a/drivers/infiniband/hw/hfi1/ruc.c b/drivers/infiniband/hw/hfi1/ruc.c
index 124a3ec1e15c..23ac6057b211 100644
--- a/drivers/infiniband/hw/hfi1/ruc.c
+++ b/drivers/infiniband/hw/hfi1/ruc.c
@@ -524,7 +524,7 @@ void _hfi1_do_send(struct work_struct *work)
 
 /**
  * hfi1_do_send - perform a send on a QP
- * @work: contains a pointer to the QP
+ * @qp: a pointer to the QP
  * @in_thread: true if in a workqueue thread
  *
  * Process entries in the send work queue until credit or queue is
diff --git a/drivers/infiniband/hw/hfi1/tid_rdma.c b/drivers/infiniband/hw/hfi1/tid_rdma.c
index 43cbce7a19ea..6fb93032fbef 100644
--- a/drivers/infiniband/hw/hfi1/tid_rdma.c
+++ b/drivers/infiniband/hw/hfi1/tid_rdma.c
@@ -67,8 +67,6 @@ static u32 mask_generation(u32 a)
 #define TID_RDMA_DESTQP_FLOW_SHIFT      11
 #define TID_RDMA_DESTQP_FLOW_MASK       0x1f
 
-#define TID_FLOW_SW_PSN BIT(0)
-
 #define TID_OPFN_QP_CTXT_MASK 0xff
 #define TID_OPFN_QP_CTXT_SHIFT 56
 #define TID_OPFN_QP_KDETH_MASK 0xff
@@ -128,6 +126,15 @@ static int make_tid_rdma_ack(struct rvt_qp *qp,
 			     struct ib_other_headers *ohdr,
 			     struct hfi1_pkt_state *ps);
 static void hfi1_do_tid_send(struct rvt_qp *qp);
+static u32 read_r_next_psn(struct hfi1_devdata *dd, u8 ctxt, u8 fidx);
+static void tid_rdma_rcv_err(struct hfi1_packet *packet,
+			     struct ib_other_headers *ohdr,
+			     struct rvt_qp *qp, u32 psn, int diff, bool fecn);
+static void update_r_next_psn_fecn(struct hfi1_packet *packet,
+				   struct hfi1_qp_priv *priv,
+				   struct hfi1_ctxtdata *rcd,
+				   struct tid_rdma_flow *flow,
+				   bool fecn);
 
 static u64 tid_rdma_opfn_encode(struct tid_rdma_params *p)
 {
@@ -776,7 +783,6 @@ int hfi1_kern_setup_hw_flow(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp)
 		rcd->flows[fs->index].generation = fs->generation;
 	fs->generation = kern_setup_hw_flow(rcd, fs->index);
 	fs->psn = 0;
-	fs->flags = 0;
 	dequeue_tid_waiter(rcd, &rcd->flow_queue, qp);
 	/* get head before dropping lock */
 	fqp = first_qp(rcd, &rcd->flow_queue);
@@ -1807,6 +1813,7 @@ sync_check:
 			goto done;
 
 		hfi1_kern_clear_hw_flow(req->rcd, qp);
+		qpriv->s_flags &= ~HFI1_R_TID_SW_PSN;
 		req->state = TID_REQUEST_ACTIVE;
 	}
 
@@ -2036,10 +2043,7 @@ static int tid_rdma_rcv_error(struct hfi1_packet *packet,
 		if (psn != e->psn || len != req->total_len)
 			goto unlock;
 
-		if (e->rdma_sge.mr) {
-			rvt_put_mr(e->rdma_sge.mr);
-			e->rdma_sge.mr = NULL;
-		}
+		release_rdma_sge_mr(e);
 
 		rkey = be32_to_cpu(reth->rkey);
 		vaddr = get_ib_reth_vaddr(reth);
@@ -2238,7 +2242,7 @@ void hfi1_rc_rcv_tid_rdma_read_req(struct hfi1_packet *packet)
 	struct ib_reth *reth;
 	struct hfi1_qp_priv *qpriv = qp->priv;
 	u32 bth0, psn, len, rkey;
-	bool is_fecn;
+	bool fecn;
 	u8 next;
 	u64 vaddr;
 	int diff;
@@ -2248,7 +2252,7 @@ void hfi1_rc_rcv_tid_rdma_read_req(struct hfi1_packet *packet)
 	if (hfi1_ruc_check_hdr(ibp, packet))
 		return;
 
-	is_fecn = process_ecn(qp, packet);
+	fecn = process_ecn(qp, packet);
 	psn = mask_psn(be32_to_cpu(ohdr->bth[2]));
 	trace_hfi1_rsp_rcv_tid_read_req(qp, psn);
 
@@ -2267,9 +2271,8 @@ void hfi1_rc_rcv_tid_rdma_read_req(struct hfi1_packet *packet)
 
 	diff = delta_psn(psn, qp->r_psn);
 	if (unlikely(diff)) {
-		if (tid_rdma_rcv_error(packet, ohdr, qp, psn, diff))
-			return;
-		goto send_ack;
+		tid_rdma_rcv_err(packet, ohdr, qp, psn, diff, fecn);
+		return;
 	}
 
 	/* We've verified the request, insert it into the ack queue. */
@@ -2285,10 +2288,7 @@ void hfi1_rc_rcv_tid_rdma_read_req(struct hfi1_packet *packet)
 		update_ack_queue(qp, next);
 	}
 	e = &qp->s_ack_queue[qp->r_head_ack_queue];
-	if (e->rdma_sge.mr) {
-		rvt_put_mr(e->rdma_sge.mr);
-		e->rdma_sge.mr = NULL;
-	}
+	release_rdma_sge_mr(e);
 
 	rkey = be32_to_cpu(reth->rkey);
 	qp->r_len = len;
@@ -2324,11 +2324,11 @@ void hfi1_rc_rcv_tid_rdma_read_req(struct hfi1_packet *packet)
 
 	/* Schedule the send tasklet. */
 	qp->s_flags |= RVT_S_RESP_PENDING;
+	if (fecn)
+		qp->s_flags |= RVT_S_ECN;
 	hfi1_schedule_send(qp);
 
 	spin_unlock_irqrestore(&qp->s_lock, flags);
-	if (is_fecn)
-		goto send_ack;
 	return;
 
 nack_inv_unlock:
@@ -2345,8 +2345,6 @@ nack_acc:
 	rvt_rc_error(qp, IB_WC_LOC_PROT_ERR);
 	qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR;
 	qp->r_ack_psn = qp->r_psn;
-send_ack:
-	hfi1_send_rc_ack(packet, is_fecn);
 }
 
 u32 hfi1_build_tid_rdma_read_resp(struct rvt_qp *qp, struct rvt_ack_entry *e,
@@ -2463,12 +2461,12 @@ void hfi1_rc_rcv_tid_rdma_read_resp(struct hfi1_packet *packet)
 	struct tid_rdma_request *req;
 	struct tid_rdma_flow *flow;
 	u32 opcode, aeth;
-	bool is_fecn;
+	bool fecn;
 	unsigned long flags;
 	u32 kpsn, ipsn;
 
 	trace_hfi1_sender_rcv_tid_read_resp(qp);
-	is_fecn = process_ecn(qp, packet);
+	fecn = process_ecn(qp, packet);
 	kpsn = mask_psn(be32_to_cpu(ohdr->bth[2]));
 	aeth = be32_to_cpu(ohdr->u.tid_rdma.r_rsp.aeth);
 	opcode = (be32_to_cpu(ohdr->bth[0]) >> 24) & 0xff;
@@ -2481,8 +2479,43 @@ void hfi1_rc_rcv_tid_rdma_read_resp(struct hfi1_packet *packet)
 
 	flow = &req->flows[req->clear_tail];
 	/* When header suppression is disabled */
-	if (cmp_psn(ipsn, flow->flow_state.ib_lpsn))
+	if (cmp_psn(ipsn, flow->flow_state.ib_lpsn)) {
+		update_r_next_psn_fecn(packet, priv, rcd, flow, fecn);
+
+		if (cmp_psn(kpsn, flow->flow_state.r_next_psn))
+			goto ack_done;
+		flow->flow_state.r_next_psn = mask_psn(kpsn + 1);
+		/*
+		 * Copy the payload to destination buffer if this packet is
+		 * delivered as an eager packet due to RSM rule and FECN.
+		 * The RSM rule selects FECN bit in BTH and SH bit in
+		 * KDETH header and therefore will not match the last
+		 * packet of each segment that has SH bit cleared.
+		 */
+		if (fecn && packet->etype == RHF_RCV_TYPE_EAGER) {
+			struct rvt_sge_state ss;
+			u32 len;
+			u32 tlen = packet->tlen;
+			u16 hdrsize = packet->hlen;
+			u8 pad = packet->pad;
+			u8 extra_bytes = pad + packet->extra_byte +
+				(SIZE_OF_CRC << 2);
+			u32 pmtu = qp->pmtu;
+
+			if (unlikely(tlen != (hdrsize + pmtu + extra_bytes)))
+				goto ack_op_err;
+			len = restart_sge(&ss, req->e.swqe, ipsn, pmtu);
+			if (unlikely(len < pmtu))
+				goto ack_op_err;
+			rvt_copy_sge(qp, &ss, packet->payload, pmtu, false,
+				     false);
+			/* Raise the sw sequence check flag for next packet */
+			priv->s_flags |= HFI1_R_TID_SW_PSN;
+		}
+
 		goto ack_done;
+	}
+	flow->flow_state.r_next_psn = mask_psn(kpsn + 1);
 	req->ack_pending--;
 	priv->pending_tid_r_segs--;
 	qp->s_num_rd_atomic--;
@@ -2524,6 +2557,7 @@ void hfi1_rc_rcv_tid_rdma_read_resp(struct hfi1_packet *packet)
 	     req->comp_seg == req->cur_seg) ||
 	    priv->tid_r_comp == priv->tid_r_reqs) {
 		hfi1_kern_clear_hw_flow(priv->rcd, qp);
+		priv->s_flags &= ~HFI1_R_TID_SW_PSN;
 		if (req->state == TID_REQUEST_SYNC)
 			req->state = TID_REQUEST_ACTIVE;
 	}
@@ -2545,8 +2579,6 @@ ack_op_err:
 
 ack_done:
 	spin_unlock_irqrestore(&qp->s_lock, flags);
-	if (is_fecn)
-		hfi1_send_rc_ack(packet, is_fecn);
 }
 
 void hfi1_kern_read_tid_flow_free(struct rvt_qp *qp)
@@ -2773,9 +2805,9 @@ static bool handle_read_kdeth_eflags(struct hfi1_ctxtdata *rcd,
 				rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR);
 				return ret;
 			}
-			if (priv->flow_state.flags & TID_FLOW_SW_PSN) {
+			if (priv->s_flags & HFI1_R_TID_SW_PSN) {
 				diff = cmp_psn(psn,
-					       priv->flow_state.r_next_psn);
+					       flow->flow_state.r_next_psn);
 				if (diff > 0) {
 					if (!(qp->r_flags & RVT_R_RDMAR_SEQ))
 						restart_tid_rdma_read_req(rcd,
@@ -2811,22 +2843,15 @@ static bool handle_read_kdeth_eflags(struct hfi1_ctxtdata *rcd,
 						qp->r_flags &=
 							~RVT_R_RDMAR_SEQ;
 				}
-				priv->flow_state.r_next_psn++;
+				flow->flow_state.r_next_psn =
+					mask_psn(psn + 1);
 			} else {
-				u64 reg;
 				u32 last_psn;
 
-				/*
-				 * The only sane way to get the amount of
-				 * progress is to read the HW flow state.
-				 */
-				reg = read_uctxt_csr(dd, rcd->ctxt,
-						     RCV_TID_FLOW_TABLE +
-						     (8 * flow->idx));
-				last_psn = mask_psn(reg);
-
-				priv->flow_state.r_next_psn = last_psn;
-				priv->flow_state.flags |= TID_FLOW_SW_PSN;
+				last_psn = read_r_next_psn(dd, rcd->ctxt,
+							   flow->idx);
+				flow->flow_state.r_next_psn = last_psn;
+				priv->s_flags |= HFI1_R_TID_SW_PSN;
 				/*
 				 * If no request has been restarted yet,
 				 * restart the current one.
@@ -2891,10 +2916,11 @@ bool hfi1_handle_kdeth_eflags(struct hfi1_ctxtdata *rcd,
 	struct rvt_ack_entry *e;
 	struct tid_rdma_request *req;
 	struct tid_rdma_flow *flow;
+	int diff = 0;
 
 	trace_hfi1_msg_handle_kdeth_eflags(NULL, "Kdeth error: rhf ",
 					   packet->rhf);
-	if (packet->rhf & (RHF_VCRC_ERR | RHF_ICRC_ERR))
+	if (packet->rhf & RHF_ICRC_ERR)
 		return ret;
 
 	packet->ohdr = &hdr->u.oth;
@@ -2974,17 +3000,10 @@ bool hfi1_handle_kdeth_eflags(struct hfi1_ctxtdata *rcd,
 		switch (rte) {
 		case RHF_RTE_EXPECTED_FLOW_SEQ_ERR:
 			if (!(qpriv->s_flags & HFI1_R_TID_SW_PSN)) {
-				u64 reg;
-
 				qpriv->s_flags |= HFI1_R_TID_SW_PSN;
-				/*
-				 * The only sane way to get the amount of
-				 * progress is to read the HW flow state.
-				 */
-				reg = read_uctxt_csr(dd, rcd->ctxt,
-						     RCV_TID_FLOW_TABLE +
-						     (8 * flow->idx));
-				flow->flow_state.r_next_psn = mask_psn(reg);
+				flow->flow_state.r_next_psn =
+					read_r_next_psn(dd, rcd->ctxt,
+							flow->idx);
 				qpriv->r_next_psn_kdeth =
 					flow->flow_state.r_next_psn;
 				goto nak_psn;
@@ -2997,10 +3016,12 @@ bool hfi1_handle_kdeth_eflags(struct hfi1_ctxtdata *rcd,
 				 * mismatch could be due to packets that were
 				 * already in flight.
 				 */
-				if (psn != flow->flow_state.r_next_psn) {
-					psn = flow->flow_state.r_next_psn;
+				diff = cmp_psn(psn,
+					       flow->flow_state.r_next_psn);
+				if (diff > 0)
 					goto nak_psn;
-				}
+				else if (diff < 0)
+					break;
 
 				qpriv->s_nak_state = 0;
 				/*
@@ -3011,8 +3032,10 @@ bool hfi1_handle_kdeth_eflags(struct hfi1_ctxtdata *rcd,
 				if (psn == full_flow_psn(flow,
 							 flow->flow_state.lpsn))
 					ret = false;
+				flow->flow_state.r_next_psn =
+					mask_psn(psn + 1);
 				qpriv->r_next_psn_kdeth =
-					++flow->flow_state.r_next_psn;
+					flow->flow_state.r_next_psn;
 			}
 			break;
 
@@ -3517,8 +3540,10 @@ static void hfi1_tid_write_alloc_resources(struct rvt_qp *qp, bool intr_ctx)
 		if (qpriv->r_tid_alloc == qpriv->r_tid_head) {
 			/* If all data has been received, clear the flow */
 			if (qpriv->flow_state.index < RXE_NUM_TID_FLOWS &&
-			    !qpriv->alloc_w_segs)
+			    !qpriv->alloc_w_segs) {
 				hfi1_kern_clear_hw_flow(rcd, qp);
+				qpriv->s_flags &= ~HFI1_R_TID_SW_PSN;
+			}
 			break;
 		}
 
@@ -3544,8 +3569,7 @@ static void hfi1_tid_write_alloc_resources(struct rvt_qp *qp, bool intr_ctx)
 		if (qpriv->sync_pt && !qpriv->alloc_w_segs) {
 			hfi1_kern_clear_hw_flow(rcd, qp);
 			qpriv->sync_pt = false;
-			if (qpriv->s_flags & HFI1_R_TID_SW_PSN)
-				qpriv->s_flags &= ~HFI1_R_TID_SW_PSN;
+			qpriv->s_flags &= ~HFI1_R_TID_SW_PSN;
 		}
 
 		/* Allocate flow if we don't have one */
@@ -3687,7 +3711,7 @@ void hfi1_rc_rcv_tid_rdma_write_req(struct hfi1_packet *packet)
 	struct hfi1_qp_priv *qpriv = qp->priv;
 	struct tid_rdma_request *req;
 	u32 bth0, psn, len, rkey, num_segs;
-	bool is_fecn;
+	bool fecn;
 	u8 next;
 	u64 vaddr;
 	int diff;
@@ -3696,7 +3720,7 @@ void hfi1_rc_rcv_tid_rdma_write_req(struct hfi1_packet *packet)
 	if (hfi1_ruc_check_hdr(ibp, packet))
 		return;
 
-	is_fecn = process_ecn(qp, packet);
+	fecn = process_ecn(qp, packet);
 	psn = mask_psn(be32_to_cpu(ohdr->bth[2]));
 	trace_hfi1_rsp_rcv_tid_write_req(qp, psn);
 
@@ -3713,9 +3737,8 @@ void hfi1_rc_rcv_tid_rdma_write_req(struct hfi1_packet *packet)
 	num_segs = DIV_ROUND_UP(len, qpriv->tid_rdma.local.max_len);
 	diff = delta_psn(psn, qp->r_psn);
 	if (unlikely(diff)) {
-		if (tid_rdma_rcv_error(packet, ohdr, qp, psn, diff))
-			return;
-		goto send_ack;
+		tid_rdma_rcv_err(packet, ohdr, qp, psn, diff, fecn);
+		return;
 	}
 
 	/*
@@ -3751,10 +3774,7 @@ void hfi1_rc_rcv_tid_rdma_write_req(struct hfi1_packet *packet)
 		goto update_head;
 	}
 
-	if (e->rdma_sge.mr) {
-		rvt_put_mr(e->rdma_sge.mr);
-		e->rdma_sge.mr = NULL;
-	}
+	release_rdma_sge_mr(e);
 
 	/* The length needs to be in multiples of PAGE_SIZE */
 	if (!len || len & ~PAGE_MASK)
@@ -3834,11 +3854,11 @@ update_head:
 
 	/* Schedule the send tasklet. */
 	qp->s_flags |= RVT_S_RESP_PENDING;
+	if (fecn)
+		qp->s_flags |= RVT_S_ECN;
 	hfi1_schedule_send(qp);
 
 	spin_unlock_irqrestore(&qp->s_lock, flags);
-	if (is_fecn)
-		goto send_ack;
 	return;
 
 nack_inv_unlock:
@@ -3855,8 +3875,6 @@ nack_acc:
 	rvt_rc_error(qp, IB_WC_LOC_PROT_ERR);
 	qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR;
 	qp->r_ack_psn = qp->r_psn;
-send_ack:
-	hfi1_send_rc_ack(packet, is_fecn);
 }
 
 u32 hfi1_build_tid_rdma_write_resp(struct rvt_qp *qp, struct rvt_ack_entry *e,
@@ -4073,10 +4091,10 @@ void hfi1_rc_rcv_tid_rdma_write_resp(struct hfi1_packet *packet)
 	struct tid_rdma_flow *flow;
 	enum ib_wc_status status;
 	u32 opcode, aeth, psn, flow_psn, i, tidlen = 0, pktlen;
-	bool is_fecn;
+	bool fecn;
 	unsigned long flags;
 
-	is_fecn = process_ecn(qp, packet);
+	fecn = process_ecn(qp, packet);
 	psn = mask_psn(be32_to_cpu(ohdr->bth[2]));
 	aeth = be32_to_cpu(ohdr->u.tid_rdma.w_rsp.aeth);
 	opcode = (be32_to_cpu(ohdr->bth[0]) >> 24) & 0xff;
@@ -4216,7 +4234,6 @@ void hfi1_rc_rcv_tid_rdma_write_resp(struct hfi1_packet *packet)
 		qpriv->s_tid_cur = i;
 	}
 	qp->s_flags &= ~HFI1_S_WAIT_TID_RESP;
-
 	hfi1_schedule_tid_send(qp);
 	goto ack_done;
 
@@ -4225,9 +4242,9 @@ ack_op_err:
 ack_err:
 	rvt_error_qp(qp, status);
 ack_done:
+	if (fecn)
+		qp->s_flags |= RVT_S_ECN;
 	spin_unlock_irqrestore(&qp->s_lock, flags);
-	if (is_fecn)
-		hfi1_send_rc_ack(packet, is_fecn);
 }
 
 bool hfi1_build_tid_rdma_packet(struct rvt_swqe *wqe,
@@ -4307,7 +4324,9 @@ void hfi1_rc_rcv_tid_rdma_write_data(struct hfi1_packet *packet)
 	unsigned long flags;
 	u32 psn, next;
 	u8 opcode;
+	bool fecn;
 
+	fecn = process_ecn(qp, packet);
 	psn = mask_psn(be32_to_cpu(ohdr->bth[2]));
 	opcode = (be32_to_cpu(ohdr->bth[0]) >> 24) & 0xff;
 
@@ -4320,9 +4339,53 @@ void hfi1_rc_rcv_tid_rdma_write_data(struct hfi1_packet *packet)
 	req = ack_to_tid_req(e);
 	flow = &req->flows[req->clear_tail];
 	if (cmp_psn(psn, full_flow_psn(flow, flow->flow_state.lpsn))) {
+		update_r_next_psn_fecn(packet, priv, rcd, flow, fecn);
+
 		if (cmp_psn(psn, flow->flow_state.r_next_psn))
 			goto send_nak;
-		flow->flow_state.r_next_psn++;
+
+		flow->flow_state.r_next_psn = mask_psn(psn + 1);
+		/*
+		 * Copy the payload to destination buffer if this packet is
+		 * delivered as an eager packet due to RSM rule and FECN.
+		 * The RSM rule selects FECN bit in BTH and SH bit in
+		 * KDETH header and therefore will not match the last
+		 * packet of each segment that has SH bit cleared.
+		 */
+		if (fecn && packet->etype == RHF_RCV_TYPE_EAGER) {
+			struct rvt_sge_state ss;
+			u32 len;
+			u32 tlen = packet->tlen;
+			u16 hdrsize = packet->hlen;
+			u8 pad = packet->pad;
+			u8 extra_bytes = pad + packet->extra_byte +
+				(SIZE_OF_CRC << 2);
+			u32 pmtu = qp->pmtu;
+
+			if (unlikely(tlen != (hdrsize + pmtu + extra_bytes)))
+				goto send_nak;
+			len = req->comp_seg * req->seg_len;
+			len += delta_psn(psn,
+				full_flow_psn(flow, flow->flow_state.spsn)) *
+				pmtu;
+			if (unlikely(req->total_len - len < pmtu))
+				goto send_nak;
+
+			/*
+			 * The e->rdma_sge field is set when TID RDMA WRITE REQ
+			 * is first received and is never modified thereafter.
+			 */
+			ss.sge = e->rdma_sge;
+			ss.sg_list = NULL;
+			ss.num_sge = 1;
+			ss.total_len = req->total_len;
+			rvt_skip_sge(&ss, len, false);
+			rvt_copy_sge(qp, &ss, packet->payload, pmtu, false,
+				     false);
+			/* Raise the sw sequence check flag for next packet */
+			priv->r_next_psn_kdeth = mask_psn(psn + 1);
+			priv->s_flags |= HFI1_R_TID_SW_PSN;
+		}
 		goto exit;
 	}
 	flow->flow_state.r_next_psn = mask_psn(psn + 1);
@@ -4347,6 +4410,7 @@ void hfi1_rc_rcv_tid_rdma_write_data(struct hfi1_packet *packet)
 		priv->r_tid_ack = priv->r_tid_tail;
 
 	if (opcode == TID_OP(WRITE_DATA_LAST)) {
+		release_rdma_sge_mr(e);
 		for (next = priv->r_tid_tail + 1; ; next++) {
 			if (next > rvt_size_atomic(&dev->rdi))
 				next = 0;
@@ -4386,6 +4450,8 @@ done:
 	hfi1_schedule_tid_send(qp);
 exit:
 	priv->r_next_psn_kdeth = flow->flow_state.r_next_psn;
+	if (fecn)
+		qp->s_flags |= RVT_S_ECN;
 	spin_unlock_irqrestore(&qp->s_lock, flags);
 	return;
 
@@ -4487,12 +4553,11 @@ void hfi1_rc_rcv_tid_rdma_ack(struct hfi1_packet *packet)
 	struct tid_rdma_request *req;
 	struct tid_rdma_flow *flow;
 	u32 aeth, psn, req_psn, ack_psn, fspsn, resync_psn, ack_kpsn;
-	bool is_fecn;
 	unsigned long flags;
 	u16 fidx;
 
 	trace_hfi1_tid_write_sender_rcv_tid_ack(qp, 0);
-	is_fecn = process_ecn(qp, packet);
+	process_ecn(qp, packet);
 	psn = mask_psn(be32_to_cpu(ohdr->bth[2]));
 	aeth = be32_to_cpu(ohdr->u.tid_rdma.ack.aeth);
 	req_psn = mask_psn(be32_to_cpu(ohdr->u.tid_rdma.ack.verbs_psn));
@@ -4846,10 +4911,10 @@ void hfi1_rc_rcv_tid_rdma_resync(struct hfi1_packet *packet)
 	struct tid_rdma_flow *flow;
 	struct tid_flow_state *fs = &qpriv->flow_state;
 	u32 psn, generation, idx, gen_next;
-	bool is_fecn;
+	bool fecn;
 	unsigned long flags;
 
-	is_fecn = process_ecn(qp, packet);
+	fecn = process_ecn(qp, packet);
 	psn = mask_psn(be32_to_cpu(ohdr->bth[2]));
 
 	generation = mask_psn(psn + 1) >> HFI1_KDETH_BTH_SEQ_SHIFT;
@@ -4940,6 +5005,8 @@ void hfi1_rc_rcv_tid_rdma_resync(struct hfi1_packet *packet)
 	qpriv->s_flags |= RVT_S_ACK_PENDING;
 	hfi1_schedule_tid_send(qp);
 bail:
+	if (fecn)
+		qp->s_flags |= RVT_S_ECN;
 	spin_unlock_irqrestore(&qp->s_lock, flags);
 }
 
@@ -5449,3 +5516,48 @@ bool hfi1_tid_rdma_ack_interlock(struct rvt_qp *qp, struct rvt_ack_entry *e)
 	}
 	return false;
 }
+
+static u32 read_r_next_psn(struct hfi1_devdata *dd, u8 ctxt, u8 fidx)
+{
+	u64 reg;
+
+	/*
+	 * The only sane way to get the amount of
+	 * progress is to read the HW flow state.
+	 */
+	reg = read_uctxt_csr(dd, ctxt, RCV_TID_FLOW_TABLE + (8 * fidx));
+	return mask_psn(reg);
+}
+
+static void tid_rdma_rcv_err(struct hfi1_packet *packet,
+			     struct ib_other_headers *ohdr,
+			     struct rvt_qp *qp, u32 psn, int diff, bool fecn)
+{
+	unsigned long flags;
+
+	tid_rdma_rcv_error(packet, ohdr, qp, psn, diff);
+	if (fecn) {
+		spin_lock_irqsave(&qp->s_lock, flags);
+		qp->s_flags |= RVT_S_ECN;
+		spin_unlock_irqrestore(&qp->s_lock, flags);
+	}
+}
+
+static void update_r_next_psn_fecn(struct hfi1_packet *packet,
+				   struct hfi1_qp_priv *priv,
+				   struct hfi1_ctxtdata *rcd,
+				   struct tid_rdma_flow *flow,
+				   bool fecn)
+{
+	/*
+	 * If a start/middle packet is delivered here due to
+	 * RSM rule and FECN, we need to update the r_next_psn.
+	 */
+	if (fecn && packet->etype == RHF_RCV_TYPE_EAGER &&
+	    !(priv->s_flags & HFI1_R_TID_SW_PSN)) {
+		struct hfi1_devdata *dd = rcd->dd;
+
+		flow->flow_state.r_next_psn =
+			read_r_next_psn(dd, rcd->ctxt, flow->idx);
+	}
+}
diff --git a/drivers/infiniband/hw/hfi1/tid_rdma.h b/drivers/infiniband/hw/hfi1/tid_rdma.h
index 53ab24ef4f02..1c536185261e 100644
--- a/drivers/infiniband/hw/hfi1/tid_rdma.h
+++ b/drivers/infiniband/hw/hfi1/tid_rdma.h
@@ -76,10 +76,8 @@ struct tid_rdma_qp_params {
 struct tid_flow_state {
 	u32 generation;
 	u32 psn;
-	u32 r_next_psn;      /* next PSN to be received (in TID space) */
 	u8 index;
 	u8 last_index;
-	u8 flags;
 };
 
 enum tid_rdma_req_state {
diff --git a/drivers/infiniband/hw/hfi1/trace_dbg.h b/drivers/infiniband/hw/hfi1/trace_dbg.h
index e62171fb7379..de7a87392b8d 100644
--- a/drivers/infiniband/hw/hfi1/trace_dbg.h
+++ b/drivers/infiniband/hw/hfi1/trace_dbg.h
@@ -86,14 +86,14 @@ DECLARE_EVENT_CLASS(hfi1_trace_template,
  * actual function to work and can not be in a macro.
  */
 #define __hfi1_trace_def(lvl) \
-void __hfi1_trace_##lvl(const char *funct, char *fmt, ...);		\
+void __printf(2, 3) __hfi1_trace_##lvl(const char *funct, char *fmt, ...); \
 									\
 DEFINE_EVENT(hfi1_trace_template, hfi1_ ##lvl,				\
 	TP_PROTO(const char *function, struct va_format *vaf),		\
 	TP_ARGS(function, vaf))
 
 #define __hfi1_trace_fn(lvl) \
-void __hfi1_trace_##lvl(const char *func, char *fmt, ...)		\
+void __printf(2, 3) __hfi1_trace_##lvl(const char *func, char *fmt, ...)\
 {									\
 	struct va_format vaf = {					\
 		.fmt = fmt,						\
diff --git a/drivers/infiniband/hw/hfi1/trace_tid.h b/drivers/infiniband/hw/hfi1/trace_tid.h
index 548dfc45a407..4388b594ed1b 100644
--- a/drivers/infiniband/hw/hfi1/trace_tid.h
+++ b/drivers/infiniband/hw/hfi1/trace_tid.h
@@ -53,7 +53,7 @@ u16 hfi1_trace_get_tid_idx(u32 ent);
 			    "tid_r_comp %u pending_tid_r_segs %u " \
 			    "s_flags 0x%x ps_flags 0x%x iow_flags 0x%lx " \
 			    "s_state 0x%x hw_flow_index %u generation 0x%x " \
-			    "fpsn 0x%x flow_flags 0x%x"
+			    "fpsn 0x%x"
 
 #define TID_REQ_PRN "[%s] qpn 0x%x newreq %u opcode 0x%x psn 0x%x lpsn 0x%x " \
 		    "cur_seg %u comp_seg %u ack_seg %u alloc_seg %u " \
@@ -71,7 +71,7 @@ u16 hfi1_trace_get_tid_idx(u32 ent);
 			    "pending_tid_w_segs %u sync_pt %s " \
 			    "ps_nak_psn 0x%x ps_nak_state 0x%x " \
 			    "prnr_nak_state 0x%x hw_flow_index %u generation "\
-			    "0x%x fpsn 0x%x flow_flags 0x%x resync %s" \
+			    "0x%x fpsn 0x%x resync %s" \
 			    "r_next_psn_kdeth 0x%x"
 
 #define TID_WRITE_SENDER_PRN "[%s] qpn 0x%x newreq %u s_tid_cur %u " \
@@ -973,7 +973,6 @@ DECLARE_EVENT_CLASS(/* tid_read_sender */
 		__field(u32, hw_flow_index)
 		__field(u32, generation)
 		__field(u32, fpsn)
-		__field(u32, flow_flags)
 	),
 	TP_fast_assign(/* assign */
 		struct hfi1_qp_priv *priv = qp->priv;
@@ -991,7 +990,6 @@ DECLARE_EVENT_CLASS(/* tid_read_sender */
 		__entry->hw_flow_index = priv->flow_state.index;
 		__entry->generation = priv->flow_state.generation;
 		__entry->fpsn = priv->flow_state.psn;
-		__entry->flow_flags = priv->flow_state.flags;
 	),
 	TP_printk(/* print */
 		TID_READ_SENDER_PRN,
@@ -1007,8 +1005,7 @@ DECLARE_EVENT_CLASS(/* tid_read_sender */
 		__entry->s_state,
 		__entry->hw_flow_index,
 		__entry->generation,
-		__entry->fpsn,
-		__entry->flow_flags
+		__entry->fpsn
 	)
 );
 
@@ -1338,7 +1335,6 @@ DECLARE_EVENT_CLASS(/* tid_write_sp */
 		__field(u32, hw_flow_index)
 		__field(u32, generation)
 		__field(u32, fpsn)
-		__field(u32, flow_flags)
 		__field(bool, resync)
 		__field(u32, r_next_psn_kdeth)
 	),
@@ -1360,7 +1356,6 @@ DECLARE_EVENT_CLASS(/* tid_write_sp */
 		__entry->hw_flow_index = priv->flow_state.index;
 		__entry->generation = priv->flow_state.generation;
 		__entry->fpsn = priv->flow_state.psn;
-		__entry->flow_flags = priv->flow_state.flags;
 		__entry->resync = priv->resync;
 		__entry->r_next_psn_kdeth = priv->r_next_psn_kdeth;
 	),
@@ -1381,7 +1376,6 @@ DECLARE_EVENT_CLASS(/* tid_write_sp */
 		__entry->hw_flow_index,
 		__entry->generation,
 		__entry->fpsn,
-		__entry->flow_flags,
 		__entry->resync ? "yes" : "no",
 		__entry->r_next_psn_kdeth
 	)
diff --git a/drivers/infiniband/hw/hfi1/verbs.c b/drivers/infiniband/hw/hfi1/verbs.c
index 55a56b3d7f83..1eb4105b2d22 100644
--- a/drivers/infiniband/hw/hfi1/verbs.c
+++ b/drivers/infiniband/hw/hfi1/verbs.c
@@ -1223,15 +1223,16 @@ static inline send_routine get_send_routine(struct rvt_qp *qp,
 	case IB_QPT_UD:
 		break;
 	case IB_QPT_UC:
-	case IB_QPT_RC: {
+	case IB_QPT_RC:
+		priv->s_running_pkt_size =
+			(tx->s_cur_size + priv->s_running_pkt_size) / 2;
 		if (piothreshold &&
-		    tx->s_cur_size <= min(piothreshold, qp->pmtu) &&
+		    priv->s_running_pkt_size <= min(piothreshold, qp->pmtu) &&
 		    (BIT(ps->opcode & OPMASK) & pio_opmask[ps->opcode >> 5]) &&
 		    iowait_sdma_pending(&priv->s_iowait) == 0 &&
 		    !sdma_txreq_built(&tx->txreq))
 			return dd->process_pio_send;
 		break;
-	}
 	default:
 		break;
 	}
@@ -1739,15 +1740,15 @@ static struct rdma_hw_stats *alloc_hw_stats(struct ib_device *ibdev,
 
 static u64 hfi1_sps_ints(void)
 {
-	unsigned long flags;
+	unsigned long index, flags;
 	struct hfi1_devdata *dd;
 	u64 sps_ints = 0;
 
-	spin_lock_irqsave(&hfi1_devs_lock, flags);
-	list_for_each_entry(dd, &hfi1_dev_list, list) {
+	xa_lock_irqsave(&hfi1_dev_table, flags);
+	xa_for_each(&hfi1_dev_table, index, dd) {
 		sps_ints += get_all_cpu_total(dd->int_counter);
 	}
-	spin_unlock_irqrestore(&hfi1_devs_lock, flags);
+	xa_unlock_irqrestore(&hfi1_dev_table, flags);
 	return sps_ints;
 }
 
diff --git a/drivers/infiniband/hw/hfi1/verbs.h b/drivers/infiniband/hw/hfi1/verbs.h
index 62ace0b2d17a..7ecb8ed4a1d9 100644
--- a/drivers/infiniband/hw/hfi1/verbs.h
+++ b/drivers/infiniband/hw/hfi1/verbs.h
@@ -170,6 +170,7 @@ struct hfi1_qp_priv {
 	struct tid_flow_state flow_state;
 	struct tid_rdma_qp_params tid_rdma;
 	struct rvt_qp *owner;
+	u16 s_running_pkt_size;
 	u8 hdr_type; /* 9B or 16B */
 	struct rvt_sge_state tid_ss;       /* SGE state pointer for 2nd leg */
 	atomic_t n_requests;               /* # of TID RDMA requests in the */
diff --git a/drivers/infiniband/hw/hfi1/vnic_main.c b/drivers/infiniband/hw/hfi1/vnic_main.c
index 2b07032dbdda..b49e60e8397d 100644
--- a/drivers/infiniband/hw/hfi1/vnic_main.c
+++ b/drivers/infiniband/hw/hfi1/vnic_main.c
@@ -162,12 +162,12 @@ static void deallocate_vnic_ctxt(struct hfi1_devdata *dd,
 
 void hfi1_vnic_setup(struct hfi1_devdata *dd)
 {
-	idr_init(&dd->vnic.vesw_idr);
+	xa_init(&dd->vnic.vesws);
 }
 
 void hfi1_vnic_cleanup(struct hfi1_devdata *dd)
 {
-	idr_destroy(&dd->vnic.vesw_idr);
+	WARN_ON(!xa_empty(&dd->vnic.vesws));
 }
 
 #define SUM_GRP_COUNTERS(stats, qstats, x_grp) do {            \
@@ -533,7 +533,7 @@ void hfi1_vnic_bypass_rcv(struct hfi1_packet *packet)
 	l4_type = hfi1_16B_get_l4(packet->ebuf);
 	if (likely(l4_type == OPA_16B_L4_ETHR)) {
 		vesw_id = HFI1_VNIC_GET_VESWID(packet->ebuf);
-		vinfo = idr_find(&dd->vnic.vesw_idr, vesw_id);
+		vinfo = xa_load(&dd->vnic.vesws, vesw_id);
 
 		/*
 		 * In case of invalid vesw id, count the error on
@@ -541,9 +541,10 @@ void hfi1_vnic_bypass_rcv(struct hfi1_packet *packet)
 		 */
 		if (unlikely(!vinfo)) {
 			struct hfi1_vnic_vport_info *vinfo_tmp;
-			int id_tmp = 0;
+			unsigned long index = 0;
 
-			vinfo_tmp =  idr_get_next(&dd->vnic.vesw_idr, &id_tmp);
+			vinfo_tmp = xa_find(&dd->vnic.vesws, &index, ULONG_MAX,
+					XA_PRESENT);
 			if (vinfo_tmp) {
 				spin_lock(&vport_cntr_lock);
 				vinfo_tmp->stats[0].netstats.rx_nohandler++;
@@ -597,8 +598,7 @@ static int hfi1_vnic_up(struct hfi1_vnic_vport_info *vinfo)
 	if (!vinfo->vesw_id)
 		return -EINVAL;
 
-	rc = idr_alloc(&dd->vnic.vesw_idr, vinfo, vinfo->vesw_id,
-		       vinfo->vesw_id + 1, GFP_NOWAIT);
+	rc = xa_insert(&dd->vnic.vesws, vinfo->vesw_id, vinfo, GFP_KERNEL);
 	if (rc < 0)
 		return rc;
 
@@ -624,7 +624,7 @@ static void hfi1_vnic_down(struct hfi1_vnic_vport_info *vinfo)
 	clear_bit(HFI1_VNIC_UP, &vinfo->flags);
 	netif_carrier_off(vinfo->netdev);
 	netif_tx_disable(vinfo->netdev);
-	idr_remove(&dd->vnic.vesw_idr, vinfo->vesw_id);
+	xa_erase(&dd->vnic.vesws, vinfo->vesw_id);
 
 	/* ensure irqs see the change */
 	msix_vnic_synchronize_irq(dd);
diff --git a/drivers/infiniband/hw/hns/Makefile b/drivers/infiniband/hw/hns/Makefile
index e2a7f1488f76..eee5205f936f 100644
--- a/drivers/infiniband/hw/hns/Makefile
+++ b/drivers/infiniband/hw/hns/Makefile
@@ -7,8 +7,8 @@ ccflags-y :=  -I $(srctree)/drivers/net/ethernet/hisilicon/hns3
 obj-$(CONFIG_INFINIBAND_HNS) += hns-roce.o
 hns-roce-objs := hns_roce_main.o hns_roce_cmd.o hns_roce_pd.o \
 	hns_roce_ah.o hns_roce_hem.o hns_roce_mr.o hns_roce_qp.o \
-	hns_roce_cq.o hns_roce_alloc.o hns_roce_db.o hns_roce_srq.o
+	hns_roce_cq.o hns_roce_alloc.o hns_roce_db.o hns_roce_srq.o hns_roce_restrack.o
 obj-$(CONFIG_INFINIBAND_HNS_HIP06) += hns-roce-hw-v1.o
 hns-roce-hw-v1-objs := hns_roce_hw_v1.o
 obj-$(CONFIG_INFINIBAND_HNS_HIP08) += hns-roce-hw-v2.o
-hns-roce-hw-v2-objs := hns_roce_hw_v2.o
+hns-roce-hw-v2-objs := hns_roce_hw_v2.o hns_roce_hw_v2_dfx.o
diff --git a/drivers/infiniband/hw/hns/hns_roce_ah.c b/drivers/infiniband/hw/hns/hns_roce_ah.c
index b3c8c45ec1e3..cdd2ac24fc2a 100644
--- a/drivers/infiniband/hw/hns/hns_roce_ah.c
+++ b/drivers/infiniband/hw/hns/hns_roce_ah.c
@@ -39,38 +39,34 @@
 #define HNS_ROCE_VLAN_SL_BIT_MASK	7
 #define HNS_ROCE_VLAN_SL_SHIFT		13
 
-struct ib_ah *hns_roce_create_ah(struct ib_pd *ibpd,
-				 struct rdma_ah_attr *ah_attr,
-				 u32 flags,
-				 struct ib_udata *udata)
+int hns_roce_create_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr,
+		       u32 flags, struct ib_udata *udata)
 {
-	struct hns_roce_dev *hr_dev = to_hr_dev(ibpd->device);
+	struct hns_roce_dev *hr_dev = to_hr_dev(ibah->device);
 	const struct ib_gid_attr *gid_attr;
 	struct device *dev = hr_dev->dev;
-	struct hns_roce_ah *ah;
+	struct hns_roce_ah *ah = to_hr_ah(ibah);
 	u16 vlan_tag = 0xffff;
 	const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr);
 	bool vlan_en = false;
+	int ret;
 
-	ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
-	if (!ah)
-		return ERR_PTR(-ENOMEM);
+	gid_attr = ah_attr->grh.sgid_attr;
+	ret = rdma_read_gid_l2_fields(gid_attr, &vlan_tag, NULL);
+	if (ret)
+		return ret;
 
 	/* Get mac address */
 	memcpy(ah->av.mac, ah_attr->roce.dmac, ETH_ALEN);
 
-	gid_attr = ah_attr->grh.sgid_attr;
-	if (is_vlan_dev(gid_attr->ndev)) {
-		vlan_tag = vlan_dev_vlan_id(gid_attr->ndev);
+	if (vlan_tag < VLAN_CFI_MASK) {
 		vlan_en = true;
-	}
-
-	if (vlan_tag < 0x1000)
 		vlan_tag |= (rdma_ah_get_sl(ah_attr) &
 			     HNS_ROCE_VLAN_SL_BIT_MASK) <<
 			     HNS_ROCE_VLAN_SL_SHIFT;
+	}
 
-	ah->av.port_pd = cpu_to_be32(to_hr_pd(ibpd)->pdn |
+	ah->av.port_pd = cpu_to_le32(to_hr_pd(ibah->pd)->pdn |
 				     (rdma_ah_get_port_num(ah_attr) <<
 				     HNS_ROCE_PORT_NUM_SHIFT));
 	ah->av.gid_index = grh->sgid_index;
@@ -86,7 +82,7 @@ struct ib_ah *hns_roce_create_ah(struct ib_pd *ibpd,
 	ah->av.sl_tclass_flowlabel = cpu_to_le32(rdma_ah_get_sl(ah_attr) <<
 						 HNS_ROCE_SL_SHIFT);
 
-	return &ah->ibah;
+	return 0;
 }
 
 int hns_roce_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr)
@@ -111,9 +107,7 @@ int hns_roce_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr)
 	return 0;
 }
 
-int hns_roce_destroy_ah(struct ib_ah *ah, u32 flags)
+void hns_roce_destroy_ah(struct ib_ah *ah, u32 flags)
 {
-	kfree(to_hr_ah(ah));
-
-	return 0;
+	return;
 }
diff --git a/drivers/infiniband/hw/hns/hns_roce_cmd.h b/drivers/infiniband/hw/hns/hns_roce_cmd.h
index 059fd1da493e..2b6ac646ca9a 100644
--- a/drivers/infiniband/hw/hns/hns_roce_cmd.h
+++ b/drivers/infiniband/hw/hns/hns_roce_cmd.h
@@ -53,6 +53,7 @@ enum {
 	HNS_ROCE_CMD_QUERY_QPC		= 0x42,
 
 	HNS_ROCE_CMD_MODIFY_CQC		= 0x52,
+	HNS_ROCE_CMD_QUERY_CQC		= 0x53,
 	/* CQC BT commands */
 	HNS_ROCE_CMD_WRITE_CQC_BT0	= 0x10,
 	HNS_ROCE_CMD_WRITE_CQC_BT1	= 0x11,
diff --git a/drivers/infiniband/hw/hns/hns_roce_common.h b/drivers/infiniband/hw/hns/hns_roce_common.h
index f4c92a7ac1ce..8e95a1aa1b4f 100644
--- a/drivers/infiniband/hw/hns/hns_roce_common.h
+++ b/drivers/infiniband/hw/hns/hns_roce_common.h
@@ -57,32 +57,6 @@
 #define roce_set_bit(origin, shift, val) \
 	roce_set_field((origin), (1ul << (shift)), (shift), (val))
 
-/*
- * roce_hw_index_cmp_lt - Compare two hardware index values in hisilicon
- *                        SOC, check if a is less than b.
- * @a: hardware index value
- * @b: hardware index value
- * @bits: the number of bits of a and b, range: 0~31.
- *
- * Hardware index increases continuously till max value, and then restart
- * from zero, again and again. Because the bits of reg field is often
- * limited, the reg field can only hold the low bits of the hardware index
- * in hisilicon SOC.
- * In some scenes we need to compare two values(a,b) getted from two reg
- * fields in this driver, for example:
- * If a equals 0xfffe, b equals 0x1 and bits equals 16, we think b has
- * incresed from 0xffff to 0x1 and a is less than b.
- * If a equals 0xfffe, b equals 0x0xf001 and bits equals 16, we think a
- * is bigger than b.
- *
- * Return true on a less than b, otherwise false.
- */
-#define roce_hw_index_mask(bits)	((1ul << (bits)) - 1)
-#define roce_hw_index_shift(bits)	(32 - (bits))
-#define roce_hw_index_cmp_lt(a, b, bits) \
-	((int)((((a) - (b)) & roce_hw_index_mask(bits)) << \
-		roce_hw_index_shift(bits)) < 0)
-
 #define ROCEE_GLB_CFG_ROCEE_DB_SQ_MODE_S 3
 #define ROCEE_GLB_CFG_ROCEE_DB_OTH_MODE_S 4
 
@@ -271,8 +245,6 @@
 #define ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M   \
 	(((1UL << 28) - 1) << ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S)
 
-#define ROCEE_SDB_PTR_CMP_BITS 28
-
 #define ROCEE_SDB_INV_CNT_SDB_INV_CNT_S 0
 #define ROCEE_SDB_INV_CNT_SDB_INV_CNT_M   \
 	(((1UL << 16) - 1) << ROCEE_SDB_INV_CNT_SDB_INV_CNT_S)
@@ -353,13 +325,8 @@
 #define ROCEE_CAEP_AE_MASK_REG			0x6C8
 #define ROCEE_CAEP_AE_ST_REG			0x6CC
 
-#define ROCEE_SDB_ISSUE_PTR_REG			0x758
-#define ROCEE_SDB_SEND_PTR_REG			0x75C
 #define ROCEE_CAEP_CQE_WCMD_EMPTY		0x850
 #define ROCEE_SCAEP_WR_CQE_CNT			0x8D0
-#define ROCEE_SDB_INV_CNT_REG			0x9A4
-#define ROCEE_SDB_RETRY_CNT_REG			0x9AC
-#define ROCEE_TSP_BP_ST_REG			0x9EC
 #define ROCEE_ECC_UCERR_ALM0_REG		0xB34
 #define ROCEE_ECC_CERR_ALM0_REG			0xB40
 
diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c
index 1dfe5627006c..9caf35061721 100644
--- a/drivers/infiniband/hw/hns/hns_roce_cq.c
+++ b/drivers/infiniband/hw/hns/hns_roce_cq.c
@@ -32,6 +32,7 @@
 
 #include <linux/platform_device.h>
 #include <rdma/ib_umem.h>
+#include <rdma/uverbs_ioctl.h>
 #include "hns_roce_device.h"
 #include "hns_roce_cmd.h"
 #include "hns_roce_hem.h"
@@ -127,13 +128,9 @@ static int hns_roce_cq_alloc(struct hns_roce_dev *hr_dev, int nent,
 		goto err_out;
 	}
 
-	/* The cq insert radix tree */
-	spin_lock_irq(&cq_table->lock);
-	/* Radix_tree: The associated pointer and long integer key value like */
-	ret = radix_tree_insert(&cq_table->tree, hr_cq->cqn, hr_cq);
-	spin_unlock_irq(&cq_table->lock);
+	ret = xa_err(xa_store(&cq_table->array, hr_cq->cqn, hr_cq, GFP_KERNEL));
 	if (ret) {
-		dev_err(dev, "CQ alloc.Failed to radix_tree_insert.\n");
+		dev_err(dev, "CQ alloc failed xa_store.\n");
 		goto err_put;
 	}
 
@@ -141,7 +138,7 @@ static int hns_roce_cq_alloc(struct hns_roce_dev *hr_dev, int nent,
 	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
 	if (IS_ERR(mailbox)) {
 		ret = PTR_ERR(mailbox);
-		goto err_radix;
+		goto err_xa;
 	}
 
 	hr_dev->hw->write_cqc(hr_dev, hr_cq, mailbox->buf, mtts, dma_handle,
@@ -152,7 +149,7 @@ static int hns_roce_cq_alloc(struct hns_roce_dev *hr_dev, int nent,
 	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
 	if (ret) {
 		dev_err(dev, "CQ alloc.Failed to cmd mailbox.\n");
-		goto err_radix;
+		goto err_xa;
 	}
 
 	hr_cq->cons_index = 0;
@@ -164,10 +161,8 @@ static int hns_roce_cq_alloc(struct hns_roce_dev *hr_dev, int nent,
 
 	return 0;
 
-err_radix:
-	spin_lock_irq(&cq_table->lock);
-	radix_tree_delete(&cq_table->tree, hr_cq->cqn);
-	spin_unlock_irq(&cq_table->lock);
+err_xa:
+	xa_erase(&cq_table->array, hr_cq->cqn);
 
 err_put:
 	hns_roce_table_put(hr_dev, &cq_table->table, hr_cq->cqn);
@@ -197,6 +192,8 @@ void hns_roce_free_cq(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq)
 		dev_err(dev, "HW2SW_CQ failed (%d) for CQN %06lx\n", ret,
 			hr_cq->cqn);
 
+	xa_erase(&cq_table->array, hr_cq->cqn);
+
 	/* Waiting interrupt process procedure carried out */
 	synchronize_irq(hr_dev->eq_table.eq[hr_cq->vector].irq);
 
@@ -205,10 +202,6 @@ void hns_roce_free_cq(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq)
 		complete(&hr_cq->free);
 	wait_for_completion(&hr_cq->free);
 
-	spin_lock_irq(&cq_table->lock);
-	radix_tree_delete(&cq_table->tree, hr_cq->cqn);
-	spin_unlock_irq(&cq_table->lock);
-
 	hns_roce_table_put(hr_dev, &cq_table->table, hr_cq->cqn);
 	hns_roce_bitmap_free(&cq_table->bitmap, hr_cq->cqn, BITMAP_NO_RR);
 }
@@ -309,7 +302,6 @@ static void hns_roce_ib_free_cq_buf(struct hns_roce_dev *hr_dev,
 
 struct ib_cq *hns_roce_ib_create_cq(struct ib_device *ib_dev,
 				    const struct ib_cq_init_attr *attr,
-				    struct ib_ucontext *context,
 				    struct ib_udata *udata)
 {
 	struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev);
@@ -321,6 +313,8 @@ struct ib_cq *hns_roce_ib_create_cq(struct ib_device *ib_dev,
 	int vector = attr->comp_vector;
 	int cq_entries = attr->cqe;
 	int ret;
+	struct hns_roce_ucontext *context = rdma_udata_to_drv_context(
+		udata, struct hns_roce_ucontext, ibucontext);
 
 	if (cq_entries < 1 || cq_entries > hr_dev->caps.max_cqes) {
 		dev_err(dev, "Creat CQ failed. entries=%d, max=%d\n",
@@ -339,7 +333,7 @@ struct ib_cq *hns_roce_ib_create_cq(struct ib_device *ib_dev,
 	hr_cq->ib_cq.cqe = cq_entries - 1;
 	spin_lock_init(&hr_cq->lock);
 
-	if (context) {
+	if (udata) {
 		if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) {
 			dev_err(dev, "Failed to copy_from_udata.\n");
 			ret = -EFAULT;
@@ -357,8 +351,7 @@ struct ib_cq *hns_roce_ib_create_cq(struct ib_device *ib_dev,
 
 		if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) &&
 		    (udata->outlen >= sizeof(resp))) {
-			ret = hns_roce_db_map_user(to_hr_ucontext(context),
-						   udata, ucmd.db_addr,
+			ret = hns_roce_db_map_user(context, udata, ucmd.db_addr,
 						   &hr_cq->db);
 			if (ret) {
 				dev_err(dev, "cq record doorbell map failed!\n");
@@ -369,7 +362,7 @@ struct ib_cq *hns_roce_ib_create_cq(struct ib_device *ib_dev,
 		}
 
 		/* Get user space parameters */
-		uar = &to_hr_ucontext(context)->uar;
+		uar = &context->uar;
 	} else {
 		if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) {
 			ret = hns_roce_alloc_db(hr_dev, &hr_cq->db, 1);
@@ -408,7 +401,7 @@ struct ib_cq *hns_roce_ib_create_cq(struct ib_device *ib_dev,
 	 * problems if tptr is set to zero here, so we initialze it in user
 	 * space.
 	 */
-	if (!context && hr_cq->tptr_addr)
+	if (!udata && hr_cq->tptr_addr)
 		*hr_cq->tptr_addr = 0;
 
 	/* Get created cq handler and carry out event */
@@ -416,7 +409,7 @@ struct ib_cq *hns_roce_ib_create_cq(struct ib_device *ib_dev,
 	hr_cq->event = hns_roce_ib_cq_event;
 	hr_cq->cq_depth = cq_entries;
 
-	if (context) {
+	if (udata) {
 		resp.cqn = hr_cq->cqn;
 		ret = ib_copy_to_udata(udata, &resp, sizeof(resp));
 		if (ret)
@@ -429,21 +422,20 @@ err_cqc:
 	hns_roce_free_cq(hr_dev, hr_cq);
 
 err_dbmap:
-	if (context && (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) &&
+	if (udata && (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) &&
 	    (udata->outlen >= sizeof(resp)))
-		hns_roce_db_unmap_user(to_hr_ucontext(context),
-				       &hr_cq->db);
+		hns_roce_db_unmap_user(context, &hr_cq->db);
 
 err_mtt:
 	hns_roce_mtt_cleanup(hr_dev, &hr_cq->hr_buf.hr_mtt);
-	if (context)
+	if (udata)
 		ib_umem_release(hr_cq->umem);
 	else
 		hns_roce_ib_free_cq_buf(hr_dev, &hr_cq->hr_buf,
 					hr_cq->ib_cq.cqe);
 
 err_db:
-	if (!context && (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB))
+	if (!udata && (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB))
 		hns_roce_free_db(hr_dev, &hr_cq->db);
 
 err_cq:
@@ -452,24 +444,27 @@ err_cq:
 }
 EXPORT_SYMBOL_GPL(hns_roce_ib_create_cq);
 
-int hns_roce_ib_destroy_cq(struct ib_cq *ib_cq)
+int hns_roce_ib_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
 {
 	struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device);
 	struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq);
 	int ret = 0;
 
 	if (hr_dev->hw->destroy_cq) {
-		ret = hr_dev->hw->destroy_cq(ib_cq);
+		ret = hr_dev->hw->destroy_cq(ib_cq, udata);
 	} else {
 		hns_roce_free_cq(hr_dev, hr_cq);
 		hns_roce_mtt_cleanup(hr_dev, &hr_cq->hr_buf.hr_mtt);
 
-		if (ib_cq->uobject) {
+		if (udata) {
 			ib_umem_release(hr_cq->umem);
 
 			if (hr_cq->db_en == 1)
 				hns_roce_db_unmap_user(
-					to_hr_ucontext(ib_cq->uobject->context),
+					rdma_udata_to_drv_context(
+						udata,
+						struct hns_roce_ucontext,
+						ibucontext),
 					&hr_cq->db);
 		} else {
 			/* Free the buff of stored cq */
@@ -491,8 +486,7 @@ void hns_roce_cq_completion(struct hns_roce_dev *hr_dev, u32 cqn)
 	struct device *dev = hr_dev->dev;
 	struct hns_roce_cq *cq;
 
-	cq = radix_tree_lookup(&hr_dev->cq_table.tree,
-			       cqn & (hr_dev->caps.num_cqs - 1));
+	cq = xa_load(&hr_dev->cq_table.array, cqn & (hr_dev->caps.num_cqs - 1));
 	if (!cq) {
 		dev_warn(dev, "Completion event for bogus CQ 0x%08x\n", cqn);
 		return;
@@ -509,8 +503,7 @@ void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type)
 	struct device *dev = hr_dev->dev;
 	struct hns_roce_cq *cq;
 
-	cq = radix_tree_lookup(&cq_table->tree,
-			       cqn & (hr_dev->caps.num_cqs - 1));
+	cq = xa_load(&cq_table->array, cqn & (hr_dev->caps.num_cqs - 1));
 	if (cq)
 		atomic_inc(&cq->refcount);
 
@@ -530,8 +523,7 @@ int hns_roce_init_cq_table(struct hns_roce_dev *hr_dev)
 {
 	struct hns_roce_cq_table *cq_table = &hr_dev->cq_table;
 
-	spin_lock_init(&cq_table->lock);
-	INIT_RADIX_TREE(&cq_table->tree, GFP_ATOMIC);
+	xa_init(&cq_table->array);
 
 	return hns_roce_bitmap_init(&cq_table->bitmap, hr_dev->caps.num_cqs,
 				    hr_dev->caps.num_cqs - 1,
diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h
index 9ee86daf1700..563cf39df6d5 100644
--- a/drivers/infiniband/hw/hns/hns_roce_device.h
+++ b/drivers/infiniband/hw/hns/hns_roce_device.h
@@ -505,7 +505,6 @@ struct hns_roce_uar_table {
 
 struct hns_roce_qp_table {
 	struct hns_roce_bitmap		bitmap;
-	spinlock_t			lock;
 	struct hns_roce_hem_table	qp_table;
 	struct hns_roce_hem_table	irrl_table;
 	struct hns_roce_hem_table	trrl_table;
@@ -515,8 +514,7 @@ struct hns_roce_qp_table {
 
 struct hns_roce_cq_table {
 	struct hns_roce_bitmap		bitmap;
-	spinlock_t			lock;
-	struct radix_tree_root		tree;
+	struct xarray			array;
 	struct hns_roce_hem_table	table;
 };
 
@@ -869,6 +867,11 @@ struct hns_roce_work {
 	int sub_type;
 };
 
+struct hns_roce_dfx_hw {
+	int (*query_cqc_info)(struct hns_roce_dev *hr_dev, u32 cqn,
+			      int *buffer);
+};
+
 struct hns_roce_hw {
 	int (*reset)(struct hns_roce_dev *hr_dev, bool enable);
 	int (*cmq_init)(struct hns_roce_dev *hr_dev);
@@ -907,7 +910,7 @@ struct hns_roce_hw {
 	int (*modify_qp)(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
 			 int attr_mask, enum ib_qp_state cur_state,
 			 enum ib_qp_state new_state);
-	int (*destroy_qp)(struct ib_qp *ibqp);
+	int (*destroy_qp)(struct ib_qp *ibqp, struct ib_udata *udata);
 	int (*qp_flow_control_init)(struct hns_roce_dev *hr_dev,
 			 struct hns_roce_qp *hr_qp);
 	int (*post_send)(struct ib_qp *ibqp, const struct ib_send_wr *wr,
@@ -916,8 +919,9 @@ struct hns_roce_hw {
 			 const struct ib_recv_wr **bad_recv_wr);
 	int (*req_notify_cq)(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
 	int (*poll_cq)(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
-	int (*dereg_mr)(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr);
-	int (*destroy_cq)(struct ib_cq *ibcq);
+	int (*dereg_mr)(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr,
+			struct ib_udata *udata);
+	int (*destroy_cq)(struct ib_cq *ibcq, struct ib_udata *udata);
 	int (*modify_cq)(struct ib_cq *cq, u16 cq_count, u16 cq_period);
 	int (*init_eq)(struct hns_roce_dev *hr_dev);
 	void (*cleanup_eq)(struct hns_roce_dev *hr_dev);
@@ -956,7 +960,7 @@ struct hns_roce_dev {
 	int			irq[HNS_ROCE_MAX_IRQ_NUM];
 	u8 __iomem		*reg_base;
 	struct hns_roce_caps	caps;
-	struct radix_tree_root  qp_table_tree;
+	struct xarray		qp_table_xa;
 
 	unsigned char	dev_addr[HNS_ROCE_MAX_PORTS][MAC_ADDR_OCTET_NUM];
 	u64			sys_image_guid;
@@ -985,6 +989,7 @@ struct hns_roce_dev {
 	const struct hns_roce_hw *hw;
 	void			*priv;
 	struct workqueue_struct *irq_workq;
+	const struct hns_roce_dfx_hw *dfx;
 };
 
 static inline struct hns_roce_dev *to_hr_dev(struct ib_device *ib_dev)
@@ -1046,8 +1051,7 @@ static inline void hns_roce_write64_k(__le32 val[2], void __iomem *dest)
 static inline struct hns_roce_qp
 	*__hns_roce_qp_lookup(struct hns_roce_dev *hr_dev, u32 qpn)
 {
-	return radix_tree_lookup(&hr_dev->qp_table_tree,
-				 qpn & (hr_dev->caps.num_qps - 1));
+	return xa_load(&hr_dev->qp_table_xa, qpn & (hr_dev->caps.num_qps - 1));
 }
 
 static inline void *hns_roce_buf_offset(struct hns_roce_buf *buf, int offset)
@@ -1107,16 +1111,13 @@ void hns_roce_bitmap_free_range(struct hns_roce_bitmap *bitmap,
 				unsigned long obj, int cnt,
 				int rr);
 
-struct ib_ah *hns_roce_create_ah(struct ib_pd *pd,
-				 struct rdma_ah_attr *ah_attr,
-				 u32 flags,
-				 struct ib_udata *udata);
+int hns_roce_create_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr,
+		       u32 flags, struct ib_udata *udata);
 int hns_roce_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr);
-int hns_roce_destroy_ah(struct ib_ah *ah, u32 flags);
+void hns_roce_destroy_ah(struct ib_ah *ah, u32 flags);
 
-int hns_roce_alloc_pd(struct ib_pd *pd, struct ib_ucontext *context,
-		      struct ib_udata *udata);
-void hns_roce_dealloc_pd(struct ib_pd *pd);
+int hns_roce_alloc_pd(struct ib_pd *pd, struct ib_udata *udata);
+void hns_roce_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata);
 
 struct ib_mr *hns_roce_get_dma_mr(struct ib_pd *pd, int acc);
 struct ib_mr *hns_roce_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
@@ -1126,10 +1127,10 @@ int hns_roce_rereg_user_mr(struct ib_mr *mr, int flags, u64 start, u64 length,
 			   u64 virt_addr, int mr_access_flags, struct ib_pd *pd,
 			   struct ib_udata *udata);
 struct ib_mr *hns_roce_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
-				u32 max_num_sg);
+				u32 max_num_sg, struct ib_udata *udata);
 int hns_roce_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
 		       unsigned int *sg_offset);
-int hns_roce_dereg_mr(struct ib_mr *ibmr);
+int hns_roce_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
 int hns_roce_hw2sw_mpt(struct hns_roce_dev *hr_dev,
 		       struct hns_roce_cmd_mailbox *mailbox,
 		       unsigned long mpt_index);
@@ -1147,13 +1148,13 @@ int hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size, u32 max_direct,
 int hns_roce_ib_umem_write_mtt(struct hns_roce_dev *hr_dev,
 			       struct hns_roce_mtt *mtt, struct ib_umem *umem);
 
-struct ib_srq *hns_roce_create_srq(struct ib_pd *pd,
-				   struct ib_srq_init_attr *srq_init_attr,
-				   struct ib_udata *udata);
+int hns_roce_create_srq(struct ib_srq *srq,
+			struct ib_srq_init_attr *srq_init_attr,
+			struct ib_udata *udata);
 int hns_roce_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr,
 			enum ib_srq_attr_mask srq_attr_mask,
 			struct ib_udata *udata);
-int hns_roce_destroy_srq(struct ib_srq *ibsrq);
+void hns_roce_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata);
 
 struct ib_qp *hns_roce_create_qp(struct ib_pd *ib_pd,
 				 struct ib_qp_init_attr *init_attr,
@@ -1179,10 +1180,9 @@ int to_hr_qp_type(int qp_type);
 
 struct ib_cq *hns_roce_ib_create_cq(struct ib_device *ib_dev,
 				    const struct ib_cq_init_attr *attr,
-				    struct ib_ucontext *context,
 				    struct ib_udata *udata);
 
-int hns_roce_ib_destroy_cq(struct ib_cq *ib_cq);
+int hns_roce_ib_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata);
 void hns_roce_free_cq(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq);
 
 int hns_roce_db_map_user(struct hns_roce_ucontext *context,
@@ -1202,4 +1202,6 @@ int hns_get_gid_index(struct hns_roce_dev *hr_dev, u8 port, int gid_index);
 int hns_roce_init(struct hns_roce_dev *hr_dev);
 void hns_roce_exit(struct hns_roce_dev *hr_dev);
 
+int hns_roce_fill_res_entry(struct sk_buff *msg,
+			    struct rdma_restrack_entry *res);
 #endif /* _HNS_ROCE_DEVICE_H */
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c
index c8555f7704d8..4c5d0f160c10 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c
@@ -730,7 +730,7 @@ static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev)
 	/* Reserved cq for loop qp */
 	cq_init_attr.cqe		= HNS_ROCE_MIN_WQE_NUM * 2;
 	cq_init_attr.comp_vector	= 0;
-	cq = hns_roce_ib_create_cq(&hr_dev->ib_dev, &cq_init_attr, NULL, NULL);
+	cq = hns_roce_ib_create_cq(&hr_dev->ib_dev, &cq_init_attr, NULL);
 	if (IS_ERR(cq)) {
 		dev_err(dev, "Create cq for reserved loop qp failed!");
 		return -ENOMEM;
@@ -749,7 +749,7 @@ static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev)
 		goto alloc_mem_failed;
 
 	pd->device  = ibdev;
-	ret = hns_roce_alloc_pd(pd, NULL, NULL);
+	ret = hns_roce_alloc_pd(pd, NULL);
 	if (ret)
 		goto alloc_pd_failed;
 
@@ -855,17 +855,17 @@ static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev)
 create_lp_qp_failed:
 	for (i -= 1; i >= 0; i--) {
 		hr_qp = free_mr->mr_free_qp[i];
-		if (hns_roce_v1_destroy_qp(&hr_qp->ibqp))
+		if (hns_roce_v1_destroy_qp(&hr_qp->ibqp, NULL))
 			dev_err(dev, "Destroy qp %d for mr free failed!\n", i);
 	}
 
-	hns_roce_dealloc_pd(pd);
+	hns_roce_dealloc_pd(pd, NULL);
 
 alloc_pd_failed:
 	kfree(pd);
 
 alloc_mem_failed:
-	if (hns_roce_ib_destroy_cq(cq))
+	if (hns_roce_ib_destroy_cq(cq, NULL))
 		dev_err(dev, "Destroy cq for create_lp_qp failed!\n");
 
 	return ret;
@@ -888,17 +888,17 @@ static void hns_roce_v1_release_lp_qp(struct hns_roce_dev *hr_dev)
 		if (!hr_qp)
 			continue;
 
-		ret = hns_roce_v1_destroy_qp(&hr_qp->ibqp);
+		ret = hns_roce_v1_destroy_qp(&hr_qp->ibqp, NULL);
 		if (ret)
 			dev_err(dev, "Destroy qp %d for mr free failed(%d)!\n",
 				i, ret);
 	}
 
-	ret = hns_roce_ib_destroy_cq(&free_mr->mr_free_cq->ib_cq);
+	ret = hns_roce_ib_destroy_cq(&free_mr->mr_free_cq->ib_cq, NULL);
 	if (ret)
 		dev_err(dev, "Destroy cq for mr_free failed(%d)!\n", ret);
 
-	hns_roce_dealloc_pd(&free_mr->mr_free_pd->ibpd);
+	hns_roce_dealloc_pd(&free_mr->mr_free_pd->ibpd, NULL);
 }
 
 static int hns_roce_db_init(struct hns_roce_dev *hr_dev)
@@ -1096,7 +1096,7 @@ free_work:
 }
 
 static int hns_roce_v1_dereg_mr(struct hns_roce_dev *hr_dev,
-				struct hns_roce_mr *mr)
+				struct hns_roce_mr *mr, struct ib_udata *udata)
 {
 	struct device *dev = &hr_dev->pdev->dev;
 	struct hns_roce_mr_free_work *mr_work;
@@ -1511,38 +1511,6 @@ static int hns_roce_v1_reset(struct hns_roce_dev *hr_dev, bool dereset)
 	return ret;
 }
 
-static int hns_roce_des_qp_init(struct hns_roce_dev *hr_dev)
-{
-	struct device *dev = &hr_dev->pdev->dev;
-	struct hns_roce_v1_priv *priv;
-	struct hns_roce_des_qp *des_qp;
-
-	priv = (struct hns_roce_v1_priv *)hr_dev->priv;
-	des_qp = &priv->des_qp;
-
-	des_qp->requeue_flag = 1;
-	des_qp->qp_wq = create_singlethread_workqueue("hns_roce_destroy_qp");
-	if (!des_qp->qp_wq) {
-		dev_err(dev, "Create destroy qp workqueue failed!\n");
-		return -ENOMEM;
-	}
-
-	return 0;
-}
-
-static void hns_roce_des_qp_free(struct hns_roce_dev *hr_dev)
-{
-	struct hns_roce_v1_priv *priv;
-	struct hns_roce_des_qp *des_qp;
-
-	priv = (struct hns_roce_v1_priv *)hr_dev->priv;
-	des_qp = &priv->des_qp;
-
-	des_qp->requeue_flag = 0;
-	flush_workqueue(des_qp->qp_wq);
-	destroy_workqueue(des_qp->qp_wq);
-}
-
 static int hns_roce_v1_profile(struct hns_roce_dev *hr_dev)
 {
 	int i = 0;
@@ -1661,12 +1629,6 @@ static int hns_roce_v1_init(struct hns_roce_dev *hr_dev)
 		goto error_failed_tptr_init;
 	}
 
-	ret = hns_roce_des_qp_init(hr_dev);
-	if (ret) {
-		dev_err(dev, "des qp init failed!\n");
-		goto error_failed_des_qp_init;
-	}
-
 	ret = hns_roce_free_mr_init(hr_dev);
 	if (ret) {
 		dev_err(dev, "free mr init failed!\n");
@@ -1678,9 +1640,6 @@ static int hns_roce_v1_init(struct hns_roce_dev *hr_dev)
 	return 0;
 
 error_failed_free_mr_init:
-	hns_roce_des_qp_free(hr_dev);
-
-error_failed_des_qp_init:
 	hns_roce_tptr_free(hr_dev);
 
 error_failed_tptr_init:
@@ -1698,7 +1657,6 @@ static void hns_roce_v1_exit(struct hns_roce_dev *hr_dev)
 {
 	hns_roce_port_enable(hr_dev, HNS_ROCE_PORT_DOWN);
 	hns_roce_free_mr_free(hr_dev);
-	hns_roce_des_qp_free(hr_dev);
 	hns_roce_tptr_free(hr_dev);
 	hns_roce_bt_free(hr_dev);
 	hns_roce_raq_free(hr_dev);
@@ -3642,307 +3600,22 @@ static int hns_roce_v1_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
 		hns_roce_v1_q_qp(ibqp, qp_attr, qp_attr_mask, qp_init_attr);
 }
 
-static void hns_roce_check_sdb_status(struct hns_roce_dev *hr_dev,
-				      u32 *old_send, u32 *old_retry,
-				      u32 *tsp_st, u32 *success_flags)
-{
-	__le32 *old_send_tmp, *old_retry_tmp;
-	u32 sdb_retry_cnt;
-	u32 sdb_send_ptr;
-	u32 cur_cnt, old_cnt;
-	__le32 tmp, tmp1;
-	u32 send_ptr;
-
-	sdb_send_ptr = roce_read(hr_dev, ROCEE_SDB_SEND_PTR_REG);
-	sdb_retry_cnt =	roce_read(hr_dev, ROCEE_SDB_RETRY_CNT_REG);
-	tmp = cpu_to_le32(sdb_send_ptr);
-	tmp1 = cpu_to_le32(sdb_retry_cnt);
-	cur_cnt = roce_get_field(tmp, ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M,
-				 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S) +
-		  roce_get_field(tmp1, ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_M,
-				 ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_S);
-
-	old_send_tmp = (__le32 *)old_send;
-	old_retry_tmp = (__le32 *)old_retry;
-	if (!roce_get_bit(*tsp_st, ROCEE_CNT_CLR_CE_CNT_CLR_CE_S)) {
-		old_cnt = roce_get_field(*old_send_tmp,
-					 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M,
-					 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S) +
-			  roce_get_field(*old_retry_tmp,
-					 ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_M,
-					 ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_S);
-		if (cur_cnt - old_cnt > SDB_ST_CMP_VAL)
-			*success_flags = 1;
-	} else {
-		old_cnt = roce_get_field(*old_send_tmp,
-					 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M,
-					 ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S);
-		if (cur_cnt - old_cnt > SDB_ST_CMP_VAL) {
-			*success_flags = 1;
-		} else {
-			send_ptr = roce_get_field(*old_send_tmp,
-					    ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M,
-					    ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S) +
-				   roce_get_field(tmp1,
-					    ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_M,
-					    ROCEE_SDB_RETRY_CNT_SDB_RETRY_CT_S);
-			roce_set_field(*old_send_tmp,
-				       ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M,
-				       ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S,
-				       send_ptr);
-		}
-	}
-}
-
-static int check_qp_db_process_status(struct hns_roce_dev *hr_dev,
-				      struct hns_roce_qp *hr_qp,
-				      u32 sdb_issue_ptr,
-				      u32 *sdb_inv_cnt,
-				      u32 *wait_stage)
-{
-	struct device *dev = &hr_dev->pdev->dev;
-	u32 sdb_send_ptr, old_send;
-	__le32 sdb_issue_ptr_tmp;
-	__le32 sdb_send_ptr_tmp;
-	u32 success_flags = 0;
-	unsigned long end;
-	u32 old_retry;
-	u32 inv_cnt;
-	u32 tsp_st;
-	__le32 tmp;
-
-	if (*wait_stage > HNS_ROCE_V1_DB_STAGE2 ||
-	    *wait_stage < HNS_ROCE_V1_DB_STAGE1) {
-		dev_err(dev, "QP(0x%lx) db status wait stage(%d) error!\n",
-			hr_qp->qpn, *wait_stage);
-		return -EINVAL;
-	}
-
-	/* Calculate the total timeout for the entire verification process */
-	end = msecs_to_jiffies(HNS_ROCE_V1_CHECK_DB_TIMEOUT_MSECS) + jiffies;
-
-	if (*wait_stage == HNS_ROCE_V1_DB_STAGE1) {
-		/* Query db process status, until hw process completely */
-		sdb_send_ptr = roce_read(hr_dev, ROCEE_SDB_SEND_PTR_REG);
-		while (roce_hw_index_cmp_lt(sdb_send_ptr, sdb_issue_ptr,
-					    ROCEE_SDB_PTR_CMP_BITS)) {
-			if (!time_before(jiffies, end)) {
-				dev_dbg(dev, "QP(0x%lx) db process stage1 timeout. issue 0x%x send 0x%x.\n",
-					hr_qp->qpn, sdb_issue_ptr,
-					sdb_send_ptr);
-				return 0;
-			}
-
-			msleep(HNS_ROCE_V1_CHECK_DB_SLEEP_MSECS);
-			sdb_send_ptr = roce_read(hr_dev,
-						 ROCEE_SDB_SEND_PTR_REG);
-		}
-
-		sdb_send_ptr_tmp = cpu_to_le32(sdb_send_ptr);
-		sdb_issue_ptr_tmp = cpu_to_le32(sdb_issue_ptr);
-		if (roce_get_field(sdb_issue_ptr_tmp,
-				   ROCEE_SDB_ISSUE_PTR_SDB_ISSUE_PTR_M,
-				   ROCEE_SDB_ISSUE_PTR_SDB_ISSUE_PTR_S) ==
-		    roce_get_field(sdb_send_ptr_tmp,
-				   ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M,
-				   ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S)) {
-			old_send = roce_read(hr_dev, ROCEE_SDB_SEND_PTR_REG);
-			old_retry = roce_read(hr_dev, ROCEE_SDB_RETRY_CNT_REG);
-
-			do {
-				tsp_st = roce_read(hr_dev, ROCEE_TSP_BP_ST_REG);
-				tmp = cpu_to_le32(tsp_st);
-				if (roce_get_bit(tmp,
-					ROCEE_TSP_BP_ST_QH_FIFO_ENTRY_S) == 1) {
-					*wait_stage = HNS_ROCE_V1_DB_WAIT_OK;
-					return 0;
-				}
-
-				if (!time_before(jiffies, end)) {
-					dev_dbg(dev, "QP(0x%lx) db process stage1 timeout when send ptr equals issue ptr.\n"
-						     "issue 0x%x send 0x%x.\n",
-						hr_qp->qpn,
-						le32_to_cpu(sdb_issue_ptr_tmp),
-						le32_to_cpu(sdb_send_ptr_tmp));
-					return 0;
-				}
-
-				msleep(HNS_ROCE_V1_CHECK_DB_SLEEP_MSECS);
-
-				hns_roce_check_sdb_status(hr_dev, &old_send,
-							  &old_retry, &tsp_st,
-							  &success_flags);
-			} while (!success_flags);
-		}
-
-		*wait_stage = HNS_ROCE_V1_DB_STAGE2;
-
-		/* Get list pointer */
-		*sdb_inv_cnt = roce_read(hr_dev, ROCEE_SDB_INV_CNT_REG);
-		dev_dbg(dev, "QP(0x%lx) db process stage2. inv cnt = 0x%x.\n",
-			hr_qp->qpn, *sdb_inv_cnt);
-	}
-
-	if (*wait_stage == HNS_ROCE_V1_DB_STAGE2) {
-		/* Query db's list status, until hw reversal */
-		inv_cnt = roce_read(hr_dev, ROCEE_SDB_INV_CNT_REG);
-		while (roce_hw_index_cmp_lt(inv_cnt,
-					    *sdb_inv_cnt + SDB_INV_CNT_OFFSET,
-					    ROCEE_SDB_CNT_CMP_BITS)) {
-			if (!time_before(jiffies, end)) {
-				dev_dbg(dev, "QP(0x%lx) db process stage2 timeout. inv cnt 0x%x.\n",
-					hr_qp->qpn, inv_cnt);
-				return 0;
-			}
-
-			msleep(HNS_ROCE_V1_CHECK_DB_SLEEP_MSECS);
-			inv_cnt = roce_read(hr_dev, ROCEE_SDB_INV_CNT_REG);
-		}
-
-		*wait_stage = HNS_ROCE_V1_DB_WAIT_OK;
-	}
-
-	return 0;
-}
-
-static int check_qp_reset_state(struct hns_roce_dev *hr_dev,
-				struct hns_roce_qp *hr_qp,
-				struct hns_roce_qp_work *qp_work_entry,
-				int *is_timeout)
-{
-	struct device *dev = &hr_dev->pdev->dev;
-	u32 sdb_issue_ptr;
-	int ret;
-
-	if (hr_qp->state != IB_QPS_RESET) {
-		/* Set qp to ERR, waiting for hw complete processing all dbs */
-		ret = hns_roce_v1_modify_qp(&hr_qp->ibqp, NULL, 0, hr_qp->state,
-					    IB_QPS_ERR);
-		if (ret) {
-			dev_err(dev, "Modify QP(0x%lx) to ERR failed!\n",
-				hr_qp->qpn);
-			return ret;
-		}
-
-		/* Record issued doorbell */
-		sdb_issue_ptr = roce_read(hr_dev, ROCEE_SDB_ISSUE_PTR_REG);
-		qp_work_entry->sdb_issue_ptr = sdb_issue_ptr;
-		qp_work_entry->db_wait_stage = HNS_ROCE_V1_DB_STAGE1;
-
-		/* Query db process status, until hw process completely */
-		ret = check_qp_db_process_status(hr_dev, hr_qp, sdb_issue_ptr,
-						 &qp_work_entry->sdb_inv_cnt,
-						 &qp_work_entry->db_wait_stage);
-		if (ret) {
-			dev_err(dev, "Check QP(0x%lx) db process status failed!\n",
-				hr_qp->qpn);
-			return ret;
-		}
-
-		if (qp_work_entry->db_wait_stage != HNS_ROCE_V1_DB_WAIT_OK) {
-			qp_work_entry->sche_cnt = 0;
-			*is_timeout = 1;
-			return 0;
-		}
-
-		/* Modify qp to reset before destroying qp */
-		ret = hns_roce_v1_modify_qp(&hr_qp->ibqp, NULL, 0, hr_qp->state,
-					    IB_QPS_RESET);
-		if (ret) {
-			dev_err(dev, "Modify QP(0x%lx) to RST failed!\n",
-				hr_qp->qpn);
-			return ret;
-		}
-	}
-
-	return 0;
-}
-
-static void hns_roce_v1_destroy_qp_work_fn(struct work_struct *work)
-{
-	struct hns_roce_qp_work *qp_work_entry;
-	struct hns_roce_v1_priv *priv;
-	struct hns_roce_dev *hr_dev;
-	struct hns_roce_qp *hr_qp;
-	struct device *dev;
-	unsigned long qpn;
-	int ret;
-
-	qp_work_entry = container_of(work, struct hns_roce_qp_work, work);
-	hr_dev = to_hr_dev(qp_work_entry->ib_dev);
-	dev = &hr_dev->pdev->dev;
-	priv = (struct hns_roce_v1_priv *)hr_dev->priv;
-	hr_qp = qp_work_entry->qp;
-	qpn = hr_qp->qpn;
-
-	dev_dbg(dev, "Schedule destroy QP(0x%lx) work.\n", qpn);
-
-	qp_work_entry->sche_cnt++;
-
-	/* Query db process status, until hw process completely */
-	ret = check_qp_db_process_status(hr_dev, hr_qp,
-					 qp_work_entry->sdb_issue_ptr,
-					 &qp_work_entry->sdb_inv_cnt,
-					 &qp_work_entry->db_wait_stage);
-	if (ret) {
-		dev_err(dev, "Check QP(0x%lx) db process status failed!\n",
-			qpn);
-		return;
-	}
-
-	if (qp_work_entry->db_wait_stage != HNS_ROCE_V1_DB_WAIT_OK &&
-	    priv->des_qp.requeue_flag) {
-		queue_work(priv->des_qp.qp_wq, work);
-		return;
-	}
-
-	/* Modify qp to reset before destroying qp */
-	ret = hns_roce_v1_modify_qp(&hr_qp->ibqp, NULL, 0, hr_qp->state,
-				    IB_QPS_RESET);
-	if (ret) {
-		dev_err(dev, "Modify QP(0x%lx) to RST failed!\n", qpn);
-		return;
-	}
-
-	hns_roce_qp_remove(hr_dev, hr_qp);
-	hns_roce_qp_free(hr_dev, hr_qp);
-
-	if (hr_qp->ibqp.qp_type == IB_QPT_RC) {
-		/* RC QP, release QPN */
-		hns_roce_release_range_qp(hr_dev, qpn, 1);
-		kfree(hr_qp);
-	} else
-		kfree(hr_to_hr_sqp(hr_qp));
-
-	kfree(qp_work_entry);
-
-	dev_dbg(dev, "Accomplished destroy QP(0x%lx) work.\n", qpn);
-}
-
-int hns_roce_v1_destroy_qp(struct ib_qp *ibqp)
+int hns_roce_v1_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
 {
 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
 	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
-	struct device *dev = &hr_dev->pdev->dev;
-	struct hns_roce_qp_work qp_work_entry;
-	struct hns_roce_qp_work *qp_work;
-	struct hns_roce_v1_priv *priv;
 	struct hns_roce_cq *send_cq, *recv_cq;
-	bool is_user = ibqp->uobject;
-	int is_timeout = 0;
 	int ret;
 
-	ret = check_qp_reset_state(hr_dev, hr_qp, &qp_work_entry, &is_timeout);
-	if (ret) {
-		dev_err(dev, "QP reset state check failed(%d)!\n", ret);
+	ret = hns_roce_v1_modify_qp(ibqp, NULL, 0, hr_qp->state, IB_QPS_RESET);
+	if (ret)
 		return ret;
-	}
 
 	send_cq = to_hr_cq(hr_qp->ibqp.send_cq);
 	recv_cq = to_hr_cq(hr_qp->ibqp.recv_cq);
 
 	hns_roce_lock_cqs(send_cq, recv_cq);
-	if (!is_user) {
+	if (!udata) {
 		__hns_roce_v1_cq_clean(recv_cq, hr_qp->qpn, hr_qp->ibqp.srq ?
 				       to_hr_srq(hr_qp->ibqp.srq) : NULL);
 		if (send_cq != recv_cq)
@@ -3950,18 +3623,16 @@ int hns_roce_v1_destroy_qp(struct ib_qp *ibqp)
 	}
 	hns_roce_unlock_cqs(send_cq, recv_cq);
 
-	if (!is_timeout) {
-		hns_roce_qp_remove(hr_dev, hr_qp);
-		hns_roce_qp_free(hr_dev, hr_qp);
+	hns_roce_qp_remove(hr_dev, hr_qp);
+	hns_roce_qp_free(hr_dev, hr_qp);
 
-		/* RC QP, release QPN */
-		if (hr_qp->ibqp.qp_type == IB_QPT_RC)
-			hns_roce_release_range_qp(hr_dev, hr_qp->qpn, 1);
-	}
+	/* RC QP, release QPN */
+	if (hr_qp->ibqp.qp_type == IB_QPT_RC)
+		hns_roce_release_range_qp(hr_dev, hr_qp->qpn, 1);
 
 	hns_roce_mtt_cleanup(hr_dev, &hr_qp->mtt);
 
-	if (is_user)
+	if (udata)
 		ib_umem_release(hr_qp->umem);
 	else {
 		kfree(hr_qp->sq.wrid);
@@ -3970,33 +3641,14 @@ int hns_roce_v1_destroy_qp(struct ib_qp *ibqp)
 		hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf);
 	}
 
-	if (!is_timeout) {
-		if (hr_qp->ibqp.qp_type == IB_QPT_RC)
-			kfree(hr_qp);
-		else
-			kfree(hr_to_hr_sqp(hr_qp));
-	} else {
-		qp_work = kzalloc(sizeof(*qp_work), GFP_KERNEL);
-		if (!qp_work)
-			return -ENOMEM;
-
-		INIT_WORK(&qp_work->work, hns_roce_v1_destroy_qp_work_fn);
-		qp_work->ib_dev	= &hr_dev->ib_dev;
-		qp_work->qp		= hr_qp;
-		qp_work->db_wait_stage	= qp_work_entry.db_wait_stage;
-		qp_work->sdb_issue_ptr	= qp_work_entry.sdb_issue_ptr;
-		qp_work->sdb_inv_cnt	= qp_work_entry.sdb_inv_cnt;
-		qp_work->sche_cnt	= qp_work_entry.sche_cnt;
-
-		priv = (struct hns_roce_v1_priv *)hr_dev->priv;
-		queue_work(priv->des_qp.qp_wq, &qp_work->work);
-		dev_dbg(dev, "Begin destroy QP(0x%lx) work.\n", hr_qp->qpn);
-	}
-
+	if (hr_qp->ibqp.qp_type == IB_QPT_RC)
+		kfree(hr_qp);
+	else
+		kfree(hr_to_hr_sqp(hr_qp));
 	return 0;
 }
 
-static int hns_roce_v1_destroy_cq(struct ib_cq *ibcq)
+static int hns_roce_v1_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
 {
 	struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device);
 	struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.h b/drivers/infiniband/hw/hns/hns_roce_hw_v1.h
index 66440147d9eb..52307b2c7100 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.h
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.h
@@ -110,11 +110,6 @@
 #define HNS_ROCE_V1_EXT_ODB_ALFUL	\
 	(HNS_ROCE_V1_EXT_ODB_DEPTH - HNS_ROCE_V1_DB_RSVD)
 
-#define HNS_ROCE_V1_DB_WAIT_OK				0
-#define HNS_ROCE_V1_DB_STAGE1				1
-#define HNS_ROCE_V1_DB_STAGE2				2
-#define HNS_ROCE_V1_CHECK_DB_TIMEOUT_MSECS		10000
-#define HNS_ROCE_V1_CHECK_DB_SLEEP_MSECS		20
 #define HNS_ROCE_V1_FREE_MR_TIMEOUT_MSECS		50000
 #define HNS_ROCE_V1_RECREATE_LP_QP_TIMEOUT_MSECS	10000
 #define HNS_ROCE_V1_FREE_MR_WAIT_VALUE			5
@@ -162,7 +157,6 @@
 #define SQ_PSN_SHIFT					8
 #define QKEY_VAL					0x80010000
 #define SDB_INV_CNT_OFFSET				8
-#define SDB_ST_CMP_VAL					8
 
 #define HNS_ROCE_CEQ_DEFAULT_INTERVAL			0x10
 #define HNS_ROCE_CEQ_DEFAULT_BURST_NUM			0x10
@@ -1068,11 +1062,6 @@ struct hns_roce_qp_work {
 	u32	sche_cnt;
 };
 
-struct hns_roce_des_qp {
-	struct workqueue_struct	*qp_wq;
-	int	requeue_flag;
-};
-
 struct hns_roce_mr_free_work {
 	struct	work_struct work;
 	struct	ib_device *ib_dev;
@@ -1100,12 +1089,11 @@ struct hns_roce_v1_priv {
 	struct hns_roce_raq_table raq_table;
 	struct hns_roce_bt_table  bt_table;
 	struct hns_roce_tptr_table tptr_table;
-	struct hns_roce_des_qp des_qp;
 	struct hns_roce_free_mr free_mr;
 };
 
 int hns_dsaf_roce_reset(struct fwnode_handle *dsaf_fwnode, bool dereset);
 int hns_roce_v1_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
-int hns_roce_v1_destroy_qp(struct ib_qp *ibqp);
+int hns_roce_v1_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);
 
 #endif
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
index 1c54390e1c85..b5392cb5b20f 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
@@ -37,7 +37,9 @@
 #include <linux/types.h>
 #include <net/addrconf.h>
 #include <rdma/ib_addr.h>
+#include <rdma/ib_cache.h>
 #include <rdma/ib_umem.h>
+#include <rdma/uverbs_ioctl.h>
 
 #include "hnae3.h"
 #include "hns_roce_common.h"
@@ -1086,7 +1088,7 @@ static int __hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
 	return ret;
 }
 
-int hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
+static int hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
 			     struct hns_roce_cmq_desc *desc, int num)
 {
 	int retval;
@@ -1559,7 +1561,7 @@ static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev)
 	caps->qpc_hop_num	= HNS_ROCE_CONTEXT_HOP_NUM;
 	caps->srqc_ba_pg_sz	= 0;
 	caps->srqc_buf_pg_sz	= 0;
-	caps->srqc_hop_num	= HNS_ROCE_HOP_NUM_0;
+	caps->srqc_hop_num	= HNS_ROCE_CONTEXT_HOP_NUM;
 	caps->cqc_ba_pg_sz	= 0;
 	caps->cqc_buf_pg_sz	= 0;
 	caps->cqc_hop_num	= HNS_ROCE_CONTEXT_HOP_NUM;
@@ -2150,7 +2152,7 @@ static int hns_roce_v2_write_mtpt(void *mb_buf, struct hns_roce_mr *mr,
 		       V2_MPT_BYTE_4_PD_S, mr->pd);
 
 	roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RA_EN_S, 0);
-	roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 1);
+	roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 0);
 	roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1);
 	roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_BIND_EN_S,
 		     (mr->access & IB_ACCESS_MW_BIND ? 1 : 0));
@@ -3171,12 +3173,6 @@ static void modify_qp_reset_to_init(struct ib_qp *ibqp,
 	roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_CNP_TX_FLAG_S, 0);
 	roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_CE_FLAG_S, 0);
 
-	if (attr_mask & IB_QP_QKEY) {
-		context->qkey_xrcd = attr->qkey;
-		qpc_mask->qkey_xrcd = 0;
-		hr_qp->qkey = attr->qkey;
-	}
-
 	if (hr_qp->rdb_en) {
 		roce_set_bit(context->byte_68_rq_db,
 			     V2_QPC_BYTE_68_RQ_RECORD_EN_S, 1);
@@ -3388,7 +3384,6 @@ static void modify_qp_reset_to_init(struct ib_qp *ibqp,
 		     0);
 
 	hr_qp->access_flags = attr->qp_access_flags;
-	hr_qp->pkey_index = attr->pkey_index;
 	roce_set_field(context->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
 		       V2_QPC_BYTE_252_TX_CQN_S, to_hr_cq(ibqp->send_cq)->cqn);
 	roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
@@ -3512,11 +3507,6 @@ static void modify_qp_init_to_init(struct ib_qp *ibqp,
 			       V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, 0);
 	}
 
-	if (attr_mask & IB_QP_QKEY) {
-		context->qkey_xrcd = attr->qkey;
-		qpc_mask->qkey_xrcd = 0;
-	}
-
 	roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
 		       V2_QPC_BYTE_4_SQPN_S, hr_qp->qpn);
 	roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
@@ -3636,13 +3626,6 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
 		       V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M,
 		       V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S, 0);
 
-	roce_set_field(context->byte_80_rnr_rx_cqn,
-		       V2_QPC_BYTE_80_MIN_RNR_TIME_M,
-		       V2_QPC_BYTE_80_MIN_RNR_TIME_S, attr->min_rnr_timer);
-	roce_set_field(qpc_mask->byte_80_rnr_rx_cqn,
-		       V2_QPC_BYTE_80_MIN_RNR_TIME_M,
-		       V2_QPC_BYTE_80_MIN_RNR_TIME_S, 0);
-
 	page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT);
 	context->rq_cur_blk_addr = (u32)(mtts[hr_qp->rq.offset / page_size]
 				    >> PAGE_ADDR_SHIFT);
@@ -3670,13 +3653,6 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
 		       V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M,
 		       V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S, 0);
 
-	roce_set_field(context->byte_108_rx_reqepsn,
-		       V2_QPC_BYTE_108_RX_REQ_EPSN_M,
-		       V2_QPC_BYTE_108_RX_REQ_EPSN_S, attr->rq_psn);
-	roce_set_field(qpc_mask->byte_108_rx_reqepsn,
-		       V2_QPC_BYTE_108_RX_REQ_EPSN_M,
-		       V2_QPC_BYTE_108_RX_REQ_EPSN_S, 0);
-
 	roce_set_field(context->byte_132_trrl, V2_QPC_BYTE_132_TRRL_BA_M,
 		       V2_QPC_BYTE_132_TRRL_BA_S, dma_handle_3 >> 4);
 	roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_BA_M,
@@ -3715,15 +3691,6 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
 		roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_LBI_S, 0);
 	}
 
-	if ((attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) &&
-	     attr->max_dest_rd_atomic) {
-		roce_set_field(context->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M,
-			       V2_QPC_BYTE_140_RR_MAX_S,
-			       fls(attr->max_dest_rd_atomic - 1));
-		roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M,
-			       V2_QPC_BYTE_140_RR_MAX_S, 0);
-	}
-
 	if (attr_mask & IB_QP_DEST_QPN) {
 		roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_DQPN_M,
 			       V2_QPC_BYTE_56_DQPN_S, attr->dest_qp_num);
@@ -3784,11 +3751,6 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
 	context->rq_rnr_timer = 0;
 	qpc_mask->rq_rnr_timer = 0;
 
-	roce_set_field(context->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M,
-		       V2_QPC_BYTE_152_RAQ_PSN_S, attr->rq_psn - 1);
-	roce_set_field(qpc_mask->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M,
-		       V2_QPC_BYTE_152_RAQ_PSN_S, 0);
-
 	roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_HEAD_MAX_M,
 		       V2_QPC_BYTE_132_TRRL_HEAD_MAX_S, 0);
 	roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_TAIL_MAX_M,
@@ -3886,13 +3848,6 @@ static int modify_qp_rtr_to_rts(struct ib_qp *ibqp,
 		       V2_QPC_BYTE_240_RX_ACK_MSN_M,
 		       V2_QPC_BYTE_240_RX_ACK_MSN_S, 0);
 
-	roce_set_field(context->byte_244_rnr_rxack,
-		       V2_QPC_BYTE_244_RX_ACK_EPSN_M,
-		       V2_QPC_BYTE_244_RX_ACK_EPSN_S, attr->sq_psn);
-	roce_set_field(qpc_mask->byte_244_rnr_rxack,
-		       V2_QPC_BYTE_244_RX_ACK_EPSN_M,
-		       V2_QPC_BYTE_244_RX_ACK_EPSN_S, 0);
-
 	roce_set_field(qpc_mask->byte_248_ack_psn,
 		       V2_QPC_BYTE_248_ACK_LAST_OPTYPE_M,
 		       V2_QPC_BYTE_248_ACK_LAST_OPTYPE_S, 0);
@@ -3906,27 +3861,6 @@ static int modify_qp_rtr_to_rts(struct ib_qp *ibqp,
 		       V2_QPC_BYTE_240_IRRL_TAIL_REAL_M,
 		       V2_QPC_BYTE_240_IRRL_TAIL_REAL_S, 0);
 
-	roce_set_field(context->byte_220_retry_psn_msn,
-		       V2_QPC_BYTE_220_RETRY_MSG_PSN_M,
-		       V2_QPC_BYTE_220_RETRY_MSG_PSN_S, attr->sq_psn);
-	roce_set_field(qpc_mask->byte_220_retry_psn_msn,
-		       V2_QPC_BYTE_220_RETRY_MSG_PSN_M,
-		       V2_QPC_BYTE_220_RETRY_MSG_PSN_S, 0);
-
-	roce_set_field(context->byte_224_retry_msg,
-		       V2_QPC_BYTE_224_RETRY_MSG_PSN_M,
-		       V2_QPC_BYTE_224_RETRY_MSG_PSN_S, attr->sq_psn >> 16);
-	roce_set_field(qpc_mask->byte_224_retry_msg,
-		       V2_QPC_BYTE_224_RETRY_MSG_PSN_M,
-		       V2_QPC_BYTE_224_RETRY_MSG_PSN_S, 0);
-
-	roce_set_field(context->byte_224_retry_msg,
-		       V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M,
-		       V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S, attr->sq_psn);
-	roce_set_field(qpc_mask->byte_224_retry_msg,
-		       V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M,
-		       V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S, 0);
-
 	roce_set_field(qpc_mask->byte_220_retry_psn_msn,
 		       V2_QPC_BYTE_220_RETRY_MSG_MSN_M,
 		       V2_QPC_BYTE_220_RETRY_MSG_MSN_S, 0);
@@ -3937,66 +3871,14 @@ static int modify_qp_rtr_to_rts(struct ib_qp *ibqp,
 	roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_CHECK_FLG_M,
 		       V2_QPC_BYTE_212_CHECK_FLG_S, 0);
 
-	roce_set_field(context->byte_212_lsn, V2_QPC_BYTE_212_RETRY_CNT_M,
-		       V2_QPC_BYTE_212_RETRY_CNT_S, attr->retry_cnt);
-	roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_RETRY_CNT_M,
-		       V2_QPC_BYTE_212_RETRY_CNT_S, 0);
-
-	roce_set_field(context->byte_212_lsn, V2_QPC_BYTE_212_RETRY_NUM_INIT_M,
-		       V2_QPC_BYTE_212_RETRY_NUM_INIT_S, attr->retry_cnt);
-	roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_RETRY_NUM_INIT_M,
-		       V2_QPC_BYTE_212_RETRY_NUM_INIT_S, 0);
-
-	roce_set_field(context->byte_244_rnr_rxack,
-		       V2_QPC_BYTE_244_RNR_NUM_INIT_M,
-		       V2_QPC_BYTE_244_RNR_NUM_INIT_S, attr->rnr_retry);
-	roce_set_field(qpc_mask->byte_244_rnr_rxack,
-		       V2_QPC_BYTE_244_RNR_NUM_INIT_M,
-		       V2_QPC_BYTE_244_RNR_NUM_INIT_S, 0);
-
-	roce_set_field(context->byte_244_rnr_rxack, V2_QPC_BYTE_244_RNR_CNT_M,
-		       V2_QPC_BYTE_244_RNR_CNT_S, attr->rnr_retry);
-	roce_set_field(qpc_mask->byte_244_rnr_rxack, V2_QPC_BYTE_244_RNR_CNT_M,
-		       V2_QPC_BYTE_244_RNR_CNT_S, 0);
-
 	roce_set_field(context->byte_212_lsn, V2_QPC_BYTE_212_LSN_M,
 		       V2_QPC_BYTE_212_LSN_S, 0x100);
 	roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_LSN_M,
 		       V2_QPC_BYTE_212_LSN_S, 0);
 
-	if (attr_mask & IB_QP_TIMEOUT) {
-		if (attr->timeout < 31) {
-			roce_set_field(context->byte_28_at_fl,
-				       V2_QPC_BYTE_28_AT_M, V2_QPC_BYTE_28_AT_S,
-				       attr->timeout);
-			roce_set_field(qpc_mask->byte_28_at_fl,
-				       V2_QPC_BYTE_28_AT_M, V2_QPC_BYTE_28_AT_S,
-				       0);
-		} else {
-			dev_warn(dev, "Local ACK timeout shall be 0 to 30.\n");
-		}
-	}
-
-	roce_set_field(context->byte_172_sq_psn, V2_QPC_BYTE_172_SQ_CUR_PSN_M,
-		       V2_QPC_BYTE_172_SQ_CUR_PSN_S, attr->sq_psn);
-	roce_set_field(qpc_mask->byte_172_sq_psn, V2_QPC_BYTE_172_SQ_CUR_PSN_M,
-		       V2_QPC_BYTE_172_SQ_CUR_PSN_S, 0);
-
 	roce_set_field(qpc_mask->byte_196_sq_psn, V2_QPC_BYTE_196_IRRL_HEAD_M,
 		       V2_QPC_BYTE_196_IRRL_HEAD_S, 0);
-	roce_set_field(context->byte_196_sq_psn, V2_QPC_BYTE_196_SQ_MAX_PSN_M,
-		       V2_QPC_BYTE_196_SQ_MAX_PSN_S, attr->sq_psn);
-	roce_set_field(qpc_mask->byte_196_sq_psn, V2_QPC_BYTE_196_SQ_MAX_PSN_M,
-		       V2_QPC_BYTE_196_SQ_MAX_PSN_S, 0);
 
-	if ((attr_mask & IB_QP_MAX_QP_RD_ATOMIC) && attr->max_rd_atomic) {
-		roce_set_field(context->byte_208_irrl, V2_QPC_BYTE_208_SR_MAX_M,
-			       V2_QPC_BYTE_208_SR_MAX_S,
-			       fls(attr->max_rd_atomic - 1));
-		roce_set_field(qpc_mask->byte_208_irrl,
-			       V2_QPC_BYTE_208_SR_MAX_M,
-			       V2_QPC_BYTE_208_SR_MAX_S, 0);
-	}
 	return 0;
 }
 
@@ -4090,7 +3972,6 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,
 		const struct ib_global_route *grh =
 					    rdma_ah_read_grh(&attr->ah_attr);
 		const struct ib_gid_attr *gid_attr = NULL;
-		u8 src_mac[ETH_ALEN];
 		int is_roce_protocol;
 		u16 vlan = 0xffff;
 		u8 ib_port;
@@ -4104,11 +3985,12 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,
 
 		if (is_roce_protocol) {
 			gid_attr = attr->ah_attr.grh.sgid_attr;
-			vlan = rdma_vlan_dev_vlan_id(gid_attr->ndev);
-			memcpy(src_mac, gid_attr->ndev->dev_addr, ETH_ALEN);
+			ret = rdma_read_gid_l2_fields(gid_attr, &vlan, NULL);
+			if (ret)
+				goto out;
 		}
 
-		if (is_vlan_dev(gid_attr->ndev)) {
+		if (vlan < VLAN_CFI_MASK) {
 			roce_set_bit(context->byte_76_srqn_op_en,
 				     V2_QPC_BYTE_76_RQ_VLAN_EN_S, 1);
 			roce_set_bit(qpc_mask->byte_76_srqn_op_en,
@@ -4190,9 +4072,152 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,
 		hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr);
 	}
 
+	if (attr_mask & IB_QP_TIMEOUT) {
+		if (attr->timeout < 31) {
+			roce_set_field(context->byte_28_at_fl,
+				       V2_QPC_BYTE_28_AT_M, V2_QPC_BYTE_28_AT_S,
+				       attr->timeout);
+			roce_set_field(qpc_mask->byte_28_at_fl,
+				       V2_QPC_BYTE_28_AT_M, V2_QPC_BYTE_28_AT_S,
+				       0);
+		} else {
+			dev_warn(dev, "Local ACK timeout shall be 0 to 30.\n");
+		}
+	}
+
+	if (attr_mask & IB_QP_RETRY_CNT) {
+		roce_set_field(context->byte_212_lsn,
+			       V2_QPC_BYTE_212_RETRY_NUM_INIT_M,
+			       V2_QPC_BYTE_212_RETRY_NUM_INIT_S,
+			       attr->retry_cnt);
+		roce_set_field(qpc_mask->byte_212_lsn,
+			       V2_QPC_BYTE_212_RETRY_NUM_INIT_M,
+			       V2_QPC_BYTE_212_RETRY_NUM_INIT_S, 0);
+
+		roce_set_field(context->byte_212_lsn,
+			       V2_QPC_BYTE_212_RETRY_CNT_M,
+			       V2_QPC_BYTE_212_RETRY_CNT_S,
+			       attr->retry_cnt);
+		roce_set_field(qpc_mask->byte_212_lsn,
+			       V2_QPC_BYTE_212_RETRY_CNT_M,
+			       V2_QPC_BYTE_212_RETRY_CNT_S, 0);
+	}
+
+	if (attr_mask & IB_QP_RNR_RETRY) {
+		roce_set_field(context->byte_244_rnr_rxack,
+			       V2_QPC_BYTE_244_RNR_NUM_INIT_M,
+			       V2_QPC_BYTE_244_RNR_NUM_INIT_S, attr->rnr_retry);
+		roce_set_field(qpc_mask->byte_244_rnr_rxack,
+			       V2_QPC_BYTE_244_RNR_NUM_INIT_M,
+			       V2_QPC_BYTE_244_RNR_NUM_INIT_S, 0);
+
+		roce_set_field(context->byte_244_rnr_rxack,
+			       V2_QPC_BYTE_244_RNR_CNT_M,
+			       V2_QPC_BYTE_244_RNR_CNT_S, attr->rnr_retry);
+		roce_set_field(qpc_mask->byte_244_rnr_rxack,
+			       V2_QPC_BYTE_244_RNR_CNT_M,
+			       V2_QPC_BYTE_244_RNR_CNT_S, 0);
+	}
+
+	if (attr_mask & IB_QP_SQ_PSN) {
+		roce_set_field(context->byte_172_sq_psn,
+			       V2_QPC_BYTE_172_SQ_CUR_PSN_M,
+			       V2_QPC_BYTE_172_SQ_CUR_PSN_S, attr->sq_psn);
+		roce_set_field(qpc_mask->byte_172_sq_psn,
+			       V2_QPC_BYTE_172_SQ_CUR_PSN_M,
+			       V2_QPC_BYTE_172_SQ_CUR_PSN_S, 0);
+
+		roce_set_field(context->byte_196_sq_psn,
+			       V2_QPC_BYTE_196_SQ_MAX_PSN_M,
+			       V2_QPC_BYTE_196_SQ_MAX_PSN_S, attr->sq_psn);
+		roce_set_field(qpc_mask->byte_196_sq_psn,
+			       V2_QPC_BYTE_196_SQ_MAX_PSN_M,
+			       V2_QPC_BYTE_196_SQ_MAX_PSN_S, 0);
+
+		roce_set_field(context->byte_220_retry_psn_msn,
+			       V2_QPC_BYTE_220_RETRY_MSG_PSN_M,
+			       V2_QPC_BYTE_220_RETRY_MSG_PSN_S, attr->sq_psn);
+		roce_set_field(qpc_mask->byte_220_retry_psn_msn,
+			       V2_QPC_BYTE_220_RETRY_MSG_PSN_M,
+			       V2_QPC_BYTE_220_RETRY_MSG_PSN_S, 0);
+
+		roce_set_field(context->byte_224_retry_msg,
+			       V2_QPC_BYTE_224_RETRY_MSG_PSN_M,
+			       V2_QPC_BYTE_224_RETRY_MSG_PSN_S,
+			       attr->sq_psn >> 16);
+		roce_set_field(qpc_mask->byte_224_retry_msg,
+			       V2_QPC_BYTE_224_RETRY_MSG_PSN_M,
+			       V2_QPC_BYTE_224_RETRY_MSG_PSN_S, 0);
+
+		roce_set_field(context->byte_224_retry_msg,
+			       V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M,
+			       V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S,
+			       attr->sq_psn);
+		roce_set_field(qpc_mask->byte_224_retry_msg,
+			       V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M,
+			       V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S, 0);
+
+		roce_set_field(context->byte_244_rnr_rxack,
+			       V2_QPC_BYTE_244_RX_ACK_EPSN_M,
+			       V2_QPC_BYTE_244_RX_ACK_EPSN_S, attr->sq_psn);
+		roce_set_field(qpc_mask->byte_244_rnr_rxack,
+			       V2_QPC_BYTE_244_RX_ACK_EPSN_M,
+			       V2_QPC_BYTE_244_RX_ACK_EPSN_S, 0);
+	}
+
+	if ((attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) &&
+	     attr->max_dest_rd_atomic) {
+		roce_set_field(context->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M,
+			       V2_QPC_BYTE_140_RR_MAX_S,
+			       fls(attr->max_dest_rd_atomic - 1));
+		roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M,
+			       V2_QPC_BYTE_140_RR_MAX_S, 0);
+	}
+
+	if ((attr_mask & IB_QP_MAX_QP_RD_ATOMIC) && attr->max_rd_atomic) {
+		roce_set_field(context->byte_208_irrl, V2_QPC_BYTE_208_SR_MAX_M,
+			       V2_QPC_BYTE_208_SR_MAX_S,
+			       fls(attr->max_rd_atomic - 1));
+		roce_set_field(qpc_mask->byte_208_irrl,
+			       V2_QPC_BYTE_208_SR_MAX_M,
+			       V2_QPC_BYTE_208_SR_MAX_S, 0);
+	}
+
 	if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC))
 		set_access_flags(hr_qp, context, qpc_mask, attr, attr_mask);
 
+	if (attr_mask & IB_QP_MIN_RNR_TIMER) {
+		roce_set_field(context->byte_80_rnr_rx_cqn,
+			       V2_QPC_BYTE_80_MIN_RNR_TIME_M,
+			       V2_QPC_BYTE_80_MIN_RNR_TIME_S,
+			       attr->min_rnr_timer);
+		roce_set_field(qpc_mask->byte_80_rnr_rx_cqn,
+			       V2_QPC_BYTE_80_MIN_RNR_TIME_M,
+			       V2_QPC_BYTE_80_MIN_RNR_TIME_S, 0);
+	}
+
+	/* RC&UC required attr */
+	if (attr_mask & IB_QP_RQ_PSN) {
+		roce_set_field(context->byte_108_rx_reqepsn,
+			       V2_QPC_BYTE_108_RX_REQ_EPSN_M,
+			       V2_QPC_BYTE_108_RX_REQ_EPSN_S, attr->rq_psn);
+		roce_set_field(qpc_mask->byte_108_rx_reqepsn,
+			       V2_QPC_BYTE_108_RX_REQ_EPSN_M,
+			       V2_QPC_BYTE_108_RX_REQ_EPSN_S, 0);
+
+		roce_set_field(context->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M,
+			       V2_QPC_BYTE_152_RAQ_PSN_S, attr->rq_psn - 1);
+		roce_set_field(qpc_mask->byte_152_raq,
+			       V2_QPC_BYTE_152_RAQ_PSN_M,
+			       V2_QPC_BYTE_152_RAQ_PSN_S, 0);
+	}
+
+	if (attr_mask & IB_QP_QKEY) {
+		context->qkey_xrcd = attr->qkey;
+		qpc_mask->qkey_xrcd = 0;
+		hr_qp->qkey = attr->qkey;
+	}
+
 	roce_set_bit(context->byte_108_rx_reqepsn, V2_QPC_BYTE_108_INV_CREDIT_S,
 		     ibqp->srq ? 1 : 0);
 	roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
@@ -4421,7 +4446,7 @@ out:
 
 static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev,
 					 struct hns_roce_qp *hr_qp,
-					 bool is_user)
+					 struct ib_udata *udata)
 {
 	struct hns_roce_cq *send_cq, *recv_cq;
 	struct device *dev = hr_dev->dev;
@@ -4443,7 +4468,7 @@ static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev,
 
 	hns_roce_lock_cqs(send_cq, recv_cq);
 
-	if (!is_user) {
+	if (!udata) {
 		__hns_roce_v2_cq_clean(recv_cq, hr_qp->qpn, hr_qp->ibqp.srq ?
 				       to_hr_srq(hr_qp->ibqp.srq) : NULL);
 		if (send_cq != recv_cq)
@@ -4464,16 +4489,18 @@ static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev,
 
 	hns_roce_mtt_cleanup(hr_dev, &hr_qp->mtt);
 
-	if (is_user) {
+	if (udata) {
+		struct hns_roce_ucontext *context =
+			rdma_udata_to_drv_context(
+				udata,
+				struct hns_roce_ucontext,
+				ibucontext);
+
 		if (hr_qp->sq.wqe_cnt && (hr_qp->sdb_en == 1))
-			hns_roce_db_unmap_user(
-				to_hr_ucontext(hr_qp->ibqp.uobject->context),
-				&hr_qp->sdb);
+			hns_roce_db_unmap_user(context, &hr_qp->sdb);
 
 		if (hr_qp->rq.wqe_cnt && (hr_qp->rdb_en == 1))
-			hns_roce_db_unmap_user(
-				to_hr_ucontext(hr_qp->ibqp.uobject->context),
-				&hr_qp->rdb);
+			hns_roce_db_unmap_user(context, &hr_qp->rdb);
 		ib_umem_release(hr_qp->umem);
 	} else {
 		kfree(hr_qp->sq.wrid);
@@ -4492,13 +4519,13 @@ static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev,
 	return 0;
 }
 
-static int hns_roce_v2_destroy_qp(struct ib_qp *ibqp)
+static int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
 {
 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
 	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
 	int ret;
 
-	ret = hns_roce_v2_destroy_qp_common(hr_dev, hr_qp, ibqp->uobject);
+	ret = hns_roce_v2_destroy_qp_common(hr_dev, hr_qp, udata);
 	if (ret) {
 		dev_err(hr_dev->dev, "Destroy qp failed(%d)\n", ret);
 		return ret;
@@ -6044,6 +6071,10 @@ static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq,
 	return ret;
 }
 
+static const struct hns_roce_dfx_hw hns_roce_dfx_hw_v2 = {
+	.query_cqc_info = hns_roce_v2_query_cqc_info,
+};
+
 static const struct ib_device_ops hns_roce_v2_dev_ops = {
 	.destroy_qp = hns_roce_v2_destroy_qp,
 	.modify_cq = hns_roce_v2_modify_cq,
@@ -6113,16 +6144,10 @@ static int hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev,
 				  struct hnae3_handle *handle)
 {
 	struct hns_roce_v2_priv *priv = hr_dev->priv;
-	const struct pci_device_id *id;
 	int i;
 
-	id = pci_match_id(hns_roce_hw_v2_pci_tbl, hr_dev->pci_dev);
-	if (!id) {
-		dev_err(hr_dev->dev, "device is not compatible!\n");
-		return -ENXIO;
-	}
-
 	hr_dev->hw = &hns_roce_hw_v2;
+	hr_dev->dfx = &hns_roce_dfx_hw_v2;
 	hr_dev->sdb_offset = ROCEE_DB_SQ_L_0_REG;
 	hr_dev->odb_offset = hr_dev->sdb_offset;
 
@@ -6209,6 +6234,7 @@ static void __hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle,
 static int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
 {
 	const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
+	const struct pci_device_id *id;
 	struct device *dev = &handle->pdev->dev;
 	int ret;
 
@@ -6219,6 +6245,10 @@ static int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
 		goto reset_chk_err;
 	}
 
+	id = pci_match_id(hns_roce_hw_v2_pci_tbl, handle->pdev);
+	if (!id)
+		return 0;
+
 	ret = __hns_roce_hw_v2_init_instance(handle);
 	if (ret) {
 		handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
index f1f1b75812f9..edfdbe2ce0db 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
@@ -719,8 +719,8 @@ struct hns_roce_v2_qp_context {
 #define	V2_QPC_BYTE_148_RAQ_SYNDROME_S 24
 #define V2_QPC_BYTE_148_RAQ_SYNDROME_M GENMASK(31, 24)
 
-#define	V2_QPC_BYTE_152_RAQ_PSN_S 8
-#define V2_QPC_BYTE_152_RAQ_PSN_M GENMASK(31, 8)
+#define	V2_QPC_BYTE_152_RAQ_PSN_S 0
+#define V2_QPC_BYTE_152_RAQ_PSN_M GENMASK(23, 0)
 
 #define	V2_QPC_BYTE_152_RAQ_TRRL_RTY_HEAD_S 24
 #define V2_QPC_BYTE_152_RAQ_TRRL_RTY_HEAD_M GENMASK(31, 24)
@@ -1799,6 +1799,9 @@ struct hns_roce_sccc_clr_done {
 	__le32 rsv[5];
 };
 
+int hns_roce_v2_query_cqc_info(struct hns_roce_dev *hr_dev, u32 cqn,
+			       int *buffer);
+
 static inline void hns_roce_write64(struct hns_roce_dev *hr_dev, __le32 val[2],
 				    void __iomem *dest)
 {
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c
new file mode 100644
index 000000000000..5a97b5a0b7be
--- /dev/null
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c
@@ -0,0 +1,35 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+// Copyright (c) 2019 Hisilicon Limited.
+
+#include "hnae3.h"
+#include "hns_roce_device.h"
+#include "hns_roce_cmd.h"
+#include "hns_roce_hw_v2.h"
+
+int hns_roce_v2_query_cqc_info(struct hns_roce_dev *hr_dev, u32 cqn,
+			       int *buffer)
+{
+	struct hns_roce_v2_cq_context *cq_context;
+	struct hns_roce_cmd_mailbox *mailbox;
+	int ret;
+
+	mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
+	if (IS_ERR(mailbox))
+		return PTR_ERR(mailbox);
+
+	cq_context = mailbox->buf;
+	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, cqn, 0,
+				HNS_ROCE_CMD_QUERY_CQC,
+				HNS_ROCE_CMD_TIMEOUT_MSECS);
+	if (ret) {
+		dev_err(hr_dev->dev, "QUERY cqc cmd process error\n");
+		goto err_mailbox;
+	}
+
+	memcpy(buffer, cq_context, sizeof(*cq_context));
+
+err_mailbox:
+	hns_roce_free_cmd_mailbox(hr_dev, mailbox);
+
+	return ret;
+}
diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c
index c929125da84b..8da5f18bf820 100644
--- a/drivers/infiniband/hw/hns/hns_roce_main.c
+++ b/drivers/infiniband/hw/hns/hns_roce_main.c
@@ -234,25 +234,6 @@ static int hns_roce_query_device(struct ib_device *ib_dev,
 	return 0;
 }
 
-static struct net_device *hns_roce_get_netdev(struct ib_device *ib_dev,
-					      u8 port_num)
-{
-	struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev);
-	struct net_device *ndev;
-
-	if (port_num < 1 || port_num > hr_dev->caps.num_ports)
-		return NULL;
-
-	rcu_read_lock();
-
-	ndev = hr_dev->iboe.netdevs[port_num - 1];
-	if (ndev)
-		dev_hold(ndev);
-
-	rcu_read_unlock();
-	return ndev;
-}
-
 static int hns_roce_query_port(struct ib_device *ib_dev, u8 port_num,
 			       struct ib_port_attr *props)
 {
@@ -455,9 +436,9 @@ static const struct ib_device_ops hns_roce_dev_ops = {
 	.destroy_ah = hns_roce_destroy_ah,
 	.destroy_cq = hns_roce_ib_destroy_cq,
 	.disassociate_ucontext = hns_roce_disassociate_ucontext,
+	.fill_res_entry = hns_roce_fill_res_entry,
 	.get_dma_mr = hns_roce_get_dma_mr,
 	.get_link_layer = hns_roce_get_link_layer,
-	.get_netdev = hns_roce_get_netdev,
 	.get_port_immutable = hns_roce_port_immutable,
 	.mmap = hns_roce_mmap,
 	.modify_device = hns_roce_modify_device,
@@ -468,6 +449,8 @@ static const struct ib_device_ops hns_roce_dev_ops = {
 	.query_pkey = hns_roce_query_pkey,
 	.query_port = hns_roce_query_port,
 	.reg_user_mr = hns_roce_reg_user_mr,
+
+	INIT_RDMA_OBJ_SIZE(ib_ah, hns_roce_ah, ibah),
 	INIT_RDMA_OBJ_SIZE(ib_pd, hns_roce_pd, ibpd),
 	INIT_RDMA_OBJ_SIZE(ib_ucontext, hns_roce_ucontext, ibucontext),
 };
@@ -489,6 +472,8 @@ static const struct ib_device_ops hns_roce_dev_frmr_ops = {
 static const struct ib_device_ops hns_roce_dev_srq_ops = {
 	.create_srq = hns_roce_create_srq,
 	.destroy_srq = hns_roce_destroy_srq,
+
+	INIT_RDMA_OBJ_SIZE(ib_srq, hns_roce_srq, ibsrq),
 };
 
 static int hns_roce_register_device(struct hns_roce_dev *hr_dev)
@@ -497,6 +482,7 @@ static int hns_roce_register_device(struct hns_roce_dev *hr_dev)
 	struct hns_roce_ib_iboe *iboe = NULL;
 	struct ib_device *ib_dev = NULL;
 	struct device *dev = hr_dev->dev;
+	unsigned int i;
 
 	iboe = &hr_dev->iboe;
 	spin_lock_init(&iboe->lock);
@@ -562,6 +548,15 @@ static int hns_roce_register_device(struct hns_roce_dev *hr_dev)
 	ib_dev->driver_id = RDMA_DRIVER_HNS;
 	ib_set_device_ops(ib_dev, hr_dev->hw->hns_roce_dev_ops);
 	ib_set_device_ops(ib_dev, &hns_roce_dev_ops);
+	for (i = 0; i < hr_dev->caps.num_ports; i++) {
+		if (!hr_dev->iboe.netdevs[i])
+			continue;
+
+		ret = ib_device_set_netdev(ib_dev, hr_dev->iboe.netdevs[i],
+					   i + 1);
+		if (ret)
+			return ret;
+	}
 	ret = ib_register_device(ib_dev, "hns_%d");
 	if (ret) {
 		dev_err(dev, "ib_register_device failed!\n");
diff --git a/drivers/infiniband/hw/hns/hns_roce_mr.c b/drivers/infiniband/hw/hns/hns_roce_mr.c
index 08be0e4eabcd..6110ec408626 100644
--- a/drivers/infiniband/hw/hns/hns_roce_mr.c
+++ b/drivers/infiniband/hw/hns/hns_roce_mr.c
@@ -1282,14 +1282,14 @@ free_cmd_mbox:
 	return ret;
 }
 
-int hns_roce_dereg_mr(struct ib_mr *ibmr)
+int hns_roce_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
 {
 	struct hns_roce_dev *hr_dev = to_hr_dev(ibmr->device);
 	struct hns_roce_mr *mr = to_hr_mr(ibmr);
 	int ret = 0;
 
 	if (hr_dev->hw->dereg_mr) {
-		ret = hr_dev->hw->dereg_mr(hr_dev, mr);
+		ret = hr_dev->hw->dereg_mr(hr_dev, mr, udata);
 	} else {
 		hns_roce_mr_free(hr_dev, mr);
 
@@ -1303,7 +1303,7 @@ int hns_roce_dereg_mr(struct ib_mr *ibmr)
 }
 
 struct ib_mr *hns_roce_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
-				u32 max_num_sg)
+				u32 max_num_sg, struct ib_udata *udata)
 {
 	struct hns_roce_dev *hr_dev = to_hr_dev(pd->device);
 	struct device *dev = hr_dev->dev;
diff --git a/drivers/infiniband/hw/hns/hns_roce_pd.c b/drivers/infiniband/hw/hns/hns_roce_pd.c
index b9b97c5e97e6..813401384d78 100644
--- a/drivers/infiniband/hw/hns/hns_roce_pd.c
+++ b/drivers/infiniband/hw/hns/hns_roce_pd.c
@@ -57,8 +57,7 @@ void hns_roce_cleanup_pd_table(struct hns_roce_dev *hr_dev)
 	hns_roce_bitmap_cleanup(&hr_dev->pd_bitmap);
 }
 
-int hns_roce_alloc_pd(struct ib_pd *ibpd, struct ib_ucontext *context,
-		      struct ib_udata *udata)
+int hns_roce_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 {
 	struct ib_device *ib_dev = ibpd->device;
 	struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev);
@@ -72,7 +71,7 @@ int hns_roce_alloc_pd(struct ib_pd *ibpd, struct ib_ucontext *context,
 		return ret;
 	}
 
-	if (context) {
+	if (udata) {
 		struct hns_roce_ib_alloc_pd_resp uresp = {.pdn = pd->pdn};
 
 		if (ib_copy_to_udata(udata, &uresp, sizeof(uresp))) {
@@ -86,7 +85,7 @@ int hns_roce_alloc_pd(struct ib_pd *ibpd, struct ib_ucontext *context,
 }
 EXPORT_SYMBOL_GPL(hns_roce_alloc_pd);
 
-void hns_roce_dealloc_pd(struct ib_pd *pd)
+void hns_roce_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)
 {
 	hns_roce_pd_free(to_hr_dev(pd->device), to_hr_pd(pd)->pdn);
 }
diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c
index 60cf9f03e941..8db2817a249e 100644
--- a/drivers/infiniband/hw/hns/hns_roce_qp.c
+++ b/drivers/infiniband/hw/hns/hns_roce_qp.c
@@ -45,17 +45,14 @@
 
 void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type)
 {
-	struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
 	struct device *dev = hr_dev->dev;
 	struct hns_roce_qp *qp;
 
-	spin_lock(&qp_table->lock);
-
+	xa_lock(&hr_dev->qp_table_xa);
 	qp = __hns_roce_qp_lookup(hr_dev, qpn);
 	if (qp)
 		atomic_inc(&qp->refcount);
-
-	spin_unlock(&qp_table->lock);
+	xa_unlock(&hr_dev->qp_table_xa);
 
 	if (!qp) {
 		dev_warn(dev, "Async event for bogus QP %08x\n", qpn);
@@ -147,29 +144,20 @@ EXPORT_SYMBOL_GPL(to_hns_roce_state);
 static int hns_roce_gsi_qp_alloc(struct hns_roce_dev *hr_dev, unsigned long qpn,
 				 struct hns_roce_qp *hr_qp)
 {
-	struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
+	struct xarray *xa = &hr_dev->qp_table_xa;
 	int ret;
 
 	if (!qpn)
 		return -EINVAL;
 
 	hr_qp->qpn = qpn;
-
-	spin_lock_irq(&qp_table->lock);
-	ret = radix_tree_insert(&hr_dev->qp_table_tree,
-				hr_qp->qpn & (hr_dev->caps.num_qps - 1), hr_qp);
-	spin_unlock_irq(&qp_table->lock);
-	if (ret) {
-		dev_err(hr_dev->dev, "QPC radix_tree_insert failed\n");
-		goto err_put_irrl;
-	}
-
 	atomic_set(&hr_qp->refcount, 1);
 	init_completion(&hr_qp->free);
 
-	return 0;
-
-err_put_irrl:
+	ret = xa_err(xa_store_irq(xa, hr_qp->qpn & (hr_dev->caps.num_qps - 1),
+				hr_qp, GFP_KERNEL));
+	if (ret)
+		dev_err(hr_dev->dev, "QPC xa_store failed\n");
 
 	return ret;
 }
@@ -220,17 +208,9 @@ static int hns_roce_qp_alloc(struct hns_roce_dev *hr_dev, unsigned long qpn,
 		}
 	}
 
-	spin_lock_irq(&qp_table->lock);
-	ret = radix_tree_insert(&hr_dev->qp_table_tree,
-				hr_qp->qpn & (hr_dev->caps.num_qps - 1), hr_qp);
-	spin_unlock_irq(&qp_table->lock);
-	if (ret) {
-		dev_err(dev, "QPC radix_tree_insert failed\n");
+	ret = hns_roce_gsi_qp_alloc(hr_dev, qpn, hr_qp);
+	if (ret)
 		goto err_put_sccc;
-	}
-
-	atomic_set(&hr_qp->refcount, 1);
-	init_completion(&hr_qp->free);
 
 	return 0;
 
@@ -255,13 +235,12 @@ err_out:
 
 void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
 {
-	struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
+	struct xarray *xa = &hr_dev->qp_table_xa;
 	unsigned long flags;
 
-	spin_lock_irqsave(&qp_table->lock, flags);
-	radix_tree_delete(&hr_dev->qp_table_tree,
-			  hr_qp->qpn & (hr_dev->caps.num_qps - 1));
-	spin_unlock_irqrestore(&qp_table->lock, flags);
+	xa_lock_irqsave(xa, flags);
+	__xa_erase(xa, hr_qp->qpn & (hr_dev->caps.num_qps - 1));
+	xa_unlock_irqrestore(xa, flags);
 }
 EXPORT_SYMBOL_GPL(hns_roce_qp_remove);
 
@@ -1154,8 +1133,7 @@ int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev)
 	int ret;
 
 	mutex_init(&qp_table->scc_mutex);
-	spin_lock_init(&qp_table->lock);
-	INIT_RADIX_TREE(&hr_dev->qp_table_tree, GFP_ATOMIC);
+	xa_init(&hr_dev->qp_table_xa);
 
 	/* In hw v1, a port include two SQP, six ports total 12 */
 	if (hr_dev->caps.max_sq_sg <= 2)
diff --git a/drivers/infiniband/hw/hns/hns_roce_restrack.c b/drivers/infiniband/hw/hns/hns_roce_restrack.c
new file mode 100644
index 000000000000..0a31d0a3d657
--- /dev/null
+++ b/drivers/infiniband/hw/hns/hns_roce_restrack.c
@@ -0,0 +1,126 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+// Copyright (c) 2019 Hisilicon Limited.
+
+#include <rdma/rdma_cm.h>
+#include <rdma/restrack.h>
+#include <uapi/rdma/rdma_netlink.h>
+#include "hnae3.h"
+#include "hns_roce_common.h"
+#include "hns_roce_device.h"
+#include "hns_roce_hw_v2.h"
+
+static int hns_roce_fill_cq(struct sk_buff *msg,
+			    struct hns_roce_v2_cq_context *context)
+{
+	if (rdma_nl_put_driver_u32(msg, "state",
+				   roce_get_field(context->byte_4_pg_ceqn,
+						  V2_CQC_BYTE_4_ARM_ST_M,
+						  V2_CQC_BYTE_4_ARM_ST_S)))
+		goto err;
+
+	if (rdma_nl_put_driver_u32(msg, "ceqn",
+				   roce_get_field(context->byte_4_pg_ceqn,
+						  V2_CQC_BYTE_4_CEQN_M,
+						  V2_CQC_BYTE_4_CEQN_S)))
+		goto err;
+
+	if (rdma_nl_put_driver_u32(msg, "cqn",
+				   roce_get_field(context->byte_8_cqn,
+						  V2_CQC_BYTE_8_CQN_M,
+						  V2_CQC_BYTE_8_CQN_S)))
+		goto err;
+
+	if (rdma_nl_put_driver_u32(msg, "hopnum",
+				   roce_get_field(context->byte_16_hop_addr,
+						  V2_CQC_BYTE_16_CQE_HOP_NUM_M,
+						  V2_CQC_BYTE_16_CQE_HOP_NUM_S)))
+		goto err;
+
+	if (rdma_nl_put_driver_u32(
+		    msg, "pi",
+		    roce_get_field(context->byte_28_cq_pi,
+				   V2_CQC_BYTE_28_CQ_PRODUCER_IDX_M,
+				   V2_CQC_BYTE_28_CQ_PRODUCER_IDX_S)))
+		goto err;
+
+	if (rdma_nl_put_driver_u32(
+		    msg, "ci",
+		    roce_get_field(context->byte_32_cq_ci,
+				   V2_CQC_BYTE_32_CQ_CONSUMER_IDX_M,
+				   V2_CQC_BYTE_32_CQ_CONSUMER_IDX_S)))
+		goto err;
+
+	if (rdma_nl_put_driver_u32(
+		    msg, "coalesce",
+		    roce_get_field(context->byte_56_cqe_period_maxcnt,
+				   V2_CQC_BYTE_56_CQ_MAX_CNT_M,
+				   V2_CQC_BYTE_56_CQ_MAX_CNT_S)))
+		goto err;
+
+	if (rdma_nl_put_driver_u32(
+		    msg, "period",
+		    roce_get_field(context->byte_56_cqe_period_maxcnt,
+				   V2_CQC_BYTE_56_CQ_PERIOD_M,
+				   V2_CQC_BYTE_56_CQ_PERIOD_S)))
+		goto err;
+
+	if (rdma_nl_put_driver_u32(msg, "cnt",
+				   roce_get_field(context->byte_52_cqe_cnt,
+						  V2_CQC_BYTE_52_CQE_CNT_M,
+						  V2_CQC_BYTE_52_CQE_CNT_S)))
+		goto err;
+
+	return 0;
+
+err:
+	return -EMSGSIZE;
+}
+
+static int hns_roce_fill_res_cq_entry(struct sk_buff *msg,
+				      struct rdma_restrack_entry *res)
+{
+	struct ib_cq *ib_cq = container_of(res, struct ib_cq, res);
+	struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device);
+	struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq);
+	struct hns_roce_v2_cq_context *context;
+	struct nlattr *table_attr;
+	int ret;
+
+	if (!hr_dev->dfx->query_cqc_info)
+		return -EINVAL;
+
+	context = kzalloc(sizeof(struct hns_roce_v2_cq_context), GFP_KERNEL);
+	if (!context)
+		return -ENOMEM;
+
+	ret = hr_dev->dfx->query_cqc_info(hr_dev, hr_cq->cqn, (int *)context);
+	if (ret)
+		goto err;
+
+	table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_DRIVER);
+	if (!table_attr)
+		goto err;
+
+	if (hns_roce_fill_cq(msg, context))
+		goto err_cancel_table;
+
+	nla_nest_end(msg, table_attr);
+	kfree(context);
+
+	return 0;
+
+err_cancel_table:
+	nla_nest_cancel(msg, table_attr);
+err:
+	kfree(context);
+	return -EMSGSIZE;
+}
+
+int hns_roce_fill_res_entry(struct sk_buff *msg,
+			    struct rdma_restrack_entry *res)
+{
+	if (res->type == RDMA_RESTRACK_CQ)
+		return hns_roce_fill_res_cq_entry(msg, res);
+
+	return 0;
+}
diff --git a/drivers/infiniband/hw/hns/hns_roce_srq.c b/drivers/infiniband/hw/hns/hns_roce_srq.c
index a8ee2f6da967..b3421b1f21e0 100644
--- a/drivers/infiniband/hw/hns/hns_roce_srq.c
+++ b/drivers/infiniband/hw/hns/hns_roce_srq.c
@@ -206,13 +206,13 @@ static int hns_roce_create_idx_que(struct ib_pd *pd, struct hns_roce_srq *srq,
 	return 0;
 }
 
-struct ib_srq *hns_roce_create_srq(struct ib_pd *pd,
-				   struct ib_srq_init_attr *srq_init_attr,
-				   struct ib_udata *udata)
+int hns_roce_create_srq(struct ib_srq *ib_srq,
+			struct ib_srq_init_attr *srq_init_attr,
+			struct ib_udata *udata)
 {
-	struct hns_roce_dev *hr_dev = to_hr_dev(pd->device);
+	struct hns_roce_dev *hr_dev = to_hr_dev(ib_srq->device);
 	struct hns_roce_ib_create_srq_resp resp = {};
-	struct hns_roce_srq *srq;
+	struct hns_roce_srq *srq = to_hr_srq(ib_srq);
 	int srq_desc_size;
 	int srq_buf_size;
 	u32 page_shift;
@@ -223,11 +223,7 @@ struct ib_srq *hns_roce_create_srq(struct ib_pd *pd,
 	/* Check the actual SRQ wqe and SRQ sge num */
 	if (srq_init_attr->attr.max_wr >= hr_dev->caps.max_srq_wrs ||
 	    srq_init_attr->attr.max_sge > hr_dev->caps.max_srq_sges)
-		return ERR_PTR(-EINVAL);
-
-	srq = kzalloc(sizeof(*srq), GFP_KERNEL);
-	if (!srq)
-		return ERR_PTR(-ENOMEM);
+		return -EINVAL;
 
 	mutex_init(&srq->mutex);
 	spin_lock_init(&srq->lock);
@@ -249,17 +245,13 @@ struct ib_srq *hns_roce_create_srq(struct ib_pd *pd,
 	if (udata) {
 		struct hns_roce_ib_create_srq  ucmd;
 
-		if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) {
-			ret = -EFAULT;
-			goto err_srq;
-		}
+		if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)))
+			return -EFAULT;
 
 		srq->umem =
 			ib_umem_get(udata, ucmd.buf_addr, srq_buf_size, 0, 0);
-		if (IS_ERR(srq->umem)) {
-			ret = PTR_ERR(srq->umem);
-			goto err_srq;
-		}
+		if (IS_ERR(srq->umem))
+			return PTR_ERR(srq->umem);
 
 		if (hr_dev->caps.srqwqe_buf_pg_sz) {
 			npages = (ib_umem_page_count(srq->umem) +
@@ -321,11 +313,9 @@ struct ib_srq *hns_roce_create_srq(struct ib_pd *pd,
 	} else {
 		page_shift = PAGE_SHIFT + hr_dev->caps.srqwqe_buf_pg_sz;
 		if (hns_roce_buf_alloc(hr_dev, srq_buf_size,
-				      (1 << page_shift) * 2,
-				      &srq->buf, page_shift)) {
-			ret = -ENOMEM;
-			goto err_srq;
-		}
+				       (1 << page_shift) * 2, &srq->buf,
+				       page_shift))
+			return -ENOMEM;
 
 		srq->head = 0;
 		srq->tail = srq->max - 1;
@@ -340,7 +330,7 @@ struct ib_srq *hns_roce_create_srq(struct ib_pd *pd,
 			goto err_srq_mtt;
 
 		page_shift = PAGE_SHIFT + hr_dev->caps.idx_buf_pg_sz;
-		ret = hns_roce_create_idx_que(pd, srq, page_shift);
+		ret = hns_roce_create_idx_que(ib_srq->pd, srq, page_shift);
 		if (ret) {
 			dev_err(hr_dev->dev, "Create idx queue fail(%d)!\n",
 				ret);
@@ -372,7 +362,7 @@ struct ib_srq *hns_roce_create_srq(struct ib_pd *pd,
 
 	srq->db_reg_l = hr_dev->reg_base + SRQ_DB_REG;
 
-	ret = hns_roce_srq_alloc(hr_dev, to_hr_pd(pd)->pdn, cqn, 0,
+	ret = hns_roce_srq_alloc(hr_dev, to_hr_pd(ib_srq->pd)->pdn, cqn, 0,
 				 &srq->mtt, 0, srq);
 	if (ret)
 		goto err_wrid;
@@ -389,7 +379,7 @@ struct ib_srq *hns_roce_create_srq(struct ib_pd *pd,
 		}
 	}
 
-	return &srq->ibsrq;
+	return 0;
 
 err_srqc_alloc:
 	hns_roce_srq_free(hr_dev, srq);
@@ -418,12 +408,10 @@ err_buf:
 	else
 		hns_roce_buf_free(hr_dev, srq_buf_size, &srq->buf);
 
-err_srq:
-	kfree(srq);
-	return ERR_PTR(ret);
+	return ret;
 }
 
-int hns_roce_destroy_srq(struct ib_srq *ibsrq)
+void hns_roce_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
 {
 	struct hns_roce_dev *hr_dev = to_hr_dev(ibsrq->device);
 	struct hns_roce_srq *srq = to_hr_srq(ibsrq);
@@ -440,10 +428,6 @@ int hns_roce_destroy_srq(struct ib_srq *ibsrq)
 		hns_roce_buf_free(hr_dev, srq->max << srq->wqe_shift,
 				  &srq->buf);
 	}
-
-	kfree(srq);
-
-	return 0;
 }
 
 int hns_roce_init_srq_table(struct hns_roce_dev *hr_dev)
diff --git a/drivers/infiniband/hw/i40iw/i40iw.h b/drivers/infiniband/hw/i40iw/i40iw.h
index 2f2b4426ded7..8feec35f95a7 100644
--- a/drivers/infiniband/hw/i40iw/i40iw.h
+++ b/drivers/infiniband/hw/i40iw/i40iw.h
@@ -552,7 +552,7 @@ enum i40iw_status_code i40iw_obj_aligned_mem(struct i40iw_device *iwdev,
 
 void i40iw_request_reset(struct i40iw_device *iwdev);
 void i40iw_destroy_rdma_device(struct i40iw_ib_device *iwibdev);
-void i40iw_setup_cm_core(struct i40iw_device *iwdev);
+int i40iw_setup_cm_core(struct i40iw_device *iwdev);
 void i40iw_cleanup_cm_core(struct i40iw_cm_core *cm_core);
 void i40iw_process_ceq(struct i40iw_device *, struct i40iw_ceq *iwceq);
 void i40iw_process_aeq(struct i40iw_device *);
diff --git a/drivers/infiniband/hw/i40iw/i40iw_cm.c b/drivers/infiniband/hw/i40iw/i40iw_cm.c
index 206cfb0016f8..8233f5a4e623 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_cm.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_cm.c
@@ -3237,7 +3237,7 @@ void i40iw_receive_ilq(struct i40iw_sc_vsi *vsi, struct i40iw_puda_buf *rbuf)
  * core
  * @iwdev: iwarp device structure
  */
-void i40iw_setup_cm_core(struct i40iw_device *iwdev)
+int i40iw_setup_cm_core(struct i40iw_device *iwdev)
 {
 	struct i40iw_cm_core *cm_core = &iwdev->cm_core;
 
@@ -3256,9 +3256,19 @@ void i40iw_setup_cm_core(struct i40iw_device *iwdev)
 
 	cm_core->event_wq = alloc_ordered_workqueue("iwewq",
 						    WQ_MEM_RECLAIM);
+	if (!cm_core->event_wq)
+		goto error;
 
 	cm_core->disconn_wq = alloc_ordered_workqueue("iwdwq",
 						      WQ_MEM_RECLAIM);
+	if (!cm_core->disconn_wq)
+		goto error;
+
+	return 0;
+error:
+	i40iw_cleanup_cm_core(&iwdev->cm_core);
+
+	return -ENOMEM;
 }
 
 /**
@@ -3278,8 +3288,10 @@ void i40iw_cleanup_cm_core(struct i40iw_cm_core *cm_core)
 		del_timer_sync(&cm_core->tcp_timer);
 	spin_unlock_irqrestore(&cm_core->ht_lock, flags);
 
-	destroy_workqueue(cm_core->event_wq);
-	destroy_workqueue(cm_core->disconn_wq);
+	if (cm_core->event_wq)
+		destroy_workqueue(cm_core->event_wq);
+	if (cm_core->disconn_wq)
+		destroy_workqueue(cm_core->disconn_wq);
 }
 
 /**
@@ -3478,7 +3490,8 @@ static void i40iw_qp_disconnect(struct i40iw_qp *iwqp)
 		/* Need to free the Last Streaming Mode Message */
 		if (iwqp->ietf_mem.va) {
 			if (iwqp->lsmm_mr)
-				iwibdev->ibdev.ops.dereg_mr(iwqp->lsmm_mr);
+				iwibdev->ibdev.ops.dereg_mr(iwqp->lsmm_mr,
+							    NULL);
 			i40iw_free_dma_mem(iwdev->sc_dev.hw, &iwqp->ietf_mem);
 		}
 	}
diff --git a/drivers/infiniband/hw/i40iw/i40iw_main.c b/drivers/infiniband/hw/i40iw/i40iw_main.c
index 68095f00d08f..10932baee279 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_main.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_main.c
@@ -1641,7 +1641,10 @@ static int i40iw_open(struct i40e_info *ldev, struct i40e_client *client)
 	iwdev = &hdl->device;
 	iwdev->hdl = hdl;
 	dev = &iwdev->sc_dev;
-	i40iw_setup_cm_core(iwdev);
+	if (i40iw_setup_cm_core(iwdev)) {
+		kfree(iwdev->hdl);
+		return -ENOMEM;
+	}
 
 	dev->back_dev = (void *)iwdev;
 	iwdev->ldev = &hdl->ldev;
diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.c b/drivers/infiniband/hw/i40iw/i40iw_verbs.c
index a8352e3ca23d..5689d742bafb 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_verbs.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_verbs.c
@@ -291,18 +291,15 @@ static void i40iw_dealloc_push_page(struct i40iw_device *iwdev, struct i40iw_sc_
 /**
  * i40iw_alloc_pd - allocate protection domain
  * @pd: PD pointer
- * @context: user context created during alloc
  * @udata: user data
  */
-static int i40iw_alloc_pd(struct ib_pd *pd, struct ib_ucontext *context,
-			  struct ib_udata *udata)
+static int i40iw_alloc_pd(struct ib_pd *pd, struct ib_udata *udata)
 {
 	struct i40iw_pd *iwpd = to_iwpd(pd);
 	struct i40iw_device *iwdev = to_iwdev(pd->device);
 	struct i40iw_sc_dev *dev = &iwdev->sc_dev;
 	struct i40iw_alloc_pd_resp uresp;
 	struct i40iw_sc_pd *sc_pd;
-	struct i40iw_ucontext *ucontext;
 	u32 pd_id = 0;
 	int err;
 
@@ -318,8 +315,9 @@ static int i40iw_alloc_pd(struct ib_pd *pd, struct ib_ucontext *context,
 
 	sc_pd = &iwpd->sc_pd;
 
-	if (context) {
-		ucontext = to_ucontext(context);
+	if (udata) {
+		struct i40iw_ucontext *ucontext = rdma_udata_to_drv_context(
+			udata, struct i40iw_ucontext, ibucontext);
 		dev->iw_pd_ops->pd_init(dev, sc_pd, pd_id, ucontext->abi_ver);
 		memset(&uresp, 0, sizeof(uresp));
 		uresp.pd_id = pd_id;
@@ -342,8 +340,9 @@ error:
 /**
  * i40iw_dealloc_pd - deallocate pd
  * @ibpd: ptr of pd to be deallocated
+ * @udata: user data or null for kernel object
  */
-static void i40iw_dealloc_pd(struct ib_pd *ibpd)
+static void i40iw_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 {
 	struct i40iw_pd *iwpd = to_iwpd(ibpd);
 	struct i40iw_device *iwdev = to_iwdev(ibpd->device);
@@ -413,7 +412,7 @@ static void i40iw_clean_cqes(struct i40iw_qp *iwqp, struct i40iw_cq *iwcq)
  * i40iw_destroy_qp - destroy qp
  * @ibqp: qp's ib pointer also to get to device's qp address
  */
-static int i40iw_destroy_qp(struct ib_qp *ibqp)
+static int i40iw_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
 {
 	struct i40iw_qp *iwqp = to_iwqp(ibqp);
 
@@ -744,8 +743,8 @@ static struct ib_qp *i40iw_create_qp(struct ib_pd *ibpd,
 		err_code = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 		if (err_code) {
 			i40iw_pr_err("copy_to_udata failed\n");
-			i40iw_destroy_qp(&iwqp->ibqp);
-			   /* let the completion of the qp destroy free the qp */
+			i40iw_destroy_qp(&iwqp->ibqp, udata);
+			/* let the completion of the qp destroy free the qp */
 			return ERR_PTR(err_code);
 		}
 	}
@@ -1063,8 +1062,9 @@ void i40iw_cq_wq_destroy(struct i40iw_device *iwdev, struct i40iw_sc_cq *cq)
 /**
  * i40iw_destroy_cq - destroy cq
  * @ib_cq: cq pointer
+ * @udata: user data or NULL for kernel object
  */
-static int i40iw_destroy_cq(struct ib_cq *ib_cq)
+static int i40iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
 {
 	struct i40iw_cq *iwcq;
 	struct i40iw_device *iwdev;
@@ -1089,12 +1089,10 @@ static int i40iw_destroy_cq(struct ib_cq *ib_cq)
  * i40iw_create_cq - create cq
  * @ibdev: device pointer from stack
  * @attr: attributes for cq
- * @context: user context created during alloc
  * @udata: user data
  */
 static struct ib_cq *i40iw_create_cq(struct ib_device *ibdev,
 				     const struct ib_cq_init_attr *attr,
-				     struct ib_ucontext *context,
 				     struct ib_udata *udata)
 {
 	struct i40iw_device *iwdev = to_iwdev(ibdev);
@@ -1144,14 +1142,14 @@ static struct ib_cq *i40iw_create_cq(struct ib_device *ibdev,
 	info.ceq_id_valid = true;
 	info.ceqe_mask = 1;
 	info.type = I40IW_CQ_TYPE_IWARP;
-	if (context) {
-		struct i40iw_ucontext *ucontext;
+	if (udata) {
+		struct i40iw_ucontext *ucontext = rdma_udata_to_drv_context(
+			udata, struct i40iw_ucontext, ibucontext);
 		struct i40iw_create_cq_req req;
 		struct i40iw_cq_mr *cqmr;
 
 		memset(&req, 0, sizeof(req));
 		iwcq->user_mode = true;
-		ucontext = to_ucontext(context);
 		if (ib_copy_from_udata(&req, udata, sizeof(struct i40iw_create_cq_req))) {
 			err_code = -EFAULT;
 			goto cq_free_resources;
@@ -1221,7 +1219,7 @@ static struct ib_cq *i40iw_create_cq(struct ib_device *ibdev,
 		goto cq_free_resources;
 	}
 
-	if (context) {
+	if (udata) {
 		struct i40iw_create_cq_resp resp;
 
 		memset(&resp, 0, sizeof(resp));
@@ -1340,53 +1338,22 @@ static void i40iw_copy_user_pgaddrs(struct i40iw_mr *iwmr,
 	struct i40iw_pbl *iwpbl = &iwmr->iwpbl;
 	struct i40iw_pble_alloc *palloc = &iwpbl->pble_alloc;
 	struct i40iw_pble_info *pinfo;
-	struct sg_dma_page_iter sg_iter;
-	u64 pg_addr = 0;
+	struct ib_block_iter biter;
 	u32 idx = 0;
-	bool first_pg = true;
 
 	pinfo = (level == I40IW_LEVEL_1) ? NULL : palloc->level2.leaf;
 
 	if (iwmr->type == IW_MEMREG_TYPE_QP)
 		iwpbl->qp_mr.sq_page = sg_page(region->sg_head.sgl);
 
-	for_each_sg_dma_page (region->sg_head.sgl, &sg_iter, region->nmap, 0) {
-		pg_addr = sg_page_iter_dma_address(&sg_iter);
-		if (first_pg)
-			*pbl = cpu_to_le64(pg_addr & iwmr->page_msk);
-		else if (!(pg_addr & ~iwmr->page_msk))
-			*pbl = cpu_to_le64(pg_addr);
-		else
-			continue;
-
-		first_pg = false;
+	rdma_for_each_block(region->sg_head.sgl, &biter, region->nmap,
+			    iwmr->page_size) {
+		*pbl = rdma_block_iter_dma_address(&biter);
 		pbl = i40iw_next_pbl_addr(pbl, &pinfo, &idx);
 	}
 }
 
 /**
- * i40iw_set_hugetlb_params - set MR pg size and mask to huge pg values.
- * @addr: virtual address
- * @iwmr: mr pointer for this memory registration
- */
-static void i40iw_set_hugetlb_values(u64 addr, struct i40iw_mr *iwmr)
-{
-	struct vm_area_struct *vma;
-	struct hstate *h;
-
-	down_read(&current->mm->mmap_sem);
-	vma = find_vma(current->mm, addr);
-	if (vma && is_vm_hugetlb_page(vma)) {
-		h = hstate_vma(vma);
-		if (huge_page_size(h) == 0x200000) {
-			iwmr->page_size = huge_page_size(h);
-			iwmr->page_msk = huge_page_mask(h);
-		}
-	}
-	up_read(&current->mm->mmap_sem);
-}
-
-/**
  * i40iw_check_mem_contiguous - check if pbls stored in arr are contiguous
  * @arr: lvl1 pbl array
  * @npages: page count
@@ -1601,10 +1568,10 @@ static int i40iw_hw_alloc_stag(struct i40iw_device *iwdev, struct i40iw_mr *iwmr
  * @pd: ibpd pointer
  * @mr_type: memory for stag registrion
  * @max_num_sg: man number of pages
+ * @udata: user data or NULL for kernel objects
  */
-static struct ib_mr *i40iw_alloc_mr(struct ib_pd *pd,
-				    enum ib_mr_type mr_type,
-				    u32 max_num_sg)
+static struct ib_mr *i40iw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
+				    u32 max_num_sg, struct ib_udata *udata)
 {
 	struct i40iw_pd *iwpd = to_iwpd(pd);
 	struct i40iw_device *iwdev = to_iwdev(pd->device);
@@ -1841,10 +1808,9 @@ static struct ib_mr *i40iw_reg_user_mr(struct ib_pd *pd,
 	iwmr->ibmr.device = pd->device;
 
 	iwmr->page_size = PAGE_SIZE;
-	iwmr->page_msk = PAGE_MASK;
-
-	if (region->hugetlb && (req.reg_type == IW_MEMREG_TYPE_MEM))
-		i40iw_set_hugetlb_values(start, iwmr);
+	if (req.reg_type == IW_MEMREG_TYPE_MEM)
+		iwmr->page_size = ib_umem_find_best_pgsz(region, SZ_4K | SZ_2M,
+							 virt);
 
 	region_length = region->length + (start & (iwmr->page_size - 1));
 	pg_shift = ffs(iwmr->page_size) - 1;
@@ -2038,7 +2004,7 @@ static void i40iw_del_memlist(struct i40iw_mr *iwmr,
  * i40iw_dereg_mr - deregister mr
  * @ib_mr: mr ptr for dereg
  */
-static int i40iw_dereg_mr(struct ib_mr *ib_mr)
+static int i40iw_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
 {
 	struct ib_pd *ibpd = ib_mr->pd;
 	struct i40iw_pd *iwpd = to_iwpd(ibpd);
@@ -2058,9 +2024,12 @@ static int i40iw_dereg_mr(struct ib_mr *ib_mr)
 	if (iwmr->type != IW_MEMREG_TYPE_MEM) {
 		/* region is released. only test for userness. */
 		if (iwmr->region) {
-			struct i40iw_ucontext *ucontext;
+			struct i40iw_ucontext *ucontext =
+				rdma_udata_to_drv_context(
+					udata,
+					struct i40iw_ucontext,
+					ibucontext);
 
-			ucontext = to_ucontext(ibpd->uobject->context);
 			i40iw_del_memlist(iwmr, ucontext);
 		}
 		if (iwpbl->pbl_allocated && iwmr->type != IW_MEMREG_TYPE_QP)
@@ -2703,6 +2672,14 @@ static const struct ib_device_ops i40iw_dev_ops = {
 	.get_dma_mr = i40iw_get_dma_mr,
 	.get_hw_stats = i40iw_get_hw_stats,
 	.get_port_immutable = i40iw_port_immutable,
+	.iw_accept = i40iw_accept,
+	.iw_add_ref = i40iw_add_ref,
+	.iw_connect = i40iw_connect,
+	.iw_create_listen = i40iw_create_listen,
+	.iw_destroy_listen = i40iw_destroy_listen,
+	.iw_get_qp = i40iw_get_qp,
+	.iw_reject = i40iw_reject,
+	.iw_rem_ref = i40iw_rem_ref,
 	.map_mr_sg = i40iw_map_mr_sg,
 	.mmap = i40iw_mmap,
 	.modify_qp = i40iw_modify_qp,
@@ -2766,22 +2743,8 @@ static struct i40iw_ib_device *i40iw_init_rdma_device(struct i40iw_device *iwdev
 	iwibdev->ibdev.phys_port_cnt = 1;
 	iwibdev->ibdev.num_comp_vectors = iwdev->ceqs_count;
 	iwibdev->ibdev.dev.parent = &pcidev->dev;
-	iwibdev->ibdev.iwcm = kzalloc(sizeof(*iwibdev->ibdev.iwcm), GFP_KERNEL);
-	if (!iwibdev->ibdev.iwcm) {
-		ib_dealloc_device(&iwibdev->ibdev);
-		return NULL;
-	}
-
-	iwibdev->ibdev.iwcm->add_ref = i40iw_add_ref;
-	iwibdev->ibdev.iwcm->rem_ref = i40iw_rem_ref;
-	iwibdev->ibdev.iwcm->get_qp = i40iw_get_qp;
-	iwibdev->ibdev.iwcm->connect = i40iw_connect;
-	iwibdev->ibdev.iwcm->accept = i40iw_accept;
-	iwibdev->ibdev.iwcm->reject = i40iw_reject;
-	iwibdev->ibdev.iwcm->create_listen = i40iw_create_listen;
-	iwibdev->ibdev.iwcm->destroy_listen = i40iw_destroy_listen;
-	memcpy(iwibdev->ibdev.iwcm->ifname, netdev->name,
-	       sizeof(iwibdev->ibdev.iwcm->ifname));
+	memcpy(iwibdev->ibdev.iw_ifname, netdev->name,
+	       sizeof(iwibdev->ibdev.iw_ifname));
 	ib_set_device_ops(&iwibdev->ibdev, &i40iw_dev_ops);
 
 	return iwibdev;
@@ -2812,8 +2775,6 @@ void i40iw_destroy_rdma_device(struct i40iw_ib_device *iwibdev)
 		return;
 
 	ib_unregister_device(&iwibdev->ibdev);
-	kfree(iwibdev->ibdev.iwcm);
-	iwibdev->ibdev.iwcm = NULL;
 	wait_event_timeout(iwibdev->iwdev->close_wq,
 			   !atomic64_read(&iwibdev->iwdev->use_count),
 			   I40IW_EVENT_TIMEOUT);
@@ -2841,8 +2802,6 @@ int i40iw_register_rdma_device(struct i40iw_device *iwdev)
 
 	return 0;
 error:
-	kfree(iwdev->iwibdev->ibdev.iwcm);
-	iwdev->iwibdev->ibdev.iwcm = NULL;
 	ib_dealloc_device(&iwdev->iwibdev->ibdev);
 	return ret;
 }
diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.h b/drivers/infiniband/hw/i40iw/i40iw_verbs.h
index 76cf173377ab..3a413752ccc3 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_verbs.h
+++ b/drivers/infiniband/hw/i40iw/i40iw_verbs.h
@@ -94,8 +94,7 @@ struct i40iw_mr {
 	struct ib_umem *region;
 	u16 type;
 	u32 page_cnt;
-	u32 page_size;
-	u64 page_msk;
+	u64 page_size;
 	u32 npages;
 	u32 stag;
 	u64 length;
diff --git a/drivers/infiniband/hw/mlx4/ah.c b/drivers/infiniband/hw/mlx4/ah.c
index 1672808262ba..02a169f8027b 100644
--- a/drivers/infiniband/hw/mlx4/ah.c
+++ b/drivers/infiniband/hw/mlx4/ah.c
@@ -40,13 +40,12 @@
 
 #include "mlx4_ib.h"
 
-static struct ib_ah *create_ib_ah(struct ib_pd *pd,
-				  struct rdma_ah_attr *ah_attr,
-				  struct mlx4_ib_ah *ah)
+static void create_ib_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr)
 {
-	struct mlx4_dev *dev = to_mdev(pd->device)->dev;
+	struct mlx4_ib_ah *ah = to_mah(ib_ah);
+	struct mlx4_dev *dev = to_mdev(ib_ah->device)->dev;
 
-	ah->av.ib.port_pd = cpu_to_be32(to_mpd(pd)->pdn |
+	ah->av.ib.port_pd = cpu_to_be32(to_mpd(ib_ah->pd)->pdn |
 			    (rdma_ah_get_port_num(ah_attr) << 24));
 	ah->av.ib.g_slid  = rdma_ah_get_path_bits(ah_attr);
 	ah->av.ib.sl_tclass_flowlabel =
@@ -73,15 +72,12 @@ static struct ib_ah *create_ib_ah(struct ib_pd *pd,
 			--static_rate;
 		ah->av.ib.stat_rate = static_rate;
 	}
-
-	return &ah->ibah;
 }
 
-static struct ib_ah *create_iboe_ah(struct ib_pd *pd,
-				    struct rdma_ah_attr *ah_attr,
-				    struct mlx4_ib_ah *ah)
+static int create_iboe_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr)
 {
-	struct mlx4_ib_dev *ibdev = to_mdev(pd->device);
+	struct mlx4_ib_dev *ibdev = to_mdev(ib_ah->device);
+	struct mlx4_ib_ah *ah = to_mah(ib_ah);
 	const struct ib_gid_attr *gid_attr;
 	struct mlx4_dev *dev = ibdev->dev;
 	int is_mcast = 0;
@@ -103,12 +99,14 @@ static struct ib_ah *create_iboe_ah(struct ib_pd *pd,
 	 */
 	gid_attr = ah_attr->grh.sgid_attr;
 	if (gid_attr) {
-		if (is_vlan_dev(gid_attr->ndev))
-			vlan_tag = vlan_dev_vlan_id(gid_attr->ndev);
-		memcpy(ah->av.eth.s_mac, gid_attr->ndev->dev_addr, ETH_ALEN);
+		ret = rdma_read_gid_l2_fields(gid_attr, &vlan_tag,
+					      &ah->av.eth.s_mac[0]);
+		if (ret)
+			return ret;
+
 		ret = mlx4_ib_gid_index_to_real_index(ibdev, gid_attr);
 		if (ret < 0)
-			return ERR_PTR(ret);
+			return ret;
 		ah->av.eth.gid_index = ret;
 	} else {
 		/* mlx4_ib_create_ah_slave fills in the s_mac and the vlan */
@@ -117,7 +115,7 @@ static struct ib_ah *create_iboe_ah(struct ib_pd *pd,
 
 	if (vlan_tag < 0x1000)
 		vlan_tag |= (rdma_ah_get_sl(ah_attr) & 7) << 13;
-	ah->av.eth.port_pd = cpu_to_be32(to_mpd(pd)->pdn |
+	ah->av.eth.port_pd = cpu_to_be32(to_mpd(ib_ah->pd)->pdn |
 					 (rdma_ah_get_port_num(ah_attr) << 24));
 	ah->av.eth.vlan = cpu_to_be16(vlan_tag);
 	ah->av.eth.hop_limit = grh->hop_limit;
@@ -140,63 +138,45 @@ static struct ib_ah *create_iboe_ah(struct ib_pd *pd,
 	memcpy(ah->av.eth.dgid, grh->dgid.raw, 16);
 	ah->av.eth.sl_tclass_flowlabel |= cpu_to_be32(rdma_ah_get_sl(ah_attr)
 						      << 29);
-	return &ah->ibah;
+	return 0;
 }
 
-struct ib_ah *mlx4_ib_create_ah(struct ib_pd *pd, struct rdma_ah_attr *ah_attr,
-				u32 flags, struct ib_udata *udata)
+int mlx4_ib_create_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr,
+		      u32 flags, struct ib_udata *udata)
 
 {
-	struct mlx4_ib_ah *ah;
-	struct ib_ah *ret;
-
-	ah = kzalloc(sizeof *ah, GFP_ATOMIC);
-	if (!ah)
-		return ERR_PTR(-ENOMEM);
-
 	if (ah_attr->type == RDMA_AH_ATTR_TYPE_ROCE) {
-		if (!(rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH)) {
-			ret = ERR_PTR(-EINVAL);
-		} else {
-			/*
-			 * TBD: need to handle the case when we get
-			 * called in an atomic context and there we
-			 * might sleep.  We don't expect this
-			 * currently since we're working with link
-			 * local addresses which we can translate
-			 * without going to sleep.
-			 */
-			ret = create_iboe_ah(pd, ah_attr, ah);
-		}
-
-		if (IS_ERR(ret))
-			kfree(ah);
+		if (!(rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH))
+			return -EINVAL;
+		/*
+		 * TBD: need to handle the case when we get
+		 * called in an atomic context and there we
+		 * might sleep.  We don't expect this
+		 * currently since we're working with link
+		 * local addresses which we can translate
+		 * without going to sleep.
+		 */
+		return create_iboe_ah(ib_ah, ah_attr);
+	}
 
-		return ret;
-	} else
-		return create_ib_ah(pd, ah_attr, ah); /* never fails */
+	create_ib_ah(ib_ah, ah_attr);
+	return 0;
 }
 
-/* AH's created via this call must be free'd by mlx4_ib_destroy_ah. */
-struct ib_ah *mlx4_ib_create_ah_slave(struct ib_pd *pd,
-				      struct rdma_ah_attr *ah_attr,
-				      int slave_sgid_index, u8 *s_mac,
-				      u16 vlan_tag)
+int mlx4_ib_create_ah_slave(struct ib_ah *ah, struct rdma_ah_attr *ah_attr,
+			    int slave_sgid_index, u8 *s_mac, u16 vlan_tag)
 {
 	struct rdma_ah_attr slave_attr = *ah_attr;
-	struct mlx4_ib_ah *mah;
-	struct ib_ah *ah;
+	struct mlx4_ib_ah *mah = to_mah(ah);
+	int ret;
 
 	slave_attr.grh.sgid_attr = NULL;
 	slave_attr.grh.sgid_index = slave_sgid_index;
-	ah = mlx4_ib_create_ah(pd, &slave_attr, 0, NULL);
-	if (IS_ERR(ah))
-		return ah;
+	ret = mlx4_ib_create_ah(ah, &slave_attr, 0, NULL);
+	if (ret)
+		return ret;
 
-	ah->device = pd->device;
-	ah->pd = pd;
 	ah->type = ah_attr->type;
-	mah = to_mah(ah);
 
 	/* get rid of force-loopback bit */
 	mah->av.ib.port_pd &= cpu_to_be32(0x7FFFFFFF);
@@ -208,7 +188,7 @@ struct ib_ah *mlx4_ib_create_ah_slave(struct ib_pd *pd,
 		vlan_tag |= (rdma_ah_get_sl(ah_attr) & 7) << 13;
 	mah->av.eth.vlan = cpu_to_be16(vlan_tag);
 
-	return ah;
+	return 0;
 }
 
 int mlx4_ib_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr)
@@ -250,8 +230,7 @@ int mlx4_ib_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr)
 	return 0;
 }
 
-int mlx4_ib_destroy_ah(struct ib_ah *ah, u32 flags)
+void mlx4_ib_destroy_ah(struct ib_ah *ah, u32 flags)
 {
-	kfree(to_mah(ah));
-	return 0;
+	return;
 }
diff --git a/drivers/infiniband/hw/mlx4/cm.c b/drivers/infiniband/hw/mlx4/cm.c
index 8c79a480f2b7..ecd6cadd529a 100644
--- a/drivers/infiniband/hw/mlx4/cm.c
+++ b/drivers/infiniband/hw/mlx4/cm.c
@@ -168,20 +168,17 @@ static void id_map_ent_timeout(struct work_struct *work)
 {
 	struct delayed_work *delay = to_delayed_work(work);
 	struct id_map_entry *ent = container_of(delay, struct id_map_entry, timeout);
-	struct id_map_entry *db_ent, *found_ent;
+	struct id_map_entry *found_ent;
 	struct mlx4_ib_dev *dev = ent->dev;
 	struct mlx4_ib_sriov *sriov = &dev->sriov;
 	struct rb_root *sl_id_map = &sriov->sl_id_map;
-	int pv_id = (int) ent->pv_cm_id;
 
 	spin_lock(&sriov->id_map_lock);
-	db_ent = (struct id_map_entry *)idr_find(&sriov->pv_id_table, pv_id);
-	if (!db_ent)
+	if (!xa_erase(&sriov->pv_id_table, ent->pv_cm_id))
 		goto out;
 	found_ent = id_map_find_by_sl_id(&dev->ib_dev, ent->slave_id, ent->sl_cm_id);
 	if (found_ent && found_ent == ent)
 		rb_erase(&found_ent->node, sl_id_map);
-	idr_remove(&sriov->pv_id_table, pv_id);
 
 out:
 	list_del(&ent->list);
@@ -196,13 +193,12 @@ static void id_map_find_del(struct ib_device *ibdev, int pv_cm_id)
 	struct id_map_entry *ent, *found_ent;
 
 	spin_lock(&sriov->id_map_lock);
-	ent = (struct id_map_entry *)idr_find(&sriov->pv_id_table, pv_cm_id);
+	ent = xa_erase(&sriov->pv_id_table, pv_cm_id);
 	if (!ent)
 		goto out;
 	found_ent = id_map_find_by_sl_id(ibdev, ent->slave_id, ent->sl_cm_id);
 	if (found_ent && found_ent == ent)
 		rb_erase(&found_ent->node, sl_id_map);
-	idr_remove(&sriov->pv_id_table, pv_cm_id);
 out:
 	spin_unlock(&sriov->id_map_lock);
 }
@@ -256,25 +252,19 @@ id_map_alloc(struct ib_device *ibdev, int slave_id, u32 sl_cm_id)
 	ent->dev = to_mdev(ibdev);
 	INIT_DELAYED_WORK(&ent->timeout, id_map_ent_timeout);
 
-	idr_preload(GFP_KERNEL);
-	spin_lock(&to_mdev(ibdev)->sriov.id_map_lock);
-
-	ret = idr_alloc_cyclic(&sriov->pv_id_table, ent, 0, 0, GFP_NOWAIT);
+	ret = xa_alloc_cyclic(&sriov->pv_id_table, &ent->pv_cm_id, ent,
+			xa_limit_32b, &sriov->pv_id_next, GFP_KERNEL);
 	if (ret >= 0) {
-		ent->pv_cm_id = (u32)ret;
+		spin_lock(&sriov->id_map_lock);
 		sl_id_map_add(ibdev, ent);
 		list_add_tail(&ent->list, &sriov->cm_list);
-	}
-
-	spin_unlock(&sriov->id_map_lock);
-	idr_preload_end();
-
-	if (ret >= 0)
+		spin_unlock(&sriov->id_map_lock);
 		return ent;
+	}
 
 	/*error flow*/
 	kfree(ent);
-	mlx4_ib_warn(ibdev, "No more space in the idr (err:0x%x)\n", ret);
+	mlx4_ib_warn(ibdev, "Allocation failed (err:0x%x)\n", ret);
 	return ERR_PTR(-ENOMEM);
 }
 
@@ -290,7 +280,7 @@ id_map_get(struct ib_device *ibdev, int *pv_cm_id, int slave_id, int sl_cm_id)
 		if (ent)
 			*pv_cm_id = (int) ent->pv_cm_id;
 	} else
-		ent = (struct id_map_entry *)idr_find(&sriov->pv_id_table, *pv_cm_id);
+		ent = xa_load(&sriov->pv_id_table, *pv_cm_id);
 	spin_unlock(&sriov->id_map_lock);
 
 	return ent;
@@ -407,7 +397,7 @@ void mlx4_ib_cm_paravirt_init(struct mlx4_ib_dev *dev)
 	spin_lock_init(&dev->sriov.id_map_lock);
 	INIT_LIST_HEAD(&dev->sriov.cm_list);
 	dev->sriov.sl_id_map = RB_ROOT;
-	idr_init(&dev->sriov.pv_id_table);
+	xa_init_flags(&dev->sriov.pv_id_table, XA_FLAGS_ALLOC);
 }
 
 /* slave = -1 ==> all slaves */
@@ -444,7 +434,7 @@ void mlx4_ib_cm_paravirt_clean(struct mlx4_ib_dev *dev, int slave)
 					 struct id_map_entry, node);
 
 			rb_erase(&ent->node, sl_id_map);
-			idr_remove(&sriov->pv_id_table, (int) ent->pv_cm_id);
+			xa_erase(&sriov->pv_id_table, ent->pv_cm_id);
 		}
 		list_splice_init(&dev->sriov.cm_list, &lh);
 	} else {
@@ -460,7 +450,7 @@ void mlx4_ib_cm_paravirt_clean(struct mlx4_ib_dev *dev, int slave)
 		/* remove those nodes from databases */
 		list_for_each_entry_safe(map, tmp_map, &lh, list) {
 			rb_erase(&map->node, sl_id_map);
-			idr_remove(&sriov->pv_id_table, (int) map->pv_cm_id);
+			xa_erase(&sriov->pv_id_table, map->pv_cm_id);
 		}
 
 		/* add remaining nodes from cm_list */
diff --git a/drivers/infiniband/hw/mlx4/cq.c b/drivers/infiniband/hw/mlx4/cq.c
index 03ac72339dd2..022a0b4ea452 100644
--- a/drivers/infiniband/hw/mlx4/cq.c
+++ b/drivers/infiniband/hw/mlx4/cq.c
@@ -38,6 +38,7 @@
 
 #include "mlx4_ib.h"
 #include <rdma/mlx4-abi.h>
+#include <rdma/uverbs_ioctl.h>
 
 static void mlx4_ib_cq_comp(struct mlx4_cq *cq)
 {
@@ -173,7 +174,6 @@ err_buf:
 #define CQ_CREATE_FLAGS_SUPPORTED IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION
 struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev,
 				const struct ib_cq_init_attr *attr,
-				struct ib_ucontext *context,
 				struct ib_udata *udata)
 {
 	int entries = attr->cqe;
@@ -183,6 +183,8 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev,
 	struct mlx4_uar *uar;
 	void *buf_addr;
 	int err;
+	struct mlx4_ib_ucontext *context = rdma_udata_to_drv_context(
+		udata, struct mlx4_ib_ucontext, ibucontext);
 
 	if (entries < 1 || entries > dev->dev->caps.max_cqes)
 		return ERR_PTR(-EINVAL);
@@ -204,7 +206,7 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev,
 	INIT_LIST_HEAD(&cq->send_qp_list);
 	INIT_LIST_HEAD(&cq->recv_qp_list);
 
-	if (context) {
+	if (udata) {
 		struct mlx4_ib_create_cq ucmd;
 
 		if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) {
@@ -218,12 +220,11 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev,
 		if (err)
 			goto err_cq;
 
-		err = mlx4_ib_db_map_user(to_mucontext(context), udata,
-					  ucmd.db_addr, &cq->db);
+		err = mlx4_ib_db_map_user(udata, ucmd.db_addr, &cq->db);
 		if (err)
 			goto err_mtt;
 
-		uar = &to_mucontext(context)->uar;
+		uar = &context->uar;
 		cq->mcq.usage = MLX4_RES_USAGE_USER_VERBS;
 	} else {
 		err = mlx4_db_alloc(dev->dev, &cq->db, 1);
@@ -248,21 +249,21 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev,
 	if (dev->eq_table)
 		vector = dev->eq_table[vector % ibdev->num_comp_vectors];
 
-	err = mlx4_cq_alloc(dev->dev, entries, &cq->buf.mtt, uar,
-			    cq->db.dma, &cq->mcq, vector, 0,
+	err = mlx4_cq_alloc(dev->dev, entries, &cq->buf.mtt, uar, cq->db.dma,
+			    &cq->mcq, vector, 0,
 			    !!(cq->create_flags &
 			       IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION),
-			    buf_addr, !!context);
+			    buf_addr, !!udata);
 	if (err)
 		goto err_dbmap;
 
-	if (context)
+	if (udata)
 		cq->mcq.tasklet_ctx.comp = mlx4_ib_cq_comp;
 	else
 		cq->mcq.comp = mlx4_ib_cq_comp;
 	cq->mcq.event = mlx4_ib_cq_event;
 
-	if (context)
+	if (udata)
 		if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof (__u32))) {
 			err = -EFAULT;
 			goto err_cq_free;
@@ -274,19 +275,19 @@ err_cq_free:
 	mlx4_cq_free(dev->dev, &cq->mcq);
 
 err_dbmap:
-	if (context)
-		mlx4_ib_db_unmap_user(to_mucontext(context), &cq->db);
+	if (udata)
+		mlx4_ib_db_unmap_user(context, &cq->db);
 
 err_mtt:
 	mlx4_mtt_cleanup(dev->dev, &cq->buf.mtt);
 
-	if (context)
+	if (udata)
 		ib_umem_release(cq->umem);
 	else
 		mlx4_ib_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe);
 
 err_db:
-	if (!context)
+	if (!udata)
 		mlx4_db_free(dev->dev, &cq->db);
 
 err_cq:
@@ -485,7 +486,7 @@ out:
 	return err;
 }
 
-int mlx4_ib_destroy_cq(struct ib_cq *cq)
+int mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata)
 {
 	struct mlx4_ib_dev *dev = to_mdev(cq->device);
 	struct mlx4_ib_cq *mcq = to_mcq(cq);
@@ -493,8 +494,13 @@ int mlx4_ib_destroy_cq(struct ib_cq *cq)
 	mlx4_cq_free(dev->dev, &mcq->mcq);
 	mlx4_mtt_cleanup(dev->dev, &mcq->buf.mtt);
 
-	if (cq->uobject) {
-		mlx4_ib_db_unmap_user(to_mucontext(cq->uobject->context), &mcq->db);
+	if (udata) {
+		mlx4_ib_db_unmap_user(
+			rdma_udata_to_drv_context(
+				udata,
+				struct mlx4_ib_ucontext,
+				ibucontext),
+			&mcq->db);
 		ib_umem_release(mcq->umem);
 	} else {
 		mlx4_ib_free_cq_buf(dev, &mcq->buf, cq->cqe);
diff --git a/drivers/infiniband/hw/mlx4/doorbell.c b/drivers/infiniband/hw/mlx4/doorbell.c
index 3aab71b29ce8..0f390351cef0 100644
--- a/drivers/infiniband/hw/mlx4/doorbell.c
+++ b/drivers/infiniband/hw/mlx4/doorbell.c
@@ -31,6 +31,7 @@
  */
 
 #include <linux/slab.h>
+#include <rdma/uverbs_ioctl.h>
 
 #include "mlx4_ib.h"
 
@@ -41,12 +42,13 @@ struct mlx4_ib_user_db_page {
 	int			refcnt;
 };
 
-int mlx4_ib_db_map_user(struct mlx4_ib_ucontext *context,
-			struct ib_udata *udata, unsigned long virt,
+int mlx4_ib_db_map_user(struct ib_udata *udata, unsigned long virt,
 			struct mlx4_db *db)
 {
 	struct mlx4_ib_user_db_page *page;
 	int err = 0;
+	struct mlx4_ib_ucontext *context = rdma_udata_to_drv_context(
+		udata, struct mlx4_ib_ucontext, ibucontext);
 
 	mutex_lock(&context->db_page_mutex);
 
diff --git a/drivers/infiniband/hw/mlx4/mad.c b/drivers/infiniband/hw/mlx4/mad.c
index 936ee1314bcd..68c951491a08 100644
--- a/drivers/infiniband/hw/mlx4/mad.c
+++ b/drivers/infiniband/hw/mlx4/mad.c
@@ -1371,9 +1371,9 @@ int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev, int slave, u8 port,
 	struct ib_ah *ah;
 	struct ib_qp *send_qp = NULL;
 	unsigned wire_tx_ix = 0;
-	int ret = 0;
 	u16 wire_pkey_ix;
 	int src_qpnum;
+	int ret;
 
 	sqp_ctx = dev->sriov.sqps[port-1];
 
@@ -1393,12 +1393,20 @@ int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev, int slave, u8 port,
 
 	send_qp = sqp->qp;
 
-	/* create ah */
-	ah = mlx4_ib_create_ah_slave(sqp_ctx->pd, attr,
-				     rdma_ah_retrieve_grh(attr)->sgid_index,
-				     s_mac, vlan_id);
-	if (IS_ERR(ah))
+	ah = rdma_zalloc_drv_obj(sqp_ctx->pd->device, ib_ah);
+	if (!ah)
 		return -ENOMEM;
+
+	ah->device = sqp_ctx->pd->device;
+	ah->pd = sqp_ctx->pd;
+
+	/* create ah */
+	ret = mlx4_ib_create_ah_slave(ah, attr,
+				      rdma_ah_retrieve_grh(attr)->sgid_index,
+				      s_mac, vlan_id);
+	if (ret)
+		goto out;
+
 	spin_lock(&sqp->tx_lock);
 	if (sqp->tx_ix_head - sqp->tx_ix_tail >=
 	    (MLX4_NUM_TUNNEL_BUFS - 1))
@@ -1410,8 +1418,7 @@ int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev, int slave, u8 port,
 		goto out;
 
 	sqp_mad = (struct mlx4_mad_snd_buf *) (sqp->tx_ring[wire_tx_ix].buf.addr);
-	if (sqp->tx_ring[wire_tx_ix].ah)
-		mlx4_ib_destroy_ah(sqp->tx_ring[wire_tx_ix].ah, 0);
+	kfree(sqp->tx_ring[wire_tx_ix].ah);
 	sqp->tx_ring[wire_tx_ix].ah = ah;
 	ib_dma_sync_single_for_cpu(&dev->ib_dev,
 				   sqp->tx_ring[wire_tx_ix].buf.map,
@@ -1450,7 +1457,7 @@ int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev, int slave, u8 port,
 	spin_unlock(&sqp->tx_lock);
 	sqp->tx_ring[wire_tx_ix].ah = NULL;
 out:
-	mlx4_ib_destroy_ah(ah, 0);
+	kfree(ah);
 	return ret;
 }
 
@@ -1902,8 +1909,8 @@ static void mlx4_ib_sqp_comp_worker(struct work_struct *work)
 		if (wc.status == IB_WC_SUCCESS) {
 			switch (wc.opcode) {
 			case IB_WC_SEND:
-				mlx4_ib_destroy_ah(sqp->tx_ring[wc.wr_id &
-					      (MLX4_NUM_TUNNEL_BUFS - 1)].ah, 0);
+				kfree(sqp->tx_ring[wc.wr_id &
+				      (MLX4_NUM_TUNNEL_BUFS - 1)].ah);
 				sqp->tx_ring[wc.wr_id & (MLX4_NUM_TUNNEL_BUFS - 1)].ah
 					= NULL;
 				spin_lock(&sqp->tx_lock);
@@ -1931,8 +1938,8 @@ static void mlx4_ib_sqp_comp_worker(struct work_struct *work)
 				 " status = %d, wrid = 0x%llx\n",
 				 ctx->slave, wc.status, wc.wr_id);
 			if (!MLX4_TUN_IS_RECV(wc.wr_id)) {
-				mlx4_ib_destroy_ah(sqp->tx_ring[wc.wr_id &
-					      (MLX4_NUM_TUNNEL_BUFS - 1)].ah, 0);
+				kfree(sqp->tx_ring[wc.wr_id &
+				      (MLX4_NUM_TUNNEL_BUFS - 1)].ah);
 				sqp->tx_ring[wc.wr_id & (MLX4_NUM_TUNNEL_BUFS - 1)].ah
 					= NULL;
 				spin_lock(&sqp->tx_lock);
diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c
index 733f7bbd5901..25d09d53b51c 100644
--- a/drivers/infiniband/hw/mlx4/main.c
+++ b/drivers/infiniband/hw/mlx4/main.c
@@ -1177,8 +1177,7 @@ static int mlx4_ib_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
 	}
 }
 
-static int mlx4_ib_alloc_pd(struct ib_pd *ibpd, struct ib_ucontext *context,
-			    struct ib_udata *udata)
+static int mlx4_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 {
 	struct mlx4_ib_pd *pd = to_mpd(ibpd);
 	struct ib_device *ibdev = ibpd->device;
@@ -1188,20 +1187,19 @@ static int mlx4_ib_alloc_pd(struct ib_pd *ibpd, struct ib_ucontext *context,
 	if (err)
 		return err;
 
-	if (context && ib_copy_to_udata(udata, &pd->pdn, sizeof(__u32))) {
+	if (udata && ib_copy_to_udata(udata, &pd->pdn, sizeof(__u32))) {
 		mlx4_pd_free(to_mdev(ibdev)->dev, pd->pdn);
 		return -EFAULT;
 	}
 	return 0;
 }
 
-static void mlx4_ib_dealloc_pd(struct ib_pd *pd)
+static void mlx4_ib_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)
 {
 	mlx4_pd_free(to_mdev(pd->device)->dev, to_mpd(pd)->pdn);
 }
 
 static struct ib_xrcd *mlx4_ib_alloc_xrcd(struct ib_device *ibdev,
-					  struct ib_ucontext *context,
 					  struct ib_udata *udata)
 {
 	struct mlx4_ib_xrcd *xrcd;
@@ -1243,7 +1241,7 @@ err1:
 	return ERR_PTR(err);
 }
 
-static int mlx4_ib_dealloc_xrcd(struct ib_xrcd *xrcd)
+static int mlx4_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata)
 {
 	ib_destroy_cq(to_mxrcd(xrcd)->cq);
 	ib_dealloc_pd(to_mxrcd(xrcd)->pd);
@@ -2560,7 +2558,10 @@ static const struct ib_device_ops mlx4_ib_dev_ops = {
 	.req_notify_cq = mlx4_ib_arm_cq,
 	.rereg_user_mr = mlx4_ib_rereg_user_mr,
 	.resize_cq = mlx4_ib_resize_cq,
+
+	INIT_RDMA_OBJ_SIZE(ib_ah, mlx4_ib_ah, ibah),
 	INIT_RDMA_OBJ_SIZE(ib_pd, mlx4_ib_pd, ibpd),
+	INIT_RDMA_OBJ_SIZE(ib_srq, mlx4_ib_srq, ibsrq),
 	INIT_RDMA_OBJ_SIZE(ib_ucontext, mlx4_ib_ucontext, ibucontext),
 };
 
diff --git a/drivers/infiniband/hw/mlx4/mlx4_ib.h b/drivers/infiniband/hw/mlx4/mlx4_ib.h
index 60dc1347c5ab..26897102057d 100644
--- a/drivers/infiniband/hw/mlx4/mlx4_ib.h
+++ b/drivers/infiniband/hw/mlx4/mlx4_ib.h
@@ -492,10 +492,11 @@ struct mlx4_ib_sriov {
 	struct mlx4_sriov_alias_guid alias_guid;
 
 	/* CM paravirtualization fields */
-	struct list_head cm_list;
+	struct xarray pv_id_table;
+	u32 pv_id_next;
 	spinlock_t id_map_lock;
 	struct rb_root sl_id_map;
-	struct idr pv_id_table;
+	struct list_head cm_list;
 };
 
 struct gid_cache_context {
@@ -722,8 +723,7 @@ static inline u8 mlx4_ib_bond_next_port(struct mlx4_ib_dev *dev)
 int mlx4_ib_init_sriov(struct mlx4_ib_dev *dev);
 void mlx4_ib_close_sriov(struct mlx4_ib_dev *dev);
 
-int mlx4_ib_db_map_user(struct mlx4_ib_ucontext *context,
-			struct ib_udata *udata, unsigned long virt,
+int mlx4_ib_db_map_user(struct ib_udata *udata, unsigned long virt,
 			struct mlx4_db *db);
 void mlx4_ib_db_unmap_user(struct mlx4_ib_ucontext *context, struct mlx4_db *db);
 
@@ -733,43 +733,38 @@ int mlx4_ib_umem_write_mtt(struct mlx4_ib_dev *dev, struct mlx4_mtt *mtt,
 struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 				  u64 virt_addr, int access_flags,
 				  struct ib_udata *udata);
-int mlx4_ib_dereg_mr(struct ib_mr *mr);
+int mlx4_ib_dereg_mr(struct ib_mr *mr, struct ib_udata *udata);
 struct ib_mw *mlx4_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
 			       struct ib_udata *udata);
 int mlx4_ib_dealloc_mw(struct ib_mw *mw);
-struct ib_mr *mlx4_ib_alloc_mr(struct ib_pd *pd,
-			       enum ib_mr_type mr_type,
-			       u32 max_num_sg);
+struct ib_mr *mlx4_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
+			       u32 max_num_sg, struct ib_udata *udata);
 int mlx4_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
 		      unsigned int *sg_offset);
 int mlx4_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period);
 int mlx4_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata);
 struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev,
 				const struct ib_cq_init_attr *attr,
-				struct ib_ucontext *context,
 				struct ib_udata *udata);
-int mlx4_ib_destroy_cq(struct ib_cq *cq);
+int mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata);
 int mlx4_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
 int mlx4_ib_arm_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags);
 void __mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq);
 void mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq);
 
-struct ib_ah *mlx4_ib_create_ah(struct ib_pd *pd, struct rdma_ah_attr *ah_attr,
-				u32 flags, struct ib_udata *udata);
-struct ib_ah *mlx4_ib_create_ah_slave(struct ib_pd *pd,
-				      struct rdma_ah_attr *ah_attr,
-				      int slave_sgid_index, u8 *s_mac,
-				      u16 vlan_tag);
+int mlx4_ib_create_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr, u32 flags,
+		      struct ib_udata *udata);
+int mlx4_ib_create_ah_slave(struct ib_ah *ah, struct rdma_ah_attr *ah_attr,
+			    int slave_sgid_index, u8 *s_mac, u16 vlan_tag);
 int mlx4_ib_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr);
-int mlx4_ib_destroy_ah(struct ib_ah *ah, u32 flags);
+void mlx4_ib_destroy_ah(struct ib_ah *ah, u32 flags);
 
-struct ib_srq *mlx4_ib_create_srq(struct ib_pd *pd,
-				  struct ib_srq_init_attr *init_attr,
-				  struct ib_udata *udata);
+int mlx4_ib_create_srq(struct ib_srq *srq, struct ib_srq_init_attr *init_attr,
+		       struct ib_udata *udata);
 int mlx4_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
 		       enum ib_srq_attr_mask attr_mask, struct ib_udata *udata);
 int mlx4_ib_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr);
-int mlx4_ib_destroy_srq(struct ib_srq *srq);
+void mlx4_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata);
 void mlx4_ib_free_srq_wqe(struct mlx4_ib_srq *srq, int wqe_index);
 int mlx4_ib_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
 			  const struct ib_recv_wr **bad_wr);
@@ -777,7 +772,7 @@ int mlx4_ib_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
 struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd,
 				struct ib_qp_init_attr *init_attr,
 				struct ib_udata *udata);
-int mlx4_ib_destroy_qp(struct ib_qp *qp);
+int mlx4_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata);
 void mlx4_ib_drain_sq(struct ib_qp *qp);
 void mlx4_ib_drain_rq(struct ib_qp *qp);
 int mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
@@ -912,7 +907,7 @@ void mlx4_ib_sl2vl_update(struct mlx4_ib_dev *mdev, int port);
 struct ib_wq *mlx4_ib_create_wq(struct ib_pd *pd,
 				struct ib_wq_init_attr *init_attr,
 				struct ib_udata *udata);
-int mlx4_ib_destroy_wq(struct ib_wq *wq);
+int mlx4_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata);
 int mlx4_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,
 		      u32 wq_attr_mask, struct ib_udata *udata);
 
diff --git a/drivers/infiniband/hw/mlx4/mr.c b/drivers/infiniband/hw/mlx4/mr.c
index 395379a480cb..355205a28544 100644
--- a/drivers/infiniband/hw/mlx4/mr.c
+++ b/drivers/infiniband/hw/mlx4/mr.c
@@ -595,7 +595,7 @@ mlx4_free_priv_pages(struct mlx4_ib_mr *mr)
 	}
 }
 
-int mlx4_ib_dereg_mr(struct ib_mr *ibmr)
+int mlx4_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
 {
 	struct mlx4_ib_mr *mr = to_mmr(ibmr);
 	int ret;
@@ -655,9 +655,8 @@ int mlx4_ib_dealloc_mw(struct ib_mw *ibmw)
 	return 0;
 }
 
-struct ib_mr *mlx4_ib_alloc_mr(struct ib_pd *pd,
-			       enum ib_mr_type mr_type,
-			       u32 max_num_sg)
+struct ib_mr *mlx4_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
+			       u32 max_num_sg, struct ib_udata *udata)
 {
 	struct mlx4_ib_dev *dev = to_mdev(pd->device);
 	struct mlx4_ib_mr *mr;
diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c
index 9426936460f8..5221c0794d1d 100644
--- a/drivers/infiniband/hw/mlx4/qp.c
+++ b/drivers/infiniband/hw/mlx4/qp.c
@@ -1041,11 +1041,11 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,
 			goto err_mtt;
 
 		if (qp_has_rq(init_attr)) {
-			err = mlx4_ib_db_map_user(
-				context, udata,
-				(src == MLX4_IB_QP_SRC) ? ucmd.qp.db_addr :
+			err = mlx4_ib_db_map_user(udata,
+						  (src == MLX4_IB_QP_SRC) ?
+							  ucmd.qp.db_addr :
 							  ucmd.wq.db_addr,
-				&qp->db);
+						  &qp->db);
 			if (err)
 				goto err_mtt;
 		}
@@ -1338,7 +1338,8 @@ static void destroy_qp_rss(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
 }
 
 static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp,
-			      enum mlx4_ib_source_type src, bool is_user)
+			      enum mlx4_ib_source_type src,
+			      struct ib_udata *udata)
 {
 	struct mlx4_ib_cq *send_cq, *recv_cq;
 	unsigned long flags;
@@ -1380,7 +1381,7 @@ static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp,
 	list_del(&qp->qps_list);
 	list_del(&qp->cq_send_list);
 	list_del(&qp->cq_recv_list);
-	if (!is_user) {
+	if (!udata) {
 		__mlx4_ib_cq_clean(recv_cq, qp->mqp.qpn,
 				 qp->ibqp.srq ? to_msrq(qp->ibqp.srq): NULL);
 		if (send_cq != recv_cq)
@@ -1398,19 +1399,26 @@ static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp,
 		if (qp->flags & MLX4_IB_QP_NETIF)
 			mlx4_ib_steer_qp_free(dev, qp->mqp.qpn, 1);
 		else if (src == MLX4_IB_RWQ_SRC)
-			mlx4_ib_release_wqn(to_mucontext(
-					    qp->ibwq.uobject->context), qp, 1);
+			mlx4_ib_release_wqn(
+				rdma_udata_to_drv_context(
+					udata,
+					struct mlx4_ib_ucontext,
+					ibucontext),
+				qp, 1);
 		else
 			mlx4_qp_release_range(dev->dev, qp->mqp.qpn, 1);
 	}
 
 	mlx4_mtt_cleanup(dev->dev, &qp->mtt);
 
-	if (is_user) {
+	if (udata) {
 		if (qp->rq.wqe_cnt) {
-			struct mlx4_ib_ucontext *mcontext = !src ?
-				to_mucontext(qp->ibqp.uobject->context) :
-				to_mucontext(qp->ibwq.uobject->context);
+			struct mlx4_ib_ucontext *mcontext =
+				rdma_udata_to_drv_context(
+					udata,
+					struct mlx4_ib_ucontext,
+					ibucontext);
+
 			mlx4_ib_db_unmap_user(mcontext, &qp->db);
 		}
 		ib_umem_release(qp->umem);
@@ -1594,7 +1602,7 @@ struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd,
 	return ibqp;
 }
 
-static int _mlx4_ib_destroy_qp(struct ib_qp *qp)
+static int _mlx4_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata)
 {
 	struct mlx4_ib_dev *dev = to_mdev(qp->device);
 	struct mlx4_ib_qp *mqp = to_mqp(qp);
@@ -1615,7 +1623,7 @@ static int _mlx4_ib_destroy_qp(struct ib_qp *qp)
 	if (qp->rwq_ind_tbl) {
 		destroy_qp_rss(dev, mqp);
 	} else {
-		destroy_qp_common(dev, mqp, MLX4_IB_QP_SRC, qp->uobject);
+		destroy_qp_common(dev, mqp, MLX4_IB_QP_SRC, udata);
 	}
 
 	if (is_sqp(dev, mqp))
@@ -1626,7 +1634,7 @@ static int _mlx4_ib_destroy_qp(struct ib_qp *qp)
 	return 0;
 }
 
-int mlx4_ib_destroy_qp(struct ib_qp *qp)
+int mlx4_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata)
 {
 	struct mlx4_ib_qp *mqp = to_mqp(qp);
 
@@ -1637,7 +1645,7 @@ int mlx4_ib_destroy_qp(struct ib_qp *qp)
 			ib_destroy_qp(sqp->roce_v2_gsi);
 	}
 
-	return _mlx4_ib_destroy_qp(qp);
+	return _mlx4_ib_destroy_qp(qp, udata);
 }
 
 static int to_mlx4_st(struct mlx4_ib_dev *dev, enum mlx4_ib_qp_type type)
@@ -2240,8 +2248,10 @@ static int __mlx4_ib_modify_qp(void *src, enum mlx4_ib_source_type src_type,
 
 		if (is_eth) {
 			gid_attr = attr->ah_attr.grh.sgid_attr;
-			vlan = rdma_vlan_dev_vlan_id(gid_attr->ndev);
-			memcpy(smac, gid_attr->ndev->dev_addr, ETH_ALEN);
+			err = rdma_read_gid_l2_fields(gid_attr, &vlan,
+						      &smac[0]);
+			if (err)
+				goto out;
 		}
 
 		if (mlx4_set_path(dev, attr, attr_mask, qp, &context->pri_path,
@@ -4238,7 +4248,7 @@ int mlx4_ib_modify_wq(struct ib_wq *ibwq, struct ib_wq_attr *wq_attr,
 	return err;
 }
 
-int mlx4_ib_destroy_wq(struct ib_wq *ibwq)
+int mlx4_ib_destroy_wq(struct ib_wq *ibwq, struct ib_udata *udata)
 {
 	struct mlx4_ib_dev *dev = to_mdev(ibwq->device);
 	struct mlx4_ib_qp *qp = to_mqp((struct ib_qp *)ibwq);
@@ -4246,7 +4256,7 @@ int mlx4_ib_destroy_wq(struct ib_wq *ibwq)
 	if (qp->counter_index)
 		mlx4_ib_free_qp_counter(dev, qp);
 
-	destroy_qp_common(dev, qp, MLX4_IB_RWQ_SRC, 1);
+	destroy_qp_common(dev, qp, MLX4_IB_RWQ_SRC, udata);
 
 	kfree(qp);
 
diff --git a/drivers/infiniband/hw/mlx4/srq.c b/drivers/infiniband/hw/mlx4/srq.c
index 381cf899bcef..4bf2946b9759 100644
--- a/drivers/infiniband/hw/mlx4/srq.c
+++ b/drivers/infiniband/hw/mlx4/srq.c
@@ -69,14 +69,14 @@ static void mlx4_ib_srq_event(struct mlx4_srq *srq, enum mlx4_event type)
 	}
 }
 
-struct ib_srq *mlx4_ib_create_srq(struct ib_pd *pd,
-				  struct ib_srq_init_attr *init_attr,
-				  struct ib_udata *udata)
+int mlx4_ib_create_srq(struct ib_srq *ib_srq,
+		       struct ib_srq_init_attr *init_attr,
+		       struct ib_udata *udata)
 {
-	struct mlx4_ib_dev *dev = to_mdev(pd->device);
+	struct mlx4_ib_dev *dev = to_mdev(ib_srq->device);
 	struct mlx4_ib_ucontext *ucontext = rdma_udata_to_drv_context(
 		udata, struct mlx4_ib_ucontext, ibucontext);
-	struct mlx4_ib_srq *srq;
+	struct mlx4_ib_srq *srq = to_msrq(ib_srq);
 	struct mlx4_wqe_srq_next_seg *next;
 	struct mlx4_wqe_data_seg *scatter;
 	u32 cqn;
@@ -89,11 +89,7 @@ struct ib_srq *mlx4_ib_create_srq(struct ib_pd *pd,
 	/* Sanity check SRQ size before proceeding */
 	if (init_attr->attr.max_wr  >= dev->dev->caps.max_srq_wqes ||
 	    init_attr->attr.max_sge >  dev->dev->caps.max_srq_sge)
-		return ERR_PTR(-EINVAL);
-
-	srq = kmalloc(sizeof *srq, GFP_KERNEL);
-	if (!srq)
-		return ERR_PTR(-ENOMEM);
+		return -EINVAL;
 
 	mutex_init(&srq->mutex);
 	spin_lock_init(&srq->lock);
@@ -111,16 +107,12 @@ struct ib_srq *mlx4_ib_create_srq(struct ib_pd *pd,
 	if (udata) {
 		struct mlx4_ib_create_srq ucmd;
 
-		if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) {
-			err = -EFAULT;
-			goto err_srq;
-		}
+		if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)))
+			return -EFAULT;
 
 		srq->umem = ib_umem_get(udata, ucmd.buf_addr, buf_size, 0, 0);
-		if (IS_ERR(srq->umem)) {
-			err = PTR_ERR(srq->umem);
-			goto err_srq;
-		}
+		if (IS_ERR(srq->umem))
+			return PTR_ERR(srq->umem);
 
 		err = mlx4_mtt_init(dev->dev, ib_umem_page_count(srq->umem),
 				    srq->umem->page_shift, &srq->mtt);
@@ -131,14 +123,13 @@ struct ib_srq *mlx4_ib_create_srq(struct ib_pd *pd,
 		if (err)
 			goto err_mtt;
 
-		err = mlx4_ib_db_map_user(ucontext, udata, ucmd.db_addr,
-					  &srq->db);
+		err = mlx4_ib_db_map_user(udata, ucmd.db_addr, &srq->db);
 		if (err)
 			goto err_mtt;
 	} else {
 		err = mlx4_db_alloc(dev->dev, &srq->db, 0);
 		if (err)
-			goto err_srq;
+			return err;
 
 		*srq->db.db = 0;
 
@@ -185,8 +176,8 @@ struct ib_srq *mlx4_ib_create_srq(struct ib_pd *pd,
 	xrcdn = (init_attr->srq_type == IB_SRQT_XRC) ?
 		to_mxrcd(init_attr->ext.xrc.xrcd)->xrcdn :
 		(u16) dev->dev->caps.reserved_xrcds;
-	err = mlx4_srq_alloc(dev->dev, to_mpd(pd)->pdn, cqn, xrcdn, &srq->mtt,
-			     srq->db.dma, &srq->msrq);
+	err = mlx4_srq_alloc(dev->dev, to_mpd(ib_srq->pd)->pdn, cqn, xrcdn,
+			     &srq->mtt, srq->db.dma, &srq->msrq);
 	if (err)
 		goto err_wrid;
 
@@ -201,7 +192,7 @@ struct ib_srq *mlx4_ib_create_srq(struct ib_pd *pd,
 
 	init_attr->attr.max_wr = srq->msrq.max - 1;
 
-	return &srq->ibsrq;
+	return 0;
 
 err_wrid:
 	if (udata)
@@ -222,10 +213,7 @@ err_db:
 	if (!udata)
 		mlx4_db_free(dev->dev, &srq->db);
 
-err_srq:
-	kfree(srq);
-
-	return ERR_PTR(err);
+	return err;
 }
 
 int mlx4_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
@@ -272,7 +260,7 @@ int mlx4_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
 	return 0;
 }
 
-int mlx4_ib_destroy_srq(struct ib_srq *srq)
+void mlx4_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata)
 {
 	struct mlx4_ib_dev *dev = to_mdev(srq->device);
 	struct mlx4_ib_srq *msrq = to_msrq(srq);
@@ -280,8 +268,13 @@ int mlx4_ib_destroy_srq(struct ib_srq *srq)
 	mlx4_srq_free(dev->dev, &msrq->msrq);
 	mlx4_mtt_cleanup(dev->dev, &msrq->mtt);
 
-	if (srq->uobject) {
-		mlx4_ib_db_unmap_user(to_mucontext(srq->uobject->context), &msrq->db);
+	if (udata) {
+		mlx4_ib_db_unmap_user(
+			rdma_udata_to_drv_context(
+				udata,
+				struct mlx4_ib_ucontext,
+				ibucontext),
+			&msrq->db);
 		ib_umem_release(msrq->umem);
 	} else {
 		kvfree(msrq->wrid);
@@ -289,10 +282,6 @@ int mlx4_ib_destroy_srq(struct ib_srq *srq)
 			      &msrq->buf);
 		mlx4_db_free(dev->dev, &msrq->db);
 	}
-
-	kfree(msrq);
-
-	return 0;
 }
 
 void mlx4_ib_free_srq_wqe(struct mlx4_ib_srq *srq, int wqe_index)
diff --git a/drivers/infiniband/hw/mlx5/ah.c b/drivers/infiniband/hw/mlx5/ah.c
index 420ae0897333..80642dd359bc 100644
--- a/drivers/infiniband/hw/mlx5/ah.c
+++ b/drivers/infiniband/hw/mlx5/ah.c
@@ -32,9 +32,8 @@
 
 #include "mlx5_ib.h"
 
-static struct ib_ah *create_ib_ah(struct mlx5_ib_dev *dev,
-				  struct mlx5_ib_ah *ah,
-				  struct rdma_ah_attr *ah_attr)
+static void create_ib_ah(struct mlx5_ib_dev *dev, struct mlx5_ib_ah *ah,
+			 struct rdma_ah_attr *ah_attr)
 {
 	enum ib_gid_type gid_type;
 
@@ -67,21 +66,19 @@ static struct ib_ah *create_ib_ah(struct mlx5_ib_dev *dev,
 		ah->av.fl_mlid = rdma_ah_get_path_bits(ah_attr) & 0x7f;
 		ah->av.stat_rate_sl |= (rdma_ah_get_sl(ah_attr) & 0xf);
 	}
-
-	return &ah->ibah;
 }
 
-struct ib_ah *mlx5_ib_create_ah(struct ib_pd *pd, struct rdma_ah_attr *ah_attr,
-				u32 flags, struct ib_udata *udata)
+int mlx5_ib_create_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr,
+		      u32 flags, struct ib_udata *udata)
 
 {
-	struct mlx5_ib_ah *ah;
-	struct mlx5_ib_dev *dev = to_mdev(pd->device);
+	struct mlx5_ib_ah *ah = to_mah(ibah);
+	struct mlx5_ib_dev *dev = to_mdev(ibah->device);
 	enum rdma_ah_attr_type ah_type = ah_attr->type;
 
 	if ((ah_type == RDMA_AH_ATTR_TYPE_ROCE) &&
 	    !(rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH))
-		return ERR_PTR(-EINVAL);
+		return -EINVAL;
 
 	if (ah_type == RDMA_AH_ATTR_TYPE_ROCE && udata) {
 		int err;
@@ -90,21 +87,18 @@ struct ib_ah *mlx5_ib_create_ah(struct ib_pd *pd, struct rdma_ah_attr *ah_attr,
 				   sizeof(resp.dmac);
 
 		if (udata->outlen < min_resp_len)
-			return ERR_PTR(-EINVAL);
+			return -EINVAL;
 
 		resp.response_length = min_resp_len;
 
 		memcpy(resp.dmac, ah_attr->roce.dmac, ETH_ALEN);
 		err = ib_copy_to_udata(udata, &resp, resp.response_length);
 		if (err)
-			return ERR_PTR(err);
+			return err;
 	}
 
-	ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
-	if (!ah)
-		return ERR_PTR(-ENOMEM);
-
-	return create_ib_ah(dev, ah, ah_attr); /* never fails */
+	create_ib_ah(dev, ah, ah_attr);
+	return 0;
 }
 
 int mlx5_ib_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr)
@@ -131,8 +125,7 @@ int mlx5_ib_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr)
 	return 0;
 }
 
-int mlx5_ib_destroy_ah(struct ib_ah *ah, u32 flags)
+void mlx5_ib_destroy_ah(struct ib_ah *ah, u32 flags)
 {
-	kfree(to_mah(ah));
-	return 0;
+	return;
 }
diff --git a/drivers/infiniband/hw/mlx5/cmd.c b/drivers/infiniband/hw/mlx5/cmd.c
index be95ac5aeb30..e3ec79b8f7f5 100644
--- a/drivers/infiniband/hw/mlx5/cmd.c
+++ b/drivers/infiniband/hw/mlx5/cmd.c
@@ -82,10 +82,10 @@ int mlx5_cmd_modify_cong_params(struct mlx5_core_dev *dev,
 	return mlx5_cmd_exec(dev, in, in_size, out, sizeof(out));
 }
 
-int mlx5_cmd_alloc_memic(struct mlx5_memic *memic, phys_addr_t *addr,
-			  u64 length, u32 alignment)
+int mlx5_cmd_alloc_memic(struct mlx5_dm *dm, phys_addr_t *addr,
+			 u64 length, u32 alignment)
 {
-	struct mlx5_core_dev *dev = memic->dev;
+	struct mlx5_core_dev *dev = dm->dev;
 	u64 num_memic_hw_pages = MLX5_CAP_DEV_MEM(dev, memic_bar_size)
 					>> PAGE_SHIFT;
 	u64 hw_start_addr = MLX5_CAP64_DEV_MEM(dev, memic_bar_start_addr);
@@ -115,17 +115,17 @@ int mlx5_cmd_alloc_memic(struct mlx5_memic *memic, phys_addr_t *addr,
 		 mlx5_alignment);
 
 	while (page_idx < num_memic_hw_pages) {
-		spin_lock(&memic->memic_lock);
-		page_idx = bitmap_find_next_zero_area(memic->memic_alloc_pages,
+		spin_lock(&dm->lock);
+		page_idx = bitmap_find_next_zero_area(dm->memic_alloc_pages,
 						      num_memic_hw_pages,
 						      page_idx,
 						      num_pages, 0);
 
 		if (page_idx < num_memic_hw_pages)
-			bitmap_set(memic->memic_alloc_pages,
+			bitmap_set(dm->memic_alloc_pages,
 				   page_idx, num_pages);
 
-		spin_unlock(&memic->memic_lock);
+		spin_unlock(&dm->lock);
 
 		if (page_idx >= num_memic_hw_pages)
 			break;
@@ -135,10 +135,10 @@ int mlx5_cmd_alloc_memic(struct mlx5_memic *memic, phys_addr_t *addr,
 
 		ret = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 		if (ret) {
-			spin_lock(&memic->memic_lock);
-			bitmap_clear(memic->memic_alloc_pages,
+			spin_lock(&dm->lock);
+			bitmap_clear(dm->memic_alloc_pages,
 				     page_idx, num_pages);
-			spin_unlock(&memic->memic_lock);
+			spin_unlock(&dm->lock);
 
 			if (ret == -EAGAIN) {
 				page_idx++;
@@ -157,9 +157,9 @@ int mlx5_cmd_alloc_memic(struct mlx5_memic *memic, phys_addr_t *addr,
 	return -ENOMEM;
 }
 
-int mlx5_cmd_dealloc_memic(struct mlx5_memic *memic, u64 addr, u64 length)
+int mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr, u64 length)
 {
-	struct mlx5_core_dev *dev = memic->dev;
+	struct mlx5_core_dev *dev = dm->dev;
 	u64 hw_start_addr = MLX5_CAP64_DEV_MEM(dev, memic_bar_start_addr);
 	u32 num_pages = DIV_ROUND_UP(length, PAGE_SIZE);
 	u32 out[MLX5_ST_SZ_DW(dealloc_memic_out)] = {0};
@@ -177,15 +177,140 @@ int mlx5_cmd_dealloc_memic(struct mlx5_memic *memic, u64 addr, u64 length)
 	err =  mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
 
 	if (!err) {
-		spin_lock(&memic->memic_lock);
-		bitmap_clear(memic->memic_alloc_pages,
+		spin_lock(&dm->lock);
+		bitmap_clear(dm->memic_alloc_pages,
 			     start_page_idx, num_pages);
-		spin_unlock(&memic->memic_lock);
+		spin_unlock(&dm->lock);
 	}
 
 	return err;
 }
 
+int mlx5_cmd_alloc_sw_icm(struct mlx5_dm *dm, int type, u64 length,
+			  u16 uid, phys_addr_t *addr, u32 *obj_id)
+{
+	struct mlx5_core_dev *dev = dm->dev;
+	u32 num_blocks = DIV_ROUND_UP(length, MLX5_SW_ICM_BLOCK_SIZE(dev));
+	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {};
+	u32 in[MLX5_ST_SZ_DW(create_sw_icm_in)] = {};
+	unsigned long *block_map;
+	u64 icm_start_addr;
+	u32 log_icm_size;
+	u32 max_blocks;
+	u64 block_idx;
+	void *sw_icm;
+	int ret;
+
+	MLX5_SET(general_obj_in_cmd_hdr, in, opcode,
+		 MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
+	MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_OBJ_TYPE_SW_ICM);
+	MLX5_SET(general_obj_in_cmd_hdr, in, uid, uid);
+
+	switch (type) {
+	case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:
+		icm_start_addr = MLX5_CAP64_DEV_MEM(dev,
+						steering_sw_icm_start_address);
+		log_icm_size = MLX5_CAP_DEV_MEM(dev, log_steering_sw_icm_size);
+		block_map = dm->steering_sw_icm_alloc_blocks;
+		break;
+	case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:
+		icm_start_addr = MLX5_CAP64_DEV_MEM(dev,
+					header_modify_sw_icm_start_address);
+		log_icm_size = MLX5_CAP_DEV_MEM(dev,
+						log_header_modify_sw_icm_size);
+		block_map = dm->header_modify_sw_icm_alloc_blocks;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	max_blocks = BIT(log_icm_size - MLX5_LOG_SW_ICM_BLOCK_SIZE(dev));
+	spin_lock(&dm->lock);
+	block_idx = bitmap_find_next_zero_area(block_map,
+					       max_blocks,
+					       0,
+					       num_blocks, 0);
+
+	if (block_idx < max_blocks)
+		bitmap_set(block_map,
+			   block_idx, num_blocks);
+
+	spin_unlock(&dm->lock);
+
+	if (block_idx >= max_blocks)
+		return -ENOMEM;
+
+	sw_icm = MLX5_ADDR_OF(create_sw_icm_in, in, sw_icm);
+	icm_start_addr += block_idx << MLX5_LOG_SW_ICM_BLOCK_SIZE(dev);
+	MLX5_SET64(sw_icm, sw_icm, sw_icm_start_addr,
+		   icm_start_addr);
+	MLX5_SET(sw_icm, sw_icm, log_sw_icm_size, ilog2(length));
+
+	ret = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+	if (ret) {
+		spin_lock(&dm->lock);
+		bitmap_clear(block_map,
+			     block_idx, num_blocks);
+		spin_unlock(&dm->lock);
+
+		return ret;
+	}
+
+	*addr = icm_start_addr;
+	*obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
+
+	return 0;
+}
+
+int mlx5_cmd_dealloc_sw_icm(struct mlx5_dm *dm, int type, u64 length,
+			    u16 uid, phys_addr_t addr, u32 obj_id)
+{
+	struct mlx5_core_dev *dev = dm->dev;
+	u32 num_blocks = DIV_ROUND_UP(length, MLX5_SW_ICM_BLOCK_SIZE(dev));
+	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {};
+	u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {};
+	unsigned long *block_map;
+	u64 start_idx;
+	int err;
+
+	switch (type) {
+	case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:
+		start_idx =
+			(addr - MLX5_CAP64_DEV_MEM(
+					dev, steering_sw_icm_start_address)) >>
+			MLX5_LOG_SW_ICM_BLOCK_SIZE(dev);
+		block_map = dm->steering_sw_icm_alloc_blocks;
+		break;
+	case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:
+		start_idx =
+			(addr -
+			 MLX5_CAP64_DEV_MEM(
+				 dev, header_modify_sw_icm_start_address)) >>
+			MLX5_LOG_SW_ICM_BLOCK_SIZE(dev);
+		block_map = dm->header_modify_sw_icm_alloc_blocks;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	MLX5_SET(general_obj_in_cmd_hdr, in, opcode,
+		 MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
+	MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_OBJ_TYPE_SW_ICM);
+	MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, obj_id);
+	MLX5_SET(general_obj_in_cmd_hdr, in, uid, uid);
+
+	err =  mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+	if (err)
+		return err;
+
+	spin_lock(&dm->lock);
+	bitmap_clear(block_map,
+		     start_idx, num_blocks);
+	spin_unlock(&dm->lock);
+
+	return 0;
+}
+
 int mlx5_cmd_query_ext_ppcnt_counters(struct mlx5_core_dev *dev, void *out)
 {
 	u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {};
diff --git a/drivers/infiniband/hw/mlx5/cmd.h b/drivers/infiniband/hw/mlx5/cmd.h
index 923a7b93f507..0572dcba6eae 100644
--- a/drivers/infiniband/hw/mlx5/cmd.h
+++ b/drivers/infiniband/hw/mlx5/cmd.h
@@ -44,9 +44,9 @@ int mlx5_cmd_query_cong_params(struct mlx5_core_dev *dev, int cong_point,
 int mlx5_cmd_query_ext_ppcnt_counters(struct mlx5_core_dev *dev, void *out);
 int mlx5_cmd_modify_cong_params(struct mlx5_core_dev *mdev,
 				void *in, int in_size);
-int mlx5_cmd_alloc_memic(struct mlx5_memic *memic, phys_addr_t *addr,
+int mlx5_cmd_alloc_memic(struct mlx5_dm *dm, phys_addr_t *addr,
 			 u64 length, u32 alignment);
-int mlx5_cmd_dealloc_memic(struct mlx5_memic *memic, u64 addr, u64 length);
+int mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr, u64 length);
 void mlx5_cmd_dealloc_pd(struct mlx5_core_dev *dev, u32 pdn, u16 uid);
 void mlx5_cmd_destroy_tir(struct mlx5_core_dev *dev, u32 tirn, u16 uid);
 void mlx5_cmd_destroy_tis(struct mlx5_core_dev *dev, u32 tisn, u16 uid);
@@ -65,4 +65,8 @@ int mlx5_cmd_alloc_q_counter(struct mlx5_core_dev *dev, u16 *counter_id,
 			     u16 uid);
 int mlx5_cmd_mad_ifc(struct mlx5_core_dev *dev, const void *inb, void *outb,
 		     u16 opmod, u8 port);
+int mlx5_cmd_alloc_sw_icm(struct mlx5_dm *dm, int type, u64 length,
+			  u16 uid, phys_addr_t *addr, u32 *obj_id);
+int mlx5_cmd_dealloc_sw_icm(struct mlx5_dm *dm, int type, u64 length,
+			    u16 uid, phys_addr_t addr, u32 obj_id);
 #endif /* MLX5_IB_CMD_H */
diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c
index 18704e503508..2e2e65f00257 100644
--- a/drivers/infiniband/hw/mlx5/cq.c
+++ b/drivers/infiniband/hw/mlx5/cq.c
@@ -679,8 +679,7 @@ static int mini_cqe_res_format_to_hw(struct mlx5_ib_dev *dev, u8 format)
 }
 
 static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,
-			  struct ib_ucontext *context, struct mlx5_ib_cq *cq,
-			  int entries, u32 **cqb,
+			  struct mlx5_ib_cq *cq, int entries, u32 **cqb,
 			  int *cqe_size, int *index, int *inlen)
 {
 	struct mlx5_ib_create_cq ucmd = {};
@@ -691,6 +690,8 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,
 	int ncont;
 	void *cqc;
 	int err;
+	struct mlx5_ib_ucontext *context = rdma_udata_to_drv_context(
+		udata, struct mlx5_ib_ucontext, ibucontext);
 
 	ucmdlen = udata->inlen < sizeof(ucmd) ?
 		  (sizeof(ucmd) - sizeof(ucmd.flags)) : sizeof(ucmd);
@@ -715,8 +716,7 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,
 		return err;
 	}
 
-	err = mlx5_ib_db_map_user(to_mucontext(context), udata, ucmd.db_addr,
-				  &cq->db);
+	err = mlx5_ib_db_map_user(context, udata, ucmd.db_addr, &cq->db);
 	if (err)
 		goto err_umem;
 
@@ -740,7 +740,7 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,
 	MLX5_SET(cqc, cqc, log_page_size,
 		 page_shift - MLX5_ADAPTER_PAGE_SHIFT);
 
-	*index = to_mucontext(context)->bfregi.sys_pages[0];
+	*index = context->bfregi.sys_pages[0];
 
 	if (ucmd.cqe_comp_en == 1) {
 		int mini_cqe_format;
@@ -782,23 +782,26 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,
 		cq->private_flags |= MLX5_IB_CQ_PR_FLAGS_CQE_128_PAD;
 	}
 
-	MLX5_SET(create_cq_in, *cqb, uid, to_mucontext(context)->devx_uid);
+	MLX5_SET(create_cq_in, *cqb, uid, context->devx_uid);
 	return 0;
 
 err_cqb:
 	kvfree(*cqb);
 
 err_db:
-	mlx5_ib_db_unmap_user(to_mucontext(context), &cq->db);
+	mlx5_ib_db_unmap_user(context, &cq->db);
 
 err_umem:
 	ib_umem_release(cq->buf.umem);
 	return err;
 }
 
-static void destroy_cq_user(struct mlx5_ib_cq *cq, struct ib_ucontext *context)
+static void destroy_cq_user(struct mlx5_ib_cq *cq, struct ib_udata *udata)
 {
-	mlx5_ib_db_unmap_user(to_mucontext(context), &cq->db);
+	struct mlx5_ib_ucontext *context = rdma_udata_to_drv_context(
+		udata, struct mlx5_ib_ucontext, ibucontext);
+
+	mlx5_ib_db_unmap_user(context, &cq->db);
 	ib_umem_release(cq->buf.umem);
 }
 
@@ -883,7 +886,6 @@ static void notify_soft_wc_handler(struct work_struct *work)
 
 struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
 				const struct ib_cq_init_attr *attr,
-				struct ib_ucontext *context,
 				struct ib_udata *udata)
 {
 	int entries = attr->cqe;
@@ -923,9 +925,9 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
 	INIT_LIST_HEAD(&cq->list_send_qp);
 	INIT_LIST_HEAD(&cq->list_recv_qp);
 
-	if (context) {
-		err = create_cq_user(dev, udata, context, cq, entries,
-				     &cqb, &cqe_size, &index, &inlen);
+	if (udata) {
+		err = create_cq_user(dev, udata, cq, entries, &cqb, &cqe_size,
+				     &index, &inlen);
 		if (err)
 			goto err_create;
 	} else {
@@ -962,7 +964,7 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
 
 	mlx5_ib_dbg(dev, "cqn 0x%x\n", cq->mcq.cqn);
 	cq->mcq.irqn = irqn;
-	if (context)
+	if (udata)
 		cq->mcq.tasklet_ctx.comp = mlx5_ib_cq_comp;
 	else
 		cq->mcq.comp  = mlx5_ib_cq_comp;
@@ -970,7 +972,7 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
 
 	INIT_LIST_HEAD(&cq->wc_list);
 
-	if (context)
+	if (udata)
 		if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof(__u32))) {
 			err = -EFAULT;
 			goto err_cmd;
@@ -985,8 +987,8 @@ err_cmd:
 
 err_cqb:
 	kvfree(cqb);
-	if (context)
-		destroy_cq_user(cq, context);
+	if (udata)
+		destroy_cq_user(cq, udata);
 	else
 		destroy_cq_kernel(dev, cq);
 
@@ -996,19 +998,14 @@ err_create:
 	return ERR_PTR(err);
 }
 
-
-int mlx5_ib_destroy_cq(struct ib_cq *cq)
+int mlx5_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata)
 {
 	struct mlx5_ib_dev *dev = to_mdev(cq->device);
 	struct mlx5_ib_cq *mcq = to_mcq(cq);
-	struct ib_ucontext *context = NULL;
-
-	if (cq->uobject)
-		context = cq->uobject->context;
 
 	mlx5_core_destroy_cq(dev->mdev, &mcq->mcq);
-	if (context)
-		destroy_cq_user(mcq, context);
+	if (udata)
+		destroy_cq_user(mcq, udata);
 	else
 		destroy_cq_kernel(dev, mcq);
 
diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c
index 9e08df7914aa..169ffffcf5ed 100644
--- a/drivers/infiniband/hw/mlx5/devx.c
+++ b/drivers/infiniband/hw/mlx5/devx.c
@@ -85,6 +85,10 @@ int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user)
 	if (is_user && capable(CAP_NET_RAW) &&
 	    (MLX5_CAP_GEN(dev->mdev, uctx_cap) & MLX5_UCTX_CAP_RAW_TX))
 		cap |= MLX5_UCTX_CAP_RAW_TX;
+	if (is_user && capable(CAP_SYS_RAWIO) &&
+	    (MLX5_CAP_GEN(dev->mdev, uctx_cap) &
+	     MLX5_UCTX_CAP_INTERNAL_DEV_RES))
+		cap |= MLX5_UCTX_CAP_INTERNAL_DEV_RES;
 
 	MLX5_SET(create_uctx_in, in, opcode, MLX5_CMD_OP_CREATE_UCTX);
 	MLX5_SET(uctx, uctx, cap, cap);
@@ -373,8 +377,10 @@ static u64 devx_get_obj_id(const void *in)
 	return obj_id;
 }
 
-static bool devx_is_valid_obj_id(struct ib_uobject *uobj, const void *in)
+static bool devx_is_valid_obj_id(struct uverbs_attr_bundle *attrs,
+				 struct ib_uobject *uobj, const void *in)
 {
+	struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata);
 	u64 obj_id = devx_get_obj_id(in);
 
 	if (!obj_id)
@@ -389,7 +395,6 @@ static bool devx_is_valid_obj_id(struct ib_uobject *uobj, const void *in)
 	case UVERBS_OBJECT_SRQ:
 	{
 		struct mlx5_core_srq *srq = &(to_msrq(uobj->object)->msrq);
-		struct mlx5_ib_dev *dev = to_mdev(uobj->context->device);
 		u16 opcode;
 
 		switch (srq->common.res) {
@@ -681,6 +686,7 @@ static bool devx_is_whitelist_cmd(void *in)
 	switch (opcode) {
 	case MLX5_CMD_OP_QUERY_HCA_CAP:
 	case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:
+	case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT:
 		return true;
 	default:
 		return false;
@@ -718,6 +724,7 @@ static bool devx_is_general_cmd(void *in)
 	switch (opcode) {
 	case MLX5_CMD_OP_QUERY_HCA_CAP:
 	case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:
+	case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT:
 	case MLX5_CMD_OP_QUERY_VPORT_STATE:
 	case MLX5_CMD_OP_QUERY_ADAPTER:
 	case MLX5_CMD_OP_QUERY_ISSI:
@@ -1117,7 +1124,8 @@ static void devx_cleanup_mkey(struct devx_obj *obj)
 }
 
 static int devx_obj_cleanup(struct ib_uobject *uobject,
-			    enum rdma_remove_reason why)
+			    enum rdma_remove_reason why,
+			    struct uverbs_attr_bundle *attrs)
 {
 	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
 	struct devx_obj *obj = uobject->object;
@@ -1135,7 +1143,8 @@ static int devx_obj_cleanup(struct ib_uobject *uobject,
 		return ret;
 
 	if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) {
-		struct mlx5_ib_dev *dev = to_mdev(uobject->context->device);
+		struct mlx5_ib_dev *dev =
+			mlx5_udata_to_mdev(&attrs->driver_udata);
 
 		call_srcu(&dev->mr_srcu, &obj->devx_mr.rcu,
 			  devx_free_indirect_mkey);
@@ -1260,7 +1269,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_MODIFY)(
 	if (!devx_is_obj_modify_cmd(cmd_in))
 		return -EINVAL;
 
-	if (!devx_is_valid_obj_id(uobj, cmd_in))
+	if (!devx_is_valid_obj_id(attrs, uobj, cmd_in))
 		return -EINVAL;
 
 	cmd_out = uverbs_zalloc(attrs, cmd_out_len);
@@ -1302,7 +1311,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_QUERY)(
 	if (!devx_is_obj_query_cmd(cmd_in))
 		return -EINVAL;
 
-	if (!devx_is_valid_obj_id(uobj, cmd_in))
+	if (!devx_is_valid_obj_id(attrs, uobj, cmd_in))
 		return -EINVAL;
 
 	cmd_out = uverbs_zalloc(attrs, cmd_out_len);
@@ -1350,7 +1359,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC)(
 
 	struct ib_uobject *uobj = uverbs_attr_get_uobject(
 		attrs, MLX5_IB_ATTR_DEVX_ASYNC_CMD_FD_ALLOC_HANDLE);
-	struct mlx5_ib_dev *mdev = to_mdev(uobj->context->device);
+	struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata);
 
 	ev_file = container_of(uobj, struct devx_async_cmd_event_file,
 			       uobj);
@@ -1412,7 +1421,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY)(
 	if (err)
 		return err;
 
-	if (!devx_is_valid_obj_id(uobj, cmd_in))
+	if (!devx_is_valid_obj_id(attrs, uobj, cmd_in))
 		return -EINVAL;
 
 	fd_uobj = uverbs_attr_get_uobject(attrs,
@@ -1599,7 +1608,8 @@ err_obj_free:
 }
 
 static int devx_umem_cleanup(struct ib_uobject *uobject,
-			     enum rdma_remove_reason why)
+			     enum rdma_remove_reason why,
+			     struct uverbs_attr_bundle *attrs)
 {
 	struct devx_umem *obj = uobject->object;
 	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
@@ -1704,7 +1714,7 @@ static __poll_t devx_async_cmd_event_poll(struct file *filp,
 	return pollflags;
 }
 
-const struct file_operations devx_async_cmd_event_fops = {
+static const struct file_operations devx_async_cmd_event_fops = {
 	.owner	 = THIS_MODULE,
 	.read	 = devx_async_cmd_event_read,
 	.poll    = devx_async_cmd_event_poll,
@@ -1900,7 +1910,7 @@ static bool devx_is_supported(struct ib_device *device)
 {
 	struct mlx5_ib_dev *dev = to_mdev(device);
 
-	return !dev->rep && MLX5_CAP_GEN(dev->mdev, log_max_uctx);
+	return MLX5_CAP_GEN(dev->mdev, log_max_uctx);
 }
 
 const struct uapi_definition mlx5_ib_devx_defs[] = {
diff --git a/drivers/infiniband/hw/mlx5/flow.c b/drivers/infiniband/hw/mlx5/flow.c
index 798591a18484..1fc302d41a53 100644
--- a/drivers/infiniband/hw/mlx5/flow.c
+++ b/drivers/infiniband/hw/mlx5/flow.c
@@ -29,6 +29,9 @@ mlx5_ib_ft_type_to_namespace(enum mlx5_ib_uapi_flow_table_type table_type,
 	case MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX:
 		*namespace = MLX5_FLOW_NAMESPACE_EGRESS;
 		break;
+	case MLX5_IB_UAPI_FLOW_TABLE_TYPE_FDB:
+		*namespace = MLX5_FLOW_NAMESPACE_FDB;
+		break;
 	default:
 		return -EINVAL;
 	}
@@ -75,7 +78,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_CREATE_FLOW)(
 	struct ib_qp *qp = NULL;
 	struct ib_uobject *uobj =
 		uverbs_attr_get_uobject(attrs, MLX5_IB_ATTR_CREATE_FLOW_HANDLE);
-	struct mlx5_ib_dev *dev = to_mdev(uobj->context->device);
+	struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata);
 	int len, ret, i;
 	u32 counter_id = 0;
 
@@ -93,6 +96,10 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_CREATE_FLOW)(
 	    ((dest_devx && dest_qp) || (!dest_devx && !dest_qp)))
 		return -EINVAL;
 
+	/* Allow only DEVX object as dest when inserting to FDB */
+	if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB && !dest_devx)
+		return -EINVAL;
+
 	if (dest_devx) {
 		devx_obj = uverbs_attr_get_obj(
 			attrs, MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX);
@@ -104,6 +111,10 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_CREATE_FLOW)(
 		 */
 		if (!mlx5_ib_devx_is_flow_dest(devx_obj, &dest_id, &dest_type))
 			return -EINVAL;
+		/* Allow only flow table as dest when inserting to FDB */
+		if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB &&
+		    dest_type != MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE)
+			return -EINVAL;
 	} else if (dest_qp) {
 		struct mlx5_ib_qp *mqp;
 
@@ -189,7 +200,8 @@ err_out:
 }
 
 static int flow_matcher_cleanup(struct ib_uobject *uobject,
-				enum rdma_remove_reason why)
+				enum rdma_remove_reason why,
+				struct uverbs_attr_bundle *attrs)
 {
 	struct mlx5_ib_flow_matcher *obj = uobject->object;
 	int ret;
@@ -202,21 +214,67 @@ static int flow_matcher_cleanup(struct ib_uobject *uobject,
 	return 0;
 }
 
+static int mlx5_ib_matcher_ns(struct uverbs_attr_bundle *attrs,
+			      struct mlx5_ib_flow_matcher *obj)
+{
+	enum mlx5_ib_uapi_flow_table_type ft_type =
+		MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX;
+	u32 flags;
+	int err;
+
+	/* New users should use MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE and older
+	 * users should switch to it. We leave this to not break userspace
+	 */
+	if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE) &&
+	    uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS))
+		return -EINVAL;
+
+	if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE)) {
+		err = uverbs_get_const(&ft_type, attrs,
+				       MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE);
+		if (err)
+			return err;
+
+		err = mlx5_ib_ft_type_to_namespace(ft_type, &obj->ns_type);
+		if (err)
+			return err;
+
+		return 0;
+	}
+
+	if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS)) {
+		err = uverbs_get_flags32(&flags, attrs,
+					 MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS,
+					 IB_FLOW_ATTR_FLAGS_EGRESS);
+		if (err)
+			return err;
+
+		if (flags) {
+			mlx5_ib_ft_type_to_namespace(
+				MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX,
+				&obj->ns_type);
+			return 0;
+		}
+	}
+
+	obj->ns_type = MLX5_FLOW_NAMESPACE_BYPASS;
+
+	return 0;
+}
+
 static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_MATCHER_CREATE)(
 	struct uverbs_attr_bundle *attrs)
 {
 	struct ib_uobject *uobj = uverbs_attr_get_uobject(
 		attrs, MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE);
-	struct mlx5_ib_dev *dev = to_mdev(uobj->context->device);
+	struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata);
 	struct mlx5_ib_flow_matcher *obj;
-	u32 flags;
 	int err;
 
 	obj = kzalloc(sizeof(struct mlx5_ib_flow_matcher), GFP_KERNEL);
 	if (!obj)
 		return -ENOMEM;
 
-	obj->ns_type = MLX5_FLOW_NAMESPACE_BYPASS;
 	obj->mask_len = uverbs_attr_get_len(
 		attrs, MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK);
 	err = uverbs_copy_from(&obj->matcher_mask,
@@ -242,19 +300,10 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_MATCHER_CREATE)(
 	if (err)
 		goto end;
 
-	err = uverbs_get_flags32(&flags, attrs,
-				 MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS,
-				 IB_FLOW_ATTR_FLAGS_EGRESS);
+	err = mlx5_ib_matcher_ns(attrs, obj);
 	if (err)
 		goto end;
 
-	if (flags) {
-		err = mlx5_ib_ft_type_to_namespace(
-			MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX, &obj->ns_type);
-		if (err)
-			goto end;
-	}
-
 	uobj->object = obj;
 	obj->mdev = dev->mdev;
 	atomic_set(&obj->usecnt, 0);
@@ -326,7 +375,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER)(
 {
 	struct ib_uobject *uobj = uverbs_attr_get_uobject(
 		attrs, MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE);
-	struct mlx5_ib_dev *mdev = to_mdev(uobj->context->device);
+	struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata);
 	enum mlx5_ib_uapi_flow_table_type ft_type;
 	struct ib_flow_action *action;
 	int num_actions;
@@ -353,7 +402,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER)(
 	if (IS_ERR(action))
 		return PTR_ERR(action);
 
-	uverbs_flow_action_fill_action(action, uobj, uobj->context->device,
+	uverbs_flow_action_fill_action(action, uobj, &mdev->ib_dev,
 				       IB_FLOW_ACTION_UNSPECIFIED);
 
 	return 0;
@@ -445,7 +494,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)(
 {
 	struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,
 		MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE);
-	struct mlx5_ib_dev *mdev = to_mdev(uobj->context->device);
+	struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata);
 	enum mlx5_ib_uapi_flow_action_packet_reformat_type dv_prt;
 	enum mlx5_ib_uapi_flow_table_type ft_type;
 	struct mlx5_ib_flow_action *maction;
@@ -493,8 +542,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT)(
 			goto free_maction;
 	}
 
-	uverbs_flow_action_fill_action(&maction->ib_action, uobj,
-				       uobj->context->device,
+	uverbs_flow_action_fill_action(&maction->ib_action, uobj, &mdev->ib_dev,
 				       IB_FLOW_ACTION_UNSPECIFIED);
 	return 0;
 
@@ -605,6 +653,9 @@ DECLARE_UVERBS_NAMED_METHOD(
 			   UA_MANDATORY),
 	UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS,
 			     enum ib_flow_flags,
+			     UA_OPTIONAL),
+	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE,
+			     enum mlx5_ib_uapi_flow_table_type,
 			     UA_OPTIONAL));
 
 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
@@ -619,15 +670,9 @@ DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_FLOW_MATCHER,
 			    &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_CREATE),
 			    &UVERBS_METHOD(MLX5_IB_METHOD_FLOW_MATCHER_DESTROY));
 
-static bool flow_is_supported(struct ib_device *device)
-{
-	return !to_mdev(device)->rep;
-}
-
 const struct uapi_definition mlx5_ib_flow_defs[] = {
 	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
-		MLX5_IB_OBJECT_FLOW_MATCHER,
-		UAPI_DEF_IS_OBJ_SUPPORTED(flow_is_supported)),
+		MLX5_IB_OBJECT_FLOW_MATCHER),
 	UAPI_DEF_CHAIN_OBJ_TREE(
 		UVERBS_OBJECT_FLOW,
 		&mlx5_ib_fs),
diff --git a/drivers/infiniband/hw/mlx5/ib_rep.c b/drivers/infiniband/hw/mlx5/ib_rep.c
index b8639ac71336..cbcc40d776b9 100644
--- a/drivers/infiniband/hw/mlx5/ib_rep.c
+++ b/drivers/infiniband/hw/mlx5/ib_rep.c
@@ -7,69 +7,59 @@
 #include "ib_rep.h"
 #include "srq.h"
 
-static const struct mlx5_ib_profile vf_rep_profile = {
-	STAGE_CREATE(MLX5_IB_STAGE_INIT,
-		     mlx5_ib_stage_init_init,
-		     mlx5_ib_stage_init_cleanup),
-	STAGE_CREATE(MLX5_IB_STAGE_FLOW_DB,
-		     mlx5_ib_stage_rep_flow_db_init,
-		     NULL),
-	STAGE_CREATE(MLX5_IB_STAGE_CAPS,
-		     mlx5_ib_stage_caps_init,
-		     NULL),
-	STAGE_CREATE(MLX5_IB_STAGE_NON_DEFAULT_CB,
-		     mlx5_ib_stage_rep_non_default_cb,
-		     NULL),
-	STAGE_CREATE(MLX5_IB_STAGE_ROCE,
-		     mlx5_ib_stage_rep_roce_init,
-		     mlx5_ib_stage_rep_roce_cleanup),
-	STAGE_CREATE(MLX5_IB_STAGE_SRQ,
-		     mlx5_init_srq_table,
-		     mlx5_cleanup_srq_table),
-	STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES,
-		     mlx5_ib_stage_dev_res_init,
-		     mlx5_ib_stage_dev_res_cleanup),
-	STAGE_CREATE(MLX5_IB_STAGE_COUNTERS,
-		     mlx5_ib_stage_counters_init,
-		     mlx5_ib_stage_counters_cleanup),
-	STAGE_CREATE(MLX5_IB_STAGE_BFREG,
-		     mlx5_ib_stage_bfrag_init,
-		     mlx5_ib_stage_bfrag_cleanup),
-	STAGE_CREATE(MLX5_IB_STAGE_PRE_IB_REG_UMR,
-		     NULL,
-		     mlx5_ib_stage_pre_ib_reg_umr_cleanup),
-	STAGE_CREATE(MLX5_IB_STAGE_IB_REG,
-		     mlx5_ib_stage_ib_reg_init,
-		     mlx5_ib_stage_ib_reg_cleanup),
-	STAGE_CREATE(MLX5_IB_STAGE_POST_IB_REG_UMR,
-		     mlx5_ib_stage_post_ib_reg_umr_init,
-		     NULL),
-};
+static int
+mlx5_ib_set_vport_rep(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep)
+{
+	struct mlx5_ib_dev *ibdev;
+	int vport_index;
+
+	ibdev = mlx5_ib_get_uplink_ibdev(dev->priv.eswitch);
+	vport_index = ibdev->free_port++;
+
+	ibdev->port[vport_index].rep = rep;
+	write_lock(&ibdev->port[vport_index].roce.netdev_lock);
+	ibdev->port[vport_index].roce.netdev =
+		mlx5_ib_get_rep_netdev(dev->priv.eswitch, rep->vport);
+	write_unlock(&ibdev->port[vport_index].roce.netdev_lock);
+
+	return 0;
+}
 
 static int
 mlx5_ib_vport_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep)
 {
+	int num_ports = MLX5_TOTAL_VPORTS(dev);
 	const struct mlx5_ib_profile *profile;
 	struct mlx5_ib_dev *ibdev;
+	int vport_index;
 
 	if (rep->vport == MLX5_VPORT_UPLINK)
 		profile = &uplink_rep_profile;
 	else
-		profile = &vf_rep_profile;
+		return mlx5_ib_set_vport_rep(dev, rep);
 
 	ibdev = ib_alloc_device(mlx5_ib_dev, ib_dev);
 	if (!ibdev)
 		return -ENOMEM;
 
-	ibdev->rep = rep;
-	ibdev->mdev = dev;
-	ibdev->num_ports = max(MLX5_CAP_GEN(dev, num_ports),
-			       MLX5_CAP_GEN(dev, num_vhca_ports));
-	if (!__mlx5_ib_add(ibdev, profile)) {
+	ibdev->port = kcalloc(num_ports, sizeof(*ibdev->port),
+			      GFP_KERNEL);
+	if (!ibdev->port) {
 		ib_dealloc_device(&ibdev->ib_dev);
-		return -EINVAL;
+		return -ENOMEM;
 	}
 
+	ibdev->is_rep = true;
+	vport_index = ibdev->free_port++;
+	ibdev->port[vport_index].rep = rep;
+	ibdev->port[vport_index].roce.netdev =
+		mlx5_ib_get_rep_netdev(dev->priv.eswitch, rep->vport);
+	ibdev->mdev = dev;
+	ibdev->num_ports = num_ports;
+
+	if (!__mlx5_ib_add(ibdev, profile))
+		return -EINVAL;
+
 	rep->rep_if[REP_IB].priv = ibdev;
 
 	return 0;
@@ -80,13 +70,13 @@ mlx5_ib_vport_rep_unload(struct mlx5_eswitch_rep *rep)
 {
 	struct mlx5_ib_dev *dev;
 
-	if (!rep->rep_if[REP_IB].priv)
+	if (!rep->rep_if[REP_IB].priv ||
+	    rep->vport != MLX5_VPORT_UPLINK)
 		return;
 
 	dev = mlx5_ib_rep_to_dev(rep);
 	__mlx5_ib_remove(dev, dev->profile, MLX5_IB_STAGE_MAX);
 	rep->rep_if[REP_IB].priv = NULL;
-	ib_dealloc_device(&dev->ib_dev);
 }
 
 static void *mlx5_ib_vport_get_proto_dev(struct mlx5_eswitch_rep *rep)
@@ -140,22 +130,21 @@ struct mlx5_eswitch_rep *mlx5_ib_vport_rep(struct mlx5_eswitch *esw, int vport)
 	return mlx5_eswitch_vport_rep(esw, vport);
 }
 
-int create_flow_rule_vport_sq(struct mlx5_ib_dev *dev,
-			      struct mlx5_ib_sq *sq)
+struct mlx5_flow_handle *create_flow_rule_vport_sq(struct mlx5_ib_dev *dev,
+						   struct mlx5_ib_sq *sq,
+						   u16 port)
 {
-	struct mlx5_flow_handle *flow_rule;
 	struct mlx5_eswitch *esw = dev->mdev->priv.eswitch;
+	struct mlx5_eswitch_rep *rep;
 
-	if (!dev->rep)
-		return 0;
+	if (!dev->is_rep || !port)
+		return NULL;
 
-	flow_rule =
-		mlx5_eswitch_add_send_to_vport_rule(esw,
-						    dev->rep->vport,
-						    sq->base.mqp.qpn);
-	if (IS_ERR(flow_rule))
-		return PTR_ERR(flow_rule);
-	sq->flow_rule = flow_rule;
+	if (!dev->port[port - 1].rep)
+		return ERR_PTR(-EINVAL);
 
-	return 0;
+	rep = dev->port[port - 1].rep;
+
+	return mlx5_eswitch_add_send_to_vport_rule(esw, rep->vport,
+						   sq->base.mqp.qpn);
 }
diff --git a/drivers/infiniband/hw/mlx5/ib_rep.h b/drivers/infiniband/hw/mlx5/ib_rep.h
index 798d41e61fb4..1d9778da8a50 100644
--- a/drivers/infiniband/hw/mlx5/ib_rep.h
+++ b/drivers/infiniband/hw/mlx5/ib_rep.h
@@ -20,8 +20,9 @@ struct mlx5_eswitch_rep *mlx5_ib_vport_rep(struct mlx5_eswitch *esw,
 					   int vport_index);
 void mlx5_ib_register_vport_reps(struct mlx5_core_dev *mdev);
 void mlx5_ib_unregister_vport_reps(struct mlx5_core_dev *mdev);
-int create_flow_rule_vport_sq(struct mlx5_ib_dev *dev,
-			      struct mlx5_ib_sq *sq);
+struct mlx5_flow_handle *create_flow_rule_vport_sq(struct mlx5_ib_dev *dev,
+						   struct mlx5_ib_sq *sq,
+						   u16 port);
 struct net_device *mlx5_ib_get_rep_netdev(struct mlx5_eswitch *esw,
 					  int vport_index);
 #else /* CONFIG_MLX5_ESWITCH */
@@ -52,10 +53,12 @@ struct mlx5_eswitch_rep *mlx5_ib_vport_rep(struct mlx5_eswitch *esw,
 
 static inline void mlx5_ib_register_vport_reps(struct mlx5_core_dev *mdev) {}
 static inline void mlx5_ib_unregister_vport_reps(struct mlx5_core_dev *mdev) {}
-static inline int create_flow_rule_vport_sq(struct mlx5_ib_dev *dev,
-					    struct mlx5_ib_sq *sq)
+static inline
+struct mlx5_flow_handle *create_flow_rule_vport_sq(struct mlx5_ib_dev *dev,
+						   struct mlx5_ib_sq *sq,
+						   u16 port)
 {
-	return 0;
+	return NULL;
 }
 
 static inline
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 1aaa2056d188..abac70ad5c7c 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -156,6 +156,34 @@ static int get_port_state(struct ib_device *ibdev,
 	return ret;
 }
 
+static struct mlx5_roce *mlx5_get_rep_roce(struct mlx5_ib_dev *dev,
+					   struct net_device *ndev,
+					   u8 *port_num)
+{
+	struct mlx5_eswitch *esw = dev->mdev->priv.eswitch;
+	struct net_device *rep_ndev;
+	struct mlx5_ib_port *port;
+	int i;
+
+	for (i = 0; i < dev->num_ports; i++) {
+		port  = &dev->port[i];
+		if (!port->rep)
+			continue;
+
+		read_lock(&port->roce.netdev_lock);
+		rep_ndev = mlx5_ib_get_rep_netdev(esw,
+						  port->rep->vport);
+		if (rep_ndev == ndev) {
+			read_unlock(&port->roce.netdev_lock);
+			*port_num = i + 1;
+			return &port->roce;
+		}
+		read_unlock(&port->roce.netdev_lock);
+	}
+
+	return NULL;
+}
+
 static int mlx5_netdev_event(struct notifier_block *this,
 			     unsigned long event, void *ptr)
 {
@@ -172,22 +200,17 @@ static int mlx5_netdev_event(struct notifier_block *this,
 
 	switch (event) {
 	case NETDEV_REGISTER:
+		/* Should already be registered during the load */
+		if (ibdev->is_rep)
+			break;
 		write_lock(&roce->netdev_lock);
-		if (ibdev->rep) {
-			struct mlx5_eswitch *esw = ibdev->mdev->priv.eswitch;
-			struct net_device *rep_ndev;
-
-			rep_ndev = mlx5_ib_get_rep_netdev(esw,
-							  ibdev->rep->vport);
-			if (rep_ndev == ndev)
-				roce->netdev = ndev;
-		} else if (ndev->dev.parent == mdev->device) {
+		if (ndev->dev.parent == mdev->device)
 			roce->netdev = ndev;
-		}
 		write_unlock(&roce->netdev_lock);
 		break;
 
 	case NETDEV_UNREGISTER:
+		/* In case of reps, ib device goes away before the netdevs */
 		write_lock(&roce->netdev_lock);
 		if (roce->netdev == ndev)
 			roce->netdev = NULL;
@@ -205,6 +228,10 @@ static int mlx5_netdev_event(struct notifier_block *this,
 			dev_put(lag_ndev);
 		}
 
+		if (ibdev->is_rep)
+			roce = mlx5_get_rep_roce(ibdev, ndev, &port_num);
+		if (!roce)
+			return NOTIFY_DONE;
 		if ((upper == ndev || (!upper && ndev == roce->netdev))
 		    && ibdev->ib_active) {
 			struct ib_event ibev = { };
@@ -257,11 +284,11 @@ static struct net_device *mlx5_ib_get_netdev(struct ib_device *device,
 
 	/* Ensure ndev does not disappear before we invoke dev_hold()
 	 */
-	read_lock(&ibdev->roce[port_num - 1].netdev_lock);
-	ndev = ibdev->roce[port_num - 1].netdev;
+	read_lock(&ibdev->port[port_num - 1].roce.netdev_lock);
+	ndev = ibdev->port[port_num - 1].roce.netdev;
 	if (ndev)
 		dev_hold(ndev);
-	read_unlock(&ibdev->roce[port_num - 1].netdev_lock);
+	read_unlock(&ibdev->port[port_num - 1].roce.netdev_lock);
 
 out:
 	mlx5_ib_put_native_port_mdev(ibdev, port_num);
@@ -479,9 +506,14 @@ static int mlx5_query_port_roce(struct ib_device *device, u8 port_num,
 
 	/* Possible bad flows are checked before filling out props so in case
 	 * of an error it will still be zeroed out.
+	 * Use native port in case of reps
 	 */
-	err = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN,
-				   mdev_port_num);
+	if (dev->is_rep)
+		err = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN,
+					   1);
+	else
+		err = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN,
+					   mdev_port_num);
 	if (err)
 		goto out;
 	ext = MLX5_CAP_PCAM_FEATURE(dev->mdev, ptys_extended_ethernet);
@@ -542,52 +574,22 @@ out:
 	return err;
 }
 
-struct mlx5_ib_vlan_info {
-	u16 vlan_id;
-	bool vlan;
-};
-
-static int get_lower_dev_vlan(struct net_device *lower_dev, void *data)
-{
-	struct mlx5_ib_vlan_info *vlan_info = data;
-
-	if (is_vlan_dev(lower_dev)) {
-		vlan_info->vlan = true;
-		vlan_info->vlan_id = vlan_dev_vlan_id(lower_dev);
-	}
-	/* We are interested only in first level vlan device, so
-	 * always return 1 to stop iterating over next level devices.
-	 */
-	return 1;
-}
-
 static int set_roce_addr(struct mlx5_ib_dev *dev, u8 port_num,
 			 unsigned int index, const union ib_gid *gid,
 			 const struct ib_gid_attr *attr)
 {
 	enum ib_gid_type gid_type = IB_GID_TYPE_IB;
-	struct mlx5_ib_vlan_info vlan_info = { };
+	u16 vlan_id = 0xffff;
 	u8 roce_version = 0;
 	u8 roce_l3_type = 0;
 	u8 mac[ETH_ALEN];
+	int ret;
 
 	if (gid) {
 		gid_type = attr->gid_type;
-		ether_addr_copy(mac, attr->ndev->dev_addr);
-
-		if (is_vlan_dev(attr->ndev)) {
-			vlan_info.vlan = true;
-			vlan_info.vlan_id = vlan_dev_vlan_id(attr->ndev);
-		} else {
-			/* If the netdev is upper device and if it's lower
-			 * lower device is vlan device, consider vlan id of
-			 * the lower vlan device for this gid entry.
-			 */
-			rcu_read_lock();
-			netdev_walk_all_lower_dev_rcu(attr->ndev,
-					get_lower_dev_vlan, &vlan_info);
-			rcu_read_unlock();
-		}
+		ret = rdma_read_gid_l2_fields(attr, &vlan_id, &mac[0]);
+		if (ret)
+			return ret;
 	}
 
 	switch (gid_type) {
@@ -608,7 +610,7 @@ static int set_roce_addr(struct mlx5_ib_dev *dev, u8 port_num,
 
 	return mlx5_core_roce_gid_set(dev->mdev, index, roce_version,
 				      roce_l3_type, gid->raw, mac,
-				      vlan_info.vlan, vlan_info.vlan_id,
+				      vlan_id < VLAN_CFI_MASK, vlan_id,
 				      port_num);
 }
 
@@ -1407,7 +1409,9 @@ static int mlx5_ib_rep_query_port(struct ib_device *ibdev, u8 port,
 {
 	int ret;
 
-	/* Only link layer == ethernet is valid for representors */
+	/* Only link layer == ethernet is valid for representors
+	 * and we always use port 1
+	 */
 	ret = mlx5_query_port_roce(ibdev, port, props);
 	if (ret || !props)
 		return ret;
@@ -1954,11 +1958,11 @@ static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx,
 	print_lib_caps(dev, context->lib_caps);
 
 	if (dev->lag_active) {
-		u8 port = mlx5_core_native_port_num(dev->mdev);
+		u8 port = mlx5_core_native_port_num(dev->mdev) - 1;
 
 		atomic_set(&context->tx_port_affinity,
 			   atomic_add_return(
-				   1, &dev->roce[port].tx_port_affinity));
+				   1, &dev->port[port].roce.tx_port_affinity));
 	}
 
 	return 0;
@@ -2060,21 +2064,22 @@ static int mlx5_ib_mmap_clock_info_page(struct mlx5_ib_dev *dev,
 					struct vm_area_struct *vma,
 					struct mlx5_ib_ucontext *context)
 {
-	if (vma->vm_end - vma->vm_start != PAGE_SIZE)
+	if ((vma->vm_end - vma->vm_start != PAGE_SIZE) ||
+	    !(vma->vm_flags & VM_SHARED))
 		return -EINVAL;
 
 	if (get_index(vma->vm_pgoff) != MLX5_IB_CLOCK_INFO_V1)
 		return -EOPNOTSUPP;
 
-	if (vma->vm_flags & VM_WRITE)
+	if (vma->vm_flags & (VM_WRITE | VM_EXEC))
 		return -EPERM;
 	vma->vm_flags &= ~VM_MAYWRITE;
 
-	if (!dev->mdev->clock_info_page)
+	if (!dev->mdev->clock_info)
 		return -EOPNOTSUPP;
 
-	return rdma_user_mmap_page(&context->ibucontext, vma,
-				   dev->mdev->clock_info_page, PAGE_SIZE);
+	return vm_insert_page(vma, vma->vm_start,
+			      virt_to_page(dev->mdev->clock_info));
 }
 
 static int uar_mmap(struct mlx5_ib_dev *dev, enum mlx5_ib_mmap_cmd cmd,
@@ -2259,89 +2264,200 @@ static int mlx5_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vm
 	return 0;
 }
 
-struct ib_dm *mlx5_ib_alloc_dm(struct ib_device *ibdev,
-			       struct ib_ucontext *context,
-			       struct ib_dm_alloc_attr *attr,
-			       struct uverbs_attr_bundle *attrs)
+static inline int check_dm_type_support(struct mlx5_ib_dev *dev,
+					u32 type)
 {
-	u64 act_size = roundup(attr->length, MLX5_MEMIC_BASE_SIZE);
-	struct mlx5_memic *memic = &to_mdev(ibdev)->memic;
-	phys_addr_t memic_addr;
-	struct mlx5_ib_dm *dm;
+	switch (type) {
+	case MLX5_IB_UAPI_DM_TYPE_MEMIC:
+		if (!MLX5_CAP_DEV_MEM(dev->mdev, memic))
+			return -EOPNOTSUPP;
+		break;
+	case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:
+		if (!capable(CAP_SYS_RAWIO) ||
+		    !capable(CAP_NET_RAW))
+			return -EPERM;
+
+		if (!(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner) ||
+		      MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, sw_owner)))
+			return -EOPNOTSUPP;
+		break;
+	}
+
+	return 0;
+}
+
+static int handle_alloc_dm_memic(struct ib_ucontext *ctx,
+				 struct mlx5_ib_dm *dm,
+				 struct ib_dm_alloc_attr *attr,
+				 struct uverbs_attr_bundle *attrs)
+{
+	struct mlx5_dm *dm_db = &to_mdev(ctx->device)->dm;
 	u64 start_offset;
 	u32 page_idx;
 	int err;
 
-	dm = kzalloc(sizeof(*dm), GFP_KERNEL);
-	if (!dm)
-		return ERR_PTR(-ENOMEM);
-
-	mlx5_ib_dbg(to_mdev(ibdev), "alloc_memic req: user_length=0x%llx act_length=0x%llx log_alignment=%d\n",
-		    attr->length, act_size, attr->alignment);
+	dm->size = roundup(attr->length, MLX5_MEMIC_BASE_SIZE);
 
-	err = mlx5_cmd_alloc_memic(memic, &memic_addr,
-				   act_size, attr->alignment);
+	err = mlx5_cmd_alloc_memic(dm_db, &dm->dev_addr,
+				   dm->size, attr->alignment);
 	if (err)
-		goto err_free;
+		return err;
 
-	start_offset = memic_addr & ~PAGE_MASK;
-	page_idx = (memic_addr - memic->dev->bar_addr -
-		    MLX5_CAP64_DEV_MEM(memic->dev, memic_bar_start_addr)) >>
+	page_idx = (dm->dev_addr - pci_resource_start(dm_db->dev->pdev, 0) -
+		    MLX5_CAP64_DEV_MEM(dm_db->dev, memic_bar_start_addr)) >>
 		    PAGE_SHIFT;
 
 	err = uverbs_copy_to(attrs,
+			     MLX5_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX,
+			     &page_idx, sizeof(page_idx));
+	if (err)
+		goto err_dealloc;
+
+	start_offset = dm->dev_addr & ~PAGE_MASK;
+	err = uverbs_copy_to(attrs,
 			     MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET,
 			     &start_offset, sizeof(start_offset));
 	if (err)
 		goto err_dealloc;
 
+	bitmap_set(to_mucontext(ctx)->dm_pages, page_idx,
+		   DIV_ROUND_UP(dm->size, PAGE_SIZE));
+
+	return 0;
+
+err_dealloc:
+	mlx5_cmd_dealloc_memic(dm_db, dm->dev_addr, dm->size);
+
+	return err;
+}
+
+static int handle_alloc_dm_sw_icm(struct ib_ucontext *ctx,
+				  struct mlx5_ib_dm *dm,
+				  struct ib_dm_alloc_attr *attr,
+				  struct uverbs_attr_bundle *attrs,
+				  int type)
+{
+	struct mlx5_dm *dm_db = &to_mdev(ctx->device)->dm;
+	u64 act_size;
+	int err;
+
+	/* Allocation size must a multiple of the basic block size
+	 * and a power of 2.
+	 */
+	act_size = roundup(attr->length, MLX5_SW_ICM_BLOCK_SIZE(dm_db->dev));
+	act_size = roundup_pow_of_two(act_size);
+
+	dm->size = act_size;
+	err = mlx5_cmd_alloc_sw_icm(dm_db, type, act_size,
+				    to_mucontext(ctx)->devx_uid, &dm->dev_addr,
+				    &dm->icm_dm.obj_id);
+	if (err)
+		return err;
+
 	err = uverbs_copy_to(attrs,
-			     MLX5_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX,
-			     &page_idx, sizeof(page_idx));
+			     MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET,
+			     &dm->dev_addr, sizeof(dm->dev_addr));
 	if (err)
-		goto err_dealloc;
+		mlx5_cmd_dealloc_sw_icm(dm_db, type, dm->size,
+					to_mucontext(ctx)->devx_uid,
+					dm->dev_addr, dm->icm_dm.obj_id);
+
+	return err;
+}
+
+struct ib_dm *mlx5_ib_alloc_dm(struct ib_device *ibdev,
+			       struct ib_ucontext *context,
+			       struct ib_dm_alloc_attr *attr,
+			       struct uverbs_attr_bundle *attrs)
+{
+	struct mlx5_ib_dm *dm;
+	enum mlx5_ib_uapi_dm_type type;
+	int err;
 
-	bitmap_set(to_mucontext(context)->dm_pages, page_idx,
-		   DIV_ROUND_UP(act_size, PAGE_SIZE));
+	err = uverbs_get_const_default(&type, attrs,
+				       MLX5_IB_ATTR_ALLOC_DM_REQ_TYPE,
+				       MLX5_IB_UAPI_DM_TYPE_MEMIC);
+	if (err)
+		return ERR_PTR(err);
 
-	dm->dev_addr = memic_addr;
+	mlx5_ib_dbg(to_mdev(ibdev), "alloc_dm req: dm_type=%d user_length=0x%llx log_alignment=%d\n",
+		    type, attr->length, attr->alignment);
+
+	err = check_dm_type_support(to_mdev(ibdev), type);
+	if (err)
+		return ERR_PTR(err);
+
+	dm = kzalloc(sizeof(*dm), GFP_KERNEL);
+	if (!dm)
+		return ERR_PTR(-ENOMEM);
+
+	dm->type = type;
+
+	switch (type) {
+	case MLX5_IB_UAPI_DM_TYPE_MEMIC:
+		err = handle_alloc_dm_memic(context, dm,
+					    attr,
+					    attrs);
+		break;
+	case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:
+	case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:
+		err = handle_alloc_dm_sw_icm(context, dm, attr, attrs, type);
+		break;
+	default:
+		err = -EOPNOTSUPP;
+	}
+
+	if (err)
+		goto err_free;
 
 	return &dm->ibdm;
 
-err_dealloc:
-	mlx5_cmd_dealloc_memic(memic, memic_addr,
-			       act_size);
 err_free:
 	kfree(dm);
 	return ERR_PTR(err);
 }
 
-int mlx5_ib_dealloc_dm(struct ib_dm *ibdm)
+int mlx5_ib_dealloc_dm(struct ib_dm *ibdm, struct uverbs_attr_bundle *attrs)
 {
-	struct mlx5_memic *memic = &to_mdev(ibdm->device)->memic;
+	struct mlx5_ib_ucontext *ctx = rdma_udata_to_drv_context(
+		&attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
+	struct mlx5_dm *dm_db = &to_mdev(ibdm->device)->dm;
 	struct mlx5_ib_dm *dm = to_mdm(ibdm);
-	u64 act_size = roundup(dm->ibdm.length, MLX5_MEMIC_BASE_SIZE);
 	u32 page_idx;
 	int ret;
 
-	ret = mlx5_cmd_dealloc_memic(memic, dm->dev_addr, act_size);
-	if (ret)
-		return ret;
+	switch (dm->type) {
+	case MLX5_IB_UAPI_DM_TYPE_MEMIC:
+		ret = mlx5_cmd_dealloc_memic(dm_db, dm->dev_addr, dm->size);
+		if (ret)
+			return ret;
 
-	page_idx = (dm->dev_addr - memic->dev->bar_addr -
-		    MLX5_CAP64_DEV_MEM(memic->dev, memic_bar_start_addr)) >>
-		    PAGE_SHIFT;
-	bitmap_clear(to_mucontext(ibdm->uobject->context)->dm_pages,
-		     page_idx,
-		     DIV_ROUND_UP(act_size, PAGE_SIZE));
+		page_idx = (dm->dev_addr -
+			    pci_resource_start(dm_db->dev->pdev, 0) -
+			    MLX5_CAP64_DEV_MEM(dm_db->dev,
+					       memic_bar_start_addr)) >>
+			   PAGE_SHIFT;
+		bitmap_clear(ctx->dm_pages, page_idx,
+			     DIV_ROUND_UP(dm->size, PAGE_SIZE));
+		break;
+	case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:
+	case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:
+		ret = mlx5_cmd_dealloc_sw_icm(dm_db, dm->type, dm->size,
+					      ctx->devx_uid, dm->dev_addr,
+					      dm->icm_dm.obj_id);
+		if (ret)
+			return ret;
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
 
 	kfree(dm);
 
 	return 0;
 }
 
-static int mlx5_ib_alloc_pd(struct ib_pd *ibpd, struct ib_ucontext *context,
-			    struct ib_udata *udata)
+static int mlx5_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 {
 	struct mlx5_ib_pd *pd = to_mpd(ibpd);
 	struct ib_device *ibdev = ibpd->device;
@@ -2350,8 +2466,10 @@ static int mlx5_ib_alloc_pd(struct ib_pd *ibpd, struct ib_ucontext *context,
 	u32 out[MLX5_ST_SZ_DW(alloc_pd_out)] = {};
 	u32 in[MLX5_ST_SZ_DW(alloc_pd_in)]   = {};
 	u16 uid = 0;
+	struct mlx5_ib_ucontext *context = rdma_udata_to_drv_context(
+		udata, struct mlx5_ib_ucontext, ibucontext);
 
-	uid = context ? to_mucontext(context)->devx_uid : 0;
+	uid = context ? context->devx_uid : 0;
 	MLX5_SET(alloc_pd_in, in, opcode, MLX5_CMD_OP_ALLOC_PD);
 	MLX5_SET(alloc_pd_in, in, uid, uid);
 	err = mlx5_cmd_exec(to_mdev(ibdev)->mdev, in, sizeof(in),
@@ -2361,7 +2479,7 @@ static int mlx5_ib_alloc_pd(struct ib_pd *ibpd, struct ib_ucontext *context,
 
 	pd->pdn = MLX5_GET(alloc_pd_out, out, pd);
 	pd->uid = uid;
-	if (context) {
+	if (udata) {
 		resp.pdn = pd->pdn;
 		if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
 			mlx5_cmd_dealloc_pd(to_mdev(ibdev)->mdev, pd->pdn, uid);
@@ -2372,7 +2490,7 @@ static int mlx5_ib_alloc_pd(struct ib_pd *ibpd, struct ib_ucontext *context,
 	return 0;
 }
 
-static void mlx5_ib_dealloc_pd(struct ib_pd *pd)
+static void mlx5_ib_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)
 {
 	struct mlx5_ib_dev *mdev = to_mdev(pd->device);
 	struct mlx5_ib_pd *mpd = to_mpd(pd);
@@ -3151,10 +3269,10 @@ static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev,
 		if (ft_type == MLX5_IB_FT_RX) {
 			fn_type = MLX5_FLOW_NAMESPACE_BYPASS;
 			prio = &dev->flow_db->prios[priority];
-			if (!dev->rep &&
+			if (!dev->is_rep &&
 			    MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap))
 				flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP;
-			if (!dev->rep &&
+			if (!dev->is_rep &&
 			    MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
 					reformat_l3_tunnel_to_l2))
 				flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
@@ -3164,7 +3282,7 @@ static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev,
 							      log_max_ft_size));
 			fn_type = MLX5_FLOW_NAMESPACE_EGRESS;
 			prio = &dev->flow_db->egress_prios[priority];
-			if (!dev->rep &&
+			if (!dev->is_rep &&
 			    MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat))
 				flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
 		}
@@ -3197,12 +3315,11 @@ static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev,
 	if (!ns)
 		return ERR_PTR(-ENOTSUPP);
 
-	if (num_entries > max_table_size)
-		return ERR_PTR(-ENOMEM);
+	max_table_size = min_t(int, num_entries, max_table_size);
 
 	ft = prio->flow_table;
 	if (!ft)
-		return _get_prio(ns, prio, priority, num_entries, num_groups,
+		return _get_prio(ns, prio, priority, max_table_size, num_groups,
 				 flags);
 
 	return prio;
@@ -3370,7 +3487,7 @@ static struct mlx5_ib_flow_handler *_create_flow_rule(struct mlx5_ib_dev *dev,
 	if (!is_valid_attr(dev->mdev, flow_attr))
 		return ERR_PTR(-EINVAL);
 
-	if (dev->rep && is_egress)
+	if (dev->is_rep && is_egress)
 		return ERR_PTR(-EINVAL);
 
 	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
@@ -3401,13 +3518,17 @@ static struct mlx5_ib_flow_handler *_create_flow_rule(struct mlx5_ib_dev *dev,
 	if (!flow_is_multicast_only(flow_attr))
 		set_underlay_qp(dev, spec, underlay_qpn);
 
-	if (dev->rep) {
+	if (dev->is_rep) {
 		void *misc;
 
+		if (!dev->port[flow_attr->port - 1].rep) {
+			err = -EINVAL;
+			goto free;
+		}
 		misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
 				    misc_parameters);
 		MLX5_SET(fte_match_set_misc, misc, source_port,
-			 dev->rep->vport);
+			 dev->port[flow_attr->port - 1].rep->vport);
 		misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
 				    misc_parameters);
 		MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
@@ -3769,11 +3890,16 @@ _get_flow_table(struct mlx5_ib_dev *dev,
 		bool mcast)
 {
 	struct mlx5_flow_namespace *ns = NULL;
-	struct mlx5_ib_flow_prio *prio;
-	int max_table_size;
+	struct mlx5_ib_flow_prio *prio = NULL;
+	int max_table_size = 0;
 	u32 flags = 0;
 	int priority;
 
+	if (mcast)
+		priority = MLX5_IB_FLOW_MCAST_PRIO;
+	else
+		priority = ib_prio_to_core_prio(fs_matcher->priority, false);
+
 	if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) {
 		max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
 					log_max_ft_size));
@@ -3782,20 +3908,18 @@ _get_flow_table(struct mlx5_ib_dev *dev,
 		if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev,
 					      reformat_l3_tunnel_to_l2))
 			flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
-	} else { /* Can only be MLX5_FLOW_NAMESPACE_EGRESS */
-		max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev,
-					log_max_ft_size));
+	} else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS) {
+		max_table_size = BIT(
+			MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, log_max_ft_size));
 		if (MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat))
 			flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
+	} else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB) {
+		max_table_size = BIT(
+			MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, log_max_ft_size));
+		priority = FDB_BYPASS_PATH;
 	}
 
-	if (max_table_size < MLX5_FS_MAX_ENTRIES)
-		return ERR_PTR(-ENOMEM);
-
-	if (mcast)
-		priority = MLX5_IB_FLOW_MCAST_PRIO;
-	else
-		priority = ib_prio_to_core_prio(fs_matcher->priority, false);
+	max_table_size = min_t(int, max_table_size, MLX5_FS_MAX_ENTRIES);
 
 	ns = mlx5_get_flow_namespace(dev->mdev, fs_matcher->ns_type);
 	if (!ns)
@@ -3803,13 +3927,18 @@ _get_flow_table(struct mlx5_ib_dev *dev,
 
 	if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS)
 		prio = &dev->flow_db->prios[priority];
-	else
+	else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS)
 		prio = &dev->flow_db->egress_prios[priority];
+	else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB)
+		prio = &dev->flow_db->fdb;
+
+	if (!prio)
+		return ERR_PTR(-EINVAL);
 
 	if (prio->flow_table)
 		return prio;
 
-	return _get_prio(ns, prio, priority, MLX5_FS_MAX_ENTRIES,
+	return _get_prio(ns, prio, priority, max_table_size,
 			 MLX5_FS_MAX_TYPES, flags);
 }
 
@@ -4509,7 +4638,7 @@ static int set_has_smi_cap(struct mlx5_ib_dev *dev)
 	int err;
 	int port;
 
-	for (port = 1; port <= dev->num_ports; port++) {
+	for (port = 1; port <= ARRAY_SIZE(dev->mdev->port_caps); port++) {
 		dev->mdev->port_caps[port - 1].has_smi = false;
 		if (MLX5_CAP_GEN(dev->mdev, port_type) ==
 		    MLX5_CAP_PORT_TYPE_IB) {
@@ -4540,7 +4669,7 @@ static void get_ext_port_caps(struct mlx5_ib_dev *dev)
 		mlx5_query_ext_port_caps(dev, port);
 }
 
-static int get_port_caps(struct mlx5_ib_dev *dev, u8 port)
+static int __get_port_caps(struct mlx5_ib_dev *dev, u8 port)
 {
 	struct ib_device_attr *dprops = NULL;
 	struct ib_port_attr *pprops = NULL;
@@ -4555,10 +4684,6 @@ static int get_port_caps(struct mlx5_ib_dev *dev, u8 port)
 	if (!dprops)
 		goto out;
 
-	err = set_has_smi_cap(dev);
-	if (err)
-		goto out;
-
 	err = mlx5_ib_query_device(&dev->ib_dev, dprops, &uhw);
 	if (err) {
 		mlx5_ib_warn(dev, "query_device failed %d\n", err);
@@ -4587,6 +4712,16 @@ out:
 	return err;
 }
 
+static int get_port_caps(struct mlx5_ib_dev *dev, u8 port)
+{
+	/* For representors use port 1, is this is the only native
+	 * port
+	 */
+	if (dev->is_rep)
+		return __get_port_caps(dev, 1);
+	return __get_port_caps(dev, port);
+}
+
 static void destroy_umrc_res(struct mlx5_ib_dev *dev)
 {
 	int err;
@@ -4596,7 +4731,7 @@ static void destroy_umrc_res(struct mlx5_ib_dev *dev)
 		mlx5_ib_warn(dev, "mr cache cleanup failed\n");
 
 	if (dev->umrc.qp)
-		mlx5_ib_destroy_qp(dev->umrc.qp);
+		mlx5_ib_destroy_qp(dev->umrc.qp, NULL);
 	if (dev->umrc.cq)
 		ib_free_cq(dev->umrc.cq);
 	if (dev->umrc.pd)
@@ -4701,7 +4836,7 @@ static int create_umr_res(struct mlx5_ib_dev *dev)
 	return 0;
 
 error_4:
-	mlx5_ib_destroy_qp(qp);
+	mlx5_ib_destroy_qp(qp, NULL);
 	dev->umrc.qp = NULL;
 
 error_3:
@@ -4752,11 +4887,11 @@ static int create_dev_resources(struct mlx5_ib_resources *devr)
 	devr->p0->uobject = NULL;
 	atomic_set(&devr->p0->usecnt, 0);
 
-	ret = mlx5_ib_alloc_pd(devr->p0, NULL, NULL);
+	ret = mlx5_ib_alloc_pd(devr->p0, NULL);
 	if (ret)
 		goto error0;
 
-	devr->c0 = mlx5_ib_create_cq(&dev->ib_dev, &cq_attr, NULL, NULL);
+	devr->c0 = mlx5_ib_create_cq(&dev->ib_dev, &cq_attr, NULL);
 	if (IS_ERR(devr->c0)) {
 		ret = PTR_ERR(devr->c0);
 		goto error1;
@@ -4768,7 +4903,7 @@ static int create_dev_resources(struct mlx5_ib_resources *devr)
 	devr->c0->cq_context    = NULL;
 	atomic_set(&devr->c0->usecnt, 0);
 
-	devr->x0 = mlx5_ib_alloc_xrcd(&dev->ib_dev, NULL, NULL);
+	devr->x0 = mlx5_ib_alloc_xrcd(&dev->ib_dev, NULL);
 	if (IS_ERR(devr->x0)) {
 		ret = PTR_ERR(devr->x0);
 		goto error2;
@@ -4779,7 +4914,7 @@ static int create_dev_resources(struct mlx5_ib_resources *devr)
 	mutex_init(&devr->x0->tgt_qp_mutex);
 	INIT_LIST_HEAD(&devr->x0->tgt_qp_list);
 
-	devr->x1 = mlx5_ib_alloc_xrcd(&dev->ib_dev, NULL, NULL);
+	devr->x1 = mlx5_ib_alloc_xrcd(&dev->ib_dev, NULL);
 	if (IS_ERR(devr->x1)) {
 		ret = PTR_ERR(devr->x1);
 		goto error3;
@@ -4797,19 +4932,21 @@ static int create_dev_resources(struct mlx5_ib_resources *devr)
 	attr.ext.cq = devr->c0;
 	attr.ext.xrc.xrcd = devr->x0;
 
-	devr->s0 = mlx5_ib_create_srq(devr->p0, &attr, NULL);
-	if (IS_ERR(devr->s0)) {
-		ret = PTR_ERR(devr->s0);
+	devr->s0 = rdma_zalloc_drv_obj(ibdev, ib_srq);
+	if (!devr->s0) {
+		ret = -ENOMEM;
 		goto error4;
 	}
+
 	devr->s0->device	= &dev->ib_dev;
 	devr->s0->pd		= devr->p0;
-	devr->s0->uobject       = NULL;
-	devr->s0->event_handler = NULL;
-	devr->s0->srq_context   = NULL;
 	devr->s0->srq_type      = IB_SRQT_XRC;
 	devr->s0->ext.xrc.xrcd	= devr->x0;
 	devr->s0->ext.cq	= devr->c0;
+	ret = mlx5_ib_create_srq(devr->s0, &attr, NULL);
+	if (ret)
+		goto err_create;
+
 	atomic_inc(&devr->s0->ext.xrc.xrcd->usecnt);
 	atomic_inc(&devr->s0->ext.cq->usecnt);
 	atomic_inc(&devr->p0->usecnt);
@@ -4819,18 +4956,21 @@ static int create_dev_resources(struct mlx5_ib_resources *devr)
 	attr.attr.max_sge = 1;
 	attr.attr.max_wr = 1;
 	attr.srq_type = IB_SRQT_BASIC;
-	devr->s1 = mlx5_ib_create_srq(devr->p0, &attr, NULL);
-	if (IS_ERR(devr->s1)) {
-		ret = PTR_ERR(devr->s1);
+	devr->s1 = rdma_zalloc_drv_obj(ibdev, ib_srq);
+	if (!devr->s1) {
+		ret = -ENOMEM;
 		goto error5;
 	}
+
 	devr->s1->device	= &dev->ib_dev;
 	devr->s1->pd		= devr->p0;
-	devr->s1->uobject       = NULL;
-	devr->s1->event_handler = NULL;
-	devr->s1->srq_context   = NULL;
 	devr->s1->srq_type      = IB_SRQT_BASIC;
 	devr->s1->ext.cq	= devr->c0;
+
+	ret = mlx5_ib_create_srq(devr->s1, &attr, NULL);
+	if (ret)
+		goto error6;
+
 	atomic_inc(&devr->p0->usecnt);
 	atomic_set(&devr->s1->usecnt, 0);
 
@@ -4842,16 +4982,20 @@ static int create_dev_resources(struct mlx5_ib_resources *devr)
 
 	return 0;
 
+error6:
+	kfree(devr->s1);
 error5:
-	mlx5_ib_destroy_srq(devr->s0);
+	mlx5_ib_destroy_srq(devr->s0, NULL);
+err_create:
+	kfree(devr->s0);
 error4:
-	mlx5_ib_dealloc_xrcd(devr->x1);
+	mlx5_ib_dealloc_xrcd(devr->x1, NULL);
 error3:
-	mlx5_ib_dealloc_xrcd(devr->x0);
+	mlx5_ib_dealloc_xrcd(devr->x0, NULL);
 error2:
-	mlx5_ib_destroy_cq(devr->c0);
+	mlx5_ib_destroy_cq(devr->c0, NULL);
 error1:
-	mlx5_ib_dealloc_pd(devr->p0);
+	mlx5_ib_dealloc_pd(devr->p0, NULL);
 error0:
 	kfree(devr->p0);
 	return ret;
@@ -4859,20 +5003,20 @@ error0:
 
 static void destroy_dev_resources(struct mlx5_ib_resources *devr)
 {
-	struct mlx5_ib_dev *dev =
-		container_of(devr, struct mlx5_ib_dev, devr);
 	int port;
 
-	mlx5_ib_destroy_srq(devr->s1);
-	mlx5_ib_destroy_srq(devr->s0);
-	mlx5_ib_dealloc_xrcd(devr->x0);
-	mlx5_ib_dealloc_xrcd(devr->x1);
-	mlx5_ib_destroy_cq(devr->c0);
-	mlx5_ib_dealloc_pd(devr->p0);
+	mlx5_ib_destroy_srq(devr->s1, NULL);
+	kfree(devr->s1);
+	mlx5_ib_destroy_srq(devr->s0, NULL);
+	kfree(devr->s0);
+	mlx5_ib_dealloc_xrcd(devr->x0, NULL);
+	mlx5_ib_dealloc_xrcd(devr->x1, NULL);
+	mlx5_ib_destroy_cq(devr->c0, NULL);
+	mlx5_ib_dealloc_pd(devr->p0, NULL);
 	kfree(devr->p0);
 
 	/* Make sure no change P_Key work items are still executing */
-	for (port = 0; port < dev->num_ports; ++port)
+	for (port = 0; port < ARRAY_SIZE(devr->ports); ++port)
 		cancel_work_sync(&devr->ports[port].pkey_change_work);
 }
 
@@ -5015,10 +5159,10 @@ static int mlx5_add_netdev_notifier(struct mlx5_ib_dev *dev, u8 port_num)
 {
 	int err;
 
-	dev->roce[port_num].nb.notifier_call = mlx5_netdev_event;
-	err = register_netdevice_notifier(&dev->roce[port_num].nb);
+	dev->port[port_num].roce.nb.notifier_call = mlx5_netdev_event;
+	err = register_netdevice_notifier(&dev->port[port_num].roce.nb);
 	if (err) {
-		dev->roce[port_num].nb.notifier_call = NULL;
+		dev->port[port_num].roce.nb.notifier_call = NULL;
 		return err;
 	}
 
@@ -5027,9 +5171,9 @@ static int mlx5_add_netdev_notifier(struct mlx5_ib_dev *dev, u8 port_num)
 
 static void mlx5_remove_netdev_notifier(struct mlx5_ib_dev *dev, u8 port_num)
 {
-	if (dev->roce[port_num].nb.notifier_call) {
-		unregister_netdevice_notifier(&dev->roce[port_num].nb);
-		dev->roce[port_num].nb.notifier_call = NULL;
+	if (dev->port[port_num].roce.nb.notifier_call) {
+		unregister_netdevice_notifier(&dev->port[port_num].roce.nb);
+		dev->port[port_num].roce.nb.notifier_call = NULL;
 	}
 }
 
@@ -5578,7 +5722,7 @@ static void mlx5_ib_unbind_slave_port(struct mlx5_ib_dev *ibdev,
 		mlx5_ib_err(ibdev, "Failed to unaffiliate port %u\n",
 			    port_num + 1);
 
-	ibdev->roce[port_num].last_port_state = IB_PORT_DOWN;
+	ibdev->port[port_num].roce.last_port_state = IB_PORT_DOWN;
 }
 
 /* The mlx5_ib_multiport_mutex should be held when calling this function */
@@ -5738,7 +5882,10 @@ ADD_UVERBS_ATTRIBUTES_SIMPLE(
 			    UA_MANDATORY),
 	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX,
 			    UVERBS_ATTR_TYPE(u16),
-			    UA_MANDATORY));
+			    UA_OPTIONAL),
+	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_ALLOC_DM_REQ_TYPE,
+			     enum mlx5_ib_uapi_dm_type,
+			     UA_OPTIONAL));
 
 ADD_UVERBS_ATTRIBUTES_SIMPLE(
 	mlx5_ib_flow_action,
@@ -5829,35 +5976,58 @@ static struct ib_counters *mlx5_ib_create_counters(struct ib_device *device,
 	return &mcounters->ibcntrs;
 }
 
-void mlx5_ib_stage_init_cleanup(struct mlx5_ib_dev *dev)
+static void mlx5_ib_stage_init_cleanup(struct mlx5_ib_dev *dev)
 {
+	struct mlx5_core_dev *mdev = dev->mdev;
+
 	mlx5_ib_cleanup_multiport_master(dev);
 	if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) {
 		srcu_barrier(&dev->mr_srcu);
 		cleanup_srcu_struct(&dev->mr_srcu);
 	}
-	kfree(dev->port);
+
+	WARN_ON(!bitmap_empty(dev->dm.memic_alloc_pages, MLX5_MAX_MEMIC_PAGES));
+
+	WARN_ON(dev->dm.steering_sw_icm_alloc_blocks &&
+		!bitmap_empty(
+			dev->dm.steering_sw_icm_alloc_blocks,
+			BIT(MLX5_CAP_DEV_MEM(mdev, log_steering_sw_icm_size) -
+			    MLX5_LOG_SW_ICM_BLOCK_SIZE(mdev))));
+
+	kfree(dev->dm.steering_sw_icm_alloc_blocks);
+
+	WARN_ON(dev->dm.header_modify_sw_icm_alloc_blocks &&
+		!bitmap_empty(dev->dm.header_modify_sw_icm_alloc_blocks,
+			      BIT(MLX5_CAP_DEV_MEM(
+					  mdev, log_header_modify_sw_icm_size) -
+				  MLX5_LOG_SW_ICM_BLOCK_SIZE(mdev))));
+
+	kfree(dev->dm.header_modify_sw_icm_alloc_blocks);
 }
 
-int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev)
+static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev)
 {
 	struct mlx5_core_dev *mdev = dev->mdev;
+	u64 header_modify_icm_blocks = 0;
+	u64 steering_icm_blocks = 0;
 	int err;
 	int i;
 
-	dev->port = kcalloc(dev->num_ports, sizeof(*dev->port),
-			    GFP_KERNEL);
-	if (!dev->port)
-		return -ENOMEM;
-
 	for (i = 0; i < dev->num_ports; i++) {
 		spin_lock_init(&dev->port[i].mp.mpi_lock);
-		rwlock_init(&dev->roce[i].netdev_lock);
+		rwlock_init(&dev->port[i].roce.netdev_lock);
+		dev->port[i].roce.dev = dev;
+		dev->port[i].roce.native_port_num = i + 1;
+		dev->port[i].roce.last_port_state = IB_PORT_DOWN;
 	}
 
 	err = mlx5_ib_init_multiport_master(dev);
 	if (err)
-		goto err_free_port;
+		return err;
+
+	err = set_has_smi_cap(dev);
+	if (err)
+		return err;
 
 	if (!mlx5_core_mp_enabled(mdev)) {
 		for (i = 1; i <= dev->num_ports; i++) {
@@ -5885,22 +6055,54 @@ int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev)
 	INIT_LIST_HEAD(&dev->qp_list);
 	spin_lock_init(&dev->reset_flow_resource_lock);
 
-	spin_lock_init(&dev->memic.memic_lock);
-	dev->memic.dev = mdev;
+	if (MLX5_CAP_GEN_64(mdev, general_obj_types) &
+	    MLX5_GENERAL_OBJ_TYPES_CAP_SW_ICM) {
+		if (MLX5_CAP64_DEV_MEM(mdev, steering_sw_icm_start_address)) {
+			steering_icm_blocks =
+				BIT(MLX5_CAP_DEV_MEM(mdev,
+						     log_steering_sw_icm_size) -
+				    MLX5_LOG_SW_ICM_BLOCK_SIZE(mdev));
+
+			dev->dm.steering_sw_icm_alloc_blocks =
+				kcalloc(BITS_TO_LONGS(steering_icm_blocks),
+					sizeof(unsigned long), GFP_KERNEL);
+			if (!dev->dm.steering_sw_icm_alloc_blocks)
+				goto err_mp;
+		}
+
+		if (MLX5_CAP64_DEV_MEM(mdev,
+				       header_modify_sw_icm_start_address)) {
+			header_modify_icm_blocks = BIT(
+				MLX5_CAP_DEV_MEM(
+					mdev, log_header_modify_sw_icm_size) -
+				MLX5_LOG_SW_ICM_BLOCK_SIZE(mdev));
+
+			dev->dm.header_modify_sw_icm_alloc_blocks =
+				kcalloc(BITS_TO_LONGS(header_modify_icm_blocks),
+					sizeof(unsigned long), GFP_KERNEL);
+			if (!dev->dm.header_modify_sw_icm_alloc_blocks)
+				goto err_dm;
+		}
+	}
+
+	spin_lock_init(&dev->dm.lock);
+	dev->dm.dev = mdev;
 
 	if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) {
 		err = init_srcu_struct(&dev->mr_srcu);
 		if (err)
-			goto err_mp;
+			goto err_dm;
 	}
 
 	return 0;
+
+err_dm:
+	kfree(dev->dm.steering_sw_icm_alloc_blocks);
+	kfree(dev->dm.header_modify_sw_icm_alloc_blocks);
+
 err_mp:
 	mlx5_ib_cleanup_multiport_master(dev);
 
-err_free_port:
-	kfree(dev->port);
-
 	return -ENOMEM;
 }
 
@@ -5916,20 +6118,6 @@ static int mlx5_ib_stage_flow_db_init(struct mlx5_ib_dev *dev)
 	return 0;
 }
 
-int mlx5_ib_stage_rep_flow_db_init(struct mlx5_ib_dev *dev)
-{
-	struct mlx5_ib_dev *nic_dev;
-
-	nic_dev = mlx5_ib_get_uplink_ibdev(dev->mdev->priv.eswitch);
-
-	if (!nic_dev)
-		return -EINVAL;
-
-	dev->flow_db = nic_dev->flow_db;
-
-	return 0;
-}
-
 static void mlx5_ib_stage_flow_db_cleanup(struct mlx5_ib_dev *dev)
 {
 	kfree(dev->flow_db);
@@ -5989,7 +6177,10 @@ static const struct ib_device_ops mlx5_ib_dev_ops = {
 	.req_notify_cq = mlx5_ib_arm_cq,
 	.rereg_user_mr = mlx5_ib_rereg_user_mr,
 	.resize_cq = mlx5_ib_resize_cq,
+
+	INIT_RDMA_OBJ_SIZE(ib_ah, mlx5_ib_ah, ibah),
 	INIT_RDMA_OBJ_SIZE(ib_pd, mlx5_ib_pd, ibpd),
+	INIT_RDMA_OBJ_SIZE(ib_srq, mlx5_ib_srq, ibsrq),
 	INIT_RDMA_OBJ_SIZE(ib_ucontext, mlx5_ib_ucontext, ibucontext),
 };
 
@@ -6025,7 +6216,7 @@ static const struct ib_device_ops mlx5_ib_dev_dm_ops = {
 	.reg_dm_mr = mlx5_ib_reg_dm_mr,
 };
 
-int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev)
+static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev)
 {
 	struct mlx5_core_dev *mdev = dev->mdev;
 	int err;
@@ -6091,7 +6282,9 @@ int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev)
 		ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_xrc_ops);
 	}
 
-	if (MLX5_CAP_DEV_MEM(mdev, memic))
+	if (MLX5_CAP_DEV_MEM(mdev, memic) ||
+	    MLX5_CAP_GEN_64(dev->mdev, general_obj_types) &
+	    MLX5_GENERAL_OBJ_TYPES_CAP_SW_ICM)
 		ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_dm_ops);
 
 	if (mlx5_accel_ipsec_device_caps(dev->mdev) &
@@ -6131,7 +6324,7 @@ static const struct ib_device_ops mlx5_ib_dev_port_rep_ops = {
 	.query_port = mlx5_ib_rep_query_port,
 };
 
-int mlx5_ib_stage_rep_non_default_cb(struct mlx5_ib_dev *dev)
+static int mlx5_ib_stage_rep_non_default_cb(struct mlx5_ib_dev *dev)
 {
 	ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_port_rep_ops);
 	return 0;
@@ -6149,13 +6342,6 @@ static const struct ib_device_ops mlx5_ib_dev_common_roce_ops = {
 static int mlx5_ib_stage_common_roce_init(struct mlx5_ib_dev *dev)
 {
 	u8 port_num;
-	int i;
-
-	for (i = 0; i < dev->num_ports; i++) {
-		dev->roce[i].dev = dev;
-		dev->roce[i].native_port_num = i + 1;
-		dev->roce[i].last_port_state = IB_PORT_DOWN;
-	}
 
 	dev->ib_dev.uverbs_ex_cmd_mask |=
 			(1ull << IB_USER_VERBS_EX_CMD_CREATE_WQ) |
@@ -6167,6 +6353,7 @@ static int mlx5_ib_stage_common_roce_init(struct mlx5_ib_dev *dev)
 
 	port_num = mlx5_core_native_port_num(dev->mdev) - 1;
 
+	/* Register only for native ports */
 	return mlx5_add_netdev_notifier(dev, port_num);
 }
 
@@ -6177,7 +6364,7 @@ static void mlx5_ib_stage_common_roce_cleanup(struct mlx5_ib_dev *dev)
 	mlx5_remove_netdev_notifier(dev, port_num);
 }
 
-int mlx5_ib_stage_rep_roce_init(struct mlx5_ib_dev *dev)
+static int mlx5_ib_stage_rep_roce_init(struct mlx5_ib_dev *dev)
 {
 	struct mlx5_core_dev *mdev = dev->mdev;
 	enum rdma_link_layer ll;
@@ -6193,7 +6380,7 @@ int mlx5_ib_stage_rep_roce_init(struct mlx5_ib_dev *dev)
 	return err;
 }
 
-void mlx5_ib_stage_rep_roce_cleanup(struct mlx5_ib_dev *dev)
+static void mlx5_ib_stage_rep_roce_cleanup(struct mlx5_ib_dev *dev)
 {
 	mlx5_ib_stage_common_roce_cleanup(dev);
 }
@@ -6240,12 +6427,12 @@ static void mlx5_ib_stage_roce_cleanup(struct mlx5_ib_dev *dev)
 	}
 }
 
-int mlx5_ib_stage_dev_res_init(struct mlx5_ib_dev *dev)
+static int mlx5_ib_stage_dev_res_init(struct mlx5_ib_dev *dev)
 {
 	return create_dev_resources(&dev->devr);
 }
 
-void mlx5_ib_stage_dev_res_cleanup(struct mlx5_ib_dev *dev)
+static void mlx5_ib_stage_dev_res_cleanup(struct mlx5_ib_dev *dev)
 {
 	destroy_dev_resources(&dev->devr);
 }
@@ -6267,7 +6454,7 @@ static const struct ib_device_ops mlx5_ib_dev_hw_stats_ops = {
 	.get_hw_stats = mlx5_ib_get_hw_stats,
 };
 
-int mlx5_ib_stage_counters_init(struct mlx5_ib_dev *dev)
+static int mlx5_ib_stage_counters_init(struct mlx5_ib_dev *dev)
 {
 	if (MLX5_CAP_GEN(dev->mdev, max_qp_cnt)) {
 		ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_hw_stats_ops);
@@ -6278,7 +6465,7 @@ int mlx5_ib_stage_counters_init(struct mlx5_ib_dev *dev)
 	return 0;
 }
 
-void mlx5_ib_stage_counters_cleanup(struct mlx5_ib_dev *dev)
+static void mlx5_ib_stage_counters_cleanup(struct mlx5_ib_dev *dev)
 {
 	if (MLX5_CAP_GEN(dev->mdev, max_qp_cnt))
 		mlx5_ib_dealloc_counters(dev);
@@ -6308,7 +6495,7 @@ static void mlx5_ib_stage_uar_cleanup(struct mlx5_ib_dev *dev)
 	mlx5_put_uars_page(dev->mdev, dev->mdev->priv.uar);
 }
 
-int mlx5_ib_stage_bfrag_init(struct mlx5_ib_dev *dev)
+static int mlx5_ib_stage_bfrag_init(struct mlx5_ib_dev *dev)
 {
 	int err;
 
@@ -6323,13 +6510,13 @@ int mlx5_ib_stage_bfrag_init(struct mlx5_ib_dev *dev)
 	return err;
 }
 
-void mlx5_ib_stage_bfrag_cleanup(struct mlx5_ib_dev *dev)
+static void mlx5_ib_stage_bfrag_cleanup(struct mlx5_ib_dev *dev)
 {
 	mlx5_free_bfreg(dev->mdev, &dev->fp_bfreg);
 	mlx5_free_bfreg(dev->mdev, &dev->bfreg);
 }
 
-int mlx5_ib_stage_ib_reg_init(struct mlx5_ib_dev *dev)
+static int mlx5_ib_stage_ib_reg_init(struct mlx5_ib_dev *dev)
 {
 	const char *name;
 
@@ -6341,17 +6528,17 @@ int mlx5_ib_stage_ib_reg_init(struct mlx5_ib_dev *dev)
 	return ib_register_device(&dev->ib_dev, name);
 }
 
-void mlx5_ib_stage_pre_ib_reg_umr_cleanup(struct mlx5_ib_dev *dev)
+static void mlx5_ib_stage_pre_ib_reg_umr_cleanup(struct mlx5_ib_dev *dev)
 {
 	destroy_umrc_res(dev);
 }
 
-void mlx5_ib_stage_ib_reg_cleanup(struct mlx5_ib_dev *dev)
+static void mlx5_ib_stage_ib_reg_cleanup(struct mlx5_ib_dev *dev)
 {
 	ib_unregister_device(&dev->ib_dev);
 }
 
-int mlx5_ib_stage_post_ib_reg_umr_init(struct mlx5_ib_dev *dev)
+static int mlx5_ib_stage_post_ib_reg_umr_init(struct mlx5_ib_dev *dev)
 {
 	return create_umr_res(dev);
 }
@@ -6406,6 +6593,9 @@ void __mlx5_ib_remove(struct mlx5_ib_dev *dev,
 		if (profile->stage[stage].cleanup)
 			profile->stage[stage].cleanup(dev);
 	}
+
+	kfree(dev->port);
+	ib_dealloc_device(&dev->ib_dev);
 }
 
 void *__mlx5_ib_add(struct mlx5_ib_dev *dev,
@@ -6527,6 +6717,9 @@ const struct mlx5_ib_profile uplink_rep_profile = {
 	STAGE_CREATE(MLX5_IB_STAGE_PRE_IB_REG_UMR,
 		     NULL,
 		     mlx5_ib_stage_pre_ib_reg_umr_cleanup),
+	STAGE_CREATE(MLX5_IB_STAGE_WHITELIST_UID,
+		     mlx5_ib_stage_devx_init,
+		     mlx5_ib_stage_devx_cleanup),
 	STAGE_CREATE(MLX5_IB_STAGE_IB_REG,
 		     mlx5_ib_stage_ib_reg_init,
 		     mlx5_ib_stage_ib_reg_cleanup),
@@ -6581,12 +6774,14 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
 	enum rdma_link_layer ll;
 	struct mlx5_ib_dev *dev;
 	int port_type_cap;
+	int num_ports;
 
 	printk_once(KERN_INFO "%s", mlx5_version);
 
 	if (MLX5_ESWITCH_MANAGER(mdev) &&
 	    mlx5_ib_eswitch_mode(mdev->priv.eswitch) == SRIOV_OFFLOADS) {
-		mlx5_ib_register_vport_reps(mdev);
+		if (!mlx5_core_mp_enabled(mdev))
+			mlx5_ib_register_vport_reps(mdev);
 		return mdev;
 	}
 
@@ -6596,13 +6791,20 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
 	if (mlx5_core_is_mp_slave(mdev) && ll == IB_LINK_LAYER_ETHERNET)
 		return mlx5_ib_add_slave_port(mdev);
 
+	num_ports = max(MLX5_CAP_GEN(mdev, num_ports),
+			MLX5_CAP_GEN(mdev, num_vhca_ports));
 	dev = ib_alloc_device(mlx5_ib_dev, ib_dev);
 	if (!dev)
 		return NULL;
+	dev->port = kcalloc(num_ports, sizeof(*dev->port),
+			     GFP_KERNEL);
+	if (!dev->port) {
+		ib_dealloc_device((struct ib_device *)dev);
+		return NULL;
+	}
 
 	dev->mdev = mdev;
-	dev->num_ports = max(MLX5_CAP_GEN(mdev, num_ports),
-			     MLX5_CAP_GEN(mdev, num_vhca_ports));
+	dev->num_ports = num_ports;
 
 	return __mlx5_ib_add(dev, &pf_profile);
 }
@@ -6629,8 +6831,6 @@ static void mlx5_ib_remove(struct mlx5_core_dev *mdev, void *context)
 
 	dev = context;
 	__mlx5_ib_remove(dev, dev->profile, MLX5_IB_STAGE_MAX);
-
-	ib_dealloc_device((struct ib_device *)dev);
 }
 
 static struct mlx5_interface mlx5_ib_interface = {
diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index 4a617d78eae1..40eb8be482e4 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -48,6 +48,7 @@
 #include <rdma/mlx5-abi.h>
 #include <rdma/uverbs_ioctl.h>
 #include <rdma/mlx5_user_ioctl_cmds.h>
+#include <rdma/mlx5_user_ioctl_verbs.h>
 
 #include "srq.h"
 
@@ -117,6 +118,10 @@ enum {
 	MLX5_MEMIC_BASE_SIZE	= 1 << MLX5_MEMIC_BASE_ALIGN,
 };
 
+#define MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)                                        \
+	(MLX5_CAP_DEV_MEM(dev, log_sw_icm_alloc_granularity))
+#define MLX5_SW_ICM_BLOCK_SIZE(dev) (1 << MLX5_LOG_SW_ICM_BLOCK_SIZE(dev))
+
 struct mlx5_ib_ucontext {
 	struct ib_ucontext	ibucontext;
 	struct list_head	db_page_list;
@@ -194,6 +199,7 @@ struct mlx5_ib_flow_db {
 	struct mlx5_ib_flow_prio	egress_prios[MLX5_IB_NUM_FLOW_FT];
 	struct mlx5_ib_flow_prio	sniffer[MLX5_IB_NUM_SNIFFER_FTS];
 	struct mlx5_ib_flow_prio	egress[MLX5_IB_NUM_EGRESS_FTS];
+	struct mlx5_ib_flow_prio	fdb;
 	struct mlx5_flow_table		*lag_demux_ft;
 	/* Protect flow steering bypass flow tables
 	 * when add/del flow rules.
@@ -553,15 +559,28 @@ enum mlx5_ib_mtt_access_flags {
 struct mlx5_ib_dm {
 	struct ib_dm		ibdm;
 	phys_addr_t		dev_addr;
+	u32			type;
+	size_t			size;
+	union {
+		struct {
+			u32	obj_id;
+		} icm_dm;
+		/* other dm types specific params should be added here */
+	};
 };
 
 #define MLX5_IB_MTT_PRESENT (MLX5_IB_MTT_READ | MLX5_IB_MTT_WRITE)
 
-#define MLX5_IB_DM_ALLOWED_ACCESS (IB_ACCESS_LOCAL_WRITE   |\
-				   IB_ACCESS_REMOTE_WRITE  |\
-				   IB_ACCESS_REMOTE_READ   |\
-				   IB_ACCESS_REMOTE_ATOMIC |\
-				   IB_ZERO_BASED)
+#define MLX5_IB_DM_MEMIC_ALLOWED_ACCESS (IB_ACCESS_LOCAL_WRITE   |\
+					 IB_ACCESS_REMOTE_WRITE  |\
+					 IB_ACCESS_REMOTE_READ   |\
+					 IB_ACCESS_REMOTE_ATOMIC |\
+					 IB_ZERO_BASED)
+
+#define MLX5_IB_DM_SW_ICM_ALLOWED_ACCESS (IB_ACCESS_LOCAL_WRITE   |\
+					  IB_ACCESS_REMOTE_WRITE  |\
+					  IB_ACCESS_REMOTE_READ   |\
+					  IB_ZERO_BASED)
 
 struct mlx5_ib_mr {
 	struct ib_mr		ibmr;
@@ -702,12 +721,6 @@ struct mlx5_ib_multiport {
 	spinlock_t mpi_lock;
 };
 
-struct mlx5_ib_port {
-	struct mlx5_ib_counters cnts;
-	struct mlx5_ib_multiport mp;
-	struct mlx5_ib_dbg_cc_params	*dbg_cc_params;
-};
-
 struct mlx5_roce {
 	/* Protect mlx5_ib_get_netdev from invoking dev_hold() with a NULL
 	 * netdev pointer
@@ -721,6 +734,14 @@ struct mlx5_roce {
 	u8			native_port_num;
 };
 
+struct mlx5_ib_port {
+	struct mlx5_ib_counters cnts;
+	struct mlx5_ib_multiport mp;
+	struct mlx5_ib_dbg_cc_params *dbg_cc_params;
+	struct mlx5_roce roce;
+	struct mlx5_eswitch_rep		*rep;
+};
+
 struct mlx5_ib_dbg_param {
 	int			offset;
 	struct mlx5_ib_dev	*dev;
@@ -840,10 +861,16 @@ struct mlx5_ib_flow_action {
 	};
 };
 
-struct mlx5_memic {
+struct mlx5_dm {
 	struct mlx5_core_dev *dev;
-	spinlock_t		memic_lock;
+	/* This lock is used to protect the access to the shared
+	 * allocation map when concurrent requests by different
+	 * processes are handled.
+	 */
+	spinlock_t lock;
 	DECLARE_BITMAP(memic_alloc_pages, MLX5_MAX_MEMIC_PAGES);
+	unsigned long *steering_sw_icm_alloc_blocks;
+	unsigned long *header_modify_sw_icm_alloc_blocks;
 };
 
 struct mlx5_read_counters_attr {
@@ -905,7 +932,6 @@ struct mlx5_ib_dev {
 	struct ib_device		ib_dev;
 	struct mlx5_core_dev		*mdev;
 	struct notifier_block		mdev_events;
-	struct mlx5_roce		roce[MLX5_MAX_PORTS];
 	int				num_ports;
 	/* serialize update of capability mask
 	 */
@@ -940,17 +966,18 @@ struct mlx5_ib_dev {
 	struct mlx5_sq_bfreg	fp_bfreg;
 	struct mlx5_ib_delay_drop	delay_drop;
 	const struct mlx5_ib_profile	*profile;
-	struct mlx5_eswitch_rep		*rep;
+	bool			is_rep;
 	int				lag_active;
 
 	struct mlx5_ib_lb_state		lb;
 	u8			umr_fence;
 	struct list_head	ib_dev_list;
 	u64			sys_image_guid;
-	struct mlx5_memic	memic;
+	struct mlx5_dm		dm;
 	u16			devx_whitelist_uid;
 	struct mlx5_srq_table   srq_table;
 	struct mlx5_async_ctx   async_ctx;
+	int			free_port;
 };
 
 static inline struct mlx5_ib_cq *to_mibcq(struct mlx5_core_cq *mcq)
@@ -968,6 +995,14 @@ static inline struct mlx5_ib_dev *to_mdev(struct ib_device *ibdev)
 	return container_of(ibdev, struct mlx5_ib_dev, ib_dev);
 }
 
+static inline struct mlx5_ib_dev *mlx5_udata_to_mdev(struct ib_udata *udata)
+{
+	struct mlx5_ib_ucontext *context = rdma_udata_to_drv_context(
+		udata, struct mlx5_ib_ucontext, ibucontext);
+
+	return to_mdev(context->ibucontext.device);
+}
+
 static inline struct mlx5_ib_cq *to_mcq(struct ib_cq *ibcq)
 {
 	return container_of(ibcq, struct mlx5_ib_cq, ibcq);
@@ -1046,17 +1081,16 @@ void mlx5_ib_db_unmap_user(struct mlx5_ib_ucontext *context, struct mlx5_db *db)
 void __mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 qpn, struct mlx5_ib_srq *srq);
 void mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 qpn, struct mlx5_ib_srq *srq);
 void mlx5_ib_free_srq_wqe(struct mlx5_ib_srq *srq, int wqe_index);
-struct ib_ah *mlx5_ib_create_ah(struct ib_pd *pd, struct rdma_ah_attr *ah_attr,
-				u32 flags, struct ib_udata *udata);
+int mlx5_ib_create_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr, u32 flags,
+		      struct ib_udata *udata);
 int mlx5_ib_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr);
-int mlx5_ib_destroy_ah(struct ib_ah *ah, u32 flags);
-struct ib_srq *mlx5_ib_create_srq(struct ib_pd *pd,
-				  struct ib_srq_init_attr *init_attr,
-				  struct ib_udata *udata);
+void mlx5_ib_destroy_ah(struct ib_ah *ah, u32 flags);
+int mlx5_ib_create_srq(struct ib_srq *srq, struct ib_srq_init_attr *init_attr,
+		       struct ib_udata *udata);
 int mlx5_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
 		       enum ib_srq_attr_mask attr_mask, struct ib_udata *udata);
 int mlx5_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr);
-int mlx5_ib_destroy_srq(struct ib_srq *srq);
+void mlx5_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata);
 int mlx5_ib_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
 			  const struct ib_recv_wr **bad_wr);
 int mlx5_ib_enable_lb(struct mlx5_ib_dev *dev, bool td, bool qp);
@@ -1068,7 +1102,7 @@ int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 		      int attr_mask, struct ib_udata *udata);
 int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask,
 		     struct ib_qp_init_attr *qp_init_attr);
-int mlx5_ib_destroy_qp(struct ib_qp *qp);
+int mlx5_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata);
 void mlx5_ib_drain_sq(struct ib_qp *qp);
 void mlx5_ib_drain_rq(struct ib_qp *qp);
 int mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
@@ -1083,9 +1117,8 @@ int mlx5_ib_read_user_wqe_srq(struct mlx5_ib_srq *srq, int wqe_index,
 			      void *buffer, int buflen, size_t *bc);
 struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
 				const struct ib_cq_init_attr *attr,
-				struct ib_ucontext *context,
 				struct ib_udata *udata);
-int mlx5_ib_destroy_cq(struct ib_cq *cq);
+int mlx5_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata);
 int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
 int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
 int mlx5_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period);
@@ -1112,10 +1145,9 @@ void mlx5_ib_free_implicit_mr(struct mlx5_ib_mr *mr);
 int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
 			  u64 length, u64 virt_addr, int access_flags,
 			  struct ib_pd *pd, struct ib_udata *udata);
-int mlx5_ib_dereg_mr(struct ib_mr *ibmr);
-struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd,
-			       enum ib_mr_type mr_type,
-			       u32 max_num_sg);
+int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
+struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
+			       u32 max_num_sg, struct ib_udata *udata);
 int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
 		      unsigned int *sg_offset);
 int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
@@ -1124,9 +1156,8 @@ int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
 			struct ib_mad_hdr *out, size_t *out_mad_size,
 			u16 *out_mad_pkey_index);
 struct ib_xrcd *mlx5_ib_alloc_xrcd(struct ib_device *ibdev,
-					  struct ib_ucontext *context,
-					  struct ib_udata *udata);
-int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd);
+				   struct ib_udata *udata);
+int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata);
 int mlx5_ib_get_buf_offset(u64 addr, int page_shift, u32 *offset);
 int mlx5_query_ext_port_caps(struct mlx5_ib_dev *dev, u8 port);
 int mlx5_query_mad_ifc_smp_attr_node_info(struct ib_device *ibdev,
@@ -1170,7 +1201,7 @@ int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask,
 struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,
 				struct ib_wq_init_attr *init_attr,
 				struct ib_udata *udata);
-int mlx5_ib_destroy_wq(struct ib_wq *wq);
+int mlx5_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata);
 int mlx5_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,
 		      u32 wq_attr_mask, struct ib_udata *udata);
 struct ib_rwq_ind_table *mlx5_ib_create_rwq_ind_table(struct ib_device *device,
@@ -1182,7 +1213,7 @@ struct ib_dm *mlx5_ib_alloc_dm(struct ib_device *ibdev,
 			       struct ib_ucontext *context,
 			       struct ib_dm_alloc_attr *attr,
 			       struct uverbs_attr_bundle *attrs);
-int mlx5_ib_dealloc_dm(struct ib_dm *ibdm);
+int mlx5_ib_dealloc_dm(struct ib_dm *ibdm, struct uverbs_attr_bundle *attrs);
 struct ib_mr *mlx5_ib_reg_dm_mr(struct ib_pd *pd, struct ib_dm *dm,
 				struct ib_dm_mr_attr *attr,
 				struct uverbs_attr_bundle *attrs);
@@ -1230,23 +1261,6 @@ static inline void mlx5_ib_invalidate_range(struct ib_umem_odp *umem_odp,
 #endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
 
 /* Needed for rep profile */
-int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev);
-void mlx5_ib_stage_init_cleanup(struct mlx5_ib_dev *dev);
-int mlx5_ib_stage_rep_flow_db_init(struct mlx5_ib_dev *dev);
-int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev);
-int mlx5_ib_stage_rep_non_default_cb(struct mlx5_ib_dev *dev);
-int mlx5_ib_stage_rep_roce_init(struct mlx5_ib_dev *dev);
-void mlx5_ib_stage_rep_roce_cleanup(struct mlx5_ib_dev *dev);
-int mlx5_ib_stage_dev_res_init(struct mlx5_ib_dev *dev);
-void mlx5_ib_stage_dev_res_cleanup(struct mlx5_ib_dev *dev);
-int mlx5_ib_stage_counters_init(struct mlx5_ib_dev *dev);
-void mlx5_ib_stage_counters_cleanup(struct mlx5_ib_dev *dev);
-int mlx5_ib_stage_bfrag_init(struct mlx5_ib_dev *dev);
-void mlx5_ib_stage_bfrag_cleanup(struct mlx5_ib_dev *dev);
-void mlx5_ib_stage_pre_ib_reg_umr_cleanup(struct mlx5_ib_dev *dev);
-int mlx5_ib_stage_ib_reg_init(struct mlx5_ib_dev *dev);
-void mlx5_ib_stage_ib_reg_cleanup(struct mlx5_ib_dev *dev);
-int mlx5_ib_stage_post_ib_reg_umr_init(struct mlx5_ib_dev *dev);
 void __mlx5_ib_remove(struct mlx5_ib_dev *dev,
 		      const struct mlx5_ib_profile *profile,
 		      int stage);
diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c
index ca921fd40499..5f09699fab98 100644
--- a/drivers/infiniband/hw/mlx5/mr.c
+++ b/drivers/infiniband/hw/mlx5/mr.c
@@ -600,7 +600,7 @@ static void clean_keys(struct mlx5_ib_dev *dev, int c)
 
 static void mlx5_mr_cache_debugfs_cleanup(struct mlx5_ib_dev *dev)
 {
-	if (!mlx5_debugfs_root || dev->rep)
+	if (!mlx5_debugfs_root || dev->is_rep)
 		return;
 
 	debugfs_remove_recursive(dev->cache.root);
@@ -614,7 +614,7 @@ static void mlx5_mr_cache_debugfs_init(struct mlx5_ib_dev *dev)
 	struct dentry *dir;
 	int i;
 
-	if (!mlx5_debugfs_root || dev->rep)
+	if (!mlx5_debugfs_root || dev->is_rep)
 		return;
 
 	cache->root = debugfs_create_dir("mr_cache", dev->mdev->priv.dbg_root);
@@ -677,7 +677,7 @@ int mlx5_mr_cache_init(struct mlx5_ib_dev *dev)
 			   MLX5_IB_UMR_OCTOWORD;
 		ent->access_mode = MLX5_MKC_ACCESS_MODE_MTT;
 		if ((dev->mdev->profile->mask & MLX5_PROF_MASK_MR_CACHE) &&
-		    !dev->rep &&
+		    !dev->is_rep &&
 		    mlx5_core_is_pf(dev->mdev))
 			ent->limit = dev->mdev->profile->mr_cache[i].limit;
 		else
@@ -1159,8 +1159,8 @@ static void set_mr_fields(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr,
 	mr->access_flags = access_flags;
 }
 
-static struct ib_mr *mlx5_ib_get_memic_mr(struct ib_pd *pd, u64 memic_addr,
-					  u64 length, int acc)
+static struct ib_mr *mlx5_ib_get_dm_mr(struct ib_pd *pd, u64 start_addr,
+				       u64 length, int acc, int mode)
 {
 	struct mlx5_ib_dev *dev = to_mdev(pd->device);
 	int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
@@ -1182,9 +1182,8 @@ static struct ib_mr *mlx5_ib_get_memic_mr(struct ib_pd *pd, u64 memic_addr,
 
 	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
 
-	MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MEMIC & 0x3);
-	MLX5_SET(mkc, mkc, access_mode_4_2,
-		 (MLX5_MKC_ACCESS_MODE_MEMIC >> 2) & 0x7);
+	MLX5_SET(mkc, mkc, access_mode_1_0, mode & 0x3);
+	MLX5_SET(mkc, mkc, access_mode_4_2, (mode >> 2) & 0x7);
 	MLX5_SET(mkc, mkc, a, !!(acc & IB_ACCESS_REMOTE_ATOMIC));
 	MLX5_SET(mkc, mkc, rw, !!(acc & IB_ACCESS_REMOTE_WRITE));
 	MLX5_SET(mkc, mkc, rr, !!(acc & IB_ACCESS_REMOTE_READ));
@@ -1194,7 +1193,7 @@ static struct ib_mr *mlx5_ib_get_memic_mr(struct ib_pd *pd, u64 memic_addr,
 	MLX5_SET64(mkc, mkc, len, length);
 	MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);
 	MLX5_SET(mkc, mkc, qpn, 0xffffff);
-	MLX5_SET64(mkc, mkc, start_addr, memic_addr - dev->mdev->bar_addr);
+	MLX5_SET64(mkc, mkc, start_addr, start_addr);
 
 	err = mlx5_core_create_mkey(mdev, &mr->mmkey, in, inlen);
 	if (err)
@@ -1236,15 +1235,31 @@ struct ib_mr *mlx5_ib_reg_dm_mr(struct ib_pd *pd, struct ib_dm *dm,
 				struct uverbs_attr_bundle *attrs)
 {
 	struct mlx5_ib_dm *mdm = to_mdm(dm);
-	u64 memic_addr;
+	struct mlx5_core_dev *dev = to_mdev(dm->device)->mdev;
+	u64 start_addr = mdm->dev_addr + attr->offset;
+	int mode;
 
-	if (attr->access_flags & ~MLX5_IB_DM_ALLOWED_ACCESS)
-		return ERR_PTR(-EINVAL);
+	switch (mdm->type) {
+	case MLX5_IB_UAPI_DM_TYPE_MEMIC:
+		if (attr->access_flags & ~MLX5_IB_DM_MEMIC_ALLOWED_ACCESS)
+			return ERR_PTR(-EINVAL);
+
+		mode = MLX5_MKC_ACCESS_MODE_MEMIC;
+		start_addr -= pci_resource_start(dev->pdev, 0);
+		break;
+	case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:
+	case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:
+		if (attr->access_flags & ~MLX5_IB_DM_SW_ICM_ALLOWED_ACCESS)
+			return ERR_PTR(-EINVAL);
 
-	memic_addr = mdm->dev_addr + attr->offset;
+		mode = MLX5_MKC_ACCESS_MODE_SW_ICM;
+		break;
+	default:
+		return ERR_PTR(-EINVAL);
+	}
 
-	return mlx5_ib_get_memic_mr(pd, memic_addr, attr->length,
-				    attr->access_flags);
+	return mlx5_ib_get_dm_mr(pd, start_addr, attr->length,
+				 attr->access_flags, mode);
 }
 
 struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
@@ -1622,15 +1637,14 @@ static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
 		kfree(mr);
 }
 
-int mlx5_ib_dereg_mr(struct ib_mr *ibmr)
+int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
 {
 	dereg_mr(to_mdev(ibmr->device), to_mmr(ibmr));
 	return 0;
 }
 
-struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd,
-			       enum ib_mr_type mr_type,
-			       u32 max_num_sg)
+struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
+			       u32 max_num_sg, struct ib_udata *udata)
 {
 	struct mlx5_ib_dev *dev = to_mdev(pd->device);
 	int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c
index 0aa10ebda5d9..91507a2e9290 100644
--- a/drivers/infiniband/hw/mlx5/odp.c
+++ b/drivers/infiniband/hw/mlx5/odp.c
@@ -288,7 +288,7 @@ void mlx5_ib_invalidate_range(struct ib_umem_odp *umem_odp, unsigned long start,
 
 	ib_umem_odp_unmap_dma_pages(umem_odp, start, end);
 
-	if (unlikely(!umem->npages && mr->parent &&
+	if (unlikely(!umem_odp->npages && mr->parent &&
 		     !umem_odp->dying)) {
 		WRITE_ONCE(umem_odp->dying, 1);
 		atomic_inc(&mr->parent->num_leaf_free);
@@ -711,6 +711,15 @@ struct pf_frame {
 	int depth;
 };
 
+static bool mkey_is_eq(struct mlx5_core_mkey *mmkey, u32 key)
+{
+	if (!mmkey)
+		return false;
+	if (mmkey->type == MLX5_MKEY_MW)
+		return mlx5_base_mkey(mmkey->key) == mlx5_base_mkey(key);
+	return mmkey->key == key;
+}
+
 static int get_indirect_num_descs(struct mlx5_core_mkey *mmkey)
 {
 	struct mlx5_ib_mw *mw;
@@ -760,7 +769,7 @@ static int pagefault_single_data_segment(struct mlx5_ib_dev *dev,
 
 next_mr:
 	mmkey = __mlx5_mr_lookup(dev->mdev, mlx5_base_mkey(key));
-	if (!mmkey || mmkey->key != key) {
+	if (!mkey_is_eq(mmkey, key)) {
 		mlx5_ib_dbg(dev, "failed to find mkey %x\n", key);
 		ret = -EFAULT;
 		goto srcu_unlock;
@@ -920,7 +929,7 @@ static int pagefault_data_segments(struct mlx5_ib_dev *dev,
 				   struct mlx5_pagefault *pfault,
 				   void *wqe,
 				   void *wqe_end, u32 *bytes_mapped,
-				   u32 *total_wqe_bytes, int receive_queue)
+				   u32 *total_wqe_bytes, bool receive_queue)
 {
 	int ret = 0, npages = 0;
 	u64 io_virt;
@@ -1200,17 +1209,15 @@ static inline struct mlx5_ib_srq *res_to_srq(struct mlx5_core_rsc_common *res)
 static void mlx5_ib_mr_wqe_pfault_handler(struct mlx5_ib_dev *dev,
 					  struct mlx5_pagefault *pfault)
 {
-	int ret;
-	void *wqe, *wqe_end;
+	bool sq = pfault->type & MLX5_PFAULT_REQUESTOR;
+	u16 wqe_index = pfault->wqe.wqe_index;
+	void *wqe = NULL, *wqe_end = NULL;
 	u32 bytes_mapped, total_wqe_bytes;
-	char *buffer = NULL;
+	struct mlx5_core_rsc_common *res;
 	int resume_with_error = 1;
-	u16 wqe_index = pfault->wqe.wqe_index;
-	int requestor = pfault->type & MLX5_PFAULT_REQUESTOR;
-	struct mlx5_core_rsc_common *res = NULL;
-	struct mlx5_ib_qp *qp = NULL;
-	struct mlx5_ib_srq *srq = NULL;
+	struct mlx5_ib_qp *qp;
 	size_t bytes_copied;
+	int ret = 0;
 
 	res = odp_get_rsc(dev, pfault->wqe.wq_num, pfault->type);
 	if (!res) {
@@ -1218,87 +1225,74 @@ static void mlx5_ib_mr_wqe_pfault_handler(struct mlx5_ib_dev *dev,
 		return;
 	}
 
-	switch (res->res) {
-	case MLX5_RES_QP:
-		qp = res_to_qp(res);
-		break;
-	case MLX5_RES_SRQ:
-	case MLX5_RES_XSRQ:
-		srq = res_to_srq(res);
-		break;
-	default:
-		mlx5_ib_err(dev, "wqe page fault for unsupported type %d\n", pfault->type);
+	if (res->res != MLX5_RES_QP && res->res != MLX5_RES_SRQ &&
+	    res->res != MLX5_RES_XSRQ) {
+		mlx5_ib_err(dev, "wqe page fault for unsupported type %d\n",
+			    pfault->type);
 		goto resolve_page_fault;
 	}
 
-	buffer = (char *)__get_free_page(GFP_KERNEL);
-	if (!buffer) {
+	wqe = (void *)__get_free_page(GFP_KERNEL);
+	if (!wqe) {
 		mlx5_ib_err(dev, "Error allocating memory for IO page fault handling.\n");
 		goto resolve_page_fault;
 	}
 
-	if (qp) {
-		if (requestor) {
-			ret = mlx5_ib_read_user_wqe_sq(qp, wqe_index,
-					buffer, PAGE_SIZE,
-					&bytes_copied);
-		} else {
-			ret = mlx5_ib_read_user_wqe_rq(qp, wqe_index,
-					buffer, PAGE_SIZE,
-					&bytes_copied);
-		}
-	} else {
-		ret = mlx5_ib_read_user_wqe_srq(srq, wqe_index,
-						buffer, PAGE_SIZE,
+	qp = (res->res == MLX5_RES_QP) ? res_to_qp(res) : NULL;
+	if (qp && sq) {
+		ret = mlx5_ib_read_user_wqe_sq(qp, wqe_index, wqe, PAGE_SIZE,
+					       &bytes_copied);
+		if (ret)
+			goto read_user;
+		ret = mlx5_ib_mr_initiator_pfault_handler(
+			dev, pfault, qp, &wqe, &wqe_end, bytes_copied);
+	} else if (qp && !sq) {
+		ret = mlx5_ib_read_user_wqe_rq(qp, wqe_index, wqe, PAGE_SIZE,
+					       &bytes_copied);
+		if (ret)
+			goto read_user;
+		ret = mlx5_ib_mr_responder_pfault_handler_rq(
+			dev, qp, wqe, &wqe_end, bytes_copied);
+	} else if (!qp) {
+		struct mlx5_ib_srq *srq = res_to_srq(res);
+
+		ret = mlx5_ib_read_user_wqe_srq(srq, wqe_index, wqe, PAGE_SIZE,
 						&bytes_copied);
+		if (ret)
+			goto read_user;
+		ret = mlx5_ib_mr_responder_pfault_handler_srq(
+			dev, srq, &wqe, &wqe_end, bytes_copied);
 	}
 
-	if (ret) {
-		mlx5_ib_err(dev, "Failed reading a WQE following page fault, error=%d, wqe_index=%x, qpn=%x\n",
-			    ret, wqe_index, pfault->token);
+	if (ret < 0 || wqe >= wqe_end)
 		goto resolve_page_fault;
-	}
 
-	wqe = buffer;
-	if (requestor)
-		ret = mlx5_ib_mr_initiator_pfault_handler(dev, pfault, qp,
-							  &wqe,  &wqe_end,
-							  bytes_copied);
-	else if (qp)
-		ret = mlx5_ib_mr_responder_pfault_handler_rq(dev, qp,
-							     wqe, &wqe_end,
-							     bytes_copied);
-	else
-		ret = mlx5_ib_mr_responder_pfault_handler_srq(dev, srq,
-							      &wqe, &wqe_end,
-							      bytes_copied);
+	ret = pagefault_data_segments(dev, pfault, wqe, wqe_end, &bytes_mapped,
+				      &total_wqe_bytes, !sq);
+	if (ret == -EAGAIN)
+		goto out;
 
-	if (ret < 0)
+	if (ret < 0 || total_wqe_bytes > bytes_mapped)
 		goto resolve_page_fault;
 
-	if (wqe >= wqe_end) {
-		mlx5_ib_err(dev, "ODP fault on invalid WQE.\n");
-		goto resolve_page_fault;
-	}
+out:
+	ret = 0;
+	resume_with_error = 0;
 
-	ret = pagefault_data_segments(dev, pfault, wqe, wqe_end,
-				      &bytes_mapped, &total_wqe_bytes,
-				      !requestor);
-	if (ret == -EAGAIN) {
-		resume_with_error = 0;
-		goto resolve_page_fault;
-	} else if (ret < 0 || total_wqe_bytes > bytes_mapped) {
-		goto resolve_page_fault;
-	}
+read_user:
+	if (ret)
+		mlx5_ib_err(
+			dev,
+			"Failed reading a WQE following page fault, error %d, wqe_index %x, qpn %x\n",
+			ret, wqe_index, pfault->token);
 
-	resume_with_error = 0;
 resolve_page_fault:
 	mlx5_ib_page_fault_resume(dev, pfault, resume_with_error);
 	mlx5_ib_dbg(dev, "PAGE FAULT completed. QP 0x%x resume_with_error=%d, type: 0x%x\n",
 		    pfault->wqe.wq_num, resume_with_error,
 		    pfault->type);
 	mlx5_core_res_put(res);
-	free_page((unsigned long)buffer);
+	free_page((unsigned long)wqe);
 }
 
 static int pages_in_range(u64 address, u32 length)
diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c
index 581144e224e2..f6623c77443a 100644
--- a/drivers/infiniband/hw/mlx5/qp.c
+++ b/drivers/infiniband/hw/mlx5/qp.c
@@ -92,6 +92,7 @@ struct mlx5_modify_raw_qp_param {
 	struct mlx5_rate_limit rl;
 
 	u8 rq_q_ctr_id;
+	u16 port;
 };
 
 static void get_cqs(enum ib_qp_type qp_type,
@@ -777,14 +778,17 @@ err_umem:
 }
 
 static void destroy_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd,
-			    struct mlx5_ib_rwq *rwq)
+			    struct mlx5_ib_rwq *rwq, struct ib_udata *udata)
 {
-	struct mlx5_ib_ucontext *context;
+	struct mlx5_ib_ucontext *context =
+		rdma_udata_to_drv_context(
+			udata,
+			struct mlx5_ib_ucontext,
+			ibucontext);
 
 	if (rwq->create_flags & MLX5_IB_WQ_FLAGS_DELAY_DROP)
 		atomic_dec(&dev->delay_drop.rqs_cnt);
 
-	context = to_mucontext(pd->uobject->context);
 	mlx5_ib_db_unmap_user(context, &rwq->db);
 	if (rwq->umem)
 		ib_umem_release(rwq->umem);
@@ -983,11 +987,15 @@ err_bfreg:
 }
 
 static void destroy_qp_user(struct mlx5_ib_dev *dev, struct ib_pd *pd,
-			    struct mlx5_ib_qp *qp, struct mlx5_ib_qp_base *base)
+			    struct mlx5_ib_qp *qp, struct mlx5_ib_qp_base *base,
+			    struct ib_udata *udata)
 {
-	struct mlx5_ib_ucontext *context;
+	struct mlx5_ib_ucontext *context =
+		rdma_udata_to_drv_context(
+			udata,
+			struct mlx5_ib_ucontext,
+			ibucontext);
 
-	context = to_mucontext(pd->uobject->context);
 	mlx5_ib_db_unmap_user(context, &qp->db);
 	if (base->ubuffer.umem)
 		ib_umem_release(base->ubuffer.umem);
@@ -1206,11 +1214,11 @@ static void destroy_raw_packet_qp_tis(struct mlx5_ib_dev *dev,
 	mlx5_cmd_destroy_tis(dev->mdev, sq->tisn, to_mpd(pd)->uid);
 }
 
-static void destroy_flow_rule_vport_sq(struct mlx5_ib_dev *dev,
-				       struct mlx5_ib_sq *sq)
+static void destroy_flow_rule_vport_sq(struct mlx5_ib_sq *sq)
 {
 	if (sq->flow_rule)
 		mlx5_del_flow_rules(sq->flow_rule);
+	sq->flow_rule = NULL;
 }
 
 static int create_raw_packet_qp_sq(struct mlx5_ib_dev *dev,
@@ -1278,15 +1286,8 @@ static int create_raw_packet_qp_sq(struct mlx5_ib_dev *dev,
 	if (err)
 		goto err_umem;
 
-	err = create_flow_rule_vport_sq(dev, sq);
-	if (err)
-		goto err_flow;
-
 	return 0;
 
-err_flow:
-	mlx5_core_destroy_sq_tracked(dev->mdev, &sq->base.mqp);
-
 err_umem:
 	ib_umem_release(sq->ubuffer.umem);
 	sq->ubuffer.umem = NULL;
@@ -1297,7 +1298,7 @@ err_umem:
 static void destroy_raw_packet_qp_sq(struct mlx5_ib_dev *dev,
 				     struct mlx5_ib_sq *sq)
 {
-	destroy_flow_rule_vport_sq(dev, sq);
+	destroy_flow_rule_vport_sq(sq);
 	mlx5_core_destroy_sq_tracked(dev->mdev, &sq->base.mqp);
 	ib_umem_release(sq->ubuffer.umem);
 }
@@ -1402,7 +1403,8 @@ static void destroy_raw_packet_qp_tir(struct mlx5_ib_dev *dev,
 static int create_raw_packet_qp_tir(struct mlx5_ib_dev *dev,
 				    struct mlx5_ib_rq *rq, u32 tdn,
 				    u32 *qp_flags_en,
-				    struct ib_pd *pd)
+				    struct ib_pd *pd,
+				    u32 *out, int outlen)
 {
 	u8 lb_flag = 0;
 	u32 *in;
@@ -1429,15 +1431,16 @@ static int create_raw_packet_qp_tir(struct mlx5_ib_dev *dev,
 	if (*qp_flags_en & MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC)
 		lb_flag |= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_MULTICAST;
 
-	if (dev->rep) {
+	if (dev->is_rep) {
 		lb_flag |= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST;
 		*qp_flags_en |= MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC;
 	}
 
 	MLX5_SET(tirc, tirc, self_lb_block, lb_flag);
 
-	err = mlx5_core_create_tir(dev->mdev, in, inlen, &rq->tirn);
+	err = mlx5_core_create_tir_out(dev->mdev, in, inlen, out, outlen);
 
+	rq->tirn = MLX5_GET(create_tir_out, out, tirn);
 	if (!err && MLX5_GET(tirc, tirc, self_lb_block)) {
 		err = mlx5_ib_enable_lb(dev, false, true);
 
@@ -1463,6 +1466,7 @@ static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 	int err;
 	u32 tdn = mucontext->tdn;
 	u16 uid = to_mpd(pd)->uid;
+	u32 out[MLX5_ST_SZ_DW(create_tir_out)] = {};
 
 	if (qp->sq.wqe_cnt) {
 		err = create_raw_packet_qp_tis(dev, qp, sq, tdn, pd);
@@ -1495,7 +1499,9 @@ static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 		if (err)
 			goto err_destroy_sq;
 
-		err = create_raw_packet_qp_tir(dev, rq, tdn, &qp->flags_en, pd);
+		err = create_raw_packet_qp_tir(
+			dev, rq, tdn, &qp->flags_en, pd, out,
+			MLX5_ST_SZ_BYTES(create_tir_out));
 		if (err)
 			goto err_destroy_rq;
 
@@ -1504,6 +1510,20 @@ static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 			resp->comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_RQN;
 			resp->tirn = rq->tirn;
 			resp->comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_TIRN;
+			if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner)) {
+				resp->tir_icm_addr = MLX5_GET(
+					create_tir_out, out, icm_address_31_0);
+				resp->tir_icm_addr |=
+					(u64)MLX5_GET(create_tir_out, out,
+						      icm_address_39_32)
+					<< 32;
+				resp->tir_icm_addr |=
+					(u64)MLX5_GET(create_tir_out, out,
+						      icm_address_63_40)
+					<< 40;
+				resp->comp_mask |=
+					MLX5_IB_CREATE_QP_RESP_MASK_TIR_ICM_ADDR;
+			}
 		}
 	}
 
@@ -1577,8 +1597,10 @@ static int create_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 		udata, struct mlx5_ib_ucontext, ibucontext);
 	struct mlx5_ib_create_qp_resp resp = {};
 	int inlen;
+	int outlen;
 	int err;
 	u32 *in;
+	u32 *out;
 	void *tirc;
 	void *hfso;
 	u32 selected_fields = 0;
@@ -1641,7 +1663,7 @@ static int create_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 		return -EOPNOTSUPP;
 	}
 
-	if (ucmd.flags & MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC || dev->rep) {
+	if (ucmd.flags & MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC || dev->is_rep) {
 		lb_flag |= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST;
 		qp->flags_en |= MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC;
 	}
@@ -1658,10 +1680,12 @@ static int create_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 	}
 
 	inlen = MLX5_ST_SZ_BYTES(create_tir_in);
-	in = kvzalloc(inlen, GFP_KERNEL);
+	outlen = MLX5_ST_SZ_BYTES(create_tir_out);
+	in = kvzalloc(inlen + outlen, GFP_KERNEL);
 	if (!in)
 		return -ENOMEM;
 
+	out = in + MLX5_ST_SZ_DW(create_tir_in);
 	MLX5_SET(create_tir_in, in, uid, to_mpd(pd)->uid);
 	tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
 	MLX5_SET(tirc, tirc, disp_type,
@@ -1773,8 +1797,9 @@ static int create_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 	MLX5_SET(rx_hash_field_select, hfso, selected_fields, selected_fields);
 
 create_tir:
-	err = mlx5_core_create_tir(dev->mdev, in, inlen, &qp->rss_qp.tirn);
+	err = mlx5_core_create_tir_out(dev->mdev, in, inlen, out, outlen);
 
+	qp->rss_qp.tirn = MLX5_GET(create_tir_out, out, tirn);
 	if (!err && MLX5_GET(tirc, tirc, self_lb_block)) {
 		err = mlx5_ib_enable_lb(dev, false, true);
 
@@ -1789,6 +1814,18 @@ create_tir:
 	if (mucontext->devx_uid) {
 		resp.comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_TIRN;
 		resp.tirn = qp->rss_qp.tirn;
+		if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner)) {
+			resp.tir_icm_addr =
+				MLX5_GET(create_tir_out, out, icm_address_31_0);
+			resp.tir_icm_addr |= (u64)MLX5_GET(create_tir_out, out,
+							   icm_address_39_32)
+					     << 32;
+			resp.tir_icm_addr |= (u64)MLX5_GET(create_tir_out, out,
+							   icm_address_63_40)
+					     << 40;
+			resp.comp_mask |=
+				MLX5_IB_CREATE_QP_RESP_MASK_TIR_ICM_ADDR;
+		}
 	}
 
 	err = ib_copy_to_udata(udata, &resp, min(udata->outlen, sizeof(resp)));
@@ -2287,7 +2324,7 @@ static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 
 err_create:
 	if (qp->create_type == MLX5_QP_USER)
-		destroy_qp_user(dev, pd, qp, base);
+		destroy_qp_user(dev, pd, qp, base, udata);
 	else if (qp->create_type == MLX5_QP_KERNEL)
 		destroy_qp_kernel(dev, qp);
 
@@ -2398,7 +2435,8 @@ static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 				const struct mlx5_modify_raw_qp_param *raw_qp_param,
 				u8 lag_tx_affinity);
 
-static void destroy_qp_common(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp)
+static void destroy_qp_common(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
+			      struct ib_udata *udata)
 {
 	struct mlx5_ib_cq *send_cq, *recv_cq;
 	struct mlx5_ib_qp_base *base;
@@ -2469,7 +2507,7 @@ static void destroy_qp_common(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp)
 	if (qp->create_type == MLX5_QP_KERNEL)
 		destroy_qp_kernel(dev, qp);
 	else if (qp->create_type == MLX5_QP_USER)
-		destroy_qp_user(dev, &get_pd(qp)->ibpd, qp, base);
+		destroy_qp_user(dev, &get_pd(qp)->ibpd, qp, base, udata);
 }
 
 static const char *ib_qp_type_str(enum ib_qp_type type)
@@ -2735,7 +2773,7 @@ static int mlx5_ib_destroy_dct(struct mlx5_ib_qp *mqp)
 	return 0;
 }
 
-int mlx5_ib_destroy_qp(struct ib_qp *qp)
+int mlx5_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata)
 {
 	struct mlx5_ib_dev *dev = to_mdev(qp->device);
 	struct mlx5_ib_qp *mqp = to_mqp(qp);
@@ -2746,7 +2784,7 @@ int mlx5_ib_destroy_qp(struct ib_qp *qp)
 	if (mqp->qp_sub_type == MLX5_IB_QPT_DCT)
 		return mlx5_ib_destroy_dct(mqp);
 
-	destroy_qp_common(dev, mqp);
+	destroy_qp_common(dev, mqp, udata);
 
 	kfree(mqp);
 
@@ -2964,6 +3002,11 @@ static enum mlx5_qp_optpar opt_mask[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE][MLX5_Q
 			[MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_PKEY_INDEX	|
 					  MLX5_QP_OPTPAR_Q_KEY		|
 					  MLX5_QP_OPTPAR_PRI_PORT,
+			[MLX5_QP_ST_XRC] = MLX5_QP_OPTPAR_RRE		|
+					  MLX5_QP_OPTPAR_RAE		|
+					  MLX5_QP_OPTPAR_RWE		|
+					  MLX5_QP_OPTPAR_PKEY_INDEX	|
+					  MLX5_QP_OPTPAR_PRI_PORT,
 		},
 		[MLX5_QP_STATE_RTR] = {
 			[MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH  |
@@ -2997,6 +3040,12 @@ static enum mlx5_qp_optpar opt_mask[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE][MLX5_Q
 					  MLX5_QP_OPTPAR_RWE		|
 					  MLX5_QP_OPTPAR_PM_STATE,
 			[MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY,
+			[MLX5_QP_ST_XRC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH	|
+					  MLX5_QP_OPTPAR_RRE		|
+					  MLX5_QP_OPTPAR_RAE		|
+					  MLX5_QP_OPTPAR_RWE		|
+					  MLX5_QP_OPTPAR_PM_STATE	|
+					  MLX5_QP_OPTPAR_RNR_TIMEOUT,
 		},
 	},
 	[MLX5_QP_STATE_RTS] = {
@@ -3013,6 +3062,12 @@ static enum mlx5_qp_optpar opt_mask[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE][MLX5_Q
 			[MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY		|
 					  MLX5_QP_OPTPAR_SRQN		|
 					  MLX5_QP_OPTPAR_CQN_RCV,
+			[MLX5_QP_ST_XRC] = MLX5_QP_OPTPAR_RRE		|
+					  MLX5_QP_OPTPAR_RAE		|
+					  MLX5_QP_OPTPAR_RWE		|
+					  MLX5_QP_OPTPAR_RNR_TIMEOUT	|
+					  MLX5_QP_OPTPAR_PM_STATE	|
+					  MLX5_QP_OPTPAR_ALT_ADDR_PATH,
 		},
 	},
 	[MLX5_QP_STATE_SQER] = {
@@ -3024,6 +3079,10 @@ static enum mlx5_qp_optpar opt_mask[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE][MLX5_Q
 					   MLX5_QP_OPTPAR_RWE		|
 					   MLX5_QP_OPTPAR_RAE		|
 					   MLX5_QP_OPTPAR_RRE,
+			[MLX5_QP_ST_XRC]  = MLX5_QP_OPTPAR_RNR_TIMEOUT	|
+					   MLX5_QP_OPTPAR_RWE		|
+					   MLX5_QP_OPTPAR_RAE		|
+					   MLX5_QP_OPTPAR_RRE,
 		},
 	},
 };
@@ -3264,6 +3323,8 @@ static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 	}
 
 	if (modify_sq) {
+		struct mlx5_flow_handle *flow_rule;
+
 		if (tx_affinity) {
 			err = modify_raw_packet_tx_affinity(dev->mdev, sq,
 							    tx_affinity,
@@ -3272,8 +3333,25 @@ static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
 				return err;
 		}
 
-		return modify_raw_packet_qp_sq(dev->mdev, sq, sq_state,
-					       raw_qp_param, qp->ibqp.pd);
+		flow_rule = create_flow_rule_vport_sq(dev, sq,
+						      raw_qp_param->port);
+		if (IS_ERR(flow_rule))
+			return PTR_ERR(flow_rule);
+
+		err = modify_raw_packet_qp_sq(dev->mdev, sq, sq_state,
+					      raw_qp_param, qp->ibqp.pd);
+		if (err) {
+			if (flow_rule)
+				mlx5_del_flow_rules(flow_rule);
+			return err;
+		}
+
+		if (flow_rule) {
+			destroy_flow_rule_vport_sq(sq);
+			sq->flow_rule = flow_rule;
+		}
+
+		return err;
 	}
 
 	return 0;
@@ -3298,7 +3376,7 @@ static unsigned int get_tx_affinity(struct mlx5_ib_dev *dev,
 	} else {
 		tx_port_affinity =
 			(unsigned int)atomic_add_return(
-				1, &dev->roce[port_num].tx_port_affinity) %
+				1, &dev->port[port_num].roce.tx_port_affinity) %
 				MLX5_MAX_PORTS +
 			1;
 		mlx5_ib_dbg(dev, "Set tx affinity 0x%x to qpn 0x%x\n",
@@ -3403,7 +3481,7 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
 		    (ibqp->qp_type == IB_QPT_XRC_INI) ||
 		    (ibqp->qp_type == IB_QPT_XRC_TGT)) {
 			if (dev->lag_active) {
-				u8 p = mlx5_core_native_port_num(dev->mdev);
+				u8 p = mlx5_core_native_port_num(dev->mdev) - 1;
 				tx_affinity = get_tx_affinity(dev, pd, base, p,
 							      udata);
 				context->flags |= cpu_to_be32(tx_affinity << 24);
@@ -3556,6 +3634,9 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
 			raw_qp_param.set_mask |= MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID;
 		}
 
+		if (attr_mask & IB_QP_PORT)
+			raw_qp_param.port = attr->port_num;
+
 		if (attr_mask & IB_QP_RATE_LIMIT) {
 			raw_qp_param.rl.rate = attr->rate_limit;
 
@@ -4729,16 +4810,15 @@ static void set_linv_wr(struct mlx5_ib_qp *qp, void **seg, int *size,
 static void dump_wqe(struct mlx5_ib_qp *qp, u32 idx, int size_16)
 {
 	__be32 *p = NULL;
-	u32 tidx = idx;
 	int i, j;
 
 	pr_debug("dump WQE index %u:\n", idx);
 	for (i = 0, j = 0; i < size_16 * 4; i += 4, j += 4) {
 		if ((i & 0xf) == 0) {
-			tidx = (tidx + 1) & (qp->sq.wqe_cnt - 1);
-			p = mlx5_frag_buf_get_wqe(&qp->sq.fbc, tidx);
+			p = mlx5_frag_buf_get_wqe(&qp->sq.fbc, idx);
 			pr_debug("WQBB at %p:\n", (void *)p);
 			j = 0;
+			idx = (idx + 1) & (qp->sq.wqe_cnt - 1);
 		}
 		pr_debug("%08x %08x %08x %08x\n", be32_to_cpu(p[j]),
 			 be32_to_cpu(p[j + 1]), be32_to_cpu(p[j + 2]),
@@ -5627,8 +5707,7 @@ out:
 }
 
 struct ib_xrcd *mlx5_ib_alloc_xrcd(struct ib_device *ibdev,
-					  struct ib_ucontext *context,
-					  struct ib_udata *udata)
+				   struct ib_udata *udata)
 {
 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
 	struct mlx5_ib_xrcd *xrcd;
@@ -5650,7 +5729,7 @@ struct ib_xrcd *mlx5_ib_alloc_xrcd(struct ib_device *ibdev,
 	return &xrcd->ibxrcd;
 }
 
-int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd)
+int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata)
 {
 	struct mlx5_ib_dev *dev = to_mdev(xrcd->device);
 	u32 xrcdn = to_mxrcd(xrcd)->xrcdn;
@@ -5962,19 +6041,19 @@ struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,
 err_copy:
 	mlx5_core_destroy_rq_tracked(dev->mdev, &rwq->core_qp);
 err_user_rq:
-	destroy_user_rq(dev, pd, rwq);
+	destroy_user_rq(dev, pd, rwq, udata);
 err:
 	kfree(rwq);
 	return ERR_PTR(err);
 }
 
-int mlx5_ib_destroy_wq(struct ib_wq *wq)
+int mlx5_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata)
 {
 	struct mlx5_ib_dev *dev = to_mdev(wq->device);
 	struct mlx5_ib_rwq *rwq = to_mrwq(wq);
 
 	mlx5_core_destroy_rq_tracked(dev->mdev, &rwq->core_qp);
-	destroy_user_rq(dev, wq->pd, rwq);
+	destroy_user_rq(dev, wq->pd, rwq, udata);
 	kfree(rwq);
 
 	return 0;
diff --git a/drivers/infiniband/hw/mlx5/srq.c b/drivers/infiniband/hw/mlx5/srq.c
index 1ec1beb1296b..4e7fde86c96b 100644
--- a/drivers/infiniband/hw/mlx5/srq.c
+++ b/drivers/infiniband/hw/mlx5/srq.c
@@ -194,9 +194,15 @@ err_db:
 	return err;
 }
 
-static void destroy_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq)
+static void destroy_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq,
+			     struct ib_udata *udata)
 {
-	mlx5_ib_db_unmap_user(to_mucontext(pd->uobject->context), &srq->db);
+	mlx5_ib_db_unmap_user(
+		rdma_udata_to_drv_context(
+			udata,
+			struct mlx5_ib_ucontext,
+			ibucontext),
+		&srq->db);
 	ib_umem_release(srq->umem);
 }
 
@@ -208,16 +214,16 @@ static void destroy_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq)
 	mlx5_db_free(dev->mdev, &srq->db);
 }
 
-struct ib_srq *mlx5_ib_create_srq(struct ib_pd *pd,
-				  struct ib_srq_init_attr *init_attr,
-				  struct ib_udata *udata)
+int mlx5_ib_create_srq(struct ib_srq *ib_srq,
+		       struct ib_srq_init_attr *init_attr,
+		       struct ib_udata *udata)
 {
-	struct mlx5_ib_dev *dev = to_mdev(pd->device);
-	struct mlx5_ib_srq *srq;
+	struct mlx5_ib_dev *dev = to_mdev(ib_srq->device);
+	struct mlx5_ib_srq *srq = to_msrq(ib_srq);
 	size_t desc_size;
 	size_t buf_size;
 	int err;
-	struct mlx5_srq_attr in = {0};
+	struct mlx5_srq_attr in = {};
 	__u32 max_srq_wqes = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz);
 
 	/* Sanity check SRQ size before proceeding */
@@ -225,13 +231,9 @@ struct ib_srq *mlx5_ib_create_srq(struct ib_pd *pd,
 		mlx5_ib_dbg(dev, "max_wr %d, cap %d\n",
 			    init_attr->attr.max_wr,
 			    max_srq_wqes);
-		return ERR_PTR(-EINVAL);
+		return -EINVAL;
 	}
 
-	srq = kmalloc(sizeof(*srq), GFP_KERNEL);
-	if (!srq)
-		return ERR_PTR(-ENOMEM);
-
 	mutex_init(&srq->mutex);
 	spin_lock_init(&srq->lock);
 	srq->msrq.max    = roundup_pow_of_two(init_attr->attr.max_wr + 1);
@@ -239,35 +241,32 @@ struct ib_srq *mlx5_ib_create_srq(struct ib_pd *pd,
 
 	desc_size = sizeof(struct mlx5_wqe_srq_next_seg) +
 		    srq->msrq.max_gs * sizeof(struct mlx5_wqe_data_seg);
-	if (desc_size == 0 || srq->msrq.max_gs > desc_size) {
-		err = -EINVAL;
-		goto err_srq;
-	}
+	if (desc_size == 0 || srq->msrq.max_gs > desc_size)
+		return -EINVAL;
+
 	desc_size = roundup_pow_of_two(desc_size);
 	desc_size = max_t(size_t, 32, desc_size);
-	if (desc_size < sizeof(struct mlx5_wqe_srq_next_seg)) {
-		err = -EINVAL;
-		goto err_srq;
-	}
+	if (desc_size < sizeof(struct mlx5_wqe_srq_next_seg))
+		return -EINVAL;
+
 	srq->msrq.max_avail_gather = (desc_size - sizeof(struct mlx5_wqe_srq_next_seg)) /
 		sizeof(struct mlx5_wqe_data_seg);
 	srq->msrq.wqe_shift = ilog2(desc_size);
 	buf_size = srq->msrq.max * desc_size;
-	if (buf_size < desc_size) {
-		err = -EINVAL;
-		goto err_srq;
-	}
+	if (buf_size < desc_size)
+		return -EINVAL;
+
 	in.type = init_attr->srq_type;
 
 	if (udata)
-		err = create_srq_user(pd, srq, &in, udata, buf_size);
+		err = create_srq_user(ib_srq->pd, srq, &in, udata, buf_size);
 	else
 		err = create_srq_kernel(dev, srq, &in, buf_size);
 
 	if (err) {
 		mlx5_ib_warn(dev, "create srq %s failed, err %d\n",
 			     udata ? "user" : "kernel", err);
-		goto err_srq;
+		return err;
 	}
 
 	in.log_size = ilog2(srq->msrq.max);
@@ -297,7 +296,7 @@ struct ib_srq *mlx5_ib_create_srq(struct ib_pd *pd,
 	else
 		in.cqn = to_mcq(dev->devr.c0)->mcq.cqn;
 
-	in.pd = to_mpd(pd)->pdn;
+	in.pd = to_mpd(ib_srq->pd)->pdn;
 	in.db_record = srq->db.dma;
 	err = mlx5_cmd_create_srq(dev, &srq->msrq, &in);
 	kvfree(in.pas);
@@ -320,21 +319,18 @@ struct ib_srq *mlx5_ib_create_srq(struct ib_pd *pd,
 
 	init_attr->attr.max_wr = srq->msrq.max - 1;
 
-	return &srq->ibsrq;
+	return 0;
 
 err_core:
 	mlx5_cmd_destroy_srq(dev, &srq->msrq);
 
 err_usr_kern_srq:
 	if (udata)
-		destroy_srq_user(pd, srq);
+		destroy_srq_user(ib_srq->pd, srq, udata);
 	else
 		destroy_srq_kernel(dev, srq);
 
-err_srq:
-	kfree(srq);
-
-	return ERR_PTR(err);
+	return err;
 }
 
 int mlx5_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
@@ -387,7 +383,7 @@ out_box:
 	return ret;
 }
 
-int mlx5_ib_destroy_srq(struct ib_srq *srq)
+void mlx5_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata)
 {
 	struct mlx5_ib_dev *dev = to_mdev(srq->device);
 	struct mlx5_ib_srq *msrq = to_msrq(srq);
@@ -395,14 +391,16 @@ int mlx5_ib_destroy_srq(struct ib_srq *srq)
 	mlx5_cmd_destroy_srq(dev, &msrq->msrq);
 
 	if (srq->uobject) {
-		mlx5_ib_db_unmap_user(to_mucontext(srq->uobject->context), &msrq->db);
+		mlx5_ib_db_unmap_user(
+			rdma_udata_to_drv_context(
+				udata,
+				struct mlx5_ib_ucontext,
+				ibucontext),
+			&msrq->db);
 		ib_umem_release(msrq->umem);
 	} else {
 		destroy_srq_kernel(dev, msrq);
 	}
-
-	kfree(srq);
-	return 0;
 }
 
 void mlx5_ib_free_srq_wqe(struct mlx5_ib_srq *srq, int wqe_index)
diff --git a/drivers/infiniband/hw/mlx5/srq.h b/drivers/infiniband/hw/mlx5/srq.h
index c330af35ff10..af197c36d757 100644
--- a/drivers/infiniband/hw/mlx5/srq.h
+++ b/drivers/infiniband/hw/mlx5/srq.h
@@ -51,15 +51,12 @@ struct mlx5_core_srq {
 
 struct mlx5_srq_table {
 	struct notifier_block nb;
-	/* protect radix tree
-	 */
-	spinlock_t lock;
-	struct radix_tree_root tree;
+	struct xarray array;
 };
 
 int mlx5_cmd_create_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
 			struct mlx5_srq_attr *in);
-int mlx5_cmd_destroy_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq);
+void mlx5_cmd_destroy_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq);
 int mlx5_cmd_query_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
 		       struct mlx5_srq_attr *out);
 int mlx5_cmd_arm_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
diff --git a/drivers/infiniband/hw/mlx5/srq_cmd.c b/drivers/infiniband/hw/mlx5/srq_cmd.c
index 63ac38bb3498..b0d0687c7a68 100644
--- a/drivers/infiniband/hw/mlx5/srq_cmd.c
+++ b/drivers/infiniband/hw/mlx5/srq_cmd.c
@@ -83,13 +83,11 @@ struct mlx5_core_srq *mlx5_cmd_get_srq(struct mlx5_ib_dev *dev, u32 srqn)
 	struct mlx5_srq_table *table = &dev->srq_table;
 	struct mlx5_core_srq *srq;
 
-	spin_lock(&table->lock);
-
-	srq = radix_tree_lookup(&table->tree, srqn);
+	xa_lock(&table->array);
+	srq = xa_load(&table->array, srqn);
 	if (srq)
 		atomic_inc(&srq->common.refcount);
-
-	spin_unlock(&table->lock);
+	xa_unlock(&table->array);
 
 	return srq;
 }
@@ -597,9 +595,7 @@ int mlx5_cmd_create_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
 	atomic_set(&srq->common.refcount, 1);
 	init_completion(&srq->common.free);
 
-	spin_lock_irq(&table->lock);
-	err = radix_tree_insert(&table->tree, srq->srqn, srq);
-	spin_unlock_irq(&table->lock);
+	err = xa_err(xa_store_irq(&table->array, srq->srqn, srq, GFP_KERNEL));
 	if (err)
 		goto err_destroy_srq_split;
 
@@ -611,26 +607,22 @@ err_destroy_srq_split:
 	return err;
 }
 
-int mlx5_cmd_destroy_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq)
+void mlx5_cmd_destroy_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq)
 {
 	struct mlx5_srq_table *table = &dev->srq_table;
 	struct mlx5_core_srq *tmp;
 	int err;
 
-	spin_lock_irq(&table->lock);
-	tmp = radix_tree_delete(&table->tree, srq->srqn);
-	spin_unlock_irq(&table->lock);
+	tmp = xa_erase_irq(&table->array, srq->srqn);
 	if (!tmp || tmp != srq)
-		return -EINVAL;
+		return;
 
 	err = destroy_srq_split(dev, srq);
 	if (err)
-		return err;
+		return;
 
 	mlx5_core_res_put(&srq->common);
 	wait_for_completion(&srq->common.free);
-
-	return 0;
 }
 
 int mlx5_cmd_query_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
@@ -680,13 +672,11 @@ static int srq_event_notifier(struct notifier_block *nb,
 	eqe = data;
 	srqn = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff;
 
-	spin_lock(&table->lock);
-
-	srq = radix_tree_lookup(&table->tree, srqn);
+	xa_lock(&table->array);
+	srq = xa_load(&table->array, srqn);
 	if (srq)
 		atomic_inc(&srq->common.refcount);
-
-	spin_unlock(&table->lock);
+	xa_unlock(&table->array);
 
 	if (!srq)
 		return NOTIFY_OK;
@@ -703,8 +693,7 @@ int mlx5_init_srq_table(struct mlx5_ib_dev *dev)
 	struct mlx5_srq_table *table = &dev->srq_table;
 
 	memset(table, 0, sizeof(*table));
-	spin_lock_init(&table->lock);
-	INIT_RADIX_TREE(&table->tree, GFP_ATOMIC);
+	xa_init_flags(&table->array, XA_FLAGS_LOCK_IRQ);
 
 	table->nb.notifier_call = srq_event_notifier;
 	mlx5_notifier_register(dev->mdev, &table->nb);
diff --git a/drivers/infiniband/hw/mthca/mthca_cq.c b/drivers/infiniband/hw/mthca/mthca_cq.c
index 877a6daffa98..c3cfea243af8 100644
--- a/drivers/infiniband/hw/mthca/mthca_cq.c
+++ b/drivers/infiniband/hw/mthca/mthca_cq.c
@@ -77,7 +77,7 @@ struct mthca_cq_context {
 	__be32 ci_db;		/* Arbel only */
 	__be32 state_db;	/* Arbel only */
 	u32    reserved;
-} __attribute__((packed));
+} __packed;
 
 #define MTHCA_CQ_STATUS_OK          ( 0 << 28)
 #define MTHCA_CQ_STATUS_OVERFLOW    ( 9 << 28)
diff --git a/drivers/infiniband/hw/mthca/mthca_eq.c b/drivers/infiniband/hw/mthca/mthca_eq.c
index 30400ea4808b..2cdf686203c1 100644
--- a/drivers/infiniband/hw/mthca/mthca_eq.c
+++ b/drivers/infiniband/hw/mthca/mthca_eq.c
@@ -63,7 +63,7 @@ struct mthca_eq_context {
 	__be32 consumer_index;
 	__be32 producer_index;
 	u32    reserved3[4];
-} __attribute__((packed));
+} __packed;
 
 #define MTHCA_EQ_STATUS_OK          ( 0 << 28)
 #define MTHCA_EQ_STATUS_OVERFLOW    ( 9 << 28)
@@ -130,7 +130,7 @@ struct mthca_eqe {
 		u32 raw[6];
 		struct {
 			__be32 cqn;
-		} __attribute__((packed)) comp;
+		} __packed comp;
 		struct {
 			u16    reserved1;
 			__be16 token;
@@ -138,27 +138,27 @@ struct mthca_eqe {
 			u8     reserved3[3];
 			u8     status;
 			__be64 out_param;
-		} __attribute__((packed)) cmd;
+		} __packed cmd;
 		struct {
 			__be32 qpn;
-		} __attribute__((packed)) qp;
+		} __packed qp;
 		struct {
 			__be32 srqn;
-		} __attribute__((packed)) srq;
+		} __packed srq;
 		struct {
 			__be32 cqn;
 			u32    reserved1;
 			u8     reserved2[3];
 			u8     syndrome;
-		} __attribute__((packed)) cq_err;
+		} __packed cq_err;
 		struct {
 			u32    reserved1[2];
 			__be32 port;
-		} __attribute__((packed)) port_change;
+		} __packed port_change;
 	} event;
 	u8 reserved3[3];
 	u8 owner;
-} __attribute__((packed));
+} __packed;
 
 #define  MTHCA_EQ_ENTRY_OWNER_SW      (0 << 7)
 #define  MTHCA_EQ_ENTRY_OWNER_HW      (1 << 7)
diff --git a/drivers/infiniband/hw/mthca/mthca_mr.c b/drivers/infiniband/hw/mthca/mthca_mr.c
index 6686042aafb4..4250b2c18c64 100644
--- a/drivers/infiniband/hw/mthca/mthca_mr.c
+++ b/drivers/infiniband/hw/mthca/mthca_mr.c
@@ -60,7 +60,7 @@ struct mthca_mpt_entry {
 	__be64 mtt_seg;
 	__be32 mtt_sz;		/* Arbel only */
 	u32    reserved[2];
-} __attribute__((packed));
+} __packed;
 
 #define MTHCA_MPT_FLAG_SW_OWNS       (0xfUL << 28)
 #define MTHCA_MPT_FLAG_MIO           (1 << 17)
diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c
index d063d7a37762..4f40dfedf920 100644
--- a/drivers/infiniband/hw/mthca/mthca_provider.c
+++ b/drivers/infiniband/hw/mthca/mthca_provider.c
@@ -363,18 +363,17 @@ static int mthca_mmap_uar(struct ib_ucontext *context,
 	return 0;
 }
 
-static int mthca_alloc_pd(struct ib_pd *ibpd, struct ib_ucontext *context,
-			  struct ib_udata *udata)
+static int mthca_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 {
 	struct ib_device *ibdev = ibpd->device;
 	struct mthca_pd *pd = to_mpd(ibpd);
 	int err;
 
-	err = mthca_pd_alloc(to_mdev(ibdev), !context, pd);
+	err = mthca_pd_alloc(to_mdev(ibdev), !udata, pd);
 	if (err)
 		return err;
 
-	if (context) {
+	if (udata) {
 		if (ib_copy_to_udata(udata, &pd->pd_num, sizeof (__u32))) {
 			mthca_pd_free(to_mdev(ibdev), pd);
 			return -EFAULT;
@@ -384,114 +383,86 @@ static int mthca_alloc_pd(struct ib_pd *ibpd, struct ib_ucontext *context,
 	return 0;
 }
 
-static void mthca_dealloc_pd(struct ib_pd *pd)
+static void mthca_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)
 {
 	mthca_pd_free(to_mdev(pd->device), to_mpd(pd));
 }
 
-static struct ib_ah *mthca_ah_create(struct ib_pd *pd,
-				     struct rdma_ah_attr *ah_attr,
-				     u32 flags,
-				     struct ib_udata *udata)
+static int mthca_ah_create(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr,
+			   u32 flags, struct ib_udata *udata)
 
 {
-	int err;
-	struct mthca_ah *ah;
+	struct mthca_ah *ah = to_mah(ibah);
 
-	ah = kmalloc(sizeof *ah, GFP_ATOMIC);
-	if (!ah)
-		return ERR_PTR(-ENOMEM);
-
-	err = mthca_create_ah(to_mdev(pd->device), to_mpd(pd), ah_attr, ah);
-	if (err) {
-		kfree(ah);
-		return ERR_PTR(err);
-	}
-
-	return &ah->ibah;
+	return mthca_create_ah(to_mdev(ibah->device), to_mpd(ibah->pd), ah_attr,
+			       ah);
 }
 
-static int mthca_ah_destroy(struct ib_ah *ah, u32 flags)
+static void mthca_ah_destroy(struct ib_ah *ah, u32 flags)
 {
 	mthca_destroy_ah(to_mdev(ah->device), to_mah(ah));
-	kfree(ah);
-
-	return 0;
 }
 
-static struct ib_srq *mthca_create_srq(struct ib_pd *pd,
-				       struct ib_srq_init_attr *init_attr,
-				       struct ib_udata *udata)
+static int mthca_create_srq(struct ib_srq *ibsrq,
+			    struct ib_srq_init_attr *init_attr,
+			    struct ib_udata *udata)
 {
 	struct mthca_create_srq ucmd;
 	struct mthca_ucontext *context = rdma_udata_to_drv_context(
 		udata, struct mthca_ucontext, ibucontext);
-	struct mthca_srq *srq;
+	struct mthca_srq *srq = to_msrq(ibsrq);
 	int err;
 
 	if (init_attr->srq_type != IB_SRQT_BASIC)
-		return ERR_PTR(-EOPNOTSUPP);
-
-	srq = kmalloc(sizeof *srq, GFP_KERNEL);
-	if (!srq)
-		return ERR_PTR(-ENOMEM);
+		return -EOPNOTSUPP;
 
 	if (udata) {
-		if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) {
-			err = -EFAULT;
-			goto err_free;
-		}
+		if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)))
+			return -EFAULT;
 
-		err = mthca_map_user_db(to_mdev(pd->device), &context->uar,
+		err = mthca_map_user_db(to_mdev(ibsrq->device), &context->uar,
 					context->db_tab, ucmd.db_index,
 					ucmd.db_page);
 
 		if (err)
-			goto err_free;
+			return err;
 
 		srq->mr.ibmr.lkey = ucmd.lkey;
 		srq->db_index     = ucmd.db_index;
 	}
 
-	err = mthca_alloc_srq(to_mdev(pd->device), to_mpd(pd),
+	err = mthca_alloc_srq(to_mdev(ibsrq->device), to_mpd(ibsrq->pd),
 			      &init_attr->attr, srq, udata);
 
 	if (err && udata)
-		mthca_unmap_user_db(to_mdev(pd->device), &context->uar,
+		mthca_unmap_user_db(to_mdev(ibsrq->device), &context->uar,
 				    context->db_tab, ucmd.db_index);
 
 	if (err)
-		goto err_free;
+		return err;
 
-	if (context && ib_copy_to_udata(udata, &srq->srqn, sizeof (__u32))) {
-		mthca_free_srq(to_mdev(pd->device), srq);
-		err = -EFAULT;
-		goto err_free;
+	if (context && ib_copy_to_udata(udata, &srq->srqn, sizeof(__u32))) {
+		mthca_free_srq(to_mdev(ibsrq->device), srq);
+		return -EFAULT;
 	}
 
-	return &srq->ibsrq;
-
-err_free:
-	kfree(srq);
-
-	return ERR_PTR(err);
+	return 0;
 }
 
-static int mthca_destroy_srq(struct ib_srq *srq)
+static void mthca_destroy_srq(struct ib_srq *srq, struct ib_udata *udata)
 {
-	struct mthca_ucontext *context;
-
-	if (srq->uobject) {
-		context = to_mucontext(srq->uobject->context);
+	if (udata) {
+		struct mthca_ucontext *context =
+			rdma_udata_to_drv_context(
+				udata,
+				struct mthca_ucontext,
+				ibucontext);
 
 		mthca_unmap_user_db(to_mdev(srq->device), &context->uar,
 				    context->db_tab, to_msrq(srq)->db_index);
 	}
 
 	mthca_free_srq(to_mdev(srq->device), to_msrq(srq));
-	kfree(srq);
-
-	return 0;
 }
 
 static struct ib_qp *mthca_create_qp(struct ib_pd *pd,
@@ -607,16 +578,22 @@ static struct ib_qp *mthca_create_qp(struct ib_pd *pd,
 	return &qp->ibqp;
 }
 
-static int mthca_destroy_qp(struct ib_qp *qp)
+static int mthca_destroy_qp(struct ib_qp *qp, struct ib_udata *udata)
 {
-	if (qp->uobject) {
+	if (udata) {
+		struct mthca_ucontext *context =
+			rdma_udata_to_drv_context(
+				udata,
+				struct mthca_ucontext,
+				ibucontext);
+
 		mthca_unmap_user_db(to_mdev(qp->device),
-				    &to_mucontext(qp->uobject->context)->uar,
-				    to_mucontext(qp->uobject->context)->db_tab,
+				    &context->uar,
+				    context->db_tab,
 				    to_mqp(qp)->sq.db_index);
 		mthca_unmap_user_db(to_mdev(qp->device),
-				    &to_mucontext(qp->uobject->context)->uar,
-				    to_mucontext(qp->uobject->context)->db_tab,
+				    &context->uar,
+				    context->db_tab,
 				    to_mqp(qp)->rq.db_index);
 	}
 	mthca_free_qp(to_mdev(qp->device), to_mqp(qp));
@@ -626,7 +603,6 @@ static int mthca_destroy_qp(struct ib_qp *qp)
 
 static struct ib_cq *mthca_create_cq(struct ib_device *ibdev,
 				     const struct ib_cq_init_attr *attr,
-				     struct ib_ucontext *context,
 				     struct ib_udata *udata)
 {
 	int entries = attr->cqe;
@@ -634,6 +610,8 @@ static struct ib_cq *mthca_create_cq(struct ib_device *ibdev,
 	struct mthca_cq *cq;
 	int nent;
 	int err;
+	struct mthca_ucontext *context = rdma_udata_to_drv_context(
+		udata, struct mthca_ucontext, ibucontext);
 
 	if (attr->flags)
 		return ERR_PTR(-EINVAL);
@@ -641,19 +619,19 @@ static struct ib_cq *mthca_create_cq(struct ib_device *ibdev,
 	if (entries < 1 || entries > to_mdev(ibdev)->limits.max_cqes)
 		return ERR_PTR(-EINVAL);
 
-	if (context) {
+	if (udata) {
 		if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd))
 			return ERR_PTR(-EFAULT);
 
-		err = mthca_map_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
-					to_mucontext(context)->db_tab,
-					ucmd.set_db_index, ucmd.set_db_page);
+		err = mthca_map_user_db(to_mdev(ibdev), &context->uar,
+					context->db_tab, ucmd.set_db_index,
+					ucmd.set_db_page);
 		if (err)
 			return ERR_PTR(err);
 
-		err = mthca_map_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
-					to_mucontext(context)->db_tab,
-					ucmd.arm_db_index, ucmd.arm_db_page);
+		err = mthca_map_user_db(to_mdev(ibdev), &context->uar,
+					context->db_tab, ucmd.arm_db_index,
+					ucmd.arm_db_page);
 		if (err)
 			goto err_unmap_set;
 	}
@@ -664,7 +642,7 @@ static struct ib_cq *mthca_create_cq(struct ib_device *ibdev,
 		goto err_unmap_arm;
 	}
 
-	if (context) {
+	if (udata) {
 		cq->buf.mr.ibmr.lkey = ucmd.lkey;
 		cq->set_ci_db_index  = ucmd.set_db_index;
 		cq->arm_db_index     = ucmd.arm_db_index;
@@ -673,14 +651,13 @@ static struct ib_cq *mthca_create_cq(struct ib_device *ibdev,
 	for (nent = 1; nent <= entries; nent <<= 1)
 		; /* nothing */
 
-	err = mthca_init_cq(to_mdev(ibdev), nent,
-			    context ? to_mucontext(context) : NULL,
-			    context ? ucmd.pdn : to_mdev(ibdev)->driver_pd.pd_num,
+	err = mthca_init_cq(to_mdev(ibdev), nent, context,
+			    udata ? ucmd.pdn : to_mdev(ibdev)->driver_pd.pd_num,
 			    cq);
 	if (err)
 		goto err_free;
 
-	if (context && ib_copy_to_udata(udata, &cq->cqn, sizeof (__u32))) {
+	if (udata && ib_copy_to_udata(udata, &cq->cqn, sizeof(__u32))) {
 		mthca_free_cq(to_mdev(ibdev), cq);
 		err = -EFAULT;
 		goto err_free;
@@ -694,14 +671,14 @@ err_free:
 	kfree(cq);
 
 err_unmap_arm:
-	if (context)
-		mthca_unmap_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
-				    to_mucontext(context)->db_tab, ucmd.arm_db_index);
+	if (udata)
+		mthca_unmap_user_db(to_mdev(ibdev), &context->uar,
+				    context->db_tab, ucmd.arm_db_index);
 
 err_unmap_set:
-	if (context)
-		mthca_unmap_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
-				    to_mucontext(context)->db_tab, ucmd.set_db_index);
+	if (udata)
+		mthca_unmap_user_db(to_mdev(ibdev), &context->uar,
+				    context->db_tab, ucmd.set_db_index);
 
 	return ERR_PTR(err);
 }
@@ -827,16 +804,22 @@ out:
 	return ret;
 }
 
-static int mthca_destroy_cq(struct ib_cq *cq)
+static int mthca_destroy_cq(struct ib_cq *cq, struct ib_udata *udata)
 {
-	if (cq->uobject) {
+	if (udata) {
+		struct mthca_ucontext *context =
+			rdma_udata_to_drv_context(
+				udata,
+				struct mthca_ucontext,
+				ibucontext);
+
 		mthca_unmap_user_db(to_mdev(cq->device),
-				    &to_mucontext(cq->uobject->context)->uar,
-				    to_mucontext(cq->uobject->context)->db_tab,
+				    &context->uar,
+				    context->db_tab,
 				    to_mcq(cq)->arm_db_index);
 		mthca_unmap_user_db(to_mdev(cq->device),
-				    &to_mucontext(cq->uobject->context)->uar,
-				    to_mucontext(cq->uobject->context)->db_tab,
+				    &context->uar,
+				    context->db_tab,
 				    to_mcq(cq)->set_ci_db_index);
 	}
 	mthca_free_cq(to_mdev(cq->device), to_mcq(cq));
@@ -914,7 +897,7 @@ static struct ib_mr *mthca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 		goto err;
 	}
 
-	n = mr->umem->nmap;
+	n = ib_umem_num_pages(mr->umem);
 
 	mr->mtt = mthca_alloc_mtt(dev, n);
 	if (IS_ERR(mr->mtt)) {
@@ -974,7 +957,7 @@ err:
 	return ERR_PTR(err);
 }
 
-static int mthca_dereg_mr(struct ib_mr *mr)
+static int mthca_dereg_mr(struct ib_mr *mr, struct ib_udata *udata)
 {
 	struct mthca_mr *mmr = to_mmr(mr);
 
@@ -1200,6 +1183,8 @@ static const struct ib_device_ops mthca_dev_ops = {
 	.query_qp = mthca_query_qp,
 	.reg_user_mr = mthca_reg_user_mr,
 	.resize_cq = mthca_resize_cq,
+
+	INIT_RDMA_OBJ_SIZE(ib_ah, mthca_ah, ibah),
 	INIT_RDMA_OBJ_SIZE(ib_pd, mthca_pd, ibpd),
 	INIT_RDMA_OBJ_SIZE(ib_ucontext, mthca_ucontext, ibucontext),
 };
@@ -1210,6 +1195,8 @@ static const struct ib_device_ops mthca_dev_arbel_srq_ops = {
 	.modify_srq = mthca_modify_srq,
 	.post_srq_recv = mthca_arbel_post_srq_recv,
 	.query_srq = mthca_query_srq,
+
+	INIT_RDMA_OBJ_SIZE(ib_srq, mthca_srq, ibsrq),
 };
 
 static const struct ib_device_ops mthca_dev_tavor_srq_ops = {
@@ -1218,6 +1205,8 @@ static const struct ib_device_ops mthca_dev_tavor_srq_ops = {
 	.modify_srq = mthca_modify_srq,
 	.post_srq_recv = mthca_tavor_post_srq_recv,
 	.query_srq = mthca_query_srq,
+
+	INIT_RDMA_OBJ_SIZE(ib_srq, mthca_srq, ibsrq),
 };
 
 static const struct ib_device_ops mthca_dev_arbel_fmr_ops = {
diff --git a/drivers/infiniband/hw/mthca/mthca_qp.c b/drivers/infiniband/hw/mthca/mthca_qp.c
index d65b189f20ea..d04c245359eb 100644
--- a/drivers/infiniband/hw/mthca/mthca_qp.c
+++ b/drivers/infiniband/hw/mthca/mthca_qp.c
@@ -115,7 +115,7 @@ struct mthca_qp_path {
 	u8     hop_limit;
 	__be32 sl_tclass_flowlabel;
 	u8     rgid[16];
-} __attribute__((packed));
+} __packed;
 
 struct mthca_qp_context {
 	__be32 flags;
@@ -154,14 +154,14 @@ struct mthca_qp_context {
 	__be16 rq_wqe_counter;	/* reserved on Tavor */
 	__be16 sq_wqe_counter;	/* reserved on Tavor */
 	u32    reserved3[18];
-} __attribute__((packed));
+} __packed;
 
 struct mthca_qp_param {
 	__be32 opt_param_mask;
 	u32    reserved1;
 	struct mthca_qp_context context;
 	u32    reserved2[62];
-} __attribute__((packed));
+} __packed;
 
 enum {
 	MTHCA_QP_OPTPAR_ALT_ADDR_PATH     = 1 << 0,
diff --git a/drivers/infiniband/hw/nes/nes_cm.c b/drivers/infiniband/hw/nes/nes_cm.c
index 0010a3ed64f1..62bf986eba67 100644
--- a/drivers/infiniband/hw/nes/nes_cm.c
+++ b/drivers/infiniband/hw/nes/nes_cm.c
@@ -3033,7 +3033,8 @@ static int nes_disconnect(struct nes_qp *nesqp, int abrupt)
 		/* Need to free the Last Streaming Mode Message */
 		if (nesqp->ietf_frame) {
 			if (nesqp->lsmm_mr)
-				nesibdev->ibdev.ops.dereg_mr(nesqp->lsmm_mr);
+				nesibdev->ibdev.ops.dereg_mr(nesqp->lsmm_mr,
+							     NULL);
 			pci_free_consistent(nesdev->pcidev,
 					    nesqp->private_data_len + nesqp->ietf_frame_size,
 					    nesqp->ietf_frame, nesqp->ietf_frame_pbase);
diff --git a/drivers/infiniband/hw/nes/nes_verbs.c b/drivers/infiniband/hw/nes/nes_verbs.c
index 828e4af3f951..49024326a518 100644
--- a/drivers/infiniband/hw/nes/nes_verbs.c
+++ b/drivers/infiniband/hw/nes/nes_verbs.c
@@ -52,7 +52,7 @@ atomic_t qps_created;
 atomic_t sw_qps_destroyed;
 
 static void nes_unregister_ofa_device(struct nes_ib_device *nesibdev);
-static int nes_dereg_mr(struct ib_mr *ib_mr);
+static int nes_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata);
 
 /**
  * nes_alloc_mw
@@ -306,9 +306,8 @@ static int alloc_fast_reg_mr(struct nes_device *nesdev, struct nes_pd *nespd,
 /*
  * nes_alloc_mr
  */
-static struct ib_mr *nes_alloc_mr(struct ib_pd *ibpd,
-				  enum ib_mr_type mr_type,
-				  u32 max_num_sg)
+static struct ib_mr *nes_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
+				  u32 max_num_sg, struct ib_udata *udata)
 {
 	struct nes_pd *nespd = to_nespd(ibpd);
 	struct nes_vnic *nesvnic = to_nesvnic(ibpd->device);
@@ -386,7 +385,7 @@ static struct ib_mr *nes_alloc_mr(struct ib_pd *ibpd,
 	return ibmr;
 
 err:
-	nes_dereg_mr(ibmr);
+	nes_dereg_mr(ibmr, udata);
 
 	return ERR_PTR(-ENOMEM);
 }
@@ -641,22 +640,24 @@ static int nes_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
 /**
  * nes_alloc_pd
  */
-static int nes_alloc_pd(struct ib_pd *pd, struct ib_ucontext *context,
-			struct ib_udata *udata)
+static int nes_alloc_pd(struct ib_pd *pd, struct ib_udata *udata)
 {
 	struct ib_device *ibdev = pd->device;
 	struct nes_pd *nespd = to_nespd(pd);
 	struct nes_vnic *nesvnic = to_nesvnic(ibdev);
 	struct nes_device *nesdev = nesvnic->nesdev;
 	struct nes_adapter *nesadapter = nesdev->nesadapter;
-	struct nes_ucontext *nesucontext;
 	struct nes_alloc_pd_resp uresp;
 	u32 pd_num = 0;
 	int err;
+	struct nes_ucontext *nesucontext = rdma_udata_to_drv_context(
+		udata, struct nes_ucontext, ibucontext);
 
-	nes_debug(NES_DBG_PD, "nesvnic=%p, netdev=%p %s, ibdev=%p, context=%p, netdev refcnt=%u\n",
-			nesvnic, nesdev->netdev[0], nesdev->netdev[0]->name, ibdev, context,
-			netdev_refcnt_read(nesvnic->netdev));
+	nes_debug(
+		NES_DBG_PD,
+		"nesvnic=%p, netdev=%p %s, ibdev=%p, context=%p, netdev refcnt=%u\n",
+		nesvnic, nesdev->netdev[0], nesdev->netdev[0]->name, ibdev,
+		&nesucontext->ibucontext, netdev_refcnt_read(nesvnic->netdev));
 
 	err = nes_alloc_resource(nesadapter, nesadapter->allocated_pds,
 			nesadapter->max_pd, &pd_num, &nesadapter->next_pd, NES_RESOURCE_PD);
@@ -668,8 +669,7 @@ static int nes_alloc_pd(struct ib_pd *pd, struct ib_ucontext *context,
 
 	nespd->pd_id = (pd_num << (PAGE_SHIFT-12)) + nesadapter->base_pd;
 
-	if (context) {
-		nesucontext = to_nesucontext(context);
+	if (udata) {
 		nespd->mmap_db_index = find_next_zero_bit(nesucontext->allocated_doorbells,
 				NES_MAX_USER_DB_REGIONS, nesucontext->first_free_db);
 		nes_debug(NES_DBG_PD, "find_first_zero_biton doorbells returned %u, mapping pd_id %u.\n",
@@ -700,7 +700,7 @@ static int nes_alloc_pd(struct ib_pd *pd, struct ib_ucontext *context,
 /**
  * nes_dealloc_pd
  */
-static void nes_dealloc_pd(struct ib_pd *ibpd)
+static void nes_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 {
 	struct nes_ucontext *nesucontext;
 	struct nes_pd *nespd = to_nespd(ibpd);
@@ -708,8 +708,12 @@ static void nes_dealloc_pd(struct ib_pd *ibpd)
 	struct nes_device *nesdev = nesvnic->nesdev;
 	struct nes_adapter *nesadapter = nesdev->nesadapter;
 
-	if ((ibpd->uobject) && (ibpd->uobject->context)) {
-		nesucontext = to_nesucontext(ibpd->uobject->context);
+	if (udata) {
+		nesucontext =
+			rdma_udata_to_drv_context(
+				udata,
+				struct nes_ucontext,
+				ibucontext);
 		nes_debug(NES_DBG_PD, "Clearing bit %u from allocated doorbells\n",
 				nespd->mmap_db_index);
 		clear_bit(nespd->mmap_db_index, nesucontext->allocated_doorbells);
@@ -1039,53 +1043,48 @@ static struct ib_qp *nes_create_qp(struct ib_pd *ibpd,
 				}
 				if (req.user_qp_buffer)
 					nesqp->nesuqp_addr = req.user_qp_buffer;
-				if (udata) {
-					nesqp->user_mode = 1;
-					if (virt_wqs) {
-						err = 1;
-						list_for_each_entry(nespbl, &nes_ucontext->qp_reg_mem_list, list) {
-							if (nespbl->user_base == (unsigned long )req.user_wqe_buffers) {
-								list_del(&nespbl->list);
-								err = 0;
-								nes_debug(NES_DBG_QP, "Found PBL for virtual QP. nespbl=%p. user_base=0x%lx\n",
-									  nespbl, nespbl->user_base);
-								break;
-							}
-						}
-						if (err) {
-							nes_debug(NES_DBG_QP, "Didn't Find PBL for virtual QP. address = %llx.\n",
-								  (long long unsigned int)req.user_wqe_buffers);
-							nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num);
-							kfree(nesqp->allocated_buffer);
-							return ERR_PTR(-EFAULT);
+
+				nesqp->user_mode = 1;
+				if (virt_wqs) {
+					err = 1;
+					list_for_each_entry(nespbl, &nes_ucontext->qp_reg_mem_list, list) {
+						if (nespbl->user_base == (unsigned long )req.user_wqe_buffers) {
+							list_del(&nespbl->list);
+							err = 0;
+							nes_debug(NES_DBG_QP, "Found PBL for virtual QP. nespbl=%p. user_base=0x%lx\n",
+								  nespbl, nespbl->user_base);
+							break;
 						}
 					}
-
-					nesqp->mmap_sq_db_index =
-						find_next_zero_bit(nes_ucontext->allocated_wqs,
-								   NES_MAX_USER_WQ_REGIONS, nes_ucontext->first_free_wq);
-					/* nes_debug(NES_DBG_QP, "find_first_zero_biton wqs returned %u\n",
-							nespd->mmap_db_index); */
-					if (nesqp->mmap_sq_db_index >= NES_MAX_USER_WQ_REGIONS) {
-						nes_debug(NES_DBG_QP,
-							  "db index > max user regions, failing create QP\n");
+					if (err) {
+						nes_debug(NES_DBG_QP, "Didn't Find PBL for virtual QP. address = %llx.\n",
+							  (long long unsigned int)req.user_wqe_buffers);
 						nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num);
-						if (virt_wqs) {
-							pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, nespbl->pbl_vbase,
-									    nespbl->pbl_pbase);
-							kfree(nespbl);
-						}
 						kfree(nesqp->allocated_buffer);
-						return ERR_PTR(-ENOMEM);
+						return ERR_PTR(-EFAULT);
 					}
-					set_bit(nesqp->mmap_sq_db_index, nes_ucontext->allocated_wqs);
-					nes_ucontext->mmap_nesqp[nesqp->mmap_sq_db_index] = nesqp;
-					nes_ucontext->first_free_wq = nesqp->mmap_sq_db_index + 1;
-				} else {
+				}
+
+				nesqp->mmap_sq_db_index =
+					find_next_zero_bit(nes_ucontext->allocated_wqs,
+							   NES_MAX_USER_WQ_REGIONS, nes_ucontext->first_free_wq);
+				/* nes_debug(NES_DBG_QP, "find_first_zero_biton wqs returned %u\n",
+						nespd->mmap_db_index); */
+				if (nesqp->mmap_sq_db_index >= NES_MAX_USER_WQ_REGIONS) {
+					nes_debug(NES_DBG_QP,
+						  "db index > max user regions, failing create QP\n");
 					nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num);
+					if (virt_wqs) {
+						pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, nespbl->pbl_vbase,
+								    nespbl->pbl_pbase);
+						kfree(nespbl);
+					}
 					kfree(nesqp->allocated_buffer);
-					return ERR_PTR(-EFAULT);
+					return ERR_PTR(-ENOMEM);
 				}
+				set_bit(nesqp->mmap_sq_db_index, nes_ucontext->allocated_wqs);
+				nes_ucontext->mmap_nesqp[nesqp->mmap_sq_db_index] = nesqp;
+				nes_ucontext->first_free_wq = nesqp->mmap_sq_db_index + 1;
 			}
 			err = (!virt_wqs) ? nes_setup_mmap_qp(nesqp, nesvnic, sq_size, rq_size) :
 					nes_setup_virt_qp(nesqp, nespbl, nesvnic, sq_size, rq_size);
@@ -1303,7 +1302,7 @@ static void nes_clean_cq(struct nes_qp *nesqp, struct nes_cq *nescq)
 /**
  * nes_destroy_qp
  */
-static int nes_destroy_qp(struct ib_qp *ibqp)
+static int nes_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
 {
 	struct nes_qp *nesqp = to_nesqp(ibqp);
 	struct nes_ucontext *nes_ucontext;
@@ -1343,8 +1342,12 @@ static int nes_destroy_qp(struct ib_qp *ibqp)
 	}
 
 	if (nesqp->user_mode) {
-		if ((ibqp->uobject)&&(ibqp->uobject->context)) {
-			nes_ucontext = to_nesucontext(ibqp->uobject->context);
+		if (udata) {
+			nes_ucontext =
+				rdma_udata_to_drv_context(
+					udata,
+					struct nes_ucontext,
+					ibucontext);
 			clear_bit(nesqp->mmap_sq_db_index, nes_ucontext->allocated_wqs);
 			nes_ucontext->mmap_nesqp[nesqp->mmap_sq_db_index] = NULL;
 			if (nes_ucontext->first_free_wq > nesqp->mmap_sq_db_index) {
@@ -1373,7 +1376,6 @@ static int nes_destroy_qp(struct ib_qp *ibqp)
  */
 static struct ib_cq *nes_create_cq(struct ib_device *ibdev,
 				   const struct ib_cq_init_attr *attr,
-				   struct ib_ucontext *context,
 				   struct ib_udata *udata)
 {
 	int entries = attr->cqe;
@@ -1418,9 +1420,10 @@ static struct ib_cq *nes_create_cq(struct ib_device *ibdev,
 	nescq->hw_cq.cq_number = cq_num;
 	nescq->ibcq.cqe = nescq->hw_cq.cq_size - 1;
 
+	if (udata) {
+		struct nes_ucontext *nes_ucontext = rdma_udata_to_drv_context(
+			udata, struct nes_ucontext, ibucontext);
 
-	if (context) {
-		nes_ucontext = to_nesucontext(context);
 		if (ib_copy_from_udata(&req, udata, sizeof (struct nes_create_cq_req))) {
 			nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num);
 			kfree(nescq);
@@ -1487,7 +1490,7 @@ static struct ib_cq *nes_create_cq(struct ib_device *ibdev,
 	cqp_request = nes_get_cqp_request(nesdev);
 	if (cqp_request == NULL) {
 		nes_debug(NES_DBG_CQ, "Failed to get a cqp_request.\n");
-		if (!context)
+		if (!udata)
 			pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size, mem,
 					nescq->hw_cq.cq_pbase);
 		else {
@@ -1516,7 +1519,7 @@ static struct ib_cq *nes_create_cq(struct ib_device *ibdev,
 			if (nesadapter->free_4kpbl == 0) {
 				spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
 				nes_free_cqp_request(nesdev, cqp_request);
-				if (!context)
+				if (!udata)
 					pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size, mem,
 							nescq->hw_cq.cq_pbase);
 				else {
@@ -1538,7 +1541,7 @@ static struct ib_cq *nes_create_cq(struct ib_device *ibdev,
 			if (nesadapter->free_256pbl == 0) {
 				spin_unlock_irqrestore(&nesadapter->pbl_lock, flags);
 				nes_free_cqp_request(nesdev, cqp_request);
-				if (!context)
+				if (!udata)
 					pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size, mem,
 							nescq->hw_cq.cq_pbase);
 				else {
@@ -1564,7 +1567,7 @@ static struct ib_cq *nes_create_cq(struct ib_device *ibdev,
 	set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
 			(nescq->hw_cq.cq_number | ((u32)nesdev->ceq_index << 16)));
 
-	if (context) {
+	if (udata) {
 		if (pbl_entries != 1)
 			u64temp = (u64)nespbl->pbl_pbase;
 		else
@@ -1595,7 +1598,7 @@ static struct ib_cq *nes_create_cq(struct ib_device *ibdev,
 			nescq->hw_cq.cq_number, ret);
 	if ((!ret) || (cqp_request->major_code)) {
 		nes_put_cqp_request(nesdev, cqp_request);
-		if (!context)
+		if (!udata)
 			pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size, mem,
 					nescq->hw_cq.cq_pbase);
 		else {
@@ -1609,7 +1612,7 @@ static struct ib_cq *nes_create_cq(struct ib_device *ibdev,
 	}
 	nes_put_cqp_request(nesdev, cqp_request);
 
-	if (context) {
+	if (udata) {
 		/* free the nespbl */
 		pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, nespbl->pbl_vbase,
 				nespbl->pbl_pbase);
@@ -1631,7 +1634,7 @@ static struct ib_cq *nes_create_cq(struct ib_device *ibdev,
 /**
  * nes_destroy_cq
  */
-static int nes_destroy_cq(struct ib_cq *ib_cq)
+static int nes_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
 {
 	struct nes_cq *nescq;
 	struct nes_device *nesdev;
@@ -2382,7 +2385,7 @@ reg_user_mr_err:
 /**
  * nes_dereg_mr
  */
-static int nes_dereg_mr(struct ib_mr *ib_mr)
+static int nes_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
 {
 	struct nes_mr *nesmr = to_nesmr(ib_mr);
 	struct nes_vnic *nesvnic = to_nesvnic(ib_mr->device);
@@ -3574,6 +3577,14 @@ static const struct ib_device_ops nes_dev_ops = {
 	.get_dev_fw_str = get_dev_fw_str,
 	.get_dma_mr = nes_get_dma_mr,
 	.get_port_immutable = nes_port_immutable,
+	.iw_accept = nes_accept,
+	.iw_add_ref = nes_add_ref,
+	.iw_connect = nes_connect,
+	.iw_create_listen = nes_create_listen,
+	.iw_destroy_listen = nes_destroy_listen,
+	.iw_get_qp = nes_get_qp,
+	.iw_reject = nes_reject,
+	.iw_rem_ref = nes_rem_ref,
 	.map_mr_sg = nes_map_mr_sg,
 	.mmap = nes_mmap,
 	.modify_qp = nes_modify_qp,
@@ -3638,23 +3649,9 @@ struct nes_ib_device *nes_init_ofa_device(struct net_device *netdev)
 	nesibdev->ibdev.num_comp_vectors = 1;
 	nesibdev->ibdev.dev.parent = &nesdev->pcidev->dev;
 
-	nesibdev->ibdev.iwcm = kzalloc(sizeof(*nesibdev->ibdev.iwcm), GFP_KERNEL);
-	if (nesibdev->ibdev.iwcm == NULL) {
-		ib_dealloc_device(&nesibdev->ibdev);
-		return NULL;
-	}
-	nesibdev->ibdev.iwcm->add_ref = nes_add_ref;
-	nesibdev->ibdev.iwcm->rem_ref = nes_rem_ref;
-	nesibdev->ibdev.iwcm->get_qp = nes_get_qp;
-	nesibdev->ibdev.iwcm->connect = nes_connect;
-	nesibdev->ibdev.iwcm->accept = nes_accept;
-	nesibdev->ibdev.iwcm->reject = nes_reject;
-	nesibdev->ibdev.iwcm->create_listen = nes_create_listen;
-	nesibdev->ibdev.iwcm->destroy_listen = nes_destroy_listen;
-
 	ib_set_device_ops(&nesibdev->ibdev, &nes_dev_ops);
-	memcpy(nesibdev->ibdev.iwcm->ifname, netdev->name,
-	       sizeof(nesibdev->ibdev.iwcm->ifname));
+	memcpy(nesibdev->ibdev.iw_ifname, netdev->name,
+	       sizeof(nesibdev->ibdev.iw_ifname));
 
 	return nesibdev;
 }
@@ -3715,7 +3712,6 @@ void nes_destroy_ofa_device(struct nes_ib_device *nesibdev)
 
 	nes_unregister_ofa_device(nesibdev);
 
-	kfree(nesibdev->ibdev.iwcm);
 	ib_dealloc_device(&nesibdev->ibdev);
 }
 
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_ah.c b/drivers/infiniband/hw/ocrdma/ocrdma_ah.c
index a7295322efbc..1d4ea135c28f 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_ah.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_ah.c
@@ -156,37 +156,34 @@ static inline int set_av_attr(struct ocrdma_dev *dev, struct ocrdma_ah *ah,
 	return status;
 }
 
-struct ib_ah *ocrdma_create_ah(struct ib_pd *ibpd, struct rdma_ah_attr *attr,
-			       u32 flags, struct ib_udata *udata)
+int ocrdma_create_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr, u32 flags,
+		     struct ib_udata *udata)
 {
 	u32 *ahid_addr;
 	int status;
-	struct ocrdma_ah *ah;
+	struct ocrdma_ah *ah = get_ocrdma_ah(ibah);
 	bool isvlan = false;
 	u16 vlan_tag = 0xffff;
 	const struct ib_gid_attr *sgid_attr;
-	struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
-	struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
+	struct ocrdma_pd *pd = get_ocrdma_pd(ibah->pd);
+	struct ocrdma_dev *dev = get_ocrdma_dev(ibah->device);
 
 	if ((attr->type != RDMA_AH_ATTR_TYPE_ROCE) ||
 	    !(rdma_ah_get_ah_flags(attr) & IB_AH_GRH))
-		return ERR_PTR(-EINVAL);
+		return -EINVAL;
 
 	if (atomic_cmpxchg(&dev->update_sl, 1, 0))
 		ocrdma_init_service_level(dev);
 
-	ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
-	if (!ah)
-		return ERR_PTR(-ENOMEM);
+	sgid_attr = attr->grh.sgid_attr;
+	status = rdma_read_gid_l2_fields(sgid_attr, &vlan_tag, NULL);
+	if (status)
+		return status;
 
 	status = ocrdma_alloc_av(dev, ah);
 	if (status)
 		goto av_err;
 
-	sgid_attr = attr->grh.sgid_attr;
-	if (is_vlan_dev(sgid_attr->ndev))
-		vlan_tag = vlan_dev_vlan_id(sgid_attr->ndev);
-
 	/* Get network header type for this GID */
 	ah->hdr_type = rdma_gid_attr_network_type(sgid_attr);
 
@@ -210,23 +207,20 @@ struct ib_ah *ocrdma_create_ah(struct ib_pd *ibpd, struct rdma_ah_attr *attr,
 				       OCRDMA_AH_VLAN_VALID_SHIFT);
 	}
 
-	return &ah->ibah;
+	return 0;
 
 av_conf_err:
 	ocrdma_free_av(dev, ah);
 av_err:
-	kfree(ah);
-	return ERR_PTR(status);
+	return status;
 }
 
-int ocrdma_destroy_ah(struct ib_ah *ibah, u32 flags)
+void ocrdma_destroy_ah(struct ib_ah *ibah, u32 flags)
 {
 	struct ocrdma_ah *ah = get_ocrdma_ah(ibah);
 	struct ocrdma_dev *dev = get_ocrdma_dev(ibah->device);
 
 	ocrdma_free_av(dev, ah);
-	kfree(ah);
-	return 0;
 }
 
 int ocrdma_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr)
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_ah.h b/drivers/infiniband/hw/ocrdma/ocrdma_ah.h
index eb996e14b520..64cb82c08664 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_ah.h
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_ah.h
@@ -51,9 +51,9 @@ enum {
 	OCRDMA_AH_L3_TYPE_SHIFT		= 0x1D /* 29 bits */
 };
 
-struct ib_ah *ocrdma_create_ah(struct ib_pd *pd, struct rdma_ah_attr *ah_attr,
-			       u32 flags, struct ib_udata *udata);
-int ocrdma_destroy_ah(struct ib_ah *ah, u32 flags);
+int ocrdma_create_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr, u32 flags,
+		     struct ib_udata *udata);
+void ocrdma_destroy_ah(struct ib_ah *ah, u32 flags);
 int ocrdma_query_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);
 
 int ocrdma_process_mad(struct ib_device *,
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_hw.c b/drivers/infiniband/hw/ocrdma/ocrdma_hw.c
index 097e5ab2a19f..32674b291f60 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_hw.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_hw.c
@@ -2496,7 +2496,7 @@ static int ocrdma_set_av_params(struct ocrdma_qp *qp,
 	int status;
 	struct rdma_ah_attr *ah_attr = &attrs->ah_attr;
 	const struct ib_gid_attr *sgid_attr;
-	u32 vlan_id = 0xFFFF;
+	u16 vlan_id = 0xFFFF;
 	u8 mac_addr[6], hdr_type;
 	union {
 		struct sockaddr     _sockaddr;
@@ -2526,8 +2526,9 @@ static int ocrdma_set_av_params(struct ocrdma_qp *qp,
 	       sizeof(cmd->params.dgid));
 
 	sgid_attr = ah_attr->grh.sgid_attr;
-	vlan_id = rdma_vlan_dev_vlan_id(sgid_attr->ndev);
-	memcpy(mac_addr, sgid_attr->ndev->dev_addr, ETH_ALEN);
+	status = rdma_read_gid_l2_fields(sgid_attr, &vlan_id, &mac_addr[0]);
+	if (status)
+		return status;
 
 	qp->sgid_idx = grh->sgid_index;
 	memcpy(&cmd->params.sgid[0], &sgid_attr->gid.raw[0],
@@ -2863,21 +2864,19 @@ int ocrdma_mbx_query_srq(struct ocrdma_srq *srq, struct ib_srq_attr *srq_attr)
 	return status;
 }
 
-int ocrdma_mbx_destroy_srq(struct ocrdma_dev *dev, struct ocrdma_srq *srq)
+void ocrdma_mbx_destroy_srq(struct ocrdma_dev *dev, struct ocrdma_srq *srq)
 {
-	int status = -ENOMEM;
 	struct ocrdma_destroy_srq *cmd;
 	struct pci_dev *pdev = dev->nic_info.pdev;
 	cmd = ocrdma_init_emb_mqe(OCRDMA_CMD_DELETE_SRQ, sizeof(*cmd));
 	if (!cmd)
-		return status;
+		return;
 	cmd->id = srq->id;
-	status = ocrdma_mbx_cmd(dev, (struct ocrdma_mqe *)cmd);
+	ocrdma_mbx_cmd(dev, (struct ocrdma_mqe *)cmd);
 	if (srq->rq.va)
 		dma_free_coherent(&pdev->dev, srq->rq.len,
 				  srq->rq.va, srq->rq.pa);
 	kfree(cmd);
-	return status;
 }
 
 static int ocrdma_mbx_get_dcbx_config(struct ocrdma_dev *dev, u32 ptype,
@@ -3067,13 +3066,12 @@ int ocrdma_alloc_av(struct ocrdma_dev *dev, struct ocrdma_ah *ah)
 	return status;
 }
 
-int ocrdma_free_av(struct ocrdma_dev *dev, struct ocrdma_ah *ah)
+void ocrdma_free_av(struct ocrdma_dev *dev, struct ocrdma_ah *ah)
 {
 	unsigned long flags;
 	spin_lock_irqsave(&dev->av_tbl.lock, flags);
 	ah->av->valid = 0;
 	spin_unlock_irqrestore(&dev->av_tbl.lock, flags);
-	return 0;
 }
 
 static int ocrdma_create_eqs(struct ocrdma_dev *dev)
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_hw.h b/drivers/infiniband/hw/ocrdma/ocrdma_hw.h
index ebc1f442aec3..06ec59326a90 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_hw.h
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_hw.h
@@ -137,10 +137,10 @@ int ocrdma_mbx_create_srq(struct ocrdma_dev *, struct ocrdma_srq *,
 			  struct ocrdma_pd *);
 int ocrdma_mbx_modify_srq(struct ocrdma_srq *, struct ib_srq_attr *);
 int ocrdma_mbx_query_srq(struct ocrdma_srq *, struct ib_srq_attr *);
-int ocrdma_mbx_destroy_srq(struct ocrdma_dev *, struct ocrdma_srq *);
+void ocrdma_mbx_destroy_srq(struct ocrdma_dev *dev, struct ocrdma_srq *srq);
 
-int ocrdma_alloc_av(struct ocrdma_dev *, struct ocrdma_ah *);
-int ocrdma_free_av(struct ocrdma_dev *, struct ocrdma_ah *);
+int ocrdma_alloc_av(struct ocrdma_dev *dev, struct ocrdma_ah *ah);
+void ocrdma_free_av(struct ocrdma_dev *dev, struct ocrdma_ah *ah);
 
 int ocrdma_qp_state_change(struct ocrdma_qp *, enum ib_qp_state new_state,
 			    enum ib_qp_state *old_ib_state);
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_main.c b/drivers/infiniband/hw/ocrdma/ocrdma_main.c
index b9e10d55a58e..fc6c0962dea9 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_main.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_main.c
@@ -62,8 +62,6 @@ MODULE_DESCRIPTION(OCRDMA_ROCE_DRV_DESC " " OCRDMA_ROCE_DRV_VERSION);
 MODULE_AUTHOR("Emulex Corporation");
 MODULE_LICENSE("Dual BSD/GPL");
 
-static DEFINE_IDR(ocrdma_dev_id);
-
 void ocrdma_get_guid(struct ocrdma_dev *dev, u8 *guid)
 {
 	u8 mac_addr[6];
@@ -161,7 +159,6 @@ static const struct ib_device_ops ocrdma_dev_ops = {
 	.get_dev_fw_str = get_dev_fw_str,
 	.get_dma_mr = ocrdma_get_dma_mr,
 	.get_link_layer = ocrdma_link_layer,
-	.get_netdev = ocrdma_get_netdev,
 	.get_port_immutable = ocrdma_port_immutable,
 	.map_mr_sg = ocrdma_map_mr_sg,
 	.mmap = ocrdma_mmap,
@@ -179,6 +176,8 @@ static const struct ib_device_ops ocrdma_dev_ops = {
 	.reg_user_mr = ocrdma_reg_user_mr,
 	.req_notify_cq = ocrdma_arm_cq,
 	.resize_cq = ocrdma_resize_cq,
+
+	INIT_RDMA_OBJ_SIZE(ib_ah, ocrdma_ah, ibah),
 	INIT_RDMA_OBJ_SIZE(ib_pd, ocrdma_pd, ibpd),
 	INIT_RDMA_OBJ_SIZE(ib_ucontext, ocrdma_ucontext, ibucontext),
 };
@@ -189,10 +188,14 @@ static const struct ib_device_ops ocrdma_dev_srq_ops = {
 	.modify_srq = ocrdma_modify_srq,
 	.post_srq_recv = ocrdma_post_srq_recv,
 	.query_srq = ocrdma_query_srq,
+
+	INIT_RDMA_OBJ_SIZE(ib_srq, ocrdma_srq, ibsrq),
 };
 
 static int ocrdma_register_device(struct ocrdma_dev *dev)
 {
+	int ret;
+
 	ocrdma_get_guid(dev, (u8 *)&dev->ibdev.node_guid);
 	BUILD_BUG_ON(sizeof(OCRDMA_NODE_DESC) > IB_DEVICE_NODE_DESC_MAX);
 	memcpy(dev->ibdev.node_desc, OCRDMA_NODE_DESC,
@@ -247,6 +250,10 @@ static int ocrdma_register_device(struct ocrdma_dev *dev)
 	}
 	rdma_set_device_sysfs_group(&dev->ibdev, &ocrdma_attr_group);
 	dev->ibdev.driver_id = RDMA_DRIVER_OCRDMA;
+	ret = ib_device_set_netdev(&dev->ibdev, dev->nic_info.netdev, 1);
+	if (ret)
+		return ret;
+
 	return ib_register_device(&dev->ibdev, "ocrdma%d");
 }
 
@@ -304,15 +311,13 @@ static struct ocrdma_dev *ocrdma_add(struct be_dev_info *dev_info)
 		pr_err("Unable to allocate ib device\n");
 		return NULL;
 	}
+
 	dev->mbx_cmd = kzalloc(sizeof(struct ocrdma_mqe_emb_cmd), GFP_KERNEL);
 	if (!dev->mbx_cmd)
-		goto idr_err;
+		goto init_err;
 
 	memcpy(&dev->nic_info, dev_info, sizeof(*dev_info));
-	dev->id = idr_alloc(&ocrdma_dev_id, NULL, 0, 0, GFP_KERNEL);
-	if (dev->id < 0)
-		goto idr_err;
-
+	dev->id = PCI_FUNC(dev->nic_info.pdev->devfn);
 	status = ocrdma_init_hw(dev);
 	if (status)
 		goto init_err;
@@ -349,8 +354,6 @@ alloc_err:
 	ocrdma_free_resources(dev);
 	ocrdma_cleanup_hw(dev);
 init_err:
-	idr_remove(&ocrdma_dev_id, dev->id);
-idr_err:
 	kfree(dev->mbx_cmd);
 	ib_dealloc_device(&dev->ibdev);
 	pr_err("%s() leaving. ret=%d\n", __func__, status);
@@ -360,7 +363,6 @@ idr_err:
 static void ocrdma_remove_free(struct ocrdma_dev *dev)
 {
 
-	idr_remove(&ocrdma_dev_id, dev->id);
 	kfree(dev->mbx_cmd);
 	ib_dealloc_device(&dev->ibdev);
 }
@@ -465,7 +467,6 @@ static void __exit ocrdma_exit_module(void)
 {
 	be_roce_unregister_driver(&ocrdma_drv);
 	ocrdma_rem_debugfs();
-	idr_destroy(&ocrdma_dev_id);
 }
 
 module_init(ocrdma_init_module);
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
index b4e1777c2c97..35ec87015792 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
@@ -47,6 +47,7 @@
 #include <rdma/ib_umem.h>
 #include <rdma/ib_addr.h>
 #include <rdma/ib_cache.h>
+#include <rdma/uverbs_ioctl.h>
 
 #include "ocrdma.h"
 #include "ocrdma_hw.h"
@@ -112,24 +113,6 @@ int ocrdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr,
 	return 0;
 }
 
-struct net_device *ocrdma_get_netdev(struct ib_device *ibdev, u8 port_num)
-{
-	struct ocrdma_dev *dev;
-	struct net_device *ndev = NULL;
-
-	rcu_read_lock();
-
-	dev = get_ocrdma_dev(ibdev);
-	if (dev)
-		ndev = dev->nic_info.netdev;
-	if (ndev)
-		dev_hold(ndev);
-
-	rcu_read_unlock();
-
-	return ndev;
-}
-
 static inline void get_link_speed_and_width(struct ocrdma_dev *dev,
 					    u8 *ib_speed, u8 *ib_width)
 {
@@ -367,6 +350,16 @@ static int ocrdma_get_pd_num(struct ocrdma_dev *dev, struct ocrdma_pd *pd)
 	return status;
 }
 
+/*
+ * NOTE:
+ *
+ * ocrdma_ucontext must be used here because this function is also
+ * called from ocrdma_alloc_ucontext where ib_udata does not have
+ * valid ib_ucontext pointer. ib_uverbs_get_context does not call
+ * uobj_{alloc|get_xxx} helpers which are used to store the
+ * ib_ucontext in uverbs_attr_bundle wrapping the ib_udata. so
+ * ib_udata does NOT imply valid ib_ucontext here!
+ */
 static int _ocrdma_alloc_pd(struct ocrdma_dev *dev, struct ocrdma_pd *pd,
 			    struct ocrdma_ucontext *uctx,
 			    struct ib_udata *udata)
@@ -593,7 +586,6 @@ int ocrdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
 }
 
 static int ocrdma_copy_pd_uresp(struct ocrdma_dev *dev, struct ocrdma_pd *pd,
-				struct ib_ucontext *ib_ctx,
 				struct ib_udata *udata)
 {
 	int status;
@@ -601,7 +593,8 @@ static int ocrdma_copy_pd_uresp(struct ocrdma_dev *dev, struct ocrdma_pd *pd,
 	u64 dpp_page_addr = 0;
 	u32 db_page_size;
 	struct ocrdma_alloc_pd_uresp rsp;
-	struct ocrdma_ucontext *uctx = get_ocrdma_ucontext(ib_ctx);
+	struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context(
+		udata, struct ocrdma_ucontext, ibucontext);
 
 	memset(&rsp, 0, sizeof(rsp));
 	rsp.id = pd->id;
@@ -639,18 +632,17 @@ dpp_map_err:
 	return status;
 }
 
-int ocrdma_alloc_pd(struct ib_pd *ibpd, struct ib_ucontext *context,
-		    struct ib_udata *udata)
+int ocrdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 {
 	struct ib_device *ibdev = ibpd->device;
 	struct ocrdma_dev *dev = get_ocrdma_dev(ibdev);
 	struct ocrdma_pd *pd;
-	struct ocrdma_ucontext *uctx = NULL;
 	int status;
 	u8 is_uctx_pd = false;
+	struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context(
+		udata, struct ocrdma_ucontext, ibucontext);
 
-	if (udata && context) {
-		uctx = get_ocrdma_ucontext(context);
+	if (udata) {
 		pd = ocrdma_get_ucontext_pd(uctx);
 		if (pd) {
 			is_uctx_pd = true;
@@ -664,8 +656,8 @@ int ocrdma_alloc_pd(struct ib_pd *ibpd, struct ib_ucontext *context,
 		goto exit;
 
 pd_mapping:
-	if (udata && context) {
-		status = ocrdma_copy_pd_uresp(dev, pd, context, udata);
+	if (udata) {
+		status = ocrdma_copy_pd_uresp(dev, pd, udata);
 		if (status)
 			goto err;
 	}
@@ -680,7 +672,7 @@ exit:
 	return status;
 }
 
-void ocrdma_dealloc_pd(struct ib_pd *ibpd)
+void ocrdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 {
 	struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
 	struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
@@ -922,7 +914,7 @@ umem_err:
 	return ERR_PTR(status);
 }
 
-int ocrdma_dereg_mr(struct ib_mr *ib_mr)
+int ocrdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
 {
 	struct ocrdma_mr *mr = get_ocrdma_mr(ib_mr);
 	struct ocrdma_dev *dev = get_ocrdma_dev(ib_mr->device);
@@ -946,13 +938,17 @@ int ocrdma_dereg_mr(struct ib_mr *ib_mr)
 }
 
 static int ocrdma_copy_cq_uresp(struct ocrdma_dev *dev, struct ocrdma_cq *cq,
-				struct ib_udata *udata,
-				struct ib_ucontext *ib_ctx)
+				struct ib_udata *udata)
 {
 	int status;
-	struct ocrdma_ucontext *uctx = get_ocrdma_ucontext(ib_ctx);
+	struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context(
+		udata, struct ocrdma_ucontext, ibucontext);
 	struct ocrdma_create_cq_uresp uresp;
 
+	/* this must be user flow! */
+	if (!udata)
+		return -EINVAL;
+
 	memset(&uresp, 0, sizeof(uresp));
 	uresp.cq_id = cq->id;
 	uresp.page_size = PAGE_ALIGN(cq->len);
@@ -983,13 +979,13 @@ err:
 
 struct ib_cq *ocrdma_create_cq(struct ib_device *ibdev,
 			       const struct ib_cq_init_attr *attr,
-			       struct ib_ucontext *ib_ctx,
 			       struct ib_udata *udata)
 {
 	int entries = attr->cqe;
 	struct ocrdma_cq *cq;
 	struct ocrdma_dev *dev = get_ocrdma_dev(ibdev);
-	struct ocrdma_ucontext *uctx = NULL;
+	struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context(
+		udata, struct ocrdma_ucontext, ibucontext);
 	u16 pd_id = 0;
 	int status;
 	struct ocrdma_create_cq_ureq ureq;
@@ -1011,18 +1007,16 @@ struct ib_cq *ocrdma_create_cq(struct ib_device *ibdev,
 	INIT_LIST_HEAD(&cq->sq_head);
 	INIT_LIST_HEAD(&cq->rq_head);
 
-	if (ib_ctx) {
-		uctx = get_ocrdma_ucontext(ib_ctx);
+	if (udata)
 		pd_id = uctx->cntxt_pd->id;
-	}
 
 	status = ocrdma_mbx_create_cq(dev, cq, entries, ureq.dpp_cq, pd_id);
 	if (status) {
 		kfree(cq);
 		return ERR_PTR(status);
 	}
-	if (ib_ctx) {
-		status = ocrdma_copy_cq_uresp(dev, cq, udata, ib_ctx);
+	if (udata) {
+		status = ocrdma_copy_cq_uresp(dev, cq, udata);
 		if (status)
 			goto ctx_err;
 	}
@@ -1076,7 +1070,7 @@ static void ocrdma_flush_cq(struct ocrdma_cq *cq)
 	spin_unlock_irqrestore(&cq->cq_lock, flags);
 }
 
-int ocrdma_destroy_cq(struct ib_cq *ibcq)
+int ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
 {
 	struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);
 	struct ocrdma_eq *eq = NULL;
@@ -1697,7 +1691,7 @@ void ocrdma_del_flush_qp(struct ocrdma_qp *qp)
 	spin_unlock_irqrestore(&dev->flush_q_lock, flags);
 }
 
-int ocrdma_destroy_qp(struct ib_qp *ibqp)
+int ocrdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
 {
 	struct ocrdma_pd *pd;
 	struct ocrdma_qp *qp;
@@ -1793,45 +1787,43 @@ static int ocrdma_copy_srq_uresp(struct ocrdma_dev *dev, struct ocrdma_srq *srq,
 	return status;
 }
 
-struct ib_srq *ocrdma_create_srq(struct ib_pd *ibpd,
-				 struct ib_srq_init_attr *init_attr,
-				 struct ib_udata *udata)
+int ocrdma_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr,
+		      struct ib_udata *udata)
 {
-	int status = -ENOMEM;
-	struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
-	struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
-	struct ocrdma_srq *srq;
+	int status;
+	struct ocrdma_pd *pd = get_ocrdma_pd(ibsrq->pd);
+	struct ocrdma_dev *dev = get_ocrdma_dev(ibsrq->device);
+	struct ocrdma_srq *srq = get_ocrdma_srq(ibsrq);
 
 	if (init_attr->attr.max_sge > dev->attr.max_recv_sge)
-		return ERR_PTR(-EINVAL);
+		return -EINVAL;
 	if (init_attr->attr.max_wr > dev->attr.max_rqe)
-		return ERR_PTR(-EINVAL);
-
-	srq = kzalloc(sizeof(*srq), GFP_KERNEL);
-	if (!srq)
-		return ERR_PTR(status);
+		return -EINVAL;
 
 	spin_lock_init(&srq->q_lock);
 	srq->pd = pd;
 	srq->db = dev->nic_info.db + (pd->id * dev->nic_info.db_page_size);
 	status = ocrdma_mbx_create_srq(dev, srq, init_attr, pd);
 	if (status)
-		goto err;
+		return status;
 
-	if (udata == NULL) {
-		status = -ENOMEM;
+	if (!udata) {
 		srq->rqe_wr_id_tbl = kcalloc(srq->rq.max_cnt, sizeof(u64),
 					     GFP_KERNEL);
-		if (srq->rqe_wr_id_tbl == NULL)
+		if (!srq->rqe_wr_id_tbl) {
+			status = -ENOMEM;
 			goto arm_err;
+		}
 
 		srq->bit_fields_len = (srq->rq.max_cnt / 32) +
 		    (srq->rq.max_cnt % 32 ? 1 : 0);
 		srq->idx_bit_fields =
 		    kmalloc_array(srq->bit_fields_len, sizeof(u32),
 				  GFP_KERNEL);
-		if (srq->idx_bit_fields == NULL)
+		if (!srq->idx_bit_fields) {
+			status = -ENOMEM;
 			goto arm_err;
+		}
 		memset(srq->idx_bit_fields, 0xff,
 		       srq->bit_fields_len * sizeof(u32));
 	}
@@ -1848,15 +1840,13 @@ struct ib_srq *ocrdma_create_srq(struct ib_pd *ibpd,
 			goto arm_err;
 	}
 
-	return &srq->ibsrq;
+	return 0;
 
 arm_err:
 	ocrdma_mbx_destroy_srq(dev, srq);
-err:
 	kfree(srq->rqe_wr_id_tbl);
 	kfree(srq->idx_bit_fields);
-	kfree(srq);
-	return ERR_PTR(status);
+	return status;
 }
 
 int ocrdma_modify_srq(struct ib_srq *ibsrq,
@@ -1885,15 +1875,14 @@ int ocrdma_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
 	return status;
 }
 
-int ocrdma_destroy_srq(struct ib_srq *ibsrq)
+void ocrdma_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
 {
-	int status;
 	struct ocrdma_srq *srq;
 	struct ocrdma_dev *dev = get_ocrdma_dev(ibsrq->device);
 
 	srq = get_ocrdma_srq(ibsrq);
 
-	status = ocrdma_mbx_destroy_srq(dev, srq);
+	ocrdma_mbx_destroy_srq(dev, srq);
 
 	if (srq->pd->uctx)
 		ocrdma_del_mmap(srq->pd->uctx, (u64) srq->rq.pa,
@@ -1901,8 +1890,6 @@ int ocrdma_destroy_srq(struct ib_srq *ibsrq)
 
 	kfree(srq->idx_bit_fields);
 	kfree(srq->rqe_wr_id_tbl);
-	kfree(srq);
-	return status;
 }
 
 /* unprivileged verbs and their support functions. */
@@ -2931,9 +2918,8 @@ int ocrdma_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags cq_flags)
 	return 0;
 }
 
-struct ib_mr *ocrdma_alloc_mr(struct ib_pd *ibpd,
-			      enum ib_mr_type mr_type,
-			      u32 max_num_sg)
+struct ib_mr *ocrdma_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
+			      u32 max_num_sg, struct ib_udata *udata)
 {
 	int status;
 	struct ocrdma_mr *mr;
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h
index 4c04ab40798e..d76aae7ed0d3 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h
@@ -61,7 +61,6 @@ enum rdma_protocol_type
 ocrdma_query_protocol(struct ib_device *device, u8 port_num);
 
 void ocrdma_get_guid(struct ocrdma_dev *, u8 *guid);
-struct net_device *ocrdma_get_netdev(struct ib_device *device, u8 port_num);
 int ocrdma_query_pkey(struct ib_device *, u8 port, u16 index, u16 *pkey);
 
 int ocrdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata);
@@ -69,16 +68,14 @@ void ocrdma_dealloc_ucontext(struct ib_ucontext *uctx);
 
 int ocrdma_mmap(struct ib_ucontext *, struct vm_area_struct *vma);
 
-int ocrdma_alloc_pd(struct ib_pd *pd, struct ib_ucontext *uctx,
-		    struct ib_udata *udata);
-void ocrdma_dealloc_pd(struct ib_pd *pd);
+int ocrdma_alloc_pd(struct ib_pd *pd, struct ib_udata *udata);
+void ocrdma_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata);
 
 struct ib_cq *ocrdma_create_cq(struct ib_device *ibdev,
 			       const struct ib_cq_init_attr *attr,
-			       struct ib_ucontext *ib_ctx,
 			       struct ib_udata *udata);
 int ocrdma_resize_cq(struct ib_cq *, int cqe, struct ib_udata *);
-int ocrdma_destroy_cq(struct ib_cq *);
+int ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
 
 struct ib_qp *ocrdma_create_qp(struct ib_pd *,
 			       struct ib_qp_init_attr *attrs,
@@ -90,25 +87,24 @@ int ocrdma_modify_qp(struct ib_qp *, struct ib_qp_attr *attr,
 int ocrdma_query_qp(struct ib_qp *,
 		    struct ib_qp_attr *qp_attr,
 		    int qp_attr_mask, struct ib_qp_init_attr *);
-int ocrdma_destroy_qp(struct ib_qp *);
+int ocrdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);
 void ocrdma_del_flush_qp(struct ocrdma_qp *qp);
 
-struct ib_srq *ocrdma_create_srq(struct ib_pd *, struct ib_srq_init_attr *,
-				 struct ib_udata *);
+int ocrdma_create_srq(struct ib_srq *srq, struct ib_srq_init_attr *attr,
+		      struct ib_udata *udata);
 int ocrdma_modify_srq(struct ib_srq *, struct ib_srq_attr *,
 		      enum ib_srq_attr_mask, struct ib_udata *);
 int ocrdma_query_srq(struct ib_srq *, struct ib_srq_attr *);
-int ocrdma_destroy_srq(struct ib_srq *);
+void ocrdma_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata);
 int ocrdma_post_srq_recv(struct ib_srq *, const struct ib_recv_wr *,
 			 const struct ib_recv_wr **bad_recv_wr);
 
-int ocrdma_dereg_mr(struct ib_mr *);
+int ocrdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata);
 struct ib_mr *ocrdma_get_dma_mr(struct ib_pd *, int acc);
 struct ib_mr *ocrdma_reg_user_mr(struct ib_pd *, u64 start, u64 length,
 				 u64 virt, int acc, struct ib_udata *);
-struct ib_mr *ocrdma_alloc_mr(struct ib_pd *pd,
-			      enum ib_mr_type mr_type,
-			      u32 max_num_sg);
+struct ib_mr *ocrdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
+			      u32 max_num_sg, struct ib_udata *udata);
 int ocrdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
 		     unsigned int *sg_offset);
 
diff --git a/drivers/infiniband/hw/qedr/main.c b/drivers/infiniband/hw/qedr/main.c
index 996d9ecd93e0..083c2c00a8e9 100644
--- a/drivers/infiniband/hw/qedr/main.c
+++ b/drivers/infiniband/hw/qedr/main.c
@@ -39,7 +39,6 @@
 #include <linux/iommu.h>
 #include <linux/pci.h>
 #include <net/addrconf.h>
-#include <linux/idr.h>
 
 #include <linux/qed/qed_chain.h>
 #include <linux/qed/qed_if.h>
@@ -82,20 +81,6 @@ static void qedr_get_dev_fw_str(struct ib_device *ibdev, char *str)
 		 (fw_ver >> 8) & 0xFF, fw_ver & 0xFF);
 }
 
-static struct net_device *qedr_get_netdev(struct ib_device *dev, u8 port_num)
-{
-	struct qedr_dev *qdev;
-
-	qdev = get_qedr_dev(dev);
-	dev_hold(qdev->ndev);
-
-	/* The HW vendor's device driver must guarantee
-	 * that this function returns NULL before the net device has finished
-	 * NETDEV_UNREGISTER state.
-	 */
-	return qdev->ndev;
-}
-
 static int qedr_roce_port_immutable(struct ib_device *ibdev, u8 port_num,
 				    struct ib_port_immutable *immutable)
 {
@@ -163,6 +148,14 @@ static const struct attribute_group qedr_attr_group = {
 
 static const struct ib_device_ops qedr_iw_dev_ops = {
 	.get_port_immutable = qedr_iw_port_immutable,
+	.iw_accept = qedr_iw_accept,
+	.iw_add_ref = qedr_iw_qp_add_ref,
+	.iw_connect = qedr_iw_connect,
+	.iw_create_listen = qedr_iw_create_listen,
+	.iw_destroy_listen = qedr_iw_destroy_listen,
+	.iw_get_qp = qedr_iw_get_qp,
+	.iw_reject = qedr_iw_reject,
+	.iw_rem_ref = qedr_iw_qp_rem_ref,
 	.query_gid = qedr_iw_query_gid,
 };
 
@@ -172,21 +165,8 @@ static int qedr_iw_register_device(struct qedr_dev *dev)
 
 	ib_set_device_ops(&dev->ibdev, &qedr_iw_dev_ops);
 
-	dev->ibdev.iwcm = kzalloc(sizeof(*dev->ibdev.iwcm), GFP_KERNEL);
-	if (!dev->ibdev.iwcm)
-		return -ENOMEM;
-
-	dev->ibdev.iwcm->connect = qedr_iw_connect;
-	dev->ibdev.iwcm->accept = qedr_iw_accept;
-	dev->ibdev.iwcm->reject = qedr_iw_reject;
-	dev->ibdev.iwcm->create_listen = qedr_iw_create_listen;
-	dev->ibdev.iwcm->destroy_listen = qedr_iw_destroy_listen;
-	dev->ibdev.iwcm->add_ref = qedr_iw_qp_add_ref;
-	dev->ibdev.iwcm->rem_ref = qedr_iw_qp_rem_ref;
-	dev->ibdev.iwcm->get_qp = qedr_iw_get_qp;
-
-	memcpy(dev->ibdev.iwcm->ifname,
-	       dev->ndev->name, sizeof(dev->ibdev.iwcm->ifname));
+	memcpy(dev->ibdev.iw_ifname,
+	       dev->ndev->name, sizeof(dev->ibdev.iw_ifname));
 
 	return 0;
 }
@@ -220,7 +200,6 @@ static const struct ib_device_ops qedr_dev_ops = {
 	.get_dev_fw_str = qedr_get_dev_fw_str,
 	.get_dma_mr = qedr_get_dma_mr,
 	.get_link_layer = qedr_link_layer,
-	.get_netdev = qedr_get_netdev,
 	.map_mr_sg = qedr_map_mr_sg,
 	.mmap = qedr_mmap,
 	.modify_port = qedr_modify_port,
@@ -239,7 +218,10 @@ static const struct ib_device_ops qedr_dev_ops = {
 	.reg_user_mr = qedr_reg_user_mr,
 	.req_notify_cq = qedr_arm_cq,
 	.resize_cq = qedr_resize_cq,
+
+	INIT_RDMA_OBJ_SIZE(ib_ah, qedr_ah, ibah),
 	INIT_RDMA_OBJ_SIZE(ib_pd, qedr_pd, ibpd),
+	INIT_RDMA_OBJ_SIZE(ib_srq, qedr_srq, ibsrq),
 	INIT_RDMA_OBJ_SIZE(ib_ucontext, qedr_ucontext, ibucontext),
 };
 
@@ -293,6 +275,10 @@ static int qedr_register_device(struct qedr_dev *dev)
 	ib_set_device_ops(&dev->ibdev, &qedr_dev_ops);
 
 	dev->ibdev.driver_id = RDMA_DRIVER_QEDR;
+	rc = ib_device_set_netdev(&dev->ibdev, dev->ndev, 1);
+	if (rc)
+		return rc;
+
 	return ib_register_device(&dev->ibdev, "qedr%d");
 }
 
@@ -364,8 +350,7 @@ static int qedr_alloc_resources(struct qedr_dev *dev)
 	spin_lock_init(&dev->sgid_lock);
 
 	if (IS_IWARP(dev)) {
-		spin_lock_init(&dev->qpidr.idr_lock);
-		idr_init(&dev->qpidr.idr);
+		xa_init_flags(&dev->qps, XA_FLAGS_LOCK_IRQ);
 		dev->iwarp_wq = create_singlethread_workqueue("qedr_iwarpq");
 	}
 
@@ -760,8 +745,8 @@ static void qedr_affiliated_event(void *context, u8 e_code, void *fw_handle)
 		break;
 	case EVENT_TYPE_SRQ:
 		srq_id = (u16)roce_handle64;
-		spin_lock_irqsave(&dev->srqidr.idr_lock, flags);
-		srq = idr_find(&dev->srqidr.idr, srq_id);
+		xa_lock_irqsave(&dev->srqs, flags);
+		srq = xa_load(&dev->srqs, srq_id);
 		if (srq) {
 			ibsrq = &srq->ibsrq;
 			if (ibsrq->event_handler) {
@@ -775,7 +760,7 @@ static void qedr_affiliated_event(void *context, u8 e_code, void *fw_handle)
 				  "SRQ event with NULL pointer ibsrq. Handle=%llx\n",
 				  roce_handle64);
 		}
-		spin_unlock_irqrestore(&dev->srqidr.idr_lock, flags);
+		xa_unlock_irqrestore(&dev->srqs, flags);
 		DP_NOTICE(dev, "SRQ event %d on handle %p\n", e_code, srq);
 	default:
 		break;
diff --git a/drivers/infiniband/hw/qedr/qedr.h b/drivers/infiniband/hw/qedr/qedr.h
index 53bbe6b4e6e6..6175d1e98717 100644
--- a/drivers/infiniband/hw/qedr/qedr.h
+++ b/drivers/infiniband/hw/qedr/qedr.h
@@ -33,7 +33,7 @@
 #define __QEDR_H__
 
 #include <linux/pci.h>
-#include <linux/idr.h>
+#include <linux/xarray.h>
 #include <rdma/ib_addr.h>
 #include <linux/qed/qed_if.h>
 #include <linux/qed/qed_chain.h>
@@ -123,11 +123,6 @@ struct qedr_device_attr {
 
 #define QEDR_ENET_STATE_BIT	(0)
 
-struct qedr_idr {
-	spinlock_t idr_lock; /* Protect idr data-structure */
-	struct idr idr;
-};
-
 struct qedr_dev {
 	struct ib_device	ibdev;
 	struct qed_dev		*cdev;
@@ -171,8 +166,8 @@ struct qedr_dev {
 	struct qedr_cq		*gsi_rqcq;
 	struct qedr_qp		*gsi_qp;
 	enum qed_rdma_type	rdma_type;
-	struct qedr_idr		qpidr;
-	struct qedr_idr		srqidr;
+	struct xarray		qps;
+	struct xarray		srqs;
 	struct workqueue_struct *iwarp_wq;
 	u16			iwarp_max_mtu;
 
diff --git a/drivers/infiniband/hw/qedr/qedr_iw_cm.c b/drivers/infiniband/hw/qedr/qedr_iw_cm.c
index 0555e5a8c9ed..22881d4442b9 100644
--- a/drivers/infiniband/hw/qedr/qedr_iw_cm.c
+++ b/drivers/infiniband/hw/qedr/qedr_iw_cm.c
@@ -491,7 +491,7 @@ int qedr_iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
 	int rc = 0;
 	int i;
 
-	qp = idr_find(&dev->qpidr.idr, conn_param->qpn);
+	qp = xa_load(&dev->qps, conn_param->qpn);
 	if (unlikely(!qp))
 		return -EINVAL;
 
@@ -681,7 +681,7 @@ int qedr_iw_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
 
 	DP_DEBUG(dev, QEDR_MSG_IWARP, "Accept on qpid=%d\n", conn_param->qpn);
 
-	qp = idr_find(&dev->qpidr.idr, conn_param->qpn);
+	qp = xa_load(&dev->qps, conn_param->qpn);
 	if (!qp) {
 		DP_ERR(dev, "Invalid QP number %d\n", conn_param->qpn);
 		return -EINVAL;
@@ -739,9 +739,7 @@ void qedr_iw_qp_rem_ref(struct ib_qp *ibqp)
 	struct qedr_qp *qp = get_qedr_qp(ibqp);
 
 	if (atomic_dec_and_test(&qp->refcnt)) {
-		spin_lock_irq(&qp->dev->qpidr.idr_lock);
-		idr_remove(&qp->dev->qpidr.idr, qp->qp_id);
-		spin_unlock_irq(&qp->dev->qpidr.idr_lock);
+		xa_erase_irq(&qp->dev->qps, qp->qp_id);
 		kfree(qp);
 	}
 }
@@ -750,5 +748,5 @@ struct ib_qp *qedr_iw_get_qp(struct ib_device *ibdev, int qpn)
 {
 	struct qedr_dev *dev = get_qedr_dev(ibdev);
 
-	return idr_find(&dev->qpidr.idr, qpn);
+	return xa_load(&dev->qps, qpn);
 }
diff --git a/drivers/infiniband/hw/qedr/qedr_roce_cm.c b/drivers/infiniband/hw/qedr/qedr_roce_cm.c
index e1ac2fd60bb1..f5542d703ef9 100644
--- a/drivers/infiniband/hw/qedr/qedr_roce_cm.c
+++ b/drivers/infiniband/hw/qedr/qedr_roce_cm.c
@@ -397,14 +397,17 @@ static inline int qedr_gsi_build_header(struct qedr_dev *dev,
 	bool has_udp = false;
 	int i;
 
-	send_size = 0;
-	for (i = 0; i < swr->num_sge; ++i)
-		send_size += swr->sg_list[i].length;
+	rc = rdma_read_gid_l2_fields(sgid_attr, &vlan_id, NULL);
+	if (rc)
+		return rc;
 
-	vlan_id = rdma_vlan_dev_vlan_id(sgid_attr->ndev);
 	if (vlan_id < VLAN_CFI_MASK)
 		has_vlan = true;
 
+	send_size = 0;
+	for (i = 0; i < swr->num_sge; ++i)
+		send_size += swr->sg_list[i].length;
+
 	has_udp = (sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP);
 	if (!has_udp) {
 		/* RoCE v1 */
diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c
index 8686a98e113d..3d7bde19838e 100644
--- a/drivers/infiniband/hw/qedr/verbs.c
+++ b/drivers/infiniband/hw/qedr/verbs.c
@@ -42,6 +42,7 @@
 #include <rdma/ib_umem.h>
 #include <rdma/ib_addr.h>
 #include <rdma/ib_cache.h>
+#include <rdma/uverbs_ioctl.h>
 
 #include <linux/qed/common_hsi.h>
 #include "qedr_hsi_rdma.h"
@@ -436,8 +437,7 @@ int qedr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
 				  vma->vm_page_prot);
 }
 
-int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_ucontext *context,
-		  struct ib_udata *udata)
+int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 {
 	struct ib_device *ibdev = ibpd->device;
 	struct qedr_dev *dev = get_qedr_dev(ibdev);
@@ -446,7 +446,7 @@ int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_ucontext *context,
 	int rc;
 
 	DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
-		 (udata && context) ? "User Lib" : "Kernel");
+		 udata ? "User Lib" : "Kernel");
 
 	if (!dev->rdma_ctx) {
 		DP_ERR(dev, "invalid RDMA context\n");
@@ -459,10 +459,12 @@ int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_ucontext *context,
 
 	pd->pd_id = pd_id;
 
-	if (udata && context) {
+	if (udata) {
 		struct qedr_alloc_pd_uresp uresp = {
 			.pd_id = pd_id,
 		};
+		struct qedr_ucontext *context = rdma_udata_to_drv_context(
+			udata, struct qedr_ucontext, ibucontext);
 
 		rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 		if (rc) {
@@ -471,14 +473,14 @@ int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_ucontext *context,
 			return rc;
 		}
 
-		pd->uctx = get_qedr_ucontext(context);
+		pd->uctx = context;
 		pd->uctx->pd = pd;
 	}
 
 	return 0;
 }
 
-void qedr_dealloc_pd(struct ib_pd *ibpd)
+void qedr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 {
 	struct qedr_dev *dev = get_qedr_dev(ibpd->device);
 	struct qedr_pd *pd = get_qedr_pd(ibpd);
@@ -813,9 +815,10 @@ int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
 
 struct ib_cq *qedr_create_cq(struct ib_device *ibdev,
 			     const struct ib_cq_init_attr *attr,
-			     struct ib_ucontext *ib_ctx, struct ib_udata *udata)
+			     struct ib_udata *udata)
 {
-	struct qedr_ucontext *ctx = get_qedr_ucontext(ib_ctx);
+	struct qedr_ucontext *ctx = rdma_udata_to_drv_context(
+		udata, struct qedr_ucontext, ibucontext);
 	struct qed_rdma_destroy_cq_out_params destroy_oparams;
 	struct qed_rdma_destroy_cq_in_params destroy_iparams;
 	struct qedr_dev *dev = get_qedr_dev(ibdev);
@@ -903,7 +906,7 @@ struct ib_cq *qedr_create_cq(struct ib_device *ibdev,
 	cq->sig = QEDR_CQ_MAGIC_NUMBER;
 	spin_lock_init(&cq->cq_lock);
 
-	if (ib_ctx) {
+	if (udata) {
 		rc = qedr_copy_cq_uresp(dev, cq, udata);
 		if (rc)
 			goto err3;
@@ -959,7 +962,7 @@ int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
 #define QEDR_DESTROY_CQ_MAX_ITERATIONS		(10)
 #define QEDR_DESTROY_CQ_ITER_DURATION		(10)
 
-int qedr_destroy_cq(struct ib_cq *ibcq)
+int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
 {
 	struct qedr_dev *dev = get_qedr_dev(ibcq->device);
 	struct qed_rdma_destroy_cq_out_params oparams;
@@ -983,7 +986,7 @@ int qedr_destroy_cq(struct ib_cq *ibcq)
 
 	dev->ops->common->chain_free(dev->cdev, &cq->pbl);
 
-	if (ibcq->uobject && ibcq->uobject->context) {
+	if (udata) {
 		qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
 		ib_umem_release(cq->q.umem);
 	}
@@ -1044,10 +1047,13 @@ static inline int get_gid_info_from_table(struct ib_qp *ibqp,
 	enum rdma_network_type nw_type;
 	const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
 	u32 ipv4_addr;
+	int ret;
 	int i;
 
 	gid_attr = grh->sgid_attr;
-	qp_params->vlan_id = rdma_vlan_dev_vlan_id(gid_attr->ndev);
+	ret = rdma_read_gid_l2_fields(gid_attr, &qp_params->vlan_id, NULL);
+	if (ret)
+		return ret;
 
 	nw_type = rdma_gid_attr_network_type(gid_attr);
 	switch (nw_type) {
@@ -1261,7 +1267,7 @@ static void qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
 	}
 }
 
-static int qedr_check_srq_params(struct ib_pd *ibpd, struct qedr_dev *dev,
+static int qedr_check_srq_params(struct qedr_dev *dev,
 				 struct ib_srq_init_attr *attrs,
 				 struct ib_udata *udata)
 {
@@ -1377,38 +1383,28 @@ err0:
 	return rc;
 }
 
-static int qedr_idr_add(struct qedr_dev *dev, struct qedr_idr *qidr,
-			void *ptr, u32 id);
-static void qedr_idr_remove(struct qedr_dev *dev,
-			    struct qedr_idr *qidr, u32 id);
-
-struct ib_srq *qedr_create_srq(struct ib_pd *ibpd,
-			       struct ib_srq_init_attr *init_attr,
-			       struct ib_udata *udata)
+int qedr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr,
+		    struct ib_udata *udata)
 {
 	struct qed_rdma_destroy_srq_in_params destroy_in_params;
 	struct qed_rdma_create_srq_in_params in_params = {};
-	struct qedr_dev *dev = get_qedr_dev(ibpd->device);
+	struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
 	struct qed_rdma_create_srq_out_params out_params;
-	struct qedr_pd *pd = get_qedr_pd(ibpd);
+	struct qedr_pd *pd = get_qedr_pd(ibsrq->pd);
 	struct qedr_create_srq_ureq ureq = {};
 	u64 pbl_base_addr, phy_prod_pair_addr;
 	struct qedr_srq_hwq_info *hw_srq;
 	u32 page_cnt, page_size;
-	struct qedr_srq *srq;
+	struct qedr_srq *srq = get_qedr_srq(ibsrq);
 	int rc = 0;
 
 	DP_DEBUG(dev, QEDR_MSG_QP,
 		 "create SRQ called from %s (pd %p)\n",
 		 (udata) ? "User lib" : "kernel", pd);
 
-	rc = qedr_check_srq_params(ibpd, dev, init_attr, udata);
+	rc = qedr_check_srq_params(dev, init_attr, udata);
 	if (rc)
-		return ERR_PTR(-EINVAL);
-
-	srq = kzalloc(sizeof(*srq), GFP_KERNEL);
-	if (!srq)
-		return ERR_PTR(-ENOMEM);
+		return -EINVAL;
 
 	srq->dev = dev;
 	hw_srq = &srq->hw_srq;
@@ -1464,13 +1460,13 @@ struct ib_srq *qedr_create_srq(struct ib_pd *ibpd,
 			goto err2;
 	}
 
-	rc = qedr_idr_add(dev, &dev->srqidr, srq, srq->srq_id);
+	rc = xa_insert_irq(&dev->srqs, srq->srq_id, srq, GFP_KERNEL);
 	if (rc)
 		goto err2;
 
 	DP_DEBUG(dev, QEDR_MSG_SRQ,
 		 "create srq: created srq with srq_id=0x%0x\n", srq->srq_id);
-	return &srq->ibsrq;
+	return 0;
 
 err2:
 	destroy_in_params.srq_id = srq->srq_id;
@@ -1482,18 +1478,16 @@ err1:
 	else
 		qedr_free_srq_kernel_params(srq);
 err0:
-	kfree(srq);
-
-	return ERR_PTR(-EFAULT);
+	return -EFAULT;
 }
 
-int qedr_destroy_srq(struct ib_srq *ibsrq)
+void qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
 {
 	struct qed_rdma_destroy_srq_in_params in_params = {};
 	struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
 	struct qedr_srq *srq = get_qedr_srq(ibsrq);
 
-	qedr_idr_remove(dev, &dev->srqidr, srq->srq_id);
+	xa_erase_irq(&dev->srqs, srq->srq_id);
 	in_params.srq_id = srq->srq_id;
 	dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params);
 
@@ -1505,9 +1499,6 @@ int qedr_destroy_srq(struct ib_srq *ibsrq)
 	DP_DEBUG(dev, QEDR_MSG_SRQ,
 		 "destroy srq: destroyed srq with srq_id=0x%0x\n",
 		 srq->srq_id);
-	kfree(srq);
-
-	return 0;
 }
 
 int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
@@ -1593,29 +1584,6 @@ static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
 		 qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len);
 }
 
-static int qedr_idr_add(struct qedr_dev *dev, struct qedr_idr *qidr,
-			void *ptr, u32 id)
-{
-	int rc;
-
-	idr_preload(GFP_KERNEL);
-	spin_lock_irq(&qidr->idr_lock);
-
-	rc = idr_alloc(&qidr->idr, ptr, id, id + 1, GFP_ATOMIC);
-
-	spin_unlock_irq(&qidr->idr_lock);
-	idr_preload_end();
-
-	return rc < 0 ? rc : 0;
-}
-
-static void qedr_idr_remove(struct qedr_dev *dev, struct qedr_idr *qidr, u32 id)
-{
-	spin_lock_irq(&qidr->idr_lock);
-	idr_remove(&qidr->idr, id);
-	spin_unlock_irq(&qidr->idr_lock);
-}
-
 static inline void
 qedr_iwarp_populate_user_qp(struct qedr_dev *dev,
 			    struct qedr_qp *qp,
@@ -1985,7 +1953,7 @@ struct ib_qp *qedr_create_qp(struct ib_pd *ibpd,
 	qp->ibqp.qp_num = qp->qp_id;
 
 	if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
-		rc = qedr_idr_add(dev, &dev->qpidr, qp, qp->qp_id);
+		rc = xa_insert_irq(&dev->qps, qp->qp_id, qp, GFP_KERNEL);
 		if (rc)
 			goto err;
 	}
@@ -2493,7 +2461,8 @@ err:
 	return rc;
 }
 
-static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp)
+static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp,
+				  struct ib_udata *udata)
 {
 	int rc = 0;
 
@@ -2503,7 +2472,7 @@ static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp)
 			return rc;
 	}
 
-	if (qp->ibqp.uobject && qp->ibqp.uobject->context)
+	if (udata)
 		qedr_cleanup_user(dev, qp);
 	else
 		qedr_cleanup_kernel(dev, qp);
@@ -2511,7 +2480,7 @@ static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp)
 	return 0;
 }
 
-int qedr_destroy_qp(struct ib_qp *ibqp)
+int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
 {
 	struct qedr_qp *qp = get_qedr_qp(ibqp);
 	struct qedr_dev *dev = qp->dev;
@@ -2555,37 +2524,31 @@ int qedr_destroy_qp(struct ib_qp *ibqp)
 	if (qp->qp_type == IB_QPT_GSI)
 		qedr_destroy_gsi_qp(dev);
 
-	qedr_free_qp_resources(dev, qp);
+	qedr_free_qp_resources(dev, qp, udata);
 
 	if (atomic_dec_and_test(&qp->refcnt) &&
 	    rdma_protocol_iwarp(&dev->ibdev, 1)) {
-		qedr_idr_remove(dev, &dev->qpidr, qp->qp_id);
+		xa_erase_irq(&dev->qps, qp->qp_id);
 		kfree(qp);
 	}
 	return rc;
 }
 
-struct ib_ah *qedr_create_ah(struct ib_pd *ibpd, struct rdma_ah_attr *attr,
-			     u32 flags, struct ib_udata *udata)
+int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr, u32 flags,
+		   struct ib_udata *udata)
 {
-	struct qedr_ah *ah;
-
-	ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
-	if (!ah)
-		return ERR_PTR(-ENOMEM);
+	struct qedr_ah *ah = get_qedr_ah(ibah);
 
 	rdma_copy_ah_attr(&ah->attr, attr);
 
-	return &ah->ibah;
+	return 0;
 }
 
-int qedr_destroy_ah(struct ib_ah *ibah, u32 flags)
+void qedr_destroy_ah(struct ib_ah *ibah, u32 flags)
 {
 	struct qedr_ah *ah = get_qedr_ah(ibah);
 
 	rdma_destroy_ah_attr(&ah->attr);
-	kfree(ah);
-	return 0;
 }
 
 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
@@ -2734,7 +2697,7 @@ err0:
 	return ERR_PTR(rc);
 }
 
-int qedr_dereg_mr(struct ib_mr *ib_mr)
+int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
 {
 	struct qedr_mr *mr = get_qedr_mr(ib_mr);
 	struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
@@ -2826,8 +2789,8 @@ err0:
 	return ERR_PTR(rc);
 }
 
-struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd,
-			    enum ib_mr_type mr_type, u32 max_num_sg)
+struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
+			    u32 max_num_sg, struct ib_udata *udata)
 {
 	struct qedr_mr *mr;
 
diff --git a/drivers/infiniband/hw/qedr/verbs.h b/drivers/infiniband/hw/qedr/verbs.h
index f0c05f4771ac..9328c80375ef 100644
--- a/drivers/infiniband/hw/qedr/verbs.h
+++ b/drivers/infiniband/hw/qedr/verbs.h
@@ -47,16 +47,14 @@ int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata);
 void qedr_dealloc_ucontext(struct ib_ucontext *uctx);
 
 int qedr_mmap(struct ib_ucontext *, struct vm_area_struct *vma);
-int qedr_alloc_pd(struct ib_pd *pd, struct ib_ucontext *uctx,
-		  struct ib_udata *udata);
-void qedr_dealloc_pd(struct ib_pd *pd);
+int qedr_alloc_pd(struct ib_pd *pd, struct ib_udata *udata);
+void qedr_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata);
 
 struct ib_cq *qedr_create_cq(struct ib_device *ibdev,
 			     const struct ib_cq_init_attr *attr,
-			     struct ib_ucontext *ib_ctx,
 			     struct ib_udata *udata);
 int qedr_resize_cq(struct ib_cq *, int cqe, struct ib_udata *);
-int qedr_destroy_cq(struct ib_cq *);
+int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
 struct ib_qp *qedr_create_qp(struct ib_pd *, struct ib_qp_init_attr *attrs,
 			     struct ib_udata *);
@@ -64,22 +62,21 @@ int qedr_modify_qp(struct ib_qp *, struct ib_qp_attr *attr,
 		   int attr_mask, struct ib_udata *udata);
 int qedr_query_qp(struct ib_qp *, struct ib_qp_attr *qp_attr,
 		  int qp_attr_mask, struct ib_qp_init_attr *);
-int qedr_destroy_qp(struct ib_qp *ibqp);
+int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);
 
-struct ib_srq *qedr_create_srq(struct ib_pd *ibpd,
-			       struct ib_srq_init_attr *attr,
-			       struct ib_udata *udata);
+int qedr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *attr,
+		    struct ib_udata *udata);
 int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
 		    enum ib_srq_attr_mask attr_mask, struct ib_udata *udata);
 int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr);
-int qedr_destroy_srq(struct ib_srq *ibsrq);
+void qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata);
 int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
 		       const struct ib_recv_wr **bad_recv_wr);
-struct ib_ah *qedr_create_ah(struct ib_pd *ibpd, struct rdma_ah_attr *attr,
-			     u32 flags, struct ib_udata *udata);
-int qedr_destroy_ah(struct ib_ah *ibah, u32 flags);
+int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr, u32 flags,
+		   struct ib_udata *udata);
+void qedr_destroy_ah(struct ib_ah *ibah, u32 flags);
 
-int qedr_dereg_mr(struct ib_mr *);
+int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata);
 struct ib_mr *qedr_get_dma_mr(struct ib_pd *, int acc);
 
 struct ib_mr *qedr_reg_user_mr(struct ib_pd *, u64 start, u64 length,
@@ -89,7 +86,7 @@ int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
 		   int sg_nents, unsigned int *sg_offset);
 
 struct ib_mr *qedr_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
-			    u32 max_num_sg);
+			    u32 max_num_sg, struct ib_udata *udata);
 int qedr_poll_cq(struct ib_cq *, int num_entries, struct ib_wc *wc);
 int qedr_post_send(struct ib_qp *, const struct ib_send_wr *,
 		   const struct ib_send_wr **bad_wr);
diff --git a/drivers/infiniband/hw/qib/qib.h b/drivers/infiniband/hw/qib/qib.h
index 83d2349188db..432d6d0fd7f4 100644
--- a/drivers/infiniband/hw/qib/qib.h
+++ b/drivers/infiniband/hw/qib/qib.h
@@ -52,6 +52,7 @@
 #include <linux/kref.h>
 #include <linux/sched.h>
 #include <linux/kthread.h>
+#include <linux/xarray.h>
 #include <rdma/ib_hdrs.h>
 #include <rdma/rdma_vt.h>
 
@@ -1105,8 +1106,7 @@ struct qib_filedata {
 	int rec_cpu_num; /* for cpu affinity; -1 if none */
 };
 
-extern struct list_head qib_dev_list;
-extern spinlock_t qib_devs_lock;
+extern struct xarray qib_dev_table;
 extern struct qib_devdata *qib_lookup(int unit);
 extern u32 qib_cpulist_count;
 extern unsigned long *qib_cpulist;
diff --git a/drivers/infiniband/hw/qib/qib_common.h b/drivers/infiniband/hw/qib/qib_common.h
index a4a1f56ce824..f91f23e02283 100644
--- a/drivers/infiniband/hw/qib/qib_common.h
+++ b/drivers/infiniband/hw/qib/qib_common.h
@@ -57,7 +57,7 @@
  * QIB_VERBOSE_TRACING define as 1 if you want additional tracing in
  * fastpath code
  * QIB_TRACE_REGWRITES define as 1 if you want register writes to be
- * traced in faspath code
+ * traced in fastpath code
  * _QIB_TRACING define as 0 if you want to remove all tracing in a
  * compilation unit
  */
diff --git a/drivers/infiniband/hw/qib/qib_driver.c b/drivers/infiniband/hw/qib/qib_driver.c
index 3117cc5f2a9a..92eeea5679e2 100644
--- a/drivers/infiniband/hw/qib/qib_driver.c
+++ b/drivers/infiniband/hw/qib/qib_driver.c
@@ -49,8 +49,6 @@
  */
 const char ib_qib_version[] = QIB_DRIVER_VERSION "\n";
 
-DEFINE_SPINLOCK(qib_devs_lock);
-LIST_HEAD(qib_dev_list);
 DEFINE_MUTEX(qib_mutex);	/* general driver use */
 
 unsigned qib_ibmtu;
@@ -96,11 +94,11 @@ int qib_count_active_units(void)
 {
 	struct qib_devdata *dd;
 	struct qib_pportdata *ppd;
-	unsigned long flags;
+	unsigned long index, flags;
 	int pidx, nunits_active = 0;
 
-	spin_lock_irqsave(&qib_devs_lock, flags);
-	list_for_each_entry(dd, &qib_dev_list, list) {
+	xa_lock_irqsave(&qib_dev_table, flags);
+	xa_for_each(&qib_dev_table, index, dd) {
 		if (!(dd->flags & QIB_PRESENT) || !dd->kregbase)
 			continue;
 		for (pidx = 0; pidx < dd->num_pports; ++pidx) {
@@ -112,7 +110,7 @@ int qib_count_active_units(void)
 			}
 		}
 	}
-	spin_unlock_irqrestore(&qib_devs_lock, flags);
+	xa_unlock_irqrestore(&qib_dev_table, flags);
 	return nunits_active;
 }
 
@@ -125,13 +123,12 @@ int qib_count_units(int *npresentp, int *nupp)
 {
 	int nunits = 0, npresent = 0, nup = 0;
 	struct qib_devdata *dd;
-	unsigned long flags;
+	unsigned long index, flags;
 	int pidx;
 	struct qib_pportdata *ppd;
 
-	spin_lock_irqsave(&qib_devs_lock, flags);
-
-	list_for_each_entry(dd, &qib_dev_list, list) {
+	xa_lock_irqsave(&qib_dev_table, flags);
+	xa_for_each(&qib_dev_table, index, dd) {
 		nunits++;
 		if ((dd->flags & QIB_PRESENT) && dd->kregbase)
 			npresent++;
@@ -142,8 +139,7 @@ int qib_count_units(int *npresentp, int *nupp)
 				nup++;
 		}
 	}
-
-	spin_unlock_irqrestore(&qib_devs_lock, flags);
+	xa_unlock_irqrestore(&qib_dev_table, flags);
 
 	if (npresentp)
 		*npresentp = npresent;
diff --git a/drivers/infiniband/hw/qib/qib_fs.c b/drivers/infiniband/hw/qib/qib_fs.c
index 1d940a2885c9..ceb42d948412 100644
--- a/drivers/infiniband/hw/qib/qib_fs.c
+++ b/drivers/infiniband/hw/qib/qib_fs.c
@@ -508,8 +508,8 @@ bail:
  */
 static int qibfs_fill_super(struct super_block *sb, void *data, int silent)
 {
-	struct qib_devdata *dd, *tmp;
-	unsigned long flags;
+	struct qib_devdata *dd;
+	unsigned long index;
 	int ret;
 
 	static const struct tree_descr files[] = {
@@ -524,18 +524,12 @@ static int qibfs_fill_super(struct super_block *sb, void *data, int silent)
 		goto bail;
 	}
 
-	spin_lock_irqsave(&qib_devs_lock, flags);
-
-	list_for_each_entry_safe(dd, tmp, &qib_dev_list, list) {
-		spin_unlock_irqrestore(&qib_devs_lock, flags);
+	xa_for_each(&qib_dev_table, index, dd) {
 		ret = add_cntr_files(sb, dd);
 		if (ret)
 			goto bail;
-		spin_lock_irqsave(&qib_devs_lock, flags);
 	}
 
-	spin_unlock_irqrestore(&qib_devs_lock, flags);
-
 bail:
 	return ret;
 }
diff --git a/drivers/infiniband/hw/qib/qib_iba7322.c b/drivers/infiniband/hw/qib/qib_iba7322.c
index ac6a84f11ad0..dd4843379f51 100644
--- a/drivers/infiniband/hw/qib/qib_iba7322.c
+++ b/drivers/infiniband/hw/qib/qib_iba7322.c
@@ -6137,7 +6137,7 @@ static void set_no_qsfp_atten(struct qib_devdata *dd, int change)
 static int setup_txselect(const char *str, const struct kernel_param *kp)
 {
 	struct qib_devdata *dd;
-	unsigned long val;
+	unsigned long index, val;
 	char *n;
 
 	if (strlen(str) >= ARRAY_SIZE(txselect_list)) {
@@ -6153,7 +6153,7 @@ static int setup_txselect(const char *str, const struct kernel_param *kp)
 	}
 	strncpy(txselect_list, str, ARRAY_SIZE(txselect_list) - 1);
 
-	list_for_each_entry(dd, &qib_dev_list, list)
+	xa_for_each(&qib_dev_table, index, dd)
 		if (dd->deviceid == PCI_DEVICE_ID_QLOGIC_IB_7322)
 			set_no_qsfp_atten(dd, 1);
 	return 0;
diff --git a/drivers/infiniband/hw/qib/qib_init.c b/drivers/infiniband/hw/qib/qib_init.c
index 9fd69903ca57..d4fd8a6cff7b 100644
--- a/drivers/infiniband/hw/qib/qib_init.c
+++ b/drivers/infiniband/hw/qib/qib_init.c
@@ -36,7 +36,6 @@
 #include <linux/netdevice.h>
 #include <linux/vmalloc.h>
 #include <linux/delay.h>
-#include <linux/idr.h>
 #include <linux/module.h>
 #include <linux/printk.h>
 #ifdef CONFIG_INFINIBAND_QIB_DCA
@@ -95,7 +94,7 @@ MODULE_PARM_DESC(cc_table_size, "Congestion control table entries 0 (CCA disable
 
 static void verify_interrupt(struct timer_list *);
 
-static struct idr qib_unit_table;
+DEFINE_XARRAY_FLAGS(qib_dev_table, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_IRQ);
 u32 qib_cpulist_count;
 unsigned long *qib_cpulist;
 
@@ -785,21 +784,9 @@ void __attribute__((weak)) qib_disable_wc(struct qib_devdata *dd)
 {
 }
 
-static inline struct qib_devdata *__qib_lookup(int unit)
-{
-	return idr_find(&qib_unit_table, unit);
-}
-
 struct qib_devdata *qib_lookup(int unit)
 {
-	struct qib_devdata *dd;
-	unsigned long flags;
-
-	spin_lock_irqsave(&qib_devs_lock, flags);
-	dd = __qib_lookup(unit);
-	spin_unlock_irqrestore(&qib_devs_lock, flags);
-
-	return dd;
+	return xa_load(&qib_dev_table, unit);
 }
 
 /*
@@ -1046,10 +1033,9 @@ void qib_free_devdata(struct qib_devdata *dd)
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(&qib_devs_lock, flags);
-	idr_remove(&qib_unit_table, dd->unit);
-	list_del(&dd->list);
-	spin_unlock_irqrestore(&qib_devs_lock, flags);
+	xa_lock_irqsave(&qib_dev_table, flags);
+	__xa_erase(&qib_dev_table, dd->unit);
+	xa_unlock_irqrestore(&qib_dev_table, flags);
 
 #ifdef CONFIG_DEBUG_FS
 	qib_dbg_ibdev_exit(&dd->verbs_dev);
@@ -1070,15 +1056,15 @@ u64 qib_int_counter(struct qib_devdata *dd)
 
 u64 qib_sps_ints(void)
 {
-	unsigned long flags;
+	unsigned long index, flags;
 	struct qib_devdata *dd;
 	u64 sps_ints = 0;
 
-	spin_lock_irqsave(&qib_devs_lock, flags);
-	list_for_each_entry(dd, &qib_dev_list, list) {
+	xa_lock_irqsave(&qib_dev_table, flags);
+	xa_for_each(&qib_dev_table, index, dd) {
 		sps_ints += qib_int_counter(dd);
 	}
-	spin_unlock_irqrestore(&qib_devs_lock, flags);
+	xa_unlock_irqrestore(&qib_dev_table, flags);
 	return sps_ints;
 }
 
@@ -1087,12 +1073,9 @@ u64 qib_sps_ints(void)
  * allocator, because the verbs cleanup process both does cleanup and
  * free of the data structure.
  * "extra" is for chip-specific data.
- *
- * Use the idr mechanism to get a unit number for this unit.
  */
 struct qib_devdata *qib_alloc_devdata(struct pci_dev *pdev, size_t extra)
 {
-	unsigned long flags;
 	struct qib_devdata *dd;
 	int ret, nports;
 
@@ -1103,20 +1086,8 @@ struct qib_devdata *qib_alloc_devdata(struct pci_dev *pdev, size_t extra)
 	if (!dd)
 		return ERR_PTR(-ENOMEM);
 
-	INIT_LIST_HEAD(&dd->list);
-
-	idr_preload(GFP_KERNEL);
-	spin_lock_irqsave(&qib_devs_lock, flags);
-
-	ret = idr_alloc(&qib_unit_table, dd, 0, 0, GFP_NOWAIT);
-	if (ret >= 0) {
-		dd->unit = ret;
-		list_add(&dd->list, &qib_dev_list);
-	}
-
-	spin_unlock_irqrestore(&qib_devs_lock, flags);
-	idr_preload_end();
-
+	ret = xa_alloc_irq(&qib_dev_table, &dd->unit, dd, xa_limit_32b,
+			GFP_KERNEL);
 	if (ret < 0) {
 		qib_early_err(&pdev->dev,
 			      "Could not allocate unit ID: error %d\n", -ret);
@@ -1255,8 +1226,6 @@ static int __init qib_ib_init(void)
 	 * These must be called before the driver is registered with
 	 * the PCI subsystem.
 	 */
-	idr_init(&qib_unit_table);
-
 #ifdef CONFIG_INFINIBAND_QIB_DCA
 	dca_register_notify(&dca_notifier);
 #endif
@@ -1281,7 +1250,6 @@ bail_dev:
 #ifdef CONFIG_DEBUG_FS
 	qib_dbg_exit();
 #endif
-	idr_destroy(&qib_unit_table);
 	qib_dev_cleanup();
 bail:
 	return ret;
@@ -1313,7 +1281,7 @@ static void __exit qib_ib_cleanup(void)
 	qib_cpulist_count = 0;
 	kfree(qib_cpulist);
 
-	idr_destroy(&qib_unit_table);
+	WARN_ON(!xa_empty(&qib_dev_table));
 	qib_dev_cleanup();
 }
 
diff --git a/drivers/infiniband/hw/qib/qib_rc.c b/drivers/infiniband/hw/qib/qib_rc.c
index 50dd9811b088..2ac4c67f5ba1 100644
--- a/drivers/infiniband/hw/qib/qib_rc.c
+++ b/drivers/infiniband/hw/qib/qib_rc.c
@@ -933,7 +933,7 @@ void qib_rc_send_complete(struct rvt_qp *qp, struct ib_header *hdr)
 		qp->s_last = s_last;
 		/* see post_send() */
 		barrier();
-		rvt_put_swqe(wqe);
+		rvt_put_qp_swqe(qp, wqe);
 		rvt_qp_swqe_complete(qp,
 				     wqe,
 				     ib_qib_wc_opcode[wqe->wr.opcode],
@@ -975,7 +975,7 @@ static struct rvt_swqe *do_rc_completion(struct rvt_qp *qp,
 	    qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0) {
 		u32 s_last;
 
-		rvt_put_swqe(wqe);
+		rvt_put_qp_swqe(qp, wqe);
 		s_last = qp->s_last;
 		if (++s_last >= qp->s_size)
 			s_last = 0;
diff --git a/drivers/infiniband/hw/qib/qib_user_sdma.c b/drivers/infiniband/hw/qib/qib_user_sdma.c
index 31c523b2a9f5..ef19d39a44b1 100644
--- a/drivers/infiniband/hw/qib/qib_user_sdma.c
+++ b/drivers/infiniband/hw/qib/qib_user_sdma.c
@@ -225,8 +225,6 @@ qib_user_sdma_queue_create(struct device *dev, int unit, int ctxt, int sctxt)
 	if (sdma_rb_node) {
 		sdma_rb_node->refcount++;
 	} else {
-		int ret;
-
 		sdma_rb_node = kmalloc(sizeof(
 			struct qib_user_sdma_rb_node), GFP_KERNEL);
 		if (!sdma_rb_node)
@@ -235,8 +233,7 @@ qib_user_sdma_queue_create(struct device *dev, int unit, int ctxt, int sctxt)
 		sdma_rb_node->refcount = 1;
 		sdma_rb_node->pid = current->pid;
 
-		ret = qib_user_sdma_rb_insert(&qib_user_sdma_rb_root,
-					sdma_rb_node);
+		qib_user_sdma_rb_insert(&qib_user_sdma_rb_root, sdma_rb_node);
 	}
 	pq->sdma_rb_node = sdma_rb_node;
 
diff --git a/drivers/infiniband/hw/qib/qib_verbs.h b/drivers/infiniband/hw/qib/qib_verbs.h
index a4426c24b0d1..17bdf8acee2f 100644
--- a/drivers/infiniband/hw/qib/qib_verbs.h
+++ b/drivers/infiniband/hw/qib/qib_verbs.h
@@ -46,7 +46,7 @@
 #include <rdma/ib_pack.h>
 #include <rdma/ib_user_verbs.h>
 #include <rdma/ib_hdrs.h>
-#include <rdma/rdma_vt.h>
+#include <rdma/rdmavt_qp.h>
 #include <rdma/rdmavt_cq.h>
 
 struct qib_ctxtdata;
diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
index bd4521b2cc5f..e9352750e029 100644
--- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
+++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
@@ -447,8 +447,7 @@ int usnic_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index,
 	return 0;
 }
 
-int usnic_ib_alloc_pd(struct ib_pd *ibpd, struct ib_ucontext *context,
-		      struct ib_udata *udata)
+int usnic_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 {
 	struct usnic_ib_pd *pd = to_upd(ibpd);
 	void *umem_pd;
@@ -461,7 +460,7 @@ int usnic_ib_alloc_pd(struct ib_pd *ibpd, struct ib_ucontext *context,
 	return 0;
 }
 
-void usnic_ib_dealloc_pd(struct ib_pd *pd)
+void usnic_ib_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)
 {
 	usnic_uiom_dealloc_pd((to_upd(pd))->umem_pd);
 }
@@ -539,7 +538,7 @@ out_release_mutex:
 	return ERR_PTR(err);
 }
 
-int usnic_ib_destroy_qp(struct ib_qp *qp)
+int usnic_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata)
 {
 	struct usnic_ib_qp_grp *qp_grp;
 	struct usnic_ib_vf *vf;
@@ -590,7 +589,6 @@ out_unlock:
 
 struct ib_cq *usnic_ib_create_cq(struct ib_device *ibdev,
 				 const struct ib_cq_init_attr *attr,
-				 struct ib_ucontext *context,
 				 struct ib_udata *udata)
 {
 	struct ib_cq *cq;
@@ -606,7 +604,7 @@ struct ib_cq *usnic_ib_create_cq(struct ib_device *ibdev,
 	return cq;
 }
 
-int usnic_ib_destroy_cq(struct ib_cq *cq)
+int usnic_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata)
 {
 	usnic_dbg("\n");
 	kfree(cq);
@@ -642,13 +640,13 @@ err_free:
 	return ERR_PTR(err);
 }
 
-int usnic_ib_dereg_mr(struct ib_mr *ibmr)
+int usnic_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
 {
 	struct usnic_ib_mr *mr = to_umr(ibmr);
 
 	usnic_dbg("va 0x%lx length 0x%zx\n", mr->umem->va, mr->umem->length);
 
-	usnic_uiom_reg_release(mr->umem, ibmr->uobject->context);
+	usnic_uiom_reg_release(mr->umem);
 	kfree(mr);
 	return 0;
 }
@@ -731,4 +729,3 @@ int usnic_ib_mmap(struct ib_ucontext *context,
 	return -EINVAL;
 }
 
-/* End of ib callbacks section */
diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.h b/drivers/infiniband/hw/usnic/usnic_ib_verbs.h
index c40e89b6246f..028f322f8e9b 100644
--- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.h
+++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.h
@@ -50,24 +50,22 @@ int usnic_ib_query_gid(struct ib_device *ibdev, u8 port, int index,
 				union ib_gid *gid);
 int usnic_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index,
 				u16 *pkey);
-int usnic_ib_alloc_pd(struct ib_pd *ibpd, struct ib_ucontext *context,
-		      struct ib_udata *udata);
-void usnic_ib_dealloc_pd(struct ib_pd *pd);
+int usnic_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
+void usnic_ib_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata);
 struct ib_qp *usnic_ib_create_qp(struct ib_pd *pd,
 					struct ib_qp_init_attr *init_attr,
 					struct ib_udata *udata);
-int usnic_ib_destroy_qp(struct ib_qp *qp);
+int usnic_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata);
 int usnic_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 				int attr_mask, struct ib_udata *udata);
 struct ib_cq *usnic_ib_create_cq(struct ib_device *ibdev,
 				 const struct ib_cq_init_attr *attr,
-				 struct ib_ucontext *context,
 				 struct ib_udata *udata);
-int usnic_ib_destroy_cq(struct ib_cq *cq);
+int usnic_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata);
 struct ib_mr *usnic_ib_reg_mr(struct ib_pd *pd, u64 start, u64 length,
 				u64 virt_addr, int access_flags,
 				struct ib_udata *udata);
-int usnic_ib_dereg_mr(struct ib_mr *ibmr);
+int usnic_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
 int usnic_ib_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata);
 void usnic_ib_dealloc_ucontext(struct ib_ucontext *ibcontext);
 int usnic_ib_mmap(struct ib_ucontext *context,
diff --git a/drivers/infiniband/hw/usnic/usnic_uiom.c b/drivers/infiniband/hw/usnic/usnic_uiom.c
index 06862a6af185..da35d6fdfc5e 100644
--- a/drivers/infiniband/hw/usnic/usnic_uiom.c
+++ b/drivers/infiniband/hw/usnic/usnic_uiom.c
@@ -432,8 +432,7 @@ static inline size_t usnic_uiom_num_pages(struct usnic_uiom_reg *uiomr)
 	return PAGE_ALIGN(uiomr->length + uiomr->offset) >> PAGE_SHIFT;
 }
 
-void usnic_uiom_reg_release(struct usnic_uiom_reg *uiomr,
-			    struct ib_ucontext *context)
+void usnic_uiom_reg_release(struct usnic_uiom_reg *uiomr)
 {
 	__usnic_uiom_reg_release(uiomr->pd, uiomr, 1);
 
diff --git a/drivers/infiniband/hw/usnic/usnic_uiom.h b/drivers/infiniband/hw/usnic/usnic_uiom.h
index c88cfa087e3a..70be49b1ca05 100644
--- a/drivers/infiniband/hw/usnic/usnic_uiom.h
+++ b/drivers/infiniband/hw/usnic/usnic_uiom.h
@@ -90,7 +90,6 @@ void usnic_uiom_free_dev_list(struct device **devs);
 struct usnic_uiom_reg *usnic_uiom_reg_get(struct usnic_uiom_pd *pd,
 						unsigned long addr, size_t size,
 						int access, int dmasync);
-void usnic_uiom_reg_release(struct usnic_uiom_reg *uiomr,
-			    struct ib_ucontext *ucontext);
+void usnic_uiom_reg_release(struct usnic_uiom_reg *uiomr);
 int usnic_uiom_init(char *drv_name);
 #endif /* USNIC_UIOM_H_ */
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c
index 104c7db4704f..d7deb19a2800 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c
@@ -49,6 +49,7 @@
 #include <rdma/ib_addr.h>
 #include <rdma/ib_smi.h>
 #include <rdma/ib_user_verbs.h>
+#include <rdma/uverbs_ioctl.h>
 
 #include "pvrdma.h"
 
@@ -93,7 +94,6 @@ int pvrdma_req_notify_cq(struct ib_cq *ibcq,
  * pvrdma_create_cq - create completion queue
  * @ibdev: the device
  * @attr: completion queue attributes
- * @context: user context
  * @udata: user data
  *
  * @return: ib_cq completion queue pointer on success,
@@ -101,7 +101,6 @@ int pvrdma_req_notify_cq(struct ib_cq *ibcq,
  */
 struct ib_cq *pvrdma_create_cq(struct ib_device *ibdev,
 			       const struct ib_cq_init_attr *attr,
-			       struct ib_ucontext *context,
 			       struct ib_udata *udata)
 {
 	int entries = attr->cqe;
@@ -116,6 +115,8 @@ struct ib_cq *pvrdma_create_cq(struct ib_device *ibdev,
 	struct pvrdma_cmd_create_cq_resp *resp = &rsp.create_cq_resp;
 	struct pvrdma_create_cq_resp cq_resp = {0};
 	struct pvrdma_create_cq ucmd;
+	struct pvrdma_ucontext *context = rdma_udata_to_drv_context(
+		udata, struct pvrdma_ucontext, ibucontext);
 
 	BUILD_BUG_ON(sizeof(struct pvrdma_cqe) != 64);
 
@@ -133,7 +134,7 @@ struct ib_cq *pvrdma_create_cq(struct ib_device *ibdev,
 	}
 
 	cq->ibcq.cqe = entries;
-	cq->is_kernel = !context;
+	cq->is_kernel = !udata;
 
 	if (!cq->is_kernel) {
 		if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) {
@@ -185,8 +186,7 @@ struct ib_cq *pvrdma_create_cq(struct ib_device *ibdev,
 	memset(cmd, 0, sizeof(*cmd));
 	cmd->hdr.cmd = PVRDMA_CMD_CREATE_CQ;
 	cmd->nchunks = npages;
-	cmd->ctx_handle = (context) ?
-		(u64)to_vucontext(context)->ctx_handle : 0;
+	cmd->ctx_handle = context ? context->ctx_handle : 0;
 	cmd->cqe = entries;
 	cmd->pdir_dma = cq->pdir.dir_dma;
 	ret = pvrdma_cmd_post(dev, &req, &rsp, PVRDMA_CMD_CREATE_CQ_RESP);
@@ -204,13 +204,13 @@ struct ib_cq *pvrdma_create_cq(struct ib_device *ibdev,
 	spin_unlock_irqrestore(&dev->cq_tbl_lock, flags);
 
 	if (!cq->is_kernel) {
-		cq->uar = &(to_vucontext(context)->uar);
+		cq->uar = &context->uar;
 
 		/* Copy udata back. */
 		if (ib_copy_to_udata(udata, &cq_resp, sizeof(cq_resp))) {
 			dev_warn(&dev->pdev->dev,
 				 "failed to copy back udata\n");
-			pvrdma_destroy_cq(&cq->ibcq);
+			pvrdma_destroy_cq(&cq->ibcq, udata);
 			return ERR_PTR(-EINVAL);
 		}
 	}
@@ -245,10 +245,11 @@ static void pvrdma_free_cq(struct pvrdma_dev *dev, struct pvrdma_cq *cq)
 /**
  * pvrdma_destroy_cq - destroy completion queue
  * @cq: the completion queue to destroy.
+ * @udata: user data or null for kernel object
  *
  * @return: 0 for success.
  */
-int pvrdma_destroy_cq(struct ib_cq *cq)
+int pvrdma_destroy_cq(struct ib_cq *cq, struct ib_udata *udata)
 {
 	struct pvrdma_cq *vcq = to_vcq(cq);
 	union pvrdma_cmd_req req;
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c
index ec41400fec0c..40182297f87f 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c
@@ -143,24 +143,6 @@ static int pvrdma_port_immutable(struct ib_device *ibdev, u8 port_num,
 	return 0;
 }
 
-static struct net_device *pvrdma_get_netdev(struct ib_device *ibdev,
-					    u8 port_num)
-{
-	struct net_device *netdev;
-	struct pvrdma_dev *dev = to_vdev(ibdev);
-
-	if (port_num != 1)
-		return NULL;
-
-	rcu_read_lock();
-	netdev = dev->netdev;
-	if (netdev)
-		dev_hold(netdev);
-	rcu_read_unlock();
-
-	return netdev;
-}
-
 static const struct ib_device_ops pvrdma_dev_ops = {
 	.add_gid = pvrdma_add_gid,
 	.alloc_mr = pvrdma_alloc_mr,
@@ -179,7 +161,6 @@ static const struct ib_device_ops pvrdma_dev_ops = {
 	.get_dev_fw_str = pvrdma_get_fw_ver_str,
 	.get_dma_mr = pvrdma_get_dma_mr,
 	.get_link_layer = pvrdma_port_link_layer,
-	.get_netdev = pvrdma_get_netdev,
 	.get_port_immutable = pvrdma_port_immutable,
 	.map_mr_sg = pvrdma_map_mr_sg,
 	.mmap = pvrdma_mmap,
@@ -195,6 +176,8 @@ static const struct ib_device_ops pvrdma_dev_ops = {
 	.query_qp = pvrdma_query_qp,
 	.reg_user_mr = pvrdma_reg_user_mr,
 	.req_notify_cq = pvrdma_req_notify_cq,
+
+	INIT_RDMA_OBJ_SIZE(ib_ah, pvrdma_ah, ibah),
 	INIT_RDMA_OBJ_SIZE(ib_pd, pvrdma_pd, ibpd),
 	INIT_RDMA_OBJ_SIZE(ib_ucontext, pvrdma_ucontext, ibucontext),
 };
@@ -204,6 +187,8 @@ static const struct ib_device_ops pvrdma_dev_srq_ops = {
 	.destroy_srq = pvrdma_destroy_srq,
 	.modify_srq = pvrdma_modify_srq,
 	.query_srq = pvrdma_query_srq,
+
+	INIT_RDMA_OBJ_SIZE(ib_srq, pvrdma_srq, ibsrq),
 };
 
 static int pvrdma_register_device(struct pvrdma_dev *dev)
@@ -277,6 +262,9 @@ static int pvrdma_register_device(struct pvrdma_dev *dev)
 			goto err_qp_free;
 	}
 	dev->ib_dev.driver_id = RDMA_DRIVER_VMW_PVRDMA;
+	ret = ib_device_set_netdev(&dev->ib_dev, dev->netdev, 1);
+	if (ret)
+		return ret;
 	spin_lock_init(&dev->srq_tbl_lock);
 	rdma_set_device_sysfs_group(&dev->ib_dev, &pvrdma_attr_group);
 
@@ -720,6 +708,7 @@ static void pvrdma_netdevice_event_handle(struct pvrdma_dev *dev,
 			pvrdma_dispatch_event(dev, 1, IB_EVENT_PORT_ACTIVE);
 		break;
 	case NETDEV_UNREGISTER:
+		ib_device_set_netdev(&dev->ib_dev, NULL, 1);
 		dev_put(dev->netdev);
 		dev->netdev = NULL;
 		break;
@@ -731,6 +720,7 @@ static void pvrdma_netdevice_event_handle(struct pvrdma_dev *dev,
 		if ((dev->netdev == NULL) &&
 		    (pci_get_drvdata(pdev_net) == ndev)) {
 			/* this is our netdev */
+			ib_device_set_netdev(&dev->ib_dev, ndev, 1);
 			dev->netdev = ndev;
 			dev_hold(ndev);
 		}
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c
index a85884e90e84..65dc47ffb8f3 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c
@@ -119,7 +119,7 @@ struct ib_mr *pvrdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 	union pvrdma_cmd_resp rsp;
 	struct pvrdma_cmd_create_mr *cmd = &req.create_mr;
 	struct pvrdma_cmd_create_mr_resp *resp = &rsp.create_mr_resp;
-	int ret;
+	int ret, npages;
 
 	if (length == 0 || length > dev->dsr->caps.max_mr_size) {
 		dev_warn(&dev->pdev->dev, "invalid mem region length\n");
@@ -133,9 +133,10 @@ struct ib_mr *pvrdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 		return ERR_CAST(umem);
 	}
 
-	if (umem->npages < 0 || umem->npages > PVRDMA_PAGE_DIR_MAX_PAGES) {
+	npages = ib_umem_num_pages(umem);
+	if (npages < 0 || npages > PVRDMA_PAGE_DIR_MAX_PAGES) {
 		dev_warn(&dev->pdev->dev, "overflow %d pages in mem region\n",
-			 umem->npages);
+			 npages);
 		ret = -EINVAL;
 		goto err_umem;
 	}
@@ -150,7 +151,7 @@ struct ib_mr *pvrdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 	mr->mmr.size = length;
 	mr->umem = umem;
 
-	ret = pvrdma_page_dir_init(dev, &mr->pdir, umem->npages, false);
+	ret = pvrdma_page_dir_init(dev, &mr->pdir, npages, false);
 	if (ret) {
 		dev_warn(&dev->pdev->dev,
 			 "could not allocate page directory\n");
@@ -167,7 +168,7 @@ struct ib_mr *pvrdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 	cmd->length = length;
 	cmd->pd_handle = to_vpd(pd)->pd_handle;
 	cmd->access_flags = access_flags;
-	cmd->nchunks = umem->npages;
+	cmd->nchunks = npages;
 	cmd->pdir_dma = mr->pdir.dir_dma;
 
 	ret = pvrdma_cmd_post(dev, &req, &rsp, PVRDMA_CMD_CREATE_MR_RESP);
@@ -201,7 +202,7 @@ err_umem:
  * @return: ib_mr pointer on success, otherwise returns an errno.
  */
 struct ib_mr *pvrdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
-			      u32 max_num_sg)
+			      u32 max_num_sg, struct ib_udata *udata)
 {
 	struct pvrdma_dev *dev = to_vdev(pd->device);
 	struct pvrdma_user_mr *mr;
@@ -272,7 +273,7 @@ freemr:
  *
  * @return: 0 on success.
  */
-int pvrdma_dereg_mr(struct ib_mr *ibmr)
+int pvrdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
 {
 	struct pvrdma_user_mr *mr = to_vmr(ibmr);
 	struct pvrdma_dev *dev = to_vdev(ibmr->device);
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c
index 08f4257169bd..0eaaead5baec 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c
@@ -446,10 +446,11 @@ static void pvrdma_free_qp(struct pvrdma_qp *qp)
 /**
  * pvrdma_destroy_qp - destroy a queue pair
  * @qp: the queue pair to destroy
+ * @udata: user data or null for kernel object
  *
  * @return: 0 on success.
  */
-int pvrdma_destroy_qp(struct ib_qp *qp)
+int pvrdma_destroy_qp(struct ib_qp *qp, struct ib_udata *udata)
 {
 	struct pvrdma_qp *vqp = to_vqp(qp);
 	union pvrdma_cmd_req req;
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c
index 951d9d68107a..6cac0c88cf39 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c
@@ -94,19 +94,18 @@ int pvrdma_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
  * @init_attr: shared receive queue attributes
  * @udata: user data
  *
- * @return: the ib_srq pointer on success, otherwise returns an errno.
+ * @return: 0 on success, otherwise returns an errno.
  */
-struct ib_srq *pvrdma_create_srq(struct ib_pd *pd,
-				 struct ib_srq_init_attr *init_attr,
-				 struct ib_udata *udata)
+int pvrdma_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr,
+		      struct ib_udata *udata)
 {
-	struct pvrdma_srq *srq = NULL;
-	struct pvrdma_dev *dev = to_vdev(pd->device);
+	struct pvrdma_srq *srq = to_vsrq(ibsrq);
+	struct pvrdma_dev *dev = to_vdev(ibsrq->device);
 	union pvrdma_cmd_req req;
 	union pvrdma_cmd_resp rsp;
 	struct pvrdma_cmd_create_srq *cmd = &req.create_srq;
 	struct pvrdma_cmd_create_srq_resp *resp = &rsp.create_srq_resp;
-	struct pvrdma_create_srq_resp srq_resp = {0};
+	struct pvrdma_create_srq_resp srq_resp = {};
 	struct pvrdma_create_srq ucmd;
 	unsigned long flags;
 	int ret;
@@ -115,31 +114,25 @@ struct ib_srq *pvrdma_create_srq(struct ib_pd *pd,
 		/* No support for kernel clients. */
 		dev_warn(&dev->pdev->dev,
 			 "no shared receive queue support for kernel client\n");
-		return ERR_PTR(-EOPNOTSUPP);
+		return -EOPNOTSUPP;
 	}
 
 	if (init_attr->srq_type != IB_SRQT_BASIC) {
 		dev_warn(&dev->pdev->dev,
 			 "shared receive queue type %d not supported\n",
 			 init_attr->srq_type);
-		return ERR_PTR(-EINVAL);
+		return -EINVAL;
 	}
 
 	if (init_attr->attr.max_wr  > dev->dsr->caps.max_srq_wr ||
 	    init_attr->attr.max_sge > dev->dsr->caps.max_srq_sge) {
 		dev_warn(&dev->pdev->dev,
 			 "shared receive queue size invalid\n");
-		return ERR_PTR(-EINVAL);
+		return -EINVAL;
 	}
 
 	if (!atomic_add_unless(&dev->num_srqs, 1, dev->dsr->caps.max_srq))
-		return ERR_PTR(-ENOMEM);
-
-	srq = kmalloc(sizeof(*srq), GFP_KERNEL);
-	if (!srq) {
-		ret = -ENOMEM;
-		goto err_srq;
-	}
+		return -ENOMEM;
 
 	spin_lock_init(&srq->lock);
 	refcount_set(&srq->refcnt, 1);
@@ -181,7 +174,7 @@ struct ib_srq *pvrdma_create_srq(struct ib_pd *pd,
 	cmd->hdr.cmd = PVRDMA_CMD_CREATE_SRQ;
 	cmd->srq_type = init_attr->srq_type;
 	cmd->nchunks = srq->npages;
-	cmd->pd_handle = to_vpd(pd)->pd_handle;
+	cmd->pd_handle = to_vpd(ibsrq->pd)->pd_handle;
 	cmd->attrs.max_wr = init_attr->attr.max_wr;
 	cmd->attrs.max_sge = init_attr->attr.max_sge;
 	cmd->attrs.srq_limit = init_attr->attr.srq_limit;
@@ -204,21 +197,20 @@ struct ib_srq *pvrdma_create_srq(struct ib_pd *pd,
 	/* Copy udata back. */
 	if (ib_copy_to_udata(udata, &srq_resp, sizeof(srq_resp))) {
 		dev_warn(&dev->pdev->dev, "failed to copy back udata\n");
-		pvrdma_destroy_srq(&srq->ibsrq);
-		return ERR_PTR(-EINVAL);
+		pvrdma_destroy_srq(&srq->ibsrq, udata);
+		return -EINVAL;
 	}
 
-	return &srq->ibsrq;
+	return 0;
 
 err_page_dir:
 	pvrdma_page_dir_cleanup(dev, &srq->pdir);
 err_umem:
 	ib_umem_release(srq->umem);
 err_srq:
-	kfree(srq);
 	atomic_dec(&dev->num_srqs);
 
-	return ERR_PTR(ret);
+	return ret;
 }
 
 static void pvrdma_free_srq(struct pvrdma_dev *dev, struct pvrdma_srq *srq)
@@ -246,10 +238,11 @@ static void pvrdma_free_srq(struct pvrdma_dev *dev, struct pvrdma_srq *srq)
 /**
  * pvrdma_destroy_srq - destroy shared receive queue
  * @srq: the shared receive queue to destroy
+ * @udata: user data or null for kernel object
  *
  * @return: 0 for success.
  */
-int pvrdma_destroy_srq(struct ib_srq *srq)
+void pvrdma_destroy_srq(struct ib_srq *srq, struct ib_udata *udata)
 {
 	struct pvrdma_srq *vsrq = to_vsrq(srq);
 	union pvrdma_cmd_req req;
@@ -268,8 +261,6 @@ int pvrdma_destroy_srq(struct ib_srq *srq)
 			 ret);
 
 	pvrdma_free_srq(dev, vsrq);
-
-	return 0;
 }
 
 /**
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
index 42fe821f8d58..faf7ecd7b3fa 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
@@ -50,6 +50,7 @@
 #include <rdma/ib_smi.h>
 #include <rdma/ib_user_verbs.h>
 #include <rdma/vmw_pvrdma-abi.h>
+#include <rdma/uverbs_ioctl.h>
 
 #include "pvrdma.h"
 
@@ -70,8 +71,6 @@ int pvrdma_query_device(struct ib_device *ibdev,
 	if (uhw->inlen || uhw->outlen)
 		return -EINVAL;
 
-	memset(props, 0, sizeof(*props));
-
 	props->fw_ver = dev->dsr->caps.fw_ver;
 	props->sys_image_guid = dev->dsr->caps.sys_image_guid;
 	props->max_mr_size = dev->dsr->caps.max_mr_size;
@@ -421,13 +420,11 @@ int pvrdma_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma)
 /**
  * pvrdma_alloc_pd - allocate protection domain
  * @ibpd: PD pointer
- * @context: user context
  * @udata: user data
  *
  * @return: the ib_pd protection domain pointer on success, otherwise errno.
  */
-int pvrdma_alloc_pd(struct ib_pd *ibpd, struct ib_ucontext *context,
-		    struct ib_udata *udata)
+int pvrdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 {
 	struct ib_device *ibdev = ibpd->device;
 	struct pvrdma_pd *pd = to_vpd(ibpd);
@@ -438,13 +435,15 @@ int pvrdma_alloc_pd(struct ib_pd *ibpd, struct ib_ucontext *context,
 	struct pvrdma_cmd_create_pd_resp *resp = &rsp.create_pd_resp;
 	struct pvrdma_alloc_pd_resp pd_resp = {0};
 	int ret;
+	struct pvrdma_ucontext *context = rdma_udata_to_drv_context(
+		udata, struct pvrdma_ucontext, ibucontext);
 
 	/* Check allowed max pds */
 	if (!atomic_add_unless(&dev->num_pds, 1, dev->dsr->caps.max_pd))
 		return -ENOMEM;
 
 	cmd->hdr.cmd = PVRDMA_CMD_CREATE_PD;
-	cmd->ctx_handle = (context) ? to_vucontext(context)->ctx_handle : 0;
+	cmd->ctx_handle = context ? context->ctx_handle : 0;
 	ret = pvrdma_cmd_post(dev, &req, &rsp, PVRDMA_CMD_CREATE_PD_RESP);
 	if (ret < 0) {
 		dev_warn(&dev->pdev->dev,
@@ -453,16 +452,16 @@ int pvrdma_alloc_pd(struct ib_pd *ibpd, struct ib_ucontext *context,
 		goto err;
 	}
 
-	pd->privileged = !context;
+	pd->privileged = !udata;
 	pd->pd_handle = resp->pd_handle;
 	pd->pdn = resp->pd_handle;
 	pd_resp.pdn = resp->pd_handle;
 
-	if (context) {
+	if (udata) {
 		if (ib_copy_to_udata(udata, &pd_resp, sizeof(pd_resp))) {
 			dev_warn(&dev->pdev->dev,
 				 "failed to copy back protection domain\n");
-			pvrdma_dealloc_pd(&pd->ibpd);
+			pvrdma_dealloc_pd(&pd->ibpd, udata);
 			return -EFAULT;
 		}
 	}
@@ -478,10 +477,11 @@ err:
 /**
  * pvrdma_dealloc_pd - deallocate protection domain
  * @pd: the protection domain to be released
+ * @udata: user data or null for kernel object
  *
  * @return: 0 on success, otherwise errno.
  */
-void pvrdma_dealloc_pd(struct ib_pd *pd)
+void pvrdma_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)
 {
 	struct pvrdma_dev *dev = to_vdev(pd->device);
 	union pvrdma_cmd_req req = {};
@@ -507,34 +507,28 @@ void pvrdma_dealloc_pd(struct ib_pd *pd)
  * @udata: user data blob
  * @flags: create address handle flags (see enum rdma_create_ah_flags)
  *
- * @return: the ib_ah pointer on success, otherwise errno.
+ * @return: 0 on success, otherwise errno.
  */
-struct ib_ah *pvrdma_create_ah(struct ib_pd *pd, struct rdma_ah_attr *ah_attr,
-			       u32 flags, struct ib_udata *udata)
+int pvrdma_create_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr,
+		     u32 flags, struct ib_udata *udata)
 {
-	struct pvrdma_dev *dev = to_vdev(pd->device);
-	struct pvrdma_ah *ah;
+	struct pvrdma_dev *dev = to_vdev(ibah->device);
+	struct pvrdma_ah *ah = to_vah(ibah);
 	const struct ib_global_route *grh;
 	u8 port_num = rdma_ah_get_port_num(ah_attr);
 
 	if (!(rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH))
-		return ERR_PTR(-EINVAL);
+		return -EINVAL;
 
 	grh = rdma_ah_read_grh(ah_attr);
 	if ((ah_attr->type != RDMA_AH_ATTR_TYPE_ROCE)  ||
 	    rdma_is_multicast_addr((struct in6_addr *)grh->dgid.raw))
-		return ERR_PTR(-EINVAL);
+		return -EINVAL;
 
 	if (!atomic_add_unless(&dev->num_ahs, 1, dev->dsr->caps.max_ah))
-		return ERR_PTR(-ENOMEM);
-
-	ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
-	if (!ah) {
-		atomic_dec(&dev->num_ahs);
-		return ERR_PTR(-ENOMEM);
-	}
+		return -ENOMEM;
 
-	ah->av.port_pd = to_vpd(pd)->pd_handle | (port_num << 24);
+	ah->av.port_pd = to_vpd(ibah->pd)->pd_handle | (port_num << 24);
 	ah->av.src_path_bits = rdma_ah_get_path_bits(ah_attr);
 	ah->av.src_path_bits |= 0x80;
 	ah->av.gid_index = grh->sgid_index;
@@ -544,11 +538,7 @@ struct ib_ah *pvrdma_create_ah(struct ib_pd *pd, struct rdma_ah_attr *ah_attr,
 	memcpy(ah->av.dgid, grh->dgid.raw, 16);
 	memcpy(ah->av.dmac, ah_attr->roce.dmac, ETH_ALEN);
 
-	ah->ibah.device = pd->device;
-	ah->ibah.pd = pd;
-	ah->ibah.uobject = NULL;
-
-	return &ah->ibah;
+	return 0;
 }
 
 /**
@@ -556,14 +546,10 @@ struct ib_ah *pvrdma_create_ah(struct ib_pd *pd, struct rdma_ah_attr *ah_attr,
  * @ah: the address handle to destroyed
  * @flags: destroy address handle flags (see enum rdma_destroy_ah_flags)
  *
- * @return: 0 on success.
  */
-int pvrdma_destroy_ah(struct ib_ah *ah, u32 flags)
+void pvrdma_destroy_ah(struct ib_ah *ah, u32 flags)
 {
 	struct pvrdma_dev *dev = to_vdev(ah->device);
 
-	kfree(to_vah(ah));
 	atomic_dec(&dev->num_ahs);
-
-	return 0;
 }
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h
index 607aa131d67c..9d7b021e1c59 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h
@@ -398,36 +398,33 @@ int pvrdma_modify_port(struct ib_device *ibdev, u8 port,
 int pvrdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma);
 int pvrdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata);
 void pvrdma_dealloc_ucontext(struct ib_ucontext *context);
-int pvrdma_alloc_pd(struct ib_pd *pd, struct ib_ucontext *context,
-		    struct ib_udata *udata);
-void pvrdma_dealloc_pd(struct ib_pd *ibpd);
+int pvrdma_alloc_pd(struct ib_pd *pd, struct ib_udata *udata);
+void pvrdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
 struct ib_mr *pvrdma_get_dma_mr(struct ib_pd *pd, int acc);
 struct ib_mr *pvrdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 				 u64 virt_addr, int access_flags,
 				 struct ib_udata *udata);
-int pvrdma_dereg_mr(struct ib_mr *mr);
+int pvrdma_dereg_mr(struct ib_mr *mr, struct ib_udata *udata);
 struct ib_mr *pvrdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
-			      u32 max_num_sg);
+			      u32 max_num_sg, struct ib_udata *udata);
 int pvrdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
 		     int sg_nents, unsigned int *sg_offset);
 struct ib_cq *pvrdma_create_cq(struct ib_device *ibdev,
 			       const struct ib_cq_init_attr *attr,
-			       struct ib_ucontext *context,
 			       struct ib_udata *udata);
-int pvrdma_destroy_cq(struct ib_cq *cq);
+int pvrdma_destroy_cq(struct ib_cq *cq, struct ib_udata *udata);
 int pvrdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
 int pvrdma_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags);
-struct ib_ah *pvrdma_create_ah(struct ib_pd *pd, struct rdma_ah_attr *ah_attr,
-			       u32 flags, struct ib_udata *udata);
-int pvrdma_destroy_ah(struct ib_ah *ah, u32 flags);
+int pvrdma_create_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr, u32 flags,
+		     struct ib_udata *udata);
+void pvrdma_destroy_ah(struct ib_ah *ah, u32 flags);
 
-struct ib_srq *pvrdma_create_srq(struct ib_pd *pd,
-				 struct ib_srq_init_attr *init_attr,
-				 struct ib_udata *udata);
+int pvrdma_create_srq(struct ib_srq *srq, struct ib_srq_init_attr *init_attr,
+		      struct ib_udata *udata);
 int pvrdma_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
 		      enum ib_srq_attr_mask attr_mask, struct ib_udata *udata);
 int pvrdma_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr);
-int pvrdma_destroy_srq(struct ib_srq *srq);
+void pvrdma_destroy_srq(struct ib_srq *srq, struct ib_udata *udata);
 
 struct ib_qp *pvrdma_create_qp(struct ib_pd *pd,
 			       struct ib_qp_init_attr *init_attr,
@@ -436,7 +433,7 @@ int pvrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 		     int attr_mask, struct ib_udata *udata);
 int pvrdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
 		    int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr);
-int pvrdma_destroy_qp(struct ib_qp *qp);
+int pvrdma_destroy_qp(struct ib_qp *qp, struct ib_udata *udata);
 int pvrdma_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
 		     const struct ib_send_wr **bad_wr);
 int pvrdma_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
diff --git a/drivers/infiniband/sw/rdmavt/ah.c b/drivers/infiniband/sw/rdmavt/ah.c
index fc10e4e26ca7..0e147b32cbe9 100644
--- a/drivers/infiniband/sw/rdmavt/ah.c
+++ b/drivers/infiniband/sw/rdmavt/ah.c
@@ -89,36 +89,29 @@ EXPORT_SYMBOL(rvt_check_ah);
 
 /**
  * rvt_create_ah - create an address handle
- * @pd: the protection domain
+ * @ibah: the IB address handle
  * @ah_attr: the attributes of the AH
  * @create_flags: create address handle flags (see enum rdma_create_ah_flags)
  * @udata: pointer to user's input output buffer information.
  *
  * This may be called from interrupt context.
  *
- * Return: newly allocated ah
+ * Return: 0 on success
  */
-struct ib_ah *rvt_create_ah(struct ib_pd *pd,
-			    struct rdma_ah_attr *ah_attr,
-			    u32 create_flags,
-			    struct ib_udata *udata)
+int rvt_create_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr,
+		  u32 create_flags, struct ib_udata *udata)
 {
-	struct rvt_ah *ah;
-	struct rvt_dev_info *dev = ib_to_rvt(pd->device);
+	struct rvt_ah *ah = ibah_to_rvtah(ibah);
+	struct rvt_dev_info *dev = ib_to_rvt(ibah->device);
 	unsigned long flags;
 
-	if (rvt_check_ah(pd->device, ah_attr))
-		return ERR_PTR(-EINVAL);
-
-	ah = kmalloc(sizeof(*ah), GFP_ATOMIC);
-	if (!ah)
-		return ERR_PTR(-ENOMEM);
+	if (rvt_check_ah(ibah->device, ah_attr))
+		return -EINVAL;
 
 	spin_lock_irqsave(&dev->n_ahs_lock, flags);
 	if (dev->n_ahs_allocated == dev->dparms.props.max_ah) {
 		spin_unlock_irqrestore(&dev->n_ahs_lock, flags);
-		kfree(ah);
-		return ERR_PTR(-ENOMEM);
+		return -ENOMEM;
 	}
 
 	dev->n_ahs_allocated++;
@@ -129,35 +122,32 @@ struct ib_ah *rvt_create_ah(struct ib_pd *pd,
 	atomic_set(&ah->refcount, 0);
 
 	if (dev->driver_f.notify_new_ah)
-		dev->driver_f.notify_new_ah(pd->device, ah_attr, ah);
+		dev->driver_f.notify_new_ah(ibah->device, ah_attr, ah);
 
-	return &ah->ibah;
+	return 0;
 }
 
 /**
  * rvt_destory_ah - Destory an address handle
  * @ibah: address handle
  * @destroy_flags: destroy address handle flags (see enum rdma_destroy_ah_flags)
+ * @udata: user data or NULL for kernel object
  *
  * Return: 0 on success
  */
-int rvt_destroy_ah(struct ib_ah *ibah, u32 destroy_flags)
+void rvt_destroy_ah(struct ib_ah *ibah, u32 destroy_flags)
 {
 	struct rvt_dev_info *dev = ib_to_rvt(ibah->device);
 	struct rvt_ah *ah = ibah_to_rvtah(ibah);
 	unsigned long flags;
 
-	if (atomic_read(&ah->refcount) != 0)
-		return -EBUSY;
+	WARN_ON_ONCE(atomic_read(&ah->refcount));
 
 	spin_lock_irqsave(&dev->n_ahs_lock, flags);
 	dev->n_ahs_allocated--;
 	spin_unlock_irqrestore(&dev->n_ahs_lock, flags);
 
 	rdma_destroy_ah_attr(&ah->attr);
-	kfree(ah);
-
-	return 0;
 }
 
 /**
diff --git a/drivers/infiniband/sw/rdmavt/ah.h b/drivers/infiniband/sw/rdmavt/ah.h
index 72431a618d5d..bbb4d3bdec4e 100644
--- a/drivers/infiniband/sw/rdmavt/ah.h
+++ b/drivers/infiniband/sw/rdmavt/ah.h
@@ -50,11 +50,9 @@
 
 #include <rdma/rdma_vt.h>
 
-struct ib_ah *rvt_create_ah(struct ib_pd *pd,
-			    struct rdma_ah_attr *ah_attr,
-			    u32 create_flags,
-			    struct ib_udata *udata);
-int rvt_destroy_ah(struct ib_ah *ibah, u32 destroy_flags);
+int rvt_create_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr,
+		  u32 create_flags, struct ib_udata *udata);
+void rvt_destroy_ah(struct ib_ah *ibah, u32 destroy_flags);
 int rvt_modify_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr);
 int rvt_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr);
 
diff --git a/drivers/infiniband/sw/rdmavt/cq.c b/drivers/infiniband/sw/rdmavt/cq.c
index 4f1544ad4aff..a06e6da7a026 100644
--- a/drivers/infiniband/sw/rdmavt/cq.c
+++ b/drivers/infiniband/sw/rdmavt/cq.c
@@ -168,7 +168,6 @@ static void send_complete(struct work_struct *work)
  * rvt_create_cq - create a completion queue
  * @ibdev: the device this completion queue is attached to
  * @attr: creation attributes
- * @context: unused by the QLogic_IB driver
  * @udata: user data for libibverbs.so
  *
  * Called by ib_create_cq() in the generic verbs code.
@@ -178,7 +177,6 @@ static void send_complete(struct work_struct *work)
  */
 struct ib_cq *rvt_create_cq(struct ib_device *ibdev,
 			    const struct ib_cq_init_attr *attr,
-			    struct ib_ucontext *context,
 			    struct ib_udata *udata)
 {
 	struct rvt_dev_info *rdi = ib_to_rvt(ibdev);
@@ -232,7 +230,7 @@ struct ib_cq *rvt_create_cq(struct ib_device *ibdev,
 	if (udata && udata->outlen >= sizeof(__u64)) {
 		int err;
 
-		cq->ip = rvt_create_mmap_info(rdi, sz, context, wc);
+		cq->ip = rvt_create_mmap_info(rdi, sz, udata, wc);
 		if (!cq->ip) {
 			ret = ERR_PTR(-ENOMEM);
 			goto bail_wc;
@@ -299,12 +297,13 @@ done:
 /**
  * rvt_destroy_cq - destroy a completion queue
  * @ibcq: the completion queue to destroy.
+ * @udata: user data or NULL for kernel object
  *
  * Called by ib_destroy_cq() in the generic verbs code.
  *
  * Return: always 0
  */
-int rvt_destroy_cq(struct ib_cq *ibcq)
+int rvt_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
 {
 	struct rvt_cq *cq = ibcq_to_rvtcq(ibcq);
 	struct rvt_dev_info *rdi = cq->rdi;
diff --git a/drivers/infiniband/sw/rdmavt/cq.h b/drivers/infiniband/sw/rdmavt/cq.h
index 72184b1c176b..3ad6faf18ecb 100644
--- a/drivers/infiniband/sw/rdmavt/cq.h
+++ b/drivers/infiniband/sw/rdmavt/cq.h
@@ -53,9 +53,8 @@
 
 struct ib_cq *rvt_create_cq(struct ib_device *ibdev,
 			    const struct ib_cq_init_attr *attr,
-			    struct ib_ucontext *context,
 			    struct ib_udata *udata);
-int rvt_destroy_cq(struct ib_cq *ibcq);
+int rvt_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
 int rvt_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags notify_flags);
 int rvt_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata);
 int rvt_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry);
diff --git a/drivers/infiniband/sw/rdmavt/mmap.c b/drivers/infiniband/sw/rdmavt/mmap.c
index 6b712eecbd37..652f4a7efc1b 100644
--- a/drivers/infiniband/sw/rdmavt/mmap.c
+++ b/drivers/infiniband/sw/rdmavt/mmap.c
@@ -49,6 +49,7 @@
 #include <linux/vmalloc.h>
 #include <linux/mm.h>
 #include <asm/pgtable.h>
+#include <rdma/uverbs_ioctl.h>
 #include "mmap.h"
 
 /**
@@ -150,18 +151,19 @@ done:
  * rvt_create_mmap_info - allocate information for hfi1_mmap
  * @rdi: rvt dev struct
  * @size: size in bytes to map
- * @context: user context
+ * @udata: user data (must be valid!)
  * @obj: opaque pointer to a cq, wq etc
  *
  * Return: rvt_mmap struct on success
  */
-struct rvt_mmap_info *rvt_create_mmap_info(struct rvt_dev_info *rdi,
-					   u32 size,
-					   struct ib_ucontext *context,
-					   void *obj)
+struct rvt_mmap_info *rvt_create_mmap_info(struct rvt_dev_info *rdi, u32 size,
+					   struct ib_udata *udata, void *obj)
 {
 	struct rvt_mmap_info *ip;
 
+	if (!udata)
+		return ERR_PTR(-EINVAL);
+
 	ip = kmalloc_node(sizeof(*ip), GFP_KERNEL, rdi->dparms.node);
 	if (!ip)
 		return ip;
@@ -177,7 +179,9 @@ struct rvt_mmap_info *rvt_create_mmap_info(struct rvt_dev_info *rdi,
 
 	INIT_LIST_HEAD(&ip->pending_mmaps);
 	ip->size = size;
-	ip->context = context;
+	ip->context =
+		container_of(udata, struct uverbs_attr_bundle, driver_udata)
+			->context;
 	ip->obj = obj;
 	kref_init(&ip->ref);
 
diff --git a/drivers/infiniband/sw/rdmavt/mmap.h b/drivers/infiniband/sw/rdmavt/mmap.h
index fab0e7b1daf9..02466c40bc1e 100644
--- a/drivers/infiniband/sw/rdmavt/mmap.h
+++ b/drivers/infiniband/sw/rdmavt/mmap.h
@@ -53,10 +53,8 @@
 void rvt_mmap_init(struct rvt_dev_info *rdi);
 void rvt_release_mmap_info(struct kref *ref);
 int rvt_mmap(struct ib_ucontext *context, struct vm_area_struct *vma);
-struct rvt_mmap_info *rvt_create_mmap_info(struct rvt_dev_info *rdi,
-					   u32 size,
-					   struct ib_ucontext *context,
-					   void *obj);
+struct rvt_mmap_info *rvt_create_mmap_info(struct rvt_dev_info *rdi, u32 size,
+					   struct ib_udata *udata, void *obj);
 void rvt_update_mmap_info(struct rvt_dev_info *rdi, struct rvt_mmap_info *ip,
 			  u32 size, void *obj);
 
diff --git a/drivers/infiniband/sw/rdmavt/mr.c b/drivers/infiniband/sw/rdmavt/mr.c
index 0bb6e39dd03a..54f3f9c27552 100644
--- a/drivers/infiniband/sw/rdmavt/mr.c
+++ b/drivers/infiniband/sw/rdmavt/mr.c
@@ -392,7 +392,7 @@ struct ib_mr *rvt_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 	if (IS_ERR(umem))
 		return (void *)umem;
 
-	n = umem->nmap;
+	n = ib_umem_num_pages(umem);
 
 	mr = __rvt_alloc_mr(n, pd);
 	if (IS_ERR(mr)) {
@@ -548,7 +548,7 @@ bool rvt_ss_has_lkey(struct rvt_sge_state *ss, u32 lkey)
  *
  * Returns 0 on success.
  */
-int rvt_dereg_mr(struct ib_mr *ibmr)
+int rvt_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
 {
 	struct rvt_mr *mr = to_imr(ibmr);
 	int ret;
@@ -575,9 +575,8 @@ out:
  *
  * Return: the memory region on success, otherwise return an errno.
  */
-struct ib_mr *rvt_alloc_mr(struct ib_pd *pd,
-			   enum ib_mr_type mr_type,
-			   u32 max_num_sg)
+struct ib_mr *rvt_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
+			   u32 max_num_sg, struct ib_udata *udata)
 {
 	struct rvt_mr *mr;
 
diff --git a/drivers/infiniband/sw/rdmavt/mr.h b/drivers/infiniband/sw/rdmavt/mr.h
index 132800ee0205..2c8d0752e8e3 100644
--- a/drivers/infiniband/sw/rdmavt/mr.h
+++ b/drivers/infiniband/sw/rdmavt/mr.h
@@ -78,10 +78,9 @@ struct ib_mr *rvt_get_dma_mr(struct ib_pd *pd, int acc);
 struct ib_mr *rvt_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 			      u64 virt_addr, int mr_access_flags,
 			      struct ib_udata *udata);
-int rvt_dereg_mr(struct ib_mr *ibmr);
-struct ib_mr *rvt_alloc_mr(struct ib_pd *pd,
-			   enum ib_mr_type mr_type,
-			   u32 max_num_sg);
+int rvt_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
+struct ib_mr *rvt_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
+			   u32 max_num_sg, struct ib_udata *udata);
 int rvt_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
 		  int sg_nents, unsigned int *sg_offset);
 struct ib_fmr *rvt_alloc_fmr(struct ib_pd *pd, int mr_access_flags,
diff --git a/drivers/infiniband/sw/rdmavt/pd.c b/drivers/infiniband/sw/rdmavt/pd.c
index 6033054b22fa..a403718f0b5e 100644
--- a/drivers/infiniband/sw/rdmavt/pd.c
+++ b/drivers/infiniband/sw/rdmavt/pd.c
@@ -51,15 +51,13 @@
 /**
  * rvt_alloc_pd - allocate a protection domain
  * @ibpd: PD
- * @context: optional user context
  * @udata: optional user data
  *
  * Allocate and keep track of a PD.
  *
  * Return: 0 on success
  */
-int rvt_alloc_pd(struct ib_pd *ibpd, struct ib_ucontext *context,
-		 struct ib_udata *udata)
+int rvt_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 {
 	struct ib_device *ibdev = ibpd->device;
 	struct rvt_dev_info *dev = ib_to_rvt(ibdev);
@@ -93,10 +91,11 @@ bail:
 /**
  * rvt_dealloc_pd - Free PD
  * @ibpd: Free up PD
+ * @udata: Valid user data or NULL for kernel object
  *
  * Return: always 0
  */
-void rvt_dealloc_pd(struct ib_pd *ibpd)
+void rvt_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 {
 	struct rvt_dev_info *dev = ib_to_rvt(ibpd->device);
 
diff --git a/drivers/infiniband/sw/rdmavt/pd.h b/drivers/infiniband/sw/rdmavt/pd.h
index 7a887e4a45e7..71ba76d72b1d 100644
--- a/drivers/infiniband/sw/rdmavt/pd.h
+++ b/drivers/infiniband/sw/rdmavt/pd.h
@@ -50,8 +50,7 @@
 
 #include <rdma/rdma_vt.h>
 
-int rvt_alloc_pd(struct ib_pd *pd, struct ib_ucontext *context,
-		 struct ib_udata *udata);
-void rvt_dealloc_pd(struct ib_pd *ibpd);
+int rvt_alloc_pd(struct ib_pd *pd, struct ib_udata *udata);
+void rvt_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
 
 #endif          /* DEF_RDMAVTPD_H */
diff --git a/drivers/infiniband/sw/rdmavt/qp.c b/drivers/infiniband/sw/rdmavt/qp.c
index a34b9a2a32b6..31a2e65e4906 100644
--- a/drivers/infiniband/sw/rdmavt/qp.c
+++ b/drivers/infiniband/sw/rdmavt/qp.c
@@ -623,13 +623,7 @@ static void rvt_clear_mr_refs(struct rvt_qp *qp, int clr_sends)
 		while (qp->s_last != qp->s_head) {
 			struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, qp->s_last);
 
-			rvt_put_swqe(wqe);
-
-			if (qp->ibqp.qp_type == IB_QPT_UD ||
-			    qp->ibqp.qp_type == IB_QPT_SMI ||
-			    qp->ibqp.qp_type == IB_QPT_GSI)
-				atomic_dec(&ibah_to_rvtah(
-						wqe->ud_wr.ah)->refcount);
+			rvt_put_qp_swqe(qp, wqe);
 			if (++qp->s_last >= qp->s_size)
 				qp->s_last = 0;
 			smp_wmb(); /* see qp_set_savail */
@@ -957,8 +951,6 @@ struct ib_qp *rvt_create_qp(struct ib_pd *ibpd,
 	size_t sg_list_sz;
 	struct ib_qp *ret = ERR_PTR(-ENOMEM);
 	struct rvt_dev_info *rdi = ib_to_rvt(ibpd->device);
-	struct rvt_ucontext *ucontext = rdma_udata_to_drv_context(
-		udata, struct rvt_ucontext, ibucontext);
 	void *priv = NULL;
 	size_t sqsize;
 
@@ -1131,8 +1123,7 @@ struct ib_qp *rvt_create_qp(struct ib_pd *ibpd,
 		} else {
 			u32 s = sizeof(struct rvt_rwq) + qp->r_rq.size * sz;
 
-			qp->ip = rvt_create_mmap_info(rdi, s,
-						      &ucontext->ibucontext,
+			qp->ip = rvt_create_mmap_info(rdi, s, udata,
 						      qp->r_rq.wq);
 			if (!qp->ip) {
 				ret = ERR_PTR(-ENOMEM);
@@ -1617,7 +1608,7 @@ inval:
  *
  * Return: 0 on success.
  */
-int rvt_destroy_qp(struct ib_qp *ibqp)
+int rvt_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
 {
 	struct rvt_qp *qp = ibqp_to_rvtqp(ibqp);
 	struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device);
@@ -2018,8 +2009,7 @@ static int rvt_post_one_wr(struct rvt_qp *qp,
 	 * opportunity to adjust PSN values based on internal checks.
 	 */
 	log_pmtu = qp->log_pmtu;
-	if (qp->ibqp.qp_type != IB_QPT_UC &&
-	    qp->ibqp.qp_type != IB_QPT_RC) {
+	if (qp->allowed_ops == IB_OPCODE_UD) {
 		struct rvt_ah *ah = ibah_to_rvtah(wqe->ud_wr.ah);
 
 		log_pmtu = ah->log_pmtu;
@@ -2067,8 +2057,7 @@ static int rvt_post_one_wr(struct rvt_qp *qp,
 	return 0;
 
 bail_inval_free_ref:
-	if (qp->ibqp.qp_type != IB_QPT_UC &&
-	    qp->ibqp.qp_type != IB_QPT_RC)
+	if (qp->allowed_ops == IB_OPCODE_UD)
 		atomic_dec(&ibah_to_rvtah(ud_wr(wr)->ah)->refcount);
 bail_inval_free:
 	/* release mr holds */
@@ -2691,11 +2680,7 @@ void rvt_send_complete(struct rvt_qp *qp, struct rvt_swqe *wqe,
 	qp->s_last = last;
 	/* See post_send() */
 	barrier();
-	rvt_put_swqe(wqe);
-	if (qp->ibqp.qp_type == IB_QPT_UD ||
-	    qp->ibqp.qp_type == IB_QPT_SMI ||
-	    qp->ibqp.qp_type == IB_QPT_GSI)
-		atomic_dec(&ibah_to_rvtah(wqe->ud_wr.ah)->refcount);
+	rvt_put_qp_swqe(qp, wqe);
 
 	rvt_qp_swqe_complete(qp,
 			     wqe,
diff --git a/drivers/infiniband/sw/rdmavt/qp.h b/drivers/infiniband/sw/rdmavt/qp.h
index 6d883972e0b8..6db1619389b0 100644
--- a/drivers/infiniband/sw/rdmavt/qp.h
+++ b/drivers/infiniband/sw/rdmavt/qp.h
@@ -48,7 +48,7 @@
  *
  */
 
-#include <rdma/rdma_vt.h>
+#include <rdma/rdmavt_qp.h>
 
 int rvt_driver_qp_init(struct rvt_dev_info *rdi);
 void rvt_qp_exit(struct rvt_dev_info *rdi);
@@ -57,7 +57,7 @@ struct ib_qp *rvt_create_qp(struct ib_pd *ibpd,
 			    struct ib_udata *udata);
 int rvt_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 		  int attr_mask, struct ib_udata *udata);
-int rvt_destroy_qp(struct ib_qp *ibqp);
+int rvt_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);
 int rvt_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 		 int attr_mask, struct ib_qp_init_attr *init_attr);
 int rvt_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
diff --git a/drivers/infiniband/sw/rdmavt/rc.c b/drivers/infiniband/sw/rdmavt/rc.c
index 8d71647820a8..09f0cf538be6 100644
--- a/drivers/infiniband/sw/rdmavt/rc.c
+++ b/drivers/infiniband/sw/rdmavt/rc.c
@@ -45,7 +45,7 @@
  *
  */
 
-#include <rdma/rdma_vt.h>
+#include <rdma/rdmavt_qp.h>
 #include <rdma/ib_hdrs.h>
 
 /*
diff --git a/drivers/infiniband/sw/rdmavt/srq.c b/drivers/infiniband/sw/rdmavt/srq.c
index 895b3fabd0bf..8d6b3e764255 100644
--- a/drivers/infiniband/sw/rdmavt/srq.c
+++ b/drivers/infiniband/sw/rdmavt/srq.c
@@ -71,31 +71,24 @@ void rvt_driver_srq_init(struct rvt_dev_info *rdi)
  * @srq_init_attr: the attributes of the SRQ
  * @udata: data from libibverbs when creating a user SRQ
  *
- * Return: Allocated srq object
+ * Return: 0 on success
  */
-struct ib_srq *rvt_create_srq(struct ib_pd *ibpd,
-			      struct ib_srq_init_attr *srq_init_attr,
-			      struct ib_udata *udata)
+int rvt_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *srq_init_attr,
+		   struct ib_udata *udata)
 {
-	struct rvt_dev_info *dev = ib_to_rvt(ibpd->device);
-	struct rvt_ucontext *ucontext = rdma_udata_to_drv_context(
-		udata, struct rvt_ucontext, ibucontext);
-	struct rvt_srq *srq;
+	struct rvt_dev_info *dev = ib_to_rvt(ibsrq->device);
+	struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq);
 	u32 sz;
-	struct ib_srq *ret;
+	int ret;
 
 	if (srq_init_attr->srq_type != IB_SRQT_BASIC)
-		return ERR_PTR(-EOPNOTSUPP);
+		return -EOPNOTSUPP;
 
 	if (srq_init_attr->attr.max_sge == 0 ||
 	    srq_init_attr->attr.max_sge > dev->dparms.props.max_srq_sge ||
 	    srq_init_attr->attr.max_wr == 0 ||
 	    srq_init_attr->attr.max_wr > dev->dparms.props.max_srq_wr)
-		return ERR_PTR(-EINVAL);
-
-	srq = kzalloc_node(sizeof(*srq), GFP_KERNEL, dev->dparms.node);
-	if (!srq)
-		return ERR_PTR(-ENOMEM);
+		return -EINVAL;
 
 	/*
 	 * Need to use vmalloc() if we want to support large #s of entries.
@@ -109,7 +102,7 @@ struct ib_srq *rvt_create_srq(struct ib_pd *ibpd,
 		vzalloc_node(sizeof(struct rvt_rwq) + srq->rq.size * sz,
 			     dev->dparms.node);
 	if (!srq->rq.wq) {
-		ret = ERR_PTR(-ENOMEM);
+		ret = -ENOMEM;
 		goto bail_srq;
 	}
 
@@ -118,23 +111,18 @@ struct ib_srq *rvt_create_srq(struct ib_pd *ibpd,
 	 * See rvt_mmap() for details.
 	 */
 	if (udata && udata->outlen >= sizeof(__u64)) {
-		int err;
 		u32 s = sizeof(struct rvt_rwq) + srq->rq.size * sz;
 
-		srq->ip =
-		    rvt_create_mmap_info(dev, s, &ucontext->ibucontext,
-					 srq->rq.wq);
+		srq->ip = rvt_create_mmap_info(dev, s, udata, srq->rq.wq);
 		if (!srq->ip) {
-			ret = ERR_PTR(-ENOMEM);
+			ret = -ENOMEM;
 			goto bail_wq;
 		}
 
-		err = ib_copy_to_udata(udata, &srq->ip->offset,
+		ret = ib_copy_to_udata(udata, &srq->ip->offset,
 				       sizeof(srq->ip->offset));
-		if (err) {
-			ret = ERR_PTR(err);
+		if (ret)
 			goto bail_ip;
-		}
 	}
 
 	/*
@@ -146,7 +134,7 @@ struct ib_srq *rvt_create_srq(struct ib_pd *ibpd,
 	spin_lock(&dev->n_srqs_lock);
 	if (dev->n_srqs_allocated == dev->dparms.props.max_srq) {
 		spin_unlock(&dev->n_srqs_lock);
-		ret = ERR_PTR(-ENOMEM);
+		ret = -ENOMEM;
 		goto bail_ip;
 	}
 
@@ -159,14 +147,13 @@ struct ib_srq *rvt_create_srq(struct ib_pd *ibpd,
 		spin_unlock_irq(&dev->pending_lock);
 	}
 
-	return &srq->ibsrq;
+	return 0;
 
 bail_ip:
 	kfree(srq->ip);
 bail_wq:
 	vfree(srq->rq.wq);
 bail_srq:
-	kfree(srq);
 	return ret;
 }
 
@@ -338,9 +325,8 @@ int rvt_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr)
  * rvt_destroy_srq - destory an srq
  * @ibsrq: srq object to destroy
  *
- * Return always 0
  */
-int rvt_destroy_srq(struct ib_srq *ibsrq)
+void rvt_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
 {
 	struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq);
 	struct rvt_dev_info *dev = ib_to_rvt(ibsrq->device);
@@ -352,7 +338,4 @@ int rvt_destroy_srq(struct ib_srq *ibsrq)
 		kref_put(&srq->ip->ref, rvt_release_mmap_info);
 	else
 		vfree(srq->rq.wq);
-	kfree(srq);
-
-	return 0;
 }
diff --git a/drivers/infiniband/sw/rdmavt/srq.h b/drivers/infiniband/sw/rdmavt/srq.h
index bf0eaaf56465..6427d7d62a9a 100644
--- a/drivers/infiniband/sw/rdmavt/srq.h
+++ b/drivers/infiniband/sw/rdmavt/srq.h
@@ -50,13 +50,12 @@
 
 #include <rdma/rdma_vt.h>
 void rvt_driver_srq_init(struct rvt_dev_info *rdi);
-struct ib_srq *rvt_create_srq(struct ib_pd *ibpd,
-			      struct ib_srq_init_attr *srq_init_attr,
-			      struct ib_udata *udata);
+int rvt_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *srq_init_attr,
+		   struct ib_udata *udata);
 int rvt_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
 		   enum ib_srq_attr_mask attr_mask,
 		   struct ib_udata *udata);
 int rvt_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr);
-int rvt_destroy_srq(struct ib_srq *ibsrq);
+void rvt_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata);
 
 #endif          /* DEF_RVTSRQ_H */
diff --git a/drivers/infiniband/sw/rdmavt/trace_qp.h b/drivers/infiniband/sw/rdmavt/trace_qp.h
index efc9d814b032..c32d21cc615e 100644
--- a/drivers/infiniband/sw/rdmavt/trace_qp.h
+++ b/drivers/infiniband/sw/rdmavt/trace_qp.h
@@ -51,7 +51,7 @@
 #include <linux/trace_seq.h>
 
 #include <rdma/ib_verbs.h>
-#include <rdma/rdma_vt.h>
+#include <rdma/rdmavt_qp.h>
 
 #undef TRACE_SYSTEM
 #define TRACE_SYSTEM rvt_qp
diff --git a/drivers/infiniband/sw/rdmavt/trace_rc.h b/drivers/infiniband/sw/rdmavt/trace_rc.h
index 995276933a55..c47357af2099 100644
--- a/drivers/infiniband/sw/rdmavt/trace_rc.h
+++ b/drivers/infiniband/sw/rdmavt/trace_rc.h
@@ -51,7 +51,7 @@
 #include <linux/trace_seq.h>
 
 #include <rdma/ib_verbs.h>
-#include <rdma/rdma_vt.h>
+#include <rdma/rdmavt_qp.h>
 
 #undef TRACE_SYSTEM
 #define TRACE_SYSTEM rvt_rc
diff --git a/drivers/infiniband/sw/rdmavt/trace_tx.h b/drivers/infiniband/sw/rdmavt/trace_tx.h
index d5df352eadb1..d963ca755828 100644
--- a/drivers/infiniband/sw/rdmavt/trace_tx.h
+++ b/drivers/infiniband/sw/rdmavt/trace_tx.h
@@ -51,7 +51,7 @@
 #include <linux/trace_seq.h>
 
 #include <rdma/ib_verbs.h>
-#include <rdma/rdma_vt.h>
+#include <rdma/rdmavt_qp.h>
 
 #undef TRACE_SYSTEM
 #define TRACE_SYSTEM rvt_tx
diff --git a/drivers/infiniband/sw/rdmavt/vt.c b/drivers/infiniband/sw/rdmavt/vt.c
index 42c9d35f832d..9546a837a8ac 100644
--- a/drivers/infiniband/sw/rdmavt/vt.c
+++ b/drivers/infiniband/sw/rdmavt/vt.c
@@ -425,7 +425,10 @@ static const struct ib_device_ops rvt_dev_ops = {
 	.req_notify_cq = rvt_req_notify_cq,
 	.resize_cq = rvt_resize_cq,
 	.unmap_fmr = rvt_unmap_fmr,
+
+	INIT_RDMA_OBJ_SIZE(ib_ah, rvt_ah, ibah),
 	INIT_RDMA_OBJ_SIZE(ib_pd, rvt_pd, ibpd),
+	INIT_RDMA_OBJ_SIZE(ib_srq, rvt_srq, ibsrq),
 	INIT_RDMA_OBJ_SIZE(ib_ucontext, rvt_ucontext, ibucontext),
 };
 
diff --git a/drivers/infiniband/sw/rxe/rxe_cq.c b/drivers/infiniband/sw/rxe/rxe_cq.c
index a57276f2cb84..ad3090131126 100644
--- a/drivers/infiniband/sw/rxe/rxe_cq.c
+++ b/drivers/infiniband/sw/rxe/rxe_cq.c
@@ -82,7 +82,7 @@ static void rxe_send_complete(unsigned long data)
 }
 
 int rxe_cq_from_init(struct rxe_dev *rxe, struct rxe_cq *cq, int cqe,
-		     int comp_vector, struct ib_ucontext *context,
+		     int comp_vector, struct ib_udata *udata,
 		     struct rxe_create_cq_resp __user *uresp)
 {
 	int err;
@@ -94,7 +94,7 @@ int rxe_cq_from_init(struct rxe_dev *rxe, struct rxe_cq *cq, int cqe,
 		return -ENOMEM;
 	}
 
-	err = do_mmap_info(rxe, uresp ? &uresp->mi : NULL, context,
+	err = do_mmap_info(rxe, uresp ? &uresp->mi : NULL, udata,
 			   cq->queue->buf, cq->queue->buf_size, &cq->queue->ip);
 	if (err) {
 		vfree(cq->queue->buf);
@@ -115,13 +115,13 @@ int rxe_cq_from_init(struct rxe_dev *rxe, struct rxe_cq *cq, int cqe,
 }
 
 int rxe_cq_resize_queue(struct rxe_cq *cq, int cqe,
-			struct rxe_resize_cq_resp __user *uresp)
+			struct rxe_resize_cq_resp __user *uresp,
+			struct ib_udata *udata)
 {
 	int err;
 
 	err = rxe_queue_resize(cq->queue, (unsigned int *)&cqe,
-			       sizeof(struct rxe_cqe),
-			       cq->queue->ip ? cq->queue->ip->context : NULL,
+			       sizeof(struct rxe_cqe), udata,
 			       uresp ? &uresp->mi : NULL, NULL, &cq->cq_lock);
 	if (!err)
 		cq->ibcq.cqe = cqe;
diff --git a/drivers/infiniband/sw/rxe/rxe_hdr.h b/drivers/infiniband/sw/rxe/rxe_hdr.h
index 6cb18406f5b8..ce003666b800 100644
--- a/drivers/infiniband/sw/rxe/rxe_hdr.h
+++ b/drivers/infiniband/sw/rxe/rxe_hdr.h
@@ -643,7 +643,7 @@ struct rxe_atmeth {
 	__be32			rkey;
 	__be64			swap_add;
 	__be64			comp;
-} __attribute__((__packed__));
+} __packed;
 
 static inline u64 __atmeth_va(void *arg)
 {
diff --git a/drivers/infiniband/sw/rxe/rxe_loc.h b/drivers/infiniband/sw/rxe/rxe_loc.h
index 3d8cef836f0d..775c23becaec 100644
--- a/drivers/infiniband/sw/rxe/rxe_loc.h
+++ b/drivers/infiniband/sw/rxe/rxe_loc.h
@@ -53,11 +53,12 @@ int rxe_cq_chk_attr(struct rxe_dev *rxe, struct rxe_cq *cq,
 		    int cqe, int comp_vector);
 
 int rxe_cq_from_init(struct rxe_dev *rxe, struct rxe_cq *cq, int cqe,
-		     int comp_vector, struct ib_ucontext *context,
+		     int comp_vector, struct ib_udata *udata,
 		     struct rxe_create_cq_resp __user *uresp);
 
 int rxe_cq_resize_queue(struct rxe_cq *cq, int new_cqe,
-			struct rxe_resize_cq_resp __user *uresp);
+			struct rxe_resize_cq_resp __user *uresp,
+			struct ib_udata *udata);
 
 int rxe_cq_post(struct rxe_cq *cq, struct rxe_cqe *cqe, int solicited);
 
@@ -91,10 +92,8 @@ struct rxe_mmap_info {
 
 void rxe_mmap_release(struct kref *ref);
 
-struct rxe_mmap_info *rxe_create_mmap_info(struct rxe_dev *dev,
-					   u32 size,
-					   struct ib_ucontext *context,
-					   void *obj);
+struct rxe_mmap_info *rxe_create_mmap_info(struct rxe_dev *dev, u32 size,
+					   struct ib_udata *udata, void *obj);
 
 int rxe_mmap(struct ib_ucontext *context, struct vm_area_struct *vma);
 
@@ -224,13 +223,12 @@ int rxe_srq_chk_attr(struct rxe_dev *rxe, struct rxe_srq *srq,
 		     struct ib_srq_attr *attr, enum ib_srq_attr_mask mask);
 
 int rxe_srq_from_init(struct rxe_dev *rxe, struct rxe_srq *srq,
-		      struct ib_srq_init_attr *init,
-		      struct ib_ucontext *context,
+		      struct ib_srq_init_attr *init, struct ib_udata *udata,
 		      struct rxe_create_srq_resp __user *uresp);
 
 int rxe_srq_from_attr(struct rxe_dev *rxe, struct rxe_srq *srq,
 		      struct ib_srq_attr *attr, enum ib_srq_attr_mask mask,
-		      struct rxe_modify_srq_cmd *ucmd);
+		      struct rxe_modify_srq_cmd *ucmd, struct ib_udata *udata);
 
 void rxe_dealloc(struct ib_device *ib_dev);
 
diff --git a/drivers/infiniband/sw/rxe/rxe_mmap.c b/drivers/infiniband/sw/rxe/rxe_mmap.c
index d22431e3a908..48f48122ddcb 100644
--- a/drivers/infiniband/sw/rxe/rxe_mmap.c
+++ b/drivers/infiniband/sw/rxe/rxe_mmap.c
@@ -36,6 +36,7 @@
 #include <linux/mm.h>
 #include <linux/errno.h>
 #include <asm/pgtable.h>
+#include <rdma/uverbs_ioctl.h>
 
 #include "rxe.h"
 #include "rxe_loc.h"
@@ -140,13 +141,14 @@ done:
 /*
  * Allocate information for rxe_mmap
  */
-struct rxe_mmap_info *rxe_create_mmap_info(struct rxe_dev *rxe,
-					   u32 size,
-					   struct ib_ucontext *context,
-					   void *obj)
+struct rxe_mmap_info *rxe_create_mmap_info(struct rxe_dev *rxe, u32 size,
+					   struct ib_udata *udata, void *obj)
 {
 	struct rxe_mmap_info *ip;
 
+	if (!udata)
+		return ERR_PTR(-EINVAL);
+
 	ip = kmalloc(sizeof(*ip), GFP_KERNEL);
 	if (!ip)
 		return NULL;
@@ -165,7 +167,9 @@ struct rxe_mmap_info *rxe_create_mmap_info(struct rxe_dev *rxe,
 
 	INIT_LIST_HEAD(&ip->pending_mmaps);
 	ip->info.size = size;
-	ip->context = context;
+	ip->context =
+		container_of(udata, struct uverbs_attr_bundle, driver_udata)
+			->context;
 	ip->obj = obj;
 	kref_init(&ip->ref);
 
diff --git a/drivers/infiniband/sw/rxe/rxe_mr.c b/drivers/infiniband/sw/rxe/rxe_mr.c
index 42f0f25e396c..f501f72489d8 100644
--- a/drivers/infiniband/sw/rxe/rxe_mr.c
+++ b/drivers/infiniband/sw/rxe/rxe_mr.c
@@ -179,7 +179,7 @@ int rxe_mem_init_user(struct rxe_pd *pd, u64 start,
 	}
 
 	mem->umem = umem;
-	num_buf = umem->nmap;
+	num_buf = ib_umem_num_pages(umem);
 
 	rxe_mem_init(access, mem);
 
@@ -199,6 +199,12 @@ int rxe_mem_init_user(struct rxe_pd *pd, u64 start,
 		buf = map[0]->buf;
 
 		for_each_sg_page(umem->sg_head.sgl, &sg_iter, umem->nmap, 0) {
+			if (num_buf >= RXE_BUF_PER_MAP) {
+				map++;
+				buf = map[0]->buf;
+				num_buf = 0;
+			}
+
 			vaddr = page_address(sg_page_iter_page(&sg_iter));
 			if (!vaddr) {
 				pr_warn("null vaddr\n");
@@ -211,11 +217,6 @@ int rxe_mem_init_user(struct rxe_pd *pd, u64 start,
 			num_buf++;
 			buf++;
 
-			if (num_buf >= RXE_BUF_PER_MAP) {
-				map++;
-				buf = map[0]->buf;
-				num_buf = 0;
-			}
 		}
 	}
 
diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c
index 753cabcd441c..5a3474f9351b 100644
--- a/drivers/infiniband/sw/rxe/rxe_net.c
+++ b/drivers/infiniband/sw/rxe/rxe_net.c
@@ -338,13 +338,13 @@ static void prepare_ipv6_hdr(struct dst_entry *dst, struct sk_buff *skb,
 	ip6h->payload_len = htons(skb->len - sizeof(*ip6h));
 }
 
-static int prepare4(struct rxe_pkt_info *pkt, struct sk_buff *skb,
-		    struct rxe_av *av)
+static int prepare4(struct rxe_pkt_info *pkt, struct sk_buff *skb)
 {
 	struct rxe_qp *qp = pkt->qp;
 	struct dst_entry *dst;
 	bool xnet = false;
 	__be16 df = htons(IP_DF);
+	struct rxe_av *av = rxe_get_av(pkt);
 	struct in_addr *saddr = &av->sgid_addr._sockaddr_in.sin_addr;
 	struct in_addr *daddr = &av->dgid_addr._sockaddr_in.sin_addr;
 
@@ -364,11 +364,11 @@ static int prepare4(struct rxe_pkt_info *pkt, struct sk_buff *skb,
 	return 0;
 }
 
-static int prepare6(struct rxe_pkt_info *pkt, struct sk_buff *skb,
-		    struct rxe_av *av)
+static int prepare6(struct rxe_pkt_info *pkt, struct sk_buff *skb)
 {
 	struct rxe_qp *qp = pkt->qp;
 	struct dst_entry *dst;
+	struct rxe_av *av = rxe_get_av(pkt);
 	struct in6_addr *saddr = &av->sgid_addr._sockaddr_in6.sin6_addr;
 	struct in6_addr *daddr = &av->dgid_addr._sockaddr_in6.sin6_addr;
 
@@ -392,16 +392,15 @@ static int prepare6(struct rxe_pkt_info *pkt, struct sk_buff *skb,
 int rxe_prepare(struct rxe_pkt_info *pkt, struct sk_buff *skb, u32 *crc)
 {
 	int err = 0;
-	struct rxe_av *av = rxe_get_av(pkt);
 
-	if (av->network_type == RDMA_NETWORK_IPV4)
-		err = prepare4(pkt, skb, av);
-	else if (av->network_type == RDMA_NETWORK_IPV6)
-		err = prepare6(pkt, skb, av);
+	if (skb->protocol == htons(ETH_P_IP))
+		err = prepare4(pkt, skb);
+	else if (skb->protocol == htons(ETH_P_IPV6))
+		err = prepare6(pkt, skb);
 
 	*crc = rxe_icrc_hdr(pkt, skb);
 
-	if (ether_addr_equal(skb->dev->dev_addr, av->dmac))
+	if (ether_addr_equal(skb->dev->dev_addr, rxe_get_av(pkt)->dmac))
 		pkt->mask |= RXE_LOOPBACK_MASK;
 
 	return err;
@@ -422,23 +421,20 @@ static void rxe_skb_tx_dtor(struct sk_buff *skb)
 
 int rxe_send(struct rxe_pkt_info *pkt, struct sk_buff *skb)
 {
-	struct rxe_av *av;
 	int err;
 
-	av = rxe_get_av(pkt);
-
 	skb->destructor = rxe_skb_tx_dtor;
 	skb->sk = pkt->qp->sk->sk;
 
 	rxe_add_ref(pkt->qp);
 	atomic_inc(&pkt->qp->skb_out);
 
-	if (av->network_type == RDMA_NETWORK_IPV4) {
+	if (skb->protocol == htons(ETH_P_IP)) {
 		err = ip_local_out(dev_net(skb_dst(skb)->dev), skb->sk, skb);
-	} else if (av->network_type == RDMA_NETWORK_IPV6) {
+	} else if (skb->protocol == htons(ETH_P_IPV6)) {
 		err = ip6_local_out(dev_net(skb_dst(skb)->dev), skb->sk, skb);
 	} else {
-		pr_err("Unknown layer 3 protocol: %d\n", av->network_type);
+		pr_err("Unknown layer 3 protocol: %d\n", skb->protocol);
 		atomic_dec(&pkt->qp->skb_out);
 		rxe_drop_ref(pkt->qp);
 		kfree_skb(skb);
@@ -462,7 +458,7 @@ struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av,
 				int paylen, struct rxe_pkt_info *pkt)
 {
 	unsigned int hdr_len;
-	struct sk_buff *skb;
+	struct sk_buff *skb = NULL;
 	struct net_device *ndev;
 	const struct ib_gid_attr *attr;
 	const int port_num = 1;
@@ -470,7 +466,6 @@ struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av,
 	attr = rdma_get_gid_attr(&rxe->ib_dev, port_num, av->grh.sgid_index);
 	if (IS_ERR(attr))
 		return NULL;
-	ndev = attr->ndev;
 
 	if (av->network_type == RDMA_NETWORK_IPV4)
 		hdr_len = ETH_HLEN + sizeof(struct udphdr) +
@@ -479,15 +474,26 @@ struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av,
 		hdr_len = ETH_HLEN + sizeof(struct udphdr) +
 			sizeof(struct ipv6hdr);
 
+	rcu_read_lock();
+	ndev = rdma_read_gid_attr_ndev_rcu(attr);
+	if (IS_ERR(ndev)) {
+		rcu_read_unlock();
+		goto out;
+	}
 	skb = alloc_skb(paylen + hdr_len + LL_RESERVED_SPACE(ndev),
 			GFP_ATOMIC);
 
-	if (unlikely(!skb))
+	if (unlikely(!skb)) {
+		rcu_read_unlock();
 		goto out;
+	}
 
-	skb_reserve(skb, hdr_len + LL_RESERVED_SPACE(rxe->ndev));
+	skb_reserve(skb, hdr_len + LL_RESERVED_SPACE(ndev));
 
+	/* FIXME: hold reference to this netdev until life of this skb. */
 	skb->dev	= ndev;
+	rcu_read_unlock();
+
 	if (av->network_type == RDMA_NETWORK_IPV4)
 		skb->protocol = htons(ETH_P_IP);
 	else
diff --git a/drivers/infiniband/sw/rxe/rxe_pool.c b/drivers/infiniband/sw/rxe/rxe_pool.c
index 120fa9005954..56cf18af016a 100644
--- a/drivers/infiniband/sw/rxe/rxe_pool.c
+++ b/drivers/infiniband/sw/rxe/rxe_pool.c
@@ -52,12 +52,12 @@ struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = {
 	[RXE_TYPE_AH] = {
 		.name		= "rxe-ah",
 		.size		= sizeof(struct rxe_ah),
-		.flags		= RXE_POOL_ATOMIC,
+		.flags		= RXE_POOL_ATOMIC | RXE_POOL_NO_ALLOC,
 	},
 	[RXE_TYPE_SRQ] = {
 		.name		= "rxe-srq",
 		.size		= sizeof(struct rxe_srq),
-		.flags		= RXE_POOL_INDEX,
+		.flags		= RXE_POOL_INDEX | RXE_POOL_NO_ALLOC,
 		.min_index	= RXE_MIN_SRQ_INDEX,
 		.max_index	= RXE_MAX_SRQ_INDEX,
 	},
diff --git a/drivers/infiniband/sw/rxe/rxe_qp.c b/drivers/infiniband/sw/rxe/rxe_qp.c
index 09ede70dc1e8..e2c6d1cedf41 100644
--- a/drivers/infiniband/sw/rxe/rxe_qp.c
+++ b/drivers/infiniband/sw/rxe/rxe_qp.c
@@ -217,8 +217,7 @@ static void rxe_qp_init_misc(struct rxe_dev *rxe, struct rxe_qp *qp,
 }
 
 static int rxe_qp_init_req(struct rxe_dev *rxe, struct rxe_qp *qp,
-			   struct ib_qp_init_attr *init,
-			   struct ib_ucontext *context,
+			   struct ib_qp_init_attr *init, struct ib_udata *udata,
 			   struct rxe_create_qp_resp __user *uresp)
 {
 	int err;
@@ -254,7 +253,7 @@ static int rxe_qp_init_req(struct rxe_dev *rxe, struct rxe_qp *qp,
 	if (!qp->sq.queue)
 		return -ENOMEM;
 
-	err = do_mmap_info(rxe, uresp ? &uresp->sq_mi : NULL, context,
+	err = do_mmap_info(rxe, uresp ? &uresp->sq_mi : NULL, udata,
 			   qp->sq.queue->buf, qp->sq.queue->buf_size,
 			   &qp->sq.queue->ip);
 
@@ -287,7 +286,7 @@ static int rxe_qp_init_req(struct rxe_dev *rxe, struct rxe_qp *qp,
 
 static int rxe_qp_init_resp(struct rxe_dev *rxe, struct rxe_qp *qp,
 			    struct ib_qp_init_attr *init,
-			    struct ib_ucontext *context,
+			    struct ib_udata *udata,
 			    struct rxe_create_qp_resp __user *uresp)
 {
 	int err;
@@ -308,7 +307,7 @@ static int rxe_qp_init_resp(struct rxe_dev *rxe, struct rxe_qp *qp,
 		if (!qp->rq.queue)
 			return -ENOMEM;
 
-		err = do_mmap_info(rxe, uresp ? &uresp->rq_mi : NULL, context,
+		err = do_mmap_info(rxe, uresp ? &uresp->rq_mi : NULL, udata,
 				   qp->rq.queue->buf, qp->rq.queue->buf_size,
 				   &qp->rq.queue->ip);
 		if (err) {
@@ -344,8 +343,6 @@ int rxe_qp_from_init(struct rxe_dev *rxe, struct rxe_qp *qp, struct rxe_pd *pd,
 	struct rxe_cq *rcq = to_rcq(init->recv_cq);
 	struct rxe_cq *scq = to_rcq(init->send_cq);
 	struct rxe_srq *srq = init->srq ? to_rsrq(init->srq) : NULL;
-	struct rxe_ucontext *ucontext =
-		rdma_udata_to_drv_context(udata, struct rxe_ucontext, ibuc);
 
 	rxe_add_ref(pd);
 	rxe_add_ref(rcq);
@@ -360,11 +357,11 @@ int rxe_qp_from_init(struct rxe_dev *rxe, struct rxe_qp *qp, struct rxe_pd *pd,
 
 	rxe_qp_init_misc(rxe, qp, init);
 
-	err = rxe_qp_init_req(rxe, qp, init, &ucontext->ibuc, uresp);
+	err = rxe_qp_init_req(rxe, qp, init, udata, uresp);
 	if (err)
 		goto err1;
 
-	err = rxe_qp_init_resp(rxe, qp, init, &ucontext->ibuc, uresp);
+	err = rxe_qp_init_resp(rxe, qp, init, udata, uresp);
 	if (err)
 		goto err2;
 
diff --git a/drivers/infiniband/sw/rxe/rxe_queue.c b/drivers/infiniband/sw/rxe/rxe_queue.c
index f84ab4469261..ff92704de32f 100644
--- a/drivers/infiniband/sw/rxe/rxe_queue.c
+++ b/drivers/infiniband/sw/rxe/rxe_queue.c
@@ -36,18 +36,15 @@
 #include "rxe_loc.h"
 #include "rxe_queue.h"
 
-int do_mmap_info(struct rxe_dev *rxe,
-		 struct mminfo __user *outbuf,
-		 struct ib_ucontext *context,
-		 struct rxe_queue_buf *buf,
-		 size_t buf_size,
-		 struct rxe_mmap_info **ip_p)
+int do_mmap_info(struct rxe_dev *rxe, struct mminfo __user *outbuf,
+		 struct ib_udata *udata, struct rxe_queue_buf *buf,
+		 size_t buf_size, struct rxe_mmap_info **ip_p)
 {
 	int err;
 	struct rxe_mmap_info *ip = NULL;
 
 	if (outbuf) {
-		ip = rxe_create_mmap_info(rxe, buf_size, context, buf);
+		ip = rxe_create_mmap_info(rxe, buf_size, udata, buf);
 		if (!ip)
 			goto err1;
 
@@ -153,12 +150,9 @@ static int resize_finish(struct rxe_queue *q, struct rxe_queue *new_q,
 	return 0;
 }
 
-int rxe_queue_resize(struct rxe_queue *q,
-		     unsigned int *num_elem_p,
-		     unsigned int elem_size,
-		     struct ib_ucontext *context,
-		     struct mminfo __user *outbuf,
-		     spinlock_t *producer_lock,
+int rxe_queue_resize(struct rxe_queue *q, unsigned int *num_elem_p,
+		     unsigned int elem_size, struct ib_udata *udata,
+		     struct mminfo __user *outbuf, spinlock_t *producer_lock,
 		     spinlock_t *consumer_lock)
 {
 	struct rxe_queue *new_q;
@@ -170,7 +164,7 @@ int rxe_queue_resize(struct rxe_queue *q,
 	if (!new_q)
 		return -ENOMEM;
 
-	err = do_mmap_info(new_q->rxe, outbuf, context, new_q->buf,
+	err = do_mmap_info(new_q->rxe, outbuf, udata, new_q->buf,
 			   new_q->buf_size, &new_q->ip);
 	if (err) {
 		vfree(new_q->buf);
diff --git a/drivers/infiniband/sw/rxe/rxe_queue.h b/drivers/infiniband/sw/rxe/rxe_queue.h
index 79ba4b320054..acd0a925481c 100644
--- a/drivers/infiniband/sw/rxe/rxe_queue.h
+++ b/drivers/infiniband/sw/rxe/rxe_queue.h
@@ -76,12 +76,9 @@ struct rxe_queue {
 	unsigned int		index_mask;
 };
 
-int do_mmap_info(struct rxe_dev *rxe,
-		 struct mminfo __user *outbuf,
-		 struct ib_ucontext *context,
-		 struct rxe_queue_buf *buf,
-		 size_t buf_size,
-		 struct rxe_mmap_info **ip_p);
+int do_mmap_info(struct rxe_dev *rxe, struct mminfo __user *outbuf,
+		 struct ib_udata *udata, struct rxe_queue_buf *buf,
+		 size_t buf_size, struct rxe_mmap_info **ip_p);
 
 void rxe_queue_reset(struct rxe_queue *q);
 
@@ -89,10 +86,8 @@ struct rxe_queue *rxe_queue_init(struct rxe_dev *rxe,
 				 int *num_elem,
 				 unsigned int elem_size);
 
-int rxe_queue_resize(struct rxe_queue *q,
-		     unsigned int *num_elem_p,
-		     unsigned int elem_size,
-		     struct ib_ucontext *context,
+int rxe_queue_resize(struct rxe_queue *q, unsigned int *num_elem_p,
+		     unsigned int elem_size, struct ib_udata *udata,
 		     struct mminfo __user *outbuf,
 		     /* Protect producers while resizing queue */
 		     spinlock_t *producer_lock,
diff --git a/drivers/infiniband/sw/rxe/rxe_srq.c b/drivers/infiniband/sw/rxe/rxe_srq.c
index c41a5fee81f7..d8459431534e 100644
--- a/drivers/infiniband/sw/rxe/rxe_srq.c
+++ b/drivers/infiniband/sw/rxe/rxe_srq.c
@@ -99,8 +99,7 @@ err1:
 }
 
 int rxe_srq_from_init(struct rxe_dev *rxe, struct rxe_srq *srq,
-		      struct ib_srq_init_attr *init,
-		      struct ib_ucontext *context,
+		      struct ib_srq_init_attr *init, struct ib_udata *udata,
 		      struct rxe_create_srq_resp __user *uresp)
 {
 	int err;
@@ -128,7 +127,7 @@ int rxe_srq_from_init(struct rxe_dev *rxe, struct rxe_srq *srq,
 
 	srq->rq.queue = q;
 
-	err = do_mmap_info(rxe, uresp ? &uresp->mi : NULL, context, q->buf,
+	err = do_mmap_info(rxe, uresp ? &uresp->mi : NULL, udata, q->buf,
 			   q->buf_size, &q->ip);
 	if (err) {
 		vfree(q->buf);
@@ -149,7 +148,7 @@ int rxe_srq_from_init(struct rxe_dev *rxe, struct rxe_srq *srq,
 
 int rxe_srq_from_attr(struct rxe_dev *rxe, struct rxe_srq *srq,
 		      struct ib_srq_attr *attr, enum ib_srq_attr_mask mask,
-		      struct rxe_modify_srq_cmd *ucmd)
+		      struct rxe_modify_srq_cmd *ucmd, struct ib_udata *udata)
 {
 	int err;
 	struct rxe_queue *q = srq->rq.queue;
@@ -163,11 +162,8 @@ int rxe_srq_from_attr(struct rxe_dev *rxe, struct rxe_srq *srq,
 		mi = u64_to_user_ptr(ucmd->mmap_info_addr);
 
 		err = rxe_queue_resize(q, &attr->max_wr,
-				       rcv_wqe_size(srq->rq.max_sge),
-				       srq->rq.queue->ip ?
-						srq->rq.queue->ip->context :
-						NULL,
-				       mi, &srq->rq.producer_lock,
+				       rcv_wqe_size(srq->rq.max_sge), udata, mi,
+				       &srq->rq.producer_lock,
 				       &srq->rq.consumer_lock);
 		if (err)
 			goto err2;
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c
index 6ecf28570ff0..8c3e2a18cfe4 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.c
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.c
@@ -176,8 +176,7 @@ static int rxe_port_immutable(struct ib_device *dev, u8 port_num,
 	return 0;
 }
 
-static int rxe_alloc_pd(struct ib_pd *ibpd, struct ib_ucontext *context,
-			struct ib_udata *udata)
+static int rxe_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 {
 	struct rxe_dev *rxe = to_rdev(ibpd->device);
 	struct rxe_pd *pd = to_rpd(ibpd);
@@ -185,37 +184,31 @@ static int rxe_alloc_pd(struct ib_pd *ibpd, struct ib_ucontext *context,
 	return rxe_add_to_pool(&rxe->pd_pool, &pd->pelem);
 }
 
-static void rxe_dealloc_pd(struct ib_pd *ibpd)
+static void rxe_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 {
 	struct rxe_pd *pd = to_rpd(ibpd);
 
 	rxe_drop_ref(pd);
 }
 
-static struct ib_ah *rxe_create_ah(struct ib_pd *ibpd,
-				   struct rdma_ah_attr *attr,
-				   u32 flags,
-				   struct ib_udata *udata)
+static int rxe_create_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr,
+			 u32 flags, struct ib_udata *udata)
 
 {
 	int err;
-	struct rxe_dev *rxe = to_rdev(ibpd->device);
-	struct rxe_pd *pd = to_rpd(ibpd);
-	struct rxe_ah *ah;
+	struct rxe_dev *rxe = to_rdev(ibah->device);
+	struct rxe_ah *ah = to_rah(ibah);
 
 	err = rxe_av_chk_attr(rxe, attr);
 	if (err)
-		return ERR_PTR(err);
-
-	ah = rxe_alloc(&rxe->ah_pool);
-	if (!ah)
-		return ERR_PTR(-ENOMEM);
+		return err;
 
-	rxe_add_ref(pd);
-	ah->pd = pd;
+	err = rxe_add_to_pool(&rxe->ah_pool, &ah->pelem);
+	if (err)
+		return err;
 
 	rxe_init_av(attr, &ah->av);
-	return &ah->ibah;
+	return 0;
 }
 
 static int rxe_modify_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr)
@@ -242,13 +235,11 @@ static int rxe_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr)
 	return 0;
 }
 
-static int rxe_destroy_ah(struct ib_ah *ibah, u32 flags)
+static void rxe_destroy_ah(struct ib_ah *ibah, u32 flags)
 {
 	struct rxe_ah *ah = to_rah(ibah);
 
-	rxe_drop_ref(ah->pd);
 	rxe_drop_ref(ah);
-	return 0;
 }
 
 static int post_one_recv(struct rxe_rq *rq, const struct ib_recv_wr *ibwr)
@@ -298,21 +289,18 @@ err1:
 	return err;
 }
 
-static struct ib_srq *rxe_create_srq(struct ib_pd *ibpd,
-				     struct ib_srq_init_attr *init,
-				     struct ib_udata *udata)
+static int rxe_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init,
+			  struct ib_udata *udata)
 {
 	int err;
-	struct rxe_dev *rxe = to_rdev(ibpd->device);
-	struct rxe_pd *pd = to_rpd(ibpd);
-	struct rxe_ucontext *ucontext =
-		rdma_udata_to_drv_context(udata, struct rxe_ucontext, ibuc);
-	struct rxe_srq *srq;
+	struct rxe_dev *rxe = to_rdev(ibsrq->device);
+	struct rxe_pd *pd = to_rpd(ibsrq->pd);
+	struct rxe_srq *srq = to_rsrq(ibsrq);
 	struct rxe_create_srq_resp __user *uresp = NULL;
 
 	if (udata) {
 		if (udata->outlen < sizeof(*uresp))
-			return ERR_PTR(-EINVAL);
+			return -EINVAL;
 		uresp = udata->outbuf;
 	}
 
@@ -320,28 +308,24 @@ static struct ib_srq *rxe_create_srq(struct ib_pd *ibpd,
 	if (err)
 		goto err1;
 
-	srq = rxe_alloc(&rxe->srq_pool);
-	if (!srq) {
-		err = -ENOMEM;
+	err = rxe_add_to_pool(&rxe->srq_pool, &srq->pelem);
+	if (err)
 		goto err1;
-	}
 
-	rxe_add_index(srq);
 	rxe_add_ref(pd);
 	srq->pd = pd;
 
-	err = rxe_srq_from_init(rxe, srq, init, &ucontext->ibuc, uresp);
+	err = rxe_srq_from_init(rxe, srq, init, udata, uresp);
 	if (err)
 		goto err2;
 
-	return &srq->ibsrq;
+	return 0;
 
 err2:
 	rxe_drop_ref(pd);
-	rxe_drop_index(srq);
 	rxe_drop_ref(srq);
 err1:
-	return ERR_PTR(err);
+	return err;
 }
 
 static int rxe_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
@@ -366,7 +350,7 @@ static int rxe_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
 	if (err)
 		goto err1;
 
-	err = rxe_srq_from_attr(rxe, srq, attr, mask, &ucmd);
+	err = rxe_srq_from_attr(rxe, srq, attr, mask, &ucmd, udata);
 	if (err)
 		goto err1;
 
@@ -389,7 +373,7 @@ static int rxe_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr)
 	return 0;
 }
 
-static int rxe_destroy_srq(struct ib_srq *ibsrq)
+static void rxe_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
 {
 	struct rxe_srq *srq = to_rsrq(ibsrq);
 
@@ -397,10 +381,7 @@ static int rxe_destroy_srq(struct ib_srq *ibsrq)
 		rxe_queue_cleanup(srq->rq.queue);
 
 	rxe_drop_ref(srq->pd);
-	rxe_drop_index(srq);
 	rxe_drop_ref(srq);
-
-	return 0;
 }
 
 static int rxe_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
@@ -509,7 +490,7 @@ static int rxe_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 	return 0;
 }
 
-static int rxe_destroy_qp(struct ib_qp *ibqp)
+static int rxe_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
 {
 	struct rxe_qp *qp = to_rqp(ibqp);
 
@@ -799,7 +780,6 @@ err1:
 
 static struct ib_cq *rxe_create_cq(struct ib_device *dev,
 				   const struct ib_cq_init_attr *attr,
-				   struct ib_ucontext *context,
 				   struct ib_udata *udata)
 {
 	int err;
@@ -826,8 +806,8 @@ static struct ib_cq *rxe_create_cq(struct ib_device *dev,
 		goto err1;
 	}
 
-	err = rxe_cq_from_init(rxe, cq, attr->cqe, attr->comp_vector,
-			       context, uresp);
+	err = rxe_cq_from_init(rxe, cq, attr->cqe, attr->comp_vector, udata,
+			       uresp);
 	if (err)
 		goto err2;
 
@@ -839,7 +819,7 @@ err1:
 	return ERR_PTR(err);
 }
 
-static int rxe_destroy_cq(struct ib_cq *ibcq)
+static int rxe_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
 {
 	struct rxe_cq *cq = to_rcq(ibcq);
 
@@ -866,7 +846,7 @@ static int rxe_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata)
 	if (err)
 		goto err1;
 
-	err = rxe_cq_resize_queue(cq, cqe, uresp);
+	err = rxe_cq_resize_queue(cq, cqe, uresp, udata);
 	if (err)
 		goto err1;
 
@@ -990,7 +970,7 @@ err2:
 	return ERR_PTR(err);
 }
 
-static int rxe_dereg_mr(struct ib_mr *ibmr)
+static int rxe_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
 {
 	struct rxe_mem *mr = to_rmr(ibmr);
 
@@ -1001,9 +981,8 @@ static int rxe_dereg_mr(struct ib_mr *ibmr)
 	return 0;
 }
 
-static struct ib_mr *rxe_alloc_mr(struct ib_pd *ibpd,
-				  enum ib_mr_type mr_type,
-				  u32 max_num_sg)
+static struct ib_mr *rxe_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
+				  u32 max_num_sg, struct ib_udata *udata)
 {
 	struct rxe_dev *rxe = to_rdev(ibpd->device);
 	struct rxe_pd *pd = to_rpd(ibpd);
@@ -1176,7 +1155,10 @@ static const struct ib_device_ops rxe_dev_ops = {
 	.reg_user_mr = rxe_reg_user_mr,
 	.req_notify_cq = rxe_req_notify_cq,
 	.resize_cq = rxe_resize_cq,
+
+	INIT_RDMA_OBJ_SIZE(ib_ah, rxe_ah, ibah),
 	INIT_RDMA_OBJ_SIZE(ib_pd, rxe_pd, ibpd),
+	INIT_RDMA_OBJ_SIZE(ib_srq, rxe_srq, ibsrq),
 	INIT_RDMA_OBJ_SIZE(ib_ucontext, rxe_ucontext, ibuc),
 };
 
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h
index 157e51aeb1e1..e8be7f44e3be 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.h
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
@@ -71,8 +71,8 @@ struct rxe_pd {
 };
 
 struct rxe_ah {
-	struct rxe_pool_entry	pelem;
 	struct ib_ah		ibah;
+	struct rxe_pool_entry	pelem;
 	struct rxe_pd		*pd;
 	struct rxe_av		av;
 };
@@ -120,8 +120,8 @@ struct rxe_rq {
 };
 
 struct rxe_srq {
-	struct rxe_pool_entry	pelem;
 	struct ib_srq		ibsrq;
+	struct rxe_pool_entry	pelem;
 	struct rxe_pd		*pd;
 	struct rxe_rq		rq;
 	u32			srq_num;
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c
index 48eda16db1a7..9b5e11d3fb85 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_main.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c
@@ -2402,7 +2402,18 @@ static ssize_t dev_id_show(struct device *dev,
 {
 	struct net_device *ndev = to_net_dev(dev);
 
-	if (ndev->dev_id == ndev->dev_port)
+	/*
+	 * ndev->dev_port will be equal to 0 in old kernel prior to commit
+	 * 9b8b2a323008 ("IB/ipoib: Use dev_port to expose network interface
+	 * port numbers") Zero was chosen as special case for user space
+	 * applications to fallback and query dev_id to check if it has
+	 * different value or not.
+	 *
+	 * Don't print warning in such scenario.
+	 *
+	 * https://github.com/systemd/systemd/blob/master/src/udev/udev-builtin-net_id.c#L358
+	 */
+	if (ndev->dev_port && ndev->dev_id == ndev->dev_port)
 		netdev_info_once(ndev,
 			"\"%s\" wants to know my dev_id. Should it look at dev_port instead? See Documentation/ABI/testing/sysfs-class-net for more info.\n",
 			current->comm);
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_verbs.c b/drivers/infiniband/ulp/ipoib/ipoib_verbs.c
index 1e88213459f2..ba09068f6200 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_verbs.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_verbs.c
@@ -279,8 +279,7 @@ void ipoib_event(struct ib_event_handler *handler,
 	ipoib_dbg(priv, "Event %d on device %s port %d\n", record->event,
 		  dev_name(&record->device->dev), record->element.port_num);
 
-	if (record->event == IB_EVENT_SM_CHANGE ||
-	    record->event == IB_EVENT_CLIENT_REREGISTER) {
+	if (record->event == IB_EVENT_CLIENT_REREGISTER) {
 		queue_work(ipoib_workqueue, &priv->flush_light);
 	} else if (record->event == IB_EVENT_PORT_ERR ||
 		   record->event == IB_EVENT_PORT_ACTIVE ||
diff --git a/drivers/infiniband/ulp/iser/Kconfig b/drivers/infiniband/ulp/iser/Kconfig
index d00af71a2cfc..299268f261ee 100644
--- a/drivers/infiniband/ulp/iser/Kconfig
+++ b/drivers/infiniband/ulp/iser/Kconfig
@@ -4,8 +4,8 @@ config INFINIBAND_ISER
 	select SCSI_ISCSI_ATTRS
 	---help---
 	  Support for the iSCSI Extensions for RDMA (iSER) Protocol
-          over InfiniBand. This allows you to access storage devices
-          that speak iSCSI over iSER over InfiniBand.
+	  over InfiniBand. This allows you to access storage devices
+	  that speak iSCSI over iSER over InfiniBand.
 
 	  The iSER protocol is defined by IETF.
 	  See <http://www.ietf.org/rfc/rfc5046.txt>
diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.c b/drivers/infiniband/ulp/iser/iscsi_iser.c
index 8c707accd148..9c185a8dabd3 100644
--- a/drivers/infiniband/ulp/iser/iscsi_iser.c
+++ b/drivers/infiniband/ulp/iser/iscsi_iser.c
@@ -763,7 +763,6 @@ static int iscsi_iser_get_ep_param(struct iscsi_endpoint *ep,
 				   enum iscsi_param param, char *buf)
 {
 	struct iser_conn *iser_conn = ep->dd_data;
-	int len;
 
 	switch (param) {
 	case ISCSI_PARAM_CONN_PORT:
@@ -774,12 +773,10 @@ static int iscsi_iser_get_ep_param(struct iscsi_endpoint *ep,
 		return iscsi_conn_get_addr_param((struct sockaddr_storage *)
 				&iser_conn->ib_conn.cma_id->route.addr.dst_addr,
 				param, buf);
-		break;
 	default:
-		return -ENOSYS;
+		break;
 	}
-
-	return len;
+	return -ENOSYS;
 }
 
 /**
diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.h b/drivers/infiniband/ulp/iser/iscsi_iser.h
index a7aeaa0c6fbc..36d525110fd2 100644
--- a/drivers/infiniband/ulp/iser/iscsi_iser.h
+++ b/drivers/infiniband/ulp/iser/iscsi_iser.h
@@ -311,7 +311,7 @@ struct iser_login_desc {
 	u64                          rsp_dma;
 	struct ib_sge                sge;
 	struct ib_cqe		     cqe;
-} __attribute__((packed));
+} __packed;
 
 struct iser_conn;
 struct ib_conn;
diff --git a/drivers/infiniband/ulp/opa_vnic/opa_vnic_vema.c b/drivers/infiniband/ulp/opa_vnic/opa_vnic_vema.c
index 560e4f2d466e..be5befd92d16 100644
--- a/drivers/infiniband/ulp/opa_vnic/opa_vnic_vema.c
+++ b/drivers/infiniband/ulp/opa_vnic/opa_vnic_vema.c
@@ -51,6 +51,7 @@
  */
 
 #include <linux/module.h>
+#include <linux/xarray.h>
 #include <rdma/ib_addr.h>
 #include <rdma/ib_verbs.h>
 #include <rdma/opa_smi.h>
@@ -97,7 +98,7 @@ const char opa_vnic_driver_version[] = DRV_VERSION;
  * @class_port_info: Class port info information.
  * @tid: Transaction id
  * @port_num: OPA port number
- * @vport_idr: vnic ports idr
+ * @vports: vnic ports
  * @event_handler: ib event handler
  * @lock: adapter interface lock
  */
@@ -107,7 +108,7 @@ struct opa_vnic_vema_port {
 	struct opa_class_port_info      class_port_info;
 	u64                             tid;
 	u8                              port_num;
-	struct idr                      vport_idr;
+	struct xarray                   vports;
 	struct ib_event_handler         event_handler;
 
 	/* Lock to query/update network adapter */
@@ -148,7 +149,7 @@ vema_get_vport_adapter(struct opa_vnic_vema_mad *recvd_mad,
 {
 	u8 vport_num = vema_get_vport_num(recvd_mad);
 
-	return idr_find(&port->vport_idr, vport_num);
+	return xa_load(&port->vports, vport_num);
 }
 
 /**
@@ -207,8 +208,7 @@ static struct opa_vnic_adapter *vema_add_vport(struct opa_vnic_vema_port *port,
 		int rc;
 
 		adapter->cport = cport;
-		rc = idr_alloc(&port->vport_idr, adapter, vport_num,
-			       vport_num + 1, GFP_NOWAIT);
+		rc = xa_insert(&port->vports, vport_num, adapter, GFP_KERNEL);
 		if (rc < 0) {
 			opa_vnic_rem_netdev(adapter);
 			adapter = ERR_PTR(rc);
@@ -853,36 +853,14 @@ err_exit:
 	v_err("Aborting trap\n");
 }
 
-static int vema_rem_vport(int id, void *p, void *data)
-{
-	struct opa_vnic_adapter *adapter = p;
-
-	opa_vnic_rem_netdev(adapter);
-	return 0;
-}
-
-static int vema_enable_vport(int id, void *p, void *data)
-{
-	struct opa_vnic_adapter *adapter = p;
-
-	netif_carrier_on(adapter->netdev);
-	return 0;
-}
-
-static int vema_disable_vport(int id, void *p, void *data)
-{
-	struct opa_vnic_adapter *adapter = p;
-
-	netif_carrier_off(adapter->netdev);
-	return 0;
-}
-
 static void opa_vnic_event(struct ib_event_handler *handler,
 			   struct ib_event *record)
 {
 	struct opa_vnic_vema_port *port =
 		container_of(handler, struct opa_vnic_vema_port, event_handler);
 	struct opa_vnic_ctrl_port *cport = port->cport;
+	struct opa_vnic_adapter *adapter;
+	unsigned long index;
 
 	if (record->element.port_num != port->port_num)
 		return;
@@ -891,10 +869,16 @@ static void opa_vnic_event(struct ib_event_handler *handler,
 	      record->event, dev_name(&record->device->dev),
 	      record->element.port_num);
 
-	if (record->event == IB_EVENT_PORT_ERR)
-		idr_for_each(&port->vport_idr, vema_disable_vport, NULL);
-	if (record->event == IB_EVENT_PORT_ACTIVE)
-		idr_for_each(&port->vport_idr, vema_enable_vport, NULL);
+	if (record->event != IB_EVENT_PORT_ERR &&
+	    record->event != IB_EVENT_PORT_ACTIVE)
+		return;
+
+	xa_for_each(&port->vports, index, adapter) {
+		if (record->event == IB_EVENT_PORT_ACTIVE)
+			netif_carrier_on(adapter->netdev);
+		else
+			netif_carrier_off(adapter->netdev);
+	}
 }
 
 /**
@@ -905,6 +889,8 @@ static void opa_vnic_event(struct ib_event_handler *handler,
  */
 static void vema_unregister(struct opa_vnic_ctrl_port *cport)
 {
+	struct opa_vnic_adapter *adapter;
+	unsigned long index;
 	int i;
 
 	for (i = 1; i <= cport->num_ports; i++) {
@@ -915,13 +901,14 @@ static void vema_unregister(struct opa_vnic_ctrl_port *cport)
 
 		/* Lock ensures no MAD is being processed */
 		mutex_lock(&port->lock);
-		idr_for_each(&port->vport_idr, vema_rem_vport, NULL);
+		xa_for_each(&port->vports, index, adapter)
+			opa_vnic_rem_netdev(adapter);
 		mutex_unlock(&port->lock);
 
 		ib_unregister_mad_agent(port->mad_agent);
 		port->mad_agent = NULL;
 		mutex_destroy(&port->lock);
-		idr_destroy(&port->vport_idr);
+		xa_destroy(&port->vports);
 		ib_unregister_event_handler(&port->event_handler);
 	}
 }
@@ -958,7 +945,7 @@ static int vema_register(struct opa_vnic_ctrl_port *cport)
 				      cport->ibdev, opa_vnic_event);
 		ib_register_event_handler(&port->event_handler);
 
-		idr_init(&port->vport_idr);
+		xa_init(&port->vports);
 		mutex_init(&port->lock);
 		port->mad_agent = ib_register_mad_agent(cport->ibdev, i,
 							IB_QPT_GSI, &reg_req,
@@ -969,7 +956,6 @@ static int vema_register(struct opa_vnic_ctrl_port *cport)
 			ret = PTR_ERR(port->mad_agent);
 			port->mad_agent = NULL;
 			mutex_destroy(&port->lock);
-			idr_destroy(&port->vport_idr);
 			vema_unregister(cport);
 			return ret;
 		}
diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c
index d3700ec15cbd..4e21efbc4459 100644
--- a/drivers/iommu/io-pgtable-arm.c
+++ b/drivers/iommu/io-pgtable-arm.c
@@ -172,6 +172,10 @@
 #define ARM_LPAE_MAIR_ATTR_IDX_CACHE	1
 #define ARM_LPAE_MAIR_ATTR_IDX_DEV	2
 
+#define ARM_MALI_LPAE_TTBR_ADRMODE_TABLE (3u << 0)
+#define ARM_MALI_LPAE_TTBR_READ_INNER	BIT(2)
+#define ARM_MALI_LPAE_TTBR_SHARE_OUTER	BIT(4)
+
 /* IOPTE accessors */
 #define iopte_deref(pte,d) __va(iopte_to_paddr(pte, d))
 
@@ -180,11 +184,6 @@
 
 #define iopte_prot(pte)	((pte) & ARM_LPAE_PTE_ATTR_MASK)
 
-#define iopte_leaf(pte,l)					\
-	(l == (ARM_LPAE_MAX_LEVELS - 1) ?			\
-		(iopte_type(pte,l) == ARM_LPAE_PTE_TYPE_PAGE) :	\
-		(iopte_type(pte,l) == ARM_LPAE_PTE_TYPE_BLOCK))
-
 struct arm_lpae_io_pgtable {
 	struct io_pgtable	iop;
 
@@ -198,6 +197,15 @@ struct arm_lpae_io_pgtable {
 
 typedef u64 arm_lpae_iopte;
 
+static inline bool iopte_leaf(arm_lpae_iopte pte, int lvl,
+			      enum io_pgtable_fmt fmt)
+{
+	if (lvl == (ARM_LPAE_MAX_LEVELS - 1) && fmt != ARM_MALI_LPAE)
+		return iopte_type(pte, lvl) == ARM_LPAE_PTE_TYPE_PAGE;
+
+	return iopte_type(pte, lvl) == ARM_LPAE_PTE_TYPE_BLOCK;
+}
+
 static arm_lpae_iopte paddr_to_iopte(phys_addr_t paddr,
 				     struct arm_lpae_io_pgtable *data)
 {
@@ -303,12 +311,14 @@ static void __arm_lpae_init_pte(struct arm_lpae_io_pgtable *data,
 	if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS)
 		pte |= ARM_LPAE_PTE_NS;
 
-	if (lvl == ARM_LPAE_MAX_LEVELS - 1)
+	if (data->iop.fmt != ARM_MALI_LPAE && lvl == ARM_LPAE_MAX_LEVELS - 1)
 		pte |= ARM_LPAE_PTE_TYPE_PAGE;
 	else
 		pte |= ARM_LPAE_PTE_TYPE_BLOCK;
 
-	pte |= ARM_LPAE_PTE_AF | ARM_LPAE_PTE_SH_IS;
+	if (data->iop.fmt != ARM_MALI_LPAE)
+		pte |= ARM_LPAE_PTE_AF;
+	pte |= ARM_LPAE_PTE_SH_IS;
 	pte |= paddr_to_iopte(paddr, data);
 
 	__arm_lpae_set_pte(ptep, pte, &data->iop.cfg);
@@ -321,7 +331,7 @@ static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data,
 {
 	arm_lpae_iopte pte = *ptep;
 
-	if (iopte_leaf(pte, lvl)) {
+	if (iopte_leaf(pte, lvl, data->iop.fmt)) {
 		/* We require an unmap first */
 		WARN_ON(!selftest_running);
 		return -EEXIST;
@@ -409,7 +419,7 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova,
 		__arm_lpae_sync_pte(ptep, cfg);
 	}
 
-	if (pte && !iopte_leaf(pte, lvl)) {
+	if (pte && !iopte_leaf(pte, lvl, data->iop.fmt)) {
 		cptep = iopte_deref(pte, data);
 	} else if (pte) {
 		/* We require an unmap first */
@@ -429,31 +439,37 @@ static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data,
 	if (data->iop.fmt == ARM_64_LPAE_S1 ||
 	    data->iop.fmt == ARM_32_LPAE_S1) {
 		pte = ARM_LPAE_PTE_nG;
-
 		if (!(prot & IOMMU_WRITE) && (prot & IOMMU_READ))
 			pte |= ARM_LPAE_PTE_AP_RDONLY;
-
 		if (!(prot & IOMMU_PRIV))
 			pte |= ARM_LPAE_PTE_AP_UNPRIV;
-
-		if (prot & IOMMU_MMIO)
-			pte |= (ARM_LPAE_MAIR_ATTR_IDX_DEV
-				<< ARM_LPAE_PTE_ATTRINDX_SHIFT);
-		else if (prot & IOMMU_CACHE)
-			pte |= (ARM_LPAE_MAIR_ATTR_IDX_CACHE
-				<< ARM_LPAE_PTE_ATTRINDX_SHIFT);
 	} else {
 		pte = ARM_LPAE_PTE_HAP_FAULT;
 		if (prot & IOMMU_READ)
 			pte |= ARM_LPAE_PTE_HAP_READ;
 		if (prot & IOMMU_WRITE)
 			pte |= ARM_LPAE_PTE_HAP_WRITE;
+	}
+
+	/*
+	 * Note that this logic is structured to accommodate Mali LPAE
+	 * having stage-1-like attributes but stage-2-like permissions.
+	 */
+	if (data->iop.fmt == ARM_64_LPAE_S2 ||
+	    data->iop.fmt == ARM_32_LPAE_S2) {
 		if (prot & IOMMU_MMIO)
 			pte |= ARM_LPAE_PTE_MEMATTR_DEV;
 		else if (prot & IOMMU_CACHE)
 			pte |= ARM_LPAE_PTE_MEMATTR_OIWB;
 		else
 			pte |= ARM_LPAE_PTE_MEMATTR_NC;
+	} else {
+		if (prot & IOMMU_MMIO)
+			pte |= (ARM_LPAE_MAIR_ATTR_IDX_DEV
+				<< ARM_LPAE_PTE_ATTRINDX_SHIFT);
+		else if (prot & IOMMU_CACHE)
+			pte |= (ARM_LPAE_MAIR_ATTR_IDX_CACHE
+				<< ARM_LPAE_PTE_ATTRINDX_SHIFT);
 	}
 
 	if (prot & IOMMU_NOEXEC)
@@ -511,7 +527,7 @@ static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl,
 	while (ptep != end) {
 		arm_lpae_iopte pte = *ptep++;
 
-		if (!pte || iopte_leaf(pte, lvl))
+		if (!pte || iopte_leaf(pte, lvl, data->iop.fmt))
 			continue;
 
 		__arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data));
@@ -602,7 +618,7 @@ static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
 	if (size == ARM_LPAE_BLOCK_SIZE(lvl, data)) {
 		__arm_lpae_set_pte(ptep, 0, &iop->cfg);
 
-		if (!iopte_leaf(pte, lvl)) {
+		if (!iopte_leaf(pte, lvl, iop->fmt)) {
 			/* Also flush any partial walks */
 			io_pgtable_tlb_add_flush(iop, iova, size,
 						ARM_LPAE_GRANULE(data), false);
@@ -621,7 +637,7 @@ static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
 		}
 
 		return size;
-	} else if (iopte_leaf(pte, lvl)) {
+	} else if (iopte_leaf(pte, lvl, iop->fmt)) {
 		/*
 		 * Insert a table at the next level to map the old region,
 		 * minus the part we want to unmap
@@ -669,7 +685,7 @@ static phys_addr_t arm_lpae_iova_to_phys(struct io_pgtable_ops *ops,
 			return 0;
 
 		/* Leaf entry? */
-		if (iopte_leaf(pte,lvl))
+		if (iopte_leaf(pte, lvl, data->iop.fmt))
 			goto found_translation;
 
 		/* Take it to the next level */
@@ -995,6 +1011,32 @@ arm_32_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie)
 	return iop;
 }
 
+static struct io_pgtable *
+arm_mali_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie)
+{
+	struct io_pgtable *iop;
+
+	if (cfg->ias != 48 || cfg->oas > 40)
+		return NULL;
+
+	cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);
+	iop = arm_64_lpae_alloc_pgtable_s1(cfg, cookie);
+	if (iop) {
+		u64 mair, ttbr;
+
+		/* Copy values as union fields overlap */
+		mair = cfg->arm_lpae_s1_cfg.mair[0];
+		ttbr = cfg->arm_lpae_s1_cfg.ttbr[0];
+
+		cfg->arm_mali_lpae_cfg.memattr = mair;
+		cfg->arm_mali_lpae_cfg.transtab = ttbr |
+			ARM_MALI_LPAE_TTBR_READ_INNER |
+			ARM_MALI_LPAE_TTBR_ADRMODE_TABLE;
+	}
+
+	return iop;
+}
+
 struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns = {
 	.alloc	= arm_64_lpae_alloc_pgtable_s1,
 	.free	= arm_lpae_free_pgtable,
@@ -1015,6 +1057,11 @@ struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns = {
 	.free	= arm_lpae_free_pgtable,
 };
 
+struct io_pgtable_init_fns io_pgtable_arm_mali_lpae_init_fns = {
+	.alloc	= arm_mali_lpae_alloc_pgtable,
+	.free	= arm_lpae_free_pgtable,
+};
+
 #ifdef CONFIG_IOMMU_IO_PGTABLE_LPAE_SELFTEST
 
 static struct io_pgtable_cfg *cfg_cookie;
diff --git a/drivers/iommu/io-pgtable.c b/drivers/iommu/io-pgtable.c
index 93f2880be6c6..5227cfdbb65b 100644
--- a/drivers/iommu/io-pgtable.c
+++ b/drivers/iommu/io-pgtable.c
@@ -30,6 +30,7 @@ io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] = {
 	[ARM_32_LPAE_S2] = &io_pgtable_arm_32_lpae_s2_init_fns,
 	[ARM_64_LPAE_S1] = &io_pgtable_arm_64_lpae_s1_init_fns,
 	[ARM_64_LPAE_S2] = &io_pgtable_arm_64_lpae_s2_init_fns,
+	[ARM_MALI_LPAE] = &io_pgtable_arm_mali_lpae_init_fns,
 #endif
 #ifdef CONFIG_IOMMU_IO_PGTABLE_ARMV7S
 	[ARM_V7S] = &io_pgtable_arm_v7s_init_fns,
diff --git a/drivers/media/Kconfig b/drivers/media/Kconfig
index 102eb35fcf3f..8efaf99243e0 100644
--- a/drivers/media/Kconfig
+++ b/drivers/media/Kconfig
@@ -31,14 +31,14 @@ comment "Multimedia core support"
 #
 config MEDIA_CAMERA_SUPPORT
 	bool "Cameras/video grabbers support"
-	---help---
+	help
 	  Enable support for webcams and video grabbers.
 
 	  Say Y when you have a webcam or a video capture grabber board.
 
 config MEDIA_ANALOG_TV_SUPPORT
 	bool "Analog TV support"
-	---help---
+	help
 	  Enable analog TV support.
 
 	  Say Y when you have a TV board with analog support or with a
@@ -50,7 +50,7 @@ config MEDIA_ANALOG_TV_SUPPORT
 
 config MEDIA_DIGITAL_TV_SUPPORT
 	bool "Digital TV support"
-	---help---
+	help
 	  Enable digital TV support.
 
 	  Say Y when you have a board with digital support or a board with
@@ -58,7 +58,7 @@ config MEDIA_DIGITAL_TV_SUPPORT
 
 config MEDIA_RADIO_SUPPORT
 	bool "AM/FM radio receivers/transmitters support"
-	---help---
+	help
 	  Enable AM/FM radio support.
 
 	  Additional info and docs are available on the web at
@@ -72,14 +72,14 @@ config MEDIA_RADIO_SUPPORT
 
 config MEDIA_SDR_SUPPORT
 	bool "Software defined radio support"
-	---help---
+	help
 	  Enable software defined radio support.
 
 	  Say Y when you have a software defined radio device.
 
 config MEDIA_CEC_SUPPORT
 	bool "HDMI CEC support"
-	---help---
+	help
 	  Enable support for HDMI CEC (Consumer Electronics Control),
 	  which is an optional HDMI feature.
 
@@ -96,7 +96,7 @@ source "drivers/media/cec/Kconfig"
 config MEDIA_CONTROLLER
 	bool "Media Controller API"
 	depends on MEDIA_CAMERA_SUPPORT || MEDIA_ANALOG_TV_SUPPORT || MEDIA_DIGITAL_TV_SUPPORT
-	---help---
+	help
 	  Enable the media controller API used to query media devices internal
 	  topology and configure it dynamically.
 
@@ -105,7 +105,7 @@ config MEDIA_CONTROLLER
 config MEDIA_CONTROLLER_DVB
 	bool "Enable Media controller for DVB (EXPERIMENTAL)"
 	depends on MEDIA_CONTROLLER && DVB_CORE
-	---help---
+	help
 	  Enable the media controller API support for DVB.
 
 	  This is currently experimental.
@@ -114,7 +114,7 @@ config MEDIA_CONTROLLER_REQUEST_API
 	bool "Enable Media controller Request API (EXPERIMENTAL)"
 	depends on MEDIA_CONTROLLER && STAGING_MEDIA
 	default n
-	---help---
+	help
 	  DO NOT ENABLE THIS OPTION UNLESS YOU KNOW WHAT YOU'RE DOING.
 
 	  This option enables the Request API for the Media controller and V4L2
@@ -137,7 +137,7 @@ config VIDEO_DEV
 config VIDEO_V4L2_SUBDEV_API
 	bool "V4L2 sub-device userspace API"
 	depends on VIDEO_DEV && MEDIA_CONTROLLER
-	---help---
+	help
 	  Enables the V4L2 sub-device pad-level userspace API used to configure
 	  video format, size and frame rate between hardware blocks.
 
diff --git a/drivers/media/Makefile b/drivers/media/Makefile
index 985d35ec6b29..4a330d0e5e40 100644
--- a/drivers/media/Makefile
+++ b/drivers/media/Makefile
@@ -6,6 +6,12 @@
 media-objs	:= media-device.o media-devnode.o media-entity.o \
 		   media-request.o
 
+ifeq ($(CONFIG_MEDIA_CONTROLLER),y)
+	ifeq ($(CONFIG_USB),y)
+		media-objs += media-dev-allocator.o
+	endif
+endif
+
 #
 # I2C drivers should come before other drivers, otherwise they'll fail
 # when compiled as builtin drivers
diff --git a/drivers/media/cec/Kconfig b/drivers/media/cec/Kconfig
index 9c2b108c613a..b5aadacf335a 100644
--- a/drivers/media/cec/Kconfig
+++ b/drivers/media/cec/Kconfig
@@ -2,11 +2,11 @@ config MEDIA_CEC_RC
 	bool "HDMI CEC RC integration"
 	depends on CEC_CORE && RC_CORE
 	depends on CEC_CORE=m || RC_CORE=y
-	---help---
+	help
 	  Pass on CEC remote control messages to the RC framework.
 
 config CEC_PIN_ERROR_INJ
 	bool "Enable CEC error injection support"
 	depends on CEC_PIN && DEBUG_FS
-	---help---
+	help
 	  This option enables CEC error injection using debugfs.
diff --git a/drivers/media/cec/cec-core.c b/drivers/media/cec/cec-core.c
index cc875dabd765..f5d1578e256a 100644
--- a/drivers/media/cec/cec-core.c
+++ b/drivers/media/cec/cec-core.c
@@ -126,6 +126,7 @@ static int __must_check cec_devnode_register(struct cec_devnode *devnode,
 	/* Part 2: Initialize and register the character device */
 	cdev_init(&devnode->cdev, &cec_devnode_fops);
 	devnode->cdev.owner = owner;
+	kobject_set_name(&devnode->cdev.kobj, "cec%d", devnode->minor);
 
 	ret = cdev_device_add(&devnode->cdev, &devnode->dev);
 	if (ret) {
diff --git a/drivers/media/cec/cec-notifier.c b/drivers/media/cec/cec-notifier.c
index dd2078b27a41..9598c7778871 100644
--- a/drivers/media/cec/cec-notifier.c
+++ b/drivers/media/cec/cec-notifier.c
@@ -11,6 +11,7 @@
 #include <linux/slab.h>
 #include <linux/list.h>
 #include <linux/kref.h>
+#include <linux/of_platform.h>
 
 #include <media/cec.h>
 #include <media/cec-notifier.h>
@@ -127,3 +128,32 @@ void cec_notifier_unregister(struct cec_notifier *n)
 	cec_notifier_put(n);
 }
 EXPORT_SYMBOL_GPL(cec_notifier_unregister);
+
+struct device *cec_notifier_parse_hdmi_phandle(struct device *dev)
+{
+	struct platform_device *hdmi_pdev;
+	struct device *hdmi_dev = NULL;
+	struct device_node *np;
+
+	np = of_parse_phandle(dev->of_node, "hdmi-phandle", 0);
+
+	if (!np) {
+		dev_err(dev, "Failed to find HDMI node in device tree\n");
+		return ERR_PTR(-ENODEV);
+	}
+	hdmi_pdev = of_find_device_by_node(np);
+	of_node_put(np);
+	if (hdmi_pdev) {
+		hdmi_dev = &hdmi_pdev->dev;
+		/*
+		 * Note that the device struct is only used as a key into the
+		 * cec_notifiers list, it is never actually accessed.
+		 * So we decrement the reference here so we don't leak
+		 * memory.
+		 */
+		put_device(hdmi_dev);
+		return hdmi_dev;
+	}
+	return ERR_PTR(-EPROBE_DEFER);
+}
+EXPORT_SYMBOL_GPL(cec_notifier_parse_hdmi_phandle);
diff --git a/drivers/media/common/cx2341x.c b/drivers/media/common/cx2341x.c
index 1dcc39b87bb7..121cda73ff88 100644
--- a/drivers/media/common/cx2341x.c
+++ b/drivers/media/common/cx2341x.c
@@ -1028,7 +1028,7 @@ static int cx2341x_api(void *priv, cx2341x_mbox_func func,
 	return func(priv, cmd, args, 0, data);
 }
 
-#define NEQ(field) (old->field != new->field)
+#define CMP_FIELD(__old, __new, __field) (__old->__field != __new->__field)
 
 int cx2341x_update(void *priv, cx2341x_mbox_func func,
 		   const struct cx2341x_mpeg_params *old,
@@ -1042,20 +1042,22 @@ int cx2341x_update(void *priv, cx2341x_mbox_func func,
 		11,	/* VCD */
 		12,	/* SVCD */
 	};
-
-	int err = 0;
-	int force = (old == NULL);
-	u16 temporal = new->video_temporal_filter;
+	int err;
 
 	cx2341x_api(priv, func, CX2341X_ENC_SET_OUTPUT_PORT, 2, new->port, 0);
 
-	if (force || NEQ(is_50hz)) {
+	if (!old ||
+	    CMP_FIELD(old, new, is_50hz)) {
 		err = cx2341x_api(priv, func, CX2341X_ENC_SET_FRAME_RATE, 1,
 				  new->is_50hz);
-		if (err) return err;
+		if (err)
+			return err;
 	}
 
-	if (force || NEQ(width) || NEQ(height) || NEQ(video_encoding)) {
+	if (!old ||
+	    CMP_FIELD(old, new, width) ||
+	    CMP_FIELD(old, new, height) ||
+	    CMP_FIELD(old, new, video_encoding)) {
 		u16 w = new->width;
 		u16 h = new->height;
 
@@ -1065,94 +1067,127 @@ int cx2341x_update(void *priv, cx2341x_mbox_func func,
 		}
 		err = cx2341x_api(priv, func, CX2341X_ENC_SET_FRAME_SIZE, 2,
 				  h, w);
-		if (err) return err;
+		if (err)
+			return err;
 	}
-	if (force || NEQ(stream_type)) {
+	if (!old ||
+	    CMP_FIELD(old, new, stream_type)) {
 		err = cx2341x_api(priv, func, CX2341X_ENC_SET_STREAM_TYPE, 1,
 				  mpeg_stream_type[new->stream_type]);
-		if (err) return err;
+		if (err)
+			return err;
 	}
-	if (force || NEQ(video_aspect)) {
+	if (!old ||
+	    CMP_FIELD(old, new, video_aspect)) {
 		err = cx2341x_api(priv, func, CX2341X_ENC_SET_ASPECT_RATIO, 1,
 				  1 + new->video_aspect);
-		if (err) return err;
+		if (err)
+			return err;
 	}
-	if (force || NEQ(video_b_frames) || NEQ(video_gop_size)) {
+	if (!old ||
+	    CMP_FIELD(old, new, video_b_frames) ||
+	    CMP_FIELD(old, new, video_gop_size)) {
 		err = cx2341x_api(priv, func, CX2341X_ENC_SET_GOP_PROPERTIES, 2,
-				new->video_gop_size, new->video_b_frames + 1);
-		if (err) return err;
+				  new->video_gop_size, new->video_b_frames + 1);
+		if (err)
+			return err;
 	}
-	if (force || NEQ(video_gop_closure)) {
+	if (!old ||
+	    CMP_FIELD(old, new, video_gop_closure)) {
 		err = cx2341x_api(priv, func, CX2341X_ENC_SET_GOP_CLOSURE, 1,
 				  new->video_gop_closure);
-		if (err) return err;
+		if (err)
+			return err;
 	}
-	if (force || NEQ(audio_properties)) {
+	if (!old ||
+	    CMP_FIELD(old, new, audio_properties)) {
 		err = cx2341x_api(priv, func, CX2341X_ENC_SET_AUDIO_PROPERTIES,
 				  1, new->audio_properties);
-		if (err) return err;
+		if (err)
+			return err;
 	}
-	if (force || NEQ(audio_mute)) {
+	if (!old ||
+	    CMP_FIELD(old, new, audio_mute)) {
 		err = cx2341x_api(priv, func, CX2341X_ENC_MUTE_AUDIO, 1,
 				  new->audio_mute);
-		if (err) return err;
+		if (err)
+			return err;
 	}
-	if (force || NEQ(video_bitrate_mode) || NEQ(video_bitrate) ||
-						NEQ(video_bitrate_peak)) {
+	if (!old ||
+	    CMP_FIELD(old, new, video_bitrate_mode) ||
+	    CMP_FIELD(old, new, video_bitrate) ||
+	    CMP_FIELD(old, new, video_bitrate_peak)) {
 		err = cx2341x_api(priv, func, CX2341X_ENC_SET_BIT_RATE, 5,
-				new->video_bitrate_mode, new->video_bitrate,
-				new->video_bitrate_peak / 400, 0, 0);
-		if (err) return err;
+				  new->video_bitrate_mode, new->video_bitrate,
+				  new->video_bitrate_peak / 400, 0, 0);
+		if (err)
+			return err;
 	}
-	if (force || NEQ(video_spatial_filter_mode) ||
-		     NEQ(video_temporal_filter_mode) ||
-		     NEQ(video_median_filter_type)) {
+	if (!old ||
+	    CMP_FIELD(old, new, video_spatial_filter_mode) ||
+	    CMP_FIELD(old, new, video_temporal_filter_mode) ||
+	    CMP_FIELD(old, new, video_median_filter_type)) {
 		err = cx2341x_api(priv, func, CX2341X_ENC_SET_DNR_FILTER_MODE,
-				  2, new->video_spatial_filter_mode |
+				  2,
+				  new->video_spatial_filter_mode |
 					(new->video_temporal_filter_mode << 1),
-				new->video_median_filter_type);
-		if (err) return err;
+				  new->video_median_filter_type);
+		if (err)
+			return err;
 	}
-	if (force || NEQ(video_luma_median_filter_bottom) ||
-		     NEQ(video_luma_median_filter_top) ||
-		     NEQ(video_chroma_median_filter_bottom) ||
-		     NEQ(video_chroma_median_filter_top)) {
+	if (!old ||
+	    CMP_FIELD(old, new, video_luma_median_filter_bottom) ||
+	    CMP_FIELD(old, new, video_luma_median_filter_top) ||
+	    CMP_FIELD(old, new, video_chroma_median_filter_bottom) ||
+	    CMP_FIELD(old, new, video_chroma_median_filter_top)) {
 		err = cx2341x_api(priv, func, CX2341X_ENC_SET_CORING_LEVELS, 4,
-				new->video_luma_median_filter_bottom,
-				new->video_luma_median_filter_top,
-				new->video_chroma_median_filter_bottom,
-				new->video_chroma_median_filter_top);
-		if (err) return err;
+				  new->video_luma_median_filter_bottom,
+				  new->video_luma_median_filter_top,
+				  new->video_chroma_median_filter_bottom,
+				  new->video_chroma_median_filter_top);
+		if (err)
+			return err;
 	}
-	if (force || NEQ(video_luma_spatial_filter_type) ||
-		     NEQ(video_chroma_spatial_filter_type)) {
+	if (!old ||
+	    CMP_FIELD(old, new, video_luma_spatial_filter_type) ||
+	    CMP_FIELD(old, new, video_chroma_spatial_filter_type)) {
 		err = cx2341x_api(priv, func,
 				  CX2341X_ENC_SET_SPATIAL_FILTER_TYPE,
 				  2, new->video_luma_spatial_filter_type,
 				  new->video_chroma_spatial_filter_type);
-		if (err) return err;
+		if (err)
+			return err;
 	}
-	if (force || NEQ(video_spatial_filter) ||
-		     old->video_temporal_filter != temporal) {
+	if (!old ||
+	    CMP_FIELD(old, new, video_spatial_filter) ||
+	    CMP_FIELD(old, new, video_temporal_filter)) {
 		err = cx2341x_api(priv, func, CX2341X_ENC_SET_DNR_FILTER_PROPS,
-				  2, new->video_spatial_filter, temporal);
-		if (err) return err;
+				  2, new->video_spatial_filter,
+				  new->video_temporal_filter);
+		if (err)
+			return err;
 	}
-	if (force || NEQ(video_temporal_decimation)) {
+	if (!old ||
+	    CMP_FIELD(old, new, video_temporal_decimation)) {
 		err = cx2341x_api(priv, func, CX2341X_ENC_SET_FRAME_DROP_RATE,
 				  1, new->video_temporal_decimation);
-		if (err) return err;
+		if (err)
+			return err;
 	}
-	if (force || NEQ(video_mute) ||
-		(new->video_mute && NEQ(video_mute_yuv))) {
+	if (!old ||
+	    CMP_FIELD(old, new, video_mute) ||
+	    (new->video_mute && CMP_FIELD(old, new, video_mute_yuv))) {
 		err = cx2341x_api(priv, func, CX2341X_ENC_MUTE_VIDEO, 1,
-				new->video_mute | (new->video_mute_yuv << 8));
-		if (err) return err;
+				  new->video_mute | (new->video_mute_yuv << 8));
+		if (err)
+			return err;
 	}
-	if (force || NEQ(stream_insert_nav_packets)) {
+	if (!old ||
+	    CMP_FIELD(old, new, stream_insert_nav_packets)) {
 		err = cx2341x_api(priv, func, CX2341X_ENC_MISC, 2,
-				7, new->stream_insert_nav_packets);
-		if (err) return err;
+				  7, new->stream_insert_nav_packets);
+		if (err)
+			return err;
 	}
 	return 0;
 }
diff --git a/drivers/media/common/siano/Kconfig b/drivers/media/common/siano/Kconfig
index 4bfbd5f463d1..577880b133eb 100644
--- a/drivers/media/common/siano/Kconfig
+++ b/drivers/media/common/siano/Kconfig
@@ -15,7 +15,7 @@ config SMS_SIANO_RC
 	depends on SMS_USB_DRV || SMS_SDIO_DRV
 	depends on MEDIA_COMMON_OPTIONS
 	default y
-	---help---
+	help
 	  Choose Y to select Remote Controller support for Siano driver.
 
 config SMS_SIANO_DEBUGFS
@@ -24,7 +24,7 @@ config SMS_SIANO_DEBUGFS
 	depends on DEBUG_FS
 	depends on SMS_USB_DRV = SMS_SDIO_DRV
 
-	---help---
+	help
 	  Choose Y to enable visualizing a dump of the frontend
 	  statistics response packets via debugfs. Currently, works
 	  only with Siano USB devices.
diff --git a/drivers/media/common/videobuf2/videobuf2-core.c b/drivers/media/common/videobuf2/videobuf2-core.c
index 15b6b9c0a2e4..7ebd58a1c431 100644
--- a/drivers/media/common/videobuf2/videobuf2-core.c
+++ b/drivers/media/common/videobuf2/videobuf2-core.c
@@ -672,6 +672,11 @@ int vb2_core_reqbufs(struct vb2_queue *q, enum vb2_memory memory,
 		return -EBUSY;
 	}
 
+	if (q->waiting_in_dqbuf && *count) {
+		dprintk(1, "another dup()ped fd is waiting for a buffer\n");
+		return -EBUSY;
+	}
+
 	if (*count == 0 || q->num_buffers != 0 ||
 	    (q->memory != VB2_MEMORY_UNKNOWN && q->memory != memory)) {
 		/*
@@ -807,6 +812,10 @@ int vb2_core_create_bufs(struct vb2_queue *q, enum vb2_memory memory,
 	}
 
 	if (!q->num_buffers) {
+		if (q->waiting_in_dqbuf && *count) {
+			dprintk(1, "another dup()ped fd is waiting for a buffer\n");
+			return -EBUSY;
+		}
 		memset(q->alloc_devs, 0, sizeof(q->alloc_devs));
 		q->memory = memory;
 		q->waiting_for_buffers = !q->is_output;
@@ -915,8 +924,7 @@ void vb2_buffer_done(struct vb2_buffer *vb, enum vb2_buffer_state state)
 
 	if (WARN_ON(state != VB2_BUF_STATE_DONE &&
 		    state != VB2_BUF_STATE_ERROR &&
-		    state != VB2_BUF_STATE_QUEUED &&
-		    state != VB2_BUF_STATE_REQUEUEING))
+		    state != VB2_BUF_STATE_QUEUED))
 		state = VB2_BUF_STATE_ERROR;
 
 #ifdef CONFIG_VIDEO_ADV_DEBUG
@@ -929,8 +937,7 @@ void vb2_buffer_done(struct vb2_buffer *vb, enum vb2_buffer_state state)
 	dprintk(4, "done processing on buffer %d, state: %d\n",
 			vb->index, state);
 
-	if (state != VB2_BUF_STATE_QUEUED &&
-	    state != VB2_BUF_STATE_REQUEUEING) {
+	if (state != VB2_BUF_STATE_QUEUED) {
 		/* sync buffers */
 		for (plane = 0; plane < vb->num_planes; ++plane)
 			call_void_memop(vb, finish, vb->planes[plane].mem_priv);
@@ -938,8 +945,7 @@ void vb2_buffer_done(struct vb2_buffer *vb, enum vb2_buffer_state state)
 	}
 
 	spin_lock_irqsave(&q->done_lock, flags);
-	if (state == VB2_BUF_STATE_QUEUED ||
-	    state == VB2_BUF_STATE_REQUEUEING) {
+	if (state == VB2_BUF_STATE_QUEUED) {
 		vb->state = VB2_BUF_STATE_QUEUED;
 	} else {
 		/* Add the buffer to the done buffers list */
@@ -949,8 +955,6 @@ void vb2_buffer_done(struct vb2_buffer *vb, enum vb2_buffer_state state)
 	atomic_dec(&q->owned_by_drv_count);
 
 	if (state != VB2_BUF_STATE_QUEUED && vb->req_obj.req) {
-		/* This is not supported at the moment */
-		WARN_ON(state == VB2_BUF_STATE_REQUEUEING);
 		media_request_object_unbind(&vb->req_obj);
 		media_request_object_put(&vb->req_obj);
 	}
@@ -962,10 +966,6 @@ void vb2_buffer_done(struct vb2_buffer *vb, enum vb2_buffer_state state)
 	switch (state) {
 	case VB2_BUF_STATE_QUEUED:
 		return;
-	case VB2_BUF_STATE_REQUEUEING:
-		if (q->start_streaming_called)
-			__enqueue_in_driver(vb);
-		return;
 	default:
 		/* Inform any processes that may be waiting for buffers */
 		wake_up(&q->done_wq);
@@ -1516,6 +1516,12 @@ int vb2_core_qbuf(struct vb2_queue *q, unsigned int index, void *pb,
 
 	vb = q->bufs[index];
 
+	if (!req && vb->state != VB2_BUF_STATE_IN_REQUEST &&
+	    q->requires_requests) {
+		dprintk(1, "qbuf requires a request\n");
+		return -EBADR;
+	}
+
 	if ((req && q->uses_qbuf) ||
 	    (!req && vb->state != VB2_BUF_STATE_IN_REQUEST &&
 	     q->uses_requests)) {
@@ -1659,6 +1665,11 @@ static int __vb2_wait_for_done_vb(struct vb2_queue *q, int nonblocking)
 	for (;;) {
 		int ret;
 
+		if (q->waiting_in_dqbuf) {
+			dprintk(1, "another dup()ped fd is waiting for a buffer\n");
+			return -EBUSY;
+		}
+
 		if (!q->streaming) {
 			dprintk(1, "streaming off, will not wait for buffers\n");
 			return -EINVAL;
@@ -1686,6 +1697,7 @@ static int __vb2_wait_for_done_vb(struct vb2_queue *q, int nonblocking)
 			return -EAGAIN;
 		}
 
+		q->waiting_in_dqbuf = 1;
 		/*
 		 * We are streaming and blocking, wait for another buffer to
 		 * become ready or for streamoff. Driver's lock is released to
@@ -1706,6 +1718,7 @@ static int __vb2_wait_for_done_vb(struct vb2_queue *q, int nonblocking)
 		 * the locks or return an error if one occurred.
 		 */
 		call_void_qop(q, wait_finish, q);
+		q->waiting_in_dqbuf = 0;
 		if (ret) {
 			dprintk(1, "sleep was interrupted\n");
 			return ret;
@@ -2247,6 +2260,9 @@ int vb2_core_queue_init(struct vb2_queue *q)
 	    WARN_ON(!q->ops->buf_queue))
 		return -EINVAL;
 
+	if (WARN_ON(q->requires_requests && !q->supports_requests))
+		return -EINVAL;
+
 	INIT_LIST_HEAD(&q->queued_list);
 	INIT_LIST_HEAD(&q->done_list);
 	spin_lock_init(&q->done_lock);
@@ -2585,6 +2601,12 @@ static size_t __vb2_perform_fileio(struct vb2_queue *q, char __user *data, size_
 	if (!data)
 		return -EINVAL;
 
+	if (q->waiting_in_dqbuf) {
+		dprintk(3, "another dup()ped fd is %s\n",
+			read ? "reading" : "writing");
+		return -EBUSY;
+	}
+
 	/*
 	 * Initialize emulator on first call.
 	 */
diff --git a/drivers/media/common/videobuf2/videobuf2-v4l2.c b/drivers/media/common/videobuf2/videobuf2-v4l2.c
index d09dee20e421..fb9ac7696fc6 100644
--- a/drivers/media/common/videobuf2/videobuf2-v4l2.c
+++ b/drivers/media/common/videobuf2/videobuf2-v4l2.c
@@ -122,9 +122,9 @@ static int __verify_length(struct vb2_buffer *vb, const struct v4l2_buffer *b)
 }
 
 /*
- * __init_v4l2_vb2_buffer() - initialize the v4l2_vb2_buffer struct
+ * __init_vb2_v4l2_buffer() - initialize the vb2_v4l2_buffer struct
  */
-static void __init_v4l2_vb2_buffer(struct vb2_buffer *vb)
+static void __init_vb2_v4l2_buffer(struct vb2_buffer *vb)
 {
 	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
 
@@ -368,6 +368,12 @@ static int vb2_queue_or_prepare_buf(struct vb2_queue *q, struct media_device *md
 	if (ret)
 		return ret;
 
+	if (!is_prepare && (b->flags & V4L2_BUF_FLAG_REQUEST_FD) &&
+	    vb->state != VB2_BUF_STATE_DEQUEUED) {
+		dprintk(1, "%s: buffer is not in dequeued state\n", opname);
+		return -EINVAL;
+	}
+
 	if (!vb->prepared) {
 		/* Copy relevant information provided by the userspace */
 		memset(vbuf->planes, 0,
@@ -381,6 +387,10 @@ static int vb2_queue_or_prepare_buf(struct vb2_queue *q, struct media_device *md
 		return 0;
 
 	if (!(b->flags & V4L2_BUF_FLAG_REQUEST_FD)) {
+		if (q->requires_requests) {
+			dprintk(1, "%s: queue requires requests\n", opname);
+			return -EBADR;
+		}
 		if (q->uses_requests) {
 			dprintk(1, "%s: queue uses requests\n", opname);
 			return -EBUSY;
@@ -388,7 +398,7 @@ static int vb2_queue_or_prepare_buf(struct vb2_queue *q, struct media_device *md
 		return 0;
 	} else if (!q->supports_requests) {
 		dprintk(1, "%s: queue does not support requests\n", opname);
-		return -EACCES;
+		return -EBADR;
 	} else if (q->uses_qbuf) {
 		dprintk(1, "%s: queue does not use requests\n", opname);
 		return -EBUSY;
@@ -419,11 +429,6 @@ static int vb2_queue_or_prepare_buf(struct vb2_queue *q, struct media_device *md
 		    !q->ops->buf_out_validate))
 		return -EINVAL;
 
-	if (vb->state != VB2_BUF_STATE_DEQUEUED) {
-		dprintk(1, "%s: buffer is not in dequeued state\n", opname);
-		return -EINVAL;
-	}
-
 	if (b->request_fd < 0) {
 		dprintk(1, "%s: request_fd < 0\n", opname);
 		return -EINVAL;
@@ -543,7 +548,6 @@ static void __fill_v4l2_buffer(struct vb2_buffer *vb, void *pb)
 		break;
 	case VB2_BUF_STATE_PREPARING:
 	case VB2_BUF_STATE_DEQUEUED:
-	case VB2_BUF_STATE_REQUEUEING:
 		/* nothing */
 		break;
 	}
@@ -592,7 +596,7 @@ static int __fill_vb2_buffer(struct vb2_buffer *vb, struct vb2_plane *planes)
 
 static const struct vb2_buf_ops v4l2_buf_ops = {
 	.verify_planes_array	= __verify_planes_array_core,
-	.init_buffer		= __init_v4l2_vb2_buffer,
+	.init_buffer		= __init_vb2_v4l2_buffer,
 	.fill_user_buffer	= __fill_v4l2_buffer,
 	.fill_vb2_buffer	= __fill_vb2_buffer,
 	.copy_timestamp		= __copy_timestamp,
diff --git a/drivers/media/common/videobuf2/videobuf2-vmalloc.c b/drivers/media/common/videobuf2/videobuf2-vmalloc.c
index 6dfbd5b05907..1c6659f7c394 100644
--- a/drivers/media/common/videobuf2/videobuf2-vmalloc.c
+++ b/drivers/media/common/videobuf2/videobuf2-vmalloc.c
@@ -46,17 +46,17 @@ static void *vb2_vmalloc_alloc(struct device *dev, unsigned long attrs,
 
 	buf->size = size;
 	buf->vaddr = vmalloc_user(buf->size);
-	buf->dma_dir = dma_dir;
-	buf->handler.refcount = &buf->refcount;
-	buf->handler.put = vb2_vmalloc_put;
-	buf->handler.arg = buf;
-
 	if (!buf->vaddr) {
 		pr_debug("vmalloc of size %ld failed\n", buf->size);
 		kfree(buf);
 		return ERR_PTR(-ENOMEM);
 	}
 
+	buf->dma_dir = dma_dir;
+	buf->handler.refcount = &buf->refcount;
+	buf->handler.put = vb2_vmalloc_put;
+	buf->handler.arg = buf;
+
 	refcount_set(&buf->refcount, 1);
 	return buf;
 }
diff --git a/drivers/media/dvb-core/dvbdev.c b/drivers/media/dvb-core/dvbdev.c
index 4a5834a1c3b7..a3393cd4e584 100644
--- a/drivers/media/dvb-core/dvbdev.c
+++ b/drivers/media/dvb-core/dvbdev.c
@@ -526,7 +526,6 @@ int dvb_register_device(struct dvb_adapter *adap, struct dvb_device **pdvbdev,
 		dvb_media_device_free(dvbdev);
 		kfree(dvbdevfops);
 		kfree(dvbdev);
-		up_write(&minor_rwsem);
 		mutex_unlock(&dvbdev_register_lock);
 		return ret;
 	}
diff --git a/drivers/media/dvb-frontends/as102_fe.c b/drivers/media/dvb-frontends/as102_fe.c
index f59a102b0a64..9ba8f39fe310 100644
--- a/drivers/media/dvb-frontends/as102_fe.c
+++ b/drivers/media/dvb-frontends/as102_fe.c
@@ -467,7 +467,7 @@ struct dvb_frontend *as102_attach(const char *name,
 
 	/* init frontend callback ops */
 	memcpy(&fe->ops, &as102_fe_ops, sizeof(struct dvb_frontend_ops));
-	strncpy(fe->ops.info.name, name, sizeof(fe->ops.info.name));
+	strscpy(fe->ops.info.name, name, sizeof(fe->ops.info.name));
 
 	return fe;
 
diff --git a/drivers/media/dvb-frontends/dib7000p.c b/drivers/media/dvb-frontends/dib7000p.c
index f8040f6def62..d869029ca87d 100644
--- a/drivers/media/dvb-frontends/dib7000p.c
+++ b/drivers/media/dvb-frontends/dib7000p.c
@@ -2774,7 +2774,8 @@ static struct dvb_frontend *dib7000p_init(struct i2c_adapter *i2c_adap, u8 i2c_a
 	dibx000_init_i2c_master(&st->i2c_master, DIB7000P, st->i2c_adap, st->i2c_addr);
 
 	/* init 7090 tuner adapter */
-	strncpy(st->dib7090_tuner_adap.name, "DiB7090 tuner interface", sizeof(st->dib7090_tuner_adap.name));
+	strscpy(st->dib7090_tuner_adap.name, "DiB7090 tuner interface",
+		sizeof(st->dib7090_tuner_adap.name));
 	st->dib7090_tuner_adap.algo = &dib7090_tuner_xfer_algo;
 	st->dib7090_tuner_adap.algo_data = NULL;
 	st->dib7090_tuner_adap.dev.parent = st->i2c_adap->dev.parent;
diff --git a/drivers/media/dvb-frontends/dib8000.c b/drivers/media/dvb-frontends/dib8000.c
index 85c429cce23e..564669338dc6 100644
--- a/drivers/media/dvb-frontends/dib8000.c
+++ b/drivers/media/dvb-frontends/dib8000.c
@@ -4458,8 +4458,8 @@ static struct dvb_frontend *dib8000_init(struct i2c_adapter *i2c_adap, u8 i2c_ad
 	dibx000_init_i2c_master(&state->i2c_master, DIB8000, state->i2c.adap, state->i2c.addr);
 
 	/* init 8096p tuner adapter */
-	strncpy(state->dib8096p_tuner_adap.name, "DiB8096P tuner interface",
-			sizeof(state->dib8096p_tuner_adap.name));
+	strscpy(state->dib8096p_tuner_adap.name, "DiB8096P tuner interface",
+		sizeof(state->dib8096p_tuner_adap.name));
 	state->dib8096p_tuner_adap.algo = &dib8096p_tuner_xfer_algo;
 	state->dib8096p_tuner_adap.algo_data = NULL;
 	state->dib8096p_tuner_adap.dev.parent = state->i2c.adap->dev.parent;
diff --git a/drivers/media/dvb-frontends/dib9000.c b/drivers/media/dvb-frontends/dib9000.c
index 1875da07c150..e7838926e6bc 100644
--- a/drivers/media/dvb-frontends/dib9000.c
+++ b/drivers/media/dvb-frontends/dib9000.c
@@ -2521,7 +2521,8 @@ struct dvb_frontend *dib9000_attach(struct i2c_adapter *i2c_adap, u8 i2c_addr, c
 	dibx000_init_i2c_master(&st->i2c_master, DIB7000MC, st->i2c.i2c_adap, st->i2c.i2c_addr);
 
 	st->tuner_adap.dev.parent = i2c_adap->dev.parent;
-	strncpy(st->tuner_adap.name, "DIB9000_FW TUNER ACCESS", sizeof(st->tuner_adap.name));
+	strscpy(st->tuner_adap.name, "DIB9000_FW TUNER ACCESS",
+		sizeof(st->tuner_adap.name));
 	st->tuner_adap.algo = &dib9000_tuner_algo;
 	st->tuner_adap.algo_data = NULL;
 	i2c_set_adapdata(&st->tuner_adap, st);
@@ -2529,7 +2530,8 @@ struct dvb_frontend *dib9000_attach(struct i2c_adapter *i2c_adap, u8 i2c_addr, c
 		goto error;
 
 	st->component_bus.dev.parent = i2c_adap->dev.parent;
-	strncpy(st->component_bus.name, "DIB9000_FW COMPONENT BUS ACCESS", sizeof(st->component_bus.name));
+	strscpy(st->component_bus.name, "DIB9000_FW COMPONENT BUS ACCESS",
+		sizeof(st->component_bus.name));
 	st->component_bus.algo = &dib9000_component_bus_algo;
 	st->component_bus.algo_data = NULL;
 	st->component_bus_speed = 340;
diff --git a/drivers/media/dvb-frontends/drxd_hard.c b/drivers/media/dvb-frontends/drxd_hard.c
index 0a5b15bee1d7..c597d6fc3046 100644
--- a/drivers/media/dvb-frontends/drxd_hard.c
+++ b/drivers/media/dvb-frontends/drxd_hard.c
@@ -2263,61 +2263,41 @@ static int DRX_Start(struct drxd_state *state, s32 off)
 		case DRX_CHANNEL_LOW:
 			transmissionParams |= SC_RA_RAM_OP_PARAM_PRIO_LO;
 			status = Write16(state, EC_SB_REG_PRIOR__A, EC_SB_REG_PRIOR_LO, 0x0000);
-			if (status < 0)
-				break;
 			break;
 		case DRX_CHANNEL_HIGH:
 			transmissionParams |= SC_RA_RAM_OP_PARAM_PRIO_HI;
 			status = Write16(state, EC_SB_REG_PRIOR__A, EC_SB_REG_PRIOR_HI, 0x0000);
-			if (status < 0)
-				break;
 			break;
-
 		}
 
 		switch (p->code_rate_HP) {
 		case FEC_1_2:
 			transmissionParams |= SC_RA_RAM_OP_PARAM_RATE_1_2;
-			if (state->type_A) {
+			if (state->type_A)
 				status = Write16(state, EC_VD_REG_SET_CODERATE__A, EC_VD_REG_SET_CODERATE_C1_2, 0x0000);
-				if (status < 0)
-					break;
-			}
 			break;
 		default:
 			operationMode |= SC_RA_RAM_OP_AUTO_RATE__M;
 			/* fall through */
 		case FEC_2_3:
 			transmissionParams |= SC_RA_RAM_OP_PARAM_RATE_2_3;
-			if (state->type_A) {
+			if (state->type_A)
 				status = Write16(state, EC_VD_REG_SET_CODERATE__A, EC_VD_REG_SET_CODERATE_C2_3, 0x0000);
-				if (status < 0)
-					break;
-			}
 			break;
 		case FEC_3_4:
 			transmissionParams |= SC_RA_RAM_OP_PARAM_RATE_3_4;
-			if (state->type_A) {
+			if (state->type_A)
 				status = Write16(state, EC_VD_REG_SET_CODERATE__A, EC_VD_REG_SET_CODERATE_C3_4, 0x0000);
-				if (status < 0)
-					break;
-			}
 			break;
 		case FEC_5_6:
 			transmissionParams |= SC_RA_RAM_OP_PARAM_RATE_5_6;
-			if (state->type_A) {
+			if (state->type_A)
 				status = Write16(state, EC_VD_REG_SET_CODERATE__A, EC_VD_REG_SET_CODERATE_C5_6, 0x0000);
-				if (status < 0)
-					break;
-			}
 			break;
 		case FEC_7_8:
 			transmissionParams |= SC_RA_RAM_OP_PARAM_RATE_7_8;
-			if (state->type_A) {
+			if (state->type_A)
 				status = Write16(state, EC_VD_REG_SET_CODERATE__A, EC_VD_REG_SET_CODERATE_C7_8, 0x0000);
-				if (status < 0)
-					break;
-			}
 			break;
 		}
 		if (status < 0)
diff --git a/drivers/media/dvb-frontends/dvb-pll.c b/drivers/media/dvb-frontends/dvb-pll.c
index 29836c1a40e9..da1b48a18515 100644
--- a/drivers/media/dvb-frontends/dvb-pll.c
+++ b/drivers/media/dvb-frontends/dvb-pll.c
@@ -839,7 +839,7 @@ struct dvb_frontend *dvb_pll_attach(struct dvb_frontend *fe, int pll_addr,
 	memcpy(&fe->ops.tuner_ops, &dvb_pll_tuner_ops,
 	       sizeof(struct dvb_tuner_ops));
 
-	strncpy(fe->ops.tuner_ops.info.name, desc->name,
+	strscpy(fe->ops.tuner_ops.info.name, desc->name,
 		sizeof(fe->ops.tuner_ops.info.name));
 
 	fe->ops.tuner_ops.info.frequency_min_hz = desc->min;
diff --git a/drivers/media/dvb-frontends/m88ds3103.c b/drivers/media/dvb-frontends/m88ds3103.c
index 123f2a33738b..b543e1c4c4f9 100644
--- a/drivers/media/dvb-frontends/m88ds3103.c
+++ b/drivers/media/dvb-frontends/m88ds3103.c
@@ -309,6 +309,9 @@ static int m88ds3103_set_frontend(struct dvb_frontend *fe)
 	u16 u16tmp;
 	u32 tuner_frequency_khz, target_mclk;
 	s32 s32tmp;
+	static const struct reg_sequence reset_buf[] = {
+		{0x07, 0x80}, {0x07, 0x00}
+	};
 
 	dev_dbg(&client->dev,
 		"delivery_system=%d modulation=%d frequency=%u symbol_rate=%d inversion=%d pilot=%d rolloff=%d\n",
@@ -321,11 +324,7 @@ static int m88ds3103_set_frontend(struct dvb_frontend *fe)
 	}
 
 	/* reset */
-	ret = regmap_write(dev->regmap, 0x07, 0x80);
-	if (ret)
-		goto err;
-
-	ret = regmap_write(dev->regmap, 0x07, 0x00);
+	ret = regmap_multi_reg_write(dev->regmap, reset_buf, 2);
 	if (ret)
 		goto err;
 
@@ -1470,7 +1469,7 @@ static int m88ds3103_probe(struct i2c_client *client,
 	/* create dvb_frontend */
 	memcpy(&dev->fe.ops, &m88ds3103_ops, sizeof(struct dvb_frontend_ops));
 	if (dev->chip_id == M88RS6000_CHIP_ID)
-		strncpy(dev->fe.ops.info.name, "Montage Technology M88RS6000",
+		strscpy(dev->fe.ops.info.name, "Montage Technology M88RS6000",
 			sizeof(dev->fe.ops.info.name));
 	if (!pdata->attach_in_use)
 		dev->fe.ops.release = NULL;
diff --git a/drivers/media/dvb-frontends/si2165.c b/drivers/media/dvb-frontends/si2165.c
index feacd8da421d..c9bcd2e95417 100644
--- a/drivers/media/dvb-frontends/si2165.c
+++ b/drivers/media/dvb-frontends/si2165.c
@@ -275,18 +275,20 @@ static u32 si2165_get_fe_clk(struct si2165_state *state)
 
 static int si2165_wait_init_done(struct si2165_state *state)
 {
-	int ret = -EINVAL;
+	int ret;
 	u8 val = 0;
 	int i;
 
 	for (i = 0; i < 3; ++i) {
-		si2165_readreg8(state, REG_INIT_DONE, &val);
+		ret = si2165_readreg8(state, REG_INIT_DONE, &val);
+		if (ret < 0)
+			return ret;
 		if (val == 0x01)
 			return 0;
 		usleep_range(1000, 50000);
 	}
 	dev_err(&state->client->dev, "init_done was not set\n");
-	return ret;
+	return -EINVAL;
 }
 
 static int si2165_upload_firmware_block(struct si2165_state *state,
@@ -1299,7 +1301,6 @@ MODULE_DEVICE_TABLE(i2c, si2165_id_table);
 
 static struct i2c_driver si2165_driver = {
 	.driver = {
-		.owner	= THIS_MODULE,
 		.name	= "si2165",
 	},
 	.probe		= si2165_probe,
diff --git a/drivers/media/dvb-frontends/ts2020.c b/drivers/media/dvb-frontends/ts2020.c
index e5cd2cd414f4..0af9b335be12 100644
--- a/drivers/media/dvb-frontends/ts2020.c
+++ b/drivers/media/dvb-frontends/ts2020.c
@@ -180,6 +180,9 @@ static int ts2020_set_tuner_rf(struct dvb_frontend *fe)
 	unsigned int utmp;
 
 	ret = regmap_read(dev->regmap, 0x3d, &utmp);
+	if (ret)
+		return ret;
+
 	utmp &= 0x7f;
 	if (utmp < 0x16)
 		utmp = 0xa1;
diff --git a/drivers/media/i2c/Kconfig b/drivers/media/i2c/Kconfig
index 6d32f8dcf83b..7793358ab8b3 100644
--- a/drivers/media/i2c/Kconfig
+++ b/drivers/media/i2c/Kconfig
@@ -8,7 +8,7 @@ config VIDEO_IR_I2C
 	tristate "I2C module for IR" if !MEDIA_SUBDRV_AUTOSELECT
 	depends on I2C && RC_CORE
 	default y
-	---help---
+	help
 	  Most boards have an IR chip directly connected via GPIO. However,
 	  some video boards have the IR connected via I2C bus.
 
@@ -29,7 +29,7 @@ comment "Audio decoders, processors and mixers"
 config VIDEO_TVAUDIO
 	tristate "Simple audio decoder chips"
 	depends on VIDEO_V4L2 && I2C
-	---help---
+	help
 	  Support for several audio decoder chips found on some bt8xx boards:
 	  Philips: tda9840, tda9873h, tda9874h/a, tda9850, tda985x, tea6300,
 		   tea6320, tea6420, tda8425, ta8874z.
@@ -41,7 +41,7 @@ config VIDEO_TVAUDIO
 config VIDEO_TDA7432
 	tristate "Philips TDA7432 audio processor"
 	depends on VIDEO_V4L2 && I2C
-	---help---
+	help
 	  Support for tda7432 audio decoder chip found on some bt8xx boards.
 
 	  To compile this driver as a module, choose M here: the
@@ -50,7 +50,7 @@ config VIDEO_TDA7432
 config VIDEO_TDA9840
 	tristate "Philips TDA9840 audio processor"
 	depends on I2C
-	---help---
+	help
 	  Support for tda9840 audio decoder chip found on some Zoran boards.
 
 	  To compile this driver as a module, choose M here: the
@@ -60,9 +60,10 @@ config VIDEO_TDA1997X
 	tristate "NXP TDA1997x HDMI receiver"
 	depends on VIDEO_V4L2 && I2C && VIDEO_V4L2_SUBDEV_API
 	depends on SND_SOC
-	select SND_PCM
 	select HDMI
-	---help---
+	select SND_PCM
+	select V4L2_FWNODE
+	help
 	  V4L2 subdevice driver for the NXP TDA1997x HDMI receivers.
 
 	  To compile this driver as a module, choose M here: the
@@ -71,7 +72,7 @@ config VIDEO_TDA1997X
 config VIDEO_TEA6415C
 	tristate "Philips TEA6415C audio processor"
 	depends on I2C
-	---help---
+	help
 	  Support for tea6415c audio decoder chip found on some bt8xx boards.
 
 	  To compile this driver as a module, choose M here: the
@@ -80,7 +81,7 @@ config VIDEO_TEA6415C
 config VIDEO_TEA6420
 	tristate "Philips TEA6420 audio processor"
 	depends on I2C
-	---help---
+	help
 	  Support for tea6420 audio decoder chip found on some bt8xx boards.
 
 	  To compile this driver as a module, choose M here: the
@@ -89,7 +90,7 @@ config VIDEO_TEA6420
 config VIDEO_MSP3400
 	tristate "Micronas MSP34xx audio decoders"
 	depends on VIDEO_V4L2 && I2C
-	---help---
+	help
 	  Support for the Micronas MSP34xx series of audio decoders.
 
 	  To compile this driver as a module, choose M here: the
@@ -98,7 +99,7 @@ config VIDEO_MSP3400
 config VIDEO_CS3308
 	tristate "Cirrus Logic CS3308 audio ADC"
 	depends on VIDEO_V4L2 && I2C
-	---help---
+	help
 	  Support for the Cirrus Logic CS3308 High Performance 8-Channel
 	  Analog Volume Control
 
@@ -108,7 +109,7 @@ config VIDEO_CS3308
 config VIDEO_CS5345
 	tristate "Cirrus Logic CS5345 audio ADC"
 	depends on VIDEO_V4L2 && I2C
-	---help---
+	help
 	  Support for the Cirrus Logic CS5345 24-bit, 192 kHz
 	  stereo A/D converter.
 
@@ -118,7 +119,7 @@ config VIDEO_CS5345
 config VIDEO_CS53L32A
 	tristate "Cirrus Logic CS53L32A audio ADC"
 	depends on VIDEO_V4L2 && I2C
-	---help---
+	help
 	  Support for the Cirrus Logic CS53L32A low voltage
 	  stereo A/D converter.
 
@@ -128,7 +129,7 @@ config VIDEO_CS53L32A
 config VIDEO_TLV320AIC23B
 	tristate "Texas Instruments TLV320AIC23B audio codec"
 	depends on VIDEO_V4L2 && I2C
-	---help---
+	help
 	  Support for the Texas Instruments TLV320AIC23B audio codec.
 
 	  To compile this driver as a module, choose M here: the
@@ -137,7 +138,7 @@ config VIDEO_TLV320AIC23B
 config VIDEO_UDA1342
 	tristate "Philips UDA1342 audio codec"
 	depends on VIDEO_V4L2 && I2C
-	---help---
+	help
 	  Support for the Philips UDA1342 audio codec.
 
 	  To compile this driver as a module, choose M here: the
@@ -146,7 +147,7 @@ config VIDEO_UDA1342
 config VIDEO_WM8775
 	tristate "Wolfson Microelectronics WM8775 audio ADC with input mixer"
 	depends on VIDEO_V4L2 && I2C
-	---help---
+	help
 	  Support for the Wolfson Microelectronics WM8775 high
 	  performance stereo A/D Converter with a 4 channel input mixer.
 
@@ -156,7 +157,7 @@ config VIDEO_WM8775
 config VIDEO_WM8739
 	tristate "Wolfson Microelectronics WM8739 stereo audio ADC"
 	depends on VIDEO_V4L2 && I2C
-	---help---
+	help
 	  Support for the Wolfson Microelectronics WM8739
 	  stereo A/D Converter.
 
@@ -166,7 +167,7 @@ config VIDEO_WM8739
 config VIDEO_VP27SMPX
 	tristate "Panasonic VP27's internal MPX"
 	depends on VIDEO_V4L2 && I2C
-	---help---
+	help
 	  Support for the internal MPX of the Panasonic VP27s tuner.
 
 	  To compile this driver as a module, choose M here: the
@@ -200,7 +201,7 @@ comment "Video decoders"
 config VIDEO_ADV7180
 	tristate "Analog Devices ADV7180 decoder"
 	depends on GPIOLIB && VIDEO_V4L2 && I2C && VIDEO_V4L2_SUBDEV_API
-	---help---
+	help
 	  Support for the Analog Devices ADV7180 video decoder.
 
 	  To compile this driver as a module, choose M here: the
@@ -209,7 +210,7 @@ config VIDEO_ADV7180
 config VIDEO_ADV7183
 	tristate "Analog Devices ADV7183 decoder"
 	depends on VIDEO_V4L2 && I2C
-	---help---
+	help
 	  V4l2 subdevice driver for the Analog Devices
 	  ADV7183 video decoder.
 
@@ -221,7 +222,8 @@ config VIDEO_ADV748X
 	depends on VIDEO_V4L2 && I2C && VIDEO_V4L2_SUBDEV_API
 	depends on OF
 	select REGMAP_I2C
-	---help---
+	select V4L2_FWNODE
+	help
 	  V4L2 subdevice driver for the Analog Devices
 	  ADV7481 and ADV7482 HDMI/Analog video decoders.
 
@@ -234,7 +236,7 @@ config VIDEO_ADV7604
 	depends on GPIOLIB || COMPILE_TEST
 	select HDMI
 	select V4L2_FWNODE
-	---help---
+	help
 	  Support for the Analog Devices ADV7604 video decoder.
 
 	  This is a Analog Devices Component/Graphics Digitizer
@@ -247,7 +249,7 @@ config VIDEO_ADV7604_CEC
 	bool "Enable Analog Devices ADV7604 CEC support"
 	depends on VIDEO_ADV7604
 	select CEC_CORE
-	---help---
+	help
 	  When selected the adv7604 will support the optional
 	  HDMI CEC feature.
 
@@ -255,7 +257,7 @@ config VIDEO_ADV7842
 	tristate "Analog Devices ADV7842 decoder"
 	depends on VIDEO_V4L2 && I2C && VIDEO_V4L2_SUBDEV_API
 	select HDMI
-	---help---
+	help
 	  Support for the Analog Devices ADV7842 video decoder.
 
 	  This is a Analog Devices Component/Graphics/SD Digitizer
@@ -268,14 +270,14 @@ config VIDEO_ADV7842_CEC
 	bool "Enable Analog Devices ADV7842 CEC support"
 	depends on VIDEO_ADV7842
 	select CEC_CORE
-	---help---
+	help
 	  When selected the adv7842 will support the optional
 	  HDMI CEC feature.
 
 config VIDEO_BT819
 	tristate "BT819A VideoStream decoder"
 	depends on VIDEO_V4L2 && I2C
-	---help---
+	help
 	  Support for BT819A video decoder.
 
 	  To compile this driver as a module, choose M here: the
@@ -284,7 +286,7 @@ config VIDEO_BT819
 config VIDEO_BT856
 	tristate "BT856 VideoStream decoder"
 	depends on VIDEO_V4L2 && I2C
-	---help---
+	help
 	  Support for BT856 video decoder.
 
 	  To compile this driver as a module, choose M here: the
@@ -293,7 +295,7 @@ config VIDEO_BT856
 config VIDEO_BT866
 	tristate "BT866 VideoStream decoder"
 	depends on VIDEO_V4L2 && I2C
-	---help---
+	help
 	  Support for BT866 video decoder.
 
 	  To compile this driver as a module, choose M here: the
@@ -302,7 +304,7 @@ config VIDEO_BT866
 config VIDEO_KS0127
 	tristate "KS0127 video decoder"
 	depends on VIDEO_V4L2 && I2C
-	---help---
+	help
 	  Support for KS0127 video decoder.
 
 	  This chip is used on AverMedia AVS6EYES Zoran-based MJPEG
@@ -314,53 +316,16 @@ config VIDEO_KS0127
 config VIDEO_ML86V7667
 	tristate "OKI ML86V7667 video decoder"
 	depends on VIDEO_V4L2 && I2C
-	---help---
+	help
 	  Support for the OKI Semiconductor ML86V7667 video decoder.
 
 	  To compile this driver as a module, choose M here: the
 	  module will be called ml86v7667.
 
-config VIDEO_AD5820
-	tristate "AD5820 lens voice coil support"
-	depends on I2C && VIDEO_V4L2 && MEDIA_CONTROLLER
-	---help---
-	  This is a driver for the AD5820 camera lens voice coil.
-	  It is used for example in Nokia N900 (RX-51).
-
-config VIDEO_AK7375
-	tristate "AK7375 lens voice coil support"
-	depends on I2C && VIDEO_V4L2 && MEDIA_CONTROLLER
-	depends on VIDEO_V4L2_SUBDEV_API
-	help
-	  This is a driver for the AK7375 camera lens voice coil.
-	  AK7375 is a 12 bit DAC with 120mA output current sink
-	  capability. This is designed for linear control of
-	  voice coil motors, controlled via I2C serial interface.
-
-config VIDEO_DW9714
-	tristate "DW9714 lens voice coil support"
-	depends on I2C && VIDEO_V4L2 && MEDIA_CONTROLLER
-	depends on VIDEO_V4L2_SUBDEV_API
-	---help---
-	  This is a driver for the DW9714 camera lens voice coil.
-	  DW9714 is a 10 bit DAC with 120mA output current sink
-	  capability. This is designed for linear control of
-	  voice coil motors, controlled via I2C serial interface.
-
-config VIDEO_DW9807_VCM
-	tristate "DW9807 lens voice coil support"
-	depends on I2C && VIDEO_V4L2 && MEDIA_CONTROLLER
-	depends on VIDEO_V4L2_SUBDEV_API
-	---help---
-	  This is a driver for the DW9807 camera lens voice coil.
-	  DW9807 is a 10 bit DAC with 100mA output current sink
-	  capability. This is designed for linear control of
-	  voice coil motors, controlled via I2C serial interface.
-
 config VIDEO_SAA7110
 	tristate "Philips SAA7110 video decoder"
 	depends on VIDEO_V4L2 && I2C
-	---help---
+	help
 	  Support for the Philips SAA7110 video decoders.
 
 	  To compile this driver as a module, choose M here: the
@@ -369,7 +334,7 @@ config VIDEO_SAA7110
 config VIDEO_SAA711X
 	tristate "Philips SAA7111/3/4/5 video decoders"
 	depends on VIDEO_V4L2 && I2C
-	---help---
+	help
 	  Support for the Philips SAA7111/3/4/5 video decoders.
 
 	  To compile this driver as a module, choose M here: the
@@ -380,7 +345,7 @@ config VIDEO_TC358743
 	depends on VIDEO_V4L2 && I2C && VIDEO_V4L2_SUBDEV_API
 	select HDMI
 	select V4L2_FWNODE
-	---help---
+	help
 	  Support for the Toshiba TC358743 HDMI to MIPI CSI-2 bridge.
 
 	  To compile this driver as a module, choose M here: the
@@ -390,7 +355,7 @@ config VIDEO_TC358743_CEC
 	bool "Enable Toshiba TC358743 CEC support"
 	depends on VIDEO_TC358743
 	select CEC_CORE
-	---help---
+	help
 	  When selected the tc358743 will support the optional
 	  HDMI CEC feature.
 
@@ -398,7 +363,7 @@ config VIDEO_TVP514X
 	tristate "Texas Instruments TVP514x video decoder"
 	depends on VIDEO_V4L2 && I2C
 	select V4L2_FWNODE
-	---help---
+	help
 	  This is a Video4Linux2 sensor driver for the TI TVP5146/47
 	  decoder. It is currently working with the TI OMAP3 camera
 	  controller.
@@ -410,7 +375,7 @@ config VIDEO_TVP5150
 	tristate "Texas Instruments TVP5150 video decoder"
 	depends on VIDEO_V4L2 && I2C
 	select V4L2_FWNODE
-	---help---
+	help
 	  Support for the Texas Instruments TVP5150 video decoder.
 
 	  To compile this driver as a module, choose M here: the
@@ -420,7 +385,7 @@ config VIDEO_TVP7002
 	tristate "Texas Instruments TVP7002 video decoder"
 	depends on VIDEO_V4L2 && I2C
 	select V4L2_FWNODE
-	---help---
+	help
 	  Support for the Texas Instruments TVP7002 video decoder.
 
 	  To compile this driver as a module, choose M here: the
@@ -429,7 +394,7 @@ config VIDEO_TVP7002
 config VIDEO_TW2804
 	tristate "Techwell TW2804 multiple video decoder"
 	depends on VIDEO_V4L2 && I2C
-	---help---
+	help
 	  Support for the Techwell tw2804 multiple video decoder.
 
 	  To compile this driver as a module, choose M here: the
@@ -438,7 +403,7 @@ config VIDEO_TW2804
 config VIDEO_TW9903
 	tristate "Techwell TW9903 video decoder"
 	depends on VIDEO_V4L2 && I2C
-	---help---
+	help
 	  Support for the Techwell tw9903 multi-standard video decoder
 	  with high quality down scaler.
 
@@ -448,7 +413,7 @@ config VIDEO_TW9903
 config VIDEO_TW9906
 	tristate "Techwell TW9906 video decoder"
 	depends on VIDEO_V4L2 && I2C
-	---help---
+	help
 	  Support for the Techwell tw9906 enhanced multi-standard comb filter
 	  video decoder with YCbCr input support.
 
@@ -458,7 +423,7 @@ config VIDEO_TW9906
 config VIDEO_TW9910
 	tristate "Techwell TW9910 video decoder"
 	depends on VIDEO_V4L2 && I2C
-	---help---
+	help
 	  Support for Techwell TW9910 NTSC/PAL/SECAM video decoder.
 
 	  To compile this driver as a module, choose M here: the
@@ -467,7 +432,7 @@ config VIDEO_TW9910
 config VIDEO_VPX3220
 	tristate "vpx3220a, vpx3216b & vpx3214c video decoders"
 	depends on VIDEO_V4L2 && I2C
-	---help---
+	help
 	  Support for VPX322x video decoders.
 
 	  To compile this driver as a module, choose M here: the
@@ -478,7 +443,7 @@ comment "Video and audio decoders"
 config VIDEO_SAA717X
 	tristate "Philips SAA7171/3/4 audio/video decoders"
 	depends on VIDEO_V4L2 && I2C
-	---help---
+	help
 	  Support for the Philips SAA7171/3/4 audio/video decoders.
 
 	  To compile this driver as a module, choose M here: the
@@ -491,7 +456,7 @@ comment "Video encoders"
 config VIDEO_SAA7127
 	tristate "Philips SAA7127/9 digital video encoders"
 	depends on VIDEO_V4L2 && I2C
-	---help---
+	help
 	  Support for the Philips SAA7127/9 digital video encoders.
 
 	  To compile this driver as a module, choose M here: the
@@ -500,7 +465,7 @@ config VIDEO_SAA7127
 config VIDEO_SAA7185
 	tristate "Philips SAA7185 video encoder"
 	depends on VIDEO_V4L2 && I2C
-	---help---
+	help
 	  Support for the Philips SAA7185 video encoder.
 
 	  To compile this driver as a module, choose M here: the
@@ -509,7 +474,7 @@ config VIDEO_SAA7185
 config VIDEO_ADV7170
 	tristate "Analog Devices ADV7170 video encoder"
 	depends on VIDEO_V4L2 && I2C
-	---help---
+	help
 	  Support for the Analog Devices ADV7170 video encoder driver
 
 	  To compile this driver as a module, choose M here: the
@@ -518,7 +483,7 @@ config VIDEO_ADV7170
 config VIDEO_ADV7175
 	tristate "Analog Devices ADV7175 video encoder"
 	depends on VIDEO_V4L2 && I2C
-	---help---
+	help
 	  Support for the Analog Devices ADV7175 video encoder driver
 
 	  To compile this driver as a module, choose M here: the
@@ -546,7 +511,7 @@ config VIDEO_ADV7511
 	tristate "Analog Devices ADV7511 encoder"
 	depends on VIDEO_V4L2 && I2C && VIDEO_V4L2_SUBDEV_API
 	select HDMI
-	---help---
+	help
 	  Support for the Analog Devices ADV7511 video encoder.
 
 	  This is a Analog Devices HDMI transmitter.
@@ -558,14 +523,14 @@ config VIDEO_ADV7511_CEC
 	bool "Enable Analog Devices ADV7511 CEC support"
 	depends on VIDEO_ADV7511
 	select CEC_CORE
-	---help---
+	help
 	  When selected the adv7511 will support the optional
 	  HDMI CEC feature.
 
 config VIDEO_AD9389B
 	tristate "Analog Devices AD9389B encoder"
 	depends on VIDEO_V4L2 && I2C && VIDEO_V4L2_SUBDEV_API
-	---help---
+	help
 	  Support for the Analog Devices AD9389B video encoder.
 
 	  This is a Analog Devices HDMI transmitter.
@@ -582,7 +547,7 @@ config VIDEO_AK881X
 config VIDEO_THS8200
 	tristate "Texas Instruments THS8200 video encoder"
 	depends on VIDEO_V4L2 && I2C
-	---help---
+	help
 	  Support for the Texas Instruments THS8200 video encoder.
 
 	  To compile this driver as a module, choose M here: the
@@ -612,7 +577,7 @@ config VIDEO_IMX258
 	tristate "Sony IMX258 sensor support"
 	depends on I2C && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
 	depends on MEDIA_CAMERA_SUPPORT
-	---help---
+	help
 	  This is a Video4Linux2 sensor driver for the Sony
 	  IMX258 camera.
 
@@ -624,7 +589,7 @@ config VIDEO_IMX274
 	depends on I2C && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
 	depends on MEDIA_CAMERA_SUPPORT
 	select REGMAP_I2C
-	---help---
+	help
 	  This is a V4L2 sensor driver for the Sony IMX274
 	  CMOS image sensor.
 
@@ -666,7 +631,7 @@ config VIDEO_OV2659
 	depends on VIDEO_V4L2 && I2C
 	depends on MEDIA_CAMERA_SUPPORT
 	select V4L2_FWNODE
-	---help---
+	help
 	  This is a Video4Linux2 sensor driver for the OmniVision
 	  OV2659 camera.
 
@@ -678,7 +643,7 @@ config VIDEO_OV2680
 	depends on VIDEO_V4L2 && I2C && MEDIA_CONTROLLER
 	depends on MEDIA_CAMERA_SUPPORT
 	select V4L2_FWNODE
-	---help---
+	help
 	  This is a Video4Linux2 sensor driver for the OmniVision
 	  OV2680 camera.
 
@@ -690,7 +655,7 @@ config VIDEO_OV2685
 	depends on VIDEO_V4L2 && I2C && MEDIA_CONTROLLER
 	depends on MEDIA_CAMERA_SUPPORT
 	select V4L2_FWNODE
-	---help---
+	help
 	  This is a Video4Linux2 sensor driver for the OmniVision
 	  OV2685 camera.
 
@@ -703,7 +668,7 @@ config VIDEO_OV5640
 	depends on GPIOLIB && VIDEO_V4L2 && I2C && VIDEO_V4L2_SUBDEV_API
 	depends on MEDIA_CAMERA_SUPPORT
 	select V4L2_FWNODE
-	---help---
+	help
 	  This is a Video4Linux2 sensor driver for the Omnivision
 	  OV5640 camera sensor with a MIPI CSI-2 interface.
 
@@ -713,7 +678,7 @@ config VIDEO_OV5645
 	depends on I2C && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
 	depends on MEDIA_CAMERA_SUPPORT
 	select V4L2_FWNODE
-	---help---
+	help
 	  This is a Video4Linux2 sensor driver for the OmniVision
 	  OV5645 camera.
 
@@ -725,7 +690,7 @@ config VIDEO_OV5647
 	depends on I2C && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
 	depends on MEDIA_CAMERA_SUPPORT
 	select V4L2_FWNODE
-	---help---
+	help
 	  This is a Video4Linux2 sensor driver for the OmniVision
 	  OV5647 camera.
 
@@ -736,7 +701,7 @@ config VIDEO_OV6650
 	tristate "OmniVision OV6650 sensor support"
 	depends on I2C && VIDEO_V4L2
 	depends on MEDIA_CAMERA_SUPPORT
-	---help---
+	help
 	  This is a Video4Linux2 sensor driver for the OmniVision
 	  OV6650 camera.
 
@@ -749,7 +714,7 @@ config VIDEO_OV5670
 	depends on MEDIA_CAMERA_SUPPORT
 	depends on MEDIA_CONTROLLER
 	select V4L2_FWNODE
-	---help---
+	help
 	  This is a Video4Linux2 sensor driver for the OmniVision
 	  OV5670 camera.
 
@@ -760,7 +725,7 @@ config VIDEO_OV5695
 	tristate "OmniVision OV5695 sensor support"
 	depends on I2C && VIDEO_V4L2
 	depends on MEDIA_CAMERA_SUPPORT
-	---help---
+	help
 	  This is a Video4Linux2 sensor driver for the OmniVision
 	  OV5695 camera.
 
@@ -784,7 +749,7 @@ config VIDEO_OV772X
 	depends on I2C && VIDEO_V4L2
 	depends on MEDIA_CAMERA_SUPPORT
 	select REGMAP_SCCB
-	---help---
+	help
 	  This is a Video4Linux2 sensor driver for the OmniVision
 	  OV772x camera.
 
@@ -795,7 +760,7 @@ config VIDEO_OV7640
 	tristate "OmniVision OV7640 sensor support"
 	depends on I2C && VIDEO_V4L2
 	depends on MEDIA_CAMERA_SUPPORT
-	---help---
+	help
 	  This is a Video4Linux2 sensor driver for the OmniVision
 	  OV7640 camera.
 
@@ -807,7 +772,7 @@ config VIDEO_OV7670
 	depends on I2C && VIDEO_V4L2
 	depends on MEDIA_CAMERA_SUPPORT
 	select V4L2_FWNODE
-	---help---
+	help
 	  This is a Video4Linux2 sensor driver for the OmniVision
 	  OV7670 VGA camera.  It currently only works with the M88ALP01
 	  controller.
@@ -816,7 +781,7 @@ config VIDEO_OV7740
 	tristate "OmniVision OV7740 sensor support"
 	depends on I2C && VIDEO_V4L2
 	depends on MEDIA_CAMERA_SUPPORT
-	---help---
+	help
 	  This is a Video4Linux2 sensor driver for the OmniVision
 	  OV7740 VGA camera sensor.
 
@@ -843,7 +808,7 @@ config VIDEO_OV9650
 	tristate "OmniVision OV9650/OV9652 sensor support"
 	depends on I2C && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
 	select REGMAP_SCCB
-	---help---
+	help
 	  This is a V4L2 sensor driver for the Omnivision
 	  OV9650 and OV9652 camera sensors.
 
@@ -852,7 +817,7 @@ config VIDEO_OV13858
 	depends on I2C && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
 	depends on MEDIA_CAMERA_SUPPORT
 	select V4L2_FWNODE
-	---help---
+	help
 	  This is a Video4Linux2 sensor driver for the OmniVision
 	  OV13858 camera.
 
@@ -860,7 +825,7 @@ config VIDEO_VS6624
 	tristate "ST VS6624 sensor support"
 	depends on VIDEO_V4L2 && I2C
 	depends on MEDIA_CAMERA_SUPPORT
-	---help---
+	help
 	  This is a Video4Linux2 sensor driver for the ST VS6624
 	  camera.
 
@@ -880,7 +845,7 @@ config VIDEO_MT9M032
 	depends on I2C && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
 	depends on MEDIA_CAMERA_SUPPORT
 	select VIDEO_APTINA_PLL
-	---help---
+	help
 	  This driver supports MT9M032 camera sensors from Aptina, monochrome
 	  models only.
 
@@ -897,7 +862,7 @@ config VIDEO_MT9P031
 	depends on I2C && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
 	depends on MEDIA_CAMERA_SUPPORT
 	select VIDEO_APTINA_PLL
-	---help---
+	help
 	  This is a Video4Linux2 sensor driver for the Aptina
 	  (Micron) mt9p031 5 Mpixel camera.
 
@@ -905,7 +870,7 @@ config VIDEO_MT9T001
 	tristate "Aptina MT9T001 support"
 	depends on I2C && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
 	depends on MEDIA_CAMERA_SUPPORT
-	---help---
+	help
 	  This is a Video4Linux2 sensor driver for the Aptina
 	  (Micron) mt0t001 3 Mpixel camera.
 
@@ -913,7 +878,7 @@ config VIDEO_MT9T112
 	tristate "Aptina MT9T111/MT9T112 support"
 	depends on I2C && VIDEO_V4L2
 	depends on MEDIA_CAMERA_SUPPORT
-	---help---
+	help
 	  This is a Video4Linux2 sensor driver for the Aptina
 	  (Micron) MT9T111 and MT9T112 3 Mpixel camera.
 
@@ -924,7 +889,7 @@ config VIDEO_MT9V011
 	tristate "Micron mt9v011 sensor support"
 	depends on I2C && VIDEO_V4L2
 	depends on MEDIA_CAMERA_SUPPORT
-	---help---
+	help
 	  This is a Video4Linux2 sensor driver for the Micron
 	  mt0v011 1.3 Mpixel camera.  It currently only works with the
 	  em28xx driver.
@@ -935,7 +900,7 @@ config VIDEO_MT9V032
 	depends on MEDIA_CAMERA_SUPPORT
 	select REGMAP_I2C
 	select V4L2_FWNODE
-	---help---
+	help
 	  This is a Video4Linux2 sensor driver for the Micron
 	  MT9V032 752x480 CMOS sensor.
 
@@ -954,14 +919,14 @@ config VIDEO_SR030PC30
 	tristate "Siliconfile SR030PC30 sensor support"
 	depends on I2C && VIDEO_V4L2
 	depends on MEDIA_CAMERA_SUPPORT
-	---help---
+	help
 	  This driver supports SR030PC30 VGA camera from Siliconfile
 
 config VIDEO_NOON010PC30
 	tristate "Siliconfile NOON010PC30 sensor support"
 	depends on I2C && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
 	depends on MEDIA_CAMERA_SUPPORT
-	---help---
+	help
 	  This driver supports NOON010PC30 CIF camera from Siliconfile
 
 source "drivers/media/i2c/m5mols/Kconfig"
@@ -981,7 +946,7 @@ config VIDEO_S5K6AA
 	tristate "Samsung S5K6AAFX sensor support"
 	depends on MEDIA_CAMERA_SUPPORT
 	depends on I2C && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
-	---help---
+	help
 	  This is a V4L2 sensor driver for Samsung S5K6AA(FX) 1.3M
 	  camera sensor with an embedded SoC image signal processor.
 
@@ -989,7 +954,7 @@ config VIDEO_S5K6A3
 	tristate "Samsung S5K6A3 sensor support"
 	depends on MEDIA_CAMERA_SUPPORT
 	depends on I2C && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
-	---help---
+	help
 	  This is a V4L2 sensor driver for Samsung S5K6A3 raw
 	  camera sensor.
 
@@ -997,7 +962,7 @@ config VIDEO_S5K4ECGX
 	tristate "Samsung S5K4ECGX sensor support"
 	depends on I2C && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
 	select CRC32
-	---help---
+	help
 	  This is a V4L2 sensor driver for Samsung S5K4ECGX 5M
 	  camera sensor with an embedded SoC image signal processor.
 
@@ -1005,7 +970,7 @@ config VIDEO_S5K5BAF
 	tristate "Samsung S5K5BAF sensor support"
 	depends on I2C && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
 	select V4L2_FWNODE
-	---help---
+	help
 	  This is a V4L2 sensor driver for Samsung S5K5BAF 2M
 	  camera sensor with an embedded SoC image signal processor.
 
@@ -1016,17 +981,56 @@ config VIDEO_S5C73M3
 	tristate "Samsung S5C73M3 sensor support"
 	depends on I2C && SPI && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
 	select V4L2_FWNODE
-	---help---
+	help
 	  This is a V4L2 sensor driver for Samsung S5C73M3
 	  8 Mpixel camera.
 
+comment "Lens drivers"
+
+config VIDEO_AD5820
+	tristate "AD5820 lens voice coil support"
+	depends on I2C && VIDEO_V4L2 && MEDIA_CONTROLLER
+	help
+	  This is a driver for the AD5820 camera lens voice coil.
+	  It is used for example in Nokia N900 (RX-51).
+
+config VIDEO_AK7375
+	tristate "AK7375 lens voice coil support"
+	depends on I2C && VIDEO_V4L2 && MEDIA_CONTROLLER
+	depends on VIDEO_V4L2_SUBDEV_API
+	help
+	  This is a driver for the AK7375 camera lens voice coil.
+	  AK7375 is a 12 bit DAC with 120mA output current sink
+	  capability. This is designed for linear control of
+	  voice coil motors, controlled via I2C serial interface.
+
+config VIDEO_DW9714
+	tristate "DW9714 lens voice coil support"
+	depends on I2C && VIDEO_V4L2 && MEDIA_CONTROLLER
+	depends on VIDEO_V4L2_SUBDEV_API
+	help
+	  This is a driver for the DW9714 camera lens voice coil.
+	  DW9714 is a 10 bit DAC with 120mA output current sink
+	  capability. This is designed for linear control of
+	  voice coil motors, controlled via I2C serial interface.
+
+config VIDEO_DW9807_VCM
+	tristate "DW9807 lens voice coil support"
+	depends on I2C && VIDEO_V4L2 && MEDIA_CONTROLLER
+	depends on VIDEO_V4L2_SUBDEV_API
+	help
+	  This is a driver for the DW9807 camera lens voice coil.
+	  DW9807 is a 10 bit DAC with 100mA output current sink
+	  capability. This is designed for linear control of
+	  voice coil motors, controlled via I2C serial interface.
+
 comment "Flash devices"
 
 config VIDEO_ADP1653
 	tristate "ADP1653 flash support"
 	depends on I2C && VIDEO_V4L2 && MEDIA_CONTROLLER
 	depends on MEDIA_CAMERA_SUPPORT
-	---help---
+	help
 	  This is a driver for the ADP1653 flash controller. It is used for
 	  example in Nokia N900.
 
@@ -1035,7 +1039,7 @@ config VIDEO_LM3560
 	depends on I2C && VIDEO_V4L2 && MEDIA_CONTROLLER
 	depends on MEDIA_CAMERA_SUPPORT
 	select REGMAP_I2C
-	---help---
+	help
 	  This is a driver for the lm3560 dual flash controllers. It controls
 	  flash, torch LEDs.
 
@@ -1044,7 +1048,7 @@ config VIDEO_LM3646
 	depends on I2C && VIDEO_V4L2 && MEDIA_CONTROLLER
 	depends on MEDIA_CAMERA_SUPPORT
 	select REGMAP_I2C
-	---help---
+	help
 	  This is a driver for the lm3646 dual flash controllers. It controls
 	  flash, torch LEDs.
 
@@ -1053,7 +1057,7 @@ comment "Video improvement chips"
 config VIDEO_UPD64031A
 	tristate "NEC Electronics uPD64031A Ghost Reduction"
 	depends on VIDEO_V4L2 && I2C
-	---help---
+	help
 	  Support for the NEC Electronics uPD64031A Ghost Reduction
 	  video chip. It is most often found in NTSC TV cards made for
 	  Japan and is used to reduce the 'ghosting' effect that can
@@ -1065,7 +1069,7 @@ config VIDEO_UPD64031A
 config VIDEO_UPD64083
 	tristate "NEC Electronics uPD64083 3-Dimensional Y/C separation"
 	depends on VIDEO_V4L2 && I2C
-	---help---
+	help
 	  Support for the NEC Electronics uPD64083 3-Dimensional Y/C
 	  separation video chip. It is used to improve the quality of
 	  the colors of a composite signal.
@@ -1079,7 +1083,7 @@ config VIDEO_SAA6752HS
 	tristate "Philips SAA6752HS MPEG-2 Audio/Video Encoder"
 	depends on VIDEO_V4L2 && I2C
 	select CRC32
-	---help---
+	help
 	  Support for the Philips SAA6752HS MPEG-2 video and MPEG-audio/AC-3
 	  audio encoder with multiplexer.
 
@@ -1091,7 +1095,7 @@ comment "SDR tuner chips"
 config SDR_MAX2175
 	tristate "Maxim 2175 RF to Bits tuner"
 	depends on VIDEO_V4L2 && MEDIA_SDR_SUPPORT && I2C
-	---help---
+	help
 	  Support for Maxim 2175 tuner. It is an advanced analog/digital
 	  radio receiver with RF-to-Bits front-end designed for SDR solutions.
 
@@ -1112,7 +1116,7 @@ config VIDEO_THS7303
 config VIDEO_M52790
 	tristate "Mitsubishi M52790 A/V switch"
 	depends on VIDEO_V4L2 && I2C
-	---help---
+	help
 	 Support for the Mitsubishi M52790 A/V switch.
 
 	 To compile this driver as a module, choose M here: the
@@ -1123,7 +1127,7 @@ config VIDEO_I2C
 	depends on VIDEO_V4L2 && I2C
 	select VIDEOBUF2_VMALLOC
 	imply HWMON
-	---help---
+	help
 	  Enable the I2C transport video support which supports the
 	  following:
 	   * Panasonic AMG88xx Grid-Eye Sensors
@@ -1132,6 +1136,19 @@ config VIDEO_I2C
 	  To compile this driver as a module, choose M here: the
 	  module will be called video-i2c
 
+config VIDEO_ST_MIPID02
+	tristate "STMicroelectronics MIPID02 CSI-2 to PARALLEL bridge"
+	depends on I2C && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
+	depends on MEDIA_CAMERA_SUPPORT
+	select V4L2_FWNODE
+	help
+	  Support for STMicroelectronics MIPID02 CSI-2 to PARALLEL bridge.
+	  It is used to allow usage of CSI-2 sensor with PARALLEL port
+	  controller.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called st-mipid02.
+
 endmenu
 
 endif
diff --git a/drivers/media/i2c/Makefile b/drivers/media/i2c/Makefile
index a64fca82e0c4..d8ad9dad495d 100644
--- a/drivers/media/i2c/Makefile
+++ b/drivers/media/i2c/Makefile
@@ -113,5 +113,6 @@ obj-$(CONFIG_VIDEO_IMX258)	+= imx258.o
 obj-$(CONFIG_VIDEO_IMX274)	+= imx274.o
 obj-$(CONFIG_VIDEO_IMX319)	+= imx319.o
 obj-$(CONFIG_VIDEO_IMX355)	+= imx355.o
+obj-$(CONFIG_VIDEO_ST_MIPID02) += st-mipid02.o
 
 obj-$(CONFIG_SDR_MAX2175) += max2175.o
diff --git a/drivers/media/i2c/cx25840/Kconfig b/drivers/media/i2c/cx25840/Kconfig
index 451133ad41ff..f4b31d7cb440 100644
--- a/drivers/media/i2c/cx25840/Kconfig
+++ b/drivers/media/i2c/cx25840/Kconfig
@@ -1,7 +1,7 @@
 config VIDEO_CX25840
 	tristate "Conexant CX2584x audio/video decoders"
 	depends on VIDEO_V4L2 && I2C
-	---help---
+	help
 	  Support for the Conexant CX2584x audio/video decoders.
 
 	  To compile this driver as a module, choose M here: the
diff --git a/drivers/media/i2c/et8ek8/Kconfig b/drivers/media/i2c/et8ek8/Kconfig
index 9fe409e95666..ab23b41bf353 100644
--- a/drivers/media/i2c/et8ek8/Kconfig
+++ b/drivers/media/i2c/et8ek8/Kconfig
@@ -2,6 +2,6 @@ config VIDEO_ET8EK8
 	tristate "ET8EK8 camera sensor support"
 	depends on I2C && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
 	select V4L2_FWNODE
-	---help---
+	help
 	  This is a driver for the Toshiba ET8EK8 5 MP camera sensor.
 	  It is used for example in Nokia N900 (RX-51).
diff --git a/drivers/media/i2c/imx214.c b/drivers/media/i2c/imx214.c
index 9857e151db46..83e9961b0505 100644
--- a/drivers/media/i2c/imx214.c
+++ b/drivers/media/i2c/imx214.c
@@ -588,12 +588,10 @@ static int imx214_set_format(struct v4l2_subdev *sd,
 
 	__crop = __imx214_get_pad_crop(imx214, cfg, format->pad, format->which);
 
-	if (format)
-		mode = v4l2_find_nearest_size(imx214_modes,
-				ARRAY_SIZE(imx214_modes), width, height,
-				format->format.width, format->format.height);
-	else
-		mode = &imx214_modes[0];
+	mode = v4l2_find_nearest_size(imx214_modes,
+				      ARRAY_SIZE(imx214_modes), width, height,
+				      format->format.width,
+				      format->format.height);
 
 	__crop->width = mode->width;
 	__crop->height = mode->height;
diff --git a/drivers/media/i2c/m5mols/Kconfig b/drivers/media/i2c/m5mols/Kconfig
index dc8c2505907e..be0bb3f1bc22 100644
--- a/drivers/media/i2c/m5mols/Kconfig
+++ b/drivers/media/i2c/m5mols/Kconfig
@@ -2,5 +2,5 @@ config VIDEO_M5MOLS
 	tristate "Fujitsu M-5MOLS 8MP sensor support"
 	depends on I2C && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
 	depends on MEDIA_CAMERA_SUPPORT
-	---help---
+	help
 	  This driver supports Fujitsu M-5MOLS camera sensor with ISP
diff --git a/drivers/media/i2c/ov2659.c b/drivers/media/i2c/ov2659.c
index 799acce803fe..5ed2413eac8a 100644
--- a/drivers/media/i2c/ov2659.c
+++ b/drivers/media/i2c/ov2659.c
@@ -1117,8 +1117,10 @@ static int ov2659_set_fmt(struct v4l2_subdev *sd,
 		if (ov2659_formats[index].code == mf->code)
 			break;
 
-	if (index < 0)
-		return -EINVAL;
+	if (index < 0) {
+		index = 0;
+		mf->code = ov2659_formats[index].code;
+	}
 
 	mf->colorspace = V4L2_COLORSPACE_SRGB;
 	mf->field = V4L2_FIELD_NONE;
@@ -1130,7 +1132,7 @@ static int ov2659_set_fmt(struct v4l2_subdev *sd,
 		mf = v4l2_subdev_get_try_format(sd, cfg, fmt->pad);
 		*mf = fmt->format;
 #else
-		return -ENOTTY;
+		ret = -ENOTTY;
 #endif
 	} else {
 		s64 val;
diff --git a/drivers/media/i2c/ov6650.c b/drivers/media/i2c/ov6650.c
index c33fd584cb44..1b972e591b48 100644
--- a/drivers/media/i2c/ov6650.c
+++ b/drivers/media/i2c/ov6650.c
@@ -804,15 +804,25 @@ static int ov6650_prog_dflt(struct i2c_client *client)
 	return ret;
 }
 
-static int ov6650_video_probe(struct i2c_client *client)
+static int ov6650_video_probe(struct v4l2_subdev *sd)
 {
+	struct i2c_client *client = v4l2_get_subdevdata(sd);
 	struct ov6650 *priv = to_ov6650(client);
 	u8		pidh, pidl, midh, midl;
 	int		ret;
 
-	ret = ov6650_s_power(&priv->subdev, 1);
-	if (ret < 0)
+	priv->clk = v4l2_clk_get(&client->dev, NULL);
+	if (IS_ERR(priv->clk)) {
+		ret = PTR_ERR(priv->clk);
+		dev_err(&client->dev, "v4l2_clk request err: %d\n", ret);
 		return ret;
+	}
+
+	ret = ov6650_s_power(sd, 1);
+	if (ret < 0)
+		goto eclkput;
+
+	msleep(20);
 
 	/*
 	 * check and show product ID and manufacturer ID
@@ -846,7 +856,12 @@ static int ov6650_video_probe(struct i2c_client *client)
 		ret = v4l2_ctrl_handler_setup(&priv->hdl);
 
 done:
-	ov6650_s_power(&priv->subdev, 0);
+	ov6650_s_power(sd, 0);
+	if (!ret)
+		return 0;
+eclkput:
+	v4l2_clk_put(priv->clk);
+
 	return ret;
 }
 
@@ -929,6 +944,10 @@ static const struct v4l2_subdev_ops ov6650_subdev_ops = {
 	.pad	= &ov6650_pad_ops,
 };
 
+static const struct v4l2_subdev_internal_ops ov6650_internal_ops = {
+	.registered = ov6650_video_probe,
+};
+
 /*
  * i2c_driver function
  */
@@ -989,18 +1008,12 @@ static int ov6650_probe(struct i2c_client *client,
 	priv->code	  = MEDIA_BUS_FMT_YUYV8_2X8;
 	priv->colorspace  = V4L2_COLORSPACE_JPEG;
 
-	priv->clk = v4l2_clk_get(&client->dev, NULL);
-	if (IS_ERR(priv->clk)) {
-		ret = PTR_ERR(priv->clk);
-		goto eclkget;
-	}
+	priv->subdev.internal_ops = &ov6650_internal_ops;
+	priv->subdev.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
 
-	ret = ov6650_video_probe(client);
-	if (ret) {
-		v4l2_clk_put(priv->clk);
-eclkget:
+	ret = v4l2_async_register_subdev(&priv->subdev);
+	if (ret)
 		v4l2_ctrl_handler_free(&priv->hdl);
-	}
 
 	return ret;
 }
@@ -1010,7 +1023,7 @@ static int ov6650_remove(struct i2c_client *client)
 	struct ov6650 *priv = to_ov6650(client);
 
 	v4l2_clk_put(priv->clk);
-	v4l2_device_unregister_subdev(&priv->subdev);
+	v4l2_async_unregister_subdev(&priv->subdev);
 	v4l2_ctrl_handler_free(&priv->hdl);
 	return 0;
 }
diff --git a/drivers/media/i2c/ov7670.c b/drivers/media/i2c/ov7670.c
index a7d26b294eb5..44c3eed8a858 100644
--- a/drivers/media/i2c/ov7670.c
+++ b/drivers/media/i2c/ov7670.c
@@ -864,7 +864,15 @@ static int ov7675_set_framerate(struct v4l2_subdev *sd,
 	/* Recalculate frame rate */
 	ov7675_get_framerate(sd, tpf);
 
-	return ov7675_apply_framerate(sd);
+	/*
+	 * If the device is not powered up by the host driver do
+	 * not apply any changes to H/W at this time. Instead
+	 * the framerate will be restored right after power-up.
+	 */
+	if (info->on)
+		return ov7675_apply_framerate(sd);
+
+	return 0;
 }
 
 static void ov7670_get_framerate_legacy(struct v4l2_subdev *sd,
@@ -895,7 +903,16 @@ static int ov7670_set_framerate_legacy(struct v4l2_subdev *sd,
 	info->clkrc = (info->clkrc & 0x80) | div;
 	tpf->numerator = 1;
 	tpf->denominator = info->clock_speed / div;
-	return ov7670_write(sd, REG_CLKRC, info->clkrc);
+
+	/*
+	 * If the device is not powered up by the host driver do
+	 * not apply any changes to H/W at this time. Instead
+	 * the framerate will be restored right after power-up.
+	 */
+	if (info->on)
+		return ov7670_write(sd, REG_CLKRC, info->clkrc);
+
+	return 0;
 }
 
 /*
@@ -1105,9 +1122,13 @@ static int ov7670_set_fmt(struct v4l2_subdev *sd,
 	if (ret)
 		return ret;
 
-	ret = ov7670_apply_fmt(sd);
-	if (ret)
-		return ret;
+	/*
+	 * If the device is not powered up by the host driver do
+	 * not apply any changes to H/W at this time. Instead
+	 * the frame format will be restored right after power-up.
+	 */
+	if (info->on)
+		return ov7670_apply_fmt(sd);
 
 	return 0;
 }
@@ -1664,6 +1685,7 @@ static int ov7670_s_power(struct v4l2_subdev *sd, int on)
 
 	if (on) {
 		ov7670_power_on (sd);
+		ov7670_init(sd, 0);
 		ov7670_apply_fmt(sd);
 		ov7675_apply_framerate(sd);
 		v4l2_ctrl_handler_setup(&info->hdl);
diff --git a/drivers/media/i2c/ov7740.c b/drivers/media/i2c/ov7740.c
index dfece91ce96b..54e80a60aa57 100644
--- a/drivers/media/i2c/ov7740.c
+++ b/drivers/media/i2c/ov7740.c
@@ -448,6 +448,27 @@ static int ov7740_get_gain(struct ov7740 *ov7740, struct v4l2_ctrl *ctrl)
 	return 0;
 }
 
+static int ov7740_get_exp(struct ov7740 *ov7740, struct v4l2_ctrl *ctrl)
+{
+	struct regmap *regmap = ov7740->regmap;
+	unsigned int value0, value1;
+	int ret;
+
+	if (ctrl->val == V4L2_EXPOSURE_MANUAL)
+		return 0;
+
+	ret = regmap_read(regmap, REG_AEC, &value0);
+	if (ret)
+		return ret;
+	ret = regmap_read(regmap, REG_HAEC, &value1);
+	if (ret)
+		return ret;
+
+	ov7740->exposure->val = (value1 << 8) | (value0 & 0xff);
+
+	return 0;
+}
+
 static int ov7740_set_exp(struct regmap *regmap, int value)
 {
 	int ret;
@@ -494,6 +515,9 @@ static int ov7740_get_volatile_ctrl(struct v4l2_ctrl *ctrl)
 	case V4L2_CID_AUTOGAIN:
 		ret = ov7740_get_gain(ov7740, ctrl);
 		break;
+	case V4L2_CID_EXPOSURE_AUTO:
+		ret = ov7740_get_exp(ov7740, ctrl);
+		break;
 	default:
 		ret = -EINVAL;
 		break;
@@ -991,8 +1015,6 @@ static int ov7740_init_controls(struct ov7740 *ov7740)
 
 	ov7740->exposure = v4l2_ctrl_new_std(ctrl_hdlr, &ov7740_ctrl_ops,
 					   V4L2_CID_EXPOSURE, 0, 65535, 1, 500);
-	if (ov7740->exposure)
-		ov7740->exposure->flags |= V4L2_CTRL_FLAG_VOLATILE;
 
 	ov7740->auto_exposure = v4l2_ctrl_new_std_menu(ctrl_hdlr,
 					&ov7740_ctrl_ops,
@@ -1003,7 +1025,7 @@ static int ov7740_init_controls(struct ov7740 *ov7740)
 	v4l2_ctrl_auto_cluster(3, &ov7740->auto_wb, 0, false);
 	v4l2_ctrl_auto_cluster(2, &ov7740->auto_gain, 0, true);
 	v4l2_ctrl_auto_cluster(2, &ov7740->auto_exposure,
-			       V4L2_EXPOSURE_MANUAL, false);
+			       V4L2_EXPOSURE_MANUAL, true);
 	v4l2_ctrl_cluster(2, &ov7740->hflip);
 
 	if (ctrl_hdlr->error) {
diff --git a/drivers/media/i2c/smiapp/Kconfig b/drivers/media/i2c/smiapp/Kconfig
index f59718d8e51e..26b54f2aa95b 100644
--- a/drivers/media/i2c/smiapp/Kconfig
+++ b/drivers/media/i2c/smiapp/Kconfig
@@ -4,5 +4,5 @@ config VIDEO_SMIAPP
 	depends on MEDIA_CAMERA_SUPPORT
 	select VIDEO_SMIAPP_PLL
 	select V4L2_FWNODE
-	---help---
+	help
 	  This is a generic driver for SMIA++/SMIA camera modules.
diff --git a/drivers/media/i2c/st-mipid02.c b/drivers/media/i2c/st-mipid02.c
new file mode 100644
index 000000000000..9369f38dbf3d
--- /dev/null
+++ b/drivers/media/i2c/st-mipid02.c
@@ -0,0 +1,1033 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Driver for ST MIPID02 CSI-2 to PARALLEL bridge
+ *
+ * Copyright (C) STMicroelectronics SA 2019
+ * Authors: Mickael Guene <mickael.guene@st.com>
+ *          for STMicroelectronics.
+ *
+ *
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/i2c.h>
+#include <linux/module.h>
+#include <linux/of_graph.h>
+#include <linux/regulator/consumer.h>
+#include <media/v4l2-async.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-fwnode.h>
+#include <media/v4l2-subdev.h>
+
+#define MIPID02_CLK_LANE_WR_REG1			0x01
+#define MIPID02_CLK_LANE_REG1				0x02
+#define MIPID02_CLK_LANE_REG3				0x04
+#define MIPID02_DATA_LANE0_REG1				0x05
+#define MIPID02_DATA_LANE0_REG2				0x06
+#define MIPID02_DATA_LANE1_REG1				0x09
+#define MIPID02_DATA_LANE1_REG2				0x0a
+#define MIPID02_MODE_REG1				0x14
+#define MIPID02_MODE_REG2				0x15
+#define MIPID02_DATA_ID_RREG				0x17
+#define MIPID02_DATA_SELECTION_CTRL			0x19
+#define MIPID02_PIX_WIDTH_CTRL				0x1e
+#define MIPID02_PIX_WIDTH_CTRL_EMB			0x1f
+
+/* Bits definition for MIPID02_CLK_LANE_REG1 */
+#define CLK_ENABLE					BIT(0)
+/* Bits definition for MIPID02_CLK_LANE_REG3 */
+#define CLK_MIPI_CSI					BIT(1)
+/* Bits definition for MIPID02_DATA_LANE0_REG1 */
+#define DATA_ENABLE					BIT(0)
+/* Bits definition for MIPID02_DATA_LANEx_REG2 */
+#define DATA_MIPI_CSI					BIT(0)
+/* Bits definition for MIPID02_MODE_REG1 */
+#define MODE_DATA_SWAP					BIT(2)
+#define MODE_NO_BYPASS					BIT(6)
+/* Bits definition for MIPID02_MODE_REG2 */
+#define MODE_HSYNC_ACTIVE_HIGH				BIT(1)
+#define MODE_VSYNC_ACTIVE_HIGH				BIT(2)
+/* Bits definition for MIPID02_DATA_SELECTION_CTRL */
+#define SELECTION_MANUAL_DATA				BIT(2)
+#define SELECTION_MANUAL_WIDTH				BIT(3)
+
+static const u32 mipid02_supported_fmt_codes[] = {
+	MEDIA_BUS_FMT_SBGGR8_1X8, MEDIA_BUS_FMT_SGBRG8_1X8,
+	MEDIA_BUS_FMT_SGRBG8_1X8, MEDIA_BUS_FMT_SRGGB8_1X8,
+	MEDIA_BUS_FMT_SBGGR10_1X10, MEDIA_BUS_FMT_SGBRG10_1X10,
+	MEDIA_BUS_FMT_SGRBG10_1X10, MEDIA_BUS_FMT_SRGGB10_1X10,
+	MEDIA_BUS_FMT_SBGGR12_1X12, MEDIA_BUS_FMT_SGBRG12_1X12,
+	MEDIA_BUS_FMT_SGRBG12_1X12, MEDIA_BUS_FMT_SRGGB12_1X12,
+	MEDIA_BUS_FMT_UYVY8_1X16, MEDIA_BUS_FMT_BGR888_1X24
+};
+
+/* regulator supplies */
+static const char * const mipid02_supply_name[] = {
+	"VDDE", /* 1.8V digital I/O supply */
+	"VDDIN", /* 1V8 voltage regulator supply */
+};
+
+#define MIPID02_NUM_SUPPLIES		ARRAY_SIZE(mipid02_supply_name)
+
+#define MIPID02_SINK_0			0
+#define MIPID02_SINK_1			1
+#define MIPID02_SOURCE			2
+#define MIPID02_PAD_NB			3
+
+struct mipid02_dev {
+	struct i2c_client *i2c_client;
+	struct regulator_bulk_data supplies[MIPID02_NUM_SUPPLIES];
+	struct v4l2_subdev sd;
+	struct media_pad pad[MIPID02_PAD_NB];
+	struct clk *xclk;
+	struct gpio_desc *reset_gpio;
+	/* endpoints info */
+	struct v4l2_fwnode_endpoint rx;
+	u64 link_frequency;
+	struct v4l2_fwnode_endpoint tx;
+	/* remote source */
+	struct v4l2_async_subdev asd;
+	struct v4l2_async_notifier notifier;
+	struct v4l2_subdev *s_subdev;
+	/* registers */
+	struct {
+		u8 clk_lane_reg1;
+		u8 data_lane0_reg1;
+		u8 data_lane1_reg1;
+		u8 mode_reg1;
+		u8 mode_reg2;
+		u8 data_id_rreg;
+		u8 pix_width_ctrl;
+		u8 pix_width_ctrl_emb;
+	} r;
+	/* lock to protect all members below */
+	struct mutex lock;
+	bool streaming;
+	struct v4l2_mbus_framefmt fmt;
+};
+
+static int bpp_from_code(__u32 code)
+{
+	switch (code) {
+	case MEDIA_BUS_FMT_SBGGR8_1X8:
+	case MEDIA_BUS_FMT_SGBRG8_1X8:
+	case MEDIA_BUS_FMT_SGRBG8_1X8:
+	case MEDIA_BUS_FMT_SRGGB8_1X8:
+		return 8;
+	case MEDIA_BUS_FMT_SBGGR10_1X10:
+	case MEDIA_BUS_FMT_SGBRG10_1X10:
+	case MEDIA_BUS_FMT_SGRBG10_1X10:
+	case MEDIA_BUS_FMT_SRGGB10_1X10:
+		return 10;
+	case MEDIA_BUS_FMT_SBGGR12_1X12:
+	case MEDIA_BUS_FMT_SGBRG12_1X12:
+	case MEDIA_BUS_FMT_SGRBG12_1X12:
+	case MEDIA_BUS_FMT_SRGGB12_1X12:
+		return 12;
+	case MEDIA_BUS_FMT_UYVY8_1X16:
+		return 16;
+	case MEDIA_BUS_FMT_BGR888_1X24:
+		return 24;
+	default:
+		return 0;
+	}
+}
+
+static u8 data_type_from_code(__u32 code)
+{
+	switch (code) {
+	case MEDIA_BUS_FMT_SBGGR8_1X8:
+	case MEDIA_BUS_FMT_SGBRG8_1X8:
+	case MEDIA_BUS_FMT_SGRBG8_1X8:
+	case MEDIA_BUS_FMT_SRGGB8_1X8:
+		return 0x2a;
+	case MEDIA_BUS_FMT_SBGGR10_1X10:
+	case MEDIA_BUS_FMT_SGBRG10_1X10:
+	case MEDIA_BUS_FMT_SGRBG10_1X10:
+	case MEDIA_BUS_FMT_SRGGB10_1X10:
+		return 0x2b;
+	case MEDIA_BUS_FMT_SBGGR12_1X12:
+	case MEDIA_BUS_FMT_SGBRG12_1X12:
+	case MEDIA_BUS_FMT_SGRBG12_1X12:
+	case MEDIA_BUS_FMT_SRGGB12_1X12:
+		return 0x2c;
+	case MEDIA_BUS_FMT_UYVY8_1X16:
+		return 0x1e;
+	case MEDIA_BUS_FMT_BGR888_1X24:
+		return 0x24;
+	default:
+		return 0;
+	}
+}
+
+static void init_format(struct v4l2_mbus_framefmt *fmt)
+{
+	fmt->code = MEDIA_BUS_FMT_SBGGR8_1X8;
+	fmt->field = V4L2_FIELD_NONE;
+	fmt->colorspace = V4L2_COLORSPACE_SRGB;
+	fmt->ycbcr_enc = V4L2_MAP_YCBCR_ENC_DEFAULT(V4L2_COLORSPACE_SRGB);
+	fmt->quantization = V4L2_QUANTIZATION_FULL_RANGE;
+	fmt->xfer_func = V4L2_MAP_XFER_FUNC_DEFAULT(V4L2_COLORSPACE_SRGB);
+	fmt->width = 640;
+	fmt->height = 480;
+}
+
+static __u32 get_fmt_code(__u32 code)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(mipid02_supported_fmt_codes); i++) {
+		if (code == mipid02_supported_fmt_codes[i])
+			return code;
+	}
+
+	return mipid02_supported_fmt_codes[0];
+}
+
+static __u32 serial_to_parallel_code(__u32 serial)
+{
+	if (serial == MEDIA_BUS_FMT_UYVY8_1X16)
+		return MEDIA_BUS_FMT_UYVY8_2X8;
+	if (serial == MEDIA_BUS_FMT_BGR888_1X24)
+		return MEDIA_BUS_FMT_BGR888_3X8;
+
+	return serial;
+}
+
+static inline struct mipid02_dev *to_mipid02_dev(struct v4l2_subdev *sd)
+{
+	return container_of(sd, struct mipid02_dev, sd);
+}
+
+static int mipid02_read_reg(struct mipid02_dev *bridge, u16 reg, u8 *val)
+{
+	struct i2c_client *client = bridge->i2c_client;
+	struct i2c_msg msg[2];
+	u8 buf[2];
+	int ret;
+
+	buf[0] = reg >> 8;
+	buf[1] = reg & 0xff;
+
+	msg[0].addr = client->addr;
+	msg[0].flags = client->flags;
+	msg[0].buf = buf;
+	msg[0].len = sizeof(buf);
+
+	msg[1].addr = client->addr;
+	msg[1].flags = client->flags | I2C_M_RD;
+	msg[1].buf = val;
+	msg[1].len = 1;
+
+	ret = i2c_transfer(client->adapter, msg, 2);
+	if (ret < 0) {
+		dev_dbg(&client->dev, "%s: %x i2c_transfer, reg: %x => %d\n",
+			    __func__, client->addr, reg, ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int mipid02_write_reg(struct mipid02_dev *bridge, u16 reg, u8 val)
+{
+	struct i2c_client *client = bridge->i2c_client;
+	struct i2c_msg msg;
+	u8 buf[3];
+	int ret;
+
+	buf[0] = reg >> 8;
+	buf[1] = reg & 0xff;
+	buf[2] = val;
+
+	msg.addr = client->addr;
+	msg.flags = client->flags;
+	msg.buf = buf;
+	msg.len = sizeof(buf);
+
+	ret = i2c_transfer(client->adapter, &msg, 1);
+	if (ret < 0) {
+		dev_dbg(&client->dev, "%s: i2c_transfer, reg: %x => %d\n",
+			    __func__, reg, ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int mipid02_get_regulators(struct mipid02_dev *bridge)
+{
+	unsigned int i;
+
+	for (i = 0; i < MIPID02_NUM_SUPPLIES; i++)
+		bridge->supplies[i].supply = mipid02_supply_name[i];
+
+	return devm_regulator_bulk_get(&bridge->i2c_client->dev,
+				       MIPID02_NUM_SUPPLIES,
+				       bridge->supplies);
+}
+
+static void mipid02_apply_reset(struct mipid02_dev *bridge)
+{
+	gpiod_set_value_cansleep(bridge->reset_gpio, 0);
+	usleep_range(5000, 10000);
+	gpiod_set_value_cansleep(bridge->reset_gpio, 1);
+	usleep_range(5000, 10000);
+	gpiod_set_value_cansleep(bridge->reset_gpio, 0);
+	usleep_range(5000, 10000);
+}
+
+static int mipid02_set_power_on(struct mipid02_dev *bridge)
+{
+	struct i2c_client *client = bridge->i2c_client;
+	int ret;
+
+	ret = clk_prepare_enable(bridge->xclk);
+	if (ret) {
+		dev_err(&client->dev, "%s: failed to enable clock\n", __func__);
+		return ret;
+	}
+
+	ret = regulator_bulk_enable(MIPID02_NUM_SUPPLIES,
+				    bridge->supplies);
+	if (ret) {
+		dev_err(&client->dev, "%s: failed to enable regulators\n",
+			    __func__);
+		goto xclk_off;
+	}
+
+	if (bridge->reset_gpio) {
+		dev_dbg(&client->dev, "apply reset");
+		mipid02_apply_reset(bridge);
+	} else {
+		dev_dbg(&client->dev, "don't apply reset");
+		usleep_range(5000, 10000);
+	}
+
+	return 0;
+
+xclk_off:
+	clk_disable_unprepare(bridge->xclk);
+	return ret;
+}
+
+static void mipid02_set_power_off(struct mipid02_dev *bridge)
+{
+	regulator_bulk_disable(MIPID02_NUM_SUPPLIES, bridge->supplies);
+	clk_disable_unprepare(bridge->xclk);
+}
+
+static int mipid02_detect(struct mipid02_dev *bridge)
+{
+	u8 reg;
+
+	/*
+	 * There is no version registers. Just try to read register
+	 * MIPID02_CLK_LANE_WR_REG1.
+	 */
+	return mipid02_read_reg(bridge, MIPID02_CLK_LANE_WR_REG1, &reg);
+}
+
+static u32 mipid02_get_link_freq_from_cid_pixel_rate(struct mipid02_dev *bridge,
+						     struct v4l2_subdev *subdev)
+{
+	struct v4l2_fwnode_endpoint *ep = &bridge->rx;
+	struct v4l2_ctrl *ctrl;
+	u32 pixel_clock;
+	u32 bpp = bpp_from_code(bridge->fmt.code);
+
+	ctrl = v4l2_ctrl_find(subdev->ctrl_handler, V4L2_CID_PIXEL_RATE);
+	if (!ctrl)
+		return 0;
+	pixel_clock = v4l2_ctrl_g_ctrl_int64(ctrl);
+
+	return pixel_clock * bpp / (2 * ep->bus.mipi_csi2.num_data_lanes);
+}
+
+/*
+ * We need to know link frequency to setup clk_lane_reg1 timings. Link frequency
+ * will be computed using connected device V4L2_CID_PIXEL_RATE, bit per pixel
+ * and number of lanes.
+ */
+static int mipid02_configure_from_rx_speed(struct mipid02_dev *bridge)
+{
+	struct i2c_client *client = bridge->i2c_client;
+	struct v4l2_subdev *subdev = bridge->s_subdev;
+	u32 link_freq;
+
+	link_freq = mipid02_get_link_freq_from_cid_pixel_rate(bridge, subdev);
+	if (!link_freq) {
+		dev_err(&client->dev, "Failed to detect link frequency");
+		return -EINVAL;
+	}
+
+	dev_dbg(&client->dev, "detect link_freq = %d Hz", link_freq);
+	bridge->r.clk_lane_reg1 |= (2000000000 / link_freq) << 2;
+
+	return 0;
+}
+
+static int mipid02_configure_clk_lane(struct mipid02_dev *bridge)
+{
+	struct i2c_client *client = bridge->i2c_client;
+	struct v4l2_fwnode_endpoint *ep = &bridge->rx;
+	bool *polarities = ep->bus.mipi_csi2.lane_polarities;
+
+	/* midid02 doesn't support clock lane remapping */
+	if (ep->bus.mipi_csi2.clock_lane != 0) {
+		dev_err(&client->dev, "clk lane must be map to lane 0\n");
+		return -EINVAL;
+	}
+	bridge->r.clk_lane_reg1 |= (polarities[0] << 1) | CLK_ENABLE;
+
+	return 0;
+}
+
+static int mipid02_configure_data0_lane(struct mipid02_dev *bridge, int nb,
+					bool are_lanes_swap, bool *polarities)
+{
+	bool are_pin_swap = are_lanes_swap ? polarities[2] : polarities[1];
+
+	if (nb == 1 && are_lanes_swap)
+		return 0;
+
+	/*
+	 * data lane 0 as pin swap polarity reversed compared to clock and
+	 * data lane 1
+	 */
+	if (!are_pin_swap)
+		bridge->r.data_lane0_reg1 = 1 << 1;
+	bridge->r.data_lane0_reg1 |= DATA_ENABLE;
+
+	return 0;
+}
+
+static int mipid02_configure_data1_lane(struct mipid02_dev *bridge, int nb,
+					bool are_lanes_swap, bool *polarities)
+{
+	bool are_pin_swap = are_lanes_swap ? polarities[1] : polarities[2];
+
+	if (nb == 1 && !are_lanes_swap)
+		return 0;
+
+	if (are_pin_swap)
+		bridge->r.data_lane1_reg1 = 1 << 1;
+	bridge->r.data_lane1_reg1 |= DATA_ENABLE;
+
+	return 0;
+}
+
+static int mipid02_configure_from_rx(struct mipid02_dev *bridge)
+{
+	struct v4l2_fwnode_endpoint *ep = &bridge->rx;
+	bool are_lanes_swap = ep->bus.mipi_csi2.data_lanes[0] == 2;
+	bool *polarities = ep->bus.mipi_csi2.lane_polarities;
+	int nb = ep->bus.mipi_csi2.num_data_lanes;
+	int ret;
+
+	ret = mipid02_configure_clk_lane(bridge);
+	if (ret)
+		return ret;
+
+	ret = mipid02_configure_data0_lane(bridge, nb, are_lanes_swap,
+					   polarities);
+	if (ret)
+		return ret;
+
+	ret = mipid02_configure_data1_lane(bridge, nb, are_lanes_swap,
+					   polarities);
+	if (ret)
+		return ret;
+
+	bridge->r.mode_reg1 |= are_lanes_swap ? MODE_DATA_SWAP : 0;
+	bridge->r.mode_reg1 |= (nb - 1) << 1;
+
+	return mipid02_configure_from_rx_speed(bridge);
+}
+
+static int mipid02_configure_from_tx(struct mipid02_dev *bridge)
+{
+	struct v4l2_fwnode_endpoint *ep = &bridge->tx;
+
+	bridge->r.pix_width_ctrl = ep->bus.parallel.bus_width;
+	bridge->r.pix_width_ctrl_emb = ep->bus.parallel.bus_width;
+	if (ep->bus.parallel.flags & V4L2_MBUS_HSYNC_ACTIVE_HIGH)
+		bridge->r.mode_reg2 |= MODE_HSYNC_ACTIVE_HIGH;
+	if (ep->bus.parallel.flags & V4L2_MBUS_VSYNC_ACTIVE_HIGH)
+		bridge->r.mode_reg2 |= MODE_VSYNC_ACTIVE_HIGH;
+
+	return 0;
+}
+
+static int mipid02_configure_from_code(struct mipid02_dev *bridge)
+{
+	u8 data_type;
+
+	bridge->r.data_id_rreg = 0;
+	data_type = data_type_from_code(bridge->fmt.code);
+	if (!data_type)
+		return -EINVAL;
+	bridge->r.data_id_rreg = data_type;
+
+	return 0;
+}
+
+static int mipid02_stream_disable(struct mipid02_dev *bridge)
+{
+	struct i2c_client *client = bridge->i2c_client;
+	int ret;
+
+	/* Disable all lanes */
+	ret = mipid02_write_reg(bridge, MIPID02_CLK_LANE_REG1, 0);
+	if (ret)
+		goto error;
+	ret = mipid02_write_reg(bridge, MIPID02_DATA_LANE0_REG1, 0);
+	if (ret)
+		goto error;
+	ret = mipid02_write_reg(bridge, MIPID02_DATA_LANE1_REG1, 0);
+	if (ret)
+		goto error;
+error:
+	if (ret)
+		dev_err(&client->dev, "failed to stream off %d", ret);
+
+	return ret;
+}
+
+static int mipid02_stream_enable(struct mipid02_dev *bridge)
+{
+	struct i2c_client *client = bridge->i2c_client;
+	int ret = -EINVAL;
+
+	if (!bridge->s_subdev)
+		goto error;
+
+	memset(&bridge->r, 0, sizeof(bridge->r));
+	/* build registers content */
+	ret = mipid02_configure_from_rx(bridge);
+	if (ret)
+		goto error;
+	ret = mipid02_configure_from_tx(bridge);
+	if (ret)
+		goto error;
+	ret = mipid02_configure_from_code(bridge);
+	if (ret)
+		goto error;
+
+	/* write mipi registers */
+	ret = mipid02_write_reg(bridge, MIPID02_CLK_LANE_REG1,
+		bridge->r.clk_lane_reg1);
+	if (ret)
+		goto error;
+	ret = mipid02_write_reg(bridge, MIPID02_CLK_LANE_REG3, CLK_MIPI_CSI);
+	if (ret)
+		goto error;
+	ret = mipid02_write_reg(bridge, MIPID02_DATA_LANE0_REG1,
+		bridge->r.data_lane0_reg1);
+	if (ret)
+		goto error;
+	ret = mipid02_write_reg(bridge, MIPID02_DATA_LANE0_REG2,
+		DATA_MIPI_CSI);
+	if (ret)
+		goto error;
+	ret = mipid02_write_reg(bridge, MIPID02_DATA_LANE1_REG1,
+		bridge->r.data_lane1_reg1);
+	if (ret)
+		goto error;
+	ret = mipid02_write_reg(bridge, MIPID02_DATA_LANE1_REG2,
+		DATA_MIPI_CSI);
+	if (ret)
+		goto error;
+	ret = mipid02_write_reg(bridge, MIPID02_MODE_REG1,
+		MODE_NO_BYPASS | bridge->r.mode_reg1);
+	if (ret)
+		goto error;
+	ret = mipid02_write_reg(bridge, MIPID02_MODE_REG2,
+		bridge->r.mode_reg2);
+	if (ret)
+		goto error;
+	ret = mipid02_write_reg(bridge, MIPID02_DATA_ID_RREG,
+		bridge->r.data_id_rreg);
+	if (ret)
+		goto error;
+	ret = mipid02_write_reg(bridge, MIPID02_DATA_SELECTION_CTRL,
+		SELECTION_MANUAL_DATA | SELECTION_MANUAL_WIDTH);
+	if (ret)
+		goto error;
+	ret = mipid02_write_reg(bridge, MIPID02_PIX_WIDTH_CTRL,
+		bridge->r.pix_width_ctrl);
+	if (ret)
+		goto error;
+	ret = mipid02_write_reg(bridge, MIPID02_PIX_WIDTH_CTRL_EMB,
+		bridge->r.pix_width_ctrl_emb);
+	if (ret)
+		goto error;
+
+	return 0;
+
+error:
+	dev_err(&client->dev, "failed to stream on %d", ret);
+	mipid02_stream_disable(bridge);
+
+	return ret;
+}
+
+static int mipid02_s_stream(struct v4l2_subdev *sd, int enable)
+{
+	struct mipid02_dev *bridge = to_mipid02_dev(sd);
+	struct i2c_client *client = bridge->i2c_client;
+	int ret = 0;
+
+	dev_dbg(&client->dev, "%s : requested %d / current = %d", __func__,
+		    enable, bridge->streaming);
+	mutex_lock(&bridge->lock);
+
+	if (bridge->streaming == enable)
+		goto out;
+
+	ret = enable ? mipid02_stream_enable(bridge) :
+		       mipid02_stream_disable(bridge);
+	if (!ret)
+		bridge->streaming = enable;
+
+out:
+	dev_dbg(&client->dev, "%s current now = %d / %d", __func__,
+		    bridge->streaming, ret);
+	mutex_unlock(&bridge->lock);
+
+	return ret;
+}
+
+static int mipid02_enum_mbus_code(struct v4l2_subdev *sd,
+				 struct v4l2_subdev_pad_config *cfg,
+				 struct v4l2_subdev_mbus_code_enum *code)
+{
+	struct mipid02_dev *bridge = to_mipid02_dev(sd);
+	int ret = 0;
+
+	switch (code->pad) {
+	case MIPID02_SINK_0:
+		if (code->index >= ARRAY_SIZE(mipid02_supported_fmt_codes))
+			ret = -EINVAL;
+		else
+			code->code = mipid02_supported_fmt_codes[code->index];
+		break;
+	case MIPID02_SOURCE:
+		if (code->index == 0)
+			code->code = serial_to_parallel_code(bridge->fmt.code);
+		else
+			ret = -EINVAL;
+		break;
+	default:
+		ret = -EINVAL;
+	}
+
+	return ret;
+}
+
+static int mipid02_get_fmt(struct v4l2_subdev *sd,
+			   struct v4l2_subdev_pad_config *cfg,
+			   struct v4l2_subdev_format *format)
+{
+	struct v4l2_mbus_framefmt *mbus_fmt = &format->format;
+	struct mipid02_dev *bridge = to_mipid02_dev(sd);
+	struct i2c_client *client = bridge->i2c_client;
+	struct v4l2_mbus_framefmt *fmt;
+
+	dev_dbg(&client->dev, "%s probe %d", __func__, format->pad);
+
+	if (format->pad >= MIPID02_PAD_NB)
+		return -EINVAL;
+	/* second CSI-2 pad not yet supported */
+	if (format->pad == MIPID02_SINK_1)
+		return -EINVAL;
+
+	if (format->which == V4L2_SUBDEV_FORMAT_TRY)
+		fmt = v4l2_subdev_get_try_format(&bridge->sd, cfg, format->pad);
+	else
+		fmt = &bridge->fmt;
+
+	mutex_lock(&bridge->lock);
+
+	*mbus_fmt = *fmt;
+	/* code may need to be converted for source */
+	if (format->pad == MIPID02_SOURCE)
+		mbus_fmt->code = serial_to_parallel_code(mbus_fmt->code);
+
+	mutex_unlock(&bridge->lock);
+
+	return 0;
+}
+
+static void mipid02_set_fmt_source(struct v4l2_subdev *sd,
+				   struct v4l2_subdev_pad_config *cfg,
+				   struct v4l2_subdev_format *format)
+{
+	struct mipid02_dev *bridge = to_mipid02_dev(sd);
+
+	/* source pad mirror active sink pad */
+	format->format = bridge->fmt;
+	/* but code may need to be converted */
+	format->format.code = serial_to_parallel_code(format->format.code);
+
+	/* only apply format for V4L2_SUBDEV_FORMAT_TRY case */
+	if (format->which != V4L2_SUBDEV_FORMAT_TRY)
+		return;
+
+	*v4l2_subdev_get_try_format(sd, cfg, format->pad) = format->format;
+}
+
+static void mipid02_set_fmt_sink(struct v4l2_subdev *sd,
+				 struct v4l2_subdev_pad_config *cfg,
+				 struct v4l2_subdev_format *format)
+{
+	struct mipid02_dev *bridge = to_mipid02_dev(sd);
+	struct v4l2_mbus_framefmt *fmt;
+
+	format->format.code = get_fmt_code(format->format.code);
+
+	if (format->which == V4L2_SUBDEV_FORMAT_TRY)
+		fmt = v4l2_subdev_get_try_format(sd, cfg, format->pad);
+	else
+		fmt = &bridge->fmt;
+
+	*fmt = format->format;
+}
+
+static int mipid02_set_fmt(struct v4l2_subdev *sd,
+			   struct v4l2_subdev_pad_config *cfg,
+			   struct v4l2_subdev_format *format)
+{
+	struct mipid02_dev *bridge = to_mipid02_dev(sd);
+	struct i2c_client *client = bridge->i2c_client;
+	int ret = 0;
+
+	dev_dbg(&client->dev, "%s for %d", __func__, format->pad);
+
+	if (format->pad >= MIPID02_PAD_NB)
+		return -EINVAL;
+	/* second CSI-2 pad not yet supported */
+	if (format->pad == MIPID02_SINK_1)
+		return -EINVAL;
+
+	mutex_lock(&bridge->lock);
+
+	if (bridge->streaming) {
+		ret = -EBUSY;
+		goto error;
+	}
+
+	if (format->pad == MIPID02_SOURCE)
+		mipid02_set_fmt_source(sd, cfg, format);
+	else
+		mipid02_set_fmt_sink(sd, cfg, format);
+
+error:
+	mutex_unlock(&bridge->lock);
+
+	return ret;
+}
+
+static const struct v4l2_subdev_video_ops mipid02_video_ops = {
+	.s_stream = mipid02_s_stream,
+};
+
+static const struct v4l2_subdev_pad_ops mipid02_pad_ops = {
+	.enum_mbus_code = mipid02_enum_mbus_code,
+	.get_fmt = mipid02_get_fmt,
+	.set_fmt = mipid02_set_fmt,
+};
+
+static const struct v4l2_subdev_ops mipid02_subdev_ops = {
+	.video = &mipid02_video_ops,
+	.pad = &mipid02_pad_ops,
+};
+
+static const struct media_entity_operations mipid02_subdev_entity_ops = {
+	.link_validate = v4l2_subdev_link_validate,
+};
+
+static int mipid02_async_bound(struct v4l2_async_notifier *notifier,
+			       struct v4l2_subdev *s_subdev,
+			       struct v4l2_async_subdev *asd)
+{
+	struct mipid02_dev *bridge = to_mipid02_dev(notifier->sd);
+	struct i2c_client *client = bridge->i2c_client;
+	int source_pad;
+	int ret;
+
+	dev_dbg(&client->dev, "sensor_async_bound call %p", s_subdev);
+
+	source_pad = media_entity_get_fwnode_pad(&s_subdev->entity,
+						 s_subdev->fwnode,
+						 MEDIA_PAD_FL_SOURCE);
+	if (source_pad < 0) {
+		dev_err(&client->dev, "Couldn't find output pad for subdev %s\n",
+			s_subdev->name);
+		return source_pad;
+	}
+
+	ret = media_create_pad_link(&s_subdev->entity, source_pad,
+				    &bridge->sd.entity, 0,
+				    MEDIA_LNK_FL_ENABLED |
+				    MEDIA_LNK_FL_IMMUTABLE);
+	if (ret) {
+		dev_err(&client->dev, "Couldn't create media link %d", ret);
+		return ret;
+	}
+
+	bridge->s_subdev = s_subdev;
+
+	return 0;
+}
+
+static void mipid02_async_unbind(struct v4l2_async_notifier *notifier,
+				 struct v4l2_subdev *s_subdev,
+				 struct v4l2_async_subdev *asd)
+{
+	struct mipid02_dev *bridge = to_mipid02_dev(notifier->sd);
+
+	bridge->s_subdev = NULL;
+}
+
+static const struct v4l2_async_notifier_operations mipid02_notifier_ops = {
+	.bound		= mipid02_async_bound,
+	.unbind		= mipid02_async_unbind,
+};
+
+static int mipid02_parse_rx_ep(struct mipid02_dev *bridge)
+{
+	struct v4l2_fwnode_endpoint ep = { .bus_type = V4L2_MBUS_CSI2_DPHY };
+	struct i2c_client *client = bridge->i2c_client;
+	struct device_node *ep_node;
+	int ret;
+
+	/* parse rx (endpoint 0) */
+	ep_node = of_graph_get_endpoint_by_regs(bridge->i2c_client->dev.of_node,
+						0, 0);
+	if (!ep_node) {
+		dev_err(&client->dev, "unable to find port0 ep");
+		ret = -EINVAL;
+		goto error;
+	}
+
+	ret = v4l2_fwnode_endpoint_parse(of_fwnode_handle(ep_node), &ep);
+	if (ret) {
+		dev_err(&client->dev, "Could not parse v4l2 endpoint %d\n",
+			ret);
+		goto error_of_node_put;
+	}
+
+	/* do some sanity checks */
+	if (ep.bus.mipi_csi2.num_data_lanes > 2) {
+		dev_err(&client->dev, "max supported data lanes is 2 / got %d",
+			ep.bus.mipi_csi2.num_data_lanes);
+		ret = -EINVAL;
+		goto error_of_node_put;
+	}
+
+	/* register it for later use */
+	bridge->rx = ep;
+
+	/* register async notifier so we get noticed when sensor is connected */
+	bridge->asd.match.fwnode =
+		fwnode_graph_get_remote_port_parent(of_fwnode_handle(ep_node));
+	bridge->asd.match_type = V4L2_ASYNC_MATCH_FWNODE;
+	of_node_put(ep_node);
+
+	v4l2_async_notifier_init(&bridge->notifier);
+	ret = v4l2_async_notifier_add_subdev(&bridge->notifier, &bridge->asd);
+	if (ret) {
+		dev_err(&client->dev, "fail to register asd to notifier %d",
+			ret);
+		fwnode_handle_put(bridge->asd.match.fwnode);
+		return ret;
+	}
+	bridge->notifier.ops = &mipid02_notifier_ops;
+
+	ret = v4l2_async_subdev_notifier_register(&bridge->sd,
+						  &bridge->notifier);
+	if (ret)
+		v4l2_async_notifier_cleanup(&bridge->notifier);
+
+	return ret;
+
+error_of_node_put:
+	of_node_put(ep_node);
+error:
+
+	return ret;
+}
+
+static int mipid02_parse_tx_ep(struct mipid02_dev *bridge)
+{
+	struct v4l2_fwnode_endpoint ep = { .bus_type = V4L2_MBUS_PARALLEL };
+	struct i2c_client *client = bridge->i2c_client;
+	struct device_node *ep_node;
+	int ret;
+
+	/* parse tx (endpoint 2) */
+	ep_node = of_graph_get_endpoint_by_regs(bridge->i2c_client->dev.of_node,
+						2, 0);
+	if (!ep_node) {
+		dev_err(&client->dev, "unable to find port1 ep");
+		ret = -EINVAL;
+		goto error;
+	}
+
+	ret = v4l2_fwnode_endpoint_parse(of_fwnode_handle(ep_node), &ep);
+	if (ret) {
+		dev_err(&client->dev, "Could not parse v4l2 endpoint\n");
+		goto error_of_node_put;
+	}
+
+	of_node_put(ep_node);
+	bridge->tx = ep;
+
+	return 0;
+
+error_of_node_put:
+	of_node_put(ep_node);
+error:
+
+	return -EINVAL;
+}
+
+static int mipid02_probe(struct i2c_client *client)
+{
+	struct device *dev = &client->dev;
+	struct mipid02_dev *bridge;
+	u32 clk_freq;
+	int ret;
+
+	bridge = devm_kzalloc(dev, sizeof(*bridge), GFP_KERNEL);
+	if (!bridge)
+		return -ENOMEM;
+
+	init_format(&bridge->fmt);
+
+	bridge->i2c_client = client;
+	v4l2_i2c_subdev_init(&bridge->sd, client, &mipid02_subdev_ops);
+
+	/* got and check clock */
+	bridge->xclk = devm_clk_get(dev, "xclk");
+	if (IS_ERR(bridge->xclk)) {
+		dev_err(dev, "failed to get xclk\n");
+		return PTR_ERR(bridge->xclk);
+	}
+
+	clk_freq = clk_get_rate(bridge->xclk);
+	if (clk_freq < 6000000 || clk_freq > 27000000) {
+		dev_err(dev, "xclk freq must be in 6-27 Mhz range. got %d Hz\n",
+			clk_freq);
+		return -EINVAL;
+	}
+
+	bridge->reset_gpio = devm_gpiod_get_optional(dev, "reset",
+						     GPIOD_OUT_HIGH);
+
+	ret = mipid02_get_regulators(bridge);
+	if (ret) {
+		dev_err(dev, "failed to get regulators %d", ret);
+		return ret;
+	}
+
+	mutex_init(&bridge->lock);
+	bridge->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
+	bridge->sd.entity.function = MEDIA_ENT_F_VID_IF_BRIDGE;
+	bridge->sd.entity.ops = &mipid02_subdev_entity_ops;
+	bridge->pad[0].flags = MEDIA_PAD_FL_SINK;
+	bridge->pad[1].flags = MEDIA_PAD_FL_SINK;
+	bridge->pad[2].flags = MEDIA_PAD_FL_SOURCE;
+	ret = media_entity_pads_init(&bridge->sd.entity, MIPID02_PAD_NB,
+				     bridge->pad);
+	if (ret) {
+		dev_err(&client->dev, "pads init failed %d", ret);
+		goto mutex_cleanup;
+	}
+
+	/* enable clock, power and reset device if available */
+	ret = mipid02_set_power_on(bridge);
+	if (ret)
+		goto entity_cleanup;
+
+	ret = mipid02_detect(bridge);
+	if (ret) {
+		dev_err(&client->dev, "failed to detect mipid02 %d", ret);
+		goto power_off;
+	}
+
+	ret = mipid02_parse_tx_ep(bridge);
+	if (ret) {
+		dev_err(&client->dev, "failed to parse tx %d", ret);
+		goto power_off;
+	}
+
+	ret = mipid02_parse_rx_ep(bridge);
+	if (ret) {
+		dev_err(&client->dev, "failed to parse rx %d", ret);
+		goto power_off;
+	}
+
+	ret = v4l2_async_register_subdev(&bridge->sd);
+	if (ret < 0) {
+		dev_err(&client->dev, "v4l2_async_register_subdev failed %d",
+			    ret);
+		goto unregister_notifier;
+	}
+
+	dev_info(&client->dev, "mipid02 device probe successfully");
+
+	return 0;
+
+unregister_notifier:
+	v4l2_async_notifier_unregister(&bridge->notifier);
+	v4l2_async_notifier_cleanup(&bridge->notifier);
+power_off:
+	mipid02_set_power_off(bridge);
+entity_cleanup:
+	media_entity_cleanup(&bridge->sd.entity);
+mutex_cleanup:
+	mutex_destroy(&bridge->lock);
+
+	return ret;
+}
+
+static int mipid02_remove(struct i2c_client *client)
+{
+	struct v4l2_subdev *sd = i2c_get_clientdata(client);
+	struct mipid02_dev *bridge = to_mipid02_dev(sd);
+
+	v4l2_async_notifier_unregister(&bridge->notifier);
+	v4l2_async_notifier_cleanup(&bridge->notifier);
+	v4l2_async_unregister_subdev(&bridge->sd);
+	mipid02_set_power_off(bridge);
+	media_entity_cleanup(&bridge->sd.entity);
+	mutex_destroy(&bridge->lock);
+
+	return 0;
+}
+
+static const struct of_device_id mipid02_dt_ids[] = {
+	{ .compatible = "st,st-mipid02" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, mipid02_dt_ids);
+
+static struct i2c_driver mipid02_i2c_driver = {
+	.driver = {
+		.name  = "st-mipid02",
+		.of_match_table = mipid02_dt_ids,
+	},
+	.probe_new = mipid02_probe,
+	.remove = mipid02_remove,
+};
+
+module_i2c_driver(mipid02_i2c_driver);
+
+MODULE_AUTHOR("Mickael Guene <mickael.guene@st.com>");
+MODULE_DESCRIPTION("STMicroelectronics MIPID02 CSI-2 bridge driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/media/media-dev-allocator.c b/drivers/media/media-dev-allocator.c
new file mode 100644
index 000000000000..ae17887dec59
--- /dev/null
+++ b/drivers/media/media-dev-allocator.c
@@ -0,0 +1,135 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * media-dev-allocator.c - Media Controller Device Allocator API
+ *
+ * Copyright (c) 2019 Shuah Khan <shuah@kernel.org>
+ *
+ * Credits: Suggested by Laurent Pinchart <laurent.pinchart@ideasonboard.com>
+ */
+
+/*
+ * This file adds a global refcounted Media Controller Device Instance API.
+ * A system wide global media device list is managed and each media device
+ * includes a kref count. The last put on the media device releases the media
+ * device instance.
+ *
+ */
+
+#include <linux/kref.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/usb.h>
+
+#include <media/media-device.h>
+#include <media/media-dev-allocator.h>
+
+static LIST_HEAD(media_device_list);
+static DEFINE_MUTEX(media_device_lock);
+
+struct media_device_instance {
+	struct media_device mdev;
+	struct module *owner;
+	struct list_head list;
+	struct kref refcount;
+};
+
+static inline struct media_device_instance *
+to_media_device_instance(struct media_device *mdev)
+{
+	return container_of(mdev, struct media_device_instance, mdev);
+}
+
+static void media_device_instance_release(struct kref *kref)
+{
+	struct media_device_instance *mdi =
+		container_of(kref, struct media_device_instance, refcount);
+
+	dev_dbg(mdi->mdev.dev, "%s: releasing Media Device\n", __func__);
+
+	mutex_lock(&media_device_lock);
+
+	media_device_unregister(&mdi->mdev);
+	media_device_cleanup(&mdi->mdev);
+
+	list_del(&mdi->list);
+	mutex_unlock(&media_device_lock);
+
+	kfree(mdi);
+}
+
+/* Callers should hold media_device_lock when calling this function */
+static struct media_device *__media_device_get(struct device *dev,
+						const char *module_name,
+						struct module *owner)
+{
+	struct media_device_instance *mdi;
+
+	list_for_each_entry(mdi, &media_device_list, list) {
+		if (mdi->mdev.dev != dev)
+			continue;
+
+		kref_get(&mdi->refcount);
+
+		/* get module reference for the media_device owner */
+		if (owner != mdi->owner && !try_module_get(mdi->owner))
+			dev_err(dev,
+				"%s: module %s get owner reference error\n",
+					__func__, module_name);
+		else
+			dev_dbg(dev, "%s: module %s got owner reference\n",
+					__func__, module_name);
+		return &mdi->mdev;
+	}
+
+	mdi = kzalloc(sizeof(*mdi), GFP_KERNEL);
+	if (!mdi)
+		return NULL;
+
+	mdi->owner = owner;
+	kref_init(&mdi->refcount);
+	list_add_tail(&mdi->list, &media_device_list);
+
+	dev_dbg(dev, "%s: Allocated media device for owner %s\n",
+			__func__, module_name);
+	return &mdi->mdev;
+}
+
+struct media_device *media_device_usb_allocate(struct usb_device *udev,
+					       const char *module_name,
+					       struct module *owner)
+{
+	struct media_device *mdev;
+
+	mutex_lock(&media_device_lock);
+	mdev = __media_device_get(&udev->dev, module_name, owner);
+	if (!mdev) {
+		mutex_unlock(&media_device_lock);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	/* check if media device is already initialized */
+	if (!mdev->dev)
+		__media_device_usb_init(mdev, udev, udev->product,
+					module_name);
+	mutex_unlock(&media_device_lock);
+	return mdev;
+}
+EXPORT_SYMBOL_GPL(media_device_usb_allocate);
+
+void media_device_delete(struct media_device *mdev, const char *module_name,
+			 struct module *owner)
+{
+	struct media_device_instance *mdi = to_media_device_instance(mdev);
+
+	mutex_lock(&media_device_lock);
+	/* put module reference for the media_device owner */
+	if (mdi->owner != owner) {
+		module_put(mdi->owner);
+		dev_dbg(mdi->mdev.dev,
+			"%s: module %s put owner module reference\n",
+			__func__, module_name);
+	}
+	mutex_unlock(&media_device_lock);
+	kref_put(&mdi->refcount, media_device_instance_release);
+}
+EXPORT_SYMBOL_GPL(media_device_delete);
diff --git a/drivers/media/media-devnode.c b/drivers/media/media-devnode.c
index 6b87a721dc49..d5aa30eeff4a 100644
--- a/drivers/media/media-devnode.c
+++ b/drivers/media/media-devnode.c
@@ -251,6 +251,7 @@ int __must_check media_devnode_register(struct media_device *mdev,
 	/* Part 2: Initialize the character device */
 	cdev_init(&devnode->cdev, &media_devnode_fops);
 	devnode->cdev.owner = owner;
+	kobject_set_name(&devnode->cdev.kobj, "media%d", devnode->minor);
 
 	/* Part 3: Add the media and char device */
 	ret = cdev_device_add(&devnode->cdev, &devnode->dev);
@@ -290,8 +291,9 @@ void media_devnode_unregister(struct media_devnode *devnode)
 	mutex_lock(&media_devnode_lock);
 	/* Delete the cdev on this minor as well */
 	cdev_device_del(&devnode->cdev, &devnode->dev);
-	mutex_unlock(&media_devnode_lock);
 	devnode->media_dev = NULL;
+	mutex_unlock(&media_devnode_lock);
+
 	put_device(&devnode->dev);
 }
 
diff --git a/drivers/media/media-entity.c b/drivers/media/media-entity.c
index 0b1cb3559140..a998a2e0ea1d 100644
--- a/drivers/media/media-entity.c
+++ b/drivers/media/media-entity.c
@@ -17,7 +17,6 @@
  */
 
 #include <linux/bitmap.h>
-#include <linux/module.h>
 #include <linux/property.h>
 #include <linux/slab.h>
 #include <media/media-entity.h>
@@ -436,7 +435,10 @@ __must_check int __media_pipeline_start(struct media_entity *entity,
 
 		entity->stream_count++;
 
-		if (WARN_ON(entity->pipe && entity->pipe != pipe)) {
+		if (entity->pipe && entity->pipe != pipe) {
+			pr_err("Pipe active for %s. Can't start for %s\n",
+				entity->name,
+				entity_err->name);
 			ret = -EBUSY;
 			goto error;
 		}
@@ -588,33 +590,6 @@ void media_pipeline_stop(struct media_entity *entity)
 EXPORT_SYMBOL_GPL(media_pipeline_stop);
 
 /* -----------------------------------------------------------------------------
- * Module use count
- */
-
-struct media_entity *media_entity_get(struct media_entity *entity)
-{
-	if (entity == NULL)
-		return NULL;
-
-	if (entity->graph_obj.mdev->dev &&
-	    !try_module_get(entity->graph_obj.mdev->dev->driver->owner))
-		return NULL;
-
-	return entity;
-}
-EXPORT_SYMBOL_GPL(media_entity_get);
-
-void media_entity_put(struct media_entity *entity)
-{
-	if (entity == NULL)
-		return;
-
-	if (entity->graph_obj.mdev->dev)
-		module_put(entity->graph_obj.mdev->dev->driver->owner);
-}
-EXPORT_SYMBOL_GPL(media_entity_put);
-
-/* -----------------------------------------------------------------------------
  * Links management
  */
 
diff --git a/drivers/media/media-request.c b/drivers/media/media-request.c
index 9e5fd2ac769e..e3fca436c75b 100644
--- a/drivers/media/media-request.c
+++ b/drivers/media/media-request.c
@@ -251,7 +251,7 @@ media_request_get_by_fd(struct media_device *mdev, int request_fd)
 
 	if (!mdev || !mdev->ops ||
 	    !mdev->ops->req_validate || !mdev->ops->req_queue)
-		return ERR_PTR(-EACCES);
+		return ERR_PTR(-EBADR);
 
 	f = fdget(request_fd);
 	if (!f.file)
@@ -407,7 +407,7 @@ int media_request_object_bind(struct media_request *req,
 	int ret = -EBUSY;
 
 	if (WARN_ON(!ops->release))
-		return -EACCES;
+		return -EBADR;
 
 	spin_lock_irqsave(&req->lock, flags);
 
diff --git a/drivers/media/mmc/siano/Kconfig b/drivers/media/mmc/siano/Kconfig
index 7693487e2f63..3941ee8352bb 100644
--- a/drivers/media/mmc/siano/Kconfig
+++ b/drivers/media/mmc/siano/Kconfig
@@ -9,5 +9,5 @@ config SMS_SDIO_DRV
 	depends on !RC_CORE || RC_CORE
 	select MEDIA_COMMON_OPTIONS
 	select SMS_SIANO_MDTV
-	---help---
+	help
 	  Choose if you would like to have Siano's support for SDIO interface
diff --git a/drivers/media/pci/bt8xx/Kconfig b/drivers/media/pci/bt8xx/Kconfig
index bc89e37608cd..0f46db7d5ffc 100644
--- a/drivers/media/pci/bt8xx/Kconfig
+++ b/drivers/media/pci/bt8xx/Kconfig
@@ -13,7 +13,7 @@ config VIDEO_BT848
 	select VIDEO_SAA6588 if MEDIA_SUBDRV_AUTOSELECT
 	select RADIO_ADAPTERS
 	select RADIO_TEA575X
-	---help---
+	help
 	  Support for BT848 based frame grabber/overlay boards. This includes
 	  the Miro, Hauppauge and STB boards. Please read the material in
 	  <file:Documentation/media/v4l-drivers/bttv.rst> for more information.
diff --git a/drivers/media/pci/bt8xx/dst.c b/drivers/media/pci/bt8xx/dst.c
index c94318c71a0c..e929797206f6 100644
--- a/drivers/media/pci/bt8xx/dst.c
+++ b/drivers/media/pci/bt8xx/dst.c
@@ -1100,7 +1100,8 @@ static int dst_get_device_id(struct dst_state *state)
 			/*	Card capabilities	*/
 			state->dst_hw_cap = p_dst_type->dst_feature;
 			pr_err("Recognise [%s]\n", p_dst_type->device_id);
-			strncpy(&state->fw_name[0], p_dst_type->device_id, 6);
+			strscpy(state->fw_name, p_dst_type->device_id,
+			        sizeof(state->fw_name));
 			/*	Multiple tuners		*/
 			if (p_dst_type->tuner_type & TUNER_TYPE_MULTI) {
 				switch (use_dst_type) {
diff --git a/drivers/media/pci/bt8xx/dst_common.h b/drivers/media/pci/bt8xx/dst_common.h
index 6a2cfdd44e3e..79dec1b1722c 100644
--- a/drivers/media/pci/bt8xx/dst_common.h
+++ b/drivers/media/pci/bt8xx/dst_common.h
@@ -138,7 +138,7 @@ struct dst_state {
 	u32 tuner_type;
 	char *tuner_name;
 	struct mutex dst_mutex;
-	u8 fw_name[8];
+	char fw_name[8];
 	struct dvb_device *dst_ca;
 };
 
diff --git a/drivers/media/pci/cobalt/Kconfig b/drivers/media/pci/cobalt/Kconfig
index aa35cbc0a904..9a544bab3178 100644
--- a/drivers/media/pci/cobalt/Kconfig
+++ b/drivers/media/pci/cobalt/Kconfig
@@ -11,7 +11,7 @@ config VIDEO_COBALT
 	select VIDEO_ADV7511
 	select VIDEO_ADV7842
 	select VIDEOBUF2_DMA_SG
-	---help---
+	help
 	  This is a video4linux driver for the Cisco PCIe Cobalt card.
 
 	  This board is sadly not available outside of Cisco, but it is
diff --git a/drivers/media/pci/cobalt/cobalt-irq.c b/drivers/media/pci/cobalt/cobalt-irq.c
index 04783e78cc12..a518927abae1 100644
--- a/drivers/media/pci/cobalt/cobalt-irq.c
+++ b/drivers/media/pci/cobalt/cobalt-irq.c
@@ -128,7 +128,7 @@ done:
 	cb->vb.sequence = s->sequence++;
 	vb2_buffer_done(&cb->vb.vb2_buf,
 			(skip || s->unstable_frame) ?
-			VB2_BUF_STATE_REQUEUEING : VB2_BUF_STATE_DONE);
+			VB2_BUF_STATE_ERROR : VB2_BUF_STATE_DONE);
 }
 
 irqreturn_t cobalt_irq_handler(int irq, void *dev_id)
diff --git a/drivers/media/pci/cx18/Kconfig b/drivers/media/pci/cx18/Kconfig
index c675b83c43a9..96477bba0d5c 100644
--- a/drivers/media/pci/cx18/Kconfig
+++ b/drivers/media/pci/cx18/Kconfig
@@ -13,7 +13,7 @@ config VIDEO_CX18
 	select DVB_S5H1411 if MEDIA_SUBDRV_AUTOSELECT
 	select MEDIA_TUNER_TDA18271 if MEDIA_SUBDRV_AUTOSELECT
 	select MEDIA_TUNER_TDA8290 if MEDIA_SUBDRV_AUTOSELECT
-	---help---
+	help
 	  This is a video4linux driver for Conexant cx23418 based
 	  PCI combo video recorder devices.
 
@@ -27,7 +27,7 @@ config VIDEO_CX18_ALSA
 	tristate "Conexant 23418 DMA audio support"
 	depends on VIDEO_CX18 && SND
 	select SND_PCM
-	---help---
+	help
 	  This is a video4linux driver for direct (DMA) audio on
 	  Conexant 23418 based TV cards using ALSA.
 
diff --git a/drivers/media/pci/cx18/cx18-fileops.c b/drivers/media/pci/cx18/cx18-fileops.c
index f778965a2eb8..59e78fb17575 100644
--- a/drivers/media/pci/cx18/cx18-fileops.c
+++ b/drivers/media/pci/cx18/cx18-fileops.c
@@ -484,7 +484,7 @@ static ssize_t cx18_read_pos(struct cx18_stream *s, char __user *ubuf,
 
 	CX18_DEBUG_HI_FILE("read %zd from %s, got %zd\n", count, s->name, rc);
 	if (rc > 0)
-		pos += rc;
+		*pos += rc;
 	return rc;
 }
 
diff --git a/drivers/media/pci/cx23885/Kconfig b/drivers/media/pci/cx23885/Kconfig
index 3435bbaa3167..1bba9e497915 100644
--- a/drivers/media/pci/cx23885/Kconfig
+++ b/drivers/media/pci/cx23885/Kconfig
@@ -43,7 +43,7 @@ config VIDEO_CX23885
 	select MEDIA_TUNER_SI2157 if MEDIA_SUBDRV_AUTOSELECT
 	select MEDIA_TUNER_M88RS6000T if MEDIA_SUBDRV_AUTOSELECT
 	select DVB_TUNER_DIB0070 if MEDIA_SUBDRV_AUTOSELECT
-	---help---
+	help
 	  This is a video4linux driver for Conexant 23885 based
 	  TV cards.
 
@@ -54,7 +54,7 @@ config MEDIA_ALTERA_CI
 	tristate "Altera FPGA based CI module"
 	depends on VIDEO_CX23885 && DVB_CORE
 	select ALTERA_STAPL
-	---help---
+	help
 	  An Altera FPGA CI module for NetUP Dual DVB-T/C RF CI card.
 
 	  To compile this driver as a module, choose M here: the
diff --git a/drivers/media/pci/cx23885/cx23885-dvb.c b/drivers/media/pci/cx23885/cx23885-dvb.c
index 0d0929c54f93..e2e63f05645e 100644
--- a/drivers/media/pci/cx23885/cx23885-dvb.c
+++ b/drivers/media/pci/cx23885/cx23885-dvb.c
@@ -1474,8 +1474,9 @@ static int dvb_register(struct cx23885_tsport *port)
 		if (fe0->dvb.frontend != NULL) {
 			struct i2c_adapter *tun_i2c;
 
-			fe0->dvb.frontend->sec_priv = kmalloc(sizeof(dib7000p_ops), GFP_KERNEL);
-			memcpy(fe0->dvb.frontend->sec_priv, &dib7000p_ops, sizeof(dib7000p_ops));
+			fe0->dvb.frontend->sec_priv = kmemdup(&dib7000p_ops, sizeof(dib7000p_ops), GFP_KERNEL);
+			if (!fe0->dvb.frontend->sec_priv)
+				return -ENOMEM;
 			tun_i2c = dib7000p_ops.get_i2c_master(fe0->dvb.frontend, DIBX000_I2C_INTERFACE_TUNER, 1);
 			if (!dvb_attach(dib0070_attach, fe0->dvb.frontend, tun_i2c, &dib7070p_dib0070_config))
 				return -ENODEV;
diff --git a/drivers/media/pci/cx25821/Kconfig b/drivers/media/pci/cx25821/Kconfig
index 1755d3d2feaa..a64fa9a6d5d5 100644
--- a/drivers/media/pci/cx25821/Kconfig
+++ b/drivers/media/pci/cx25821/Kconfig
@@ -3,7 +3,7 @@ config VIDEO_CX25821
 	depends on VIDEO_DEV && PCI && I2C
 	select I2C_ALGOBIT
 	select VIDEOBUF2_DMA_SG
-	---help---
+	help
 	  This is a video4linux driver for Conexant 25821 based
 	  TV cards.
 
@@ -14,7 +14,7 @@ config VIDEO_CX25821_ALSA
 	tristate "Conexant 25821 DMA audio support"
 	depends on VIDEO_CX25821 && SND
 	select SND_PCM
-	---help---
+	help
 	  This is a video4linux driver for direct (DMA) audio on
 	  Conexant 25821 based capture cards using ALSA.
 
diff --git a/drivers/media/pci/cx88/Kconfig b/drivers/media/pci/cx88/Kconfig
index 14b813d634a8..fbb17ddb6bc3 100644
--- a/drivers/media/pci/cx88/Kconfig
+++ b/drivers/media/pci/cx88/Kconfig
@@ -6,7 +6,7 @@ config VIDEO_CX88
 	select VIDEO_TUNER
 	select VIDEO_TVEEPROM
 	select VIDEO_WM8775 if MEDIA_SUBDRV_AUTOSELECT
-	---help---
+	help
 	  This is a video4linux driver for Conexant 2388x based
 	  TV cards.
 
@@ -17,7 +17,7 @@ config VIDEO_CX88_ALSA
 	tristate "Conexant 2388x DMA audio support"
 	depends on VIDEO_CX88 && SND
 	select SND_PCM
-	---help---
+	help
 	  This is a video4linux driver for direct (DMA) audio on
 	  Conexant 2388x based TV cards using ALSA.
 
@@ -33,7 +33,7 @@ config VIDEO_CX88_BLACKBIRD
 	tristate "Blackbird MPEG encoder support (cx2388x + cx23416)"
 	depends on VIDEO_CX88
 	select VIDEO_CX2341X
-	---help---
+	help
 	  This adds support for MPEG encoder cards based on the
 	  Blackbird reference design, using the Conexant 2388x
 	  and 23416 chips.
@@ -64,7 +64,7 @@ config VIDEO_CX88_DVB
 	select DVB_DS3000 if MEDIA_SUBDRV_AUTOSELECT
 	select DVB_TS2020 if MEDIA_SUBDRV_AUTOSELECT
 	select MEDIA_TUNER_SIMPLE if MEDIA_SUBDRV_AUTOSELECT
-	---help---
+	help
 	  This adds support for DVB/ATSC cards based on the
 	  Conexant 2388x chip.
 
@@ -75,7 +75,7 @@ config VIDEO_CX88_ENABLE_VP3054
 	bool "VP-3054 Secondary I2C Bus Support"
 	default y
 	depends on VIDEO_CX88_DVB && DVB_MT352
-	---help---
+	help
 	  This adds DVB-T support for cards based on the
 	  Conexant 2388x chip and the MT352 demodulator,
 	  which also require support for the VP-3054
diff --git a/drivers/media/pci/ddbridge/Kconfig b/drivers/media/pci/ddbridge/Kconfig
index 16faef265e97..fc98b6d575d9 100644
--- a/drivers/media/pci/ddbridge/Kconfig
+++ b/drivers/media/pci/ddbridge/Kconfig
@@ -15,7 +15,7 @@ config DVB_DDBRIDGE
 	select DVB_MXL5XX if MEDIA_SUBDRV_AUTOSELECT
 	select DVB_CXD2099 if MEDIA_SUBDRV_AUTOSELECT
 	select DVB_DUMMY_FE if MEDIA_SUBDRV_AUTOSELECT
-	---help---
+	help
 	  Support for cards with the Digital Devices PCI express bridge:
 	  - Octopus PCIe Bridge
 	  - Octopus mini PCIe Bridge
@@ -36,7 +36,7 @@ config DVB_DDBRIDGE_MSIENABLE
 	depends on DVB_DDBRIDGE
 	depends on PCI_MSI
 	default n
-	---help---
+	help
 	  Use PCI MSI (Message Signaled Interrupts) per default. Enabling this
 	  might lead to I2C errors originating from the bridge in conjunction
 	  with certain SATA controllers, requiring a reload of the ddbridge
diff --git a/drivers/media/pci/dt3155/Kconfig b/drivers/media/pci/dt3155/Kconfig
index 858b0f2f15be..d770eec541d4 100644
--- a/drivers/media/pci/dt3155/Kconfig
+++ b/drivers/media/pci/dt3155/Kconfig
@@ -3,7 +3,7 @@ config VIDEO_DT3155
 	depends on PCI && VIDEO_DEV && VIDEO_V4L2
 	select VIDEOBUF2_DMA_CONTIG
 	default n
-	---help---
+	help
 	  Enables dt3155 device driver for the DataTranslation DT3155 frame grabber.
 	  Say Y here if you have this hardware.
 	  In doubt, say N.
diff --git a/drivers/media/pci/intel/ipu3/Kconfig b/drivers/media/pci/intel/ipu3/Kconfig
index 715f77651482..bd518bdc9f5f 100644
--- a/drivers/media/pci/intel/ipu3/Kconfig
+++ b/drivers/media/pci/intel/ipu3/Kconfig
@@ -7,7 +7,7 @@ config VIDEO_IPU3_CIO2
 	select V4L2_FWNODE
 	select VIDEOBUF2_DMA_SG
 
-	---help---
+	help
 	  This is the Intel IPU3 CIO2 CSI-2 receiver unit, found in Intel
 	  Skylake and Kaby Lake SoCs and used for capturing images and
 	  video from a camera sensor.
diff --git a/drivers/media/pci/intel/ipu3/ipu3-cio2.c b/drivers/media/pci/intel/ipu3/ipu3-cio2.c
index f8020ebe9f05..2a52a393fe74 100644
--- a/drivers/media/pci/intel/ipu3/ipu3-cio2.c
+++ b/drivers/media/pci/intel/ipu3/ipu3-cio2.c
@@ -1601,6 +1601,7 @@ static int cio2_queue_init(struct cio2_device *cio2, struct cio2_queue *q)
 	subdev->owner = THIS_MODULE;
 	snprintf(subdev->name, sizeof(subdev->name),
 		 CIO2_ENTITY_NAME " %td", q - cio2->queue);
+	subdev->entity.function = MEDIA_ENT_F_VID_IF_BRIDGE;
 	v4l2_set_subdevdata(subdev, cio2);
 	r = v4l2_device_register_subdev(&cio2->v4l2_dev, subdev);
 	if (r) {
diff --git a/drivers/media/pci/ivtv/Kconfig b/drivers/media/pci/ivtv/Kconfig
index 06ca4e23f9fb..e96b3c182a2f 100644
--- a/drivers/media/pci/ivtv/Kconfig
+++ b/drivers/media/pci/ivtv/Kconfig
@@ -18,7 +18,7 @@ config VIDEO_IVTV
 	select VIDEO_VP27SMPX
 	select VIDEO_UPD64031A
 	select VIDEO_UPD64083
-	---help---
+	help
 	  This is a video4linux driver for Conexant cx23416 or cx23415 based
 	  PCI personal video recorder devices.
 
@@ -32,7 +32,7 @@ config VIDEO_IVTV_DEPRECATED_IOCTLS
 	bool "enable the DVB ioctls abuse on ivtv driver"
 	depends on VIDEO_IVTV
 	default n
-	---help---
+	help
 	  Enable the usage of the a DVB set of ioctls that were abused by
 	  IVTV driver for a while.
 
@@ -45,7 +45,7 @@ config VIDEO_IVTV_ALSA
 	tristate "Conexant cx23415/cx23416 ALSA interface for PCM audio capture"
 	depends on VIDEO_IVTV && SND
 	select SND_PCM
-	---help---
+	help
 	  This driver provides an ALSA interface as another method for user
 	  applications to obtain PCM audio data from Conexant cx23415/cx23416
 	  based PCI TV cards supported by the ivtv driver.
@@ -63,7 +63,7 @@ config VIDEO_FB_IVTV
 	select FB_CFB_FILLRECT
 	select FB_CFB_COPYAREA
 	select FB_CFB_IMAGEBLIT
-	---help---
+	help
 	  This is a framebuffer driver for the Conexant cx23415 MPEG
 	  encoder/decoder.
 
@@ -77,7 +77,7 @@ config VIDEO_FB_IVTV_FORCE_PAT
 	bool "force cx23415 framebuffer init with x86 PAT enabled"
 	depends on VIDEO_FB_IVTV && X86_PAT
 	default n
-	---help---
+	help
 	  With PAT enabled, the cx23415 framebuffer driver does not
 	  utilize write-combined caching on the framebuffer memory.
 	  For this reason, the driver will by default disable itself
diff --git a/drivers/media/pci/ivtv/ivtv-fileops.c b/drivers/media/pci/ivtv/ivtv-fileops.c
index 6196daae4b3e..043ac0ae9ed0 100644
--- a/drivers/media/pci/ivtv/ivtv-fileops.c
+++ b/drivers/media/pci/ivtv/ivtv-fileops.c
@@ -420,7 +420,7 @@ static ssize_t ivtv_read_pos(struct ivtv_stream *s, char __user *ubuf, size_t co
 
 	IVTV_DEBUG_HI_FILE("read %zd from %s, got %zd\n", count, s->name, rc);
 	if (rc > 0)
-		pos += rc;
+		*pos += rc;
 	return rc;
 }
 
diff --git a/drivers/media/pci/mantis/mantis_i2c.c b/drivers/media/pci/mantis/mantis_i2c.c
index 6528a2180119..f8b503ef42bc 100644
--- a/drivers/media/pci/mantis/mantis_i2c.c
+++ b/drivers/media/pci/mantis/mantis_i2c.c
@@ -225,7 +225,7 @@ int mantis_i2c_init(struct mantis_pci *mantis)
 
 	init_waitqueue_head(&mantis->i2c_wq);
 	mutex_init(&mantis->i2c_lock);
-	strncpy(i2c_adapter->name, "Mantis I2C", sizeof(i2c_adapter->name));
+	strscpy(i2c_adapter->name, "Mantis I2C", sizeof(i2c_adapter->name));
 	i2c_set_adapdata(i2c_adapter, mantis);
 
 	i2c_adapter->owner	= THIS_MODULE;
diff --git a/drivers/media/pci/meye/Kconfig b/drivers/media/pci/meye/Kconfig
index 9a50f54231ad..ce0463c81886 100644
--- a/drivers/media/pci/meye/Kconfig
+++ b/drivers/media/pci/meye/Kconfig
@@ -2,7 +2,7 @@ config VIDEO_MEYE
 	tristate "Sony Vaio Picturebook Motion Eye Video For Linux"
 	depends on PCI && VIDEO_V4L2
 	depends on SONY_LAPTOP || COMPILE_TEST
-	---help---
+	help
 	  This is the video4linux driver for the Motion Eye camera found
 	  in the Vaio Picturebook laptops. Please read the material in
 	  <file:Documentation/media/v4l-drivers/meye.rst> for more information.
diff --git a/drivers/media/pci/netup_unidvb/Kconfig b/drivers/media/pci/netup_unidvb/Kconfig
index b663154d0cc4..60057585f04c 100644
--- a/drivers/media/pci/netup_unidvb/Kconfig
+++ b/drivers/media/pci/netup_unidvb/Kconfig
@@ -8,7 +8,7 @@ config DVB_NETUP_UNIDVB
 	select DVB_HELENE if MEDIA_SUBDRV_AUTOSELECT
 	select DVB_LNBH25 if MEDIA_SUBDRV_AUTOSELECT
 	select DVB_CXD2841ER if MEDIA_SUBDRV_AUTOSELECT
-	---help---
+	help
 	  Support for NetUP PCI express Universal DVB card.
 
 	  Say Y when you want to support NetUP Dual Universal DVB card.
diff --git a/drivers/media/pci/ngene/Kconfig b/drivers/media/pci/ngene/Kconfig
index e06d019996f3..8a80a5bab8e9 100644
--- a/drivers/media/pci/ngene/Kconfig
+++ b/drivers/media/pci/ngene/Kconfig
@@ -15,6 +15,6 @@ config DVB_NGENE
 	select DVB_STV6111 if MEDIA_SUBDRV_AUTOSELECT
 	select DVB_LNBH25 if MEDIA_SUBDRV_AUTOSELECT
 	select DVB_CXD2099 if MEDIA_SUBDRV_AUTOSELECT
-	---help---
+	help
 	  Support for Micronas PCI express cards with nGene bridge.
 
diff --git a/drivers/media/pci/saa7134/Kconfig b/drivers/media/pci/saa7134/Kconfig
index b44e0d70907e..8b28783b3fcd 100644
--- a/drivers/media/pci/saa7134/Kconfig
+++ b/drivers/media/pci/saa7134/Kconfig
@@ -7,7 +7,7 @@ config VIDEO_SAA7134
 	select CRC32
 	select VIDEO_SAA6588 if MEDIA_SUBDRV_AUTOSELECT
 	select VIDEO_SAA6752HS if MEDIA_SUBDRV_AUTOSELECT
-	---help---
+	help
 	  This is a video4linux driver for Philips SAA713x based
 	  TV cards.
 
@@ -18,7 +18,7 @@ config VIDEO_SAA7134_ALSA
 	tristate "Philips SAA7134 DMA audio support"
 	depends on VIDEO_SAA7134 && SND
 	select SND_PCM
-	---help---
+	help
 	  This is a video4linux driver for direct (DMA) audio in
 	  Philips SAA713x based TV cards using ALSA
 
@@ -31,7 +31,7 @@ config VIDEO_SAA7134_RC
 	depends on VIDEO_SAA7134
 	depends on !(RC_CORE=m && VIDEO_SAA7134=y)
 	default y
-	---help---
+	help
 	  Enables Remote Controller support on saa7134 driver.
 
 config VIDEO_SAA7134_DVB
@@ -57,7 +57,7 @@ config VIDEO_SAA7134_DVB
 	select MEDIA_TUNER_TDA18271 if MEDIA_SUBDRV_AUTOSELECT
 	select MEDIA_TUNER_TDA8290 if MEDIA_SUBDRV_AUTOSELECT
 	select DVB_ZL10039 if MEDIA_SUBDRV_AUTOSELECT
-	---help---
+	help
 	  This adds support for DVB cards based on the
 	  Philips saa7134 chip.
 
@@ -68,6 +68,6 @@ config VIDEO_SAA7134_GO7007
 	tristate "go7007 support for saa7134 based TV cards"
 	depends on VIDEO_SAA7134
 	depends on VIDEO_GO7007
-	---help---
+	help
 	  Enables saa7134 driver support for boards with go7007
 	  MPEG encoder (WIS Voyager or compatible).
diff --git a/drivers/media/pci/saa7134/saa7134-go7007.c b/drivers/media/pci/saa7134/saa7134-go7007.c
index 275c5e151818..626e130a9770 100644
--- a/drivers/media/pci/saa7134/saa7134-go7007.c
+++ b/drivers/media/pci/saa7134/saa7134-go7007.c
@@ -444,7 +444,7 @@ static int saa7134_go7007_init(struct saa7134_dev *dev)
 	sd = &saa->sd;
 	v4l2_subdev_init(sd, &saa7134_go7007_sd_ops);
 	v4l2_set_subdevdata(sd, saa);
-	strncpy(sd->name, "saa7134-go7007", sizeof(sd->name));
+	strscpy(sd->name, "saa7134-go7007", sizeof(sd->name));
 
 	/* Allocate a couple pages for receiving the compressed stream */
 	saa->top = (u8 *)get_zeroed_page(GFP_KERNEL);
diff --git a/drivers/media/pci/saa7146/Kconfig b/drivers/media/pci/saa7146/Kconfig
index da88b77a916c..60d9862580ff 100644
--- a/drivers/media/pci/saa7146/Kconfig
+++ b/drivers/media/pci/saa7146/Kconfig
@@ -2,7 +2,7 @@ config VIDEO_HEXIUM_GEMINI
 	tristate "Hexium Gemini frame grabber"
 	depends on PCI && VIDEO_V4L2 && I2C
 	select VIDEO_SAA7146_VV
-	---help---
+	help
 	  This is a video4linux driver for the Hexium Gemini frame
 	  grabber card by Hexium. Please note that the Gemini Dual
 	  card is *not* fully supported.
@@ -14,7 +14,7 @@ config VIDEO_HEXIUM_ORION
 	tristate "Hexium HV-PCI6 and Orion frame grabber"
 	depends on PCI && VIDEO_V4L2 && I2C
 	select VIDEO_SAA7146_VV
-	---help---
+	help
 	  This is a video4linux driver for the Hexium HV-PCI6 and
 	  Orion frame grabber cards by Hexium.
 
@@ -30,7 +30,7 @@ config VIDEO_MXB
 	select VIDEO_TDA9840 if MEDIA_SUBDRV_AUTOSELECT
 	select VIDEO_TEA6415C if MEDIA_SUBDRV_AUTOSELECT
 	select VIDEO_TEA6420 if MEDIA_SUBDRV_AUTOSELECT
-	---help---
+	help
 	  This is a video4linux driver for the 'Multimedia eXtension Board'
 	  TV card by Siemens-Nixdorf.
 
diff --git a/drivers/media/pci/saa7146/hexium_gemini.c b/drivers/media/pci/saa7146/hexium_gemini.c
index 5817d9cde4d0..6d8e4afe9673 100644
--- a/drivers/media/pci/saa7146/hexium_gemini.c
+++ b/drivers/media/pci/saa7146/hexium_gemini.c
@@ -270,9 +270,8 @@ static int hexium_attach(struct saa7146_dev *dev, struct saa7146_pci_extension_d
 	/* enable i2c-port pins */
 	saa7146_write(dev, MC1, (MASK_08 | MASK_24 | MASK_10 | MASK_26));
 
-	hexium->i2c_adapter = (struct i2c_adapter) {
-		.name = "hexium gemini",
-	};
+	strscpy(hexium->i2c_adapter.name, "hexium gemini",
+		sizeof(hexium->i2c_adapter.name));
 	saa7146_i2c_adapter_prepare(dev, &hexium->i2c_adapter, SAA7146_I2C_BUS_BIT_RATE_480);
 	if (i2c_add_adapter(&hexium->i2c_adapter) < 0) {
 		DEB_S("cannot register i2c-device. skipping.\n");
diff --git a/drivers/media/pci/saa7146/hexium_orion.c b/drivers/media/pci/saa7146/hexium_orion.c
index 0a05176c18ab..a794f9e5f990 100644
--- a/drivers/media/pci/saa7146/hexium_orion.c
+++ b/drivers/media/pci/saa7146/hexium_orion.c
@@ -231,9 +231,8 @@ static int hexium_probe(struct saa7146_dev *dev)
 	saa7146_write(dev, DD1_STREAM_B, 0x00000000);
 	saa7146_write(dev, MC2, (MASK_09 | MASK_25 | MASK_10 | MASK_26));
 
-	hexium->i2c_adapter = (struct i2c_adapter) {
-		.name = "hexium orion",
-	};
+	strscpy(hexium->i2c_adapter.name, "hexium orion",
+		sizeof(hexium->i2c_adapter.name));
 	saa7146_i2c_adapter_prepare(dev, &hexium->i2c_adapter, SAA7146_I2C_BUS_BIT_RATE_480);
 	if (i2c_add_adapter(&hexium->i2c_adapter) < 0) {
 		DEB_S("cannot register i2c-device. skipping.\n");
diff --git a/drivers/media/pci/saa7164/Kconfig b/drivers/media/pci/saa7164/Kconfig
index 9098ef5feca4..265c5a4fd823 100644
--- a/drivers/media/pci/saa7164/Kconfig
+++ b/drivers/media/pci/saa7164/Kconfig
@@ -8,7 +8,7 @@ config VIDEO_SAA7164
 	select DVB_TDA10048 if MEDIA_SUBDRV_AUTOSELECT
 	select DVB_S5H1411 if MEDIA_SUBDRV_AUTOSELECT
 	select MEDIA_TUNER_TDA18271 if MEDIA_SUBDRV_AUTOSELECT
-	---help---
+	help
 	  This is a video4linux driver for NXP SAA7164 based
 	  TV cards.
 
diff --git a/drivers/media/pci/solo6x10/Kconfig b/drivers/media/pci/solo6x10/Kconfig
index d9e06a6bf1eb..2061d02a82d0 100644
--- a/drivers/media/pci/solo6x10/Kconfig
+++ b/drivers/media/pci/solo6x10/Kconfig
@@ -8,7 +8,7 @@ config VIDEO_SOLO6X10
 	select VIDEOBUF2_DMA_CONTIG
 	select SND_PCM
 	select FONT_8x16
-	---help---
+	help
 	  This driver supports the Bluecherry H.264 and MPEG-4 hardware
 	  compression capture cards and other Softlogic-based ones.
 
diff --git a/drivers/media/pci/tw5864/Kconfig b/drivers/media/pci/tw5864/Kconfig
index 760fb11dfeae..e5d52f076232 100644
--- a/drivers/media/pci/tw5864/Kconfig
+++ b/drivers/media/pci/tw5864/Kconfig
@@ -2,7 +2,7 @@ config VIDEO_TW5864
 	tristate "Techwell TW5864 video/audio grabber and encoder"
 	depends on VIDEO_DEV && PCI && VIDEO_V4L2
 	select VIDEOBUF2_DMA_CONTIG
-	---help---
+	help
 	  Support for boards based on Techwell TW5864 chip which provides
 	  multichannel video & audio grabbing and encoding (H.264, MJPEG,
 	  ADPCM G.726).
diff --git a/drivers/media/pci/tw5864/tw5864-video.c b/drivers/media/pci/tw5864/tw5864-video.c
index 5a1f3aa4101a..434d313a3c15 100644
--- a/drivers/media/pci/tw5864/tw5864-video.c
+++ b/drivers/media/pci/tw5864/tw5864-video.c
@@ -1395,13 +1395,13 @@ static void tw5864_handle_frame(struct tw5864_h264_frame *frame)
 	input->vb = NULL;
 	spin_unlock_irqrestore(&input->slock, flags);
 
-	v4l2_buf = to_vb2_v4l2_buffer(&vb->vb.vb2_buf);
-
 	if (!vb) { /* Gone because of disabling */
 		dev_dbg(&dev->pci->dev, "vb is empty, dropping frame\n");
 		return;
 	}
 
+	v4l2_buf = to_vb2_v4l2_buffer(&vb->vb.vb2_buf);
+
 	/*
 	 * Check for space.
 	 * Mind the overhead of startcode emulation prevention.
diff --git a/drivers/media/pci/tw68/Kconfig b/drivers/media/pci/tw68/Kconfig
index 95d5d5202048..4bfc4fa416e5 100644
--- a/drivers/media/pci/tw68/Kconfig
+++ b/drivers/media/pci/tw68/Kconfig
@@ -2,7 +2,7 @@ config VIDEO_TW68
 	tristate "Techwell tw68x Video For Linux"
 	depends on VIDEO_DEV && PCI && VIDEO_V4L2
 	select VIDEOBUF2_DMA_SG
-	---help---
+	help
 	  Support for Techwell tw68xx based frame grabber boards.
 
 	  To compile this driver as a module, choose M here: the
diff --git a/drivers/media/platform/Kconfig b/drivers/media/platform/Kconfig
index 4acbed189644..011c1c2fcf19 100644
--- a/drivers/media/platform/Kconfig
+++ b/drivers/media/platform/Kconfig
@@ -6,7 +6,7 @@ menuconfig V4L_PLATFORM_DRIVERS
 	bool "V4L platform devices"
 	depends on MEDIA_CAMERA_SUPPORT
 	default n
-	---help---
+	help
 	  Say Y here to enable support for platform-specific V4L drivers.
 
 if V4L_PLATFORM_DRIVERS
@@ -55,7 +55,7 @@ config VIDEO_VIU
 	depends on VIDEO_V4L2 && (PPC_MPC512x || COMPILE_TEST) && I2C
 	select VIDEOBUF_DMA_CONTIG
 	default y
-	---help---
+	help
 	  Support for Freescale VIU video driver. This device captures
 	  video data, or overlays video on DIU frame buffer.
 
@@ -80,13 +80,13 @@ config VIDEO_OMAP3
 	select VIDEOBUF2_DMA_CONTIG
 	select MFD_SYSCON
 	select V4L2_FWNODE
-	---help---
+	help
 	  Driver for an OMAP 3 camera controller.
 
 config VIDEO_OMAP3_DEBUG
 	bool "OMAP 3 Camera debug messages"
 	depends on VIDEO_OMAP3
-	---help---
+	help
 	  Enable debug messages on OMAP 3 camera controller driver.
 
 config VIDEO_PXA27x
@@ -96,7 +96,7 @@ config VIDEO_PXA27x
 	select VIDEOBUF2_DMA_SG
 	select SG_SPLIT
 	select V4L2_FWNODE
-	---help---
+	help
 	  This is a v4l2 driver for the PXA27x Quick Capture Interface
 
 config VIDEO_QCOM_CAMSS
@@ -112,7 +112,7 @@ config VIDEO_S3C_CAMIF
 	depends on PM
 	depends on ARCH_S3C64XX || PLAT_S3C24XX || COMPILE_TEST
 	select VIDEOBUF2_DMA_CONTIG
-	---help---
+	help
 	  This is a v4l2 driver for s3c24xx and s3c64xx SoC series camera
 	  host interface (CAMIF).
 
@@ -125,7 +125,7 @@ config VIDEO_STM32_DCMI
 	depends on ARCH_STM32 || COMPILE_TEST
 	select VIDEOBUF2_DMA_CONTIG
 	select V4L2_FWNODE
-	---help---
+	help
 	  This module makes the STM32 Digital Camera Memory Interface (DCMI)
 	  available as a v4l2 device.
 
@@ -138,7 +138,7 @@ config VIDEO_RENESAS_CEU
 	depends on ARCH_SHMOBILE || ARCH_R7S72100 || COMPILE_TEST
 	select VIDEOBUF2_DMA_CONTIG
 	select V4L2_FWNODE
-	---help---
+	help
 	  This is a v4l2 driver for the Renesas CEU Interface
 
 source "drivers/media/platform/exynos4-is/Kconfig"
@@ -155,7 +155,7 @@ config VIDEO_TI_CAL
 	select VIDEOBUF2_DMA_CONTIG
 	select V4L2_FWNODE
 	default n
-	---help---
+	help
 	  Support for the TI CAL (Camera Adaptation Layer) block
 	  found on DRA72X SoC.
 	  In TI Technical Reference Manual this module is referred as
@@ -168,7 +168,7 @@ menuconfig V4L_MEM2MEM_DRIVERS
 	depends on VIDEO_V4L2
 	depends on MEDIA_CAMERA_SUPPORT
 	default n
-	---help---
+	help
 	  Say Y here to enable selecting drivers for V4L devices that
 	  use system memory for both source and destination buffers, as opposed
 	  to capture and output drivers, which use memory buffers for just
@@ -184,7 +184,7 @@ config VIDEO_CODA
 	select VIDEOBUF2_VMALLOC
 	select V4L2_MEM2MEM_DEV
 	select GENERIC_ALLOCATOR
-	---help---
+	help
 	   Coda is a range of video codec IPs that supports
 	   H.264, MPEG-4, and other video formats.
 
@@ -207,7 +207,7 @@ config VIDEO_MEDIATEK_JPEG
 	depends on ARCH_MEDIATEK || COMPILE_TEST
 	select VIDEOBUF2_DMA_CONTIG
 	select V4L2_MEM2MEM_DEV
-	---help---
+	help
 	  Mediatek jpeg codec driver provides HW capability to decode
 	  JPEG format
 
@@ -218,7 +218,7 @@ config VIDEO_MEDIATEK_VPU
 	tristate "Mediatek Video Processor Unit"
 	depends on VIDEO_DEV && VIDEO_V4L2
 	depends on ARCH_MEDIATEK || COMPILE_TEST
-	---help---
+	help
 	    This driver provides downloading VPU firmware and
 	    communicating with VPU. This driver for hw video
 	    codec embedded in Mediatek's MT8173 SOCs. It is able
@@ -236,7 +236,7 @@ config VIDEO_MEDIATEK_MDP
 	select V4L2_MEM2MEM_DEV
 	select VIDEO_MEDIATEK_VPU
 	default n
-	---help---
+	help
 	    It is a v4l2 driver and present in Mediatek MT8173 SoCs.
 	    The driver supports for scaling and color space conversion.
 
@@ -252,7 +252,7 @@ config VIDEO_MEDIATEK_VCODEC
 	select V4L2_MEM2MEM_DEV
 	select VIDEO_MEDIATEK_VPU
 	default n
-	---help---
+	help
 	    Mediatek video codec driver provides HW capability to
 	    encode and decode in a range of video formats
 	    This driver rely on VPU driver to communicate with VPU.
@@ -276,7 +276,7 @@ config VIDEO_SAMSUNG_S5P_G2D
 	select VIDEOBUF2_DMA_CONTIG
 	select V4L2_MEM2MEM_DEV
 	default n
-	---help---
+	help
 	  This is a v4l2 driver for Samsung S5P and EXYNOS4 G2D
 	  2d graphics accelerator.
 
@@ -286,7 +286,7 @@ config VIDEO_SAMSUNG_S5P_JPEG
 	depends on ARCH_S5PV210 || ARCH_EXYNOS || COMPILE_TEST
 	select VIDEOBUF2_DMA_CONTIG
 	select V4L2_MEM2MEM_DEV
-	---help---
+	help
 	  This is a v4l2 driver for Samsung S5P, EXYNOS3250
 	  and EXYNOS4 JPEG codec
 
@@ -407,7 +407,7 @@ config VIDEO_RENESAS_FDP1
 	depends on (!ARM64 && !VIDEO_RENESAS_FCP) || VIDEO_RENESAS_FCP
 	select VIDEOBUF2_DMA_CONTIG
 	select V4L2_MEM2MEM_DEV
-	---help---
+	help
 	  This is a V4L2 driver for the Renesas Fine Display Processor
 	  providing colour space conversion, and de-interlacing features.
 
@@ -420,7 +420,7 @@ config VIDEO_RENESAS_JPU
 	depends on ARCH_RENESAS || COMPILE_TEST
 	select VIDEOBUF2_DMA_CONTIG
 	select V4L2_MEM2MEM_DEV
-	---help---
+	help
 	  This is a V4L2 driver for the Renesas JPEG Processing Unit.
 
 	  To compile this driver as a module, choose M here: the module
@@ -430,7 +430,7 @@ config VIDEO_RENESAS_FCP
 	tristate "Renesas Frame Compression Processor"
 	depends on ARCH_RENESAS || COMPILE_TEST
 	depends on OF
-	---help---
+	help
 	  This is a driver for the Renesas Frame Compression Processor (FCP).
 	  The FCP is a companion module of video processing modules in the
 	  Renesas R-Car Gen3 SoCs. It handles memory access for the codec,
@@ -446,7 +446,7 @@ config VIDEO_RENESAS_VSP1
 	depends on (!ARM64 && !VIDEO_RENESAS_FCP) || VIDEO_RENESAS_FCP
 	select VIDEOBUF2_DMA_CONTIG
 	select VIDEOBUF2_VMALLOC
-	---help---
+	help
 	  This is a V4L2 driver for the Renesas VSP1 video processing engine.
 
 	  To compile this driver as a module, choose M here: the module
@@ -459,7 +459,7 @@ config VIDEO_ROCKCHIP_RGA
 	select VIDEOBUF2_DMA_SG
 	select V4L2_MEM2MEM_DEV
 	default n
-	---help---
+	help
 	  This is a v4l2 driver for Rockchip SOC RGA 2d graphics accelerator.
 	  Rockchip RGA is a separate 2D raster graphic acceleration unit.
 	  It accelerates 2D graphics operations, such as point/line drawing,
@@ -477,14 +477,14 @@ config VIDEO_TI_VPE
 	select VIDEO_TI_SC
 	select VIDEO_TI_CSC
 	default n
-	---help---
+	help
 	  Support for the TI VPE(Video Processing Engine) block
 	  found on DRA7XX SoC.
 
 config VIDEO_TI_VPE_DEBUG
 	bool "VPE debug messages"
 	depends on VIDEO_TI_VPE
-	---help---
+	help
 	  Enable debug messages on VPE driver.
 
 config VIDEO_QCOM_VENUS
@@ -495,7 +495,7 @@ config VIDEO_QCOM_VENUS
 	select QCOM_SCM if ARCH_QCOM
 	select VIDEOBUF2_DMA_SG
 	select V4L2_MEM2MEM_DEV
-	---help---
+	help
 	  This is a V4L2 driver for Qualcomm Venus video accelerator
 	  hardware. It accelerates encoding and decoding operations
 	  on various Qualcomm SoCs.
@@ -530,7 +530,7 @@ config VIDEO_VIM2M
 	select VIDEOBUF2_VMALLOC
 	select V4L2_MEM2MEM_DEV
 	default n
-	---help---
+	help
 	  This is a virtual test device for the memory-to-memory driver
 	  framework.
 
@@ -542,7 +542,7 @@ menuconfig DVB_PLATFORM_DRIVERS
 	bool "DVB platform devices"
 	depends on MEDIA_DIGITAL_TV_SUPPORT
 	default n
-	---help---
+	help
 	  Say Y here to enable support for platform-specific Digital TV drivers.
 
 if DVB_PLATFORM_DRIVERS
@@ -562,7 +562,7 @@ config VIDEO_CROS_EC_CEC
 	select CEC_NOTIFIER
 	select CHROME_PLATFORMS
 	select CROS_EC_PROTO
-	---help---
+	help
 	  If you say yes here you will get support for the
 	  ChromeOS Embedded Controller's CEC.
 	  The CEC bus is present in the HDMI connector and enables communication
@@ -573,18 +573,34 @@ config VIDEO_MESON_AO_CEC
 	depends on ARCH_MESON || COMPILE_TEST
 	select CEC_CORE
 	select CEC_NOTIFIER
-	---help---
+	help
 	  This is a driver for Amlogic Meson SoCs AO CEC interface. It uses the
 	  generic CEC framework interface.
 	  CEC bus is present in the HDMI connector and enables communication
 
+config VIDEO_MESON_G12A_AO_CEC
+	tristate "Amlogic Meson G12A AO CEC driver"
+	depends on ARCH_MESON || COMPILE_TEST
+	depends on COMMON_CLK && OF
+	select REGMAP
+	select REGMAP_MMIO
+	select CEC_CORE
+	select CEC_NOTIFIER
+	---help---
+	  This is a driver for Amlogic Meson G12A SoCs AO CEC interface.
+	  This driver if for the new AO-CEC module found in G12A SoCs,
+	  usually named AO_CEC_B in documentation.
+	  It uses the generic CEC framework interface.
+	  CEC bus is present in the HDMI connector and enables communication
+	  between compatible devices.
+
 config CEC_GPIO
 	tristate "Generic GPIO-based CEC driver"
 	depends on PREEMPT || COMPILE_TEST
 	select CEC_CORE
 	select CEC_PIN
 	select GPIOLIB
-	---help---
+	help
 	  This is a generic GPIO-based CEC driver.
 	  The CEC bus is present in the HDMI connector and enables communication
 	  between compatible devices.
@@ -594,7 +610,7 @@ config VIDEO_SAMSUNG_S5P_CEC
        depends on ARCH_EXYNOS || COMPILE_TEST
        select CEC_CORE
        select CEC_NOTIFIER
-       ---help---
+       help
 	 This is a driver for Samsung S5P HDMI CEC interface. It uses the
 	 generic CEC framework interface.
 	 CEC bus is present in the HDMI connector and enables communication
@@ -605,7 +621,7 @@ config VIDEO_STI_HDMI_CEC
        depends on ARCH_STI || COMPILE_TEST
        select CEC_CORE
        select CEC_NOTIFIER
-       ---help---
+       help
 	 This is a driver for STIH4xx HDMI CEC interface. It uses the
 	 generic CEC framework interface.
 	 CEC bus is present in the HDMI connector and enables communication
@@ -617,7 +633,7 @@ config VIDEO_STM32_HDMI_CEC
        select REGMAP
        select REGMAP_MMIO
        select CEC_CORE
-       ---help---
+       help
 	 This is a driver for STM32 interface. It uses the
 	 generic CEC framework interface.
 	 CEC bus is present in the HDMI connector and enables communication
@@ -628,7 +644,7 @@ config VIDEO_TEGRA_HDMI_CEC
        depends on ARCH_TEGRA || COMPILE_TEST
        select CEC_CORE
        select CEC_NOTIFIER
-       ---help---
+       help
 	 This is a driver for the Tegra HDMI CEC interface. It uses the
 	 generic CEC framework interface.
 	 The CEC bus is present in the HDMI connector and enables communication
@@ -649,7 +665,7 @@ config VIDEO_SECO_CEC
 config VIDEO_SECO_RC
 	bool "SECO Boards IR RC5 support"
 	depends on VIDEO_SECO_CEC
-	depends on RC_CORE
+	depends on RC_CORE=y || RC_CORE = VIDEO_SECO_CEC
 	help
 	  If you say yes here you will get support for the
 	  SECO Boards Consumer-IR in seco-cec driver.
@@ -662,7 +678,7 @@ menuconfig SDR_PLATFORM_DRIVERS
 	bool "SDR platform devices"
 	depends on MEDIA_SDR_SUPPORT
 	default n
-	---help---
+	help
 	  Say Y here to enable support for platform-specific SDR Drivers.
 
 if SDR_PLATFORM_DRIVERS
@@ -672,7 +688,7 @@ config VIDEO_RCAR_DRIF
 	depends on VIDEO_V4L2
 	depends on ARCH_RENESAS || COMPILE_TEST
 	select VIDEOBUF2_VMALLOC
-	---help---
+	help
 	  Say Y if you want to enable R-Car Gen3 DRIF support. DRIF is Digital
 	  Radio Interface that interfaces with an RF front end chip. It is a
 	  receiver of digital data which uses DMA to transfer received data to
diff --git a/drivers/media/platform/am437x/am437x-vpfe.c b/drivers/media/platform/am437x/am437x-vpfe.c
index 5c17624aaade..fe7b937eb5f2 100644
--- a/drivers/media/platform/am437x/am437x-vpfe.c
+++ b/drivers/media/platform/am437x/am437x-vpfe.c
@@ -1540,7 +1540,7 @@ static int vpfe_enum_fmt(struct file *file, void  *priv,
 	if (!fmt)
 		return -EINVAL;
 
-	strncpy(f->description, fmt->name, sizeof(f->description) - 1);
+	strscpy(f->description, fmt->name, sizeof(f->description));
 	f->pixelformat = fmt->fourcc;
 	f->type = vpfe->fmt.type;
 
diff --git a/drivers/media/platform/aspeed-video.c b/drivers/media/platform/aspeed-video.c
index 692e08ef38c0..8144fe36ad48 100644
--- a/drivers/media/platform/aspeed-video.c
+++ b/drivers/media/platform/aspeed-video.c
@@ -14,7 +14,6 @@
 #include <linux/of_irq.h>
 #include <linux/of_reserved_mem.h>
 #include <linux/platform_device.h>
-#include <linux/reset.h>
 #include <linux/sched.h>
 #include <linux/spinlock.h>
 #include <linux/string.h>
@@ -208,7 +207,6 @@ struct aspeed_video {
 	void __iomem *base;
 	struct clk *eclk;
 	struct clk *vclk;
-	struct reset_control *rst;
 
 	struct device *dev;
 	struct v4l2_ctrl_handler ctrl_handler;
@@ -483,19 +481,10 @@ static void aspeed_video_enable_mode_detect(struct aspeed_video *video)
 	aspeed_video_update(video, VE_SEQ_CTRL, 0, VE_SEQ_CTRL_TRIG_MODE_DET);
 }
 
-static void aspeed_video_reset(struct aspeed_video *video)
-{
-	/* Reset the engine */
-	reset_control_assert(video->rst);
-
-	/* Don't usleep here; function may be called in interrupt context */
-	udelay(100);
-	reset_control_deassert(video->rst);
-}
-
 static void aspeed_video_off(struct aspeed_video *video)
 {
-	aspeed_video_reset(video);
+	/* Disable interrupts */
+	aspeed_video_write(video, VE_INTERRUPT_CTRL, 0);
 
 	/* Turn off the relevant clocks */
 	clk_disable_unprepare(video->vclk);
@@ -507,8 +496,6 @@ static void aspeed_video_on(struct aspeed_video *video)
 	/* Turn on the relevant clocks */
 	clk_prepare_enable(video->eclk);
 	clk_prepare_enable(video->vclk);
-
-	aspeed_video_reset(video);
 }
 
 static void aspeed_video_bufs_done(struct aspeed_video *video,
@@ -1464,7 +1451,9 @@ static void aspeed_video_stop_streaming(struct vb2_queue *q)
 		 * Need to force stop any DMA and try and get HW into a good
 		 * state for future calls to start streaming again.
 		 */
-		aspeed_video_reset(video);
+		aspeed_video_off(video);
+		aspeed_video_on(video);
+
 		aspeed_video_init_regs(video);
 
 		aspeed_video_get_resolution(video);
@@ -1619,17 +1608,7 @@ static int aspeed_video_init(struct aspeed_video *video)
 		return PTR_ERR(video->vclk);
 	}
 
-	video->rst = devm_reset_control_get_exclusive(dev, NULL);
-	if (IS_ERR(video->rst)) {
-		dev_err(dev, "Unable to get VE reset\n");
-		return PTR_ERR(video->rst);
-	}
-
-	rc = of_reserved_mem_device_init(dev);
-	if (rc) {
-		dev_err(dev, "Unable to reserve memory\n");
-		return rc;
-	}
+	of_reserved_mem_device_init(dev);
 
 	rc = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
 	if (rc) {
diff --git a/drivers/media/platform/atmel/Kconfig b/drivers/media/platform/atmel/Kconfig
index a211ef20f77e..c3f6a47cdc0e 100644
--- a/drivers/media/platform/atmel/Kconfig
+++ b/drivers/media/platform/atmel/Kconfig
@@ -15,6 +15,6 @@ config VIDEO_ATMEL_ISI
 	depends on ARCH_AT91 || COMPILE_TEST
 	select VIDEOBUF2_DMA_CONTIG
 	select V4L2_FWNODE
-	---help---
+	help
 	  This module makes the ATMEL Image Sensor Interface available
 	  as a v4l2 device.
diff --git a/drivers/media/platform/atmel/atmel-isc-regs.h b/drivers/media/platform/atmel/atmel-isc-regs.h
index 2aadc19235ea..d730693f299c 100644
--- a/drivers/media/platform/atmel/atmel-isc-regs.h
+++ b/drivers/media/platform/atmel/atmel-isc-regs.h
@@ -24,6 +24,8 @@
 #define ISC_PFE_CFG0_HPOL_LOW   BIT(0)
 #define ISC_PFE_CFG0_VPOL_LOW   BIT(1)
 #define ISC_PFE_CFG0_PPOL_LOW   BIT(2)
+#define ISC_PFE_CFG0_CCIR656    BIT(9)
+#define ISC_PFE_CFG0_CCIR_CRC   BIT(10)
 
 #define ISC_PFE_CFG0_MODE_PROGRESSIVE   (0x0 << 4)
 #define ISC_PFE_CFG0_MODE_MASK          GENMASK(6, 4)
diff --git a/drivers/media/platform/atmel/atmel-isc.c b/drivers/media/platform/atmel/atmel-isc.c
index 50178968b8a6..4bba9da206e4 100644
--- a/drivers/media/platform/atmel/atmel-isc.c
+++ b/drivers/media/platform/atmel/atmel-isc.c
@@ -89,35 +89,25 @@ struct isc_subdev_entity {
 	struct list_head list;
 };
 
-/* Indicate the format is generated by the sensor */
-#define FMT_FLAG_FROM_SENSOR		BIT(0)
-/* Indicate the format is produced by ISC itself */
-#define FMT_FLAG_FROM_CONTROLLER	BIT(1)
-/* Indicate a Raw Bayer format */
-#define FMT_FLAG_RAW_FORMAT		BIT(2)
-
-#define FMT_FLAG_RAW_FROM_SENSOR	(FMT_FLAG_FROM_SENSOR | \
-					 FMT_FLAG_RAW_FORMAT)
-
 /*
  * struct isc_format - ISC media bus format information
+			This structure represents the interface between the ISC
+			and the sensor. It's the input format received by
+			the ISC.
  * @fourcc:		Fourcc code for this format
  * @mbus_code:		V4L2 media bus format code.
- * flags:		Indicate format from sensor or converted by controller
- * @bpp:		Bits per pixel (when stored in memory)
- *			(when transferred over a bus)
- * @sd_support:		Subdev supports this format
- * @isc_support:	ISC can convert raw format to this format
+ * @cfa_baycfg:		If this format is RAW BAYER, indicate the type of bayer.
+			this is either BGBG, RGRG, etc.
+ * @pfe_cfg0_bps:	Number of hardware data lines connected to the ISC
  */
 
 struct isc_format {
 	u32	fourcc;
 	u32	mbus_code;
-	u32	flags;
-	u8	bpp;
+	u32	cfa_baycfg;
 
 	bool	sd_support;
-	bool	isc_support;
+	u32	pfe_cfg0_bps;
 };
 
 /* Pipeline bitmap */
@@ -135,16 +125,31 @@ struct isc_format {
 
 #define GAM_ENABLES	(GAM_RENABLE | GAM_GENABLE | GAM_BENABLE | GAM_ENABLE)
 
+/*
+ * struct fmt_config - ISC format configuration and internal pipeline
+			This structure represents the internal configuration
+			of the ISC.
+			It also holds the format that ISC will present to v4l2.
+ * @sd_format:		Pointer to an isc_format struct that holds the sensor
+			configuration.
+ * @fourcc:		Fourcc code for this format.
+ * @bpp:		Bytes per pixel in the current format.
+ * @rlp_cfg_mode:	Configuration of the RLP (rounding, limiting packaging)
+ * @dcfg_imode:		Configuration of the input of the DMA module
+ * @dctrl_dview:	Configuration of the output of the DMA module
+ * @bits_pipeline:	Configuration of the pipeline, which modules are enabled
+ */
 struct fmt_config {
-	u32	fourcc;
+	struct isc_format	*sd_format;
 
-	u32	pfe_cfg0_bps;
-	u32	cfa_baycfg;
-	u32	rlp_cfg_mode;
-	u32	dcfg_imode;
-	u32	dctrl_dview;
+	u32			fourcc;
+	u8			bpp;
 
-	u32	bits_pipeline;
+	u32			rlp_cfg_mode;
+	u32			dcfg_imode;
+	u32			dctrl_dview;
+
+	u32			bits_pipeline;
 };
 
 #define HIST_ENTRIES		512
@@ -196,8 +201,9 @@ struct isc_device {
 	struct v4l2_format	fmt;
 	struct isc_format	**user_formats;
 	unsigned int		num_user_formats;
-	const struct isc_format	*current_fmt;
-	const struct isc_format	*raw_fmt;
+
+	struct fmt_config	config;
+	struct fmt_config	try_config;
 
 	struct isc_ctrls	ctrls;
 	struct work_struct	awb_work;
@@ -210,319 +216,125 @@ struct isc_device {
 	struct list_head		subdev_entities;
 };
 
-static struct isc_format formats_list[] = {
-	{
-		.fourcc		= V4L2_PIX_FMT_SBGGR8,
-		.mbus_code	= MEDIA_BUS_FMT_SBGGR8_1X8,
-		.flags		= FMT_FLAG_RAW_FROM_SENSOR,
-		.bpp		= 8,
-	},
+/* This is a list of the formats that the ISC can *output* */
+static struct isc_format controller_formats[] = {
 	{
-		.fourcc		= V4L2_PIX_FMT_SGBRG8,
-		.mbus_code	= MEDIA_BUS_FMT_SGBRG8_1X8,
-		.flags		= FMT_FLAG_RAW_FROM_SENSOR,
-		.bpp		= 8,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_SGRBG8,
-		.mbus_code	= MEDIA_BUS_FMT_SGRBG8_1X8,
-		.flags		= FMT_FLAG_RAW_FROM_SENSOR,
-		.bpp		= 8,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_SRGGB8,
-		.mbus_code	= MEDIA_BUS_FMT_SRGGB8_1X8,
-		.flags		= FMT_FLAG_RAW_FROM_SENSOR,
-		.bpp		= 8,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_SBGGR10,
-		.mbus_code	= MEDIA_BUS_FMT_SBGGR10_1X10,
-		.flags		= FMT_FLAG_RAW_FROM_SENSOR,
-		.bpp		= 16,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_SGBRG10,
-		.mbus_code	= MEDIA_BUS_FMT_SGBRG10_1X10,
-		.flags		= FMT_FLAG_RAW_FROM_SENSOR,
-		.bpp		= 16,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_SGRBG10,
-		.mbus_code	= MEDIA_BUS_FMT_SGRBG10_1X10,
-		.flags		= FMT_FLAG_RAW_FROM_SENSOR,
-		.bpp		= 16,
+		.fourcc		= V4L2_PIX_FMT_ARGB444,
 	},
 	{
-		.fourcc		= V4L2_PIX_FMT_SRGGB10,
-		.mbus_code	= MEDIA_BUS_FMT_SRGGB10_1X10,
-		.flags		= FMT_FLAG_RAW_FROM_SENSOR,
-		.bpp		= 16,
+		.fourcc		= V4L2_PIX_FMT_ARGB555,
 	},
 	{
-		.fourcc		= V4L2_PIX_FMT_SBGGR12,
-		.mbus_code	= MEDIA_BUS_FMT_SBGGR12_1X12,
-		.flags		= FMT_FLAG_RAW_FROM_SENSOR,
-		.bpp		= 16,
+		.fourcc		= V4L2_PIX_FMT_RGB565,
 	},
 	{
-		.fourcc		= V4L2_PIX_FMT_SGBRG12,
-		.mbus_code	= MEDIA_BUS_FMT_SGBRG12_1X12,
-		.flags		= FMT_FLAG_RAW_FROM_SENSOR,
-		.bpp		= 16,
+		.fourcc		= V4L2_PIX_FMT_ABGR32,
 	},
 	{
-		.fourcc		= V4L2_PIX_FMT_SGRBG12,
-		.mbus_code	= MEDIA_BUS_FMT_SGRBG12_1X12,
-		.flags		= FMT_FLAG_RAW_FROM_SENSOR,
-		.bpp		= 16,
+		.fourcc		= V4L2_PIX_FMT_XBGR32,
 	},
 	{
-		.fourcc		= V4L2_PIX_FMT_SRGGB12,
-		.mbus_code	= MEDIA_BUS_FMT_SRGGB12_1X12,
-		.flags		= FMT_FLAG_RAW_FROM_SENSOR,
-		.bpp		= 16,
+		.fourcc		= V4L2_PIX_FMT_YUV420,
 	},
 	{
-		.fourcc		= V4L2_PIX_FMT_YUV420,
-		.mbus_code	= 0x0,
-		.flags		= FMT_FLAG_FROM_CONTROLLER,
-		.bpp		= 12,
+		.fourcc		= V4L2_PIX_FMT_YUYV,
 	},
 	{
 		.fourcc		= V4L2_PIX_FMT_YUV422P,
-		.mbus_code	= 0x0,
-		.flags		= FMT_FLAG_FROM_CONTROLLER,
-		.bpp		= 16,
 	},
 	{
 		.fourcc		= V4L2_PIX_FMT_GREY,
-		.mbus_code	= MEDIA_BUS_FMT_Y8_1X8,
-		.flags		= FMT_FLAG_FROM_CONTROLLER |
-				  FMT_FLAG_FROM_SENSOR,
-		.bpp		= 8,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_ARGB444,
-		.mbus_code	= MEDIA_BUS_FMT_RGB444_2X8_PADHI_LE,
-		.flags		= FMT_FLAG_FROM_CONTROLLER,
-		.bpp		= 16,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_ARGB555,
-		.mbus_code	= MEDIA_BUS_FMT_RGB555_2X8_PADHI_LE,
-		.flags		= FMT_FLAG_FROM_CONTROLLER,
-		.bpp		= 16,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_RGB565,
-		.mbus_code	= MEDIA_BUS_FMT_RGB565_2X8_LE,
-		.flags		= FMT_FLAG_FROM_CONTROLLER,
-		.bpp		= 16,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_ARGB32,
-		.mbus_code	= MEDIA_BUS_FMT_ARGB8888_1X32,
-		.flags		= FMT_FLAG_FROM_CONTROLLER,
-		.bpp		= 32,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_YUYV,
-		.mbus_code	= MEDIA_BUS_FMT_YUYV8_2X8,
-		.flags		= FMT_FLAG_FROM_CONTROLLER |
-				  FMT_FLAG_FROM_SENSOR,
-		.bpp		= 16,
 	},
 };
 
-static struct fmt_config fmt_configs_list[] = {
+/* This is a list of formats that the ISC can receive as *input* */
+static struct isc_format formats_list[] = {
 	{
 		.fourcc		= V4L2_PIX_FMT_SBGGR8,
+		.mbus_code	= MEDIA_BUS_FMT_SBGGR8_1X8,
 		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_EIGHT,
 		.cfa_baycfg	= ISC_BAY_CFG_BGBG,
-		.rlp_cfg_mode	= ISC_RLP_CFG_MODE_DAT8,
-		.dcfg_imode	= ISC_DCFG_IMODE_PACKED8,
-		.dctrl_dview	= ISC_DCTRL_DVIEW_PACKED,
-		.bits_pipeline	= 0x0,
 	},
 	{
 		.fourcc		= V4L2_PIX_FMT_SGBRG8,
+		.mbus_code	= MEDIA_BUS_FMT_SGBRG8_1X8,
 		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_EIGHT,
 		.cfa_baycfg	= ISC_BAY_CFG_GBGB,
-		.rlp_cfg_mode	= ISC_RLP_CFG_MODE_DAT8,
-		.dcfg_imode	= ISC_DCFG_IMODE_PACKED8,
-		.dctrl_dview	= ISC_DCTRL_DVIEW_PACKED,
-		.bits_pipeline	= 0x0,
 	},
 	{
 		.fourcc		= V4L2_PIX_FMT_SGRBG8,
+		.mbus_code	= MEDIA_BUS_FMT_SGRBG8_1X8,
 		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_EIGHT,
 		.cfa_baycfg	= ISC_BAY_CFG_GRGR,
-		.rlp_cfg_mode	= ISC_RLP_CFG_MODE_DAT8,
-		.dcfg_imode	= ISC_DCFG_IMODE_PACKED8,
-		.dctrl_dview	= ISC_DCTRL_DVIEW_PACKED,
-		.bits_pipeline	= 0x0,
 	},
 	{
 		.fourcc		= V4L2_PIX_FMT_SRGGB8,
+		.mbus_code	= MEDIA_BUS_FMT_SRGGB8_1X8,
 		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_EIGHT,
 		.cfa_baycfg	= ISC_BAY_CFG_RGRG,
-		.rlp_cfg_mode	= ISC_RLP_CFG_MODE_DAT8,
-		.dcfg_imode	= ISC_DCFG_IMODE_PACKED8,
-		.dctrl_dview	= ISC_DCTRL_DVIEW_PACKED,
-		.bits_pipeline	= 0x0,
 	},
 	{
 		.fourcc		= V4L2_PIX_FMT_SBGGR10,
+		.mbus_code	= MEDIA_BUS_FMT_SBGGR10_1X10,
 		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TEN,
-		.cfa_baycfg	= ISC_BAY_CFG_BGBG,
-		.rlp_cfg_mode	= ISC_RLP_CFG_MODE_DAT10,
-		.dcfg_imode	= ISC_DCFG_IMODE_PACKED16,
-		.dctrl_dview	= ISC_DCTRL_DVIEW_PACKED,
-		.bits_pipeline	= 0x0,
+		.cfa_baycfg	= ISC_BAY_CFG_RGRG,
 	},
 	{
 		.fourcc		= V4L2_PIX_FMT_SGBRG10,
+		.mbus_code	= MEDIA_BUS_FMT_SGBRG10_1X10,
 		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TEN,
 		.cfa_baycfg	= ISC_BAY_CFG_GBGB,
-		.rlp_cfg_mode	= ISC_RLP_CFG_MODE_DAT10,
-		.dcfg_imode	= ISC_DCFG_IMODE_PACKED16,
-		.dctrl_dview	= ISC_DCTRL_DVIEW_PACKED,
-		.bits_pipeline	= 0x0,
 	},
 	{
 		.fourcc		= V4L2_PIX_FMT_SGRBG10,
+		.mbus_code	= MEDIA_BUS_FMT_SGRBG10_1X10,
 		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TEN,
 		.cfa_baycfg	= ISC_BAY_CFG_GRGR,
-		.rlp_cfg_mode	= ISC_RLP_CFG_MODE_DAT10,
-		.dcfg_imode	= ISC_DCFG_IMODE_PACKED16,
-		.dctrl_dview	= ISC_DCTRL_DVIEW_PACKED,
-		.bits_pipeline	= 0x0,
 	},
 	{
 		.fourcc		= V4L2_PIX_FMT_SRGGB10,
+		.mbus_code	= MEDIA_BUS_FMT_SRGGB10_1X10,
 		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TEN,
 		.cfa_baycfg	= ISC_BAY_CFG_RGRG,
-		.rlp_cfg_mode	= ISC_RLP_CFG_MODE_DAT10,
-		.dcfg_imode	= ISC_DCFG_IMODE_PACKED16,
-		.dctrl_dview	= ISC_DCTRL_DVIEW_PACKED,
-		.bits_pipeline	= 0x0,
 	},
 	{
 		.fourcc		= V4L2_PIX_FMT_SBGGR12,
+		.mbus_code	= MEDIA_BUS_FMT_SBGGR12_1X12,
 		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TWELVE,
 		.cfa_baycfg	= ISC_BAY_CFG_BGBG,
-		.rlp_cfg_mode	= ISC_RLP_CFG_MODE_DAT12,
-		.dcfg_imode	= ISC_DCFG_IMODE_PACKED16,
-		.dctrl_dview	= ISC_DCTRL_DVIEW_PACKED,
-		.bits_pipeline	= 0x0,
 	},
 	{
 		.fourcc		= V4L2_PIX_FMT_SGBRG12,
+		.mbus_code	= MEDIA_BUS_FMT_SGBRG12_1X12,
 		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TWELVE,
 		.cfa_baycfg	= ISC_BAY_CFG_GBGB,
-		.rlp_cfg_mode	= ISC_RLP_CFG_MODE_DAT12,
-		.dcfg_imode	= ISC_DCFG_IMODE_PACKED16,
-		.dctrl_dview	= ISC_DCTRL_DVIEW_PACKED,
-		.bits_pipeline	= 0x0
 	},
 	{
 		.fourcc		= V4L2_PIX_FMT_SGRBG12,
+		.mbus_code	= MEDIA_BUS_FMT_SGRBG12_1X12,
 		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TWELVE,
 		.cfa_baycfg	= ISC_BAY_CFG_GRGR,
-		.rlp_cfg_mode	= ISC_RLP_CFG_MODE_DAT12,
-		.dcfg_imode	= ISC_DCFG_IMODE_PACKED16,
-		.dctrl_dview	= ISC_DCTRL_DVIEW_PACKED,
-		.bits_pipeline	= 0x0,
 	},
 	{
 		.fourcc		= V4L2_PIX_FMT_SRGGB12,
+		.mbus_code	= MEDIA_BUS_FMT_SRGGB12_1X12,
 		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TWELVE,
 		.cfa_baycfg	= ISC_BAY_CFG_RGRG,
-		.rlp_cfg_mode	= ISC_RLP_CFG_MODE_DAT12,
-		.dcfg_imode	= ISC_DCFG_IMODE_PACKED16,
-		.dctrl_dview	= ISC_DCTRL_DVIEW_PACKED,
-		.bits_pipeline	= 0x0,
-	},
-	{
-		.fourcc = V4L2_PIX_FMT_YUV420,
-		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_EIGHT,
-		.cfa_baycfg	= ISC_BAY_CFG_BGBG,
-		.rlp_cfg_mode	= ISC_RLP_CFG_MODE_YYCC,
-		.dcfg_imode	= ISC_DCFG_IMODE_YC420P,
-		.dctrl_dview	= ISC_DCTRL_DVIEW_PLANAR,
-		.bits_pipeline	= SUB420_ENABLE | SUB422_ENABLE |
-				  CBC_ENABLE | CSC_ENABLE |
-				  GAM_ENABLES |
-				  CFA_ENABLE | WB_ENABLE,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_YUV422P,
-		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_EIGHT,
-		.cfa_baycfg	= ISC_BAY_CFG_BGBG,
-		.rlp_cfg_mode	= ISC_RLP_CFG_MODE_YYCC,
-		.dcfg_imode	= ISC_DCFG_IMODE_YC422P,
-		.dctrl_dview	= ISC_DCTRL_DVIEW_PLANAR,
-		.bits_pipeline	= SUB422_ENABLE |
-				  CBC_ENABLE | CSC_ENABLE |
-				  GAM_ENABLES |
-				  CFA_ENABLE | WB_ENABLE,
 	},
 	{
 		.fourcc		= V4L2_PIX_FMT_GREY,
+		.mbus_code	= MEDIA_BUS_FMT_Y8_1X8,
 		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_EIGHT,
-		.cfa_baycfg	= ISC_BAY_CFG_BGBG,
-		.rlp_cfg_mode	= ISC_RLP_CFG_MODE_DATY8,
-		.dcfg_imode	= ISC_DCFG_IMODE_PACKED8,
-		.dctrl_dview	= ISC_DCTRL_DVIEW_PACKED,
-		.bits_pipeline	= CBC_ENABLE | CSC_ENABLE |
-				  GAM_ENABLES |
-				  CFA_ENABLE | WB_ENABLE,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_ARGB444,
-		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_EIGHT,
-		.cfa_baycfg	= ISC_BAY_CFG_BGBG,
-		.rlp_cfg_mode	= ISC_RLP_CFG_MODE_ARGB444,
-		.dcfg_imode	= ISC_DCFG_IMODE_PACKED16,
-		.dctrl_dview	= ISC_DCTRL_DVIEW_PACKED,
-		.bits_pipeline	= GAM_ENABLES | CFA_ENABLE | WB_ENABLE,
 	},
 	{
-		.fourcc		= V4L2_PIX_FMT_ARGB555,
+		.fourcc		= V4L2_PIX_FMT_YUYV,
+		.mbus_code	= MEDIA_BUS_FMT_YUYV8_2X8,
 		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_EIGHT,
-		.cfa_baycfg	= ISC_BAY_CFG_BGBG,
-		.rlp_cfg_mode	= ISC_RLP_CFG_MODE_ARGB555,
-		.dcfg_imode	= ISC_DCFG_IMODE_PACKED16,
-		.dctrl_dview	= ISC_DCTRL_DVIEW_PACKED,
-		.bits_pipeline	= GAM_ENABLES | CFA_ENABLE | WB_ENABLE,
 	},
 	{
 		.fourcc		= V4L2_PIX_FMT_RGB565,
+		.mbus_code	= MEDIA_BUS_FMT_RGB565_2X8_LE,
 		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_EIGHT,
-		.cfa_baycfg	= ISC_BAY_CFG_BGBG,
-		.rlp_cfg_mode	= ISC_RLP_CFG_MODE_RGB565,
-		.dcfg_imode	= ISC_DCFG_IMODE_PACKED16,
-		.dctrl_dview	= ISC_DCTRL_DVIEW_PACKED,
-		.bits_pipeline	= GAM_ENABLES | CFA_ENABLE | WB_ENABLE,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_ARGB32,
-		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_EIGHT,
-		.cfa_baycfg	= ISC_BAY_CFG_BGBG,
-		.rlp_cfg_mode	= ISC_RLP_CFG_MODE_ARGB32,
-		.dcfg_imode	= ISC_DCFG_IMODE_PACKED32,
-		.dctrl_dview	= ISC_DCTRL_DVIEW_PACKED,
-		.bits_pipeline	= GAM_ENABLES | CFA_ENABLE | WB_ENABLE,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_YUYV,
-		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_EIGHT,
-		.cfa_baycfg	= ISC_BAY_CFG_BGBG,
-		.rlp_cfg_mode	= ISC_RLP_CFG_MODE_DAT8,
-		.dcfg_imode	= ISC_DCFG_IMODE_PACKED8,
-		.dctrl_dview	= ISC_DCTRL_DVIEW_PACKED,
-		.bits_pipeline	= 0x0
 	},
 };
 
@@ -571,6 +383,13 @@ static const u32 isc_gamma_table[GAMMA_MAX + 1][GAMMA_ENTRIES] = {
 	  0x3E20007, 0x3E90007, 0x3F00008, 0x3F80007 },
 };
 
+#define ISC_IS_FORMAT_RAW(mbus_code) \
+	(((mbus_code) & 0xf000) == 0x3000)
+
+static unsigned int debug;
+module_param(debug, int, 0644);
+MODULE_PARM_DESC(debug, "debug level (0-2)");
+
 static unsigned int sensor_preferred = 1;
 module_param(sensor_preferred, uint, 0644);
 MODULE_PARM_DESC(sensor_preferred,
@@ -896,40 +715,17 @@ static int isc_buffer_prepare(struct vb2_buffer *vb)
 	return 0;
 }
 
-static inline bool sensor_is_preferred(const struct isc_format *isc_fmt)
-{
-	return (sensor_preferred && isc_fmt->sd_support) ||
-		!isc_fmt->isc_support;
-}
-
-static struct fmt_config *get_fmt_config(u32 fourcc)
-{
-	struct fmt_config *config;
-	int i;
-
-	config = &fmt_configs_list[0];
-	for (i = 0; i < ARRAY_SIZE(fmt_configs_list); i++) {
-		if (config->fourcc == fourcc)
-			return config;
-
-		config++;
-	}
-	return NULL;
-}
-
 static void isc_start_dma(struct isc_device *isc)
 {
 	struct regmap *regmap = isc->regmap;
-	struct v4l2_pix_format *pixfmt = &isc->fmt.fmt.pix;
-	u32 sizeimage = pixfmt->sizeimage;
-	struct fmt_config *config = get_fmt_config(isc->current_fmt->fourcc);
+	u32 sizeimage = isc->fmt.fmt.pix.sizeimage;
 	u32 dctrl_dview;
 	dma_addr_t addr0;
 
 	addr0 = vb2_dma_contig_plane_dma_addr(&isc->cur_frm->vb.vb2_buf, 0);
 	regmap_write(regmap, ISC_DAD0, addr0);
 
-	switch (pixfmt->pixelformat) {
+	switch (isc->config.fourcc) {
 	case V4L2_PIX_FMT_YUV420:
 		regmap_write(regmap, ISC_DAD1, addr0 + (sizeimage * 2) / 3);
 		regmap_write(regmap, ISC_DAD2, addr0 + (sizeimage * 5) / 6);
@@ -942,10 +738,7 @@ static void isc_start_dma(struct isc_device *isc)
 		break;
 	}
 
-	if (sensor_is_preferred(isc->current_fmt))
-		dctrl_dview = ISC_DCTRL_DVIEW_PACKED;
-	else
-		dctrl_dview = config->dctrl_dview;
+	dctrl_dview = isc->config.dctrl_dview;
 
 	regmap_write(regmap, ISC_DCTRL, dctrl_dview | ISC_DCTRL_IE_IS);
 	regmap_write(regmap, ISC_CTRLEN, ISC_CTRL_CAPTURE);
@@ -955,7 +748,6 @@ static void isc_set_pipeline(struct isc_device *isc, u32 pipeline)
 {
 	struct regmap *regmap = isc->regmap;
 	struct isc_ctrls *ctrls = &isc->ctrls;
-	struct fmt_config *config = get_fmt_config(isc->raw_fmt->fourcc);
 	u32 val, bay_cfg;
 	const u32 *gamma;
 	unsigned int i;
@@ -969,7 +761,7 @@ static void isc_set_pipeline(struct isc_device *isc, u32 pipeline)
 	if (!pipeline)
 		return;
 
-	bay_cfg = config->cfa_baycfg;
+	bay_cfg = isc->config.sd_format->cfa_baycfg;
 
 	regmap_write(regmap, ISC_WB_CFG, bay_cfg);
 	regmap_write(regmap, ISC_WB_O_RGR, 0x0);
@@ -1011,24 +803,24 @@ static int isc_update_profile(struct isc_device *isc)
 	}
 
 	if (counter < 0) {
-		v4l2_warn(&isc->v4l2_dev, "Time out to update profie\n");
+		v4l2_warn(&isc->v4l2_dev, "Time out to update profile\n");
 		return -ETIMEDOUT;
 	}
 
 	return 0;
 }
 
-static void isc_set_histogram(struct isc_device *isc)
+static void isc_set_histogram(struct isc_device *isc, bool enable)
 {
 	struct regmap *regmap = isc->regmap;
 	struct isc_ctrls *ctrls = &isc->ctrls;
-	struct fmt_config *config = get_fmt_config(isc->raw_fmt->fourcc);
 
-	if (ctrls->awb && (ctrls->hist_stat != HIST_ENABLED)) {
+	if (enable) {
 		regmap_write(regmap, ISC_HIS_CFG,
 			     ISC_HIS_CFG_MODE_R |
-			     (config->cfa_baycfg << ISC_HIS_CFG_BAYSEL_SHIFT) |
-			     ISC_HIS_CFG_RAR);
+			     (isc->config.sd_format->cfa_baycfg
+					<< ISC_HIS_CFG_BAYSEL_SHIFT) |
+					ISC_HIS_CFG_RAR);
 		regmap_write(regmap, ISC_HIS_CTRL, ISC_HIS_CTRL_EN);
 		regmap_write(regmap, ISC_INTEN, ISC_INT_HISDONE);
 		ctrls->hist_id = ISC_HIS_CFG_MODE_R;
@@ -1036,7 +828,7 @@ static void isc_set_histogram(struct isc_device *isc)
 		regmap_write(regmap, ISC_CTRLEN, ISC_CTRL_HISREQ);
 
 		ctrls->hist_stat = HIST_ENABLED;
-	} else if (!ctrls->awb && (ctrls->hist_stat != HIST_DISABLED)) {
+	} else {
 		regmap_write(regmap, ISC_INTDIS, ISC_INT_HISDONE);
 		regmap_write(regmap, ISC_HIS_CTRL, ISC_HIS_CTRL_DIS);
 
@@ -1044,58 +836,24 @@ static void isc_set_histogram(struct isc_device *isc)
 	}
 }
 
-static inline void isc_get_param(const struct isc_format *fmt,
-				 u32 *rlp_mode, u32 *dcfg)
-{
-	struct fmt_config *config = get_fmt_config(fmt->fourcc);
-
-	*dcfg = ISC_DCFG_YMBSIZE_BEATS8;
-
-	switch (fmt->fourcc) {
-	case V4L2_PIX_FMT_SBGGR10:
-	case V4L2_PIX_FMT_SGBRG10:
-	case V4L2_PIX_FMT_SGRBG10:
-	case V4L2_PIX_FMT_SRGGB10:
-	case V4L2_PIX_FMT_SBGGR12:
-	case V4L2_PIX_FMT_SGBRG12:
-	case V4L2_PIX_FMT_SGRBG12:
-	case V4L2_PIX_FMT_SRGGB12:
-		*rlp_mode = config->rlp_cfg_mode;
-		*dcfg |= config->dcfg_imode;
-		break;
-	default:
-		*rlp_mode = ISC_RLP_CFG_MODE_DAT8;
-		*dcfg |= ISC_DCFG_IMODE_PACKED8;
-		break;
-	}
-}
-
 static int isc_configure(struct isc_device *isc)
 {
 	struct regmap *regmap = isc->regmap;
-	const struct isc_format *current_fmt = isc->current_fmt;
-	struct fmt_config *curfmt_config = get_fmt_config(current_fmt->fourcc);
-	struct fmt_config *rawfmt_config = get_fmt_config(isc->raw_fmt->fourcc);
-	struct isc_subdev_entity *subdev = isc->current_subdev;
 	u32 pfe_cfg0, rlp_mode, dcfg, mask, pipeline;
+	struct isc_subdev_entity *subdev = isc->current_subdev;
 
-	if (sensor_is_preferred(current_fmt)) {
-		pfe_cfg0 = curfmt_config->pfe_cfg0_bps;
-		pipeline = 0x0;
-		isc_get_param(current_fmt, &rlp_mode, &dcfg);
-		isc->ctrls.hist_stat = HIST_INIT;
-	} else {
-		pfe_cfg0 = rawfmt_config->pfe_cfg0_bps;
-		pipeline = curfmt_config->bits_pipeline;
-		rlp_mode = curfmt_config->rlp_cfg_mode;
-		dcfg = curfmt_config->dcfg_imode |
+	pfe_cfg0 = isc->config.sd_format->pfe_cfg0_bps;
+	rlp_mode = isc->config.rlp_cfg_mode;
+	pipeline = isc->config.bits_pipeline;
+
+	dcfg = isc->config.dcfg_imode |
 		       ISC_DCFG_YMBSIZE_BEATS8 | ISC_DCFG_CMBSIZE_BEATS8;
-	}
 
 	pfe_cfg0  |= subdev->pfe_cfg0 | ISC_PFE_CFG0_MODE_PROGRESSIVE;
 	mask = ISC_PFE_CFG0_BPS_MASK | ISC_PFE_CFG0_HPOL_LOW |
 	       ISC_PFE_CFG0_VPOL_LOW | ISC_PFE_CFG0_PPOL_LOW |
-	       ISC_PFE_CFG0_MODE_MASK;
+	       ISC_PFE_CFG0_MODE_MASK | ISC_PFE_CFG0_CCIR_CRC |
+		   ISC_PFE_CFG0_CCIR656;
 
 	regmap_update_bits(regmap, ISC_PFE_CFG0, mask, pfe_cfg0);
 
@@ -1107,8 +865,15 @@ static int isc_configure(struct isc_device *isc)
 	/* Set the pipeline */
 	isc_set_pipeline(isc, pipeline);
 
-	if (pipeline)
-		isc_set_histogram(isc);
+	/*
+	 * The current implemented histogram is available for RAW R, B, GB
+	 * channels. We need to check if sensor is outputting RAW BAYER
+	 */
+	if (isc->ctrls.awb &&
+	    ISC_IS_FORMAT_RAW(isc->config.sd_format->mbus_code))
+		isc_set_histogram(isc, true);
+	else
+		isc_set_histogram(isc, false);
 
 	/* Update profile */
 	return isc_update_profile(isc);
@@ -1125,7 +890,8 @@ static int isc_start_streaming(struct vb2_queue *vq, unsigned int count)
 	/* Enable stream on the sub device */
 	ret = v4l2_subdev_call(isc->current_subdev->sd, video, s_stream, 1);
 	if (ret && ret != -ENOIOCTLCMD) {
-		v4l2_err(&isc->v4l2_dev, "stream on failed in subdev\n");
+		v4l2_err(&isc->v4l2_dev, "stream on failed in subdev %d\n",
+			 ret);
 		goto err_start_stream;
 	}
 
@@ -1223,6 +989,22 @@ static void isc_buffer_queue(struct vb2_buffer *vb)
 	spin_unlock_irqrestore(&isc->dma_queue_lock, flags);
 }
 
+static struct isc_format *find_format_by_fourcc(struct isc_device *isc,
+						 unsigned int fourcc)
+{
+	unsigned int num_formats = isc->num_user_formats;
+	struct isc_format *fmt;
+	unsigned int i;
+
+	for (i = 0; i < num_formats; i++) {
+		fmt = isc->user_formats[i];
+		if (fmt->fourcc == fourcc)
+			return fmt;
+	}
+
+	return NULL;
+}
+
 static const struct vb2_ops isc_vb2_ops = {
 	.queue_setup		= isc_queue_setup,
 	.wait_prepare		= vb2_ops_wait_prepare,
@@ -1249,15 +1031,31 @@ static int isc_querycap(struct file *file, void *priv,
 static int isc_enum_fmt_vid_cap(struct file *file, void *priv,
 				 struct v4l2_fmtdesc *f)
 {
-	struct isc_device *isc = video_drvdata(file);
 	u32 index = f->index;
+	u32 i, supported_index;
 
-	if (index >= isc->num_user_formats)
-		return -EINVAL;
+	if (index < ARRAY_SIZE(controller_formats)) {
+		f->pixelformat = controller_formats[index].fourcc;
+		return 0;
+	}
 
-	f->pixelformat = isc->user_formats[index]->fourcc;
+	index -= ARRAY_SIZE(controller_formats);
 
-	return 0;
+	i = 0;
+	supported_index = 0;
+
+	for (i = 0; i < ARRAY_SIZE(formats_list); i++) {
+		if (!ISC_IS_FORMAT_RAW(formats_list[i].mbus_code) ||
+		    !formats_list[i].sd_support)
+			continue;
+		if (supported_index == index) {
+			f->pixelformat = formats_list[i].fourcc;
+			return 0;
+		}
+		supported_index++;
+	}
+
+	return -EINVAL;
 }
 
 static int isc_g_fmt_vid_cap(struct file *file, void *priv,
@@ -1270,26 +1068,238 @@ static int isc_g_fmt_vid_cap(struct file *file, void *priv,
 	return 0;
 }
 
-static struct isc_format *find_format_by_fourcc(struct isc_device *isc,
-						 unsigned int fourcc)
+/*
+ * Checks the current configured format, if ISC can output it,
+ * considering which type of format the ISC receives from the sensor
+ */
+static int isc_try_validate_formats(struct isc_device *isc)
 {
-	unsigned int num_formats = isc->num_user_formats;
-	struct isc_format *fmt;
-	unsigned int i;
+	int ret;
+	bool bayer = false, yuv = false, rgb = false, grey = false;
+
+	/* all formats supported by the RLP module are OK */
+	switch (isc->try_config.fourcc) {
+	case V4L2_PIX_FMT_SBGGR8:
+	case V4L2_PIX_FMT_SGBRG8:
+	case V4L2_PIX_FMT_SGRBG8:
+	case V4L2_PIX_FMT_SRGGB8:
+	case V4L2_PIX_FMT_SBGGR10:
+	case V4L2_PIX_FMT_SGBRG10:
+	case V4L2_PIX_FMT_SGRBG10:
+	case V4L2_PIX_FMT_SRGGB10:
+	case V4L2_PIX_FMT_SBGGR12:
+	case V4L2_PIX_FMT_SGBRG12:
+	case V4L2_PIX_FMT_SGRBG12:
+	case V4L2_PIX_FMT_SRGGB12:
+		ret = 0;
+		bayer = true;
+		break;
 
-	for (i = 0; i < num_formats; i++) {
-		fmt = isc->user_formats[i];
-		if (fmt->fourcc == fourcc)
-			return fmt;
+	case V4L2_PIX_FMT_YUV420:
+	case V4L2_PIX_FMT_YUV422P:
+	case V4L2_PIX_FMT_YUYV:
+		ret = 0;
+		yuv = true;
+		break;
+
+	case V4L2_PIX_FMT_RGB565:
+	case V4L2_PIX_FMT_ABGR32:
+	case V4L2_PIX_FMT_XBGR32:
+	case V4L2_PIX_FMT_ARGB444:
+	case V4L2_PIX_FMT_ARGB555:
+		ret = 0;
+		rgb = true;
+		break;
+	case V4L2_PIX_FMT_GREY:
+		ret = 0;
+		grey = true;
+		break;
+	default:
+	/* any other different formats are not supported */
+		ret = -EINVAL;
 	}
 
-	return NULL;
+	/* we cannot output RAW/Grey if we do not receive RAW */
+	if ((bayer || grey) &&
+	    !ISC_IS_FORMAT_RAW(isc->try_config.sd_format->mbus_code))
+		return -EINVAL;
+
+	v4l2_dbg(1, debug, &isc->v4l2_dev,
+		 "Format validation, requested rgb=%u, yuv=%u, grey=%u, bayer=%u\n",
+		 rgb, yuv, grey, bayer);
+
+	return ret;
+}
+
+/*
+ * Configures the RLP and DMA modules, depending on the output format
+ * configured for the ISC.
+ * If direct_dump == true, just dump raw data 8 bits.
+ */
+static int isc_try_configure_rlp_dma(struct isc_device *isc, bool direct_dump)
+{
+	if (direct_dump) {
+		isc->try_config.rlp_cfg_mode = ISC_RLP_CFG_MODE_DAT8;
+		isc->try_config.dcfg_imode = ISC_DCFG_IMODE_PACKED8;
+		isc->try_config.dctrl_dview = ISC_DCTRL_DVIEW_PACKED;
+		isc->try_config.bpp = 16;
+		return 0;
+	}
+
+	switch (isc->try_config.fourcc) {
+	case V4L2_PIX_FMT_SBGGR8:
+	case V4L2_PIX_FMT_SGBRG8:
+	case V4L2_PIX_FMT_SGRBG8:
+	case V4L2_PIX_FMT_SRGGB8:
+		isc->try_config.rlp_cfg_mode = ISC_RLP_CFG_MODE_DAT8;
+		isc->try_config.dcfg_imode = ISC_DCFG_IMODE_PACKED8;
+		isc->try_config.dctrl_dview = ISC_DCTRL_DVIEW_PACKED;
+		isc->try_config.bpp = 8;
+		break;
+	case V4L2_PIX_FMT_SBGGR10:
+	case V4L2_PIX_FMT_SGBRG10:
+	case V4L2_PIX_FMT_SGRBG10:
+	case V4L2_PIX_FMT_SRGGB10:
+		isc->try_config.rlp_cfg_mode = ISC_RLP_CFG_MODE_DAT10;
+		isc->try_config.dcfg_imode = ISC_DCFG_IMODE_PACKED16;
+		isc->try_config.dctrl_dview = ISC_DCTRL_DVIEW_PACKED;
+		isc->try_config.bpp = 16;
+		break;
+	case V4L2_PIX_FMT_SBGGR12:
+	case V4L2_PIX_FMT_SGBRG12:
+	case V4L2_PIX_FMT_SGRBG12:
+	case V4L2_PIX_FMT_SRGGB12:
+		isc->try_config.rlp_cfg_mode = ISC_RLP_CFG_MODE_DAT12;
+		isc->try_config.dcfg_imode = ISC_DCFG_IMODE_PACKED16;
+		isc->try_config.dctrl_dview = ISC_DCTRL_DVIEW_PACKED;
+		isc->try_config.bpp = 16;
+		break;
+	case V4L2_PIX_FMT_RGB565:
+		isc->try_config.rlp_cfg_mode = ISC_RLP_CFG_MODE_RGB565;
+		isc->try_config.dcfg_imode = ISC_DCFG_IMODE_PACKED16;
+		isc->try_config.dctrl_dview = ISC_DCTRL_DVIEW_PACKED;
+		isc->try_config.bpp = 16;
+		break;
+	case V4L2_PIX_FMT_ARGB444:
+		isc->try_config.rlp_cfg_mode = ISC_RLP_CFG_MODE_ARGB444;
+		isc->try_config.dcfg_imode = ISC_DCFG_IMODE_PACKED16;
+		isc->try_config.dctrl_dview = ISC_DCTRL_DVIEW_PACKED;
+		isc->try_config.bpp = 16;
+		break;
+	case V4L2_PIX_FMT_ARGB555:
+		isc->try_config.rlp_cfg_mode = ISC_RLP_CFG_MODE_ARGB555;
+		isc->try_config.dcfg_imode = ISC_DCFG_IMODE_PACKED16;
+		isc->try_config.dctrl_dview = ISC_DCTRL_DVIEW_PACKED;
+		isc->try_config.bpp = 16;
+		break;
+	case V4L2_PIX_FMT_ABGR32:
+	case V4L2_PIX_FMT_XBGR32:
+		isc->try_config.rlp_cfg_mode = ISC_RLP_CFG_MODE_ARGB32;
+		isc->try_config.dcfg_imode = ISC_DCFG_IMODE_PACKED32;
+		isc->try_config.dctrl_dview = ISC_DCTRL_DVIEW_PACKED;
+		isc->try_config.bpp = 32;
+		break;
+	case V4L2_PIX_FMT_YUV420:
+		isc->try_config.rlp_cfg_mode = ISC_RLP_CFG_MODE_YYCC;
+		isc->try_config.dcfg_imode = ISC_DCFG_IMODE_YC420P;
+		isc->try_config.dctrl_dview = ISC_DCTRL_DVIEW_PLANAR;
+		isc->try_config.bpp = 12;
+		break;
+	case V4L2_PIX_FMT_YUV422P:
+		isc->try_config.rlp_cfg_mode = ISC_RLP_CFG_MODE_YYCC;
+		isc->try_config.dcfg_imode = ISC_DCFG_IMODE_YC422P;
+		isc->try_config.dctrl_dview = ISC_DCTRL_DVIEW_PLANAR;
+		isc->try_config.bpp = 16;
+		break;
+	case V4L2_PIX_FMT_YUYV:
+		isc->try_config.rlp_cfg_mode = ISC_RLP_CFG_MODE_YYCC;
+		isc->try_config.dcfg_imode = ISC_DCFG_IMODE_PACKED32;
+		isc->try_config.dctrl_dview = ISC_DCTRL_DVIEW_PACKED;
+		isc->try_config.bpp = 16;
+		break;
+	case V4L2_PIX_FMT_GREY:
+		isc->try_config.rlp_cfg_mode = ISC_RLP_CFG_MODE_DATY8;
+		isc->try_config.dcfg_imode = ISC_DCFG_IMODE_PACKED8;
+		isc->try_config.dctrl_dview = ISC_DCTRL_DVIEW_PACKED;
+		isc->try_config.bpp = 8;
+		break;
+	default:
+		return -EINVAL;
+	}
+	return 0;
+}
+
+/*
+ * Configuring pipeline modules, depending on which format the ISC outputs
+ * and considering which format it has as input from the sensor.
+ */
+static int isc_try_configure_pipeline(struct isc_device *isc)
+{
+	switch (isc->try_config.fourcc) {
+	case V4L2_PIX_FMT_RGB565:
+	case V4L2_PIX_FMT_ARGB555:
+	case V4L2_PIX_FMT_ARGB444:
+	case V4L2_PIX_FMT_ABGR32:
+	case V4L2_PIX_FMT_XBGR32:
+		/* if sensor format is RAW, we convert inside ISC */
+		if (ISC_IS_FORMAT_RAW(isc->try_config.sd_format->mbus_code)) {
+			isc->try_config.bits_pipeline = CFA_ENABLE |
+				WB_ENABLE | GAM_ENABLES;
+		} else {
+			isc->try_config.bits_pipeline = 0x0;
+		}
+		break;
+	case V4L2_PIX_FMT_YUV420:
+		/* if sensor format is RAW, we convert inside ISC */
+		if (ISC_IS_FORMAT_RAW(isc->try_config.sd_format->mbus_code)) {
+			isc->try_config.bits_pipeline = CFA_ENABLE |
+				CSC_ENABLE | WB_ENABLE | GAM_ENABLES |
+				SUB420_ENABLE | SUB422_ENABLE | CBC_ENABLE;
+		} else {
+			isc->try_config.bits_pipeline = 0x0;
+		}
+		break;
+	case V4L2_PIX_FMT_YUV422P:
+		/* if sensor format is RAW, we convert inside ISC */
+		if (ISC_IS_FORMAT_RAW(isc->try_config.sd_format->mbus_code)) {
+			isc->try_config.bits_pipeline = CFA_ENABLE |
+				CSC_ENABLE | WB_ENABLE | GAM_ENABLES |
+				SUB422_ENABLE | CBC_ENABLE;
+		} else {
+			isc->try_config.bits_pipeline = 0x0;
+		}
+		break;
+	case V4L2_PIX_FMT_YUYV:
+		/* if sensor format is RAW, we convert inside ISC */
+		if (ISC_IS_FORMAT_RAW(isc->try_config.sd_format->mbus_code)) {
+			isc->try_config.bits_pipeline = CFA_ENABLE |
+				CSC_ENABLE | WB_ENABLE | GAM_ENABLES |
+				SUB422_ENABLE | CBC_ENABLE;
+		} else {
+			isc->try_config.bits_pipeline = 0x0;
+		}
+		break;
+	case V4L2_PIX_FMT_GREY:
+		if (ISC_IS_FORMAT_RAW(isc->try_config.sd_format->mbus_code)) {
+		/* if sensor format is RAW, we convert inside ISC */
+			isc->try_config.bits_pipeline = CFA_ENABLE |
+				CSC_ENABLE | WB_ENABLE | GAM_ENABLES |
+				CBC_ENABLE;
+		} else {
+			isc->try_config.bits_pipeline = 0x0;
+		}
+		break;
+	default:
+		isc->try_config.bits_pipeline = 0x0;
+	}
+	return 0;
 }
 
 static int isc_try_fmt(struct isc_device *isc, struct v4l2_format *f,
-			struct isc_format **current_fmt, u32 *code)
+			u32 *code)
 {
-	struct isc_format *isc_fmt;
+	int i;
+	struct isc_format *sd_fmt = NULL, *direct_fmt = NULL;
 	struct v4l2_pix_format *pixfmt = &f->fmt.pix;
 	struct v4l2_subdev_pad_config pad_cfg;
 	struct v4l2_subdev_format format = {
@@ -1297,48 +1307,119 @@ static int isc_try_fmt(struct isc_device *isc, struct v4l2_format *f,
 	};
 	u32 mbus_code;
 	int ret;
+	bool rlp_dma_direct_dump = false;
 
 	if (f->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
 		return -EINVAL;
 
-	isc_fmt = find_format_by_fourcc(isc, pixfmt->pixelformat);
-	if (!isc_fmt) {
-		v4l2_warn(&isc->v4l2_dev, "Format 0x%x not found\n",
-			  pixfmt->pixelformat);
-		isc_fmt = isc->user_formats[isc->num_user_formats - 1];
-		pixfmt->pixelformat = isc_fmt->fourcc;
+	/* Step 1: find a RAW format that is supported */
+	for (i = 0; i < isc->num_user_formats; i++) {
+		if (ISC_IS_FORMAT_RAW(isc->user_formats[i]->mbus_code)) {
+			sd_fmt = isc->user_formats[i];
+			break;
+		}
+	}
+	/* Step 2: We can continue with this RAW format, or we can look
+	 * for better: maybe sensor supports directly what we need.
+	 */
+	direct_fmt = find_format_by_fourcc(isc, pixfmt->pixelformat);
+
+	/* Step 3: We have both. We decide given the module parameter which
+	 * one to use.
+	 */
+	if (direct_fmt && sd_fmt && sensor_preferred)
+		sd_fmt = direct_fmt;
+
+	/* Step 4: we do not have RAW but we have a direct format. Use it. */
+	if (direct_fmt && !sd_fmt)
+		sd_fmt = direct_fmt;
+
+	/* Step 5: if we are using a direct format, we need to package
+	 * everything as 8 bit data and just dump it
+	 */
+	if (sd_fmt == direct_fmt)
+		rlp_dma_direct_dump = true;
+
+	/* Step 6: We have no format. This can happen if the userspace
+	 * requests some weird/invalid format.
+	 * In this case, default to whatever we have
+	 */
+	if (!sd_fmt && !direct_fmt) {
+		sd_fmt = isc->user_formats[isc->num_user_formats - 1];
+		v4l2_dbg(1, debug, &isc->v4l2_dev,
+			 "Sensor not supporting %.4s, using %.4s\n",
+			 (char *)&pixfmt->pixelformat, (char *)&sd_fmt->fourcc);
+	}
+
+	if (!sd_fmt) {
+		ret = -EINVAL;
+		goto isc_try_fmt_err;
 	}
 
+	/* Step 7: Print out what we decided for debugging */
+	v4l2_dbg(1, debug, &isc->v4l2_dev,
+		 "Preferring to have sensor using format %.4s\n",
+		 (char *)&sd_fmt->fourcc);
+
+	/* Step 8: at this moment we decided which format the subdev will use */
+	isc->try_config.sd_format = sd_fmt;
+
 	/* Limit to Atmel ISC hardware capabilities */
 	if (pixfmt->width > ISC_MAX_SUPPORT_WIDTH)
 		pixfmt->width = ISC_MAX_SUPPORT_WIDTH;
 	if (pixfmt->height > ISC_MAX_SUPPORT_HEIGHT)
 		pixfmt->height = ISC_MAX_SUPPORT_HEIGHT;
 
-	if (sensor_is_preferred(isc_fmt))
-		mbus_code = isc_fmt->mbus_code;
-	else
-		mbus_code = isc->raw_fmt->mbus_code;
+	/*
+	 * The mbus format is the one the subdev outputs.
+	 * The pixels will be transferred in this format Sensor -> ISC
+	 */
+	mbus_code = sd_fmt->mbus_code;
+
+	/*
+	 * Validate formats. If the required format is not OK, default to raw.
+	 */
+
+	isc->try_config.fourcc = pixfmt->pixelformat;
+
+	if (isc_try_validate_formats(isc)) {
+		pixfmt->pixelformat = isc->try_config.fourcc = sd_fmt->fourcc;
+		/* Re-try to validate the new format */
+		ret = isc_try_validate_formats(isc);
+		if (ret)
+			goto isc_try_fmt_err;
+	}
+
+	ret = isc_try_configure_rlp_dma(isc, rlp_dma_direct_dump);
+	if (ret)
+		goto isc_try_fmt_err;
+
+	ret = isc_try_configure_pipeline(isc);
+	if (ret)
+		goto isc_try_fmt_err;
 
 	v4l2_fill_mbus_format(&format.format, pixfmt, mbus_code);
 	ret = v4l2_subdev_call(isc->current_subdev->sd, pad, set_fmt,
 			       &pad_cfg, &format);
 	if (ret < 0)
-		return ret;
+		goto isc_try_fmt_err;
 
 	v4l2_fill_pix_format(pixfmt, &format.format);
 
 	pixfmt->field = V4L2_FIELD_NONE;
-	pixfmt->bytesperline = (pixfmt->width * isc_fmt->bpp) >> 3;
+	pixfmt->bytesperline = (pixfmt->width * isc->try_config.bpp) >> 3;
 	pixfmt->sizeimage = pixfmt->bytesperline * pixfmt->height;
 
-	if (current_fmt)
-		*current_fmt = isc_fmt;
-
 	if (code)
 		*code = mbus_code;
 
 	return 0;
+
+isc_try_fmt_err:
+	v4l2_err(&isc->v4l2_dev, "Could not find any possible format for a working pipeline\n");
+	memset(&isc->try_config, 0, sizeof(isc->try_config));
+
+	return ret;
 }
 
 static int isc_set_fmt(struct isc_device *isc, struct v4l2_format *f)
@@ -1346,11 +1427,10 @@ static int isc_set_fmt(struct isc_device *isc, struct v4l2_format *f)
 	struct v4l2_subdev_format format = {
 		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
 	};
-	struct isc_format *current_fmt;
-	u32 mbus_code;
+	u32 mbus_code = 0;
 	int ret;
 
-	ret = isc_try_fmt(isc, f, &current_fmt, &mbus_code);
+	ret = isc_try_fmt(isc, f, &mbus_code);
 	if (ret)
 		return ret;
 
@@ -1361,7 +1441,10 @@ static int isc_set_fmt(struct isc_device *isc, struct v4l2_format *f)
 		return ret;
 
 	isc->fmt = *f;
-	isc->current_fmt = current_fmt;
+	/* make the try configuration active */
+	isc->config = isc->try_config;
+
+	v4l2_dbg(1, debug, &isc->v4l2_dev, "New ISC configuration in place\n");
 
 	return 0;
 }
@@ -1382,7 +1465,7 @@ static int isc_try_fmt_vid_cap(struct file *file, void *priv,
 {
 	struct isc_device *isc = video_drvdata(file);
 
-	return isc_try_fmt(isc, f, NULL, NULL);
+	return isc_try_fmt(isc, f, NULL);
 }
 
 static int isc_enum_input(struct file *file, void *priv,
@@ -1431,27 +1514,31 @@ static int isc_enum_framesizes(struct file *file, void *fh,
 			       struct v4l2_frmsizeenum *fsize)
 {
 	struct isc_device *isc = video_drvdata(file);
-	const struct isc_format *isc_fmt;
 	struct v4l2_subdev_frame_size_enum fse = {
 		.index = fsize->index,
 		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
 	};
-	int ret;
+	int ret = -EINVAL;
+	int i;
 
-	isc_fmt = find_format_by_fourcc(isc, fsize->pixel_format);
-	if (!isc_fmt)
-		return -EINVAL;
+	for (i = 0; i < isc->num_user_formats; i++)
+		if (isc->user_formats[i]->fourcc == fsize->pixel_format)
+			ret = 0;
 
-	if (sensor_is_preferred(isc_fmt))
-		fse.code = isc_fmt->mbus_code;
-	else
-		fse.code = isc->raw_fmt->mbus_code;
+	for (i = 0; i < ARRAY_SIZE(controller_formats); i++)
+		if (controller_formats[i].fourcc == fsize->pixel_format)
+			ret = 0;
+
+	if (ret)
+		return ret;
 
 	ret = v4l2_subdev_call(isc->current_subdev->sd, pad, enum_frame_size,
 			       NULL, &fse);
 	if (ret)
 		return ret;
 
+	fse.code = isc->config.sd_format->mbus_code;
+
 	fsize->type = V4L2_FRMSIZE_TYPE_DISCRETE;
 	fsize->discrete.width = fse.max_width;
 	fsize->discrete.height = fse.max_height;
@@ -1463,29 +1550,32 @@ static int isc_enum_frameintervals(struct file *file, void *fh,
 				    struct v4l2_frmivalenum *fival)
 {
 	struct isc_device *isc = video_drvdata(file);
-	const struct isc_format *isc_fmt;
 	struct v4l2_subdev_frame_interval_enum fie = {
 		.index = fival->index,
 		.width = fival->width,
 		.height = fival->height,
 		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
 	};
-	int ret;
+	int ret = -EINVAL;
+	int i;
 
-	isc_fmt = find_format_by_fourcc(isc, fival->pixel_format);
-	if (!isc_fmt)
-		return -EINVAL;
+	for (i = 0; i < isc->num_user_formats; i++)
+		if (isc->user_formats[i]->fourcc == fival->pixel_format)
+			ret = 0;
 
-	if (sensor_is_preferred(isc_fmt))
-		fie.code = isc_fmt->mbus_code;
-	else
-		fie.code = isc->raw_fmt->mbus_code;
+	for (i = 0; i < ARRAY_SIZE(controller_formats); i++)
+		if (controller_formats[i].fourcc == fival->pixel_format)
+			ret = 0;
+
+	if (ret)
+		return ret;
 
 	ret = v4l2_subdev_call(isc->current_subdev->sd, pad,
 			       enum_frame_interval, NULL, &fie);
 	if (ret)
 		return ret;
 
+	fie.code = isc->config.sd_format->mbus_code;
 	fival->type = V4L2_FRMIVAL_TYPE_DISCRETE;
 	fival->discrete = fie.interval;
 
@@ -1668,7 +1758,6 @@ static void isc_awb_work(struct work_struct *w)
 	struct isc_device *isc =
 		container_of(w, struct isc_device, awb_work);
 	struct regmap *regmap = isc->regmap;
-	struct fmt_config *config = get_fmt_config(isc->raw_fmt->fourcc);
 	struct isc_ctrls *ctrls = &isc->ctrls;
 	u32 hist_id = ctrls->hist_id;
 	u32 baysel;
@@ -1686,7 +1775,7 @@ static void isc_awb_work(struct work_struct *w)
 	}
 
 	ctrls->hist_id = hist_id;
-	baysel = config->cfa_baycfg << ISC_HIS_CFG_BAYSEL_SHIFT;
+	baysel = isc->config.sd_format->cfa_baycfg << ISC_HIS_CFG_BAYSEL_SHIFT;
 
 	pm_runtime_get_sync(isc->dev);
 
@@ -1754,7 +1843,6 @@ static int isc_ctrl_init(struct isc_device *isc)
 	return 0;
 }
 
-
 static int isc_async_bound(struct v4l2_async_notifier *notifier,
 			    struct v4l2_subdev *subdev,
 			    struct v4l2_async_subdev *asd)
@@ -1812,35 +1900,20 @@ static int isc_formats_init(struct isc_device *isc)
 		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
 	};
 
+	num_fmts = 0;
 	while (!v4l2_subdev_call(subdev, pad, enum_mbus_code,
 	       NULL, &mbus_code)) {
 		mbus_code.index++;
 
 		fmt = find_format_by_code(mbus_code.code, &i);
-		if ((!fmt) || (!(fmt->flags & FMT_FLAG_FROM_SENSOR)))
+		if (!fmt) {
+			v4l2_warn(&isc->v4l2_dev, "Mbus code %x not supported\n",
+				  mbus_code.code);
 			continue;
+		}
 
 		fmt->sd_support = true;
-
-		if (fmt->flags & FMT_FLAG_RAW_FORMAT)
-			isc->raw_fmt = fmt;
-	}
-
-	fmt = &formats_list[0];
-	for (i = 0; i < list_size; i++) {
-		if (fmt->flags & FMT_FLAG_FROM_CONTROLLER)
-			fmt->isc_support = true;
-
-		fmt++;
-	}
-
-	fmt = &formats_list[0];
-	num_fmts = 0;
-	for (i = 0; i < list_size; i++) {
-		if (fmt->isc_support || fmt->sd_support)
-			num_fmts++;
-
-		fmt++;
+		num_fmts++;
 	}
 
 	if (!num_fmts)
@@ -1855,9 +1928,8 @@ static int isc_formats_init(struct isc_device *isc)
 
 	fmt = &formats_list[0];
 	for (i = 0, j = 0; i < list_size; i++) {
-		if (fmt->isc_support || fmt->sd_support)
+		if (fmt->sd_support)
 			isc->user_formats[j++] = fmt;
-
 		fmt++;
 	}
 
@@ -1877,13 +1949,11 @@ static int isc_set_default_fmt(struct isc_device *isc)
 	};
 	int ret;
 
-	ret = isc_try_fmt(isc, &f, NULL, NULL);
+	ret = isc_try_fmt(isc, &f, NULL);
 	if (ret)
 		return ret;
 
-	isc->current_fmt = isc->user_formats[0];
 	isc->fmt = f;
-
 	return 0;
 }
 
@@ -2084,6 +2154,10 @@ static int isc_parse_dt(struct device *dev, struct isc_device *isc)
 		if (flags & V4L2_MBUS_PCLK_SAMPLE_FALLING)
 			subdev_entity->pfe_cfg0 |= ISC_PFE_CFG0_PPOL_LOW;
 
+		if (v4l2_epn.bus_type == V4L2_MBUS_BT656)
+			subdev_entity->pfe_cfg0 |= ISC_PFE_CFG0_CCIR_CRC |
+					ISC_PFE_CFG0_CCIR656;
+
 		subdev_entity->asd->match_type = V4L2_ASYNC_MATCH_FWNODE;
 		subdev_entity->asd->match.fwnode =
 			of_fwnode_handle(rem);
diff --git a/drivers/media/platform/coda/coda-bit.c b/drivers/media/platform/coda/coda-bit.c
index b4f396c2e72c..eaa86737fa04 100644
--- a/drivers/media/platform/coda/coda-bit.c
+++ b/drivers/media/platform/coda/coda-bit.c
@@ -2010,6 +2010,9 @@ static int coda_prepare_decode(struct coda_ctx *ctx)
 	/* Clear decode success flag */
 	coda_write(dev, 0, CODA_RET_DEC_PIC_SUCCESS);
 
+	/* Clear error return value */
+	coda_write(dev, 0, CODA_RET_DEC_PIC_ERR_MB);
+
 	trace_coda_dec_pic_run(ctx, meta);
 
 	coda_command_async(ctx, CODA_COMMAND_PIC_RUN);
diff --git a/drivers/media/platform/coda/coda-common.c b/drivers/media/platform/coda/coda-common.c
index fa0b22fb7991..3ce58dee4422 100644
--- a/drivers/media/platform/coda/coda-common.c
+++ b/drivers/media/platform/coda/coda-common.c
@@ -764,6 +764,7 @@ static int coda_s_fmt_vid_cap(struct file *file, void *priv,
 {
 	struct coda_ctx *ctx = fh_to_ctx(priv);
 	struct coda_q_data *q_data_src;
+	const struct coda_codec *codec;
 	struct v4l2_rect r;
 	int ret;
 
@@ -784,6 +785,15 @@ static int coda_s_fmt_vid_cap(struct file *file, void *priv,
 	if (ctx->inst_type != CODA_INST_ENCODER)
 		return 0;
 
+	/* Setting the coded format determines the selected codec */
+	codec = coda_find_codec(ctx->dev, q_data_src->fourcc,
+				f->fmt.pix.pixelformat);
+	if (!codec) {
+		v4l2_err(&ctx->dev->v4l2_dev, "failed to determine codec\n");
+		return -EINVAL;
+	}
+	ctx->codec = codec;
+
 	ctx->colorspace = f->fmt.pix.colorspace;
 	ctx->xfer_func = f->fmt.pix.xfer_func;
 	ctx->ycbcr_enc = f->fmt.pix.ycbcr_enc;
@@ -796,6 +806,7 @@ static int coda_s_fmt_vid_out(struct file *file, void *priv,
 			      struct v4l2_format *f)
 {
 	struct coda_ctx *ctx = fh_to_ctx(priv);
+	const struct coda_codec *codec;
 	struct v4l2_format f_cap;
 	struct vb2_queue *dst_vq;
 	int ret;
@@ -808,14 +819,23 @@ static int coda_s_fmt_vid_out(struct file *file, void *priv,
 	if (ret)
 		return ret;
 
-	if (ctx->inst_type != CODA_INST_DECODER)
-		return 0;
-
 	ctx->colorspace = f->fmt.pix.colorspace;
 	ctx->xfer_func = f->fmt.pix.xfer_func;
 	ctx->ycbcr_enc = f->fmt.pix.ycbcr_enc;
 	ctx->quantization = f->fmt.pix.quantization;
 
+	if (ctx->inst_type != CODA_INST_DECODER)
+		return 0;
+
+	/* Setting the coded format determines the selected codec */
+	codec = coda_find_codec(ctx->dev, f->fmt.pix.pixelformat,
+				V4L2_PIX_FMT_YUV420);
+	if (!codec) {
+		v4l2_err(&ctx->dev->v4l2_dev, "failed to determine codec\n");
+		return -EINVAL;
+	}
+	ctx->codec = codec;
+
 	dst_vq = v4l2_m2m_get_vq(ctx->fh.m2m_ctx, V4L2_BUF_TYPE_VIDEO_CAPTURE);
 	if (!dst_vq)
 		return -EINVAL;
@@ -980,6 +1000,11 @@ static int coda_s_selection(struct file *file, void *fh,
 static int coda_try_encoder_cmd(struct file *file, void *fh,
 				struct v4l2_encoder_cmd *ec)
 {
+	struct coda_ctx *ctx = fh_to_ctx(fh);
+
+	if (ctx->inst_type != CODA_INST_ENCODER)
+		return -ENOTTY;
+
 	if (ec->cmd != V4L2_ENC_CMD_STOP)
 		return -EINVAL;
 
@@ -1000,10 +1025,6 @@ static int coda_encoder_cmd(struct file *file, void *fh,
 	if (ret < 0)
 		return ret;
 
-	/* Ignore encoder stop command silently in decoder context */
-	if (ctx->inst_type != CODA_INST_ENCODER)
-		return 0;
-
 	/* Set the stream-end flag on this context */
 	ctx->bit_stream_param |= CODA_BIT_STREAM_END_FLAG;
 
@@ -1021,6 +1042,11 @@ static int coda_encoder_cmd(struct file *file, void *fh,
 static int coda_try_decoder_cmd(struct file *file, void *fh,
 				struct v4l2_decoder_cmd *dc)
 {
+	struct coda_ctx *ctx = fh_to_ctx(fh);
+
+	if (ctx->inst_type != CODA_INST_DECODER)
+		return -ENOTTY;
+
 	if (dc->cmd != V4L2_DEC_CMD_STOP)
 		return -EINVAL;
 
@@ -1043,10 +1069,6 @@ static int coda_decoder_cmd(struct file *file, void *fh,
 	if (ret < 0)
 		return ret;
 
-	/* Ignore decoder stop command silently in encoder context */
-	if (ctx->inst_type != CODA_INST_DECODER)
-		return 0;
-
 	/* Set the stream-end flag on this context */
 	coda_bit_stream_end_flag(ctx);
 	ctx->hold = false;
@@ -1055,6 +1077,42 @@ static int coda_decoder_cmd(struct file *file, void *fh,
 	return 0;
 }
 
+static int coda_enum_framesizes(struct file *file, void *fh,
+				struct v4l2_frmsizeenum *fsize)
+{
+	struct coda_ctx *ctx = fh_to_ctx(fh);
+	struct coda_q_data *q_data_dst;
+	const struct coda_codec *codec;
+
+	if (ctx->inst_type != CODA_INST_ENCODER)
+		return -ENOTTY;
+
+	if (fsize->index)
+		return -EINVAL;
+
+	if (coda_format_normalize_yuv(fsize->pixel_format) ==
+	    V4L2_PIX_FMT_YUV420) {
+		q_data_dst = get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_CAPTURE);
+		codec = coda_find_codec(ctx->dev, fsize->pixel_format,
+					q_data_dst->fourcc);
+	} else {
+		codec = coda_find_codec(ctx->dev, V4L2_PIX_FMT_YUV420,
+					fsize->pixel_format);
+	}
+	if (!codec)
+		return -EINVAL;
+
+	fsize->type = V4L2_FRMSIZE_TYPE_CONTINUOUS;
+	fsize->stepwise.min_width = MIN_W;
+	fsize->stepwise.max_width = codec->max_w;
+	fsize->stepwise.step_width = 1;
+	fsize->stepwise.min_height = MIN_H;
+	fsize->stepwise.max_height = codec->max_h;
+	fsize->stepwise.step_height = 1;
+
+	return 0;
+}
+
 static int coda_enum_frameintervals(struct file *file, void *fh,
 				    struct v4l2_frmivalenum *f)
 {
@@ -1233,6 +1291,7 @@ static const struct v4l2_ioctl_ops coda_ioctl_ops = {
 	.vidioc_g_parm		= coda_g_parm,
 	.vidioc_s_parm		= coda_s_parm,
 
+	.vidioc_enum_framesizes	= coda_enum_framesizes,
 	.vidioc_enum_frameintervals = coda_enum_frameintervals,
 
 	.vidioc_subscribe_event = coda_subscribe_event,
@@ -1442,6 +1501,9 @@ static int coda_queue_setup(struct vb2_queue *vq,
 	q_data = get_q_data(ctx, vq->type);
 	size = q_data->sizeimage;
 
+	if (*nplanes)
+		return sizes[0] < size ? -EINVAL : 0;
+
 	*nplanes = 1;
 	sizes[0] = size;
 
@@ -1680,14 +1742,6 @@ static int coda_start_streaming(struct vb2_queue *q, unsigned int count)
 
 	ctx->gopcounter = ctx->params.gop_size - 1;
 
-	ctx->codec = coda_find_codec(ctx->dev, q_data_src->fourcc,
-				     q_data_dst->fourcc);
-	if (!ctx->codec) {
-		v4l2_err(v4l2_dev, "couldn't tell instance type.\n");
-		ret = -EINVAL;
-		goto err;
-	}
-
 	if (q_data_dst->fourcc == V4L2_PIX_FMT_JPEG)
 		ctx->params.gop_size = 1;
 	ctx->gopcounter = ctx->params.gop_size - 1;
@@ -2015,7 +2069,6 @@ static void coda_jpeg_encode_ctrls(struct coda_ctx *ctx)
 
 static void coda_decode_ctrls(struct coda_ctx *ctx)
 {
-	u64 mask;
 	u8 max;
 
 	ctx->h264_profile_ctrl = v4l2_ctrl_new_std_menu(&ctx->ctrls,
@@ -2029,27 +2082,14 @@ static void coda_decode_ctrls(struct coda_ctx *ctx)
 		ctx->h264_profile_ctrl->flags |= V4L2_CTRL_FLAG_READ_ONLY;
 
 	if (ctx->dev->devtype->product == CODA_HX4 ||
-	    ctx->dev->devtype->product == CODA_7541) {
+	    ctx->dev->devtype->product == CODA_7541)
 		max = V4L2_MPEG_VIDEO_H264_LEVEL_4_0;
-		mask = ~((1 << V4L2_MPEG_VIDEO_H264_LEVEL_2_0) |
-			 (1 << V4L2_MPEG_VIDEO_H264_LEVEL_3_0) |
-			 (1 << V4L2_MPEG_VIDEO_H264_LEVEL_3_1) |
-			 (1 << V4L2_MPEG_VIDEO_H264_LEVEL_3_2) |
-			 (1 << V4L2_MPEG_VIDEO_H264_LEVEL_4_0));
-	} else if (ctx->dev->devtype->product == CODA_960) {
+	else if (ctx->dev->devtype->product == CODA_960)
 		max = V4L2_MPEG_VIDEO_H264_LEVEL_4_1;
-		mask = ~((1 << V4L2_MPEG_VIDEO_H264_LEVEL_2_0) |
-			 (1 << V4L2_MPEG_VIDEO_H264_LEVEL_3_0) |
-			 (1 << V4L2_MPEG_VIDEO_H264_LEVEL_3_1) |
-			 (1 << V4L2_MPEG_VIDEO_H264_LEVEL_3_2) |
-			 (1 << V4L2_MPEG_VIDEO_H264_LEVEL_4_0) |
-			 (1 << V4L2_MPEG_VIDEO_H264_LEVEL_4_1));
-	} else {
+	else
 		return;
-	}
 	ctx->h264_level_ctrl = v4l2_ctrl_new_std_menu(&ctx->ctrls,
-		&coda_ctrl_ops, V4L2_CID_MPEG_VIDEO_H264_LEVEL, max, mask,
-		max);
+		&coda_ctrl_ops, V4L2_CID_MPEG_VIDEO_H264_LEVEL, max, 0, max);
 	if (ctx->h264_level_ctrl)
 		ctx->h264_level_ctrl->flags |= V4L2_CTRL_FLAG_READ_ONLY;
 }
@@ -2063,11 +2103,17 @@ static int coda_ctrls_setup(struct coda_ctx *ctx)
 	v4l2_ctrl_new_std(&ctx->ctrls, &coda_ctrl_ops,
 		V4L2_CID_VFLIP, 0, 1, 1, 0);
 	if (ctx->inst_type == CODA_INST_ENCODER) {
+		v4l2_ctrl_new_std(&ctx->ctrls, &coda_ctrl_ops,
+				  V4L2_CID_MIN_BUFFERS_FOR_OUTPUT,
+				  1, 1, 1, 1);
 		if (ctx->cvd->dst_formats[0] == V4L2_PIX_FMT_JPEG)
 			coda_jpeg_encode_ctrls(ctx);
 		else
 			coda_encode_ctrls(ctx);
 	} else {
+		v4l2_ctrl_new_std(&ctx->ctrls, &coda_ctrl_ops,
+				  V4L2_CID_MIN_BUFFERS_FOR_CAPTURE,
+				  1, 1, 1, 1);
 		if (ctx->cvd->src_formats[0] == V4L2_PIX_FMT_H264)
 			coda_decode_ctrls(ctx);
 	}
diff --git a/drivers/media/platform/cros-ec-cec/cros-ec-cec.c b/drivers/media/platform/cros-ec-cec/cros-ec-cec.c
index 7bc4d8a9af28..068df9888dbf 100644
--- a/drivers/media/platform/cros-ec-cec/cros-ec-cec.c
+++ b/drivers/media/platform/cros-ec-cec/cros-ec-cec.c
@@ -236,6 +236,7 @@ static int cros_ec_cec_get_notifier(struct device *dev,
 				return -EPROBE_DEFER;
 
 			*notify = cec_notifier_get_conn(d, m->conn);
+			put_device(d);
 			return 0;
 		}
 	}
diff --git a/drivers/media/platform/davinci/isif.c b/drivers/media/platform/davinci/isif.c
index 47cecd10eb9f..2dee9af6d413 100644
--- a/drivers/media/platform/davinci/isif.c
+++ b/drivers/media/platform/davinci/isif.c
@@ -884,9 +884,7 @@ static int isif_set_hw_if_params(struct vpfe_hw_if_param *params)
 static int isif_config_ycbcr(void)
 {
 	struct isif_ycbcr_config *params = &isif_cfg.ycbcr;
-	struct vpss_pg_frame_size frame_size;
 	u32 modeset = 0, ccdcfg = 0;
-	struct vpss_sync_pol sync;
 
 	dev_dbg(isif_cfg.dev, "\nStarting isif_config_ycbcr...");
 
@@ -974,13 +972,6 @@ static int isif_config_ycbcr(void)
 		/* two fields are interleaved in memory */
 		regw(0x00000249, SDOFST);
 
-	/* Setup test pattern if enabled */
-	if (isif_cfg.bayer.config_params.test_pat_gen) {
-		sync.ccdpg_hdpol = params->hd_pol;
-		sync.ccdpg_vdpol = params->vd_pol;
-		dm365_vpss_set_sync_pol(sync);
-		dm365_vpss_set_pg_frame_size(frame_size);
-	}
 	return 0;
 }
 
diff --git a/drivers/media/platform/davinci/vpfe_capture.c b/drivers/media/platform/davinci/vpfe_capture.c
index 26dadbba930f..1e3a13830544 100644
--- a/drivers/media/platform/davinci/vpfe_capture.c
+++ b/drivers/media/platform/davinci/vpfe_capture.c
@@ -1759,7 +1759,7 @@ static int vpfe_probe(struct platform_device *pdev)
 
 	mutex_lock(&ccdc_lock);
 
-	strncpy(ccdc_cfg->name, vpfe_cfg->ccdc, 32);
+	strscpy(ccdc_cfg->name, vpfe_cfg->ccdc, sizeof(ccdc_cfg->name));
 	/* Get VINT0 irq resource */
 	res1 = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
 	if (!res1) {
diff --git a/drivers/media/platform/davinci/vpif_capture.c b/drivers/media/platform/davinci/vpif_capture.c
index 6216b7ac6875..b5aacb0fb96b 100644
--- a/drivers/media/platform/davinci/vpif_capture.c
+++ b/drivers/media/platform/davinci/vpif_capture.c
@@ -1254,7 +1254,8 @@ static int vpif_s_dv_timings(struct file *file, void *priv,
 	} else {
 		std_info->l5 = std_info->vsize - (bt->vfrontporch - 1);
 	}
-	strncpy(std_info->name, "Custom timings BT656/1120", VPIF_MAX_NAME);
+	strscpy(std_info->name, "Custom timings BT656/1120",
+		sizeof(std_info->name));
 	std_info->width = bt->width;
 	std_info->height = bt->height;
 	std_info->frm_fmt = bt->interlaced ? 0 : 1;
diff --git a/drivers/media/platform/davinci/vpif_display.c b/drivers/media/platform/davinci/vpif_display.c
index f4b4f2a1dfc0..a69897c68a50 100644
--- a/drivers/media/platform/davinci/vpif_display.c
+++ b/drivers/media/platform/davinci/vpif_display.c
@@ -987,8 +987,8 @@ static int vpif_s_dv_timings(struct file *file, void *priv,
 	} else {
 		std_info->l5 = std_info->vsize - (bt->vfrontporch - 1);
 	}
-	strncpy(std_info->name, "Custom timings BT656/1120",
-			VPIF_MAX_NAME);
+	strscpy(std_info->name, "Custom timings BT656/1120",
+		sizeof(std_info->name));
 	std_info->width = bt->width;
 	std_info->height = bt->height;
 	std_info->frm_fmt = bt->interlaced ? 0 : 1;
diff --git a/drivers/media/platform/exynos4-is/fimc-capture.c b/drivers/media/platform/exynos4-is/fimc-capture.c
index 3e9fcf4f8a13..de4af0357a3c 100644
--- a/drivers/media/platform/exynos4-is/fimc-capture.c
+++ b/drivers/media/platform/exynos4-is/fimc-capture.c
@@ -742,7 +742,7 @@ static int fimc_cap_enum_fmt_mplane(struct file *file, void *priv,
 			       f->index);
 	if (!fmt)
 		return -EINVAL;
-	strncpy(f->description, fmt->name, sizeof(f->description) - 1);
+	strscpy(f->description, fmt->name, sizeof(f->description));
 	f->pixelformat = fmt->fourcc;
 	if (fmt->fourcc == MEDIA_BUS_FMT_JPEG_1X8)
 		f->flags |= V4L2_FMT_FLAG_COMPRESSED;
diff --git a/drivers/media/platform/exynos4-is/fimc-m2m.c b/drivers/media/platform/exynos4-is/fimc-m2m.c
index 61c8177409cf..1bea1ce4091e 100644
--- a/drivers/media/platform/exynos4-is/fimc-m2m.c
+++ b/drivers/media/platform/exynos4-is/fimc-m2m.c
@@ -252,7 +252,7 @@ static int fimc_m2m_enum_fmt_mplane(struct file *file, void *priv,
 	if (!fmt)
 		return -EINVAL;
 
-	strncpy(f->description, fmt->name, sizeof(f->description) - 1);
+	strscpy(f->description, fmt->name, sizeof(f->description));
 	f->pixelformat = fmt->fourcc;
 	return 0;
 }
diff --git a/drivers/media/platform/imx-pxp.c b/drivers/media/platform/imx-pxp.c
index 0bcfc5aa8f3d..8e7ef23b9a7e 100644
--- a/drivers/media/platform/imx-pxp.c
+++ b/drivers/media/platform/imx-pxp.c
@@ -1025,8 +1025,8 @@ static irqreturn_t pxp_irq_handler(int irq, void *dev_id)
 static int pxp_querycap(struct file *file, void *priv,
 			   struct v4l2_capability *cap)
 {
-	strlcpy(cap->driver, MEM2MEM_NAME, sizeof(cap->driver));
-	strlcpy(cap->card, MEM2MEM_NAME, sizeof(cap->card));
+	strscpy(cap->driver, MEM2MEM_NAME, sizeof(cap->driver));
+	strscpy(cap->card, MEM2MEM_NAME, sizeof(cap->card));
 	snprintf(cap->bus_info, sizeof(cap->bus_info),
 			"platform:%s", MEM2MEM_NAME);
 	return 0;
diff --git a/drivers/media/platform/marvell-ccic/Kconfig b/drivers/media/platform/marvell-ccic/Kconfig
index cf12e077203a..cd88e2eed749 100644
--- a/drivers/media/platform/marvell-ccic/Kconfig
+++ b/drivers/media/platform/marvell-ccic/Kconfig
@@ -5,7 +5,7 @@ config VIDEO_CAFE_CCIC
 	select VIDEOBUF2_VMALLOC
 	select VIDEOBUF2_DMA_CONTIG
 	select VIDEOBUF2_DMA_SG
-	---help---
+	help
 	  This is a video4linux2 driver for the Marvell 88ALP01 integrated
 	  CMOS camera controller.  This is the controller found on first-
 	  generation OLPC systems.
@@ -19,7 +19,7 @@ config VIDEO_MMP_CAMERA
 	select VIDEOBUF2_VMALLOC
 	select VIDEOBUF2_DMA_CONTIG
 	select VIDEOBUF2_DMA_SG
-	---help---
+	help
 	  This is a Video4Linux2 driver for the integrated camera
 	  controller found on Marvell Armada 610 application
 	  processors (and likely beyond).  This is the controller found
diff --git a/drivers/media/platform/meson/Makefile b/drivers/media/platform/meson/Makefile
index 597beb8f34d1..f611c23c3718 100644
--- a/drivers/media/platform/meson/Makefile
+++ b/drivers/media/platform/meson/Makefile
@@ -1 +1,2 @@
 obj-$(CONFIG_VIDEO_MESON_AO_CEC)	+= ao-cec.o
+obj-$(CONFIG_VIDEO_MESON_G12A_AO_CEC)	+= ao-cec-g12a.o
diff --git a/drivers/media/platform/meson/ao-cec-g12a.c b/drivers/media/platform/meson/ao-cec-g12a.c
new file mode 100644
index 000000000000..3620a1e310f5
--- /dev/null
+++ b/drivers/media/platform/meson/ao-cec-g12a.c
@@ -0,0 +1,779 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Driver for Amlogic Meson AO CEC G12A Controller
+ *
+ * Copyright (C) 2017 Amlogic, Inc. All rights reserved
+ * Copyright (C) 2019 BayLibre, SAS
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ */
+
+#include <linux/bitfield.h>
+#include <linux/clk.h>
+#include <linux/device.h>
+#include <linux/io.h>
+#include <linux/delay.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/types.h>
+#include <linux/interrupt.h>
+#include <linux/reset.h>
+#include <linux/slab.h>
+#include <linux/regmap.h>
+#include <media/cec.h>
+#include <media/cec-notifier.h>
+#include <linux/clk-provider.h>
+
+/* CEC Registers */
+
+#define CECB_CLK_CNTL_REG0		0x00
+
+#define CECB_CLK_CNTL_N1		GENMASK(11, 0)
+#define CECB_CLK_CNTL_N2		GENMASK(23, 12)
+#define CECB_CLK_CNTL_DUAL_EN		BIT(28)
+#define CECB_CLK_CNTL_OUTPUT_EN		BIT(30)
+#define CECB_CLK_CNTL_INPUT_EN		BIT(31)
+
+#define CECB_CLK_CNTL_REG1		0x04
+
+#define CECB_CLK_CNTL_M1		GENMASK(11, 0)
+#define CECB_CLK_CNTL_M2		GENMASK(23, 12)
+#define CECB_CLK_CNTL_BYPASS_EN		BIT(24)
+
+/*
+ * [14:12] Filter_del. For glitch-filtering CEC line, ignore signal
+ *       change pulse width < filter_del * T(filter_tick) * 3.
+ * [9:8] Filter_tick_sel: Select which periodical pulse for
+ *       glitch-filtering CEC line signal.
+ *  - 0=Use T(xtal)*3 = 125ns;
+ *  - 1=Use once-per-1us pulse;
+ *  - 2=Use once-per-10us pulse;
+ *  - 3=Use once-per-100us pulse.
+ * [3]   Sysclk_en. 0=Disable system clock; 1=Enable system clock.
+ * [2:1] cntl_clk
+ *  - 0 = Disable clk (Power-off mode)
+ *  - 1 = Enable gated clock (Normal mode)
+ *  - 2 = Enable free-run clk (Debug mode)
+ * [0] SW_RESET 1=Apply reset; 0=No reset.
+ */
+#define CECB_GEN_CNTL_REG		0x08
+
+#define CECB_GEN_CNTL_RESET		BIT(0)
+#define CECB_GEN_CNTL_CLK_DISABLE	0
+#define CECB_GEN_CNTL_CLK_ENABLE	1
+#define CECB_GEN_CNTL_CLK_ENABLE_DBG	2
+#define CECB_GEN_CNTL_CLK_CTRL_MASK	GENMASK(2, 1)
+#define CECB_GEN_CNTL_SYS_CLK_EN	BIT(3)
+#define CECB_GEN_CNTL_FILTER_TICK_125NS	0
+#define CECB_GEN_CNTL_FILTER_TICK_1US	1
+#define CECB_GEN_CNTL_FILTER_TICK_10US	2
+#define CECB_GEN_CNTL_FILTER_TICK_100US	3
+#define CECB_GEN_CNTL_FILTER_TICK_SEL	GENMASK(9, 8)
+#define CECB_GEN_CNTL_FILTER_DEL	GENMASK(14, 12)
+
+/*
+ * [7:0] cec_reg_addr
+ * [15:8] cec_reg_wrdata
+ * [16] cec_reg_wr
+ *  - 0 = Read
+ *  - 1 = Write
+ * [31:24] cec_reg_rddata
+ */
+#define CECB_RW_REG			0x0c
+
+#define CECB_RW_ADDR			GENMASK(7, 0)
+#define CECB_RW_WR_DATA			GENMASK(15, 8)
+#define CECB_RW_WRITE_EN		BIT(16)
+#define CECB_RW_BUS_BUSY		BIT(23)
+#define CECB_RW_RD_DATA			GENMASK(31, 24)
+
+/*
+ * [0] DONE Interrupt
+ * [1] End Of Message Interrupt
+ * [2] Not Acknowlegde Interrupt
+ * [3] Arbitration Loss Interrupt
+ * [4] Initiator Error Interrupt
+ * [5] Follower Error Interrupt
+ * [6] Wake-Up Interrupt
+ */
+#define CECB_INTR_MASKN_REG		0x10
+#define CECB_INTR_CLR_REG		0x14
+#define CECB_INTR_STAT_REG		0x18
+
+#define CECB_INTR_DONE			BIT(0)
+#define CECB_INTR_EOM			BIT(1)
+#define CECB_INTR_NACK			BIT(2)
+#define CECB_INTR_ARB_LOSS		BIT(3)
+#define CECB_INTR_INITIATOR_ERR		BIT(4)
+#define CECB_INTR_FOLLOWER_ERR		BIT(5)
+#define CECB_INTR_WAKE_UP		BIT(6)
+
+/* CEC Commands */
+
+#define CECB_CTRL		0x00
+
+#define CECB_CTRL_SEND		BIT(0)
+#define CECB_CTRL_TYPE		GENMASK(2, 1)
+#define CECB_CTRL_TYPE_RETRY	0
+#define CECB_CTRL_TYPE_NEW	1
+#define CECB_CTRL_TYPE_NEXT	2
+
+#define CECB_CTRL2		0x01
+#define CECB_INTR_MASK		0x02
+#define CECB_LADD_LOW		0x05
+#define CECB_LADD_HIGH		0x06
+#define CECB_TX_CNT		0x07
+#define CECB_RX_CNT		0x08
+#define CECB_STAT0		0x09
+#define CECB_TX_DATA00		0x10
+#define CECB_TX_DATA01		0x11
+#define CECB_TX_DATA02		0x12
+#define CECB_TX_DATA03		0x13
+#define CECB_TX_DATA04		0x14
+#define CECB_TX_DATA05		0x15
+#define CECB_TX_DATA06		0x16
+#define CECB_TX_DATA07		0x17
+#define CECB_TX_DATA08		0x18
+#define CECB_TX_DATA09		0x19
+#define CECB_TX_DATA10		0x1A
+#define CECB_TX_DATA11		0x1B
+#define CECB_TX_DATA12		0x1C
+#define CECB_TX_DATA13		0x1D
+#define CECB_TX_DATA14		0x1E
+#define CECB_TX_DATA15		0x1F
+#define CECB_RX_DATA00		0x20
+#define CECB_RX_DATA01		0x21
+#define CECB_RX_DATA02		0x22
+#define CECB_RX_DATA03		0x23
+#define CECB_RX_DATA04		0x24
+#define CECB_RX_DATA05		0x25
+#define CECB_RX_DATA06		0x26
+#define CECB_RX_DATA07		0x27
+#define CECB_RX_DATA08		0x28
+#define CECB_RX_DATA09		0x29
+#define CECB_RX_DATA10		0x2A
+#define CECB_RX_DATA11		0x2B
+#define CECB_RX_DATA12		0x2C
+#define CECB_RX_DATA13		0x2D
+#define CECB_RX_DATA14		0x2E
+#define CECB_RX_DATA15		0x2F
+#define CECB_LOCK_BUF		0x30
+
+#define CECB_LOCK_BUF_EN	BIT(0)
+
+#define CECB_WAKEUPCTRL		0x31
+
+struct meson_ao_cec_g12a_device {
+	struct platform_device		*pdev;
+	struct regmap			*regmap;
+	struct regmap			*regmap_cec;
+	spinlock_t			cec_reg_lock;
+	struct cec_notifier		*notify;
+	struct cec_adapter		*adap;
+	struct cec_msg			rx_msg;
+	struct clk			*oscin;
+	struct clk			*core;
+};
+
+static const struct regmap_config meson_ao_cec_g12a_regmap_conf = {
+	.reg_bits = 8,
+	.val_bits = 32,
+	.reg_stride = 4,
+	.max_register = CECB_INTR_STAT_REG,
+};
+
+/*
+ * The AO-CECB embeds a dual/divider to generate a more precise
+ * 32,768KHz clock for CEC core clock.
+ *                      ______   ______
+ *                     |      | |      |
+ *         ______      | Div1 |-| Cnt1 |       ______
+ *        |      |    /|______| |______|\     |      |
+ * Xtal-->| Gate |---|  ______   ______  X-X--| Gate |-->
+ *        |______| |  \|      | |      |/  |  |______|
+ *                 |   | Div2 |-| Cnt2 |   |
+ *                 |   |______| |______|   |
+ *                 |_______________________|
+ *
+ * The dividing can be switched to single or dual, with a counter
+ * for each divider to set when the switching is done.
+ * The entire dividing mechanism can be also bypassed.
+ */
+
+struct meson_ao_cec_g12a_dualdiv_clk {
+	struct clk_hw hw;
+	struct regmap *regmap;
+};
+
+#define hw_to_meson_ao_cec_g12a_dualdiv_clk(_hw)			\
+	container_of(_hw, struct meson_ao_cec_g12a_dualdiv_clk, hw)	\
+
+static unsigned long
+meson_ao_cec_g12a_dualdiv_clk_recalc_rate(struct clk_hw *hw,
+					  unsigned long parent_rate)
+{
+	struct meson_ao_cec_g12a_dualdiv_clk *dualdiv_clk =
+		hw_to_meson_ao_cec_g12a_dualdiv_clk(hw);
+	unsigned long n1;
+	u32 reg0, reg1;
+
+	regmap_read(dualdiv_clk->regmap, CECB_CLK_CNTL_REG0, &reg0);
+	regmap_read(dualdiv_clk->regmap, CECB_CLK_CNTL_REG0, &reg1);
+
+	if (reg1 & CECB_CLK_CNTL_BYPASS_EN)
+		return parent_rate;
+
+	if (reg0 & CECB_CLK_CNTL_DUAL_EN) {
+		unsigned long n2, m1, m2, f1, f2, p1, p2;
+
+		n1 = FIELD_GET(CECB_CLK_CNTL_N1, reg0) + 1;
+		n2 = FIELD_GET(CECB_CLK_CNTL_N2, reg0) + 1;
+
+		m1 = FIELD_GET(CECB_CLK_CNTL_M1, reg1) + 1;
+		m2 = FIELD_GET(CECB_CLK_CNTL_M1, reg1) + 1;
+
+		f1 = DIV_ROUND_CLOSEST(parent_rate, n1);
+		f2 = DIV_ROUND_CLOSEST(parent_rate, n2);
+
+		p1 = DIV_ROUND_CLOSEST(100000000 * m1, f1 * (m1 + m2));
+		p2 = DIV_ROUND_CLOSEST(100000000 * m2, f2 * (m1 + m2));
+
+		return DIV_ROUND_UP(100000000, p1 + p2);
+	}
+
+	n1 = FIELD_GET(CECB_CLK_CNTL_N1, reg0) + 1;
+
+	return DIV_ROUND_CLOSEST(parent_rate, n1);
+}
+
+static int meson_ao_cec_g12a_dualdiv_clk_enable(struct clk_hw *hw)
+{
+	struct meson_ao_cec_g12a_dualdiv_clk *dualdiv_clk =
+		hw_to_meson_ao_cec_g12a_dualdiv_clk(hw);
+
+
+	/* Disable Input & Output */
+	regmap_update_bits(dualdiv_clk->regmap, CECB_CLK_CNTL_REG0,
+			   CECB_CLK_CNTL_INPUT_EN | CECB_CLK_CNTL_OUTPUT_EN,
+			   0);
+
+	/* Set N1 & N2 */
+	regmap_update_bits(dualdiv_clk->regmap, CECB_CLK_CNTL_REG0,
+			   CECB_CLK_CNTL_N1,
+			   FIELD_PREP(CECB_CLK_CNTL_N1, 733 - 1));
+
+	regmap_update_bits(dualdiv_clk->regmap, CECB_CLK_CNTL_REG0,
+			   CECB_CLK_CNTL_N2,
+			   FIELD_PREP(CECB_CLK_CNTL_N2, 732 - 1));
+
+	/* Set M1 & M2 */
+	regmap_update_bits(dualdiv_clk->regmap, CECB_CLK_CNTL_REG1,
+			   CECB_CLK_CNTL_M1,
+			   FIELD_PREP(CECB_CLK_CNTL_M1, 8 - 1));
+
+	regmap_update_bits(dualdiv_clk->regmap, CECB_CLK_CNTL_REG1,
+			   CECB_CLK_CNTL_M2,
+			   FIELD_PREP(CECB_CLK_CNTL_M2, 11 - 1));
+
+	/* Enable Dual divisor */
+	regmap_update_bits(dualdiv_clk->regmap, CECB_CLK_CNTL_REG0,
+			   CECB_CLK_CNTL_DUAL_EN, CECB_CLK_CNTL_DUAL_EN);
+
+	/* Disable divisor bypass */
+	regmap_update_bits(dualdiv_clk->regmap, CECB_CLK_CNTL_REG1,
+			   CECB_CLK_CNTL_BYPASS_EN, 0);
+
+	/* Enable Input & Output */
+	regmap_update_bits(dualdiv_clk->regmap, CECB_CLK_CNTL_REG0,
+			   CECB_CLK_CNTL_INPUT_EN | CECB_CLK_CNTL_OUTPUT_EN,
+			   CECB_CLK_CNTL_INPUT_EN | CECB_CLK_CNTL_OUTPUT_EN);
+
+	return 0;
+}
+
+static void meson_ao_cec_g12a_dualdiv_clk_disable(struct clk_hw *hw)
+{
+	struct meson_ao_cec_g12a_dualdiv_clk *dualdiv_clk =
+		hw_to_meson_ao_cec_g12a_dualdiv_clk(hw);
+
+	regmap_update_bits(dualdiv_clk->regmap, CECB_CLK_CNTL_REG0,
+			   CECB_CLK_CNTL_INPUT_EN | CECB_CLK_CNTL_OUTPUT_EN,
+			   0);
+}
+
+static int meson_ao_cec_g12a_dualdiv_clk_is_enabled(struct clk_hw *hw)
+{
+	struct meson_ao_cec_g12a_dualdiv_clk *dualdiv_clk =
+		hw_to_meson_ao_cec_g12a_dualdiv_clk(hw);
+	int val;
+
+	regmap_read(dualdiv_clk->regmap, CECB_CLK_CNTL_REG0, &val);
+
+	return !!(val & (CECB_CLK_CNTL_INPUT_EN | CECB_CLK_CNTL_OUTPUT_EN));
+}
+
+static const struct clk_ops meson_ao_cec_g12a_dualdiv_clk_ops = {
+	.recalc_rate	= meson_ao_cec_g12a_dualdiv_clk_recalc_rate,
+	.is_enabled	= meson_ao_cec_g12a_dualdiv_clk_is_enabled,
+	.enable		= meson_ao_cec_g12a_dualdiv_clk_enable,
+	.disable	= meson_ao_cec_g12a_dualdiv_clk_disable,
+};
+
+static int meson_ao_cec_g12a_setup_clk(struct meson_ao_cec_g12a_device *ao_cec)
+{
+	struct meson_ao_cec_g12a_dualdiv_clk *dualdiv_clk;
+	struct device *dev = &ao_cec->pdev->dev;
+	struct clk_init_data init;
+	const char *parent_name;
+	struct clk *clk;
+	char *name;
+
+	dualdiv_clk = devm_kzalloc(dev, sizeof(*dualdiv_clk), GFP_KERNEL);
+	if (!dualdiv_clk)
+		return -ENOMEM;
+
+	name = kasprintf(GFP_KERNEL, "%s#dualdiv_clk", dev_name(dev));
+	if (!name)
+		return -ENOMEM;
+
+	parent_name = __clk_get_name(ao_cec->oscin);
+
+	init.name = name;
+	init.ops = &meson_ao_cec_g12a_dualdiv_clk_ops;
+	init.flags = 0;
+	init.parent_names = &parent_name;
+	init.num_parents = 1;
+	dualdiv_clk->regmap = ao_cec->regmap;
+	dualdiv_clk->hw.init = &init;
+
+	clk = devm_clk_register(dev, &dualdiv_clk->hw);
+	kfree(name);
+	if (IS_ERR(clk)) {
+		dev_err(dev, "failed to register clock\n");
+		return PTR_ERR(clk);
+	}
+
+	ao_cec->core = clk;
+
+	return 0;
+}
+
+static int meson_ao_cec_g12a_read(void *context, unsigned int addr,
+				  unsigned int *data)
+{
+	struct meson_ao_cec_g12a_device *ao_cec = context;
+	u32 reg = FIELD_PREP(CECB_RW_ADDR, addr);
+	unsigned long flags;
+	int ret = 0;
+
+	spin_lock_irqsave(&ao_cec->cec_reg_lock, flags);
+
+	ret = regmap_write(ao_cec->regmap, CECB_RW_REG, reg);
+	if (ret)
+		goto read_out;
+
+	ret = regmap_read_poll_timeout(ao_cec->regmap, CECB_RW_REG, reg,
+				       !(reg & CECB_RW_BUS_BUSY),
+				       5, 1000);
+	if (ret)
+		goto read_out;
+
+	ret = regmap_read(ao_cec->regmap, CECB_RW_REG, &reg);
+
+	*data = FIELD_GET(CECB_RW_RD_DATA, reg);
+
+read_out:
+	spin_unlock_irqrestore(&ao_cec->cec_reg_lock, flags);
+
+	return ret;
+}
+
+static int meson_ao_cec_g12a_write(void *context, unsigned int addr,
+				   unsigned int data)
+{
+	struct meson_ao_cec_g12a_device *ao_cec = context;
+	unsigned long flags;
+	u32 reg = FIELD_PREP(CECB_RW_ADDR, addr) |
+		  FIELD_PREP(CECB_RW_WR_DATA, data) |
+		  CECB_RW_WRITE_EN;
+	int ret = 0;
+
+	spin_lock_irqsave(&ao_cec->cec_reg_lock, flags);
+
+	ret = regmap_write(ao_cec->regmap, CECB_RW_REG, reg);
+
+	spin_unlock_irqrestore(&ao_cec->cec_reg_lock, flags);
+
+	return ret;
+}
+
+static const struct regmap_config meson_ao_cec_g12a_cec_regmap_conf = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.reg_read = meson_ao_cec_g12a_read,
+	.reg_write = meson_ao_cec_g12a_write,
+	.max_register = 0xffff,
+	.fast_io = true,
+};
+
+static inline void
+meson_ao_cec_g12a_irq_setup(struct meson_ao_cec_g12a_device *ao_cec,
+			    bool enable)
+{
+	u32 cfg = CECB_INTR_DONE | CECB_INTR_EOM | CECB_INTR_NACK |
+		  CECB_INTR_ARB_LOSS | CECB_INTR_INITIATOR_ERR |
+		  CECB_INTR_FOLLOWER_ERR;
+
+	regmap_write(ao_cec->regmap, CECB_INTR_MASKN_REG,
+		     enable ? cfg : 0);
+}
+
+static void meson_ao_cec_g12a_irq_rx(struct meson_ao_cec_g12a_device *ao_cec)
+{
+	int i, ret = 0;
+	u32 val;
+
+	ret = regmap_read(ao_cec->regmap_cec, CECB_RX_CNT, &val);
+
+	ao_cec->rx_msg.len = val;
+	if (ao_cec->rx_msg.len > CEC_MAX_MSG_SIZE)
+		ao_cec->rx_msg.len = CEC_MAX_MSG_SIZE;
+
+	for (i = 0; i < ao_cec->rx_msg.len; i++) {
+		ret |= regmap_read(ao_cec->regmap_cec,
+				   CECB_RX_DATA00 + i, &val);
+
+		ao_cec->rx_msg.msg[i] = val & 0xff;
+	}
+
+	ret |= regmap_write(ao_cec->regmap_cec, CECB_LOCK_BUF, 0);
+	if (ret)
+		return;
+
+	cec_received_msg(ao_cec->adap, &ao_cec->rx_msg);
+}
+
+static irqreturn_t meson_ao_cec_g12a_irq(int irq, void *data)
+{
+	struct meson_ao_cec_g12a_device *ao_cec = data;
+	u32 stat;
+
+	regmap_read(ao_cec->regmap, CECB_INTR_STAT_REG, &stat);
+	if (stat)
+		return IRQ_WAKE_THREAD;
+
+	return IRQ_NONE;
+}
+
+static irqreturn_t meson_ao_cec_g12a_irq_thread(int irq, void *data)
+{
+	struct meson_ao_cec_g12a_device *ao_cec = data;
+	u32 stat;
+
+	regmap_read(ao_cec->regmap, CECB_INTR_STAT_REG, &stat);
+	regmap_write(ao_cec->regmap, CECB_INTR_CLR_REG, stat);
+
+	if (stat & CECB_INTR_DONE)
+		cec_transmit_attempt_done(ao_cec->adap, CEC_TX_STATUS_OK);
+
+	if (stat & CECB_INTR_EOM)
+		meson_ao_cec_g12a_irq_rx(ao_cec);
+
+	if (stat & CECB_INTR_NACK)
+		cec_transmit_attempt_done(ao_cec->adap, CEC_TX_STATUS_NACK);
+
+	if (stat & CECB_INTR_ARB_LOSS) {
+		regmap_write(ao_cec->regmap_cec, CECB_TX_CNT, 0);
+		regmap_update_bits(ao_cec->regmap_cec, CECB_CTRL,
+				   CECB_CTRL_SEND | CECB_CTRL_TYPE, 0);
+		cec_transmit_attempt_done(ao_cec->adap, CEC_TX_STATUS_ARB_LOST);
+	}
+
+	/* Initiator reports an error on the CEC bus */
+	if (stat & CECB_INTR_INITIATOR_ERR)
+		cec_transmit_attempt_done(ao_cec->adap, CEC_TX_STATUS_ERROR);
+
+	/* Follower reports a receive error, just reset RX buffer */
+	if (stat & CECB_INTR_FOLLOWER_ERR)
+		regmap_write(ao_cec->regmap_cec, CECB_LOCK_BUF, 0);
+
+	return IRQ_HANDLED;
+}
+
+static int
+meson_ao_cec_g12a_set_log_addr(struct cec_adapter *adap, u8 logical_addr)
+{
+	struct meson_ao_cec_g12a_device *ao_cec = adap->priv;
+	int ret = 0;
+
+	if (logical_addr == CEC_LOG_ADDR_INVALID) {
+		/* Assume this will allways succeed */
+		regmap_write(ao_cec->regmap_cec, CECB_LADD_LOW, 0);
+		regmap_write(ao_cec->regmap_cec, CECB_LADD_HIGH, 0);
+
+		return 0;
+	} else if (logical_addr < 8) {
+		ret = regmap_update_bits(ao_cec->regmap_cec, CECB_LADD_LOW,
+					 BIT(logical_addr),
+					 BIT(logical_addr));
+	} else {
+		ret = regmap_update_bits(ao_cec->regmap_cec, CECB_LADD_HIGH,
+					 BIT(logical_addr - 8),
+					 BIT(logical_addr - 8));
+	}
+
+	/* Always set Broadcast/Unregistered 15 address */
+	ret |= regmap_update_bits(ao_cec->regmap_cec, CECB_LADD_HIGH,
+				  BIT(CEC_LOG_ADDR_UNREGISTERED - 8),
+				  BIT(CEC_LOG_ADDR_UNREGISTERED - 8));
+
+	return ret ? -EIO : 0;
+}
+
+static int meson_ao_cec_g12a_transmit(struct cec_adapter *adap, u8 attempts,
+				 u32 signal_free_time, struct cec_msg *msg)
+{
+	struct meson_ao_cec_g12a_device *ao_cec = adap->priv;
+	unsigned int type;
+	int ret = 0;
+	u32 val;
+	int i;
+
+	/* Check if RX is in progress */
+	ret = regmap_read(ao_cec->regmap_cec, CECB_LOCK_BUF, &val);
+	if (ret)
+		return ret;
+	if (val & CECB_LOCK_BUF_EN)
+		return -EBUSY;
+
+	/* Check if TX Busy */
+	ret = regmap_read(ao_cec->regmap_cec, CECB_CTRL, &val);
+	if (ret)
+		return ret;
+	if (val & CECB_CTRL_SEND)
+		return -EBUSY;
+
+	switch (signal_free_time) {
+	case CEC_SIGNAL_FREE_TIME_RETRY:
+		type = CECB_CTRL_TYPE_RETRY;
+		break;
+	case CEC_SIGNAL_FREE_TIME_NEXT_XFER:
+		type = CECB_CTRL_TYPE_NEXT;
+		break;
+	case CEC_SIGNAL_FREE_TIME_NEW_INITIATOR:
+	default:
+		type = CECB_CTRL_TYPE_NEW;
+		break;
+	}
+
+	for (i = 0; i < msg->len; i++)
+		ret |= regmap_write(ao_cec->regmap_cec, CECB_TX_DATA00 + i,
+				    msg->msg[i]);
+
+	ret |= regmap_write(ao_cec->regmap_cec, CECB_TX_CNT, msg->len);
+	if (ret)
+		return -EIO;
+
+	ret = regmap_update_bits(ao_cec->regmap_cec, CECB_CTRL,
+				 CECB_CTRL_SEND |
+				 CECB_CTRL_TYPE,
+				 CECB_CTRL_SEND |
+				 FIELD_PREP(CECB_CTRL_TYPE, type));
+
+	return ret;
+}
+
+static int meson_ao_cec_g12a_adap_enable(struct cec_adapter *adap, bool enable)
+{
+	struct meson_ao_cec_g12a_device *ao_cec = adap->priv;
+
+	meson_ao_cec_g12a_irq_setup(ao_cec, false);
+
+	regmap_update_bits(ao_cec->regmap, CECB_GEN_CNTL_REG,
+			   CECB_GEN_CNTL_RESET, CECB_GEN_CNTL_RESET);
+
+	if (!enable)
+		return 0;
+
+	/* Setup Filter */
+	regmap_update_bits(ao_cec->regmap, CECB_GEN_CNTL_REG,
+			   CECB_GEN_CNTL_FILTER_TICK_SEL |
+			   CECB_GEN_CNTL_FILTER_DEL,
+			   FIELD_PREP(CECB_GEN_CNTL_FILTER_TICK_SEL,
+				      CECB_GEN_CNTL_FILTER_TICK_1US) |
+			   FIELD_PREP(CECB_GEN_CNTL_FILTER_DEL, 7));
+
+	/* Enable System Clock */
+	regmap_update_bits(ao_cec->regmap, CECB_GEN_CNTL_REG,
+			   CECB_GEN_CNTL_SYS_CLK_EN,
+			   CECB_GEN_CNTL_SYS_CLK_EN);
+
+	/* Enable gated clock (Normal mode). */
+	regmap_update_bits(ao_cec->regmap, CECB_GEN_CNTL_REG,
+			   CECB_GEN_CNTL_CLK_CTRL_MASK,
+			    FIELD_PREP(CECB_GEN_CNTL_CLK_CTRL_MASK,
+				       CECB_GEN_CNTL_CLK_ENABLE));
+
+	/* Release Reset */
+	regmap_update_bits(ao_cec->regmap, CECB_GEN_CNTL_REG,
+			   CECB_GEN_CNTL_RESET, 0);
+
+	meson_ao_cec_g12a_irq_setup(ao_cec, true);
+
+	return 0;
+}
+
+static const struct cec_adap_ops meson_ao_cec_g12a_ops = {
+	.adap_enable = meson_ao_cec_g12a_adap_enable,
+	.adap_log_addr = meson_ao_cec_g12a_set_log_addr,
+	.adap_transmit = meson_ao_cec_g12a_transmit,
+};
+
+static int meson_ao_cec_g12a_probe(struct platform_device *pdev)
+{
+	struct meson_ao_cec_g12a_device *ao_cec;
+	struct device *hdmi_dev;
+	struct resource *res;
+	void __iomem *base;
+	int ret, irq;
+
+	hdmi_dev = cec_notifier_parse_hdmi_phandle(&pdev->dev);
+	if (IS_ERR(hdmi_dev))
+		return PTR_ERR(hdmi_dev);
+
+	ao_cec = devm_kzalloc(&pdev->dev, sizeof(*ao_cec), GFP_KERNEL);
+	if (!ao_cec)
+		return -ENOMEM;
+
+	spin_lock_init(&ao_cec->cec_reg_lock);
+	ao_cec->pdev = pdev;
+
+	ao_cec->notify = cec_notifier_get(hdmi_dev);
+	if (!ao_cec->notify)
+		return -ENOMEM;
+
+	ao_cec->adap = cec_allocate_adapter(&meson_ao_cec_g12a_ops, ao_cec,
+					    "meson_g12a_ao_cec",
+					    CEC_CAP_DEFAULTS,
+					    CEC_MAX_LOG_ADDRS);
+	if (IS_ERR(ao_cec->adap)) {
+		ret = PTR_ERR(ao_cec->adap);
+		goto out_probe_notify;
+	}
+
+	ao_cec->adap->owner = THIS_MODULE;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(base)) {
+		ret = PTR_ERR(base);
+		goto out_probe_adapter;
+	}
+
+	ao_cec->regmap = devm_regmap_init_mmio(&pdev->dev, base,
+					       &meson_ao_cec_g12a_regmap_conf);
+	if (IS_ERR(ao_cec->regmap)) {
+		ret = PTR_ERR(ao_cec->regmap);
+		goto out_probe_adapter;
+	}
+
+	ao_cec->regmap_cec = devm_regmap_init(&pdev->dev, NULL, ao_cec,
+					   &meson_ao_cec_g12a_cec_regmap_conf);
+	if (IS_ERR(ao_cec->regmap_cec)) {
+		ret = PTR_ERR(ao_cec->regmap_cec);
+		goto out_probe_adapter;
+	}
+
+	irq = platform_get_irq(pdev, 0);
+	ret = devm_request_threaded_irq(&pdev->dev, irq,
+					meson_ao_cec_g12a_irq,
+					meson_ao_cec_g12a_irq_thread,
+					0, NULL, ao_cec);
+	if (ret) {
+		dev_err(&pdev->dev, "irq request failed\n");
+		goto out_probe_adapter;
+	}
+
+	ao_cec->oscin = devm_clk_get(&pdev->dev, "oscin");
+	if (IS_ERR(ao_cec->oscin)) {
+		dev_err(&pdev->dev, "oscin clock request failed\n");
+		ret = PTR_ERR(ao_cec->oscin);
+		goto out_probe_adapter;
+	}
+
+	ret = meson_ao_cec_g12a_setup_clk(ao_cec);
+	if (ret)
+		goto out_probe_adapter;
+
+	ret = clk_prepare_enable(ao_cec->core);
+	if (ret) {
+		dev_err(&pdev->dev, "core clock enable failed\n");
+		goto out_probe_adapter;
+	}
+
+	device_reset_optional(&pdev->dev);
+
+	platform_set_drvdata(pdev, ao_cec);
+
+	ret = cec_register_adapter(ao_cec->adap, &pdev->dev);
+	if (ret < 0) {
+		cec_notifier_put(ao_cec->notify);
+		goto out_probe_core_clk;
+	}
+
+	/* Setup Hardware */
+	regmap_write(ao_cec->regmap, CECB_GEN_CNTL_REG, CECB_GEN_CNTL_RESET);
+
+	cec_register_cec_notifier(ao_cec->adap, ao_cec->notify);
+
+	return 0;
+
+out_probe_core_clk:
+	clk_disable_unprepare(ao_cec->core);
+
+out_probe_adapter:
+	cec_delete_adapter(ao_cec->adap);
+
+out_probe_notify:
+	cec_notifier_put(ao_cec->notify);
+
+	dev_err(&pdev->dev, "CEC controller registration failed\n");
+
+	return ret;
+}
+
+static int meson_ao_cec_g12a_remove(struct platform_device *pdev)
+{
+	struct meson_ao_cec_g12a_device *ao_cec = platform_get_drvdata(pdev);
+
+	clk_disable_unprepare(ao_cec->core);
+
+	cec_unregister_adapter(ao_cec->adap);
+
+	cec_notifier_put(ao_cec->notify);
+
+	return 0;
+}
+
+static const struct of_device_id meson_ao_cec_g12a_of_match[] = {
+	{ .compatible = "amlogic,meson-g12a-ao-cec", },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, meson_ao_cec_g12a_of_match);
+
+static struct platform_driver meson_ao_cec_g12a_driver = {
+	.probe   = meson_ao_cec_g12a_probe,
+	.remove  = meson_ao_cec_g12a_remove,
+	.driver  = {
+		.name = "meson-ao-cec-g12a",
+		.of_match_table = of_match_ptr(meson_ao_cec_g12a_of_match),
+	},
+};
+
+module_platform_driver(meson_ao_cec_g12a_driver);
+
+MODULE_DESCRIPTION("Meson AO CEC G12A Controller driver");
+MODULE_AUTHOR("Neil Armstrong <narmstrong@baylibre.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/platform/meson/ao-cec.c b/drivers/media/platform/meson/ao-cec.c
index cd4be38ab5ac..facf9b029e79 100644
--- a/drivers/media/platform/meson/ao-cec.c
+++ b/drivers/media/platform/meson/ao-cec.c
@@ -601,20 +601,14 @@ static const struct cec_adap_ops meson_ao_cec_ops = {
 static int meson_ao_cec_probe(struct platform_device *pdev)
 {
 	struct meson_ao_cec_device *ao_cec;
-	struct platform_device *hdmi_dev;
-	struct device_node *np;
+	struct device *hdmi_dev;
 	struct resource *res;
 	int ret, irq;
 
-	np = of_parse_phandle(pdev->dev.of_node, "hdmi-phandle", 0);
-	if (!np) {
-		dev_err(&pdev->dev, "Failed to find hdmi node\n");
-		return -ENODEV;
-	}
+	hdmi_dev = cec_notifier_parse_hdmi_phandle(&pdev->dev);
 
-	hdmi_dev = of_find_device_by_node(np);
-	if (hdmi_dev == NULL)
-		return -EPROBE_DEFER;
+	if (IS_ERR(hdmi_dev))
+		return PTR_ERR(hdmi_dev);
 
 	ao_cec = devm_kzalloc(&pdev->dev, sizeof(*ao_cec), GFP_KERNEL);
 	if (!ao_cec)
@@ -622,7 +616,7 @@ static int meson_ao_cec_probe(struct platform_device *pdev)
 
 	spin_lock_init(&ao_cec->cec_reg_lock);
 
-	ao_cec->notify = cec_notifier_get(&hdmi_dev->dev);
+	ao_cec->notify = cec_notifier_get(hdmi_dev);
 	if (!ao_cec->notify)
 		return -ENOMEM;
 
diff --git a/drivers/media/platform/mtk-vcodec/mtk_vcodec_dec.c b/drivers/media/platform/mtk-vcodec/mtk_vcodec_dec.c
index d022c65bb34c..851903867bc9 100644
--- a/drivers/media/platform/mtk-vcodec/mtk_vcodec_dec.c
+++ b/drivers/media/platform/mtk-vcodec/mtk_vcodec_dec.c
@@ -129,11 +129,9 @@ static struct vb2_buffer *get_display_buffer(struct mtk_vcodec_ctx *ctx)
 	mutex_lock(&ctx->lock);
 	if (dstbuf->used) {
 		vb2_set_plane_payload(&dstbuf->vb.vb2_buf, 0,
-					ctx->picinfo.y_bs_sz);
+					ctx->picinfo.fb_sz[0]);
 		vb2_set_plane_payload(&dstbuf->vb.vb2_buf, 1,
-					ctx->picinfo.c_bs_sz);
-
-		dstbuf->ready_to_display = true;
+					ctx->picinfo.fb_sz[1]);
 
 		mtk_v4l2_debug(2,
 				"[%d]status=%x queue id=%d to done_list %d",
@@ -278,6 +276,27 @@ static void mtk_vdec_flush_decoder(struct mtk_vcodec_ctx *ctx)
 	clean_free_buffer(ctx);
 }
 
+static void mtk_vdec_update_fmt(struct mtk_vcodec_ctx *ctx,
+				unsigned int pixelformat)
+{
+	struct mtk_video_fmt *fmt;
+	struct mtk_q_data *dst_q_data;
+	unsigned int k;
+
+	dst_q_data = &ctx->q_data[MTK_Q_DATA_DST];
+	for (k = 0; k < NUM_FORMATS; k++) {
+		fmt = &mtk_video_formats[k];
+		if (fmt->fourcc == pixelformat) {
+			mtk_v4l2_debug(1, "Update cap fourcc(%d -> %d)",
+				dst_q_data->fmt.fourcc, pixelformat);
+			dst_q_data->fmt = fmt;
+			return;
+		}
+	}
+
+	mtk_v4l2_err("Cannot get fourcc(%d), using init value", pixelformat);
+}
+
 static int mtk_vdec_pic_info_update(struct mtk_vcodec_ctx *ctx)
 {
 	unsigned int dpbsize = 0;
@@ -299,6 +318,10 @@ static int mtk_vdec_pic_info_update(struct mtk_vcodec_ctx *ctx)
 		return -EINVAL;
 	}
 
+	if (ctx->last_decoded_picinfo.cap_fourcc != ctx->picinfo.cap_fourcc &&
+		ctx->picinfo.cap_fourcc != 0)
+		mtk_vdec_update_fmt(ctx, ctx->picinfo.cap_fourcc);
+
 	if ((ctx->last_decoded_picinfo.pic_w == ctx->picinfo.pic_w) ||
 	    (ctx->last_decoded_picinfo.pic_h == ctx->picinfo.pic_h))
 		return 0;
@@ -352,11 +375,11 @@ static void mtk_vdec_worker(struct work_struct *work)
 	pfb = &dst_buf_info->frame_buffer;
 	pfb->base_y.va = vb2_plane_vaddr(&dst_buf->vb2_buf, 0);
 	pfb->base_y.dma_addr = vb2_dma_contig_plane_dma_addr(&dst_buf->vb2_buf, 0);
-	pfb->base_y.size = ctx->picinfo.y_bs_sz + ctx->picinfo.y_len_sz;
+	pfb->base_y.size = ctx->picinfo.fb_sz[0];
 
 	pfb->base_c.va = vb2_plane_vaddr(&dst_buf->vb2_buf, 1);
 	pfb->base_c.dma_addr = vb2_dma_contig_plane_dma_addr(&dst_buf->vb2_buf, 1);
-	pfb->base_c.size = ctx->picinfo.c_bs_sz + ctx->picinfo.c_len_sz;
+	pfb->base_c.size = ctx->picinfo.fb_sz[1];
 	pfb->status = 0;
 	mtk_v4l2_debug(3, "===>[%d] vdec_if_decode() ===>", ctx->id);
 
@@ -388,7 +411,7 @@ static void mtk_vdec_worker(struct work_struct *work)
 	}
 	buf.va = vb2_plane_vaddr(&src_buf->vb2_buf, 0);
 	buf.dma_addr = vb2_dma_contig_plane_dma_addr(&src_buf->vb2_buf, 0);
-	buf.size = (size_t)src_buf->planes[0].bytesused;
+	buf.size = (size_t)src_buf->vb2_buf.planes[0].bytesused;
 	if (!buf.va) {
 		v4l2_m2m_job_finish(dev->m2m_dev_dec, ctx->m2m_ctx);
 		mtk_v4l2_err("[%d] id=%d src_addr is NULL!!",
@@ -976,14 +999,13 @@ static int vidioc_vdec_g_fmt(struct file *file, void *priv,
 		 * So we just return picinfo yet, and update picinfo in
 		 * stop_streaming hook function
 		 */
-		q_data->sizeimage[0] = ctx->picinfo.y_bs_sz +
-					ctx->picinfo.y_len_sz;
-		q_data->sizeimage[1] = ctx->picinfo.c_bs_sz +
-					ctx->picinfo.c_len_sz;
+		q_data->sizeimage[0] = ctx->picinfo.fb_sz[0];
+		q_data->sizeimage[1] = ctx->picinfo.fb_sz[1];
 		q_data->bytesperline[0] = ctx->last_decoded_picinfo.buf_w;
 		q_data->bytesperline[1] = ctx->last_decoded_picinfo.buf_w;
 		q_data->coded_width = ctx->picinfo.buf_w;
 		q_data->coded_height = ctx->picinfo.buf_h;
+		ctx->last_decoded_picinfo.cap_fourcc = q_data->fmt->fourcc;
 
 		/*
 		 * Width and height are set to the dimensions
@@ -1103,10 +1125,11 @@ static void vb2ops_vdec_buf_queue(struct vb2_buffer *vb)
 	struct mtk_vcodec_mem src_mem;
 	bool res_chg = false;
 	int ret = 0;
-	unsigned int dpbsize = 1;
+	unsigned int dpbsize = 1, i = 0;
 	struct mtk_vcodec_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue);
 	struct vb2_v4l2_buffer *vb2_v4l2 = NULL;
 	struct mtk_video_dec_buf *buf = NULL;
+	struct mtk_q_data *dst_q_data;
 
 	mtk_v4l2_debug(3, "[%d] (%d) id=%d, vb=%p",
 			ctx->id, vb->vb2_queue->type,
@@ -1122,11 +1145,9 @@ static void vb2ops_vdec_buf_queue(struct vb2_buffer *vb)
 			v4l2_m2m_buf_queue(ctx->m2m_ctx, vb2_v4l2);
 			buf->queued_in_vb2 = true;
 			buf->queued_in_v4l2 = true;
-			buf->ready_to_display = false;
 		} else {
 			buf->queued_in_vb2 = false;
 			buf->queued_in_v4l2 = true;
-			buf->ready_to_display = false;
 		}
 		mutex_unlock(&ctx->lock);
 		return;
@@ -1155,10 +1176,10 @@ static void vb2ops_vdec_buf_queue(struct vb2_buffer *vb)
 
 	src_mem.va = vb2_plane_vaddr(&src_buf->vb2_buf, 0);
 	src_mem.dma_addr = vb2_dma_contig_plane_dma_addr(&src_buf->vb2_buf, 0);
-	src_mem.size = (size_t)src_buf->planes[0].bytesused;
+	src_mem.size = (size_t)src_buf->vb2_buf.planes[0].bytesused;
 	mtk_v4l2_debug(2,
 			"[%d] buf id=%d va=%p dma=%pad size=%zx",
-			ctx->id, src_buf->index,
+			ctx->id, src_buf->vb2_buf.index,
 			src_mem.va, &src_mem.dma_addr,
 			src_mem.size);
 
@@ -1182,7 +1203,7 @@ static void vb2ops_vdec_buf_queue(struct vb2_buffer *vb)
 		}
 		mtk_v4l2_debug(ret ? 0 : 1,
 			       "[%d] vdec_if_decode() src_buf=%d, size=%zu, fail=%d, res_chg=%d",
-			       ctx->id, src_buf->index,
+			       ctx->id, src_buf->vb2_buf.index,
 			       src_mem.size, ret, res_chg);
 		return;
 	}
@@ -1194,21 +1215,18 @@ static void vb2ops_vdec_buf_queue(struct vb2_buffer *vb)
 	}
 
 	ctx->last_decoded_picinfo = ctx->picinfo;
-	ctx->q_data[MTK_Q_DATA_DST].sizeimage[0] =
-						ctx->picinfo.y_bs_sz +
-						ctx->picinfo.y_len_sz;
-	ctx->q_data[MTK_Q_DATA_DST].bytesperline[0] =
-						ctx->picinfo.buf_w;
-	ctx->q_data[MTK_Q_DATA_DST].sizeimage[1] =
-						ctx->picinfo.c_bs_sz +
-						ctx->picinfo.c_len_sz;
-	ctx->q_data[MTK_Q_DATA_DST].bytesperline[1] = ctx->picinfo.buf_w;
+	dst_q_data = &ctx->q_data[MTK_Q_DATA_DST];
+	for (i = 0; i < dst_q_data->fmt->num_planes; i++) {
+		dst_q_data->sizeimage[i] = ctx->picinfo.fb_sz[i];
+		dst_q_data->bytesperline[i] = ctx->picinfo.buf_w;
+	}
+
 	mtk_v4l2_debug(2, "[%d] vdec_if_init() OK wxh=%dx%d pic wxh=%dx%d sz[0]=0x%x sz[1]=0x%x",
 			ctx->id,
 			ctx->picinfo.buf_w, ctx->picinfo.buf_h,
 			ctx->picinfo.pic_w, ctx->picinfo.pic_h,
-			ctx->q_data[MTK_Q_DATA_DST].sizeimage[0],
-			ctx->q_data[MTK_Q_DATA_DST].sizeimage[1]);
+			dst_q_data->sizeimage[0],
+			dst_q_data->sizeimage[1]);
 
 	ret = vdec_if_get_param(ctx, GET_PARAM_DPB_SIZE, &dpbsize);
 	if (dpbsize == 0)
@@ -1253,7 +1271,6 @@ static int vb2ops_vdec_buf_init(struct vb2_buffer *vb)
 
 	if (vb->vb2_queue->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) {
 		buf->used = false;
-		buf->ready_to_display = false;
 		buf->queued_in_v4l2 = false;
 	} else {
 		buf->lastframe = false;
diff --git a/drivers/media/platform/mtk-vcodec/mtk_vcodec_dec.h b/drivers/media/platform/mtk-vcodec/mtk_vcodec_dec.h
index dc4fc1df63c5..e4984edec4f8 100644
--- a/drivers/media/platform/mtk-vcodec/mtk_vcodec_dec.h
+++ b/drivers/media/platform/mtk-vcodec/mtk_vcodec_dec.h
@@ -45,7 +45,6 @@ struct vdec_fb {
  * @list:	link list
  * @used:	Capture buffer contain decoded frame data and keep in
  *			codec data structure
- * @ready_to_display:	Capture buffer not display yet
  * @queued_in_vb2:	Capture buffer is queue in vb2
  * @queued_in_v4l2:	Capture buffer is in v4l2 driver, but not in vb2
  *			queue yet
@@ -60,7 +59,6 @@ struct mtk_video_dec_buf {
 	struct list_head	list;
 
 	bool	used;
-	bool	ready_to_display;
 	bool	queued_in_vb2;
 	bool	queued_in_v4l2;
 	bool	lastframe;
diff --git a/drivers/media/platform/mtk-vcodec/mtk_vcodec_drv.h b/drivers/media/platform/mtk-vcodec/mtk_vcodec_drv.h
index e7e2a108def9..662a84b822af 100644
--- a/drivers/media/platform/mtk-vcodec/mtk_vcodec_drv.h
+++ b/drivers/media/platform/mtk-vcodec/mtk_vcodec_drv.h
@@ -211,24 +211,20 @@ struct mtk_vcodec_pm {
  * @pic_h: picture height
  * @buf_w: picture buffer width (64 aligned up from pic_w)
  * @buf_h: picture buffer heiht (64 aligned up from pic_h)
- * @y_bs_sz: Y bitstream size
- * @c_bs_sz: CbCr bitstream size
- * @y_len_sz: additional size required to store decompress information for y
- *		plane
- * @c_len_sz: additional size required to store decompress information for cbcr
- *		plane
+ * @fb_sz: bitstream size of each plane
  * E.g. suppose picture size is 176x144,
  *      buffer size will be aligned to 176x160.
+ * @cap_fourcc: fourcc number(may changed when resolution change)
+ * @reserved: align struct to 64-bit in order to adjust 32-bit and 64-bit os.
  */
 struct vdec_pic_info {
 	unsigned int pic_w;
 	unsigned int pic_h;
 	unsigned int buf_w;
 	unsigned int buf_h;
-	unsigned int y_bs_sz;
-	unsigned int c_bs_sz;
-	unsigned int y_len_sz;
-	unsigned int c_len_sz;
+	unsigned int fb_sz[VIDEO_MAX_PLANES];
+	unsigned int cap_fourcc;
+	unsigned int reserved;
 };
 
 /**
diff --git a/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc.c b/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc.c
index c6b48b5925fb..50351adafc47 100644
--- a/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc.c
+++ b/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc.c
@@ -894,7 +894,7 @@ static void vb2ops_venc_stop_streaming(struct vb2_queue *q)
 
 	if (q->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) {
 		while ((dst_buf = v4l2_m2m_dst_buf_remove(ctx->m2m_ctx))) {
-			dst_buf->planes[0].bytesused = 0;
+			dst_buf->vb2_buf.planes[0].bytesused = 0;
 			v4l2_m2m_buf_done(dst_buf, VB2_BUF_STATE_ERROR);
 		}
 	} else {
@@ -947,7 +947,7 @@ static int mtk_venc_encode_header(void *priv)
 
 	bs_buf.va = vb2_plane_vaddr(&dst_buf->vb2_buf, 0);
 	bs_buf.dma_addr = vb2_dma_contig_plane_dma_addr(&dst_buf->vb2_buf, 0);
-	bs_buf.size = (size_t)dst_buf->planes[0].length;
+	bs_buf.size = (size_t)dst_buf->vb2_buf.planes[0].length;
 
 	mtk_v4l2_debug(1,
 			"[%d] buf id=%d va=0x%p dma_addr=0x%llx size=%zu",
@@ -976,7 +976,7 @@ static int mtk_venc_encode_header(void *priv)
 	}
 
 	ctx->state = MTK_STATE_HEADER;
-	dst_buf->planes[0].bytesused = enc_result.bs_size;
+	dst_buf->vb2_buf.planes[0].bytesused = enc_result.bs_size;
 	v4l2_m2m_buf_done(dst_buf, VB2_BUF_STATE_DONE);
 
 	return 0;
@@ -1107,12 +1107,12 @@ static void mtk_venc_worker(struct work_struct *work)
 
 	if (ret) {
 		v4l2_m2m_buf_done(src_buf, VB2_BUF_STATE_ERROR);
-		dst_buf->planes[0].bytesused = 0;
+		dst_buf->vb2_buf.planes[0].bytesused = 0;
 		v4l2_m2m_buf_done(dst_buf, VB2_BUF_STATE_ERROR);
 		mtk_v4l2_err("venc_if_encode failed=%d", ret);
 	} else {
 		v4l2_m2m_buf_done(src_buf, VB2_BUF_STATE_DONE);
-		dst_buf->planes[0].bytesused = enc_result.bs_size;
+		dst_buf->vb2_buf.planes[0].bytesused = enc_result.bs_size;
 		v4l2_m2m_buf_done(dst_buf, VB2_BUF_STATE_DONE);
 		mtk_v4l2_debug(2, "venc_if_encode bs size=%d",
 				 enc_result.bs_size);
diff --git a/drivers/media/platform/mtk-vcodec/vdec/vdec_h264_if.c b/drivers/media/platform/mtk-vcodec/vdec/vdec_h264_if.c
index 02c960c63ac0..cdbcd6909728 100644
--- a/drivers/media/platform/mtk-vcodec/vdec/vdec_h264_if.c
+++ b/drivers/media/platform/mtk-vcodec/vdec/vdec_h264_if.c
@@ -253,8 +253,8 @@ static void get_pic_info(struct vdec_h264_inst *inst,
 	*pic = inst->vsi->pic;
 	mtk_vcodec_debug(inst, "pic(%d, %d), buf(%d, %d)",
 			 pic->pic_w, pic->pic_h, pic->buf_w, pic->buf_h);
-	mtk_vcodec_debug(inst, "Y(%d, %d), C(%d, %d)", pic->y_bs_sz,
-			 pic->y_len_sz, pic->c_bs_sz, pic->c_len_sz);
+	mtk_vcodec_debug(inst, "fb size: Y(%d), C(%d)",
+			 pic->fb_sz[0], pic->fb_sz[1]);
 }
 
 static void get_crop_info(struct vdec_h264_inst *inst, struct v4l2_rect *cr)
diff --git a/drivers/media/platform/mtk-vcodec/vdec/vdec_vp8_if.c b/drivers/media/platform/mtk-vcodec/vdec/vdec_vp8_if.c
index bac3723038de..ba79136421ef 100644
--- a/drivers/media/platform/mtk-vcodec/vdec/vdec_vp8_if.c
+++ b/drivers/media/platform/mtk-vcodec/vdec/vdec_vp8_if.c
@@ -294,8 +294,8 @@ static void get_pic_info(struct vdec_vp8_inst *inst, struct vdec_pic_info *pic)
 
 	mtk_vcodec_debug(inst, "pic(%d, %d), buf(%d, %d)",
 			 pic->pic_w, pic->pic_h, pic->buf_w, pic->buf_h);
-	mtk_vcodec_debug(inst, "Y(%d, %d), C(%d, %d)", pic->y_bs_sz,
-			 pic->y_len_sz, pic->c_bs_sz, pic->c_len_sz);
+	mtk_vcodec_debug(inst, "fb size: Y(%d), C(%d)",
+			 pic->fb_sz[0], pic->fb_sz[1]);
 }
 
 static void vp8_dec_finish(struct vdec_vp8_inst *inst)
diff --git a/drivers/media/platform/mtk-vcodec/vdec/vdec_vp9_if.c b/drivers/media/platform/mtk-vcodec/vdec/vdec_vp9_if.c
index bc8349bc2e80..939ea14bf6c5 100644
--- a/drivers/media/platform/mtk-vcodec/vdec/vdec_vp9_if.c
+++ b/drivers/media/platform/mtk-vcodec/vdec/vdec_vp9_if.c
@@ -481,15 +481,15 @@ static void vp9_swap_frm_bufs(struct vdec_vp9_inst *inst)
 		 */
 		if ((frm_to_show->fb != NULL) &&
 			(inst->cur_fb->base_y.size >=
-			frm_to_show->fb->base_y.size)) {
+			frm_to_show->fb->base_y.size) &&
+			(inst->cur_fb->base_c.size >=
+			frm_to_show->fb->base_c.size)) {
 			memcpy((void *)inst->cur_fb->base_y.va,
 				(void *)frm_to_show->fb->base_y.va,
-				vsi->buf_w *
-				vsi->buf_h);
+				frm_to_show->fb->base_y.size);
 			memcpy((void *)inst->cur_fb->base_c.va,
 				(void *)frm_to_show->fb->base_c.va,
-				vsi->buf_w *
-				vsi->buf_h / 2);
+				frm_to_show->fb->base_c.size);
 		} else {
 			/* After resolution change case, current CAPTURE buffer
 			 * may have less buffer size than frm_to_show buffer
@@ -702,10 +702,8 @@ static void init_all_fb_lists(struct vdec_vp9_inst *inst)
 
 static void get_pic_info(struct vdec_vp9_inst *inst, struct vdec_pic_info *pic)
 {
-	pic->y_bs_sz = inst->vsi->buf_sz_y_bs;
-	pic->c_bs_sz = inst->vsi->buf_sz_c_bs;
-	pic->y_len_sz = inst->vsi->buf_len_sz_y;
-	pic->c_len_sz = inst->vsi->buf_len_sz_c;
+	pic->fb_sz[0] = inst->vsi->buf_sz_y_bs + inst->vsi->buf_len_sz_y;
+	pic->fb_sz[1] = inst->vsi->buf_sz_c_bs + inst->vsi->buf_len_sz_c;
 
 	pic->pic_w = inst->vsi->pic_w;
 	pic->pic_h = inst->vsi->pic_h;
@@ -714,8 +712,9 @@ static void get_pic_info(struct vdec_vp9_inst *inst, struct vdec_pic_info *pic)
 
 	mtk_vcodec_debug(inst, "pic(%d, %d), buf(%d, %d)",
 		 pic->pic_w, pic->pic_h, pic->buf_w, pic->buf_h);
-	mtk_vcodec_debug(inst, "Y(%d, %d), C(%d, %d)", pic->y_bs_sz,
-		 pic->y_len_sz, pic->c_bs_sz, pic->c_len_sz);
+	mtk_vcodec_debug(inst, "fb size: Y(%d), C(%d)",
+		pic->fb_sz[0],
+		pic->fb_sz[1]);
 }
 
 static void get_disp_fb(struct vdec_vp9_inst *inst, struct vdec_fb **out_fb)
@@ -895,7 +894,7 @@ static int vdec_vp9_decode(unsigned long h_vdec, struct mtk_vcodec_mem *bs,
 
 		if (vsi->resolution_changed) {
 			if (!vp9_alloc_work_buf(inst)) {
-				ret = -EINVAL;
+				ret = -EIO;
 				goto DECODE_ERROR;
 			}
 		}
@@ -924,14 +923,12 @@ static int vdec_vp9_decode(unsigned long h_vdec, struct mtk_vcodec_mem *bs,
 
 		if (vsi->show_existing_frame && (vsi->frm_to_show_idx <
 					VP9_MAX_FRM_BUF_NUM)) {
-			mtk_vcodec_err(inst,
+			mtk_vcodec_debug(inst,
 				"Skip Decode drv->new_fb_idx=%d, drv->frm_to_show_idx=%d",
 				vsi->new_fb_idx, vsi->frm_to_show_idx);
 
 			vp9_ref_cnt_fb(inst, &vsi->new_fb_idx,
 					vsi->frm_to_show_idx);
-			ret = -EINVAL;
-			goto DECODE_ERROR;
 		}
 
 		/* VPU assign the buffer pointer in its address space,
diff --git a/drivers/media/platform/mtk-vpu/mtk_vpu.c b/drivers/media/platform/mtk-vpu/mtk_vpu.c
index b6602490a247..46c45f93c977 100644
--- a/drivers/media/platform/mtk-vpu/mtk_vpu.c
+++ b/drivers/media/platform/mtk-vpu/mtk_vpu.c
@@ -614,7 +614,7 @@ static void vpu_init_ipi_handler(void *data, unsigned int len, void *priv)
 	struct vpu_run *run = (struct vpu_run *)data;
 
 	vpu->run.signaled = run->signaled;
-	strncpy(vpu->run.fw_ver, run->fw_ver, VPU_FW_VER_LEN);
+	strscpy(vpu->run.fw_ver, run->fw_ver, sizeof(vpu->run.fw_ver));
 	vpu->run.dec_capability = run->dec_capability;
 	vpu->run.enc_capability = run->enc_capability;
 	wake_up_interruptible(&vpu->run.wq);
diff --git a/drivers/media/platform/mx2_emmaprp.c b/drivers/media/platform/mx2_emmaprp.c
index f60f499c596b..e100b30bb6f5 100644
--- a/drivers/media/platform/mx2_emmaprp.c
+++ b/drivers/media/platform/mx2_emmaprp.c
@@ -385,8 +385,8 @@ static irqreturn_t emmaprp_irq(int irq_emma, void *data)
 static int vidioc_querycap(struct file *file, void *priv,
 			   struct v4l2_capability *cap)
 {
-	strncpy(cap->driver, MEM2MEM_NAME, sizeof(cap->driver) - 1);
-	strncpy(cap->card, MEM2MEM_NAME, sizeof(cap->card) - 1);
+	strscpy(cap->driver, MEM2MEM_NAME, sizeof(cap->driver));
+	strscpy(cap->card, MEM2MEM_NAME, sizeof(cap->card));
 	cap->device_caps = V4L2_CAP_VIDEO_M2M | V4L2_CAP_STREAMING;
 	cap->capabilities = cap->device_caps | V4L2_CAP_DEVICE_CAPS;
 	return 0;
diff --git a/drivers/media/platform/omap/Kconfig b/drivers/media/platform/omap/Kconfig
index 4b5e55d41ad4..30ce2ba120a1 100644
--- a/drivers/media/platform/omap/Kconfig
+++ b/drivers/media/platform/omap/Kconfig
@@ -14,5 +14,5 @@ config VIDEO_OMAP2_VOUT
 	select OMAP2_VRFB if ARCH_OMAP2 || ARCH_OMAP3
 	select FRAME_VECTOR
 	default n
-	---help---
+	help
 	  V4L2 Display driver support for OMAP2/3 based boards.
diff --git a/drivers/media/platform/pxa_camera.c b/drivers/media/platform/pxa_camera.c
index 4fe228752a43..a632f06d9fff 100644
--- a/drivers/media/platform/pxa_camera.c
+++ b/drivers/media/platform/pxa_camera.c
@@ -2350,7 +2350,7 @@ static int pxa_camera_pdata_from_dt(struct device *dev,
 		pcdev->platform_flags |= PXA_CAMERA_PCLK_EN;
 
 	asd->match_type = V4L2_ASYNC_MATCH_FWNODE;
-	remote = of_graph_get_remote_port(np);
+	remote = of_graph_get_remote_port_parent(np);
 	if (remote)
 		asd->match.fwnode = of_fwnode_handle(remote);
 	else
diff --git a/drivers/media/platform/qcom/venus/hfi_helper.h b/drivers/media/platform/qcom/venus/hfi_helper.h
index 15804ad7e65d..34ea503a9842 100644
--- a/drivers/media/platform/qcom/venus/hfi_helper.h
+++ b/drivers/media/platform/qcom/venus/hfi_helper.h
@@ -569,7 +569,7 @@ struct hfi_capability {
 
 struct hfi_capabilities {
 	u32 num_capabilities;
-	struct hfi_capability data[1];
+	struct hfi_capability *data;
 };
 
 #define HFI_DEBUG_MSG_LOW	0x01
@@ -726,7 +726,7 @@ struct hfi_profile_level {
 
 struct hfi_profile_level_supported {
 	u32 profile_count;
-	struct hfi_profile_level profile_level[1];
+	struct hfi_profile_level *profile_level;
 };
 
 struct hfi_quality_vs_speed {
diff --git a/drivers/media/platform/rcar-vin/Kconfig b/drivers/media/platform/rcar-vin/Kconfig
index e3eb8fee2536..240ac3f3c941 100644
--- a/drivers/media/platform/rcar-vin/Kconfig
+++ b/drivers/media/platform/rcar-vin/Kconfig
@@ -3,6 +3,7 @@ config VIDEO_RCAR_CSI2
 	tristate "R-Car MIPI CSI-2 Receiver"
 	depends on VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API && OF
 	depends on ARCH_RENESAS || COMPILE_TEST
+	select RESET_CONTROLLER
 	select V4L2_FWNODE
 	help
 	  Support for Renesas R-Car MIPI CSI-2 receiver.
@@ -17,7 +18,7 @@ config VIDEO_RCAR_VIN
 	depends on ARCH_RENESAS || COMPILE_TEST
 	select VIDEOBUF2_DMA_CONTIG
 	select V4L2_FWNODE
-	---help---
+	help
 	  Support for Renesas R-Car Video Input (VIN) driver.
 	  Supports R-Car Gen2 and Gen3 SoCs.
 
diff --git a/drivers/media/platform/rcar-vin/rcar-core.c b/drivers/media/platform/rcar-vin/rcar-core.c
index 594d80434004..64f9cf790445 100644
--- a/drivers/media/platform/rcar-vin/rcar-core.c
+++ b/drivers/media/platform/rcar-vin/rcar-core.c
@@ -546,7 +546,9 @@ static void rvin_parallel_notify_unbind(struct v4l2_async_notifier *notifier,
 
 	vin_dbg(vin, "unbind parallel subdev %s\n", subdev->name);
 
+	mutex_lock(&vin->lock);
 	rvin_parallel_subdevice_detach(vin);
+	mutex_unlock(&vin->lock);
 }
 
 static int rvin_parallel_notify_bound(struct v4l2_async_notifier *notifier,
@@ -556,7 +558,9 @@ static int rvin_parallel_notify_bound(struct v4l2_async_notifier *notifier,
 	struct rvin_dev *vin = v4l2_dev_to_vin(notifier->v4l2_dev);
 	int ret;
 
+	mutex_lock(&vin->lock);
 	ret = rvin_parallel_subdevice_attach(vin, subdev);
+	mutex_unlock(&vin->lock);
 	if (ret)
 		return ret;
 
@@ -664,6 +668,7 @@ static int rvin_group_notify_complete(struct v4l2_async_notifier *notifier)
 	}
 
 	/* Create all media device links between VINs and CSI-2's. */
+	mutex_lock(&vin->group->lock);
 	for (route = vin->info->routes; route->mask; route++) {
 		struct media_pad *source_pad, *sink_pad;
 		struct media_entity *source, *sink;
@@ -699,6 +704,7 @@ static int rvin_group_notify_complete(struct v4l2_async_notifier *notifier)
 			break;
 		}
 	}
+	mutex_unlock(&vin->group->lock);
 
 	return ret;
 }
@@ -714,6 +720,8 @@ static void rvin_group_notify_unbind(struct v4l2_async_notifier *notifier,
 		if (vin->group->vin[i])
 			rvin_v4l2_unregister(vin->group->vin[i]);
 
+	mutex_lock(&vin->group->lock);
+
 	for (i = 0; i < RVIN_CSI_MAX; i++) {
 		if (vin->group->csi[i].fwnode != asd->match.fwnode)
 			continue;
@@ -721,6 +729,8 @@ static void rvin_group_notify_unbind(struct v4l2_async_notifier *notifier,
 		vin_dbg(vin, "Unbind CSI-2 %s from slot %u\n", subdev->name, i);
 		break;
 	}
+
+	mutex_unlock(&vin->group->lock);
 }
 
 static int rvin_group_notify_bound(struct v4l2_async_notifier *notifier,
@@ -730,6 +740,8 @@ static int rvin_group_notify_bound(struct v4l2_async_notifier *notifier,
 	struct rvin_dev *vin = v4l2_dev_to_vin(notifier->v4l2_dev);
 	unsigned int i;
 
+	mutex_lock(&vin->group->lock);
+
 	for (i = 0; i < RVIN_CSI_MAX; i++) {
 		if (vin->group->csi[i].fwnode != asd->match.fwnode)
 			continue;
@@ -738,6 +750,8 @@ static int rvin_group_notify_bound(struct v4l2_async_notifier *notifier,
 		break;
 	}
 
+	mutex_unlock(&vin->group->lock);
+
 	return 0;
 }
 
@@ -752,6 +766,7 @@ static int rvin_mc_parse_of_endpoint(struct device *dev,
 				     struct v4l2_async_subdev *asd)
 {
 	struct rvin_dev *vin = dev_get_drvdata(dev);
+	int ret = 0;
 
 	if (vep->base.port != 1 || vep->base.id >= RVIN_CSI_MAX)
 		return -EINVAL;
@@ -762,38 +777,48 @@ static int rvin_mc_parse_of_endpoint(struct device *dev,
 		return -ENOTCONN;
 	}
 
+	mutex_lock(&vin->group->lock);
+
 	if (vin->group->csi[vep->base.id].fwnode) {
 		vin_dbg(vin, "OF device %pOF already handled\n",
 			to_of_node(asd->match.fwnode));
-		return -ENOTCONN;
+		ret = -ENOTCONN;
+		goto out;
 	}
 
 	vin->group->csi[vep->base.id].fwnode = asd->match.fwnode;
 
 	vin_dbg(vin, "Add group OF device %pOF to slot %u\n",
 		to_of_node(asd->match.fwnode), vep->base.id);
+out:
+	mutex_unlock(&vin->group->lock);
 
-	return 0;
+	return ret;
 }
 
 static int rvin_mc_parse_of_graph(struct rvin_dev *vin)
 {
-	unsigned int count = 0;
+	unsigned int count = 0, vin_mask = 0;
 	unsigned int i;
 	int ret;
 
 	mutex_lock(&vin->group->lock);
 
 	/* If not all VIN's are registered don't register the notifier. */
-	for (i = 0; i < RCAR_VIN_NUM; i++)
-		if (vin->group->vin[i])
+	for (i = 0; i < RCAR_VIN_NUM; i++) {
+		if (vin->group->vin[i]) {
 			count++;
+			vin_mask |= BIT(i);
+		}
+	}
 
 	if (vin->group->count != count) {
 		mutex_unlock(&vin->group->lock);
 		return 0;
 	}
 
+	mutex_unlock(&vin->group->lock);
+
 	v4l2_async_notifier_init(&vin->group->notifier);
 
 	/*
@@ -802,21 +827,17 @@ static int rvin_mc_parse_of_graph(struct rvin_dev *vin)
 	 * will only be registered once with the group notifier.
 	 */
 	for (i = 0; i < RCAR_VIN_NUM; i++) {
-		if (!vin->group->vin[i])
+		if (!(vin_mask & BIT(i)))
 			continue;
 
 		ret = v4l2_async_notifier_parse_fwnode_endpoints_by_port(
 				vin->group->vin[i]->dev, &vin->group->notifier,
 				sizeof(struct v4l2_async_subdev), 1,
 				rvin_mc_parse_of_endpoint);
-		if (ret) {
-			mutex_unlock(&vin->group->lock);
+		if (ret)
 			return ret;
-		}
 	}
 
-	mutex_unlock(&vin->group->lock);
-
 	if (list_empty(&vin->group->notifier.asd_list))
 		return 0;
 
@@ -1136,6 +1157,10 @@ static const struct rvin_info rcar_info_r8a77995 = {
 
 static const struct of_device_id rvin_of_id_table[] = {
 	{
+		.compatible = "renesas,vin-r8a774a1",
+		.data = &rcar_info_r8a7796,
+	},
+	{
 		.compatible = "renesas,vin-r8a774c0",
 		.data = &rcar_info_r8a77990,
 	},
diff --git a/drivers/media/platform/rcar-vin/rcar-csi2.c b/drivers/media/platform/rcar-vin/rcar-csi2.c
index f64528d2be3c..799e526fd3df 100644
--- a/drivers/media/platform/rcar-vin/rcar-csi2.c
+++ b/drivers/media/platform/rcar-vin/rcar-csi2.c
@@ -14,6 +14,7 @@
 #include <linux/of_graph.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
+#include <linux/reset.h>
 #include <linux/sys_soc.h>
 
 #include <media/v4l2-ctrls.h>
@@ -350,6 +351,7 @@ struct rcar_csi2 {
 	struct device *dev;
 	void __iomem *base;
 	const struct rcar_csi2_info *info;
+	struct reset_control *rstc;
 
 	struct v4l2_subdev subdev;
 	struct media_pad pads[NR_OF_RCAR_CSI2_PAD];
@@ -387,11 +389,19 @@ static void rcsi2_write(struct rcar_csi2 *priv, unsigned int reg, u32 data)
 	iowrite32(data, priv->base + reg);
 }
 
-static void rcsi2_reset(struct rcar_csi2 *priv)
+static void rcsi2_enter_standby(struct rcar_csi2 *priv)
 {
-	rcsi2_write(priv, SRST_REG, SRST_SRST);
+	rcsi2_write(priv, PHYCNT_REG, 0);
+	rcsi2_write(priv, PHTC_REG, PHTC_TESTCLR);
+	reset_control_assert(priv->rstc);
 	usleep_range(100, 150);
-	rcsi2_write(priv, SRST_REG, 0);
+	pm_runtime_put(priv->dev);
+}
+
+static void rcsi2_exit_standby(struct rcar_csi2 *priv)
+{
+	pm_runtime_get_sync(priv->dev);
+	reset_control_deassert(priv->rstc);
 }
 
 static int rcsi2_wait_phy_start(struct rcar_csi2 *priv)
@@ -462,7 +472,7 @@ static int rcsi2_calc_mbps(struct rcar_csi2 *priv, unsigned int bpp)
 	return mbps;
 }
 
-static int rcsi2_start(struct rcar_csi2 *priv)
+static int rcsi2_start_receiver(struct rcar_csi2 *priv)
 {
 	const struct rcar_csi2_format *format;
 	u32 phycnt, vcdt = 0, vcdt2 = 0;
@@ -506,12 +516,9 @@ static int rcsi2_start(struct rcar_csi2 *priv)
 
 	/* Init */
 	rcsi2_write(priv, TREF_REG, TREF_TREF);
-	rcsi2_reset(priv);
 	rcsi2_write(priv, PHTC_REG, 0);
 
 	/* Configure */
-	rcsi2_write(priv, FLD_REG, FLD_FLD_NUM(2) | FLD_FLD_EN4 |
-		    FLD_FLD_EN3 | FLD_FLD_EN2 | FLD_FLD_EN);
 	rcsi2_write(priv, VCDT_REG, vcdt);
 	if (vcdt2)
 		rcsi2_write(priv, VCDT2_REG, vcdt2);
@@ -542,6 +549,8 @@ static int rcsi2_start(struct rcar_csi2 *priv)
 	rcsi2_write(priv, PHYCNT_REG, phycnt);
 	rcsi2_write(priv, LINKCNT_REG, LINKCNT_MONITOR_EN |
 		    LINKCNT_REG_MONI_PACT_EN | LINKCNT_ICLK_NONSTOP);
+	rcsi2_write(priv, FLD_REG, FLD_FLD_NUM(2) | FLD_FLD_EN4 |
+		    FLD_FLD_EN3 | FLD_FLD_EN2 | FLD_FLD_EN);
 	rcsi2_write(priv, PHYCNT_REG, phycnt | PHYCNT_SHUTDOWNZ);
 	rcsi2_write(priv, PHYCNT_REG, phycnt | PHYCNT_SHUTDOWNZ | PHYCNT_RSTZ);
 
@@ -564,19 +573,36 @@ static int rcsi2_start(struct rcar_csi2 *priv)
 	return 0;
 }
 
-static void rcsi2_stop(struct rcar_csi2 *priv)
+static int rcsi2_start(struct rcar_csi2 *priv)
 {
-	rcsi2_write(priv, PHYCNT_REG, 0);
+	int ret;
 
-	rcsi2_reset(priv);
+	rcsi2_exit_standby(priv);
 
-	rcsi2_write(priv, PHTC_REG, PHTC_TESTCLR);
+	ret = rcsi2_start_receiver(priv);
+	if (ret) {
+		rcsi2_enter_standby(priv);
+		return ret;
+	}
+
+	ret = v4l2_subdev_call(priv->remote, video, s_stream, 1);
+	if (ret) {
+		rcsi2_enter_standby(priv);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void rcsi2_stop(struct rcar_csi2 *priv)
+{
+	rcsi2_enter_standby(priv);
+	v4l2_subdev_call(priv->remote, video, s_stream, 0);
 }
 
 static int rcsi2_s_stream(struct v4l2_subdev *sd, int enable)
 {
 	struct rcar_csi2 *priv = sd_to_csi2(sd);
-	struct v4l2_subdev *nextsd;
 	int ret = 0;
 
 	mutex_lock(&priv->lock);
@@ -586,27 +612,12 @@ static int rcsi2_s_stream(struct v4l2_subdev *sd, int enable)
 		goto out;
 	}
 
-	nextsd = priv->remote;
-
 	if (enable && priv->stream_count == 0) {
-		pm_runtime_get_sync(priv->dev);
-
 		ret = rcsi2_start(priv);
-		if (ret) {
-			pm_runtime_put(priv->dev);
-			goto out;
-		}
-
-		ret = v4l2_subdev_call(nextsd, video, s_stream, 1);
-		if (ret) {
-			rcsi2_stop(priv);
-			pm_runtime_put(priv->dev);
+		if (ret)
 			goto out;
-		}
 	} else if (!enable && priv->stream_count == 1) {
 		rcsi2_stop(priv);
-		v4l2_subdev_call(nextsd, video, s_stream, 0);
-		pm_runtime_put(priv->dev);
 	}
 
 	priv->stream_count += enable ? 1 : -1;
@@ -854,7 +865,8 @@ static int rcsi2_phtw_write_mbps(struct rcar_csi2 *priv, unsigned int mbps,
 	return rcsi2_phtw_write(priv, value->reg, code);
 }
 
-static int rcsi2_init_phtw_h3_v3h_m3n(struct rcar_csi2 *priv, unsigned int mbps)
+static int __rcsi2_init_phtw_h3_v3h_m3n(struct rcar_csi2 *priv,
+					unsigned int mbps)
 {
 	static const struct phtw_value step1[] = {
 		{ .data = 0xcc, .code = 0xe2 },
@@ -880,7 +892,7 @@ static int rcsi2_init_phtw_h3_v3h_m3n(struct rcar_csi2 *priv, unsigned int mbps)
 	if (ret)
 		return ret;
 
-	if (mbps <= 250) {
+	if (mbps != 0 && mbps <= 250) {
 		ret = rcsi2_phtw_write(priv, 0x39, 0x05);
 		if (ret)
 			return ret;
@@ -894,6 +906,16 @@ static int rcsi2_init_phtw_h3_v3h_m3n(struct rcar_csi2 *priv, unsigned int mbps)
 	return rcsi2_phtw_write_array(priv, step2);
 }
 
+static int rcsi2_init_phtw_h3_v3h_m3n(struct rcar_csi2 *priv, unsigned int mbps)
+{
+	return __rcsi2_init_phtw_h3_v3h_m3n(priv, mbps);
+}
+
+static int rcsi2_init_phtw_h3es2(struct rcar_csi2 *priv, unsigned int mbps)
+{
+	return __rcsi2_init_phtw_h3_v3h_m3n(priv, 0);
+}
+
 static int rcsi2_init_phtw_v3m_e3(struct rcar_csi2 *priv, unsigned int mbps)
 {
 	return rcsi2_phtw_write_mbps(priv, mbps, phtw_mbps_v3m_e3, 0x44);
@@ -902,11 +924,11 @@ static int rcsi2_init_phtw_v3m_e3(struct rcar_csi2 *priv, unsigned int mbps)
 static int rcsi2_confirm_start_v3m_e3(struct rcar_csi2 *priv)
 {
 	static const struct phtw_value step1[] = {
-		{ .data = 0xed, .code = 0x34 },
-		{ .data = 0xed, .code = 0x44 },
-		{ .data = 0xed, .code = 0x54 },
-		{ .data = 0xed, .code = 0x84 },
-		{ .data = 0xed, .code = 0x94 },
+		{ .data = 0xee, .code = 0x34 },
+		{ .data = 0xee, .code = 0x44 },
+		{ .data = 0xee, .code = 0x54 },
+		{ .data = 0xee, .code = 0x84 },
+		{ .data = 0xee, .code = 0x94 },
 		{ /* sentinel */ },
 	};
 
@@ -936,6 +958,10 @@ static int rcsi2_probe_resources(struct rcar_csi2 *priv,
 	if (irq < 0)
 		return irq;
 
+	priv->rstc = devm_reset_control_get(&pdev->dev, NULL);
+	if (IS_ERR(priv->rstc))
+		return PTR_ERR(priv->rstc);
+
 	return 0;
 }
 
@@ -952,6 +978,14 @@ static const struct rcar_csi2_info rcar_csi2_info_r8a7795es1 = {
 	.num_channels = 4,
 };
 
+static const struct rcar_csi2_info rcar_csi2_info_r8a7795es2 = {
+	.init_phtw = rcsi2_init_phtw_h3es2,
+	.hsfreqrange = hsfreqrange_h3_v3h_m3n,
+	.csi0clkfreqrange = 0x20,
+	.num_channels = 4,
+	.clear_ulps = true,
+};
+
 static const struct rcar_csi2_info rcar_csi2_info_r8a7796 = {
 	.hsfreqrange = hsfreqrange_m3w_h3es1,
 	.num_channels = 4,
@@ -986,6 +1020,10 @@ static const struct rcar_csi2_info rcar_csi2_info_r8a77990 = {
 
 static const struct of_device_id rcar_csi2_of_table[] = {
 	{
+		.compatible = "renesas,r8a774a1-csi2",
+		.data = &rcar_csi2_info_r8a7796,
+	},
+	{
 		.compatible = "renesas,r8a774c0-csi2",
 		.data = &rcar_csi2_info_r8a77990,
 	},
@@ -1017,11 +1055,15 @@ static const struct of_device_id rcar_csi2_of_table[] = {
 };
 MODULE_DEVICE_TABLE(of, rcar_csi2_of_table);
 
-static const struct soc_device_attribute r8a7795es1[] = {
+static const struct soc_device_attribute r8a7795[] = {
 	{
 		.soc_id = "r8a7795", .revision = "ES1.*",
 		.data = &rcar_csi2_info_r8a7795es1,
 	},
+	{
+		.soc_id = "r8a7795", .revision = "ES2.*",
+		.data = &rcar_csi2_info_r8a7795es2,
+	},
 	{ /* sentinel */ },
 };
 
@@ -1039,10 +1081,10 @@ static int rcsi2_probe(struct platform_device *pdev)
 	priv->info = of_device_get_match_data(&pdev->dev);
 
 	/*
-	 * r8a7795 ES1.x behaves differently than the ES2.0+ but doesn't
-	 * have it's own compatible string.
+	 * The different ES versions of r8a7795 (H3) behave differently but
+	 * share the same compatible string.
 	 */
-	attr = soc_device_match(r8a7795es1);
+	attr = soc_device_match(r8a7795);
 	if (attr)
 		priv->info = attr->data;
 
diff --git a/drivers/media/platform/rcar-vin/rcar-dma.c b/drivers/media/platform/rcar-vin/rcar-dma.c
index 2207a31d355e..91ab064404a1 100644
--- a/drivers/media/platform/rcar-vin/rcar-dma.c
+++ b/drivers/media/platform/rcar-vin/rcar-dma.c
@@ -486,7 +486,7 @@ static void rvin_set_coeff(struct rvin_dev *vin, unsigned short xs)
 	}
 
 	/* Use previous value if its XS value is closer */
-	if (p_prev_set && p_set &&
+	if (p_prev_set &&
 	    xs - p_prev_set->xs_value < p_set->xs_value - xs)
 		p_set = p_prev_set;
 
diff --git a/drivers/media/platform/rcar_drif.c b/drivers/media/platform/rcar_drif.c
index c417ff8f6fe5..608e5217ccd5 100644
--- a/drivers/media/platform/rcar_drif.c
+++ b/drivers/media/platform/rcar_drif.c
@@ -1405,11 +1405,9 @@ static int rcar_drif_probe(struct platform_device *pdev)
 	/* Register map */
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	ch->base = devm_ioremap_resource(&pdev->dev, res);
-	if (IS_ERR(ch->base)) {
-		ret = PTR_ERR(ch->base);
-		dev_err(&pdev->dev, "ioremap failed (%d)\n", ret);
-		return ret;
-	}
+	if (IS_ERR(ch->base))
+		return PTR_ERR(ch->base);
+
 	ch->start = res->start;
 	platform_set_drvdata(pdev, ch);
 
diff --git a/drivers/media/platform/rcar_fdp1.c b/drivers/media/platform/rcar_fdp1.c
index 6bda1eee9170..6a90bc4c476e 100644
--- a/drivers/media/platform/rcar_fdp1.c
+++ b/drivers/media/platform/rcar_fdp1.c
@@ -945,7 +945,7 @@ static void fdp1_configure_wpf(struct fdp1_ctx *ctx,
 	u32 rndctl;
 
 	pstride = q_data->format.plane_fmt[0].bytesperline
-			<< FD1_WPF_PSTRIDE_Y_SHIFT;
+		<< FD1_WPF_PSTRIDE_Y_SHIFT;
 
 	if (q_data->format.num_planes > 1)
 		pstride |= q_data->format.plane_fmt[1].bytesperline
@@ -1139,8 +1139,8 @@ static int fdp1_m2m_job_ready(void *priv)
 	int dstbufs = 1;
 
 	dprintk(ctx->fdp1, "+ Src: %d : Dst: %d\n",
-			v4l2_m2m_num_src_bufs_ready(ctx->fh.m2m_ctx),
-			v4l2_m2m_num_dst_bufs_ready(ctx->fh.m2m_ctx));
+		v4l2_m2m_num_src_bufs_ready(ctx->fh.m2m_ctx),
+		v4l2_m2m_num_dst_bufs_ready(ctx->fh.m2m_ctx));
 
 	/* One output buffer is required for each field */
 	if (V4L2_FIELD_HAS_BOTH(src_q_data->format.field))
@@ -1278,7 +1278,7 @@ static void fdp1_m2m_device_run(void *priv)
 
 		fdp1_queue_field(ctx, fbuf);
 		dprintk(fdp1, "Queued Buffer [%d] last_field:%d\n",
-				i, fbuf->last_field);
+			i, fbuf->last_field);
 	}
 
 	/* Queue as many jobs as our data provides for */
@@ -1337,7 +1337,7 @@ static void device_frame_end(struct fdp1_dev *fdp1,
 	fdp1_job_free(fdp1, job);
 
 	dprintk(fdp1, "curr_ctx->num_processed %d curr_ctx->translen %d\n",
-			ctx->num_processed, ctx->translen);
+		ctx->num_processed, ctx->translen);
 
 	if (ctx->num_processed == ctx->translen ||
 			ctx->aborting) {
@@ -1362,7 +1362,7 @@ static int fdp1_vidioc_querycap(struct file *file, void *priv,
 	strscpy(cap->driver, DRIVER_NAME, sizeof(cap->driver));
 	strscpy(cap->card, DRIVER_NAME, sizeof(cap->card));
 	snprintf(cap->bus_info, sizeof(cap->bus_info),
-			"platform:%s", DRIVER_NAME);
+		 "platform:%s", DRIVER_NAME);
 	return 0;
 }
 
@@ -1993,13 +1993,13 @@ static void fdp1_stop_streaming(struct vb2_queue *q)
 		/* Free smsk_data */
 		if (ctx->smsk_cpu) {
 			dma_free_coherent(ctx->fdp1->dev, ctx->smsk_size,
-					ctx->smsk_cpu, ctx->smsk_addr[0]);
+					  ctx->smsk_cpu, ctx->smsk_addr[0]);
 			ctx->smsk_addr[0] = ctx->smsk_addr[1] = 0;
 			ctx->smsk_cpu = NULL;
 		}
 
 		WARN(!list_empty(&ctx->fields_queue),
-				"Buffer queue not empty");
+		     "Buffer queue not empty");
 	} else {
 		/* Empty Capture queues (Jobs) */
 		struct fdp1_job *job;
@@ -2021,10 +2021,10 @@ static void fdp1_stop_streaming(struct vb2_queue *q)
 		fdp1_field_complete(ctx, ctx->previous);
 
 		WARN(!list_empty(&ctx->fdp1->queued_job_list),
-				"Queued Job List not empty");
+		     "Queued Job List not empty");
 
 		WARN(!list_empty(&ctx->fdp1->hw_job_list),
-				"HW Job list not empty");
+		     "HW Job list not empty");
 	}
 }
 
@@ -2110,7 +2110,7 @@ static int fdp1_open(struct file *file)
 				     fdp1_ctrl_deint_menu);
 
 	ctrl = v4l2_ctrl_new_std(&ctx->hdl, &fdp1_ctrl_ops,
-			V4L2_CID_MIN_BUFFERS_FOR_CAPTURE, 1, 2, 1, 1);
+				 V4L2_CID_MIN_BUFFERS_FOR_CAPTURE, 1, 2, 1, 1);
 	if (ctrl)
 		ctrl->flags |= V4L2_CTRL_FLAG_VOLATILE;
 
@@ -2347,8 +2347,8 @@ static int fdp1_probe(struct platform_device *pdev)
 		goto release_m2m;
 	}
 
-	v4l2_info(&fdp1->v4l2_dev,
-			"Device registered as /dev/video%d\n", vfd->num);
+	v4l2_info(&fdp1->v4l2_dev, "Device registered as /dev/video%d\n",
+		  vfd->num);
 
 	/* Power up the cells to read HW */
 	pm_runtime_enable(&pdev->dev);
@@ -2367,7 +2367,7 @@ static int fdp1_probe(struct platform_device *pdev)
 		break;
 	default:
 		dev_err(fdp1->dev, "FDP1 Unidentifiable (0x%08x)\n",
-				hw_version);
+			hw_version);
 	}
 
 	/* Allow the hw to sleep until an open call puts it to use */
diff --git a/drivers/media/platform/s5p-cec/s5p_cec.c b/drivers/media/platform/s5p-cec/s5p_cec.c
index 8837e2678bde..7f62f5ef0086 100644
--- a/drivers/media/platform/s5p-cec/s5p_cec.c
+++ b/drivers/media/platform/s5p-cec/s5p_cec.c
@@ -178,22 +178,16 @@ static const struct cec_adap_ops s5p_cec_adap_ops = {
 static int s5p_cec_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
-	struct device_node *np;
-	struct platform_device *hdmi_dev;
+	struct device *hdmi_dev;
 	struct resource *res;
 	struct s5p_cec_dev *cec;
 	bool needs_hpd = of_property_read_bool(pdev->dev.of_node, "needs-hpd");
 	int ret;
 
-	np = of_parse_phandle(pdev->dev.of_node, "hdmi-phandle", 0);
+	hdmi_dev = cec_notifier_parse_hdmi_phandle(dev);
 
-	if (!np) {
-		dev_err(&pdev->dev, "Failed to find hdmi node in device tree\n");
-		return -ENODEV;
-	}
-	hdmi_dev = of_find_device_by_node(np);
-	if (hdmi_dev == NULL)
-		return -EPROBE_DEFER;
+	if (IS_ERR(hdmi_dev))
+		return PTR_ERR(hdmi_dev);
 
 	cec = devm_kzalloc(&pdev->dev, sizeof(*cec), GFP_KERNEL);
 	if (!cec)
@@ -224,7 +218,7 @@ static int s5p_cec_probe(struct platform_device *pdev)
 	if (IS_ERR(cec->reg))
 		return PTR_ERR(cec->reg);
 
-	cec->notifier = cec_notifier_get(&hdmi_dev->dev);
+	cec->notifier = cec_notifier_get(hdmi_dev);
 	if (cec->notifier == NULL)
 		return -ENOMEM;
 
diff --git a/drivers/media/platform/s5p-g2d/g2d.c b/drivers/media/platform/s5p-g2d/g2d.c
index 971c47165010..c8f394e1aff4 100644
--- a/drivers/media/platform/s5p-g2d/g2d.c
+++ b/drivers/media/platform/s5p-g2d/g2d.c
@@ -297,8 +297,8 @@ static int g2d_release(struct file *file)
 static int vidioc_querycap(struct file *file, void *priv,
 				struct v4l2_capability *cap)
 {
-	strncpy(cap->driver, G2D_NAME, sizeof(cap->driver) - 1);
-	strncpy(cap->card, G2D_NAME, sizeof(cap->card) - 1);
+	strscpy(cap->driver, G2D_NAME, sizeof(cap->driver));
+	strscpy(cap->card, G2D_NAME, sizeof(cap->card));
 	cap->bus_info[0] = 0;
 	cap->device_caps = V4L2_CAP_VIDEO_M2M | V4L2_CAP_STREAMING;
 	cap->capabilities = cap->device_caps | V4L2_CAP_DEVICE_CAPS;
@@ -312,7 +312,7 @@ static int vidioc_enum_fmt(struct file *file, void *prv, struct v4l2_fmtdesc *f)
 		return -EINVAL;
 	fmt = &formats[f->index];
 	f->pixelformat = fmt->fourcc;
-	strncpy(f->description, fmt->name, sizeof(f->description) - 1);
+	strscpy(f->description, fmt->name, sizeof(f->description));
 	return 0;
 }
 
diff --git a/drivers/media/platform/seco-cec/seco-cec.c b/drivers/media/platform/seco-cec/seco-cec.c
index a425a10540c1..e5080d6f5b2d 100644
--- a/drivers/media/platform/seco-cec/seco-cec.c
+++ b/drivers/media/platform/seco-cec/seco-cec.c
@@ -536,6 +536,7 @@ static int secocec_cec_get_notifier(struct cec_notifier **notify)
 				return -EPROBE_DEFER;
 
 			*notify = cec_notifier_get_conn(d, m->conn);
+			put_device(d);
 
 			return 0;
 		}
diff --git a/drivers/media/platform/sh_veu.c b/drivers/media/platform/sh_veu.c
index d277cc674349..5a9ba05c996e 100644
--- a/drivers/media/platform/sh_veu.c
+++ b/drivers/media/platform/sh_veu.c
@@ -493,9 +493,6 @@ static int sh_veu_try_fmt_vid_cap(struct file *file, void *priv,
 	const struct sh_veu_format *fmt;
 
 	fmt = sh_veu_find_fmt(f);
-	if (!fmt)
-		/* wrong buffer type */
-		return -EINVAL;
 
 	return sh_veu_try_fmt(f, fmt);
 }
@@ -506,9 +503,6 @@ static int sh_veu_try_fmt_vid_out(struct file *file, void *priv,
 	const struct sh_veu_format *fmt;
 
 	fmt = sh_veu_find_fmt(f);
-	if (!fmt)
-		/* wrong buffer type */
-		return -EINVAL;
 
 	return sh_veu_try_fmt(f, fmt);
 }
diff --git a/drivers/media/platform/sti/c8sectpfe/Kconfig b/drivers/media/platform/sti/c8sectpfe/Kconfig
index 7420a50572d3..93eaabfd5437 100644
--- a/drivers/media/platform/sti/c8sectpfe/Kconfig
+++ b/drivers/media/platform/sti/c8sectpfe/Kconfig
@@ -12,7 +12,7 @@ config DVB_C8SECTPFE
 	select DVB_STV0367 if MEDIA_SUBDRV_AUTOSELECT
 	select MEDIA_TUNER_TDA18212 if MEDIA_SUBDRV_AUTOSELECT
 
-	---help---
+	help
 	  This adds support for DVB front-end cards connected
 	  to TS inputs of STiH407/410 SoC.
 
diff --git a/drivers/media/platform/sti/cec/stih-cec.c b/drivers/media/platform/sti/cec/stih-cec.c
index d34099f75990..fc37efe1d554 100644
--- a/drivers/media/platform/sti/cec/stih-cec.c
+++ b/drivers/media/platform/sti/cec/stih-cec.c
@@ -301,26 +301,19 @@ static int stih_cec_probe(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	struct resource *res;
 	struct stih_cec *cec;
-	struct device_node *np;
-	struct platform_device *hdmi_dev;
+	struct device *hdmi_dev;
 	int ret;
 
+	hdmi_dev = cec_notifier_parse_hdmi_phandle(dev);
+
+	if (IS_ERR(hdmi_dev))
+		return PTR_ERR(hdmi_dev);
+
 	cec = devm_kzalloc(dev, sizeof(*cec), GFP_KERNEL);
 	if (!cec)
 		return -ENOMEM;
 
-	np = of_parse_phandle(pdev->dev.of_node, "hdmi-phandle", 0);
-
-	if (!np) {
-		dev_err(&pdev->dev, "Failed to find hdmi node in device tree\n");
-		return -ENODEV;
-	}
-
-	hdmi_dev = of_find_device_by_node(np);
-	if (!hdmi_dev)
-		return -EPROBE_DEFER;
-
-	cec->notifier = cec_notifier_get(&hdmi_dev->dev);
+	cec->notifier = cec_notifier_get(hdmi_dev);
 	if (!cec->notifier)
 		return -ENOMEM;
 
diff --git a/drivers/media/platform/sti/delta/delta-ipc.c b/drivers/media/platform/sti/delta/delta-ipc.c
index a4603d573c34..186d88f02ecd 100644
--- a/drivers/media/platform/sti/delta/delta-ipc.c
+++ b/drivers/media/platform/sti/delta/delta-ipc.c
@@ -220,10 +220,8 @@ int delta_ipc_open(struct delta_ctx *pctx, const char *name,
 
 err:
 	pctx->sys_errors++;
-	if (ctx->ipc_buf) {
-		hw_free(pctx, ctx->ipc_buf);
-		ctx->ipc_buf = NULL;
-	}
+	hw_free(pctx, ctx->ipc_buf);
+	ctx->ipc_buf = NULL;
 
 	return ret;
 };
diff --git a/drivers/media/platform/stm32/stm32-cec.c b/drivers/media/platform/stm32/stm32-cec.c
index 7c496bc1cf38..8a86b2cc22fa 100644
--- a/drivers/media/platform/stm32/stm32-cec.c
+++ b/drivers/media/platform/stm32/stm32-cec.c
@@ -56,6 +56,13 @@
 #define ALL_TX_IT	(TXEND | TXBR | TXACKE | TXERR | TXUDR | ARBLST)
 #define ALL_RX_IT	(RXEND | RXBR | RXACKE | RXOVR)
 
+/*
+ * 400 ms is the time it takes for one 16 byte message to be
+ * transferred and 5 is the maximum number of retries. Add
+ * another 100 ms as a margin.
+ */
+#define CEC_XFER_TIMEOUT_MS (5 * 400 + 100)
+
 struct stm32_cec {
 	struct cec_adapter	*adap;
 	struct device		*dev;
@@ -188,7 +195,11 @@ static int stm32_cec_adap_log_addr(struct cec_adapter *adap, u8 logical_addr)
 {
 	struct stm32_cec *cec = adap->priv;
 	u32 oar = (1 << logical_addr) << 16;
+	u32 val;
 
+	/* Poll every 100µs the register CEC_CR to wait end of transmission */
+	regmap_read_poll_timeout(cec->regmap, CEC_CR, val, !(val & TXSOM),
+				 100, CEC_XFER_TIMEOUT_MS * 1000);
 	regmap_update_bits(cec->regmap, CEC_CR, CECEN, 0);
 
 	if (logical_addr == CEC_LOG_ADDR_INVALID)
diff --git a/drivers/media/platform/stm32/stm32-dcmi.c b/drivers/media/platform/stm32/stm32-dcmi.c
index 5fe5b38fa901..b9dad0accd1b 100644
--- a/drivers/media/platform/stm32/stm32-dcmi.c
+++ b/drivers/media/platform/stm32/stm32-dcmi.c
@@ -97,6 +97,8 @@ enum state {
 
 #define TIMEOUT_MS	1000
 
+#define OVERRUN_ERROR_THRESHOLD	3
+
 struct dcmi_graph_entity {
 	struct device_node *node;
 
@@ -164,6 +166,9 @@ struct stm32_dcmi {
 	int				errors_count;
 	int				overrun_count;
 	int				buffers_count;
+
+	/* Ensure DMA operations atomicity */
+	struct mutex			dma_lock;
 };
 
 static inline struct stm32_dcmi *notifier_to_dcmi(struct v4l2_async_notifier *n)
@@ -314,6 +319,13 @@ static int dcmi_start_dma(struct stm32_dcmi *dcmi,
 		return ret;
 	}
 
+	/*
+	 * Avoid call of dmaengine_terminate_all() between
+	 * dmaengine_prep_slave_single() and dmaengine_submit()
+	 * by locking the whole DMA submission sequence
+	 */
+	mutex_lock(&dcmi->dma_lock);
+
 	/* Prepare a DMA transaction */
 	desc = dmaengine_prep_slave_single(dcmi->dma_chan, buf->paddr,
 					   buf->size,
@@ -322,6 +334,7 @@ static int dcmi_start_dma(struct stm32_dcmi *dcmi,
 	if (!desc) {
 		dev_err(dcmi->dev, "%s: DMA dmaengine_prep_slave_single failed for buffer phy=%pad size=%zu\n",
 			__func__, &buf->paddr, buf->size);
+		mutex_unlock(&dcmi->dma_lock);
 		return -EINVAL;
 	}
 
@@ -333,9 +346,12 @@ static int dcmi_start_dma(struct stm32_dcmi *dcmi,
 	dcmi->dma_cookie = dmaengine_submit(desc);
 	if (dma_submit_error(dcmi->dma_cookie)) {
 		dev_err(dcmi->dev, "%s: DMA submission failed\n", __func__);
+		mutex_unlock(&dcmi->dma_lock);
 		return -ENXIO;
 	}
 
+	mutex_unlock(&dcmi->dma_lock);
+
 	dma_async_issue_pending(dcmi->dma_chan);
 
 	return 0;
@@ -432,11 +448,13 @@ static irqreturn_t dcmi_irq_thread(int irq, void *arg)
 
 	spin_lock_irq(&dcmi->irqlock);
 
-	if ((dcmi->misr & IT_OVR) || (dcmi->misr & IT_ERR)) {
-		dcmi->errors_count++;
-		if (dcmi->misr & IT_OVR)
-			dcmi->overrun_count++;
+	if (dcmi->misr & IT_OVR) {
+		dcmi->overrun_count++;
+		if (dcmi->overrun_count > OVERRUN_ERROR_THRESHOLD)
+			dcmi->errors_count++;
 	}
+	if (dcmi->misr & IT_ERR)
+		dcmi->errors_count++;
 
 	if (dcmi->sd_format->fourcc == V4L2_PIX_FMT_JPEG &&
 	    dcmi->misr & IT_FRAME) {
@@ -570,9 +588,9 @@ static int dcmi_start_streaming(struct vb2_queue *vq, unsigned int count)
 	int ret;
 
 	ret = pm_runtime_get_sync(dcmi->dev);
-	if (ret) {
-		dev_err(dcmi->dev, "%s: Failed to start streaming, cannot get sync\n",
-			__func__);
+	if (ret < 0) {
+		dev_err(dcmi->dev, "%s: Failed to start streaming, cannot get sync (%d)\n",
+			__func__, ret);
 		goto err_release_buffers;
 	}
 
@@ -720,7 +738,9 @@ static void dcmi_stop_streaming(struct vb2_queue *vq)
 	spin_unlock_irq(&dcmi->irqlock);
 
 	/* Stop all pending DMA operations */
+	mutex_lock(&dcmi->dma_lock);
 	dmaengine_terminate_all(dcmi->dma_chan);
+	mutex_unlock(&dcmi->dma_lock);
 
 	pm_runtime_put(dcmi->dev);
 
@@ -811,6 +831,9 @@ static int dcmi_try_fmt(struct stm32_dcmi *dcmi, struct v4l2_format *f,
 
 	sd_fmt = find_format_by_fourcc(dcmi, pix->pixelformat);
 	if (!sd_fmt) {
+		if (!dcmi->num_of_sd_formats)
+			return -ENODATA;
+
 		sd_fmt = dcmi->sd_formats[dcmi->num_of_sd_formats - 1];
 		pix->pixelformat = sd_fmt->fourcc;
 	}
@@ -989,6 +1012,9 @@ static int dcmi_set_sensor_format(struct stm32_dcmi *dcmi,
 
 	sd_fmt = find_format_by_fourcc(dcmi, pix->pixelformat);
 	if (!sd_fmt) {
+		if (!dcmi->num_of_sd_formats)
+			return -ENODATA;
+
 		sd_fmt = dcmi->sd_formats[dcmi->num_of_sd_formats - 1];
 		pix->pixelformat = sd_fmt->fourcc;
 	}
@@ -1595,7 +1621,7 @@ static int dcmi_graph_init(struct stm32_dcmi *dcmi)
 	/* Parse the graph to extract a list of subdevice DT nodes. */
 	ret = dcmi_graph_parse(dcmi, dcmi->dev->of_node);
 	if (ret < 0) {
-		dev_err(dcmi->dev, "Graph parsing failed\n");
+		dev_err(dcmi->dev, "Failed to parse graph\n");
 		return ret;
 	}
 
@@ -1604,6 +1630,7 @@ static int dcmi_graph_init(struct stm32_dcmi *dcmi)
 	ret = v4l2_async_notifier_add_subdev(&dcmi->notifier,
 					     &dcmi->entity.asd);
 	if (ret) {
+		dev_err(dcmi->dev, "Failed to add subdev notifier\n");
 		of_node_put(dcmi->entity.node);
 		return ret;
 	}
@@ -1612,7 +1639,7 @@ static int dcmi_graph_init(struct stm32_dcmi *dcmi)
 
 	ret = v4l2_async_notifier_register(&dcmi->v4l2_dev, &dcmi->notifier);
 	if (ret < 0) {
-		dev_err(dcmi->dev, "Notifier registration failed\n");
+		dev_err(dcmi->dev, "Failed to register notifier\n");
 		v4l2_async_notifier_cleanup(&dcmi->notifier);
 		return ret;
 	}
@@ -1645,7 +1672,7 @@ static int dcmi_probe(struct platform_device *pdev)
 	dcmi->rstc = devm_reset_control_get_exclusive(&pdev->dev, NULL);
 	if (IS_ERR(dcmi->rstc)) {
 		dev_err(&pdev->dev, "Could not get reset control\n");
-		return -ENODEV;
+		return PTR_ERR(dcmi->rstc);
 	}
 
 	/* Get bus characteristics from devicetree */
@@ -1660,7 +1687,7 @@ static int dcmi_probe(struct platform_device *pdev)
 	of_node_put(np);
 	if (ret) {
 		dev_err(&pdev->dev, "Could not parse the endpoint\n");
-		return -ENODEV;
+		return ret;
 	}
 
 	if (ep.bus_type == V4L2_MBUS_CSI2_DPHY) {
@@ -1673,8 +1700,9 @@ static int dcmi_probe(struct platform_device *pdev)
 
 	irq = platform_get_irq(pdev, 0);
 	if (irq <= 0) {
-		dev_err(&pdev->dev, "Could not get irq\n");
-		return -ENODEV;
+		if (irq != -EPROBE_DEFER)
+			dev_err(&pdev->dev, "Could not get irq\n");
+		return irq;
 	}
 
 	dcmi->res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
@@ -1694,12 +1722,13 @@ static int dcmi_probe(struct platform_device *pdev)
 					dev_name(&pdev->dev), dcmi);
 	if (ret) {
 		dev_err(&pdev->dev, "Unable to request irq %d\n", irq);
-		return -ENODEV;
+		return ret;
 	}
 
 	mclk = devm_clk_get(&pdev->dev, "mclk");
 	if (IS_ERR(mclk)) {
-		dev_err(&pdev->dev, "Unable to get mclk\n");
+		if (PTR_ERR(mclk) != -EPROBE_DEFER)
+			dev_err(&pdev->dev, "Unable to get mclk\n");
 		return PTR_ERR(mclk);
 	}
 
@@ -1711,6 +1740,7 @@ static int dcmi_probe(struct platform_device *pdev)
 
 	spin_lock_init(&dcmi->irqlock);
 	mutex_init(&dcmi->lock);
+	mutex_init(&dcmi->dma_lock);
 	init_completion(&dcmi->complete);
 	INIT_LIST_HEAD(&dcmi->buffers);
 
diff --git a/drivers/media/platform/sunxi/sun6i-csi/sun6i_video.c b/drivers/media/platform/sunxi/sun6i-csi/sun6i_video.c
index 1fd16861f111..f0dfe68486d1 100644
--- a/drivers/media/platform/sunxi/sun6i-csi/sun6i_video.c
+++ b/drivers/media/platform/sunxi/sun6i-csi/sun6i_video.c
@@ -412,7 +412,7 @@ static int vidioc_enum_input(struct file *file, void *fh,
 	if (inp->index != 0)
 		return -EINVAL;
 
-	strlcpy(inp->name, "camera", sizeof(inp->name));
+	strscpy(inp->name, "camera", sizeof(inp->name));
 	inp->type = V4L2_INPUT_TYPE_CAMERA;
 
 	return 0;
@@ -644,7 +644,7 @@ int sun6i_video_init(struct sun6i_video *video, struct sun6i_csi *csi,
 	}
 
 	/* Register video device */
-	strlcpy(vdev->name, name, sizeof(vdev->name));
+	strscpy(vdev->name, name, sizeof(vdev->name));
 	vdev->release		= video_device_release_empty;
 	vdev->fops		= &sun6i_video_fops;
 	vdev->ioctl_ops		= &sun6i_video_ioctl_ops;
diff --git a/drivers/media/platform/tegra-cec/tegra_cec.c b/drivers/media/platform/tegra-cec/tegra_cec.c
index aba488cd0e64..7fb3a4fa07c1 100644
--- a/drivers/media/platform/tegra-cec/tegra_cec.c
+++ b/drivers/media/platform/tegra-cec/tegra_cec.c
@@ -327,21 +327,15 @@ static const struct cec_adap_ops tegra_cec_ops = {
 
 static int tegra_cec_probe(struct platform_device *pdev)
 {
-	struct platform_device *hdmi_dev;
-	struct device_node *np;
+	struct device *hdmi_dev;
 	struct tegra_cec *cec;
 	struct resource *res;
 	int ret = 0;
 
-	np = of_parse_phandle(pdev->dev.of_node, "hdmi-phandle", 0);
+	hdmi_dev = cec_notifier_parse_hdmi_phandle(&pdev->dev);
 
-	if (!np) {
-		dev_err(&pdev->dev, "Failed to find hdmi node in device tree\n");
+	if (!hdmi_dev)
 		return -ENODEV;
-	}
-	hdmi_dev = of_find_device_by_node(np);
-	if (hdmi_dev == NULL)
-		return -EPROBE_DEFER;
 
 	cec = devm_kzalloc(&pdev->dev, sizeof(struct tegra_cec), GFP_KERNEL);
 
@@ -400,7 +394,7 @@ static int tegra_cec_probe(struct platform_device *pdev)
 		goto clk_error;
 	}
 
-	cec->notifier = cec_notifier_get(&hdmi_dev->dev);
+	cec->notifier = cec_notifier_get(hdmi_dev);
 	if (!cec->notifier) {
 		ret = -ENOMEM;
 		goto clk_error;
diff --git a/drivers/media/platform/ti-vpe/cal.c b/drivers/media/platform/ti-vpe/cal.c
index fc3c212b96e1..8d075683e448 100644
--- a/drivers/media/platform/ti-vpe/cal.c
+++ b/drivers/media/platform/ti-vpe/cal.c
@@ -1643,8 +1643,7 @@ of_get_next_endpoint(const struct device_node *parent,
 static int of_cal_create_instance(struct cal_ctx *ctx, int inst)
 {
 	struct platform_device *pdev = ctx->dev->pdev;
-	struct device_node *ep_node, *port, *remote_ep,
-			*sensor_node, *parent;
+	struct device_node *ep_node, *port, *sensor_node, *parent;
 	struct v4l2_fwnode_endpoint *endpoint;
 	struct v4l2_async_subdev *asd;
 	u32 regval = 0;
@@ -1657,7 +1656,6 @@ static int of_cal_create_instance(struct cal_ctx *ctx, int inst)
 
 	ep_node = NULL;
 	port = NULL;
-	remote_ep = NULL;
 	sensor_node = NULL;
 	ret = -EINVAL;
 
@@ -1703,12 +1701,7 @@ static int of_cal_create_instance(struct cal_ctx *ctx, int inst)
 	asd->match_type = V4L2_ASYNC_MATCH_FWNODE;
 	asd->match.fwnode = of_fwnode_handle(sensor_node);
 
-	remote_ep = of_graph_get_remote_endpoint(ep_node);
-	if (!remote_ep) {
-		ctx_dbg(3, ctx, "can't get remote-endpoint\n");
-		goto cleanup_exit;
-	}
-	v4l2_fwnode_endpoint_parse(of_fwnode_handle(remote_ep), endpoint);
+	v4l2_fwnode_endpoint_parse(of_fwnode_handle(ep_node), endpoint);
 
 	if (endpoint->bus_type != V4L2_MBUS_CSI2_DPHY) {
 		ctx_err(ctx, "Port:%d sub-device %pOFn is not a CSI2 device\n",
@@ -1759,7 +1752,6 @@ static int of_cal_create_instance(struct cal_ctx *ctx, int inst)
 	sensor_node = NULL;
 
 cleanup_exit:
-	of_node_put(remote_ep);
 	of_node_put(sensor_node);
 	of_node_put(ep_node);
 	of_node_put(port);
diff --git a/drivers/media/platform/ti-vpe/vpe.c b/drivers/media/platform/ti-vpe/vpe.c
index 207e7e76c048..1e40eafec284 100644
--- a/drivers/media/platform/ti-vpe/vpe.c
+++ b/drivers/media/platform/ti-vpe/vpe.c
@@ -1491,8 +1491,8 @@ handled:
 static int vpe_querycap(struct file *file, void *priv,
 			struct v4l2_capability *cap)
 {
-	strncpy(cap->driver, VPE_MODULE_NAME, sizeof(cap->driver) - 1);
-	strncpy(cap->card, VPE_MODULE_NAME, sizeof(cap->card) - 1);
+	strscpy(cap->driver, VPE_MODULE_NAME, sizeof(cap->driver));
+	strscpy(cap->card, VPE_MODULE_NAME, sizeof(cap->card));
 	snprintf(cap->bus_info, sizeof(cap->bus_info), "platform:%s",
 		VPE_MODULE_NAME);
 	cap->device_caps  = V4L2_CAP_VIDEO_M2M_MPLANE | V4L2_CAP_STREAMING;
@@ -1519,7 +1519,7 @@ static int __enum_fmt(struct v4l2_fmtdesc *f, u32 type)
 	if (!fmt)
 		return -EINVAL;
 
-	strncpy(f->description, fmt->name, sizeof(f->description) - 1);
+	strscpy(f->description, fmt->name, sizeof(f->description));
 	f->pixelformat = fmt->fourcc;
 	return 0;
 }
diff --git a/drivers/media/platform/vicodec/codec-fwht.c b/drivers/media/platform/vicodec/codec-fwht.c
index d1d6085da9f1..31faf319e508 100644
--- a/drivers/media/platform/vicodec/codec-fwht.c
+++ b/drivers/media/platform/vicodec/codec-fwht.c
@@ -46,8 +46,12 @@ static const uint8_t zigzag[64] = {
 	63,
 };
 
-
-static int rlc(const s16 *in, __be16 *output, int blocktype)
+/*
+ * noinline_for_stack to work around
+ * https://bugs.llvm.org/show_bug.cgi?id=38809
+ */
+static int noinline_for_stack
+rlc(const s16 *in, __be16 *output, int blocktype)
 {
 	s16 block[8 * 8];
 	s16 *wp = block;
@@ -106,8 +110,8 @@ static int rlc(const s16 *in, __be16 *output, int blocktype)
  * This function will worst-case increase rlc_in by 65*2 bytes:
  * one s16 value for the header and 8 * 8 coefficients of type s16.
  */
-static u16 derlc(const __be16 **rlc_in, s16 *dwht_out,
-		 const __be16 *end_of_input)
+static noinline_for_stack u16
+derlc(const __be16 **rlc_in, s16 *dwht_out, const __be16 *end_of_input)
 {
 	/* header */
 	const __be16 *input = *rlc_in;
@@ -240,8 +244,9 @@ static void dequantize_inter(s16 *coeff)
 			*coeff <<= *quant;
 }
 
-static void fwht(const u8 *block, s16 *output_block, unsigned int stride,
-		 unsigned int input_step, bool intra)
+static void noinline_for_stack fwht(const u8 *block, s16 *output_block,
+				    unsigned int stride,
+				    unsigned int input_step, bool intra)
 {
 	/* we'll need more than 8 bits for the transformed coefficients */
 	s32 workspace1[8], workspace2[8];
@@ -373,7 +378,8 @@ static void fwht(const u8 *block, s16 *output_block, unsigned int stride,
  * Furthermore values can be negative... This is just a version that
  * works with 16 signed data
  */
-static void fwht16(const s16 *block, s16 *output_block, int stride, int intra)
+static void noinline_for_stack
+fwht16(const s16 *block, s16 *output_block, int stride, int intra)
 {
 	/* we'll need more than 8 bits for the transformed coefficients */
 	s32 workspace1[8], workspace2[8];
@@ -456,7 +462,8 @@ static void fwht16(const s16 *block, s16 *output_block, int stride, int intra)
 	}
 }
 
-static void ifwht(const s16 *block, s16 *output_block, int intra)
+static noinline_for_stack void
+ifwht(const s16 *block, s16 *output_block, int intra)
 {
 	/*
 	 * we'll need more than 8 bits for the transformed coefficients
@@ -604,9 +611,9 @@ static int var_inter(const s16 *old, const s16 *new)
 	return ret;
 }
 
-static int decide_blocktype(const u8 *cur, const u8 *reference,
-			    s16 *deltablock, unsigned int stride,
-			    unsigned int input_step)
+static noinline_for_stack int
+decide_blocktype(const u8 *cur, const u8 *reference, s16 *deltablock,
+		 unsigned int stride, unsigned int input_step)
 {
 	s16 tmp[64];
 	s16 old[64];
@@ -632,12 +639,13 @@ static int decide_blocktype(const u8 *cur, const u8 *reference,
 	return vari <= vard ? IBLOCK : PBLOCK;
 }
 
-static void fill_decoder_block(u8 *dst, const s16 *input, int stride)
+static void fill_decoder_block(u8 *dst, const s16 *input, int stride,
+			       unsigned int dst_step)
 {
 	int i, j;
 
 	for (i = 0; i < 8; i++) {
-		for (j = 0; j < 8; j++, input++, dst++) {
+		for (j = 0; j < 8; j++, input++, dst += dst_step) {
 			if (*input < 0)
 				*dst = 0;
 			else if (*input > 255)
@@ -645,17 +653,19 @@ static void fill_decoder_block(u8 *dst, const s16 *input, int stride)
 			else
 				*dst = *input;
 		}
-		dst += stride - 8;
+		dst += stride - (8 * dst_step);
 	}
 }
 
-static void add_deltas(s16 *deltas, const u8 *ref, int stride)
+static void add_deltas(s16 *deltas, const u8 *ref, int stride,
+		       unsigned int ref_step)
 {
 	int k, l;
 
 	for (k = 0; k < 8; k++) {
 		for (l = 0; l < 8; l++) {
-			*deltas += *ref++;
+			*deltas += *ref;
+			ref += ref_step;
 			/*
 			 * Due to quantizing, it might possible that the
 			 * decoded coefficients are slightly out of range
@@ -666,7 +676,7 @@ static void add_deltas(s16 *deltas, const u8 *ref, int stride)
 				*deltas = 255;
 			deltas++;
 		}
-		ref += stride - 8;
+		ref += stride - (8 * ref_step);
 	}
 }
 
@@ -711,8 +721,8 @@ static u32 encode_plane(u8 *input, u8 *refp, __be16 **rlco, __be16 *rlco_max,
 				ifwht(cf->de_coeffs, cf->de_fwht, blocktype);
 
 				if (blocktype == PBLOCK)
-					add_deltas(cf->de_fwht, refp, 8);
-				fill_decoder_block(refp, cf->de_fwht, 8);
+					add_deltas(cf->de_fwht, refp, 8, 1);
+				fill_decoder_block(refp, cf->de_fwht, 8, 1);
 			}
 
 			input += 8 * input_step;
@@ -821,23 +831,31 @@ u32 fwht_encode_frame(struct fwht_raw_frame *frm,
 	return encoding;
 }
 
-static bool decode_plane(struct fwht_cframe *cf, const __be16 **rlco, u8 *ref,
-			 u32 height, u32 width, u32 coded_width,
+static bool decode_plane(struct fwht_cframe *cf, const __be16 **rlco,
+			 u32 height, u32 width, const u8 *ref, u32 ref_stride,
+			 unsigned int ref_step, u8 *dst,
+			 unsigned int dst_stride, unsigned int dst_step,
 			 bool uncompressed, const __be16 *end_of_rlco_buf)
 {
 	unsigned int copies = 0;
 	s16 copy[8 * 8];
 	u16 stat;
 	unsigned int i, j;
+	bool is_intra = !ref;
 
 	width = round_up(width, 8);
 	height = round_up(height, 8);
 
 	if (uncompressed) {
+		int i;
+
 		if (end_of_rlco_buf + 1 < *rlco + width * height / 2)
 			return false;
-		memcpy(ref, *rlco, width * height);
-		*rlco += width * height / 2;
+		for (i = 0; i < height; i++) {
+			memcpy(dst, *rlco, width);
+			dst += dst_stride;
+			*rlco += width / 2;
+		}
 		return true;
 	}
 
@@ -849,15 +867,17 @@ static bool decode_plane(struct fwht_cframe *cf, const __be16 **rlco, u8 *ref,
 	 */
 	for (j = 0; j < height / 8; j++) {
 		for (i = 0; i < width / 8; i++) {
-			u8 *refp = ref + j * 8 * coded_width + i * 8;
+			const u8 *refp = ref + j * 8 * ref_stride +
+				i * 8 * ref_step;
+			u8 *dstp = dst + j * 8 * dst_stride + i * 8 * dst_step;
 
 			if (copies) {
 				memcpy(cf->de_fwht, copy, sizeof(copy));
-				if (stat & PFRAME_BIT)
+				if ((stat & PFRAME_BIT) && !is_intra)
 					add_deltas(cf->de_fwht, refp,
-						   coded_width);
-				fill_decoder_block(refp, cf->de_fwht,
-						   coded_width);
+						   ref_stride, ref_step);
+				fill_decoder_block(dstp, cf->de_fwht,
+						   dst_stride, dst_step);
 				copies--;
 				continue;
 			}
@@ -865,35 +885,41 @@ static bool decode_plane(struct fwht_cframe *cf, const __be16 **rlco, u8 *ref,
 			stat = derlc(rlco, cf->coeffs, end_of_rlco_buf);
 			if (stat & OVERFLOW_BIT)
 				return false;
-			if (stat & PFRAME_BIT)
+			if ((stat & PFRAME_BIT) && !is_intra)
 				dequantize_inter(cf->coeffs);
 			else
 				dequantize_intra(cf->coeffs);
 
 			ifwht(cf->coeffs, cf->de_fwht,
-			      (stat & PFRAME_BIT) ? 0 : 1);
+			      ((stat & PFRAME_BIT) && !is_intra) ? 0 : 1);
 
 			copies = (stat & DUPS_MASK) >> 1;
 			if (copies)
 				memcpy(copy, cf->de_fwht, sizeof(copy));
-			if (stat & PFRAME_BIT)
-				add_deltas(cf->de_fwht, refp, coded_width);
-			fill_decoder_block(refp, cf->de_fwht, coded_width);
+			if ((stat & PFRAME_BIT) && !is_intra)
+				add_deltas(cf->de_fwht, refp,
+					   ref_stride, ref_step);
+			fill_decoder_block(dstp, cf->de_fwht, dst_stride,
+					   dst_step);
 		}
 	}
 	return true;
 }
 
-bool fwht_decode_frame(struct fwht_cframe *cf, struct fwht_raw_frame *ref,
-		       u32 hdr_flags, unsigned int components_num,
-		       unsigned int width, unsigned int height,
-		       unsigned int coded_width)
+bool fwht_decode_frame(struct fwht_cframe *cf, u32 hdr_flags,
+		       unsigned int components_num, unsigned int width,
+		       unsigned int height, const struct fwht_raw_frame *ref,
+		       unsigned int ref_stride, unsigned int ref_chroma_stride,
+		       struct fwht_raw_frame *dst, unsigned int dst_stride,
+		       unsigned int dst_chroma_stride)
 {
 	const __be16 *rlco = cf->rlc_data;
 	const __be16 *end_of_rlco_buf = cf->rlc_data +
 			(cf->size / sizeof(*rlco)) - 1;
 
-	if (!decode_plane(cf, &rlco, ref->luma, height, width, coded_width,
+	if (!decode_plane(cf, &rlco, height, width, ref->luma, ref_stride,
+			  ref->luma_alpha_step, dst->luma, dst_stride,
+			  dst->luma_alpha_step,
 			  hdr_flags & FWHT_FL_LUMA_IS_UNCOMPRESSED,
 			  end_of_rlco_buf))
 		return false;
@@ -901,27 +927,30 @@ bool fwht_decode_frame(struct fwht_cframe *cf, struct fwht_raw_frame *ref,
 	if (components_num >= 3) {
 		u32 h = height;
 		u32 w = width;
-		u32 c = coded_width;
 
 		if (!(hdr_flags & FWHT_FL_CHROMA_FULL_HEIGHT))
 			h /= 2;
-		if (!(hdr_flags & FWHT_FL_CHROMA_FULL_WIDTH)) {
+		if (!(hdr_flags & FWHT_FL_CHROMA_FULL_WIDTH))
 			w /= 2;
-			c /= 2;
-		}
-		if (!decode_plane(cf, &rlco, ref->cb, h, w, c,
+
+		if (!decode_plane(cf, &rlco, h, w, ref->cb, ref_chroma_stride,
+				  ref->chroma_step, dst->cb, dst_chroma_stride,
+				  dst->chroma_step,
 				  hdr_flags & FWHT_FL_CB_IS_UNCOMPRESSED,
 				  end_of_rlco_buf))
 			return false;
-		if (!decode_plane(cf, &rlco, ref->cr, h, w, c,
+		if (!decode_plane(cf, &rlco, h, w, ref->cr, ref_chroma_stride,
+				  ref->chroma_step, dst->cr, dst_chroma_stride,
+				  dst->chroma_step,
 				  hdr_flags & FWHT_FL_CR_IS_UNCOMPRESSED,
 				  end_of_rlco_buf))
 			return false;
 	}
 
 	if (components_num == 4)
-		if (!decode_plane(cf, &rlco, ref->alpha, height, width,
-				  coded_width,
+		if (!decode_plane(cf, &rlco, height, width, ref->alpha, ref_stride,
+				  ref->luma_alpha_step, dst->alpha, dst_stride,
+				  dst->luma_alpha_step,
 				  hdr_flags & FWHT_FL_ALPHA_IS_UNCOMPRESSED,
 				  end_of_rlco_buf))
 			return false;
diff --git a/drivers/media/platform/vicodec/codec-fwht.h b/drivers/media/platform/vicodec/codec-fwht.h
index c410512d47c5..b6fec2b1cbca 100644
--- a/drivers/media/platform/vicodec/codec-fwht.h
+++ b/drivers/media/platform/vicodec/codec-fwht.h
@@ -124,6 +124,7 @@ struct fwht_raw_frame {
 	unsigned int luma_alpha_step;
 	unsigned int chroma_step;
 	unsigned int components_num;
+	u8 *buf;
 	u8 *luma, *cb, *cr, *alpha;
 };
 
@@ -140,9 +141,10 @@ u32 fwht_encode_frame(struct fwht_raw_frame *frm,
 		      bool is_intra, bool next_is_intra,
 		      unsigned int width, unsigned int height,
 		      unsigned int stride, unsigned int chroma_stride);
-bool fwht_decode_frame(struct fwht_cframe *cf, struct fwht_raw_frame *ref,
-		       u32 hdr_flags, unsigned int components_num,
-		       unsigned int width, unsigned int height,
-		       unsigned int coded_width);
-
+bool fwht_decode_frame(struct fwht_cframe *cf, u32 hdr_flags,
+		unsigned int components_num, unsigned int width,
+		unsigned int height, const struct fwht_raw_frame *ref,
+		unsigned int ref_stride, unsigned int ref_chroma_stride,
+		struct fwht_raw_frame *dst, unsigned int dst_stride,
+		unsigned int dst_chroma_stride);
 #endif
diff --git a/drivers/media/platform/vicodec/codec-v4l2-fwht.c b/drivers/media/platform/vicodec/codec-v4l2-fwht.c
index 6573a471c5ca..01e7f09efc4e 100644
--- a/drivers/media/platform/vicodec/codec-v4l2-fwht.c
+++ b/drivers/media/platform/vicodec/codec-v4l2-fwht.c
@@ -37,7 +37,19 @@ static const struct v4l2_fwht_pixfmt_info v4l2_fwht_pixfmts[] = {
 	{ V4L2_PIX_FMT_GREY,    1, 1, 1, 1, 0, 1, 1, 1, 1, FWHT_FL_PIXENC_RGB},
 };
 
-const struct v4l2_fwht_pixfmt_info *v4l2_fwht_default_fmt(u32 width_div,
+bool v4l2_fwht_validate_fmt(const struct v4l2_fwht_pixfmt_info *info,
+			    u32 width_div, u32 height_div, u32 components_num,
+			    u32 pixenc)
+{
+	if (info->width_div == width_div &&
+	    info->height_div == height_div &&
+	    (!pixenc || info->pixenc == pixenc) &&
+	    info->components_num == components_num)
+		return true;
+	return false;
+}
+
+const struct v4l2_fwht_pixfmt_info *v4l2_fwht_find_nth_fmt(u32 width_div,
 							  u32 height_div,
 							  u32 components_num,
 							  u32 pixenc,
@@ -46,10 +58,10 @@ const struct v4l2_fwht_pixfmt_info *v4l2_fwht_default_fmt(u32 width_div,
 	unsigned int i;
 
 	for (i = 0; i < ARRAY_SIZE(v4l2_fwht_pixfmts); i++) {
-		if (v4l2_fwht_pixfmts[i].width_div == width_div &&
-		    v4l2_fwht_pixfmts[i].height_div == height_div &&
-		    (!pixenc || v4l2_fwht_pixfmts[i].pixenc == pixenc) &&
-		    v4l2_fwht_pixfmts[i].components_num == components_num) {
+		bool is_valid = v4l2_fwht_validate_fmt(&v4l2_fwht_pixfmts[i],
+						       width_div, height_div,
+						       components_num, pixenc);
+		if (is_valid) {
 			if (start_idx == 0)
 				return v4l2_fwht_pixfmts + i;
 			start_idx--;
@@ -75,117 +87,141 @@ const struct v4l2_fwht_pixfmt_info *v4l2_fwht_get_pixfmt(u32 idx)
 	return v4l2_fwht_pixfmts + idx;
 }
 
-int v4l2_fwht_encode(struct v4l2_fwht_state *state, u8 *p_in, u8 *p_out)
+static int prepare_raw_frame(struct fwht_raw_frame *rf,
+			 const struct v4l2_fwht_pixfmt_info *info, u8 *buf,
+			 unsigned int size)
 {
-	unsigned int size = state->stride * state->coded_height;
-	unsigned int chroma_stride = state->stride;
-	const struct v4l2_fwht_pixfmt_info *info = state->info;
-	struct fwht_cframe_hdr *p_hdr;
-	struct fwht_cframe cf;
-	struct fwht_raw_frame rf;
-	u32 encoding;
-	u32 flags = 0;
-
-	if (!info)
-		return -EINVAL;
-
-	rf.luma = p_in;
-	rf.width_div = info->width_div;
-	rf.height_div = info->height_div;
-	rf.luma_alpha_step = info->luma_alpha_step;
-	rf.chroma_step = info->chroma_step;
-	rf.alpha = NULL;
-	rf.components_num = info->components_num;
+	rf->luma = buf;
+	rf->width_div = info->width_div;
+	rf->height_div = info->height_div;
+	rf->luma_alpha_step = info->luma_alpha_step;
+	rf->chroma_step = info->chroma_step;
+	rf->alpha = NULL;
+	rf->components_num = info->components_num;
 
+	/*
+	 * The buffer is NULL if it is the reference
+	 * frame of an I-frame in the stateless decoder
+	 */
+	if (!buf) {
+		rf->luma = NULL;
+		rf->cb = NULL;
+		rf->cr = NULL;
+		rf->alpha = NULL;
+		return 0;
+	}
 	switch (info->id) {
 	case V4L2_PIX_FMT_GREY:
-		rf.cb = NULL;
-		rf.cr = NULL;
+		rf->cb = NULL;
+		rf->cr = NULL;
 		break;
 	case V4L2_PIX_FMT_YUV420:
-		rf.cb = rf.luma + size;
-		rf.cr = rf.cb + size / 4;
-		chroma_stride /= 2;
+		rf->cb = rf->luma + size;
+		rf->cr = rf->cb + size / 4;
 		break;
 	case V4L2_PIX_FMT_YVU420:
-		rf.cr = rf.luma + size;
-		rf.cb = rf.cr + size / 4;
-		chroma_stride /= 2;
+		rf->cr = rf->luma + size;
+		rf->cb = rf->cr + size / 4;
 		break;
 	case V4L2_PIX_FMT_YUV422P:
-		rf.cb = rf.luma + size;
-		rf.cr = rf.cb + size / 2;
-		chroma_stride /= 2;
+		rf->cb = rf->luma + size;
+		rf->cr = rf->cb + size / 2;
 		break;
 	case V4L2_PIX_FMT_NV12:
 	case V4L2_PIX_FMT_NV16:
 	case V4L2_PIX_FMT_NV24:
-		rf.cb = rf.luma + size;
-		rf.cr = rf.cb + 1;
+		rf->cb = rf->luma + size;
+		rf->cr = rf->cb + 1;
 		break;
 	case V4L2_PIX_FMT_NV21:
 	case V4L2_PIX_FMT_NV61:
 	case V4L2_PIX_FMT_NV42:
-		rf.cr = rf.luma + size;
-		rf.cb = rf.cr + 1;
+		rf->cr = rf->luma + size;
+		rf->cb = rf->cr + 1;
 		break;
 	case V4L2_PIX_FMT_YUYV:
-		rf.cb = rf.luma + 1;
-		rf.cr = rf.cb + 2;
+		rf->cb = rf->luma + 1;
+		rf->cr = rf->cb + 2;
 		break;
 	case V4L2_PIX_FMT_YVYU:
-		rf.cr = rf.luma + 1;
-		rf.cb = rf.cr + 2;
+		rf->cr = rf->luma + 1;
+		rf->cb = rf->cr + 2;
 		break;
 	case V4L2_PIX_FMT_UYVY:
-		rf.cb = rf.luma;
-		rf.cr = rf.cb + 2;
-		rf.luma++;
+		rf->cb = rf->luma;
+		rf->cr = rf->cb + 2;
+		rf->luma++;
 		break;
 	case V4L2_PIX_FMT_VYUY:
-		rf.cr = rf.luma;
-		rf.cb = rf.cr + 2;
-		rf.luma++;
+		rf->cr = rf->luma;
+		rf->cb = rf->cr + 2;
+		rf->luma++;
 		break;
 	case V4L2_PIX_FMT_RGB24:
 	case V4L2_PIX_FMT_HSV24:
-		rf.cr = rf.luma;
-		rf.cb = rf.cr + 2;
-		rf.luma++;
+		rf->cr = rf->luma;
+		rf->cb = rf->cr + 2;
+		rf->luma++;
 		break;
 	case V4L2_PIX_FMT_BGR24:
-		rf.cb = rf.luma;
-		rf.cr = rf.cb + 2;
-		rf.luma++;
+		rf->cb = rf->luma;
+		rf->cr = rf->cb + 2;
+		rf->luma++;
 		break;
 	case V4L2_PIX_FMT_RGB32:
 	case V4L2_PIX_FMT_XRGB32:
 	case V4L2_PIX_FMT_HSV32:
-		rf.cr = rf.luma + 1;
-		rf.cb = rf.cr + 2;
-		rf.luma += 2;
+		rf->cr = rf->luma + 1;
+		rf->cb = rf->cr + 2;
+		rf->luma += 2;
 		break;
 	case V4L2_PIX_FMT_BGR32:
 	case V4L2_PIX_FMT_XBGR32:
-		rf.cb = rf.luma;
-		rf.cr = rf.cb + 2;
-		rf.luma++;
+		rf->cb = rf->luma;
+		rf->cr = rf->cb + 2;
+		rf->luma++;
 		break;
 	case V4L2_PIX_FMT_ARGB32:
-		rf.alpha = rf.luma;
-		rf.cr = rf.luma + 1;
-		rf.cb = rf.cr + 2;
-		rf.luma += 2;
+		rf->alpha = rf->luma;
+		rf->cr = rf->luma + 1;
+		rf->cb = rf->cr + 2;
+		rf->luma += 2;
 		break;
 	case V4L2_PIX_FMT_ABGR32:
-		rf.cb = rf.luma;
-		rf.cr = rf.cb + 2;
-		rf.luma++;
-		rf.alpha = rf.cr + 1;
+		rf->cb = rf->luma;
+		rf->cr = rf->cb + 2;
+		rf->luma++;
+		rf->alpha = rf->cr + 1;
 		break;
 	default:
 		return -EINVAL;
 	}
+	return 0;
+}
+
+int v4l2_fwht_encode(struct v4l2_fwht_state *state, u8 *p_in, u8 *p_out)
+{
+	unsigned int size = state->stride * state->coded_height;
+	unsigned int chroma_stride = state->stride;
+	const struct v4l2_fwht_pixfmt_info *info = state->info;
+	struct fwht_cframe_hdr *p_hdr;
+	struct fwht_cframe cf;
+	struct fwht_raw_frame rf;
+	u32 encoding;
+	u32 flags = 0;
+
+	if (!info)
+		return -EINVAL;
+
+	if (prepare_raw_frame(&rf, info, p_in, size))
+		return -EINVAL;
+
+	if (info->planes_num == 3)
+		chroma_stride /= 2;
+
+	if (info->id == V4L2_PIX_FMT_NV24 ||
+	    info->id == V4L2_PIX_FMT_NV42)
+		chroma_stride *= 2;
 
 	cf.i_frame_qp = state->i_frame_qp;
 	cf.p_frame_qp = state->p_frame_qp;
@@ -235,14 +271,17 @@ int v4l2_fwht_encode(struct v4l2_fwht_state *state, u8 *p_in, u8 *p_out)
 
 int v4l2_fwht_decode(struct v4l2_fwht_state *state, u8 *p_in, u8 *p_out)
 {
-	unsigned int i, j, k;
 	u32 flags;
 	struct fwht_cframe cf;
-	u8 *p, *ref_p;
 	unsigned int components_num = 3;
 	unsigned int version;
 	const struct v4l2_fwht_pixfmt_info *info;
 	unsigned int hdr_width_div, hdr_height_div;
+	struct fwht_raw_frame dst_rf;
+	unsigned int dst_chroma_stride = state->stride;
+	unsigned int ref_chroma_stride = state->ref_stride;
+	unsigned int dst_size = state->stride * state->coded_height;
+	unsigned int ref_size;
 
 	if (!state->info)
 		return -EINVAL;
@@ -290,241 +329,29 @@ int v4l2_fwht_decode(struct v4l2_fwht_state *state, u8 *p_in, u8 *p_out)
 	    hdr_height_div != info->height_div)
 		return -EINVAL;
 
-	if (!fwht_decode_frame(&cf, &state->ref_frame, flags, components_num,
-			       state->visible_width, state->visible_height,
-			       state->coded_width))
+	if (prepare_raw_frame(&dst_rf, info, p_out, dst_size))
 		return -EINVAL;
+	if (info->planes_num == 3) {
+		dst_chroma_stride /= 2;
+		ref_chroma_stride /= 2;
+	}
+	if (info->id == V4L2_PIX_FMT_NV24 ||
+	    info->id == V4L2_PIX_FMT_NV42) {
+		dst_chroma_stride *= 2;
+		ref_chroma_stride *= 2;
+	}
 
-	/*
-	 * TODO - handle the case where the compressed stream encodes a
-	 * different format than the requested decoded format.
-	 */
-	switch (state->info->id) {
-	case V4L2_PIX_FMT_GREY:
-		ref_p = state->ref_frame.luma;
-		for (i = 0; i < state->coded_height; i++)  {
-			memcpy(p_out, ref_p, state->visible_width);
-			p_out += state->stride;
-			ref_p += state->coded_width;
-		}
-		break;
-	case V4L2_PIX_FMT_YUV420:
-	case V4L2_PIX_FMT_YUV422P:
-		ref_p = state->ref_frame.luma;
-		for (i = 0; i < state->coded_height; i++)  {
-			memcpy(p_out, ref_p, state->visible_width);
-			p_out += state->stride;
-			ref_p += state->coded_width;
-		}
 
-		ref_p = state->ref_frame.cb;
-		for (i = 0; i < state->coded_height / 2; i++)  {
-			memcpy(p_out, ref_p, state->visible_width / 2);
-			p_out += state->stride / 2;
-			ref_p += state->coded_width / 2;
-		}
-		ref_p = state->ref_frame.cr;
-		for (i = 0; i < state->coded_height / 2; i++)  {
-			memcpy(p_out, ref_p, state->visible_width / 2);
-			p_out += state->stride / 2;
-			ref_p += state->coded_width / 2;
-		}
-		break;
-	case V4L2_PIX_FMT_YVU420:
-		ref_p = state->ref_frame.luma;
-		for (i = 0; i < state->coded_height; i++)  {
-			memcpy(p_out, ref_p, state->visible_width);
-			p_out += state->stride;
-			ref_p += state->coded_width;
-		}
-
-		ref_p = state->ref_frame.cr;
-		for (i = 0; i < state->coded_height / 2; i++)  {
-			memcpy(p_out, ref_p, state->visible_width / 2);
-			p_out += state->stride / 2;
-			ref_p += state->coded_width / 2;
-		}
-		ref_p = state->ref_frame.cb;
-		for (i = 0; i < state->coded_height / 2; i++)  {
-			memcpy(p_out, ref_p, state->visible_width / 2);
-			p_out += state->stride / 2;
-			ref_p += state->coded_width / 2;
-		}
-		break;
-	case V4L2_PIX_FMT_NV12:
-	case V4L2_PIX_FMT_NV16:
-	case V4L2_PIX_FMT_NV24:
-		ref_p = state->ref_frame.luma;
-		for (i = 0; i < state->coded_height; i++)  {
-			memcpy(p_out, ref_p, state->visible_width);
-			p_out += state->stride;
-			ref_p += state->coded_width;
-		}
+	ref_size = state->ref_stride * state->coded_height;
 
-		k = 0;
-		for (i = 0; i < state->coded_height / 2; i++) {
-			for (j = 0, p = p_out; j < state->coded_width / 2; j++) {
-				*p++ = state->ref_frame.cb[k];
-				*p++ = state->ref_frame.cr[k];
-				k++;
-			}
-			p_out += state->stride;
-		}
-		break;
-	case V4L2_PIX_FMT_NV21:
-	case V4L2_PIX_FMT_NV61:
-	case V4L2_PIX_FMT_NV42:
-		ref_p = state->ref_frame.luma;
-		for (i = 0; i < state->coded_height; i++)  {
-			memcpy(p_out, ref_p, state->visible_width);
-			p_out += state->stride;
-			ref_p += state->coded_width;
-		}
+	if (prepare_raw_frame(&state->ref_frame, info, state->ref_frame.buf,
+			      ref_size))
+		return -EINVAL;
 
-		k = 0;
-		for (i = 0; i < state->coded_height / 2; i++) {
-			for (j = 0, p = p_out; j < state->coded_width / 2; j++) {
-				*p++ = state->ref_frame.cr[k];
-				*p++ = state->ref_frame.cb[k];
-				k++;
-			}
-			p_out += state->stride;
-		}
-		break;
-	case V4L2_PIX_FMT_YUYV:
-		k = 0;
-		for (i = 0; i < state->coded_height; i++) {
-			for (j = 0, p = p_out; j < state->coded_width / 2; j++) {
-				*p++ = state->ref_frame.luma[k];
-				*p++ = state->ref_frame.cb[k / 2];
-				*p++ = state->ref_frame.luma[k + 1];
-				*p++ = state->ref_frame.cr[k / 2];
-				k += 2;
-			}
-			p_out += state->stride;
-		}
-		break;
-	case V4L2_PIX_FMT_YVYU:
-		k = 0;
-		for (i = 0; i < state->coded_height; i++) {
-			for (j = 0, p = p_out; j < state->coded_width / 2; j++) {
-				*p++ = state->ref_frame.luma[k];
-				*p++ = state->ref_frame.cr[k / 2];
-				*p++ = state->ref_frame.luma[k + 1];
-				*p++ = state->ref_frame.cb[k / 2];
-				k += 2;
-			}
-			p_out += state->stride;
-		}
-		break;
-	case V4L2_PIX_FMT_UYVY:
-		k = 0;
-		for (i = 0; i < state->coded_height; i++) {
-			for (j = 0, p = p_out; j < state->coded_width / 2; j++) {
-				*p++ = state->ref_frame.cb[k / 2];
-				*p++ = state->ref_frame.luma[k];
-				*p++ = state->ref_frame.cr[k / 2];
-				*p++ = state->ref_frame.luma[k + 1];
-				k += 2;
-			}
-			p_out += state->stride;
-		}
-		break;
-	case V4L2_PIX_FMT_VYUY:
-		k = 0;
-		for (i = 0; i < state->coded_height; i++) {
-			for (j = 0, p = p_out; j < state->coded_width / 2; j++) {
-				*p++ = state->ref_frame.cr[k / 2];
-				*p++ = state->ref_frame.luma[k];
-				*p++ = state->ref_frame.cb[k / 2];
-				*p++ = state->ref_frame.luma[k + 1];
-				k += 2;
-			}
-			p_out += state->stride;
-		}
-		break;
-	case V4L2_PIX_FMT_RGB24:
-	case V4L2_PIX_FMT_HSV24:
-		k = 0;
-		for (i = 0; i < state->coded_height; i++) {
-			for (j = 0, p = p_out; j < state->coded_width; j++) {
-				*p++ = state->ref_frame.cr[k];
-				*p++ = state->ref_frame.luma[k];
-				*p++ = state->ref_frame.cb[k];
-				k++;
-			}
-			p_out += state->stride;
-		}
-		break;
-	case V4L2_PIX_FMT_BGR24:
-		k = 0;
-		for (i = 0; i < state->coded_height; i++) {
-			for (j = 0, p = p_out; j < state->coded_width; j++) {
-				*p++ = state->ref_frame.cb[k];
-				*p++ = state->ref_frame.luma[k];
-				*p++ = state->ref_frame.cr[k];
-				k++;
-			}
-			p_out += state->stride;
-		}
-		break;
-	case V4L2_PIX_FMT_RGB32:
-	case V4L2_PIX_FMT_XRGB32:
-	case V4L2_PIX_FMT_HSV32:
-		k = 0;
-		for (i = 0; i < state->coded_height; i++) {
-			for (j = 0, p = p_out; j < state->coded_width; j++) {
-				*p++ = 0;
-				*p++ = state->ref_frame.cr[k];
-				*p++ = state->ref_frame.luma[k];
-				*p++ = state->ref_frame.cb[k];
-				k++;
-			}
-			p_out += state->stride;
-		}
-		break;
-	case V4L2_PIX_FMT_BGR32:
-	case V4L2_PIX_FMT_XBGR32:
-		k = 0;
-		for (i = 0; i < state->coded_height; i++) {
-			for (j = 0, p = p_out; j < state->coded_width; j++) {
-				*p++ = state->ref_frame.cb[k];
-				*p++ = state->ref_frame.luma[k];
-				*p++ = state->ref_frame.cr[k];
-				*p++ = 0;
-				k++;
-			}
-			p_out += state->stride;
-		}
-		break;
-	case V4L2_PIX_FMT_ARGB32:
-		k = 0;
-		for (i = 0; i < state->coded_height; i++) {
-			for (j = 0, p = p_out; j < state->coded_width; j++) {
-				*p++ = state->ref_frame.alpha[k];
-				*p++ = state->ref_frame.cr[k];
-				*p++ = state->ref_frame.luma[k];
-				*p++ = state->ref_frame.cb[k];
-				k++;
-			}
-			p_out += state->stride;
-		}
-		break;
-	case V4L2_PIX_FMT_ABGR32:
-		k = 0;
-		for (i = 0; i < state->coded_height; i++) {
-			for (j = 0, p = p_out; j < state->coded_width; j++) {
-				*p++ = state->ref_frame.cb[k];
-				*p++ = state->ref_frame.luma[k];
-				*p++ = state->ref_frame.cr[k];
-				*p++ = state->ref_frame.alpha[k];
-				k++;
-			}
-			p_out += state->stride;
-		}
-		break;
-	default:
+	if (!fwht_decode_frame(&cf, flags, components_num,
+			state->visible_width, state->visible_height,
+			&state->ref_frame, state->ref_stride, ref_chroma_stride,
+			&dst_rf, state->stride, dst_chroma_stride))
 		return -EINVAL;
-	}
 	return 0;
 }
diff --git a/drivers/media/platform/vicodec/codec-v4l2-fwht.h b/drivers/media/platform/vicodec/codec-v4l2-fwht.h
index aa6fa90a48be..1a0d2a9f931a 100644
--- a/drivers/media/platform/vicodec/codec-v4l2-fwht.h
+++ b/drivers/media/platform/vicodec/codec-v4l2-fwht.h
@@ -30,6 +30,7 @@ struct v4l2_fwht_state {
 	unsigned int coded_width;
 	unsigned int coded_height;
 	unsigned int stride;
+	unsigned int ref_stride;
 	unsigned int gop_size;
 	unsigned int gop_cnt;
 	u16 i_frame_qp;
@@ -43,11 +44,15 @@ struct v4l2_fwht_state {
 	struct fwht_raw_frame ref_frame;
 	struct fwht_cframe_hdr header;
 	u8 *compressed_frame;
+	u64 ref_frame_ts;
 };
 
 const struct v4l2_fwht_pixfmt_info *v4l2_fwht_find_pixfmt(u32 pixelformat);
 const struct v4l2_fwht_pixfmt_info *v4l2_fwht_get_pixfmt(u32 idx);
-const struct v4l2_fwht_pixfmt_info *v4l2_fwht_default_fmt(u32 width_div,
+bool v4l2_fwht_validate_fmt(const struct v4l2_fwht_pixfmt_info *info,
+			    u32 width_div, u32 height_div, u32 components_num,
+			    u32 pixenc);
+const struct v4l2_fwht_pixfmt_info *v4l2_fwht_find_nth_fmt(u32 width_div,
 							  u32 height_div,
 							  u32 components_num,
 							  u32 pixenc,
diff --git a/drivers/media/platform/vicodec/vicodec-core.c b/drivers/media/platform/vicodec/vicodec-core.c
index d7636fe9e174..bd01a9206aa6 100644
--- a/drivers/media/platform/vicodec/vicodec-core.c
+++ b/drivers/media/platform/vicodec/vicodec-core.c
@@ -64,6 +64,10 @@ static const struct v4l2_fwht_pixfmt_info pixfmt_fwht = {
 	V4L2_PIX_FMT_FWHT, 0, 3, 1, 1, 1, 1, 1, 0, 1
 };
 
+static const struct v4l2_fwht_pixfmt_info pixfmt_stateless_fwht = {
+	V4L2_PIX_FMT_FWHT_STATELESS, 0, 3, 1, 1, 1, 1, 1, 0, 1
+};
+
 static void vicodec_dev_release(struct device *dev)
 {
 }
@@ -89,27 +93,29 @@ enum {
 	V4L2_M2M_DST = 1,
 };
 
+struct vicodec_dev_instance {
+	struct video_device     vfd;
+	struct mutex            mutex;
+	spinlock_t              lock;
+	struct v4l2_m2m_dev     *m2m_dev;
+};
+
 struct vicodec_dev {
 	struct v4l2_device	v4l2_dev;
-	struct video_device	enc_vfd;
-	struct video_device	dec_vfd;
+	struct vicodec_dev_instance stateful_enc;
+	struct vicodec_dev_instance stateful_dec;
+	struct vicodec_dev_instance stateless_dec;
 #ifdef CONFIG_MEDIA_CONTROLLER
 	struct media_device	mdev;
 #endif
 
-	struct mutex		enc_mutex;
-	struct mutex		dec_mutex;
-	spinlock_t		enc_lock;
-	spinlock_t		dec_lock;
-
-	struct v4l2_m2m_dev	*enc_dev;
-	struct v4l2_m2m_dev	*dec_dev;
 };
 
 struct vicodec_ctx {
 	struct v4l2_fh		fh;
 	struct vicodec_dev	*dev;
 	bool			is_enc;
+	bool			is_stateless;
 	spinlock_t		*lock;
 
 	struct v4l2_ctrl_handler hdl;
@@ -148,27 +154,149 @@ static struct vicodec_q_data *get_q_data(struct vicodec_ctx *ctx,
 	case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
 		return &ctx->q_data[V4L2_M2M_DST];
 	default:
-		WARN_ON(1);
 		break;
 	}
 	return NULL;
 }
 
+static void copy_cap_to_ref(const u8 *cap, const struct v4l2_fwht_pixfmt_info *info,
+		struct v4l2_fwht_state *state)
+{
+	int plane_idx;
+	u8 *p_ref = state->ref_frame.buf;
+	unsigned int cap_stride = state->stride;
+	unsigned int ref_stride = state->ref_stride;
+
+	for (plane_idx = 0; plane_idx < info->planes_num; plane_idx++) {
+		int i;
+		unsigned int h_div = (plane_idx == 1 || plane_idx == 2) ?
+			info->height_div : 1;
+		const u8 *row_cap = cap;
+		u8 *row_ref = p_ref;
+
+		if (info->planes_num == 3 && plane_idx == 1) {
+			cap_stride /= 2;
+			ref_stride /= 2;
+		}
+
+		if (plane_idx == 1 &&
+		    (info->id == V4L2_PIX_FMT_NV24 ||
+		     info->id == V4L2_PIX_FMT_NV42)) {
+			cap_stride *= 2;
+			ref_stride *= 2;
+		}
+
+		for (i = 0; i < state->visible_height / h_div; i++) {
+			memcpy(row_ref, row_cap, ref_stride);
+			row_ref += ref_stride;
+			row_cap += cap_stride;
+		}
+		cap += cap_stride * (state->coded_height / h_div);
+		p_ref += ref_stride * (state->coded_height / h_div);
+	}
+}
+
+static bool validate_by_version(unsigned int flags, unsigned int version)
+{
+	if (!version || version > FWHT_VERSION)
+		return false;
+
+	if (version >= 2) {
+		unsigned int components_num = 1 +
+			((flags & FWHT_FL_COMPONENTS_NUM_MSK) >>
+			 FWHT_FL_COMPONENTS_NUM_OFFSET);
+		unsigned int pixenc = flags & FWHT_FL_PIXENC_MSK;
+
+		if (components_num == 0 || components_num > 4 || !pixenc)
+			return false;
+	}
+	return true;
+}
+
+static bool validate_stateless_params_flags(const struct v4l2_ctrl_fwht_params *params,
+					    const struct v4l2_fwht_pixfmt_info *cur_info)
+{
+	unsigned int width_div =
+		(params->flags & FWHT_FL_CHROMA_FULL_WIDTH) ? 1 : 2;
+	unsigned int height_div =
+		(params->flags & FWHT_FL_CHROMA_FULL_HEIGHT) ? 1 : 2;
+	unsigned int components_num = 3;
+	unsigned int pixenc = 0;
+
+	if (params->version < 3)
+		return false;
+
+	components_num = 1 + ((params->flags & FWHT_FL_COMPONENTS_NUM_MSK) >>
+			      FWHT_FL_COMPONENTS_NUM_OFFSET);
+	pixenc = (params->flags & FWHT_FL_PIXENC_MSK);
+	if (v4l2_fwht_validate_fmt(cur_info, width_div, height_div,
+				    components_num, pixenc))
+		return true;
+	return false;
+}
+
+
+static void update_state_from_header(struct vicodec_ctx *ctx)
+{
+	const struct fwht_cframe_hdr *p_hdr = &ctx->state.header;
+
+	ctx->state.visible_width = ntohl(p_hdr->width);
+	ctx->state.visible_height = ntohl(p_hdr->height);
+	ctx->state.colorspace = ntohl(p_hdr->colorspace);
+	ctx->state.xfer_func = ntohl(p_hdr->xfer_func);
+	ctx->state.ycbcr_enc = ntohl(p_hdr->ycbcr_enc);
+	ctx->state.quantization = ntohl(p_hdr->quantization);
+}
+
 static int device_process(struct vicodec_ctx *ctx,
 			  struct vb2_v4l2_buffer *src_vb,
 			  struct vb2_v4l2_buffer *dst_vb)
 {
 	struct vicodec_dev *dev = ctx->dev;
-	struct vicodec_q_data *q_dst;
 	struct v4l2_fwht_state *state = &ctx->state;
 	u8 *p_src, *p_dst;
-	int ret;
+	int ret = 0;
 
-	q_dst = get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_CAPTURE);
-	if (ctx->is_enc)
+	if (ctx->is_enc || ctx->is_stateless)
 		p_src = vb2_plane_vaddr(&src_vb->vb2_buf, 0);
 	else
 		p_src = state->compressed_frame;
+
+	if (ctx->is_stateless) {
+		struct media_request *src_req = src_vb->vb2_buf.req_obj.req;
+
+		ret = v4l2_ctrl_request_setup(src_req, &ctx->hdl);
+		if (ret)
+			return ret;
+		update_state_from_header(ctx);
+
+		ctx->state.header.size =
+			htonl(vb2_get_plane_payload(&src_vb->vb2_buf, 0));
+		/*
+		 * set the reference buffer from the reference timestamp
+		 * only if this is a P-frame
+		 */
+		if (!(ntohl(ctx->state.header.flags) & FWHT_FL_I_FRAME)) {
+			struct vb2_buffer *ref_vb2_buf;
+			int ref_buf_idx;
+			struct vb2_queue *vq_cap =
+				v4l2_m2m_get_vq(ctx->fh.m2m_ctx,
+						V4L2_BUF_TYPE_VIDEO_CAPTURE);
+
+			ref_buf_idx = vb2_find_timestamp(vq_cap,
+							 ctx->state.ref_frame_ts, 0);
+			if (ref_buf_idx < 0)
+				return -EINVAL;
+
+			ref_vb2_buf = vq_cap->bufs[ref_buf_idx];
+			if (ref_vb2_buf->state == VB2_BUF_STATE_ERROR)
+				ret = -EINVAL;
+			ctx->state.ref_frame.buf =
+				vb2_plane_vaddr(ref_vb2_buf, 0);
+		} else {
+			ctx->state.ref_frame.buf = NULL;
+		}
+	}
 	p_dst = vb2_plane_vaddr(&dst_vb->vb2_buf, 0);
 	if (!p_src || !p_dst) {
 		v4l2_err(&dev->v4l2_dev,
@@ -178,30 +306,31 @@ static int device_process(struct vicodec_ctx *ctx,
 
 	if (ctx->is_enc) {
 		struct vicodec_q_data *q_src;
+		int comp_sz_or_errcode;
 
 		q_src = get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_OUTPUT);
 		state->info = q_src->info;
-		ret = v4l2_fwht_encode(state, p_src, p_dst);
-		if (ret < 0)
-			return ret;
-		vb2_set_plane_payload(&dst_vb->vb2_buf, 0, ret);
+		comp_sz_or_errcode = v4l2_fwht_encode(state, p_src, p_dst);
+		if (comp_sz_or_errcode < 0)
+			return comp_sz_or_errcode;
+		vb2_set_plane_payload(&dst_vb->vb2_buf, 0, comp_sz_or_errcode);
 	} else {
+		struct vicodec_q_data *q_dst;
 		unsigned int comp_frame_size = ntohl(ctx->state.header.size);
 
+		q_dst = get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_CAPTURE);
 		if (comp_frame_size > ctx->comp_max_size)
 			return -EINVAL;
 		state->info = q_dst->info;
 		ret = v4l2_fwht_decode(state, p_src, p_dst);
 		if (ret < 0)
 			return ret;
+		if (!ctx->is_stateless)
+			copy_cap_to_ref(p_dst, ctx->state.info, &ctx->state);
+
 		vb2_set_plane_payload(&dst_vb->vb2_buf, 0, q_dst->sizeimage);
 	}
-
-	dst_vb->sequence = q_dst->sequence++;
-	dst_vb->flags &= ~V4L2_BUF_FLAG_LAST;
-	v4l2_m2m_buf_copy_metadata(src_vb, dst_vb, !ctx->is_enc);
-
-	return 0;
+	return ret;
 }
 
 /*
@@ -274,16 +403,26 @@ static void device_run(void *priv)
 	struct vicodec_ctx *ctx = priv;
 	struct vicodec_dev *dev = ctx->dev;
 	struct vb2_v4l2_buffer *src_buf, *dst_buf;
-	struct vicodec_q_data *q_src;
+	struct vicodec_q_data *q_src, *q_dst;
 	u32 state;
+	struct media_request *src_req;
+
 
 	src_buf = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx);
 	dst_buf = v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx);
+	src_req = src_buf->vb2_buf.req_obj.req;
+
 	q_src = get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_OUTPUT);
+	q_dst = get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_CAPTURE);
 
 	state = VB2_BUF_STATE_DONE;
 	if (device_process(ctx, src_buf, dst_buf))
 		state = VB2_BUF_STATE_ERROR;
+	else
+		dst_buf->sequence = q_dst->sequence++;
+	dst_buf->flags &= ~V4L2_BUF_FLAG_LAST;
+	v4l2_m2m_buf_copy_metadata(src_buf, dst_buf, !ctx->is_enc);
+
 	ctx->last_dst_buf = dst_buf;
 
 	spin_lock(ctx->lock);
@@ -291,7 +430,7 @@ static void device_run(void *priv)
 		dst_buf->flags |= V4L2_BUF_FLAG_LAST;
 		v4l2_event_queue_fh(&ctx->fh, &eos_event);
 	}
-	if (ctx->is_enc) {
+	if (ctx->is_enc || ctx->is_stateless) {
 		src_buf->sequence = q_src->sequence++;
 		src_buf = v4l2_m2m_src_buf_remove(ctx->fh.m2m_ctx);
 		v4l2_m2m_buf_done(src_buf, state);
@@ -303,6 +442,9 @@ static void device_run(void *priv)
 		ctx->comp_has_next_frame = false;
 	}
 	v4l2_m2m_buf_done(dst_buf, state);
+	if (ctx->is_stateless && src_req)
+		v4l2_ctrl_request_complete(src_req, &ctx->hdl);
+
 	ctx->comp_size = 0;
 	ctx->header_size = 0;
 	ctx->comp_magic_cnt = 0;
@@ -310,9 +452,12 @@ static void device_run(void *priv)
 	spin_unlock(ctx->lock);
 
 	if (ctx->is_enc)
-		v4l2_m2m_job_finish(dev->enc_dev, ctx->fh.m2m_ctx);
+		v4l2_m2m_job_finish(dev->stateful_enc.m2m_dev, ctx->fh.m2m_ctx);
+	else if (ctx->is_stateless)
+		v4l2_m2m_job_finish(dev->stateless_dec.m2m_dev,
+				    ctx->fh.m2m_ctx);
 	else
-		v4l2_m2m_job_finish(dev->dec_dev, ctx->fh.m2m_ctx);
+		v4l2_m2m_job_finish(dev->stateful_dec.m2m_dev, ctx->fh.m2m_ctx);
 }
 
 static void job_remove_src_buf(struct vicodec_ctx *ctx, u32 state)
@@ -344,7 +489,7 @@ info_from_header(const struct fwht_cframe_hdr *p_hdr)
 				FWHT_FL_COMPONENTS_NUM_OFFSET);
 		pixenc = (flags & FWHT_FL_PIXENC_MSK);
 	}
-	return v4l2_fwht_default_fmt(width_div, height_div,
+	return v4l2_fwht_find_nth_fmt(width_div, height_div,
 				     components_num, pixenc, 0);
 }
 
@@ -356,21 +501,11 @@ static bool is_header_valid(const struct fwht_cframe_hdr *p_hdr)
 	unsigned int version = ntohl(p_hdr->version);
 	unsigned int flags = ntohl(p_hdr->flags);
 
-	if (!version || version > FWHT_VERSION)
-		return false;
-
 	if (w < MIN_WIDTH || w > MAX_WIDTH || h < MIN_HEIGHT || h > MAX_HEIGHT)
 		return false;
 
-	if (version >= 2) {
-		unsigned int components_num = 1 +
-			((flags & FWHT_FL_COMPONENTS_NUM_MSK) >>
-			FWHT_FL_COMPONENTS_NUM_OFFSET);
-		unsigned int pixenc = flags & FWHT_FL_PIXENC_MSK;
-
-		if (components_num == 0 || components_num > 4 || !pixenc)
-			return false;
-	}
+	if (!validate_by_version(flags, version))
+		return false;
 
 	info = info_from_header(p_hdr);
 	if (!info)
@@ -388,6 +523,12 @@ static void update_capture_data_from_header(struct vicodec_ctx *ctx)
 	unsigned int hdr_width_div = (flags & FWHT_FL_CHROMA_FULL_WIDTH) ? 1 : 2;
 	unsigned int hdr_height_div = (flags & FWHT_FL_CHROMA_FULL_HEIGHT) ? 1 : 2;
 
+	/*
+	 * This function should not be used by a stateless codec since
+	 * it changes values in q_data that are not request specific
+	 */
+	WARN_ON(ctx->is_stateless);
+
 	q_dst->info = info;
 	q_dst->visible_width = ntohl(p_hdr->width);
 	q_dst->visible_height = ntohl(p_hdr->height);
@@ -440,7 +581,7 @@ static int job_ready(void *priv)
 
 	if (ctx->source_changed)
 		return 0;
-	if (ctx->is_enc || ctx->comp_has_frame)
+	if (ctx->is_stateless || ctx->is_enc || ctx->comp_has_frame)
 		return 1;
 
 restart:
@@ -551,8 +692,8 @@ static const struct v4l2_fwht_pixfmt_info *find_fmt(u32 fmt)
 static int vidioc_querycap(struct file *file, void *priv,
 			   struct v4l2_capability *cap)
 {
-	strncpy(cap->driver, VICODEC_NAME, sizeof(cap->driver) - 1);
-	strncpy(cap->card, VICODEC_NAME, sizeof(cap->card) - 1);
+	strscpy(cap->driver, VICODEC_NAME, sizeof(cap->driver));
+	strscpy(cap->card, VICODEC_NAME, sizeof(cap->card));
 	snprintf(cap->bus_info, sizeof(cap->bus_info),
 			"platform:%s", VICODEC_NAME);
 	return 0;
@@ -575,7 +716,7 @@ static int enum_fmt(struct v4l2_fmtdesc *f, struct vicodec_ctx *ctx,
 		if (!info || ctx->is_enc)
 			info = v4l2_fwht_get_pixfmt(f->index);
 		else
-			info = v4l2_fwht_default_fmt(info->width_div,
+			info = v4l2_fwht_find_nth_fmt(info->width_div,
 						     info->height_div,
 						     info->components_num,
 						     info->pixenc,
@@ -586,7 +727,8 @@ static int enum_fmt(struct v4l2_fmtdesc *f, struct vicodec_ctx *ctx,
 	} else {
 		if (f->index)
 			return -EINVAL;
-		f->pixelformat = V4L2_PIX_FMT_FWHT;
+		f->pixelformat = ctx->is_stateless ?
+			V4L2_PIX_FMT_FWHT_STATELESS : V4L2_PIX_FMT_FWHT;
 	}
 	return 0;
 }
@@ -688,13 +830,15 @@ static int vidioc_try_fmt(struct vicodec_ctx *ctx, struct v4l2_format *f)
 	struct v4l2_pix_format_mplane *pix_mp;
 	struct v4l2_pix_format *pix;
 	struct v4l2_plane_pix_format *plane;
-	const struct v4l2_fwht_pixfmt_info *info = &pixfmt_fwht;
+	const struct v4l2_fwht_pixfmt_info *info = ctx->is_stateless ?
+		&pixfmt_stateless_fwht : &pixfmt_fwht;
 
 	switch (f->type) {
 	case V4L2_BUF_TYPE_VIDEO_CAPTURE:
 	case V4L2_BUF_TYPE_VIDEO_OUTPUT:
 		pix = &f->fmt.pix;
-		if (pix->pixelformat != V4L2_PIX_FMT_FWHT)
+		if (pix->pixelformat != V4L2_PIX_FMT_FWHT &&
+		    pix->pixelformat != V4L2_PIX_FMT_FWHT_STATELESS)
 			info = find_fmt(pix->pixelformat);
 
 		pix->width = clamp(pix->width, MIN_WIDTH, MAX_WIDTH);
@@ -715,7 +859,8 @@ static int vidioc_try_fmt(struct vicodec_ctx *ctx, struct v4l2_format *f)
 	case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
 		pix_mp = &f->fmt.pix_mp;
 		plane = pix_mp->plane_fmt;
-		if (pix_mp->pixelformat != V4L2_PIX_FMT_FWHT)
+		if (pix_mp->pixelformat != V4L2_PIX_FMT_FWHT &&
+		    pix_mp->pixelformat != V4L2_PIX_FMT_FWHT_STATELESS)
 			info = find_fmt(pix_mp->pixelformat);
 		pix_mp->num_planes = 1;
 
@@ -792,8 +937,12 @@ static int vidioc_try_fmt_vid_out(struct file *file, void *priv,
 		if (multiplanar)
 			return -EINVAL;
 		pix = &f->fmt.pix;
-		pix->pixelformat = !ctx->is_enc ? V4L2_PIX_FMT_FWHT :
-				   find_fmt(pix->pixelformat)->id;
+		if (ctx->is_enc)
+			pix->pixelformat = find_fmt(pix->pixelformat)->id;
+		else if (ctx->is_stateless)
+			pix->pixelformat = V4L2_PIX_FMT_FWHT_STATELESS;
+		else
+			pix->pixelformat = V4L2_PIX_FMT_FWHT;
 		if (!pix->colorspace)
 			pix->colorspace = V4L2_COLORSPACE_REC709;
 		break;
@@ -801,8 +950,12 @@ static int vidioc_try_fmt_vid_out(struct file *file, void *priv,
 		if (!multiplanar)
 			return -EINVAL;
 		pix_mp = &f->fmt.pix_mp;
-		pix_mp->pixelformat = !ctx->is_enc ? V4L2_PIX_FMT_FWHT :
-				      find_fmt(pix_mp->pixelformat)->id;
+		if (ctx->is_enc)
+			pix_mp->pixelformat = find_fmt(pix_mp->pixelformat)->id;
+		else if (ctx->is_stateless)
+			pix_mp->pixelformat = V4L2_PIX_FMT_FWHT_STATELESS;
+		else
+			pix_mp->pixelformat = V4L2_PIX_FMT_FWHT;
 		if (!pix_mp->colorspace)
 			pix_mp->colorspace = V4L2_COLORSPACE_REC709;
 		break;
@@ -845,6 +998,8 @@ static int vidioc_s_fmt(struct vicodec_ctx *ctx, struct v4l2_format *f)
 
 		if (pix->pixelformat == V4L2_PIX_FMT_FWHT)
 			q_data->info = &pixfmt_fwht;
+		else if (pix->pixelformat == V4L2_PIX_FMT_FWHT_STATELESS)
+			q_data->info = &pixfmt_stateless_fwht;
 		else
 			q_data->info = find_fmt(pix->pixelformat);
 		q_data->coded_width = pix->width;
@@ -866,6 +1021,8 @@ static int vidioc_s_fmt(struct vicodec_ctx *ctx, struct v4l2_format *f)
 
 		if (pix_mp->pixelformat == V4L2_PIX_FMT_FWHT)
 			q_data->info = &pixfmt_fwht;
+		else if (pix_mp->pixelformat == V4L2_PIX_FMT_FWHT_STATELESS)
+			q_data->info = &pixfmt_stateless_fwht;
 		else
 			q_data->info = find_fmt(pix_mp->pixelformat);
 		q_data->coded_width = pix_mp->width;
@@ -945,16 +1102,6 @@ static int vidioc_g_selection(struct file *file, void *priv,
 {
 	struct vicodec_ctx *ctx = file2ctx(file);
 	struct vicodec_q_data *q_data;
-	enum v4l2_buf_type valid_cap_type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
-	enum v4l2_buf_type valid_out_type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
-
-	if (multiplanar) {
-		valid_cap_type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
-		valid_out_type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE;
-	}
-
-	if (s->type != valid_cap_type && s->type != valid_out_type)
-		return -EINVAL;
 
 	q_data = get_q_data(ctx, s->type);
 	if (!q_data)
@@ -963,18 +1110,14 @@ static int vidioc_g_selection(struct file *file, void *priv,
 	 * encoder supports only cropping on the OUTPUT buffer
 	 * decoder supports only composing on the CAPTURE buffer
 	 */
-	if ((ctx->is_enc && s->type == valid_out_type) ||
-	    (!ctx->is_enc && s->type == valid_cap_type)) {
+	if (ctx->is_enc && s->type == V4L2_BUF_TYPE_VIDEO_OUTPUT) {
 		switch (s->target) {
-		case V4L2_SEL_TGT_COMPOSE:
 		case V4L2_SEL_TGT_CROP:
 			s->r.left = 0;
 			s->r.top = 0;
 			s->r.width = q_data->visible_width;
 			s->r.height = q_data->visible_height;
 			return 0;
-		case V4L2_SEL_TGT_COMPOSE_DEFAULT:
-		case V4L2_SEL_TGT_COMPOSE_BOUNDS:
 		case V4L2_SEL_TGT_CROP_DEFAULT:
 		case V4L2_SEL_TGT_CROP_BOUNDS:
 			s->r.left = 0;
@@ -983,6 +1126,22 @@ static int vidioc_g_selection(struct file *file, void *priv,
 			s->r.height = q_data->coded_height;
 			return 0;
 		}
+	} else if (!ctx->is_enc && s->type == V4L2_BUF_TYPE_VIDEO_CAPTURE) {
+		switch (s->target) {
+		case V4L2_SEL_TGT_COMPOSE:
+			s->r.left = 0;
+			s->r.top = 0;
+			s->r.width = q_data->visible_width;
+			s->r.height = q_data->visible_height;
+			return 0;
+		case V4L2_SEL_TGT_COMPOSE_DEFAULT:
+		case V4L2_SEL_TGT_COMPOSE_BOUNDS:
+			s->r.left = 0;
+			s->r.top = 0;
+			s->r.width = q_data->coded_width;
+			s->r.height = q_data->coded_height;
+			return 0;
+		}
 	}
 	return -EINVAL;
 }
@@ -992,12 +1151,8 @@ static int vidioc_s_selection(struct file *file, void *priv,
 {
 	struct vicodec_ctx *ctx = file2ctx(file);
 	struct vicodec_q_data *q_data;
-	enum v4l2_buf_type out_type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
-
-	if (multiplanar)
-		out_type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE;
 
-	if (s->type != out_type)
+	if (s->type != V4L2_BUF_TYPE_VIDEO_OUTPUT)
 		return -EINVAL;
 
 	q_data = get_q_data(ctx, s->type);
@@ -1092,6 +1247,8 @@ static int vicodec_enum_framesizes(struct file *file, void *fh,
 				   struct v4l2_frmsizeenum *fsize)
 {
 	switch (fsize->pixel_format) {
+	case V4L2_PIX_FMT_FWHT_STATELESS:
+		break;
 	case V4L2_PIX_FMT_FWHT:
 		break;
 	default:
@@ -1200,6 +1357,14 @@ static int vicodec_queue_setup(struct vb2_queue *vq, unsigned int *nbuffers,
 	return 0;
 }
 
+static int vicodec_buf_out_validate(struct vb2_buffer *vb)
+{
+	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
+
+	vbuf->field = V4L2_FIELD_NONE;
+	return 0;
+}
+
 static int vicodec_buf_prepare(struct vb2_buffer *vb)
 {
 	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
@@ -1263,10 +1428,11 @@ static void vicodec_buf_queue(struct vb2_buffer *vb)
 	}
 
 	/*
-	 * source change event is relevant only for the decoder
+	 * source change event is relevant only for the stateful decoder
 	 * in the compressed stream
 	 */
-	if (ctx->is_enc || !V4L2_TYPE_IS_OUTPUT(vb->vb2_queue->type)) {
+	if (ctx->is_stateless || ctx->is_enc ||
+	    !V4L2_TYPE_IS_OUTPUT(vb->vb2_queue->type)) {
 		v4l2_m2m_buf_queue(ctx->fh.m2m_ctx, vbuf);
 		return;
 	}
@@ -1314,12 +1480,33 @@ static void vicodec_return_bufs(struct vb2_queue *q, u32 state)
 			vbuf = v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx);
 		if (vbuf == NULL)
 			return;
+		v4l2_ctrl_request_complete(vbuf->vb2_buf.req_obj.req,
+					   &ctx->hdl);
 		spin_lock(ctx->lock);
 		v4l2_m2m_buf_done(vbuf, state);
 		spin_unlock(ctx->lock);
 	}
 }
 
+static unsigned int total_frame_size(struct vicodec_q_data *q_data)
+{
+	unsigned int size;
+	unsigned int chroma_div;
+
+	if (!q_data->info) {
+		WARN_ON(1);
+		return 0;
+	}
+	size = q_data->coded_width * q_data->coded_height;
+	chroma_div = q_data->info->width_div * q_data->info->height_div;
+
+	if (q_data->info->components_num == 4)
+		return 2 * size + 2 * (size / chroma_div);
+	else if (q_data->info->components_num == 3)
+		return size + 2 * (size / chroma_div);
+	return size;
+}
+
 static int vicodec_start_streaming(struct vb2_queue *q,
 				   unsigned int count)
 {
@@ -1330,7 +1517,7 @@ static int vicodec_start_streaming(struct vb2_queue *q,
 	unsigned int size = q_data->coded_width * q_data->coded_height;
 	unsigned int chroma_div;
 	unsigned int total_planes_size;
-	u8 *new_comp_frame;
+	u8 *new_comp_frame = NULL;
 
 	if (!info)
 		return -EINVAL;
@@ -1338,24 +1525,24 @@ static int vicodec_start_streaming(struct vb2_queue *q,
 	chroma_div = info->width_div * info->height_div;
 	q_data->sequence = 0;
 
-	ctx->last_src_buf = NULL;
-	ctx->last_dst_buf = NULL;
+	if (V4L2_TYPE_IS_OUTPUT(q->type))
+		ctx->last_src_buf = NULL;
+	else
+		ctx->last_dst_buf = NULL;
+
 	state->gop_cnt = 0;
 
 	if ((V4L2_TYPE_IS_OUTPUT(q->type) && !ctx->is_enc) ||
 	    (!V4L2_TYPE_IS_OUTPUT(q->type) && ctx->is_enc))
 		return 0;
 
-	if (info->id == V4L2_PIX_FMT_FWHT) {
+	if (info->id == V4L2_PIX_FMT_FWHT ||
+	    info->id == V4L2_PIX_FMT_FWHT_STATELESS) {
 		vicodec_return_bufs(q, VB2_BUF_STATE_QUEUED);
 		return -EINVAL;
 	}
-	if (info->components_num == 4)
-		total_planes_size = 2 * size + 2 * (size / chroma_div);
-	else if (info->components_num == 3)
-		total_planes_size = size + 2 * (size / chroma_div);
-	else
-		total_planes_size = size;
+	total_planes_size = total_frame_size(q_data);
+	ctx->comp_max_size = total_planes_size;
 
 	state->visible_width = q_data->visible_width;
 	state->visible_height = q_data->visible_height;
@@ -1364,8 +1551,14 @@ static int vicodec_start_streaming(struct vb2_queue *q,
 	state->stride = q_data->coded_width *
 				info->bytesperline_mult;
 
-	state->ref_frame.luma = kvmalloc(total_planes_size, GFP_KERNEL);
-	ctx->comp_max_size = total_planes_size;
+	if (ctx->is_stateless) {
+		state->ref_stride = state->stride;
+		return 0;
+	}
+	state->ref_stride = q_data->coded_width * info->luma_alpha_step;
+
+	state->ref_frame.buf = kvmalloc(total_planes_size, GFP_KERNEL);
+	state->ref_frame.luma = state->ref_frame.buf;
 	new_comp_frame = kvmalloc(ctx->comp_max_size, GFP_KERNEL);
 
 	if (!state->ref_frame.luma || !new_comp_frame) {
@@ -1413,7 +1606,10 @@ static void vicodec_stop_streaming(struct vb2_queue *q)
 
 	if ((!V4L2_TYPE_IS_OUTPUT(q->type) && !ctx->is_enc) ||
 	    (V4L2_TYPE_IS_OUTPUT(q->type) && ctx->is_enc)) {
-		kvfree(ctx->state.ref_frame.luma);
+		if (!ctx->is_stateless)
+			kvfree(ctx->state.ref_frame.buf);
+		ctx->state.ref_frame.buf = NULL;
+		ctx->state.ref_frame.luma = NULL;
 		ctx->comp_max_size = 0;
 		ctx->source_changed = false;
 	}
@@ -1427,14 +1623,24 @@ static void vicodec_stop_streaming(struct vb2_queue *q)
 	}
 }
 
+static void vicodec_buf_request_complete(struct vb2_buffer *vb)
+{
+	struct vicodec_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue);
+
+	v4l2_ctrl_request_complete(vb->req_obj.req, &ctx->hdl);
+}
+
+
 static const struct vb2_ops vicodec_qops = {
-	.queue_setup	 = vicodec_queue_setup,
-	.buf_prepare	 = vicodec_buf_prepare,
-	.buf_queue	 = vicodec_buf_queue,
-	.start_streaming = vicodec_start_streaming,
-	.stop_streaming  = vicodec_stop_streaming,
-	.wait_prepare	 = vb2_ops_wait_prepare,
-	.wait_finish	 = vb2_ops_wait_finish,
+	.queue_setup		= vicodec_queue_setup,
+	.buf_out_validate	= vicodec_buf_out_validate,
+	.buf_prepare		= vicodec_buf_prepare,
+	.buf_queue		= vicodec_buf_queue,
+	.buf_request_complete	= vicodec_buf_request_complete,
+	.start_streaming	= vicodec_start_streaming,
+	.stop_streaming		= vicodec_stop_streaming,
+	.wait_prepare		= vb2_ops_wait_prepare,
+	.wait_finish		= vb2_ops_wait_finish,
 };
 
 static int queue_init(void *priv, struct vb2_queue *src_vq,
@@ -1452,9 +1658,14 @@ static int queue_init(void *priv, struct vb2_queue *src_vq,
 	src_vq->ops = &vicodec_qops;
 	src_vq->mem_ops = &vb2_vmalloc_memops;
 	src_vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY;
-	src_vq->lock = ctx->is_enc ? &ctx->dev->enc_mutex :
-		&ctx->dev->dec_mutex;
-
+	if (ctx->is_enc)
+		src_vq->lock = &ctx->dev->stateful_enc.mutex;
+	else if (ctx->is_stateless)
+		src_vq->lock = &ctx->dev->stateless_dec.mutex;
+	else
+		src_vq->lock = &ctx->dev->stateful_dec.mutex;
+	src_vq->supports_requests = ctx->is_stateless;
+	src_vq->requires_requests = ctx->is_stateless;
 	ret = vb2_queue_init(src_vq);
 	if (ret)
 		return ret;
@@ -1473,53 +1684,86 @@ static int queue_init(void *priv, struct vb2_queue *src_vq,
 	return vb2_queue_init(dst_vq);
 }
 
-#define VICODEC_CID_CUSTOM_BASE		(V4L2_CID_MPEG_BASE | 0xf000)
-#define VICODEC_CID_I_FRAME_QP		(VICODEC_CID_CUSTOM_BASE + 0)
-#define VICODEC_CID_P_FRAME_QP		(VICODEC_CID_CUSTOM_BASE + 1)
+static int vicodec_try_ctrl(struct v4l2_ctrl *ctrl)
+{
+	struct vicodec_ctx *ctx = container_of(ctrl->handler,
+			struct vicodec_ctx, hdl);
+	const struct v4l2_ctrl_fwht_params *params;
+	struct vicodec_q_data *q_dst = get_q_data(ctx,
+			V4L2_BUF_TYPE_VIDEO_CAPTURE);
+
+	switch (ctrl->id) {
+	case V4L2_CID_MPEG_VIDEO_FWHT_PARAMS:
+		if (!q_dst->info)
+			return -EINVAL;
+		params = ctrl->p_new.p_fwht_params;
+		if (params->width > q_dst->coded_width ||
+		    params->width < MIN_WIDTH ||
+		    params->height > q_dst->coded_height ||
+		    params->height < MIN_HEIGHT)
+			return -EINVAL;
+		if (!validate_by_version(params->flags, params->version))
+			return -EINVAL;
+		if (!validate_stateless_params_flags(params, q_dst->info))
+			return -EINVAL;
+		return 0;
+	default:
+		return 0;
+	}
+	return 0;
+}
+
+static void update_header_from_stateless_params(struct vicodec_ctx *ctx,
+						const struct v4l2_ctrl_fwht_params *params)
+{
+	struct fwht_cframe_hdr *p_hdr = &ctx->state.header;
+
+	p_hdr->magic1 = FWHT_MAGIC1;
+	p_hdr->magic2 = FWHT_MAGIC2;
+	p_hdr->version = htonl(params->version);
+	p_hdr->width = htonl(params->width);
+	p_hdr->height = htonl(params->height);
+	p_hdr->flags = htonl(params->flags);
+	p_hdr->colorspace = htonl(params->colorspace);
+	p_hdr->xfer_func = htonl(params->xfer_func);
+	p_hdr->ycbcr_enc = htonl(params->ycbcr_enc);
+	p_hdr->quantization = htonl(params->quantization);
+}
 
 static int vicodec_s_ctrl(struct v4l2_ctrl *ctrl)
 {
 	struct vicodec_ctx *ctx = container_of(ctrl->handler,
 					       struct vicodec_ctx, hdl);
+	const struct v4l2_ctrl_fwht_params *params;
 
 	switch (ctrl->id) {
 	case V4L2_CID_MPEG_VIDEO_GOP_SIZE:
 		ctx->state.gop_size = ctrl->val;
 		return 0;
-	case VICODEC_CID_I_FRAME_QP:
+	case V4L2_CID_FWHT_I_FRAME_QP:
 		ctx->state.i_frame_qp = ctrl->val;
 		return 0;
-	case VICODEC_CID_P_FRAME_QP:
+	case V4L2_CID_FWHT_P_FRAME_QP:
 		ctx->state.p_frame_qp = ctrl->val;
 		return 0;
+	case V4L2_CID_MPEG_VIDEO_FWHT_PARAMS:
+		params = ctrl->p_new.p_fwht_params;
+		update_header_from_stateless_params(ctx, params);
+		ctx->state.ref_frame_ts = params->backward_ref_ts;
+		return 0;
 	}
 	return -EINVAL;
 }
 
 static const struct v4l2_ctrl_ops vicodec_ctrl_ops = {
 	.s_ctrl = vicodec_s_ctrl,
+	.try_ctrl = vicodec_try_ctrl,
 };
 
-static const struct v4l2_ctrl_config vicodec_ctrl_i_frame = {
-	.ops = &vicodec_ctrl_ops,
-	.id = VICODEC_CID_I_FRAME_QP,
-	.name = "FWHT I-Frame QP Value",
-	.type = V4L2_CTRL_TYPE_INTEGER,
-	.min = 1,
-	.max = 31,
-	.def = 20,
-	.step = 1,
-};
-
-static const struct v4l2_ctrl_config vicodec_ctrl_p_frame = {
-	.ops = &vicodec_ctrl_ops,
-	.id = VICODEC_CID_P_FRAME_QP,
-	.name = "FWHT P-Frame QP Value",
-	.type = V4L2_CTRL_TYPE_INTEGER,
-	.min = 1,
-	.max = 31,
-	.def = 20,
-	.step = 1,
+static const struct v4l2_ctrl_config vicodec_ctrl_stateless_state = {
+	.ops		= &vicodec_ctrl_ops,
+	.id		= V4L2_CID_MPEG_VIDEO_FWHT_PARAMS,
+	.elem_size      = sizeof(struct v4l2_ctrl_fwht_params),
 };
 
 /*
@@ -1542,8 +1786,10 @@ static int vicodec_open(struct file *file)
 		goto open_unlock;
 	}
 
-	if (vfd == &dev->enc_vfd)
+	if (vfd == &dev->stateful_enc.vfd)
 		ctx->is_enc = true;
+	else if (vfd == &dev->stateless_dec.vfd)
+		ctx->is_stateless = true;
 
 	v4l2_fh_init(&ctx->fh, video_devdata(file));
 	file->private_data = &ctx->fh;
@@ -1552,8 +1798,12 @@ static int vicodec_open(struct file *file)
 	v4l2_ctrl_handler_init(hdl, 4);
 	v4l2_ctrl_new_std(hdl, &vicodec_ctrl_ops, V4L2_CID_MPEG_VIDEO_GOP_SIZE,
 			  1, 16, 1, 10);
-	v4l2_ctrl_new_custom(hdl, &vicodec_ctrl_i_frame, NULL);
-	v4l2_ctrl_new_custom(hdl, &vicodec_ctrl_p_frame, NULL);
+	v4l2_ctrl_new_std(hdl, &vicodec_ctrl_ops, V4L2_CID_FWHT_I_FRAME_QP,
+			  1, 31, 1, 20);
+	v4l2_ctrl_new_std(hdl, &vicodec_ctrl_ops, V4L2_CID_FWHT_P_FRAME_QP,
+			  1, 31, 1, 20);
+	if (ctx->is_stateless)
+		v4l2_ctrl_new_custom(hdl, &vicodec_ctrl_stateless_state, NULL);
 	if (hdl->error) {
 		rc = hdl->error;
 		v4l2_ctrl_handler_free(hdl);
@@ -1563,15 +1813,19 @@ static int vicodec_open(struct file *file)
 	ctx->fh.ctrl_handler = hdl;
 	v4l2_ctrl_handler_setup(hdl);
 
-	ctx->q_data[V4L2_M2M_SRC].info =
-		ctx->is_enc ? v4l2_fwht_get_pixfmt(0) : &pixfmt_fwht;
+	if (ctx->is_enc)
+		ctx->q_data[V4L2_M2M_SRC].info = v4l2_fwht_get_pixfmt(0);
+	else if (ctx->is_stateless)
+		ctx->q_data[V4L2_M2M_SRC].info = &pixfmt_stateless_fwht;
+	else
+		ctx->q_data[V4L2_M2M_SRC].info = &pixfmt_fwht;
 	ctx->q_data[V4L2_M2M_SRC].coded_width = 1280;
 	ctx->q_data[V4L2_M2M_SRC].coded_height = 720;
 	ctx->q_data[V4L2_M2M_SRC].visible_width = 1280;
 	ctx->q_data[V4L2_M2M_SRC].visible_height = 720;
 	size = 1280 * 720 * ctx->q_data[V4L2_M2M_SRC].info->sizeimage_mult /
 		ctx->q_data[V4L2_M2M_SRC].info->sizeimage_div;
-	if (ctx->is_enc)
+	if (ctx->is_enc || ctx->is_stateless)
 		ctx->q_data[V4L2_M2M_SRC].sizeimage = size;
 	else
 		ctx->q_data[V4L2_M2M_SRC].sizeimage =
@@ -1590,13 +1844,17 @@ static int vicodec_open(struct file *file)
 	ctx->state.colorspace = V4L2_COLORSPACE_REC709;
 
 	if (ctx->is_enc) {
-		ctx->fh.m2m_ctx = v4l2_m2m_ctx_init(dev->enc_dev, ctx,
-						    &queue_init);
-		ctx->lock = &dev->enc_lock;
+		ctx->fh.m2m_ctx = v4l2_m2m_ctx_init(dev->stateful_enc.m2m_dev,
+						    ctx, &queue_init);
+		ctx->lock = &dev->stateful_enc.lock;
+	} else if (ctx->is_stateless) {
+		ctx->fh.m2m_ctx = v4l2_m2m_ctx_init(dev->stateless_dec.m2m_dev,
+						    ctx, &queue_init);
+		ctx->lock = &dev->stateless_dec.lock;
 	} else {
-		ctx->fh.m2m_ctx = v4l2_m2m_ctx_init(dev->dec_dev, ctx,
-						    &queue_init);
-		ctx->lock = &dev->dec_lock;
+		ctx->fh.m2m_ctx = v4l2_m2m_ctx_init(dev->stateful_dec.m2m_dev,
+						    ctx, &queue_init);
+		ctx->lock = &dev->stateful_dec.lock;
 	}
 
 	if (IS_ERR(ctx->fh.m2m_ctx)) {
@@ -1620,17 +1878,71 @@ static int vicodec_release(struct file *file)
 	struct video_device *vfd = video_devdata(file);
 	struct vicodec_ctx *ctx = file2ctx(file);
 
-	v4l2_fh_del(&ctx->fh);
-	v4l2_fh_exit(&ctx->fh);
-	v4l2_ctrl_handler_free(&ctx->hdl);
 	mutex_lock(vfd->lock);
 	v4l2_m2m_ctx_release(ctx->fh.m2m_ctx);
 	mutex_unlock(vfd->lock);
+	v4l2_fh_del(&ctx->fh);
+	v4l2_fh_exit(&ctx->fh);
+	v4l2_ctrl_handler_free(&ctx->hdl);
+	kvfree(ctx->state.compressed_frame);
 	kfree(ctx);
 
 	return 0;
 }
 
+static int vicodec_request_validate(struct media_request *req)
+{
+	struct media_request_object *obj;
+	struct v4l2_ctrl_handler *parent_hdl, *hdl;
+	struct vicodec_ctx *ctx = NULL;
+	struct v4l2_ctrl *ctrl;
+	unsigned int count;
+
+	list_for_each_entry(obj, &req->objects, list) {
+		struct vb2_buffer *vb;
+
+		if (vb2_request_object_is_buffer(obj)) {
+			vb = container_of(obj, struct vb2_buffer, req_obj);
+			ctx = vb2_get_drv_priv(vb->vb2_queue);
+
+			break;
+		}
+	}
+
+	if (!ctx) {
+		pr_err("No buffer was provided with the request\n");
+		return -ENOENT;
+	}
+
+	count = vb2_request_buffer_cnt(req);
+	if (!count) {
+		v4l2_info(&ctx->dev->v4l2_dev,
+			  "No buffer was provided with the request\n");
+		return -ENOENT;
+	} else if (count > 1) {
+		v4l2_info(&ctx->dev->v4l2_dev,
+			  "More than one buffer was provided with the request\n");
+		return -EINVAL;
+	}
+
+	parent_hdl = &ctx->hdl;
+
+	hdl = v4l2_ctrl_request_hdl_find(req, parent_hdl);
+	if (!hdl) {
+		v4l2_info(&ctx->dev->v4l2_dev, "Missing codec control\n");
+		return -ENOENT;
+	}
+	ctrl = v4l2_ctrl_request_hdl_ctrl_find(hdl,
+					       vicodec_ctrl_stateless_state.id);
+	if (!ctrl) {
+		v4l2_info(&ctx->dev->v4l2_dev,
+			  "Missing required codec control\n");
+		return -ENOENT;
+	}
+
+	return vb2_request_validate(req);
+}
+
 static const struct v4l2_file_operations vicodec_fops = {
 	.owner		= THIS_MODULE,
 	.open		= vicodec_open,
@@ -1649,24 +1961,67 @@ static const struct video_device vicodec_videodev = {
 	.release	= video_device_release_empty,
 };
 
+static const struct media_device_ops vicodec_m2m_media_ops = {
+	.req_validate	= vicodec_request_validate,
+	.req_queue	= v4l2_m2m_request_queue,
+};
+
 static const struct v4l2_m2m_ops m2m_ops = {
 	.device_run	= device_run,
 	.job_ready	= job_ready,
 };
 
+static int register_instance(struct vicodec_dev *dev,
+			     struct vicodec_dev_instance *dev_instance,
+			     const char *name, bool is_enc)
+{
+	struct video_device *vfd;
+	int ret;
+
+	spin_lock_init(&dev_instance->lock);
+	mutex_init(&dev_instance->mutex);
+	dev_instance->m2m_dev = v4l2_m2m_init(&m2m_ops);
+	if (IS_ERR(dev_instance->m2m_dev)) {
+		v4l2_err(&dev->v4l2_dev, "Failed to init vicodec enc device\n");
+		return PTR_ERR(dev_instance->m2m_dev);
+	}
+
+	dev_instance->vfd = vicodec_videodev;
+	vfd = &dev_instance->vfd;
+	vfd->lock = &dev_instance->mutex;
+	vfd->v4l2_dev = &dev->v4l2_dev;
+	strscpy(vfd->name, name, sizeof(vfd->name));
+	vfd->device_caps = V4L2_CAP_STREAMING |
+		(multiplanar ? V4L2_CAP_VIDEO_M2M_MPLANE : V4L2_CAP_VIDEO_M2M);
+	if (is_enc) {
+		v4l2_disable_ioctl(vfd, VIDIOC_DECODER_CMD);
+		v4l2_disable_ioctl(vfd, VIDIOC_TRY_DECODER_CMD);
+	} else {
+		v4l2_disable_ioctl(vfd, VIDIOC_ENCODER_CMD);
+		v4l2_disable_ioctl(vfd, VIDIOC_TRY_ENCODER_CMD);
+	}
+	video_set_drvdata(vfd, dev);
+
+	ret = video_register_device(vfd, VFL_TYPE_GRABBER, 0);
+	if (ret) {
+		v4l2_err(&dev->v4l2_dev, "Failed to register video device '%s'\n", name);
+		v4l2_m2m_release(dev_instance->m2m_dev);
+		return ret;
+	}
+	v4l2_info(&dev->v4l2_dev, "Device '%s' registered as /dev/video%d\n",
+		  name, vfd->num);
+	return 0;
+}
+
 static int vicodec_probe(struct platform_device *pdev)
 {
 	struct vicodec_dev *dev;
-	struct video_device *vfd;
 	int ret;
 
 	dev = devm_kzalloc(&pdev->dev, sizeof(*dev), GFP_KERNEL);
 	if (!dev)
 		return -ENOMEM;
 
-	spin_lock_init(&dev->enc_lock);
-	spin_lock_init(&dev->dec_lock);
-
 	ret = v4l2_device_register(&pdev->dev, &dev->v4l2_dev);
 	if (ret)
 		return ret;
@@ -1677,103 +2032,74 @@ static int vicodec_probe(struct platform_device *pdev)
 	strscpy(dev->mdev.bus_info, "platform:vicodec",
 		sizeof(dev->mdev.bus_info));
 	media_device_init(&dev->mdev);
+	dev->mdev.ops = &vicodec_m2m_media_ops;
 	dev->v4l2_dev.mdev = &dev->mdev;
 #endif
 
-	mutex_init(&dev->enc_mutex);
-	mutex_init(&dev->dec_mutex);
-
 	platform_set_drvdata(pdev, dev);
 
-	dev->enc_dev = v4l2_m2m_init(&m2m_ops);
-	if (IS_ERR(dev->enc_dev)) {
-		v4l2_err(&dev->v4l2_dev, "Failed to init vicodec device\n");
-		ret = PTR_ERR(dev->enc_dev);
+	if (register_instance(dev, &dev->stateful_enc,
+			      "stateful-encoder", true))
 		goto unreg_dev;
-	}
 
-	dev->dec_dev = v4l2_m2m_init(&m2m_ops);
-	if (IS_ERR(dev->dec_dev)) {
-		v4l2_err(&dev->v4l2_dev, "Failed to init vicodec device\n");
-		ret = PTR_ERR(dev->dec_dev);
-		goto err_enc_m2m;
-	}
+	if (register_instance(dev, &dev->stateful_dec,
+			      "stateful-decoder", false))
+		goto unreg_sf_enc;
 
-	dev->enc_vfd = vicodec_videodev;
-	vfd = &dev->enc_vfd;
-	vfd->lock = &dev->enc_mutex;
-	vfd->v4l2_dev = &dev->v4l2_dev;
-	strscpy(vfd->name, "vicodec-enc", sizeof(vfd->name));
-	vfd->device_caps = V4L2_CAP_STREAMING |
-		(multiplanar ? V4L2_CAP_VIDEO_M2M_MPLANE : V4L2_CAP_VIDEO_M2M);
-	v4l2_disable_ioctl(vfd, VIDIOC_DECODER_CMD);
-	v4l2_disable_ioctl(vfd, VIDIOC_TRY_DECODER_CMD);
-	video_set_drvdata(vfd, dev);
-
-	ret = video_register_device(vfd, VFL_TYPE_GRABBER, 0);
-	if (ret) {
-		v4l2_err(&dev->v4l2_dev, "Failed to register video device\n");
-		goto err_dec_m2m;
-	}
-	v4l2_info(&dev->v4l2_dev,
-			"Device registered as /dev/video%d\n", vfd->num);
+	if (register_instance(dev, &dev->stateless_dec,
+			      "stateless-decoder", false))
+		goto unreg_sf_dec;
 
-	dev->dec_vfd = vicodec_videodev;
-	vfd = &dev->dec_vfd;
-	vfd->lock = &dev->dec_mutex;
-	vfd->v4l2_dev = &dev->v4l2_dev;
-	vfd->device_caps = V4L2_CAP_STREAMING |
-		(multiplanar ? V4L2_CAP_VIDEO_M2M_MPLANE : V4L2_CAP_VIDEO_M2M);
-	strscpy(vfd->name, "vicodec-dec", sizeof(vfd->name));
-	v4l2_disable_ioctl(vfd, VIDIOC_ENCODER_CMD);
-	v4l2_disable_ioctl(vfd, VIDIOC_TRY_ENCODER_CMD);
-	video_set_drvdata(vfd, dev);
-
-	ret = video_register_device(vfd, VFL_TYPE_GRABBER, 0);
+#ifdef CONFIG_MEDIA_CONTROLLER
+	ret = v4l2_m2m_register_media_controller(dev->stateful_enc.m2m_dev,
+						 &dev->stateful_enc.vfd,
+						 MEDIA_ENT_F_PROC_VIDEO_ENCODER);
 	if (ret) {
-		v4l2_err(&dev->v4l2_dev, "Failed to register video device\n");
-		goto unreg_enc;
+		v4l2_err(&dev->v4l2_dev, "Failed to init mem2mem media controller for enc\n");
+		goto unreg_m2m;
 	}
-	v4l2_info(&dev->v4l2_dev,
-			"Device registered as /dev/video%d\n", vfd->num);
 
-#ifdef CONFIG_MEDIA_CONTROLLER
-	ret = v4l2_m2m_register_media_controller(dev->enc_dev,
-			&dev->enc_vfd, MEDIA_ENT_F_PROC_VIDEO_ENCODER);
+	ret = v4l2_m2m_register_media_controller(dev->stateful_dec.m2m_dev,
+						 &dev->stateful_dec.vfd,
+						 MEDIA_ENT_F_PROC_VIDEO_DECODER);
 	if (ret) {
-		v4l2_err(&dev->v4l2_dev, "Failed to init mem2mem media controller\n");
-		goto unreg_m2m;
+		v4l2_err(&dev->v4l2_dev, "Failed to init mem2mem media controller for dec\n");
+		goto unreg_m2m_sf_enc_mc;
 	}
 
-	ret = v4l2_m2m_register_media_controller(dev->dec_dev,
-			&dev->dec_vfd, MEDIA_ENT_F_PROC_VIDEO_DECODER);
+	ret = v4l2_m2m_register_media_controller(dev->stateless_dec.m2m_dev,
+						 &dev->stateless_dec.vfd,
+						 MEDIA_ENT_F_PROC_VIDEO_DECODER);
 	if (ret) {
-		v4l2_err(&dev->v4l2_dev, "Failed to init mem2mem media controller\n");
-		goto unreg_m2m_enc_mc;
+		v4l2_err(&dev->v4l2_dev, "Failed to init mem2mem media controller for stateless dec\n");
+		goto unreg_m2m_sf_dec_mc;
 	}
 
 	ret = media_device_register(&dev->mdev);
 	if (ret) {
 		v4l2_err(&dev->v4l2_dev, "Failed to register mem2mem media device\n");
-		goto unreg_m2m_dec_mc;
+		goto unreg_m2m_sl_dec_mc;
 	}
 #endif
 	return 0;
 
 #ifdef CONFIG_MEDIA_CONTROLLER
-unreg_m2m_dec_mc:
-	v4l2_m2m_unregister_media_controller(dev->dec_dev);
-unreg_m2m_enc_mc:
-	v4l2_m2m_unregister_media_controller(dev->enc_dev);
+unreg_m2m_sl_dec_mc:
+	v4l2_m2m_unregister_media_controller(dev->stateless_dec.m2m_dev);
+unreg_m2m_sf_dec_mc:
+	v4l2_m2m_unregister_media_controller(dev->stateful_dec.m2m_dev);
+unreg_m2m_sf_enc_mc:
+	v4l2_m2m_unregister_media_controller(dev->stateful_enc.m2m_dev);
 unreg_m2m:
-	video_unregister_device(&dev->dec_vfd);
+	video_unregister_device(&dev->stateless_dec.vfd);
+	v4l2_m2m_release(dev->stateless_dec.m2m_dev);
 #endif
-unreg_enc:
-	video_unregister_device(&dev->enc_vfd);
-err_dec_m2m:
-	v4l2_m2m_release(dev->dec_dev);
-err_enc_m2m:
-	v4l2_m2m_release(dev->enc_dev);
+unreg_sf_dec:
+	video_unregister_device(&dev->stateful_dec.vfd);
+	v4l2_m2m_release(dev->stateful_dec.m2m_dev);
+unreg_sf_enc:
+	video_unregister_device(&dev->stateful_enc.vfd);
+	v4l2_m2m_release(dev->stateful_enc.m2m_dev);
 unreg_dev:
 	v4l2_device_unregister(&dev->v4l2_dev);
 
@@ -1788,15 +2114,17 @@ static int vicodec_remove(struct platform_device *pdev)
 
 #ifdef CONFIG_MEDIA_CONTROLLER
 	media_device_unregister(&dev->mdev);
-	v4l2_m2m_unregister_media_controller(dev->enc_dev);
-	v4l2_m2m_unregister_media_controller(dev->dec_dev);
+	v4l2_m2m_unregister_media_controller(dev->stateful_enc.m2m_dev);
+	v4l2_m2m_unregister_media_controller(dev->stateful_dec.m2m_dev);
+	v4l2_m2m_unregister_media_controller(dev->stateless_dec.m2m_dev);
 	media_device_cleanup(&dev->mdev);
 #endif
 
-	v4l2_m2m_release(dev->enc_dev);
-	v4l2_m2m_release(dev->dec_dev);
-	video_unregister_device(&dev->enc_vfd);
-	video_unregister_device(&dev->dec_vfd);
+	v4l2_m2m_release(dev->stateful_enc.m2m_dev);
+	v4l2_m2m_release(dev->stateful_dec.m2m_dev);
+	video_unregister_device(&dev->stateful_enc.vfd);
+	video_unregister_device(&dev->stateful_dec.vfd);
+	video_unregister_device(&dev->stateless_dec.vfd);
 	v4l2_device_unregister(&dev->v4l2_dev);
 
 	return 0;
diff --git a/drivers/media/platform/video-mux.c b/drivers/media/platform/video-mux.c
index 0ba30756e1e4..d8cd5f5cb10d 100644
--- a/drivers/media/platform/video-mux.c
+++ b/drivers/media/platform/video-mux.c
@@ -419,9 +419,14 @@ static int video_mux_probe(struct platform_device *pdev)
 	vmux->active = -1;
 	vmux->pads = devm_kcalloc(dev, num_pads, sizeof(*vmux->pads),
 				  GFP_KERNEL);
+	if (!vmux->pads)
+		return -ENOMEM;
+
 	vmux->format_mbus = devm_kcalloc(dev, num_pads,
 					 sizeof(*vmux->format_mbus),
 					 GFP_KERNEL);
+	if (!vmux->format_mbus)
+		return -ENOMEM;
 
 	for (i = 0; i < num_pads; i++) {
 		vmux->pads[i].flags = (i < num_pads - 1) ? MEDIA_PAD_FL_SINK
diff --git a/drivers/media/platform/vim2m.c b/drivers/media/platform/vim2m.c
index 34dcaca45d8b..243c82b5d537 100644
--- a/drivers/media/platform/vim2m.c
+++ b/drivers/media/platform/vim2m.c
@@ -302,7 +302,7 @@ static void copy_two_pixels(struct vim2m_q_data *q_data_in,
 	switch (in->fourcc) {
 	case V4L2_PIX_FMT_RGB565: /* rrrrrggg gggbbbbb */
 		for (i = 0; i < 2; i++) {
-			u16 pix = *(u16 *)(src[i]);
+			u16 pix = le16_to_cpu(*(__le16 *)(src[i]));
 
 			*r++ = (u8)(((pix & 0xf800) >> 11) << 3) | 0x07;
 			*g++ = (u8)((((pix & 0x07e0) >> 5)) << 2) | 0x03;
@@ -311,12 +311,11 @@ static void copy_two_pixels(struct vim2m_q_data *q_data_in,
 		break;
 	case V4L2_PIX_FMT_RGB565X: /* gggbbbbb rrrrrggg */
 		for (i = 0; i < 2; i++) {
-			u16 pix = *(u16 *)(src[i]);
+			u16 pix = be16_to_cpu(*(__be16 *)(src[i]));
 
-			*r++ = (u8)(((0x00f8 & pix) >> 3) << 3) | 0x07;
-			*g++ = (u8)(((pix & 0x7) << 2) |
-				    ((pix & 0xe000) >> 5)) | 0x03;
-			*b++ = (u8)(((pix & 0x1f00) >> 8) << 3) | 0x07;
+			*r++ = (u8)(((pix & 0xf800) >> 11) << 3) | 0x07;
+			*g++ = (u8)((((pix & 0x07e0) >> 5)) << 2) | 0x03;
+			*b++ = (u8)((pix & 0x1f) << 3) | 0x07;
 		}
 		break;
 	default:
@@ -345,21 +344,26 @@ static void copy_two_pixels(struct vim2m_q_data *q_data_in,
 	switch (out->fourcc) {
 	case V4L2_PIX_FMT_RGB565: /* rrrrrggg gggbbbbb */
 		for (i = 0; i < 2; i++) {
-			u16 *pix = (u16 *)*dst;
+			u16 pix;
+			__le16 *dst_pix = (__le16 *)*dst;
+
+			pix = ((*r << 8) & 0xf800) | ((*g << 3) & 0x07e0) |
+			      (*b >> 3);
 
-			*pix = ((*r << 8) & 0xf800) | ((*g << 3) & 0x07e0) |
-			       (*b >> 3);
+			*dst_pix = cpu_to_le16(pix);
 
 			*dst += 2;
 		}
 		return;
 	case V4L2_PIX_FMT_RGB565X: /* gggbbbbb rrrrrggg */
 		for (i = 0; i < 2; i++) {
-			u16 *pix = (u16 *)*dst;
-			u8 green = *g++ >> 2;
+			u16 pix;
+			__be16 *dst_pix = (__be16 *)*dst;
 
-			*pix = ((green << 8) & 0xe000) | (green & 0x07) |
-			       ((*b++ << 5) & 0x1f00) | ((*r++ & 0xf8));
+			pix = ((*r << 8) & 0xf800) | ((*g << 3) & 0x07e0) |
+			      (*b >> 3);
+
+			*dst_pix = cpu_to_be16(pix);
 
 			*dst += 2;
 		}
@@ -655,8 +659,8 @@ static void device_work(struct work_struct *w)
 static int vidioc_querycap(struct file *file, void *priv,
 			   struct v4l2_capability *cap)
 {
-	strncpy(cap->driver, MEM2MEM_NAME, sizeof(cap->driver) - 1);
-	strncpy(cap->card, MEM2MEM_NAME, sizeof(cap->card) - 1);
+	strscpy(cap->driver, MEM2MEM_NAME, sizeof(cap->driver));
+	strscpy(cap->card, MEM2MEM_NAME, sizeof(cap->card));
 	snprintf(cap->bus_info, sizeof(cap->bus_info),
 		 "platform:%s", MEM2MEM_NAME);
 	return 0;
@@ -1262,6 +1266,15 @@ static int vim2m_release(struct file *file)
 	return 0;
 }
 
+static void vim2m_device_release(struct video_device *vdev)
+{
+	struct vim2m_dev *dev = container_of(vdev, struct vim2m_dev, vfd);
+
+	v4l2_device_unregister(&dev->v4l2_dev);
+	v4l2_m2m_release(dev->m2m_dev);
+	kfree(dev);
+}
+
 static const struct v4l2_file_operations vim2m_fops = {
 	.owner		= THIS_MODULE,
 	.open		= vim2m_open,
@@ -1277,7 +1290,7 @@ static const struct video_device vim2m_videodev = {
 	.fops		= &vim2m_fops,
 	.ioctl_ops	= &vim2m_ioctl_ops,
 	.minor		= -1,
-	.release	= video_device_release_empty,
+	.release	= vim2m_device_release,
 	.device_caps	= V4L2_CAP_VIDEO_M2M | V4L2_CAP_STREAMING,
 };
 
@@ -1298,13 +1311,13 @@ static int vim2m_probe(struct platform_device *pdev)
 	struct video_device *vfd;
 	int ret;
 
-	dev = devm_kzalloc(&pdev->dev, sizeof(*dev), GFP_KERNEL);
+	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
 	if (!dev)
 		return -ENOMEM;
 
 	ret = v4l2_device_register(&pdev->dev, &dev->v4l2_dev);
 	if (ret)
-		return ret;
+		goto error_free;
 
 	atomic_set(&dev->num_inst, 0);
 	mutex_init(&dev->dev_mutex);
@@ -1317,7 +1330,7 @@ static int vim2m_probe(struct platform_device *pdev)
 	ret = video_register_device(vfd, VFL_TYPE_GRABBER, 0);
 	if (ret) {
 		v4l2_err(&dev->v4l2_dev, "Failed to register video device\n");
-		goto unreg_v4l2;
+		goto error_v4l2;
 	}
 
 	video_set_drvdata(vfd, dev);
@@ -1330,7 +1343,7 @@ static int vim2m_probe(struct platform_device *pdev)
 	if (IS_ERR(dev->m2m_dev)) {
 		v4l2_err(&dev->v4l2_dev, "Failed to init mem2mem device\n");
 		ret = PTR_ERR(dev->m2m_dev);
-		goto unreg_dev;
+		goto error_dev;
 	}
 
 #ifdef CONFIG_MEDIA_CONTROLLER
@@ -1346,27 +1359,29 @@ static int vim2m_probe(struct platform_device *pdev)
 						 MEDIA_ENT_F_PROC_VIDEO_SCALER);
 	if (ret) {
 		v4l2_err(&dev->v4l2_dev, "Failed to init mem2mem media controller\n");
-		goto unreg_m2m;
+		goto error_m2m;
 	}
 
 	ret = media_device_register(&dev->mdev);
 	if (ret) {
 		v4l2_err(&dev->v4l2_dev, "Failed to register mem2mem media device\n");
-		goto unreg_m2m_mc;
+		goto error_m2m_mc;
 	}
 #endif
 	return 0;
 
 #ifdef CONFIG_MEDIA_CONTROLLER
-unreg_m2m_mc:
+error_m2m_mc:
 	v4l2_m2m_unregister_media_controller(dev->m2m_dev);
-unreg_m2m:
+error_m2m:
 	v4l2_m2m_release(dev->m2m_dev);
 #endif
-unreg_dev:
+error_dev:
 	video_unregister_device(&dev->vfd);
-unreg_v4l2:
+error_v4l2:
 	v4l2_device_unregister(&dev->v4l2_dev);
+error_free:
+	kfree(dev);
 
 	return ret;
 }
@@ -1382,9 +1397,7 @@ static int vim2m_remove(struct platform_device *pdev)
 	v4l2_m2m_unregister_media_controller(dev->m2m_dev);
 	media_device_cleanup(&dev->mdev);
 #endif
-	v4l2_m2m_release(dev->m2m_dev);
 	video_unregister_device(&dev->vfd);
-	v4l2_device_unregister(&dev->v4l2_dev);
 
 	return 0;
 }
diff --git a/drivers/media/platform/vimc/Kconfig b/drivers/media/platform/vimc/Kconfig
index 71c9fe7d3370..1de9bc9aa49b 100644
--- a/drivers/media/platform/vimc/Kconfig
+++ b/drivers/media/platform/vimc/Kconfig
@@ -4,7 +4,7 @@ config VIDEO_VIMC
 	select VIDEOBUF2_VMALLOC
 	select VIDEO_V4L2_TPG
 	default n
-	---help---
+	help
 	  Skeleton driver for Virtual Media Controller
 
 	  This driver can be compared to the vivid driver for emulating
diff --git a/drivers/media/platform/vimc/vimc-capture.c b/drivers/media/platform/vimc/vimc-capture.c
index ea869631a3f6..e7d0fc2228a6 100644
--- a/drivers/media/platform/vimc/vimc-capture.c
+++ b/drivers/media/platform/vimc/vimc-capture.c
@@ -28,6 +28,32 @@
 
 #define VIMC_CAP_DRV_NAME "vimc-capture"
 
+static const u32 vimc_cap_supported_pixfmt[] = {
+	V4L2_PIX_FMT_BGR24,
+	V4L2_PIX_FMT_RGB24,
+	V4L2_PIX_FMT_ARGB32,
+	V4L2_PIX_FMT_SBGGR8,
+	V4L2_PIX_FMT_SGBRG8,
+	V4L2_PIX_FMT_SGRBG8,
+	V4L2_PIX_FMT_SRGGB8,
+	V4L2_PIX_FMT_SBGGR10,
+	V4L2_PIX_FMT_SGBRG10,
+	V4L2_PIX_FMT_SGRBG10,
+	V4L2_PIX_FMT_SRGGB10,
+	V4L2_PIX_FMT_SBGGR10ALAW8,
+	V4L2_PIX_FMT_SGBRG10ALAW8,
+	V4L2_PIX_FMT_SGRBG10ALAW8,
+	V4L2_PIX_FMT_SRGGB10ALAW8,
+	V4L2_PIX_FMT_SBGGR10DPCM8,
+	V4L2_PIX_FMT_SGBRG10DPCM8,
+	V4L2_PIX_FMT_SGRBG10DPCM8,
+	V4L2_PIX_FMT_SRGGB10DPCM8,
+	V4L2_PIX_FMT_SBGGR12,
+	V4L2_PIX_FMT_SGBRG12,
+	V4L2_PIX_FMT_SGRBG12,
+	V4L2_PIX_FMT_SRGGB12,
+};
+
 struct vimc_cap_device {
 	struct vimc_ent_device ved;
 	struct video_device vdev;
@@ -101,29 +127,25 @@ static int vimc_cap_try_fmt_vid_cap(struct file *file, void *priv,
 				    struct v4l2_format *f)
 {
 	struct v4l2_pix_format *format = &f->fmt.pix;
-	const struct vimc_pix_map *vpix;
 
 	format->width = clamp_t(u32, format->width, VIMC_FRAME_MIN_WIDTH,
 				VIMC_FRAME_MAX_WIDTH) & ~1;
 	format->height = clamp_t(u32, format->height, VIMC_FRAME_MIN_HEIGHT,
 				 VIMC_FRAME_MAX_HEIGHT) & ~1;
 
-	/* Don't accept a pixelformat that is not on the table */
-	vpix = vimc_pix_map_by_pixelformat(format->pixelformat);
-	if (!vpix) {
-		format->pixelformat = fmt_default.pixelformat;
-		vpix = vimc_pix_map_by_pixelformat(format->pixelformat);
-	}
-	/* TODO: Add support for custom bytesperline values */
-	format->bytesperline = format->width * vpix->bpp;
-	format->sizeimage = format->bytesperline * format->height;
+	vimc_colorimetry_clamp(format);
 
 	if (format->field == V4L2_FIELD_ANY)
 		format->field = fmt_default.field;
 
-	vimc_colorimetry_clamp(format);
+	/* TODO: Add support for custom bytesperline values */
 
-	return 0;
+	/* Don't accept a pixelformat that is not on the table */
+	if (!v4l2_format_info(format->pixelformat))
+		format->pixelformat = fmt_default.pixelformat;
+
+	return v4l2_fill_pixfmt(format, format->pixelformat,
+				format->width, format->height);
 }
 
 static int vimc_cap_s_fmt_vid_cap(struct file *file, void *priv,
@@ -159,27 +181,31 @@ static int vimc_cap_s_fmt_vid_cap(struct file *file, void *priv,
 static int vimc_cap_enum_fmt_vid_cap(struct file *file, void *priv,
 				     struct v4l2_fmtdesc *f)
 {
-	const struct vimc_pix_map *vpix = vimc_pix_map_by_index(f->index);
-
-	if (!vpix)
+	if (f->index >= ARRAY_SIZE(vimc_cap_supported_pixfmt))
 		return -EINVAL;
 
-	f->pixelformat = vpix->pixelformat;
+	f->pixelformat = vimc_cap_supported_pixfmt[f->index];
 
 	return 0;
 }
 
+static bool vimc_cap_is_pixfmt_supported(u32 pixelformat)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(vimc_cap_supported_pixfmt); i++)
+		if (vimc_cap_supported_pixfmt[i] == pixelformat)
+			return true;
+	return false;
+}
+
 static int vimc_cap_enum_framesizes(struct file *file, void *fh,
 				    struct v4l2_frmsizeenum *fsize)
 {
-	const struct vimc_pix_map *vpix;
-
 	if (fsize->index)
 		return -EINVAL;
 
-	/* Only accept code in the pix map table */
-	vpix = vimc_pix_map_by_code(fsize->pixel_format);
-	if (!vpix)
+	if (!vimc_cap_is_pixfmt_supported(fsize->pixel_format))
 		return -EINVAL;
 
 	fsize->type = V4L2_FRMSIZE_TYPE_CONTINUOUS;
@@ -187,8 +213,8 @@ static int vimc_cap_enum_framesizes(struct file *file, void *fh,
 	fsize->stepwise.max_width = VIMC_FRAME_MAX_WIDTH;
 	fsize->stepwise.min_height = VIMC_FRAME_MIN_HEIGHT;
 	fsize->stepwise.max_height = VIMC_FRAME_MAX_HEIGHT;
-	fsize->stepwise.step_width = 2;
-	fsize->stepwise.step_height = 2;
+	fsize->stepwise.step_width = 1;
+	fsize->stepwise.step_height = 1;
 
 	return 0;
 }
@@ -253,6 +279,7 @@ static int vimc_cap_start_streaming(struct vb2_queue *vq, unsigned int count)
 		return ret;
 	}
 
+	vcap->stream.producer_pixfmt = vcap->format.pixelformat;
 	ret = vimc_streamer_s_stream(&vcap->stream, &vcap->ved, 1);
 	if (ret) {
 		media_pipeline_stop(entity);
@@ -338,6 +365,15 @@ static const struct media_entity_operations vimc_cap_mops = {
 	.link_validate		= vimc_link_validate,
 };
 
+static void vimc_cap_release(struct video_device *vdev)
+{
+	struct vimc_cap_device *vcap =
+		container_of(vdev, struct vimc_cap_device, vdev);
+
+	vimc_pads_cleanup(vcap->ved.pads);
+	kfree(vcap);
+}
+
 static void vimc_cap_comp_unbind(struct device *comp, struct device *master,
 				 void *master_data)
 {
@@ -348,8 +384,6 @@ static void vimc_cap_comp_unbind(struct device *comp, struct device *master,
 	vb2_queue_release(&vcap->queue);
 	media_entity_cleanup(ved->ent);
 	video_unregister_device(&vcap->vdev);
-	vimc_pads_cleanup(vcap->ved.pads);
-	kfree(vcap);
 }
 
 static void *vimc_cap_process_frame(struct vimc_ent_device *ved,
@@ -396,7 +430,6 @@ static int vimc_cap_comp_bind(struct device *comp, struct device *master,
 {
 	struct v4l2_device *v4l2_dev = master_data;
 	struct vimc_platform_data *pdata = comp->platform_data;
-	const struct vimc_pix_map *vpix;
 	struct vimc_cap_device *vcap;
 	struct video_device *vdev;
 	struct vb2_queue *q;
@@ -451,10 +484,8 @@ static int vimc_cap_comp_bind(struct device *comp, struct device *master,
 
 	/* Set default frame format */
 	vcap->format = fmt_default;
-	vpix = vimc_pix_map_by_pixelformat(vcap->format.pixelformat);
-	vcap->format.bytesperline = vcap->format.width * vpix->bpp;
-	vcap->format.sizeimage = vcap->format.bytesperline *
-				 vcap->format.height;
+	v4l2_fill_pixfmt(&vcap->format, vcap->format.pixelformat,
+			 vcap->format.width, vcap->format.height);
 
 	/* Fill the vimc_ent_device struct */
 	vcap->ved.ent = &vcap->vdev.entity;
@@ -467,7 +498,7 @@ static int vimc_cap_comp_bind(struct device *comp, struct device *master,
 	vdev = &vcap->vdev;
 	vdev->device_caps = V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_STREAMING;
 	vdev->entity.ops = &vimc_cap_mops;
-	vdev->release = video_device_release_empty;
+	vdev->release = vimc_cap_release;
 	vdev->fops = &vimc_cap_fops;
 	vdev->ioctl_ops = &vimc_cap_ioctl_ops;
 	vdev->lock = &vcap->lock;
diff --git a/drivers/media/platform/vimc/vimc-common.c b/drivers/media/platform/vimc/vimc-common.c
index c1a74bb2df58..fad7c7c6de93 100644
--- a/drivers/media/platform/vimc/vimc-common.c
+++ b/drivers/media/platform/vimc/vimc-common.c
@@ -20,192 +20,139 @@
 
 #include "vimc-common.h"
 
-/*
- * NOTE: non-bayer formats need to come first (necessary for enum_mbus_code
- * in the scaler)
- */
-static const struct vimc_pix_map vimc_pix_map_list[] = {
-	/* TODO: add all missing formats */
-
-	/* RGB formats */
-	{
-		.code = MEDIA_BUS_FMT_BGR888_1X24,
-		.pixelformat = V4L2_PIX_FMT_BGR24,
-		.bpp = 3,
-		.bayer = false,
-	},
-	{
-		.code = MEDIA_BUS_FMT_RGB888_1X24,
-		.pixelformat = V4L2_PIX_FMT_RGB24,
-		.bpp = 3,
-		.bayer = false,
-	},
-	{
-		.code = MEDIA_BUS_FMT_ARGB8888_1X32,
-		.pixelformat = V4L2_PIX_FMT_ARGB32,
-		.bpp = 4,
-		.bayer = false,
-	},
-
-	/* Bayer formats */
-	{
-		.code = MEDIA_BUS_FMT_SBGGR8_1X8,
-		.pixelformat = V4L2_PIX_FMT_SBGGR8,
-		.bpp = 1,
-		.bayer = true,
-	},
-	{
-		.code = MEDIA_BUS_FMT_SGBRG8_1X8,
-		.pixelformat = V4L2_PIX_FMT_SGBRG8,
-		.bpp = 1,
-		.bayer = true,
-	},
-	{
-		.code = MEDIA_BUS_FMT_SGRBG8_1X8,
-		.pixelformat = V4L2_PIX_FMT_SGRBG8,
-		.bpp = 1,
-		.bayer = true,
-	},
-	{
-		.code = MEDIA_BUS_FMT_SRGGB8_1X8,
-		.pixelformat = V4L2_PIX_FMT_SRGGB8,
-		.bpp = 1,
-		.bayer = true,
-	},
-	{
-		.code = MEDIA_BUS_FMT_SBGGR10_1X10,
-		.pixelformat = V4L2_PIX_FMT_SBGGR10,
-		.bpp = 2,
-		.bayer = true,
-	},
-	{
-		.code = MEDIA_BUS_FMT_SGBRG10_1X10,
-		.pixelformat = V4L2_PIX_FMT_SGBRG10,
-		.bpp = 2,
-		.bayer = true,
-	},
-	{
-		.code = MEDIA_BUS_FMT_SGRBG10_1X10,
-		.pixelformat = V4L2_PIX_FMT_SGRBG10,
-		.bpp = 2,
-		.bayer = true,
-	},
-	{
-		.code = MEDIA_BUS_FMT_SRGGB10_1X10,
-		.pixelformat = V4L2_PIX_FMT_SRGGB10,
-		.bpp = 2,
-		.bayer = true,
-	},
-
-	/* 10bit raw bayer a-law compressed to 8 bits */
-	{
-		.code = MEDIA_BUS_FMT_SBGGR10_ALAW8_1X8,
-		.pixelformat = V4L2_PIX_FMT_SBGGR10ALAW8,
-		.bpp = 1,
-		.bayer = true,
-	},
-	{
-		.code = MEDIA_BUS_FMT_SGBRG10_ALAW8_1X8,
-		.pixelformat = V4L2_PIX_FMT_SGBRG10ALAW8,
-		.bpp = 1,
-		.bayer = true,
-	},
-	{
-		.code = MEDIA_BUS_FMT_SGRBG10_ALAW8_1X8,
-		.pixelformat = V4L2_PIX_FMT_SGRBG10ALAW8,
-		.bpp = 1,
-		.bayer = true,
-	},
-	{
-		.code = MEDIA_BUS_FMT_SRGGB10_ALAW8_1X8,
-		.pixelformat = V4L2_PIX_FMT_SRGGB10ALAW8,
-		.bpp = 1,
-		.bayer = true,
-	},
-
-	/* 10bit raw bayer DPCM compressed to 8 bits */
-	{
-		.code = MEDIA_BUS_FMT_SBGGR10_DPCM8_1X8,
-		.pixelformat = V4L2_PIX_FMT_SBGGR10DPCM8,
-		.bpp = 1,
-		.bayer = true,
-	},
-	{
-		.code = MEDIA_BUS_FMT_SGBRG10_DPCM8_1X8,
-		.pixelformat = V4L2_PIX_FMT_SGBRG10DPCM8,
-		.bpp = 1,
-		.bayer = true,
-	},
-	{
-		.code = MEDIA_BUS_FMT_SGRBG10_DPCM8_1X8,
-		.pixelformat = V4L2_PIX_FMT_SGRBG10DPCM8,
-		.bpp = 1,
-		.bayer = true,
-	},
-	{
-		.code = MEDIA_BUS_FMT_SRGGB10_DPCM8_1X8,
-		.pixelformat = V4L2_PIX_FMT_SRGGB10DPCM8,
-		.bpp = 1,
-		.bayer = true,
-	},
-	{
-		.code = MEDIA_BUS_FMT_SBGGR12_1X12,
-		.pixelformat = V4L2_PIX_FMT_SBGGR12,
-		.bpp = 2,
-		.bayer = true,
-	},
-	{
-		.code = MEDIA_BUS_FMT_SGBRG12_1X12,
-		.pixelformat = V4L2_PIX_FMT_SGBRG12,
-		.bpp = 2,
-		.bayer = true,
-	},
-	{
-		.code = MEDIA_BUS_FMT_SGRBG12_1X12,
-		.pixelformat = V4L2_PIX_FMT_SGRBG12,
-		.bpp = 2,
-		.bayer = true,
-	},
-	{
-		.code = MEDIA_BUS_FMT_SRGGB12_1X12,
-		.pixelformat = V4L2_PIX_FMT_SRGGB12,
-		.bpp = 2,
-		.bayer = true,
-	},
+static const __u32 vimc_mbus_list[] = {
+	MEDIA_BUS_FMT_FIXED,
+	MEDIA_BUS_FMT_RGB444_1X12,
+	MEDIA_BUS_FMT_RGB444_2X8_PADHI_BE,
+	MEDIA_BUS_FMT_RGB444_2X8_PADHI_LE,
+	MEDIA_BUS_FMT_RGB555_2X8_PADHI_BE,
+	MEDIA_BUS_FMT_RGB555_2X8_PADHI_LE,
+	MEDIA_BUS_FMT_RGB565_1X16,
+	MEDIA_BUS_FMT_BGR565_2X8_BE,
+	MEDIA_BUS_FMT_BGR565_2X8_LE,
+	MEDIA_BUS_FMT_RGB565_2X8_BE,
+	MEDIA_BUS_FMT_RGB565_2X8_LE,
+	MEDIA_BUS_FMT_RGB666_1X18,
+	MEDIA_BUS_FMT_RBG888_1X24,
+	MEDIA_BUS_FMT_RGB666_1X24_CPADHI,
+	MEDIA_BUS_FMT_RGB666_1X7X3_SPWG,
+	MEDIA_BUS_FMT_BGR888_1X24,
+	MEDIA_BUS_FMT_GBR888_1X24,
+	MEDIA_BUS_FMT_RGB888_1X24,
+	MEDIA_BUS_FMT_RGB888_2X12_BE,
+	MEDIA_BUS_FMT_RGB888_2X12_LE,
+	MEDIA_BUS_FMT_RGB888_1X7X4_SPWG,
+	MEDIA_BUS_FMT_RGB888_1X7X4_JEIDA,
+	MEDIA_BUS_FMT_ARGB8888_1X32,
+	MEDIA_BUS_FMT_RGB888_1X32_PADHI,
+	MEDIA_BUS_FMT_RGB101010_1X30,
+	MEDIA_BUS_FMT_RGB121212_1X36,
+	MEDIA_BUS_FMT_RGB161616_1X48,
+	MEDIA_BUS_FMT_Y8_1X8,
+	MEDIA_BUS_FMT_UV8_1X8,
+	MEDIA_BUS_FMT_UYVY8_1_5X8,
+	MEDIA_BUS_FMT_VYUY8_1_5X8,
+	MEDIA_BUS_FMT_YUYV8_1_5X8,
+	MEDIA_BUS_FMT_YVYU8_1_5X8,
+	MEDIA_BUS_FMT_UYVY8_2X8,
+	MEDIA_BUS_FMT_VYUY8_2X8,
+	MEDIA_BUS_FMT_YUYV8_2X8,
+	MEDIA_BUS_FMT_YVYU8_2X8,
+	MEDIA_BUS_FMT_Y10_1X10,
+	MEDIA_BUS_FMT_Y10_2X8_PADHI_LE,
+	MEDIA_BUS_FMT_UYVY10_2X10,
+	MEDIA_BUS_FMT_VYUY10_2X10,
+	MEDIA_BUS_FMT_YUYV10_2X10,
+	MEDIA_BUS_FMT_YVYU10_2X10,
+	MEDIA_BUS_FMT_Y12_1X12,
+	MEDIA_BUS_FMT_UYVY12_2X12,
+	MEDIA_BUS_FMT_VYUY12_2X12,
+	MEDIA_BUS_FMT_YUYV12_2X12,
+	MEDIA_BUS_FMT_YVYU12_2X12,
+	MEDIA_BUS_FMT_UYVY8_1X16,
+	MEDIA_BUS_FMT_VYUY8_1X16,
+	MEDIA_BUS_FMT_YUYV8_1X16,
+	MEDIA_BUS_FMT_YVYU8_1X16,
+	MEDIA_BUS_FMT_YDYUYDYV8_1X16,
+	MEDIA_BUS_FMT_UYVY10_1X20,
+	MEDIA_BUS_FMT_VYUY10_1X20,
+	MEDIA_BUS_FMT_YUYV10_1X20,
+	MEDIA_BUS_FMT_YVYU10_1X20,
+	MEDIA_BUS_FMT_VUY8_1X24,
+	MEDIA_BUS_FMT_YUV8_1X24,
+	MEDIA_BUS_FMT_UYYVYY8_0_5X24,
+	MEDIA_BUS_FMT_UYVY12_1X24,
+	MEDIA_BUS_FMT_VYUY12_1X24,
+	MEDIA_BUS_FMT_YUYV12_1X24,
+	MEDIA_BUS_FMT_YVYU12_1X24,
+	MEDIA_BUS_FMT_YUV10_1X30,
+	MEDIA_BUS_FMT_UYYVYY10_0_5X30,
+	MEDIA_BUS_FMT_AYUV8_1X32,
+	MEDIA_BUS_FMT_UYYVYY12_0_5X36,
+	MEDIA_BUS_FMT_YUV12_1X36,
+	MEDIA_BUS_FMT_YUV16_1X48,
+	MEDIA_BUS_FMT_UYYVYY16_0_5X48,
+	MEDIA_BUS_FMT_SBGGR8_1X8,
+	MEDIA_BUS_FMT_SGBRG8_1X8,
+	MEDIA_BUS_FMT_SGRBG8_1X8,
+	MEDIA_BUS_FMT_SRGGB8_1X8,
+	MEDIA_BUS_FMT_SBGGR10_ALAW8_1X8,
+	MEDIA_BUS_FMT_SGBRG10_ALAW8_1X8,
+	MEDIA_BUS_FMT_SGRBG10_ALAW8_1X8,
+	MEDIA_BUS_FMT_SRGGB10_ALAW8_1X8,
+	MEDIA_BUS_FMT_SBGGR10_DPCM8_1X8,
+	MEDIA_BUS_FMT_SGBRG10_DPCM8_1X8,
+	MEDIA_BUS_FMT_SGRBG10_DPCM8_1X8,
+	MEDIA_BUS_FMT_SRGGB10_DPCM8_1X8,
+	MEDIA_BUS_FMT_SBGGR10_2X8_PADHI_BE,
+	MEDIA_BUS_FMT_SBGGR10_2X8_PADHI_LE,
+	MEDIA_BUS_FMT_SBGGR10_2X8_PADLO_BE,
+	MEDIA_BUS_FMT_SBGGR10_2X8_PADLO_LE,
+	MEDIA_BUS_FMT_SBGGR10_1X10,
+	MEDIA_BUS_FMT_SGBRG10_1X10,
+	MEDIA_BUS_FMT_SGRBG10_1X10,
+	MEDIA_BUS_FMT_SRGGB10_1X10,
+	MEDIA_BUS_FMT_SBGGR12_1X12,
+	MEDIA_BUS_FMT_SGBRG12_1X12,
+	MEDIA_BUS_FMT_SGRBG12_1X12,
+	MEDIA_BUS_FMT_SRGGB12_1X12,
+	MEDIA_BUS_FMT_SBGGR14_1X14,
+	MEDIA_BUS_FMT_SGBRG14_1X14,
+	MEDIA_BUS_FMT_SGRBG14_1X14,
+	MEDIA_BUS_FMT_SRGGB14_1X14,
+	MEDIA_BUS_FMT_SBGGR16_1X16,
+	MEDIA_BUS_FMT_SGBRG16_1X16,
+	MEDIA_BUS_FMT_SGRBG16_1X16,
+	MEDIA_BUS_FMT_SRGGB16_1X16,
+	MEDIA_BUS_FMT_JPEG_1X8,
+	MEDIA_BUS_FMT_S5C_UYVY_JPEG_1X8,
+	MEDIA_BUS_FMT_AHSV8888_1X32,
 };
 
-const struct vimc_pix_map *vimc_pix_map_by_index(unsigned int i)
-{
-	if (i >= ARRAY_SIZE(vimc_pix_map_list))
-		return NULL;
-
-	return &vimc_pix_map_list[i];
-}
-EXPORT_SYMBOL_GPL(vimc_pix_map_by_index);
-
-const struct vimc_pix_map *vimc_pix_map_by_code(u32 code)
+/* Helper function to check mbus codes */
+bool vimc_mbus_code_supported(__u32 code)
 {
 	unsigned int i;
 
-	for (i = 0; i < ARRAY_SIZE(vimc_pix_map_list); i++) {
-		if (vimc_pix_map_list[i].code == code)
-			return &vimc_pix_map_list[i];
-	}
-	return NULL;
+	for (i = 0; i < ARRAY_SIZE(vimc_mbus_list); i++)
+		if (code == vimc_mbus_list[i])
+			return true;
+	return false;
 }
-EXPORT_SYMBOL_GPL(vimc_pix_map_by_code);
+EXPORT_SYMBOL_GPL(vimc_mbus_code_supported);
 
-const struct vimc_pix_map *vimc_pix_map_by_pixelformat(u32 pixelformat)
+/* Helper function to enumerate mbus codes */
+int vimc_enum_mbus_code(struct v4l2_subdev *sd,
+			struct v4l2_subdev_pad_config *cfg,
+			struct v4l2_subdev_mbus_code_enum *code)
 {
-	unsigned int i;
+	if (code->index >= ARRAY_SIZE(vimc_mbus_list))
+		return -EINVAL;
 
-	for (i = 0; i < ARRAY_SIZE(vimc_pix_map_list); i++) {
-		if (vimc_pix_map_list[i].pixelformat == pixelformat)
-			return &vimc_pix_map_list[i];
-	}
-	return NULL;
+	code->code = vimc_mbus_list[code->index];
+	return 0;
 }
-EXPORT_SYMBOL_GPL(vimc_pix_map_by_pixelformat);
+EXPORT_SYMBOL_GPL(vimc_enum_mbus_code);
 
 /* Helper function to allocate and initialize pads */
 struct media_pad *vimc_pads_init(u16 num_pads, const unsigned long *pads_flag)
@@ -277,15 +224,13 @@ static int vimc_get_mbus_format(struct media_pad *pad,
 							 struct video_device,
 							 entity);
 		struct vimc_ent_device *ved = video_get_drvdata(vdev);
-		const struct vimc_pix_map *vpix;
 		struct v4l2_pix_format vdev_fmt;
 
 		if (!ved->vdev_get_format)
 			return -ENOIOCTLCMD;
 
 		ved->vdev_get_format(ved, &vdev_fmt);
-		vpix = vimc_pix_map_by_pixelformat(vdev_fmt.pixelformat);
-		v4l2_fill_mbus_format(&fmt->format, &vdev_fmt, vpix->code);
+		v4l2_fill_mbus_format(&fmt->format, &vdev_fmt, 0);
 	} else {
 		return -EINVAL;
 	}
@@ -325,8 +270,12 @@ int vimc_link_validate(struct media_link *link)
 	/* The width, height and code must match. */
 	if (source_fmt.format.width != sink_fmt.format.width
 	    || source_fmt.format.height != sink_fmt.format.height
-	    || source_fmt.format.code != sink_fmt.format.code)
+	    || (source_fmt.format.code && sink_fmt.format.code &&
+		source_fmt.format.code != sink_fmt.format.code)) {
+		pr_err("vimc: format doesn't match in link %s->%s\n",
+			link->source->entity->name, link->sink->entity->name);
 		return -EPIPE;
+	}
 
 	/*
 	 * The field order must match, or the sink field order must be NONE
@@ -380,6 +329,7 @@ int vimc_ent_sd_register(struct vimc_ent_device *ved,
 			 u32 function,
 			 u16 num_pads,
 			 const unsigned long *pads_flag,
+			 const struct v4l2_subdev_internal_ops *sd_int_ops,
 			 const struct v4l2_subdev_ops *sd_ops)
 {
 	int ret;
@@ -394,6 +344,7 @@ int vimc_ent_sd_register(struct vimc_ent_device *ved,
 
 	/* Initialize the subdev */
 	v4l2_subdev_init(sd, sd_ops);
+	sd->internal_ops = sd_int_ops;
 	sd->entity.function = function;
 	sd->entity.ops = &vimc_ent_sd_mops;
 	sd->owner = THIS_MODULE;
@@ -431,9 +382,9 @@ EXPORT_SYMBOL_GPL(vimc_ent_sd_register);
 
 void vimc_ent_sd_unregister(struct vimc_ent_device *ved, struct v4l2_subdev *sd)
 {
-	v4l2_device_unregister_subdev(sd);
 	media_entity_cleanup(ved->ent);
 	vimc_pads_cleanup(ved->pads);
+	v4l2_device_unregister_subdev(sd);
 }
 EXPORT_SYMBOL_GPL(vimc_ent_sd_unregister);
 
diff --git a/drivers/media/platform/vimc/vimc-common.h b/drivers/media/platform/vimc/vimc-common.h
index 84539430b5e7..142ddfa69b3d 100644
--- a/drivers/media/platform/vimc/vimc-common.h
+++ b/drivers/media/platform/vimc/vimc-common.h
@@ -22,6 +22,8 @@
 #include <media/media-device.h>
 #include <media/v4l2-device.h>
 
+#include "vimc-streamer.h"
+
 #define VIMC_PDEV_NAME "vimc"
 
 /* VIMC-specific controls */
@@ -78,23 +80,6 @@ struct vimc_platform_data {
 };
 
 /**
- * struct vimc_pix_map - maps media bus code with v4l2 pixel format
- *
- * @code:		media bus format code defined by MEDIA_BUS_FMT_* macros
- * @bbp:		number of bytes each pixel occupies
- * @pixelformat:	pixel format devined by V4L2_PIX_FMT_* macros
- *
- * Struct which matches the MEDIA_BUS_FMT_* codes with the corresponding
- * V4L2_PIX_FMT_* fourcc pixelformat and its bytes per pixel (bpp)
- */
-struct vimc_pix_map {
-	unsigned int code;
-	unsigned int bpp;
-	u32 pixelformat;
-	bool bayer;
-};
-
-/**
  * struct vimc_ent_device - core struct that represents a node in the topology
  *
  * @ent:		the pointer to struct media_entity for the node
@@ -115,6 +100,7 @@ struct vimc_pix_map {
 struct vimc_ent_device {
 	struct media_entity *ent;
 	struct media_pad *pads;
+	struct vimc_stream *stream;
 	void * (*process_frame)(struct vimc_ent_device *ved,
 				const void *frame);
 	void (*vdev_get_format)(struct vimc_ent_device *ved,
@@ -122,6 +108,23 @@ struct vimc_ent_device {
 };
 
 /**
+ * vimc_mbus_code_supported - helper to check supported mbus codes
+ *
+ * Helper function to check if mbus code is enumerated by vimc_enum_mbus_code()
+ */
+bool vimc_mbus_code_supported(__u32 code);
+
+/**
+ * vimc_enum_mbus_code - enumerate mbus codes
+ *
+ * Helper function to be pluged in .enum_mbus_code from
+ * struct v4l2_subdev_pad_ops.
+ */
+int vimc_enum_mbus_code(struct v4l2_subdev *sd,
+			struct v4l2_subdev_pad_config *cfg,
+			struct v4l2_subdev_mbus_code_enum *code);
+
+/**
  * vimc_pads_init - initialize pads
  *
  * @num_pads:	number of pads to initialize
@@ -156,27 +159,6 @@ static inline void vimc_pads_cleanup(struct media_pad *pads)
 int vimc_pipeline_s_stream(struct media_entity *ent, int enable);
 
 /**
- * vimc_pix_map_by_index - get vimc_pix_map struct by its index
- *
- * @i:			index of the vimc_pix_map struct in vimc_pix_map_list
- */
-const struct vimc_pix_map *vimc_pix_map_by_index(unsigned int i);
-
-/**
- * vimc_pix_map_by_code - get vimc_pix_map struct by media bus code
- *
- * @code:		media bus format code defined by MEDIA_BUS_FMT_* macros
- */
-const struct vimc_pix_map *vimc_pix_map_by_code(u32 code);
-
-/**
- * vimc_pix_map_by_pixelformat - get vimc_pix_map struct by v4l2 pixel format
- *
- * @pixelformat:	pixel format devined by V4L2_PIX_FMT_* macros
- */
-const struct vimc_pix_map *vimc_pix_map_by_pixelformat(u32 pixelformat);
-
-/**
  * vimc_ent_sd_register - initialize and register a subdev node
  *
  * @ved:	the vimc_ent_device struct to be initialize
@@ -187,6 +169,7 @@ const struct vimc_pix_map *vimc_pix_map_by_pixelformat(u32 pixelformat);
  * @function:	media entity function defined by MEDIA_ENT_F_* macros
  * @num_pads:	number of pads to initialize
  * @pads_flag:	flags to use in each pad
+ * @sd_int_ops:	pointer to &struct v4l2_subdev_internal_ops
  * @sd_ops:	pointer to &struct v4l2_subdev_ops.
  *
  * Helper function initialize and register the struct vimc_ent_device and struct
@@ -199,6 +182,7 @@ int vimc_ent_sd_register(struct vimc_ent_device *ved,
 			 u32 function,
 			 u16 num_pads,
 			 const unsigned long *pads_flag,
+			 const struct v4l2_subdev_internal_ops *sd_int_ops,
 			 const struct v4l2_subdev_ops *sd_ops);
 
 /**
diff --git a/drivers/media/platform/vimc/vimc-core.c b/drivers/media/platform/vimc/vimc-core.c
index 0fbb7914098f..3aa62d7e3d0e 100644
--- a/drivers/media/platform/vimc/vimc-core.c
+++ b/drivers/media/platform/vimc/vimc-core.c
@@ -304,6 +304,8 @@ static int vimc_probe(struct platform_device *pdev)
 
 	dev_dbg(&pdev->dev, "probe");
 
+	memset(&vimc->mdev, 0, sizeof(vimc->mdev));
+
 	/* Create platform_device for each entity in the topology*/
 	vimc->subdevs = devm_kcalloc(&vimc->pdev.dev, vimc->pipe_cfg->num_ents,
 				     sizeof(*vimc->subdevs), GFP_KERNEL);
diff --git a/drivers/media/platform/vimc/vimc-debayer.c b/drivers/media/platform/vimc/vimc-debayer.c
index 7d77c63b99d2..281f9c1a7249 100644
--- a/drivers/media/platform/vimc/vimc-debayer.c
+++ b/drivers/media/platform/vimc/vimc-debayer.c
@@ -26,6 +26,9 @@
 #include "vimc-common.h"
 
 #define VIMC_DEB_DRV_NAME "vimc-debayer"
+/* This module only supports tranforming a bayer format to V4L2_PIX_FMT_RGB24 */
+#define VIMC_DEB_SRC_PIXFMT V4L2_PIX_FMT_RGB24
+#define VIMC_DEB_SRC_MBUS_FMT_DEFAULT MEDIA_BUS_FMT_RGB888_1X24
 
 static unsigned int deb_mean_win_size = 3;
 module_param(deb_mean_win_size, uint, 0000);
@@ -44,6 +47,7 @@ enum vimc_deb_rgb_colors {
 };
 
 struct vimc_deb_pix_map {
+	u32 pixelformat;
 	u32 code;
 	enum vimc_deb_rgb_colors order[2][2];
 };
@@ -66,68 +70,80 @@ struct vimc_deb_device {
 static const struct v4l2_mbus_framefmt sink_fmt_default = {
 	.width = 640,
 	.height = 480,
-	.code = MEDIA_BUS_FMT_RGB888_1X24,
+	.code = MEDIA_BUS_FMT_SRGGB8_1X8,
 	.field = V4L2_FIELD_NONE,
 	.colorspace = V4L2_COLORSPACE_DEFAULT,
 };
 
 static const struct vimc_deb_pix_map vimc_deb_pix_map_list[] = {
 	{
+		.pixelformat = V4L2_PIX_FMT_SBGGR8,
 		.code = MEDIA_BUS_FMT_SBGGR8_1X8,
 		.order = { { VIMC_DEB_BLUE, VIMC_DEB_GREEN },
 			   { VIMC_DEB_GREEN, VIMC_DEB_RED } }
 	},
 	{
+		.pixelformat = V4L2_PIX_FMT_SGBRG8,
 		.code = MEDIA_BUS_FMT_SGBRG8_1X8,
 		.order = { { VIMC_DEB_GREEN, VIMC_DEB_BLUE },
 			   { VIMC_DEB_RED, VIMC_DEB_GREEN } }
 	},
 	{
+		.pixelformat = V4L2_PIX_FMT_SGRBG8,
 		.code = MEDIA_BUS_FMT_SGRBG8_1X8,
 		.order = { { VIMC_DEB_GREEN, VIMC_DEB_RED },
 			   { VIMC_DEB_BLUE, VIMC_DEB_GREEN } }
 	},
 	{
+		.pixelformat = V4L2_PIX_FMT_SRGGB8,
 		.code = MEDIA_BUS_FMT_SRGGB8_1X8,
 		.order = { { VIMC_DEB_RED, VIMC_DEB_GREEN },
 			   { VIMC_DEB_GREEN, VIMC_DEB_BLUE } }
 	},
 	{
+		.pixelformat = V4L2_PIX_FMT_SBGGR10,
 		.code = MEDIA_BUS_FMT_SBGGR10_1X10,
 		.order = { { VIMC_DEB_BLUE, VIMC_DEB_GREEN },
 			   { VIMC_DEB_GREEN, VIMC_DEB_RED } }
 	},
 	{
+		.pixelformat = V4L2_PIX_FMT_SGBRG10,
 		.code = MEDIA_BUS_FMT_SGBRG10_1X10,
 		.order = { { VIMC_DEB_GREEN, VIMC_DEB_BLUE },
 			   { VIMC_DEB_RED, VIMC_DEB_GREEN } }
 	},
 	{
+		.pixelformat = V4L2_PIX_FMT_SGRBG10,
 		.code = MEDIA_BUS_FMT_SGRBG10_1X10,
 		.order = { { VIMC_DEB_GREEN, VIMC_DEB_RED },
 			   { VIMC_DEB_BLUE, VIMC_DEB_GREEN } }
 	},
 	{
+		.pixelformat = V4L2_PIX_FMT_SRGGB10,
 		.code = MEDIA_BUS_FMT_SRGGB10_1X10,
 		.order = { { VIMC_DEB_RED, VIMC_DEB_GREEN },
 			   { VIMC_DEB_GREEN, VIMC_DEB_BLUE } }
 	},
 	{
+		.pixelformat = V4L2_PIX_FMT_SBGGR12,
 		.code = MEDIA_BUS_FMT_SBGGR12_1X12,
 		.order = { { VIMC_DEB_BLUE, VIMC_DEB_GREEN },
 			   { VIMC_DEB_GREEN, VIMC_DEB_RED } }
 	},
 	{
+		.pixelformat = V4L2_PIX_FMT_SGBRG12,
 		.code = MEDIA_BUS_FMT_SGBRG12_1X12,
 		.order = { { VIMC_DEB_GREEN, VIMC_DEB_BLUE },
 			   { VIMC_DEB_RED, VIMC_DEB_GREEN } }
 	},
 	{
+		.pixelformat = V4L2_PIX_FMT_SGRBG12,
 		.code = MEDIA_BUS_FMT_SGRBG12_1X12,
 		.order = { { VIMC_DEB_GREEN, VIMC_DEB_RED },
 			   { VIMC_DEB_BLUE, VIMC_DEB_GREEN } }
 	},
 	{
+		.pixelformat = V4L2_PIX_FMT_SRGGB12,
 		.code = MEDIA_BUS_FMT_SRGGB12_1X12,
 		.order = { { VIMC_DEB_RED, VIMC_DEB_GREEN },
 			   { VIMC_DEB_GREEN, VIMC_DEB_BLUE } }
@@ -168,41 +184,32 @@ static int vimc_deb_enum_mbus_code(struct v4l2_subdev *sd,
 				   struct v4l2_subdev_pad_config *cfg,
 				   struct v4l2_subdev_mbus_code_enum *code)
 {
-	/* We only support one format for source pads */
-	if (IS_SRC(code->pad)) {
-		struct vimc_deb_device *vdeb = v4l2_get_subdevdata(sd);
-
-		if (code->index)
-			return -EINVAL;
-
-		code->code = vdeb->src_code;
-	} else {
+	/* For the sink pad we only support codes in the map_list */
+	if (IS_SINK(code->pad)) {
 		if (code->index >= ARRAY_SIZE(vimc_deb_pix_map_list))
 			return -EINVAL;
 
 		code->code = vimc_deb_pix_map_list[code->index].code;
+		return 0;
 	}
 
-	return 0;
+	return vimc_enum_mbus_code(sd, cfg, code);
 }
 
 static int vimc_deb_enum_frame_size(struct v4l2_subdev *sd,
 				    struct v4l2_subdev_pad_config *cfg,
 				    struct v4l2_subdev_frame_size_enum *fse)
 {
-	struct vimc_deb_device *vdeb = v4l2_get_subdevdata(sd);
-
 	if (fse->index)
 		return -EINVAL;
 
+	/* For the sink pad we only support codes in the map_list */
 	if (IS_SINK(fse->pad)) {
 		const struct vimc_deb_pix_map *vpix =
 			vimc_deb_pix_map_by_code(fse->code);
 
 		if (!vpix)
 			return -EINVAL;
-	} else if (fse->code != vdeb->src_code) {
-		return -EINVAL;
 	}
 
 	fse->min_width = VIMC_FRAME_MIN_WIDTH;
@@ -258,6 +265,9 @@ static int vimc_deb_set_fmt(struct v4l2_subdev *sd,
 	struct vimc_deb_device *vdeb = v4l2_get_subdevdata(sd);
 	struct v4l2_mbus_framefmt *sink_fmt;
 
+	if (!vimc_mbus_code_supported(fmt->format.code))
+		fmt->format.code = sink_fmt_default.code;
+
 	if (fmt->which == V4L2_SUBDEV_FORMAT_ACTIVE) {
 		/* Do not change the format while stream is on */
 		if (vdeb->src_frame)
@@ -270,11 +280,11 @@ static int vimc_deb_set_fmt(struct v4l2_subdev *sd,
 
 	/*
 	 * Do not change the format of the source pad,
-	 * it is propagated from the sink
+	 * it is propagated from the sink (except for the code)
 	 */
 	if (IS_SRC(fmt->pad)) {
+		vdeb->src_code = fmt->format.code;
 		fmt->format = *sink_fmt;
-		/* TODO: Add support for other formats */
 		fmt->format.code = vdeb->src_code;
 	} else {
 		/* Set the new format in the sink pad */
@@ -306,7 +316,7 @@ static const struct v4l2_subdev_pad_ops vimc_deb_pad_ops = {
 	.set_fmt		= vimc_deb_set_fmt,
 };
 
-static void vimc_deb_set_rgb_mbus_fmt_rgb888_1x24(struct vimc_deb_device *vdeb,
+static void vimc_deb_set_rgb_pix_rgb24(struct vimc_deb_device *vdeb,
 						  unsigned int lin,
 						  unsigned int col,
 						  unsigned int rgb[3])
@@ -323,25 +333,38 @@ static int vimc_deb_s_stream(struct v4l2_subdev *sd, int enable)
 	struct vimc_deb_device *vdeb = v4l2_get_subdevdata(sd);
 
 	if (enable) {
-		const struct vimc_pix_map *vpix;
+		u32 src_pixelformat = vdeb->ved.stream->producer_pixfmt;
+		const struct v4l2_format_info *pix_info;
 		unsigned int frame_size;
 
 		if (vdeb->src_frame)
 			return 0;
 
-		/* Calculate the frame size of the source pad */
-		vpix = vimc_pix_map_by_code(vdeb->src_code);
-		frame_size = vdeb->sink_fmt.width * vdeb->sink_fmt.height *
-				vpix->bpp;
-
-		/* Save the bytes per pixel of the sink */
-		vpix = vimc_pix_map_by_code(vdeb->sink_fmt.code);
-		vdeb->sink_bpp = vpix->bpp;
+		/* We only support translating bayer to RGB24 */
+		if (src_pixelformat != V4L2_PIX_FMT_RGB24) {
+			dev_err(vdeb->dev,
+				"translating to pixfmt (0x%08x) is not supported\n",
+				src_pixelformat);
+			return -EINVAL;
+		}
 
 		/* Get the corresponding pixel map from the table */
 		vdeb->sink_pix_map =
 			vimc_deb_pix_map_by_code(vdeb->sink_fmt.code);
 
+		/* Request bayer format from the pipeline for the sink pad */
+		vdeb->ved.stream->producer_pixfmt =
+			vdeb->sink_pix_map->pixelformat;
+
+		/* Calculate frame_size of the source */
+		pix_info = v4l2_format_info(src_pixelformat);
+		frame_size = vdeb->sink_fmt.width * vdeb->sink_fmt.height *
+			     pix_info->bpp[0];
+
+		/* Get bpp from the sink */
+		pix_info = v4l2_format_info(vdeb->sink_pix_map->pixelformat);
+		vdeb->sink_bpp = pix_info->bpp[0];
+
 		/*
 		 * Allocate the frame buffer. Use vmalloc to be able to
 		 * allocate a large amount of memory
@@ -489,6 +512,18 @@ static void *vimc_deb_process_frame(struct vimc_ent_device *ved,
 
 }
 
+static void vimc_deb_release(struct v4l2_subdev *sd)
+{
+	struct vimc_deb_device *vdeb =
+				container_of(sd, struct vimc_deb_device, sd);
+
+	kfree(vdeb);
+}
+
+static const struct v4l2_subdev_internal_ops vimc_deb_int_ops = {
+	.release = vimc_deb_release,
+};
+
 static void vimc_deb_comp_unbind(struct device *comp, struct device *master,
 				 void *master_data)
 {
@@ -497,7 +532,6 @@ static void vimc_deb_comp_unbind(struct device *comp, struct device *master,
 						    ved);
 
 	vimc_ent_sd_unregister(ved, &vdeb->sd);
-	kfree(vdeb);
 }
 
 static int vimc_deb_comp_bind(struct device *comp, struct device *master,
@@ -519,7 +553,7 @@ static int vimc_deb_comp_bind(struct device *comp, struct device *master,
 				   MEDIA_ENT_F_PROC_VIDEO_PIXEL_ENC_CONV, 2,
 				   (const unsigned long[2]) {MEDIA_PAD_FL_SINK,
 				   MEDIA_PAD_FL_SOURCE},
-				   &vimc_deb_ops);
+				   &vimc_deb_int_ops, &vimc_deb_ops);
 	if (ret) {
 		kfree(vdeb);
 		return ret;
@@ -531,14 +565,14 @@ static int vimc_deb_comp_bind(struct device *comp, struct device *master,
 
 	/* Initialize the frame format */
 	vdeb->sink_fmt = sink_fmt_default;
+	vdeb->src_code = VIMC_DEB_SRC_MBUS_FMT_DEFAULT;
 	/*
 	 * TODO: Add support for more output formats, we only support
-	 * RGB888 for now
+	 * RGB24 for now.
 	 * NOTE: the src format is always the same as the sink, except
 	 * for the code
 	 */
-	vdeb->src_code = MEDIA_BUS_FMT_RGB888_1X24;
-	vdeb->set_rgb_src = vimc_deb_set_rgb_mbus_fmt_rgb888_1x24;
+	vdeb->set_rgb_src = vimc_deb_set_rgb_pix_rgb24;
 
 	return 0;
 }
diff --git a/drivers/media/platform/vimc/vimc-scaler.c b/drivers/media/platform/vimc/vimc-scaler.c
index 39b2a73dfcc1..8aecf8e92031 100644
--- a/drivers/media/platform/vimc/vimc-scaler.c
+++ b/drivers/media/platform/vimc/vimc-scaler.c
@@ -35,6 +35,12 @@ MODULE_PARM_DESC(sca_mult, " the image size multiplier");
 #define IS_SRC(pad)	(pad)
 #define MAX_ZOOM	8
 
+static const u32 vimc_sca_supported_pixfmt[] = {
+	V4L2_PIX_FMT_BGR24,
+	V4L2_PIX_FMT_RGB24,
+	V4L2_PIX_FMT_ARGB32,
+};
+
 struct vimc_sca_device {
 	struct vimc_ent_device ved;
 	struct v4l2_subdev sd;
@@ -57,6 +63,16 @@ static const struct v4l2_mbus_framefmt sink_fmt_default = {
 	.colorspace = V4L2_COLORSPACE_DEFAULT,
 };
 
+static bool vimc_sca_is_pixfmt_supported(u32 pixelformat)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(vimc_sca_supported_pixfmt); i++)
+		if (vimc_sca_supported_pixfmt[i] == pixelformat)
+			return true;
+	return false;
+}
+
 static int vimc_sca_init_cfg(struct v4l2_subdev *sd,
 			     struct v4l2_subdev_pad_config *cfg)
 {
@@ -76,35 +92,13 @@ static int vimc_sca_init_cfg(struct v4l2_subdev *sd,
 	return 0;
 }
 
-static int vimc_sca_enum_mbus_code(struct v4l2_subdev *sd,
-				   struct v4l2_subdev_pad_config *cfg,
-				   struct v4l2_subdev_mbus_code_enum *code)
-{
-	const struct vimc_pix_map *vpix = vimc_pix_map_by_index(code->index);
-
-	/* We don't support bayer format */
-	if (!vpix || vpix->bayer)
-		return -EINVAL;
-
-	code->code = vpix->code;
-
-	return 0;
-}
-
 static int vimc_sca_enum_frame_size(struct v4l2_subdev *sd,
 				    struct v4l2_subdev_pad_config *cfg,
 				    struct v4l2_subdev_frame_size_enum *fse)
 {
-	const struct vimc_pix_map *vpix;
-
 	if (fse->index)
 		return -EINVAL;
 
-	/* Only accept code in the pix map table in non bayer format */
-	vpix = vimc_pix_map_by_code(fse->code);
-	if (!vpix || vpix->bayer)
-		return -EINVAL;
-
 	fse->min_width = VIMC_FRAME_MIN_WIDTH;
 	fse->min_height = VIMC_FRAME_MIN_HEIGHT;
 
@@ -141,13 +135,6 @@ static int vimc_sca_get_fmt(struct v4l2_subdev *sd,
 
 static void vimc_sca_adjust_sink_fmt(struct v4l2_mbus_framefmt *fmt)
 {
-	const struct vimc_pix_map *vpix;
-
-	/* Only accept code in the pix map table in non bayer format */
-	vpix = vimc_pix_map_by_code(fmt->code);
-	if (!vpix || vpix->bayer)
-		fmt->code = sink_fmt_default.code;
-
 	fmt->width = clamp_t(u32, fmt->width, VIMC_FRAME_MIN_WIDTH,
 			     VIMC_FRAME_MAX_WIDTH) & ~1;
 	fmt->height = clamp_t(u32, fmt->height, VIMC_FRAME_MIN_HEIGHT,
@@ -166,6 +153,9 @@ static int vimc_sca_set_fmt(struct v4l2_subdev *sd,
 	struct vimc_sca_device *vsca = v4l2_get_subdevdata(sd);
 	struct v4l2_mbus_framefmt *sink_fmt;
 
+	if (!vimc_mbus_code_supported(fmt->format.code))
+		fmt->format.code = sink_fmt_default.code;
+
 	if (fmt->which == V4L2_SUBDEV_FORMAT_ACTIVE) {
 		/* Do not change the format while stream is on */
 		if (vsca->src_frame)
@@ -208,7 +198,7 @@ static int vimc_sca_set_fmt(struct v4l2_subdev *sd,
 
 static const struct v4l2_subdev_pad_ops vimc_sca_pad_ops = {
 	.init_cfg		= vimc_sca_init_cfg,
-	.enum_mbus_code		= vimc_sca_enum_mbus_code,
+	.enum_mbus_code		= vimc_enum_mbus_code,
 	.enum_frame_size	= vimc_sca_enum_frame_size,
 	.get_fmt		= vimc_sca_get_fmt,
 	.set_fmt		= vimc_sca_set_fmt,
@@ -219,15 +209,22 @@ static int vimc_sca_s_stream(struct v4l2_subdev *sd, int enable)
 	struct vimc_sca_device *vsca = v4l2_get_subdevdata(sd);
 
 	if (enable) {
-		const struct vimc_pix_map *vpix;
+		u32 pixelformat = vsca->ved.stream->producer_pixfmt;
+		const struct v4l2_format_info *pix_info;
 		unsigned int frame_size;
 
 		if (vsca->src_frame)
 			return 0;
 
+		if (!vimc_sca_is_pixfmt_supported(pixelformat)) {
+			dev_err(vsca->dev, "pixfmt (0x%08x) is not supported\n",
+				pixelformat);
+			return -EINVAL;
+		}
+
 		/* Save the bytes per pixel of the sink */
-		vpix = vimc_pix_map_by_code(vsca->sink_fmt.code);
-		vsca->bpp = vpix->bpp;
+		pix_info = v4l2_format_info(pixelformat);
+		vsca->bpp = pix_info->bpp[0];
 
 		/* Calculate the width in bytes of the src frame */
 		vsca->src_line_size = vsca->sink_fmt.width *
@@ -348,6 +345,18 @@ static void *vimc_sca_process_frame(struct vimc_ent_device *ved,
 	return vsca->src_frame;
 };
 
+static void vimc_sca_release(struct v4l2_subdev *sd)
+{
+	struct vimc_sca_device *vsca =
+				container_of(sd, struct vimc_sca_device, sd);
+
+	kfree(vsca);
+}
+
+static const struct v4l2_subdev_internal_ops vimc_sca_int_ops = {
+	.release = vimc_sca_release,
+};
+
 static void vimc_sca_comp_unbind(struct device *comp, struct device *master,
 				 void *master_data)
 {
@@ -356,7 +365,6 @@ static void vimc_sca_comp_unbind(struct device *comp, struct device *master,
 						    ved);
 
 	vimc_ent_sd_unregister(ved, &vsca->sd);
-	kfree(vsca);
 }
 
 
@@ -379,7 +387,7 @@ static int vimc_sca_comp_bind(struct device *comp, struct device *master,
 				   MEDIA_ENT_F_PROC_VIDEO_SCALER, 2,
 				   (const unsigned long[2]) {MEDIA_PAD_FL_SINK,
 				   MEDIA_PAD_FL_SOURCE},
-				   &vimc_sca_ops);
+				   &vimc_sca_int_ops, &vimc_sca_ops);
 	if (ret) {
 		kfree(vsca);
 		return ret;
diff --git a/drivers/media/platform/vimc/vimc-sensor.c b/drivers/media/platform/vimc/vimc-sensor.c
index 59195f262623..081e54204c9f 100644
--- a/drivers/media/platform/vimc/vimc-sensor.c
+++ b/drivers/media/platform/vimc/vimc-sensor.c
@@ -65,34 +65,13 @@ static int vimc_sen_init_cfg(struct v4l2_subdev *sd,
 	return 0;
 }
 
-static int vimc_sen_enum_mbus_code(struct v4l2_subdev *sd,
-				   struct v4l2_subdev_pad_config *cfg,
-				   struct v4l2_subdev_mbus_code_enum *code)
-{
-	const struct vimc_pix_map *vpix = vimc_pix_map_by_index(code->index);
-
-	if (!vpix)
-		return -EINVAL;
-
-	code->code = vpix->code;
-
-	return 0;
-}
-
 static int vimc_sen_enum_frame_size(struct v4l2_subdev *sd,
 				    struct v4l2_subdev_pad_config *cfg,
 				    struct v4l2_subdev_frame_size_enum *fse)
 {
-	const struct vimc_pix_map *vpix;
-
 	if (fse->index)
 		return -EINVAL;
 
-	/* Only accept code in the pix map table */
-	vpix = vimc_pix_map_by_code(fse->code);
-	if (!vpix)
-		return -EINVAL;
-
 	fse->min_width = VIMC_FRAME_MIN_WIDTH;
 	fse->max_width = VIMC_FRAME_MAX_WIDTH;
 	fse->min_height = VIMC_FRAME_MIN_HEIGHT;
@@ -117,14 +96,17 @@ static int vimc_sen_get_fmt(struct v4l2_subdev *sd,
 
 static void vimc_sen_tpg_s_format(struct vimc_sen_device *vsen)
 {
-	const struct vimc_pix_map *vpix =
-				vimc_pix_map_by_code(vsen->mbus_format.code);
+	u32 pixelformat = vsen->ved.stream->producer_pixfmt;
+	const struct v4l2_format_info *pix_info;
+
+	pix_info = v4l2_format_info(pixelformat);
 
 	tpg_reset_source(&vsen->tpg, vsen->mbus_format.width,
 			 vsen->mbus_format.height, vsen->mbus_format.field);
-	tpg_s_bytesperline(&vsen->tpg, 0, vsen->mbus_format.width * vpix->bpp);
+	tpg_s_bytesperline(&vsen->tpg, 0,
+			   vsen->mbus_format.width * pix_info->bpp[0]);
 	tpg_s_buf_height(&vsen->tpg, vsen->mbus_format.height);
-	tpg_s_fourcc(&vsen->tpg, vpix->pixelformat);
+	tpg_s_fourcc(&vsen->tpg, pixelformat);
 	/* TODO: add support for V4L2_FIELD_ALTERNATE */
 	tpg_s_field(&vsen->tpg, vsen->mbus_format.field, false);
 	tpg_s_colorspace(&vsen->tpg, vsen->mbus_format.colorspace);
@@ -135,13 +117,6 @@ static void vimc_sen_tpg_s_format(struct vimc_sen_device *vsen)
 
 static void vimc_sen_adjust_fmt(struct v4l2_mbus_framefmt *fmt)
 {
-	const struct vimc_pix_map *vpix;
-
-	/* Only accept code in the pix map table */
-	vpix = vimc_pix_map_by_code(fmt->code);
-	if (!vpix)
-		fmt->code = fmt_default.code;
-
 	fmt->width = clamp_t(u32, fmt->width, VIMC_FRAME_MIN_WIDTH,
 			     VIMC_FRAME_MAX_WIDTH) & ~1;
 	fmt->height = clamp_t(u32, fmt->height, VIMC_FRAME_MIN_HEIGHT,
@@ -161,6 +136,9 @@ static int vimc_sen_set_fmt(struct v4l2_subdev *sd,
 	struct vimc_sen_device *vsen = v4l2_get_subdevdata(sd);
 	struct v4l2_mbus_framefmt *mf;
 
+	if (!vimc_mbus_code_supported(fmt->format.code))
+		fmt->format.code = fmt_default.code;
+
 	if (fmt->which == V4L2_SUBDEV_FORMAT_ACTIVE) {
 		/* Do not change the format while stream is on */
 		if (vsen->frame)
@@ -193,7 +171,7 @@ static int vimc_sen_set_fmt(struct v4l2_subdev *sd,
 
 static const struct v4l2_subdev_pad_ops vimc_sen_pad_ops = {
 	.init_cfg		= vimc_sen_init_cfg,
-	.enum_mbus_code		= vimc_sen_enum_mbus_code,
+	.enum_mbus_code		= vimc_enum_mbus_code,
 	.enum_frame_size	= vimc_sen_enum_frame_size,
 	.get_fmt		= vimc_sen_get_fmt,
 	.set_fmt		= vimc_sen_set_fmt,
@@ -215,7 +193,8 @@ static int vimc_sen_s_stream(struct v4l2_subdev *sd, int enable)
 				container_of(sd, struct vimc_sen_device, sd);
 
 	if (enable) {
-		const struct vimc_pix_map *vpix;
+		u32 pixelformat = vsen->ved.stream->producer_pixfmt;
+		const struct v4l2_format_info *pix_info;
 		unsigned int frame_size;
 
 		if (vsen->kthread_sen)
@@ -223,8 +202,8 @@ static int vimc_sen_s_stream(struct v4l2_subdev *sd, int enable)
 			return 0;
 
 		/* Calculate the frame size */
-		vpix = vimc_pix_map_by_code(vsen->mbus_format.code);
-		frame_size = vsen->mbus_format.width * vpix->bpp *
+		pix_info = v4l2_format_info(pixelformat);
+		frame_size = vsen->mbus_format.width * pix_info->bpp[0] *
 			     vsen->mbus_format.height;
 
 		/*
@@ -301,6 +280,20 @@ static const struct v4l2_ctrl_ops vimc_sen_ctrl_ops = {
 	.s_ctrl = vimc_sen_s_ctrl,
 };
 
+static void vimc_sen_release(struct v4l2_subdev *sd)
+{
+	struct vimc_sen_device *vsen =
+				container_of(sd, struct vimc_sen_device, sd);
+
+	v4l2_ctrl_handler_free(&vsen->hdl);
+	tpg_free(&vsen->tpg);
+	kfree(vsen);
+}
+
+static const struct v4l2_subdev_internal_ops vimc_sen_int_ops = {
+	.release = vimc_sen_release,
+};
+
 static void vimc_sen_comp_unbind(struct device *comp, struct device *master,
 				 void *master_data)
 {
@@ -309,9 +302,6 @@ static void vimc_sen_comp_unbind(struct device *comp, struct device *master,
 				container_of(ved, struct vimc_sen_device, ved);
 
 	vimc_ent_sd_unregister(ved, &vsen->sd);
-	v4l2_ctrl_handler_free(&vsen->hdl);
-	tpg_free(&vsen->tpg);
-	kfree(vsen);
 }
 
 /* Image Processing Controls */
@@ -371,7 +361,7 @@ static int vimc_sen_comp_bind(struct device *comp, struct device *master,
 				   pdata->entity_name,
 				   MEDIA_ENT_F_CAM_SENSOR, 1,
 				   (const unsigned long[1]) {MEDIA_PAD_FL_SOURCE},
-				   &vimc_sen_ops);
+				   &vimc_sen_int_ops, &vimc_sen_ops);
 	if (ret)
 		goto err_free_hdl;
 
diff --git a/drivers/media/platform/vimc/vimc-streamer.c b/drivers/media/platform/vimc/vimc-streamer.c
index fcc897fb247b..26b674259489 100644
--- a/drivers/media/platform/vimc/vimc-streamer.c
+++ b/drivers/media/platform/vimc/vimc-streamer.c
@@ -46,19 +46,19 @@ static struct media_entity *vimc_get_source_entity(struct media_entity *ent)
  */
 static void vimc_streamer_pipeline_terminate(struct vimc_stream *stream)
 {
-	struct media_entity *entity;
+	struct vimc_ent_device *ved;
 	struct v4l2_subdev *sd;
 
 	while (stream->pipe_size) {
 		stream->pipe_size--;
-		entity = stream->ved_pipeline[stream->pipe_size]->ent;
-		entity = vimc_get_source_entity(entity);
+		ved = stream->ved_pipeline[stream->pipe_size];
+		ved->stream = NULL;
 		stream->ved_pipeline[stream->pipe_size] = NULL;
 
-		if (!is_media_entity_v4l2_subdev(entity))
+		if (!is_media_entity_v4l2_subdev(ved->ent))
 			continue;
 
-		sd = media_entity_to_v4l2_subdev(entity);
+		sd = media_entity_to_v4l2_subdev(ved->ent);
 		v4l2_subdev_call(sd, video, s_stream, 0);
 	}
 }
@@ -88,19 +88,27 @@ static int vimc_streamer_pipeline_init(struct vimc_stream *stream,
 			return -EINVAL;
 		}
 		stream->ved_pipeline[stream->pipe_size++] = ved;
+		ved->stream = stream;
+
+		if (is_media_entity_v4l2_subdev(ved->ent)) {
+			sd = media_entity_to_v4l2_subdev(ved->ent);
+			ret = v4l2_subdev_call(sd, video, s_stream, 1);
+			if (ret && ret != -ENOIOCTLCMD) {
+				pr_err("subdev_call error %s\n",
+				       ved->ent->name);
+				vimc_streamer_pipeline_terminate(stream);
+				return ret;
+			}
+		}
 
 		entity = vimc_get_source_entity(ved->ent);
 		/* Check if the end of the pipeline was reached*/
 		if (!entity)
 			return 0;
 
+		/* Get the next device in the pipeline */
 		if (is_media_entity_v4l2_subdev(entity)) {
 			sd = media_entity_to_v4l2_subdev(entity);
-			ret = v4l2_subdev_call(sd, video, s_stream, 1);
-			if (ret && ret != -ENOIOCTLCMD) {
-				vimc_streamer_pipeline_terminate(stream);
-				return ret;
-			}
 			ved = v4l2_get_subdevdata(sd);
 		} else {
 			vdev = container_of(entity,
@@ -117,10 +125,10 @@ static int vimc_streamer_pipeline_init(struct vimc_stream *stream,
 static int vimc_streamer_thread(void *data)
 {
 	struct vimc_stream *stream = data;
+	u8 *frame = NULL;
 	int i;
 
 	set_freezable();
-	set_current_state(TASK_UNINTERRUPTIBLE);
 
 	for (;;) {
 		try_to_freeze();
@@ -128,15 +136,13 @@ static int vimc_streamer_thread(void *data)
 			break;
 
 		for (i = stream->pipe_size - 1; i >= 0; i--) {
-			stream->frame = stream->ved_pipeline[i]->process_frame(
-					stream->ved_pipeline[i],
-					stream->frame);
-			if (!stream->frame)
-				break;
-			if (IS_ERR(stream->frame))
+			frame = stream->ved_pipeline[i]->process_frame(
+					stream->ved_pipeline[i], frame);
+			if (!frame || IS_ERR(frame))
 				break;
 		}
 		//wait for 60hz
+		set_current_state(TASK_UNINTERRUPTIBLE);
 		schedule_timeout(HZ / 60);
 	}
 
diff --git a/drivers/media/platform/vimc/vimc-streamer.h b/drivers/media/platform/vimc/vimc-streamer.h
index 752af2e2d5a2..2b3667408794 100644
--- a/drivers/media/platform/vimc/vimc-streamer.h
+++ b/drivers/media/platform/vimc/vimc-streamer.h
@@ -15,12 +15,32 @@
 
 #define VIMC_STREAMER_PIPELINE_MAX_SIZE 16
 
+/**
+ * struct vimc_stream - struct that represents a stream in the pipeline
+ *
+ * @pipe:		the media pipeline object associated with this stream
+ * @ved_pipeline:	array containing all the entities participating in the
+ * stream. The order is from a video device (usually a capture device) where
+ * stream_on was called, to the entity generating the first base image to be
+ * processed in the pipeline.
+ * @pipe_size:		size of @ved_pipeline
+ * @kthread:		thread that generates the frames of the stream.
+ * @producer_pixfmt:	the pixel format requested from the pipeline. This must
+ * be set just before calling vimc_streamer_s_stream(ent, 1). This value is
+ * propagated up to the source of the base image (usually a sensor node) and
+ * can be modified by entities during s_stream callback to request a different
+ * format from rest of the pipeline.
+ *
+ * When the user call stream_on in a video device, struct vimc_stream is
+ * used to keep track of all entities and subdevices that generates and
+ * process frames for the stream.
+ */
 struct vimc_stream {
 	struct media_pipeline pipe;
 	struct vimc_ent_device *ved_pipeline[VIMC_STREAMER_PIPELINE_MAX_SIZE];
 	unsigned int pipe_size;
-	u8 *frame;
 	struct task_struct *kthread;
+	u32 producer_pixfmt;
 };
 
 /**
diff --git a/drivers/media/platform/vivid/Kconfig b/drivers/media/platform/vivid/Kconfig
index 154de92dd809..4b51d4d6cf93 100644
--- a/drivers/media/platform/vivid/Kconfig
+++ b/drivers/media/platform/vivid/Kconfig
@@ -11,7 +11,7 @@ config VIDEO_VIVID
 	select VIDEOBUF2_DMA_CONTIG
 	select VIDEO_V4L2_TPG
 	default n
-	---help---
+	help
 	  Enables a virtual video driver. This driver emulates a webcam,
 	  TV, S-Video and HDMI capture hardware, including VBI support for
 	  the SDTV inputs. Also video output, VBI output, radio receivers,
@@ -28,7 +28,7 @@ config VIDEO_VIVID_CEC
 	bool "Enable CEC emulation support"
 	depends on VIDEO_VIVID
 	select CEC_CORE
-	---help---
+	help
 	  When selected the vivid module will emulate the optional
 	  HDMI CEC feature.
 
@@ -36,6 +36,6 @@ config VIDEO_VIVID_MAX_DEVS
 	int "Maximum number of devices"
 	depends on VIDEO_VIVID
 	default "64"
-	---help---
+	help
 	  This allows you to specify the maximum number of devices supported
 	  by the vivid driver.
diff --git a/drivers/media/platform/vivid/vivid-core.c b/drivers/media/platform/vivid/vivid-core.c
index 342e0e6c103b..7047df6f0e0e 100644
--- a/drivers/media/platform/vivid/vivid-core.c
+++ b/drivers/media/platform/vivid/vivid-core.c
@@ -681,7 +681,7 @@ static int vivid_create_instance(struct platform_device *pdev, int inst)
 	dev->v4l2_dev.mdev = &dev->mdev;
 
 	/* Initialize media device */
-	strlcpy(dev->mdev.model, VIVID_MODULE_NAME, sizeof(dev->mdev.model));
+	strscpy(dev->mdev.model, VIVID_MODULE_NAME, sizeof(dev->mdev.model));
 	snprintf(dev->mdev.bus_info, sizeof(dev->mdev.bus_info),
 		 "platform:%s-%03d", VIVID_MODULE_NAME, inst);
 	dev->mdev.dev = &pdev->dev;
diff --git a/drivers/media/platform/vivid/vivid-vid-cap.c b/drivers/media/platform/vivid/vivid-vid-cap.c
index 52eeda624d7e..530ac8decb25 100644
--- a/drivers/media/platform/vivid/vivid-vid-cap.c
+++ b/drivers/media/platform/vivid/vivid-vid-cap.c
@@ -1007,7 +1007,7 @@ int vivid_vid_cap_s_selection(struct file *file, void *fh, struct v4l2_selection
 		v4l2_rect_map_inside(&s->r, &dev->fmt_cap_rect);
 		if (dev->bitmap_cap && (compose->width != s->r.width ||
 					compose->height != s->r.height)) {
-			kfree(dev->bitmap_cap);
+			vfree(dev->bitmap_cap);
 			dev->bitmap_cap = NULL;
 		}
 		*compose = s->r;
diff --git a/drivers/media/platform/vivid/vivid-vid-out.c b/drivers/media/platform/vivid/vivid-vid-out.c
index e61b91b414f9..9350ca65dd91 100644
--- a/drivers/media/platform/vivid/vivid-vid-out.c
+++ b/drivers/media/platform/vivid/vivid-vid-out.c
@@ -798,7 +798,7 @@ int vivid_vid_out_s_selection(struct file *file, void *fh, struct v4l2_selection
 		s->r.height *= factor;
 		if (dev->bitmap_out && (compose->width != s->r.width ||
 					compose->height != s->r.height)) {
-			kfree(dev->bitmap_out);
+			vfree(dev->bitmap_out);
 			dev->bitmap_out = NULL;
 		}
 		*compose = s->r;
@@ -941,15 +941,19 @@ int vidioc_s_fmt_vid_out_overlay(struct file *file, void *priv,
 		return ret;
 
 	if (win->bitmap) {
-		new_bitmap = memdup_user(win->bitmap, bitmap_size);
+		new_bitmap = vzalloc(bitmap_size);
 
-		if (IS_ERR(new_bitmap))
-			return PTR_ERR(new_bitmap);
+		if (!new_bitmap)
+			return -ENOMEM;
+		if (copy_from_user(new_bitmap, win->bitmap, bitmap_size)) {
+			vfree(new_bitmap);
+			return -EFAULT;
+		}
 	}
 
 	dev->overlay_out_top = win->w.top;
 	dev->overlay_out_left = win->w.left;
-	kfree(dev->bitmap_out);
+	vfree(dev->bitmap_out);
 	dev->bitmap_out = new_bitmap;
 	dev->clipcount_out = win->clipcount;
 	if (dev->clipcount_out)
diff --git a/drivers/media/platform/vsp1/vsp1_brx.c b/drivers/media/platform/vsp1/vsp1_brx.c
index 58ad248cd7f7..2d86c718a5cf 100644
--- a/drivers/media/platform/vsp1/vsp1_brx.c
+++ b/drivers/media/platform/vsp1/vsp1_brx.c
@@ -283,6 +283,7 @@ static const struct v4l2_subdev_ops brx_ops = {
 
 static void brx_configure_stream(struct vsp1_entity *entity,
 				 struct vsp1_pipeline *pipe,
+				 struct vsp1_dl_list *dl,
 				 struct vsp1_dl_body *dlb)
 {
 	struct vsp1_brx *brx = to_brx(&entity->subdev);
diff --git a/drivers/media/platform/vsp1/vsp1_clu.c b/drivers/media/platform/vsp1/vsp1_clu.c
index 942fc14c19d1..a47b23bf5abf 100644
--- a/drivers/media/platform/vsp1/vsp1_clu.c
+++ b/drivers/media/platform/vsp1/vsp1_clu.c
@@ -171,6 +171,7 @@ static const struct v4l2_subdev_ops clu_ops = {
 
 static void clu_configure_stream(struct vsp1_entity *entity,
 				 struct vsp1_pipeline *pipe,
+				 struct vsp1_dl_list *dl,
 				 struct vsp1_dl_body *dlb)
 {
 	struct vsp1_clu *clu = to_clu(&entity->subdev);
diff --git a/drivers/media/platform/vsp1/vsp1_dl.c b/drivers/media/platform/vsp1/vsp1_dl.c
index 26289adaf658..104b6f514536 100644
--- a/drivers/media/platform/vsp1/vsp1_dl.c
+++ b/drivers/media/platform/vsp1/vsp1_dl.c
@@ -178,7 +178,7 @@ struct vsp1_dl_cmd_pool {
  * @post_cmd: post command to be issued through extended dl header
  * @has_chain: if true, indicates that there's a partition chain
  * @chain: entry in the display list partition chain
- * @internal: whether the display list is used for internal purpose
+ * @flags: display list flags, a combination of VSP1_DL_FRAME_END_*
  */
 struct vsp1_dl_list {
 	struct list_head list;
@@ -197,7 +197,7 @@ struct vsp1_dl_list {
 	bool has_chain;
 	struct list_head chain;
 
-	bool internal;
+	unsigned int flags;
 };
 
 /**
@@ -699,8 +699,8 @@ struct vsp1_dl_body *vsp1_dl_list_get_body0(struct vsp1_dl_list *dl)
  * which bodies are added.
  *
  * Adding a body to a display list passes ownership of the body to the list. The
- * caller retains its reference to the fragment when adding it to the display
- * list, but is not allowed to add new entries to the body.
+ * caller retains its reference to the body when adding it to the display list,
+ * but is not allowed to add new entries to the body.
  *
  * The reference must be explicitly released by a call to vsp1_dl_body_put()
  * when the body isn't needed anymore.
@@ -770,17 +770,35 @@ static void vsp1_dl_list_fill_header(struct vsp1_dl_list *dl, bool is_last)
 	}
 
 	dl->header->num_lists = num_lists;
+	dl->header->flags = 0;
 
-	if (!list_empty(&dl->chain) && !is_last) {
+	/*
+	 * Enable the interrupt for the end of each frame. In continuous mode
+	 * chained lists are used with one list per frame, so enable the
+	 * interrupt for each list. In singleshot mode chained lists are used
+	 * to partition a single frame, so enable the interrupt for the last
+	 * list only.
+	 */
+	if (!dlm->singleshot || is_last)
+		dl->header->flags |= VSP1_DLH_INT_ENABLE;
+
+	/*
+	 * In continuous mode enable auto-start for all lists, as the VSP must
+	 * loop on the same list until a new one is queued. In singleshot mode
+	 * enable auto-start for all lists but the last to chain processing of
+	 * partitions without software intervention.
+	 */
+	if (!dlm->singleshot || !is_last)
+		dl->header->flags |= VSP1_DLH_AUTO_START;
+
+	if (!is_last) {
 		/*
-		 * If this display list's chain is not empty, we are on a list,
-		 * and the next item is the display list that we must queue for
-		 * automatic processing by the hardware.
+		 * If this is not the last display list in the chain, queue the
+		 * next item for automatic processing by the hardware.
 		 */
 		struct vsp1_dl_list *next = list_next_entry(dl, chain);
 
 		dl->header->next_header = next->dma;
-		dl->header->flags = VSP1_DLH_AUTO_START;
 	} else if (!dlm->singleshot) {
 		/*
 		 * if the display list manager works in continuous mode, the VSP
@@ -788,13 +806,6 @@ static void vsp1_dl_list_fill_header(struct vsp1_dl_list *dl, bool is_last)
 		 * instructed to do otherwise.
 		 */
 		dl->header->next_header = dl->dma;
-		dl->header->flags = VSP1_DLH_INT_ENABLE | VSP1_DLH_AUTO_START;
-	} else {
-		/*
-		 * Otherwise, in mem-to-mem mode, we work in single-shot mode
-		 * and the next display list must not be started automatically.
-		 */
-		dl->header->flags = VSP1_DLH_INT_ENABLE;
 	}
 
 	if (!dl->extension)
@@ -861,13 +872,15 @@ static void vsp1_dl_list_commit_continuous(struct vsp1_dl_list *dl)
 	 *
 	 * If a display list is already pending we simply drop it as the new
 	 * display list is assumed to contain a more recent configuration. It is
-	 * an error if the already pending list has the internal flag set, as
-	 * there is then a process waiting for that list to complete. This
-	 * shouldn't happen as the waiting process should perform proper
-	 * locking, but warn just in case.
+	 * an error if the already pending list has the
+	 * VSP1_DL_FRAME_END_INTERNAL flag set, as there is then a process
+	 * waiting for that list to complete. This shouldn't happen as the
+	 * waiting process should perform proper locking, but warn just in
+	 * case.
 	 */
 	if (vsp1_dl_list_hw_update_pending(dlm)) {
-		WARN_ON(dlm->pending && dlm->pending->internal);
+		WARN_ON(dlm->pending &&
+			(dlm->pending->flags & VSP1_DL_FRAME_END_INTERNAL));
 		__vsp1_dl_list_put(dlm->pending);
 		dlm->pending = dl;
 		return;
@@ -897,7 +910,7 @@ static void vsp1_dl_list_commit_singleshot(struct vsp1_dl_list *dl)
 	dlm->active = dl;
 }
 
-void vsp1_dl_list_commit(struct vsp1_dl_list *dl, bool internal)
+void vsp1_dl_list_commit(struct vsp1_dl_list *dl, unsigned int dl_flags)
 {
 	struct vsp1_dl_manager *dlm = dl->dlm;
 	struct vsp1_dl_list *dl_next;
@@ -912,7 +925,7 @@ void vsp1_dl_list_commit(struct vsp1_dl_list *dl, bool internal)
 		vsp1_dl_list_fill_header(dl_next, last);
 	}
 
-	dl->internal = internal;
+	dl->flags = dl_flags & ~VSP1_DL_FRAME_END_COMPLETED;
 
 	spin_lock_irqsave(&dlm->lock, flags);
 
@@ -941,9 +954,13 @@ void vsp1_dl_list_commit(struct vsp1_dl_list *dl, bool internal)
  * set in single-shot mode as display list processing is then not continuous and
  * races never occur.
  *
- * The VSP1_DL_FRAME_END_INTERNAL flag indicates that the previous display list
- * has completed and had been queued with the internal notification flag.
- * Internal notification is only supported for continuous mode.
+ * The following flags are only supported for continuous mode.
+ *
+ * The VSP1_DL_FRAME_END_INTERNAL flag indicates that the display list that just
+ * became active had been queued with the internal notification flag.
+ *
+ * The VSP1_DL_FRAME_END_WRITEBACK flag indicates that the previously active
+ * display list had been queued with the writeback flag.
  */
 unsigned int vsp1_dlm_irq_frame_end(struct vsp1_dl_manager *dlm)
 {
@@ -982,13 +999,24 @@ unsigned int vsp1_dlm_irq_frame_end(struct vsp1_dl_manager *dlm)
 		goto done;
 
 	/*
+	 * If the active display list has the writeback flag set, the frame
+	 * completion marks the end of the writeback capture. Return the
+	 * VSP1_DL_FRAME_END_WRITEBACK flag and reset the display list's
+	 * writeback flag.
+	 */
+	if (dlm->active && (dlm->active->flags & VSP1_DL_FRAME_END_WRITEBACK)) {
+		flags |= VSP1_DL_FRAME_END_WRITEBACK;
+		dlm->active->flags &= ~VSP1_DL_FRAME_END_WRITEBACK;
+	}
+
+	/*
 	 * The device starts processing the queued display list right after the
 	 * frame end interrupt. The display list thus becomes active.
 	 */
 	if (dlm->queued) {
-		if (dlm->queued->internal)
+		if (dlm->queued->flags & VSP1_DL_FRAME_END_INTERNAL)
 			flags |= VSP1_DL_FRAME_END_INTERNAL;
-		dlm->queued->internal = false;
+		dlm->queued->flags &= ~VSP1_DL_FRAME_END_INTERNAL;
 
 		__vsp1_dl_list_put(dlm->active);
 		dlm->active = dlm->queued;
diff --git a/drivers/media/platform/vsp1/vsp1_dl.h b/drivers/media/platform/vsp1/vsp1_dl.h
index 125750dc8b5c..bebe16483ca5 100644
--- a/drivers/media/platform/vsp1/vsp1_dl.h
+++ b/drivers/media/platform/vsp1/vsp1_dl.h
@@ -17,8 +17,10 @@ struct vsp1_dl_body_pool;
 struct vsp1_dl_list;
 struct vsp1_dl_manager;
 
+/* Keep these flags in sync with VSP1_DU_STATUS_* in include/media/vsp1.h. */
 #define VSP1_DL_FRAME_END_COMPLETED		BIT(0)
-#define VSP1_DL_FRAME_END_INTERNAL		BIT(1)
+#define VSP1_DL_FRAME_END_WRITEBACK		BIT(1)
+#define VSP1_DL_FRAME_END_INTERNAL		BIT(2)
 
 /**
  * struct vsp1_dl_ext_cmd - Extended Display command
@@ -61,7 +63,7 @@ struct vsp1_dl_list *vsp1_dl_list_get(struct vsp1_dl_manager *dlm);
 void vsp1_dl_list_put(struct vsp1_dl_list *dl);
 struct vsp1_dl_body *vsp1_dl_list_get_body0(struct vsp1_dl_list *dl);
 struct vsp1_dl_ext_cmd *vsp1_dl_get_pre_cmd(struct vsp1_dl_list *dl);
-void vsp1_dl_list_commit(struct vsp1_dl_list *dl, bool internal);
+void vsp1_dl_list_commit(struct vsp1_dl_list *dl, unsigned int dl_flags);
 
 struct vsp1_dl_body_pool *
 vsp1_dl_body_pool_create(struct vsp1_device *vsp1, unsigned int num_bodies,
diff --git a/drivers/media/platform/vsp1/vsp1_drm.c b/drivers/media/platform/vsp1/vsp1_drm.c
index 84895385d2e5..a4a45d68a6ef 100644
--- a/drivers/media/platform/vsp1/vsp1_drm.c
+++ b/drivers/media/platform/vsp1/vsp1_drm.c
@@ -34,14 +34,16 @@ static void vsp1_du_pipeline_frame_end(struct vsp1_pipeline *pipe,
 				       unsigned int completion)
 {
 	struct vsp1_drm_pipeline *drm_pipe = to_vsp1_drm_pipeline(pipe);
-	bool complete = completion == VSP1_DL_FRAME_END_COMPLETED;
 
 	if (drm_pipe->du_complete) {
 		struct vsp1_entity *uif = drm_pipe->uif;
+		unsigned int status = completion
+				    & (VSP1_DU_STATUS_COMPLETE |
+				       VSP1_DU_STATUS_WRITEBACK);
 		u32 crc;
 
 		crc = uif ? vsp1_uif_get_crc(to_uif(&uif->subdev)) : 0;
-		drm_pipe->du_complete(drm_pipe->du_private, complete, crc);
+		drm_pipe->du_complete(drm_pipe->du_private, status, crc);
 	}
 
 	if (completion & VSP1_DL_FRAME_END_INTERNAL) {
@@ -537,6 +539,12 @@ static void vsp1_du_pipeline_configure(struct vsp1_pipeline *pipe)
 	struct vsp1_entity *next;
 	struct vsp1_dl_list *dl;
 	struct vsp1_dl_body *dlb;
+	unsigned int dl_flags = 0;
+
+	if (drm_pipe->force_brx_release)
+		dl_flags |= VSP1_DL_FRAME_END_INTERNAL;
+	if (pipe->output->writeback)
+		dl_flags |= VSP1_DL_FRAME_END_WRITEBACK;
 
 	dl = vsp1_dl_list_get(pipe->output->dlm);
 	dlb = vsp1_dl_list_get_body0(dl);
@@ -554,12 +562,42 @@ static void vsp1_du_pipeline_configure(struct vsp1_pipeline *pipe)
 		}
 
 		vsp1_entity_route_setup(entity, pipe, dlb);
-		vsp1_entity_configure_stream(entity, pipe, dlb);
+		vsp1_entity_configure_stream(entity, pipe, dl, dlb);
 		vsp1_entity_configure_frame(entity, pipe, dl, dlb);
 		vsp1_entity_configure_partition(entity, pipe, dl, dlb);
 	}
 
-	vsp1_dl_list_commit(dl, drm_pipe->force_brx_release);
+	vsp1_dl_list_commit(dl, dl_flags);
+}
+
+static int vsp1_du_pipeline_set_rwpf_format(struct vsp1_device *vsp1,
+					    struct vsp1_rwpf *rwpf,
+					    u32 pixelformat, unsigned int pitch)
+{
+	const struct vsp1_format_info *fmtinfo;
+	unsigned int chroma_hsub;
+
+	fmtinfo = vsp1_get_format_info(vsp1, pixelformat);
+	if (!fmtinfo) {
+		dev_dbg(vsp1->dev, "Unsupported pixel format %08x\n",
+			pixelformat);
+		return -EINVAL;
+	}
+
+	/*
+	 * Only formats with three planes can affect the chroma planes pitch.
+	 * All formats with two planes have a horizontal subsampling value of 2,
+	 * but combine U and V in a single chroma plane, which thus results in
+	 * the luma plane and chroma plane having the same pitch.
+	 */
+	chroma_hsub = (fmtinfo->planes == 3) ? fmtinfo->hsub : 1;
+
+	rwpf->fmtinfo = fmtinfo;
+	rwpf->format.num_planes = fmtinfo->planes;
+	rwpf->format.plane_fmt[0].bytesperline = pitch;
+	rwpf->format.plane_fmt[1].bytesperline = pitch / chroma_hsub;
+
+	return 0;
 }
 
 /* -----------------------------------------------------------------------------
@@ -700,8 +738,8 @@ int vsp1_du_setup_lif(struct device *dev, unsigned int pipe_index,
 	drm_pipe->du_private = cfg->callback_data;
 
 	/* Disable the display interrupts. */
-	vsp1_write(vsp1, VI6_DISP_IRQ_STA, 0);
-	vsp1_write(vsp1, VI6_DISP_IRQ_ENB, 0);
+	vsp1_write(vsp1, VI6_DISP_IRQ_STA(pipe_index), 0);
+	vsp1_write(vsp1, VI6_DISP_IRQ_ENB(pipe_index), 0);
 
 	/* Configure all entities in the pipeline. */
 	vsp1_du_pipeline_configure(pipe);
@@ -769,9 +807,8 @@ int vsp1_du_atomic_update(struct device *dev, unsigned int pipe_index,
 {
 	struct vsp1_device *vsp1 = dev_get_drvdata(dev);
 	struct vsp1_drm_pipeline *drm_pipe = &vsp1->drm->pipe[pipe_index];
-	const struct vsp1_format_info *fmtinfo;
-	unsigned int chroma_hsub;
 	struct vsp1_rwpf *rpf;
+	int ret;
 
 	if (rpf_index >= vsp1->info->rpf_count)
 		return -EINVAL;
@@ -804,25 +841,11 @@ int vsp1_du_atomic_update(struct device *dev, unsigned int pipe_index,
 	 * Store the format, stride, memory buffer address, crop and compose
 	 * rectangles and Z-order position and for the input.
 	 */
-	fmtinfo = vsp1_get_format_info(vsp1, cfg->pixelformat);
-	if (!fmtinfo) {
-		dev_dbg(vsp1->dev, "Unsupported pixel format %08x for RPF\n",
-			cfg->pixelformat);
-		return -EINVAL;
-	}
-
-	/*
-	 * Only formats with three planes can affect the chroma planes pitch.
-	 * All formats with two planes have a horizontal subsampling value of 2,
-	 * but combine U and V in a single chroma plane, which thus results in
-	 * the luma plane and chroma plane having the same pitch.
-	 */
-	chroma_hsub = (fmtinfo->planes == 3) ? fmtinfo->hsub : 1;
+	ret = vsp1_du_pipeline_set_rwpf_format(vsp1, rpf, cfg->pixelformat,
+					       cfg->pitch);
+	if (ret < 0)
+		return ret;
 
-	rpf->fmtinfo = fmtinfo;
-	rpf->format.num_planes = fmtinfo->planes;
-	rpf->format.plane_fmt[0].bytesperline = cfg->pitch;
-	rpf->format.plane_fmt[1].bytesperline = cfg->pitch / chroma_hsub;
 	rpf->alpha = cfg->alpha;
 
 	rpf->mem.addr[0] = cfg->mem[0];
@@ -851,12 +874,31 @@ void vsp1_du_atomic_flush(struct device *dev, unsigned int pipe_index,
 	struct vsp1_device *vsp1 = dev_get_drvdata(dev);
 	struct vsp1_drm_pipeline *drm_pipe = &vsp1->drm->pipe[pipe_index];
 	struct vsp1_pipeline *pipe = &drm_pipe->pipe;
+	int ret;
 
 	drm_pipe->crc = cfg->crc;
 
 	mutex_lock(&vsp1->drm->lock);
+
+	if (cfg->writeback.pixelformat) {
+		const struct vsp1_du_writeback_config *wb_cfg = &cfg->writeback;
+
+		ret = vsp1_du_pipeline_set_rwpf_format(vsp1, pipe->output,
+						       wb_cfg->pixelformat,
+						       wb_cfg->pitch);
+		if (WARN_ON(ret < 0))
+			goto done;
+
+		pipe->output->mem.addr[0] = wb_cfg->mem[0];
+		pipe->output->mem.addr[1] = wb_cfg->mem[1];
+		pipe->output->mem.addr[2] = wb_cfg->mem[2];
+		pipe->output->writeback = true;
+	}
+
 	vsp1_du_pipeline_setup_inputs(vsp1, pipe);
 	vsp1_du_pipeline_configure(pipe);
+
+done:
 	mutex_unlock(&vsp1->drm->lock);
 }
 EXPORT_SYMBOL_GPL(vsp1_du_atomic_flush);
diff --git a/drivers/media/platform/vsp1/vsp1_drm.h b/drivers/media/platform/vsp1/vsp1_drm.h
index 8dfd274a59e2..e85ad4366fbb 100644
--- a/drivers/media/platform/vsp1/vsp1_drm.h
+++ b/drivers/media/platform/vsp1/vsp1_drm.h
@@ -42,7 +42,7 @@ struct vsp1_drm_pipeline {
 	struct vsp1_du_crc_config crc;
 
 	/* Frame synchronisation */
-	void (*du_complete)(void *data, bool completed, u32 crc);
+	void (*du_complete)(void *data, unsigned int status, u32 crc);
 	void *du_private;
 };
 
diff --git a/drivers/media/platform/vsp1/vsp1_entity.c b/drivers/media/platform/vsp1/vsp1_entity.c
index a54ab528b060..aa9d2286056e 100644
--- a/drivers/media/platform/vsp1/vsp1_entity.c
+++ b/drivers/media/platform/vsp1/vsp1_entity.c
@@ -71,10 +71,11 @@ void vsp1_entity_route_setup(struct vsp1_entity *entity,
 
 void vsp1_entity_configure_stream(struct vsp1_entity *entity,
 				  struct vsp1_pipeline *pipe,
+				  struct vsp1_dl_list *dl,
 				  struct vsp1_dl_body *dlb)
 {
 	if (entity->ops->configure_stream)
-		entity->ops->configure_stream(entity, pipe, dlb);
+		entity->ops->configure_stream(entity, pipe, dl, dlb);
 }
 
 void vsp1_entity_configure_frame(struct vsp1_entity *entity,
diff --git a/drivers/media/platform/vsp1/vsp1_entity.h b/drivers/media/platform/vsp1/vsp1_entity.h
index 97acb7795cf1..a1ceb37bb837 100644
--- a/drivers/media/platform/vsp1/vsp1_entity.h
+++ b/drivers/media/platform/vsp1/vsp1_entity.h
@@ -67,7 +67,9 @@ struct vsp1_route {
  * struct vsp1_entity_operations - Entity operations
  * @destroy:	Destroy the entity.
  * @configure_stream:	Setup the hardware parameters for the stream which do
- *			not vary between frames (pipeline, formats).
+ *			not vary between frames (pipeline, formats). Note that
+ *			the vsp1_dl_list argument is only valid for display
+ *			pipeline and will be NULL for mem-to-mem pipelines.
  * @configure_frame:	Configure the runtime parameters for each frame.
  * @configure_partition: Configure partition specific parameters.
  * @max_width:	Return the max supported width of data that the entity can
@@ -78,7 +80,7 @@ struct vsp1_route {
 struct vsp1_entity_operations {
 	void (*destroy)(struct vsp1_entity *);
 	void (*configure_stream)(struct vsp1_entity *, struct vsp1_pipeline *,
-				 struct vsp1_dl_body *);
+				 struct vsp1_dl_list *, struct vsp1_dl_body *);
 	void (*configure_frame)(struct vsp1_entity *, struct vsp1_pipeline *,
 				struct vsp1_dl_list *, struct vsp1_dl_body *);
 	void (*configure_partition)(struct vsp1_entity *,
@@ -155,6 +157,7 @@ void vsp1_entity_route_setup(struct vsp1_entity *entity,
 
 void vsp1_entity_configure_stream(struct vsp1_entity *entity,
 				  struct vsp1_pipeline *pipe,
+				  struct vsp1_dl_list *dl,
 				  struct vsp1_dl_body *dlb);
 
 void vsp1_entity_configure_frame(struct vsp1_entity *entity,
diff --git a/drivers/media/platform/vsp1/vsp1_hgo.c b/drivers/media/platform/vsp1/vsp1_hgo.c
index 827373c25351..bf3f981f93a1 100644
--- a/drivers/media/platform/vsp1/vsp1_hgo.c
+++ b/drivers/media/platform/vsp1/vsp1_hgo.c
@@ -131,6 +131,7 @@ static const struct v4l2_ctrl_config hgo_num_bins_control = {
 
 static void hgo_configure_stream(struct vsp1_entity *entity,
 				 struct vsp1_pipeline *pipe,
+				 struct vsp1_dl_list *dl,
 				 struct vsp1_dl_body *dlb)
 {
 	struct vsp1_hgo *hgo = to_hgo(&entity->subdev);
diff --git a/drivers/media/platform/vsp1/vsp1_hgt.c b/drivers/media/platform/vsp1/vsp1_hgt.c
index bb6ce6fdd5f4..aa1c718e0453 100644
--- a/drivers/media/platform/vsp1/vsp1_hgt.c
+++ b/drivers/media/platform/vsp1/vsp1_hgt.c
@@ -127,6 +127,7 @@ static const struct v4l2_ctrl_config hgt_hue_areas = {
 
 static void hgt_configure_stream(struct vsp1_entity *entity,
 				 struct vsp1_pipeline *pipe,
+				 struct vsp1_dl_list *dl,
 				 struct vsp1_dl_body *dlb)
 {
 	struct vsp1_hgt *hgt = to_hgt(&entity->subdev);
diff --git a/drivers/media/platform/vsp1/vsp1_hsit.c b/drivers/media/platform/vsp1/vsp1_hsit.c
index 39ab2e0c7c18..d5ebd9d08c8a 100644
--- a/drivers/media/platform/vsp1/vsp1_hsit.c
+++ b/drivers/media/platform/vsp1/vsp1_hsit.c
@@ -129,6 +129,7 @@ static const struct v4l2_subdev_ops hsit_ops = {
 
 static void hsit_configure_stream(struct vsp1_entity *entity,
 				  struct vsp1_pipeline *pipe,
+				  struct vsp1_dl_list *dl,
 				  struct vsp1_dl_body *dlb)
 {
 	struct vsp1_hsit *hsit = to_hsit(&entity->subdev);
diff --git a/drivers/media/platform/vsp1/vsp1_lif.c b/drivers/media/platform/vsp1/vsp1_lif.c
index 8b0a26335d70..14ed5d7bd061 100644
--- a/drivers/media/platform/vsp1/vsp1_lif.c
+++ b/drivers/media/platform/vsp1/vsp1_lif.c
@@ -84,6 +84,7 @@ static const struct v4l2_subdev_ops lif_ops = {
 
 static void lif_configure_stream(struct vsp1_entity *entity,
 				 struct vsp1_pipeline *pipe,
+				 struct vsp1_dl_list *dl,
 				 struct vsp1_dl_body *dlb)
 {
 	const struct v4l2_mbus_framefmt *format;
diff --git a/drivers/media/platform/vsp1/vsp1_lut.c b/drivers/media/platform/vsp1/vsp1_lut.c
index 64c48d9459b0..9f88842d7048 100644
--- a/drivers/media/platform/vsp1/vsp1_lut.c
+++ b/drivers/media/platform/vsp1/vsp1_lut.c
@@ -147,6 +147,7 @@ static const struct v4l2_subdev_ops lut_ops = {
 
 static void lut_configure_stream(struct vsp1_entity *entity,
 				 struct vsp1_pipeline *pipe,
+				 struct vsp1_dl_list *dl,
 				 struct vsp1_dl_body *dlb)
 {
 	struct vsp1_lut *lut = to_lut(&entity->subdev);
diff --git a/drivers/media/platform/vsp1/vsp1_pipe.c b/drivers/media/platform/vsp1/vsp1_pipe.c
index 54ff539ffea0..f72ac01c21ea 100644
--- a/drivers/media/platform/vsp1/vsp1_pipe.c
+++ b/drivers/media/platform/vsp1/vsp1_pipe.c
@@ -42,6 +42,30 @@ static const struct vsp1_format_info vsp1_video_formats[] = {
 	  VI6_FMT_XRGB_4444, VI6_RPF_DSWAP_P_LLS | VI6_RPF_DSWAP_P_LWS |
 	  VI6_RPF_DSWAP_P_WDS,
 	  1, { 16, 0, 0 }, false, false, 1, 1, false },
+	{ V4L2_PIX_FMT_RGBA444, MEDIA_BUS_FMT_ARGB8888_1X32,
+	  VI6_FMT_RGBA_4444, VI6_RPF_DSWAP_P_LLS | VI6_RPF_DSWAP_P_LWS |
+	  VI6_RPF_DSWAP_P_WDS,
+	  1, { 16, 0, 0 }, false, false, 1, 1, true },
+	{ V4L2_PIX_FMT_RGBX444, MEDIA_BUS_FMT_ARGB8888_1X32,
+	  VI6_FMT_RGBX_4444, VI6_RPF_DSWAP_P_LLS | VI6_RPF_DSWAP_P_LWS |
+	  VI6_RPF_DSWAP_P_WDS,
+	  1, { 16, 0, 0 }, false, false, 1, 1, false },
+	{ V4L2_PIX_FMT_ABGR444, MEDIA_BUS_FMT_ARGB8888_1X32,
+	  VI6_FMT_ABGR_4444, VI6_RPF_DSWAP_P_LLS | VI6_RPF_DSWAP_P_LWS |
+	  VI6_RPF_DSWAP_P_WDS,
+	  1, { 16, 0, 0 }, false, false, 1, 1, true },
+	{ V4L2_PIX_FMT_XBGR444, MEDIA_BUS_FMT_ARGB8888_1X32,
+	  VI6_FMT_ABGR_4444, VI6_RPF_DSWAP_P_LLS | VI6_RPF_DSWAP_P_LWS |
+	  VI6_RPF_DSWAP_P_WDS,
+	  1, { 16, 0, 0 }, false, false, 1, 1, false },
+	{ V4L2_PIX_FMT_BGRA444, MEDIA_BUS_FMT_ARGB8888_1X32,
+	  VI6_FMT_BGRA_4444, VI6_RPF_DSWAP_P_LLS | VI6_RPF_DSWAP_P_LWS |
+	  VI6_RPF_DSWAP_P_WDS,
+	  1, { 16, 0, 0 }, false, false, 1, 1, true },
+	{ V4L2_PIX_FMT_BGRX444, MEDIA_BUS_FMT_ARGB8888_1X32,
+	  VI6_FMT_BGRA_4444, VI6_RPF_DSWAP_P_LLS | VI6_RPF_DSWAP_P_LWS |
+	  VI6_RPF_DSWAP_P_WDS,
+	  1, { 16, 0, 0 }, false, false, 1, 1, false },
 	{ V4L2_PIX_FMT_ARGB555, MEDIA_BUS_FMT_ARGB8888_1X32,
 	  VI6_FMT_ARGB_1555, VI6_RPF_DSWAP_P_LLS | VI6_RPF_DSWAP_P_LWS |
 	  VI6_RPF_DSWAP_P_WDS,
@@ -50,6 +74,30 @@ static const struct vsp1_format_info vsp1_video_formats[] = {
 	  VI6_FMT_XRGB_1555, VI6_RPF_DSWAP_P_LLS | VI6_RPF_DSWAP_P_LWS |
 	  VI6_RPF_DSWAP_P_WDS,
 	  1, { 16, 0, 0 }, false, false, 1, 1, false },
+	{ V4L2_PIX_FMT_RGBA555, MEDIA_BUS_FMT_ARGB8888_1X32,
+	  VI6_FMT_RGBA_5551, VI6_RPF_DSWAP_P_LLS | VI6_RPF_DSWAP_P_LWS |
+	  VI6_RPF_DSWAP_P_WDS,
+	  1, { 16, 0, 0 }, false, false, 1, 1, true },
+	{ V4L2_PIX_FMT_RGBX555, MEDIA_BUS_FMT_ARGB8888_1X32,
+	  VI6_FMT_RGBX_5551, VI6_RPF_DSWAP_P_LLS | VI6_RPF_DSWAP_P_LWS |
+	  VI6_RPF_DSWAP_P_WDS,
+	  1, { 16, 0, 0 }, false, false, 1, 1, false },
+	{ V4L2_PIX_FMT_ABGR555, MEDIA_BUS_FMT_ARGB8888_1X32,
+	  VI6_FMT_ABGR_1555, VI6_RPF_DSWAP_P_LLS | VI6_RPF_DSWAP_P_LWS |
+	  VI6_RPF_DSWAP_P_WDS,
+	  1, { 16, 0, 0 }, false, false, 1, 1, true },
+	{ V4L2_PIX_FMT_XBGR555, MEDIA_BUS_FMT_ARGB8888_1X32,
+	  VI6_FMT_ABGR_1555, VI6_RPF_DSWAP_P_LLS | VI6_RPF_DSWAP_P_LWS |
+	  VI6_RPF_DSWAP_P_WDS,
+	  1, { 16, 0, 0 }, false, false, 1, 1, false },
+	{ V4L2_PIX_FMT_BGRA555, MEDIA_BUS_FMT_ARGB8888_1X32,
+	  VI6_FMT_BGRA_5551, VI6_RPF_DSWAP_P_LLS | VI6_RPF_DSWAP_P_LWS |
+	  VI6_RPF_DSWAP_P_WDS,
+	  1, { 16, 0, 0 }, false, false, 1, 1, true },
+	{ V4L2_PIX_FMT_BGRX555, MEDIA_BUS_FMT_ARGB8888_1X32,
+	  VI6_FMT_BGRA_5551, VI6_RPF_DSWAP_P_LLS | VI6_RPF_DSWAP_P_LWS |
+	  VI6_RPF_DSWAP_P_WDS,
+	  1, { 16, 0, 0 }, false, false, 1, 1, false },
 	{ V4L2_PIX_FMT_RGB565, MEDIA_BUS_FMT_ARGB8888_1X32,
 	  VI6_FMT_RGB_565, VI6_RPF_DSWAP_P_LLS | VI6_RPF_DSWAP_P_LWS |
 	  VI6_RPF_DSWAP_P_WDS,
@@ -68,6 +116,20 @@ static const struct vsp1_format_info vsp1_video_formats[] = {
 	{ V4L2_PIX_FMT_XBGR32, MEDIA_BUS_FMT_ARGB8888_1X32,
 	  VI6_FMT_ARGB_8888, VI6_RPF_DSWAP_P_LLS | VI6_RPF_DSWAP_P_LWS,
 	  1, { 32, 0, 0 }, false, false, 1, 1, false },
+	{ V4L2_PIX_FMT_BGRA32, MEDIA_BUS_FMT_ARGB8888_1X32,
+	  VI6_FMT_RGBA_8888, VI6_RPF_DSWAP_P_LLS | VI6_RPF_DSWAP_P_LWS,
+	  1, { 32, 0, 0 }, false, false, 1, 1, true },
+	{ V4L2_PIX_FMT_BGRX32, MEDIA_BUS_FMT_ARGB8888_1X32,
+	  VI6_FMT_RGBA_8888, VI6_RPF_DSWAP_P_LLS | VI6_RPF_DSWAP_P_LWS,
+	  1, { 32, 0, 0 }, false, false, 1, 1, false },
+	{ V4L2_PIX_FMT_RGBA32, MEDIA_BUS_FMT_ARGB8888_1X32,
+	  VI6_FMT_RGBA_8888, VI6_RPF_DSWAP_P_LLS | VI6_RPF_DSWAP_P_LWS |
+	  VI6_RPF_DSWAP_P_WDS | VI6_RPF_DSWAP_P_BTS,
+	  1, { 32, 0, 0 }, false, false, 1, 1, true },
+	{ V4L2_PIX_FMT_RGBX32, MEDIA_BUS_FMT_ARGB8888_1X32,
+	  VI6_FMT_RGBA_8888, VI6_RPF_DSWAP_P_LLS | VI6_RPF_DSWAP_P_LWS |
+	  VI6_RPF_DSWAP_P_WDS | VI6_RPF_DSWAP_P_BTS,
+	  1, { 32, 0, 0 }, false, false, 1, 1, false },
 	{ V4L2_PIX_FMT_ARGB32, MEDIA_BUS_FMT_ARGB8888_1X32,
 	  VI6_FMT_ARGB_8888, VI6_RPF_DSWAP_P_LLS | VI6_RPF_DSWAP_P_LWS |
 	  VI6_RPF_DSWAP_P_WDS | VI6_RPF_DSWAP_P_BTS,
diff --git a/drivers/media/platform/vsp1/vsp1_regs.h b/drivers/media/platform/vsp1/vsp1_regs.h
index f6e4157095cc..1bb1d39c60d9 100644
--- a/drivers/media/platform/vsp1/vsp1_regs.h
+++ b/drivers/media/platform/vsp1/vsp1_regs.h
@@ -39,12 +39,12 @@
 #define VI6_WFP_IRQ_STA_DFE		(1 << 1)
 #define VI6_WFP_IRQ_STA_FRE		(1 << 0)
 
-#define VI6_DISP_IRQ_ENB		0x0078
+#define VI6_DISP_IRQ_ENB(n)		(0x0078 + (n) * 60)
 #define VI6_DISP_IRQ_ENB_DSTE		(1 << 8)
 #define VI6_DISP_IRQ_ENB_MAEE		(1 << 5)
 #define VI6_DISP_IRQ_ENB_LNEE(n)	(1 << (n))
 
-#define VI6_DISP_IRQ_STA		0x007c
+#define VI6_DISP_IRQ_STA(n)		(0x007c + (n) * 60)
 #define VI6_DISP_IRQ_STA_DST		(1 << 8)
 #define VI6_DISP_IRQ_STA_MAE		(1 << 5)
 #define VI6_DISP_IRQ_STA_LNE(n)		(1 << (n))
@@ -307,7 +307,7 @@
 #define VI6_WPF_DSTM_ADDR_C0		0x1028
 #define VI6_WPF_DSTM_ADDR_C1		0x102c
 
-#define VI6_WPF_WRBCK_CTRL		0x1034
+#define VI6_WPF_WRBCK_CTRL(n)		(0x1034 + (n) * 0x100)
 #define VI6_WPF_WRBCK_CTRL_WBMD		(1 << 0)
 
 /* -----------------------------------------------------------------------------
diff --git a/drivers/media/platform/vsp1/vsp1_rpf.c b/drivers/media/platform/vsp1/vsp1_rpf.c
index 616afa7e165f..85587c1b6a37 100644
--- a/drivers/media/platform/vsp1/vsp1_rpf.c
+++ b/drivers/media/platform/vsp1/vsp1_rpf.c
@@ -57,6 +57,7 @@ static const struct v4l2_subdev_ops rpf_ops = {
 
 static void rpf_configure_stream(struct vsp1_entity *entity,
 				 struct vsp1_pipeline *pipe,
+				 struct vsp1_dl_list *dl,
 				 struct vsp1_dl_body *dlb)
 {
 	struct vsp1_rwpf *rpf = to_rwpf(&entity->subdev);
diff --git a/drivers/media/platform/vsp1/vsp1_rwpf.h b/drivers/media/platform/vsp1/vsp1_rwpf.h
index 70742ecf766f..2f3582590618 100644
--- a/drivers/media/platform/vsp1/vsp1_rwpf.h
+++ b/drivers/media/platform/vsp1/vsp1_rwpf.h
@@ -61,6 +61,7 @@ struct vsp1_rwpf {
 	} flip;
 
 	struct vsp1_rwpf_memory mem;
+	bool writeback;
 
 	struct vsp1_dl_manager *dlm;
 };
diff --git a/drivers/media/platform/vsp1/vsp1_sru.c b/drivers/media/platform/vsp1/vsp1_sru.c
index b1617cb1f2b9..2b65457ee12f 100644
--- a/drivers/media/platform/vsp1/vsp1_sru.c
+++ b/drivers/media/platform/vsp1/vsp1_sru.c
@@ -269,6 +269,7 @@ static const struct v4l2_subdev_ops sru_ops = {
 
 static void sru_configure_stream(struct vsp1_entity *entity,
 				 struct vsp1_pipeline *pipe,
+				 struct vsp1_dl_list *dl,
 				 struct vsp1_dl_body *dlb)
 {
 	const struct vsp1_sru_param *param;
diff --git a/drivers/media/platform/vsp1/vsp1_uds.c b/drivers/media/platform/vsp1/vsp1_uds.c
index 27012af973b2..5fc04c082d1a 100644
--- a/drivers/media/platform/vsp1/vsp1_uds.c
+++ b/drivers/media/platform/vsp1/vsp1_uds.c
@@ -257,6 +257,7 @@ static const struct v4l2_subdev_ops uds_ops = {
 
 static void uds_configure_stream(struct vsp1_entity *entity,
 				 struct vsp1_pipeline *pipe,
+				 struct vsp1_dl_list *dl,
 				 struct vsp1_dl_body *dlb)
 {
 	struct vsp1_uds *uds = to_uds(&entity->subdev);
diff --git a/drivers/media/platform/vsp1/vsp1_uif.c b/drivers/media/platform/vsp1/vsp1_uif.c
index 4b58d51df231..467d1072577b 100644
--- a/drivers/media/platform/vsp1/vsp1_uif.c
+++ b/drivers/media/platform/vsp1/vsp1_uif.c
@@ -192,6 +192,7 @@ static const struct v4l2_subdev_ops uif_ops = {
 
 static void uif_configure_stream(struct vsp1_entity *entity,
 				 struct vsp1_pipeline *pipe,
+				 struct vsp1_dl_list *dl,
 				 struct vsp1_dl_body *dlb)
 {
 	struct vsp1_uif *uif = to_uif(&entity->subdev);
diff --git a/drivers/media/platform/vsp1/vsp1_video.c b/drivers/media/platform/vsp1/vsp1_video.c
index 7ceaf3222145..fd98e483b2f4 100644
--- a/drivers/media/platform/vsp1/vsp1_video.c
+++ b/drivers/media/platform/vsp1/vsp1_video.c
@@ -307,11 +307,6 @@ static int vsp1_video_pipeline_setup_partitions(struct vsp1_pipeline *pipe)
  * This function completes the current buffer by filling its sequence number,
  * time stamp and payload size, and hands it back to the videobuf core.
  *
- * When operating in DU output mode (deep pipeline to the DU through the LIF),
- * the VSP1 needs to constantly supply frames to the display. In that case, if
- * no other buffer is queued, reuse the one that has just been processed instead
- * of handing it back to the videobuf core.
- *
  * Return the next queued buffer or NULL if the queue is empty.
  */
 static struct vsp1_vb2_buffer *
@@ -333,12 +328,6 @@ vsp1_video_complete_buffer(struct vsp1_video *video)
 	done = list_first_entry(&video->irqqueue,
 				struct vsp1_vb2_buffer, queue);
 
-	/* In DU output mode reuse the buffer if the list is singular. */
-	if (pipe->lif && list_is_singular(&video->irqqueue)) {
-		spin_unlock_irqrestore(&video->irqlock, flags);
-		return done;
-	}
-
 	list_del(&done->queue);
 
 	if (!list_empty(&video->irqqueue))
@@ -432,7 +421,7 @@ static void vsp1_video_pipeline_run(struct vsp1_pipeline *pipe)
 	}
 
 	/* Complete, and commit the head display list. */
-	vsp1_dl_list_commit(dl, false);
+	vsp1_dl_list_commit(dl, 0);
 	pipe->configured = true;
 
 	vsp1_pipeline_run(pipe);
@@ -836,7 +825,8 @@ static int vsp1_video_setup_pipeline(struct vsp1_pipeline *pipe)
 
 	list_for_each_entry(entity, &pipe->entities, list_pipe) {
 		vsp1_entity_route_setup(entity, pipe, pipe->stream_config);
-		vsp1_entity_configure_stream(entity, pipe, pipe->stream_config);
+		vsp1_entity_configure_stream(entity, pipe, NULL,
+					     pipe->stream_config);
 	}
 
 	return 0;
diff --git a/drivers/media/platform/vsp1/vsp1_wpf.c b/drivers/media/platform/vsp1/vsp1_wpf.c
index 32bb207b2007..208498fa6ed7 100644
--- a/drivers/media/platform/vsp1/vsp1_wpf.c
+++ b/drivers/media/platform/vsp1/vsp1_wpf.c
@@ -232,17 +232,41 @@ static void vsp1_wpf_destroy(struct vsp1_entity *entity)
 	vsp1_dlm_destroy(wpf->dlm);
 }
 
+static int wpf_configure_writeback_chain(struct vsp1_rwpf *wpf,
+					 struct vsp1_dl_list *dl)
+{
+	unsigned int index = wpf->entity.index;
+	struct vsp1_dl_list *dl_next;
+	struct vsp1_dl_body *dlb;
+
+	dl_next = vsp1_dl_list_get(wpf->dlm);
+	if (!dl_next) {
+		dev_err(wpf->entity.vsp1->dev,
+			"Failed to obtain a dl list, disabling writeback\n");
+		return -ENOMEM;
+	}
+
+	dlb = vsp1_dl_list_get_body0(dl_next);
+	vsp1_dl_body_write(dlb, VI6_WPF_WRBCK_CTRL(index), 0);
+	vsp1_dl_list_add_chain(dl, dl_next);
+
+	return 0;
+}
+
 static void wpf_configure_stream(struct vsp1_entity *entity,
 				 struct vsp1_pipeline *pipe,
+				 struct vsp1_dl_list *dl,
 				 struct vsp1_dl_body *dlb)
 {
 	struct vsp1_rwpf *wpf = to_rwpf(&entity->subdev);
 	struct vsp1_device *vsp1 = wpf->entity.vsp1;
 	const struct v4l2_mbus_framefmt *source_format;
 	const struct v4l2_mbus_framefmt *sink_format;
+	unsigned int index = wpf->entity.index;
 	unsigned int i;
 	u32 outfmt = 0;
 	u32 srcrpf = 0;
+	int ret;
 
 	sink_format = vsp1_entity_get_pad_format(&wpf->entity,
 						 wpf->entity.config,
@@ -250,8 +274,9 @@ static void wpf_configure_stream(struct vsp1_entity *entity,
 	source_format = vsp1_entity_get_pad_format(&wpf->entity,
 						   wpf->entity.config,
 						   RWPF_PAD_SOURCE);
+
 	/* Format */
-	if (!pipe->lif) {
+	if (!pipe->lif || wpf->writeback) {
 		const struct v4l2_pix_format_mplane *format = &wpf->format;
 		const struct vsp1_format_info *fmtinfo = wpf->fmtinfo;
 
@@ -276,8 +301,7 @@ static void wpf_configure_stream(struct vsp1_entity *entity,
 
 		vsp1_wpf_write(wpf, dlb, VI6_WPF_DSWAP, fmtinfo->swap);
 
-		if (vsp1_feature(vsp1, VSP1_HAS_WPF_HFLIP) &&
-		    wpf->entity.index == 0)
+		if (vsp1_feature(vsp1, VSP1_HAS_WPF_HFLIP) && index == 0)
 			vsp1_wpf_write(wpf, dlb, VI6_WPF_ROT_CTRL,
 				       VI6_WPF_ROT_CTRL_LN16 |
 				       (256 << VI6_WPF_ROT_CTRL_LMEM_WD_SHIFT));
@@ -288,11 +312,9 @@ static void wpf_configure_stream(struct vsp1_entity *entity,
 
 	wpf->outfmt = outfmt;
 
-	vsp1_dl_body_write(dlb, VI6_DPR_WPF_FPORCH(wpf->entity.index),
+	vsp1_dl_body_write(dlb, VI6_DPR_WPF_FPORCH(index),
 			   VI6_DPR_WPF_FPORCH_FP_WPFN);
 
-	vsp1_dl_body_write(dlb, VI6_WPF_WRBCK_CTRL, 0);
-
 	/*
 	 * Sources. If the pipeline has a single input and BRx is not used,
 	 * configure it as the master layer. Otherwise configure all
@@ -318,9 +340,26 @@ static void wpf_configure_stream(struct vsp1_entity *entity,
 	vsp1_wpf_write(wpf, dlb, VI6_WPF_SRCRPF, srcrpf);
 
 	/* Enable interrupts. */
-	vsp1_dl_body_write(dlb, VI6_WPF_IRQ_STA(wpf->entity.index), 0);
-	vsp1_dl_body_write(dlb, VI6_WPF_IRQ_ENB(wpf->entity.index),
+	vsp1_dl_body_write(dlb, VI6_WPF_IRQ_STA(index), 0);
+	vsp1_dl_body_write(dlb, VI6_WPF_IRQ_ENB(index),
 			   VI6_WFP_IRQ_ENB_DFEE);
+
+	/*
+	 * Configure writeback for display pipelines (the wpf writeback flag is
+	 * never set for memory-to-memory pipelines). Start by adding a chained
+	 * display list to disable writeback after a single frame, and process
+	 * to enable writeback. If the display list allocation fails don't
+	 * enable writeback as we wouldn't be able to safely disable it,
+	 * resulting in possible memory corruption.
+	 */
+	if (wpf->writeback) {
+		ret = wpf_configure_writeback_chain(wpf, dl);
+		if (ret < 0)
+			wpf->writeback = false;
+	}
+
+	vsp1_dl_body_write(dlb, VI6_WPF_WRBCK_CTRL(index),
+			   wpf->writeback ? VI6_WPF_WRBCK_CTRL_WBMD : 0);
 }
 
 static void wpf_configure_frame(struct vsp1_entity *entity,
@@ -362,6 +401,7 @@ static void wpf_configure_partition(struct vsp1_entity *entity,
 	const struct vsp1_format_info *fmtinfo = wpf->fmtinfo;
 	unsigned int width;
 	unsigned int height;
+	unsigned int left;
 	unsigned int offset;
 	unsigned int flip;
 	unsigned int i;
@@ -371,13 +411,16 @@ static void wpf_configure_partition(struct vsp1_entity *entity,
 						 RWPF_PAD_SINK);
 	width = sink_format->width;
 	height = sink_format->height;
+	left = 0;
 
 	/*
 	 * Cropping. The partition algorithm can split the image into
 	 * multiple slices.
 	 */
-	if (pipe->partitions > 1)
+	if (pipe->partitions > 1) {
 		width = pipe->partition->wpf.width;
+		left = pipe->partition->wpf.left;
+	}
 
 	vsp1_wpf_write(wpf, dlb, VI6_WPF_HSZCLIP, VI6_WPF_SZCLIP_EN |
 		       (0 << VI6_WPF_SZCLIP_OFST_SHIFT) |
@@ -386,7 +429,11 @@ static void wpf_configure_partition(struct vsp1_entity *entity,
 		       (0 << VI6_WPF_SZCLIP_OFST_SHIFT) |
 		       (height << VI6_WPF_SZCLIP_SIZE_SHIFT));
 
-	if (pipe->lif)
+	/*
+	 * For display pipelines without writeback enabled there's no memory
+	 * address to configure, return now.
+	 */
+	if (pipe->lif && !wpf->writeback)
 		return;
 
 	/*
@@ -408,13 +455,11 @@ static void wpf_configure_partition(struct vsp1_entity *entity,
 	flip = wpf->flip.active;
 
 	if (flip & BIT(WPF_CTRL_HFLIP) && !wpf->flip.rotate)
-		offset = format->width - pipe->partition->wpf.left
-			- pipe->partition->wpf.width;
+		offset = format->width - left - width;
 	else if (flip & BIT(WPF_CTRL_VFLIP) && wpf->flip.rotate)
-		offset = format->height - pipe->partition->wpf.left
-			- pipe->partition->wpf.width;
+		offset = format->height - left - width;
 	else
-		offset = pipe->partition->wpf.left;
+		offset = left;
 
 	for (i = 0; i < format->num_planes; ++i) {
 		unsigned int hsub = i > 0 ? fmtinfo->hsub : 1;
@@ -436,7 +481,7 @@ static void wpf_configure_partition(struct vsp1_entity *entity,
 		 * image height.
 		 */
 		if (wpf->flip.rotate)
-			height = pipe->partition->wpf.width;
+			height = width;
 		else
 			height = format->height;
 
@@ -477,6 +522,12 @@ static void wpf_configure_partition(struct vsp1_entity *entity,
 	vsp1_wpf_write(wpf, dlb, VI6_WPF_DSTM_ADDR_Y, mem.addr[0]);
 	vsp1_wpf_write(wpf, dlb, VI6_WPF_DSTM_ADDR_C0, mem.addr[1]);
 	vsp1_wpf_write(wpf, dlb, VI6_WPF_DSTM_ADDR_C1, mem.addr[2]);
+
+	/*
+	 * Writeback operates in single-shot mode and lasts for a single frame,
+	 * reset the writeback flag to false for the next frame.
+	 */
+	wpf->writeback = false;
 }
 
 static unsigned int wpf_max_width(struct vsp1_entity *entity,
diff --git a/drivers/media/platform/xilinx/Kconfig b/drivers/media/platform/xilinx/Kconfig
index 74ec8aaa5ae0..a2773ad7c185 100644
--- a/drivers/media/platform/xilinx/Kconfig
+++ b/drivers/media/platform/xilinx/Kconfig
@@ -5,7 +5,7 @@ config VIDEO_XILINX
 	depends on VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API && OF && HAS_DMA
 	select VIDEOBUF2_DMA_CONTIG
 	select V4L2_FWNODE
-	---help---
+	help
 	  Driver for Xilinx Video IP Pipelines
 
 if VIDEO_XILINX
@@ -14,13 +14,13 @@ config VIDEO_XILINX_TPG
 	tristate "Xilinx Video Test Pattern Generator"
 	depends on VIDEO_XILINX
 	select VIDEO_XILINX_VTC
-	---help---
+	help
 	   Driver for the Xilinx Video Test Pattern Generator
 
 config VIDEO_XILINX_VTC
 	tristate "Xilinx Video Timing Controller"
 	depends on VIDEO_XILINX
-	---help---
+	help
 	   Driver for the Xilinx Video Timing Controller
 
 endif #VIDEO_XILINX
diff --git a/drivers/media/radio/Kconfig b/drivers/media/radio/Kconfig
index 9b99dfb2d0c6..9cd00f64af32 100644
--- a/drivers/media/radio/Kconfig
+++ b/drivers/media/radio/Kconfig
@@ -7,7 +7,7 @@ menuconfig RADIO_ADAPTERS
 	depends on VIDEO_V4L2
 	depends on MEDIA_RADIO_SUPPORT
 	default y
-	---help---
+	help
 	  Say Y here to enable selecting AM/FM radio adapters.
 
 if RADIO_ADAPTERS && VIDEO_V4L2
@@ -29,7 +29,7 @@ config RADIO_SI476X
 	depends on MFD_SI476X_CORE
 	depends on SND_SOC
 	select SND_SOC_SI476X
-	---help---
+	help
 	  Choose Y here if you have this FM radio chip.
 
 	  In order to control your radio card, you will need to use programs
@@ -43,7 +43,7 @@ config RADIO_SI476X
 config USB_MR800
 	tristate "AverMedia MR 800 USB FM radio support"
 	depends on USB && VIDEO_V4L2
-	---help---
+	help
 	  Say Y here if you want to connect this type of radio to your
 	  computer's USB port. Note that the audio is not digital, and
 	  you must connect the line out connector to a sound card or a
@@ -55,7 +55,7 @@ config USB_MR800
 config USB_DSBR
 	tristate "D-Link/GemTek USB FM radio support"
 	depends on USB && VIDEO_V4L2
-	---help---
+	help
 	  Say Y here if you want to connect this type of radio to your
 	  computer's USB port. Note that the audio is not digital, and
 	  you must connect the line out connector to a sound card or a
@@ -68,7 +68,7 @@ config RADIO_MAXIRADIO
 	tristate "Guillemot MAXI Radio FM 2000 radio"
 	depends on VIDEO_V4L2 && PCI
 	select RADIO_TEA575X
-	---help---
+	help
 	  Choose Y here if you have this radio card.  This card may also be
 	  found as Gemtek PCI FM.
 
@@ -84,7 +84,7 @@ config RADIO_SHARK
 	tristate "Griffin radioSHARK USB radio receiver"
 	depends on USB
 	select RADIO_TEA575X
-	---help---
+	help
 	  Choose Y here if you have this radio receiver.
 
 	  There are 2 versions of this device, this driver is for version 1,
@@ -101,7 +101,7 @@ config RADIO_SHARK
 config RADIO_SHARK2
 	tristate "Griffin radioSHARK2 USB radio receiver"
 	depends on USB
-	---help---
+	help
 	  Choose Y here if you have this radio receiver.
 
 	  There are 2 versions of this device, this driver is for version 2,
@@ -118,7 +118,7 @@ config RADIO_SHARK2
 config USB_KEENE
 	tristate "Keene FM Transmitter USB support"
 	depends on USB && VIDEO_V4L2
-	---help---
+	help
 	  Say Y here if you want to connect this type of FM transmitter
 	  to your computer's USB port.
 
@@ -128,7 +128,7 @@ config USB_KEENE
 config USB_RAREMONO
 	tristate "Thanko's Raremono AM/FM/SW radio support"
 	depends on USB && VIDEO_V4L2
-	---help---
+	help
 	  The 'Thanko's Raremono' device contains the Si4734 chip from Silicon Labs Inc.
 	  It is one of the very few or perhaps the only consumer USB radio device
 	  to receive the AM/FM/SW bands.
@@ -142,7 +142,7 @@ config USB_RAREMONO
 config USB_MA901
 	tristate "Masterkit MA901 USB FM radio support"
 	depends on USB && VIDEO_V4L2
-	---help---
+	help
 	  Say Y here if you want to connect this type of radio to your
 	  computer's USB port. Note that the audio is not digital, and
 	  you must connect the line out connector to a sound card or a
@@ -154,7 +154,7 @@ config USB_MA901
 config RADIO_TEA5764
 	tristate "TEA5764 I2C FM radio support"
 	depends on I2C && VIDEO_V4L2
-	---help---
+	help
 	  Say Y here if you want to use the TEA5764 FM chip found in
 	  EZX phones. This FM chip is present in EZX phones from Motorola,
 	  connected to internal pxa I2C bus.
@@ -173,7 +173,7 @@ config RADIO_TEA5764_XTAL
 config RADIO_SAA7706H
 	tristate "SAA7706H Car Radio DSP"
 	depends on I2C && VIDEO_V4L2
-	---help---
+	help
 	  Say Y here if you want to use the SAA7706H Car radio Digital
 	  Signal Processor, found for instance on the Russellville development
 	  board. On the russellville the device is connected to internal
@@ -185,7 +185,7 @@ config RADIO_SAA7706H
 config RADIO_TEF6862
 	tristate "TEF6862 Car Radio Enhanced Selectivity Tuner"
 	depends on I2C && VIDEO_V4L2
-	---help---
+	help
 	  Say Y here if you want to use the TEF6862 Car Radio Enhanced
 	  Selectivity Tuner, found for instance on the Russellville development
 	  board. On the russellville the device is connected to internal
@@ -200,7 +200,7 @@ config RADIO_TIMBERDALE
 	depends on I2C	# for RADIO_SAA7706H
 	select RADIO_TEF6862
 	select RADIO_SAA7706H
-	---help---
+	help
 	  This is a kind of umbrella driver for the Radio Tuner and DSP
 	  found behind the Timberdale FPGA on the Russellville board.
 	  Enabling this driver will automatically select the DSP and tuner.
@@ -211,7 +211,7 @@ config RADIO_WL1273
 	select MFD_CORE
 	select MFD_WL1273_CORE
 	select FW_LOADER
-	---help---
+	help
 	  Choose Y here if you have this FM radio chip.
 
 	  In order to control your radio card, you will need to use programs
@@ -233,7 +233,7 @@ menuconfig V4L_RADIO_ISA_DRIVERS
 	bool "ISA radio devices"
 	depends on ISA || COMPILE_TEST
 	default n
-	---help---
+	help
 	  Say Y here to enable support for these ISA drivers.
 
 if V4L_RADIO_ISA_DRIVERS
@@ -246,7 +246,7 @@ config RADIO_CADET
 	tristate "ADS Cadet AM/FM Tuner"
 	depends on ISA || COMPILE_TEST
 	depends on VIDEO_V4L2
-	---help---
+	help
 	  Choose Y here if you have one of these AM/FM radio cards, and then
 	  fill in the port address below.
 
@@ -258,7 +258,7 @@ config RADIO_RTRACK
 	depends on ISA || COMPILE_TEST
 	depends on VIDEO_V4L2
 	select RADIO_ISA
-	---help---
+	help
 	  Choose Y here if you have one of these FM radio cards, and then fill
 	  in the port address below.
 
@@ -290,7 +290,7 @@ config RADIO_RTRACK2
 	depends on ISA || COMPILE_TEST
 	depends on VIDEO_V4L2
 	select RADIO_ISA
-	---help---
+	help
 	  Choose Y here if you have this FM radio card, and then fill in the
 	  port address below.
 
@@ -314,7 +314,7 @@ config RADIO_AZTECH
 	depends on ISA || COMPILE_TEST
 	depends on VIDEO_V4L2
 	select RADIO_ISA
-	---help---
+	help
 	  Choose Y here if you have one of these FM radio cards, and then fill
 	  in the port address below.
 
@@ -335,7 +335,7 @@ config RADIO_GEMTEK
 	depends on ISA || COMPILE_TEST
 	depends on VIDEO_V4L2
 	select RADIO_ISA
-	---help---
+	help
 	  Choose Y here if you have this FM radio card, and then fill in the
 	  I/O port address and settings below. The following cards either have
 	  GemTek Radio tuner or are rebranded GemTek Radio cards:
@@ -377,7 +377,7 @@ config RADIO_MIROPCM20
 	depends on ISA_DMA_API && VIDEO_V4L2 && SND
 	select SND_ISA
 	select SND_MIRO
-	---help---
+	help
 	  Choose Y here if you have this FM radio card. You also need to enable
 	  the ALSA sound system. This choice automatically selects the ALSA
 	  sound card driver "Miro miroSOUND PCM1pro/PCM12/PCM20radio" as this
@@ -390,7 +390,7 @@ config RADIO_SF16FMI
 	tristate "SF16-FMI/SF16-FMP/SF16-FMD Radio"
 	depends on ISA || COMPILE_TEST
 	depends on VIDEO_V4L2
-	---help---
+	help
 	  Choose Y here if you have one of these FM radio cards.
 
 	  To compile this driver as a module, choose M here: the
@@ -401,7 +401,7 @@ config RADIO_SF16FMR2
 	depends on ISA || COMPILE_TEST
 	depends on VIDEO_V4L2
 	select RADIO_TEA575X
-	---help---
+	help
 	  Choose Y here if you have one of these FM radio cards.
 
 	  To compile this driver as a module, choose M here: the
@@ -412,7 +412,7 @@ config RADIO_TERRATEC
 	depends on ISA || COMPILE_TEST
 	depends on VIDEO_V4L2
 	select RADIO_ISA
-	---help---
+	help
 	  Choose Y here if you have this FM radio card.
 
 	  Note: this driver hasn't been tested since a long time due to lack
@@ -451,7 +451,7 @@ config RADIO_TYPHOON
 	depends on ISA || COMPILE_TEST
 	depends on VIDEO_V4L2
 	select RADIO_ISA
-	---help---
+	help
 	  Choose Y here if you have one of these FM radio cards, and then fill
 	  in the port address and the frequency used for muting below.
 
@@ -486,7 +486,7 @@ config RADIO_ZOLTRIX
 	depends on ISA || COMPILE_TEST
 	depends on VIDEO_V4L2
 	select RADIO_ISA
-	---help---
+	help
 	  Choose Y here if you have one of these FM radio cards, and then fill
 	  in the port address below.
 
diff --git a/drivers/media/radio/si470x/Kconfig b/drivers/media/radio/si470x/Kconfig
index 6dbb158cd2a0..21026488de90 100644
--- a/drivers/media/radio/si470x/Kconfig
+++ b/drivers/media/radio/si470x/Kconfig
@@ -1,7 +1,7 @@
 config RADIO_SI470X
         tristate "Silicon Labs Si470x FM Radio Receiver support"
         depends on VIDEO_V4L2
-	---help---
+	help
 	  This is a driver for devices with the Silicon Labs SI470x
 	  chip (either via USB or I2C buses).
 
@@ -15,7 +15,7 @@ config RADIO_SI470X
 config USB_SI470X
 	tristate "Silicon Labs Si470x FM Radio Receiver support with USB"
 	depends on USB && RADIO_SI470X
-	---help---
+	help
 	  This is a driver for USB devices with the Silicon Labs SI470x
 	  chip. Currently these devices are known to work:
 	  - 10c4:818a: Silicon Labs USB FM Radio Reference Design
@@ -40,7 +40,7 @@ config USB_SI470X
 config I2C_SI470X
 	tristate "Silicon Labs Si470x FM Radio Receiver support with I2C"
 	depends on I2C && RADIO_SI470X
-	---help---
+	help
 	  This is a driver for I2C devices with the Silicon Labs SI470x
 	  chip.
 
diff --git a/drivers/media/radio/si4713/Kconfig b/drivers/media/radio/si4713/Kconfig
index 9c8b887cff75..17567c917554 100644
--- a/drivers/media/radio/si4713/Kconfig
+++ b/drivers/media/radio/si4713/Kconfig
@@ -2,7 +2,7 @@ config USB_SI4713
 	tristate "Silicon Labs Si4713 FM Radio Transmitter support with USB"
 	depends on USB && I2C && RADIO_SI4713
 	select I2C_SI4713
-	---help---
+	help
 	  This is a driver for USB devices with the Silicon Labs SI4713
 	  chip. Currently these devices are known to work.
 	  - 10c4:8244: Silicon Labs FM Transmitter USB device.
@@ -17,7 +17,7 @@ config PLATFORM_SI4713
 	tristate "Silicon Labs Si4713 FM Radio Transmitter support with I2C"
 	depends on I2C && RADIO_SI4713
 	select I2C_SI4713
-	---help---
+	help
 	  This is a driver for I2C devices with the Silicon Labs SI4713
 	  chip.
 
@@ -30,7 +30,7 @@ config PLATFORM_SI4713
 config I2C_SI4713
 	tristate "Silicon Labs Si4713 FM Radio Transmitter support"
 	depends on I2C && RADIO_SI4713
-	---help---
+	help
 	  Say Y here if you want support to Si4713 FM Radio Transmitter.
 	  This device can transmit audio through FM. It can transmit
 	  RDS and RBDS signals as well. This module is the v4l2 radio
diff --git a/drivers/media/radio/si4713/si4713.c b/drivers/media/radio/si4713/si4713.c
index f4a53f1e856e..a8584371a32d 100644
--- a/drivers/media/radio/si4713/si4713.c
+++ b/drivers/media/radio/si4713/si4713.c
@@ -1272,7 +1272,7 @@ static int si4713_g_modulator(struct v4l2_subdev *sd, struct v4l2_modulator *vm)
 	if (vm->index > 0)
 		return -EINVAL;
 
-	strncpy(vm->name, "FM Modulator", 32);
+	strscpy(vm->name, "FM Modulator", sizeof(vm->name));
 	vm->capability = V4L2_TUNER_CAP_STEREO | V4L2_TUNER_CAP_LOW |
 		V4L2_TUNER_CAP_RDS | V4L2_TUNER_CAP_RDS_CONTROLS;
 
diff --git a/drivers/media/radio/wl128x/fmdrv_common.c b/drivers/media/radio/wl128x/fmdrv_common.c
index 3c8987af3772..053d51a5890a 100644
--- a/drivers/media/radio/wl128x/fmdrv_common.c
+++ b/drivers/media/radio/wl128x/fmdrv_common.c
@@ -489,7 +489,8 @@ int fmc_send_cmd(struct fmdev *fmdev, u8 fm_op, u16 type, void *payload,
 		return -EIO;
 	}
 	/* Send response data to caller */
-	if (response != NULL && response_len != NULL && evt_hdr->dlen) {
+	if (response != NULL && response_len != NULL && evt_hdr->dlen &&
+	    evt_hdr->dlen <= payload_len) {
 		/* Skip header info and copy only response data */
 		skb_pull(skb, sizeof(struct fm_event_msg_hdr));
 		memcpy(response, skb->data, evt_hdr->dlen);
@@ -583,6 +584,8 @@ static void fm_irq_handle_flag_getcmd_resp(struct fmdev *fmdev)
 		return;
 
 	fm_evt_hdr = (void *)skb->data;
+	if (fm_evt_hdr->dlen > sizeof(fmdev->irq_info.flag))
+		return;
 
 	/* Skip header info and copy only response data */
 	skb_pull(skb, sizeof(struct fm_event_msg_hdr));
@@ -1268,8 +1271,9 @@ static int fm_download_firmware(struct fmdev *fmdev, const u8 *fw_name)
 
 		switch (action->type) {
 		case ACTION_SEND_COMMAND:	/* Send */
-			if (fmc_send_cmd(fmdev, 0, 0, action->data,
-						action->size, NULL, NULL))
+			ret = fmc_send_cmd(fmdev, 0, 0, action->data,
+					   action->size, NULL, NULL);
+			if (ret)
 				goto rel_fw;
 
 			cmd_cnt++;
@@ -1308,7 +1312,7 @@ static int load_default_rx_configuration(struct fmdev *fmdev)
 static int fm_power_up(struct fmdev *fmdev, u8 mode)
 {
 	u16 payload;
-	__be16 asic_id, asic_ver;
+	__be16 asic_id = 0, asic_ver = 0;
 	int resp_len, ret;
 	u8 fw_name[50];
 
@@ -1520,7 +1524,7 @@ int fmc_prepare(struct fmdev *fmdev)
 		}
 
 		ret = 0;
-	} else if (ret == -1) {
+	} else if (ret < 0) {
 		fmerr("st_register failed %d\n", ret);
 		return -EAGAIN;
 	}
diff --git a/drivers/media/rc/Kconfig b/drivers/media/rc/Kconfig
index 96ce3e5524e0..3fc6ac15c66d 100644
--- a/drivers/media/rc/Kconfig
+++ b/drivers/media/rc/Kconfig
@@ -2,7 +2,7 @@
 menuconfig RC_CORE
 	tristate "Remote Controller support"
 	depends on INPUT
-	---help---
+	help
 	  Enable support for Remote Controllers on Linux. This is
 	  needed in order to support several video capture adapters,
 	  standalone IR receivers/transmitters, and RF receivers.
@@ -19,7 +19,7 @@ source "drivers/media/rc/keymaps/Kconfig"
 config LIRC
 	bool "LIRC user interface"
 	depends on RC_CORE
-	---help---
+	help
 	   Enable this option to enable the Linux Infrared Remote
 	   Control user interface (e.g. /dev/lirc*). This interface
 	   passes raw IR to and from userspace, which is needed for
@@ -48,7 +48,7 @@ config IR_NEC_DECODER
 	depends on RC_CORE
 	select BITREVERSE
 
-	---help---
+	help
 	   Enable this option if you have IR with NEC protocol, and
 	   if the IR is decoded in software
 
@@ -57,7 +57,7 @@ config IR_RC5_DECODER
 	depends on RC_CORE
 	select BITREVERSE
 
-	---help---
+	help
 	   Enable this option if you have IR with RC-5 protocol, and
 	   if the IR is decoded in software
 
@@ -66,7 +66,7 @@ config IR_RC6_DECODER
 	depends on RC_CORE
 	select BITREVERSE
 
-	---help---
+	help
 	   Enable this option if you have an infrared remote control which
 	   uses the RC6 protocol, and you need software decoding support.
 
@@ -75,7 +75,7 @@ config IR_JVC_DECODER
 	depends on RC_CORE
 	select BITREVERSE
 
-	---help---
+	help
 	   Enable this option if you have an infrared remote control which
 	   uses the JVC protocol, and you need software decoding support.
 
@@ -84,7 +84,7 @@ config IR_SONY_DECODER
 	depends on RC_CORE
 	select BITREVERSE
 
-	---help---
+	help
 	   Enable this option if you have an infrared remote control which
 	   uses the Sony protocol, and you need software decoding support.
 
@@ -92,7 +92,7 @@ config IR_SANYO_DECODER
 	tristate "Enable IR raw decoder for the Sanyo protocol"
 	depends on RC_CORE
 
-	---help---
+	help
 	   Enable this option if you have an infrared remote control which
 	   uses the Sanyo protocol (Sanyo, Aiwa, Chinon remotes),
 	   and you need software decoding support.
@@ -101,7 +101,7 @@ config IR_SHARP_DECODER
 	tristate "Enable IR raw decoder for the Sharp protocol"
 	depends on RC_CORE
 
-	---help---
+	help
 	   Enable this option if you have an infrared remote control which
 	   uses the Sharp protocol (Sharp, Denon), and you need software
 	   decoding support.
@@ -111,7 +111,7 @@ config IR_MCE_KBD_DECODER
 	depends on RC_CORE
 	select BITREVERSE
 
-	---help---
+	help
 	   Enable this option if you have a Microsoft Remote Keyboard for
 	   Windows Media Center Edition, which you would like to use with
 	   a raw IR receiver in your system.
@@ -121,14 +121,14 @@ config IR_XMP_DECODER
 	depends on RC_CORE
 	select BITREVERSE
 
-	---help---
+	help
 	   Enable this option if you have IR with XMP protocol, and
 	   if the IR is decoded in software
 
 config IR_IMON_DECODER
 	tristate "Enable IR raw decoder for the iMON protocol"
 	depends on RC_CORE
-	---help---
+	help
 	   Enable this option if you have iMON PAD or Antec Veris infrared
 	   remote control and you would like to use it with a raw IR
 	   receiver, or if you wish to use an encoder to transmit this IR.
@@ -177,7 +177,7 @@ config IR_ENE
 	tristate "ENE eHome Receiver/Transceiver (pnp id: ENE0100/ENE02xxx)"
 	depends on PNP || COMPILE_TEST
 	depends on RC_CORE
-	---help---
+	help
 	   Say Y here to enable support for integrated infrared receiver
 	   /transceiver made by ENE.
 
@@ -203,7 +203,7 @@ config IR_IMON
 	depends on USB_ARCH_HAS_HCD
 	depends on RC_CORE
 	select USB
-	---help---
+	help
 	   Say Y here if you want to use a SoundGraph iMON (aka Antec Veris)
 	   IR Receiver and/or LCD/VFD/VGA display.
 
@@ -215,7 +215,7 @@ config IR_IMON_RAW
 	depends on USB_ARCH_HAS_HCD
 	depends on RC_CORE
 	select USB
-	---help---
+	help
 	   Say Y here if you want to use a SoundGraph iMON IR Receiver,
 	   early raw models.
 
@@ -227,7 +227,7 @@ config IR_MCEUSB
 	depends on USB_ARCH_HAS_HCD
 	depends on RC_CORE
 	select USB
-	---help---
+	help
 	   Say Y here if you want to use a Windows Media Center Edition
 	   eHome Infrared Transceiver.
 
@@ -238,7 +238,7 @@ config IR_ITE_CIR
 	tristate "ITE Tech Inc. IT8712/IT8512 Consumer Infrared Transceiver"
 	depends on PNP || COMPILE_TEST
 	depends on RC_CORE
-	---help---
+	help
 	   Say Y here to enable support for integrated infrared receivers
 	   /transceivers made by ITE Tech Inc. These are found in
 	   several ASUS devices, like the ASUS Digimatrix or the ASUS
@@ -251,7 +251,7 @@ config IR_FINTEK
 	tristate "Fintek Consumer Infrared Transceiver"
 	depends on PNP || COMPILE_TEST
 	depends on RC_CORE
-	---help---
+	help
 	   Say Y here to enable support for integrated infrared receiver
 	   /transceiver made by Fintek. This chip is found on assorted
 	   Jetway motherboards (and of course, possibly others).
@@ -263,7 +263,7 @@ config IR_MESON
 	tristate "Amlogic Meson IR remote receiver"
 	depends on RC_CORE
 	depends on ARCH_MESON || COMPILE_TEST
-	---help---
+	help
 	   Say Y if you want to use the IR remote receiver available
 	   on Amlogic Meson SoCs.
 
@@ -274,7 +274,7 @@ config IR_MTK
 	tristate "Mediatek IR remote receiver"
 	depends on RC_CORE
 	depends on ARCH_MEDIATEK || COMPILE_TEST
-	---help---
+	help
 	   Say Y if you want to use the IR remote receiver available
 	   on Mediatek SoCs.
 
@@ -285,7 +285,7 @@ config IR_NUVOTON
 	tristate "Nuvoton w836x7hg Consumer Infrared Transceiver"
 	depends on PNP || COMPILE_TEST
 	depends on RC_CORE
-	---help---
+	help
 	   Say Y here to enable support for integrated infrared receiver
 	   /transceiver made by Nuvoton (formerly Winbond). This chip is
 	   found in the ASRock ION 330HT, as well as assorted Intel
@@ -301,7 +301,7 @@ config IR_REDRAT3
 	select NEW_LEDS
 	select LEDS_CLASS
 	select USB
-	---help---
+	help
 	   Say Y here if you want to use a RedRat3 Infrared Transceiver.
 
 	   To compile this driver as a module, choose M here: the
@@ -311,7 +311,7 @@ config IR_SPI
 	tristate "SPI connected IR LED"
 	depends on SPI && LIRC
 	depends on OF || COMPILE_TEST
-	---help---
+	help
 	  Say Y if you want to use an IR LED connected through SPI bus.
 
 	  To compile this driver as a module, choose M here: the module will be
@@ -322,7 +322,7 @@ config IR_STREAMZAP
 	depends on USB_ARCH_HAS_HCD
 	depends on RC_CORE
 	select USB
-	---help---
+	help
 	   Say Y here if you want to use a Streamzap PC Remote
 	   Infrared Receiver.
 
@@ -336,7 +336,7 @@ config IR_WINBOND_CIR
 	select NEW_LEDS
 	select LEDS_CLASS
 	select BITREVERSE
-	---help---
+	help
 	   Say Y here if you want to use the IR remote functionality found
 	   in some Winbond SuperI/O chips. Currently only the WPCD376I
 	   chip is supported (included in some Intel Media series
@@ -350,7 +350,7 @@ config IR_IGORPLUGUSB
 	depends on USB_ARCH_HAS_HCD
 	depends on RC_CORE
 	select USB
-	---help---
+	help
 	   Say Y here if you want to use the IgorPlug-USB IR Receiver by
 	   Igor Cesko. This device is included on the Fit-PC2.
 
@@ -365,7 +365,7 @@ config IR_IGUANA
 	depends on USB_ARCH_HAS_HCD
 	depends on RC_CORE
 	select USB
-	---help---
+	help
 	   Say Y here if you want to use the IguanaWorks USB IR Transceiver.
 	   Both infrared receive and send are supported. If you want to
 	   change the ID or the pin config, use the user space driver from
@@ -383,7 +383,7 @@ config IR_TTUSBIR
 	select USB
 	select NEW_LEDS
 	select LEDS_CLASS
-	---help---
+	help
 	   Say Y here if you want to use the TechnoTrend USB IR Receiver. The
 	   driver can control the led.
 
@@ -393,7 +393,7 @@ config IR_TTUSBIR
 config IR_RX51
 	tristate "Nokia N900 IR transmitter diode"
 	depends on (OMAP_DM_TIMER && PWM_OMAP_DMTIMER && ARCH_OMAP2PLUS || COMPILE_TEST) && RC_CORE
-	---help---
+	help
 	   Say Y or M here if you want to enable support for the IR
 	   transmitter diode built in the Nokia N900 (RX51) device.
 
@@ -405,7 +405,7 @@ source "drivers/media/rc/img-ir/Kconfig"
 config RC_LOOPBACK
 	tristate "Remote Control Loopback Driver"
 	depends on RC_CORE
-	---help---
+	help
 	   Say Y here if you want support for the remote control loopback
 	   driver which allows TX data to be sent back as RX data.
 	   This is mostly useful for debugging purposes.
@@ -419,7 +419,7 @@ config IR_GPIO_CIR
 	tristate "GPIO IR remote control"
 	depends on RC_CORE
 	depends on (OF && GPIOLIB) || COMPILE_TEST
-	---help---
+	help
 	   Say Y if you want to use GPIO based IR Receiver.
 
 	   To compile this driver as a module, choose M here: the module will
@@ -430,7 +430,7 @@ config IR_GPIO_TX
 	depends on RC_CORE
 	depends on LIRC
 	depends on (OF && GPIOLIB) || COMPILE_TEST
-	---help---
+	help
 	   Say Y if you want to a GPIO based IR transmitter. This is a
 	   bit banging driver.
 
@@ -443,7 +443,7 @@ config IR_PWM_TX
 	depends on LIRC
 	depends on PWM
 	depends on OF || COMPILE_TEST
-	---help---
+	help
 	   Say Y if you want to use a PWM based IR transmitter. This is
 	   more power efficient than the bit banging gpio driver.
 
@@ -454,7 +454,7 @@ config RC_ST
 	tristate "ST remote control receiver"
 	depends on RC_CORE
 	depends on ARCH_STI || COMPILE_TEST
-	---help---
+	help
 	   Say Y here if you want support for ST remote control driver
 	   which allows both IR and UHF RX.
 	   The driver passes raw pulse and space information to the LIRC decoder.
@@ -465,7 +465,7 @@ config IR_SUNXI
 	tristate "SUNXI IR remote control"
 	depends on RC_CORE
 	depends on ARCH_SUNXI || COMPILE_TEST
-	---help---
+	help
 	   Say Y if you want to use sunXi internal IR Controller
 
 	   To compile this driver as a module, choose M here: the module will
@@ -474,7 +474,7 @@ config IR_SUNXI
 config IR_SERIAL
 	tristate "Homebrew Serial Port Receiver"
 	depends on RC_CORE
-	---help---
+	help
 	   Say Y if you want to use Homebrew Serial Port Receivers and
 	   Transceivers.
 
@@ -484,13 +484,13 @@ config IR_SERIAL
 config IR_SERIAL_TRANSMITTER
 	bool "Serial Port Transmitter"
 	depends on IR_SERIAL
-	---help---
+	help
 	   Serial Port Transmitter support
 
 config IR_SIR
 	tristate "Built-in SIR IrDA port"
 	depends on RC_CORE
-	---help---
+	help
 	   Say Y if you want to use a IrDA SIR port Transceivers.
 
 	   To compile this driver as a module, choose M here: the module will
@@ -500,7 +500,7 @@ config IR_TANGO
 	tristate "Sigma Designs SMP86xx IR decoder"
 	depends on RC_CORE
 	depends on ARCH_TANGO || COMPILE_TEST
-	---help---
+	help
 	   Adds support for the HW IR decoder embedded on Sigma Designs
 	   Tango-based systems (SMP86xx, SMP87xx).
 	   The HW decoder supports NEC, RC-5, RC-6 IR protocols.
@@ -522,7 +522,7 @@ config IR_ZX
 	tristate "ZTE ZX IR remote control"
 	depends on RC_CORE
 	depends on ARCH_ZX || COMPILE_TEST
-	---help---
+	help
 	   Say Y if you want to use the IR remote control available
 	   on ZTE ZX family SoCs.
 
diff --git a/drivers/media/rc/ir-rcmm-decoder.c b/drivers/media/rc/ir-rcmm-decoder.c
index f1096ac1e5c5..64fb65a9a19f 100644
--- a/drivers/media/rc/ir-rcmm-decoder.c
+++ b/drivers/media/rc/ir-rcmm-decoder.c
@@ -5,7 +5,6 @@
 
 #include "rc-core-priv.h"
 #include <linux/module.h>
-#include <linux/version.h>
 
 #define RCMM_UNIT		166667	/* nanosecs */
 #define RCMM_PREFIX_PULSE	416666  /* 166666.666666666*2.5 */
diff --git a/drivers/media/rc/keymaps/Kconfig b/drivers/media/rc/keymaps/Kconfig
index 767423bbbdd0..f459096d8e9c 100644
--- a/drivers/media/rc/keymaps/Kconfig
+++ b/drivers/media/rc/keymaps/Kconfig
@@ -3,7 +3,7 @@ config RC_MAP
 	depends on RC_CORE
 	default y
 
-	---help---
+	help
 	   This option enables the compilation of lots of Remote
 	   Controller tables. They are short tables, but if you
 	   don't use a remote controller, or prefer to load the
diff --git a/drivers/media/rc/keymaps/rc-xbox-dvd.c b/drivers/media/rc/keymaps/rc-xbox-dvd.c
index af387244636b..42815ab57bff 100644
--- a/drivers/media/rc/keymaps/rc-xbox-dvd.c
+++ b/drivers/media/rc/keymaps/rc-xbox-dvd.c
@@ -42,7 +42,7 @@ static struct rc_map_list xbox_dvd_map = {
 	.map = {
 		.scan     = xbox_dvd,
 		.size     = ARRAY_SIZE(xbox_dvd),
-		.rc_proto = RC_PROTO_UNKNOWN,
+		.rc_proto = RC_PROTO_XBOX_DVD,
 		.name     = RC_MAP_XBOX_DVD,
 	}
 };
diff --git a/drivers/media/rc/rc-main.c b/drivers/media/rc/rc-main.c
index e8fa28e20192..be5fd129d728 100644
--- a/drivers/media/rc/rc-main.c
+++ b/drivers/media/rc/rc-main.c
@@ -76,6 +76,7 @@ static const struct {
 		.scancode_bits = 0x00ffffff, .repeat_period = 114 },
 	[RC_PROTO_RCMM32] = { .name = "rc-mm-32",
 		.scancode_bits = 0xffffffff, .repeat_period = 114 },
+	[RC_PROTO_XBOX_DVD] = { .name = "xbox-dvd", .repeat_period = 64 },
 };
 
 /* Used to keep track of known keymaps */
@@ -1027,6 +1028,7 @@ static const struct {
 	{ RC_PROTO_BIT_RCMM12 |
 	  RC_PROTO_BIT_RCMM24 |
 	  RC_PROTO_BIT_RCMM32,	"rc-mm",	"ir-rcmm-decoder"	},
+	{ RC_PROTO_BIT_XBOX_DVD, "xbox-dvd",	NULL			},
 };
 
 /**
diff --git a/drivers/media/rc/serial_ir.c b/drivers/media/rc/serial_ir.c
index ffe2c672d105..3998ba29beb6 100644
--- a/drivers/media/rc/serial_ir.c
+++ b/drivers/media/rc/serial_ir.c
@@ -773,8 +773,6 @@ static void serial_ir_exit(void)
 
 static int __init serial_ir_init_module(void)
 {
-	int result;
-
 	switch (type) {
 	case IR_HOMEBREW:
 	case IR_IRDEO:
@@ -802,12 +800,7 @@ static int __init serial_ir_init_module(void)
 	if (sense != -1)
 		sense = !!sense;
 
-	result = serial_ir_init();
-	if (!result)
-		return 0;
-
-	serial_ir_exit();
-	return result;
+	return serial_ir_init();
 }
 
 static void __exit serial_ir_exit_module(void)
diff --git a/drivers/media/rc/xbox_remote.c b/drivers/media/rc/xbox_remote.c
index f959cbb94744..4a3f2cc4ef18 100644
--- a/drivers/media/rc/xbox_remote.c
+++ b/drivers/media/rc/xbox_remote.c
@@ -107,7 +107,7 @@ static void xbox_remote_input_report(struct urb *urb)
 		return;
 	}
 
-	rc_keydown(xbox_remote->rdev, RC_PROTO_UNKNOWN,
+	rc_keydown(xbox_remote->rdev, RC_PROTO_XBOX_DVD,
 		   le16_to_cpup((__le16 *)(data + 2)), 0);
 }
 
@@ -148,7 +148,7 @@ static void xbox_remote_rc_init(struct xbox_remote *xbox_remote)
 	struct rc_dev *rdev = xbox_remote->rdev;
 
 	rdev->priv = xbox_remote;
-	rdev->allowed_protocols = RC_PROTO_BIT_UNKNOWN;
+	rdev->allowed_protocols = RC_PROTO_BIT_XBOX_DVD;
 	rdev->driver_name = "xbox_remote";
 
 	rdev->open = xbox_remote_rc_open;
@@ -157,6 +157,8 @@ static void xbox_remote_rc_init(struct xbox_remote *xbox_remote)
 	rdev->device_name = xbox_remote->rc_name;
 	rdev->input_phys = xbox_remote->rc_phys;
 
+	rdev->timeout = MS_TO_NS(10);
+
 	usb_to_input_id(xbox_remote->udev, &rdev->input_id);
 	rdev->dev.parent = &xbox_remote->interface->dev;
 }
diff --git a/drivers/media/spi/Kconfig b/drivers/media/spi/Kconfig
index b07ac86fc53c..df169ecf0c27 100644
--- a/drivers/media/spi/Kconfig
+++ b/drivers/media/spi/Kconfig
@@ -6,7 +6,7 @@ menu "SPI helper chips"
 config VIDEO_GS1662
 	tristate "Gennum Serializers video"
 	depends on SPI && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
-	---help---
+	help
 	  Enable the GS1662 driver which serializes video streams.
 
 endmenu
diff --git a/drivers/media/usb/airspy/Kconfig b/drivers/media/usb/airspy/Kconfig
index 10b204cf4dbc..67578511bb9a 100644
--- a/drivers/media/usb/airspy/Kconfig
+++ b/drivers/media/usb/airspy/Kconfig
@@ -2,7 +2,7 @@ config USB_AIRSPY
 	tristate "AirSpy"
 	depends on VIDEO_V4L2
 	select VIDEOBUF2_VMALLOC
-	---help---
+	help
 	  This is a video4linux2 driver for AirSpy SDR device.
 
 	  To compile this driver as a module, choose M here: the
diff --git a/drivers/media/usb/au0828/Kconfig b/drivers/media/usb/au0828/Kconfig
index 65fc067eb864..0ad985542c60 100644
--- a/drivers/media/usb/au0828/Kconfig
+++ b/drivers/media/usb/au0828/Kconfig
@@ -2,6 +2,8 @@
 config VIDEO_AU0828
 	tristate "Auvitek AU0828 support"
 	depends on I2C && INPUT && DVB_CORE && USB && VIDEO_V4L2
+	select MEDIA_CONTROLLER
+	select MEDIA_CONTROLLER_DVB
 	select I2C_ALGOBIT
 	select VIDEO_TVEEPROM
 	select VIDEOBUF2_VMALLOC if VIDEO_V4L2
@@ -9,7 +11,7 @@ config VIDEO_AU0828
 	select MEDIA_TUNER_XC5000 if MEDIA_SUBDRV_AUTOSELECT
 	select MEDIA_TUNER_MXL5007T if MEDIA_SUBDRV_AUTOSELECT
 	select MEDIA_TUNER_TDA18271 if MEDIA_SUBDRV_AUTOSELECT
-	---help---
+	help
 	  This is a hybrid analog/digital tv capture driver for
 	  Auvitek's AU0828 USB device.
 
@@ -23,7 +25,7 @@ config VIDEO_AU0828_V4L2
 	select DVB_AU8522_V4L if MEDIA_SUBDRV_AUTOSELECT
 	select VIDEO_TUNER
 	default y
-	---help---
+	help
 	  This is a video4linux driver for Auvitek's USB device.
 
 	  Choose Y here to include support for v4l2 analog video
@@ -34,5 +36,5 @@ config VIDEO_AU0828_RC
 	depends on RC_CORE
 	depends on !(RC_CORE=m && VIDEO_AU0828=y)
 	depends on VIDEO_AU0828
-	---help---
+	help
 	   Enables Remote Controller support on au0828 driver.
diff --git a/drivers/media/usb/au0828/au0828-core.c b/drivers/media/usb/au0828/au0828-core.c
index 3f8c92a70116..925a80437822 100644
--- a/drivers/media/usb/au0828/au0828-core.c
+++ b/drivers/media/usb/au0828/au0828-core.c
@@ -155,9 +155,7 @@ static void au0828_unregister_media_device(struct au0828_dev *dev)
 	dev->media_dev->disable_source = NULL;
 	mutex_unlock(&mdev->graph_mutex);
 
-	media_device_unregister(dev->media_dev);
-	media_device_cleanup(dev->media_dev);
-	kfree(dev->media_dev);
+	media_device_delete(dev->media_dev, KBUILD_MODNAME, THIS_MODULE);
 	dev->media_dev = NULL;
 #endif
 }
@@ -210,14 +208,10 @@ static int au0828_media_device_init(struct au0828_dev *dev,
 #ifdef CONFIG_MEDIA_CONTROLLER
 	struct media_device *mdev;
 
-	mdev = kzalloc(sizeof(*mdev), GFP_KERNEL);
+	mdev = media_device_usb_allocate(udev, KBUILD_MODNAME, THIS_MODULE);
 	if (!mdev)
 		return -ENOMEM;
 
-	/* check if media device is already initialized */
-	if (!mdev->dev)
-		media_device_usb_init(mdev, udev, udev->product);
-
 	dev->media_dev = mdev;
 #endif
 	return 0;
@@ -278,6 +272,28 @@ create_link:
 	}
 }
 
+static bool au0828_is_link_shareable(struct media_entity *owner,
+				     struct media_entity *entity)
+{
+	bool shareable = false;
+
+	/* Tuner link can be shared by audio, video, and VBI */
+	switch (owner->function) {
+	case MEDIA_ENT_F_IO_V4L:
+	case MEDIA_ENT_F_AUDIO_CAPTURE:
+	case MEDIA_ENT_F_IO_VBI:
+		if (entity->function == MEDIA_ENT_F_IO_V4L ||
+		    entity->function == MEDIA_ENT_F_AUDIO_CAPTURE ||
+		    entity->function == MEDIA_ENT_F_IO_VBI)
+			shareable = true;
+		break;
+	case MEDIA_ENT_F_DTV_DEMOD:
+	default:
+		break;
+	}
+	return shareable;
+}
+
 /* Callers should hold graph_mutex */
 static int au0828_enable_source(struct media_entity *entity,
 				struct media_pipeline *pipe)
@@ -320,18 +336,20 @@ static int au0828_enable_source(struct media_entity *entity,
 		/*
 		 * Default input is tuner and default input_type
 		 * is AU0828_VMUX_TELEVISION.
-		 * FIXME:
+		 *
 		 * There is a problem when s_input is called to
 		 * change the default input. s_input will try to
 		 * enable_source before attempting to change the
 		 * input on the device, and will end up enabling
 		 * default source which is tuner.
 		 *
-		 * Additional logic is necessary in au0828
-		 * to detect that the input has changed and
-		 * enable the right source.
+		 * Additional logic is necessary in au0828 to detect
+		 * that the input has changed and enable the right
+		 * source. au0828 handles this case in its s_input.
+		 * It will disable the old source and enable the new
+		 * source.
+		 *
 		*/
-
 		if (dev->input_type == AU0828_VMUX_TELEVISION)
 			find_source = dev->tuner;
 		else if (dev->input_type == AU0828_VMUX_SVIDEO ||
@@ -344,27 +362,33 @@ static int au0828_enable_source(struct media_entity *entity,
 		}
 	}
 
-	/* Is an active link between sink and source */
+	/* Is there an active link between sink and source */
 	if (dev->active_link) {
-		/*
-		 * If DVB is using the tuner and calling entity is
-		 * audio/video, the following check will be false,
-		 * since sink is different. Result is Busy.
-		 */
-		if (dev->active_link->sink->entity == sink &&
-		    dev->active_link->source->entity == find_source) {
-			/*
-			 * Either ALSA or Video own tuner. sink is
-			 * the same for both. Prevent Video stepping
-			 * on ALSA when ALSA owns the source.
+		if (dev->active_link_owner == entity) {
+			/* This check is necessary to handle multiple
+			 * enable_source calls from v4l_ioctls during
+			 * the course of video/vbi application run-time.
 			*/
-			if (dev->active_link_owner != entity &&
-			    dev->active_link_owner->function ==
-						MEDIA_ENT_F_AUDIO_CAPTURE) {
-				pr_debug("ALSA has the tuner\n");
-				ret = -EBUSY;
-				goto end;
-			}
+			pr_debug("%s already owns the tuner\n", entity->name);
+			ret = 0;
+			goto end;
+		} else if (au0828_is_link_shareable(dev->active_link_owner,
+			   entity)) {
+			/* Either ALSA or Video own tuner. Sink is the same
+			 * for both. Allow sharing the active link between
+			 * their common source (tuner) and sink (decoder).
+			 * Starting pipeline between sharing entity and sink
+			 * will fail with pipe mismatch, while owner has an
+			 * active pipeline. Switch pipeline ownership from
+			 * user to owner when owner disables the source.
+			 */
+			dev->active_link_shared = true;
+			/* save the user info to use from disable */
+			dev->active_link_user = entity;
+			dev->active_link_user_pipe = pipe;
+			pr_debug("%s owns the tuner %s can share!\n",
+				 dev->active_link_owner->name,
+				 entity->name);
 			ret = 0;
 			goto end;
 		} else {
@@ -391,7 +415,7 @@ static int au0828_enable_source(struct media_entity *entity,
 	source = found_link->source->entity;
 	ret = __media_entity_setup_link(found_link, MEDIA_LNK_FL_ENABLED);
 	if (ret) {
-		pr_err("Activate tuner link %s->%s. Error %d\n",
+		pr_err("Activate link from %s->%s. Error %d\n",
 			source->name, sink->name, ret);
 		goto end;
 	}
@@ -401,25 +425,26 @@ static int au0828_enable_source(struct media_entity *entity,
 		pr_err("Start Pipeline: %s->%s Error %d\n",
 			source->name, entity->name, ret);
 		ret = __media_entity_setup_link(found_link, 0);
-		pr_err("Deactivate link Error %d\n", ret);
+		if (ret)
+			pr_err("Deactivate link Error %d\n", ret);
 		goto end;
 	}
-	/*
-	 * save active link and active link owner to avoid audio
-	 * deactivating video owned link from disable_source and
-	 * vice versa
+
+	/* save link state to allow audio and video share the link
+	 * and not disable the link while the other is using it.
+	 * active_link_owner is used to deactivate the link.
 	*/
 	dev->active_link = found_link;
 	dev->active_link_owner = entity;
 	dev->active_source = source;
 	dev->active_sink = sink;
 
-	pr_debug("Enabled Source: %s->%s->%s Ret %d\n",
+	pr_info("Enabled Source: %s->%s->%s Ret %d\n",
 		 dev->active_source->name, dev->active_sink->name,
 		 dev->active_link_owner->name, ret);
 end:
-	pr_debug("au0828_enable_source() end %s %d %d\n",
-		 entity->name, entity->function, ret);
+	pr_debug("%s end: ent:%s fnc:%d ret %d\n",
+		 __func__, entity->name, entity->function, ret);
 	return ret;
 }
 
@@ -438,21 +463,95 @@ static void au0828_disable_source(struct media_entity *entity)
 	if (!dev->active_link)
 		return;
 
-	/* link is active - stop pipeline from source (tuner) */
+	/* link is active - stop pipeline from source
+	 * (tuner/s-video/Composite) to the entity
+	 * When DVB/s-video/Composite owns tuner, it won't be in
+	 * shared state.
+	 */
 	if (dev->active_link->sink->entity == dev->active_sink &&
 	    dev->active_link->source->entity == dev->active_source) {
 		/*
-		 * prevent video from deactivating link when audio
-		 * has active pipeline
+		 * Prevent video from deactivating link when audio
+		 * has active pipeline and vice versa. In addition
+		 * handle the case when more than one video/vbi
+		 * application is sharing the link.
 		*/
+		bool owner_is_audio = false;
+
+		if (dev->active_link_owner->function ==
+		    MEDIA_ENT_F_AUDIO_CAPTURE)
+			owner_is_audio = true;
+
+		if (dev->active_link_shared) {
+			pr_debug("Shared link owner %s user %s %d\n",
+				 dev->active_link_owner->name,
+				 entity->name, dev->users);
+
+			/* Handle video device users > 1
+			 * When audio owns the shared link with
+			 * more than one video users, avoid
+			 * disabling the source and/or switching
+			 * the owner until the last disable_source
+			 * call from video _close(). Use dev->users to
+			 * determine when to switch/disable.
+			 */
+			if (dev->active_link_owner != entity) {
+				/* video device has users > 1 */
+				if (owner_is_audio && dev->users > 1)
+					return;
+
+				dev->active_link_user = NULL;
+				dev->active_link_user_pipe = NULL;
+				dev->active_link_shared = false;
+				return;
+			}
+
+			/* video owns the link and has users > 1 */
+			if (!owner_is_audio && dev->users > 1)
+				return;
+
+			/* stop pipeline */
+			__media_pipeline_stop(dev->active_link_owner);
+			pr_debug("Pipeline stop for %s\n",
+				dev->active_link_owner->name);
+
+			ret = __media_pipeline_start(
+					dev->active_link_user,
+					dev->active_link_user_pipe);
+			if (ret) {
+				pr_err("Start Pipeline: %s->%s %d\n",
+					dev->active_source->name,
+					dev->active_link_user->name,
+					ret);
+				goto deactivate_link;
+			}
+			/* link user is now the owner */
+			dev->active_link_owner = dev->active_link_user;
+			dev->active_link_user = NULL;
+			dev->active_link_user_pipe = NULL;
+			dev->active_link_shared = false;
+
+			pr_debug("Pipeline started for %s\n",
+				dev->active_link_owner->name);
+			return;
+		} else if (!owner_is_audio && dev->users > 1)
+			/* video/vbi owns the link and has users > 1 */
+			return;
+
 		if (dev->active_link_owner != entity)
 			return;
-		__media_pipeline_stop(entity);
+
+		/* stop pipeline */
+		__media_pipeline_stop(dev->active_link_owner);
+		pr_debug("Pipeline stop for %s\n",
+			dev->active_link_owner->name);
+
+deactivate_link:
 		ret = __media_entity_setup_link(dev->active_link, 0);
 		if (ret)
 			pr_err("Deactivate link Error %d\n", ret);
 
-		pr_debug("Disabled Source: %s->%s->%s Ret %d\n",
+		pr_info("Disabled Source: %s->%s->%s Ret %d\n",
 			 dev->active_source->name, dev->active_sink->name,
 			 dev->active_link_owner->name, ret);
 
@@ -460,6 +559,8 @@ static void au0828_disable_source(struct media_entity *entity)
 		dev->active_link_owner = NULL;
 		dev->active_source = NULL;
 		dev->active_sink = NULL;
+		dev->active_link_shared = false;
+		dev->active_link_user = NULL;
 	}
 }
 #endif
@@ -480,6 +581,9 @@ static int au0828_media_device_register(struct au0828_dev *dev,
 		/* register media device */
 		ret = media_device_register(dev->media_dev);
 		if (ret) {
+			media_device_delete(dev->media_dev, KBUILD_MODNAME,
+					    THIS_MODULE);
+			dev->media_dev = NULL;
 			dev_err(&udev->dev,
 				"Media Device Register Error: %d\n", ret);
 			return ret;
diff --git a/drivers/media/usb/au0828/au0828-video.c b/drivers/media/usb/au0828/au0828-video.c
index 7876c897cc1d..4bde3db83aa2 100644
--- a/drivers/media/usb/au0828/au0828-video.c
+++ b/drivers/media/usb/au0828/au0828-video.c
@@ -758,6 +758,9 @@ static int au0828_analog_stream_enable(struct au0828_dev *d)
 
 	dprintk(1, "au0828_analog_stream_enable called\n");
 
+	if (test_bit(DEV_DISCONNECTED, &d->dev_state))
+		return -ENODEV;
+
 	iface = usb_ifnum_to_if(d->usbdev, 0);
 	if (iface && iface->cur_altsetting->desc.bAlternateSetting != 5) {
 		dprintk(1, "Changing intf#0 to alt 5\n");
@@ -839,9 +842,9 @@ int au0828_start_analog_streaming(struct vb2_queue *vq, unsigned int count)
 			return rc;
 		}
 
+		v4l2_device_call_all(&dev->v4l2_dev, 0, video, s_stream, 1);
+
 		if (vq->type == V4L2_BUF_TYPE_VIDEO_CAPTURE) {
-			v4l2_device_call_all(&dev->v4l2_dev, 0, video,
-						s_stream, 1);
 			dev->vid_timeout_running = 1;
 			mod_timer(&dev->vid_timeout, jiffies + (HZ / 10));
 		} else if (vq->type == V4L2_BUF_TYPE_VBI_CAPTURE) {
@@ -861,10 +864,11 @@ static void au0828_stop_streaming(struct vb2_queue *vq)
 
 	dprintk(1, "au0828_stop_streaming called %d\n", dev->streaming_users);
 
-	if (dev->streaming_users-- == 1)
+	if (dev->streaming_users-- == 1) {
 		au0828_uninit_isoc(dev);
+		v4l2_device_call_all(&dev->v4l2_dev, 0, video, s_stream, 0);
+	}
 
-	v4l2_device_call_all(&dev->v4l2_dev, 0, video, s_stream, 0);
 	dev->vid_timeout_running = 0;
 	del_timer_sync(&dev->vid_timeout);
 
@@ -893,8 +897,10 @@ void au0828_stop_vbi_streaming(struct vb2_queue *vq)
 	dprintk(1, "au0828_stop_vbi_streaming called %d\n",
 		dev->streaming_users);
 
-	if (dev->streaming_users-- == 1)
+	if (dev->streaming_users-- == 1) {
 		au0828_uninit_isoc(dev);
+		v4l2_device_call_all(&dev->v4l2_dev, 0, video, s_stream, 0);
+	}
 
 	spin_lock_irqsave(&dev->slock, flags);
 	if (dev->isoc_ctl.vbi_buf != NULL) {
@@ -1065,7 +1071,7 @@ static int au0828_v4l2_close(struct file *filp)
 		 * streaming.
 		 *
 		 * On most USB devices  like au0828 the tuner can
-		 * be safely put in sleep stare here if ALSA isn't
+		 * be safely put in sleep state here if ALSA isn't
 		 * streaming. Exceptions are some very old USB tuner
 		 * models such as em28xx-based WinTV USB2 which have
 		 * a separate audio output jack. The devices that have
@@ -1074,7 +1080,7 @@ static int au0828_v4l2_close(struct file *filp)
 		 * so the s_power callback are silently ignored.
 		 * So, the current logic here does the following:
 		 * Disable (put tuner to sleep) when
-		 * - ALSA and DVB aren't not streaming;
+		 * - ALSA and DVB aren't streaming.
 		 * - the last V4L2 file handler is closed.
 		 *
 		 * FIXME:
diff --git a/drivers/media/usb/au0828/au0828.h b/drivers/media/usb/au0828/au0828.h
index 425c35d16057..b47ecc9affd8 100644
--- a/drivers/media/usb/au0828/au0828.h
+++ b/drivers/media/usb/au0828/au0828.h
@@ -31,6 +31,7 @@
 #include <media/v4l2-ctrls.h>
 #include <media/v4l2-fh.h>
 #include <media/media-device.h>
+#include <media/media-dev-allocator.h>
 
 /* DVB */
 #include <media/demux.h>
@@ -283,9 +284,12 @@ struct au0828_dev {
 	struct media_entity_notify entity_notify;
 	struct media_entity *tuner;
 	struct media_link *active_link;
-	struct media_entity *active_link_owner;
 	struct media_entity *active_source;
 	struct media_entity *active_sink;
+	struct media_entity *active_link_owner;
+	struct media_entity *active_link_user;
+	struct media_pipeline *active_link_user_pipe;
+	bool active_link_shared;
 #endif
 };
 
diff --git a/drivers/media/usb/cpia2/Kconfig b/drivers/media/usb/cpia2/Kconfig
index 66e9283f5993..7029a04f3ffd 100644
--- a/drivers/media/usb/cpia2/Kconfig
+++ b/drivers/media/usb/cpia2/Kconfig
@@ -1,7 +1,7 @@
 config VIDEO_CPIA2
 	tristate "CPiA2 Video For Linux"
 	depends on VIDEO_DEV && USB && VIDEO_V4L2
-	---help---
+	help
 	  This is the video4linux driver for cameras based on Vision's CPiA2
 	  (Colour Processor Interface ASIC), such as the Digital Blue QX5
 	  Microscope. If you have one of these cameras, say Y here
diff --git a/drivers/media/usb/cpia2/cpia2_v4l.c b/drivers/media/usb/cpia2/cpia2_v4l.c
index 95c0bd4a19dc..45caf78119c4 100644
--- a/drivers/media/usb/cpia2/cpia2_v4l.c
+++ b/drivers/media/usb/cpia2/cpia2_v4l.c
@@ -1240,8 +1240,7 @@ static int __init cpia2_init(void)
 	LOG("%s v%s\n",
 	    ABOUT, CPIA_VERSION);
 	check_parameters();
-	cpia2_usb_init();
-	return 0;
+	return cpia2_usb_init();
 }
 
 
diff --git a/drivers/media/usb/cx231xx/Kconfig b/drivers/media/usb/cx231xx/Kconfig
index 9e5b3e7c3ef5..9262d0d7439a 100644
--- a/drivers/media/usb/cx231xx/Kconfig
+++ b/drivers/media/usb/cx231xx/Kconfig
@@ -7,7 +7,7 @@ config VIDEO_CX231XX
 	select VIDEO_CX25840
 	select VIDEO_CX2341X
 
-	---help---
+	help
 	  This is a video4linux driver for Conexant 231xx USB based TV cards.
 
 	  To compile this driver as a module, choose M here: the
@@ -18,7 +18,7 @@ config VIDEO_CX231XX_RC
 	depends on RC_CORE=y || RC_CORE=VIDEO_CX231XX
 	depends on VIDEO_CX231XX
 	default y
-	---help---
+	help
 	  cx231xx hardware has a builtin RX/TX support. However, a few
 	  designs opted to not use it, but, instead, some other hardware.
 	  This module enables the usage of those other hardware, like the
@@ -31,7 +31,7 @@ config VIDEO_CX231XX_ALSA
 	depends on VIDEO_CX231XX && SND
 	select SND_PCM
 
-	---help---
+	help
 	  This is an ALSA driver for Cx231xx USB based TV cards.
 
 	  To compile this driver as a module, choose M here: the
@@ -52,6 +52,6 @@ config VIDEO_CX231XX_DVB
 	select DVB_MN88473 if MEDIA_SUBDRV_AUTOSELECT
 	select MEDIA_TUNER_R820T if MEDIA_SUBDRV_AUTOSELECT
 
-	---help---
+	help
 	  This adds support for DVB cards based on the
 	  Conexant cx231xx chips.
diff --git a/drivers/media/usb/dvb-usb-v2/af9035.c b/drivers/media/usb/dvb-usb-v2/af9035.c
index 80d3bd3a0f24..1b7f1af399fb 100644
--- a/drivers/media/usb/dvb-usb-v2/af9035.c
+++ b/drivers/media/usb/dvb-usb-v2/af9035.c
@@ -846,6 +846,7 @@ static int af9035_read_config(struct dvb_usb_device *d)
 	state->af9033_config[1].adc_multiplier = AF9033_ADC_MULTIPLIER_2X;
 	state->af9033_config[0].ts_mode = AF9033_TS_MODE_USB;
 	state->af9033_config[1].ts_mode = AF9033_TS_MODE_SERIAL;
+	state->it930x_addresses = 0;
 
 	if (state->chip_type == 0x9135) {
 		/* feed clock for integrated RF tuner */
@@ -872,6 +873,10 @@ static int af9035_read_config(struct dvb_usb_device *d)
 		 * IT930x is an USB bridge, only single demod-single tuner
 		 * configurations seen so far.
 		 */
+		if ((le16_to_cpu(d->udev->descriptor.idVendor) == USB_VID_AVERMEDIA) &&
+		    (le16_to_cpu(d->udev->descriptor.idProduct) == USB_PID_AVERMEDIA_TD310)) {
+			state->it930x_addresses = 1;
+		}
 		return 0;
 	}
 
@@ -1218,6 +1223,48 @@ static int it930x_frontend_attach(struct dvb_usb_adapter *adap)
 
 	dev_dbg(&intf->dev, "adap->id=%d\n", adap->id);
 
+	/* I2C master bus 2 clock speed 300k */
+	ret = af9035_wr_reg(d, 0x00f6a7, 0x07);
+	if (ret < 0)
+		goto err;
+
+	/* I2C master bus 1,3 clock speed 300k */
+	ret = af9035_wr_reg(d, 0x00f103, 0x07);
+	if (ret < 0)
+		goto err;
+
+	/* set gpio11 low */
+	ret = af9035_wr_reg_mask(d, 0xd8d4, 0x01, 0x01);
+	if (ret < 0)
+		goto err;
+
+	ret = af9035_wr_reg_mask(d, 0xd8d5, 0x01, 0x01);
+	if (ret < 0)
+		goto err;
+
+	ret = af9035_wr_reg_mask(d, 0xd8d3, 0x01, 0x01);
+	if (ret < 0)
+		goto err;
+
+	/* Tuner enable using gpiot2_en, gpiot2_on and gpiot2_o (reset) */
+	ret = af9035_wr_reg_mask(d, 0xd8b8, 0x01, 0x01);
+	if (ret < 0)
+		goto err;
+
+	ret = af9035_wr_reg_mask(d, 0xd8b9, 0x01, 0x01);
+	if (ret < 0)
+		goto err;
+
+	ret = af9035_wr_reg_mask(d, 0xd8b7, 0x00, 0x01);
+	if (ret < 0)
+		goto err;
+
+	msleep(200);
+
+	ret = af9035_wr_reg_mask(d, 0xd8b7, 0x01, 0x01);
+	if (ret < 0)
+		goto err;
+
 	memset(&si2168_config, 0, sizeof(si2168_config));
 	si2168_config.i2c_adapter = &adapter;
 	si2168_config.fe = &adap->fe[0];
@@ -1225,8 +1272,9 @@ static int it930x_frontend_attach(struct dvb_usb_adapter *adap)
 
 	state->af9033_config[adap->id].fe = &adap->fe[0];
 	state->af9033_config[adap->id].ops = &state->ops;
-	ret = af9035_add_i2c_dev(d, "si2168", 0x67, &si2168_config,
-				&d->i2c_adap);
+	ret = af9035_add_i2c_dev(d, "si2168",
+				 it930x_addresses_table[state->it930x_addresses].frontend_i2c_addr,
+				 &si2168_config, &d->i2c_adap);
 	if (ret)
 		goto err;
 
@@ -1575,54 +1623,12 @@ static int it930x_tuner_attach(struct dvb_usb_adapter *adap)
 
 	dev_dbg(&intf->dev, "adap->id=%d\n", adap->id);
 
-	/* I2C master bus 2 clock speed 300k */
-	ret = af9035_wr_reg(d, 0x00f6a7, 0x07);
-	if (ret < 0)
-		goto err;
-
-	/* I2C master bus 1,3 clock speed 300k */
-	ret = af9035_wr_reg(d, 0x00f103, 0x07);
-	if (ret < 0)
-		goto err;
-
-	/* set gpio11 low */
-	ret = af9035_wr_reg_mask(d, 0xd8d4, 0x01, 0x01);
-	if (ret < 0)
-		goto err;
-
-	ret = af9035_wr_reg_mask(d, 0xd8d5, 0x01, 0x01);
-	if (ret < 0)
-		goto err;
-
-	ret = af9035_wr_reg_mask(d, 0xd8d3, 0x01, 0x01);
-	if (ret < 0)
-		goto err;
-
-	/* Tuner enable using gpiot2_en, gpiot2_on and gpiot2_o (reset) */
-	ret = af9035_wr_reg_mask(d, 0xd8b8, 0x01, 0x01);
-	if (ret < 0)
-		goto err;
-
-	ret = af9035_wr_reg_mask(d, 0xd8b9, 0x01, 0x01);
-	if (ret < 0)
-		goto err;
-
-	ret = af9035_wr_reg_mask(d, 0xd8b7, 0x00, 0x01);
-	if (ret < 0)
-		goto err;
-
-	msleep(200);
-
-	ret = af9035_wr_reg_mask(d, 0xd8b7, 0x01, 0x01);
-	if (ret < 0)
-		goto err;
-
 	memset(&si2157_config, 0, sizeof(si2157_config));
 	si2157_config.fe = adap->fe[0];
-	si2157_config.if_port = 1;
-	ret = af9035_add_i2c_dev(d, "si2157", 0x63,
-			&si2157_config, state->i2c_adapter_demod);
-
+	si2157_config.if_port = it930x_addresses_table[state->it930x_addresses].tuner_if_port;
+	ret = af9035_add_i2c_dev(d, "si2157",
+				 it930x_addresses_table[state->it930x_addresses].tuner_i2c_addr,
+				 &si2157_config, state->i2c_adapter_demod);
 	if (ret)
 		goto err;
 
@@ -2128,6 +2134,8 @@ static const struct usb_device_id af9035_id_table[] = {
 	/* IT930x devices */
 	{ DVB_USB_DEVICE(USB_VID_ITETECH, USB_PID_ITETECH_IT9303,
 		&it930x_props, "ITE 9303 Generic", NULL) },
+	{ DVB_USB_DEVICE(USB_VID_AVERMEDIA, USB_PID_AVERMEDIA_TD310,
+		&it930x_props, "AVerMedia TD310 DVB-T2", NULL) },
 	{ }
 };
 MODULE_DEVICE_TABLE(usb, af9035_id_table);
diff --git a/drivers/media/usb/dvb-usb-v2/af9035.h b/drivers/media/usb/dvb-usb-v2/af9035.h
index a76e6bf0ab1e..bc41c16f9727 100644
--- a/drivers/media/usb/dvb-usb-v2/af9035.h
+++ b/drivers/media/usb/dvb-usb-v2/af9035.h
@@ -69,6 +69,7 @@ struct state {
 	u8 dual_mode:1;
 	u8 no_read:1;
 	u8 af9033_i2c_addr[2];
+	u8 it930x_addresses;
 	struct af9033_config af9033_config[2];
 	struct af9033_ops ops;
 	#define AF9035_I2C_CLIENT_MAX 4
@@ -77,6 +78,17 @@ struct state {
 	struct platform_device *platform_device_tuner[2];
 };
 
+struct address_table {
+	u8 frontend_i2c_addr;
+	u8 tuner_i2c_addr;
+	u8 tuner_if_port;
+};
+
+static const struct address_table it930x_addresses_table[] = {
+	{ 0x67, 0x63, 1 },
+	{ 0x64, 0x60, 0 },
+};
+
 static const u32 clock_lut_af9035[] = {
 	20480000, /*      FPGA */
 	16384000, /* 16.38 MHz */
diff --git a/drivers/media/usb/dvb-usb-v2/dvbsky.c b/drivers/media/usb/dvb-usb-v2/dvbsky.c
index e28bd8836751..ae0814dd202a 100644
--- a/drivers/media/usb/dvb-usb-v2/dvbsky.c
+++ b/drivers/media/usb/dvb-usb-v2/dvbsky.c
@@ -615,16 +615,18 @@ static int dvbsky_init(struct dvb_usb_device *d)
 	return 0;
 }
 
-static void dvbsky_exit(struct dvb_usb_device *d)
+static int dvbsky_frontend_detach(struct dvb_usb_adapter *adap)
 {
+	struct dvb_usb_device *d = adap_to_d(adap);
 	struct dvbsky_state *state = d_to_priv(d);
-	struct dvb_usb_adapter *adap = &d->adapter[0];
+
+	dev_dbg(&d->udev->dev, "%s: adap=%d\n", __func__, adap->id);
 
 	dvb_module_release(state->i2c_client_tuner);
 	dvb_module_release(state->i2c_client_demod);
 	dvb_module_release(state->i2c_client_ci);
 
-	adap->fe[0] = NULL;
+	return 0;
 }
 
 /* DVB USB Driver stuff */
@@ -640,11 +642,11 @@ static struct dvb_usb_device_properties dvbsky_s960_props = {
 
 	.i2c_algo         = &dvbsky_i2c_algo,
 	.frontend_attach  = dvbsky_s960_attach,
+	.frontend_detach  = dvbsky_frontend_detach,
 	.init             = dvbsky_init,
 	.get_rc_config    = dvbsky_get_rc_config,
 	.streaming_ctrl   = dvbsky_streaming_ctrl,
 	.identify_state	  = dvbsky_identify_state,
-	.exit             = dvbsky_exit,
 	.read_mac_address = dvbsky_read_mac_addr,
 
 	.num_adapters = 1,
@@ -667,11 +669,11 @@ static struct dvb_usb_device_properties dvbsky_s960c_props = {
 
 	.i2c_algo         = &dvbsky_i2c_algo,
 	.frontend_attach  = dvbsky_s960c_attach,
+	.frontend_detach  = dvbsky_frontend_detach,
 	.init             = dvbsky_init,
 	.get_rc_config    = dvbsky_get_rc_config,
 	.streaming_ctrl   = dvbsky_streaming_ctrl,
 	.identify_state	  = dvbsky_identify_state,
-	.exit             = dvbsky_exit,
 	.read_mac_address = dvbsky_read_mac_addr,
 
 	.num_adapters = 1,
@@ -694,11 +696,11 @@ static struct dvb_usb_device_properties dvbsky_t680c_props = {
 
 	.i2c_algo         = &dvbsky_i2c_algo,
 	.frontend_attach  = dvbsky_t680c_attach,
+	.frontend_detach  = dvbsky_frontend_detach,
 	.init             = dvbsky_init,
 	.get_rc_config    = dvbsky_get_rc_config,
 	.streaming_ctrl   = dvbsky_streaming_ctrl,
 	.identify_state	  = dvbsky_identify_state,
-	.exit             = dvbsky_exit,
 	.read_mac_address = dvbsky_read_mac_addr,
 
 	.num_adapters = 1,
@@ -721,11 +723,11 @@ static struct dvb_usb_device_properties dvbsky_t330_props = {
 
 	.i2c_algo         = &dvbsky_i2c_algo,
 	.frontend_attach  = dvbsky_t330_attach,
+	.frontend_detach  = dvbsky_frontend_detach,
 	.init             = dvbsky_init,
 	.get_rc_config    = dvbsky_get_rc_config,
 	.streaming_ctrl   = dvbsky_streaming_ctrl,
 	.identify_state	  = dvbsky_identify_state,
-	.exit             = dvbsky_exit,
 	.read_mac_address = dvbsky_read_mac_addr,
 
 	.num_adapters = 1,
@@ -748,11 +750,11 @@ static struct dvb_usb_device_properties mygica_t230c_props = {
 
 	.i2c_algo         = &dvbsky_i2c_algo,
 	.frontend_attach  = dvbsky_mygica_t230c_attach,
+	.frontend_detach  = dvbsky_frontend_detach,
 	.init             = dvbsky_init,
 	.get_rc_config    = dvbsky_get_rc_config,
 	.streaming_ctrl   = dvbsky_streaming_ctrl,
 	.identify_state	  = dvbsky_identify_state,
-	.exit             = dvbsky_exit,
 
 	.num_adapters = 1,
 	.adapter = {
diff --git a/drivers/media/usb/em28xx/Kconfig b/drivers/media/usb/em28xx/Kconfig
index 451e076525d3..639da7e24066 100644
--- a/drivers/media/usb/em28xx/Kconfig
+++ b/drivers/media/usb/em28xx/Kconfig
@@ -13,7 +13,7 @@ config VIDEO_EM28XX_V4L2
 	select VIDEO_MSP3400 if MEDIA_SUBDRV_AUTOSELECT
 	select VIDEO_MT9V011 if MEDIA_SUBDRV_AUTOSELECT && MEDIA_CAMERA_SUPPORT
 	select VIDEO_OV2640 if MEDIA_SUBDRV_AUTOSELECT && MEDIA_CAMERA_SUPPORT
-	---help---
+	help
 	  This is a video4linux driver for Empia 28xx based TV cards.
 
 	  To compile this driver as a module, choose M here: the
@@ -23,7 +23,7 @@ config VIDEO_EM28XX_ALSA
 	depends on VIDEO_EM28XX && SND
 	select SND_PCM
 	tristate "Empia EM28xx ALSA audio module"
-	---help---
+	help
 	  This is an ALSA driver for some Empia 28xx based TV cards.
 
 	  This is not required for em2800/em2820/em2821 boards. However,
@@ -66,7 +66,7 @@ config VIDEO_EM28XX_DVB
 	select MEDIA_TUNER_XC2028 if MEDIA_SUBDRV_AUTOSELECT
 	select MEDIA_TUNER_XC5000 if MEDIA_SUBDRV_AUTOSELECT
 	select MEDIA_TUNER_MT2060 if MEDIA_SUBDRV_AUTOSELECT
-	---help---
+	help
 	  This adds support for DVB cards based on the
 	  Empiatech em28xx chips.
 
@@ -76,5 +76,5 @@ config VIDEO_EM28XX_RC
 	depends on VIDEO_EM28XX
 	depends on !(RC_CORE=m && VIDEO_EM28XX=y)
 	default VIDEO_EM28XX
-	---help---
+	help
 	  Enables Remote Controller support on em28xx driver.
diff --git a/drivers/media/usb/em28xx/em28xx-input.c b/drivers/media/usb/em28xx/em28xx-input.c
index f84a1208d5d3..d85ea1af6aa1 100644
--- a/drivers/media/usb/em28xx/em28xx-input.c
+++ b/drivers/media/usb/em28xx/em28xx-input.c
@@ -499,7 +499,7 @@ static int em28xx_probe_i2c_ir(struct em28xx *dev)
 	 * at address 0x18, so if that address is needed for another board in
 	 * the future, please put it after 0x1f.
 	 */
-	const unsigned short addr_list[] = {
+	static const unsigned short addr_list[] = {
 		 0x1f, 0x30, 0x47, I2C_CLIENT_END
 	};
 
diff --git a/drivers/media/usb/go7007/Kconfig b/drivers/media/usb/go7007/Kconfig
index af1d02430931..beab257c092f 100644
--- a/drivers/media/usb/go7007/Kconfig
+++ b/drivers/media/usb/go7007/Kconfig
@@ -13,7 +13,7 @@ config VIDEO_GO7007
 	select VIDEO_TW9906 if MEDIA_SUBDRV_AUTOSELECT
 	select VIDEO_OV7640 if MEDIA_SUBDRV_AUTOSELECT && MEDIA_CAMERA_SUPPORT
 	select VIDEO_UDA1342 if MEDIA_SUBDRV_AUTOSELECT
-	---help---
+	help
 	  This is a video4linux driver for the WIS GO7007 MPEG
 	  encoder chip.
 
@@ -23,7 +23,7 @@ config VIDEO_GO7007
 config VIDEO_GO7007_USB
 	tristate "WIS GO7007 USB support"
 	depends on VIDEO_GO7007 && USB
-	---help---
+	help
 	  This is a video4linux driver for the WIS GO7007 MPEG
 	  encoder chip over USB.
 
@@ -34,7 +34,7 @@ config VIDEO_GO7007_LOADER
 	tristate "WIS GO7007 Loader support"
 	depends on VIDEO_GO7007
 	default y
-	---help---
+	help
 	  This is a go7007 firmware loader driver for the WIS GO7007
 	  MPEG encoder chip over USB.
 
@@ -44,7 +44,7 @@ config VIDEO_GO7007_LOADER
 config VIDEO_GO7007_USB_S2250_BOARD
 	tristate "Sensoray 2250/2251 support"
 	depends on VIDEO_GO7007_USB && USB
-	---help---
+	help
 	  This is a video4linux driver for the Sensoray 2250/2251 device.
 
 	  To compile this driver as a module, choose M here: the
diff --git a/drivers/media/usb/go7007/go7007-fw.c b/drivers/media/usb/go7007/go7007-fw.c
index 24f5b615dc7a..dfa9f899d0c2 100644
--- a/drivers/media/usb/go7007/go7007-fw.c
+++ b/drivers/media/usb/go7007/go7007-fw.c
@@ -1499,8 +1499,8 @@ static int modet_to_package(struct go7007 *go, __le16 *code, int space)
 	return cnt;
 }
 
-static int do_special(struct go7007 *go, u16 type, __le16 *code, int space,
-			int *framelen)
+static noinline_for_stack int do_special(struct go7007 *go, u16 type,
+					 __le16 *code, int space, int *framelen)
 {
 	switch (type) {
 	case SPECIAL_FRM_HEAD:
diff --git a/drivers/media/usb/go7007/go7007-usb.c b/drivers/media/usb/go7007/go7007-usb.c
index 19c6a0354ce0..abe98488be23 100644
--- a/drivers/media/usb/go7007/go7007-usb.c
+++ b/drivers/media/usb/go7007/go7007-usb.c
@@ -1132,7 +1132,7 @@ static int go7007_usb_probe(struct usb_interface *intf,
 	usb->usbdev = usbdev;
 	usb_make_path(usbdev, go->bus_info, sizeof(go->bus_info));
 	go->board_id = id->driver_info;
-	strncpy(go->name, name, sizeof(go->name));
+	strscpy(go->name, name, sizeof(go->name));
 	if (board->flags & GO7007_USB_EZUSB)
 		go->hpi_ops = &go7007_usb_ezusb_hpi_ops;
 	else
@@ -1198,7 +1198,7 @@ static int go7007_usb_probe(struct usb_interface *intf,
 				go->board_id = GO7007_BOARDID_ENDURA;
 				usb->board = board = &board_endura;
 				go->board_info = &board->main_info;
-				strncpy(go->name, "Pelco Endura",
+				strscpy(go->name, "Pelco Endura",
 					sizeof(go->name));
 			} else {
 				u16 channel;
@@ -1232,21 +1232,21 @@ static int go7007_usb_probe(struct usb_interface *intf,
 		case 1:
 			go->tuner_type = TUNER_SONY_BTF_PG472Z;
 			go->std = V4L2_STD_PAL;
-			strncpy(go->name, "Plextor PX-TV402U-EU",
-					sizeof(go->name));
+			strscpy(go->name, "Plextor PX-TV402U-EU",
+				sizeof(go->name));
 			break;
 		case 2:
 			go->tuner_type = TUNER_SONY_BTF_PK467Z;
 			go->std = V4L2_STD_NTSC_M_JP;
 			num_i2c_devs -= 2;
-			strncpy(go->name, "Plextor PX-TV402U-JP",
-					sizeof(go->name));
+			strscpy(go->name, "Plextor PX-TV402U-JP",
+				sizeof(go->name));
 			break;
 		case 3:
 			go->tuner_type = TUNER_SONY_BTF_PB463Z;
 			num_i2c_devs -= 2;
-			strncpy(go->name, "Plextor PX-TV402U-NA",
-					sizeof(go->name));
+			strscpy(go->name, "Plextor PX-TV402U-NA",
+				sizeof(go->name));
 			break;
 		default:
 			pr_debug("unable to detect tuner type!\n");
diff --git a/drivers/media/usb/go7007/go7007-v4l2.c b/drivers/media/usb/go7007/go7007-v4l2.c
index 7a2781fa83e7..bebdfcecf600 100644
--- a/drivers/media/usb/go7007/go7007-v4l2.c
+++ b/drivers/media/usb/go7007/go7007-v4l2.c
@@ -327,7 +327,7 @@ static int vidioc_enum_fmt_vid_cap(struct file *file, void  *priv,
 	fmt->type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
 	fmt->flags = V4L2_FMT_FLAG_COMPRESSED;
 
-	strncpy(fmt->description, desc, sizeof(fmt->description));
+	strscpy(fmt->description, desc, sizeof(fmt->description));
 
 	return 0;
 }
diff --git a/drivers/media/usb/gspca/Kconfig b/drivers/media/usb/gspca/Kconfig
index 088566e88467..0e6f36cb46e6 100644
--- a/drivers/media/usb/gspca/Kconfig
+++ b/drivers/media/usb/gspca/Kconfig
@@ -4,7 +4,7 @@ menuconfig USB_GSPCA
 	depends on INPUT || INPUT=n
 	select VIDEOBUF2_VMALLOC
 	default m
-	---help---
+	help
 	  Say Y here if you want to enable selecting webcams based
 	  on the GSPCA framework.
 
diff --git a/drivers/media/usb/gspca/gspca.c b/drivers/media/usb/gspca/gspca.c
index ac70b36d67b7..4d7517411cc2 100644
--- a/drivers/media/usb/gspca/gspca.c
+++ b/drivers/media/usb/gspca/gspca.c
@@ -294,7 +294,7 @@ static void fill_frame(struct gspca_dev *gspca_dev,
 		/* check the packet status and length */
 		st = urb->iso_frame_desc[i].status;
 		if (st) {
-			pr_err("ISOC data error: [%d] len=%d, status=%d\n",
+			gspca_dbg(gspca_dev, D_PACK, "ISOC data error: [%d] len=%d, status=%d\n",
 			       i, len, st);
 			gspca_dev->last_packet_type = DISCARD_PACKET;
 			continue;
@@ -314,6 +314,8 @@ static void fill_frame(struct gspca_dev *gspca_dev,
 	}
 
 resubmit:
+	if (!gspca_dev->streaming)
+		return;
 	/* resubmit the URB */
 	st = usb_submit_urb(urb, GFP_ATOMIC);
 	if (st < 0)
@@ -330,7 +332,7 @@ static void isoc_irq(struct urb *urb)
 	struct gspca_dev *gspca_dev = (struct gspca_dev *) urb->context;
 
 	gspca_dbg(gspca_dev, D_PACK, "isoc irq\n");
-	if (!vb2_start_streaming_called(&gspca_dev->queue))
+	if (!gspca_dev->streaming)
 		return;
 	fill_frame(gspca_dev, urb);
 }
@@ -344,7 +346,7 @@ static void bulk_irq(struct urb *urb)
 	int st;
 
 	gspca_dbg(gspca_dev, D_PACK, "bulk irq\n");
-	if (!vb2_start_streaming_called(&gspca_dev->queue))
+	if (!gspca_dev->streaming)
 		return;
 	switch (urb->status) {
 	case 0:
@@ -367,6 +369,8 @@ static void bulk_irq(struct urb *urb)
 				urb->actual_length);
 
 resubmit:
+	if (!gspca_dev->streaming)
+		return;
 	/* resubmit the URB */
 	if (gspca_dev->cam.bulk_nurbs != 0) {
 		st = usb_submit_urb(urb, GFP_ATOMIC);
@@ -1638,6 +1642,8 @@ void gspca_disconnect(struct usb_interface *intf)
 
 	mutex_lock(&gspca_dev->usb_lock);
 	gspca_dev->present = false;
+	destroy_urbs(gspca_dev);
+	gspca_input_destroy_urb(gspca_dev);
 
 	vb2_queue_error(&gspca_dev->queue);
 
diff --git a/drivers/media/usb/hackrf/Kconfig b/drivers/media/usb/hackrf/Kconfig
index 937e6f5c1e8e..072e186018f5 100644
--- a/drivers/media/usb/hackrf/Kconfig
+++ b/drivers/media/usb/hackrf/Kconfig
@@ -2,7 +2,7 @@ config USB_HACKRF
 	tristate "HackRF"
 	depends on VIDEO_V4L2
 	select VIDEOBUF2_VMALLOC
-	---help---
+	help
 	  This is a video4linux2 driver for HackRF SDR device.
 
 	  To compile this driver as a module, choose M here: the
diff --git a/drivers/media/usb/hdpvr/Kconfig b/drivers/media/usb/hdpvr/Kconfig
index d73d9a1952b4..9e78c0c32651 100644
--- a/drivers/media/usb/hdpvr/Kconfig
+++ b/drivers/media/usb/hdpvr/Kconfig
@@ -2,7 +2,7 @@
 config VIDEO_HDPVR
 	tristate "Hauppauge HD PVR support"
 	depends on VIDEO_DEV && VIDEO_V4L2
-	---help---
+	help
 	  This is a video4linux driver for Hauppauge's HD PVR USB device.
 
 	  To compile this driver as a module, choose M here: the
diff --git a/drivers/media/usb/hdpvr/hdpvr-video.c b/drivers/media/usb/hdpvr/hdpvr-video.c
index e082086428a4..3804aa3fb50f 100644
--- a/drivers/media/usb/hdpvr/hdpvr-video.c
+++ b/drivers/media/usb/hdpvr/hdpvr-video.c
@@ -769,8 +769,7 @@ static int vidioc_enum_input(struct file *file, void *_fh, struct v4l2_input *i)
 
 	i->type = V4L2_INPUT_TYPE_CAMERA;
 
-	strncpy(i->name, iname[n], sizeof(i->name) - 1);
-	i->name[sizeof(i->name) - 1] = '\0';
+	strscpy(i->name, iname[n], sizeof(i->name));
 
 	i->audioset = 1<<HDPVR_RCA_FRONT | 1<<HDPVR_RCA_BACK | 1<<HDPVR_SPDIF;
 
@@ -841,8 +840,7 @@ static int vidioc_enumaudio(struct file *file, void *priv,
 
 	audio->capability = V4L2_AUDCAP_STEREO;
 
-	strncpy(audio->name, audio_iname[n], sizeof(audio->name) - 1);
-	audio->name[sizeof(audio->name) - 1] = '\0';
+	strscpy(audio->name, audio_iname[n], sizeof(audio->name));
 
 	return 0;
 }
@@ -874,7 +872,6 @@ static int vidioc_g_audio(struct file *file, void *private_data,
 	audio->index = dev->options.audio_input;
 	audio->capability = V4L2_AUDCAP_STEREO;
 	strscpy(audio->name, audio_iname[audio->index], sizeof(audio->name));
-	audio->name[sizeof(audio->name) - 1] = '\0';
 	return 0;
 }
 
@@ -991,7 +988,8 @@ static int vidioc_enum_fmt_vid_cap(struct file *file, void *private_data,
 		return -EINVAL;
 
 	f->flags = V4L2_FMT_FLAG_COMPRESSED;
-	strncpy(f->description, "MPEG2-TS with AVC/AAC streams", 32);
+	strscpy(f->description, "MPEG2-TS with AVC/AAC streams",
+		sizeof(f->description));
 	f->pixelformat = V4L2_PIX_FMT_MPEG;
 
 	return 0;
diff --git a/drivers/media/usb/pulse8-cec/Kconfig b/drivers/media/usb/pulse8-cec/Kconfig
index 18ead44824ba..11f1b75d3efd 100644
--- a/drivers/media/usb/pulse8-cec/Kconfig
+++ b/drivers/media/usb/pulse8-cec/Kconfig
@@ -4,7 +4,7 @@ config USB_PULSE8_CEC
 	select CEC_CORE
 	select SERIO
 	select SERIO_SERPORT
-	---help---
+	help
 	  This is a cec driver for the Pulse Eight HDMI CEC device.
 
 	  To compile this driver as a module, choose M here: the
diff --git a/drivers/media/usb/pulse8-cec/pulse8-cec.c b/drivers/media/usb/pulse8-cec/pulse8-cec.c
index b085b14f3f87..ea9ee74fa336 100644
--- a/drivers/media/usb/pulse8-cec/pulse8-cec.c
+++ b/drivers/media/usb/pulse8-cec/pulse8-cec.c
@@ -435,7 +435,7 @@ static int pulse8_setup(struct pulse8 *pulse8, struct serio *serio,
 	err = pulse8_send_and_wait(pulse8, cmd, 1, cmd[0], 0);
 	if (err)
 		return err;
-	strncpy(log_addrs->osd_name, data, 13);
+	strscpy(log_addrs->osd_name, data, sizeof(log_addrs->osd_name));
 	dev_dbg(pulse8->dev, "OSD name: %s\n", log_addrs->osd_name);
 
 	return 0;
@@ -566,7 +566,7 @@ static int pulse8_cec_adap_log_addr(struct cec_adapter *adap, u8 log_addr)
 		char *osd_str = cmd + 1;
 
 		cmd[0] = MSGCODE_SET_OSD_NAME;
-		strncpy(cmd + 1, adap->log_addrs.osd_name, 13);
+		strscpy(cmd + 1, adap->log_addrs.osd_name, sizeof(cmd) - 1);
 		if (osd_len < 4) {
 			memset(osd_str + osd_len, ' ', 4 - osd_len);
 			osd_len = 4;
diff --git a/drivers/media/usb/pvrusb2/Kconfig b/drivers/media/usb/pvrusb2/Kconfig
index 1ad913fc30bf..ac6612cf1bec 100644
--- a/drivers/media/usb/pvrusb2/Kconfig
+++ b/drivers/media/usb/pvrusb2/Kconfig
@@ -9,7 +9,7 @@ config VIDEO_PVRUSB2
 	select VIDEO_MSP3400
 	select VIDEO_WM8775
 	select VIDEO_CS53L32A
-	---help---
+	help
 	  This is a video4linux driver for Conexant 23416 based
 	  usb2 personal video recorder devices.
 
@@ -20,7 +20,7 @@ config VIDEO_PVRUSB2_SYSFS
 	bool "pvrusb2 sysfs support"
 	default y
 	depends on VIDEO_PVRUSB2 && SYSFS
-	---help---
+	help
 	  This option enables the operation of a sysfs based
 	  interface for query and control of the pvrusb2 driver.
 
@@ -43,7 +43,7 @@ config VIDEO_PVRUSB2_DVB
 	select MEDIA_TUNER_TDA18271 if MEDIA_SUBDRV_AUTOSELECT
 	select MEDIA_TUNER_SIMPLE if MEDIA_SUBDRV_AUTOSELECT
 	select MEDIA_TUNER_TDA8290 if MEDIA_SUBDRV_AUTOSELECT
-	---help---
+	help
 	  This option enables a DVB interface for the pvrusb2 driver.
 	  If your device does not support digital television, this
 	  feature will have no affect on the driver's operation.
@@ -53,7 +53,7 @@ config VIDEO_PVRUSB2_DVB
 config VIDEO_PVRUSB2_DEBUGIFC
 	bool "pvrusb2 debug interface"
 	depends on VIDEO_PVRUSB2_SYSFS
-	---help---
+	help
 	  This option enables the inclusion of a debug interface
 	  in the pvrusb2 driver, hosted through sysfs.
 
diff --git a/drivers/media/usb/pvrusb2/pvrusb2-hdw.c b/drivers/media/usb/pvrusb2/pvrusb2-hdw.c
index 446a999dd2ce..816c85786c2a 100644
--- a/drivers/media/usb/pvrusb2/pvrusb2-hdw.c
+++ b/drivers/media/usb/pvrusb2/pvrusb2-hdw.c
@@ -666,6 +666,8 @@ static int ctrl_get_input(struct pvr2_ctrl *cptr,int *vp)
 
 static int ctrl_check_input(struct pvr2_ctrl *cptr,int v)
 {
+	if (v < 0 || v > PVR2_CVAL_INPUT_MAX)
+		return 0;
 	return ((1 << v) & cptr->hdw->input_allowed_mask) != 0;
 }
 
@@ -2459,9 +2461,8 @@ struct pvr2_hdw *pvr2_hdw_create(struct usb_interface *intf,
 		if (!(qctrl.flags & V4L2_CTRL_FLAG_READ_ONLY)) {
 			ciptr->set_value = ctrl_cx2341x_set;
 		}
-		strncpy(hdw->mpeg_ctrl_info[idx].desc,qctrl.name,
-			PVR2_CTLD_INFO_DESC_SIZE);
-		hdw->mpeg_ctrl_info[idx].desc[PVR2_CTLD_INFO_DESC_SIZE-1] = 0;
+		strscpy(hdw->mpeg_ctrl_info[idx].desc, qctrl.name,
+			sizeof(hdw->mpeg_ctrl_info[idx].desc));
 		ciptr->default_value = qctrl.default_value;
 		switch (qctrl.type) {
 		default:
diff --git a/drivers/media/usb/pvrusb2/pvrusb2-hdw.h b/drivers/media/usb/pvrusb2/pvrusb2-hdw.h
index 25648add77e5..bd2b7a67b732 100644
--- a/drivers/media/usb/pvrusb2/pvrusb2-hdw.h
+++ b/drivers/media/usb/pvrusb2/pvrusb2-hdw.h
@@ -50,6 +50,7 @@
 #define PVR2_CVAL_INPUT_COMPOSITE 2
 #define PVR2_CVAL_INPUT_SVIDEO 3
 #define PVR2_CVAL_INPUT_RADIO 4
+#define PVR2_CVAL_INPUT_MAX PVR2_CVAL_INPUT_RADIO
 
 enum pvr2_config {
 	pvr2_config_empty,    /* No configuration */
diff --git a/drivers/media/usb/pvrusb2/pvrusb2-v4l2.c b/drivers/media/usb/pvrusb2/pvrusb2-v4l2.c
index 08d5b7aa3537..cb6668580d77 100644
--- a/drivers/media/usb/pvrusb2/pvrusb2-v4l2.c
+++ b/drivers/media/usb/pvrusb2/pvrusb2-v4l2.c
@@ -284,7 +284,7 @@ static int pvr2_enumaudio(struct file *file, void *priv, struct v4l2_audio *vin)
 
 	if (vin->index > 0)
 		return -EINVAL;
-	strncpy(vin->name, "PVRUSB2 Audio", 14);
+	strscpy(vin->name, "PVRUSB2 Audio", sizeof(vin->name));
 	vin->capability = V4L2_AUDCAP_STEREO;
 	return 0;
 }
@@ -293,7 +293,7 @@ static int pvr2_g_audio(struct file *file, void *priv, struct v4l2_audio *vin)
 {
 	/* pkt: FIXME: see above comment (VIDIOC_ENUMAUDIO) */
 	vin->index = 0;
-	strncpy(vin->name, "PVRUSB2 Audio", 14);
+	strscpy(vin->name, "PVRUSB2 Audio", sizeof(vin->name));
 	vin->capability = V4L2_AUDCAP_STEREO;
 	return 0;
 }
diff --git a/drivers/media/usb/pwc/Kconfig b/drivers/media/usb/pwc/Kconfig
index d63d0a850035..5f6d91edca41 100644
--- a/drivers/media/usb/pwc/Kconfig
+++ b/drivers/media/usb/pwc/Kconfig
@@ -2,7 +2,7 @@ config USB_PWC
 	tristate "USB Philips Cameras"
 	depends on VIDEO_V4L2
 	select VIDEOBUF2_VMALLOC
-	---help---
+	help
 	  Say Y or M here if you want to use one of these Philips & OEM
 	  webcams:
 	   * Philips PCA645, PCA646
@@ -41,7 +41,7 @@ config USB_PWC_INPUT_EVDEV
 	bool "USB Philips Cameras input events device support"
 	default y
 	depends on USB_PWC && (USB_PWC=INPUT || INPUT=y)
-	---help---
+	help
 	  This option makes USB Philips cameras register the snapshot button as
 	  an input device to report button events.
 
diff --git a/drivers/media/usb/pwc/pwc-ctrl.c b/drivers/media/usb/pwc/pwc-ctrl.c
index 655cef39eb3d..b681a184ef87 100644
--- a/drivers/media/usb/pwc/pwc-ctrl.c
+++ b/drivers/media/usb/pwc/pwc-ctrl.c
@@ -242,14 +242,14 @@ static int set_video_mode_Timon(struct pwc_device *pdev, int size, int pixfmt,
 	fps = (frames / 5) - 1;
 
 	/* Find a supported framerate with progressively higher compression */
-	pChoose = NULL;
-	while (*compression <= 3) {
+	do {
 		pChoose = &Timon_table[size][fps][*compression];
 		if (pChoose->alternate != 0)
 			break;
 		(*compression)++;
-	}
-	if (pChoose == NULL || pChoose->alternate == 0)
+	} while (*compression <= 3);
+
+	if (pChoose->alternate == 0)
 		return -ENOENT; /* Not supported. */
 
 	if (send_to_cam)
@@ -279,7 +279,7 @@ static int set_video_mode_Timon(struct pwc_device *pdev, int size, int pixfmt,
 static int set_video_mode_Kiara(struct pwc_device *pdev, int size, int pixfmt,
 				int frames, int *compression, int send_to_cam)
 {
-	const struct Kiara_table_entry *pChoose = NULL;
+	const struct Kiara_table_entry *pChoose;
 	int fps, ret = 0;
 
 	if (size >= PSZ_MAX || *compression < 0 || *compression > 3)
@@ -293,13 +293,14 @@ static int set_video_mode_Kiara(struct pwc_device *pdev, int size, int pixfmt,
 	fps = (frames / 5) - 1;
 
 	/* Find a supported framerate with progressively higher compression */
-	while (*compression <= 3) {
+	do {
 		pChoose = &Kiara_table[size][fps][*compression];
 		if (pChoose->alternate != 0)
 			break;
 		(*compression)++;
-	}
-	if (pChoose == NULL || pChoose->alternate == 0)
+	} while (*compression <= 3);
+
+	if (pChoose->alternate == 0)
 		return -ENOENT; /* Not supported. */
 
 	/* Firmware bug: video endpoint is 5, but commands are sent to endpoint 4 */
diff --git a/drivers/media/usb/rainshadow-cec/Kconfig b/drivers/media/usb/rainshadow-cec/Kconfig
index 030ef01b1ff0..6b00be618db8 100644
--- a/drivers/media/usb/rainshadow-cec/Kconfig
+++ b/drivers/media/usb/rainshadow-cec/Kconfig
@@ -4,7 +4,7 @@ config USB_RAINSHADOW_CEC
 	select CEC_CORE
 	select SERIO
 	select SERIO_SERPORT
-	---help---
+	help
 	  This is a cec driver for the RainShadow Tech HDMI CEC device.
 
 	  To compile this driver as a module, choose M here: the
diff --git a/drivers/media/usb/siano/Kconfig b/drivers/media/usb/siano/Kconfig
index d37b742d4f7a..cc5e5aa3c93a 100644
--- a/drivers/media/usb/siano/Kconfig
+++ b/drivers/media/usb/siano/Kconfig
@@ -8,6 +8,6 @@ config SMS_USB_DRV
 	depends on !RC_CORE || RC_CORE
 	select MEDIA_COMMON_OPTIONS
 	select SMS_SIANO_MDTV
-	---help---
+	help
 	  Choose if you would like to have Siano's support for USB interface
 
diff --git a/drivers/media/usb/stk1160/Kconfig b/drivers/media/usb/stk1160/Kconfig
index 425ed00e2599..03426e4437ea 100644
--- a/drivers/media/usb/stk1160/Kconfig
+++ b/drivers/media/usb/stk1160/Kconfig
@@ -2,7 +2,7 @@ config VIDEO_STK1160_COMMON
 	tristate "STK1160 USB video capture support"
 	depends on VIDEO_DEV && I2C
 
-	---help---
+	help
 	  This is a video4linux driver for STK1160 based video capture devices.
 
 	  To compile this driver as a module, choose M here: the
diff --git a/drivers/media/usb/stkwebcam/Kconfig b/drivers/media/usb/stkwebcam/Kconfig
index a6a00aa4fce6..ea9e04b3caaf 100644
--- a/drivers/media/usb/stkwebcam/Kconfig
+++ b/drivers/media/usb/stkwebcam/Kconfig
@@ -1,7 +1,7 @@
 config USB_STKWEBCAM
 	tristate "USB Syntek DC1125 Camera support"
 	depends on VIDEO_V4L2
-	---help---
+	help
 	  Say Y here if you want to use this type of camera.
 	  Supported devices are typically found in some Asus laptops,
 	  with USB id 174f:a311 and 05e1:0501. Other Syntek cameras
diff --git a/drivers/media/usb/tm6000/Kconfig b/drivers/media/usb/tm6000/Kconfig
index a43b77abd931..321ae691f4d9 100644
--- a/drivers/media/usb/tm6000/Kconfig
+++ b/drivers/media/usb/tm6000/Kconfig
@@ -18,7 +18,7 @@ config VIDEO_TM6000_ALSA
 	tristate "TV Master TM5600/6000/6010 audio support"
 	depends on VIDEO_TM6000 && SND
 	select SND_PCM
-	---help---
+	help
 	  This is a video4linux driver for direct (DMA) audio for
 	  TM5600/TM6000/TM6010 USB Devices.
 
@@ -29,5 +29,5 @@ config VIDEO_TM6000_DVB
 	tristate "DVB Support for tm6000 based TV cards"
 	depends on VIDEO_TM6000 && DVB_CORE && USB
 	select DVB_ZL10353
-	---help---
+	help
 	  This adds support for DVB cards based on the tm5600/tm6000 chip.
diff --git a/drivers/media/usb/usbtv/Kconfig b/drivers/media/usb/usbtv/Kconfig
index 14a0941fa0d0..2b4ac0848469 100644
--- a/drivers/media/usb/usbtv/Kconfig
+++ b/drivers/media/usb/usbtv/Kconfig
@@ -4,7 +4,7 @@ config VIDEO_USBTV
 	select SND_PCM
 	select VIDEOBUF2_VMALLOC
 
-	---help---
+	help
 	  This is a video4linux2 driver for USBTV007 based video capture devices.
 
 	  To compile this driver as a module, choose M here: the
diff --git a/drivers/media/usb/usbvision/Kconfig b/drivers/media/usb/usbvision/Kconfig
index 6b6afc5d8f7e..7aa080cb9884 100644
--- a/drivers/media/usb/usbvision/Kconfig
+++ b/drivers/media/usb/usbvision/Kconfig
@@ -3,7 +3,7 @@ config VIDEO_USBVISION
 	depends on I2C && VIDEO_V4L2
 	select VIDEO_TUNER
 	select VIDEO_SAA711X if MEDIA_SUBDRV_AUTOSELECT
-	---help---
+	help
 	  There are more than 50 different USB video devices based on
 	  NT1003/1004/1005 USB Bridges. This driver enables using those
 	  devices.
diff --git a/drivers/media/usb/usbvision/usbvision-core.c b/drivers/media/usb/usbvision/usbvision-core.c
index 92d166bf8c12..abc4eed832a3 100644
--- a/drivers/media/usb/usbvision/usbvision-core.c
+++ b/drivers/media/usb/usbvision/usbvision-core.c
@@ -2302,6 +2302,9 @@ int usbvision_init_isoc(struct usb_usbvision *usbvision)
 					   sb_size,
 					   GFP_KERNEL,
 					   &urb->transfer_dma);
+		if (!usbvision->sbuf[buf_idx].data)
+			return -ENOMEM;
+
 		urb->dev = dev;
 		urb->context = usbvision;
 		urb->pipe = usb_rcvisocpipe(dev, usbvision->video_endp);
diff --git a/drivers/media/usb/uvc/Kconfig b/drivers/media/usb/uvc/Kconfig
index 6ed85efabcaa..94937d0cc2e3 100644
--- a/drivers/media/usb/uvc/Kconfig
+++ b/drivers/media/usb/uvc/Kconfig
@@ -2,7 +2,7 @@ config USB_VIDEO_CLASS
 	tristate "USB Video Class (UVC)"
 	depends on VIDEO_V4L2
 	select VIDEOBUF2_VMALLOC
-	---help---
+	help
 	  Support for the USB Video Class (UVC).  Currently only video
 	  input devices, such as webcams, are supported.
 
@@ -13,7 +13,7 @@ config USB_VIDEO_CLASS_INPUT_EVDEV
 	default y
 	depends on USB_VIDEO_CLASS
 	depends on USB_VIDEO_CLASS=INPUT || INPUT=y
-	---help---
+	help
 	  This option makes USB Video Class devices register an input device
 	  to report button events.
 
diff --git a/drivers/media/usb/zr364xx/Kconfig b/drivers/media/usb/zr364xx/Kconfig
index ac429bca70e8..979b1d4f3f68 100644
--- a/drivers/media/usb/zr364xx/Kconfig
+++ b/drivers/media/usb/zr364xx/Kconfig
@@ -3,7 +3,7 @@ config USB_ZR364XX
 	depends on VIDEO_V4L2
 	select VIDEOBUF_GEN
 	select VIDEOBUF_VMALLOC
-	---help---
+	help
 	  Say Y here if you want to connect this type of camera to your
 	  computer's USB port.
 	  See <file:Documentation/media/v4l-drivers/zr364xx.rst> for more info
diff --git a/drivers/media/v4l2-core/Kconfig b/drivers/media/v4l2-core/Kconfig
index c0940f5c69b4..8402096f7796 100644
--- a/drivers/media/v4l2-core/Kconfig
+++ b/drivers/media/v4l2-core/Kconfig
@@ -13,7 +13,7 @@ config VIDEO_V4L2
 config VIDEO_ADV_DEBUG
 	bool "Enable advanced debug functionality on V4L2 drivers"
 	default n
-	---help---
+	help
 	  Say Y here to enable advanced debugging functionality on some
 	  V4L devices.
 	  In doubt, say N.
@@ -21,7 +21,7 @@ config VIDEO_ADV_DEBUG
 config VIDEO_FIXED_MINOR_RANGES
 	bool "Enable old-style fixed minor ranges on drivers/video devices"
 	default n
-	---help---
+	help
 	  Say Y here to enable the old-style fixed-range minor assignments.
 	  Only useful if you rely on the old behavior and use mknod instead of udev.
 
@@ -33,7 +33,7 @@ config VIDEO_PCI_SKELETON
 	depends on SAMPLES
 	depends on VIDEO_V4L2 && VIDEOBUF2_CORE
 	depends on VIDEOBUF2_MEMOPS && VIDEOBUF2_DMA_CONTIG
-	---help---
+	help
 	  Enable build of the skeleton PCI driver, used as a reference
 	  when developing new drivers.
 
@@ -51,7 +51,7 @@ config V4L2_FLASH_LED_CLASS
 	tristate "V4L2 flash API for LED flash class devices"
 	depends on VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API
 	depends on LEDS_CLASS_FLASH
-	---help---
+	help
 	  Say Y here to enable V4L2 flash API support for LED flash
 	  class drivers.
 
diff --git a/drivers/media/v4l2-core/v4l2-common.c b/drivers/media/v4l2-core/v4l2-common.c
index 663730f088cd..b5778b2ffa27 100644
--- a/drivers/media/v4l2-core/v4l2-common.c
+++ b/drivers/media/v4l2-core/v4l2-common.c
@@ -445,3 +445,189 @@ int v4l2_s_parm_cap(struct video_device *vdev,
 	return ret;
 }
 EXPORT_SYMBOL_GPL(v4l2_s_parm_cap);
+
+const struct v4l2_format_info *v4l2_format_info(u32 format)
+{
+	static const struct v4l2_format_info formats[] = {
+		/* RGB formats */
+		{ .format = V4L2_PIX_FMT_BGR24,   .mem_planes = 1, .comp_planes = 1, .bpp = { 3, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_RGB24,   .mem_planes = 1, .comp_planes = 1, .bpp = { 3, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_HSV24,   .mem_planes = 1, .comp_planes = 1, .bpp = { 3, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_BGR32,   .mem_planes = 1, .comp_planes = 1, .bpp = { 4, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_XBGR32,  .mem_planes = 1, .comp_planes = 1, .bpp = { 4, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_RGB32,   .mem_planes = 1, .comp_planes = 1, .bpp = { 4, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_XRGB32,  .mem_planes = 1, .comp_planes = 1, .bpp = { 4, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_HSV32,   .mem_planes = 1, .comp_planes = 1, .bpp = { 4, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_ARGB32,  .mem_planes = 1, .comp_planes = 1, .bpp = { 4, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_ABGR32,  .mem_planes = 1, .comp_planes = 1, .bpp = { 4, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_GREY,    .mem_planes = 1, .comp_planes = 1, .bpp = { 1, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
+
+		/* YUV packed formats */
+		{ .format = V4L2_PIX_FMT_YUYV,    .mem_planes = 1, .comp_planes = 1, .bpp = { 2, 0, 0, 0 }, .hdiv = 2, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_YVYU,    .mem_planes = 1, .comp_planes = 1, .bpp = { 2, 0, 0, 0 }, .hdiv = 2, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_UYVY,    .mem_planes = 1, .comp_planes = 1, .bpp = { 2, 0, 0, 0 }, .hdiv = 2, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_VYUY,    .mem_planes = 1, .comp_planes = 1, .bpp = { 2, 0, 0, 0 }, .hdiv = 2, .vdiv = 1 },
+
+		/* YUV planar formats */
+		{ .format = V4L2_PIX_FMT_NV12,    .mem_planes = 1, .comp_planes = 2, .bpp = { 1, 2, 0, 0 }, .hdiv = 2, .vdiv = 2 },
+		{ .format = V4L2_PIX_FMT_NV21,    .mem_planes = 1, .comp_planes = 2, .bpp = { 1, 2, 0, 0 }, .hdiv = 2, .vdiv = 2 },
+		{ .format = V4L2_PIX_FMT_NV16,    .mem_planes = 1, .comp_planes = 2, .bpp = { 1, 2, 0, 0 }, .hdiv = 2, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_NV61,    .mem_planes = 1, .comp_planes = 2, .bpp = { 1, 2, 0, 0 }, .hdiv = 2, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_NV24,    .mem_planes = 1, .comp_planes = 2, .bpp = { 1, 2, 0, 0 }, .hdiv = 1, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_NV42,    .mem_planes = 1, .comp_planes = 2, .bpp = { 1, 2, 0, 0 }, .hdiv = 1, .vdiv = 1 },
+
+		{ .format = V4L2_PIX_FMT_YUV410,  .mem_planes = 1, .comp_planes = 3, .bpp = { 1, 1, 1, 0 }, .hdiv = 4, .vdiv = 4 },
+		{ .format = V4L2_PIX_FMT_YVU410,  .mem_planes = 1, .comp_planes = 3, .bpp = { 1, 1, 1, 0 }, .hdiv = 4, .vdiv = 4 },
+		{ .format = V4L2_PIX_FMT_YUV411P, .mem_planes = 1, .comp_planes = 3, .bpp = { 1, 1, 1, 0 }, .hdiv = 4, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_YUV420,  .mem_planes = 1, .comp_planes = 3, .bpp = { 1, 1, 1, 0 }, .hdiv = 2, .vdiv = 2 },
+		{ .format = V4L2_PIX_FMT_YVU420,  .mem_planes = 1, .comp_planes = 3, .bpp = { 1, 1, 1, 0 }, .hdiv = 2, .vdiv = 2 },
+		{ .format = V4L2_PIX_FMT_YUV422P, .mem_planes = 1, .comp_planes = 3, .bpp = { 1, 1, 1, 0 }, .hdiv = 2, .vdiv = 1 },
+
+		/* YUV planar formats, non contiguous variant */
+		{ .format = V4L2_PIX_FMT_YUV420M, .mem_planes = 3, .comp_planes = 3, .bpp = { 1, 1, 1, 0 }, .hdiv = 2, .vdiv = 2 },
+		{ .format = V4L2_PIX_FMT_YVU420M, .mem_planes = 3, .comp_planes = 3, .bpp = { 1, 1, 1, 0 }, .hdiv = 2, .vdiv = 2 },
+		{ .format = V4L2_PIX_FMT_YUV422M, .mem_planes = 3, .comp_planes = 3, .bpp = { 1, 1, 1, 0 }, .hdiv = 2, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_YVU422M, .mem_planes = 3, .comp_planes = 3, .bpp = { 1, 1, 1, 0 }, .hdiv = 2, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_YUV444M, .mem_planes = 3, .comp_planes = 3, .bpp = { 1, 1, 1, 0 }, .hdiv = 1, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_YVU444M, .mem_planes = 3, .comp_planes = 3, .bpp = { 1, 1, 1, 0 }, .hdiv = 1, .vdiv = 1 },
+
+		{ .format = V4L2_PIX_FMT_NV12M,   .mem_planes = 2, .comp_planes = 2, .bpp = { 1, 2, 0, 0 }, .hdiv = 2, .vdiv = 2 },
+		{ .format = V4L2_PIX_FMT_NV21M,   .mem_planes = 2, .comp_planes = 2, .bpp = { 1, 2, 0, 0 }, .hdiv = 2, .vdiv = 2 },
+		{ .format = V4L2_PIX_FMT_NV16M,   .mem_planes = 2, .comp_planes = 2, .bpp = { 1, 2, 0, 0 }, .hdiv = 2, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_NV61M,   .mem_planes = 2, .comp_planes = 2, .bpp = { 1, 2, 0, 0 }, .hdiv = 2, .vdiv = 1 },
+
+		/* Bayer RGB formats */
+		{ .format = V4L2_PIX_FMT_SBGGR8,	.mem_planes = 1, .comp_planes = 1, .bpp = { 1, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_SGBRG8,	.mem_planes = 1, .comp_planes = 1, .bpp = { 1, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_SGRBG8,	.mem_planes = 1, .comp_planes = 1, .bpp = { 1, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_SRGGB8,	.mem_planes = 1, .comp_planes = 1, .bpp = { 1, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_SBGGR10,	.mem_planes = 1, .comp_planes = 1, .bpp = { 2, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_SGBRG10,	.mem_planes = 1, .comp_planes = 1, .bpp = { 2, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_SGRBG10,	.mem_planes = 1, .comp_planes = 1, .bpp = { 2, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_SRGGB10,	.mem_planes = 1, .comp_planes = 1, .bpp = { 2, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_SBGGR10ALAW8,	.mem_planes = 1, .comp_planes = 1, .bpp = { 1, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_SGBRG10ALAW8,	.mem_planes = 1, .comp_planes = 1, .bpp = { 1, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_SGRBG10ALAW8,	.mem_planes = 1, .comp_planes = 1, .bpp = { 1, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_SRGGB10ALAW8,	.mem_planes = 1, .comp_planes = 1, .bpp = { 1, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_SBGGR10DPCM8,	.mem_planes = 1, .comp_planes = 1, .bpp = { 1, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_SGBRG10DPCM8,	.mem_planes = 1, .comp_planes = 1, .bpp = { 1, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_SGRBG10DPCM8,	.mem_planes = 1, .comp_planes = 1, .bpp = { 1, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_SRGGB10DPCM8,	.mem_planes = 1, .comp_planes = 1, .bpp = { 1, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_SBGGR12,	.mem_planes = 1, .comp_planes = 1, .bpp = { 2, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_SGBRG12,	.mem_planes = 1, .comp_planes = 1, .bpp = { 2, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_SGRBG12,	.mem_planes = 1, .comp_planes = 1, .bpp = { 2, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_SRGGB12,	.mem_planes = 1, .comp_planes = 1, .bpp = { 2, 0, 0, 0 }, .hdiv = 1, .vdiv = 1 },
+	};
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(formats); ++i)
+		if (formats[i].format == format)
+			return &formats[i];
+	return NULL;
+}
+EXPORT_SYMBOL(v4l2_format_info);
+
+static inline unsigned int v4l2_format_block_width(const struct v4l2_format_info *info, int plane)
+{
+	if (!info->block_w[plane])
+		return 1;
+	return info->block_w[plane];
+}
+
+static inline unsigned int v4l2_format_block_height(const struct v4l2_format_info *info, int plane)
+{
+	if (!info->block_h[plane])
+		return 1;
+	return info->block_h[plane];
+}
+
+int v4l2_fill_pixfmt_mp(struct v4l2_pix_format_mplane *pixfmt,
+			 int pixelformat, int width, int height)
+{
+	const struct v4l2_format_info *info;
+	struct v4l2_plane_pix_format *plane;
+	int i;
+
+	info = v4l2_format_info(pixelformat);
+	if (!info)
+		return -EINVAL;
+
+	pixfmt->width = width;
+	pixfmt->height = height;
+	pixfmt->pixelformat = pixelformat;
+	pixfmt->num_planes = info->mem_planes;
+
+	if (info->mem_planes == 1) {
+		plane = &pixfmt->plane_fmt[0];
+		plane->bytesperline = ALIGN(width, v4l2_format_block_width(info, 0)) * info->bpp[0];
+		plane->sizeimage = 0;
+
+		for (i = 0; i < info->comp_planes; i++) {
+			unsigned int hdiv = (i == 0) ? 1 : info->hdiv;
+			unsigned int vdiv = (i == 0) ? 1 : info->vdiv;
+			unsigned int aligned_width;
+			unsigned int aligned_height;
+
+			aligned_width = ALIGN(width, v4l2_format_block_width(info, i));
+			aligned_height = ALIGN(height, v4l2_format_block_height(info, i));
+
+			plane->sizeimage += info->bpp[i] *
+				DIV_ROUND_UP(aligned_width, hdiv) *
+				DIV_ROUND_UP(aligned_height, vdiv);
+		}
+	} else {
+		for (i = 0; i < info->comp_planes; i++) {
+			unsigned int hdiv = (i == 0) ? 1 : info->hdiv;
+			unsigned int vdiv = (i == 0) ? 1 : info->vdiv;
+			unsigned int aligned_width;
+			unsigned int aligned_height;
+
+			aligned_width = ALIGN(width, v4l2_format_block_width(info, i));
+			aligned_height = ALIGN(height, v4l2_format_block_height(info, i));
+
+			plane = &pixfmt->plane_fmt[i];
+			plane->bytesperline =
+				info->bpp[i] * DIV_ROUND_UP(aligned_width, hdiv);
+			plane->sizeimage =
+				plane->bytesperline * DIV_ROUND_UP(aligned_height, vdiv);
+		}
+	}
+	return 0;
+}
+EXPORT_SYMBOL_GPL(v4l2_fill_pixfmt_mp);
+
+int v4l2_fill_pixfmt(struct v4l2_pix_format *pixfmt, int pixelformat, int width, int height)
+{
+	const struct v4l2_format_info *info;
+	int i;
+
+	info = v4l2_format_info(pixelformat);
+	if (!info)
+		return -EINVAL;
+
+	/* Single planar API cannot be used for multi plane formats. */
+	if (info->mem_planes > 1)
+		return -EINVAL;
+
+	pixfmt->width = width;
+	pixfmt->height = height;
+	pixfmt->pixelformat = pixelformat;
+	pixfmt->bytesperline = ALIGN(width, v4l2_format_block_width(info, 0)) * info->bpp[0];
+	pixfmt->sizeimage = 0;
+
+	for (i = 0; i < info->comp_planes; i++) {
+		unsigned int hdiv = (i == 0) ? 1 : info->hdiv;
+		unsigned int vdiv = (i == 0) ? 1 : info->vdiv;
+		unsigned int aligned_width;
+		unsigned int aligned_height;
+
+		aligned_width = ALIGN(width, v4l2_format_block_width(info, i));
+		aligned_height = ALIGN(height, v4l2_format_block_height(info, i));
+
+		pixfmt->sizeimage += info->bpp[i] *
+			DIV_ROUND_UP(aligned_width, hdiv) *
+			DIV_ROUND_UP(aligned_height, vdiv);
+	}
+	return 0;
+}
+EXPORT_SYMBOL_GPL(v4l2_fill_pixfmt);
diff --git a/drivers/media/v4l2-core/v4l2-ctrls.c b/drivers/media/v4l2-core/v4l2-ctrls.c
index b79d3bbd8350..89a1fe564675 100644
--- a/drivers/media/v4l2-core/v4l2-ctrls.c
+++ b/drivers/media/v4l2-core/v4l2-ctrls.c
@@ -828,6 +828,10 @@ const char *v4l2_ctrl_get_name(u32 id)
 	case V4L2_CID_MPEG_VIDEO_H264_CONSTRAINED_INTRA_PREDICTION:
 								return "H264 Constrained Intra Pred";
 	case V4L2_CID_MPEG_VIDEO_H264_CHROMA_QP_INDEX_OFFSET:	return "H264 Chroma QP Index Offset";
+	case V4L2_CID_MPEG_VIDEO_H264_I_FRAME_MIN_QP:		return "H264 I-Frame Minimum QP Value";
+	case V4L2_CID_MPEG_VIDEO_H264_I_FRAME_MAX_QP:		return "H264 I-Frame Maximum QP Value";
+	case V4L2_CID_MPEG_VIDEO_H264_P_FRAME_MIN_QP:		return "H264 P-Frame Minimum QP Value";
+	case V4L2_CID_MPEG_VIDEO_H264_P_FRAME_MAX_QP:		return "H264 P-Frame Maximum QP Value";
 	case V4L2_CID_MPEG_VIDEO_MPEG4_I_FRAME_QP:		return "MPEG4 I-Frame QP Value";
 	case V4L2_CID_MPEG_VIDEO_MPEG4_P_FRAME_QP:		return "MPEG4 P-Frame QP Value";
 	case V4L2_CID_MPEG_VIDEO_MPEG4_B_FRAME_QP:		return "MPEG4 B-Frame QP Value";
@@ -849,6 +853,9 @@ const char *v4l2_ctrl_get_name(u32 id)
 	case V4L2_CID_MPEG_VIDEO_FORCE_KEY_FRAME:		return "Force Key Frame";
 	case V4L2_CID_MPEG_VIDEO_MPEG2_SLICE_PARAMS:		return "MPEG-2 Slice Parameters";
 	case V4L2_CID_MPEG_VIDEO_MPEG2_QUANTIZATION:		return "MPEG-2 Quantization Matrices";
+	case V4L2_CID_MPEG_VIDEO_FWHT_PARAMS:			return "FWHT Stateless Parameters";
+	case V4L2_CID_FWHT_I_FRAME_QP:				return "FWHT I-Frame QP Value";
+	case V4L2_CID_FWHT_P_FRAME_QP:				return "FWHT P-Frame QP Value";
 
 	/* VPX controls */
 	case V4L2_CID_MPEG_VIDEO_VPX_NUM_PARTITIONS:		return "VPX Number of Partitions";
@@ -1303,6 +1310,9 @@ void v4l2_ctrl_fill(u32 id, const char **name, enum v4l2_ctrl_type *type,
 	case V4L2_CID_MPEG_VIDEO_MPEG2_QUANTIZATION:
 		*type = V4L2_CTRL_TYPE_MPEG2_QUANTIZATION;
 		break;
+	case V4L2_CID_MPEG_VIDEO_FWHT_PARAMS:
+		*type = V4L2_CTRL_TYPE_FWHT_PARAMS;
+		break;
 	default:
 		*type = V4L2_CTRL_TYPE_INTEGER;
 		break;
@@ -1599,7 +1609,7 @@ static int std_validate(const struct v4l2_ctrl *ctrl, u32 idx,
 	case V4L2_CTRL_TYPE_INTEGER_MENU:
 		if (ptr.p_s32[idx] < ctrl->minimum || ptr.p_s32[idx] > ctrl->maximum)
 			return -ERANGE;
-		if (ctrl->menu_skip_mask & (1 << ptr.p_s32[idx]))
+		if (ctrl->menu_skip_mask & (1ULL << ptr.p_s32[idx]))
 			return -EINVAL;
 		if (ctrl->type == V4L2_CTRL_TYPE_MENU &&
 		    ctrl->qmenu[ptr.p_s32[idx]][0] == '\0')
@@ -1669,6 +1679,9 @@ static int std_validate(const struct v4l2_ctrl *ctrl, u32 idx,
 	case V4L2_CTRL_TYPE_MPEG2_QUANTIZATION:
 		return 0;
 
+	case V4L2_CTRL_TYPE_FWHT_PARAMS:
+		return 0;
+
 	default:
 		return -EINVAL;
 	}
@@ -2249,6 +2262,9 @@ static struct v4l2_ctrl *v4l2_ctrl_new(struct v4l2_ctrl_handler *hdl,
 	case V4L2_CTRL_TYPE_MPEG2_QUANTIZATION:
 		elem_size = sizeof(struct v4l2_ctrl_mpeg2_quantization);
 		break;
+	case V4L2_CTRL_TYPE_FWHT_PARAMS:
+		elem_size = sizeof(struct v4l2_ctrl_fwht_params);
+		break;
 	default:
 		if (type < V4L2_CTRL_COMPOUND_TYPES)
 			elem_size = sizeof(s32);
@@ -2918,7 +2934,7 @@ int v4l2_querymenu(struct v4l2_ctrl_handler *hdl, struct v4l2_querymenu *qm)
 		return -EINVAL;
 
 	/* Use mask to see if this menu item should be skipped */
-	if (ctrl->menu_skip_mask & (1 << i))
+	if (ctrl->menu_skip_mask & (1ULL << i))
 		return -EINVAL;
 	/* Empty menu items should also be skipped */
 	if (ctrl->type == V4L2_CTRL_TYPE_MENU) {
@@ -3899,18 +3915,19 @@ void v4l2_ctrl_request_complete(struct media_request *req,
 }
 EXPORT_SYMBOL(v4l2_ctrl_request_complete);
 
-void v4l2_ctrl_request_setup(struct media_request *req,
+int v4l2_ctrl_request_setup(struct media_request *req,
 			     struct v4l2_ctrl_handler *main_hdl)
 {
 	struct media_request_object *obj;
 	struct v4l2_ctrl_handler *hdl;
 	struct v4l2_ctrl_ref *ref;
+	int ret = 0;
 
 	if (!req || !main_hdl)
-		return;
+		return 0;
 
 	if (WARN_ON(req->state != MEDIA_REQUEST_STATE_QUEUED))
-		return;
+		return -EBUSY;
 
 	/*
 	 * Note that it is valid if nothing was found. It means
@@ -3919,10 +3936,10 @@ void v4l2_ctrl_request_setup(struct media_request *req,
 	 */
 	obj = media_request_object_find(req, &req_ops, main_hdl);
 	if (!obj)
-		return;
+		return 0;
 	if (obj->completed) {
 		media_request_object_put(obj);
-		return;
+		return -EBUSY;
 	}
 	hdl = container_of(obj, struct v4l2_ctrl_handler, req_obj);
 
@@ -3990,12 +4007,15 @@ void v4l2_ctrl_request_setup(struct media_request *req,
 				update_from_auto_cluster(master);
 		}
 
-		try_or_set_cluster(NULL, master, true, 0);
-
+		ret = try_or_set_cluster(NULL, master, true, 0);
 		v4l2_ctrl_unlock(master);
+
+		if (ret)
+			break;
 	}
 
 	media_request_object_put(obj);
+	return ret;
 }
 EXPORT_SYMBOL(v4l2_ctrl_request_setup);
 
diff --git a/drivers/media/v4l2-core/v4l2-device.c b/drivers/media/v4l2-core/v4l2-device.c
index e0ddb9a52bd1..7cca0de1b730 100644
--- a/drivers/media/v4l2-core/v4l2-device.c
+++ b/drivers/media/v4l2-core/v4l2-device.c
@@ -216,10 +216,18 @@ error_module:
 }
 EXPORT_SYMBOL_GPL(v4l2_device_register_subdev);
 
+static void v4l2_subdev_release(struct v4l2_subdev *sd)
+{
+	struct module *owner = !sd->owner_v4l2_dev ? sd->owner : NULL;
+
+	if (sd->internal_ops && sd->internal_ops->release)
+		sd->internal_ops->release(sd);
+	module_put(owner);
+}
+
 static void v4l2_device_release_subdev_node(struct video_device *vdev)
 {
-	struct v4l2_subdev *sd = video_get_drvdata(vdev);
-	sd->devnode = NULL;
+	v4l2_subdev_release(video_get_drvdata(vdev));
 	kfree(vdev);
 }
 
@@ -318,8 +326,9 @@ void v4l2_device_unregister_subdev(struct v4l2_subdev *sd)
 		media_device_unregister_entity(&sd->entity);
 	}
 #endif
-	video_unregister_device(sd->devnode);
-	if (!sd->owner_v4l2_dev)
-		module_put(sd->owner);
+	if (sd->devnode)
+		video_unregister_device(sd->devnode);
+	else
+		v4l2_subdev_release(sd);
 }
 EXPORT_SYMBOL_GPL(v4l2_device_unregister_subdev);
diff --git a/drivers/media/v4l2-core/v4l2-fwnode.c b/drivers/media/v4l2-core/v4l2-fwnode.c
index 20571846e636..ea1ed88f9dc8 100644
--- a/drivers/media/v4l2-core/v4l2-fwnode.c
+++ b/drivers/media/v4l2-core/v4l2-fwnode.c
@@ -163,7 +163,7 @@ static int v4l2_fwnode_endpoint_parse_csi2_bus(struct fwnode_handle *fwnode,
 		}
 
 		if (use_default_lane_mapping)
-			pr_debug("using default lane mapping\n");
+			pr_debug("no lane mapping given, using defaults\n");
 	}
 
 	rval = fwnode_property_read_u32_array(fwnode, "data-lanes", NULL, 0);
@@ -175,6 +175,10 @@ static int v4l2_fwnode_endpoint_parse_csi2_bus(struct fwnode_handle *fwnode,
 					       num_data_lanes);
 
 		have_data_lanes = true;
+		if (use_default_lane_mapping) {
+			pr_debug("data-lanes property exists; disabling default mapping\n");
+			use_default_lane_mapping = false;
+		}
 	}
 
 	for (i = 0; i < num_data_lanes; i++) {
@@ -225,6 +229,10 @@ static int v4l2_fwnode_endpoint_parse_csi2_bus(struct fwnode_handle *fwnode,
 	if (bus_type == V4L2_MBUS_CSI2_DPHY ||
 	    bus_type == V4L2_MBUS_CSI2_CPHY || lanes_used ||
 	    have_clk_lane || (flags & ~V4L2_MBUS_CSI2_CONTINUOUS_CLOCK)) {
+		/* Only D-PHY has a clock lane. */
+		unsigned int dfl_data_lane_index =
+			bus_type == V4L2_MBUS_CSI2_DPHY;
+
 		bus->flags = flags;
 		if (bus_type == V4L2_MBUS_UNKNOWN)
 			vep->bus_type = V4L2_MBUS_CSI2_DPHY;
@@ -233,7 +241,7 @@ static int v4l2_fwnode_endpoint_parse_csi2_bus(struct fwnode_handle *fwnode,
 		if (use_default_lane_mapping) {
 			bus->clock_lane = 0;
 			for (i = 0; i < num_data_lanes; i++)
-				bus->data_lanes[i] = 1 + i;
+				bus->data_lanes[i] = dfl_data_lane_index + i;
 		} else {
 			bus->clock_lane = clock_lane;
 			for (i = 0; i < num_data_lanes; i++)
@@ -820,7 +828,10 @@ error:
  * underneath the fwnode identified by the previous tuple, etc. until you
  * reached the fwnode you need.
  *
- * An example with a graph, as defined in Documentation/acpi/dsd/graph.txt:
+ * THIS EXAMPLE EXISTS MERELY TO DOCUMENT THIS FUNCTION. DO NOT USE IT AS A
+ * REFERENCE IN HOW ACPI TABLES SHOULD BE WRITTEN!! See documentation under
+ * Documentation/acpi/dsd instead and especially graph.txt,
+ * data-node-references.txt and leds.txt .
  *
  *	Scope (\_SB.PCI0.I2C2)
  *	{
diff --git a/drivers/media/v4l2-core/v4l2-ioctl.c b/drivers/media/v4l2-core/v4l2-ioctl.c
index f6d663934648..ac87c3e37280 100644
--- a/drivers/media/v4l2-core/v4l2-ioctl.c
+++ b/drivers/media/v4l2-core/v4l2-ioctl.c
@@ -1337,6 +1337,7 @@ static void v4l_fill_fmtdesc(struct v4l2_fmtdesc *fmt)
 		case V4L2_PIX_FMT_VP9:		descr = "VP9"; break;
 		case V4L2_PIX_FMT_HEVC:		descr = "HEVC"; break; /* aka H.265 */
 		case V4L2_PIX_FMT_FWHT:		descr = "FWHT"; break; /* used in vicodec */
+		case V4L2_PIX_FMT_FWHT_STATELESS:	descr = "FWHT Stateless"; break; /* used in vicodec */
 		case V4L2_PIX_FMT_CPIA1:	descr = "GSPCA CPiA YUV"; break;
 		case V4L2_PIX_FMT_WNVA:		descr = "WNVA"; break;
 		case V4L2_PIX_FMT_SN9C10X:	descr = "GSPCA SN9C10X"; break;
@@ -1373,7 +1374,7 @@ static void v4l_fill_fmtdesc(struct v4l2_fmtdesc *fmt)
 	}
 
 	if (descr)
-		WARN_ON(strscpy(fmt->description, descr, sz) >= sz);
+		WARN_ON(strscpy(fmt->description, descr, sz) < 0);
 	fmt->flags = flags;
 }
 
diff --git a/drivers/media/v4l2-core/v4l2-subdev.c b/drivers/media/v4l2-core/v4l2-subdev.c
index f5f0d71ec745..d75815ab0d7b 100644
--- a/drivers/media/v4l2-core/v4l2-subdev.c
+++ b/drivers/media/v4l2-core/v4l2-subdev.c
@@ -18,6 +18,7 @@
 
 #include <linux/ioctl.h>
 #include <linux/mm.h>
+#include <linux/module.h>
 #include <linux/slab.h>
 #include <linux/types.h>
 #include <linux/videodev2.h>
@@ -54,9 +55,6 @@ static int subdev_open(struct file *file)
 	struct video_device *vdev = video_devdata(file);
 	struct v4l2_subdev *sd = vdev_to_v4l2_subdev(vdev);
 	struct v4l2_subdev_fh *subdev_fh;
-#if defined(CONFIG_MEDIA_CONTROLLER)
-	struct media_entity *entity = NULL;
-#endif
 	int ret;
 
 	subdev_fh = kzalloc(sizeof(*subdev_fh), GFP_KERNEL);
@@ -73,12 +71,15 @@ static int subdev_open(struct file *file)
 	v4l2_fh_add(&subdev_fh->vfh);
 	file->private_data = &subdev_fh->vfh;
 #if defined(CONFIG_MEDIA_CONTROLLER)
-	if (sd->v4l2_dev->mdev) {
-		entity = media_entity_get(&sd->entity);
-		if (!entity) {
+	if (sd->v4l2_dev->mdev && sd->entity.graph_obj.mdev->dev) {
+		struct module *owner;
+
+		owner = sd->entity.graph_obj.mdev->dev->driver->owner;
+		if (!try_module_get(owner)) {
 			ret = -EBUSY;
 			goto err;
 		}
+		subdev_fh->owner = owner;
 	}
 #endif
 
@@ -91,9 +92,7 @@ static int subdev_open(struct file *file)
 	return 0;
 
 err:
-#if defined(CONFIG_MEDIA_CONTROLLER)
-	media_entity_put(entity);
-#endif
+	module_put(subdev_fh->owner);
 	v4l2_fh_del(&subdev_fh->vfh);
 	v4l2_fh_exit(&subdev_fh->vfh);
 	subdev_fh_free(subdev_fh);
@@ -111,10 +110,7 @@ static int subdev_close(struct file *file)
 
 	if (sd->internal_ops && sd->internal_ops->close)
 		sd->internal_ops->close(sd, subdev_fh);
-#if defined(CONFIG_MEDIA_CONTROLLER)
-	if (sd->v4l2_dev->mdev)
-		media_entity_put(&sd->entity);
-#endif
+	module_put(subdev_fh->owner);
 	v4l2_fh_del(vfh);
 	v4l2_fh_exit(vfh);
 	subdev_fh_free(subdev_fh);
diff --git a/drivers/memory/Makefile b/drivers/memory/Makefile
index 90161dec6fa5..91ae4eb0e913 100644
--- a/drivers/memory/Makefile
+++ b/drivers/memory/Makefile
@@ -28,6 +28,9 @@ ti-emif-sram-objs		:= ti-emif-pm.o ti-emif-sram-pm.o
 
 AFLAGS_ti-emif-sram-pm.o	:=-Wa,-march=armv7-a
 
-include drivers/memory/Makefile.asm-offsets
-
 drivers/memory/ti-emif-sram-pm.o: include/generated/ti-emif-asm-offsets.h
+
+include/generated/ti-emif-asm-offsets.h: drivers/memory/emif-asm-offsets.s FORCE
+	$(call filechk,offsets,__TI_EMIF_ASM_OFFSETS_H__)
+
+targets += emif-asm-offsets.s
diff --git a/drivers/memory/Makefile.asm-offsets b/drivers/memory/Makefile.asm-offsets
deleted file mode 100644
index 0447e174c752..000000000000
--- a/drivers/memory/Makefile.asm-offsets
+++ /dev/null
@@ -1,4 +0,0 @@
-include/generated/ti-emif-asm-offsets.h: drivers/memory/emif-asm-offsets.s FORCE
-	$(call filechk,offsets,__TI_EMIF_ASM_OFFSETS_H__)
-
-targets += emif-asm-offsets.s
diff --git a/drivers/message/fusion/mptbase.c b/drivers/message/fusion/mptbase.c
index ba551d8dfba4..d8882b0a1338 100644
--- a/drivers/message/fusion/mptbase.c
+++ b/drivers/message/fusion/mptbase.c
@@ -642,7 +642,7 @@ mptbase_reply(MPT_ADAPTER *ioc, MPT_FRAME_HDR *req, MPT_FRAME_HDR *reply)
 			freereq = 0;
 		if (event != MPI_EVENT_EVENT_CHANGE)
 			break;
-		/* else: fall through */
+		/* fall through */
 	case MPI_FUNCTION_CONFIG:
 	case MPI_FUNCTION_SAS_IO_UNIT_CONTROL:
 		ioc->mptbase_cmds.status |= MPT_MGMT_STATUS_COMMAND_GOOD;
diff --git a/drivers/message/fusion/mptctl.c b/drivers/message/fusion/mptctl.c
index 8d22d6134a89..f9ac22413000 100644
--- a/drivers/message/fusion/mptctl.c
+++ b/drivers/message/fusion/mptctl.c
@@ -565,7 +565,7 @@ mptctl_event_process(MPT_ADAPTER *ioc, EventNotificationReply_t *pEvReply)
 	 * TODO - this define is not in MPI spec yet,
 	 * but they plan to set it to 0x21
 	 */
-	 if (event == 0x21 ) {
+	if (event == 0x21) {
 		ioc->aen_event_read_flag=1;
 		dctlprintk(ioc, printk(MYIOC_s_DEBUG_FMT "Raised SIGIO to application\n",
 		    ioc->name));
diff --git a/drivers/message/fusion/mptsas.c b/drivers/message/fusion/mptsas.c
index 612cb5bc1333..6a79cd0ebe2b 100644
--- a/drivers/message/fusion/mptsas.c
+++ b/drivers/message/fusion/mptsas.c
@@ -2928,27 +2928,27 @@ mptsas_exp_repmanufacture_info(MPT_ADAPTER *ioc,
 	if (ioc->sas_mgmt.status & MPT_MGMT_STATUS_RF_VALID) {
 		u8 *tmp;
 
-	smprep = (SmpPassthroughReply_t *)ioc->sas_mgmt.reply;
-	if (le16_to_cpu(smprep->ResponseDataLength) !=
-		sizeof(struct rep_manu_reply))
+		smprep = (SmpPassthroughReply_t *)ioc->sas_mgmt.reply;
+		if (le16_to_cpu(smprep->ResponseDataLength) !=
+		    sizeof(struct rep_manu_reply))
 			goto out_free;
 
-	manufacture_reply = data_out + sizeof(struct rep_manu_request);
-	strncpy(edev->vendor_id, manufacture_reply->vendor_id,
-		SAS_EXPANDER_VENDOR_ID_LEN);
-	strncpy(edev->product_id, manufacture_reply->product_id,
-		SAS_EXPANDER_PRODUCT_ID_LEN);
-	strncpy(edev->product_rev, manufacture_reply->product_rev,
-		SAS_EXPANDER_PRODUCT_REV_LEN);
-	edev->level = manufacture_reply->sas_format;
-	if (manufacture_reply->sas_format) {
-		strncpy(edev->component_vendor_id,
-			manufacture_reply->component_vendor_id,
+		manufacture_reply = data_out + sizeof(struct rep_manu_request);
+		strncpy(edev->vendor_id, manufacture_reply->vendor_id,
+			SAS_EXPANDER_VENDOR_ID_LEN);
+		strncpy(edev->product_id, manufacture_reply->product_id,
+			SAS_EXPANDER_PRODUCT_ID_LEN);
+		strncpy(edev->product_rev, manufacture_reply->product_rev,
+			SAS_EXPANDER_PRODUCT_REV_LEN);
+		edev->level = manufacture_reply->sas_format;
+		if (manufacture_reply->sas_format) {
+			strncpy(edev->component_vendor_id,
+				manufacture_reply->component_vendor_id,
 				SAS_EXPANDER_COMPONENT_VENDOR_ID_LEN);
-		tmp = (u8 *)&manufacture_reply->component_id;
-		edev->component_id = tmp[0] << 8 | tmp[1];
-		edev->component_revision_id =
-			manufacture_reply->component_revision_id;
+			tmp = (u8 *)&manufacture_reply->component_id;
+			edev->component_id = tmp[0] << 8 | tmp[1];
+			edev->component_revision_id =
+				manufacture_reply->component_revision_id;
 		}
 	} else {
 		printk(MYIOC_s_ERR_FMT
diff --git a/drivers/message/fusion/mptscsih.c b/drivers/message/fusion/mptscsih.c
index 6ba07c7feb92..f0737c57ed5f 100644
--- a/drivers/message/fusion/mptscsih.c
+++ b/drivers/message/fusion/mptscsih.c
@@ -786,6 +786,7 @@ mptscsih_io_done(MPT_ADAPTER *ioc, MPT_FRAME_HDR *mf, MPT_FRAME_HDR *mr)
 			/*
 			 * Allow non-SAS & non-NEXUS_LOSS to drop into below code
 			 */
+			/* Fall through */
 
 		case MPI_IOCSTATUS_SCSI_TASK_TERMINATED:	/* 0x0048 */
 			/* Linux handles an unsolicited DID_RESET better
@@ -882,6 +883,7 @@ mptscsih_io_done(MPT_ADAPTER *ioc, MPT_FRAME_HDR *mf, MPT_FRAME_HDR *mr)
 
 		case MPI_IOCSTATUS_SCSI_DATA_OVERRUN:		/* 0x0044 */
 			scsi_set_resid(sc, 0);
+			/* Fall through */
 		case MPI_IOCSTATUS_SCSI_RECOVERED_ERROR:	/* 0x0040 */
 		case MPI_IOCSTATUS_SUCCESS:			/* 0x0000 */
 			sc->result = (DID_OK << 16) | scsi_status;
@@ -1934,7 +1936,7 @@ mptscsih_host_reset(struct scsi_cmnd *SCpnt)
 	/*  If our attempts to reset the host failed, then return a failed
 	 *  status.  The host will be taken off line by the SCSI mid-layer.
 	 */
-    retval = mpt_Soft_Hard_ResetHandler(ioc, CAN_SLEEP);
+	retval = mpt_Soft_Hard_ResetHandler(ioc, CAN_SLEEP);
 	if (retval < 0)
 		status = FAILED;
 	else
diff --git a/drivers/message/fusion/mptspi.c b/drivers/message/fusion/mptspi.c
index 7172b0b16bdd..eabc4de5816c 100644
--- a/drivers/message/fusion/mptspi.c
+++ b/drivers/message/fusion/mptspi.c
@@ -258,8 +258,6 @@ mptspi_writeIOCPage4(MPT_SCSI_HOST *hd, u8 channel , u8 id)
 	IOCPage4_t		*IOCPage4Ptr;
 	MPT_FRAME_HDR		*mf;
 	dma_addr_t		 dataDma;
-	u16			 req_idx;
-	u32			 frameOffset;
 	u32			 flagsLength;
 	int			 ii;
 
@@ -276,9 +274,6 @@ mptspi_writeIOCPage4(MPT_SCSI_HOST *hd, u8 channel , u8 id)
 	 */
 	pReq = (Config_t *)mf;
 
-	req_idx = le16_to_cpu(mf->u.frame.hwhdr.msgctxu.fld.req_idx);
-	frameOffset = ioc->req_sz - sizeof(IOCPage4_t);
-
 	/* Complete the request frame (same for all requests).
 	 */
 	pReq->Action = MPI_CONFIG_ACTION_PAGE_WRITE_CURRENT;
diff --git a/drivers/mfd/intel-lpss.c b/drivers/mfd/intel-lpss.c
index 50bffc3382d7..45221e092ecf 100644
--- a/drivers/mfd/intel-lpss.c
+++ b/drivers/mfd/intel-lpss.c
@@ -28,6 +28,8 @@
 #include <linux/seq_file.h>
 #include <linux/io-64-nonatomic-lo-hi.h>
 
+#include <linux/dma/idma64.h>
+
 #include "intel-lpss.h"
 
 #define LPSS_DEV_OFFSET		0x000
@@ -96,8 +98,6 @@ static const struct resource intel_lpss_idma64_resources[] = {
 	DEFINE_RES_IRQ(0),
 };
 
-#define LPSS_IDMA64_DRIVER_NAME		"idma64"
-
 /*
  * Cells needs to be ordered so that the iDMA is created first. This is
  * because we need to be sure the DMA is available when the host controller
diff --git a/drivers/mfd/omap-usb-tll.c b/drivers/mfd/omap-usb-tll.c
index 446713dbee27..93177d85e7a9 100644
--- a/drivers/mfd/omap-usb-tll.c
+++ b/drivers/mfd/omap-usb-tll.c
@@ -459,7 +459,6 @@ EXPORT_SYMBOL_GPL(omap_tll_disable);
 
 MODULE_AUTHOR("Keshava Munegowda <keshava_mgowda@ti.com>");
 MODULE_AUTHOR("Roger Quadros <rogerq@ti.com>");
-MODULE_ALIAS("platform:" USBHS_DRIVER_NAME);
 MODULE_LICENSE("GPL v2");
 MODULE_DESCRIPTION("usb tll driver for TI OMAP EHCI and OHCI controllers");
 
diff --git a/drivers/net/dsa/lantiq_gswip.c b/drivers/net/dsa/lantiq_gswip.c
index 553831df58fe..4e64835deac2 100644
--- a/drivers/net/dsa/lantiq_gswip.c
+++ b/drivers/net/dsa/lantiq_gswip.c
@@ -1235,7 +1235,7 @@ static void gswip_port_fast_age(struct dsa_switch *ds, int port)
 
 		err = gswip_pce_table_entry_read(priv, &mac_bridge);
 		if (err) {
-			dev_err(priv->dev, "failed to read mac brigde: %d\n",
+			dev_err(priv->dev, "failed to read mac bridge: %d\n",
 				err);
 			return;
 		}
@@ -1252,7 +1252,7 @@ static void gswip_port_fast_age(struct dsa_switch *ds, int port)
 		mac_bridge.valid = false;
 		err = gswip_pce_table_entry_write(priv, &mac_bridge);
 		if (err) {
-			dev_err(priv->dev, "failed to write mac brigde: %d\n",
+			dev_err(priv->dev, "failed to write mac bridge: %d\n",
 				err);
 			return;
 		}
@@ -1328,7 +1328,7 @@ static int gswip_port_fdb(struct dsa_switch *ds, int port,
 
 	err = gswip_pce_table_entry_write(priv, &mac_bridge);
 	if (err)
-		dev_err(priv->dev, "failed to write mac brigde: %d\n", err);
+		dev_err(priv->dev, "failed to write mac bridge: %d\n", err);
 
 	return err;
 }
@@ -1360,7 +1360,7 @@ static int gswip_port_fdb_dump(struct dsa_switch *ds, int port,
 
 		err = gswip_pce_table_entry_read(priv, &mac_bridge);
 		if (err) {
-			dev_err(priv->dev, "failed to write mac brigde: %d\n",
+			dev_err(priv->dev, "failed to write mac bridge: %d\n",
 				err);
 			return err;
 		}
diff --git a/drivers/net/dsa/sja1105/sja1105_main.c b/drivers/net/dsa/sja1105/sja1105_main.c
index 50ff625c85d6..0663b78a2f6c 100644
--- a/drivers/net/dsa/sja1105/sja1105_main.c
+++ b/drivers/net/dsa/sja1105/sja1105_main.c
@@ -1070,7 +1070,11 @@ static u8 sja1105_stp_state_get(struct sja1105_private *priv, int port)
 		return BR_STATE_LEARNING;
 	if (mac[port].ingress && mac[port].egress && mac[port].dyn_learn)
 		return BR_STATE_FORWARDING;
-	return -EINVAL;
+	/* This is really an error condition if the MAC was in none of the STP
+	 * states above. But treating the port as disabled does nothing, which
+	 * is adequate, and it also resets the MAC to a known state later on.
+	 */
+	return BR_STATE_DISABLED;
 }
 
 /* For situations where we need to change a setting at runtime that is only
diff --git a/drivers/net/dsa/sja1105/sja1105_spi.c b/drivers/net/dsa/sja1105/sja1105_spi.c
index 244a94ccfc18..2eb70b8acfc3 100644
--- a/drivers/net/dsa/sja1105/sja1105_spi.c
+++ b/drivers/net/dsa/sja1105/sja1105_spi.c
@@ -466,14 +466,15 @@ int sja1105_static_config_upload(struct sja1105_private *priv)
 				"invalid, retrying...\n");
 			continue;
 		}
-	} while (--retries && (status.crcchkl == 1 || status.crcchkg == 1 ||
-		 status.configs == 0 || status.ids == 1));
+		/* Success! */
+		break;
+	} while (--retries);
 
 	if (!retries) {
 		rc = -EIO;
 		dev_err(dev, "Failed to upload config to device, giving up\n");
 		goto out;
-	} else if (retries != RETRIES - 1) {
+	} else if (retries != RETRIES) {
 		dev_info(dev, "Succeeded after %d tried\n", RETRIES - retries);
 	}
 
@@ -483,7 +484,7 @@ out:
 	return rc;
 }
 
-struct sja1105_regs sja1105et_regs = {
+static struct sja1105_regs sja1105et_regs = {
 	.device_id = 0x0,
 	.prod_id = 0x100BC3,
 	.status = 0x1,
@@ -508,7 +509,7 @@ struct sja1105_regs sja1105et_regs = {
 	.rmii_ext_tx_clk = {0x100018, 0x10001F, 0x100026, 0x10002D, 0x100034},
 };
 
-struct sja1105_regs sja1105pqrs_regs = {
+static struct sja1105_regs sja1105pqrs_regs = {
 	.device_id = 0x0,
 	.prod_id = 0x100BC3,
 	.status = 0x1,
diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_drvinfo.c b/drivers/net/ethernet/aquantia/atlantic/aq_drvinfo.c
index f5a92b2a5cd6..adad6a7acabe 100644
--- a/drivers/net/ethernet/aquantia/atlantic/aq_drvinfo.c
+++ b/drivers/net/ethernet/aquantia/atlantic/aq_drvinfo.c
@@ -13,6 +13,7 @@
 
 #include "aq_drvinfo.h"
 
+#if IS_REACHABLE(CONFIG_HWMON)
 static int aq_hwmon_read(struct device *dev, enum hwmon_sensor_types type,
 			 u32 attr, int channel, long *value)
 {
@@ -123,3 +124,7 @@ int aq_drvinfo_init(struct net_device *ndev)
 
 	return err;
 }
+
+#else
+int aq_drvinfo_init(struct net_device *ndev) { return 0; }
+#endif
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c
index 18711e0f9bdf..196a3d780dcf 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c
@@ -810,7 +810,7 @@ static int hns3_set_l2l3l4(struct sk_buff *skb, u8 ol4_proto,
 			   u8 il4_proto, u32 *type_cs_vlan_tso,
 			   u32 *ol_type_vlan_len_msec)
 {
-	unsigned char *l2_hdr = l2_hdr = skb->data;
+	unsigned char *l2_hdr = skb->data;
 	u32 l4_proto = ol4_proto;
 	union l4_hdr_info l4;
 	union l3_hdr_info l3;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c
index ca0ee9916e9e..0059b290e095 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c
@@ -535,23 +535,16 @@ void mlx5_init_clock(struct mlx5_core_dev *mdev)
 	do_div(ns, NSEC_PER_SEC / HZ);
 	clock->overflow_period = ns;
 
-	mdev->clock_info_page = alloc_page(GFP_KERNEL);
-	if (mdev->clock_info_page) {
-		mdev->clock_info = kmap(mdev->clock_info_page);
-		if (!mdev->clock_info) {
-			__free_page(mdev->clock_info_page);
-			mlx5_core_warn(mdev, "failed to map clock page\n");
-		} else {
-			mdev->clock_info->sign   = 0;
-			mdev->clock_info->nsec   = clock->tc.nsec;
-			mdev->clock_info->cycles = clock->tc.cycle_last;
-			mdev->clock_info->mask   = clock->cycles.mask;
-			mdev->clock_info->mult   = clock->nominal_c_mult;
-			mdev->clock_info->shift  = clock->cycles.shift;
-			mdev->clock_info->frac   = clock->tc.frac;
-			mdev->clock_info->overflow_period =
-						clock->overflow_period;
-		}
+	mdev->clock_info =
+		(struct mlx5_ib_clock_info *)get_zeroed_page(GFP_KERNEL);
+	if (mdev->clock_info) {
+		mdev->clock_info->nsec = clock->tc.nsec;
+		mdev->clock_info->cycles = clock->tc.cycle_last;
+		mdev->clock_info->mask = clock->cycles.mask;
+		mdev->clock_info->mult = clock->nominal_c_mult;
+		mdev->clock_info->shift = clock->cycles.shift;
+		mdev->clock_info->frac = clock->tc.frac;
+		mdev->clock_info->overflow_period = clock->overflow_period;
 	}
 
 	INIT_WORK(&clock->pps_info.out_work, mlx5_pps_out);
@@ -599,8 +592,7 @@ void mlx5_cleanup_clock(struct mlx5_core_dev *mdev)
 	cancel_delayed_work_sync(&clock->overflow_work);
 
 	if (mdev->clock_info) {
-		kunmap(mdev->clock_info_page);
-		__free_page(mdev->clock_info_page);
+		free_page((unsigned long)mdev->clock_info);
 		mdev->clock_info = NULL;
 	}
 
diff --git a/drivers/net/ethernet/netronome/nfp/bpf/jit.c b/drivers/net/ethernet/netronome/nfp/bpf/jit.c
index f272247d1708..d4bf0e694541 100644
--- a/drivers/net/ethernet/netronome/nfp/bpf/jit.c
+++ b/drivers/net/ethernet/netronome/nfp/bpf/jit.c
@@ -328,7 +328,18 @@ __emit_shf(struct nfp_prog *nfp_prog, u16 dst, enum alu_dst_ab dst_ab,
 		return;
 	}
 
-	if (sc == SHF_SC_L_SHF)
+	/* NFP shift instruction has something special. If shift direction is
+	 * left then shift amount of 1 to 31 is specified as 32 minus the amount
+	 * to shift.
+	 *
+	 * But no need to do this for indirect shift which has shift amount be
+	 * 0. Even after we do this subtraction, shift amount 0 will be turned
+	 * into 32 which will eventually be encoded the same as 0 because only
+	 * low 5 bits are encoded, but shift amount be 32 will fail the
+	 * FIELD_PREP check done later on shift mask (0x1f), due to 32 is out of
+	 * mask range.
+	 */
+	if (sc == SHF_SC_L_SHF && shift)
 		shift = 32 - shift;
 
 	insn = OP_SHF_BASE |
diff --git a/drivers/net/ethernet/netronome/nfp/ccm.h b/drivers/net/ethernet/netronome/nfp/ccm.h
index e2fe4b867958..ac963b128203 100644
--- a/drivers/net/ethernet/netronome/nfp/ccm.h
+++ b/drivers/net/ethernet/netronome/nfp/ccm.h
@@ -54,6 +54,8 @@ static inline unsigned int nfp_ccm_get_tag(struct sk_buff *skb)
 
 /**
  * struct nfp_ccm - common control message handling
+ * @app:		APP handle
+ *
  * @tag_allocator:	bitmap of control message tags in use
  * @tag_alloc_next:	next tag bit to allocate
  * @tag_alloc_last:	next tag bit to be freed
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c b/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c
index 036edcc1fa18..1eef446036d6 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c
@@ -273,6 +273,7 @@ const struct net_device_ops nfp_repr_netdev_ops = {
 	.ndo_fix_features	= nfp_repr_fix_features,
 	.ndo_set_features	= nfp_port_set_features,
 	.ndo_set_mac_address    = eth_mac_addr,
+	.ndo_get_port_parent_id	= nfp_port_get_port_parent_id,
 	.ndo_get_devlink_port	= nfp_devlink_get_devlink_port,
 };
 
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_port.c b/drivers/net/ethernet/netronome/nfp/nfp_port.c
index fcd16877e6e0..93c5bfc0510b 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_port.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_port.c
@@ -30,6 +30,22 @@ struct nfp_port *nfp_port_from_netdev(struct net_device *netdev)
 	return NULL;
 }
 
+int nfp_port_get_port_parent_id(struct net_device *netdev,
+				struct netdev_phys_item_id *ppid)
+{
+	struct nfp_port *port;
+	const u8 *serial;
+
+	port = nfp_port_from_netdev(netdev);
+	if (!port)
+		return -EOPNOTSUPP;
+
+	ppid->id_len = nfp_cpp_serial(port->app->cpp, &serial);
+	memcpy(&ppid->id, serial, ppid->id_len);
+
+	return 0;
+}
+
 int nfp_port_setup_tc(struct net_device *netdev, enum tc_setup_type type,
 		      void *type_data)
 {
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c
index 7e5d5db0d516..b4bb5629de38 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c
@@ -192,6 +192,8 @@ static void dwmac4_prog_mtl_tx_algorithms(struct mac_device_info *hw,
 	default:
 		break;
 	}
+
+	writel(value, ioaddr + MTL_OPERATION_MODE);
 }
 
 static void dwmac4_set_mtl_tx_queue_weight(struct mac_device_info *hw,
diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c
index b395423b19bc..92efa93649f0 100644
--- a/drivers/net/macvlan.c
+++ b/drivers/net/macvlan.c
@@ -836,6 +836,8 @@ static int macvlan_do_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
 
 	switch (cmd) {
 	case SIOCSHWTSTAMP:
+		if (!net_eq(dev_net(dev), &init_net))
+			break;
 	case SIOCGHWTSTAMP:
 		if (netif_device_present(real_dev) && ops->ndo_do_ioctl)
 			err = ops->ndo_do_ioctl(real_dev, &ifrr, cmd);
diff --git a/drivers/net/phy/realtek.c b/drivers/net/phy/realtek.c
index d6a10f323117..761ce3b1e7bd 100644
--- a/drivers/net/phy/realtek.c
+++ b/drivers/net/phy/realtek.c
@@ -23,11 +23,15 @@
 
 #define RTL821x_INSR				0x13
 
+#define RTL821x_EXT_PAGE_SELECT			0x1e
 #define RTL821x_PAGE_SELECT			0x1f
 
 #define RTL8211F_INSR				0x1d
 
 #define RTL8211F_TX_DELAY			BIT(8)
+#define RTL8211E_TX_DELAY			BIT(1)
+#define RTL8211E_RX_DELAY			BIT(2)
+#define RTL8211E_MODE_MII_GMII			BIT(3)
 
 #define RTL8201F_ISR				0x1e
 #define RTL8201F_IER				0x13
@@ -157,16 +161,73 @@ static int rtl8211c_config_init(struct phy_device *phydev)
 
 static int rtl8211f_config_init(struct phy_device *phydev)
 {
-	u16 val = 0;
+	u16 val;
 
-	/* enable TX-delay for rgmii-id and rgmii-txid, otherwise disable it */
-	if (phydev->interface == PHY_INTERFACE_MODE_RGMII_ID ||
-	    phydev->interface == PHY_INTERFACE_MODE_RGMII_TXID)
+	/* enable TX-delay for rgmii-{id,txid}, and disable it for rgmii and
+	 * rgmii-rxid. The RX-delay can be enabled by the external RXDLY pin.
+	 */
+	switch (phydev->interface) {
+	case PHY_INTERFACE_MODE_RGMII:
+	case PHY_INTERFACE_MODE_RGMII_RXID:
+		val = 0;
+		break;
+	case PHY_INTERFACE_MODE_RGMII_ID:
+	case PHY_INTERFACE_MODE_RGMII_TXID:
 		val = RTL8211F_TX_DELAY;
+		break;
+	default: /* the rest of the modes imply leaving delay as is. */
+		return 0;
+	}
 
 	return phy_modify_paged(phydev, 0xd08, 0x11, RTL8211F_TX_DELAY, val);
 }
 
+static int rtl8211e_config_init(struct phy_device *phydev)
+{
+	int ret = 0, oldpage;
+	u16 val;
+
+	/* enable TX/RX delay for rgmii-* modes, and disable them for rgmii. */
+	switch (phydev->interface) {
+	case PHY_INTERFACE_MODE_RGMII:
+		val = 0;
+		break;
+	case PHY_INTERFACE_MODE_RGMII_ID:
+		val = RTL8211E_TX_DELAY | RTL8211E_RX_DELAY;
+		break;
+	case PHY_INTERFACE_MODE_RGMII_RXID:
+		val = RTL8211E_RX_DELAY;
+		break;
+	case PHY_INTERFACE_MODE_RGMII_TXID:
+		val = RTL8211E_TX_DELAY;
+		break;
+	default: /* the rest of the modes imply leaving delays as is. */
+		return 0;
+	}
+
+	/* According to a sample driver there is a 0x1c config register on the
+	 * 0xa4 extension page (0x7) layout. It can be used to disable/enable
+	 * the RX/TX delays otherwise controlled by RXDLY/TXDLY pins. It can
+	 * also be used to customize the whole configuration register:
+	 * 8:6 = PHY Address, 5:4 = Auto-Negotiation, 3 = Interface Mode Select,
+	 * 2 = RX Delay, 1 = TX Delay, 0 = SELRGV (see original PHY datasheet
+	 * for details).
+	 */
+	oldpage = phy_select_page(phydev, 0x7);
+	if (oldpage < 0)
+		goto err_restore_page;
+
+	ret = phy_write(phydev, RTL821x_EXT_PAGE_SELECT, 0xa4);
+	if (ret)
+		goto err_restore_page;
+
+	ret = phy_modify(phydev, 0x1c, RTL8211E_TX_DELAY | RTL8211E_RX_DELAY,
+			 val);
+
+err_restore_page:
+	return phy_restore_page(phydev, oldpage, ret);
+}
+
 static int rtl8211b_suspend(struct phy_device *phydev)
 {
 	phy_write(phydev, MII_MMD_DATA, BIT(9));
@@ -239,6 +300,7 @@ static struct phy_driver realtek_drvs[] = {
 	}, {
 		PHY_ID_MATCH_EXACT(0x001cc915),
 		.name		= "RTL8211E Gigabit Ethernet",
+		.config_init	= &rtl8211e_config_init,
 		.ack_interrupt	= &rtl821x_ack_interrupt,
 		.config_intr	= &rtl8211e_config_intr,
 		.suspend	= genphy_suspend,
diff --git a/drivers/net/tun.c b/drivers/net/tun.c
index 9d72f8c76c15..abae165dcca5 100644
--- a/drivers/net/tun.c
+++ b/drivers/net/tun.c
@@ -596,13 +596,18 @@ static u16 tun_automq_select_queue(struct tun_struct *tun, struct sk_buff *skb)
 static u16 tun_ebpf_select_queue(struct tun_struct *tun, struct sk_buff *skb)
 {
 	struct tun_prog *prog;
+	u32 numqueues;
 	u16 ret = 0;
 
+	numqueues = READ_ONCE(tun->numqueues);
+	if (!numqueues)
+		return 0;
+
 	prog = rcu_dereference(tun->steering_prog);
 	if (prog)
 		ret = bpf_prog_run_clear_cb(prog->prog, skb);
 
-	return ret % tun->numqueues;
+	return ret % numqueues;
 }
 
 static u16 tun_select_queue(struct net_device *dev, struct sk_buff *skb,
@@ -699,6 +704,8 @@ static void __tun_detach(struct tun_file *tfile, bool clean)
 				   tun->tfiles[tun->numqueues - 1]);
 		ntfile = rtnl_dereference(tun->tfiles[index]);
 		ntfile->queue_index = index;
+		rcu_assign_pointer(tun->tfiles[tun->numqueues - 1],
+				   NULL);
 
 		--tun->numqueues;
 		if (clean) {
@@ -1081,7 +1088,7 @@ static netdev_tx_t tun_net_xmit(struct sk_buff *skb, struct net_device *dev)
 	tfile = rcu_dereference(tun->tfiles[txq]);
 
 	/* Drop packet if interface is not attached */
-	if (txq >= tun->numqueues)
+	if (!tfile)
 		goto drop;
 
 	if (!rcu_dereference(tun->steering_prog))
@@ -1304,6 +1311,7 @@ static int tun_xdp_xmit(struct net_device *dev, int n,
 
 	rcu_read_lock();
 
+resample:
 	numqueues = READ_ONCE(tun->numqueues);
 	if (!numqueues) {
 		rcu_read_unlock();
@@ -1312,6 +1320,8 @@ static int tun_xdp_xmit(struct net_device *dev, int n,
 
 	tfile = rcu_dereference(tun->tfiles[smp_processor_id() %
 					    numqueues]);
+	if (unlikely(!tfile))
+		goto resample;
 
 	spin_lock(&tfile->tx_ring.producer_lock);
 	for (i = 0; i < n; i++) {
diff --git a/drivers/net/usb/aqc111.c b/drivers/net/usb/aqc111.c
index aff995be2a31..b86c5ce9a92a 100644
--- a/drivers/net/usb/aqc111.c
+++ b/drivers/net/usb/aqc111.c
@@ -320,6 +320,7 @@ static int aqc111_get_link_ksettings(struct net_device *net,
 static void aqc111_set_phy_speed(struct usbnet *dev, u8 autoneg, u16 speed)
 {
 	struct aqc111_data *aqc111_data = dev->driver_priv;
+	u32 phy_on_the_wire;
 
 	aqc111_data->phy_cfg &= ~AQ_ADV_MASK;
 	aqc111_data->phy_cfg |= AQ_PAUSE;
@@ -361,7 +362,8 @@ static void aqc111_set_phy_speed(struct usbnet *dev, u8 autoneg, u16 speed)
 		}
 	}
 
-	aqc111_write32_cmd(dev, AQ_PHY_OPS, 0, 0, &aqc111_data->phy_cfg);
+	phy_on_the_wire = aqc111_data->phy_cfg;
+	aqc111_write32_cmd(dev, AQ_PHY_OPS, 0, 0, &phy_on_the_wire);
 }
 
 static int aqc111_set_link_ksettings(struct net_device *net,
@@ -453,6 +455,8 @@ static int aqc111_change_mtu(struct net_device *net, int new_mtu)
 		reg16 = 0x1420;
 	else if (dev->net->mtu <= 16334)
 		reg16 = 0x1A20;
+	else
+		return 0;
 
 	aqc111_write16_cmd(dev, AQ_ACCESS_MAC, SFR_PAUSE_WATERLVL_LOW,
 			   2, &reg16);
@@ -753,6 +757,7 @@ static void aqc111_unbind(struct usbnet *dev, struct usb_interface *intf)
 {
 	struct aqc111_data *aqc111_data = dev->driver_priv;
 	u16 reg16;
+	u32 phy_on_the_wire;
 
 	/* Force bz */
 	reg16 = SFR_PHYPWR_RSTCTL_BZ;
@@ -766,8 +771,9 @@ static void aqc111_unbind(struct usbnet *dev, struct usb_interface *intf)
 	aqc111_data->phy_cfg &= ~AQ_ADV_MASK;
 	aqc111_data->phy_cfg |= AQ_LOW_POWER;
 	aqc111_data->phy_cfg &= ~AQ_PHY_POWER_EN;
+	phy_on_the_wire = aqc111_data->phy_cfg;
 	aqc111_write32_cmd_nopm(dev, AQ_PHY_OPS, 0, 0,
-				&aqc111_data->phy_cfg);
+				&phy_on_the_wire);
 
 	kfree(aqc111_data);
 }
@@ -990,6 +996,7 @@ static int aqc111_reset(struct usbnet *dev)
 {
 	struct aqc111_data *aqc111_data = dev->driver_priv;
 	u8 reg8 = 0;
+	u32 phy_on_the_wire;
 
 	dev->rx_urb_size = URB_SIZE;
 
@@ -1002,8 +1009,9 @@ static int aqc111_reset(struct usbnet *dev)
 
 	/* Power up ethernet PHY */
 	aqc111_data->phy_cfg = AQ_PHY_POWER_EN;
+	phy_on_the_wire = aqc111_data->phy_cfg;
 	aqc111_write32_cmd(dev, AQ_PHY_OPS, 0, 0,
-			   &aqc111_data->phy_cfg);
+			   &phy_on_the_wire);
 
 	/* Set the MAC address */
 	aqc111_write_cmd(dev, AQ_ACCESS_MAC, SFR_NODE_ID, ETH_ALEN,
@@ -1034,6 +1042,7 @@ static int aqc111_stop(struct usbnet *dev)
 {
 	struct aqc111_data *aqc111_data = dev->driver_priv;
 	u16 reg16 = 0;
+	u32 phy_on_the_wire;
 
 	aqc111_read16_cmd(dev, AQ_ACCESS_MAC, SFR_MEDIUM_STATUS_MODE,
 			  2, &reg16);
@@ -1045,8 +1054,9 @@ static int aqc111_stop(struct usbnet *dev)
 
 	/* Put PHY to low power*/
 	aqc111_data->phy_cfg |= AQ_LOW_POWER;
+	phy_on_the_wire = aqc111_data->phy_cfg;
 	aqc111_write32_cmd(dev, AQ_PHY_OPS, 0, 0,
-			   &aqc111_data->phy_cfg);
+			   &phy_on_the_wire);
 
 	netif_carrier_off(dev->net);
 
@@ -1322,6 +1332,7 @@ static int aqc111_suspend(struct usb_interface *intf, pm_message_t message)
 	u16 temp_rx_ctrl = 0x00;
 	u16 reg16;
 	u8 reg8;
+	u32 phy_on_the_wire;
 
 	usbnet_suspend(intf, message);
 
@@ -1393,12 +1404,14 @@ static int aqc111_suspend(struct usb_interface *intf, pm_message_t message)
 
 		aqc111_write_cmd(dev, AQ_WOL_CFG, 0, 0,
 				 WOL_CFG_SIZE, &wol_cfg);
+		phy_on_the_wire = aqc111_data->phy_cfg;
 		aqc111_write32_cmd(dev, AQ_PHY_OPS, 0, 0,
-				   &aqc111_data->phy_cfg);
+				   &phy_on_the_wire);
 	} else {
 		aqc111_data->phy_cfg |= AQ_LOW_POWER;
+		phy_on_the_wire = aqc111_data->phy_cfg;
 		aqc111_write32_cmd(dev, AQ_PHY_OPS, 0, 0,
-				   &aqc111_data->phy_cfg);
+				   &phy_on_the_wire);
 
 		/* Disable RX path */
 		aqc111_read16_cmd_nopm(dev, AQ_ACCESS_MAC,
@@ -1415,7 +1428,7 @@ static int aqc111_resume(struct usb_interface *intf)
 {
 	struct usbnet *dev = usb_get_intfdata(intf);
 	struct aqc111_data *aqc111_data = dev->driver_priv;
-	u16 reg16;
+	u16 reg16, oldreg16;
 	u8 reg8;
 
 	netif_carrier_off(dev->net);
@@ -1431,9 +1444,11 @@ static int aqc111_resume(struct usb_interface *intf)
 	/* Configure RX control register => start operation */
 	reg16 = aqc111_data->rxctl;
 	reg16 &= ~SFR_RX_CTL_START;
+	/* needs to be saved in case endianness is swapped */
+	oldreg16 = reg16;
 	aqc111_write16_cmd_nopm(dev, AQ_ACCESS_MAC, SFR_RX_CTL, 2, &reg16);
 
-	reg16 |= SFR_RX_CTL_START;
+	reg16 = oldreg16 | SFR_RX_CTL_START;
 	aqc111_write16_cmd_nopm(dev, AQ_ACCESS_MAC, SFR_RX_CTL, 2, &reg16);
 
 	aqc111_set_phy_speed(dev, aqc111_data->autoneg,
diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c
index 6d8451356eac..9544eb60f725 100644
--- a/drivers/nvme/host/fc.c
+++ b/drivers/nvme/host/fc.c
@@ -14,7 +14,7 @@
 #include "fabrics.h"
 #include <linux/nvme-fc-driver.h>
 #include <linux/nvme-fc.h>
-
+#include <scsi/scsi_transport_fc.h>
 
 /* *************************** Data Structures/Defines ****************** */
 
diff --git a/drivers/parport/parport_ip32.c b/drivers/parport/parport_ip32.c
index 62873070f988..b7a892791c3e 100644
--- a/drivers/parport/parport_ip32.c
+++ b/drivers/parport/parport_ip32.c
@@ -568,6 +568,7 @@ static irqreturn_t parport_ip32_merr_interrupt(int irq, void *dev_id)
 
 /**
  * parport_ip32_dma_start - begins a DMA transfer
+ * @p:		partport to work on
  * @dir:	DMA direction: DMA_TO_DEVICE or DMA_FROM_DEVICE
  * @addr:	pointer to data buffer
  * @count:	buffer size
@@ -575,8 +576,8 @@ static irqreturn_t parport_ip32_merr_interrupt(int irq, void *dev_id)
  * Calls to parport_ip32_dma_start() and parport_ip32_dma_stop() must be
  * correctly balanced.
  */
-static int parport_ip32_dma_start(enum dma_data_direction dir,
-				  void *addr, size_t count)
+static int parport_ip32_dma_start(struct parport *p,
+		enum dma_data_direction dir, void *addr, size_t count)
 {
 	unsigned int limit;
 	u64 ctrl;
@@ -601,7 +602,7 @@ static int parport_ip32_dma_start(enum dma_data_direction dir,
 
 	/* Prepare DMA pointers */
 	parport_ip32_dma.dir = dir;
-	parport_ip32_dma.buf = dma_map_single(NULL, addr, count, dir);
+	parport_ip32_dma.buf = dma_map_single(&p->bus_dev, addr, count, dir);
 	parport_ip32_dma.len = count;
 	parport_ip32_dma.next = parport_ip32_dma.buf;
 	parport_ip32_dma.left = parport_ip32_dma.len;
@@ -625,11 +626,12 @@ static int parport_ip32_dma_start(enum dma_data_direction dir,
 
 /**
  * parport_ip32_dma_stop - ends a running DMA transfer
+ * @p:		partport to work on
  *
  * Calls to parport_ip32_dma_start() and parport_ip32_dma_stop() must be
  * correctly balanced.
  */
-static void parport_ip32_dma_stop(void)
+static void parport_ip32_dma_stop(struct parport *p)
 {
 	u64 ctx_a;
 	u64 ctx_b;
@@ -685,8 +687,8 @@ static void parport_ip32_dma_stop(void)
 	enable_irq(MACEISA_PAR_CTXB_IRQ);
 	parport_ip32_dma.irq_on = 1;
 
-	dma_unmap_single(NULL, parport_ip32_dma.buf, parport_ip32_dma.len,
-			 parport_ip32_dma.dir);
+	dma_unmap_single(&p->bus_dev, parport_ip32_dma.buf,
+			 parport_ip32_dma.len, parport_ip32_dma.dir);
 }
 
 /**
@@ -1445,7 +1447,7 @@ static size_t parport_ip32_fifo_write_block_dma(struct parport *p,
 
 	priv->irq_mode = PARPORT_IP32_IRQ_HERE;
 
-	parport_ip32_dma_start(DMA_TO_DEVICE, (void *)buf, len);
+	parport_ip32_dma_start(p, DMA_TO_DEVICE, (void *)buf, len);
 	reinit_completion(&priv->irq_complete);
 	parport_ip32_frob_econtrol(p, ECR_DMAEN | ECR_SERVINTR, ECR_DMAEN);
 
@@ -1461,7 +1463,7 @@ static size_t parport_ip32_fifo_write_block_dma(struct parport *p,
 		if (ecr & ECR_SERVINTR)
 			break;	/* DMA transfer just finished */
 	}
-	parport_ip32_dma_stop();
+	parport_ip32_dma_stop(p);
 	written = len - parport_ip32_dma_get_residue();
 
 	priv->irq_mode = PARPORT_IP32_IRQ_FWD;
diff --git a/drivers/phy/amlogic/Kconfig b/drivers/phy/amlogic/Kconfig
index 23fe1cda2f70..4c08c1ccdd04 100644
--- a/drivers/phy/amlogic/Kconfig
+++ b/drivers/phy/amlogic/Kconfig
@@ -36,3 +36,25 @@ config PHY_MESON_GXL_USB3
 	  Enable this to support the Meson USB3 PHY and OTG detection
 	  IP block found in Meson GXL and GXM SoCs.
 	  If unsure, say N.
+
+config PHY_MESON_G12A_USB2
+	tristate "Meson G12A USB2 PHY driver"
+	default ARCH_MESON
+	depends on OF && (ARCH_MESON || COMPILE_TEST)
+	select GENERIC_PHY
+	select REGMAP_MMIO
+	help
+	  Enable this to support the Meson USB2 PHYs found in Meson
+	  G12A SoCs.
+	  If unsure, say N.
+
+config PHY_MESON_G12A_USB3_PCIE
+	tristate "Meson G12A USB3+PCIE Combo PHY driver"
+	default ARCH_MESON
+	depends on OF && (ARCH_MESON || COMPILE_TEST)
+	select GENERIC_PHY
+	select REGMAP_MMIO
+	help
+	  Enable this to support the Meson USB3 + PCIE Combo PHY found
+	  in Meson G12A SoCs.
+	  If unsure, say N.
diff --git a/drivers/phy/amlogic/Makefile b/drivers/phy/amlogic/Makefile
index 4fd8848c194d..fdd008e1b19b 100644
--- a/drivers/phy/amlogic/Makefile
+++ b/drivers/phy/amlogic/Makefile
@@ -1,3 +1,5 @@
 obj-$(CONFIG_PHY_MESON8B_USB2)		+= phy-meson8b-usb2.o
 obj-$(CONFIG_PHY_MESON_GXL_USB2)	+= phy-meson-gxl-usb2.o
+obj-$(CONFIG_PHY_MESON_G12A_USB2)	+= phy-meson-g12a-usb2.o
 obj-$(CONFIG_PHY_MESON_GXL_USB3)	+= phy-meson-gxl-usb3.o
+obj-$(CONFIG_PHY_MESON_G12A_USB3_PCIE)	+= phy-meson-g12a-usb3-pcie.o
diff --git a/drivers/phy/amlogic/phy-meson-g12a-usb2.c b/drivers/phy/amlogic/phy-meson-g12a-usb2.c
new file mode 100644
index 000000000000..9065ffc85eb4
--- /dev/null
+++ b/drivers/phy/amlogic/phy-meson-g12a-usb2.c
@@ -0,0 +1,341 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Meson G12A USB2 PHY driver
+ *
+ * Copyright (C) 2017 Martin Blumenstingl <martin.blumenstingl@googlemail.com>
+ * Copyright (C) 2017 Amlogic, Inc. All rights reserved
+ * Copyright (C) 2019 BayLibre, SAS
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ */
+
+#include <linux/bitfield.h>
+#include <linux/bitops.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/regmap.h>
+#include <linux/reset.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+
+#define PHY_CTRL_R0						0x0
+#define PHY_CTRL_R1						0x4
+#define PHY_CTRL_R2						0x8
+#define PHY_CTRL_R3						0xc
+	#define PHY_CTRL_R3_SQUELCH_REF				GENMASK(1, 0)
+	#define PHY_CTRL_R3_HSDIC_REF				GENMASK(3, 2)
+	#define PHY_CTRL_R3_DISC_THRESH				GENMASK(7, 4)
+
+#define PHY_CTRL_R4						0x10
+	#define PHY_CTRL_R4_CALIB_CODE_7_0			GENMASK(7, 0)
+	#define PHY_CTRL_R4_CALIB_CODE_15_8			GENMASK(15, 8)
+	#define PHY_CTRL_R4_CALIB_CODE_23_16			GENMASK(23, 16)
+	#define PHY_CTRL_R4_I_C2L_CAL_EN			BIT(24)
+	#define PHY_CTRL_R4_I_C2L_CAL_RESET_N			BIT(25)
+	#define PHY_CTRL_R4_I_C2L_CAL_DONE			BIT(26)
+	#define PHY_CTRL_R4_TEST_BYPASS_MODE_EN			BIT(27)
+	#define PHY_CTRL_R4_I_C2L_BIAS_TRIM_1_0			GENMASK(29, 28)
+	#define PHY_CTRL_R4_I_C2L_BIAS_TRIM_3_2			GENMASK(31, 30)
+
+#define PHY_CTRL_R5						0x14
+#define PHY_CTRL_R6						0x18
+#define PHY_CTRL_R7						0x1c
+#define PHY_CTRL_R8						0x20
+#define PHY_CTRL_R9						0x24
+#define PHY_CTRL_R10						0x28
+#define PHY_CTRL_R11						0x2c
+#define PHY_CTRL_R12						0x30
+#define PHY_CTRL_R13						0x34
+	#define PHY_CTRL_R13_CUSTOM_PATTERN_19			GENMASK(7, 0)
+	#define PHY_CTRL_R13_LOAD_STAT				BIT(14)
+	#define PHY_CTRL_R13_UPDATE_PMA_SIGNALS			BIT(15)
+	#define PHY_CTRL_R13_MIN_COUNT_FOR_SYNC_DET		GENMASK(20, 16)
+	#define PHY_CTRL_R13_CLEAR_HOLD_HS_DISCONNECT		BIT(21)
+	#define PHY_CTRL_R13_BYPASS_HOST_DISCONNECT_VAL		BIT(22)
+	#define PHY_CTRL_R13_BYPASS_HOST_DISCONNECT_EN		BIT(23)
+	#define PHY_CTRL_R13_I_C2L_HS_EN			BIT(24)
+	#define PHY_CTRL_R13_I_C2L_FS_EN			BIT(25)
+	#define PHY_CTRL_R13_I_C2L_LS_EN			BIT(26)
+	#define PHY_CTRL_R13_I_C2L_HS_OE			BIT(27)
+	#define PHY_CTRL_R13_I_C2L_FS_OE			BIT(28)
+	#define PHY_CTRL_R13_I_C2L_HS_RX_EN			BIT(29)
+	#define PHY_CTRL_R13_I_C2L_FSLS_RX_EN			BIT(30)
+
+#define PHY_CTRL_R14						0x38
+	#define PHY_CTRL_R14_I_RDP_EN				BIT(0)
+	#define PHY_CTRL_R14_I_RPU_SW1_EN			BIT(1)
+	#define PHY_CTRL_R14_I_RPU_SW2_EN			GENMASK(2, 3)
+	#define PHY_CTRL_R14_PG_RSTN				BIT(4)
+	#define PHY_CTRL_R14_I_C2L_DATA_16_8			BIT(5)
+	#define PHY_CTRL_R14_I_C2L_ASSERT_SINGLE_EN_ZERO	BIT(6)
+	#define PHY_CTRL_R14_BYPASS_CTRL_7_0			GENMASK(15, 8)
+	#define PHY_CTRL_R14_BYPASS_CTRL_15_8			GENMASK(23, 16)
+
+#define PHY_CTRL_R15						0x3c
+#define PHY_CTRL_R16						0x40
+	#define PHY_CTRL_R16_MPLL_M				GENMASK(8, 0)
+	#define PHY_CTRL_R16_MPLL_N				GENMASK(14, 10)
+	#define PHY_CTRL_R16_MPLL_TDC_MODE			BIT(20)
+	#define PHY_CTRL_R16_MPLL_SDM_EN			BIT(21)
+	#define PHY_CTRL_R16_MPLL_LOAD				BIT(22)
+	#define PHY_CTRL_R16_MPLL_DCO_SDM_EN			BIT(23)
+	#define PHY_CTRL_R16_MPLL_LOCK_LONG			GENMASK(25, 24)
+	#define PHY_CTRL_R16_MPLL_LOCK_F			BIT(26)
+	#define PHY_CTRL_R16_MPLL_FAST_LOCK			BIT(27)
+	#define PHY_CTRL_R16_MPLL_EN				BIT(28)
+	#define PHY_CTRL_R16_MPLL_RESET				BIT(29)
+	#define PHY_CTRL_R16_MPLL_LOCK				BIT(30)
+	#define PHY_CTRL_R16_MPLL_LOCK_DIG			BIT(31)
+
+#define PHY_CTRL_R17						0x44
+	#define PHY_CTRL_R17_MPLL_FRAC_IN			GENMASK(13, 0)
+	#define PHY_CTRL_R17_MPLL_FIX_EN			BIT(16)
+	#define PHY_CTRL_R17_MPLL_LAMBDA1			GENMASK(19, 17)
+	#define PHY_CTRL_R17_MPLL_LAMBDA0			GENMASK(22, 20)
+	#define PHY_CTRL_R17_MPLL_FILTER_MODE			BIT(23)
+	#define PHY_CTRL_R17_MPLL_FILTER_PVT2			GENMASK(27, 24)
+	#define PHY_CTRL_R17_MPLL_FILTER_PVT1			GENMASK(31, 28)
+
+#define PHY_CTRL_R18						0x48
+	#define PHY_CTRL_R18_MPLL_LKW_SEL			GENMASK(1, 0)
+	#define PHY_CTRL_R18_MPLL_LK_W				GENMASK(5, 2)
+	#define PHY_CTRL_R18_MPLL_LK_S				GENMASK(11, 6)
+	#define PHY_CTRL_R18_MPLL_DCO_M_EN			BIT(12)
+	#define PHY_CTRL_R18_MPLL_DCO_CLK_SEL			BIT(13)
+	#define PHY_CTRL_R18_MPLL_PFD_GAIN			GENMASK(15, 14)
+	#define PHY_CTRL_R18_MPLL_ROU				GENMASK(18, 16)
+	#define PHY_CTRL_R18_MPLL_DATA_SEL			GENMASK(21, 19)
+	#define PHY_CTRL_R18_MPLL_BIAS_ADJ			GENMASK(23, 22)
+	#define PHY_CTRL_R18_MPLL_BB_MODE			GENMASK(25, 24)
+	#define PHY_CTRL_R18_MPLL_ALPHA				GENMASK(28, 26)
+	#define PHY_CTRL_R18_MPLL_ADJ_LDO			GENMASK(30, 29)
+	#define PHY_CTRL_R18_MPLL_ACG_RANGE			BIT(31)
+
+#define PHY_CTRL_R19						0x4c
+#define PHY_CTRL_R20						0x50
+	#define PHY_CTRL_R20_USB2_IDDET_EN			BIT(0)
+	#define PHY_CTRL_R20_USB2_OTG_VBUS_TRIM_2_0		GENMASK(3, 1)
+	#define PHY_CTRL_R20_USB2_OTG_VBUSDET_EN		BIT(4)
+	#define PHY_CTRL_R20_USB2_AMON_EN			BIT(5)
+	#define PHY_CTRL_R20_USB2_CAL_CODE_R5			BIT(6)
+	#define PHY_CTRL_R20_BYPASS_OTG_DET			BIT(7)
+	#define PHY_CTRL_R20_USB2_DMON_EN			BIT(8)
+	#define PHY_CTRL_R20_USB2_DMON_SEL_3_0			GENMASK(12, 9)
+	#define PHY_CTRL_R20_USB2_EDGE_DRV_EN			BIT(13)
+	#define PHY_CTRL_R20_USB2_EDGE_DRV_TRIM_1_0		GENMASK(15, 14)
+	#define PHY_CTRL_R20_USB2_BGR_ADJ_4_0			GENMASK(20, 16)
+	#define PHY_CTRL_R20_USB2_BGR_START			BIT(21)
+	#define PHY_CTRL_R20_USB2_BGR_VREF_4_0			GENMASK(28, 24)
+	#define PHY_CTRL_R20_USB2_BGR_DBG_1_0			GENMASK(30, 29)
+	#define PHY_CTRL_R20_BYPASS_CAL_DONE_R5			BIT(31)
+
+#define PHY_CTRL_R21						0x54
+	#define PHY_CTRL_R21_USB2_BGR_FORCE			BIT(0)
+	#define PHY_CTRL_R21_USB2_CAL_ACK_EN			BIT(1)
+	#define PHY_CTRL_R21_USB2_OTG_ACA_EN			BIT(2)
+	#define PHY_CTRL_R21_USB2_TX_STRG_PD			BIT(3)
+	#define PHY_CTRL_R21_USB2_OTG_ACA_TRIM_1_0		GENMASK(5, 4)
+	#define PHY_CTRL_R21_BYPASS_UTMI_CNTR			GENMASK(15, 6)
+	#define PHY_CTRL_R21_BYPASS_UTMI_REG			GENMASK(25, 20)
+
+#define PHY_CTRL_R22						0x58
+#define PHY_CTRL_R23						0x5c
+
+#define RESET_COMPLETE_TIME					1000
+#define PLL_RESET_COMPLETE_TIME					100
+
+struct phy_meson_g12a_usb2_priv {
+	struct device		*dev;
+	struct regmap		*regmap;
+	struct clk		*clk;
+	struct reset_control	*reset;
+};
+
+static const struct regmap_config phy_meson_g12a_usb2_regmap_conf = {
+	.reg_bits = 8,
+	.val_bits = 32,
+	.reg_stride = 4,
+	.max_register = PHY_CTRL_R23,
+};
+
+static int phy_meson_g12a_usb2_init(struct phy *phy)
+{
+	struct phy_meson_g12a_usb2_priv *priv = phy_get_drvdata(phy);
+	int ret;
+
+	ret = reset_control_reset(priv->reset);
+	if (ret)
+		return ret;
+
+	udelay(RESET_COMPLETE_TIME);
+
+	/* usb2_otg_aca_en == 0 */
+	regmap_update_bits(priv->regmap, PHY_CTRL_R21,
+			   PHY_CTRL_R21_USB2_OTG_ACA_EN, 0);
+
+	/* PLL Setup : 24MHz * 20 / 1 = 480MHz */
+	regmap_write(priv->regmap, PHY_CTRL_R16,
+		     FIELD_PREP(PHY_CTRL_R16_MPLL_M, 20) |
+		     FIELD_PREP(PHY_CTRL_R16_MPLL_N, 1) |
+		     PHY_CTRL_R16_MPLL_LOAD |
+		     FIELD_PREP(PHY_CTRL_R16_MPLL_LOCK_LONG, 1) |
+		     PHY_CTRL_R16_MPLL_FAST_LOCK |
+		     PHY_CTRL_R16_MPLL_EN |
+		     PHY_CTRL_R16_MPLL_RESET);
+
+	regmap_write(priv->regmap, PHY_CTRL_R17,
+		     FIELD_PREP(PHY_CTRL_R17_MPLL_FRAC_IN, 0) |
+		     FIELD_PREP(PHY_CTRL_R17_MPLL_LAMBDA1, 7) |
+		     FIELD_PREP(PHY_CTRL_R17_MPLL_LAMBDA0, 7) |
+		     FIELD_PREP(PHY_CTRL_R17_MPLL_FILTER_PVT2, 2) |
+		     FIELD_PREP(PHY_CTRL_R17_MPLL_FILTER_PVT1, 9));
+
+	regmap_write(priv->regmap, PHY_CTRL_R18,
+		     FIELD_PREP(PHY_CTRL_R18_MPLL_LKW_SEL, 1) |
+		     FIELD_PREP(PHY_CTRL_R18_MPLL_LK_W, 9) |
+		     FIELD_PREP(PHY_CTRL_R18_MPLL_LK_S, 0x27) |
+		     FIELD_PREP(PHY_CTRL_R18_MPLL_PFD_GAIN, 1) |
+		     FIELD_PREP(PHY_CTRL_R18_MPLL_ROU, 7) |
+		     FIELD_PREP(PHY_CTRL_R18_MPLL_DATA_SEL, 3) |
+		     FIELD_PREP(PHY_CTRL_R18_MPLL_BIAS_ADJ, 1) |
+		     FIELD_PREP(PHY_CTRL_R18_MPLL_BB_MODE, 0) |
+		     FIELD_PREP(PHY_CTRL_R18_MPLL_ALPHA, 3) |
+		     FIELD_PREP(PHY_CTRL_R18_MPLL_ADJ_LDO, 1) |
+		     PHY_CTRL_R18_MPLL_ACG_RANGE);
+
+	udelay(PLL_RESET_COMPLETE_TIME);
+
+	/* UnReset PLL */
+	regmap_write(priv->regmap, PHY_CTRL_R16,
+		     FIELD_PREP(PHY_CTRL_R16_MPLL_M, 20) |
+		     FIELD_PREP(PHY_CTRL_R16_MPLL_N, 1) |
+		     PHY_CTRL_R16_MPLL_LOAD |
+		     FIELD_PREP(PHY_CTRL_R16_MPLL_LOCK_LONG, 1) |
+		     PHY_CTRL_R16_MPLL_FAST_LOCK |
+		     PHY_CTRL_R16_MPLL_EN);
+
+	/* PHY Tuning */
+	regmap_write(priv->regmap, PHY_CTRL_R20,
+		     FIELD_PREP(PHY_CTRL_R20_USB2_OTG_VBUS_TRIM_2_0, 4) |
+		     PHY_CTRL_R20_USB2_OTG_VBUSDET_EN |
+		     FIELD_PREP(PHY_CTRL_R20_USB2_DMON_SEL_3_0, 15) |
+		     PHY_CTRL_R20_USB2_EDGE_DRV_EN |
+		     FIELD_PREP(PHY_CTRL_R20_USB2_EDGE_DRV_TRIM_1_0, 3) |
+		     FIELD_PREP(PHY_CTRL_R20_USB2_BGR_ADJ_4_0, 0) |
+		     FIELD_PREP(PHY_CTRL_R20_USB2_BGR_VREF_4_0, 0) |
+		     FIELD_PREP(PHY_CTRL_R20_USB2_BGR_DBG_1_0, 0));
+
+	regmap_write(priv->regmap, PHY_CTRL_R4,
+		     FIELD_PREP(PHY_CTRL_R4_CALIB_CODE_7_0, 0xf) |
+		     FIELD_PREP(PHY_CTRL_R4_CALIB_CODE_15_8, 0xf) |
+		     FIELD_PREP(PHY_CTRL_R4_CALIB_CODE_23_16, 0xf) |
+		     PHY_CTRL_R4_TEST_BYPASS_MODE_EN |
+		     FIELD_PREP(PHY_CTRL_R4_I_C2L_BIAS_TRIM_1_0, 0) |
+		     FIELD_PREP(PHY_CTRL_R4_I_C2L_BIAS_TRIM_3_2, 0));
+
+	/* Tuning Disconnect Threshold */
+	regmap_write(priv->regmap, PHY_CTRL_R3,
+		     FIELD_PREP(PHY_CTRL_R3_SQUELCH_REF, 0) |
+		     FIELD_PREP(PHY_CTRL_R3_HSDIC_REF, 1) |
+		     FIELD_PREP(PHY_CTRL_R3_DISC_THRESH, 3));
+
+	/* Analog Settings */
+	regmap_write(priv->regmap, PHY_CTRL_R14, 0);
+	regmap_write(priv->regmap, PHY_CTRL_R13,
+		     PHY_CTRL_R13_UPDATE_PMA_SIGNALS |
+		     FIELD_PREP(PHY_CTRL_R13_MIN_COUNT_FOR_SYNC_DET, 7));
+
+	return 0;
+}
+
+static int phy_meson_g12a_usb2_exit(struct phy *phy)
+{
+	struct phy_meson_g12a_usb2_priv *priv = phy_get_drvdata(phy);
+
+	return reset_control_reset(priv->reset);
+}
+
+/* set_mode is not needed, mode setting is handled via the UTMI bus */
+static const struct phy_ops phy_meson_g12a_usb2_ops = {
+	.init		= phy_meson_g12a_usb2_init,
+	.exit		= phy_meson_g12a_usb2_exit,
+	.owner		= THIS_MODULE,
+};
+
+static int phy_meson_g12a_usb2_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct phy_provider *phy_provider;
+	struct resource *res;
+	struct phy_meson_g12a_usb2_priv *priv;
+	struct phy *phy;
+	void __iomem *base;
+	int ret;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->dev = dev;
+	platform_set_drvdata(pdev, priv);
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(base))
+		return PTR_ERR(base);
+
+	priv->regmap = devm_regmap_init_mmio(dev, base,
+					     &phy_meson_g12a_usb2_regmap_conf);
+	if (IS_ERR(priv->regmap))
+		return PTR_ERR(priv->regmap);
+
+	priv->clk = devm_clk_get(dev, "xtal");
+	if (IS_ERR(priv->clk))
+		return PTR_ERR(priv->clk);
+
+	priv->reset = devm_reset_control_get(dev, "phy");
+	if (IS_ERR(priv->reset))
+		return PTR_ERR(priv->reset);
+
+	ret = reset_control_deassert(priv->reset);
+	if (ret)
+		return ret;
+
+	phy = devm_phy_create(dev, NULL, &phy_meson_g12a_usb2_ops);
+	if (IS_ERR(phy)) {
+		ret = PTR_ERR(phy);
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "failed to create PHY\n");
+
+		return ret;
+	}
+
+	phy_set_bus_width(phy, 8);
+	phy_set_drvdata(phy, priv);
+
+	phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate);
+
+	return PTR_ERR_OR_ZERO(phy_provider);
+}
+
+static const struct of_device_id phy_meson_g12a_usb2_of_match[] = {
+	{ .compatible = "amlogic,g12a-usb2-phy", },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, phy_meson_g12a_usb2_of_match);
+
+static struct platform_driver phy_meson_g12a_usb2_driver = {
+	.probe	= phy_meson_g12a_usb2_probe,
+	.driver	= {
+		.name		= "phy-meson-g12a-usb2",
+		.of_match_table	= phy_meson_g12a_usb2_of_match,
+	},
+};
+module_platform_driver(phy_meson_g12a_usb2_driver);
+
+MODULE_AUTHOR("Martin Blumenstingl <martin.blumenstingl@googlemail.com>");
+MODULE_AUTHOR("Neil Armstrong <narmstrong@baylibre.com>");
+MODULE_DESCRIPTION("Meson G12A USB2 PHY driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/phy/amlogic/phy-meson-g12a-usb3-pcie.c b/drivers/phy/amlogic/phy-meson-g12a-usb3-pcie.c
new file mode 100644
index 000000000000..6233a7979a93
--- /dev/null
+++ b/drivers/phy/amlogic/phy-meson-g12a-usb3-pcie.c
@@ -0,0 +1,413 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Amlogic G12A USB3 + PCIE Combo PHY driver
+ *
+ * Copyright (C) 2017 Amlogic, Inc. All rights reserved
+ * Copyright (C) 2019 BayLibre, SAS
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ */
+
+#include <linux/bitfield.h>
+#include <linux/bitops.h>
+#include <linux/clk.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/phy/phy.h>
+#include <linux/regmap.h>
+#include <linux/reset.h>
+#include <linux/platform_device.h>
+#include <dt-bindings/phy/phy.h>
+
+#define PHY_R0							0x00
+	#define PHY_R0_PCIE_POWER_STATE				GENMASK(4, 0)
+	#define PHY_R0_PCIE_USB3_SWITCH				GENMASK(6, 5)
+
+#define PHY_R1							0x04
+	#define PHY_R1_PHY_TX1_TERM_OFFSET			GENMASK(4, 0)
+	#define PHY_R1_PHY_TX0_TERM_OFFSET			GENMASK(9, 5)
+	#define PHY_R1_PHY_RX1_EQ				GENMASK(12, 10)
+	#define PHY_R1_PHY_RX0_EQ				GENMASK(15, 13)
+	#define PHY_R1_PHY_LOS_LEVEL				GENMASK(20, 16)
+	#define PHY_R1_PHY_LOS_BIAS				GENMASK(23, 21)
+	#define PHY_R1_PHY_REF_CLKDIV2				BIT(24)
+	#define PHY_R1_PHY_MPLL_MULTIPLIER			GENMASK(31, 25)
+
+#define PHY_R2							0x08
+	#define PHY_R2_PCS_TX_DEEMPH_GEN2_6DB			GENMASK(5, 0)
+	#define PHY_R2_PCS_TX_DEEMPH_GEN2_3P5DB			GENMASK(11, 6)
+	#define PHY_R2_PCS_TX_DEEMPH_GEN1			GENMASK(17, 12)
+	#define PHY_R2_PHY_TX_VBOOST_LVL			GENMASK(20, 18)
+
+#define PHY_R4							0x10
+	#define PHY_R4_PHY_CR_WRITE				BIT(0)
+	#define PHY_R4_PHY_CR_READ				BIT(1)
+	#define PHY_R4_PHY_CR_DATA_IN				GENMASK(17, 2)
+	#define PHY_R4_PHY_CR_CAP_DATA				BIT(18)
+	#define PHY_R4_PHY_CR_CAP_ADDR				BIT(19)
+
+#define PHY_R5							0x14
+	#define PHY_R5_PHY_CR_DATA_OUT				GENMASK(15, 0)
+	#define PHY_R5_PHY_CR_ACK				BIT(16)
+	#define PHY_R5_PHY_BS_OUT				BIT(17)
+
+struct phy_g12a_usb3_pcie_priv {
+	struct regmap		*regmap;
+	struct regmap		*regmap_cr;
+	struct clk		*clk_ref;
+	struct reset_control	*reset;
+	struct phy		*phy;
+	unsigned int		mode;
+};
+
+static const struct regmap_config phy_g12a_usb3_pcie_regmap_conf = {
+	.reg_bits = 8,
+	.val_bits = 32,
+	.reg_stride = 4,
+	.max_register = PHY_R5,
+};
+
+static int phy_g12a_usb3_pcie_cr_bus_addr(struct phy_g12a_usb3_pcie_priv *priv,
+					  unsigned int addr)
+{
+	unsigned int val, reg;
+	int ret;
+
+	reg = FIELD_PREP(PHY_R4_PHY_CR_DATA_IN, addr);
+
+	regmap_write(priv->regmap, PHY_R4, reg);
+	regmap_write(priv->regmap, PHY_R4, reg);
+
+	regmap_write(priv->regmap, PHY_R4, reg | PHY_R4_PHY_CR_CAP_ADDR);
+
+	ret = regmap_read_poll_timeout(priv->regmap, PHY_R5, val,
+				       (val & PHY_R5_PHY_CR_ACK),
+				       5, 1000);
+	if (ret)
+		return ret;
+
+	regmap_write(priv->regmap, PHY_R4, reg);
+
+	ret = regmap_read_poll_timeout(priv->regmap, PHY_R5, val,
+				       !(val & PHY_R5_PHY_CR_ACK),
+				       5, 1000);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int phy_g12a_usb3_pcie_cr_bus_read(void *context, unsigned int addr,
+					  unsigned int *data)
+{
+	struct phy_g12a_usb3_pcie_priv *priv = context;
+	unsigned int val;
+	int ret;
+
+	ret = phy_g12a_usb3_pcie_cr_bus_addr(priv, addr);
+	if (ret)
+		return ret;
+
+	regmap_write(priv->regmap, PHY_R4, 0);
+	regmap_write(priv->regmap, PHY_R4, PHY_R4_PHY_CR_READ);
+
+	ret = regmap_read_poll_timeout(priv->regmap, PHY_R5, val,
+				       (val & PHY_R5_PHY_CR_ACK),
+				       5, 1000);
+	if (ret)
+		return ret;
+
+	*data = FIELD_GET(PHY_R5_PHY_CR_DATA_OUT, val);
+
+	regmap_write(priv->regmap, PHY_R4, 0);
+
+	ret = regmap_read_poll_timeout(priv->regmap, PHY_R5, val,
+				       !(val & PHY_R5_PHY_CR_ACK),
+				       5, 1000);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int phy_g12a_usb3_pcie_cr_bus_write(void *context, unsigned int addr,
+					   unsigned int data)
+{
+	struct phy_g12a_usb3_pcie_priv *priv = context;
+	unsigned int val, reg;
+	int ret;
+
+	ret = phy_g12a_usb3_pcie_cr_bus_addr(priv, addr);
+	if (ret)
+		return ret;
+
+	reg = FIELD_PREP(PHY_R4_PHY_CR_DATA_IN, data);
+
+	regmap_write(priv->regmap, PHY_R4, reg);
+	regmap_write(priv->regmap, PHY_R4, reg);
+
+	regmap_write(priv->regmap, PHY_R4, reg | PHY_R4_PHY_CR_CAP_DATA);
+
+	ret = regmap_read_poll_timeout(priv->regmap, PHY_R5, val,
+				       (val & PHY_R5_PHY_CR_ACK),
+				       5, 1000);
+	if (ret)
+		return ret;
+
+	regmap_write(priv->regmap, PHY_R4, reg);
+
+	ret = regmap_read_poll_timeout(priv->regmap, PHY_R5, val,
+				       (val & PHY_R5_PHY_CR_ACK) == 0,
+				       5, 1000);
+	if (ret)
+		return ret;
+
+	regmap_write(priv->regmap, PHY_R4, reg);
+
+	regmap_write(priv->regmap, PHY_R4, reg | PHY_R4_PHY_CR_WRITE);
+
+	ret = regmap_read_poll_timeout(priv->regmap, PHY_R5, val,
+				       (val & PHY_R5_PHY_CR_ACK),
+				       5, 1000);
+	if (ret)
+		return ret;
+
+	regmap_write(priv->regmap, PHY_R4, reg);
+
+	ret = regmap_read_poll_timeout(priv->regmap, PHY_R5, val,
+				       (val & PHY_R5_PHY_CR_ACK) == 0,
+				       5, 1000);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static const struct regmap_config phy_g12a_usb3_pcie_cr_regmap_conf = {
+	.reg_bits = 16,
+	.val_bits = 16,
+	.reg_read = phy_g12a_usb3_pcie_cr_bus_read,
+	.reg_write = phy_g12a_usb3_pcie_cr_bus_write,
+	.max_register = 0xffff,
+	.fast_io = true,
+};
+
+static int phy_g12a_usb3_init(struct phy *phy)
+{
+	struct phy_g12a_usb3_pcie_priv *priv = phy_get_drvdata(phy);
+	int data, ret;
+
+	/* Switch PHY to USB3 */
+	/* TODO figure out how to handle when PCIe was set in the bootloader */
+	regmap_update_bits(priv->regmap, PHY_R0,
+			   PHY_R0_PCIE_USB3_SWITCH,
+			   PHY_R0_PCIE_USB3_SWITCH);
+
+	/*
+	 * WORKAROUND: There is SSPHY suspend bug due to
+	 * which USB enumerates
+	 * in HS mode instead of SS mode. Workaround it by asserting
+	 * LANE0.TX_ALT_BLOCK.EN_ALT_BUS to enable TX to use alt bus
+	 * mode
+	 */
+	ret = regmap_update_bits(priv->regmap_cr, 0x102d, BIT(7), BIT(7));
+	if (ret)
+		return ret;
+
+	ret = regmap_update_bits(priv->regmap_cr, 0x1010, 0xff0, 20);
+	if (ret)
+		return ret;
+
+	/*
+	 * Fix RX Equalization setting as follows
+	 * LANE0.RX_OVRD_IN_HI. RX_EQ_EN set to 0
+	 * LANE0.RX_OVRD_IN_HI.RX_EQ_EN_OVRD set to 1
+	 * LANE0.RX_OVRD_IN_HI.RX_EQ set to 3
+	 * LANE0.RX_OVRD_IN_HI.RX_EQ_OVRD set to 1
+	 */
+	ret = regmap_read(priv->regmap_cr, 0x1006, &data);
+	if (ret)
+		return ret;
+
+	data &= ~BIT(6);
+	data |= BIT(7);
+	data &= ~(0x7 << 8);
+	data |= (0x3 << 8);
+	data |= (1 << 11);
+	ret = regmap_write(priv->regmap_cr, 0x1006, data);
+	if (ret)
+		return ret;
+
+	/*
+	 * Set EQ and TX launch amplitudes as follows
+	 * LANE0.TX_OVRD_DRV_LO.PREEMPH set to 22
+	 * LANE0.TX_OVRD_DRV_LO.AMPLITUDE set to 127
+	 * LANE0.TX_OVRD_DRV_LO.EN set to 1.
+	 */
+	ret = regmap_read(priv->regmap_cr, 0x1002, &data);
+	if (ret)
+		return ret;
+
+	data &= ~0x3f80;
+	data |= (0x16 << 7);
+	data &= ~0x7f;
+	data |= (0x7f | BIT(14));
+	ret = regmap_write(priv->regmap_cr, 0x1002, data);
+	if (ret)
+		return ret;
+
+	/* MPLL_LOOP_CTL.PROP_CNTRL = 8 */
+	ret = regmap_update_bits(priv->regmap_cr, 0x30, 0xf << 4, 8 << 4);
+	if (ret)
+		return ret;
+
+	regmap_update_bits(priv->regmap, PHY_R2,
+			PHY_R2_PHY_TX_VBOOST_LVL,
+			FIELD_PREP(PHY_R2_PHY_TX_VBOOST_LVL, 0x4));
+
+	regmap_update_bits(priv->regmap, PHY_R1,
+			PHY_R1_PHY_LOS_BIAS | PHY_R1_PHY_LOS_LEVEL,
+			FIELD_PREP(PHY_R1_PHY_LOS_BIAS, 4) |
+			FIELD_PREP(PHY_R1_PHY_LOS_LEVEL, 9));
+
+	return 0;
+}
+
+static int phy_g12a_usb3_pcie_init(struct phy *phy)
+{
+	struct phy_g12a_usb3_pcie_priv *priv = phy_get_drvdata(phy);
+	int ret;
+
+	ret = reset_control_reset(priv->reset);
+	if (ret)
+		return ret;
+
+	if (priv->mode == PHY_TYPE_USB3)
+		return phy_g12a_usb3_init(phy);
+
+	/* Power UP PCIE */
+	/* TODO figure out when the bootloader has set USB3 mode before */
+	regmap_update_bits(priv->regmap, PHY_R0,
+			   PHY_R0_PCIE_POWER_STATE,
+			   FIELD_PREP(PHY_R0_PCIE_POWER_STATE, 0x1c));
+
+	return 0;
+}
+
+static int phy_g12a_usb3_pcie_exit(struct phy *phy)
+{
+	struct phy_g12a_usb3_pcie_priv *priv = phy_get_drvdata(phy);
+
+	return reset_control_reset(priv->reset);
+}
+
+static struct phy *phy_g12a_usb3_pcie_xlate(struct device *dev,
+					    struct of_phandle_args *args)
+{
+	struct phy_g12a_usb3_pcie_priv *priv = dev_get_drvdata(dev);
+	unsigned int mode;
+
+	if (args->args_count < 1) {
+		dev_err(dev, "invalid number of arguments\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	mode = args->args[0];
+
+	if (mode != PHY_TYPE_USB3 && mode != PHY_TYPE_PCIE) {
+		dev_err(dev, "invalid phy mode select argument\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	priv->mode = mode;
+
+	return priv->phy;
+}
+
+static const struct phy_ops phy_g12a_usb3_pcie_ops = {
+	.init		= phy_g12a_usb3_pcie_init,
+	.exit		= phy_g12a_usb3_pcie_exit,
+	.owner		= THIS_MODULE,
+};
+
+static int phy_g12a_usb3_pcie_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	struct phy_g12a_usb3_pcie_priv *priv;
+	struct resource *res;
+	struct phy_provider *phy_provider;
+	void __iomem *base;
+	int ret;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(base))
+		return PTR_ERR(base);
+
+	priv->regmap = devm_regmap_init_mmio(dev, base,
+					     &phy_g12a_usb3_pcie_regmap_conf);
+	if (IS_ERR(priv->regmap))
+		return PTR_ERR(priv->regmap);
+
+	priv->regmap_cr = devm_regmap_init(dev, NULL, priv,
+					   &phy_g12a_usb3_pcie_cr_regmap_conf);
+	if (IS_ERR(priv->regmap_cr))
+		return PTR_ERR(priv->regmap_cr);
+
+	priv->clk_ref = devm_clk_get(dev, "ref_clk");
+	if (IS_ERR(priv->clk_ref))
+		return PTR_ERR(priv->clk_ref);
+
+	ret = clk_prepare_enable(priv->clk_ref);
+	if (ret)
+		goto err_disable_clk_ref;
+
+	priv->reset = devm_reset_control_array_get(dev, false, false);
+	if (IS_ERR(priv->reset))
+		return PTR_ERR(priv->reset);
+
+	priv->phy = devm_phy_create(dev, np, &phy_g12a_usb3_pcie_ops);
+	if (IS_ERR(priv->phy)) {
+		ret = PTR_ERR(priv->phy);
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "failed to create PHY\n");
+
+		return ret;
+	}
+
+	phy_set_drvdata(priv->phy, priv);
+	dev_set_drvdata(dev, priv);
+
+	phy_provider = devm_of_phy_provider_register(dev,
+						     phy_g12a_usb3_pcie_xlate);
+
+	return PTR_ERR_OR_ZERO(phy_provider);
+
+err_disable_clk_ref:
+	clk_disable_unprepare(priv->clk_ref);
+
+	return ret;
+}
+
+static const struct of_device_id phy_g12a_usb3_pcie_of_match[] = {
+	{ .compatible = "amlogic,g12a-usb3-pcie-phy", },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, phy_g12a_usb3_pcie_of_match);
+
+static struct platform_driver phy_g12a_usb3_pcie_driver = {
+	.probe	= phy_g12a_usb3_pcie_probe,
+	.driver	= {
+		.name		= "phy-g12a-usb3-pcie",
+		.of_match_table	= phy_g12a_usb3_pcie_of_match,
+	},
+};
+module_platform_driver(phy_g12a_usb3_pcie_driver);
+
+MODULE_AUTHOR("Neil Armstrong <narmstrong@baylibre.com>");
+MODULE_DESCRIPTION("Amlogic G12A USB3 + PCIE Combo PHY driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/phy/amlogic/phy-meson-gxl-usb2.c b/drivers/phy/amlogic/phy-meson-gxl-usb2.c
index 148ef0bdb9c1..4cbee412f2b0 100644
--- a/drivers/phy/amlogic/phy-meson-gxl-usb2.c
+++ b/drivers/phy/amlogic/phy-meson-gxl-usb2.c
@@ -261,14 +261,9 @@ static int phy_meson_gxl_usb2_probe(struct platform_device *pdev)
 	if (IS_ERR(priv->regmap))
 		return PTR_ERR(priv->regmap);
 
-	priv->clk = devm_clk_get(dev, "phy");
-	if (IS_ERR(priv->clk)) {
-		ret = PTR_ERR(priv->clk);
-		if (ret == -ENOENT)
-			priv->clk = NULL;
-		else
-			return ret;
-	}
+	priv->clk = devm_clk_get_optional(dev, "phy");
+	if (IS_ERR(priv->clk))
+		return PTR_ERR(priv->clk);
 
 	priv->reset = devm_reset_control_get_optional_shared(dev, "phy");
 	if (IS_ERR(priv->reset))
diff --git a/drivers/phy/broadcom/Kconfig b/drivers/phy/broadcom/Kconfig
index aa917a61071d..f30f4819c3bb 100644
--- a/drivers/phy/broadcom/Kconfig
+++ b/drivers/phy/broadcom/Kconfig
@@ -10,6 +10,17 @@ config PHY_CYGNUS_PCIE
 	  Enable this to support the Broadcom Cygnus PCIe PHY.
 	  If unsure, say N.
 
+config PHY_BCM_SR_USB
+	tristate "Broadcom Stingray USB PHY driver"
+	depends on OF && (ARCH_BCM_IPROC || COMPILE_TEST)
+	select GENERIC_PHY
+	default ARCH_BCM_IPROC
+	help
+	  Enable this to support the Broadcom Stingray USB PHY
+	  driver. It supports all versions of Superspeed and
+	  Highspeed PHYs.
+	  If unsure, say N.
+
 config BCM_KONA_USB2_PHY
 	tristate "Broadcom Kona USB2 PHY Driver"
 	depends on HAS_IOMEM
diff --git a/drivers/phy/broadcom/Makefile b/drivers/phy/broadcom/Makefile
index 0f60184e6662..f453c7d3ffff 100644
--- a/drivers/phy/broadcom/Makefile
+++ b/drivers/phy/broadcom/Makefile
@@ -11,3 +11,4 @@ obj-$(CONFIG_PHY_BRCM_USB)		+= phy-brcm-usb-dvr.o
 phy-brcm-usb-dvr-objs := phy-brcm-usb.o phy-brcm-usb-init.o
 
 obj-$(CONFIG_PHY_BCM_SR_PCIE)		+= phy-bcm-sr-pcie.o
+obj-$(CONFIG_PHY_BCM_SR_USB)		+= phy-bcm-sr-usb.o
diff --git a/drivers/phy/broadcom/phy-bcm-sr-usb.c b/drivers/phy/broadcom/phy-bcm-sr-usb.c
new file mode 100644
index 000000000000..fe6c58910e4c
--- /dev/null
+++ b/drivers/phy/broadcom/phy-bcm-sr-usb.c
@@ -0,0 +1,394 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2016-2018 Broadcom
+ */
+
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+
+enum bcm_usb_phy_version {
+	BCM_SR_USB_COMBO_PHY,
+	BCM_SR_USB_HS_PHY,
+};
+
+enum bcm_usb_phy_reg {
+	PLL_NDIV_FRAC,
+	PLL_NDIV_INT,
+	PLL_CTRL,
+	PHY_CTRL,
+	PHY_PLL_CTRL,
+};
+
+/* USB PHY registers */
+
+static const u8 bcm_usb_combo_phy_ss[] = {
+	[PLL_CTRL]		= 0x18,
+	[PHY_CTRL]		= 0x14,
+};
+
+static const u8 bcm_usb_combo_phy_hs[] = {
+	[PLL_NDIV_FRAC]	= 0x04,
+	[PLL_NDIV_INT]	= 0x08,
+	[PLL_CTRL]	= 0x0c,
+	[PHY_CTRL]	= 0x10,
+};
+
+#define HSPLL_NDIV_INT_VAL	0x13
+#define HSPLL_NDIV_FRAC_VAL	0x1005
+
+static const u8 bcm_usb_hs_phy[] = {
+	[PLL_NDIV_FRAC]	= 0x0,
+	[PLL_NDIV_INT]	= 0x4,
+	[PLL_CTRL]	= 0x8,
+	[PHY_CTRL]	= 0xc,
+};
+
+enum pll_ctrl_bits {
+	PLL_RESETB,
+	SSPLL_SUSPEND_EN,
+	PLL_SEQ_START,
+	PLL_LOCK,
+	PLL_PDIV,
+};
+
+static const u8 u3pll_ctrl[] = {
+	[PLL_RESETB]		= 0,
+	[SSPLL_SUSPEND_EN]	= 1,
+	[PLL_SEQ_START]		= 2,
+	[PLL_LOCK]		= 3,
+};
+
+#define HSPLL_PDIV_MASK		0xF
+#define HSPLL_PDIV_VAL		0x1
+
+static const u8 u2pll_ctrl[] = {
+	[PLL_PDIV]	= 1,
+	[PLL_RESETB]	= 5,
+	[PLL_LOCK]	= 6,
+};
+
+enum bcm_usb_phy_ctrl_bits {
+	CORERDY,
+	AFE_LDO_PWRDWNB,
+	AFE_PLL_PWRDWNB,
+	AFE_BG_PWRDWNB,
+	PHY_ISO,
+	PHY_RESETB,
+	PHY_PCTL,
+};
+
+#define PHY_PCTL_MASK	0xffff
+/*
+ * 0x0806 of PCTL_VAL has below bits set
+ * BIT-8 : refclk divider 1
+ * BIT-3:2: device mode; mode is not effect
+ * BIT-1: soft reset active low
+ */
+#define HSPHY_PCTL_VAL	0x0806
+#define SSPHY_PCTL_VAL	0x0006
+
+static const u8 u3phy_ctrl[] = {
+	[PHY_RESETB]	= 1,
+	[PHY_PCTL]	= 2,
+};
+
+static const u8 u2phy_ctrl[] = {
+	[CORERDY]		= 0,
+	[AFE_LDO_PWRDWNB]	= 1,
+	[AFE_PLL_PWRDWNB]	= 2,
+	[AFE_BG_PWRDWNB]	= 3,
+	[PHY_ISO]		= 4,
+	[PHY_RESETB]		= 5,
+	[PHY_PCTL]		= 6,
+};
+
+struct bcm_usb_phy_cfg {
+	uint32_t type;
+	uint32_t version;
+	void __iomem *regs;
+	struct phy *phy;
+	const u8 *offset;
+};
+
+#define PLL_LOCK_RETRY_COUNT	1000
+
+enum bcm_usb_phy_type {
+	USB_HS_PHY,
+	USB_SS_PHY,
+};
+
+#define NUM_BCM_SR_USB_COMBO_PHYS	2
+
+static inline void bcm_usb_reg32_clrbits(void __iomem *addr, uint32_t clear)
+{
+	writel(readl(addr) & ~clear, addr);
+}
+
+static inline void bcm_usb_reg32_setbits(void __iomem *addr, uint32_t set)
+{
+	writel(readl(addr) | set, addr);
+}
+
+static int bcm_usb_pll_lock_check(void __iomem *addr, u32 bit)
+{
+	int retry;
+	u32 rd_data;
+
+	retry = PLL_LOCK_RETRY_COUNT;
+	do {
+		rd_data = readl(addr);
+		if (rd_data & bit)
+			return 0;
+		udelay(1);
+	} while (--retry > 0);
+
+	pr_err("%s: FAIL\n", __func__);
+	return -ETIMEDOUT;
+}
+
+static int bcm_usb_ss_phy_init(struct bcm_usb_phy_cfg *phy_cfg)
+{
+	int ret = 0;
+	void __iomem *regs = phy_cfg->regs;
+	const u8 *offset;
+	u32 rd_data;
+
+	offset = phy_cfg->offset;
+
+	/* Set pctl with mode and soft reset */
+	rd_data = readl(regs + offset[PHY_CTRL]);
+	rd_data &= ~(PHY_PCTL_MASK << u3phy_ctrl[PHY_PCTL]);
+	rd_data |= (SSPHY_PCTL_VAL << u3phy_ctrl[PHY_PCTL]);
+	writel(rd_data, regs + offset[PHY_CTRL]);
+
+	bcm_usb_reg32_clrbits(regs + offset[PLL_CTRL],
+			      BIT(u3pll_ctrl[SSPLL_SUSPEND_EN]));
+	bcm_usb_reg32_setbits(regs + offset[PLL_CTRL],
+			      BIT(u3pll_ctrl[PLL_SEQ_START]));
+	bcm_usb_reg32_setbits(regs + offset[PLL_CTRL],
+			      BIT(u3pll_ctrl[PLL_RESETB]));
+
+	/* Maximum timeout for PLL reset done */
+	msleep(30);
+
+	ret = bcm_usb_pll_lock_check(regs + offset[PLL_CTRL],
+				     BIT(u3pll_ctrl[PLL_LOCK]));
+
+	return ret;
+}
+
+static int bcm_usb_hs_phy_init(struct bcm_usb_phy_cfg *phy_cfg)
+{
+	int ret = 0;
+	void __iomem *regs = phy_cfg->regs;
+	const u8 *offset;
+	u32 rd_data;
+
+	offset = phy_cfg->offset;
+
+	writel(HSPLL_NDIV_INT_VAL, regs + offset[PLL_NDIV_INT]);
+	writel(HSPLL_NDIV_FRAC_VAL, regs + offset[PLL_NDIV_FRAC]);
+
+	rd_data = readl(regs + offset[PLL_CTRL]);
+	rd_data &= ~(HSPLL_PDIV_MASK << u2pll_ctrl[PLL_PDIV]);
+	rd_data |= (HSPLL_PDIV_VAL << u2pll_ctrl[PLL_PDIV]);
+	writel(rd_data, regs + offset[PLL_CTRL]);
+
+	/* Set Core Ready high */
+	bcm_usb_reg32_setbits(regs + offset[PHY_CTRL],
+			      BIT(u2phy_ctrl[CORERDY]));
+
+	/* Maximum timeout for Core Ready done */
+	msleep(30);
+
+	bcm_usb_reg32_setbits(regs + offset[PLL_CTRL],
+			      BIT(u2pll_ctrl[PLL_RESETB]));
+	bcm_usb_reg32_setbits(regs + offset[PHY_CTRL],
+			      BIT(u2phy_ctrl[PHY_RESETB]));
+
+
+	rd_data = readl(regs + offset[PHY_CTRL]);
+	rd_data &= ~(PHY_PCTL_MASK << u2phy_ctrl[PHY_PCTL]);
+	rd_data |= (HSPHY_PCTL_VAL << u2phy_ctrl[PHY_PCTL]);
+	writel(rd_data, regs + offset[PHY_CTRL]);
+
+	/* Maximum timeout for PLL reset done */
+	msleep(30);
+
+	ret = bcm_usb_pll_lock_check(regs + offset[PLL_CTRL],
+				     BIT(u2pll_ctrl[PLL_LOCK]));
+
+	return ret;
+}
+
+static int bcm_usb_phy_reset(struct phy *phy)
+{
+	struct bcm_usb_phy_cfg *phy_cfg = phy_get_drvdata(phy);
+	void __iomem *regs = phy_cfg->regs;
+	const u8 *offset;
+
+	offset = phy_cfg->offset;
+
+	if (phy_cfg->type == USB_HS_PHY) {
+		bcm_usb_reg32_clrbits(regs + offset[PHY_CTRL],
+				      BIT(u2phy_ctrl[CORERDY]));
+		bcm_usb_reg32_setbits(regs + offset[PHY_CTRL],
+				      BIT(u2phy_ctrl[CORERDY]));
+	}
+
+	return 0;
+}
+
+static int bcm_usb_phy_init(struct phy *phy)
+{
+	struct bcm_usb_phy_cfg *phy_cfg = phy_get_drvdata(phy);
+	int ret = -EINVAL;
+
+	if (phy_cfg->type == USB_SS_PHY)
+		ret = bcm_usb_ss_phy_init(phy_cfg);
+	else if (phy_cfg->type == USB_HS_PHY)
+		ret = bcm_usb_hs_phy_init(phy_cfg);
+
+	return ret;
+}
+
+static struct phy_ops sr_phy_ops = {
+	.init		= bcm_usb_phy_init,
+	.reset		= bcm_usb_phy_reset,
+	.owner		= THIS_MODULE,
+};
+
+static struct phy *bcm_usb_phy_xlate(struct device *dev,
+				     struct of_phandle_args *args)
+{
+	struct bcm_usb_phy_cfg *phy_cfg;
+	int phy_idx;
+
+	phy_cfg = dev_get_drvdata(dev);
+	if (!phy_cfg)
+		return ERR_PTR(-EINVAL);
+
+	if (phy_cfg->version == BCM_SR_USB_COMBO_PHY) {
+		phy_idx = args->args[0];
+
+		if (WARN_ON(phy_idx > 1))
+			return ERR_PTR(-ENODEV);
+
+		return phy_cfg[phy_idx].phy;
+	} else
+		return phy_cfg->phy;
+}
+
+static int bcm_usb_phy_create(struct device *dev, struct device_node *node,
+			      void __iomem *regs, uint32_t version)
+{
+	struct bcm_usb_phy_cfg *phy_cfg;
+	int idx;
+
+	if (version == BCM_SR_USB_COMBO_PHY) {
+		phy_cfg = devm_kzalloc(dev, NUM_BCM_SR_USB_COMBO_PHYS *
+				       sizeof(struct bcm_usb_phy_cfg),
+				       GFP_KERNEL);
+		if (!phy_cfg)
+			return -ENOMEM;
+
+		for (idx = 0; idx < NUM_BCM_SR_USB_COMBO_PHYS; idx++) {
+			phy_cfg[idx].regs = regs;
+			phy_cfg[idx].version = version;
+			if (idx == 0) {
+				phy_cfg[idx].offset = bcm_usb_combo_phy_hs;
+				phy_cfg[idx].type = USB_HS_PHY;
+			} else {
+				phy_cfg[idx].offset = bcm_usb_combo_phy_ss;
+				phy_cfg[idx].type = USB_SS_PHY;
+			}
+			phy_cfg[idx].phy = devm_phy_create(dev, node,
+							   &sr_phy_ops);
+			if (IS_ERR(phy_cfg[idx].phy))
+				return PTR_ERR(phy_cfg[idx].phy);
+
+			phy_set_drvdata(phy_cfg[idx].phy, &phy_cfg[idx]);
+		}
+	} else if (version == BCM_SR_USB_HS_PHY) {
+		phy_cfg = devm_kzalloc(dev, sizeof(struct bcm_usb_phy_cfg),
+				       GFP_KERNEL);
+		if (!phy_cfg)
+			return -ENOMEM;
+
+		phy_cfg->regs = regs;
+		phy_cfg->version = version;
+		phy_cfg->offset = bcm_usb_hs_phy;
+		phy_cfg->type = USB_HS_PHY;
+		phy_cfg->phy = devm_phy_create(dev, node, &sr_phy_ops);
+		if (IS_ERR(phy_cfg->phy))
+			return PTR_ERR(phy_cfg->phy);
+
+		phy_set_drvdata(phy_cfg->phy, phy_cfg);
+	} else
+		return -ENODEV;
+
+	dev_set_drvdata(dev, phy_cfg);
+
+	return 0;
+}
+
+static const struct of_device_id bcm_usb_phy_of_match[] = {
+	{
+		.compatible = "brcm,sr-usb-combo-phy",
+		.data = (void *)BCM_SR_USB_COMBO_PHY,
+	},
+	{
+		.compatible = "brcm,sr-usb-hs-phy",
+		.data = (void *)BCM_SR_USB_HS_PHY,
+	},
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, bcm_usb_phy_of_match);
+
+static int bcm_usb_phy_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *dn = dev->of_node;
+	const struct of_device_id *of_id;
+	struct resource *res;
+	void __iomem *regs;
+	int ret;
+	enum bcm_usb_phy_version version;
+	struct phy_provider *phy_provider;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	regs = devm_ioremap_resource(dev, res);
+	if (IS_ERR(regs))
+		return PTR_ERR(regs);
+
+	of_id = of_match_node(bcm_usb_phy_of_match, dn);
+	if (of_id)
+		version = (enum bcm_usb_phy_version)of_id->data;
+	else
+		return -ENODEV;
+
+	ret = bcm_usb_phy_create(dev, dn, regs, version);
+	if (ret)
+		return ret;
+
+	phy_provider = devm_of_phy_provider_register(dev, bcm_usb_phy_xlate);
+
+	return PTR_ERR_OR_ZERO(phy_provider);
+}
+
+static struct platform_driver bcm_usb_phy_driver = {
+	.driver = {
+		.name = "phy-bcm-sr-usb",
+		.of_match_table = bcm_usb_phy_of_match,
+	},
+	.probe = bcm_usb_phy_probe,
+};
+module_platform_driver(bcm_usb_phy_driver);
+
+MODULE_AUTHOR("Broadcom");
+MODULE_DESCRIPTION("Broadcom stingray USB Phy driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/phy/freescale/phy-fsl-imx8mq-usb.c b/drivers/phy/freescale/phy-fsl-imx8mq-usb.c
index d6ea5ce8afa5..0c4833da7be0 100644
--- a/drivers/phy/freescale/phy-fsl-imx8mq-usb.c
+++ b/drivers/phy/freescale/phy-fsl-imx8mq-usb.c
@@ -6,6 +6,7 @@
 #include <linux/module.h>
 #include <linux/phy/phy.h>
 #include <linux/platform_device.h>
+#include <linux/regulator/consumer.h>
 
 #define PHY_CTRL0			0x0
 #define PHY_CTRL0_REF_SSP_EN		BIT(2)
@@ -24,6 +25,7 @@ struct imx8mq_usb_phy {
 	struct phy *phy;
 	struct clk *clk;
 	void __iomem *base;
+	struct regulator *vbus;
 };
 
 static int imx8mq_usb_phy_init(struct phy *phy)
@@ -55,6 +57,11 @@ static int imx8mq_usb_phy_init(struct phy *phy)
 static int imx8mq_phy_power_on(struct phy *phy)
 {
 	struct imx8mq_usb_phy *imx_phy = phy_get_drvdata(phy);
+	int ret;
+
+	ret = regulator_enable(imx_phy->vbus);
+	if (ret)
+		return ret;
 
 	return clk_prepare_enable(imx_phy->clk);
 }
@@ -64,6 +71,7 @@ static int imx8mq_phy_power_off(struct phy *phy)
 	struct imx8mq_usb_phy *imx_phy = phy_get_drvdata(phy);
 
 	clk_disable_unprepare(imx_phy->clk);
+	regulator_disable(imx_phy->vbus);
 
 	return 0;
 }
@@ -101,6 +109,10 @@ static int imx8mq_usb_phy_probe(struct platform_device *pdev)
 	if (IS_ERR(imx_phy->phy))
 		return PTR_ERR(imx_phy->phy);
 
+	imx_phy->vbus = devm_regulator_get(dev, "vbus");
+	if (IS_ERR(imx_phy->vbus))
+		return PTR_ERR(imx_phy->vbus);
+
 	phy_set_drvdata(imx_phy->phy, imx_phy);
 
 	phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate);
diff --git a/drivers/phy/hisilicon/Kconfig b/drivers/phy/hisilicon/Kconfig
index b40ee54a1a50..3c142f08987c 100644
--- a/drivers/phy/hisilicon/Kconfig
+++ b/drivers/phy/hisilicon/Kconfig
@@ -12,6 +12,16 @@ config PHY_HI6220_USB
 
 	  To compile this driver as a module, choose M here.
 
+config PHY_HI3660_USB
+	tristate "hi3660 USB PHY support"
+	depends on (ARCH_HISI && ARM64) || COMPILE_TEST
+	select GENERIC_PHY
+	select MFD_SYSCON
+	help
+	  Enable this to support the HISILICON HI3660 USB PHY.
+
+	  To compile this driver as a module, choose M here.
+
 config PHY_HISTB_COMBPHY
 	tristate "HiSilicon STB SoCs COMBPHY support"
 	depends on (ARCH_HISI && ARM64) || COMPILE_TEST
diff --git a/drivers/phy/hisilicon/Makefile b/drivers/phy/hisilicon/Makefile
index f662a4fe18d8..75ba64e2faf8 100644
--- a/drivers/phy/hisilicon/Makefile
+++ b/drivers/phy/hisilicon/Makefile
@@ -1,4 +1,5 @@
 obj-$(CONFIG_PHY_HI6220_USB)		+= phy-hi6220-usb.o
+obj-$(CONFIG_PHY_HI3660_USB)		+= phy-hi3660-usb3.o
 obj-$(CONFIG_PHY_HISTB_COMBPHY)		+= phy-histb-combphy.o
 obj-$(CONFIG_PHY_HISI_INNO_USB2)	+= phy-hisi-inno-usb2.o
 obj-$(CONFIG_PHY_HIX5HD2_SATA)		+= phy-hix5hd2-sata.o
diff --git a/drivers/phy/hisilicon/phy-hi3660-usb3.c b/drivers/phy/hisilicon/phy-hi3660-usb3.c
new file mode 100644
index 000000000000..cc0af2c044d0
--- /dev/null
+++ b/drivers/phy/hisilicon/phy-hi3660-usb3.c
@@ -0,0 +1,233 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Phy provider for USB 3.0 controller on HiSilicon 3660 platform
+ *
+ * Copyright (C) 2017-2018 Hilisicon Electronics Co., Ltd.
+ *		http://www.huawei.com
+ *
+ * Authors: Yu Chen <chenyu56@huawei.com>
+ */
+
+#include <linux/kernel.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#define PERI_CRG_CLK_EN4			0x40
+#define PERI_CRG_CLK_DIS4			0x44
+#define GT_CLK_USB3OTG_REF			BIT(0)
+#define GT_ACLK_USB3OTG				BIT(1)
+
+#define PERI_CRG_RSTEN4				0x90
+#define PERI_CRG_RSTDIS4			0x94
+#define IP_RST_USB3OTGPHY_POR			BIT(3)
+#define IP_RST_USB3OTG				BIT(5)
+
+#define PERI_CRG_ISODIS				0x148
+#define USB_REFCLK_ISO_EN			BIT(25)
+
+#define PCTRL_PERI_CTRL3			0x10
+#define PCTRL_PERI_CTRL3_MSK_START		16
+#define USB_TCXO_EN				BIT(1)
+
+#define PCTRL_PERI_CTRL24			0x64
+#define SC_CLK_USB3PHY_3MUX1_SEL		BIT(25)
+
+#define USBOTG3_CTRL0				0x00
+#define SC_USB3PHY_ABB_GT_EN			BIT(15)
+
+#define USBOTG3_CTRL2				0x08
+#define USBOTG3CTRL2_POWERDOWN_HSP		BIT(0)
+#define USBOTG3CTRL2_POWERDOWN_SSP		BIT(1)
+
+#define USBOTG3_CTRL3				0x0C
+#define USBOTG3_CTRL3_VBUSVLDEXT		BIT(6)
+#define USBOTG3_CTRL3_VBUSVLDEXTSEL		BIT(5)
+
+#define USBOTG3_CTRL4				0x10
+
+#define USBOTG3_CTRL7				0x1c
+#define REF_SSP_EN				BIT(16)
+
+/* This value config the default txtune parameter of the usb 2.0 phy */
+#define HI3660_USB_DEFAULT_PHY_PARAM		0x1c466e3
+
+struct hi3660_priv {
+	struct device *dev;
+	struct regmap *peri_crg;
+	struct regmap *pctrl;
+	struct regmap *otg_bc;
+	u32 eye_diagram_param;
+};
+
+static int hi3660_phy_init(struct phy *phy)
+{
+	struct hi3660_priv *priv = phy_get_drvdata(phy);
+	u32 val, mask;
+	int ret;
+
+	/* usb refclk iso disable */
+	ret = regmap_write(priv->peri_crg, PERI_CRG_ISODIS, USB_REFCLK_ISO_EN);
+	if (ret)
+		goto out;
+
+	/* enable usb_tcxo_en */
+	val = USB_TCXO_EN | (USB_TCXO_EN << PCTRL_PERI_CTRL3_MSK_START);
+	ret = regmap_write(priv->pctrl, PCTRL_PERI_CTRL3, val);
+	if (ret)
+		goto out;
+
+	/* assert phy */
+	val = IP_RST_USB3OTGPHY_POR | IP_RST_USB3OTG;
+	ret = regmap_write(priv->peri_crg, PERI_CRG_RSTEN4, val);
+	if (ret)
+		goto out;
+
+	/* enable phy ref clk */
+	val = SC_USB3PHY_ABB_GT_EN;
+	mask = val;
+	ret = regmap_update_bits(priv->otg_bc, USBOTG3_CTRL0, mask, val);
+	if (ret)
+		goto out;
+
+	val = REF_SSP_EN;
+	mask = val;
+	ret = regmap_update_bits(priv->otg_bc, USBOTG3_CTRL7, mask, val);
+	if (ret)
+		goto out;
+
+	/* exit from IDDQ mode */
+	mask = USBOTG3CTRL2_POWERDOWN_HSP | USBOTG3CTRL2_POWERDOWN_SSP;
+	ret = regmap_update_bits(priv->otg_bc, USBOTG3_CTRL2, mask, 0);
+	if (ret)
+		goto out;
+
+	/* delay for exit from IDDQ mode */
+	usleep_range(100, 120);
+
+	/* deassert phy */
+	val = IP_RST_USB3OTGPHY_POR | IP_RST_USB3OTG;
+	ret = regmap_write(priv->peri_crg, PERI_CRG_RSTDIS4, val);
+	if (ret)
+		goto out;
+
+	/* delay for phy deasserted */
+	usleep_range(10000, 15000);
+
+	/* fake vbus valid signal */
+	val = USBOTG3_CTRL3_VBUSVLDEXT | USBOTG3_CTRL3_VBUSVLDEXTSEL;
+	mask = val;
+	ret = regmap_update_bits(priv->otg_bc, USBOTG3_CTRL3, mask, val);
+	if (ret)
+		goto out;
+
+	/* delay for vbus valid */
+	usleep_range(100, 120);
+
+	ret = regmap_write(priv->otg_bc, USBOTG3_CTRL4,
+			priv->eye_diagram_param);
+	if (ret)
+		goto out;
+
+	return 0;
+out:
+	dev_err(priv->dev, "failed to init phy ret: %d\n", ret);
+	return ret;
+}
+
+static int hi3660_phy_exit(struct phy *phy)
+{
+	struct hi3660_priv *priv = phy_get_drvdata(phy);
+	u32 val;
+	int ret;
+
+	/* assert phy */
+	val = IP_RST_USB3OTGPHY_POR;
+	ret = regmap_write(priv->peri_crg, PERI_CRG_RSTEN4, val);
+	if (ret)
+		goto out;
+
+	/* disable usb_tcxo_en */
+	val = USB_TCXO_EN << PCTRL_PERI_CTRL3_MSK_START;
+	ret = regmap_write(priv->pctrl, PCTRL_PERI_CTRL3, val);
+	if (ret)
+		goto out;
+
+	return 0;
+out:
+	dev_err(priv->dev, "failed to exit phy ret: %d\n", ret);
+	return ret;
+}
+
+static struct phy_ops hi3660_phy_ops = {
+	.init		= hi3660_phy_init,
+	.exit		= hi3660_phy_exit,
+	.owner		= THIS_MODULE,
+};
+
+static int hi3660_phy_probe(struct platform_device *pdev)
+{
+	struct phy_provider *phy_provider;
+	struct device *dev = &pdev->dev;
+	struct phy *phy;
+	struct hi3660_priv *priv;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->dev = dev;
+	priv->peri_crg = syscon_regmap_lookup_by_phandle(dev->of_node,
+					"hisilicon,pericrg-syscon");
+	if (IS_ERR(priv->peri_crg)) {
+		dev_err(dev, "no hisilicon,pericrg-syscon\n");
+		return PTR_ERR(priv->peri_crg);
+	}
+
+	priv->pctrl = syscon_regmap_lookup_by_phandle(dev->of_node,
+					"hisilicon,pctrl-syscon");
+	if (IS_ERR(priv->pctrl)) {
+		dev_err(dev, "no hisilicon,pctrl-syscon\n");
+		return PTR_ERR(priv->pctrl);
+	}
+
+	/* node of hi3660 phy is a sub-node of usb3_otg_bc */
+	priv->otg_bc = syscon_node_to_regmap(dev->parent->of_node);
+	if (IS_ERR(priv->otg_bc)) {
+		dev_err(dev, "no hisilicon,usb3-otg-bc-syscon\n");
+		return PTR_ERR(priv->otg_bc);
+	}
+
+	if (of_property_read_u32(dev->of_node, "hisilicon,eye-diagram-param",
+		&(priv->eye_diagram_param)))
+		priv->eye_diagram_param = HI3660_USB_DEFAULT_PHY_PARAM;
+
+	phy = devm_phy_create(dev, NULL, &hi3660_phy_ops);
+	if (IS_ERR(phy))
+		return PTR_ERR(phy);
+
+	phy_set_drvdata(phy, priv);
+	phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate);
+	return PTR_ERR_OR_ZERO(phy_provider);
+}
+
+static const struct of_device_id hi3660_phy_of_match[] = {
+	{.compatible = "hisilicon,hi3660-usb-phy",},
+	{ }
+};
+MODULE_DEVICE_TABLE(of, hi3660_phy_of_match);
+
+static struct platform_driver hi3660_phy_driver = {
+	.probe	= hi3660_phy_probe,
+	.driver = {
+		.name	= "hi3660-usb-phy",
+		.of_match_table	= hi3660_phy_of_match,
+	}
+};
+module_platform_driver(hi3660_phy_driver);
+
+MODULE_AUTHOR("Yu Chen <chenyu56@huawei.com>");
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("Hilisicon Hi3660 USB3 PHY Driver");
diff --git a/drivers/phy/mediatek/Kconfig b/drivers/phy/mediatek/Kconfig
index 8857d00b3c65..b5a89dbd3fe7 100644
--- a/drivers/phy/mediatek/Kconfig
+++ b/drivers/phy/mediatek/Kconfig
@@ -13,6 +13,16 @@ config PHY_MTK_TPHY
 	  multi-ports is first version, otherwise is second veriosn,
 	  so you can easily distinguish them by banks layout.
 
+config PHY_MTK_UFS
+	tristate "MediaTek UFS M-PHY driver"
+	depends on ARCH_MEDIATEK && OF
+	select GENERIC_PHY
+	help
+	  Support for UFS M-PHY on MediaTek chipsets.
+	  Enable this to provide vendor-specific probing,
+	  initialization, power on and power off flow of
+	  specified M-PHYs.
+
 config PHY_MTK_XSPHY
     tristate "MediaTek XS-PHY Driver"
     depends on ARCH_MEDIATEK && OF
diff --git a/drivers/phy/mediatek/Makefile b/drivers/phy/mediatek/Makefile
index ee49edc97ee9..08a8e6a97b1e 100644
--- a/drivers/phy/mediatek/Makefile
+++ b/drivers/phy/mediatek/Makefile
@@ -4,4 +4,5 @@
 #
 
 obj-$(CONFIG_PHY_MTK_TPHY)		+= phy-mtk-tphy.o
+obj-$(CONFIG_PHY_MTK_UFS)		+= phy-mtk-ufs.o
 obj-$(CONFIG_PHY_MTK_XSPHY)		+= phy-mtk-xsphy.o
diff --git a/drivers/phy/mediatek/phy-mtk-tphy.c b/drivers/phy/mediatek/phy-mtk-tphy.c
index 5b6a470ca145..cb2ed3b25068 100644
--- a/drivers/phy/mediatek/phy-mtk-tphy.c
+++ b/drivers/phy/mediatek/phy-mtk-tphy.c
@@ -1103,13 +1103,9 @@ static int mtk_tphy_probe(struct platform_device *pdev)
 	}
 
 	/* it's deprecated, make it optional for backward compatibility */
-	tphy->u3phya_ref = devm_clk_get(dev, "u3phya_ref");
-	if (IS_ERR(tphy->u3phya_ref)) {
-		if (PTR_ERR(tphy->u3phya_ref) == -EPROBE_DEFER)
-			return -EPROBE_DEFER;
-
-		tphy->u3phya_ref = NULL;
-	}
+	tphy->u3phya_ref = devm_clk_get_optional(dev, "u3phya_ref");
+	if (IS_ERR(tphy->u3phya_ref))
+		return PTR_ERR(tphy->u3phya_ref);
 
 	tphy->src_ref_clk = U3P_REF_CLK;
 	tphy->src_coef = U3P_SLEW_RATE_COEF;
diff --git a/drivers/phy/mediatek/phy-mtk-ufs.c b/drivers/phy/mediatek/phy-mtk-ufs.c
new file mode 100644
index 000000000000..cf94f5c35dc5
--- /dev/null
+++ b/drivers/phy/mediatek/phy-mtk-ufs.c
@@ -0,0 +1,245 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2019 MediaTek Inc.
+ * Author: Stanley Chu <stanley.chu@mediatek.com>
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+
+/* mphy register and offsets */
+#define MP_GLB_DIG_8C               0x008C
+#define FRC_PLL_ISO_EN              BIT(8)
+#define PLL_ISO_EN                  BIT(9)
+#define FRC_FRC_PWR_ON              BIT(10)
+#define PLL_PWR_ON                  BIT(11)
+
+#define MP_LN_DIG_RX_9C             0xA09C
+#define FSM_DIFZ_FRC                BIT(18)
+
+#define MP_LN_DIG_RX_AC             0xA0AC
+#define FRC_RX_SQ_EN                BIT(0)
+#define RX_SQ_EN                    BIT(1)
+
+#define MP_LN_RX_44                 0xB044
+#define FRC_CDR_PWR_ON              BIT(17)
+#define CDR_PWR_ON                  BIT(18)
+#define FRC_CDR_ISO_EN              BIT(19)
+#define CDR_ISO_EN                  BIT(20)
+
+struct ufs_mtk_phy {
+	struct device *dev;
+	void __iomem *mmio;
+	struct clk *mp_clk;
+	struct clk *unipro_clk;
+};
+
+static inline u32 mphy_readl(struct ufs_mtk_phy *phy, u32 reg)
+{
+	return readl(phy->mmio + reg);
+}
+
+static inline void mphy_writel(struct ufs_mtk_phy *phy, u32 val, u32 reg)
+{
+	writel(val, phy->mmio + reg);
+}
+
+static void mphy_set_bit(struct ufs_mtk_phy *phy, u32 reg, u32 bit)
+{
+	u32 val;
+
+	val = mphy_readl(phy, reg);
+	val |= bit;
+	mphy_writel(phy, val, reg);
+}
+
+static void mphy_clr_bit(struct ufs_mtk_phy *phy, u32 reg, u32 bit)
+{
+	u32 val;
+
+	val = mphy_readl(phy, reg);
+	val &= ~bit;
+	mphy_writel(phy, val, reg);
+}
+
+static struct ufs_mtk_phy *get_ufs_mtk_phy(struct phy *generic_phy)
+{
+	return (struct ufs_mtk_phy *)phy_get_drvdata(generic_phy);
+}
+
+static int ufs_mtk_phy_clk_init(struct ufs_mtk_phy *phy)
+{
+	struct device *dev = phy->dev;
+
+	phy->unipro_clk = devm_clk_get(dev, "unipro");
+	if (IS_ERR(phy->unipro_clk)) {
+		dev_err(dev, "failed to get clock: unipro");
+		return PTR_ERR(phy->unipro_clk);
+	}
+
+	phy->mp_clk = devm_clk_get(dev, "mp");
+	if (IS_ERR(phy->mp_clk)) {
+		dev_err(dev, "failed to get clock: mp");
+		return PTR_ERR(phy->mp_clk);
+	}
+
+	return 0;
+}
+
+static void ufs_mtk_phy_set_active(struct ufs_mtk_phy *phy)
+{
+	/* release DA_MP_PLL_PWR_ON */
+	mphy_set_bit(phy, MP_GLB_DIG_8C, PLL_PWR_ON);
+	mphy_clr_bit(phy, MP_GLB_DIG_8C, FRC_FRC_PWR_ON);
+
+	/* release DA_MP_PLL_ISO_EN */
+	mphy_clr_bit(phy, MP_GLB_DIG_8C, PLL_ISO_EN);
+	mphy_clr_bit(phy, MP_GLB_DIG_8C, FRC_PLL_ISO_EN);
+
+	/* release DA_MP_CDR_PWR_ON */
+	mphy_set_bit(phy, MP_LN_RX_44, CDR_PWR_ON);
+	mphy_clr_bit(phy, MP_LN_RX_44, FRC_CDR_PWR_ON);
+
+	/* release DA_MP_CDR_ISO_EN */
+	mphy_clr_bit(phy, MP_LN_RX_44, CDR_ISO_EN);
+	mphy_clr_bit(phy, MP_LN_RX_44, FRC_CDR_ISO_EN);
+
+	/* release DA_MP_RX0_SQ_EN */
+	mphy_set_bit(phy, MP_LN_DIG_RX_AC, RX_SQ_EN);
+	mphy_clr_bit(phy, MP_LN_DIG_RX_AC, FRC_RX_SQ_EN);
+
+	/* delay 1us to wait DIFZ stable */
+	udelay(1);
+
+	/* release DIFZ */
+	mphy_clr_bit(phy, MP_LN_DIG_RX_9C, FSM_DIFZ_FRC);
+}
+
+static void ufs_mtk_phy_set_deep_hibern(struct ufs_mtk_phy *phy)
+{
+	/* force DIFZ */
+	mphy_set_bit(phy, MP_LN_DIG_RX_9C, FSM_DIFZ_FRC);
+
+	/* force DA_MP_RX0_SQ_EN */
+	mphy_set_bit(phy, MP_LN_DIG_RX_AC, FRC_RX_SQ_EN);
+	mphy_clr_bit(phy, MP_LN_DIG_RX_AC, RX_SQ_EN);
+
+	/* force DA_MP_CDR_ISO_EN */
+	mphy_set_bit(phy, MP_LN_RX_44, FRC_CDR_ISO_EN);
+	mphy_set_bit(phy, MP_LN_RX_44, CDR_ISO_EN);
+
+	/* force DA_MP_CDR_PWR_ON */
+	mphy_set_bit(phy, MP_LN_RX_44, FRC_CDR_PWR_ON);
+	mphy_clr_bit(phy, MP_LN_RX_44, CDR_PWR_ON);
+
+	/* force DA_MP_PLL_ISO_EN */
+	mphy_set_bit(phy, MP_GLB_DIG_8C, FRC_PLL_ISO_EN);
+	mphy_set_bit(phy, MP_GLB_DIG_8C, PLL_ISO_EN);
+
+	/* force DA_MP_PLL_PWR_ON */
+	mphy_set_bit(phy, MP_GLB_DIG_8C, FRC_FRC_PWR_ON);
+	mphy_clr_bit(phy, MP_GLB_DIG_8C, PLL_PWR_ON);
+}
+
+static int ufs_mtk_phy_power_on(struct phy *generic_phy)
+{
+	struct ufs_mtk_phy *phy = get_ufs_mtk_phy(generic_phy);
+	int ret;
+
+	ret = clk_prepare_enable(phy->unipro_clk);
+	if (ret) {
+		dev_err(phy->dev, "unipro_clk enable failed %d\n", ret);
+		goto out;
+	}
+
+	ret = clk_prepare_enable(phy->mp_clk);
+	if (ret) {
+		dev_err(phy->dev, "mp_clk enable failed %d\n", ret);
+		goto out_unprepare_unipro_clk;
+	}
+
+	ufs_mtk_phy_set_active(phy);
+
+	return 0;
+
+out_unprepare_unipro_clk:
+	clk_disable_unprepare(phy->unipro_clk);
+out:
+	return ret;
+}
+
+static int ufs_mtk_phy_power_off(struct phy *generic_phy)
+{
+	struct ufs_mtk_phy *phy = get_ufs_mtk_phy(generic_phy);
+
+	ufs_mtk_phy_set_deep_hibern(phy);
+
+	clk_disable_unprepare(phy->unipro_clk);
+	clk_disable_unprepare(phy->mp_clk);
+
+	return 0;
+}
+
+static const struct phy_ops ufs_mtk_phy_ops = {
+	.power_on       = ufs_mtk_phy_power_on,
+	.power_off      = ufs_mtk_phy_power_off,
+	.owner          = THIS_MODULE,
+};
+
+static int ufs_mtk_phy_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct phy *generic_phy;
+	struct phy_provider *phy_provider;
+	struct resource *res;
+	struct ufs_mtk_phy *phy;
+	int ret;
+
+	phy = devm_kzalloc(dev, sizeof(*phy), GFP_KERNEL);
+	if (!phy)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	phy->mmio = devm_ioremap_resource(dev, res);
+	if (IS_ERR(phy->mmio))
+		return PTR_ERR(phy->mmio);
+
+	phy->dev = dev;
+
+	ret = ufs_mtk_phy_clk_init(phy);
+	if (ret)
+		return ret;
+
+	generic_phy = devm_phy_create(dev, NULL, &ufs_mtk_phy_ops);
+	if (IS_ERR(generic_phy))
+		return PTR_ERR(generic_phy);
+
+	phy_set_drvdata(generic_phy, phy);
+
+	phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate);
+
+	return PTR_ERR_OR_ZERO(phy_provider);
+}
+
+static const struct of_device_id ufs_mtk_phy_of_match[] = {
+	{.compatible = "mediatek,mt8183-ufsphy"},
+	{},
+};
+MODULE_DEVICE_TABLE(of, ufs_mtk_phy_of_match);
+
+static struct platform_driver ufs_mtk_phy_driver = {
+	.probe = ufs_mtk_phy_probe,
+	.driver = {
+		.of_match_table = ufs_mtk_phy_of_match,
+		.name = "ufs_mtk_phy",
+	},
+};
+module_platform_driver(ufs_mtk_phy_driver);
+
+MODULE_DESCRIPTION("Universal Flash Storage (UFS) MediaTek MPHY");
+MODULE_AUTHOR("Stanley Chu <stanley.chu@mediatek.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/phy/mscc/phy-ocelot-serdes.c b/drivers/phy/mscc/phy-ocelot-serdes.c
index 77c46f639fbf..76f596365176 100644
--- a/drivers/phy/mscc/phy-ocelot-serdes.c
+++ b/drivers/phy/mscc/phy-ocelot-serdes.c
@@ -31,6 +31,238 @@ struct serdes_macro {
 	struct serdes_ctrl	*ctrl;
 };
 
+#define MCB_S6G_CFG_TIMEOUT     50
+
+static int __serdes_write_mcb_s6g(struct regmap *regmap, u8 macro, u32 op)
+{
+	unsigned int regval = 0;
+
+	regmap_write(regmap, HSIO_MCB_S6G_ADDR_CFG, op |
+		     HSIO_MCB_S6G_ADDR_CFG_SERDES6G_ADDR(BIT(macro)));
+
+	return regmap_read_poll_timeout(regmap, HSIO_MCB_S6G_ADDR_CFG, regval,
+					(regval & op) != op, 100,
+					MCB_S6G_CFG_TIMEOUT * 1000);
+}
+
+static int serdes_commit_mcb_s6g(struct regmap *regmap, u8 macro)
+{
+	return __serdes_write_mcb_s6g(regmap, macro,
+		HSIO_MCB_S6G_ADDR_CFG_SERDES6G_WR_ONE_SHOT);
+}
+
+static int serdes_update_mcb_s6g(struct regmap *regmap, u8 macro)
+{
+	return __serdes_write_mcb_s6g(regmap, macro,
+		HSIO_MCB_S6G_ADDR_CFG_SERDES6G_RD_ONE_SHOT);
+}
+
+static int serdes_init_s6g(struct regmap *regmap, u8 serdes, int mode)
+{
+	u32 pll_fsm_ctrl_data;
+	u32 ob_ena1v_mode;
+	u32 des_bw_ana;
+	u32 ob_ena_cas;
+	u32 if_mode;
+	u32 ob_lev;
+	u32 qrate;
+	int ret;
+
+	if (mode == PHY_INTERFACE_MODE_QSGMII) {
+		pll_fsm_ctrl_data = 120;
+		ob_ena1v_mode = 0;
+		ob_ena_cas = 0;
+		des_bw_ana = 5;
+		ob_lev = 24;
+		if_mode = 3;
+		qrate = 0;
+	} else {
+		pll_fsm_ctrl_data = 60;
+		ob_ena1v_mode = 1;
+		ob_ena_cas = 2;
+		des_bw_ana = 3;
+		ob_lev = 48;
+		if_mode = 1;
+		qrate = 1;
+	}
+
+	ret = serdes_update_mcb_s6g(regmap, serdes);
+	if (ret)
+		return ret;
+
+	/* Test pattern */
+
+	regmap_update_bits(regmap, HSIO_S6G_COMMON_CFG,
+			   HSIO_S6G_COMMON_CFG_SYS_RST, 0);
+
+	regmap_update_bits(regmap, HSIO_S6G_PLL_CFG,
+			   HSIO_S6G_PLL_CFG_PLL_FSM_ENA, 0);
+
+	regmap_update_bits(regmap, HSIO_S6G_IB_CFG,
+			   HSIO_S6G_IB_CFG_IB_SIG_DET_ENA |
+			   HSIO_S6G_IB_CFG_IB_REG_ENA |
+			   HSIO_S6G_IB_CFG_IB_SAM_ENA |
+			   HSIO_S6G_IB_CFG_IB_EQZ_ENA |
+			   HSIO_S6G_IB_CFG_IB_CONCUR |
+			   HSIO_S6G_IB_CFG_IB_CAL_ENA,
+			   HSIO_S6G_IB_CFG_IB_SIG_DET_ENA |
+			   HSIO_S6G_IB_CFG_IB_REG_ENA |
+			   HSIO_S6G_IB_CFG_IB_SAM_ENA |
+			   HSIO_S6G_IB_CFG_IB_EQZ_ENA |
+			   HSIO_S6G_IB_CFG_IB_CONCUR);
+
+	regmap_update_bits(regmap, HSIO_S6G_IB_CFG1,
+			   HSIO_S6G_IB_CFG1_IB_FRC_OFFSET |
+			   HSIO_S6G_IB_CFG1_IB_FRC_LP |
+			   HSIO_S6G_IB_CFG1_IB_FRC_MID |
+			   HSIO_S6G_IB_CFG1_IB_FRC_HP |
+			   HSIO_S6G_IB_CFG1_IB_FILT_OFFSET |
+			   HSIO_S6G_IB_CFG1_IB_FILT_LP |
+			   HSIO_S6G_IB_CFG1_IB_FILT_MID |
+			   HSIO_S6G_IB_CFG1_IB_FILT_HP,
+			   HSIO_S6G_IB_CFG1_IB_FILT_OFFSET |
+			   HSIO_S6G_IB_CFG1_IB_FILT_HP |
+			   HSIO_S6G_IB_CFG1_IB_FILT_LP |
+			   HSIO_S6G_IB_CFG1_IB_FILT_MID);
+
+	regmap_update_bits(regmap, HSIO_S6G_IB_CFG2,
+			   HSIO_S6G_IB_CFG2_IB_UREG_M,
+			   HSIO_S6G_IB_CFG2_IB_UREG(4));
+
+	regmap_update_bits(regmap, HSIO_S6G_IB_CFG3,
+			   HSIO_S6G_IB_CFG3_IB_INI_OFFSET_M |
+			   HSIO_S6G_IB_CFG3_IB_INI_LP_M |
+			   HSIO_S6G_IB_CFG3_IB_INI_MID_M |
+			   HSIO_S6G_IB_CFG3_IB_INI_HP_M,
+			   HSIO_S6G_IB_CFG3_IB_INI_OFFSET(31) |
+			   HSIO_S6G_IB_CFG3_IB_INI_LP(1) |
+			   HSIO_S6G_IB_CFG3_IB_INI_MID(31) |
+			   HSIO_S6G_IB_CFG3_IB_INI_HP(0));
+
+	regmap_update_bits(regmap, HSIO_S6G_MISC_CFG,
+			   HSIO_S6G_MISC_CFG_LANE_RST,
+			   HSIO_S6G_MISC_CFG_LANE_RST);
+
+	ret = serdes_commit_mcb_s6g(regmap, serdes);
+	if (ret)
+		return ret;
+
+	/* OB + DES + IB + SER CFG */
+	regmap_update_bits(regmap, HSIO_S6G_OB_CFG,
+			   HSIO_S6G_OB_CFG_OB_IDLE |
+			   HSIO_S6G_OB_CFG_OB_ENA1V_MODE |
+			   HSIO_S6G_OB_CFG_OB_POST0_M |
+			   HSIO_S6G_OB_CFG_OB_PREC_M,
+			   (ob_ena1v_mode ? HSIO_S6G_OB_CFG_OB_ENA1V_MODE : 0) |
+			   HSIO_S6G_OB_CFG_OB_POST0(0) |
+			   HSIO_S6G_OB_CFG_OB_PREC(0));
+
+	regmap_update_bits(regmap, HSIO_S6G_OB_CFG1,
+			   HSIO_S6G_OB_CFG1_OB_ENA_CAS_M |
+			   HSIO_S6G_OB_CFG1_OB_LEV_M,
+			   HSIO_S6G_OB_CFG1_OB_LEV(ob_lev) |
+			   HSIO_S6G_OB_CFG1_OB_ENA_CAS(ob_ena_cas));
+
+	regmap_update_bits(regmap, HSIO_S6G_DES_CFG,
+			   HSIO_S6G_DES_CFG_DES_PHS_CTRL_M |
+			   HSIO_S6G_DES_CFG_DES_CPMD_SEL_M |
+			   HSIO_S6G_DES_CFG_DES_BW_ANA_M,
+			   HSIO_S6G_DES_CFG_DES_PHS_CTRL(2) |
+			   HSIO_S6G_DES_CFG_DES_CPMD_SEL(0) |
+			   HSIO_S6G_DES_CFG_DES_BW_ANA(des_bw_ana));
+
+	regmap_update_bits(regmap, HSIO_S6G_IB_CFG,
+			   HSIO_S6G_IB_CFG_IB_SIG_DET_CLK_SEL_M |
+			   HSIO_S6G_IB_CFG_IB_REG_PAT_SEL_OFFSET_M,
+			   HSIO_S6G_IB_CFG_IB_REG_PAT_SEL_OFFSET(0) |
+			   HSIO_S6G_IB_CFG_IB_SIG_DET_CLK_SEL(0));
+
+	regmap_update_bits(regmap, HSIO_S6G_IB_CFG1,
+			   HSIO_S6G_IB_CFG1_IB_TSDET_M,
+			   HSIO_S6G_IB_CFG1_IB_TSDET(16));
+
+	regmap_update_bits(regmap, HSIO_S6G_SER_CFG,
+			   HSIO_S6G_SER_CFG_SER_ALISEL_M |
+			   HSIO_S6G_SER_CFG_SER_ENALI,
+			   HSIO_S6G_SER_CFG_SER_ALISEL(0));
+
+	regmap_update_bits(regmap, HSIO_S6G_PLL_CFG,
+			   HSIO_S6G_PLL_CFG_PLL_DIV4 |
+			   HSIO_S6G_PLL_CFG_PLL_ENA_ROT |
+			   HSIO_S6G_PLL_CFG_PLL_FSM_CTRL_DATA_M |
+			   HSIO_S6G_PLL_CFG_PLL_ROT_DIR |
+			   HSIO_S6G_PLL_CFG_PLL_ROT_FRQ,
+			   HSIO_S6G_PLL_CFG_PLL_FSM_CTRL_DATA
+			   (pll_fsm_ctrl_data));
+
+	regmap_update_bits(regmap, HSIO_S6G_COMMON_CFG,
+			   HSIO_S6G_COMMON_CFG_SYS_RST |
+			   HSIO_S6G_COMMON_CFG_ENA_LANE |
+			   HSIO_S6G_COMMON_CFG_PWD_RX |
+			   HSIO_S6G_COMMON_CFG_PWD_TX |
+			   HSIO_S6G_COMMON_CFG_HRATE |
+			   HSIO_S6G_COMMON_CFG_QRATE |
+			   HSIO_S6G_COMMON_CFG_ENA_ELOOP |
+			   HSIO_S6G_COMMON_CFG_ENA_FLOOP |
+			   HSIO_S6G_COMMON_CFG_IF_MODE_M,
+			   HSIO_S6G_COMMON_CFG_SYS_RST |
+			   HSIO_S6G_COMMON_CFG_ENA_LANE |
+			   (qrate ? HSIO_S6G_COMMON_CFG_QRATE : 0) |
+			   HSIO_S6G_COMMON_CFG_IF_MODE(if_mode));
+
+	regmap_update_bits(regmap, HSIO_S6G_MISC_CFG,
+			   HSIO_S6G_MISC_CFG_LANE_RST |
+			   HSIO_S6G_MISC_CFG_DES_100FX_CPMD_ENA |
+			   HSIO_S6G_MISC_CFG_RX_LPI_MODE_ENA |
+			   HSIO_S6G_MISC_CFG_TX_LPI_MODE_ENA,
+			   HSIO_S6G_MISC_CFG_LANE_RST |
+			   HSIO_S6G_MISC_CFG_RX_LPI_MODE_ENA);
+
+
+	ret = serdes_commit_mcb_s6g(regmap, serdes);
+	if (ret)
+		return ret;
+
+	regmap_update_bits(regmap, HSIO_S6G_PLL_CFG,
+			   HSIO_S6G_PLL_CFG_PLL_FSM_ENA,
+			   HSIO_S6G_PLL_CFG_PLL_FSM_ENA);
+
+	ret = serdes_commit_mcb_s6g(regmap, serdes);
+	if (ret)
+		return ret;
+
+	/* Wait for PLL bringup */
+	msleep(20);
+
+	regmap_update_bits(regmap, HSIO_S6G_IB_CFG,
+			   HSIO_S6G_IB_CFG_IB_CAL_ENA,
+			   HSIO_S6G_IB_CFG_IB_CAL_ENA);
+
+	regmap_update_bits(regmap, HSIO_S6G_MISC_CFG,
+			   HSIO_S6G_MISC_CFG_LANE_RST, 0);
+
+	ret = serdes_commit_mcb_s6g(regmap, serdes);
+	if (ret)
+		return ret;
+
+	/* Wait for calibration */
+	msleep(60);
+
+	regmap_update_bits(regmap, HSIO_S6G_IB_CFG,
+			   HSIO_S6G_IB_CFG_IB_REG_PAT_SEL_OFFSET_M |
+			   HSIO_S6G_IB_CFG_IB_SIG_DET_CLK_SEL_M,
+			   HSIO_S6G_IB_CFG_IB_REG_PAT_SEL_OFFSET(0) |
+			   HSIO_S6G_IB_CFG_IB_SIG_DET_CLK_SEL(7));
+
+	regmap_update_bits(regmap, HSIO_S6G_IB_CFG1,
+			   HSIO_S6G_IB_CFG1_IB_TSDET_M,
+			   HSIO_S6G_IB_CFG1_IB_TSDET(3));
+
+	/* IB CFG */
+
+	return 0;
+}
+
 #define MCB_S1G_CFG_TIMEOUT     50
 
 static int __serdes_write_mcb_s1g(struct regmap *regmap, u8 macro, u32 op)
@@ -110,7 +342,7 @@ struct serdes_mux {
 	u32			mux;
 };
 
-#define SERDES_MUX(_idx, _port, _mode, _submode, _mask, _mux) {		\
+#define SERDES_MUX(_idx, _port, _mode, _submode, _mask, _mux) { \
 	.idx = _idx,						\
 	.port = _port,						\
 	.mode = _mode,						\
@@ -191,8 +423,12 @@ static int serdes_set_mode(struct phy *phy, enum phy_mode mode, int submode)
 
 		if (macro->idx <= SERDES1G_MAX)
 			return serdes_init_s1g(macro->ctrl->regs, macro->idx);
+		else if (macro->idx <= SERDES6G_MAX)
+			return serdes_init_s6g(macro->ctrl->regs,
+					       macro->idx - (SERDES1G_MAX + 1),
+					       ocelot_serdes_muxes[i].submode);
 
-		/* SERDES6G and PCIe not supported yet */
+		/* PCIe not supported yet */
 		return -EOPNOTSUPP;
 	}
 
diff --git a/drivers/phy/phy-core.c b/drivers/phy/phy-core.c
index cb38f6e8614c..c147ba843f0b 100644
--- a/drivers/phy/phy-core.c
+++ b/drivers/phy/phy-core.c
@@ -384,10 +384,16 @@ int phy_reset(struct phy *phy)
 	if (!phy || !phy->ops->reset)
 		return 0;
 
+	ret = phy_pm_runtime_get_sync(phy);
+	if (ret < 0 && ret != -ENOTSUPP)
+		return ret;
+
 	mutex_lock(&phy->mutex);
 	ret = phy->ops->reset(phy);
 	mutex_unlock(&phy->mutex);
 
+	phy_pm_runtime_put(phy);
+
 	return ret;
 }
 EXPORT_SYMBOL_GPL(phy_reset);
@@ -564,6 +570,11 @@ void phy_put(struct phy *phy)
 	if (!phy || IS_ERR(phy))
 		return;
 
+	mutex_lock(&phy->mutex);
+	if (phy->ops->release)
+		phy->ops->release(phy);
+	mutex_unlock(&phy->mutex);
+
 	module_put(phy->ops->owner);
 	put_device(&phy->dev);
 }
diff --git a/drivers/phy/qualcomm/phy-qcom-qmp.c b/drivers/phy/qualcomm/phy-qcom-qmp.c
index 08d6f6f7f039..cd91b4179b10 100644
--- a/drivers/phy/qualcomm/phy-qcom-qmp.c
+++ b/drivers/phy/qualcomm/phy-qcom-qmp.c
@@ -242,6 +242,88 @@ static const struct qmp_phy_init_tbl msm8996_pcie_pcs_tbl[] = {
 	QMP_PHY_INIT_CFG(QPHY_TXDEEMPH_M3P5DB_V0, 0x0e),
 };
 
+static const struct qmp_phy_init_tbl msm8998_pcie_serdes_tbl[] = {
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_BIAS_EN_CLKBUFLR_EN, 0x14),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_CLK_SELECT, 0x30),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_PLL_IVCO, 0x0f),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_CMN_CONFIG, 0x06),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_LOCK_CMP_EN, 0x01),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_RESETSM_CNTRL, 0x20),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_VCO_TUNE_MAP, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_VCO_TUNE2_MODE0, 0x01),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_VCO_TUNE1_MODE0, 0xc9),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_VCO_TUNE_TIMER1, 0xff),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_VCO_TUNE_TIMER2, 0x3f),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_SVS_MODE_CLK_SEL, 0x01),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_CORE_CLK_EN, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_CORECLK_DIV_MODE0, 0x0a),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_CLK_EP_DIV, 0x19),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_CLK_ENABLE1, 0x90),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_DEC_START_MODE0, 0x82),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_DIV_FRAC_START3_MODE0, 0x03),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_DIV_FRAC_START2_MODE0, 0x55),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_DIV_FRAC_START1_MODE0, 0x55),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_LOCK_CMP3_MODE0, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_LOCK_CMP2_MODE0, 0x0d),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_LOCK_CMP1_MODE0, 0x04),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_HSCLK_SEL, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_CP_CTRL_MODE0, 0x08),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_PLL_RCTRL_MODE0, 0x16),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_PLL_CCTRL_MODE0, 0x34),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_CMN_CONFIG, 0x06),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_CLK_SELECT, 0x33),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_SYS_CLK_CTRL, 0x02),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_SYSCLK_BUF_ENABLE, 0x07),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_SYSCLK_EN_SEL, 0x04),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_INTEGLOOP_GAIN1_MODE0, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_INTEGLOOP_GAIN0_MODE0, 0x3f),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_BG_TIMER, 0x09),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_SSC_EN_CENTER, 0x01),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_SSC_PER1, 0x40),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_SSC_PER2, 0x01),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_SSC_ADJ_PER1, 0x02),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_SSC_ADJ_PER2, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_SSC_STEP_SIZE1, 0x7e),
+	QMP_PHY_INIT_CFG(QSERDES_V3_COM_SSC_STEP_SIZE2, 0x15),
+};
+
+static const struct qmp_phy_init_tbl msm8998_pcie_tx_tbl[] = {
+	QMP_PHY_INIT_CFG(QSERDES_V3_TX_RES_CODE_LANE_OFFSET_TX, 0x02),
+	QMP_PHY_INIT_CFG(QSERDES_V3_TX_RCV_DETECT_LVL_2, 0x12),
+	QMP_PHY_INIT_CFG(QSERDES_V3_TX_HIGHZ_DRVR_EN, 0x10),
+	QMP_PHY_INIT_CFG(QSERDES_V3_TX_LANE_MODE_1, 0x06),
+};
+
+static const struct qmp_phy_init_tbl msm8998_pcie_rx_tbl[] = {
+	QMP_PHY_INIT_CFG(QSERDES_V3_RX_SIGDET_CNTRL, 0x03),
+	QMP_PHY_INIT_CFG(QSERDES_V3_RX_SIGDET_ENABLES, 0x1c),
+	QMP_PHY_INIT_CFG(QSERDES_V3_RX_SIGDET_DEGLITCH_CNTRL, 0x14),
+	QMP_PHY_INIT_CFG(QSERDES_V3_RX_RX_EQU_ADAPTOR_CNTRL2, 0x0a),
+	QMP_PHY_INIT_CFG(QSERDES_V3_RX_RX_EQU_ADAPTOR_CNTRL3, 0x04),
+	QMP_PHY_INIT_CFG(QSERDES_V3_RX_RX_EQU_ADAPTOR_CNTRL4, 0x1a),
+	QMP_PHY_INIT_CFG(QSERDES_V3_RX_UCDR_SO_SATURATION_AND_ENABLE, 0x4b),
+	QMP_PHY_INIT_CFG(QSERDES_V3_RX_UCDR_SO_GAIN, 0x04),
+	QMP_PHY_INIT_CFG(QSERDES_V3_RX_UCDR_SO_GAIN_HALF, 0x04),
+	QMP_PHY_INIT_CFG(QSERDES_V3_RX_RX_EQ_OFFSET_ADAPTOR_CNTRL1, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V3_RX_RX_OFFSET_ADAPTOR_CNTRL2, 0x80),
+	QMP_PHY_INIT_CFG(QSERDES_V3_RX_RX_INTERFACE_MODE, 0x40),
+	QMP_PHY_INIT_CFG(QSERDES_V3_RX_UCDR_PI_CONTROLS, 0x71),
+	QMP_PHY_INIT_CFG(QSERDES_V3_RX_UCDR_FASTLOCK_COUNT_LOW, 0x40),
+};
+
+static const struct qmp_phy_init_tbl msm8998_pcie_pcs_tbl[] = {
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_ENDPOINT_REFCLK_DRIVE, 0x04),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_OSC_DTCT_ACTIONS, 0x00),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_PWRUP_RESET_DLY_TIME_AUXCLK, 0x01),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_L1SS_WAKEUP_DLY_TIME_AUXCLK_MSB, 0x00),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_L1SS_WAKEUP_DLY_TIME_AUXCLK_LSB, 0x20),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_LP_WAKEUP_DLY_TIME_AUXCLK_MSB, 0x00),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_LP_WAKEUP_DLY_TIME_AUXCLK, 0x01),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_PLL_LOCK_CHK_DLY_TIME, 0x73),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_RX_SIGDET_LVL, 0x99),
+	QMP_PHY_INIT_CFG(QPHY_V3_PCS_SIGDET_CNTRL, 0x03),
+};
+
 static const struct qmp_phy_init_tbl msm8996_usb3_serdes_tbl[] = {
 	QMP_PHY_INIT_CFG(QSERDES_COM_SYSCLK_EN_SEL, 0x14),
 	QMP_PHY_INIT_CFG(QSERDES_COM_BIAS_EN_CLKBUFLR_EN, 0x08),
@@ -897,6 +979,7 @@ struct qmp_phy {
  * @init_count: phy common block initialization count
  * @phy_initialized: indicate if PHY has been initialized
  * @mode: current PHY mode
+ * @ufs_reset: optional UFS PHY reset handle
  */
 struct qcom_qmp {
 	struct device *dev;
@@ -914,6 +997,8 @@ struct qcom_qmp {
 	int init_count;
 	bool phy_initialized;
 	enum phy_mode mode;
+
+	struct reset_control *ufs_reset;
 };
 
 static inline void qphy_setbits(void __iomem *base, u32 offset, u32 val)
@@ -1146,6 +1231,31 @@ static const struct qmp_phy_cfg sdm845_ufsphy_cfg = {
 	.no_pcs_sw_reset	= true,
 };
 
+static const struct qmp_phy_cfg msm8998_pciephy_cfg = {
+	.type			= PHY_TYPE_PCIE,
+	.nlanes			= 1,
+
+	.serdes_tbl		= msm8998_pcie_serdes_tbl,
+	.serdes_tbl_num		= ARRAY_SIZE(msm8998_pcie_serdes_tbl),
+	.tx_tbl			= msm8998_pcie_tx_tbl,
+	.tx_tbl_num		= ARRAY_SIZE(msm8998_pcie_tx_tbl),
+	.rx_tbl			= msm8998_pcie_rx_tbl,
+	.rx_tbl_num		= ARRAY_SIZE(msm8998_pcie_rx_tbl),
+	.pcs_tbl		= msm8998_pcie_pcs_tbl,
+	.pcs_tbl_num		= ARRAY_SIZE(msm8998_pcie_pcs_tbl),
+	.clk_list		= msm8996_phy_clk_l,
+	.num_clks		= ARRAY_SIZE(msm8996_phy_clk_l),
+	.reset_list		= ipq8074_pciephy_reset_l,
+	.num_resets		= ARRAY_SIZE(ipq8074_pciephy_reset_l),
+	.vreg_list		= qmp_phy_vreg_l,
+	.num_vregs		= ARRAY_SIZE(qmp_phy_vreg_l),
+	.regs			= pciephy_regs_layout,
+
+	.start_ctrl             = SERDES_START | PCS_START,
+	.pwrdn_ctrl		= SW_PWRDN | REFCLK_DRV_DSBL,
+	.mask_com_pcs_ready	= PCS_READY,
+};
+
 static const struct qmp_phy_cfg msm8998_usb3phy_cfg = {
 	.type                   = PHY_TYPE_USB3,
 	.nlanes                 = 1,
@@ -1314,6 +1424,7 @@ static int qcom_qmp_phy_com_exit(struct qcom_qmp *qmp)
 		return 0;
 	}
 
+	reset_control_assert(qmp->ufs_reset);
 	if (cfg->has_phy_com_ctrl) {
 		qphy_setbits(serdes, cfg->regs[QPHY_COM_START_CONTROL],
 			     SERDES_START | PCS_START);
@@ -1335,8 +1446,7 @@ static int qcom_qmp_phy_com_exit(struct qcom_qmp *qmp)
 	return 0;
 }
 
-/* PHY Initialization */
-static int qcom_qmp_phy_init(struct phy *phy)
+static int qcom_qmp_phy_enable(struct phy *phy)
 {
 	struct qmp_phy *qphy = phy_get_drvdata(phy);
 	struct qcom_qmp *qmp = qphy->qmp;
@@ -1351,6 +1461,33 @@ static int qcom_qmp_phy_init(struct phy *phy)
 
 	dev_vdbg(qmp->dev, "Initializing QMP phy\n");
 
+	if (cfg->no_pcs_sw_reset) {
+		/*
+		 * Get UFS reset, which is delayed until now to avoid a
+		 * circular dependency where UFS needs its PHY, but the PHY
+		 * needs this UFS reset.
+		 */
+		if (!qmp->ufs_reset) {
+			qmp->ufs_reset =
+				devm_reset_control_get_exclusive(qmp->dev,
+								 "ufsphy");
+
+			if (IS_ERR(qmp->ufs_reset)) {
+				ret = PTR_ERR(qmp->ufs_reset);
+				dev_err(qmp->dev,
+					"failed to get UFS reset: %d\n",
+					ret);
+
+				qmp->ufs_reset = NULL;
+				return ret;
+			}
+		}
+
+		ret = reset_control_assert(qmp->ufs_reset);
+		if (ret)
+			goto err_lane_rst;
+	}
+
 	ret = qcom_qmp_phy_com_init(qphy);
 	if (ret)
 		return ret;
@@ -1383,14 +1520,9 @@ static int qcom_qmp_phy_init(struct phy *phy)
 				       cfg->rx_tbl, cfg->rx_tbl_num);
 
 	qcom_qmp_phy_configure(pcs, cfg->regs, cfg->pcs_tbl, cfg->pcs_tbl_num);
-
-	/*
-	 * UFS PHY requires the deassert of software reset before serdes start.
-	 * For UFS PHYs that do not have software reset control bits, defer
-	 * starting serdes until the power on callback.
-	 */
-	if ((cfg->type == PHY_TYPE_UFS) && cfg->no_pcs_sw_reset)
-		goto out;
+	ret = reset_control_deassert(qmp->ufs_reset);
+	if (ret)
+		goto err_lane_rst;
 
 	/*
 	 * Pull out PHY from POWER DOWN state.
@@ -1403,7 +1535,9 @@ static int qcom_qmp_phy_init(struct phy *phy)
 		usleep_range(cfg->pwrdn_delay_min, cfg->pwrdn_delay_max);
 
 	/* Pull PHY out of reset state */
-	qphy_clrbits(pcs, cfg->regs[QPHY_SW_RESET], SW_RESET);
+	if (!cfg->no_pcs_sw_reset)
+		qphy_clrbits(pcs, cfg->regs[QPHY_SW_RESET], SW_RESET);
+
 	if (cfg->has_phy_dp_com_ctrl)
 		qphy_clrbits(dp_com, QPHY_V3_DP_COM_SW_RESET, SW_RESET);
 
@@ -1420,11 +1554,10 @@ static int qcom_qmp_phy_init(struct phy *phy)
 		goto err_pcs_ready;
 	}
 	qmp->phy_initialized = true;
-
-out:
-	return ret;
+	return 0;
 
 err_pcs_ready:
+	reset_control_assert(qmp->ufs_reset);
 	clk_disable_unprepare(qphy->pipe_clk);
 err_clk_enable:
 	if (cfg->has_lane_rst)
@@ -1435,7 +1568,7 @@ err_lane_rst:
 	return ret;
 }
 
-static int qcom_qmp_phy_exit(struct phy *phy)
+static int qcom_qmp_phy_disable(struct phy *phy)
 {
 	struct qmp_phy *qphy = phy_get_drvdata(phy);
 	struct qcom_qmp *qmp = qphy->qmp;
@@ -1463,44 +1596,6 @@ static int qcom_qmp_phy_exit(struct phy *phy)
 	return 0;
 }
 
-static int qcom_qmp_phy_poweron(struct phy *phy)
-{
-	struct qmp_phy *qphy = phy_get_drvdata(phy);
-	struct qcom_qmp *qmp = qphy->qmp;
-	const struct qmp_phy_cfg *cfg = qmp->cfg;
-	void __iomem *pcs = qphy->pcs;
-	void __iomem *status;
-	unsigned int mask, val;
-	int ret = 0;
-
-	if (cfg->type != PHY_TYPE_UFS)
-		return 0;
-
-	/*
-	 * For UFS PHY that has not software reset control, serdes start
-	 * should only happen when UFS driver explicitly calls phy_power_on
-	 * after it deasserts software reset.
-	 */
-	if (cfg->no_pcs_sw_reset && !qmp->phy_initialized &&
-	    (qmp->init_count != 0)) {
-		/* start SerDes and Phy-Coding-Sublayer */
-		qphy_setbits(pcs, cfg->regs[QPHY_START_CTRL], cfg->start_ctrl);
-
-		status = pcs + cfg->regs[QPHY_PCS_READY_STATUS];
-		mask = cfg->mask_pcs_ready;
-
-		ret = readl_poll_timeout(status, val, !(val & mask), 1,
-					 PHY_INIT_COMPLETE_TIMEOUT);
-		if (ret) {
-			dev_err(qmp->dev, "phy initialization timed-out\n");
-			return ret;
-		}
-		qmp->phy_initialized = true;
-	}
-
-	return ret;
-}
-
 static int qcom_qmp_phy_set_mode(struct phy *phy,
 				 enum phy_mode mode, int submode)
 {
@@ -1750,9 +1845,15 @@ static int phy_pipe_clk_register(struct qcom_qmp *qmp, struct device_node *np)
 }
 
 static const struct phy_ops qcom_qmp_phy_gen_ops = {
-	.init		= qcom_qmp_phy_init,
-	.exit		= qcom_qmp_phy_exit,
-	.power_on	= qcom_qmp_phy_poweron,
+	.init		= qcom_qmp_phy_enable,
+	.exit		= qcom_qmp_phy_disable,
+	.set_mode	= qcom_qmp_phy_set_mode,
+	.owner		= THIS_MODULE,
+};
+
+static const struct phy_ops qcom_qmp_ufs_ops = {
+	.power_on	= qcom_qmp_phy_enable,
+	.power_off	= qcom_qmp_phy_disable,
 	.set_mode	= qcom_qmp_phy_set_mode,
 	.owner		= THIS_MODULE,
 };
@@ -1763,6 +1864,7 @@ int qcom_qmp_phy_create(struct device *dev, struct device_node *np, int id)
 	struct qcom_qmp *qmp = dev_get_drvdata(dev);
 	struct phy *generic_phy;
 	struct qmp_phy *qphy;
+	const struct phy_ops *ops = &qcom_qmp_phy_gen_ops;
 	char prop_name[MAX_PROP_NAME];
 	int ret;
 
@@ -1849,7 +1951,10 @@ int qcom_qmp_phy_create(struct device *dev, struct device_node *np, int id)
 		}
 	}
 
-	generic_phy = devm_phy_create(dev, np, &qcom_qmp_phy_gen_ops);
+	if (qmp->cfg->type == PHY_TYPE_UFS)
+		ops = &qcom_qmp_ufs_ops;
+
+	generic_phy = devm_phy_create(dev, np, ops);
 	if (IS_ERR(generic_phy)) {
 		ret = PTR_ERR(generic_phy);
 		dev_err(dev, "failed to create qphy %d\n", ret);
@@ -1873,6 +1978,9 @@ static const struct of_device_id qcom_qmp_phy_of_match_table[] = {
 		.compatible = "qcom,msm8996-qmp-usb3-phy",
 		.data = &msm8996_usb3phy_cfg,
 	}, {
+		.compatible = "qcom,msm8998-qmp-pcie-phy",
+		.data = &msm8998_pciephy_cfg,
+	}, {
 		.compatible = "qcom,msm8998-qmp-ufs-phy",
 		.data = &sdm845_ufsphy_cfg,
 	}, {
diff --git a/drivers/phy/qualcomm/phy-qcom-qmp.h b/drivers/phy/qualcomm/phy-qcom-qmp.h
index a1b6cdee9a08..335ea5d7ef40 100644
--- a/drivers/phy/qualcomm/phy-qcom-qmp.h
+++ b/drivers/phy/qualcomm/phy-qcom-qmp.h
@@ -241,6 +241,7 @@
 #define QSERDES_V3_RX_RX_BAND				0x110
 #define QSERDES_V3_RX_RX_INTERFACE_MODE			0x11c
 #define QSERDES_V3_RX_RX_MODE_00			0x164
+#define QSERDES_V3_RX_RX_MODE_01			0x168
 
 /* Only for QMP V3 PHY - PCS registers */
 #define QPHY_V3_PCS_POWER_DOWN_CONTROL			0x004
@@ -280,6 +281,7 @@
 #define QPHY_V3_PCS_TSYNC_RSYNC_TIME			0x08c
 #define QPHY_V3_PCS_PWRUP_RESET_DLY_TIME_AUXCLK		0x0a0
 #define QPHY_V3_PCS_LP_WAKEUP_DLY_TIME_AUXCLK		0x0a4
+#define QPHY_V3_PCS_PLL_LOCK_CHK_DLY_TIME		0x0a8
 #define QPHY_V3_PCS_LFPS_TX_ECSTART_EQTLOCK		0x0b0
 #define QPHY_V3_PCS_RXEQTRAINING_WAIT_TIME		0x0b8
 #define QPHY_V3_PCS_RXEQTRAINING_RUN_TIME		0x0bc
@@ -292,13 +294,23 @@
 #define QPHY_V3_PCS_RX_MIN_HIBERN8_TIME			0x138
 #define QPHY_V3_PCS_RX_SIGDET_CTRL1			0x13c
 #define QPHY_V3_PCS_RX_SIGDET_CTRL2			0x140
+#define QPHY_V3_PCS_LP_WAKEUP_DLY_TIME_AUXCLK_MSB	0x1a8
+#define QPHY_V3_PCS_OSC_DTCT_ACTIONS			0x1ac
+#define QPHY_V3_PCS_SIGDET_CNTRL			0x1b0
 #define QPHY_V3_PCS_TX_MID_TERM_CTRL1			0x1bc
 #define QPHY_V3_PCS_MULTI_LANE_CTRL1			0x1c4
 #define QPHY_V3_PCS_RX_SIGDET_LVL			0x1d8
+#define QPHY_V3_PCS_L1SS_WAKEUP_DLY_TIME_AUXCLK_LSB	0x1dc
+#define QPHY_V3_PCS_L1SS_WAKEUP_DLY_TIME_AUXCLK_MSB	0x1e0
 #define QPHY_V3_PCS_REFGEN_REQ_CONFIG1			0x20c
 #define QPHY_V3_PCS_REFGEN_REQ_CONFIG2			0x210
 
 /* Only for QMP V3 PHY - PCS_MISC registers */
 #define QPHY_V3_PCS_MISC_CLAMP_ENABLE			0x0c
+#define QPHY_V3_PCS_MISC_OSC_DTCT_CONFIG2		0x2c
+#define QPHY_V3_PCS_MISC_PCIE_INT_AUX_CLK_CONFIG1	0x44
+#define QPHY_V3_PCS_MISC_OSC_DTCT_MODE2_CONFIG2		0x54
+#define QPHY_V3_PCS_MISC_OSC_DTCT_MODE2_CONFIG4		0x5c
+#define QPHY_V3_PCS_MISC_OSC_DTCT_MODE2_CONFIG5		0x60
 
 #endif
diff --git a/drivers/phy/qualcomm/phy-qcom-qusb2.c b/drivers/phy/qualcomm/phy-qcom-qusb2.c
index 8fd7ce139772..1cbf1d6f28ce 100644
--- a/drivers/phy/qualcomm/phy-qcom-qusb2.c
+++ b/drivers/phy/qualcomm/phy-qcom-qusb2.c
@@ -822,14 +822,9 @@ static int qusb2_phy_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	qphy->iface_clk = devm_clk_get(dev, "iface");
-	if (IS_ERR(qphy->iface_clk)) {
-		ret = PTR_ERR(qphy->iface_clk);
-		if (ret == -EPROBE_DEFER)
-			return ret;
-		qphy->iface_clk = NULL;
-		dev_dbg(dev, "failed to get iface clk, %d\n", ret);
-	}
+	qphy->iface_clk = devm_clk_get_optional(dev, "iface");
+	if (IS_ERR(qphy->iface_clk))
+		return PTR_ERR(qphy->iface_clk);
 
 	qphy->phy_reset = devm_reset_control_get_by_index(&pdev->dev, 0);
 	if (IS_ERR(qphy->phy_reset)) {
diff --git a/drivers/phy/qualcomm/phy-qcom-ufs-i.h b/drivers/phy/qualcomm/phy-qcom-ufs-i.h
index f798fb64de94..109ddd67be82 100644
--- a/drivers/phy/qualcomm/phy-qcom-ufs-i.h
+++ b/drivers/phy/qualcomm/phy-qcom-ufs-i.h
@@ -19,6 +19,7 @@
 #include <linux/clk.h>
 #include <linux/phy/phy.h>
 #include <linux/regulator/consumer.h>
+#include <linux/reset.h>
 #include <linux/slab.h>
 #include <linux/platform_device.h>
 #include <linux/io.h>
@@ -96,11 +97,10 @@ struct ufs_qcom_phy {
 	char name[UFS_QCOM_PHY_NAME_LEN];
 	struct ufs_qcom_phy_calibration *cached_regs;
 	int cached_regs_table_size;
-	bool is_powered_on;
-	bool is_started;
 	struct ufs_qcom_phy_specific_ops *phy_spec_ops;
 
 	enum phy_mode mode;
+	struct reset_control *ufs_reset;
 };
 
 /**
@@ -115,6 +115,7 @@ struct ufs_qcom_phy {
  * and writes to QSERDES_RX_SIGDET_CNTRL attribute
  */
 struct ufs_qcom_phy_specific_ops {
+	int (*calibrate)(struct ufs_qcom_phy *ufs_qcom_phy, bool is_rate_B);
 	void (*start_serdes)(struct ufs_qcom_phy *phy);
 	int (*is_physical_coding_sublayer_ready)(struct ufs_qcom_phy *phy);
 	void (*set_tx_lane_enable)(struct ufs_qcom_phy *phy, u32 val);
diff --git a/drivers/phy/qualcomm/phy-qcom-ufs-qmp-14nm.c b/drivers/phy/qualcomm/phy-qcom-ufs-qmp-14nm.c
index 1e0d4f2046a4..4815546f228c 100644
--- a/drivers/phy/qualcomm/phy-qcom-ufs-qmp-14nm.c
+++ b/drivers/phy/qualcomm/phy-qcom-ufs-qmp-14nm.c
@@ -42,28 +42,6 @@ void ufs_qcom_phy_qmp_14nm_advertise_quirks(struct ufs_qcom_phy *phy_common)
 		UFS_QCOM_PHY_QUIRK_HIBERN8_EXIT_AFTER_PHY_PWR_COLLAPSE;
 }
 
-static int ufs_qcom_phy_qmp_14nm_init(struct phy *generic_phy)
-{
-	struct ufs_qcom_phy *phy_common = get_ufs_qcom_phy(generic_phy);
-	bool is_rate_B = false;
-	int ret;
-
-	if (phy_common->mode == PHY_MODE_UFS_HS_B)
-		is_rate_B = true;
-
-	ret = ufs_qcom_phy_qmp_14nm_phy_calibrate(phy_common, is_rate_B);
-	if (!ret)
-		/* phy calibrated, but yet to be started */
-		phy_common->is_started = false;
-
-	return ret;
-}
-
-static int ufs_qcom_phy_qmp_14nm_exit(struct phy *generic_phy)
-{
-	return 0;
-}
-
 static
 int ufs_qcom_phy_qmp_14nm_set_mode(struct phy *generic_phy,
 				   enum phy_mode mode, int submode)
@@ -124,8 +102,6 @@ static int ufs_qcom_phy_qmp_14nm_is_pcs_ready(struct ufs_qcom_phy *phy_common)
 }
 
 static const struct phy_ops ufs_qcom_phy_qmp_14nm_phy_ops = {
-	.init		= ufs_qcom_phy_qmp_14nm_init,
-	.exit		= ufs_qcom_phy_qmp_14nm_exit,
 	.power_on	= ufs_qcom_phy_power_on,
 	.power_off	= ufs_qcom_phy_power_off,
 	.set_mode	= ufs_qcom_phy_qmp_14nm_set_mode,
@@ -133,6 +109,7 @@ static const struct phy_ops ufs_qcom_phy_qmp_14nm_phy_ops = {
 };
 
 static struct ufs_qcom_phy_specific_ops phy_14nm_ops = {
+	.calibrate		= ufs_qcom_phy_qmp_14nm_phy_calibrate,
 	.start_serdes		= ufs_qcom_phy_qmp_14nm_start_serdes,
 	.is_physical_coding_sublayer_ready = ufs_qcom_phy_qmp_14nm_is_pcs_ready,
 	.set_tx_lane_enable	= ufs_qcom_phy_qmp_14nm_set_tx_lane_enable,
diff --git a/drivers/phy/qualcomm/phy-qcom-ufs-qmp-20nm.c b/drivers/phy/qualcomm/phy-qcom-ufs-qmp-20nm.c
index aef40f7a41d4..f1cf819e12ea 100644
--- a/drivers/phy/qualcomm/phy-qcom-ufs-qmp-20nm.c
+++ b/drivers/phy/qualcomm/phy-qcom-ufs-qmp-20nm.c
@@ -61,28 +61,6 @@ void ufs_qcom_phy_qmp_20nm_advertise_quirks(struct ufs_qcom_phy *phy_common)
 		UFS_QCOM_PHY_QUIRK_HIBERN8_EXIT_AFTER_PHY_PWR_COLLAPSE;
 }
 
-static int ufs_qcom_phy_qmp_20nm_init(struct phy *generic_phy)
-{
-	struct ufs_qcom_phy *phy_common = get_ufs_qcom_phy(generic_phy);
-	bool is_rate_B = false;
-	int ret;
-
-	if (phy_common->mode == PHY_MODE_UFS_HS_B)
-		is_rate_B = true;
-
-	ret = ufs_qcom_phy_qmp_20nm_phy_calibrate(phy_common, is_rate_B);
-	if (!ret)
-		/* phy calibrated, but yet to be started */
-		phy_common->is_started = false;
-
-	return ret;
-}
-
-static int ufs_qcom_phy_qmp_20nm_exit(struct phy *generic_phy)
-{
-	return 0;
-}
-
 static
 int ufs_qcom_phy_qmp_20nm_set_mode(struct phy *generic_phy,
 				   enum phy_mode mode, int submode)
@@ -182,8 +160,6 @@ static int ufs_qcom_phy_qmp_20nm_is_pcs_ready(struct ufs_qcom_phy *phy_common)
 }
 
 static const struct phy_ops ufs_qcom_phy_qmp_20nm_phy_ops = {
-	.init		= ufs_qcom_phy_qmp_20nm_init,
-	.exit		= ufs_qcom_phy_qmp_20nm_exit,
 	.power_on	= ufs_qcom_phy_power_on,
 	.power_off	= ufs_qcom_phy_power_off,
 	.set_mode	= ufs_qcom_phy_qmp_20nm_set_mode,
@@ -191,6 +167,7 @@ static const struct phy_ops ufs_qcom_phy_qmp_20nm_phy_ops = {
 };
 
 static struct ufs_qcom_phy_specific_ops phy_20nm_ops = {
+	.calibrate		= ufs_qcom_phy_qmp_20nm_phy_calibrate,
 	.start_serdes		= ufs_qcom_phy_qmp_20nm_start_serdes,
 	.is_physical_coding_sublayer_ready = ufs_qcom_phy_qmp_20nm_is_pcs_ready,
 	.set_tx_lane_enable	= ufs_qcom_phy_qmp_20nm_set_tx_lane_enable,
diff --git a/drivers/phy/qualcomm/phy-qcom-ufs.c b/drivers/phy/qualcomm/phy-qcom-ufs.c
index 78c339b2fc8b..45404e31e672 100644
--- a/drivers/phy/qualcomm/phy-qcom-ufs.c
+++ b/drivers/phy/qualcomm/phy-qcom-ufs.c
@@ -147,6 +147,21 @@ out:
 }
 EXPORT_SYMBOL_GPL(ufs_qcom_phy_generic_probe);
 
+static int ufs_qcom_phy_get_reset(struct ufs_qcom_phy *phy_common)
+{
+	struct reset_control *reset;
+
+	if (phy_common->ufs_reset)
+		return 0;
+
+	reset = devm_reset_control_get_exclusive_by_index(phy_common->dev, 0);
+	if (IS_ERR(reset))
+		return PTR_ERR(reset);
+
+	phy_common->ufs_reset = reset;
+	return 0;
+}
+
 static int __ufs_qcom_phy_clk_get(struct device *dev,
 			 const char *name, struct clk **clk_out, bool err_print)
 {
@@ -528,23 +543,38 @@ int ufs_qcom_phy_power_on(struct phy *generic_phy)
 {
 	struct ufs_qcom_phy *phy_common = get_ufs_qcom_phy(generic_phy);
 	struct device *dev = phy_common->dev;
+	bool is_rate_B = false;
 	int err;
 
-	if (phy_common->is_powered_on)
-		return 0;
+	err = ufs_qcom_phy_get_reset(phy_common);
+	if (err)
+		return err;
 
-	if (!phy_common->is_started) {
-		err = ufs_qcom_phy_start_serdes(phy_common);
-		if (err)
-			return err;
+	err = reset_control_assert(phy_common->ufs_reset);
+	if (err)
+		return err;
 
-		err = ufs_qcom_phy_is_pcs_ready(phy_common);
-		if (err)
-			return err;
+	if (phy_common->mode == PHY_MODE_UFS_HS_B)
+		is_rate_B = true;
 
-		phy_common->is_started = true;
+	err = phy_common->phy_spec_ops->calibrate(phy_common, is_rate_B);
+	if (err)
+		return err;
+
+	err = reset_control_deassert(phy_common->ufs_reset);
+	if (err) {
+		dev_err(dev, "Failed to assert UFS PHY reset");
+		return err;
 	}
 
+	err = ufs_qcom_phy_start_serdes(phy_common);
+	if (err)
+		return err;
+
+	err = ufs_qcom_phy_is_pcs_ready(phy_common);
+	if (err)
+		return err;
+
 	err = ufs_qcom_phy_enable_vreg(dev, &phy_common->vdda_phy);
 	if (err) {
 		dev_err(dev, "%s enable vdda_phy failed, err=%d\n",
@@ -587,7 +617,6 @@ int ufs_qcom_phy_power_on(struct phy *generic_phy)
 		}
 	}
 
-	phy_common->is_powered_on = true;
 	goto out;
 
 out_disable_ref_clk:
@@ -607,9 +636,6 @@ int ufs_qcom_phy_power_off(struct phy *generic_phy)
 {
 	struct ufs_qcom_phy *phy_common = get_ufs_qcom_phy(generic_phy);
 
-	if (!phy_common->is_powered_on)
-		return 0;
-
 	phy_common->phy_spec_ops->power_control(phy_common, false);
 
 	if (phy_common->vddp_ref_clk.reg)
@@ -620,8 +646,7 @@ int ufs_qcom_phy_power_off(struct phy *generic_phy)
 
 	ufs_qcom_phy_disable_vreg(phy_common->dev, &phy_common->vdda_pll);
 	ufs_qcom_phy_disable_vreg(phy_common->dev, &phy_common->vdda_phy);
-	phy_common->is_powered_on = false;
-
+	reset_control_assert(phy_common->ufs_reset);
 	return 0;
 }
 EXPORT_SYMBOL_GPL(ufs_qcom_phy_power_off);
diff --git a/drivers/phy/renesas/Kconfig b/drivers/phy/renesas/Kconfig
index e340a925bbb1..111bdcae775c 100644
--- a/drivers/phy/renesas/Kconfig
+++ b/drivers/phy/renesas/Kconfig
@@ -19,7 +19,7 @@ config PHY_RCAR_GEN3_PCIE
 config PHY_RCAR_GEN3_USB2
 	tristate "Renesas R-Car generation 3 USB 2.0 PHY driver"
 	depends on ARCH_RENESAS
-	depends on EXTCON
+	depends on EXTCON || !EXTCON # if EXTCON=m, this cannot be built-in
 	depends on USB_SUPPORT
 	select GENERIC_PHY
 	select USB_COMMON
diff --git a/drivers/phy/renesas/phy-rcar-gen2.c b/drivers/phy/renesas/phy-rcar-gen2.c
index 72eeb066912d..8dc5710d9c98 100644
--- a/drivers/phy/renesas/phy-rcar-gen2.c
+++ b/drivers/phy/renesas/phy-rcar-gen2.c
@@ -4,6 +4,7 @@
  *
  * Copyright (C) 2014 Renesas Solutions Corp.
  * Copyright (C) 2014 Cogent Embedded, Inc.
+ * Copyright (C) 2019 Renesas Electronics Corp.
  */
 
 #include <linux/clk.h>
@@ -15,6 +16,7 @@
 #include <linux/platform_device.h>
 #include <linux/spinlock.h>
 #include <linux/atomic.h>
+#include <linux/of_device.h>
 
 #define USBHS_LPSTS			0x02
 #define USBHS_UGCTRL			0x80
@@ -35,6 +37,8 @@
 #define USBHS_UGCTRL2_USB0SEL		0x00000030
 #define USBHS_UGCTRL2_USB0SEL_PCI	0x00000010
 #define USBHS_UGCTRL2_USB0SEL_HS_USB	0x00000030
+#define USBHS_UGCTRL2_USB0SEL_USB20	0x00000010
+#define USBHS_UGCTRL2_USB0SEL_HS_USB20	0x00000020
 
 /* USB General status register (UGSTS) */
 #define USBHS_UGSTS_LOCK		0x00000100 /* From technical update */
@@ -64,6 +68,11 @@ struct rcar_gen2_phy_driver {
 	struct rcar_gen2_channel *channels;
 };
 
+struct rcar_gen2_phy_data {
+	const struct phy_ops *gen2_phy_ops;
+	const u32 (*select_value)[PHYS_PER_CHANNEL];
+};
+
 static int rcar_gen2_phy_init(struct phy *p)
 {
 	struct rcar_gen2_phy *phy = phy_get_drvdata(p);
@@ -180,6 +189,60 @@ static int rcar_gen2_phy_power_off(struct phy *p)
 	return 0;
 }
 
+static int rz_g1c_phy_power_on(struct phy *p)
+{
+	struct rcar_gen2_phy *phy = phy_get_drvdata(p);
+	struct rcar_gen2_phy_driver *drv = phy->channel->drv;
+	void __iomem *base = drv->base;
+	unsigned long flags;
+	u32 value;
+
+	spin_lock_irqsave(&drv->lock, flags);
+
+	/* Power on USBHS PHY */
+	value = readl(base + USBHS_UGCTRL);
+	value &= ~USBHS_UGCTRL_PLLRESET;
+	writel(value, base + USBHS_UGCTRL);
+
+	/* As per the data sheet wait 340 micro sec for power stable */
+	udelay(340);
+
+	if (phy->select_value == USBHS_UGCTRL2_USB0SEL_HS_USB20) {
+		value = readw(base + USBHS_LPSTS);
+		value |= USBHS_LPSTS_SUSPM;
+		writew(value, base + USBHS_LPSTS);
+	}
+
+	spin_unlock_irqrestore(&drv->lock, flags);
+
+	return 0;
+}
+
+static int rz_g1c_phy_power_off(struct phy *p)
+{
+	struct rcar_gen2_phy *phy = phy_get_drvdata(p);
+	struct rcar_gen2_phy_driver *drv = phy->channel->drv;
+	void __iomem *base = drv->base;
+	unsigned long flags;
+	u32 value;
+
+	spin_lock_irqsave(&drv->lock, flags);
+	/* Power off USBHS PHY */
+	if (phy->select_value == USBHS_UGCTRL2_USB0SEL_HS_USB20) {
+		value = readw(base + USBHS_LPSTS);
+		value &= ~USBHS_LPSTS_SUSPM;
+		writew(value, base + USBHS_LPSTS);
+	}
+
+	value = readl(base + USBHS_UGCTRL);
+	value |= USBHS_UGCTRL_PLLRESET;
+	writel(value, base + USBHS_UGCTRL);
+
+	spin_unlock_irqrestore(&drv->lock, flags);
+
+	return 0;
+}
+
 static const struct phy_ops rcar_gen2_phy_ops = {
 	.init		= rcar_gen2_phy_init,
 	.exit		= rcar_gen2_phy_exit,
@@ -188,12 +251,55 @@ static const struct phy_ops rcar_gen2_phy_ops = {
 	.owner		= THIS_MODULE,
 };
 
+static const struct phy_ops rz_g1c_phy_ops = {
+	.init		= rcar_gen2_phy_init,
+	.exit		= rcar_gen2_phy_exit,
+	.power_on	= rz_g1c_phy_power_on,
+	.power_off	= rz_g1c_phy_power_off,
+	.owner		= THIS_MODULE,
+};
+
+static const u32 pci_select_value[][PHYS_PER_CHANNEL] = {
+	[0]	= { USBHS_UGCTRL2_USB0SEL_PCI, USBHS_UGCTRL2_USB0SEL_HS_USB },
+	[2]	= { USBHS_UGCTRL2_USB2SEL_PCI, USBHS_UGCTRL2_USB2SEL_USB30 },
+};
+
+static const u32 usb20_select_value[][PHYS_PER_CHANNEL] = {
+	{ USBHS_UGCTRL2_USB0SEL_USB20, USBHS_UGCTRL2_USB0SEL_HS_USB20 },
+};
+
+static const struct rcar_gen2_phy_data rcar_gen2_usb_phy_data = {
+	.gen2_phy_ops = &rcar_gen2_phy_ops,
+	.select_value = pci_select_value,
+};
+
+static const struct rcar_gen2_phy_data rz_g1c_usb_phy_data = {
+	.gen2_phy_ops = &rz_g1c_phy_ops,
+	.select_value = usb20_select_value,
+};
+
 static const struct of_device_id rcar_gen2_phy_match_table[] = {
-	{ .compatible = "renesas,usb-phy-r8a7790" },
-	{ .compatible = "renesas,usb-phy-r8a7791" },
-	{ .compatible = "renesas,usb-phy-r8a7794" },
-	{ .compatible = "renesas,rcar-gen2-usb-phy" },
-	{ }
+	{
+		.compatible = "renesas,usb-phy-r8a77470",
+		.data = &rz_g1c_usb_phy_data,
+	},
+	{
+		.compatible = "renesas,usb-phy-r8a7790",
+		.data = &rcar_gen2_usb_phy_data,
+	},
+	{
+		.compatible = "renesas,usb-phy-r8a7791",
+		.data = &rcar_gen2_usb_phy_data,
+	},
+	{
+		.compatible = "renesas,usb-phy-r8a7794",
+		.data = &rcar_gen2_usb_phy_data,
+	},
+	{
+		.compatible = "renesas,rcar-gen2-usb-phy",
+		.data = &rcar_gen2_usb_phy_data,
+	},
+	{ /* sentinel */ },
 };
 MODULE_DEVICE_TABLE(of, rcar_gen2_phy_match_table);
 
@@ -224,11 +330,6 @@ static const u32 select_mask[] = {
 	[2]	= USBHS_UGCTRL2_USB2SEL,
 };
 
-static const u32 select_value[][PHYS_PER_CHANNEL] = {
-	[0]	= { USBHS_UGCTRL2_USB0SEL_PCI, USBHS_UGCTRL2_USB0SEL_HS_USB },
-	[2]	= { USBHS_UGCTRL2_USB2SEL_PCI, USBHS_UGCTRL2_USB2SEL_USB30 },
-};
-
 static int rcar_gen2_phy_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
@@ -238,6 +339,7 @@ static int rcar_gen2_phy_probe(struct platform_device *pdev)
 	struct resource *res;
 	void __iomem *base;
 	struct clk *clk;
+	const struct rcar_gen2_phy_data *data;
 	int i = 0;
 
 	if (!dev->of_node) {
@@ -266,6 +368,10 @@ static int rcar_gen2_phy_probe(struct platform_device *pdev)
 	drv->clk = clk;
 	drv->base = base;
 
+	data = of_device_get_match_data(dev);
+	if (!data)
+		return -EINVAL;
+
 	drv->num_channels = of_get_child_count(dev->of_node);
 	drv->channels = devm_kcalloc(dev, drv->num_channels,
 				     sizeof(struct rcar_gen2_channel),
@@ -294,10 +400,10 @@ static int rcar_gen2_phy_probe(struct platform_device *pdev)
 
 			phy->channel = channel;
 			phy->number = n;
-			phy->select_value = select_value[channel_num][n];
+			phy->select_value = data->select_value[channel_num][n];
 
 			phy->phy = devm_phy_create(dev, NULL,
-						   &rcar_gen2_phy_ops);
+						   data->gen2_phy_ops);
 			if (IS_ERR(phy->phy)) {
 				dev_err(dev, "Failed to create PHY\n");
 				return PTR_ERR(phy->phy);
diff --git a/drivers/phy/renesas/phy-rcar-gen3-usb2.c b/drivers/phy/renesas/phy-rcar-gen3-usb2.c
index 0a34782aaaa2..1322185a00a2 100644
--- a/drivers/phy/renesas/phy-rcar-gen3-usb2.c
+++ b/drivers/phy/renesas/phy-rcar-gen3-usb2.c
@@ -37,11 +37,8 @@
 
 /* INT_ENABLE */
 #define USB2_INT_ENABLE_UCOM_INTEN	BIT(3)
-#define USB2_INT_ENABLE_USBH_INTB_EN	BIT(2)
-#define USB2_INT_ENABLE_USBH_INTA_EN	BIT(1)
-#define USB2_INT_ENABLE_INIT		(USB2_INT_ENABLE_UCOM_INTEN | \
-					 USB2_INT_ENABLE_USBH_INTB_EN | \
-					 USB2_INT_ENABLE_USBH_INTA_EN)
+#define USB2_INT_ENABLE_USBH_INTB_EN	BIT(2)	/* For EHCI */
+#define USB2_INT_ENABLE_USBH_INTA_EN	BIT(1)	/* For OHCI */
 
 /* USBCTR */
 #define USB2_USBCTR_DIRPD	BIT(2)
@@ -78,10 +75,35 @@
 #define USB2_ADPCTRL_IDPULLUP		BIT(5)	/* 1 = ID sampling is enabled */
 #define USB2_ADPCTRL_DRVVBUS		BIT(4)
 
+#define NUM_OF_PHYS			4
+enum rcar_gen3_phy_index {
+	PHY_INDEX_BOTH_HC,
+	PHY_INDEX_OHCI,
+	PHY_INDEX_EHCI,
+	PHY_INDEX_HSUSB
+};
+
+static const u32 rcar_gen3_int_enable[NUM_OF_PHYS] = {
+	USB2_INT_ENABLE_USBH_INTB_EN | USB2_INT_ENABLE_USBH_INTA_EN,
+	USB2_INT_ENABLE_USBH_INTA_EN,
+	USB2_INT_ENABLE_USBH_INTB_EN,
+	0
+};
+
+struct rcar_gen3_phy {
+	struct phy *phy;
+	struct rcar_gen3_chan *ch;
+	u32 int_enable_bits;
+	bool initialized;
+	bool otg_initialized;
+	bool powered;
+};
+
 struct rcar_gen3_chan {
 	void __iomem *base;
+	struct device *dev;	/* platform_device's device */
 	struct extcon_dev *extcon;
-	struct phy *phy;
+	struct rcar_gen3_phy rphys[NUM_OF_PHYS];
 	struct regulator *vbus;
 	struct work_struct work;
 	enum usb_dr_mode dr_mode;
@@ -120,7 +142,7 @@ static void rcar_gen3_set_host_mode(struct rcar_gen3_chan *ch, int host)
 	void __iomem *usb2_base = ch->base;
 	u32 val = readl(usb2_base + USB2_COMMCTRL);
 
-	dev_vdbg(&ch->phy->dev, "%s: %08x, %d\n", __func__, val, host);
+	dev_vdbg(ch->dev, "%s: %08x, %d\n", __func__, val, host);
 	if (host)
 		val &= ~USB2_COMMCTRL_OTG_PERI;
 	else
@@ -133,7 +155,7 @@ static void rcar_gen3_set_linectrl(struct rcar_gen3_chan *ch, int dp, int dm)
 	void __iomem *usb2_base = ch->base;
 	u32 val = readl(usb2_base + USB2_LINECTRL1);
 
-	dev_vdbg(&ch->phy->dev, "%s: %08x, %d, %d\n", __func__, val, dp, dm);
+	dev_vdbg(ch->dev, "%s: %08x, %d, %d\n", __func__, val, dp, dm);
 	val &= ~(USB2_LINECTRL1_DP_RPD | USB2_LINECTRL1_DM_RPD);
 	if (dp)
 		val |= USB2_LINECTRL1_DP_RPD;
@@ -147,7 +169,7 @@ static void rcar_gen3_enable_vbus_ctrl(struct rcar_gen3_chan *ch, int vbus)
 	void __iomem *usb2_base = ch->base;
 	u32 val = readl(usb2_base + USB2_ADPCTRL);
 
-	dev_vdbg(&ch->phy->dev, "%s: %08x, %d\n", __func__, val, vbus);
+	dev_vdbg(ch->dev, "%s: %08x, %d\n", __func__, val, vbus);
 	if (vbus)
 		val |= USB2_ADPCTRL_DRVVBUS;
 	else
@@ -249,6 +271,42 @@ static enum phy_mode rcar_gen3_get_phy_mode(struct rcar_gen3_chan *ch)
 	return PHY_MODE_USB_DEVICE;
 }
 
+static bool rcar_gen3_is_any_rphy_initialized(struct rcar_gen3_chan *ch)
+{
+	int i;
+
+	for (i = 0; i < NUM_OF_PHYS; i++) {
+		if (ch->rphys[i].initialized)
+			return true;
+	}
+
+	return false;
+}
+
+static bool rcar_gen3_needs_init_otg(struct rcar_gen3_chan *ch)
+{
+	int i;
+
+	for (i = 0; i < NUM_OF_PHYS; i++) {
+		if (ch->rphys[i].otg_initialized)
+			return false;
+	}
+
+	return true;
+}
+
+static bool rcar_gen3_are_all_rphys_power_off(struct rcar_gen3_chan *ch)
+{
+	int i;
+
+	for (i = 0; i < NUM_OF_PHYS; i++) {
+		if (ch->rphys[i].powered)
+			return false;
+	}
+
+	return true;
+}
+
 static ssize_t role_store(struct device *dev, struct device_attribute *attr,
 			  const char *buf, size_t count)
 {
@@ -256,7 +314,7 @@ static ssize_t role_store(struct device *dev, struct device_attribute *attr,
 	bool is_b_device;
 	enum phy_mode cur_mode, new_mode;
 
-	if (!ch->is_otg_channel || !ch->phy->init_count)
+	if (!ch->is_otg_channel || !rcar_gen3_is_any_rphy_initialized(ch))
 		return -EIO;
 
 	if (!strncmp(buf, "host", strlen("host")))
@@ -294,7 +352,7 @@ static ssize_t role_show(struct device *dev, struct device_attribute *attr,
 {
 	struct rcar_gen3_chan *ch = dev_get_drvdata(dev);
 
-	if (!ch->is_otg_channel || !ch->phy->init_count)
+	if (!ch->is_otg_channel || !rcar_gen3_is_any_rphy_initialized(ch))
 		return -EIO;
 
 	return sprintf(buf, "%s\n", rcar_gen3_is_host(ch) ? "host" :
@@ -328,37 +386,62 @@ static void rcar_gen3_init_otg(struct rcar_gen3_chan *ch)
 
 static int rcar_gen3_phy_usb2_init(struct phy *p)
 {
-	struct rcar_gen3_chan *channel = phy_get_drvdata(p);
+	struct rcar_gen3_phy *rphy = phy_get_drvdata(p);
+	struct rcar_gen3_chan *channel = rphy->ch;
 	void __iomem *usb2_base = channel->base;
+	u32 val;
 
 	/* Initialize USB2 part */
-	writel(USB2_INT_ENABLE_INIT, usb2_base + USB2_INT_ENABLE);
+	val = readl(usb2_base + USB2_INT_ENABLE);
+	val |= USB2_INT_ENABLE_UCOM_INTEN | rphy->int_enable_bits;
+	writel(val, usb2_base + USB2_INT_ENABLE);
 	writel(USB2_SPD_RSM_TIMSET_INIT, usb2_base + USB2_SPD_RSM_TIMSET);
 	writel(USB2_OC_TIMSET_INIT, usb2_base + USB2_OC_TIMSET);
 
 	/* Initialize otg part */
-	if (channel->is_otg_channel)
-		rcar_gen3_init_otg(channel);
+	if (channel->is_otg_channel) {
+		if (rcar_gen3_needs_init_otg(channel))
+			rcar_gen3_init_otg(channel);
+		rphy->otg_initialized = true;
+	}
+
+	rphy->initialized = true;
 
 	return 0;
 }
 
 static int rcar_gen3_phy_usb2_exit(struct phy *p)
 {
-	struct rcar_gen3_chan *channel = phy_get_drvdata(p);
+	struct rcar_gen3_phy *rphy = phy_get_drvdata(p);
+	struct rcar_gen3_chan *channel = rphy->ch;
+	void __iomem *usb2_base = channel->base;
+	u32 val;
+
+	rphy->initialized = false;
+
+	if (channel->is_otg_channel)
+		rphy->otg_initialized = false;
 
-	writel(0, channel->base + USB2_INT_ENABLE);
+	val = readl(usb2_base + USB2_INT_ENABLE);
+	val &= ~rphy->int_enable_bits;
+	if (!rcar_gen3_is_any_rphy_initialized(channel))
+		val &= ~USB2_INT_ENABLE_UCOM_INTEN;
+	writel(val, usb2_base + USB2_INT_ENABLE);
 
 	return 0;
 }
 
 static int rcar_gen3_phy_usb2_power_on(struct phy *p)
 {
-	struct rcar_gen3_chan *channel = phy_get_drvdata(p);
+	struct rcar_gen3_phy *rphy = phy_get_drvdata(p);
+	struct rcar_gen3_chan *channel = rphy->ch;
 	void __iomem *usb2_base = channel->base;
 	u32 val;
 	int ret;
 
+	if (!rcar_gen3_are_all_rphys_power_off(channel))
+		return 0;
+
 	if (channel->vbus) {
 		ret = regulator_enable(channel->vbus);
 		if (ret)
@@ -371,14 +454,22 @@ static int rcar_gen3_phy_usb2_power_on(struct phy *p)
 	val &= ~USB2_USBCTR_PLL_RST;
 	writel(val, usb2_base + USB2_USBCTR);
 
+	rphy->powered = true;
+
 	return 0;
 }
 
 static int rcar_gen3_phy_usb2_power_off(struct phy *p)
 {
-	struct rcar_gen3_chan *channel = phy_get_drvdata(p);
+	struct rcar_gen3_phy *rphy = phy_get_drvdata(p);
+	struct rcar_gen3_chan *channel = rphy->ch;
 	int ret = 0;
 
+	rphy->powered = false;
+
+	if (!rcar_gen3_are_all_rphys_power_off(channel))
+		return 0;
+
 	if (channel->vbus)
 		ret = regulator_disable(channel->vbus);
 
@@ -393,6 +484,12 @@ static const struct phy_ops rcar_gen3_phy_usb2_ops = {
 	.owner		= THIS_MODULE,
 };
 
+static const struct phy_ops rz_g1c_phy_usb2_ops = {
+	.init		= rcar_gen3_phy_usb2_init,
+	.exit		= rcar_gen3_phy_usb2_exit,
+	.owner		= THIS_MODULE,
+};
+
 static irqreturn_t rcar_gen3_phy_usb2_irq(int irq, void *_ch)
 {
 	struct rcar_gen3_chan *ch = _ch;
@@ -401,7 +498,7 @@ static irqreturn_t rcar_gen3_phy_usb2_irq(int irq, void *_ch)
 	irqreturn_t ret = IRQ_NONE;
 
 	if (status & USB2_OBINT_BITS) {
-		dev_vdbg(&ch->phy->dev, "%s: %08x\n", __func__, status);
+		dev_vdbg(ch->dev, "%s: %08x\n", __func__, status);
 		writel(USB2_OBINT_BITS, usb2_base + USB2_OBINTSTA);
 		rcar_gen3_device_recognition(ch);
 		ret = IRQ_HANDLED;
@@ -411,11 +508,27 @@ static irqreturn_t rcar_gen3_phy_usb2_irq(int irq, void *_ch)
 }
 
 static const struct of_device_id rcar_gen3_phy_usb2_match_table[] = {
-	{ .compatible = "renesas,usb2-phy-r8a7795" },
-	{ .compatible = "renesas,usb2-phy-r8a7796" },
-	{ .compatible = "renesas,usb2-phy-r8a77965" },
-	{ .compatible = "renesas,rcar-gen3-usb2-phy" },
-	{ }
+	{
+		.compatible = "renesas,usb2-phy-r8a77470",
+		.data = &rz_g1c_phy_usb2_ops,
+	},
+	{
+		.compatible = "renesas,usb2-phy-r8a7795",
+		.data = &rcar_gen3_phy_usb2_ops,
+	},
+	{
+		.compatible = "renesas,usb2-phy-r8a7796",
+		.data = &rcar_gen3_phy_usb2_ops,
+	},
+	{
+		.compatible = "renesas,usb2-phy-r8a77965",
+		.data = &rcar_gen3_phy_usb2_ops,
+	},
+	{
+		.compatible = "renesas,rcar-gen3-usb2-phy",
+		.data = &rcar_gen3_phy_usb2_ops,
+	},
+	{ /* sentinel */ },
 };
 MODULE_DEVICE_TABLE(of, rcar_gen3_phy_usb2_match_table);
 
@@ -425,13 +538,54 @@ static const unsigned int rcar_gen3_phy_cable[] = {
 	EXTCON_NONE,
 };
 
+static struct phy *rcar_gen3_phy_usb2_xlate(struct device *dev,
+					    struct of_phandle_args *args)
+{
+	struct rcar_gen3_chan *ch = dev_get_drvdata(dev);
+
+	if (args->args_count == 0)	/* For old version dts */
+		return ch->rphys[PHY_INDEX_BOTH_HC].phy;
+	else if (args->args_count > 1)	/* Prevent invalid args count */
+		return ERR_PTR(-ENODEV);
+
+	if (args->args[0] >= NUM_OF_PHYS)
+		return ERR_PTR(-ENODEV);
+
+	return ch->rphys[args->args[0]].phy;
+}
+
+static enum usb_dr_mode rcar_gen3_get_dr_mode(struct device_node *np)
+{
+	enum usb_dr_mode candidate = USB_DR_MODE_UNKNOWN;
+	int i;
+
+	/*
+	 * If one of device nodes has other dr_mode except UNKNOWN,
+	 * this function returns UNKNOWN. To achieve backward compatibility,
+	 * this loop starts the index as 0.
+	 */
+	for (i = 0; i < NUM_OF_PHYS; i++) {
+		enum usb_dr_mode mode = of_usb_get_dr_mode_by_phy(np, i);
+
+		if (mode != USB_DR_MODE_UNKNOWN) {
+			if (candidate == USB_DR_MODE_UNKNOWN)
+				candidate = mode;
+			else if (candidate != mode)
+				return USB_DR_MODE_UNKNOWN;
+		}
+	}
+
+	return candidate;
+}
+
 static int rcar_gen3_phy_usb2_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct rcar_gen3_chan *channel;
 	struct phy_provider *provider;
 	struct resource *res;
-	int irq, ret = 0;
+	const struct phy_ops *phy_usb2_ops;
+	int irq, ret = 0, i;
 
 	if (!dev->of_node) {
 		dev_err(dev, "This driver needs device tree\n");
@@ -457,7 +611,7 @@ static int rcar_gen3_phy_usb2_probe(struct platform_device *pdev)
 			dev_err(dev, "No irq handler (%d)\n", irq);
 	}
 
-	channel->dr_mode = of_usb_get_dr_mode_by_phy(dev->of_node, 0);
+	channel->dr_mode = rcar_gen3_get_dr_mode(dev->of_node);
 	if (channel->dr_mode != USB_DR_MODE_UNKNOWN) {
 		int ret;
 
@@ -481,11 +635,21 @@ static int rcar_gen3_phy_usb2_probe(struct platform_device *pdev)
 	 * And then, phy-core will manage runtime pm for this device.
 	 */
 	pm_runtime_enable(dev);
-	channel->phy = devm_phy_create(dev, NULL, &rcar_gen3_phy_usb2_ops);
-	if (IS_ERR(channel->phy)) {
-		dev_err(dev, "Failed to create USB2 PHY\n");
-		ret = PTR_ERR(channel->phy);
-		goto error;
+	phy_usb2_ops = of_device_get_match_data(dev);
+	if (!phy_usb2_ops)
+		return -EINVAL;
+
+	for (i = 0; i < NUM_OF_PHYS; i++) {
+		channel->rphys[i].phy = devm_phy_create(dev, NULL,
+							phy_usb2_ops);
+		if (IS_ERR(channel->rphys[i].phy)) {
+			dev_err(dev, "Failed to create USB2 PHY\n");
+			ret = PTR_ERR(channel->rphys[i].phy);
+			goto error;
+		}
+		channel->rphys[i].ch = channel;
+		channel->rphys[i].int_enable_bits = rcar_gen3_int_enable[i];
+		phy_set_drvdata(channel->rphys[i].phy, &channel->rphys[i]);
 	}
 
 	channel->vbus = devm_regulator_get_optional(dev, "vbus");
@@ -498,9 +662,9 @@ static int rcar_gen3_phy_usb2_probe(struct platform_device *pdev)
 	}
 
 	platform_set_drvdata(pdev, channel);
-	phy_set_drvdata(channel->phy, channel);
+	channel->dev = dev;
 
-	provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate);
+	provider = devm_of_phy_provider_register(dev, rcar_gen3_phy_usb2_xlate);
 	if (IS_ERR(provider)) {
 		dev_err(dev, "Failed to register PHY provider\n");
 		ret = PTR_ERR(provider);
diff --git a/drivers/phy/rockchip/phy-rockchip-emmc.c b/drivers/phy/rockchip/phy-rockchip-emmc.c
index 19bf84f0bc67..b804c6bf4b55 100644
--- a/drivers/phy/rockchip/phy-rockchip-emmc.c
+++ b/drivers/phy/rockchip/phy-rockchip-emmc.c
@@ -87,6 +87,7 @@ struct rockchip_emmc_phy {
 	unsigned int	reg_offset;
 	struct regmap	*reg_base;
 	struct clk	*emmcclk;
+	unsigned int drive_impedance;
 };
 
 static int rockchip_emmc_phy_power(struct phy *phy, bool on_off)
@@ -281,10 +282,10 @@ static int rockchip_emmc_phy_power_on(struct phy *phy)
 {
 	struct rockchip_emmc_phy *rk_phy = phy_get_drvdata(phy);
 
-	/* Drive impedance: 50 Ohm */
+	/* Drive impedance: from DTS */
 	regmap_write(rk_phy->reg_base,
 		     rk_phy->reg_offset + GRF_EMMCPHY_CON6,
-		     HIWORD_UPDATE(PHYCTRL_DR_50OHM,
+		     HIWORD_UPDATE(rk_phy->drive_impedance,
 				   PHYCTRL_DR_MASK,
 				   PHYCTRL_DR_SHIFT));
 
@@ -314,6 +315,26 @@ static const struct phy_ops ops = {
 	.owner		= THIS_MODULE,
 };
 
+static u32 convert_drive_impedance_ohm(struct platform_device *pdev, u32 dr_ohm)
+{
+	switch (dr_ohm) {
+	case 100:
+		return PHYCTRL_DR_100OHM;
+	case 66:
+		return PHYCTRL_DR_66OHM;
+	case 50:
+		return PHYCTRL_DR_50OHM;
+	case 40:
+		return PHYCTRL_DR_40OHM;
+	case 33:
+		return PHYCTRL_DR_33OHM;
+	}
+
+	dev_warn(&pdev->dev, "Invalid value %u for drive-impedance-ohm.\n",
+		 dr_ohm);
+	return PHYCTRL_DR_50OHM;
+}
+
 static int rockchip_emmc_phy_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
@@ -322,6 +343,7 @@ static int rockchip_emmc_phy_probe(struct platform_device *pdev)
 	struct phy_provider *phy_provider;
 	struct regmap *grf;
 	unsigned int reg_offset;
+	u32 val;
 
 	if (!dev->parent || !dev->parent->of_node)
 		return -ENODEV;
@@ -344,6 +366,10 @@ static int rockchip_emmc_phy_probe(struct platform_device *pdev)
 
 	rk_phy->reg_offset = reg_offset;
 	rk_phy->reg_base = grf;
+	rk_phy->drive_impedance = PHYCTRL_DR_50OHM;
+
+	if (!of_property_read_u32(dev->of_node, "drive-impedance-ohm", &val))
+		rk_phy->drive_impedance = convert_drive_impedance_ohm(pdev, val);
 
 	generic_phy = devm_phy_create(dev, dev->of_node, &ops);
 	if (IS_ERR(generic_phy)) {
diff --git a/drivers/phy/socionext/phy-uniphier-usb3hs.c b/drivers/phy/socionext/phy-uniphier-usb3hs.c
index b1b048be6166..50f379fc4e06 100644
--- a/drivers/phy/socionext/phy-uniphier-usb3hs.c
+++ b/drivers/phy/socionext/phy-uniphier-usb3hs.c
@@ -335,13 +335,9 @@ static int uniphier_u3hsphy_probe(struct platform_device *pdev)
 	if (IS_ERR(priv->clk_parent))
 		return PTR_ERR(priv->clk_parent);
 
-	priv->clk_ext = devm_clk_get(dev, "phy-ext");
-	if (IS_ERR(priv->clk_ext)) {
-		if (PTR_ERR(priv->clk_ext) == -ENOENT)
-			priv->clk_ext = NULL;
-		else
-			return PTR_ERR(priv->clk_ext);
-	}
+	priv->clk_ext = devm_clk_get_optional(dev, "phy-ext");
+	if (IS_ERR(priv->clk_ext))
+		return PTR_ERR(priv->clk_ext);
 
 	priv->rst = devm_reset_control_get_shared(dev, "phy");
 	if (IS_ERR(priv->rst))
diff --git a/drivers/phy/socionext/phy-uniphier-usb3ss.c b/drivers/phy/socionext/phy-uniphier-usb3ss.c
index 4be95679c7d8..ec231e40ef2a 100644
--- a/drivers/phy/socionext/phy-uniphier-usb3ss.c
+++ b/drivers/phy/socionext/phy-uniphier-usb3ss.c
@@ -238,13 +238,9 @@ static int uniphier_u3ssphy_probe(struct platform_device *pdev)
 		if (IS_ERR(priv->clk))
 			return PTR_ERR(priv->clk);
 
-		priv->clk_ext = devm_clk_get(dev, "phy-ext");
-		if (IS_ERR(priv->clk_ext)) {
-			if (PTR_ERR(priv->clk_ext) == -ENOENT)
-				priv->clk_ext = NULL;
-			else
-				return PTR_ERR(priv->clk_ext);
-		}
+		priv->clk_ext = devm_clk_get_optional(dev, "phy-ext");
+		if (IS_ERR(priv->clk_ext))
+			return PTR_ERR(priv->clk_ext);
 
 		priv->rst = devm_reset_control_get_shared(dev, "phy");
 		if (IS_ERR(priv->rst))
diff --git a/drivers/phy/tegra/Makefile b/drivers/phy/tegra/Makefile
index 898589238fd9..a93cd9a499b2 100644
--- a/drivers/phy/tegra/Makefile
+++ b/drivers/phy/tegra/Makefile
@@ -4,3 +4,4 @@ phy-tegra-xusb-y += xusb.o
 phy-tegra-xusb-$(CONFIG_ARCH_TEGRA_124_SOC) += xusb-tegra124.o
 phy-tegra-xusb-$(CONFIG_ARCH_TEGRA_132_SOC) += xusb-tegra124.o
 phy-tegra-xusb-$(CONFIG_ARCH_TEGRA_210_SOC) += xusb-tegra210.o
+phy-tegra-xusb-$(CONFIG_ARCH_TEGRA_186_SOC) += xusb-tegra186.o
diff --git a/drivers/phy/tegra/xusb-tegra186.c b/drivers/phy/tegra/xusb-tegra186.c
new file mode 100644
index 000000000000..6f3afaf9398f
--- /dev/null
+++ b/drivers/phy/tegra/xusb-tegra186.c
@@ -0,0 +1,899 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2016-2019, NVIDIA CORPORATION.  All rights reserved.
+ */
+
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/phy/phy.h>
+#include <linux/regulator/consumer.h>
+#include <linux/platform_device.h>
+#include <linux/clk.h>
+#include <linux/slab.h>
+
+#include <soc/tegra/fuse.h>
+
+#include "xusb.h"
+
+/* FUSE USB_CALIB registers */
+#define HS_CURR_LEVEL_PADX_SHIFT(x)	((x) ? (11 + (x - 1) * 6) : 0)
+#define HS_CURR_LEVEL_PAD_MASK		0x3f
+#define HS_TERM_RANGE_ADJ_SHIFT		7
+#define HS_TERM_RANGE_ADJ_MASK		0xf
+#define HS_SQUELCH_SHIFT		29
+#define HS_SQUELCH_MASK			0x7
+
+#define RPD_CTRL_SHIFT			0
+#define RPD_CTRL_MASK			0x1f
+
+/* XUSB PADCTL registers */
+#define XUSB_PADCTL_USB2_PAD_MUX	0x4
+#define  USB2_PORT_SHIFT(x)		((x) * 2)
+#define  USB2_PORT_MASK			0x3
+#define   PORT_XUSB			1
+#define  HSIC_PORT_SHIFT(x)		((x) + 20)
+#define  HSIC_PORT_MASK			0x1
+#define   PORT_HSIC			0
+
+#define XUSB_PADCTL_USB2_PORT_CAP	0x8
+#define XUSB_PADCTL_SS_PORT_CAP		0xc
+#define  PORTX_CAP_SHIFT(x)		((x) * 4)
+#define  PORT_CAP_MASK			0x3
+#define   PORT_CAP_DISABLED		0x0
+#define   PORT_CAP_HOST			0x1
+#define   PORT_CAP_DEVICE		0x2
+#define   PORT_CAP_OTG			0x3
+
+#define XUSB_PADCTL_ELPG_PROGRAM		0x20
+#define  USB2_PORT_WAKE_INTERRUPT_ENABLE(x)		BIT(x)
+#define  USB2_PORT_WAKEUP_EVENT(x)			BIT((x) +  7)
+#define  SS_PORT_WAKE_INTERRUPT_ENABLE(x)		BIT((x) + 14)
+#define  SS_PORT_WAKEUP_EVENT(x)			BIT((x) + 21)
+#define  USB2_HSIC_PORT_WAKE_INTERRUPT_ENABLE(x)	BIT((x) + 28)
+#define  USB2_HSIC_PORT_WAKEUP_EVENT(x)			BIT((x) + 30)
+#define  ALL_WAKE_EVENTS						\
+	(USB2_PORT_WAKEUP_EVENT(0) | USB2_PORT_WAKEUP_EVENT(1) |	\
+	USB2_PORT_WAKEUP_EVENT(2) | SS_PORT_WAKEUP_EVENT(0) |		\
+	SS_PORT_WAKEUP_EVENT(1) | SS_PORT_WAKEUP_EVENT(2) |		\
+	USB2_HSIC_PORT_WAKEUP_EVENT(0))
+
+#define XUSB_PADCTL_ELPG_PROGRAM_1		0x24
+#define  SSPX_ELPG_CLAMP_EN(x)			BIT(0 + (x) * 3)
+#define  SSPX_ELPG_CLAMP_EN_EARLY(x)		BIT(1 + (x) * 3)
+#define  SSPX_ELPG_VCORE_DOWN(x)		BIT(2 + (x) * 3)
+
+#define XUSB_PADCTL_USB2_OTG_PADX_CTL0(x)	(0x88 + (x) * 0x40)
+#define  HS_CURR_LEVEL(x)			((x) & 0x3f)
+#define  TERM_SEL				BIT(25)
+#define  USB2_OTG_PD				BIT(26)
+#define  USB2_OTG_PD2				BIT(27)
+#define  USB2_OTG_PD2_OVRD_EN			BIT(28)
+#define  USB2_OTG_PD_ZI				BIT(29)
+
+#define XUSB_PADCTL_USB2_OTG_PADX_CTL1(x)	(0x8c + (x) * 0x40)
+#define  USB2_OTG_PD_DR				BIT(2)
+#define  TERM_RANGE_ADJ(x)			(((x) & 0xf) << 3)
+#define  RPD_CTRL(x)				(((x) & 0x1f) << 26)
+
+#define XUSB_PADCTL_USB2_BIAS_PAD_CTL0		0x284
+#define  BIAS_PAD_PD				BIT(11)
+#define  HS_SQUELCH_LEVEL(x)			(((x) & 0x7) << 0)
+
+#define XUSB_PADCTL_USB2_BIAS_PAD_CTL1		0x288
+#define  USB2_TRK_START_TIMER(x)		(((x) & 0x7f) << 12)
+#define  USB2_TRK_DONE_RESET_TIMER(x)		(((x) & 0x7f) << 19)
+#define  USB2_PD_TRK				BIT(26)
+
+#define XUSB_PADCTL_HSIC_PADX_CTL0(x)		(0x300 + (x) * 0x20)
+#define  HSIC_PD_TX_DATA0			BIT(1)
+#define  HSIC_PD_TX_STROBE			BIT(3)
+#define  HSIC_PD_RX_DATA0			BIT(4)
+#define  HSIC_PD_RX_STROBE			BIT(6)
+#define  HSIC_PD_ZI_DATA0			BIT(7)
+#define  HSIC_PD_ZI_STROBE			BIT(9)
+#define  HSIC_RPD_DATA0				BIT(13)
+#define  HSIC_RPD_STROBE			BIT(15)
+#define  HSIC_RPU_DATA0				BIT(16)
+#define  HSIC_RPU_STROBE			BIT(18)
+
+#define XUSB_PADCTL_HSIC_PAD_TRK_CTL0		0x340
+#define  HSIC_TRK_START_TIMER(x)		(((x) & 0x7f) << 5)
+#define  HSIC_TRK_DONE_RESET_TIMER(x)		(((x) & 0x7f) << 12)
+#define  HSIC_PD_TRK				BIT(19)
+
+#define USB2_VBUS_ID				0x360
+#define  VBUS_OVERRIDE				BIT(14)
+#define  ID_OVERRIDE(x)				(((x) & 0xf) << 18)
+#define  ID_OVERRIDE_FLOATING			ID_OVERRIDE(8)
+#define  ID_OVERRIDE_GROUNDED			ID_OVERRIDE(0)
+
+#define TEGRA186_LANE(_name, _offset, _shift, _mask, _type)		\
+	{								\
+		.name = _name,						\
+		.offset = _offset,					\
+		.shift = _shift,					\
+		.mask = _mask,						\
+		.num_funcs = ARRAY_SIZE(tegra186_##_type##_functions),	\
+		.funcs = tegra186_##_type##_functions,			\
+	}
+
+struct tegra_xusb_fuse_calibration {
+	u32 *hs_curr_level;
+	u32 hs_squelch;
+	u32 hs_term_range_adj;
+	u32 rpd_ctrl;
+};
+
+struct tegra186_xusb_padctl {
+	struct tegra_xusb_padctl base;
+
+	struct tegra_xusb_fuse_calibration calib;
+
+	/* UTMI bias and tracking */
+	struct clk *usb2_trk_clk;
+	unsigned int bias_pad_enable;
+};
+
+static inline struct tegra186_xusb_padctl *
+to_tegra186_xusb_padctl(struct tegra_xusb_padctl *padctl)
+{
+	return container_of(padctl, struct tegra186_xusb_padctl, base);
+}
+
+/* USB 2.0 UTMI PHY support */
+static struct tegra_xusb_lane *
+tegra186_usb2_lane_probe(struct tegra_xusb_pad *pad, struct device_node *np,
+			 unsigned int index)
+{
+	struct tegra_xusb_usb2_lane *usb2;
+	int err;
+
+	usb2 = kzalloc(sizeof(*usb2), GFP_KERNEL);
+	if (!usb2)
+		return ERR_PTR(-ENOMEM);
+
+	INIT_LIST_HEAD(&usb2->base.list);
+	usb2->base.soc = &pad->soc->lanes[index];
+	usb2->base.index = index;
+	usb2->base.pad = pad;
+	usb2->base.np = np;
+
+	err = tegra_xusb_lane_parse_dt(&usb2->base, np);
+	if (err < 0) {
+		kfree(usb2);
+		return ERR_PTR(err);
+	}
+
+	return &usb2->base;
+}
+
+static void tegra186_usb2_lane_remove(struct tegra_xusb_lane *lane)
+{
+	struct tegra_xusb_usb2_lane *usb2 = to_usb2_lane(lane);
+
+	kfree(usb2);
+}
+
+static const struct tegra_xusb_lane_ops tegra186_usb2_lane_ops = {
+	.probe = tegra186_usb2_lane_probe,
+	.remove = tegra186_usb2_lane_remove,
+};
+
+static void tegra186_utmi_bias_pad_power_on(struct tegra_xusb_padctl *padctl)
+{
+	struct tegra186_xusb_padctl *priv = to_tegra186_xusb_padctl(padctl);
+	struct device *dev = padctl->dev;
+	u32 value;
+	int err;
+
+	mutex_lock(&padctl->lock);
+
+	if (priv->bias_pad_enable++ > 0) {
+		mutex_unlock(&padctl->lock);
+		return;
+	}
+
+	err = clk_prepare_enable(priv->usb2_trk_clk);
+	if (err < 0)
+		dev_warn(dev, "failed to enable USB2 trk clock: %d\n", err);
+
+	value = padctl_readl(padctl, XUSB_PADCTL_USB2_BIAS_PAD_CTL1);
+	value &= ~USB2_TRK_START_TIMER(~0);
+	value |= USB2_TRK_START_TIMER(0x1e);
+	value &= ~USB2_TRK_DONE_RESET_TIMER(~0);
+	value |= USB2_TRK_DONE_RESET_TIMER(0xa);
+	padctl_writel(padctl, value, XUSB_PADCTL_USB2_BIAS_PAD_CTL1);
+
+	value = padctl_readl(padctl, XUSB_PADCTL_USB2_BIAS_PAD_CTL0);
+	value &= ~BIAS_PAD_PD;
+	value &= ~HS_SQUELCH_LEVEL(~0);
+	value |= HS_SQUELCH_LEVEL(priv->calib.hs_squelch);
+	padctl_writel(padctl, value, XUSB_PADCTL_USB2_BIAS_PAD_CTL0);
+
+	udelay(1);
+
+	value = padctl_readl(padctl, XUSB_PADCTL_USB2_BIAS_PAD_CTL1);
+	value &= ~USB2_PD_TRK;
+	padctl_writel(padctl, value, XUSB_PADCTL_USB2_BIAS_PAD_CTL1);
+
+	mutex_unlock(&padctl->lock);
+}
+
+static void tegra186_utmi_bias_pad_power_off(struct tegra_xusb_padctl *padctl)
+{
+	struct tegra186_xusb_padctl *priv = to_tegra186_xusb_padctl(padctl);
+	u32 value;
+
+	mutex_lock(&padctl->lock);
+
+	if (WARN_ON(priv->bias_pad_enable == 0)) {
+		mutex_unlock(&padctl->lock);
+		return;
+	}
+
+	if (--priv->bias_pad_enable > 0) {
+		mutex_unlock(&padctl->lock);
+		return;
+	}
+
+	value = padctl_readl(padctl, XUSB_PADCTL_USB2_BIAS_PAD_CTL1);
+	value |= USB2_PD_TRK;
+	padctl_writel(padctl, value, XUSB_PADCTL_USB2_BIAS_PAD_CTL1);
+
+	clk_disable_unprepare(priv->usb2_trk_clk);
+
+	mutex_unlock(&padctl->lock);
+}
+
+static void tegra_phy_xusb_utmi_pad_power_on(struct phy *phy)
+{
+	struct tegra_xusb_lane *lane = phy_get_drvdata(phy);
+	struct tegra_xusb_padctl *padctl = lane->pad->padctl;
+	struct tegra_xusb_usb2_port *port;
+	struct device *dev = padctl->dev;
+	unsigned int index = lane->index;
+	u32 value;
+
+	if (!phy)
+		return;
+
+	port = tegra_xusb_find_usb2_port(padctl, index);
+	if (!port) {
+		dev_err(dev, "no port found for USB2 lane %u\n", index);
+		return;
+	}
+
+	tegra186_utmi_bias_pad_power_on(padctl);
+
+	udelay(2);
+
+	value = padctl_readl(padctl, XUSB_PADCTL_USB2_OTG_PADX_CTL0(index));
+	value &= ~USB2_OTG_PD;
+	padctl_writel(padctl, value, XUSB_PADCTL_USB2_OTG_PADX_CTL0(index));
+
+	value = padctl_readl(padctl, XUSB_PADCTL_USB2_OTG_PADX_CTL1(index));
+	value &= ~USB2_OTG_PD_DR;
+	padctl_writel(padctl, value, XUSB_PADCTL_USB2_OTG_PADX_CTL1(index));
+}
+
+static void tegra_phy_xusb_utmi_pad_power_down(struct phy *phy)
+{
+	struct tegra_xusb_lane *lane = phy_get_drvdata(phy);
+	struct tegra_xusb_padctl *padctl = lane->pad->padctl;
+	unsigned int index = lane->index;
+	u32 value;
+
+	if (!phy)
+		return;
+
+	value = padctl_readl(padctl, XUSB_PADCTL_USB2_OTG_PADX_CTL0(index));
+	value |= USB2_OTG_PD;
+	padctl_writel(padctl, value, XUSB_PADCTL_USB2_OTG_PADX_CTL0(index));
+
+	value = padctl_readl(padctl, XUSB_PADCTL_USB2_OTG_PADX_CTL1(index));
+	value |= USB2_OTG_PD_DR;
+	padctl_writel(padctl, value, XUSB_PADCTL_USB2_OTG_PADX_CTL1(index));
+
+	udelay(2);
+
+	tegra186_utmi_bias_pad_power_off(padctl);
+}
+
+static int tegra186_utmi_phy_power_on(struct phy *phy)
+{
+	struct tegra_xusb_lane *lane = phy_get_drvdata(phy);
+	struct tegra_xusb_usb2_lane *usb2 = to_usb2_lane(lane);
+	struct tegra_xusb_padctl *padctl = lane->pad->padctl;
+	struct tegra186_xusb_padctl *priv = to_tegra186_xusb_padctl(padctl);
+	struct tegra_xusb_usb2_port *port;
+	unsigned int index = lane->index;
+	struct device *dev = padctl->dev;
+	u32 value;
+
+	port = tegra_xusb_find_usb2_port(padctl, index);
+	if (!port) {
+		dev_err(dev, "no port found for USB2 lane %u\n", index);
+		return -ENODEV;
+	}
+
+	value = padctl_readl(padctl, XUSB_PADCTL_USB2_PAD_MUX);
+	value &= ~(USB2_PORT_MASK << USB2_PORT_SHIFT(index));
+	value |= (PORT_XUSB << USB2_PORT_SHIFT(index));
+	padctl_writel(padctl, value, XUSB_PADCTL_USB2_PAD_MUX);
+
+	value = padctl_readl(padctl, XUSB_PADCTL_USB2_PORT_CAP);
+	value &= ~(PORT_CAP_MASK << PORTX_CAP_SHIFT(index));
+
+	if (port->mode == USB_DR_MODE_UNKNOWN)
+		value |= (PORT_CAP_DISABLED << PORTX_CAP_SHIFT(index));
+	else if (port->mode == USB_DR_MODE_PERIPHERAL)
+		value |= (PORT_CAP_DEVICE << PORTX_CAP_SHIFT(index));
+	else if (port->mode == USB_DR_MODE_HOST)
+		value |= (PORT_CAP_HOST << PORTX_CAP_SHIFT(index));
+	else if (port->mode == USB_DR_MODE_OTG)
+		value |= (PORT_CAP_OTG << PORTX_CAP_SHIFT(index));
+
+	padctl_writel(padctl, value, XUSB_PADCTL_USB2_PORT_CAP);
+
+	value = padctl_readl(padctl, XUSB_PADCTL_USB2_OTG_PADX_CTL0(index));
+	value &= ~USB2_OTG_PD_ZI;
+	value |= TERM_SEL;
+	value &= ~HS_CURR_LEVEL(~0);
+
+	if (usb2->hs_curr_level_offset) {
+		int hs_current_level;
+
+		hs_current_level = (int)priv->calib.hs_curr_level[index] +
+						usb2->hs_curr_level_offset;
+
+		if (hs_current_level < 0)
+			hs_current_level = 0;
+		if (hs_current_level > 0x3f)
+			hs_current_level = 0x3f;
+
+		value |= HS_CURR_LEVEL(hs_current_level);
+	} else {
+		value |= HS_CURR_LEVEL(priv->calib.hs_curr_level[index]);
+	}
+
+	padctl_writel(padctl, value, XUSB_PADCTL_USB2_OTG_PADX_CTL0(index));
+
+	value = padctl_readl(padctl, XUSB_PADCTL_USB2_OTG_PADX_CTL1(index));
+	value &= ~TERM_RANGE_ADJ(~0);
+	value |= TERM_RANGE_ADJ(priv->calib.hs_term_range_adj);
+	value &= ~RPD_CTRL(~0);
+	value |= RPD_CTRL(priv->calib.rpd_ctrl);
+	padctl_writel(padctl, value, XUSB_PADCTL_USB2_OTG_PADX_CTL1(index));
+
+	/* TODO: pad power saving */
+	tegra_phy_xusb_utmi_pad_power_on(phy);
+	return 0;
+}
+
+static int tegra186_utmi_phy_power_off(struct phy *phy)
+{
+	/* TODO: pad power saving */
+	tegra_phy_xusb_utmi_pad_power_down(phy);
+
+	return 0;
+}
+
+static int tegra186_utmi_phy_init(struct phy *phy)
+{
+	struct tegra_xusb_lane *lane = phy_get_drvdata(phy);
+	struct tegra_xusb_padctl *padctl = lane->pad->padctl;
+	struct tegra_xusb_usb2_port *port;
+	unsigned int index = lane->index;
+	struct device *dev = padctl->dev;
+	int err;
+
+	port = tegra_xusb_find_usb2_port(padctl, index);
+	if (!port) {
+		dev_err(dev, "no port found for USB2 lane %u\n", index);
+		return -ENODEV;
+	}
+
+	if (port->supply && port->mode == USB_DR_MODE_HOST) {
+		err = regulator_enable(port->supply);
+		if (err) {
+			dev_err(dev, "failed to enable port %u VBUS: %d\n",
+				index, err);
+			return err;
+		}
+	}
+
+	return 0;
+}
+
+static int tegra186_utmi_phy_exit(struct phy *phy)
+{
+	struct tegra_xusb_lane *lane = phy_get_drvdata(phy);
+	struct tegra_xusb_padctl *padctl = lane->pad->padctl;
+	struct tegra_xusb_usb2_port *port;
+	unsigned int index = lane->index;
+	struct device *dev = padctl->dev;
+	int err;
+
+	port = tegra_xusb_find_usb2_port(padctl, index);
+	if (!port) {
+		dev_err(dev, "no port found for USB2 lane %u\n", index);
+		return -ENODEV;
+	}
+
+	if (port->supply && port->mode == USB_DR_MODE_HOST) {
+		err = regulator_disable(port->supply);
+		if (err) {
+			dev_err(dev, "failed to disable port %u VBUS: %d\n",
+				index, err);
+			return err;
+		}
+	}
+
+	return 0;
+}
+
+static const struct phy_ops utmi_phy_ops = {
+	.init = tegra186_utmi_phy_init,
+	.exit = tegra186_utmi_phy_exit,
+	.power_on = tegra186_utmi_phy_power_on,
+	.power_off = tegra186_utmi_phy_power_off,
+	.owner = THIS_MODULE,
+};
+
+static struct tegra_xusb_pad *
+tegra186_usb2_pad_probe(struct tegra_xusb_padctl *padctl,
+			const struct tegra_xusb_pad_soc *soc,
+			struct device_node *np)
+{
+	struct tegra186_xusb_padctl *priv = to_tegra186_xusb_padctl(padctl);
+	struct tegra_xusb_usb2_pad *usb2;
+	struct tegra_xusb_pad *pad;
+	int err;
+
+	usb2 = kzalloc(sizeof(*usb2), GFP_KERNEL);
+	if (!usb2)
+		return ERR_PTR(-ENOMEM);
+
+	pad = &usb2->base;
+	pad->ops = &tegra186_usb2_lane_ops;
+	pad->soc = soc;
+
+	err = tegra_xusb_pad_init(pad, padctl, np);
+	if (err < 0) {
+		kfree(usb2);
+		goto out;
+	}
+
+	priv->usb2_trk_clk = devm_clk_get(&pad->dev, "trk");
+	if (IS_ERR(priv->usb2_trk_clk)) {
+		err = PTR_ERR(priv->usb2_trk_clk);
+		dev_dbg(&pad->dev, "failed to get usb2 trk clock: %d\n", err);
+		goto unregister;
+	}
+
+	err = tegra_xusb_pad_register(pad, &utmi_phy_ops);
+	if (err < 0)
+		goto unregister;
+
+	dev_set_drvdata(&pad->dev, pad);
+
+	return pad;
+
+unregister:
+	device_unregister(&pad->dev);
+out:
+	return ERR_PTR(err);
+}
+
+static void tegra186_usb2_pad_remove(struct tegra_xusb_pad *pad)
+{
+	struct tegra_xusb_usb2_pad *usb2 = to_usb2_pad(pad);
+
+	kfree(usb2);
+}
+
+static const struct tegra_xusb_pad_ops tegra186_usb2_pad_ops = {
+	.probe = tegra186_usb2_pad_probe,
+	.remove = tegra186_usb2_pad_remove,
+};
+
+static const char * const tegra186_usb2_functions[] = {
+	"xusb",
+};
+
+static const struct tegra_xusb_lane_soc tegra186_usb2_lanes[] = {
+	TEGRA186_LANE("usb2-0", 0,  0, 0, usb2),
+	TEGRA186_LANE("usb2-1", 0,  0, 0, usb2),
+	TEGRA186_LANE("usb2-2", 0,  0, 0, usb2),
+};
+
+static const struct tegra_xusb_pad_soc tegra186_usb2_pad = {
+	.name = "usb2",
+	.num_lanes = ARRAY_SIZE(tegra186_usb2_lanes),
+	.lanes = tegra186_usb2_lanes,
+	.ops = &tegra186_usb2_pad_ops,
+};
+
+static int tegra186_usb2_port_enable(struct tegra_xusb_port *port)
+{
+	return 0;
+}
+
+static void tegra186_usb2_port_disable(struct tegra_xusb_port *port)
+{
+}
+
+static struct tegra_xusb_lane *
+tegra186_usb2_port_map(struct tegra_xusb_port *port)
+{
+	return tegra_xusb_find_lane(port->padctl, "usb2", port->index);
+}
+
+static const struct tegra_xusb_port_ops tegra186_usb2_port_ops = {
+	.enable = tegra186_usb2_port_enable,
+	.disable = tegra186_usb2_port_disable,
+	.map = tegra186_usb2_port_map,
+};
+
+/* SuperSpeed PHY support */
+static struct tegra_xusb_lane *
+tegra186_usb3_lane_probe(struct tegra_xusb_pad *pad, struct device_node *np,
+			 unsigned int index)
+{
+	struct tegra_xusb_usb3_lane *usb3;
+	int err;
+
+	usb3 = kzalloc(sizeof(*usb3), GFP_KERNEL);
+	if (!usb3)
+		return ERR_PTR(-ENOMEM);
+
+	INIT_LIST_HEAD(&usb3->base.list);
+	usb3->base.soc = &pad->soc->lanes[index];
+	usb3->base.index = index;
+	usb3->base.pad = pad;
+	usb3->base.np = np;
+
+	err = tegra_xusb_lane_parse_dt(&usb3->base, np);
+	if (err < 0) {
+		kfree(usb3);
+		return ERR_PTR(err);
+	}
+
+	return &usb3->base;
+}
+
+static void tegra186_usb3_lane_remove(struct tegra_xusb_lane *lane)
+{
+	struct tegra_xusb_usb3_lane *usb3 = to_usb3_lane(lane);
+
+	kfree(usb3);
+}
+
+static const struct tegra_xusb_lane_ops tegra186_usb3_lane_ops = {
+	.probe = tegra186_usb3_lane_probe,
+	.remove = tegra186_usb3_lane_remove,
+};
+static int tegra186_usb3_port_enable(struct tegra_xusb_port *port)
+{
+	return 0;
+}
+
+static void tegra186_usb3_port_disable(struct tegra_xusb_port *port)
+{
+}
+
+static struct tegra_xusb_lane *
+tegra186_usb3_port_map(struct tegra_xusb_port *port)
+{
+	return tegra_xusb_find_lane(port->padctl, "usb3", port->index);
+}
+
+static const struct tegra_xusb_port_ops tegra186_usb3_port_ops = {
+	.enable = tegra186_usb3_port_enable,
+	.disable = tegra186_usb3_port_disable,
+	.map = tegra186_usb3_port_map,
+};
+
+static int tegra186_usb3_phy_power_on(struct phy *phy)
+{
+	struct tegra_xusb_lane *lane = phy_get_drvdata(phy);
+	struct tegra_xusb_padctl *padctl = lane->pad->padctl;
+	struct tegra_xusb_usb3_port *port;
+	struct tegra_xusb_usb2_port *usb2;
+	unsigned int index = lane->index;
+	struct device *dev = padctl->dev;
+	u32 value;
+
+	port = tegra_xusb_find_usb3_port(padctl, index);
+	if (!port) {
+		dev_err(dev, "no port found for USB3 lane %u\n", index);
+		return -ENODEV;
+	}
+
+	usb2 = tegra_xusb_find_usb2_port(padctl, port->port);
+	if (!usb2) {
+		dev_err(dev, "no companion port found for USB3 lane %u\n",
+			index);
+		return -ENODEV;
+	}
+
+	mutex_lock(&padctl->lock);
+
+	value = padctl_readl(padctl, XUSB_PADCTL_SS_PORT_CAP);
+	value &= ~(PORT_CAP_MASK << PORTX_CAP_SHIFT(index));
+
+	if (usb2->mode == USB_DR_MODE_UNKNOWN)
+		value |= (PORT_CAP_DISABLED << PORTX_CAP_SHIFT(index));
+	else if (usb2->mode == USB_DR_MODE_PERIPHERAL)
+		value |= (PORT_CAP_DEVICE << PORTX_CAP_SHIFT(index));
+	else if (usb2->mode == USB_DR_MODE_HOST)
+		value |= (PORT_CAP_HOST << PORTX_CAP_SHIFT(index));
+	else if (usb2->mode == USB_DR_MODE_OTG)
+		value |= (PORT_CAP_OTG << PORTX_CAP_SHIFT(index));
+
+	padctl_writel(padctl, value, XUSB_PADCTL_SS_PORT_CAP);
+
+	value = padctl_readl(padctl, XUSB_PADCTL_ELPG_PROGRAM_1);
+	value &= ~SSPX_ELPG_VCORE_DOWN(index);
+	padctl_writel(padctl, value, XUSB_PADCTL_ELPG_PROGRAM_1);
+
+	usleep_range(100, 200);
+
+	value = padctl_readl(padctl, XUSB_PADCTL_ELPG_PROGRAM_1);
+	value &= ~SSPX_ELPG_CLAMP_EN_EARLY(index);
+	padctl_writel(padctl, value, XUSB_PADCTL_ELPG_PROGRAM_1);
+
+	usleep_range(100, 200);
+
+	value = padctl_readl(padctl, XUSB_PADCTL_ELPG_PROGRAM_1);
+	value &= ~SSPX_ELPG_CLAMP_EN(index);
+	padctl_writel(padctl, value, XUSB_PADCTL_ELPG_PROGRAM_1);
+
+	mutex_unlock(&padctl->lock);
+
+	return 0;
+}
+
+static int tegra186_usb3_phy_power_off(struct phy *phy)
+{
+	struct tegra_xusb_lane *lane = phy_get_drvdata(phy);
+	struct tegra_xusb_padctl *padctl = lane->pad->padctl;
+	struct tegra_xusb_usb3_port *port;
+	unsigned int index = lane->index;
+	struct device *dev = padctl->dev;
+	u32 value;
+
+	port = tegra_xusb_find_usb3_port(padctl, index);
+	if (!port) {
+		dev_err(dev, "no port found for USB3 lane %u\n", index);
+		return -ENODEV;
+	}
+
+	mutex_lock(&padctl->lock);
+
+	value = padctl_readl(padctl, XUSB_PADCTL_ELPG_PROGRAM_1);
+	value |= SSPX_ELPG_CLAMP_EN_EARLY(index);
+	padctl_writel(padctl, value, XUSB_PADCTL_ELPG_PROGRAM_1);
+
+	usleep_range(100, 200);
+
+	value = padctl_readl(padctl, XUSB_PADCTL_ELPG_PROGRAM_1);
+	value |= SSPX_ELPG_CLAMP_EN(index);
+	padctl_writel(padctl, value, XUSB_PADCTL_ELPG_PROGRAM_1);
+
+	usleep_range(250, 350);
+
+	value = padctl_readl(padctl, XUSB_PADCTL_ELPG_PROGRAM_1);
+	value |= SSPX_ELPG_VCORE_DOWN(index);
+	padctl_writel(padctl, value, XUSB_PADCTL_ELPG_PROGRAM_1);
+
+	mutex_unlock(&padctl->lock);
+
+	return 0;
+}
+
+static int tegra186_usb3_phy_init(struct phy *phy)
+{
+	return 0;
+}
+
+static int tegra186_usb3_phy_exit(struct phy *phy)
+{
+	return 0;
+}
+
+static const struct phy_ops usb3_phy_ops = {
+	.init = tegra186_usb3_phy_init,
+	.exit = tegra186_usb3_phy_exit,
+	.power_on = tegra186_usb3_phy_power_on,
+	.power_off = tegra186_usb3_phy_power_off,
+	.owner = THIS_MODULE,
+};
+
+static struct tegra_xusb_pad *
+tegra186_usb3_pad_probe(struct tegra_xusb_padctl *padctl,
+			const struct tegra_xusb_pad_soc *soc,
+			struct device_node *np)
+{
+	struct tegra_xusb_usb3_pad *usb3;
+	struct tegra_xusb_pad *pad;
+	int err;
+
+	usb3 = kzalloc(sizeof(*usb3), GFP_KERNEL);
+	if (!usb3)
+		return ERR_PTR(-ENOMEM);
+
+	pad = &usb3->base;
+	pad->ops = &tegra186_usb3_lane_ops;
+	pad->soc = soc;
+
+	err = tegra_xusb_pad_init(pad, padctl, np);
+	if (err < 0) {
+		kfree(usb3);
+		goto out;
+	}
+
+	err = tegra_xusb_pad_register(pad, &usb3_phy_ops);
+	if (err < 0)
+		goto unregister;
+
+	dev_set_drvdata(&pad->dev, pad);
+
+	return pad;
+
+unregister:
+	device_unregister(&pad->dev);
+out:
+	return ERR_PTR(err);
+}
+
+static void tegra186_usb3_pad_remove(struct tegra_xusb_pad *pad)
+{
+	struct tegra_xusb_usb2_pad *usb2 = to_usb2_pad(pad);
+
+	kfree(usb2);
+}
+
+static const struct tegra_xusb_pad_ops tegra186_usb3_pad_ops = {
+	.probe = tegra186_usb3_pad_probe,
+	.remove = tegra186_usb3_pad_remove,
+};
+
+static const char * const tegra186_usb3_functions[] = {
+	"xusb",
+};
+
+static const struct tegra_xusb_lane_soc tegra186_usb3_lanes[] = {
+	TEGRA186_LANE("usb3-0", 0,  0, 0, usb3),
+	TEGRA186_LANE("usb3-1", 0,  0, 0, usb3),
+	TEGRA186_LANE("usb3-2", 0,  0, 0, usb3),
+};
+
+static const struct tegra_xusb_pad_soc tegra186_usb3_pad = {
+	.name = "usb3",
+	.num_lanes = ARRAY_SIZE(tegra186_usb3_lanes),
+	.lanes = tegra186_usb3_lanes,
+	.ops = &tegra186_usb3_pad_ops,
+};
+
+static const struct tegra_xusb_pad_soc * const tegra186_pads[] = {
+	&tegra186_usb2_pad,
+	&tegra186_usb3_pad,
+#if 0 /* TODO implement */
+	&tegra186_hsic_pad,
+#endif
+};
+
+static int
+tegra186_xusb_read_fuse_calibration(struct tegra186_xusb_padctl *padctl)
+{
+	struct device *dev = padctl->base.dev;
+	unsigned int i, count;
+	u32 value, *level;
+	int err;
+
+	count = padctl->base.soc->ports.usb2.count;
+
+	level = devm_kcalloc(dev, count, sizeof(u32), GFP_KERNEL);
+	if (!level)
+		return -ENOMEM;
+
+	err = tegra_fuse_readl(TEGRA_FUSE_SKU_CALIB_0, &value);
+	if (err) {
+		dev_err(dev, "failed to read calibration fuse: %d\n", err);
+		return err;
+	}
+
+	dev_dbg(dev, "FUSE_USB_CALIB_0 %#x\n", value);
+
+	for (i = 0; i < count; i++)
+		level[i] = (value >> HS_CURR_LEVEL_PADX_SHIFT(i)) &
+				HS_CURR_LEVEL_PAD_MASK;
+
+	padctl->calib.hs_curr_level = level;
+
+	padctl->calib.hs_squelch = (value >> HS_SQUELCH_SHIFT) &
+					HS_SQUELCH_MASK;
+	padctl->calib.hs_term_range_adj = (value >> HS_TERM_RANGE_ADJ_SHIFT) &
+						HS_TERM_RANGE_ADJ_MASK;
+
+	err = tegra_fuse_readl(TEGRA_FUSE_USB_CALIB_EXT_0, &value);
+	if (err) {
+		dev_err(dev, "failed to read calibration fuse: %d\n", err);
+		return err;
+	}
+
+	dev_dbg(dev, "FUSE_USB_CALIB_EXT_0 %#x\n", value);
+
+	padctl->calib.rpd_ctrl = (value >> RPD_CTRL_SHIFT) & RPD_CTRL_MASK;
+
+	return 0;
+}
+
+static struct tegra_xusb_padctl *
+tegra186_xusb_padctl_probe(struct device *dev,
+			   const struct tegra_xusb_padctl_soc *soc)
+{
+	struct tegra186_xusb_padctl *priv;
+	int err;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return ERR_PTR(-ENOMEM);
+
+	priv->base.dev = dev;
+	priv->base.soc = soc;
+
+	err = tegra186_xusb_read_fuse_calibration(priv);
+	if (err < 0)
+		return ERR_PTR(err);
+
+	return &priv->base;
+}
+
+static void tegra186_xusb_padctl_remove(struct tegra_xusb_padctl *padctl)
+{
+}
+
+static const struct tegra_xusb_padctl_ops tegra186_xusb_padctl_ops = {
+	.probe = tegra186_xusb_padctl_probe,
+	.remove = tegra186_xusb_padctl_remove,
+};
+
+static const char * const tegra186_xusb_padctl_supply_names[] = {
+	"avdd-pll-erefeut",
+	"avdd-usb",
+	"vclamp-usb",
+	"vddio-hsic",
+};
+
+const struct tegra_xusb_padctl_soc tegra186_xusb_padctl_soc = {
+	.num_pads = ARRAY_SIZE(tegra186_pads),
+	.pads = tegra186_pads,
+	.ports = {
+		.usb2 = {
+			.ops = &tegra186_usb2_port_ops,
+			.count = 3,
+		},
+#if 0 /* TODO implement */
+		.hsic = {
+			.ops = &tegra186_hsic_port_ops,
+			.count = 1,
+		},
+#endif
+		.usb3 = {
+			.ops = &tegra186_usb3_port_ops,
+			.count = 3,
+		},
+	},
+	.ops = &tegra186_xusb_padctl_ops,
+	.supply_names = tegra186_xusb_padctl_supply_names,
+	.num_supplies = ARRAY_SIZE(tegra186_xusb_padctl_supply_names),
+};
+EXPORT_SYMBOL_GPL(tegra186_xusb_padctl_soc);
+
+MODULE_AUTHOR("JC Kuo <jckuo@nvidia.com>");
+MODULE_DESCRIPTION("NVIDIA Tegra186 XUSB Pad Controller driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/phy/tegra/xusb.c b/drivers/phy/tegra/xusb.c
index 5b3b8863363e..0417213ed68b 100644
--- a/drivers/phy/tegra/xusb.c
+++ b/drivers/phy/tegra/xusb.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2015, NVIDIA CORPORATION.  All rights reserved.
+ * Copyright (c) 2014-2016, NVIDIA CORPORATION.  All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -68,6 +68,12 @@ static const struct of_device_id tegra_xusb_padctl_of_match[] = {
 		.data = &tegra210_xusb_padctl_soc,
 	},
 #endif
+#if defined(CONFIG_ARCH_TEGRA_186_SOC)
+	{
+		.compatible = "nvidia,tegra186-xusb-padctl",
+		.data = &tegra186_xusb_padctl_soc,
+	},
+#endif
 	{ }
 };
 MODULE_DEVICE_TABLE(of, tegra_xusb_padctl_of_match);
@@ -313,6 +319,10 @@ static void tegra_xusb_lane_program(struct tegra_xusb_lane *lane)
 	const struct tegra_xusb_lane_soc *soc = lane->soc;
 	u32 value;
 
+	/* skip single function lanes */
+	if (soc->num_funcs < 2)
+		return;
+
 	/* choose function */
 	value = padctl_readl(padctl, soc->offset);
 	value &= ~(soc->mask << soc->shift);
@@ -542,13 +552,34 @@ static void tegra_xusb_port_unregister(struct tegra_xusb_port *port)
 	device_unregister(&port->dev);
 }
 
+static const char *const modes[] = {
+	[USB_DR_MODE_UNKNOWN] = "",
+	[USB_DR_MODE_HOST] = "host",
+	[USB_DR_MODE_PERIPHERAL] = "peripheral",
+	[USB_DR_MODE_OTG] = "otg",
+};
+
 static int tegra_xusb_usb2_port_parse_dt(struct tegra_xusb_usb2_port *usb2)
 {
 	struct tegra_xusb_port *port = &usb2->base;
 	struct device_node *np = port->dev.of_node;
+	const char *mode;
 
 	usb2->internal = of_property_read_bool(np, "nvidia,internal");
 
+	if (!of_property_read_string(np, "mode", &mode)) {
+		int err = match_string(modes, ARRAY_SIZE(modes), mode);
+		if (err < 0) {
+			dev_err(&port->dev, "invalid value %s for \"mode\"\n",
+				mode);
+			usb2->mode = USB_DR_MODE_UNKNOWN;
+		} else {
+			usb2->mode = err;
+		}
+	} else {
+		usb2->mode = USB_DR_MODE_HOST;
+	}
+
 	usb2->supply = devm_regulator_get(&port->dev, "vbus");
 	return PTR_ERR_OR_ZERO(usb2->supply);
 }
@@ -839,6 +870,7 @@ static int tegra_xusb_padctl_probe(struct platform_device *pdev)
 	struct tegra_xusb_padctl *padctl;
 	const struct of_device_id *match;
 	struct resource *res;
+	unsigned int i;
 	int err;
 
 	/* for backwards compatibility with old device trees */
@@ -876,14 +908,38 @@ static int tegra_xusb_padctl_probe(struct platform_device *pdev)
 		goto remove;
 	}
 
+	padctl->supplies = devm_kcalloc(&pdev->dev, padctl->soc->num_supplies,
+					sizeof(*padctl->supplies), GFP_KERNEL);
+	if (!padctl->supplies) {
+		err = -ENOMEM;
+		goto remove;
+	}
+
+	for (i = 0; i < padctl->soc->num_supplies; i++)
+		padctl->supplies[i].supply = padctl->soc->supply_names[i];
+
+	err = devm_regulator_bulk_get(&pdev->dev, padctl->soc->num_supplies,
+				      padctl->supplies);
+	if (err < 0) {
+		dev_err(&pdev->dev, "failed to get regulators: %d\n", err);
+		goto remove;
+	}
+
 	err = reset_control_deassert(padctl->rst);
 	if (err < 0)
 		goto remove;
 
+	err = regulator_bulk_enable(padctl->soc->num_supplies,
+				    padctl->supplies);
+	if (err < 0) {
+		dev_err(&pdev->dev, "failed to enable supplies: %d\n", err);
+		goto reset;
+	}
+
 	err = tegra_xusb_setup_pads(padctl);
 	if (err < 0) {
 		dev_err(&pdev->dev, "failed to setup pads: %d\n", err);
-		goto reset;
+		goto power_down;
 	}
 
 	err = tegra_xusb_setup_ports(padctl);
@@ -896,6 +952,8 @@ static int tegra_xusb_padctl_probe(struct platform_device *pdev)
 
 remove_pads:
 	tegra_xusb_remove_pads(padctl);
+power_down:
+	regulator_bulk_disable(padctl->soc->num_supplies, padctl->supplies);
 reset:
 	reset_control_assert(padctl->rst);
 remove:
@@ -911,6 +969,11 @@ static int tegra_xusb_padctl_remove(struct platform_device *pdev)
 	tegra_xusb_remove_ports(padctl);
 	tegra_xusb_remove_pads(padctl);
 
+	err = regulator_bulk_disable(padctl->soc->num_supplies,
+				     padctl->supplies);
+	if (err < 0)
+		dev_err(&pdev->dev, "failed to disable supplies: %d\n", err);
+
 	err = reset_control_assert(padctl->rst);
 	if (err < 0)
 		dev_err(&pdev->dev, "failed to assert reset: %d\n", err);
diff --git a/drivers/phy/tegra/xusb.h b/drivers/phy/tegra/xusb.h
index b49dbc36efa3..e0028b9fe702 100644
--- a/drivers/phy/tegra/xusb.h
+++ b/drivers/phy/tegra/xusb.h
@@ -19,6 +19,8 @@
 #include <linux/mutex.h>
 #include <linux/workqueue.h>
 
+#include <linux/usb/otg.h>
+
 /* legacy entry points for backwards-compatibility */
 int tegra_xusb_padctl_legacy_probe(struct platform_device *pdev);
 int tegra_xusb_padctl_legacy_remove(struct platform_device *pdev);
@@ -54,10 +56,21 @@ struct tegra_xusb_lane {
 int tegra_xusb_lane_parse_dt(struct tegra_xusb_lane *lane,
 			     struct device_node *np);
 
+struct tegra_xusb_usb3_lane {
+	struct tegra_xusb_lane base;
+};
+
+static inline struct tegra_xusb_usb3_lane *
+to_usb3_lane(struct tegra_xusb_lane *lane)
+{
+	return container_of(lane, struct tegra_xusb_usb3_lane, base);
+}
+
 struct tegra_xusb_usb2_lane {
 	struct tegra_xusb_lane base;
 
 	u32 hs_curr_level_offset;
+	bool powered_on;
 };
 
 static inline struct tegra_xusb_usb2_lane *
@@ -168,6 +181,19 @@ int tegra_xusb_pad_register(struct tegra_xusb_pad *pad,
 			    const struct phy_ops *ops);
 void tegra_xusb_pad_unregister(struct tegra_xusb_pad *pad);
 
+struct tegra_xusb_usb3_pad {
+	struct tegra_xusb_pad base;
+
+	unsigned int enable;
+	struct mutex lock;
+};
+
+static inline struct tegra_xusb_usb3_pad *
+to_usb3_pad(struct tegra_xusb_pad *pad)
+{
+	return container_of(pad, struct tegra_xusb_usb3_pad, base);
+}
+
 struct tegra_xusb_usb2_pad {
 	struct tegra_xusb_pad base;
 
@@ -271,6 +297,7 @@ struct tegra_xusb_usb2_port {
 	struct tegra_xusb_port base;
 
 	struct regulator *supply;
+	enum usb_dr_mode mode;
 	bool internal;
 };
 
@@ -367,6 +394,9 @@ struct tegra_xusb_padctl_soc {
 	} ports;
 
 	const struct tegra_xusb_padctl_ops *ops;
+
+	const char * const *supply_names;
+	unsigned int num_supplies;
 };
 
 struct tegra_xusb_padctl {
@@ -390,6 +420,8 @@ struct tegra_xusb_padctl {
 	unsigned int enable;
 
 	struct clk *clk;
+
+	struct regulator_bulk_data *supplies;
 };
 
 static inline void padctl_writel(struct tegra_xusb_padctl *padctl, u32 value,
@@ -417,5 +449,8 @@ extern const struct tegra_xusb_padctl_soc tegra124_xusb_padctl_soc;
 #if defined(CONFIG_ARCH_TEGRA_210_SOC)
 extern const struct tegra_xusb_padctl_soc tegra210_xusb_padctl_soc;
 #endif
+#if defined(CONFIG_ARCH_TEGRA_186_SOC)
+extern const struct tegra_xusb_padctl_soc tegra186_xusb_padctl_soc;
+#endif
 
 #endif /* __PHY_TEGRA_XUSB_H */
diff --git a/drivers/phy/ti/Kconfig b/drivers/phy/ti/Kconfig
index 022ac16f626c..781514efded3 100644
--- a/drivers/phy/ti/Kconfig
+++ b/drivers/phy/ti/Kconfig
@@ -20,6 +20,18 @@ config PHY_DM816X_USB
 	help
 	  Enable this for dm816x USB to work.
 
+config PHY_AM654_SERDES
+	tristate "TI AM654 SERDES support"
+	depends on OF && ARCH_K3 || COMPILE_TEST
+	depends on COMMON_CLK
+	select GENERIC_PHY
+	select MULTIPLEXER
+	select REGMAP_MMIO
+	select MUX_MMIO
+	help
+	  This option enables support for TI AM654 SerDes PHY used for
+	  PCIe.
+
 config OMAP_CONTROL_PHY
 	tristate "OMAP CONTROL PHY Driver"
 	depends on ARCH_OMAP2PLUS || COMPILE_TEST
diff --git a/drivers/phy/ti/Makefile b/drivers/phy/ti/Makefile
index bea8f25a137a..bff901eb0ecc 100644
--- a/drivers/phy/ti/Makefile
+++ b/drivers/phy/ti/Makefile
@@ -6,4 +6,5 @@ obj-$(CONFIG_OMAP_USB2)			+= phy-omap-usb2.o
 obj-$(CONFIG_TI_PIPE3)			+= phy-ti-pipe3.o
 obj-$(CONFIG_PHY_TUSB1210)		+= phy-tusb1210.o
 obj-$(CONFIG_TWL4030_USB)		+= phy-twl4030-usb.o
+obj-$(CONFIG_PHY_AM654_SERDES)		+= phy-am654-serdes.o
 obj-$(CONFIG_PHY_TI_GMII_SEL)		+= phy-gmii-sel.o
diff --git a/drivers/phy/ti/phy-am654-serdes.c b/drivers/phy/ti/phy-am654-serdes.c
new file mode 100644
index 000000000000..d3769200cb9b
--- /dev/null
+++ b/drivers/phy/ti/phy-am654-serdes.c
@@ -0,0 +1,658 @@
+// SPDX-License-Identifier: GPL-2.0
+/**
+ * PCIe SERDES driver for AM654x SoC
+ *
+ * Copyright (C) 2018 - 2019 Texas Instruments Incorporated - http://www.ti.com/
+ * Author: Kishon Vijay Abraham I <kishon@ti.com>
+ */
+
+#include <dt-bindings/phy/phy.h>
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/delay.h>
+#include <linux/module.h>
+#include <linux/mfd/syscon.h>
+#include <linux/mux/consumer.h>
+#include <linux/of_address.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/regmap.h>
+
+#define CMU_R07C		0x7c
+
+#define COMLANE_R138		0xb38
+#define VERSION			0x70
+
+#define COMLANE_R190		0xb90
+
+#define COMLANE_R194		0xb94
+
+#define SERDES_CTRL		0x1fd0
+
+#define WIZ_LANEXCTL_STS	0x1fe0
+#define TX0_DISABLE_STATE	0x4
+#define TX0_SLEEP_STATE		0x5
+#define TX0_SNOOZE_STATE	0x6
+#define TX0_ENABLE_STATE	0x7
+
+#define RX0_DISABLE_STATE	0x4
+#define RX0_SLEEP_STATE		0x5
+#define RX0_SNOOZE_STATE	0x6
+#define RX0_ENABLE_STATE	0x7
+
+#define WIZ_PLL_CTRL		0x1ff4
+#define PLL_DISABLE_STATE	0x4
+#define PLL_SLEEP_STATE		0x5
+#define PLL_SNOOZE_STATE	0x6
+#define PLL_ENABLE_STATE	0x7
+
+#define PLL_LOCK_TIME		100000	/* in microseconds */
+#define SLEEP_TIME		100	/* in microseconds */
+
+#define LANE_USB3		0x0
+#define LANE_PCIE0_LANE0	0x1
+
+#define LANE_PCIE1_LANE0	0x0
+#define LANE_PCIE0_LANE1	0x1
+
+#define SERDES_NUM_CLOCKS	3
+
+#define AM654_SERDES_CTRL_CLKSEL_MASK	GENMASK(7, 4)
+#define AM654_SERDES_CTRL_CLKSEL_SHIFT	4
+
+struct serdes_am654_clk_mux {
+	struct clk_hw	hw;
+	struct regmap	*regmap;
+	unsigned int	reg;
+	int		clk_id;
+	struct clk_init_data clk_data;
+};
+
+#define to_serdes_am654_clk_mux(_hw)	\
+		container_of(_hw, struct serdes_am654_clk_mux, hw)
+
+static struct regmap_config serdes_am654_regmap_config = {
+	.reg_bits = 32,
+	.val_bits = 32,
+	.reg_stride = 4,
+	.fast_io = true,
+};
+
+static const struct reg_field cmu_master_cdn_o = REG_FIELD(CMU_R07C, 24, 24);
+static const struct reg_field config_version = REG_FIELD(COMLANE_R138, 16, 23);
+static const struct reg_field l1_master_cdn_o = REG_FIELD(COMLANE_R190, 9, 9);
+static const struct reg_field cmu_ok_i_0 = REG_FIELD(COMLANE_R194, 19, 19);
+static const struct reg_field por_en = REG_FIELD(SERDES_CTRL, 29, 29);
+static const struct reg_field tx0_enable = REG_FIELD(WIZ_LANEXCTL_STS, 29, 31);
+static const struct reg_field rx0_enable = REG_FIELD(WIZ_LANEXCTL_STS, 13, 15);
+static const struct reg_field pll_enable = REG_FIELD(WIZ_PLL_CTRL, 29, 31);
+static const struct reg_field pll_ok = REG_FIELD(WIZ_PLL_CTRL, 28, 28);
+
+struct serdes_am654 {
+	struct regmap		*regmap;
+	struct regmap_field	*cmu_master_cdn_o;
+	struct regmap_field	*config_version;
+	struct regmap_field	*l1_master_cdn_o;
+	struct regmap_field	*cmu_ok_i_0;
+	struct regmap_field	*por_en;
+	struct regmap_field	*tx0_enable;
+	struct regmap_field	*rx0_enable;
+	struct regmap_field	*pll_enable;
+	struct regmap_field	*pll_ok;
+
+	struct device		*dev;
+	struct mux_control	*control;
+	bool			busy;
+	u32			type;
+	struct device_node	*of_node;
+	struct clk_onecell_data	clk_data;
+	struct clk		*clks[SERDES_NUM_CLOCKS];
+};
+
+static int serdes_am654_enable_pll(struct serdes_am654 *phy)
+{
+	int ret;
+	u32 val;
+
+	ret = regmap_field_write(phy->pll_enable, PLL_ENABLE_STATE);
+	if (ret)
+		return ret;
+
+	return regmap_field_read_poll_timeout(phy->pll_ok, val, val, 1000,
+					      PLL_LOCK_TIME);
+}
+
+static void serdes_am654_disable_pll(struct serdes_am654 *phy)
+{
+	struct device *dev = phy->dev;
+	int ret;
+
+	ret = regmap_field_write(phy->pll_enable, PLL_DISABLE_STATE);
+	if (ret)
+		dev_err(dev, "Failed to disable PLL\n");
+}
+
+static int serdes_am654_enable_txrx(struct serdes_am654 *phy)
+{
+	int ret;
+
+	/* Enable TX */
+	ret = regmap_field_write(phy->tx0_enable, TX0_ENABLE_STATE);
+	if (ret)
+		return ret;
+
+	/* Enable RX */
+	ret = regmap_field_write(phy->rx0_enable, RX0_ENABLE_STATE);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int serdes_am654_disable_txrx(struct serdes_am654 *phy)
+{
+	int ret;
+
+	/* Disable TX */
+	ret = regmap_field_write(phy->tx0_enable, TX0_DISABLE_STATE);
+	if (ret)
+		return ret;
+
+	/* Disable RX */
+	ret = regmap_field_write(phy->rx0_enable, RX0_DISABLE_STATE);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int serdes_am654_power_on(struct phy *x)
+{
+	struct serdes_am654 *phy = phy_get_drvdata(x);
+	struct device *dev = phy->dev;
+	int ret;
+	u32 val;
+
+	ret = serdes_am654_enable_pll(phy);
+	if (ret) {
+		dev_err(dev, "Failed to enable PLL\n");
+		return ret;
+	}
+
+	ret = serdes_am654_enable_txrx(phy);
+	if (ret) {
+		dev_err(dev, "Failed to enable TX RX\n");
+		return ret;
+	}
+
+	return regmap_field_read_poll_timeout(phy->cmu_ok_i_0, val, val,
+					      SLEEP_TIME, PLL_LOCK_TIME);
+}
+
+static int serdes_am654_power_off(struct phy *x)
+{
+	struct serdes_am654 *phy = phy_get_drvdata(x);
+
+	serdes_am654_disable_txrx(phy);
+	serdes_am654_disable_pll(phy);
+
+	return 0;
+}
+
+static int serdes_am654_init(struct phy *x)
+{
+	struct serdes_am654 *phy = phy_get_drvdata(x);
+	int ret;
+
+	ret = regmap_field_write(phy->config_version, VERSION);
+	if (ret)
+		return ret;
+
+	ret = regmap_field_write(phy->cmu_master_cdn_o, 0x1);
+	if (ret)
+		return ret;
+
+	ret = regmap_field_write(phy->l1_master_cdn_o, 0x1);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int serdes_am654_reset(struct phy *x)
+{
+	struct serdes_am654 *phy = phy_get_drvdata(x);
+	int ret;
+
+	ret = regmap_field_write(phy->por_en, 0x1);
+	if (ret)
+		return ret;
+
+	mdelay(1);
+
+	ret = regmap_field_write(phy->por_en, 0x0);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static void serdes_am654_release(struct phy *x)
+{
+	struct serdes_am654 *phy = phy_get_drvdata(x);
+
+	phy->type = PHY_NONE;
+	phy->busy = false;
+	mux_control_deselect(phy->control);
+}
+
+struct phy *serdes_am654_xlate(struct device *dev, struct of_phandle_args
+				 *args)
+{
+	struct serdes_am654 *am654_phy;
+	struct phy *phy;
+	int ret;
+
+	phy = of_phy_simple_xlate(dev, args);
+	if (IS_ERR(phy))
+		return phy;
+
+	am654_phy = phy_get_drvdata(phy);
+	if (am654_phy->busy)
+		return ERR_PTR(-EBUSY);
+
+	ret = mux_control_select(am654_phy->control, args->args[1]);
+	if (ret) {
+		dev_err(dev, "Failed to select SERDES Lane Function\n");
+		return ERR_PTR(ret);
+	}
+
+	am654_phy->busy = true;
+	am654_phy->type = args->args[0];
+
+	return phy;
+}
+
+static const struct phy_ops ops = {
+	.reset		= serdes_am654_reset,
+	.init		= serdes_am654_init,
+	.power_on	= serdes_am654_power_on,
+	.power_off	= serdes_am654_power_off,
+	.release	= serdes_am654_release,
+	.owner		= THIS_MODULE,
+};
+
+#define SERDES_NUM_MUX_COMBINATIONS 16
+
+#define LICLK 0
+#define EXT_REFCLK 1
+#define RICLK 2
+
+static const int
+serdes_am654_mux_table[SERDES_NUM_MUX_COMBINATIONS][SERDES_NUM_CLOCKS] = {
+	/*
+	 * Each combination maps to one of
+	 * "Figure 12-1986. SerDes Reference Clock Distribution"
+	 * in TRM.
+	 */
+	 /* Parent of CMU refclk, Left output, Right output
+	  * either of EXT_REFCLK, LICLK, RICLK
+	  */
+	{ EXT_REFCLK, EXT_REFCLK, EXT_REFCLK },	/* 0000 */
+	{ RICLK, EXT_REFCLK, EXT_REFCLK },	/* 0001 */
+	{ EXT_REFCLK, RICLK, LICLK },		/* 0010 */
+	{ RICLK, RICLK, EXT_REFCLK },		/* 0011 */
+	{ LICLK, EXT_REFCLK, EXT_REFCLK },	/* 0100 */
+	{ EXT_REFCLK, EXT_REFCLK, EXT_REFCLK },	/* 0101 */
+	{ LICLK, RICLK, LICLK },		/* 0110 */
+	{ EXT_REFCLK, RICLK, LICLK },		/* 0111 */
+	{ EXT_REFCLK, EXT_REFCLK, LICLK },	/* 1000 */
+	{ RICLK, EXT_REFCLK, LICLK },		/* 1001 */
+	{ EXT_REFCLK, RICLK, EXT_REFCLK },	/* 1010 */
+	{ RICLK, RICLK, EXT_REFCLK },		/* 1011 */
+	{ LICLK, EXT_REFCLK, LICLK },		/* 1100 */
+	{ EXT_REFCLK, EXT_REFCLK, LICLK },	/* 1101 */
+	{ LICLK, RICLK, EXT_REFCLK },		/* 1110 */
+	{ EXT_REFCLK, RICLK, EXT_REFCLK },	/* 1111 */
+};
+
+static u8 serdes_am654_clk_mux_get_parent(struct clk_hw *hw)
+{
+	struct serdes_am654_clk_mux *mux = to_serdes_am654_clk_mux(hw);
+	struct regmap *regmap = mux->regmap;
+	unsigned int reg = mux->reg;
+	unsigned int val;
+
+	regmap_read(regmap, reg, &val);
+	val &= AM654_SERDES_CTRL_CLKSEL_MASK;
+	val >>= AM654_SERDES_CTRL_CLKSEL_SHIFT;
+
+	return serdes_am654_mux_table[val][mux->clk_id];
+}
+
+static int serdes_am654_clk_mux_set_parent(struct clk_hw *hw, u8 index)
+{
+	struct serdes_am654_clk_mux *mux = to_serdes_am654_clk_mux(hw);
+	struct regmap *regmap = mux->regmap;
+	unsigned int reg = mux->reg;
+	int clk_id = mux->clk_id;
+	int parents[SERDES_NUM_CLOCKS];
+	const int *p;
+	u32 val;
+	int found, i;
+	int ret;
+
+	/* get existing setting */
+	regmap_read(regmap, reg, &val);
+	val &= AM654_SERDES_CTRL_CLKSEL_MASK;
+	val >>= AM654_SERDES_CTRL_CLKSEL_SHIFT;
+
+	for (i = 0; i < SERDES_NUM_CLOCKS; i++)
+		parents[i] = serdes_am654_mux_table[val][i];
+
+	/* change parent of this clock. others left intact */
+	parents[clk_id] = index;
+
+	/* Find the match */
+	for (val = 0; val < SERDES_NUM_MUX_COMBINATIONS; val++) {
+		p = serdes_am654_mux_table[val];
+		found = 1;
+		for (i = 0; i < SERDES_NUM_CLOCKS; i++) {
+			if (parents[i] != p[i]) {
+				found = 0;
+				break;
+			}
+		}
+
+		if (found)
+			break;
+	}
+
+	if (!found) {
+		/*
+		 * This can never happen, unless we missed
+		 * a valid combination in serdes_am654_mux_table.
+		 */
+		WARN(1, "Failed to find the parent of %s clock\n",
+		     hw->init->name);
+		return -EINVAL;
+	}
+
+	val <<= AM654_SERDES_CTRL_CLKSEL_SHIFT;
+	ret = regmap_update_bits(regmap, reg, AM654_SERDES_CTRL_CLKSEL_MASK,
+				 val);
+
+	return ret;
+}
+
+static const struct clk_ops serdes_am654_clk_mux_ops = {
+	.set_parent = serdes_am654_clk_mux_set_parent,
+	.get_parent = serdes_am654_clk_mux_get_parent,
+};
+
+static int serdes_am654_clk_register(struct serdes_am654 *am654_phy,
+				     const char *clock_name, int clock_num)
+{
+	struct device_node *node = am654_phy->of_node;
+	struct device *dev = am654_phy->dev;
+	struct serdes_am654_clk_mux *mux;
+	struct device_node *regmap_node;
+	const char **parent_names;
+	struct clk_init_data *init;
+	unsigned int num_parents;
+	struct regmap *regmap;
+	const __be32 *addr;
+	unsigned int reg;
+	struct clk *clk;
+
+	mux = devm_kzalloc(dev, sizeof(*mux), GFP_KERNEL);
+	if (!mux)
+		return -ENOMEM;
+
+	init = &mux->clk_data;
+
+	regmap_node = of_parse_phandle(node, "ti,serdes-clk", 0);
+	of_node_put(regmap_node);
+	if (!regmap_node) {
+		dev_err(dev, "Fail to get serdes-clk node\n");
+		return -ENODEV;
+	}
+
+	regmap = syscon_node_to_regmap(regmap_node->parent);
+	if (IS_ERR(regmap)) {
+		dev_err(dev, "Fail to get Syscon regmap\n");
+		return PTR_ERR(regmap);
+	}
+
+	num_parents = of_clk_get_parent_count(node);
+	if (num_parents < 2) {
+		dev_err(dev, "SERDES clock must have parents\n");
+		return -EINVAL;
+	}
+
+	parent_names = devm_kzalloc(dev, (sizeof(char *) * num_parents),
+				    GFP_KERNEL);
+	if (!parent_names)
+		return -ENOMEM;
+
+	of_clk_parent_fill(node, parent_names, num_parents);
+
+	addr = of_get_address(regmap_node, 0, NULL, NULL);
+	if (!addr)
+		return -EINVAL;
+
+	reg = be32_to_cpu(*addr);
+
+	init->ops = &serdes_am654_clk_mux_ops;
+	init->flags = CLK_SET_RATE_NO_REPARENT;
+	init->parent_names = parent_names;
+	init->num_parents = num_parents;
+	init->name = clock_name;
+
+	mux->regmap = regmap;
+	mux->reg = reg;
+	mux->clk_id = clock_num;
+	mux->hw.init = init;
+
+	clk = devm_clk_register(dev, &mux->hw);
+	if (IS_ERR(clk))
+		return PTR_ERR(clk);
+
+	am654_phy->clks[clock_num] = clk;
+
+	return 0;
+}
+
+static const struct of_device_id serdes_am654_id_table[] = {
+	{
+		.compatible = "ti,phy-am654-serdes",
+	},
+	{}
+};
+MODULE_DEVICE_TABLE(of, serdes_am654_id_table);
+
+static int serdes_am654_regfield_init(struct serdes_am654 *am654_phy)
+{
+	struct regmap *regmap = am654_phy->regmap;
+	struct device *dev = am654_phy->dev;
+
+	am654_phy->cmu_master_cdn_o = devm_regmap_field_alloc(dev, regmap,
+							      cmu_master_cdn_o);
+	if (IS_ERR(am654_phy->cmu_master_cdn_o)) {
+		dev_err(dev, "CMU_MASTER_CDN_O reg field init failed\n");
+		return PTR_ERR(am654_phy->cmu_master_cdn_o);
+	}
+
+	am654_phy->config_version = devm_regmap_field_alloc(dev, regmap,
+							    config_version);
+	if (IS_ERR(am654_phy->config_version)) {
+		dev_err(dev, "CONFIG_VERSION reg field init failed\n");
+		return PTR_ERR(am654_phy->config_version);
+	}
+
+	am654_phy->l1_master_cdn_o = devm_regmap_field_alloc(dev, regmap,
+							     l1_master_cdn_o);
+	if (IS_ERR(am654_phy->l1_master_cdn_o)) {
+		dev_err(dev, "L1_MASTER_CDN_O reg field init failed\n");
+		return PTR_ERR(am654_phy->l1_master_cdn_o);
+	}
+
+	am654_phy->cmu_ok_i_0 = devm_regmap_field_alloc(dev, regmap,
+							cmu_ok_i_0);
+	if (IS_ERR(am654_phy->cmu_ok_i_0)) {
+		dev_err(dev, "CMU_OK_I_0 reg field init failed\n");
+		return PTR_ERR(am654_phy->cmu_ok_i_0);
+	}
+
+	am654_phy->por_en = devm_regmap_field_alloc(dev, regmap, por_en);
+	if (IS_ERR(am654_phy->por_en)) {
+		dev_err(dev, "POR_EN reg field init failed\n");
+		return PTR_ERR(am654_phy->por_en);
+	}
+
+	am654_phy->tx0_enable = devm_regmap_field_alloc(dev, regmap,
+							tx0_enable);
+	if (IS_ERR(am654_phy->tx0_enable)) {
+		dev_err(dev, "TX0_ENABLE reg field init failed\n");
+		return PTR_ERR(am654_phy->tx0_enable);
+	}
+
+	am654_phy->rx0_enable = devm_regmap_field_alloc(dev, regmap,
+							rx0_enable);
+	if (IS_ERR(am654_phy->rx0_enable)) {
+		dev_err(dev, "RX0_ENABLE reg field init failed\n");
+		return PTR_ERR(am654_phy->rx0_enable);
+	}
+
+	am654_phy->pll_enable = devm_regmap_field_alloc(dev, regmap,
+							pll_enable);
+	if (IS_ERR(am654_phy->pll_enable)) {
+		dev_err(dev, "PLL_ENABLE reg field init failed\n");
+		return PTR_ERR(am654_phy->pll_enable);
+	}
+
+	am654_phy->pll_ok = devm_regmap_field_alloc(dev, regmap, pll_ok);
+	if (IS_ERR(am654_phy->pll_ok)) {
+		dev_err(dev, "PLL_OK reg field init failed\n");
+		return PTR_ERR(am654_phy->pll_ok);
+	}
+
+	return 0;
+}
+
+static int serdes_am654_probe(struct platform_device *pdev)
+{
+	struct phy_provider *phy_provider;
+	struct device *dev = &pdev->dev;
+	struct device_node *node = dev->of_node;
+	struct clk_onecell_data *clk_data;
+	struct serdes_am654 *am654_phy;
+	struct mux_control *control;
+	const char *clock_name;
+	struct regmap *regmap;
+	void __iomem *base;
+	struct phy *phy;
+	int ret;
+	int i;
+
+	am654_phy = devm_kzalloc(dev, sizeof(*am654_phy), GFP_KERNEL);
+	if (!am654_phy)
+		return -ENOMEM;
+
+	base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(base))
+		return PTR_ERR(base);
+
+	regmap = devm_regmap_init_mmio(dev, base, &serdes_am654_regmap_config);
+	if (IS_ERR(regmap)) {
+		dev_err(dev, "Failed to initialize regmap\n");
+		return PTR_ERR(regmap);
+	}
+
+	control = devm_mux_control_get(dev, NULL);
+	if (IS_ERR(control))
+		return PTR_ERR(control);
+
+	am654_phy->dev = dev;
+	am654_phy->of_node = node;
+	am654_phy->regmap = regmap;
+	am654_phy->control = control;
+	am654_phy->type = PHY_NONE;
+
+	ret = serdes_am654_regfield_init(am654_phy);
+	if (ret) {
+		dev_err(dev, "Failed to initialize regfields\n");
+		return ret;
+	}
+
+	platform_set_drvdata(pdev, am654_phy);
+
+	for (i = 0; i < SERDES_NUM_CLOCKS; i++) {
+		ret = of_property_read_string_index(node, "clock-output-names",
+						    i, &clock_name);
+		if (ret) {
+			dev_err(dev, "Failed to get clock name\n");
+			return ret;
+		}
+
+		ret = serdes_am654_clk_register(am654_phy, clock_name, i);
+		if (ret) {
+			dev_err(dev, "Failed to initialize clock %s\n",
+				clock_name);
+			return ret;
+		}
+	}
+
+	clk_data = &am654_phy->clk_data;
+	clk_data->clks = am654_phy->clks;
+	clk_data->clk_num = SERDES_NUM_CLOCKS;
+	ret = of_clk_add_provider(node, of_clk_src_onecell_get, clk_data);
+	if (ret)
+		return ret;
+
+	pm_runtime_enable(dev);
+
+	phy = devm_phy_create(dev, NULL, &ops);
+	if (IS_ERR(phy))
+		return PTR_ERR(phy);
+
+	phy_set_drvdata(phy, am654_phy);
+	phy_provider = devm_of_phy_provider_register(dev, serdes_am654_xlate);
+	if (IS_ERR(phy_provider)) {
+		ret = PTR_ERR(phy_provider);
+		goto clk_err;
+	}
+
+	return 0;
+
+clk_err:
+	of_clk_del_provider(node);
+
+	return ret;
+}
+
+static int serdes_am654_remove(struct platform_device *pdev)
+{
+	struct serdes_am654 *am654_phy = platform_get_drvdata(pdev);
+	struct device_node *node = am654_phy->of_node;
+
+	pm_runtime_disable(&pdev->dev);
+	of_clk_del_provider(node);
+
+	return 0;
+}
+
+static struct platform_driver serdes_am654_driver = {
+	.probe		= serdes_am654_probe,
+	.remove		= serdes_am654_remove,
+	.driver		= {
+		.name	= "phy-am654",
+		.of_match_table = serdes_am654_id_table,
+	},
+};
+module_platform_driver(serdes_am654_driver);
+
+MODULE_AUTHOR("Texas Instruments Inc.");
+MODULE_DESCRIPTION("TI AM654x SERDES driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/phy/ti/phy-ti-pipe3.c b/drivers/phy/ti/phy-ti-pipe3.c
index 68ce4a082b9b..739aaa0eb0ef 100644
--- a/drivers/phy/ti/phy-ti-pipe3.c
+++ b/drivers/phy/ti/phy-ti-pipe3.c
@@ -56,51 +56,73 @@
 
 #define SATA_PLL_SOFT_RESET	BIT(18)
 
-#define PIPE3_PHY_PWRCTL_CLK_CMD_MASK	0x003FC000
+#define PIPE3_PHY_PWRCTL_CLK_CMD_MASK	GENMASK(21, 14)
 #define PIPE3_PHY_PWRCTL_CLK_CMD_SHIFT	14
 
-#define PIPE3_PHY_PWRCTL_CLK_FREQ_MASK	0xFFC00000
+#define PIPE3_PHY_PWRCTL_CLK_FREQ_MASK	GENMASK(31, 22)
 #define PIPE3_PHY_PWRCTL_CLK_FREQ_SHIFT	22
 
-#define PIPE3_PHY_TX_RX_POWERON		0x3
-#define PIPE3_PHY_TX_RX_POWEROFF	0x0
+#define PIPE3_PHY_RX_POWERON       (0x1 << PIPE3_PHY_PWRCTL_CLK_CMD_SHIFT)
+#define PIPE3_PHY_TX_POWERON       (0x2 << PIPE3_PHY_PWRCTL_CLK_CMD_SHIFT)
 
 #define PCIE_PCS_MASK			0xFF0000
 #define PCIE_PCS_DELAY_COUNT_SHIFT	0x10
 
-#define PCIEPHYRX_ANA_PROGRAMMABILITY	0x0000000C
+#define PIPE3_PHY_RX_ANA_PROGRAMMABILITY	0x0000000C
 #define INTERFACE_MASK			GENMASK(31, 27)
 #define INTERFACE_SHIFT			27
+#define INTERFACE_MODE_USBSS		BIT(4)
+#define INTERFACE_MODE_SATA_1P5		BIT(3)
+#define INTERFACE_MODE_SATA_3P0		BIT(2)
+#define INTERFACE_MODE_PCIE		BIT(0)
+
 #define LOSD_MASK			GENMASK(17, 14)
 #define LOSD_SHIFT			14
 #define MEM_PLLDIV			GENMASK(6, 5)
 
-#define PCIEPHYRX_TRIM			0x0000001C
-#define MEM_DLL_TRIM_SEL		GENMASK(31, 30)
+#define PIPE3_PHY_RX_TRIM		0x0000001C
+#define MEM_DLL_TRIM_SEL_MASK		GENMASK(31, 30)
 #define MEM_DLL_TRIM_SHIFT		30
 
-#define PCIEPHYRX_DLL			0x00000024
-#define MEM_DLL_PHINT_RATE		GENMASK(31, 30)
+#define PIPE3_PHY_RX_DLL		0x00000024
+#define MEM_DLL_PHINT_RATE_MASK		GENMASK(31, 30)
+#define MEM_DLL_PHINT_RATE_SHIFT	30
 
-#define PCIEPHYRX_DIGITAL_MODES		0x00000028
+#define PIPE3_PHY_RX_DIGITAL_MODES		0x00000028
+#define MEM_HS_RATE_MASK		GENMASK(28, 27)
+#define MEM_HS_RATE_SHIFT		27
+#define MEM_OVRD_HS_RATE		BIT(26)
+#define MEM_OVRD_HS_RATE_SHIFT		26
 #define MEM_CDR_FASTLOCK		BIT(23)
-#define MEM_CDR_LBW			GENMASK(22, 21)
-#define MEM_CDR_STEPCNT			GENMASK(20, 19)
+#define MEM_CDR_FASTLOCK_SHIFT		23
+#define MEM_CDR_LBW_MASK		GENMASK(22, 21)
+#define MEM_CDR_LBW_SHIFT		21
+#define MEM_CDR_STEPCNT_MASK		GENMASK(20, 19)
+#define MEM_CDR_STEPCNT_SHIFT		19
 #define MEM_CDR_STL_MASK		GENMASK(18, 16)
 #define MEM_CDR_STL_SHIFT		16
 #define MEM_CDR_THR_MASK		GENMASK(15, 13)
 #define MEM_CDR_THR_SHIFT		13
 #define MEM_CDR_THR_MODE		BIT(12)
-#define MEM_CDR_CDR_2NDO_SDM_MODE	BIT(11)
-#define MEM_OVRD_HS_RATE		BIT(26)
-
-#define PCIEPHYRX_EQUALIZER		0x00000038
-#define MEM_EQLEV			GENMASK(31, 16)
-#define MEM_EQFTC			GENMASK(15, 11)
-#define MEM_EQCTL			GENMASK(10, 7)
+#define MEM_CDR_THR_MODE_SHIFT		12
+#define MEM_CDR_2NDO_SDM_MODE		BIT(11)
+#define MEM_CDR_2NDO_SDM_MODE_SHIFT	11
+
+#define PIPE3_PHY_RX_EQUALIZER		0x00000038
+#define MEM_EQLEV_MASK			GENMASK(31, 16)
+#define MEM_EQLEV_SHIFT			16
+#define MEM_EQFTC_MASK			GENMASK(15, 11)
+#define MEM_EQFTC_SHIFT			11
+#define MEM_EQCTL_MASK			GENMASK(10, 7)
 #define MEM_EQCTL_SHIFT			7
 #define MEM_OVRD_EQLEV			BIT(2)
+#define MEM_OVRD_EQLEV_SHIFT		2
 #define MEM_OVRD_EQFTC			BIT(1)
+#define MEM_OVRD_EQFTC_SHIFT		1
+
+#define SATA_PHY_RX_IO_AND_A2D_OVERRIDES	0x44
+#define MEM_CDR_LOS_SOURCE_MASK		GENMASK(10, 9)
+#define MEM_CDR_LOS_SOURCE_SHIFT	9
 
 /*
  * This is an Empirical value that works, need to confirm the actual
@@ -110,6 +132,10 @@
 #define PLL_IDLE_TIME	100	/* in milliseconds */
 #define PLL_LOCK_TIME	100	/* in milliseconds */
 
+enum pipe3_mode { PIPE3_MODE_PCIE = 1,
+		  PIPE3_MODE_SATA,
+		  PIPE3_MODE_USBSS };
+
 struct pipe3_dpll_params {
 	u16	m;
 	u8	n;
@@ -123,6 +149,27 @@ struct pipe3_dpll_map {
 	struct pipe3_dpll_params params;
 };
 
+struct pipe3_settings {
+	u8 ana_interface;
+	u8 ana_losd;
+	u8 dig_fastlock;
+	u8 dig_lbw;
+	u8 dig_stepcnt;
+	u8 dig_stl;
+	u8 dig_thr;
+	u8 dig_thr_mode;
+	u8 dig_2ndo_sdm_mode;
+	u8 dig_hs_rate;
+	u8 dig_ovrd_hs_rate;
+	u8 dll_trim_sel;
+	u8 dll_phint_rate;
+	u8 eq_lev;
+	u8 eq_ftc;
+	u8 eq_ctl;
+	u8 eq_ovrd_lev;
+	u8 eq_ovrd_ftc;
+};
+
 struct ti_pipe3 {
 	void __iomem		*pll_ctrl_base;
 	void __iomem		*phy_rx;
@@ -141,6 +188,8 @@ struct ti_pipe3 {
 	unsigned int		power_reg; /* power reg. index within syscon */
 	unsigned int		pcie_pcs_reg; /* pcs reg. index in syscon */
 	bool			sata_refclk_enabled;
+	enum pipe3_mode		mode;
+	struct pipe3_settings	settings;
 };
 
 static struct pipe3_dpll_map dpll_map_usb[] = {
@@ -163,6 +212,89 @@ static struct pipe3_dpll_map dpll_map_sata[] = {
 	{ },					/* Terminator */
 };
 
+struct pipe3_data {
+	enum pipe3_mode mode;
+	struct pipe3_dpll_map *dpll_map;
+	struct pipe3_settings settings;
+};
+
+static struct pipe3_data data_usb = {
+	.mode = PIPE3_MODE_USBSS,
+	.dpll_map = dpll_map_usb,
+	.settings = {
+	/* DRA75x TRM Table 26-17 Preferred USB3_PHY_RX SCP Register Settings */
+		.ana_interface = INTERFACE_MODE_USBSS,
+		.ana_losd = 0xa,
+		.dig_fastlock = 1,
+		.dig_lbw = 3,
+		.dig_stepcnt = 0,
+		.dig_stl = 0x3,
+		.dig_thr = 1,
+		.dig_thr_mode = 1,
+		.dig_2ndo_sdm_mode = 0,
+		.dig_hs_rate = 0,
+		.dig_ovrd_hs_rate = 1,
+		.dll_trim_sel = 0x2,
+		.dll_phint_rate = 0x3,
+		.eq_lev = 0,
+		.eq_ftc = 0,
+		.eq_ctl = 0x9,
+		.eq_ovrd_lev = 0,
+		.eq_ovrd_ftc = 0,
+	},
+};
+
+static struct pipe3_data data_sata = {
+	.mode = PIPE3_MODE_SATA,
+	.dpll_map = dpll_map_sata,
+	.settings = {
+	/* DRA75x TRM Table 26-9 Preferred SATA_PHY_RX SCP Register Settings */
+		.ana_interface = INTERFACE_MODE_SATA_3P0,
+		.ana_losd = 0x5,
+		.dig_fastlock = 1,
+		.dig_lbw = 3,
+		.dig_stepcnt = 0,
+		.dig_stl = 0x3,
+		.dig_thr = 1,
+		.dig_thr_mode = 1,
+		.dig_2ndo_sdm_mode = 0,
+		.dig_hs_rate = 0,	/* Not in TRM preferred settings */
+		.dig_ovrd_hs_rate = 0,	/* Not in TRM preferred settings */
+		.dll_trim_sel = 0x1,
+		.dll_phint_rate = 0x2,	/* for 1.5 GHz DPLL clock */
+		.eq_lev = 0,
+		.eq_ftc = 0x1f,
+		.eq_ctl = 0,
+		.eq_ovrd_lev = 1,
+		.eq_ovrd_ftc = 1,
+	},
+};
+
+static struct pipe3_data data_pcie = {
+	.mode = PIPE3_MODE_PCIE,
+	.settings = {
+	/* DRA75x TRM Table 26-62 Preferred PCIe_PHY_RX SCP Register Settings */
+		.ana_interface = INTERFACE_MODE_PCIE,
+		.ana_losd = 0xa,
+		.dig_fastlock = 1,
+		.dig_lbw = 3,
+		.dig_stepcnt = 0,
+		.dig_stl = 0x3,
+		.dig_thr = 1,
+		.dig_thr_mode = 1,
+		.dig_2ndo_sdm_mode = 0,
+		.dig_hs_rate = 0,
+		.dig_ovrd_hs_rate = 0,
+		.dll_trim_sel = 0x2,
+		.dll_phint_rate = 0x3,
+		.eq_lev = 0,
+		.eq_ftc = 0x1f,
+		.eq_ctl = 1,
+		.eq_ovrd_lev = 0,
+		.eq_ovrd_ftc = 0,
+	},
+};
+
 static inline u32 ti_pipe3_readl(void __iomem *addr, unsigned offset)
 {
 	return __raw_readl(addr + offset);
@@ -196,7 +328,6 @@ static void ti_pipe3_disable_clocks(struct ti_pipe3 *phy);
 
 static int ti_pipe3_power_off(struct phy *x)
 {
-	u32 val;
 	int ret;
 	struct ti_pipe3 *phy = phy_get_drvdata(x);
 
@@ -205,13 +336,13 @@ static int ti_pipe3_power_off(struct phy *x)
 		return 0;
 	}
 
-	val = PIPE3_PHY_TX_RX_POWEROFF << PIPE3_PHY_PWRCTL_CLK_CMD_SHIFT;
-
 	ret = regmap_update_bits(phy->phy_power_syscon, phy->power_reg,
-				 PIPE3_PHY_PWRCTL_CLK_CMD_MASK, val);
+				 PIPE3_PHY_PWRCTL_CLK_CMD_MASK, 0);
 	return ret;
 }
 
+static void ti_pipe3_calibrate(struct ti_pipe3 *phy);
+
 static int ti_pipe3_power_on(struct phy *x)
 {
 	u32 val;
@@ -219,6 +350,7 @@ static int ti_pipe3_power_on(struct phy *x)
 	int ret;
 	unsigned long rate;
 	struct ti_pipe3 *phy = phy_get_drvdata(x);
+	bool rx_pending = false;
 
 	if (!phy->phy_power_syscon) {
 		omap_control_phy_power(phy->control_dev, 1);
@@ -231,14 +363,35 @@ static int ti_pipe3_power_on(struct phy *x)
 		return -EINVAL;
 	}
 	rate = rate / 1000000;
-	mask = OMAP_CTRL_PIPE3_PHY_PWRCTL_CLK_CMD_MASK |
-		  OMAP_CTRL_PIPE3_PHY_PWRCTL_CLK_FREQ_MASK;
-	val = PIPE3_PHY_TX_RX_POWERON << PIPE3_PHY_PWRCTL_CLK_CMD_SHIFT;
-	val |= rate << OMAP_CTRL_PIPE3_PHY_PWRCTL_CLK_FREQ_SHIFT;
-
+	mask = OMAP_CTRL_PIPE3_PHY_PWRCTL_CLK_FREQ_MASK;
+	val = rate << OMAP_CTRL_PIPE3_PHY_PWRCTL_CLK_FREQ_SHIFT;
 	ret = regmap_update_bits(phy->phy_power_syscon, phy->power_reg,
 				 mask, val);
-	return ret;
+	/*
+	 * For PCIe, TX and RX must be powered on simultaneously.
+	 * For USB and SATA, TX must be powered on before RX
+	 */
+	mask = OMAP_CTRL_PIPE3_PHY_PWRCTL_CLK_CMD_MASK;
+	if (phy->mode == PIPE3_MODE_SATA || phy->mode == PIPE3_MODE_USBSS) {
+		val = PIPE3_PHY_TX_POWERON;
+		rx_pending = true;
+	} else {
+		val = PIPE3_PHY_TX_POWERON | PIPE3_PHY_RX_POWERON;
+	}
+
+	regmap_update_bits(phy->phy_power_syscon, phy->power_reg,
+			   mask, val);
+
+	if (rx_pending) {
+		val = PIPE3_PHY_TX_POWERON | PIPE3_PHY_RX_POWERON;
+		regmap_update_bits(phy->phy_power_syscon, phy->power_reg,
+				   mask, val);
+	}
+
+	if (phy->mode == PIPE3_MODE_PCIE)
+		ti_pipe3_calibrate(phy);
+
+	return 0;
 }
 
 static int ti_pipe3_dpll_wait_lock(struct ti_pipe3 *phy)
@@ -300,32 +453,55 @@ static int ti_pipe3_dpll_program(struct ti_pipe3 *phy)
 static void ti_pipe3_calibrate(struct ti_pipe3 *phy)
 {
 	u32 val;
+	struct pipe3_settings *s = &phy->settings;
 
-	val = ti_pipe3_readl(phy->phy_rx, PCIEPHYRX_ANA_PROGRAMMABILITY);
+	val = ti_pipe3_readl(phy->phy_rx, PIPE3_PHY_RX_ANA_PROGRAMMABILITY);
 	val &= ~(INTERFACE_MASK | LOSD_MASK | MEM_PLLDIV);
-	val = (0x1 << INTERFACE_SHIFT | 0xA << LOSD_SHIFT);
-	ti_pipe3_writel(phy->phy_rx, PCIEPHYRX_ANA_PROGRAMMABILITY, val);
-
-	val = ti_pipe3_readl(phy->phy_rx, PCIEPHYRX_DIGITAL_MODES);
-	val &= ~(MEM_CDR_STEPCNT | MEM_CDR_STL_MASK | MEM_CDR_THR_MASK |
-		 MEM_CDR_CDR_2NDO_SDM_MODE | MEM_OVRD_HS_RATE);
-	val |= (MEM_CDR_FASTLOCK | MEM_CDR_LBW | 0x3 << MEM_CDR_STL_SHIFT |
-		0x1 << MEM_CDR_THR_SHIFT | MEM_CDR_THR_MODE);
-	ti_pipe3_writel(phy->phy_rx, PCIEPHYRX_DIGITAL_MODES, val);
-
-	val = ti_pipe3_readl(phy->phy_rx, PCIEPHYRX_TRIM);
-	val &= ~MEM_DLL_TRIM_SEL;
-	val |= 0x2 << MEM_DLL_TRIM_SHIFT;
-	ti_pipe3_writel(phy->phy_rx, PCIEPHYRX_TRIM, val);
-
-	val = ti_pipe3_readl(phy->phy_rx, PCIEPHYRX_DLL);
-	val |= MEM_DLL_PHINT_RATE;
-	ti_pipe3_writel(phy->phy_rx, PCIEPHYRX_DLL, val);
-
-	val = ti_pipe3_readl(phy->phy_rx, PCIEPHYRX_EQUALIZER);
-	val &= ~(MEM_EQLEV | MEM_EQCTL | MEM_OVRD_EQLEV | MEM_OVRD_EQFTC);
-	val |= MEM_EQFTC | 0x1 << MEM_EQCTL_SHIFT;
-	ti_pipe3_writel(phy->phy_rx, PCIEPHYRX_EQUALIZER, val);
+	val |= (s->ana_interface << INTERFACE_SHIFT | s->ana_losd << LOSD_SHIFT);
+	ti_pipe3_writel(phy->phy_rx, PIPE3_PHY_RX_ANA_PROGRAMMABILITY, val);
+
+	val = ti_pipe3_readl(phy->phy_rx, PIPE3_PHY_RX_DIGITAL_MODES);
+	val &= ~(MEM_HS_RATE_MASK | MEM_OVRD_HS_RATE | MEM_CDR_FASTLOCK |
+		 MEM_CDR_LBW_MASK | MEM_CDR_STEPCNT_MASK | MEM_CDR_STL_MASK |
+		 MEM_CDR_THR_MASK | MEM_CDR_THR_MODE | MEM_CDR_2NDO_SDM_MODE);
+	val |= s->dig_hs_rate << MEM_HS_RATE_SHIFT |
+		s->dig_ovrd_hs_rate << MEM_OVRD_HS_RATE_SHIFT |
+		s->dig_fastlock << MEM_CDR_FASTLOCK_SHIFT |
+		s->dig_lbw << MEM_CDR_LBW_SHIFT |
+		s->dig_stepcnt << MEM_CDR_STEPCNT_SHIFT |
+		s->dig_stl << MEM_CDR_STL_SHIFT |
+		s->dig_thr << MEM_CDR_THR_SHIFT |
+		s->dig_thr_mode << MEM_CDR_THR_MODE_SHIFT |
+		s->dig_2ndo_sdm_mode << MEM_CDR_2NDO_SDM_MODE_SHIFT;
+	ti_pipe3_writel(phy->phy_rx, PIPE3_PHY_RX_DIGITAL_MODES, val);
+
+	val = ti_pipe3_readl(phy->phy_rx, PIPE3_PHY_RX_TRIM);
+	val &= ~MEM_DLL_TRIM_SEL_MASK;
+	val |= s->dll_trim_sel << MEM_DLL_TRIM_SHIFT;
+	ti_pipe3_writel(phy->phy_rx, PIPE3_PHY_RX_TRIM, val);
+
+	val = ti_pipe3_readl(phy->phy_rx, PIPE3_PHY_RX_DLL);
+	val &= ~MEM_DLL_PHINT_RATE_MASK;
+	val |= s->dll_phint_rate << MEM_DLL_PHINT_RATE_SHIFT;
+	ti_pipe3_writel(phy->phy_rx, PIPE3_PHY_RX_DLL, val);
+
+	val = ti_pipe3_readl(phy->phy_rx, PIPE3_PHY_RX_EQUALIZER);
+	val &= ~(MEM_EQLEV_MASK | MEM_EQFTC_MASK | MEM_EQCTL_MASK |
+		 MEM_OVRD_EQLEV | MEM_OVRD_EQFTC);
+	val |= s->eq_lev << MEM_EQLEV_SHIFT |
+		s->eq_ftc << MEM_EQFTC_SHIFT |
+		s->eq_ctl << MEM_EQCTL_SHIFT |
+		s->eq_ovrd_lev << MEM_OVRD_EQLEV_SHIFT |
+		s->eq_ovrd_ftc << MEM_OVRD_EQFTC_SHIFT;
+	ti_pipe3_writel(phy->phy_rx, PIPE3_PHY_RX_EQUALIZER, val);
+
+	if (phy->mode == PIPE3_MODE_SATA) {
+		val = ti_pipe3_readl(phy->phy_rx,
+				     SATA_PHY_RX_IO_AND_A2D_OVERRIDES);
+		val &= ~MEM_CDR_LOS_SOURCE_MASK;
+		ti_pipe3_writel(phy->phy_rx, SATA_PHY_RX_IO_AND_A2D_OVERRIDES,
+				val);
+	}
 }
 
 static int ti_pipe3_init(struct phy *x)
@@ -340,7 +516,7 @@ static int ti_pipe3_init(struct phy *x)
 	 * as recommended in AM572x TRM SPRUHZ6, section 18.5.2.2, table
 	 * 18-1804.
 	 */
-	if (of_device_is_compatible(phy->dev->of_node, "ti,phy-pipe3-pcie")) {
+	if (phy->mode == PIPE3_MODE_PCIE) {
 		if (!phy->pcs_syscon) {
 			omap_control_pcie_pcs(phy->control_dev, 0x96);
 			return 0;
@@ -349,12 +525,7 @@ static int ti_pipe3_init(struct phy *x)
 		val = 0x96 << OMAP_CTRL_PCIE_PCS_DELAY_COUNT_SHIFT;
 		ret = regmap_update_bits(phy->pcs_syscon, phy->pcie_pcs_reg,
 					 PCIE_PCS_MASK, val);
-		if (ret)
-			return ret;
-
-		ti_pipe3_calibrate(phy);
-
-		return 0;
+		return ret;
 	}
 
 	/* Bring it out of IDLE if it is IDLE */
@@ -367,8 +538,7 @@ static int ti_pipe3_init(struct phy *x)
 
 	/* SATA has issues if re-programmed when locked */
 	val = ti_pipe3_readl(phy->pll_ctrl_base, PLL_STATUS);
-	if ((val & PLL_LOCK) && of_device_is_compatible(phy->dev->of_node,
-							"ti,phy-pipe3-sata"))
+	if ((val & PLL_LOCK) && phy->mode == PIPE3_MODE_SATA)
 		return ret;
 
 	/* Program the DPLL */
@@ -378,6 +548,8 @@ static int ti_pipe3_init(struct phy *x)
 		return -EINVAL;
 	}
 
+	ti_pipe3_calibrate(phy);
+
 	return ret;
 }
 
@@ -390,12 +562,11 @@ static int ti_pipe3_exit(struct phy *x)
 	/* If dpll_reset_syscon is not present we wont power down SATA DPLL
 	 * due to Errata i783
 	 */
-	if (of_device_is_compatible(phy->dev->of_node, "ti,phy-pipe3-sata") &&
-	    !phy->dpll_reset_syscon)
+	if (phy->mode == PIPE3_MODE_SATA && !phy->dpll_reset_syscon)
 		return 0;
 
 	/* PCIe doesn't have internal DPLL */
-	if (!of_device_is_compatible(phy->dev->of_node, "ti,phy-pipe3-pcie")) {
+	if (phy->mode != PIPE3_MODE_PCIE) {
 		/* Put DPLL in IDLE mode */
 		val = ti_pipe3_readl(phy->pll_ctrl_base, PLL_CONFIGURATION2);
 		val |= PLL_IDLE;
@@ -418,7 +589,7 @@ static int ti_pipe3_exit(struct phy *x)
 	}
 
 	/* i783: SATA needs control bit toggle after PLL unlock */
-	if (of_device_is_compatible(phy->dev->of_node, "ti,phy-pipe3-sata")) {
+	if (phy->mode == PIPE3_MODE_SATA) {
 		regmap_update_bits(phy->dpll_reset_syscon, phy->dpll_reset_reg,
 				   SATA_PLL_SOFT_RESET, SATA_PLL_SOFT_RESET);
 		regmap_update_bits(phy->dpll_reset_syscon, phy->dpll_reset_reg,
@@ -443,7 +614,6 @@ static int ti_pipe3_get_clk(struct ti_pipe3 *phy)
 {
 	struct clk *clk;
 	struct device *dev = phy->dev;
-	struct device_node *node = dev->of_node;
 
 	phy->refclk = devm_clk_get(dev, "refclk");
 	if (IS_ERR(phy->refclk)) {
@@ -451,11 +621,11 @@ static int ti_pipe3_get_clk(struct ti_pipe3 *phy)
 		/* older DTBs have missing refclk in SATA PHY
 		 * so don't bail out in case of SATA PHY.
 		 */
-		if (!of_device_is_compatible(node, "ti,phy-pipe3-sata"))
+		if (phy->mode != PIPE3_MODE_SATA)
 			return PTR_ERR(phy->refclk);
 	}
 
-	if (!of_device_is_compatible(node, "ti,phy-pipe3-sata")) {
+	if (phy->mode != PIPE3_MODE_SATA) {
 		phy->wkupclk = devm_clk_get(dev, "wkupclk");
 		if (IS_ERR(phy->wkupclk)) {
 			dev_err(dev, "unable to get wkupclk\n");
@@ -465,8 +635,7 @@ static int ti_pipe3_get_clk(struct ti_pipe3 *phy)
 		phy->wkupclk = ERR_PTR(-ENODEV);
 	}
 
-	if (!of_device_is_compatible(node, "ti,phy-pipe3-pcie") ||
-	    phy->phy_power_syscon) {
+	if (phy->mode != PIPE3_MODE_PCIE || phy->phy_power_syscon) {
 		phy->sys_clk = devm_clk_get(dev, "sysclk");
 		if (IS_ERR(phy->sys_clk)) {
 			dev_err(dev, "unable to get sysclk\n");
@@ -474,7 +643,7 @@ static int ti_pipe3_get_clk(struct ti_pipe3 *phy)
 		}
 	}
 
-	if (of_device_is_compatible(node, "ti,phy-pipe3-pcie")) {
+	if (phy->mode == PIPE3_MODE_PCIE) {
 		clk = devm_clk_get(dev, "dpll_ref");
 		if (IS_ERR(clk)) {
 			dev_err(dev, "unable to get dpll ref clk\n");
@@ -546,7 +715,7 @@ static int ti_pipe3_get_sysctrl(struct ti_pipe3 *phy)
 		phy->control_dev = &control_pdev->dev;
 	}
 
-	if (of_device_is_compatible(node, "ti,phy-pipe3-pcie")) {
+	if (phy->mode == PIPE3_MODE_PCIE) {
 		phy->pcs_syscon = syscon_regmap_lookup_by_phandle(node,
 								  "syscon-pcs");
 		if (IS_ERR(phy->pcs_syscon)) {
@@ -564,7 +733,7 @@ static int ti_pipe3_get_sysctrl(struct ti_pipe3 *phy)
 		}
 	}
 
-	if (of_device_is_compatible(node, "ti,phy-pipe3-sata")) {
+	if (phy->mode == PIPE3_MODE_SATA) {
 		phy->dpll_reset_syscon = syscon_regmap_lookup_by_phandle(node,
 							"syscon-pllreset");
 		if (IS_ERR(phy->dpll_reset_syscon)) {
@@ -589,12 +758,8 @@ static int ti_pipe3_get_tx_rx_base(struct ti_pipe3 *phy)
 {
 	struct resource *res;
 	struct device *dev = phy->dev;
-	struct device_node *node = dev->of_node;
 	struct platform_device *pdev = to_platform_device(dev);
 
-	if (!of_device_is_compatible(node, "ti,phy-pipe3-pcie"))
-		return 0;
-
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
 					   "phy_rx");
 	phy->phy_rx = devm_ioremap_resource(dev, res);
@@ -611,24 +776,12 @@ static int ti_pipe3_get_tx_rx_base(struct ti_pipe3 *phy)
 static int ti_pipe3_get_pll_base(struct ti_pipe3 *phy)
 {
 	struct resource *res;
-	const struct of_device_id *match;
 	struct device *dev = phy->dev;
-	struct device_node *node = dev->of_node;
 	struct platform_device *pdev = to_platform_device(dev);
 
-	if (of_device_is_compatible(node, "ti,phy-pipe3-pcie"))
+	if (phy->mode == PIPE3_MODE_PCIE)
 		return 0;
 
-	match = of_match_device(ti_pipe3_id_table, dev);
-	if (!match)
-		return -EINVAL;
-
-	phy->dpll_map = (struct pipe3_dpll_map *)match->data;
-	if (!phy->dpll_map) {
-		dev_err(dev, "no DPLL data\n");
-		return -EINVAL;
-	}
-
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
 					   "pll_ctrl");
 	phy->pll_ctrl_base = devm_ioremap_resource(dev, res);
@@ -640,15 +793,29 @@ static int ti_pipe3_probe(struct platform_device *pdev)
 	struct ti_pipe3 *phy;
 	struct phy *generic_phy;
 	struct phy_provider *phy_provider;
-	struct device_node *node = pdev->dev.of_node;
 	struct device *dev = &pdev->dev;
 	int ret;
+	const struct of_device_id *match;
+	struct pipe3_data *data;
 
 	phy = devm_kzalloc(dev, sizeof(*phy), GFP_KERNEL);
 	if (!phy)
 		return -ENOMEM;
 
-	phy->dev		= dev;
+	match = of_match_device(ti_pipe3_id_table, dev);
+	if (!match)
+		return -EINVAL;
+
+	data = (struct pipe3_data *)match->data;
+	if (!data) {
+		dev_err(dev, "no driver data\n");
+		return -EINVAL;
+	}
+
+	phy->dev = dev;
+	phy->mode = data->mode;
+	phy->dpll_map = data->dpll_map;
+	phy->settings = data->settings;
 
 	ret = ti_pipe3_get_pll_base(phy);
 	if (ret)
@@ -672,7 +839,7 @@ static int ti_pipe3_probe(struct platform_device *pdev)
 	/*
 	 * Prevent auto-disable of refclk for SATA PHY due to Errata i783
 	 */
-	if (of_device_is_compatible(node, "ti,phy-pipe3-sata")) {
+	if (phy->mode == PIPE3_MODE_SATA) {
 		if (!IS_ERR(phy->refclk)) {
 			clk_prepare_enable(phy->refclk);
 			phy->sata_refclk_enabled = true;
@@ -762,18 +929,19 @@ static void ti_pipe3_disable_clocks(struct ti_pipe3 *phy)
 static const struct of_device_id ti_pipe3_id_table[] = {
 	{
 		.compatible = "ti,phy-usb3",
-		.data = dpll_map_usb,
+		.data = &data_usb,
 	},
 	{
 		.compatible = "ti,omap-usb3",
-		.data = dpll_map_usb,
+		.data = &data_usb,
 	},
 	{
 		.compatible = "ti,phy-pipe3-sata",
-		.data = dpll_map_sata,
+		.data = &data_sata,
 	},
 	{
 		.compatible = "ti,phy-pipe3-pcie",
+		.data = &data_pcie,
 	},
 	{}
 };
diff --git a/drivers/pinctrl/Kconfig b/drivers/pinctrl/Kconfig
index 2764d713fea8..19d8af9a36a2 100644
--- a/drivers/pinctrl/Kconfig
+++ b/drivers/pinctrl/Kconfig
@@ -108,6 +108,14 @@ config PINCTRL_AMD
 	  Requires ACPI/FDT device enumeration code to set up a platform
 	  device.
 
+config PINCTRL_BM1880
+	bool "Bitmain BM1880 Pinctrl driver"
+	depends on OF && (ARCH_BITMAIN || COMPILE_TEST)
+	default ARCH_BITMAIN
+	select PINMUX
+	help
+	  Pinctrl driver for Bitmain BM1880 SoC.
+
 config PINCTRL_DA850_PUPD
 	tristate "TI DA850/OMAP-L138/AM18XX pullup/pulldown groups"
 	depends on OF && (ARCH_DAVINCI_DA850 || COMPILE_TEST)
diff --git a/drivers/pinctrl/Makefile b/drivers/pinctrl/Makefile
index 712184b74a5c..62df40647e02 100644
--- a/drivers/pinctrl/Makefile
+++ b/drivers/pinctrl/Makefile
@@ -14,6 +14,7 @@ obj-$(CONFIG_PINCTRL_AXP209)	+= pinctrl-axp209.o
 obj-$(CONFIG_PINCTRL_AT91)	+= pinctrl-at91.o
 obj-$(CONFIG_PINCTRL_AT91PIO4)	+= pinctrl-at91-pio4.o
 obj-$(CONFIG_PINCTRL_AMD)	+= pinctrl-amd.o
+obj-$(CONFIG_PINCTRL_BM1880)	+= pinctrl-bm1880.o
 obj-$(CONFIG_PINCTRL_DA850_PUPD) += pinctrl-da850-pupd.o
 obj-$(CONFIG_PINCTRL_DIGICOLOR)	+= pinctrl-digicolor.o
 obj-$(CONFIG_PINCTRL_FALCON)	+= pinctrl-falcon.o
diff --git a/drivers/pinctrl/cirrus/Kconfig b/drivers/pinctrl/cirrus/Kconfig
index 27013e5949bc..74af07e25174 100644
--- a/drivers/pinctrl/cirrus/Kconfig
+++ b/drivers/pinctrl/cirrus/Kconfig
@@ -1,3 +1,13 @@
+config PINCTRL_LOCHNAGAR
+	tristate "Cirrus Logic Lochnagar pinctrl driver"
+	depends on MFD_LOCHNAGAR
+	select PINMUX
+	select PINCONF
+	select GENERIC_PINCONF
+	help
+	  This driver supports configuring the GPIO and other pin configuration
+	  of the Cirrus Logic Lochnagar audio development board.
+
 # This is all selected by the Madera MFD driver Kconfig options
 config PINCTRL_MADERA
 	tristate
diff --git a/drivers/pinctrl/cirrus/Makefile b/drivers/pinctrl/cirrus/Makefile
index 6e4938cde9e3..20baebf438f6 100644
--- a/drivers/pinctrl/cirrus/Makefile
+++ b/drivers/pinctrl/cirrus/Makefile
@@ -1,4 +1,6 @@
 # Cirrus Logic pinctrl drivers
+obj-$(CONFIG_PINCTRL_LOCHNAGAR)	+= pinctrl-lochnagar.o
+
 pinctrl-madera-objs		:= pinctrl-madera-core.o
 ifeq ($(CONFIG_PINCTRL_CS47L35),y)
 pinctrl-madera-objs		+= pinctrl-cs47l35.o
diff --git a/drivers/pinctrl/cirrus/pinctrl-lochnagar.c b/drivers/pinctrl/cirrus/pinctrl-lochnagar.c
new file mode 100644
index 000000000000..670ac53a3141
--- /dev/null
+++ b/drivers/pinctrl/cirrus/pinctrl-lochnagar.c
@@ -0,0 +1,1235 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Lochnagar pin and GPIO control
+ *
+ * Copyright (c) 2017-2018 Cirrus Logic, Inc. and
+ *                         Cirrus Logic International Semiconductor Ltd.
+ *
+ * Author: Charles Keepax <ckeepax@opensource.cirrus.com>
+ */
+
+#include <linux/err.h>
+#include <linux/errno.h>
+#include <linux/gpio/driver.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/pinctrl/pinctrl.h>
+#include <linux/pinctrl/pinmux.h>
+#include <linux/pinctrl/pinconf.h>
+#include <linux/pinctrl/pinconf-generic.h>
+
+#include <linux/mfd/lochnagar.h>
+#include <linux/mfd/lochnagar1_regs.h>
+#include <linux/mfd/lochnagar2_regs.h>
+
+#include <dt-bindings/pinctrl/lochnagar.h>
+
+#include "../pinctrl-utils.h"
+
+#define LN2_NUM_GPIO_CHANNELS	16
+
+#define LN_CDC_AIF1_STR		"codec-aif1"
+#define LN_CDC_AIF2_STR		"codec-aif2"
+#define LN_CDC_AIF3_STR		"codec-aif3"
+#define LN_DSP_AIF1_STR		"dsp-aif1"
+#define LN_DSP_AIF2_STR		"dsp-aif2"
+#define LN_PSIA1_STR		"psia1"
+#define LN_PSIA2_STR		"psia2"
+#define LN_GF_AIF1_STR		"gf-aif1"
+#define LN_GF_AIF2_STR		"gf-aif2"
+#define LN_GF_AIF3_STR		"gf-aif3"
+#define LN_GF_AIF4_STR		"gf-aif4"
+#define LN_SPDIF_AIF_STR	"spdif-aif"
+#define LN_USB_AIF1_STR		"usb-aif1"
+#define LN_USB_AIF2_STR		"usb-aif2"
+#define LN_ADAT_AIF_STR		"adat-aif"
+#define LN_SOUNDCARD_AIF_STR	"soundcard-aif"
+
+#define LN_PIN_GPIO(REV, ID, NAME, REG, SHIFT, INVERT) \
+static const struct lochnagar_pin lochnagar##REV##_##ID##_pin = { \
+	.name = NAME, .type = LN_PTYPE_GPIO, .reg = LOCHNAGAR##REV##_##REG, \
+	.shift = LOCHNAGAR##REV##_##SHIFT##_SHIFT, .invert = INVERT, \
+}
+
+#define LN_PIN_SAIF(REV, ID, NAME) \
+static const struct lochnagar_pin lochnagar##REV##_##ID##_pin = \
+	{ .name = NAME, .type = LN_PTYPE_AIF, }
+
+#define LN_PIN_AIF(REV, ID) \
+	LN_PIN_SAIF(REV, ID##_BCLK,  LN_##ID##_STR"-bclk"); \
+	LN_PIN_SAIF(REV, ID##_LRCLK, LN_##ID##_STR"-lrclk"); \
+	LN_PIN_SAIF(REV, ID##_RXDAT, LN_##ID##_STR"-rxdat"); \
+	LN_PIN_SAIF(REV, ID##_TXDAT, LN_##ID##_STR"-txdat")
+
+#define LN1_PIN_GPIO(ID, NAME, REG, SHIFT, INVERT) \
+	LN_PIN_GPIO(1, ID, NAME, REG, SHIFT, INVERT)
+
+#define LN1_PIN_MUX(ID, NAME) \
+static const struct lochnagar_pin lochnagar1_##ID##_pin = \
+	{ .name = NAME, .type = LN_PTYPE_MUX, .reg = LOCHNAGAR1_##ID, }
+
+#define LN1_PIN_AIF(ID) LN_PIN_AIF(1, ID)
+
+#define LN2_PIN_GPIO(ID, NAME, REG, SHIFT, INVERT) \
+	LN_PIN_GPIO(2, ID, NAME, REG, SHIFT, INVERT)
+
+#define LN2_PIN_MUX(ID, NAME) \
+static const struct lochnagar_pin lochnagar2_##ID##_pin = \
+	{ .name = NAME, .type = LN_PTYPE_MUX, .reg = LOCHNAGAR2_GPIO_##ID, }
+
+#define LN2_PIN_AIF(ID) LN_PIN_AIF(2, ID)
+
+#define LN2_PIN_GAI(ID) \
+	LN2_PIN_MUX(ID##_BCLK,  LN_##ID##_STR"-bclk"); \
+	LN2_PIN_MUX(ID##_LRCLK, LN_##ID##_STR"-lrclk"); \
+	LN2_PIN_MUX(ID##_RXDAT, LN_##ID##_STR"-rxdat"); \
+	LN2_PIN_MUX(ID##_TXDAT, LN_##ID##_STR"-txdat")
+
+#define LN_PIN(REV, ID) [LOCHNAGAR##REV##_PIN_##ID] = { \
+	.number = LOCHNAGAR##REV##_PIN_##ID, \
+	.name = lochnagar##REV##_##ID##_pin.name, \
+	.drv_data = (void *)&lochnagar##REV##_##ID##_pin, \
+}
+
+#define LN1_PIN(ID) LN_PIN(1, ID)
+#define LN2_PIN(ID) LN_PIN(2, ID)
+
+#define LN_PINS(REV, ID) \
+	LN_PIN(REV, ID##_BCLK), LN_PIN(REV, ID##_LRCLK), \
+	LN_PIN(REV, ID##_RXDAT), LN_PIN(REV, ID##_TXDAT)
+
+#define LN1_PINS(ID) LN_PINS(1, ID)
+#define LN2_PINS(ID) LN_PINS(2, ID)
+
+enum {
+	LOCHNAGAR1_PIN_GF_GPIO2 = LOCHNAGAR1_PIN_NUM_GPIOS,
+	LOCHNAGAR1_PIN_GF_GPIO3,
+	LOCHNAGAR1_PIN_GF_GPIO7,
+	LOCHNAGAR1_PIN_LED1,
+	LOCHNAGAR1_PIN_LED2,
+	LOCHNAGAR1_PIN_CDC_AIF1_BCLK,
+	LOCHNAGAR1_PIN_CDC_AIF1_LRCLK,
+	LOCHNAGAR1_PIN_CDC_AIF1_RXDAT,
+	LOCHNAGAR1_PIN_CDC_AIF1_TXDAT,
+	LOCHNAGAR1_PIN_CDC_AIF2_BCLK,
+	LOCHNAGAR1_PIN_CDC_AIF2_LRCLK,
+	LOCHNAGAR1_PIN_CDC_AIF2_RXDAT,
+	LOCHNAGAR1_PIN_CDC_AIF2_TXDAT,
+	LOCHNAGAR1_PIN_CDC_AIF3_BCLK,
+	LOCHNAGAR1_PIN_CDC_AIF3_LRCLK,
+	LOCHNAGAR1_PIN_CDC_AIF3_RXDAT,
+	LOCHNAGAR1_PIN_CDC_AIF3_TXDAT,
+	LOCHNAGAR1_PIN_DSP_AIF1_BCLK,
+	LOCHNAGAR1_PIN_DSP_AIF1_LRCLK,
+	LOCHNAGAR1_PIN_DSP_AIF1_RXDAT,
+	LOCHNAGAR1_PIN_DSP_AIF1_TXDAT,
+	LOCHNAGAR1_PIN_DSP_AIF2_BCLK,
+	LOCHNAGAR1_PIN_DSP_AIF2_LRCLK,
+	LOCHNAGAR1_PIN_DSP_AIF2_RXDAT,
+	LOCHNAGAR1_PIN_DSP_AIF2_TXDAT,
+	LOCHNAGAR1_PIN_PSIA1_BCLK,
+	LOCHNAGAR1_PIN_PSIA1_LRCLK,
+	LOCHNAGAR1_PIN_PSIA1_RXDAT,
+	LOCHNAGAR1_PIN_PSIA1_TXDAT,
+	LOCHNAGAR1_PIN_PSIA2_BCLK,
+	LOCHNAGAR1_PIN_PSIA2_LRCLK,
+	LOCHNAGAR1_PIN_PSIA2_RXDAT,
+	LOCHNAGAR1_PIN_PSIA2_TXDAT,
+	LOCHNAGAR1_PIN_SPDIF_AIF_BCLK,
+	LOCHNAGAR1_PIN_SPDIF_AIF_LRCLK,
+	LOCHNAGAR1_PIN_SPDIF_AIF_RXDAT,
+	LOCHNAGAR1_PIN_SPDIF_AIF_TXDAT,
+	LOCHNAGAR1_PIN_GF_AIF3_BCLK,
+	LOCHNAGAR1_PIN_GF_AIF3_RXDAT,
+	LOCHNAGAR1_PIN_GF_AIF3_LRCLK,
+	LOCHNAGAR1_PIN_GF_AIF3_TXDAT,
+	LOCHNAGAR1_PIN_GF_AIF4_BCLK,
+	LOCHNAGAR1_PIN_GF_AIF4_RXDAT,
+	LOCHNAGAR1_PIN_GF_AIF4_LRCLK,
+	LOCHNAGAR1_PIN_GF_AIF4_TXDAT,
+	LOCHNAGAR1_PIN_GF_AIF1_BCLK,
+	LOCHNAGAR1_PIN_GF_AIF1_RXDAT,
+	LOCHNAGAR1_PIN_GF_AIF1_LRCLK,
+	LOCHNAGAR1_PIN_GF_AIF1_TXDAT,
+	LOCHNAGAR1_PIN_GF_AIF2_BCLK,
+	LOCHNAGAR1_PIN_GF_AIF2_RXDAT,
+	LOCHNAGAR1_PIN_GF_AIF2_LRCLK,
+	LOCHNAGAR1_PIN_GF_AIF2_TXDAT,
+
+	LOCHNAGAR2_PIN_SPDIF_AIF_BCLK = LOCHNAGAR2_PIN_NUM_GPIOS,
+	LOCHNAGAR2_PIN_SPDIF_AIF_LRCLK,
+	LOCHNAGAR2_PIN_SPDIF_AIF_RXDAT,
+	LOCHNAGAR2_PIN_SPDIF_AIF_TXDAT,
+	LOCHNAGAR2_PIN_USB_AIF1_BCLK,
+	LOCHNAGAR2_PIN_USB_AIF1_LRCLK,
+	LOCHNAGAR2_PIN_USB_AIF1_RXDAT,
+	LOCHNAGAR2_PIN_USB_AIF1_TXDAT,
+	LOCHNAGAR2_PIN_USB_AIF2_BCLK,
+	LOCHNAGAR2_PIN_USB_AIF2_LRCLK,
+	LOCHNAGAR2_PIN_USB_AIF2_RXDAT,
+	LOCHNAGAR2_PIN_USB_AIF2_TXDAT,
+	LOCHNAGAR2_PIN_ADAT_AIF_BCLK,
+	LOCHNAGAR2_PIN_ADAT_AIF_LRCLK,
+	LOCHNAGAR2_PIN_ADAT_AIF_RXDAT,
+	LOCHNAGAR2_PIN_ADAT_AIF_TXDAT,
+	LOCHNAGAR2_PIN_SOUNDCARD_AIF_BCLK,
+	LOCHNAGAR2_PIN_SOUNDCARD_AIF_LRCLK,
+	LOCHNAGAR2_PIN_SOUNDCARD_AIF_RXDAT,
+	LOCHNAGAR2_PIN_SOUNDCARD_AIF_TXDAT,
+};
+
+enum lochnagar_pin_type {
+	LN_PTYPE_GPIO,
+	LN_PTYPE_MUX,
+	LN_PTYPE_AIF,
+	LN_PTYPE_COUNT,
+};
+
+struct lochnagar_pin {
+	const char name[20];
+
+	enum lochnagar_pin_type type;
+
+	unsigned int reg;
+	int shift;
+	bool invert;
+};
+
+LN1_PIN_GPIO(CDC_RESET,    "codec-reset",    RST,      CDC_RESET,    1);
+LN1_PIN_GPIO(DSP_RESET,    "dsp-reset",      RST,      DSP_RESET,    1);
+LN1_PIN_GPIO(CDC_CIF1MODE, "codec-cif1mode", I2C_CTRL, CDC_CIF_MODE, 0);
+LN1_PIN_MUX(GF_GPIO2,      "gf-gpio2");
+LN1_PIN_MUX(GF_GPIO3,      "gf-gpio3");
+LN1_PIN_MUX(GF_GPIO7,      "gf-gpio7");
+LN1_PIN_MUX(LED1,          "led1");
+LN1_PIN_MUX(LED2,          "led2");
+LN1_PIN_AIF(CDC_AIF1);
+LN1_PIN_AIF(CDC_AIF2);
+LN1_PIN_AIF(CDC_AIF3);
+LN1_PIN_AIF(DSP_AIF1);
+LN1_PIN_AIF(DSP_AIF2);
+LN1_PIN_AIF(PSIA1);
+LN1_PIN_AIF(PSIA2);
+LN1_PIN_AIF(SPDIF_AIF);
+LN1_PIN_AIF(GF_AIF1);
+LN1_PIN_AIF(GF_AIF2);
+LN1_PIN_AIF(GF_AIF3);
+LN1_PIN_AIF(GF_AIF4);
+
+LN2_PIN_GPIO(CDC_RESET,    "codec-reset",    MINICARD_RESETS, CDC_RESET,     1);
+LN2_PIN_GPIO(DSP_RESET,    "dsp-reset",      MINICARD_RESETS, DSP_RESET,     1);
+LN2_PIN_GPIO(CDC_CIF1MODE, "codec-cif1mode", COMMS_CTRL4,     CDC_CIF1MODE,  0);
+LN2_PIN_GPIO(CDC_LDOENA,   "codec-ldoena",   POWER_CTRL,      PWR_ENA,       0);
+LN2_PIN_GPIO(SPDIF_HWMODE, "spdif-hwmode",   SPDIF_CTRL,      SPDIF_HWMODE,  0);
+LN2_PIN_GPIO(SPDIF_RESET,  "spdif-reset",    SPDIF_CTRL,      SPDIF_RESET,   1);
+LN2_PIN_MUX(FPGA_GPIO1,    "fpga-gpio1");
+LN2_PIN_MUX(FPGA_GPIO2,    "fpga-gpio2");
+LN2_PIN_MUX(FPGA_GPIO3,    "fpga-gpio3");
+LN2_PIN_MUX(FPGA_GPIO4,    "fpga-gpio4");
+LN2_PIN_MUX(FPGA_GPIO5,    "fpga-gpio5");
+LN2_PIN_MUX(FPGA_GPIO6,    "fpga-gpio6");
+LN2_PIN_MUX(CDC_GPIO1,     "codec-gpio1");
+LN2_PIN_MUX(CDC_GPIO2,     "codec-gpio2");
+LN2_PIN_MUX(CDC_GPIO3,     "codec-gpio3");
+LN2_PIN_MUX(CDC_GPIO4,     "codec-gpio4");
+LN2_PIN_MUX(CDC_GPIO5,     "codec-gpio5");
+LN2_PIN_MUX(CDC_GPIO6,     "codec-gpio6");
+LN2_PIN_MUX(CDC_GPIO7,     "codec-gpio7");
+LN2_PIN_MUX(CDC_GPIO8,     "codec-gpio8");
+LN2_PIN_MUX(DSP_GPIO1,     "dsp-gpio1");
+LN2_PIN_MUX(DSP_GPIO2,     "dsp-gpio2");
+LN2_PIN_MUX(DSP_GPIO3,     "dsp-gpio3");
+LN2_PIN_MUX(DSP_GPIO4,     "dsp-gpio4");
+LN2_PIN_MUX(DSP_GPIO5,     "dsp-gpio5");
+LN2_PIN_MUX(DSP_GPIO6,     "dsp-gpio6");
+LN2_PIN_MUX(GF_GPIO2,      "gf-gpio2");
+LN2_PIN_MUX(GF_GPIO3,      "gf-gpio3");
+LN2_PIN_MUX(GF_GPIO7,      "gf-gpio7");
+LN2_PIN_MUX(DSP_UART1_RX,  "dsp-uart1-rx");
+LN2_PIN_MUX(DSP_UART1_TX,  "dsp-uart1-tx");
+LN2_PIN_MUX(DSP_UART2_RX,  "dsp-uart2-rx");
+LN2_PIN_MUX(DSP_UART2_TX,  "dsp-uart2-tx");
+LN2_PIN_MUX(GF_UART2_RX,   "gf-uart2-rx");
+LN2_PIN_MUX(GF_UART2_TX,   "gf-uart2-tx");
+LN2_PIN_MUX(USB_UART_RX,   "usb-uart-rx");
+LN2_PIN_MUX(CDC_PDMCLK1,   "codec-pdmclk1");
+LN2_PIN_MUX(CDC_PDMDAT1,   "codec-pdmdat1");
+LN2_PIN_MUX(CDC_PDMCLK2,   "codec-pdmclk2");
+LN2_PIN_MUX(CDC_PDMDAT2,   "codec-pdmdat2");
+LN2_PIN_MUX(CDC_DMICCLK1,  "codec-dmicclk1");
+LN2_PIN_MUX(CDC_DMICDAT1,  "codec-dmicdat1");
+LN2_PIN_MUX(CDC_DMICCLK2,  "codec-dmicclk2");
+LN2_PIN_MUX(CDC_DMICDAT2,  "codec-dmicdat2");
+LN2_PIN_MUX(CDC_DMICCLK3,  "codec-dmicclk3");
+LN2_PIN_MUX(CDC_DMICDAT3,  "codec-dmicdat3");
+LN2_PIN_MUX(CDC_DMICCLK4,  "codec-dmicclk4");
+LN2_PIN_MUX(CDC_DMICDAT4,  "codec-dmicdat4");
+LN2_PIN_MUX(DSP_DMICCLK1,  "dsp-dmicclk1");
+LN2_PIN_MUX(DSP_DMICDAT1,  "dsp-dmicdat1");
+LN2_PIN_MUX(DSP_DMICCLK2,  "dsp-dmicclk2");
+LN2_PIN_MUX(DSP_DMICDAT2,  "dsp-dmicdat2");
+LN2_PIN_MUX(I2C2_SCL,      "i2c2-scl");
+LN2_PIN_MUX(I2C2_SDA,      "i2c2-sda");
+LN2_PIN_MUX(I2C3_SCL,      "i2c3-scl");
+LN2_PIN_MUX(I2C3_SDA,      "i2c3-sda");
+LN2_PIN_MUX(I2C4_SCL,      "i2c4-scl");
+LN2_PIN_MUX(I2C4_SDA,      "i2c4-sda");
+LN2_PIN_MUX(DSP_STANDBY,   "dsp-standby");
+LN2_PIN_MUX(CDC_MCLK1,     "codec-mclk1");
+LN2_PIN_MUX(CDC_MCLK2,     "codec-mclk2");
+LN2_PIN_MUX(DSP_CLKIN,     "dsp-clkin");
+LN2_PIN_MUX(PSIA1_MCLK,    "psia1-mclk");
+LN2_PIN_MUX(PSIA2_MCLK,    "psia2-mclk");
+LN2_PIN_MUX(GF_GPIO1,      "gf-gpio1");
+LN2_PIN_MUX(GF_GPIO5,      "gf-gpio5");
+LN2_PIN_MUX(DSP_GPIO20,    "dsp-gpio20");
+LN2_PIN_GAI(CDC_AIF1);
+LN2_PIN_GAI(CDC_AIF2);
+LN2_PIN_GAI(CDC_AIF3);
+LN2_PIN_GAI(DSP_AIF1);
+LN2_PIN_GAI(DSP_AIF2);
+LN2_PIN_GAI(PSIA1);
+LN2_PIN_GAI(PSIA2);
+LN2_PIN_GAI(GF_AIF1);
+LN2_PIN_GAI(GF_AIF2);
+LN2_PIN_GAI(GF_AIF3);
+LN2_PIN_GAI(GF_AIF4);
+LN2_PIN_AIF(SPDIF_AIF);
+LN2_PIN_AIF(USB_AIF1);
+LN2_PIN_AIF(USB_AIF2);
+LN2_PIN_AIF(ADAT_AIF);
+LN2_PIN_AIF(SOUNDCARD_AIF);
+
+static const struct pinctrl_pin_desc lochnagar1_pins[] = {
+	LN1_PIN(CDC_RESET),      LN1_PIN(DSP_RESET),    LN1_PIN(CDC_CIF1MODE),
+	LN1_PIN(GF_GPIO2),       LN1_PIN(GF_GPIO3),     LN1_PIN(GF_GPIO7),
+	LN1_PIN(LED1),           LN1_PIN(LED2),
+	LN1_PINS(CDC_AIF1),      LN1_PINS(CDC_AIF2),    LN1_PINS(CDC_AIF3),
+	LN1_PINS(DSP_AIF1),      LN1_PINS(DSP_AIF2),
+	LN1_PINS(PSIA1),         LN1_PINS(PSIA2),
+	LN1_PINS(SPDIF_AIF),
+	LN1_PINS(GF_AIF1),       LN1_PINS(GF_AIF2),
+	LN1_PINS(GF_AIF3),       LN1_PINS(GF_AIF4),
+};
+
+static const struct pinctrl_pin_desc lochnagar2_pins[] = {
+	LN2_PIN(CDC_RESET),      LN2_PIN(DSP_RESET),    LN2_PIN(CDC_CIF1MODE),
+	LN2_PIN(CDC_LDOENA),
+	LN2_PIN(SPDIF_HWMODE),   LN2_PIN(SPDIF_RESET),
+	LN2_PIN(FPGA_GPIO1),     LN2_PIN(FPGA_GPIO2),   LN2_PIN(FPGA_GPIO3),
+	LN2_PIN(FPGA_GPIO4),     LN2_PIN(FPGA_GPIO5),   LN2_PIN(FPGA_GPIO6),
+	LN2_PIN(CDC_GPIO1),      LN2_PIN(CDC_GPIO2),    LN2_PIN(CDC_GPIO3),
+	LN2_PIN(CDC_GPIO4),      LN2_PIN(CDC_GPIO5),    LN2_PIN(CDC_GPIO6),
+	LN2_PIN(CDC_GPIO7),      LN2_PIN(CDC_GPIO8),
+	LN2_PIN(DSP_GPIO1),      LN2_PIN(DSP_GPIO2),    LN2_PIN(DSP_GPIO3),
+	LN2_PIN(DSP_GPIO4),      LN2_PIN(DSP_GPIO5),    LN2_PIN(DSP_GPIO6),
+	LN2_PIN(DSP_GPIO20),
+	LN2_PIN(GF_GPIO1),       LN2_PIN(GF_GPIO2),     LN2_PIN(GF_GPIO3),
+	LN2_PIN(GF_GPIO5),       LN2_PIN(GF_GPIO7),
+	LN2_PINS(CDC_AIF1),      LN2_PINS(CDC_AIF2),    LN2_PINS(CDC_AIF3),
+	LN2_PINS(DSP_AIF1),      LN2_PINS(DSP_AIF2),
+	LN2_PINS(PSIA1),         LN2_PINS(PSIA2),
+	LN2_PINS(GF_AIF1),       LN2_PINS(GF_AIF2),
+	LN2_PINS(GF_AIF3),       LN2_PINS(GF_AIF4),
+	LN2_PIN(DSP_UART1_RX),   LN2_PIN(DSP_UART1_TX),
+	LN2_PIN(DSP_UART2_RX),   LN2_PIN(DSP_UART2_TX),
+	LN2_PIN(GF_UART2_RX),    LN2_PIN(GF_UART2_TX),
+	LN2_PIN(USB_UART_RX),
+	LN2_PIN(CDC_PDMCLK1),    LN2_PIN(CDC_PDMDAT1),
+	LN2_PIN(CDC_PDMCLK2),    LN2_PIN(CDC_PDMDAT2),
+	LN2_PIN(CDC_DMICCLK1),   LN2_PIN(CDC_DMICDAT1),
+	LN2_PIN(CDC_DMICCLK2),   LN2_PIN(CDC_DMICDAT2),
+	LN2_PIN(CDC_DMICCLK3),   LN2_PIN(CDC_DMICDAT3),
+	LN2_PIN(CDC_DMICCLK4),   LN2_PIN(CDC_DMICDAT4),
+	LN2_PIN(DSP_DMICCLK1),   LN2_PIN(DSP_DMICDAT1),
+	LN2_PIN(DSP_DMICCLK2),   LN2_PIN(DSP_DMICDAT2),
+	LN2_PIN(I2C2_SCL),       LN2_PIN(I2C2_SDA),
+	LN2_PIN(I2C3_SCL),       LN2_PIN(I2C3_SDA),
+	LN2_PIN(I2C4_SCL),       LN2_PIN(I2C4_SDA),
+	LN2_PIN(DSP_STANDBY),
+	LN2_PIN(CDC_MCLK1),      LN2_PIN(CDC_MCLK2),
+	LN2_PIN(DSP_CLKIN),
+	LN2_PIN(PSIA1_MCLK),     LN2_PIN(PSIA2_MCLK),
+	LN2_PINS(SPDIF_AIF),
+	LN2_PINS(USB_AIF1),      LN2_PINS(USB_AIF2),
+	LN2_PINS(ADAT_AIF),
+	LN2_PINS(SOUNDCARD_AIF),
+};
+
+#define LN_AIF_PINS(REV, ID) \
+	LOCHNAGAR##REV##_PIN_##ID##_BCLK, \
+	LOCHNAGAR##REV##_PIN_##ID##_LRCLK, \
+	LOCHNAGAR##REV##_PIN_##ID##_TXDAT, \
+	LOCHNAGAR##REV##_PIN_##ID##_RXDAT,
+
+#define LN1_AIF(ID, CTRL) \
+static const struct lochnagar_aif lochnagar1_##ID##_aif = { \
+	.name = LN_##ID##_STR, \
+	.pins = { LN_AIF_PINS(1, ID) }, \
+	.src_reg = LOCHNAGAR1_##ID##_SEL, \
+	.src_mask = LOCHNAGAR1_SRC_MASK, \
+	.ctrl_reg = LOCHNAGAR1_##CTRL, \
+	.ena_mask = LOCHNAGAR1_##ID##_ENA_MASK, \
+	.master_mask = LOCHNAGAR1_##ID##_LRCLK_DIR_MASK | \
+		       LOCHNAGAR1_##ID##_BCLK_DIR_MASK, \
+}
+
+#define LN2_AIF(ID) \
+static const struct lochnagar_aif lochnagar2_##ID##_aif = { \
+	.name = LN_##ID##_STR, \
+	.pins = { LN_AIF_PINS(2, ID) }, \
+	.src_reg = LOCHNAGAR2_##ID##_CTRL,  \
+	.src_mask = LOCHNAGAR2_AIF_SRC_MASK, \
+	.ctrl_reg = LOCHNAGAR2_##ID##_CTRL, \
+	.ena_mask = LOCHNAGAR2_AIF_ENA_MASK, \
+	.master_mask = LOCHNAGAR2_AIF_LRCLK_DIR_MASK | \
+		       LOCHNAGAR2_AIF_BCLK_DIR_MASK, \
+}
+
+struct lochnagar_aif {
+	const char name[16];
+
+	unsigned int pins[4];
+
+	u16 src_reg;
+	u16 src_mask;
+
+	u16 ctrl_reg;
+	u16 ena_mask;
+	u16 master_mask;
+};
+
+LN1_AIF(CDC_AIF1,      CDC_AIF_CTRL1);
+LN1_AIF(CDC_AIF2,      CDC_AIF_CTRL1);
+LN1_AIF(CDC_AIF3,      CDC_AIF_CTRL2);
+LN1_AIF(DSP_AIF1,      DSP_AIF);
+LN1_AIF(DSP_AIF2,      DSP_AIF);
+LN1_AIF(PSIA1,         PSIA_AIF);
+LN1_AIF(PSIA2,         PSIA_AIF);
+LN1_AIF(GF_AIF1,       GF_AIF1);
+LN1_AIF(GF_AIF2,       GF_AIF2);
+LN1_AIF(GF_AIF3,       GF_AIF1);
+LN1_AIF(GF_AIF4,       GF_AIF2);
+LN1_AIF(SPDIF_AIF,     EXT_AIF_CTRL);
+
+LN2_AIF(CDC_AIF1);
+LN2_AIF(CDC_AIF2);
+LN2_AIF(CDC_AIF3);
+LN2_AIF(DSP_AIF1);
+LN2_AIF(DSP_AIF2);
+LN2_AIF(PSIA1);
+LN2_AIF(PSIA2);
+LN2_AIF(GF_AIF1);
+LN2_AIF(GF_AIF2);
+LN2_AIF(GF_AIF3);
+LN2_AIF(GF_AIF4);
+LN2_AIF(SPDIF_AIF);
+LN2_AIF(USB_AIF1);
+LN2_AIF(USB_AIF2);
+LN2_AIF(ADAT_AIF);
+LN2_AIF(SOUNDCARD_AIF);
+
+#define LN2_OP_AIF	0x00
+#define LN2_OP_GPIO	0xFE
+
+#define LN_FUNC(NAME, TYPE, OP) \
+	{ .name = NAME, .type = LN_FTYPE_##TYPE, .op = OP }
+
+#define LN_FUNC_PIN(REV, ID, OP) \
+	LN_FUNC(lochnagar##REV##_##ID##_pin.name, PIN, OP)
+
+#define LN1_FUNC_PIN(ID, OP) LN_FUNC_PIN(1, ID, OP)
+#define LN2_FUNC_PIN(ID, OP) LN_FUNC_PIN(2, ID, OP)
+
+#define LN_FUNC_AIF(REV, ID, OP) \
+	LN_FUNC(lochnagar##REV##_##ID##_aif.name, AIF, OP)
+
+#define LN1_FUNC_AIF(ID, OP) LN_FUNC_AIF(1, ID, OP)
+#define LN2_FUNC_AIF(ID, OP) LN_FUNC_AIF(2, ID, OP)
+
+#define LN2_FUNC_GAI(ID, OP, BOP, LROP, RXOP, TXOP) \
+	LN2_FUNC_AIF(ID, OP), \
+	LN_FUNC(lochnagar2_##ID##_BCLK_pin.name, PIN, BOP), \
+	LN_FUNC(lochnagar2_##ID##_LRCLK_pin.name, PIN, LROP), \
+	LN_FUNC(lochnagar2_##ID##_RXDAT_pin.name, PIN, RXOP), \
+	LN_FUNC(lochnagar2_##ID##_TXDAT_pin.name, PIN, TXOP)
+
+enum lochnagar_func_type {
+	LN_FTYPE_PIN,
+	LN_FTYPE_AIF,
+	LN_FTYPE_COUNT,
+};
+
+struct lochnagar_func {
+	const char * const name;
+
+	enum lochnagar_func_type type;
+
+	u8 op;
+};
+
+static const struct lochnagar_func lochnagar1_funcs[] = {
+	LN_FUNC("dsp-gpio1",       PIN, 0x01),
+	LN_FUNC("dsp-gpio2",       PIN, 0x02),
+	LN_FUNC("dsp-gpio3",       PIN, 0x03),
+	LN_FUNC("codec-gpio1",     PIN, 0x04),
+	LN_FUNC("codec-gpio2",     PIN, 0x05),
+	LN_FUNC("codec-gpio3",     PIN, 0x06),
+	LN_FUNC("codec-gpio4",     PIN, 0x07),
+	LN_FUNC("codec-gpio5",     PIN, 0x08),
+	LN_FUNC("codec-gpio6",     PIN, 0x09),
+	LN_FUNC("codec-gpio7",     PIN, 0x0A),
+	LN_FUNC("codec-gpio8",     PIN, 0x0B),
+	LN1_FUNC_PIN(GF_GPIO2,          0x0C),
+	LN1_FUNC_PIN(GF_GPIO3,          0x0D),
+	LN1_FUNC_PIN(GF_GPIO7,          0x0E),
+
+	LN1_FUNC_AIF(SPDIF_AIF,         0x01),
+	LN1_FUNC_AIF(PSIA1,             0x02),
+	LN1_FUNC_AIF(PSIA2,             0x03),
+	LN1_FUNC_AIF(CDC_AIF1,          0x04),
+	LN1_FUNC_AIF(CDC_AIF2,          0x05),
+	LN1_FUNC_AIF(CDC_AIF3,          0x06),
+	LN1_FUNC_AIF(DSP_AIF1,          0x07),
+	LN1_FUNC_AIF(DSP_AIF2,          0x08),
+	LN1_FUNC_AIF(GF_AIF3,           0x09),
+	LN1_FUNC_AIF(GF_AIF4,           0x0A),
+	LN1_FUNC_AIF(GF_AIF1,           0x0B),
+	LN1_FUNC_AIF(GF_AIF2,           0x0C),
+};
+
+static const struct lochnagar_func lochnagar2_funcs[] = {
+	LN_FUNC("aif",             PIN, LN2_OP_AIF),
+	LN2_FUNC_PIN(FPGA_GPIO1,        0x01),
+	LN2_FUNC_PIN(FPGA_GPIO2,        0x02),
+	LN2_FUNC_PIN(FPGA_GPIO3,        0x03),
+	LN2_FUNC_PIN(FPGA_GPIO4,        0x04),
+	LN2_FUNC_PIN(FPGA_GPIO5,        0x05),
+	LN2_FUNC_PIN(FPGA_GPIO6,        0x06),
+	LN2_FUNC_PIN(CDC_GPIO1,         0x07),
+	LN2_FUNC_PIN(CDC_GPIO2,         0x08),
+	LN2_FUNC_PIN(CDC_GPIO3,         0x09),
+	LN2_FUNC_PIN(CDC_GPIO4,         0x0A),
+	LN2_FUNC_PIN(CDC_GPIO5,         0x0B),
+	LN2_FUNC_PIN(CDC_GPIO6,         0x0C),
+	LN2_FUNC_PIN(CDC_GPIO7,         0x0D),
+	LN2_FUNC_PIN(CDC_GPIO8,         0x0E),
+	LN2_FUNC_PIN(DSP_GPIO1,         0x0F),
+	LN2_FUNC_PIN(DSP_GPIO2,         0x10),
+	LN2_FUNC_PIN(DSP_GPIO3,         0x11),
+	LN2_FUNC_PIN(DSP_GPIO4,         0x12),
+	LN2_FUNC_PIN(DSP_GPIO5,         0x13),
+	LN2_FUNC_PIN(DSP_GPIO6,         0x14),
+	LN2_FUNC_PIN(GF_GPIO2,          0x15),
+	LN2_FUNC_PIN(GF_GPIO3,          0x16),
+	LN2_FUNC_PIN(GF_GPIO7,          0x17),
+	LN2_FUNC_PIN(GF_GPIO1,          0x18),
+	LN2_FUNC_PIN(GF_GPIO5,          0x19),
+	LN2_FUNC_PIN(DSP_GPIO20,        0x1A),
+	LN_FUNC("codec-clkout",    PIN, 0x20),
+	LN_FUNC("dsp-clkout",      PIN, 0x21),
+	LN_FUNC("pmic-32k",        PIN, 0x22),
+	LN_FUNC("spdif-clkout",    PIN, 0x23),
+	LN_FUNC("clk-12m288",      PIN, 0x24),
+	LN_FUNC("clk-11m2986",     PIN, 0x25),
+	LN_FUNC("clk-24m576",      PIN, 0x26),
+	LN_FUNC("clk-22m5792",     PIN, 0x27),
+	LN_FUNC("xmos-mclk",       PIN, 0x29),
+	LN_FUNC("gf-clkout1",      PIN, 0x2A),
+	LN_FUNC("gf-mclk1",        PIN, 0x2B),
+	LN_FUNC("gf-mclk3",        PIN, 0x2C),
+	LN_FUNC("gf-mclk2",        PIN, 0x2D),
+	LN_FUNC("gf-clkout2",      PIN, 0x2E),
+	LN2_FUNC_PIN(CDC_MCLK1,         0x2F),
+	LN2_FUNC_PIN(CDC_MCLK2,         0x30),
+	LN2_FUNC_PIN(DSP_CLKIN,         0x31),
+	LN2_FUNC_PIN(PSIA1_MCLK,        0x32),
+	LN2_FUNC_PIN(PSIA2_MCLK,        0x33),
+	LN_FUNC("spdif-mclk",      PIN, 0x34),
+	LN_FUNC("codec-irq",       PIN, 0x42),
+	LN2_FUNC_PIN(CDC_RESET,         0x43),
+	LN2_FUNC_PIN(DSP_RESET,         0x44),
+	LN_FUNC("dsp-irq",         PIN, 0x45),
+	LN2_FUNC_PIN(DSP_STANDBY,       0x46),
+	LN2_FUNC_PIN(CDC_PDMCLK1,       0x90),
+	LN2_FUNC_PIN(CDC_PDMDAT1,       0x91),
+	LN2_FUNC_PIN(CDC_PDMCLK2,       0x92),
+	LN2_FUNC_PIN(CDC_PDMDAT2,       0x93),
+	LN2_FUNC_PIN(CDC_DMICCLK1,      0xA0),
+	LN2_FUNC_PIN(CDC_DMICDAT1,      0xA1),
+	LN2_FUNC_PIN(CDC_DMICCLK2,      0xA2),
+	LN2_FUNC_PIN(CDC_DMICDAT2,      0xA3),
+	LN2_FUNC_PIN(CDC_DMICCLK3,      0xA4),
+	LN2_FUNC_PIN(CDC_DMICDAT3,      0xA5),
+	LN2_FUNC_PIN(CDC_DMICCLK4,      0xA6),
+	LN2_FUNC_PIN(CDC_DMICDAT4,      0xA7),
+	LN2_FUNC_PIN(DSP_DMICCLK1,      0xA8),
+	LN2_FUNC_PIN(DSP_DMICDAT1,      0xA9),
+	LN2_FUNC_PIN(DSP_DMICCLK2,      0xAA),
+	LN2_FUNC_PIN(DSP_DMICDAT2,      0xAB),
+	LN2_FUNC_PIN(DSP_UART1_RX,      0xC0),
+	LN2_FUNC_PIN(DSP_UART1_TX,      0xC1),
+	LN2_FUNC_PIN(DSP_UART2_RX,      0xC2),
+	LN2_FUNC_PIN(DSP_UART2_TX,      0xC3),
+	LN2_FUNC_PIN(GF_UART2_RX,       0xC4),
+	LN2_FUNC_PIN(GF_UART2_TX,       0xC5),
+	LN2_FUNC_PIN(USB_UART_RX,       0xC6),
+	LN_FUNC("usb-uart-tx",     PIN, 0xC7),
+	LN2_FUNC_PIN(I2C2_SCL,          0xE0),
+	LN2_FUNC_PIN(I2C2_SDA,          0xE1),
+	LN2_FUNC_PIN(I2C3_SCL,          0xE2),
+	LN2_FUNC_PIN(I2C3_SDA,          0xE3),
+	LN2_FUNC_PIN(I2C4_SCL,          0xE4),
+	LN2_FUNC_PIN(I2C4_SDA,          0xE5),
+
+	LN2_FUNC_AIF(SPDIF_AIF,         0x01),
+	LN2_FUNC_GAI(PSIA1,             0x02, 0x50, 0x51, 0x52, 0x53),
+	LN2_FUNC_GAI(PSIA2,             0x03, 0x54, 0x55, 0x56, 0x57),
+	LN2_FUNC_GAI(CDC_AIF1,          0x04, 0x59, 0x5B, 0x5A, 0x58),
+	LN2_FUNC_GAI(CDC_AIF2,          0x05, 0x5D, 0x5F, 0x5E, 0x5C),
+	LN2_FUNC_GAI(CDC_AIF3,          0x06, 0x61, 0x62, 0x63, 0x60),
+	LN2_FUNC_GAI(DSP_AIF1,          0x07, 0x65, 0x67, 0x66, 0x64),
+	LN2_FUNC_GAI(DSP_AIF2,          0x08, 0x69, 0x6B, 0x6A, 0x68),
+	LN2_FUNC_GAI(GF_AIF3,           0x09, 0x6D, 0x6F, 0x6C, 0x6E),
+	LN2_FUNC_GAI(GF_AIF4,           0x0A, 0x71, 0x73, 0x70, 0x72),
+	LN2_FUNC_GAI(GF_AIF1,           0x0B, 0x75, 0x77, 0x74, 0x76),
+	LN2_FUNC_GAI(GF_AIF2,           0x0C, 0x79, 0x7B, 0x78, 0x7A),
+	LN2_FUNC_AIF(USB_AIF1,          0x0D),
+	LN2_FUNC_AIF(USB_AIF2,          0x0E),
+	LN2_FUNC_AIF(ADAT_AIF,          0x0F),
+	LN2_FUNC_AIF(SOUNDCARD_AIF,     0x10),
+};
+
+#define LN_GROUP_PIN(REV, ID) { \
+	.name = lochnagar##REV##_##ID##_pin.name, \
+	.type = LN_FTYPE_PIN, \
+	.pins = &lochnagar##REV##_pins[LOCHNAGAR##REV##_PIN_##ID].number, \
+	.npins = 1, \
+	.priv = &lochnagar##REV##_pins[LOCHNAGAR##REV##_PIN_##ID], \
+}
+
+#define LN_GROUP_AIF(REV, ID) { \
+	.name = lochnagar##REV##_##ID##_aif.name, \
+	.type = LN_FTYPE_AIF, \
+	.pins = lochnagar##REV##_##ID##_aif.pins, \
+	.npins = ARRAY_SIZE(lochnagar##REV##_##ID##_aif.pins), \
+	.priv = &lochnagar##REV##_##ID##_aif, \
+}
+
+#define LN1_GROUP_PIN(ID) LN_GROUP_PIN(1, ID)
+#define LN2_GROUP_PIN(ID) LN_GROUP_PIN(2, ID)
+
+#define LN1_GROUP_AIF(ID) LN_GROUP_AIF(1, ID)
+#define LN2_GROUP_AIF(ID) LN_GROUP_AIF(2, ID)
+
+#define LN2_GROUP_GAI(ID) \
+	LN2_GROUP_AIF(ID), \
+	LN2_GROUP_PIN(ID##_BCLK), LN2_GROUP_PIN(ID##_LRCLK), \
+	LN2_GROUP_PIN(ID##_RXDAT), LN2_GROUP_PIN(ID##_TXDAT)
+
+struct lochnagar_group {
+	const char * const name;
+
+	enum lochnagar_func_type type;
+
+	const unsigned int *pins;
+	unsigned int npins;
+
+	const void *priv;
+};
+
+static const struct lochnagar_group lochnagar1_groups[] = {
+	LN1_GROUP_PIN(GF_GPIO2),       LN1_GROUP_PIN(GF_GPIO3),
+	LN1_GROUP_PIN(GF_GPIO7),
+	LN1_GROUP_PIN(LED1),           LN1_GROUP_PIN(LED2),
+	LN1_GROUP_AIF(CDC_AIF1),       LN1_GROUP_AIF(CDC_AIF2),
+	LN1_GROUP_AIF(CDC_AIF3),
+	LN1_GROUP_AIF(DSP_AIF1),       LN1_GROUP_AIF(DSP_AIF2),
+	LN1_GROUP_AIF(PSIA1),          LN1_GROUP_AIF(PSIA2),
+	LN1_GROUP_AIF(GF_AIF1),        LN1_GROUP_AIF(GF_AIF2),
+	LN1_GROUP_AIF(GF_AIF3),        LN1_GROUP_AIF(GF_AIF4),
+	LN1_GROUP_AIF(SPDIF_AIF),
+};
+
+static const struct lochnagar_group lochnagar2_groups[] = {
+	LN2_GROUP_PIN(FPGA_GPIO1),     LN2_GROUP_PIN(FPGA_GPIO2),
+	LN2_GROUP_PIN(FPGA_GPIO3),     LN2_GROUP_PIN(FPGA_GPIO4),
+	LN2_GROUP_PIN(FPGA_GPIO5),     LN2_GROUP_PIN(FPGA_GPIO6),
+	LN2_GROUP_PIN(CDC_GPIO1),      LN2_GROUP_PIN(CDC_GPIO2),
+	LN2_GROUP_PIN(CDC_GPIO3),      LN2_GROUP_PIN(CDC_GPIO4),
+	LN2_GROUP_PIN(CDC_GPIO5),      LN2_GROUP_PIN(CDC_GPIO6),
+	LN2_GROUP_PIN(CDC_GPIO7),      LN2_GROUP_PIN(CDC_GPIO8),
+	LN2_GROUP_PIN(DSP_GPIO1),      LN2_GROUP_PIN(DSP_GPIO2),
+	LN2_GROUP_PIN(DSP_GPIO3),      LN2_GROUP_PIN(DSP_GPIO4),
+	LN2_GROUP_PIN(DSP_GPIO5),      LN2_GROUP_PIN(DSP_GPIO6),
+	LN2_GROUP_PIN(DSP_GPIO20),
+	LN2_GROUP_PIN(GF_GPIO1),
+	LN2_GROUP_PIN(GF_GPIO2),       LN2_GROUP_PIN(GF_GPIO5),
+	LN2_GROUP_PIN(GF_GPIO3),       LN2_GROUP_PIN(GF_GPIO7),
+	LN2_GROUP_PIN(DSP_UART1_RX),   LN2_GROUP_PIN(DSP_UART1_TX),
+	LN2_GROUP_PIN(DSP_UART2_RX),   LN2_GROUP_PIN(DSP_UART2_TX),
+	LN2_GROUP_PIN(GF_UART2_RX),    LN2_GROUP_PIN(GF_UART2_TX),
+	LN2_GROUP_PIN(USB_UART_RX),
+	LN2_GROUP_PIN(CDC_PDMCLK1),    LN2_GROUP_PIN(CDC_PDMDAT1),
+	LN2_GROUP_PIN(CDC_PDMCLK2),    LN2_GROUP_PIN(CDC_PDMDAT2),
+	LN2_GROUP_PIN(CDC_DMICCLK1),   LN2_GROUP_PIN(CDC_DMICDAT1),
+	LN2_GROUP_PIN(CDC_DMICCLK2),   LN2_GROUP_PIN(CDC_DMICDAT2),
+	LN2_GROUP_PIN(CDC_DMICCLK3),   LN2_GROUP_PIN(CDC_DMICDAT3),
+	LN2_GROUP_PIN(CDC_DMICCLK4),   LN2_GROUP_PIN(CDC_DMICDAT4),
+	LN2_GROUP_PIN(DSP_DMICCLK1),   LN2_GROUP_PIN(DSP_DMICDAT1),
+	LN2_GROUP_PIN(DSP_DMICCLK2),   LN2_GROUP_PIN(DSP_DMICDAT2),
+	LN2_GROUP_PIN(I2C2_SCL),       LN2_GROUP_PIN(I2C2_SDA),
+	LN2_GROUP_PIN(I2C3_SCL),       LN2_GROUP_PIN(I2C3_SDA),
+	LN2_GROUP_PIN(I2C4_SCL),       LN2_GROUP_PIN(I2C4_SDA),
+	LN2_GROUP_PIN(DSP_STANDBY),
+	LN2_GROUP_PIN(CDC_MCLK1),      LN2_GROUP_PIN(CDC_MCLK2),
+	LN2_GROUP_PIN(DSP_CLKIN),
+	LN2_GROUP_PIN(PSIA1_MCLK),     LN2_GROUP_PIN(PSIA2_MCLK),
+	LN2_GROUP_GAI(CDC_AIF1),       LN2_GROUP_GAI(CDC_AIF2),
+	LN2_GROUP_GAI(CDC_AIF3),
+	LN2_GROUP_GAI(DSP_AIF1),       LN2_GROUP_GAI(DSP_AIF2),
+	LN2_GROUP_GAI(PSIA1),          LN2_GROUP_GAI(PSIA2),
+	LN2_GROUP_GAI(GF_AIF1),        LN2_GROUP_GAI(GF_AIF2),
+	LN2_GROUP_GAI(GF_AIF3),        LN2_GROUP_GAI(GF_AIF4),
+	LN2_GROUP_AIF(SPDIF_AIF),
+	LN2_GROUP_AIF(USB_AIF1),       LN2_GROUP_AIF(USB_AIF2),
+	LN2_GROUP_AIF(ADAT_AIF),
+	LN2_GROUP_AIF(SOUNDCARD_AIF),
+};
+
+struct lochnagar_func_groups {
+	const char **groups;
+	unsigned int ngroups;
+};
+
+struct lochnagar_pin_priv {
+	struct lochnagar *lochnagar;
+	struct device *dev;
+
+	const struct lochnagar_func *funcs;
+	unsigned int nfuncs;
+
+	const struct pinctrl_pin_desc *pins;
+	unsigned int npins;
+
+	const struct lochnagar_group *groups;
+	unsigned int ngroups;
+
+	struct lochnagar_func_groups func_groups[LN_FTYPE_COUNT];
+
+	struct gpio_chip gpio_chip;
+};
+
+static int lochnagar_get_groups_count(struct pinctrl_dev *pctldev)
+{
+	struct lochnagar_pin_priv *priv = pinctrl_dev_get_drvdata(pctldev);
+
+	return priv->ngroups;
+}
+
+static const char *lochnagar_get_group_name(struct pinctrl_dev *pctldev,
+					    unsigned int group_idx)
+{
+	struct lochnagar_pin_priv *priv = pinctrl_dev_get_drvdata(pctldev);
+
+	return priv->groups[group_idx].name;
+}
+
+static int lochnagar_get_group_pins(struct pinctrl_dev *pctldev,
+				    unsigned int group_idx,
+				    const unsigned int **pins,
+				    unsigned int *num_pins)
+{
+	struct lochnagar_pin_priv *priv = pinctrl_dev_get_drvdata(pctldev);
+
+	*pins = priv->groups[group_idx].pins;
+	*num_pins = priv->groups[group_idx].npins;
+
+	return 0;
+}
+
+static const struct pinctrl_ops lochnagar_pin_group_ops = {
+	.get_groups_count = lochnagar_get_groups_count,
+	.get_group_name = lochnagar_get_group_name,
+	.get_group_pins = lochnagar_get_group_pins,
+	.dt_node_to_map = pinconf_generic_dt_node_to_map_all,
+	.dt_free_map = pinctrl_utils_free_map,
+};
+
+static int lochnagar_get_funcs_count(struct pinctrl_dev *pctldev)
+{
+	struct lochnagar_pin_priv *priv = pinctrl_dev_get_drvdata(pctldev);
+
+	return priv->nfuncs;
+}
+
+static const char *lochnagar_get_func_name(struct pinctrl_dev *pctldev,
+					   unsigned int func_idx)
+{
+	struct lochnagar_pin_priv *priv = pinctrl_dev_get_drvdata(pctldev);
+
+	return priv->funcs[func_idx].name;
+}
+
+static int lochnagar_get_func_groups(struct pinctrl_dev *pctldev,
+				     unsigned int func_idx,
+				     const char * const **groups,
+				     unsigned int * const num_groups)
+{
+	struct lochnagar_pin_priv *priv = pinctrl_dev_get_drvdata(pctldev);
+	int func_type;
+
+	func_type = priv->funcs[func_idx].type;
+
+	*groups = priv->func_groups[func_type].groups;
+	*num_groups = priv->func_groups[func_type].ngroups;
+
+	return 0;
+}
+
+static int lochnagar2_get_gpio_chan(struct lochnagar_pin_priv *priv,
+				    unsigned int op)
+{
+	struct regmap *regmap = priv->lochnagar->regmap;
+	unsigned int val;
+	int free = -1;
+	int i, ret;
+
+	for (i = 0; i < LN2_NUM_GPIO_CHANNELS; i++) {
+		ret = regmap_read(regmap, LOCHNAGAR2_GPIO_CHANNEL1 + i, &val);
+		if (ret)
+			return ret;
+
+		val &= LOCHNAGAR2_GPIO_CHANNEL_SRC_MASK;
+
+		if (val == op)
+			return i + 1;
+
+		if (free < 0 && !val)
+			free = i;
+	}
+
+	if (free >= 0) {
+		ret = regmap_update_bits(regmap,
+					 LOCHNAGAR2_GPIO_CHANNEL1 + free,
+					 LOCHNAGAR2_GPIO_CHANNEL_SRC_MASK, op);
+		if (ret)
+			return ret;
+
+		free++;
+
+		dev_dbg(priv->dev, "Set channel %d to 0x%x\n", free, op);
+
+		return free;
+	}
+
+	return -ENOSPC;
+}
+
+static int lochnagar_pin_set_mux(struct lochnagar_pin_priv *priv,
+				 const struct lochnagar_pin *pin,
+				 unsigned int op)
+{
+	int ret;
+
+	switch (priv->lochnagar->type) {
+	case LOCHNAGAR1:
+		break;
+	default:
+		ret = lochnagar2_get_gpio_chan(priv, op);
+		if (ret < 0) {
+			dev_err(priv->dev, "Failed to get channel for %s: %d\n",
+				pin->name, ret);
+			return ret;
+		}
+
+		op = ret;
+		break;
+	}
+
+	dev_dbg(priv->dev, "Set pin %s to 0x%x\n", pin->name, op);
+
+	ret = regmap_write(priv->lochnagar->regmap, pin->reg, op);
+	if (ret)
+		dev_err(priv->dev, "Failed to set %s mux: %d\n",
+			pin->name, ret);
+
+	return 0;
+}
+
+static int lochnagar_aif_set_mux(struct lochnagar_pin_priv *priv,
+				 const struct lochnagar_group *group,
+				 unsigned int op)
+{
+	struct regmap *regmap = priv->lochnagar->regmap;
+	const struct lochnagar_aif *aif = group->priv;
+	const struct lochnagar_pin *pin;
+	int i, ret;
+
+	ret = regmap_update_bits(regmap, aif->src_reg, aif->src_mask, op);
+	if (ret) {
+		dev_err(priv->dev, "Failed to set %s source: %d\n",
+			group->name, ret);
+		return ret;
+	}
+
+	ret = regmap_update_bits(regmap, aif->ctrl_reg,
+				 aif->ena_mask, aif->ena_mask);
+	if (ret) {
+		dev_err(priv->dev, "Failed to set %s enable: %d\n",
+			group->name, ret);
+		return ret;
+	}
+
+	for (i = 0; i < group->npins; i++) {
+		pin = priv->pins[group->pins[i]].drv_data;
+
+		if (pin->type != LN_PTYPE_MUX)
+			continue;
+
+		dev_dbg(priv->dev, "Set pin %s to AIF\n", pin->name);
+
+		ret = regmap_update_bits(regmap, pin->reg,
+					 LOCHNAGAR2_GPIO_SRC_MASK,
+					 LN2_OP_AIF);
+		if (ret) {
+			dev_err(priv->dev, "Failed to set %s to AIF: %d\n",
+				pin->name, ret);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int lochnagar_set_mux(struct pinctrl_dev *pctldev,
+			     unsigned int func_idx, unsigned int group_idx)
+{
+	struct lochnagar_pin_priv *priv = pinctrl_dev_get_drvdata(pctldev);
+	const struct lochnagar_func *func = &priv->funcs[func_idx];
+	const struct lochnagar_group *group = &priv->groups[group_idx];
+	const struct lochnagar_pin *pin;
+
+	switch (func->type) {
+	case LN_FTYPE_AIF:
+		dev_dbg(priv->dev, "Set group %s to %s\n",
+			group->name, func->name);
+
+		return lochnagar_aif_set_mux(priv, group, func->op);
+	case LN_FTYPE_PIN:
+		pin = priv->pins[*group->pins].drv_data;
+
+		dev_dbg(priv->dev, "Set pin %s to %s\n", pin->name, func->name);
+
+		return lochnagar_pin_set_mux(priv, pin, func->op);
+	default:
+		return -EINVAL;
+	}
+}
+
+static int lochnagar_gpio_request(struct pinctrl_dev *pctldev,
+				  struct pinctrl_gpio_range *range,
+				  unsigned int offset)
+{
+	struct lochnagar_pin_priv *priv = pinctrl_dev_get_drvdata(pctldev);
+	struct lochnagar *lochnagar = priv->lochnagar;
+	const struct lochnagar_pin *pin = priv->pins[offset].drv_data;
+	int ret;
+
+	dev_dbg(priv->dev, "Requesting GPIO %s\n", pin->name);
+
+	if (lochnagar->type == LOCHNAGAR1 || pin->type != LN_PTYPE_MUX)
+		return 0;
+
+	ret = lochnagar2_get_gpio_chan(priv, LN2_OP_GPIO);
+	if (ret < 0) {
+		dev_err(priv->dev, "Failed to get low channel: %d\n", ret);
+		return ret;
+	}
+
+	ret = lochnagar2_get_gpio_chan(priv, LN2_OP_GPIO | 0x1);
+	if (ret < 0) {
+		dev_err(priv->dev, "Failed to get high channel: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int lochnagar_gpio_set_direction(struct pinctrl_dev *pctldev,
+					struct pinctrl_gpio_range *range,
+					unsigned int offset,
+					bool input)
+{
+	/* The GPIOs only support output */
+	if (input)
+		return -EINVAL;
+
+	return 0;
+}
+
+static const struct pinmux_ops lochnagar_pin_mux_ops = {
+	.get_functions_count = lochnagar_get_funcs_count,
+	.get_function_name = lochnagar_get_func_name,
+	.get_function_groups = lochnagar_get_func_groups,
+	.set_mux = lochnagar_set_mux,
+
+	.gpio_request_enable = lochnagar_gpio_request,
+	.gpio_set_direction = lochnagar_gpio_set_direction,
+
+	.strict = true,
+};
+
+static int lochnagar_aif_set_master(struct lochnagar_pin_priv *priv,
+				    unsigned int group_idx, bool master)
+{
+	struct regmap *regmap = priv->lochnagar->regmap;
+	const struct lochnagar_group *group = &priv->groups[group_idx];
+	const struct lochnagar_aif *aif = group->priv;
+	unsigned int val = 0;
+	int ret;
+
+	if (group->type != LN_FTYPE_AIF)
+		return -EINVAL;
+
+	if (!master)
+		val = aif->master_mask;
+
+	dev_dbg(priv->dev, "Set AIF %s to %s\n",
+		group->name, master ? "master" : "slave");
+
+	ret = regmap_update_bits(regmap, aif->ctrl_reg, aif->master_mask, val);
+	if (ret) {
+		dev_err(priv->dev, "Failed to set %s mode: %d\n",
+			group->name, ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int lochnagar_conf_group_set(struct pinctrl_dev *pctldev,
+				    unsigned int group_idx,
+				    unsigned long *configs,
+				    unsigned int num_configs)
+{
+	struct lochnagar_pin_priv *priv = pinctrl_dev_get_drvdata(pctldev);
+	int i, ret;
+
+	for (i = 0; i < num_configs; i++) {
+		unsigned int param = pinconf_to_config_param(*configs);
+
+		switch (param) {
+		case PIN_CONFIG_OUTPUT_ENABLE:
+			ret = lochnagar_aif_set_master(priv, group_idx, true);
+			if (ret)
+				return ret;
+			break;
+		case PIN_CONFIG_INPUT_ENABLE:
+			ret = lochnagar_aif_set_master(priv, group_idx, false);
+			if (ret)
+				return ret;
+			break;
+		default:
+			return -ENOTSUPP;
+		}
+
+		configs++;
+	}
+
+	return 0;
+}
+
+static const struct pinconf_ops lochnagar_pin_conf_ops = {
+	.pin_config_group_set = lochnagar_conf_group_set,
+};
+
+static const struct pinctrl_desc lochnagar_pin_desc = {
+	.name = "lochnagar-pinctrl",
+	.owner = THIS_MODULE,
+
+	.pctlops = &lochnagar_pin_group_ops,
+	.pmxops = &lochnagar_pin_mux_ops,
+	.confops = &lochnagar_pin_conf_ops,
+};
+
+static void lochnagar_gpio_set(struct gpio_chip *chip,
+			       unsigned int offset, int value)
+{
+	struct lochnagar_pin_priv *priv = gpiochip_get_data(chip);
+	struct lochnagar *lochnagar = priv->lochnagar;
+	const struct lochnagar_pin *pin = priv->pins[offset].drv_data;
+	int ret;
+
+	value = !!value;
+
+	dev_dbg(priv->dev, "Set GPIO %s to %s\n",
+		pin->name, value ? "high" : "low");
+
+	switch (pin->type) {
+	case LN_PTYPE_MUX:
+		value |= LN2_OP_GPIO;
+
+		ret = lochnagar_pin_set_mux(priv, pin, value);
+		break;
+	case LN_PTYPE_GPIO:
+		if (pin->invert)
+			value = !value;
+
+		ret = regmap_update_bits(lochnagar->regmap, pin->reg,
+					 BIT(pin->shift), value << pin->shift);
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	if (ret < 0)
+		dev_err(chip->parent, "Failed to set %s value: %d\n",
+			pin->name, ret);
+}
+
+static int lochnagar_gpio_direction_out(struct gpio_chip *chip,
+					unsigned int offset, int value)
+{
+	lochnagar_gpio_set(chip, offset, value);
+
+	return pinctrl_gpio_direction_output(chip->base + offset);
+}
+
+static int lochnagar_fill_func_groups(struct lochnagar_pin_priv *priv)
+{
+	struct lochnagar_func_groups *funcs;
+	int i;
+
+	for (i = 0; i < priv->ngroups; i++)
+		priv->func_groups[priv->groups[i].type].ngroups++;
+
+	for (i = 0; i < LN_FTYPE_COUNT; i++) {
+		funcs = &priv->func_groups[i];
+
+		if (!funcs->ngroups)
+			continue;
+
+		funcs->groups = devm_kcalloc(priv->dev, funcs->ngroups,
+					     sizeof(*funcs->groups),
+					     GFP_KERNEL);
+		if (!funcs->groups)
+			return -ENOMEM;
+
+		funcs->ngroups = 0;
+	}
+
+	for (i = 0; i < priv->ngroups; i++) {
+		funcs = &priv->func_groups[priv->groups[i].type];
+
+		funcs->groups[funcs->ngroups++] = priv->groups[i].name;
+	}
+
+	return 0;
+}
+
+static int lochnagar_pin_probe(struct platform_device *pdev)
+{
+	struct lochnagar *lochnagar = dev_get_drvdata(pdev->dev.parent);
+	struct lochnagar_pin_priv *priv;
+	struct pinctrl_desc *desc;
+	struct pinctrl_dev *pctl;
+	struct device *dev = &pdev->dev;
+	int ret;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->dev = dev;
+	priv->lochnagar = lochnagar;
+
+	desc = devm_kzalloc(dev, sizeof(*desc), GFP_KERNEL);
+	if (!desc)
+		return -ENOMEM;
+
+	*desc = lochnagar_pin_desc;
+
+	priv->gpio_chip.label = dev_name(dev);
+	priv->gpio_chip.request = gpiochip_generic_request;
+	priv->gpio_chip.free = gpiochip_generic_free;
+	priv->gpio_chip.direction_output = lochnagar_gpio_direction_out;
+	priv->gpio_chip.set = lochnagar_gpio_set;
+	priv->gpio_chip.can_sleep = true;
+	priv->gpio_chip.parent = dev;
+	priv->gpio_chip.base = -1;
+#ifdef CONFIG_OF_GPIO
+	priv->gpio_chip.of_node = dev->of_node;
+#endif
+
+	switch (lochnagar->type) {
+	case LOCHNAGAR1:
+		priv->funcs = lochnagar1_funcs;
+		priv->nfuncs = ARRAY_SIZE(lochnagar1_funcs);
+		priv->pins = lochnagar1_pins;
+		priv->npins = ARRAY_SIZE(lochnagar1_pins);
+		priv->groups = lochnagar1_groups;
+		priv->ngroups = ARRAY_SIZE(lochnagar1_groups);
+
+		priv->gpio_chip.ngpio = LOCHNAGAR1_PIN_NUM_GPIOS;
+		break;
+	case LOCHNAGAR2:
+		priv->funcs = lochnagar2_funcs;
+		priv->nfuncs = ARRAY_SIZE(lochnagar2_funcs);
+		priv->pins = lochnagar2_pins;
+		priv->npins = ARRAY_SIZE(lochnagar2_pins);
+		priv->groups = lochnagar2_groups;
+		priv->ngroups = ARRAY_SIZE(lochnagar2_groups);
+
+		priv->gpio_chip.ngpio = LOCHNAGAR2_PIN_NUM_GPIOS;
+		break;
+	default:
+		dev_err(dev, "Unknown Lochnagar type: %d\n", lochnagar->type);
+		return -EINVAL;
+	}
+
+	ret = lochnagar_fill_func_groups(priv);
+	if (ret < 0)
+		return ret;
+
+	desc->pins = priv->pins;
+	desc->npins = priv->npins;
+
+	pctl = devm_pinctrl_register(dev, desc, priv);
+	if (IS_ERR(pctl)) {
+		ret = PTR_ERR(pctl);
+		dev_err(priv->dev, "Failed to register pinctrl: %d\n", ret);
+		return ret;
+	}
+
+	ret = devm_gpiochip_add_data(dev, &priv->gpio_chip, priv);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "Failed to register gpiochip: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static const struct of_device_id lochnagar_of_match[] = {
+	{ .compatible = "cirrus,lochnagar-pinctrl" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, lochnagar_of_match);
+
+static struct platform_driver lochnagar_pin_driver = {
+	.driver = {
+		.name = "lochnagar-pinctrl",
+		.of_match_table = of_match_ptr(lochnagar_of_match),
+	},
+
+	.probe = lochnagar_pin_probe,
+};
+module_platform_driver(lochnagar_pin_driver);
+
+MODULE_AUTHOR("Charles Keepax <ckeepax@opensource.cirrus.com>");
+MODULE_DESCRIPTION("Pinctrl driver for Cirrus Logic Lochnagar Board");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/pinctrl/freescale/pinctrl-imx.c b/drivers/pinctrl/freescale/pinctrl-imx.c
index 188001beb298..83ff9532bae6 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx.c
+++ b/drivers/pinctrl/freescale/pinctrl-imx.c
@@ -449,7 +449,7 @@ static void imx_pinconf_dbg_show(struct pinctrl_dev *pctldev,
 		}
 	} else {
 		pin_reg = &ipctl->pin_regs[pin_id];
-		if (!pin_reg || pin_reg->conf_reg == -1) {
+		if (pin_reg->conf_reg == -1) {
 			seq_puts(s, "N/A");
 			return;
 		}
@@ -785,7 +785,6 @@ int imx_pinctrl_probe(struct platform_device *pdev,
 	struct pinctrl_desc *imx_pinctrl_desc;
 	struct device_node *np;
 	struct imx_pinctrl *ipctl;
-	struct resource *res;
 	struct regmap *gpr;
 	int ret, i;
 
@@ -817,8 +816,7 @@ int imx_pinctrl_probe(struct platform_device *pdev,
 			ipctl->pin_regs[i].conf_reg = -1;
 		}
 
-		res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-		ipctl->base = devm_ioremap_resource(&pdev->dev, res);
+		ipctl->base = devm_platform_ioremap_resource(pdev, 0);
 		if (IS_ERR(ipctl->base))
 			return PTR_ERR(ipctl->base);
 
@@ -887,3 +885,22 @@ free:
 
 	return ret;
 }
+
+static int __maybe_unused imx_pinctrl_suspend(struct device *dev)
+{
+	struct imx_pinctrl *ipctl = dev_get_drvdata(dev);
+
+	return pinctrl_force_sleep(ipctl->pctl);
+}
+
+static int __maybe_unused imx_pinctrl_resume(struct device *dev)
+{
+	struct imx_pinctrl *ipctl = dev_get_drvdata(dev);
+
+	return pinctrl_force_default(ipctl->pctl);
+}
+
+const struct dev_pm_ops imx_pinctrl_pm_ops = {
+	SET_LATE_SYSTEM_SLEEP_PM_OPS(imx_pinctrl_suspend,
+					imx_pinctrl_resume)
+};
diff --git a/drivers/pinctrl/freescale/pinctrl-imx.h b/drivers/pinctrl/freescale/pinctrl-imx.h
index 98a4889af4ef..333d32b947b1 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx.h
+++ b/drivers/pinctrl/freescale/pinctrl-imx.h
@@ -17,6 +17,7 @@
 struct platform_device;
 
 extern struct pinmux_ops imx_pmx_ops;
+extern const struct dev_pm_ops imx_pinctrl_pm_ops;
 
 /**
  * struct imx_pin_mmio - MMIO pin configurations
diff --git a/drivers/pinctrl/freescale/pinctrl-imx8mq.c b/drivers/pinctrl/freescale/pinctrl-imx8mq.c
index 8d39af541d5f..50aa1c00c4b2 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx8mq.c
+++ b/drivers/pinctrl/freescale/pinctrl-imx8mq.c
@@ -339,6 +339,7 @@ static struct platform_driver imx8mq_pinctrl_driver = {
 	.driver = {
 		.name = "imx8mq-pinctrl",
 		.of_match_table = of_match_ptr(imx8mq_pinctrl_of_match),
+		.pm = &imx_pinctrl_pm_ops,
 		.suppress_bind_attrs = true,
 	},
 	.probe = imx8mq_pinctrl_probe,
diff --git a/drivers/pinctrl/freescale/pinctrl-scu.c b/drivers/pinctrl/freescale/pinctrl-scu.c
index 83e69c0f39e6..73bf1d9f9cc6 100644
--- a/drivers/pinctrl/freescale/pinctrl-scu.c
+++ b/drivers/pinctrl/freescale/pinctrl-scu.c
@@ -35,7 +35,7 @@ struct imx_sc_msg_resp_pad_get {
 	u32 val;
 } __packed;
 
-struct imx_sc_ipc *pinctrl_ipc_handle;
+static struct imx_sc_ipc *pinctrl_ipc_handle;
 
 int imx_pinctrl_sc_ipc_init(struct platform_device *pdev)
 {
diff --git a/drivers/pinctrl/intel/pinctrl-baytrail.c b/drivers/pinctrl/intel/pinctrl-baytrail.c
index 241384ead4ed..18d9ad504194 100644
--- a/drivers/pinctrl/intel/pinctrl-baytrail.c
+++ b/drivers/pinctrl/intel/pinctrl-baytrail.c
@@ -1710,6 +1710,8 @@ static int byt_gpio_probe(struct byt_gpio *vg)
 #ifdef CONFIG_PM_SLEEP
 	vg->saved_context = devm_kcalloc(&vg->pdev->dev, gc->ngpio,
 				       sizeof(*vg->saved_context), GFP_KERNEL);
+	if (!vg->saved_context)
+		return -ENOMEM;
 #endif
 	ret = devm_gpiochip_add_data(&vg->pdev->dev, gc, vg);
 	if (ret) {
diff --git a/drivers/pinctrl/intel/pinctrl-cedarfork.c b/drivers/pinctrl/intel/pinctrl-cedarfork.c
index b7d632f1dbf6..aa6f9040d3d8 100644
--- a/drivers/pinctrl/intel/pinctrl-cedarfork.c
+++ b/drivers/pinctrl/intel/pinctrl-cedarfork.c
@@ -91,13 +91,13 @@ static const struct pinctrl_pin_desc cdf_pins[] = {
 	PINCTRL_PIN(43, "MEMTRIP_N"),
 	PINCTRL_PIN(44, "UART0_RXD"),
 	PINCTRL_PIN(45, "UART0_TXD"),
-	PINCTRL_PIN(46, "UART1_RXD"),
-	PINCTRL_PIN(47, "UART1_TXD"),
+	PINCTRL_PIN(46, "GBE_UART_RXD"),
+	PINCTRL_PIN(47, "GBE_UART_TXD"),
 	/* WEST01 */
 	PINCTRL_PIN(48, "GBE_GPIO13"),
 	PINCTRL_PIN(49, "AUX_PWR"),
-	PINCTRL_PIN(50, "CPU_GP_2"),
-	PINCTRL_PIN(51, "CPU_GP_3"),
+	PINCTRL_PIN(50, "UART0_RTS"),
+	PINCTRL_PIN(51, "UART0_CTS"),
 	PINCTRL_PIN(52, "FAN_PWM_0"),
 	PINCTRL_PIN(53, "FAN_PWM_1"),
 	PINCTRL_PIN(54, "FAN_PWM_2"),
@@ -201,8 +201,8 @@ static const struct pinctrl_pin_desc cdf_pins[] = {
 	/* WESTF */
 	PINCTRL_PIN(145, "NAC_RMII_CLK"),
 	PINCTRL_PIN(146, "NAC_RGMII_CLK"),
-	PINCTRL_PIN(147, "NAC_SPARE0"),
-	PINCTRL_PIN(148, "NAC_SPARE1"),
+	PINCTRL_PIN(147, "NAC_GBE_SMB_CLK_TX_N2S"),
+	PINCTRL_PIN(148, "NAC_GBE_SMB_DATA_TX_N2S"),
 	PINCTRL_PIN(149, "NAC_SPARE2"),
 	PINCTRL_PIN(150, "NAC_INIT_SX_WAKE_N"),
 	PINCTRL_PIN(151, "NAC_GBE_GPIO0_S2N"),
@@ -219,8 +219,8 @@ static const struct pinctrl_pin_desc cdf_pins[] = {
 	PINCTRL_PIN(162, "NAC_NCSI_TXD1"),
 	PINCTRL_PIN(163, "NAC_NCSI_ARB_OUT"),
 	PINCTRL_PIN(164, "NAC_NCSI_OE_N"),
-	PINCTRL_PIN(165, "NAC_GBE_SMB_CLK"),
-	PINCTRL_PIN(166, "NAC_GBE_SMB_DATA"),
+	PINCTRL_PIN(165, "NAC_GBE_SMB_CLK_RX_S2N"),
+	PINCTRL_PIN(166, "NAC_GBE_SMB_DATA_RX_S2N"),
 	PINCTRL_PIN(167, "NAC_GBE_SMB_ALRT_N"),
 	/* EAST2 */
 	PINCTRL_PIN(168, "USB_OC0_N"),
@@ -232,7 +232,7 @@ static const struct pinctrl_pin_desc cdf_pins[] = {
 	PINCTRL_PIN(174, "GBE_GPIO5"),
 	PINCTRL_PIN(175, "GBE_GPIO6"),
 	PINCTRL_PIN(176, "GBE_GPIO7"),
-	PINCTRL_PIN(177, "GBE_GPIO8"),
+	PINCTRL_PIN(177, "SPI_TPM_CS_N"),
 	PINCTRL_PIN(178, "GBE_GPIO9"),
 	PINCTRL_PIN(179, "GBE_GPIO10"),
 	PINCTRL_PIN(180, "GBE_GPIO11"),
diff --git a/drivers/pinctrl/intel/pinctrl-intel.c b/drivers/pinctrl/intel/pinctrl-intel.c
index 3b1818184207..d7acbb79cdf7 100644
--- a/drivers/pinctrl/intel/pinctrl-intel.c
+++ b/drivers/pinctrl/intel/pinctrl-intel.c
@@ -81,6 +81,7 @@ struct intel_pad_context {
 
 struct intel_community_context {
 	u32 *intmask;
+	u32 *hostown;
 };
 
 struct intel_pinctrl_context {
@@ -1284,7 +1285,7 @@ static int intel_pinctrl_pm_init(struct intel_pinctrl *pctrl)
 
 	for (i = 0; i < pctrl->ncommunities; i++) {
 		struct intel_community *community = &pctrl->communities[i];
-		u32 *intmask;
+		u32 *intmask, *hostown;
 
 		intmask = devm_kcalloc(pctrl->dev, community->ngpps,
 				       sizeof(*intmask), GFP_KERNEL);
@@ -1292,6 +1293,13 @@ static int intel_pinctrl_pm_init(struct intel_pinctrl *pctrl)
 			return -ENOMEM;
 
 		communities[i].intmask = intmask;
+
+		hostown = devm_kcalloc(pctrl->dev, community->ngpps,
+				       sizeof(*hostown), GFP_KERNEL);
+		if (!hostown)
+			return -ENOMEM;
+
+		communities[i].hostown = hostown;
 	}
 
 	pctrl->context.pads = pads;
@@ -1466,7 +1474,7 @@ static bool intel_pinctrl_should_save(struct intel_pinctrl *pctrl, unsigned int
 	return false;
 }
 
-int intel_pinctrl_suspend(struct device *dev)
+int intel_pinctrl_suspend_noirq(struct device *dev)
 {
 	struct intel_pinctrl *pctrl = dev_get_drvdata(dev);
 	struct intel_community_context *communities;
@@ -1501,11 +1509,15 @@ int intel_pinctrl_suspend(struct device *dev)
 		base = community->regs + community->ie_offset;
 		for (gpp = 0; gpp < community->ngpps; gpp++)
 			communities[i].intmask[gpp] = readl(base + gpp * 4);
+
+		base = community->regs + community->hostown_offset;
+		for (gpp = 0; gpp < community->ngpps; gpp++)
+			communities[i].hostown[gpp] = readl(base + gpp * 4);
 	}
 
 	return 0;
 }
-EXPORT_SYMBOL_GPL(intel_pinctrl_suspend);
+EXPORT_SYMBOL_GPL(intel_pinctrl_suspend_noirq);
 
 static void intel_gpio_irq_init(struct intel_pinctrl *pctrl)
 {
@@ -1527,7 +1539,32 @@ static void intel_gpio_irq_init(struct intel_pinctrl *pctrl)
 	}
 }
 
-int intel_pinctrl_resume(struct device *dev)
+static u32
+intel_gpio_is_requested(struct gpio_chip *chip, int base, unsigned int size)
+{
+	u32 requested = 0;
+	unsigned int i;
+
+	for (i = 0; i < size; i++)
+		if (gpiochip_is_requested(chip, base + i))
+			requested |= BIT(i);
+
+	return requested;
+}
+
+static u32
+intel_gpio_update_pad_mode(void __iomem *hostown, u32 mask, u32 value)
+{
+	u32 curr, updated;
+
+	curr = readl(hostown);
+	updated = (curr & ~mask) | (value & mask);
+	writel(updated, hostown);
+
+	return curr;
+}
+
+int intel_pinctrl_resume_noirq(struct device *dev)
 {
 	struct intel_pinctrl *pctrl = dev_get_drvdata(dev);
 	const struct intel_community_context *communities;
@@ -1585,11 +1622,30 @@ int intel_pinctrl_resume(struct device *dev)
 			dev_dbg(dev, "restored mask %d/%u %#08x\n", i, gpp,
 				readl(base + gpp * 4));
 		}
+
+		base = community->regs + community->hostown_offset;
+		for (gpp = 0; gpp < community->ngpps; gpp++) {
+			const struct intel_padgroup *padgrp = &community->gpps[gpp];
+			u32 requested = 0, value = 0;
+			u32 saved = communities[i].hostown[gpp];
+
+			if (padgrp->gpio_base < 0)
+				continue;
+
+			requested = intel_gpio_is_requested(&pctrl->chip,
+					padgrp->gpio_base, padgrp->size);
+			value = intel_gpio_update_pad_mode(base + gpp * 4,
+					requested, saved);
+			if ((value ^ saved) & requested) {
+				dev_warn(dev, "restore hostown %d/%u %#8x->%#8x\n",
+					i, gpp, value, saved);
+			}
+		}
 	}
 
 	return 0;
 }
-EXPORT_SYMBOL_GPL(intel_pinctrl_resume);
+EXPORT_SYMBOL_GPL(intel_pinctrl_resume_noirq);
 #endif
 
 MODULE_AUTHOR("Mathias Nyman <mathias.nyman@linux.intel.com>");
diff --git a/drivers/pinctrl/intel/pinctrl-intel.h b/drivers/pinctrl/intel/pinctrl-intel.h
index b8a07d37d18f..a8e958f1dcf5 100644
--- a/drivers/pinctrl/intel/pinctrl-intel.h
+++ b/drivers/pinctrl/intel/pinctrl-intel.h
@@ -177,13 +177,14 @@ int intel_pinctrl_probe_by_hid(struct platform_device *pdev);
 int intel_pinctrl_probe_by_uid(struct platform_device *pdev);
 
 #ifdef CONFIG_PM_SLEEP
-int intel_pinctrl_suspend(struct device *dev);
-int intel_pinctrl_resume(struct device *dev);
+int intel_pinctrl_suspend_noirq(struct device *dev);
+int intel_pinctrl_resume_noirq(struct device *dev);
 #endif
 
-#define INTEL_PINCTRL_PM_OPS(_name)						  \
-const struct dev_pm_ops _name = {						  \
-	SET_LATE_SYSTEM_SLEEP_PM_OPS(intel_pinctrl_suspend, intel_pinctrl_resume) \
+#define INTEL_PINCTRL_PM_OPS(_name)					\
+const struct dev_pm_ops _name = {					\
+	SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(intel_pinctrl_suspend_noirq,	\
+				      intel_pinctrl_resume_noirq)	\
 }
 
 #endif /* PINCTRL_INTEL_H */
diff --git a/drivers/pinctrl/mediatek/Kconfig b/drivers/pinctrl/mediatek/Kconfig
index a005cbccb4f7..26ed5dca1460 100644
--- a/drivers/pinctrl/mediatek/Kconfig
+++ b/drivers/pinctrl/mediatek/Kconfig
@@ -113,6 +113,13 @@ config PINCTRL_MT8183
 	default ARM64 && ARCH_MEDIATEK
 	select PINCTRL_MTK_PARIS
 
+config PINCTRL_MT8516
+	bool "Mediatek MT8516 pin control"
+	depends on OF
+	depends on ARM64 || COMPILE_TEST
+	default ARM64 && ARCH_MEDIATEK
+	select PINCTRL_MTK
+
 # For PMIC
 config PINCTRL_MT6397
 	bool "Mediatek MT6397 pin control"
diff --git a/drivers/pinctrl/mediatek/Makefile b/drivers/pinctrl/mediatek/Makefile
index 4b4e2eaf6f2d..a74325abd877 100644
--- a/drivers/pinctrl/mediatek/Makefile
+++ b/drivers/pinctrl/mediatek/Makefile
@@ -17,4 +17,5 @@ obj-$(CONFIG_PINCTRL_MT7623)	+= pinctrl-mt7623.o
 obj-$(CONFIG_PINCTRL_MT7629)	+= pinctrl-mt7629.o
 obj-$(CONFIG_PINCTRL_MT8173)	+= pinctrl-mt8173.o
 obj-$(CONFIG_PINCTRL_MT8183)	+= pinctrl-mt8183.o
+obj-$(CONFIG_PINCTRL_MT8516)	+= pinctrl-mt8516.o
 obj-$(CONFIG_PINCTRL_MT6397)	+= pinctrl-mt6397.o
diff --git a/drivers/pinctrl/mediatek/pinctrl-mt8183.c b/drivers/pinctrl/mediatek/pinctrl-mt8183.c
index 6262fd3678ea..2c7409ed16fa 100644
--- a/drivers/pinctrl/mediatek/pinctrl-mt8183.c
+++ b/drivers/pinctrl/mediatek/pinctrl-mt8183.c
@@ -472,6 +472,51 @@ static const struct mtk_pin_field_calc mt8183_pin_r1_range[] = {
 	PIN_FIELD_BASE(133, 133, 8, 0x0D0, 0x10, 13, 1),
 };
 
+static const struct mtk_pin_field_calc mt8183_pin_e1e0en_range[] = {
+	PIN_FIELD_BASE(48, 48, 3, 0x0F0, 0x10, 20, 1),
+	PIN_FIELD_BASE(49, 49, 3, 0x0F0, 0x10, 15, 1),
+	PIN_FIELD_BASE(50, 50, 4, 0x0F0, 0x10, 12, 1),
+	PIN_FIELD_BASE(51, 51, 4, 0x0F0, 0x10, 7, 1),
+	PIN_FIELD_BASE(81, 81, 5, 0x0F0, 0x10, 12, 1),
+	PIN_FIELD_BASE(82, 82, 5, 0x0F0, 0x10, 9, 1),
+	PIN_FIELD_BASE(83, 83, 5, 0x0F0, 0x10, 19, 1),
+	PIN_FIELD_BASE(84, 84, 5, 0x0F0, 0x10, 22, 1),
+	PIN_FIELD_BASE(103, 103, 6, 0x0F0, 0x10, 24, 1),
+	PIN_FIELD_BASE(104, 104, 6, 0x0F0, 0x10, 14, 1),
+	PIN_FIELD_BASE(105, 105, 6, 0x0F0, 0x10, 27, 1),
+	PIN_FIELD_BASE(106, 106, 6, 0x0F0, 0x10, 17, 1),
+};
+
+static const struct mtk_pin_field_calc mt8183_pin_e0_range[] = {
+	PIN_FIELD_BASE(48, 48, 3, 0x0F0, 0x10, 21, 1),
+	PIN_FIELD_BASE(49, 49, 3, 0x0F0, 0x10, 16, 1),
+	PIN_FIELD_BASE(50, 50, 4, 0x0F0, 0x10, 13, 1),
+	PIN_FIELD_BASE(51, 51, 4, 0x0F0, 0x10, 8, 1),
+	PIN_FIELD_BASE(81, 81, 5, 0x0F0, 0x10, 13, 1),
+	PIN_FIELD_BASE(82, 82, 5, 0x0F0, 0x10, 10, 1),
+	PIN_FIELD_BASE(83, 83, 5, 0x0F0, 0x10, 20, 1),
+	PIN_FIELD_BASE(84, 84, 5, 0x0F0, 0x10, 23, 1),
+	PIN_FIELD_BASE(103, 103, 6, 0x0F0, 0x10, 25, 1),
+	PIN_FIELD_BASE(104, 104, 6, 0x0F0, 0x10, 15, 1),
+	PIN_FIELD_BASE(105, 105, 6, 0x0F0, 0x10, 28, 1),
+	PIN_FIELD_BASE(106, 106, 6, 0x0F0, 0x10, 18, 1),
+};
+
+static const struct mtk_pin_field_calc mt8183_pin_e1_range[] = {
+	PIN_FIELD_BASE(48, 48, 3, 0x0F0, 0x10, 22, 1),
+	PIN_FIELD_BASE(49, 49, 3, 0x0F0, 0x10, 17, 1),
+	PIN_FIELD_BASE(50, 50, 4, 0x0F0, 0x10, 14, 1),
+	PIN_FIELD_BASE(51, 51, 4, 0x0F0, 0x10, 9, 1),
+	PIN_FIELD_BASE(81, 81, 5, 0x0F0, 0x10, 14, 1),
+	PIN_FIELD_BASE(82, 82, 5, 0x0F0, 0x10, 11, 1),
+	PIN_FIELD_BASE(83, 83, 5, 0x0F0, 0x10, 21, 1),
+	PIN_FIELD_BASE(84, 84, 5, 0x0F0, 0x10, 24, 1),
+	PIN_FIELD_BASE(103, 103, 6, 0x0F0, 0x10, 26, 1),
+	PIN_FIELD_BASE(104, 104, 6, 0x0F0, 0x10, 16, 1),
+	PIN_FIELD_BASE(105, 105, 6, 0x0F0, 0x10, 29, 1),
+	PIN_FIELD_BASE(106, 106, 6, 0x0F0, 0x10, 19, 1),
+};
+
 static const struct mtk_pin_reg_calc mt8183_reg_cals[PINCTRL_PIN_REG_MAX] = {
 	[PINCTRL_PIN_REG_MODE] = MTK_RANGE(mt8183_pin_mode_range),
 	[PINCTRL_PIN_REG_DIR] = MTK_RANGE(mt8183_pin_dir_range),
@@ -485,6 +530,9 @@ static const struct mtk_pin_reg_calc mt8183_reg_cals[PINCTRL_PIN_REG_MAX] = {
 	[PINCTRL_PIN_REG_PUPD] = MTK_RANGE(mt8183_pin_pupd_range),
 	[PINCTRL_PIN_REG_R0] = MTK_RANGE(mt8183_pin_r0_range),
 	[PINCTRL_PIN_REG_R1] = MTK_RANGE(mt8183_pin_r1_range),
+	[PINCTRL_PIN_REG_DRV_EN] = MTK_RANGE(mt8183_pin_e1e0en_range),
+	[PINCTRL_PIN_REG_DRV_E0] = MTK_RANGE(mt8183_pin_e0_range),
+	[PINCTRL_PIN_REG_DRV_E1] = MTK_RANGE(mt8183_pin_e1_range),
 };
 
 static const char * const mt8183_pinctrl_register_base_names[] = {
@@ -517,6 +565,8 @@ static const struct mtk_pin_soc mt8183_data = {
 	.drive_get = mtk_pinconf_drive_get_rev1,
 	.adv_pull_get = mtk_pinconf_adv_pull_get,
 	.adv_pull_set = mtk_pinconf_adv_pull_set,
+	.adv_drive_get = mtk_pinconf_adv_drive_get,
+	.adv_drive_set = mtk_pinconf_adv_drive_set,
 };
 
 static const struct of_device_id mt8183_pinctrl_of_match[] = {
diff --git a/drivers/pinctrl/mediatek/pinctrl-mt8516.c b/drivers/pinctrl/mediatek/pinctrl-mt8516.c
new file mode 100644
index 000000000000..b375426aa61e
--- /dev/null
+++ b/drivers/pinctrl/mediatek/pinctrl-mt8516.c
@@ -0,0 +1,362 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2019 MediaTek Inc.
+ * Author: Min.Guo <min.guo@mediatek.com>
+ */
+
+#include <dt-bindings/pinctrl/mt65xx.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/module.h>
+#include <linux/pinctrl/pinctrl.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#include "pinctrl-mtk-common.h"
+#include "pinctrl-mtk-mt8516.h"
+
+static const struct mtk_drv_group_desc mt8516_drv_grp[] = {
+	/* 0E4E8SR 4/8/12/16 */
+	MTK_DRV_GRP(4, 16, 1, 2, 4),
+	/* 0E2E4SR  2/4/6/8 */
+	MTK_DRV_GRP(2, 8, 1, 2, 2),
+	/* E8E4E2  2/4/6/8/10/12/14/16 */
+	MTK_DRV_GRP(2, 16, 0, 2, 2)
+};
+
+static const struct mtk_pin_drv_grp mt8516_pin_drv[] = {
+	MTK_PIN_DRV_GRP(0, 0xd00, 0, 0),
+	MTK_PIN_DRV_GRP(1, 0xd00, 0, 0),
+	MTK_PIN_DRV_GRP(2, 0xd00, 0, 0),
+	MTK_PIN_DRV_GRP(3, 0xd00, 0, 0),
+	MTK_PIN_DRV_GRP(4, 0xd00, 0, 0),
+
+	MTK_PIN_DRV_GRP(5, 0xd00, 4, 0),
+	MTK_PIN_DRV_GRP(6, 0xd00, 4, 0),
+	MTK_PIN_DRV_GRP(7, 0xd00, 4, 0),
+	MTK_PIN_DRV_GRP(8, 0xd00, 4, 0),
+	MTK_PIN_DRV_GRP(9, 0xd00, 4, 0),
+	MTK_PIN_DRV_GRP(10, 0xd00, 4, 0),
+
+	MTK_PIN_DRV_GRP(11, 0xd00, 8, 0),
+	MTK_PIN_DRV_GRP(12, 0xd00, 8, 0),
+	MTK_PIN_DRV_GRP(13, 0xd00, 8, 0),
+
+	MTK_PIN_DRV_GRP(14, 0xd00, 12, 2),
+	MTK_PIN_DRV_GRP(15, 0xd00, 12, 2),
+	MTK_PIN_DRV_GRP(16, 0xd00, 12, 2),
+	MTK_PIN_DRV_GRP(17, 0xd00, 12, 2),
+
+	MTK_PIN_DRV_GRP(18, 0xd10, 0, 0),
+	MTK_PIN_DRV_GRP(19, 0xd10, 0, 0),
+	MTK_PIN_DRV_GRP(20, 0xd10, 0, 0),
+
+	MTK_PIN_DRV_GRP(21, 0xd00, 12, 2),
+	MTK_PIN_DRV_GRP(22, 0xd00, 12, 2),
+	MTK_PIN_DRV_GRP(23, 0xd00, 12, 2),
+
+	MTK_PIN_DRV_GRP(24, 0xd00, 8, 0),
+	MTK_PIN_DRV_GRP(25, 0xd00, 8, 0),
+
+	MTK_PIN_DRV_GRP(26, 0xd10, 4, 1),
+	MTK_PIN_DRV_GRP(27, 0xd10, 4, 1),
+	MTK_PIN_DRV_GRP(28, 0xd10, 4, 1),
+	MTK_PIN_DRV_GRP(29, 0xd10, 4, 1),
+	MTK_PIN_DRV_GRP(30, 0xd10, 4, 1),
+
+	MTK_PIN_DRV_GRP(31, 0xd10, 8, 1),
+	MTK_PIN_DRV_GRP(32, 0xd10, 8, 1),
+	MTK_PIN_DRV_GRP(33, 0xd10, 8, 1),
+
+	MTK_PIN_DRV_GRP(34, 0xd10, 12, 0),
+	MTK_PIN_DRV_GRP(35, 0xd10, 12, 0),
+
+	MTK_PIN_DRV_GRP(36, 0xd20, 0, 0),
+	MTK_PIN_DRV_GRP(37, 0xd20, 0, 0),
+	MTK_PIN_DRV_GRP(38, 0xd20, 0, 0),
+	MTK_PIN_DRV_GRP(39, 0xd20, 0, 0),
+
+	MTK_PIN_DRV_GRP(40, 0xd20, 4, 1),
+
+	MTK_PIN_DRV_GRP(41, 0xd20, 8, 1),
+	MTK_PIN_DRV_GRP(42, 0xd20, 8, 1),
+	MTK_PIN_DRV_GRP(43, 0xd20, 8, 1),
+
+	MTK_PIN_DRV_GRP(44, 0xd20, 12, 1),
+	MTK_PIN_DRV_GRP(45, 0xd20, 12, 1),
+	MTK_PIN_DRV_GRP(46, 0xd20, 12, 1),
+	MTK_PIN_DRV_GRP(47, 0xd20, 12, 1),
+
+	MTK_PIN_DRV_GRP(48, 0xd30, 0, 1),
+	MTK_PIN_DRV_GRP(49, 0xd30, 0, 1),
+	MTK_PIN_DRV_GRP(50, 0xd30, 0, 1),
+	MTK_PIN_DRV_GRP(51, 0xd30, 0, 1),
+
+	MTK_PIN_DRV_GRP(54, 0xd30, 8, 1),
+
+	MTK_PIN_DRV_GRP(55, 0xd30, 12, 1),
+	MTK_PIN_DRV_GRP(56, 0xd30, 12, 1),
+	MTK_PIN_DRV_GRP(57, 0xd30, 12, 1),
+
+	MTK_PIN_DRV_GRP(62, 0xd40, 8, 1),
+	MTK_PIN_DRV_GRP(63, 0xd40, 8, 1),
+	MTK_PIN_DRV_GRP(64, 0xd40, 8, 1),
+	MTK_PIN_DRV_GRP(65, 0xd40, 8, 1),
+	MTK_PIN_DRV_GRP(66, 0xd40, 8, 1),
+	MTK_PIN_DRV_GRP(67, 0xd40, 8, 1),
+
+	MTK_PIN_DRV_GRP(68, 0xd40, 12, 2),
+
+	MTK_PIN_DRV_GRP(69, 0xd50, 0, 2),
+
+	MTK_PIN_DRV_GRP(70, 0xd50, 4, 2),
+	MTK_PIN_DRV_GRP(71, 0xd50, 4, 2),
+	MTK_PIN_DRV_GRP(72, 0xd50, 4, 2),
+	MTK_PIN_DRV_GRP(73, 0xd50, 4, 2),
+
+	MTK_PIN_DRV_GRP(100, 0xd50, 8, 1),
+	MTK_PIN_DRV_GRP(101, 0xd50, 8, 1),
+	MTK_PIN_DRV_GRP(102, 0xd50, 8, 1),
+	MTK_PIN_DRV_GRP(103, 0xd50, 8, 1),
+
+	MTK_PIN_DRV_GRP(104, 0xd50, 12, 2),
+
+	MTK_PIN_DRV_GRP(105, 0xd60, 0, 2),
+
+	MTK_PIN_DRV_GRP(106, 0xd60, 4, 2),
+	MTK_PIN_DRV_GRP(107, 0xd60, 4, 2),
+	MTK_PIN_DRV_GRP(108, 0xd60, 4, 2),
+	MTK_PIN_DRV_GRP(109, 0xd60, 4, 2),
+
+	MTK_PIN_DRV_GRP(110, 0xd70, 0, 2),
+	MTK_PIN_DRV_GRP(111, 0xd70, 0, 2),
+	MTK_PIN_DRV_GRP(112, 0xd70, 0, 2),
+	MTK_PIN_DRV_GRP(113, 0xd70, 0, 2),
+
+	MTK_PIN_DRV_GRP(114, 0xd70, 4, 2),
+
+	MTK_PIN_DRV_GRP(115, 0xd60, 12, 2),
+
+	MTK_PIN_DRV_GRP(116, 0xd60, 8, 2),
+
+	MTK_PIN_DRV_GRP(117, 0xd70, 0, 2),
+	MTK_PIN_DRV_GRP(118, 0xd70, 0, 2),
+	MTK_PIN_DRV_GRP(119, 0xd70, 0, 2),
+	MTK_PIN_DRV_GRP(120, 0xd70, 0, 2),
+};
+
+static const struct mtk_pin_spec_pupd_set_samereg mt8516_spec_pupd[] = {
+	MTK_PIN_PUPD_SPEC_SR(14, 0xe50, 14, 13, 12),
+	MTK_PIN_PUPD_SPEC_SR(15, 0xe60, 2, 1, 0),
+	MTK_PIN_PUPD_SPEC_SR(16, 0xe60, 6, 5, 4),
+	MTK_PIN_PUPD_SPEC_SR(17, 0xe60, 10, 9, 8),
+
+	MTK_PIN_PUPD_SPEC_SR(21, 0xe60, 14, 13, 12),
+	MTK_PIN_PUPD_SPEC_SR(22, 0xe70, 2, 1, 0),
+	MTK_PIN_PUPD_SPEC_SR(23, 0xe70, 6, 5, 4),
+
+	MTK_PIN_PUPD_SPEC_SR(40, 0xe80, 2, 1, 0),
+	MTK_PIN_PUPD_SPEC_SR(41, 0xe80, 6, 5, 4),
+	MTK_PIN_PUPD_SPEC_SR(42, 0xe90, 2, 1, 0),
+	MTK_PIN_PUPD_SPEC_SR(43, 0xe90, 6, 5, 4),
+
+	MTK_PIN_PUPD_SPEC_SR(68, 0xe50, 10, 9, 8),
+	MTK_PIN_PUPD_SPEC_SR(69, 0xe50, 6, 5, 4),
+	MTK_PIN_PUPD_SPEC_SR(70, 0xe40, 6, 5, 4),
+	MTK_PIN_PUPD_SPEC_SR(71, 0xe40, 10, 9, 8),
+	MTK_PIN_PUPD_SPEC_SR(72, 0xe40, 14, 13, 12),
+	MTK_PIN_PUPD_SPEC_SR(73, 0xe50, 2, 1, 0),
+
+	MTK_PIN_PUPD_SPEC_SR(104, 0xe40, 2, 1, 0),
+	MTK_PIN_PUPD_SPEC_SR(105, 0xe30, 14, 13, 12),
+	MTK_PIN_PUPD_SPEC_SR(106, 0xe20, 14, 13, 12),
+	MTK_PIN_PUPD_SPEC_SR(107, 0xe30, 2, 1, 0),
+	MTK_PIN_PUPD_SPEC_SR(108, 0xe30, 6, 5, 4),
+	MTK_PIN_PUPD_SPEC_SR(109, 0xe30, 10, 9, 8),
+	MTK_PIN_PUPD_SPEC_SR(110, 0xe10, 14, 13, 12),
+	MTK_PIN_PUPD_SPEC_SR(111, 0xe10, 10, 9, 8),
+	MTK_PIN_PUPD_SPEC_SR(112, 0xe10, 6, 5, 4),
+	MTK_PIN_PUPD_SPEC_SR(113, 0xe10, 2, 1, 0),
+	MTK_PIN_PUPD_SPEC_SR(114, 0xe20, 10, 9, 8),
+	MTK_PIN_PUPD_SPEC_SR(115, 0xe20, 2, 1, 0),
+	MTK_PIN_PUPD_SPEC_SR(116, 0xe20, 6, 5, 4),
+	MTK_PIN_PUPD_SPEC_SR(117, 0xe00, 14, 13, 12),
+	MTK_PIN_PUPD_SPEC_SR(118, 0xe00, 10, 9, 8),
+	MTK_PIN_PUPD_SPEC_SR(119, 0xe00, 6, 5, 4),
+	MTK_PIN_PUPD_SPEC_SR(120, 0xe00, 2, 1, 0),
+};
+
+static int mt8516_spec_pull_set(struct regmap *regmap, unsigned int pin,
+			unsigned char align, bool isup, unsigned int r1r0)
+{
+	return mtk_pctrl_spec_pull_set_samereg(regmap, mt8516_spec_pupd,
+		ARRAY_SIZE(mt8516_spec_pupd), pin, align, isup, r1r0);
+}
+
+static const struct mtk_pin_ies_smt_set mt8516_ies_set[] = {
+	MTK_PIN_IES_SMT_SPEC(0, 6, 0x900, 2),
+	MTK_PIN_IES_SMT_SPEC(7, 10, 0x900, 3),
+	MTK_PIN_IES_SMT_SPEC(11, 13, 0x900, 12),
+	MTK_PIN_IES_SMT_SPEC(14, 17, 0x900, 13),
+	MTK_PIN_IES_SMT_SPEC(18, 20, 0x910, 10),
+	MTK_PIN_IES_SMT_SPEC(21, 23, 0x900, 13),
+	MTK_PIN_IES_SMT_SPEC(24, 25, 0x900, 12),
+	MTK_PIN_IES_SMT_SPEC(26, 30, 0x900, 0),
+	MTK_PIN_IES_SMT_SPEC(31, 33, 0x900, 1),
+	MTK_PIN_IES_SMT_SPEC(34, 39, 0x900, 2),
+	MTK_PIN_IES_SMT_SPEC(40, 40, 0x910, 11),
+	MTK_PIN_IES_SMT_SPEC(41, 43, 0x900, 10),
+	MTK_PIN_IES_SMT_SPEC(44, 47, 0x900, 11),
+	MTK_PIN_IES_SMT_SPEC(48, 51, 0x900, 14),
+	MTK_PIN_IES_SMT_SPEC(52, 53, 0x910, 0),
+	MTK_PIN_IES_SMT_SPEC(54, 54, 0x910, 2),
+	MTK_PIN_IES_SMT_SPEC(55, 57, 0x910, 4),
+	MTK_PIN_IES_SMT_SPEC(58, 59, 0x900, 15),
+	MTK_PIN_IES_SMT_SPEC(60, 61, 0x910, 1),
+	MTK_PIN_IES_SMT_SPEC(62, 65, 0x910, 5),
+	MTK_PIN_IES_SMT_SPEC(66, 67, 0x910, 6),
+	MTK_PIN_IES_SMT_SPEC(68, 68, 0x930, 2),
+	MTK_PIN_IES_SMT_SPEC(69, 69, 0x930, 1),
+	MTK_PIN_IES_SMT_SPEC(70, 70, 0x930, 6),
+	MTK_PIN_IES_SMT_SPEC(71, 71, 0x930, 5),
+	MTK_PIN_IES_SMT_SPEC(72, 72, 0x930, 4),
+	MTK_PIN_IES_SMT_SPEC(73, 73, 0x930, 3),
+	MTK_PIN_IES_SMT_SPEC(100, 103, 0x910, 7),
+	MTK_PIN_IES_SMT_SPEC(104, 104, 0x920, 12),
+	MTK_PIN_IES_SMT_SPEC(105, 105, 0x920, 11),
+	MTK_PIN_IES_SMT_SPEC(106, 106, 0x930, 0),
+	MTK_PIN_IES_SMT_SPEC(107, 107, 0x920, 15),
+	MTK_PIN_IES_SMT_SPEC(108, 108, 0x920, 14),
+	MTK_PIN_IES_SMT_SPEC(109, 109, 0x920, 13),
+	MTK_PIN_IES_SMT_SPEC(110, 110, 0x920, 9),
+	MTK_PIN_IES_SMT_SPEC(111, 111, 0x920, 8),
+	MTK_PIN_IES_SMT_SPEC(112, 112, 0x920, 7),
+	MTK_PIN_IES_SMT_SPEC(113, 113, 0x920, 6),
+	MTK_PIN_IES_SMT_SPEC(114, 114, 0x920, 10),
+	MTK_PIN_IES_SMT_SPEC(115, 115, 0x920, 1),
+	MTK_PIN_IES_SMT_SPEC(116, 116, 0x920, 0),
+	MTK_PIN_IES_SMT_SPEC(117, 117, 0x920, 5),
+	MTK_PIN_IES_SMT_SPEC(118, 118, 0x920, 4),
+	MTK_PIN_IES_SMT_SPEC(119, 119, 0x920, 3),
+	MTK_PIN_IES_SMT_SPEC(120, 120, 0x920, 2),
+	MTK_PIN_IES_SMT_SPEC(121, 124, 0x910, 9),
+};
+
+static const struct mtk_pin_ies_smt_set mt8516_smt_set[] = {
+	MTK_PIN_IES_SMT_SPEC(0, 6, 0xA00, 2),
+	MTK_PIN_IES_SMT_SPEC(7, 10, 0xA00, 3),
+	MTK_PIN_IES_SMT_SPEC(11, 13, 0xA00, 12),
+	MTK_PIN_IES_SMT_SPEC(14, 17, 0xA00, 13),
+	MTK_PIN_IES_SMT_SPEC(18, 20, 0xA10, 10),
+	MTK_PIN_IES_SMT_SPEC(21, 23, 0xA00, 13),
+	MTK_PIN_IES_SMT_SPEC(24, 25, 0xA00, 12),
+	MTK_PIN_IES_SMT_SPEC(26, 30, 0xA00, 0),
+	MTK_PIN_IES_SMT_SPEC(31, 33, 0xA00, 1),
+	MTK_PIN_IES_SMT_SPEC(34, 39, 0xA900, 2),
+	MTK_PIN_IES_SMT_SPEC(40, 40, 0xA10, 11),
+	MTK_PIN_IES_SMT_SPEC(41, 43, 0xA00, 10),
+	MTK_PIN_IES_SMT_SPEC(44, 47, 0xA00, 11),
+	MTK_PIN_IES_SMT_SPEC(48, 51, 0xA00, 14),
+	MTK_PIN_IES_SMT_SPEC(52, 53, 0xA10, 0),
+	MTK_PIN_IES_SMT_SPEC(54, 54, 0xA10, 2),
+	MTK_PIN_IES_SMT_SPEC(55, 57, 0xA10, 4),
+	MTK_PIN_IES_SMT_SPEC(58, 59, 0xA00, 15),
+	MTK_PIN_IES_SMT_SPEC(60, 61, 0xA10, 1),
+	MTK_PIN_IES_SMT_SPEC(62, 65, 0xA10, 5),
+	MTK_PIN_IES_SMT_SPEC(66, 67, 0xA10, 6),
+	MTK_PIN_IES_SMT_SPEC(68, 68, 0xA30, 2),
+	MTK_PIN_IES_SMT_SPEC(69, 69, 0xA30, 1),
+	MTK_PIN_IES_SMT_SPEC(70, 70, 0xA30, 3),
+	MTK_PIN_IES_SMT_SPEC(71, 71, 0xA30, 4),
+	MTK_PIN_IES_SMT_SPEC(72, 72, 0xA30, 5),
+	MTK_PIN_IES_SMT_SPEC(73, 73, 0xA30, 6),
+
+	MTK_PIN_IES_SMT_SPEC(100, 103, 0xA10, 7),
+	MTK_PIN_IES_SMT_SPEC(104, 104, 0xA20, 12),
+	MTK_PIN_IES_SMT_SPEC(105, 105, 0xA20, 11),
+	MTK_PIN_IES_SMT_SPEC(106, 106, 0xA30, 13),
+	MTK_PIN_IES_SMT_SPEC(107, 107, 0xA20, 14),
+	MTK_PIN_IES_SMT_SPEC(108, 108, 0xA20, 15),
+	MTK_PIN_IES_SMT_SPEC(109, 109, 0xA30, 0),
+	MTK_PIN_IES_SMT_SPEC(110, 110, 0xA20, 9),
+	MTK_PIN_IES_SMT_SPEC(111, 111, 0xA20, 8),
+	MTK_PIN_IES_SMT_SPEC(112, 112, 0xA20, 7),
+	MTK_PIN_IES_SMT_SPEC(113, 113, 0xA20, 6),
+	MTK_PIN_IES_SMT_SPEC(114, 114, 0xA20, 10),
+	MTK_PIN_IES_SMT_SPEC(115, 115, 0xA20, 1),
+	MTK_PIN_IES_SMT_SPEC(116, 116, 0xA20, 0),
+	MTK_PIN_IES_SMT_SPEC(117, 117, 0xA20, 5),
+	MTK_PIN_IES_SMT_SPEC(118, 118, 0xA20, 4),
+	MTK_PIN_IES_SMT_SPEC(119, 119, 0xA20, 3),
+	MTK_PIN_IES_SMT_SPEC(120, 120, 0xA20, 2),
+	MTK_PIN_IES_SMT_SPEC(121, 124, 0xA10, 9),
+};
+
+static int mt8516_ies_smt_set(struct regmap *regmap, unsigned int pin,
+		unsigned char align, int value, enum pin_config_param arg)
+{
+	if (arg == PIN_CONFIG_INPUT_ENABLE)
+		return mtk_pconf_spec_set_ies_smt_range(regmap, mt8516_ies_set,
+			ARRAY_SIZE(mt8516_ies_set), pin, align, value);
+	else if (arg == PIN_CONFIG_INPUT_SCHMITT_ENABLE)
+		return mtk_pconf_spec_set_ies_smt_range(regmap, mt8516_smt_set,
+			ARRAY_SIZE(mt8516_smt_set), pin, align, value);
+	return -EINVAL;
+}
+
+static const struct mtk_pinctrl_devdata mt8516_pinctrl_data = {
+	.pins = mtk_pins_mt8516,
+	.npins = ARRAY_SIZE(mtk_pins_mt8516),
+	.grp_desc = mt8516_drv_grp,
+	.n_grp_cls = ARRAY_SIZE(mt8516_drv_grp),
+	.pin_drv_grp = mt8516_pin_drv,
+	.n_pin_drv_grps = ARRAY_SIZE(mt8516_pin_drv),
+	.spec_pull_set = mt8516_spec_pull_set,
+	.spec_ies_smt_set = mt8516_ies_smt_set,
+	.dir_offset = 0x0000,
+	.pullen_offset = 0x0500,
+	.pullsel_offset = 0x0600,
+	.dout_offset = 0x0100,
+	.din_offset = 0x0200,
+	.pinmux_offset = 0x0300,
+	.type1_start = 125,
+	.type1_end = 125,
+	.port_shf = 4,
+	.port_mask = 0xf,
+	.port_align = 4,
+	.eint_hw = {
+		.port_mask = 7,
+		.ports     = 6,
+		.ap_num    = 169,
+		.db_cnt    = 64,
+	},
+};
+
+static int mt8516_pinctrl_probe(struct platform_device *pdev)
+{
+	return mtk_pctrl_init(pdev, &mt8516_pinctrl_data, NULL);
+}
+
+static const struct of_device_id mt8516_pctrl_match[] = {
+	{
+		.compatible = "mediatek,mt8516-pinctrl",
+	},
+	{}
+};
+
+MODULE_DEVICE_TABLE(of, mt8516_pctrl_match);
+
+static struct platform_driver mtk_pinctrl_driver = {
+	.probe = mt8516_pinctrl_probe,
+	.driver = {
+		.name = "mediatek-mt8516-pinctrl",
+		.of_match_table = mt8516_pctrl_match,
+		.pm = &mtk_eint_pm_ops,
+	},
+};
+
+static int __init mtk_pinctrl_init(void)
+{
+	return platform_driver_register(&mtk_pinctrl_driver);
+}
+arch_initcall(mtk_pinctrl_init);
diff --git a/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.c b/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.c
index b1c368455d30..20e1c890e73b 100644
--- a/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.c
+++ b/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.c
@@ -674,3 +674,52 @@ int mtk_pinconf_adv_pull_get(struct mtk_pinctrl *hw,
 
 	return 0;
 }
+
+int mtk_pinconf_adv_drive_set(struct mtk_pinctrl *hw,
+			      const struct mtk_pin_desc *desc, u32 arg)
+{
+	int err;
+	int en = arg & 1;
+	int e0 = !!(arg & 2);
+	int e1 = !!(arg & 4);
+
+	err = mtk_hw_set_value(hw, desc, PINCTRL_PIN_REG_DRV_EN, en);
+	if (err)
+		return err;
+
+	if (!en)
+		return err;
+
+	err = mtk_hw_set_value(hw, desc, PINCTRL_PIN_REG_DRV_E0, e0);
+	if (err)
+		return err;
+
+	err = mtk_hw_set_value(hw, desc, PINCTRL_PIN_REG_DRV_E1, e1);
+	if (err)
+		return err;
+
+	return err;
+}
+
+int mtk_pinconf_adv_drive_get(struct mtk_pinctrl *hw,
+			      const struct mtk_pin_desc *desc, u32 *val)
+{
+	u32 en, e0, e1;
+	int err;
+
+	err = mtk_hw_get_value(hw, desc, PINCTRL_PIN_REG_DRV_EN, &en);
+	if (err)
+		return err;
+
+	err = mtk_hw_get_value(hw, desc, PINCTRL_PIN_REG_DRV_E0, &e0);
+	if (err)
+		return err;
+
+	err = mtk_hw_get_value(hw, desc, PINCTRL_PIN_REG_DRV_E1, &e1);
+	if (err)
+		return err;
+
+	*val = (en | e0 << 1 | e1 << 2) & 0x7;
+
+	return 0;
+}
diff --git a/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.h b/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.h
index 6d24522739d9..1b7da42aa1d5 100644
--- a/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.h
+++ b/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.h
@@ -63,6 +63,9 @@ enum {
 	PINCTRL_PIN_REG_IES,
 	PINCTRL_PIN_REG_PULLEN,
 	PINCTRL_PIN_REG_PULLSEL,
+	PINCTRL_PIN_REG_DRV_EN,
+	PINCTRL_PIN_REG_DRV_E0,
+	PINCTRL_PIN_REG_DRV_E1,
 	PINCTRL_PIN_REG_MAX,
 };
 
@@ -224,6 +227,10 @@ struct mtk_pin_soc {
 	int (*adv_pull_get)(struct mtk_pinctrl *hw,
 			    const struct mtk_pin_desc *desc, bool pullup,
 			    u32 *val);
+	int (*adv_drive_set)(struct mtk_pinctrl *hw,
+			     const struct mtk_pin_desc *desc, u32 arg);
+	int (*adv_drive_get)(struct mtk_pinctrl *hw,
+			     const struct mtk_pin_desc *desc, u32 *val);
 
 	/* Specific driver data */
 	void				*driver_data;
@@ -287,5 +294,9 @@ int mtk_pinconf_adv_pull_set(struct mtk_pinctrl *hw,
 int mtk_pinconf_adv_pull_get(struct mtk_pinctrl *hw,
 			     const struct mtk_pin_desc *desc, bool pullup,
 			     u32 *val);
+int mtk_pinconf_adv_drive_set(struct mtk_pinctrl *hw,
+			      const struct mtk_pin_desc *desc, u32 arg);
+int mtk_pinconf_adv_drive_get(struct mtk_pinctrl *hw,
+			      const struct mtk_pin_desc *desc, u32 *val);
 
 #endif /* __PINCTRL_MTK_COMMON_V2_H */
diff --git a/drivers/pinctrl/mediatek/pinctrl-mtk-mt8516.h b/drivers/pinctrl/mediatek/pinctrl-mtk-mt8516.h
new file mode 100644
index 000000000000..f7a4c6e4a026
--- /dev/null
+++ b/drivers/pinctrl/mediatek/pinctrl-mtk-mt8516.h
@@ -0,0 +1,1182 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2019 MediaTek Inc.
+ */
+#ifndef __PINCTRL_MTK_MT8516_H
+#define __PINCTRL_MTK_MT8516_H
+
+#include <linux/pinctrl/pinctrl.h>
+#include "pinctrl-mtk-common.h"
+
+static const struct mtk_desc_pin mtk_pins_mt8516[] = {
+	MTK_PIN(
+		PINCTRL_PIN(0, "EINT0"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 0),
+		MTK_FUNCTION(0, "GPIO0"),
+		MTK_FUNCTION(1, "PWM_B"),
+		MTK_FUNCTION(3, "I2S2_BCK"),
+		MTK_FUNCTION(4, "EXT_TXD0"),
+		MTK_FUNCTION(6, "SQICS"),
+		MTK_FUNCTION(7, "DBG_MON_A[6]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(1, "EINT1"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 1),
+		MTK_FUNCTION(0, "GPIO1"),
+		MTK_FUNCTION(1, "PWM_C"),
+		MTK_FUNCTION(3, "I2S2_DI"),
+		MTK_FUNCTION(4, "EXT_TXD1"),
+		MTK_FUNCTION(5, "CONN_MCU_TDO"),
+		MTK_FUNCTION(6, "SQISO"),
+		MTK_FUNCTION(7, "DBG_MON_A[7]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(2, "EINT2"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 2),
+		MTK_FUNCTION(0, "GPIO2"),
+		MTK_FUNCTION(1, "CLKM0"),
+		MTK_FUNCTION(3, "I2S2_LRCK"),
+		MTK_FUNCTION(4, "EXT_TXD2"),
+		MTK_FUNCTION(5, "CONN_MCU_DBGACK_N"),
+		MTK_FUNCTION(6, "SQISI"),
+		MTK_FUNCTION(7, "DBG_MON_A[8]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(3, "EINT3"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 3),
+		MTK_FUNCTION(0, "GPIO3"),
+		MTK_FUNCTION(1, "CLKM1"),
+		MTK_FUNCTION(3, "SPI_MI"),
+		MTK_FUNCTION(4, "EXT_TXD3"),
+		MTK_FUNCTION(5, "CONN_MCU_DBGI_N"),
+		MTK_FUNCTION(6, "SQIWP"),
+		MTK_FUNCTION(7, "DBG_MON_A[9]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(4, "EINT4"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 4),
+		MTK_FUNCTION(0, "GPIO4"),
+		MTK_FUNCTION(1, "CLKM2"),
+		MTK_FUNCTION(3, "SPI_MO"),
+		MTK_FUNCTION(4, "EXT_TXC"),
+		MTK_FUNCTION(5, "CONN_MCU_TCK"),
+		MTK_FUNCTION(6, "CONN_MCU_AICE_JCKC"),
+		MTK_FUNCTION(7, "DBG_MON_A[10]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(5, "EINT5"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 5),
+		MTK_FUNCTION(0, "GPIO5"),
+		MTK_FUNCTION(1, "UCTS2"),
+		MTK_FUNCTION(3, "SPI_CSB"),
+		MTK_FUNCTION(4, "EXT_RXER"),
+		MTK_FUNCTION(5, "CONN_MCU_TDI"),
+		MTK_FUNCTION(6, "CONN_TEST_CK"),
+		MTK_FUNCTION(7, "DBG_MON_A[11]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(6, "EINT6"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 6),
+		MTK_FUNCTION(0, "GPIO6"),
+		MTK_FUNCTION(1, "URTS2"),
+		MTK_FUNCTION(3, "SPI_CLK"),
+		MTK_FUNCTION(4, "EXT_RXC"),
+		MTK_FUNCTION(5, "CONN_MCU_TRST_B"),
+		MTK_FUNCTION(7, "DBG_MON_A[12]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(7, "EINT7"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 7),
+		MTK_FUNCTION(0, "GPIO7"),
+		MTK_FUNCTION(1, "SQIRST"),
+		MTK_FUNCTION(3, "SDA1_0"),
+		MTK_FUNCTION(4, "EXT_RXDV"),
+		MTK_FUNCTION(5, "CONN_MCU_TMS"),
+		MTK_FUNCTION(6, "CONN_MCU_AICE_JMSC"),
+		MTK_FUNCTION(7, "DBG_MON_A[13]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(8, "EINT8"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 8),
+		MTK_FUNCTION(0, "GPIO8"),
+		MTK_FUNCTION(1, "SQICK"),
+		MTK_FUNCTION(2, "CLKM3"),
+		MTK_FUNCTION(3, "SCL1_0"),
+		MTK_FUNCTION(4, "EXT_RXD0"),
+		MTK_FUNCTION(5, "ANT_SEL0"),
+		MTK_FUNCTION(7, "DBG_MON_A[14]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(9, "EINT9"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 9),
+		MTK_FUNCTION(0, "GPIO9"),
+		MTK_FUNCTION(1, "CLKM4"),
+		MTK_FUNCTION(2, "SDA2_0"),
+		MTK_FUNCTION(3, "EXT_FRAME_SYNC"),
+		MTK_FUNCTION(4, "EXT_RXD1"),
+		MTK_FUNCTION(5, "ANT_SEL1"),
+		MTK_FUNCTION(7, "DBG_MON_A[15]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(10, "EINT10"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 10),
+		MTK_FUNCTION(0, "GPIO10"),
+		MTK_FUNCTION(1, "CLKM5"),
+		MTK_FUNCTION(2, "SCL2_0"),
+		MTK_FUNCTION(3, "EXT_FRAME_SYNC"),
+		MTK_FUNCTION(4, "EXT_RXD2"),
+		MTK_FUNCTION(5, "ANT_SEL2"),
+		MTK_FUNCTION(7, "DBG_MON_A[16]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(11, "EINT11"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 11),
+		MTK_FUNCTION(0, "GPIO11"),
+		MTK_FUNCTION(1, "CLKM4"),
+		MTK_FUNCTION(2, "PWM_C"),
+		MTK_FUNCTION(3, "CONN_TEST_CK"),
+		MTK_FUNCTION(4, "ANT_SEL3"),
+		MTK_FUNCTION(6, "EXT_RXD3"),
+		MTK_FUNCTION(7, "DBG_MON_A[17]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(12, "EINT12"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 12),
+		MTK_FUNCTION(0, "GPIO12"),
+		MTK_FUNCTION(1, "CLKM5"),
+		MTK_FUNCTION(2, "PWM_A"),
+		MTK_FUNCTION(3, "SPDIF_OUT"),
+		MTK_FUNCTION(4, "ANT_SEL4"),
+		MTK_FUNCTION(6, "EXT_TXEN"),
+		MTK_FUNCTION(7, "DBG_MON_A[18]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(13, "EINT13"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 13),
+		MTK_FUNCTION(0, "GPIO13"),
+		MTK_FUNCTION(3, "TSF_IN"),
+		MTK_FUNCTION(4, "ANT_SEL5"),
+		MTK_FUNCTION(6, "SPDIF_IN"),
+		MTK_FUNCTION(7, "DBG_MON_A[19]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(14, "EINT14"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 14),
+		MTK_FUNCTION(0, "GPIO14"),
+		MTK_FUNCTION(2, "I2S_8CH_DO1"),
+		MTK_FUNCTION(3, "TDM_RX_MCK"),
+		MTK_FUNCTION(4, "ANT_SEL1"),
+		MTK_FUNCTION(5, "CONN_MCU_DBGACK_N"),
+		MTK_FUNCTION(6, "NCLE"),
+		MTK_FUNCTION(7, "DBG_MON_B[8]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(15, "EINT15"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 15),
+		MTK_FUNCTION(0, "GPIO15"),
+		MTK_FUNCTION(2, "I2S_8CH_LRCK"),
+		MTK_FUNCTION(3, "TDM_RX_BCK"),
+		MTK_FUNCTION(4, "ANT_SEL2"),
+		MTK_FUNCTION(5, "CONN_MCU_DBGI_N"),
+		MTK_FUNCTION(6, "NCEB1"),
+		MTK_FUNCTION(7, "DBG_MON_B[9]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(16, "EINT16"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 16),
+		MTK_FUNCTION(0, "GPIO16"),
+		MTK_FUNCTION(2, "I2S_8CH_BCK"),
+		MTK_FUNCTION(3, "TDM_RX_LRCK"),
+		MTK_FUNCTION(4, "ANT_SEL3"),
+		MTK_FUNCTION(5, "CONN_MCU_TRST_B"),
+		MTK_FUNCTION(6, "NCEB0"),
+		MTK_FUNCTION(7, "DBG_MON_B[10]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(17, "EINT17"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 17),
+		MTK_FUNCTION(0, "GPIO17"),
+		MTK_FUNCTION(2, "I2S_8CH_MCK"),
+		MTK_FUNCTION(3, "TDM_RX_DI"),
+		MTK_FUNCTION(4, "IDDIG"),
+		MTK_FUNCTION(5, "ANT_SEL4"),
+		MTK_FUNCTION(6, "NREB"),
+		MTK_FUNCTION(7, "DBG_MON_B[11]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(18, "EINT18"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 18),
+		MTK_FUNCTION(0, "GPIO18"),
+		MTK_FUNCTION(2, "USB_DRVVBUS"),
+		MTK_FUNCTION(3, "I2S3_LRCK"),
+		MTK_FUNCTION(4, "CLKM1"),
+		MTK_FUNCTION(5, "ANT_SEL3"),
+		MTK_FUNCTION(6, "I2S2_BCK"),
+		MTK_FUNCTION(7, "DBG_MON_A[20]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(19, "EINT19"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 19),
+		MTK_FUNCTION(0, "GPIO19"),
+		MTK_FUNCTION(1, "UCTS1"),
+		MTK_FUNCTION(2, "IDDIG"),
+		MTK_FUNCTION(3, "I2S3_BCK"),
+		MTK_FUNCTION(4, "CLKM2"),
+		MTK_FUNCTION(5, "ANT_SEL4"),
+		MTK_FUNCTION(6, "I2S2_DI"),
+		MTK_FUNCTION(7, "DBG_MON_A[21]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(20, "EINT20"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 20),
+		MTK_FUNCTION(0, "GPIO20"),
+		MTK_FUNCTION(1, "URTS1"),
+		MTK_FUNCTION(3, "I2S3_DO"),
+		MTK_FUNCTION(4, "CLKM3"),
+		MTK_FUNCTION(5, "ANT_SEL5"),
+		MTK_FUNCTION(6, "I2S2_LRCK"),
+		MTK_FUNCTION(7, "DBG_MON_A[22]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(21, "EINT21"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 21),
+		MTK_FUNCTION(0, "GPIO21"),
+		MTK_FUNCTION(1, "NRNB"),
+		MTK_FUNCTION(2, "ANT_SEL0"),
+		MTK_FUNCTION(3, "I2S_8CH_DO4"),
+		MTK_FUNCTION(7, "DBG_MON_B[31]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(22, "EINT22"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 22),
+		MTK_FUNCTION(0, "GPIO22"),
+		MTK_FUNCTION(2, "I2S_8CH_DO2"),
+		MTK_FUNCTION(3, "TSF_IN"),
+		MTK_FUNCTION(4, "USB_DRVVBUS"),
+		MTK_FUNCTION(5, "SPDIF_OUT"),
+		MTK_FUNCTION(6, "NRE_C"),
+		MTK_FUNCTION(7, "DBG_MON_B[12]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(23, "EINT23"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 23),
+		MTK_FUNCTION(0, "GPIO23"),
+		MTK_FUNCTION(2, "I2S_8CH_DO3"),
+		MTK_FUNCTION(3, "CLKM0"),
+		MTK_FUNCTION(4, "IR"),
+		MTK_FUNCTION(5, "SPDIF_IN"),
+		MTK_FUNCTION(6, "NDQS_C"),
+		MTK_FUNCTION(7, "DBG_MON_B[13]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(24, "EINT24"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 24),
+		MTK_FUNCTION(0, "GPIO24"),
+		MTK_FUNCTION(3, "ANT_SEL1"),
+		MTK_FUNCTION(4, "UCTS2"),
+		MTK_FUNCTION(5, "PWM_A"),
+		MTK_FUNCTION(6, "I2S0_MCK"),
+		MTK_FUNCTION(7, "DBG_MON_A[0]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(25, "EINT25"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 25),
+		MTK_FUNCTION(0, "GPIO25"),
+		MTK_FUNCTION(3, "ANT_SEL0"),
+		MTK_FUNCTION(4, "URTS2"),
+		MTK_FUNCTION(5, "PWM_B"),
+		MTK_FUNCTION(6, "I2S_8CH_MCK"),
+		MTK_FUNCTION(7, "DBG_MON_A[1]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(26, "PWRAP_SPI0_MI"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 26),
+		MTK_FUNCTION(0, "GPIO26"),
+		MTK_FUNCTION(1, "PWRAP_SPI0_MO"),
+		MTK_FUNCTION(2, "PWRAP_SPI0_MI")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(27, "PWRAP_SPI0_MO"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 27),
+		MTK_FUNCTION(0, "GPIO27"),
+		MTK_FUNCTION(1, "PWRAP_SPI0_MI"),
+		MTK_FUNCTION(2, "PWRAP_SPI0_MO")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(28, "PWRAP_INT"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 28),
+		MTK_FUNCTION(0, "GPIO28"),
+		MTK_FUNCTION(1, "I2S0_MCK"),
+		MTK_FUNCTION(4, "I2S_8CH_MCK"),
+		MTK_FUNCTION(5, "I2S2_MCK"),
+		MTK_FUNCTION(6, "I2S3_MCK")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(29, "PWRAP_SPI0_CK"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 29),
+		MTK_FUNCTION(0, "GPIO29"),
+		MTK_FUNCTION(1, "PWRAP_SPI0_CK")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(30, "PWRAP_SPI0_CSN"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 30),
+		MTK_FUNCTION(0, "GPIO30"),
+		MTK_FUNCTION(1, "PWRAP_SPI0_CSN")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(31, "RTC32K_CK"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 31),
+		MTK_FUNCTION(0, "GPIO31"),
+		MTK_FUNCTION(1, "RTC32K_CK")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(32, "WATCHDOG"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 32),
+		MTK_FUNCTION(0, "GPIO32"),
+		MTK_FUNCTION(1, "WATCHDOG")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(33, "SRCLKENA"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 33),
+		MTK_FUNCTION(0, "GPIO33"),
+		MTK_FUNCTION(1, "SRCLKENA0")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(34, "URXD2"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 34),
+		MTK_FUNCTION(0, "GPIO34"),
+		MTK_FUNCTION(1, "URXD2"),
+		MTK_FUNCTION(3, "UTXD2"),
+		MTK_FUNCTION(4, "DBG_SCL"),
+		MTK_FUNCTION(6, "I2S2_MCK"),
+		MTK_FUNCTION(7, "DBG_MON_B[0]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(35, "UTXD2"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 35),
+		MTK_FUNCTION(0, "GPIO35"),
+		MTK_FUNCTION(1, "UTXD2"),
+		MTK_FUNCTION(3, "URXD2"),
+		MTK_FUNCTION(4, "DBG_SDA"),
+		MTK_FUNCTION(6, "I2S3_MCK"),
+		MTK_FUNCTION(7, "DBG_MON_B[1]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(36, "MRG_CLK"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 36),
+		MTK_FUNCTION(0, "GPIO36"),
+		MTK_FUNCTION(1, "MRG_CLK"),
+		MTK_FUNCTION(3, "I2S0_BCK"),
+		MTK_FUNCTION(4, "I2S3_BCK"),
+		MTK_FUNCTION(5, "PCM0_CLK"),
+		MTK_FUNCTION(6, "IR"),
+		MTK_FUNCTION(7, "DBG_MON_A[2]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(37, "MRG_SYNC"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 37),
+		MTK_FUNCTION(0, "GPIO37"),
+		MTK_FUNCTION(1, "MRG_SYNC"),
+		MTK_FUNCTION(3, "I2S0_LRCK"),
+		MTK_FUNCTION(4, "I2S3_LRCK"),
+		MTK_FUNCTION(5, "PCM0_SYNC"),
+		MTK_FUNCTION(6, "EXT_COL"),
+		MTK_FUNCTION(7, "DBG_MON_A[3]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(38, "MRG_DI"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 38),
+		MTK_FUNCTION(0, "GPIO38"),
+		MTK_FUNCTION(1, "MRG_DI"),
+		MTK_FUNCTION(3, "I2S0_DI"),
+		MTK_FUNCTION(4, "I2S3_DO"),
+		MTK_FUNCTION(5, "PCM0_DI"),
+		MTK_FUNCTION(6, "EXT_MDIO"),
+		MTK_FUNCTION(7, "DBG_MON_A[4]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(39, "MRG_DO"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 39),
+		MTK_FUNCTION(0, "GPIO39"),
+		MTK_FUNCTION(1, "MRG_DO"),
+		MTK_FUNCTION(3, "I2S0_MCK"),
+		MTK_FUNCTION(4, "I2S3_MCK"),
+		MTK_FUNCTION(5, "PCM0_DO"),
+		MTK_FUNCTION(6, "EXT_MDC"),
+		MTK_FUNCTION(7, "DBG_MON_A[5]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(40, "KPROW0"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 40),
+		MTK_FUNCTION(0, "GPIO40"),
+		MTK_FUNCTION(1, "KPROW0"),
+		MTK_FUNCTION(7, "DBG_MON_B[4]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(41, "KPROW1"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 41),
+		MTK_FUNCTION(0, "GPIO41"),
+		MTK_FUNCTION(1, "KPROW1"),
+		MTK_FUNCTION(2, "IDDIG"),
+		MTK_FUNCTION(3, "EXT_FRAME_SYNC"),
+		MTK_FUNCTION(7, "DBG_MON_B[5]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(42, "KPCOL0"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 42),
+		MTK_FUNCTION(0, "GPIO42"),
+		MTK_FUNCTION(1, "KPCOL0"),
+		MTK_FUNCTION(7, "DBG_MON_B[6]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(43, "KPCOL1"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 43),
+		MTK_FUNCTION(0, "GPIO43"),
+		MTK_FUNCTION(1, "KPCOL1"),
+		MTK_FUNCTION(2, "USB_DRVVBUS"),
+		MTK_FUNCTION(3, "EXT_FRAME_SYNC"),
+		MTK_FUNCTION(4, "TSF_IN"),
+		MTK_FUNCTION(7, "DBG_MON_B[7]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(44, "JTMS"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 44),
+		MTK_FUNCTION(0, "GPIO44"),
+		MTK_FUNCTION(1, "JTMS"),
+		MTK_FUNCTION(2, "CONN_MCU_TMS"),
+		MTK_FUNCTION(3, "CONN_MCU_AICE_JMSC")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(45, "JTCK"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 45),
+		MTK_FUNCTION(0, "GPIO45"),
+		MTK_FUNCTION(1, "JTCK"),
+		MTK_FUNCTION(2, "CONN_MCU_TCK"),
+		MTK_FUNCTION(3, "CONN_MCU_AICE_JCKC")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(46, "JTDI"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 46),
+		MTK_FUNCTION(0, "GPIO46"),
+		MTK_FUNCTION(1, "JTDI"),
+		MTK_FUNCTION(2, "CONN_MCU_TDI")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(47, "JTDO"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 47),
+		MTK_FUNCTION(0, "GPIO47"),
+		MTK_FUNCTION(1, "JTDO"),
+		MTK_FUNCTION(2, "CONN_MCU_TDO")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(48, "SPI_CS"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 48),
+		MTK_FUNCTION(0, "GPIO48"),
+		MTK_FUNCTION(1, "SPI_CSB"),
+		MTK_FUNCTION(3, "I2S0_DI"),
+		MTK_FUNCTION(4, "I2S2_BCK"),
+		MTK_FUNCTION(7, "DBG_MON_A[23]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(49, "SPI_CK"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 49),
+		MTK_FUNCTION(0, "GPIO49"),
+		MTK_FUNCTION(1, "SPI_CLK"),
+		MTK_FUNCTION(3, "I2S0_LRCK"),
+		MTK_FUNCTION(4, "I2S2_DI"),
+		MTK_FUNCTION(7, "DBG_MON_A[24]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(50, "SPI_MI"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 50),
+		MTK_FUNCTION(0, "GPIO50"),
+		MTK_FUNCTION(1, "SPI_MI"),
+		MTK_FUNCTION(2, "SPI_MO"),
+		MTK_FUNCTION(3, "I2S0_BCK"),
+		MTK_FUNCTION(4, "I2S2_LRCK"),
+		MTK_FUNCTION(7, "DBG_MON_A[25]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(51, "SPI_MO"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 51),
+		MTK_FUNCTION(0, "GPIO51"),
+		MTK_FUNCTION(1, "SPI_MO"),
+		MTK_FUNCTION(2, "SPI_MI"),
+		MTK_FUNCTION(3, "I2S0_MCK"),
+		MTK_FUNCTION(4, "I2S2_MCK"),
+		MTK_FUNCTION(7, "DBG_MON_A[26]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(52, "SDA1"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 52),
+		MTK_FUNCTION(0, "GPIO52"),
+		MTK_FUNCTION(1, "SDA1_0")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(53, "SCL1"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 53),
+		MTK_FUNCTION(0, "GPIO53"),
+		MTK_FUNCTION(1, "SCL1_0")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(54, "GPIO54"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 54),
+		MTK_FUNCTION(0, "GPIO54"),
+		MTK_FUNCTION(2, "PWM_B"),
+		MTK_FUNCTION(7, "DBG_MON_B[2]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(55, "I2S_DATA_IN"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 55),
+		MTK_FUNCTION(0, "GPIO55"),
+		MTK_FUNCTION(1, "I2S0_DI"),
+		MTK_FUNCTION(2, "UCTS0"),
+		MTK_FUNCTION(3, "I2S3_DO"),
+		MTK_FUNCTION(4, "I2S_8CH_DO1"),
+		MTK_FUNCTION(5, "PWM_A"),
+		MTK_FUNCTION(6, "I2S2_BCK"),
+		MTK_FUNCTION(7, "DBG_MON_A[28]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(56, "I2S_LRCK"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 56),
+		MTK_FUNCTION(0, "GPIO56"),
+		MTK_FUNCTION(1, "I2S0_LRCK"),
+		MTK_FUNCTION(3, "I2S3_LRCK"),
+		MTK_FUNCTION(4, "I2S_8CH_LRCK"),
+		MTK_FUNCTION(5, "PWM_B"),
+		MTK_FUNCTION(6, "I2S2_DI"),
+		MTK_FUNCTION(7, "DBG_MON_A[29]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(57, "I2S_BCK"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 57),
+		MTK_FUNCTION(0, "GPIO57"),
+		MTK_FUNCTION(1, "I2S0_BCK"),
+		MTK_FUNCTION(2, "URTS0"),
+		MTK_FUNCTION(3, "I2S3_BCK"),
+		MTK_FUNCTION(4, "I2S_8CH_BCK"),
+		MTK_FUNCTION(5, "PWM_C"),
+		MTK_FUNCTION(6, "I2S2_LRCK"),
+		MTK_FUNCTION(7, "DBG_MON_A[30]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(58, "SDA0"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 58),
+		MTK_FUNCTION(0, "GPIO58"),
+		MTK_FUNCTION(1, "SDA0_0")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(59, "SCL0"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 59),
+		MTK_FUNCTION(0, "GPIO59"),
+		MTK_FUNCTION(1, "SCL0_0")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(60, "SDA2"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 60),
+		MTK_FUNCTION(0, "GPIO60"),
+		MTK_FUNCTION(1, "SDA2_0"),
+		MTK_FUNCTION(2, "PWM_B")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(61, "SCL2"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 61),
+		MTK_FUNCTION(0, "GPIO61"),
+		MTK_FUNCTION(1, "SCL2_0"),
+		MTK_FUNCTION(2, "PWM_C")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(62, "URXD0"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 62),
+		MTK_FUNCTION(0, "GPIO62"),
+		MTK_FUNCTION(1, "URXD0"),
+		MTK_FUNCTION(2, "UTXD0")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(63, "UTXD0"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 63),
+		MTK_FUNCTION(0, "GPIO63"),
+		MTK_FUNCTION(1, "UTXD0"),
+		MTK_FUNCTION(2, "URXD0")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(64, "URXD1"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 64),
+		MTK_FUNCTION(0, "GPIO64"),
+		MTK_FUNCTION(1, "URXD1"),
+		MTK_FUNCTION(2, "UTXD1"),
+		MTK_FUNCTION(7, "DBG_MON_A[27]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(65, "UTXD1"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 65),
+		MTK_FUNCTION(0, "GPIO65"),
+		MTK_FUNCTION(1, "UTXD1"),
+		MTK_FUNCTION(2, "URXD1"),
+		MTK_FUNCTION(7, "DBG_MON_A[31]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(66, "LCM_RST"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 66),
+		MTK_FUNCTION(0, "GPIO66"),
+		MTK_FUNCTION(1, "LCM_RST"),
+		MTK_FUNCTION(3, "I2S0_MCK"),
+		MTK_FUNCTION(7, "DBG_MON_B[3]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(67, "GPIO67"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 67),
+		MTK_FUNCTION(0, "GPIO67"),
+		MTK_FUNCTION(3, "I2S_8CH_MCK"),
+		MTK_FUNCTION(7, "DBG_MON_B[14]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(68, "MSDC2_CMD"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 68),
+		MTK_FUNCTION(0, "GPIO68"),
+		MTK_FUNCTION(1, "MSDC2_CMD"),
+		MTK_FUNCTION(2, "I2S_8CH_DO4"),
+		MTK_FUNCTION(3, "SDA1_0"),
+		MTK_FUNCTION(5, "USB_SDA"),
+		MTK_FUNCTION(6, "I2S3_BCK"),
+		MTK_FUNCTION(7, "DBG_MON_B[15]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(69, "MSDC2_CLK"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 69),
+		MTK_FUNCTION(0, "GPIO69"),
+		MTK_FUNCTION(1, "MSDC2_CLK"),
+		MTK_FUNCTION(2, "I2S_8CH_DO3"),
+		MTK_FUNCTION(3, "SCL1_0"),
+		MTK_FUNCTION(5, "USB_SCL"),
+		MTK_FUNCTION(6, "I2S3_LRCK"),
+		MTK_FUNCTION(7, "DBG_MON_B[16]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(70, "MSDC2_DAT0"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 70),
+		MTK_FUNCTION(0, "GPIO70"),
+		MTK_FUNCTION(1, "MSDC2_DAT0"),
+		MTK_FUNCTION(2, "I2S_8CH_DO2"),
+		MTK_FUNCTION(5, "UTXD0"),
+		MTK_FUNCTION(6, "I2S3_DO"),
+		MTK_FUNCTION(7, "DBG_MON_B[17]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(71, "MSDC2_DAT1"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 71),
+		MTK_FUNCTION(0, "GPIO71"),
+		MTK_FUNCTION(1, "MSDC2_DAT1"),
+		MTK_FUNCTION(2, "I2S_8CH_DO1"),
+		MTK_FUNCTION(3, "PWM_A"),
+		MTK_FUNCTION(4, "I2S3_MCK"),
+		MTK_FUNCTION(5, "URXD0"),
+		MTK_FUNCTION(6, "PWM_B"),
+		MTK_FUNCTION(7, "DBG_MON_B[18]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(72, "MSDC2_DAT2"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 72),
+		MTK_FUNCTION(0, "GPIO72"),
+		MTK_FUNCTION(1, "MSDC2_DAT2"),
+		MTK_FUNCTION(2, "I2S_8CH_LRCK"),
+		MTK_FUNCTION(3, "SDA2_0"),
+		MTK_FUNCTION(5, "UTXD1"),
+		MTK_FUNCTION(6, "PWM_C"),
+		MTK_FUNCTION(7, "DBG_MON_B[19]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(73, "MSDC2_DAT3"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 73),
+		MTK_FUNCTION(0, "GPIO73"),
+		MTK_FUNCTION(1, "MSDC2_DAT3"),
+		MTK_FUNCTION(2, "I2S_8CH_BCK"),
+		MTK_FUNCTION(3, "SCL2_0"),
+		MTK_FUNCTION(4, "EXT_FRAME_SYNC"),
+		MTK_FUNCTION(5, "URXD1"),
+		MTK_FUNCTION(6, "PWM_A"),
+		MTK_FUNCTION(7, "DBG_MON_B[20]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(74, "TDN3"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 74),
+		MTK_FUNCTION(0, "GPIO74"),
+		MTK_FUNCTION(1, "TDN3")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(75, "TDP3"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 75),
+		MTK_FUNCTION(0, "GPIO75"),
+		MTK_FUNCTION(1, "TDP3")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(76, "TDN2"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 76),
+		MTK_FUNCTION(0, "GPIO76"),
+		MTK_FUNCTION(1, "TDN2")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(77, "TDP2"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 77),
+		MTK_FUNCTION(0, "GPIO77"),
+		MTK_FUNCTION(1, "TDP2")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(78, "TCN"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 78),
+		MTK_FUNCTION(0, "GPIO78"),
+		MTK_FUNCTION(1, "TCN")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(79, "TCP"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 79),
+		MTK_FUNCTION(0, "GPIO79"),
+		MTK_FUNCTION(1, "TCP")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(80, "TDN1"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 80),
+		MTK_FUNCTION(0, "GPIO80"),
+		MTK_FUNCTION(1, "TDN1")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(81, "TDP1"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 81),
+		MTK_FUNCTION(0, "GPIO81"),
+		MTK_FUNCTION(1, "TDP1")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(82, "TDN0"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 82),
+		MTK_FUNCTION(0, "GPIO82"),
+		MTK_FUNCTION(1, "TDN0")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(83, "TDP0"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 83),
+		MTK_FUNCTION(0, "GPIO83"),
+		MTK_FUNCTION(1, "TDP0")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(84, "RDN0"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 84),
+		MTK_FUNCTION(0, "GPIO84"),
+		MTK_FUNCTION(1, "RDN0")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(85, "RDP0"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 85),
+		MTK_FUNCTION(0, "GPIO85"),
+		MTK_FUNCTION(1, "RDP0")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(86, "RDN1"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 86),
+		MTK_FUNCTION(0, "GPIO86"),
+		MTK_FUNCTION(1, "RDN1")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(87, "RDP1"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 87),
+		MTK_FUNCTION(0, "GPIO87"),
+		MTK_FUNCTION(1, "RDP1")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(88, "RCN"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 88),
+		MTK_FUNCTION(0, "GPIO88"),
+		MTK_FUNCTION(1, "RCN")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(89, "RCP"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 89),
+		MTK_FUNCTION(0, "GPIO89"),
+		MTK_FUNCTION(1, "RCP")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(90, "RDN2"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 90),
+		MTK_FUNCTION(0, "GPIO90"),
+		MTK_FUNCTION(1, "RDN2"),
+		MTK_FUNCTION(2, "CMDAT8")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(91, "RDP2"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 91),
+		MTK_FUNCTION(0, "GPIO91"),
+		MTK_FUNCTION(1, "RDP2"),
+		MTK_FUNCTION(2, "CMDAT9")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(92, "RDN3"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 92),
+		MTK_FUNCTION(0, "GPIO92"),
+		MTK_FUNCTION(1, "RDN3"),
+		MTK_FUNCTION(2, "CMDAT4")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(93, "RDP3"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 93),
+		MTK_FUNCTION(0, "GPIO93"),
+		MTK_FUNCTION(1, "RDP3"),
+		MTK_FUNCTION(2, "CMDAT5")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(94, "RCN_A"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 94),
+		MTK_FUNCTION(0, "GPIO94"),
+		MTK_FUNCTION(1, "RCN_A"),
+		MTK_FUNCTION(2, "CMDAT6")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(95, "RCP_A"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 95),
+		MTK_FUNCTION(0, "GPIO95"),
+		MTK_FUNCTION(1, "RCP_A"),
+		MTK_FUNCTION(2, "CMDAT7")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(96, "RDN1_A"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 96),
+		MTK_FUNCTION(0, "GPIO96"),
+		MTK_FUNCTION(1, "RDN1_A"),
+		MTK_FUNCTION(2, "CMDAT2"),
+		MTK_FUNCTION(3, "CMCSD2")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(97, "RDP1_A"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 97),
+		MTK_FUNCTION(0, "GPIO97"),
+		MTK_FUNCTION(1, "RDP1_A"),
+		MTK_FUNCTION(2, "CMDAT3"),
+		MTK_FUNCTION(3, "CMCSD3")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(98, "RDN0_A"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 98),
+		MTK_FUNCTION(0, "GPIO98"),
+		MTK_FUNCTION(1, "RDN0_A"),
+		MTK_FUNCTION(2, "CMHSYNC")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(99, "RDP0_A"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 99),
+		MTK_FUNCTION(0, "GPIO99"),
+		MTK_FUNCTION(1, "RDP0_A"),
+		MTK_FUNCTION(2, "CMVSYNC")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(100, "CMDAT0"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 100),
+		MTK_FUNCTION(0, "GPIO100"),
+		MTK_FUNCTION(1, "CMDAT0"),
+		MTK_FUNCTION(2, "CMCSD0"),
+		MTK_FUNCTION(3, "ANT_SEL2"),
+		MTK_FUNCTION(5, "TDM_RX_MCK"),
+		MTK_FUNCTION(7, "DBG_MON_B[21]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(101, "CMDAT1"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 101),
+		MTK_FUNCTION(0, "GPIO101"),
+		MTK_FUNCTION(1, "CMDAT1"),
+		MTK_FUNCTION(2, "CMCSD1"),
+		MTK_FUNCTION(3, "ANT_SEL3"),
+		MTK_FUNCTION(4, "CMFLASH"),
+		MTK_FUNCTION(5, "TDM_RX_BCK"),
+		MTK_FUNCTION(7, "DBG_MON_B[22]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(102, "CMMCLK"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 102),
+		MTK_FUNCTION(0, "GPIO102"),
+		MTK_FUNCTION(1, "CMMCLK"),
+		MTK_FUNCTION(3, "ANT_SEL4"),
+		MTK_FUNCTION(5, "TDM_RX_LRCK"),
+		MTK_FUNCTION(7, "DBG_MON_B[23]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(103, "CMPCLK"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 103),
+		MTK_FUNCTION(0, "GPIO103"),
+		MTK_FUNCTION(1, "CMPCLK"),
+		MTK_FUNCTION(2, "CMCSK"),
+		MTK_FUNCTION(3, "ANT_SEL5"),
+		MTK_FUNCTION(5, " TDM_RX_DI"),
+		MTK_FUNCTION(7, "DBG_MON_B[24]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(104, "MSDC1_CMD"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 104),
+		MTK_FUNCTION(0, "GPIO104"),
+		MTK_FUNCTION(1, "MSDC1_CMD"),
+		MTK_FUNCTION(4, "SQICS"),
+		MTK_FUNCTION(7, "DBG_MON_B[25]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(105, "MSDC1_CLK"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 105),
+		MTK_FUNCTION(0, "GPIO105"),
+		MTK_FUNCTION(1, "MSDC1_CLK"),
+		MTK_FUNCTION(4, "SQISO"),
+		MTK_FUNCTION(7, "DBG_MON_B[26]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(106, "MSDC1_DAT0"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 106),
+		MTK_FUNCTION(0, "GPIO106"),
+		MTK_FUNCTION(1, "MSDC1_DAT0"),
+		MTK_FUNCTION(4, "SQISI"),
+		MTK_FUNCTION(7, "DBG_MON_B[27]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(107, "MSDC1_DAT1"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 107),
+		MTK_FUNCTION(0, "GPIO107"),
+		MTK_FUNCTION(1, "MSDC1_DAT1"),
+		MTK_FUNCTION(4, "SQIWP"),
+		MTK_FUNCTION(7, "DBG_MON_B[28]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(108, "MSDC1_DAT2"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 108),
+		MTK_FUNCTION(0, "GPIO108"),
+		MTK_FUNCTION(1, "MSDC1_DAT2"),
+		MTK_FUNCTION(4, "SQIRST"),
+		MTK_FUNCTION(7, "DBG_MON_B[29]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(109, "MSDC1_DAT3"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 109),
+		MTK_FUNCTION(0, "GPIO109"),
+		MTK_FUNCTION(1, "MSDC1_DAT3"),
+		MTK_FUNCTION(4, "SQICK"), /* WIP */
+		MTK_FUNCTION(7, "DBG_MON_B[30]")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(110, "MSDC0_DAT7"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 110),
+		MTK_FUNCTION(0, "GPIO110"),
+		MTK_FUNCTION(1, "MSDC0_DAT7"),
+		MTK_FUNCTION(4, "NLD7")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(111, "MSDC0_DAT6"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 111),
+		MTK_FUNCTION(0, "GPIO111"),
+		MTK_FUNCTION(1, "MSDC0_DAT6"),
+		MTK_FUNCTION(4, "NLD6")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(112, "MSDC0_DAT5"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 112),
+		MTK_FUNCTION(0, "GPIO112"),
+		MTK_FUNCTION(1, "MSDC0_DAT5"),
+		MTK_FUNCTION(4, "NLD4")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(113, "MSDC0_DAT4"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 113),
+		MTK_FUNCTION(0, "GPIO113"),
+		MTK_FUNCTION(1, "MSDC0_DAT4"),
+		MTK_FUNCTION(4, "NLD3")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(114, "MSDC0_RSTB"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 114),
+		MTK_FUNCTION(0, "GPIO114"),
+		MTK_FUNCTION(1, "MSDC0_RSTB"),
+		MTK_FUNCTION(4, "NLD0")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(115, "MSDC0_CMD"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 115),
+		MTK_FUNCTION(0, "GPIO115"),
+		MTK_FUNCTION(1, "MSDC0_CMD"),
+		MTK_FUNCTION(4, "NALE")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(116, "MSDC0_CLK"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 116),
+		MTK_FUNCTION(0, "GPIO116"),
+		MTK_FUNCTION(1, "MSDC0_CLK"),
+		MTK_FUNCTION(4, "NWEB")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(117, "MSDC0_DAT3"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 117),
+		MTK_FUNCTION(0, "GPIO117"),
+		MTK_FUNCTION(1, "MSDC0_DAT3"),
+		MTK_FUNCTION(4, "NLD1")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(118, "MSDC0_DAT2"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 118),
+		MTK_FUNCTION(0, "GPIO118"),
+		MTK_FUNCTION(1, "MSDC0_DAT2"),
+		MTK_FUNCTION(4, "NLD5")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(119, "MSDC0_DAT1"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 119),
+		MTK_FUNCTION(0, "GPIO119"),
+		MTK_FUNCTION(1, "MSDC0_DAT1"),
+		MTK_FUNCTION(4, "NLD8")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(120, "MSDC0_DAT0"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 120),
+		MTK_FUNCTION(0, "GPIO120"),
+		MTK_FUNCTION(1, "MSDC0_DAT0"),
+		MTK_FUNCTION(4, "WATCHDOG"),
+		MTK_FUNCTION(5, "NLD2")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(121, "GPIO121"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 121),
+		MTK_FUNCTION(0, "GPIO121")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(122, "GPIO122"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 122),
+		MTK_FUNCTION(0, "GPIO122")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(123, "GPIO123"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 123),
+		MTK_FUNCTION(0, "GPIO123")
+	),
+	MTK_PIN(
+		PINCTRL_PIN(124, "GPIO124"),
+		NULL, "mt8516",
+		MTK_EINT_FUNCTION(0, 124),
+		MTK_FUNCTION(0, "GPIO124")
+	),
+};
+
+#endif /* __PINCTRL_MTK_MT8516_H */
diff --git a/drivers/pinctrl/mediatek/pinctrl-paris.c b/drivers/pinctrl/mediatek/pinctrl-paris.c
index b59e10852bfb..d3b34e9a7507 100644
--- a/drivers/pinctrl/mediatek/pinctrl-paris.c
+++ b/drivers/pinctrl/mediatek/pinctrl-paris.c
@@ -20,12 +20,14 @@
 #define MTK_PIN_CONFIG_RDSEL	(PIN_CONFIG_END + 2)
 #define MTK_PIN_CONFIG_PU_ADV	(PIN_CONFIG_END + 3)
 #define MTK_PIN_CONFIG_PD_ADV	(PIN_CONFIG_END + 4)
+#define MTK_PIN_CONFIG_DRV_ADV	(PIN_CONFIG_END + 5)
 
 static const struct pinconf_generic_params mtk_custom_bindings[] = {
 	{"mediatek,tdsel",	MTK_PIN_CONFIG_TDSEL,		0},
 	{"mediatek,rdsel",	MTK_PIN_CONFIG_RDSEL,		0},
 	{"mediatek,pull-up-adv", MTK_PIN_CONFIG_PU_ADV,		1},
 	{"mediatek,pull-down-adv", MTK_PIN_CONFIG_PD_ADV,	1},
+	{"mediatek,drive-strength-adv", MTK_PIN_CONFIG_DRV_ADV,	2},
 };
 
 #ifdef CONFIG_DEBUG_FS
@@ -34,6 +36,7 @@ static const struct pin_config_item mtk_conf_items[] = {
 	PCONFDUMP(MTK_PIN_CONFIG_RDSEL, "rdsel", NULL, true),
 	PCONFDUMP(MTK_PIN_CONFIG_PU_ADV, "pu-adv", NULL, true),
 	PCONFDUMP(MTK_PIN_CONFIG_PD_ADV, "pd-adv", NULL, true),
+	PCONFDUMP(MTK_PIN_CONFIG_DRV_ADV, "drive-strength-adv", NULL, true),
 };
 #endif
 
@@ -176,6 +179,15 @@ static int mtk_pinconf_get(struct pinctrl_dev *pctldev,
 			return -ENOTSUPP;
 		}
 		break;
+	case MTK_PIN_CONFIG_DRV_ADV:
+		if (hw->soc->adv_drive_get) {
+			err = hw->soc->adv_drive_get(hw, desc, &ret);
+			if (err)
+				return err;
+		} else {
+			return -ENOTSUPP;
+		}
+		break;
 	default:
 		return -ENOTSUPP;
 	}
@@ -311,6 +323,15 @@ static int mtk_pinconf_set(struct pinctrl_dev *pctldev, unsigned int pin,
 			return -ENOTSUPP;
 		}
 		break;
+	case MTK_PIN_CONFIG_DRV_ADV:
+		if (hw->soc->adv_drive_set) {
+			err = hw->soc->adv_drive_set(hw, desc, arg);
+			if (err)
+				return err;
+		} else {
+			return -ENOTSUPP;
+		}
+		break;
 	default:
 		err = -ENOTSUPP;
 	}
diff --git a/drivers/pinctrl/pinctrl-amd.c b/drivers/pinctrl/pinctrl-amd.c
index 6689995fa3aa..e18ebb2c78d9 100644
--- a/drivers/pinctrl/pinctrl-amd.c
+++ b/drivers/pinctrl/pinctrl-amd.c
@@ -930,8 +930,8 @@ static int amd_gpio_probe(struct platform_device *pdev)
 		goto out2;
 	}
 
-	ret = devm_request_irq(&pdev->dev, irq_base, amd_gpio_irq_handler, 0,
-			       KBUILD_MODNAME, gpio_dev);
+	ret = devm_request_irq(&pdev->dev, irq_base, amd_gpio_irq_handler,
+			       IRQF_SHARED, KBUILD_MODNAME, gpio_dev);
 	if (ret)
 		goto out2;
 
diff --git a/drivers/pinctrl/pinctrl-artpec6.c b/drivers/pinctrl/pinctrl-artpec6.c
index d89dc43c5757..e3239cf926f9 100644
--- a/drivers/pinctrl/pinctrl-artpec6.c
+++ b/drivers/pinctrl/pinctrl-artpec6.c
@@ -688,8 +688,9 @@ static void artpec6_pmx_select_func(struct pinctrl_dev *pctldev,
 	}
 }
 
-int artpec6_pmx_enable(struct pinctrl_dev *pctldev, unsigned int function,
-		       unsigned int group)
+static int artpec6_pmx_set(struct pinctrl_dev *pctldev,
+			   unsigned int function,
+			   unsigned int group)
 {
 	struct artpec6_pmx *pmx = pinctrl_dev_get_drvdata(pctldev);
 
@@ -702,18 +703,6 @@ int artpec6_pmx_enable(struct pinctrl_dev *pctldev, unsigned int function,
 	return 0;
 }
 
-void artpec6_pmx_disable(struct pinctrl_dev *pctldev, unsigned int function,
-			 unsigned int group)
-{
-	struct artpec6_pmx *pmx = pinctrl_dev_get_drvdata(pctldev);
-
-	dev_dbg(pmx->dev, "disabling %s function for pin group %s\n",
-		artpec6_pmx_get_fname(pctldev, function),
-		artpec6_get_group_name(pctldev, group));
-
-	artpec6_pmx_select_func(pctldev, function, group, false);
-}
-
 static int artpec6_pmx_request_gpio(struct pinctrl_dev *pctldev,
 				    struct pinctrl_gpio_range *range,
 				    unsigned int pin)
@@ -737,7 +726,7 @@ static const struct pinmux_ops artpec6_pmx_ops = {
 	.get_functions_count	= artpec6_pmx_get_functions_count,
 	.get_function_name	= artpec6_pmx_get_fname,
 	.get_function_groups	= artpec6_pmx_get_fgroups,
-	.set_mux		= artpec6_pmx_enable,
+	.set_mux		= artpec6_pmx_set,
 	.gpio_request_enable = artpec6_pmx_request_gpio,
 };
 
diff --git a/drivers/pinctrl/pinctrl-axp209.c b/drivers/pinctrl/pinctrl-axp209.c
index afd0b533c40a..4fcf7262bed9 100644
--- a/drivers/pinctrl/pinctrl-axp209.c
+++ b/drivers/pinctrl/pinctrl-axp209.c
@@ -366,6 +366,8 @@ static int axp20x_build_funcs_groups(struct platform_device *pdev)
 		pctl->funcs[i].groups = devm_kcalloc(&pdev->dev,
 						     npins, sizeof(char *),
 						     GFP_KERNEL);
+		if (!pctl->funcs[i].groups)
+			return -ENOMEM;
 		for (pin = 0; pin < npins; pin++)
 			pctl->funcs[i].groups[pin] = pctl->desc->pins[pin].name;
 	}
diff --git a/drivers/pinctrl/pinctrl-bm1880.c b/drivers/pinctrl/pinctrl-bm1880.c
new file mode 100644
index 000000000000..446b07d8fbfc
--- /dev/null
+++ b/drivers/pinctrl/pinctrl-bm1880.c
@@ -0,0 +1,965 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Bitmain BM1880 SoC Pinctrl driver
+ *
+ * Copyright (c) 2019 Linaro Ltd.
+ * Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
+ */
+
+#include <linux/io.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pinctrl/pinctrl.h>
+#include <linux/pinctrl/pinmux.h>
+#include <linux/pinctrl/pinconf-generic.h>
+#include <linux/slab.h>
+
+#include "core.h"
+#include "pinctrl-utils.h"
+
+#define BM1880_REG_MUX 0x20
+
+/**
+ * struct bm1880_pinctrl - driver data
+ * @base:	Pinctrl base address
+ * @pctrl:	Pinctrl device
+ * @groups:	Pingroups
+ * @ngroups:	Number of @groups
+ * @funcs:	Pinmux functions
+ * @nfuncs:	Number of @funcs
+ */
+struct bm1880_pinctrl {
+	void __iomem *base;
+	struct pinctrl_dev *pctrldev;
+	const struct bm1880_pctrl_group *groups;
+	unsigned int ngroups;
+	const struct bm1880_pinmux_function *funcs;
+	unsigned int nfuncs;
+};
+
+/**
+ * struct bm1880_pctrl_group - pinctrl group
+ * @name:	Name of the group
+ * @pins:	Array of pins belonging to this group
+ * @npins:	Number of @pins
+ */
+struct bm1880_pctrl_group {
+	const char *name;
+	const unsigned int *pins;
+	const unsigned int npins;
+};
+
+/**
+ * struct bm1880_pinmux_function - a pinmux function
+ * @name:	Name of the pinmux function.
+ * @groups:	List of pingroups for this function.
+ * @ngroups:	Number of entries in @groups.
+ * @mux_val:	Selector for this function
+ * @mux_mask:   Mask for function specific selector
+ * @mux:	Offset of function specific mux
+ * @mux_shift:	Shift for function specific selector
+ */
+struct bm1880_pinmux_function {
+	const char *name;
+	const char * const *groups;
+	unsigned int ngroups;
+	u32 mux_val;
+	u32 mux_mask;
+	u32 mux;
+	u8 mux_shift;
+};
+
+static const struct pinctrl_pin_desc bm1880_pins[] = {
+	PINCTRL_PIN(0,   "MIO0"),
+	PINCTRL_PIN(1,   "MIO1"),
+	PINCTRL_PIN(2,   "MIO2"),
+	PINCTRL_PIN(3,   "MIO3"),
+	PINCTRL_PIN(4,   "MIO4"),
+	PINCTRL_PIN(5,   "MIO5"),
+	PINCTRL_PIN(6,   "MIO6"),
+	PINCTRL_PIN(7,   "MIO7"),
+	PINCTRL_PIN(8,   "MIO8"),
+	PINCTRL_PIN(9,   "MIO9"),
+	PINCTRL_PIN(10,   "MIO10"),
+	PINCTRL_PIN(11,   "MIO11"),
+	PINCTRL_PIN(12,   "MIO12"),
+	PINCTRL_PIN(13,   "MIO13"),
+	PINCTRL_PIN(14,   "MIO14"),
+	PINCTRL_PIN(15,   "MIO15"),
+	PINCTRL_PIN(16,   "MIO16"),
+	PINCTRL_PIN(17,   "MIO17"),
+	PINCTRL_PIN(18,   "MIO18"),
+	PINCTRL_PIN(19,   "MIO19"),
+	PINCTRL_PIN(20,   "MIO20"),
+	PINCTRL_PIN(21,   "MIO21"),
+	PINCTRL_PIN(22,   "MIO22"),
+	PINCTRL_PIN(23,   "MIO23"),
+	PINCTRL_PIN(24,   "MIO24"),
+	PINCTRL_PIN(25,   "MIO25"),
+	PINCTRL_PIN(26,   "MIO26"),
+	PINCTRL_PIN(27,   "MIO27"),
+	PINCTRL_PIN(28,   "MIO28"),
+	PINCTRL_PIN(29,   "MIO29"),
+	PINCTRL_PIN(30,   "MIO30"),
+	PINCTRL_PIN(31,   "MIO31"),
+	PINCTRL_PIN(32,   "MIO32"),
+	PINCTRL_PIN(33,   "MIO33"),
+	PINCTRL_PIN(34,   "MIO34"),
+	PINCTRL_PIN(35,   "MIO35"),
+	PINCTRL_PIN(36,   "MIO36"),
+	PINCTRL_PIN(37,   "MIO37"),
+	PINCTRL_PIN(38,   "MIO38"),
+	PINCTRL_PIN(39,   "MIO39"),
+	PINCTRL_PIN(40,   "MIO40"),
+	PINCTRL_PIN(41,   "MIO41"),
+	PINCTRL_PIN(42,   "MIO42"),
+	PINCTRL_PIN(43,   "MIO43"),
+	PINCTRL_PIN(44,   "MIO44"),
+	PINCTRL_PIN(45,   "MIO45"),
+	PINCTRL_PIN(46,   "MIO46"),
+	PINCTRL_PIN(47,   "MIO47"),
+	PINCTRL_PIN(48,   "MIO48"),
+	PINCTRL_PIN(49,   "MIO49"),
+	PINCTRL_PIN(50,   "MIO50"),
+	PINCTRL_PIN(51,   "MIO51"),
+	PINCTRL_PIN(52,   "MIO52"),
+	PINCTRL_PIN(53,   "MIO53"),
+	PINCTRL_PIN(54,   "MIO54"),
+	PINCTRL_PIN(55,   "MIO55"),
+	PINCTRL_PIN(56,   "MIO56"),
+	PINCTRL_PIN(57,   "MIO57"),
+	PINCTRL_PIN(58,   "MIO58"),
+	PINCTRL_PIN(59,   "MIO59"),
+	PINCTRL_PIN(60,   "MIO60"),
+	PINCTRL_PIN(61,   "MIO61"),
+	PINCTRL_PIN(62,   "MIO62"),
+	PINCTRL_PIN(63,   "MIO63"),
+	PINCTRL_PIN(64,   "MIO64"),
+	PINCTRL_PIN(65,   "MIO65"),
+	PINCTRL_PIN(66,   "MIO66"),
+	PINCTRL_PIN(67,   "MIO67"),
+	PINCTRL_PIN(68,   "MIO68"),
+	PINCTRL_PIN(69,   "MIO69"),
+	PINCTRL_PIN(70,   "MIO70"),
+	PINCTRL_PIN(71,   "MIO71"),
+	PINCTRL_PIN(72,   "MIO72"),
+	PINCTRL_PIN(73,   "MIO73"),
+	PINCTRL_PIN(74,   "MIO74"),
+	PINCTRL_PIN(75,   "MIO75"),
+	PINCTRL_PIN(76,   "MIO76"),
+	PINCTRL_PIN(77,   "MIO77"),
+	PINCTRL_PIN(78,   "MIO78"),
+	PINCTRL_PIN(79,   "MIO79"),
+	PINCTRL_PIN(80,   "MIO80"),
+	PINCTRL_PIN(81,   "MIO81"),
+	PINCTRL_PIN(82,   "MIO82"),
+	PINCTRL_PIN(83,   "MIO83"),
+	PINCTRL_PIN(84,   "MIO84"),
+	PINCTRL_PIN(85,   "MIO85"),
+	PINCTRL_PIN(86,   "MIO86"),
+	PINCTRL_PIN(87,   "MIO87"),
+	PINCTRL_PIN(88,   "MIO88"),
+	PINCTRL_PIN(89,   "MIO89"),
+	PINCTRL_PIN(90,   "MIO90"),
+	PINCTRL_PIN(91,   "MIO91"),
+	PINCTRL_PIN(92,   "MIO92"),
+	PINCTRL_PIN(93,   "MIO93"),
+	PINCTRL_PIN(94,   "MIO94"),
+	PINCTRL_PIN(95,   "MIO95"),
+	PINCTRL_PIN(96,   "MIO96"),
+	PINCTRL_PIN(97,   "MIO97"),
+	PINCTRL_PIN(98,   "MIO98"),
+	PINCTRL_PIN(99,   "MIO99"),
+	PINCTRL_PIN(100,   "MIO100"),
+	PINCTRL_PIN(101,   "MIO101"),
+	PINCTRL_PIN(102,   "MIO102"),
+	PINCTRL_PIN(103,   "MIO103"),
+	PINCTRL_PIN(104,   "MIO104"),
+	PINCTRL_PIN(105,   "MIO105"),
+	PINCTRL_PIN(106,   "MIO106"),
+	PINCTRL_PIN(107,   "MIO107"),
+	PINCTRL_PIN(108,   "MIO108"),
+	PINCTRL_PIN(109,   "MIO109"),
+	PINCTRL_PIN(110,   "MIO110"),
+	PINCTRL_PIN(111,   "MIO111"),
+};
+
+enum bm1880_pinmux_functions {
+	F_nand, F_spi, F_emmc, F_sdio, F_eth0, F_pwm0, F_pwm1, F_pwm2,
+	F_pwm3, F_pwm4, F_pwm5, F_pwm6, F_pwm7, F_pwm8, F_pwm9, F_pwm10,
+	F_pwm11, F_pwm12, F_pwm13, F_pwm14, F_pwm15, F_pwm16, F_pwm17,
+	F_pwm18, F_pwm19, F_pwm20, F_pwm21, F_pwm22, F_pwm23, F_pwm24,
+	F_pwm25, F_pwm26, F_pwm27, F_pwm28, F_pwm29, F_pwm30, F_pwm31,
+	F_pwm32, F_pwm33, F_pwm34, F_pwm35, F_pwm36, F_pwm37, F_i2c0, F_i2c1,
+	F_i2c2, F_i2c3, F_i2c4, F_uart0, F_uart1, F_uart2, F_uart3, F_uart4,
+	F_uart5, F_uart6, F_uart7, F_uart8, F_uart9, F_uart10, F_uart11,
+	F_uart12, F_uart13, F_uart14, F_uart15, F_gpio0, F_gpio1, F_gpio2,
+	F_gpio3, F_gpio4, F_gpio5, F_gpio6, F_gpio7, F_gpio8, F_gpio9, F_gpio10,
+	F_gpio11, F_gpio12, F_gpio13, F_gpio14, F_gpio15, F_gpio16, F_gpio17,
+	F_gpio18, F_gpio19, F_gpio20, F_gpio21, F_gpio22, F_gpio23, F_gpio24,
+	F_gpio25, F_gpio26, F_gpio27, F_gpio28, F_gpio29, F_gpio30, F_gpio31,
+	F_gpio32, F_gpio33, F_gpio34, F_gpio35, F_gpio36, F_gpio37, F_gpio38,
+	F_gpio39, F_gpio40, F_gpio41, F_gpio42, F_gpio43, F_gpio44, F_gpio45,
+	F_gpio46, F_gpio47, F_gpio48, F_gpio49, F_gpio50, F_gpio51, F_gpio52,
+	F_gpio53, F_gpio54, F_gpio55, F_gpio56, F_gpio57, F_gpio58, F_gpio59,
+	F_gpio60, F_gpio61, F_gpio62, F_gpio63, F_gpio64, F_gpio65, F_gpio66,
+	F_gpio67, F_eth1, F_i2s0, F_i2s0_mclkin, F_i2s1, F_i2s1_mclkin, F_spi0,
+	F_max
+};
+
+static const unsigned int nand_pins[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
+					  10, 11, 12, 13, 14, 15, 16 };
+static const unsigned int spi_pins[] = { 0, 1, 8, 10, 11, 12, 13 };
+static const unsigned int emmc_pins[] = { 2, 3, 4, 5, 6, 7, 9, 14, 15, 16 };
+static const unsigned int sdio_pins[] = { 17, 18, 19, 20, 21, 22, 23, 24,
+					  25, 26 };
+static const unsigned int eth0_pins[] = { 27, 28, 29, 30, 31, 32, 33, 34, 35,
+					  36, 37, 38, 39, 40, 41, 42 };
+static const unsigned int pwm0_pins[] = { 29 };
+static const unsigned int pwm1_pins[] = { 30 };
+static const unsigned int pwm2_pins[] = { 34 };
+static const unsigned int pwm3_pins[] = { 35 };
+static const unsigned int pwm4_pins[] = { 43 };
+static const unsigned int pwm5_pins[] = { 44 };
+static const unsigned int pwm6_pins[] = { 45 };
+static const unsigned int pwm7_pins[] = { 46 };
+static const unsigned int pwm8_pins[] = { 47 };
+static const unsigned int pwm9_pins[] = { 48 };
+static const unsigned int pwm10_pins[] = { 49 };
+static const unsigned int pwm11_pins[] = { 50 };
+static const unsigned int pwm12_pins[] = { 51 };
+static const unsigned int pwm13_pins[] = { 52 };
+static const unsigned int pwm14_pins[] = { 53 };
+static const unsigned int pwm15_pins[] = { 54 };
+static const unsigned int pwm16_pins[] = { 55 };
+static const unsigned int pwm17_pins[] = { 56 };
+static const unsigned int pwm18_pins[] = { 57 };
+static const unsigned int pwm19_pins[] = { 58 };
+static const unsigned int pwm20_pins[] = { 59 };
+static const unsigned int pwm21_pins[] = { 60 };
+static const unsigned int pwm22_pins[] = { 61 };
+static const unsigned int pwm23_pins[] = { 62 };
+static const unsigned int pwm24_pins[] = { 97 };
+static const unsigned int pwm25_pins[] = { 98 };
+static const unsigned int pwm26_pins[] = { 99 };
+static const unsigned int pwm27_pins[] = { 100 };
+static const unsigned int pwm28_pins[] = { 101 };
+static const unsigned int pwm29_pins[] = { 102 };
+static const unsigned int pwm30_pins[] = { 103 };
+static const unsigned int pwm31_pins[] = { 104 };
+static const unsigned int pwm32_pins[] = { 105 };
+static const unsigned int pwm33_pins[] = { 106 };
+static const unsigned int pwm34_pins[] = { 107 };
+static const unsigned int pwm35_pins[] = { 108 };
+static const unsigned int pwm36_pins[] = { 109 };
+static const unsigned int pwm37_pins[] = { 110 };
+static const unsigned int i2c0_pins[] = { 63, 64 };
+static const unsigned int i2c1_pins[] = { 65, 66 };
+static const unsigned int i2c2_pins[] = { 67, 68 };
+static const unsigned int i2c3_pins[] = { 69, 70 };
+static const unsigned int i2c4_pins[] = { 71, 72 };
+static const unsigned int uart0_pins[] = { 73, 74 };
+static const unsigned int uart1_pins[] = { 75, 76 };
+static const unsigned int uart2_pins[] = { 77, 78 };
+static const unsigned int uart3_pins[] = { 79, 80 };
+static const unsigned int uart4_pins[] = { 81, 82 };
+static const unsigned int uart5_pins[] = { 83, 84 };
+static const unsigned int uart6_pins[] = { 85, 86 };
+static const unsigned int uart7_pins[] = { 87, 88 };
+static const unsigned int uart8_pins[] = { 89, 90 };
+static const unsigned int uart9_pins[] = { 91, 92 };
+static const unsigned int uart10_pins[] = { 93, 94 };
+static const unsigned int uart11_pins[] = { 95, 96 };
+static const unsigned int uart12_pins[] = { 73, 74, 75, 76 };
+static const unsigned int uart13_pins[] = { 77, 78, 83, 84 };
+static const unsigned int uart14_pins[] = { 79, 80, 85, 86 };
+static const unsigned int uart15_pins[] = { 81, 82, 87, 88 };
+static const unsigned int gpio0_pins[] = { 97 };
+static const unsigned int gpio1_pins[] = { 98 };
+static const unsigned int gpio2_pins[] = { 99 };
+static const unsigned int gpio3_pins[] = { 100 };
+static const unsigned int gpio4_pins[] = { 101 };
+static const unsigned int gpio5_pins[] = { 102 };
+static const unsigned int gpio6_pins[] = { 103 };
+static const unsigned int gpio7_pins[] = { 104 };
+static const unsigned int gpio8_pins[] = { 105 };
+static const unsigned int gpio9_pins[] = { 106 };
+static const unsigned int gpio10_pins[] = { 107 };
+static const unsigned int gpio11_pins[] = { 108 };
+static const unsigned int gpio12_pins[] = { 109 };
+static const unsigned int gpio13_pins[] = { 110 };
+static const unsigned int gpio14_pins[] = { 43 };
+static const unsigned int gpio15_pins[] = { 44 };
+static const unsigned int gpio16_pins[] = { 45 };
+static const unsigned int gpio17_pins[] = { 46 };
+static const unsigned int gpio18_pins[] = { 47 };
+static const unsigned int gpio19_pins[] = { 48 };
+static const unsigned int gpio20_pins[] = { 49 };
+static const unsigned int gpio21_pins[] = { 50 };
+static const unsigned int gpio22_pins[] = { 51 };
+static const unsigned int gpio23_pins[] = { 52 };
+static const unsigned int gpio24_pins[] = { 53 };
+static const unsigned int gpio25_pins[] = { 54 };
+static const unsigned int gpio26_pins[] = { 55 };
+static const unsigned int gpio27_pins[] = { 56 };
+static const unsigned int gpio28_pins[] = { 57 };
+static const unsigned int gpio29_pins[] = { 58 };
+static const unsigned int gpio30_pins[] = { 59 };
+static const unsigned int gpio31_pins[] = { 60 };
+static const unsigned int gpio32_pins[] = { 61 };
+static const unsigned int gpio33_pins[] = { 62 };
+static const unsigned int gpio34_pins[] = { 63 };
+static const unsigned int gpio35_pins[] = { 64 };
+static const unsigned int gpio36_pins[] = { 65 };
+static const unsigned int gpio37_pins[] = { 66 };
+static const unsigned int gpio38_pins[] = { 67 };
+static const unsigned int gpio39_pins[] = { 68 };
+static const unsigned int gpio40_pins[] = { 69 };
+static const unsigned int gpio41_pins[] = { 70 };
+static const unsigned int gpio42_pins[] = { 71 };
+static const unsigned int gpio43_pins[] = { 72 };
+static const unsigned int gpio44_pins[] = { 73 };
+static const unsigned int gpio45_pins[] = { 74 };
+static const unsigned int gpio46_pins[] = { 75 };
+static const unsigned int gpio47_pins[] = { 76 };
+static const unsigned int gpio48_pins[] = { 77 };
+static const unsigned int gpio49_pins[] = { 78 };
+static const unsigned int gpio50_pins[] = { 79 };
+static const unsigned int gpio51_pins[] = { 80 };
+static const unsigned int gpio52_pins[] = { 81 };
+static const unsigned int gpio53_pins[] = { 82 };
+static const unsigned int gpio54_pins[] = { 83 };
+static const unsigned int gpio55_pins[] = { 84 };
+static const unsigned int gpio56_pins[] = { 85 };
+static const unsigned int gpio57_pins[] = { 86 };
+static const unsigned int gpio58_pins[] = { 87 };
+static const unsigned int gpio59_pins[] = { 88 };
+static const unsigned int gpio60_pins[] = { 89 };
+static const unsigned int gpio61_pins[] = { 90 };
+static const unsigned int gpio62_pins[] = { 91 };
+static const unsigned int gpio63_pins[] = { 92 };
+static const unsigned int gpio64_pins[] = { 93 };
+static const unsigned int gpio65_pins[] = { 94 };
+static const unsigned int gpio66_pins[] = { 95 };
+static const unsigned int gpio67_pins[] = { 96 };
+static const unsigned int eth1_pins[] = { 43, 44, 45, 46, 47, 48, 49, 50, 51,
+					  52, 53, 54, 55, 56, 57, 58 };
+static const unsigned int i2s0_pins[] = { 87, 88, 89, 90, 91 };
+static const unsigned int i2s0_mclkin_pins[] = { 97 };
+static const unsigned int i2s1_pins[] = { 92, 93, 94, 95, 96 };
+static const unsigned int i2s1_mclkin_pins[] = { 98 };
+static const unsigned int spi0_pins[] = { 59, 60, 61, 62 };
+
+#define BM1880_PINCTRL_GRP(nm) \
+	{ \
+		.name = #nm "_grp", \
+		.pins = nm ## _pins, \
+		.npins = ARRAY_SIZE(nm ## _pins), \
+	}
+
+static const struct bm1880_pctrl_group bm1880_pctrl_groups[] = {
+	BM1880_PINCTRL_GRP(nand),
+	BM1880_PINCTRL_GRP(spi),
+	BM1880_PINCTRL_GRP(emmc),
+	BM1880_PINCTRL_GRP(sdio),
+	BM1880_PINCTRL_GRP(eth0),
+	BM1880_PINCTRL_GRP(pwm0),
+	BM1880_PINCTRL_GRP(pwm1),
+	BM1880_PINCTRL_GRP(pwm2),
+	BM1880_PINCTRL_GRP(pwm3),
+	BM1880_PINCTRL_GRP(pwm4),
+	BM1880_PINCTRL_GRP(pwm5),
+	BM1880_PINCTRL_GRP(pwm6),
+	BM1880_PINCTRL_GRP(pwm7),
+	BM1880_PINCTRL_GRP(pwm8),
+	BM1880_PINCTRL_GRP(pwm9),
+	BM1880_PINCTRL_GRP(pwm10),
+	BM1880_PINCTRL_GRP(pwm11),
+	BM1880_PINCTRL_GRP(pwm12),
+	BM1880_PINCTRL_GRP(pwm13),
+	BM1880_PINCTRL_GRP(pwm14),
+	BM1880_PINCTRL_GRP(pwm15),
+	BM1880_PINCTRL_GRP(pwm16),
+	BM1880_PINCTRL_GRP(pwm17),
+	BM1880_PINCTRL_GRP(pwm18),
+	BM1880_PINCTRL_GRP(pwm19),
+	BM1880_PINCTRL_GRP(pwm20),
+	BM1880_PINCTRL_GRP(pwm21),
+	BM1880_PINCTRL_GRP(pwm22),
+	BM1880_PINCTRL_GRP(pwm23),
+	BM1880_PINCTRL_GRP(pwm24),
+	BM1880_PINCTRL_GRP(pwm25),
+	BM1880_PINCTRL_GRP(pwm26),
+	BM1880_PINCTRL_GRP(pwm27),
+	BM1880_PINCTRL_GRP(pwm28),
+	BM1880_PINCTRL_GRP(pwm29),
+	BM1880_PINCTRL_GRP(pwm30),
+	BM1880_PINCTRL_GRP(pwm31),
+	BM1880_PINCTRL_GRP(pwm32),
+	BM1880_PINCTRL_GRP(pwm33),
+	BM1880_PINCTRL_GRP(pwm34),
+	BM1880_PINCTRL_GRP(pwm35),
+	BM1880_PINCTRL_GRP(pwm36),
+	BM1880_PINCTRL_GRP(i2c0),
+	BM1880_PINCTRL_GRP(i2c1),
+	BM1880_PINCTRL_GRP(i2c2),
+	BM1880_PINCTRL_GRP(i2c3),
+	BM1880_PINCTRL_GRP(i2c4),
+	BM1880_PINCTRL_GRP(uart0),
+	BM1880_PINCTRL_GRP(uart1),
+	BM1880_PINCTRL_GRP(uart2),
+	BM1880_PINCTRL_GRP(uart3),
+	BM1880_PINCTRL_GRP(uart4),
+	BM1880_PINCTRL_GRP(uart5),
+	BM1880_PINCTRL_GRP(uart6),
+	BM1880_PINCTRL_GRP(uart7),
+	BM1880_PINCTRL_GRP(uart8),
+	BM1880_PINCTRL_GRP(uart9),
+	BM1880_PINCTRL_GRP(uart10),
+	BM1880_PINCTRL_GRP(uart11),
+	BM1880_PINCTRL_GRP(uart12),
+	BM1880_PINCTRL_GRP(uart13),
+	BM1880_PINCTRL_GRP(uart14),
+	BM1880_PINCTRL_GRP(uart15),
+	BM1880_PINCTRL_GRP(gpio0),
+	BM1880_PINCTRL_GRP(gpio1),
+	BM1880_PINCTRL_GRP(gpio2),
+	BM1880_PINCTRL_GRP(gpio3),
+	BM1880_PINCTRL_GRP(gpio4),
+	BM1880_PINCTRL_GRP(gpio5),
+	BM1880_PINCTRL_GRP(gpio6),
+	BM1880_PINCTRL_GRP(gpio7),
+	BM1880_PINCTRL_GRP(gpio8),
+	BM1880_PINCTRL_GRP(gpio9),
+	BM1880_PINCTRL_GRP(gpio10),
+	BM1880_PINCTRL_GRP(gpio11),
+	BM1880_PINCTRL_GRP(gpio12),
+	BM1880_PINCTRL_GRP(gpio13),
+	BM1880_PINCTRL_GRP(gpio14),
+	BM1880_PINCTRL_GRP(gpio15),
+	BM1880_PINCTRL_GRP(gpio16),
+	BM1880_PINCTRL_GRP(gpio17),
+	BM1880_PINCTRL_GRP(gpio18),
+	BM1880_PINCTRL_GRP(gpio19),
+	BM1880_PINCTRL_GRP(gpio20),
+	BM1880_PINCTRL_GRP(gpio21),
+	BM1880_PINCTRL_GRP(gpio22),
+	BM1880_PINCTRL_GRP(gpio23),
+	BM1880_PINCTRL_GRP(gpio24),
+	BM1880_PINCTRL_GRP(gpio25),
+	BM1880_PINCTRL_GRP(gpio26),
+	BM1880_PINCTRL_GRP(gpio27),
+	BM1880_PINCTRL_GRP(gpio28),
+	BM1880_PINCTRL_GRP(gpio29),
+	BM1880_PINCTRL_GRP(gpio30),
+	BM1880_PINCTRL_GRP(gpio31),
+	BM1880_PINCTRL_GRP(gpio32),
+	BM1880_PINCTRL_GRP(gpio33),
+	BM1880_PINCTRL_GRP(gpio34),
+	BM1880_PINCTRL_GRP(gpio35),
+	BM1880_PINCTRL_GRP(gpio36),
+	BM1880_PINCTRL_GRP(gpio37),
+	BM1880_PINCTRL_GRP(gpio38),
+	BM1880_PINCTRL_GRP(gpio39),
+	BM1880_PINCTRL_GRP(gpio40),
+	BM1880_PINCTRL_GRP(gpio41),
+	BM1880_PINCTRL_GRP(gpio42),
+	BM1880_PINCTRL_GRP(gpio43),
+	BM1880_PINCTRL_GRP(gpio44),
+	BM1880_PINCTRL_GRP(gpio45),
+	BM1880_PINCTRL_GRP(gpio46),
+	BM1880_PINCTRL_GRP(gpio47),
+	BM1880_PINCTRL_GRP(gpio48),
+	BM1880_PINCTRL_GRP(gpio49),
+	BM1880_PINCTRL_GRP(gpio50),
+	BM1880_PINCTRL_GRP(gpio51),
+	BM1880_PINCTRL_GRP(gpio52),
+	BM1880_PINCTRL_GRP(gpio53),
+	BM1880_PINCTRL_GRP(gpio54),
+	BM1880_PINCTRL_GRP(gpio55),
+	BM1880_PINCTRL_GRP(gpio56),
+	BM1880_PINCTRL_GRP(gpio57),
+	BM1880_PINCTRL_GRP(gpio58),
+	BM1880_PINCTRL_GRP(gpio59),
+	BM1880_PINCTRL_GRP(gpio60),
+	BM1880_PINCTRL_GRP(gpio61),
+	BM1880_PINCTRL_GRP(gpio62),
+	BM1880_PINCTRL_GRP(gpio63),
+	BM1880_PINCTRL_GRP(gpio64),
+	BM1880_PINCTRL_GRP(gpio65),
+	BM1880_PINCTRL_GRP(gpio66),
+	BM1880_PINCTRL_GRP(gpio67),
+	BM1880_PINCTRL_GRP(eth1),
+	BM1880_PINCTRL_GRP(i2s0),
+	BM1880_PINCTRL_GRP(i2s0_mclkin),
+	BM1880_PINCTRL_GRP(i2s1),
+	BM1880_PINCTRL_GRP(i2s1_mclkin),
+	BM1880_PINCTRL_GRP(spi0),
+};
+
+static const char * const nand_group[] = { "nand_grp" };
+static const char * const spi_group[] = { "spi_grp" };
+static const char * const emmc_group[] = { "emmc_grp" };
+static const char * const sdio_group[] = { "sdio_grp" };
+static const char * const eth0_group[] = { "eth0_grp" };
+static const char * const pwm0_group[] = { "pwm0_grp" };
+static const char * const pwm1_group[] = { "pwm1_grp" };
+static const char * const pwm2_group[] = { "pwm2_grp" };
+static const char * const pwm3_group[] = { "pwm3_grp" };
+static const char * const pwm4_group[] = { "pwm4_grp" };
+static const char * const pwm5_group[] = { "pwm5_grp" };
+static const char * const pwm6_group[] = { "pwm6_grp" };
+static const char * const pwm7_group[] = { "pwm7_grp" };
+static const char * const pwm8_group[] = { "pwm8_grp" };
+static const char * const pwm9_group[] = { "pwm9_grp" };
+static const char * const pwm10_group[] = { "pwm10_grp" };
+static const char * const pwm11_group[] = { "pwm11_grp" };
+static const char * const pwm12_group[] = { "pwm12_grp" };
+static const char * const pwm13_group[] = { "pwm13_grp" };
+static const char * const pwm14_group[] = { "pwm14_grp" };
+static const char * const pwm15_group[] = { "pwm15_grp" };
+static const char * const pwm16_group[] = { "pwm16_grp" };
+static const char * const pwm17_group[] = { "pwm17_grp" };
+static const char * const pwm18_group[] = { "pwm18_grp" };
+static const char * const pwm19_group[] = { "pwm19_grp" };
+static const char * const pwm20_group[] = { "pwm20_grp" };
+static const char * const pwm21_group[] = { "pwm21_grp" };
+static const char * const pwm22_group[] = { "pwm22_grp" };
+static const char * const pwm23_group[] = { "pwm23_grp" };
+static const char * const pwm24_group[] = { "pwm24_grp" };
+static const char * const pwm25_group[] = { "pwm25_grp" };
+static const char * const pwm26_group[] = { "pwm26_grp" };
+static const char * const pwm27_group[] = { "pwm27_grp" };
+static const char * const pwm28_group[] = { "pwm28_grp" };
+static const char * const pwm29_group[] = { "pwm29_grp" };
+static const char * const pwm30_group[] = { "pwm30_grp" };
+static const char * const pwm31_group[] = { "pwm31_grp" };
+static const char * const pwm32_group[] = { "pwm32_grp" };
+static const char * const pwm33_group[] = { "pwm33_grp" };
+static const char * const pwm34_group[] = { "pwm34_grp" };
+static const char * const pwm35_group[] = { "pwm35_grp" };
+static const char * const pwm36_group[] = { "pwm36_grp" };
+static const char * const pwm37_group[] = { "pwm37_grp" };
+static const char * const i2c0_group[] = { "i2c0_grp" };
+static const char * const i2c1_group[] = { "i2c1_grp" };
+static const char * const i2c2_group[] = { "i2c2_grp" };
+static const char * const i2c3_group[] = { "i2c3_grp" };
+static const char * const i2c4_group[] = { "i2c4_grp" };
+static const char * const uart0_group[] = { "uart0_grp" };
+static const char * const uart1_group[] = { "uart1_grp" };
+static const char * const uart2_group[] = { "uart2_grp" };
+static const char * const uart3_group[] = { "uart3_grp" };
+static const char * const uart4_group[] = { "uart4_grp" };
+static const char * const uart5_group[] = { "uart5_grp" };
+static const char * const uart6_group[] = { "uart6_grp" };
+static const char * const uart7_group[] = { "uart7_grp" };
+static const char * const uart8_group[] = { "uart8_grp" };
+static const char * const uart9_group[] = { "uart9_grp" };
+static const char * const uart10_group[] = { "uart10_grp" };
+static const char * const uart11_group[] = { "uart11_grp" };
+static const char * const uart12_group[] = { "uart12_grp" };
+static const char * const uart13_group[] = { "uart13_grp" };
+static const char * const uart14_group[] = { "uart14_grp" };
+static const char * const uart15_group[] = { "uart15_grp" };
+static const char * const gpio0_group[] = { "gpio0_grp" };
+static const char * const gpio1_group[] = { "gpio1_grp" };
+static const char * const gpio2_group[] = { "gpio2_grp" };
+static const char * const gpio3_group[] = { "gpio3_grp" };
+static const char * const gpio4_group[] = { "gpio4_grp" };
+static const char * const gpio5_group[] = { "gpio5_grp" };
+static const char * const gpio6_group[] = { "gpio6_grp" };
+static const char * const gpio7_group[] = { "gpio7_grp" };
+static const char * const gpio8_group[] = { "gpio8_grp" };
+static const char * const gpio9_group[] = { "gpio9_grp" };
+static const char * const gpio10_group[] = { "gpio10_grp" };
+static const char * const gpio11_group[] = { "gpio11_grp" };
+static const char * const gpio12_group[] = { "gpio12_grp" };
+static const char * const gpio13_group[] = { "gpio13_grp" };
+static const char * const gpio14_group[] = { "gpio14_grp" };
+static const char * const gpio15_group[] = { "gpio15_grp" };
+static const char * const gpio16_group[] = { "gpio16_grp" };
+static const char * const gpio17_group[] = { "gpio17_grp" };
+static const char * const gpio18_group[] = { "gpio18_grp" };
+static const char * const gpio19_group[] = { "gpio19_grp" };
+static const char * const gpio20_group[] = { "gpio20_grp" };
+static const char * const gpio21_group[] = { "gpio21_grp" };
+static const char * const gpio22_group[] = { "gpio22_grp" };
+static const char * const gpio23_group[] = { "gpio23_grp" };
+static const char * const gpio24_group[] = { "gpio24_grp" };
+static const char * const gpio25_group[] = { "gpio25_grp" };
+static const char * const gpio26_group[] = { "gpio26_grp" };
+static const char * const gpio27_group[] = { "gpio27_grp" };
+static const char * const gpio28_group[] = { "gpio28_grp" };
+static const char * const gpio29_group[] = { "gpio29_grp" };
+static const char * const gpio30_group[] = { "gpio30_grp" };
+static const char * const gpio31_group[] = { "gpio31_grp" };
+static const char * const gpio32_group[] = { "gpio32_grp" };
+static const char * const gpio33_group[] = { "gpio33_grp" };
+static const char * const gpio34_group[] = { "gpio34_grp" };
+static const char * const gpio35_group[] = { "gpio35_grp" };
+static const char * const gpio36_group[] = { "gpio36_grp" };
+static const char * const gpio37_group[] = { "gpio37_grp" };
+static const char * const gpio38_group[] = { "gpio38_grp" };
+static const char * const gpio39_group[] = { "gpio39_grp" };
+static const char * const gpio40_group[] = { "gpio40_grp" };
+static const char * const gpio41_group[] = { "gpio41_grp" };
+static const char * const gpio42_group[] = { "gpio42_grp" };
+static const char * const gpio43_group[] = { "gpio43_grp" };
+static const char * const gpio44_group[] = { "gpio44_grp" };
+static const char * const gpio45_group[] = { "gpio45_grp" };
+static const char * const gpio46_group[] = { "gpio46_grp" };
+static const char * const gpio47_group[] = { "gpio47_grp" };
+static const char * const gpio48_group[] = { "gpio48_grp" };
+static const char * const gpio49_group[] = { "gpio49_grp" };
+static const char * const gpio50_group[] = { "gpio50_grp" };
+static const char * const gpio51_group[] = { "gpio51_grp" };
+static const char * const gpio52_group[] = { "gpio52_grp" };
+static const char * const gpio53_group[] = { "gpio53_grp" };
+static const char * const gpio54_group[] = { "gpio54_grp" };
+static const char * const gpio55_group[] = { "gpio55_grp" };
+static const char * const gpio56_group[] = { "gpio56_grp" };
+static const char * const gpio57_group[] = { "gpio57_grp" };
+static const char * const gpio58_group[] = { "gpio58_grp" };
+static const char * const gpio59_group[] = { "gpio59_grp" };
+static const char * const gpio60_group[] = { "gpio60_grp" };
+static const char * const gpio61_group[] = { "gpio61_grp" };
+static const char * const gpio62_group[] = { "gpio62_grp" };
+static const char * const gpio63_group[] = { "gpio63_grp" };
+static const char * const gpio64_group[] = { "gpio64_grp" };
+static const char * const gpio65_group[] = { "gpio65_grp" };
+static const char * const gpio66_group[] = { "gpio66_grp" };
+static const char * const gpio67_group[] = { "gpio67_grp" };
+static const char * const eth1_group[] = { "eth1_grp" };
+static const char * const i2s0_group[] = { "i2s0_grp" };
+static const char * const i2s0_mclkin_group[] = { "i2s0_mclkin_grp" };
+static const char * const i2s1_group[] = { "i2s1_grp" };
+static const char * const i2s1_mclkin_group[] = { "i2s1_mclkin_grp" };
+static const char * const spi0_group[] = { "spi0_grp" };
+
+#define BM1880_PINMUX_FUNCTION(fname, mval, mask)	\
+	[F_##fname] = {					\
+		.name = #fname,				\
+		.groups = fname##_group,		\
+		.ngroups = ARRAY_SIZE(fname##_group),	\
+		.mux_val = mval,			\
+		.mux_mask = mask,			\
+	}
+
+#define BM1880_PINMUX_FUNCTION_MUX(fname, mval, mask, offset, shift)\
+	[F_##fname] = {					\
+		.name = #fname,				\
+		.groups = fname##_group,		\
+		.ngroups = ARRAY_SIZE(fname##_group),	\
+		.mux_val = mval,			\
+		.mux_mask = mask,			\
+		.mux = offset,				\
+		.mux_shift = shift,			\
+	}
+
+static const struct bm1880_pinmux_function bm1880_pmux_functions[] = {
+	BM1880_PINMUX_FUNCTION(nand, 2, 0x03),
+	BM1880_PINMUX_FUNCTION(spi, 0, 0x03),
+	BM1880_PINMUX_FUNCTION(emmc, 1, 0x03),
+	BM1880_PINMUX_FUNCTION(sdio, 0, 0x03),
+	BM1880_PINMUX_FUNCTION(eth0, 0, 0x03),
+	BM1880_PINMUX_FUNCTION_MUX(pwm0, 2, 0x0F, 0x50, 0x00),
+	BM1880_PINMUX_FUNCTION_MUX(pwm1, 2, 0x0F, 0x50, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(pwm2, 2, 0x0F, 0x50, 0x08),
+	BM1880_PINMUX_FUNCTION_MUX(pwm3, 2, 0x0F, 0x50, 0x0C),
+	BM1880_PINMUX_FUNCTION_MUX(pwm4, 2, 0x0F, 0x50, 0x10),
+	BM1880_PINMUX_FUNCTION_MUX(pwm5, 2, 0x0F, 0x50, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(pwm6, 2, 0x0F, 0x50, 0x18),
+	BM1880_PINMUX_FUNCTION_MUX(pwm7, 2, 0x0F, 0x50, 0x1C),
+	BM1880_PINMUX_FUNCTION_MUX(pwm8, 2, 0x0F, 0x54, 0x00),
+	BM1880_PINMUX_FUNCTION_MUX(pwm9, 2, 0x0F, 0x54, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(pwm10, 2, 0x0F, 0x54, 0x08),
+	BM1880_PINMUX_FUNCTION_MUX(pwm11, 2, 0x0F, 0x54, 0x0C),
+	BM1880_PINMUX_FUNCTION_MUX(pwm12, 2, 0x0F, 0x54, 0x10),
+	BM1880_PINMUX_FUNCTION_MUX(pwm13, 2, 0x0F, 0x54, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(pwm14, 2, 0x0F, 0x54, 0x18),
+	BM1880_PINMUX_FUNCTION_MUX(pwm15, 2, 0x0F, 0x54, 0x1C),
+	BM1880_PINMUX_FUNCTION_MUX(pwm16, 2, 0x0F, 0x58, 0x00),
+	BM1880_PINMUX_FUNCTION_MUX(pwm17, 2, 0x0F, 0x58, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(pwm18, 2, 0x0F, 0x58, 0x08),
+	BM1880_PINMUX_FUNCTION_MUX(pwm19, 2, 0x0F, 0x58, 0x0C),
+	BM1880_PINMUX_FUNCTION_MUX(pwm20, 2, 0x0F, 0x58, 0x10),
+	BM1880_PINMUX_FUNCTION_MUX(pwm21, 2, 0x0F, 0x58, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(pwm22, 2, 0x0F, 0x58, 0x18),
+	BM1880_PINMUX_FUNCTION_MUX(pwm23, 2, 0x0F, 0x58, 0x1C),
+	BM1880_PINMUX_FUNCTION_MUX(pwm24, 2, 0x0F, 0x5C, 0x00),
+	BM1880_PINMUX_FUNCTION_MUX(pwm25, 2, 0x0F, 0x5C, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(pwm26, 2, 0x0F, 0x5C, 0x08),
+	BM1880_PINMUX_FUNCTION_MUX(pwm27, 2, 0x0F, 0x5C, 0x0C),
+	BM1880_PINMUX_FUNCTION_MUX(pwm28, 2, 0x0F, 0x5C, 0x10),
+	BM1880_PINMUX_FUNCTION_MUX(pwm29, 2, 0x0F, 0x5C, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(pwm30, 2, 0x0F, 0x5C, 0x18),
+	BM1880_PINMUX_FUNCTION_MUX(pwm31, 2, 0x0F, 0x5C, 0x1C),
+	BM1880_PINMUX_FUNCTION_MUX(pwm32, 2, 0x0F, 0x60, 0x00),
+	BM1880_PINMUX_FUNCTION_MUX(pwm33, 2, 0x0F, 0x60, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(pwm34, 2, 0x0F, 0x60, 0x08),
+	BM1880_PINMUX_FUNCTION_MUX(pwm35, 2, 0x0F, 0x60, 0x0C),
+	BM1880_PINMUX_FUNCTION_MUX(pwm36, 2, 0x0F, 0x60, 0x10),
+	BM1880_PINMUX_FUNCTION_MUX(pwm37, 2, 0x0F, 0x60, 0x1C),
+	BM1880_PINMUX_FUNCTION(i2c0, 1, 0x03),
+	BM1880_PINMUX_FUNCTION(i2c1, 1, 0x03),
+	BM1880_PINMUX_FUNCTION(i2c2, 1, 0x03),
+	BM1880_PINMUX_FUNCTION(i2c3, 1, 0x03),
+	BM1880_PINMUX_FUNCTION(i2c4, 1, 0x03),
+	BM1880_PINMUX_FUNCTION(uart0, 1, 0x03),
+	BM1880_PINMUX_FUNCTION(uart1, 1, 0x03),
+	BM1880_PINMUX_FUNCTION(uart2, 1, 0x03),
+	BM1880_PINMUX_FUNCTION(uart3, 1, 0x03),
+	BM1880_PINMUX_FUNCTION(uart4, 1, 0x03),
+	BM1880_PINMUX_FUNCTION(uart5, 1, 0x03),
+	BM1880_PINMUX_FUNCTION(uart6, 1, 0x03),
+	BM1880_PINMUX_FUNCTION(uart7, 1, 0x03),
+	BM1880_PINMUX_FUNCTION(uart8, 1, 0x03),
+	BM1880_PINMUX_FUNCTION(uart9, 1, 0x03),
+	BM1880_PINMUX_FUNCTION(uart10, 1, 0x03),
+	BM1880_PINMUX_FUNCTION(uart11, 1, 0x03),
+	BM1880_PINMUX_FUNCTION(uart12, 3, 0x03),
+	BM1880_PINMUX_FUNCTION(uart13, 3, 0x03),
+	BM1880_PINMUX_FUNCTION(uart14, 3, 0x03),
+	BM1880_PINMUX_FUNCTION(uart15, 3, 0x03),
+	BM1880_PINMUX_FUNCTION_MUX(gpio0, 0, 0x03, 0x4E0, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(gpio1, 0, 0x03, 0x4E4, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(gpio2, 0, 0x03, 0x4E4, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(gpio3, 0, 0x03, 0x4E8, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(gpio4, 0, 0x03, 0x4E8, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(gpio5, 0, 0x03, 0x4EC, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(gpio6, 0, 0x03, 0x4EC, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(gpio7, 0, 0x03, 0x4F0, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(gpio8, 0, 0x03, 0x4F0, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(gpio9, 0, 0x03, 0x4F4, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(gpio10, 0, 0x03, 0x4F4, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(gpio11, 0, 0x03, 0x4F8, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(gpio12, 1, 0x03, 0x4F8, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(gpio13, 1, 0x03, 0x4FC, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(gpio14, 0, 0x03, 0x474, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(gpio15, 0, 0x03, 0x478, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(gpio16, 0, 0x03, 0x478, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(gpio17, 0, 0x03, 0x47C, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(gpio18, 0, 0x03, 0x47C, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(gpio19, 0, 0x03, 0x480, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(gpio20, 0, 0x03, 0x480, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(gpio21, 0, 0x03, 0x484, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(gpio22, 0, 0x03, 0x484, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(gpio23, 0, 0x03, 0x488, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(gpio24, 0, 0x03, 0x488, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(gpio25, 0, 0x03, 0x48C, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(gpio26, 0, 0x03, 0x48C, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(gpio27, 0, 0x03, 0x490, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(gpio28, 0, 0x03, 0x490, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(gpio29, 0, 0x03, 0x494, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(gpio30, 0, 0x03, 0x494, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(gpio31, 0, 0x03, 0x498, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(gpio32, 0, 0x03, 0x498, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(gpio33, 0, 0x03, 0x49C, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(gpio34, 0, 0x03, 0x49C, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(gpio35, 0, 0x03, 0x4A0, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(gpio36, 0, 0x03, 0x4A0, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(gpio37, 0, 0x03, 0x4A4, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(gpio38, 0, 0x03, 0x4A4, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(gpio39, 0, 0x03, 0x4A8, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(gpio40, 0, 0x03, 0x4A8, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(gpio41, 0, 0x03, 0x4AC, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(gpio42, 0, 0x03, 0x4AC, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(gpio43, 0, 0x03, 0x4B0, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(gpio44, 0, 0x03, 0x4B0, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(gpio45, 0, 0x03, 0x4B4, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(gpio46, 0, 0x03, 0x4B4, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(gpio47, 0, 0x03, 0x4B8, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(gpio48, 0, 0x03, 0x4B8, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(gpio49, 0, 0x03, 0x4BC, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(gpio50, 0, 0x03, 0x4BC, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(gpio51, 0, 0x03, 0x4C0, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(gpio52, 0, 0x03, 0x4C0, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(gpio53, 0, 0x03, 0x4C4, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(gpio54, 0, 0x03, 0x4C4, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(gpio55, 0, 0x03, 0x4C8, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(gpio56, 0, 0x03, 0x4C8, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(gpio57, 0, 0x03, 0x4CC, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(gpio58, 0, 0x03, 0x4CC, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(gpio59, 0, 0x03, 0x4D0, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(gpio60, 0, 0x03, 0x4D0, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(gpio61, 0, 0x03, 0x4D4, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(gpio62, 0, 0x03, 0x4D4, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(gpio63, 0, 0x03, 0x4D8, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(gpio64, 0, 0x03, 0x4D8, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(gpio65, 0, 0x03, 0x4DC, 0x04),
+	BM1880_PINMUX_FUNCTION_MUX(gpio66, 0, 0x03, 0x4DC, 0x14),
+	BM1880_PINMUX_FUNCTION_MUX(gpio67, 0, 0x03, 0x4E0, 0x04),
+	BM1880_PINMUX_FUNCTION(eth1, 1, 0x03),
+	BM1880_PINMUX_FUNCTION(i2s0, 2, 0x03),
+	BM1880_PINMUX_FUNCTION(i2s0_mclkin, 1, 0x03),
+	BM1880_PINMUX_FUNCTION(i2s1, 2, 0x03),
+	BM1880_PINMUX_FUNCTION(i2s1_mclkin, 1, 0x03),
+	BM1880_PINMUX_FUNCTION(spi0, 1, 0x03),
+};
+
+static int bm1880_pctrl_get_groups_count(struct pinctrl_dev *pctldev)
+{
+	struct bm1880_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev);
+
+	return pctrl->ngroups;
+}
+
+static const char *bm1880_pctrl_get_group_name(struct pinctrl_dev *pctldev,
+					       unsigned int selector)
+{
+	struct bm1880_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev);
+
+	return pctrl->groups[selector].name;
+}
+
+static int bm1880_pctrl_get_group_pins(struct pinctrl_dev *pctldev,
+				       unsigned int selector,
+				       const unsigned int **pins,
+				       unsigned int *num_pins)
+{
+	struct bm1880_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev);
+
+	*pins = pctrl->groups[selector].pins;
+	*num_pins = pctrl->groups[selector].npins;
+
+	return 0;
+}
+
+static const struct pinctrl_ops bm1880_pctrl_ops = {
+	.get_groups_count = bm1880_pctrl_get_groups_count,
+	.get_group_name = bm1880_pctrl_get_group_name,
+	.get_group_pins = bm1880_pctrl_get_group_pins,
+	.dt_node_to_map = pinconf_generic_dt_node_to_map_all,
+	.dt_free_map = pinctrl_utils_free_map,
+};
+
+/* pinmux */
+static int bm1880_pmux_get_functions_count(struct pinctrl_dev *pctldev)
+{
+	struct bm1880_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev);
+
+	return pctrl->nfuncs;
+}
+
+static const char *bm1880_pmux_get_function_name(struct pinctrl_dev *pctldev,
+						 unsigned int selector)
+{
+	struct bm1880_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev);
+
+	return pctrl->funcs[selector].name;
+}
+
+static int bm1880_pmux_get_function_groups(struct pinctrl_dev *pctldev,
+					   unsigned int selector,
+					   const char * const **groups,
+					   unsigned * const num_groups)
+{
+	struct bm1880_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev);
+
+	*groups = pctrl->funcs[selector].groups;
+	*num_groups = pctrl->funcs[selector].ngroups;
+	return 0;
+}
+
+static int bm1880_pinmux_set_mux(struct pinctrl_dev *pctldev,
+				 unsigned int function,
+				 unsigned int  group)
+{
+	struct bm1880_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev);
+	const struct bm1880_pctrl_group *pgrp = &pctrl->groups[group];
+	const struct bm1880_pinmux_function *func = &pctrl->funcs[function];
+	int i;
+
+	if (func->mux) {
+		u32 regval = readl_relaxed(pctrl->base + BM1880_REG_MUX +
+					   func->mux);
+
+		regval &= ~(func->mux_mask << func->mux_shift);
+		regval |= func->mux_val << func->mux_shift;
+		writel_relaxed(regval, pctrl->base + BM1880_REG_MUX +
+			       func->mux);
+	} else {
+		for (i = 0; i < pgrp->npins; i++) {
+			unsigned int pin = pgrp->pins[i];
+			u32 offset = (pin >> 1) << 2;
+			u32 mux_offset = ((!((pin + 1) & 1) << 4) + 4);
+			u32 regval = readl_relaxed(pctrl->base +
+						   BM1880_REG_MUX + offset);
+
+			regval &= ~(func->mux_mask << mux_offset);
+			regval |= func->mux_val << mux_offset;
+
+			writel_relaxed(regval, pctrl->base +
+				       BM1880_REG_MUX + offset);
+		}
+	}
+
+	return 0;
+}
+
+static const struct pinmux_ops bm1880_pinmux_ops = {
+	.get_functions_count = bm1880_pmux_get_functions_count,
+	.get_function_name = bm1880_pmux_get_function_name,
+	.get_function_groups = bm1880_pmux_get_function_groups,
+	.set_mux = bm1880_pinmux_set_mux,
+};
+
+static struct pinctrl_desc bm1880_desc = {
+	.name = "bm1880_pinctrl",
+	.pins = bm1880_pins,
+	.npins = ARRAY_SIZE(bm1880_pins),
+	.pctlops = &bm1880_pctrl_ops,
+	.pmxops = &bm1880_pinmux_ops,
+	.owner = THIS_MODULE,
+};
+
+static int bm1880_pinctrl_probe(struct platform_device *pdev)
+
+{
+	struct resource *res;
+	struct bm1880_pinctrl *pctrl;
+
+	pctrl = devm_kzalloc(&pdev->dev, sizeof(*pctrl), GFP_KERNEL);
+	if (!pctrl)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	pctrl->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(pctrl->base))
+		return PTR_ERR(pctrl->base);
+
+	pctrl->groups = bm1880_pctrl_groups;
+	pctrl->ngroups = ARRAY_SIZE(bm1880_pctrl_groups);
+	pctrl->funcs = bm1880_pmux_functions;
+	pctrl->nfuncs = ARRAY_SIZE(bm1880_pmux_functions);
+
+	pctrl->pctrldev = devm_pinctrl_register(&pdev->dev, &bm1880_desc,
+						pctrl);
+	if (IS_ERR(pctrl->pctrldev))
+		return PTR_ERR(pctrl->pctrldev);
+
+	platform_set_drvdata(pdev, pctrl);
+
+	dev_info(&pdev->dev, "BM1880 pinctrl driver initialized\n");
+
+	return 0;
+}
+
+static const struct of_device_id bm1880_pinctrl_of_match[] = {
+	{ .compatible = "bitmain,bm1880-pinctrl" },
+	{ }
+};
+
+static struct platform_driver bm1880_pinctrl_driver = {
+	.driver = {
+		.name = "pinctrl-bm1880",
+		.of_match_table = of_match_ptr(bm1880_pinctrl_of_match),
+	},
+	.probe = bm1880_pinctrl_probe,
+};
+
+static int __init bm1880_pinctrl_init(void)
+{
+	return platform_driver_register(&bm1880_pinctrl_driver);
+}
+arch_initcall(bm1880_pinctrl_init);
diff --git a/drivers/pinctrl/pinctrl-mcp23s08.c b/drivers/pinctrl/pinctrl-mcp23s08.c
index 5d7a8514def9..fd9d6f026d70 100644
--- a/drivers/pinctrl/pinctrl-mcp23s08.c
+++ b/drivers/pinctrl/pinctrl-mcp23s08.c
@@ -266,7 +266,6 @@ static int mcp_pinconf_get(struct pinctrl_dev *pctldev, unsigned int pin,
 		status = (data & BIT(pin)) ? 1 : 0;
 		break;
 	default:
-		dev_err(mcp->dev, "Invalid config param %04x\n", param);
 		return -ENOTSUPP;
 	}
 
@@ -293,7 +292,7 @@ static int mcp_pinconf_set(struct pinctrl_dev *pctldev, unsigned int pin,
 			ret = mcp_set_bit(mcp, MCP_GPPU, pin, arg);
 			break;
 		default:
-			dev_err(mcp->dev, "Invalid config param %04x\n", param);
+			dev_dbg(mcp->dev, "Invalid config param %04x\n", param);
 			return -ENOTSUPP;
 		}
 	}
diff --git a/drivers/pinctrl/pinctrl-pistachio.c b/drivers/pinctrl/pinctrl-pistachio.c
index aa5f949ef219..5b0678f310e5 100644
--- a/drivers/pinctrl/pinctrl-pistachio.c
+++ b/drivers/pinctrl/pinctrl-pistachio.c
@@ -1367,6 +1367,7 @@ static int pistachio_gpio_register(struct pistachio_pinctrl *pctl)
 		if (!of_find_property(child, "gpio-controller", NULL)) {
 			dev_err(pctl->dev,
 				"No gpio-controller property for bank %u\n", i);
+			of_node_put(child);
 			ret = -ENODEV;
 			goto err;
 		}
@@ -1374,6 +1375,7 @@ static int pistachio_gpio_register(struct pistachio_pinctrl *pctl)
 		irq = irq_of_parse_and_map(child, 0);
 		if (irq < 0) {
 			dev_err(pctl->dev, "No IRQ for bank %u: %d\n", i, irq);
+			of_node_put(child);
 			ret = irq;
 			goto err;
 		}
diff --git a/drivers/pinctrl/pinctrl-rza1.c b/drivers/pinctrl/pinctrl-rza1.c
index 9cfe9d0520ac..021e37b7689e 100644
--- a/drivers/pinctrl/pinctrl-rza1.c
+++ b/drivers/pinctrl/pinctrl-rza1.c
@@ -620,14 +620,7 @@ static void rza1_pin_reset(struct rza1_port *port, unsigned int pin)
 static inline int rza1_pin_get_direction(struct rza1_port *port,
 					 unsigned int pin)
 {
-	unsigned long irqflags;
-	int input;
-
-	spin_lock_irqsave(&port->lock, irqflags);
-	input = rza1_get_bit(port, RZA1_PM_REG, pin);
-	spin_unlock_irqrestore(&port->lock, irqflags);
-
-	return !!input;
+	return !!rza1_get_bit(port, RZA1_PM_REG, pin);
 }
 
 /**
@@ -671,14 +664,7 @@ static inline void rza1_pin_set(struct rza1_port *port, unsigned int pin,
 
 static inline int rza1_pin_get(struct rza1_port *port, unsigned int pin)
 {
-	unsigned long irqflags;
-	int val;
-
-	spin_lock_irqsave(&port->lock, irqflags);
-	val = rza1_get_bit(port, RZA1_PPR_REG, pin);
-	spin_unlock_irqrestore(&port->lock, irqflags);
-
-	return val;
+	return rza1_get_bit(port, RZA1_PPR_REG, pin);
 }
 
 /**
diff --git a/drivers/pinctrl/pinctrl-st.c b/drivers/pinctrl/pinctrl-st.c
index e66af93f2cbf..195b442a2343 100644
--- a/drivers/pinctrl/pinctrl-st.c
+++ b/drivers/pinctrl/pinctrl-st.c
@@ -1170,7 +1170,7 @@ static int st_pctl_dt_parse_groups(struct device_node *np,
 	struct property *pp;
 	struct st_pinconf *conf;
 	struct device_node *pins;
-	int i = 0, npins = 0, nr_props;
+	int i = 0, npins = 0, nr_props, ret = 0;
 
 	pins = of_get_child_by_name(np, "st,pins");
 	if (!pins)
@@ -1185,7 +1185,8 @@ static int st_pctl_dt_parse_groups(struct device_node *np,
 			npins++;
 		} else {
 			pr_warn("Invalid st,pins in %pOFn node\n", np);
-			return -EINVAL;
+			ret = -EINVAL;
+			goto out_put_node;
 		}
 	}
 
@@ -1195,8 +1196,10 @@ static int st_pctl_dt_parse_groups(struct device_node *np,
 	grp->pin_conf = devm_kcalloc(info->dev,
 					npins, sizeof(*conf), GFP_KERNEL);
 
-	if (!grp->pins || !grp->pin_conf)
-		return -ENOMEM;
+	if (!grp->pins || !grp->pin_conf) {
+		ret = -ENOMEM;
+		goto out_put_node;
+	}
 
 	/* <bank offset mux direction rt_type rt_delay rt_clk> */
 	for_each_property_of_node(pins, pp) {
@@ -1229,9 +1232,11 @@ static int st_pctl_dt_parse_groups(struct device_node *np,
 		}
 		i++;
 	}
+
+out_put_node:
 	of_node_put(pins);
 
-	return 0;
+	return ret;
 }
 
 static int st_pctl_parse_functions(struct device_node *np,
diff --git a/drivers/pinctrl/samsung/pinctrl-exynos-arm.c b/drivers/pinctrl/samsung/pinctrl-exynos-arm.c
index 44c6b753f692..85ddf49a5188 100644
--- a/drivers/pinctrl/samsung/pinctrl-exynos-arm.c
+++ b/drivers/pinctrl/samsung/pinctrl-exynos-arm.c
@@ -71,6 +71,7 @@ s5pv210_retention_init(struct samsung_pinctrl_drv_data *drvdata,
 	}
 
 	clk_base = of_iomap(np, 0);
+	of_node_put(np);
 	if (!clk_base) {
 		pr_err("%s: failed to map clock registers\n", __func__);
 		return ERR_PTR(-EINVAL);
diff --git a/drivers/pinctrl/sh-pfc/Kconfig b/drivers/pinctrl/sh-pfc/Kconfig
index e941ba60d4b7..2dd716b016a3 100644
--- a/drivers/pinctrl/sh-pfc/Kconfig
+++ b/drivers/pinctrl/sh-pfc/Kconfig
@@ -3,201 +3,183 @@
 # Renesas SH and SH Mobile PINCTRL drivers
 #
 
-if ARCH_RENESAS || SUPERH
-
 config PINCTRL_SH_PFC
+	bool "Renesas SoC pin control support" if COMPILE_TEST && !(ARCH_RENESAS || SUPERH)
+	default y if ARCH_RENESAS || SUPERH
 	select PINMUX
 	select PINCONF
 	select GENERIC_PINCONF
-	def_bool y
+	select PINCTRL_PFC_EMEV2 if ARCH_EMEV2
+	select PINCTRL_PFC_R8A73A4 if ARCH_R8A73A4
+	select PINCTRL_PFC_R8A7740 if ARCH_R8A7740
+	select PINCTRL_PFC_R8A7743 if ARCH_R8A7743
+	select PINCTRL_PFC_R8A7744 if ARCH_R8A7744
+	select PINCTRL_PFC_R8A7745 if ARCH_R8A7745
+	select PINCTRL_PFC_R8A77470 if ARCH_R8A77470
+	select PINCTRL_PFC_R8A774A1 if ARCH_R8A774A1
+	select PINCTRL_PFC_R8A774C0 if ARCH_R8A774C0
+	select PINCTRL_PFC_R8A7778 if ARCH_R8A7778
+	select PINCTRL_PFC_R8A7779 if ARCH_R8A7779
+	select PINCTRL_PFC_R8A7790 if ARCH_R8A7790
+	select PINCTRL_PFC_R8A7791 if ARCH_R8A7791
+	select PINCTRL_PFC_R8A7792 if ARCH_R8A7792
+	select PINCTRL_PFC_R8A7793 if ARCH_R8A7793
+	select PINCTRL_PFC_R8A7794 if ARCH_R8A7794
+	select PINCTRL_PFC_R8A7795 if ARCH_R8A7795
+	select PINCTRL_PFC_R8A7796 if ARCH_R8A7796
+	select PINCTRL_PFC_R8A77965 if ARCH_R8A77965
+	select PINCTRL_PFC_R8A77970 if ARCH_R8A77970
+	select PINCTRL_PFC_R8A77980 if ARCH_R8A77980
+	select PINCTRL_PFC_R8A77990 if ARCH_R8A77990
+	select PINCTRL_PFC_R8A77995 if ARCH_R8A77995
+	select PINCTRL_PFC_SH7203 if CPU_SUBTYPE_SH7203
+	select PINCTRL_PFC_SH7264 if CPU_SUBTYPE_SH7264
+	select PINCTRL_PFC_SH7269 if CPU_SUBTYPE_SH7269
+	select PINCTRL_PFC_SH73A0 if ARCH_SH73A0
+	select PINCTRL_PFC_SH7720 if CPU_SUBTYPE_SH7720
+	select PINCTRL_PFC_SH7722 if CPU_SUBTYPE_SH7722
+	select PINCTRL_PFC_SH7723 if CPU_SUBTYPE_SH7723
+	select PINCTRL_PFC_SH7724 if CPU_SUBTYPE_SH7724
+	select PINCTRL_PFC_SH7734 if CPU_SUBTYPE_SH7734
+	select PINCTRL_PFC_SH7757 if CPU_SUBTYPE_SH7757
+	select PINCTRL_PFC_SH7785 if CPU_SUBTYPE_SH7785
+	select PINCTRL_PFC_SH7786 if CPU_SUBTYPE_SH7786
+	select PINCTRL_PFC_SHX3 if CPU_SUBTYPE_SHX3
 	help
-	  This enables pin control drivers for SH and SH Mobile platforms
+	  This enables pin control drivers for Renesas SuperH and ARM platforms
 
 config PINCTRL_SH_PFC_GPIO
 	select GPIOLIB
-	select PINCTRL_SH_PFC
 	bool
 	help
 	  This enables pin control and GPIO drivers for SH/SH Mobile platforms
 
+config PINCTRL_SH_FUNC_GPIO
+	select PINCTRL_SH_PFC_GPIO
+	bool
+	help
+	  This enables legacy function GPIOs for SH platforms
+
 config PINCTRL_PFC_EMEV2
-	def_bool y
-	depends on ARCH_EMEV2
-	select PINCTRL_SH_PFC
+	bool "Emma Mobile AV2 pin control support" if COMPILE_TEST
 
 config PINCTRL_PFC_R8A73A4
-	def_bool y
-	depends on ARCH_R8A73A4
+	bool "R-Mobile APE6 pin control support" if COMPILE_TEST
 	select PINCTRL_SH_PFC_GPIO
 
 config PINCTRL_PFC_R8A7740
-	def_bool y
-	depends on ARCH_R8A7740
+	bool "R-Mobile A1 pin control support" if COMPILE_TEST
 	select PINCTRL_SH_PFC_GPIO
 
 config PINCTRL_PFC_R8A7743
-	def_bool y
-	depends on ARCH_R8A7743
-	select PINCTRL_SH_PFC
+	bool "RZ/G1M pin control support" if COMPILE_TEST
 
 config PINCTRL_PFC_R8A7744
-	def_bool y
-	depends on ARCH_R8A7744
-	select PINCTRL_SH_PFC
+	bool "RZ/G1N pin control support" if COMPILE_TEST
 
 config PINCTRL_PFC_R8A7745
-        def_bool y
-        depends on ARCH_R8A7745
-        select PINCTRL_SH_PFC
+	bool "RZ/G1E pin control support" if COMPILE_TEST
 
 config PINCTRL_PFC_R8A77470
-        def_bool y
-        depends on ARCH_R8A77470
-        select PINCTRL_SH_PFC
+	bool "RZ/G1C pin control support" if COMPILE_TEST
 
 config PINCTRL_PFC_R8A774A1
-        def_bool y
-        depends on ARCH_R8A774A1
-        select PINCTRL_SH_PFC
+	bool "RZ/G2M pin control support" if COMPILE_TEST
 
 config PINCTRL_PFC_R8A774C0
-        def_bool y
-        depends on ARCH_R8A774C0
-        select PINCTRL_SH_PFC
+	bool "RZ/G2E pin control support" if COMPILE_TEST
 
 config PINCTRL_PFC_R8A7778
-	def_bool y
-	depends on ARCH_R8A7778
-	select PINCTRL_SH_PFC
+	bool "R-Car M1A pin control support" if COMPILE_TEST
 
 config PINCTRL_PFC_R8A7779
-	def_bool y
-	depends on ARCH_R8A7779
-	select PINCTRL_SH_PFC
+	bool "R-Car H1 pin control support" if COMPILE_TEST
 
 config PINCTRL_PFC_R8A7790
-	def_bool y
-	depends on ARCH_R8A7790
-	select PINCTRL_SH_PFC
+	bool "R-Car H2 pin control support" if COMPILE_TEST
 
 config PINCTRL_PFC_R8A7791
-	def_bool y
-	depends on ARCH_R8A7791
-	select PINCTRL_SH_PFC
+	bool "R-Car M2-W pin control support" if COMPILE_TEST
 
 config PINCTRL_PFC_R8A7792
-	def_bool y
-	depends on ARCH_R8A7792
-	select PINCTRL_SH_PFC
+	bool "R-Car V2H pin control support" if COMPILE_TEST
 
 config PINCTRL_PFC_R8A7793
-	def_bool y
-	depends on ARCH_R8A7793
-	select PINCTRL_SH_PFC
+	bool "R-Car M2-N pin control support" if COMPILE_TEST
 
 config PINCTRL_PFC_R8A7794
-	def_bool y
-	depends on ARCH_R8A7794
-	select PINCTRL_SH_PFC
+	bool "R-Car E2 pin control support" if COMPILE_TEST
 
 config PINCTRL_PFC_R8A7795
-	def_bool y
-	depends on ARCH_R8A7795
-	select PINCTRL_SH_PFC
+	bool "R-Car H3 pin control support" if COMPILE_TEST
 
 config PINCTRL_PFC_R8A7796
-        def_bool y
-        depends on ARCH_R8A7796
-        select PINCTRL_SH_PFC
+	bool "R-Car M3-W pin control support" if COMPILE_TEST
 
 config PINCTRL_PFC_R8A77965
-        def_bool y
-        depends on ARCH_R8A77965
-        select PINCTRL_SH_PFC
+	bool "R-Car M3-N pin control support" if COMPILE_TEST
 
 config PINCTRL_PFC_R8A77970
-	def_bool y
-	depends on ARCH_R8A77970
-	select PINCTRL_SH_PFC
+	bool "R-Car V3M pin control support" if COMPILE_TEST
 
 config PINCTRL_PFC_R8A77980
-	def_bool y
-	depends on ARCH_R8A77980
-	select PINCTRL_SH_PFC
+	bool "R-Car V3H pin control support" if COMPILE_TEST
 
 config PINCTRL_PFC_R8A77990
-        def_bool y
-        depends on ARCH_R8A77990
-        select PINCTRL_SH_PFC
+	bool "R-Car E3 pin control support" if COMPILE_TEST
 
 config PINCTRL_PFC_R8A77995
-        def_bool y
-        depends on ARCH_R8A77995
-        select PINCTRL_SH_PFC
+	bool "R-Car D3 pin control support" if COMPILE_TEST
 
 config PINCTRL_PFC_SH7203
-	def_bool y
-	depends on CPU_SUBTYPE_SH7203
-	select PINCTRL_SH_PFC_GPIO
+	bool "SH7203 pin control support" if COMPILE_TEST
+	select PINCTRL_SH_FUNC_GPIO
 
 config PINCTRL_PFC_SH7264
-	def_bool y
-	depends on CPU_SUBTYPE_SH7264
-	select PINCTRL_SH_PFC_GPIO
+	bool "SH7264 pin control support" if COMPILE_TEST
+	select PINCTRL_SH_FUNC_GPIO
 
 config PINCTRL_PFC_SH7269
-	def_bool y
-	depends on CPU_SUBTYPE_SH7269
-	select PINCTRL_SH_PFC_GPIO
+	bool "SH7269 pin control support" if COMPILE_TEST
+	select PINCTRL_SH_FUNC_GPIO
 
 config PINCTRL_PFC_SH73A0
-	def_bool y
-	depends on ARCH_SH73A0
+	bool "SH-Mobile AG5 pin control support" if COMPILE_TEST
 	select PINCTRL_SH_PFC_GPIO
 	select REGULATOR
 
 config PINCTRL_PFC_SH7720
-	def_bool y
-	depends on CPU_SUBTYPE_SH7720
-	select PINCTRL_SH_PFC_GPIO
+	bool "SH7720 pin control support" if COMPILE_TEST
+	select PINCTRL_SH_FUNC_GPIO
 
 config PINCTRL_PFC_SH7722
-	def_bool y
-	depends on CPU_SUBTYPE_SH7722
-	select PINCTRL_SH_PFC_GPIO
+	bool "SH7722 pin control support" if COMPILE_TEST
+	select PINCTRL_SH_FUNC_GPIO
 
 config PINCTRL_PFC_SH7723
-	def_bool y
-	depends on CPU_SUBTYPE_SH7723
-	select PINCTRL_SH_PFC_GPIO
+	bool "SH-Mobile R2 pin control support" if COMPILE_TEST
+	select PINCTRL_SH_FUNC_GPIO
 
 config PINCTRL_PFC_SH7724
-	def_bool y
-	depends on CPU_SUBTYPE_SH7724
-	select PINCTRL_SH_PFC_GPIO
+	bool "SH-Mobile R2R pin control support" if COMPILE_TEST
+	select PINCTRL_SH_FUNC_GPIO
 
 config PINCTRL_PFC_SH7734
-	def_bool y
-	depends on CPU_SUBTYPE_SH7734
-	select PINCTRL_SH_PFC_GPIO
+	bool "SH7734 pin control support" if COMPILE_TEST
+	select PINCTRL_SH_FUNC_GPIO
 
 config PINCTRL_PFC_SH7757
-	def_bool y
-	depends on CPU_SUBTYPE_SH7757
-	select PINCTRL_SH_PFC_GPIO
+	bool "SH7757 pin control support" if COMPILE_TEST
+	select PINCTRL_SH_FUNC_GPIO
 
 config PINCTRL_PFC_SH7785
-	def_bool y
-	depends on CPU_SUBTYPE_SH7785
-	select PINCTRL_SH_PFC_GPIO
+	bool "SH7785 pin control support" if COMPILE_TEST
+	select PINCTRL_SH_FUNC_GPIO
 
 config PINCTRL_PFC_SH7786
-	def_bool y
-	depends on CPU_SUBTYPE_SH7786
-	select PINCTRL_SH_PFC_GPIO
+	bool "SH7786 pin control support" if COMPILE_TEST
+	select PINCTRL_SH_FUNC_GPIO
 
 config PINCTRL_PFC_SHX3
-	def_bool y
-	depends on CPU_SUBTYPE_SHX3
-	select PINCTRL_SH_PFC_GPIO
-endif
+	bool "SH-X3 pin control support" if COMPILE_TEST
+	select PINCTRL_SH_FUNC_GPIO
diff --git a/drivers/pinctrl/sh-pfc/Makefile b/drivers/pinctrl/sh-pfc/Makefile
index 82ebb2a91ee0..8c95abcfcc00 100644
--- a/drivers/pinctrl/sh-pfc/Makefile
+++ b/drivers/pinctrl/sh-pfc/Makefile
@@ -38,3 +38,18 @@ obj-$(CONFIG_PINCTRL_PFC_SH7757)	+= pfc-sh7757.o
 obj-$(CONFIG_PINCTRL_PFC_SH7785)	+= pfc-sh7785.o
 obj-$(CONFIG_PINCTRL_PFC_SH7786)	+= pfc-sh7786.o
 obj-$(CONFIG_PINCTRL_PFC_SHX3)		+= pfc-shx3.o
+
+ifeq ($(CONFIG_COMPILE_TEST),y)
+CFLAGS_pfc-sh7203.o	+= -I$(srctree)/arch/sh/include/cpu-sh2a
+CFLAGS_pfc-sh7264.o	+= -I$(srctree)/arch/sh/include/cpu-sh2a
+CFLAGS_pfc-sh7269.o	+= -I$(srctree)/arch/sh/include/cpu-sh2a
+CFLAGS_pfc-sh7720.o	+= -I$(srctree)/arch/sh/include/cpu-sh3
+CFLAGS_pfc-sh7722.o	+= -I$(srctree)/arch/sh/include/cpu-sh4
+CFLAGS_pfc-sh7723.o	+= -I$(srctree)/arch/sh/include/cpu-sh4
+CFLAGS_pfc-sh7724.o	+= -I$(srctree)/arch/sh/include/cpu-sh4
+CFLAGS_pfc-sh7734.o	+= -I$(srctree)/arch/sh/include/cpu-sh4
+CFLAGS_pfc-sh7757.o	+= -I$(srctree)/arch/sh/include/cpu-sh4
+CFLAGS_pfc-sh7785.o	+= -I$(srctree)/arch/sh/include/cpu-sh4
+CFLAGS_pfc-sh7786.o	+= -I$(srctree)/arch/sh/include/cpu-sh4
+CFLAGS_pfc-shx3.o	+= -I$(srctree)/arch/sh/include/cpu-sh4
+endif
diff --git a/drivers/pinctrl/sh-pfc/core.c b/drivers/pinctrl/sh-pfc/core.c
index f1cfcc8c6544..3f989f5cb021 100644
--- a/drivers/pinctrl/sh-pfc/core.c
+++ b/drivers/pinctrl/sh-pfc/core.c
@@ -571,6 +571,13 @@ static const struct of_device_id sh_pfc_of_table[] = {
 		.compatible = "renesas,pfc-r8a7795",
 		.data = &r8a7795_pinmux_info,
 	},
+#ifdef DEBUG
+	{
+		/* For sanity checks only (nothing matches against this) */
+		.compatible = "renesas,pfc-r8a77950",	/* R-Car H3 ES1.0 */
+		.data = &r8a7795es1_pinmux_info,
+	},
+#endif /* DEBUG */
 #endif
 #ifdef CONFIG_PINCTRL_PFC_R8A7796
 	{
@@ -709,6 +716,128 @@ static int sh_pfc_suspend_init(struct sh_pfc *pfc) { return 0; }
 #define DEV_PM_OPS	NULL
 #endif /* CONFIG_PM_SLEEP && CONFIG_ARM_PSCI_FW */
 
+#ifdef DEBUG
+static bool is0s(const u16 *enum_ids, unsigned int n)
+{
+	unsigned int i;
+
+	for (i = 0; i < n; i++)
+		if (enum_ids[i])
+			return false;
+
+	return true;
+}
+
+static unsigned int sh_pfc_errors;
+static unsigned int sh_pfc_warnings;
+
+static void sh_pfc_check_cfg_reg(const char *drvname,
+				 const struct pinmux_cfg_reg *cfg_reg)
+{
+	unsigned int i, n, rw, fw;
+
+	if (cfg_reg->field_width) {
+		/* Checked at build time */
+		return;
+	}
+
+	for (i = 0, n = 0, rw = 0; (fw = cfg_reg->var_field_width[i]); i++) {
+		if (fw > 3 && is0s(&cfg_reg->enum_ids[n], 1 << fw)) {
+			pr_warn("%s: reg 0x%x: reserved field [%u:%u] can be split to reduce table size\n",
+				drvname, cfg_reg->reg, rw, rw + fw - 1);
+			sh_pfc_warnings++;
+		}
+		n += 1 << fw;
+		rw += fw;
+	}
+
+	if (rw != cfg_reg->reg_width) {
+		pr_err("%s: reg 0x%x: var_field_width declares %u instead of %u bits\n",
+		       drvname, cfg_reg->reg, rw, cfg_reg->reg_width);
+		sh_pfc_errors++;
+	}
+
+	if (n != cfg_reg->nr_enum_ids) {
+		pr_err("%s: reg 0x%x: enum_ids[] has %u instead of %u values\n",
+		       drvname, cfg_reg->reg, cfg_reg->nr_enum_ids, n);
+		sh_pfc_errors++;
+	}
+}
+
+static void sh_pfc_check_info(const struct sh_pfc_soc_info *info)
+{
+	const struct sh_pfc_function *func;
+	const char *drvname = info->name;
+	unsigned int *refcnts;
+	unsigned int i, j, k;
+
+	pr_info("Checking %s\n", drvname);
+
+	/* Check groups and functions */
+	refcnts = kcalloc(info->nr_groups, sizeof(*refcnts), GFP_KERNEL);
+	if (!refcnts)
+		return;
+
+	for (i = 0; i < info->nr_functions; i++) {
+		func = &info->functions[i];
+		for (j = 0; j < func->nr_groups; j++) {
+			for (k = 0; k < info->nr_groups; k++) {
+				if (!strcmp(func->groups[j],
+					    info->groups[k].name)) {
+					refcnts[k]++;
+					break;
+				}
+			}
+
+			if (k == info->nr_groups) {
+				pr_err("%s: function %s: group %s not found\n",
+				       drvname, func->name, func->groups[j]);
+				sh_pfc_errors++;
+			}
+		}
+	}
+
+	for (i = 0; i < info->nr_groups; i++) {
+		if (!refcnts[i]) {
+			pr_err("%s: orphan group %s\n", drvname,
+			       info->groups[i].name);
+			sh_pfc_errors++;
+		} else if (refcnts[i] > 1) {
+			pr_err("%s: group %s referred by %u functions\n",
+			       drvname, info->groups[i].name, refcnts[i]);
+			sh_pfc_warnings++;
+		}
+	}
+
+	kfree(refcnts);
+
+	/* Check config register descriptions */
+	for (i = 0; info->cfg_regs && info->cfg_regs[i].reg; i++)
+		sh_pfc_check_cfg_reg(drvname, &info->cfg_regs[i]);
+}
+
+static void sh_pfc_check_driver(const struct platform_driver *pdrv)
+{
+	unsigned int i;
+
+	pr_warn("Checking builtin pinmux tables\n");
+
+	for (i = 0; pdrv->id_table[i].name[0]; i++)
+		sh_pfc_check_info((void *)pdrv->id_table[i].driver_data);
+
+#ifdef CONFIG_OF
+	for (i = 0; pdrv->driver.of_match_table[i].compatible[0]; i++)
+		sh_pfc_check_info(pdrv->driver.of_match_table[i].data);
+#endif
+
+	pr_warn("Detected %u errors and %u warnings\n", sh_pfc_errors,
+		sh_pfc_warnings);
+}
+
+#else /* !DEBUG */
+static inline void sh_pfc_check_driver(struct platform_driver *pdrv) {}
+#endif /* !DEBUG */
+
 static int sh_pfc_probe(struct platform_device *pdev)
 {
 #ifdef CONFIG_OF
@@ -840,6 +969,7 @@ static struct platform_driver sh_pfc_driver = {
 
 static int __init sh_pfc_init(void)
 {
+	sh_pfc_check_driver(&sh_pfc_driver);
 	return platform_driver_register(&sh_pfc_driver);
 }
 postcore_initcall(sh_pfc_init);
diff --git a/drivers/pinctrl/sh-pfc/gpio.c b/drivers/pinctrl/sh-pfc/gpio.c
index 4f3a34ee1cd4..97c1332c1045 100644
--- a/drivers/pinctrl/sh-pfc/gpio.c
+++ b/drivers/pinctrl/sh-pfc/gpio.c
@@ -252,7 +252,7 @@ static int gpio_pin_setup(struct sh_pfc_chip *chip)
  * Function GPIOs
  */
 
-#ifdef CONFIG_SUPERH
+#ifdef CONFIG_PINCTRL_SH_FUNC_GPIO
 static int gpio_function_request(struct gpio_chip *gc, unsigned offset)
 {
 	static bool __print_once;
@@ -292,7 +292,7 @@ static int gpio_function_setup(struct sh_pfc_chip *chip)
 
 	return 0;
 }
-#endif
+#endif /* CONFIG_PINCTRL_SH_FUNC_GPIO */
 
 /* -----------------------------------------------------------------------------
  * Register/unregister
@@ -369,7 +369,7 @@ int sh_pfc_register_gpiochip(struct sh_pfc *pfc)
 	if (IS_ENABLED(CONFIG_OF) && pfc->dev->of_node)
 		return 0;
 
-#ifdef CONFIG_SUPERH
+#ifdef CONFIG_PINCTRL_SH_FUNC_GPIO
 	/*
 	 * Register the GPIO to pin mappings. As pins with GPIO ports
 	 * must come first in the ranges, skip the pins without GPIO
@@ -397,7 +397,7 @@ int sh_pfc_register_gpiochip(struct sh_pfc *pfc)
 	chip = sh_pfc_add_gpiochip(pfc, gpio_function_setup, NULL);
 	if (IS_ERR(chip))
 		return PTR_ERR(chip);
-#endif /* CONFIG_SUPERH */
+#endif /* CONFIG_PINCTRL_SH_FUNC_GPIO */
 
 	return 0;
 }
diff --git a/drivers/pinctrl/sh-pfc/pfc-emev2.c b/drivers/pinctrl/sh-pfc/pfc-emev2.c
index 310c6f3ee7cc..0af1ef82a1a8 100644
--- a/drivers/pinctrl/sh-pfc/pfc-emev2.c
+++ b/drivers/pinctrl/sh-pfc/pfc-emev2.c
@@ -1433,7 +1433,7 @@ static const struct sh_pfc_function pinmux_functions[] = {
 };
 
 static const struct pinmux_cfg_reg pinmux_config_regs[] = {
-	{ PINMUX_CFG_REG("GPSR0", 0xe0140200, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR0", 0xe0140200, 32, 1, GROUP(
 		0, PORT31_FN,				/* PIN: J18  */
 		0, PORT30_FN,				/* PIN: H18  */
 		0, PORT29_FN,				/* PIN: G18  */
@@ -1466,9 +1466,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_JT_SEL, PORT2_FN,			/* PIN: V9   */
 		0, PORT1_FN,				/* PIN: U10  */
 		0, PORT0_FN,				/* PIN: V10  */
-		}
+		))
 	},
-	{ PINMUX_CFG_REG("GPSR1", 0xe0140204, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR1", 0xe0140204, 32, 1, GROUP(
 		FN_SDI1_CMD, PORT63_FN,			/* PIN: AC21 */
 		FN_SDI1_CKI, PORT62_FN,			/* PIN: AA23 */
 		FN_SDI1_CKO, PORT61_FN,			/* PIN: AB22 */
@@ -1501,9 +1501,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_LCD3_R2, PORT34_FN,			/* PIN: A19  */
 		FN_LCD3_R1, PORT33_FN,			/* PIN: B20  */
 		FN_LCD3_R0, PORT32_FN,			/* PIN: A20  */
-		}
+		))
 	},
-	{ PINMUX_CFG_REG("GPSR2", 0xe0140208, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR2", 0xe0140208, 32, 1, GROUP(
 		FN_AB_1_0_PORT95, PORT95_FN,		/* PIN: L21  */
 		FN_AB_1_0_PORT94, PORT94_FN,		/* PIN: K21  */
 		FN_AB_1_0_PORT93, PORT93_FN,		/* PIN: J21  */
@@ -1536,9 +1536,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_SDI1_DATA2, PORT66_FN,		/* PIN: AB19 */
 		FN_SDI1_DATA1, PORT65_FN,		/* PIN: AB20 */
 		FN_SDI1_DATA0, PORT64_FN,		/* PIN: AB21 */
-		}
+		))
 	},
-	{ PINMUX_CFG_REG("GPSR3", 0xe014020c, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR3", 0xe014020c, 32, 1, GROUP(
 		FN_NTSC_DATA4, PORT127_FN,		/* PIN: T20  */
 		FN_NTSC_DATA3, PORT126_FN,		/* PIN: R18  */
 		FN_NTSC_DATA2, PORT125_FN,		/* PIN: R20  */
@@ -1571,9 +1571,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_AB_9_8_PORT98, PORT98_FN,		/* PIN: M20  */
 		FN_AB_9_8_PORT97, PORT97_FN,		/* PIN: N21  */
 		FN_AB_A20, PORT96_FN,			/* PIN: M21  */
-		}
+		))
 	},
-	{ PINMUX_CFG_REG("GPSR4", 0xe0140210, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR4", 0xe0140210, 32, 1, GROUP(
 		0, 0,
 		FN_UART_1_0_PORT158, PORT158_FN,	/* PIN: AB10 */
 		FN_UART_1_0_PORT157, PORT157_FN,	/* PIN: AA10 */
@@ -1606,11 +1606,13 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_NTSC_DATA7, PORT130_FN,		/* PIN: U18  */
 		FN_NTSC_DATA6, PORT129_FN,		/* PIN: U20  */
 		FN_NTSC_DATA5, PORT128_FN,		/* PIN: T18  */
-		}
+		))
 	},
 	{ PINMUX_CFG_REG_VAR("CHG_PINSEL_LCD3", 0xe0140284, 32,
-			1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-			1, 1, 1, 1, 2, 2, 2, 2, 2, 2) {
+			     GROUP(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+				   1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2,
+				   2, 2),
+			     GROUP(
 		/* 31 - 12 */
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -1624,11 +1626,13 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* 1 - 0 */
 		FN_SEL_LCD3_1_0_00, FN_SEL_LCD3_1_0_01, 0, 0,
-		}
+		))
 	},
 	{ PINMUX_CFG_REG_VAR("CHG_PINSEL_UART", 0xe0140288, 32,
-			1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-			1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2) {
+			     GROUP(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+				   1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+				   1, 1, 1, 1, 1, 1, 2),
+			     GROUP(
 		/* 31 - 2 */
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -1636,11 +1640,13 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* 1 - 0 */
 		FN_SEL_UART_1_0_00, FN_SEL_UART_1_0_01, 0, 0,
-		}
+		))
 	},
 	{ PINMUX_CFG_REG_VAR("CHG_PINSEL_IIC", 0xe014028c, 32,
-			1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-			1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2) {
+			     GROUP(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+				   1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+				   1, 1, 1, 1, 1, 1, 2),
+			     GROUP(
 		/* 31 - 2 */
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -1648,11 +1654,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* 1 - 0 */
 		FN_SEL_IIC_1_0_00, FN_SEL_IIC_1_0_01, 0, 0,
-		}
+		))
 	},
 	{ PINMUX_CFG_REG_VAR("CHG_PINSEL_AB", 0xe0140294, 32,
-			1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-			2, 2, 2, 2, 2, 2, 2, 2) {
+			     GROUP(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+				   1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2),
+			     GROUP(
 		/* 31 - 14 */
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -1673,11 +1680,13 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_SEL_AB_3_2_10, FN_SEL_AB_3_2_11,
 		/* 1 - 0 */
 		FN_SEL_AB_1_0_00, 0, FN_SEL_AB_1_0_10, 0,
-		}
+		))
 	},
 	{ PINMUX_CFG_REG_VAR("CHG_PINSEL_USI", 0xe0140298, 32,
-			1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-			1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2) {
+			     GROUP(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+				   1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2,
+				   2, 2, 2),
+			     GROUP(
 		/* 31 - 10 */
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -1692,11 +1701,13 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_SEL_USI_3_2_00, FN_SEL_USI_3_2_01, 0, 0,
 		/* 1 - 0 */
 		FN_SEL_USI_1_0_00, FN_SEL_USI_1_0_01, 0, 0,
-		}
+		))
 	},
 	{ PINMUX_CFG_REG_VAR("CHG_PINSEL_HSI", 0xe01402a8, 32,
-			1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-			1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2) {
+			     GROUP(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+				   1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+				   1, 1, 1, 1, 1, 1, 2),
+			     GROUP(
 		/* 31 - 2 */
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -1704,7 +1715,7 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* 1 - 0 */
 		FN_SEL_HSI_1_0_00, FN_SEL_HSI_1_0_01, 0, 0,
-		}
+		))
 	},
 	{ },
 };
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a73a4.c b/drivers/pinctrl/sh-pfc/pfc-r8a73a4.c
index 5acbacb3727f..bf12849defdb 100644
--- a/drivers/pinctrl/sh-pfc/pfc-r8a73a4.c
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a73a4.c
@@ -2284,7 +2284,7 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 	PORTCR(328, 0xe6053148),
 	PORTCR(329, 0xe6053149),
 
-	{ PINMUX_CFG_REG("MSEL1CR", 0xe605800c, 32, 1) {
+	{ PINMUX_CFG_REG("MSEL1CR", 0xe605800c, 32, 1, GROUP(
 			MSEL1CR_31_0, MSEL1CR_31_1,
 			0, 0,
 			0, 0,
@@ -2317,9 +2317,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 			MSEL1CR_02_0, MSEL1CR_02_1,
 			MSEL1CR_01_0, MSEL1CR_01_1,
 			MSEL1CR_00_0, MSEL1CR_00_1,
-		}
+		))
 	},
-	{ PINMUX_CFG_REG("MSEL3CR", 0xe6058020, 32, 1) {
+	{ PINMUX_CFG_REG("MSEL3CR", 0xe6058020, 32, 1, GROUP(
 			MSEL3CR_31_0, MSEL3CR_31_1,
 			0, 0,
 			0, 0,
@@ -2352,9 +2352,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 			0, 0,
 			MSEL3CR_01_0, MSEL3CR_01_1,
 			MSEL3CR_00_0, MSEL3CR_00_1,
-			}
+			))
 	},
-	{ PINMUX_CFG_REG("MSEL4CR", 0xe6058024, 32, 1) {
+	{ PINMUX_CFG_REG("MSEL4CR", 0xe6058024, 32, 1, GROUP(
 			0, 0,
 			MSEL4CR_30_0, MSEL4CR_30_1,
 			MSEL4CR_29_0, MSEL4CR_29_1,
@@ -2387,9 +2387,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 			0, 0,
 			MSEL4CR_01_0, MSEL4CR_01_1,
 			0, 0,
-		}
+		))
 	},
-	{ PINMUX_CFG_REG("MSEL5CR", 0xe6058028, 32, 1) {
+	{ PINMUX_CFG_REG("MSEL5CR", 0xe6058028, 32, 1, GROUP(
 			MSEL5CR_31_0, MSEL5CR_31_1,
 			MSEL5CR_30_0, MSEL5CR_30_1,
 			MSEL5CR_29_0, MSEL5CR_29_1,
@@ -2422,9 +2422,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 			0, 0,
 			0, 0,
 			0, 0,
-		}
+		))
 	},
-	{ PINMUX_CFG_REG("MSEL8CR", 0xe6058034, 32, 1) {
+	{ PINMUX_CFG_REG("MSEL8CR", 0xe6058034, 32, 1, GROUP(
 			0, 0,
 			0, 0,
 			0, 0,
@@ -2457,14 +2457,14 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 			0, 0,
 			MSEL8CR_01_0, MSEL8CR_01_1,
 			MSEL8CR_00_0, MSEL8CR_00_1,
-		}
+		))
 	},
 	{ },
 };
 
 static const struct pinmux_data_reg pinmux_data_regs[] = {
 
-	{ PINMUX_DATA_REG("PORTL031_000DR", 0xe6054000, 32) {
+	{ PINMUX_DATA_REG("PORTL031_000DR", 0xe6054000, 32, GROUP(
 			0, PORT30_DATA, PORT29_DATA, PORT28_DATA,
 			PORT27_DATA, PORT26_DATA, PORT25_DATA, PORT24_DATA,
 			PORT23_DATA, PORT22_DATA, PORT21_DATA, PORT20_DATA,
@@ -2473,9 +2473,9 @@ static const struct pinmux_data_reg pinmux_data_regs[] = {
 			PORT11_DATA, PORT10_DATA, PORT9_DATA, PORT8_DATA,
 			PORT7_DATA, PORT6_DATA, PORT5_DATA, PORT4_DATA,
 			PORT3_DATA, PORT2_DATA, PORT1_DATA, PORT0_DATA,
-		}
+		))
 	},
-	{ PINMUX_DATA_REG("PORTD063_032DR", 0xe6055000, 32) {
+	{ PINMUX_DATA_REG("PORTD063_032DR", 0xe6055000, 32, GROUP(
 			0, 0, 0, 0,
 			0, 0, 0, 0,
 			0, 0, 0, 0,
@@ -2484,9 +2484,9 @@ static const struct pinmux_data_reg pinmux_data_regs[] = {
 			0, 0, 0, PORT40_DATA,
 			PORT39_DATA, PORT38_DATA, PORT37_DATA, PORT36_DATA,
 			PORT35_DATA, PORT34_DATA, PORT33_DATA, PORT32_DATA,
-		}
+		))
 	},
-	{ PINMUX_DATA_REG("PORTL095_064DR", 0xe6054004, 32) {
+	{ PINMUX_DATA_REG("PORTL095_064DR", 0xe6054004, 32, GROUP(
 			0, 0, 0, 0,
 			0, 0, 0, 0,
 			0, 0, PORT85_DATA, PORT84_DATA,
@@ -2495,9 +2495,9 @@ static const struct pinmux_data_reg pinmux_data_regs[] = {
 			PORT75_DATA, PORT74_DATA, PORT73_DATA, PORT72_DATA,
 			PORT71_DATA, PORT70_DATA, PORT69_DATA, PORT68_DATA,
 			PORT67_DATA, PORT66_DATA, PORT65_DATA, PORT64_DATA,
-		}
+		))
 	},
-	{ PINMUX_DATA_REG("PORTD127_096DR", 0xe6055004, 32) {
+	{ PINMUX_DATA_REG("PORTD127_096DR", 0xe6055004, 32, GROUP(
 			0, PORT126_DATA, PORT125_DATA, PORT124_DATA,
 			PORT123_DATA, PORT122_DATA, PORT121_DATA, PORT120_DATA,
 			PORT119_DATA, PORT118_DATA, PORT117_DATA, PORT116_DATA,
@@ -2506,9 +2506,9 @@ static const struct pinmux_data_reg pinmux_data_regs[] = {
 			PORT107_DATA, PORT106_DATA, PORT105_DATA, PORT104_DATA,
 			PORT103_DATA, PORT102_DATA, PORT101_DATA, PORT100_DATA,
 			PORT99_DATA, PORT98_DATA, PORT97_DATA, PORT96_DATA,
-		}
+		))
 	},
-	{ PINMUX_DATA_REG("PORTD159_128DR", 0xe6055008, 32) {
+	{ PINMUX_DATA_REG("PORTD159_128DR", 0xe6055008, 32, GROUP(
 			0, 0, 0, 0,
 			0, 0, 0, 0,
 			0, 0, 0, 0,
@@ -2517,9 +2517,9 @@ static const struct pinmux_data_reg pinmux_data_regs[] = {
 			0, 0, 0, 0,
 			0, PORT134_DATA, PORT133_DATA, PORT132_DATA,
 			PORT131_DATA, PORT130_DATA, PORT129_DATA, PORT128_DATA,
-		}
+		))
 	},
-	{ PINMUX_DATA_REG("PORTR191_160DR", 0xe6056000, 32) {
+	{ PINMUX_DATA_REG("PORTR191_160DR", 0xe6056000, 32, GROUP(
 			0, 0, 0, 0,
 			0, 0, 0, 0,
 			0, 0, 0, 0,
@@ -2528,9 +2528,9 @@ static const struct pinmux_data_reg pinmux_data_regs[] = {
 			PORT171_DATA, PORT170_DATA, PORT169_DATA, PORT168_DATA,
 			PORT167_DATA, PORT166_DATA, PORT165_DATA, PORT164_DATA,
 			PORT163_DATA, PORT162_DATA, PORT161_DATA, PORT160_DATA,
-		}
+		))
 	},
-	{ PINMUX_DATA_REG("PORTR223_192DR", 0xe6056004, 32) {
+	{ PINMUX_DATA_REG("PORTR223_192DR", 0xe6056004, 32, GROUP(
 			0, PORT222_DATA, PORT221_DATA, PORT220_DATA,
 			PORT219_DATA, PORT218_DATA, PORT217_DATA, PORT216_DATA,
 			PORT215_DATA, PORT214_DATA, PORT213_DATA, PORT212_DATA,
@@ -2539,9 +2539,9 @@ static const struct pinmux_data_reg pinmux_data_regs[] = {
 			PORT203_DATA, PORT202_DATA, PORT201_DATA, PORT200_DATA,
 			PORT199_DATA, PORT198_DATA, PORT197_DATA, PORT196_DATA,
 			PORT195_DATA, PORT194_DATA, PORT193_DATA, PORT192_DATA,
-		}
+		))
 	},
-	{ PINMUX_DATA_REG("PORTR255_224DR", 0xe6056008, 32) {
+	{ PINMUX_DATA_REG("PORTR255_224DR", 0xe6056008, 32, GROUP(
 			0, 0, 0, 0,
 			0, PORT250_DATA, PORT249_DATA, PORT248_DATA,
 			PORT247_DATA, PORT246_DATA, PORT245_DATA, PORT244_DATA,
@@ -2550,9 +2550,9 @@ static const struct pinmux_data_reg pinmux_data_regs[] = {
 			PORT235_DATA, PORT234_DATA, PORT233_DATA, PORT232_DATA,
 			PORT231_DATA, PORT230_DATA, PORT229_DATA, PORT228_DATA,
 			PORT227_DATA, PORT226_DATA, PORT225_DATA, PORT224_DATA,
-		}
+		))
 	},
-	{ PINMUX_DATA_REG("PORTR287_256DR", 0xe605600C, 32) {
+	{ PINMUX_DATA_REG("PORTR287_256DR", 0xe605600C, 32, GROUP(
 			0, 0, 0, 0,
 			PORT283_DATA, PORT282_DATA, PORT281_DATA, PORT280_DATA,
 			PORT279_DATA, PORT278_DATA, PORT277_DATA, PORT276_DATA,
@@ -2561,9 +2561,9 @@ static const struct pinmux_data_reg pinmux_data_regs[] = {
 			PORT267_DATA, PORT266_DATA, PORT265_DATA, PORT264_DATA,
 			PORT263_DATA, PORT262_DATA, PORT261_DATA, PORT260_DATA,
 			PORT259_DATA, PORT258_DATA, PORT257_DATA, PORT256_DATA,
-		}
+		))
 	},
-	{ PINMUX_DATA_REG("PORTU319_288DR", 0xe6057000, 32) {
+	{ PINMUX_DATA_REG("PORTU319_288DR", 0xe6057000, 32, GROUP(
 			0, 0, 0, 0,
 			0, 0, 0, 0,
 			0, 0, 0, PORT308_DATA,
@@ -2572,9 +2572,9 @@ static const struct pinmux_data_reg pinmux_data_regs[] = {
 			PORT299_DATA, PORT298_DATA, PORT297_DATA, PORT296_DATA,
 			PORT295_DATA, PORT294_DATA, PORT293_DATA, PORT292_DATA,
 			PORT291_DATA, PORT290_DATA, PORT289_DATA, PORT288_DATA,
-		}
+		))
 	},
-	{ PINMUX_DATA_REG("PORTU351_320DR", 0xe6057004, 32) {
+	{ PINMUX_DATA_REG("PORTU351_320DR", 0xe6057004, 32, GROUP(
 			0, 0, 0, 0,
 			0, 0, 0, 0,
 			0, 0, 0, 0,
@@ -2583,7 +2583,7 @@ static const struct pinmux_data_reg pinmux_data_regs[] = {
 			0, 0, PORT329_DATA, PORT328_DATA,
 			PORT327_DATA, PORT326_DATA, PORT325_DATA, PORT324_DATA,
 			PORT323_DATA, PORT322_DATA, PORT321_DATA, PORT320_DATA,
-		}
+		))
 	},
 	{ },
 };
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a7740.c b/drivers/pinctrl/sh-pfc/pfc-r8a7740.c
index 6d761e62c6c8..696a0f6fc1da 100644
--- a/drivers/pinctrl/sh-pfc/pfc-r8a7740.c
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a7740.c
@@ -3436,7 +3436,7 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 	PORTCR(210,	0xe60530d2), /* PORT210CR */
 	PORTCR(211,	0xe60530d3), /* PORT211CR */
 
-	{ PINMUX_CFG_REG("MSEL1CR", 0xe605800c, 32, 1) {
+	{ PINMUX_CFG_REG("MSEL1CR", 0xe605800c, 32, 1, GROUP(
 			MSEL1CR_31_0,	MSEL1CR_31_1,
 			MSEL1CR_30_0,	MSEL1CR_30_1,
 			MSEL1CR_29_0,	MSEL1CR_29_1,
@@ -3461,9 +3461,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 			MSEL1CR_2_0,	MSEL1CR_2_1,
 			0, 0,
 			MSEL1CR_0_0,	MSEL1CR_0_1,
-		}
+		))
 	},
-	{ PINMUX_CFG_REG("MSEL3CR", 0xE6058020, 32, 1) {
+	{ PINMUX_CFG_REG("MSEL3CR", 0xE6058020, 32, 1, GROUP(
 			0, 0, 0, 0, 0, 0, 0, 0,
 			0, 0, 0, 0, 0, 0, 0, 0,
 			0, 0, 0, 0, 0, 0, 0, 0,
@@ -3474,9 +3474,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 			MSEL3CR_6_0,	MSEL3CR_6_1,
 			0, 0, 0, 0, 0, 0, 0, 0,
 			0, 0, 0, 0,
-			}
+			))
 	},
-	{ PINMUX_CFG_REG("MSEL4CR", 0xE6058024, 32, 1) {
+	{ PINMUX_CFG_REG("MSEL4CR", 0xE6058024, 32, 1, GROUP(
 			0, 0, 0, 0, 0, 0, 0, 0,
 			0, 0, 0, 0, 0, 0, 0, 0,
 			0, 0, 0, 0, 0, 0, 0, 0,
@@ -3493,9 +3493,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 			0, 0, 0, 0,
 			MSEL4CR_1_0,	MSEL4CR_1_1,
 			0, 0,
-		}
+		))
 	},
-	{ PINMUX_CFG_REG("MSEL5CR", 0xE6058028, 32, 1) {
+	{ PINMUX_CFG_REG("MSEL5CR", 0xE6058028, 32, 1, GROUP(
 			MSEL5CR_31_0,	MSEL5CR_31_1,
 			MSEL5CR_30_0,	MSEL5CR_30_1,
 			MSEL5CR_29_0,	MSEL5CR_29_1,
@@ -3528,13 +3528,13 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 			MSEL5CR_2_0,	MSEL5CR_2_1,
 			0, 0,
 			MSEL5CR_0_0,	MSEL5CR_0_1,
-		}
+		))
 	},
 	{ },
 };
 
 static const struct pinmux_data_reg pinmux_data_regs[] = {
-	{ PINMUX_DATA_REG("PORTL031_000DR", 0xe6054800, 32) {
+	{ PINMUX_DATA_REG("PORTL031_000DR", 0xe6054800, 32, GROUP(
 		PORT31_DATA,	PORT30_DATA,	PORT29_DATA,	PORT28_DATA,
 		PORT27_DATA,	PORT26_DATA,	PORT25_DATA,	PORT24_DATA,
 		PORT23_DATA,	PORT22_DATA,	PORT21_DATA,	PORT20_DATA,
@@ -3542,9 +3542,9 @@ static const struct pinmux_data_reg pinmux_data_regs[] = {
 		PORT15_DATA,	PORT14_DATA,	PORT13_DATA,	PORT12_DATA,
 		PORT11_DATA,	PORT10_DATA,	PORT9_DATA,	PORT8_DATA,
 		PORT7_DATA,	PORT6_DATA,	PORT5_DATA,	PORT4_DATA,
-		PORT3_DATA,	PORT2_DATA,	PORT1_DATA,	PORT0_DATA }
+		PORT3_DATA,	PORT2_DATA,	PORT1_DATA,	PORT0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PORTL063_032DR", 0xe6054804, 32) {
+	{ PINMUX_DATA_REG("PORTL063_032DR", 0xe6054804, 32, GROUP(
 		PORT63_DATA,	PORT62_DATA,	PORT61_DATA,	PORT60_DATA,
 		PORT59_DATA,	PORT58_DATA,	PORT57_DATA,	PORT56_DATA,
 		PORT55_DATA,	PORT54_DATA,	PORT53_DATA,	PORT52_DATA,
@@ -3552,9 +3552,9 @@ static const struct pinmux_data_reg pinmux_data_regs[] = {
 		PORT47_DATA,	PORT46_DATA,	PORT45_DATA,	PORT44_DATA,
 		PORT43_DATA,	PORT42_DATA,	PORT41_DATA,	PORT40_DATA,
 		PORT39_DATA,	PORT38_DATA,	PORT37_DATA,	PORT36_DATA,
-		PORT35_DATA,	PORT34_DATA,	PORT33_DATA,	PORT32_DATA }
+		PORT35_DATA,	PORT34_DATA,	PORT33_DATA,	PORT32_DATA ))
 	},
-	{ PINMUX_DATA_REG("PORTL095_064DR", 0xe6054808, 32) {
+	{ PINMUX_DATA_REG("PORTL095_064DR", 0xe6054808, 32, GROUP(
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		0, 0, 0, 0,
@@ -3562,9 +3562,9 @@ static const struct pinmux_data_reg pinmux_data_regs[] = {
 		PORT79_DATA,	PORT78_DATA,	PORT77_DATA,	PORT76_DATA,
 		PORT75_DATA,	PORT74_DATA,	PORT73_DATA,	PORT72_DATA,
 		PORT71_DATA,	PORT70_DATA,	PORT69_DATA,	PORT68_DATA,
-		PORT67_DATA,	PORT66_DATA,	PORT65_DATA,	PORT64_DATA }
+		PORT67_DATA,	PORT66_DATA,	PORT65_DATA,	PORT64_DATA ))
 	},
-	{ PINMUX_DATA_REG("PORTD095_064DR", 0xe6055808, 32) {
+	{ PINMUX_DATA_REG("PORTD095_064DR", 0xe6055808, 32, GROUP(
 		PORT95_DATA,	PORT94_DATA,	PORT93_DATA,	PORT92_DATA,
 		PORT91_DATA,	PORT90_DATA,	PORT89_DATA,	PORT88_DATA,
 		PORT87_DATA,	PORT86_DATA,	PORT85_DATA,	PORT84_DATA,
@@ -3572,9 +3572,9 @@ static const struct pinmux_data_reg pinmux_data_regs[] = {
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		0, 0, 0, 0,
-		0, 0, 0, 0 }
+		0, 0, 0, 0 ))
 	},
-	{ PINMUX_DATA_REG("PORTD127_096DR", 0xe605580c, 32) {
+	{ PINMUX_DATA_REG("PORTD127_096DR", 0xe605580c, 32, GROUP(
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		0, 0, 0, 0,
@@ -3582,9 +3582,9 @@ static const struct pinmux_data_reg pinmux_data_regs[] = {
 		PORT111_DATA,	PORT110_DATA,	PORT109_DATA,	PORT108_DATA,
 		PORT107_DATA,	PORT106_DATA,	PORT105_DATA,	PORT104_DATA,
 		PORT103_DATA,	PORT102_DATA,	PORT101_DATA,	PORT100_DATA,
-		PORT99_DATA,	PORT98_DATA,	PORT97_DATA,	PORT96_DATA }
+		PORT99_DATA,	PORT98_DATA,	PORT97_DATA,	PORT96_DATA ))
 	},
-	{ PINMUX_DATA_REG("PORTR127_096DR", 0xe605680C, 32) {
+	{ PINMUX_DATA_REG("PORTR127_096DR", 0xe605680C, 32, GROUP(
 		PORT127_DATA,	PORT126_DATA,	PORT125_DATA,	PORT124_DATA,
 		PORT123_DATA,	PORT122_DATA,	PORT121_DATA,	PORT120_DATA,
 		PORT119_DATA,	PORT118_DATA,	PORT117_DATA,	PORT116_DATA,
@@ -3592,9 +3592,9 @@ static const struct pinmux_data_reg pinmux_data_regs[] = {
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		0, 0, 0, 0,
-		0, 0, 0, 0 }
+		0, 0, 0, 0 ))
 	},
-	{ PINMUX_DATA_REG("PORTR159_128DR", 0xe6056810, 32) {
+	{ PINMUX_DATA_REG("PORTR159_128DR", 0xe6056810, 32, GROUP(
 		PORT159_DATA,	PORT158_DATA,	PORT157_DATA,	PORT156_DATA,
 		PORT155_DATA,	PORT154_DATA,	PORT153_DATA,	PORT152_DATA,
 		PORT151_DATA,	PORT150_DATA,	PORT149_DATA,	PORT148_DATA,
@@ -3602,9 +3602,9 @@ static const struct pinmux_data_reg pinmux_data_regs[] = {
 		PORT143_DATA,	PORT142_DATA,	PORT141_DATA,	PORT140_DATA,
 		PORT139_DATA,	PORT138_DATA,	PORT137_DATA,	PORT136_DATA,
 		PORT135_DATA,	PORT134_DATA,	PORT133_DATA,	PORT132_DATA,
-		PORT131_DATA,	PORT130_DATA,	PORT129_DATA,	PORT128_DATA }
+		PORT131_DATA,	PORT130_DATA,	PORT129_DATA,	PORT128_DATA ))
 	},
-	{ PINMUX_DATA_REG("PORTR191_160DR", 0xe6056814, 32) {
+	{ PINMUX_DATA_REG("PORTR191_160DR", 0xe6056814, 32, GROUP(
 		PORT191_DATA,	PORT190_DATA,	PORT189_DATA,	PORT188_DATA,
 		PORT187_DATA,	PORT186_DATA,	PORT185_DATA,	PORT184_DATA,
 		PORT183_DATA,	PORT182_DATA,	PORT181_DATA,	PORT180_DATA,
@@ -3612,9 +3612,9 @@ static const struct pinmux_data_reg pinmux_data_regs[] = {
 		PORT175_DATA,	PORT174_DATA,	PORT173_DATA,	PORT172_DATA,
 		PORT171_DATA,	PORT170_DATA,	PORT169_DATA,	PORT168_DATA,
 		PORT167_DATA,	PORT166_DATA,	PORT165_DATA,	PORT164_DATA,
-		PORT163_DATA,	PORT162_DATA,	PORT161_DATA,	PORT160_DATA }
+		PORT163_DATA,	PORT162_DATA,	PORT161_DATA,	PORT160_DATA ))
 	},
-	{ PINMUX_DATA_REG("PORTR223_192DR", 0xe6056818, 32) {
+	{ PINMUX_DATA_REG("PORTR223_192DR", 0xe6056818, 32, GROUP(
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		0, 0, 0, 0,
@@ -3622,9 +3622,9 @@ static const struct pinmux_data_reg pinmux_data_regs[] = {
 		PORT207_DATA,	PORT206_DATA,	PORT205_DATA,	PORT204_DATA,
 		PORT203_DATA,	PORT202_DATA,	PORT201_DATA,	PORT200_DATA,
 		PORT199_DATA,	PORT198_DATA,	PORT197_DATA,	PORT196_DATA,
-		PORT195_DATA,	PORT194_DATA,	PORT193_DATA,	PORT192_DATA }
+		PORT195_DATA,	PORT194_DATA,	PORT193_DATA,	PORT192_DATA ))
 	},
-	{ PINMUX_DATA_REG("PORTU223_192DR", 0xe6057818, 32) {
+	{ PINMUX_DATA_REG("PORTU223_192DR", 0xe6057818, 32, GROUP(
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		0, 0, 0, 0,
@@ -3632,7 +3632,7 @@ static const struct pinmux_data_reg pinmux_data_regs[] = {
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		0, 0, 0, 0,
-		0, 0, 0, 0 }
+		0, 0, 0, 0 ))
 	},
 	{ },
 };
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a77470.c b/drivers/pinctrl/sh-pfc/pfc-r8a77470.c
index 4359aeb35dbd..c05dc1490486 100644
--- a/drivers/pinctrl/sh-pfc/pfc-r8a77470.c
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a77470.c
@@ -5,6 +5,7 @@
  * Copyright (C) 2018 Renesas Electronics Corp.
  */
 
+#include <linux/errno.h>
 #include <linux/kernel.h>
 
 #include "sh_pfc.h"
@@ -2540,7 +2541,7 @@ static const struct sh_pfc_function pinmux_functions[] = {
 };
 
 static const struct pinmux_cfg_reg pinmux_config_regs[] = {
-	{ PINMUX_CFG_REG("GPSR0", 0xE6060004, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR0", 0xE6060004, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -2572,9 +2573,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_0_3_FN, FN_USB1_OVC,
 		GP_0_2_FN, FN_USB1_PWEN,
 		GP_0_1_FN, FN_USB0_OVC,
-		GP_0_0_FN, FN_USB0_PWEN, }
+		GP_0_0_FN, FN_USB0_PWEN, ))
 	},
-	{ PINMUX_CFG_REG("GPSR1", 0xE6060008, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR1", 0xE6060008, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -2606,9 +2607,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_1_3_FN, FN_IP1_23_20,
 		GP_1_2_FN, FN_IP1_19_16,
 		GP_1_1_FN, FN_IP1_15_12,
-		GP_1_0_FN, FN_IP1_11_8, }
+		GP_1_0_FN, FN_IP1_11_8, ))
 	},
-	{ PINMUX_CFG_REG("GPSR2", 0xE606000C, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR2", 0xE606000C, 32, 1, GROUP(
 		GP_2_31_FN, FN_IP8_3_0,
 		GP_2_30_FN, FN_IP7_31_28,
 		GP_2_29_FN, FN_IP7_27_24,
@@ -2640,9 +2641,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_2_3_FN, FN_IP4_19_16,
 		GP_2_2_FN, FN_IP4_15_12,
 		GP_2_1_FN, FN_IP4_11_8,
-		GP_2_0_FN, FN_IP4_7_4, }
+		GP_2_0_FN, FN_IP4_7_4, ))
 	},
-	{ PINMUX_CFG_REG("GPSR3", 0xE6060010, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR3", 0xE6060010, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		GP_3_29_FN, FN_IP10_19_16,
@@ -2674,9 +2675,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_3_3_FN, FN_IP8_19_16,
 		GP_3_2_FN, FN_IP8_15_12,
 		GP_3_1_FN, FN_IP8_11_8,
-		GP_3_0_FN, FN_IP8_7_4, }
+		GP_3_0_FN, FN_IP8_7_4, ))
 	},
-	{ PINMUX_CFG_REG("GPSR4", 0xE6060014, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR4", 0xE6060014, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -2708,9 +2709,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_4_3_FN, FN_IP11_3_0,
 		GP_4_2_FN, FN_IP10_31_28,
 		GP_4_1_FN, FN_IP10_27_24,
-		GP_4_0_FN, FN_IP10_23_20, }
+		GP_4_0_FN, FN_IP10_23_20, ))
 	},
-	{ PINMUX_CFG_REG("GPSR5", 0xE6060018, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR5", 0xE6060018, 32, 1, GROUP(
 		GP_5_31_FN, FN_IP17_27_24,
 		GP_5_30_FN, FN_IP17_23_20,
 		GP_5_29_FN, FN_IP17_19_16,
@@ -2742,10 +2743,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_5_3_FN, FN_IP14_11_8,
 		GP_5_2_FN, FN_IP14_7_4,
 		GP_5_1_FN, FN_IP14_3_0,
-		GP_5_0_FN, FN_IP13_31_28, }
+		GP_5_0_FN, FN_IP13_31_28, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR0", 0xE6060040, 32,
-			     4, 4, 4, 4, 4, 4, 4, 4) {
+			     GROUP(4, 4, 4, 4, 4, 4, 4, 4),
+			     GROUP(
 		/* IP0_31_28 [4] */
 		FN_SD0_WP, FN_IRQ7, FN_CAN0_TX_A, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -2769,10 +2771,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP0_3_0 [4] */
 		FN_SD0_CLK, 0, 0, FN_SSI_SCK1_C, FN_RX3_C, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0, }
+		0, 0, 0, 0, 0, 0, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR1", 0xE6060044, 32,
-			     4, 4, 4, 4, 4, 4, 4, 4) {
+			     GROUP(4, 4, 4, 4, 4, 4, 4, 4),
+			     GROUP(
 		/* IP1_31_28 [4] */
 		FN_D5, FN_HRX2, FN_SCL1_B, FN_PWM2_C, FN_TCLK2_B, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -2796,10 +2799,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP1_3_0 [4] */
 		FN_MMC0_D4, FN_SD1_CD, 0, 0, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0, }
+		0, 0, 0, 0, 0, 0, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR2", 0xE6060048, 32,
-			     4, 4, 4, 4, 4, 4, 4, 4) {
+			     GROUP(4, 4, 4, 4, 4, 4, 4, 4),
+			     GROUP(
 		/* IP2_31_28 [4] */
 		FN_D13, FN_MSIOF2_SYNC_A, 0, FN_RX4_C, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0,
@@ -2823,10 +2827,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP2_3_0 [4] */
 		FN_D6, FN_HTX2, FN_SDA1_B, FN_PWM4_C, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0, }
+		0, 0, 0, 0, 0, 0, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR3", 0xE606004C, 32,
-			     4, 4, 4, 4, 4, 4, 4, 4) {
+			     GROUP(4, 4, 4, 4, 4, 4, 4, 4),
+			     GROUP(
 		/* IP3_31_28 [4] */
 		FN_QSPI0_SSL, FN_WE1_N, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0,
@@ -2851,10 +2856,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP3_3_0 [4] */
 		FN_D14, FN_MSIOF2_SS1, 0, FN_TX4_C, FN_CAN1_RX_B,
 		0, FN_AVB_AVTP_CAPTURE_A,
-		0, 0, 0, 0, 0, 0, 0, 0, 0, }
+		0, 0, 0, 0, 0, 0, 0, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR4", 0xE6060050, 32,
-			     4, 4, 4, 4, 4, 4, 4, 4) {
+			     GROUP(4, 4, 4, 4, 4, 4, 4, 4),
+			     GROUP(
 		/* IP4_31_28 [4] */
 		FN_DU0_DR6, 0, FN_RX2_C, 0, 0, 0, FN_A6, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -2878,10 +2884,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP4_3_0 [4] */
 		FN_EX_WAIT0, FN_CAN_CLK_B, FN_SCIF_CLK_A, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0, 0, }
+		0, 0, 0, 0, 0, 0, 0, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR5", 0xE6060054, 32,
-			     4, 4, 4, 4, 4, 4, 4, 4) {
+			     GROUP(4, 4, 4, 4, 4, 4, 4, 4),
+			     GROUP(
 		/* IP5_31_28 [4] */
 		FN_DU0_DG6, 0, FN_HRX1_C, 0, 0, 0, FN_A14,  0, 0, 0,
 		0, 0, 0, 0, 0, 0,
@@ -2905,10 +2912,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP5_3_0 [4] */
 		FN_DU0_DR7, 0, FN_TX2_C, 0, FN_PWM2_B, 0, FN_A7, 0,
-		0, 0, 0, 0, 0, 0, 0, 0, }
+		0, 0, 0, 0, 0, 0, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR6", 0xE6060058, 32,
-			     4, 4, 4, 4, 4, 4, 4, 4) {
+			     GROUP(4, 4, 4, 4, 4, 4, 4, 4),
+			     GROUP(
 		/* IP6_31_28 [4] */
 		FN_DU0_DB6, 0, 0, 0, 0, 0, FN_A22, 0, 0,
 		0, 0, 0, 0, 0, 0, 0,
@@ -2932,10 +2940,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP6_3_0 [4] */
 		FN_DU0_DG7, 0, FN_HTX1_C, 0,  FN_PWM6_B, 0, FN_A15,
-		0, 0, 0, 0, 0, 0, 0, 0, 0, }
+		0, 0, 0, 0, 0, 0, 0, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR7", 0xE606005C, 32,
-			     4, 4, 4, 4, 4, 4, 4, 4) {
+			     GROUP(4, 4, 4, 4, 4, 4, 4, 4),
+			     GROUP(
 		/* IP7_31_28 [4] */
 		FN_DU0_DISP, 0, 0, 0, FN_CAN1_RX_C, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0,
@@ -2959,10 +2968,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0,
 		/* IP7_3_0 [4] */
 		FN_DU0_DB7, 0, 0, 0, 0, 0, FN_A23, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, }
+		0, 0, 0, 0, 0, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR8", 0xE6060060, 32,
-			     4, 4, 4, 4, 4, 4, 4, 4) {
+			     GROUP(4, 4, 4, 4, 4, 4, 4, 4),
+			     GROUP(
 		/* IP8_31_28 [4] */
 		FN_VI1_DATA5, 0, 0, 0, FN_AVB_RXD4, FN_ETH_LINK, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0,
@@ -2986,10 +2996,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0,
 		/* IP8_3_0 [4] */
 		FN_DU0_CDE, 0, 0, 0, FN_CAN1_TX_C, 0, 0, 0, 0, 0, 0, 0,
-		0, 0, 0, 0, }
+		0, 0, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR9", 0xE6060064, 32,
-			     4, 4, 4, 4, 4, 4, 4, 4) {
+			     GROUP(4, 4, 4, 4, 4, 4, 4, 4),
+			     GROUP(
 		/* IP9_31_28 [4] */
 		FN_VI1_DATA9, 0, 0, FN_SDA2_B, FN_AVB_TXD0, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0,
@@ -3013,10 +3024,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0,
 		/* IP9_3_0 [4] */
 		FN_VI1_DATA6, 0, 0, 0, FN_AVB_RXD5, FN_ETH_TXD1, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, }
+		0, 0, 0, 0, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR10", 0xE6060068, 32,
-			     4, 4, 4, 4, 4, 4, 4, 4) {
+			     GROUP(4, 4, 4, 4, 4, 4, 4, 4),
+			     GROUP(
 		/* IP10_31_28 [4] */
 		FN_SCL1_A, FN_RX4_A, FN_PWM5_D, FN_DU1_DR0, 0, 0,
 		FN_SSI_SCK6_B, FN_VI0_G0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -3041,10 +3053,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0,
 		/* IP10_3_0 [4] */
 		FN_VI1_DATA10, 0, 0, FN_CAN0_RX_B, FN_AVB_TXD1, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, }
+		0, 0, 0, 0, 0, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR11", 0xE606006C, 32,
-			     4, 4, 4, 4, 4, 4, 4, 4) {
+			     GROUP(4, 4, 4, 4, 4, 4, 4, 4),
+			     GROUP(
 		/* IP11_31_28 [4] */
 		FN_HRX1_A, FN_SCL4_A, FN_PWM6_A, FN_DU1_DG0, FN_RX0_A, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -3072,10 +3085,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP11_3_0 [4] */
 		FN_SDA1_A, FN_TX4_A, 0, FN_DU1_DR1, 0, 0, FN_SSI_WS6_B,
-		FN_VI0_G1, 0, 0, 0, 0, 0, 0, 0, 0, }
+		FN_VI0_G1, 0, 0, 0, 0, 0, 0, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR12", 0xE6060070, 32,
-			     4, 4, 4, 4, 4, 4, 4, 4) {
+			     GROUP(4, 4, 4, 4, 4, 4, 4, 4),
+			     GROUP(
 		/* IP12_31_28 [4] */
 		FN_SD2_DAT2, FN_RX2_A, 0, FN_DU1_DB0, FN_SSI_SDATA2_B, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -3099,10 +3113,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP12_3_0 [4] */
 		FN_HTX1_A, FN_SDA4_A, 0, FN_DU1_DG1, FN_TX0_A, 0, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, }
+		0, 0, 0, 0, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR13", 0xE6060074, 32,
-			     4, 4, 4, 4, 4, 4, 4, 4) {
+			     GROUP(4, 4, 4, 4, 4, 4, 4, 4),
+			     GROUP(
 		/* IP13_31_28 [4] */
 		FN_SSI_SCK5_A, 0, 0, FN_DU1_DOTCLKOUT1, 0, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0,
@@ -3127,10 +3142,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP13_3_0 [4] */
 		FN_SD2_DAT3, FN_TX2_A, 0, FN_DU1_DB1, FN_SSI_WS9_B, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0, }
+		0, 0, 0, 0, 0, 0, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR14", 0xE6060078, 32,
-			     4, 4, 4, 4, 4, 4, 4, 4) {
+			     GROUP(4, 4, 4, 4, 4, 4, 4, 4),
+			     GROUP(
 		/* IP14_31_28 [4] */
 		FN_SSI_SDATA7_A, 0, 0, FN_IRQ8, FN_AUDIO_CLKA_D, FN_CAN_CLK_D,
 		FN_VI0_G5, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -3154,10 +3170,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP14_3_0 [4] */
 		FN_SSI_WS5_A, 0, FN_SCL3_C, FN_DU1_DOTCLKIN, 0, 0, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, }
+		0, 0, 0, 0, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR15", 0xE606007C, 32,
-			     4, 4, 4, 4, 4, 4, 4, 4) {
+			     GROUP(4, 4, 4, 4, 4, 4, 4, 4),
+			     GROUP(
 		/* IP15_31_28 [4] */
 		FN_SSI_WS4_A, 0, FN_AVB_PHY_INT, 0, 0, 0, FN_VI0_R5, 0, 0, 0,
 		0, 0, 0, 0, 0, 0,
@@ -3181,10 +3198,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_VI0_G7, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP15_3_0 [4] */
 		FN_SSI_SCK0129_A, FN_MSIOF1_RXD_A, FN_RX5_D, 0, 0, 0,
-		FN_VI0_G6, 0, 0, 0, 0, 0, 0, 0, 0, 0, }
+		FN_VI0_G6, 0, 0, 0, 0, 0, 0, 0, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR16", 0xE6060080, 32,
-			     4, 4, 4, 4, 4, 4, 4, 4) {
+			     GROUP(4, 4, 4, 4, 4, 4, 4, 4),
+			     GROUP(
 		/* IP16_31_28 [4] */
 		FN_SSI_SDATA2_A, FN_HRTS1_N_B, 0, 0, 0, 0,
 		FN_VI0_DATA4_VI0_B4, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -3209,10 +3227,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_DACK2, FN_VI0_CLK, FN_AVB_COL, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP16_3_0 [4] */
 		FN_SSI_SDATA4_A, 0, FN_AVB_CRS, 0, 0, 0, FN_VI0_R6, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, }
+		0, 0, 0, 0, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR17", 0xE6060084, 32,
-			     4, 4, 4, 4, 4, 4, 4, 4) {
+			     GROUP(4, 4, 4, 4, 4, 4, 4, 4),
+			     GROUP(
 		/* IP17_31_28 [4] */
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP17_27_24 [4] */
@@ -3235,11 +3254,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_VI0_DATA6_VI0_B6, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP17_3_0 [4] */
 		FN_SSI_SCK9_A, FN_RX2_B, FN_SCL3_E, 0, 0, FN_EX_WAIT1,
-		FN_VI0_DATA5_VI0_B5, 0, 0, 0, 0, 0, 0, 0, 0, 0, }
+		FN_VI0_DATA5_VI0_B5, 0, 0, 0, 0, 0, 0, 0, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("MOD_SEL0", 0xE60600C0, 32,
-			     1, 1, 1, 1, 1, 2, 1, 1, 2, 2, 2, 1, 3, 3,
-				 1, 2, 3, 3, 1) {
+			     GROUP(1, 1, 1, 1, 1, 2, 1, 1, 2, 2, 2, 1,
+				   3, 3, 1, 2, 3, 3, 1),
+			     GROUP(
 		/* RESERVED [1] */
 		0, 0,
 		/* RESERVED [1] */
@@ -3282,11 +3302,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_SEL_I2C00_0, FN_SEL_I2C00_1, FN_SEL_I2C00_2, FN_SEL_I2C00_3,
 		FN_SEL_I2C00_4, 0, 0, 0,
 		/* SEL_AVB [1] */
-		FN_SEL_AVB_0, FN_SEL_AVB_1, }
+		FN_SEL_AVB_0, FN_SEL_AVB_1, ))
 	},
 	{ PINMUX_CFG_REG_VAR("MOD_SEL1", 0xE60600C4, 32,
-			     1, 3, 3, 2, 2, 1, 2, 2,
-			     2, 1, 1, 1, 1, 1, 2, 1, 1, 2, 2, 1) {
+			     GROUP(1, 3, 3, 2, 2, 1, 2, 2, 2, 1, 1, 1,
+				   1, 1, 2, 1, 1, 2, 2, 1),
+			     GROUP(
 		/* SEL_SCIFCLK [1] */
 		FN_SEL_SCIFCLK_0, FN_SEL_SCIFCLK_1,
 		/* SEL_SCIF5 [3] */
@@ -3328,11 +3349,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* SEL_HSCIF1 [2] */
 		FN_SEL_HSCIF1_0, FN_SEL_HSCIF1_1, FN_SEL_HSCIF1_2, 0,
 		/* SEL_HSCIF0 [1] */
-		FN_SEL_HSCIF0_0, FN_SEL_HSCIF0_1,}
+		FN_SEL_HSCIF0_0, FN_SEL_HSCIF0_1, ))
 	},
 	{ PINMUX_CFG_REG_VAR("MOD_SEL2", 0xE60600C8, 32,
-			     1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-				 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2) {
+			     GROUP(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2,
+				   2, 2, 2, 2, 2, 2, 2, 2, 2),
+			     GROUP(
 		/* RESERVED [1] */
 		0, 0,
 		/* RESERVED [1] */
@@ -3374,7 +3396,7 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* SEL_SSI1 [2] */
 		FN_SEL_SSI1_0, FN_SEL_SSI1_1, FN_SEL_SSI1_2, FN_SEL_SSI1_3,
 		/* SEL_SSI0 [2] */
-		FN_SEL_SSI0_0, FN_SEL_SSI0_1, 0, 0, }
+		FN_SEL_SSI0_0, FN_SEL_SSI0_1, 0, 0, ))
 	},
 	{ },
 };
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a7778.c b/drivers/pinctrl/sh-pfc/pfc-r8a7778.c
index 068b5e6334d1..49fe52d35f30 100644
--- a/drivers/pinctrl/sh-pfc/pfc-r8a7778.c
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a7778.c
@@ -2104,7 +2104,7 @@ static const struct sh_pfc_function pinmux_functions[] = {
 };
 
 static const struct pinmux_cfg_reg pinmux_config_regs[] = {
-	{ PINMUX_CFG_REG("GPSR0", 0xfffc0004, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR0", 0xfffc0004, 32, 1, GROUP(
 		GP_0_31_FN,	FN_IP1_14_11,
 		GP_0_30_FN,	FN_IP1_10_8,
 		GP_0_29_FN,	FN_IP1_7_5,
@@ -2136,9 +2136,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_0_3_FN,	FN_IP0_4_2,
 		GP_0_2_FN,	FN_PENC1,
 		GP_0_1_FN,	FN_PENC0,
-		GP_0_0_FN,	FN_IP0_1_0 }
+		GP_0_0_FN,	FN_IP0_1_0 ))
 	},
-	{ PINMUX_CFG_REG("GPSR1", 0xfffc0008, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR1", 0xfffc0008, 32, 1, GROUP(
 		GP_1_31_FN,	FN_IP4_6_4,
 		GP_1_30_FN,	FN_IP4_3_1,
 		GP_1_29_FN,	FN_IP4_0,
@@ -2170,9 +2170,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_1_3_FN,	FN_IP1_27_25,
 		GP_1_2_FN,	FN_IP1_24,
 		GP_1_1_FN,	FN_WE0,
-		GP_1_0_FN,	FN_IP1_23_21 }
+		GP_1_0_FN,	FN_IP1_23_21 ))
 	},
-	{ PINMUX_CFG_REG("GPSR2", 0xfffc000c, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR2", 0xfffc000c, 32, 1, GROUP(
 		GP_2_31_FN,	FN_IP6_7,
 		GP_2_30_FN,	FN_IP6_6_5,
 		GP_2_29_FN,	FN_IP6_4_2,
@@ -2204,9 +2204,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_2_3_FN,	FN_IP4_12_11,
 		GP_2_2_FN,	FN_IP4_10_9,
 		GP_2_1_FN,	FN_IP4_8,
-		GP_2_0_FN,	FN_IP4_7 }
+		GP_2_0_FN,	FN_IP4_7 ))
 	},
-	{ PINMUX_CFG_REG("GPSR3", 0xfffc0010, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR3", 0xfffc0010, 32, 1, GROUP(
 		GP_3_31_FN,	FN_IP8_10_9,
 		GP_3_30_FN,	FN_IP8_8_6,
 		GP_3_29_FN,	FN_IP8_5_3,
@@ -2238,9 +2238,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_3_3_FN,	FN_IP6_10,
 		GP_3_2_FN,	FN_SSI_SCK34,
 		GP_3_1_FN,	FN_IP6_9,
-		GP_3_0_FN,	FN_IP6_8 }
+		GP_3_0_FN,	FN_IP6_8 ))
 	},
-	{ PINMUX_CFG_REG("GPSR4", 0xfffc0014, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR4", 0xfffc0014, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -2272,12 +2272,13 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_4_3_FN,	FN_IP8_21_19,
 		GP_4_2_FN,	FN_IP8_18_16,
 		GP_4_1_FN,	FN_IP8_15_14,
-		GP_4_0_FN,	FN_IP8_13_11 }
+		GP_4_0_FN,	FN_IP8_13_11 ))
 	},
 
 	{ PINMUX_CFG_REG_VAR("IPSR0", 0xfffc0020, 32,
-			     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-			     1, 1, 1, 1, 1, 1, 3, 4, 3, 3, 2) {
+			     GROUP(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+				   1, 1, 1, 1, 1, 3, 4, 3, 3, 2),
+			     GROUP(
 		/* IP0_31 [1] */
 		0,	0,
 		/* IP0_30 [1] */
@@ -2328,10 +2329,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_TX2_E,	FN_SDA2_B,		0,		0,
 		/* IP0_1_0 [2] */
 		FN_PRESETOUT,	0,	FN_PWM1,	0,
-		}
+		))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR1", 0xfffc0024, 32,
-			     1, 1, 2, 3, 1, 3, 3, 1, 2, 4, 3, 3, 3, 1, 1) {
+			     GROUP(1, 1, 2, 3, 1, 3, 3, 1, 2, 4, 3, 3,
+				   3, 1, 1),
+			     GROUP(
 		/* IP1_31 [1] */
 		0,	0,
 		/* IP1_30 [1] */
@@ -2371,11 +2374,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_A21,		FN_HSPI_CLK1_B,
 		/* IP1_0 [1] */
 		FN_A20,		FN_HSPI_CS1_B,
-		}
+		))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR2", 0xfffc0028, 32,
-			     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-			     1, 1, 1, 1, 3, 2, 3, 3, 3, 3) {
+			     GROUP(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+				   1, 1, 1, 3, 2, 3, 3, 3, 3),
+			     GROUP(
 		/* IP2_31 [1] */
 		FN_MLB_CLK,	FN_IRQ1_A,
 		/* IP2_30 [1] */
@@ -2423,11 +2427,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP2_2_0 [3] */
 		FN_SD1_CLK_A,	FN_MMC_CLK,	0,	FN_ATACS00,
 		FN_EX_CS2,	0,		0,	0,
-		}
+		))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR3", 0xfffc002c, 32,
-			     1, 1, 1, 1, 1, 3, 3, 2,
-			     3, 3, 3, 2, 3, 3, 2) {
+			     GROUP(1, 1, 1, 1, 1, 3, 3, 2, 3, 3, 3, 2,
+				   3, 3, 2),
+			     GROUP(
 		/* IP3_31 [1] */
 		FN_DU0_DR6,	FN_LCDOUT6,
 		/* IP3_30 [1] */
@@ -2465,10 +2470,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_SDSELF_B,	0,		0,		0,
 		/* IP3_1_0 [2] */
 		FN_MLB_SIG,	FN_RX5_B,	FN_SDA3_A,	FN_IRQ2_A,
-		}
+		))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR4", 0xfffc0030, 32,
-			     1, 2, 2, 2, 4, 4, 2, 2, 2, 2, 1, 1, 3, 3, 1) {
+			     GROUP(1, 2, 2, 2, 4, 4, 2, 2, 2, 2, 1, 1,
+				   3, 3, 1),
+			     GROUP(
 		/* IP4_31 [1] */
 		0,	0,
 		/* IP4_30_29 [2] */
@@ -2507,10 +2514,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_TX1_D,	FN_CAN0_TX_A,	FN_ADICHS0,	0,
 		/* IP4_0 [1] */
 		FN_DU0_DR7,	FN_LCDOUT7,
-		}
+		))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR5", 0xfffc0034, 32,
-			     1, 2, 3, 3, 2, 3, 3, 2, 1, 2, 2, 1, 1, 2, 2, 2) {
+			     GROUP(1, 2, 3, 3, 2, 3, 3, 2, 1, 2, 2, 1,
+				   1, 2, 2, 2),
+			     GROUP(
 
 		/* IP5_31 [1] */
 		0, 0,
@@ -2551,11 +2560,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_VI1_DATA10_B,	FN_DU0_DB6,	FN_LCDOUT22,	0,
 		/* IP5_1_0 [2] */
 		FN_VI0_R5_B,		FN_DU0_DB5,	FN_LCDOUT21,	0,
-		}
+		))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR6", 0xfffc0038, 32,
-			     2, 2, 2, 2, 2, 1, 2, 2, 1, 2,
-			     1, 2, 1, 1, 1, 1, 2, 3, 2) {
+			     GROUP(2, 2, 2, 2, 2, 1, 2, 2, 1, 2, 1, 2,
+				   1, 1, 1, 1, 2, 3, 2),
+			     GROUP(
 		/* IP6_31_30 [2] */
 		FN_SD0_DAT2,	0,	FN_SUB_TDI,	0,
 		/* IP6_29_28 [2] */
@@ -2602,10 +2612,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP6_1_0 [2] */
 		FN_SSI_SCK6,		FN_HSPI_RX2_A,
 		FN_FMCLK_B,		FN_CAN1_TX_B,
-		}
+		))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR7", 0xfffc003c, 32,
-			     3, 4, 3, 1, 3, 3, 3, 3, 3, 2, 2, 2) {
+			     GROUP(3, 4, 3, 1, 3, 3, 3, 3, 3, 2, 2, 2),
+			     GROUP(
 
 		/* IP7_31_29 [3] */
 		FN_VI0_HSYNC,	FN_SD2_CD_B,	FN_VI1_DATA2,	FN_DU1_DR2,
@@ -2641,10 +2652,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_SD0_CD,	0,		FN_TX5_A,	0,
 		/* IP7_1_0 [2] */
 		FN_SD0_DAT3,	0,		FN_IRQ1_B,	0,
-		}
+		))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR8", 0xfffc0040, 32,
-			     1, 1, 3, 3, 2, 3, 3, 2, 3, 2, 3, 3, 3) {
+			     GROUP(1, 1, 3, 3, 2, 3, 3, 2, 3, 2, 3, 3, 3),
+			     GROUP(
 		/* IP8_31 [1] */
 		0, 0,
 		/* IP8_30 [1] */
@@ -2681,10 +2693,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP8_2_0 [3] */
 		FN_VI0_VSYNC,		FN_SD2_WP_B,	FN_VI1_DATA3,	FN_DU1_DR3,
 		0,			FN_HSPI_TX1_A,	FN_TX3_B,	0,
-		}
+		))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR9", 0xfffc0044, 32,
-			     1, 1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3) {
+			     GROUP(1, 1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3),
+			     GROUP(
 		/* IP9_31 [1] */
 		0, 0,
 		/* IP9_30 [1] */
@@ -2723,10 +2736,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP9_2_0 [3] */
 		FN_VI0_G4,	FN_SD2_DAT0_B,	FN_VI1_DATA6,	FN_DU1_DR6,
 		0,		FN_HRTS1_B,	0,		0,
-		}
+		))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR10", 0xfffc0048, 32,
-			     1, 1, 1, 1, 1, 1, 1, 3, 3, 3, 3, 4, 3, 3, 3) {
+			     GROUP(1, 1, 1, 1, 1, 1, 1, 3, 3, 3, 3, 4,
+				   3, 3, 3),
+			     GROUP(
 
 		/* IP10_31 [1] */
 		0, 0,
@@ -2772,11 +2787,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_ATARD1,	FN_ETH_MDC,
 		FN_SDA1_B,	0,
 		0,		0,
-		}
+		))
 	},
 	{ PINMUX_CFG_REG_VAR("MOD_SEL0", 0xfffc0050, 32,
-			     1, 1, 2, 2, 3, 2, 2, 1, 1, 1, 1, 2,
-			     1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1) {
+			     GROUP(1, 1, 2, 2, 3, 2, 2, 1, 1, 1, 1, 2,
+				   1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1),
+			     GROUP(
 
 		/* SEL 31  [1] */
 		0, 0,
@@ -2835,11 +2851,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_SEL_WAIT2_A,		FN_SEL_WAIT2_B,
 		/* SEL_0 (WAIT1) [1] */
 		FN_SEL_WAIT1_A,		FN_SEL_WAIT1_B,
-		}
+		))
 	},
 	{ PINMUX_CFG_REG_VAR("MOD_SEL1", 0xfffc0054, 32,
-			     1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1,
-			     1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 2, 2, 1) {
+			     GROUP(1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1,
+				   1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 2, 2, 1),
+			     GROUP(
 
 		/* SEL_31 [1] */
 		0, 0,
@@ -2899,7 +2916,7 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_SEL_I2C2_C,		0,
 		/* SEL_0 (I2C1) [1] */
 		FN_SEL_I2C1_A,		FN_SEL_I2C1_B,
-		}
+		))
 	},
 	{ },
 };
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a7779.c b/drivers/pinctrl/sh-pfc/pfc-r8a7779.c
index 64bace100316..0c121b28ec3f 100644
--- a/drivers/pinctrl/sh-pfc/pfc-r8a7779.c
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a7779.c
@@ -1595,6 +1595,92 @@ static const unsigned int ether_magic_pins[] = {
 static const unsigned int ether_magic_mux[] = {
 	ETH_MAGIC_MARK,
 };
+/* - HSCIF0 ----------------------------------------------------------------- */
+static const unsigned int hscif0_data_pins[] = {
+	/* TX, RX */
+	RCAR_GP_PIN(4, 20), RCAR_GP_PIN(4, 21)
+};
+static const unsigned int hscif0_data_mux[] = {
+	HTX0_MARK, HRX0_MARK
+};
+static const unsigned int hscif0_data_b_pins[] = {
+	/* TX, RX */
+	RCAR_GP_PIN(3, 12), RCAR_GP_PIN(3, 13)
+};
+static const unsigned int hscif0_data_b_mux[] = {
+	HTX0_B_MARK, HRX0_B_MARK
+};
+static const unsigned int hscif0_ctrl_pins[] = {
+	/* CTS, RTS */
+	RCAR_GP_PIN(4, 18), RCAR_GP_PIN(4, 19)
+};
+static const unsigned int hscif0_ctrl_mux[] = {
+	HCTS0_MARK, HRTS0_MARK
+};
+static const unsigned int hscif0_ctrl_b_pins[] = {
+	/* CTS, RTS */
+	RCAR_GP_PIN(3, 9), RCAR_GP_PIN(3, 10)
+};
+static const unsigned int hscif0_ctrl_b_mux[] = {
+	HCTS0_B_MARK, HRTS0_B_MARK
+};
+static const unsigned int hscif0_clk_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(4, 17)
+};
+static const unsigned int hscif0_clk_mux[] = {
+	HSCK0_MARK
+};
+static const unsigned int hscif0_clk_b_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(3, 11)
+};
+static const unsigned int hscif0_clk_b_mux[] = {
+	HSCK0_B_MARK
+};
+/* - HSCIF1 ----------------------------------------------------------------- */
+static const unsigned int hscif1_data_pins[] = {
+	/* TX, RX */
+	RCAR_GP_PIN(0, 19), RCAR_GP_PIN(0, 20)
+};
+static const unsigned int hscif1_data_mux[] = {
+	HTX1_MARK, HRX1_MARK
+};
+static const unsigned int hscif1_data_b_pins[] = {
+	/* TX, RX */
+	RCAR_GP_PIN(2, 2), RCAR_GP_PIN(2, 3)
+};
+static const unsigned int hscif1_data_b_mux[] = {
+	HTX1_B_MARK, HRX1_B_MARK
+};
+static const unsigned int hscif1_ctrl_pins[] = {
+	/* CTS, RTS */
+	RCAR_GP_PIN(0, 21), RCAR_GP_PIN(0, 22)
+};
+static const unsigned int hscif1_ctrl_mux[] = {
+	HCTS1_MARK, HRTS1_MARK
+};
+static const unsigned int hscif1_ctrl_b_pins[] = {
+	/* CTS, RTS */
+	RCAR_GP_PIN(2, 7), RCAR_GP_PIN(2, 6)
+};
+static const unsigned int hscif1_ctrl_b_mux[] = {
+	HCTS1_B_MARK, HRTS1_B_MARK
+};
+static const unsigned int hscif1_clk_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(0, 18)
+};
+static const unsigned int hscif1_clk_mux[] = {
+	HSCK1_MARK
+};
+static const unsigned int hscif1_clk_b_pins[] = {
+	/* SCK */
+	RCAR_GP_PIN(2, 4)
+};
+static const unsigned int hscif1_clk_b_mux[] = {
+	HSCK1_B_MARK
+};
 /* - HSPI0 ------------------------------------------------------------------ */
 static const unsigned int hspi0_pins[] = {
 	/* CLK, CS, RX, TX */
@@ -2618,6 +2704,18 @@ static const struct sh_pfc_pin_group pinmux_groups[] = {
 	SH_PFC_PIN_GROUP(ether_rmii),
 	SH_PFC_PIN_GROUP(ether_link),
 	SH_PFC_PIN_GROUP(ether_magic),
+	SH_PFC_PIN_GROUP(hscif0_data),
+	SH_PFC_PIN_GROUP(hscif0_data_b),
+	SH_PFC_PIN_GROUP(hscif0_ctrl),
+	SH_PFC_PIN_GROUP(hscif0_ctrl_b),
+	SH_PFC_PIN_GROUP(hscif0_clk),
+	SH_PFC_PIN_GROUP(hscif0_clk_b),
+	SH_PFC_PIN_GROUP(hscif1_data),
+	SH_PFC_PIN_GROUP(hscif1_data_b),
+	SH_PFC_PIN_GROUP(hscif1_ctrl),
+	SH_PFC_PIN_GROUP(hscif1_ctrl_b),
+	SH_PFC_PIN_GROUP(hscif1_clk),
+	SH_PFC_PIN_GROUP(hscif1_clk_b),
 	SH_PFC_PIN_GROUP(hspi0),
 	SH_PFC_PIN_GROUP(hspi1),
 	SH_PFC_PIN_GROUP(hspi1_b),
@@ -2783,6 +2881,24 @@ static const char * const ether_groups[] = {
 	"ether_magic",
 };
 
+static const char * const hscif0_groups[] = {
+	"hscif0_data",
+	"hscif0_data_b",
+	"hscif0_ctrl",
+	"hscif0_ctrl_b",
+	"hscif0_clk",
+	"hscif0_clk_b",
+};
+
+static const char * const hscif1_groups[] = {
+	"hscif1_data",
+	"hscif1_data_b",
+	"hscif1_ctrl",
+	"hscif1_ctrl_b",
+	"hscif1_clk",
+	"hscif1_clk_b",
+};
+
 static const char * const hspi0_groups[] = {
 	"hspi0",
 };
@@ -3005,6 +3121,8 @@ static const struct sh_pfc_function pinmux_functions[] = {
 	SH_PFC_FUNCTION(du0),
 	SH_PFC_FUNCTION(du1),
 	SH_PFC_FUNCTION(ether),
+	SH_PFC_FUNCTION(hscif0),
+	SH_PFC_FUNCTION(hscif1),
 	SH_PFC_FUNCTION(hspi0),
 	SH_PFC_FUNCTION(hspi1),
 	SH_PFC_FUNCTION(hspi2),
@@ -3036,7 +3154,7 @@ static const struct sh_pfc_function pinmux_functions[] = {
 };
 
 static const struct pinmux_cfg_reg pinmux_config_regs[] = {
-	{ PINMUX_CFG_REG("GPSR0", 0xfffc0004, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR0", 0xfffc0004, 32, 1, GROUP(
 		GP_0_31_FN, FN_IP3_31_29,
 		GP_0_30_FN, FN_IP3_26_24,
 		GP_0_29_FN, FN_IP3_22_21,
@@ -3068,9 +3186,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_0_3_FN, FN_A17,
 		GP_0_2_FN, FN_IP0_7_6,
 		GP_0_1_FN, FN_AVS2,
-		GP_0_0_FN, FN_AVS1 }
+		GP_0_0_FN, FN_AVS1 ))
 	},
-	{ PINMUX_CFG_REG("GPSR1", 0xfffc0008, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR1", 0xfffc0008, 32, 1, GROUP(
 		GP_1_31_FN, FN_IP5_23_21,
 		GP_1_30_FN, FN_IP5_20_17,
 		GP_1_29_FN, FN_IP5_16_15,
@@ -3102,9 +3220,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_1_3_FN, FN_IP4_10_8,
 		GP_1_2_FN, FN_IP4_7_5,
 		GP_1_1_FN, FN_IP4_4_2,
-		GP_1_0_FN, FN_IP4_1_0 }
+		GP_1_0_FN, FN_IP4_1_0 ))
 	},
-	{ PINMUX_CFG_REG("GPSR2", 0xfffc000c, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR2", 0xfffc000c, 32, 1, GROUP(
 		GP_2_31_FN, FN_IP10_28_26,
 		GP_2_30_FN, FN_IP10_25_24,
 		GP_2_29_FN, FN_IP10_23_21,
@@ -3136,9 +3254,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_2_3_FN, FN_IP8_24_23,
 		GP_2_2_FN, FN_IP8_22_21,
 		GP_2_1_FN, FN_IP8_20,
-		GP_2_0_FN, FN_IP5_27_24 }
+		GP_2_0_FN, FN_IP5_27_24 ))
 	},
-	{ PINMUX_CFG_REG("GPSR3", 0xfffc0010, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR3", 0xfffc0010, 32, 1, GROUP(
 		GP_3_31_FN, FN_IP6_3_2,
 		GP_3_30_FN, FN_IP6_1_0,
 		GP_3_29_FN, FN_IP5_30_29,
@@ -3170,9 +3288,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_3_3_FN, FN_IP11_8_6,
 		GP_3_2_FN, FN_IP11_5_3,
 		GP_3_1_FN, FN_IP11_2_0,
-		GP_3_0_FN, FN_IP10_31_29 }
+		GP_3_0_FN, FN_IP10_31_29 ))
 	},
-	{ PINMUX_CFG_REG("GPSR4", 0xfffc0014, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR4", 0xfffc0014, 32, 1, GROUP(
 		GP_4_31_FN, FN_IP8_19,
 		GP_4_30_FN, FN_IP8_18,
 		GP_4_29_FN, FN_IP8_17_16,
@@ -3204,9 +3322,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_4_3_FN, FN_IP6_11_9,
 		GP_4_2_FN, FN_IP6_8,
 		GP_4_1_FN, FN_IP6_7_6,
-		GP_4_0_FN, FN_IP6_5_4 }
+		GP_4_0_FN, FN_IP6_5_4 ))
 	},
-	{ PINMUX_CFG_REG("GPSR5", 0xfffc0018, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR5", 0xfffc0018, 32, 1, GROUP(
 		GP_5_31_FN, FN_IP3_5,
 		GP_5_30_FN, FN_IP3_4,
 		GP_5_29_FN, FN_IP3_3,
@@ -3238,9 +3356,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_5_3_FN, FN_A4,
 		GP_5_2_FN, FN_A3,
 		GP_5_1_FN, FN_A2,
-		GP_5_0_FN, FN_A1 }
+		GP_5_0_FN, FN_A1 ))
 	},
-	{ PINMUX_CFG_REG("GPSR6", 0xfffc001c, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR6", 0xfffc001c, 32, 1, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0,	0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,	0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -3255,11 +3373,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_6_3_FN, FN_IP3_15,
 		GP_6_2_FN, FN_IP3_8,
 		GP_6_1_FN, FN_IP3_7,
-		GP_6_0_FN, FN_IP3_6 }
+		GP_6_0_FN, FN_IP3_6 ))
 	},
 
 	{ PINMUX_CFG_REG_VAR("IPSR0", 0xfffc0020, 32,
-			     1, 3, 2, 1, 2, 4, 3, 2, 2, 2, 2, 2, 3, 3) {
+			     GROUP(1, 3, 2, 1, 2, 4, 3, 2, 2, 2, 2, 2, 3, 3),
+			     GROUP(
 		/* IP0_31 [1] */
 		0, 0,
 		/* IP0_30_28 [3] */
@@ -3294,10 +3413,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_ATADIR0, FN_SDSELF, FN_HCTS1, FN_TX4_C,
 		/* IP0_2_0 [3] */
 		FN_USB_PENC2, FN_SCK0, FN_PWM1, FN_PWMFSW0,
-		FN_SCIF_CLK, FN_TCLK0_C, 0, 0 }
+		FN_SCIF_CLK, FN_TCLK0_C, 0, 0 ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR1", 0xfffc0024, 32,
-			     3, 4, 2, 2, 2, 4, 4, 4, 3, 2, 2) {
+			     GROUP(3, 4, 2, 2, 2, 4, 4, 4, 3, 2, 2),
+			     GROUP(
 		/* IP1_31_29 [3] */
 		0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP1_28_25 [4] */
@@ -3332,10 +3452,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP1_3_2 [2] */
 		FN_EX_CS1, FN_MMC0_D7, FN_FD7, 0,
 		/* IP1_1_0 [2] */
-		FN_EX_CS0, FN_RX3_C_IRDA_RX_C, FN_MMC0_D6, FN_FD6 }
+		FN_EX_CS0, FN_RX3_C_IRDA_RX_C, FN_MMC0_D6, FN_FD6 ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR2", 0xfffc0028, 32,
-			     1, 3, 1, 1, 1, 1, 1, 1, 3, 3, 4, 4, 4, 4) {
+			     GROUP(1, 3, 1, 1, 1, 1, 1, 1, 3, 3, 4, 4, 4, 4),
+			     GROUP(
 		/* IP2_31 [1] */
 		0, 0,
 		/* IP2_30_28 [3] */
@@ -3378,11 +3499,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_HRX0, FN_RX1, FN_SCKZ, FN_RTS0_C_TANS_C,
 		FN_SUB_TDI, FN_CC5_STATE3, FN_CC5_STATE11, FN_CC5_STATE19,
 		FN_CC5_STATE27, FN_CC5_STATE35, 0, 0,
-		0, 0, 0, 0 }
+		0, 0, 0, 0 ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR3", 0xfffc002c, 32,
-			     3, 1, 1, 3, 1, 2, 1, 1, 1, 1, 1,
-			     1, 3, 3, 1, 1, 1, 1, 1, 1, 3) {
+			     GROUP(3, 1, 1, 3, 1, 2, 1, 1, 1, 1, 1, 1,
+				   3, 3, 1, 1, 1, 1, 1, 1, 3),
+			     GROUP(
 	    /* IP3_31_29 [3] */
 	    FN_DU0_EXODDF_DU0_ODDF_DISP_CDE, FN_QCPV_QDE, FN_CAN1_TX, FN_TX2_C,
 	    FN_SCL2_C, FN_REMOCON, 0, 0,
@@ -3429,11 +3551,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 	    FN_DU0_DG2, FN_LCDOUT10,
 	    /* IP3_2_0 [3] */
 	    FN_DU0_DG1, FN_LCDOUT9, FN_DACK1, FN_SDA2,
-	    FN_AUDATA3, 0, 0, 0 }
+	    FN_AUDATA3, 0, 0, 0 ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR4", 0xfffc0030, 32,
-			     3, 1, 1, 1, 1, 1, 1, 3, 3,
-			     1, 1, 1, 1, 1, 1, 3, 3, 3, 2) {
+			     GROUP(3, 1, 1, 1, 1, 1, 1, 3, 3, 1, 1, 1,
+				   1, 1, 1, 3, 3, 3, 2),
+			     GROUP(
 	    /* IP4_31_29 [3] */
 	    FN_DU1_DB0, FN_VI2_DATA4_VI2_B4, FN_SCL2_B, FN_SD3_DAT0,
 	    FN_TX5, FN_SCK0_D, 0, 0,
@@ -3477,11 +3600,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 	    FN_DU0_CDE, FN_QPOLB, FN_CAN1_RX, FN_RX2_C,
 	    FN_DREQ0_B, FN_SSI_SCK78_B, FN_SCK0_B, 0,
 	    /* IP4_1_0 [2] */
-	    FN_DU0_DISP, FN_QPOLA, FN_CAN_CLK_C, FN_SCK2_C }
+	    FN_DU0_DISP, FN_QPOLA, FN_CAN_CLK_C, FN_SCK2_C ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR5", 0xfffc0034, 32,
-			     1, 2, 1, 4, 3, 4, 2, 2,
-			     2, 2, 1, 1, 1, 1, 1, 1, 3) {
+			     GROUP(1, 2, 1, 4, 3, 4, 2, 2, 2, 2, 1, 1,
+				   1, 1, 1, 1, 3),
+			     GROUP(
 	    /* IP5_31 [1] */
 	    0, 0,
 	    /* IP5_30_29 [2] */
@@ -3523,10 +3647,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 	    FN_DU1_DB2, FN_VI2_R4,
 	    /* IP5_2_0 [3] */
 	    FN_DU1_DB1, FN_VI2_DATA5_VI2_B5, FN_SDA2_B, FN_SD3_DAT1,
-	    FN_RX5, FN_RTS0_D_TANS_D, 0, 0 }
+	    FN_RX5, FN_RTS0_D_TANS_D, 0, 0 ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR6", 0xfffc0038, 32,
-			     1, 2, 2, 2, 2, 3, 2, 3, 3, 3, 1, 2, 2, 2, 2) {
+			     GROUP(1, 2, 2, 2, 2, 3, 2, 3, 3, 3, 1, 2,
+				   2, 2, 2),
+			     GROUP(
 	    /* IP6_31 [1] */
 	    0, 0,
 	    /* IP6_30_29 [2] */
@@ -3560,10 +3686,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 	    /* IP6_3_2 [2] */
 	    FN_SSI_WS0129, FN_CAN_DEBUGOUT2, FN_MOUT2, 0,
 	    /* IP6_1_0 [2] */
-	    FN_SSI_SCK0129, FN_CAN_DEBUGOUT1, FN_MOUT1, 0 }
+	    FN_SSI_SCK0129, FN_CAN_DEBUGOUT1, FN_MOUT1, 0 ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR7", 0xfffc003c, 32,
-			     1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 2, 2) {
+			     GROUP(1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3,
+				   3, 2, 2),
+			     GROUP(
 	    /* IP7_31 [1] */
 	    0, 0,
 	    /* IP7_30_29 [2] */
@@ -3596,10 +3724,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 	    /* IP7_3_2 [2] */
 	    FN_SSI_SDATA6, FN_ADICHS2, FN_CAN_CLK, FN_IECLK_B,
 	    /* IP7_1_0 [2] */
-	    FN_SSI_WS6, FN_ADICHS1, FN_CAN0_RX, FN_IETX_B }
+	    FN_SSI_WS6, FN_ADICHS1, FN_CAN0_RX, FN_IETX_B ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR8", 0xfffc0040, 32,
-			     1, 3, 3, 2, 2, 1, 1, 1, 2, 4, 4, 4, 4) {
+			     GROUP(1, 3, 3, 2, 2, 1, 1, 1, 2, 4, 4, 4, 4),
+			     GROUP(
 	    /* IP8_31 [1] */
 	    0, 0,
 	    /* IP8_30_28 [3] */
@@ -3639,11 +3768,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 	    FN_HSPI_CLK0, FN_CTS0, FN_USB_OVC0, FN_AD_CLK,
 	    FN_CC5_STATE4, FN_CC5_STATE12, FN_CC5_STATE20, FN_CC5_STATE28,
 	    FN_CC5_STATE36, 0, 0, 0,
-	    0, 0, 0, 0 }
+	    0, 0, 0, 0 ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR9", 0xfffc0044, 32,
-			     2, 2, 2, 2, 2, 3, 3, 2, 2,
-			     2, 2, 1, 1, 1, 1, 2, 2) {
+			     GROUP(2, 2, 2, 2, 2, 3, 3, 2, 2, 2, 2, 1,
+				   1, 1, 1, 2, 2),
+			     GROUP(
 	    /* IP9_31_30 [2] */
 	    0, 0, 0, 0,
 	    /* IP9_29_28 [2] */
@@ -3679,10 +3809,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 	    /* IP9_3_2 [2] */
 	    FN_VI0_DATA1_VI0_B1, FN_HCTS1_B, FN_MT1_PWM, 0,
 	    /* IP9_1_0 [2] */
-	    FN_VI0_DATA0_VI0_B0, FN_HRTS1_B, FN_MT1_VCXO, 0 }
+	    FN_VI0_DATA0_VI0_B0, FN_HRTS1_B, FN_MT1_VCXO, 0 ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR10", 0xfffc0048, 32,
-			     3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3) {
+			     GROUP(3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3),
+			     GROUP(
 	    /* IP10_31_29 [3] */
 	    FN_VI1_VSYNC, FN_AUDIO_CLKOUT_C, FN_SSI_WS4, FN_SIM_CLK,
 	    FN_GPS_MAG_C, FN_SPV_TRST, FN_SCL3, 0,
@@ -3714,10 +3845,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 	    FN_DACK0_C, FN_DRACK0_C, 0, 0,
 	    /* IP10_2_0 [3] */
 	    FN_VI0_R0, FN_SSI_SDATA7_C, FN_SCK1_C, FN_DREQ1_B,
-	    FN_ARM_TRACEDATA_10, FN_DREQ0_C, 0, 0 }
+	    FN_ARM_TRACEDATA_10, FN_DREQ0_C, 0, 0 ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR11", 0xfffc004c, 32,
-			     2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3) {
+			     GROUP(2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3),
+			     GROUP(
 	    /* IP11_31_30 [2] */
 	    0, 0, 0, 0,
 	    /* IP11_29_27 [3] */
@@ -3749,10 +3881,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 	    FN_ADICS_B_SAMP_B, 0, 0, 0,
 	    /* IP11_2_0 [3] */
 	    FN_VI1_DATA0_VI1_B0, FN_SD2_DAT0, FN_SIM_RST, FN_SPV_TCK,
-	    FN_ADICLK_B, 0, 0, 0 }
+	    FN_ADICLK_B, 0, 0, 0 ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR12", 0xfffc0050, 32,
-			     4, 4, 4, 2, 3, 3, 3, 3, 3, 3) {
+			     GROUP(4, 4, 4, 2, 3, 3, 3, 3, 3, 3),
+			     GROUP(
 	    /* IP12_31_28 [4] */
 	    0, 0, 0, 0, 0, 0, 0, 0,
 	    0, 0, 0, 0, 0, 0, 0, 0,
@@ -3781,11 +3914,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 	    FN_SCL1_C, FN_HTX0_B, 0, 0,
 	    /* IP12_2_0 [3] */
 	    FN_VI1_G2, FN_VI3_DATA2, FN_SSI_WS1, FN_TS_SPSYNC1,
-	    FN_SCK2, FN_HSCK0_B, 0, 0 }
+	    FN_SCK2, FN_HSCK0_B, 0, 0 ))
 	},
 	{ PINMUX_CFG_REG_VAR("MOD_SEL", 0xfffc0090, 32,
-			     2, 2, 3, 3, 2, 2, 2, 2, 2,
-			     1, 1, 1, 1, 1, 1, 1, 2, 1, 2) {
+			     GROUP(2, 2, 3, 3, 2, 2, 2, 2, 2, 1, 1, 1,
+				   1, 1, 1, 1, 2, 1, 2),
+			     GROUP(
 	    /* SEL_SCIF5 [2] */
 	    FN_SEL_SCIF5_0, FN_SEL_SCIF5_1, FN_SEL_SCIF5_2, FN_SEL_SCIF5_3,
 	    /* SEL_SCIF4 [2] */
@@ -3825,11 +3959,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 	    /* SEL_EXBUS1 [1] */
 	    FN_SEL_EXBUS1_0, FN_SEL_EXBUS1_1,
 	    /* SEL_EXBUS0 [2] */
-	    FN_SEL_EXBUS0_0, FN_SEL_EXBUS0_1, FN_SEL_EXBUS0_2, 0 }
+	    FN_SEL_EXBUS0_0, FN_SEL_EXBUS0_1, FN_SEL_EXBUS0_2, 0 ))
 	},
 	{ PINMUX_CFG_REG_VAR("MOD_SEL2", 0xfffc0094, 32,
-			     2, 2, 2, 2, 1, 1, 1, 3, 1,
-			     2, 2, 2, 2, 1, 1, 2, 1, 2, 2) {
+			     GROUP(2, 2, 2, 2, 1, 1, 1, 3, 1, 2, 2, 2,
+				   2, 1, 1, 2, 1, 2, 2),
+			     GROUP(
 	    /* SEL_TMU1 [2] */
 	    FN_SEL_TMU1_0, FN_SEL_TMU1_1, FN_SEL_TMU1_2, 0,
 	    /* SEL_TMU0 [2] */
@@ -3868,7 +4003,7 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 	    /* SEL_I2C2 [2] */
 	    FN_SEL_I2C2_0, FN_SEL_I2C2_1, FN_SEL_I2C2_2, FN_SEL_I2C2_3,
 	    /* SEL_I2C1 [2] */
-	    FN_SEL_I2C1_0, FN_SEL_I2C1_1, FN_SEL_I2C1_2, FN_SEL_I2C1_3 }
+	    FN_SEL_I2C1_0, FN_SEL_I2C1_1, FN_SEL_I2C1_2, FN_SEL_I2C1_3 ))
 	},
 	{ },
 };
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a7790.c b/drivers/pinctrl/sh-pfc/pfc-r8a7790.c
index a84229cb8cd4..c41a6761cf9d 100644
--- a/drivers/pinctrl/sh-pfc/pfc-r8a7790.c
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a7790.c
@@ -8,6 +8,7 @@
  * Copyright (C) 2012  Kuninori Morimoto <kuninori.morimoto.gx@renesas.com>
  */
 
+#include <linux/errno.h>
 #include <linux/io.h>
 #include <linux/kernel.h>
 #include <linux/sys_soc.h>
@@ -4744,7 +4745,7 @@ static const struct sh_pfc_function pinmux_functions[] = {
 };
 
 static const struct pinmux_cfg_reg pinmux_config_regs[] = {
-	{ PINMUX_CFG_REG("GPSR0", 0xE6060004, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR0", 0xE6060004, 32, 1, GROUP(
 		GP_0_31_FN, FN_IP3_17_15,
 		GP_0_30_FN, FN_IP3_14_12,
 		GP_0_29_FN, FN_IP3_11_8,
@@ -4776,9 +4777,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_0_3_FN, FN_IP0_11_9,
 		GP_0_2_FN, FN_IP0_8_6,
 		GP_0_1_FN, FN_IP0_5_3,
-		GP_0_0_FN, FN_IP0_2_0 }
+		GP_0_0_FN, FN_IP0_2_0 ))
 	},
-	{ PINMUX_CFG_REG("GPSR1", 0xE6060008, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR1", 0xE6060008, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		GP_1_29_FN, FN_IP6_13_11,
@@ -4810,9 +4811,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_1_3_FN, FN_IP3_28_26,
 		GP_1_2_FN, FN_IP3_25_23,
 		GP_1_1_FN, FN_IP3_22_20,
-		GP_1_0_FN, FN_IP3_19_18, }
+		GP_1_0_FN, FN_IP3_19_18, ))
 	},
-	{ PINMUX_CFG_REG("GPSR2", 0xE606000C, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR2", 0xE606000C, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		GP_2_29_FN, FN_IP7_15_13,
@@ -4844,9 +4845,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_2_3_FN, FN_IP8_3_2,
 		GP_2_2_FN, FN_IP8_1_0,
 		GP_2_1_FN, FN_IP7_30_29,
-		GP_2_0_FN, FN_IP7_28_27 }
+		GP_2_0_FN, FN_IP7_28_27 ))
 	},
-	{ PINMUX_CFG_REG("GPSR3", 0xE6060010, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR3", 0xE6060010, 32, 1, GROUP(
 		GP_3_31_FN, FN_IP11_21_18,
 		GP_3_30_FN, FN_IP11_17_15,
 		GP_3_29_FN, FN_IP11_14_13,
@@ -4878,9 +4879,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_3_3_FN, FN_IP9_3_2,
 		GP_3_2_FN, FN_IP9_1_0,
 		GP_3_1_FN, FN_IP8_30_29,
-		GP_3_0_FN, FN_IP8_28 }
+		GP_3_0_FN, FN_IP8_28 ))
 	},
-	{ PINMUX_CFG_REG("GPSR4", 0xE6060014, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR4", 0xE6060014, 32, 1, GROUP(
 		GP_4_31_FN, FN_IP14_18_16,
 		GP_4_30_FN, FN_IP14_15_12,
 		GP_4_29_FN, FN_IP14_11_9,
@@ -4912,9 +4913,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_4_3_FN, FN_IP11_31_30,
 		GP_4_2_FN, FN_IP11_29_27,
 		GP_4_1_FN, FN_IP11_26_24,
-		GP_4_0_FN, FN_IP11_23_22 }
+		GP_4_0_FN, FN_IP11_23_22 ))
 	},
-	{ PINMUX_CFG_REG("GPSR5", 0xE6060018, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR5", 0xE6060018, 32, 1, GROUP(
 		GP_5_31_FN, FN_IP7_24_22,
 		GP_5_30_FN, FN_IP7_21_19,
 		GP_5_29_FN, FN_IP7_18_16,
@@ -4946,10 +4947,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_5_3_FN, FN_IP14_30_28,
 		GP_5_2_FN, FN_IP14_27_25,
 		GP_5_1_FN, FN_IP14_24_22,
-		GP_5_0_FN, FN_IP14_21_19 }
+		GP_5_0_FN, FN_IP14_21_19 ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR0", 0xE6060020, 32,
-			     1, 4, 4, 3, 4, 4, 3, 3, 3, 3) {
+			     GROUP(1, 4, 4, 3, 4, 4, 3, 3, 3, 3),
+			     GROUP(
 		/* IP0_31 [1] */
 		0, 0,
 		/* IP0_30_27 [4] */
@@ -4982,10 +4984,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0,
 		/* IP0_2_0 [3] */
 		FN_D0, FN_MSIOF3_SCK_B, FN_VI3_DATA0, FN_VI0_G4, FN_VI0_G4_B,
-		0, 0, 0, }
+		0, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR1", 0xE6060024, 32,
-			     2, 2, 2, 4, 4, 3, 3, 4, 4, 4) {
+			     GROUP(2, 2, 2, 4, 4, 3, 3, 4, 4, 4),
+			     GROUP(
 		/* IP1_31_30 [2] */
 		0, 0, 0, 0,
 		/* IP1_29_28 [2] */
@@ -5019,10 +5022,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP1_3_0 [4] */
 		FN_D9, FN_SCIFA1_RXD_C, FN_AVB_TXD1, 0,
 		FN_VI0_G1, FN_VI0_G1_B, FN_VI2_DATA1_VI2_B1,
-		0, 0, 0, 0, 0, 0, 0, 0, 0, }
+		0, 0, 0, 0, 0, 0, 0, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR2", 0xE6060028, 32,
-			     3, 3, 4, 4, 3, 3, 3, 3, 3, 3) {
+			     GROUP(3, 3, 4, 4, 3, 3, 3, 3, 3, 3),
+			     GROUP(
 		/* IP2_31_29 [3] */
 		0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP2_28_26 [3] */
@@ -5048,10 +5052,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP2_5_3 [3] */
 		FN_A3, FN_PWM6, FN_MSIOF1_SS2_B, 0, 0, 0, 0, 0,
 		/* IP2_2_0 [3] */
-		FN_A2, FN_PWM5, FN_MSIOF1_SS1_B, 0, 0, 0, 0, 0,	}
+		FN_A2, FN_PWM5, FN_MSIOF1_SS1_B, 0, 0, 0, 0, 0,	))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR3", 0xE606002C, 32,
-			     3, 3, 3, 3, 2, 3, 3, 4, 4, 4) {
+			     GROUP(3, 3, 3, 3, 2, 3, 3, 4, 4, 4),
+			     GROUP(
 		/* IP3_31_29 [3] */
 		FN_A20, FN_SPCLK, FN_VI1_R3, FN_VI1_R3_B, FN_VI2_G4,
 		0, 0, 0,
@@ -5081,10 +5086,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP3_3_0 [4] */
 		FN_A11, FN_SCIFB2_CTS_N_B, FN_MSIOF2_SCK, FN_VI1_R0,
 		FN_VI1_R0_B, FN_VI2_G0, FN_VI2_DATA3_VI2_B3_B, 0,
-		0, 0, 0, 0, 0, 0, 0, 0, }
+		0, 0, 0, 0, 0, 0, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR4", 0xE6060030, 32,
-			     2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3) {
+			     GROUP(2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3),
+			     GROUP(
 		/* IP4_31_30 [2] */
 		0, 0, 0, 0,
 		/* IP4_29_27 [3] */
@@ -5114,10 +5120,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_A22, FN_MISO_IO1, FN_VI1_R5, FN_VI1_R5_B, FN_VI2_G6, 0, 0, 0,
 		/* IP4_2_0 [3] */
 		FN_A21, FN_MOSI_IO0, FN_VI1_R4, FN_VI1_R4_B, FN_VI2_G5, 0, 0, 0,
-		}
+		))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR5", 0xE6060034, 32,
-			     2, 3, 3, 3, 3, 3, 2, 3, 4, 3, 3) {
+			     GROUP(2, 3, 3, 3, 3, 3, 2, 3, 4, 3, 3),
+			     GROUP(
 		/* IP5_31_30 [2] */
 		0, 0, 0, 0,
 		/* IP5_29_27 [3] */
@@ -5151,10 +5158,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_INTC_EN0_N, FN_I2C1_SCL,
 		/* IP5_2_0 [3] */
 		FN_EX_CS3_N, FN_GPS_MAG, FN_VI3_FIELD, FN_VI1_G1, FN_VI1_G1_B,
-		FN_VI2_R3, 0, 0, }
+		FN_VI2_R3, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR6", 0xE6060038, 32,
-			     3, 3, 3, 3, 3, 3, 3, 2, 3, 3, 3) {
+			     GROUP(3, 3, 3, 3, 3, 3, 3, 2, 3, 3, 3),
+			     GROUP(
 		/* IP6_31_29 [3] */
 		FN_ETH_REF_CLK, 0, FN_HCTS0_N_E,
 		FN_STP_IVCXO27_1_B, FN_HRX0_F, 0, 0, 0,
@@ -5187,10 +5195,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_SSI_SDATA7_C, FN_SSI_SCK78_B, 0, 0, 0,
 		/* IP6_2_0 [3] */
 		FN_DACK0, FN_IRQ0, FN_INTC_IRQ0_N, FN_SSI_SCK6_B,
-		FN_VI1_VSYNC_N, FN_VI1_VSYNC_N_B, FN_SSI_WS78_C, 0, }
+		FN_VI1_VSYNC_N, FN_VI1_VSYNC_N_B, FN_SSI_WS78_C, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR7", 0xE606003C, 32,
-			     1, 2, 2, 2, 3, 3, 3, 3, 3, 2, 2, 3, 3) {
+			     GROUP(1, 2, 2, 2, 3, 3, 3, 3, 3, 2, 2, 3, 3),
+			     GROUP(
 		/* IP7_31 [1] */
 		0, 0,
 		/* IP7_30_29 [2] */
@@ -5222,11 +5231,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_ETH_TXD1, 0, FN_HTX0_F, FN_BPFCLK_G, 0, 0, 0, 0,
 		/* IP7_2_0 [3] */
 		FN_ETH_MDIO, 0, FN_HRTS0_N_E,
-		FN_SIM0_D_C, FN_HCTS0_N_F, 0, 0, 0, }
+		FN_SIM0_D_C, FN_HCTS0_N_F, 0, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR8", 0xE6060040, 32,
-			     1, 2, 1, 1, 1, 2, 2, 2, 2, 2, 2,
-			     2, 2, 2, 2, 2, 2, 2) {
+			     GROUP(1, 2, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2,
+				   2, 2, 2, 2, 2, 2),
+			     GROUP(
 		/* IP8_31 [1] */
 		0, 0,
 		/* IP8_30_29 [2] */
@@ -5263,10 +5273,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP8_3_2 [2] */
 		FN_VI0_DATA2_VI0_B2, FN_ATAWR0_N, FN_AVB_RXD4, 0,
 		/* IP8_1_0 [2] */
-		FN_VI0_DATA1_VI0_B1, FN_ATARD0_N, FN_AVB_RXD3, 0, }
+		FN_VI0_DATA1_VI0_B1, FN_ATARD0_N, FN_AVB_RXD3, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR9", 0xE6060044, 32,
-			     4, 2, 2, 2, 2, 2, 2, 4, 4, 2, 2, 2, 2) {
+			     GROUP(4, 2, 2, 2, 2, 2, 2, 4, 4, 2, 2, 2, 2),
+			     GROUP(
 		/* IP9_31_28 [4] */
 		FN_SD1_CD, FN_MMC1_D6, FN_TS_SDEN1, FN_USB1_EXTP,
 		FN_GLO_SS, FN_VI0_CLK_B, FN_IIC2_SCL_D, FN_I2C2_SCL_D,
@@ -5298,10 +5309,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP9_3_2 [2] */
 		FN_SD0_DAT1, FN_SCIFB1_TXD_B, FN_VI1_DATA3_VI1_B3_B, 0,
 		/* IP9_1_0 [2] */
-		FN_SD0_DAT0, FN_SCIFB1_RXD_B, FN_VI1_DATA2_VI1_B2_B, 0, }
+		FN_SD0_DAT0, FN_SCIFB1_RXD_B, FN_VI1_DATA2_VI1_B2_B, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR10", 0xE6060048, 32,
-			     2, 4, 3, 4, 4, 4, 4, 3, 4) {
+			     GROUP(2, 4, 3, 4, 4, 4, 4, 3, 4),
+			     GROUP(
 		/* IP10_31_30 [2] */
 		0, 0, 0, 0,
 		/* IP10_29_26 [4] */
@@ -5337,10 +5349,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP10_3_0 [4] */
 		FN_SD1_WP, FN_MMC1_D7, FN_TS_SPSYNC1, FN_USB1_IDIN,
 		FN_GLO_RFON, FN_VI1_CLK_B, FN_IIC2_SDA_D, FN_I2C2_SDA_D,
-		FN_SIM0_D_B, 0, 0, 0, 0, 0, 0, 0, }
+		FN_SIM0_D_B, 0, 0, 0, 0, 0, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR11", 0xE606004C, 32,
-			     2, 3, 3, 2, 4, 3, 2, 2, 2, 2, 2, 1, 4) {
+			     GROUP(2, 3, 3, 2, 4, 3, 2, 2, 2, 2, 2, 1, 4),
+			     GROUP(
 		/* IP11_31_30 [2] */
 		FN_SSI_SCK0129, FN_CAN_CLK_B, FN_MOUT0, 0,
 		/* IP11_29_27 [3] */
@@ -5372,10 +5385,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP11_3_0 [4] */
 		FN_SD2_WP, FN_MMC0_D5, FN_TS_SCK0_B, FN_USB2_IDIN,
 		FN_GLO_I1, FN_VI0_DATA7_VI0_B7_B, FN_HRTS0_N_D,
-		FN_TS_SCK1_B, FN_GLO_I1_B, FN_VI3_DATA7_B, 0, 0, 0, 0, 0, 0, }
+		FN_TS_SCK1_B, FN_GLO_I1_B, FN_VI3_DATA7_B, 0, 0, 0, 0, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR12", 0xE6060050, 32,
-			     1, 3, 3, 2, 3, 3, 3, 3, 3, 2, 2, 2, 2) {
+			     GROUP(1, 3, 3, 2, 3, 3, 3, 3, 3, 2, 2, 2, 2),
+			     GROUP(
 		/* IP12_31 [1] */
 		0, 0,
 		/* IP12_30_28 [3] */
@@ -5411,10 +5425,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP12_3_2 [2] */
 		FN_SSI_SDATA0, FN_CAN0_RX_B, FN_MOUT2, 0,
 		/* IP12_1_0 [2] */
-		FN_SSI_WS0129, FN_CAN0_TX_B, FN_MOUT1, 0, }
+		FN_SSI_WS0129, FN_CAN0_TX_B, FN_MOUT1, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR13", 0xE6060054, 32,
-			     1, 2, 3, 3, 4, 3, 3, 3, 3, 4, 3) {
+			     GROUP(1, 2, 3, 3, 4, 3, 3, 3, 3, 4, 3),
+			     GROUP(
 		/* IP13_31 [1] */
 		0, 0,
 		/* IP13_30_29 [2] */
@@ -5447,10 +5462,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_BPFCLK_F, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP13_2_0 [3] */
 		FN_SSI_SDATA5, FN_SCIFB1_TXD, FN_IETX_B, FN_DU2_DR2,
-		FN_LCDOUT2, FN_CAN_DEBUGOUT5, 0, 0, }
+		FN_LCDOUT2, FN_CAN_DEBUGOUT5, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR14", 0xE6060058, 32,
-			     1, 3, 3, 3, 3, 3, 4, 3, 3, 3, 3) {
+			     GROUP(1, 3, 3, 3, 3, 3, 4, 3, 3, 3, 3),
+			     GROUP(
 		/* IP14_30 [1] */
 		0, 0,
 		/* IP14_30_28 [3] */
@@ -5485,10 +5501,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP14_2_0 [3] */
 		FN_AUDIO_CLKB, FN_SCIF_CLK, FN_CAN0_RX_D,
 		FN_DVC_MUTE, FN_CAN0_RX_C, FN_CAN_DEBUGOUT15,
-		FN_REMOCON, 0, }
+		FN_REMOCON, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR15", 0xE606005C, 32,
-			     2, 2, 2, 3, 3, 2, 2, 2, 2, 3, 3, 3, 3) {
+			     GROUP(2, 2, 2, 3, 3, 2, 2, 2, 2, 3, 3, 3, 3),
+			     GROUP(
 		/* IP15_31_30 [2] */
 		0, 0, 0, 0,
 		/* IP15_29_28 [2] */
@@ -5520,10 +5537,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_IIC2_SCL, FN_I2C2_SCL, 0,
 		/* IP15_2_0 [3] */
 		FN_SCIFA2_SCK, FN_FMCLK, FN_SCK2, FN_MSIOF3_SCK, FN_DU2_DG7,
-		FN_LCDOUT15, FN_SCIF_CLK_B, 0, }
+		FN_LCDOUT15, FN_SCIF_CLK_B, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR16", 0xE6060160, 32,
-			     4, 4, 4, 4, 4, 4, 1, 1, 3, 3) {
+			     GROUP(4, 4, 4, 4, 4, 4, 1, 1, 3, 3),
+			     GROUP(
 		/* IP16_31_28 [4] */
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -5551,11 +5569,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_ADICS_SAMP, FN_DU2_CDE, FN_QPOLB, FN_SCIFA2_RXD_B, 0,
 		/* IP16_2_0 [3] */
 		FN_MSIOF0_SS2, FN_AUDIO_CLKOUT, FN_ADICHS2,
-		FN_DU2_DISP, FN_QPOLA, FN_HTX0_C, FN_SCIFA2_TXD_B, 0, }
+		FN_DU2_DISP, FN_QPOLA, FN_HTX0_C, FN_SCIFA2_TXD_B, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("MOD_SEL", 0xE6060090, 32,
-			     3, 2, 2, 3, 2, 1, 1, 1, 2, 1,
-			     2, 1, 1, 1, 1, 2, 1, 1, 2, 1, 1) {
+			     GROUP(3, 2, 2, 3, 2, 1, 1, 1, 2, 1, 2, 1,
+				   1, 1, 1, 2, 1, 1, 2, 1, 1),
+			     GROUP(
 		/* SEL_SCIF1 [3] */
 		FN_SEL_SCIF1_0, FN_SEL_SCIF1_1, FN_SEL_SCIF1_2, FN_SEL_SCIF1_3,
 		FN_SEL_SCIF1_4, 0, 0, 0,
@@ -5601,11 +5620,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* SEL_SOF3 [1] */
 		FN_SEL_SOF3_0, FN_SEL_SOF3_1,
 		/* SEL_SOF0 [1] */
-		FN_SEL_SOF0_0, FN_SEL_SOF0_1, }
+		FN_SEL_SOF0_0, FN_SEL_SOF0_1, ))
 	},
 	{ PINMUX_CFG_REG_VAR("MOD_SEL2", 0xE6060094, 32,
-			     3, 1, 1, 1, 2, 1, 2, 1, 2,
-			     1, 1, 1, 3, 3, 2, 3, 2, 2) {
+			     GROUP(3, 1, 1, 1, 2, 1, 2, 1, 2, 1, 1, 1,
+				   3, 3, 2, 3, 2, 2),
+			     GROUP(
 		/* RESERVED [3] */
 		0, 0, 0, 0, 0, 0, 0, 0,
 		/* SEL_TMU1 [1] */
@@ -5643,11 +5663,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* SEL_SIM [2] */
 		FN_SEL_SIM_0, FN_SEL_SIM_1, FN_SEL_SIM_2, 0,
 		/* SEL_SSI8 [2] */
-		FN_SEL_SSI8_0, FN_SEL_SSI8_1, FN_SEL_SSI8_2, 0, }
+		FN_SEL_SSI8_0, FN_SEL_SSI8_1, FN_SEL_SSI8_2, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("MOD_SEL3", 0xE6060098, 32,
-			     1, 1, 2, 4, 4, 2, 2,
-			     4, 2, 3, 2, 3, 2) {
+			     GROUP(1, 1, 2, 4, 4, 2, 2, 4, 2, 3, 2, 3, 2),
+			     GROUP(
 		/* SEL_IICDVFS [1] */
 		FN_SEL_IICDVFS_0, FN_SEL_IICDVFS_1,
 		/* SEL_IIC0 [1] */
@@ -5678,7 +5698,7 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_SEL_I2C2_0, FN_SEL_I2C2_1, FN_SEL_I2C2_2, FN_SEL_I2C2_3,
 		FN_SEL_I2C2_4, 0, 0, 0,
 		/* SEL_I2C1 [2] */
-		FN_SEL_I2C1_0, FN_SEL_I2C1_1, FN_SEL_I2C1_2, 0, }
+		FN_SEL_I2C1_0, FN_SEL_I2C1_1, FN_SEL_I2C1_2, 0, ))
 	},
 	{ },
 };
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a7791.c b/drivers/pinctrl/sh-pfc/pfc-r8a7791.c
index d8b13d4e9bbf..1292ec8d268f 100644
--- a/drivers/pinctrl/sh-pfc/pfc-r8a7791.c
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a7791.c
@@ -6,6 +6,7 @@
  * Copyright (C) 2014-2017 Cogent Embedded, Inc.
  */
 
+#include <linux/errno.h>
 #include <linux/kernel.h>
 
 #include "sh_pfc.h"
@@ -5427,7 +5428,7 @@ static const struct {
 };
 
 static const struct pinmux_cfg_reg pinmux_config_regs[] = {
-	{ PINMUX_CFG_REG("GPSR0", 0xE6060004, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR0", 0xE6060004, 32, 1, GROUP(
 		GP_0_31_FN, FN_IP1_22_20,
 		GP_0_30_FN, FN_IP1_19_17,
 		GP_0_29_FN, FN_IP1_16_14,
@@ -5459,9 +5460,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_0_3_FN, FN_IP0_3,
 		GP_0_2_FN, FN_IP0_2,
 		GP_0_1_FN, FN_IP0_1,
-		GP_0_0_FN, FN_IP0_0, }
+		GP_0_0_FN, FN_IP0_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR1", 0xE6060008, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR1", 0xE6060008, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -5493,9 +5494,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_1_3_FN, FN_IP2_2_0,
 		GP_1_2_FN, FN_IP1_31_29,
 		GP_1_1_FN, FN_IP1_28_26,
-		GP_1_0_FN, FN_IP1_25_23, }
+		GP_1_0_FN, FN_IP1_25_23, ))
 	},
-	{ PINMUX_CFG_REG("GPSR2", 0xE606000C, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR2", 0xE606000C, 32, 1, GROUP(
 		GP_2_31_FN, FN_IP6_7_6,
 		GP_2_30_FN, FN_IP6_5_3,
 		GP_2_29_FN, FN_IP6_2_0,
@@ -5527,9 +5528,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_2_3_FN, FN_IP4_4_2,
 		GP_2_2_FN, FN_IP4_1_0,
 		GP_2_1_FN, FN_IP3_30_28,
-		GP_2_0_FN, FN_IP3_27_25 }
+		GP_2_0_FN, FN_IP3_27_25 ))
 	},
-	{ PINMUX_CFG_REG("GPSR3", 0xE6060010, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR3", 0xE6060010, 32, 1, GROUP(
 		GP_3_31_FN, FN_IP9_18_17,
 		GP_3_30_FN, FN_IP9_16,
 		GP_3_29_FN, FN_IP9_15_13,
@@ -5561,9 +5562,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_3_3_FN, FN_IP7_12_11,
 		GP_3_2_FN, FN_IP7_10_9,
 		GP_3_1_FN, FN_IP7_8_6,
-		GP_3_0_FN, FN_IP7_5_3 }
+		GP_3_0_FN, FN_IP7_5_3 ))
 	},
-	{ PINMUX_CFG_REG("GPSR4", 0xE6060014, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR4", 0xE6060014, 32, 1, GROUP(
 		GP_4_31_FN, FN_IP15_5_4,
 		GP_4_30_FN, FN_IP15_3_2,
 		GP_4_29_FN, FN_IP15_1_0,
@@ -5595,9 +5596,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_4_3_FN, FN_IP9_24_23,
 		GP_4_2_FN, FN_IP9_22_21,
 		GP_4_1_FN, FN_IP9_20_19,
-		GP_4_0_FN, FN_VI0_CLK }
+		GP_4_0_FN, FN_VI0_CLK ))
 	},
-	{ PINMUX_CFG_REG("GPSR5", 0xE6060018, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR5", 0xE6060018, 32, 1, GROUP(
 		GP_5_31_FN, FN_IP3_24_22,
 		GP_5_30_FN, FN_IP13_9_7,
 		GP_5_29_FN, FN_IP13_6_5,
@@ -5629,9 +5630,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_5_3_FN, FN_IP11_18_17,
 		GP_5_2_FN, FN_IP11_16_15,
 		GP_5_1_FN, FN_IP11_14_12,
-		GP_5_0_FN, FN_IP11_11_9 }
+		GP_5_0_FN, FN_IP11_11_9 ))
 	},
-	{ PINMUX_CFG_REG("GPSR6", 0xE606001C, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR6", 0xE606001C, 32, 1, GROUP(
 		GP_6_31_FN, FN_DU0_DOTCLKIN,
 		GP_6_30_FN, FN_USB1_OVC,
 		GP_6_29_FN, FN_IP14_31_29,
@@ -5663,9 +5664,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_6_3_FN, FN_IP13_13,
 		GP_6_2_FN, FN_IP13_12,
 		GP_6_1_FN, FN_IP13_11,
-		GP_6_0_FN, FN_IP13_10 }
+		GP_6_0_FN, FN_IP13_10 ))
 	},
-	{ PINMUX_CFG_REG("GPSR7", 0xE6060074, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR7", 0xE6060074, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -5697,11 +5698,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_7_3_FN, FN_IP15_26_24,
 		GP_7_2_FN, FN_IP15_23_21,
 		GP_7_1_FN, FN_IP15_20_18,
-		GP_7_0_FN, FN_IP15_17_15 }
+		GP_7_0_FN, FN_IP15_17_15 ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR0", 0xE6060020, 32,
-			     1, 2, 2, 2, 2, 2, 2, 3, 1, 1, 1, 1, 1, 1, 1, 1,
-			     1, 1, 1, 1, 1, 1, 1, 1) {
+			     GROUP(1, 2, 2, 2, 2, 2, 2, 3, 1, 1, 1, 1,
+				   1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1),
+			     GROUP(
 		/* IP0_31 [1] */
 		0, 0,
 		/* IP0_30_29 [2] */
@@ -5756,10 +5758,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP0_1 [1] */
 		FN_D1, 0,
 		/* IP0_0 [1] */
-		FN_D0, 0, }
+		FN_D0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR1", 0xE6060024, 32,
-			     3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2) {
+			     GROUP(3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2),
+			     GROUP(
 		/* IP1_31_29 [3] */
 		FN_A18, FN_DREQ1, FN_SCIFA1_RXD_C, 0, FN_SCIFB1_RXD_C,
 		0, 0, 0,
@@ -5792,10 +5795,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_A8, FN_MSIOF1_SS1, FN_I2C0_SCL, 0,
 		/* IP1_1_0 [2] */
 		FN_A7, FN_MSIOF1_SYNC,
-		0, 0, }
+		0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR2", 0xE6060028, 32,
-			     2, 3, 2, 2, 2, 2, 3, 3, 3, 3, 2, 2, 3) {
+			     GROUP(2, 3, 2, 2, 2, 2, 3, 3, 3, 3, 2, 2, 3),
+			     GROUP(
 		/* IP2_31_30 [2] */
 		0, 0, 0, 0,
 		/* IP2_29_27 [3] */
@@ -5828,10 +5832,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_A20, FN_SPCLK, 0, 0,
 		/* IP2_2_0 [3] */
 		FN_A19, FN_DACK1, FN_SCIFA1_TXD_C, 0,
-		FN_SCIFB1_TXD_C, 0, FN_SCIFB1_SCK_B, 0, }
+		FN_SCIFB1_TXD_C, 0, FN_SCIFB1_SCK_B, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR3", 0xE606002C, 32,
-			     1, 3, 3, 3, 2, 2, 2, 2, 2, 3, 3, 3, 3) {
+			     GROUP(1, 3, 3, 3, 2, 2, 2, 2, 2, 3, 3, 3, 3),
+			     GROUP(
 		/* IP3_31 [1] */
 		0, 0,
 		/* IP3_30_28 [3] */
@@ -5866,10 +5871,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_SCIFB1_RXD_B, FN_PWM1, FN_TPU_TO1, 0,
 		/* IP3_2_0 [3] */
 		FN_EX_CS4_N, FN_ATARD0_N, FN_MSIOF2_RXD, 0, FN_EX_WAIT2,
-		0, 0, 0, }
+		0, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR4", 0xE6060030, 32,
-			     1, 3, 2, 2, 2, 1, 1, 1, 3, 3, 3, 2, 3, 3, 2) {
+			     GROUP(1, 3, 2, 2, 2, 1, 1, 1, 3, 3, 3, 2,
+				   3, 3, 2),
+			     GROUP(
 		/* IP4_31 [1] */
 		0, 0,
 		/* IP4_30_28 [3] */
@@ -5908,10 +5915,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_MSIOF2_SYNC_C, FN_GLO_I0_D,
 		0, 0, 0,
 		/* IP4_1_0 [2] */
-		FN_SSI_SDATA0, FN_I2C0_SCL_B, FN_IIC0_SCL_B, FN_MSIOF2_SCK_C, }
+		FN_SSI_SDATA0, FN_I2C0_SCL_B, FN_IIC0_SCL_B, FN_MSIOF2_SCK_C,
+		))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR5", 0xE6060034, 32,
-			     3, 3, 2, 2, 2, 3, 2, 3, 3, 3, 3, 3) {
+			     GROUP(3, 3, 2, 2, 2, 3, 2, 3, 3, 3, 3, 3),
+			     GROUP(
 		/* IP5_31_29 [3] */
 		FN_SSI_SDATA9, FN_RX3_D, FN_CAN0_RX_D,
 		0, 0, 0, 0, 0,
@@ -5946,10 +5955,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP5_2_0 [3] */
 		FN_SSI_WS5, FN_MSIOF1_SYNC_C, FN_TS_SCK0, FN_GLO_I1,
 		FN_MSIOF2_TXD_D, FN_VI1_R3_B,
-		0, 0, }
+		0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR6", 0xE6060038, 32,
-			     2, 3, 3, 3, 2, 3, 2, 2, 2, 2, 2, 3, 3) {
+			     GROUP(2, 3, 3, 3, 2, 3, 2, 2, 2, 2, 2, 3, 3),
+			     GROUP(
 		/* IP6_31_30 [2] */
 		0, 0, 0, 0,
 		/* IP6_29_27 [3] */
@@ -5986,10 +5996,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP6_2_0 [3] */
 		FN_AUDIO_CLKB, FN_STP_OPWM_0_B, FN_MSIOF1_SCK_B,
 		FN_SCIF_CLK, FN_DVC_MUTE, FN_BPFCLK_E,
-		0, 0, }
+		0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR7", 0xE606003C, 32,
-			     2, 3, 3, 3, 2, 2, 2, 2, 2, 2, 3, 3, 3) {
+			     GROUP(2, 3, 3, 3, 2, 2, 2, 2, 2, 2, 3, 3, 3),
+			     GROUP(
 		/* IP7_31_30 [2] */
 		0, 0, 0, 0,
 		/* IP7_29_27 [3] */
@@ -6027,10 +6038,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP7_2_0 [3] */
 		FN_IRQ9, FN_DU1_DOTCLKIN_B, FN_CAN_CLK_D, FN_GPS_MAG_C,
 		FN_SCIF_CLK_B, FN_GPS_MAG_D,
-		0, 0, }
+		0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR8", 0xE6060040, 32,
-			     1, 3, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3) {
+			     GROUP(1, 3, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3),
+			     GROUP(
 		/* IP8_31 [1] */
 		0, 0,
 		/* IP8_30_28 [3] */
@@ -6070,10 +6082,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0,
 		/* IP8_2_0 [3] */
 		FN_DU1_DG3, FN_LCDOUT11, FN_VI1_DATA5_B, 0, FN_SSI_WS78_B,
-		0, 0, 0, }
+		0, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR9", 0xE6060044, 32,
-			     3, 2, 2, 2, 2, 2, 2, 1, 3, 1, 1, 3, 1, 1, 3, 3) {
+			     GROUP(3, 2, 2, 2, 2, 2, 2, 1, 3, 1, 1, 3,
+				   1, 1, 3, 3),
+			     GROUP(
 		/* IP9_31_29 [3] */
 		FN_VI0_G0, FN_IIC1_SCL, FN_STP_IVCXO27_0_C, FN_I2C4_SCL,
 		FN_HCTS2_N, FN_SCIFB2_CTS_N, FN_ATAWR1_N, 0,
@@ -6113,10 +6127,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0,
 		/* IP9_2_0 [3] */
 		FN_DU1_DB6, FN_LCDOUT22, FN_I2C3_SCL_C, FN_RX3, FN_SCIFA3_RXD,
-		0, 0, 0, }
+		0, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR10", 0xE6060048, 32,
-			     3, 2, 2, 3, 3, 2, 2, 3, 3, 3, 3, 3) {
+			     GROUP(3, 2, 2, 3, 3, 2, 2, 3, 3, 3, 3, 3),
+			     GROUP(
 		/* IP10_31_29 [3] */
 		FN_VI0_R4, FN_VI2_DATA5, FN_GLO_SCLK_B, FN_TX0_C, FN_I2C1_SCL_D,
 		0, 0, 0,
@@ -6150,11 +6165,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_HSCK2, FN_SCIFB2_SCK, FN_ATARD1_N, 0,
 		/* IP10_2_0 [3] */
 		FN_VI0_G1, FN_IIC1_SDA, FN_STP_ISCLK_0_C, FN_I2C4_SDA,
-		FN_HRTS2_N, FN_SCIFB2_RTS_N, FN_ATADIR1_N, 0, }
+		FN_HRTS2_N, FN_SCIFB2_RTS_N, FN_ATADIR1_N, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR11", 0xE606004C, 32,
-			     2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2,
-			     3, 3, 3, 3, 3) {
+			     GROUP(2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2,
+				   2, 3, 3, 3, 3, 3),
+			     GROUP(
 		/* IP11_31_30 [2] */
 		FN_ETH_CRS_DV, FN_AVB_LINK, FN_I2C2_SDA_C, 0,
 		/* IP11_29_28 [2] */
@@ -6197,10 +6213,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0,
 		/* IP11_2_0 [3] */
 		FN_VI0_R5, FN_VI2_DATA6, FN_GLO_SDATA_B, FN_RX0_C,
-		FN_I2C1_SDA_D, 0, 0, 0, }
+		FN_I2C1_SDA_D, 0, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR12", 0xE6060050, 32,
-			     2, 3, 3, 2, 2, 2, 2, 3, 3, 3, 3, 2, 2) {
+			     GROUP(2, 3, 3, 2, 2, 2, 2, 3, 3, 3, 3, 2, 2),
+			     GROUP(
 		/* IP12_31_30 [2] */
 		0, 0, 0, 0,
 		/* IP12_29_27 [3] */
@@ -6238,11 +6255,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP12_3_2 [2] */
 		FN_ETH_RXD0, FN_AVB_PHY_INT, FN_I2C3_SDA, FN_IIC0_SDA,
 		/* IP12_1_0 [2] */
-		FN_ETH_RX_ER, FN_AVB_CRS, FN_I2C3_SCL, FN_IIC0_SCL, }
+		FN_ETH_RX_ER, FN_AVB_CRS, FN_I2C3_SCL, FN_IIC0_SCL, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR13", 0xE6060054, 32,
-			     1, 3, 1, 1, 1, 2, 1, 3, 3, 1, 1, 1, 1, 1, 1,
-			     3, 2, 2, 3) {
+			     GROUP(1, 3, 1, 1, 1, 2, 1, 3, 3, 1, 1, 1,
+				   1, 1, 1, 3, 2, 2, 3),
+			     GROUP(
 		/* IP13_31 [1] */
 		0, 0,
 		/* IP13_30_28 [3] */
@@ -6289,10 +6307,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP13_2_0 [3] */
 		FN_STP_ISD_0, FN_AVB_TX_ER, FN_SCIFB2_SCK_C,
 		FN_ADICLK_B, FN_MSIOF0_SS1_C,
-		0, 0, 0, }
+		0, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR14", 0xE6060058, 32,
-			     3, 3, 3, 3, 3, 3, 3, 3, 1, 1, 1, 1, 1, 1, 2) {
+			     GROUP(3, 3, 3, 3, 3, 3, 3, 3, 1, 1, 1, 1,
+				   1, 1, 2),
+			     GROUP(
 		/* IP14_31_29 [3] */
 		FN_MSIOF0_SS2, FN_MMC_D7, FN_ADICHS2, FN_RX0_E,
 		FN_VI1_VSYNC_N_C, FN_IIC0_SDA_C, FN_VI1_G5_B, 0,
@@ -6332,10 +6352,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP14_2 [1] */
 		FN_SD2_CLK, FN_MMC_CLK,
 		/* IP14_1_0 [2] */
-		FN_SD1_WP, FN_PWM1_B, FN_I2C1_SDA_C, 0, }
+		FN_SD1_WP, FN_PWM1_B, FN_I2C1_SDA_C, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR15", 0xE606005C, 32,
-			     2, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2) {
+			     GROUP(2, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2),
+			     GROUP(
 		/* IP15_31_30 [2] */
 		0, 0, 0, 0,
 		/* IP15_29_27 [3] */
@@ -6373,10 +6394,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP15_3_2 [2] */
 		FN_SIM0_CLK, FN_IECLK, FN_CAN_CLK_C, 0,
 		/* IP15_1_0 [2] */
-		FN_SIM0_RST, FN_IETX, FN_CAN1_TX_D, 0, }
+		FN_SIM0_RST, FN_IETX, FN_CAN1_TX_D, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR16", 0xE6060160, 32,
-			     4, 4, 4, 4, 4, 2, 2, 2, 3, 3) {
+			     GROUP(4, 4, 4, 4, 4, 2, 2, 2, 3, 3),
+			     GROUP(
 		/* IP16_31_28 [4] */
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -6405,11 +6427,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP16_2_0 [3] */
 		FN_HRX1, FN_SCIFB1_RXD, FN_VI1_R0_B,
 		FN_GLO_SDATA_C, FN_VI1_DATA6_C,
-		0, 0, 0, }
+		0, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("MOD_SEL", 0xE6060090, 32,
-			     1, 2, 2, 2, 3, 2, 1, 1, 1, 1,
-			     3, 2, 2, 2, 1, 2, 2, 2) {
+			     GROUP(1, 2, 2, 2, 3, 2, 1, 1, 1, 1, 3, 2,
+				   2, 2, 1, 2, 2, 2),
+			     GROUP(
 		/* RESERVED [1] */
 		0, 0,
 		/* SEL_SCIF1 [2] */
@@ -6450,11 +6473,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* SEL_TSIF0 [2] */
 		FN_SEL_TSIF0_0, FN_SEL_TSIF0_1, FN_SEL_TSIF0_2, FN_SEL_TSIF0_3,
 		/* SEL_SOF0 [2] */
-		FN_SEL_SOF0_0, FN_SEL_SOF0_1, FN_SEL_SOF0_2, 0, }
+		FN_SEL_SOF0_0, FN_SEL_SOF0_1, FN_SEL_SOF0_2, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("MOD_SEL2", 0xE6060094, 32,
-			     3, 1, 1, 3, 2, 1, 1, 2, 2,
-			     1, 3, 2, 1, 2, 2, 2, 1, 1, 1) {
+			     GROUP(3, 1, 1, 3, 2, 1, 1, 2, 2, 1, 3, 2,
+				   1, 2, 2, 2, 1, 1, 1),
+			     GROUP(
 		/* SEL_SCIF0 [3] */
 		FN_SEL_SCIF0_0, FN_SEL_SCIF0_1, FN_SEL_SCIF0_2,
 		FN_SEL_SCIF0_3, FN_SEL_SCIF0_4,
@@ -6498,11 +6522,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* RESERVED [1] */
 		0, 0,
 		/* SEL_SSI8 [1] */
-		FN_SEL_SSI8_0, FN_SEL_SSI8_1, }
+		FN_SEL_SSI8_0, FN_SEL_SSI8_1, ))
 	},
 	{ PINMUX_CFG_REG_VAR("MOD_SEL3", 0xE6060098, 32,
-			     2, 2, 2, 2, 2, 2, 2, 2,
-			     1, 1, 2, 2, 3, 2, 2, 2, 1) {
+			     GROUP(2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 2, 2,
+				   3, 2, 2, 2, 1),
+			     GROUP(
 		/* SEL_HSCIF2 [2] */
 		FN_SEL_HSCIF2_0, FN_SEL_HSCIF2_1,
 		FN_SEL_HSCIF2_2, FN_SEL_HSCIF2_3,
@@ -6540,11 +6565,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* RESERVED [2] */
 		0, 0, 0, 0,
 		/* RESERVED [1] */
-		0, 0, }
+		0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("MOD_SEL4", 0xE606009C, 32,
-			     3, 2, 2, 1, 1, 1, 1, 3, 2,
-			     2, 3, 1, 1, 1, 2, 2, 2, 2) {
+			     GROUP(3, 2, 2, 1, 1, 1, 1, 3, 2, 2, 3, 1,
+				   1, 1, 2, 2, 2, 2),
+			     GROUP(
 		/* SEL_SOF1 [3] */
 		FN_SEL_SOF1_0, FN_SEL_SOF1_1, FN_SEL_SOF1_2, FN_SEL_SOF1_3,
 		FN_SEL_SOF1_4,
@@ -6586,7 +6612,7 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* RESERVED [2] */
 		0, 0, 0, 0,
 		/* RESERVED [2] */
-		0, 0, 0, 0, }
+		0, 0, 0, 0, ))
 	},
 	{ },
 };
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a7792.c b/drivers/pinctrl/sh-pfc/pfc-r8a7792.c
index d36da5652de6..bbace1478613 100644
--- a/drivers/pinctrl/sh-pfc/pfc-r8a7792.c
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a7792.c
@@ -1988,7 +1988,7 @@ static const struct sh_pfc_function pinmux_functions[] = {
 };
 
 static const struct pinmux_cfg_reg pinmux_config_regs[] = {
-	{ PINMUX_CFG_REG("GPSR0", 0xE6060004, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR0", 0xE6060004, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -2020,9 +2020,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_0_3_FN, FN_IP0_3,
 		GP_0_2_FN, FN_IP0_2,
 		GP_0_1_FN, FN_IP0_1,
-		GP_0_0_FN, FN_IP0_0 }
+		GP_0_0_FN, FN_IP0_0 ))
 	},
-	{ PINMUX_CFG_REG("GPSR1", 0xE6060008, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR1", 0xE6060008, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -2054,9 +2054,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_1_3_FN, FN_IP1_8,
 		GP_1_2_FN, FN_IP1_7,
 		GP_1_1_FN, FN_IP1_6,
-		GP_1_0_FN, FN_IP1_5, }
+		GP_1_0_FN, FN_IP1_5, ))
 	},
-	{ PINMUX_CFG_REG("GPSR2", 0xE606000C, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR2", 0xE606000C, 32, 1, GROUP(
 		GP_2_31_FN, FN_A15,
 		GP_2_30_FN, FN_A14,
 		GP_2_29_FN, FN_A13,
@@ -2088,9 +2088,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_2_3_FN, FN_D3,
 		GP_2_2_FN, FN_D2,
 		GP_2_1_FN, FN_D1,
-		GP_2_0_FN, FN_D0 }
+		GP_2_0_FN, FN_D0 ))
 	},
-	{ PINMUX_CFG_REG("GPSR3", 0xE6060010, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR3", 0xE6060010, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -2122,9 +2122,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_3_3_FN, FN_A19,
 		GP_3_2_FN, FN_A18,
 		GP_3_1_FN, FN_A17,
-		GP_3_0_FN, FN_A16 }
+		GP_3_0_FN, FN_A16 ))
 	},
-	{ PINMUX_CFG_REG("GPSR4", 0xE6060014, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR4", 0xE6060014, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -2156,9 +2156,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_4_3_FN, FN_VI0_VSYNC_N,
 		GP_4_2_FN, FN_VI0_HSYNC_N,
 		GP_4_1_FN, FN_VI0_CLKENB,
-		GP_4_0_FN, FN_VI0_CLK }
+		GP_4_0_FN, FN_VI0_CLK ))
 	},
-	{ PINMUX_CFG_REG("GPSR5", 0xE6060018, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR5", 0xE6060018, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -2190,9 +2190,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_5_3_FN, FN_VI1_VSYNC_N,
 		GP_5_2_FN, FN_VI1_HSYNC_N,
 		GP_5_1_FN, FN_VI1_CLKENB,
-		GP_5_0_FN, FN_VI1_CLK }
+		GP_5_0_FN, FN_VI1_CLK ))
 	},
-	{ PINMUX_CFG_REG("GPSR6", 0xE606001C, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR6", 0xE606001C, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -2224,9 +2224,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_6_3_FN, FN_IP2_3,
 		GP_6_2_FN, FN_IP2_2,
 		GP_6_1_FN, FN_IP2_1,
-		GP_6_0_FN, FN_IP2_0 }
+		GP_6_0_FN, FN_IP2_0 ))
 	},
-	{ PINMUX_CFG_REG("GPSR7", 0xE6060020, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR7", 0xE6060020, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -2258,9 +2258,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_7_3_FN, FN_IP3_3,
 		GP_7_2_FN, FN_IP3_2,
 		GP_7_1_FN, FN_IP3_1,
-		GP_7_0_FN, FN_IP3_0 }
+		GP_7_0_FN, FN_IP3_0 ))
 	},
-	{ PINMUX_CFG_REG("GPSR8", 0xE6060024, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR8", 0xE6060024, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -2292,9 +2292,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_8_3_FN, FN_IP4_3_2,
 		GP_8_2_FN, FN_IP4_1,
 		GP_8_1_FN, FN_IP4_0,
-		GP_8_0_FN, FN_VI4_CLK }
+		GP_8_0_FN, FN_VI4_CLK ))
 	},
-	{ PINMUX_CFG_REG("GPSR9", 0xE6060028, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR9", 0xE6060028, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -2326,9 +2326,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_9_3_FN, FN_IP5_2,
 		GP_9_2_FN, FN_IP5_1,
 		GP_9_1_FN, FN_IP5_0,
-		GP_9_0_FN, FN_VI5_CLK }
+		GP_9_0_FN, FN_VI5_CLK ))
 	},
-	{ PINMUX_CFG_REG("GPSR10", 0xE606002C, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR10", 0xE606002C, 32, 1, GROUP(
 		GP_10_31_FN, FN_CAN1_RX,
 		GP_10_30_FN, FN_CAN1_TX,
 		GP_10_29_FN, FN_CAN_CLK,
@@ -2360,9 +2360,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_10_3_FN, FN_IP6_2,
 		GP_10_2_FN, FN_HRTS0_N,
 		GP_10_1_FN, FN_IP6_1,
-		GP_10_0_FN, FN_IP6_0 }
+		GP_10_0_FN, FN_IP6_0 ))
 	},
-	{ PINMUX_CFG_REG("GPSR11", 0xE6060030, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR11", 0xE6060030, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		GP_11_29_FN, FN_AVS2,
@@ -2394,13 +2394,14 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_11_3_FN, FN_IP7_6,
 		GP_11_2_FN, FN_IP7_5_4,
 		GP_11_1_FN, FN_IP7_3_2,
-		GP_11_0_FN, FN_IP7_1_0 }
+		GP_11_0_FN, FN_IP7_1_0 ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR0", 0xE6060040, 32,
-			     4, 4,
-			     1, 1, 1, 1, 1, 1, 1, 1,
-			     1, 1, 1, 1, 1, 1, 1, 1,
-			     1, 1, 1, 1, 1, 1, 1, 1) {
+			     GROUP(4, 4,
+				   1, 1, 1, 1, 1, 1, 1, 1,
+				   1, 1, 1, 1, 1, 1, 1, 1,
+				   1, 1, 1, 1, 1, 1, 1, 1),
+			     GROUP(
 		/* IP0_31_28 [4] */
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP0_27_24 [4] */
@@ -2452,13 +2453,14 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP0_1 [1] */
 		FN_DU0_DR1_DATA1, 0,
 		/* IP0_0 [1] */
-		FN_DU0_DR0_DATA0, 0 }
+		FN_DU0_DR0_DATA0, 0 ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR1", 0xE6060044, 32,
-			     4, 4,
-			     1, 1, 1, 1, 1, 1, 1, 1,
-			     1, 1, 1, 1, 1, 1, 1, 1,
-			     1, 1, 1, 1, 1, 1, 1, 1) {
+			     GROUP(4, 4,
+				   1, 1, 1, 1, 1, 1, 1, 1,
+				   1, 1, 1, 1, 1, 1, 1, 1,
+				   1, 1, 1, 1, 1, 1, 1, 1),
+			     GROUP(
 		/* IP1_31_28 [4] */
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP1_27_24 [4] */
@@ -2510,13 +2512,14 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP1_1 [1] */
 		FN_DU0_EXVSYNC_DU0_VSYNC, 0,
 		/* IP1_0 [1] */
-		FN_DU0_EXHSYNC_DU0_HSYNC, 0 }
+		FN_DU0_EXHSYNC_DU0_HSYNC, 0 ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR2", 0xE6060048, 32,
-			     4, 4,
-			     4, 3, 1,
-			     1, 1, 1, 1, 1, 1, 1, 1,
-			     1, 1, 1, 1, 1, 1, 1, 1) {
+			     GROUP(4, 4,
+				   4, 3, 1,
+				   1, 1, 1, 1, 1, 1, 1, 1,
+				   1, 1, 1, 1, 1, 1, 1, 1),
+			     GROUP(
 		/* IP2_31_28 [4] */
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP2_27_24 [4] */
@@ -2558,13 +2561,14 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP2_1 [1] */
 		FN_VI2_CLKENB, FN_AVB_RX_DV,
 		/* IP2_0 [1] */
-		FN_VI2_CLK, FN_AVB_RX_CLK }
+		FN_VI2_CLK, FN_AVB_RX_CLK ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR3", 0xE606004C, 32,
-			     4, 4,
-			     4, 4,
-			     1, 1, 1, 1, 1, 1, 1, 1,
-			     1, 1, 1, 1, 1, 1, 1, 1) {
+			     GROUP(4, 4,
+				   4, 4,
+				   1, 1, 1, 1, 1, 1, 1, 1,
+				   1, 1, 1, 1, 1, 1, 1, 1),
+			     GROUP(
 		/* IP3_31_28 [4] */
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP3_27_24 [4] */
@@ -2604,12 +2608,13 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP3_1 [1] */
 		FN_VI3_CLKENB, FN_AVB_TXD4,
 		/* IP3_0 [1] */
-		FN_VI3_CLK, FN_AVB_TX_CLK }
+		FN_VI3_CLK, FN_AVB_TX_CLK ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR4", 0xE6060050, 32,
-			     4, 3, 1,
-			     1, 1, 1, 2, 2, 2,
-			     2, 2, 2, 2, 2, 1, 2, 1, 1) {
+			     GROUP(4, 3, 1,
+				   1, 1, 1, 2, 2, 2,
+				   2, 2, 2, 2, 2, 1, 2, 1, 1),
+			     GROUP(
 		/* IP4_31_28 [4] */
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP4_27_25 [3] */
@@ -2645,13 +2650,14 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP4_1 [1] */
 		FN_VI4_HSYNC_N, FN_VI0_D13_G5_Y5,
 		/* IP4_0 [1] */
-		FN_VI4_CLKENB, FN_VI0_D12_G4_Y4 }
+		FN_VI4_CLKENB, FN_VI0_D12_G4_Y4 ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR5", 0xE6060054, 32,
-			     4, 4,
-			     4, 4,
-			     4, 1, 1, 1, 1,
-			     1, 1, 1, 1, 1, 1, 1, 1) {
+			     GROUP(4, 4,
+				   4, 4,
+				   4, 1, 1, 1, 1,
+				   1, 1, 1, 1, 1, 1, 1, 1),
+			     GROUP(
 		/* IP5_31_28 [4] */
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP5_27_24 [4] */
@@ -2685,13 +2691,14 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP5_1 [1] */
 		FN_VI5_HSYNC_N, FN_VI1_D13_G5_Y5_B,
 		/* IP5_0 [1] */
-		FN_VI5_CLKENB, FN_VI1_D12_G4_Y4_B }
+		FN_VI5_CLKENB, FN_VI1_D12_G4_Y4_B ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR6", 0xE6060058, 32,
-			     4, 4,
-			     4, 1, 2, 1,
-			     2, 2, 2, 2,
-			     1, 1, 1, 1, 1, 1, 1, 1) {
+			     GROUP(4, 4,
+				   4, 1, 2, 1,
+				   2, 2, 2, 2,
+				   1, 1, 1, 1, 1, 1, 1, 1),
+			     GROUP(
 		/* IP6_31_28 [4] */
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP6_27_24 [4] */
@@ -2727,13 +2734,14 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP6_1 [1] */
 		FN_MSIOF0_SYNC, FN_HCTS0_N,
 		/* IP6_0 [1] */
-		FN_MSIOF0_SCK, FN_HSCK0 }
+		FN_MSIOF0_SCK, FN_HSCK0 ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR7", 0xE606005C, 32,
-			     4, 4,
-			     3, 1, 1, 1, 1, 1,
-			     2, 2, 2, 2,
-			     1, 1, 2, 2, 2) {
+			     GROUP(4, 4,
+				   3, 1, 1, 1, 1, 1,
+				   2, 2, 2, 2,
+				   1, 1, 2, 2, 2),
+			     GROUP(
 		/* IP7_31_28 [4] */
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP7_27_24 [4] */
@@ -2767,7 +2775,7 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP7_3_2 [2] */
 		FN_PWM1, FN_TCLK2, FN_FSO_CFE_1, 0,
 		/* IP7_1_0 [2] */
-		FN_PWM0, FN_TCLK1, FN_FSO_CFE_0, 0 }
+		FN_PWM0, FN_TCLK1, FN_FSO_CFE_0, 0 ))
 	},
 	{ },
 };
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a7794.c b/drivers/pinctrl/sh-pfc/pfc-r8a7794.c
index 958a5f714c93..1ff4969d8381 100644
--- a/drivers/pinctrl/sh-pfc/pfc-r8a7794.c
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a7794.c
@@ -7,6 +7,7 @@
  * Copyright (C) 2015-2017 Cogent Embedded, Inc. <source@cogentembedded.com>
  */
 
+#include <linux/errno.h>
 #include <linux/kernel.h>
 #include <linux/sys_soc.h>
 
@@ -4617,7 +4618,7 @@ static const struct sh_pfc_function pinmux_functions[] = {
 };
 
 static const struct pinmux_cfg_reg pinmux_config_regs[] = {
-	{ PINMUX_CFG_REG("GPSR0", 0xE6060004, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR0", 0xE6060004, 32, 1, GROUP(
 		GP_0_31_FN, FN_IP2_17_16,
 		GP_0_30_FN, FN_IP2_15_14,
 		GP_0_29_FN, FN_IP2_13_12,
@@ -4649,9 +4650,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_0_3_FN, FN_IP0_27_26,
 		GP_0_2_FN, FN_IP0_25,
 		GP_0_1_FN, FN_IP0_24,
-		GP_0_0_FN, FN_IP0_23_22, }
+		GP_0_0_FN, FN_IP0_23_22, ))
 	},
-	{ PINMUX_CFG_REG("GPSR1", 0xE6060008, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR1", 0xE6060008, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -4683,9 +4684,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_1_3_FN, FN_IP2_29_27,
 		GP_1_2_FN, FN_IP2_26_24,
 		GP_1_1_FN, FN_IP2_23_21,
-		GP_1_0_FN, FN_IP2_20_18, }
+		GP_1_0_FN, FN_IP2_20_18, ))
 	},
-	{ PINMUX_CFG_REG("GPSR2", 0xE606000C, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR2", 0xE606000C, 32, 1, GROUP(
 		GP_2_31_FN, FN_IP6_7_6,
 		GP_2_30_FN, FN_IP6_5_4,
 		GP_2_29_FN, FN_IP6_3_2,
@@ -4717,9 +4718,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_2_3_FN, FN_IP4_11_10,
 		GP_2_2_FN, FN_IP4_9_8,
 		GP_2_1_FN, FN_IP4_7_5,
-		GP_2_0_FN, FN_IP4_4_2 }
+		GP_2_0_FN, FN_IP4_4_2 ))
 	},
-	{ PINMUX_CFG_REG("GPSR3", 0xE6060010, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR3", 0xE6060010, 32, 1, GROUP(
 		GP_3_31_FN, FN_IP8_22_20,
 		GP_3_30_FN, FN_IP8_19_17,
 		GP_3_29_FN, FN_IP8_16_15,
@@ -4751,9 +4752,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_3_3_FN, FN_IP6_11,
 		GP_3_2_FN, FN_IP6_10,
 		GP_3_1_FN, FN_IP6_9,
-		GP_3_0_FN, FN_IP6_8 }
+		GP_3_0_FN, FN_IP6_8 ))
 	},
-	{ PINMUX_CFG_REG("GPSR4", 0xE6060014, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR4", 0xE6060014, 32, 1, GROUP(
 		GP_4_31_FN, FN_IP11_17_16,
 		GP_4_30_FN, FN_IP11_15_14,
 		GP_4_29_FN, FN_IP11_13_11,
@@ -4785,9 +4786,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_4_3_FN, FN_IP9_2_0,
 		GP_4_2_FN, FN_IP8_31_29,
 		GP_4_1_FN, FN_IP8_28_26,
-		GP_4_0_FN, FN_IP8_25_23 }
+		GP_4_0_FN, FN_IP8_25_23 ))
 	},
-	{ PINMUX_CFG_REG("GPSR5", 0xE6060018, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR5", 0xE6060018, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -4819,9 +4820,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_5_3_FN, FN_IP11_29_27,
 		GP_5_2_FN, FN_IP11_26_24,
 		GP_5_1_FN, FN_IP11_23_21,
-		GP_5_0_FN, FN_IP11_20_18 }
+		GP_5_0_FN, FN_IP11_20_18 ))
 	},
-	{ PINMUX_CFG_REG("GPSR6", 0xE606001C, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR6", 0xE606001C, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -4853,11 +4854,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_6_3_FN, FN_SD0_DATA1,
 		GP_6_2_FN, FN_SD0_DATA0,
 		GP_6_1_FN, FN_SD0_CMD,
-		GP_6_0_FN, FN_SD0_CLK }
+		GP_6_0_FN, FN_SD0_CLK ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR0", 0xE6060020, 32,
-			     2, 2, 2, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1,
-			     2, 1, 1, 1, 1, 1, 1, 1, 1) {
+			     GROUP(2, 2, 2, 1, 1, 2, 2, 2, 1, 1, 1, 1,
+				   1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1),
+			     GROUP(
 		/* IP0_31_30 [2] */
 		FN_D5, FN_SCIF4_RXD_B, FN_I2C0_SCL_D, 0,
 		/* IP0_29_28 [2] */
@@ -4907,11 +4909,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP0_1 [1] */
 		0, 0,
 		/* IP0_0 [1] */
-		FN_SD1_CD, FN_CAN0_RX, }
+		FN_SD1_CD, FN_CAN0_RX, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR1", 0xE6060024, 32,
-			     2, 2, 1, 1, 1, 1, 2, 2, 2, 3, 2, 2, 3, 2, 2,
-			     2, 2) {
+			     GROUP(2, 2, 1, 1, 1, 1, 2, 2, 2, 3, 2, 2,
+				   3, 2, 2, 2, 2),
+			     GROUP(
 		/* IP1_31_30 [2] */
 		FN_A6, FN_SCIFB0_CTS_N, FN_SCIFA4_RXD_B, FN_TPUTO2_C,
 		/* IP1_29_28 [2] */
@@ -4947,10 +4950,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP1_3_2 [2] */
 		FN_D7, FN_IRQ3, FN_TCLK1, FN_PWM6_B,
 		/* IP1_1_0 [2] */
-		FN_D6, FN_SCIF4_TXD_B, FN_I2C0_SDA_D, 0, }
+		FN_D6, FN_SCIF4_TXD_B, FN_I2C0_SDA_D, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR2", 0xE6060028, 32,
-			     2, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2) {
+			     GROUP(2, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2),
+			     GROUP(
 		/* IP2_31_30 [2] */
 		FN_A20, FN_SPCLK, 0, 0,
 		/* IP2_29_27 [3] */
@@ -4982,10 +4986,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP2_3_2 [2] */
 		FN_A8, FN_MSIOF1_RXD, FN_SCIFA0_RXD_B, 0,
 		/* IP2_1_0 [2] */
-		FN_A7, FN_SCIFB0_RTS_N, FN_SCIFA4_TXD_B, 0, }
+		FN_A7, FN_SCIFB0_RTS_N, FN_SCIFA4_TXD_B, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR3", 0xE606002C, 32,
-			     1, 1, 3, 3, 3, 3, 3, 2, 1, 1, 1, 2, 2, 2, 2, 2) {
+			     GROUP(1, 1, 3, 3, 3, 3, 3, 2, 1, 1, 1, 2,
+				   2, 2, 2, 2),
+			     GROUP(
 		/* IP3_31 [1] */
 		FN_RD_WR_N, FN_ATAG1_N,
 		/* IP3_30 [1] */
@@ -5022,10 +5028,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP3_3_2 [2] */
 		FN_A22, FN_MISO_IO1, 0, FN_ATADIR1_N,
 		/* IP3_1_0 [2] */
-		FN_A21, FN_MOSI_IO0, 0, 0, }
+		FN_A21, FN_MOSI_IO0, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR4", 0xE6060030, 32,
-			     2, 2, 2, 3, 3, 2, 2, 2, 2, 2, 2, 3, 3, 2) {
+			     GROUP(2, 2, 2, 3, 3, 2, 2, 2, 2, 2, 2, 3, 3, 2),
+			     GROUP(
 		/* IP4_31_30 [2] */
 		FN_DU0_DG4, FN_LCDOUT12, 0, 0,
 		/* IP4_29_28 [2] */
@@ -5057,10 +5064,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_DU0_DR0, FN_LCDOUT16, FN_SCIF5_RXD_C, FN_I2C2_SCL_D,
 		0, 0, 0, 0,
 		/* IP4_1_0 [2] */
-		FN_EX_WAIT0, FN_CAN_CLK_B, FN_SCIF_CLK, 0, }
+		FN_EX_WAIT0, FN_CAN_CLK_B, FN_SCIF_CLK, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR5", 0xE6060034, 32,
-			     2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 2, 2, 2) {
+			     GROUP(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3,
+				   2, 2, 2),
+			     GROUP(
 		/* IP5_31_30 [2] */
 		FN_DU0_EXHSYNC_DU0_HSYNC, FN_QSTH_QHS, 0, 0,
 		/* IP5_29_28 [2] */
@@ -5092,11 +5101,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP5_3_2 [2] */
 		FN_DU0_DG6, FN_LCDOUT14, 0, 0,
 		/* IP5_1_0 [2] */
-		FN_DU0_DG5, FN_LCDOUT13, 0, 0, }
+		FN_DU0_DG5, FN_LCDOUT13, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR6", 0xE6060038, 32,
-			     3, 3, 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2,
-			     2, 2) {
+			     GROUP(3, 3, 3, 3, 3, 1, 1, 1, 1, 1, 1, 1,
+				   1, 1, 2, 2, 2, 2),
+			     GROUP(
 		/* IP6_31_29 [3] */
 		FN_ETH_MDIO, FN_VI0_G0, FN_MSIOF2_RXD_B, FN_I2C5_SCL_D,
 		FN_AVB_TX_CLK, FN_ADIDATA, 0, 0,
@@ -5138,10 +5148,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_DU0_EXODDF_DU0_ODDF_DISP_CDE, FN_QCPV_QDE, 0,
 		0,
 		/* IP6_1_0 [2] */
-		FN_DU0_EXVSYNC_DU0_VSYNC, FN_QSTB_QHE, 0, 0, }
+		FN_DU0_EXVSYNC_DU0_VSYNC, FN_QSTB_QHE, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR7", 0xE606003C, 32,
-			     1, 1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3) {
+			     GROUP(1, 1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3),
+			     GROUP(
 		/* IP7_31 [1] */
 		FN_DREQ0_N, FN_SCIFB1_RXD,
 		/* IP7_30 [1] */
@@ -5175,10 +5186,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_AVB_TXD1, FN_ADICLK, 0, 0,
 		/* IP7_2_0 [3] */
 		FN_ETH_CRS_DV, FN_VI0_G1, FN_MSIOF2_TXD_B, FN_I2C5_SDA_D,
-		FN_AVB_TXD0, FN_ADICS_SAMP, 0, 0, }
+		FN_AVB_TXD0, FN_ADICS_SAMP, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR8", 0xE6060040, 32,
-			     3, 3, 3, 3, 3, 2, 3, 3, 3, 3, 3) {
+			     GROUP(3, 3, 3, 3, 3, 2, 3, 3, 3, 3, 3),
+			     GROUP(
 		/* IP8_31_29 [3] */
 		FN_MSIOF0_RXD, FN_SCIF5_RXD, FN_I2C2_SCL_C, FN_DU1_DR2,
 		0, FN_TS_SDEN_D, FN_FMCLK_C, 0,
@@ -5210,10 +5222,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_AVB_MDIO, FN_SSI_SCK78_B, 0, 0,
 		/* IP8_2_0 [3] */
 		FN_ETH_MDC, FN_VI0_R3, FN_SCIF3_TXD_B, FN_I2C4_SDA_E,
-		FN_AVB_MDC, FN_SSI_SDATA6_B, 0, 0, }
+		FN_AVB_MDC, FN_SSI_SDATA6_B, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR9", 0xE6060044, 32,
-			     1, 3, 3, 3, 3, 2, 2, 3, 3, 3, 3, 3) {
+			     GROUP(1, 3, 3, 3, 3, 2, 2, 3, 3, 3, 3, 3),
+			     GROUP(
 		/* IP9_31 [1] */
 		0, 0,
 		/* IP9_30_28 [3] */
@@ -5246,10 +5259,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, FN_TPUTO1_C, 0, 0,
 		/* IP9_2_0 [3] */
 		FN_MSIOF0_TXD, FN_SCIF5_TXD, FN_I2C2_SDA_C, FN_DU1_DR3,
-		0, FN_TS_SPSYNC_D, FN_FMIN_C, 0, }
+		0, FN_TS_SPSYNC_D, FN_FMIN_C, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR10", 0xE6060048, 32,
-			     2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3) {
+			     GROUP(2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3),
+			     GROUP(
 		/* IP10_31_30 [2] */
 		FN_SSI_SCK5, FN_SCIFA3_SCK, FN_DU1_DOTCLKIN, 0,
 		/* IP10_29_27 [3] */
@@ -5281,10 +5295,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0,
 		/* IP10_2_0 [3] */
 		FN_SCIF1_RXD, FN_I2C5_SCL, FN_DU1_DG6, FN_SSI_SCK2_B,
-		0, 0, 0, 0, }
+		0, 0, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR11", 0xE606004C, 32,
-			     2, 3, 3, 3, 3, 2, 2, 3, 3, 2, 3, 3) {
+			     GROUP(2, 3, 3, 3, 3, 2, 2, 3, 3, 2, 3, 3),
+			     GROUP(
 		/* IP11_31_30 [2] */
 		0, 0, 0, 0,
 		/* IP11_29_27 [3] */
@@ -5316,10 +5331,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0,
 		/* IP11_2_0 [3] */
 		FN_SSI_WS5, FN_SCIFA3_RXD, FN_I2C3_SCL_C, FN_DU1_DOTCLKOUT0,
-		0, 0, 0, 0, }
+		0, 0, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR12", 0xE6060050, 32,
-			     2, 3, 3, 3, 3, 3, 2, 2, 2, 3, 3, 3) {
+			     GROUP(2, 3, 3, 3, 3, 3, 2, 2, 2, 3, 3, 3),
+			     GROUP(
 		/* IP12_31_30 [2] */
 		0, 0, 0, 0,
 		/* IP12_29_27 [3] */
@@ -5351,10 +5367,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_CAN1_RX_C, FN_DACK1_B, 0, 0,
 		/* IP12_2_0 [3] */
 		FN_SSI_SCK34, FN_MSIOF1_SYNC_B, FN_SCIFA1_SCK_C, FN_ADICHS0_B,
-		0, FN_DREQ1_N_B, 0, 0, }
+		0, FN_DREQ1_N_B, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR13", 0xE6060054, 32,
-			     1, 1, 1, 1, 1, 3, 3, 3, 3, 3, 3, 3, 3, 3) {
+			     GROUP(1, 1, 1, 1, 1, 3, 3, 3, 3, 3, 3, 3, 3, 3),
+			     GROUP(
 		/* IP13_31 [1] */
 		0, 0,
 		/* IP13_30 [1] */
@@ -5391,11 +5408,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_VI1_DATA4, 0, FN_ATACS10_N, FN_ETH_REFCLK_B, 0,
 		/* IP13_2_0 [3] */
 		FN_SSI_WS2, FN_HSCIF1_HCTS_N_B, FN_SCIFA0_RXD_D, FN_VI1_DATA3,
-		0, FN_ATACS00_N, FN_ETH_LINK_B, 0, }
+		0, FN_ATACS00_N, FN_ETH_LINK_B, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("MOD_SEL", 0xE6060090, 32,
-			     2, 1, 2, 3, 4, 1, 1, 3, 3, 3, 3, 3,
-			     2, 1) {
+			     GROUP(2, 1, 2, 3, 4, 1, 1, 3, 3, 3, 3, 3, 2, 1),
+			     GROUP(
 		/* SEL_ADG [2] */
 		FN_SEL_ADG_0, FN_SEL_ADG_1, FN_SEL_ADG_2, FN_SEL_ADG_3,
 		/* RESERVED [1] */
@@ -5429,11 +5446,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* SEL_I2C05 [2] */
 		FN_SEL_I2C05_0, FN_SEL_I2C05_1, FN_SEL_I2C05_2, FN_SEL_I2C05_3,
 		/* RESERVED [1] */
-		0, 0, }
+		0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("MOD_SEL2", 0xE6060094, 32,
-			     2, 2, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 2, 2, 1, 1,
-			     2, 2, 2, 1, 1, 2) {
+			     GROUP(2, 2, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1,
+				   2, 2, 1, 1, 2, 2, 2, 1, 1, 2),
+			     GROUP(
 		/* SEL_IEB [2] */
 		FN_SEL_IEB_0, FN_SEL_IEB_1, FN_SEL_IEB_2, 0,
 		/* SEL_IIC0 [2] */
@@ -5480,11 +5498,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* SEL_HSCIF1 [1] */
 		FN_SEL_HSCIF1_0, FN_SEL_HSCIF1_1,
 		/* RESERVED [2] */
-		0, 0, 0, 0, }
+		0, 0, 0, 0, ))
 	},
 	{ PINMUX_CFG_REG_VAR("MOD_SEL3", 0xE6060098, 32,
-			     2, 2, 2, 1, 3, 2, 1, 1, 1, 1, 1, 1, 1, 1,
-			     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) {
+			     GROUP(2, 2, 2, 1, 3, 2, 1, 1, 1, 1, 1, 1,
+				   1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1),
+			     GROUP(
 		/* SEL_SCIF0 [2] */
 		FN_SEL_SCIF0_0, FN_SEL_SCIF0_1, FN_SEL_SCIF0_2, FN_SEL_SCIF0_3,
 		/* SEL_SCIF1 [2] */
@@ -5537,7 +5556,7 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* RESERVED [1] */
 		0, 0,
 		/* RESERVED [1] */
-		0, 0, }
+		0, 0, ))
 	},
 	{ },
 };
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a7795-es1.c b/drivers/pinctrl/sh-pfc/pfc-r8a7795-es1.c
index 287cfbb7e992..f16dfbad3f17 100644
--- a/drivers/pinctrl/sh-pfc/pfc-r8a7795-es1.c
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a7795-es1.c
@@ -5,6 +5,7 @@
  * Copyright (C) 2015-2017  Renesas Electronics Corporation
  */
 
+#include <linux/errno.h>
 #include <linux/kernel.h>
 
 #include "core.h"
@@ -153,11 +154,11 @@
 #define GPSR5_11	F_(RX2_A,		IP12_7_4)
 #define GPSR5_10	F_(TX2_A,		IP12_3_0)
 #define GPSR5_9		F_(SCK2,		IP11_31_28)
-#define GPSR5_8		F_(RTS1_N_TANS,		IP11_27_24)
+#define GPSR5_8		F_(RTS1_N,		IP11_27_24)
 #define GPSR5_7		F_(CTS1_N,		IP11_23_20)
 #define GPSR5_6		F_(TX1_A,		IP11_19_16)
 #define GPSR5_5		F_(RX1_A,		IP11_15_12)
-#define GPSR5_4		F_(RTS0_N_TANS,		IP11_11_8)
+#define GPSR5_4		F_(RTS0_N,		IP11_11_8)
 #define GPSR5_3		F_(CTS0_N,		IP11_7_4)
 #define GPSR5_2		F_(TX0,			IP11_3_0)
 #define GPSR5_1		F_(RX0,			IP10_31_28)
@@ -198,8 +199,8 @@
 #define GPSR6_0		F_(SSI_SCK01239,	IP13_23_20)
 
 /* GPSR7 */
-#define GPSR7_3		FM(HDMI1_CEC)
-#define GPSR7_2		FM(HDMI0_CEC)
+#define GPSR7_3		FM(GP7_03)
+#define GPSR7_2		FM(GP7_02)
 #define GPSR7_1		FM(AVS2)
 #define GPSR7_0		FM(AVS1)
 
@@ -210,7 +211,7 @@
 #define IP0_11_8	FM(AVB_PHY_INT)		F_(0, 0)	FM(MSIOF2_SYNC_C)	FM(RX4_A)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP0_15_12	FM(AVB_LINK)		F_(0, 0)	FM(MSIOF2_SCK_C)	FM(TX4_A)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP0_19_16	FM(AVB_AVTP_MATCH_A)	F_(0, 0)	FM(MSIOF2_RXD_C)	FM(CTS4_N_A)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP0_23_20	FM(AVB_AVTP_CAPTURE_A)	F_(0, 0)	FM(MSIOF2_TXD_C)	FM(RTS4_N_TANS_A)		F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP0_23_20	FM(AVB_AVTP_CAPTURE_A)	F_(0, 0)	FM(MSIOF2_TXD_C)	FM(RTS4_N_A)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP0_27_24	FM(IRQ0)		FM(QPOLB)	F_(0, 0)		FM(DU_CDE)			FM(VI4_DATA0_B)	FM(CAN0_TX_B)	FM(CANFD0_TX_B)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP0_31_28	FM(IRQ1)		FM(QPOLA)	F_(0, 0)		FM(DU_DISP)			FM(VI4_DATA1_B)	FM(CAN0_RX_B)	FM(CANFD0_RX_B)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP1_3_0		FM(IRQ2)		FM(QCPV_QDE)	F_(0, 0)		FM(DU_EXODDF_DU_ODDF_DISP_CDE)	FM(VI4_DATA2_B)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		FM(PWM3_B)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
@@ -232,7 +233,7 @@
 #define IP2_27_24	FM(A7)			FM(LCDOUT23)	FM(MSIOF2_SS2_A)	FM(TX4_B)			FM(VI4_DATA15)	FM(VI5_DATA15)	FM(DU_DB7)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP2_31_28	FM(A8)			FM(RX3_B)	FM(MSIOF2_SYNC_A)	FM(HRX4_B)			F_(0, 0)	F_(0, 0)	F_(0, 0)		FM(SDA6_A)	FM(AVB_AVTP_MATCH_B)	FM(PWM1_B)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP3_3_0		FM(A9)			F_(0, 0)	FM(MSIOF2_SCK_A)	FM(CTS4_N_B)			F_(0, 0)	FM(VI5_VSYNC_N)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP3_7_4		FM(A10)			F_(0, 0)	FM(MSIOF2_RXD_A)	FM(RTS4_N_TANS_B)		F_(0, 0)	FM(VI5_HSYNC_N)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP3_7_4		FM(A10)			F_(0, 0)	FM(MSIOF2_RXD_A)	FM(RTS4_N_B)			F_(0, 0)	FM(VI5_HSYNC_N)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP3_11_8	FM(A11)			FM(TX3_B)	FM(MSIOF2_TXD_A)	FM(HTX4_B)			FM(HSCK4)	FM(VI5_FIELD)	F_(0, 0)		FM(SCL6_A)	FM(AVB_AVTP_CAPTURE_B)	FM(PWM2_B)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP3_15_12	FM(A12)			FM(LCDOUT12)	FM(MSIOF3_SCK_C)	F_(0, 0)			FM(HRX4_A)	FM(VI5_DATA8)	FM(DU_DG4)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP3_19_16	FM(A13)			FM(LCDOUT13)	FM(MSIOF3_SYNC_C)	F_(0, 0)			FM(HTX4_A)	FM(VI5_DATA9)	FM(DU_DG5)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
@@ -248,7 +249,7 @@
 #define IP4_27_24	FM(RD_N)		F_(0, 0)	FM(MSIOF3_SYNC_D)	FM(RX3_A)			FM(HRX3_A)	F_(0, 0)	F_(0, 0)		F_(0, 0)	FM(CAN0_TX_A)		FM(CANFD0_TX_A)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP4_31_28	FM(RD_WR_N)		F_(0, 0)	FM(MSIOF3_RXD_D)	FM(TX3_A)			FM(HTX3_A)	F_(0, 0)	F_(0, 0)		F_(0, 0)	FM(CAN0_RX_A)		FM(CANFD0_RX_A)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP5_3_0		FM(WE0_N)		F_(0, 0)	FM(MSIOF3_TXD_D)	FM(CTS3_N)			FM(HCTS3_N)	F_(0, 0)	F_(0, 0)		FM(SCL6_B)	FM(CAN_CLK)		F_(0, 0)	FM(IECLK_A)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP5_7_4		FM(WE1_N)		F_(0, 0)	FM(MSIOF3_SS1_D)	FM(RTS3_N_TANS)			FM(HRTS3_N)	F_(0, 0)	F_(0, 0)		FM(SDA6_B)	FM(CAN1_RX)		FM(CANFD1_RX)	FM(IERX_A)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP5_7_4		FM(WE1_N)		F_(0, 0)	FM(MSIOF3_SS1_D)	FM(RTS3_N)			FM(HRTS3_N)	F_(0, 0)	F_(0, 0)		FM(SDA6_B)	FM(CAN1_RX)		FM(CANFD1_RX)	FM(IERX_A)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP5_11_8	FM(EX_WAIT0_A)		FM(QCLK)	F_(0, 0)		F_(0, 0)			FM(VI4_CLK)	F_(0, 0)	FM(DU_DOTCLKOUT0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP5_15_12	FM(D0)			FM(MSIOF2_SS1_B)FM(MSIOF3_SCK_A)	F_(0, 0)			FM(VI4_DATA16)	FM(VI5_DATA0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP5_19_16	FM(D1)			FM(MSIOF2_SS2_B)FM(MSIOF3_SYNC_A)	F_(0, 0)			FM(VI4_DATA17)	FM(VI5_DATA1)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
@@ -261,7 +262,7 @@
 #define IP6_15_12	FM(D8)			FM(LCDOUT0)	FM(MSIOF2_SCK_D)	FM(SCK4_C)			FM(VI4_DATA0_A)	F_(0, 0)	FM(DU_DR0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP6_19_16	FM(D9)			FM(LCDOUT1)	FM(MSIOF2_SYNC_D)	F_(0, 0)			FM(VI4_DATA1_A)	F_(0, 0)	FM(DU_DR1)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP6_23_20	FM(D10)			FM(LCDOUT2)	FM(MSIOF2_RXD_D)	FM(HRX3_B)			FM(VI4_DATA2_A)	FM(CTS4_N_C)	FM(DU_DR2)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP6_27_24	FM(D11)			FM(LCDOUT3)	FM(MSIOF2_TXD_D)	FM(HTX3_B)			FM(VI4_DATA3_A)	FM(RTS4_N_TANS_C)FM(DU_DR3)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP6_27_24	FM(D11)			FM(LCDOUT3)	FM(MSIOF2_TXD_D)	FM(HTX3_B)			FM(VI4_DATA3_A)	FM(RTS4_N_C)	FM(DU_DR3)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP6_31_28	FM(D12)			FM(LCDOUT4)	FM(MSIOF2_SS1_D)	FM(RX4_C)			FM(VI4_DATA4_A)	F_(0, 0)	FM(DU_DR4)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP7_3_0		FM(D13)			FM(LCDOUT5)	FM(MSIOF2_SS2_D)	FM(TX4_C)			FM(VI4_DATA5_A)	F_(0, 0)	FM(DU_DR5)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP7_7_4		FM(D14)			FM(LCDOUT6)	FM(MSIOF3_SS1_A)	FM(HRX3_C)			FM(VI4_DATA6_A)	F_(0, 0)	FM(DU_DR6)		FM(SCL6_C)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
@@ -299,11 +300,11 @@
 #define IP10_31_28	FM(RX0)			FM(HRX1_B)	F_(0, 0)		F_(0, 0)			F_(0, 0)	FM(TS_SCK0_C)	FM(STP_ISCLK_0_C)	FM(RIF0_D0_B)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP11_3_0	FM(TX0)			FM(HTX1_B)	F_(0, 0)		F_(0, 0)			F_(0, 0)	FM(TS_SPSYNC0_C)FM(STP_ISSYNC_0_C)	FM(RIF0_D1_B)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP11_7_4	FM(CTS0_N)		FM(HCTS1_N_B)	FM(MSIOF1_SYNC_B)	F_(0, 0)			F_(0, 0)	FM(TS_SPSYNC1_C)FM(STP_ISSYNC_1_C)	FM(RIF1_SYNC_B)	FM(AUDIO_CLKOUT_C)	FM(ADICS_SAMP)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP11_11_8	FM(RTS0_N_TANS)		FM(HRTS1_N_B)	FM(MSIOF1_SS1_B)	FM(AUDIO_CLKA_B)		FM(SCL2_A)	F_(0, 0)	FM(STP_IVCXO27_1_C)	FM(RIF0_SYNC_B)	F_(0, 0)		FM(ADICHS1)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP11_11_8	FM(RTS0_N)		FM(HRTS1_N_B)	FM(MSIOF1_SS1_B)	FM(AUDIO_CLKA_B)		FM(SCL2_A)	F_(0, 0)	FM(STP_IVCXO27_1_C)	FM(RIF0_SYNC_B)	F_(0, 0)		FM(ADICHS1)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP11_15_12	FM(RX1_A)		FM(HRX1_A)	F_(0, 0)		F_(0, 0)			F_(0, 0)	FM(TS_SDAT0_C)	FM(STP_ISD_0_C)		FM(RIF1_CLK_C)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP11_19_16	FM(TX1_A)		FM(HTX1_A)	F_(0, 0)		F_(0, 0)			F_(0, 0)	FM(TS_SDEN0_C)	FM(STP_ISEN_0_C)	FM(RIF1_D0_C)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP11_23_20	FM(CTS1_N)		FM(HCTS1_N_A)	FM(MSIOF1_RXD_B)	F_(0, 0)			F_(0, 0)	FM(TS_SDEN1_C)	FM(STP_ISEN_1_C)	FM(RIF1_D0_B)	F_(0, 0)		FM(ADIDATA)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP11_27_24	FM(RTS1_N_TANS)		FM(HRTS1_N_A)	FM(MSIOF1_TXD_B)	F_(0, 0)			F_(0, 0)	FM(TS_SDAT1_C)	FM(STP_ISD_1_C)		FM(RIF1_D1_B)	F_(0, 0)		FM(ADICHS0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP11_27_24	FM(RTS1_N)		FM(HRTS1_N_A)	FM(MSIOF1_TXD_B)	F_(0, 0)			F_(0, 0)	FM(TS_SDAT1_C)	FM(STP_ISD_1_C)		FM(RIF1_D1_B)	F_(0, 0)		FM(ADICHS0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP11_31_28	FM(SCK2)		FM(SCIF_CLK_B)	FM(MSIOF1_SCK_B)	F_(0, 0)			F_(0, 0)	FM(TS_SCK1_C)	FM(STP_ISCLK_1_C)	FM(RIF1_CLK_B)	F_(0, 0)		FM(ADICLK)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP12_3_0	FM(TX2_A)		F_(0, 0)	F_(0, 0)		FM(SD2_CD_B)			FM(SCL1_A)	F_(0, 0)	FM(FMCLK_A)		FM(RIF1_D1_C)	F_(0, 0)		FM(FSO_CFE_0_B)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP12_7_4	FM(RX2_A)		F_(0, 0)	F_(0, 0)		FM(SD2_WP_B)			FM(SDA1_A)	F_(0, 0)	FM(FMIN_A)		FM(RIF1_SYNC_C)	F_(0, 0)		FM(FSO_CFE_1_B)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
@@ -339,7 +340,7 @@
 #define IP15_23_20	FM(SSI_SDATA7)		FM(HCTS2_N_B)	FM(MSIOF1_RXD_C)	F_(0, 0)			F_(0, 0)	FM(TS_SDEN1_A)	FM(STP_ISEN_1_A)	FM(RIF1_D0_A)	FM(RIF3_D0_A)		F_(0, 0)	FM(TCLK2_A)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP15_27_24	FM(SSI_SDATA8)		FM(HRTS2_N_B)	FM(MSIOF1_TXD_C)	F_(0, 0)			F_(0, 0)	FM(TS_SPSYNC1_A)FM(STP_ISSYNC_1_A)	FM(RIF1_D1_A)	FM(RIF3_D1_A)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP15_31_28	FM(SSI_SDATA9_A)	FM(HSCK2_B)	FM(MSIOF1_SS1_C)	FM(HSCK1_A)			FM(SSI_WS1_B)	FM(SCK1)	FM(STP_IVCXO27_1_A)	FM(SCK5)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP16_3_0	FM(AUDIO_CLKA_A)	F_(0, 0)	F_(0, 0)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	FM(CC5_OSCOUT)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP16_3_0	FM(AUDIO_CLKA_A)	F_(0, 0)	F_(0, 0)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP16_7_4	FM(AUDIO_CLKB_B)	FM(SCIF_CLK_A)	F_(0, 0)		F_(0, 0)			F_(0, 0)	F_(0, 0)	FM(STP_IVCXO27_1_D)	FM(REMOCON_A)	F_(0, 0)		F_(0, 0)	FM(TCLK1_A)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP16_11_8	FM(USB0_PWEN)		F_(0, 0)	F_(0, 0)		FM(SIM0_RST_C)			F_(0, 0)	FM(TS_SCK1_D)	FM(STP_ISCLK_1_D)	FM(BPFCLK_B)	FM(RIF3_CLK_B)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP16_15_12	FM(USB0_OVC)		F_(0, 0)	F_(0, 0)		FM(SIM0_D_C)			F_(0, 0)	FM(TS_SDAT1_D)	FM(STP_ISD_1_D)		F_(0, 0)	FM(RIF3_SYNC_B)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
@@ -576,8 +577,8 @@ static const u16 pinmux_data[] = {
 
 	PINMUX_SINGLE(AVS1),
 	PINMUX_SINGLE(AVS2),
-	PINMUX_SINGLE(HDMI0_CEC),
-	PINMUX_SINGLE(HDMI1_CEC),
+	PINMUX_SINGLE(GP7_02),
+	PINMUX_SINGLE(GP7_03),
 	PINMUX_SINGLE(MSIOF0_RXD),
 	PINMUX_SINGLE(MSIOF0_SCK),
 	PINMUX_SINGLE(MSIOF0_TXD),
@@ -616,7 +617,7 @@ static const u16 pinmux_data[] = {
 
 	PINMUX_IPSR_PHYS_MSEL(IP0_23_20, AVB_AVTP_CAPTURE_A,	I2C_SEL_5_0,	SEL_ETHERAVB_0),
 	PINMUX_IPSR_PHYS_MSEL(IP0_23_20, MSIOF2_TXD_C,		I2C_SEL_5_0,	SEL_MSIOF2_2),
-	PINMUX_IPSR_PHYS_MSEL(IP0_23_20, RTS4_N_TANS_A,		I2C_SEL_5_0,	SEL_SCIF4_0),
+	PINMUX_IPSR_PHYS_MSEL(IP0_23_20, RTS4_N_A,		I2C_SEL_5_0,	SEL_SCIF4_0),
 	PINMUX_IPSR_PHYS(IP0_23_20,	SDA5,			I2C_SEL_5_1),
 
 	PINMUX_IPSR_GPSR(IP0_27_24,	IRQ0),
@@ -756,7 +757,7 @@ static const u16 pinmux_data[] = {
 
 	PINMUX_IPSR_GPSR(IP3_7_4,	A10),
 	PINMUX_IPSR_MSEL(IP3_7_4,	MSIOF2_RXD_A,		SEL_MSIOF2_0),
-	PINMUX_IPSR_MSEL(IP3_7_4,	RTS4_N_TANS_B,		SEL_SCIF4_1),
+	PINMUX_IPSR_MSEL(IP3_7_4,	RTS4_N_B,		SEL_SCIF4_1),
 	PINMUX_IPSR_GPSR(IP3_7_4,	VI5_HSYNC_N),
 
 	PINMUX_IPSR_GPSR(IP3_11_8,	A11),
@@ -859,7 +860,7 @@ static const u16 pinmux_data[] = {
 
 	PINMUX_IPSR_GPSR(IP5_7_4,	WE1_N),
 	PINMUX_IPSR_MSEL(IP5_7_4,	MSIOF3_SS1_D,		SEL_MSIOF3_3),
-	PINMUX_IPSR_GPSR(IP5_7_4,	RTS3_N_TANS),
+	PINMUX_IPSR_GPSR(IP5_7_4,	RTS3_N),
 	PINMUX_IPSR_GPSR(IP5_7_4,	HRTS3_N),
 	PINMUX_IPSR_MSEL(IP5_7_4,	SDA6_B,			SEL_I2C6_1),
 	PINMUX_IPSR_GPSR(IP5_7_4,	CAN1_RX),
@@ -940,7 +941,7 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_MSEL(IP6_27_24,	MSIOF2_TXD_D,		SEL_MSIOF2_3),
 	PINMUX_IPSR_MSEL(IP6_27_24,	HTX3_B,			SEL_HSCIF3_1),
 	PINMUX_IPSR_MSEL(IP6_27_24,	VI4_DATA3_A,		SEL_VIN4_0),
-	PINMUX_IPSR_MSEL(IP6_27_24,	RTS4_N_TANS_C,		SEL_SCIF4_2),
+	PINMUX_IPSR_MSEL(IP6_27_24,	RTS4_N_C,		SEL_SCIF4_2),
 	PINMUX_IPSR_GPSR(IP6_27_24,	DU_DR3),
 
 	PINMUX_IPSR_GPSR(IP6_31_28,	D12),
@@ -1112,7 +1113,7 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_MSEL(IP11_7_4,	AUDIO_CLKOUT_C,		SEL_ADG_2),
 	PINMUX_IPSR_GPSR(IP11_7_4,	ADICS_SAMP),
 
-	PINMUX_IPSR_GPSR(IP11_11_8,	RTS0_N_TANS),
+	PINMUX_IPSR_GPSR(IP11_11_8,	RTS0_N),
 	PINMUX_IPSR_MSEL(IP11_11_8,	HRTS1_N_B,		SEL_HSCIF1_1),
 	PINMUX_IPSR_MSEL(IP11_11_8,	MSIOF1_SS1_B,		SEL_MSIOF1_1),
 	PINMUX_IPSR_MSEL(IP11_11_8,	AUDIO_CLKA_B,		SEL_ADG_1),
@@ -1141,7 +1142,7 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_MSEL(IP11_23_20,	RIF1_D0_B,		SEL_DRIF1_1),
 	PINMUX_IPSR_GPSR(IP11_23_20,	ADIDATA),
 
-	PINMUX_IPSR_GPSR(IP11_27_24,	RTS1_N_TANS),
+	PINMUX_IPSR_GPSR(IP11_27_24,	RTS1_N),
 	PINMUX_IPSR_MSEL(IP11_27_24,	HRTS1_N_A,		SEL_HSCIF1_0),
 	PINMUX_IPSR_MSEL(IP11_27_24,	MSIOF1_TXD_B,		SEL_MSIOF1_1),
 	PINMUX_IPSR_MSEL(IP11_27_24,	TS_SDAT1_C,		SEL_TSIF1_2),
@@ -1358,7 +1359,6 @@ static const u16 pinmux_data[] = {
 
 	/* IPSR16 */
 	PINMUX_IPSR_MSEL(IP16_3_0,	AUDIO_CLKA_A,		SEL_ADG_0),
-	PINMUX_IPSR_GPSR(IP16_3_0,	CC5_OSCOUT),
 
 	PINMUX_IPSR_MSEL(IP16_7_4,	AUDIO_CLKB_B,		SEL_ADG_1),
 	PINMUX_IPSR_MSEL(IP16_7_4,	SCIF_CLK_A,		SEL_SCIF1_0),
@@ -2071,22 +2071,6 @@ static const unsigned int du_disp_pins[] = {
 static const unsigned int du_disp_mux[] = {
 	DU_DISP_MARK,
 };
-/* - HDMI ------------------------------------------------------------------- */
-static const unsigned int hdmi0_cec_pins[] = {
-	/* HDMI0_CEC */
-	RCAR_GP_PIN(7, 2),
-};
-static const unsigned int hdmi0_cec_mux[] = {
-	HDMI0_CEC_MARK,
-};
-static const unsigned int hdmi1_cec_pins[] = {
-	/* HDMI1_CEC */
-	RCAR_GP_PIN(7, 3),
-};
-static const unsigned int hdmi1_cec_mux[] = {
-	HDMI1_CEC_MARK,
-};
-
 /* - HSCIF0 ----------------------------------------------------------------- */
 static const unsigned int hscif0_data_pins[] = {
 	/* RX, TX */
@@ -3235,7 +3219,7 @@ static const unsigned int scif0_ctrl_pins[] = {
 	RCAR_GP_PIN(5, 4), RCAR_GP_PIN(5, 3),
 };
 static const unsigned int scif0_ctrl_mux[] = {
-	RTS0_N_TANS_MARK, CTS0_N_MARK,
+	RTS0_N_MARK, CTS0_N_MARK,
 };
 /* - SCIF1 ------------------------------------------------------------------ */
 static const unsigned int scif1_data_a_pins[] = {
@@ -3257,7 +3241,7 @@ static const unsigned int scif1_ctrl_pins[] = {
 	RCAR_GP_PIN(5, 8), RCAR_GP_PIN(5, 7),
 };
 static const unsigned int scif1_ctrl_mux[] = {
-	RTS1_N_TANS_MARK, CTS1_N_MARK,
+	RTS1_N_MARK, CTS1_N_MARK,
 };
 
 static const unsigned int scif1_data_b_pins[] = {
@@ -3309,7 +3293,7 @@ static const unsigned int scif3_ctrl_pins[] = {
 	RCAR_GP_PIN(1, 26), RCAR_GP_PIN(1, 25),
 };
 static const unsigned int scif3_ctrl_mux[] = {
-	RTS3_N_TANS_MARK, CTS3_N_MARK,
+	RTS3_N_MARK, CTS3_N_MARK,
 };
 static const unsigned int scif3_data_b_pins[] = {
 	/* RX, TX */
@@ -3338,7 +3322,7 @@ static const unsigned int scif4_ctrl_a_pins[] = {
 	RCAR_GP_PIN(2, 14), RCAR_GP_PIN(2, 13),
 };
 static const unsigned int scif4_ctrl_a_mux[] = {
-	RTS4_N_TANS_A_MARK, CTS4_N_A_MARK,
+	RTS4_N_A_MARK, CTS4_N_A_MARK,
 };
 static const unsigned int scif4_data_b_pins[] = {
 	/* RX, TX */
@@ -3359,7 +3343,7 @@ static const unsigned int scif4_ctrl_b_pins[] = {
 	RCAR_GP_PIN(1, 10), RCAR_GP_PIN(1, 9),
 };
 static const unsigned int scif4_ctrl_b_mux[] = {
-	RTS4_N_TANS_B_MARK, CTS4_N_B_MARK,
+	RTS4_N_B_MARK, CTS4_N_B_MARK,
 };
 static const unsigned int scif4_data_c_pins[] = {
 	/* RX, TX */
@@ -3380,7 +3364,7 @@ static const unsigned int scif4_ctrl_c_pins[] = {
 	RCAR_GP_PIN(0, 11), RCAR_GP_PIN(0, 10),
 };
 static const unsigned int scif4_ctrl_c_mux[] = {
-	RTS4_N_TANS_C_MARK, CTS4_N_C_MARK,
+	RTS4_N_C_MARK, CTS4_N_C_MARK,
 };
 /* - SCIF5 ------------------------------------------------------------------ */
 static const unsigned int scif5_data_pins[] = {
@@ -3944,8 +3928,6 @@ static const struct sh_pfc_pin_group pinmux_groups[] = {
 	SH_PFC_PIN_GROUP(du_oddf),
 	SH_PFC_PIN_GROUP(du_cde),
 	SH_PFC_PIN_GROUP(du_disp),
-	SH_PFC_PIN_GROUP(hdmi0_cec),
-	SH_PFC_PIN_GROUP(hdmi1_cec),
 	SH_PFC_PIN_GROUP(hscif0_data),
 	SH_PFC_PIN_GROUP(hscif0_clk),
 	SH_PFC_PIN_GROUP(hscif0_ctrl),
@@ -4299,14 +4281,6 @@ static const char * const du_groups[] = {
 	"du_disp",
 };
 
-static const char * const hdmi0_groups[] = {
-	"hdmi0_cec",
-};
-
-static const char * const hdmi1_groups[] = {
-	"hdmi1_cec",
-};
-
 static const char * const hscif0_groups[] = {
 	"hscif0_data",
 	"hscif0_clk",
@@ -4694,8 +4668,6 @@ static const struct sh_pfc_function pinmux_functions[] = {
 	SH_PFC_FUNCTION(drif2),
 	SH_PFC_FUNCTION(drif3),
 	SH_PFC_FUNCTION(du),
-	SH_PFC_FUNCTION(hdmi0),
-	SH_PFC_FUNCTION(hdmi1),
 	SH_PFC_FUNCTION(hscif0),
 	SH_PFC_FUNCTION(hscif1),
 	SH_PFC_FUNCTION(hscif2),
@@ -4745,7 +4717,7 @@ static const struct sh_pfc_function pinmux_functions[] = {
 static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 #define F_(x, y)	FN_##y
 #define FM(x)		FN_##x
-	{ PINMUX_CFG_REG("GPSR0", 0xe6060100, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR0", 0xe6060100, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -4777,9 +4749,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_0_3_FN,	GPSR0_3,
 		GP_0_2_FN,	GPSR0_2,
 		GP_0_1_FN,	GPSR0_1,
-		GP_0_0_FN,	GPSR0_0, }
+		GP_0_0_FN,	GPSR0_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR1", 0xe6060104, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR1", 0xe6060104, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -4811,9 +4783,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_1_3_FN,	GPSR1_3,
 		GP_1_2_FN,	GPSR1_2,
 		GP_1_1_FN,	GPSR1_1,
-		GP_1_0_FN,	GPSR1_0, }
+		GP_1_0_FN,	GPSR1_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR2", 0xe6060108, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR2", 0xe6060108, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -4845,9 +4817,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_2_3_FN,	GPSR2_3,
 		GP_2_2_FN,	GPSR2_2,
 		GP_2_1_FN,	GPSR2_1,
-		GP_2_0_FN,	GPSR2_0, }
+		GP_2_0_FN,	GPSR2_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR3", 0xe606010c, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR3", 0xe606010c, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -4879,9 +4851,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_3_3_FN,	GPSR3_3,
 		GP_3_2_FN,	GPSR3_2,
 		GP_3_1_FN,	GPSR3_1,
-		GP_3_0_FN,	GPSR3_0, }
+		GP_3_0_FN,	GPSR3_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR4", 0xe6060110, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR4", 0xe6060110, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -4913,9 +4885,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_4_3_FN,	GPSR4_3,
 		GP_4_2_FN,	GPSR4_2,
 		GP_4_1_FN,	GPSR4_1,
-		GP_4_0_FN,	GPSR4_0, }
+		GP_4_0_FN,	GPSR4_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR5", 0xe6060114, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR5", 0xe6060114, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -4947,9 +4919,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_5_3_FN,	GPSR5_3,
 		GP_5_2_FN,	GPSR5_2,
 		GP_5_1_FN,	GPSR5_1,
-		GP_5_0_FN,	GPSR5_0, }
+		GP_5_0_FN,	GPSR5_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR6", 0xe6060118, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR6", 0xe6060118, 32, 1, GROUP(
 		GP_6_31_FN,	GPSR6_31,
 		GP_6_30_FN,	GPSR6_30,
 		GP_6_29_FN,	GPSR6_29,
@@ -4981,9 +4953,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_6_3_FN,	GPSR6_3,
 		GP_6_2_FN,	GPSR6_2,
 		GP_6_1_FN,	GPSR6_1,
-		GP_6_0_FN,	GPSR6_0, }
+		GP_6_0_FN,	GPSR6_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR7", 0xe606011c, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR7", 0xe606011c, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -5015,14 +4987,14 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_7_3_FN, GPSR7_3,
 		GP_7_2_FN, GPSR7_2,
 		GP_7_1_FN, GPSR7_1,
-		GP_7_0_FN, GPSR7_0, }
+		GP_7_0_FN, GPSR7_0, ))
 	},
 #undef F_
 #undef FM
 
 #define F_(x, y)	x,
 #define FM(x)		FN_##x,
-	{ PINMUX_CFG_REG("IPSR0", 0xe6060200, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR0", 0xe6060200, 32, 4, GROUP(
 		IP0_31_28
 		IP0_27_24
 		IP0_23_20
@@ -5030,9 +5002,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP0_15_12
 		IP0_11_8
 		IP0_7_4
-		IP0_3_0 }
+		IP0_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR1", 0xe6060204, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR1", 0xe6060204, 32, 4, GROUP(
 		IP1_31_28
 		IP1_27_24
 		IP1_23_20
@@ -5040,9 +5012,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP1_15_12
 		IP1_11_8
 		IP1_7_4
-		IP1_3_0 }
+		IP1_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR2", 0xe6060208, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR2", 0xe6060208, 32, 4, GROUP(
 		IP2_31_28
 		IP2_27_24
 		IP2_23_20
@@ -5050,9 +5022,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP2_15_12
 		IP2_11_8
 		IP2_7_4
-		IP2_3_0 }
+		IP2_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR3", 0xe606020c, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR3", 0xe606020c, 32, 4, GROUP(
 		IP3_31_28
 		IP3_27_24
 		IP3_23_20
@@ -5060,9 +5032,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP3_15_12
 		IP3_11_8
 		IP3_7_4
-		IP3_3_0 }
+		IP3_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR4", 0xe6060210, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR4", 0xe6060210, 32, 4, GROUP(
 		IP4_31_28
 		IP4_27_24
 		IP4_23_20
@@ -5070,9 +5042,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP4_15_12
 		IP4_11_8
 		IP4_7_4
-		IP4_3_0 }
+		IP4_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR5", 0xe6060214, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR5", 0xe6060214, 32, 4, GROUP(
 		IP5_31_28
 		IP5_27_24
 		IP5_23_20
@@ -5080,9 +5052,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP5_15_12
 		IP5_11_8
 		IP5_7_4
-		IP5_3_0 }
+		IP5_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR6", 0xe6060218, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR6", 0xe6060218, 32, 4, GROUP(
 		IP6_31_28
 		IP6_27_24
 		IP6_23_20
@@ -5090,9 +5062,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP6_15_12
 		IP6_11_8
 		IP6_7_4
-		IP6_3_0 }
+		IP6_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR7", 0xe606021c, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR7", 0xe606021c, 32, 4, GROUP(
 		IP7_31_28
 		IP7_27_24
 		IP7_23_20
@@ -5100,9 +5072,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP7_15_12
 		IP7_11_8
 		IP7_7_4
-		IP7_3_0 }
+		IP7_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR8", 0xe6060220, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR8", 0xe6060220, 32, 4, GROUP(
 		IP8_31_28
 		IP8_27_24
 		IP8_23_20
@@ -5110,9 +5082,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP8_15_12
 		IP8_11_8
 		IP8_7_4
-		IP8_3_0 }
+		IP8_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR9", 0xe6060224, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR9", 0xe6060224, 32, 4, GROUP(
 		IP9_31_28
 		IP9_27_24
 		IP9_23_20
@@ -5120,9 +5092,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP9_15_12
 		IP9_11_8
 		IP9_7_4
-		IP9_3_0 }
+		IP9_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR10", 0xe6060228, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR10", 0xe6060228, 32, 4, GROUP(
 		IP10_31_28
 		IP10_27_24
 		IP10_23_20
@@ -5130,9 +5102,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP10_15_12
 		IP10_11_8
 		IP10_7_4
-		IP10_3_0 }
+		IP10_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR11", 0xe606022c, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR11", 0xe606022c, 32, 4, GROUP(
 		IP11_31_28
 		IP11_27_24
 		IP11_23_20
@@ -5140,9 +5112,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP11_15_12
 		IP11_11_8
 		IP11_7_4
-		IP11_3_0 }
+		IP11_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR12", 0xe6060230, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR12", 0xe6060230, 32, 4, GROUP(
 		IP12_31_28
 		IP12_27_24
 		IP12_23_20
@@ -5150,9 +5122,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP12_15_12
 		IP12_11_8
 		IP12_7_4
-		IP12_3_0 }
+		IP12_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR13", 0xe6060234, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR13", 0xe6060234, 32, 4, GROUP(
 		IP13_31_28
 		IP13_27_24
 		IP13_23_20
@@ -5160,9 +5132,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP13_15_12
 		IP13_11_8
 		IP13_7_4
-		IP13_3_0 }
+		IP13_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR14", 0xe6060238, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR14", 0xe6060238, 32, 4, GROUP(
 		IP14_31_28
 		IP14_27_24
 		IP14_23_20
@@ -5170,9 +5142,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP14_15_12
 		IP14_11_8
 		IP14_7_4
-		IP14_3_0 }
+		IP14_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR15", 0xe606023c, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR15", 0xe606023c, 32, 4, GROUP(
 		IP15_31_28
 		IP15_27_24
 		IP15_23_20
@@ -5180,9 +5152,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP15_15_12
 		IP15_11_8
 		IP15_7_4
-		IP15_3_0 }
+		IP15_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR16", 0xe6060240, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR16", 0xe6060240, 32, 4, GROUP(
 		IP16_31_28
 		IP16_27_24
 		IP16_23_20
@@ -5190,9 +5162,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP16_15_12
 		IP16_11_8
 		IP16_7_4
-		IP16_3_0 }
+		IP16_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR17", 0xe6060244, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR17", 0xe6060244, 32, 4, GROUP(
 		/* IP17_31_28 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP17_27_24 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP17_23_20 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -5200,7 +5172,7 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP17_15_12 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP17_11_8  */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		IP17_7_4
-		IP17_3_0 }
+		IP17_3_0 ))
 	},
 #undef F_
 #undef FM
@@ -5208,8 +5180,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 #define F_(x, y)	x,
 #define FM(x)		FN_##x,
 	{ PINMUX_CFG_REG_VAR("MOD_SEL0", 0xe6060500, 32,
-			     1, 2, 2, 3, 1, 1, 2, 1, 1, 1,
-			     2, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 2, 1) {
+			     GROUP(1, 2, 2, 3, 1, 1, 2, 1, 1, 1, 2, 1,
+				   1, 1, 1, 1, 1, 1, 2, 2, 1, 2, 1),
+			     GROUP(
 		0, 0, /* RESERVED 31 */
 		MOD_SEL0_30_29
 		MOD_SEL0_28_27
@@ -5232,11 +5205,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		MOD_SEL0_5_4
 		MOD_SEL0_3
 		MOD_SEL0_2_1
-		0, 0, /* RESERVED 0 */ }
+		0, 0, /* RESERVED 0 */ ))
 	},
 	{ PINMUX_CFG_REG_VAR("MOD_SEL1", 0xe6060504, 32,
-			     2, 3, 1, 2, 3, 1, 1, 2, 1,
-			     2, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1) {
+			     GROUP(2, 3, 1, 2, 3, 1, 1, 2, 1, 2, 1, 1,
+				   1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1),
+			     GROUP(
 		MOD_SEL1_31_30
 		MOD_SEL1_29_28_27
 		MOD_SEL1_26
@@ -5259,11 +5233,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		MOD_SEL1_3
 		MOD_SEL1_2
 		MOD_SEL1_1
-		MOD_SEL1_0 }
+		MOD_SEL1_0 ))
 	},
 	{ PINMUX_CFG_REG_VAR("MOD_SEL2", 0xe6060508, 32,
-			     1, 1, 1, 1, 4, 4, 4,
-			     4, 4, 4, 1, 2, 1) {
+			     GROUP(1, 1, 1, 1, 4, 4, 4, 4, 4, 4, 1, 2, 1),
+			     GROUP(
 		MOD_SEL2_31
 		MOD_SEL2_30
 		MOD_SEL2_29
@@ -5291,7 +5265,7 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0,
 		/* RESERVED 2, 1 */
 		0, 0, 0, 0,
-		MOD_SEL2_0 }
+		MOD_SEL2_0 ))
 	},
 	{ },
 };
@@ -5412,8 +5386,8 @@ static const struct pinmux_drive_reg pinmux_drive_regs[] = {
 		{ RCAR_GP_PIN(0, 15),   24, 3 },	/* D15 */
 		{ RCAR_GP_PIN(7,  0),   20, 3 },	/* AVS1 */
 		{ RCAR_GP_PIN(7,  1),   16, 3 },	/* AVS2 */
-		{ RCAR_GP_PIN(7,  2),   12, 3 },	/* HDMI0_CEC */
-		{ RCAR_GP_PIN(7,  3),    8, 3 },	/* HDMI1_CEC */
+		{ RCAR_GP_PIN(7,  2),   12, 3 },	/* GP7_02 */
+		{ RCAR_GP_PIN(7,  3),    8, 3 },	/* GP7_03 */
 		{ PIN_A_NUMBER('P', 7),  4, 2 },	/* DU_DOTCLKIN0 */
 		{ PIN_A_NUMBER('P', 8),  0, 2 },	/* DU_DOTCLKIN1 */
 	} },
@@ -5474,11 +5448,11 @@ static const struct pinmux_drive_reg pinmux_drive_regs[] = {
 		{ RCAR_GP_PIN(5,  3),  0, 3 },	/* CTS0 */
 	} },
 	{ PINMUX_DRIVE_REG("DRVCTRL18", 0xe6060348) {
-		{ RCAR_GP_PIN(5,  4), 28, 3 },	/* RTS0_TANS */
+		{ RCAR_GP_PIN(5,  4), 28, 3 },	/* RTS0 */
 		{ RCAR_GP_PIN(5,  5), 24, 3 },	/* RX1 */
 		{ RCAR_GP_PIN(5,  6), 20, 3 },	/* TX1 */
 		{ RCAR_GP_PIN(5,  7), 16, 3 },	/* CTS1 */
-		{ RCAR_GP_PIN(5,  8), 12, 3 },	/* RTS1_TANS */
+		{ RCAR_GP_PIN(5,  8), 12, 3 },	/* RTS1 */
 		{ RCAR_GP_PIN(5,  9),  8, 3 },	/* SCK2 */
 		{ RCAR_GP_PIN(5, 10),  4, 3 },	/* TX2 */
 		{ RCAR_GP_PIN(5, 11),  0, 3 },	/* RX2 */
@@ -5547,10 +5521,12 @@ static const struct pinmux_drive_reg pinmux_drive_regs[] = {
 
 enum ioctrl_regs {
 	POCCTRL,
+	TDSELCTRL,
 };
 
 static const struct pinmux_ioctrl_reg pinmux_ioctrl_regs[] = {
 	[POCCTRL] = { 0xe6060380, },
+	[TDSELCTRL] = { 0xe60603c0, },
 	{ /* sentinel */ },
 };
 
@@ -5668,8 +5644,8 @@ static const struct pinmux_bias_reg pinmux_bias_regs[] = {
 		[25] = RCAR_GP_PIN(0, 15),	/* D15 */
 		[26] = RCAR_GP_PIN(7,  0),	/* AVS1 */
 		[27] = RCAR_GP_PIN(7,  1),	/* AVS2 */
-		[28] = RCAR_GP_PIN(7,  2),	/* HDMI0_CEC */
-		[29] = RCAR_GP_PIN(7,  3),	/* HDMI1_CEC */
+		[28] = RCAR_GP_PIN(7,  2),	/* GP7_02 */
+		[29] = RCAR_GP_PIN(7,  3),	/* GP7_03 */
 		[30] = PIN_A_NUMBER('P', 7),	/* DU_DOTCLKIN0 */
 		[31] = PIN_A_NUMBER('P', 8),	/* DU_DOTCLKIN1 */
 	} },
@@ -5724,11 +5700,11 @@ static const struct pinmux_bias_reg pinmux_bias_regs[] = {
 		[13] = RCAR_GP_PIN(5,  1),	/* RX0 */
 		[14] = RCAR_GP_PIN(5,  2),	/* TX0 */
 		[15] = RCAR_GP_PIN(5,  3),	/* CTS0_N */
-		[16] = RCAR_GP_PIN(5,  4),	/* RTS0_N_TANS */
+		[16] = RCAR_GP_PIN(5,  4),	/* RTS0_N */
 		[17] = RCAR_GP_PIN(5,  5),	/* RX1_A */
 		[18] = RCAR_GP_PIN(5,  6),	/* TX1_A */
 		[19] = RCAR_GP_PIN(5,  7),	/* CTS1_N */
-		[20] = RCAR_GP_PIN(5,  8),	/* RTS1_N_TANS */
+		[20] = RCAR_GP_PIN(5,  8),	/* RTS1_N */
 		[21] = RCAR_GP_PIN(5,  9),	/* SCK2 */
 		[22] = RCAR_GP_PIN(5, 10),	/* TX2_A */
 		[23] = RCAR_GP_PIN(5, 11),	/* RX2_A */
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a7795.c b/drivers/pinctrl/sh-pfc/pfc-r8a7795.c
index db9add1405c5..68bcb8980b16 100644
--- a/drivers/pinctrl/sh-pfc/pfc-r8a7795.c
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a7795.c
@@ -2,9 +2,10 @@
 /*
  * R8A7795 ES2.0+ processor support - PFC hardware block.
  *
- * Copyright (C) 2015-2017 Renesas Electronics Corporation
+ * Copyright (C) 2015-2019 Renesas Electronics Corporation
  */
 
+#include <linux/errno.h>
 #include <linux/kernel.h>
 #include <linux/sys_soc.h>
 
@@ -200,8 +201,8 @@
 #define GPSR6_0		F_(SSI_SCK01239,		IP14_23_20)
 
 /* GPSR7 */
-#define GPSR7_3		FM(HDMI1_CEC)
-#define GPSR7_2		FM(HDMI0_CEC)
+#define GPSR7_3		FM(GP7_03)
+#define GPSR7_2		FM(GP7_02)
 #define GPSR7_1		FM(AVS2)
 #define GPSR7_0		FM(AVS1)
 
@@ -350,7 +351,7 @@
 #define IP16_23_20	FM(SSI_SDATA7)		FM(HCTS2_N_B)	FM(MSIOF1_RXD_C)	F_(0, 0)			F_(0, 0)	FM(TS_SDEN1_A)	FM(STP_ISEN_1_A)	FM(RIF1_D0_A)	FM(RIF3_D0_A)		F_(0, 0)	FM(TCLK2_A)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP16_27_24	FM(SSI_SDATA8)		FM(HRTS2_N_B)	FM(MSIOF1_TXD_C)	F_(0, 0)			F_(0, 0)	FM(TS_SPSYNC1_A)FM(STP_ISSYNC_1_A)	FM(RIF1_D1_A)	FM(RIF3_D1_A)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP16_31_28	FM(SSI_SDATA9_A)	FM(HSCK2_B)	FM(MSIOF1_SS1_C)	FM(HSCK1_A)			FM(SSI_WS1_B)	FM(SCK1)	FM(STP_IVCXO27_1_A)	FM(SCK5_A)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP17_3_0	FM(AUDIO_CLKA_A)	F_(0, 0)	F_(0, 0)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	FM(CC5_OSCOUT)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP17_3_0	FM(AUDIO_CLKA_A)	F_(0, 0)	F_(0, 0)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP17_7_4	FM(AUDIO_CLKB_B)	FM(SCIF_CLK_A)	F_(0, 0)		F_(0, 0)			F_(0, 0)	F_(0, 0)	FM(STP_IVCXO27_1_D)	FM(REMOCON_A)	F_(0, 0)		F_(0, 0)	FM(TCLK1_A)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP17_11_8	FM(USB0_PWEN)		F_(0, 0)	F_(0, 0)		FM(SIM0_RST_C)			F_(0, 0)	FM(TS_SCK1_D)	FM(STP_ISCLK_1_D)	FM(BPFCLK_B)	FM(RIF3_CLK_B)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) FM(HSCK2_C) F_(0, 0) F_(0, 0)
 #define IP17_15_12	FM(USB0_OVC)		F_(0, 0)	F_(0, 0)		FM(SIM0_D_C)			F_(0, 0)	FM(TS_SDAT1_D)	FM(STP_ISD_1_D)		F_(0, 0)	FM(RIF3_SYNC_B)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) FM(HRX2_C) F_(0, 0) F_(0, 0)
@@ -461,7 +462,7 @@ FM(IP16_31_28)	IP16_31_28	FM(IP17_31_28)	IP17_31_28
 #define MOD_SEL0_9_8		FM(SEL_DRIF1_0)		FM(SEL_DRIF1_1)		FM(SEL_DRIF1_2)		F_(0, 0)
 #define MOD_SEL0_7_6		FM(SEL_DRIF0_0)		FM(SEL_DRIF0_1)		FM(SEL_DRIF0_2)		F_(0, 0)
 #define MOD_SEL0_5		FM(SEL_CANFD0_0)	FM(SEL_CANFD0_1)
-#define MOD_SEL0_4_3		FM(SEL_ADG_A_0)		FM(SEL_ADG_A_1)		FM(SEL_ADG_A_2)		FM(SEL_ADG_A_3)
+#define MOD_SEL0_4_3		FM(SEL_ADGA_0)		FM(SEL_ADGA_1)		FM(SEL_ADGA_2)		FM(SEL_ADGA_3)
 
 /* MOD_SEL1 */			/* 0 */			/* 1 */			/* 2 */			/* 3 */			/* 4 */			/* 5 */			/* 6 */			/* 7 */
 #define MOD_SEL1_31_30		FM(SEL_TSIF1_0)		FM(SEL_TSIF1_1)		FM(SEL_TSIF1_2)		FM(SEL_TSIF1_3)
@@ -497,8 +498,8 @@ FM(IP16_31_28)	IP16_31_28	FM(IP17_31_28)	IP17_31_28
 #define MOD_SEL2_21		FM(SEL_SSI2_0)		FM(SEL_SSI2_1)
 #define MOD_SEL2_20		FM(SEL_SSI9_0)		FM(SEL_SSI9_1)
 #define MOD_SEL2_19		FM(SEL_TIMER_TMU2_0)	FM(SEL_TIMER_TMU2_1)
-#define MOD_SEL2_18		FM(SEL_ADG_B_0)		FM(SEL_ADG_B_1)
-#define MOD_SEL2_17		FM(SEL_ADG_C_0)		FM(SEL_ADG_C_1)
+#define MOD_SEL2_18		FM(SEL_ADGB_0)		FM(SEL_ADGB_1)
+#define MOD_SEL2_17		FM(SEL_ADGC_0)		FM(SEL_ADGC_1)
 #define MOD_SEL2_0		FM(SEL_VIN4_0)		FM(SEL_VIN4_1)
 
 #define PINMUX_MOD_SELS	\
@@ -590,8 +591,8 @@ static const u16 pinmux_data[] = {
 	PINMUX_SINGLE(AVS1),
 	PINMUX_SINGLE(AVS2),
 	PINMUX_SINGLE(CLKOUT),
-	PINMUX_SINGLE(HDMI0_CEC),
-	PINMUX_SINGLE(HDMI1_CEC),
+	PINMUX_SINGLE(GP7_02),
+	PINMUX_SINGLE(GP7_03),
 	PINMUX_SINGLE(MSIOF0_RXD),
 	PINMUX_SINGLE(MSIOF0_SCK),
 	PINMUX_SINGLE(MSIOF0_TXD),
@@ -1129,7 +1130,7 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_GPSR(IP11_27_24,	SCK0),
 	PINMUX_IPSR_MSEL(IP11_27_24,	HSCK1_B,		SEL_HSCIF1_1),
 	PINMUX_IPSR_MSEL(IP11_27_24,	MSIOF1_SS2_B,		SEL_MSIOF1_1),
-	PINMUX_IPSR_MSEL(IP11_27_24,	AUDIO_CLKC_B,		SEL_ADG_C_1),
+	PINMUX_IPSR_MSEL(IP11_27_24,	AUDIO_CLKC_B,		SEL_ADGC_1),
 	PINMUX_IPSR_MSEL(IP11_27_24,	SDA2_A,			SEL_I2C2_0),
 	PINMUX_IPSR_MSEL(IP11_27_24,	SIM0_RST_B,		SEL_SIMCARD_1),
 	PINMUX_IPSR_MSEL(IP11_27_24,	STP_OPWM_0_C,		SEL_SSP1_0_2),
@@ -1162,7 +1163,7 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_GPSR(IP12_11_8,	RTS0_N),
 	PINMUX_IPSR_MSEL(IP12_11_8,	HRTS1_N_B,		SEL_HSCIF1_1),
 	PINMUX_IPSR_MSEL(IP12_11_8,	MSIOF1_SS1_B,		SEL_MSIOF1_1),
-	PINMUX_IPSR_MSEL(IP12_11_8,	AUDIO_CLKA_B,		SEL_ADG_A_1),
+	PINMUX_IPSR_MSEL(IP12_11_8,	AUDIO_CLKA_B,		SEL_ADGA_1),
 	PINMUX_IPSR_MSEL(IP12_11_8,	SCL2_A,			SEL_I2C2_0),
 	PINMUX_IPSR_MSEL(IP12_11_8,	STP_IVCXO27_1_C,	SEL_SSP1_1_2),
 	PINMUX_IPSR_MSEL(IP12_11_8,	RIF0_SYNC_B,		SEL_DRIF0_1),
@@ -1221,7 +1222,7 @@ static const u16 pinmux_data[] = {
 
 	PINMUX_IPSR_GPSR(IP13_11_8,	HSCK0),
 	PINMUX_IPSR_MSEL(IP13_11_8,	MSIOF1_SCK_D,		SEL_MSIOF1_3),
-	PINMUX_IPSR_MSEL(IP13_11_8,	AUDIO_CLKB_A,		SEL_ADG_B_0),
+	PINMUX_IPSR_MSEL(IP13_11_8,	AUDIO_CLKB_A,		SEL_ADGB_0),
 	PINMUX_IPSR_MSEL(IP13_11_8,	SSI_SDATA1_B,		SEL_SSI1_1),
 	PINMUX_IPSR_MSEL(IP13_11_8,	TS_SCK0_D,		SEL_TSIF0_3),
 	PINMUX_IPSR_MSEL(IP13_11_8,	STP_ISCLK_0_D,		SEL_SSP1_0_3),
@@ -1268,7 +1269,7 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_GPSR(IP14_3_0,	MSIOF0_SS1),
 	PINMUX_IPSR_MSEL(IP14_3_0,	RX5_A,			SEL_SCIF5_0),
 	PINMUX_IPSR_GPSR(IP14_3_0,	NFWP_N_A),
-	PINMUX_IPSR_MSEL(IP14_3_0,	AUDIO_CLKA_C,		SEL_ADG_A_2),
+	PINMUX_IPSR_MSEL(IP14_3_0,	AUDIO_CLKA_C,		SEL_ADGA_2),
 	PINMUX_IPSR_MSEL(IP14_3_0,	SSI_SCK2_A,		SEL_SSI2_0),
 	PINMUX_IPSR_MSEL(IP14_3_0,	STP_IVCXO27_0_C,	SEL_SSP1_0_2),
 	PINMUX_IPSR_GPSR(IP14_3_0,	AUDIO_CLKOUT3_A),
@@ -1277,7 +1278,7 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_GPSR(IP14_7_4,	MSIOF0_SS2),
 	PINMUX_IPSR_MSEL(IP14_7_4,	TX5_A,			SEL_SCIF5_0),
 	PINMUX_IPSR_MSEL(IP14_7_4,	MSIOF1_SS2_D,		SEL_MSIOF1_3),
-	PINMUX_IPSR_MSEL(IP14_7_4,	AUDIO_CLKC_A,		SEL_ADG_C_0),
+	PINMUX_IPSR_MSEL(IP14_7_4,	AUDIO_CLKC_A,		SEL_ADGC_0),
 	PINMUX_IPSR_MSEL(IP14_7_4,	SSI_WS2_A,		SEL_SSI2_0),
 	PINMUX_IPSR_MSEL(IP14_7_4,	STP_OPWM_0_D,		SEL_SSP1_0_3),
 	PINMUX_IPSR_GPSR(IP14_7_4,	AUDIO_CLKOUT_D),
@@ -1408,10 +1409,9 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_MSEL(IP16_31_28,	SCK5_A,			SEL_SCIF5_0),
 
 	/* IPSR17 */
-	PINMUX_IPSR_MSEL(IP17_3_0,	AUDIO_CLKA_A,		SEL_ADG_A_0),
-	PINMUX_IPSR_GPSR(IP17_3_0,	CC5_OSCOUT),
+	PINMUX_IPSR_MSEL(IP17_3_0,	AUDIO_CLKA_A,		SEL_ADGA_0),
 
-	PINMUX_IPSR_MSEL(IP17_7_4,	AUDIO_CLKB_B,		SEL_ADG_B_1),
+	PINMUX_IPSR_MSEL(IP17_7_4,	AUDIO_CLKB_B,		SEL_ADGB_1),
 	PINMUX_IPSR_MSEL(IP17_7_4,	SCIF_CLK_A,		SEL_SCIF_0),
 	PINMUX_IPSR_MSEL(IP17_7_4,	STP_IVCXO27_1_D,	SEL_SSP1_1_3),
 	PINMUX_IPSR_MSEL(IP17_7_4,	REMOCON_A,		SEL_REMOCON_0),
@@ -2131,22 +2131,6 @@ static const unsigned int du_disp_mux[] = {
 	DU_DISP_MARK,
 };
 
-/* - HDMI ------------------------------------------------------------------- */
-static const unsigned int hdmi0_cec_pins[] = {
-	/* HDMI0_CEC */
-	RCAR_GP_PIN(7, 2),
-};
-static const unsigned int hdmi0_cec_mux[] = {
-	HDMI0_CEC_MARK,
-};
-static const unsigned int hdmi1_cec_pins[] = {
-	/* HDMI1_CEC */
-	RCAR_GP_PIN(7, 3),
-};
-static const unsigned int hdmi1_cec_mux[] = {
-	HDMI1_CEC_MARK,
-};
-
 /* - HSCIF0 ----------------------------------------------------------------- */
 static const unsigned int hscif0_data_pins[] = {
 	/* RX, TX */
@@ -4225,8 +4209,6 @@ static const struct sh_pfc_pin_group pinmux_groups[] = {
 	SH_PFC_PIN_GROUP(du_oddf),
 	SH_PFC_PIN_GROUP(du_cde),
 	SH_PFC_PIN_GROUP(du_disp),
-	SH_PFC_PIN_GROUP(hdmi0_cec),
-	SH_PFC_PIN_GROUP(hdmi1_cec),
 	SH_PFC_PIN_GROUP(hscif0_data),
 	SH_PFC_PIN_GROUP(hscif0_clk),
 	SH_PFC_PIN_GROUP(hscif0_ctrl),
@@ -4611,14 +4593,6 @@ static const char * const du_groups[] = {
 	"du_disp",
 };
 
-static const char * const hdmi0_groups[] = {
-	"hdmi0_cec",
-};
-
-static const char * const hdmi1_groups[] = {
-	"hdmi1_cec",
-};
-
 static const char * const hscif0_groups[] = {
 	"hscif0_data",
 	"hscif0_clk",
@@ -5037,8 +5011,6 @@ static const struct sh_pfc_function pinmux_functions[] = {
 	SH_PFC_FUNCTION(drif2),
 	SH_PFC_FUNCTION(drif3),
 	SH_PFC_FUNCTION(du),
-	SH_PFC_FUNCTION(hdmi0),
-	SH_PFC_FUNCTION(hdmi1),
 	SH_PFC_FUNCTION(hscif0),
 	SH_PFC_FUNCTION(hscif1),
 	SH_PFC_FUNCTION(hscif2),
@@ -5088,7 +5060,7 @@ static const struct sh_pfc_function pinmux_functions[] = {
 static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 #define F_(x, y)	FN_##y
 #define FM(x)		FN_##x
-	{ PINMUX_CFG_REG("GPSR0", 0xe6060100, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR0", 0xe6060100, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -5120,9 +5092,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_0_3_FN,	GPSR0_3,
 		GP_0_2_FN,	GPSR0_2,
 		GP_0_1_FN,	GPSR0_1,
-		GP_0_0_FN,	GPSR0_0, }
+		GP_0_0_FN,	GPSR0_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR1", 0xe6060104, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR1", 0xe6060104, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -5154,9 +5126,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_1_3_FN,	GPSR1_3,
 		GP_1_2_FN,	GPSR1_2,
 		GP_1_1_FN,	GPSR1_1,
-		GP_1_0_FN,	GPSR1_0, }
+		GP_1_0_FN,	GPSR1_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR2", 0xe6060108, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR2", 0xe6060108, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -5188,9 +5160,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_2_3_FN,	GPSR2_3,
 		GP_2_2_FN,	GPSR2_2,
 		GP_2_1_FN,	GPSR2_1,
-		GP_2_0_FN,	GPSR2_0, }
+		GP_2_0_FN,	GPSR2_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR3", 0xe606010c, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR3", 0xe606010c, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -5222,9 +5194,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_3_3_FN,	GPSR3_3,
 		GP_3_2_FN,	GPSR3_2,
 		GP_3_1_FN,	GPSR3_1,
-		GP_3_0_FN,	GPSR3_0, }
+		GP_3_0_FN,	GPSR3_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR4", 0xe6060110, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR4", 0xe6060110, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -5256,9 +5228,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_4_3_FN,	GPSR4_3,
 		GP_4_2_FN,	GPSR4_2,
 		GP_4_1_FN,	GPSR4_1,
-		GP_4_0_FN,	GPSR4_0, }
+		GP_4_0_FN,	GPSR4_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR5", 0xe6060114, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR5", 0xe6060114, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -5290,9 +5262,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_5_3_FN,	GPSR5_3,
 		GP_5_2_FN,	GPSR5_2,
 		GP_5_1_FN,	GPSR5_1,
-		GP_5_0_FN,	GPSR5_0, }
+		GP_5_0_FN,	GPSR5_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR6", 0xe6060118, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR6", 0xe6060118, 32, 1, GROUP(
 		GP_6_31_FN,	GPSR6_31,
 		GP_6_30_FN,	GPSR6_30,
 		GP_6_29_FN,	GPSR6_29,
@@ -5324,9 +5296,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_6_3_FN,	GPSR6_3,
 		GP_6_2_FN,	GPSR6_2,
 		GP_6_1_FN,	GPSR6_1,
-		GP_6_0_FN,	GPSR6_0, }
+		GP_6_0_FN,	GPSR6_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR7", 0xe606011c, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR7", 0xe606011c, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -5358,14 +5330,14 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_7_3_FN, GPSR7_3,
 		GP_7_2_FN, GPSR7_2,
 		GP_7_1_FN, GPSR7_1,
-		GP_7_0_FN, GPSR7_0, }
+		GP_7_0_FN, GPSR7_0, ))
 	},
 #undef F_
 #undef FM
 
 #define F_(x, y)	x,
 #define FM(x)		FN_##x,
-	{ PINMUX_CFG_REG("IPSR0", 0xe6060200, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR0", 0xe6060200, 32, 4, GROUP(
 		IP0_31_28
 		IP0_27_24
 		IP0_23_20
@@ -5373,9 +5345,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP0_15_12
 		IP0_11_8
 		IP0_7_4
-		IP0_3_0 }
+		IP0_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR1", 0xe6060204, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR1", 0xe6060204, 32, 4, GROUP(
 		IP1_31_28
 		IP1_27_24
 		IP1_23_20
@@ -5383,9 +5355,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP1_15_12
 		IP1_11_8
 		IP1_7_4
-		IP1_3_0 }
+		IP1_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR2", 0xe6060208, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR2", 0xe6060208, 32, 4, GROUP(
 		IP2_31_28
 		IP2_27_24
 		IP2_23_20
@@ -5393,9 +5365,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP2_15_12
 		IP2_11_8
 		IP2_7_4
-		IP2_3_0 }
+		IP2_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR3", 0xe606020c, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR3", 0xe606020c, 32, 4, GROUP(
 		IP3_31_28
 		IP3_27_24
 		IP3_23_20
@@ -5403,9 +5375,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP3_15_12
 		IP3_11_8
 		IP3_7_4
-		IP3_3_0 }
+		IP3_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR4", 0xe6060210, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR4", 0xe6060210, 32, 4, GROUP(
 		IP4_31_28
 		IP4_27_24
 		IP4_23_20
@@ -5413,9 +5385,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP4_15_12
 		IP4_11_8
 		IP4_7_4
-		IP4_3_0 }
+		IP4_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR5", 0xe6060214, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR5", 0xe6060214, 32, 4, GROUP(
 		IP5_31_28
 		IP5_27_24
 		IP5_23_20
@@ -5423,9 +5395,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP5_15_12
 		IP5_11_8
 		IP5_7_4
-		IP5_3_0 }
+		IP5_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR6", 0xe6060218, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR6", 0xe6060218, 32, 4, GROUP(
 		IP6_31_28
 		IP6_27_24
 		IP6_23_20
@@ -5433,9 +5405,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP6_15_12
 		IP6_11_8
 		IP6_7_4
-		IP6_3_0 }
+		IP6_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR7", 0xe606021c, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR7", 0xe606021c, 32, 4, GROUP(
 		IP7_31_28
 		IP7_27_24
 		IP7_23_20
@@ -5443,9 +5415,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP7_15_12 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		IP7_11_8
 		IP7_7_4
-		IP7_3_0 }
+		IP7_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR8", 0xe6060220, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR8", 0xe6060220, 32, 4, GROUP(
 		IP8_31_28
 		IP8_27_24
 		IP8_23_20
@@ -5453,9 +5425,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP8_15_12
 		IP8_11_8
 		IP8_7_4
-		IP8_3_0 }
+		IP8_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR9", 0xe6060224, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR9", 0xe6060224, 32, 4, GROUP(
 		IP9_31_28
 		IP9_27_24
 		IP9_23_20
@@ -5463,9 +5435,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP9_15_12
 		IP9_11_8
 		IP9_7_4
-		IP9_3_0 }
+		IP9_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR10", 0xe6060228, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR10", 0xe6060228, 32, 4, GROUP(
 		IP10_31_28
 		IP10_27_24
 		IP10_23_20
@@ -5473,9 +5445,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP10_15_12
 		IP10_11_8
 		IP10_7_4
-		IP10_3_0 }
+		IP10_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR11", 0xe606022c, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR11", 0xe606022c, 32, 4, GROUP(
 		IP11_31_28
 		IP11_27_24
 		IP11_23_20
@@ -5483,9 +5455,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP11_15_12
 		IP11_11_8
 		IP11_7_4
-		IP11_3_0 }
+		IP11_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR12", 0xe6060230, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR12", 0xe6060230, 32, 4, GROUP(
 		IP12_31_28
 		IP12_27_24
 		IP12_23_20
@@ -5493,9 +5465,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP12_15_12
 		IP12_11_8
 		IP12_7_4
-		IP12_3_0 }
+		IP12_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR13", 0xe6060234, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR13", 0xe6060234, 32, 4, GROUP(
 		IP13_31_28
 		IP13_27_24
 		IP13_23_20
@@ -5503,9 +5475,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP13_15_12
 		IP13_11_8
 		IP13_7_4
-		IP13_3_0 }
+		IP13_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR14", 0xe6060238, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR14", 0xe6060238, 32, 4, GROUP(
 		IP14_31_28
 		IP14_27_24
 		IP14_23_20
@@ -5513,9 +5485,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP14_15_12
 		IP14_11_8
 		IP14_7_4
-		IP14_3_0 }
+		IP14_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR15", 0xe606023c, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR15", 0xe606023c, 32, 4, GROUP(
 		IP15_31_28
 		IP15_27_24
 		IP15_23_20
@@ -5523,9 +5495,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP15_15_12
 		IP15_11_8
 		IP15_7_4
-		IP15_3_0 }
+		IP15_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR16", 0xe6060240, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR16", 0xe6060240, 32, 4, GROUP(
 		IP16_31_28
 		IP16_27_24
 		IP16_23_20
@@ -5533,9 +5505,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP16_15_12
 		IP16_11_8
 		IP16_7_4
-		IP16_3_0 }
+		IP16_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR17", 0xe6060244, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR17", 0xe6060244, 32, 4, GROUP(
 		IP17_31_28
 		IP17_27_24
 		IP17_23_20
@@ -5543,9 +5515,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP17_15_12
 		IP17_11_8
 		IP17_7_4
-		IP17_3_0 }
+		IP17_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR18", 0xe6060248, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR18", 0xe6060248, 32, 4, GROUP(
 		/* IP18_31_28 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP18_27_24 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP18_23_20 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -5553,7 +5525,7 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP18_15_12 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP18_11_8  */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		IP18_7_4
-		IP18_3_0 }
+		IP18_3_0 ))
 	},
 #undef F_
 #undef FM
@@ -5561,8 +5533,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 #define F_(x, y)	x,
 #define FM(x)		FN_##x,
 	{ PINMUX_CFG_REG_VAR("MOD_SEL0", 0xe6060500, 32,
-			     3, 2, 3, 1, 1, 1, 1, 1, 2, 1,
-			     1, 2, 1, 1, 1, 2, 2, 1, 2, 3) {
+			     GROUP(3, 2, 3, 1, 1, 1, 1, 1, 2, 1, 1, 2,
+				   1, 1, 1, 2, 2, 1, 2, 3),
+			     GROUP(
 		MOD_SEL0_31_30_29
 		MOD_SEL0_28_27
 		MOD_SEL0_26_25_24
@@ -5583,11 +5556,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		MOD_SEL0_5
 		MOD_SEL0_4_3
 		/* RESERVED 2, 1, 0 */
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
 	{ PINMUX_CFG_REG_VAR("MOD_SEL1", 0xe6060504, 32,
-			     2, 3, 1, 2, 3, 1, 1, 2, 1,
-			     2, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1) {
+			     GROUP(2, 3, 1, 2, 3, 1, 1, 2, 1, 2, 1, 1,
+				   1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1),
+			     GROUP(
 		MOD_SEL1_31_30
 		MOD_SEL1_29_28_27
 		MOD_SEL1_26
@@ -5610,11 +5584,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		MOD_SEL1_3
 		MOD_SEL1_2
 		MOD_SEL1_1
-		MOD_SEL1_0 }
+		MOD_SEL1_0 ))
 	},
 	{ PINMUX_CFG_REG_VAR("MOD_SEL2", 0xe6060508, 32,
-			     1, 1, 1, 2, 1, 3, 1, 1, 1, 1, 1, 1, 1,
-			     4, 4, 4, 3, 1) {
+			     GROUP(1, 1, 1, 2, 1, 3, 1, 1, 1, 1, 1, 1,
+				   1, 4, 4, 4, 3, 1),
+			     GROUP(
 		MOD_SEL2_31
 		MOD_SEL2_30
 		MOD_SEL2_29
@@ -5641,7 +5616,7 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0,
 		/* RESERVED 3, 2, 1 */
 		0, 0, 0, 0, 0, 0, 0, 0,
-		MOD_SEL2_0 }
+		MOD_SEL2_0 ))
 	},
 	{ },
 };
@@ -5762,8 +5737,8 @@ static const struct pinmux_drive_reg pinmux_drive_regs[] = {
 		{ RCAR_GP_PIN(0, 15),   24, 3 },	/* D15 */
 		{ RCAR_GP_PIN(7,  0),   20, 3 },	/* AVS1 */
 		{ RCAR_GP_PIN(7,  1),   16, 3 },	/* AVS2 */
-		{ RCAR_GP_PIN(7,  2),   12, 3 },	/* HDMI0_CEC */
-		{ RCAR_GP_PIN(7,  3),    8, 3 },	/* HDMI1_CEC */
+		{ RCAR_GP_PIN(7,  2),   12, 3 },	/* GP7_02 */
+		{ RCAR_GP_PIN(7,  3),    8, 3 },	/* GP7_03 */
 		{ PIN_A_NUMBER('P', 7),  4, 2 },	/* DU_DOTCLKIN0 */
 		{ PIN_A_NUMBER('P', 8),  0, 2 },	/* DU_DOTCLKIN1 */
 	} },
@@ -5897,10 +5872,12 @@ static const struct pinmux_drive_reg pinmux_drive_regs[] = {
 
 enum ioctrl_regs {
 	POCCTRL,
+	TDSELCTRL,
 };
 
 static const struct pinmux_ioctrl_reg pinmux_ioctrl_regs[] = {
 	[POCCTRL] = { 0xe6060380, },
+	[TDSELCTRL] = { 0xe60603c0, },
 	{ /* sentinel */ },
 };
 
@@ -6017,8 +5994,8 @@ static const struct pinmux_bias_reg pinmux_bias_regs[] = {
 		[25] = RCAR_GP_PIN(0, 15),	/* D15 */
 		[26] = RCAR_GP_PIN(7,  0),	/* AVS1 */
 		[27] = RCAR_GP_PIN(7,  1),	/* AVS2 */
-		[28] = RCAR_GP_PIN(7,  2),	/* HDMI0_CEC */
-		[29] = RCAR_GP_PIN(7,  3),	/* HDMI1_CEC */
+		[28] = RCAR_GP_PIN(7,  2),	/* GP7_02 */
+		[29] = RCAR_GP_PIN(7,  3),	/* GP7_03 */
 		[30] = PIN_A_NUMBER('P', 7),	/* DU_DOTCLKIN0 */
 		[31] = PIN_A_NUMBER('P', 8),	/* DU_DOTCLKIN1 */
 	} },
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a7796.c b/drivers/pinctrl/sh-pfc/pfc-r8a7796.c
index 72348a4f2ece..38cce690db70 100644
--- a/drivers/pinctrl/sh-pfc/pfc-r8a7796.c
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a7796.c
@@ -2,7 +2,7 @@
 /*
  * R8A7796 processor support - PFC hardware block.
  *
- * Copyright (C) 2016-2017 Renesas Electronics Corp.
+ * Copyright (C) 2016-2019 Renesas Electronics Corp.
  *
  * This file is based on the drivers/pinctrl/sh-pfc/pfc-r8a7795.c
  *
@@ -11,6 +11,7 @@
  * Copyright (C) 2015  Renesas Electronics Corporation
  */
 
+#include <linux/errno.h>
 #include <linux/kernel.h>
 
 #include "core.h"
@@ -206,7 +207,7 @@
 
 /* GPSR7 */
 #define GPSR7_3		FM(GP7_03)
-#define GPSR7_2		FM(HDMI0_CEC)
+#define GPSR7_2		FM(GP7_02)
 #define GPSR7_1		FM(AVS2)
 #define GPSR7_0		FM(AVS1)
 
@@ -355,7 +356,7 @@
 #define IP16_23_20	FM(SSI_SDATA7)		FM(HCTS2_N_B)	FM(MSIOF1_RXD_C)	F_(0, 0)			F_(0, 0)	FM(TS_SDEN1_A)	FM(STP_ISEN_1_A)	FM(RIF1_D0_A)	FM(RIF3_D0_A)		F_(0, 0)	FM(TCLK2_A)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP16_27_24	FM(SSI_SDATA8)		FM(HRTS2_N_B)	FM(MSIOF1_TXD_C)	F_(0, 0)			F_(0, 0)	FM(TS_SPSYNC1_A)FM(STP_ISSYNC_1_A)	FM(RIF1_D1_A)	FM(RIF3_D1_A)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP16_31_28	FM(SSI_SDATA9_A)	FM(HSCK2_B)	FM(MSIOF1_SS1_C)	FM(HSCK1_A)			FM(SSI_WS1_B)	FM(SCK1)	FM(STP_IVCXO27_1_A)	FM(SCK5_A)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP17_3_0	FM(AUDIO_CLKA_A)	F_(0, 0)	F_(0, 0)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	FM(CC5_OSCOUT)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP17_3_0	FM(AUDIO_CLKA_A)	F_(0, 0)	F_(0, 0)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP17_7_4	FM(AUDIO_CLKB_B)	FM(SCIF_CLK_A)	F_(0, 0)		F_(0, 0)			F_(0, 0)	F_(0, 0)	FM(STP_IVCXO27_1_D)	FM(REMOCON_A)	F_(0, 0)		F_(0, 0)	FM(TCLK1_A)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP17_11_8	FM(USB0_PWEN)		F_(0, 0)	F_(0, 0)		FM(SIM0_RST_C)			F_(0, 0)	FM(TS_SCK1_D)	FM(STP_ISCLK_1_D)	FM(BPFCLK_B)	FM(RIF3_CLK_B)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) FM(HSCK2_C) F_(0, 0) F_(0, 0)
 #define IP17_15_12	FM(USB0_OVC)		F_(0, 0)	F_(0, 0)		FM(SIM0_D_C)			F_(0, 0)	FM(TS_SDAT1_D)	FM(STP_ISD_1_D)		F_(0, 0)	FM(RIF3_SYNC_B)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) FM(HRX2_C) F_(0, 0) F_(0, 0)
@@ -466,7 +467,7 @@ FM(IP16_31_28)	IP16_31_28	FM(IP17_31_28)	IP17_31_28
 #define MOD_SEL0_9_8		FM(SEL_DRIF1_0)		FM(SEL_DRIF1_1)		FM(SEL_DRIF1_2)		F_(0, 0)
 #define MOD_SEL0_7_6		FM(SEL_DRIF0_0)		FM(SEL_DRIF0_1)		FM(SEL_DRIF0_2)		F_(0, 0)
 #define MOD_SEL0_5		FM(SEL_CANFD0_0)	FM(SEL_CANFD0_1)
-#define MOD_SEL0_4_3		FM(SEL_ADG_A_0)		FM(SEL_ADG_A_1)		FM(SEL_ADG_A_2)		FM(SEL_ADG_A_3)
+#define MOD_SEL0_4_3		FM(SEL_ADGA_0)		FM(SEL_ADGA_1)		FM(SEL_ADGA_2)		FM(SEL_ADGA_3)
 
 /* MOD_SEL1 */			/* 0 */			/* 1 */			/* 2 */			/* 3 */			/* 4 */			/* 5 */			/* 6 */			/* 7 */
 #define MOD_SEL1_31_30		FM(SEL_TSIF1_0)		FM(SEL_TSIF1_1)		FM(SEL_TSIF1_2)		FM(SEL_TSIF1_3)
@@ -499,12 +500,12 @@ FM(IP16_31_28)	IP16_31_28	FM(IP17_31_28)	IP17_31_28
 #define MOD_SEL2_28_27		FM(SEL_FM_0)		FM(SEL_FM_1)		FM(SEL_FM_2)		FM(SEL_FM_3)
 #define MOD_SEL2_26		FM(SEL_SCIF5_0)		FM(SEL_SCIF5_1)
 #define MOD_SEL2_25_24_23	FM(SEL_I2C6_0)		FM(SEL_I2C6_1)		FM(SEL_I2C6_2)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)
-#define MOD_SEL2_22		FM(SEL_NDFC_0)		FM(SEL_NDFC_1)
+#define MOD_SEL2_22		FM(SEL_NDF_0)		FM(SEL_NDF_1)
 #define MOD_SEL2_21		FM(SEL_SSI2_0)		FM(SEL_SSI2_1)
 #define MOD_SEL2_20		FM(SEL_SSI9_0)		FM(SEL_SSI9_1)
 #define MOD_SEL2_19		FM(SEL_TIMER_TMU2_0)	FM(SEL_TIMER_TMU2_1)
-#define MOD_SEL2_18		FM(SEL_ADG_B_0)		FM(SEL_ADG_B_1)
-#define MOD_SEL2_17		FM(SEL_ADG_C_0)		FM(SEL_ADG_C_1)
+#define MOD_SEL2_18		FM(SEL_ADGB_0)		FM(SEL_ADGB_1)
+#define MOD_SEL2_17		FM(SEL_ADGC_0)		FM(SEL_ADGC_1)
 #define MOD_SEL2_0		FM(SEL_VIN4_0)		FM(SEL_VIN4_1)
 
 #define PINMUX_MOD_SELS	\
@@ -597,7 +598,7 @@ static const u16 pinmux_data[] = {
 	PINMUX_SINGLE(AVS2),
 	PINMUX_SINGLE(CLKOUT),
 	PINMUX_SINGLE(GP7_03),
-	PINMUX_SINGLE(HDMI0_CEC),
+	PINMUX_SINGLE(GP7_02),
 	PINMUX_SINGLE(MSIOF0_RXD),
 	PINMUX_SINGLE(MSIOF0_SCK),
 	PINMUX_SINGLE(MSIOF0_TXD),
@@ -1021,35 +1022,35 @@ static const u16 pinmux_data[] = {
 
 	PINMUX_IPSR_GPSR(IP8_15_12,	SD1_CMD),
 	PINMUX_IPSR_MSEL(IP8_15_12,	MSIOF1_SYNC_G,		SEL_MSIOF1_6),
-	PINMUX_IPSR_MSEL(IP8_15_12,	NFCE_N_B,		SEL_NDFC_1),
+	PINMUX_IPSR_MSEL(IP8_15_12,	NFCE_N_B,		SEL_NDF_1),
 	PINMUX_IPSR_MSEL(IP8_15_12,	SIM0_D_A,		SEL_SIMCARD_0),
 	PINMUX_IPSR_MSEL(IP8_15_12,	STP_IVCXO27_1_B,	SEL_SSP1_1_1),
 
 	PINMUX_IPSR_GPSR(IP8_19_16,	SD1_DAT0),
 	PINMUX_IPSR_GPSR(IP8_19_16,	SD2_DAT4),
 	PINMUX_IPSR_MSEL(IP8_19_16,	MSIOF1_RXD_G,		SEL_MSIOF1_6),
-	PINMUX_IPSR_MSEL(IP8_19_16,	NFWP_N_B,		SEL_NDFC_1),
+	PINMUX_IPSR_MSEL(IP8_19_16,	NFWP_N_B,		SEL_NDF_1),
 	PINMUX_IPSR_MSEL(IP8_19_16,	TS_SCK1_B,		SEL_TSIF1_1),
 	PINMUX_IPSR_MSEL(IP8_19_16,	STP_ISCLK_1_B,		SEL_SSP1_1_1),
 
 	PINMUX_IPSR_GPSR(IP8_23_20,	SD1_DAT1),
 	PINMUX_IPSR_GPSR(IP8_23_20,	SD2_DAT5),
 	PINMUX_IPSR_MSEL(IP8_23_20,	MSIOF1_TXD_G,		SEL_MSIOF1_6),
-	PINMUX_IPSR_MSEL(IP8_23_20,	NFDATA14_B,		SEL_NDFC_1),
+	PINMUX_IPSR_MSEL(IP8_23_20,	NFDATA14_B,		SEL_NDF_1),
 	PINMUX_IPSR_MSEL(IP8_23_20,	TS_SPSYNC1_B,		SEL_TSIF1_1),
 	PINMUX_IPSR_MSEL(IP8_23_20,	STP_ISSYNC_1_B,		SEL_SSP1_1_1),
 
 	PINMUX_IPSR_GPSR(IP8_27_24,	SD1_DAT2),
 	PINMUX_IPSR_GPSR(IP8_27_24,	SD2_DAT6),
 	PINMUX_IPSR_MSEL(IP8_27_24,	MSIOF1_SS1_G,		SEL_MSIOF1_6),
-	PINMUX_IPSR_MSEL(IP8_27_24,	NFDATA15_B,		SEL_NDFC_1),
+	PINMUX_IPSR_MSEL(IP8_27_24,	NFDATA15_B,		SEL_NDF_1),
 	PINMUX_IPSR_MSEL(IP8_27_24,	TS_SDAT1_B,		SEL_TSIF1_1),
 	PINMUX_IPSR_MSEL(IP8_27_24,	STP_ISD_1_B,		SEL_SSP1_1_1),
 
 	PINMUX_IPSR_GPSR(IP8_31_28,	SD1_DAT3),
 	PINMUX_IPSR_GPSR(IP8_31_28,	SD2_DAT7),
 	PINMUX_IPSR_MSEL(IP8_31_28,	MSIOF1_SS2_G,		SEL_MSIOF1_6),
-	PINMUX_IPSR_MSEL(IP8_31_28,	NFRB_N_B,		SEL_NDFC_1),
+	PINMUX_IPSR_MSEL(IP8_31_28,	NFRB_N_B,		SEL_NDF_1),
 	PINMUX_IPSR_MSEL(IP8_31_28,	TS_SDEN1_B,		SEL_TSIF1_1),
 	PINMUX_IPSR_MSEL(IP8_31_28,	STP_ISEN_1_B,		SEL_SSP1_1_1),
 
@@ -1115,28 +1116,28 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_GPSR(IP11_7_4,	NFCLE),
 
 	PINMUX_IPSR_GPSR(IP11_11_8,	SD0_CD),
-	PINMUX_IPSR_MSEL(IP11_11_8,	NFDATA14_A,		SEL_NDFC_0),
+	PINMUX_IPSR_MSEL(IP11_11_8,	NFDATA14_A,		SEL_NDF_0),
 	PINMUX_IPSR_MSEL(IP11_11_8,	SCL2_B,			SEL_I2C2_1),
 	PINMUX_IPSR_MSEL(IP11_11_8,	SIM0_RST_A,		SEL_SIMCARD_0),
 
 	PINMUX_IPSR_GPSR(IP11_15_12,	SD0_WP),
-	PINMUX_IPSR_MSEL(IP11_15_12,	NFDATA15_A,		SEL_NDFC_0),
+	PINMUX_IPSR_MSEL(IP11_15_12,	NFDATA15_A,		SEL_NDF_0),
 	PINMUX_IPSR_MSEL(IP11_15_12,	SDA2_B,			SEL_I2C2_1),
 
 	PINMUX_IPSR_MSEL(IP11_19_16,	SD1_CD,			I2C_SEL_0_0),
-	PINMUX_IPSR_PHYS_MSEL(IP11_19_16, NFRB_N_A,		I2C_SEL_0_0,	SEL_NDFC_0),
+	PINMUX_IPSR_PHYS_MSEL(IP11_19_16, NFRB_N_A,		I2C_SEL_0_0,	SEL_NDF_0),
 	PINMUX_IPSR_PHYS_MSEL(IP11_19_16, SIM0_CLK_B,		I2C_SEL_0_0,	SEL_SIMCARD_1),
 	PINMUX_IPSR_PHYS(IP11_19_16,	SCL0,			I2C_SEL_0_1),
 
 	PINMUX_IPSR_MSEL(IP11_23_20,	SD1_WP,			I2C_SEL_0_0),
-	PINMUX_IPSR_PHYS_MSEL(IP11_23_20, NFCE_N_A,		I2C_SEL_0_0,	SEL_NDFC_0),
+	PINMUX_IPSR_PHYS_MSEL(IP11_23_20, NFCE_N_A,		I2C_SEL_0_0,	SEL_NDF_0),
 	PINMUX_IPSR_PHYS_MSEL(IP11_23_20, SIM0_D_B,		I2C_SEL_0_0,	SEL_SIMCARD_1),
 	PINMUX_IPSR_PHYS(IP11_23_20,	SDA0,			I2C_SEL_0_1),
 
 	PINMUX_IPSR_GPSR(IP11_27_24,	SCK0),
 	PINMUX_IPSR_MSEL(IP11_27_24,	HSCK1_B,		SEL_HSCIF1_1),
 	PINMUX_IPSR_MSEL(IP11_27_24,	MSIOF1_SS2_B,		SEL_MSIOF1_1),
-	PINMUX_IPSR_MSEL(IP11_27_24,	AUDIO_CLKC_B,		SEL_ADG_C_1),
+	PINMUX_IPSR_MSEL(IP11_27_24,	AUDIO_CLKC_B,		SEL_ADGC_1),
 	PINMUX_IPSR_MSEL(IP11_27_24,	SDA2_A,			SEL_I2C2_0),
 	PINMUX_IPSR_MSEL(IP11_27_24,	SIM0_RST_B,		SEL_SIMCARD_1),
 	PINMUX_IPSR_MSEL(IP11_27_24,	STP_OPWM_0_C,		SEL_SSP1_0_2),
@@ -1169,7 +1170,7 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_GPSR(IP12_11_8,	RTS0_N),
 	PINMUX_IPSR_MSEL(IP12_11_8,	HRTS1_N_B,		SEL_HSCIF1_1),
 	PINMUX_IPSR_MSEL(IP12_11_8,	MSIOF1_SS1_B,		SEL_MSIOF1_1),
-	PINMUX_IPSR_MSEL(IP12_11_8,	AUDIO_CLKA_B,		SEL_ADG_A_1),
+	PINMUX_IPSR_MSEL(IP12_11_8,	AUDIO_CLKA_B,		SEL_ADGA_1),
 	PINMUX_IPSR_MSEL(IP12_11_8,	SCL2_A,			SEL_I2C2_0),
 	PINMUX_IPSR_MSEL(IP12_11_8,	STP_IVCXO27_1_C,	SEL_SSP1_1_2),
 	PINMUX_IPSR_MSEL(IP12_11_8,	RIF0_SYNC_B,		SEL_DRIF0_1),
@@ -1228,7 +1229,7 @@ static const u16 pinmux_data[] = {
 
 	PINMUX_IPSR_GPSR(IP13_11_8,	HSCK0),
 	PINMUX_IPSR_MSEL(IP13_11_8,	MSIOF1_SCK_D,		SEL_MSIOF1_3),
-	PINMUX_IPSR_MSEL(IP13_11_8,	AUDIO_CLKB_A,		SEL_ADG_B_0),
+	PINMUX_IPSR_MSEL(IP13_11_8,	AUDIO_CLKB_A,		SEL_ADGB_0),
 	PINMUX_IPSR_MSEL(IP13_11_8,	SSI_SDATA1_B,		SEL_SSI1_1),
 	PINMUX_IPSR_MSEL(IP13_11_8,	TS_SCK0_D,		SEL_TSIF0_3),
 	PINMUX_IPSR_MSEL(IP13_11_8,	STP_ISCLK_0_D,		SEL_SSP1_0_3),
@@ -1274,8 +1275,8 @@ static const u16 pinmux_data[] = {
 	/* IPSR14 */
 	PINMUX_IPSR_GPSR(IP14_3_0,	MSIOF0_SS1),
 	PINMUX_IPSR_MSEL(IP14_3_0,	RX5_A,			SEL_SCIF5_0),
-	PINMUX_IPSR_MSEL(IP14_3_0,	NFWP_N_A,		SEL_NDFC_0),
-	PINMUX_IPSR_MSEL(IP14_3_0,	AUDIO_CLKA_C,		SEL_ADG_A_2),
+	PINMUX_IPSR_MSEL(IP14_3_0,	NFWP_N_A,		SEL_NDF_0),
+	PINMUX_IPSR_MSEL(IP14_3_0,	AUDIO_CLKA_C,		SEL_ADGA_2),
 	PINMUX_IPSR_MSEL(IP14_3_0,	SSI_SCK2_A,		SEL_SSI2_0),
 	PINMUX_IPSR_MSEL(IP14_3_0,	STP_IVCXO27_0_C,	SEL_SSP1_0_2),
 	PINMUX_IPSR_GPSR(IP14_3_0,	AUDIO_CLKOUT3_A),
@@ -1284,7 +1285,7 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_GPSR(IP14_7_4,	MSIOF0_SS2),
 	PINMUX_IPSR_MSEL(IP14_7_4,	TX5_A,			SEL_SCIF5_0),
 	PINMUX_IPSR_MSEL(IP14_7_4,	MSIOF1_SS2_D,		SEL_MSIOF1_3),
-	PINMUX_IPSR_MSEL(IP14_7_4,	AUDIO_CLKC_A,		SEL_ADG_C_0),
+	PINMUX_IPSR_MSEL(IP14_7_4,	AUDIO_CLKC_A,		SEL_ADGC_0),
 	PINMUX_IPSR_MSEL(IP14_7_4,	SSI_WS2_A,		SEL_SSI2_0),
 	PINMUX_IPSR_MSEL(IP14_7_4,	STP_OPWM_0_D,		SEL_SSP1_0_3),
 	PINMUX_IPSR_GPSR(IP14_7_4,	AUDIO_CLKOUT_D),
@@ -1412,10 +1413,9 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_MSEL(IP16_31_28,	SCK5_A,			SEL_SCIF5_0),
 
 	/* IPSR17 */
-	PINMUX_IPSR_MSEL(IP17_3_0,	AUDIO_CLKA_A,		SEL_ADG_A_0),
-	PINMUX_IPSR_GPSR(IP17_3_0,	CC5_OSCOUT),
+	PINMUX_IPSR_MSEL(IP17_3_0,	AUDIO_CLKA_A,		SEL_ADGA_0),
 
-	PINMUX_IPSR_MSEL(IP17_7_4,	AUDIO_CLKB_B,		SEL_ADG_B_1),
+	PINMUX_IPSR_MSEL(IP17_7_4,	AUDIO_CLKB_B,		SEL_ADGB_1),
 	PINMUX_IPSR_MSEL(IP17_7_4,	SCIF_CLK_A,		SEL_SCIF_0),
 	PINMUX_IPSR_MSEL(IP17_7_4,	STP_IVCXO27_1_D,	SEL_SSP1_1_3),
 	PINMUX_IPSR_MSEL(IP17_7_4,	REMOCON_A,		SEL_REMOCON_0),
@@ -1499,11 +1499,6 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_MSEL(IP18_7_4,	FMIN_C,			SEL_FM_2),
 	PINMUX_IPSR_MSEL(IP18_7_4,	FMIN_D,			SEL_FM_3),
 
-	/* I2C */
-	PINMUX_IPSR_NOGP(0,		I2C_SEL_0_1),
-	PINMUX_IPSR_NOGP(0,		I2C_SEL_3_1),
-	PINMUX_IPSR_NOGP(0,		I2C_SEL_5_1),
-
 /*
  * Static pins can not be muxed between different functions but
  * still need mark entries in the pinmux list. Add each static
@@ -2140,15 +2135,6 @@ static const unsigned int du_disp_mux[] = {
 	DU_DISP_MARK,
 };
 
-/* - HDMI ------------------------------------------------------------------- */
-static const unsigned int hdmi0_cec_pins[] = {
-	/* HDMI0_CEC */
-	RCAR_GP_PIN(7, 2),
-};
-static const unsigned int hdmi0_cec_mux[] = {
-	HDMI0_CEC_MARK,
-};
-
 /* - HSCIF0 ----------------------------------------------------------------- */
 static const unsigned int hscif0_data_pins[] = {
 	/* RX, TX */
@@ -4124,8 +4110,8 @@ static const unsigned int vin5_clk_mux[] = {
 };
 
 static const struct {
-	struct sh_pfc_pin_group common[310];
-	struct sh_pfc_pin_group automotive[33];
+	struct sh_pfc_pin_group common[312];
+	struct sh_pfc_pin_group automotive[30];
 } pinmux_groups = {
 	.common = {
 		SH_PFC_PIN_GROUP(audio_clk_a_a),
@@ -4160,6 +4146,9 @@ static const struct {
 		SH_PFC_PIN_GROUP(can0_data_b),
 		SH_PFC_PIN_GROUP(can1_data),
 		SH_PFC_PIN_GROUP(can_clk),
+		SH_PFC_PIN_GROUP(canfd0_data_a),
+		SH_PFC_PIN_GROUP(canfd0_data_b),
+		SH_PFC_PIN_GROUP(canfd1_data),
 		SH_PFC_PIN_GROUP(du_rgb666),
 		SH_PFC_PIN_GROUP(du_rgb888),
 		SH_PFC_PIN_GROUP(du_clk_out_0),
@@ -4168,7 +4157,6 @@ static const struct {
 		SH_PFC_PIN_GROUP(du_oddf),
 		SH_PFC_PIN_GROUP(du_cde),
 		SH_PFC_PIN_GROUP(du_disp),
-		SH_PFC_PIN_GROUP(hdmi0_cec),
 		SH_PFC_PIN_GROUP(hscif0_data),
 		SH_PFC_PIN_GROUP(hscif0_clk),
 		SH_PFC_PIN_GROUP(hscif0_ctrl),
@@ -4440,9 +4428,6 @@ static const struct {
 		SH_PFC_PIN_GROUP(vin5_clk),
 	},
 	.automotive = {
-		SH_PFC_PIN_GROUP(canfd0_data_a),
-		SH_PFC_PIN_GROUP(canfd0_data_b),
-		SH_PFC_PIN_GROUP(canfd1_data),
 		SH_PFC_PIN_GROUP(drif0_ctrl_a),
 		SH_PFC_PIN_GROUP(drif0_data0_a),
 		SH_PFC_PIN_GROUP(drif0_data1_a),
@@ -4585,10 +4570,6 @@ static const char * const du_groups[] = {
 	"du_disp",
 };
 
-static const char * const hdmi0_groups[] = {
-	"hdmi0_cec",
-};
-
 static const char * const hscif0_groups[] = {
 	"hscif0_data",
 	"hscif0_clk",
@@ -4982,8 +4963,8 @@ static const char * const vin5_groups[] = {
 };
 
 static const struct {
-	struct sh_pfc_function common[48];
-	struct sh_pfc_function automotive[6];
+	struct sh_pfc_function common[49];
+	struct sh_pfc_function automotive[4];
 } pinmux_functions = {
 	.common = {
 		SH_PFC_FUNCTION(audio_clk),
@@ -4991,8 +4972,9 @@ static const struct {
 		SH_PFC_FUNCTION(can0),
 		SH_PFC_FUNCTION(can1),
 		SH_PFC_FUNCTION(can_clk),
+		SH_PFC_FUNCTION(canfd0),
+		SH_PFC_FUNCTION(canfd1),
 		SH_PFC_FUNCTION(du),
-		SH_PFC_FUNCTION(hdmi0),
 		SH_PFC_FUNCTION(hscif0),
 		SH_PFC_FUNCTION(hscif1),
 		SH_PFC_FUNCTION(hscif2),
@@ -5036,8 +5018,6 @@ static const struct {
 		SH_PFC_FUNCTION(vin5),
 	},
 	.automotive = {
-		SH_PFC_FUNCTION(canfd0),
-		SH_PFC_FUNCTION(canfd1),
 		SH_PFC_FUNCTION(drif0),
 		SH_PFC_FUNCTION(drif1),
 		SH_PFC_FUNCTION(drif2),
@@ -5048,7 +5028,7 @@ static const struct {
 static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 #define F_(x, y)	FN_##y
 #define FM(x)		FN_##x
-	{ PINMUX_CFG_REG("GPSR0", 0xe6060100, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR0", 0xe6060100, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -5080,9 +5060,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_0_3_FN,	GPSR0_3,
 		GP_0_2_FN,	GPSR0_2,
 		GP_0_1_FN,	GPSR0_1,
-		GP_0_0_FN,	GPSR0_0, }
+		GP_0_0_FN,	GPSR0_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR1", 0xe6060104, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR1", 0xe6060104, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -5114,9 +5094,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_1_3_FN,	GPSR1_3,
 		GP_1_2_FN,	GPSR1_2,
 		GP_1_1_FN,	GPSR1_1,
-		GP_1_0_FN,	GPSR1_0, }
+		GP_1_0_FN,	GPSR1_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR2", 0xe6060108, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR2", 0xe6060108, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -5148,9 +5128,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_2_3_FN,	GPSR2_3,
 		GP_2_2_FN,	GPSR2_2,
 		GP_2_1_FN,	GPSR2_1,
-		GP_2_0_FN,	GPSR2_0, }
+		GP_2_0_FN,	GPSR2_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR3", 0xe606010c, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR3", 0xe606010c, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -5182,9 +5162,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_3_3_FN,	GPSR3_3,
 		GP_3_2_FN,	GPSR3_2,
 		GP_3_1_FN,	GPSR3_1,
-		GP_3_0_FN,	GPSR3_0, }
+		GP_3_0_FN,	GPSR3_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR4", 0xe6060110, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR4", 0xe6060110, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -5216,9 +5196,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_4_3_FN,	GPSR4_3,
 		GP_4_2_FN,	GPSR4_2,
 		GP_4_1_FN,	GPSR4_1,
-		GP_4_0_FN,	GPSR4_0, }
+		GP_4_0_FN,	GPSR4_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR5", 0xe6060114, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR5", 0xe6060114, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -5250,9 +5230,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_5_3_FN,	GPSR5_3,
 		GP_5_2_FN,	GPSR5_2,
 		GP_5_1_FN,	GPSR5_1,
-		GP_5_0_FN,	GPSR5_0, }
+		GP_5_0_FN,	GPSR5_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR6", 0xe6060118, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR6", 0xe6060118, 32, 1, GROUP(
 		GP_6_31_FN,	GPSR6_31,
 		GP_6_30_FN,	GPSR6_30,
 		GP_6_29_FN,	GPSR6_29,
@@ -5284,9 +5264,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_6_3_FN,	GPSR6_3,
 		GP_6_2_FN,	GPSR6_2,
 		GP_6_1_FN,	GPSR6_1,
-		GP_6_0_FN,	GPSR6_0, }
+		GP_6_0_FN,	GPSR6_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR7", 0xe606011c, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR7", 0xe606011c, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -5318,14 +5298,14 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_7_3_FN, GPSR7_3,
 		GP_7_2_FN, GPSR7_2,
 		GP_7_1_FN, GPSR7_1,
-		GP_7_0_FN, GPSR7_0, }
+		GP_7_0_FN, GPSR7_0, ))
 	},
 #undef F_
 #undef FM
 
 #define F_(x, y)	x,
 #define FM(x)		FN_##x,
-	{ PINMUX_CFG_REG("IPSR0", 0xe6060200, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR0", 0xe6060200, 32, 4, GROUP(
 		IP0_31_28
 		IP0_27_24
 		IP0_23_20
@@ -5333,9 +5313,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP0_15_12
 		IP0_11_8
 		IP0_7_4
-		IP0_3_0 }
+		IP0_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR1", 0xe6060204, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR1", 0xe6060204, 32, 4, GROUP(
 		IP1_31_28
 		IP1_27_24
 		IP1_23_20
@@ -5343,9 +5323,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP1_15_12
 		IP1_11_8
 		IP1_7_4
-		IP1_3_0 }
+		IP1_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR2", 0xe6060208, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR2", 0xe6060208, 32, 4, GROUP(
 		IP2_31_28
 		IP2_27_24
 		IP2_23_20
@@ -5353,9 +5333,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP2_15_12
 		IP2_11_8
 		IP2_7_4
-		IP2_3_0 }
+		IP2_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR3", 0xe606020c, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR3", 0xe606020c, 32, 4, GROUP(
 		IP3_31_28
 		IP3_27_24
 		IP3_23_20
@@ -5363,9 +5343,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP3_15_12
 		IP3_11_8
 		IP3_7_4
-		IP3_3_0 }
+		IP3_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR4", 0xe6060210, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR4", 0xe6060210, 32, 4, GROUP(
 		IP4_31_28
 		IP4_27_24
 		IP4_23_20
@@ -5373,9 +5353,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP4_15_12
 		IP4_11_8
 		IP4_7_4
-		IP4_3_0 }
+		IP4_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR5", 0xe6060214, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR5", 0xe6060214, 32, 4, GROUP(
 		IP5_31_28
 		IP5_27_24
 		IP5_23_20
@@ -5383,9 +5363,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP5_15_12
 		IP5_11_8
 		IP5_7_4
-		IP5_3_0 }
+		IP5_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR6", 0xe6060218, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR6", 0xe6060218, 32, 4, GROUP(
 		IP6_31_28
 		IP6_27_24
 		IP6_23_20
@@ -5393,9 +5373,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP6_15_12
 		IP6_11_8
 		IP6_7_4
-		IP6_3_0 }
+		IP6_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR7", 0xe606021c, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR7", 0xe606021c, 32, 4, GROUP(
 		IP7_31_28
 		IP7_27_24
 		IP7_23_20
@@ -5403,9 +5383,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP7_15_12 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		IP7_11_8
 		IP7_7_4
-		IP7_3_0 }
+		IP7_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR8", 0xe6060220, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR8", 0xe6060220, 32, 4, GROUP(
 		IP8_31_28
 		IP8_27_24
 		IP8_23_20
@@ -5413,9 +5393,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP8_15_12
 		IP8_11_8
 		IP8_7_4
-		IP8_3_0 }
+		IP8_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR9", 0xe6060224, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR9", 0xe6060224, 32, 4, GROUP(
 		IP9_31_28
 		IP9_27_24
 		IP9_23_20
@@ -5423,9 +5403,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP9_15_12
 		IP9_11_8
 		IP9_7_4
-		IP9_3_0 }
+		IP9_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR10", 0xe6060228, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR10", 0xe6060228, 32, 4, GROUP(
 		IP10_31_28
 		IP10_27_24
 		IP10_23_20
@@ -5433,9 +5413,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP10_15_12
 		IP10_11_8
 		IP10_7_4
-		IP10_3_0 }
+		IP10_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR11", 0xe606022c, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR11", 0xe606022c, 32, 4, GROUP(
 		IP11_31_28
 		IP11_27_24
 		IP11_23_20
@@ -5443,9 +5423,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP11_15_12
 		IP11_11_8
 		IP11_7_4
-		IP11_3_0 }
+		IP11_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR12", 0xe6060230, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR12", 0xe6060230, 32, 4, GROUP(
 		IP12_31_28
 		IP12_27_24
 		IP12_23_20
@@ -5453,9 +5433,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP12_15_12
 		IP12_11_8
 		IP12_7_4
-		IP12_3_0 }
+		IP12_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR13", 0xe6060234, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR13", 0xe6060234, 32, 4, GROUP(
 		IP13_31_28
 		IP13_27_24
 		IP13_23_20
@@ -5463,9 +5443,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP13_15_12
 		IP13_11_8
 		IP13_7_4
-		IP13_3_0 }
+		IP13_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR14", 0xe6060238, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR14", 0xe6060238, 32, 4, GROUP(
 		IP14_31_28
 		IP14_27_24
 		IP14_23_20
@@ -5473,9 +5453,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP14_15_12
 		IP14_11_8
 		IP14_7_4
-		IP14_3_0 }
+		IP14_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR15", 0xe606023c, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR15", 0xe606023c, 32, 4, GROUP(
 		IP15_31_28
 		IP15_27_24
 		IP15_23_20
@@ -5483,9 +5463,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP15_15_12
 		IP15_11_8
 		IP15_7_4
-		IP15_3_0 }
+		IP15_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR16", 0xe6060240, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR16", 0xe6060240, 32, 4, GROUP(
 		IP16_31_28
 		IP16_27_24
 		IP16_23_20
@@ -5493,9 +5473,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP16_15_12
 		IP16_11_8
 		IP16_7_4
-		IP16_3_0 }
+		IP16_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR17", 0xe6060244, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR17", 0xe6060244, 32, 4, GROUP(
 		IP17_31_28
 		IP17_27_24
 		IP17_23_20
@@ -5503,9 +5483,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP17_15_12
 		IP17_11_8
 		IP17_7_4
-		IP17_3_0 }
+		IP17_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR18", 0xe6060248, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR18", 0xe6060248, 32, 4, GROUP(
 		/* IP18_31_28 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP18_27_24 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP18_23_20 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -5513,7 +5493,7 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP18_15_12 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP18_11_8  */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		IP18_7_4
-		IP18_3_0 }
+		IP18_3_0 ))
 	},
 #undef F_
 #undef FM
@@ -5521,8 +5501,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 #define F_(x, y)	x,
 #define FM(x)		FN_##x,
 	{ PINMUX_CFG_REG_VAR("MOD_SEL0", 0xe6060500, 32,
-			     3, 2, 3, 1, 1, 1, 1, 1, 2, 1,
-			     1, 2, 1, 1, 1, 2, 2, 1, 2, 3) {
+			     GROUP(3, 2, 3, 1, 1, 1, 1, 1, 2, 1, 1, 2,
+				   1, 1, 1, 2, 2, 1, 2, 3),
+			     GROUP(
 		MOD_SEL0_31_30_29
 		MOD_SEL0_28_27
 		MOD_SEL0_26_25_24
@@ -5543,11 +5524,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		MOD_SEL0_5
 		MOD_SEL0_4_3
 		/* RESERVED 2, 1, 0 */
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
 	{ PINMUX_CFG_REG_VAR("MOD_SEL1", 0xe6060504, 32,
-			     2, 3, 1, 2, 3, 1, 1, 2, 1,
-			     2, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1) {
+			     GROUP(2, 3, 1, 2, 3, 1, 1, 2, 1, 2, 1, 1,
+				   1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1),
+			     GROUP(
 		MOD_SEL1_31_30
 		MOD_SEL1_29_28_27
 		MOD_SEL1_26
@@ -5570,11 +5552,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		MOD_SEL1_3
 		MOD_SEL1_2
 		MOD_SEL1_1
-		MOD_SEL1_0 }
+		MOD_SEL1_0 ))
 	},
 	{ PINMUX_CFG_REG_VAR("MOD_SEL2", 0xe6060508, 32,
-			     1, 1, 1, 2, 1, 3, 1, 1, 1, 1, 1, 1, 1,
-			     4, 4, 4, 3, 1) {
+			     GROUP(1, 1, 1, 2, 1, 3, 1, 1, 1, 1, 1, 1,
+				   1, 4, 4, 4, 3, 1),
+			     GROUP(
 		MOD_SEL2_31
 		MOD_SEL2_30
 		MOD_SEL2_29
@@ -5600,7 +5583,7 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0,
 		/* RESERVED 3, 2, 1 */
 		0, 0, 0, 0, 0, 0, 0, 0,
-		MOD_SEL2_0 }
+		MOD_SEL2_0 ))
 	},
 	{ },
 };
@@ -5721,7 +5704,7 @@ static const struct pinmux_drive_reg pinmux_drive_regs[] = {
 		{ RCAR_GP_PIN(0, 15),   24, 3 },	/* D15 */
 		{ RCAR_GP_PIN(7,  0),   20, 3 },	/* AVS1 */
 		{ RCAR_GP_PIN(7,  1),   16, 3 },	/* AVS2 */
-		{ RCAR_GP_PIN(7,  2),   12, 3 },	/* HDMI0_CEC */
+		{ RCAR_GP_PIN(7,  2),   12, 3 },	/* GP7_02 */
 		{ RCAR_GP_PIN(7,  3),    8, 3 },	/* GP7_03 */
 		{ PIN_A_NUMBER('P', 7),  4, 2 },	/* DU_DOTCLKIN0 */
 		{ PIN_A_NUMBER('P', 8),  0, 2 },	/* DU_DOTCLKIN1 */
@@ -5855,10 +5838,12 @@ static const struct pinmux_drive_reg pinmux_drive_regs[] = {
 
 enum ioctrl_regs {
 	POCCTRL,
+	TDSELCTRL,
 };
 
 static const struct pinmux_ioctrl_reg pinmux_ioctrl_regs[] = {
 	[POCCTRL] = { 0xe6060380, },
+	[TDSELCTRL] = { 0xe60603c0, },
 	{ /* sentinel */ },
 };
 
@@ -5975,7 +5960,7 @@ static const struct pinmux_bias_reg pinmux_bias_regs[] = {
 		[25] = RCAR_GP_PIN(0, 15),	/* D15 */
 		[26] = RCAR_GP_PIN(7,  0),	/* AVS1 */
 		[27] = RCAR_GP_PIN(7,  1),	/* AVS2 */
-		[28] = RCAR_GP_PIN(7,  2),	/* HDMI0_CEC */
+		[28] = RCAR_GP_PIN(7,  2),	/* GP7_02 */
 		[29] = RCAR_GP_PIN(7,  3),	/* GP7_03 */
 		[30] = PIN_A_NUMBER('P', 7),	/* DU_DOTCLKIN0 */
 		[31] = PIN_A_NUMBER('P', 8),	/* DU_DOTCLKIN1 */
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a77965.c b/drivers/pinctrl/sh-pfc/pfc-r8a77965.c
index 14c4b671cddf..090024355eba 100644
--- a/drivers/pinctrl/sh-pfc/pfc-r8a77965.c
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a77965.c
@@ -3,7 +3,7 @@
  * R8A77965 processor support - PFC hardware block.
  *
  * Copyright (C) 2018 Jacopo Mondi <jacopo+renesas@jmondi.org>
- * Copyright (C) 2016 Renesas Electronics Corp.
+ * Copyright (C) 2016-2019 Renesas Electronics Corp.
  *
  * This file is based on the drivers/pinctrl/sh-pfc/pfc-r8a7796.c
  *
@@ -12,6 +12,7 @@
  * Copyright (C) 2015  Renesas Electronics Corporation
  */
 
+#include <linux/errno.h>
 #include <linux/kernel.h>
 
 #include "core.h"
@@ -207,7 +208,7 @@
 
 /* GPSR7 */
 #define GPSR7_3		FM(GP7_03)
-#define GPSR7_2		FM(HDMI0_CEC)
+#define GPSR7_2		FM(GP7_02)
 #define GPSR7_1		FM(AVS2)
 #define GPSR7_0		FM(AVS1)
 
@@ -356,7 +357,7 @@
 #define IP16_23_20	FM(SSI_SDATA7)		FM(HCTS2_N_B)	FM(MSIOF1_RXD_C)	F_(0, 0)			F_(0, 0)	FM(TS_SDEN1_A)	FM(STP_ISEN_1_A)	FM(RIF1_D0_A)	FM(RIF3_D0_A)		F_(0, 0)	FM(TCLK2_A)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP16_27_24	FM(SSI_SDATA8)		FM(HRTS2_N_B)	FM(MSIOF1_TXD_C)	F_(0, 0)			F_(0, 0)	FM(TS_SPSYNC1_A)FM(STP_ISSYNC_1_A)	FM(RIF1_D1_A)	FM(RIF3_D1_A)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP16_31_28	FM(SSI_SDATA9_A)	FM(HSCK2_B)	FM(MSIOF1_SS1_C)	FM(HSCK1_A)			FM(SSI_WS1_B)	FM(SCK1)	FM(STP_IVCXO27_1_A)	FM(SCK5_A)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP17_3_0	FM(AUDIO_CLKA_A)	F_(0, 0)	F_(0, 0)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	FM(CC5_OSCOUT)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP17_3_0	FM(AUDIO_CLKA_A)	F_(0, 0)	F_(0, 0)		F_(0, 0)			F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP17_7_4	FM(AUDIO_CLKB_B)	FM(SCIF_CLK_A)	F_(0, 0)		F_(0, 0)			F_(0, 0)	F_(0, 0)	FM(STP_IVCXO27_1_D)	FM(REMOCON_A)	F_(0, 0)		F_(0, 0)	FM(TCLK1_A)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP17_11_8	FM(USB0_PWEN)		F_(0, 0)	F_(0, 0)		FM(SIM0_RST_C)			F_(0, 0)	FM(TS_SCK1_D)	FM(STP_ISCLK_1_D)	FM(BPFCLK_B)	FM(RIF3_CLK_B)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) FM(HSCK2_C) F_(0, 0) F_(0, 0)
 #define IP17_15_12	FM(USB0_OVC)		F_(0, 0)	F_(0, 0)		FM(SIM0_D_C)			F_(0, 0)	FM(TS_SDAT1_D)	FM(STP_ISD_1_D)		F_(0, 0)	FM(RIF3_SYNC_B)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) FM(HRX2_C) F_(0, 0) F_(0, 0)
@@ -467,7 +468,7 @@ FM(IP16_31_28)	IP16_31_28	FM(IP17_31_28)	IP17_31_28
 #define MOD_SEL0_9_8		FM(SEL_DRIF1_0)		FM(SEL_DRIF1_1)		FM(SEL_DRIF1_2)		F_(0, 0)
 #define MOD_SEL0_7_6		FM(SEL_DRIF0_0)		FM(SEL_DRIF0_1)		FM(SEL_DRIF0_2)		F_(0, 0)
 #define MOD_SEL0_5		FM(SEL_CANFD0_0)	FM(SEL_CANFD0_1)
-#define MOD_SEL0_4_3		FM(SEL_ADG_A_0)		FM(SEL_ADG_A_1)		FM(SEL_ADG_A_2)		FM(SEL_ADG_A_3)
+#define MOD_SEL0_4_3		FM(SEL_ADGA_0)		FM(SEL_ADGA_1)		FM(SEL_ADGA_2)		FM(SEL_ADGA_3)
 
 /* MOD_SEL1 */			/* 0 */			/* 1 */			/* 2 */			/* 3 */			/* 4 */			/* 5 */			/* 6 */			/* 7 */
 #define MOD_SEL1_31_30		FM(SEL_TSIF1_0)		FM(SEL_TSIF1_1)		FM(SEL_TSIF1_2)		FM(SEL_TSIF1_3)
@@ -500,12 +501,12 @@ FM(IP16_31_28)	IP16_31_28	FM(IP17_31_28)	IP17_31_28
 #define MOD_SEL2_28_27		FM(SEL_FM_0)		FM(SEL_FM_1)		FM(SEL_FM_2)		FM(SEL_FM_3)
 #define MOD_SEL2_26		FM(SEL_SCIF5_0)		FM(SEL_SCIF5_1)
 #define MOD_SEL2_25_24_23	FM(SEL_I2C6_0)		FM(SEL_I2C6_1)		FM(SEL_I2C6_2)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)		F_(0, 0)
-#define MOD_SEL2_22		FM(SEL_NDFC_0)		FM(SEL_NDFC_1)
+#define MOD_SEL2_22		FM(SEL_NDF_0)		FM(SEL_NDF_1)
 #define MOD_SEL2_21		FM(SEL_SSI2_0)		FM(SEL_SSI2_1)
 #define MOD_SEL2_20		FM(SEL_SSI9_0)		FM(SEL_SSI9_1)
 #define MOD_SEL2_19		FM(SEL_TIMER_TMU2_0)	FM(SEL_TIMER_TMU2_1)
-#define MOD_SEL2_18		FM(SEL_ADG_B_0)		FM(SEL_ADG_B_1)
-#define MOD_SEL2_17		FM(SEL_ADG_C_0)		FM(SEL_ADG_C_1)
+#define MOD_SEL2_18		FM(SEL_ADGB_0)		FM(SEL_ADGB_1)
+#define MOD_SEL2_17		FM(SEL_ADGC_0)		FM(SEL_ADGC_1)
 #define MOD_SEL2_0		FM(SEL_VIN4_0)		FM(SEL_VIN4_1)
 
 #define PINMUX_MOD_SELS	\
@@ -557,6 +558,9 @@ MOD_SEL0_4_3		MOD_SEL1_4 \
 	FM(DU_DOTCLKIN0) FM(DU_DOTCLKIN1) FM(DU_DOTCLKIN3) \
 	FM(TMS) FM(TDO) FM(ASEBRK) FM(MLB_REF) FM(TDI) FM(TCK) FM(TRST) FM(EXTALR)
 
+#define PINMUX_PHYS \
+	FM(SCL0) FM(SDA0) FM(SCL3) FM(SDA3) FM(SCL5) FM(SDA5)
+
 enum {
 	PINMUX_RESERVED = 0,
 
@@ -582,6 +586,7 @@ enum {
 	PINMUX_IPSR
 	PINMUX_MOD_SELS
 	PINMUX_STATIC
+	PINMUX_PHYS
 	PINMUX_MARK_END,
 #undef F_
 #undef FM
@@ -594,7 +599,7 @@ static const u16 pinmux_data[] = {
 	PINMUX_SINGLE(AVS2),
 	PINMUX_SINGLE(CLKOUT),
 	PINMUX_SINGLE(GP7_03),
-	PINMUX_SINGLE(HDMI0_CEC),
+	PINMUX_SINGLE(GP7_02),
 	PINMUX_SINGLE(MSIOF0_RXD),
 	PINMUX_SINGLE(MSIOF0_SCK),
 	PINMUX_SINGLE(MSIOF0_TXD),
@@ -619,13 +624,15 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_MSEL(IP0_15_12,	TX4_A,			SEL_SCIF4_0),
 	PINMUX_IPSR_GPSR(IP0_19_16,	FSCLKST2_N_A),
 
-	PINMUX_IPSR_MSEL(IP0_19_16,	AVB_AVTP_MATCH_A,	SEL_ETHERAVB_0),
-	PINMUX_IPSR_MSEL(IP0_19_16,	MSIOF2_RXD_C,		SEL_MSIOF2_2),
-	PINMUX_IPSR_MSEL(IP0_19_16,	CTS4_N_A,		SEL_SCIF4_0),
+	PINMUX_IPSR_PHYS_MSEL(IP0_19_16,	AVB_AVTP_MATCH_A,	I2C_SEL_5_0, SEL_ETHERAVB_0),
+	PINMUX_IPSR_PHYS_MSEL(IP0_19_16,	MSIOF2_RXD_C,	I2C_SEL_5_0, SEL_MSIOF2_2),
+	PINMUX_IPSR_PHYS_MSEL(IP0_19_16,	CTS4_N_A,	I2C_SEL_5_0, SEL_SCIF4_0),
+	PINMUX_IPSR_PHYS(IP0_19_16,	SCL5,			I2C_SEL_5_1),
 
-	PINMUX_IPSR_MSEL(IP0_23_20,	AVB_AVTP_CAPTURE_A,	SEL_ETHERAVB_0),
-	PINMUX_IPSR_MSEL(IP0_23_20,	MSIOF2_TXD_C,		SEL_MSIOF2_2),
-	PINMUX_IPSR_MSEL(IP0_23_20,	RTS4_N_A,		SEL_SCIF4_0),
+	PINMUX_IPSR_PHYS_MSEL(IP0_23_20,	AVB_AVTP_CAPTURE_A,	I2C_SEL_5_0, SEL_ETHERAVB_0),
+	PINMUX_IPSR_PHYS_MSEL(IP0_23_20,	MSIOF2_TXD_C,		I2C_SEL_5_0, SEL_MSIOF2_2),
+	PINMUX_IPSR_PHYS_MSEL(IP0_23_20,	RTS4_N_A,		I2C_SEL_5_0, SEL_SCIF4_0),
+	PINMUX_IPSR_PHYS(IP0_23_20,	SDA5,			I2C_SEL_5_1),
 
 	PINMUX_IPSR_GPSR(IP0_27_24,	IRQ0),
 	PINMUX_IPSR_GPSR(IP0_27_24,	QPOLB),
@@ -678,14 +685,16 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_MSEL(IP1_19_16,	VI4_DATA6_B,		SEL_VIN4_1),
 	PINMUX_IPSR_MSEL(IP1_19_16,	IECLK_B,		SEL_IEBUS_1),
 
-	PINMUX_IPSR_MSEL(IP1_23_20,	PWM1_A,			SEL_PWM1_0),
-	PINMUX_IPSR_MSEL(IP1_23_20,	HRX3_D,			SEL_HSCIF3_3),
-	PINMUX_IPSR_MSEL(IP1_23_20,	VI4_DATA7_B,		SEL_VIN4_1),
-	PINMUX_IPSR_MSEL(IP1_23_20,	IERX_B,			SEL_IEBUS_1),
+	PINMUX_IPSR_PHYS_MSEL(IP1_23_20,	PWM1_A,		I2C_SEL_3_0,	SEL_PWM1_0),
+	PINMUX_IPSR_PHYS_MSEL(IP1_23_20,	HRX3_D,		I2C_SEL_3_0,	SEL_HSCIF3_3),
+	PINMUX_IPSR_PHYS_MSEL(IP1_23_20,	VI4_DATA7_B,	I2C_SEL_3_0,	SEL_VIN4_1),
+	PINMUX_IPSR_PHYS_MSEL(IP1_23_20,	IERX_B,		I2C_SEL_3_0,	SEL_IEBUS_1),
+	PINMUX_IPSR_PHYS(IP0_23_20,	SCL3,		I2C_SEL_3_1),
 
-	PINMUX_IPSR_MSEL(IP1_27_24,	PWM2_A,			SEL_PWM2_0),
-	PINMUX_IPSR_MSEL(IP1_27_24,	HTX3_D,			SEL_HSCIF3_3),
-	PINMUX_IPSR_MSEL(IP1_27_24,	IETX_B,			SEL_IEBUS_1),
+	PINMUX_IPSR_PHYS_MSEL(IP1_27_24,	PWM2_A,		I2C_SEL_3_0,	SEL_PWM2_0),
+	PINMUX_IPSR_PHYS_MSEL(IP1_27_24,	HTX3_D,		I2C_SEL_3_0,	SEL_HSCIF3_3),
+	PINMUX_IPSR_PHYS_MSEL(IP1_27_24,	IETX_B,		I2C_SEL_3_0,	SEL_IEBUS_1),
+	PINMUX_IPSR_PHYS(IP1_27_24,	SDA3,		I2C_SEL_3_1),
 
 	PINMUX_IPSR_GPSR(IP1_31_28,	A0),
 	PINMUX_IPSR_GPSR(IP1_31_28,	LCDOUT16),
@@ -1016,35 +1025,35 @@ static const u16 pinmux_data[] = {
 
 	PINMUX_IPSR_GPSR(IP8_15_12,	SD1_CMD),
 	PINMUX_IPSR_MSEL(IP8_15_12,	MSIOF1_SYNC_G,		SEL_MSIOF1_6),
-	PINMUX_IPSR_MSEL(IP8_15_12,	NFCE_N_B,		SEL_NDFC_1),
+	PINMUX_IPSR_MSEL(IP8_15_12,	NFCE_N_B,		SEL_NDF_1),
 	PINMUX_IPSR_MSEL(IP8_15_12,	SIM0_D_A,		SEL_SIMCARD_0),
 	PINMUX_IPSR_MSEL(IP8_15_12,	STP_IVCXO27_1_B,	SEL_SSP1_1_1),
 
 	PINMUX_IPSR_GPSR(IP8_19_16,	SD1_DAT0),
 	PINMUX_IPSR_GPSR(IP8_19_16,	SD2_DAT4),
 	PINMUX_IPSR_MSEL(IP8_19_16,	MSIOF1_RXD_G,		SEL_MSIOF1_6),
-	PINMUX_IPSR_MSEL(IP8_19_16,	NFWP_N_B,		SEL_NDFC_1),
+	PINMUX_IPSR_MSEL(IP8_19_16,	NFWP_N_B,		SEL_NDF_1),
 	PINMUX_IPSR_MSEL(IP8_19_16,	TS_SCK1_B,		SEL_TSIF1_1),
 	PINMUX_IPSR_MSEL(IP8_19_16,	STP_ISCLK_1_B,		SEL_SSP1_1_1),
 
 	PINMUX_IPSR_GPSR(IP8_23_20,	SD1_DAT1),
 	PINMUX_IPSR_GPSR(IP8_23_20,	SD2_DAT5),
 	PINMUX_IPSR_MSEL(IP8_23_20,	MSIOF1_TXD_G,		SEL_MSIOF1_6),
-	PINMUX_IPSR_MSEL(IP8_23_20,	NFDATA14_B,		SEL_NDFC_1),
+	PINMUX_IPSR_MSEL(IP8_23_20,	NFDATA14_B,		SEL_NDF_1),
 	PINMUX_IPSR_MSEL(IP8_23_20,	TS_SPSYNC1_B,		SEL_TSIF1_1),
 	PINMUX_IPSR_MSEL(IP8_23_20,	STP_ISSYNC_1_B,		SEL_SSP1_1_1),
 
 	PINMUX_IPSR_GPSR(IP8_27_24,	SD1_DAT2),
 	PINMUX_IPSR_GPSR(IP8_27_24,	SD2_DAT6),
 	PINMUX_IPSR_MSEL(IP8_27_24,	MSIOF1_SS1_G,		SEL_MSIOF1_6),
-	PINMUX_IPSR_MSEL(IP8_27_24,	NFDATA15_B,		SEL_NDFC_1),
+	PINMUX_IPSR_MSEL(IP8_27_24,	NFDATA15_B,		SEL_NDF_1),
 	PINMUX_IPSR_MSEL(IP8_27_24,	TS_SDAT1_B,		SEL_TSIF1_1),
 	PINMUX_IPSR_MSEL(IP8_27_24,	STP_ISD_1_B,		SEL_SSP1_1_1),
 
 	PINMUX_IPSR_GPSR(IP8_31_28,	SD1_DAT3),
 	PINMUX_IPSR_GPSR(IP8_31_28,	SD2_DAT7),
 	PINMUX_IPSR_MSEL(IP8_31_28,	MSIOF1_SS2_G,		SEL_MSIOF1_6),
-	PINMUX_IPSR_MSEL(IP8_31_28,	NFRB_N_B,		SEL_NDFC_1),
+	PINMUX_IPSR_MSEL(IP8_31_28,	NFRB_N_B,		SEL_NDF_1),
 	PINMUX_IPSR_MSEL(IP8_31_28,	TS_SDEN1_B,		SEL_TSIF1_1),
 	PINMUX_IPSR_MSEL(IP8_31_28,	STP_ISEN_1_B,		SEL_SSP1_1_1),
 
@@ -1111,26 +1120,28 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_GPSR(IP11_7_4,	NFCLE),
 
 	PINMUX_IPSR_GPSR(IP11_11_8,	SD0_CD),
-	PINMUX_IPSR_MSEL(IP11_11_8,	NFDATA14_A,		SEL_NDFC_0),
+	PINMUX_IPSR_MSEL(IP11_11_8,	NFDATA14_A,		SEL_NDF_0),
 	PINMUX_IPSR_MSEL(IP11_11_8,	SCL2_B,			SEL_I2C2_1),
 	PINMUX_IPSR_MSEL(IP11_11_8,	SIM0_RST_A,		SEL_SIMCARD_0),
 
 	PINMUX_IPSR_GPSR(IP11_15_12,	SD0_WP),
-	PINMUX_IPSR_MSEL(IP11_15_12,	NFDATA15_A,		SEL_NDFC_0),
+	PINMUX_IPSR_MSEL(IP11_15_12,	NFDATA15_A,		SEL_NDF_0),
 	PINMUX_IPSR_MSEL(IP11_15_12,	SDA2_B,			SEL_I2C2_1),
 
-	PINMUX_IPSR_GPSR(IP11_19_16,	SD1_CD),
-	PINMUX_IPSR_MSEL(IP11_19_16,	NFRB_N_A,		SEL_NDFC_0),
-	PINMUX_IPSR_MSEL(IP11_19_16,	SIM0_CLK_B,		SEL_SIMCARD_1),
+	PINMUX_IPSR_MSEL(IP11_19_16,	SD1_CD,			I2C_SEL_0_0),
+	PINMUX_IPSR_PHYS_MSEL(IP11_19_16,	NFRB_N_A,	I2C_SEL_0_0, SEL_NDF_0),
+	PINMUX_IPSR_PHYS_MSEL(IP11_19_16,	SIM0_CLK_B,	I2C_SEL_0_0, SEL_SIMCARD_1),
+	PINMUX_IPSR_PHYS(IP11_19_16,	SCL0,			I2C_SEL_0_1),
 
-	PINMUX_IPSR_GPSR(IP11_23_20,	SD1_WP),
-	PINMUX_IPSR_MSEL(IP11_23_20,	NFCE_N_A,		SEL_NDFC_0),
-	PINMUX_IPSR_MSEL(IP11_23_20,	SIM0_D_B,		SEL_SIMCARD_1),
+	PINMUX_IPSR_MSEL(IP11_23_20,	SD1_WP,			I2C_SEL_0_0),
+	PINMUX_IPSR_PHYS_MSEL(IP11_23_20,	NFCE_N_A,	I2C_SEL_0_0, SEL_NDF_0),
+	PINMUX_IPSR_PHYS_MSEL(IP11_23_20,	SIM0_D_B,	I2C_SEL_0_0, SEL_SIMCARD_1),
+	PINMUX_IPSR_PHYS(IP11_23_20,	SDA0,			I2C_SEL_0_1),
 
 	PINMUX_IPSR_GPSR(IP11_27_24,	SCK0),
 	PINMUX_IPSR_MSEL(IP11_27_24,	HSCK1_B,		SEL_HSCIF1_1),
 	PINMUX_IPSR_MSEL(IP11_27_24,	MSIOF1_SS2_B,		SEL_MSIOF1_1),
-	PINMUX_IPSR_MSEL(IP11_27_24,	AUDIO_CLKC_B,		SEL_ADG_C_1),
+	PINMUX_IPSR_MSEL(IP11_27_24,	AUDIO_CLKC_B,		SEL_ADGC_1),
 	PINMUX_IPSR_MSEL(IP11_27_24,	SDA2_A,			SEL_I2C2_0),
 	PINMUX_IPSR_MSEL(IP11_27_24,	SIM0_RST_B,		SEL_SIMCARD_1),
 	PINMUX_IPSR_MSEL(IP11_27_24,	STP_OPWM_0_C,		SEL_SSP1_0_2),
@@ -1163,7 +1174,7 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_GPSR(IP12_11_8,	RTS0_N),
 	PINMUX_IPSR_MSEL(IP12_11_8,	HRTS1_N_B,		SEL_HSCIF1_1),
 	PINMUX_IPSR_MSEL(IP12_11_8,	MSIOF1_SS1_B,		SEL_MSIOF1_1),
-	PINMUX_IPSR_MSEL(IP12_11_8,	AUDIO_CLKA_B,		SEL_ADG_A_1),
+	PINMUX_IPSR_MSEL(IP12_11_8,	AUDIO_CLKA_B,		SEL_ADGA_1),
 	PINMUX_IPSR_MSEL(IP12_11_8,	SCL2_A,			SEL_I2C2_0),
 	PINMUX_IPSR_MSEL(IP12_11_8,	STP_IVCXO27_1_C,	SEL_SSP1_1_2),
 	PINMUX_IPSR_MSEL(IP12_11_8,	RIF0_SYNC_B,		SEL_DRIF0_1),
@@ -1222,7 +1233,7 @@ static const u16 pinmux_data[] = {
 
 	PINMUX_IPSR_GPSR(IP13_11_8,	HSCK0),
 	PINMUX_IPSR_MSEL(IP13_11_8,	MSIOF1_SCK_D,		SEL_MSIOF1_3),
-	PINMUX_IPSR_MSEL(IP13_11_8,	AUDIO_CLKB_A,		SEL_ADG_B_0),
+	PINMUX_IPSR_MSEL(IP13_11_8,	AUDIO_CLKB_A,		SEL_ADGB_0),
 	PINMUX_IPSR_MSEL(IP13_11_8,	SSI_SDATA1_B,		SEL_SSI1_1),
 	PINMUX_IPSR_MSEL(IP13_11_8,	TS_SCK0_D,		SEL_TSIF0_3),
 	PINMUX_IPSR_MSEL(IP13_11_8,	STP_ISCLK_0_D,		SEL_SSP1_0_3),
@@ -1268,8 +1279,8 @@ static const u16 pinmux_data[] = {
 	/* IPSR14 */
 	PINMUX_IPSR_GPSR(IP14_3_0,	MSIOF0_SS1),
 	PINMUX_IPSR_MSEL(IP14_3_0,	RX5_A,			SEL_SCIF5_0),
-	PINMUX_IPSR_MSEL(IP14_3_0,	NFWP_N_A,		SEL_NDFC_0),
-	PINMUX_IPSR_MSEL(IP14_3_0,	AUDIO_CLKA_C,		SEL_ADG_A_2),
+	PINMUX_IPSR_MSEL(IP14_3_0,	NFWP_N_A,		SEL_NDF_0),
+	PINMUX_IPSR_MSEL(IP14_3_0,	AUDIO_CLKA_C,		SEL_ADGA_2),
 	PINMUX_IPSR_MSEL(IP14_3_0,	SSI_SCK2_A,		SEL_SSI2_0),
 	PINMUX_IPSR_MSEL(IP14_3_0,	STP_IVCXO27_0_C,	SEL_SSP1_0_2),
 	PINMUX_IPSR_GPSR(IP14_3_0,	AUDIO_CLKOUT3_A),
@@ -1278,7 +1289,7 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_GPSR(IP14_7_4,	MSIOF0_SS2),
 	PINMUX_IPSR_MSEL(IP14_7_4,	TX5_A,			SEL_SCIF5_0),
 	PINMUX_IPSR_MSEL(IP14_7_4,	MSIOF1_SS2_D,		SEL_MSIOF1_3),
-	PINMUX_IPSR_MSEL(IP14_7_4,	AUDIO_CLKC_A,		SEL_ADG_C_0),
+	PINMUX_IPSR_MSEL(IP14_7_4,	AUDIO_CLKC_A,		SEL_ADGC_0),
 	PINMUX_IPSR_MSEL(IP14_7_4,	SSI_WS2_A,		SEL_SSI2_0),
 	PINMUX_IPSR_MSEL(IP14_7_4,	STP_OPWM_0_D,		SEL_SSP1_0_3),
 	PINMUX_IPSR_GPSR(IP14_7_4,	AUDIO_CLKOUT_D),
@@ -1407,10 +1418,9 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_MSEL(IP16_31_28,	SCK5_A,			SEL_SCIF5_0),
 
 	/* IPSR17 */
-	PINMUX_IPSR_MSEL(IP17_3_0,	AUDIO_CLKA_A,		SEL_ADG_A_0),
-	PINMUX_IPSR_GPSR(IP17_3_0,	CC5_OSCOUT),
+	PINMUX_IPSR_MSEL(IP17_3_0,	AUDIO_CLKA_A,		SEL_ADGA_0),
 
-	PINMUX_IPSR_MSEL(IP17_7_4,	AUDIO_CLKB_B,		SEL_ADG_B_1),
+	PINMUX_IPSR_MSEL(IP17_7_4,	AUDIO_CLKB_B,		SEL_ADGB_1),
 	PINMUX_IPSR_MSEL(IP17_7_4,	SCIF_CLK_A,		SEL_SCIF_0),
 	PINMUX_IPSR_MSEL(IP17_7_4,	STP_IVCXO27_1_D,	SEL_SSP1_1_3),
 	PINMUX_IPSR_MSEL(IP17_7_4,	REMOCON_A,		SEL_REMOCON_0),
@@ -1494,11 +1504,6 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_MSEL(IP18_7_4,	FMIN_C,			SEL_FM_2),
 	PINMUX_IPSR_MSEL(IP18_7_4,	FMIN_D,			SEL_FM_3),
 
-	/* I2C */
-	PINMUX_IPSR_NOGP(0,		I2C_SEL_0_1),
-	PINMUX_IPSR_NOGP(0,		I2C_SEL_3_1),
-	PINMUX_IPSR_NOGP(0,		I2C_SEL_5_1),
-
 /*
  * Static pins can not be muxed between different functions but
  * still need mark entries in the pinmux list. Add each static
@@ -2478,52 +2483,92 @@ static const unsigned int hscif4_data_b_mux[] = {
 };
 
 /* - I2C -------------------------------------------------------------------- */
+static const unsigned int i2c0_pins[] = {
+	/* SCL, SDA */
+	RCAR_GP_PIN(3, 14), RCAR_GP_PIN(3, 15),
+};
+
+static const unsigned int i2c0_mux[] = {
+	SCL0_MARK, SDA0_MARK,
+};
+
 static const unsigned int i2c1_a_pins[] = {
 	/* SDA, SCL */
 	RCAR_GP_PIN(5, 11), RCAR_GP_PIN(5, 10),
 };
+
 static const unsigned int i2c1_a_mux[] = {
 	SDA1_A_MARK, SCL1_A_MARK,
 };
+
 static const unsigned int i2c1_b_pins[] = {
 	/* SDA, SCL */
 	RCAR_GP_PIN(5, 24), RCAR_GP_PIN(5, 23),
 };
+
 static const unsigned int i2c1_b_mux[] = {
 	SDA1_B_MARK, SCL1_B_MARK,
 };
+
 static const unsigned int i2c2_a_pins[] = {
 	/* SDA, SCL */
 	RCAR_GP_PIN(5, 0), RCAR_GP_PIN(5, 4),
 };
+
 static const unsigned int i2c2_a_mux[] = {
 	SDA2_A_MARK, SCL2_A_MARK,
 };
+
 static const unsigned int i2c2_b_pins[] = {
 	/* SDA, SCL */
 	RCAR_GP_PIN(3, 13), RCAR_GP_PIN(3, 12),
 };
+
 static const unsigned int i2c2_b_mux[] = {
 	SDA2_B_MARK, SCL2_B_MARK,
 };
+
+static const unsigned int i2c3_pins[] = {
+	/* SCL, SDA */
+	RCAR_GP_PIN(2, 7), RCAR_GP_PIN(2, 8),
+};
+
+static const unsigned int i2c3_mux[] = {
+	SCL3_MARK, SDA3_MARK,
+};
+
+static const unsigned int i2c5_pins[] = {
+	/* SCL, SDA */
+	RCAR_GP_PIN(2, 13), RCAR_GP_PIN(2, 14),
+};
+
+static const unsigned int i2c5_mux[] = {
+	SCL5_MARK, SDA5_MARK,
+};
+
 static const unsigned int i2c6_a_pins[] = {
 	/* SDA, SCL */
 	RCAR_GP_PIN(1, 8), RCAR_GP_PIN(1, 11),
 };
+
 static const unsigned int i2c6_a_mux[] = {
 	SDA6_A_MARK, SCL6_A_MARK,
 };
+
 static const unsigned int i2c6_b_pins[] = {
 	/* SDA, SCL */
 	RCAR_GP_PIN(1, 26), RCAR_GP_PIN(1, 25),
 };
+
 static const unsigned int i2c6_b_mux[] = {
 	SDA6_B_MARK, SCL6_B_MARK,
 };
+
 static const unsigned int i2c6_c_pins[] = {
 	/* SDA, SCL */
 	RCAR_GP_PIN(0, 15), RCAR_GP_PIN(0, 14),
 };
+
 static const unsigned int i2c6_c_mux[] = {
 	SDA6_C_MARK, SCL6_C_MARK,
 };
@@ -4413,10 +4458,13 @@ static const struct sh_pfc_pin_group pinmux_groups[] = {
 	SH_PFC_PIN_GROUP(hscif4_clk),
 	SH_PFC_PIN_GROUP(hscif4_ctrl),
 	SH_PFC_PIN_GROUP(hscif4_data_b),
+	SH_PFC_PIN_GROUP(i2c0),
 	SH_PFC_PIN_GROUP(i2c1_a),
 	SH_PFC_PIN_GROUP(i2c1_b),
 	SH_PFC_PIN_GROUP(i2c2_a),
 	SH_PFC_PIN_GROUP(i2c2_b),
+	SH_PFC_PIN_GROUP(i2c3),
+	SH_PFC_PIN_GROUP(i2c5),
 	SH_PFC_PIN_GROUP(i2c6_a),
 	SH_PFC_PIN_GROUP(i2c6_b),
 	SH_PFC_PIN_GROUP(i2c6_c),
@@ -4807,6 +4855,10 @@ static const char * const hscif4_groups[] = {
 	"hscif4_data_b",
 };
 
+static const char * const i2c0_groups[] = {
+	"i2c0",
+};
+
 static const char * const i2c1_groups[] = {
 	"i2c1_a",
 	"i2c1_b",
@@ -4817,6 +4869,14 @@ static const char * const i2c2_groups[] = {
 	"i2c2_b",
 };
 
+static const char * const i2c3_groups[] = {
+	"i2c3",
+};
+
+static const char * const i2c5_groups[] = {
+	"i2c5",
+};
+
 static const char * const i2c6_groups[] = {
 	"i2c6_a",
 	"i2c6_b",
@@ -5166,8 +5226,11 @@ static const struct sh_pfc_function pinmux_functions[] = {
 	SH_PFC_FUNCTION(hscif2),
 	SH_PFC_FUNCTION(hscif3),
 	SH_PFC_FUNCTION(hscif4),
+	SH_PFC_FUNCTION(i2c0),
 	SH_PFC_FUNCTION(i2c1),
 	SH_PFC_FUNCTION(i2c2),
+	SH_PFC_FUNCTION(i2c3),
+	SH_PFC_FUNCTION(i2c5),
 	SH_PFC_FUNCTION(i2c6),
 	SH_PFC_FUNCTION(intc_ex),
 	SH_PFC_FUNCTION(msiof0),
@@ -5205,7 +5268,7 @@ static const struct sh_pfc_function pinmux_functions[] = {
 static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 #define F_(x, y)	FN_##y
 #define FM(x)		FN_##x
-	{ PINMUX_CFG_REG("GPSR0", 0xe6060100, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR0", 0xe6060100, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -5237,9 +5300,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_0_3_FN,	GPSR0_3,
 		GP_0_2_FN,	GPSR0_2,
 		GP_0_1_FN,	GPSR0_1,
-		GP_0_0_FN,	GPSR0_0, }
+		GP_0_0_FN,	GPSR0_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR1", 0xe6060104, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR1", 0xe6060104, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -5271,9 +5334,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_1_3_FN,	GPSR1_3,
 		GP_1_2_FN,	GPSR1_2,
 		GP_1_1_FN,	GPSR1_1,
-		GP_1_0_FN,	GPSR1_0, }
+		GP_1_0_FN,	GPSR1_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR2", 0xe6060108, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR2", 0xe6060108, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -5305,9 +5368,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_2_3_FN,	GPSR2_3,
 		GP_2_2_FN,	GPSR2_2,
 		GP_2_1_FN,	GPSR2_1,
-		GP_2_0_FN,	GPSR2_0, }
+		GP_2_0_FN,	GPSR2_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR3", 0xe606010c, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR3", 0xe606010c, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -5339,9 +5402,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_3_3_FN,	GPSR3_3,
 		GP_3_2_FN,	GPSR3_2,
 		GP_3_1_FN,	GPSR3_1,
-		GP_3_0_FN,	GPSR3_0, }
+		GP_3_0_FN,	GPSR3_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR4", 0xe6060110, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR4", 0xe6060110, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -5373,9 +5436,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_4_3_FN,	GPSR4_3,
 		GP_4_2_FN,	GPSR4_2,
 		GP_4_1_FN,	GPSR4_1,
-		GP_4_0_FN,	GPSR4_0, }
+		GP_4_0_FN,	GPSR4_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR5", 0xe6060114, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR5", 0xe6060114, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -5407,9 +5470,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_5_3_FN,	GPSR5_3,
 		GP_5_2_FN,	GPSR5_2,
 		GP_5_1_FN,	GPSR5_1,
-		GP_5_0_FN,	GPSR5_0, }
+		GP_5_0_FN,	GPSR5_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR6", 0xe6060118, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR6", 0xe6060118, 32, 1, GROUP(
 		GP_6_31_FN,	GPSR6_31,
 		GP_6_30_FN,	GPSR6_30,
 		GP_6_29_FN,	GPSR6_29,
@@ -5441,9 +5504,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_6_3_FN,	GPSR6_3,
 		GP_6_2_FN,	GPSR6_2,
 		GP_6_1_FN,	GPSR6_1,
-		GP_6_0_FN,	GPSR6_0, }
+		GP_6_0_FN,	GPSR6_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR7", 0xe606011c, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR7", 0xe606011c, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -5475,14 +5538,14 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_7_3_FN, GPSR7_3,
 		GP_7_2_FN, GPSR7_2,
 		GP_7_1_FN, GPSR7_1,
-		GP_7_0_FN, GPSR7_0, }
+		GP_7_0_FN, GPSR7_0, ))
 	},
 #undef F_
 #undef FM
 
 #define F_(x, y)	x,
 #define FM(x)		FN_##x,
-	{ PINMUX_CFG_REG("IPSR0", 0xe6060200, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR0", 0xe6060200, 32, 4, GROUP(
 		IP0_31_28
 		IP0_27_24
 		IP0_23_20
@@ -5490,9 +5553,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP0_15_12
 		IP0_11_8
 		IP0_7_4
-		IP0_3_0 }
+		IP0_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR1", 0xe6060204, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR1", 0xe6060204, 32, 4, GROUP(
 		IP1_31_28
 		IP1_27_24
 		IP1_23_20
@@ -5500,9 +5563,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP1_15_12
 		IP1_11_8
 		IP1_7_4
-		IP1_3_0 }
+		IP1_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR2", 0xe6060208, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR2", 0xe6060208, 32, 4, GROUP(
 		IP2_31_28
 		IP2_27_24
 		IP2_23_20
@@ -5510,9 +5573,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP2_15_12
 		IP2_11_8
 		IP2_7_4
-		IP2_3_0 }
+		IP2_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR3", 0xe606020c, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR3", 0xe606020c, 32, 4, GROUP(
 		IP3_31_28
 		IP3_27_24
 		IP3_23_20
@@ -5520,9 +5583,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP3_15_12
 		IP3_11_8
 		IP3_7_4
-		IP3_3_0 }
+		IP3_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR4", 0xe6060210, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR4", 0xe6060210, 32, 4, GROUP(
 		IP4_31_28
 		IP4_27_24
 		IP4_23_20
@@ -5530,9 +5593,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP4_15_12
 		IP4_11_8
 		IP4_7_4
-		IP4_3_0 }
+		IP4_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR5", 0xe6060214, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR5", 0xe6060214, 32, 4, GROUP(
 		IP5_31_28
 		IP5_27_24
 		IP5_23_20
@@ -5540,9 +5603,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP5_15_12
 		IP5_11_8
 		IP5_7_4
-		IP5_3_0 }
+		IP5_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR6", 0xe6060218, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR6", 0xe6060218, 32, 4, GROUP(
 		IP6_31_28
 		IP6_27_24
 		IP6_23_20
@@ -5550,9 +5613,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP6_15_12
 		IP6_11_8
 		IP6_7_4
-		IP6_3_0 }
+		IP6_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR7", 0xe606021c, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR7", 0xe606021c, 32, 4, GROUP(
 		IP7_31_28
 		IP7_27_24
 		IP7_23_20
@@ -5560,9 +5623,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP7_15_12 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		IP7_11_8
 		IP7_7_4
-		IP7_3_0 }
+		IP7_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR8", 0xe6060220, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR8", 0xe6060220, 32, 4, GROUP(
 		IP8_31_28
 		IP8_27_24
 		IP8_23_20
@@ -5570,9 +5633,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP8_15_12
 		IP8_11_8
 		IP8_7_4
-		IP8_3_0 }
+		IP8_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR9", 0xe6060224, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR9", 0xe6060224, 32, 4, GROUP(
 		IP9_31_28
 		IP9_27_24
 		IP9_23_20
@@ -5580,9 +5643,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP9_15_12
 		IP9_11_8
 		IP9_7_4
-		IP9_3_0 }
+		IP9_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR10", 0xe6060228, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR10", 0xe6060228, 32, 4, GROUP(
 		IP10_31_28
 		IP10_27_24
 		IP10_23_20
@@ -5590,9 +5653,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP10_15_12
 		IP10_11_8
 		IP10_7_4
-		IP10_3_0 }
+		IP10_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR11", 0xe606022c, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR11", 0xe606022c, 32, 4, GROUP(
 		IP11_31_28
 		IP11_27_24
 		IP11_23_20
@@ -5600,9 +5663,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP11_15_12
 		IP11_11_8
 		IP11_7_4
-		IP11_3_0 }
+		IP11_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR12", 0xe6060230, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR12", 0xe6060230, 32, 4, GROUP(
 		IP12_31_28
 		IP12_27_24
 		IP12_23_20
@@ -5610,9 +5673,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP12_15_12
 		IP12_11_8
 		IP12_7_4
-		IP12_3_0 }
+		IP12_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR13", 0xe6060234, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR13", 0xe6060234, 32, 4, GROUP(
 		IP13_31_28
 		IP13_27_24
 		IP13_23_20
@@ -5620,9 +5683,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP13_15_12
 		IP13_11_8
 		IP13_7_4
-		IP13_3_0 }
+		IP13_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR14", 0xe6060238, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR14", 0xe6060238, 32, 4, GROUP(
 		IP14_31_28
 		IP14_27_24
 		IP14_23_20
@@ -5630,9 +5693,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP14_15_12
 		IP14_11_8
 		IP14_7_4
-		IP14_3_0 }
+		IP14_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR15", 0xe606023c, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR15", 0xe606023c, 32, 4, GROUP(
 		IP15_31_28
 		IP15_27_24
 		IP15_23_20
@@ -5640,9 +5703,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP15_15_12
 		IP15_11_8
 		IP15_7_4
-		IP15_3_0 }
+		IP15_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR16", 0xe6060240, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR16", 0xe6060240, 32, 4, GROUP(
 		IP16_31_28
 		IP16_27_24
 		IP16_23_20
@@ -5650,9 +5713,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP16_15_12
 		IP16_11_8
 		IP16_7_4
-		IP16_3_0 }
+		IP16_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR17", 0xe6060244, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR17", 0xe6060244, 32, 4, GROUP(
 		IP17_31_28
 		IP17_27_24
 		IP17_23_20
@@ -5660,9 +5723,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP17_15_12
 		IP17_11_8
 		IP17_7_4
-		IP17_3_0 }
+		IP17_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR18", 0xe6060248, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR18", 0xe6060248, 32, 4, GROUP(
 		/* IP18_31_28 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP18_27_24 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP18_23_20 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -5670,7 +5733,7 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP18_15_12 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP18_11_8  */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		IP18_7_4
-		IP18_3_0 }
+		IP18_3_0 ))
 	},
 #undef F_
 #undef FM
@@ -5678,8 +5741,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 #define F_(x, y)	x,
 #define FM(x)		FN_##x,
 	{ PINMUX_CFG_REG_VAR("MOD_SEL0", 0xe6060500, 32,
-			     3, 2, 3, 1, 1, 1, 1, 1, 2, 1,
-			     1, 2, 1, 1, 1, 2, 2, 1, 2, 3) {
+			     GROUP(3, 2, 3, 1, 1, 1, 1, 1, 2, 1, 1, 2,
+				   1, 1, 1, 2, 2, 1, 2, 3),
+			     GROUP(
 		MOD_SEL0_31_30_29
 		MOD_SEL0_28_27
 		MOD_SEL0_26_25_24
@@ -5700,11 +5764,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		MOD_SEL0_5
 		MOD_SEL0_4_3
 		/* RESERVED 2, 1, 0 */
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
 	{ PINMUX_CFG_REG_VAR("MOD_SEL1", 0xe6060504, 32,
-			     2, 3, 1, 2, 3, 1, 1, 2, 1,
-			     2, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1) {
+			     GROUP(2, 3, 1, 2, 3, 1, 1, 2, 1, 2, 1, 1,
+				   1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1),
+			     GROUP(
 		MOD_SEL1_31_30
 		MOD_SEL1_29_28_27
 		MOD_SEL1_26
@@ -5727,11 +5792,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		MOD_SEL1_3
 		MOD_SEL1_2
 		MOD_SEL1_1
-		MOD_SEL1_0 }
+		MOD_SEL1_0 ))
 	},
 	{ PINMUX_CFG_REG_VAR("MOD_SEL2", 0xe6060508, 32,
-			     1, 1, 1, 2, 1, 3, 1, 1, 1, 1, 1, 1, 1,
-			     4, 4, 4, 3, 1) {
+			     GROUP(1, 1, 1, 2, 1, 3, 1, 1, 1, 1, 1, 1,
+				   1, 4, 4, 4, 3, 1),
+			     GROUP(
 		MOD_SEL2_31
 		MOD_SEL2_30
 		MOD_SEL2_29
@@ -5757,7 +5823,7 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0,
 		/* RESERVED 3, 2, 1 */
 		0, 0, 0, 0, 0, 0, 0, 0,
-		MOD_SEL2_0 }
+		MOD_SEL2_0 ))
 	},
 	{ },
 };
@@ -5878,7 +5944,7 @@ static const struct pinmux_drive_reg pinmux_drive_regs[] = {
 		{ RCAR_GP_PIN(0, 15),   24, 3 },	/* D15 */
 		{ RCAR_GP_PIN(7,  0),   20, 3 },	/* AVS1 */
 		{ RCAR_GP_PIN(7,  1),   16, 3 },	/* AVS2 */
-		{ RCAR_GP_PIN(7,  2),   12, 3 },	/* HDMI0_CEC */
+		{ RCAR_GP_PIN(7,  2),   12, 3 },	/* GP7_02 */
 		{ RCAR_GP_PIN(7,  3),    8, 3 },	/* GP7_03 */
 		{ PIN_A_NUMBER('P', 7),  4, 2 },	/* DU_DOTCLKIN0 */
 		{ PIN_A_NUMBER('P', 8),  0, 2 },	/* DU_DOTCLKIN1 */
@@ -6012,10 +6078,12 @@ static const struct pinmux_drive_reg pinmux_drive_regs[] = {
 
 enum ioctrl_regs {
 	POCCTRL,
+	TDSELCTRL,
 };
 
 static const struct pinmux_ioctrl_reg pinmux_ioctrl_regs[] = {
 	[POCCTRL] = { 0xe6060380, },
+	[TDSELCTRL] = { 0xe60603c0, },
 	{ /* sentinel */ },
 };
 
@@ -6132,7 +6200,7 @@ static const struct pinmux_bias_reg pinmux_bias_regs[] = {
 		[25] = RCAR_GP_PIN(0, 15),	/* D15 */
 		[26] = RCAR_GP_PIN(7,  0),	/* AVS1 */
 		[27] = RCAR_GP_PIN(7,  1),	/* AVS2 */
-		[28] = RCAR_GP_PIN(7,  2),	/* HDMI0_CEC */
+		[28] = RCAR_GP_PIN(7,  2),	/* GP7_02 */
 		[29] = RCAR_GP_PIN(7,  3),	/* GP7_03 */
 		[30] = PIN_A_NUMBER('P', 7),	/* DU_DOTCLKIN0 */
 		[31] = PIN_A_NUMBER('P', 8),	/* DU_DOTCLKIN1 */
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a77970.c b/drivers/pinctrl/sh-pfc/pfc-r8a77970.c
index c5e67ba29f7c..2d76b548b942 100644
--- a/drivers/pinctrl/sh-pfc/pfc-r8a77970.c
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a77970.c
@@ -12,6 +12,7 @@
  * Copyright (C) 2015  Renesas Electronics Corporation
  */
 
+#include <linux/errno.h>
 #include <linux/io.h>
 #include <linux/kernel.h>
 
@@ -171,19 +172,19 @@
 #define IP2_15_12	FM(DU_EXHSYNC_DU_HSYNC)		FM(HRX0)		F_(0, 0)	FM(A19)		FM(IRQ3)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP2_19_16	FM(DU_EXVSYNC_DU_VSYNC)		FM(MSIOF3_SCK)		F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP2_23_20	FM(DU_EXODDF_DU_ODDF_DISP_CDE)	FM(MSIOF3_SYNC)		F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP2_27_24	FM(IRQ0)			FM(CC5_OSCOUT)		F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP2_27_24	FM(IRQ0)			F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP2_31_28	FM(VI0_CLK)			FM(MSIOF2_SCK)		FM(SCK3)	F_(0, 0)	FM(HSCK3)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP3_3_0		FM(VI0_CLKENB)			FM(MSIOF2_RXD)		FM(RX3)		FM(RD_WR_N)	FM(HCTS3_N)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP3_7_4		FM(VI0_HSYNC_N)			FM(MSIOF2_TXD)		FM(TX3)		F_(0, 0)	FM(HRTS3_N)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP3_11_8	FM(VI0_VSYNC_N)			FM(MSIOF2_SYNC)		FM(CTS3_N)	F_(0, 0)	FM(HTX3)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP3_15_12	FM(VI0_DATA0)			FM(MSIOF2_SS1)		FM(RTS3_N_TANS)	F_(0, 0)	FM(HRX3)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP3_15_12	FM(VI0_DATA0)			FM(MSIOF2_SS1)		FM(RTS3_N)	F_(0, 0)	FM(HRX3)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP3_19_16	FM(VI0_DATA1)			FM(MSIOF2_SS2)		FM(SCK1)	F_(0, 0)	FM(SPEEDIN_A)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP3_23_20	FM(VI0_DATA2)			FM(AVB0_AVTP_PPS)	FM(SDA3_A)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP3_27_24	FM(VI0_DATA3)			FM(HSCK1)		FM(SCL3_A)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP3_31_28	FM(VI0_DATA4)			FM(HRTS1_N)		FM(RX1_A)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP4_3_0		FM(VI0_DATA5)			FM(HCTS1_N)		FM(TX1_A)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP4_7_4		FM(VI0_DATA6)			FM(HTX1)		FM(CTS1_N)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP4_11_8	FM(VI0_DATA7)			FM(HRX1)		FM(RTS1_N_TANS)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP4_11_8	FM(VI0_DATA7)			FM(HRX1)		FM(RTS1_N)	F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP4_15_12	FM(VI0_DATA8)			FM(HSCK2)		FM(PWM0_A)	FM(A22)		F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP4_19_16	FM(VI0_DATA9)			FM(HCTS2_N)		FM(PWM1_A)	FM(A23)		FM(FSO_CFE_0_N_B)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP4_23_20	FM(VI0_DATA10)			FM(HRTS2_N)		FM(PWM2_A)	FM(A24)		FM(FSO_CFE_1_N_B)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
@@ -198,18 +199,18 @@
 #define IP5_27_24	FM(VI1_DATA2)			FM(CANFD0_TX_B)		F_(0, 0)	FM(D5)		FM(MMC_D0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP5_31_28	FM(VI1_DATA3)			FM(CANFD0_RX_B)		F_(0, 0)	FM(D6)		FM(MMC_D1)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP6_3_0		FM(VI1_DATA4)			FM(CANFD_CLK_B)		F_(0, 0)	FM(D7)		FM(MMC_D2)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP6_7_4		FM(VI1_DATA5)			F_(0,0)			FM(SCK4)	FM(D8)		FM(MMC_D3)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP6_11_8	FM(VI1_DATA6)			F_(0,0)			FM(RX4)		FM(D9)		FM(MMC_CLK)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP6_15_12	FM(VI1_DATA7)			F_(0,0)			FM(TX4)		FM(D10)		FM(MMC_D4)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP6_19_16	FM(VI1_DATA8)			F_(0,0)			FM(CTS4_N)	FM(D11)		FM(MMC_D5)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP6_23_20	FM(VI1_DATA9)			F_(0,0)			FM(RTS4_N_TANS)	FM(D12)		FM(MMC_D6)		FM(SCL3_B)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP6_27_24	FM(VI1_DATA10)			F_(0,0)			F_(0, 0)	FM(D13)		FM(MMC_D7)		FM(SDA3_B)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP6_7_4		FM(VI1_DATA5)			F_(0, 0)		FM(SCK4)	FM(D8)		FM(MMC_D3)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP6_11_8	FM(VI1_DATA6)			F_(0, 0)		FM(RX4)		FM(D9)		FM(MMC_CLK)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP6_15_12	FM(VI1_DATA7)			F_(0, 0)		FM(TX4)		FM(D10)		FM(MMC_D4)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP6_19_16	FM(VI1_DATA8)			F_(0, 0)		FM(CTS4_N)	FM(D11)		FM(MMC_D5)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP6_23_20	FM(VI1_DATA9)			F_(0, 0)		FM(RTS4_N)	FM(D12)		FM(MMC_D6)		FM(SCL3_B)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP6_27_24	FM(VI1_DATA10)			F_(0, 0)		F_(0, 0)	FM(D13)		FM(MMC_D7)		FM(SDA3_B)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP6_31_28	FM(VI1_DATA11)			FM(SCL4)		FM(IRQ4)	FM(D14)		FM(MMC_WP)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP7_3_0		FM(VI1_FIELD)			FM(SDA4)		FM(IRQ5)	FM(D15)		FM(MMC_CD)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP7_7_4		FM(SCL0)			FM(DU_DR0)		FM(TPU0TO0)	FM(CLKOUT)	F_(0, 0)		FM(MSIOF0_RXD)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP7_11_8	FM(SDA0)			FM(DU_DR1)		FM(TPU0TO1)	FM(BS_N)	FM(SCK0)		FM(MSIOF0_TXD)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP7_15_12	FM(SCL1)			FM(DU_DG0)		FM(TPU0TO2)	FM(RD_N)	FM(CTS0_N)		FM(MSIOF0_SCK)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP7_19_16	FM(SDA1)			FM(DU_DG1)		FM(TPU0TO3)	FM(WE0_N)	FM(RTS0_N_TANS)		FM(MSIOF0_SYNC)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP7_19_16	FM(SDA1)			FM(DU_DG1)		FM(TPU0TO3)	FM(WE0_N)	FM(RTS0_N)		FM(MSIOF0_SYNC)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP7_23_20	FM(SCL2)			FM(DU_DB0)		FM(TCLK1_A)	FM(WE1_N)	FM(RX0)			FM(MSIOF0_SS1)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP7_27_24	FM(SDA2)			FM(DU_DB1)		FM(TCLK2_A)	FM(EX_WAIT0)	FM(TX0)			FM(MSIOF0_SS2)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP7_31_28	FM(AVB0_AVTP_CAPTURE)		F_(0, 0)		F_(0, 0)	F_(0, 0)	FM(FSCLKST2_N_B)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
@@ -470,7 +471,6 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_GPSR(IP2_23_20,	MSIOF3_SYNC),
 
 	PINMUX_IPSR_GPSR(IP2_27_24,	IRQ0),
-	PINMUX_IPSR_GPSR(IP2_27_24,	CC5_OSCOUT),
 
 	PINMUX_IPSR_GPSR(IP2_31_28,	VI0_CLK),
 	PINMUX_IPSR_GPSR(IP2_31_28,	MSIOF2_SCK),
@@ -496,7 +496,7 @@ static const u16 pinmux_data[] = {
 
 	PINMUX_IPSR_GPSR(IP3_15_12,	VI0_DATA0),
 	PINMUX_IPSR_GPSR(IP3_15_12,	MSIOF2_SS1),
-	PINMUX_IPSR_GPSR(IP3_15_12,	RTS3_N_TANS),
+	PINMUX_IPSR_GPSR(IP3_15_12,	RTS3_N),
 	PINMUX_IPSR_GPSR(IP3_15_12,	HRX3),
 
 	PINMUX_IPSR_GPSR(IP3_19_16,	VI0_DATA1),
@@ -527,7 +527,7 @@ static const u16 pinmux_data[] = {
 
 	PINMUX_IPSR_GPSR(IP4_11_8,	VI0_DATA7),
 	PINMUX_IPSR_GPSR(IP4_11_8,	HRX1),
-	PINMUX_IPSR_GPSR(IP4_11_8,	RTS1_N_TANS),
+	PINMUX_IPSR_GPSR(IP4_11_8,	RTS1_N),
 
 	PINMUX_IPSR_GPSR(IP4_15_12,	VI0_DATA8),
 	PINMUX_IPSR_GPSR(IP4_15_12,	HSCK2),
@@ -617,7 +617,7 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_GPSR(IP6_19_16,	MMC_D5),
 
 	PINMUX_IPSR_GPSR(IP6_23_20,	VI1_DATA9),
-	PINMUX_IPSR_GPSR(IP6_23_20,	RTS4_N_TANS),
+	PINMUX_IPSR_GPSR(IP6_23_20,	RTS4_N),
 	PINMUX_IPSR_GPSR(IP6_23_20,	D12),
 	PINMUX_IPSR_GPSR(IP6_23_20,	MMC_D6),
 	PINMUX_IPSR_MSEL(IP6_23_20,	SCL3_B,	SEL_I2C3_1),
@@ -664,7 +664,7 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_GPSR(IP7_19_16,	DU_DG1),
 	PINMUX_IPSR_GPSR(IP7_19_16,	TPU0TO3),
 	PINMUX_IPSR_GPSR(IP7_19_16,	WE0_N),
-	PINMUX_IPSR_GPSR(IP7_19_16,	RTS0_N_TANS),
+	PINMUX_IPSR_GPSR(IP7_19_16,	RTS0_N),
 	PINMUX_IPSR_GPSR(IP7_19_16,	MSIOF0_SYNC),
 
 	PINMUX_IPSR_GPSR(IP7_23_20,	SCL2),
@@ -1468,7 +1468,7 @@ static const unsigned int scif0_ctrl_pins[] = {
 	RCAR_GP_PIN(4, 3), RCAR_GP_PIN(4, 2),
 };
 static const unsigned int scif0_ctrl_mux[] = {
-	RTS0_N_TANS_MARK, CTS0_N_MARK,
+	RTS0_N_MARK, CTS0_N_MARK,
 };
 
 /* - SCIF1 ------------------------------------------------------------------ */
@@ -1491,7 +1491,7 @@ static const unsigned int scif1_ctrl_pins[] = {
 	RCAR_GP_PIN(2, 11), RCAR_GP_PIN(2, 10),
 };
 static const unsigned int scif1_ctrl_mux[] = {
-	RTS1_N_TANS_MARK, CTS1_N_MARK,
+	RTS1_N_MARK, CTS1_N_MARK,
 };
 static const unsigned int scif1_data_b_pins[] = {
 	/* RX, TX */
@@ -1521,7 +1521,7 @@ static const unsigned int scif3_ctrl_pins[] = {
 	RCAR_GP_PIN(2, 4), RCAR_GP_PIN(2, 3),
 };
 static const unsigned int scif3_ctrl_mux[] = {
-	RTS3_N_TANS_MARK, CTS3_N_MARK,
+	RTS3_N_MARK, CTS3_N_MARK,
 };
 
 /* - SCIF4 ------------------------------------------------------------------ */
@@ -1544,7 +1544,7 @@ static const unsigned int scif4_ctrl_pins[] = {
 	RCAR_GP_PIN(3, 13), RCAR_GP_PIN(3, 12),
 };
 static const unsigned int scif4_ctrl_mux[] = {
-	RTS4_N_TANS_MARK, CTS4_N_MARK,
+	RTS4_N_MARK, CTS4_N_MARK,
 };
 
 /* - TMU -------------------------------------------------------------------- */
@@ -2072,7 +2072,7 @@ static const struct sh_pfc_function pinmux_functions[] = {
 static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 #define F_(x, y)	FN_##y
 #define FM(x)		FN_##x
-	{ PINMUX_CFG_REG("GPSR0", 0xe6060100, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR0", 0xe6060100, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -2104,9 +2104,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_0_3_FN,	GPSR0_3,
 		GP_0_2_FN,	GPSR0_2,
 		GP_0_1_FN,	GPSR0_1,
-		GP_0_0_FN,	GPSR0_0, }
+		GP_0_0_FN,	GPSR0_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR1", 0xe6060104, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR1", 0xe6060104, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -2138,9 +2138,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_1_3_FN,	GPSR1_3,
 		GP_1_2_FN,	GPSR1_2,
 		GP_1_1_FN,	GPSR1_1,
-		GP_1_0_FN,	GPSR1_0, }
+		GP_1_0_FN,	GPSR1_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR2", 0xe6060108, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR2", 0xe6060108, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -2172,9 +2172,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_2_3_FN,	GPSR2_3,
 		GP_2_2_FN,	GPSR2_2,
 		GP_2_1_FN,	GPSR2_1,
-		GP_2_0_FN,	GPSR2_0, }
+		GP_2_0_FN,	GPSR2_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR3", 0xe606010c, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR3", 0xe606010c, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -2206,9 +2206,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_3_3_FN,	GPSR3_3,
 		GP_3_2_FN,	GPSR3_2,
 		GP_3_1_FN,	GPSR3_1,
-		GP_3_0_FN,	GPSR3_0, }
+		GP_3_0_FN,	GPSR3_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR4", 0xe6060110, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR4", 0xe6060110, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -2240,9 +2240,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_4_3_FN,	GPSR4_3,
 		GP_4_2_FN,	GPSR4_2,
 		GP_4_1_FN,	GPSR4_1,
-		GP_4_0_FN,	GPSR4_0, }
+		GP_4_0_FN,	GPSR4_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR5", 0xe6060114, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR5", 0xe6060114, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -2274,14 +2274,14 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_5_3_FN,	GPSR5_3,
 		GP_5_2_FN,	GPSR5_2,
 		GP_5_1_FN,	GPSR5_1,
-		GP_5_0_FN,	GPSR5_0, }
+		GP_5_0_FN,	GPSR5_0, ))
 	},
 #undef F_
 #undef FM
 
 #define F_(x, y)	x,
 #define FM(x)		FN_##x,
-	{ PINMUX_CFG_REG("IPSR0", 0xe6060200, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR0", 0xe6060200, 32, 4, GROUP(
 		IP0_31_28
 		IP0_27_24
 		IP0_23_20
@@ -2289,9 +2289,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP0_15_12
 		IP0_11_8
 		IP0_7_4
-		IP0_3_0 }
+		IP0_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR1", 0xe6060204, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR1", 0xe6060204, 32, 4, GROUP(
 		IP1_31_28
 		IP1_27_24
 		IP1_23_20
@@ -2299,9 +2299,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP1_15_12
 		IP1_11_8
 		IP1_7_4
-		IP1_3_0 }
+		IP1_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR2", 0xe6060208, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR2", 0xe6060208, 32, 4, GROUP(
 		IP2_31_28
 		IP2_27_24
 		IP2_23_20
@@ -2309,9 +2309,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP2_15_12
 		IP2_11_8
 		IP2_7_4
-		IP2_3_0 }
+		IP2_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR3", 0xe606020c, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR3", 0xe606020c, 32, 4, GROUP(
 		IP3_31_28
 		IP3_27_24
 		IP3_23_20
@@ -2319,9 +2319,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP3_15_12
 		IP3_11_8
 		IP3_7_4
-		IP3_3_0 }
+		IP3_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR4", 0xe6060210, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR4", 0xe6060210, 32, 4, GROUP(
 		IP4_31_28
 		IP4_27_24
 		IP4_23_20
@@ -2329,9 +2329,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP4_15_12
 		IP4_11_8
 		IP4_7_4
-		IP4_3_0 }
+		IP4_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR5", 0xe6060214, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR5", 0xe6060214, 32, 4, GROUP(
 		IP5_31_28
 		IP5_27_24
 		IP5_23_20
@@ -2339,9 +2339,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP5_15_12
 		IP5_11_8
 		IP5_7_4
-		IP5_3_0 }
+		IP5_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR6", 0xe6060218, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR6", 0xe6060218, 32, 4, GROUP(
 		IP6_31_28
 		IP6_27_24
 		IP6_23_20
@@ -2349,9 +2349,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP6_15_12
 		IP6_11_8
 		IP6_7_4
-		IP6_3_0 }
+		IP6_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR7", 0xe606021c, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR7", 0xe606021c, 32, 4, GROUP(
 		IP7_31_28
 		IP7_27_24
 		IP7_23_20
@@ -2359,9 +2359,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP7_15_12
 		IP7_11_8
 		IP7_7_4
-		IP7_3_0 }
+		IP7_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR8", 0xe6060220, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR8", 0xe6060220, 32, 4, GROUP(
 		IP8_31_28
 		IP8_27_24
 		IP8_23_20
@@ -2369,7 +2369,7 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP8_15_12
 		IP8_11_8
 		IP8_7_4
-		IP8_3_0 }
+		IP8_3_0 ))
 	},
 #undef F_
 #undef FM
@@ -2377,8 +2377,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 #define F_(x, y)	x,
 #define FM(x)		FN_##x,
 	{ PINMUX_CFG_REG_VAR("MOD_SEL0", 0xe6060500, 32,
-			     4, 4, 4, 4, 4,
-			     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) {
+			     GROUP(4, 4, 4, 4, 4, 1, 1, 1, 1, 1, 1, 1,
+				   1, 1, 1, 1, 1),
+			     GROUP(
 		/* RESERVED 31, 30, 29, 28 */
 		0, 0, 0, 0, 0, 0, 0, 0,	0, 0, 0, 0, 0, 0, 0, 0,
 		/* RESERVED 27, 26, 25, 24 */
@@ -2400,21 +2401,23 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		MOD_SEL0_3
 		MOD_SEL0_2
 		MOD_SEL0_1
-		MOD_SEL0_0 }
+		MOD_SEL0_0 ))
 	},
 	{ },
 };
 
 enum ioctrl_regs {
-	IOCTRL30,
-	IOCTRL31,
-	IOCTRL32,
+	POCCTRL0,
+	POCCTRL1,
+	POCCTRL2,
+	TDSELCTRL,
 };
 
 static const struct pinmux_ioctrl_reg pinmux_ioctrl_regs[] = {
-	[IOCTRL30] = { 0xe6060380 },
-	[IOCTRL31] = { 0xe6060384 },
-	[IOCTRL32] = { 0xe6060388 },
+	[POCCTRL0] = { 0xe6060380 },
+	[POCCTRL1] = { 0xe6060384 },
+	[POCCTRL2] = { 0xe6060388 },
+	[TDSELCTRL] = { 0xe60603c0, },
 	{ /* sentinel */ },
 };
 
@@ -2423,13 +2426,13 @@ static int r8a77970_pin_to_pocctrl(struct sh_pfc *pfc, unsigned int pin,
 {
 	int bit = pin & 0x1f;
 
-	*pocctrl = pinmux_ioctrl_regs[IOCTRL30].reg;
+	*pocctrl = pinmux_ioctrl_regs[POCCTRL0].reg;
 	if (pin >= RCAR_GP_PIN(0, 0) && pin <= RCAR_GP_PIN(0, 21))
 		return bit;
 	if (pin >= RCAR_GP_PIN(2, 0) && pin <= RCAR_GP_PIN(2, 9))
 		return bit + 22;
 
-	*pocctrl = pinmux_ioctrl_regs[IOCTRL31].reg;
+	*pocctrl = pinmux_ioctrl_regs[POCCTRL1].reg;
 	if (pin >= RCAR_GP_PIN(2, 10) && pin <= RCAR_GP_PIN(2, 16))
 		return bit - 10;
 	if (pin >= RCAR_GP_PIN(3, 0) && pin <= RCAR_GP_PIN(3, 16))
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a77980.c b/drivers/pinctrl/sh-pfc/pfc-r8a77980.c
index b807b67ae143..473da65890a7 100644
--- a/drivers/pinctrl/sh-pfc/pfc-r8a77980.c
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a77980.c
@@ -12,6 +12,7 @@
  * Copyright (C) 2015 Renesas Electronics Corporation
  */
 
+#include <linux/errno.h>
 #include <linux/io.h>
 #include <linux/kernel.h>
 
@@ -186,7 +187,7 @@
 #define IP0_7_4		FM(DU_DR3)			FM(RX4)			FM(GETHER_RMII_RX_ER)	FM(A1)		F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP0_11_8	FM(DU_DR4)			FM(TX4)			FM(GETHER_RMII_RXD0)	FM(A2)		F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP0_15_12	FM(DU_DR5)			FM(CTS4_N)		FM(GETHER_RMII_RXD1)	FM(A3)		F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP0_19_16	FM(DU_DR6)			FM(RTS4_N_TANS)		FM(GETHER_RMII_TXD_EN)	FM(A4)		F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP0_19_16	FM(DU_DR6)			FM(RTS4_N)		FM(GETHER_RMII_TXD_EN)	FM(A4)		F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP0_23_20	FM(DU_DR7)			F_(0, 0)		FM(GETHER_RMII_TXD0)	FM(A5)		F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP0_27_24	FM(DU_DG2)			F_(0, 0)		FM(GETHER_RMII_TXD1)	FM(A6)		F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP0_31_28	FM(DU_DG3)			FM(CPG_CPCKOUT)		FM(GETHER_RMII_REFCLK)	FM(A7)		FM(PWMFSW0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
@@ -204,19 +205,19 @@
 #define IP2_15_12	FM(DU_EXHSYNC_DU_HSYNC)		FM(MSIOF3_SS2)		FM(GETHER_PHY_INT_B)	FM(A19)		FM(FXR_TXENA_N)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP2_19_16	FM(DU_EXVSYNC_DU_VSYNC)		FM(MSIOF3_SCK)		F_(0, 0)		F_(0, 0)	FM(FXR_TXENB_N)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP2_23_20	FM(DU_EXODDF_DU_ODDF_DISP_CDE)	FM(MSIOF3_SYNC)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP2_27_24	FM(IRQ0)			FM(CC5_OSCOUT)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP2_27_24	FM(IRQ0)			F_(0, 0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP2_31_28	FM(VI0_CLK)			FM(MSIOF2_SCK)		FM(SCK3)		F_(0, 0)	FM(HSCK3)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP3_3_0		FM(VI0_CLKENB)			FM(MSIOF2_RXD)		FM(RX3)			FM(RD_WR_N)	FM(HCTS3_N)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP3_7_4		FM(VI0_HSYNC_N)			FM(MSIOF2_TXD)		FM(TX3)			F_(0, 0)	FM(HRTS3_N)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP3_11_8	FM(VI0_VSYNC_N)			FM(MSIOF2_SYNC)		FM(CTS3_N)		F_(0, 0)	FM(HTX3)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP3_15_12	FM(VI0_DATA0)			FM(MSIOF2_SS1)		FM(RTS3_N_TANS)		F_(0, 0)	FM(HRX3)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP3_15_12	FM(VI0_DATA0)			FM(MSIOF2_SS1)		FM(RTS3_N)		F_(0, 0)	FM(HRX3)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP3_19_16	FM(VI0_DATA1)			FM(MSIOF2_SS2)		FM(SCK1)		F_(0, 0)	FM(SPEEDIN_A)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP3_23_20	FM(VI0_DATA2)			FM(AVB_AVTP_PPS)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP3_27_24	FM(VI0_DATA3)			FM(HSCK1)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP3_31_28	FM(VI0_DATA4)			FM(HRTS1_N)		FM(RX1_A)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP4_3_0		FM(VI0_DATA5)			FM(HCTS1_N)		FM(TX1_A)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP4_7_4		FM(VI0_DATA6)			FM(HTX1)		FM(CTS1_N)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP4_11_8	FM(VI0_DATA7)			FM(HRX1)		FM(RTS1_N_TANS)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP4_11_8	FM(VI0_DATA7)			FM(HRX1)		FM(RTS1_N)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP4_15_12	FM(VI0_DATA8)			FM(HSCK2)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP4_19_16	FM(VI0_DATA9)			FM(HCTS2_N)		FM(PWM1_A)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP4_23_20	FM(VI0_DATA10)			FM(HRTS2_N)		FM(PWM2_A)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
@@ -242,7 +243,7 @@
 #define IP7_7_4		FM(SCL0)			F_(0, 0)		F_(0, 0)		FM(CLKOUT)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP7_11_8	FM(SDA0)			F_(0, 0)		F_(0, 0)		FM(BS_N)	FM(SCK0)	FM(HSCK0_B)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP7_15_12	FM(SCL1)			F_(0, 0)		FM(TPU0TO2)		FM(RD_N)	FM(CTS0_N)	FM(HCTS0_N_B)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP7_19_16	FM(SDA1)			F_(0, 0)		FM(TPU0TO3)		FM(WE0_N)	FM(RTS0_N_TANS)	FM(HRTS0_N_B)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP7_19_16	FM(SDA1)			F_(0, 0)		FM(TPU0TO3)		FM(WE0_N)	FM(RTS0_N)	FM(HRTS0_N_B)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP7_23_20	FM(SCL2)			F_(0, 0)		F_(0, 0)		FM(WE1_N)	FM(RX0)		FM(HRX0_B)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP7_27_24	FM(SDA2)			F_(0, 0)		F_(0, 0)		FM(EX_WAIT0)	FM(TX0)		FM(HTX0_B)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP7_31_28	FM(AVB_AVTP_MATCH)		FM(TPU0TO0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
@@ -469,7 +470,7 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_GPSR(IP0_15_12,	A3),
 
 	PINMUX_IPSR_GPSR(IP0_19_16,	DU_DR6),
-	PINMUX_IPSR_GPSR(IP0_19_16,	RTS4_N_TANS),
+	PINMUX_IPSR_GPSR(IP0_19_16,	RTS4_N),
 	PINMUX_IPSR_GPSR(IP0_19_16,	GETHER_RMII_TXD_EN),
 	PINMUX_IPSR_GPSR(IP0_19_16,	A4),
 
@@ -554,7 +555,6 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_GPSR(IP2_23_20,	MSIOF3_SYNC),
 
 	PINMUX_IPSR_GPSR(IP2_27_24,	IRQ0),
-	PINMUX_IPSR_GPSR(IP2_27_24,	CC5_OSCOUT),
 
 	PINMUX_IPSR_GPSR(IP2_31_28,	VI0_CLK),
 	PINMUX_IPSR_GPSR(IP2_31_28,	MSIOF2_SCK),
@@ -580,7 +580,7 @@ static const u16 pinmux_data[] = {
 
 	PINMUX_IPSR_GPSR(IP3_15_12,	VI0_DATA0),
 	PINMUX_IPSR_GPSR(IP3_15_12,	MSIOF2_SS1),
-	PINMUX_IPSR_GPSR(IP3_15_12,	RTS3_N_TANS),
+	PINMUX_IPSR_GPSR(IP3_15_12,	RTS3_N),
 	PINMUX_IPSR_GPSR(IP3_15_12,	HRX3),
 
 	PINMUX_IPSR_GPSR(IP3_19_16,	VI0_DATA1),
@@ -609,7 +609,7 @@ static const u16 pinmux_data[] = {
 
 	PINMUX_IPSR_GPSR(IP4_11_8,	VI0_DATA7),
 	PINMUX_IPSR_GPSR(IP4_11_8,	HRX1),
-	PINMUX_IPSR_GPSR(IP4_11_8,	RTS1_N_TANS),
+	PINMUX_IPSR_GPSR(IP4_11_8,	RTS1_N),
 
 	PINMUX_IPSR_GPSR(IP4_15_12,	VI0_DATA8),
 	PINMUX_IPSR_GPSR(IP4_15_12,	HSCK2),
@@ -728,7 +728,7 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_GPSR(IP7_19_16,	SDA1),
 	PINMUX_IPSR_GPSR(IP7_19_16,	TPU0TO3),
 	PINMUX_IPSR_GPSR(IP7_19_16,	WE0_N),
-	PINMUX_IPSR_GPSR(IP7_19_16,	RTS0_N_TANS),
+	PINMUX_IPSR_GPSR(IP7_19_16,	RTS0_N),
 	PINMUX_IPSR_MSEL(IP1_23_20,	HRTS0_N_B, SEL_HSCIF0_1),
 
 	PINMUX_IPSR_GPSR(IP7_23_20,	SCL2),
@@ -1726,11 +1726,11 @@ static const unsigned int scif0_clk_mux[] = {
 	SCK0_MARK,
 };
 static const unsigned int scif0_ctrl_pins[] = {
-	/* RTS0#/TANS, CTS0# */
+	/* RTS0#, CTS0# */
 	RCAR_GP_PIN(4, 3), RCAR_GP_PIN(4, 2),
 };
 static const unsigned int scif0_ctrl_mux[] = {
-	RTS0_N_TANS_MARK, CTS0_N_MARK,
+	RTS0_N_MARK, CTS0_N_MARK,
 };
 
 /* - SCIF1 ------------------------------------------------------------------ */
@@ -1749,11 +1749,11 @@ static const unsigned int scif1_clk_mux[] = {
 	SCK1_MARK,
 };
 static const unsigned int scif1_ctrl_pins[] = {
-	/* RTS1#/TANS, CTS1# */
+	/* RTS1#, CTS1# */
 	RCAR_GP_PIN(2, 11), RCAR_GP_PIN(2, 10),
 };
 static const unsigned int scif1_ctrl_mux[] = {
-	RTS1_N_TANS_MARK, CTS1_N_MARK,
+	RTS1_N_MARK, CTS1_N_MARK,
 };
 static const unsigned int scif1_data_b_pins[] = {
 	/* RX1, TX1 */
@@ -1779,11 +1779,11 @@ static const unsigned int scif3_clk_mux[] = {
 	SCK3_MARK,
 };
 static const unsigned int scif3_ctrl_pins[] = {
-	/* RTS3#/TANS, CTS3# */
+	/* RTS3#, CTS3# */
 	RCAR_GP_PIN(2, 4), RCAR_GP_PIN(2, 3),
 };
 static const unsigned int scif3_ctrl_mux[] = {
-	RTS3_N_TANS_MARK, CTS3_N_MARK,
+	RTS3_N_MARK, CTS3_N_MARK,
 };
 
 /* - SCIF4 ------------------------------------------------------------------ */
@@ -1802,11 +1802,11 @@ static const unsigned int scif4_clk_mux[] = {
 	SCK4_MARK,
 };
 static const unsigned int scif4_ctrl_pins[] = {
-	/* RTS4#/TANS, CTS4# */
+	/* RTS4#, CTS4# */
 	RCAR_GP_PIN(0, 4), RCAR_GP_PIN(0, 3),
 };
 static const unsigned int scif4_ctrl_mux[] = {
-	RTS4_N_TANS_MARK, CTS4_N_MARK,
+	RTS4_N_MARK, CTS4_N_MARK,
 };
 
 /* - SCIF Clock ------------------------------------------------------------- */
@@ -2474,7 +2474,7 @@ static const struct sh_pfc_function pinmux_functions[] = {
 static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 #define F_(x, y)	FN_##y
 #define FM(x)		FN_##x
-	{ PINMUX_CFG_REG("GPSR0", 0xe6060100, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR0", 0xe6060100, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -2506,9 +2506,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_0_3_FN,	GPSR0_3,
 		GP_0_2_FN,	GPSR0_2,
 		GP_0_1_FN,	GPSR0_1,
-		GP_0_0_FN,	GPSR0_0, }
+		GP_0_0_FN,	GPSR0_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR1", 0xe6060104, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR1", 0xe6060104, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -2540,9 +2540,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_1_3_FN,	GPSR1_3,
 		GP_1_2_FN,	GPSR1_2,
 		GP_1_1_FN,	GPSR1_1,
-		GP_1_0_FN,	GPSR1_0, }
+		GP_1_0_FN,	GPSR1_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR2", 0xe6060108, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR2", 0xe6060108, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		GP_2_29_FN,	GPSR2_29,
@@ -2574,9 +2574,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_2_3_FN,	GPSR2_3,
 		GP_2_2_FN,	GPSR2_2,
 		GP_2_1_FN,	GPSR2_1,
-		GP_2_0_FN,	GPSR2_0, }
+		GP_2_0_FN,	GPSR2_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR3", 0xe606010c, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR3", 0xe606010c, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -2608,9 +2608,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_3_3_FN,	GPSR3_3,
 		GP_3_2_FN,	GPSR3_2,
 		GP_3_1_FN,	GPSR3_1,
-		GP_3_0_FN,	GPSR3_0, }
+		GP_3_0_FN,	GPSR3_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR4", 0xe6060110, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR4", 0xe6060110, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -2642,9 +2642,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_4_3_FN,	GPSR4_3,
 		GP_4_2_FN,	GPSR4_2,
 		GP_4_1_FN,	GPSR4_1,
-		GP_4_0_FN,	GPSR4_0, }
+		GP_4_0_FN,	GPSR4_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR5", 0xe6060114, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR5", 0xe6060114, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -2676,14 +2676,14 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_5_3_FN,	GPSR5_3,
 		GP_5_2_FN,	GPSR5_2,
 		GP_5_1_FN,	GPSR5_1,
-		GP_5_0_FN,	GPSR5_0, }
+		GP_5_0_FN,	GPSR5_0, ))
 	},
 #undef F_
 #undef FM
 
 #define F_(x, y)	x,
 #define FM(x)		FN_##x,
-	{ PINMUX_CFG_REG("IPSR0", 0xe6060200, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR0", 0xe6060200, 32, 4, GROUP(
 		IP0_31_28
 		IP0_27_24
 		IP0_23_20
@@ -2691,9 +2691,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP0_15_12
 		IP0_11_8
 		IP0_7_4
-		IP0_3_0 }
+		IP0_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR1", 0xe6060204, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR1", 0xe6060204, 32, 4, GROUP(
 		IP1_31_28
 		IP1_27_24
 		IP1_23_20
@@ -2701,9 +2701,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP1_15_12
 		IP1_11_8
 		IP1_7_4
-		IP1_3_0 }
+		IP1_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR2", 0xe6060208, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR2", 0xe6060208, 32, 4, GROUP(
 		IP2_31_28
 		IP2_27_24
 		IP2_23_20
@@ -2711,9 +2711,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP2_15_12
 		IP2_11_8
 		IP2_7_4
-		IP2_3_0 }
+		IP2_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR3", 0xe606020c, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR3", 0xe606020c, 32, 4, GROUP(
 		IP3_31_28
 		IP3_27_24
 		IP3_23_20
@@ -2721,9 +2721,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP3_15_12
 		IP3_11_8
 		IP3_7_4
-		IP3_3_0 }
+		IP3_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR4", 0xe6060210, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR4", 0xe6060210, 32, 4, GROUP(
 		IP4_31_28
 		IP4_27_24
 		IP4_23_20
@@ -2731,9 +2731,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP4_15_12
 		IP4_11_8
 		IP4_7_4
-		IP4_3_0 }
+		IP4_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR5", 0xe6060214, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR5", 0xe6060214, 32, 4, GROUP(
 		IP5_31_28
 		IP5_27_24
 		IP5_23_20
@@ -2741,9 +2741,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP5_15_12
 		IP5_11_8
 		IP5_7_4
-		IP5_3_0 }
+		IP5_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR6", 0xe6060218, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR6", 0xe6060218, 32, 4, GROUP(
 		IP6_31_28
 		IP6_27_24
 		IP6_23_20
@@ -2751,9 +2751,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP6_15_12
 		IP6_11_8
 		IP6_7_4
-		IP6_3_0 }
+		IP6_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR7", 0xe606021c, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR7", 0xe606021c, 32, 4, GROUP(
 		IP7_31_28
 		IP7_27_24
 		IP7_23_20
@@ -2761,9 +2761,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP7_15_12
 		IP7_11_8
 		IP7_7_4
-		IP7_3_0 }
+		IP7_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR8", 0xe6060220, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR8", 0xe6060220, 32, 4, GROUP(
 		IP8_31_28
 		IP8_27_24
 		IP8_23_20
@@ -2771,9 +2771,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP8_15_12
 		IP8_11_8
 		IP8_7_4
-		IP8_3_0 }
+		IP8_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR9", 0xe6060224, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR9", 0xe6060224, 32, 4, GROUP(
 		IP9_31_28
 		IP9_27_24
 		IP9_23_20
@@ -2781,9 +2781,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP9_15_12
 		IP9_11_8
 		IP9_7_4
-		IP9_3_0 }
+		IP9_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR10", 0xe6060228, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR10", 0xe6060228, 32, 4, GROUP(
 		IP10_31_28
 		IP10_27_24
 		IP10_23_20
@@ -2791,7 +2791,7 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP10_15_12
 		IP10_11_8
 		IP10_7_4
-		IP10_3_0 }
+		IP10_3_0 ))
 	},
 #undef F_
 #undef FM
@@ -2799,8 +2799,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 #define F_(x, y)	x,
 #define FM(x)		FN_##x,
 	{ PINMUX_CFG_REG_VAR("MOD_SEL0", 0xe6060500, 32,
-			     4, 4, 4, 4, 4,
-			     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) {
+			     GROUP(4, 4, 4, 4, 4, 1, 1, 1, 1, 1, 1, 1,
+				   1, 1, 1, 1, 1),
+			     GROUP(
 		/* RESERVED 31, 30, 29, 28 */
 		0, 0, 0, 0, 0, 0, 0, 0,	0, 0, 0, 0, 0, 0, 0, 0,
 		/* RESERVED 27, 26, 25, 24 */
@@ -2822,23 +2823,25 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0,
 		MOD_SEL0_2
 		MOD_SEL0_1
-		MOD_SEL0_0 }
+		MOD_SEL0_0 ))
 	},
 	{ },
 };
 
 enum ioctrl_regs {
-	IOCTRL30,
-	IOCTRL31,
-	IOCTRL32,
-	IOCTRL33,
+	POCCTRL0,
+	POCCTRL1,
+	POCCTRL2,
+	POCCTRL3,
+	TDSELCTRL,
 };
 
 static const struct pinmux_ioctrl_reg pinmux_ioctrl_regs[] = {
-	[IOCTRL30] = { 0xe6060380, },
-	[IOCTRL31] = { 0xe6060384, },
-	[IOCTRL32] = { 0xe6060388, },
-	[IOCTRL33] = { 0xe606038c, },
+	[POCCTRL0] = { 0xe6060380, },
+	[POCCTRL1] = { 0xe6060384, },
+	[POCCTRL2] = { 0xe6060388, },
+	[POCCTRL3] = { 0xe606038c, },
+	[TDSELCTRL] = { 0xe60603c0, },
 	{ /* sentinel */ },
 };
 
@@ -2847,20 +2850,20 @@ static int r8a77980_pin_to_pocctrl(struct sh_pfc *pfc, unsigned int pin,
 {
 	int bit = pin & 0x1f;
 
-	*pocctrl = pinmux_ioctrl_regs[IOCTRL30].reg;
+	*pocctrl = pinmux_ioctrl_regs[POCCTRL0].reg;
 	if (pin >= RCAR_GP_PIN(0, 0) && pin <= RCAR_GP_PIN(0, 21))
 		return bit;
 	else if (pin >= RCAR_GP_PIN(2, 0) && pin <= RCAR_GP_PIN(2, 9))
 		return bit + 22;
 
-	*pocctrl = pinmux_ioctrl_regs[IOCTRL31].reg;
+	*pocctrl = pinmux_ioctrl_regs[POCCTRL1].reg;
 	if (pin >= RCAR_GP_PIN(2, 10) && pin <= RCAR_GP_PIN(2, 16))
 		return bit - 10;
 	if ((pin >= RCAR_GP_PIN(2, 17) && pin <= RCAR_GP_PIN(2, 24)) ||
 	    (pin >= RCAR_GP_PIN(3,  0) && pin <= RCAR_GP_PIN(3, 16)))
 		return bit + 7;
 
-	*pocctrl = pinmux_ioctrl_regs[IOCTRL32].reg;
+	*pocctrl = pinmux_ioctrl_regs[POCCTRL2].reg;
 	if (pin >= RCAR_GP_PIN(2, 25) && pin <= RCAR_GP_PIN(2, 29))
 		return pin - 25;
 
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a77990.c b/drivers/pinctrl/sh-pfc/pfc-r8a77990.c
index 151640c30e9d..91a837b02a36 100644
--- a/drivers/pinctrl/sh-pfc/pfc-r8a77990.c
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a77990.c
@@ -2,7 +2,7 @@
 /*
  * R8A77990 processor support - PFC hardware block.
  *
- * Copyright (C) 2018 Renesas Electronics Corp.
+ * Copyright (C) 2018-2019 Renesas Electronics Corp.
  *
  * This file is based on the drivers/pinctrl/sh-pfc/pfc-r8a7796.c
  *
@@ -11,6 +11,7 @@
  * Copyright (C) 2016-2017 Renesas Electronics Corp.
  */
 
+#include <linux/errno.h>
 #include <linux/kernel.h>
 
 #include "core.h"
@@ -165,7 +166,7 @@
 #define GPSR5_7		F_(SCK2_A,		IP12_7_4)
 #define GPSR5_6		F_(TX1,			IP12_3_0)
 #define GPSR5_5		F_(RX1,			IP11_31_28)
-#define GPSR5_4		F_(RTS0_N_TANS_A,	IP11_23_20)
+#define GPSR5_4		F_(RTS0_N_A,		IP11_23_20)
 #define GPSR5_3		F_(CTS0_N_A,		IP11_19_16)
 #define GPSR5_2		F_(TX0_A,		IP11_15_12)
 #define GPSR5_1		F_(RX0_A,		IP11_11_8)
@@ -219,7 +220,7 @@
 #define IP3_3_0		FM(A1)			FM(IRQ1)		FM(PWM3_A)		FM(DU_DOTCLKIN1)	FM(VI5_DATA0_A)		FM(DU_DISP_CDE) FM(SDA6_B)	FM(IETX)	FM(QCPV_QDE)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP3_7_4		FM(A2)			FM(IRQ2)		FM(AVB_AVTP_PPS)	FM(VI4_CLKENB)		FM(VI5_DATA1_A)		FM(DU_DISP)	FM(SCL6_B)	F_(0, 0)	FM(QSTVB_QVE)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP3_11_8	FM(A3)			FM(CTS4_N_A)		FM(PWM4_A)		FM(VI4_DATA12)		F_(0, 0)		FM(DU_DOTCLKOUT0) FM(HTX3_D)	FM(IECLK)	FM(LCDOUT12)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP3_15_12	FM(A4)			FM(RTS4_N_TANS_A)	FM(MSIOF3_SYNC_B)	FM(VI4_DATA8)		FM(PWM2_B)		FM(DU_DG4)	FM(RIF2_CLK_B)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP3_15_12	FM(A4)			FM(RTS4_N_A)		FM(MSIOF3_SYNC_B)	FM(VI4_DATA8)		FM(PWM2_B)		FM(DU_DG4)	FM(RIF2_CLK_B)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP3_19_16	FM(A5)			FM(SCK4_A)		FM(MSIOF3_SCK_B)	FM(VI4_DATA9)		FM(PWM3_B)		F_(0, 0)	FM(RIF2_SYNC_B)	F_(0, 0)	FM(QPOLA)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP3_23_20	FM(A6)			FM(RX4_A)		FM(MSIOF3_RXD_B)	FM(VI4_DATA10)		F_(0, 0)		F_(0, 0)	FM(RIF2_D0_B)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP3_27_24	FM(A7)			FM(TX4_A)		FM(MSIOF3_TXD_B)	FM(VI4_DATA11)		F_(0, 0)		F_(0, 0)	FM(RIF2_D1_B)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
@@ -240,10 +241,10 @@
 #define IP5_15_12	FM(CS0_N)		FM(SCL5)		F_(0, 0)		F_(0, 0)		F_(0, 0)		FM(DU_DR0)	FM(VI4_DATA2_B)	F_(0, 0)	FM(LCDOUT16)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP5_19_16	FM(WE0_N)		FM(SDA5)		F_(0, 0)		F_(0, 0)		F_(0, 0)		FM(DU_DR1)	FM(VI4_DATA3_B)	F_(0, 0)	FM(LCDOUT17)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP5_23_20	FM(D0)			FM(MSIOF3_SCK_A)	F_(0, 0)		F_(0, 0)		F_(0, 0)		FM(DU_DR2)	FM(CTS4_N_C)	F_(0, 0)	FM(LCDOUT18)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP5_27_24	FM(D1)			FM(MSIOF3_SYNC_A)	FM(SCK3_A)		FM(VI4_DATA23)		FM(VI5_CLKENB_A)	FM(DU_DB7)	FM(RTS4_N_TANS_C) F_(0, 0)	FM(LCDOUT7)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP5_27_24	FM(D1)			FM(MSIOF3_SYNC_A)	FM(SCK3_A)		FM(VI4_DATA23)		FM(VI5_CLKENB_A)	FM(DU_DB7)	FM(RTS4_N_C)	F_(0, 0)	FM(LCDOUT7)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP5_31_28	FM(D2)			FM(MSIOF3_RXD_A)	FM(RX5_C)		F_(0, 0)		FM(VI5_DATA14_A)	FM(DU_DR3)	FM(RX4_C)	F_(0, 0)	FM(LCDOUT19)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP6_3_0		FM(D3)			FM(MSIOF3_TXD_A)	FM(TX5_C)		F_(0, 0)		FM(VI5_DATA15_A)	FM(DU_DR4)	FM(TX4_C)	F_(0, 0)	FM(LCDOUT20)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP6_7_4		FM(D4)			FM(CANFD1_TX)		FM(HSCK3_B)		FM(CAN1_TX)		FM(RTS3_N_TANS_A)	FM(MSIOF3_SS2_A) F_(0, 0)	FM(VI5_DATA1_B)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP6_7_4		FM(D4)			FM(CANFD1_TX)		FM(HSCK3_B)		FM(CAN1_TX)		FM(RTS3_N_A)		FM(MSIOF3_SS2_A) F_(0, 0)	FM(VI5_DATA1_B)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP6_11_8	FM(D5)			FM(RX3_A)		FM(HRX3_B)		F_(0, 0)		F_(0, 0)		FM(DU_DR5)	FM(VI4_DATA4_B)	F_(0, 0)	FM(LCDOUT21)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP6_15_12	FM(D6)			FM(TX3_A)		FM(HTX3_B)		F_(0, 0)		F_(0, 0)		FM(DU_DR6)	FM(VI4_DATA5_B)	F_(0, 0)	FM(LCDOUT22)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP6_19_16	FM(D7)			FM(CANFD1_RX)		FM(IRQ5)		FM(CAN1_RX)		FM(CTS3_N_A)		F_(0, 0)	F_(0, 0)	FM(VI5_DATA2_B)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
@@ -289,8 +290,8 @@
 #define IP11_11_8	FM(RX0_A)		FM(HRX1_A)		FM(SSI_SCK2_A)		FM(RIF1_SYNC)		F_(0, 0)		F_(0, 0)	F_(0, 0)	FM(TS_SCK1)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP11_15_12	FM(TX0_A)		FM(HTX1_A)		FM(SSI_WS2_A)		FM(RIF1_D0)		F_(0, 0)		F_(0, 0)	F_(0, 0)	FM(TS_SDAT1)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP11_19_16	FM(CTS0_N_A)		FM(NFDATA14_A)		FM(AUDIO_CLKOUT_A)	FM(RIF1_D1)		FM(SCIF_CLK_A)		FM(FMCLK_A)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP11_23_20	FM(RTS0_N_TANS_A)	FM(NFDATA15_A)		FM(AUDIO_CLKOUT1_A)	FM(RIF1_CLK)		FM(SCL2_A)		FM(FMIN_A)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP11_27_24	FM(SCK0_A)		FM(HSCK1_A)		FM(USB3HS0_ID)		FM(RTS1_N_TANS)		FM(SDA2_A)		FM(FMCLK_C)	F_(0, 0)	F_(0, 0)	FM(USB0_ID)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP11_23_20	FM(RTS0_N_A)		FM(NFDATA15_A)		FM(AUDIO_CLKOUT1_A)	FM(RIF1_CLK)		FM(SCL2_A)		FM(FMIN_A)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP11_27_24	FM(SCK0_A)		FM(HSCK1_A)		FM(USB3HS0_ID)		FM(RTS1_N)		FM(SDA2_A)		FM(FMCLK_C)	F_(0, 0)	F_(0, 0)	FM(USB0_ID)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP11_31_28	FM(RX1)			FM(HRX2_B)		FM(SSI_SCK9_B)		FM(AUDIO_CLKOUT1_B)	F_(0, 0)		F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 
 /* IPSRx */		/* 0 */			/* 1 */			/* 2 */			/* 3 */			/* 4 */			/* 5 */		/* 6 */		/* 7 */		/* 8 */		/* 9 - F */
@@ -414,7 +415,7 @@ FM(IP12_31_28)	IP12_31_28	FM(IP13_31_28)	IP13_31_28	FM(IP14_31_28)	IP14_31_28	FM
 #define MOD_SEL0_22		FM(SEL_HSCIF2_0)		FM(SEL_HSCIF2_1)
 #define MOD_SEL0_21_20	   REV4(FM(SEL_I2C1_0),			FM(SEL_I2C1_1),			FM(SEL_I2C1_2),			FM(SEL_I2C1_3))
 #define MOD_SEL0_19_18_17  REV8(FM(SEL_I2C2_0),			FM(SEL_I2C2_1),			FM(SEL_I2C2_2),			FM(SEL_I2C2_3),		FM(SEL_I2C2_4),		F_(0, 0),	F_(0, 0),	F_(0, 0))
-#define MOD_SEL0_16		FM(SEL_NDFC_0)			FM(SEL_NDFC_1)
+#define MOD_SEL0_16		FM(SEL_NDF_0)			FM(SEL_NDF_1)
 #define MOD_SEL0_15		FM(SEL_PWM0_0)			FM(SEL_PWM0_1)
 #define MOD_SEL0_14		FM(SEL_PWM1_0)			FM(SEL_PWM1_1)
 #define MOD_SEL0_13_12	   REV4(FM(SEL_PWM2_0),			FM(SEL_PWM2_1),			FM(SEL_PWM2_2),			F_(0, 0))
@@ -429,8 +430,6 @@ FM(IP12_31_28)	IP12_31_28	FM(IP13_31_28)	IP13_31_28	FM(IP14_31_28)	IP14_31_28	FM
 #define MOD_SEL0_1_0	   REV4(FM(SEL_SPEED_PULSE_IF_0),	FM(SEL_SPEED_PULSE_IF_1),	FM(SEL_SPEED_PULSE_IF_2),	F_(0, 0))
 
 /* MOD_SEL1 */			/* 0 */				/* 1 */				/* 2 */				/* 3 */			/* 4 */			/* 5 */		/* 6 */		/* 7 */
-#define MOD_SEL1_31		FM(SEL_SIMCARD_0)		FM(SEL_SIMCARD_1)
-#define MOD_SEL1_30		FM(SEL_SSI2_0)			FM(SEL_SSI2_1)
 #define MOD_SEL1_29		FM(SEL_TIMER_TMU_0)		FM(SEL_TIMER_TMU_1)
 #define MOD_SEL1_28		FM(SEL_USB_20_CH0_0)		FM(SEL_USB_20_CH0_1)
 #define MOD_SEL1_26		FM(SEL_DRIF2_0)			FM(SEL_DRIF2_1)
@@ -451,8 +450,7 @@ FM(IP12_31_28)	IP12_31_28	FM(IP13_31_28)	IP13_31_28	FM(IP14_31_28)	IP14_31_28	FM
 
 #define PINMUX_MOD_SELS	\
 \
-			MOD_SEL1_31 \
-MOD_SEL0_30_29		MOD_SEL1_30 \
+MOD_SEL0_30_29 \
 			MOD_SEL1_29 \
 MOD_SEL0_28		MOD_SEL1_28 \
 MOD_SEL0_27_26 \
@@ -671,7 +669,7 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_GPSR(IP3_11_8,		LCDOUT12),
 
 	PINMUX_IPSR_GPSR(IP3_15_12,		A4),
-	PINMUX_IPSR_MSEL(IP3_15_12,		RTS4_N_TANS_A,	SEL_SCIF4_0),
+	PINMUX_IPSR_MSEL(IP3_15_12,		RTS4_N_A,	SEL_SCIF4_0),
 	PINMUX_IPSR_MSEL(IP3_15_12,		MSIOF3_SYNC_B,	SEL_MSIOF3_1),
 	PINMUX_IPSR_GPSR(IP3_15_12,		VI4_DATA8),
 	PINMUX_IPSR_MSEL(IP3_15_12,		PWM2_B,		SEL_PWM2_1),
@@ -821,7 +819,7 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_GPSR(IP5_27_24,		VI4_DATA23),
 	PINMUX_IPSR_MSEL(IP5_27_24,		VI5_CLKENB_A,	SEL_VIN5_0),
 	PINMUX_IPSR_GPSR(IP5_27_24,		DU_DB7),
-	PINMUX_IPSR_MSEL(IP5_27_24,		RTS4_N_TANS_C,	SEL_SCIF4_2),
+	PINMUX_IPSR_MSEL(IP5_27_24,		RTS4_N_C,	SEL_SCIF4_2),
 	PINMUX_IPSR_GPSR(IP5_27_24,		LCDOUT7),
 
 	PINMUX_IPSR_GPSR(IP5_31_28,		D2),
@@ -845,7 +843,7 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_GPSR(IP6_7_4,		CANFD1_TX),
 	PINMUX_IPSR_MSEL(IP6_7_4,		HSCK3_B,	SEL_HSCIF3_1),
 	PINMUX_IPSR_GPSR(IP6_7_4,		CAN1_TX),
-	PINMUX_IPSR_MSEL(IP6_7_4,		RTS3_N_TANS_A,	SEL_SCIF3_0),
+	PINMUX_IPSR_MSEL(IP6_7_4,		RTS3_N_A,	SEL_SCIF3_0),
 	PINMUX_IPSR_GPSR(IP6_7_4,		MSIOF3_SS2_A),
 	PINMUX_IPSR_MSEL(IP6_7_4,		VI5_DATA1_B,	SEL_VIN5_1),
 
@@ -984,23 +982,23 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_MSEL(IP8_19_16,		REMOCON_C,	SEL_REMOCON_2),
 
 	PINMUX_IPSR_GPSR(IP8_23_20,		SD1_CLK),
-	PINMUX_IPSR_MSEL(IP8_23_20,		NFDATA14_B,	SEL_NDFC_1),
+	PINMUX_IPSR_MSEL(IP8_23_20,		NFDATA14_B,	SEL_NDF_1),
 
 	PINMUX_IPSR_GPSR(IP8_27_24,		SD1_CMD),
-	PINMUX_IPSR_MSEL(IP8_27_24,		NFDATA15_B,	SEL_NDFC_1),
+	PINMUX_IPSR_MSEL(IP8_27_24,		NFDATA15_B,	SEL_NDF_1),
 
 	PINMUX_IPSR_GPSR(IP8_31_28,		SD1_DAT0),
-	PINMUX_IPSR_MSEL(IP8_31_28,		NFWP_N_B,	SEL_NDFC_1),
+	PINMUX_IPSR_MSEL(IP8_31_28,		NFWP_N_B,	SEL_NDF_1),
 
 	/* IPSR9 */
 	PINMUX_IPSR_GPSR(IP9_3_0,		SD1_DAT1),
-	PINMUX_IPSR_MSEL(IP9_3_0,		NFCE_N_B,	SEL_NDFC_1),
+	PINMUX_IPSR_MSEL(IP9_3_0,		NFCE_N_B,	SEL_NDF_1),
 
 	PINMUX_IPSR_GPSR(IP9_7_4,		SD1_DAT2),
-	PINMUX_IPSR_MSEL(IP9_7_4,		NFALE_B,	SEL_NDFC_1),
+	PINMUX_IPSR_MSEL(IP9_7_4,		NFALE_B,	SEL_NDF_1),
 
 	PINMUX_IPSR_GPSR(IP9_11_8,		SD1_DAT3),
-	PINMUX_IPSR_MSEL(IP9_11_8,		NFRB_N_B,	SEL_NDFC_1),
+	PINMUX_IPSR_MSEL(IP9_11_8,		NFRB_N_B,	SEL_NDF_1),
 
 	PINMUX_IPSR_GPSR(IP9_15_12,		SD3_CLK),
 	PINMUX_IPSR_GPSR(IP9_15_12,		NFWE_N),
@@ -1037,57 +1035,57 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_GPSR(IP10_23_20,		NFCLE),
 
 	PINMUX_IPSR_GPSR(IP10_27_24,		SD0_CD),
-	PINMUX_IPSR_GPSR(IP10_27_24,		NFALE_A),
+	PINMUX_IPSR_MSEL(IP10_27_24,		NFALE_A,	SEL_NDF_0),
 	PINMUX_IPSR_GPSR(IP10_27_24,		SD3_CD),
 	PINMUX_IPSR_MSEL(IP10_27_24,		RIF0_CLK_B,	SEL_DRIF0_1),
 	PINMUX_IPSR_MSEL(IP10_27_24,		SCL2_B,		SEL_I2C2_1),
 	PINMUX_IPSR_MSEL(IP10_27_24,		TCLK1_A,	SEL_TIMER_TMU_0),
-	PINMUX_IPSR_MSEL(IP10_27_24,		SSI_SCK2_B,	SEL_SSI2_1),
+	PINMUX_IPSR_GPSR(IP10_27_24,		SSI_SCK2_B),
 	PINMUX_IPSR_GPSR(IP10_27_24,		TS_SCK0),
 
 	PINMUX_IPSR_GPSR(IP10_31_28,		SD0_WP),
-	PINMUX_IPSR_GPSR(IP10_31_28,		NFRB_N_A),
+	PINMUX_IPSR_MSEL(IP10_31_28,		NFRB_N_A,	SEL_NDF_0),
 	PINMUX_IPSR_GPSR(IP10_31_28,		SD3_WP),
 	PINMUX_IPSR_MSEL(IP10_31_28,		RIF0_D0_B,	SEL_DRIF0_1),
 	PINMUX_IPSR_MSEL(IP10_31_28,		SDA2_B,		SEL_I2C2_1),
 	PINMUX_IPSR_MSEL(IP10_31_28,		TCLK2_A,	SEL_TIMER_TMU_0),
-	PINMUX_IPSR_MSEL(IP10_31_28,		SSI_WS2_B,	SEL_SSI2_1),
+	PINMUX_IPSR_GPSR(IP10_31_28,		SSI_WS2_B),
 	PINMUX_IPSR_GPSR(IP10_31_28,		TS_SDAT0),
 
 	/* IPSR11 */
 	PINMUX_IPSR_GPSR(IP11_3_0,		SD1_CD),
-	PINMUX_IPSR_MSEL(IP11_3_0,		NFCE_N_A,	SEL_NDFC_0),
+	PINMUX_IPSR_MSEL(IP11_3_0,		NFCE_N_A,	SEL_NDF_0),
 	PINMUX_IPSR_GPSR(IP11_3_0,		SSI_SCK1),
 	PINMUX_IPSR_MSEL(IP11_3_0,		RIF0_D1_B,	SEL_DRIF0_1),
 	PINMUX_IPSR_GPSR(IP11_3_0,		TS_SDEN0),
 
 	PINMUX_IPSR_GPSR(IP11_7_4,		SD1_WP),
-	PINMUX_IPSR_MSEL(IP11_7_4,		NFWP_N_A,	SEL_NDFC_0),
+	PINMUX_IPSR_MSEL(IP11_7_4,		NFWP_N_A,	SEL_NDF_0),
 	PINMUX_IPSR_GPSR(IP11_7_4,		SSI_WS1),
 	PINMUX_IPSR_MSEL(IP11_7_4,		RIF0_SYNC_B,	SEL_DRIF0_1),
 	PINMUX_IPSR_GPSR(IP11_7_4,		TS_SPSYNC0),
 
 	PINMUX_IPSR_MSEL(IP11_11_8,		RX0_A,		SEL_SCIF0_0),
 	PINMUX_IPSR_MSEL(IP11_11_8,		HRX1_A,		SEL_HSCIF1_0),
-	PINMUX_IPSR_MSEL(IP11_11_8,		SSI_SCK2_A,	SEL_SSI2_0),
+	PINMUX_IPSR_GPSR(IP11_11_8,		SSI_SCK2_A),
 	PINMUX_IPSR_GPSR(IP11_11_8,		RIF1_SYNC),
 	PINMUX_IPSR_GPSR(IP11_11_8,		TS_SCK1),
 
 	PINMUX_IPSR_MSEL(IP11_15_12,		TX0_A,		SEL_SCIF0_0),
 	PINMUX_IPSR_GPSR(IP11_15_12,		HTX1_A),
-	PINMUX_IPSR_MSEL(IP11_15_12,		SSI_WS2_A,	SEL_SSI2_0),
+	PINMUX_IPSR_GPSR(IP11_15_12,		SSI_WS2_A),
 	PINMUX_IPSR_GPSR(IP11_15_12,		RIF1_D0),
 	PINMUX_IPSR_GPSR(IP11_15_12,		TS_SDAT1),
 
 	PINMUX_IPSR_MSEL(IP11_19_16,		CTS0_N_A,	SEL_SCIF0_0),
-	PINMUX_IPSR_MSEL(IP11_19_16,		NFDATA14_A,	SEL_NDFC_0),
+	PINMUX_IPSR_MSEL(IP11_19_16,		NFDATA14_A,	SEL_NDF_0),
 	PINMUX_IPSR_GPSR(IP11_19_16,		AUDIO_CLKOUT_A),
 	PINMUX_IPSR_GPSR(IP11_19_16,		RIF1_D1),
 	PINMUX_IPSR_MSEL(IP11_19_16,		SCIF_CLK_A,	SEL_SCIF_0),
 	PINMUX_IPSR_MSEL(IP11_19_16,		FMCLK_A,	SEL_FM_0),
 
-	PINMUX_IPSR_MSEL(IP11_23_20,		RTS0_N_TANS_A,	SEL_SCIF0_0),
-	PINMUX_IPSR_MSEL(IP11_23_20,		NFDATA15_A,	SEL_NDFC_0),
+	PINMUX_IPSR_MSEL(IP11_23_20,		RTS0_N_A,	SEL_SCIF0_0),
+	PINMUX_IPSR_MSEL(IP11_23_20,		NFDATA15_A,	SEL_NDF_0),
 	PINMUX_IPSR_GPSR(IP11_23_20,		AUDIO_CLKOUT1_A),
 	PINMUX_IPSR_GPSR(IP11_23_20,		RIF1_CLK),
 	PINMUX_IPSR_MSEL(IP11_23_20,		SCL2_A,		SEL_I2C2_0),
@@ -1096,7 +1094,7 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_MSEL(IP11_27_24,		SCK0_A,		SEL_SCIF0_0),
 	PINMUX_IPSR_MSEL(IP11_27_24,		HSCK1_A,	SEL_HSCIF1_0),
 	PINMUX_IPSR_GPSR(IP11_27_24,		USB3HS0_ID),
-	PINMUX_IPSR_GPSR(IP11_27_24,		RTS1_N_TANS),
+	PINMUX_IPSR_GPSR(IP11_27_24,		RTS1_N),
 	PINMUX_IPSR_MSEL(IP11_27_24,		SDA2_A,		SEL_I2C2_0),
 	PINMUX_IPSR_MSEL(IP11_27_24,		FMCLK_C,	SEL_FM_2),
 	PINMUX_IPSR_GPSR(IP11_27_24,		USB0_ID),
@@ -1180,7 +1178,7 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_MSEL(IP13_19_16,		RIF0_D1_A,	SEL_DRIF0_0),
 	PINMUX_IPSR_MSEL(IP13_19_16,		SDA1_B,		SEL_I2C1_1),
 	PINMUX_IPSR_MSEL(IP13_19_16,		TCLK2_B,	SEL_TIMER_TMU_1),
-	PINMUX_IPSR_MSEL(IP13_19_16,		SIM0_D_A,	SEL_SIMCARD_0),
+	PINMUX_IPSR_GPSR(IP13_19_16,		SIM0_D_A),
 
 	PINMUX_IPSR_GPSR(IP13_23_20,		MLB_DAT),
 	PINMUX_IPSR_MSEL(IP13_23_20,		TX0_B,		SEL_SCIF0_1),
@@ -1248,7 +1246,7 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_GPSR(IP15_15_12,		TPU0TO2),
 	PINMUX_IPSR_MSEL(IP15_15_12,		SDA1_D,		SEL_I2C1_3),
 	PINMUX_IPSR_MSEL(IP15_15_12,		FSO_CFE_1_N_B,	SEL_FSO_1),
-	PINMUX_IPSR_MSEL(IP15_15_12,		SIM0_D_B,	SEL_SIMCARD_1),
+	PINMUX_IPSR_GPSR(IP15_15_12,		SIM0_D_B),
 
 	PINMUX_IPSR_GPSR(IP15_19_16,		SSI_SDATA6),
 	PINMUX_IPSR_MSEL(IP15_19_16,		HRTS2_N_A,	SEL_HSCIF2_0),
@@ -2839,7 +2837,7 @@ static const unsigned int scif0_ctrl_a_pins[] = {
 };
 
 static const unsigned int scif0_ctrl_a_mux[] = {
-	RTS0_N_TANS_A_MARK, CTS0_N_A_MARK,
+	RTS0_N_A_MARK, CTS0_N_A_MARK,
 };
 
 static const unsigned int scif0_data_b_pins[] = {
@@ -2885,7 +2883,7 @@ static const unsigned int scif1_ctrl_pins[] = {
 };
 
 static const unsigned int scif1_ctrl_mux[] = {
-	RTS1_N_TANS_MARK, CTS1_N_MARK,
+	RTS1_N_MARK, CTS1_N_MARK,
 };
 
 /* - SCIF2 ------------------------------------------------------------------ */
@@ -2941,7 +2939,7 @@ static const unsigned int scif3_ctrl_a_pins[] = {
 };
 
 static const unsigned int scif3_ctrl_a_mux[] = {
-	RTS3_N_TANS_A_MARK, CTS3_N_A_MARK,
+	RTS3_N_A_MARK, CTS3_N_A_MARK,
 };
 
 static const unsigned int scif3_data_b_pins[] = {
@@ -2996,7 +2994,7 @@ static const unsigned int scif4_ctrl_a_pins[] = {
 };
 
 static const unsigned int scif4_ctrl_a_mux[] = {
-	RTS4_N_TANS_A_MARK, CTS4_N_A_MARK,
+	RTS4_N_A_MARK, CTS4_N_A_MARK,
 };
 
 static const unsigned int scif4_data_b_pins[] = {
@@ -3032,7 +3030,7 @@ static const unsigned int scif4_ctrl_c_pins[] = {
 };
 
 static const unsigned int scif4_ctrl_c_mux[] = {
-	RTS4_N_TANS_C_MARK, CTS4_N_C_MARK,
+	RTS4_N_C_MARK, CTS4_N_C_MARK,
 };
 
 /* - SCIF5 ------------------------------------------------------------------ */
@@ -3766,8 +3764,8 @@ static const unsigned int vin5_clk_b_mux[] = {
 };
 
 static const struct {
-	struct sh_pfc_pin_group common[245];
-	struct sh_pfc_pin_group automotive[23];
+	struct sh_pfc_pin_group common[247];
+	struct sh_pfc_pin_group automotive[21];
 } pinmux_groups = {
 	.common = {
 		SH_PFC_PIN_GROUP(audio_clk_a),
@@ -3798,6 +3796,8 @@ static const struct {
 		SH_PFC_PIN_GROUP(can0_data),
 		SH_PFC_PIN_GROUP(can1_data),
 		SH_PFC_PIN_GROUP(can_clk),
+		SH_PFC_PIN_GROUP(canfd0_data),
+		SH_PFC_PIN_GROUP(canfd1_data),
 		SH_PFC_PIN_GROUP(du_rgb666),
 		SH_PFC_PIN_GROUP(du_rgb888),
 		SH_PFC_PIN_GROUP(du_clk_in_0),
@@ -4017,8 +4017,6 @@ static const struct {
 		SH_PFC_PIN_GROUP(vin5_clk_b),
 	},
 	.automotive = {
-		SH_PFC_PIN_GROUP(canfd0_data),
-		SH_PFC_PIN_GROUP(canfd1_data),
 		SH_PFC_PIN_GROUP(drif0_ctrl_a),
 		SH_PFC_PIN_GROUP(drif0_data0_a),
 		SH_PFC_PIN_GROUP(drif0_data1_a),
@@ -4465,8 +4463,8 @@ static const char * const vin5_groups[] = {
 };
 
 static const struct {
-	struct sh_pfc_function common[45];
-	struct sh_pfc_function automotive[6];
+	struct sh_pfc_function common[47];
+	struct sh_pfc_function automotive[4];
 } pinmux_functions = {
 	.common = {
 		SH_PFC_FUNCTION(audio_clk),
@@ -4474,6 +4472,8 @@ static const struct {
 		SH_PFC_FUNCTION(can0),
 		SH_PFC_FUNCTION(can1),
 		SH_PFC_FUNCTION(can_clk),
+		SH_PFC_FUNCTION(canfd0),
+		SH_PFC_FUNCTION(canfd1),
 		SH_PFC_FUNCTION(du),
 		SH_PFC_FUNCTION(hscif0),
 		SH_PFC_FUNCTION(hscif1),
@@ -4516,8 +4516,6 @@ static const struct {
 		SH_PFC_FUNCTION(vin5),
 	},
 	.automotive = {
-		SH_PFC_FUNCTION(canfd0),
-		SH_PFC_FUNCTION(canfd1),
 		SH_PFC_FUNCTION(drif0),
 		SH_PFC_FUNCTION(drif1),
 		SH_PFC_FUNCTION(drif2),
@@ -4528,7 +4526,7 @@ static const struct {
 static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 #define F_(x, y)	FN_##y
 #define FM(x)		FN_##x
-	{ PINMUX_CFG_REG("GPSR0", 0xe6060100, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR0", 0xe6060100, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -4560,9 +4558,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_0_3_FN,	GPSR0_3,
 		GP_0_2_FN,	GPSR0_2,
 		GP_0_1_FN,	GPSR0_1,
-		GP_0_0_FN,	GPSR0_0, }
+		GP_0_0_FN,	GPSR0_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR1", 0xe6060104, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR1", 0xe6060104, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -4594,9 +4592,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_1_3_FN,	GPSR1_3,
 		GP_1_2_FN,	GPSR1_2,
 		GP_1_1_FN,	GPSR1_1,
-		GP_1_0_FN,	GPSR1_0, }
+		GP_1_0_FN,	GPSR1_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR2", 0xe6060108, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR2", 0xe6060108, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -4628,9 +4626,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_2_3_FN,	GPSR2_3,
 		GP_2_2_FN,	GPSR2_2,
 		GP_2_1_FN,	GPSR2_1,
-		GP_2_0_FN,	GPSR2_0, }
+		GP_2_0_FN,	GPSR2_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR3", 0xe606010c, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR3", 0xe606010c, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -4662,9 +4660,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_3_3_FN,	GPSR3_3,
 		GP_3_2_FN,	GPSR3_2,
 		GP_3_1_FN,	GPSR3_1,
-		GP_3_0_FN,	GPSR3_0, }
+		GP_3_0_FN,	GPSR3_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR4", 0xe6060110, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR4", 0xe6060110, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -4696,9 +4694,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_4_3_FN,	GPSR4_3,
 		GP_4_2_FN,	GPSR4_2,
 		GP_4_1_FN,	GPSR4_1,
-		GP_4_0_FN,	GPSR4_0, }
+		GP_4_0_FN,	GPSR4_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR5", 0xe6060114, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR5", 0xe6060114, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -4730,9 +4728,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_5_3_FN,	GPSR5_3,
 		GP_5_2_FN,	GPSR5_2,
 		GP_5_1_FN,	GPSR5_1,
-		GP_5_0_FN,	GPSR5_0, }
+		GP_5_0_FN,	GPSR5_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR6", 0xe6060118, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR6", 0xe6060118, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -4764,14 +4762,14 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_6_3_FN,	GPSR6_3,
 		GP_6_2_FN,	GPSR6_2,
 		GP_6_1_FN,	GPSR6_1,
-		GP_6_0_FN,	GPSR6_0, }
+		GP_6_0_FN,	GPSR6_0, ))
 	},
 #undef F_
 #undef FM
 
 #define F_(x, y)	x,
 #define FM(x)		FN_##x,
-	{ PINMUX_CFG_REG("IPSR0", 0xe6060200, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR0", 0xe6060200, 32, 4, GROUP(
 		IP0_31_28
 		IP0_27_24
 		IP0_23_20
@@ -4779,9 +4777,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP0_15_12
 		IP0_11_8
 		IP0_7_4
-		IP0_3_0 }
+		IP0_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR1", 0xe6060204, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR1", 0xe6060204, 32, 4, GROUP(
 		IP1_31_28
 		IP1_27_24
 		IP1_23_20
@@ -4789,9 +4787,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP1_15_12
 		IP1_11_8
 		IP1_7_4
-		IP1_3_0 }
+		IP1_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR2", 0xe6060208, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR2", 0xe6060208, 32, 4, GROUP(
 		IP2_31_28
 		IP2_27_24
 		IP2_23_20
@@ -4799,9 +4797,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP2_15_12
 		IP2_11_8
 		IP2_7_4
-		IP2_3_0 }
+		IP2_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR3", 0xe606020c, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR3", 0xe606020c, 32, 4, GROUP(
 		IP3_31_28
 		IP3_27_24
 		IP3_23_20
@@ -4809,9 +4807,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP3_15_12
 		IP3_11_8
 		IP3_7_4
-		IP3_3_0 }
+		IP3_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR4", 0xe6060210, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR4", 0xe6060210, 32, 4, GROUP(
 		IP4_31_28
 		IP4_27_24
 		IP4_23_20
@@ -4819,9 +4817,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP4_15_12
 		IP4_11_8
 		IP4_7_4
-		IP4_3_0 }
+		IP4_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR5", 0xe6060214, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR5", 0xe6060214, 32, 4, GROUP(
 		IP5_31_28
 		IP5_27_24
 		IP5_23_20
@@ -4829,9 +4827,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP5_15_12
 		IP5_11_8
 		IP5_7_4
-		IP5_3_0 }
+		IP5_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR6", 0xe6060218, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR6", 0xe6060218, 32, 4, GROUP(
 		IP6_31_28
 		IP6_27_24
 		IP6_23_20
@@ -4839,9 +4837,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP6_15_12
 		IP6_11_8
 		IP6_7_4
-		IP6_3_0 }
+		IP6_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR7", 0xe606021c, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR7", 0xe606021c, 32, 4, GROUP(
 		IP7_31_28
 		IP7_27_24
 		IP7_23_20
@@ -4849,9 +4847,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP7_15_12
 		IP7_11_8
 		IP7_7_4
-		IP7_3_0 }
+		IP7_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR8", 0xe6060220, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR8", 0xe6060220, 32, 4, GROUP(
 		IP8_31_28
 		IP8_27_24
 		IP8_23_20
@@ -4859,9 +4857,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP8_15_12
 		IP8_11_8
 		IP8_7_4
-		IP8_3_0 }
+		IP8_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR9", 0xe6060224, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR9", 0xe6060224, 32, 4, GROUP(
 		IP9_31_28
 		IP9_27_24
 		IP9_23_20
@@ -4869,9 +4867,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP9_15_12
 		IP9_11_8
 		IP9_7_4
-		IP9_3_0 }
+		IP9_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR10", 0xe6060228, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR10", 0xe6060228, 32, 4, GROUP(
 		IP10_31_28
 		IP10_27_24
 		IP10_23_20
@@ -4879,9 +4877,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP10_15_12
 		IP10_11_8
 		IP10_7_4
-		IP10_3_0 }
+		IP10_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR11", 0xe606022c, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR11", 0xe606022c, 32, 4, GROUP(
 		IP11_31_28
 		IP11_27_24
 		IP11_23_20
@@ -4889,9 +4887,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP11_15_12
 		IP11_11_8
 		IP11_7_4
-		IP11_3_0 }
+		IP11_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR12", 0xe6060230, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR12", 0xe6060230, 32, 4, GROUP(
 		IP12_31_28
 		IP12_27_24
 		IP12_23_20
@@ -4899,9 +4897,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP12_15_12
 		IP12_11_8
 		IP12_7_4
-		IP12_3_0 }
+		IP12_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR13", 0xe6060234, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR13", 0xe6060234, 32, 4, GROUP(
 		IP13_31_28
 		IP13_27_24
 		IP13_23_20
@@ -4909,9 +4907,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP13_15_12
 		IP13_11_8
 		IP13_7_4
-		IP13_3_0 }
+		IP13_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR14", 0xe6060238, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR14", 0xe6060238, 32, 4, GROUP(
 		IP14_31_28
 		IP14_27_24
 		IP14_23_20
@@ -4919,9 +4917,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP14_15_12
 		IP14_11_8
 		IP14_7_4
-		IP14_3_0 }
+		IP14_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR15", 0xe606023c, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR15", 0xe606023c, 32, 4, GROUP(
 		IP15_31_28
 		IP15_27_24
 		IP15_23_20
@@ -4929,7 +4927,7 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP15_15_12
 		IP15_11_8
 		IP15_7_4
-		IP15_3_0 }
+		IP15_3_0 ))
 	},
 #undef F_
 #undef FM
@@ -4937,8 +4935,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 #define F_(x, y)	x,
 #define FM(x)		FN_##x,
 	{ PINMUX_CFG_REG_VAR("MOD_SEL0", 0xe6060500, 32,
-			     1, 2, 1, 2, 1, 1, 1, 1, 2, 3, 1,
-			     1, 1, 2, 2, 1, 1, 1, 2, 1, 1, 1, 2) {
+			     GROUP(1, 2, 1, 2, 1, 1, 1, 1, 2, 3, 1, 1,
+				   1, 2, 2, 1, 1, 1, 2, 1, 1, 1, 2),
+			     GROUP(
 		/* RESERVED 31 */
 		0, 0,
 		MOD_SEL0_30_29
@@ -4962,13 +4961,14 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		MOD_SEL0_4
 		MOD_SEL0_3
 		MOD_SEL0_2
-		MOD_SEL0_1_0 }
+		MOD_SEL0_1_0 ))
 	},
 	{ PINMUX_CFG_REG_VAR("MOD_SEL1", 0xe6060504, 32,
-			     1, 1, 1, 1, 1, 1, 1, 3, 3, 1, 1, 1,
-			     1, 2, 2, 2, 1, 1, 2, 1, 4) {
-		MOD_SEL1_31
-		MOD_SEL1_30
+			     GROUP(2, 1, 1, 1, 1, 1, 3, 3, 1, 1, 1, 1,
+				   2, 2, 2, 1, 1, 2, 1, 4),
+			     GROUP(
+		/* RESERVED 31, 30 */
+		0, 0, 0, 0,
 		MOD_SEL1_29
 		MOD_SEL1_28
 		/* RESERVED 27 */
@@ -4989,17 +4989,19 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		MOD_SEL1_6_5
 		MOD_SEL1_4
 		/* RESERVED 3, 2, 1, 0  */
-		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
 	{ },
 };
 
 enum ioctrl_regs {
-	IOCTRL30,
+	POCCTRL0,
+	TDSELCTRL,
 };
 
 static const struct pinmux_ioctrl_reg pinmux_ioctrl_regs[] = {
-	[IOCTRL30] = { 0xe6060380, },
+	[POCCTRL0] = { 0xe6060380, },
+	[TDSELCTRL] = { 0xe60603c0, },
 	{ /* sentinel */ },
 };
 
@@ -5008,7 +5010,7 @@ static int r8a77990_pin_to_pocctrl(struct sh_pfc *pfc, unsigned int pin,
 {
 	int bit = -EINVAL;
 
-	*pocctrl = pinmux_ioctrl_regs[IOCTRL30].reg;
+	*pocctrl = pinmux_ioctrl_regs[POCCTRL0].reg;
 
 	if (pin >= RCAR_GP_PIN(3, 0) && pin <= RCAR_GP_PIN(3, 11))
 		bit = pin & 0x1f;
@@ -5124,7 +5126,7 @@ static const struct pinmux_bias_reg pinmux_bias_regs[] = {
 	} },
 	{ PINMUX_BIAS_REG("PUEN3", 0xe606040c, "PUD3", 0xe606044c) {
 		 [0] = RCAR_GP_PIN(5,  0),	/* SCK0_A */
-		 [1] = RCAR_GP_PIN(5,  4),	/* RTS0#/TANS_A */
+		 [1] = RCAR_GP_PIN(5,  4),	/* RTS0#_A */
 		 [2] = RCAR_GP_PIN(5,  3),	/* CTS0#_A */
 		 [3] = RCAR_GP_PIN(5,  2),	/* TX0_A */
 		 [4] = RCAR_GP_PIN(5,  1),	/* RX0_A */
diff --git a/drivers/pinctrl/sh-pfc/pfc-r8a77995.c b/drivers/pinctrl/sh-pfc/pfc-r8a77995.c
index 9e377e3b9cb3..dd87085d48cb 100644
--- a/drivers/pinctrl/sh-pfc/pfc-r8a77995.c
+++ b/drivers/pinctrl/sh-pfc/pfc-r8a77995.c
@@ -11,6 +11,7 @@
  * Copyright (C) 2015  Renesas Electronics Corporation
  */
 
+#include <linux/errno.h>
 #include <linux/kernel.h>
 
 #include "core.h"
@@ -287,7 +288,7 @@
 #define IP10_23_20	FM(SSI_SDATA4_A)	FM(HTX0)		FM(SCL2_A)		FM(CAN1_RX_B)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP10_27_24	FM(SSI_WS4_A)		FM(HRX0)		FM(SDA2_A)		FM(CAN1_TX_B)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP10_31_28	FM(SCL1)		FM(CTS1_N)		F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP11_3_0	FM(SDA1)		FM(RTS1_N_TANS)		F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP11_3_0	FM(SDA1)		FM(RTS1_N)		F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP11_7_4	FM(MSIOF1_SCK)		FM(AVB0_AVTP_PPS_B)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP11_11_8	FM(MSIOF1_TXD)		FM(AVB0_AVTP_CAPTURE_B)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP11_15_12	FM(MSIOF1_RXD)		FM(AVB0_AVTP_MATCH_B)	F_(0, 0)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
@@ -298,7 +299,7 @@
 
 /* IPSRx */		/* 0 */			/* 1 */			/* 2 */			/* 3 */		/* 4 */			/* 5 */		/* 6  - F */
 #define IP12_3_0	FM(RX1_A)		FM(CTS0_N)		FM(TPU0TO0_B)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
-#define IP12_7_4	FM(TX1_A)		FM(RTS0_N_TANS)		FM(TPU0TO1_B)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
+#define IP12_7_4	FM(TX1_A)		FM(RTS0_N)		FM(TPU0TO1_B)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP12_11_8	FM(SCK2)		FM(MSIOF1_SS1)		FM(TPU0TO3_B)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP12_15_12	FM(TPU0TO0_A)		FM(AVB0_AVTP_CAPTURE_A)	FM(HCTS0_N)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
 #define IP12_19_16	FM(TPU0TO1_A)		FM(AVB0_AVTP_MATCH_A)	FM(HRTS0_N)		F_(0, 0)	F_(0, 0)		F_(0, 0)	F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0) F_(0, 0)
@@ -857,7 +858,7 @@ static const u16 pinmux_data[] = {
 
 	/* IPSR11 */
 	PINMUX_IPSR_GPSR(IP11_3_0,	SDA1),
-	PINMUX_IPSR_GPSR(IP11_3_0,	RTS1_N_TANS),
+	PINMUX_IPSR_GPSR(IP11_3_0,	RTS1_N),
 
 	PINMUX_IPSR_GPSR(IP11_7_4,	MSIOF1_SCK),
 	PINMUX_IPSR_MSEL(IP11_7_4,	AVB0_AVTP_PPS_B, SEL_ETHERAVB_1),
@@ -892,7 +893,7 @@ static const u16 pinmux_data[] = {
 	PINMUX_IPSR_GPSR(IP12_3_0,	TPU0TO0_B),
 
 	PINMUX_IPSR_MSEL(IP12_7_4,	TX1_A, SEL_SCIF1_0),
-	PINMUX_IPSR_GPSR(IP12_7_4,	RTS0_N_TANS),
+	PINMUX_IPSR_GPSR(IP12_7_4,	RTS0_N),
 	PINMUX_IPSR_GPSR(IP12_7_4,	TPU0TO1_B),
 
 	PINMUX_IPSR_GPSR(IP12_11_8,	SCK2),
@@ -1704,7 +1705,7 @@ static const unsigned int scif0_ctrl_pins[] = {
 	RCAR_GP_PIN(4, 24), RCAR_GP_PIN(4, 23),
 };
 static const unsigned int scif0_ctrl_mux[] = {
-	RTS0_N_TANS_MARK, CTS0_N_MARK,
+	RTS0_N_MARK, CTS0_N_MARK,
 };
 /* - SCIF1 ------------------------------------------------------------------ */
 static const unsigned int scif1_data_a_pins[] = {
@@ -1740,7 +1741,7 @@ static const unsigned int scif1_ctrl_pins[] = {
 	RCAR_GP_PIN(4, 11), RCAR_GP_PIN(4, 10),
 };
 static const unsigned int scif1_ctrl_mux[] = {
-	RTS1_N_TANS_MARK, CTS1_N_MARK,
+	RTS1_N_MARK, CTS1_N_MARK,
 };
 
 /* - SCIF2 ------------------------------------------------------------------ */
@@ -2374,7 +2375,7 @@ static const struct sh_pfc_function pinmux_functions[] = {
 static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 #define F_(x, y)	FN_##y
 #define FM(x)		FN_##x
-	{ PINMUX_CFG_REG("GPSR0", 0xe6060100, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR0", 0xe6060100, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -2406,9 +2407,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_0_3_FN,	GPSR0_3,
 		GP_0_2_FN,	GPSR0_2,
 		GP_0_1_FN,	GPSR0_1,
-		GP_0_0_FN,	GPSR0_0, }
+		GP_0_0_FN,	GPSR0_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR1", 0xe6060104, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR1", 0xe6060104, 32, 1, GROUP(
 		GP_1_31_FN,	GPSR1_31,
 		GP_1_30_FN,	GPSR1_30,
 		GP_1_29_FN,	GPSR1_29,
@@ -2440,9 +2441,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_1_3_FN,	GPSR1_3,
 		GP_1_2_FN,	GPSR1_2,
 		GP_1_1_FN,	GPSR1_1,
-		GP_1_0_FN,	GPSR1_0, }
+		GP_1_0_FN,	GPSR1_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR2", 0xe6060108, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR2", 0xe6060108, 32, 1, GROUP(
 		GP_2_31_FN,	GPSR2_31,
 		GP_2_30_FN,	GPSR2_30,
 		GP_2_29_FN,	GPSR2_29,
@@ -2474,9 +2475,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_2_3_FN,	GPSR2_3,
 		GP_2_2_FN,	GPSR2_2,
 		GP_2_1_FN,	GPSR2_1,
-		GP_2_0_FN,	GPSR2_0, }
+		GP_2_0_FN,	GPSR2_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR3", 0xe606010c, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR3", 0xe606010c, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -2508,9 +2509,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_3_3_FN,	GPSR3_3,
 		GP_3_2_FN,	GPSR3_2,
 		GP_3_1_FN,	GPSR3_1,
-		GP_3_0_FN,	GPSR3_0, }
+		GP_3_0_FN,	GPSR3_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR4", 0xe6060110, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR4", 0xe6060110, 32, 1, GROUP(
 		GP_4_31_FN,	GPSR4_31,
 		GP_4_30_FN,	GPSR4_30,
 		GP_4_29_FN,	GPSR4_29,
@@ -2542,9 +2543,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_4_3_FN,	GPSR4_3,
 		GP_4_2_FN,	GPSR4_2,
 		GP_4_1_FN,	GPSR4_1,
-		GP_4_0_FN,	GPSR4_0, }
+		GP_4_0_FN,	GPSR4_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR5", 0xe6060114, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR5", 0xe6060114, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -2576,9 +2577,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_5_3_FN,	GPSR5_3,
 		GP_5_2_FN,	GPSR5_2,
 		GP_5_1_FN,	GPSR5_1,
-		GP_5_0_FN,	GPSR5_0, }
+		GP_5_0_FN,	GPSR5_0, ))
 	},
-	{ PINMUX_CFG_REG("GPSR6", 0xe6060118, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR6", 0xe6060118, 32, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -2610,14 +2611,14 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_6_3_FN,	GPSR6_3,
 		GP_6_2_FN,	GPSR6_2,
 		GP_6_1_FN,	GPSR6_1,
-		GP_6_0_FN,	GPSR6_0, }
+		GP_6_0_FN,	GPSR6_0, ))
 	},
 #undef F_
 #undef FM
 
 #define F_(x, y)	x,
 #define FM(x)		FN_##x,
-	{ PINMUX_CFG_REG("IPSR0", 0xe6060200, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR0", 0xe6060200, 32, 4, GROUP(
 		IP0_31_28
 		IP0_27_24
 		IP0_23_20
@@ -2625,9 +2626,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP0_15_12
 		IP0_11_8
 		IP0_7_4
-		IP0_3_0 }
+		IP0_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR1", 0xe6060204, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR1", 0xe6060204, 32, 4, GROUP(
 		IP1_31_28
 		IP1_27_24
 		IP1_23_20
@@ -2635,9 +2636,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP1_15_12
 		IP1_11_8
 		IP1_7_4
-		IP1_3_0 }
+		IP1_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR2", 0xe6060208, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR2", 0xe6060208, 32, 4, GROUP(
 		IP2_31_28
 		IP2_27_24
 		IP2_23_20
@@ -2645,9 +2646,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP2_15_12
 		IP2_11_8
 		IP2_7_4
-		IP2_3_0 }
+		IP2_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR3", 0xe606020c, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR3", 0xe606020c, 32, 4, GROUP(
 		IP3_31_28
 		IP3_27_24
 		IP3_23_20
@@ -2655,9 +2656,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP3_15_12
 		IP3_11_8
 		IP3_7_4
-		IP3_3_0 }
+		IP3_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR4", 0xe6060210, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR4", 0xe6060210, 32, 4, GROUP(
 		IP4_31_28
 		IP4_27_24
 		IP4_23_20
@@ -2665,9 +2666,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP4_15_12
 		IP4_11_8
 		IP4_7_4
-		IP4_3_0 }
+		IP4_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR5", 0xe6060214, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR5", 0xe6060214, 32, 4, GROUP(
 		IP5_31_28
 		IP5_27_24
 		IP5_23_20
@@ -2675,9 +2676,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP5_15_12
 		IP5_11_8
 		IP5_7_4
-		IP5_3_0 }
+		IP5_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR6", 0xe6060218, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR6", 0xe6060218, 32, 4, GROUP(
 		IP6_31_28
 		IP6_27_24
 		IP6_23_20
@@ -2685,9 +2686,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP6_15_12
 		IP6_11_8
 		IP6_7_4
-		IP6_3_0 }
+		IP6_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR7", 0xe606021c, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR7", 0xe606021c, 32, 4, GROUP(
 		IP7_31_28
 		IP7_27_24
 		IP7_23_20
@@ -2695,9 +2696,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP7_15_12
 		IP7_11_8
 		IP7_7_4
-		IP7_3_0 }
+		IP7_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR8", 0xe6060220, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR8", 0xe6060220, 32, 4, GROUP(
 		IP8_31_28
 		IP8_27_24
 		IP8_23_20
@@ -2705,9 +2706,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP8_15_12
 		IP8_11_8
 		IP8_7_4
-		IP8_3_0 }
+		IP8_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR9", 0xe6060224, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR9", 0xe6060224, 32, 4, GROUP(
 		IP9_31_28
 		IP9_27_24
 		IP9_23_20
@@ -2715,9 +2716,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP9_15_12
 		IP9_11_8
 		IP9_7_4
-		IP9_3_0 }
+		IP9_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR10", 0xe6060228, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR10", 0xe6060228, 32, 4, GROUP(
 		IP10_31_28
 		IP10_27_24
 		IP10_23_20
@@ -2725,9 +2726,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP10_15_12
 		IP10_11_8
 		IP10_7_4
-		IP10_3_0 }
+		IP10_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR11", 0xe606022c, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR11", 0xe606022c, 32, 4, GROUP(
 		IP11_31_28
 		IP11_27_24
 		IP11_23_20
@@ -2735,9 +2736,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP11_15_12
 		IP11_11_8
 		IP11_7_4
-		IP11_3_0 }
+		IP11_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR12", 0xe6060230, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR12", 0xe6060230, 32, 4, GROUP(
 		IP12_31_28
 		IP12_27_24
 		IP12_23_20
@@ -2745,9 +2746,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		IP12_15_12
 		IP12_11_8
 		IP12_7_4
-		IP12_3_0 }
+		IP12_3_0 ))
 	},
-	{ PINMUX_CFG_REG("IPSR13", 0xe6060234, 32, 4) {
+	{ PINMUX_CFG_REG("IPSR13", 0xe6060234, 32, 4, GROUP(
 		/* IP13_31_28 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP13_27_24 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP13_23_20 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -2755,7 +2756,7 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP13_15_12 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* IP13_11_8  */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		IP13_7_4
-		IP13_3_0 }
+		IP13_3_0 ))
 	},
 #undef F_
 #undef FM
@@ -2763,8 +2764,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 #define F_(x, y)	x,
 #define FM(x)		FN_##x,
 	{ PINMUX_CFG_REG_VAR("MOD_SEL0", 0xe6060500, 32,
-			     1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 1,
-			     1, 1, 1, 1, 1, 1, 4, 1, 1, 1, 1, 1, 1) {
+			     GROUP(1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 1,
+				   1, 1, 1, 1, 1, 1, 4, 1, 1, 1, 1, 1, 1),
+			     GROUP(
 		/* RESERVED 31 */
 		0, 0,
 		MOD_SEL0_30
@@ -2792,11 +2794,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		MOD_SEL0_3
 		MOD_SEL0_2
 		MOD_SEL0_1
-		MOD_SEL0_0 }
+		MOD_SEL0_0 ))
 	},
 	{ PINMUX_CFG_REG_VAR("MOD_SEL1", 0xe6060504, 32,
-			     1, 1, 1, 1, 1, 1, 2, 4, 4,
-			     4, 4, 4, 4) {
+			     GROUP(1, 1, 1, 1, 1, 1, 2, 4, 4, 4, 4, 4, 4),
+			     GROUP(
 		MOD_SEL1_31
 		MOD_SEL1_30
 		MOD_SEL1_29
@@ -2816,7 +2818,7 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* RESERVED 7, 6, 5, 4  */
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		/* RESERVED 3, 2, 1, 0  */
-		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
 	{ },
 };
@@ -2833,6 +2835,15 @@ static int r8a77995_pin_to_pocctrl(struct sh_pfc *pfc, unsigned int pin, u32 *po
 	return bit;
 }
 
+enum ioctrl_regs {
+	TDSELCTRL,
+};
+
+static const struct pinmux_ioctrl_reg pinmux_ioctrl_regs[] = {
+	[TDSELCTRL] = { 0xe60603c0, },
+	{ /* sentinel */ },
+};
+
 static const struct sh_pfc_soc_operations r8a77995_pinmux_ops = {
 	.pin_to_pocctrl = r8a77995_pin_to_pocctrl,
 };
@@ -2852,6 +2863,7 @@ const struct sh_pfc_soc_info r8a77995_pinmux_info = {
 	.nr_functions = ARRAY_SIZE(pinmux_functions),
 
 	.cfg_regs = pinmux_config_regs,
+	.ioctrl_regs = pinmux_ioctrl_regs,
 
 	.pinmux_data = pinmux_data,
 	.pinmux_data_size = ARRAY_SIZE(pinmux_data),
diff --git a/drivers/pinctrl/sh-pfc/pfc-sh7203.c b/drivers/pinctrl/sh-pfc/pfc-sh7203.c
index 9ee468a9bd0e..811a6f2cb1fc 100644
--- a/drivers/pinctrl/sh-pfc/pfc-sh7203.c
+++ b/drivers/pinctrl/sh-pfc/pfc-sh7203.c
@@ -1073,7 +1073,7 @@ static const struct pinmux_func pinmux_func_gpios[] = {
 };
 
 static const struct pinmux_cfg_reg pinmux_config_regs[] = {
-	{ PINMUX_CFG_REG("PBIORL", 0xfffe3886, 16, 1) {
+	{ PINMUX_CFG_REG("PBIORL", 0xfffe3886, 16, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -1089,9 +1089,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0,
 		0, 0,
 		0, 0,
-		0, 0 }
+		0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PBCRL4", 0xfffe3890, 16, 4) {
+	{ PINMUX_CFG_REG("PBCRL4", 0xfffe3890, 16, 4, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -1099,9 +1099,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
 		PB12MD_00, PB12MD_01, PB12MD_10, PB12MD_11,
-		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PBCRL3", 0xfffe3892, 16, 4) {
+	{ PINMUX_CFG_REG("PBCRL3", 0xfffe3892, 16, 4, GROUP(
 		PB11MD_0, PB11MD_1,
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
@@ -1112,9 +1112,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
 		PB8MD_00, PB8MD_01, PB8MD_10, 0,
-		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PBCRL2", 0xfffe3894, 16, 4) {
+	{ PINMUX_CFG_REG("PBCRL2", 0xfffe3894, 16, 4, GROUP(
 		PB7MD_00, PB7MD_01, PB7MD_10, PB7MD_11,
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
@@ -1125,9 +1125,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
 		PB4MD_00, PB4MD_01, PB4MD_10, PB4MD_11,
-		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PBCRL1", 0xfffe3896, 16, 4) {
+	{ PINMUX_CFG_REG("PBCRL1", 0xfffe3896, 16, 4, GROUP(
 		PB3MD_00, PB3MD_01, PB3MD_10, PB3MD_11,
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
@@ -1138,9 +1138,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
 		PB0MD_00, PB0MD_01, PB0MD_10, PB0MD_11,
-		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("IFCR", 0xfffe38a2, 16, 4) {
+	{ PINMUX_CFG_REG("IFCR", 0xfffe38a2, 16, 4, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -1148,9 +1148,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
 		PB12IRQ_00, PB12IRQ_01, PB12IRQ_10, 0,
-		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PCIORL", 0xfffe3906, 16, 1) {
+	{ PINMUX_CFG_REG("PCIORL", 0xfffe3906, 16, 1, GROUP(
 		0, 0,
 		PC14_IN, PC14_OUT,
 		PC13_IN, PC13_OUT,
@@ -1166,9 +1166,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PC3_IN, PC3_OUT,
 		PC2_IN, PC2_OUT,
 		PC1_IN, PC1_OUT,
-		PC0_IN, PC0_OUT }
+		PC0_IN, PC0_OUT ))
 	},
-	{ PINMUX_CFG_REG("PCCRL4", 0xfffe3910, 16, 4) {
+	{ PINMUX_CFG_REG("PCCRL4", 0xfffe3910, 16, 4, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
 		PC14MD_0, PC14MD_1,
@@ -1178,9 +1178,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
 		PC12MD_0, PC12MD_1,
-		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PCCRL3", 0xfffe3912, 16, 4) {
+	{ PINMUX_CFG_REG("PCCRL3", 0xfffe3912, 16, 4, GROUP(
 		PC11MD_00, PC11MD_01, PC11MD_10, 0,
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
@@ -1191,9 +1191,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
 		PC8MD_0, PC8MD_1,
-		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PCCRL2", 0xfffe3914, 16, 4) {
+	{ PINMUX_CFG_REG("PCCRL2", 0xfffe3914, 16, 4, GROUP(
 		PC7MD_0, PC7MD_1,
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
@@ -1204,9 +1204,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
 		PC4MD_0, PC4MD_1,
-		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PCCRL1", 0xfffe3916, 16, 4) {
+	{ PINMUX_CFG_REG("PCCRL1", 0xfffe3916, 16, 4, GROUP(
 		PC3MD_0, PC3MD_1,
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
@@ -1217,9 +1217,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
 		PC0MD_00, PC0MD_01, PC0MD_10, 0,
-		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PDIORL", 0xfffe3986, 16, 1) {
+	{ PINMUX_CFG_REG("PDIORL", 0xfffe3986, 16, 1, GROUP(
 		PD15_IN, PD15_OUT,
 		PD14_IN, PD14_OUT,
 		PD13_IN, PD13_OUT,
@@ -1235,9 +1235,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PD3_IN, PD3_OUT,
 		PD2_IN, PD2_OUT,
 		PD1_IN, PD1_OUT,
-		PD0_IN, PD0_OUT }
+		PD0_IN, PD0_OUT ))
 	},
-	{ PINMUX_CFG_REG("PDCRL4", 0xfffe3990, 16, 4) {
+	{ PINMUX_CFG_REG("PDCRL4", 0xfffe3990, 16, 4, GROUP(
 		PD15MD_000, PD15MD_001, PD15MD_010, 0,
 		PD15MD_100, PD15MD_101, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -1252,9 +1252,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 
 		PD12MD_000, PD12MD_001, PD12MD_010, 0,
 		PD12MD_100, PD12MD_101, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PDCRL3", 0xfffe3992, 16, 4) {
+	{ PINMUX_CFG_REG("PDCRL3", 0xfffe3992, 16, 4, GROUP(
 		PD11MD_000, PD11MD_001, PD11MD_010, 0,
 		PD11MD_100, PD11MD_101, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -1269,9 +1269,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 
 		PD8MD_000, PD8MD_001, PD8MD_010, 0,
 		PD8MD_100, PD8MD_101, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PDCRL2", 0xfffe3994, 16, 4) {
+	{ PINMUX_CFG_REG("PDCRL2", 0xfffe3994, 16, 4, GROUP(
 		PD7MD_000, PD7MD_001, PD7MD_010, PD7MD_011,
 		PD7MD_100, PD7MD_101, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -1286,9 +1286,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 
 		PD4MD_000, PD4MD_001, PD4MD_010, PD4MD_011,
 		PD4MD_100, PD4MD_101, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PDCRL1", 0xfffe3996, 16, 4) {
+	{ PINMUX_CFG_REG("PDCRL1", 0xfffe3996, 16, 4, GROUP(
 		PD3MD_000, PD3MD_001, PD3MD_010, PD3MD_011,
 		PD3MD_100, PD3MD_101, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -1303,9 +1303,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 
 		PD0MD_000, PD0MD_001, PD0MD_010, PD0MD_011,
 		PD0MD_100, PD0MD_101, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PEIORL", 0xfffe3a06, 16, 1) {
+	{ PINMUX_CFG_REG("PEIORL", 0xfffe3a06, 16, 1, GROUP(
 		PE15_IN, PE15_OUT,
 		PE14_IN, PE14_OUT,
 		PE13_IN, PE13_OUT,
@@ -1321,9 +1321,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PE3_IN, PE3_OUT,
 		PE2_IN, PE2_OUT,
 		PE1_IN, PE1_OUT,
-		PE0_IN, PE0_OUT }
+		PE0_IN, PE0_OUT ))
 	},
-	{ PINMUX_CFG_REG("PECRL4", 0xfffe3a10, 16, 4) {
+	{ PINMUX_CFG_REG("PECRL4", 0xfffe3a10, 16, 4, GROUP(
 		PE15MD_00, PE15MD_01, 0, PE15MD_11,
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
@@ -1334,9 +1334,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
 		PE12MD_00, 0, 0, PE12MD_11,
-		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PECRL3", 0xfffe3a12, 16, 4) {
+	{ PINMUX_CFG_REG("PECRL3", 0xfffe3a12, 16, 4, GROUP(
 		PE11MD_000, PE11MD_001, PE11MD_010, 0,
 		PE11MD_100, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -1349,9 +1349,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
 		PE8MD_00, PE8MD_01, PE8MD_10, PE8MD_11,
-		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PECRL2", 0xfffe3a14, 16, 4) {
+	{ PINMUX_CFG_REG("PECRL2", 0xfffe3a14, 16, 4, GROUP(
 		PE7MD_000, PE7MD_001, PE7MD_010, PE7MD_011,
 		PE7MD_100, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -1366,9 +1366,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 
 		PE4MD_000, PE4MD_001, PE4MD_010, PE4MD_011,
 		PE4MD_100, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PECRL1", 0xfffe3a16, 16, 4) {
+	{ PINMUX_CFG_REG("PECRL1", 0xfffe3a16, 16, 4, GROUP(
 		PE3MD_00, PE3MD_01, 0, PE3MD_11,
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
@@ -1380,9 +1380,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 
 		PE0MD_000, PE0MD_001, 0, PE0MD_011,
 		PE0MD_100, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PFIORH", 0xfffe3a84, 16, 1) {
+	{ PINMUX_CFG_REG("PFIORH", 0xfffe3a84, 16, 1, GROUP(
 		0, 0,
 		PF30_IN, PF30_OUT,
 		PF29_IN, PF29_OUT,
@@ -1398,9 +1398,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PF19_IN, PF19_OUT,
 		PF18_IN, PF18_OUT,
 		PF17_IN, PF17_OUT,
-		PF16_IN, PF16_OUT }
+		PF16_IN, PF16_OUT ))
 	},
-	{ PINMUX_CFG_REG("PFIORL", 0xfffe3a86, 16, 1) {
+	{ PINMUX_CFG_REG("PFIORL", 0xfffe3a86, 16, 1, GROUP(
 		PF15_IN, PF15_OUT,
 		PF14_IN, PF14_OUT,
 		PF13_IN, PF13_OUT,
@@ -1416,9 +1416,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PF3_IN, PF3_OUT,
 		PF2_IN, PF2_OUT,
 		PF1_IN, PF1_OUT,
-		PF0_IN, PF0_OUT }
+		PF0_IN, PF0_OUT ))
 	},
-	{ PINMUX_CFG_REG("PFCRH4", 0xfffe3a88, 16, 4) {
+	{ PINMUX_CFG_REG("PFCRH4", 0xfffe3a88, 16, 4, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
 		PF30MD_0, PF30MD_1,
@@ -1428,9 +1428,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
 		PF28MD_0, PF28MD_1,
-		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PFCRH3", 0xfffe3a8a, 16, 4) {
+	{ PINMUX_CFG_REG("PFCRH3", 0xfffe3a8a, 16, 4, GROUP(
 		PF27MD_0, PF27MD_1,
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
@@ -1441,9 +1441,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
 		PF24MD_0, PF24MD_1,
-		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PFCRH2", 0xfffe3a8c, 16, 4) {
+	{ PINMUX_CFG_REG("PFCRH2", 0xfffe3a8c, 16, 4, GROUP(
 		PF23MD_00, PF23MD_01, PF23MD_10, 0,
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
@@ -1454,9 +1454,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
 		PF20MD_00, PF20MD_01, PF20MD_10, 0,
-		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PFCRH1", 0xfffe3a8e, 16, 4) {
+	{ PINMUX_CFG_REG("PFCRH1", 0xfffe3a8e, 16, 4, GROUP(
 		PF19MD_00, PF19MD_01, PF19MD_10, 0,
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
@@ -1467,9 +1467,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
 		PF16MD_00, PF16MD_01, PF16MD_10, 0,
-		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PFCRL4", 0xfffe3a90, 16, 4) {
+	{ PINMUX_CFG_REG("PFCRL4", 0xfffe3a90, 16, 4, GROUP(
 		PF15MD_00, PF15MD_01, PF15MD_10, 0,
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
@@ -1480,9 +1480,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
 		PF12MD_00, PF12MD_01, PF12MD_10, 0,
-		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PFCRL3", 0xfffe3a92, 16, 4) {
+	{ PINMUX_CFG_REG("PFCRL3", 0xfffe3a92, 16, 4, GROUP(
 		PF11MD_00, PF11MD_01, PF11MD_10, 0,
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
@@ -1493,9 +1493,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
 		PF8MD_00, PF8MD_01, PF8MD_10, 0,
-		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PFCRL2", 0xfffe3a94, 16, 4) {
+	{ PINMUX_CFG_REG("PFCRL2", 0xfffe3a94, 16, 4, GROUP(
 		PF7MD_00, PF7MD_01, PF7MD_10, PF7MD_11,
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
@@ -1506,9 +1506,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
 		PF4MD_00, PF4MD_01, PF4MD_10, PF4MD_11,
-		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PFCRL1", 0xfffe3a96, 16, 4) {
+	{ PINMUX_CFG_REG("PFCRL1", 0xfffe3a96, 16, 4, GROUP(
 		PF3MD_00, PF3MD_01, PF3MD_10, PF3MD_11,
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
@@ -1519,53 +1519,53 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
 		PF0MD_00, PF0MD_01, PF0MD_10, PF0MD_11,
-		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
 	{}
 };
 
 static const struct pinmux_data_reg pinmux_data_regs[] = {
-	{ PINMUX_DATA_REG("PADRL", 0xfffe3802, 16) {
+	{ PINMUX_DATA_REG("PADRL", 0xfffe3802, 16, GROUP(
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		PA7_DATA, PA6_DATA, PA5_DATA, PA4_DATA,
-		PA3_DATA, PA2_DATA, PA1_DATA, PA0_DATA }
+		PA3_DATA, PA2_DATA, PA1_DATA, PA0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PBDRL", 0xfffe3882, 16) {
+	{ PINMUX_DATA_REG("PBDRL", 0xfffe3882, 16, GROUP(
 		0, 0, 0, PB12_DATA,
 		PB11_DATA, PB10_DATA, PB9_DATA, PB8_DATA,
 		PB7_DATA, PB6_DATA, PB5_DATA, PB4_DATA,
-		PB3_DATA, PB2_DATA, PB1_DATA, PB0_DATA }
+		PB3_DATA, PB2_DATA, PB1_DATA, PB0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PCDRL", 0xfffe3902, 16) {
+	{ PINMUX_DATA_REG("PCDRL", 0xfffe3902, 16, GROUP(
 		0, PC14_DATA, PC13_DATA, PC12_DATA,
 		PC11_DATA, PC10_DATA, PC9_DATA, PC8_DATA,
 		PC7_DATA, PC6_DATA, PC5_DATA, PC4_DATA,
-		PC3_DATA, PC2_DATA, PC1_DATA, PC0_DATA }
+		PC3_DATA, PC2_DATA, PC1_DATA, PC0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PDDRL", 0xfffe3982, 16) {
+	{ PINMUX_DATA_REG("PDDRL", 0xfffe3982, 16, GROUP(
 		PD15_DATA, PD14_DATA, PD13_DATA, PD12_DATA,
 		PD11_DATA, PD10_DATA, PD9_DATA, PD8_DATA,
 		PD7_DATA, PD6_DATA, PD5_DATA, PD4_DATA,
-		PD3_DATA, PD2_DATA, PD1_DATA, PD0_DATA }
+		PD3_DATA, PD2_DATA, PD1_DATA, PD0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PEDRL", 0xfffe3a02, 16) {
+	{ PINMUX_DATA_REG("PEDRL", 0xfffe3a02, 16, GROUP(
 		PE15_DATA, PE14_DATA, PE13_DATA, PE12_DATA,
 		PE11_DATA, PE10_DATA, PE9_DATA, PE8_DATA,
 		PE7_DATA, PE6_DATA, PE5_DATA, PE4_DATA,
-		PE3_DATA, PE2_DATA, PE1_DATA, PE0_DATA }
+		PE3_DATA, PE2_DATA, PE1_DATA, PE0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PFDRH", 0xfffe3a80, 16) {
+	{ PINMUX_DATA_REG("PFDRH", 0xfffe3a80, 16, GROUP(
 		0, PF30_DATA, PF29_DATA, PF28_DATA,
 		PF27_DATA, PF26_DATA, PF25_DATA, PF24_DATA,
 		PF23_DATA, PF22_DATA, PF21_DATA, PF20_DATA,
-		PF19_DATA, PF18_DATA, PF17_DATA, PF16_DATA }
+		PF19_DATA, PF18_DATA, PF17_DATA, PF16_DATA ))
 	},
-	{ PINMUX_DATA_REG("PFDRL", 0xfffe3a82, 16) {
+	{ PINMUX_DATA_REG("PFDRL", 0xfffe3a82, 16, GROUP(
 		PF15_DATA, PF14_DATA, PF13_DATA, PF12_DATA,
 		PF11_DATA, PF10_DATA, PF9_DATA, PF8_DATA,
 		PF7_DATA, PF6_DATA, PF5_DATA, PF4_DATA,
-		PF3_DATA, PF2_DATA, PF1_DATA, PF0_DATA }
+		PF3_DATA, PF2_DATA, PF1_DATA, PF0_DATA ))
 	},
 	{ },
 };
diff --git a/drivers/pinctrl/sh-pfc/pfc-sh7264.c b/drivers/pinctrl/sh-pfc/pfc-sh7264.c
index 501de63e6c5f..4a95867deb8a 100644
--- a/drivers/pinctrl/sh-pfc/pfc-sh7264.c
+++ b/drivers/pinctrl/sh-pfc/pfc-sh7264.c
@@ -1466,17 +1466,17 @@ static const struct pinmux_func pinmux_func_gpios[] = {
 };
 
 static const struct pinmux_cfg_reg pinmux_config_regs[] = {
-	{ PINMUX_CFG_REG("PAIOR0", 0xfffe3812, 16, 1) {
+	{ PINMUX_CFG_REG("PAIOR0", 0xfffe3812, 16, 1, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PA3_IN, PA3_OUT,
 		PA2_IN, PA2_OUT,
 		PA1_IN, PA1_OUT,
-		PA0_IN,	PA0_OUT }
+		PA0_IN,	PA0_OUT ))
 	},
 
-	{ PINMUX_CFG_REG("PBCR5", 0xfffe3824, 16, 4) {
+	{ PINMUX_CFG_REG("PBCR5", 0xfffe3824, 16, 4, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PB22MD_00, PB22MD_01, PB22MD_10, 0, 0, 0, 0, 0,
@@ -1484,10 +1484,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PB21MD_0, PB21MD_1, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, PB20MD_1, 0, 0, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 
 	},
-	{ PINMUX_CFG_REG("PBCR4", 0xfffe3826, 16, 4) {
+	{ PINMUX_CFG_REG("PBCR4", 0xfffe3826, 16, 4, GROUP(
 		0, PB19MD_01, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, PB18MD_01, 0, 0, 0, 0, 0, 0,
@@ -1495,9 +1495,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, PB17MD_01, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, PB16MD_01, 0, 0, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PBCR3", 0xfffe3828, 16, 4) {
+	{ PINMUX_CFG_REG("PBCR3", 0xfffe3828, 16, 4, GROUP(
 		0, PB15MD_01, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, PB14MD_01, 0, 0, 0, 0, 0, 0,
@@ -1505,9 +1505,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, PB13MD_01, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, PB12MD_01, 0, 0, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PBCR2", 0xfffe382a, 16, 4) {
+	{ PINMUX_CFG_REG("PBCR2", 0xfffe382a, 16, 4, GROUP(
 		0, PB11MD_01, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, PB10MD_01, 0, 0, 0, 0, 0, 0,
@@ -1515,9 +1515,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, PB9MD_01, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, PB8MD_01, 0, 0, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PBCR1", 0xfffe382c, 16, 4) {
+	{ PINMUX_CFG_REG("PBCR1", 0xfffe382c, 16, 4, GROUP(
 		0, PB7MD_01, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, PB6MD_01, 0, 0, 0, 0, 0, 0,
@@ -1525,9 +1525,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, PB5MD_01, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, PB4MD_01, 0, 0, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PBCR0", 0xfffe382e, 16, 4) {
+	{ PINMUX_CFG_REG("PBCR0", 0xfffe382e, 16, 4, GROUP(
 		0, PB3MD_1, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, PB2MD_1, 0, 0, 0, 0, 0, 0,
@@ -1535,10 +1535,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, PB1MD_1, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
 
-	{ PINMUX_CFG_REG("PBIOR1", 0xfffe3830, 16, 1) {
+	{ PINMUX_CFG_REG("PBIOR1", 0xfffe3830, 16, 1, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0,
@@ -1548,10 +1548,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PB19_IN, PB19_OUT,
 		PB18_IN, PB18_OUT,
 		PB17_IN, PB17_OUT,
-		PB16_IN, PB16_OUT }
+		PB16_IN, PB16_OUT ))
 	},
 
-	{ PINMUX_CFG_REG("PBIOR0", 0xfffe3832, 16, 1) {
+	{ PINMUX_CFG_REG("PBIOR0", 0xfffe3832, 16, 1, GROUP(
 		PB15_IN, PB15_OUT,
 		PB14_IN, PB14_OUT,
 		PB13_IN, PB13_OUT,
@@ -1567,10 +1567,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PB3_IN, PB3_OUT,
 		PB2_IN, PB2_OUT,
 		PB1_IN, PB1_OUT,
-		0, 0 }
+		0, 0 ))
 	},
 
-	{ PINMUX_CFG_REG("PCCR2", 0xfffe384a, 16, 4) {
+	{ PINMUX_CFG_REG("PCCR2", 0xfffe384a, 16, 4, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PC10MD_0, PC10MD_1, 0, 0, 0, 0, 0, 0,
@@ -1578,9 +1578,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PC9MD_0, PC9MD_1, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PC8MD_00, PC8MD_01, PC8MD_10, PC8MD_11, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PCCR1", 0xfffe384c, 16, 4) {
+	{ PINMUX_CFG_REG("PCCR1", 0xfffe384c, 16, 4, GROUP(
 		PC7MD_00, PC7MD_01, PC7MD_10, PC7MD_11, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PC6MD_00, PC6MD_01, PC6MD_10, PC6MD_11, 0, 0, 0, 0,
@@ -1588,9 +1588,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PC5MD_00, PC5MD_01, PC5MD_10, PC5MD_11, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PC4MD_0, PC4MD_1, 0, 0, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PCCR0", 0xfffe384e, 16, 4) {
+	{ PINMUX_CFG_REG("PCCR0", 0xfffe384e, 16, 4, GROUP(
 		PC3MD_0, PC3MD_1, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PC2MD_0, PC2MD_1, 0, 0, 0, 0, 0, 0,
@@ -1598,10 +1598,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PC1MD_0, PC1MD_1, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PC0MD_0, PC0MD_1, 0, 0, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
 
-	{ PINMUX_CFG_REG("PCIOR0", 0xfffe3852, 16, 1) {
+	{ PINMUX_CFG_REG("PCIOR0", 0xfffe3852, 16, 1, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		PC10_IN, PC10_OUT,
 		PC9_IN, PC9_OUT,
@@ -1614,10 +1614,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PC2_IN, PC2_OUT,
 		PC1_IN, PC1_OUT,
 		PC0_IN, PC0_OUT
-	 }
+	 ))
 	},
 
-	{ PINMUX_CFG_REG("PDCR3", 0xfffe3868, 16, 4) {
+	{ PINMUX_CFG_REG("PDCR3", 0xfffe3868, 16, 4, GROUP(
 		0, PD15MD_01, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, PD14MD_01, 0, 0, 0, 0, 0, 0,
@@ -1625,9 +1625,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, PD13MD_01, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, PD12MD_01, 0, 0, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PDCR2", 0xfffe386a, 16, 4) {
+	{ PINMUX_CFG_REG("PDCR2", 0xfffe386a, 16, 4, GROUP(
 		0, PD11MD_01, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, PD10MD_01, 0, 0, 0, 0, 0, 0,
@@ -1635,9 +1635,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, PD9MD_01, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, PD8MD_01, 0, 0, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PDCR1", 0xfffe386c, 16, 4) {
+	{ PINMUX_CFG_REG("PDCR1", 0xfffe386c, 16, 4, GROUP(
 		0, PD7MD_01, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, PD6MD_01, 0, 0, 0, 0, 0, 0,
@@ -1645,9 +1645,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, PD5MD_01, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, PD4MD_01, 0, 0, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PDCR0", 0xfffe386e, 16, 4) {
+	{ PINMUX_CFG_REG("PDCR0", 0xfffe386e, 16, 4, GROUP(
 		0, PD3MD_01, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, PD2MD_01, 0, 0, 0, 0, 0, 0,
@@ -1655,10 +1655,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, PD1MD_01, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, PD0MD_01, 0, 0, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
 
-	{ PINMUX_CFG_REG("PDIOR0", 0xfffe3872, 16, 1) {
+	{ PINMUX_CFG_REG("PDIOR0", 0xfffe3872, 16, 1, GROUP(
 		PD15_IN, PD15_OUT,
 		PD14_IN, PD14_OUT,
 		PD13_IN, PD13_OUT,
@@ -1674,10 +1674,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PD3_IN, PD3_OUT,
 		PD2_IN, PD2_OUT,
 		PD1_IN, PD1_OUT,
-		PD0_IN, PD0_OUT }
+		PD0_IN, PD0_OUT ))
 	},
 
-	{ PINMUX_CFG_REG("PECR1", 0xfffe388c, 16, 4) {
+	{ PINMUX_CFG_REG("PECR1", 0xfffe388c, 16, 4, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -1685,10 +1685,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PE5MD_00, PE5MD_01, 0, PE5MD_11, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PE4MD_00, PE4MD_01, 0, PE4MD_11, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
 
-	{ PINMUX_CFG_REG("PECR0", 0xfffe388e, 16, 4) {
+	{ PINMUX_CFG_REG("PECR0", 0xfffe388e, 16, 4, GROUP(
 		PE3MD_00, PE3MD_01, 0, PE3MD_11, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PE2MD_00, PE2MD_01, 0, PE2MD_11, 0, 0, 0, 0,
@@ -1697,10 +1697,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PE1MD_100, PE1MD_101, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PE0MD_00, PE0MD_01, PE0MD_10, PE0MD_11, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
 
-	{ PINMUX_CFG_REG("PEIOR0", 0xfffe3892, 16, 1) {
+	{ PINMUX_CFG_REG("PEIOR0", 0xfffe3892, 16, 1, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0,
@@ -1709,19 +1709,19 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PE3_IN, PE3_OUT,
 		PE2_IN, PE2_OUT,
 		PE1_IN, PE1_OUT,
-		PE0_IN, PE0_OUT }
+		PE0_IN, PE0_OUT ))
 	},
 
-	{ PINMUX_CFG_REG("PFCR3", 0xfffe38a8, 16, 4) {
+	{ PINMUX_CFG_REG("PFCR3", 0xfffe38a8, 16, 4, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		PF12MD_000, PF12MD_001, 0, PF12MD_011,
 		PF12MD_100, PF12MD_101, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
 
-	{ PINMUX_CFG_REG("PFCR2", 0xfffe38aa, 16, 4) {
+	{ PINMUX_CFG_REG("PFCR2", 0xfffe38aa, 16, 4, GROUP(
 		PF11MD_000, PF11MD_001, PF11MD_010, PF11MD_011,
 		PF11MD_100, PF11MD_101, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -1732,10 +1732,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PF9MD_100, PF9MD_101, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PF8MD_00, PF8MD_01, PF8MD_10, PF8MD_11, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
 
-	{ PINMUX_CFG_REG("PFCR1", 0xfffe38ac, 16, 4) {
+	{ PINMUX_CFG_REG("PFCR1", 0xfffe38ac, 16, 4, GROUP(
 		PF7MD_000, PF7MD_001, PF7MD_010, PF7MD_011,
 		PF7MD_100, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -1747,10 +1747,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PF4MD_000, PF4MD_001, PF4MD_010, PF4MD_011,
 		PF4MD_100, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
 
-	{ PINMUX_CFG_REG("PFCR0", 0xfffe38ae, 16, 4) {
+	{ PINMUX_CFG_REG("PFCR0", 0xfffe38ae, 16, 4, GROUP(
 		PF3MD_000, PF3MD_001, PF3MD_010, PF3MD_011,
 		PF3MD_100, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -1762,10 +1762,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PF0MD_000, PF0MD_001, PF0MD_010, PF0MD_011,
 		PF0MD_100, PF0MD_101, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
 
-	{ PINMUX_CFG_REG("PFIOR0", 0xfffe38b2, 16, 1) {
+	{ PINMUX_CFG_REG("PFIOR0", 0xfffe38b2, 16, 1, GROUP(
 		0, 0, 0, 0, 0, 0,
 		PF12_IN, PF12_OUT,
 		PF11_IN, PF11_OUT,
@@ -1779,10 +1779,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PF3_IN, PF3_OUT,
 		PF2_IN, PF2_OUT,
 		PF1_IN, PF1_OUT,
-		PF0_IN, PF0_OUT }
+		PF0_IN, PF0_OUT ))
 	},
 
-	{ PINMUX_CFG_REG("PGCR7", 0xfffe38c0, 16, 4) {
+	{ PINMUX_CFG_REG("PGCR7", 0xfffe38c0, 16, 4, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -1791,10 +1791,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PG0MD_000, PG0MD_001, PG0MD_010, PG0MD_011,
 		PG0MD_100, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
 
-	{ PINMUX_CFG_REG("PGCR6", 0xfffe38c2, 16, 4) {
+	{ PINMUX_CFG_REG("PGCR6", 0xfffe38c2, 16, 4, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -1802,10 +1802,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PG24MD_00, PG24MD_01, PG24MD_10, PG24MD_11, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
 
-	{ PINMUX_CFG_REG("PGCR5", 0xfffe38c4, 16, 4) {
+	{ PINMUX_CFG_REG("PGCR5", 0xfffe38c4, 16, 4, GROUP(
 		PG23MD_00, PG23MD_01, PG23MD_10, PG23MD_11, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PG22MD_00, PG22MD_01, PG22MD_10, PG22MD_11, 0, 0, 0, 0,
@@ -1814,10 +1814,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PG20MD_000, PG20MD_001, PG20MD_010, PG20MD_011,
 		PG20MD_100, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
 
-	{ PINMUX_CFG_REG("PGCR4", 0xfffe38c6, 16, 4) {
+	{ PINMUX_CFG_REG("PGCR4", 0xfffe38c6, 16, 4, GROUP(
 		PG19MD_000, PG19MD_001, PG19MD_010, PG19MD_011,
 		PG19MD_100, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -1829,10 +1829,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PG16MD_000, PG16MD_001, PG16MD_010, PG16MD_011,
 		PG16MD_100, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
 
-	{ PINMUX_CFG_REG("PGCR3", 0xfffe38c8, 16, 4) {
+	{ PINMUX_CFG_REG("PGCR3", 0xfffe38c8, 16, 4, GROUP(
 		PG15MD_000, PG15MD_001, PG15MD_010, PG15MD_011,
 		PG15MD_100, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -1844,9 +1844,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PG12MD_000, PG12MD_001, PG12MD_010, 0,
 		PG12MD_100, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PGCR2", 0xfffe38ca, 16, 4) {
+	{ PINMUX_CFG_REG("PGCR2", 0xfffe38ca, 16, 4, GROUP(
 		PG11MD_000, PG11MD_001, PG11MD_010, PG11MD_011,
 		PG11MD_100, PG11MD_101, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -1858,10 +1858,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PG8MD_000, PG8MD_001, PG8MD_010, PG8MD_011,
 		PG8MD_100, PG8MD_101, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
 
-	{ PINMUX_CFG_REG("PGCR1", 0xfffe38cc, 16, 4) {
+	{ PINMUX_CFG_REG("PGCR1", 0xfffe38cc, 16, 4, GROUP(
 		PG7MD_00, PG7MD_01, PG7MD_10, PG7MD_11, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PG6MD_00, PG6MD_01, PG6MD_10, PG6MD_11, 0, 0, 0, 0,
@@ -1869,9 +1869,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PG5MD_00, PG5MD_01, PG5MD_10, PG5MD_11, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PG4MD_00, PG4MD_01, PG4MD_10, PG4MD_11, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PGCR0", 0xfffe38ce, 16, 4) {
+	{ PINMUX_CFG_REG("PGCR0", 0xfffe38ce, 16, 4, GROUP(
 		PG3MD_00, PG3MD_01, PG3MD_10, PG3MD_11, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PG2MD_00, PG2MD_01, PG2MD_10, PG2MD_11, 0, 0, 0, 0,
@@ -1879,9 +1879,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PG1MD_00, PG1MD_01, PG1MD_10, PG1MD_11, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PGIOR1", 0xfffe38d0, 16, 1) {
+	{ PINMUX_CFG_REG("PGIOR1", 0xfffe38d0, 16, 1, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0,
 		PG24_IN, PG24_OUT,
@@ -1892,10 +1892,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PG19_IN, PG19_OUT,
 		PG18_IN, PG18_OUT,
 		PG17_IN, PG17_OUT,
-		PG16_IN, PG16_OUT }
+		PG16_IN, PG16_OUT ))
 	},
 
-	{ PINMUX_CFG_REG("PGIOR0", 0xfffe38d2, 16, 1) {
+	{ PINMUX_CFG_REG("PGIOR0", 0xfffe38d2, 16, 1, GROUP(
 		PG15_IN, PG15_OUT,
 		PG14_IN, PG14_OUT,
 		PG13_IN, PG13_OUT,
@@ -1912,10 +1912,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PG2_IN, PG2_OUT,
 		PG1_IN, PG1_OUT,
 		PG0_IN, PG0_OUT
-	 }
+	 ))
 	},
 
-	{ PINMUX_CFG_REG("PHCR1", 0xfffe38ec, 16, 4) {
+	{ PINMUX_CFG_REG("PHCR1", 0xfffe38ec, 16, 4, GROUP(
 		PH7MD_0, PH7MD_1, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PH6MD_0, PH6MD_1, 0, 0, 0, 0, 0, 0,
@@ -1923,10 +1923,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PH5MD_0, PH5MD_1, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PH4MD_0, PH4MD_1, 0, 0, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
 
-	{ PINMUX_CFG_REG("PHCR0", 0xfffe38ee, 16, 4) {
+	{ PINMUX_CFG_REG("PHCR0", 0xfffe38ee, 16, 4, GROUP(
 		PH3MD_0, PH3MD_1, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PH2MD_0, PH2MD_1, 0, 0, 0, 0, 0, 0,
@@ -1934,10 +1934,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PH1MD_0, PH1MD_1, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PH0MD_0, PH0MD_1, 0, 0, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
 
-	{ PINMUX_CFG_REG("PJCR2", 0xfffe390a, 16, 4) {
+	{ PINMUX_CFG_REG("PJCR2", 0xfffe390a, 16, 4, GROUP(
 		PJ11MD_00, PJ11MD_01, PJ11MD_10, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PJ10MD_00, PJ10MD_01, PJ10MD_10, 0, 0, 0, 0, 0,
@@ -1945,9 +1945,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PJ9MD_00, PJ9MD_01, PJ9MD_10, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PJ8MD_00, PJ8MD_01, PJ8MD_10, 0, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PJCR1", 0xfffe390c, 16, 4) {
+	{ PINMUX_CFG_REG("PJCR1", 0xfffe390c, 16, 4, GROUP(
 		PJ7MD_00, PJ7MD_01, PJ7MD_10, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PJ6MD_00, PJ6MD_01, PJ6MD_10, 0, 0, 0, 0, 0,
@@ -1955,9 +1955,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PJ5MD_00, PJ5MD_01, PJ5MD_10, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PJ4MD_00, PJ4MD_01, PJ4MD_10, 0, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PJCR0", 0xfffe390e, 16, 4) {
+	{ PINMUX_CFG_REG("PJCR0", 0xfffe390e, 16, 4, GROUP(
 		PJ3MD_00, PJ3MD_01, PJ3MD_10, PJ3MD_11, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PJ2MD_000, PJ2MD_001, PJ2MD_010, PJ2MD_011,
@@ -1968,9 +1968,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PJ0MD_000, PJ0MD_001, PJ0MD_010, PJ0MD_011,
 		PJ0MD_100, PJ0MD_101, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0, }
+		0, 0, 0, 0, 0, 0, 0, 0, ))
 	},
-	{ PINMUX_CFG_REG("PJIOR0", 0xfffe3912, 16, 1) {
+	{ PINMUX_CFG_REG("PJIOR0", 0xfffe3912, 16, 1, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PJ11_IN, PJ11_OUT,
 		PJ10_IN, PJ10_OUT,
@@ -1983,10 +1983,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PJ3_IN, PJ3_OUT,
 		PJ2_IN, PJ2_OUT,
 		PJ1_IN, PJ1_OUT,
-		PJ0_IN, PJ0_OUT }
+		PJ0_IN, PJ0_OUT ))
 	},
 
-	{ PINMUX_CFG_REG("PKCR2", 0xfffe392a, 16, 4) {
+	{ PINMUX_CFG_REG("PKCR2", 0xfffe392a, 16, 4, GROUP(
 		PK11MD_00, PK11MD_01, PK11MD_10, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PK10MD_00, PK10MD_01, PK10MD_10, 0, 0, 0, 0, 0,
@@ -1994,10 +1994,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PK9MD_00, PK9MD_01, PK9MD_10, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PK8MD_00, PK8MD_01, PK8MD_10, 0, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
 
-	{ PINMUX_CFG_REG("PKCR1", 0xfffe392c, 16, 4) {
+	{ PINMUX_CFG_REG("PKCR1", 0xfffe392c, 16, 4, GROUP(
 		PK7MD_00, PK7MD_01, PK7MD_10, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PK6MD_00, PK6MD_01, PK6MD_10, 0,  0, 0, 0, 0,
@@ -2005,9 +2005,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PK5MD_00, PK5MD_01, PK5MD_10, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PK4MD_00, PK4MD_01, PK4MD_10, 0, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PKCR0", 0xfffe392e, 16, 4) {
+	{ PINMUX_CFG_REG("PKCR0", 0xfffe392e, 16, 4, GROUP(
 		PK3MD_00, PK3MD_01, PK3MD_10, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PK2MD_00, PK2MD_01, PK2MD_10, 0, 0, 0, 0, 0,
@@ -2015,10 +2015,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PK1MD_00, PK1MD_01, PK1MD_10, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PK0MD_00, PK0MD_01, PK0MD_10, 0, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
 
-	{ PINMUX_CFG_REG("PKIOR0", 0xfffe3932, 16, 1) {
+	{ PINMUX_CFG_REG("PKIOR0", 0xfffe3932, 16, 1, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PJ11_IN, PJ11_OUT,
 		PJ10_IN, PJ10_OUT,
@@ -2031,85 +2031,85 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PJ3_IN, PJ3_OUT,
 		PJ2_IN, PJ2_OUT,
 		PJ1_IN, PJ1_OUT,
-		PJ0_IN, PJ0_OUT }
+		PJ0_IN, PJ0_OUT ))
 	},
 	{}
 };
 
 static const struct pinmux_data_reg pinmux_data_regs[] = {
-	{ PINMUX_DATA_REG("PADR1", 0xfffe3814, 16) {
+	{ PINMUX_DATA_REG("PADR1", 0xfffe3814, 16, GROUP(
 		0, 0, 0, 0, 0, 0, 0, PA3_DATA,
-		0, 0, 0, 0, 0, 0, 0, PA2_DATA }
+		0, 0, 0, 0, 0, 0, 0, PA2_DATA ))
 	},
 
-	{ PINMUX_DATA_REG("PADR0", 0xfffe3816, 16) {
+	{ PINMUX_DATA_REG("PADR0", 0xfffe3816, 16, GROUP(
 		0, 0, 0, 0, 0, 0, 0, PA1_DATA,
-		0, 0, 0, 0, 0, 0, 0, PA0_DATA }
+		0, 0, 0, 0, 0, 0, 0, PA0_DATA ))
 	},
 
-	{ PINMUX_DATA_REG("PBDR1", 0xfffe3834, 16) {
+	{ PINMUX_DATA_REG("PBDR1", 0xfffe3834, 16, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, PB22_DATA, PB21_DATA, PB20_DATA,
-		PB19_DATA, PB18_DATA, PB17_DATA, PB16_DATA }
+		PB19_DATA, PB18_DATA, PB17_DATA, PB16_DATA ))
 	},
 
-	{ PINMUX_DATA_REG("PBDR0", 0xfffe3836, 16) {
+	{ PINMUX_DATA_REG("PBDR0", 0xfffe3836, 16, GROUP(
 		PB15_DATA, PB14_DATA, PB13_DATA, PB12_DATA,
 		PB11_DATA, PB10_DATA, PB9_DATA, PB8_DATA,
 		PB7_DATA, PB6_DATA, PB5_DATA, PB4_DATA,
-		PB3_DATA, PB2_DATA, PB1_DATA, 0 }
+		PB3_DATA, PB2_DATA, PB1_DATA, 0 ))
 	},
 
-	{ PINMUX_DATA_REG("PCDR0", 0xfffe3856, 16) {
+	{ PINMUX_DATA_REG("PCDR0", 0xfffe3856, 16, GROUP(
 		0, 0, 0, 0,
 		0, PC10_DATA, PC9_DATA, PC8_DATA,
 		PC7_DATA, PC6_DATA, PC5_DATA, PC4_DATA,
-		PC3_DATA, PC2_DATA, PC1_DATA, PC0_DATA }
+		PC3_DATA, PC2_DATA, PC1_DATA, PC0_DATA ))
 	},
 
-	{ PINMUX_DATA_REG("PDDR0", 0xfffe3876, 16) {
+	{ PINMUX_DATA_REG("PDDR0", 0xfffe3876, 16, GROUP(
 		PD15_DATA, PD14_DATA, PD13_DATA, PD12_DATA,
 		PD11_DATA, PD10_DATA, PD9_DATA, PD8_DATA,
 		PD7_DATA, PD6_DATA, PD5_DATA, PD4_DATA,
-		PD3_DATA, PD2_DATA, PD1_DATA, PD0_DATA }
+		PD3_DATA, PD2_DATA, PD1_DATA, PD0_DATA ))
 	},
 
-	{ PINMUX_DATA_REG("PEDR0", 0xfffe3896, 16) {
+	{ PINMUX_DATA_REG("PEDR0", 0xfffe3896, 16, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0, PE5_DATA, PE4_DATA,
-		PE3_DATA, PE2_DATA, PE1_DATA, PE0_DATA }
+		PE3_DATA, PE2_DATA, PE1_DATA, PE0_DATA ))
 	},
 
-	{ PINMUX_DATA_REG("PFDR0", 0xfffe38b6, 16) {
+	{ PINMUX_DATA_REG("PFDR0", 0xfffe38b6, 16, GROUP(
 		0, 0, 0, PF12_DATA,
 		PF11_DATA, PF10_DATA, PF9_DATA, PF8_DATA,
 		PF7_DATA, PF6_DATA, PF5_DATA, PF4_DATA,
-		PF3_DATA, PF2_DATA, PF1_DATA, PF0_DATA }
+		PF3_DATA, PF2_DATA, PF1_DATA, PF0_DATA ))
 	},
 
-	{ PINMUX_DATA_REG("PGDR1", 0xfffe38d4, 16) {
+	{ PINMUX_DATA_REG("PGDR1", 0xfffe38d4, 16, GROUP(
 		0, 0, 0, 0, 0, 0, 0, PG24_DATA,
 		PG23_DATA, PG22_DATA, PG21_DATA, PG20_DATA,
-		PG19_DATA, PG18_DATA, PG17_DATA, PG16_DATA }
+		PG19_DATA, PG18_DATA, PG17_DATA, PG16_DATA ))
 	},
 
-	{ PINMUX_DATA_REG("PGDR0", 0xfffe38d6, 16) {
+	{ PINMUX_DATA_REG("PGDR0", 0xfffe38d6, 16, GROUP(
 		PG15_DATA, PG14_DATA, PG13_DATA, PG12_DATA,
 		PG11_DATA, PG10_DATA, PG9_DATA, PG8_DATA,
 		PG7_DATA, PG6_DATA, PG5_DATA, PG4_DATA,
-		PG3_DATA, PG2_DATA, PG1_DATA, PG0_DATA }
+		PG3_DATA, PG2_DATA, PG1_DATA, PG0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PJDR0", 0xfffe3916, 16) {
+	{ PINMUX_DATA_REG("PJDR0", 0xfffe3916, 16, GROUP(
 		0, 0, 0, PJ12_DATA,
 		PJ11_DATA, PJ10_DATA, PJ9_DATA, PJ8_DATA,
 		PJ7_DATA, PJ6_DATA, PJ5_DATA, PJ4_DATA,
-		PJ3_DATA, PJ2_DATA, PJ1_DATA, PJ0_DATA }
+		PJ3_DATA, PJ2_DATA, PJ1_DATA, PJ0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PKDR0", 0xfffe3936, 16) {
+	{ PINMUX_DATA_REG("PKDR0", 0xfffe3936, 16, GROUP(
 		0, 0, 0, PK12_DATA,
 		PK11_DATA, PK10_DATA, PK9_DATA, PK8_DATA,
 		PK7_DATA, PK6_DATA, PK5_DATA, PK4_DATA,
-		PK3_DATA, PK2_DATA, PK1_DATA, PK0_DATA }
+		PK3_DATA, PK2_DATA, PK1_DATA, PK0_DATA ))
 	},
 	{ }
 };
diff --git a/drivers/pinctrl/sh-pfc/pfc-sh7269.c b/drivers/pinctrl/sh-pfc/pfc-sh7269.c
index a95997a389a4..6cbb18ef77dc 100644
--- a/drivers/pinctrl/sh-pfc/pfc-sh7269.c
+++ b/drivers/pinctrl/sh-pfc/pfc-sh7269.c
@@ -1951,13 +1951,13 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 	/* where Field_Width is 1 for single mode registers or 4 for upto 16
 	   mode registers and modes are described in assending order [0..16] */
 
-	{ PINMUX_CFG_REG("PAIOR0", 0xfffe3812, 16, 1) {
+	{ PINMUX_CFG_REG("PAIOR0", 0xfffe3812, 16, 1, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, PA1_IN, PA1_OUT,
 		0, 0, 0, 0, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, PA0_IN, PA0_OUT }
+		0, 0, 0, 0, 0, 0, PA0_IN, PA0_OUT ))
 	},
-	{ PINMUX_CFG_REG("PBCR5", 0xfffe3824, 16, 4) {
+	{ PINMUX_CFG_REG("PBCR5", 0xfffe3824, 16, 4, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
 		PB22MD_000, PB22MD_001, PB22MD_010, PB22MD_011,
@@ -1969,9 +1969,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 
 		PB20MD_000, PB20MD_001, PB20MD_010, PB20MD_011,
 		PB20MD_100, PB20MD_101, PB20MD_110, PB20MD_111,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PBCR4", 0xfffe3826, 16, 4) {
+	{ PINMUX_CFG_REG("PBCR4", 0xfffe3826, 16, 4, GROUP(
 		PB19MD_000, PB19MD_001, PB19MD_010, PB19MD_011,
 		PB19MD_100, PB19MD_101, PB19MD_110, PB19MD_111,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -1986,9 +1986,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 
 		PB16MD_000, PB16MD_001, PB16MD_010, PB16MD_011,
 		PB16MD_100, PB16MD_101, PB16MD_110, PB16MD_111,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PBCR3", 0xfffe3828, 16, 4) {
+	{ PINMUX_CFG_REG("PBCR3", 0xfffe3828, 16, 4, GROUP(
 		PB15MD_000, PB15MD_001, PB15MD_010, PB15MD_011,
 		PB15MD_100, PB15MD_101, PB15MD_110, PB15MD_111,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -2002,9 +2002,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0,
 
 		PB12MD_00, PB12MD_01, PB12MD_10, PB12MD_11, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PBCR2", 0xfffe382a, 16, 4) {
+	{ PINMUX_CFG_REG("PBCR2", 0xfffe382a, 16, 4, GROUP(
 		PB11MD_00, PB11MD_01, PB11MD_10, PB11MD_11, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 
@@ -2015,9 +2015,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0,
 
 		PB8MD_00, PB8MD_01, PB8MD_10, PB8MD_11, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PBCR1", 0xfffe382c, 16, 4) {
+	{ PINMUX_CFG_REG("PBCR1", 0xfffe382c, 16, 4, GROUP(
 		PB7MD_00, PB7MD_01, PB7MD_10, PB7MD_11, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 
@@ -2028,9 +2028,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0,
 
 		PB4MD_00, PB4MD_01, PB4MD_10, PB4MD_11, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PBCR0", 0xfffe382e, 16, 4) {
+	{ PINMUX_CFG_REG("PBCR0", 0xfffe382e, 16, 4, GROUP(
 		PB3MD_00, PB3MD_01, PB3MD_10, PB3MD_11, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 
@@ -2040,10 +2040,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PB1MD_00, PB1MD_01, PB1MD_10, PB1MD_11, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 
-		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
 
-	{ PINMUX_CFG_REG("PBIOR1", 0xfffe3830, 16, 1) {
+	{ PINMUX_CFG_REG("PBIOR1", 0xfffe3830, 16, 1, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0,
@@ -2053,9 +2053,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PB19_IN, PB19_OUT,
 		PB18_IN, PB18_OUT,
 		PB17_IN, PB17_OUT,
-		PB16_IN, PB16_OUT }
+		PB16_IN, PB16_OUT ))
 	},
-	{ PINMUX_CFG_REG("PBIOR0", 0xfffe3832, 16, 1) {
+	{ PINMUX_CFG_REG("PBIOR0", 0xfffe3832, 16, 1, GROUP(
 		PB15_IN, PB15_OUT,
 		PB14_IN, PB14_OUT,
 		PB13_IN, PB13_OUT,
@@ -2071,10 +2071,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PB3_IN, PB3_OUT,
 		PB2_IN, PB2_OUT,
 		PB1_IN, PB1_OUT,
-		0, 0 }
+		0, 0 ))
 	},
 
-	{ PINMUX_CFG_REG("PCCR2", 0xfffe384a, 16, 4) {
+	{ PINMUX_CFG_REG("PCCR2", 0xfffe384a, 16, 4, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -2083,9 +2083,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 
 		PC8MD_000, PC8MD_001, PC8MD_010, PC8MD_011,
 		PC8MD_100, PC8MD_101, PC8MD_110, PC8MD_111,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PCCR1", 0xfffe384c, 16, 4) {
+	{ PINMUX_CFG_REG("PCCR1", 0xfffe384c, 16, 4, GROUP(
 		PC7MD_000, PC7MD_001, PC7MD_010, PC7MD_011,
 		PC7MD_100, PC7MD_101, PC7MD_110, PC7MD_111,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -2099,9 +2099,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0,
 
 		PC4MD_00, PC4MD_01, PC4MD_10, PC4MD_11, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PCCR0", 0xfffe384e, 16, 4) {
+	{ PINMUX_CFG_REG("PCCR0", 0xfffe384e, 16, 4, GROUP(
 		PC3MD_00, PC3MD_01, PC3MD_10, PC3MD_11, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 
@@ -2112,10 +2112,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0,
 
 		PC0MD_0, PC0MD_1, 0, 0, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
 
-	{ PINMUX_CFG_REG("PCIOR0", 0xfffe3852, 16, 1) {
+	{ PINMUX_CFG_REG("PCIOR0", 0xfffe3852, 16, 1, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		PC8_IN, PC8_OUT,
 		PC7_IN, PC7_OUT,
@@ -2125,10 +2125,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PC3_IN, PC3_OUT,
 		PC2_IN, PC2_OUT,
 		PC1_IN, PC1_OUT,
-		PC0_IN, PC0_OUT }
+		PC0_IN, PC0_OUT ))
 	},
 
-	{ PINMUX_CFG_REG("PDCR3", 0xfffe3868, 16, 4) {
+	{ PINMUX_CFG_REG("PDCR3", 0xfffe3868, 16, 4, GROUP(
 		PD15MD_00, PD15MD_01, PD15MD_10, PD15MD_11, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 
@@ -2139,9 +2139,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0,
 
 		PD12MD_00, PD12MD_01, PD12MD_10, PD12MD_11, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PDCR2", 0xfffe386a, 16, 4) {
+	{ PINMUX_CFG_REG("PDCR2", 0xfffe386a, 16, 4, GROUP(
 		PD11MD_00, PD11MD_01, PD11MD_10, PD11MD_11, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 
@@ -2152,9 +2152,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0,
 
 		PD8MD_00, PD8MD_01, PD8MD_10, PD8MD_11, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PDCR1", 0xfffe386c, 16, 4) {
+	{ PINMUX_CFG_REG("PDCR1", 0xfffe386c, 16, 4, GROUP(
 		PD7MD_00, PD7MD_01, PD7MD_10, PD7MD_11, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 
@@ -2165,9 +2165,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0,
 
 		PD4MD_00, PD4MD_01, PD4MD_10, PD4MD_11, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PDCR0", 0xfffe386e, 16, 4) {
+	{ PINMUX_CFG_REG("PDCR0", 0xfffe386e, 16, 4, GROUP(
 		PD3MD_00, PD3MD_01, PD3MD_10, PD3MD_11, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 
@@ -2178,10 +2178,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0,
 
 		PD0MD_00, PD0MD_01, PD0MD_10, PD0MD_11, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
 
-	{ PINMUX_CFG_REG("PDIOR0", 0xfffe3872, 16, 1) {
+	{ PINMUX_CFG_REG("PDIOR0", 0xfffe3872, 16, 1, GROUP(
 		PD15_IN, PD15_OUT,
 		PD14_IN, PD14_OUT,
 		PD13_IN, PD13_OUT,
@@ -2197,10 +2197,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PD3_IN, PD3_OUT,
 		PD2_IN, PD2_OUT,
 		PD1_IN, PD1_OUT,
-		PD0_IN, PD0_OUT }
+		PD0_IN, PD0_OUT ))
 	},
 
-	{ PINMUX_CFG_REG("PECR1", 0xfffe388c, 16, 4) {
+	{ PINMUX_CFG_REG("PECR1", 0xfffe388c, 16, 4, GROUP(
 		PE7MD_00, PE7MD_01, PE7MD_10, PE7MD_11, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 
@@ -2211,9 +2211,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0,
 
 		PE4MD_00, PE4MD_01, PE4MD_10, PE4MD_11, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PECR0", 0xfffe388e, 16, 4) {
+	{ PINMUX_CFG_REG("PECR0", 0xfffe388e, 16, 4, GROUP(
 		PE3MD_000, PE3MD_001, PE3MD_010, PE3MD_011,
 		PE3MD_100, PE3MD_101, PE3MD_110, PE3MD_111,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -2227,9 +2227,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0,
 
 		PE0MD_00, PE0MD_01, PE0MD_10, PE0MD_11, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PEIOR0", 0xfffe3892, 16, 1) {
+	{ PINMUX_CFG_REG("PEIOR0", 0xfffe3892, 16, 1, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PE7_IN, PE7_OUT,
@@ -2239,10 +2239,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PE3_IN, PE3_OUT,
 		PE2_IN, PE2_OUT,
 		PE1_IN, PE1_OUT,
-		PE0_IN, PE0_OUT }
+		PE0_IN, PE0_OUT ))
 	},
 
-	{ PINMUX_CFG_REG("PFCR6", 0xfffe38a2, 16, 4) {
+	{ PINMUX_CFG_REG("PFCR6", 0xfffe38a2, 16, 4, GROUP(
 		PF23MD_000, PF23MD_001, PF23MD_010, PF23MD_011,
 		PF23MD_100, PF23MD_101, PF23MD_110, PF23MD_111,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -2257,9 +2257,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 
 		PF20MD_000, PF20MD_001, PF20MD_010, PF20MD_011,
 		PF20MD_100, PF20MD_101, PF20MD_110, PF20MD_111,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PFCR5", 0xfffe38a4, 16, 4) {
+	{ PINMUX_CFG_REG("PFCR5", 0xfffe38a4, 16, 4, GROUP(
 		PF19MD_000, PF19MD_001, PF19MD_010, PF19MD_011,
 		PF19MD_100, PF19MD_101, PF19MD_110, PF19MD_111,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -2274,9 +2274,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 
 		PF16MD_000, PF16MD_001, PF16MD_010, PF16MD_011,
 		PF16MD_100, PF16MD_101, PF16MD_110, PF16MD_111,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PFCR4", 0xfffe38a6, 16, 4) {
+	{ PINMUX_CFG_REG("PFCR4", 0xfffe38a6, 16, 4, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -2285,9 +2285,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 
 		PF15MD_000, PF15MD_001, PF15MD_010, PF15MD_011,
 		PF15MD_100, PF15MD_101, PF15MD_110, PF15MD_111,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PFCR3", 0xfffe38a8, 16, 4) {
+	{ PINMUX_CFG_REG("PFCR3", 0xfffe38a8, 16, 4, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 
 		PF14MD_000, PF14MD_001, PF14MD_010, PF14MD_011,
@@ -2300,9 +2300,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 
 		PF12MD_000, PF12MD_001, PF12MD_010, PF12MD_011,
 		PF12MD_100, PF12MD_101, PF12MD_110, PF12MD_111,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PFCR2", 0xfffe38aa, 16, 4) {
+	{ PINMUX_CFG_REG("PFCR2", 0xfffe38aa, 16, 4, GROUP(
 		PF11MD_000, PF11MD_001, PF11MD_010, PF11MD_011,
 		PF11MD_100, PF11MD_101, PF11MD_110, PF11MD_111,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -2317,9 +2317,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 
 		PF8MD_000, PF8MD_001, PF8MD_010, PF8MD_011,
 		PF8MD_100, PF8MD_101, PF8MD_110, PF8MD_111,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PFCR1", 0xfffe38ac, 16, 4) {
+	{ PINMUX_CFG_REG("PFCR1", 0xfffe38ac, 16, 4, GROUP(
 		PF7MD_000, PF7MD_001, PF7MD_010, PF7MD_011,
 		PF7MD_100, PF7MD_101, PF7MD_110, PF7MD_111,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -2334,9 +2334,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 
 		PF4MD_000, PF4MD_001, PF4MD_010, PF4MD_011,
 		PF4MD_100, PF4MD_101, PF4MD_110, PF4MD_111,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PFCR0", 0xfffe38ae, 16, 4) {
+	{ PINMUX_CFG_REG("PFCR0", 0xfffe38ae, 16, 4, GROUP(
 		PF3MD_000, PF3MD_001, PF3MD_010, PF3MD_011,
 		PF3MD_100, PF3MD_101, PF3MD_110, PF3MD_111,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -2351,10 +2351,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 
 		PF0MD_000, PF0MD_001, PF0MD_010, PF0MD_011,
 		PF0MD_100, PF0MD_101, PF0MD_110, PF0MD_111,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
 
-	{ PINMUX_CFG_REG("PFIOR1", 0xfffe38b0, 16, 1) {
+	{ PINMUX_CFG_REG("PFIOR1", 0xfffe38b0, 16, 1, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PF23_IN, PF23_OUT,
@@ -2364,9 +2364,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PF19_IN, PF19_OUT,
 		PF18_IN, PF18_OUT,
 		PF17_IN, PF17_OUT,
-		PF16_IN, PF16_OUT }
+		PF16_IN, PF16_OUT ))
 	},
-	{ PINMUX_CFG_REG("PFIOR0", 0xfffe38b2, 16, 1) {
+	{ PINMUX_CFG_REG("PFIOR0", 0xfffe38b2, 16, 1, GROUP(
 		PF15_IN, PF15_OUT,
 		PF14_IN, PF14_OUT,
 		PF13_IN, PF13_OUT,
@@ -2382,10 +2382,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PF3_IN, PF3_OUT,
 		PF2_IN, PF2_OUT,
 		PF1_IN, PF1_OUT,
-		PF0_IN, PF0_OUT }
+		PF0_IN, PF0_OUT ))
 	},
 
-	{ PINMUX_CFG_REG("PGCR6", 0xfffe38c2, 16, 4) {
+	{ PINMUX_CFG_REG("PGCR6", 0xfffe38c2, 16, 4, GROUP(
 		PG27MD_00, PG27MD_01, PG27MD_10, PG27MD_11, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 
@@ -2396,9 +2396,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0,
 
 		PG24MD_00, PG24MD_01, PG24MD_10, PG24MD_11, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PGCR5", 0xfffe38c4, 16, 4) {
+	{ PINMUX_CFG_REG("PGCR5", 0xfffe38c4, 16, 4, GROUP(
 		PG23MD_000, PG23MD_001, PG23MD_010, PG23MD_011,
 		PG23MD_100, PG23MD_101, PG23MD_110, PG23MD_111,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -2413,9 +2413,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 
 		PG20MD_000, PG20MD_001, PG20MD_010, PG20MD_011,
 		PG20MD_100, PG20MD_101, PG20MD_110, PG20MD_111,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PGCR4", 0xfffe38c6, 16, 4) {
+	{ PINMUX_CFG_REG("PGCR4", 0xfffe38c6, 16, 4, GROUP(
 		PG19MD_000, PG19MD_001, PG19MD_010, PG19MD_011,
 		PG19MD_100, PG19MD_101, PG19MD_110, PG19MD_111,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -2428,9 +2428,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0,
 
 		PG16MD_00, PG16MD_01, PG16MD_10, PG16MD_11, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PGCR3", 0xfffe38c8, 16, 4) {
+	{ PINMUX_CFG_REG("PGCR3", 0xfffe38c8, 16, 4, GROUP(
 		PG15MD_00, PG15MD_01, PG15MD_10, PG15MD_11, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 
@@ -2441,9 +2441,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0,
 
 		PG12MD_00, PG12MD_01, PG12MD_10, PG12MD_11, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PGCR2", 0xfffe38ca, 16, 4) {
+	{ PINMUX_CFG_REG("PGCR2", 0xfffe38ca, 16, 4, GROUP(
 		PG11MD_000, PG11MD_001, PG11MD_010, PG11MD_011,
 		PG11MD_100, PG11MD_101, PG11MD_110, PG11MD_111,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -2458,10 +2458,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 
 		PG8MD_000, PG8MD_001, PG8MD_010, PG8MD_011,
 		PG8MD_100, PG8MD_101, PG8MD_110, PG8MD_111,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
 
-	{ PINMUX_CFG_REG("PGCR1", 0xfffe38cc, 16, 4) {
+	{ PINMUX_CFG_REG("PGCR1", 0xfffe38cc, 16, 4, GROUP(
 		PG7MD_000, PG7MD_001, PG7MD_010, PG7MD_011,
 		PG7MD_100, PG7MD_101, PG7MD_110, PG7MD_111,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -2476,9 +2476,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 
 		PG4MD_000, PG4MD_001, PG4MD_010, PG4MD_011,
 		PG4MD_100, PG4MD_101, PG4MD_110, PG4MD_111,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PGCR0", 0xfffe38ce, 16, 4) {
+	{ PINMUX_CFG_REG("PGCR0", 0xfffe38ce, 16, 4, GROUP(
 		PG3MD_000, PG3MD_001, PG3MD_010, PG3MD_011,
 		PG3MD_100, PG3MD_101, PG3MD_110, PG3MD_111,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -2493,10 +2493,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 
 		PG0MD_000, PG0MD_001, PG0MD_010, PG0MD_011,
 		PG0MD_100, PG0MD_101, PG0MD_110, PG0MD_111,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
 
-	{ PINMUX_CFG_REG("PGIOR1", 0xfffe38d0, 16, 1) {
+	{ PINMUX_CFG_REG("PGIOR1", 0xfffe38d0, 16, 1, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PG27_IN, PG27_OUT,
 		PG26_IN, PG26_OUT,
@@ -2509,9 +2509,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PG19_IN, PG19_OUT,
 		PG18_IN, PG18_OUT,
 		PG17_IN, PG17_OUT,
-		PG16_IN, PG16_OUT }
+		PG16_IN, PG16_OUT ))
 	},
-	{ PINMUX_CFG_REG("PGIOR0", 0xfffe38d2, 16, 1) {
+	{ PINMUX_CFG_REG("PGIOR0", 0xfffe38d2, 16, 1, GROUP(
 		PG15_IN, PG15_OUT,
 		PG14_IN, PG14_OUT,
 		PG13_IN, PG13_OUT,
@@ -2527,10 +2527,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PG3_IN, PG3_OUT,
 		PG2_IN, PG2_OUT,
 		PG1_IN, PG1_OUT,
-		PG0_IN, PG0_OUT }
+		PG0_IN, PG0_OUT ))
 	},
 
-	{ PINMUX_CFG_REG("PHCR1", 0xfffe38ec, 16, 4) {
+	{ PINMUX_CFG_REG("PHCR1", 0xfffe38ec, 16, 4, GROUP(
 		PH7MD_00, PH7MD_01, PH7MD_10, PH7MD_11, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 
@@ -2541,10 +2541,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0,
 
 		PH4MD_00, PH4MD_01, PH4MD_10, PH4MD_11, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
 
-	{ PINMUX_CFG_REG("PHCR0", 0xfffe38ee, 16, 4) {
+	{ PINMUX_CFG_REG("PHCR0", 0xfffe38ee, 16, 4, GROUP(
 		PH3MD_00, PH3MD_01, PH3MD_10, PH3MD_11, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 
@@ -2555,10 +2555,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0, 0, 0, 0, 0,
 
 		PH0MD_00, PH0MD_01, PH0MD_10, PH0MD_11, 0, 0, 0, 0,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
 
-	{ PINMUX_CFG_REG("PJCR7", 0xfffe3900, 16, 4) {
+	{ PINMUX_CFG_REG("PJCR7", 0xfffe3900, 16, 4, GROUP(
 		PJ31MD_0, PJ31MD_1, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0, 0, 0, 0, 0,
 
@@ -2572,9 +2572,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 
 		PJ28MD_000, PJ28MD_001, PJ28MD_010, PJ28MD_011,
 		PJ28MD_100, PJ28MD_101, PJ28MD_110, PJ28MD_111,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PJCR6", 0xfffe3902, 16, 4) {
+	{ PINMUX_CFG_REG("PJCR6", 0xfffe3902, 16, 4, GROUP(
 		PJ27MD_000, PJ27MD_001, PJ27MD_010, PJ27MD_011,
 		PJ27MD_100, PJ27MD_101, PJ27MD_110, PJ27MD_111,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -2589,9 +2589,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 
 		PJ24MD_000, PJ24MD_001, PJ24MD_010, PJ24MD_011,
 		PJ24MD_100, PJ24MD_101, PJ24MD_110, PJ24MD_111,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PJCR5", 0xfffe3904, 16, 4) {
+	{ PINMUX_CFG_REG("PJCR5", 0xfffe3904, 16, 4, GROUP(
 		PJ23MD_000, PJ23MD_001, PJ23MD_010, PJ23MD_011,
 		PJ23MD_100, PJ23MD_101, PJ23MD_110, PJ23MD_111,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -2606,9 +2606,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 
 		PJ20MD_000, PJ20MD_001, PJ20MD_010, PJ20MD_011,
 		PJ20MD_100, PJ20MD_101, PJ20MD_110, PJ20MD_111,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PJCR4", 0xfffe3906, 16, 4) {
+	{ PINMUX_CFG_REG("PJCR4", 0xfffe3906, 16, 4, GROUP(
 		PJ19MD_000, PJ19MD_001, PJ19MD_010, PJ19MD_011,
 		PJ19MD_100, PJ19MD_101, PJ19MD_110, PJ19MD_111,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -2623,9 +2623,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 
 		PJ16MD_000, PJ16MD_001, PJ16MD_010, PJ16MD_011,
 		PJ16MD_100, PJ16MD_101, PJ16MD_110, PJ16MD_111,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PJCR3", 0xfffe3908, 16, 4) {
+	{ PINMUX_CFG_REG("PJCR3", 0xfffe3908, 16, 4, GROUP(
 		PJ15MD_000, PJ15MD_001, PJ15MD_010, PJ15MD_011,
 		PJ15MD_100, PJ15MD_101, PJ15MD_110, PJ15MD_111,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -2640,9 +2640,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 
 		PJ12MD_000, PJ12MD_001, PJ12MD_010, PJ12MD_011,
 		PJ12MD_100, PJ12MD_101, PJ12MD_110, PJ12MD_111,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PJCR2", 0xfffe390a, 16, 4) {
+	{ PINMUX_CFG_REG("PJCR2", 0xfffe390a, 16, 4, GROUP(
 		PJ11MD_000, PJ11MD_001, PJ11MD_010, PJ11MD_011,
 		PJ11MD_100, PJ11MD_101, PJ11MD_110, PJ11MD_111,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -2657,9 +2657,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 
 		PJ8MD_000, PJ8MD_001, PJ8MD_010, PJ8MD_011,
 		PJ8MD_100, PJ8MD_101, PJ8MD_110, PJ8MD_111,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PJCR1", 0xfffe390c, 16, 4) {
+	{ PINMUX_CFG_REG("PJCR1", 0xfffe390c, 16, 4, GROUP(
 		PJ7MD_000, PJ7MD_001, PJ7MD_010, PJ7MD_011,
 		PJ7MD_100, PJ7MD_101, PJ7MD_110, PJ7MD_111,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -2674,9 +2674,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 
 		PJ4MD_000, PJ4MD_001, PJ4MD_010, PJ4MD_011,
 		PJ4MD_100, PJ4MD_101, PJ4MD_110, PJ4MD_111,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PJCR0", 0xfffe390e, 16, 4) {
+	{ PINMUX_CFG_REG("PJCR0", 0xfffe390e, 16, 4, GROUP(
 		PJ3MD_000, PJ3MD_001, PJ3MD_010, PJ3MD_011,
 		PJ3MD_100, PJ3MD_101, PJ3MD_110, PJ3MD_111,
 		0, 0, 0, 0, 0, 0, 0, 0,
@@ -2691,10 +2691,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 
 		PJ0MD_000, PJ0MD_001, PJ0MD_010, PJ0MD_011,
 		PJ0MD_100, PJ0MD_101, PJ0MD_110, PJ0MD_111,
-		0, 0, 0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0, 0, 0 ))
 	},
 
-	{ PINMUX_CFG_REG("PJIOR1", 0xfffe3910, 16, 1) {
+	{ PINMUX_CFG_REG("PJIOR1", 0xfffe3910, 16, 1, GROUP(
 		PJ31_IN, PJ31_OUT,
 		PJ30_IN, PJ30_OUT,
 		PJ29_IN, PJ29_OUT,
@@ -2710,9 +2710,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PJ19_IN, PJ19_OUT,
 		PJ18_IN, PJ18_OUT,
 		PJ17_IN, PJ17_OUT,
-		PJ16_IN, PJ16_OUT }
+		PJ16_IN, PJ16_OUT ))
 	},
-	{ PINMUX_CFG_REG("PJIOR0", 0xfffe3912, 16, 1) {
+	{ PINMUX_CFG_REG("PJIOR0", 0xfffe3912, 16, 1, GROUP(
 		PJ15_IN, PJ15_OUT,
 		PJ14_IN, PJ14_OUT,
 		PJ13_IN, PJ13_OUT,
@@ -2728,86 +2728,86 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PJ3_IN, PJ3_OUT,
 		PJ2_IN, PJ2_OUT,
 		PJ1_IN, PJ1_OUT,
-		PJ0_IN, PJ0_OUT }
+		PJ0_IN, PJ0_OUT ))
 	},
 
 	{}
 };
 
 static const struct pinmux_data_reg pinmux_data_regs[] = {
-	{ PINMUX_DATA_REG("PADR0", 0xfffe3816, 16) {
+	{ PINMUX_DATA_REG("PADR0", 0xfffe3816, 16, GROUP(
 		0, 0, 0, 0, 0, 0, 0, PA1_DATA,
-		0, 0, 0, 0, 0, 0, 0, PA0_DATA }
+		0, 0, 0, 0, 0, 0, 0, PA0_DATA ))
 	},
 
-	{ PINMUX_DATA_REG("PBDR1", 0xfffe3834, 16) {
+	{ PINMUX_DATA_REG("PBDR1", 0xfffe3834, 16, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, PB22_DATA, PB21_DATA, PB20_DATA,
-		PB19_DATA, PB18_DATA, PB17_DATA, PB16_DATA }
+		PB19_DATA, PB18_DATA, PB17_DATA, PB16_DATA ))
 	},
-	{ PINMUX_DATA_REG("PBDR0", 0xfffe3836, 16) {
+	{ PINMUX_DATA_REG("PBDR0", 0xfffe3836, 16, GROUP(
 		PB15_DATA, PB14_DATA, PB13_DATA, PB12_DATA,
 		PB11_DATA, PB10_DATA, PB9_DATA, PB8_DATA,
 		PB7_DATA, PB6_DATA, PB5_DATA, PB4_DATA,
-		PB3_DATA, PB2_DATA, PB1_DATA, 0 }
+		PB3_DATA, PB2_DATA, PB1_DATA, 0 ))
 	},
 
-	{ PINMUX_DATA_REG("PCDR0", 0xfffe3856, 16) {
+	{ PINMUX_DATA_REG("PCDR0", 0xfffe3856, 16, GROUP(
 		0, 0, 0, 0,
 		0, 0, 0, PC8_DATA,
 		PC7_DATA, PC6_DATA, PC5_DATA, PC4_DATA,
-		PC3_DATA, PC2_DATA, PC1_DATA, PC0_DATA }
+		PC3_DATA, PC2_DATA, PC1_DATA, PC0_DATA ))
 	},
 
-	{ PINMUX_DATA_REG("PDDR0", 0xfffe3876, 16) {
+	{ PINMUX_DATA_REG("PDDR0", 0xfffe3876, 16, GROUP(
 		PD15_DATA, PD14_DATA, PD13_DATA, PD12_DATA,
 		PD11_DATA, PD10_DATA, PD9_DATA, PD8_DATA,
 		PD7_DATA, PD6_DATA, PD5_DATA, PD4_DATA,
-		PD3_DATA, PD2_DATA, PD1_DATA, PD0_DATA }
+		PD3_DATA, PD2_DATA, PD1_DATA, PD0_DATA ))
 	},
 
-	{ PINMUX_DATA_REG("PEDR0", 0xfffe3896, 16) {
+	{ PINMUX_DATA_REG("PEDR0", 0xfffe3896, 16, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PE7_DATA, PE6_DATA, PE5_DATA, PE4_DATA,
-		PE3_DATA, PE2_DATA, PE1_DATA, PE0_DATA }
+		PE3_DATA, PE2_DATA, PE1_DATA, PE0_DATA ))
 	},
 
-	{ PINMUX_DATA_REG("PFDR1", 0xfffe38b4, 16) {
+	{ PINMUX_DATA_REG("PFDR1", 0xfffe38b4, 16, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PF23_DATA, PF22_DATA, PF21_DATA, PF20_DATA,
-		PF19_DATA, PF18_DATA, PF17_DATA, PF16_DATA }
+		PF19_DATA, PF18_DATA, PF17_DATA, PF16_DATA ))
 	},
-	{ PINMUX_DATA_REG("PFDR0", 0xfffe38b6, 16) {
+	{ PINMUX_DATA_REG("PFDR0", 0xfffe38b6, 16, GROUP(
 		PF15_DATA, PF14_DATA, PF13_DATA, PF12_DATA,
 		PF11_DATA, PF10_DATA, PF9_DATA, PF8_DATA,
 		PF7_DATA, PF6_DATA, PF5_DATA, PF4_DATA,
-		PF3_DATA, PF2_DATA, PF1_DATA, PF0_DATA }
+		PF3_DATA, PF2_DATA, PF1_DATA, PF0_DATA ))
 	},
 
-	{ PINMUX_DATA_REG("PGDR1", 0xfffe38d4, 16) {
+	{ PINMUX_DATA_REG("PGDR1", 0xfffe38d4, 16, GROUP(
 		0, 0, 0, 0,
 		PG27_DATA, PG26_DATA, PG25_DATA, PG24_DATA,
 		PG23_DATA, PG22_DATA, PG21_DATA, PG20_DATA,
-		PG19_DATA, PG18_DATA, PG17_DATA, PG16_DATA }
+		PG19_DATA, PG18_DATA, PG17_DATA, PG16_DATA ))
 	},
-	{ PINMUX_DATA_REG("PGDR0", 0xfffe38d6, 16) {
+	{ PINMUX_DATA_REG("PGDR0", 0xfffe38d6, 16, GROUP(
 		PG15_DATA, PG14_DATA, PG13_DATA, PG12_DATA,
 		PG11_DATA, PG10_DATA, PG9_DATA, PG8_DATA,
 		PG7_DATA, PG6_DATA, PG5_DATA, PG4_DATA,
-		PG3_DATA, PG2_DATA, PG1_DATA, PG0_DATA }
+		PG3_DATA, PG2_DATA, PG1_DATA, PG0_DATA ))
 	},
 
-	{ PINMUX_DATA_REG("PJDR1", 0xfffe3914, 16) {
+	{ PINMUX_DATA_REG("PJDR1", 0xfffe3914, 16, GROUP(
 		PJ31_DATA, PJ30_DATA, PJ29_DATA, PJ28_DATA,
 		PJ27_DATA, PJ26_DATA, PJ25_DATA, PJ24_DATA,
 		PJ23_DATA, PJ22_DATA, PJ21_DATA, PJ20_DATA,
-		PJ19_DATA, PJ18_DATA, PJ17_DATA, PJ16_DATA }
+		PJ19_DATA, PJ18_DATA, PJ17_DATA, PJ16_DATA ))
 	},
-	{ PINMUX_DATA_REG("PJDR0", 0xfffe3916, 16) {
+	{ PINMUX_DATA_REG("PJDR0", 0xfffe3916, 16, GROUP(
 		PJ15_DATA, PJ14_DATA, PJ13_DATA, PJ12_DATA,
 		PJ11_DATA, PJ10_DATA, PJ9_DATA, PJ8_DATA,
 		PJ7_DATA, PJ6_DATA, PJ5_DATA, PJ4_DATA,
-		PJ3_DATA, PJ2_DATA, PJ1_DATA, PJ0_DATA }
+		PJ3_DATA, PJ2_DATA, PJ1_DATA, PJ0_DATA ))
 	},
 
 	{ }
diff --git a/drivers/pinctrl/sh-pfc/pfc-sh73a0.c b/drivers/pinctrl/sh-pfc/pfc-sh73a0.c
index ef3da8bf1d87..e1276d143117 100644
--- a/drivers/pinctrl/sh-pfc/pfc-sh73a0.c
+++ b/drivers/pinctrl/sh-pfc/pfc-sh73a0.c
@@ -3971,7 +3971,7 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 	PORTCR(308, 0xe6052134), /* PORT308CR */
 	PORTCR(309, 0xe6052135), /* PORT309CR */
 
-	{ PINMUX_CFG_REG("MSEL2CR", 0xe605801c, 32, 1) {
+	{ PINMUX_CFG_REG("MSEL2CR", 0xe605801c, 32, 1, GROUP(
 			0, 0,
 			0, 0,
 			0, 0,
@@ -4004,9 +4004,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 			MSEL2CR_MSEL2_0, MSEL2CR_MSEL2_1,
 			MSEL2CR_MSEL1_0, MSEL2CR_MSEL1_1,
 			MSEL2CR_MSEL0_0, MSEL2CR_MSEL0_1,
-		}
+		))
 	},
-	{ PINMUX_CFG_REG("MSEL3CR", 0xe6058020, 32, 1) {
+	{ PINMUX_CFG_REG("MSEL3CR", 0xe6058020, 32, 1, GROUP(
 			0, 0,
 			0, 0,
 			0, 0,
@@ -4039,9 +4039,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 			MSEL3CR_MSEL2_0, MSEL3CR_MSEL2_1,
 			0, 0,
 			0, 0,
-		}
+		))
 	},
-	{ PINMUX_CFG_REG("MSEL4CR", 0xe6058024, 32, 1) {
+	{ PINMUX_CFG_REG("MSEL4CR", 0xe6058024, 32, 1, GROUP(
 			0, 0,
 			0, 0,
 			MSEL4CR_MSEL29_0, MSEL4CR_MSEL29_1,
@@ -4074,13 +4074,13 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 			0, 0,
 			MSEL4CR_MSEL1_0, MSEL4CR_MSEL1_1,
 			0, 0,
-		}
+		))
 	},
 	{ },
 };
 
 static const struct pinmux_data_reg pinmux_data_regs[] = {
-	{ PINMUX_DATA_REG("PORTL031_000DR", 0xe6054000, 32) {
+	{ PINMUX_DATA_REG("PORTL031_000DR", 0xe6054000, 32, GROUP(
 			PORT31_DATA, PORT30_DATA, PORT29_DATA, PORT28_DATA,
 			PORT27_DATA, PORT26_DATA, PORT25_DATA, PORT24_DATA,
 			PORT23_DATA, PORT22_DATA, PORT21_DATA, PORT20_DATA,
@@ -4088,9 +4088,9 @@ static const struct pinmux_data_reg pinmux_data_regs[] = {
 			PORT15_DATA, PORT14_DATA, PORT13_DATA, PORT12_DATA,
 			PORT11_DATA, PORT10_DATA, PORT9_DATA, PORT8_DATA,
 			PORT7_DATA, PORT6_DATA, PORT5_DATA, PORT4_DATA,
-			PORT3_DATA, PORT2_DATA, PORT1_DATA, PORT0_DATA }
+			PORT3_DATA, PORT2_DATA, PORT1_DATA, PORT0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PORTD063_032DR", 0xe6055000, 32) {
+	{ PINMUX_DATA_REG("PORTD063_032DR", 0xe6055000, 32, GROUP(
 			PORT63_DATA, PORT62_DATA, PORT61_DATA, PORT60_DATA,
 			PORT59_DATA, PORT58_DATA, PORT57_DATA, PORT56_DATA,
 			PORT55_DATA, PORT54_DATA, PORT53_DATA, PORT52_DATA,
@@ -4098,9 +4098,9 @@ static const struct pinmux_data_reg pinmux_data_regs[] = {
 			PORT47_DATA, PORT46_DATA, PORT45_DATA, PORT44_DATA,
 			PORT43_DATA, PORT42_DATA, PORT41_DATA, PORT40_DATA,
 			PORT39_DATA, PORT38_DATA, PORT37_DATA, PORT36_DATA,
-			PORT35_DATA, PORT34_DATA, PORT33_DATA, PORT32_DATA }
+			PORT35_DATA, PORT34_DATA, PORT33_DATA, PORT32_DATA ))
 	},
-	{ PINMUX_DATA_REG("PORTD095_064DR", 0xe6055004, 32) {
+	{ PINMUX_DATA_REG("PORTD095_064DR", 0xe6055004, 32, GROUP(
 			PORT95_DATA, PORT94_DATA, PORT93_DATA, PORT92_DATA,
 			PORT91_DATA, PORT90_DATA, PORT89_DATA, PORT88_DATA,
 			PORT87_DATA, PORT86_DATA, PORT85_DATA, PORT84_DATA,
@@ -4108,9 +4108,9 @@ static const struct pinmux_data_reg pinmux_data_regs[] = {
 			PORT79_DATA, PORT78_DATA, PORT77_DATA, PORT76_DATA,
 			PORT75_DATA, PORT74_DATA, PORT73_DATA, PORT72_DATA,
 			PORT71_DATA, PORT70_DATA, PORT69_DATA, PORT68_DATA,
-			PORT67_DATA, PORT66_DATA, PORT65_DATA, PORT64_DATA }
+			PORT67_DATA, PORT66_DATA, PORT65_DATA, PORT64_DATA ))
 	},
-	{ PINMUX_DATA_REG("PORTR127_096DR", 0xe6056000, 32) {
+	{ PINMUX_DATA_REG("PORTR127_096DR", 0xe6056000, 32, GROUP(
 			0, 0, 0, 0,
 			0, 0, 0, 0,
 			0, PORT118_DATA, PORT117_DATA, PORT116_DATA,
@@ -4118,9 +4118,9 @@ static const struct pinmux_data_reg pinmux_data_regs[] = {
 			PORT111_DATA, PORT110_DATA, PORT109_DATA, PORT108_DATA,
 			PORT107_DATA, PORT106_DATA, PORT105_DATA, PORT104_DATA,
 			PORT103_DATA, PORT102_DATA, PORT101_DATA, PORT100_DATA,
-			PORT99_DATA, PORT98_DATA, PORT97_DATA, PORT96_DATA }
+			PORT99_DATA, PORT98_DATA, PORT97_DATA, PORT96_DATA ))
 	},
-	{ PINMUX_DATA_REG("PORTR159_128DR", 0xe6056004, 32) {
+	{ PINMUX_DATA_REG("PORTR159_128DR", 0xe6056004, 32, GROUP(
 			PORT159_DATA, PORT158_DATA, PORT157_DATA, PORT156_DATA,
 			PORT155_DATA, PORT154_DATA, PORT153_DATA, PORT152_DATA,
 			PORT151_DATA, PORT150_DATA, PORT149_DATA, PORT148_DATA,
@@ -4128,9 +4128,9 @@ static const struct pinmux_data_reg pinmux_data_regs[] = {
 			PORT143_DATA, PORT142_DATA, PORT141_DATA, PORT140_DATA,
 			PORT139_DATA, PORT138_DATA, PORT137_DATA, PORT136_DATA,
 			PORT135_DATA, PORT134_DATA, PORT133_DATA, PORT132_DATA,
-			PORT131_DATA, PORT130_DATA, PORT129_DATA, PORT128_DATA }
+			PORT131_DATA, PORT130_DATA, PORT129_DATA, PORT128_DATA ))
 	},
-	{ PINMUX_DATA_REG("PORTR191_160DR", 0xe6056008, 32) {
+	{ PINMUX_DATA_REG("PORTR191_160DR", 0xe6056008, 32, GROUP(
 			0, 0, 0, 0,
 			0, 0, 0, 0,
 			0, 0, 0, 0,
@@ -4138,9 +4138,9 @@ static const struct pinmux_data_reg pinmux_data_regs[] = {
 			0, 0, 0, 0,
 			0, 0, 0, 0,
 			0, 0, 0, PORT164_DATA,
-			PORT163_DATA, PORT162_DATA, PORT161_DATA, PORT160_DATA }
+			PORT163_DATA, PORT162_DATA, PORT161_DATA, PORT160_DATA ))
 	},
-	{ PINMUX_DATA_REG("PORTR223_192DR", 0xe605600C, 32) {
+	{ PINMUX_DATA_REG("PORTR223_192DR", 0xe605600C, 32, GROUP(
 			PORT223_DATA, PORT222_DATA, PORT221_DATA, PORT220_DATA,
 			PORT219_DATA, PORT218_DATA, PORT217_DATA, PORT216_DATA,
 			PORT215_DATA, PORT214_DATA, PORT213_DATA, PORT212_DATA,
@@ -4148,9 +4148,9 @@ static const struct pinmux_data_reg pinmux_data_regs[] = {
 			PORT207_DATA, PORT206_DATA, PORT205_DATA, PORT204_DATA,
 			PORT203_DATA, PORT202_DATA, PORT201_DATA, PORT200_DATA,
 			PORT199_DATA, PORT198_DATA, PORT197_DATA, PORT196_DATA,
-			PORT195_DATA, PORT194_DATA, PORT193_DATA, PORT192_DATA }
+			PORT195_DATA, PORT194_DATA, PORT193_DATA, PORT192_DATA ))
 	},
-	{ PINMUX_DATA_REG("PORTU255_224DR", 0xe6057000, 32) {
+	{ PINMUX_DATA_REG("PORTU255_224DR", 0xe6057000, 32, GROUP(
 			PORT255_DATA, PORT254_DATA, PORT253_DATA, PORT252_DATA,
 			PORT251_DATA, PORT250_DATA, PORT249_DATA, PORT248_DATA,
 			PORT247_DATA, PORT246_DATA, PORT245_DATA, PORT244_DATA,
@@ -4158,9 +4158,9 @@ static const struct pinmux_data_reg pinmux_data_regs[] = {
 			PORT239_DATA, PORT238_DATA, PORT237_DATA, PORT236_DATA,
 			PORT235_DATA, PORT234_DATA, PORT233_DATA, PORT232_DATA,
 			PORT231_DATA, PORT230_DATA, PORT229_DATA, PORT228_DATA,
-			PORT227_DATA, PORT226_DATA, PORT225_DATA, PORT224_DATA }
+			PORT227_DATA, PORT226_DATA, PORT225_DATA, PORT224_DATA ))
 	},
-	{ PINMUX_DATA_REG("PORTU287_256DR", 0xe6057004, 32) {
+	{ PINMUX_DATA_REG("PORTU287_256DR", 0xe6057004, 32, GROUP(
 			0, 0, 0, 0,
 			0, PORT282_DATA, PORT281_DATA, PORT280_DATA,
 			PORT279_DATA, PORT278_DATA, PORT277_DATA, PORT276_DATA,
@@ -4168,9 +4168,9 @@ static const struct pinmux_data_reg pinmux_data_regs[] = {
 			PORT271_DATA, PORT270_DATA, PORT269_DATA, PORT268_DATA,
 			PORT267_DATA, PORT266_DATA, PORT265_DATA, PORT264_DATA,
 			PORT263_DATA, PORT262_DATA, PORT261_DATA, PORT260_DATA,
-			PORT259_DATA, PORT258_DATA, PORT257_DATA, PORT256_DATA }
+			PORT259_DATA, PORT258_DATA, PORT257_DATA, PORT256_DATA ))
 	},
-	{ PINMUX_DATA_REG("PORTR319_288DR", 0xe6056010, 32) {
+	{ PINMUX_DATA_REG("PORTR319_288DR", 0xe6056010, 32, GROUP(
 			0, 0, 0, 0,
 			0, 0, 0, 0,
 			0, 0, PORT309_DATA, PORT308_DATA,
@@ -4178,7 +4178,7 @@ static const struct pinmux_data_reg pinmux_data_regs[] = {
 			PORT303_DATA, PORT302_DATA, PORT301_DATA, PORT300_DATA,
 			PORT299_DATA, PORT298_DATA, PORT297_DATA, PORT296_DATA,
 			PORT295_DATA, PORT294_DATA, PORT293_DATA, PORT292_DATA,
-			PORT291_DATA, PORT290_DATA, PORT289_DATA, PORT288_DATA }
+			PORT291_DATA, PORT290_DATA, PORT289_DATA, PORT288_DATA ))
 	},
 	{ },
 };
diff --git a/drivers/pinctrl/sh-pfc/pfc-sh7720.c b/drivers/pinctrl/sh-pfc/pfc-sh7720.c
index 65694bfaa08d..37bcae6b3208 100644
--- a/drivers/pinctrl/sh-pfc/pfc-sh7720.c
+++ b/drivers/pinctrl/sh-pfc/pfc-sh7720.c
@@ -925,7 +925,7 @@ static const struct pinmux_func pinmux_func_gpios[] = {
 };
 
 static const struct pinmux_cfg_reg pinmux_config_regs[] = {
-	{ PINMUX_CFG_REG("PACR", 0xa4050100, 16, 2) {
+	{ PINMUX_CFG_REG("PACR", 0xa4050100, 16, 2, GROUP(
 		PTA7_FN, PTA7_OUT, 0, PTA7_IN,
 		PTA6_FN, PTA6_OUT, 0, PTA6_IN,
 		PTA5_FN, PTA5_OUT, 0, PTA5_IN,
@@ -933,9 +933,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTA3_FN, PTA3_OUT, 0, PTA3_IN,
 		PTA2_FN, PTA2_OUT, 0, PTA2_IN,
 		PTA1_FN, PTA1_OUT, 0, PTA1_IN,
-		PTA0_FN, PTA0_OUT, 0, PTA0_IN }
+		PTA0_FN, PTA0_OUT, 0, PTA0_IN ))
 	},
-	{ PINMUX_CFG_REG("PBCR", 0xa4050102, 16, 2) {
+	{ PINMUX_CFG_REG("PBCR", 0xa4050102, 16, 2, GROUP(
 		PTB7_FN, PTB7_OUT, 0, PTB7_IN,
 		PTB6_FN, PTB6_OUT, 0, PTB6_IN,
 		PTB5_FN, PTB5_OUT, 0, PTB5_IN,
@@ -943,9 +943,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTB3_FN, PTB3_OUT, 0, PTB3_IN,
 		PTB2_FN, PTB2_OUT, 0, PTB2_IN,
 		PTB1_FN, PTB1_OUT, 0, PTB1_IN,
-		PTB0_FN, PTB0_OUT, 0, PTB0_IN }
+		PTB0_FN, PTB0_OUT, 0, PTB0_IN ))
 	},
-	{ PINMUX_CFG_REG("PCCR", 0xa4050104, 16, 2) {
+	{ PINMUX_CFG_REG("PCCR", 0xa4050104, 16, 2, GROUP(
 		PTC7_FN, PTC7_OUT, 0, PTC7_IN,
 		PTC6_FN, PTC6_OUT, 0, PTC6_IN,
 		PTC5_FN, PTC5_OUT, 0, PTC5_IN,
@@ -953,9 +953,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTC3_FN, PTC3_OUT, 0, PTC3_IN,
 		PTC2_FN, PTC2_OUT, 0, PTC2_IN,
 		PTC1_FN, PTC1_OUT, 0, PTC1_IN,
-		PTC0_FN, PTC0_OUT, 0, PTC0_IN }
+		PTC0_FN, PTC0_OUT, 0, PTC0_IN ))
 	},
-	{ PINMUX_CFG_REG("PDCR", 0xa4050106, 16, 2) {
+	{ PINMUX_CFG_REG("PDCR", 0xa4050106, 16, 2, GROUP(
 		PTD7_FN, PTD7_OUT, 0, PTD7_IN,
 		PTD6_FN, PTD6_OUT, 0, PTD6_IN,
 		PTD5_FN, PTD5_OUT, 0, PTD5_IN,
@@ -963,9 +963,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTD3_FN, PTD3_OUT, 0, PTD3_IN,
 		PTD2_FN, PTD2_OUT, 0, PTD2_IN,
 		PTD1_FN, PTD1_OUT, 0, PTD1_IN,
-		PTD0_FN, PTD0_OUT, 0, PTD0_IN }
+		PTD0_FN, PTD0_OUT, 0, PTD0_IN ))
 	},
-	{ PINMUX_CFG_REG("PECR", 0xa4050108, 16, 2) {
+	{ PINMUX_CFG_REG("PECR", 0xa4050108, 16, 2, GROUP(
 		0, 0, 0, 0,
 		PTE6_FN, 0, 0, PTE6_IN,
 		PTE5_FN, 0, 0, PTE5_IN,
@@ -973,9 +973,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTE3_FN, PTE3_OUT, 0, PTE3_IN,
 		PTE2_FN, PTE2_OUT, 0, PTE2_IN,
 		PTE1_FN, PTE1_OUT, 0, PTE1_IN,
-		PTE0_FN, PTE0_OUT, 0, PTE0_IN }
+		PTE0_FN, PTE0_OUT, 0, PTE0_IN ))
 	},
-	{ PINMUX_CFG_REG("PFCR", 0xa405010a, 16, 2) {
+	{ PINMUX_CFG_REG("PFCR", 0xa405010a, 16, 2, GROUP(
 		0, 0, 0, 0,
 		PTF6_FN, 0, 0, PTF6_IN,
 		PTF5_FN, 0, 0, PTF5_IN,
@@ -983,9 +983,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTF3_FN, 0, 0, PTF3_IN,
 		PTF2_FN, 0, 0, PTF2_IN,
 		PTF1_FN, 0, 0, PTF1_IN,
-		PTF0_FN, 0, 0, PTF0_IN }
+		PTF0_FN, 0, 0, PTF0_IN ))
 	},
-	{ PINMUX_CFG_REG("PGCR", 0xa405010c, 16, 2) {
+	{ PINMUX_CFG_REG("PGCR", 0xa405010c, 16, 2, GROUP(
 		0, 0, 0, 0,
 		PTG6_FN, PTG6_OUT, 0, PTG6_IN,
 		PTG5_FN, PTG5_OUT, 0, PTG5_IN,
@@ -993,9 +993,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTG3_FN, PTG3_OUT, 0, PTG3_IN,
 		PTG2_FN, PTG2_OUT, 0, PTG2_IN,
 		PTG1_FN, PTG1_OUT, 0, PTG1_IN,
-		PTG0_FN, PTG0_OUT, 0, PTG0_IN }
+		PTG0_FN, PTG0_OUT, 0, PTG0_IN ))
 	},
-	{ PINMUX_CFG_REG("PHCR", 0xa405010e, 16, 2) {
+	{ PINMUX_CFG_REG("PHCR", 0xa405010e, 16, 2, GROUP(
 		0, 0, 0, 0,
 		PTH6_FN, PTH6_OUT, 0, PTH6_IN,
 		PTH5_FN, PTH5_OUT, 0, PTH5_IN,
@@ -1003,9 +1003,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTH3_FN, PTH3_OUT, 0, PTH3_IN,
 		PTH2_FN, PTH2_OUT, 0, PTH2_IN,
 		PTH1_FN, PTH1_OUT, 0, PTH1_IN,
-		PTH0_FN, PTH0_OUT, 0, PTH0_IN }
+		PTH0_FN, PTH0_OUT, 0, PTH0_IN ))
 	},
-	{ PINMUX_CFG_REG("PJCR", 0xa4050110, 16, 2) {
+	{ PINMUX_CFG_REG("PJCR", 0xa4050110, 16, 2, GROUP(
 		0, 0, 0, 0,
 		PTJ6_FN, PTJ6_OUT, 0, PTJ6_IN,
 		PTJ5_FN, PTJ5_OUT, 0, PTJ5_IN,
@@ -1013,9 +1013,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTJ3_FN, PTJ3_OUT, 0, PTJ3_IN,
 		PTJ2_FN, PTJ2_OUT, 0, PTJ2_IN,
 		PTJ1_FN, PTJ1_OUT, 0, PTJ1_IN,
-		PTJ0_FN, PTJ0_OUT, 0, PTJ0_IN }
+		PTJ0_FN, PTJ0_OUT, 0, PTJ0_IN ))
 	},
-	{ PINMUX_CFG_REG("PKCR", 0xa4050112, 16, 2) {
+	{ PINMUX_CFG_REG("PKCR", 0xa4050112, 16, 2, GROUP(
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		0, 0, 0, 0,
@@ -1023,9 +1023,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTK3_FN, PTK3_OUT, 0, PTK3_IN,
 		PTK2_FN, PTK2_OUT, 0, PTK2_IN,
 		PTK1_FN, PTK1_OUT, 0, PTK1_IN,
-		PTK0_FN, PTK0_OUT, 0, PTK0_IN }
+		PTK0_FN, PTK0_OUT, 0, PTK0_IN ))
 	},
-	{ PINMUX_CFG_REG("PLCR", 0xa4050114, 16, 2) {
+	{ PINMUX_CFG_REG("PLCR", 0xa4050114, 16, 2, GROUP(
 		PTL7_FN, PTL7_OUT, 0, PTL7_IN,
 		PTL6_FN, PTL6_OUT, 0, PTL6_IN,
 		PTL5_FN, PTL5_OUT, 0, PTL5_IN,
@@ -1033,9 +1033,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTL3_FN, PTL3_OUT, 0, PTL3_IN,
 		0, 0, 0, 0,
 		0, 0, 0, 0,
-		0, 0, 0, 0 }
+		0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PMCR", 0xa4050116, 16, 2) {
+	{ PINMUX_CFG_REG("PMCR", 0xa4050116, 16, 2, GROUP(
 		PTM7_FN, PTM7_OUT, 0, PTM7_IN,
 		PTM6_FN, PTM6_OUT, 0, PTM6_IN,
 		PTM5_FN, PTM5_OUT, 0, PTM5_IN,
@@ -1043,9 +1043,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTM3_FN, PTM3_OUT, 0, PTM3_IN,
 		PTM2_FN, PTM2_OUT, 0, PTM2_IN,
 		PTM1_FN, PTM1_OUT, 0, PTM1_IN,
-		PTM0_FN, PTM0_OUT, 0, PTM0_IN }
+		PTM0_FN, PTM0_OUT, 0, PTM0_IN ))
 	},
-	{ PINMUX_CFG_REG("PPCR", 0xa4050118, 16, 2) {
+	{ PINMUX_CFG_REG("PPCR", 0xa4050118, 16, 2, GROUP(
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		0, 0, 0, 0,
@@ -1053,9 +1053,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTP3_FN, PTP3_OUT, 0, PTP3_IN,
 		PTP2_FN, PTP2_OUT, 0, PTP2_IN,
 		PTP1_FN, PTP1_OUT, 0, PTP1_IN,
-		PTP0_FN, PTP0_OUT, 0, PTP0_IN }
+		PTP0_FN, PTP0_OUT, 0, PTP0_IN ))
 	},
-	{ PINMUX_CFG_REG("PRCR", 0xa405011a, 16, 2) {
+	{ PINMUX_CFG_REG("PRCR", 0xa405011a, 16, 2, GROUP(
 		PTR7_FN, PTR7_OUT, 0, PTR7_IN,
 		PTR6_FN, PTR6_OUT, 0, PTR6_IN,
 		PTR5_FN, PTR5_OUT, 0, PTR5_IN,
@@ -1063,9 +1063,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTR3_FN, PTR3_OUT, 0, PTR3_IN,
 		PTR2_FN, PTR2_OUT, 0, PTR2_IN,
 		PTR1_FN, PTR1_OUT, 0, PTR1_IN,
-		PTR0_FN, PTR0_OUT, 0, PTR0_IN }
+		PTR0_FN, PTR0_OUT, 0, PTR0_IN ))
 	},
-	{ PINMUX_CFG_REG("PSCR", 0xa405011c, 16, 2) {
+	{ PINMUX_CFG_REG("PSCR", 0xa405011c, 16, 2, GROUP(
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		0, 0, 0, 0,
@@ -1073,9 +1073,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTS3_FN, PTS3_OUT, 0, PTS3_IN,
 		PTS2_FN, PTS2_OUT, 0, PTS2_IN,
 		PTS1_FN, PTS1_OUT, 0, PTS1_IN,
-		PTS0_FN, PTS0_OUT, 0, PTS0_IN }
+		PTS0_FN, PTS0_OUT, 0, PTS0_IN ))
 	},
-	{ PINMUX_CFG_REG("PTCR", 0xa405011e, 16, 2) {
+	{ PINMUX_CFG_REG("PTCR", 0xa405011e, 16, 2, GROUP(
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		0, 0, 0, 0,
@@ -1083,9 +1083,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTT3_FN, PTT3_OUT, 0, PTT3_IN,
 		PTT2_FN, PTT2_OUT, 0, PTT2_IN,
 		PTT1_FN, PTT1_OUT, 0, PTT1_IN,
-		PTT0_FN, PTT0_OUT, 0, PTT0_IN }
+		PTT0_FN, PTT0_OUT, 0, PTT0_IN ))
 	},
-	{ PINMUX_CFG_REG("PUCR", 0xa4050120, 16, 2) {
+	{ PINMUX_CFG_REG("PUCR", 0xa4050120, 16, 2, GROUP(
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		0, 0, 0, 0,
@@ -1093,9 +1093,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTU3_FN, PTU3_OUT, 0, PTU3_IN,
 		PTU2_FN, PTU2_OUT, 0, PTU2_IN,
 		PTU1_FN, PTU1_OUT, 0, PTU1_IN,
-		PTU0_FN, PTU0_OUT, 0, PTU0_IN }
+		PTU0_FN, PTU0_OUT, 0, PTU0_IN ))
 	},
-	{ PINMUX_CFG_REG("PVCR", 0xa4050122, 16, 2) {
+	{ PINMUX_CFG_REG("PVCR", 0xa4050122, 16, 2, GROUP(
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		0, 0, 0, 0,
@@ -1103,83 +1103,83 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTV3_FN, PTV3_OUT, 0, PTV3_IN,
 		PTV2_FN, PTV2_OUT, 0, PTV2_IN,
 		PTV1_FN, PTV1_OUT, 0, PTV1_IN,
-		PTV0_FN, PTV0_OUT, 0, PTV0_IN }
+		PTV0_FN, PTV0_OUT, 0, PTV0_IN ))
 	},
 	{}
 };
 
 static const struct pinmux_data_reg pinmux_data_regs[] = {
-	{ PINMUX_DATA_REG("PADR", 0xa4050140, 8) {
+	{ PINMUX_DATA_REG("PADR", 0xa4050140, 8, GROUP(
 		PTA7_DATA, PTA6_DATA, PTA5_DATA, PTA4_DATA,
-		PTA3_DATA, PTA2_DATA, PTA1_DATA, PTA0_DATA }
+		PTA3_DATA, PTA2_DATA, PTA1_DATA, PTA0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PBDR", 0xa4050142, 8) {
+	{ PINMUX_DATA_REG("PBDR", 0xa4050142, 8, GROUP(
 		PTB7_DATA, PTB6_DATA, PTB5_DATA, PTB4_DATA,
-		PTB3_DATA, PTB2_DATA, PTB1_DATA, PTB0_DATA }
+		PTB3_DATA, PTB2_DATA, PTB1_DATA, PTB0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PCDR", 0xa4050144, 8) {
+	{ PINMUX_DATA_REG("PCDR", 0xa4050144, 8, GROUP(
 		PTC7_DATA, PTC6_DATA, PTC5_DATA, PTC4_DATA,
-		PTC3_DATA, PTC2_DATA, PTC1_DATA, PTC0_DATA }
+		PTC3_DATA, PTC2_DATA, PTC1_DATA, PTC0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PDDR", 0xa4050126, 8) {
+	{ PINMUX_DATA_REG("PDDR", 0xa4050126, 8, GROUP(
 		PTD7_DATA, PTD6_DATA, PTD5_DATA, PTD4_DATA,
-		PTD3_DATA, PTD2_DATA, PTD1_DATA, PTD0_DATA }
+		PTD3_DATA, PTD2_DATA, PTD1_DATA, PTD0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PEDR", 0xa4050148, 8) {
+	{ PINMUX_DATA_REG("PEDR", 0xa4050148, 8, GROUP(
 		0, PTE6_DATA, PTE5_DATA, PTE4_DATA,
-		PTE3_DATA, PTE2_DATA, PTE1_DATA, PTE0_DATA }
+		PTE3_DATA, PTE2_DATA, PTE1_DATA, PTE0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PFDR", 0xa405014a, 8) {
+	{ PINMUX_DATA_REG("PFDR", 0xa405014a, 8, GROUP(
 		0, PTF6_DATA, PTF5_DATA, PTF4_DATA,
-		PTF3_DATA, PTF2_DATA, PTF1_DATA, PTF0_DATA }
+		PTF3_DATA, PTF2_DATA, PTF1_DATA, PTF0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PGDR", 0xa405014c, 8) {
+	{ PINMUX_DATA_REG("PGDR", 0xa405014c, 8, GROUP(
 		0, PTG6_DATA, PTG5_DATA, PTG4_DATA,
-		PTG3_DATA, PTG2_DATA, PTG1_DATA, PTG0_DATA }
+		PTG3_DATA, PTG2_DATA, PTG1_DATA, PTG0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PHDR", 0xa405014e, 8) {
+	{ PINMUX_DATA_REG("PHDR", 0xa405014e, 8, GROUP(
 		0, PTH6_DATA, PTH5_DATA, PTH4_DATA,
-		PTH3_DATA, PTH2_DATA, PTH1_DATA, PTH0_DATA }
+		PTH3_DATA, PTH2_DATA, PTH1_DATA, PTH0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PJDR", 0xa4050150, 8) {
+	{ PINMUX_DATA_REG("PJDR", 0xa4050150, 8, GROUP(
 		0, PTJ6_DATA, PTJ5_DATA, PTJ4_DATA,
-		PTJ3_DATA, PTJ2_DATA, PTJ1_DATA, PTJ0_DATA }
+		PTJ3_DATA, PTJ2_DATA, PTJ1_DATA, PTJ0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PKDR", 0xa4050152, 8) {
+	{ PINMUX_DATA_REG("PKDR", 0xa4050152, 8, GROUP(
 		0, 0, 0, 0,
-		PTK3_DATA, PTK2_DATA, PTK1_DATA, PTK0_DATA }
+		PTK3_DATA, PTK2_DATA, PTK1_DATA, PTK0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PLDR", 0xa4050154, 8) {
+	{ PINMUX_DATA_REG("PLDR", 0xa4050154, 8, GROUP(
 		PTL7_DATA, PTL6_DATA, PTL5_DATA, PTL4_DATA,
-		PTL3_DATA, 0, 0, 0 }
+		PTL3_DATA, 0, 0, 0 ))
 	},
-	{ PINMUX_DATA_REG("PMDR", 0xa4050156, 8) {
+	{ PINMUX_DATA_REG("PMDR", 0xa4050156, 8, GROUP(
 		PTM7_DATA, PTM6_DATA, PTM5_DATA, PTM4_DATA,
-		PTM3_DATA, PTM2_DATA, PTM1_DATA, PTM0_DATA }
+		PTM3_DATA, PTM2_DATA, PTM1_DATA, PTM0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PPDR", 0xa4050158, 8) {
+	{ PINMUX_DATA_REG("PPDR", 0xa4050158, 8, GROUP(
 		0, 0, 0, PTP4_DATA,
-		PTP3_DATA, PTP2_DATA, PTP1_DATA, PTP0_DATA }
+		PTP3_DATA, PTP2_DATA, PTP1_DATA, PTP0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PRDR", 0xa405015a, 8) {
+	{ PINMUX_DATA_REG("PRDR", 0xa405015a, 8, GROUP(
 		PTR7_DATA, PTR6_DATA, PTR5_DATA, PTR4_DATA,
-		PTR3_DATA, PTR2_DATA, PTR1_DATA, PTR0_DATA }
+		PTR3_DATA, PTR2_DATA, PTR1_DATA, PTR0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PSDR", 0xa405015c, 8) {
+	{ PINMUX_DATA_REG("PSDR", 0xa405015c, 8, GROUP(
 		0, 0, 0, PTS4_DATA,
-		PTS3_DATA, PTS2_DATA, PTS1_DATA, PTS0_DATA }
+		PTS3_DATA, PTS2_DATA, PTS1_DATA, PTS0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PTDR", 0xa405015e, 8) {
+	{ PINMUX_DATA_REG("PTDR", 0xa405015e, 8, GROUP(
 		0, 0, 0, PTT4_DATA,
-		PTT3_DATA, PTT2_DATA, PTT1_DATA, PTT0_DATA }
+		PTT3_DATA, PTT2_DATA, PTT1_DATA, PTT0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PUDR", 0xa4050160, 8) {
+	{ PINMUX_DATA_REG("PUDR", 0xa4050160, 8, GROUP(
 		0, 0, 0, PTU4_DATA,
-		PTU3_DATA, PTU2_DATA, PTU1_DATA, PTU0_DATA }
+		PTU3_DATA, PTU2_DATA, PTU1_DATA, PTU0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PVDR", 0xa4050162, 8) {
+	{ PINMUX_DATA_REG("PVDR", 0xa4050162, 8, GROUP(
 		0, 0, 0, PTV4_DATA,
-		PTV3_DATA, PTV2_DATA, PTV1_DATA, PTV0_DATA }
+		PTV3_DATA, PTV2_DATA, PTV1_DATA, PTV0_DATA ))
 	},
 	{ },
 };
diff --git a/drivers/pinctrl/sh-pfc/pfc-sh7722.c b/drivers/pinctrl/sh-pfc/pfc-sh7722.c
index 0e733bffdb38..95295be4e703 100644
--- a/drivers/pinctrl/sh-pfc/pfc-sh7722.c
+++ b/drivers/pinctrl/sh-pfc/pfc-sh7722.c
@@ -1237,7 +1237,7 @@ static const struct pinmux_func pinmux_func_gpios[] = {
 };
 
 static const struct pinmux_cfg_reg pinmux_config_regs[] = {
-	{ PINMUX_CFG_REG("PACR", 0xa4050100, 16, 2) {
+	{ PINMUX_CFG_REG("PACR", 0xa4050100, 16, 2, GROUP(
 		VIO_D7_SCIF1_SCK, PTA7_OUT, 0, PTA7_IN,
 		VIO_D6_SCIF1_RXD, 0, 0, PTA6_IN,
 		VIO_D5_SCIF1_TXD, PTA5_OUT, 0, PTA5_IN,
@@ -1245,9 +1245,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		VIO_D3, 0, 0, PTA3_IN,
 		VIO_D2, 0, 0, PTA2_IN,
 		VIO_D1, 0, 0, PTA1_IN,
-		VIO_D0_LCDLCLK, 0, 0, PTA0_IN }
+		VIO_D0_LCDLCLK, 0, 0, PTA0_IN ))
 	},
-	{ PINMUX_CFG_REG("PBCR", 0xa4050102, 16, 2) {
+	{ PINMUX_CFG_REG("PBCR", 0xa4050102, 16, 2, GROUP(
 		HPD55, PTB7_OUT, 0, PTB7_IN,
 		HPD54, PTB6_OUT, 0, PTB6_IN,
 		HPD53, PTB5_OUT, 0, PTB5_IN,
@@ -1255,9 +1255,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		HPD51, PTB3_OUT, 0, PTB3_IN,
 		HPD50, PTB2_OUT, 0, PTB2_IN,
 		HPD49, PTB1_OUT, 0, PTB1_IN,
-		HPD48, PTB0_OUT, 0, PTB0_IN }
+		HPD48, PTB0_OUT, 0, PTB0_IN ))
 	},
-	{ PINMUX_CFG_REG("PCCR", 0xa4050104, 16, 2) {
+	{ PINMUX_CFG_REG("PCCR", 0xa4050104, 16, 2, GROUP(
 		0, 0, 0, PTC7_IN,
 		0, 0, 0, 0,
 		IOIS16, 0, 0, PTC5_IN,
@@ -1265,9 +1265,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		HPDQM6, PTC3_OUT, 0, PTC3_IN,
 		HPDQM5, PTC2_OUT, 0, PTC2_IN,
 		0, 0, 0, 0,
-		HPDQM4, PTC0_OUT, 0, PTC0_IN }
+		HPDQM4, PTC0_OUT, 0, PTC0_IN ))
 	},
-	{ PINMUX_CFG_REG("PDCR", 0xa4050106, 16, 2) {
+	{ PINMUX_CFG_REG("PDCR", 0xa4050106, 16, 2, GROUP(
 		SDHICD, 0, 0, PTD7_IN,
 		SDHIWP, PTD6_OUT, 0, PTD6_IN,
 		SDHID3, PTD5_OUT, 0, PTD5_IN,
@@ -1275,9 +1275,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		SDHID1, PTD3_OUT, 0, PTD3_IN,
 		SDHID0, PTD2_OUT, 0, PTD2_IN,
 		SDHICMD, PTD1_OUT, 0, PTD1_IN,
-		SDHICLK, PTD0_OUT, 0, 0 }
+		SDHICLK, PTD0_OUT, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PECR", 0xa4050108, 16, 2) {
+	{ PINMUX_CFG_REG("PECR", 0xa4050108, 16, 2, GROUP(
 		A25, PTE7_OUT, 0, PTE7_IN,
 		A24, PTE6_OUT, 0, PTE6_IN,
 		A23, PTE5_OUT, 0, PTE5_IN,
@@ -1285,9 +1285,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		IRQ5, PTE1_OUT, 0, PTE1_IN,
-		IRQ4_BS, PTE0_OUT, 0, PTE0_IN }
+		IRQ4_BS, PTE0_OUT, 0, PTE0_IN ))
 	},
-	{ PINMUX_CFG_REG("PFCR", 0xa405010a, 16, 2) {
+	{ PINMUX_CFG_REG("PFCR", 0xa405010a, 16, 2, GROUP(
 		0, 0, 0, 0,
 		PTF6, PTF6_OUT, 0, PTF6_IN,
 		SIOSCK_SIUBOBT, PTF5_OUT, 0, PTF5_IN,
@@ -1295,9 +1295,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		SIOSTRB0_SIUBIBT, PTF3_OUT, 0, PTF3_IN,
 		SIOD_SIUBILR, PTF2_OUT, 0, PTF2_IN,
 		SIORXD_SIUBISLD, 0, 0, PTF1_IN,
-		SIOTXD_SIUBOSLD, PTF0_OUT, 0, 0 }
+		SIOTXD_SIUBOSLD, PTF0_OUT, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PGCR", 0xa405010c, 16, 2) {
+	{ PINMUX_CFG_REG("PGCR", 0xa405010c, 16, 2, GROUP(
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		0, 0, 0, 0,
@@ -1305,9 +1305,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		AUDATA3, PTG3_OUT, 0, 0,
 		AUDATA2, PTG2_OUT, 0, 0,
 		AUDATA1, PTG1_OUT, 0, 0,
-		AUDATA0, PTG0_OUT, 0, 0 }
+		AUDATA0, PTG0_OUT, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PHCR", 0xa405010e, 16, 2) {
+	{ PINMUX_CFG_REG("PHCR", 0xa405010e, 16, 2, GROUP(
 		LCDVCPWC_LCDVCPWC2, PTH7_OUT, 0, 0,
 		LCDVSYN2_DACK, PTH6_OUT, 0, PTH6_IN,
 		LCDVSYN, PTH5_OUT, 0, PTH5_IN,
@@ -1315,9 +1315,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		LCDHSYN_LCDCS, PTH3_OUT, 0, 0,
 		LCDDON_LCDDON2, PTH2_OUT, 0, 0,
 		LCDD17_DV_HSYNC, PTH1_OUT, 0, PTH1_IN,
-		LCDD16_DV_VSYNC, PTH0_OUT, 0, PTH0_IN }
+		LCDD16_DV_VSYNC, PTH0_OUT, 0, PTH0_IN ))
 	},
-	{ PINMUX_CFG_REG("PJCR", 0xa4050110, 16, 2) {
+	{ PINMUX_CFG_REG("PJCR", 0xa4050110, 16, 2, GROUP(
 		STATUS0, PTJ7_OUT, 0, 0,
 		0, PTJ6_OUT, 0, 0,
 		PDSTATUS, PTJ5_OUT, 0, 0,
@@ -1325,9 +1325,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		IRQ1, PTJ1_OUT, 0, PTJ1_IN,
-		IRQ0, PTJ0_OUT, 0, PTJ0_IN }
+		IRQ0, PTJ0_OUT, 0, PTJ0_IN ))
 	},
-	{ PINMUX_CFG_REG("PKCR", 0xa4050112, 16, 2) {
+	{ PINMUX_CFG_REG("PKCR", 0xa4050112, 16, 2, GROUP(
 		0, 0, 0, 0,
 		SIUAILR_SIOF1_SS2, PTK6_OUT, 0, PTK6_IN,
 		SIUAIBT_SIOF1_SS1, PTK5_OUT, 0, PTK5_IN,
@@ -1335,9 +1335,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		SIUAOBT_SIOF1_SCK, PTK3_OUT, 0, PTK3_IN,
 		SIUAISLD_SIOF1_RXD, 0, 0, PTK2_IN,
 		SIUAOSLD_SIOF1_TXD, PTK1_OUT, 0, 0,
-		PTK0, PTK0_OUT, 0, PTK0_IN }
+		PTK0, PTK0_OUT, 0, PTK0_IN ))
 	},
-	{ PINMUX_CFG_REG("PLCR", 0xa4050114, 16, 2) {
+	{ PINMUX_CFG_REG("PLCR", 0xa4050114, 16, 2, GROUP(
 		LCDD15_DV_D15, PTL7_OUT, 0, PTL7_IN,
 		LCDD14_DV_D14, PTL6_OUT, 0, PTL6_IN,
 		LCDD13_DV_D13, PTL5_OUT, 0, PTL5_IN,
@@ -1345,9 +1345,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		LCDD11_DV_D11, PTL3_OUT, 0, PTL3_IN,
 		LCDD10_DV_D10, PTL2_OUT, 0, PTL2_IN,
 		LCDD9_DV_D9, PTL1_OUT, 0, PTL1_IN,
-		LCDD8_DV_D8, PTL0_OUT, 0, PTL0_IN }
+		LCDD8_DV_D8, PTL0_OUT, 0, PTL0_IN ))
 	},
-	{ PINMUX_CFG_REG("PMCR", 0xa4050116, 16, 2) {
+	{ PINMUX_CFG_REG("PMCR", 0xa4050116, 16, 2, GROUP(
 		LCDD7_DV_D7, PTM7_OUT, 0, PTM7_IN,
 		LCDD6_DV_D6, PTM6_OUT, 0, PTM6_IN,
 		LCDD5_DV_D5, PTM5_OUT, 0, PTM5_IN,
@@ -1355,9 +1355,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		LCDD3_DV_D3, PTM3_OUT, 0, PTM3_IN,
 		LCDD2_DV_D2, PTM2_OUT, 0, PTM2_IN,
 		LCDD1_DV_D1, PTM1_OUT, 0, PTM1_IN,
-		LCDD0_DV_D0, PTM0_OUT, 0, PTM0_IN }
+		LCDD0_DV_D0, PTM0_OUT, 0, PTM0_IN ))
 	},
-	{ PINMUX_CFG_REG("PNCR", 0xa4050118, 16, 2) {
+	{ PINMUX_CFG_REG("PNCR", 0xa4050118, 16, 2, GROUP(
 		HPD63, PTN7_OUT, 0, PTN7_IN,
 		HPD62, PTN6_OUT, 0, PTN6_IN,
 		HPD61, PTN5_OUT, 0, PTN5_IN,
@@ -1365,9 +1365,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		HPD59, PTN3_OUT, 0, PTN3_IN,
 		HPD58, PTN2_OUT, 0, PTN2_IN,
 		HPD57, PTN1_OUT, 0, PTN1_IN,
-		HPD56, PTN0_OUT, 0, PTN0_IN }
+		HPD56, PTN0_OUT, 0, PTN0_IN ))
 	},
-	{ PINMUX_CFG_REG("PQCR", 0xa405011a, 16, 2) {
+	{ PINMUX_CFG_REG("PQCR", 0xa405011a, 16, 2, GROUP(
 		0, 0, 0, 0,
 		SIOF0_SS2_SIM_RST, PTQ6_OUT, 0, 0,
 		SIOF0_SS1_TS_SPSYNC, PTQ5_OUT, 0, PTQ5_IN,
@@ -1375,9 +1375,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		SIOF0_SCK_TS_SCK, PTQ3_OUT, 0, PTQ3_IN,
 		PTQ2, 0, 0, PTQ2_IN,
 		PTQ1, PTQ1_OUT, 0, 0,
-		PTQ0, PTQ0_OUT, 0, PTQ0_IN }
+		PTQ0, PTQ0_OUT, 0, PTQ0_IN ))
 	},
-	{ PINMUX_CFG_REG("PRCR", 0xa405011c, 16, 2) {
+	{ PINMUX_CFG_REG("PRCR", 0xa405011c, 16, 2, GROUP(
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		0, 0, 0, 0,
@@ -1385,9 +1385,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		CS6B_CE1B_LCDCS2, PTR3_OUT, 0, 0,
 		WAIT, 0, 0, PTR2_IN,
 		LCDDCK_LCDWR, PTR1_OUT, 0, 0,
-		LCDVEPWC_LCDVEPWC2, PTR0_OUT, 0, 0 }
+		LCDVEPWC_LCDVEPWC2, PTR0_OUT, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PSCR", 0xa405011e, 16, 2) {
+	{ PINMUX_CFG_REG("PSCR", 0xa405011e, 16, 2, GROUP(
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		0, 0, 0, 0,
@@ -1395,9 +1395,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		SCIF0_RTS_SIUAOSPD, PTS3_OUT, 0, 0,
 		SCIF0_SCK_TPUTO, PTS2_OUT, 0, PTS2_IN,
 		SCIF0_RXD, 0, 0, PTS1_IN,
-		SCIF0_TXD, PTS0_OUT, 0, 0 }
+		SCIF0_TXD, PTS0_OUT, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PTCR", 0xa4050140, 16, 2) {
+	{ PINMUX_CFG_REG("PTCR", 0xa4050140, 16, 2, GROUP(
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		0, 0, 0, 0,
@@ -1405,9 +1405,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FWE, PTT3_OUT, 0, PTT3_IN,
 		FSC, PTT2_OUT, 0, PTT2_IN,
 		DREQ0, 0, 0, PTT1_IN,
-		FCDE, PTT0_OUT, 0, 0 }
+		FCDE, PTT0_OUT, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PUCR", 0xa4050142, 16, 2) {
+	{ PINMUX_CFG_REG("PUCR", 0xa4050142, 16, 2, GROUP(
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		0, 0, 0, 0,
@@ -1415,9 +1415,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		NAF1_VIO_D9, PTU3_OUT, 0, PTU3_IN,
 		NAF0_VIO_D8, PTU2_OUT, 0, PTU2_IN,
 		FRB_VIO_CLK2, 0, 0, PTU1_IN,
-		FCE_VIO_HD2, PTU0_OUT, 0, PTU0_IN }
+		FCE_VIO_HD2, PTU0_OUT, 0, PTU0_IN ))
 	},
-	{ PINMUX_CFG_REG("PVCR", 0xa4050144, 16, 2) {
+	{ PINMUX_CFG_REG("PVCR", 0xa4050144, 16, 2, GROUP(
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		0, 0, 0, 0,
@@ -1425,9 +1425,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		NAF6_VIO_D14, PTV3_OUT, 0, PTV3_IN,
 		NAF5_VIO_D13, PTV2_OUT, 0, PTV2_IN,
 		NAF4_VIO_D12, PTV1_OUT, 0, PTV1_IN,
-		NAF3_VIO_D11, PTV0_OUT, 0, PTV0_IN }
+		NAF3_VIO_D11, PTV0_OUT, 0, PTV0_IN ))
 	},
-	{ PINMUX_CFG_REG("PWCR", 0xa4050146, 16, 2) {
+	{ PINMUX_CFG_REG("PWCR", 0xa4050146, 16, 2, GROUP(
 		0, 0, 0, 0,
 		VIO_FLD_SCIF2_CTS, 0, 0, PTW6_IN,
 		VIO_CKO_SCIF2_RTS, PTW5_OUT, 0, 0,
@@ -1435,9 +1435,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		VIO_STEM_SCIF2_TXD, PTW3_OUT, 0, PTW3_IN,
 		VIO_HD_SCIF2_RXD, PTW2_OUT, 0, PTW2_IN,
 		VIO_VD_SCIF1_CTS, PTW1_OUT, 0, PTW1_IN,
-		VIO_CLK_SCIF1_RTS, PTW0_OUT, 0, PTW0_IN }
+		VIO_CLK_SCIF1_RTS, PTW0_OUT, 0, PTW0_IN ))
 	},
-	{ PINMUX_CFG_REG("PXCR", 0xa4050148, 16, 2) {
+	{ PINMUX_CFG_REG("PXCR", 0xa4050148, 16, 2, GROUP(
 		0, 0, 0, 0,
 		CS6A_CE2B, PTX6_OUT, 0, PTX6_IN,
 		LCDD23, PTX5_OUT, 0, PTX5_IN,
@@ -1445,9 +1445,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		LCDD21, PTX3_OUT, 0, PTX3_IN,
 		LCDD20, PTX2_OUT, 0, PTX2_IN,
 		LCDD19_DV_CLKI, PTX1_OUT, 0, PTX1_IN,
-		LCDD18_DV_CLK, PTX0_OUT, 0, PTX0_IN }
+		LCDD18_DV_CLK, PTX0_OUT, 0, PTX0_IN ))
 	},
-	{ PINMUX_CFG_REG("PYCR", 0xa405014a, 16, 2) {
+	{ PINMUX_CFG_REG("PYCR", 0xa405014a, 16, 2, GROUP(
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		KEYOUT5_IN5, PTY5_OUT, 0, PTY5_IN,
@@ -1455,9 +1455,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		KEYOUT3, PTY3_OUT, 0, PTY3_IN,
 		KEYOUT2, PTY2_OUT, 0, PTY2_IN,
 		KEYOUT1, PTY1_OUT, 0, 0,
-		KEYOUT0, PTY0_OUT, 0, PTY0_IN }
+		KEYOUT0, PTY0_OUT, 0, PTY0_IN ))
 	},
-	{ PINMUX_CFG_REG("PZCR", 0xa405014c, 16, 2) {
+	{ PINMUX_CFG_REG("PZCR", 0xa405014c, 16, 2, GROUP(
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		KEYIN4_IRQ7, 0, 0, PTZ5_IN,
@@ -1465,9 +1465,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		KEYIN2, 0, 0, PTZ3_IN,
 		KEYIN1, 0, 0, PTZ2_IN,
 		KEYIN0_IRQ6, 0, 0, PTZ1_IN,
-		0, 0, 0, 0 }
+		0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PSELA", 0xa405014e, 16, 1) {
+	{ PINMUX_CFG_REG("PSELA", 0xa405014e, 16, 1, GROUP(
 		PSA15_KEYIN0, PSA15_IRQ6,
 		PSA14_KEYIN4, PSA14_IRQ7,
 		0, 0,
@@ -1483,9 +1483,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0,
 		0, 0,
 		0, 0,
-		0, 0 }
+		0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PSELB", 0xa4050150, 16, 1) {
+	{ PINMUX_CFG_REG("PSELB", 0xa4050150, 16, 1, GROUP(
 		PSB15_SIOTXD, PSB15_SIUBOSLD,
 		PSB14_SIORXD, PSB14_SIUBISLD,
 		PSB13_SIOD, PSB13_SIUBILR,
@@ -1501,9 +1501,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PSB3_SIOF0_SS1, PSB3_TS_SPSYNC,
 		PSB2_SIOF0_SS2, PSB2_SIM_RST,
 		PSB1_SIUMCKA, PSB1_SIOF1_MCK,
-		PSB0_SIUAOSLD, PSB0_SIOF1_TXD }
+		PSB0_SIUAOSLD, PSB0_SIOF1_TXD ))
 	},
-	{ PINMUX_CFG_REG("PSELC", 0xa4050152, 16, 1) {
+	{ PINMUX_CFG_REG("PSELC", 0xa4050152, 16, 1, GROUP(
 		PSC15_SIUAISLD, PSC15_SIOF1_RXD,
 		PSC14_SIUAOBT, PSC14_SIOF1_SCK,
 		PSC13_SIUAOLR, PSC13_SIOF1_SYNC,
@@ -1519,9 +1519,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0,
 		0, 0,
 		0, 0,
-		PSC0_NAF, PSC0_VIO }
+		PSC0_NAF, PSC0_VIO ))
 	},
-	{ PINMUX_CFG_REG("PSELD", 0xa4050154, 16, 1) {
+	{ PINMUX_CFG_REG("PSELD", 0xa4050154, 16, 1, GROUP(
 		0, 0,
 		0, 0,
 		PSD13_VIO, PSD13_SCIF2,
@@ -1537,9 +1537,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PSD3_LCDVEPWC_LCDVCPWC, PSD3_LCDVEPWC2_LCDVCPWC2,
 		PSD2_LCDDON, PSD2_LCDDON2,
 		0, 0,
-		PSD0_LCDD19_LCDD0, PSD0_DV }
+		PSD0_LCDD19_LCDD0, PSD0_DV ))
 	},
-	{ PINMUX_CFG_REG("PSELE", 0xa4050156, 16, 1) {
+	{ PINMUX_CFG_REG("PSELE", 0xa4050156, 16, 1, GROUP(
 		PSE15_SIOF0_MCK_IRQ3, PSE15_SIM_D,
 		PSE14_SIOF0_TXD_IRDA_OUT, PSE14_SIM_CLK,
 		PSE13_SIOF0_RXD_IRDA_IN, PSE13_TS_SDAT,
@@ -1555,9 +1555,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PSE3_FLCTL, PSE3_VIO,
 		PSE2_NAF2, PSE2_VIO_D10,
 		PSE1_NAF1, PSE1_VIO_D9,
-		PSE0_NAF0, PSE0_VIO_D8 }
+		PSE0_NAF0, PSE0_VIO_D8 ))
 	},
-	{ PINMUX_CFG_REG("HIZCRA", 0xa4050158, 16, 1) {
+	{ PINMUX_CFG_REG("HIZCRA", 0xa4050158, 16, 1, GROUP(
 		0, 0,
 		HIZA14_KEYSC, HIZA14_HIZ,
 		0, 0,
@@ -1573,9 +1573,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0,
 		0, 0,
 		0, 0,
-		0, 0 }
+		0, 0 ))
 	},
-	{ PINMUX_CFG_REG("HIZCRB", 0xa405015a, 16, 1) {
+	{ PINMUX_CFG_REG("HIZCRB", 0xa405015a, 16, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -1591,9 +1591,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0,
 		0, 0,
 		HIZB1_VIO, HIZB1_HIZ,
-		HIZB0_VIO, HIZB0_HIZ }
+		HIZB0_VIO, HIZB0_HIZ ))
 	},
-	{ PINMUX_CFG_REG("HIZCRC", 0xa405015c, 16, 1) {
+	{ PINMUX_CFG_REG("HIZCRC", 0xa405015c, 16, 1, GROUP(
 		HIZC15_IRQ7, HIZC15_HIZ,
 		HIZC14_IRQ6, HIZC14_HIZ,
 		HIZC13_IRQ5, HIZC13_HIZ,
@@ -1609,9 +1609,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0,
 		0, 0,
 		0, 0,
-		0, 0 }
+		0, 0 ))
 	},
-	{ PINMUX_CFG_REG("MSELCRB", 0xa4050182, 16, 1) {
+	{ PINMUX_CFG_REG("MSELCRB", 0xa4050182, 16, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -1627,103 +1627,103 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0,
 		0, 0,
 		0, 0,
-		0, 0 }
+		0, 0 ))
 	},
 	{}
 };
 
 static const struct pinmux_data_reg pinmux_data_regs[] = {
-	{ PINMUX_DATA_REG("PADR", 0xa4050120, 8) {
+	{ PINMUX_DATA_REG("PADR", 0xa4050120, 8, GROUP(
 		PTA7_DATA, PTA6_DATA, PTA5_DATA, PTA4_DATA,
-		PTA3_DATA, PTA2_DATA, PTA1_DATA, PTA0_DATA }
+		PTA3_DATA, PTA2_DATA, PTA1_DATA, PTA0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PBDR", 0xa4050122, 8) {
+	{ PINMUX_DATA_REG("PBDR", 0xa4050122, 8, GROUP(
 		PTB7_DATA, PTB6_DATA, PTB5_DATA, PTB4_DATA,
-		PTB3_DATA, PTB2_DATA, PTB1_DATA, PTB0_DATA }
+		PTB3_DATA, PTB2_DATA, PTB1_DATA, PTB0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PCDR", 0xa4050124, 8) {
+	{ PINMUX_DATA_REG("PCDR", 0xa4050124, 8, GROUP(
 		PTC7_DATA, 0, PTC5_DATA, PTC4_DATA,
-		PTC3_DATA, PTC2_DATA, 0, PTC0_DATA }
+		PTC3_DATA, PTC2_DATA, 0, PTC0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PDDR", 0xa4050126, 8) {
+	{ PINMUX_DATA_REG("PDDR", 0xa4050126, 8, GROUP(
 		PTD7_DATA, PTD6_DATA, PTD5_DATA, PTD4_DATA,
-		PTD3_DATA, PTD2_DATA, PTD1_DATA, PTD0_DATA }
+		PTD3_DATA, PTD2_DATA, PTD1_DATA, PTD0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PEDR", 0xa4050128, 8) {
+	{ PINMUX_DATA_REG("PEDR", 0xa4050128, 8, GROUP(
 		PTE7_DATA, PTE6_DATA, PTE5_DATA, PTE4_DATA,
-		0, 0, PTE1_DATA, PTE0_DATA }
+		0, 0, PTE1_DATA, PTE0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PFDR", 0xa405012a, 8) {
+	{ PINMUX_DATA_REG("PFDR", 0xa405012a, 8, GROUP(
 		0, PTF6_DATA, PTF5_DATA, PTF4_DATA,
-		PTF3_DATA, PTF2_DATA, PTF1_DATA, PTF0_DATA }
+		PTF3_DATA, PTF2_DATA, PTF1_DATA, PTF0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PGDR", 0xa405012c, 8) {
+	{ PINMUX_DATA_REG("PGDR", 0xa405012c, 8, GROUP(
 		0, 0, 0, PTG4_DATA,
-		PTG3_DATA, PTG2_DATA, PTG1_DATA, PTG0_DATA }
+		PTG3_DATA, PTG2_DATA, PTG1_DATA, PTG0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PHDR", 0xa405012e, 8) {
+	{ PINMUX_DATA_REG("PHDR", 0xa405012e, 8, GROUP(
 		PTH7_DATA, PTH6_DATA, PTH5_DATA, PTH4_DATA,
-		PTH3_DATA, PTH2_DATA, PTH1_DATA, PTH0_DATA }
+		PTH3_DATA, PTH2_DATA, PTH1_DATA, PTH0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PJDR", 0xa4050130, 8) {
+	{ PINMUX_DATA_REG("PJDR", 0xa4050130, 8, GROUP(
 		PTJ7_DATA, PTJ6_DATA, PTJ5_DATA, 0,
-		0, 0, PTJ1_DATA, PTJ0_DATA }
+		0, 0, PTJ1_DATA, PTJ0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PKDR", 0xa4050132, 8) {
+	{ PINMUX_DATA_REG("PKDR", 0xa4050132, 8, GROUP(
 		0, PTK6_DATA, PTK5_DATA, PTK4_DATA,
-		PTK3_DATA, PTK2_DATA, PTK1_DATA, PTK0_DATA }
+		PTK3_DATA, PTK2_DATA, PTK1_DATA, PTK0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PLDR", 0xa4050134, 8) {
+	{ PINMUX_DATA_REG("PLDR", 0xa4050134, 8, GROUP(
 		PTL7_DATA, PTL6_DATA, PTL5_DATA, PTL4_DATA,
-		PTL3_DATA, PTL2_DATA, PTL1_DATA, PTL0_DATA }
+		PTL3_DATA, PTL2_DATA, PTL1_DATA, PTL0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PMDR", 0xa4050136, 8) {
+	{ PINMUX_DATA_REG("PMDR", 0xa4050136, 8, GROUP(
 		PTM7_DATA, PTM6_DATA, PTM5_DATA, PTM4_DATA,
-		PTM3_DATA, PTM2_DATA, PTM1_DATA, PTM0_DATA }
+		PTM3_DATA, PTM2_DATA, PTM1_DATA, PTM0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PNDR", 0xa4050138, 8) {
+	{ PINMUX_DATA_REG("PNDR", 0xa4050138, 8, GROUP(
 		PTN7_DATA, PTN6_DATA, PTN5_DATA, PTN4_DATA,
-		PTN3_DATA, PTN2_DATA, PTN1_DATA, PTN0_DATA }
+		PTN3_DATA, PTN2_DATA, PTN1_DATA, PTN0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PQDR", 0xa405013a, 8) {
+	{ PINMUX_DATA_REG("PQDR", 0xa405013a, 8, GROUP(
 		0, PTQ6_DATA, PTQ5_DATA, PTQ4_DATA,
-		PTQ3_DATA, PTQ2_DATA, PTQ1_DATA, PTQ0_DATA }
+		PTQ3_DATA, PTQ2_DATA, PTQ1_DATA, PTQ0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PRDR", 0xa405013c, 8) {
+	{ PINMUX_DATA_REG("PRDR", 0xa405013c, 8, GROUP(
 		0, 0, 0, PTR4_DATA,
-		PTR3_DATA, PTR2_DATA, PTR1_DATA, PTR0_DATA }
+		PTR3_DATA, PTR2_DATA, PTR1_DATA, PTR0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PSDR", 0xa405013e, 8) {
+	{ PINMUX_DATA_REG("PSDR", 0xa405013e, 8, GROUP(
 		0, 0, 0, PTS4_DATA,
-		PTS3_DATA, PTS2_DATA, PTS1_DATA, PTS0_DATA }
+		PTS3_DATA, PTS2_DATA, PTS1_DATA, PTS0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PTDR", 0xa4050160, 8) {
+	{ PINMUX_DATA_REG("PTDR", 0xa4050160, 8, GROUP(
 		0, 0, 0, PTT4_DATA,
-		PTT3_DATA, PTT2_DATA, PTT1_DATA, PTT0_DATA }
+		PTT3_DATA, PTT2_DATA, PTT1_DATA, PTT0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PUDR", 0xa4050162, 8) {
+	{ PINMUX_DATA_REG("PUDR", 0xa4050162, 8, GROUP(
 		0, 0, 0, PTU4_DATA,
-		PTU3_DATA, PTU2_DATA, PTU1_DATA, PTU0_DATA }
+		PTU3_DATA, PTU2_DATA, PTU1_DATA, PTU0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PVDR", 0xa4050164, 8) {
+	{ PINMUX_DATA_REG("PVDR", 0xa4050164, 8, GROUP(
 		0, 0, 0, PTV4_DATA,
-		PTV3_DATA, PTV2_DATA, PTV1_DATA, PTV0_DATA }
+		PTV3_DATA, PTV2_DATA, PTV1_DATA, PTV0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PWDR", 0xa4050166, 8) {
+	{ PINMUX_DATA_REG("PWDR", 0xa4050166, 8, GROUP(
 		0, PTW6_DATA, PTW5_DATA, PTW4_DATA,
-		PTW3_DATA, PTW2_DATA, PTW1_DATA, PTW0_DATA }
+		PTW3_DATA, PTW2_DATA, PTW1_DATA, PTW0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PXDR", 0xa4050168, 8) {
+	{ PINMUX_DATA_REG("PXDR", 0xa4050168, 8, GROUP(
 		0, PTX6_DATA, PTX5_DATA, PTX4_DATA,
-		PTX3_DATA, PTX2_DATA, PTX1_DATA, PTX0_DATA }
+		PTX3_DATA, PTX2_DATA, PTX1_DATA, PTX0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PYDR", 0xa405016a, 8) {
+	{ PINMUX_DATA_REG("PYDR", 0xa405016a, 8, GROUP(
 		0, PTY6_DATA, PTY5_DATA, PTY4_DATA,
-		PTY3_DATA, PTY2_DATA, PTY1_DATA, PTY0_DATA }
+		PTY3_DATA, PTY2_DATA, PTY1_DATA, PTY0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PZDR", 0xa405016c, 8) {
+	{ PINMUX_DATA_REG("PZDR", 0xa405016c, 8, GROUP(
 		0, 0, PTZ5_DATA, PTZ4_DATA,
-		PTZ3_DATA, PTZ2_DATA, PTZ1_DATA, PTZ0_DATA }
+		PTZ3_DATA, PTZ2_DATA, PTZ1_DATA, PTZ0_DATA ))
 	},
 	{ },
 };
diff --git a/drivers/pinctrl/sh-pfc/pfc-sh7723.c b/drivers/pinctrl/sh-pfc/pfc-sh7723.c
index 86f9a88726b7..6f08f527c010 100644
--- a/drivers/pinctrl/sh-pfc/pfc-sh7723.c
+++ b/drivers/pinctrl/sh-pfc/pfc-sh7723.c
@@ -1507,7 +1507,7 @@ static const struct pinmux_func pinmux_func_gpios[] = {
 };
 
 static const struct pinmux_cfg_reg pinmux_config_regs[] = {
-	{ PINMUX_CFG_REG("PACR", 0xa4050100, 16, 2) {
+	{ PINMUX_CFG_REG("PACR", 0xa4050100, 16, 2, GROUP(
 		PTA7_FN, PTA7_OUT, 0, PTA7_IN,
 		PTA6_FN, PTA6_OUT, 0, PTA6_IN,
 		PTA5_FN, PTA5_OUT, 0, PTA5_IN,
@@ -1515,9 +1515,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTA3_FN, PTA3_OUT, 0, PTA3_IN,
 		PTA2_FN, PTA2_OUT, 0, PTA2_IN,
 		PTA1_FN, PTA1_OUT, 0, PTA1_IN,
-		PTA0_FN, PTA0_OUT, 0, PTA0_IN }
+		PTA0_FN, PTA0_OUT, 0, PTA0_IN ))
 	},
-	{ PINMUX_CFG_REG("PBCR", 0xa4050102, 16, 2) {
+	{ PINMUX_CFG_REG("PBCR", 0xa4050102, 16, 2, GROUP(
 		PTB7_FN, PTB7_OUT, 0, PTB7_IN,
 		PTB6_FN, PTB6_OUT, 0, PTB6_IN,
 		PTB5_FN, PTB5_OUT, 0, PTB5_IN,
@@ -1525,9 +1525,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTB3_FN, PTB3_OUT, 0, PTB3_IN,
 		PTB2_FN, PTB2_OUT, 0, PTB2_IN,
 		PTB1_FN, PTB1_OUT, 0, PTB1_IN,
-		PTB0_FN, PTB0_OUT, 0, PTB0_IN }
+		PTB0_FN, PTB0_OUT, 0, PTB0_IN ))
 	},
-	{ PINMUX_CFG_REG("PCCR", 0xa4050104, 16, 2) {
+	{ PINMUX_CFG_REG("PCCR", 0xa4050104, 16, 2, GROUP(
 		PTC7_FN, PTC7_OUT, 0, PTC7_IN,
 		PTC6_FN, PTC6_OUT, 0, PTC6_IN,
 		PTC5_FN, PTC5_OUT, 0, PTC5_IN,
@@ -1535,9 +1535,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTC3_FN, PTC3_OUT, 0, PTC3_IN,
 		PTC2_FN, PTC2_OUT, 0, PTC2_IN,
 		PTC1_FN, PTC1_OUT, 0, PTC1_IN,
-		PTC0_FN, PTC0_OUT, 0, PTC0_IN }
+		PTC0_FN, PTC0_OUT, 0, PTC0_IN ))
 	},
-	{ PINMUX_CFG_REG("PDCR", 0xa4050106, 16, 2) {
+	{ PINMUX_CFG_REG("PDCR", 0xa4050106, 16, 2, GROUP(
 		PTD7_FN, PTD7_OUT, 0, PTD7_IN,
 		PTD6_FN, PTD6_OUT, 0, PTD6_IN,
 		PTD5_FN, PTD5_OUT, 0, PTD5_IN,
@@ -1545,9 +1545,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTD3_FN, PTD3_OUT, 0, PTD3_IN,
 		PTD2_FN, PTD2_OUT, 0, PTD2_IN,
 		PTD1_FN, PTD1_OUT, 0, PTD1_IN,
-		PTD0_FN, PTD0_OUT, 0, PTD0_IN }
+		PTD0_FN, PTD0_OUT, 0, PTD0_IN ))
 	},
-	{ PINMUX_CFG_REG("PECR", 0xa4050108, 16, 2) {
+	{ PINMUX_CFG_REG("PECR", 0xa4050108, 16, 2, GROUP(
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		PTE5_FN, PTE5_OUT, 0, PTE5_IN,
@@ -1555,9 +1555,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTE3_FN, PTE3_OUT, 0, PTE3_IN,
 		PTE2_FN, PTE2_OUT, 0, PTE2_IN,
 		PTE1_FN, PTE1_OUT, 0, PTE1_IN,
-		PTE0_FN, PTE0_OUT, 0, PTE0_IN }
+		PTE0_FN, PTE0_OUT, 0, PTE0_IN ))
 	},
-	{ PINMUX_CFG_REG("PFCR", 0xa405010a, 16, 2) {
+	{ PINMUX_CFG_REG("PFCR", 0xa405010a, 16, 2, GROUP(
 		PTF7_FN, PTF7_OUT, 0, PTF7_IN,
 		PTF6_FN, PTF6_OUT, 0, PTF6_IN,
 		PTF5_FN, PTF5_OUT, 0, PTF5_IN,
@@ -1565,9 +1565,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTF3_FN, PTF3_OUT, 0, PTF3_IN,
 		PTF2_FN, PTF2_OUT, 0, PTF2_IN,
 		PTF1_FN, PTF1_OUT, 0, PTF1_IN,
-		PTF0_FN, PTF0_OUT, 0, PTF0_IN }
+		PTF0_FN, PTF0_OUT, 0, PTF0_IN ))
 	},
-	{ PINMUX_CFG_REG("PGCR", 0xa405010c, 16, 2) {
+	{ PINMUX_CFG_REG("PGCR", 0xa405010c, 16, 2, GROUP(
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		PTG5_FN, PTG5_OUT, 0, 0,
@@ -1575,9 +1575,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTG3_FN, PTG3_OUT, 0, 0,
 		PTG2_FN, PTG2_OUT, 0, 0,
 		PTG1_FN, PTG1_OUT, 0, 0,
-		PTG0_FN, PTG0_OUT, 0, 0 }
+		PTG0_FN, PTG0_OUT, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PHCR", 0xa405010e, 16, 2) {
+	{ PINMUX_CFG_REG("PHCR", 0xa405010e, 16, 2, GROUP(
 		PTH7_FN, PTH7_OUT, 0, PTH7_IN,
 		PTH6_FN, PTH6_OUT, 0, PTH6_IN,
 		PTH5_FN, PTH5_OUT, 0, PTH5_IN,
@@ -1585,9 +1585,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTH3_FN, PTH3_OUT, 0, PTH3_IN,
 		PTH2_FN, PTH2_OUT, 0, PTH2_IN,
 		PTH1_FN, PTH1_OUT, 0, PTH1_IN,
-		PTH0_FN, PTH0_OUT, 0, PTH0_IN }
+		PTH0_FN, PTH0_OUT, 0, PTH0_IN ))
 	},
-	{ PINMUX_CFG_REG("PJCR", 0xa4050110, 16, 2) {
+	{ PINMUX_CFG_REG("PJCR", 0xa4050110, 16, 2, GROUP(
 		PTJ7_FN, PTJ7_OUT, 0, 0,
 		0, 0, 0, 0,
 		PTJ5_FN, PTJ5_OUT, 0, 0,
@@ -1595,9 +1595,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTJ3_FN, PTJ3_OUT, 0, PTJ3_IN,
 		PTJ2_FN, PTJ2_OUT, 0, PTJ2_IN,
 		PTJ1_FN, PTJ1_OUT, 0, PTJ1_IN,
-		PTJ0_FN, PTJ0_OUT, 0, PTJ0_IN }
+		PTJ0_FN, PTJ0_OUT, 0, PTJ0_IN ))
 	},
-	{ PINMUX_CFG_REG("PKCR", 0xa4050112, 16, 2) {
+	{ PINMUX_CFG_REG("PKCR", 0xa4050112, 16, 2, GROUP(
 		PTK7_FN, PTK7_OUT, 0, PTK7_IN,
 		PTK6_FN, PTK6_OUT, 0, PTK6_IN,
 		PTK5_FN, PTK5_OUT, 0, PTK5_IN,
@@ -1605,9 +1605,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTK3_FN, PTK3_OUT, 0, PTK3_IN,
 		PTK2_FN, PTK2_OUT, 0, PTK2_IN,
 		PTK1_FN, PTK1_OUT, 0, PTK1_IN,
-		PTK0_FN, PTK0_OUT, 0, PTK0_IN }
+		PTK0_FN, PTK0_OUT, 0, PTK0_IN ))
 	},
-	{ PINMUX_CFG_REG("PLCR", 0xa4050114, 16, 2) {
+	{ PINMUX_CFG_REG("PLCR", 0xa4050114, 16, 2, GROUP(
 		PTL7_FN, PTL7_OUT, 0, PTL7_IN,
 		PTL6_FN, PTL6_OUT, 0, PTL6_IN,
 		PTL5_FN, PTL5_OUT, 0, PTL5_IN,
@@ -1615,9 +1615,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTL3_FN, PTL3_OUT, 0, PTL3_IN,
 		PTL2_FN, PTL2_OUT, 0, PTL2_IN,
 		PTL1_FN, PTL1_OUT, 0, PTL1_IN,
-		PTL0_FN, PTL0_OUT, 0, PTL0_IN }
+		PTL0_FN, PTL0_OUT, 0, PTL0_IN ))
 	},
-	{ PINMUX_CFG_REG("PMCR", 0xa4050116, 16, 2) {
+	{ PINMUX_CFG_REG("PMCR", 0xa4050116, 16, 2, GROUP(
 		PTM7_FN, PTM7_OUT, 0, PTM7_IN,
 		PTM6_FN, PTM6_OUT, 0, PTM6_IN,
 		PTM5_FN, PTM5_OUT, 0, PTM5_IN,
@@ -1625,9 +1625,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTM3_FN, PTM3_OUT, 0, PTM3_IN,
 		PTM2_FN, PTM2_OUT, 0, PTM2_IN,
 		PTM1_FN, PTM1_OUT, 0, PTM1_IN,
-		PTM0_FN, PTM0_OUT, 0, PTM0_IN }
+		PTM0_FN, PTM0_OUT, 0, PTM0_IN ))
 	},
-	{ PINMUX_CFG_REG("PNCR", 0xa4050118, 16, 2) {
+	{ PINMUX_CFG_REG("PNCR", 0xa4050118, 16, 2, GROUP(
 		PTN7_FN, PTN7_OUT, 0, PTN7_IN,
 		PTN6_FN, PTN6_OUT, 0, PTN6_IN,
 		PTN5_FN, PTN5_OUT, 0, PTN5_IN,
@@ -1635,9 +1635,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTN3_FN, PTN3_OUT, 0, PTN3_IN,
 		PTN2_FN, PTN2_OUT, 0, PTN2_IN,
 		PTN1_FN, PTN1_OUT, 0, PTN1_IN,
-		PTN0_FN, PTN0_OUT, 0, PTN0_IN }
+		PTN0_FN, PTN0_OUT, 0, PTN0_IN ))
 	},
-	{ PINMUX_CFG_REG("PQCR", 0xa405011a, 16, 2) {
+	{ PINMUX_CFG_REG("PQCR", 0xa405011a, 16, 2, GROUP(
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		0, 0, 0, 0,
@@ -1645,9 +1645,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTQ3_FN, 0, 0, PTQ3_IN,
 		PTQ2_FN, 0, 0, PTQ2_IN,
 		PTQ1_FN, 0, 0, PTQ1_IN,
-		PTQ0_FN, 0, 0, PTQ0_IN }
+		PTQ0_FN, 0, 0, PTQ0_IN ))
 	},
-	{ PINMUX_CFG_REG("PRCR", 0xa405011c, 16, 2) {
+	{ PINMUX_CFG_REG("PRCR", 0xa405011c, 16, 2, GROUP(
 		PTR7_FN, PTR7_OUT, 0, PTR7_IN,
 		PTR6_FN, PTR6_OUT, 0, PTR6_IN,
 		PTR5_FN, PTR5_OUT, 0, PTR5_IN,
@@ -1655,9 +1655,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTR3_FN, 0, 0, PTR3_IN,
 		PTR2_FN, 0, 0, PTR2_IN,
 		PTR1_FN, PTR1_OUT, 0, PTR1_IN,
-		PTR0_FN, PTR0_OUT, 0, PTR0_IN }
+		PTR0_FN, PTR0_OUT, 0, PTR0_IN ))
 	},
-	{ PINMUX_CFG_REG("PSCR", 0xa405011e, 16, 2) {
+	{ PINMUX_CFG_REG("PSCR", 0xa405011e, 16, 2, GROUP(
 		PTS7_FN, PTS7_OUT, 0, PTS7_IN,
 		PTS6_FN, PTS6_OUT, 0, PTS6_IN,
 		PTS5_FN, PTS5_OUT, 0, PTS5_IN,
@@ -1665,9 +1665,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTS3_FN, PTS3_OUT, 0, PTS3_IN,
 		PTS2_FN, PTS2_OUT, 0, PTS2_IN,
 		PTS1_FN, PTS1_OUT, 0, PTS1_IN,
-		PTS0_FN, PTS0_OUT, 0, PTS0_IN }
+		PTS0_FN, PTS0_OUT, 0, PTS0_IN ))
 	},
-	{ PINMUX_CFG_REG("PTCR", 0xa4050140, 16, 2) {
+	{ PINMUX_CFG_REG("PTCR", 0xa4050140, 16, 2, GROUP(
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		PTT5_FN, PTT5_OUT, 0, PTT5_IN,
@@ -1675,9 +1675,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTT3_FN, PTT3_OUT, 0, PTT3_IN,
 		PTT2_FN, PTT2_OUT, 0, PTT2_IN,
 		PTT1_FN, PTT1_OUT, 0, PTT1_IN,
-		PTT0_FN, PTT0_OUT, 0, PTT0_IN }
+		PTT0_FN, PTT0_OUT, 0, PTT0_IN ))
 	},
-	{ PINMUX_CFG_REG("PUCR", 0xa4050142, 16, 2) {
+	{ PINMUX_CFG_REG("PUCR", 0xa4050142, 16, 2, GROUP(
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		PTU5_FN, PTU5_OUT, 0, PTU5_IN,
@@ -1685,9 +1685,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTU3_FN, PTU3_OUT, 0, PTU3_IN,
 		PTU2_FN, PTU2_OUT, 0, PTU2_IN,
 		PTU1_FN, PTU1_OUT, 0, PTU1_IN,
-		PTU0_FN, PTU0_OUT, 0, PTU0_IN }
+		PTU0_FN, PTU0_OUT, 0, PTU0_IN ))
 	},
-	{ PINMUX_CFG_REG("PVCR", 0xa4050144, 16, 2) {
+	{ PINMUX_CFG_REG("PVCR", 0xa4050144, 16, 2, GROUP(
 		PTV7_FN, PTV7_OUT, 0, PTV7_IN,
 		PTV6_FN, PTV6_OUT, 0, PTV6_IN,
 		PTV5_FN, PTV5_OUT, 0, PTV5_IN,
@@ -1695,9 +1695,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTV3_FN, PTV3_OUT, 0, PTV3_IN,
 		PTV2_FN, PTV2_OUT, 0, PTV2_IN,
 		PTV1_FN, PTV1_OUT, 0, PTV1_IN,
-		PTV0_FN, PTV0_OUT, 0, PTV0_IN }
+		PTV0_FN, PTV0_OUT, 0, PTV0_IN ))
 	},
-	{ PINMUX_CFG_REG("PWCR", 0xa4050146, 16, 2) {
+	{ PINMUX_CFG_REG("PWCR", 0xa4050146, 16, 2, GROUP(
 		PTW7_FN, PTW7_OUT, 0, PTW7_IN,
 		PTW6_FN, PTW6_OUT, 0, PTW6_IN,
 		PTW5_FN, PTW5_OUT, 0, PTW5_IN,
@@ -1705,9 +1705,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTW3_FN, PTW3_OUT, 0, PTW3_IN,
 		PTW2_FN, PTW2_OUT, 0, PTW2_IN,
 		PTW1_FN, PTW1_OUT, 0, PTW1_IN,
-		PTW0_FN, PTW0_OUT, 0, PTW0_IN }
+		PTW0_FN, PTW0_OUT, 0, PTW0_IN ))
 	},
-	{ PINMUX_CFG_REG("PXCR", 0xa4050148, 16, 2) {
+	{ PINMUX_CFG_REG("PXCR", 0xa4050148, 16, 2, GROUP(
 		PTX7_FN, PTX7_OUT, 0, PTX7_IN,
 		PTX6_FN, PTX6_OUT, 0, PTX6_IN,
 		PTX5_FN, PTX5_OUT, 0, PTX5_IN,
@@ -1715,9 +1715,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTX3_FN, PTX3_OUT, 0, PTX3_IN,
 		PTX2_FN, PTX2_OUT, 0, PTX2_IN,
 		PTX1_FN, PTX1_OUT, 0, PTX1_IN,
-		PTX0_FN, PTX0_OUT, 0, PTX0_IN }
+		PTX0_FN, PTX0_OUT, 0, PTX0_IN ))
 	},
-	{ PINMUX_CFG_REG("PYCR", 0xa405014a, 16, 2) {
+	{ PINMUX_CFG_REG("PYCR", 0xa405014a, 16, 2, GROUP(
 		PTY7_FN, PTY7_OUT, 0, PTY7_IN,
 		PTY6_FN, PTY6_OUT, 0, PTY6_IN,
 		PTY5_FN, PTY5_OUT, 0, PTY5_IN,
@@ -1725,9 +1725,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTY3_FN, PTY3_OUT, 0, PTY3_IN,
 		PTY2_FN, PTY2_OUT, 0, PTY2_IN,
 		PTY1_FN, PTY1_OUT, 0, PTY1_IN,
-		PTY0_FN, PTY0_OUT, 0, PTY0_IN }
+		PTY0_FN, PTY0_OUT, 0, PTY0_IN ))
 	},
-	{ PINMUX_CFG_REG("PZCR", 0xa405014c, 16, 2) {
+	{ PINMUX_CFG_REG("PZCR", 0xa405014c, 16, 2, GROUP(
 		PTZ7_FN, PTZ7_OUT, 0, PTZ7_IN,
 		PTZ6_FN, PTZ6_OUT, 0, PTZ6_IN,
 		PTZ5_FN, PTZ5_OUT, 0, PTZ5_IN,
@@ -1735,9 +1735,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTZ3_FN, PTZ3_OUT, 0, PTZ3_IN,
 		PTZ2_FN, PTZ2_OUT, 0, PTZ2_IN,
 		PTZ1_FN, PTZ1_OUT, 0, PTZ1_IN,
-		PTZ0_FN, PTZ0_OUT, 0, PTZ0_IN }
+		PTZ0_FN, PTZ0_OUT, 0, PTZ0_IN ))
 	},
-	{ PINMUX_CFG_REG("PSELA", 0xa405014e, 16, 2) {
+	{ PINMUX_CFG_REG("PSELA", 0xa405014e, 16, 2, GROUP(
 		PSA15_PSA14_FN1, PSA15_PSA14_FN2, 0, 0,
 		PSA13_PSA12_FN1, PSA13_PSA12_FN2, 0, 0,
 		PSA11_PSA10_FN1, PSA11_PSA10_FN2, 0, 0,
@@ -1745,9 +1745,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0,
 		PSA5_PSA4_FN1, PSA5_PSA4_FN2, PSA5_PSA4_FN3, 0,
 		PSA3_PSA2_FN1, PSA3_PSA2_FN2, 0, 0,
-		0, 0, 0, 0 }
+		0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PSELB", 0xa4050150, 16, 2) {
+	{ PINMUX_CFG_REG("PSELB", 0xa4050150, 16, 2, GROUP(
 		PSB15_PSB14_FN1, PSB15_PSB14_FN2, 0, 0,
 		PSB13_PSB12_LCDC_RGB, PSB13_PSB12_LCDC_SYS, 0, 0,
 		0, 0, 0, 0,
@@ -1755,9 +1755,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PSB7_PSB6_FN1, PSB7_PSB6_FN2, 0, 0,
 		PSB5_PSB4_FN1, PSB5_PSB4_FN2, 0, 0,
 		PSB3_PSB2_FN1, PSB3_PSB2_FN2, 0, 0,
-		0, 0, 0, 0 }
+		0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PSELC", 0xa4050152, 16, 2) {
+	{ PINMUX_CFG_REG("PSELC", 0xa4050152, 16, 2, GROUP(
 		PSC15_PSC14_FN1, PSC15_PSC14_FN2, 0, 0,
 		PSC13_PSC12_FN1, PSC13_PSC12_FN2, 0, 0,
 		PSC11_PSC10_FN1, PSC11_PSC10_FN2, PSC11_PSC10_FN3, 0,
@@ -1765,9 +1765,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PSC7_PSC6_FN1, PSC7_PSC6_FN2, PSC7_PSC6_FN3, 0,
 		0, 0, 0, 0,
 		0, 0, 0, 0,
-		0, 0, 0, 0 }
+		0, 0, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PSELD", 0xa4050154, 16, 2) {
+	{ PINMUX_CFG_REG("PSELD", 0xa4050154, 16, 2, GROUP(
 		PSD15_PSD14_FN1, PSD15_PSD14_FN2, 0, 0,
 		PSD13_PSD12_FN1, PSD13_PSD12_FN2, 0, 0,
 		PSD11_PSD10_FN1, PSD11_PSD10_FN2, PSD11_PSD10_FN3, 0,
@@ -1775,103 +1775,103 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PSD7_PSD6_FN1, PSD7_PSD6_FN2, 0, 0,
 		PSD5_PSD4_FN1, PSD5_PSD4_FN2, 0, 0,
 		PSD3_PSD2_FN1, PSD3_PSD2_FN2, 0, 0,
-		PSD1_PSD0_FN1, PSD1_PSD0_FN2, 0, 0 }
+		PSD1_PSD0_FN1, PSD1_PSD0_FN2, 0, 0 ))
 	},
 	{}
 };
 
 static const struct pinmux_data_reg pinmux_data_regs[] = {
-	{ PINMUX_DATA_REG("PADR", 0xa4050120, 8) {
+	{ PINMUX_DATA_REG("PADR", 0xa4050120, 8, GROUP(
 		PTA7_DATA, PTA6_DATA, PTA5_DATA, PTA4_DATA,
-		PTA3_DATA, PTA2_DATA, PTA1_DATA, PTA0_DATA }
+		PTA3_DATA, PTA2_DATA, PTA1_DATA, PTA0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PBDR", 0xa4050122, 8) {
+	{ PINMUX_DATA_REG("PBDR", 0xa4050122, 8, GROUP(
 		PTB7_DATA, PTB6_DATA, PTB5_DATA, PTB4_DATA,
-		PTB3_DATA, PTB2_DATA, PTB1_DATA, PTB0_DATA }
+		PTB3_DATA, PTB2_DATA, PTB1_DATA, PTB0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PCDR", 0xa4050124, 8) {
+	{ PINMUX_DATA_REG("PCDR", 0xa4050124, 8, GROUP(
 		PTC7_DATA, PTC6_DATA, PTC5_DATA, PTC4_DATA,
-		PTC3_DATA, PTC2_DATA, PTC1_DATA, PTC0_DATA }
+		PTC3_DATA, PTC2_DATA, PTC1_DATA, PTC0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PDDR", 0xa4050126, 8) {
+	{ PINMUX_DATA_REG("PDDR", 0xa4050126, 8, GROUP(
 		PTD7_DATA, PTD6_DATA, PTD5_DATA, PTD4_DATA,
-		PTD3_DATA, PTD2_DATA, PTD1_DATA, PTD0_DATA }
+		PTD3_DATA, PTD2_DATA, PTD1_DATA, PTD0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PEDR", 0xa4050128, 8) {
+	{ PINMUX_DATA_REG("PEDR", 0xa4050128, 8, GROUP(
 		0, 0, PTE5_DATA, PTE4_DATA,
-		PTE3_DATA, PTE2_DATA, PTE1_DATA, PTE0_DATA }
+		PTE3_DATA, PTE2_DATA, PTE1_DATA, PTE0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PFDR", 0xa405012a, 8) {
+	{ PINMUX_DATA_REG("PFDR", 0xa405012a, 8, GROUP(
 		PTF7_DATA, PTF6_DATA, PTF5_DATA, PTF4_DATA,
-		PTF3_DATA, PTF2_DATA, PTF1_DATA, PTF0_DATA }
+		PTF3_DATA, PTF2_DATA, PTF1_DATA, PTF0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PGDR", 0xa405012c, 8) {
+	{ PINMUX_DATA_REG("PGDR", 0xa405012c, 8, GROUP(
 		0, 0, PTG5_DATA, PTG4_DATA,
-		PTG3_DATA, PTG2_DATA, PTG1_DATA, PTG0_DATA }
+		PTG3_DATA, PTG2_DATA, PTG1_DATA, PTG0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PHDR", 0xa405012e, 8) {
+	{ PINMUX_DATA_REG("PHDR", 0xa405012e, 8, GROUP(
 		PTH7_DATA, PTH6_DATA, PTH5_DATA, PTH4_DATA,
-		PTH3_DATA, PTH2_DATA, PTH1_DATA, PTH0_DATA }
+		PTH3_DATA, PTH2_DATA, PTH1_DATA, PTH0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PJDR", 0xa4050130, 8) {
+	{ PINMUX_DATA_REG("PJDR", 0xa4050130, 8, GROUP(
 		PTJ7_DATA, 0, PTJ5_DATA, 0,
-		PTJ3_DATA, PTJ2_DATA, PTJ1_DATA, PTJ0_DATA }
+		PTJ3_DATA, PTJ2_DATA, PTJ1_DATA, PTJ0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PKDR", 0xa4050132, 8) {
+	{ PINMUX_DATA_REG("PKDR", 0xa4050132, 8, GROUP(
 		PTK7_DATA, PTK6_DATA, PTK5_DATA, PTK4_DATA,
-		PTK3_DATA, PTK2_DATA, PTK1_DATA, PTK0_DATA }
+		PTK3_DATA, PTK2_DATA, PTK1_DATA, PTK0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PLDR", 0xa4050134, 8) {
+	{ PINMUX_DATA_REG("PLDR", 0xa4050134, 8, GROUP(
 		PTL7_DATA, PTL6_DATA, PTL5_DATA, PTL4_DATA,
-		PTL3_DATA, PTL2_DATA, PTL1_DATA, PTL0_DATA }
+		PTL3_DATA, PTL2_DATA, PTL1_DATA, PTL0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PMDR", 0xa4050136, 8) {
+	{ PINMUX_DATA_REG("PMDR", 0xa4050136, 8, GROUP(
 		PTM7_DATA, PTM6_DATA, PTM5_DATA, PTM4_DATA,
-		PTM3_DATA, PTM2_DATA, PTM1_DATA, PTM0_DATA }
+		PTM3_DATA, PTM2_DATA, PTM1_DATA, PTM0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PNDR", 0xa4050138, 8) {
+	{ PINMUX_DATA_REG("PNDR", 0xa4050138, 8, GROUP(
 		PTN7_DATA, PTN6_DATA, PTN5_DATA, PTN4_DATA,
-		PTN3_DATA, PTN2_DATA, PTN1_DATA, PTN0_DATA }
+		PTN3_DATA, PTN2_DATA, PTN1_DATA, PTN0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PQDR", 0xa405013a, 8) {
+	{ PINMUX_DATA_REG("PQDR", 0xa405013a, 8, GROUP(
 		0, 0, 0, 0,
-		PTQ3_DATA, PTQ2_DATA, PTQ1_DATA, PTQ0_DATA }
+		PTQ3_DATA, PTQ2_DATA, PTQ1_DATA, PTQ0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PRDR", 0xa405013c, 8) {
+	{ PINMUX_DATA_REG("PRDR", 0xa405013c, 8, GROUP(
 		PTR7_DATA, PTR6_DATA, PTR5_DATA, PTR4_DATA,
-		PTR3_DATA, PTR2_DATA, PTR1_DATA, PTR0_DATA }
+		PTR3_DATA, PTR2_DATA, PTR1_DATA, PTR0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PSDR", 0xa405013e, 8) {
+	{ PINMUX_DATA_REG("PSDR", 0xa405013e, 8, GROUP(
 		PTS7_DATA, PTS6_DATA, PTS5_DATA, PTS4_DATA,
-		PTS3_DATA, PTS2_DATA, PTS1_DATA, PTS0_DATA }
+		PTS3_DATA, PTS2_DATA, PTS1_DATA, PTS0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PTDR", 0xa4050160, 8) {
+	{ PINMUX_DATA_REG("PTDR", 0xa4050160, 8, GROUP(
 		0, 0, PTT5_DATA, PTT4_DATA,
-		PTT3_DATA, PTT2_DATA, PTT1_DATA, PTT0_DATA }
+		PTT3_DATA, PTT2_DATA, PTT1_DATA, PTT0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PUDR", 0xa4050162, 8) {
+	{ PINMUX_DATA_REG("PUDR", 0xa4050162, 8, GROUP(
 		0, 0, PTU5_DATA, PTU4_DATA,
-		PTU3_DATA, PTU2_DATA, PTU1_DATA, PTU0_DATA }
+		PTU3_DATA, PTU2_DATA, PTU1_DATA, PTU0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PVDR", 0xa4050164, 8) {
+	{ PINMUX_DATA_REG("PVDR", 0xa4050164, 8, GROUP(
 		PTV7_DATA, PTV6_DATA, PTV5_DATA, PTV4_DATA,
-		PTV3_DATA, PTV2_DATA, PTV1_DATA, PTV0_DATA }
+		PTV3_DATA, PTV2_DATA, PTV1_DATA, PTV0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PWDR", 0xa4050166, 8) {
+	{ PINMUX_DATA_REG("PWDR", 0xa4050166, 8, GROUP(
 		PTW7_DATA, PTW6_DATA, PTW5_DATA, PTW4_DATA,
-		PTW3_DATA, PTW2_DATA, PTW1_DATA, PTW0_DATA }
+		PTW3_DATA, PTW2_DATA, PTW1_DATA, PTW0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PXDR", 0xa4050168, 8) {
+	{ PINMUX_DATA_REG("PXDR", 0xa4050168, 8, GROUP(
 		PTX7_DATA, PTX6_DATA, PTX5_DATA, PTX4_DATA,
-		PTX3_DATA, PTX2_DATA, PTX1_DATA, PTX0_DATA }
+		PTX3_DATA, PTX2_DATA, PTX1_DATA, PTX0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PYDR", 0xa405016a, 8) {
+	{ PINMUX_DATA_REG("PYDR", 0xa405016a, 8, GROUP(
 		PTY7_DATA, PTY6_DATA, PTY5_DATA, PTY4_DATA,
-		PTY3_DATA, PTY2_DATA, PTY1_DATA, PTY0_DATA }
+		PTY3_DATA, PTY2_DATA, PTY1_DATA, PTY0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PZDR", 0xa405016c, 8) {
+	{ PINMUX_DATA_REG("PZDR", 0xa405016c, 8, GROUP(
 		PTZ7_DATA, PTZ6_DATA, PTZ5_DATA, PTZ4_DATA,
-		PTZ3_DATA, PTZ2_DATA, PTZ1_DATA, PTZ0_DATA }
+		PTZ3_DATA, PTZ2_DATA, PTZ1_DATA, PTZ0_DATA ))
 	},
 	{ },
 };
diff --git a/drivers/pinctrl/sh-pfc/pfc-sh7724.c b/drivers/pinctrl/sh-pfc/pfc-sh7724.c
index 2cc4aa7df613..7a18afecda2c 100644
--- a/drivers/pinctrl/sh-pfc/pfc-sh7724.c
+++ b/drivers/pinctrl/sh-pfc/pfc-sh7724.c
@@ -1739,7 +1739,7 @@ static const struct pinmux_func pinmux_func_gpios[] = {
 };
 
 static const struct pinmux_cfg_reg pinmux_config_regs[] = {
-	{ PINMUX_CFG_REG("PACR", 0xa4050100, 16, 2) {
+	{ PINMUX_CFG_REG("PACR", 0xa4050100, 16, 2, GROUP(
 		PTA7_FN, PTA7_OUT, 0, PTA7_IN,
 		PTA6_FN, PTA6_OUT, 0, PTA6_IN,
 		PTA5_FN, PTA5_OUT, 0, PTA5_IN,
@@ -1747,9 +1747,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTA3_FN, PTA3_OUT, 0, PTA3_IN,
 		PTA2_FN, PTA2_OUT, 0, PTA2_IN,
 		PTA1_FN, PTA1_OUT, 0, PTA1_IN,
-		PTA0_FN, PTA0_OUT, 0, PTA0_IN }
+		PTA0_FN, PTA0_OUT, 0, PTA0_IN ))
 	},
-	{ PINMUX_CFG_REG("PBCR", 0xa4050102, 16, 2) {
+	{ PINMUX_CFG_REG("PBCR", 0xa4050102, 16, 2, GROUP(
 		PTB7_FN, PTB7_OUT, 0, PTB7_IN,
 		PTB6_FN, PTB6_OUT, 0, PTB6_IN,
 		PTB5_FN, PTB5_OUT, 0, PTB5_IN,
@@ -1757,9 +1757,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTB3_FN, PTB3_OUT, 0, PTB3_IN,
 		PTB2_FN, PTB2_OUT, 0, PTB2_IN,
 		PTB1_FN, PTB1_OUT, 0, PTB1_IN,
-		PTB0_FN, PTB0_OUT, 0, PTB0_IN }
+		PTB0_FN, PTB0_OUT, 0, PTB0_IN ))
 	},
-	{ PINMUX_CFG_REG("PCCR", 0xa4050104, 16, 2) {
+	{ PINMUX_CFG_REG("PCCR", 0xa4050104, 16, 2, GROUP(
 		PTC7_FN, PTC7_OUT, 0, PTC7_IN,
 		PTC6_FN, PTC6_OUT, 0, PTC6_IN,
 		PTC5_FN, PTC5_OUT, 0, PTC5_IN,
@@ -1767,9 +1767,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTC3_FN, PTC3_OUT, 0, PTC3_IN,
 		PTC2_FN, PTC2_OUT, 0, PTC2_IN,
 		PTC1_FN, PTC1_OUT, 0, PTC1_IN,
-		PTC0_FN, PTC0_OUT, 0, PTC0_IN }
+		PTC0_FN, PTC0_OUT, 0, PTC0_IN ))
 	},
-	{ PINMUX_CFG_REG("PDCR", 0xa4050106, 16, 2) {
+	{ PINMUX_CFG_REG("PDCR", 0xa4050106, 16, 2, GROUP(
 		PTD7_FN, PTD7_OUT, 0, PTD7_IN,
 		PTD6_FN, PTD6_OUT, 0, PTD6_IN,
 		PTD5_FN, PTD5_OUT, 0, PTD5_IN,
@@ -1777,9 +1777,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTD3_FN, PTD3_OUT, 0, PTD3_IN,
 		PTD2_FN, PTD2_OUT, 0, PTD2_IN,
 		PTD1_FN, PTD1_OUT, 0, PTD1_IN,
-		PTD0_FN, PTD0_OUT, 0, PTD0_IN }
+		PTD0_FN, PTD0_OUT, 0, PTD0_IN ))
 	},
-	{ PINMUX_CFG_REG("PECR", 0xa4050108, 16, 2) {
+	{ PINMUX_CFG_REG("PECR", 0xa4050108, 16, 2, GROUP(
 		PTE7_FN, PTE7_OUT, 0, PTE7_IN,
 		PTE6_FN, PTE6_OUT, 0, PTE6_IN,
 		PTE5_FN, PTE5_OUT, 0, PTE5_IN,
@@ -1787,9 +1787,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTE3_FN, PTE3_OUT, 0, PTE3_IN,
 		PTE2_FN, PTE2_OUT, 0, PTE2_IN,
 		PTE1_FN, PTE1_OUT, 0, PTE1_IN,
-		PTE0_FN, PTE0_OUT, 0, PTE0_IN }
+		PTE0_FN, PTE0_OUT, 0, PTE0_IN ))
 	},
-	{ PINMUX_CFG_REG("PFCR", 0xa405010a, 16, 2) {
+	{ PINMUX_CFG_REG("PFCR", 0xa405010a, 16, 2, GROUP(
 		PTF7_FN, PTF7_OUT, 0, PTF7_IN,
 		PTF6_FN, PTF6_OUT, 0, PTF6_IN,
 		PTF5_FN, PTF5_OUT, 0, PTF5_IN,
@@ -1797,9 +1797,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTF3_FN, PTF3_OUT, 0, PTF3_IN,
 		PTF2_FN, PTF2_OUT, 0, PTF2_IN,
 		PTF1_FN, PTF1_OUT, 0, PTF1_IN,
-		PTF0_FN, PTF0_OUT, 0, PTF0_IN }
+		PTF0_FN, PTF0_OUT, 0, PTF0_IN ))
 	},
-	{ PINMUX_CFG_REG("PGCR", 0xa405010c, 16, 2) {
+	{ PINMUX_CFG_REG("PGCR", 0xa405010c, 16, 2, GROUP(
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		PTG5_FN, PTG5_OUT, 0, 0,
@@ -1807,9 +1807,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTG3_FN, PTG3_OUT, 0, 0,
 		PTG2_FN, PTG2_OUT, 0, 0,
 		PTG1_FN, PTG1_OUT, 0, 0,
-		PTG0_FN, PTG0_OUT, 0, 0 }
+		PTG0_FN, PTG0_OUT, 0, 0 ))
 	},
-	{ PINMUX_CFG_REG("PHCR", 0xa405010e, 16, 2) {
+	{ PINMUX_CFG_REG("PHCR", 0xa405010e, 16, 2, GROUP(
 		PTH7_FN, PTH7_OUT, 0, PTH7_IN,
 		PTH6_FN, PTH6_OUT, 0, PTH6_IN,
 		PTH5_FN, PTH5_OUT, 0, PTH5_IN,
@@ -1817,9 +1817,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTH3_FN, PTH3_OUT, 0, PTH3_IN,
 		PTH2_FN, PTH2_OUT, 0, PTH2_IN,
 		PTH1_FN, PTH1_OUT, 0, PTH1_IN,
-		PTH0_FN, PTH0_OUT, 0, PTH0_IN }
+		PTH0_FN, PTH0_OUT, 0, PTH0_IN ))
 	},
-	{ PINMUX_CFG_REG("PJCR", 0xa4050110, 16, 2) {
+	{ PINMUX_CFG_REG("PJCR", 0xa4050110, 16, 2, GROUP(
 		PTJ7_FN, PTJ7_OUT, 0, 0,
 		PTJ6_FN, PTJ6_OUT, 0, 0,
 		PTJ5_FN, PTJ5_OUT, 0, 0,
@@ -1827,9 +1827,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTJ3_FN, PTJ3_OUT, 0, PTJ3_IN,
 		PTJ2_FN, PTJ2_OUT, 0, PTJ2_IN,
 		PTJ1_FN, PTJ1_OUT, 0, PTJ1_IN,
-		PTJ0_FN, PTJ0_OUT, 0, PTJ0_IN }
+		PTJ0_FN, PTJ0_OUT, 0, PTJ0_IN ))
 	},
-	{ PINMUX_CFG_REG("PKCR", 0xa4050112, 16, 2) {
+	{ PINMUX_CFG_REG("PKCR", 0xa4050112, 16, 2, GROUP(
 		PTK7_FN, PTK7_OUT, 0, PTK7_IN,
 		PTK6_FN, PTK6_OUT, 0, PTK6_IN,
 		PTK5_FN, PTK5_OUT, 0, PTK5_IN,
@@ -1837,9 +1837,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTK3_FN, PTK3_OUT, 0, PTK3_IN,
 		PTK2_FN, PTK2_OUT, 0, PTK2_IN,
 		PTK1_FN, PTK1_OUT, 0, PTK1_IN,
-		PTK0_FN, PTK0_OUT, 0, PTK0_IN }
+		PTK0_FN, PTK0_OUT, 0, PTK0_IN ))
 	},
-	{ PINMUX_CFG_REG("PLCR", 0xa4050114, 16, 2) {
+	{ PINMUX_CFG_REG("PLCR", 0xa4050114, 16, 2, GROUP(
 		PTL7_FN, PTL7_OUT, 0, PTL7_IN,
 		PTL6_FN, PTL6_OUT, 0, PTL6_IN,
 		PTL5_FN, PTL5_OUT, 0, PTL5_IN,
@@ -1847,9 +1847,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTL3_FN, PTL3_OUT, 0, PTL3_IN,
 		PTL2_FN, PTL2_OUT, 0, PTL2_IN,
 		PTL1_FN, PTL1_OUT, 0, PTL1_IN,
-		PTL0_FN, PTL0_OUT, 0, PTL0_IN }
+		PTL0_FN, PTL0_OUT, 0, PTL0_IN ))
 	},
-	{ PINMUX_CFG_REG("PMCR", 0xa4050116, 16, 2) {
+	{ PINMUX_CFG_REG("PMCR", 0xa4050116, 16, 2, GROUP(
 		PTM7_FN, PTM7_OUT, 0, PTM7_IN,
 		PTM6_FN, PTM6_OUT, 0, PTM6_IN,
 		PTM5_FN, PTM5_OUT, 0, PTM5_IN,
@@ -1857,9 +1857,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTM3_FN, PTM3_OUT, 0, PTM3_IN,
 		PTM2_FN, PTM2_OUT, 0, PTM2_IN,
 		PTM1_FN, PTM1_OUT, 0, PTM1_IN,
-		PTM0_FN, PTM0_OUT, 0, PTM0_IN }
+		PTM0_FN, PTM0_OUT, 0, PTM0_IN ))
 	},
-	{ PINMUX_CFG_REG("PNCR", 0xa4050118, 16, 2) {
+	{ PINMUX_CFG_REG("PNCR", 0xa4050118, 16, 2, GROUP(
 		PTN7_FN, PTN7_OUT, 0, PTN7_IN,
 		PTN6_FN, PTN6_OUT, 0, PTN6_IN,
 		PTN5_FN, PTN5_OUT, 0, PTN5_IN,
@@ -1867,9 +1867,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTN3_FN, PTN3_OUT, 0, PTN3_IN,
 		PTN2_FN, PTN2_OUT, 0, PTN2_IN,
 		PTN1_FN, PTN1_OUT, 0, PTN1_IN,
-		PTN0_FN, PTN0_OUT, 0, PTN0_IN }
+		PTN0_FN, PTN0_OUT, 0, PTN0_IN ))
 	},
-	{ PINMUX_CFG_REG("PQCR", 0xa405011a, 16, 2) {
+	{ PINMUX_CFG_REG("PQCR", 0xa405011a, 16, 2, GROUP(
 		PTQ7_FN, PTQ7_OUT, 0, PTQ7_IN,
 		PTQ6_FN, PTQ6_OUT, 0, PTQ6_IN,
 		PTQ5_FN, PTQ5_OUT, 0, PTQ5_IN,
@@ -1877,9 +1877,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTQ3_FN, PTQ3_OUT, 0, PTQ3_IN,
 		PTQ2_FN, PTQ2_OUT, 0, PTQ2_IN,
 		PTQ1_FN, PTQ1_OUT, 0, PTQ1_IN,
-		PTQ0_FN, PTQ0_OUT, 0, PTQ0_IN }
+		PTQ0_FN, PTQ0_OUT, 0, PTQ0_IN ))
 	},
-	{ PINMUX_CFG_REG("PRCR", 0xa405011c, 16, 2) {
+	{ PINMUX_CFG_REG("PRCR", 0xa405011c, 16, 2, GROUP(
 		PTR7_FN, PTR7_OUT, 0, PTR7_IN,
 		PTR6_FN, PTR6_OUT, 0, PTR6_IN,
 		PTR5_FN, PTR5_OUT, 0, PTR5_IN,
@@ -1887,9 +1887,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTR3_FN, 0,        0, PTR3_IN,
 		PTR2_FN, 0,        0, PTR2_IN,
 		PTR1_FN, PTR1_OUT, 0, PTR1_IN,
-		PTR0_FN, PTR0_OUT, 0, PTR0_IN }
+		PTR0_FN, PTR0_OUT, 0, PTR0_IN ))
 	},
-	{ PINMUX_CFG_REG("PSCR", 0xa405011e, 16, 2) {
+	{ PINMUX_CFG_REG("PSCR", 0xa405011e, 16, 2, GROUP(
 		0, 0, 0, 0,
 		PTS6_FN, PTS6_OUT, 0, PTS6_IN,
 		PTS5_FN, PTS5_OUT, 0, PTS5_IN,
@@ -1897,9 +1897,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTS3_FN, PTS3_OUT, 0, PTS3_IN,
 		PTS2_FN, PTS2_OUT, 0, PTS2_IN,
 		PTS1_FN, PTS1_OUT, 0, PTS1_IN,
-		PTS0_FN, PTS0_OUT, 0, PTS0_IN }
+		PTS0_FN, PTS0_OUT, 0, PTS0_IN ))
 	},
-	{ PINMUX_CFG_REG("PTCR", 0xa4050140, 16, 2) {
+	{ PINMUX_CFG_REG("PTCR", 0xa4050140, 16, 2, GROUP(
 		PTT7_FN, PTT7_OUT, 0, PTT7_IN,
 		PTT6_FN, PTT6_OUT, 0, PTT6_IN,
 		PTT5_FN, PTT5_OUT, 0, PTT5_IN,
@@ -1907,9 +1907,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTT3_FN, PTT3_OUT, 0, PTT3_IN,
 		PTT2_FN, PTT2_OUT, 0, PTT2_IN,
 		PTT1_FN, PTT1_OUT, 0, PTT1_IN,
-		PTT0_FN, PTT0_OUT, 0, PTT0_IN }
+		PTT0_FN, PTT0_OUT, 0, PTT0_IN ))
 	},
-	{ PINMUX_CFG_REG("PUCR", 0xa4050142, 16, 2) {
+	{ PINMUX_CFG_REG("PUCR", 0xa4050142, 16, 2, GROUP(
 		PTU7_FN, PTU7_OUT, 0, PTU7_IN,
 		PTU6_FN, PTU6_OUT, 0, PTU6_IN,
 		PTU5_FN, PTU5_OUT, 0, PTU5_IN,
@@ -1917,9 +1917,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTU3_FN, PTU3_OUT, 0, PTU3_IN,
 		PTU2_FN, PTU2_OUT, 0, PTU2_IN,
 		PTU1_FN, PTU1_OUT, 0, PTU1_IN,
-		PTU0_FN, PTU0_OUT, 0, PTU0_IN }
+		PTU0_FN, PTU0_OUT, 0, PTU0_IN ))
 	},
-	{ PINMUX_CFG_REG("PVCR", 0xa4050144, 16, 2) {
+	{ PINMUX_CFG_REG("PVCR", 0xa4050144, 16, 2, GROUP(
 		PTV7_FN, PTV7_OUT, 0, PTV7_IN,
 		PTV6_FN, PTV6_OUT, 0, PTV6_IN,
 		PTV5_FN, PTV5_OUT, 0, PTV5_IN,
@@ -1927,9 +1927,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTV3_FN, PTV3_OUT, 0, PTV3_IN,
 		PTV2_FN, PTV2_OUT, 0, PTV2_IN,
 		PTV1_FN, PTV1_OUT, 0, PTV1_IN,
-		PTV0_FN, PTV0_OUT, 0, PTV0_IN }
+		PTV0_FN, PTV0_OUT, 0, PTV0_IN ))
 	},
-	{ PINMUX_CFG_REG("PWCR", 0xa4050146, 16, 2) {
+	{ PINMUX_CFG_REG("PWCR", 0xa4050146, 16, 2, GROUP(
 		PTW7_FN, PTW7_OUT, 0, PTW7_IN,
 		PTW6_FN, PTW6_OUT, 0, PTW6_IN,
 		PTW5_FN, PTW5_OUT, 0, PTW5_IN,
@@ -1937,9 +1937,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTW3_FN, PTW3_OUT, 0, PTW3_IN,
 		PTW2_FN, PTW2_OUT, 0, PTW2_IN,
 		PTW1_FN, PTW1_OUT, 0, PTW1_IN,
-		PTW0_FN, PTW0_OUT, 0, PTW0_IN }
+		PTW0_FN, PTW0_OUT, 0, PTW0_IN ))
 	},
-	{ PINMUX_CFG_REG("PXCR", 0xa4050148, 16, 2) {
+	{ PINMUX_CFG_REG("PXCR", 0xa4050148, 16, 2, GROUP(
 		PTX7_FN, PTX7_OUT, 0, PTX7_IN,
 		PTX6_FN, PTX6_OUT, 0, PTX6_IN,
 		PTX5_FN, PTX5_OUT, 0, PTX5_IN,
@@ -1947,9 +1947,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTX3_FN, PTX3_OUT, 0, PTX3_IN,
 		PTX2_FN, PTX2_OUT, 0, PTX2_IN,
 		PTX1_FN, PTX1_OUT, 0, PTX1_IN,
-		PTX0_FN, PTX0_OUT, 0, PTX0_IN }
+		PTX0_FN, PTX0_OUT, 0, PTX0_IN ))
 	},
-	{ PINMUX_CFG_REG("PYCR", 0xa405014a, 16, 2) {
+	{ PINMUX_CFG_REG("PYCR", 0xa405014a, 16, 2, GROUP(
 		PTY7_FN, PTY7_OUT, 0, PTY7_IN,
 		PTY6_FN, PTY6_OUT, 0, PTY6_IN,
 		PTY5_FN, PTY5_OUT, 0, PTY5_IN,
@@ -1957,9 +1957,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTY3_FN, PTY3_OUT, 0, PTY3_IN,
 		PTY2_FN, PTY2_OUT, 0, PTY2_IN,
 		PTY1_FN, PTY1_OUT, 0, PTY1_IN,
-		PTY0_FN, PTY0_OUT, 0, PTY0_IN }
+		PTY0_FN, PTY0_OUT, 0, PTY0_IN ))
 	},
-	{ PINMUX_CFG_REG("PZCR", 0xa405014c, 16, 2) {
+	{ PINMUX_CFG_REG("PZCR", 0xa405014c, 16, 2, GROUP(
 		PTZ7_FN, PTZ7_OUT, 0, PTZ7_IN,
 		PTZ6_FN, PTZ6_OUT, 0, PTZ6_IN,
 		PTZ5_FN, PTZ5_OUT, 0, PTZ5_IN,
@@ -1967,9 +1967,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTZ3_FN, PTZ3_OUT, 0, PTZ3_IN,
 		PTZ2_FN, PTZ2_OUT, 0, PTZ2_IN,
 		PTZ1_FN, PTZ1_OUT, 0, PTZ1_IN,
-		PTZ0_FN, PTZ0_OUT, 0, PTZ0_IN }
+		PTZ0_FN, PTZ0_OUT, 0, PTZ0_IN ))
 	},
-	{ PINMUX_CFG_REG("PSELA", 0xa405014e, 16, 1) {
+	{ PINMUX_CFG_REG("PSELA", 0xa405014e, 16, 1, GROUP(
 		PSA15_0, PSA15_1,
 		PSA14_0, PSA14_1,
 		PSA13_0, PSA13_1,
@@ -1985,9 +1985,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PSA3_0,  PSA3_1,
 		PSA2_0,  PSA2_1,
 		PSA1_0,  PSA1_1,
-		PSA0_0,  PSA0_1}
+		PSA0_0,  PSA0_1))
 	},
-	{ PINMUX_CFG_REG("PSELB", 0xa4050150, 16, 1) {
+	{ PINMUX_CFG_REG("PSELB", 0xa4050150, 16, 1, GROUP(
 		0, 0,
 		PSB14_0, PSB14_1,
 		PSB13_0, PSB13_1,
@@ -2003,9 +2003,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PSB3_0,  PSB3_1,
 		PSB2_0,  PSB2_1,
 		PSB1_0,  PSB1_1,
-		PSB0_0,  PSB0_1}
+		PSB0_0,  PSB0_1))
 	},
-	{ PINMUX_CFG_REG("PSELC", 0xa4050152, 16, 1) {
+	{ PINMUX_CFG_REG("PSELC", 0xa4050152, 16, 1, GROUP(
 		PSC15_0, PSC15_1,
 		PSC14_0, PSC14_1,
 		PSC13_0, PSC13_1,
@@ -2021,9 +2021,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0,
 		PSC2_0,  PSC2_1,
 		PSC1_0,  PSC1_1,
-		PSC0_0,  PSC0_1}
+		PSC0_0,  PSC0_1))
 	},
-	{ PINMUX_CFG_REG("PSELD", 0xa4050154, 16, 1) {
+	{ PINMUX_CFG_REG("PSELD", 0xa4050154, 16, 1, GROUP(
 		PSD15_0, PSD15_1,
 		PSD14_0, PSD14_1,
 		PSD13_0, PSD13_1,
@@ -2039,9 +2039,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PSD3_0,  PSD3_1,
 		PSD2_0,  PSD2_1,
 		PSD1_0,  PSD1_1,
-		PSD0_0,  PSD0_1}
+		PSD0_0,  PSD0_1))
 	},
-	{ PINMUX_CFG_REG("PSELE", 0xa4050156, 16, 1) {
+	{ PINMUX_CFG_REG("PSELE", 0xa4050156, 16, 1, GROUP(
 		PSE15_0, PSE15_1,
 		PSE14_0, PSE14_1,
 		PSE13_0, PSE13_1,
@@ -2057,103 +2057,103 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PSE3_0,  PSE3_1,
 		PSE2_0,  PSE2_1,
 		PSE1_0,  PSE1_1,
-		PSE0_0,  PSE0_1}
+		PSE0_0,  PSE0_1))
 	},
 	{}
 };
 
 static const struct pinmux_data_reg pinmux_data_regs[] = {
-	{ PINMUX_DATA_REG("PADR", 0xa4050120, 8) {
+	{ PINMUX_DATA_REG("PADR", 0xa4050120, 8, GROUP(
 		PTA7_DATA, PTA6_DATA, PTA5_DATA, PTA4_DATA,
-		PTA3_DATA, PTA2_DATA, PTA1_DATA, PTA0_DATA }
+		PTA3_DATA, PTA2_DATA, PTA1_DATA, PTA0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PBDR", 0xa4050122, 8) {
+	{ PINMUX_DATA_REG("PBDR", 0xa4050122, 8, GROUP(
 		PTB7_DATA, PTB6_DATA, PTB5_DATA, PTB4_DATA,
-		PTB3_DATA, PTB2_DATA, PTB1_DATA, PTB0_DATA }
+		PTB3_DATA, PTB2_DATA, PTB1_DATA, PTB0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PCDR", 0xa4050124, 8) {
+	{ PINMUX_DATA_REG("PCDR", 0xa4050124, 8, GROUP(
 		PTC7_DATA, PTC6_DATA, PTC5_DATA, PTC4_DATA,
-		PTC3_DATA, PTC2_DATA, PTC1_DATA, PTC0_DATA }
+		PTC3_DATA, PTC2_DATA, PTC1_DATA, PTC0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PDDR", 0xa4050126, 8) {
+	{ PINMUX_DATA_REG("PDDR", 0xa4050126, 8, GROUP(
 		PTD7_DATA, PTD6_DATA, PTD5_DATA, PTD4_DATA,
-		PTD3_DATA, PTD2_DATA, PTD1_DATA, PTD0_DATA }
+		PTD3_DATA, PTD2_DATA, PTD1_DATA, PTD0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PEDR", 0xa4050128, 8) {
+	{ PINMUX_DATA_REG("PEDR", 0xa4050128, 8, GROUP(
 		PTE7_DATA, PTE6_DATA, PTE5_DATA, PTE4_DATA,
-		PTE3_DATA, PTE2_DATA, PTE1_DATA, PTE0_DATA }
+		PTE3_DATA, PTE2_DATA, PTE1_DATA, PTE0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PFDR", 0xa405012a, 8) {
+	{ PINMUX_DATA_REG("PFDR", 0xa405012a, 8, GROUP(
 		PTF7_DATA, PTF6_DATA, PTF5_DATA, PTF4_DATA,
-		PTF3_DATA, PTF2_DATA, PTF1_DATA, PTF0_DATA }
+		PTF3_DATA, PTF2_DATA, PTF1_DATA, PTF0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PGDR", 0xa405012c, 8) {
+	{ PINMUX_DATA_REG("PGDR", 0xa405012c, 8, GROUP(
 		0,         0,         PTG5_DATA, PTG4_DATA,
-		PTG3_DATA, PTG2_DATA, PTG1_DATA, PTG0_DATA }
+		PTG3_DATA, PTG2_DATA, PTG1_DATA, PTG0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PHDR", 0xa405012e, 8) {
+	{ PINMUX_DATA_REG("PHDR", 0xa405012e, 8, GROUP(
 		PTH7_DATA, PTH6_DATA, PTH5_DATA, PTH4_DATA,
-		PTH3_DATA, PTH2_DATA, PTH1_DATA, PTH0_DATA }
+		PTH3_DATA, PTH2_DATA, PTH1_DATA, PTH0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PJDR", 0xa4050130, 8) {
+	{ PINMUX_DATA_REG("PJDR", 0xa4050130, 8, GROUP(
 		PTJ7_DATA, PTJ6_DATA, PTJ5_DATA, 0,
-		PTJ3_DATA, PTJ2_DATA, PTJ1_DATA, PTJ0_DATA }
+		PTJ3_DATA, PTJ2_DATA, PTJ1_DATA, PTJ0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PKDR", 0xa4050132, 8) {
+	{ PINMUX_DATA_REG("PKDR", 0xa4050132, 8, GROUP(
 		PTK7_DATA, PTK6_DATA, PTK5_DATA, PTK4_DATA,
-		PTK3_DATA, PTK2_DATA, PTK1_DATA, PTK0_DATA }
+		PTK3_DATA, PTK2_DATA, PTK1_DATA, PTK0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PLDR", 0xa4050134, 8) {
+	{ PINMUX_DATA_REG("PLDR", 0xa4050134, 8, GROUP(
 		PTL7_DATA, PTL6_DATA, PTL5_DATA, PTL4_DATA,
-		PTL3_DATA, PTL2_DATA, PTL1_DATA, PTL0_DATA }
+		PTL3_DATA, PTL2_DATA, PTL1_DATA, PTL0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PMDR", 0xa4050136, 8) {
+	{ PINMUX_DATA_REG("PMDR", 0xa4050136, 8, GROUP(
 		PTM7_DATA, PTM6_DATA, PTM5_DATA, PTM4_DATA,
-		PTM3_DATA, PTM2_DATA, PTM1_DATA, PTM0_DATA }
+		PTM3_DATA, PTM2_DATA, PTM1_DATA, PTM0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PNDR", 0xa4050138, 8) {
+	{ PINMUX_DATA_REG("PNDR", 0xa4050138, 8, GROUP(
 		PTN7_DATA, PTN6_DATA, PTN5_DATA, PTN4_DATA,
-		PTN3_DATA, PTN2_DATA, PTN1_DATA, PTN0_DATA }
+		PTN3_DATA, PTN2_DATA, PTN1_DATA, PTN0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PQDR", 0xa405013a, 8) {
+	{ PINMUX_DATA_REG("PQDR", 0xa405013a, 8, GROUP(
 		PTQ7_DATA, PTQ6_DATA, PTQ5_DATA, PTQ4_DATA,
-		PTQ3_DATA, PTQ2_DATA, PTQ1_DATA, PTQ0_DATA }
+		PTQ3_DATA, PTQ2_DATA, PTQ1_DATA, PTQ0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PRDR", 0xa405013c, 8) {
+	{ PINMUX_DATA_REG("PRDR", 0xa405013c, 8, GROUP(
 		PTR7_DATA, PTR6_DATA, PTR5_DATA, PTR4_DATA,
-		PTR3_DATA, PTR2_DATA, PTR1_DATA, PTR0_DATA }
+		PTR3_DATA, PTR2_DATA, PTR1_DATA, PTR0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PSDR", 0xa405013e, 8) {
+	{ PINMUX_DATA_REG("PSDR", 0xa405013e, 8, GROUP(
 		0,         PTS6_DATA, PTS5_DATA, PTS4_DATA,
-		PTS3_DATA, PTS2_DATA, PTS1_DATA, PTS0_DATA }
+		PTS3_DATA, PTS2_DATA, PTS1_DATA, PTS0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PTDR", 0xa4050160, 8) {
+	{ PINMUX_DATA_REG("PTDR", 0xa4050160, 8, GROUP(
 		PTT7_DATA, PTT6_DATA, PTT5_DATA, PTT4_DATA,
-		PTT3_DATA, PTT2_DATA, PTT1_DATA, PTT0_DATA }
+		PTT3_DATA, PTT2_DATA, PTT1_DATA, PTT0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PUDR", 0xa4050162, 8) {
+	{ PINMUX_DATA_REG("PUDR", 0xa4050162, 8, GROUP(
 		PTU7_DATA, PTU6_DATA, PTU5_DATA, PTU4_DATA,
-		PTU3_DATA, PTU2_DATA, PTU1_DATA, PTU0_DATA }
+		PTU3_DATA, PTU2_DATA, PTU1_DATA, PTU0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PVDR", 0xa4050164, 8) {
+	{ PINMUX_DATA_REG("PVDR", 0xa4050164, 8, GROUP(
 		PTV7_DATA, PTV6_DATA, PTV5_DATA, PTV4_DATA,
-		PTV3_DATA, PTV2_DATA, PTV1_DATA, PTV0_DATA }
+		PTV3_DATA, PTV2_DATA, PTV1_DATA, PTV0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PWDR", 0xa4050166, 8) {
+	{ PINMUX_DATA_REG("PWDR", 0xa4050166, 8, GROUP(
 		PTW7_DATA, PTW6_DATA, PTW5_DATA, PTW4_DATA,
-		PTW3_DATA, PTW2_DATA, PTW1_DATA, PTW0_DATA }
+		PTW3_DATA, PTW2_DATA, PTW1_DATA, PTW0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PXDR", 0xa4050168, 8) {
+	{ PINMUX_DATA_REG("PXDR", 0xa4050168, 8, GROUP(
 		PTX7_DATA, PTX6_DATA, PTX5_DATA, PTX4_DATA,
-		PTX3_DATA, PTX2_DATA, PTX1_DATA, PTX0_DATA }
+		PTX3_DATA, PTX2_DATA, PTX1_DATA, PTX0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PYDR", 0xa405016a, 8) {
+	{ PINMUX_DATA_REG("PYDR", 0xa405016a, 8, GROUP(
 		PTY7_DATA, PTY6_DATA, PTY5_DATA, PTY4_DATA,
-		PTY3_DATA, PTY2_DATA, PTY1_DATA, PTY0_DATA }
+		PTY3_DATA, PTY2_DATA, PTY1_DATA, PTY0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PZDR", 0xa405016c, 8) {
+	{ PINMUX_DATA_REG("PZDR", 0xa405016c, 8, GROUP(
 		PTZ7_DATA, PTZ6_DATA, PTZ5_DATA, PTZ4_DATA,
-		PTZ3_DATA, PTZ2_DATA, PTZ1_DATA, PTZ0_DATA }
+		PTZ3_DATA, PTZ2_DATA, PTZ1_DATA, PTZ0_DATA ))
 	},
 	{ },
 };
diff --git a/drivers/pinctrl/sh-pfc/pfc-sh7734.c b/drivers/pinctrl/sh-pfc/pfc-sh7734.c
index 748a32a3af82..fac7b4699121 100644
--- a/drivers/pinctrl/sh-pfc/pfc-sh7734.c
+++ b/drivers/pinctrl/sh-pfc/pfc-sh7734.c
@@ -1635,7 +1635,7 @@ static const struct pinmux_func pinmux_func_gpios[] = {
 };
 
 static const struct pinmux_cfg_reg pinmux_config_regs[] = {
-	{ PINMUX_CFG_REG("GPSR0", 0xFFFC0004, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR0", 0xFFFC0004, 32, 1, GROUP(
 		GP_0_31_FN, FN_IP2_2_0,
 		GP_0_30_FN, FN_IP1_31_29,
 		GP_0_29_FN, FN_IP1_28_26,
@@ -1667,9 +1667,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_0_3_FN, FN_IP1_15_14,
 		GP_0_2_FN, FN_IP1_13_12,
 		GP_0_1_FN, FN_IP1_11_10,
-		GP_0_0_FN, FN_IP1_9_8 }
+		GP_0_0_FN, FN_IP1_9_8 ))
 	},
-	{ PINMUX_CFG_REG("GPSR1", 0xFFFC0008, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR1", 0xFFFC0008, 32, 1, GROUP(
 		GP_1_31_FN, FN_IP11_25_23,
 		GP_1_30_FN, FN_IP2_13_11,
 		GP_1_29_FN, FN_IP2_10_8,
@@ -1701,9 +1701,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_1_3_FN, FN_IP11_22_21,
 		GP_1_2_FN, FN_IP11_20_19,
 		GP_1_1_FN, FN_IP3_29_27,
-		GP_1_0_FN, FN_IP3_20 }
+		GP_1_0_FN, FN_IP3_20 ))
 	},
-	{ PINMUX_CFG_REG("GPSR2", 0xFFFC000C, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR2", 0xFFFC000C, 32, 1, GROUP(
 		GP_2_31_FN, FN_IP4_31_30,
 		GP_2_30_FN, FN_IP5_2_0,
 		GP_2_29_FN, FN_IP5_5_3,
@@ -1735,9 +1735,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_2_3_FN, FN_IP4_2_0,
 		GP_2_2_FN, FN_IP11_11_10,
 		GP_2_1_FN, FN_IP11_9_7,
-		GP_2_0_FN, FN_IP11_6_4 }
+		GP_2_0_FN, FN_IP11_6_4 ))
 	},
-	{ PINMUX_CFG_REG("GPSR3", 0xFFFC0010, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR3", 0xFFFC0010, 32, 1, GROUP(
 		GP_3_31_FN, FN_IP9_1_0,
 		GP_3_30_FN, FN_IP8_19_18,
 		GP_3_29_FN, FN_IP8_17_16,
@@ -1769,10 +1769,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_3_3_FN, FN_IP6_9_8,
 		GP_3_2_FN, FN_IP6_7_6,
 		GP_3_1_FN, FN_IP6_5_3,
-		GP_3_0_FN, FN_IP6_2_0 }
+		GP_3_0_FN, FN_IP6_2_0 ))
 	},
 
-	{ PINMUX_CFG_REG("GPSR4", 0xFFFC0014, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR4", 0xFFFC0014, 32, 1, GROUP(
 		GP_4_31_FN, FN_IP10_24_23,
 		GP_4_30_FN, FN_IP10_22,
 		GP_4_29_FN, FN_IP11_18_16,
@@ -1804,9 +1804,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_4_3_FN, FN_IP9_25_24,
 		GP_4_2_FN, FN_IP9_23_22,
 		GP_4_1_FN, FN_IP9_21_20,
-		GP_4_0_FN, FN_IP9_19_18 }
+		GP_4_0_FN, FN_IP9_19_18 ))
 	},
-	{ PINMUX_CFG_REG("GPSR5", 0xFFFC0018, 32, 1) {
+	{ PINMUX_CFG_REG("GPSR5", 0xFFFC0018, 32, 1, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0, /* 31 - 28 */
 		0, 0, 0, 0, 0, 0, 0, 0, /* 27 - 24 */
 		0, 0, 0, 0, 0, 0, 0, 0, /* 23 - 20 */
@@ -1819,12 +1819,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_5_3_FN, FN_IRQ3_B,
 		GP_5_2_FN, FN_IRQ2_B,
 		GP_5_1_FN, FN_IP11_3,
-		GP_5_0_FN, FN_IP10_25 }
+		GP_5_0_FN, FN_IP10_25 ))
 	},
 
 	{ PINMUX_CFG_REG_VAR("IPSR0", 0xFFFC001C, 32,
-			2, 2, 2, 2, 2, 2, 2, 2,
-			2, 2, 2, 2, 2, 2, 2, 2) {
+			GROUP(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2),
+			GROUP(
 		/* IP0_31_30 [2] */
 		FN_A15, FN_ST0_VCO_CLKIN, FN_LCD_DATA15_A,
 			FN_TIOC3D_C,
@@ -1857,10 +1857,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP0_3_2 [2] */
 		FN_A1, FN_ST0_REQ, FN_LCD_DATA1_A, FN_TCLKB_C,
 		/* IP0_1_0 [2] */
-		FN_A0, FN_ST0_CLKIN, FN_LCD_DATA0_A, FN_TCLKA_C }
+		FN_A0, FN_ST0_CLKIN, FN_LCD_DATA0_A, FN_TCLKA_C ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR1", 0xFFFC0020, 32,
-			3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2) {
+			GROUP(3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2),
+			GROUP(
 		/* IP1_31_29 [3] */
 		FN_D3, FN_SD0_DAT3_A, FN_MMC_D3_A, FN_ST1_D6,
 			FN_FD3_A, 0, 0, 0,
@@ -1892,10 +1893,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* IP1_3_2 [2] */
 		FN_A17, FN_ST1_VCO_CLKIN, FN_LCD_CL1_A,	FN_TIOC4B_C,
 		/* IP1_1_0 [2] */
-		FN_A16, FN_ST0_PWM, FN_LCD_DON_A, FN_TIOC4A_C }
+		FN_A16, FN_ST0_PWM, FN_LCD_DON_A, FN_TIOC4A_C ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR2", 0xFFFC0024, 32,
-			     1, 3, 3, 2, 3, 3, 3, 3, 3, 3, 2, 3) {
+			     GROUP(1, 3, 3, 2, 3, 3, 3, 3, 3, 3, 2, 3),
+			     GROUP(
 		/* IP2_31 [1] */
 		0, 0,
 		/* IP2_30_28 [3] */
@@ -1928,10 +1930,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_D5, FN_SD0_WP_A, FN_MMC_D5_A, FN_FD5_A,
 		/* IP2_2_0 [3] */
 		FN_D4, FN_SD0_CD_A, FN_MMC_D4_A, FN_ST1_D7,
-			FN_FD4_A, 0, 0, 0 }
+			FN_FD4_A, 0, 0, 0 ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR3", 0xFFFC0028, 32,
-				2, 3, 3, 3, 1, 2, 3, 3, 3, 3, 3, 1, 2) {
+			     GROUP(2, 3, 3, 3, 1, 2, 3, 3, 3, 3, 3, 1, 2),
+			     GROUP(
 	    /* IP3_31_30 [2] */
 		0, 0, 0, 0,
 	    /* IP3_29_27 [3] */
@@ -1965,10 +1968,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 	    /* IP3_2 [1] */
 		FN_CS1_A26, FN_QIO3_B,
 	    /* IP3_1_0 [2] */
-		FN_D15, FN_SCK2_B, 0, 0 }
+		FN_D15, FN_SCK2_B, 0, 0 ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR4", 0xFFFC002C, 32,
-				2, 2, 2, 2, 2, 2 , 2, 3, 3, 3, 3, 3, 3) {
+			     GROUP(2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3),
+			     GROUP(
 	    /* IP4_31_30 [2] */
 		0, FN_SCK2_A, FN_VI0_G3, 0,
 	    /* IP4_29_28 [2] */
@@ -2000,10 +2004,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 			FN_ET0_RX_DV, 0, 0, 0,
 	    /* IP4_2_0 [3] */
 		FN_HCTS0_A, FN_CTS1_A, FN_VI0_FIELD, FN_RMII0_RXD1_A,
-			FN_ET0_ERXD7, 0, 0, 0 }
+			FN_ET0_ERXD7, 0, 0, 0 ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR5", 0xFFFC0030, 32,
-				1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3) {
+			     GROUP(1, 1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3,
+				   3, 3, 3),
+			     GROUP(
 	    /* IP5_31 [1] */
 	    0, 0,
 	    /* IP5_30 [1] */
@@ -2040,11 +2046,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, FN_ET0_ERXD2_B,
 	    /* IP5_2_0 [3] */
 		FN_SD2_CLK_A, FN_RX2_A, FN_VI0_G4, 0,
-		FN_ET0_RX_CLK_B, 0, 0, 0 }
+		FN_ET0_RX_CLK_B, 0, 0, 0 ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR6", 0xFFFC0034, 32,
-				1, 1, 1, 1, 1, 1, 1, 1,
-				3, 3, 2, 2, 2, 2, 2, 2, 3, 3) {
+			     GROUP(1, 1, 1, 1, 1, 1, 1, 1, 3, 3, 2, 2,
+				   2, 2, 2, 2, 3, 3),
+			     GROUP(
 	    /* IP5_31 [1] */
 	    0, 0,
 	    /* IP6_30 [1] */
@@ -2084,10 +2091,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_TCLKB_A, FN_HIFD01, 0, 0,
 	    /* IP6_2_0 [3] */
 		FN_DU0_DR0, FN_SCIF_CLK_B, FN_HRX0_D, FN_IETX_A,
-		FN_TCLKA_A, FN_HIFD00, 0, 0 }
+		FN_TCLKA_A, FN_HIFD00, 0, 0 ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR7", 0xFFFC0038, 32,
-			     1, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3) {
+			     GROUP(1, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3),
+			     GROUP(
 	    /* IP7_31 [1] */
 	    0, 0,
 	    /* IP7_30_29 [2] */
@@ -2120,10 +2128,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_HIFD11, 0, 0, 0,
 	    /* IP7_2_0 [3] */
 		FN_DU0_DG2, FN_RTS1_C, FN_RMII0_MDC_B, FN_TIOC2A_A,
-		FN_HIFD10, 0, 0, 0 }
+		FN_HIFD10, 0, 0, 0 ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR8", 0xFFFC003C, 32,
-			     2, 2, 2, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2) {
+			     GROUP(2, 2, 2, 3, 3, 2, 2, 2, 2, 2, 2, 2,
+				   2, 2, 2),
+			     GROUP(
 	    /* IP9_31_30 [2] */
 	    0, 0, 0, 0,
 	    /* IP8_29_28 [2] */
@@ -2156,11 +2166,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 	    /* IP8_3_2 [2] */
 		FN_DU0_DB6, 0, FN_HIFRDY, 0,
 	    /* IP8_1_0 [2] */
-		FN_DU0_DB5, 0, FN_HIFDREQ, 0 }
+		FN_DU0_DB5, 0, FN_HIFDREQ, 0 ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR9", 0xFFFC0040, 32,
-			     2, 2, 2, 2, 2, 2, 2, 2,
-			     2, 2, 2, 2, 2, 2, 2, 2) {
+			     GROUP(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+				   2, 2, 2, 2),
+			     GROUP(
 	    /* IP9_31_30 [2] */
 	    0, 0, 0, 0,
 	    /* IP9_29_28 [2] */
@@ -2192,11 +2203,11 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 	    /* IP9_3_2 [2] */
 		FN_VI1_0_A, 0, FN_FD1_B, FN_LCD_DATA1_B,
 	    /* IP9_1_0 [2] */
-		FN_VI1_CLK_A, 0, FN_FD0_B, FN_LCD_DATA0_B }
+		FN_VI1_CLK_A, 0, FN_FD0_B, FN_LCD_DATA0_B ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR10", 0xFFFC0044, 32,
-					2, 2, 2, 1, 2, 1, 3,
-					3, 1, 3, 3, 3, 3, 3) {
+			     GROUP(2, 2, 2, 1, 2, 1, 3, 3, 1, 3, 3, 3, 3, 3),
+			     GROUP(
 	    /* IP9_31_30 [2] */
 	    0, 0, 0, 0,
 	    /* IP10_29_28 [2] */
@@ -2231,10 +2242,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		FN_LCD_DON_B, 0, 0,
 	    /* IP10_2_0 [3] */
 		FN_SSI_SCK23, FN_VI1_4_B, FN_RX1_D, FN_FCLE_B,
-		FN_LCD_DATA15_B, 0, 0, 0 }
+		FN_LCD_DATA15_B, 0, 0, 0 ))
 	},
 	{ PINMUX_CFG_REG_VAR("IPSR11", 0xFFFC0048, 32,
-			3, 1, 2, 3, 2, 2, 3, 3, 1, 2, 3, 3, 1, 1, 1, 1) {
+			     GROUP(3, 1, 2, 3, 2, 2, 3, 3, 1, 2, 3, 3,
+				   1, 1, 1, 1),
+			     GROUP(
 	    /* IP11_31_29 [3] */
 	    0, 0, 0, 0, 0, 0, 0, 0,
 	    /* IP11_28 [1] */
@@ -2271,11 +2284,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 	    /* IP11_1 [1] */
 		FN_SDA1, FN_RX1_E,
 	    /* IP11_0 [1] */
-		FN_SCL1, FN_SCIF_CLK_C }
+		FN_SCL1, FN_SCIF_CLK_C ))
 	},
 	{ PINMUX_CFG_REG_VAR("MOD_SEL1", 0xFFFC004C, 32,
-				3, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 2, 2,
-				1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) {
+			     GROUP(3, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 2,
+				   2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1),
+			     GROUP(
 		/* SEL1_31_29 [3] */
 		0, 0, 0, 0, 0, 0, 0, 0,
 		/* SEL1_28 [1] */
@@ -2327,11 +2341,12 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* SEL1_1 [1] */
 		FN_SEL_MMC_0, FN_SEL_MMC_1,
 		/* SEL1_0 [1] */
-		FN_SEL_INTC_0, FN_SEL_INTC_1 }
+		FN_SEL_INTC_0, FN_SEL_INTC_1 ))
 	},
 	{ PINMUX_CFG_REG_VAR("MOD_SEL2", 0xFFFC0050, 32,
-				1, 1, 1, 1, 1, 1, 1, 1,
-				1, 1, 1, 2, 2, 1, 2, 2, 3, 2, 3, 2, 2) {
+			     GROUP(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2,
+				   2, 1, 2, 2, 3, 2, 3, 2, 2),
+			     GROUP(
 		/* SEL2_31 [1] */
 		0, 0,
 		/* SEL2_30 [1] */
@@ -2375,15 +2390,20 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		/* SEL2_3_2 [2] */
 		FN_SEL_SCIF0_0, FN_SEL_SCIF0_1, FN_SEL_SCIF0_2, 0,
 		/* SEL2_1_0 [2] */
-		FN_SEL_SCIF_CLK_0, FN_SEL_SCIF_CLK_1, FN_SEL_SCIF_CLK_2, 0  }
+		FN_SEL_SCIF_CLK_0, FN_SEL_SCIF_CLK_1, FN_SEL_SCIF_CLK_2, 0  ))
 	},
 	/* GPIO 0 - 5*/
-	{ PINMUX_CFG_REG("INOUTSEL0", 0xFFC40004, 32, 1) { GP_INOUTSEL(0) } },
-	{ PINMUX_CFG_REG("INOUTSEL1", 0xFFC41004, 32, 1) { GP_INOUTSEL(1) } },
-	{ PINMUX_CFG_REG("INOUTSEL2", 0xFFC42004, 32, 1) { GP_INOUTSEL(2) } },
-	{ PINMUX_CFG_REG("INOUTSEL3", 0xFFC43004, 32, 1) { GP_INOUTSEL(3) } },
-	{ PINMUX_CFG_REG("INOUTSEL4", 0xFFC44004, 32, 1) { GP_INOUTSEL(4) } },
-	{ PINMUX_CFG_REG("INOUTSEL5", 0xffc45004, 32, 1) {
+	{ PINMUX_CFG_REG("INOUTSEL0", 0xFFC40004, 32, 1, GROUP(GP_INOUTSEL(0)))
+	},
+	{ PINMUX_CFG_REG("INOUTSEL1", 0xFFC41004, 32, 1, GROUP(GP_INOUTSEL(1)))
+	},
+	{ PINMUX_CFG_REG("INOUTSEL2", 0xFFC42004, 32, 1, GROUP(GP_INOUTSEL(2)))
+	},
+	{ PINMUX_CFG_REG("INOUTSEL3", 0xFFC43004, 32, 1, GROUP(GP_INOUTSEL(3)))
+	},
+	{ PINMUX_CFG_REG("INOUTSEL4", 0xFFC44004, 32, 1, GROUP(GP_INOUTSEL(4)))
+	},
+	{ PINMUX_CFG_REG("INOUTSEL5", 0xffc45004, 32, 1, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 31 - 24 */
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 23 - 16 */
 		0, 0, 0, 0, 0, 0, 0, 0, /* 15 - 12 */
@@ -2398,24 +2418,24 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		GP_5_3_IN, GP_5_3_OUT,
 		GP_5_2_IN, GP_5_2_OUT,
 		GP_5_1_IN, GP_5_1_OUT,
-		GP_5_0_IN, GP_5_0_OUT }
+		GP_5_0_IN, GP_5_0_OUT ))
 	},
 	{ },
 };
 
 static const struct pinmux_data_reg pinmux_data_regs[] = {
 	/* GPIO 0 - 5*/
-	{ PINMUX_DATA_REG("INDT0", 0xFFC4000C, 32) { GP_INDT(0) } },
-	{ PINMUX_DATA_REG("INDT1", 0xFFC4100C, 32) { GP_INDT(1) } },
-	{ PINMUX_DATA_REG("INDT2", 0xFFC4200C, 32) { GP_INDT(2) } },
-	{ PINMUX_DATA_REG("INDT3", 0xFFC4300C, 32) { GP_INDT(3) } },
-	{ PINMUX_DATA_REG("INDT4", 0xFFC4400C, 32) { GP_INDT(4) } },
-	{ PINMUX_DATA_REG("INDT5", 0xFFC4500C, 32) {
+	{ PINMUX_DATA_REG("INDT0", 0xFFC4000C, 32, GROUP(GP_INDT(0))) },
+	{ PINMUX_DATA_REG("INDT1", 0xFFC4100C, 32, GROUP(GP_INDT(1))) },
+	{ PINMUX_DATA_REG("INDT2", 0xFFC4200C, 32, GROUP(GP_INDT(2))) },
+	{ PINMUX_DATA_REG("INDT3", 0xFFC4300C, 32, GROUP(GP_INDT(3))) },
+	{ PINMUX_DATA_REG("INDT4", 0xFFC4400C, 32, GROUP(GP_INDT(4))) },
+	{ PINMUX_DATA_REG("INDT5", 0xFFC4500C, 32, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0, 0, 0,
 		GP_5_11_DATA, GP_5_10_DATA, GP_5_9_DATA, GP_5_8_DATA,
 		GP_5_7_DATA, GP_5_6_DATA, GP_5_5_DATA, GP_5_4_DATA,
-		GP_5_3_DATA, GP_5_2_DATA, GP_5_1_DATA, GP_5_0_DATA }
+		GP_5_3_DATA, GP_5_2_DATA, GP_5_1_DATA, GP_5_0_DATA ))
 	},
 	{ },
 };
diff --git a/drivers/pinctrl/sh-pfc/pfc-sh7757.c b/drivers/pinctrl/sh-pfc/pfc-sh7757.c
index b16090690ee3..064e987b09cb 100644
--- a/drivers/pinctrl/sh-pfc/pfc-sh7757.c
+++ b/drivers/pinctrl/sh-pfc/pfc-sh7757.c
@@ -1683,7 +1683,7 @@ static const struct pinmux_func pinmux_func_gpios[] = {
 };
 
 static const struct pinmux_cfg_reg pinmux_config_regs[] = {
-	{ PINMUX_CFG_REG("PACR", 0xffec0000, 16, 2) {
+	{ PINMUX_CFG_REG("PACR", 0xffec0000, 16, 2, GROUP(
 		PTA7_FN, PTA7_OUT, PTA7_IN, 0,
 		PTA6_FN, PTA6_OUT, PTA6_IN, 0,
 		PTA5_FN, PTA5_OUT, PTA5_IN, 0,
@@ -1691,9 +1691,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTA3_FN, PTA3_OUT, PTA3_IN, 0,
 		PTA2_FN, PTA2_OUT, PTA2_IN, 0,
 		PTA1_FN, PTA1_OUT, PTA1_IN, 0,
-		PTA0_FN, PTA0_OUT, PTA0_IN, 0 }
+		PTA0_FN, PTA0_OUT, PTA0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PBCR", 0xffec0002, 16, 2) {
+	{ PINMUX_CFG_REG("PBCR", 0xffec0002, 16, 2, GROUP(
 		PTB7_FN, PTB7_OUT, PTB7_IN, 0,
 		PTB6_FN, PTB6_OUT, PTB6_IN, 0,
 		PTB5_FN, PTB5_OUT, PTB5_IN, 0,
@@ -1701,9 +1701,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTB3_FN, PTB3_OUT, PTB3_IN, 0,
 		PTB2_FN, PTB2_OUT, PTB2_IN, 0,
 		PTB1_FN, PTB1_OUT, PTB1_IN, 0,
-		PTB0_FN, PTB0_OUT, PTB0_IN, 0 }
+		PTB0_FN, PTB0_OUT, PTB0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PCCR", 0xffec0004, 16, 2) {
+	{ PINMUX_CFG_REG("PCCR", 0xffec0004, 16, 2, GROUP(
 		PTC7_FN, PTC7_OUT, PTC7_IN, 0,
 		PTC6_FN, PTC6_OUT, PTC6_IN, 0,
 		PTC5_FN, PTC5_OUT, PTC5_IN, 0,
@@ -1711,9 +1711,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTC3_FN, PTC3_OUT, PTC3_IN, 0,
 		PTC2_FN, PTC2_OUT, PTC2_IN, 0,
 		PTC1_FN, PTC1_OUT, PTC1_IN, 0,
-		PTC0_FN, PTC0_OUT, PTC0_IN, 0 }
+		PTC0_FN, PTC0_OUT, PTC0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PDCR", 0xffec0006, 16, 2) {
+	{ PINMUX_CFG_REG("PDCR", 0xffec0006, 16, 2, GROUP(
 		PTD7_FN, PTD7_OUT, PTD7_IN, 0,
 		PTD6_FN, PTD6_OUT, PTD6_IN, 0,
 		PTD5_FN, PTD5_OUT, PTD5_IN, 0,
@@ -1721,9 +1721,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTD3_FN, PTD3_OUT, PTD3_IN, 0,
 		PTD2_FN, PTD2_OUT, PTD2_IN, 0,
 		PTD1_FN, PTD1_OUT, PTD1_IN, 0,
-		PTD0_FN, PTD0_OUT, PTD0_IN, 0 }
+		PTD0_FN, PTD0_OUT, PTD0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PECR", 0xffec0008, 16, 2) {
+	{ PINMUX_CFG_REG("PECR", 0xffec0008, 16, 2, GROUP(
 		PTE7_FN, PTE7_OUT, PTE7_IN, 0,
 		PTE6_FN, PTE6_OUT, PTE6_IN, 0,
 		PTE5_FN, PTE5_OUT, PTE5_IN, 0,
@@ -1731,9 +1731,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTE3_FN, PTE3_OUT, PTE3_IN, 0,
 		PTE2_FN, PTE2_OUT, PTE2_IN, 0,
 		PTE1_FN, PTE1_OUT, PTE1_IN, 0,
-		PTE0_FN, PTE0_OUT, PTE0_IN, 0 }
+		PTE0_FN, PTE0_OUT, PTE0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PFCR", 0xffec000a, 16, 2) {
+	{ PINMUX_CFG_REG("PFCR", 0xffec000a, 16, 2, GROUP(
 		PTF7_FN, PTF7_OUT, PTF7_IN, 0,
 		PTF6_FN, PTF6_OUT, PTF6_IN, 0,
 		PTF5_FN, PTF5_OUT, PTF5_IN, 0,
@@ -1741,9 +1741,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTF3_FN, PTF3_OUT, PTF3_IN, 0,
 		PTF2_FN, PTF2_OUT, PTF2_IN, 0,
 		PTF1_FN, PTF1_OUT, PTF1_IN, 0,
-		PTF0_FN, PTF0_OUT, PTF0_IN, 0 }
+		PTF0_FN, PTF0_OUT, PTF0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PGCR", 0xffec000c, 16, 2) {
+	{ PINMUX_CFG_REG("PGCR", 0xffec000c, 16, 2, GROUP(
 		PTG7_FN, PTG7_OUT, PTG7_IN, 0,
 		PTG6_FN, PTG6_OUT, PTG6_IN, 0,
 		PTG5_FN, PTG5_OUT, PTG5_IN, 0,
@@ -1751,9 +1751,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTG3_FN, PTG3_OUT, PTG3_IN, 0,
 		PTG2_FN, PTG2_OUT, PTG2_IN, 0,
 		PTG1_FN, PTG1_OUT, PTG1_IN, 0,
-		PTG0_FN, PTG0_OUT, PTG0_IN, 0 }
+		PTG0_FN, PTG0_OUT, PTG0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PHCR", 0xffec000e, 16, 2) {
+	{ PINMUX_CFG_REG("PHCR", 0xffec000e, 16, 2, GROUP(
 		PTH7_FN, PTH7_OUT, PTH7_IN, 0,
 		PTH6_FN, PTH6_OUT, PTH6_IN, 0,
 		PTH5_FN, PTH5_OUT, PTH5_IN, 0,
@@ -1761,9 +1761,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTH3_FN, PTH3_OUT, PTH3_IN, 0,
 		PTH2_FN, PTH2_OUT, PTH2_IN, 0,
 		PTH1_FN, PTH1_OUT, PTH1_IN, 0,
-		PTH0_FN, PTH0_OUT, PTH0_IN, 0 }
+		PTH0_FN, PTH0_OUT, PTH0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PICR", 0xffec0010, 16, 2) {
+	{ PINMUX_CFG_REG("PICR", 0xffec0010, 16, 2, GROUP(
 		PTI7_FN, PTI7_OUT, PTI7_IN, 0,
 		PTI6_FN, PTI6_OUT, PTI6_IN, 0,
 		PTI5_FN, PTI5_OUT, PTI5_IN, 0,
@@ -1771,9 +1771,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTI3_FN, PTI3_OUT, PTI3_IN, 0,
 		PTI2_FN, PTI2_OUT, PTI2_IN, 0,
 		PTI1_FN, PTI1_OUT, PTI1_IN, 0,
-		PTI0_FN, PTI0_OUT, PTI0_IN, 0 }
+		PTI0_FN, PTI0_OUT, PTI0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PJCR", 0xffec0012, 16, 2) {
+	{ PINMUX_CFG_REG("PJCR", 0xffec0012, 16, 2, GROUP(
 		0, 0, 0, 0,	/* reserved: always set 1 */
 		PTJ6_FN, PTJ6_OUT, PTJ6_IN, 0,
 		PTJ5_FN, PTJ5_OUT, PTJ5_IN, 0,
@@ -1781,9 +1781,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTJ3_FN, PTJ3_OUT, PTJ3_IN, 0,
 		PTJ2_FN, PTJ2_OUT, PTJ2_IN, 0,
 		PTJ1_FN, PTJ1_OUT, PTJ1_IN, 0,
-		PTJ0_FN, PTJ0_OUT, PTJ0_IN, 0 }
+		PTJ0_FN, PTJ0_OUT, PTJ0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PKCR", 0xffec0014, 16, 2) {
+	{ PINMUX_CFG_REG("PKCR", 0xffec0014, 16, 2, GROUP(
 		PTK7_FN, PTK7_OUT, PTK7_IN, 0,
 		PTK6_FN, PTK6_OUT, PTK6_IN, 0,
 		PTK5_FN, PTK5_OUT, PTK5_IN, 0,
@@ -1791,9 +1791,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTK3_FN, PTK3_OUT, PTK3_IN, 0,
 		PTK2_FN, PTK2_OUT, PTK2_IN, 0,
 		PTK1_FN, PTK1_OUT, PTK1_IN, 0,
-		PTK0_FN, PTK0_OUT, PTK0_IN, 0 }
+		PTK0_FN, PTK0_OUT, PTK0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PLCR", 0xffec0016, 16, 2) {
+	{ PINMUX_CFG_REG("PLCR", 0xffec0016, 16, 2, GROUP(
 		0, 0, 0, 0,	/* reserved: always set 1 */
 		PTL6_FN, PTL6_OUT, PTL6_IN, 0,
 		PTL5_FN, PTL5_OUT, PTL5_IN, 0,
@@ -1801,9 +1801,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTL3_FN, PTL3_OUT, PTL3_IN, 0,
 		PTL2_FN, PTL2_OUT, PTL2_IN, 0,
 		PTL1_FN, PTL1_OUT, PTL1_IN, 0,
-		PTL0_FN, PTL0_OUT, PTL0_IN, 0 }
+		PTL0_FN, PTL0_OUT, PTL0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PMCR", 0xffec0018, 16, 2) {
+	{ PINMUX_CFG_REG("PMCR", 0xffec0018, 16, 2, GROUP(
 		PTM7_FN, PTM7_OUT, PTM7_IN, 0,
 		PTM6_FN, PTM6_OUT, PTM6_IN, 0,
 		PTM5_FN, PTM5_OUT, PTM5_IN, 0,
@@ -1811,9 +1811,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTM3_FN, PTM3_OUT, PTM3_IN, 0,
 		PTM2_FN, PTM2_OUT, PTM2_IN, 0,
 		PTM1_FN, PTM1_OUT, PTM1_IN, 0,
-		PTM0_FN, PTM0_OUT, PTM0_IN, 0 }
+		PTM0_FN, PTM0_OUT, PTM0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PNCR", 0xffec001a, 16, 2) {
+	{ PINMUX_CFG_REG("PNCR", 0xffec001a, 16, 2, GROUP(
 		0, 0, 0, 0,	/* reserved: always set 1 */
 		PTN6_FN, PTN6_OUT, PTN6_IN, 0,
 		PTN5_FN, PTN5_OUT, PTN5_IN, 0,
@@ -1821,9 +1821,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTN3_FN, PTN3_OUT, PTN3_IN, 0,
 		PTN2_FN, PTN2_OUT, PTN2_IN, 0,
 		PTN1_FN, PTN1_OUT, PTN1_IN, 0,
-		PTN0_FN, PTN0_OUT, PTN0_IN, 0 }
+		PTN0_FN, PTN0_OUT, PTN0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("POCR", 0xffec001c, 16, 2) {
+	{ PINMUX_CFG_REG("POCR", 0xffec001c, 16, 2, GROUP(
 		PTO7_FN, PTO7_OUT, PTO7_IN, 0,
 		PTO6_FN, PTO6_OUT, PTO6_IN, 0,
 		PTO5_FN, PTO5_OUT, PTO5_IN, 0,
@@ -1831,10 +1831,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTO3_FN, PTO3_OUT, PTO3_IN, 0,
 		PTO2_FN, PTO2_OUT, PTO2_IN, 0,
 		PTO1_FN, PTO1_OUT, PTO1_IN, 0,
-		PTO0_FN, PTO0_OUT, PTO0_IN, 0 }
+		PTO0_FN, PTO0_OUT, PTO0_IN, 0 ))
 	},
 #if 0	/* FIXME: Remove it? */
-	{ PINMUX_CFG_REG("PPCR", 0xffec001e, 16, 2) {
+	{ PINMUX_CFG_REG("PPCR", 0xffec001e, 16, 2, GROUP(
 		0, 0, 0, 0,	/* reserved: always set 1 */
 		PTP6_FN, PTP6_OUT, PTP6_IN, 0,
 		PTP5_FN, PTP5_OUT, PTP5_IN, 0,
@@ -1842,10 +1842,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTP3_FN, PTP3_OUT, PTP3_IN, 0,
 		PTP2_FN, PTP2_OUT, PTP2_IN, 0,
 		PTP1_FN, PTP1_OUT, PTP1_IN, 0,
-		PTP0_FN, PTP0_OUT, PTP0_IN, 0 }
+		PTP0_FN, PTP0_OUT, PTP0_IN, 0 ))
 	},
 #endif
-	{ PINMUX_CFG_REG("PQCR", 0xffec0020, 16, 2) {
+	{ PINMUX_CFG_REG("PQCR", 0xffec0020, 16, 2, GROUP(
 		0, 0, 0, 0,	/* reserved: always set 1 */
 		PTQ6_FN, PTQ6_OUT, PTQ6_IN, 0,
 		PTQ5_FN, PTQ5_OUT, PTQ5_IN, 0,
@@ -1853,9 +1853,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTQ3_FN, PTQ3_OUT, PTQ3_IN, 0,
 		PTQ2_FN, PTQ2_OUT, PTQ2_IN, 0,
 		PTQ1_FN, PTQ1_OUT, PTQ1_IN, 0,
-		PTQ0_FN, PTQ0_OUT, PTQ0_IN, 0 }
+		PTQ0_FN, PTQ0_OUT, PTQ0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PRCR", 0xffec0022, 16, 2) {
+	{ PINMUX_CFG_REG("PRCR", 0xffec0022, 16, 2, GROUP(
 		PTR7_FN, PTR7_OUT, PTR7_IN, 0,
 		PTR6_FN, PTR6_OUT, PTR6_IN, 0,
 		PTR5_FN, PTR5_OUT, PTR5_IN, 0,
@@ -1863,9 +1863,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTR3_FN, PTR3_OUT, PTR3_IN, 0,
 		PTR2_FN, PTR2_OUT, PTR2_IN, 0,
 		PTR1_FN, PTR1_OUT, PTR1_IN, 0,
-		PTR0_FN, PTR0_OUT, PTR0_IN, 0 }
+		PTR0_FN, PTR0_OUT, PTR0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PSCR", 0xffec0024, 16, 2) {
+	{ PINMUX_CFG_REG("PSCR", 0xffec0024, 16, 2, GROUP(
 		PTS7_FN, PTS7_OUT, PTS7_IN, 0,
 		PTS6_FN, PTS6_OUT, PTS6_IN, 0,
 		PTS5_FN, PTS5_OUT, PTS5_IN, 0,
@@ -1873,9 +1873,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTS3_FN, PTS3_OUT, PTS3_IN, 0,
 		PTS2_FN, PTS2_OUT, PTS2_IN, 0,
 		PTS1_FN, PTS1_OUT, PTS1_IN, 0,
-		PTS0_FN, PTS0_OUT, PTS0_IN, 0 }
+		PTS0_FN, PTS0_OUT, PTS0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PTCR", 0xffec0026, 16, 2) {
+	{ PINMUX_CFG_REG("PTCR", 0xffec0026, 16, 2, GROUP(
 		PTT7_FN, PTT7_OUT, PTT7_IN, 0,
 		PTT6_FN, PTT6_OUT, PTT6_IN, 0,
 		PTT5_FN, PTT5_OUT, PTT5_IN, 0,
@@ -1883,9 +1883,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTT3_FN, PTT3_OUT, PTT3_IN, 0,
 		PTT2_FN, PTT2_OUT, PTT2_IN, 0,
 		PTT1_FN, PTT1_OUT, PTT1_IN, 0,
-		PTT0_FN, PTT0_OUT, PTT0_IN, 0 }
+		PTT0_FN, PTT0_OUT, PTT0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PUCR", 0xffec0028, 16, 2) {
+	{ PINMUX_CFG_REG("PUCR", 0xffec0028, 16, 2, GROUP(
 		PTU7_FN, PTU7_OUT, PTU7_IN, 0,
 		PTU6_FN, PTU6_OUT, PTU6_IN, 0,
 		PTU5_FN, PTU5_OUT, PTU5_IN, 0,
@@ -1893,9 +1893,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTU3_FN, PTU3_OUT, PTU3_IN, 0,
 		PTU2_FN, PTU2_OUT, PTU2_IN, 0,
 		PTU1_FN, PTU1_OUT, PTU1_IN, 0,
-		PTU0_FN, PTU0_OUT, PTU0_IN, 0 }
+		PTU0_FN, PTU0_OUT, PTU0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PVCR", 0xffec002a, 16, 2) {
+	{ PINMUX_CFG_REG("PVCR", 0xffec002a, 16, 2, GROUP(
 		PTV7_FN, PTV7_OUT, PTV7_IN, 0,
 		PTV6_FN, PTV6_OUT, PTV6_IN, 0,
 		PTV5_FN, PTV5_OUT, PTV5_IN, 0,
@@ -1903,9 +1903,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTV3_FN, PTV3_OUT, PTV3_IN, 0,
 		PTV2_FN, PTV2_OUT, PTV2_IN, 0,
 		PTV1_FN, PTV1_OUT, PTV1_IN, 0,
-		PTV0_FN, PTV0_OUT, PTV0_IN, 0 }
+		PTV0_FN, PTV0_OUT, PTV0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PWCR", 0xffec002c, 16, 2) {
+	{ PINMUX_CFG_REG("PWCR", 0xffec002c, 16, 2, GROUP(
 		PTW7_FN, PTW7_OUT, PTW7_IN, 0,
 		PTW6_FN, PTW6_OUT, PTW6_IN, 0,
 		PTW5_FN, PTW5_OUT, PTW5_IN, 0,
@@ -1913,9 +1913,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTW3_FN, PTW3_OUT, PTW3_IN, 0,
 		PTW2_FN, PTW2_OUT, PTW2_IN, 0,
 		PTW1_FN, PTW1_OUT, PTW1_IN, 0,
-		PTW0_FN, PTW0_OUT, PTW0_IN, 0 }
+		PTW0_FN, PTW0_OUT, PTW0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PXCR", 0xffec002e, 16, 2) {
+	{ PINMUX_CFG_REG("PXCR", 0xffec002e, 16, 2, GROUP(
 		PTX7_FN, PTX7_OUT, PTX7_IN, 0,
 		PTX6_FN, PTX6_OUT, PTX6_IN, 0,
 		PTX5_FN, PTX5_OUT, PTX5_IN, 0,
@@ -1923,9 +1923,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTX3_FN, PTX3_OUT, PTX3_IN, 0,
 		PTX2_FN, PTX2_OUT, PTX2_IN, 0,
 		PTX1_FN, PTX1_OUT, PTX1_IN, 0,
-		PTX0_FN, PTX0_OUT, PTX0_IN, 0 }
+		PTX0_FN, PTX0_OUT, PTX0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PYCR", 0xffec0030, 16, 2) {
+	{ PINMUX_CFG_REG("PYCR", 0xffec0030, 16, 2, GROUP(
 		PTY7_FN, PTY7_OUT, PTY7_IN, 0,
 		PTY6_FN, PTY6_OUT, PTY6_IN, 0,
 		PTY5_FN, PTY5_OUT, PTY5_IN, 0,
@@ -1933,9 +1933,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTY3_FN, PTY3_OUT, PTY3_IN, 0,
 		PTY2_FN, PTY2_OUT, PTY2_IN, 0,
 		PTY1_FN, PTY1_OUT, PTY1_IN, 0,
-		PTY0_FN, PTY0_OUT, PTY0_IN, 0 }
+		PTY0_FN, PTY0_OUT, PTY0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PZCR", 0xffec0032, 16, 2) {
+	{ PINMUX_CFG_REG("PZCR", 0xffec0032, 16, 2, GROUP(
 		PTZ7_FN, PTZ7_OUT, PTZ7_IN, 0,
 		PTZ6_FN, PTZ6_OUT, PTZ6_IN, 0,
 		PTZ5_FN, PTZ5_OUT, PTZ5_IN, 0,
@@ -1943,10 +1943,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PTZ3_FN, PTZ3_OUT, PTZ3_IN, 0,
 		PTZ2_FN, PTZ2_OUT, PTZ2_IN, 0,
 		PTZ1_FN, PTZ1_OUT, PTZ1_IN, 0,
-		PTZ0_FN, PTZ0_OUT, PTZ0_IN, 0 }
+		PTZ0_FN, PTZ0_OUT, PTZ0_IN, 0 ))
 	},
 
-	{ PINMUX_CFG_REG("PSEL0", 0xffec0070, 16, 1) {
+	{ PINMUX_CFG_REG("PSEL0", 0xffec0070, 16, 1, GROUP(
 		PS0_15_FN1, PS0_15_FN2,
 		PS0_14_FN1, PS0_14_FN2,
 		PS0_13_FN1, PS0_13_FN2,
@@ -1962,9 +1962,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PS0_3_FN1, PS0_3_FN2,
 		PS0_2_FN1, PS0_2_FN2,
 		0, 0,
-		0, 0, }
+		0, 0, ))
 	},
-	{ PINMUX_CFG_REG("PSEL1", 0xffec0072, 16, 1) {
+	{ PINMUX_CFG_REG("PSEL1", 0xffec0072, 16, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -1980,9 +1980,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0,
 		PS1_2_FN1, PS1_2_FN2,
 		0, 0,
-		0, 0, }
+		0, 0, ))
 	},
-	{ PINMUX_CFG_REG("PSEL2", 0xffec0074, 16, 1) {
+	{ PINMUX_CFG_REG("PSEL2", 0xffec0074, 16, 1, GROUP(
 		0, 0,
 		0, 0,
 		PS2_13_FN1, PS2_13_FN2,
@@ -1998,9 +1998,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0,
 		PS2_2_FN1, PS2_2_FN2,
 		0, 0,
-		0, 0, }
+		0, 0, ))
 	},
-	{ PINMUX_CFG_REG("PSEL3", 0xffec0076, 16, 1) {
+	{ PINMUX_CFG_REG("PSEL3", 0xffec0076, 16, 1, GROUP(
 		PS3_15_FN1, PS3_15_FN2,
 		PS3_14_FN1, PS3_14_FN2,
 		PS3_13_FN1, PS3_13_FN2,
@@ -2016,10 +2016,10 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0,
 		PS3_2_FN1, PS3_2_FN2,
 		PS3_1_FN1, PS3_1_FN2,
-		0, 0, }
+		0, 0, ))
 	},
 
-	{ PINMUX_CFG_REG("PSEL4", 0xffec0078, 16, 1) {
+	{ PINMUX_CFG_REG("PSEL4", 0xffec0078, 16, 1, GROUP(
 		0, 0,
 		PS4_14_FN1, PS4_14_FN2,
 		PS4_13_FN1, PS4_13_FN2,
@@ -2035,9 +2035,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PS4_3_FN1, PS4_3_FN2,
 		PS4_2_FN1, PS4_2_FN2,
 		PS4_1_FN1, PS4_1_FN2,
-		PS4_0_FN1, PS4_0_FN2, }
+		PS4_0_FN1, PS4_0_FN2, ))
 	},
-	{ PINMUX_CFG_REG("PSEL5", 0xffec007a, 16, 1) {
+	{ PINMUX_CFG_REG("PSEL5", 0xffec007a, 16, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -2053,9 +2053,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PS5_3_FN1, PS5_3_FN2,
 		PS5_2_FN1, PS5_2_FN2,
 		0, 0,
-		0, 0, }
+		0, 0, ))
 	},
-	{ PINMUX_CFG_REG("PSEL6", 0xffec007c, 16, 1) {
+	{ PINMUX_CFG_REG("PSEL6", 0xffec007c, 16, 1, GROUP(
 		PS6_15_FN1, PS6_15_FN2,
 		PS6_14_FN1, PS6_14_FN2,
 		PS6_13_FN1, PS6_13_FN2,
@@ -2071,9 +2071,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PS6_3_FN1, PS6_3_FN2,
 		PS6_2_FN1, PS6_2_FN2,
 		PS6_1_FN1, PS6_1_FN2,
-		PS6_0_FN1, PS6_0_FN2, }
+		PS6_0_FN1, PS6_0_FN2, ))
 	},
-	{ PINMUX_CFG_REG("PSEL7", 0xffec0082, 16, 1) {
+	{ PINMUX_CFG_REG("PSEL7", 0xffec0082, 16, 1, GROUP(
 		PS7_15_FN1, PS7_15_FN2,
 		PS7_14_FN1, PS7_14_FN2,
 		PS7_13_FN1, PS7_13_FN2,
@@ -2089,9 +2089,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0,
 		0, 0,
 		0, 0,
-		0, 0, }
+		0, 0, ))
 	},
-	{ PINMUX_CFG_REG("PSEL8", 0xffec0084, 16, 1) {
+	{ PINMUX_CFG_REG("PSEL8", 0xffec0084, 16, 1, GROUP(
 		PS8_15_FN1, PS8_15_FN2,
 		PS8_14_FN1, PS8_14_FN2,
 		PS8_13_FN1, PS8_13_FN2,
@@ -2107,115 +2107,115 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0,
 		0, 0,
 		0, 0,
-		0, 0, }
+		0, 0, ))
 	},
 	{}
 };
 
 static const struct pinmux_data_reg pinmux_data_regs[] = {
-	{ PINMUX_DATA_REG("PADR", 0xffec0034, 8) {
+	{ PINMUX_DATA_REG("PADR", 0xffec0034, 8, GROUP(
 		PTA7_DATA, PTA6_DATA, PTA5_DATA, PTA4_DATA,
-		PTA3_DATA, PTA2_DATA, PTA1_DATA, PTA0_DATA }
+		PTA3_DATA, PTA2_DATA, PTA1_DATA, PTA0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PBDR", 0xffec0036, 8) {
+	{ PINMUX_DATA_REG("PBDR", 0xffec0036, 8, GROUP(
 		PTB7_DATA, PTB6_DATA, PTB5_DATA, PTB4_DATA,
-		PTB3_DATA, PTB2_DATA, PTB1_DATA, PTB0_DATA }
+		PTB3_DATA, PTB2_DATA, PTB1_DATA, PTB0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PCDR", 0xffec0038, 8) {
+	{ PINMUX_DATA_REG("PCDR", 0xffec0038, 8, GROUP(
 		PTC7_DATA, PTC6_DATA, PTC5_DATA, PTC4_DATA,
-		PTC3_DATA, PTC2_DATA, PTC1_DATA, PTC0_DATA }
+		PTC3_DATA, PTC2_DATA, PTC1_DATA, PTC0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PDDR", 0xffec003a, 8) {
+	{ PINMUX_DATA_REG("PDDR", 0xffec003a, 8, GROUP(
 		PTD7_DATA, PTD6_DATA, PTD5_DATA, PTD4_DATA,
-		PTD3_DATA, PTD2_DATA, PTD1_DATA, PTD0_DATA }
+		PTD3_DATA, PTD2_DATA, PTD1_DATA, PTD0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PEDR", 0xffec003c, 8) {
+	{ PINMUX_DATA_REG("PEDR", 0xffec003c, 8, GROUP(
 		PTE7_DATA, PTE6_DATA, PTE5_DATA, PTE4_DATA,
-		PTE3_DATA, PTE2_DATA, PTE1_DATA, PTE0_DATA }
+		PTE3_DATA, PTE2_DATA, PTE1_DATA, PTE0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PFDR", 0xffec003e, 8) {
+	{ PINMUX_DATA_REG("PFDR", 0xffec003e, 8, GROUP(
 		PTF7_DATA, PTF6_DATA, PTF5_DATA, PTF4_DATA,
-		PTF3_DATA, PTF2_DATA, PTF1_DATA, PTF0_DATA }
+		PTF3_DATA, PTF2_DATA, PTF1_DATA, PTF0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PGDR", 0xffec0040, 8) {
+	{ PINMUX_DATA_REG("PGDR", 0xffec0040, 8, GROUP(
 		PTG7_DATA, PTG6_DATA, PTG5_DATA, PTG4_DATA,
-		PTG3_DATA, PTG2_DATA, PTG1_DATA, PTG0_DATA }
+		PTG3_DATA, PTG2_DATA, PTG1_DATA, PTG0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PHDR", 0xffec0042, 8) {
+	{ PINMUX_DATA_REG("PHDR", 0xffec0042, 8, GROUP(
 		PTH7_DATA, PTH6_DATA, PTH5_DATA, PTH4_DATA,
-		PTH3_DATA, PTH2_DATA, PTH1_DATA, PTH0_DATA }
+		PTH3_DATA, PTH2_DATA, PTH1_DATA, PTH0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PIDR", 0xffec0044, 8) {
+	{ PINMUX_DATA_REG("PIDR", 0xffec0044, 8, GROUP(
 		PTI7_DATA, PTI6_DATA, PTI5_DATA, PTI4_DATA,
-		PTI3_DATA, PTI2_DATA, PTI1_DATA, PTI0_DATA }
+		PTI3_DATA, PTI2_DATA, PTI1_DATA, PTI0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PJDR", 0xffec0046, 8) {
+	{ PINMUX_DATA_REG("PJDR", 0xffec0046, 8, GROUP(
 		0, PTJ6_DATA, PTJ5_DATA, PTJ4_DATA,
-		PTJ3_DATA, PTJ2_DATA, PTJ1_DATA, PTJ0_DATA }
+		PTJ3_DATA, PTJ2_DATA, PTJ1_DATA, PTJ0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PKDR", 0xffec0048, 8) {
+	{ PINMUX_DATA_REG("PKDR", 0xffec0048, 8, GROUP(
 		PTK7_DATA, PTK6_DATA, PTK5_DATA, PTK4_DATA,
-		PTK3_DATA, PTK2_DATA, PTK1_DATA, PTK0_DATA }
+		PTK3_DATA, PTK2_DATA, PTK1_DATA, PTK0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PLDR", 0xffec004a, 8) {
+	{ PINMUX_DATA_REG("PLDR", 0xffec004a, 8, GROUP(
 		0, PTL6_DATA, PTL5_DATA, PTL4_DATA,
-		PTL3_DATA, PTL2_DATA, PTL1_DATA, PTL0_DATA }
+		PTL3_DATA, PTL2_DATA, PTL1_DATA, PTL0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PMDR", 0xffec004c, 8) {
+	{ PINMUX_DATA_REG("PMDR", 0xffec004c, 8, GROUP(
 		PTM7_DATA, PTM6_DATA, PTM5_DATA, PTM4_DATA,
-		PTM3_DATA, PTM2_DATA, PTM1_DATA, PTM0_DATA }
+		PTM3_DATA, PTM2_DATA, PTM1_DATA, PTM0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PNDR", 0xffec004e, 8) {
+	{ PINMUX_DATA_REG("PNDR", 0xffec004e, 8, GROUP(
 		0, PTN6_DATA, PTN5_DATA, PTN4_DATA,
-		PTN3_DATA, PTN2_DATA, PTN1_DATA, PTN0_DATA }
+		PTN3_DATA, PTN2_DATA, PTN1_DATA, PTN0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PODR", 0xffec0050, 8) {
+	{ PINMUX_DATA_REG("PODR", 0xffec0050, 8, GROUP(
 		PTO7_DATA, PTO6_DATA, PTO5_DATA, PTO4_DATA,
-		PTO3_DATA, PTO2_DATA, PTO1_DATA, PTO0_DATA }
+		PTO3_DATA, PTO2_DATA, PTO1_DATA, PTO0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PPDR", 0xffec0052, 8) {
+	{ PINMUX_DATA_REG("PPDR", 0xffec0052, 8, GROUP(
 		PTP7_DATA, PTP6_DATA, PTP5_DATA, PTP4_DATA,
-		PTP3_DATA, PTP2_DATA, PTP1_DATA, PTP0_DATA }
+		PTP3_DATA, PTP2_DATA, PTP1_DATA, PTP0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PQDR", 0xffec0054, 8) {
+	{ PINMUX_DATA_REG("PQDR", 0xffec0054, 8, GROUP(
 		0, PTQ6_DATA, PTQ5_DATA, PTQ4_DATA,
-		PTQ3_DATA, PTQ2_DATA, PTQ1_DATA, PTQ0_DATA }
+		PTQ3_DATA, PTQ2_DATA, PTQ1_DATA, PTQ0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PRDR", 0xffec0056, 8) {
+	{ PINMUX_DATA_REG("PRDR", 0xffec0056, 8, GROUP(
 		PTR7_DATA, PTR6_DATA, PTR5_DATA, PTR4_DATA,
-		PTR3_DATA, PTR2_DATA, PTR1_DATA, PTR0_DATA }
+		PTR3_DATA, PTR2_DATA, PTR1_DATA, PTR0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PSDR", 0xffec0058, 8) {
+	{ PINMUX_DATA_REG("PSDR", 0xffec0058, 8, GROUP(
 		PTS7_DATA, PTS6_DATA, PTS5_DATA, PTS4_DATA,
-		PTS3_DATA, PTS2_DATA, PTS1_DATA, PTS0_DATA }
+		PTS3_DATA, PTS2_DATA, PTS1_DATA, PTS0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PTDR", 0xffec005a, 8) {
+	{ PINMUX_DATA_REG("PTDR", 0xffec005a, 8, GROUP(
 		PTT7_DATA, PTT6_DATA, PTT5_DATA, PTT4_DATA,
-		PTT3_DATA, PTT2_DATA, PTT1_DATA, PTT0_DATA }
+		PTT3_DATA, PTT2_DATA, PTT1_DATA, PTT0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PUDR", 0xffec005c, 8) {
+	{ PINMUX_DATA_REG("PUDR", 0xffec005c, 8, GROUP(
 		PTU7_DATA, PTU6_DATA, PTU5_DATA, PTU4_DATA,
-		PTU3_DATA, PTU2_DATA, PTU1_DATA, PTU0_DATA }
+		PTU3_DATA, PTU2_DATA, PTU1_DATA, PTU0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PVDR", 0xffec005e, 8) {
+	{ PINMUX_DATA_REG("PVDR", 0xffec005e, 8, GROUP(
 		PTV7_DATA, PTV6_DATA, PTV5_DATA, PTV4_DATA,
-		PTV3_DATA, PTV2_DATA, PTV1_DATA, PTV0_DATA }
+		PTV3_DATA, PTV2_DATA, PTV1_DATA, PTV0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PWDR", 0xffec0060, 8) {
+	{ PINMUX_DATA_REG("PWDR", 0xffec0060, 8, GROUP(
 		PTW7_DATA, PTW6_DATA, PTW5_DATA, PTW4_DATA,
-		PTW3_DATA, PTW2_DATA, PTW1_DATA, PTW0_DATA }
+		PTW3_DATA, PTW2_DATA, PTW1_DATA, PTW0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PXDR", 0xffec0062, 8) {
+	{ PINMUX_DATA_REG("PXDR", 0xffec0062, 8, GROUP(
 		PTX7_DATA, PTX6_DATA, PTX5_DATA, PTX4_DATA,
-		PTX3_DATA, PTX2_DATA, PTX1_DATA, PTX0_DATA }
+		PTX3_DATA, PTX2_DATA, PTX1_DATA, PTX0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PYDR", 0xffec0064, 8) {
+	{ PINMUX_DATA_REG("PYDR", 0xffec0064, 8, GROUP(
 		PTY7_DATA, PTY6_DATA, PTY5_DATA, PTY4_DATA,
-		PTY3_DATA, PTY2_DATA, PTY1_DATA, PTY0_DATA }
+		PTY3_DATA, PTY2_DATA, PTY1_DATA, PTY0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PZDR", 0xffec0066, 8) {
+	{ PINMUX_DATA_REG("PZDR", 0xffec0066, 8, GROUP(
 		PTZ7_DATA, PTZ6_DATA, PTZ5_DATA, PTZ4_DATA,
-		PTZ3_DATA, PTZ2_DATA, PTZ1_DATA, PTZ0_DATA }
+		PTZ3_DATA, PTZ2_DATA, PTZ1_DATA, PTZ0_DATA ))
 	},
 	{ },
 };
diff --git a/drivers/pinctrl/sh-pfc/pfc-sh7785.c b/drivers/pinctrl/sh-pfc/pfc-sh7785.c
index 193179f7fdd9..c4c1e288c53e 100644
--- a/drivers/pinctrl/sh-pfc/pfc-sh7785.c
+++ b/drivers/pinctrl/sh-pfc/pfc-sh7785.c
@@ -985,7 +985,7 @@ static const struct pinmux_func pinmux_func_gpios[] = {
 };
 
 static const struct pinmux_cfg_reg pinmux_config_regs[] = {
-	{ PINMUX_CFG_REG("PACR", 0xffe70000, 16, 2) {
+	{ PINMUX_CFG_REG("PACR", 0xffe70000, 16, 2, GROUP(
 		PA7_FN, PA7_OUT, PA7_IN, 0,
 		PA6_FN, PA6_OUT, PA6_IN, 0,
 		PA5_FN, PA5_OUT, PA5_IN, 0,
@@ -993,9 +993,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PA3_FN, PA3_OUT, PA3_IN, 0,
 		PA2_FN, PA2_OUT, PA2_IN, 0,
 		PA1_FN, PA1_OUT, PA1_IN, 0,
-		PA0_FN, PA0_OUT, PA0_IN, 0 }
+		PA0_FN, PA0_OUT, PA0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PBCR", 0xffe70002, 16, 2) {
+	{ PINMUX_CFG_REG("PBCR", 0xffe70002, 16, 2, GROUP(
 		PB7_FN, PB7_OUT, PB7_IN, 0,
 		PB6_FN, PB6_OUT, PB6_IN, 0,
 		PB5_FN, PB5_OUT, PB5_IN, 0,
@@ -1003,9 +1003,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PB3_FN, PB3_OUT, PB3_IN, 0,
 		PB2_FN, PB2_OUT, PB2_IN, 0,
 		PB1_FN, PB1_OUT, PB1_IN, 0,
-		PB0_FN, PB0_OUT, PB0_IN, 0 }
+		PB0_FN, PB0_OUT, PB0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PCCR", 0xffe70004, 16, 2) {
+	{ PINMUX_CFG_REG("PCCR", 0xffe70004, 16, 2, GROUP(
 		PC7_FN, PC7_OUT, PC7_IN, 0,
 		PC6_FN, PC6_OUT, PC6_IN, 0,
 		PC5_FN, PC5_OUT, PC5_IN, 0,
@@ -1013,9 +1013,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PC3_FN, PC3_OUT, PC3_IN, 0,
 		PC2_FN, PC2_OUT, PC2_IN, 0,
 		PC1_FN, PC1_OUT, PC1_IN, 0,
-		PC0_FN, PC0_OUT, PC0_IN, 0 }
+		PC0_FN, PC0_OUT, PC0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PDCR", 0xffe70006, 16, 2) {
+	{ PINMUX_CFG_REG("PDCR", 0xffe70006, 16, 2, GROUP(
 		PD7_FN, PD7_OUT, PD7_IN, 0,
 		PD6_FN, PD6_OUT, PD6_IN, 0,
 		PD5_FN, PD5_OUT, PD5_IN, 0,
@@ -1023,9 +1023,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PD3_FN, PD3_OUT, PD3_IN, 0,
 		PD2_FN, PD2_OUT, PD2_IN, 0,
 		PD1_FN, PD1_OUT, PD1_IN, 0,
-		PD0_FN, PD0_OUT, PD0_IN, 0 }
+		PD0_FN, PD0_OUT, PD0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PECR", 0xffe70008, 16, 2) {
+	{ PINMUX_CFG_REG("PECR", 0xffe70008, 16, 2, GROUP(
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		PE5_FN, PE5_OUT, PE5_IN, 0,
@@ -1033,9 +1033,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PE3_FN, PE3_OUT, PE3_IN, 0,
 		PE2_FN, PE2_OUT, PE2_IN, 0,
 		PE1_FN, PE1_OUT, PE1_IN, 0,
-		PE0_FN, PE0_OUT, PE0_IN, 0 }
+		PE0_FN, PE0_OUT, PE0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PFCR", 0xffe7000a, 16, 2) {
+	{ PINMUX_CFG_REG("PFCR", 0xffe7000a, 16, 2, GROUP(
 		PF7_FN, PF7_OUT, PF7_IN, 0,
 		PF6_FN, PF6_OUT, PF6_IN, 0,
 		PF5_FN, PF5_OUT, PF5_IN, 0,
@@ -1043,9 +1043,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PF3_FN, PF3_OUT, PF3_IN, 0,
 		PF2_FN, PF2_OUT, PF2_IN, 0,
 		PF1_FN, PF1_OUT, PF1_IN, 0,
-		PF0_FN, PF0_OUT, PF0_IN, 0 }
+		PF0_FN, PF0_OUT, PF0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PGCR", 0xffe7000c, 16, 2) {
+	{ PINMUX_CFG_REG("PGCR", 0xffe7000c, 16, 2, GROUP(
 		PG7_FN, PG7_OUT, PG7_IN, 0,
 		PG6_FN, PG6_OUT, PG6_IN, 0,
 		PG5_FN, PG5_OUT, PG5_IN, 0,
@@ -1053,9 +1053,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PG3_FN, PG3_OUT, PG3_IN, 0,
 		PG2_FN, PG2_OUT, PG2_IN, 0,
 		PG1_FN, PG1_OUT, PG1_IN, 0,
-		PG0_FN, PG0_OUT, PG0_IN, 0 }
+		PG0_FN, PG0_OUT, PG0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PHCR", 0xffe7000e, 16, 2) {
+	{ PINMUX_CFG_REG("PHCR", 0xffe7000e, 16, 2, GROUP(
 		PH7_FN, PH7_OUT, PH7_IN, 0,
 		PH6_FN, PH6_OUT, PH6_IN, 0,
 		PH5_FN, PH5_OUT, PH5_IN, 0,
@@ -1063,9 +1063,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PH3_FN, PH3_OUT, PH3_IN, 0,
 		PH2_FN, PH2_OUT, PH2_IN, 0,
 		PH1_FN, PH1_OUT, PH1_IN, 0,
-		PH0_FN, PH0_OUT, PH0_IN, 0 }
+		PH0_FN, PH0_OUT, PH0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PJCR", 0xffe70010, 16, 2) {
+	{ PINMUX_CFG_REG("PJCR", 0xffe70010, 16, 2, GROUP(
 		PJ7_FN, PJ7_OUT, PJ7_IN, 0,
 		PJ6_FN, PJ6_OUT, PJ6_IN, 0,
 		PJ5_FN, PJ5_OUT, PJ5_IN, 0,
@@ -1073,9 +1073,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PJ3_FN, PJ3_OUT, PJ3_IN, 0,
 		PJ2_FN, PJ2_OUT, PJ2_IN, 0,
 		PJ1_FN, PJ1_OUT, PJ1_IN, 0,
-		PJ0_FN, PJ0_OUT, PJ0_IN, 0 }
+		PJ0_FN, PJ0_OUT, PJ0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PKCR", 0xffe70012, 16, 2) {
+	{ PINMUX_CFG_REG("PKCR", 0xffe70012, 16, 2, GROUP(
 		PK7_FN, PK7_OUT, PK7_IN, 0,
 		PK6_FN, PK6_OUT, PK6_IN, 0,
 		PK5_FN, PK5_OUT, PK5_IN, 0,
@@ -1083,9 +1083,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PK3_FN, PK3_OUT, PK3_IN, 0,
 		PK2_FN, PK2_OUT, PK2_IN, 0,
 		PK1_FN, PK1_OUT, PK1_IN, 0,
-		PK0_FN, PK0_OUT, PK0_IN, 0 }
+		PK0_FN, PK0_OUT, PK0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PLCR", 0xffe70014, 16, 2) {
+	{ PINMUX_CFG_REG("PLCR", 0xffe70014, 16, 2, GROUP(
 		PL7_FN, PL7_OUT, PL7_IN, 0,
 		PL6_FN, PL6_OUT, PL6_IN, 0,
 		PL5_FN, PL5_OUT, PL5_IN, 0,
@@ -1093,9 +1093,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PL3_FN, PL3_OUT, PL3_IN, 0,
 		PL2_FN, PL2_OUT, PL2_IN, 0,
 		PL1_FN, PL1_OUT, PL1_IN, 0,
-		PL0_FN, PL0_OUT, PL0_IN, 0 }
+		PL0_FN, PL0_OUT, PL0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PMCR", 0xffe70016, 16, 2) {
+	{ PINMUX_CFG_REG("PMCR", 0xffe70016, 16, 2, GROUP(
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		0, 0, 0, 0,
@@ -1103,9 +1103,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		PM1_FN, PM1_OUT, PM1_IN, 0,
-		PM0_FN, PM0_OUT, PM0_IN, 0 }
+		PM0_FN, PM0_OUT, PM0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PNCR", 0xffe70018, 16, 2) {
+	{ PINMUX_CFG_REG("PNCR", 0xffe70018, 16, 2, GROUP(
 		PN7_FN, PN7_OUT, PN7_IN, 0,
 		PN6_FN, PN6_OUT, PN6_IN, 0,
 		PN5_FN, PN5_OUT, PN5_IN, 0,
@@ -1113,9 +1113,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PN3_FN, PN3_OUT, PN3_IN, 0,
 		PN2_FN, PN2_OUT, PN2_IN, 0,
 		PN1_FN, PN1_OUT, PN1_IN, 0,
-		PN0_FN, PN0_OUT, PN0_IN, 0 }
+		PN0_FN, PN0_OUT, PN0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PPCR", 0xffe7001a, 16, 2) {
+	{ PINMUX_CFG_REG("PPCR", 0xffe7001a, 16, 2, GROUP(
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		PP5_FN, PP5_OUT, PP5_IN, 0,
@@ -1123,9 +1123,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PP3_FN, PP3_OUT, PP3_IN, 0,
 		PP2_FN, PP2_OUT, PP2_IN, 0,
 		PP1_FN, PP1_OUT, PP1_IN, 0,
-		PP0_FN, PP0_OUT, PP0_IN, 0 }
+		PP0_FN, PP0_OUT, PP0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PQCR", 0xffe7001c, 16, 2) {
+	{ PINMUX_CFG_REG("PQCR", 0xffe7001c, 16, 2, GROUP(
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		0, 0, 0, 0,
@@ -1133,9 +1133,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PQ3_FN, PQ3_OUT, PQ3_IN, 0,
 		PQ2_FN, PQ2_OUT, PQ2_IN, 0,
 		PQ1_FN, PQ1_OUT, PQ1_IN, 0,
-		PQ0_FN, PQ0_OUT, PQ0_IN, 0 }
+		PQ0_FN, PQ0_OUT, PQ0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PRCR", 0xffe7001e, 16, 2) {
+	{ PINMUX_CFG_REG("PRCR", 0xffe7001e, 16, 2, GROUP(
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		0, 0, 0, 0,
@@ -1143,9 +1143,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PR3_FN, PR3_OUT, PR3_IN, 0,
 		PR2_FN, PR2_OUT, PR2_IN, 0,
 		PR1_FN, PR1_OUT, PR1_IN, 0,
-		PR0_FN, PR0_OUT, PR0_IN, 0 }
+		PR0_FN, PR0_OUT, PR0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("P1MSELR", 0xffe70080, 16, 1) {
+	{ PINMUX_CFG_REG("P1MSELR", 0xffe70080, 16, 1, GROUP(
 		P1MSEL15_0, P1MSEL15_1,
 		P1MSEL14_0, P1MSEL14_1,
 		P1MSEL13_0, P1MSEL13_1,
@@ -1161,9 +1161,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		P1MSEL3_0, P1MSEL3_1,
 		P1MSEL2_0, P1MSEL2_1,
 		P1MSEL1_0, P1MSEL1_1,
-		P1MSEL0_0, P1MSEL0_1 }
+		P1MSEL0_0, P1MSEL0_1 ))
 	},
-	{ PINMUX_CFG_REG("P2MSELR", 0xffe70082, 16, 1) {
+	{ PINMUX_CFG_REG("P2MSELR", 0xffe70082, 16, 1, GROUP(
 		0, 0,
 		0, 0,
 		0, 0,
@@ -1179,75 +1179,75 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0,
 		P2MSEL2_0, P2MSEL2_1,
 		P2MSEL1_0, P2MSEL1_1,
-		P2MSEL0_0, P2MSEL0_1 }
+		P2MSEL0_0, P2MSEL0_1 ))
 	},
 	{}
 };
 
 static const struct pinmux_data_reg pinmux_data_regs[] = {
-	{ PINMUX_DATA_REG("PADR", 0xffe70020, 8) {
+	{ PINMUX_DATA_REG("PADR", 0xffe70020, 8, GROUP(
 		PA7_DATA, PA6_DATA, PA5_DATA, PA4_DATA,
-		PA3_DATA, PA2_DATA, PA1_DATA, PA0_DATA }
+		PA3_DATA, PA2_DATA, PA1_DATA, PA0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PBDR", 0xffe70022, 8) {
+	{ PINMUX_DATA_REG("PBDR", 0xffe70022, 8, GROUP(
 		PB7_DATA, PB6_DATA, PB5_DATA, PB4_DATA,
-		PB3_DATA, PB2_DATA, PB1_DATA, PB0_DATA }
+		PB3_DATA, PB2_DATA, PB1_DATA, PB0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PCDR", 0xffe70024, 8) {
+	{ PINMUX_DATA_REG("PCDR", 0xffe70024, 8, GROUP(
 		PC7_DATA, PC6_DATA, PC5_DATA, PC4_DATA,
-		PC3_DATA, PC2_DATA, PC1_DATA, PC0_DATA }
+		PC3_DATA, PC2_DATA, PC1_DATA, PC0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PDDR", 0xffe70026, 8) {
+	{ PINMUX_DATA_REG("PDDR", 0xffe70026, 8, GROUP(
 		PD7_DATA, PD6_DATA, PD5_DATA, PD4_DATA,
-		PD3_DATA, PD2_DATA, PD1_DATA, PD0_DATA }
+		PD3_DATA, PD2_DATA, PD1_DATA, PD0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PEDR", 0xffe70028, 8) {
+	{ PINMUX_DATA_REG("PEDR", 0xffe70028, 8, GROUP(
 		0, 0, PE5_DATA, PE4_DATA,
-		PE3_DATA, PE2_DATA, PE1_DATA, PE0_DATA }
+		PE3_DATA, PE2_DATA, PE1_DATA, PE0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PFDR", 0xffe7002a, 8) {
+	{ PINMUX_DATA_REG("PFDR", 0xffe7002a, 8, GROUP(
 		PF7_DATA, PF6_DATA, PF5_DATA, PF4_DATA,
-		PF3_DATA, PF2_DATA, PF1_DATA, PF0_DATA }
+		PF3_DATA, PF2_DATA, PF1_DATA, PF0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PGDR", 0xffe7002c, 8) {
+	{ PINMUX_DATA_REG("PGDR", 0xffe7002c, 8, GROUP(
 		PG7_DATA, PG6_DATA, PG5_DATA, PG4_DATA,
-		PG3_DATA, PG2_DATA, PG1_DATA, PG0_DATA }
+		PG3_DATA, PG2_DATA, PG1_DATA, PG0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PHDR", 0xffe7002e, 8) {
+	{ PINMUX_DATA_REG("PHDR", 0xffe7002e, 8, GROUP(
 		PH7_DATA, PH6_DATA, PH5_DATA, PH4_DATA,
-		PH3_DATA, PH2_DATA, PH1_DATA, PH0_DATA }
+		PH3_DATA, PH2_DATA, PH1_DATA, PH0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PJDR", 0xffe70030, 8) {
+	{ PINMUX_DATA_REG("PJDR", 0xffe70030, 8, GROUP(
 		PJ7_DATA, PJ6_DATA, PJ5_DATA, PJ4_DATA,
-		PJ3_DATA, PJ2_DATA, PJ1_DATA, PJ0_DATA }
+		PJ3_DATA, PJ2_DATA, PJ1_DATA, PJ0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PKDR", 0xffe70032, 8) {
+	{ PINMUX_DATA_REG("PKDR", 0xffe70032, 8, GROUP(
 		PK7_DATA, PK6_DATA, PK5_DATA, PK4_DATA,
-		PK3_DATA, PK2_DATA, PK1_DATA, PK0_DATA }
+		PK3_DATA, PK2_DATA, PK1_DATA, PK0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PLDR", 0xffe70034, 8) {
+	{ PINMUX_DATA_REG("PLDR", 0xffe70034, 8, GROUP(
 		PL7_DATA, PL6_DATA, PL5_DATA, PL4_DATA,
-		PL3_DATA, PL2_DATA, PL1_DATA, PL0_DATA }
+		PL3_DATA, PL2_DATA, PL1_DATA, PL0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PMDR", 0xffe70036, 8) {
+	{ PINMUX_DATA_REG("PMDR", 0xffe70036, 8, GROUP(
 		0, 0, 0, 0,
-		0, 0, PM1_DATA, PM0_DATA }
+		0, 0, PM1_DATA, PM0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PNDR", 0xffe70038, 8) {
+	{ PINMUX_DATA_REG("PNDR", 0xffe70038, 8, GROUP(
 		PN7_DATA, PN6_DATA, PN5_DATA, PN4_DATA,
-		PN3_DATA, PN2_DATA, PN1_DATA, PN0_DATA }
+		PN3_DATA, PN2_DATA, PN1_DATA, PN0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PPDR", 0xffe7003a, 8) {
+	{ PINMUX_DATA_REG("PPDR", 0xffe7003a, 8, GROUP(
 		0, 0, PP5_DATA, PP4_DATA,
-		PP3_DATA, PP2_DATA, PP1_DATA, PP0_DATA }
+		PP3_DATA, PP2_DATA, PP1_DATA, PP0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PQDR", 0xffe7003c, 8) {
+	{ PINMUX_DATA_REG("PQDR", 0xffe7003c, 8, GROUP(
 		0, 0, 0, PQ4_DATA,
-		PQ3_DATA, PQ2_DATA, PQ1_DATA, PQ0_DATA }
+		PQ3_DATA, PQ2_DATA, PQ1_DATA, PQ0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PRDR", 0xffe7003e, 8) {
+	{ PINMUX_DATA_REG("PRDR", 0xffe7003e, 8, GROUP(
 		0, 0, 0, 0,
-		PR3_DATA, PR2_DATA, PR1_DATA, PR0_DATA }
+		PR3_DATA, PR2_DATA, PR1_DATA, PR0_DATA ))
 	},
 	{ },
 };
diff --git a/drivers/pinctrl/sh-pfc/pfc-sh7786.c b/drivers/pinctrl/sh-pfc/pfc-sh7786.c
index cc2657c4f85c..b8a098cd7721 100644
--- a/drivers/pinctrl/sh-pfc/pfc-sh7786.c
+++ b/drivers/pinctrl/sh-pfc/pfc-sh7786.c
@@ -627,7 +627,7 @@ static const struct pinmux_func pinmux_func_gpios[] = {
 };
 
 static const struct pinmux_cfg_reg pinmux_config_regs[] = {
-	{ PINMUX_CFG_REG("PACR", 0xffcc0000, 16, 2) {
+	{ PINMUX_CFG_REG("PACR", 0xffcc0000, 16, 2, GROUP(
 		PA7_FN, PA7_OUT, PA7_IN, 0,
 		PA6_FN, PA6_OUT, PA6_IN, 0,
 		PA5_FN, PA5_OUT, PA5_IN, 0,
@@ -635,9 +635,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PA3_FN, PA3_OUT, PA3_IN, 0,
 		PA2_FN, PA2_OUT, PA2_IN, 0,
 		PA1_FN, PA1_OUT, PA1_IN, 0,
-		PA0_FN, PA0_OUT, PA0_IN, 0 }
+		PA0_FN, PA0_OUT, PA0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PBCR", 0xffcc0002, 16, 2) {
+	{ PINMUX_CFG_REG("PBCR", 0xffcc0002, 16, 2, GROUP(
 		PB7_FN, PB7_OUT, PB7_IN, 0,
 		PB6_FN, PB6_OUT, PB6_IN, 0,
 		PB5_FN, PB5_OUT, PB5_IN, 0,
@@ -645,9 +645,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PB3_FN, PB3_OUT, PB3_IN, 0,
 		PB2_FN, PB2_OUT, PB2_IN, 0,
 		PB1_FN, PB1_OUT, PB1_IN, 0,
-		PB0_FN, PB0_OUT, PB0_IN, 0 }
+		PB0_FN, PB0_OUT, PB0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PCCR", 0xffcc0004, 16, 2) {
+	{ PINMUX_CFG_REG("PCCR", 0xffcc0004, 16, 2, GROUP(
 		PC7_FN, PC7_OUT, PC7_IN, 0,
 		PC6_FN, PC6_OUT, PC6_IN, 0,
 		PC5_FN, PC5_OUT, PC5_IN, 0,
@@ -655,9 +655,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PC3_FN, PC3_OUT, PC3_IN, 0,
 		PC2_FN, PC2_OUT, PC2_IN, 0,
 		PC1_FN, PC1_OUT, PC1_IN, 0,
-		PC0_FN, PC0_OUT, PC0_IN, 0 }
+		PC0_FN, PC0_OUT, PC0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PDCR", 0xffcc0006, 16, 2) {
+	{ PINMUX_CFG_REG("PDCR", 0xffcc0006, 16, 2, GROUP(
 		PD7_FN, PD7_OUT, PD7_IN, 0,
 		PD6_FN, PD6_OUT, PD6_IN, 0,
 		PD5_FN, PD5_OUT, PD5_IN, 0,
@@ -665,9 +665,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PD3_FN, PD3_OUT, PD3_IN, 0,
 		PD2_FN, PD2_OUT, PD2_IN, 0,
 		PD1_FN, PD1_OUT, PD1_IN, 0,
-		PD0_FN, PD0_OUT, PD0_IN, 0 }
+		PD0_FN, PD0_OUT, PD0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PECR", 0xffcc0008, 16, 2) {
+	{ PINMUX_CFG_REG("PECR", 0xffcc0008, 16, 2, GROUP(
 		PE7_FN, PE7_OUT, PE7_IN, 0,
 		PE6_FN, PE6_OUT, PE6_IN, 0,
 		0, 0, 0, 0,
@@ -675,9 +675,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		0, 0, 0, 0,
-		0, 0, 0, 0, }
+		0, 0, 0, 0, ))
 	},
-	{ PINMUX_CFG_REG("PFCR", 0xffcc000a, 16, 2) {
+	{ PINMUX_CFG_REG("PFCR", 0xffcc000a, 16, 2, GROUP(
 		PF7_FN, PF7_OUT, PF7_IN, 0,
 		PF6_FN, PF6_OUT, PF6_IN, 0,
 		PF5_FN, PF5_OUT, PF5_IN, 0,
@@ -685,9 +685,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PF3_FN, PF3_OUT, PF3_IN, 0,
 		PF2_FN, PF2_OUT, PF2_IN, 0,
 		PF1_FN, PF1_OUT, PF1_IN, 0,
-		PF0_FN, PF0_OUT, PF0_IN, 0 }
+		PF0_FN, PF0_OUT, PF0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PGCR", 0xffcc000c, 16, 2) {
+	{ PINMUX_CFG_REG("PGCR", 0xffcc000c, 16, 2, GROUP(
 		PG7_FN, PG7_OUT, PG7_IN, 0,
 		PG6_FN, PG6_OUT, PG6_IN, 0,
 		PG5_FN, PG5_OUT, PG5_IN, 0,
@@ -695,9 +695,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		0, 0, 0, 0,
 		0, 0, 0, 0,
 		0, 0, 0, 0,
-		0, 0, 0, 0, }
+		0, 0, 0, 0, ))
 	},
-	{ PINMUX_CFG_REG("PHCR", 0xffcc000e, 16, 2) {
+	{ PINMUX_CFG_REG("PHCR", 0xffcc000e, 16, 2, GROUP(
 		PH7_FN, PH7_OUT, PH7_IN, 0,
 		PH6_FN, PH6_OUT, PH6_IN, 0,
 		PH5_FN, PH5_OUT, PH5_IN, 0,
@@ -705,9 +705,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PH3_FN, PH3_OUT, PH3_IN, 0,
 		PH2_FN, PH2_OUT, PH2_IN, 0,
 		PH1_FN, PH1_OUT, PH1_IN, 0,
-		PH0_FN, PH0_OUT, PH0_IN, 0 }
+		PH0_FN, PH0_OUT, PH0_IN, 0 ))
 	},
-	{ PINMUX_CFG_REG("PJCR", 0xffcc0010, 16, 2) {
+	{ PINMUX_CFG_REG("PJCR", 0xffcc0010, 16, 2, GROUP(
 		PJ7_FN, PJ7_OUT, PJ7_IN, 0,
 		PJ6_FN, PJ6_OUT, PJ6_IN, 0,
 		PJ5_FN, PJ5_OUT, PJ5_IN, 0,
@@ -715,9 +715,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PJ3_FN, PJ3_OUT, PJ3_IN, 0,
 		PJ2_FN, PJ2_OUT, PJ2_IN, 0,
 		PJ1_FN, PJ1_OUT, PJ1_IN, 0,
-		0, 0, 0, 0, }
+		0, 0, 0, 0, ))
 	},
-	{ PINMUX_CFG_REG("P1MSELR", 0xffcc0080, 16, 1) {
+	{ PINMUX_CFG_REG("P1MSELR", 0xffcc0080, 16, 1, GROUP(
 		0, 0,
 		P1MSEL14_0, P1MSEL14_1,
 		P1MSEL13_0, P1MSEL13_1,
@@ -733,9 +733,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		P1MSEL3_0,  P1MSEL3_1,
 		P1MSEL2_0,  P1MSEL2_1,
 		P1MSEL1_0,  P1MSEL1_1,
-		P1MSEL0_0,  P1MSEL0_1 }
+		P1MSEL0_0,  P1MSEL0_1 ))
 	},
-	{ PINMUX_CFG_REG("P2MSELR", 0xffcc0082, 16, 1) {
+	{ PINMUX_CFG_REG("P2MSELR", 0xffcc0082, 16, 1, GROUP(
 		P2MSEL15_0, P2MSEL15_1,
 		P2MSEL14_0, P2MSEL14_1,
 		P2MSEL13_0, P2MSEL13_1,
@@ -751,47 +751,47 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		P2MSEL3_0,  P2MSEL3_1,
 		P2MSEL2_0,  P2MSEL2_1,
 		P2MSEL1_0,  P2MSEL1_1,
-		P2MSEL0_0,  P2MSEL0_1 }
+		P2MSEL0_0,  P2MSEL0_1 ))
 	},
 	{}
 };
 
 static const struct pinmux_data_reg pinmux_data_regs[] = {
-	{ PINMUX_DATA_REG("PADR", 0xffcc0020, 8) {
+	{ PINMUX_DATA_REG("PADR", 0xffcc0020, 8, GROUP(
 		PA7_DATA, PA6_DATA, PA5_DATA, PA4_DATA,
-		PA3_DATA, PA2_DATA, PA1_DATA, PA0_DATA }
+		PA3_DATA, PA2_DATA, PA1_DATA, PA0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PBDR", 0xffcc0022, 8) {
+	{ PINMUX_DATA_REG("PBDR", 0xffcc0022, 8, GROUP(
 		PB7_DATA, PB6_DATA, PB5_DATA, PB4_DATA,
-		PB3_DATA, PB2_DATA, PB1_DATA, PB0_DATA }
+		PB3_DATA, PB2_DATA, PB1_DATA, PB0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PCDR", 0xffcc0024, 8) {
+	{ PINMUX_DATA_REG("PCDR", 0xffcc0024, 8, GROUP(
 		PC7_DATA, PC6_DATA, PC5_DATA, PC4_DATA,
-		PC3_DATA, PC2_DATA, PC1_DATA, PC0_DATA }
+		PC3_DATA, PC2_DATA, PC1_DATA, PC0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PDDR", 0xffcc0026, 8) {
+	{ PINMUX_DATA_REG("PDDR", 0xffcc0026, 8, GROUP(
 		PD7_DATA, PD6_DATA, PD5_DATA, PD4_DATA,
-		PD3_DATA, PD2_DATA, PD1_DATA, PD0_DATA }
+		PD3_DATA, PD2_DATA, PD1_DATA, PD0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PEDR", 0xffcc0028, 8) {
+	{ PINMUX_DATA_REG("PEDR", 0xffcc0028, 8, GROUP(
 		PE7_DATA, PE6_DATA,
-		0, 0, 0, 0, 0, 0 }
+		0, 0, 0, 0, 0, 0 ))
 	},
-	{ PINMUX_DATA_REG("PFDR", 0xffcc002a, 8) {
+	{ PINMUX_DATA_REG("PFDR", 0xffcc002a, 8, GROUP(
 		PF7_DATA, PF6_DATA, PF5_DATA, PF4_DATA,
-		PF3_DATA, PF2_DATA, PF1_DATA, PF0_DATA }
+		PF3_DATA, PF2_DATA, PF1_DATA, PF0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PGDR", 0xffcc002c, 8) {
+	{ PINMUX_DATA_REG("PGDR", 0xffcc002c, 8, GROUP(
 		PG7_DATA, PG6_DATA, PG5_DATA, 0,
-		0, 0, 0, 0 }
+		0, 0, 0, 0 ))
 	},
-	{ PINMUX_DATA_REG("PHDR", 0xffcc002e, 8) {
+	{ PINMUX_DATA_REG("PHDR", 0xffcc002e, 8, GROUP(
 		PH7_DATA, PH6_DATA, PH5_DATA, PH4_DATA,
-		PH3_DATA, PH2_DATA, PH1_DATA, PH0_DATA }
+		PH3_DATA, PH2_DATA, PH1_DATA, PH0_DATA ))
 	},
-	{ PINMUX_DATA_REG("PJDR", 0xffcc0030, 8) {
+	{ PINMUX_DATA_REG("PJDR", 0xffcc0030, 8, GROUP(
 		PJ7_DATA, PJ6_DATA, PJ5_DATA, PJ4_DATA,
-		PJ3_DATA, PJ2_DATA, PJ1_DATA, 0 }
+		PJ3_DATA, PJ2_DATA, PJ1_DATA, 0 ))
 	},
 	{ },
 };
diff --git a/drivers/pinctrl/sh-pfc/pfc-shx3.c b/drivers/pinctrl/sh-pfc/pfc-shx3.c
index 905ae00cc6f1..22e812850964 100644
--- a/drivers/pinctrl/sh-pfc/pfc-shx3.c
+++ b/drivers/pinctrl/sh-pfc/pfc-shx3.c
@@ -431,7 +431,7 @@ static const struct pinmux_func pinmux_func_gpios[] = {
 };
 
 static const struct pinmux_cfg_reg pinmux_config_regs[] = {
-	{ PINMUX_CFG_REG("PABCR", 0xffc70000, 32, 2) {
+	{ PINMUX_CFG_REG("PABCR", 0xffc70000, 32, 2, GROUP(
 		PA7_FN, PA7_OUT, PA7_IN, 0,
 		PA6_FN, PA6_OUT, PA6_IN, 0,
 		PA5_FN, PA5_OUT, PA5_IN, 0,
@@ -447,9 +447,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PB3_FN, PB3_OUT, PB3_IN, 0,
 		PB2_FN, PB2_OUT, PB2_IN, 0,
 		PB1_FN, PB1_OUT, PB1_IN, 0,
-		PB0_FN, PB0_OUT, PB0_IN, 0, },
+		PB0_FN, PB0_OUT, PB0_IN, 0, ))
 	},
-	{ PINMUX_CFG_REG("PCDCR", 0xffc70004, 32, 2) {
+	{ PINMUX_CFG_REG("PCDCR", 0xffc70004, 32, 2, GROUP(
 		PC7_FN, PC7_OUT, PC7_IN, 0,
 		PC6_FN, PC6_OUT, PC6_IN, 0,
 		PC5_FN, PC5_OUT, PC5_IN, 0,
@@ -465,9 +465,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PD3_FN, PD3_OUT, PD3_IN, 0,
 		PD2_FN, PD2_OUT, PD2_IN, 0,
 		PD1_FN, PD1_OUT, PD1_IN, 0,
-		PD0_FN, PD0_OUT, PD0_IN, 0, },
+		PD0_FN, PD0_OUT, PD0_IN, 0, ))
 	},
-	{ PINMUX_CFG_REG("PEFCR", 0xffc70008, 32, 2) {
+	{ PINMUX_CFG_REG("PEFCR", 0xffc70008, 32, 2, GROUP(
 		PE7_FN, PE7_OUT, PE7_IN, 0,
 		PE6_FN, PE6_OUT, PE6_IN, 0,
 		PE5_FN, PE5_OUT, PE5_IN, 0,
@@ -483,9 +483,9 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PF3_FN, PF3_OUT, PF3_IN, 0,
 		PF2_FN, PF2_OUT, PF2_IN, 0,
 		PF1_FN, PF1_OUT, PF1_IN, 0,
-		PF0_FN, PF0_OUT, PF0_IN, 0, },
+		PF0_FN, PF0_OUT, PF0_IN, 0, ))
 	},
-	{ PINMUX_CFG_REG("PGHCR", 0xffc7000c, 32, 2) {
+	{ PINMUX_CFG_REG("PGHCR", 0xffc7000c, 32, 2, GROUP(
 		PG7_FN, PG7_OUT, PG7_IN, 0,
 		PG6_FN, PG6_OUT, PG6_IN, 0,
 		PG5_FN, PG5_OUT, PG5_IN, 0,
@@ -501,43 +501,43 @@ static const struct pinmux_cfg_reg pinmux_config_regs[] = {
 		PH3_FN, PH3_OUT, PH3_IN, 0,
 		PH2_FN, PH2_OUT, PH2_IN, 0,
 		PH1_FN, PH1_OUT, PH1_IN, 0,
-		PH0_FN, PH0_OUT, PH0_IN, 0, },
+		PH0_FN, PH0_OUT, PH0_IN, 0, ))
 	},
 	{ },
 };
 
 static const struct pinmux_data_reg pinmux_data_regs[] = {
-	{ PINMUX_DATA_REG("PABDR", 0xffc70010, 32) {
+	{ PINMUX_DATA_REG("PABDR", 0xffc70010, 32, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PA7_DATA, PA6_DATA, PA5_DATA, PA4_DATA,
 		PA3_DATA, PA2_DATA, PA1_DATA, PA0_DATA,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PB7_DATA, PB6_DATA, PB5_DATA, PB4_DATA,
-		PB3_DATA, PB2_DATA, PB1_DATA, PB0_DATA, },
+		PB3_DATA, PB2_DATA, PB1_DATA, PB0_DATA, ))
 	},
-	{ PINMUX_DATA_REG("PCDDR", 0xffc70014, 32) {
+	{ PINMUX_DATA_REG("PCDDR", 0xffc70014, 32, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PC7_DATA, PC6_DATA, PC5_DATA, PC4_DATA,
 		PC3_DATA, PC2_DATA, PC1_DATA, PC0_DATA,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PD7_DATA, PD6_DATA, PD5_DATA, PD4_DATA,
-		PD3_DATA, PD2_DATA, PD1_DATA, PD0_DATA, },
+		PD3_DATA, PD2_DATA, PD1_DATA, PD0_DATA, ))
 	},
-	{ PINMUX_DATA_REG("PEFDR", 0xffc70018, 32) {
+	{ PINMUX_DATA_REG("PEFDR", 0xffc70018, 32, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PE7_DATA, PE6_DATA, PE5_DATA, PE4_DATA,
 		PE3_DATA, PE2_DATA, PE1_DATA, PE0_DATA,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PF7_DATA, PF6_DATA, PF5_DATA, PF4_DATA,
-		PF3_DATA, PF2_DATA, PF1_DATA, PF0_DATA, },
+		PF3_DATA, PF2_DATA, PF1_DATA, PF0_DATA, ))
 	},
-	{ PINMUX_DATA_REG("PGHDR", 0xffc7001c, 32) {
+	{ PINMUX_DATA_REG("PGHDR", 0xffc7001c, 32, GROUP(
 		0, 0, 0, 0, 0, 0, 0, 0,
 		PG7_DATA, PG6_DATA, PG5_DATA, PG4_DATA,
 		PG3_DATA, PG2_DATA, PG1_DATA, PG0_DATA,
 		0, 0, 0, 0, 0, 0, 0, 0,
 		0, 0, PH5_DATA, PH4_DATA,
-		PH3_DATA, PH2_DATA, PH1_DATA, PH0_DATA, },
+		PH3_DATA, PH2_DATA, PH1_DATA, PH0_DATA, ))
 	},
 	{ },
 };
diff --git a/drivers/pinctrl/sh-pfc/sh_pfc.h b/drivers/pinctrl/sh-pfc/sh_pfc.h
index 56016cb76769..7db5819eea7e 100644
--- a/drivers/pinctrl/sh-pfc/sh_pfc.h
+++ b/drivers/pinctrl/sh-pfc/sh_pfc.h
@@ -111,40 +111,54 @@ struct pinmux_func {
 struct pinmux_cfg_reg {
 	u32 reg;
 	u8 reg_width, field_width;
+#ifdef DEBUG
+	u16 nr_enum_ids;	/* for variable width regs only */
+#define SET_NR_ENUM_IDS(n)	.nr_enum_ids = n,
+#else
+#define SET_NR_ENUM_IDS(n)
+#endif
 	const u16 *enum_ids;
 	const u8 *var_field_width;
 };
 
+#define GROUP(...)	__VA_ARGS__
+
 /*
  * Describe a config register consisting of several fields of the same width
  *   - name: Register name (unused, for documentation purposes only)
  *   - r: Physical register address
  *   - r_width: Width of the register (in bits)
  *   - f_width: Width of the fixed-width register fields (in bits)
- * This macro must be followed by initialization data: For each register field
- * (from left to right, i.e. MSB to LSB), 2^f_width enum IDs must be specified,
- * one for each possible combination of the register field bit values.
+ *   - ids: For each register field (from left to right, i.e. MSB to LSB),
+ *          2^f_width enum IDs must be specified, one for each possible
+ *          combination of the register field bit values, all wrapped using
+ *          the GROUP() macro.
  */
-#define PINMUX_CFG_REG(name, r, r_width, f_width) \
+#define PINMUX_CFG_REG(name, r, r_width, f_width, ids)			\
 	.reg = r, .reg_width = r_width,					\
-	.field_width = f_width + BUILD_BUG_ON_ZERO(r_width % f_width),	\
-	.enum_ids = (const u16 [(r_width / f_width) * (1 << f_width)])
+	.field_width = f_width + BUILD_BUG_ON_ZERO(r_width % f_width) +	\
+	BUILD_BUG_ON_ZERO(sizeof((const u16 []) { ids }) / sizeof(u16) != \
+			  (r_width / f_width) * (1 << f_width)),	\
+	.enum_ids = (const u16 [(r_width / f_width) * (1 << f_width)])	\
+		{ ids }
 
 /*
  * Describe a config register consisting of several fields of different widths
  *   - name: Register name (unused, for documentation purposes only)
  *   - r: Physical register address
  *   - r_width: Width of the register (in bits)
- *   - var_fw0, var_fwn...: List of widths of the register fields (in bits),
- *                          From left to right (i.e. MSB to LSB)
- * This macro must be followed by initialization data: For each register field
- * (from left to right, i.e. MSB to LSB), 2^var_fwi enum IDs must be specified,
- * one for each possible combination of the register field bit values.
+ *   - f_widths: List of widths of the register fields (in bits), from left
+ *               to right (i.e. MSB to LSB), wrapped using the GROUP() macro.
+ *   - ids: For each register field (from left to right, i.e. MSB to LSB),
+ *          2^f_widths[i] enum IDs must be specified, one for each possible
+ *          combination of the register field bit values, all wrapped using
+ *          the GROUP() macro.
  */
-#define PINMUX_CFG_REG_VAR(name, r, r_width, var_fw0, var_fwn...) \
-	.reg = r, .reg_width = r_width,	\
-	.var_field_width = (const u8 []) { var_fw0, var_fwn, 0 }, \
-	.enum_ids = (const u16 [])
+#define PINMUX_CFG_REG_VAR(name, r, r_width, f_widths, ids)		\
+	.reg = r, .reg_width = r_width,					\
+	.var_field_width = (const u8 []) { f_widths, 0 },		\
+	SET_NR_ENUM_IDS(sizeof((const u16 []) { ids }) / sizeof(u16))	\
+	.enum_ids = (const u16 []) { ids }
 
 struct pinmux_drive_reg_field {
 	u16 pin;
@@ -187,12 +201,14 @@ struct pinmux_data_reg {
  *   - name: Register name (unused, for documentation purposes only)
  *   - r: Physical register address
  *   - r_width: Width of the register (in bits)
- * This macro must be followed by initialization data: For each register bit
- * (from left to right, i.e. MSB to LSB), one enum ID must be specified.
+ *   - ids: For each register bit (from left to right, i.e. MSB to LSB), one
+ *          enum ID must be specified, all wrapped using the GROUP() macro.
  */
-#define PINMUX_DATA_REG(name, r, r_width) \
-	.reg = r, .reg_width = r_width,	\
-	.enum_ids = (const u16 [r_width]) \
+#define PINMUX_DATA_REG(name, r, r_width, ids)				\
+	.reg = r, .reg_width = r_width +				\
+	BUILD_BUG_ON_ZERO(sizeof((const u16 []) { ids }) / sizeof(u16) != \
+			  r_width),					\
+	.enum_ids = (const u16 [r_width]) { ids }
 
 struct pinmux_irq {
 	const short *gpios;
@@ -261,7 +277,7 @@ struct sh_pfc_soc_info {
 	const struct sh_pfc_function *functions;
 	unsigned int nr_functions;
 
-#ifdef CONFIG_SUPERH
+#ifdef CONFIG_PINCTRL_SH_FUNC_GPIO
 	const struct pinmux_func *func_gpios;
 	unsigned int nr_func_gpios;
 #endif
@@ -402,8 +418,8 @@ extern const struct sh_pfc_soc_info shx3_pinmux_info;
 /*
  * Describe a pinmux configuration in which a pin is physically multiplexed
  * with other pins.
- *   - ipsr: IPSR field
- *   - fn: Function name, also referring to the IPSR field
+ *   - ipsr: IPSR field (unused, for documentation purposes only)
+ *   - fn: Function name
  *   - psel: Physical multiplexing selector
  */
 #define PINMUX_IPSR_PHYS(ipsr, fn, psel) \
@@ -663,7 +679,9 @@ extern const struct sh_pfc_soc_info shx3_pinmux_info;
  */
 #define PORTCR(nr, reg)							\
 	{								\
-		PINMUX_CFG_REG_VAR("PORT" nr "CR", reg, 8, 2, 2, 1, 3) {\
+		PINMUX_CFG_REG_VAR("PORT" nr "CR", reg, 8,		\
+				   GROUP(2, 2, 1, 3),			\
+				   GROUP(				\
 			/* PULMD[1:0], handled by .set_bias() */	\
 			0, 0, 0, 0,					\
 			/* IE and OE */					\
@@ -675,7 +693,7 @@ extern const struct sh_pfc_soc_info shx3_pinmux_info;
 			PORT##nr##_FN2, PORT##nr##_FN3,			\
 			PORT##nr##_FN4, PORT##nr##_FN5,			\
 			PORT##nr##_FN6, PORT##nr##_FN7			\
-		}							\
+		))							\
 	}
 
 /*
diff --git a/drivers/pinctrl/stm32/pinctrl-stm32.c b/drivers/pinctrl/stm32/pinctrl-stm32.c
index 0b9ff5aa6bb5..2317ccf63321 100644
--- a/drivers/pinctrl/stm32/pinctrl-stm32.c
+++ b/drivers/pinctrl/stm32/pinctrl-stm32.c
@@ -95,6 +95,9 @@ struct stm32_pinctrl {
 	struct regmap		*regmap;
 	struct regmap_field	*irqmux[STM32_GPIO_PINS_PER_BANK];
 	struct hwspinlock *hwlock;
+	struct stm32_desc_pin *pins;
+	u32 npins;
+	u32 pkg;
 };
 
 static inline int stm32_gpio_pin(int gpio)
@@ -358,8 +361,8 @@ static bool stm32_pctrl_is_function_valid(struct stm32_pinctrl *pctl,
 {
 	int i;
 
-	for (i = 0; i < pctl->match_data->npins; i++) {
-		const struct stm32_desc_pin *pin = pctl->match_data->pins + i;
+	for (i = 0; i < pctl->npins; i++) {
+		const struct stm32_desc_pin *pin = pctl->pins + i;
 		const struct stm32_desc_function *func = pin->functions;
 
 		if (pin->pin.number != pin_num)
@@ -1119,23 +1122,35 @@ static int stm32_gpiolib_register_bank(struct stm32_pinctrl *pctl,
 	return 0;
 }
 
+static struct irq_domain *stm32_pctrl_get_irq_domain(struct device_node *np)
+{
+	struct device_node *parent;
+	struct irq_domain *domain;
+
+	if (!of_find_property(np, "interrupt-parent", NULL))
+		return NULL;
+
+	parent = of_irq_find_parent(np);
+	if (!parent)
+		return ERR_PTR(-ENXIO);
+
+	domain = irq_find_host(parent);
+	if (!domain)
+		/* domain not registered yet */
+		return ERR_PTR(-EPROBE_DEFER);
+
+	return domain;
+}
+
 static int stm32_pctrl_dt_setup_irq(struct platform_device *pdev,
 			   struct stm32_pinctrl *pctl)
 {
-	struct device_node *np = pdev->dev.of_node, *parent;
+	struct device_node *np = pdev->dev.of_node;
 	struct device *dev = &pdev->dev;
 	struct regmap *rm;
 	int offset, ret, i;
 	int mask, mask_width;
 
-	parent = of_irq_find_parent(np);
-	if (!parent)
-		return -ENXIO;
-
-	pctl->domain = irq_find_host(parent);
-	if (!pctl->domain)
-		return -ENXIO;
-
 	pctl->regmap = syscon_regmap_lookup_by_phandle(np, "st,syscfg");
 	if (IS_ERR(pctl->regmap))
 		return PTR_ERR(pctl->regmap);
@@ -1175,7 +1190,7 @@ static int stm32_pctrl_build_state(struct platform_device *pdev)
 	struct stm32_pinctrl *pctl = platform_get_drvdata(pdev);
 	int i;
 
-	pctl->ngroups = pctl->match_data->npins;
+	pctl->ngroups = pctl->npins;
 
 	/* Allocate groups */
 	pctl->groups = devm_kcalloc(&pdev->dev, pctl->ngroups,
@@ -1189,19 +1204,50 @@ static int stm32_pctrl_build_state(struct platform_device *pdev)
 	if (!pctl->grp_names)
 		return -ENOMEM;
 
-	for (i = 0; i < pctl->match_data->npins; i++) {
-		const struct stm32_desc_pin *pin = pctl->match_data->pins + i;
+	for (i = 0; i < pctl->npins; i++) {
+		const struct stm32_desc_pin *pin = pctl->pins + i;
 		struct stm32_pinctrl_group *group = pctl->groups + i;
 
 		group->name = pin->pin.name;
 		group->pin = pin->pin.number;
-
 		pctl->grp_names[i] = pin->pin.name;
 	}
 
 	return 0;
 }
 
+static int stm32_pctrl_create_pins_tab(struct stm32_pinctrl *pctl,
+				       struct stm32_desc_pin *pins)
+{
+	const struct stm32_desc_pin *p;
+	int i, nb_pins_available = 0;
+
+	for (i = 0; i < pctl->match_data->npins; i++) {
+		p = pctl->match_data->pins + i;
+		if (pctl->pkg && !(pctl->pkg & p->pkg))
+			continue;
+		pins->pin = p->pin;
+		pins->functions = p->functions;
+		pins++;
+		nb_pins_available++;
+	}
+
+	pctl->npins = nb_pins_available;
+
+	return 0;
+}
+
+static void stm32_pctl_get_package(struct device_node *np,
+				   struct stm32_pinctrl *pctl)
+{
+	if (of_property_read_u32(np, "st,package", &pctl->pkg)) {
+		pctl->pkg = 0;
+		dev_warn(pctl->dev, "No package detected, use default one\n");
+	} else {
+		dev_dbg(pctl->dev, "package detected: %x\n", pctl->pkg);
+	}
+}
+
 int stm32_pctl_probe(struct platform_device *pdev)
 {
 	struct device_node *np = pdev->dev.of_node;
@@ -1230,6 +1276,11 @@ int stm32_pctl_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, pctl);
 
+	/* check for IRQ controller (may require deferred probe) */
+	pctl->domain = stm32_pctrl_get_irq_domain(np);
+	if (IS_ERR(pctl->domain))
+		return PTR_ERR(pctl->domain);
+
 	/* hwspinlock is optional */
 	hwlock_id = of_hwspin_lock_get_id(pdev->dev.of_node, 0);
 	if (hwlock_id < 0) {
@@ -1241,30 +1292,43 @@ int stm32_pctl_probe(struct platform_device *pdev)
 
 	pctl->dev = dev;
 	pctl->match_data = match->data;
+
+	/*  get package information */
+	stm32_pctl_get_package(np, pctl);
+
+	pctl->pins = devm_kcalloc(pctl->dev, pctl->match_data->npins,
+				  sizeof(*pctl->pins), GFP_KERNEL);
+	if (!pctl->pins)
+		return -ENOMEM;
+
+	ret = stm32_pctrl_create_pins_tab(pctl, pctl->pins);
+	if (ret)
+		return ret;
+
 	ret = stm32_pctrl_build_state(pdev);
 	if (ret) {
 		dev_err(dev, "build state failed: %d\n", ret);
 		return -EINVAL;
 	}
 
-	if (of_find_property(np, "interrupt-parent", NULL)) {
+	if (pctl->domain) {
 		ret = stm32_pctrl_dt_setup_irq(pdev, pctl);
 		if (ret)
 			return ret;
 	}
 
-	pins = devm_kcalloc(&pdev->dev, pctl->match_data->npins, sizeof(*pins),
+	pins = devm_kcalloc(&pdev->dev, pctl->npins, sizeof(*pins),
 			    GFP_KERNEL);
 	if (!pins)
 		return -ENOMEM;
 
-	for (i = 0; i < pctl->match_data->npins; i++)
-		pins[i] = pctl->match_data->pins[i].pin;
+	for (i = 0; i < pctl->npins; i++)
+		pins[i] = pctl->pins[i].pin;
 
 	pctl->pctl_desc.name = dev_name(&pdev->dev);
 	pctl->pctl_desc.owner = THIS_MODULE;
 	pctl->pctl_desc.pins = pins;
-	pctl->pctl_desc.npins = pctl->match_data->npins;
+	pctl->pctl_desc.npins = pctl->npins;
 	pctl->pctl_desc.confops = &stm32_pconf_ops;
 	pctl->pctl_desc.pctlops = &stm32_pctrl_ops;
 	pctl->pctl_desc.pmxops = &stm32_pmx_ops;
@@ -1305,4 +1369,3 @@ int stm32_pctl_probe(struct platform_device *pdev)
 
 	return 0;
 }
-
diff --git a/drivers/pinctrl/stm32/pinctrl-stm32.h b/drivers/pinctrl/stm32/pinctrl-stm32.h
index 473a6238a27b..de5e7012ca03 100644
--- a/drivers/pinctrl/stm32/pinctrl-stm32.h
+++ b/drivers/pinctrl/stm32/pinctrl-stm32.h
@@ -18,6 +18,12 @@
 #define STM32_PIN_AF(x)		((x) + 1)
 #define STM32_PIN_ANALOG	(STM32_PIN_AF(15) + 1)
 
+/*  package information */
+#define STM32MP_PKG_AA		BIT(0)
+#define STM32MP_PKG_AB		BIT(1)
+#define STM32MP_PKG_AC		BIT(2)
+#define STM32MP_PKG_AD		BIT(3)
+
 struct stm32_desc_function {
 	const char *name;
 	const unsigned char num;
@@ -26,6 +32,7 @@ struct stm32_desc_function {
 struct stm32_desc_pin {
 	struct pinctrl_pin_desc pin;
 	const struct stm32_desc_function *functions;
+	const unsigned int pkg;
 };
 
 #define STM32_PIN(_pin, ...)					\
@@ -35,6 +42,13 @@ struct stm32_desc_pin {
 			__VA_ARGS__, { } },			\
 	}
 
+#define STM32_PIN_PKG(_pin, _pkg, ...)					\
+	{							\
+		.pin = _pin,					\
+		.pkg  = _pkg,				\
+		.functions = (struct stm32_desc_function[]){	\
+			__VA_ARGS__, { } },			\
+	}
 #define STM32_FUNCTION(_num, _name)		\
 	{							\
 		.num = _num,					\
diff --git a/drivers/pinctrl/stm32/pinctrl-stm32mp157.c b/drivers/pinctrl/stm32/pinctrl-stm32mp157.c
index 7c7d6284b23c..320544f69e57 100644
--- a/drivers/pinctrl/stm32/pinctrl-stm32mp157.c
+++ b/drivers/pinctrl/stm32/pinctrl-stm32mp157.c
@@ -10,77 +10,82 @@
 #include "pinctrl-stm32.h"
 
 static const struct stm32_desc_pin stm32mp157_pins[] = {
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(0, "PA0"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOA0"),
 		STM32_FUNCTION(2, "TIM2_CH1 TIM2_ETR"),
 		STM32_FUNCTION(3, "TIM5_CH1"),
 		STM32_FUNCTION(4, "TIM8_ETR"),
 		STM32_FUNCTION(5, "TIM15_BKIN"),
-		STM32_FUNCTION(8, "USART2_CTS_NSS USART_BOOT2_CTS_NSS"),
+		STM32_FUNCTION(8, "USART2_CTS USART2_NSS"),
 		STM32_FUNCTION(9, "UART4_TX"),
 		STM32_FUNCTION(10, "SDMMC2_CMD"),
 		STM32_FUNCTION(11, "SAI2_SD_B"),
-		STM32_FUNCTION(12, "ETH_GMII_CRS ETH_MII_CRS"),
+		STM32_FUNCTION(12, "ETH1_GMII_CRS ETH1_MII_CRS"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(1, "PA1"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOA1"),
 		STM32_FUNCTION(1, "ETH_CLK"),
 		STM32_FUNCTION(2, "TIM2_CH2"),
 		STM32_FUNCTION(3, "TIM5_CH2"),
 		STM32_FUNCTION(4, "LPTIM3_OUT"),
 		STM32_FUNCTION(5, "TIM15_CH1N"),
-		STM32_FUNCTION(8, "USART2_RTS USART_BOOT2_RTS"),
+		STM32_FUNCTION(8, "USART2_RTS USART2_DE"),
 		STM32_FUNCTION(9, "UART4_RX"),
-		STM32_FUNCTION(10, "QUADSPI_BK1_IO3 QUADSPI_BOOTBK1_IO3"),
+		STM32_FUNCTION(10, "QUADSPI_BK1_IO3"),
 		STM32_FUNCTION(11, "SAI2_MCLK_B"),
-		STM32_FUNCTION(12, "ETH_GMII_RX_CLK ETH_MII_RX_CLK ETH_RGMII_RX_CLK ETH_RMII_REF_CLK"),
+		STM32_FUNCTION(12, "ETH1_GMII_RX_CLK ETH1_MII_RX_CLK ETH1_RGMII_RX_CLK ETH1_RMII_REF_CLK"),
 		STM32_FUNCTION(15, "LCD_R2"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(2, "PA2"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOA2"),
 		STM32_FUNCTION(2, "TIM2_CH3"),
 		STM32_FUNCTION(3, "TIM5_CH3"),
 		STM32_FUNCTION(4, "LPTIM4_OUT"),
 		STM32_FUNCTION(5, "TIM15_CH1"),
-		STM32_FUNCTION(8, "USART2_TX USART_BOOT2_TX"),
+		STM32_FUNCTION(8, "USART2_TX"),
 		STM32_FUNCTION(9, "SAI2_SCK_B"),
-		STM32_FUNCTION(11, "SDMMC2_D0DIR SDMMC_BOOT2_D0DIR"),
-		STM32_FUNCTION(12, "ETH_MDIO"),
+		STM32_FUNCTION(11, "SDMMC2_D0DIR"),
+		STM32_FUNCTION(12, "ETH1_MDIO"),
 		STM32_FUNCTION(13, "MDIOS_MDIO"),
 		STM32_FUNCTION(15, "LCD_R1"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(3, "PA3"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOA3"),
 		STM32_FUNCTION(2, "TIM2_CH4"),
 		STM32_FUNCTION(3, "TIM5_CH4"),
 		STM32_FUNCTION(4, "LPTIM5_OUT"),
 		STM32_FUNCTION(5, "TIM15_CH2"),
-		STM32_FUNCTION(8, "USART2_RX USART_BOOT2_RX"),
+		STM32_FUNCTION(8, "USART2_RX"),
 		STM32_FUNCTION(10, "LCD_B2"),
-		STM32_FUNCTION(12, "ETH_GMII_COL ETH_MII_COL"),
+		STM32_FUNCTION(12, "ETH1_GMII_COL ETH1_MII_COL"),
 		STM32_FUNCTION(15, "LCD_B5"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(4, "PA4"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOA4"),
 		STM32_FUNCTION(1, "HDP0"),
 		STM32_FUNCTION(3, "TIM5_ETR"),
 		STM32_FUNCTION(5, "SAI4_D2"),
 		STM32_FUNCTION(6, "SPI1_NSS I2S1_WS"),
 		STM32_FUNCTION(7, "SPI3_NSS I2S3_WS"),
-		STM32_FUNCTION(8, "USART2_CK USART_BOOT2_CK"),
+		STM32_FUNCTION(8, "USART2_CK"),
 		STM32_FUNCTION(9, "SPI6_NSS"),
 		STM32_FUNCTION(13, "SAI4_FS_A"),
 		STM32_FUNCTION(14, "DCMI_HSYNC"),
@@ -88,8 +93,9 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(5, "PA5"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOA5"),
 		STM32_FUNCTION(2, "TIM2_CH1 TIM2_ETR"),
 		STM32_FUNCTION(4, "TIM8_CH1N"),
@@ -101,8 +107,9 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(6, "PA6"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOA6"),
 		STM32_FUNCTION(2, "TIM1_BKIN"),
 		STM32_FUNCTION(3, "TIM3_CH1"),
@@ -118,8 +125,9 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(7, "PA7"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOA7"),
 		STM32_FUNCTION(2, "TIM1_CH1N"),
 		STM32_FUNCTION(3, "TIM3_CH2"),
@@ -129,13 +137,14 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(9, "SPI6_MOSI"),
 		STM32_FUNCTION(10, "TIM14_CH1"),
 		STM32_FUNCTION(11, "QUADSPI_CLK"),
-		STM32_FUNCTION(12, "ETH_GMII_RX_DV ETH_MII_RX_DV ETH_RGMII_RX_CTL ETH_RMII_CRS_DV"),
+		STM32_FUNCTION(12, "ETH1_GMII_RX_DV ETH1_MII_RX_DV ETH1_RGMII_RX_CTL ETH1_RMII_CRS_DV"),
 		STM32_FUNCTION(13, "SAI4_SD_A"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(8, "PA8"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOA8"),
 		STM32_FUNCTION(1, "MCO1"),
 		STM32_FUNCTION(2, "TIM1_CH1"),
@@ -143,37 +152,37 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(5, "I2C3_SCL"),
 		STM32_FUNCTION(6, "SPI3_MOSI I2S3_SDO"),
 		STM32_FUNCTION(8, "USART1_CK"),
-		STM32_FUNCTION(9, "SDMMC2_CKIN SDMMC_BOOT2_CKIN"),
-		STM32_FUNCTION(10, "SDMMC2_D4 SDMMC_BOOT2_D4"),
-		STM32_FUNCTION(11, "USBO_SOF"),
+		STM32_FUNCTION(9, "SDMMC2_CKIN"),
+		STM32_FUNCTION(10, "SDMMC2_D4"),
+		STM32_FUNCTION(11, "OTG_FS_SOF OTG_HS_SOF"),
 		STM32_FUNCTION(13, "SAI4_SD_B"),
 		STM32_FUNCTION(14, "UART7_RX"),
 		STM32_FUNCTION(15, "LCD_R6"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(9, "PA9"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOA9"),
 		STM32_FUNCTION(2, "TIM1_CH2"),
 		STM32_FUNCTION(5, "I2C3_SMBA"),
 		STM32_FUNCTION(6, "SPI2_SCK I2S2_CK"),
 		STM32_FUNCTION(8, "USART1_TX"),
-		STM32_FUNCTION(9, "SDMMC2_CDIR SDMMC_BOOT2_CDIR"),
-		STM32_FUNCTION(10, "CAN1_RXFD"),
-		STM32_FUNCTION(11, "SDMMC2_D5 SDMMC_BOOT2_D5"),
+		STM32_FUNCTION(9, "SDMMC2_CDIR"),
+		STM32_FUNCTION(11, "SDMMC2_D5"),
 		STM32_FUNCTION(14, "DCMI_D0"),
 		STM32_FUNCTION(15, "LCD_R5"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(10, "PA10"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOA10"),
 		STM32_FUNCTION(2, "TIM1_CH3"),
 		STM32_FUNCTION(6, "SPI3_NSS I2S3_WS"),
 		STM32_FUNCTION(8, "USART1_RX"),
-		STM32_FUNCTION(10, "CAN1_TXFD"),
 		STM32_FUNCTION(12, "MDIOS_MDIO"),
 		STM32_FUNCTION(13, "SAI4_FS_B"),
 		STM32_FUNCTION(14, "DCMI_D1"),
@@ -181,37 +190,39 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(11, "PA11"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOA11"),
 		STM32_FUNCTION(2, "TIM1_CH4"),
 		STM32_FUNCTION(3, "I2C6_SCL"),
 		STM32_FUNCTION(5, "I2C5_SCL"),
 		STM32_FUNCTION(6, "SPI2_NSS I2S2_WS"),
 		STM32_FUNCTION(7, "UART4_RX"),
-		STM32_FUNCTION(8, "USART1_CTS_NSS"),
-		STM32_FUNCTION(10, "CAN1_RX"),
+		STM32_FUNCTION(8, "USART1_CTS USART1_NSS"),
+		STM32_FUNCTION(10, "FDCAN1_RX"),
 		STM32_FUNCTION(15, "LCD_R4"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(12, "PA12"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOA12"),
 		STM32_FUNCTION(2, "TIM1_ETR"),
 		STM32_FUNCTION(3, "I2C6_SDA"),
 		STM32_FUNCTION(5, "I2C5_SDA"),
-		STM32_FUNCTION(6, "SPI2_SCK I2S2_CK"),
 		STM32_FUNCTION(7, "UART4_TX"),
-		STM32_FUNCTION(8, "USART1_RTS"),
+		STM32_FUNCTION(8, "USART1_RTS USART1_DE"),
 		STM32_FUNCTION(9, "SAI2_FS_B"),
-		STM32_FUNCTION(10, "CAN1_TX"),
+		STM32_FUNCTION(10, "FDCAN1_TX"),
 		STM32_FUNCTION(15, "LCD_R5"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(13, "PA13"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOA13"),
 		STM32_FUNCTION(1, "DBTRGO"),
 		STM32_FUNCTION(2, "DBTRGI"),
@@ -220,8 +231,9 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(14, "PA14"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOA14"),
 		STM32_FUNCTION(1, "DBTRGO"),
 		STM32_FUNCTION(2, "DBTRGI"),
@@ -229,73 +241,79 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(15, "PA15"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOA15"),
 		STM32_FUNCTION(1, "DBTRGI"),
 		STM32_FUNCTION(2, "TIM2_CH1 TIM2_ETR"),
 		STM32_FUNCTION(3, "SAI4_D2"),
 		STM32_FUNCTION(4, "SDMMC1_CDIR"),
-		STM32_FUNCTION(5, "HDMI_CEC"),
+		STM32_FUNCTION(5, "CEC"),
 		STM32_FUNCTION(6, "SPI1_NSS I2S1_WS"),
 		STM32_FUNCTION(7, "SPI3_NSS I2S3_WS"),
 		STM32_FUNCTION(8, "SPI6_NSS"),
-		STM32_FUNCTION(9, "UART4_RTS UART_BOOT4_RTS"),
-		STM32_FUNCTION(10, "SDMMC2_D5 SDMMC_BOOT2_D5"),
-		STM32_FUNCTION(11, "SDMMC2_CDIR SDMMC_BOOT2_CDIR"),
-		STM32_FUNCTION(12, "SDMMC1_D5 SDMMC_BOOT1_D5"),
+		STM32_FUNCTION(9, "UART4_RTS UART4_DE"),
+		STM32_FUNCTION(10, "SDMMC2_D5"),
+		STM32_FUNCTION(11, "SDMMC2_CDIR"),
+		STM32_FUNCTION(12, "SDMMC1_D5"),
 		STM32_FUNCTION(13, "SAI4_FS_A"),
 		STM32_FUNCTION(14, "UART7_TX"),
+		STM32_FUNCTION(15, "LCD_R1"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(16, "PB0"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOB0"),
 		STM32_FUNCTION(2, "TIM1_CH2N"),
 		STM32_FUNCTION(3, "TIM3_CH3"),
 		STM32_FUNCTION(4, "TIM8_CH2N"),
-		STM32_FUNCTION(7, "DFSDM_CKOUT"),
-		STM32_FUNCTION(9, "UART4_CTS UART_BOOT4_CTS"),
+		STM32_FUNCTION(7, "DFSDM1_CKOUT"),
+		STM32_FUNCTION(9, "UART4_CTS"),
 		STM32_FUNCTION(10, "LCD_R3"),
-		STM32_FUNCTION(12, "ETH_GMII_RXD2 ETH_MII_RXD2 ETH_RGMII_RXD2"),
+		STM32_FUNCTION(12, "ETH1_GMII_RXD2 ETH1_MII_RXD2 ETH1_RGMII_RXD2"),
 		STM32_FUNCTION(13, "MDIOS_MDIO"),
 		STM32_FUNCTION(15, "LCD_G1"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(17, "PB1"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOB1"),
 		STM32_FUNCTION(2, "TIM1_CH3N"),
 		STM32_FUNCTION(3, "TIM3_CH4"),
 		STM32_FUNCTION(4, "TIM8_CH3N"),
-		STM32_FUNCTION(7, "DFSDM_DATA1"),
+		STM32_FUNCTION(7, "DFSDM1_DATIN1"),
 		STM32_FUNCTION(10, "LCD_R6"),
-		STM32_FUNCTION(12, "ETH_GMII_RXD3 ETH_MII_RXD3 ETH_RGMII_RXD3"),
+		STM32_FUNCTION(12, "ETH1_GMII_RXD3 ETH1_MII_RXD3 ETH1_RGMII_RXD3"),
 		STM32_FUNCTION(13, "MDIOS_MDC"),
 		STM32_FUNCTION(15, "LCD_G0"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(18, "PB2"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOB2"),
 		STM32_FUNCTION(1, "TRACED4"),
 		STM32_FUNCTION(2, "RTC_OUT2"),
 		STM32_FUNCTION(3, "SAI1_D1"),
-		STM32_FUNCTION(4, "DFSDM_CK1"),
+		STM32_FUNCTION(4, "DFSDM1_CKIN1"),
 		STM32_FUNCTION(5, "USART1_RX"),
 		STM32_FUNCTION(6, "I2S_CKIN"),
 		STM32_FUNCTION(7, "SAI1_SD_A"),
 		STM32_FUNCTION(8, "SPI3_MOSI I2S3_SDO"),
-		STM32_FUNCTION(9, "UART4_RX UART_BOOT4_RX"),
+		STM32_FUNCTION(9, "UART4_RX"),
 		STM32_FUNCTION(10, "QUADSPI_CLK"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(19, "PB3"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOB3"),
 		STM32_FUNCTION(1, "TRACED9"),
 		STM32_FUNCTION(2, "TIM2_CH2"),
@@ -303,14 +321,15 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(6, "SPI1_SCK I2S1_CK"),
 		STM32_FUNCTION(7, "SPI3_SCK I2S3_CK"),
 		STM32_FUNCTION(9, "SPI6_SCK"),
-		STM32_FUNCTION(10, "SDMMC2_D2 SDMMC_BOOT2_D2"),
+		STM32_FUNCTION(10, "SDMMC2_D2"),
 		STM32_FUNCTION(13, "SAI4_MCLK_A"),
 		STM32_FUNCTION(14, "UART7_RX"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(20, "PB4"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOB4"),
 		STM32_FUNCTION(1, "TRACED8"),
 		STM32_FUNCTION(2, "TIM16_BKIN"),
@@ -320,14 +339,15 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(7, "SPI3_MISO I2S3_SDI"),
 		STM32_FUNCTION(8, "SPI2_NSS I2S2_WS"),
 		STM32_FUNCTION(9, "SPI6_MISO"),
-		STM32_FUNCTION(10, "SDMMC2_D3 SDMMC_BOOT2_D3"),
+		STM32_FUNCTION(10, "SDMMC2_D3"),
 		STM32_FUNCTION(13, "SAI4_SCK_A"),
 		STM32_FUNCTION(14, "UART7_TX"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(21, "PB5"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOB5"),
 		STM32_FUNCTION(1, "ETH_CLK"),
 		STM32_FUNCTION(2, "TIM17_BKIN"),
@@ -338,166 +358,175 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(7, "I2C4_SMBA"),
 		STM32_FUNCTION(8, "SPI3_MOSI I2S3_SDO"),
 		STM32_FUNCTION(9, "SPI6_MOSI"),
-		STM32_FUNCTION(10, "CAN2_RX"),
+		STM32_FUNCTION(10, "FDCAN2_RX"),
 		STM32_FUNCTION(11, "SAI4_SD_A"),
-		STM32_FUNCTION(12, "ETH_PPS_OUT"),
-		STM32_FUNCTION(13, "UART5_RX UART_BOOT5_RX"),
+		STM32_FUNCTION(12, "ETH1_PPS_OUT"),
+		STM32_FUNCTION(13, "UART5_RX"),
 		STM32_FUNCTION(14, "DCMI_D10"),
 		STM32_FUNCTION(15, "LCD_G7"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(22, "PB6"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOB6"),
 		STM32_FUNCTION(2, "TIM16_CH1N"),
 		STM32_FUNCTION(3, "TIM4_CH1"),
 		STM32_FUNCTION(5, "I2C1_SCL"),
-		STM32_FUNCTION(6, "HDMI_CEC"),
+		STM32_FUNCTION(6, "CEC"),
 		STM32_FUNCTION(7, "I2C4_SCL"),
 		STM32_FUNCTION(8, "USART1_TX"),
-		STM32_FUNCTION(10, "CAN2_TX"),
-		STM32_FUNCTION(11, "QUADSPI_BK1_NCS QUADSPI_BOOTBK1_NCS"),
-		STM32_FUNCTION(12, "DFSDM_DATA5"),
+		STM32_FUNCTION(10, "FDCAN2_TX"),
+		STM32_FUNCTION(11, "QUADSPI_BK1_NCS"),
+		STM32_FUNCTION(12, "DFSDM1_DATIN5"),
 		STM32_FUNCTION(13, "UART5_TX"),
 		STM32_FUNCTION(14, "DCMI_D5"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(23, "PB7"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOB7"),
 		STM32_FUNCTION(2, "TIM17_CH1N"),
 		STM32_FUNCTION(3, "TIM4_CH2"),
 		STM32_FUNCTION(5, "I2C1_SDA"),
 		STM32_FUNCTION(7, "I2C4_SDA"),
 		STM32_FUNCTION(8, "USART1_RX"),
-		STM32_FUNCTION(10, "CAN2_TXFD"),
-		STM32_FUNCTION(11, "SDMMC2_D1 SDMMC_BOOT2_D1"),
-		STM32_FUNCTION(12, "DFSDM_CK5"),
+		STM32_FUNCTION(11, "SDMMC2_D1"),
+		STM32_FUNCTION(12, "DFSDM1_CKIN5"),
 		STM32_FUNCTION(13, "FMC_NL"),
 		STM32_FUNCTION(14, "DCMI_VSYNC"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(24, "PB8"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOB8"),
 		STM32_FUNCTION(1, "HDP6"),
 		STM32_FUNCTION(2, "TIM16_CH1"),
 		STM32_FUNCTION(3, "TIM4_CH3"),
-		STM32_FUNCTION(4, "DFSDM_CK7"),
+		STM32_FUNCTION(4, "DFSDM1_CKIN7"),
 		STM32_FUNCTION(5, "I2C1_SCL"),
-		STM32_FUNCTION(6, "SDMMC1_CKIN SDMMC_BOOT1_CKIN"),
+		STM32_FUNCTION(6, "SDMMC1_CKIN"),
 		STM32_FUNCTION(7, "I2C4_SCL"),
-		STM32_FUNCTION(8, "SDMMC2_CKIN SDMMC_BOOT2_CKIN"),
+		STM32_FUNCTION(8, "SDMMC2_CKIN"),
 		STM32_FUNCTION(9, "UART4_RX"),
-		STM32_FUNCTION(10, "CAN1_RX"),
-		STM32_FUNCTION(11, "SDMMC2_D4 SDMMC_BOOT2_D4"),
-		STM32_FUNCTION(12, "ETH_GMII_TXD3 ETH_MII_TXD3 ETH_RGMII_TXD3"),
-		STM32_FUNCTION(13, "SDMMC1_D4 SDMMC_BOOT1_D4"),
+		STM32_FUNCTION(10, "FDCAN1_RX"),
+		STM32_FUNCTION(11, "SDMMC2_D4"),
+		STM32_FUNCTION(12, "ETH1_GMII_TXD3 ETH1_MII_TXD3 ETH1_RGMII_TXD3"),
+		STM32_FUNCTION(13, "SDMMC1_D4"),
 		STM32_FUNCTION(14, "DCMI_D6"),
 		STM32_FUNCTION(15, "LCD_B6"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(25, "PB9"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOB9"),
 		STM32_FUNCTION(1, "HDP7"),
 		STM32_FUNCTION(2, "TIM17_CH1"),
 		STM32_FUNCTION(3, "TIM4_CH4"),
-		STM32_FUNCTION(4, "DFSDM_DATA7"),
+		STM32_FUNCTION(4, "DFSDM1_DATIN7"),
 		STM32_FUNCTION(5, "I2C1_SDA"),
 		STM32_FUNCTION(6, "SPI2_NSS I2S2_WS"),
 		STM32_FUNCTION(7, "I2C4_SDA"),
-		STM32_FUNCTION(8, "SDMMC2_CDIR SDMMC_BOOT2_CDIR"),
+		STM32_FUNCTION(8, "SDMMC2_CDIR"),
 		STM32_FUNCTION(9, "UART4_TX"),
-		STM32_FUNCTION(10, "CAN1_TX"),
-		STM32_FUNCTION(11, "SDMMC2_D5 SDMMC_BOOT2_D5"),
-		STM32_FUNCTION(12, "SDMMC1_CDIR SDMMC_BOOT1_CDIR"),
-		STM32_FUNCTION(13, "SDMMC1_D5 SDMMC_BOOT1_D5"),
+		STM32_FUNCTION(10, "FDCAN1_TX"),
+		STM32_FUNCTION(11, "SDMMC2_D5"),
+		STM32_FUNCTION(12, "SDMMC1_CDIR"),
+		STM32_FUNCTION(13, "SDMMC1_D5"),
 		STM32_FUNCTION(14, "DCMI_D7"),
 		STM32_FUNCTION(15, "LCD_B7"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(26, "PB10"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOB10"),
 		STM32_FUNCTION(2, "TIM2_CH3"),
 		STM32_FUNCTION(4, "LPTIM2_IN1"),
 		STM32_FUNCTION(5, "I2C2_SCL"),
 		STM32_FUNCTION(6, "SPI2_SCK I2S2_CK"),
-		STM32_FUNCTION(7, "DFSDM_DATA7"),
-		STM32_FUNCTION(8, "USART3_TX USART_BOOT3_TX"),
+		STM32_FUNCTION(7, "DFSDM1_DATIN7"),
+		STM32_FUNCTION(8, "USART3_TX"),
 		STM32_FUNCTION(10, "QUADSPI_BK1_NCS"),
-		STM32_FUNCTION(12, "ETH_GMII_RX_ER ETH_MII_RX_ER"),
+		STM32_FUNCTION(12, "ETH1_GMII_RX_ER ETH1_MII_RX_ER"),
 		STM32_FUNCTION(15, "LCD_G4"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(27, "PB11"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOB11"),
 		STM32_FUNCTION(2, "TIM2_CH4"),
 		STM32_FUNCTION(4, "LPTIM2_ETR"),
 		STM32_FUNCTION(5, "I2C2_SDA"),
-		STM32_FUNCTION(7, "DFSDM_CK7"),
+		STM32_FUNCTION(7, "DFSDM1_CKIN7"),
 		STM32_FUNCTION(8, "USART3_RX"),
-		STM32_FUNCTION(12, "ETH_GMII_TX_EN ETH_MII_TX_EN ETH_RGMII_TX_CTL ETH_RMII_TX_EN"),
+		STM32_FUNCTION(12, "ETH1_GMII_TX_EN ETH1_MII_TX_EN ETH1_RGMII_TX_CTL ETH1_RMII_TX_EN"),
 		STM32_FUNCTION(14, "DSI_TE"),
 		STM32_FUNCTION(15, "LCD_G5"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(28, "PB12"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOB12"),
 		STM32_FUNCTION(2, "TIM1_BKIN"),
 		STM32_FUNCTION(3, "I2C6_SMBA"),
 		STM32_FUNCTION(5, "I2C2_SMBA"),
 		STM32_FUNCTION(6, "SPI2_NSS I2S2_WS"),
-		STM32_FUNCTION(7, "DFSDM_DATA1"),
-		STM32_FUNCTION(8, "USART3_CK USART_BOOT3_CK"),
-		STM32_FUNCTION(9, "USART3_RX USART_BOOT3_RX"),
-		STM32_FUNCTION(10, "CAN2_RX"),
-		STM32_FUNCTION(12, "ETH_GMII_TXD0 ETH_MII_TXD0 ETH_RGMII_TXD0 ETH_RMII_TXD0"),
+		STM32_FUNCTION(7, "DFSDM1_DATIN1"),
+		STM32_FUNCTION(8, "USART3_CK"),
+		STM32_FUNCTION(9, "USART3_RX"),
+		STM32_FUNCTION(10, "FDCAN2_RX"),
+		STM32_FUNCTION(12, "ETH1_GMII_TXD0 ETH1_MII_TXD0 ETH1_RGMII_TXD0 ETH1_RMII_TXD0"),
 		STM32_FUNCTION(15, "UART5_RX"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(29, "PB13"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOB13"),
 		STM32_FUNCTION(2, "TIM1_CH1N"),
-		STM32_FUNCTION(4, "DFSDM_CKOUT"),
+		STM32_FUNCTION(4, "DFSDM1_CKOUT"),
 		STM32_FUNCTION(5, "LPTIM2_OUT"),
 		STM32_FUNCTION(6, "SPI2_SCK I2S2_CK"),
-		STM32_FUNCTION(7, "DFSDM_CK1"),
-		STM32_FUNCTION(8, "USART3_CTS_NSS USART_BOOT3_CTS_NSS"),
-		STM32_FUNCTION(10, "CAN2_TX"),
-		STM32_FUNCTION(12, "ETH_GMII_TXD1 ETH_MII_TXD1 ETH_RGMII_TXD1 ETH_RMII_TXD1"),
-		STM32_FUNCTION(15, "UART5_TX UART_BOOT5_TX"),
+		STM32_FUNCTION(7, "DFSDM1_CKIN1"),
+		STM32_FUNCTION(8, "USART3_CTS USART3_NSS"),
+		STM32_FUNCTION(10, "FDCAN2_TX"),
+		STM32_FUNCTION(12, "ETH1_GMII_TXD1 ETH1_MII_TXD1 ETH1_RGMII_TXD1 ETH1_RMII_TXD1"),
+		STM32_FUNCTION(15, "UART5_TX"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(30, "PB14"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOB14"),
 		STM32_FUNCTION(2, "TIM1_CH2N"),
 		STM32_FUNCTION(3, "TIM12_CH1"),
 		STM32_FUNCTION(4, "TIM8_CH2N"),
 		STM32_FUNCTION(5, "USART1_TX"),
 		STM32_FUNCTION(6, "SPI2_MISO I2S2_SDI"),
-		STM32_FUNCTION(7, "DFSDM_DATA2"),
-		STM32_FUNCTION(8, "USART3_RTS USART_BOOT3_RTS"),
-		STM32_FUNCTION(10, "SDMMC2_D0 SDMMC_BOOT2_D0"),
+		STM32_FUNCTION(7, "DFSDM1_DATIN2"),
+		STM32_FUNCTION(8, "USART3_RTS USART3_DE"),
+		STM32_FUNCTION(10, "SDMMC2_D0"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(31, "PB15"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOB15"),
 		STM32_FUNCTION(1, "RTC_REFIN"),
 		STM32_FUNCTION(2, "TIM1_CH3N"),
@@ -505,523 +534,557 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(4, "TIM8_CH3N"),
 		STM32_FUNCTION(5, "USART1_RX"),
 		STM32_FUNCTION(6, "SPI2_MOSI I2S2_SDO"),
-		STM32_FUNCTION(7, "DFSDM_CK2"),
-		STM32_FUNCTION(10, "SDMMC2_D1 SDMMC_BOOT2_D1"),
+		STM32_FUNCTION(7, "DFSDM1_CKIN2"),
+		STM32_FUNCTION(10, "SDMMC2_D1"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(32, "PC0"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOC0"),
-		STM32_FUNCTION(4, "DFSDM_CK0"),
+		STM32_FUNCTION(4, "DFSDM1_CKIN0"),
 		STM32_FUNCTION(5, "LPTIM2_IN2"),
-		STM32_FUNCTION(7, "DFSDM_DATA4"),
+		STM32_FUNCTION(7, "DFSDM1_DATIN4"),
 		STM32_FUNCTION(9, "SAI2_FS_B"),
-		STM32_FUNCTION(11, "QUADSPI_BK2_NCS QUADSPI_BOOTBK2_NCS"),
+		STM32_FUNCTION(11, "QUADSPI_BK2_NCS"),
 		STM32_FUNCTION(15, "LCD_R5"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(33, "PC1"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOC1"),
 		STM32_FUNCTION(1, "TRACED0"),
 		STM32_FUNCTION(3, "SAI1_D1"),
-		STM32_FUNCTION(4, "DFSDM_DATA0"),
-		STM32_FUNCTION(5, "DFSDM_CK4"),
+		STM32_FUNCTION(4, "DFSDM1_DATIN0"),
+		STM32_FUNCTION(5, "DFSDM1_CKIN4"),
 		STM32_FUNCTION(6, "SPI2_MOSI I2S2_SDO"),
 		STM32_FUNCTION(7, "SAI1_SD_A"),
 		STM32_FUNCTION(10, "SDMMC2_CK"),
-		STM32_FUNCTION(12, "ETH_MDC"),
+		STM32_FUNCTION(12, "ETH1_MDC"),
 		STM32_FUNCTION(13, "MDIOS_MDC"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(34, "PC2"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOC2"),
-		STM32_FUNCTION(4, "DFSDM_CK1"),
+		STM32_FUNCTION(4, "DFSDM1_CKIN1"),
 		STM32_FUNCTION(6, "SPI2_MISO I2S2_SDI"),
-		STM32_FUNCTION(7, "DFSDM_CKOUT"),
-		STM32_FUNCTION(12, "ETH_GMII_TXD2 ETH_MII_TXD2 ETH_RGMII_TXD2"),
+		STM32_FUNCTION(7, "DFSDM1_CKOUT"),
+		STM32_FUNCTION(12, "ETH1_GMII_TXD2 ETH1_MII_TXD2 ETH1_RGMII_TXD2"),
+		STM32_FUNCTION(14, "DCMI_PIXCLK"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(35, "PC3"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOC3"),
 		STM32_FUNCTION(1, "TRACECLK"),
-		STM32_FUNCTION(4, "DFSDM_DATA1"),
+		STM32_FUNCTION(4, "DFSDM1_DATIN1"),
 		STM32_FUNCTION(6, "SPI2_MOSI I2S2_SDO"),
-		STM32_FUNCTION(12, "ETH_GMII_TX_CLK ETH_MII_TX_CLK"),
+		STM32_FUNCTION(12, "ETH1_GMII_TX_CLK ETH1_MII_TX_CLK"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(36, "PC4"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOC4"),
-		STM32_FUNCTION(4, "DFSDM_CK2"),
+		STM32_FUNCTION(4, "DFSDM1_CKIN2"),
 		STM32_FUNCTION(6, "I2S1_MCK"),
-		STM32_FUNCTION(10, "SPDIF_IN2"),
-		STM32_FUNCTION(12, "ETH_GMII_RXD0 ETH_MII_RXD0 ETH_RGMII_RXD0 ETH_RMII_RXD0"),
+		STM32_FUNCTION(10, "SPDIFRX_IN2"),
+		STM32_FUNCTION(12, "ETH1_GMII_RXD0 ETH1_MII_RXD0 ETH1_RGMII_RXD0 ETH1_RMII_RXD0"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(37, "PC5"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOC5"),
 		STM32_FUNCTION(3, "SAI1_D3"),
-		STM32_FUNCTION(4, "DFSDM_DATA2"),
+		STM32_FUNCTION(4, "DFSDM1_DATIN2"),
 		STM32_FUNCTION(5, "SAI4_D4"),
 		STM32_FUNCTION(7, "SAI1_D4"),
-		STM32_FUNCTION(10, "SPDIF_IN3"),
-		STM32_FUNCTION(12, "ETH_GMII_RXD1 ETH_MII_RXD1 ETH_RGMII_RXD1 ETH_RMII_RXD1"),
+		STM32_FUNCTION(10, "SPDIFRX_IN3"),
+		STM32_FUNCTION(12, "ETH1_GMII_RXD1 ETH1_MII_RXD1 ETH1_RGMII_RXD1 ETH1_RMII_RXD1"),
 		STM32_FUNCTION(13, "SAI4_D3"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(38, "PC6"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOC6"),
 		STM32_FUNCTION(1, "HDP1"),
 		STM32_FUNCTION(3, "TIM3_CH1"),
 		STM32_FUNCTION(4, "TIM8_CH1"),
-		STM32_FUNCTION(5, "DFSDM_CK3"),
+		STM32_FUNCTION(5, "DFSDM1_CKIN3"),
 		STM32_FUNCTION(6, "I2S2_MCK"),
-		STM32_FUNCTION(8, "USART6_TX USART_BOOT6_TX"),
-		STM32_FUNCTION(9, "SDMMC1_D0DIR SDMMC_BOOT1_D0DIR"),
-		STM32_FUNCTION(10, "SDMMC2_D0DIR SDMMC_BOOT2_D0DIR"),
-		STM32_FUNCTION(11, "SDMMC2_D6 SDMMC_BOOT2_D6"),
+		STM32_FUNCTION(8, "USART6_TX"),
+		STM32_FUNCTION(9, "SDMMC1_D0DIR"),
+		STM32_FUNCTION(10, "SDMMC2_D0DIR"),
+		STM32_FUNCTION(11, "SDMMC2_D6"),
 		STM32_FUNCTION(12, "DSI_TE"),
-		STM32_FUNCTION(13, "SDMMC1_D6 SDMMC_BOOT1_D6"),
+		STM32_FUNCTION(13, "SDMMC1_D6"),
 		STM32_FUNCTION(14, "DCMI_D0"),
 		STM32_FUNCTION(15, "LCD_HSYNC"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(39, "PC7"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOC7"),
 		STM32_FUNCTION(1, "HDP4"),
 		STM32_FUNCTION(3, "TIM3_CH2"),
 		STM32_FUNCTION(4, "TIM8_CH2"),
-		STM32_FUNCTION(5, "DFSDM_DATA3"),
+		STM32_FUNCTION(5, "DFSDM1_DATIN3"),
 		STM32_FUNCTION(7, "I2S3_MCK"),
-		STM32_FUNCTION(8, "USART6_RX USART_BOOT6_RX"),
-		STM32_FUNCTION(9, "SDMMC1_D123DIR SDMMC_BOOT1_D123DIR"),
-		STM32_FUNCTION(10, "SDMMC2_D123DIR SDMMC_BOOT2_D123DIR"),
-		STM32_FUNCTION(11, "SDMMC2_D7 SDMMC_BOOT2_D7"),
-		STM32_FUNCTION(13, "SDMMC1_D7 SDMMC_BOOT1_D7"),
+		STM32_FUNCTION(8, "USART6_RX"),
+		STM32_FUNCTION(9, "SDMMC1_D123DIR"),
+		STM32_FUNCTION(10, "SDMMC2_D123DIR"),
+		STM32_FUNCTION(11, "SDMMC2_D7"),
+		STM32_FUNCTION(13, "SDMMC1_D7"),
 		STM32_FUNCTION(14, "DCMI_D1"),
 		STM32_FUNCTION(15, "LCD_G6"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(40, "PC8"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOC8"),
 		STM32_FUNCTION(1, "TRACED0"),
 		STM32_FUNCTION(3, "TIM3_CH3"),
 		STM32_FUNCTION(4, "TIM8_CH3"),
 		STM32_FUNCTION(7, "UART4_TX"),
-		STM32_FUNCTION(8, "USART6_CK USART_BOOT6_CK"),
-		STM32_FUNCTION(9, "UART5_RTS UART_BOOT5_RTS"),
-		STM32_FUNCTION(13, "SDMMC1_D0 SDMMC_BOOT1_D0"),
+		STM32_FUNCTION(8, "USART6_CK"),
+		STM32_FUNCTION(9, "UART5_RTS UART5_DE"),
+		STM32_FUNCTION(13, "SDMMC1_D0"),
 		STM32_FUNCTION(14, "DCMI_D2"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(41, "PC9"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOC9"),
 		STM32_FUNCTION(1, "TRACED1"),
 		STM32_FUNCTION(3, "TIM3_CH4"),
 		STM32_FUNCTION(4, "TIM8_CH4"),
 		STM32_FUNCTION(5, "I2C3_SDA"),
 		STM32_FUNCTION(6, "I2S_CKIN"),
-		STM32_FUNCTION(9, "UART5_CTS UART_BOOT5_CTS"),
+		STM32_FUNCTION(9, "UART5_CTS"),
 		STM32_FUNCTION(10, "QUADSPI_BK1_IO0"),
-		STM32_FUNCTION(13, "SDMMC1_D1 SDMMC_BOOT1_D1"),
+		STM32_FUNCTION(13, "SDMMC1_D1"),
 		STM32_FUNCTION(14, "DCMI_D3"),
 		STM32_FUNCTION(15, "LCD_B2"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(42, "PC10"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOC10"),
 		STM32_FUNCTION(1, "TRACED2"),
-		STM32_FUNCTION(4, "DFSDM_CK5"),
+		STM32_FUNCTION(4, "DFSDM1_CKIN5"),
 		STM32_FUNCTION(7, "SPI3_SCK I2S3_CK"),
 		STM32_FUNCTION(8, "USART3_TX"),
 		STM32_FUNCTION(9, "UART4_TX"),
 		STM32_FUNCTION(10, "QUADSPI_BK1_IO1"),
 		STM32_FUNCTION(11, "SAI4_MCLK_B"),
-		STM32_FUNCTION(13, "SDMMC1_D2 SDMMC_BOOT1_D2"),
+		STM32_FUNCTION(13, "SDMMC1_D2"),
 		STM32_FUNCTION(14, "DCMI_D8"),
 		STM32_FUNCTION(15, "LCD_R2"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(43, "PC11"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOC11"),
 		STM32_FUNCTION(1, "TRACED3"),
-		STM32_FUNCTION(4, "DFSDM_DATA5"),
+		STM32_FUNCTION(4, "DFSDM1_DATIN5"),
 		STM32_FUNCTION(7, "SPI3_MISO I2S3_SDI"),
 		STM32_FUNCTION(8, "USART3_RX"),
 		STM32_FUNCTION(9, "UART4_RX"),
-		STM32_FUNCTION(10, "QUADSPI_BK2_NCS QUADSPI_BOOTBK2_NCS"),
+		STM32_FUNCTION(10, "QUADSPI_BK2_NCS"),
 		STM32_FUNCTION(11, "SAI4_SCK_B"),
-		STM32_FUNCTION(13, "SDMMC1_D3 SDMMC_BOOT1_D3"),
+		STM32_FUNCTION(13, "SDMMC1_D3"),
 		STM32_FUNCTION(14, "DCMI_D4"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(44, "PC12"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOC12"),
 		STM32_FUNCTION(1, "TRACECLK"),
 		STM32_FUNCTION(2, "MCO2"),
 		STM32_FUNCTION(3, "SAI4_D3"),
 		STM32_FUNCTION(7, "SPI3_MOSI I2S3_SDO"),
-		STM32_FUNCTION(8, "USART3_CK USART_BOOT3_CK"),
+		STM32_FUNCTION(8, "USART3_CK"),
 		STM32_FUNCTION(9, "UART5_TX"),
 		STM32_FUNCTION(11, "SAI4_SD_B"),
-		STM32_FUNCTION(13, "SDMMC1_CK SDMMC_BOOT1_CK"),
+		STM32_FUNCTION(13, "SDMMC1_CK"),
 		STM32_FUNCTION(14, "DCMI_D9"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(45, "PC13"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOC13"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(46, "PC14"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOC14"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(47, "PC15"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOC15"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(48, "PD0"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOD0"),
 		STM32_FUNCTION(3, "I2C6_SDA"),
-		STM32_FUNCTION(4, "DFSDM_CK6"),
+		STM32_FUNCTION(4, "DFSDM1_CKIN6"),
 		STM32_FUNCTION(5, "I2C5_SDA"),
 		STM32_FUNCTION(7, "SAI3_SCK_A"),
 		STM32_FUNCTION(9, "UART4_RX"),
-		STM32_FUNCTION(10, "CAN1_RX"),
+		STM32_FUNCTION(10, "FDCAN1_RX"),
 		STM32_FUNCTION(11, "SDMMC3_CMD"),
-		STM32_FUNCTION(12, "DFSDM_DATA7"),
-		STM32_FUNCTION(13, "FMC_D2"),
+		STM32_FUNCTION(12, "DFSDM1_DATIN7"),
+		STM32_FUNCTION(13, "FMC_D2 FMC_DA2"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(49, "PD1"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOD1"),
 		STM32_FUNCTION(3, "I2C6_SCL"),
-		STM32_FUNCTION(4, "DFSDM_DATA6"),
+		STM32_FUNCTION(4, "DFSDM1_DATIN6"),
 		STM32_FUNCTION(5, "I2C5_SCL"),
 		STM32_FUNCTION(7, "SAI3_SD_A"),
 		STM32_FUNCTION(9, "UART4_TX"),
-		STM32_FUNCTION(10, "CAN1_TX"),
+		STM32_FUNCTION(10, "FDCAN1_TX"),
 		STM32_FUNCTION(11, "SDMMC3_D0"),
-		STM32_FUNCTION(12, "DFSDM_CK7"),
-		STM32_FUNCTION(13, "FMC_D3"),
+		STM32_FUNCTION(12, "DFSDM1_CKIN7"),
+		STM32_FUNCTION(13, "FMC_D3 FMC_DA3"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(50, "PD2"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOD2"),
 		STM32_FUNCTION(3, "TIM3_ETR"),
 		STM32_FUNCTION(5, "I2C5_SMBA"),
 		STM32_FUNCTION(7, "UART4_RX"),
 		STM32_FUNCTION(9, "UART5_RX"),
-		STM32_FUNCTION(13, "SDMMC1_CMD SDMMC_BOOT1_CMD"),
+		STM32_FUNCTION(13, "SDMMC1_CMD"),
 		STM32_FUNCTION(14, "DCMI_D11"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(51, "PD3"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOD3"),
 		STM32_FUNCTION(1, "HDP5"),
-		STM32_FUNCTION(4, "DFSDM_CKOUT"),
+		STM32_FUNCTION(4, "DFSDM1_CKOUT"),
 		STM32_FUNCTION(6, "SPI2_SCK I2S2_CK"),
-		STM32_FUNCTION(7, "DFSDM_DATA0"),
-		STM32_FUNCTION(8, "USART2_CTS_NSS USART_BOOT2_CTS_NSS"),
-		STM32_FUNCTION(9, "SDMMC1_D123DIR SDMMC_BOOT1_D123DIR"),
-		STM32_FUNCTION(10, "SDMMC2_D7 SDMMC_BOOT2_D7"),
-		STM32_FUNCTION(11, "SDMMC2_D123DIR SDMMC_BOOT2_D123DIR"),
-		STM32_FUNCTION(12, "SDMMC1_D7 SDMMC_BOOT1_D7"),
+		STM32_FUNCTION(7, "DFSDM1_DATIN0"),
+		STM32_FUNCTION(8, "USART2_CTS USART2_NSS"),
+		STM32_FUNCTION(9, "SDMMC1_D123DIR"),
+		STM32_FUNCTION(10, "SDMMC2_D7"),
+		STM32_FUNCTION(11, "SDMMC2_D123DIR"),
+		STM32_FUNCTION(12, "SDMMC1_D7"),
 		STM32_FUNCTION(13, "FMC_CLK"),
 		STM32_FUNCTION(14, "DCMI_D5"),
 		STM32_FUNCTION(15, "LCD_G7"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(52, "PD4"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOD4"),
 		STM32_FUNCTION(7, "SAI3_FS_A"),
-		STM32_FUNCTION(8, "USART2_RTS USART_BOOT2_RTS"),
-		STM32_FUNCTION(10, "CAN1_RXFD"),
+		STM32_FUNCTION(8, "USART2_RTS USART2_DE"),
 		STM32_FUNCTION(11, "SDMMC3_D1"),
-		STM32_FUNCTION(12, "DFSDM_CK0"),
+		STM32_FUNCTION(12, "DFSDM1_CKIN0"),
 		STM32_FUNCTION(13, "FMC_NOE"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(53, "PD5"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOD5"),
 		STM32_FUNCTION(8, "USART2_TX"),
-		STM32_FUNCTION(10, "CAN1_TXFD"),
 		STM32_FUNCTION(11, "SDMMC3_D2"),
 		STM32_FUNCTION(13, "FMC_NWE"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(54, "PD6"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOD6"),
 		STM32_FUNCTION(2, "TIM16_CH1N"),
 		STM32_FUNCTION(3, "SAI1_D1"),
-		STM32_FUNCTION(4, "DFSDM_CK4"),
-		STM32_FUNCTION(5, "DFSDM_DATA1"),
+		STM32_FUNCTION(4, "DFSDM1_CKIN4"),
+		STM32_FUNCTION(5, "DFSDM1_DATIN1"),
 		STM32_FUNCTION(6, "SPI3_MOSI I2S3_SDO"),
 		STM32_FUNCTION(7, "SAI1_SD_A"),
 		STM32_FUNCTION(8, "USART2_RX"),
-		STM32_FUNCTION(10, "CAN2_RXFD"),
-		STM32_FUNCTION(11, "FMC_INT"),
 		STM32_FUNCTION(13, "FMC_NWAIT"),
 		STM32_FUNCTION(14, "DCMI_D10"),
 		STM32_FUNCTION(15, "LCD_B2"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(55, "PD7"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOD7"),
 		STM32_FUNCTION(1, "TRACED6"),
-		STM32_FUNCTION(4, "DFSDM_DATA4"),
+		STM32_FUNCTION(4, "DFSDM1_DATIN4"),
 		STM32_FUNCTION(5, "I2C2_SCL"),
-		STM32_FUNCTION(7, "DFSDM_CK1"),
-		STM32_FUNCTION(8, "USART2_CK USART_BOOT2_CK"),
-		STM32_FUNCTION(10, "SPDIF_IN0"),
+		STM32_FUNCTION(7, "DFSDM1_CKIN1"),
+		STM32_FUNCTION(8, "USART2_CK"),
+		STM32_FUNCTION(10, "SPDIFRX_IN0"),
 		STM32_FUNCTION(11, "SDMMC3_D3"),
 		STM32_FUNCTION(13, "FMC_NE1"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(56, "PD8"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOD8"),
-		STM32_FUNCTION(4, "DFSDM_CK3"),
+		STM32_FUNCTION(4, "DFSDM1_CKIN3"),
 		STM32_FUNCTION(7, "SAI3_SCK_B"),
 		STM32_FUNCTION(8, "USART3_TX"),
-		STM32_FUNCTION(10, "SPDIF_IN1"),
-		STM32_FUNCTION(13, "FMC_D13"),
+		STM32_FUNCTION(10, "SPDIFRX_IN1"),
+		STM32_FUNCTION(13, "FMC_D13 FMC_DA13"),
 		STM32_FUNCTION(15, "LCD_B7"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(57, "PD9"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOD9"),
-		STM32_FUNCTION(4, "DFSDM_DATA3"),
+		STM32_FUNCTION(4, "DFSDM1_DATIN3"),
 		STM32_FUNCTION(7, "SAI3_SD_B"),
 		STM32_FUNCTION(8, "USART3_RX"),
-		STM32_FUNCTION(10, "CAN2_RXFD"),
-		STM32_FUNCTION(13, "FMC_D14"),
+		STM32_FUNCTION(13, "FMC_D14 FMC_DA14"),
+		STM32_FUNCTION(14, "DCMI_HSYNC"),
 		STM32_FUNCTION(15, "LCD_B0"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(58, "PD10"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOD10"),
 		STM32_FUNCTION(1, "RTC_REFIN"),
 		STM32_FUNCTION(2, "TIM16_BKIN"),
-		STM32_FUNCTION(4, "DFSDM_CKOUT"),
+		STM32_FUNCTION(4, "DFSDM1_CKOUT"),
 		STM32_FUNCTION(5, "I2C5_SMBA"),
 		STM32_FUNCTION(6, "SPI3_MISO I2S3_SDI"),
 		STM32_FUNCTION(7, "SAI3_FS_B"),
-		STM32_FUNCTION(8, "USART3_CK USART_BOOT3_CK"),
-		STM32_FUNCTION(10, "CAN2_TXFD"),
-		STM32_FUNCTION(13, "FMC_D15"),
+		STM32_FUNCTION(8, "USART3_CK"),
+		STM32_FUNCTION(13, "FMC_D15 FMC_DA15"),
 		STM32_FUNCTION(15, "LCD_B3"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(59, "PD11"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOD11"),
 		STM32_FUNCTION(4, "LPTIM2_IN2"),
 		STM32_FUNCTION(5, "I2C4_SMBA"),
 		STM32_FUNCTION(6, "I2C1_SMBA"),
-		STM32_FUNCTION(8, "USART3_CTS_NSS USART_BOOT3_CTS_NSS"),
+		STM32_FUNCTION(8, "USART3_CTS USART3_NSS"),
 		STM32_FUNCTION(10, "QUADSPI_BK1_IO0"),
 		STM32_FUNCTION(11, "SAI2_SD_A"),
-		STM32_FUNCTION(13, "FMC_A16 FMC_CLE"),
+		STM32_FUNCTION(13, "FMC_CLE FMC_A16"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(60, "PD12"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOD12"),
 		STM32_FUNCTION(2, "LPTIM1_IN1"),
 		STM32_FUNCTION(3, "TIM4_CH1"),
 		STM32_FUNCTION(4, "LPTIM2_IN1"),
 		STM32_FUNCTION(5, "I2C4_SCL"),
 		STM32_FUNCTION(6, "I2C1_SCL"),
-		STM32_FUNCTION(8, "USART3_RTS USART_BOOT3_RTS"),
+		STM32_FUNCTION(8, "USART3_RTS USART3_DE"),
 		STM32_FUNCTION(10, "QUADSPI_BK1_IO1"),
 		STM32_FUNCTION(11, "SAI2_FS_A"),
-		STM32_FUNCTION(13, "FMC_A17 FMC_ALE"),
+		STM32_FUNCTION(13, "FMC_ALE FMC_A17"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(61, "PD13"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOD13"),
 		STM32_FUNCTION(2, "LPTIM1_OUT"),
 		STM32_FUNCTION(3, "TIM4_CH2"),
 		STM32_FUNCTION(5, "I2C4_SDA"),
 		STM32_FUNCTION(6, "I2C1_SDA"),
 		STM32_FUNCTION(7, "I2S3_MCK"),
-		STM32_FUNCTION(10, "QUADSPI_BK1_IO3 QUADSPI_BOOTBK1_IO3"),
+		STM32_FUNCTION(10, "QUADSPI_BK1_IO3"),
 		STM32_FUNCTION(11, "SAI2_SCK_A"),
 		STM32_FUNCTION(13, "FMC_A18"),
 		STM32_FUNCTION(14, "DSI_TE"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(62, "PD14"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOD14"),
 		STM32_FUNCTION(3, "TIM4_CH3"),
 		STM32_FUNCTION(7, "SAI3_MCLK_B"),
-		STM32_FUNCTION(9, "UART8_CTS UART_BOOT8_CTS"),
-		STM32_FUNCTION(13, "FMC_D0"),
+		STM32_FUNCTION(9, "UART8_CTS"),
+		STM32_FUNCTION(13, "FMC_D0 FMC_DA0"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(63, "PD15"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOD15"),
 		STM32_FUNCTION(3, "TIM4_CH4"),
 		STM32_FUNCTION(7, "SAI3_MCLK_A"),
-		STM32_FUNCTION(9, "UART8_CTS UART_BOOT8_CTS"),
-		STM32_FUNCTION(13, "FMC_D1"),
+		STM32_FUNCTION(9, "UART8_CTS"),
+		STM32_FUNCTION(13, "FMC_D1 FMC_DA1"),
+		STM32_FUNCTION(15, "LCD_R1"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(64, "PE0"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOE0"),
 		STM32_FUNCTION(2, "LPTIM1_ETR"),
 		STM32_FUNCTION(3, "TIM4_ETR"),
 		STM32_FUNCTION(5, "LPTIM2_ETR"),
 		STM32_FUNCTION(6, "SPI3_SCK I2S3_CK"),
 		STM32_FUNCTION(7, "SAI4_MCLK_B"),
-		STM32_FUNCTION(9, "UART8_RX UART_BOOT8_RX"),
-		STM32_FUNCTION(10, "CAN1_RXFD"),
+		STM32_FUNCTION(9, "UART8_RX"),
 		STM32_FUNCTION(11, "SAI2_MCLK_A"),
 		STM32_FUNCTION(13, "FMC_NBL0"),
 		STM32_FUNCTION(14, "DCMI_D2"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(65, "PE1"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOE1"),
 		STM32_FUNCTION(2, "LPTIM1_IN2"),
 		STM32_FUNCTION(6, "I2S2_MCK"),
 		STM32_FUNCTION(7, "SAI3_SD_B"),
-		STM32_FUNCTION(9, "UART8_TX UART_BOOT8_TX"),
-		STM32_FUNCTION(10, "CAN1_TXFD"),
+		STM32_FUNCTION(9, "UART8_TX"),
 		STM32_FUNCTION(13, "FMC_NBL1"),
 		STM32_FUNCTION(14, "DCMI_D3"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(66, "PE2"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOE2"),
 		STM32_FUNCTION(1, "TRACECLK"),
 		STM32_FUNCTION(3, "SAI1_CK1"),
 		STM32_FUNCTION(5, "I2C4_SCL"),
 		STM32_FUNCTION(6, "SPI4_SCK"),
 		STM32_FUNCTION(7, "SAI1_MCLK_A"),
-		STM32_FUNCTION(10, "QUADSPI_BK1_IO2 QUADSPI_BOOTBK1_IO2"),
-		STM32_FUNCTION(12, "ETH_GMII_TXD3 ETH_MII_TXD3 ETH_RGMII_TXD3"),
+		STM32_FUNCTION(10, "QUADSPI_BK1_IO2"),
+		STM32_FUNCTION(12, "ETH1_GMII_TXD3 ETH1_MII_TXD3 ETH1_RGMII_TXD3"),
 		STM32_FUNCTION(13, "FMC_A23"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(67, "PE3"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOE3"),
 		STM32_FUNCTION(1, "TRACED0"),
 		STM32_FUNCTION(5, "TIM15_BKIN"),
 		STM32_FUNCTION(7, "SAI1_SD_B"),
-		STM32_FUNCTION(10, "SDMMC2_CK SDMMC_BOOT2_CK"),
+		STM32_FUNCTION(10, "SDMMC2_CK"),
 		STM32_FUNCTION(13, "FMC_A19"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(68, "PE4"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOE4"),
 		STM32_FUNCTION(1, "TRACED1"),
 		STM32_FUNCTION(3, "SAI1_D2"),
-		STM32_FUNCTION(4, "DFSDM_DATA3"),
+		STM32_FUNCTION(4, "DFSDM1_DATIN3"),
 		STM32_FUNCTION(5, "TIM15_CH1N"),
 		STM32_FUNCTION(6, "SPI4_NSS"),
 		STM32_FUNCTION(7, "SAI1_FS_A"),
-		STM32_FUNCTION(8, "SDMMC2_CKIN SDMMC_BOOT2_CKIN"),
-		STM32_FUNCTION(9, "SDMMC1_CKIN SDMMC_BOOT1_CKIN"),
-		STM32_FUNCTION(10, "SDMMC2_D4 SDMMC_BOOT2_D4"),
-		STM32_FUNCTION(12, "SDMMC1_D4 SDMMC_BOOT1_D4"),
+		STM32_FUNCTION(8, "SDMMC2_CKIN"),
+		STM32_FUNCTION(9, "SDMMC1_CKIN"),
+		STM32_FUNCTION(10, "SDMMC2_D4"),
+		STM32_FUNCTION(12, "SDMMC1_D4"),
 		STM32_FUNCTION(13, "FMC_A20"),
 		STM32_FUNCTION(14, "DCMI_D4"),
 		STM32_FUNCTION(15, "LCD_B0"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(69, "PE5"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOE5"),
 		STM32_FUNCTION(1, "TRACED3"),
 		STM32_FUNCTION(3, "SAI1_CK2"),
-		STM32_FUNCTION(4, "DFSDM_CK3"),
+		STM32_FUNCTION(4, "DFSDM1_CKIN3"),
 		STM32_FUNCTION(5, "TIM15_CH1"),
 		STM32_FUNCTION(6, "SPI4_MISO"),
 		STM32_FUNCTION(7, "SAI1_SCK_A"),
-		STM32_FUNCTION(8, "SDMMC2_D0DIR SDMMC_BOOT2_D0DIR"),
-		STM32_FUNCTION(9, "SDMMC1_D0DIR SDMMC_BOOT1_D0DIR"),
-		STM32_FUNCTION(10, "SDMMC2_D6 SDMMC_BOOT2_D6"),
-		STM32_FUNCTION(12, "SDMMC1_D6 SDMMC_BOOT1_D6"),
+		STM32_FUNCTION(8, "SDMMC2_D0DIR"),
+		STM32_FUNCTION(9, "SDMMC1_D0DIR"),
+		STM32_FUNCTION(10, "SDMMC2_D6"),
+		STM32_FUNCTION(12, "SDMMC1_D6"),
 		STM32_FUNCTION(13, "FMC_A21"),
 		STM32_FUNCTION(14, "DCMI_D6"),
 		STM32_FUNCTION(15, "LCD_G0"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(70, "PE6"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOE6"),
 		STM32_FUNCTION(1, "TRACED2"),
 		STM32_FUNCTION(2, "TIM1_BKIN2"),
@@ -1030,7 +1093,7 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(6, "SPI4_MOSI"),
 		STM32_FUNCTION(7, "SAI1_SD_A"),
 		STM32_FUNCTION(8, "SDMMC2_D0"),
-		STM32_FUNCTION(9, "SDMMC1_D2 SDMMC_BOOT1_D2"),
+		STM32_FUNCTION(9, "SDMMC1_D2"),
 		STM32_FUNCTION(11, "SAI2_MCLK_B"),
 		STM32_FUNCTION(13, "FMC_A22"),
 		STM32_FUNCTION(14, "DCMI_D7"),
@@ -1038,119 +1101,132 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(71, "PE7"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOE7"),
 		STM32_FUNCTION(2, "TIM1_ETR"),
 		STM32_FUNCTION(3, "TIM3_ETR"),
-		STM32_FUNCTION(4, "DFSDM_DATA2"),
+		STM32_FUNCTION(4, "DFSDM1_DATIN2"),
 		STM32_FUNCTION(8, "UART7_RX"),
-		STM32_FUNCTION(11, "QUADSPI_BK2_IO0 QUADSPI_BOOTBK2_IO0"),
-		STM32_FUNCTION(13, "FMC_D4"),
+		STM32_FUNCTION(11, "QUADSPI_BK2_IO0"),
+		STM32_FUNCTION(13, "FMC_D4 FMC_DA4"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(72, "PE8"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOE8"),
 		STM32_FUNCTION(2, "TIM1_CH1N"),
-		STM32_FUNCTION(4, "DFSDM_CK2"),
+		STM32_FUNCTION(4, "DFSDM1_CKIN2"),
 		STM32_FUNCTION(8, "UART7_TX"),
-		STM32_FUNCTION(11, "QUADSPI_BK2_IO1 QUADSPI_BOOTBK2_IO1"),
-		STM32_FUNCTION(13, "FMC_D5"),
+		STM32_FUNCTION(11, "QUADSPI_BK2_IO1"),
+		STM32_FUNCTION(13, "FMC_D5 FMC_DA5"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(73, "PE9"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOE9"),
 		STM32_FUNCTION(2, "TIM1_CH1"),
-		STM32_FUNCTION(4, "DFSDM_CKOUT"),
-		STM32_FUNCTION(8, "UART7_RTS UART_BOOT7_RTS"),
-		STM32_FUNCTION(11, "QUADSPI_BK2_IO2 QUADSPI_BOOTBK2_IO2"),
-		STM32_FUNCTION(13, "FMC_D6"),
+		STM32_FUNCTION(4, "DFSDM1_CKOUT"),
+		STM32_FUNCTION(8, "UART7_RTS UART7_DE"),
+		STM32_FUNCTION(11, "QUADSPI_BK2_IO2"),
+		STM32_FUNCTION(13, "FMC_D6 FMC_DA6"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(74, "PE10"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOE10"),
 		STM32_FUNCTION(2, "TIM1_CH2N"),
-		STM32_FUNCTION(4, "DFSDM_DATA4"),
-		STM32_FUNCTION(8, "UART7_CTS UART_BOOT7_CTS"),
-		STM32_FUNCTION(11, "QUADSPI_BK2_IO3 QUADSPI_BOOTBK2_IO3"),
-		STM32_FUNCTION(13, "FMC_D7"),
+		STM32_FUNCTION(4, "DFSDM1_DATIN4"),
+		STM32_FUNCTION(8, "UART7_CTS"),
+		STM32_FUNCTION(11, "QUADSPI_BK2_IO3"),
+		STM32_FUNCTION(13, "FMC_D7 FMC_DA7"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(75, "PE11"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOE11"),
 		STM32_FUNCTION(2, "TIM1_CH2"),
-		STM32_FUNCTION(4, "DFSDM_CK4"),
+		STM32_FUNCTION(4, "DFSDM1_CKIN4"),
 		STM32_FUNCTION(6, "SPI4_NSS"),
-		STM32_FUNCTION(8, "USART6_CK USART_BOOT6_CK"),
+		STM32_FUNCTION(8, "USART6_CK"),
 		STM32_FUNCTION(11, "SAI2_SD_B"),
-		STM32_FUNCTION(13, "FMC_D8"),
+		STM32_FUNCTION(13, "FMC_D8 FMC_DA8"),
+		STM32_FUNCTION(14, "DCMI_D4"),
 		STM32_FUNCTION(15, "LCD_G3"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(76, "PE12"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOE12"),
 		STM32_FUNCTION(2, "TIM1_CH3N"),
-		STM32_FUNCTION(4, "DFSDM_DATA5"),
+		STM32_FUNCTION(4, "DFSDM1_DATIN5"),
 		STM32_FUNCTION(6, "SPI4_SCK"),
-		STM32_FUNCTION(9, "SDMMC1_D0DIR SDMMC_BOOT1_D0DIR"),
+		STM32_FUNCTION(9, "SDMMC1_D0DIR"),
 		STM32_FUNCTION(11, "SAI2_SCK_B"),
-		STM32_FUNCTION(13, "FMC_D9"),
+		STM32_FUNCTION(13, "FMC_D9 FMC_DA9"),
 		STM32_FUNCTION(15, "LCD_B4"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(77, "PE13"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOE13"),
 		STM32_FUNCTION(1, "HDP2"),
 		STM32_FUNCTION(2, "TIM1_CH3"),
-		STM32_FUNCTION(4, "DFSDM_CK5"),
+		STM32_FUNCTION(4, "DFSDM1_CKIN5"),
 		STM32_FUNCTION(6, "SPI4_MISO"),
 		STM32_FUNCTION(11, "SAI2_FS_B"),
-		STM32_FUNCTION(13, "FMC_D10"),
+		STM32_FUNCTION(13, "FMC_D10 FMC_DA10"),
+		STM32_FUNCTION(14, "DCMI_D6"),
 		STM32_FUNCTION(15, "LCD_DE"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(78, "PE14"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOE14"),
 		STM32_FUNCTION(2, "TIM1_CH4"),
 		STM32_FUNCTION(6, "SPI4_MOSI"),
-		STM32_FUNCTION(9, "UART8_RTS UART_BOOT8_RTS"),
+		STM32_FUNCTION(9, "UART8_RTS UART8_DE"),
 		STM32_FUNCTION(11, "SAI2_MCLK_B"),
-		STM32_FUNCTION(12, "SDMMC1_D123DIR SDMMC_BOOT1_D123DIR"),
-		STM32_FUNCTION(13, "FMC_D11"),
+		STM32_FUNCTION(12, "SDMMC1_D123DIR"),
+		STM32_FUNCTION(13, "FMC_D11 FMC_DA11"),
 		STM32_FUNCTION(14, "LCD_G0"),
 		STM32_FUNCTION(15, "LCD_CLK"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(79, "PE15"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOE15"),
 		STM32_FUNCTION(1, "HDP3"),
 		STM32_FUNCTION(2, "TIM1_BKIN"),
 		STM32_FUNCTION(5, "TIM15_BKIN"),
-		STM32_FUNCTION(8, "USART2_CTS_NSS USART_BOOT2_CTS_NSS"),
-		STM32_FUNCTION(9, "UART8_CTS UART_BOOT8_CTS"),
-		STM32_FUNCTION(13, "FMC_D12"),
+		STM32_FUNCTION(8, "USART2_CTS USART2_NSS"),
+		STM32_FUNCTION(9, "UART8_CTS"),
+		STM32_FUNCTION(11, "FMC_NCE2"),
+		STM32_FUNCTION(13, "FMC_D12 FMC_DA12"),
 		STM32_FUNCTION(15, "LCD_R7"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(80, "PF0"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOF0"),
 		STM32_FUNCTION(5, "I2C2_SDA"),
 		STM32_FUNCTION(10, "SDMMC3_D0"),
@@ -1159,8 +1235,9 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(81, "PF1"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOF1"),
 		STM32_FUNCTION(5, "I2C2_SCL"),
 		STM32_FUNCTION(10, "SDMMC3_CMD"),
@@ -1169,27 +1246,30 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(82, "PF2"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOF2"),
 		STM32_FUNCTION(5, "I2C2_SMBA"),
-		STM32_FUNCTION(10, "SDMMC2_D0DIR SDMMC_BOOT2_D0DIR"),
+		STM32_FUNCTION(10, "SDMMC2_D0DIR"),
 		STM32_FUNCTION(11, "SDMMC3_D0DIR"),
-		STM32_FUNCTION(12, "SDMMC1_D0DIR SDMMC_BOOT1_D0DIR"),
+		STM32_FUNCTION(12, "SDMMC1_D0DIR"),
 		STM32_FUNCTION(13, "FMC_A2"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(83, "PF3"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOF3"),
-		STM32_FUNCTION(12, "ETH_GMII_TX_ER ETH_MII_TX_ER"),
+		STM32_FUNCTION(12, "ETH1_GMII_TX_ER ETH1_MII_TX_ER"),
 		STM32_FUNCTION(13, "FMC_A3"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(84, "PF4"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOF4"),
 		STM32_FUNCTION(8, "USART2_RX"),
 		STM32_FUNCTION(10, "SDMMC3_D1"),
@@ -1198,8 +1278,9 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(85, "PF5"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOF5"),
 		STM32_FUNCTION(8, "USART2_TX"),
 		STM32_FUNCTION(10, "SDMMC3_D2"),
@@ -1207,71 +1288,77 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(86, "PF6"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOF6"),
 		STM32_FUNCTION(2, "TIM16_CH1"),
 		STM32_FUNCTION(6, "SPI5_NSS"),
 		STM32_FUNCTION(7, "SAI1_SD_B"),
-		STM32_FUNCTION(8, "UART7_RX UART_BOOT7_RX"),
-		STM32_FUNCTION(10, "QUADSPI_BK1_IO3 QUADSPI_BOOTBK1_IO3"),
+		STM32_FUNCTION(8, "UART7_RX"),
+		STM32_FUNCTION(10, "QUADSPI_BK1_IO3"),
 		STM32_FUNCTION(13, "SAI4_SCK_B"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(87, "PF7"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOF7"),
 		STM32_FUNCTION(2, "TIM17_CH1"),
 		STM32_FUNCTION(6, "SPI5_SCK"),
 		STM32_FUNCTION(7, "SAI1_MCLK_B"),
-		STM32_FUNCTION(8, "UART7_TX UART_BOOT7_TX"),
-		STM32_FUNCTION(10, "QUADSPI_BK1_IO2 QUADSPI_BOOTBK1_IO2"),
+		STM32_FUNCTION(8, "UART7_TX"),
+		STM32_FUNCTION(10, "QUADSPI_BK1_IO2"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(88, "PF8"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOF8"),
 		STM32_FUNCTION(1, "TRACED12"),
 		STM32_FUNCTION(2, "TIM16_CH1N"),
 		STM32_FUNCTION(6, "SPI5_MISO"),
 		STM32_FUNCTION(7, "SAI1_SCK_B"),
-		STM32_FUNCTION(8, "UART7_RTS UART_BOOT7_RTS"),
+		STM32_FUNCTION(8, "UART7_RTS UART7_DE"),
 		STM32_FUNCTION(10, "TIM13_CH1"),
-		STM32_FUNCTION(11, "QUADSPI_BK1_IO0 QUADSPI_BOOTBK1_IO0"),
+		STM32_FUNCTION(11, "QUADSPI_BK1_IO0"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(89, "PF9"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOF9"),
 		STM32_FUNCTION(1, "TRACED13"),
 		STM32_FUNCTION(2, "TIM17_CH1N"),
 		STM32_FUNCTION(6, "SPI5_MOSI"),
 		STM32_FUNCTION(7, "SAI1_FS_B"),
-		STM32_FUNCTION(8, "UART7_CTS UART_BOOT7_CTS"),
+		STM32_FUNCTION(8, "UART7_CTS"),
 		STM32_FUNCTION(10, "TIM14_CH1"),
-		STM32_FUNCTION(11, "QUADSPI_BK1_IO1 QUADSPI_BOOTBK1_IO1"),
+		STM32_FUNCTION(11, "QUADSPI_BK1_IO1"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(90, "PF10"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOF10"),
 		STM32_FUNCTION(2, "TIM16_BKIN"),
 		STM32_FUNCTION(3, "SAI1_D3"),
 		STM32_FUNCTION(4, "SAI4_D4"),
 		STM32_FUNCTION(7, "SAI1_D4"),
-		STM32_FUNCTION(10, "QUADSPI_CLK QUADSPI_BOOTCLK"),
+		STM32_FUNCTION(10, "QUADSPI_CLK"),
 		STM32_FUNCTION(13, "SAI4_D3"),
 		STM32_FUNCTION(14, "DCMI_D11"),
 		STM32_FUNCTION(15, "LCD_DE"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(91, "PF11"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOF11"),
 		STM32_FUNCTION(6, "SPI5_MOSI"),
 		STM32_FUNCTION(11, "SAI2_SD_B"),
@@ -1280,138 +1367,151 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(92, "PF12"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOF12"),
 		STM32_FUNCTION(1, "TRACED4"),
-		STM32_FUNCTION(12, "ETH_GMII_RXD4"),
+		STM32_FUNCTION(12, "ETH1_GMII_RXD4"),
 		STM32_FUNCTION(13, "FMC_A6"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(93, "PF13"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOF13"),
 		STM32_FUNCTION(1, "TRACED5"),
-		STM32_FUNCTION(4, "DFSDM_DATA6"),
+		STM32_FUNCTION(4, "DFSDM1_DATIN6"),
 		STM32_FUNCTION(5, "I2C4_SMBA"),
 		STM32_FUNCTION(6, "I2C1_SMBA"),
-		STM32_FUNCTION(7, "DFSDM_DATA3"),
-		STM32_FUNCTION(12, "ETH_GMII_RXD5"),
+		STM32_FUNCTION(7, "DFSDM1_DATIN3"),
+		STM32_FUNCTION(12, "ETH1_GMII_RXD5"),
 		STM32_FUNCTION(13, "FMC_A7"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(94, "PF14"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOF14"),
 		STM32_FUNCTION(1, "TRACED6"),
-		STM32_FUNCTION(4, "DFSDM_CK6"),
+		STM32_FUNCTION(4, "DFSDM1_CKIN6"),
 		STM32_FUNCTION(5, "I2C4_SCL"),
 		STM32_FUNCTION(6, "I2C1_SCL"),
-		STM32_FUNCTION(12, "ETH_GMII_RXD6"),
+		STM32_FUNCTION(12, "ETH1_GMII_RXD6"),
 		STM32_FUNCTION(13, "FMC_A8"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(95, "PF15"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOF15"),
 		STM32_FUNCTION(1, "TRACED7"),
 		STM32_FUNCTION(5, "I2C4_SDA"),
 		STM32_FUNCTION(6, "I2C1_SDA"),
-		STM32_FUNCTION(12, "ETH_GMII_RXD7"),
+		STM32_FUNCTION(12, "ETH1_GMII_RXD7"),
 		STM32_FUNCTION(13, "FMC_A9"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(96, "PG0"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOG0"),
 		STM32_FUNCTION(1, "TRACED0"),
-		STM32_FUNCTION(4, "DFSDM_DATA0"),
-		STM32_FUNCTION(12, "ETH_GMII_TXD4"),
+		STM32_FUNCTION(4, "DFSDM1_DATIN0"),
+		STM32_FUNCTION(12, "ETH1_GMII_TXD4"),
 		STM32_FUNCTION(13, "FMC_A10"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(97, "PG1"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOG1"),
 		STM32_FUNCTION(1, "TRACED1"),
-		STM32_FUNCTION(12, "ETH_GMII_TXD5"),
+		STM32_FUNCTION(12, "ETH1_GMII_TXD5"),
 		STM32_FUNCTION(13, "FMC_A11"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(98, "PG2"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOG2"),
 		STM32_FUNCTION(1, "TRACED2"),
 		STM32_FUNCTION(2, "MCO2"),
 		STM32_FUNCTION(4, "TIM8_BKIN"),
-		STM32_FUNCTION(12, "ETH_GMII_TXD6"),
+		STM32_FUNCTION(12, "ETH1_GMII_TXD6"),
 		STM32_FUNCTION(13, "FMC_A12"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(99, "PG3"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOG3"),
 		STM32_FUNCTION(1, "TRACED3"),
 		STM32_FUNCTION(4, "TIM8_BKIN2"),
-		STM32_FUNCTION(5, "DFSDM_CK1"),
-		STM32_FUNCTION(12, "ETH_GMII_TXD7"),
+		STM32_FUNCTION(5, "DFSDM1_CKIN1"),
+		STM32_FUNCTION(12, "ETH1_GMII_TXD7"),
 		STM32_FUNCTION(13, "FMC_A13"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(100, "PG4"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOG4"),
 		STM32_FUNCTION(2, "TIM1_BKIN2"),
-		STM32_FUNCTION(12, "ETH_GMII_GTX_CLK ETH_RGMII_GTX_CLK"),
+		STM32_FUNCTION(12, "ETH1_GMII_GTX_CLK ETH1_RGMII_GTX_CLK"),
 		STM32_FUNCTION(13, "FMC_A14"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(101, "PG5"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOG5"),
 		STM32_FUNCTION(2, "TIM1_ETR"),
-		STM32_FUNCTION(12, "ETH_GMII_CLK125 ETH_RGMII_CLK125"),
+		STM32_FUNCTION(12, "ETH1_GMII_CLK125 ETH1_RGMII_CLK125"),
 		STM32_FUNCTION(13, "FMC_A15"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(102, "PG6"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOG6"),
 		STM32_FUNCTION(1, "TRACED14"),
 		STM32_FUNCTION(2, "TIM17_BKIN"),
-		STM32_FUNCTION(11, "SDMMC2_CMD SDMMC_BOOT2_CMD"),
+		STM32_FUNCTION(11, "SDMMC2_CMD"),
 		STM32_FUNCTION(14, "DCMI_D12"),
 		STM32_FUNCTION(15, "LCD_R7"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(103, "PG7"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOG7"),
 		STM32_FUNCTION(1, "TRACED5"),
 		STM32_FUNCTION(7, "SAI1_MCLK_A"),
-		STM32_FUNCTION(8, "USART6_CK USART_BOOT6_CK"),
-		STM32_FUNCTION(9, "UART8_RTS UART_BOOT8_RTS"),
+		STM32_FUNCTION(8, "USART6_CK"),
+		STM32_FUNCTION(9, "UART8_RTS UART8_DE"),
 		STM32_FUNCTION(10, "QUADSPI_CLK"),
-		STM32_FUNCTION(12, "QUADSPI_BK2_IO3 QUADSPI_BOOTBK2_IO3"),
+		STM32_FUNCTION(12, "QUADSPI_BK2_IO3"),
 		STM32_FUNCTION(13, "FMC_INT"),
 		STM32_FUNCTION(14, "DCMI_D13"),
 		STM32_FUNCTION(15, "LCD_CLK"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(104, "PG8"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOG8"),
 		STM32_FUNCTION(1, "TRACED15"),
 		STM32_FUNCTION(2, "TIM2_CH1 TIM2_ETR"),
@@ -1419,73 +1519,79 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(4, "TIM8_ETR"),
 		STM32_FUNCTION(6, "SPI6_NSS"),
 		STM32_FUNCTION(7, "SAI4_D2"),
-		STM32_FUNCTION(8, "USART6_RTS USART_BOOT6_RTS"),
-		STM32_FUNCTION(9, "USART3_RTS"),
-		STM32_FUNCTION(10, "SPDIF_IN2"),
+		STM32_FUNCTION(8, "USART6_RTS USART6_DE"),
+		STM32_FUNCTION(9, "USART3_RTS USART3_DE"),
+		STM32_FUNCTION(10, "SPDIFRX_IN2"),
 		STM32_FUNCTION(11, "SAI4_FS_A"),
-		STM32_FUNCTION(12, "ETH_PPS_OUT"),
+		STM32_FUNCTION(12, "ETH1_PPS_OUT"),
 		STM32_FUNCTION(15, "LCD_G7"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(105, "PG9"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOG9"),
 		STM32_FUNCTION(1, "DBTRGO"),
 		STM32_FUNCTION(8, "USART6_RX"),
-		STM32_FUNCTION(9, "SPDIF_IN3"),
-		STM32_FUNCTION(10, "QUADSPI_BK2_IO2 QUADSPI_BOOTBK2_IO2"),
+		STM32_FUNCTION(9, "SPDIFRX_IN3"),
+		STM32_FUNCTION(10, "QUADSPI_BK2_IO2"),
 		STM32_FUNCTION(11, "SAI2_FS_B"),
-		STM32_FUNCTION(13, "FMC_NE2 FMC_NCE"),
+		STM32_FUNCTION(13, "FMC_NCE FMC_NE2"),
 		STM32_FUNCTION(14, "DCMI_VSYNC"),
+		STM32_FUNCTION(15, "LCD_R1"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(106, "PG10"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOG10"),
 		STM32_FUNCTION(1, "TRACED10"),
-		STM32_FUNCTION(9, "UART8_CTS UART_BOOT8_CTS"),
+		STM32_FUNCTION(9, "UART8_CTS"),
 		STM32_FUNCTION(10, "LCD_G3"),
 		STM32_FUNCTION(11, "SAI2_SD_B"),
-		STM32_FUNCTION(12, "QUADSPI_BK2_IO2 QUADSPI_BOOTBK2_IO2"),
+		STM32_FUNCTION(12, "QUADSPI_BK2_IO2"),
 		STM32_FUNCTION(13, "FMC_NE3"),
 		STM32_FUNCTION(14, "DCMI_D2"),
 		STM32_FUNCTION(15, "LCD_B2"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(107, "PG11"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOG11"),
 		STM32_FUNCTION(1, "TRACED11"),
 		STM32_FUNCTION(5, "USART1_TX"),
-		STM32_FUNCTION(7, "UART4_TX UART_BOOT4_TX"),
-		STM32_FUNCTION(9, "SPDIF_IN0"),
-		STM32_FUNCTION(12, "ETH_GMII_TX_EN ETH_MII_TX_EN ETH_RGMII_TX_CTL ETH_RMII_TX_EN"),
+		STM32_FUNCTION(7, "UART4_TX"),
+		STM32_FUNCTION(9, "SPDIFRX_IN0"),
+		STM32_FUNCTION(12, "ETH1_GMII_TX_EN ETH1_MII_TX_EN ETH1_RGMII_TX_CTL ETH1_RMII_TX_EN"),
 		STM32_FUNCTION(14, "DCMI_D3"),
 		STM32_FUNCTION(15, "LCD_B3"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(108, "PG12"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOG12"),
 		STM32_FUNCTION(2, "LPTIM1_IN1"),
 		STM32_FUNCTION(6, "SPI6_MISO"),
 		STM32_FUNCTION(7, "SAI4_CK2"),
-		STM32_FUNCTION(8, "USART6_RTS USART_BOOT6_RTS"),
-		STM32_FUNCTION(9, "SPDIF_IN1"),
+		STM32_FUNCTION(8, "USART6_RTS USART6_DE"),
+		STM32_FUNCTION(9, "SPDIFRX_IN1"),
 		STM32_FUNCTION(10, "LCD_B4"),
 		STM32_FUNCTION(11, "SAI4_SCK_A"),
-		STM32_FUNCTION(12, "ETH_PHY_INTN"),
+		STM32_FUNCTION(12, "ETH1_PHY_INTN"),
 		STM32_FUNCTION(13, "FMC_NE4"),
 		STM32_FUNCTION(15, "LCD_B1"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(109, "PG13"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOG13"),
 		STM32_FUNCTION(1, "TRACED0"),
 		STM32_FUNCTION(2, "LPTIM1_OUT"),
@@ -1493,79 +1599,86 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(5, "SAI4_CK1"),
 		STM32_FUNCTION(6, "SPI6_SCK"),
 		STM32_FUNCTION(7, "SAI1_SCK_A"),
-		STM32_FUNCTION(8, "USART6_CTS_NSS USART_BOOT6_CTS_NSS"),
+		STM32_FUNCTION(8, "USART6_CTS USART6_NSS"),
 		STM32_FUNCTION(11, "SAI4_MCLK_A"),
-		STM32_FUNCTION(12, "ETH_GMII_TXD0 ETH_MII_TXD0 ETH_RGMII_TXD0 ETH_RMII_TXD0"),
+		STM32_FUNCTION(12, "ETH1_GMII_TXD0 ETH1_MII_TXD0 ETH1_RGMII_TXD0 ETH1_RMII_TXD0"),
 		STM32_FUNCTION(13, "FMC_A24"),
 		STM32_FUNCTION(15, "LCD_R0"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(110, "PG14"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOG14"),
 		STM32_FUNCTION(1, "TRACED1"),
 		STM32_FUNCTION(2, "LPTIM1_ETR"),
 		STM32_FUNCTION(6, "SPI6_MOSI"),
 		STM32_FUNCTION(7, "SAI4_D1"),
 		STM32_FUNCTION(8, "USART6_TX"),
-		STM32_FUNCTION(10, "QUADSPI_BK2_IO3 QUADSPI_BOOTBK2_IO3"),
+		STM32_FUNCTION(10, "QUADSPI_BK2_IO3"),
 		STM32_FUNCTION(11, "SAI4_SD_A"),
-		STM32_FUNCTION(12, "ETH_GMII_TXD1 ETH_MII_TXD1 ETH_RGMII_TXD1 ETH_RMII_TXD1"),
+		STM32_FUNCTION(12, "ETH1_GMII_TXD1 ETH1_MII_TXD1 ETH1_RGMII_TXD1 ETH1_RMII_TXD1"),
 		STM32_FUNCTION(13, "FMC_A25"),
 		STM32_FUNCTION(15, "LCD_B0"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(111, "PG15"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOG15"),
 		STM32_FUNCTION(1, "TRACED7"),
 		STM32_FUNCTION(3, "SAI1_D2"),
 		STM32_FUNCTION(5, "I2C2_SDA"),
 		STM32_FUNCTION(7, "SAI1_FS_A"),
-		STM32_FUNCTION(8, "USART6_CTS_NSS USART_BOOT6_CTS_NSS"),
+		STM32_FUNCTION(8, "USART6_CTS USART6_NSS"),
 		STM32_FUNCTION(11, "SDMMC3_CK"),
 		STM32_FUNCTION(14, "DCMI_D13"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(112, "PH0"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOH0"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(113, "PH1"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC | STM32MP_PKG_AB | STM32MP_PKG_AD,
 		STM32_FUNCTION(0, "GPIOH1"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(114, "PH2"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOH2"),
 		STM32_FUNCTION(2, "LPTIM1_IN2"),
-		STM32_FUNCTION(10, "QUADSPI_BK2_IO0 QUADSPI_BOOTBK2_IO0"),
+		STM32_FUNCTION(10, "QUADSPI_BK2_IO0"),
 		STM32_FUNCTION(11, "SAI2_SCK_B"),
-		STM32_FUNCTION(12, "ETH_GMII_CRS ETH_MII_CRS"),
+		STM32_FUNCTION(12, "ETH1_GMII_CRS ETH1_MII_CRS"),
 		STM32_FUNCTION(15, "LCD_R0"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(115, "PH3"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOH3"),
-		STM32_FUNCTION(4, "DFSDM_CK4"),
-		STM32_FUNCTION(10, "QUADSPI_BK2_IO1 QUADSPI_BOOTBK2_IO1"),
+		STM32_FUNCTION(4, "DFSDM1_CKIN4"),
+		STM32_FUNCTION(10, "QUADSPI_BK2_IO1"),
 		STM32_FUNCTION(11, "SAI2_MCLK_B"),
-		STM32_FUNCTION(12, "ETH_GMII_COL ETH_MII_COL"),
+		STM32_FUNCTION(12, "ETH1_GMII_COL ETH1_MII_COL"),
 		STM32_FUNCTION(15, "LCD_R1"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(116, "PH4"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOH4"),
 		STM32_FUNCTION(5, "I2C2_SCL"),
 		STM32_FUNCTION(10, "LCD_G5"),
@@ -1573,8 +1686,9 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(117, "PH5"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOH5"),
 		STM32_FUNCTION(5, "I2C2_SDA"),
 		STM32_FUNCTION(6, "SPI5_NSS"),
@@ -1582,31 +1696,34 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(118, "PH6"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOH6"),
 		STM32_FUNCTION(3, "TIM12_CH1"),
 		STM32_FUNCTION(5, "I2C2_SMBA"),
 		STM32_FUNCTION(6, "SPI5_SCK"),
-		STM32_FUNCTION(12, "ETH_GMII_RXD2 ETH_MII_RXD2 ETH_RGMII_RXD2"),
+		STM32_FUNCTION(12, "ETH1_GMII_RXD2 ETH1_MII_RXD2 ETH1_RGMII_RXD2"),
 		STM32_FUNCTION(13, "MDIOS_MDIO"),
 		STM32_FUNCTION(14, "DCMI_D8"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(119, "PH7"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOH7"),
 		STM32_FUNCTION(5, "I2C3_SCL"),
 		STM32_FUNCTION(6, "SPI5_MISO"),
-		STM32_FUNCTION(12, "ETH_GMII_RXD3 ETH_MII_RXD3 ETH_RGMII_RXD3"),
+		STM32_FUNCTION(12, "ETH1_GMII_RXD3 ETH1_MII_RXD3 ETH1_RGMII_RXD3"),
 		STM32_FUNCTION(13, "MDIOS_MDC"),
 		STM32_FUNCTION(14, "DCMI_D9"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(120, "PH8"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOH8"),
 		STM32_FUNCTION(3, "TIM5_ETR"),
 		STM32_FUNCTION(5, "I2C3_SDA"),
@@ -1615,8 +1732,9 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(121, "PH9"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOH9"),
 		STM32_FUNCTION(3, "TIM12_CH2"),
 		STM32_FUNCTION(5, "I2C3_SMBA"),
@@ -1625,8 +1743,9 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(122, "PH10"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOH10"),
 		STM32_FUNCTION(3, "TIM5_CH1"),
 		STM32_FUNCTION(5, "I2C4_SMBA"),
@@ -1636,8 +1755,9 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(123, "PH11"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOH11"),
 		STM32_FUNCTION(3, "TIM5_CH2"),
 		STM32_FUNCTION(5, "I2C4_SCL"),
@@ -1647,8 +1767,9 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(124, "PH12"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOH12"),
 		STM32_FUNCTION(1, "HDP2"),
 		STM32_FUNCTION(3, "TIM5_CH3"),
@@ -1659,50 +1780,53 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(125, "PH13"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOH13"),
 		STM32_FUNCTION(4, "TIM8_CH1N"),
 		STM32_FUNCTION(9, "UART4_TX"),
-		STM32_FUNCTION(10, "CAN1_TX"),
+		STM32_FUNCTION(10, "FDCAN1_TX"),
 		STM32_FUNCTION(15, "LCD_G2"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(126, "PH14"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOH14"),
 		STM32_FUNCTION(4, "TIM8_CH2N"),
 		STM32_FUNCTION(9, "UART4_RX"),
-		STM32_FUNCTION(10, "CAN1_RX"),
+		STM32_FUNCTION(10, "FDCAN1_RX"),
 		STM32_FUNCTION(14, "DCMI_D4"),
 		STM32_FUNCTION(15, "LCD_G3"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(127, "PH15"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOH15"),
 		STM32_FUNCTION(4, "TIM8_CH3N"),
-		STM32_FUNCTION(10, "CAN1_TXFD"),
 		STM32_FUNCTION(14, "DCMI_D11"),
 		STM32_FUNCTION(15, "LCD_G4"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(128, "PI0"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOI0"),
 		STM32_FUNCTION(3, "TIM5_CH4"),
 		STM32_FUNCTION(6, "SPI2_NSS I2S2_WS"),
-		STM32_FUNCTION(10, "CAN1_RXFD"),
 		STM32_FUNCTION(14, "DCMI_D13"),
 		STM32_FUNCTION(15, "LCD_G5"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(129, "PI1"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOI1"),
 		STM32_FUNCTION(4, "TIM8_BKIN2"),
 		STM32_FUNCTION(6, "SPI2_SCK I2S2_CK"),
@@ -1711,8 +1835,9 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(130, "PI2"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOI2"),
 		STM32_FUNCTION(4, "TIM8_CH4"),
 		STM32_FUNCTION(6, "SPI2_MISO I2S2_SDI"),
@@ -1721,8 +1846,9 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(131, "PI3"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOI3"),
 		STM32_FUNCTION(4, "TIM8_ETR"),
 		STM32_FUNCTION(6, "SPI2_MOSI I2S2_SDO"),
@@ -1730,8 +1856,9 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(132, "PI4"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOI4"),
 		STM32_FUNCTION(4, "TIM8_BKIN"),
 		STM32_FUNCTION(11, "SAI2_MCLK_A"),
@@ -1740,8 +1867,9 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(133, "PI5"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOI5"),
 		STM32_FUNCTION(4, "TIM8_CH1"),
 		STM32_FUNCTION(11, "SAI2_SCK_A"),
@@ -1750,8 +1878,9 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(134, "PI6"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOI6"),
 		STM32_FUNCTION(4, "TIM8_CH2"),
 		STM32_FUNCTION(11, "SAI2_SD_A"),
@@ -1760,8 +1889,9 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(135, "PI7"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOI7"),
 		STM32_FUNCTION(4, "TIM8_CH3"),
 		STM32_FUNCTION(11, "SAI2_FS_A"),
@@ -1770,35 +1900,38 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(136, "PI8"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOI8"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(137, "PI9"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOI9"),
 		STM32_FUNCTION(1, "HDP1"),
 		STM32_FUNCTION(9, "UART4_RX"),
-		STM32_FUNCTION(10, "CAN1_RX"),
+		STM32_FUNCTION(10, "FDCAN1_RX"),
 		STM32_FUNCTION(15, "LCD_VSYNC"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(138, "PI10"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOI10"),
 		STM32_FUNCTION(1, "HDP0"),
-		STM32_FUNCTION(9, "USART3_CTS_NSS USART_BOOT3_CTS_NSS"),
-		STM32_FUNCTION(10, "CAN1_RXFD"),
-		STM32_FUNCTION(12, "ETH_GMII_RX_ER ETH_MII_RX_ER"),
+		STM32_FUNCTION(9, "USART3_CTS USART3_NSS"),
+		STM32_FUNCTION(12, "ETH1_GMII_RX_ER ETH1_MII_RX_ER"),
 		STM32_FUNCTION(15, "LCD_HSYNC"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(139, "PI11"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOI11"),
 		STM32_FUNCTION(1, "MCO1"),
 		STM32_FUNCTION(6, "I2S_CKIN"),
@@ -1806,8 +1939,9 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(140, "PI12"),
+		STM32MP_PKG_AA,
 		STM32_FUNCTION(0, "GPIOI12"),
 		STM32_FUNCTION(1, "TRACED0"),
 		STM32_FUNCTION(3, "HDP0"),
@@ -1815,8 +1949,9 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(141, "PI13"),
+		STM32MP_PKG_AA,
 		STM32_FUNCTION(0, "GPIOI13"),
 		STM32_FUNCTION(1, "TRACED1"),
 		STM32_FUNCTION(3, "HDP1"),
@@ -1824,24 +1959,27 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(142, "PI14"),
+		STM32MP_PKG_AA,
 		STM32_FUNCTION(0, "GPIOI14"),
 		STM32_FUNCTION(1, "TRACECLK"),
 		STM32_FUNCTION(15, "LCD_CLK"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(143, "PI15"),
+		STM32MP_PKG_AA,
 		STM32_FUNCTION(0, "GPIOI15"),
 		STM32_FUNCTION(10, "LCD_G2"),
 		STM32_FUNCTION(15, "LCD_R0"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(144, "PJ0"),
+		STM32MP_PKG_AA,
 		STM32_FUNCTION(0, "GPIOJ0"),
 		STM32_FUNCTION(1, "TRACED8"),
 		STM32_FUNCTION(10, "LCD_R7"),
@@ -1849,16 +1987,18 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(145, "PJ1"),
+		STM32MP_PKG_AA,
 		STM32_FUNCTION(0, "GPIOJ1"),
 		STM32_FUNCTION(1, "TRACED9"),
 		STM32_FUNCTION(15, "LCD_R2"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(146, "PJ2"),
+		STM32MP_PKG_AA,
 		STM32_FUNCTION(0, "GPIOJ2"),
 		STM32_FUNCTION(1, "TRACED10"),
 		STM32_FUNCTION(14, "DSI_TE"),
@@ -1866,24 +2006,27 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(147, "PJ3"),
+		STM32MP_PKG_AA,
 		STM32_FUNCTION(0, "GPIOJ3"),
 		STM32_FUNCTION(1, "TRACED11"),
 		STM32_FUNCTION(15, "LCD_R4"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(148, "PJ4"),
+		STM32MP_PKG_AA,
 		STM32_FUNCTION(0, "GPIOJ4"),
 		STM32_FUNCTION(1, "TRACED12"),
 		STM32_FUNCTION(15, "LCD_R5"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(149, "PJ5"),
+		STM32MP_PKG_AA,
 		STM32_FUNCTION(0, "GPIOJ5"),
 		STM32_FUNCTION(1, "TRACED2"),
 		STM32_FUNCTION(3, "HDP2"),
@@ -1891,8 +2034,9 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(150, "PJ6"),
+		STM32MP_PKG_AA,
 		STM32_FUNCTION(0, "GPIOJ6"),
 		STM32_FUNCTION(1, "TRACED3"),
 		STM32_FUNCTION(3, "HDP3"),
@@ -1901,8 +2045,9 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(151, "PJ7"),
+		STM32MP_PKG_AA,
 		STM32_FUNCTION(0, "GPIOJ7"),
 		STM32_FUNCTION(1, "TRACED13"),
 		STM32_FUNCTION(4, "TIM8_CH2N"),
@@ -1910,8 +2055,9 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(152, "PJ8"),
+		STM32MP_PKG_AA,
 		STM32_FUNCTION(0, "GPIOJ8"),
 		STM32_FUNCTION(1, "TRACED14"),
 		STM32_FUNCTION(2, "TIM1_CH3N"),
@@ -1921,8 +2067,9 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(153, "PJ9"),
+		STM32MP_PKG_AA,
 		STM32_FUNCTION(0, "GPIOJ9"),
 		STM32_FUNCTION(1, "TRACED15"),
 		STM32_FUNCTION(2, "TIM1_CH3"),
@@ -1932,8 +2079,9 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(154, "PJ10"),
+		STM32MP_PKG_AA,
 		STM32_FUNCTION(0, "GPIOJ10"),
 		STM32_FUNCTION(2, "TIM1_CH2N"),
 		STM32_FUNCTION(4, "TIM8_CH2"),
@@ -1942,8 +2090,9 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(155, "PJ11"),
+		STM32MP_PKG_AA,
 		STM32_FUNCTION(0, "GPIOJ11"),
 		STM32_FUNCTION(2, "TIM1_CH2"),
 		STM32_FUNCTION(4, "TIM8_CH2N"),
@@ -1952,38 +2101,43 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(156, "PJ12"),
+		STM32MP_PKG_AA,
 		STM32_FUNCTION(0, "GPIOJ12"),
 		STM32_FUNCTION(10, "LCD_G3"),
 		STM32_FUNCTION(15, "LCD_B0"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(157, "PJ13"),
+		STM32MP_PKG_AA,
 		STM32_FUNCTION(0, "GPIOJ13"),
 		STM32_FUNCTION(10, "LCD_G4"),
 		STM32_FUNCTION(15, "LCD_B1"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(158, "PJ14"),
+		STM32MP_PKG_AA,
 		STM32_FUNCTION(0, "GPIOJ14"),
 		STM32_FUNCTION(15, "LCD_B2"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(159, "PJ15"),
+		STM32MP_PKG_AA,
 		STM32_FUNCTION(0, "GPIOJ15"),
 		STM32_FUNCTION(15, "LCD_B3"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(160, "PK0"),
+		STM32MP_PKG_AA,
 		STM32_FUNCTION(0, "GPIOK0"),
 		STM32_FUNCTION(2, "TIM1_CH1N"),
 		STM32_FUNCTION(4, "TIM8_CH3"),
@@ -1992,8 +2146,9 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(161, "PK1"),
+		STM32MP_PKG_AA,
 		STM32_FUNCTION(0, "GPIOK1"),
 		STM32_FUNCTION(1, "TRACED4"),
 		STM32_FUNCTION(2, "TIM1_CH1"),
@@ -2004,8 +2159,9 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(162, "PK2"),
+		STM32MP_PKG_AA,
 		STM32_FUNCTION(0, "GPIOK2"),
 		STM32_FUNCTION(1, "TRACED5"),
 		STM32_FUNCTION(2, "TIM1_BKIN"),
@@ -2015,22 +2171,25 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(163, "PK3"),
+		STM32MP_PKG_AA,
 		STM32_FUNCTION(0, "GPIOK3"),
 		STM32_FUNCTION(15, "LCD_B4"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(164, "PK4"),
+		STM32MP_PKG_AA,
 		STM32_FUNCTION(0, "GPIOK4"),
 		STM32_FUNCTION(15, "LCD_B5"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(165, "PK5"),
+		STM32MP_PKG_AA,
 		STM32_FUNCTION(0, "GPIOK5"),
 		STM32_FUNCTION(1, "TRACED6"),
 		STM32_FUNCTION(3, "HDP6"),
@@ -2038,8 +2197,9 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(166, "PK6"),
+		STM32MP_PKG_AA,
 		STM32_FUNCTION(0, "GPIOK6"),
 		STM32_FUNCTION(1, "TRACED7"),
 		STM32_FUNCTION(3, "HDP7"),
@@ -2047,8 +2207,9 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(167, "PK7"),
+		STM32MP_PKG_AA,
 		STM32_FUNCTION(0, "GPIOK7"),
 		STM32_FUNCTION(15, "LCD_DE"),
 		STM32_FUNCTION(16, "EVENTOUT"),
@@ -2057,8 +2218,9 @@ static const struct stm32_desc_pin stm32mp157_pins[] = {
 };
 
 static const struct stm32_desc_pin stm32mp157_z_pins[] = {
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(400, "PZ0"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOZ0"),
 		STM32_FUNCTION(3, "I2C6_SCL"),
 		STM32_FUNCTION(4, "I2C2_SCL"),
@@ -2068,8 +2230,9 @@ static const struct stm32_desc_pin stm32mp157_z_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(401, "PZ1"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOZ1"),
 		STM32_FUNCTION(3, "I2C6_SDA"),
 		STM32_FUNCTION(4, "I2C2_SDA"),
@@ -2081,8 +2244,9 @@ static const struct stm32_desc_pin stm32mp157_z_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(402, "PZ2"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOZ2"),
 		STM32_FUNCTION(3, "I2C6_SCL"),
 		STM32_FUNCTION(4, "I2C2_SCL"),
@@ -2094,21 +2258,23 @@ static const struct stm32_desc_pin stm32mp157_z_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(403, "PZ3"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOZ3"),
 		STM32_FUNCTION(3, "I2C6_SDA"),
 		STM32_FUNCTION(4, "I2C2_SDA"),
 		STM32_FUNCTION(5, "I2C5_SDA"),
 		STM32_FUNCTION(6, "SPI1_NSS I2S1_WS"),
 		STM32_FUNCTION(7, "I2C4_SDA"),
-		STM32_FUNCTION(8, "USART1_CTS_NSS"),
+		STM32_FUNCTION(8, "USART1_CTS USART1_NSS"),
 		STM32_FUNCTION(9, "SPI6_NSS"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(404, "PZ4"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOZ4"),
 		STM32_FUNCTION(3, "I2C6_SCL"),
 		STM32_FUNCTION(4, "I2C2_SCL"),
@@ -2117,19 +2283,21 @@ static const struct stm32_desc_pin stm32mp157_z_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(405, "PZ5"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOZ5"),
 		STM32_FUNCTION(3, "I2C6_SDA"),
 		STM32_FUNCTION(4, "I2C2_SDA"),
 		STM32_FUNCTION(5, "I2C5_SDA"),
 		STM32_FUNCTION(7, "I2C4_SDA"),
-		STM32_FUNCTION(8, "USART1_RTS"),
+		STM32_FUNCTION(8, "USART1_RTS USART1_DE"),
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(406, "PZ6"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOZ6"),
 		STM32_FUNCTION(3, "I2C6_SCL"),
 		STM32_FUNCTION(4, "I2C2_SCL"),
@@ -2140,8 +2308,9 @@ static const struct stm32_desc_pin stm32mp157_z_pins[] = {
 		STM32_FUNCTION(16, "EVENTOUT"),
 		STM32_FUNCTION(17, "ANALOG")
 	),
-	STM32_PIN(
+	STM32_PIN_PKG(
 		PINCTRL_PIN(407, "PZ7"),
+		STM32MP_PKG_AA | STM32MP_PKG_AC,
 		STM32_FUNCTION(0, "GPIOZ7"),
 		STM32_FUNCTION(3, "I2C6_SDA"),
 		STM32_FUNCTION(4, "I2C2_SDA"),
diff --git a/drivers/pinctrl/sunxi/Kconfig b/drivers/pinctrl/sunxi/Kconfig
index a731fc966b63..9093a420d310 100644
--- a/drivers/pinctrl/sunxi/Kconfig
+++ b/drivers/pinctrl/sunxi/Kconfig
@@ -11,82 +11,101 @@ config PINCTRL_SUNIV_F1C100S
 	select PINCTRL_SUNXI
 
 config PINCTRL_SUN4I_A10
-	def_bool MACH_SUN4I || MACH_SUN7I || MACH_SUN8I
+	bool "Support for the Allwinner A10, A20 and R40 PIO"
+	default MACH_SUN4I || MACH_SUN7I || MACH_SUN8I
 	select PINCTRL_SUNXI
 
 config PINCTRL_SUN5I
-	def_bool MACH_SUN5I
+	bool "Support for the Allwinner A10s, A13, R8 and NextThing GR8 PIO"
+	default MACH_SUN5I
 	select PINCTRL_SUNXI
 
 config PINCTRL_SUN6I_A31
-	def_bool MACH_SUN6I
+	bool "Support for the Allwinner A31 PIO"
+	default MACH_SUN6I
 	select PINCTRL_SUNXI
 
 config PINCTRL_SUN6I_A31_R
-	def_bool MACH_SUN6I
+	bool "Support for the Allwinner A31 R-PIO"
+	default MACH_SUN6I
 	depends on RESET_CONTROLLER
 	select PINCTRL_SUNXI
 
 config PINCTRL_SUN8I_A23
-	def_bool MACH_SUN8I
+	bool "Support for the Allwinner A23 PIO"
+	default MACH_SUN8I
 	select PINCTRL_SUNXI
 
 config PINCTRL_SUN8I_A33
-	def_bool MACH_SUN8I
+	bool "Support for the Allwinner A33 PIO"
+	default MACH_SUN8I
 	select PINCTRL_SUNXI
 
 config PINCTRL_SUN8I_A83T
-	def_bool MACH_SUN8I
+	bool "Support for the Allwinner A83T PIO"
+	default MACH_SUN8I
 	select PINCTRL_SUNXI
 
 config PINCTRL_SUN8I_A83T_R
-	def_bool MACH_SUN8I
+	bool "Support for the Allwinner A83T R-PIO"
+	default MACH_SUN8I
 	select PINCTRL_SUNXI
 
 config PINCTRL_SUN8I_A23_R
-	def_bool MACH_SUN8I
+	bool "Support for the Allwinner A23 and A33 R-PIO"
+	default MACH_SUN8I
 	depends on RESET_CONTROLLER
 	select PINCTRL_SUNXI
 
 config PINCTRL_SUN8I_H3
-	def_bool MACH_SUN8I
+	bool "Support for the Allwinner H3 PIO"
+	default MACH_SUN8I
 	select PINCTRL_SUNXI
 
 config PINCTRL_SUN8I_H3_R
-	def_bool MACH_SUN8I || (ARM64 && ARCH_SUNXI)
+	bool "Support for the Allwinner H3 and H5 R-PIO"
+	default MACH_SUN8I || (ARM64 && ARCH_SUNXI)
 	select PINCTRL_SUNXI
 
 config PINCTRL_SUN8I_V3S
-	def_bool MACH_SUN8I
+	bool "Support for the Allwinner V3s PIO"
+	default MACH_SUN8I
 	select PINCTRL_SUNXI
 
 config PINCTRL_SUN9I_A80
-	def_bool MACH_SUN9I
+	bool "Support for the Allwinner A80 PIO"
+	default MACH_SUN9I
 	select PINCTRL_SUNXI
 
 config PINCTRL_SUN9I_A80_R
-	def_bool MACH_SUN9I
+	bool "Support for the Allwinner A80 R-PIO"
+	default MACH_SUN9I
 	depends on RESET_CONTROLLER
 	select PINCTRL_SUNXI
 
 config PINCTRL_SUN50I_A64
-	def_bool ARM64 && ARCH_SUNXI
+	bool "Support for the Allwinner A64 PIO"
+	default ARM64 && ARCH_SUNXI
 	select PINCTRL_SUNXI
 
 config PINCTRL_SUN50I_A64_R
-	def_bool ARM64 && ARCH_SUNXI
+	bool "Support for the Allwinner A64 R-PIO"
+	default ARM64 && ARCH_SUNXI
 	select PINCTRL_SUNXI
 
 config PINCTRL_SUN50I_H5
-	def_bool ARM64 && ARCH_SUNXI
+	bool "Support for the Allwinner H5 PIO"
+	default ARM64 && ARCH_SUNXI
 	select PINCTRL_SUNXI
 
 config PINCTRL_SUN50I_H6
-	def_bool ARM64 && ARCH_SUNXI
+	bool "Support for the Allwinner H6 PIO"
+	default ARM64 && ARCH_SUNXI
 	select PINCTRL_SUNXI
 
 config PINCTRL_SUN50I_H6_R
-	def_bool ARM64 && ARCH_SUNXI
+	bool "Support for the Allwinner H6 R-PIO"
+	default ARM64 && ARCH_SUNXI
 	select PINCTRL_SUNXI
 
 endif
diff --git a/drivers/pinctrl/sunxi/pinctrl-sun50i-h6.c b/drivers/pinctrl/sunxi/pinctrl-sun50i-h6.c
index ef4268cc6227..3cc1121589c9 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sun50i-h6.c
+++ b/drivers/pinctrl/sunxi/pinctrl-sun50i-h6.c
@@ -591,6 +591,7 @@ static const struct sunxi_pinctrl_desc h6_pinctrl_data = {
 	.irq_banks = 4,
 	.irq_bank_map = h6_irq_bank_map,
 	.irq_read_needs_mux = true,
+	.io_bias_cfg_variant = BIAS_VOLTAGE_PIO_POW_MODE_SEL,
 };
 
 static int h6_pinctrl_probe(struct platform_device *pdev)
diff --git a/drivers/pinctrl/sunxi/pinctrl-sun9i-a80-r.c b/drivers/pinctrl/sunxi/pinctrl-sun9i-a80-r.c
index e05dd9a5551d..a191a65217ac 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sun9i-a80-r.c
+++ b/drivers/pinctrl/sunxi/pinctrl-sun9i-a80-r.c
@@ -153,7 +153,7 @@ static const struct sunxi_pinctrl_desc sun9i_a80_r_pinctrl_data = {
 	.pin_base = PL_BASE,
 	.irq_banks = 2,
 	.disable_strict_mode = true,
-	.has_io_bias_cfg = true,
+	.io_bias_cfg_variant = BIAS_VOLTAGE_GRP_CONFIG,
 };
 
 static int sun9i_a80_r_pinctrl_probe(struct platform_device *pdev)
diff --git a/drivers/pinctrl/sunxi/pinctrl-sun9i-a80.c b/drivers/pinctrl/sunxi/pinctrl-sun9i-a80.c
index da37d594a13d..0633a03d5e13 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sun9i-a80.c
+++ b/drivers/pinctrl/sunxi/pinctrl-sun9i-a80.c
@@ -722,7 +722,7 @@ static const struct sunxi_pinctrl_desc sun9i_a80_pinctrl_data = {
 	.npins = ARRAY_SIZE(sun9i_a80_pins),
 	.irq_banks = 5,
 	.disable_strict_mode = true,
-	.has_io_bias_cfg = true,
+	.io_bias_cfg_variant = BIAS_VOLTAGE_GRP_CONFIG,
 };
 
 static int sun9i_a80_pinctrl_probe(struct platform_device *pdev)
diff --git a/drivers/pinctrl/sunxi/pinctrl-sunxi.c b/drivers/pinctrl/sunxi/pinctrl-sunxi.c
index 8dd25caea2cf..0cbca30b75dc 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sunxi.c
+++ b/drivers/pinctrl/sunxi/pinctrl-sunxi.c
@@ -530,14 +530,10 @@ static int sunxi_pconf_group_get(struct pinctrl_dev *pctldev,
 	return sunxi_pconf_get(pctldev, g->pin, config);
 }
 
-static int sunxi_pconf_group_set(struct pinctrl_dev *pctldev,
-				 unsigned group,
-				 unsigned long *configs,
-				 unsigned num_configs)
+static int sunxi_pconf_set(struct pinctrl_dev *pctldev, unsigned pin,
+			   unsigned long *configs, unsigned num_configs)
 {
 	struct sunxi_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev);
-	struct sunxi_pinctrl_group *g = &pctl->groups[group];
-	unsigned pin = g->pin - pctl->desc->pin_base;
 	int i;
 
 	for (i = 0; i < num_configs; i++) {
@@ -596,9 +592,20 @@ static int sunxi_pconf_group_set(struct pinctrl_dev *pctldev,
 	return 0;
 }
 
+static int sunxi_pconf_group_set(struct pinctrl_dev *pctldev, unsigned group,
+				 unsigned long *configs, unsigned num_configs)
+{
+	struct sunxi_pinctrl *pctl = pinctrl_dev_get_drvdata(pctldev);
+	struct sunxi_pinctrl_group *g = &pctl->groups[group];
+
+	/* We only support 1 pin per group. Chain it to the pin callback */
+	return sunxi_pconf_set(pctldev, g->pin, configs, num_configs);
+}
+
 static const struct pinconf_ops sunxi_pconf_ops = {
 	.is_generic		= true,
 	.pin_config_get		= sunxi_pconf_get,
+	.pin_config_set		= sunxi_pconf_set,
 	.pin_config_group_get	= sunxi_pconf_group_get,
 	.pin_config_group_set	= sunxi_pconf_group_set,
 };
@@ -607,10 +614,12 @@ static int sunxi_pinctrl_set_io_bias_cfg(struct sunxi_pinctrl *pctl,
 					 unsigned pin,
 					 struct regulator *supply)
 {
+	unsigned short bank = pin / PINS_PER_BANK;
+	unsigned long flags;
 	u32 val, reg;
 	int uV;
 
-	if (!pctl->desc->has_io_bias_cfg)
+	if (!pctl->desc->io_bias_cfg_variant)
 		return 0;
 
 	uV = regulator_get_voltage(supply);
@@ -621,25 +630,41 @@ static int sunxi_pinctrl_set_io_bias_cfg(struct sunxi_pinctrl *pctl,
 	if (uV == 0)
 		return 0;
 
-	/* Configured value must be equal or greater to actual voltage */
-	if (uV <= 1800000)
-		val = 0x0; /* 1.8V */
-	else if (uV <= 2500000)
-		val = 0x6; /* 2.5V */
-	else if (uV <= 2800000)
-		val = 0x9; /* 2.8V */
-	else if (uV <= 3000000)
-		val = 0xA; /* 3.0V */
-	else
-		val = 0xD; /* 3.3V */
-
-	pin -= pctl->desc->pin_base;
-
-	reg = readl(pctl->membase + sunxi_grp_config_reg(pin));
-	reg &= ~IO_BIAS_MASK;
-	writel(reg | val, pctl->membase + sunxi_grp_config_reg(pin));
+	switch (pctl->desc->io_bias_cfg_variant) {
+	case BIAS_VOLTAGE_GRP_CONFIG:
+		/*
+		 * Configured value must be equal or greater to actual
+		 * voltage.
+		 */
+		if (uV <= 1800000)
+			val = 0x0; /* 1.8V */
+		else if (uV <= 2500000)
+			val = 0x6; /* 2.5V */
+		else if (uV <= 2800000)
+			val = 0x9; /* 2.8V */
+		else if (uV <= 3000000)
+			val = 0xA; /* 3.0V */
+		else
+			val = 0xD; /* 3.3V */
+
+		pin -= pctl->desc->pin_base;
+
+		reg = readl(pctl->membase + sunxi_grp_config_reg(pin));
+		reg &= ~IO_BIAS_MASK;
+		writel(reg | val, pctl->membase + sunxi_grp_config_reg(pin));
+		return 0;
+	case BIAS_VOLTAGE_PIO_POW_MODE_SEL:
+		val = uV <= 1800000 ? 1 : 0;
 
-	return 0;
+		raw_spin_lock_irqsave(&pctl->lock, flags);
+		reg = readl(pctl->membase + PIO_POW_MOD_SEL_REG);
+		reg &= ~(1 << bank);
+		writel(reg | val << bank, pctl->membase + PIO_POW_MOD_SEL_REG);
+		raw_spin_unlock_irqrestore(&pctl->lock, flags);
+		return 0;
+	default:
+		return -EINVAL;
+	}
 }
 
 static int sunxi_pmx_get_funcs_cnt(struct pinctrl_dev *pctldev)
@@ -1443,16 +1468,17 @@ int sunxi_pinctrl_init_with_variant(struct platform_device *pdev,
 
 	last_pin = pctl->desc->pins[pctl->desc->npins - 1].pin.number;
 	pctl->chip->owner = THIS_MODULE;
-	pctl->chip->request = gpiochip_generic_request,
-	pctl->chip->free = gpiochip_generic_free,
-	pctl->chip->direction_input = sunxi_pinctrl_gpio_direction_input,
-	pctl->chip->direction_output = sunxi_pinctrl_gpio_direction_output,
-	pctl->chip->get = sunxi_pinctrl_gpio_get,
-	pctl->chip->set = sunxi_pinctrl_gpio_set,
-	pctl->chip->of_xlate = sunxi_pinctrl_gpio_of_xlate,
-	pctl->chip->to_irq = sunxi_pinctrl_gpio_to_irq,
-	pctl->chip->of_gpio_n_cells = 3,
-	pctl->chip->can_sleep = false,
+	pctl->chip->request = gpiochip_generic_request;
+	pctl->chip->free = gpiochip_generic_free;
+	pctl->chip->set_config = gpiochip_generic_config;
+	pctl->chip->direction_input = sunxi_pinctrl_gpio_direction_input;
+	pctl->chip->direction_output = sunxi_pinctrl_gpio_direction_output;
+	pctl->chip->get = sunxi_pinctrl_gpio_get;
+	pctl->chip->set = sunxi_pinctrl_gpio_set;
+	pctl->chip->of_xlate = sunxi_pinctrl_gpio_of_xlate;
+	pctl->chip->to_irq = sunxi_pinctrl_gpio_to_irq;
+	pctl->chip->of_gpio_n_cells = 3;
+	pctl->chip->can_sleep = false;
 	pctl->chip->ngpio = round_up(last_pin, PINS_PER_BANK) -
 			    pctl->desc->pin_base;
 	pctl->chip->label = dev_name(&pdev->dev);
diff --git a/drivers/pinctrl/sunxi/pinctrl-sunxi.h b/drivers/pinctrl/sunxi/pinctrl-sunxi.h
index ee15ab067b5f..44e30deeee38 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sunxi.h
+++ b/drivers/pinctrl/sunxi/pinctrl-sunxi.h
@@ -95,6 +95,22 @@
 #define PINCTRL_SUN7I_A20	BIT(7)
 #define PINCTRL_SUN8I_R40	BIT(8)
 
+#define PIO_POW_MOD_SEL_REG	0x340
+
+enum sunxi_desc_bias_voltage {
+	BIAS_VOLTAGE_NONE,
+	/*
+	 * Bias voltage configuration is done through
+	 * Pn_GRP_CONFIG registers, as seen on A80 SoC.
+	 */
+	BIAS_VOLTAGE_GRP_CONFIG,
+	/*
+	 * Bias voltage is set through PIO_POW_MOD_SEL_REG
+	 * register, as seen on H6 SoC, for example.
+	 */
+	BIAS_VOLTAGE_PIO_POW_MODE_SEL,
+};
+
 struct sunxi_desc_function {
 	unsigned long	variant;
 	const char	*name;
@@ -117,7 +133,7 @@ struct sunxi_pinctrl_desc {
 	const unsigned int		*irq_bank_map;
 	bool				irq_read_needs_mux;
 	bool				disable_strict_mode;
-	bool				has_io_bias_cfg;
+	enum sunxi_desc_bias_voltage	io_bias_cfg_variant;
 };
 
 struct sunxi_pinctrl_function {
diff --git a/drivers/pinctrl/zte/pinctrl-zx.c b/drivers/pinctrl/zte/pinctrl-zx.c
index caa44dd2880a..3cb69309912b 100644
--- a/drivers/pinctrl/zte/pinctrl-zx.c
+++ b/drivers/pinctrl/zte/pinctrl-zx.c
@@ -411,6 +411,7 @@ int zx_pinctrl_init(struct platform_device *pdev,
 	}
 
 	zpctl->aux_base = of_iomap(np, 0);
+	of_node_put(np);
 	if (!zpctl->aux_base)
 		return -ENOMEM;
 
diff --git a/drivers/ptp/ptp_qoriq.c b/drivers/ptp/ptp_qoriq.c
index 53775362aac6..e10642403b25 100644
--- a/drivers/ptp/ptp_qoriq.c
+++ b/drivers/ptp/ptp_qoriq.c
@@ -467,6 +467,9 @@ int ptp_qoriq_init(struct ptp_qoriq *ptp_qoriq, void __iomem *base,
 	unsigned long flags;
 	u32 tmr_ctrl;
 
+	if (!node)
+		return -ENODEV;
+
 	ptp_qoriq->base = base;
 	ptp_qoriq->caps = *caps;
 
diff --git a/drivers/pwm/pwm-meson.c b/drivers/pwm/pwm-meson.c
index c1ed641b3e26..4ae5d774443e 100644
--- a/drivers/pwm/pwm-meson.c
+++ b/drivers/pwm/pwm-meson.c
@@ -470,7 +470,7 @@ static int meson_pwm_init_channels(struct meson_pwm *meson,
 
 		init.name = name;
 		init.ops = &clk_mux_ops;
-		init.flags = CLK_IS_BASIC;
+		init.flags = 0;
 		init.parent_names = meson->data->parent_names;
 		init.num_parents = meson->data->num_parents;
 
diff --git a/drivers/reset/core.c b/drivers/reset/core.c
index 9582efb70025..81ea77cba123 100644
--- a/drivers/reset/core.c
+++ b/drivers/reset/core.c
@@ -34,6 +34,7 @@ static LIST_HEAD(reset_lookup_list);
  * @id: ID of the reset controller in the reset
  *      controller device
  * @refcnt: Number of gets of this reset_control
+ * @acquired: Only one reset_control may be acquired for a given rcdev and id.
  * @shared: Is this a shared (1), or an exclusive (0) reset_control?
  * @deassert_cnt: Number of times this reset line has been deasserted
  * @triggered_count: Number of times this reset line has been reset. Currently
@@ -45,6 +46,7 @@ struct reset_control {
 	struct list_head list;
 	unsigned int id;
 	struct kref refcnt;
+	bool acquired;
 	bool shared;
 	bool array;
 	atomic_t deassert_count;
@@ -63,6 +65,17 @@ struct reset_control_array {
 	struct reset_control *rstc[];
 };
 
+static const char *rcdev_name(struct reset_controller_dev *rcdev)
+{
+	if (rcdev->dev)
+		return dev_name(rcdev->dev);
+
+	if (rcdev->of_node)
+		return rcdev->of_node->full_name;
+
+	return NULL;
+}
+
 /**
  * of_reset_simple_xlate - translate reset_spec to the reset line number
  * @rcdev: a pointer to the reset controller device
@@ -232,6 +245,34 @@ err:
 	return ret;
 }
 
+static int reset_control_array_acquire(struct reset_control_array *resets)
+{
+	unsigned int i;
+	int err;
+
+	for (i = 0; i < resets->num_rstcs; i++) {
+		err = reset_control_acquire(resets->rstc[i]);
+		if (err < 0)
+			goto release;
+	}
+
+	return 0;
+
+release:
+	while (i--)
+		reset_control_release(resets->rstc[i]);
+
+	return err;
+}
+
+static void reset_control_array_release(struct reset_control_array *resets)
+{
+	unsigned int i;
+
+	for (i = 0; i < resets->num_rstcs; i++)
+		reset_control_release(resets->rstc[i]);
+}
+
 static inline bool reset_control_is_array(struct reset_control *rstc)
 {
 	return rstc->array;
@@ -272,6 +313,9 @@ int reset_control_reset(struct reset_control *rstc)
 
 		if (atomic_inc_return(&rstc->triggered_count) != 1)
 			return 0;
+	} else {
+		if (!rstc->acquired)
+			return -EPERM;
 	}
 
 	ret = rstc->rcdev->ops->reset(rstc->rcdev, rstc->id);
@@ -334,6 +378,12 @@ int reset_control_assert(struct reset_control *rstc)
 		 */
 		if (!rstc->rcdev->ops->assert)
 			return -ENOTSUPP;
+
+		if (!rstc->acquired) {
+			WARN(1, "reset %s (ID: %u) is not acquired\n",
+			     rcdev_name(rstc->rcdev), rstc->id);
+			return -EPERM;
+		}
 	}
 
 	return rstc->rcdev->ops->assert(rstc->rcdev, rstc->id);
@@ -369,6 +419,12 @@ int reset_control_deassert(struct reset_control *rstc)
 
 		if (atomic_inc_return(&rstc->deassert_count) != 1)
 			return 0;
+	} else {
+		if (!rstc->acquired) {
+			WARN(1, "reset %s (ID: %u) is not acquired\n",
+			     rcdev_name(rstc->rcdev), rstc->id);
+			return -EPERM;
+		}
 	}
 
 	/*
@@ -406,9 +462,87 @@ int reset_control_status(struct reset_control *rstc)
 }
 EXPORT_SYMBOL_GPL(reset_control_status);
 
+/**
+ * reset_control_acquire() - acquires a reset control for exclusive use
+ * @rstc: reset control
+ *
+ * This is used to explicitly acquire a reset control for exclusive use. Note
+ * that exclusive resets are requested as acquired by default. In order for a
+ * second consumer to be able to control the reset, the first consumer has to
+ * release it first. Typically the easiest way to achieve this is to call the
+ * reset_control_get_exclusive_released() to obtain an instance of the reset
+ * control. Such reset controls are not acquired by default.
+ *
+ * Consumers implementing shared access to an exclusive reset need to follow
+ * a specific protocol in order to work together. Before consumers can change
+ * a reset they must acquire exclusive access using reset_control_acquire().
+ * After they are done operating the reset, they must release exclusive access
+ * with a call to reset_control_release(). Consumers are not granted exclusive
+ * access to the reset as long as another consumer hasn't released a reset.
+ *
+ * See also: reset_control_release()
+ */
+int reset_control_acquire(struct reset_control *rstc)
+{
+	struct reset_control *rc;
+
+	if (!rstc)
+		return 0;
+
+	if (WARN_ON(IS_ERR(rstc)))
+		return -EINVAL;
+
+	if (reset_control_is_array(rstc))
+		return reset_control_array_acquire(rstc_to_array(rstc));
+
+	mutex_lock(&reset_list_mutex);
+
+	if (rstc->acquired) {
+		mutex_unlock(&reset_list_mutex);
+		return 0;
+	}
+
+	list_for_each_entry(rc, &rstc->rcdev->reset_control_head, list) {
+		if (rstc != rc && rstc->id == rc->id) {
+			if (rc->acquired) {
+				mutex_unlock(&reset_list_mutex);
+				return -EBUSY;
+			}
+		}
+	}
+
+	rstc->acquired = true;
+
+	mutex_unlock(&reset_list_mutex);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(reset_control_acquire);
+
+/**
+ * reset_control_release() - releases exclusive access to a reset control
+ * @rstc: reset control
+ *
+ * Releases exclusive access right to a reset control previously obtained by a
+ * call to reset_control_acquire(). Until a consumer calls this function, no
+ * other consumers will be granted exclusive access.
+ *
+ * See also: reset_control_acquire()
+ */
+void reset_control_release(struct reset_control *rstc)
+{
+	if (!rstc || WARN_ON(IS_ERR(rstc)))
+		return;
+
+	if (reset_control_is_array(rstc))
+		reset_control_array_release(rstc_to_array(rstc));
+	else
+		rstc->acquired = false;
+}
+EXPORT_SYMBOL_GPL(reset_control_release);
+
 static struct reset_control *__reset_control_get_internal(
 				struct reset_controller_dev *rcdev,
-				unsigned int index, bool shared)
+				unsigned int index, bool shared, bool acquired)
 {
 	struct reset_control *rstc;
 
@@ -416,6 +550,14 @@ static struct reset_control *__reset_control_get_internal(
 
 	list_for_each_entry(rstc, &rcdev->reset_control_head, list) {
 		if (rstc->id == index) {
+			/*
+			 * Allow creating a secondary exclusive reset_control
+			 * that is initially not acquired for an already
+			 * controlled reset line.
+			 */
+			if (!rstc->shared && !shared && !acquired)
+				break;
+
 			if (WARN_ON(!rstc->shared || !shared))
 				return ERR_PTR(-EBUSY);
 
@@ -434,6 +576,7 @@ static struct reset_control *__reset_control_get_internal(
 	list_add(&rstc->list, &rcdev->reset_control_head);
 	rstc->id = index;
 	kref_init(&rstc->refcnt);
+	rstc->acquired = acquired;
 	rstc->shared = shared;
 
 	return rstc;
@@ -461,7 +604,7 @@ static void __reset_control_put_internal(struct reset_control *rstc)
 
 struct reset_control *__of_reset_control_get(struct device_node *node,
 				     const char *id, int index, bool shared,
-				     bool optional)
+				     bool optional, bool acquired)
 {
 	struct reset_control *rstc;
 	struct reset_controller_dev *r, *rcdev;
@@ -514,7 +657,7 @@ struct reset_control *__of_reset_control_get(struct device_node *node,
 	}
 
 	/* reset_list_mutex also protects the rcdev's reset_control list */
-	rstc = __reset_control_get_internal(rcdev, rstc_id, shared);
+	rstc = __reset_control_get_internal(rcdev, rstc_id, shared, acquired);
 
 out:
 	mutex_unlock(&reset_list_mutex);
@@ -544,7 +687,7 @@ __reset_controller_by_name(const char *name)
 
 static struct reset_control *
 __reset_control_get_from_lookup(struct device *dev, const char *con_id,
-				bool shared, bool optional)
+				bool shared, bool optional, bool acquired)
 {
 	const struct reset_control_lookup *lookup;
 	struct reset_controller_dev *rcdev;
@@ -574,7 +717,7 @@ __reset_control_get_from_lookup(struct device *dev, const char *con_id,
 
 			rstc = __reset_control_get_internal(rcdev,
 							    lookup->index,
-							    shared);
+							    shared, acquired);
 			mutex_unlock(&reset_list_mutex);
 			break;
 		}
@@ -589,13 +732,18 @@ __reset_control_get_from_lookup(struct device *dev, const char *con_id,
 }
 
 struct reset_control *__reset_control_get(struct device *dev, const char *id,
-					  int index, bool shared, bool optional)
+					  int index, bool shared, bool optional,
+					  bool acquired)
 {
+	if (WARN_ON(shared && acquired))
+		return ERR_PTR(-EINVAL);
+
 	if (dev->of_node)
 		return __of_reset_control_get(dev->of_node, id, index, shared,
-					      optional);
+					      optional, acquired);
 
-	return __reset_control_get_from_lookup(dev, id, shared, optional);
+	return __reset_control_get_from_lookup(dev, id, shared, optional,
+					       acquired);
 }
 EXPORT_SYMBOL_GPL(__reset_control_get);
 
@@ -636,7 +784,7 @@ static void devm_reset_control_release(struct device *dev, void *res)
 
 struct reset_control *__devm_reset_control_get(struct device *dev,
 				     const char *id, int index, bool shared,
-				     bool optional)
+				     bool optional, bool acquired)
 {
 	struct reset_control **ptr, *rstc;
 
@@ -645,7 +793,7 @@ struct reset_control *__devm_reset_control_get(struct device *dev,
 	if (!ptr)
 		return ERR_PTR(-ENOMEM);
 
-	rstc = __reset_control_get(dev, id, index, shared, optional);
+	rstc = __reset_control_get(dev, id, index, shared, optional, acquired);
 	if (!IS_ERR(rstc)) {
 		*ptr = rstc;
 		devres_add(dev, ptr);
@@ -672,7 +820,7 @@ int __device_reset(struct device *dev, bool optional)
 	struct reset_control *rstc;
 	int ret;
 
-	rstc = __reset_control_get(dev, NULL, 0, 0, optional);
+	rstc = __reset_control_get(dev, NULL, 0, 0, optional, true);
 	if (IS_ERR(rstc))
 		return PTR_ERR(rstc);
 
@@ -716,12 +864,15 @@ static int of_reset_control_get_count(struct device_node *node)
  * @np: device node for the device that requests the reset controls array
  * @shared: whether reset controls are shared or not
  * @optional: whether it is optional to get the reset controls
+ * @acquired: only one reset control may be acquired for a given controller
+ *            and ID
  *
  * Returns pointer to allocated reset_control_array on success or
  * error on failure
  */
 struct reset_control *
-of_reset_control_array_get(struct device_node *np, bool shared, bool optional)
+of_reset_control_array_get(struct device_node *np, bool shared, bool optional,
+			   bool acquired)
 {
 	struct reset_control_array *resets;
 	struct reset_control *rstc;
@@ -736,7 +887,8 @@ of_reset_control_array_get(struct device_node *np, bool shared, bool optional)
 		return ERR_PTR(-ENOMEM);
 
 	for (i = 0; i < num; i++) {
-		rstc = __of_reset_control_get(np, NULL, i, shared, optional);
+		rstc = __of_reset_control_get(np, NULL, i, shared, optional,
+					      acquired);
 		if (IS_ERR(rstc))
 			goto err_rst;
 		resets->rstc[i] = rstc;
@@ -783,7 +935,7 @@ devm_reset_control_array_get(struct device *dev, bool shared, bool optional)
 	if (!devres)
 		return ERR_PTR(-ENOMEM);
 
-	rstc = of_reset_control_array_get(dev->of_node, shared, optional);
+	rstc = of_reset_control_array_get(dev->of_node, shared, optional, true);
 	if (IS_ERR(rstc)) {
 		devres_free(devres);
 		return rstc;
diff --git a/drivers/rtc/Kconfig b/drivers/rtc/Kconfig
index f933c06bff4f..7b8e156dbf38 100644
--- a/drivers/rtc/Kconfig
+++ b/drivers/rtc/Kconfig
@@ -439,6 +439,7 @@ config RTC_DRV_PCF8523
 
 config RTC_DRV_PCF85063
 	tristate "NXP PCF85063"
+	select REGMAP_I2C
 	help
 	  If you say yes here you get support for the PCF85063 RTC chip
 
@@ -447,7 +448,6 @@ config RTC_DRV_PCF85063
 
 config RTC_DRV_PCF85363
 	tristate "NXP PCF85363"
-	depends on I2C
 	select REGMAP_I2C
 	help
 	  If you say yes here you get support for the PCF85363 RTC chip.
@@ -602,7 +602,6 @@ config RTC_DRV_FM3130
 
 config RTC_DRV_RX8010
 	tristate "Epson RX8010SJ"
-	depends on I2C
 	help
 	  If you say yes here you get support for the Epson RX8010SJ RTC
 	  chip.
@@ -1432,7 +1431,7 @@ config RTC_DRV_AT91RM9200
 config RTC_DRV_AT91SAM9
 	tristate "AT91SAM9 RTT as RTC"
 	depends on ARCH_AT91 || COMPILE_TEST
-	depends on HAS_IOMEM
+	depends on OF && HAS_IOMEM
 	select MFD_SYSCON
 	help
 	  Some AT91SAM9 SoCs provide an RTT (Real Time Timer) block which
@@ -1841,6 +1840,17 @@ config RTC_DRV_RTD119X
 	  If you say yes here, you get support for the RTD1295 SoC
 	  Real Time Clock.
 
+config RTC_DRV_ASPEED
+	tristate "ASPEED RTC"
+	depends on OF
+	depends on ARCH_ASPEED || COMPILE_TEST
+	help
+	  If you say yes here you get support for the ASPEED BMC SoC real time
+	  clocks.
+
+	  This driver can also be built as a module, if so, the module
+	  will be called "rtc-aspeed".
+
 comment "HID Sensor RTC drivers"
 
 config RTC_DRV_HID_SENSOR_TIME
@@ -1857,7 +1867,8 @@ config RTC_DRV_HID_SENSOR_TIME
 
 config RTC_DRV_GOLDFISH
 	tristate "Goldfish Real Time Clock"
-	depends on MIPS && (GOLDFISH || COMPILE_TEST)
+	depends on OF && HAS_IOMEM
+	depends on GOLDFISH || COMPILE_TEST
 	help
 	  Say yes to enable RTC driver for the Goldfish based virtual platform.
 
diff --git a/drivers/rtc/Makefile b/drivers/rtc/Makefile
index fe3962496685..9d997faa2c26 100644
--- a/drivers/rtc/Makefile
+++ b/drivers/rtc/Makefile
@@ -34,6 +34,7 @@ obj-$(CONFIG_RTC_DRV_AC100)	+= rtc-ac100.o
 obj-$(CONFIG_RTC_DRV_ARMADA38X)	+= rtc-armada38x.o
 obj-$(CONFIG_RTC_DRV_AS3722)	+= rtc-as3722.o
 obj-$(CONFIG_RTC_DRV_ASM9260)	+= rtc-asm9260.o
+obj-$(CONFIG_RTC_DRV_ASPEED)	+= rtc-aspeed.o
 obj-$(CONFIG_RTC_DRV_AT91RM9200)+= rtc-at91rm9200.o
 obj-$(CONFIG_RTC_DRV_AT91SAM9)	+= rtc-at91sam9.o
 obj-$(CONFIG_RTC_DRV_AU1XXX)	+= rtc-au1xxx.o
diff --git a/drivers/rtc/class.c b/drivers/rtc/class.c
index ac93b76f2b11..0f492b0940b3 100644
--- a/drivers/rtc/class.c
+++ b/drivers/rtc/class.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * RTC subsystem, base class
  *
@@ -5,11 +6,7 @@
  * Author: Alessandro Zummo <a.zummo@towertech.it>
  *
  * class skeleton from drivers/hwmon/hwmon.c
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
@@ -23,13 +20,13 @@
 
 #include "rtc-core.h"
 
-
 static DEFINE_IDA(rtc_ida);
 struct class *rtc_class;
 
 static void rtc_device_release(struct device *dev)
 {
 	struct rtc_device *rtc = to_rtc_device(dev);
+
 	ida_simple_remove(&rtc_ida, rtc->id);
 	kfree(rtc);
 }
@@ -47,7 +44,6 @@ int rtc_hctosys_ret = -ENODEV;
 
 static struct timespec64 old_rtc, old_system, old_delta;
 
-
 static int rtc_suspend(struct device *dev)
 {
 	struct rtc_device	*rtc = to_rtc_device(dev);
@@ -71,7 +67,6 @@ static int rtc_suspend(struct device *dev)
 	ktime_get_real_ts64(&old_system);
 	old_rtc.tv_sec = rtc_tm_to_time64(&tm);
 
-
 	/*
 	 * To avoid drift caused by repeated suspend/resumes,
 	 * which each can add ~1 second drift error,
@@ -83,7 +78,7 @@ static int rtc_suspend(struct device *dev)
 	if (delta_delta.tv_sec < -2 || delta_delta.tv_sec >= 2) {
 		/*
 		 * if delta_delta is too large, assume time correction
-		 * has occured and set old_delta to the current delta.
+		 * has occurred and set old_delta to the current delta.
 		 */
 		old_delta = delta;
 	} else {
@@ -136,7 +131,7 @@ static int rtc_resume(struct device *dev)
 	 * to keep things accurate.
 	 */
 	sleep_time = timespec64_sub(sleep_time,
-			timespec64_sub(new_system, old_system));
+				    timespec64_sub(new_system, old_system));
 
 	if (sleep_time.tv_sec >= 0)
 		timekeeping_inject_sleeptime64(&sleep_time);
@@ -397,9 +392,9 @@ EXPORT_SYMBOL_GPL(__rtc_register_device);
  * rtc_register_device instead
  */
 struct rtc_device *devm_rtc_device_register(struct device *dev,
-					const char *name,
-					const struct rtc_class_ops *ops,
-					struct module *owner)
+					    const char *name,
+					    const struct rtc_class_ops *ops,
+					    struct module *owner)
 {
 	struct rtc_device *rtc;
 	int err;
diff --git a/drivers/rtc/dev.c b/drivers/rtc/dev.c
index 1d006ef4bb57..84feb2565abd 100644
--- a/drivers/rtc/dev.c
+++ b/drivers/rtc/dev.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * RTC subsystem, dev interface
  *
@@ -5,11 +6,7 @@
  * Author: Alessandro Zummo <a.zummo@towertech.it>
  *
  * based on arch/arm/common/rtctime.c
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
@@ -60,7 +57,7 @@ static void rtc_uie_task(struct work_struct *work)
 	} else if (rtc->oldsecs != tm.tm_sec) {
 		num = (tm.tm_sec + 60 - rtc->oldsecs) % 60;
 		rtc->oldsecs = tm.tm_sec;
-		rtc->uie_timer.expires = jiffies + HZ - (HZ/10);
+		rtc->uie_timer.expires = jiffies + HZ - (HZ / 10);
 		rtc->uie_timer_active = 1;
 		rtc->uie_task_active = 0;
 		add_timer(&rtc->uie_timer);
@@ -71,6 +68,7 @@ static void rtc_uie_task(struct work_struct *work)
 	if (num)
 		rtc_handle_legacy_irq(rtc, num, RTC_UF);
 }
+
 static void rtc_uie_timer(struct timer_list *t)
 {
 	struct rtc_device *rtc = from_timer(rtc, t, uie_timer);
@@ -202,14 +200,14 @@ static __poll_t rtc_dev_poll(struct file *file, poll_table *wait)
 }
 
 static long rtc_dev_ioctl(struct file *file,
-		unsigned int cmd, unsigned long arg)
+			  unsigned int cmd, unsigned long arg)
 {
 	int err = 0;
 	struct rtc_device *rtc = file->private_data;
 	const struct rtc_class_ops *ops = rtc->ops;
 	struct rtc_time tm;
 	struct rtc_wkalrm alarm;
-	void __user *uarg = (void __user *) arg;
+	void __user *uarg = (void __user *)arg;
 
 	err = mutex_lock_interruptible(&rtc->ops_lock);
 	if (err)
@@ -233,7 +231,7 @@ static long rtc_dev_ioctl(struct file *file,
 
 	case RTC_PIE_ON:
 		if (rtc->irq_freq > rtc->max_user_freq &&
-				!capable(CAP_SYS_RESOURCE))
+		    !capable(CAP_SYS_RESOURCE))
 			err = -EACCES;
 		break;
 	}
@@ -390,8 +388,9 @@ static long rtc_dev_ioctl(struct file *file,
 			err = ops->ioctl(rtc->dev.parent, cmd, arg);
 			if (err == -ENOIOCTLCMD)
 				err = -ENOTTY;
-		} else
+		} else {
 			err = -ENOTTY;
+		}
 		break;
 	}
 
@@ -403,6 +402,7 @@ done:
 static int rtc_dev_fasync(int fd, struct file *file, int on)
 {
 	struct rtc_device *rtc = file->private_data;
+
 	return fasync_helper(fd, file, on, &rtc->async_queue);
 }
 
diff --git a/drivers/rtc/hctosys.c b/drivers/rtc/hctosys.c
index ff2092a0d38c..a74d0d890600 100644
--- a/drivers/rtc/hctosys.c
+++ b/drivers/rtc/hctosys.c
@@ -1,13 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * RTC subsystem, initialize system time on startup
  *
  * Copyright (C) 2005 Tower Technologies
  * Author: Alessandro Zummo <a.zummo@towertech.it>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
@@ -33,7 +30,7 @@ static int __init rtc_hctosys(void)
 	};
 	struct rtc_device *rtc = rtc_class_open(CONFIG_RTC_HCTOSYS_DEVICE);
 
-	if (rtc == NULL) {
+	if (!rtc) {
 		pr_info("unable to open rtc device (%s)\n",
 			CONFIG_RTC_HCTOSYS_DEVICE);
 		goto err_open;
@@ -44,7 +41,6 @@ static int __init rtc_hctosys(void)
 		dev_err(rtc->dev.parent,
 			"hctosys: unable to read the hardware clock\n");
 		goto err_read;
-
 	}
 
 	tv64.tv_sec = rtc_tm_to_time64(&tm);
diff --git a/drivers/rtc/interface.c b/drivers/rtc/interface.c
index 98d9c87b0d1b..4124f4dd376b 100644
--- a/drivers/rtc/interface.c
+++ b/drivers/rtc/interface.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * RTC subsystem, interface functions
  *
@@ -5,11 +6,7 @@
  * Author: Alessandro Zummo <a.zummo@towertech.it>
  *
  * based on arch/arm/common/rtctime.c
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #include <linux/rtc.h>
 #include <linux/sched.h>
@@ -87,11 +84,12 @@ static int rtc_valid_range(struct rtc_device *rtc, struct rtc_time *tm)
 static int __rtc_read_time(struct rtc_device *rtc, struct rtc_time *tm)
 {
 	int err;
-	if (!rtc->ops)
+
+	if (!rtc->ops) {
 		err = -ENODEV;
-	else if (!rtc->ops->read_time)
+	} else if (!rtc->ops->read_time) {
 		err = -EINVAL;
-	else {
+	} else {
 		memset(tm, 0, sizeof(struct rtc_time));
 		err = rtc->ops->read_time(rtc->dev.parent, tm);
 		if (err < 0) {
@@ -147,14 +145,7 @@ int rtc_set_time(struct rtc_device *rtc, struct rtc_time *tm)
 		err = -ENODEV;
 	else if (rtc->ops->set_time)
 		err = rtc->ops->set_time(rtc->dev.parent, tm);
-	else if (rtc->ops->set_mmss64) {
-		time64_t secs64 = rtc_tm_to_time64(tm);
-
-		err = rtc->ops->set_mmss64(rtc->dev.parent, secs64);
-	} else if (rtc->ops->set_mmss) {
-		time64_t secs64 = rtc_tm_to_time64(tm);
-		err = rtc->ops->set_mmss(rtc->dev.parent, secs64);
-	} else
+	else
 		err = -EINVAL;
 
 	pm_stay_awake(rtc->dev.parent);
@@ -167,7 +158,8 @@ int rtc_set_time(struct rtc_device *rtc, struct rtc_time *tm)
 }
 EXPORT_SYMBOL_GPL(rtc_set_time);
 
-static int rtc_read_alarm_internal(struct rtc_device *rtc, struct rtc_wkalrm *alarm)
+static int rtc_read_alarm_internal(struct rtc_device *rtc,
+				   struct rtc_wkalrm *alarm)
 {
 	int err;
 
@@ -175,11 +167,11 @@ static int rtc_read_alarm_internal(struct rtc_device *rtc, struct rtc_wkalrm *al
 	if (err)
 		return err;
 
-	if (rtc->ops == NULL)
+	if (!rtc->ops) {
 		err = -ENODEV;
-	else if (!rtc->ops->read_alarm)
+	} else if (!rtc->ops->read_alarm) {
 		err = -EINVAL;
-	else {
+	} else {
 		alarm->enabled = 0;
 		alarm->pending = 0;
 		alarm->time.tm_sec = -1;
@@ -207,7 +199,7 @@ int __rtc_read_alarm(struct rtc_device *rtc, struct rtc_wkalrm *alarm)
 	int first_time = 1;
 	time64_t t_now, t_alm;
 	enum { none, day, month, year } missing = none;
-	unsigned days;
+	unsigned int days;
 
 	/* The lower level RTC driver may return -1 in some fields,
 	 * creating invalid alarm->time values, for reasons like:
@@ -276,10 +268,10 @@ int __rtc_read_alarm(struct rtc_device *rtc, struct rtc_wkalrm *alarm)
 			return err;
 
 		/* note that tm_sec is a "don't care" value here: */
-	} while (   before.tm_min   != now.tm_min
-		 || before.tm_hour  != now.tm_hour
-		 || before.tm_mon   != now.tm_mon
-		 || before.tm_year  != now.tm_year);
+	} while (before.tm_min  != now.tm_min ||
+		 before.tm_hour != now.tm_hour ||
+		 before.tm_mon  != now.tm_mon ||
+		 before.tm_year != now.tm_year);
 
 	/* Fill in the missing alarm fields using the timestamp; we
 	 * know there's at least one since alarm->time is invalid.
@@ -296,7 +288,7 @@ int __rtc_read_alarm(struct rtc_device *rtc, struct rtc_wkalrm *alarm)
 		alarm->time.tm_mday = now.tm_mday;
 		missing = day;
 	}
-	if ((unsigned)alarm->time.tm_mon >= 12) {
+	if ((unsigned int)alarm->time.tm_mon >= 12) {
 		alarm->time.tm_mon = now.tm_mon;
 		if (missing == none)
 			missing = month;
@@ -321,7 +313,6 @@ int __rtc_read_alarm(struct rtc_device *rtc, struct rtc_wkalrm *alarm)
 		goto done;
 
 	switch (missing) {
-
 	/* 24 hour rollover ... if it's now 10am Monday, an alarm that
 	 * that will trigger at 5am will do so at 5am Tuesday, which
 	 * could also be in the next month or year.  This is a common
@@ -341,14 +332,14 @@ int __rtc_read_alarm(struct rtc_device *rtc, struct rtc_wkalrm *alarm)
 	case month:
 		dev_dbg(&rtc->dev, "alarm rollover: %s\n", "month");
 		do {
-			if (alarm->time.tm_mon < 11)
+			if (alarm->time.tm_mon < 11) {
 				alarm->time.tm_mon++;
-			else {
+			} else {
 				alarm->time.tm_mon = 0;
 				alarm->time.tm_year++;
 			}
 			days = rtc_month_days(alarm->time.tm_mon,
-					alarm->time.tm_year);
+					      alarm->time.tm_year);
 		} while (days < alarm->time.tm_mday);
 		break;
 
@@ -357,8 +348,8 @@ int __rtc_read_alarm(struct rtc_device *rtc, struct rtc_wkalrm *alarm)
 		dev_dbg(&rtc->dev, "alarm rollover: %s\n", "year");
 		do {
 			alarm->time.tm_year++;
-		} while (!is_leap_year(alarm->time.tm_year + 1900)
-			&& rtc_valid_tm(&alarm->time) != 0);
+		} while (!is_leap_year(alarm->time.tm_year + 1900) &&
+			 rtc_valid_tm(&alarm->time) != 0);
 		break;
 
 	default:
@@ -369,7 +360,8 @@ int __rtc_read_alarm(struct rtc_device *rtc, struct rtc_wkalrm *alarm)
 
 done:
 	if (err)
-		dev_warn(&rtc->dev, "invalid alarm value: %ptR\n", &alarm->time);
+		dev_warn(&rtc->dev, "invalid alarm value: %ptR\n",
+			 &alarm->time);
 
 	return err;
 }
@@ -381,11 +373,11 @@ int rtc_read_alarm(struct rtc_device *rtc, struct rtc_wkalrm *alarm)
 	err = mutex_lock_interruptible(&rtc->ops_lock);
 	if (err)
 		return err;
-	if (rtc->ops == NULL)
+	if (!rtc->ops) {
 		err = -ENODEV;
-	else if (!rtc->ops->read_alarm)
+	} else if (!rtc->ops->read_alarm) {
 		err = -EINVAL;
-	else {
+	} else {
 		memset(alarm, 0, sizeof(struct rtc_wkalrm));
 		alarm->enabled = rtc->aie_timer.enabled;
 		alarm->time = rtc_ktime_to_tm(rtc->aie_timer.node.expires);
@@ -494,7 +486,6 @@ int rtc_initialize_alarm(struct rtc_device *rtc, struct rtc_wkalrm *alarm)
 	/* Alarm has to be enabled & in the future for us to enqueue it */
 	if (alarm->enabled && (rtc_tm_to_ktime(now) <
 			 rtc->aie_timer.node.expires)) {
-
 		rtc->aie_timer.enabled = 1;
 		timerqueue_add(&rtc->timerqueue, &rtc->aie_timer.node);
 		trace_rtc_timer_enqueue(&rtc->aie_timer);
@@ -506,7 +497,9 @@ EXPORT_SYMBOL_GPL(rtc_initialize_alarm);
 
 int rtc_alarm_irq_enable(struct rtc_device *rtc, unsigned int enabled)
 {
-	int err = mutex_lock_interruptible(&rtc->ops_lock);
+	int err;
+
+	err = mutex_lock_interruptible(&rtc->ops_lock);
 	if (err)
 		return err;
 
@@ -535,7 +528,9 @@ EXPORT_SYMBOL_GPL(rtc_alarm_irq_enable);
 
 int rtc_update_irq_enable(struct rtc_device *rtc, unsigned int enabled)
 {
-	int err = mutex_lock_interruptible(&rtc->ops_lock);
+	int err;
+
+	err = mutex_lock_interruptible(&rtc->ops_lock);
 	if (err)
 		return err;
 
@@ -564,27 +559,25 @@ int rtc_update_irq_enable(struct rtc_device *rtc, unsigned int enabled)
 		rtc->uie_rtctimer.node.expires = ktime_add(now, onesec);
 		rtc->uie_rtctimer.period = ktime_set(1, 0);
 		err = rtc_timer_enqueue(rtc, &rtc->uie_rtctimer);
-	} else
+	} else {
 		rtc_timer_remove(rtc, &rtc->uie_rtctimer);
+	}
 
 out:
 	mutex_unlock(&rtc->ops_lock);
 #ifdef CONFIG_RTC_INTF_DEV_UIE_EMUL
 	/*
-	 * Enable emulation if the driver did not provide
-	 * the update_irq_enable function pointer or if returned
-	 * -EINVAL to signal that it has been configured without
-	 * interrupts or that are not available at the moment.
+	 * Enable emulation if the driver returned -EINVAL to signal that it has
+	 * been configured without interrupts or they are not available at the
+	 * moment.
 	 */
 	if (err == -EINVAL)
 		err = rtc_dev_update_irq_enable_emul(rtc, enabled);
 #endif
 	return err;
-
 }
 EXPORT_SYMBOL_GPL(rtc_update_irq_enable);
 
-
 /**
  * rtc_handle_legacy_irq - AIE, UIE and PIE event hook
  * @rtc: pointer to the rtc device
@@ -599,14 +592,13 @@ void rtc_handle_legacy_irq(struct rtc_device *rtc, int num, int mode)
 
 	/* mark one irq of the appropriate mode */
 	spin_lock_irqsave(&rtc->irq_lock, flags);
-	rtc->irq_data = (rtc->irq_data + (num << 8)) | (RTC_IRQF|mode);
+	rtc->irq_data = (rtc->irq_data + (num << 8)) | (RTC_IRQF | mode);
 	spin_unlock_irqrestore(&rtc->irq_lock, flags);
 
 	wake_up_interruptible(&rtc->irq_queue);
 	kill_fasync(&rtc->async_queue, SIGIO, POLL_IN);
 }
 
-
 /**
  * rtc_aie_update_irq - AIE mode rtctimer hook
  * @rtc: pointer to the rtc_device
@@ -618,7 +610,6 @@ void rtc_aie_update_irq(struct rtc_device *rtc)
 	rtc_handle_legacy_irq(rtc, 1, RTC_AF);
 }
 
-
 /**
  * rtc_uie_update_irq - UIE mode rtctimer hook
  * @rtc: pointer to the rtc_device
@@ -630,7 +621,6 @@ void rtc_uie_update_irq(struct rtc_device *rtc)
 	rtc_handle_legacy_irq(rtc, 1,  RTC_UF);
 }
 
-
 /**
  * rtc_pie_update_irq - PIE mode hrtimer hook
  * @timer: pointer to the pie mode hrtimer
@@ -644,6 +634,7 @@ enum hrtimer_restart rtc_pie_update_irq(struct hrtimer *timer)
 	struct rtc_device *rtc;
 	ktime_t period;
 	int count;
+
 	rtc = container_of(timer, struct rtc_device, pie_timer);
 
 	period = NSEC_PER_SEC / rtc->irq_freq;
@@ -662,7 +653,7 @@ enum hrtimer_restart rtc_pie_update_irq(struct hrtimer *timer)
  * Context: any
  */
 void rtc_update_irq(struct rtc_device *rtc,
-		unsigned long num, unsigned long events)
+		    unsigned long num, unsigned long events)
 {
 	if (IS_ERR_OR_NULL(rtc))
 		return;
@@ -811,6 +802,7 @@ static int rtc_timer_enqueue(struct rtc_device *rtc, struct rtc_timer *timer)
 	if (!next || ktime_before(timer->node.expires, next->expires)) {
 		struct rtc_wkalrm alarm;
 		int err;
+
 		alarm.time = rtc_ktime_to_tm(timer->node.expires);
 		alarm.enabled = 1;
 		err = __rtc_set_alarm(rtc, &alarm);
@@ -851,12 +843,14 @@ static void rtc_alarm_disable(struct rtc_device *rtc)
 static void rtc_timer_remove(struct rtc_device *rtc, struct rtc_timer *timer)
 {
 	struct timerqueue_node *next = timerqueue_getnext(&rtc->timerqueue);
+
 	timerqueue_del(&rtc->timerqueue, &timer->node);
 	trace_rtc_timer_dequeue(timer);
 	timer->enabled = 0;
 	if (next == &timer->node) {
 		struct rtc_wkalrm alarm;
 		int err;
+
 		next = timerqueue_getnext(&rtc->timerqueue);
 		if (!next) {
 			rtc_alarm_disable(rtc);
@@ -929,9 +923,9 @@ again:
 		alarm.enabled = 1;
 reprogram:
 		err = __rtc_set_alarm(rtc, &alarm);
-		if (err == -ETIME)
+		if (err == -ETIME) {
 			goto again;
-		else if (err) {
+		} else if (err) {
 			if (retry-- > 0)
 				goto reprogram;
 
@@ -942,14 +936,14 @@ reprogram:
 			dev_err(&rtc->dev, "__rtc_set_alarm: err=%d\n", err);
 			goto again;
 		}
-	} else
+	} else {
 		rtc_alarm_disable(rtc);
+	}
 
 	pm_relax(rtc->dev.parent);
 	mutex_unlock(&rtc->ops_lock);
 }
 
-
 /* rtc_timer_init - Initializes an rtc_timer
  * @timer: timer to be intiialized
  * @f: function pointer to be called when timer fires
@@ -975,9 +969,10 @@ void rtc_timer_init(struct rtc_timer *timer, void (*f)(struct rtc_device *r),
  * Kernel interface to set an rtc_timer
  */
 int rtc_timer_start(struct rtc_device *rtc, struct rtc_timer *timer,
-			ktime_t expires, ktime_t period)
+		    ktime_t expires, ktime_t period)
 {
 	int ret = 0;
+
 	mutex_lock(&rtc->ops_lock);
 	if (timer->enabled)
 		rtc_timer_remove(rtc, timer);
diff --git a/drivers/rtc/lib.c b/drivers/rtc/lib.c
index 9714cb3d1e29..23284580df97 100644
--- a/drivers/rtc/lib.c
+++ b/drivers/rtc/lib.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * rtc and date/time utility functions
  *
@@ -5,11 +6,7 @@
  * Author: Alessandro Zummo <a.zummo@towertech.it>
  *
  * based on arch/arm/common/rtctime.c and other bits
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #include <linux/export.h>
 #include <linux/rtc.h>
@@ -25,7 +22,7 @@ static const unsigned short rtc_ydays[2][13] = {
 	{ 0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335, 366 }
 };
 
-#define LEAPS_THRU_END_OF(y) ((y)/4 - (y)/100 + (y)/400)
+#define LEAPS_THRU_END_OF(y) ((y) / 4 - (y) / 100 + (y) / 400)
 
 /*
  * The number of days in the month.
@@ -41,11 +38,10 @@ EXPORT_SYMBOL(rtc_month_days);
  */
 int rtc_year_days(unsigned int day, unsigned int month, unsigned int year)
 {
-	return rtc_ydays[is_leap_year(year)][month] + day-1;
+	return rtc_ydays[is_leap_year(year)][month] + day - 1;
 }
 EXPORT_SYMBOL(rtc_year_days);
 
-
 /*
  * rtc_time64_to_tm - Converts time64_t to rtc_time.
  * Convert seconds since 01-01-1970 00:00:00 to Gregorian date.
@@ -97,13 +93,15 @@ EXPORT_SYMBOL(rtc_time64_to_tm);
  */
 int rtc_valid_tm(struct rtc_time *tm)
 {
-	if (tm->tm_year < 70
-		|| ((unsigned)tm->tm_mon) >= 12
-		|| tm->tm_mday < 1
-		|| tm->tm_mday > rtc_month_days(tm->tm_mon, ((unsigned)tm->tm_year + 1900))
-		|| ((unsigned)tm->tm_hour) >= 24
-		|| ((unsigned)tm->tm_min) >= 60
-		|| ((unsigned)tm->tm_sec) >= 60)
+	if (tm->tm_year < 70 ||
+	    tm->tm_year > (INT_MAX - 1900) ||
+	    ((unsigned int)tm->tm_mon) >= 12 ||
+	    tm->tm_mday < 1 ||
+	    tm->tm_mday > rtc_month_days(tm->tm_mon,
+					 ((unsigned int)tm->tm_year + 1900)) ||
+	    ((unsigned int)tm->tm_hour) >= 24 ||
+	    ((unsigned int)tm->tm_min) >= 60 ||
+	    ((unsigned int)tm->tm_sec) >= 60)
 		return -EINVAL;
 
 	return 0;
@@ -116,7 +114,7 @@ EXPORT_SYMBOL(rtc_valid_tm);
  */
 time64_t rtc_tm_to_time64(struct rtc_time *tm)
 {
-	return mktime64(((unsigned)tm->tm_year + 1900), tm->tm_mon + 1,
+	return mktime64(((unsigned int)tm->tm_year + 1900), tm->tm_mon + 1,
 			tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
 }
 EXPORT_SYMBOL(rtc_tm_to_time64);
diff --git a/drivers/rtc/nvmem.c b/drivers/rtc/nvmem.c
index dce518d5e50e..4312096c7738 100644
--- a/drivers/rtc/nvmem.c
+++ b/drivers/rtc/nvmem.c
@@ -1,11 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * RTC subsystem, nvmem interface
  *
  * Copyright (C) 2017 Alexandre Belloni
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include <linux/err.h>
@@ -46,7 +43,7 @@ static int rtc_nvram_register(struct rtc_device *rtc,
 {
 	int err;
 
-	rtc->nvram = kzalloc(sizeof(struct bin_attribute), GFP_KERNEL);
+	rtc->nvram = kzalloc(sizeof(*rtc->nvram), GFP_KERNEL);
 	if (!rtc->nvram)
 		return -ENOMEM;
 
diff --git a/drivers/rtc/proc.c b/drivers/rtc/proc.c
index 4d74e4f4ff30..73344598fc1b 100644
--- a/drivers/rtc/proc.c
+++ b/drivers/rtc/proc.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * RTC subsystem, proc interface
  *
@@ -5,11 +6,7 @@
  * Author: Alessandro Zummo <a.zummo@towertech.it>
  *
  * based on arch/arm/common/rtctime.c
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #include <linux/module.h>
 #include <linux/rtc.h>
@@ -60,17 +57,17 @@ static int rtc_proc_show(struct seq_file *seq, void *offset)
 		seq_printf(seq, "alrm_time\t: %ptRt\n", &alrm.time);
 		seq_printf(seq, "alrm_date\t: %ptRd\n", &alrm.time);
 		seq_printf(seq, "alarm_IRQ\t: %s\n",
-				alrm.enabled ? "yes" : "no");
+			   alrm.enabled ? "yes" : "no");
 		seq_printf(seq, "alrm_pending\t: %s\n",
-				alrm.pending ? "yes" : "no");
+			   alrm.pending ? "yes" : "no");
 		seq_printf(seq, "update IRQ enabled\t: %s\n",
-			(rtc->uie_rtctimer.enabled) ? "yes" : "no");
+			   (rtc->uie_rtctimer.enabled) ? "yes" : "no");
 		seq_printf(seq, "periodic IRQ enabled\t: %s\n",
-			(rtc->pie_enabled) ? "yes" : "no");
+			   (rtc->pie_enabled) ? "yes" : "no");
 		seq_printf(seq, "periodic IRQ frequency\t: %d\n",
-			rtc->irq_freq);
+			   rtc->irq_freq);
 		seq_printf(seq, "max user IRQ frequency\t: %d\n",
-			rtc->max_user_freq);
+			   rtc->max_user_freq);
 	}
 
 	seq_printf(seq, "24hr\t\t: yes\n");
@@ -85,7 +82,7 @@ void rtc_proc_add_device(struct rtc_device *rtc)
 {
 	if (is_rtc_hctosys(rtc))
 		proc_create_single_data("driver/rtc", 0, NULL, rtc_proc_show,
-				rtc);
+					rtc);
 }
 
 void rtc_proc_del_device(struct rtc_device *rtc)
diff --git a/drivers/rtc/rtc-88pm80x.c b/drivers/rtc/rtc-88pm80x.c
index 1fc48ebd3cd0..e4d5a19fd1c9 100644
--- a/drivers/rtc/rtc-88pm80x.c
+++ b/drivers/rtc/rtc-88pm80x.c
@@ -1,22 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Real Time Clock driver for Marvell 88PM80x PMIC
  *
  * Copyright (c) 2012 Marvell International Ltd.
  *  Wenzeng Chen<wzch@marvell.com>
  *  Qiao Zhou <zhouqiao@marvell.com>
- *
- * This file is subject to the terms and conditions of the GNU General
- * Public License. See the file "COPYING" in the main directory of this
- * archive for more details.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
 #include <linux/kernel.h>
diff --git a/drivers/rtc/rtc-88pm860x.c b/drivers/rtc/rtc-88pm860x.c
index d25282b4a7dd..73697e4b18a9 100644
--- a/drivers/rtc/rtc-88pm860x.c
+++ b/drivers/rtc/rtc-88pm860x.c
@@ -421,7 +421,7 @@ static int pm860x_rtc_remove(struct platform_device *pdev)
 	struct pm860x_rtc_info *info = platform_get_drvdata(pdev);
 
 #ifdef VRTC_CALIBRATION
-	flush_scheduled_work();
+	cancel_delayed_work_sync(&info->calib_work);
 	/* disable measurement */
 	pm860x_set_bits(info->i2c, PM8607_MEAS_EN2, MEAS2_VRTC, 0);
 #endif	/* VRTC_CALIBRATION */
diff --git a/drivers/rtc/rtc-ab-b5ze-s3.c b/drivers/rtc/rtc-ab-b5ze-s3.c
index 2233601761ac..cdad6f00debf 100644
--- a/drivers/rtc/rtc-ab-b5ze-s3.c
+++ b/drivers/rtc/rtc-ab-b5ze-s3.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  * rtc-ab-b5ze-s3 - Driver for Abracon AB-RTCMC-32.768Khz-B5ZE-S3
  *                  I2C RTC / Alarm chip
@@ -10,19 +11,9 @@
  *
  * This work is based on ISL12057 driver (drivers/rtc/rtc-isl12057.c).
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #include <linux/module.h>
-#include <linux/mutex.h>
 #include <linux/rtc.h>
 #include <linux/i2c.h>
 #include <linux/bcd.h>
@@ -128,7 +119,6 @@
 struct abb5zes3_rtc_data {
 	struct rtc_device *rtc;
 	struct regmap *regmap;
-	struct mutex lock;
 
 	int irq;
 
@@ -138,8 +128,7 @@ struct abb5zes3_rtc_data {
 
 /*
  * Try and match register bits w/ fixed null values to see whether we
- * are dealing with an ABB5ZES3. Note: this function is called early
- * during init and hence does need mutex protection.
+ * are dealing with an ABB5ZES3.
  */
 static int abb5zes3_i2c_validate_chip(struct regmap *regmap)
 {
@@ -230,14 +219,12 @@ static int _abb5zes3_rtc_read_time(struct device *dev, struct rtc_time *tm)
 	if (ret) {
 		dev_err(dev, "%s: reading RTC time failed (%d)\n",
 			__func__, ret);
-		goto err;
+		return ret;
 	}
 
 	/* If clock integrity is not guaranteed, do not return a time value */
-	if (regs[ABB5ZES3_REG_RTC_SC] & ABB5ZES3_REG_RTC_SC_OSC) {
-		ret = -ENODATA;
-		goto err;
-	}
+	if (regs[ABB5ZES3_REG_RTC_SC] & ABB5ZES3_REG_RTC_SC_OSC)
+		return -ENODATA;
 
 	tm->tm_sec = bcd2bin(regs[ABB5ZES3_REG_RTC_SC] & 0x7F);
 	tm->tm_min = bcd2bin(regs[ABB5ZES3_REG_RTC_MN]);
@@ -255,7 +242,6 @@ static int _abb5zes3_rtc_read_time(struct device *dev, struct rtc_time *tm)
 	tm->tm_mon  = bcd2bin(regs[ABB5ZES3_REG_RTC_MO]) - 1; /* starts at 1 */
 	tm->tm_year = bcd2bin(regs[ABB5ZES3_REG_RTC_YR]) + 100;
 
-err:
 	return ret;
 }
 
@@ -273,12 +259,9 @@ static int abb5zes3_rtc_set_time(struct device *dev, struct rtc_time *tm)
 	regs[ABB5ZES3_REG_RTC_MO] = bin2bcd(tm->tm_mon + 1);
 	regs[ABB5ZES3_REG_RTC_YR] = bin2bcd(tm->tm_year - 100);
 
-	mutex_lock(&data->lock);
 	ret = regmap_bulk_write(data->regmap, ABB5ZES3_REG_RTC_SC,
 				regs + ABB5ZES3_REG_RTC_SC,
 				ABB5ZES3_RTC_SEC_LEN);
-	mutex_unlock(&data->lock);
-
 
 	return ret;
 }
@@ -332,38 +315,35 @@ static int _abb5zes3_rtc_read_timer(struct device *dev,
 	if (ret) {
 		dev_err(dev, "%s: reading Timer A section failed (%d)\n",
 			__func__, ret);
-		goto err;
+		return ret;
 	}
 
 	/* get current time ... */
 	ret = _abb5zes3_rtc_read_time(dev, &rtc_tm);
 	if (ret)
-		goto err;
+		return ret;
 
 	/* ... convert to seconds ... */
-	ret = rtc_tm_to_time(&rtc_tm, &rtc_secs);
-	if (ret)
-		goto err;
+	rtc_secs = rtc_tm_to_time64(&rtc_tm);
 
 	/* ... add remaining timer A time ... */
 	ret = sec_from_timer_a(&timer_secs, regs[1], regs[2]);
 	if (ret)
-		goto err;
+		return ret;
 
 	/* ... and convert back. */
-	rtc_time_to_tm(rtc_secs + timer_secs, alarm_tm);
+	rtc_time64_to_tm(rtc_secs + timer_secs, alarm_tm);
 
 	ret = regmap_read(data->regmap, ABB5ZES3_REG_CTRL2, &reg);
 	if (ret) {
 		dev_err(dev, "%s: reading ctrl reg failed (%d)\n",
 			__func__, ret);
-		goto err;
+		return ret;
 	}
 
 	alarm->enabled = !!(reg & ABB5ZES3_REG_CTRL2_WTAIE);
 
-err:
-	return ret;
+	return 0;
 }
 
 /* Read alarm currently configured via a RTC alarm registers. */
@@ -382,7 +362,7 @@ static int _abb5zes3_rtc_read_alarm(struct device *dev,
 	if (ret) {
 		dev_err(dev, "%s: reading alarm section failed (%d)\n",
 			__func__, ret);
-		goto err;
+		return ret;
 	}
 
 	alarm_tm->tm_sec  = 0;
@@ -398,18 +378,13 @@ static int _abb5zes3_rtc_read_alarm(struct device *dev,
 	 */
 	ret = _abb5zes3_rtc_read_time(dev, &rtc_tm);
 	if (ret)
-		goto err;
+		return ret;
 
 	alarm_tm->tm_year = rtc_tm.tm_year;
 	alarm_tm->tm_mon = rtc_tm.tm_mon;
 
-	ret = rtc_tm_to_time(&rtc_tm, &rtc_secs);
-	if (ret)
-		goto err;
-
-	ret = rtc_tm_to_time(alarm_tm, &alarm_secs);
-	if (ret)
-		goto err;
+	rtc_secs = rtc_tm_to_time64(&rtc_tm);
+	alarm_secs = rtc_tm_to_time64(alarm_tm);
 
 	if (alarm_secs < rtc_secs) {
 		if (alarm_tm->tm_mon == 11) {
@@ -424,13 +399,12 @@ static int _abb5zes3_rtc_read_alarm(struct device *dev,
 	if (ret) {
 		dev_err(dev, "%s: reading ctrl reg failed (%d)\n",
 			__func__, ret);
-		goto err;
+		return ret;
 	}
 
 	alarm->enabled = !!(reg & ABB5ZES3_REG_CTRL1_AIE);
 
-err:
-	return ret;
+	return 0;
 }
 
 /*
@@ -447,12 +421,10 @@ static int abb5zes3_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alarm)
 	struct abb5zes3_rtc_data *data = dev_get_drvdata(dev);
 	int ret;
 
-	mutex_lock(&data->lock);
 	if (data->timer_alarm)
 		ret = _abb5zes3_rtc_read_timer(dev, alarm);
 	else
 		ret = _abb5zes3_rtc_read_alarm(dev, alarm);
-	mutex_unlock(&data->lock);
 
 	return ret;
 }
@@ -466,33 +438,25 @@ static int _abb5zes3_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alarm)
 {
 	struct abb5zes3_rtc_data *data = dev_get_drvdata(dev);
 	struct rtc_time *alarm_tm = &alarm->time;
-	unsigned long rtc_secs, alarm_secs;
 	u8 regs[ABB5ZES3_ALRM_SEC_LEN];
 	struct rtc_time rtc_tm;
 	int ret, enable = 1;
 
-	ret = _abb5zes3_rtc_read_time(dev, &rtc_tm);
-	if (ret)
-		goto err;
-
-	ret = rtc_tm_to_time(&rtc_tm, &rtc_secs);
-	if (ret)
-		goto err;
-
-	ret = rtc_tm_to_time(alarm_tm, &alarm_secs);
-	if (ret)
-		goto err;
-
-	/* If alarm time is before current time, disable the alarm */
-	if (!alarm->enabled || alarm_secs <= rtc_secs) {
+	if (!alarm->enabled) {
 		enable = 0;
 	} else {
+		unsigned long rtc_secs, alarm_secs;
+
 		/*
 		 * Chip only support alarms up to one month in the future. Let's
 		 * return an error if we get something after that limit.
 		 * Comparison is done by incrementing rtc_tm month field by one
 		 * and checking alarm value is still below.
 		 */
+		ret = _abb5zes3_rtc_read_time(dev, &rtc_tm);
+		if (ret)
+			return ret;
+
 		if (rtc_tm.tm_mon == 11) { /* handle year wrapping */
 			rtc_tm.tm_mon = 0;
 			rtc_tm.tm_year += 1;
@@ -500,15 +464,13 @@ static int _abb5zes3_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alarm)
 			rtc_tm.tm_mon += 1;
 		}
 
-		ret = rtc_tm_to_time(&rtc_tm, &rtc_secs);
-		if (ret)
-			goto err;
+		rtc_secs = rtc_tm_to_time64(&rtc_tm);
+		alarm_secs = rtc_tm_to_time64(alarm_tm);
 
 		if (alarm_secs > rtc_secs) {
-			dev_err(dev, "%s: alarm maximum is one month in the "
-				"future (%d)\n", __func__, ret);
-			ret = -EINVAL;
-			goto err;
+			dev_err(dev, "%s: alarm maximum is one month in the future (%d)\n",
+				__func__, ret);
+			return -EINVAL;
 		}
 	}
 
@@ -526,17 +488,14 @@ static int _abb5zes3_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alarm)
 	if (ret < 0) {
 		dev_err(dev, "%s: writing ALARM section failed (%d)\n",
 			__func__, ret);
-		goto err;
+		return ret;
 	}
 
 	/* Record currently configured alarm is not a timer */
 	data->timer_alarm = 0;
 
 	/* Enable or disable alarm interrupt generation */
-	ret = _abb5zes3_rtc_update_alarm(dev, enable);
-
-err:
-	return ret;
+	return _abb5zes3_rtc_update_alarm(dev, enable);
 }
 
 /*
@@ -557,7 +516,7 @@ static int _abb5zes3_rtc_set_timer(struct device *dev, struct rtc_wkalrm *alarm,
 				ABB5ZES3_TIMA_SEC_LEN);
 	if (ret < 0) {
 		dev_err(dev, "%s: writing timer section failed\n", __func__);
-		goto err;
+		return ret;
 	}
 
 	/* Configure Timer A as a watchdog timer */
@@ -570,10 +529,7 @@ static int _abb5zes3_rtc_set_timer(struct device *dev, struct rtc_wkalrm *alarm,
 	data->timer_alarm = 1;
 
 	/* Enable or disable timer interrupt generation */
-	ret = _abb5zes3_rtc_update_timer(dev, alarm->enabled);
-
-err:
-	return ret;
+	return _abb5zes3_rtc_update_timer(dev, alarm->enabled);
 }
 
 /*
@@ -590,31 +546,25 @@ static int abb5zes3_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alarm)
 	struct rtc_time rtc_tm;
 	int ret;
 
-	mutex_lock(&data->lock);
 	ret = _abb5zes3_rtc_read_time(dev, &rtc_tm);
 	if (ret)
-		goto err;
+		return ret;
 
-	ret = rtc_tm_to_time(&rtc_tm, &rtc_secs);
-	if (ret)
-		goto err;
-
-	ret = rtc_tm_to_time(alarm_tm, &alarm_secs);
-	if (ret)
-		goto err;
+	rtc_secs = rtc_tm_to_time64(&rtc_tm);
+	alarm_secs = rtc_tm_to_time64(alarm_tm);
 
 	/* Let's first disable both the alarm and the timer interrupts */
 	ret = _abb5zes3_rtc_update_alarm(dev, false);
 	if (ret < 0) {
 		dev_err(dev, "%s: unable to disable alarm (%d)\n", __func__,
 			ret);
-		goto err;
+		return ret;
 	}
 	ret = _abb5zes3_rtc_update_timer(dev, false);
 	if (ret < 0) {
 		dev_err(dev, "%s: unable to disable timer (%d)\n", __func__,
 			ret);
-		goto err;
+		return ret;
 	}
 
 	data->timer_alarm = 0;
@@ -629,9 +579,6 @@ static int abb5zes3_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alarm)
 	else
 		ret = _abb5zes3_rtc_set_alarm(dev, alarm);
 
- err:
-	mutex_unlock(&data->lock);
-
 	if (ret)
 		dev_err(dev, "%s: unable to configure alarm (%d)\n", __func__,
 			ret);
@@ -650,8 +597,7 @@ static inline int _abb5zes3_rtc_battery_low_irq_enable(struct regmap *regmap,
 
 /*
  * Check current RTC status and enable/disable what needs to be. Return 0 if
- * everything went ok and a negative value upon error. Note: this function
- * is called early during init and hence does need mutex protection.
+ * everything went ok and a negative value upon error.
  */
 static int abb5zes3_rtc_check_setup(struct device *dev)
 {
@@ -675,8 +621,9 @@ static int abb5zes3_rtc_check_setup(struct device *dev)
 		ABB5ZES3_REG_TIM_CLK_COF1 | ABB5ZES3_REG_TIM_CLK_COF2 |
 		ABB5ZES3_REG_TIM_CLK_TBM | ABB5ZES3_REG_TIM_CLK_TAM);
 	ret = regmap_update_bits(regmap, ABB5ZES3_REG_TIM_CLK, mask,
-		ABB5ZES3_REG_TIM_CLK_COF0 | ABB5ZES3_REG_TIM_CLK_COF1 |
-		ABB5ZES3_REG_TIM_CLK_COF2);
+				 ABB5ZES3_REG_TIM_CLK_COF0 |
+				 ABB5ZES3_REG_TIM_CLK_COF1 |
+				 ABB5ZES3_REG_TIM_CLK_COF2);
 	if (ret < 0) {
 		dev_err(dev, "%s: unable to initialize clkout register (%d)\n",
 			__func__, ret);
@@ -729,9 +676,9 @@ static int abb5zes3_rtc_check_setup(struct device *dev)
 	 * switchover flag but not battery low flag. The latter is checked
 	 * later below.
 	 */
-	mask = (ABB5ZES3_REG_CTRL3_PM0 | ABB5ZES3_REG_CTRL3_PM1 |
-		ABB5ZES3_REG_CTRL3_PM2 | ABB5ZES3_REG_CTRL3_BLIE |
-		ABB5ZES3_REG_CTRL3_BSIE| ABB5ZES3_REG_CTRL3_BSF);
+	mask = (ABB5ZES3_REG_CTRL3_PM0  | ABB5ZES3_REG_CTRL3_PM1 |
+		ABB5ZES3_REG_CTRL3_PM2  | ABB5ZES3_REG_CTRL3_BLIE |
+		ABB5ZES3_REG_CTRL3_BSIE | ABB5ZES3_REG_CTRL3_BSF);
 	ret = regmap_update_bits(regmap, ABB5ZES3_REG_CTRL3, mask, 0);
 	if (ret < 0) {
 		dev_err(dev, "%s: unable to initialize CTRL3 register (%d)\n",
@@ -748,10 +695,8 @@ static int abb5zes3_rtc_check_setup(struct device *dev)
 	}
 
 	if (reg & ABB5ZES3_REG_RTC_SC_OSC) {
-		dev_err(dev, "clock integrity not guaranteed. Osc. has stopped "
-			"or has been interrupted.\n");
-		dev_err(dev, "change battery (if not already done) and  "
-			"then set time to reset osc. failure flag.\n");
+		dev_err(dev, "clock integrity not guaranteed. Osc. has stopped or has been interrupted.\n");
+		dev_err(dev, "change battery (if not already done) and then set time to reset osc. failure flag.\n");
 	}
 
 	/*
@@ -769,13 +714,12 @@ static int abb5zes3_rtc_check_setup(struct device *dev)
 
 	data->battery_low = reg & ABB5ZES3_REG_CTRL3_BLF;
 	if (data->battery_low) {
-		dev_err(dev, "RTC battery is low; please, consider "
-			"changing it!\n");
+		dev_err(dev, "RTC battery is low; please, consider changing it!\n");
 
 		ret = _abb5zes3_rtc_battery_low_irq_enable(regmap, false);
 		if (ret)
-			dev_err(dev, "%s: disabling battery low interrupt "
-				"generation failed (%d)\n", __func__, ret);
+			dev_err(dev, "%s: disabling battery low interrupt generation failed (%d)\n",
+				__func__, ret);
 	}
 
 	return ret;
@@ -788,12 +732,10 @@ static int abb5zes3_rtc_alarm_irq_enable(struct device *dev,
 	int ret = 0;
 
 	if (rtc_data->irq) {
-		mutex_lock(&rtc_data->lock);
 		if (rtc_data->timer_alarm)
 			ret = _abb5zes3_rtc_update_timer(dev, enable);
 		else
 			ret = _abb5zes3_rtc_update_alarm(dev, enable);
-		mutex_unlock(&rtc_data->lock);
 	}
 
 	return ret;
@@ -885,49 +827,44 @@ static int abb5zes3_probe(struct i2c_client *client,
 
 	if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C |
 				     I2C_FUNC_SMBUS_BYTE_DATA |
-				     I2C_FUNC_SMBUS_I2C_BLOCK)) {
-		ret = -ENODEV;
-		goto err;
-	}
+				     I2C_FUNC_SMBUS_I2C_BLOCK))
+		return -ENODEV;
 
 	regmap = devm_regmap_init_i2c(client, &abb5zes3_rtc_regmap_config);
 	if (IS_ERR(regmap)) {
 		ret = PTR_ERR(regmap);
 		dev_err(dev, "%s: regmap allocation failed: %d\n",
 			__func__, ret);
-		goto err;
+		return ret;
 	}
 
 	ret = abb5zes3_i2c_validate_chip(regmap);
 	if (ret)
-		goto err;
+		return ret;
 
 	data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
-	if (!data) {
-		ret = -ENOMEM;
-		goto err;
-	}
+	if (!data)
+		return -ENOMEM;
 
-	mutex_init(&data->lock);
 	data->regmap = regmap;
 	dev_set_drvdata(dev, data);
 
 	ret = abb5zes3_rtc_check_setup(dev);
 	if (ret)
-		goto err;
+		return ret;
 
 	data->rtc = devm_rtc_allocate_device(dev);
 	ret = PTR_ERR_OR_ZERO(data->rtc);
 	if (ret) {
 		dev_err(dev, "%s: unable to allocate RTC device (%d)\n",
 			__func__, ret);
-		goto err;
+		return ret;
 	}
 
 	if (client->irq > 0) {
 		ret = devm_request_threaded_irq(dev, client->irq, NULL,
 						_abb5zes3_rtc_interrupt,
-						IRQF_SHARED|IRQF_ONESHOT,
+						IRQF_SHARED | IRQF_ONESHOT,
 						DRV_NAME, client);
 		if (!ret) {
 			device_init_wakeup(dev, true);
@@ -949,8 +886,8 @@ static int abb5zes3_probe(struct i2c_client *client,
 	if (!data->battery_low && data->irq) {
 		ret = _abb5zes3_rtc_battery_low_irq_enable(regmap, true);
 		if (ret) {
-			dev_err(dev, "%s: enabling battery low interrupt "
-				"generation failed (%d)\n", __func__, ret);
+			dev_err(dev, "%s: enabling battery low interrupt generation failed (%d)\n",
+				__func__, ret);
 			goto err;
 		}
 	}
@@ -958,7 +895,7 @@ static int abb5zes3_probe(struct i2c_client *client,
 	ret = rtc_register_device(data->rtc);
 
 err:
-	if (ret && data && data->irq)
+	if (ret && data->irq)
 		device_init_wakeup(dev, false);
 	return ret;
 }
diff --git a/drivers/rtc/rtc-ab3100.c b/drivers/rtc/rtc-ab3100.c
index 821ff52a2222..2ed6def90975 100644
--- a/drivers/rtc/rtc-ab3100.c
+++ b/drivers/rtc/rtc-ab3100.c
@@ -1,6 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) 2007-2009 ST-Ericsson AB
- * License terms: GNU General Public License (GPL) version 2
  * RTC clock driver for the AB3100 Analog Baseband Chip
  * Author: Linus Walleij <linus.walleij@stericsson.com>
  */
@@ -43,12 +43,12 @@
 /*
  * RTC clock functions and device struct declaration
  */
-static int ab3100_rtc_set_mmss(struct device *dev, time64_t secs)
+static int ab3100_rtc_set_time(struct device *dev, struct rtc_time *tm)
 {
 	u8 regs[] = {AB3100_TI0, AB3100_TI1, AB3100_TI2,
 		     AB3100_TI3, AB3100_TI4, AB3100_TI5};
 	unsigned char buf[6];
-	u64 hw_counter = secs * AB3100_RTC_CLOCK_RATE * 2;
+	u64 hw_counter = rtc_tm_to_time64(tm) * AB3100_RTC_CLOCK_RATE * 2;
 	int err = 0;
 	int i;
 
@@ -192,7 +192,7 @@ static int ab3100_rtc_irq_enable(struct device *dev, unsigned int enabled)
 
 static const struct rtc_class_ops ab3100_rtc_ops = {
 	.read_time	= ab3100_rtc_read_time,
-	.set_mmss64	= ab3100_rtc_set_mmss,
+	.set_time	= ab3100_rtc_set_time,
 	.read_alarm	= ab3100_rtc_read_alarm,
 	.set_alarm	= ab3100_rtc_set_alarm,
 	.alarm_irq_enable = ab3100_rtc_irq_enable,
@@ -228,15 +228,17 @@ static int __init ab3100_rtc_probe(struct platform_device *pdev)
 		/* Ignore any error on this write */
 	}
 
-	rtc = devm_rtc_device_register(&pdev->dev, "ab3100-rtc",
-					&ab3100_rtc_ops, THIS_MODULE);
-	if (IS_ERR(rtc)) {
-		err = PTR_ERR(rtc);
-		return err;
-	}
+	rtc = devm_rtc_allocate_device(&pdev->dev);
+	if (IS_ERR(rtc))
+		return PTR_ERR(rtc);
+
+	rtc->ops = &ab3100_rtc_ops;
+	/* 48bit counter at (AB3100_RTC_CLOCK_RATE * 2) */
+	rtc->range_max = U32_MAX;
+
 	platform_set_drvdata(pdev, rtc);
 
-	return 0;
+	return rtc_register_device(rtc);
 }
 
 static struct platform_driver ab3100_rtc_driver = {
diff --git a/drivers/rtc/rtc-abx80x.c b/drivers/rtc/rtc-abx80x.c
index 6ddcad642d1e..73830670a41f 100644
--- a/drivers/rtc/rtc-abx80x.c
+++ b/drivers/rtc/rtc-abx80x.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * A driver for the I2C members of the Abracon AB x8xx RTC family,
  * and compatible: AB 1805 and AB 0805
@@ -7,10 +8,6 @@
  * Author: Philippe De Muyter <phdm@macqel.be>
  * Author: Alexandre Belloni <alexandre.belloni@bootlin.com>
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
  */
 
 #include <linux/bcd.h>
@@ -404,7 +401,7 @@ static ssize_t autocalibration_store(struct device *dev,
 		return -EINVAL;
 	}
 
-	retval = abx80x_rtc_set_autocalibration(dev, autocalibration);
+	retval = abx80x_rtc_set_autocalibration(dev->parent, autocalibration);
 
 	return retval ? retval : count;
 }
@@ -414,7 +411,7 @@ static ssize_t autocalibration_show(struct device *dev,
 {
 	int autocalibration = 0;
 
-	autocalibration = abx80x_rtc_get_autocalibration(dev);
+	autocalibration = abx80x_rtc_get_autocalibration(dev->parent);
 	if (autocalibration < 0) {
 		dev_err(dev, "Failed to read RTC autocalibration\n");
 		sprintf(buf, "0\n");
@@ -430,7 +427,7 @@ static ssize_t oscillator_store(struct device *dev,
 				struct device_attribute *attr,
 				const char *buf, size_t count)
 {
-	struct i2c_client *client = to_i2c_client(dev);
+	struct i2c_client *client = to_i2c_client(dev->parent);
 	int retval, flags, rc_mode = 0;
 
 	if (strncmp(buf, "rc", 2) == 0) {
@@ -472,7 +469,7 @@ static ssize_t oscillator_show(struct device *dev,
 			       struct device_attribute *attr, char *buf)
 {
 	int rc_mode = 0;
-	struct i2c_client *client = to_i2c_client(dev);
+	struct i2c_client *client = to_i2c_client(dev->parent);
 
 	rc_mode = abx80x_is_rc_mode(client);
 
@@ -592,13 +589,6 @@ static int abx80x_dt_trickle_cfg(struct device_node *np)
 	return (trickle_cfg | i);
 }
 
-static void rtc_calib_remove_sysfs_group(void *_dev)
-{
-	struct device *dev = _dev;
-
-	sysfs_remove_group(&dev->kobj, &rtc_calib_attr_group);
-}
-
 #ifdef CONFIG_WATCHDOG
 
 static inline u8 timeout_bits(unsigned int timeout)
@@ -851,32 +841,14 @@ static int abx80x_probe(struct i2c_client *client,
 		}
 	}
 
-	/* Export sysfs entries */
-	err = sysfs_create_group(&(&client->dev)->kobj, &rtc_calib_attr_group);
+	err = rtc_add_group(priv->rtc, &rtc_calib_attr_group);
 	if (err) {
 		dev_err(&client->dev, "Failed to create sysfs group: %d\n",
 			err);
 		return err;
 	}
 
-	err = devm_add_action_or_reset(&client->dev,
-				       rtc_calib_remove_sysfs_group,
-				       &client->dev);
-	if (err) {
-		dev_err(&client->dev,
-			"Failed to add sysfs cleanup action: %d\n",
-			err);
-		return err;
-	}
-
-	err = rtc_register_device(priv->rtc);
-
-	return err;
-}
-
-static int abx80x_remove(struct i2c_client *client)
-{
-	return 0;
+	return rtc_register_device(priv->rtc);
 }
 
 static const struct i2c_device_id abx80x_id[] = {
@@ -899,7 +871,6 @@ static struct i2c_driver abx80x_driver = {
 		.name	= "rtc-abx80x",
 	},
 	.probe		= abx80x_probe,
-	.remove		= abx80x_remove,
 	.id_table	= abx80x_id,
 };
 
diff --git a/drivers/rtc/rtc-aspeed.c b/drivers/rtc/rtc-aspeed.c
new file mode 100644
index 000000000000..af3eb676d7c3
--- /dev/null
+++ b/drivers/rtc/rtc-aspeed.c
@@ -0,0 +1,136 @@
+// SPDX-License-Identifier: GPL-2.0+
+// Copyright 2015 IBM Corp.
+
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/rtc.h>
+#include <linux/io.h>
+
+struct aspeed_rtc {
+	struct rtc_device *rtc_dev;
+	void __iomem *base;
+};
+
+#define RTC_TIME	0x00
+#define RTC_YEAR	0x04
+#define RTC_CTRL	0x10
+
+#define RTC_UNLOCK	BIT(1)
+#define RTC_ENABLE	BIT(0)
+
+static int aspeed_rtc_read_time(struct device *dev, struct rtc_time *tm)
+{
+	struct aspeed_rtc *rtc = dev_get_drvdata(dev);
+	unsigned int cent, year;
+	u32 reg1, reg2;
+
+	if (!(readl(rtc->base + RTC_CTRL) & RTC_ENABLE)) {
+		dev_dbg(dev, "%s failing as rtc disabled\n", __func__);
+		return -EINVAL;
+	}
+
+	do {
+		reg2 = readl(rtc->base + RTC_YEAR);
+		reg1 = readl(rtc->base + RTC_TIME);
+	} while (reg2 != readl(rtc->base + RTC_YEAR));
+
+	tm->tm_mday = (reg1 >> 24) & 0x1f;
+	tm->tm_hour = (reg1 >> 16) & 0x1f;
+	tm->tm_min = (reg1 >> 8) & 0x3f;
+	tm->tm_sec = (reg1 >> 0) & 0x3f;
+
+	cent = (reg2 >> 16) & 0x1f;
+	year = (reg2 >> 8) & 0x7f;
+	tm->tm_mon = ((reg2 >>  0) & 0x0f) - 1;
+	tm->tm_year = year + (cent * 100) - 1900;
+
+	dev_dbg(dev, "%s %ptR", __func__, tm);
+
+	return 0;
+}
+
+static int aspeed_rtc_set_time(struct device *dev, struct rtc_time *tm)
+{
+	struct aspeed_rtc *rtc = dev_get_drvdata(dev);
+	u32 reg1, reg2, ctrl;
+	int year, cent;
+
+	cent = (tm->tm_year + 1900) / 100;
+	year = tm->tm_year % 100;
+
+	reg1 = (tm->tm_mday << 24) | (tm->tm_hour << 16) | (tm->tm_min << 8) |
+		tm->tm_sec;
+
+	reg2 = ((cent & 0x1f) << 16) | ((year & 0x7f) << 8) |
+		((tm->tm_mon + 1) & 0xf);
+
+	ctrl = readl(rtc->base + RTC_CTRL);
+	writel(ctrl | RTC_UNLOCK, rtc->base + RTC_CTRL);
+
+	writel(reg1, rtc->base + RTC_TIME);
+	writel(reg2, rtc->base + RTC_YEAR);
+
+	/* Re-lock and ensure enable is set now that a time is programmed */
+	writel(ctrl | RTC_ENABLE, rtc->base + RTC_CTRL);
+
+	return 0;
+}
+
+static const struct rtc_class_ops aspeed_rtc_ops = {
+	.read_time = aspeed_rtc_read_time,
+	.set_time = aspeed_rtc_set_time,
+};
+
+static int aspeed_rtc_probe(struct platform_device *pdev)
+{
+	struct aspeed_rtc *rtc;
+	struct resource *res;
+	int ret;
+
+	rtc = devm_kzalloc(&pdev->dev, sizeof(*rtc), GFP_KERNEL);
+	if (!rtc)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	rtc->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(rtc->base))
+		return PTR_ERR(rtc->base);
+
+	rtc->rtc_dev = devm_rtc_allocate_device(&pdev->dev);
+	if (IS_ERR(rtc->rtc_dev))
+		return PTR_ERR(rtc->rtc_dev);
+
+	platform_set_drvdata(pdev, rtc);
+
+	rtc->rtc_dev->ops = &aspeed_rtc_ops;
+	rtc->rtc_dev->range_min = RTC_TIMESTAMP_BEGIN_1900;
+	rtc->rtc_dev->range_max = 38814989399LL; /* 3199-12-31 23:59:59 */
+
+	ret = rtc_register_device(rtc->rtc_dev);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static const struct of_device_id aspeed_rtc_match[] = {
+	{ .compatible = "aspeed,ast2400-rtc", },
+	{ .compatible = "aspeed,ast2500-rtc", },
+	{ .compatible = "aspeed,ast2600-rtc", },
+	{}
+};
+MODULE_DEVICE_TABLE(of, aspeed_rtc_match);
+
+static struct platform_driver aspeed_rtc_driver = {
+	.driver = {
+		.name = "aspeed-rtc",
+		.of_match_table = of_match_ptr(aspeed_rtc_match),
+	},
+};
+
+module_platform_driver_probe(aspeed_rtc_driver, aspeed_rtc_probe);
+
+MODULE_DESCRIPTION("ASPEED RTC driver");
+MODULE_AUTHOR("Joel Stanley <joel@jms.id.au>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/rtc/rtc-at91sam9.c b/drivers/rtc/rtc-at91sam9.c
index 1d31c0ae6334..4daf3789b978 100644
--- a/drivers/rtc/rtc-at91sam9.c
+++ b/drivers/rtc/rtc-at91sam9.c
@@ -1,14 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  * "RTT as Real Time Clock" driver for AT91SAM9 SoC family
  *
  * (C) 2007 Michel Benoit
  *
  * Based on rtc-at91rm9200.c by Rick Bronson
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version
- * 2 of the License, or (at your option) any later version.
  */
 
 #include <linux/clk.h>
@@ -47,21 +43,21 @@
  * registers available, likewise usable for more than "RTC" support.
  */
 
-#define AT91_RTT_MR		0x00			/* Real-time Mode Register */
-#define AT91_RTT_RTPRES		(0xffff << 0)		/* Real-time Timer Prescaler Value */
-#define AT91_RTT_ALMIEN		(1 << 16)		/* Alarm Interrupt Enable */
-#define AT91_RTT_RTTINCIEN	(1 << 17)		/* Real Time Timer Increment Interrupt Enable */
-#define AT91_RTT_RTTRST		(1 << 18)		/* Real Time Timer Restart */
+#define AT91_RTT_MR		0x00		/* Real-time Mode Register */
+#define AT91_RTT_RTPRES		(0xffff << 0)	/* Timer Prescaler Value */
+#define AT91_RTT_ALMIEN		BIT(16)		/* Alarm Interrupt Enable */
+#define AT91_RTT_RTTINCIEN	BIT(17)		/* Increment Interrupt Enable */
+#define AT91_RTT_RTTRST		BIT(18)		/* Timer Restart */
 
-#define AT91_RTT_AR		0x04			/* Real-time Alarm Register */
-#define AT91_RTT_ALMV		(0xffffffff)		/* Alarm Value */
+#define AT91_RTT_AR		0x04		/* Real-time Alarm Register */
+#define AT91_RTT_ALMV		(0xffffffff)	/* Alarm Value */
 
-#define AT91_RTT_VR		0x08			/* Real-time Value Register */
-#define AT91_RTT_CRTV		(0xffffffff)		/* Current Real-time Value */
+#define AT91_RTT_VR		0x08		/* Real-time Value Register */
+#define AT91_RTT_CRTV		(0xffffffff)	/* Current Real-time Value */
 
-#define AT91_RTT_SR		0x0c			/* Real-time Status Register */
-#define AT91_RTT_ALMS		(1 << 0)		/* Real-time Alarm Status */
-#define AT91_RTT_RTTINC		(1 << 1)		/* Real-time Timer Increment */
+#define AT91_RTT_SR		0x0c		/* Real-time Status Register */
+#define AT91_RTT_ALMS		BIT(0)		/* Alarm Status */
+#define AT91_RTT_RTTINC		BIT(1)		/* Timer Increment */
 
 /*
  * We store ALARM_DISABLED in ALMV to record that no alarm is set.
@@ -69,14 +65,13 @@
  */
 #define ALARM_DISABLED	((u32)~0)
 
-
 struct sam9_rtc {
 	void __iomem		*rtt;
 	struct rtc_device	*rtcdev;
 	u32			imr;
 	struct regmap		*gpbr;
 	unsigned int		gpbr_offset;
-	int 			irq;
+	int			irq;
 	struct clk		*sclk;
 	bool			suspended;
 	unsigned long		events;
@@ -122,7 +117,7 @@ static int at91_rtc_readtime(struct device *dev, struct rtc_time *tm)
 	if (secs != secs2)
 		secs = rtt_readl(rtc, VR);
 
-	rtc_time_to_tm(offset + secs, tm);
+	rtc_time64_to_tm(offset + secs, tm);
 
 	dev_dbg(dev, "%s: %ptR\n", __func__, tm);
 
@@ -135,15 +130,12 @@ static int at91_rtc_readtime(struct device *dev, struct rtc_time *tm)
 static int at91_rtc_settime(struct device *dev, struct rtc_time *tm)
 {
 	struct sam9_rtc *rtc = dev_get_drvdata(dev);
-	int err;
 	u32 offset, alarm, mr;
 	unsigned long secs;
 
 	dev_dbg(dev, "%s: %ptR\n", __func__, tm);
 
-	err = rtc_tm_to_time(tm, &secs);
-	if (err != 0)
-		return err;
+	secs = rtc_tm_to_time64(tm);
 
 	mr = rtt_readl(rtc, MR);
 
@@ -193,7 +185,7 @@ static int at91_rtc_readalarm(struct device *dev, struct rtc_wkalrm *alrm)
 
 	memset(alrm, 0, sizeof(*alrm));
 	if (alarm != ALARM_DISABLED && offset != 0) {
-		rtc_time_to_tm(offset + alarm, tm);
+		rtc_time64_to_tm(offset + alarm, tm);
 
 		dev_dbg(dev, "%s: %ptR\n", __func__, tm);
 
@@ -211,11 +203,8 @@ static int at91_rtc_setalarm(struct device *dev, struct rtc_wkalrm *alrm)
 	unsigned long secs;
 	u32 offset;
 	u32 mr;
-	int err;
 
-	err = rtc_tm_to_time(tm, &secs);
-	if (err != 0)
-		return err;
+	secs = rtc_tm_to_time64(tm);
 
 	offset = gpbr_readl(rtc);
 	if (offset == 0) {
@@ -263,7 +252,7 @@ static int at91_rtc_proc(struct device *dev, struct seq_file *seq)
 	u32 mr = rtt_readl(rtc, MR);
 
 	seq_printf(seq, "update_IRQ\t: %s\n",
-			(mr & AT91_RTT_RTTINCIEN) ? "yes" : "no");
+		   (mr & AT91_RTT_RTTINCIEN) ? "yes" : "no");
 	return 0;
 }
 
@@ -299,7 +288,7 @@ static void at91_rtc_flush_events(struct sam9_rtc *rtc)
 	rtc->events = 0;
 
 	pr_debug("%s: num=%ld, events=0x%02lx\n", __func__,
-		rtc->events >> 8, rtc->events & 0x000000FF);
+		 rtc->events >> 8, rtc->events & 0x000000FF);
 }
 
 /*
@@ -340,13 +329,6 @@ static const struct rtc_class_ops at91_rtc_ops = {
 	.alarm_irq_enable = at91_rtc_alarm_irq_enable,
 };
 
-static const struct regmap_config gpbr_regmap_config = {
-	.name = "gpbr",
-	.reg_bits = 32,
-	.val_bits = 32,
-	.reg_stride = 4,
-};
-
 /*
  * Initialize and install RTC driver
  */
@@ -357,6 +339,7 @@ static int at91_rtc_probe(struct platform_device *pdev)
 	int		ret, irq;
 	u32		mr;
 	unsigned int	sclk_rate;
+	struct of_phandle_args args;
 
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0) {
@@ -382,34 +365,14 @@ static int at91_rtc_probe(struct platform_device *pdev)
 	if (IS_ERR(rtc->rtt))
 		return PTR_ERR(rtc->rtt);
 
-	if (!pdev->dev.of_node) {
-		/*
-		 * TODO: Remove this code chunk when removing non DT board
-		 * support. Remember to remove the gpbr_regmap_config
-		 * variable too.
-		 */
-		void __iomem *gpbr;
-
-		r = platform_get_resource(pdev, IORESOURCE_MEM, 1);
-		gpbr = devm_ioremap_resource(&pdev->dev, r);
-		if (IS_ERR(gpbr))
-			return PTR_ERR(gpbr);
-
-		rtc->gpbr = regmap_init_mmio(NULL, gpbr,
-					     &gpbr_regmap_config);
-	} else {
-		struct of_phandle_args args;
-
-		ret = of_parse_phandle_with_fixed_args(pdev->dev.of_node,
-						"atmel,rtt-rtc-time-reg", 1, 0,
-						&args);
-		if (ret)
-			return ret;
-
-		rtc->gpbr = syscon_node_to_regmap(args.np);
-		rtc->gpbr_offset = args.args[0];
-	}
+	ret = of_parse_phandle_with_fixed_args(pdev->dev.of_node,
+					       "atmel,rtt-rtc-time-reg", 1, 0,
+					       &args);
+	if (ret)
+		return ret;
 
+	rtc->gpbr = syscon_node_to_regmap(args.np);
+	rtc->gpbr_offset = args.args[0];
 	if (IS_ERR(rtc->gpbr)) {
 		dev_err(&pdev->dev, "failed to retrieve gpbr regmap, aborting.\n");
 		return -ENOMEM;
@@ -444,13 +407,15 @@ static int at91_rtc_probe(struct platform_device *pdev)
 	mr &= ~(AT91_RTT_ALMIEN | AT91_RTT_RTTINCIEN);
 	rtt_writel(rtc, MR, mr);
 
-	rtc->rtcdev = devm_rtc_device_register(&pdev->dev, pdev->name,
-					&at91_rtc_ops, THIS_MODULE);
+	rtc->rtcdev = devm_rtc_allocate_device(&pdev->dev);
 	if (IS_ERR(rtc->rtcdev)) {
 		ret = PTR_ERR(rtc->rtcdev);
 		goto err_clk;
 	}
 
+	rtc->rtcdev->ops = &at91_rtc_ops;
+	rtc->rtcdev->range_max = U32_MAX;
+
 	/* register irq handler after we know what name we'll use */
 	ret = devm_request_irq(&pdev->dev, rtc->irq, at91_rtc_interrupt,
 			       IRQF_SHARED | IRQF_COND_SUSPEND,
@@ -468,9 +433,9 @@ static int at91_rtc_probe(struct platform_device *pdev)
 
 	if (gpbr_readl(rtc) == 0)
 		dev_warn(&pdev->dev, "%s: SET TIME!\n",
-				dev_name(&rtc->rtcdev->dev));
+			 dev_name(&rtc->rtcdev->dev));
 
-	return 0;
+	return rtc_register_device(rtc->rtcdev);
 
 err_clk:
 	clk_disable_unprepare(rtc->sclk);
@@ -528,8 +493,9 @@ static int at91_rtc_suspend(struct device *dev)
 			/* don't let RTTINC cause wakeups */
 			if (mr & AT91_RTT_RTTINCIEN)
 				rtt_writel(rtc, MR, mr & ~AT91_RTT_RTTINCIEN);
-		} else
+		} else {
 			rtt_writel(rtc, MR, mr & ~rtc->imr);
+		}
 	}
 
 	return 0;
@@ -561,13 +527,11 @@ static int at91_rtc_resume(struct device *dev)
 
 static SIMPLE_DEV_PM_OPS(at91_rtc_pm_ops, at91_rtc_suspend, at91_rtc_resume);
 
-#ifdef CONFIG_OF
 static const struct of_device_id at91_rtc_dt_ids[] = {
 	{ .compatible = "atmel,at91sam9260-rtt" },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, at91_rtc_dt_ids);
-#endif
 
 static struct platform_driver at91_rtc_driver = {
 	.probe		= at91_rtc_probe,
diff --git a/drivers/rtc/rtc-brcmstb-waketimer.c b/drivers/rtc/rtc-brcmstb-waketimer.c
index f4010a75f2be..a193396a8140 100644
--- a/drivers/rtc/rtc-brcmstb-waketimer.c
+++ b/drivers/rtc/rtc-brcmstb-waketimer.c
@@ -132,7 +132,7 @@ static int brcmstb_waketmr_gettime(struct device *dev,
 
 	wktmr_read(timer, &now);
 
-	rtc_time_to_tm(now.sec, tm);
+	rtc_time64_to_tm(now.sec, tm);
 
 	return 0;
 }
diff --git a/drivers/rtc/rtc-coh901331.c b/drivers/rtc/rtc-coh901331.c
index 0b232c84f674..4ac850837153 100644
--- a/drivers/rtc/rtc-coh901331.c
+++ b/drivers/rtc/rtc-coh901331.c
@@ -1,6 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) 2007-2009 ST-Ericsson AB
- * License terms: GNU General Public License (GPL) version 2
  * Real Time Clock interface for ST-Ericsson AB COH 901 331 RTC.
  * Author: Linus Walleij <linus.walleij@stericsson.com>
  * Based on rtc-pl031.c by Deepak Saxena <dsaxena@plexity.net>
@@ -80,21 +80,22 @@ static int coh901331_read_time(struct device *dev, struct rtc_time *tm)
 
 	clk_enable(rtap->clk);
 	/* Check if the time is valid */
-	if (readl(rtap->virtbase + COH901331_VALID)) {
-		rtc_time_to_tm(readl(rtap->virtbase + COH901331_CUR_TIME), tm);
+	if (!readl(rtap->virtbase + COH901331_VALID)) {
 		clk_disable(rtap->clk);
-		return 0;
+		return -EINVAL;
 	}
+
+	rtc_time64_to_tm(readl(rtap->virtbase + COH901331_CUR_TIME), tm);
 	clk_disable(rtap->clk);
-	return -EINVAL;
+	return 0;
 }
 
-static int coh901331_set_mmss(struct device *dev, unsigned long secs)
+static int coh901331_set_time(struct device *dev, struct rtc_time *tm)
 {
 	struct coh901331_port *rtap = dev_get_drvdata(dev);
 
 	clk_enable(rtap->clk);
-	writel(secs, rtap->virtbase + COH901331_SET_TIME);
+	writel(rtc_tm_to_time64(tm), rtap->virtbase + COH901331_SET_TIME);
 	clk_disable(rtap->clk);
 
 	return 0;
@@ -105,7 +106,7 @@ static int coh901331_read_alarm(struct device *dev, struct rtc_wkalrm *alarm)
 	struct coh901331_port *rtap = dev_get_drvdata(dev);
 
 	clk_enable(rtap->clk);
-	rtc_time_to_tm(readl(rtap->virtbase + COH901331_ALARM), &alarm->time);
+	rtc_time64_to_tm(readl(rtap->virtbase + COH901331_ALARM), &alarm->time);
 	alarm->pending = readl(rtap->virtbase + COH901331_IRQ_EVENT) & 1U;
 	alarm->enabled = readl(rtap->virtbase + COH901331_IRQ_MASK) & 1U;
 	clk_disable(rtap->clk);
@@ -116,9 +117,8 @@ static int coh901331_read_alarm(struct device *dev, struct rtc_wkalrm *alarm)
 static int coh901331_set_alarm(struct device *dev, struct rtc_wkalrm *alarm)
 {
 	struct coh901331_port *rtap = dev_get_drvdata(dev);
-	unsigned long time;
+	unsigned long time =  rtc_tm_to_time64(&alarm->time);
 
-	rtc_tm_to_time(&alarm->time, &time);
 	clk_enable(rtap->clk);
 	writel(time, rtap->virtbase + COH901331_ALARM);
 	writel(alarm->enabled, rtap->virtbase + COH901331_IRQ_MASK);
@@ -143,7 +143,7 @@ static int coh901331_alarm_irq_enable(struct device *dev, unsigned int enabled)
 
 static const struct rtc_class_ops coh901331_ops = {
 	.read_time = coh901331_read_time,
-	.set_mmss = coh901331_set_mmss,
+	.set_time = coh901331_set_time,
 	.read_alarm = coh901331_read_alarm,
 	.set_alarm = coh901331_set_alarm,
 	.alarm_irq_enable = coh901331_alarm_irq_enable,
@@ -188,6 +188,13 @@ static int __init coh901331_probe(struct platform_device *pdev)
 		return ret;
 	}
 
+	rtap->rtc = devm_rtc_allocate_device(&pdev->dev);
+	if (IS_ERR(rtap->rtc))
+		return PTR_ERR(rtap->rtc);
+
+	rtap->rtc->ops = &coh901331_ops;
+	rtap->rtc->range_max = U32_MAX;
+
 	/* We enable/disable the clock only to assure it works */
 	ret = clk_prepare_enable(rtap->clk);
 	if (ret) {
@@ -197,12 +204,10 @@ static int __init coh901331_probe(struct platform_device *pdev)
 	clk_disable(rtap->clk);
 
 	platform_set_drvdata(pdev, rtap);
-	rtap->rtc = devm_rtc_device_register(&pdev->dev, "coh901331",
-					&coh901331_ops, THIS_MODULE);
-	if (IS_ERR(rtap->rtc)) {
-		ret = PTR_ERR(rtap->rtc);
+
+	ret = rtc_register_device(rtap->rtc);
+	if (ret)
 		goto out_no_rtc;
-	}
 
 	return 0;
 
diff --git a/drivers/rtc/rtc-da9063.c b/drivers/rtc/rtc-da9063.c
index 69b54e5556c0..15908d51b1cb 100644
--- a/drivers/rtc/rtc-da9063.c
+++ b/drivers/rtc/rtc-da9063.c
@@ -1,15 +1,7 @@
-/* rtc-da9063.c - Real time clock device driver for DA9063
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Real time clock device driver for DA9063
  * Copyright (C) 2013-2015  Dialog Semiconductor Ltd.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
- * of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #include <linux/delay.h>
@@ -247,8 +239,8 @@ static int da9063_rtc_read_time(struct device *dev, struct rtc_time *tm)
 
 	da9063_data_to_tm(data, tm, rtc);
 
-	rtc_tm_to_time(tm, &tm_secs);
-	rtc_tm_to_time(&rtc->alarm_time, &al_secs);
+	tm_secs = rtc_tm_to_time64(tm);
+	al_secs = rtc_tm_to_time64(&rtc->alarm_time);
 
 	/* handle the rtc synchronisation delay */
 	if (rtc->rtc_sync == true && al_secs - tm_secs == 1)
@@ -472,11 +464,14 @@ static int da9063_rtc_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, rtc);
 
-	rtc->rtc_dev = devm_rtc_device_register(&pdev->dev, DA9063_DRVNAME_RTC,
-					   &da9063_rtc_ops, THIS_MODULE);
+	rtc->rtc_dev = devm_rtc_allocate_device(&pdev->dev);
 	if (IS_ERR(rtc->rtc_dev))
 		return PTR_ERR(rtc->rtc_dev);
 
+	rtc->rtc_dev->ops = &da9063_rtc_ops;
+	rtc->rtc_dev->range_min = RTC_TIMESTAMP_BEGIN_2000;
+	rtc->rtc_dev->range_max = RTC_TIMESTAMP_END_2063;
+
 	da9063_data_to_tm(data, &rtc->alarm_time, rtc);
 	rtc->rtc_sync = false;
 
@@ -496,7 +491,7 @@ static int da9063_rtc_probe(struct platform_device *pdev)
 		dev_err(&pdev->dev, "Failed to request ALARM IRQ %d: %d\n",
 			irq_alarm, ret);
 
-	return ret;
+	return rtc_register_device(rtc->rtc_dev);
 }
 
 static struct platform_driver da9063_rtc_driver = {
diff --git a/drivers/rtc/rtc-digicolor.c b/drivers/rtc/rtc-digicolor.c
index b253bf1b3531..0aecc3f8e721 100644
--- a/drivers/rtc/rtc-digicolor.c
+++ b/drivers/rtc/rtc-digicolor.c
@@ -1,14 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  * Real Time Clock driver for Conexant Digicolor
  *
  * Copyright (C) 2015 Paradox Innovation Ltd.
  *
  * Author: Baruch Siach <baruch@tkos.co.il>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation; either version 2 of the License, or (at your
- * option) any later version.
  */
 
 #include <linux/io.h>
@@ -106,11 +102,11 @@ static int dc_rtc_read_time(struct device *dev, struct rtc_time *tm)
 	return 0;
 }
 
-static int dc_rtc_set_mmss(struct device *dev, unsigned long secs)
+static int dc_rtc_set_time(struct device *dev, struct rtc_time *tm)
 {
 	struct dc_rtc *rtc = dev_get_drvdata(dev);
 
-	return dc_rtc_write(rtc, secs);
+	return dc_rtc_write(rtc, rtc_tm_to_time64(tm));
 }
 
 static int dc_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alarm)
@@ -161,7 +157,7 @@ static int dc_rtc_alarm_irq_enable(struct device *dev, unsigned int enabled)
 
 static const struct rtc_class_ops dc_rtc_ops = {
 	.read_time		= dc_rtc_read_time,
-	.set_mmss		= dc_rtc_set_mmss,
+	.set_time		= dc_rtc_set_time,
 	.read_alarm		= dc_rtc_read_alarm,
 	.set_alarm		= dc_rtc_set_alarm,
 	.alarm_irq_enable	= dc_rtc_alarm_irq_enable,
@@ -192,6 +188,10 @@ static int __init dc_rtc_probe(struct platform_device *pdev)
 	if (IS_ERR(rtc->regs))
 		return PTR_ERR(rtc->regs);
 
+	rtc->rtc_dev = devm_rtc_allocate_device(&pdev->dev);
+	if (IS_ERR(rtc->rtc_dev))
+		return PTR_ERR(rtc->rtc_dev);
+
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0)
 		return irq;
@@ -200,12 +200,11 @@ static int __init dc_rtc_probe(struct platform_device *pdev)
 		return ret;
 
 	platform_set_drvdata(pdev, rtc);
-	rtc->rtc_dev = devm_rtc_device_register(&pdev->dev, pdev->name,
-						&dc_rtc_ops, THIS_MODULE);
-	if (IS_ERR(rtc->rtc_dev))
-		return PTR_ERR(rtc->rtc_dev);
 
-	return 0;
+	rtc->rtc_dev->ops = &dc_rtc_ops;
+	rtc->rtc_dev->range_max = U32_MAX;
+
+	return rtc_register_device(rtc->rtc_dev);
 }
 
 static const struct of_device_id dc_dt_ids[] = {
diff --git a/drivers/rtc/rtc-dm355evm.c b/drivers/rtc/rtc-dm355evm.c
index 97d8259b9494..cd947a20843b 100644
--- a/drivers/rtc/rtc-dm355evm.c
+++ b/drivers/rtc/rtc-dm355evm.c
@@ -1,12 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  * rtc-dm355evm.c - access battery-backed counter in MSP430 firmware
  *
  * Copyright (c) 2008 by David Brownell
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version
- * 2 of the License, or (at your option) any later version.
  */
 #include <linux/kernel.h>
 #include <linux/init.h>
@@ -78,7 +74,7 @@ static int dm355evm_rtc_read_time(struct device *dev, struct rtc_time *tm)
 
 	dev_dbg(dev, "read timestamp %08x\n", time.value);
 
-	rtc_time_to_tm(le32_to_cpu(time.value), tm);
+	rtc_time64_to_tm(le32_to_cpu(time.value), tm);
 	return 0;
 }
 
@@ -88,7 +84,7 @@ static int dm355evm_rtc_set_time(struct device *dev, struct rtc_time *tm)
 	unsigned long	value;
 	int		status;
 
-	rtc_tm_to_time(tm, &value);
+	value = rtc_tm_to_time64(tm);
 	time.value = cpu_to_le32(value);
 
 	dev_dbg(dev, "write timestamp %08x\n", time.value);
@@ -127,16 +123,16 @@ static int dm355evm_rtc_probe(struct platform_device *pdev)
 {
 	struct rtc_device *rtc;
 
-	rtc = devm_rtc_device_register(&pdev->dev, pdev->name,
-					&dm355evm_rtc_ops, THIS_MODULE);
-	if (IS_ERR(rtc)) {
-		dev_err(&pdev->dev, "can't register RTC device, err %ld\n",
-			PTR_ERR(rtc));
+	rtc = devm_rtc_allocate_device(&pdev->dev);
+	if (IS_ERR(rtc))
 		return PTR_ERR(rtc);
-	}
+
 	platform_set_drvdata(pdev, rtc);
 
-	return 0;
+	rtc->ops = &dm355evm_rtc_ops;
+	rtc->range_max = U32_MAX;
+
+	return rtc_register_device(rtc);
 }
 
 /*
diff --git a/drivers/rtc/rtc-ds1672.c b/drivers/rtc/rtc-ds1672.c
index b1ebca099b0d..e9e8d02743ee 100644
--- a/drivers/rtc/rtc-ds1672.c
+++ b/drivers/rtc/rtc-ds1672.c
@@ -1,12 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * An rtc/i2c driver for the Dallas DS1672
  * Copyright 2005-06 Tower Technologies
  *
  * Author: Alessandro Zummo <a.zummo@towertech.it>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include <linux/i2c.h>
@@ -21,17 +18,16 @@
 
 #define DS1672_REG_CONTROL_EOSC	0x80
 
-static struct i2c_driver ds1672_driver;
-
 /*
  * In the routines that deal directly with the ds1672 hardware, we use
  * rtc_time -- month 0-11, hour 0-23, yr = calendar year-epoch
- * Epoch is initialized as 2000. Time is set to UTC.
+ * Time is set to UTC.
  */
-static int ds1672_get_datetime(struct i2c_client *client, struct rtc_time *tm)
+static int ds1672_read_time(struct device *dev, struct rtc_time *tm)
 {
+	struct i2c_client *client = to_i2c_client(dev);
 	unsigned long time;
-	unsigned char addr = DS1672_REG_CNT_BASE;
+	unsigned char addr = DS1672_REG_CONTROL;
 	unsigned char buf[4];
 
 	struct i2c_msg msgs[] = {
@@ -43,11 +39,25 @@ static int ds1672_get_datetime(struct i2c_client *client, struct rtc_time *tm)
 		{/* read date */
 			.addr = client->addr,
 			.flags = I2C_M_RD,
-			.len = 4,
+			.len = 1,
 			.buf = buf
 		},
 	};
 
+	/* read control register */
+	if ((i2c_transfer(client->adapter, &msgs[0], 2)) != 2) {
+		dev_warn(&client->dev, "Unable to read the control register\n");
+		return -EIO;
+	}
+
+	if (buf[0] & DS1672_REG_CONTROL_EOSC) {
+		dev_warn(&client->dev, "Oscillator not enabled. Set time to enable.\n");
+		return -EINVAL;
+	}
+
+	addr = DS1672_REG_CNT_BASE;
+	msgs[1].len = 4;
+
 	/* read date registers */
 	if ((i2c_transfer(client->adapter, &msgs[0], 2)) != 2) {
 		dev_err(&client->dev, "%s: read error\n", __func__);
@@ -61,20 +71,19 @@ static int ds1672_get_datetime(struct i2c_client *client, struct rtc_time *tm)
 	time = ((unsigned long)buf[3] << 24) | (buf[2] << 16) |
 	       (buf[1] << 8) | buf[0];
 
-	rtc_time_to_tm(time, tm);
+	rtc_time64_to_tm(time, tm);
 
-	dev_dbg(&client->dev, "%s: tm is secs=%d, mins=%d, hours=%d, "
-		"mday=%d, mon=%d, year=%d, wday=%d\n",
-		__func__, tm->tm_sec, tm->tm_min, tm->tm_hour,
-		tm->tm_mday, tm->tm_mon, tm->tm_year, tm->tm_wday);
+	dev_dbg(&client->dev, "%s: tm is %ptR\n", __func__, tm);
 
 	return 0;
 }
 
-static int ds1672_set_mmss(struct i2c_client *client, unsigned long secs)
+static int ds1672_set_time(struct device *dev, struct rtc_time *tm)
 {
+	struct i2c_client *client = to_i2c_client(dev);
 	int xfer;
 	unsigned char buf[6];
+	unsigned long secs = rtc_tm_to_time64(tm);
 
 	buf[0] = DS1672_REG_CNT_BASE;
 	buf[1] = secs & 0x000000FF;
@@ -92,71 +101,15 @@ static int ds1672_set_mmss(struct i2c_client *client, unsigned long secs)
 	return 0;
 }
 
-static int ds1672_rtc_read_time(struct device *dev, struct rtc_time *tm)
-{
-	return ds1672_get_datetime(to_i2c_client(dev), tm);
-}
-
-static int ds1672_rtc_set_mmss(struct device *dev, unsigned long secs)
-{
-	return ds1672_set_mmss(to_i2c_client(dev), secs);
-}
-
-static int ds1672_get_control(struct i2c_client *client, u8 *status)
-{
-	unsigned char addr = DS1672_REG_CONTROL;
-
-	struct i2c_msg msgs[] = {
-		{/* setup read ptr */
-			.addr = client->addr,
-			.len = 1,
-			.buf = &addr
-		},
-		{/* read control */
-			.addr = client->addr,
-			.flags = I2C_M_RD,
-			.len = 1,
-			.buf = status
-		},
-	};
-
-	/* read control register */
-	if ((i2c_transfer(client->adapter, &msgs[0], 2)) != 2) {
-		dev_err(&client->dev, "%s: read error\n", __func__);
-		return -EIO;
-	}
-
-	return 0;
-}
-
-/* following are the sysfs callback functions */
-static ssize_t show_control(struct device *dev, struct device_attribute *attr,
-			    char *buf)
-{
-	struct i2c_client *client = to_i2c_client(dev);
-	u8 control;
-	int err;
-
-	err = ds1672_get_control(client, &control);
-	if (err)
-		return err;
-
-	return sprintf(buf, "%s\n", (control & DS1672_REG_CONTROL_EOSC)
-		       ? "disabled" : "enabled");
-}
-
-static DEVICE_ATTR(control, S_IRUGO, show_control, NULL);
-
 static const struct rtc_class_ops ds1672_rtc_ops = {
-	.read_time = ds1672_rtc_read_time,
-	.set_mmss = ds1672_rtc_set_mmss,
+	.read_time = ds1672_read_time,
+	.set_time = ds1672_set_time,
 };
 
 static int ds1672_probe(struct i2c_client *client,
 			const struct i2c_device_id *id)
 {
 	int err = 0;
-	u8 control;
 	struct rtc_device *rtc;
 
 	dev_dbg(&client->dev, "%s\n", __func__);
@@ -164,29 +117,21 @@ static int ds1672_probe(struct i2c_client *client,
 	if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C))
 		return -ENODEV;
 
-	rtc = devm_rtc_device_register(&client->dev, ds1672_driver.driver.name,
-				  &ds1672_rtc_ops, THIS_MODULE);
-
+	rtc = devm_rtc_allocate_device(&client->dev);
 	if (IS_ERR(rtc))
 		return PTR_ERR(rtc);
 
-	i2c_set_clientdata(client, rtc);
+	rtc->ops = &ds1672_rtc_ops;
+	rtc->range_max = U32_MAX;
 
-	/* read control register */
-	err = ds1672_get_control(client, &control);
-	if (err) {
-		dev_warn(&client->dev, "Unable to read the control register\n");
-	}
+	err = rtc_register_device(rtc);
+	if (err)
+		return err;
 
-	if (control & DS1672_REG_CONTROL_EOSC)
-		dev_warn(&client->dev, "Oscillator not enabled. "
-			 "Set time to enable.\n");
+	if (IS_ERR(rtc))
+		return PTR_ERR(rtc);
 
-	/* Register sysfs hooks */
-	err = device_create_file(&client->dev, &dev_attr_control);
-	if (err)
-		dev_err(&client->dev, "Unable to create sysfs entry: %s\n",
-			dev_attr_control.attr.name);
+	i2c_set_clientdata(client, rtc);
 
 	return 0;
 }
diff --git a/drivers/rtc/rtc-ds1685.c b/drivers/rtc/rtc-ds1685.c
index 2710f2594c42..5f4328524183 100644
--- a/drivers/rtc/rtc-ds1685.c
+++ b/drivers/rtc/rtc-ds1685.c
@@ -192,42 +192,6 @@ ds1685_rtc_end_data_access(struct ds1685_priv *rtc)
 }
 
 /**
- * ds1685_rtc_begin_ctrl_access - prepare the rtc for ctrl access.
- * @rtc: pointer to the ds1685 rtc structure.
- * @flags: irq flags variable for spin_lock_irqsave.
- *
- * This takes several steps to prepare the rtc for access to read just the
- * control registers:
- *  - Sets a spinlock on the rtc IRQ.
- *  - Switches the rtc to bank 1.  This allows access to the two extended
- *    control registers.
- *
- * Only use this where you are certain another lock will not be held.
- */
-static inline void
-ds1685_rtc_begin_ctrl_access(struct ds1685_priv *rtc, unsigned long *flags)
-{
-	spin_lock_irqsave(&rtc->lock, *flags);
-	ds1685_rtc_switch_to_bank1(rtc);
-}
-
-/**
- * ds1685_rtc_end_ctrl_access - end ctrl access on the rtc.
- * @rtc: pointer to the ds1685 rtc structure.
- * @flags: irq flags variable for spin_unlock_irqrestore.
- *
- * This ends what was started by ds1685_rtc_begin_ctrl_access:
- *  - Switches the rtc back to bank 0.
- *  - Unsets the spinlock on the rtc IRQ.
- */
-static inline void
-ds1685_rtc_end_ctrl_access(struct ds1685_priv *rtc, unsigned long flags)
-{
-	ds1685_rtc_switch_to_bank0(rtc);
-	spin_unlock_irqrestore(&rtc->lock, flags);
-}
-
-/**
  * ds1685_rtc_get_ssn - retrieve the silicon serial number.
  * @rtc: pointer to the ds1685 rtc structure.
  * @ssn: u8 array to hold the bits of the silicon serial number.
@@ -546,10 +510,6 @@ static int
 ds1685_rtc_alarm_irq_enable(struct device *dev, unsigned int enabled)
 {
 	struct ds1685_priv *rtc = dev_get_drvdata(dev);
-	unsigned long flags = 0;
-
-	/* Enable/disable the Alarm IRQ-Enable flag. */
-	spin_lock_irqsave(&rtc->lock, flags);
 
 	/* Flip the requisite interrupt-enable bit. */
 	if (enabled)
@@ -561,7 +521,6 @@ ds1685_rtc_alarm_irq_enable(struct device *dev, unsigned int enabled)
 
 	/* Read Control C to clear all the flag bits. */
 	rtc->read(rtc, RTC_CTRL_C);
-	spin_unlock_irqrestore(&rtc->lock, flags);
 
 	return 0;
 }
@@ -569,98 +528,18 @@ ds1685_rtc_alarm_irq_enable(struct device *dev, unsigned int enabled)
 
 
 /* ----------------------------------------------------------------------- */
-/* IRQ handler & workqueue. */
-
-/**
- * ds1685_rtc_irq_handler - IRQ handler.
- * @irq: IRQ number.
- * @dev_id: platform device pointer.
- */
-static irqreturn_t
-ds1685_rtc_irq_handler(int irq, void *dev_id)
-{
-	struct platform_device *pdev = dev_id;
-	struct ds1685_priv *rtc = platform_get_drvdata(pdev);
-	u8 ctrlb, ctrlc;
-	unsigned long events = 0;
-	u8 num_irqs = 0;
-
-	/* Abort early if the device isn't ready yet (i.e., DEBUG_SHIRQ). */
-	if (unlikely(!rtc))
-		return IRQ_HANDLED;
-
-	/* Ctrlb holds the interrupt-enable bits and ctrlc the flag bits. */
-	spin_lock(&rtc->lock);
-	ctrlb = rtc->read(rtc, RTC_CTRL_B);
-	ctrlc = rtc->read(rtc, RTC_CTRL_C);
-
-	/* Is the IRQF bit set? */
-	if (likely(ctrlc & RTC_CTRL_C_IRQF)) {
-		/*
-		 * We need to determine if it was one of the standard
-		 * events: PF, AF, or UF.  If so, we handle them and
-		 * update the RTC core.
-		 */
-		if (likely(ctrlc & RTC_CTRL_B_PAU_MASK)) {
-			events = RTC_IRQF;
-
-			/* Check for a periodic interrupt. */
-			if ((ctrlb & RTC_CTRL_B_PIE) &&
-			    (ctrlc & RTC_CTRL_C_PF)) {
-				events |= RTC_PF;
-				num_irqs++;
-			}
-
-			/* Check for an alarm interrupt. */
-			if ((ctrlb & RTC_CTRL_B_AIE) &&
-			    (ctrlc & RTC_CTRL_C_AF)) {
-				events |= RTC_AF;
-				num_irqs++;
-			}
-
-			/* Check for an update interrupt. */
-			if ((ctrlb & RTC_CTRL_B_UIE) &&
-			    (ctrlc & RTC_CTRL_C_UF)) {
-				events |= RTC_UF;
-				num_irqs++;
-			}
-
-			rtc_update_irq(rtc->dev, num_irqs, events);
-		} else {
-			/*
-			 * One of the "extended" interrupts was received that
-			 * is not recognized by the RTC core.  These need to
-			 * be handled in task context as they can call other
-			 * functions and the time spent in irq context needs
-			 * to be minimized.  Schedule them into a workqueue
-			 * and inform the RTC core that the IRQs were handled.
-			 */
-			spin_unlock(&rtc->lock);
-			schedule_work(&rtc->work);
-			rtc_update_irq(rtc->dev, 0, 0);
-			return IRQ_HANDLED;
-		}
-	}
-	spin_unlock(&rtc->lock);
-
-	return events ? IRQ_HANDLED : IRQ_NONE;
-}
+/* IRQ handler */
 
 /**
- * ds1685_rtc_work_queue - work queue handler.
- * @work: work_struct containing data to work on in task context.
+ * ds1685_rtc_extended_irq - take care of extended interrupts
+ * @rtc: pointer to the ds1685 rtc structure.
+ * @pdev: platform device pointer.
  */
 static void
-ds1685_rtc_work_queue(struct work_struct *work)
+ds1685_rtc_extended_irq(struct ds1685_priv *rtc, struct platform_device *pdev)
 {
-	struct ds1685_priv *rtc = container_of(work,
-					       struct ds1685_priv, work);
-	struct platform_device *pdev = to_platform_device(&rtc->dev->dev);
-	struct mutex *rtc_mutex = &rtc->dev->ops_lock;
 	u8 ctrl4a, ctrl4b;
 
-	mutex_lock(rtc_mutex);
-
 	ds1685_rtc_switch_to_bank1(rtc);
 	ctrl4a = rtc->read(rtc, RTC_EXT_CTRL_4A);
 	ctrl4b = rtc->read(rtc, RTC_EXT_CTRL_4B);
@@ -739,8 +618,76 @@ ds1685_rtc_work_queue(struct work_struct *work)
 				 "RAM-Clear IRQ just occurred!\n");
 	}
 	ds1685_rtc_switch_to_bank0(rtc);
+}
+
+/**
+ * ds1685_rtc_irq_handler - IRQ handler.
+ * @irq: IRQ number.
+ * @dev_id: platform device pointer.
+ */
+static irqreturn_t
+ds1685_rtc_irq_handler(int irq, void *dev_id)
+{
+	struct platform_device *pdev = dev_id;
+	struct ds1685_priv *rtc = platform_get_drvdata(pdev);
+	struct mutex *rtc_mutex;
+	u8 ctrlb, ctrlc;
+	unsigned long events = 0;
+	u8 num_irqs = 0;
+
+	/* Abort early if the device isn't ready yet (i.e., DEBUG_SHIRQ). */
+	if (unlikely(!rtc))
+		return IRQ_HANDLED;
+
+	rtc_mutex = &rtc->dev->ops_lock;
+	mutex_lock(rtc_mutex);
 
+	/* Ctrlb holds the interrupt-enable bits and ctrlc the flag bits. */
+	ctrlb = rtc->read(rtc, RTC_CTRL_B);
+	ctrlc = rtc->read(rtc, RTC_CTRL_C);
+
+	/* Is the IRQF bit set? */
+	if (likely(ctrlc & RTC_CTRL_C_IRQF)) {
+		/*
+		 * We need to determine if it was one of the standard
+		 * events: PF, AF, or UF.  If so, we handle them and
+		 * update the RTC core.
+		 */
+		if (likely(ctrlc & RTC_CTRL_B_PAU_MASK)) {
+			events = RTC_IRQF;
+
+			/* Check for a periodic interrupt. */
+			if ((ctrlb & RTC_CTRL_B_PIE) &&
+			    (ctrlc & RTC_CTRL_C_PF)) {
+				events |= RTC_PF;
+				num_irqs++;
+			}
+
+			/* Check for an alarm interrupt. */
+			if ((ctrlb & RTC_CTRL_B_AIE) &&
+			    (ctrlc & RTC_CTRL_C_AF)) {
+				events |= RTC_AF;
+				num_irqs++;
+			}
+
+			/* Check for an update interrupt. */
+			if ((ctrlb & RTC_CTRL_B_UIE) &&
+			    (ctrlc & RTC_CTRL_C_UF)) {
+				events |= RTC_UF;
+				num_irqs++;
+			}
+		} else {
+			/*
+			 * One of the "extended" interrupts was received that
+			 * is not recognized by the RTC core.
+			 */
+			ds1685_rtc_extended_irq(rtc, pdev);
+		}
+	}
+	rtc_update_irq(rtc->dev, num_irqs, events);
 	mutex_unlock(rtc_mutex);
+
+	return events ? IRQ_HANDLED : IRQ_NONE;
 }
 /* ----------------------------------------------------------------------- */
 
@@ -869,11 +816,15 @@ static int ds1685_nvram_read(void *priv, unsigned int pos, void *val,
 			     size_t size)
 {
 	struct ds1685_priv *rtc = priv;
+	struct mutex *rtc_mutex = &rtc->dev->ops_lock;
 	ssize_t count;
-	unsigned long flags = 0;
 	u8 *buf = val;
+	int err;
+
+	err = mutex_lock_interruptible(rtc_mutex);
+	if (err)
+		return err;
 
-	spin_lock_irqsave(&rtc->lock, flags);
 	ds1685_rtc_switch_to_bank0(rtc);
 
 	/* Read NVRAM in time and bank0 registers. */
@@ -923,7 +874,7 @@ static int ds1685_nvram_read(void *priv, unsigned int pos, void *val,
 		ds1685_rtc_switch_to_bank0(rtc);
 	}
 #endif /* !CONFIG_RTC_DRV_DS1689 */
-	spin_unlock_irqrestore(&rtc->lock, flags);
+	mutex_unlock(rtc_mutex);
 
 	return 0;
 }
@@ -932,11 +883,15 @@ static int ds1685_nvram_write(void *priv, unsigned int pos, void *val,
 			      size_t size)
 {
 	struct ds1685_priv *rtc = priv;
+	struct mutex *rtc_mutex = &rtc->dev->ops_lock;
 	ssize_t count;
-	unsigned long flags = 0;
 	u8 *buf = val;
+	int err;
+
+	err = mutex_lock_interruptible(rtc_mutex);
+	if (err)
+		return err;
 
-	spin_lock_irqsave(&rtc->lock, flags);
 	ds1685_rtc_switch_to_bank0(rtc);
 
 	/* Write NVRAM in time and bank0 registers. */
@@ -986,7 +941,7 @@ static int ds1685_nvram_write(void *priv, unsigned int pos, void *val,
 		ds1685_rtc_switch_to_bank0(rtc);
 	}
 #endif /* !CONFIG_RTC_DRV_DS1689 */
-	spin_unlock_irqrestore(&rtc->lock, flags);
+	mutex_unlock(rtc_mutex);
 
 	return 0;
 }
@@ -1004,7 +959,7 @@ static ssize_t
 ds1685_rtc_sysfs_battery_show(struct device *dev,
 			      struct device_attribute *attr, char *buf)
 {
-	struct ds1685_priv *rtc = dev_get_drvdata(dev);
+	struct ds1685_priv *rtc = dev_get_drvdata(dev->parent);
 	u8 ctrld;
 
 	ctrld = rtc->read(rtc, RTC_CTRL_D);
@@ -1024,7 +979,7 @@ static ssize_t
 ds1685_rtc_sysfs_auxbatt_show(struct device *dev,
 			      struct device_attribute *attr, char *buf)
 {
-	struct ds1685_priv *rtc = dev_get_drvdata(dev);
+	struct ds1685_priv *rtc = dev_get_drvdata(dev->parent);
 	u8 ctrl4a;
 
 	ds1685_rtc_switch_to_bank1(rtc);
@@ -1046,7 +1001,7 @@ static ssize_t
 ds1685_rtc_sysfs_serial_show(struct device *dev,
 			     struct device_attribute *attr, char *buf)
 {
-	struct ds1685_priv *rtc = dev_get_drvdata(dev);
+	struct ds1685_priv *rtc = dev_get_drvdata(dev->parent);
 	u8 ssn[8];
 
 	ds1685_rtc_switch_to_bank1(rtc);
@@ -1177,9 +1132,7 @@ ds1685_rtc_probe(struct platform_device *pdev)
 	if (pdata->plat_post_ram_clear)
 		rtc->post_ram_clear = pdata->plat_post_ram_clear;
 
-	/* Init the spinlock, workqueue, & set the driver data. */
-	spin_lock_init(&rtc->lock);
-	INIT_WORK(&rtc->work, ds1685_rtc_work_queue);
+	/* set the driver data. */
 	platform_set_drvdata(pdev, rtc);
 
 	/* Turn the oscillator on if is not already on (DV1 = 1). */
@@ -1335,22 +1288,23 @@ ds1685_rtc_probe(struct platform_device *pdev)
 	 */
 	if (!pdata->no_irq) {
 		ret = platform_get_irq(pdev, 0);
-		if (ret > 0) {
-			rtc->irq_num = ret;
-
-			/* Request an IRQ. */
-			ret = devm_request_irq(&pdev->dev, rtc->irq_num,
-					       ds1685_rtc_irq_handler,
-					       IRQF_SHARED, pdev->name, pdev);
-
-			/* Check to see if something came back. */
-			if (unlikely(ret)) {
-				dev_warn(&pdev->dev,
-					 "RTC interrupt not available\n");
-				rtc->irq_num = 0;
-			}
-		} else
+		if (ret <= 0)
 			return ret;
+
+		rtc->irq_num = ret;
+
+		/* Request an IRQ. */
+		ret = devm_request_threaded_irq(&pdev->dev, rtc->irq_num,
+				       NULL, ds1685_rtc_irq_handler,
+				       IRQF_SHARED | IRQF_ONESHOT,
+				       pdev->name, pdev);
+
+		/* Check to see if something came back. */
+		if (unlikely(ret)) {
+			dev_warn(&pdev->dev,
+				 "RTC interrupt not available\n");
+			rtc->irq_num = 0;
+		}
 	}
 	rtc->no_irq = pdata->no_irq;
 
@@ -1397,8 +1351,6 @@ ds1685_rtc_remove(struct platform_device *pdev)
 		   (rtc->read(rtc, RTC_EXT_CTRL_4A) &
 		    ~(RTC_CTRL_4A_RWK_MASK)));
 
-	cancel_work_sync(&rtc->work);
-
 	return 0;
 }
 
diff --git a/drivers/rtc/rtc-ds2404.c b/drivers/rtc/rtc-ds2404.c
index b886b6a5c178..1e9f429ada64 100644
--- a/drivers/rtc/rtc-ds2404.c
+++ b/drivers/rtc/rtc-ds2404.c
@@ -1,11 +1,5 @@
-/*
- * Copyright (C) 2012 Sven Schnelle <svens@stackframe.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- */
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2012 Sven Schnelle <svens@stackframe.org>
 
 #include <linux/platform_device.h>
 #include <linux/module.h>
@@ -29,14 +23,6 @@
 #define DS2404_COPY_SCRATCHPAD_CMD 0x55
 #define DS2404_READ_MEMORY_CMD 0xf0
 
-struct ds2404;
-
-struct ds2404_chip_ops {
-	int (*map_io)(struct ds2404 *chip, struct platform_device *pdev,
-		      struct ds2404_platform_data *pdata);
-	void (*unmap_io)(struct ds2404 *chip);
-};
-
 #define DS2404_RST	0
 #define DS2404_CLK	1
 #define DS2404_DQ	2
@@ -48,7 +34,6 @@ struct ds2404_gpio {
 
 struct ds2404 {
 	struct ds2404_gpio *gpio;
-	const struct ds2404_chip_ops *ops;
 	struct rtc_device *rtc;
 };
 
@@ -87,7 +72,7 @@ err_request:
 	return err;
 }
 
-static void ds2404_gpio_unmap(struct ds2404 *chip)
+static void ds2404_gpio_unmap(void *data)
 {
 	int i;
 
@@ -95,11 +80,6 @@ static void ds2404_gpio_unmap(struct ds2404 *chip)
 		gpio_free(ds2404_gpio[i].gpio);
 }
 
-static const struct ds2404_chip_ops ds2404_gpio_ops = {
-	.map_io		= ds2404_gpio_map,
-	.unmap_io	= ds2404_gpio_unmap,
-};
-
 static void ds2404_reset(struct device *dev)
 {
 	gpio_set_value(ds2404_gpio[DS2404_RST].gpio, 0);
@@ -206,20 +186,20 @@ static int ds2404_read_time(struct device *dev, struct rtc_time *dt)
 	ds2404_read_memory(dev, 0x203, 4, (u8 *)&time);
 	time = le32_to_cpu(time);
 
-	rtc_time_to_tm(time, dt);
+	rtc_time64_to_tm(time, dt);
 	return 0;
 }
 
-static int ds2404_set_mmss(struct device *dev, unsigned long secs)
+static int ds2404_set_time(struct device *dev, struct rtc_time *dt)
 {
-	u32 time = cpu_to_le32(secs);
+	u32 time = cpu_to_le32(rtc_tm_to_time64(dt));
 	ds2404_write_memory(dev, 0x203, 4, (u8 *)&time);
 	return 0;
 }
 
 static const struct rtc_class_ops ds2404_rtc_ops = {
 	.read_time	= ds2404_read_time,
-	.set_mmss	= ds2404_set_mmss,
+	.set_time	= ds2404_set_time,
 };
 
 static int rtc_probe(struct platform_device *pdev)
@@ -232,11 +212,17 @@ static int rtc_probe(struct platform_device *pdev)
 	if (!chip)
 		return -ENOMEM;
 
-	chip->ops = &ds2404_gpio_ops;
+	chip->rtc = devm_rtc_allocate_device(&pdev->dev);
+	if (IS_ERR(chip->rtc))
+		return PTR_ERR(chip->rtc);
 
-	retval = chip->ops->map_io(chip, pdev, pdata);
+	retval = ds2404_gpio_map(chip, pdev, pdata);
 	if (retval)
-		goto err_chip;
+		return retval;
+
+	retval = devm_add_action_or_reset(&pdev->dev, ds2404_gpio_unmap, chip);
+	if (retval)
+		return retval;
 
 	dev_info(&pdev->dev, "using GPIOs RST:%d, CLK:%d, DQ:%d\n",
 		 chip->gpio[DS2404_RST].gpio, chip->gpio[DS2404_CLK].gpio,
@@ -244,34 +230,19 @@ static int rtc_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, chip);
 
-	chip->rtc = devm_rtc_device_register(&pdev->dev, "ds2404",
-					&ds2404_rtc_ops, THIS_MODULE);
-	if (IS_ERR(chip->rtc)) {
-		retval = PTR_ERR(chip->rtc);
-		goto err_io;
-	}
+	chip->rtc->ops = &ds2404_rtc_ops;
+	chip->rtc->range_max = U32_MAX;
 
-	ds2404_enable_osc(&pdev->dev);
-	return 0;
-
-err_io:
-	chip->ops->unmap_io(chip);
-err_chip:
-	return retval;
-}
-
-static int rtc_remove(struct platform_device *dev)
-{
-	struct ds2404 *chip = platform_get_drvdata(dev);
-
-	chip->ops->unmap_io(chip);
+	retval = rtc_register_device(chip->rtc);
+	if (retval)
+		return retval;
 
+	ds2404_enable_osc(&pdev->dev);
 	return 0;
 }
 
 static struct platform_driver rtc_device_driver = {
 	.probe	= rtc_probe,
-	.remove = rtc_remove,
 	.driver = {
 		.name	= "ds2404",
 	},
diff --git a/drivers/rtc/rtc-ds3232.c b/drivers/rtc/rtc-ds3232.c
index 7184e5145f12..1e9312f96021 100644
--- a/drivers/rtc/rtc-ds3232.c
+++ b/drivers/rtc/rtc-ds3232.c
@@ -48,6 +48,10 @@
 #       define DS3232_REG_SR_A1F     0x01
 
 #define DS3232_REG_TEMPERATURE	0x11
+#define DS3232_REG_SRAM_START   0x14
+#define DS3232_REG_SRAM_END     0xFF
+
+#define DS3232_REG_SRAM_SIZE    236
 
 struct ds3232 {
 	struct device *dev;
@@ -461,11 +465,39 @@ static const struct rtc_class_ops ds3232_rtc_ops = {
 	.alarm_irq_enable = ds3232_alarm_irq_enable,
 };
 
+static int ds3232_nvmem_read(void *priv, unsigned int offset, void *val,
+			     size_t bytes)
+{
+	struct regmap *ds3232_regmap = (struct regmap *)priv;
+
+	return regmap_bulk_read(ds3232_regmap, DS3232_REG_SRAM_START + offset,
+				val, bytes);
+}
+
+static int ds3232_nvmem_write(void *priv, unsigned int offset, void *val,
+			      size_t bytes)
+{
+	struct regmap *ds3232_regmap = (struct regmap *)priv;
+
+	return regmap_bulk_write(ds3232_regmap, DS3232_REG_SRAM_START + offset,
+				 val, bytes);
+}
+
 static int ds3232_probe(struct device *dev, struct regmap *regmap, int irq,
 			const char *name)
 {
 	struct ds3232 *ds3232;
 	int ret;
+	struct nvmem_config nvmem_cfg = {
+		.name = "ds3232_sram",
+		.stride = 1,
+		.size = DS3232_REG_SRAM_SIZE,
+		.word_size = 1,
+		.reg_read = ds3232_nvmem_read,
+		.reg_write = ds3232_nvmem_write,
+		.priv = regmap,
+		.type = NVMEM_TYPE_BATTERY_BACKED
+	};
 
 	ds3232 = devm_kzalloc(dev, sizeof(*ds3232), GFP_KERNEL);
 	if (!ds3232)
@@ -490,6 +522,10 @@ static int ds3232_probe(struct device *dev, struct regmap *regmap, int irq,
 	if (IS_ERR(ds3232->rtc))
 		return PTR_ERR(ds3232->rtc);
 
+	ret = rtc_nvmem_register(ds3232->rtc, &nvmem_cfg);
+	if(ret)
+		return ret;
+
 	if (ds3232->irq > 0) {
 		ret = devm_request_threaded_irq(dev, ds3232->irq, NULL,
 						ds3232_irq,
@@ -542,7 +578,7 @@ static int ds3232_i2c_probe(struct i2c_client *client,
 	static const struct regmap_config config = {
 		.reg_bits = 8,
 		.val_bits = 8,
-		.max_register = 0x13,
+		.max_register = DS3232_REG_SRAM_END,
 	};
 
 	regmap = devm_regmap_init_i2c(client, &config);
@@ -609,7 +645,7 @@ static int ds3234_probe(struct spi_device *spi)
 	static const struct regmap_config config = {
 		.reg_bits = 8,
 		.val_bits = 8,
-		.max_register = 0x13,
+		.max_register = DS3232_REG_SRAM_END,
 		.write_flag_mask = 0x80,
 	};
 	struct regmap *regmap;
diff --git a/drivers/rtc/rtc-ep93xx.c b/drivers/rtc/rtc-ep93xx.c
index 1932a4f861d1..1766496385fe 100644
--- a/drivers/rtc/rtc-ep93xx.c
+++ b/drivers/rtc/rtc-ep93xx.c
@@ -1,12 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * A driver for the RTC embedded in the Cirrus Logic EP93XX processors
  * Copyright (c) 2006 Tower Technologies
  *
  * Author: Alessandro Zummo <a.zummo@towertech.it>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include <linux/module.h>
@@ -18,27 +15,23 @@
 #define EP93XX_RTC_DATA			0x000
 #define EP93XX_RTC_MATCH		0x004
 #define EP93XX_RTC_STATUS		0x008
-#define  EP93XX_RTC_STATUS_INTR		 (1<<0)
+#define  EP93XX_RTC_STATUS_INTR		 BIT(0)
 #define EP93XX_RTC_LOAD			0x00C
 #define EP93XX_RTC_CONTROL		0x010
-#define  EP93XX_RTC_CONTROL_MIE		 (1<<0)
+#define  EP93XX_RTC_CONTROL_MIE		 BIT(0)
 #define EP93XX_RTC_SWCOMP		0x108
 #define  EP93XX_RTC_SWCOMP_DEL_MASK	 0x001f0000
 #define  EP93XX_RTC_SWCOMP_DEL_SHIFT	 16
 #define  EP93XX_RTC_SWCOMP_INT_MASK	 0x0000ffff
 #define  EP93XX_RTC_SWCOMP_INT_SHIFT	 0
 
-/*
- * struct device dev.platform_data is used to store our private data
- * because struct rtc_device does not have a variable to hold it.
- */
 struct ep93xx_rtc {
 	void __iomem	*mmio_base;
 	struct rtc_device *rtc;
 };
 
 static int ep93xx_rtc_get_swcomp(struct device *dev, unsigned short *preload,
-				unsigned short *delete)
+				 unsigned short *delete)
 {
 	struct ep93xx_rtc *ep93xx_rtc = dev_get_platdata(dev);
 	unsigned long comp;
@@ -63,13 +56,14 @@ static int ep93xx_rtc_read_time(struct device *dev, struct rtc_time *tm)
 
 	time = readl(ep93xx_rtc->mmio_base + EP93XX_RTC_DATA);
 
-	rtc_time_to_tm(time, tm);
+	rtc_time64_to_tm(time, tm);
 	return 0;
 }
 
-static int ep93xx_rtc_set_mmss(struct device *dev, unsigned long secs)
+static int ep93xx_rtc_set_time(struct device *dev, struct rtc_time *tm)
 {
 	struct ep93xx_rtc *ep93xx_rtc = dev_get_platdata(dev);
+	unsigned long secs = rtc_tm_to_time64(tm);
 
 	writel(secs + 1, ep93xx_rtc->mmio_base + EP93XX_RTC_LOAD);
 	return 0;
@@ -89,31 +83,31 @@ static int ep93xx_rtc_proc(struct device *dev, struct seq_file *seq)
 
 static const struct rtc_class_ops ep93xx_rtc_ops = {
 	.read_time	= ep93xx_rtc_read_time,
-	.set_mmss	= ep93xx_rtc_set_mmss,
+	.set_time	= ep93xx_rtc_set_time,
 	.proc		= ep93xx_rtc_proc,
 };
 
-static ssize_t ep93xx_rtc_show_comp_preload(struct device *dev,
-			struct device_attribute *attr, char *buf)
+static ssize_t comp_preload_show(struct device *dev,
+				 struct device_attribute *attr, char *buf)
 {
 	unsigned short preload;
 
-	ep93xx_rtc_get_swcomp(dev, &preload, NULL);
+	ep93xx_rtc_get_swcomp(dev->parent, &preload, NULL);
 
 	return sprintf(buf, "%d\n", preload);
 }
-static DEVICE_ATTR(comp_preload, S_IRUGO, ep93xx_rtc_show_comp_preload, NULL);
+static DEVICE_ATTR_RO(comp_preload);
 
-static ssize_t ep93xx_rtc_show_comp_delete(struct device *dev,
-			struct device_attribute *attr, char *buf)
+static ssize_t comp_delete_show(struct device *dev,
+				struct device_attribute *attr, char *buf)
 {
 	unsigned short delete;
 
-	ep93xx_rtc_get_swcomp(dev, NULL, &delete);
+	ep93xx_rtc_get_swcomp(dev->parent, NULL, &delete);
 
 	return sprintf(buf, "%d\n", delete);
 }
-static DEVICE_ATTR(comp_delete, S_IRUGO, ep93xx_rtc_show_comp_delete, NULL);
+static DEVICE_ATTR_RO(comp_delete);
 
 static struct attribute *ep93xx_rtc_attrs[] = {
 	&dev_attr_comp_preload.attr,
@@ -140,33 +134,20 @@ static int ep93xx_rtc_probe(struct platform_device *pdev)
 	if (IS_ERR(ep93xx_rtc->mmio_base))
 		return PTR_ERR(ep93xx_rtc->mmio_base);
 
-	pdev->dev.platform_data = ep93xx_rtc;
 	platform_set_drvdata(pdev, ep93xx_rtc);
 
-	ep93xx_rtc->rtc = devm_rtc_device_register(&pdev->dev,
-				pdev->name, &ep93xx_rtc_ops, THIS_MODULE);
-	if (IS_ERR(ep93xx_rtc->rtc)) {
-		err = PTR_ERR(ep93xx_rtc->rtc);
-		goto exit;
-	}
+	ep93xx_rtc->rtc = devm_rtc_allocate_device(&pdev->dev);
+	if (IS_ERR(ep93xx_rtc->rtc))
+		return PTR_ERR(ep93xx_rtc->rtc);
 
-	err = sysfs_create_group(&pdev->dev.kobj, &ep93xx_rtc_sysfs_files);
-	if (err)
-		goto exit;
-
-	return 0;
+	ep93xx_rtc->rtc->ops = &ep93xx_rtc_ops;
+	ep93xx_rtc->rtc->range_max = U32_MAX;
 
-exit:
-	pdev->dev.platform_data = NULL;
-	return err;
-}
-
-static int ep93xx_rtc_remove(struct platform_device *pdev)
-{
-	sysfs_remove_group(&pdev->dev.kobj, &ep93xx_rtc_sysfs_files);
-	pdev->dev.platform_data = NULL;
+	err = rtc_add_group(ep93xx_rtc->rtc, &ep93xx_rtc_sysfs_files);
+	if (err)
+		return err;
 
-	return 0;
+	return rtc_register_device(ep93xx_rtc->rtc);
 }
 
 static struct platform_driver ep93xx_rtc_driver = {
@@ -174,7 +155,6 @@ static struct platform_driver ep93xx_rtc_driver = {
 		.name	= "ep93xx-rtc",
 	},
 	.probe		= ep93xx_rtc_probe,
-	.remove		= ep93xx_rtc_remove,
 };
 
 module_platform_driver(ep93xx_rtc_driver);
diff --git a/drivers/rtc/rtc-goldfish.c b/drivers/rtc/rtc-goldfish.c
index a1c44d0c8557..1a3420ee6a4d 100644
--- a/drivers/rtc/rtc-goldfish.c
+++ b/drivers/rtc/rtc-goldfish.c
@@ -1,23 +1,15 @@
+// SPDX-License-Identifier: GPL-2.0
 /* drivers/rtc/rtc-goldfish.c
  *
  * Copyright (C) 2007 Google, Inc.
  * Copyright (C) 2017 Imagination Technologies Ltd.
- *
- * This software is licensed under the terms of the GNU General Public
- * License version 2, as published by the Free Software Foundation, and
- * may be copied, distributed, and modified under those terms.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
  */
 
+#include <linux/io.h>
 #include <linux/module.h>
+#include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/rtc.h>
-#include <linux/io.h>
 
 #define TIMER_TIME_LOW		0x00	/* get low bits of current time  */
 					/*   and update TIMER_TIME_HIGH  */
@@ -56,7 +48,7 @@ static int goldfish_rtc_read_alarm(struct device *dev,
 	do_div(rtc_alarm, NSEC_PER_SEC);
 	memset(alrm, 0, sizeof(struct rtc_wkalrm));
 
-	rtc_time_to_tm(rtc_alarm, &alrm->time);
+	rtc_time64_to_tm(rtc_alarm, &alrm->time);
 
 	if (readl(base + TIMER_ALARM_STATUS))
 		alrm->enabled = 1;
@@ -70,21 +62,15 @@ static int goldfish_rtc_set_alarm(struct device *dev,
 				  struct rtc_wkalrm *alrm)
 {
 	struct goldfish_rtc *rtcdrv;
-	unsigned long rtc_alarm;
 	u64 rtc_alarm64;
 	u64 rtc_status_reg;
 	void __iomem *base;
-	int ret = 0;
 
 	rtcdrv = dev_get_drvdata(dev);
 	base = rtcdrv->base;
 
 	if (alrm->enabled) {
-		ret = rtc_tm_to_time(&alrm->time, &rtc_alarm);
-		if (ret != 0)
-			return ret;
-
-		rtc_alarm64 = rtc_alarm * NSEC_PER_SEC;
+		rtc_alarm64 = rtc_tm_to_time64(&alrm->time) * NSEC_PER_SEC;
 		writel((rtc_alarm64 >> 32), base + TIMER_ALARM_HIGH);
 		writel(rtc_alarm64, base + TIMER_ALARM_LOW);
 	} else {
@@ -98,7 +84,7 @@ static int goldfish_rtc_set_alarm(struct device *dev,
 			writel(1, base + TIMER_CLEAR_ALARM);
 	}
 
-	return ret;
+	return 0;
 }
 
 static int goldfish_rtc_alarm_irq_enable(struct device *dev,
@@ -147,7 +133,7 @@ static int goldfish_rtc_read_time(struct device *dev, struct rtc_time *tm)
 
 	do_div(time, NSEC_PER_SEC);
 
-	rtc_time_to_tm(time, tm);
+	rtc_time64_to_tm(time, tm);
 
 	return 0;
 }
@@ -156,21 +142,16 @@ static int goldfish_rtc_set_time(struct device *dev, struct rtc_time *tm)
 {
 	struct goldfish_rtc *rtcdrv;
 	void __iomem *base;
-	unsigned long now;
 	u64 now64;
-	int ret;
 
 	rtcdrv = dev_get_drvdata(dev);
 	base = rtcdrv->base;
 
-	ret = rtc_tm_to_time(tm, &now);
-	if (ret == 0) {
-		now64 = now * NSEC_PER_SEC;
-		writel((now64 >> 32), base + TIMER_TIME_HIGH);
-		writel(now64, base + TIMER_TIME_LOW);
-	}
+	now64 = rtc_tm_to_time64(tm) * NSEC_PER_SEC;
+	writel((now64 >> 32), base + TIMER_TIME_HIGH);
+	writel(now64, base + TIMER_TIME_LOW);
 
-	return ret;
+	return 0;
 }
 
 static const struct rtc_class_ops goldfish_rtc_ops = {
@@ -205,19 +186,20 @@ static int goldfish_rtc_probe(struct platform_device *pdev)
 	if (rtcdrv->irq < 0)
 		return -ENODEV;
 
-	rtcdrv->rtc = devm_rtc_device_register(&pdev->dev, pdev->name,
-					       &goldfish_rtc_ops,
-					       THIS_MODULE);
+	rtcdrv->rtc = devm_rtc_allocate_device(&pdev->dev);
 	if (IS_ERR(rtcdrv->rtc))
 		return PTR_ERR(rtcdrv->rtc);
 
+	rtcdrv->rtc->ops = &goldfish_rtc_ops;
+	rtcdrv->rtc->range_max = U64_MAX / NSEC_PER_SEC;
+
 	err = devm_request_irq(&pdev->dev, rtcdrv->irq,
 			       goldfish_rtc_interrupt,
 			       0, pdev->name, rtcdrv);
 	if (err)
 		return err;
 
-	return 0;
+	return rtc_register_device(rtcdrv->rtc);
 }
 
 static const struct of_device_id goldfish_rtc_of_match[] = {
diff --git a/drivers/rtc/rtc-hid-sensor-time.c b/drivers/rtc/rtc-hid-sensor-time.c
index 3e1abb455472..f27c40e8331f 100644
--- a/drivers/rtc/rtc-hid-sensor-time.c
+++ b/drivers/rtc/rtc-hid-sensor-time.c
@@ -205,8 +205,7 @@ static int hid_time_parse_report(struct platform_device *pdev,
 static int hid_rtc_read_time(struct device *dev, struct rtc_time *tm)
 {
 	unsigned long flags;
-	struct hid_time_state *time_state =
-		platform_get_drvdata(to_platform_device(dev));
+	struct hid_time_state *time_state = dev_get_drvdata(dev);
 	int ret;
 
 	reinit_completion(&time_state->comp_last_time);
diff --git a/drivers/rtc/rtc-imxdi.c b/drivers/rtc/rtc-imxdi.c
index 80931114c899..3f3d652a0b0f 100644
--- a/drivers/rtc/rtc-imxdi.c
+++ b/drivers/rtc/rtc-imxdi.c
@@ -1,20 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  * Copyright 2008-2009 Freescale Semiconductor, Inc. All Rights Reserved.
  * Copyright 2010 Orex Computed Radiography
  */
 
 /*
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
- */
-
-/* based on rtc-mc13892.c */
-
-/*
  * This driver uses the 47-bit 32 kHz counter in the Freescale DryIce block
  * to implement a Linux RTC. Times and alarms are truncated to seconds.
  * Since the RTC framework performs API locking via rtc->ops_lock the
@@ -552,7 +542,7 @@ static int dryice_rtc_read_time(struct device *dev, struct rtc_time *tm)
 	unsigned long now;
 
 	now = readl(imxdi->ioaddr + DTCMR);
-	rtc_time_to_tm(now, tm);
+	rtc_time64_to_tm(now, tm);
 
 	return 0;
 }
@@ -561,7 +551,7 @@ static int dryice_rtc_read_time(struct device *dev, struct rtc_time *tm)
  * set the seconds portion of dryice time counter and clear the
  * fractional part.
  */
-static int dryice_rtc_set_mmss(struct device *dev, unsigned long secs)
+static int dryice_rtc_set_time(struct device *dev, struct rtc_time *tm)
 {
 	struct imxdi_dev *imxdi = dev_get_drvdata(dev);
 	u32 dcr, dsr;
@@ -588,7 +578,7 @@ static int dryice_rtc_set_mmss(struct device *dev, unsigned long secs)
 	if (rc != 0)
 		return rc;
 
-	rc = di_write_wait(imxdi, secs, DTCMR);
+	rc = di_write_wait(imxdi, rtc_tm_to_time64(tm), DTCMR);
 	if (rc != 0)
 		return rc;
 
@@ -618,7 +608,7 @@ static int dryice_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alarm)
 	u32 dcamr;
 
 	dcamr = readl(imxdi->ioaddr + DCAMR);
-	rtc_time_to_tm(dcamr, &alarm->time);
+	rtc_time64_to_tm(dcamr, &alarm->time);
 
 	/* alarm is enabled if the interrupt is enabled */
 	alarm->enabled = (readl(imxdi->ioaddr + DIER) & DIER_CAIE) != 0;
@@ -640,21 +630,10 @@ static int dryice_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alarm)
 static int dryice_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alarm)
 {
 	struct imxdi_dev *imxdi = dev_get_drvdata(dev);
-	unsigned long now;
-	unsigned long alarm_time;
 	int rc;
 
-	rc = rtc_tm_to_time(&alarm->time, &alarm_time);
-	if (rc)
-		return rc;
-
-	/* don't allow setting alarm in the past */
-	now = readl(imxdi->ioaddr + DTCMR);
-	if (alarm_time < now)
-		return -EINVAL;
-
 	/* write the new alarm time */
-	rc = di_write_wait(imxdi, (u32)alarm_time, DCAMR);
+	rc = di_write_wait(imxdi, rtc_tm_to_time64(&alarm->time), DCAMR);
 	if (rc)
 		return rc;
 
@@ -668,7 +647,7 @@ static int dryice_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alarm)
 
 static const struct rtc_class_ops dryice_rtc_ops = {
 	.read_time		= dryice_rtc_read_time,
-	.set_mmss		= dryice_rtc_set_mmss,
+	.set_time		= dryice_rtc_set_time,
 	.alarm_irq_enable	= dryice_rtc_alarm_irq_enable,
 	.read_alarm		= dryice_rtc_read_alarm,
 	.set_alarm		= dryice_rtc_set_alarm,
@@ -796,6 +775,10 @@ static int __init dryice_rtc_probe(struct platform_device *pdev)
 
 	mutex_init(&imxdi->write_mutex);
 
+	imxdi->rtc = devm_rtc_allocate_device(&pdev->dev);
+	if (IS_ERR(imxdi->rtc))
+		return PTR_ERR(imxdi->rtc);
+
 	imxdi->clk = devm_clk_get(&pdev->dev, NULL);
 	if (IS_ERR(imxdi->clk))
 		return PTR_ERR(imxdi->clk);
@@ -829,12 +812,13 @@ static int __init dryice_rtc_probe(struct platform_device *pdev)
 	}
 
 	platform_set_drvdata(pdev, imxdi);
-	imxdi->rtc = devm_rtc_device_register(&pdev->dev, pdev->name,
-				  &dryice_rtc_ops, THIS_MODULE);
-	if (IS_ERR(imxdi->rtc)) {
-		rc = PTR_ERR(imxdi->rtc);
+
+	imxdi->rtc->ops = &dryice_rtc_ops;
+	imxdi->rtc->range_max = U32_MAX;
+
+	rc = rtc_register_device(imxdi->rtc);
+	if (rc)
 		goto err;
-	}
 
 	return 0;
 
diff --git a/drivers/rtc/rtc-jz4740.c b/drivers/rtc/rtc-jz4740.c
index d0a891777f44..9e7b3a04debc 100644
--- a/drivers/rtc/rtc-jz4740.c
+++ b/drivers/rtc/rtc-jz4740.c
@@ -1,17 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  *  Copyright (C) 2009-2010, Lars-Peter Clausen <lars@metafoo.de>
  *  Copyright (C) 2010, Paul Cercueil <paul@crapouillou.net>
  *	 JZ4740 SoC RTC driver
- *
- *  This program is free software; you can redistribute it and/or modify it
- *  under  the terms of  the GNU General Public License as published by the
- *  Free Software Foundation;  either version 2 of the License, or (at your
- *  option) any later version.
- *
- *  You should have received a copy of the GNU General Public License along
- *  with this program; if not, write to the Free Software Foundation, Inc.,
- *  675 Mass Ave, Cambridge, MA 02139, USA.
- *
  */
 
 #include <linux/clk.h>
@@ -20,6 +11,7 @@
 #include <linux/module.h>
 #include <linux/of_device.h>
 #include <linux/platform_device.h>
+#include <linux/pm_wakeirq.h>
 #include <linux/reboot.h>
 #include <linux/rtc.h>
 #include <linux/slab.h>
@@ -156,6 +148,9 @@ static int jz4740_rtc_read_time(struct device *dev, struct rtc_time *time)
 	uint32_t secs, secs2;
 	int timeout = 5;
 
+	if (jz4740_rtc_reg_read(rtc, JZ_REG_RTC_SCRATCHPAD) != 0x12345678)
+		return -EINVAL;
+
 	/* If the seconds register is read while it is updated, it can contain a
 	 * bogus value. This can be avoided by making sure that two consecutive
 	 * reads have the same value.
@@ -171,16 +166,21 @@ static int jz4740_rtc_read_time(struct device *dev, struct rtc_time *time)
 	if (timeout == 0)
 		return -EIO;
 
-	rtc_time_to_tm(secs, time);
+	rtc_time64_to_tm(secs, time);
 
 	return 0;
 }
 
-static int jz4740_rtc_set_mmss(struct device *dev, unsigned long secs)
+static int jz4740_rtc_set_time(struct device *dev, struct rtc_time *time)
 {
 	struct jz4740_rtc *rtc = dev_get_drvdata(dev);
+	int ret;
 
-	return jz4740_rtc_reg_write(rtc, JZ_REG_RTC_SEC, secs);
+	ret = jz4740_rtc_reg_write(rtc, JZ_REG_RTC_SEC, rtc_tm_to_time64(time));
+	if (ret)
+		return ret;
+
+	return jz4740_rtc_reg_write(rtc, JZ_REG_RTC_SCRATCHPAD, 0x12345678);
 }
 
 static int jz4740_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm)
@@ -196,18 +196,16 @@ static int jz4740_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 	alrm->enabled = !!(ctrl & JZ_RTC_CTRL_AE);
 	alrm->pending = !!(ctrl & JZ_RTC_CTRL_AF);
 
-	rtc_time_to_tm(secs, &alrm->time);
+	rtc_time64_to_tm(secs, &alrm->time);
 
-	return rtc_valid_tm(&alrm->time);
+	return 0;
 }
 
 static int jz4740_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 {
 	int ret;
 	struct jz4740_rtc *rtc = dev_get_drvdata(dev);
-	unsigned long secs;
-
-	rtc_tm_to_time(&alrm->time, &secs);
+	uint32_t secs = lower_32_bits(rtc_tm_to_time64(&alrm->time));
 
 	ret = jz4740_rtc_reg_write(rtc, JZ_REG_RTC_SEC_ALARM, secs);
 	if (!ret)
@@ -225,7 +223,7 @@ static int jz4740_rtc_alarm_irq_enable(struct device *dev, unsigned int enable)
 
 static const struct rtc_class_ops jz4740_rtc_ops = {
 	.read_time	= jz4740_rtc_read_time,
-	.set_mmss	= jz4740_rtc_set_mmss,
+	.set_time	= jz4740_rtc_set_time,
 	.read_alarm	= jz4740_rtc_read_alarm,
 	.set_alarm	= jz4740_rtc_set_alarm,
 	.alarm_irq_enable = jz4740_rtc_alarm_irq_enable,
@@ -309,7 +307,6 @@ static int jz4740_rtc_probe(struct platform_device *pdev)
 {
 	int ret;
 	struct jz4740_rtc *rtc;
-	uint32_t scratchpad;
 	struct resource *mem;
 	const struct platform_device_id *id = platform_get_device_id(pdev);
 	const struct of_device_id *of_id = of_match_device(
@@ -348,10 +345,24 @@ static int jz4740_rtc_probe(struct platform_device *pdev)
 
 	device_init_wakeup(&pdev->dev, 1);
 
-	rtc->rtc = devm_rtc_device_register(&pdev->dev, pdev->name,
-					&jz4740_rtc_ops, THIS_MODULE);
+	ret = dev_pm_set_wake_irq(&pdev->dev, rtc->irq);
+	if (ret) {
+		dev_err(&pdev->dev, "Failed to set wake irq: %d\n", ret);
+		return ret;
+	}
+
+	rtc->rtc = devm_rtc_allocate_device(&pdev->dev);
 	if (IS_ERR(rtc->rtc)) {
 		ret = PTR_ERR(rtc->rtc);
+		dev_err(&pdev->dev, "Failed to allocate rtc device: %d\n", ret);
+		return ret;
+	}
+
+	rtc->rtc->ops = &jz4740_rtc_ops;
+	rtc->rtc->range_max = U32_MAX;
+
+	ret = rtc_register_device(rtc->rtc);
+	if (ret) {
 		dev_err(&pdev->dev, "Failed to register rtc device: %d\n", ret);
 		return ret;
 	}
@@ -363,16 +374,6 @@ static int jz4740_rtc_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	scratchpad = jz4740_rtc_reg_read(rtc, JZ_REG_RTC_SCRATCHPAD);
-	if (scratchpad != 0x12345678) {
-		ret = jz4740_rtc_reg_write(rtc, JZ_REG_RTC_SCRATCHPAD, 0x12345678);
-		ret = jz4740_rtc_reg_write(rtc, JZ_REG_RTC_SEC, 0);
-		if (ret) {
-			dev_err(&pdev->dev, "Could not write to RTC registers\n");
-			return ret;
-		}
-	}
-
 	if (np && of_device_is_system_power_controller(np)) {
 		if (!pm_power_off) {
 			/* Default: 60ms */
@@ -397,35 +398,6 @@ static int jz4740_rtc_probe(struct platform_device *pdev)
 	return 0;
 }
 
-#ifdef CONFIG_PM
-static int jz4740_rtc_suspend(struct device *dev)
-{
-	struct jz4740_rtc *rtc = dev_get_drvdata(dev);
-
-	if (device_may_wakeup(dev))
-		enable_irq_wake(rtc->irq);
-	return 0;
-}
-
-static int jz4740_rtc_resume(struct device *dev)
-{
-	struct jz4740_rtc *rtc = dev_get_drvdata(dev);
-
-	if (device_may_wakeup(dev))
-		disable_irq_wake(rtc->irq);
-	return 0;
-}
-
-static const struct dev_pm_ops jz4740_pm_ops = {
-	.suspend = jz4740_rtc_suspend,
-	.resume  = jz4740_rtc_resume,
-};
-#define JZ4740_RTC_PM_OPS (&jz4740_pm_ops)
-
-#else
-#define JZ4740_RTC_PM_OPS NULL
-#endif  /* CONFIG_PM */
-
 static const struct platform_device_id jz4740_rtc_ids[] = {
 	{ "jz4740-rtc", ID_JZ4740 },
 	{ "jz4780-rtc", ID_JZ4780 },
@@ -437,7 +409,6 @@ static struct platform_driver jz4740_rtc_driver = {
 	.probe	 = jz4740_rtc_probe,
 	.driver	 = {
 		.name  = "jz4740-rtc",
-		.pm    = JZ4740_RTC_PM_OPS,
 		.of_match_table = of_match_ptr(jz4740_rtc_of_match),
 	},
 	.id_table = jz4740_rtc_ids,
diff --git a/drivers/rtc/rtc-lpc32xx.c b/drivers/rtc/rtc-lpc32xx.c
index 910e600275b9..ac393230e592 100644
--- a/drivers/rtc/rtc-lpc32xx.c
+++ b/drivers/rtc/rtc-lpc32xx.c
@@ -1,14 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  * Copyright (C) 2010 NXP Semiconductors
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- *  You should have received a copy of the GNU General Public License along
- *  with this program; if not, write to the Free Software Foundation, Inc.,
- *  675 Mass Ave, Cambridge, MA 02139, USA.
  */
 
 #include <linux/kernel.h>
@@ -47,8 +39,6 @@
 
 #define LPC32XX_RTC_KEY_ONSW_LOADVAL	0xB5C13F27
 
-#define RTC_NAME "rtc-lpc32xx"
-
 #define rtc_readl(dev, reg) \
 	__raw_readl((dev)->rtc_base + (reg))
 #define rtc_writel(dev, reg, val) \
@@ -68,14 +58,15 @@ static int lpc32xx_rtc_read_time(struct device *dev, struct rtc_time *time)
 	struct lpc32xx_rtc *rtc = dev_get_drvdata(dev);
 
 	elapsed_sec = rtc_readl(rtc, LPC32XX_RTC_UCOUNT);
-	rtc_time_to_tm(elapsed_sec, time);
+	rtc_time64_to_tm(elapsed_sec, time);
 
 	return 0;
 }
 
-static int lpc32xx_rtc_set_mmss(struct device *dev, unsigned long secs)
+static int lpc32xx_rtc_set_time(struct device *dev, struct rtc_time *time)
 {
 	struct lpc32xx_rtc *rtc = dev_get_drvdata(dev);
+	u32 secs = rtc_tm_to_time64(time);
 	u32 tmp;
 
 	spin_lock_irq(&rtc->lock);
@@ -97,7 +88,7 @@ static int lpc32xx_rtc_read_alarm(struct device *dev,
 {
 	struct lpc32xx_rtc *rtc = dev_get_drvdata(dev);
 
-	rtc_time_to_tm(rtc_readl(rtc, LPC32XX_RTC_MATCH0), &wkalrm->time);
+	rtc_time64_to_tm(rtc_readl(rtc, LPC32XX_RTC_MATCH0), &wkalrm->time);
 	wkalrm->enabled = rtc->alarm_enabled;
 	wkalrm->pending = !!(rtc_readl(rtc, LPC32XX_RTC_INTSTAT) &
 		LPC32XX_RTC_INTSTAT_MATCH0);
@@ -111,13 +102,8 @@ static int lpc32xx_rtc_set_alarm(struct device *dev,
 	struct lpc32xx_rtc *rtc = dev_get_drvdata(dev);
 	unsigned long alarmsecs;
 	u32 tmp;
-	int ret;
 
-	ret = rtc_tm_to_time(&wkalrm->time, &alarmsecs);
-	if (ret < 0) {
-		dev_warn(dev, "Failed to convert time: %d\n", ret);
-		return ret;
-	}
+	alarmsecs = rtc_tm_to_time64(&wkalrm->time);
 
 	spin_lock_irq(&rtc->lock);
 
@@ -191,7 +177,7 @@ static irqreturn_t lpc32xx_rtc_alarm_interrupt(int irq, void *dev)
 
 static const struct rtc_class_ops lpc32xx_rtc_ops = {
 	.read_time		= lpc32xx_rtc_read_time,
-	.set_mmss		= lpc32xx_rtc_set_mmss,
+	.set_time		= lpc32xx_rtc_set_time,
 	.read_alarm		= lpc32xx_rtc_read_alarm,
 	.set_alarm		= lpc32xx_rtc_set_alarm,
 	.alarm_irq_enable	= lpc32xx_rtc_alarm_irq_enable,
@@ -201,21 +187,13 @@ static int lpc32xx_rtc_probe(struct platform_device *pdev)
 {
 	struct resource *res;
 	struct lpc32xx_rtc *rtc;
-	int rtcirq;
+	int err;
 	u32 tmp;
 
-	rtcirq = platform_get_irq(pdev, 0);
-	if (rtcirq < 0) {
-		dev_warn(&pdev->dev, "Can't get interrupt resource\n");
-		rtcirq = -1;
-	}
-
 	rtc = devm_kzalloc(&pdev->dev, sizeof(*rtc), GFP_KERNEL);
 	if (unlikely(!rtc))
 		return -ENOMEM;
 
-	rtc->irq = rtcirq;
-
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	rtc->rtc_base = devm_ioremap_resource(&pdev->dev, res);
 	if (IS_ERR(rtc->rtc_base))
@@ -256,18 +234,25 @@ static int lpc32xx_rtc_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, rtc);
 
-	rtc->rtc = devm_rtc_device_register(&pdev->dev, RTC_NAME,
-					&lpc32xx_rtc_ops, THIS_MODULE);
-	if (IS_ERR(rtc->rtc)) {
-		dev_err(&pdev->dev, "Can't get RTC\n");
+	rtc->rtc = devm_rtc_allocate_device(&pdev->dev);
+	if (IS_ERR(rtc->rtc))
 		return PTR_ERR(rtc->rtc);
-	}
+
+	rtc->rtc->ops = &lpc32xx_rtc_ops;
+	rtc->rtc->range_max = U32_MAX;
+
+	err = rtc_register_device(rtc->rtc);
+	if (err)
+		return err;
 
 	/*
 	 * IRQ is enabled after device registration in case alarm IRQ
 	 * is pending upon suspend exit.
 	 */
-	if (rtc->irq >= 0) {
+	rtc->irq = platform_get_irq(pdev, 0);
+	if (rtc->irq < 0) {
+		dev_warn(&pdev->dev, "Can't get interrupt resource\n");
+	} else {
 		if (devm_request_irq(&pdev->dev, rtc->irq,
 				     lpc32xx_rtc_alarm_interrupt,
 				     0, pdev->name, rtc) < 0) {
@@ -374,7 +359,7 @@ static struct platform_driver lpc32xx_rtc_driver = {
 	.probe		= lpc32xx_rtc_probe,
 	.remove		= lpc32xx_rtc_remove,
 	.driver = {
-		.name	= RTC_NAME,
+		.name	= "rtc-lpc32xx",
 		.pm	= LPC32XX_RTC_PM_OPS,
 		.of_match_table = of_match_ptr(lpc32xx_rtc_match),
 	},
diff --git a/drivers/rtc/rtc-mc13xxx.c b/drivers/rtc/rtc-mc13xxx.c
index 0fa33708fc49..afce2c0b4bd6 100644
--- a/drivers/rtc/rtc-mc13xxx.c
+++ b/drivers/rtc/rtc-mc13xxx.c
@@ -1,12 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Real Time Clock driver for Freescale MC13XXX PMIC
  *
  * (C) 2009 Sascha Hauer, Pengutronix
  * (C) 2009 Uwe Kleine-Koenig, Pengutronix
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #include <linux/mfd/mc13xxx.h>
@@ -89,14 +86,14 @@ static int mc13xxx_rtc_read_time(struct device *dev, struct rtc_time *tm)
 	return 0;
 }
 
-static int mc13xxx_rtc_set_mmss(struct device *dev, time64_t secs)
+static int mc13xxx_rtc_set_time(struct device *dev, struct rtc_time *tm)
 {
 	struct mc13xxx_rtc *priv = dev_get_drvdata(dev);
 	unsigned int seconds, days;
 	unsigned int alarmseconds;
 	int ret;
 
-	days = div_s64_rem(secs, SEC_PER_DAY, &seconds);
+	days = div_s64_rem(rtc_tm_to_time64(tm), SEC_PER_DAY, &seconds);
 
 	mc13xxx_lock(priv->mc13xxx);
 
@@ -158,7 +155,7 @@ out:
 static int mc13xxx_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alarm)
 {
 	struct mc13xxx_rtc *priv = dev_get_drvdata(dev);
-	unsigned seconds, days;
+	unsigned int seconds, days;
 	time64_t s1970;
 	int enabled, pending;
 	int ret;
@@ -253,7 +250,7 @@ static irqreturn_t mc13xxx_rtc_alarm_handler(int irq, void *dev)
 
 static const struct rtc_class_ops mc13xxx_rtc_ops = {
 	.read_time = mc13xxx_rtc_read_time,
-	.set_mmss64 = mc13xxx_rtc_set_mmss,
+	.set_time = mc13xxx_rtc_set_time,
 	.read_alarm = mc13xxx_rtc_read_alarm,
 	.set_alarm = mc13xxx_rtc_set_alarm,
 	.alarm_irq_enable = mc13xxx_rtc_alarm_irq_enable,
@@ -285,8 +282,15 @@ static int __init mc13xxx_rtc_probe(struct platform_device *pdev)
 	priv->mc13xxx = mc13xxx;
 	priv->valid = 1;
 
+	priv->rtc = devm_rtc_allocate_device(&pdev->dev);
+	if (IS_ERR(priv->rtc))
+		return PTR_ERR(priv->rtc);
 	platform_set_drvdata(pdev, priv);
 
+	priv->rtc->ops = &mc13xxx_rtc_ops;
+	/* 15bit days + hours, minutes, seconds */
+	priv->rtc->range_max = (timeu64_t)(1 << 15) * SEC_PER_DAY - 1;
+
 	mc13xxx_lock(mc13xxx);
 
 	mc13xxx_irq_ack(mc13xxx, MC13XXX_IRQ_RTCRST);
@@ -303,8 +307,9 @@ static int __init mc13xxx_rtc_probe(struct platform_device *pdev)
 
 	mc13xxx_unlock(mc13xxx);
 
-	priv->rtc = devm_rtc_device_register(&pdev->dev, pdev->name,
-					     &mc13xxx_rtc_ops, THIS_MODULE);
+	ret = rtc_register_device(priv->rtc);
+	if (ret)
+		goto err_irq_request;
 
 	return 0;
 
diff --git a/drivers/rtc/rtc-mt6397.c b/drivers/rtc/rtc-mt6397.c
index e9a25ec4d434..c06cf5202e02 100644
--- a/drivers/rtc/rtc-mt6397.c
+++ b/drivers/rtc/rtc-mt6397.c
@@ -343,7 +343,7 @@ static int mtk_rtc_probe(struct platform_device *pdev)
 	if (ret) {
 		dev_err(&pdev->dev, "Failed to request alarm IRQ: %d: %d\n",
 			rtc->irq, ret);
-		goto out_dispose_irq;
+		return ret;
 	}
 
 	device_init_wakeup(&pdev->dev, 1);
@@ -359,9 +359,7 @@ static int mtk_rtc_probe(struct platform_device *pdev)
 	return 0;
 
 out_free_irq:
-	free_irq(rtc->irq, rtc->rtc_dev);
-out_dispose_irq:
-	irq_dispose_mapping(rtc->irq);
+	free_irq(rtc->irq, rtc);
 	return ret;
 }
 
@@ -369,8 +367,7 @@ static int mtk_rtc_remove(struct platform_device *pdev)
 {
 	struct mt6397_rtc *rtc = platform_get_drvdata(pdev);
 
-	free_irq(rtc->irq, rtc->rtc_dev);
-	irq_dispose_mapping(rtc->irq);
+	free_irq(rtc->irq, rtc);
 
 	return 0;
 }
diff --git a/drivers/rtc/rtc-mv.c b/drivers/rtc/rtc-mv.c
index e7f14bd12fe3..ab9db57a6834 100644
--- a/drivers/rtc/rtc-mv.c
+++ b/drivers/rtc/rtc-mv.c
@@ -1,9 +1,6 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Driver for the RTC in Marvell SoCs.
- *
- * This file is licensed under the terms of the GNU General Public
- * License version 2.  This program is licensed "as is" without any
- * warranty of any kind, whether express or implied.
  */
 
 #include <linux/init.h>
@@ -60,7 +57,7 @@ static int mv_rtc_set_time(struct device *dev, struct rtc_time *tm)
 
 	rtc_reg = (bin2bcd(tm->tm_mday) << RTC_MDAY_OFFS) |
 		(bin2bcd(tm->tm_mon + 1) << RTC_MONTH_OFFS) |
-		(bin2bcd(tm->tm_year % 100) << RTC_YEAR_OFFS);
+		(bin2bcd(tm->tm_year - 100) << RTC_YEAR_OFFS);
 	writel(rtc_reg, ioaddr + RTC_DATE_REG_OFFS);
 
 	return 0;
@@ -159,7 +156,7 @@ static int mv_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alm)
 			<< RTC_MONTH_OFFS;
 
 	if (alm->time.tm_year >= 0)
-		rtc_reg |= (RTC_ALARM_VALID | bin2bcd(alm->time.tm_year % 100))
+		rtc_reg |= (RTC_ALARM_VALID | bin2bcd(alm->time.tm_year - 100))
 			<< RTC_YEAR_OFFS;
 
 	writel(rtc_reg, ioaddr + RTC_ALARM_DATE_REG_OFFS);
@@ -257,15 +254,7 @@ static int __init mv_rtc_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, pdata);
 
-	if (pdata->irq >= 0) {
-		device_init_wakeup(&pdev->dev, 1);
-		pdata->rtc = devm_rtc_device_register(&pdev->dev, pdev->name,
-						 &mv_rtc_alarm_ops,
-						 THIS_MODULE);
-	} else {
-		pdata->rtc = devm_rtc_device_register(&pdev->dev, pdev->name,
-						 &mv_rtc_ops, THIS_MODULE);
-	}
+	pdata->rtc = devm_rtc_allocate_device(&pdev->dev);
 	if (IS_ERR(pdata->rtc)) {
 		ret = PTR_ERR(pdata->rtc);
 		goto out;
@@ -281,7 +270,19 @@ static int __init mv_rtc_probe(struct platform_device *pdev)
 		}
 	}
 
-	return 0;
+	if (pdata->irq >= 0) {
+		device_init_wakeup(&pdev->dev, 1);
+		pdata->rtc->ops = &mv_rtc_alarm_ops;
+	} else {
+		pdata->rtc->ops = &mv_rtc_ops;
+	}
+
+	pdata->rtc->range_min = RTC_TIMESTAMP_BEGIN_2000;
+	pdata->rtc->range_max = RTC_TIMESTAMP_END_2099;
+
+	ret = rtc_register_device(pdata->rtc);
+	if (!ret)
+		return 0;
 out:
 	if (!IS_ERR(pdata->clk))
 		clk_disable_unprepare(pdata->clk);
diff --git a/drivers/rtc/rtc-mxc.c b/drivers/rtc/rtc-mxc.c
index 878c6ee82901..e697e96612bb 100644
--- a/drivers/rtc/rtc-mxc.c
+++ b/drivers/rtc/rtc-mxc.c
@@ -8,6 +8,7 @@
 #include <linux/slab.h>
 #include <linux/interrupt.h>
 #include <linux/platform_device.h>
+#include <linux/pm_wakeirq.h>
 #include <linux/clk.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
@@ -253,20 +254,9 @@ static int mxc_rtc_read_time(struct device *dev, struct rtc_time *tm)
 /*
  * This function sets the internal RTC time based on tm in Gregorian date.
  */
-static int mxc_rtc_set_mmss(struct device *dev, time64_t time)
+static int mxc_rtc_set_time(struct device *dev, struct rtc_time *tm)
 {
-	struct rtc_plat_data *pdata = dev_get_drvdata(dev);
-
-	/*
-	 * TTC_DAYR register is 9-bit in MX1 SoC, save time and day of year only
-	 */
-	if (is_imx1_rtc(pdata)) {
-		struct rtc_time tm;
-
-		rtc_time64_to_tm(time, &tm);
-		tm.tm_year = 70;
-		time = rtc_tm_to_time64(&tm);
-	}
+	time64_t time = rtc_tm_to_time64(tm);
 
 	/* Avoid roll-over from reading the different registers */
 	do {
@@ -310,7 +300,7 @@ static int mxc_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 /* RTC layer */
 static const struct rtc_class_ops mxc_rtc_ops = {
 	.read_time		= mxc_rtc_read_time,
-	.set_mmss64		= mxc_rtc_set_mmss,
+	.set_time		= mxc_rtc_set_time,
 	.read_alarm		= mxc_rtc_read_alarm,
 	.set_alarm		= mxc_rtc_set_alarm,
 	.alarm_irq_enable	= mxc_rtc_alarm_irq_enable,
@@ -318,7 +308,6 @@ static const struct rtc_class_ops mxc_rtc_ops = {
 
 static int mxc_rtc_probe(struct platform_device *pdev)
 {
-	struct resource *res;
 	struct rtc_device *rtc;
 	struct rtc_plat_data *pdata = NULL;
 	u32 reg;
@@ -336,11 +325,34 @@ static int mxc_rtc_probe(struct platform_device *pdev)
 	else
 		pdata->devtype = pdev->id_entry->driver_data;
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	pdata->ioaddr = devm_ioremap_resource(&pdev->dev, res);
+	pdata->ioaddr = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(pdata->ioaddr))
 		return PTR_ERR(pdata->ioaddr);
 
+	rtc = devm_rtc_allocate_device(&pdev->dev);
+	if (IS_ERR(rtc))
+		return PTR_ERR(rtc);
+
+	pdata->rtc = rtc;
+	rtc->ops = &mxc_rtc_ops;
+	if (is_imx1_rtc(pdata)) {
+		struct rtc_time tm;
+
+		/* 9bit days + hours minutes seconds */
+		rtc->range_max = (1 << 9) * 86400 - 1;
+
+		/*
+		 * Set the start date as beginning of the current year. This can
+		 * be overridden using device tree.
+		 */
+		rtc_time64_to_tm(ktime_get_real_seconds(), &tm);
+		rtc->start_secs =  mktime64(tm.tm_year, 1, 1, 0, 0, 0);
+		rtc->set_start_time = true;
+	} else {
+		/* 16bit days + hours minutes seconds */
+		rtc->range_max = (1 << 16) * 86400ULL - 1;
+	}
+
 	pdata->clk_ipg = devm_clk_get(&pdev->dev, "ipg");
 	if (IS_ERR(pdata->clk_ipg)) {
 		dev_err(&pdev->dev, "unable to get ipg clock!\n");
@@ -396,17 +408,16 @@ static int mxc_rtc_probe(struct platform_device *pdev)
 		pdata->irq = -1;
 	}
 
-	if (pdata->irq >= 0)
+	if (pdata->irq >= 0) {
 		device_init_wakeup(&pdev->dev, 1);
-
-	rtc = devm_rtc_device_register(&pdev->dev, pdev->name, &mxc_rtc_ops,
-				  THIS_MODULE);
-	if (IS_ERR(rtc)) {
-		ret = PTR_ERR(rtc);
-		goto exit_put_clk_ref;
+		ret = dev_pm_set_wake_irq(&pdev->dev, pdata->irq);
+		if (ret)
+			dev_err(&pdev->dev, "failed to enable irq wake\n");
 	}
 
-	pdata->rtc = rtc;
+	ret = rtc_register_device(rtc);
+	if (ret)
+		goto exit_put_clk_ref;
 
 	return 0;
 
@@ -428,35 +439,10 @@ static int mxc_rtc_remove(struct platform_device *pdev)
 	return 0;
 }
 
-#ifdef CONFIG_PM_SLEEP
-static int mxc_rtc_suspend(struct device *dev)
-{
-	struct rtc_plat_data *pdata = dev_get_drvdata(dev);
-
-	if (device_may_wakeup(dev))
-		enable_irq_wake(pdata->irq);
-
-	return 0;
-}
-
-static int mxc_rtc_resume(struct device *dev)
-{
-	struct rtc_plat_data *pdata = dev_get_drvdata(dev);
-
-	if (device_may_wakeup(dev))
-		disable_irq_wake(pdata->irq);
-
-	return 0;
-}
-#endif
-
-static SIMPLE_DEV_PM_OPS(mxc_rtc_pm_ops, mxc_rtc_suspend, mxc_rtc_resume);
-
 static struct platform_driver mxc_rtc_driver = {
 	.driver = {
 		   .name	= "mxc_rtc",
 		   .of_match_table = of_match_ptr(imx_rtc_dt_ids),
-		   .pm		= &mxc_rtc_pm_ops,
 	},
 	.id_table = imx_rtc_devtype,
 	.probe = mxc_rtc_probe,
diff --git a/drivers/rtc/rtc-mxc_v2.c b/drivers/rtc/rtc-mxc_v2.c
index 007879a5042d..5b970a816631 100644
--- a/drivers/rtc/rtc-mxc_v2.c
+++ b/drivers/rtc/rtc-mxc_v2.c
@@ -10,6 +10,7 @@
 #include <linux/module.h>
 #include <linux/mod_devicetable.h>
 #include <linux/platform_device.h>
+#include <linux/pm_wakeirq.h>
 #include <linux/rtc.h>
 
 #define SRTC_LPPDR_INIT       0x41736166	/* init for glitch detect */
@@ -305,6 +306,9 @@ static int mxc_rtc_probe(struct platform_device *pdev)
 		return pdata->irq;
 
 	device_init_wakeup(&pdev->dev, 1);
+	ret = dev_pm_set_wake_irq(&pdev->dev, pdata->irq);
+	if (ret)
+		dev_err(&pdev->dev, "failed to enable irq wake\n");
 
 	ret = clk_prepare_enable(pdata->clk);
 	if (ret)
@@ -367,30 +371,6 @@ static int mxc_rtc_remove(struct platform_device *pdev)
 	return 0;
 }
 
-#ifdef CONFIG_PM_SLEEP
-static int mxc_rtc_suspend(struct device *dev)
-{
-	struct mxc_rtc_data *pdata = dev_get_drvdata(dev);
-
-	if (device_may_wakeup(dev))
-		enable_irq_wake(pdata->irq);
-
-	return 0;
-}
-
-static int mxc_rtc_resume(struct device *dev)
-{
-	struct mxc_rtc_data *pdata = dev_get_drvdata(dev);
-
-	if (device_may_wakeup(dev))
-		disable_irq_wake(pdata->irq);
-
-	return 0;
-}
-#endif
-
-static SIMPLE_DEV_PM_OPS(mxc_rtc_pm_ops, mxc_rtc_suspend, mxc_rtc_resume);
-
 static const struct of_device_id mxc_ids[] = {
 	{ .compatible = "fsl,imx53-rtc", },
 	{}
@@ -400,7 +380,6 @@ static struct platform_driver mxc_rtc_driver = {
 	.driver = {
 		.name = "mxc_rtc_v2",
 		.of_match_table = mxc_ids,
-		.pm = &mxc_rtc_pm_ops,
 	},
 	.probe = mxc_rtc_probe,
 	.remove = mxc_rtc_remove,
diff --git a/drivers/rtc/rtc-omap.c b/drivers/rtc/rtc-omap.c
index bbff0e2deb84..32994b0dd139 100644
--- a/drivers/rtc/rtc-omap.c
+++ b/drivers/rtc/rtc-omap.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  * TI OMAP Real Time Clock interface for Linux
  *
@@ -6,11 +7,6 @@
  *
  * Copyright (C) 2006 David Brownell (new RTC framework)
  * Copyright (C) 2014 Johan Hovold <johan@kernel.org>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version
- * 2 of the License, or (at your option) any later version.
  */
 
 #include <dt-bindings/gpio/gpio.h>
@@ -271,7 +267,7 @@ static int omap_rtc_alarm_irq_enable(struct device *dev, unsigned int enabled)
 }
 
 /* this hardware doesn't support "don't care" alarm fields */
-static int tm2bcd(struct rtc_time *tm)
+static void tm2bcd(struct rtc_time *tm)
 {
 	tm->tm_sec = bin2bcd(tm->tm_sec);
 	tm->tm_min = bin2bcd(tm->tm_min);
@@ -279,13 +275,7 @@ static int tm2bcd(struct rtc_time *tm)
 	tm->tm_mday = bin2bcd(tm->tm_mday);
 
 	tm->tm_mon = bin2bcd(tm->tm_mon + 1);
-
-	/* epoch == 1900 */
-	if (tm->tm_year < 100 || tm->tm_year > 199)
-		return -EINVAL;
 	tm->tm_year = bin2bcd(tm->tm_year - 100);
-
-	return 0;
 }
 
 static void bcd2tm(struct rtc_time *tm)
@@ -328,8 +318,7 @@ static int omap_rtc_set_time(struct device *dev, struct rtc_time *tm)
 {
 	struct omap_rtc *rtc = dev_get_drvdata(dev);
 
-	if (tm2bcd(tm) < 0)
-		return -EINVAL;
+	tm2bcd(tm);
 
 	local_irq_disable();
 	rtc_wait_not_busy(rtc);
@@ -378,8 +367,7 @@ static int omap_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alm)
 	struct omap_rtc *rtc = dev_get_drvdata(dev);
 	u8 reg, irqwake_reg = 0;
 
-	if (tm2bcd(&alm->time) < 0)
-		return -EINVAL;
+	tm2bcd(&alm->time);
 
 	local_irq_disable();
 	rtc_wait_not_busy(rtc);
@@ -441,14 +429,10 @@ again:
 	omap_rtc_read_time_raw(rtc, &tm);
 	seconds = tm.tm_sec;
 	bcd2tm(&tm);
-	rtc_tm_to_time(&tm, &now);
-	rtc_time_to_tm(now + 1, &tm);
+	now = rtc_tm_to_time64(&tm);
+	rtc_time64_to_tm(now + 1, &tm);
 
-	if (tm2bcd(&tm) < 0) {
-		dev_err(&rtc->rtc->dev, "power off failed\n");
-		rtc->type->lock(rtc);
-		return;
-	}
+	tm2bcd(&tm);
 
 	rtc_wait_not_busy(rtc);
 
@@ -845,6 +829,8 @@ static int omap_rtc_probe(struct platform_device *pdev)
 	}
 
 	rtc->rtc->ops = &omap_rtc_ops;
+	rtc->rtc->range_min = RTC_TIMESTAMP_BEGIN_2000;
+	rtc->rtc->range_max = RTC_TIMESTAMP_END_2099;
 	omap_rtc_nvmem_config.priv = rtc;
 
 	/* handle periodic and alarm irqs */
diff --git a/drivers/rtc/rtc-opal.c b/drivers/rtc/rtc-opal.c
index 60f2250fd96b..3dd9d266ce09 100644
--- a/drivers/rtc/rtc-opal.c
+++ b/drivers/rtc/rtc-opal.c
@@ -224,7 +224,7 @@ exit:
 	return rc;
 }
 
-int opal_tpo_alarm_irq_enable(struct device *dev, unsigned int enabled)
+static int opal_tpo_alarm_irq_enable(struct device *dev, unsigned int enabled)
 {
 	struct rtc_wkalrm alarm = { .enabled = 0 };
 
diff --git a/drivers/rtc/rtc-pcap.c b/drivers/rtc/rtc-pcap.c
index f176cb9d0dbc..178bfb1dea21 100644
--- a/drivers/rtc/rtc-pcap.c
+++ b/drivers/rtc/rtc-pcap.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  *  pcap rtc code for Motorola EZX phones
  *
@@ -5,11 +6,6 @@
  *  Copyright (c) 2009 Daniel Ribeiro <drwyrm@gmail.com>
  *
  *  Based on Motorola's rtc.c Copyright (c) 2003-2005 Motorola
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License version 2 as
- *  published by the Free Software Foundation.
- *
  */
 
 #include <linux/kernel.h>
@@ -55,7 +51,7 @@ static int pcap_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 	ezx_pcap_read(pcap_rtc->pcap, PCAP_REG_RTC_DAYA, &days);
 	secs += (days & PCAP_RTC_DAY_MASK) * SEC_PER_DAY;
 
-	rtc_time_to_tm(secs, tm);
+	rtc_time64_to_tm(secs, tm);
 
 	return 0;
 }
@@ -63,12 +59,9 @@ static int pcap_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 static int pcap_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 {
 	struct pcap_rtc *pcap_rtc = dev_get_drvdata(dev);
-	struct rtc_time *tm = &alrm->time;
-	unsigned long secs;
+	unsigned long secs = rtc_tm_to_time64(&alrm->time);
 	u32 tod, days;
 
-	rtc_tm_to_time(tm, &secs);
-
 	tod = secs % SEC_PER_DAY;
 	ezx_pcap_write(pcap_rtc->pcap, PCAP_REG_RTC_TODA, tod);
 
@@ -90,14 +83,15 @@ static int pcap_rtc_read_time(struct device *dev, struct rtc_time *tm)
 	ezx_pcap_read(pcap_rtc->pcap, PCAP_REG_RTC_DAY, &days);
 	secs += (days & PCAP_RTC_DAY_MASK) * SEC_PER_DAY;
 
-	rtc_time_to_tm(secs, tm);
+	rtc_time64_to_tm(secs, tm);
 
 	return 0;
 }
 
-static int pcap_rtc_set_mmss(struct device *dev, unsigned long secs)
+static int pcap_rtc_set_time(struct device *dev, struct rtc_time *tm)
 {
 	struct pcap_rtc *pcap_rtc = dev_get_drvdata(dev);
+	unsigned long secs = rtc_tm_to_time64(tm);
 	u32 tod, days;
 
 	tod = secs % SEC_PER_DAY;
@@ -128,9 +122,9 @@ static int pcap_rtc_alarm_irq_enable(struct device *dev, unsigned int en)
 
 static const struct rtc_class_ops pcap_rtc_ops = {
 	.read_time = pcap_rtc_read_time,
+	.set_time = pcap_rtc_set_time,
 	.read_alarm = pcap_rtc_read_alarm,
 	.set_alarm = pcap_rtc_set_alarm,
-	.set_mmss = pcap_rtc_set_mmss,
 	.alarm_irq_enable = pcap_rtc_alarm_irq_enable,
 };
 
@@ -149,11 +143,13 @@ static int __init pcap_rtc_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, pcap_rtc);
 
-	pcap_rtc->rtc = devm_rtc_device_register(&pdev->dev, "pcap",
-					&pcap_rtc_ops, THIS_MODULE);
+	pcap_rtc->rtc = devm_rtc_allocate_device(&pdev->dev);
 	if (IS_ERR(pcap_rtc->rtc))
 		return PTR_ERR(pcap_rtc->rtc);
 
+	pcap_rtc->rtc->ops = &pcap_rtc_ops;
+	pcap_rtc->rtc->range_max = (1 << 14) * 86400ULL - 1;
+
 	timer_irq = pcap_to_irq(pcap_rtc->pcap, PCAP_IRQ_1HZ);
 	alarm_irq = pcap_to_irq(pcap_rtc->pcap, PCAP_IRQ_TODA);
 
@@ -167,7 +163,7 @@ static int __init pcap_rtc_probe(struct platform_device *pdev)
 	if (err)
 		return err;
 
-	return 0;
+	return rtc_register_device(pcap_rtc->rtc);
 }
 
 static int __exit pcap_rtc_remove(struct platform_device *pdev)
diff --git a/drivers/rtc/rtc-pcf85063.c b/drivers/rtc/rtc-pcf85063.c
index f6ce63c443a0..1afa6d9fa9fb 100644
--- a/drivers/rtc/rtc-pcf85063.c
+++ b/drivers/rtc/rtc-pcf85063.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * An I2C driver for the PCF85063 RTC
  * Copyright 2014 Rose Technology
@@ -5,16 +6,16 @@
  * Author: Søren Andersen <san@rosetechnology.dk>
  * Maintainers: http://www.nslu2-linux.org/
  *
- * based on the other drivers in this same directory.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
+ * Copyright (C) 2019 Micro Crystal AG
+ * Author: Alexandre Belloni <alexandre.belloni@bootlin.com>
  */
 #include <linux/i2c.h>
 #include <linux/bcd.h>
 #include <linux/rtc.h>
 #include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/pm_wakeirq.h>
+#include <linux/regmap.h>
 
 /*
  * Information for this driver was pulled from the following datasheets.
@@ -24,61 +25,47 @@
  *
  *  PCF85063A -- Rev. 6 — 18 November 2015
  *  PCF85063TP -- Rev. 4 — 6 May 2015
-*/
+ *
+ *  https://www.microcrystal.com/fileadmin/Media/Products/RTC/App.Manual/RV-8263-C7_App-Manual.pdf
+ *  RV8263 -- Rev. 1.0 — January 2019
+ */
 
 #define PCF85063_REG_CTRL1		0x00 /* status */
 #define PCF85063_REG_CTRL1_CAP_SEL	BIT(0)
 #define PCF85063_REG_CTRL1_STOP		BIT(5)
 
-#define PCF85063_REG_SC			0x04 /* datetime */
-#define PCF85063_REG_SC_OS		0x80
-
-static struct i2c_driver pcf85063_driver;
-
-static int pcf85063_stop_clock(struct i2c_client *client, u8 *ctrl1)
-{
-	int rc;
-	u8 reg;
+#define PCF85063_REG_CTRL2		0x01
+#define PCF85063_CTRL2_AF		BIT(6)
+#define PCF85063_CTRL2_AIE		BIT(7)
 
-	rc = i2c_smbus_read_byte_data(client, PCF85063_REG_CTRL1);
-	if (rc < 0) {
-		dev_err(&client->dev, "Failing to stop the clock\n");
-		return -EIO;
-	}
+#define PCF85063_REG_OFFSET		0x02
+#define PCF85063_OFFSET_SIGN_BIT	6	/* 2's complement sign bit */
+#define PCF85063_OFFSET_MODE		BIT(7)
+#define PCF85063_OFFSET_STEP0		4340
+#define PCF85063_OFFSET_STEP1		4069
 
-	/* stop the clock */
-	reg = rc | PCF85063_REG_CTRL1_STOP;
+#define PCF85063_REG_RAM		0x03
 
-	rc = i2c_smbus_write_byte_data(client, PCF85063_REG_CTRL1, reg);
-	if (rc < 0) {
-		dev_err(&client->dev, "Failing to stop the clock\n");
-		return -EIO;
-	}
-
-	*ctrl1 = reg;
-
-	return 0;
-}
-
-static int pcf85063_start_clock(struct i2c_client *client, u8 ctrl1)
-{
-	int rc;
+#define PCF85063_REG_SC			0x04 /* datetime */
+#define PCF85063_REG_SC_OS		0x80
 
-	/* start the clock */
-	ctrl1 &= ~PCF85063_REG_CTRL1_STOP;
+#define PCF85063_REG_ALM_S		0x0b
+#define PCF85063_AEN			BIT(7)
 
-	rc = i2c_smbus_write_byte_data(client, PCF85063_REG_CTRL1, ctrl1);
-	if (rc < 0) {
-		dev_err(&client->dev, "Failing to start the clock\n");
-		return -EIO;
-	}
+struct pcf85063_config {
+	struct regmap_config regmap;
+	unsigned has_alarms:1;
+	unsigned force_cap_7000:1;
+};
 
-	return 0;
-}
+struct pcf85063 {
+	struct rtc_device	*rtc;
+	struct regmap		*regmap;
+};
 
 static int pcf85063_rtc_read_time(struct device *dev, struct rtc_time *tm)
 {
-	struct i2c_client *client = to_i2c_client(dev);
+	struct pcf85063 *pcf85063 = dev_get_drvdata(dev);
 	int rc;
 	u8 regs[7];
 
@@ -88,16 +75,14 @@ static int pcf85063_rtc_read_time(struct device *dev, struct rtc_time *tm)
 	 * event, the access must be finished within one second. So, read all
 	 * time/date registers in one turn.
 	 */
-	rc = i2c_smbus_read_i2c_block_data(client, PCF85063_REG_SC,
-					   sizeof(regs), regs);
-	if (rc != sizeof(regs)) {
-		dev_err(&client->dev, "date/time register read error\n");
-		return -EIO;
-	}
+	rc = regmap_bulk_read(pcf85063->regmap, PCF85063_REG_SC, regs,
+			      sizeof(regs));
+	if (rc)
+		return rc;
 
 	/* if the clock has lost its power it makes no sense to use its time */
 	if (regs[0] & PCF85063_REG_SC_OS) {
-		dev_warn(&client->dev, "Power loss detected, invalid time\n");
+		dev_warn(&pcf85063->rtc->dev, "Power loss detected, invalid time\n");
 		return -EINVAL;
 	}
 
@@ -115,20 +100,18 @@ static int pcf85063_rtc_read_time(struct device *dev, struct rtc_time *tm)
 
 static int pcf85063_rtc_set_time(struct device *dev, struct rtc_time *tm)
 {
-	struct i2c_client *client = to_i2c_client(dev);
+	struct pcf85063 *pcf85063 = dev_get_drvdata(dev);
 	int rc;
 	u8 regs[7];
-	u8 ctrl1;
-
-	if ((tm->tm_year < 100) || (tm->tm_year > 199))
-		return -EINVAL;
 
 	/*
 	 * to accurately set the time, reset the divider chain and keep it in
 	 * reset state until all time/date registers are written
 	 */
-	rc = pcf85063_stop_clock(client, &ctrl1);
-	if (rc != 0)
+	rc = regmap_update_bits(pcf85063->regmap, PCF85063_REG_CTRL1,
+				PCF85063_REG_CTRL1_STOP,
+				PCF85063_REG_CTRL1_STOP);
+	if (rc)
 		return rc;
 
 	/* hours, minutes and seconds */
@@ -150,101 +133,351 @@ static int pcf85063_rtc_set_time(struct device *dev, struct rtc_time *tm)
 	regs[6] = bin2bcd(tm->tm_year - 100);
 
 	/* write all registers at once */
-	rc = i2c_smbus_write_i2c_block_data(client, PCF85063_REG_SC,
-					    sizeof(regs), regs);
-	if (rc < 0) {
-		dev_err(&client->dev, "date/time register write error\n");
+	rc = regmap_bulk_write(pcf85063->regmap, PCF85063_REG_SC,
+			       regs, sizeof(regs));
+	if (rc)
 		return rc;
-	}
 
 	/*
 	 * Write the control register as a separate action since the size of
 	 * the register space is different between the PCF85063TP and
 	 * PCF85063A devices.  The rollover point can not be used.
 	 */
-	rc = pcf85063_start_clock(client, ctrl1);
-	if (rc != 0)
-		return rc;
+	return regmap_update_bits(pcf85063->regmap, PCF85063_REG_CTRL1,
+				  PCF85063_REG_CTRL1_STOP, 0);
+}
+
+static int pcf85063_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm)
+{
+	struct pcf85063 *pcf85063 = dev_get_drvdata(dev);
+	u8 buf[4];
+	unsigned int val;
+	int ret;
+
+	ret = regmap_bulk_read(pcf85063->regmap, PCF85063_REG_ALM_S,
+			       buf, sizeof(buf));
+	if (ret)
+		return ret;
+
+	alrm->time.tm_sec = bcd2bin(buf[0]);
+	alrm->time.tm_min = bcd2bin(buf[1]);
+	alrm->time.tm_hour = bcd2bin(buf[2]);
+	alrm->time.tm_mday = bcd2bin(buf[3]);
+
+	ret = regmap_read(pcf85063->regmap, PCF85063_REG_CTRL2, &val);
+	if (ret)
+		return ret;
+
+	alrm->enabled =  !!(val & PCF85063_CTRL2_AIE);
 
 	return 0;
 }
 
+static int pcf85063_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm)
+{
+	struct pcf85063 *pcf85063 = dev_get_drvdata(dev);
+	u8 buf[5];
+	int ret;
+
+	buf[0] = bin2bcd(alrm->time.tm_sec);
+	buf[1] = bin2bcd(alrm->time.tm_min);
+	buf[2] = bin2bcd(alrm->time.tm_hour);
+	buf[3] = bin2bcd(alrm->time.tm_mday);
+	buf[4] = PCF85063_AEN; /* Do not match on week day */
+
+	ret = regmap_update_bits(pcf85063->regmap, PCF85063_REG_CTRL2,
+				 PCF85063_CTRL2_AIE | PCF85063_CTRL2_AF, 0);
+	if (ret)
+		return ret;
+
+	ret = regmap_bulk_write(pcf85063->regmap, PCF85063_REG_ALM_S,
+				buf, sizeof(buf));
+	if (ret)
+		return ret;
+
+	return regmap_update_bits(pcf85063->regmap, PCF85063_REG_CTRL2,
+				  PCF85063_CTRL2_AIE | PCF85063_CTRL2_AF,
+				  alrm->enabled ? PCF85063_CTRL2_AIE | PCF85063_CTRL2_AF : PCF85063_CTRL2_AF);
+}
+
+static int pcf85063_rtc_alarm_irq_enable(struct device *dev,
+					 unsigned int enabled)
+{
+	struct pcf85063 *pcf85063 = dev_get_drvdata(dev);
+
+	return regmap_update_bits(pcf85063->regmap, PCF85063_REG_CTRL2,
+				  PCF85063_CTRL2_AIE,
+				  enabled ? PCF85063_CTRL2_AIE : 0);
+}
+
+static irqreturn_t pcf85063_rtc_handle_irq(int irq, void *dev_id)
+{
+	struct pcf85063 *pcf85063 = dev_id;
+	unsigned int val;
+	int err;
+
+	err = regmap_read(pcf85063->regmap, PCF85063_REG_CTRL2, &val);
+	if (err)
+		return IRQ_NONE;
+
+	if (val & PCF85063_CTRL2_AF) {
+		rtc_update_irq(pcf85063->rtc, 1, RTC_IRQF | RTC_AF);
+		regmap_update_bits(pcf85063->regmap, PCF85063_REG_CTRL2,
+				   PCF85063_CTRL2_AIE | PCF85063_CTRL2_AF,
+				   0);
+		return IRQ_HANDLED;
+	}
+
+	return IRQ_NONE;
+}
+
+static int pcf85063_read_offset(struct device *dev, long *offset)
+{
+	struct pcf85063 *pcf85063 = dev_get_drvdata(dev);
+	long val;
+	u32 reg;
+	int ret;
+
+	ret = regmap_read(pcf85063->regmap, PCF85063_REG_OFFSET, &reg);
+	if (ret < 0)
+		return ret;
+
+	val = sign_extend32(reg & ~PCF85063_OFFSET_MODE,
+			    PCF85063_OFFSET_SIGN_BIT);
+
+	if (reg & PCF85063_OFFSET_MODE)
+		*offset = val * PCF85063_OFFSET_STEP1;
+	else
+		*offset = val * PCF85063_OFFSET_STEP0;
+
+	return 0;
+}
+
+static int pcf85063_set_offset(struct device *dev, long offset)
+{
+	struct pcf85063 *pcf85063 = dev_get_drvdata(dev);
+	s8 mode0, mode1, reg;
+	unsigned int error0, error1;
+
+	if (offset > PCF85063_OFFSET_STEP0 * 63)
+		return -ERANGE;
+	if (offset < PCF85063_OFFSET_STEP0 * -64)
+		return -ERANGE;
+
+	mode0 = DIV_ROUND_CLOSEST(offset, PCF85063_OFFSET_STEP0);
+	mode1 = DIV_ROUND_CLOSEST(offset, PCF85063_OFFSET_STEP1);
+
+	error0 = abs(offset - (mode0 * PCF85063_OFFSET_STEP0));
+	error1 = abs(offset - (mode1 * PCF85063_OFFSET_STEP1));
+	if (mode1 > 63 || mode1 < -64 || error0 < error1)
+		reg = mode0 & ~PCF85063_OFFSET_MODE;
+	else
+		reg = mode1 | PCF85063_OFFSET_MODE;
+
+	return regmap_write(pcf85063->regmap, PCF85063_REG_OFFSET, reg);
+}
+
+static int pcf85063_ioctl(struct device *dev, unsigned int cmd,
+			  unsigned long arg)
+{
+	struct pcf85063 *pcf85063 = dev_get_drvdata(dev);
+	int status, ret = 0;
+
+	switch (cmd) {
+	case RTC_VL_READ:
+		ret = regmap_read(pcf85063->regmap, PCF85063_REG_SC, &status);
+		if (ret < 0)
+			return ret;
+
+		if (status & PCF85063_REG_SC_OS)
+			dev_warn(&pcf85063->rtc->dev, "Voltage low, data loss detected.\n");
+
+		status &= PCF85063_REG_SC_OS;
+
+		if (copy_to_user((void __user *)arg, &status, sizeof(int)))
+			return -EFAULT;
+
+		return 0;
+
+	case RTC_VL_CLR:
+		ret = regmap_update_bits(pcf85063->regmap, PCF85063_REG_SC,
+					 PCF85063_REG_SC_OS, 0);
+
+		return ret;
+
+	default:
+		return -ENOIOCTLCMD;
+	}
+}
+
 static const struct rtc_class_ops pcf85063_rtc_ops = {
 	.read_time	= pcf85063_rtc_read_time,
-	.set_time	= pcf85063_rtc_set_time
+	.set_time	= pcf85063_rtc_set_time,
+	.read_offset	= pcf85063_read_offset,
+	.set_offset	= pcf85063_set_offset,
+	.ioctl		= pcf85063_ioctl,
+};
+
+static const struct rtc_class_ops pcf85063_rtc_ops_alarm = {
+	.read_time	= pcf85063_rtc_read_time,
+	.set_time	= pcf85063_rtc_set_time,
+	.read_offset	= pcf85063_read_offset,
+	.set_offset	= pcf85063_set_offset,
+	.read_alarm	= pcf85063_rtc_read_alarm,
+	.set_alarm	= pcf85063_rtc_set_alarm,
+	.alarm_irq_enable = pcf85063_rtc_alarm_irq_enable,
+	.ioctl		= pcf85063_ioctl,
 };
 
-static int pcf85063_load_capacitance(struct i2c_client *client)
+static int pcf85063_nvmem_read(void *priv, unsigned int offset,
+			       void *val, size_t bytes)
 {
-	u32 load;
-	int rc;
-	u8 reg;
+	return regmap_read(priv, PCF85063_REG_RAM, val);
+}
 
-	rc = i2c_smbus_read_byte_data(client, PCF85063_REG_CTRL1);
-	if (rc < 0)
-		return rc;
+static int pcf85063_nvmem_write(void *priv, unsigned int offset,
+				void *val, size_t bytes)
+{
+	return regmap_write(priv, PCF85063_REG_RAM, *(u8 *)val);
+}
 
-	reg = rc;
-	load = 7000;
-	of_property_read_u32(client->dev.of_node, "quartz-load-femtofarads",
-			     &load);
+static int pcf85063_load_capacitance(struct pcf85063 *pcf85063,
+				     const struct device_node *np,
+				     unsigned int force_cap)
+{
+	u32 load = 7000;
+	u8 reg = 0;
+
+	if (force_cap)
+		load = force_cap;
+	else
+		of_property_read_u32(np, "quartz-load-femtofarads", &load);
 
 	switch (load) {
 	default:
-		dev_warn(&client->dev, "Unknown quartz-load-femtofarads value: %d. Assuming 7000",
+		dev_warn(&pcf85063->rtc->dev, "Unknown quartz-load-femtofarads value: %d. Assuming 7000",
 			 load);
 		/* fall through */
 	case 7000:
-		reg &= ~PCF85063_REG_CTRL1_CAP_SEL;
 		break;
 	case 12500:
-		reg |= PCF85063_REG_CTRL1_CAP_SEL;
+		reg = PCF85063_REG_CTRL1_CAP_SEL;
 		break;
 	}
 
-	rc = i2c_smbus_write_byte_data(client, PCF85063_REG_CTRL1, reg);
-
-	return rc;
+	return regmap_update_bits(pcf85063->regmap, PCF85063_REG_CTRL1,
+				  PCF85063_REG_CTRL1_CAP_SEL, reg);
 }
 
-static int pcf85063_probe(struct i2c_client *client,
-				const struct i2c_device_id *id)
+static const struct pcf85063_config pcf85063a_config = {
+	.regmap = {
+		.reg_bits = 8,
+		.val_bits = 8,
+		.max_register = 0x11,
+	},
+	.has_alarms = 1,
+};
+
+static const struct pcf85063_config pcf85063tp_config = {
+	.regmap = {
+		.reg_bits = 8,
+		.val_bits = 8,
+		.max_register = 0x0a,
+	},
+};
+
+static const struct pcf85063_config rv8263_config = {
+	.regmap = {
+		.reg_bits = 8,
+		.val_bits = 8,
+		.max_register = 0x11,
+	},
+	.has_alarms = 1,
+	.force_cap_7000 = 1,
+};
+
+static int pcf85063_probe(struct i2c_client *client)
 {
-	struct rtc_device *rtc;
+	struct pcf85063 *pcf85063;
+	unsigned int tmp;
 	int err;
+	const struct pcf85063_config *config = &pcf85063tp_config;
+	const void *data = of_device_get_match_data(&client->dev);
+	struct nvmem_config nvmem_cfg = {
+		.name = "pcf85063_nvram",
+		.reg_read = pcf85063_nvmem_read,
+		.reg_write = pcf85063_nvmem_write,
+		.type = NVMEM_TYPE_BATTERY_BACKED,
+		.size = 1,
+	};
 
 	dev_dbg(&client->dev, "%s\n", __func__);
 
-	if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C))
-		return -ENODEV;
+	pcf85063 = devm_kzalloc(&client->dev, sizeof(struct pcf85063),
+				GFP_KERNEL);
+	if (!pcf85063)
+		return -ENOMEM;
+
+	if (data)
+		config = data;
+
+	pcf85063->regmap = devm_regmap_init_i2c(client, &config->regmap);
+	if (IS_ERR(pcf85063->regmap))
+		return PTR_ERR(pcf85063->regmap);
 
-	err = i2c_smbus_read_byte_data(client, PCF85063_REG_CTRL1);
-	if (err < 0) {
+	i2c_set_clientdata(client, pcf85063);
+
+	err = regmap_read(pcf85063->regmap, PCF85063_REG_CTRL1, &tmp);
+	if (err) {
 		dev_err(&client->dev, "RTC chip is not present\n");
 		return err;
 	}
 
-	err = pcf85063_load_capacitance(client);
+	pcf85063->rtc = devm_rtc_allocate_device(&client->dev);
+	if (IS_ERR(pcf85063->rtc))
+		return PTR_ERR(pcf85063->rtc);
+
+	err = pcf85063_load_capacitance(pcf85063, client->dev.of_node,
+					config->force_cap_7000 ? 7000 : 0);
 	if (err < 0)
 		dev_warn(&client->dev, "failed to set xtal load capacitance: %d",
 			 err);
 
-	rtc = devm_rtc_device_register(&client->dev,
-				       pcf85063_driver.driver.name,
-				       &pcf85063_rtc_ops, THIS_MODULE);
+	pcf85063->rtc->ops = &pcf85063_rtc_ops;
+	pcf85063->rtc->range_min = RTC_TIMESTAMP_BEGIN_2000;
+	pcf85063->rtc->range_max = RTC_TIMESTAMP_END_2099;
+	pcf85063->rtc->uie_unsupported = 1;
+
+	if (config->has_alarms && client->irq > 0) {
+		err = devm_request_threaded_irq(&client->dev, client->irq,
+						NULL, pcf85063_rtc_handle_irq,
+						IRQF_TRIGGER_LOW | IRQF_ONESHOT,
+						"pcf85063", pcf85063);
+		if (err) {
+			dev_warn(&pcf85063->rtc->dev,
+				 "unable to request IRQ, alarms disabled\n");
+		} else {
+			pcf85063->rtc->ops = &pcf85063_rtc_ops_alarm;
+			device_init_wakeup(&client->dev, true);
+			err = dev_pm_set_wake_irq(&client->dev, client->irq);
+			if (err)
+				dev_err(&pcf85063->rtc->dev,
+					"failed to enable irq wake\n");
+		}
+	}
+
+	nvmem_cfg.priv = pcf85063->regmap;
+	rtc_nvmem_register(pcf85063->rtc, &nvmem_cfg);
 
-	return PTR_ERR_OR_ZERO(rtc);
+	return rtc_register_device(pcf85063->rtc);
 }
 
-static const struct i2c_device_id pcf85063_id[] = {
-	{ "pcf85063", 0 },
-	{ }
-};
-MODULE_DEVICE_TABLE(i2c, pcf85063_id);
-
 #ifdef CONFIG_OF
 static const struct of_device_id pcf85063_of_match[] = {
-	{ .compatible = "nxp,pcf85063" },
+	{ .compatible = "nxp,pcf85063", .data = &pcf85063tp_config },
+	{ .compatible = "nxp,pcf85063tp", .data = &pcf85063tp_config },
+	{ .compatible = "nxp,pcf85063a", .data = &pcf85063a_config },
+	{ .compatible = "microcrystal,rv8263", .data = &rv8263_config },
 	{}
 };
 MODULE_DEVICE_TABLE(of, pcf85063_of_match);
@@ -255,8 +488,7 @@ static struct i2c_driver pcf85063_driver = {
 		.name	= "rtc-pcf85063",
 		.of_match_table = of_match_ptr(pcf85063_of_match),
 	},
-	.probe		= pcf85063_probe,
-	.id_table	= pcf85063_id,
+	.probe_new	= pcf85063_probe,
 };
 
 module_i2c_driver(pcf85063_driver);
diff --git a/drivers/rtc/rtc-pcf85363.c b/drivers/rtc/rtc-pcf85363.c
index a3988079f60a..a075e77617dc 100644
--- a/drivers/rtc/rtc-pcf85363.c
+++ b/drivers/rtc/rtc-pcf85363.c
@@ -1,15 +1,10 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * drivers/rtc/rtc-pcf85363.c
  *
  * Driver for NXP PCF85363 real-time clock.
  *
  * Copyright (C) 2017 Eric Nelson
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- * Based loosely on rtc-8583 by Russell King, Wolfram Sang and Juergen Beisert
  */
 #include <linux/module.h>
 #include <linux/i2c.h>
@@ -112,10 +107,7 @@
 
 #define NVRAM_SIZE	0x40
 
-static struct i2c_driver pcf85363_driver;
-
 struct pcf85363 {
-	struct device		*dev;
 	struct rtc_device	*rtc;
 	struct regmap		*regmap;
 };
@@ -386,9 +378,6 @@ static int pcf85363_probe(struct i2c_client *client,
 	if (data)
 		config = data;
 
-	if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C))
-		return -ENODEV;
-
 	pcf85363 = devm_kzalloc(&client->dev, sizeof(struct pcf85363),
 				GFP_KERNEL);
 	if (!pcf85363)
@@ -400,20 +389,21 @@ static int pcf85363_probe(struct i2c_client *client,
 		return PTR_ERR(pcf85363->regmap);
 	}
 
-	pcf85363->dev = &client->dev;
 	i2c_set_clientdata(client, pcf85363);
 
-	pcf85363->rtc = devm_rtc_allocate_device(pcf85363->dev);
+	pcf85363->rtc = devm_rtc_allocate_device(&client->dev);
 	if (IS_ERR(pcf85363->rtc))
 		return PTR_ERR(pcf85363->rtc);
 
 	pcf85363->rtc->ops = &rtc_ops;
+	pcf85363->rtc->range_min = RTC_TIMESTAMP_BEGIN_2000;
+	pcf85363->rtc->range_max = RTC_TIMESTAMP_END_2099;
 
 	if (client->irq > 0) {
 		regmap_write(pcf85363->regmap, CTRL_FLAGS, 0);
 		regmap_update_bits(pcf85363->regmap, CTRL_PIN_IO,
 				   PIN_IO_INTA_OUT, PIN_IO_INTAPM);
-		ret = devm_request_threaded_irq(pcf85363->dev, client->irq,
+		ret = devm_request_threaded_irq(&client->dev, client->irq,
 						NULL, pcf85363_rtc_handle_irq,
 						IRQF_TRIGGER_LOW | IRQF_ONESHOT,
 						"pcf85363", client);
diff --git a/drivers/rtc/rtc-ps3.c b/drivers/rtc/rtc-ps3.c
index 347288bff438..f0336d691e6c 100644
--- a/drivers/rtc/rtc-ps3.c
+++ b/drivers/rtc/rtc-ps3.c
@@ -1,20 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * PS3 RTC Driver
  *
  * Copyright 2009 Sony Corporation
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; version 2 of the License.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.
- * If not, see <http://www.gnu.org/licenses/>.
  */
 
 #include <linux/kernel.h>
@@ -40,16 +28,13 @@ static u64 read_rtc(void)
 
 static int ps3_get_time(struct device *dev, struct rtc_time *tm)
 {
-	rtc_time_to_tm(read_rtc() + ps3_os_area_get_rtc_diff(), tm);
+	rtc_time64_to_tm(read_rtc() + ps3_os_area_get_rtc_diff(), tm);
 	return 0;
 }
 
 static int ps3_set_time(struct device *dev, struct rtc_time *tm)
 {
-	unsigned long now;
-
-	rtc_tm_to_time(tm, &now);
-	ps3_os_area_set_rtc_diff(now - read_rtc());
+	ps3_os_area_set_rtc_diff(rtc_tm_to_time64(tm) - read_rtc());
 	return 0;
 }
 
@@ -62,13 +47,16 @@ static int __init ps3_rtc_probe(struct platform_device *dev)
 {
 	struct rtc_device *rtc;
 
-	rtc = devm_rtc_device_register(&dev->dev, "rtc-ps3", &ps3_rtc_ops,
-				  THIS_MODULE);
+	rtc = devm_rtc_allocate_device(&dev->dev);
 	if (IS_ERR(rtc))
 		return PTR_ERR(rtc);
 
+	rtc->ops = &ps3_rtc_ops;
+	rtc->range_max = U64_MAX;
+
 	platform_set_drvdata(dev, rtc);
-	return 0;
+
+	return rtc_register_device(rtc);
 }
 
 static struct platform_driver ps3_rtc_driver = {
diff --git a/drivers/rtc/rtc-pxa.c b/drivers/rtc/rtc-pxa.c
index e1887b86fdc7..d4766734e40b 100644
--- a/drivers/rtc/rtc-pxa.c
+++ b/drivers/rtc/rtc-pxa.c
@@ -145,8 +145,7 @@ static void rtsr_set_bits(struct pxa_rtc *pxa_rtc, u32 mask)
 
 static irqreturn_t pxa_rtc_irq(int irq, void *dev_id)
 {
-	struct platform_device *pdev = to_platform_device(dev_id);
-	struct pxa_rtc *pxa_rtc = platform_get_drvdata(pdev);
+	struct pxa_rtc *pxa_rtc = dev_get_drvdata(dev_id);
 	u32 rtsr;
 	unsigned long events = 0;
 
diff --git a/drivers/rtc/rtc-rk808.c b/drivers/rtc/rtc-rk808.c
index 1fb864d4ef83..5c5d9f125669 100644
--- a/drivers/rtc/rtc-rk808.c
+++ b/drivers/rtc/rtc-rk808.c
@@ -336,8 +336,7 @@ static const struct rtc_class_ops rk808_rtc_ops = {
 /* Turn off the alarm if it should not be a wake source. */
 static int rk808_rtc_suspend(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct rk808_rtc *rk808_rtc = dev_get_drvdata(&pdev->dev);
+	struct rk808_rtc *rk808_rtc = dev_get_drvdata(dev);
 
 	if (device_may_wakeup(dev))
 		enable_irq_wake(rk808_rtc->irq);
@@ -350,8 +349,7 @@ static int rk808_rtc_suspend(struct device *dev)
  */
 static int rk808_rtc_resume(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct rk808_rtc *rk808_rtc = dev_get_drvdata(&pdev->dev);
+	struct rk808_rtc *rk808_rtc = dev_get_drvdata(dev);
 
 	if (device_may_wakeup(dev))
 		disable_irq_wake(rk808_rtc->irq);
diff --git a/drivers/rtc/rtc-rx6110.c b/drivers/rtc/rtc-rx6110.c
index 5899ca368d59..71e20a6bd387 100644
--- a/drivers/rtc/rtc-rx6110.c
+++ b/drivers/rtc/rtc-rx6110.c
@@ -21,6 +21,8 @@
 #include <linux/of_gpio.h>
 #include <linux/regmap.h>
 #include <linux/rtc.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
 #include <linux/spi/spi.h>
 
 /* RX-6110 Register definitions */
@@ -379,9 +381,16 @@ static const struct spi_device_id rx6110_id[] = {
 };
 MODULE_DEVICE_TABLE(spi, rx6110_id);
 
+static const struct of_device_id rx6110_spi_of_match[] = {
+	{ .compatible = "epson,rx6110" },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, rx6110_spi_of_match);
+
 static struct spi_driver rx6110_driver = {
 	.driver = {
 		.name = RX6110_DRIVER_NAME,
+		.of_match_table = of_match_ptr(rx6110_spi_of_match),
 	},
 	.probe		= rx6110_probe,
 	.remove		= rx6110_remove,
diff --git a/drivers/rtc/rtc-rx8025.c b/drivers/rtc/rtc-rx8025.c
index 41de38acc570..fddc996cb38d 100644
--- a/drivers/rtc/rtc-rx8025.c
+++ b/drivers/rtc/rtc-rx8025.c
@@ -311,7 +311,7 @@ static int rx8025_read_alarm(struct device *dev, struct rtc_wkalrm *t)
 		t->time.tm_hour = bcd2bin(ald[1] & 0x1f) % 12
 			+ (ald[1] & 0x20 ? 12 : 0);
 
-	dev_dbg(dev, "%s: date: %ptRr\n", __func__, t);
+	dev_dbg(dev, "%s: date: %ptRr\n", __func__, &t->time);
 	t->enabled = !!(rx8025->ctrl1 & RX8025_BIT_CTRL1_DALE);
 	t->pending = (ctrl2 & RX8025_BIT_CTRL2_DAFG) && t->enabled;
 
diff --git a/drivers/rtc/rtc-sh.c b/drivers/rtc/rtc-sh.c
index 1d3de2a3d1a4..579b3ff5c644 100644
--- a/drivers/rtc/rtc-sh.c
+++ b/drivers/rtc/rtc-sh.c
@@ -276,6 +276,9 @@ static int sh_rtc_read_time(struct device *dev, struct rtc_time *tm)
 	struct sh_rtc *rtc = dev_get_drvdata(dev);
 	unsigned int sec128, sec2, yr, yr100, cf_bit;
 
+	if (!(readb(rtc->regbase + RCR2) & RCR2_RTCEN))
+		return -EINVAL;
+
 	do {
 		unsigned int tmp;
 
@@ -466,7 +469,6 @@ static int __init sh_rtc_probe(struct platform_device *pdev)
 {
 	struct sh_rtc *rtc;
 	struct resource *res;
-	struct rtc_time r;
 	char clk_name[6];
 	int clk_id, ret;
 
@@ -528,6 +530,10 @@ static int __init sh_rtc_probe(struct platform_device *pdev)
 		rtc->clk = NULL;
 	}
 
+	rtc->rtc_dev = devm_rtc_allocate_device(&pdev->dev);
+	if (IS_ERR(rtc->rtc_dev))
+		return PTR_ERR(rtc->rtc_dev);
+
 	clk_enable(rtc->clk);
 
 	rtc->capabilities = RTC_DEF_CAPABILITIES;
@@ -591,21 +597,21 @@ static int __init sh_rtc_probe(struct platform_device *pdev)
 	sh_rtc_setaie(&pdev->dev, 0);
 	sh_rtc_setcie(&pdev->dev, 0);
 
-	rtc->rtc_dev = devm_rtc_device_register(&pdev->dev, "sh",
-					   &sh_rtc_ops, THIS_MODULE);
-	if (IS_ERR(rtc->rtc_dev)) {
-		ret = PTR_ERR(rtc->rtc_dev);
-		goto err_unmap;
-	}
-
+	rtc->rtc_dev->ops = &sh_rtc_ops;
 	rtc->rtc_dev->max_user_freq = 256;
 
-	/* reset rtc to epoch 0 if time is invalid */
-	if (rtc_read_time(rtc->rtc_dev, &r) < 0) {
-		rtc_time_to_tm(0, &r);
-		rtc_set_time(rtc->rtc_dev, &r);
+	if (rtc->capabilities & RTC_CAP_4_DIGIT_YEAR) {
+		rtc->rtc_dev->range_min = RTC_TIMESTAMP_BEGIN_1900;
+		rtc->rtc_dev->range_max = RTC_TIMESTAMP_END_9999;
+	} else {
+		rtc->rtc_dev->range_min = mktime64(1999, 1, 1, 0, 0, 0);
+		rtc->rtc_dev->range_max = mktime64(2098, 12, 31, 23, 59, 59);
 	}
 
+	ret = rtc_register_device(rtc->rtc_dev);
+	if (ret)
+		goto err_unmap;
+
 	device_init_wakeup(&pdev->dev, 1);
 	return 0;
 
diff --git a/drivers/rtc/rtc-sirfsoc.c b/drivers/rtc/rtc-sirfsoc.c
index 2a9e151cae99..9ba28d1ebd87 100644
--- a/drivers/rtc/rtc-sirfsoc.c
+++ b/drivers/rtc/rtc-sirfsoc.c
@@ -279,7 +279,7 @@ static const struct of_device_id sirfsoc_rtc_of_match[] = {
 	{},
 };
 
-const struct regmap_config sysrtc_regmap_config = {
+static const struct regmap_config sysrtc_regmap_config = {
 	.reg_bits = 32,
 	.val_bits = 32,
 	.fast_io = true,
diff --git a/drivers/rtc/rtc-snvs.c b/drivers/rtc/rtc-snvs.c
index 0b9eff19149b..7ee673a25fd0 100644
--- a/drivers/rtc/rtc-snvs.c
+++ b/drivers/rtc/rtc-snvs.c
@@ -9,6 +9,7 @@
 #include <linux/of.h>
 #include <linux/of_device.h>
 #include <linux/platform_device.h>
+#include <linux/pm_wakeirq.h>
 #include <linux/rtc.h>
 #include <linux/clk.h>
 #include <linux/mfd/syscon.h>
@@ -271,7 +272,6 @@ static const struct regmap_config snvs_rtc_config = {
 static int snvs_rtc_probe(struct platform_device *pdev)
 {
 	struct snvs_rtc_data *data;
-	struct resource *res;
 	int ret;
 	void __iomem *mmio;
 
@@ -283,9 +283,8 @@ static int snvs_rtc_probe(struct platform_device *pdev)
 
 	if (IS_ERR(data->regmap)) {
 		dev_warn(&pdev->dev, "snvs rtc: you use old dts file, please update it\n");
-		res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 
-		mmio = devm_ioremap_resource(&pdev->dev, res);
+		mmio = devm_platform_ioremap_resource(pdev, 0);
 		if (IS_ERR(mmio))
 			return PTR_ERR(mmio);
 
@@ -332,6 +331,9 @@ static int snvs_rtc_probe(struct platform_device *pdev)
 	}
 
 	device_init_wakeup(&pdev->dev, true);
+	ret = dev_pm_set_wake_irq(&pdev->dev, data->irq);
+	if (ret)
+		dev_err(&pdev->dev, "failed to enable irq wake\n");
 
 	ret = devm_request_irq(&pdev->dev, data->irq, snvs_rtc_irq_handler,
 			       IRQF_SHARED, "rtc alarm", &pdev->dev);
@@ -358,18 +360,7 @@ error_rtc_device_register:
 	return ret;
 }
 
-#ifdef CONFIG_PM_SLEEP
-static int snvs_rtc_suspend(struct device *dev)
-{
-	struct snvs_rtc_data *data = dev_get_drvdata(dev);
-
-	if (device_may_wakeup(dev))
-		return enable_irq_wake(data->irq);
-
-	return 0;
-}
-
-static int snvs_rtc_suspend_noirq(struct device *dev)
+static int __maybe_unused snvs_rtc_suspend_noirq(struct device *dev)
 {
 	struct snvs_rtc_data *data = dev_get_drvdata(dev);
 
@@ -379,17 +370,7 @@ static int snvs_rtc_suspend_noirq(struct device *dev)
 	return 0;
 }
 
-static int snvs_rtc_resume(struct device *dev)
-{
-	struct snvs_rtc_data *data = dev_get_drvdata(dev);
-
-	if (device_may_wakeup(dev))
-		return disable_irq_wake(data->irq);
-
-	return 0;
-}
-
-static int snvs_rtc_resume_noirq(struct device *dev)
+static int __maybe_unused snvs_rtc_resume_noirq(struct device *dev)
 {
 	struct snvs_rtc_data *data = dev_get_drvdata(dev);
 
@@ -400,20 +381,9 @@ static int snvs_rtc_resume_noirq(struct device *dev)
 }
 
 static const struct dev_pm_ops snvs_rtc_pm_ops = {
-	.suspend = snvs_rtc_suspend,
-	.suspend_noirq = snvs_rtc_suspend_noirq,
-	.resume = snvs_rtc_resume,
-	.resume_noirq = snvs_rtc_resume_noirq,
+	SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(snvs_rtc_suspend_noirq, snvs_rtc_resume_noirq)
 };
 
-#define SNVS_RTC_PM_OPS	(&snvs_rtc_pm_ops)
-
-#else
-
-#define SNVS_RTC_PM_OPS	NULL
-
-#endif
-
 static const struct of_device_id snvs_dt_ids[] = {
 	{ .compatible = "fsl,sec-v4.0-mon-rtc-lp", },
 	{ /* sentinel */ }
@@ -423,7 +393,7 @@ MODULE_DEVICE_TABLE(of, snvs_dt_ids);
 static struct platform_driver snvs_rtc_driver = {
 	.driver = {
 		.name	= "snvs_rtc",
-		.pm	= SNVS_RTC_PM_OPS,
+		.pm	= &snvs_rtc_pm_ops,
 		.of_match_table = snvs_dt_ids,
 	},
 	.probe		= snvs_rtc_probe,
diff --git a/drivers/rtc/rtc-stm32.c b/drivers/rtc/rtc-stm32.c
index c5908cfea234..8e6c9b3bcc29 100644
--- a/drivers/rtc/rtc-stm32.c
+++ b/drivers/rtc/rtc-stm32.c
@@ -788,11 +788,14 @@ static int stm32_rtc_probe(struct platform_device *pdev)
 	ret = device_init_wakeup(&pdev->dev, true);
 	if (rtc->data->has_wakeirq) {
 		rtc->wakeirq_alarm = platform_get_irq(pdev, 1);
-		if (rtc->wakeirq_alarm <= 0)
-			ret = rtc->wakeirq_alarm;
-		else
+		if (rtc->wakeirq_alarm > 0) {
 			ret = dev_pm_set_dedicated_wake_irq(&pdev->dev,
 							    rtc->wakeirq_alarm);
+		} else {
+			ret = rtc->wakeirq_alarm;
+			if (rtc->wakeirq_alarm == -EPROBE_DEFER)
+				goto err;
+		}
 	}
 	if (ret)
 		dev_warn(&pdev->dev, "alarm can't wake up the system: %d", ret);
diff --git a/drivers/rtc/rtc-stmp3xxx.c b/drivers/rtc/rtc-stmp3xxx.c
index b76318fd5bb0..ff6488be385f 100644
--- a/drivers/rtc/rtc-stmp3xxx.c
+++ b/drivers/rtc/rtc-stmp3xxx.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  * Freescale STMP37XX/STMP378X Real Time Clock driver
  *
@@ -8,15 +9,6 @@
  * Copyright 2008 Embedded Alley Solutions, Inc All Rights Reserved.
  * Copyright 2011 Wolfram Sang, Pengutronix e.K.
  */
-
-/*
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
- */
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/io.h>
@@ -160,15 +152,15 @@ static int stmp3xxx_rtc_gettime(struct device *dev, struct rtc_time *rtc_tm)
 	if (ret)
 		return ret;
 
-	rtc_time_to_tm(readl(rtc_data->io + STMP3XXX_RTC_SECONDS), rtc_tm);
+	rtc_time64_to_tm(readl(rtc_data->io + STMP3XXX_RTC_SECONDS), rtc_tm);
 	return 0;
 }
 
-static int stmp3xxx_rtc_set_mmss(struct device *dev, unsigned long t)
+static int stmp3xxx_rtc_settime(struct device *dev, struct rtc_time *rtc_tm)
 {
 	struct stmp3xxx_rtc_data *rtc_data = dev_get_drvdata(dev);
 
-	writel(t, rtc_data->io + STMP3XXX_RTC_SECONDS);
+	writel(rtc_tm_to_time64(rtc_tm), rtc_data->io + STMP3XXX_RTC_SECONDS);
 	return stmp3xxx_wait_time(rtc_data);
 }
 
@@ -214,17 +206,15 @@ static int stmp3xxx_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alm)
 {
 	struct stmp3xxx_rtc_data *rtc_data = dev_get_drvdata(dev);
 
-	rtc_time_to_tm(readl(rtc_data->io + STMP3XXX_RTC_ALARM), &alm->time);
+	rtc_time64_to_tm(readl(rtc_data->io + STMP3XXX_RTC_ALARM), &alm->time);
 	return 0;
 }
 
 static int stmp3xxx_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alm)
 {
-	unsigned long t;
 	struct stmp3xxx_rtc_data *rtc_data = dev_get_drvdata(dev);
 
-	rtc_tm_to_time(&alm->time, &t);
-	writel(t, rtc_data->io + STMP3XXX_RTC_ALARM);
+	writel(rtc_tm_to_time64(&alm->time), rtc_data->io + STMP3XXX_RTC_ALARM);
 
 	stmp3xxx_alarm_irq_enable(dev, alm->enabled);
 
@@ -235,7 +225,7 @@ static const struct rtc_class_ops stmp3xxx_rtc_ops = {
 	.alarm_irq_enable =
 			  stmp3xxx_alarm_irq_enable,
 	.read_time	= stmp3xxx_rtc_gettime,
-	.set_mmss	= stmp3xxx_rtc_set_mmss,
+	.set_time	= stmp3xxx_rtc_settime,
 	.read_alarm	= stmp3xxx_rtc_read_alarm,
 	.set_alarm	= stmp3xxx_rtc_set_alarm,
 };
@@ -361,8 +351,7 @@ static int stmp3xxx_rtc_probe(struct platform_device *pdev)
 			STMP3XXX_RTC_CTRL_ALARM_IRQ_EN,
 		rtc_data->io + STMP3XXX_RTC_CTRL + STMP_OFFSET_REG_CLR);
 
-	rtc_data->rtc = devm_rtc_device_register(&pdev->dev, pdev->name,
-				&stmp3xxx_rtc_ops, THIS_MODULE);
+	rtc_data->rtc = devm_rtc_allocate_device(&pdev->dev);
 	if (IS_ERR(rtc_data->rtc))
 		return PTR_ERR(rtc_data->rtc);
 
@@ -374,6 +363,13 @@ static int stmp3xxx_rtc_probe(struct platform_device *pdev)
 		return err;
 	}
 
+	rtc_data->rtc->ops = &stmp3xxx_rtc_ops;
+	rtc_data->rtc->range_max = U32_MAX;
+
+	err = rtc_register_device(rtc_data->rtc);
+	if (err)
+		return err;
+
 	stmp3xxx_wdt_register(pdev);
 	return 0;
 }
diff --git a/drivers/rtc/rtc-sun4v.c b/drivers/rtc/rtc-sun4v.c
index 11bc562eba5d..036463dfa103 100644
--- a/drivers/rtc/rtc-sun4v.c
+++ b/drivers/rtc/rtc-sun4v.c
@@ -1,7 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
 /* rtc-sun4v.c: Hypervisor based RTC for SUN4V systems.
  *
  * Author: David S. Miller
- * License: GPL
  *
  * Copyright (C) 2008 David S. Miller <davem@davemloft.net>
  */
@@ -39,7 +39,7 @@ retry:
 
 static int sun4v_read_time(struct device *dev, struct rtc_time *tm)
 {
-	rtc_time_to_tm(hypervisor_get_time(), tm);
+	rtc_time64_to_tm(hypervisor_get_time(), tm);
 	return 0;
 }
 
@@ -66,14 +66,7 @@ retry:
 
 static int sun4v_set_time(struct device *dev, struct rtc_time *tm)
 {
-	unsigned long secs;
-	int err;
-
-	err = rtc_tm_to_time(tm, &secs);
-	if (err)
-		return err;
-
-	return hypervisor_set_time(secs);
+	return hypervisor_set_time(rtc_tm_to_time64(tm));
 }
 
 static const struct rtc_class_ops sun4v_rtc_ops = {
@@ -85,13 +78,15 @@ static int __init sun4v_rtc_probe(struct platform_device *pdev)
 {
 	struct rtc_device *rtc;
 
-	rtc = devm_rtc_device_register(&pdev->dev, "sun4v",
-				&sun4v_rtc_ops, THIS_MODULE);
+	rtc = devm_rtc_allocate_device(&pdev->dev);
 	if (IS_ERR(rtc))
 		return PTR_ERR(rtc);
 
+	rtc->ops = &sun4v_rtc_ops;
+	rtc->range_max = U64_MAX;
 	platform_set_drvdata(pdev, rtc);
-	return 0;
+
+	return rtc_register_device(rtc);
 }
 
 static struct platform_driver sun4v_rtc_driver = {
diff --git a/drivers/rtc/rtc-tegra.c b/drivers/rtc/rtc-tegra.c
index c6b0a99aa3a9..f0ce76865434 100644
--- a/drivers/rtc/rtc-tegra.c
+++ b/drivers/rtc/rtc-tegra.c
@@ -1,21 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  * An RTC driver for the NVIDIA Tegra 200 series internal RTC.
  *
  * Copyright (c) 2010, NVIDIA Corporation.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 
 #include <linux/clk.h>
@@ -123,7 +110,7 @@ static int tegra_rtc_read_time(struct device *dev, struct rtc_time *tm)
 
 	spin_unlock_irqrestore(&info->tegra_rtc_lock, sl_irq_flags);
 
-	rtc_time_to_tm(sec, tm);
+	rtc_time64_to_tm(sec, tm);
 
 	dev_vdbg(dev, "time read as %lu. %ptR\n", sec, tm);
 
@@ -137,7 +124,7 @@ static int tegra_rtc_set_time(struct device *dev, struct rtc_time *tm)
 	int ret;
 
 	/* convert tm to seconds. */
-	rtc_tm_to_time(tm, &sec);
+	sec = rtc_tm_to_time64(tm);
 
 	dev_vdbg(dev, "time set to %lu. %ptR\n", sec, tm);
 
@@ -166,7 +153,7 @@ static int tegra_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alarm)
 	} else {
 		/* alarm is enabled. */
 		alarm->enabled = 1;
-		rtc_time_to_tm(sec, &alarm->time);
+		rtc_time64_to_tm(sec, &alarm->time);
 	}
 
 	tmp = readl(info->rtc_base + TEGRA_RTC_REG_INTR_STATUS);
@@ -204,7 +191,7 @@ static int tegra_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alarm)
 	unsigned long sec;
 
 	if (alarm->enabled)
-		rtc_tm_to_time(&alarm->time, &sec);
+		sec = rtc_tm_to_time64(&alarm->time);
 	else
 		sec = 0;
 
@@ -306,6 +293,13 @@ static int __init tegra_rtc_probe(struct platform_device *pdev)
 
 	info->tegra_rtc_irq = ret;
 
+	info->rtc_dev = devm_rtc_allocate_device(&pdev->dev);
+	if (IS_ERR(info->rtc_dev))
+		return PTR_ERR(info->rtc_dev);
+
+	info->rtc_dev->ops = &tegra_rtc_ops;
+	info->rtc_dev->range_max = U32_MAX;
+
 	info->clk = devm_clk_get(&pdev->dev, NULL);
 	if (IS_ERR(info->clk))
 		return PTR_ERR(info->clk);
@@ -327,16 +321,6 @@ static int __init tegra_rtc_probe(struct platform_device *pdev)
 
 	device_init_wakeup(&pdev->dev, 1);
 
-	info->rtc_dev = devm_rtc_device_register(&pdev->dev,
-				dev_name(&pdev->dev), &tegra_rtc_ops,
-				THIS_MODULE);
-	if (IS_ERR(info->rtc_dev)) {
-		ret = PTR_ERR(info->rtc_dev);
-		dev_err(&pdev->dev, "Unable to register device (err=%d).\n",
-			ret);
-		goto disable_clk;
-	}
-
 	ret = devm_request_irq(&pdev->dev, info->tegra_rtc_irq,
 			tegra_rtc_irq_handler, IRQF_TRIGGER_HIGH,
 			dev_name(&pdev->dev), &pdev->dev);
@@ -347,6 +331,13 @@ static int __init tegra_rtc_probe(struct platform_device *pdev)
 		goto disable_clk;
 	}
 
+	ret = rtc_register_device(info->rtc_dev);
+	if (ret) {
+		dev_err(&pdev->dev, "Unable to register device (err=%d).\n",
+			ret);
+		goto disable_clk;
+	}
+
 	dev_notice(&pdev->dev, "Tegra internal Real Time Clock\n");
 
 	return 0;
diff --git a/drivers/rtc/rtc-test.c b/drivers/rtc/rtc-test.c
index 6c5f09c815e8..b298e9902f45 100644
--- a/drivers/rtc/rtc-test.c
+++ b/drivers/rtc/rtc-test.c
@@ -70,11 +70,11 @@ static int test_rtc_read_time(struct device *dev, struct rtc_time *tm)
 	return 0;
 }
 
-static int test_rtc_set_mmss64(struct device *dev, time64_t secs)
+static int test_rtc_set_time(struct device *dev, struct rtc_time *tm)
 {
 	struct rtc_test_data *rtd = dev_get_drvdata(dev);
 
-	rtd->offset = secs - ktime_get_real_seconds();
+	rtd->offset = rtc_tm_to_time64(tm) - ktime_get_real_seconds();
 
 	return 0;
 }
@@ -94,15 +94,15 @@ static int test_rtc_alarm_irq_enable(struct device *dev, unsigned int enable)
 
 static const struct rtc_class_ops test_rtc_ops_noalm = {
 	.read_time = test_rtc_read_time,
-	.set_mmss64 = test_rtc_set_mmss64,
+	.set_time = test_rtc_set_time,
 	.alarm_irq_enable = test_rtc_alarm_irq_enable,
 };
 
 static const struct rtc_class_ops test_rtc_ops = {
 	.read_time = test_rtc_read_time,
+	.set_time = test_rtc_set_time,
 	.read_alarm = test_rtc_read_alarm,
 	.set_alarm = test_rtc_set_alarm,
-	.set_mmss64 = test_rtc_set_mmss64,
 	.alarm_irq_enable = test_rtc_alarm_irq_enable,
 };
 
@@ -152,7 +152,8 @@ static int __init test_init(void)
 {
 	int i, err;
 
-	if ((err = platform_driver_register(&test_driver)))
+	err = platform_driver_register(&test_driver);
+	if (err)
 		return err;
 
 	err = -ENOMEM;
diff --git a/drivers/rtc/rtc-tx4939.c b/drivers/rtc/rtc-tx4939.c
index 2d24babc4057..5a29915a06ec 100644
--- a/drivers/rtc/rtc-tx4939.c
+++ b/drivers/rtc/rtc-tx4939.c
@@ -42,11 +42,6 @@ struct tx4939rtc_plat_data {
 	spinlock_t lock;
 };
 
-static struct tx4939rtc_plat_data *get_tx4939rtc_plat_data(struct device *dev)
-{
-	return platform_get_drvdata(to_platform_device(dev));
-}
-
 static int tx4939_rtc_cmd(struct tx4939_rtc_reg __iomem *rtcreg, int cmd)
 {
 	int i = 0;
@@ -64,7 +59,7 @@ static int tx4939_rtc_cmd(struct tx4939_rtc_reg __iomem *rtcreg, int cmd)
 
 static int tx4939_rtc_set_time(struct device *dev, struct rtc_time *tm)
 {
-	struct tx4939rtc_plat_data *pdata = get_tx4939rtc_plat_data(dev);
+	struct tx4939rtc_plat_data *pdata = dev_get_drvdata(dev);
 	struct tx4939_rtc_reg __iomem *rtcreg = pdata->rtcreg;
 	unsigned long secs = rtc_tm_to_time64(tm);
 	int i, ret;
@@ -89,7 +84,7 @@ static int tx4939_rtc_set_time(struct device *dev, struct rtc_time *tm)
 
 static int tx4939_rtc_read_time(struct device *dev, struct rtc_time *tm)
 {
-	struct tx4939rtc_plat_data *pdata = get_tx4939rtc_plat_data(dev);
+	struct tx4939rtc_plat_data *pdata = dev_get_drvdata(dev);
 	struct tx4939_rtc_reg __iomem *rtcreg = pdata->rtcreg;
 	int i, ret;
 	unsigned long sec;
@@ -115,7 +110,7 @@ static int tx4939_rtc_read_time(struct device *dev, struct rtc_time *tm)
 
 static int tx4939_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 {
-	struct tx4939rtc_plat_data *pdata = get_tx4939rtc_plat_data(dev);
+	struct tx4939rtc_plat_data *pdata = dev_get_drvdata(dev);
 	struct tx4939_rtc_reg __iomem *rtcreg = pdata->rtcreg;
 	int i, ret;
 	unsigned long sec;
@@ -140,7 +135,7 @@ static int tx4939_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 
 static int tx4939_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 {
-	struct tx4939rtc_plat_data *pdata = get_tx4939rtc_plat_data(dev);
+	struct tx4939rtc_plat_data *pdata = dev_get_drvdata(dev);
 	struct tx4939_rtc_reg __iomem *rtcreg = pdata->rtcreg;
 	int i, ret;
 	unsigned long sec;
@@ -170,7 +165,7 @@ static int tx4939_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 
 static int tx4939_rtc_alarm_irq_enable(struct device *dev, unsigned int enabled)
 {
-	struct tx4939rtc_plat_data *pdata = get_tx4939rtc_plat_data(dev);
+	struct tx4939rtc_plat_data *pdata = dev_get_drvdata(dev);
 
 	spin_lock_irq(&pdata->lock);
 	tx4939_rtc_cmd(pdata->rtcreg,
@@ -182,7 +177,7 @@ static int tx4939_rtc_alarm_irq_enable(struct device *dev, unsigned int enabled)
 
 static irqreturn_t tx4939_rtc_interrupt(int irq, void *dev_id)
 {
-	struct tx4939rtc_plat_data *pdata = get_tx4939rtc_plat_data(dev_id);
+	struct tx4939rtc_plat_data *pdata = dev_get_drvdata(dev_id);
 	struct tx4939_rtc_reg __iomem *rtcreg = pdata->rtcreg;
 	unsigned long events = RTC_IRQF;
 
diff --git a/drivers/rtc/rtc-wm831x.c b/drivers/rtc/rtc-wm831x.c
index 7b824dabf104..d2e8b21c90c4 100644
--- a/drivers/rtc/rtc-wm831x.c
+++ b/drivers/rtc/rtc-wm831x.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  *	Real Time Clock driver for Wolfson Microelectronics WM831x
  *
@@ -5,11 +6,6 @@
  *
  *  Author: Mark Brown <broonie@opensource.wolfsonmicro.com>
  *
- *  This program is free software; you can redistribute  it and/or modify it
- *  under  the terms of  the GNU General  Public License as published by the
- *  Free Software Foundation;  either version 2 of the  License, or (at your
- *  option) any later version.
- *
  */
 
 #include <linux/module.h>
@@ -155,7 +151,7 @@ static int wm831x_rtc_readtime(struct device *dev, struct rtc_time *tm)
 		if (memcmp(time1, time2, sizeof(time1)) == 0) {
 			u32 time = (time1[0] << 16) | time1[1];
 
-			rtc_time_to_tm(time, tm);
+			rtc_time64_to_tm(time, tm);
 			return 0;
 		}
 
@@ -169,15 +165,17 @@ static int wm831x_rtc_readtime(struct device *dev, struct rtc_time *tm)
 /*
  * Set current time and date in RTC
  */
-static int wm831x_rtc_set_mmss(struct device *dev, unsigned long time)
+static int wm831x_rtc_settime(struct device *dev, struct rtc_time *tm)
 {
 	struct wm831x_rtc *wm831x_rtc = dev_get_drvdata(dev);
 	struct wm831x *wm831x = wm831x_rtc->wm831x;
 	struct rtc_time new_tm;
-	unsigned long new_time;
+	unsigned long time, new_time;
 	int ret;
 	int count = 0;
 
+	time = rtc_tm_to_time64(tm);
+
 	ret = wm831x_reg_write(wm831x, WM831X_RTC_TIME_1,
 			       (time >> 16) & 0xffff);
 	if (ret < 0) {
@@ -215,11 +213,7 @@ static int wm831x_rtc_set_mmss(struct device *dev, unsigned long time)
 	if (ret < 0)
 		return ret;
 
-	ret = rtc_tm_to_time(&new_tm, &new_time);
-	if (ret < 0) {
-		dev_err(dev, "Failed to convert time: %d\n", ret);
-		return ret;
-	}
+	new_time = rtc_tm_to_time64(&new_tm);
 
 	/* Allow a second of change in case of tick */
 	if (new_time - time > 1) {
@@ -249,7 +243,7 @@ static int wm831x_rtc_readalarm(struct device *dev, struct rtc_wkalrm *alrm)
 
 	time = (data[0] << 16) | data[1];
 
-	rtc_time_to_tm(time, &alrm->time);
+	rtc_time64_to_tm(time, &alrm->time);
 
 	ret = wm831x_reg_read(wm831x_rtc->wm831x, WM831X_RTC_CONTROL);
 	if (ret < 0) {
@@ -288,11 +282,7 @@ static int wm831x_rtc_setalarm(struct device *dev, struct rtc_wkalrm *alrm)
 	int ret;
 	unsigned long time;
 
-	ret = rtc_tm_to_time(&alrm->time, &time);
-	if (ret < 0) {
-		dev_err(dev, "Failed to convert time: %d\n", ret);
-		return ret;
-	}
+	time = rtc_tm_to_time64(&alrm->time);
 
 	ret = wm831x_rtc_stop_alarm(wm831x_rtc);
 	if (ret < 0) {
@@ -346,7 +336,7 @@ static irqreturn_t wm831x_alm_irq(int irq, void *data)
 
 static const struct rtc_class_ops wm831x_rtc_ops = {
 	.read_time = wm831x_rtc_readtime,
-	.set_mmss = wm831x_rtc_set_mmss,
+	.set_time = wm831x_rtc_settime,
 	.read_alarm = wm831x_rtc_readalarm,
 	.set_alarm = wm831x_rtc_setalarm,
 	.alarm_irq_enable = wm831x_rtc_alarm_irq_enable,
@@ -356,11 +346,10 @@ static const struct rtc_class_ops wm831x_rtc_ops = {
 /* Turn off the alarm if it should not be a wake source. */
 static int wm831x_rtc_suspend(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct wm831x_rtc *wm831x_rtc = dev_get_drvdata(&pdev->dev);
+	struct wm831x_rtc *wm831x_rtc = dev_get_drvdata(dev);
 	int ret, enable;
 
-	if (wm831x_rtc->alarm_enabled && device_may_wakeup(&pdev->dev))
+	if (wm831x_rtc->alarm_enabled && device_may_wakeup(dev))
 		enable = WM831X_RTC_ALM_ENA;
 	else
 		enable = 0;
@@ -368,7 +357,7 @@ static int wm831x_rtc_suspend(struct device *dev)
 	ret = wm831x_set_bits(wm831x_rtc->wm831x, WM831X_RTC_CONTROL,
 			      WM831X_RTC_ALM_ENA, enable);
 	if (ret != 0)
-		dev_err(&pdev->dev, "Failed to update RTC alarm: %d\n", ret);
+		dev_err(dev, "Failed to update RTC alarm: %d\n", ret);
 
 	return 0;
 }
@@ -378,15 +367,13 @@ static int wm831x_rtc_suspend(struct device *dev)
  */
 static int wm831x_rtc_resume(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct wm831x_rtc *wm831x_rtc = dev_get_drvdata(&pdev->dev);
+	struct wm831x_rtc *wm831x_rtc = dev_get_drvdata(dev);
 	int ret;
 
 	if (wm831x_rtc->alarm_enabled) {
 		ret = wm831x_rtc_start_alarm(wm831x_rtc);
 		if (ret != 0)
-			dev_err(&pdev->dev,
-				"Failed to restart RTC alarm: %d\n", ret);
+			dev_err(dev, "Failed to restart RTC alarm: %d\n", ret);
 	}
 
 	return 0;
@@ -395,14 +382,13 @@ static int wm831x_rtc_resume(struct device *dev)
 /* Unconditionally disable the alarm */
 static int wm831x_rtc_freeze(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct wm831x_rtc *wm831x_rtc = dev_get_drvdata(&pdev->dev);
+	struct wm831x_rtc *wm831x_rtc = dev_get_drvdata(dev);
 	int ret;
 
 	ret = wm831x_set_bits(wm831x_rtc->wm831x, WM831X_RTC_CONTROL,
 			      WM831X_RTC_ALM_ENA, 0);
 	if (ret != 0)
-		dev_err(&pdev->dev, "Failed to stop RTC alarm: %d\n", ret);
+		dev_err(dev, "Failed to stop RTC alarm: %d\n", ret);
 
 	return 0;
 }
@@ -429,19 +415,23 @@ static int wm831x_rtc_probe(struct platform_device *pdev)
 	ret = wm831x_reg_read(wm831x, WM831X_RTC_CONTROL);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Failed to read RTC control: %d\n", ret);
-		goto err;
+		return ret;
 	}
 	if (ret & WM831X_RTC_ALM_ENA)
 		wm831x_rtc->alarm_enabled = 1;
 
 	device_init_wakeup(&pdev->dev, 1);
 
-	wm831x_rtc->rtc = devm_rtc_device_register(&pdev->dev, "wm831x",
-					      &wm831x_rtc_ops, THIS_MODULE);
-	if (IS_ERR(wm831x_rtc->rtc)) {
-		ret = PTR_ERR(wm831x_rtc->rtc);
-		goto err;
-	}
+	wm831x_rtc->rtc = devm_rtc_allocate_device(&pdev->dev);
+	if (IS_ERR(wm831x_rtc->rtc))
+		return PTR_ERR(wm831x_rtc->rtc);
+
+	wm831x_rtc->rtc->ops = &wm831x_rtc_ops;
+	wm831x_rtc->rtc->range_max = U32_MAX;
+
+	ret = rtc_register_device(wm831x_rtc->rtc);
+	if (ret)
+		return ret;
 
 	ret = devm_request_threaded_irq(&pdev->dev, alm_irq, NULL,
 				wm831x_alm_irq,
@@ -455,9 +445,6 @@ static int wm831x_rtc_probe(struct platform_device *pdev)
 	wm831x_rtc_add_randomness(wm831x);
 
 	return 0;
-
-err:
-	return ret;
 }
 
 static const struct dev_pm_ops wm831x_rtc_pm_ops = {
diff --git a/drivers/rtc/rtc-wm8350.c b/drivers/rtc/rtc-wm8350.c
index 483c7993516b..f54fa12c4b4b 100644
--- a/drivers/rtc/rtc-wm8350.c
+++ b/drivers/rtc/rtc-wm8350.c
@@ -340,8 +340,7 @@ static const struct rtc_class_ops wm8350_rtc_ops = {
 #ifdef CONFIG_PM_SLEEP
 static int wm8350_rtc_suspend(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct wm8350 *wm8350 = dev_get_drvdata(&pdev->dev);
+	struct wm8350 *wm8350 = dev_get_drvdata(dev);
 	int ret = 0;
 	u16 reg;
 
@@ -351,8 +350,7 @@ static int wm8350_rtc_suspend(struct device *dev)
 	    reg & WM8350_RTC_ALMSTS) {
 		ret = wm8350_rtc_stop_alarm(wm8350);
 		if (ret != 0)
-			dev_err(&pdev->dev, "Failed to stop RTC alarm: %d\n",
-				ret);
+			dev_err(dev, "Failed to stop RTC alarm: %d\n", ret);
 	}
 
 	return ret;
@@ -360,15 +358,13 @@ static int wm8350_rtc_suspend(struct device *dev)
 
 static int wm8350_rtc_resume(struct device *dev)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct wm8350 *wm8350 = dev_get_drvdata(&pdev->dev);
+	struct wm8350 *wm8350 = dev_get_drvdata(dev);
 	int ret;
 
 	if (wm8350->rtc.alarm_enabled) {
 		ret = wm8350_rtc_start_alarm(wm8350);
 		if (ret != 0)
-			dev_err(&pdev->dev,
-				"Failed to restart RTC alarm: %d\n", ret);
+			dev_err(dev, "Failed to restart RTC alarm: %d\n", ret);
 	}
 
 	return 0;
diff --git a/drivers/rtc/rtc-x1205.c b/drivers/rtc/rtc-x1205.c
index f08f18e4fcdf..ad2ae2f0536e 100644
--- a/drivers/rtc/rtc-x1205.c
+++ b/drivers/rtc/rtc-x1205.c
@@ -673,9 +673,16 @@ static const struct i2c_device_id x1205_id[] = {
 };
 MODULE_DEVICE_TABLE(i2c, x1205_id);
 
+static const struct of_device_id x1205_dt_ids[] = {
+	{ .compatible = "xircom,x1205", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, x1205_dt_ids);
+
 static struct i2c_driver x1205_driver = {
 	.driver		= {
 		.name	= "rtc-x1205",
+		.of_match_table = x1205_dt_ids,
 	},
 	.probe		= x1205_probe,
 	.remove		= x1205_remove,
diff --git a/drivers/rtc/rtc-xgene.c b/drivers/rtc/rtc-xgene.c
index 153820876a82..9888383f0088 100644
--- a/drivers/rtc/rtc-xgene.c
+++ b/drivers/rtc/rtc-xgene.c
@@ -1,34 +1,21 @@
+// SPDX-License-Identifier: GPL-2.0+
 /*
  * APM X-Gene SoC Real Time Clock Driver
  *
  * Copyright (c) 2014, Applied Micro Circuits Corporation
  * Author: Rameshwar Prasad Sahu <rsahu@apm.com>
  *         Loc Ho <lho@apm.com>
- *
- * This program is free software; you can redistribute  it and/or modify it
- * under  the terms of  the GNU General  Public License as published by the
- * Free Software Foundation;  either version 2 of the  License, or (at your
- * option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
  */
 
+#include <linux/clk.h>
+#include <linux/delay.h>
 #include <linux/init.h>
+#include <linux/io.h>
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
-#include <linux/io.h>
-#include <linux/slab.h>
-#include <linux/clk.h>
-#include <linux/delay.h>
 #include <linux/rtc.h>
+#include <linux/slab.h>
 
 /* RTC CSR Registers */
 #define RTC_CCVR		0x00
@@ -48,7 +35,6 @@
 struct xgene_rtc_dev {
 	struct rtc_device *rtc;
 	struct device *dev;
-	unsigned long alarm_time;
 	void __iomem *csr_base;
 	struct clk *clk;
 	unsigned int irq_wake;
@@ -59,11 +45,11 @@ static int xgene_rtc_read_time(struct device *dev, struct rtc_time *tm)
 {
 	struct xgene_rtc_dev *pdata = dev_get_drvdata(dev);
 
-	rtc_time_to_tm(readl(pdata->csr_base + RTC_CCVR), tm);
+	rtc_time64_to_tm(readl(pdata->csr_base + RTC_CCVR), tm);
 	return 0;
 }
 
-static int xgene_rtc_set_mmss(struct device *dev, unsigned long secs)
+static int xgene_rtc_set_time(struct device *dev, struct rtc_time *tm)
 {
 	struct xgene_rtc_dev *pdata = dev_get_drvdata(dev);
 
@@ -71,7 +57,7 @@ static int xgene_rtc_set_mmss(struct device *dev, unsigned long secs)
 	 * NOTE: After the following write, the RTC_CCVR is only reflected
 	 *       after the update cycle of 1 seconds.
 	 */
-	writel((u32) secs, pdata->csr_base + RTC_CLR);
+	writel((u32)rtc_tm_to_time64(tm), pdata->csr_base + RTC_CLR);
 	readl(pdata->csr_base + RTC_CLR); /* Force a barrier */
 
 	return 0;
@@ -81,7 +67,8 @@ static int xgene_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 {
 	struct xgene_rtc_dev *pdata = dev_get_drvdata(dev);
 
-	rtc_time_to_tm(pdata->alarm_time, &alrm->time);
+	/* If possible, CMR should be read here */
+	rtc_time64_to_tm(0, &alrm->time);
 	alrm->enabled = readl(pdata->csr_base + RTC_CCR) & RTC_CCR_IE;
 
 	return 0;
@@ -115,11 +102,8 @@ static int xgene_rtc_alarm_irq_enabled(struct device *dev)
 static int xgene_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 {
 	struct xgene_rtc_dev *pdata = dev_get_drvdata(dev);
-	unsigned long alarm_time;
 
-	rtc_tm_to_time(&alrm->time, &alarm_time);
-	pdata->alarm_time = alarm_time;
-	writel((u32) pdata->alarm_time, pdata->csr_base + RTC_CMR);
+	writel((u32)rtc_tm_to_time64(&alrm->time), pdata->csr_base + RTC_CMR);
 
 	xgene_rtc_alarm_irq_enable(dev, alrm->enabled);
 
@@ -128,7 +112,7 @@ static int xgene_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm)
 
 static const struct rtc_class_ops xgene_rtc_ops = {
 	.read_time	= xgene_rtc_read_time,
-	.set_mmss	= xgene_rtc_set_mmss,
+	.set_time	= xgene_rtc_set_time,
 	.read_alarm	= xgene_rtc_read_alarm,
 	.set_alarm	= xgene_rtc_set_alarm,
 	.alarm_irq_enable = xgene_rtc_alarm_irq_enable,
@@ -136,7 +120,7 @@ static const struct rtc_class_ops xgene_rtc_ops = {
 
 static irqreturn_t xgene_rtc_interrupt(int irq, void *id)
 {
-	struct xgene_rtc_dev *pdata = (struct xgene_rtc_dev *) id;
+	struct xgene_rtc_dev *pdata = id;
 
 	/* Check if interrupt asserted */
 	if (!(readl(pdata->csr_base + RTC_STAT) & RTC_STAT_BIT))
@@ -168,6 +152,10 @@ static int xgene_rtc_probe(struct platform_device *pdev)
 	if (IS_ERR(pdata->csr_base))
 		return PTR_ERR(pdata->csr_base);
 
+	pdata->rtc = devm_rtc_allocate_device(&pdev->dev);
+	if (IS_ERR(pdata->rtc))
+		return PTR_ERR(pdata->rtc);
+
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0) {
 		dev_err(&pdev->dev, "No IRQ resource\n");
@@ -198,15 +186,16 @@ static int xgene_rtc_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	pdata->rtc = devm_rtc_device_register(&pdev->dev, pdev->name,
-					 &xgene_rtc_ops, THIS_MODULE);
-	if (IS_ERR(pdata->rtc)) {
-		clk_disable_unprepare(pdata->clk);
-		return PTR_ERR(pdata->rtc);
-	}
-
 	/* HW does not support update faster than 1 seconds */
 	pdata->rtc->uie_unsupported = 1;
+	pdata->rtc->ops = &xgene_rtc_ops;
+	pdata->rtc->range_max = U32_MAX;
+
+	ret = rtc_register_device(pdata->rtc);
+	if (ret) {
+		clk_disable_unprepare(pdata->clk);
+		return ret;
+	}
 
 	return 0;
 }
diff --git a/drivers/rtc/rtc-zynqmp.c b/drivers/rtc/rtc-zynqmp.c
index bb950945ec7f..00639594de0c 100644
--- a/drivers/rtc/rtc-zynqmp.c
+++ b/drivers/rtc/rtc-zynqmp.c
@@ -1,20 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * Xilinx Zynq Ultrascale+ MPSoC Real Time Clock Driver
  *
  * Copyright (C) 2015 Xilinx, Inc.
  *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program. If not, see <http://www.gnu.org/licenses/>.
- *
  */
 
 #include <linux/delay.h>
diff --git a/drivers/rtc/sysfs.c b/drivers/rtc/sysfs.c
index a8f22ee726bb..be3531e7f868 100644
--- a/drivers/rtc/sysfs.c
+++ b/drivers/rtc/sysfs.c
@@ -1,20 +1,16 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * RTC subsystem, sysfs interface
  *
  * Copyright (C) 2005 Tower Technologies
  * Author: Alessandro Zummo <a.zummo@towertech.it>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 #include <linux/module.h>
 #include <linux/rtc.h>
 
 #include "rtc-core.h"
 
-
 /* device attributes */
 
 /*
@@ -86,7 +82,7 @@ max_user_freq_show(struct device *dev, struct device_attribute *attr, char *buf)
 
 static ssize_t
 max_user_freq_store(struct device *dev, struct device_attribute *attr,
-		const char *buf, size_t n)
+		    const char *buf, size_t n)
 {
 	struct rtc_device *rtc = to_rtc_device(dev);
 	unsigned long val;
@@ -116,12 +112,11 @@ hctosys_show(struct device *dev, struct device_attribute *attr, char *buf)
 {
 #ifdef CONFIG_RTC_HCTOSYS_DEVICE
 	if (rtc_hctosys_ret == 0 &&
-			strcmp(dev_name(&to_rtc_device(dev)->dev),
-				CONFIG_RTC_HCTOSYS_DEVICE) == 0)
+	    strcmp(dev_name(&to_rtc_device(dev)->dev),
+		   CONFIG_RTC_HCTOSYS_DEVICE) == 0)
 		return sprintf(buf, "1\n");
-	else
 #endif
-		return sprintf(buf, "0\n");
+	return sprintf(buf, "0\n");
 }
 static DEVICE_ATTR_RO(hctosys);
 
@@ -175,15 +170,15 @@ wakealarm_store(struct device *dev, struct device_attribute *attr,
 		if (*buf_ptr == '=') {
 			buf_ptr++;
 			push = 1;
-		} else
+		} else {
 			adjust = 1;
+		}
 	}
 	retval = kstrtos64(buf_ptr, 0, &alarm);
 	if (retval)
 		return retval;
-	if (adjust) {
+	if (adjust)
 		alarm += now;
-	}
 	if (alarm > now || push) {
 		/* Avoid accidentally clobbering active alarms; we can't
 		 * entirely prevent that here, without even the minimal
diff --git a/drivers/rtc/systohc.c b/drivers/rtc/systohc.c
index 718293d72426..8b70f0520e13 100644
--- a/drivers/rtc/systohc.c
+++ b/drivers/rtc/systohc.c
@@ -1,9 +1,4 @@
-/*
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 as published by
- * the Free Software Foundation.
- *
- */
+// SPDX-License-Identifier: GPL-2.0
 #include <linux/rtc.h>
 #include <linux/time.h>
 
@@ -35,8 +30,7 @@ int rtc_set_ntp_time(struct timespec64 now, unsigned long *target_nsec)
 	if (!rtc)
 		goto out_err;
 
-	if (!rtc->ops || (!rtc->ops->set_time && !rtc->ops->set_mmss64 &&
-			  !rtc->ops->set_mmss))
+	if (!rtc->ops || !rtc->ops->set_time)
 		goto out_close;
 
 	/* Compute the value of tv_nsec we require the caller to supply in
@@ -58,9 +52,6 @@ int rtc_set_ntp_time(struct timespec64 now, unsigned long *target_nsec)
 
 	rtc_time64_to_tm(to_set.tv_sec, &tm);
 
-	/* rtc_hctosys exclusively uses UTC, so we call set_time here, not
-	 * set_mmss.
-	 */
 	err = rtc_set_time(rtc, &tm);
 
 out_close:
diff --git a/drivers/sbus/char/oradax.c b/drivers/sbus/char/oradax.c
index 6516bc3cb58b..acd9ba40eabe 100644
--- a/drivers/sbus/char/oradax.c
+++ b/drivers/sbus/char/oradax.c
@@ -30,7 +30,7 @@
  * the recommended way for applications to use the coprocessor, and
  * the driver interface is not intended for general use.
  *
- * See Documentation/sparc/oradax/oracle-dax.txt for more details.
+ * See Documentation/sparc/oradax/oracle-dax.rst for more details.
  */
 
 #include <linux/uaccess.h>
diff --git a/drivers/scsi/NCR5380.c b/drivers/scsi/NCR5380.c
index 01c23d27f290..fe0535affc14 100644
--- a/drivers/scsi/NCR5380.c
+++ b/drivers/scsi/NCR5380.c
@@ -272,9 +272,8 @@ mrs[] = {
 static void NCR5380_print(struct Scsi_Host *instance)
 {
 	struct NCR5380_hostdata *hostdata = shost_priv(instance);
-	unsigned char status, data, basr, mr, icr, i;
+	unsigned char status, basr, mr, icr, i;
 
-	data = NCR5380_read(CURRENT_SCSI_DATA_REG);
 	status = NCR5380_read(STATUS_REG);
 	mr = NCR5380_read(MODE_REG);
 	icr = NCR5380_read(INITIATOR_COMMAND_REG);
@@ -1933,13 +1932,13 @@ static void NCR5380_information_transfer(struct Scsi_Host *instance)
 					if (!hostdata->connected)
 						return;
 
-					/* Fall through to reject message */
-
+					/* Reject message */
+					/* Fall through */
+				default:
 					/*
 					 * If we get something weird that we aren't expecting,
-					 * reject it.
+					 * log it.
 					 */
-				default:
 					if (tmp == EXTENDED_MESSAGE)
 						scmd_printk(KERN_INFO, cmd,
 						            "rejecting unknown extended message code %02x, length %d\n",
diff --git a/drivers/scsi/aic7xxx/Kconfig.aic7xxx b/drivers/scsi/aic7xxx/Kconfig.aic7xxx
index 55ac55ee6068..40fe08a64535 100644
--- a/drivers/scsi/aic7xxx/Kconfig.aic7xxx
+++ b/drivers/scsi/aic7xxx/Kconfig.aic7xxx
@@ -3,7 +3,7 @@
 # $Id: //depot/linux-aic79xx-2.5.0/drivers/scsi/aic7xxx/Kconfig.aic7xxx#7 $
 #
 config SCSI_AIC7XXX
-	tristate "Adaptec AIC7xxx Fast -> U160 support (New Driver)"
+	tristate "Adaptec AIC7xxx Fast -> U160 support"
 	depends on (PCI || EISA) && SCSI
 	select SCSI_SPI_ATTRS
 	---help---
diff --git a/drivers/scsi/aic7xxx/aic7xxx_core.c b/drivers/scsi/aic7xxx/aic7xxx_core.c
index d4a7263e4b8f..a9d40d3b90ef 100644
--- a/drivers/scsi/aic7xxx/aic7xxx_core.c
+++ b/drivers/scsi/aic7xxx/aic7xxx_core.c
@@ -1666,7 +1666,7 @@ ahc_handle_scsiint(struct ahc_softc *ahc, u_int intstat)
 					printk("\tCRC Value Mismatch\n");
 				if ((sstat2 & CRCENDERR) != 0)
 					printk("\tNo terminal CRC packet "
-					       "recevied\n");
+					       "received\n");
 				if ((sstat2 & CRCREQERR) != 0)
 					printk("\tIllegal CRC packet "
 					       "request\n");
diff --git a/drivers/scsi/atp870u.c b/drivers/scsi/atp870u.c
index 1267200380f8..446a789cdaf5 100644
--- a/drivers/scsi/atp870u.c
+++ b/drivers/scsi/atp870u.c
@@ -194,12 +194,11 @@ static irqreturn_t atp870u_intr_handle(int irq, void *dev_id)
 				((unsigned char *) &adrcnt)[2] = atp_readb_io(dev, c, 0x12);
 				((unsigned char *) &adrcnt)[1] = atp_readb_io(dev, c, 0x13);
 				((unsigned char *) &adrcnt)[0] = atp_readb_io(dev, c, 0x14);
-				if (dev->id[c][target_id].last_len != adrcnt)
-				{
-			   		k = dev->id[c][target_id].last_len;
+				if (dev->id[c][target_id].last_len != adrcnt) {
+					k = dev->id[c][target_id].last_len;
 			   		k -= adrcnt;
 			   		dev->id[c][target_id].tran_len = k;			   
-			   	dev->id[c][target_id].last_len = adrcnt;			   
+					dev->id[c][target_id].last_len = adrcnt;
 				}
 #ifdef ED_DBGP
 				printk("dev->id[c][target_id].last_len = %d dev->id[c][target_id].tran_len = %d\n",dev->id[c][target_id].last_len,dev->id[c][target_id].tran_len);
diff --git a/drivers/scsi/be2iscsi/be_cmds.c b/drivers/scsi/be2iscsi/be_cmds.c
index 0a6972ee94d7..ea042c1fc70d 100644
--- a/drivers/scsi/be2iscsi/be_cmds.c
+++ b/drivers/scsi/be2iscsi/be_cmds.c
@@ -963,7 +963,7 @@ int beiscsi_cmd_q_destroy(struct be_ctrl_info *ctrl, struct be_queue_info *q,
  * @ctrl: ptr to ctrl_info
  * @cq: Completion Queue
  * @dq: Default Queue
- * @lenght: ring size
+ * @length: ring size
  * @entry_size: size of each entry in DEFQ
  * @is_header: Header or Data DEFQ
  * @ulp_num: Bind to which ULP
diff --git a/drivers/scsi/bnx2fc/bnx2fc_io.c b/drivers/scsi/bnx2fc/bnx2fc_io.c
index bc9f2a2365f4..8def63c0755f 100644
--- a/drivers/scsi/bnx2fc/bnx2fc_io.c
+++ b/drivers/scsi/bnx2fc/bnx2fc_io.c
@@ -1083,7 +1083,6 @@ int bnx2fc_eh_device_reset(struct scsi_cmnd *sc_cmd)
 static int bnx2fc_abts_cleanup(struct bnx2fc_cmd *io_req)
 {
 	struct bnx2fc_rport *tgt = io_req->tgt;
-	int rc = SUCCESS;
 	unsigned int time_left;
 
 	io_req->wait_for_comp = 1;
@@ -1110,7 +1109,7 @@ static int bnx2fc_abts_cleanup(struct bnx2fc_cmd *io_req)
 	kref_put(&io_req->refcount, bnx2fc_cmd_release);
 
 	spin_lock_bh(&tgt->tgt_lock);
-	return rc;
+	return SUCCESS;
 }
 
 /**
diff --git a/drivers/scsi/csiostor/csio_isr.c b/drivers/scsi/csiostor/csio_isr.c
index 7c8814715711..b2540402fafc 100644
--- a/drivers/scsi/csiostor/csio_isr.c
+++ b/drivers/scsi/csiostor/csio_isr.c
@@ -474,13 +474,39 @@ csio_reduce_sqsets(struct csio_hw *hw, int cnt)
 	csio_dbg(hw, "Reduced sqsets to %d\n", hw->num_sqsets);
 }
 
+static void csio_calc_sets(struct irq_affinity *affd, unsigned int nvecs)
+{
+	struct csio_hw *hw = affd->priv;
+	u8 i;
+
+	if (!nvecs)
+		return;
+
+	if (nvecs < hw->num_pports) {
+		affd->nr_sets = 1;
+		affd->set_size[0] = nvecs;
+		return;
+	}
+
+	affd->nr_sets = hw->num_pports;
+	for (i = 0; i < hw->num_pports; i++)
+		affd->set_size[i] = nvecs / hw->num_pports;
+}
+
 static int
 csio_enable_msix(struct csio_hw *hw)
 {
 	int i, j, k, n, min, cnt;
 	int extra = CSIO_EXTRA_VECS;
 	struct csio_scsi_cpu_info *info;
-	struct irq_affinity desc = { .pre_vectors = 2 };
+	struct irq_affinity desc = {
+		.pre_vectors = CSIO_EXTRA_VECS,
+		.calc_sets = csio_calc_sets,
+		.priv = hw,
+	};
+
+	if (hw->num_pports > IRQ_AFFINITY_MAX_SETS)
+		return -ENOSPC;
 
 	min = hw->num_pports + extra;
 	cnt = hw->num_sqsets + extra;
diff --git a/drivers/scsi/cxgbi/cxgb3i/cxgb3i.c b/drivers/scsi/cxgbi/cxgb3i/cxgb3i.c
index 75e1273a44b3..b8dd9e648dd0 100644
--- a/drivers/scsi/cxgbi/cxgb3i/cxgb3i.c
+++ b/drivers/scsi/cxgbi/cxgb3i/cxgb3i.c
@@ -979,14 +979,17 @@ static int init_act_open(struct cxgbi_sock *csk)
 	csk->atid = cxgb3_alloc_atid(t3dev, &t3_client, csk);
 	if (csk->atid < 0) {
 		pr_err("NO atid available.\n");
-		goto rel_resource;
+		return -EINVAL;
 	}
 	cxgbi_sock_set_flag(csk, CTPF_HAS_ATID);
 	cxgbi_sock_get(csk);
 
 	skb = alloc_wr(sizeof(struct cpl_act_open_req), 0, GFP_KERNEL);
-	if (!skb)
-		goto rel_resource;
+	if (!skb) {
+		cxgb3_free_atid(t3dev, csk->atid);
+		cxgbi_sock_put(csk);
+		return -ENOMEM;
+	}
 	skb->sk = (struct sock *)csk;
 	set_arp_failure_handler(skb, act_open_arp_failure);
 	csk->snd_win = cxgb3i_snd_win;
@@ -1007,11 +1010,6 @@ static int init_act_open(struct cxgbi_sock *csk)
 	cxgbi_sock_set_state(csk, CTP_ACTIVE_OPEN);
 	send_act_open_req(csk, skb, csk->l2t);
 	return 0;
-
-rel_resource:
-	if (skb)
-		__kfree_skb(skb);
-	return -EINVAL;
 }
 
 cxgb3_cpl_handler_func cxgb3i_cpl_handlers[NUM_CPL_CMDS] = {
diff --git a/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c b/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c
index d44914e5e415..124f3345420f 100644
--- a/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c
+++ b/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c
@@ -60,7 +60,7 @@ MODULE_PARM_DESC(dbg_level, "Debug flag (default=0)");
 #define CXGB4I_DEFAULT_10G_RCV_WIN (256 * 1024)
 static int cxgb4i_rcv_win = -1;
 module_param(cxgb4i_rcv_win, int, 0644);
-MODULE_PARM_DESC(cxgb4i_rcv_win, "TCP reveive window in bytes");
+MODULE_PARM_DESC(cxgb4i_rcv_win, "TCP receive window in bytes");
 
 #define CXGB4I_DEFAULT_10G_SND_WIN (128 * 1024)
 static int cxgb4i_snd_win = -1;
diff --git a/drivers/scsi/cxgbi/libcxgbi.c b/drivers/scsi/cxgbi/libcxgbi.c
index 006372b3fba2..8b915d4ed98d 100644
--- a/drivers/scsi/cxgbi/libcxgbi.c
+++ b/drivers/scsi/cxgbi/libcxgbi.c
@@ -282,7 +282,6 @@ struct cxgbi_device *cxgbi_device_find_by_netdev_rcu(struct net_device *ndev,
 }
 EXPORT_SYMBOL_GPL(cxgbi_device_find_by_netdev_rcu);
 
-#if IS_ENABLED(CONFIG_IPV6)
 static struct cxgbi_device *cxgbi_device_find_by_mac(struct net_device *ndev,
 						     int *port)
 {
@@ -315,7 +314,6 @@ static struct cxgbi_device *cxgbi_device_find_by_mac(struct net_device *ndev,
 		  ndev, ndev->name);
 	return NULL;
 }
-#endif
 
 void cxgbi_hbas_remove(struct cxgbi_device *cdev)
 {
@@ -653,6 +651,8 @@ cxgbi_check_route(struct sockaddr *dst_addr, int ifindex)
 	}
 
 	cdev = cxgbi_device_find_by_netdev(ndev, &port);
+	if (!cdev)
+		cdev = cxgbi_device_find_by_mac(ndev, &port);
 	if (!cdev) {
 		pr_info("dst %pI4, %s, NOT cxgbi device.\n",
 			&daddr->sin_addr.s_addr, ndev->name);
@@ -2310,7 +2310,6 @@ int cxgbi_get_ep_param(struct iscsi_endpoint *ep, enum iscsi_param param,
 {
 	struct cxgbi_endpoint *cep = ep->dd_data;
 	struct cxgbi_sock *csk;
-	int len;
 
 	log_debug(1 << CXGBI_DBG_ISCSI,
 		"cls_conn 0x%p, param %d.\n", ep, param);
@@ -2328,9 +2327,9 @@ int cxgbi_get_ep_param(struct iscsi_endpoint *ep, enum iscsi_param param,
 		return iscsi_conn_get_addr_param((struct sockaddr_storage *)
 						 &csk->daddr, param, buf);
 	default:
-		return -ENOSYS;
+		break;
 	}
-	return len;
+	return -ENOSYS;
 }
 EXPORT_SYMBOL_GPL(cxgbi_get_ep_param);
 
@@ -2563,13 +2562,9 @@ struct iscsi_endpoint *cxgbi_ep_connect(struct Scsi_Host *shost,
 			pr_info("shost 0x%p, priv NULL.\n", shost);
 			goto err_out;
 		}
-
-		rtnl_lock();
-		if (!vlan_uses_dev(hba->ndev))
-			ifindex = hba->ndev->ifindex;
-		rtnl_unlock();
 	}
 
+check_route:
 	if (dst_addr->sa_family == AF_INET) {
 		csk = cxgbi_check_route(dst_addr, ifindex);
 #if IS_ENABLED(CONFIG_IPV6)
@@ -2590,6 +2585,13 @@ struct iscsi_endpoint *cxgbi_ep_connect(struct Scsi_Host *shost,
 	if (!hba)
 		hba = csk->cdev->hbas[csk->port_id];
 	else if (hba != csk->cdev->hbas[csk->port_id]) {
+		if (ifindex != hba->ndev->ifindex) {
+			cxgbi_sock_put(csk);
+			cxgbi_sock_closed(csk);
+			ifindex = hba->ndev->ifindex;
+			goto check_route;
+		}
+
 		pr_info("Could not connect through requested host %u"
 			"hba 0x%p != 0x%p (%u).\n",
 			shost->host_no, hba,
diff --git a/drivers/scsi/dpt_i2o.c b/drivers/scsi/dpt_i2o.c
index abdc34affdf6..a3afd1478ab6 100644
--- a/drivers/scsi/dpt_i2o.c
+++ b/drivers/scsi/dpt_i2o.c
@@ -835,8 +835,8 @@ static void adpt_i2o_sys_shutdown(void)
 	adpt_hba *pHba, *pNext;
 	struct adpt_i2o_post_wait_data *p1, *old;
 
-	 printk(KERN_INFO"Shutting down Adaptec I2O controllers.\n");
-	 printk(KERN_INFO"   This could take a few minutes if there are many devices attached\n");
+	printk(KERN_INFO "Shutting down Adaptec I2O controllers.\n");
+	printk(KERN_INFO "   This could take a few minutes if there are many devices attached\n");
 	/* Delete all IOPs from the controller chain */
 	/* They should have already been released by the
 	 * scsi-core
@@ -859,7 +859,7 @@ static void adpt_i2o_sys_shutdown(void)
 //	spin_unlock_irqrestore(&adpt_post_wait_lock, flags);
 	adpt_post_wait_queue = NULL;
 
-	 printk(KERN_INFO "Adaptec I2O controllers down.\n");
+	printk(KERN_INFO "Adaptec I2O controllers down.\n");
 }
 
 static int adpt_install_hba(struct scsi_host_template* sht, struct pci_dev* pDev)
@@ -3390,7 +3390,7 @@ static int adpt_i2o_issue_params(int cmd, adpt_hba* pHba, int tid,
 		return -((res[1] >> 16) & 0xFF); /* -BlockStatus */
 	}
 
-	 return 4 + ((res[1] & 0x0000FFFF) << 2); /* bytes used in resblk */ 
+	return 4 + ((res[1] & 0x0000FFFF) << 2); /* bytes used in resblk */
 }
 
 
@@ -3463,8 +3463,8 @@ static int adpt_i2o_enable_hba(adpt_hba* pHba)
 
 static int adpt_i2o_systab_send(adpt_hba* pHba)
 {
-	 u32 msg[12];
-	 int ret;
+	u32 msg[12];
+	int ret;
 
 	msg[0] = I2O_MESSAGE_SIZE(12) | SGL_OFFSET_6;
 	msg[1] = I2O_CMD_SYS_TAB_SET<<24 | HOST_TID<<12 | ADAPTER_TID;
diff --git a/drivers/scsi/gdth.c b/drivers/scsi/gdth.c
index e7f1dd4f3b66..0ca9b4393770 100644
--- a/drivers/scsi/gdth.c
+++ b/drivers/scsi/gdth.c
@@ -3697,8 +3697,9 @@ static int ioc_general(void __user *arg, char *cmnd)
 
 	rval = 0;
 out_free_buf:
-	dma_free_coherent(&ha->pdev->dev, gen.data_len + gen.sense_len, buf,
-			paddr);
+	if (buf)
+		dma_free_coherent(&ha->pdev->dev, gen.data_len + gen.sense_len,
+				  buf, paddr);
 	return rval;
 }
  
diff --git a/drivers/scsi/hisi_sas/hisi_sas.h b/drivers/scsi/hisi_sas/hisi_sas.h
index 9bfa9f12d81e..fc87994b5d73 100644
--- a/drivers/scsi/hisi_sas/hisi_sas.h
+++ b/drivers/scsi/hisi_sas/hisi_sas.h
@@ -170,6 +170,7 @@ struct hisi_sas_phy {
 	u32		code_violation_err_count;
 	enum sas_linkrate	minimum_linkrate;
 	enum sas_linkrate	maximum_linkrate;
+	int enable;
 };
 
 struct hisi_sas_port {
@@ -551,6 +552,8 @@ extern int hisi_sas_slave_configure(struct scsi_device *sdev);
 extern int hisi_sas_scan_finished(struct Scsi_Host *shost, unsigned long time);
 extern void hisi_sas_scan_start(struct Scsi_Host *shost);
 extern int hisi_sas_host_reset(struct Scsi_Host *shost, int reset_type);
+extern void hisi_sas_phy_enable(struct hisi_hba *hisi_hba, int phy_no,
+				int enable);
 extern void hisi_sas_phy_down(struct hisi_hba *hisi_hba, int phy_no, int rdy);
 extern void hisi_sas_slot_task_free(struct hisi_hba *hisi_hba,
 				    struct sas_task *task,
diff --git a/drivers/scsi/hisi_sas/hisi_sas_main.c b/drivers/scsi/hisi_sas/hisi_sas_main.c
index 14bac4966c87..8a7feb8ed8d6 100644
--- a/drivers/scsi/hisi_sas/hisi_sas_main.c
+++ b/drivers/scsi/hisi_sas/hisi_sas_main.c
@@ -10,7 +10,6 @@
  */
 
 #include "hisi_sas.h"
-#include "../libsas/sas_internal.h"
 #define DRV_NAME "hisi_sas"
 
 #define DEV_IS_GONE(dev) \
@@ -171,7 +170,7 @@ void hisi_sas_stop_phys(struct hisi_hba *hisi_hba)
 	int phy_no;
 
 	for (phy_no = 0; phy_no < hisi_hba->n_phy; phy_no++)
-		hisi_hba->hw->phy_disable(hisi_hba, phy_no);
+		hisi_sas_phy_enable(hisi_hba, phy_no, 0);
 }
 EXPORT_SYMBOL_GPL(hisi_sas_stop_phys);
 
@@ -684,7 +683,7 @@ static void hisi_sas_bytes_dmaed(struct hisi_hba *hisi_hba, int phy_no)
 		id->initiator_bits = SAS_PROTOCOL_ALL;
 		id->target_bits = phy->identify.target_port_protocols;
 	} else if (phy->phy_type & PORT_TYPE_SATA) {
-		/*Nothing*/
+		/* Nothing */
 	}
 
 	sas_phy->frame_rcvd_size = phy->frame_rcvd_size;
@@ -755,7 +754,8 @@ static int hisi_sas_init_device(struct domain_device *device)
 		 * STP target port
 		 */
 		local_phy = sas_get_local_phy(device);
-		if (!scsi_is_sas_phy_local(local_phy)) {
+		if (!scsi_is_sas_phy_local(local_phy) &&
+		    !test_bit(HISI_SAS_RESET_BIT, &hisi_hba->flags)) {
 			unsigned long deadline = ata_deadline(jiffies, 20000);
 			struct sata_device *sata_dev = &device->sata_dev;
 			struct ata_host *ata_host = sata_dev->ata_host;
@@ -770,8 +770,7 @@ static int hisi_sas_init_device(struct domain_device *device)
 		}
 		sas_put_local_phy(local_phy);
 		if (rc) {
-			dev_warn(dev, "SATA disk hardreset fail: 0x%x\n",
-				 rc);
+			dev_warn(dev, "SATA disk hardreset fail: %d\n", rc);
 			return rc;
 		}
 
@@ -976,6 +975,30 @@ static void hisi_sas_phy_init(struct hisi_hba *hisi_hba, int phy_no)
 	timer_setup(&phy->timer, hisi_sas_wait_phyup_timedout, 0);
 }
 
+/* Wrapper to ensure we track hisi_sas_phy.enable properly */
+void hisi_sas_phy_enable(struct hisi_hba *hisi_hba, int phy_no, int enable)
+{
+	struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
+	struct asd_sas_phy *aphy = &phy->sas_phy;
+	struct sas_phy *sphy = aphy->phy;
+	unsigned long flags;
+
+	spin_lock_irqsave(&phy->lock, flags);
+
+	if (enable) {
+		/* We may have been enabled already; if so, don't touch */
+		if (!phy->enable)
+			sphy->negotiated_linkrate = SAS_LINK_RATE_UNKNOWN;
+		hisi_hba->hw->phy_start(hisi_hba, phy_no);
+	} else {
+		sphy->negotiated_linkrate = SAS_PHY_DISABLED;
+		hisi_hba->hw->phy_disable(hisi_hba, phy_no);
+	}
+	phy->enable = enable;
+	spin_unlock_irqrestore(&phy->lock, flags);
+}
+EXPORT_SYMBOL_GPL(hisi_sas_phy_enable);
+
 static void hisi_sas_port_notify_formed(struct asd_sas_phy *sas_phy)
 {
 	struct sas_ha_struct *sas_ha = sas_phy->ha;
@@ -1112,10 +1135,10 @@ static int hisi_sas_phy_set_linkrate(struct hisi_hba *hisi_hba, int phy_no,
 	sas_phy->phy->maximum_linkrate = max;
 	sas_phy->phy->minimum_linkrate = min;
 
-	hisi_hba->hw->phy_disable(hisi_hba, phy_no);
+	hisi_sas_phy_enable(hisi_hba, phy_no, 0);
 	msleep(100);
 	hisi_hba->hw->phy_set_linkrate(hisi_hba, phy_no, &_r);
-	hisi_hba->hw->phy_start(hisi_hba, phy_no);
+	hisi_sas_phy_enable(hisi_hba, phy_no, 1);
 
 	return 0;
 }
@@ -1133,13 +1156,13 @@ static int hisi_sas_control_phy(struct asd_sas_phy *sas_phy, enum phy_func func,
 		break;
 
 	case PHY_FUNC_LINK_RESET:
-		hisi_hba->hw->phy_disable(hisi_hba, phy_no);
+		hisi_sas_phy_enable(hisi_hba, phy_no, 0);
 		msleep(100);
-		hisi_hba->hw->phy_start(hisi_hba, phy_no);
+		hisi_sas_phy_enable(hisi_hba, phy_no, 1);
 		break;
 
 	case PHY_FUNC_DISABLE:
-		hisi_hba->hw->phy_disable(hisi_hba, phy_no);
+		hisi_sas_phy_enable(hisi_hba, phy_no, 0);
 		break;
 
 	case PHY_FUNC_SET_LINK_RATE:
@@ -1264,8 +1287,7 @@ static int hisi_sas_exec_internal_tmf_task(struct domain_device *device,
 			/* no error, but return the number of bytes of
 			 * underrun
 			 */
-			dev_warn(dev, "abort tmf: task to dev %016llx "
-				 "resp: 0x%x sts 0x%x underrun\n",
+			dev_warn(dev, "abort tmf: task to dev %016llx resp: 0x%x sts 0x%x underrun\n",
 				 SAS_ADDR(device->sas_addr),
 				 task->task_status.resp,
 				 task->task_status.stat);
@@ -1280,10 +1302,16 @@ static int hisi_sas_exec_internal_tmf_task(struct domain_device *device,
 			break;
 		}
 
-		dev_warn(dev, "abort tmf: task to dev "
-			 "%016llx resp: 0x%x status 0x%x\n",
-			 SAS_ADDR(device->sas_addr), task->task_status.resp,
-			 task->task_status.stat);
+		if (task->task_status.resp == SAS_TASK_COMPLETE &&
+		    task->task_status.stat == SAS_OPEN_REJECT) {
+			dev_warn(dev, "abort tmf: open reject failed\n");
+			res = -EIO;
+		} else {
+			dev_warn(dev, "abort tmf: task to dev %016llx resp: 0x%x status 0x%x\n",
+				 SAS_ADDR(device->sas_addr),
+				 task->task_status.resp,
+				 task->task_status.stat);
+		}
 		sas_free_task(task);
 		task = NULL;
 	}
@@ -1427,9 +1455,9 @@ static void hisi_sas_rescan_topology(struct hisi_hba *hisi_hba, u32 old_state,
 					sas_ha->notify_port_event(sas_phy,
 							PORTE_BROADCAST_RCVD);
 			}
-		} else if (old_state & (1 << phy_no))
-			/* PHY down but was up before */
+		} else {
 			hisi_sas_phy_down(hisi_hba, phy_no, 0);
+		}
 
 	}
 }
@@ -1711,7 +1739,7 @@ static int hisi_sas_abort_task_set(struct domain_device *device, u8 *lun)
 	struct hisi_hba *hisi_hba = dev_to_hisi_hba(device);
 	struct device *dev = hisi_hba->dev;
 	struct hisi_sas_tmf_task tmf_task;
-	int rc = TMF_RESP_FUNC_FAILED;
+	int rc;
 
 	rc = hisi_sas_internal_task_abort(hisi_hba, device,
 					  HISI_SAS_INT_ABT_DEV, 0);
@@ -1803,7 +1831,7 @@ static int hisi_sas_I_T_nexus_reset(struct domain_device *device)
 
 	if (dev_is_sata(device)) {
 		rc = hisi_sas_softreset_ata_disk(device);
-		if (rc)
+		if (rc == TMF_RESP_FUNC_FAILED)
 			return TMF_RESP_FUNC_FAILED;
 	}
 
@@ -2100,10 +2128,8 @@ _hisi_sas_internal_task_abort(struct hisi_hba *hisi_hba,
 	}
 
 exit:
-	dev_dbg(dev, "internal task abort: task to dev %016llx task=%p "
-		"resp: 0x%x sts 0x%x\n",
-		SAS_ADDR(device->sas_addr),
-		task,
+	dev_dbg(dev, "internal task abort: task to dev %016llx task=%p resp: 0x%x sts 0x%x\n",
+		SAS_ADDR(device->sas_addr), task,
 		task->task_status.resp, /* 0 is complete, -1 is undelivered */
 		task->task_status.stat);
 	sas_free_task(task);
@@ -2172,16 +2198,18 @@ static void hisi_sas_phy_disconnected(struct hisi_sas_phy *phy)
 {
 	struct asd_sas_phy *sas_phy = &phy->sas_phy;
 	struct sas_phy *sphy = sas_phy->phy;
-	struct sas_phy_data *d = sphy->hostdata;
+	unsigned long flags;
 
 	phy->phy_attached = 0;
 	phy->phy_type = 0;
 	phy->port = NULL;
 
-	if (d->enable)
+	spin_lock_irqsave(&phy->lock, flags);
+	if (phy->enable)
 		sphy->negotiated_linkrate = SAS_LINK_RATE_UNKNOWN;
 	else
 		sphy->negotiated_linkrate = SAS_PHY_DISABLED;
+	spin_unlock_irqrestore(&phy->lock, flags);
 }
 
 void hisi_sas_phy_down(struct hisi_hba *hisi_hba, int phy_no, int rdy)
@@ -2234,6 +2262,19 @@ void hisi_sas_kill_tasklets(struct hisi_hba *hisi_hba)
 }
 EXPORT_SYMBOL_GPL(hisi_sas_kill_tasklets);
 
+int hisi_sas_host_reset(struct Scsi_Host *shost, int reset_type)
+{
+	struct hisi_hba *hisi_hba = shost_priv(shost);
+
+	if (reset_type != SCSI_ADAPTER_RESET)
+		return -EOPNOTSUPP;
+
+	queue_work(hisi_hba->wq, &hisi_hba->rst_work);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(hisi_sas_host_reset);
+
 struct scsi_transport_template *hisi_sas_stt;
 EXPORT_SYMBOL_GPL(hisi_sas_stt);
 
@@ -2491,22 +2532,19 @@ int hisi_sas_get_fw_info(struct hisi_hba *hisi_hba)
 
 		if (device_property_read_u32(dev, "ctrl-reset-reg",
 					     &hisi_hba->ctrl_reset_reg)) {
-			dev_err(dev,
-				"could not get property ctrl-reset-reg\n");
+			dev_err(dev, "could not get property ctrl-reset-reg\n");
 			return -ENOENT;
 		}
 
 		if (device_property_read_u32(dev, "ctrl-reset-sts-reg",
 					     &hisi_hba->ctrl_reset_sts_reg)) {
-			dev_err(dev,
-				"could not get property ctrl-reset-sts-reg\n");
+			dev_err(dev, "could not get property ctrl-reset-sts-reg\n");
 			return -ENOENT;
 		}
 
 		if (device_property_read_u32(dev, "ctrl-clock-ena-reg",
 					     &hisi_hba->ctrl_clock_ena_reg)) {
-			dev_err(dev,
-				"could not get property ctrl-clock-ena-reg\n");
+			dev_err(dev, "could not get property ctrl-clock-ena-reg\n");
 			return -ENOENT;
 		}
 	}
diff --git a/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c
index 293807443480..78fe7d344848 100644
--- a/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c
+++ b/drivers/scsi/hisi_sas/hisi_sas_v1_hw.c
@@ -798,16 +798,11 @@ static void start_phy_v1_hw(struct hisi_hba *hisi_hba, int phy_no)
 	enable_phy_v1_hw(hisi_hba, phy_no);
 }
 
-static void stop_phy_v1_hw(struct hisi_hba *hisi_hba, int phy_no)
-{
-	disable_phy_v1_hw(hisi_hba, phy_no);
-}
-
 static void phy_hard_reset_v1_hw(struct hisi_hba *hisi_hba, int phy_no)
 {
-	stop_phy_v1_hw(hisi_hba, phy_no);
+	hisi_sas_phy_enable(hisi_hba, phy_no, 0);
 	msleep(100);
-	start_phy_v1_hw(hisi_hba, phy_no);
+	hisi_sas_phy_enable(hisi_hba, phy_no, 1);
 }
 
 static void start_phys_v1_hw(struct timer_list *t)
@@ -817,7 +812,7 @@ static void start_phys_v1_hw(struct timer_list *t)
 
 	for (i = 0; i < hisi_hba->n_phy; i++) {
 		hisi_sas_phy_write32(hisi_hba, i, CHL_INT2_MSK, 0x12a);
-		start_phy_v1_hw(hisi_hba, i);
+		hisi_sas_phy_enable(hisi_hba, i, 1);
 	}
 }
 
@@ -1695,8 +1690,7 @@ static int interrupt_init_v1_hw(struct hisi_hba *hisi_hba)
 		for (j = 0; j < HISI_SAS_PHY_INT_NR; j++, idx++) {
 			irq = platform_get_irq(pdev, idx);
 			if (!irq) {
-				dev_err(dev,
-					"irq init: fail map phy interrupt %d\n",
+				dev_err(dev, "irq init: fail map phy interrupt %d\n",
 					idx);
 				return -ENOENT;
 			}
@@ -1704,8 +1698,7 @@ static int interrupt_init_v1_hw(struct hisi_hba *hisi_hba)
 			rc = devm_request_irq(dev, irq, phy_interrupts[j], 0,
 					      DRV_NAME " phy", phy);
 			if (rc) {
-				dev_err(dev, "irq init: could not request "
-					"phy interrupt %d, rc=%d\n",
+				dev_err(dev, "irq init: could not request phy interrupt %d, rc=%d\n",
 					irq, rc);
 				return -ENOENT;
 			}
@@ -1742,8 +1735,7 @@ static int interrupt_init_v1_hw(struct hisi_hba *hisi_hba)
 		rc = devm_request_irq(dev, irq, fatal_interrupts[i], 0,
 				      DRV_NAME " fatal", hisi_hba);
 		if (rc) {
-			dev_err(dev,
-				"irq init: could not request fatal interrupt %d, rc=%d\n",
+			dev_err(dev, "irq init: could not request fatal interrupt %d, rc=%d\n",
 				irq, rc);
 			return -ENOENT;
 		}
@@ -1823,6 +1815,7 @@ static struct scsi_host_template sht_v1_hw = {
 	.target_destroy		= sas_target_destroy,
 	.ioctl			= sas_ioctl,
 	.shost_attrs		= host_attrs_v1_hw,
+	.host_reset             = hisi_sas_host_reset,
 };
 
 static const struct hisi_sas_hw hisi_sas_v1_hw = {
diff --git a/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c
index 89160ab3efb0..d4650bed8274 100644
--- a/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c
+++ b/drivers/scsi/hisi_sas/hisi_sas_v2_hw.c
@@ -1546,14 +1546,14 @@ static void phy_hard_reset_v2_hw(struct hisi_hba *hisi_hba, int phy_no)
 	struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
 	u32 txid_auto;
 
-	disable_phy_v2_hw(hisi_hba, phy_no);
+	hisi_sas_phy_enable(hisi_hba, phy_no, 0);
 	if (phy->identify.device_type == SAS_END_DEVICE) {
 		txid_auto = hisi_sas_phy_read32(hisi_hba, phy_no, TXID_AUTO);
 		hisi_sas_phy_write32(hisi_hba, phy_no, TXID_AUTO,
 					txid_auto | TX_HARDRST_MSK);
 	}
 	msleep(100);
-	start_phy_v2_hw(hisi_hba, phy_no);
+	hisi_sas_phy_enable(hisi_hba, phy_no, 1);
 }
 
 static void phy_get_events_v2_hw(struct hisi_hba *hisi_hba, int phy_no)
@@ -1586,7 +1586,7 @@ static void phys_init_v2_hw(struct hisi_hba *hisi_hba)
 		if (!sas_phy->phy->enabled)
 			continue;
 
-		start_phy_v2_hw(hisi_hba, i);
+		hisi_sas_phy_enable(hisi_hba, i, 1);
 	}
 }
 
@@ -2423,14 +2423,12 @@ slot_complete_v2_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot)
 			slot_err_v2_hw(hisi_hba, task, slot, 2);
 
 		if (ts->stat != SAS_DATA_UNDERRUN)
-			dev_info(dev, "erroneous completion iptt=%d task=%p dev id=%d "
-				"CQ hdr: 0x%x 0x%x 0x%x 0x%x "
-				"Error info: 0x%x 0x%x 0x%x 0x%x\n",
-				slot->idx, task, sas_dev->device_id,
-				complete_hdr->dw0, complete_hdr->dw1,
-				complete_hdr->act, complete_hdr->dw3,
-				error_info[0], error_info[1],
-				error_info[2], error_info[3]);
+			dev_info(dev, "erroneous completion iptt=%d task=%p dev id=%d CQ hdr: 0x%x 0x%x 0x%x 0x%x Error info: 0x%x 0x%x 0x%x 0x%x\n",
+				 slot->idx, task, sas_dev->device_id,
+				 complete_hdr->dw0, complete_hdr->dw1,
+				 complete_hdr->act, complete_hdr->dw3,
+				 error_info[0], error_info[1],
+				 error_info[2], error_info[3]);
 
 		if (unlikely(slot->abort))
 			return ts->stat;
@@ -2502,7 +2500,7 @@ out:
 		spin_lock_irqsave(&device->done_lock, flags);
 		if (test_bit(SAS_HA_FROZEN, &ha->state)) {
 			spin_unlock_irqrestore(&device->done_lock, flags);
-			dev_info(dev, "slot complete: task(%p) ignored\n ",
+			dev_info(dev, "slot complete: task(%p) ignored\n",
 				 task);
 			return sts;
 		}
@@ -2935,7 +2933,7 @@ static irqreturn_t int_chnl_int_v2_hw(int irq_no, void *p)
 
 			if (irq_value2 & BIT(CHL_INT2_SL_IDAF_TOUT_CONF_OFF)) {
 				dev_warn(dev, "phy%d identify timeout\n",
-						phy_no);
+					 phy_no);
 				hisi_sas_notify_phy_event(phy,
 						HISI_PHYE_LINK_RESET);
 			}
@@ -3036,7 +3034,7 @@ static const struct hisi_sas_hw_error axi_error[] = {
 	{ .msk = BIT(5), .msg = "SATA_AXI_R_ERR" },
 	{ .msk = BIT(6), .msg = "DQE_AXI_R_ERR" },
 	{ .msk = BIT(7), .msg = "CQE_AXI_W_ERR" },
-	{},
+	{}
 };
 
 static const struct hisi_sas_hw_error fifo_error[] = {
@@ -3045,7 +3043,7 @@ static const struct hisi_sas_hw_error fifo_error[] = {
 	{ .msk = BIT(10), .msg = "GETDQE_FIFO" },
 	{ .msk = BIT(11), .msg = "CMDP_FIFO" },
 	{ .msk = BIT(12), .msg = "AWTCTRL_FIFO" },
-	{},
+	{}
 };
 
 static const struct hisi_sas_hw_error fatal_axi_errors[] = {
@@ -3109,12 +3107,12 @@ static irqreturn_t fatal_axi_int_v2_hw(int irq_no, void *p)
 				if (!(err_value & sub->msk))
 					continue;
 				dev_err(dev, "%s (0x%x) found!\n",
-					 sub->msg, irq_value);
+					sub->msg, irq_value);
 				queue_work(hisi_hba->wq, &hisi_hba->rst_work);
 			}
 		} else {
 			dev_err(dev, "%s (0x%x) found!\n",
-				 axi_error->msg, irq_value);
+				axi_error->msg, irq_value);
 			queue_work(hisi_hba->wq, &hisi_hba->rst_work);
 		}
 	}
@@ -3258,7 +3256,7 @@ static irqreturn_t sata_int_v2_hw(int irq_no, void *p)
 	/* check ERR bit of Status Register */
 	if (fis->status & ATA_ERR) {
 		dev_warn(dev, "sata int: phy%d FIS status: 0x%x\n", phy_no,
-				fis->status);
+			 fis->status);
 		hisi_sas_notify_phy_event(phy, HISI_PHYE_LINK_RESET);
 		res = IRQ_NONE;
 		goto end;
@@ -3349,8 +3347,7 @@ static int interrupt_init_v2_hw(struct hisi_hba *hisi_hba)
 		rc = devm_request_irq(dev, irq, phy_interrupts[i], 0,
 				      DRV_NAME " phy", hisi_hba);
 		if (rc) {
-			dev_err(dev, "irq init: could not request "
-				"phy interrupt %d, rc=%d\n",
+			dev_err(dev, "irq init: could not request phy interrupt %d, rc=%d\n",
 				irq, rc);
 			rc = -ENOENT;
 			goto free_phy_int_irqs;
@@ -3364,8 +3361,7 @@ static int interrupt_init_v2_hw(struct hisi_hba *hisi_hba)
 		rc = devm_request_irq(dev, irq, sata_int_v2_hw, 0,
 				      DRV_NAME " sata", phy);
 		if (rc) {
-			dev_err(dev, "irq init: could not request "
-				"sata interrupt %d, rc=%d\n",
+			dev_err(dev, "irq init: could not request sata interrupt %d, rc=%d\n",
 				irq, rc);
 			rc = -ENOENT;
 			goto free_sata_int_irqs;
@@ -3377,8 +3373,7 @@ static int interrupt_init_v2_hw(struct hisi_hba *hisi_hba)
 		rc = devm_request_irq(dev, irq, fatal_interrupts[fatal_no], 0,
 				      DRV_NAME " fatal", hisi_hba);
 		if (rc) {
-			dev_err(dev,
-				"irq init: could not request fatal interrupt %d, rc=%d\n",
+			dev_err(dev, "irq init: could not request fatal interrupt %d, rc=%d\n",
 				irq, rc);
 			rc = -ENOENT;
 			goto free_fatal_int_irqs;
@@ -3393,8 +3388,7 @@ static int interrupt_init_v2_hw(struct hisi_hba *hisi_hba)
 		rc = devm_request_irq(dev, irq, cq_interrupt_v2_hw, 0,
 				      DRV_NAME " cq", cq);
 		if (rc) {
-			dev_err(dev,
-				"irq init: could not request cq interrupt %d, rc=%d\n",
+			dev_err(dev, "irq init: could not request cq interrupt %d, rc=%d\n",
 				irq, rc);
 			rc = -ENOENT;
 			goto free_cq_int_irqs;
@@ -3546,7 +3540,7 @@ static int write_gpio_v2_hw(struct hisi_hba *hisi_hba, u8 reg_type,
 		break;
 	default:
 		dev_err(dev, "write gpio: unsupported or bad reg type %d\n",
-				reg_type);
+			reg_type);
 		return -EINVAL;
 	}
 
@@ -3599,6 +3593,7 @@ static struct scsi_host_template sht_v2_hw = {
 	.target_destroy		= sas_target_destroy,
 	.ioctl			= sas_ioctl,
 	.shost_attrs		= host_attrs_v2_hw,
+	.host_reset		= hisi_sas_host_reset,
 };
 
 static const struct hisi_sas_hw hisi_sas_v2_hw = {
diff --git a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c
index 086695a4099f..49620c2411df 100644
--- a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c
+++ b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c
@@ -52,7 +52,36 @@
 #define CFG_ABT_SET_IPTT_DONE	0xd8
 #define CFG_ABT_SET_IPTT_DONE_OFF	0
 #define HGC_IOMB_PROC1_STATUS	0x104
+#define HGC_LM_DFX_STATUS2		0x128
+#define HGC_LM_DFX_STATUS2_IOSTLIST_OFF		0
+#define HGC_LM_DFX_STATUS2_IOSTLIST_MSK	(0xfff << \
+					 HGC_LM_DFX_STATUS2_IOSTLIST_OFF)
+#define HGC_LM_DFX_STATUS2_ITCTLIST_OFF		12
+#define HGC_LM_DFX_STATUS2_ITCTLIST_MSK	(0x7ff << \
+					 HGC_LM_DFX_STATUS2_ITCTLIST_OFF)
+#define HGC_CQE_ECC_ADDR		0x13c
+#define HGC_CQE_ECC_1B_ADDR_OFF	0
+#define HGC_CQE_ECC_1B_ADDR_MSK	(0x3f << HGC_CQE_ECC_1B_ADDR_OFF)
+#define HGC_CQE_ECC_MB_ADDR_OFF	8
+#define HGC_CQE_ECC_MB_ADDR_MSK (0x3f << HGC_CQE_ECC_MB_ADDR_OFF)
+#define HGC_IOST_ECC_ADDR		0x140
+#define HGC_IOST_ECC_1B_ADDR_OFF	0
+#define HGC_IOST_ECC_1B_ADDR_MSK	(0x3ff << HGC_IOST_ECC_1B_ADDR_OFF)
+#define HGC_IOST_ECC_MB_ADDR_OFF	16
+#define HGC_IOST_ECC_MB_ADDR_MSK	(0x3ff << HGC_IOST_ECC_MB_ADDR_OFF)
+#define HGC_DQE_ECC_ADDR		0x144
+#define HGC_DQE_ECC_1B_ADDR_OFF	0
+#define HGC_DQE_ECC_1B_ADDR_MSK	(0xfff << HGC_DQE_ECC_1B_ADDR_OFF)
+#define HGC_DQE_ECC_MB_ADDR_OFF	16
+#define HGC_DQE_ECC_MB_ADDR_MSK (0xfff << HGC_DQE_ECC_MB_ADDR_OFF)
 #define CHNL_INT_STATUS			0x148
+#define HGC_ITCT_ECC_ADDR		0x150
+#define HGC_ITCT_ECC_1B_ADDR_OFF		0
+#define HGC_ITCT_ECC_1B_ADDR_MSK		(0x3ff << \
+						 HGC_ITCT_ECC_1B_ADDR_OFF)
+#define HGC_ITCT_ECC_MB_ADDR_OFF		16
+#define HGC_ITCT_ECC_MB_ADDR_MSK		(0x3ff << \
+						 HGC_ITCT_ECC_MB_ADDR_OFF)
 #define HGC_AXI_FIFO_ERR_INFO  0x154
 #define AXI_ERR_INFO_OFF               0
 #define AXI_ERR_INFO_MSK               (0xff << AXI_ERR_INFO_OFF)
@@ -81,6 +110,10 @@
 #define ENT_INT_SRC3_ITC_INT_OFF	15
 #define ENT_INT_SRC3_ITC_INT_MSK	(0x1 << ENT_INT_SRC3_ITC_INT_OFF)
 #define ENT_INT_SRC3_ABT_OFF		16
+#define ENT_INT_SRC3_DQE_POISON_OFF	18
+#define ENT_INT_SRC3_IOST_POISON_OFF	19
+#define ENT_INT_SRC3_ITCT_POISON_OFF	20
+#define ENT_INT_SRC3_ITCT_NCQ_POISON_OFF	21
 #define ENT_INT_SRC_MSK1		0x1c4
 #define ENT_INT_SRC_MSK2		0x1c8
 #define ENT_INT_SRC_MSK3		0x1cc
@@ -90,6 +123,28 @@
 #define HGC_COM_INT_MSK				0x1d8
 #define ENT_INT_SRC_MSK3_ENT95_MSK_MSK	(0x1 << ENT_INT_SRC_MSK3_ENT95_MSK_OFF)
 #define SAS_ECC_INTR			0x1e8
+#define SAS_ECC_INTR_DQE_ECC_1B_OFF		0
+#define SAS_ECC_INTR_DQE_ECC_MB_OFF		1
+#define SAS_ECC_INTR_IOST_ECC_1B_OFF	2
+#define SAS_ECC_INTR_IOST_ECC_MB_OFF	3
+#define SAS_ECC_INTR_ITCT_ECC_1B_OFF	4
+#define SAS_ECC_INTR_ITCT_ECC_MB_OFF	5
+#define SAS_ECC_INTR_ITCTLIST_ECC_1B_OFF	6
+#define SAS_ECC_INTR_ITCTLIST_ECC_MB_OFF	7
+#define SAS_ECC_INTR_IOSTLIST_ECC_1B_OFF	8
+#define SAS_ECC_INTR_IOSTLIST_ECC_MB_OFF	9
+#define SAS_ECC_INTR_CQE_ECC_1B_OFF		10
+#define SAS_ECC_INTR_CQE_ECC_MB_OFF		11
+#define SAS_ECC_INTR_NCQ_MEM0_ECC_1B_OFF	12
+#define SAS_ECC_INTR_NCQ_MEM0_ECC_MB_OFF	13
+#define SAS_ECC_INTR_NCQ_MEM1_ECC_1B_OFF	14
+#define SAS_ECC_INTR_NCQ_MEM1_ECC_MB_OFF	15
+#define SAS_ECC_INTR_NCQ_MEM2_ECC_1B_OFF	16
+#define SAS_ECC_INTR_NCQ_MEM2_ECC_MB_OFF	17
+#define SAS_ECC_INTR_NCQ_MEM3_ECC_1B_OFF	18
+#define SAS_ECC_INTR_NCQ_MEM3_ECC_MB_OFF	19
+#define SAS_ECC_INTR_OOO_RAM_ECC_1B_OFF		20
+#define SAS_ECC_INTR_OOO_RAM_ECC_MB_OFF		21
 #define SAS_ECC_INTR_MSK		0x1ec
 #define HGC_ERR_STAT_EN			0x238
 #define CQE_SEND_CNT			0x248
@@ -105,6 +160,20 @@
 #define COMPL_Q_0_DEPTH			0x4e8
 #define COMPL_Q_0_WR_PTR		0x4ec
 #define COMPL_Q_0_RD_PTR		0x4f0
+#define HGC_RXM_DFX_STATUS14		0xae8
+#define HGC_RXM_DFX_STATUS14_MEM0_OFF	0
+#define HGC_RXM_DFX_STATUS14_MEM0_MSK	(0x1ff << \
+					 HGC_RXM_DFX_STATUS14_MEM0_OFF)
+#define HGC_RXM_DFX_STATUS14_MEM1_OFF	9
+#define HGC_RXM_DFX_STATUS14_MEM1_MSK	(0x1ff << \
+					 HGC_RXM_DFX_STATUS14_MEM1_OFF)
+#define HGC_RXM_DFX_STATUS14_MEM2_OFF	18
+#define HGC_RXM_DFX_STATUS14_MEM2_MSK	(0x1ff << \
+					 HGC_RXM_DFX_STATUS14_MEM2_OFF)
+#define HGC_RXM_DFX_STATUS15		0xaec
+#define HGC_RXM_DFX_STATUS15_MEM3_OFF	0
+#define HGC_RXM_DFX_STATUS15_MEM3_MSK	(0x1ff << \
+					 HGC_RXM_DFX_STATUS15_MEM3_OFF)
 #define AWQOS_AWCACHE_CFG	0xc84
 #define ARQOS_ARCACHE_CFG	0xc88
 #define HILINK_ERR_DFX		0xe04
@@ -172,14 +241,18 @@
 #define CHL_INT0_PHY_RDY_OFF		5
 #define CHL_INT0_PHY_RDY_MSK		(0x1 << CHL_INT0_PHY_RDY_OFF)
 #define CHL_INT1			(PORT_BASE + 0x1b8)
-#define CHL_INT1_DMAC_TX_ECC_ERR_OFF	15
-#define CHL_INT1_DMAC_TX_ECC_ERR_MSK	(0x1 << CHL_INT1_DMAC_TX_ECC_ERR_OFF)
-#define CHL_INT1_DMAC_RX_ECC_ERR_OFF	17
-#define CHL_INT1_DMAC_RX_ECC_ERR_MSK	(0x1 << CHL_INT1_DMAC_RX_ECC_ERR_OFF)
+#define CHL_INT1_DMAC_TX_ECC_MB_ERR_OFF	15
+#define CHL_INT1_DMAC_TX_ECC_1B_ERR_OFF	16
+#define CHL_INT1_DMAC_RX_ECC_MB_ERR_OFF	17
+#define CHL_INT1_DMAC_RX_ECC_1B_ERR_OFF	18
 #define CHL_INT1_DMAC_TX_AXI_WR_ERR_OFF	19
 #define CHL_INT1_DMAC_TX_AXI_RD_ERR_OFF	20
 #define CHL_INT1_DMAC_RX_AXI_WR_ERR_OFF	21
 #define CHL_INT1_DMAC_RX_AXI_RD_ERR_OFF	22
+#define CHL_INT1_DMAC_TX_FIFO_ERR_OFF	23
+#define CHL_INT1_DMAC_RX_FIFO_ERR_OFF	24
+#define CHL_INT1_DMAC_TX_AXI_RUSER_ERR_OFF	26
+#define CHL_INT1_DMAC_RX_AXI_RUSER_ERR_OFF	27
 #define CHL_INT2			(PORT_BASE + 0x1bc)
 #define CHL_INT2_SL_IDAF_TOUT_CONF_OFF	0
 #define CHL_INT2_RX_DISP_ERR_OFF	28
@@ -227,10 +300,8 @@
 #define AM_CFG_SINGLE_PORT_MAX_TRANS	(0x5014)
 #define AXI_CFG					(0x5100)
 #define AM_ROB_ECC_ERR_ADDR		(0x510c)
-#define AM_ROB_ECC_ONEBIT_ERR_ADDR_OFF	0
-#define AM_ROB_ECC_ONEBIT_ERR_ADDR_MSK	(0xff << AM_ROB_ECC_ONEBIT_ERR_ADDR_OFF)
-#define AM_ROB_ECC_MULBIT_ERR_ADDR_OFF	8
-#define AM_ROB_ECC_MULBIT_ERR_ADDR_MSK	(0xff << AM_ROB_ECC_MULBIT_ERR_ADDR_OFF)
+#define AM_ROB_ECC_ERR_ADDR_OFF	0
+#define AM_ROB_ECC_ERR_ADDR_MSK	0xffffffff
 
 /* RAS registers need init */
 #define RAS_BASE		(0x6000)
@@ -408,6 +479,10 @@ struct hisi_sas_err_record_v3 {
 #define BASE_VECTORS_V3_HW  16
 #define MIN_AFFINE_VECTORS_V3_HW  (BASE_VECTORS_V3_HW + 1)
 
+enum {
+	DSM_FUNC_ERR_HANDLE_MSI = 0,
+};
+
 static bool hisi_sas_intr_conv;
 MODULE_PARM_DESC(intr_conv, "interrupt converge enable (0-1)");
 
@@ -474,7 +549,6 @@ static u32 hisi_sas_phy_read32(struct hisi_hba *hisi_hba,
 
 static void init_reg_v3_hw(struct hisi_hba *hisi_hba)
 {
-	struct pci_dev *pdev = hisi_hba->pci_dev;
 	int i;
 
 	/* Global registers init */
@@ -494,14 +568,11 @@ static void init_reg_v3_hw(struct hisi_hba *hisi_hba)
 	hisi_sas_write32(hisi_hba, ENT_INT_SRC3, 0xffffffff);
 	hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK1, 0xfefefefe);
 	hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK2, 0xfefefefe);
-	if (pdev->revision >= 0x21)
-		hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0xffff7aff);
-	else
-		hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0xfffe20ff);
+	hisi_sas_write32(hisi_hba, ENT_INT_SRC_MSK3, 0xffc220ff);
 	hisi_sas_write32(hisi_hba, CHNL_PHYUPDOWN_INT_MSK, 0x0);
 	hisi_sas_write32(hisi_hba, CHNL_ENT_INT_MSK, 0x0);
 	hisi_sas_write32(hisi_hba, HGC_COM_INT_MSK, 0x0);
-	hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, 0x0);
+	hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, 0x155555);
 	hisi_sas_write32(hisi_hba, AWQOS_AWCACHE_CFG, 0xf0f0);
 	hisi_sas_write32(hisi_hba, ARQOS_ARCACHE_CFG, 0xf0f0);
 	for (i = 0; i < hisi_hba->queue_count; i++)
@@ -532,12 +603,7 @@ static void init_reg_v3_hw(struct hisi_hba *hisi_hba)
 		hisi_sas_phy_write32(hisi_hba, i, CHL_INT1, 0xffffffff);
 		hisi_sas_phy_write32(hisi_hba, i, CHL_INT2, 0xffffffff);
 		hisi_sas_phy_write32(hisi_hba, i, RXOP_CHECK_CFG_H, 0x1000);
-		if (pdev->revision >= 0x21)
-			hisi_sas_phy_write32(hisi_hba, i, CHL_INT1_MSK,
-					0xffffffff);
-		else
-			hisi_sas_phy_write32(hisi_hba, i, CHL_INT1_MSK,
-					0xff87ffff);
+		hisi_sas_phy_write32(hisi_hba, i, CHL_INT1_MSK, 0xf2057fff);
 		hisi_sas_phy_write32(hisi_hba, i, CHL_INT2_MSK, 0xffffbfe);
 		hisi_sas_phy_write32(hisi_hba, i, PHY_CTRL_RDY_MSK, 0x0);
 		hisi_sas_phy_write32(hisi_hba, i, PHYCTRL_NOT_RDY_MSK, 0x0);
@@ -804,6 +870,8 @@ static int reset_hw_v3_hw(struct hisi_hba *hisi_hba)
 static int hw_init_v3_hw(struct hisi_hba *hisi_hba)
 {
 	struct device *dev = hisi_hba->dev;
+	union acpi_object *obj;
+	guid_t guid;
 	int rc;
 
 	rc = reset_hw_v3_hw(hisi_hba);
@@ -815,6 +883,19 @@ static int hw_init_v3_hw(struct hisi_hba *hisi_hba)
 	msleep(100);
 	init_reg_v3_hw(hisi_hba);
 
+	if (guid_parse("D5918B4B-37AE-4E10-A99F-E5E8A6EF4C1F", &guid)) {
+		dev_err(dev, "Parse GUID failed\n");
+		return -EINVAL;
+	}
+
+	/* Switch over to MSI handling , from PCI AER default */
+	obj = acpi_evaluate_dsm(ACPI_HANDLE(dev), &guid, 0,
+				DSM_FUNC_ERR_HANDLE_MSI, NULL);
+	if (!obj)
+		dev_warn(dev, "Switch over to MSI handling failed\n");
+	else
+		ACPI_FREE(obj);
+
 	return 0;
 }
 
@@ -856,14 +937,14 @@ static void phy_hard_reset_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
 	struct hisi_sas_phy *phy = &hisi_hba->phy[phy_no];
 	u32 txid_auto;
 
-	disable_phy_v3_hw(hisi_hba, phy_no);
+	hisi_sas_phy_enable(hisi_hba, phy_no, 0);
 	if (phy->identify.device_type == SAS_END_DEVICE) {
 		txid_auto = hisi_sas_phy_read32(hisi_hba, phy_no, TXID_AUTO);
 		hisi_sas_phy_write32(hisi_hba, phy_no, TXID_AUTO,
 					txid_auto | TX_HARDRST_MSK);
 	}
 	msleep(100);
-	start_phy_v3_hw(hisi_hba, phy_no);
+	hisi_sas_phy_enable(hisi_hba, phy_no, 1);
 }
 
 static enum sas_linkrate phy_get_max_linkrate_v3_hw(void)
@@ -882,7 +963,7 @@ static void phys_init_v3_hw(struct hisi_hba *hisi_hba)
 		if (!sas_phy->phy->enabled)
 			continue;
 
-		start_phy_v3_hw(hisi_hba, i);
+		hisi_sas_phy_enable(hisi_hba, i, 1);
 	}
 }
 
@@ -929,7 +1010,7 @@ get_free_slot_v3_hw(struct hisi_hba *hisi_hba, struct hisi_sas_dq *dq)
 				DLVRY_Q_0_RD_PTR + (queue * 0x14));
 	if (r == (w+1) % HISI_SAS_QUEUE_SLOTS) {
 		dev_warn(dev, "full queue=%d r=%d w=%d\n",
-				queue, r, w);
+			 queue, r, w);
 		return -EAGAIN;
 	}
 
@@ -1380,6 +1461,7 @@ static irqreturn_t phy_up_v3_hw(int phy_no, struct hisi_hba *hisi_hba)
 		struct hisi_sas_initial_fis *initial_fis;
 		struct dev_to_host_fis *fis;
 		u8 attached_sas_addr[SAS_ADDR_SIZE] = {0};
+		struct Scsi_Host *shost = hisi_hba->shost;
 
 		dev_info(dev, "phyup: phy%d link_rate=%d(sata)\n", phy_no, link_rate);
 		initial_fis = &hisi_hba->initial_fis[phy_no];
@@ -1396,6 +1478,7 @@ static irqreturn_t phy_up_v3_hw(int phy_no, struct hisi_hba *hisi_hba)
 
 		sas_phy->oob_mode = SATA_OOB_MODE;
 		attached_sas_addr[0] = 0x50;
+		attached_sas_addr[6] = shost->host_no;
 		attached_sas_addr[7] = phy_no;
 		memcpy(sas_phy->attached_sas_addr,
 		       attached_sas_addr,
@@ -1540,6 +1623,14 @@ static irqreturn_t int_phy_up_down_bcast_v3_hw(int irq_no, void *p)
 
 static const struct hisi_sas_hw_error port_axi_error[] = {
 	{
+		.irq_msk = BIT(CHL_INT1_DMAC_TX_ECC_MB_ERR_OFF),
+		.msg = "dmac_tx_ecc_bad_err",
+	},
+	{
+		.irq_msk = BIT(CHL_INT1_DMAC_RX_ECC_MB_ERR_OFF),
+		.msg = "dmac_rx_ecc_bad_err",
+	},
+	{
 		.irq_msk = BIT(CHL_INT1_DMAC_TX_AXI_WR_ERR_OFF),
 		.msg = "dma_tx_axi_wr_err",
 	},
@@ -1555,6 +1646,22 @@ static const struct hisi_sas_hw_error port_axi_error[] = {
 		.irq_msk = BIT(CHL_INT1_DMAC_RX_AXI_RD_ERR_OFF),
 		.msg = "dma_rx_axi_rd_err",
 	},
+	{
+		.irq_msk = BIT(CHL_INT1_DMAC_TX_FIFO_ERR_OFF),
+		.msg = "dma_tx_fifo_err",
+	},
+	{
+		.irq_msk = BIT(CHL_INT1_DMAC_RX_FIFO_ERR_OFF),
+		.msg = "dma_rx_fifo_err",
+	},
+	{
+		.irq_msk = BIT(CHL_INT1_DMAC_TX_AXI_RUSER_ERR_OFF),
+		.msg = "dma_tx_axi_ruser_err",
+	},
+	{
+		.irq_msk = BIT(CHL_INT1_DMAC_RX_AXI_RUSER_ERR_OFF),
+		.msg = "dma_rx_axi_ruser_err",
+	},
 };
 
 static void handle_chl_int1_v3_hw(struct hisi_hba *hisi_hba, int phy_no)
@@ -1719,6 +1826,121 @@ static irqreturn_t int_chnl_int_v3_hw(int irq_no, void *p)
 	return IRQ_HANDLED;
 }
 
+static const struct hisi_sas_hw_error multi_bit_ecc_errors[] = {
+	{
+		.irq_msk = BIT(SAS_ECC_INTR_DQE_ECC_MB_OFF),
+		.msk = HGC_DQE_ECC_MB_ADDR_MSK,
+		.shift = HGC_DQE_ECC_MB_ADDR_OFF,
+		.msg = "hgc_dqe_eccbad_intr found: ram addr is 0x%08X\n",
+		.reg = HGC_DQE_ECC_ADDR,
+	},
+	{
+		.irq_msk = BIT(SAS_ECC_INTR_IOST_ECC_MB_OFF),
+		.msk = HGC_IOST_ECC_MB_ADDR_MSK,
+		.shift = HGC_IOST_ECC_MB_ADDR_OFF,
+		.msg = "hgc_iost_eccbad_intr found: ram addr is 0x%08X\n",
+		.reg = HGC_IOST_ECC_ADDR,
+	},
+	{
+		.irq_msk = BIT(SAS_ECC_INTR_ITCT_ECC_MB_OFF),
+		.msk = HGC_ITCT_ECC_MB_ADDR_MSK,
+		.shift = HGC_ITCT_ECC_MB_ADDR_OFF,
+		.msg = "hgc_itct_eccbad_intr found: ram addr is 0x%08X\n",
+		.reg = HGC_ITCT_ECC_ADDR,
+	},
+	{
+		.irq_msk = BIT(SAS_ECC_INTR_IOSTLIST_ECC_MB_OFF),
+		.msk = HGC_LM_DFX_STATUS2_IOSTLIST_MSK,
+		.shift = HGC_LM_DFX_STATUS2_IOSTLIST_OFF,
+		.msg = "hgc_iostl_eccbad_intr found: mem addr is 0x%08X\n",
+		.reg = HGC_LM_DFX_STATUS2,
+	},
+	{
+		.irq_msk = BIT(SAS_ECC_INTR_ITCTLIST_ECC_MB_OFF),
+		.msk = HGC_LM_DFX_STATUS2_ITCTLIST_MSK,
+		.shift = HGC_LM_DFX_STATUS2_ITCTLIST_OFF,
+		.msg = "hgc_itctl_eccbad_intr found: mem addr is 0x%08X\n",
+		.reg = HGC_LM_DFX_STATUS2,
+	},
+	{
+		.irq_msk = BIT(SAS_ECC_INTR_CQE_ECC_MB_OFF),
+		.msk = HGC_CQE_ECC_MB_ADDR_MSK,
+		.shift = HGC_CQE_ECC_MB_ADDR_OFF,
+		.msg = "hgc_cqe_eccbad_intr found: ram address is 0x%08X\n",
+		.reg = HGC_CQE_ECC_ADDR,
+	},
+	{
+		.irq_msk = BIT(SAS_ECC_INTR_NCQ_MEM0_ECC_MB_OFF),
+		.msk = HGC_RXM_DFX_STATUS14_MEM0_MSK,
+		.shift = HGC_RXM_DFX_STATUS14_MEM0_OFF,
+		.msg = "rxm_mem0_eccbad_intr found: mem addr is 0x%08X\n",
+		.reg = HGC_RXM_DFX_STATUS14,
+	},
+	{
+		.irq_msk = BIT(SAS_ECC_INTR_NCQ_MEM1_ECC_MB_OFF),
+		.msk = HGC_RXM_DFX_STATUS14_MEM1_MSK,
+		.shift = HGC_RXM_DFX_STATUS14_MEM1_OFF,
+		.msg = "rxm_mem1_eccbad_intr found: mem addr is 0x%08X\n",
+		.reg = HGC_RXM_DFX_STATUS14,
+	},
+	{
+		.irq_msk = BIT(SAS_ECC_INTR_NCQ_MEM2_ECC_MB_OFF),
+		.msk = HGC_RXM_DFX_STATUS14_MEM2_MSK,
+		.shift = HGC_RXM_DFX_STATUS14_MEM2_OFF,
+		.msg = "rxm_mem2_eccbad_intr found: mem addr is 0x%08X\n",
+		.reg = HGC_RXM_DFX_STATUS14,
+	},
+	{
+		.irq_msk = BIT(SAS_ECC_INTR_NCQ_MEM3_ECC_MB_OFF),
+		.msk = HGC_RXM_DFX_STATUS15_MEM3_MSK,
+		.shift = HGC_RXM_DFX_STATUS15_MEM3_OFF,
+		.msg = "rxm_mem3_eccbad_intr found: mem addr is 0x%08X\n",
+		.reg = HGC_RXM_DFX_STATUS15,
+	},
+	{
+		.irq_msk = BIT(SAS_ECC_INTR_OOO_RAM_ECC_MB_OFF),
+		.msk = AM_ROB_ECC_ERR_ADDR_MSK,
+		.shift = AM_ROB_ECC_ERR_ADDR_OFF,
+		.msg = "ooo_ram_eccbad_intr found: ROB_ECC_ERR_ADDR=0x%08X\n",
+		.reg = AM_ROB_ECC_ERR_ADDR,
+	},
+};
+
+static void multi_bit_ecc_error_process_v3_hw(struct hisi_hba *hisi_hba,
+					      u32 irq_value)
+{
+	struct device *dev = hisi_hba->dev;
+	const struct hisi_sas_hw_error *ecc_error;
+	u32 val;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(multi_bit_ecc_errors); i++) {
+		ecc_error = &multi_bit_ecc_errors[i];
+		if (irq_value & ecc_error->irq_msk) {
+			val = hisi_sas_read32(hisi_hba, ecc_error->reg);
+			val &= ecc_error->msk;
+			val >>= ecc_error->shift;
+			dev_err(dev, ecc_error->msg, irq_value, val);
+			queue_work(hisi_hba->wq, &hisi_hba->rst_work);
+		}
+	}
+}
+
+static void fatal_ecc_int_v3_hw(struct hisi_hba *hisi_hba)
+{
+	u32 irq_value, irq_msk;
+
+	irq_msk = hisi_sas_read32(hisi_hba, SAS_ECC_INTR_MSK);
+	hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, irq_msk | 0xffffffff);
+
+	irq_value = hisi_sas_read32(hisi_hba, SAS_ECC_INTR);
+	if (irq_value)
+		multi_bit_ecc_error_process_v3_hw(hisi_hba, irq_value);
+
+	hisi_sas_write32(hisi_hba, SAS_ECC_INTR, irq_value);
+	hisi_sas_write32(hisi_hba, SAS_ECC_INTR_MSK, irq_msk);
+}
+
 static const struct hisi_sas_hw_error axi_error[] = {
 	{ .msk = BIT(0), .msg = "IOST_AXI_W_ERR" },
 	{ .msk = BIT(1), .msg = "IOST_AXI_R_ERR" },
@@ -1728,7 +1950,7 @@ static const struct hisi_sas_hw_error axi_error[] = {
 	{ .msk = BIT(5), .msg = "SATA_AXI_R_ERR" },
 	{ .msk = BIT(6), .msg = "DQE_AXI_R_ERR" },
 	{ .msk = BIT(7), .msg = "CQE_AXI_W_ERR" },
-	{},
+	{}
 };
 
 static const struct hisi_sas_hw_error fifo_error[] = {
@@ -1737,7 +1959,7 @@ static const struct hisi_sas_hw_error fifo_error[] = {
 	{ .msk = BIT(10), .msg = "GETDQE_FIFO" },
 	{ .msk = BIT(11), .msg = "CMDP_FIFO" },
 	{ .msk = BIT(12), .msg = "AWTCTRL_FIFO" },
-	{},
+	{}
 };
 
 static const struct hisi_sas_hw_error fatal_axi_error[] = {
@@ -1771,6 +1993,23 @@ static const struct hisi_sas_hw_error fatal_axi_error[] = {
 		.irq_msk = BIT(ENT_INT_SRC3_ABT_OFF),
 		.msg = "SAS_HGC_ABT fetch LM list",
 	},
+	{
+		.irq_msk = BIT(ENT_INT_SRC3_DQE_POISON_OFF),
+		.msg = "read dqe poison",
+	},
+	{
+		.irq_msk = BIT(ENT_INT_SRC3_IOST_POISON_OFF),
+		.msg = "read iost poison",
+	},
+	{
+		.irq_msk = BIT(ENT_INT_SRC3_ITCT_POISON_OFF),
+		.msg = "read itct poison",
+	},
+	{
+		.irq_msk = BIT(ENT_INT_SRC3_ITCT_NCQ_POISON_OFF),
+		.msg = "read itct ncq poison",
+	},
+
 };
 
 static irqreturn_t fatal_axi_int_v3_hw(int irq_no, void *p)
@@ -1823,6 +2062,8 @@ static irqreturn_t fatal_axi_int_v3_hw(int irq_no, void *p)
 		}
 	}
 
+	fatal_ecc_int_v3_hw(hisi_hba);
+
 	if (irq_value & BIT(ENT_INT_SRC3_ITC_INT_OFF)) {
 		u32 reg_val = hisi_sas_read32(hisi_hba, ITCT_CLR);
 		u32 dev_id = reg_val & ITCT_DEV_MSK;
@@ -1966,13 +2207,11 @@ slot_complete_v3_hw(struct hisi_hba *hisi_hba, struct hisi_sas_slot *slot)
 
 		slot_err_v3_hw(hisi_hba, task, slot);
 		if (ts->stat != SAS_DATA_UNDERRUN)
-			dev_info(dev, "erroneous completion iptt=%d task=%p dev id=%d "
-				"CQ hdr: 0x%x 0x%x 0x%x 0x%x "
-				"Error info: 0x%x 0x%x 0x%x 0x%x\n",
-				slot->idx, task, sas_dev->device_id,
-				dw0, dw1, complete_hdr->act, dw3,
-				error_info[0], error_info[1],
-				error_info[2], error_info[3]);
+			dev_info(dev, "erroneous completion iptt=%d task=%p dev id=%d CQ hdr: 0x%x 0x%x 0x%x 0x%x Error info: 0x%x 0x%x 0x%x 0x%x\n",
+				 slot->idx, task, sas_dev->device_id,
+				 dw0, dw1, complete_hdr->act, dw3,
+				 error_info[0], error_info[1],
+				 error_info[2], error_info[3]);
 		if (unlikely(slot->abort))
 			return ts->stat;
 		goto out;
@@ -2205,8 +2444,7 @@ static int interrupt_init_v3_hw(struct hisi_hba *hisi_hba)
 				      cq_interrupt_v3_hw, irqflags,
 				      DRV_NAME " cq", cq);
 		if (rc) {
-			dev_err(dev,
-				"could not request cq%d interrupt, rc=%d\n",
+			dev_err(dev, "could not request cq%d interrupt, rc=%d\n",
 				i, rc);
 			rc = -ENOENT;
 			goto free_cq_irqs;
@@ -2362,7 +2600,7 @@ static int write_gpio_v3_hw(struct hisi_hba *hisi_hba, u8 reg_type,
 		break;
 	default:
 		dev_err(dev, "write gpio: unsupported or bad reg type %d\n",
-				reg_type);
+			reg_type);
 		return -EINVAL;
 	}
 
@@ -2678,6 +2916,7 @@ static struct scsi_host_template sht_v3_hw = {
 	.ioctl			= sas_ioctl,
 	.shost_attrs		= host_attrs_v3_hw,
 	.tag_alloc_policy	= BLK_TAG_ALLOC_RR,
+	.host_reset             = hisi_sas_host_reset,
 };
 
 static const struct hisi_sas_hw hisi_sas_v3_hw = {
@@ -2800,7 +3039,7 @@ hisi_sas_v3_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 
 	hisi_hba->regs = pcim_iomap(pdev, 5, 0);
 	if (!hisi_hba->regs) {
-		dev_err(dev, "cannot map register.\n");
+		dev_err(dev, "cannot map register\n");
 		rc = -ENOMEM;
 		goto err_out_ha;
 	}
@@ -2921,161 +3160,6 @@ static void hisi_sas_v3_remove(struct pci_dev *pdev)
 	scsi_host_put(shost);
 }
 
-static const struct hisi_sas_hw_error sas_ras_intr0_nfe[] = {
-	{ .irq_msk = BIT(19), .msg = "HILINK_INT" },
-	{ .irq_msk = BIT(20), .msg = "HILINK_PLL0_OUT_OF_LOCK" },
-	{ .irq_msk = BIT(21), .msg = "HILINK_PLL1_OUT_OF_LOCK" },
-	{ .irq_msk = BIT(22), .msg = "HILINK_LOSS_OF_REFCLK0" },
-	{ .irq_msk = BIT(23), .msg = "HILINK_LOSS_OF_REFCLK1" },
-	{ .irq_msk = BIT(24), .msg = "DMAC0_TX_POISON" },
-	{ .irq_msk = BIT(25), .msg = "DMAC1_TX_POISON" },
-	{ .irq_msk = BIT(26), .msg = "DMAC2_TX_POISON" },
-	{ .irq_msk = BIT(27), .msg = "DMAC3_TX_POISON" },
-	{ .irq_msk = BIT(28), .msg = "DMAC4_TX_POISON" },
-	{ .irq_msk = BIT(29), .msg = "DMAC5_TX_POISON" },
-	{ .irq_msk = BIT(30), .msg = "DMAC6_TX_POISON" },
-	{ .irq_msk = BIT(31), .msg = "DMAC7_TX_POISON" },
-};
-
-static const struct hisi_sas_hw_error sas_ras_intr1_nfe[] = {
-	{ .irq_msk = BIT(0), .msg = "RXM_CFG_MEM3_ECC2B_INTR" },
-	{ .irq_msk = BIT(1), .msg = "RXM_CFG_MEM2_ECC2B_INTR" },
-	{ .irq_msk = BIT(2), .msg = "RXM_CFG_MEM1_ECC2B_INTR" },
-	{ .irq_msk = BIT(3), .msg = "RXM_CFG_MEM0_ECC2B_INTR" },
-	{ .irq_msk = BIT(4), .msg = "HGC_CQE_ECC2B_INTR" },
-	{ .irq_msk = BIT(5), .msg = "LM_CFG_IOSTL_ECC2B_INTR" },
-	{ .irq_msk = BIT(6), .msg = "LM_CFG_ITCTL_ECC2B_INTR" },
-	{ .irq_msk = BIT(7), .msg = "HGC_ITCT_ECC2B_INTR" },
-	{ .irq_msk = BIT(8), .msg = "HGC_IOST_ECC2B_INTR" },
-	{ .irq_msk = BIT(9), .msg = "HGC_DQE_ECC2B_INTR" },
-	{ .irq_msk = BIT(10), .msg = "DMAC0_RAM_ECC2B_INTR" },
-	{ .irq_msk = BIT(11), .msg = "DMAC1_RAM_ECC2B_INTR" },
-	{ .irq_msk = BIT(12), .msg = "DMAC2_RAM_ECC2B_INTR" },
-	{ .irq_msk = BIT(13), .msg = "DMAC3_RAM_ECC2B_INTR" },
-	{ .irq_msk = BIT(14), .msg = "DMAC4_RAM_ECC2B_INTR" },
-	{ .irq_msk = BIT(15), .msg = "DMAC5_RAM_ECC2B_INTR" },
-	{ .irq_msk = BIT(16), .msg = "DMAC6_RAM_ECC2B_INTR" },
-	{ .irq_msk = BIT(17), .msg = "DMAC7_RAM_ECC2B_INTR" },
-	{ .irq_msk = BIT(18), .msg = "OOO_RAM_ECC2B_INTR" },
-	{ .irq_msk = BIT(20), .msg = "HGC_DQE_POISON_INTR" },
-	{ .irq_msk = BIT(21), .msg = "HGC_IOST_POISON_INTR" },
-	{ .irq_msk = BIT(22), .msg = "HGC_ITCT_POISON_INTR" },
-	{ .irq_msk = BIT(23), .msg = "HGC_ITCT_NCQ_POISON_INTR" },
-	{ .irq_msk = BIT(24), .msg = "DMAC0_RX_POISON" },
-	{ .irq_msk = BIT(25), .msg = "DMAC1_RX_POISON" },
-	{ .irq_msk = BIT(26), .msg = "DMAC2_RX_POISON" },
-	{ .irq_msk = BIT(27), .msg = "DMAC3_RX_POISON" },
-	{ .irq_msk = BIT(28), .msg = "DMAC4_RX_POISON" },
-	{ .irq_msk = BIT(29), .msg = "DMAC5_RX_POISON" },
-	{ .irq_msk = BIT(30), .msg = "DMAC6_RX_POISON" },
-	{ .irq_msk = BIT(31), .msg = "DMAC7_RX_POISON" },
-};
-
-static const struct hisi_sas_hw_error sas_ras_intr2_nfe[] = {
-	{ .irq_msk = BIT(0), .msg = "DMAC0_AXI_BUS_ERR" },
-	{ .irq_msk = BIT(1), .msg = "DMAC1_AXI_BUS_ERR" },
-	{ .irq_msk = BIT(2), .msg = "DMAC2_AXI_BUS_ERR" },
-	{ .irq_msk = BIT(3), .msg = "DMAC3_AXI_BUS_ERR" },
-	{ .irq_msk = BIT(4), .msg = "DMAC4_AXI_BUS_ERR" },
-	{ .irq_msk = BIT(5), .msg = "DMAC5_AXI_BUS_ERR" },
-	{ .irq_msk = BIT(6), .msg = "DMAC6_AXI_BUS_ERR" },
-	{ .irq_msk = BIT(7), .msg = "DMAC7_AXI_BUS_ERR" },
-	{ .irq_msk = BIT(8), .msg = "DMAC0_FIFO_OMIT_ERR" },
-	{ .irq_msk = BIT(9), .msg = "DMAC1_FIFO_OMIT_ERR" },
-	{ .irq_msk = BIT(10), .msg = "DMAC2_FIFO_OMIT_ERR" },
-	{ .irq_msk = BIT(11), .msg = "DMAC3_FIFO_OMIT_ERR" },
-	{ .irq_msk = BIT(12), .msg = "DMAC4_FIFO_OMIT_ERR" },
-	{ .irq_msk = BIT(13), .msg = "DMAC5_FIFO_OMIT_ERR" },
-	{ .irq_msk = BIT(14), .msg = "DMAC6_FIFO_OMIT_ERR" },
-	{ .irq_msk = BIT(15), .msg = "DMAC7_FIFO_OMIT_ERR" },
-	{ .irq_msk = BIT(16), .msg = "HGC_RLSE_SLOT_UNMATCH" },
-	{ .irq_msk = BIT(17), .msg = "HGC_LM_ADD_FCH_LIST_ERR" },
-	{ .irq_msk = BIT(18), .msg = "HGC_AXI_BUS_ERR" },
-	{ .irq_msk = BIT(19), .msg = "HGC_FIFO_OMIT_ERR" },
-};
-
-static bool process_non_fatal_error_v3_hw(struct hisi_hba *hisi_hba)
-{
-	struct device *dev = hisi_hba->dev;
-	const struct hisi_sas_hw_error *ras_error;
-	bool need_reset = false;
-	u32 irq_value;
-	int i;
-
-	irq_value = hisi_sas_read32(hisi_hba, SAS_RAS_INTR0);
-	for (i = 0; i < ARRAY_SIZE(sas_ras_intr0_nfe); i++) {
-		ras_error = &sas_ras_intr0_nfe[i];
-		if (ras_error->irq_msk & irq_value) {
-			dev_warn(dev, "SAS_RAS_INTR0: %s(irq_value=0x%x) found.\n",
-					ras_error->msg, irq_value);
-			need_reset = true;
-		}
-	}
-	hisi_sas_write32(hisi_hba, SAS_RAS_INTR0, irq_value);
-
-	irq_value = hisi_sas_read32(hisi_hba, SAS_RAS_INTR1);
-	for (i = 0; i < ARRAY_SIZE(sas_ras_intr1_nfe); i++) {
-		ras_error = &sas_ras_intr1_nfe[i];
-		if (ras_error->irq_msk & irq_value) {
-			dev_warn(dev, "SAS_RAS_INTR1: %s(irq_value=0x%x) found.\n",
-					ras_error->msg, irq_value);
-			need_reset = true;
-		}
-	}
-	hisi_sas_write32(hisi_hba, SAS_RAS_INTR1, irq_value);
-
-	irq_value = hisi_sas_read32(hisi_hba, SAS_RAS_INTR2);
-	for (i = 0; i < ARRAY_SIZE(sas_ras_intr2_nfe); i++) {
-		ras_error = &sas_ras_intr2_nfe[i];
-		if (ras_error->irq_msk & irq_value) {
-			dev_warn(dev, "SAS_RAS_INTR2: %s(irq_value=0x%x) found.\n",
-					ras_error->msg, irq_value);
-			need_reset = true;
-		}
-	}
-	hisi_sas_write32(hisi_hba, SAS_RAS_INTR2, irq_value);
-
-	return need_reset;
-}
-
-static pci_ers_result_t hisi_sas_error_detected_v3_hw(struct pci_dev *pdev,
-		pci_channel_state_t state)
-{
-	struct sas_ha_struct *sha = pci_get_drvdata(pdev);
-	struct hisi_hba *hisi_hba = sha->lldd_ha;
-	struct device *dev = hisi_hba->dev;
-
-	dev_info(dev, "PCI error: detected callback, state(%d)!!\n", state);
-	if (state == pci_channel_io_perm_failure)
-		return PCI_ERS_RESULT_DISCONNECT;
-
-	if (process_non_fatal_error_v3_hw(hisi_hba))
-		return PCI_ERS_RESULT_NEED_RESET;
-
-	return PCI_ERS_RESULT_CAN_RECOVER;
-}
-
-static pci_ers_result_t hisi_sas_mmio_enabled_v3_hw(struct pci_dev *pdev)
-{
-	return PCI_ERS_RESULT_RECOVERED;
-}
-
-static pci_ers_result_t hisi_sas_slot_reset_v3_hw(struct pci_dev *pdev)
-{
-	struct sas_ha_struct *sha = pci_get_drvdata(pdev);
-	struct hisi_hba *hisi_hba = sha->lldd_ha;
-	struct device *dev = hisi_hba->dev;
-	HISI_SAS_DECLARE_RST_WORK_ON_STACK(r);
-
-	dev_info(dev, "PCI error: slot reset callback!!\n");
-	queue_work(hisi_hba->wq, &r.work);
-	wait_for_completion(r.completion);
-	if (r.done)
-		return PCI_ERS_RESULT_RECOVERED;
-
-	return PCI_ERS_RESULT_DISCONNECT;
-}
-
 static void hisi_sas_reset_prepare_v3_hw(struct pci_dev *pdev)
 {
 	struct sas_ha_struct *sha = pci_get_drvdata(pdev);
@@ -3171,7 +3255,7 @@ static int hisi_sas_v3_resume(struct pci_dev *pdev)
 	pci_power_t device_state = pdev->current_state;
 
 	dev_warn(dev, "resuming from operating state [D%d]\n",
-			device_state);
+		 device_state);
 	pci_set_power_state(pdev, PCI_D0);
 	pci_enable_wake(pdev, PCI_D0, 0);
 	pci_restore_state(pdev);
@@ -3199,9 +3283,6 @@ static const struct pci_device_id sas_v3_pci_table[] = {
 MODULE_DEVICE_TABLE(pci, sas_v3_pci_table);
 
 static const struct pci_error_handlers hisi_sas_err_handler = {
-	.error_detected	= hisi_sas_error_detected_v3_hw,
-	.mmio_enabled	= hisi_sas_mmio_enabled_v3_hw,
-	.slot_reset	= hisi_sas_slot_reset_v3_hw,
 	.reset_prepare	= hisi_sas_reset_prepare_v3_hw,
 	.reset_done	= hisi_sas_reset_done_v3_hw,
 };
diff --git a/drivers/scsi/hpsa.c b/drivers/scsi/hpsa.c
index f044e7d10d63..1bef1da273c2 100644
--- a/drivers/scsi/hpsa.c
+++ b/drivers/scsi/hpsa.c
@@ -60,7 +60,7 @@
  * HPSA_DRIVER_VERSION must be 3 byte values (0-255) separated by '.'
  * with an optional trailing '-' followed by a byte value (0-255).
  */
-#define HPSA_DRIVER_VERSION "3.4.20-125"
+#define HPSA_DRIVER_VERSION "3.4.20-160"
 #define DRIVER_NAME "HP HPSA Driver (v " HPSA_DRIVER_VERSION ")"
 #define HPSA "hpsa"
 
@@ -2647,9 +2647,20 @@ static void complete_scsi_command(struct CommandList *cp)
 			decode_sense_data(ei->SenseInfo, sense_data_size,
 				&sense_key, &asc, &ascq);
 		if (ei->ScsiStatus == SAM_STAT_CHECK_CONDITION) {
-			if (sense_key == ABORTED_COMMAND) {
+			switch (sense_key) {
+			case ABORTED_COMMAND:
 				cmd->result |= DID_SOFT_ERROR << 16;
 				break;
+			case UNIT_ATTENTION:
+				if (asc == 0x3F && ascq == 0x0E)
+					h->drv_req_rescan = 1;
+				break;
+			case ILLEGAL_REQUEST:
+				if (asc == 0x25 && ascq == 0x00) {
+					dev->removed = 1;
+					cmd->result = DID_NO_CONNECT << 16;
+				}
+				break;
 			}
 			break;
 		}
@@ -3956,14 +3967,18 @@ static int hpsa_update_device_info(struct ctlr_info *h,
 	memset(this_device->device_id, 0,
 		sizeof(this_device->device_id));
 	if (hpsa_get_device_id(h, scsi3addr, this_device->device_id, 8,
-		sizeof(this_device->device_id)) < 0)
+		sizeof(this_device->device_id)) < 0) {
 		dev_err(&h->pdev->dev,
-			"hpsa%d: %s: can't get device id for host %d:C0:T%d:L%d\t%s\t%.16s\n",
+			"hpsa%d: %s: can't get device id for [%d:%d:%d:%d]\t%s\t%.16s\n",
 			h->ctlr, __func__,
 			h->scsi_host->host_no,
-			this_device->target, this_device->lun,
+			this_device->bus, this_device->target,
+			this_device->lun,
 			scsi_device_type(this_device->devtype),
 			this_device->model);
+		rc = HPSA_LV_FAILED;
+		goto bail_out;
+	}
 
 	if ((this_device->devtype == TYPE_DISK ||
 		this_device->devtype == TYPE_ZBC) &&
@@ -5809,7 +5824,7 @@ static int hpsa_send_test_unit_ready(struct ctlr_info *h,
 	/* Send the Test Unit Ready, fill_cmd can't fail, no mapping */
 	(void) fill_cmd(c, TEST_UNIT_READY, h,
 			NULL, 0, 0, lunaddr, TYPE_CMD);
-	rc = hpsa_scsi_do_simple_cmd(h, c, reply_queue, DEFAULT_TIMEOUT);
+	rc = hpsa_scsi_do_simple_cmd(h, c, reply_queue, NO_TIMEOUT);
 	if (rc)
 		return rc;
 	/* no unmap needed here because no data xfer. */
diff --git a/drivers/scsi/libsas/sas_ata.c b/drivers/scsi/libsas/sas_ata.c
index 6f93fee2b21b..1ecca71df8b5 100644
--- a/drivers/scsi/libsas/sas_ata.c
+++ b/drivers/scsi/libsas/sas_ata.c
@@ -281,7 +281,7 @@ int sas_get_ata_info(struct domain_device *dev, struct ex_phy *phy)
 		res = sas_get_report_phy_sata(dev->parent, phy->phy_id,
 					      &dev->sata_dev.rps_resp);
 		if (res) {
-			pr_debug("report phy sata to %016llx:0x%x returned 0x%x\n",
+			pr_debug("report phy sata to %016llx:%02d returned 0x%x\n",
 				 SAS_ADDR(dev->parent->sas_addr),
 				 phy->phy_id, res);
 			return res;
diff --git a/drivers/scsi/libsas/sas_expander.c b/drivers/scsi/libsas/sas_expander.c
index 17b45a0c7bc3..83f2fd70ce76 100644
--- a/drivers/scsi/libsas/sas_expander.c
+++ b/drivers/scsi/libsas/sas_expander.c
@@ -826,9 +826,14 @@ static struct domain_device *sas_ex_discover_end_dev(
 #ifdef CONFIG_SCSI_SAS_ATA
 	if ((phy->attached_tproto & SAS_PROTOCOL_STP) || phy->attached_sata_dev) {
 		if (child->linkrate > parent->min_linkrate) {
+			struct sas_phy *cphy = child->phy;
+			enum sas_linkrate min_prate = cphy->minimum_linkrate,
+				parent_min_lrate = parent->min_linkrate,
+				min_linkrate = (min_prate > parent_min_lrate) ?
+					       parent_min_lrate : 0;
 			struct sas_phy_linkrates rates = {
 				.maximum_linkrate = parent->min_linkrate,
-				.minimum_linkrate = parent->min_linkrate,
+				.minimum_linkrate = min_linkrate,
 			};
 			int ret;
 
@@ -865,7 +870,7 @@ static struct domain_device *sas_ex_discover_end_dev(
 
 		res = sas_discover_sata(child);
 		if (res) {
-			pr_notice("sas_discover_sata() for device %16llx at %016llx:0x%x returned 0x%x\n",
+			pr_notice("sas_discover_sata() for device %16llx at %016llx:%02d returned 0x%x\n",
 				  SAS_ADDR(child->sas_addr),
 				  SAS_ADDR(parent->sas_addr), phy_id, res);
 			goto out_list_del;
@@ -890,7 +895,7 @@ static struct domain_device *sas_ex_discover_end_dev(
 
 		res = sas_discover_end_dev(child);
 		if (res) {
-			pr_notice("sas_discover_end_dev() for device %16llx at %016llx:0x%x returned 0x%x\n",
+			pr_notice("sas_discover_end_dev() for device %16llx at %016llx:%02d returned 0x%x\n",
 				  SAS_ADDR(child->sas_addr),
 				  SAS_ADDR(parent->sas_addr), phy_id, res);
 			goto out_list_del;
@@ -955,7 +960,7 @@ static struct domain_device *sas_ex_discover_expander(
 	int res;
 
 	if (phy->routing_attr == DIRECT_ROUTING) {
-		pr_warn("ex %016llx:0x%x:D <--> ex %016llx:0x%x is not allowed\n",
+		pr_warn("ex %016llx:%02d:D <--> ex %016llx:0x%x is not allowed\n",
 			SAS_ADDR(parent->sas_addr), phy_id,
 			SAS_ADDR(phy->attached_sas_addr),
 			phy->attached_phy_id);
@@ -1065,7 +1070,7 @@ static int sas_ex_discover_dev(struct domain_device *dev, int phy_id)
 	    ex_phy->attached_dev_type != SAS_FANOUT_EXPANDER_DEVICE &&
 	    ex_phy->attached_dev_type != SAS_EDGE_EXPANDER_DEVICE &&
 	    ex_phy->attached_dev_type != SAS_SATA_PENDING) {
-		pr_warn("unknown device type(0x%x) attached to ex %016llx phy 0x%x\n",
+		pr_warn("unknown device type(0x%x) attached to ex %016llx phy%02d\n",
 			ex_phy->attached_dev_type,
 			SAS_ADDR(dev->sas_addr),
 			phy_id);
@@ -1081,7 +1086,7 @@ static int sas_ex_discover_dev(struct domain_device *dev, int phy_id)
 	}
 
 	if (sas_ex_join_wide_port(dev, phy_id)) {
-		pr_debug("Attaching ex phy%d to wide port %016llx\n",
+		pr_debug("Attaching ex phy%02d to wide port %016llx\n",
 			 phy_id, SAS_ADDR(ex_phy->attached_sas_addr));
 		return res;
 	}
@@ -1093,7 +1098,7 @@ static int sas_ex_discover_dev(struct domain_device *dev, int phy_id)
 		break;
 	case SAS_FANOUT_EXPANDER_DEVICE:
 		if (SAS_ADDR(dev->port->disc.fanout_sas_addr)) {
-			pr_debug("second fanout expander %016llx phy 0x%x attached to ex %016llx phy 0x%x\n",
+			pr_debug("second fanout expander %016llx phy%02d attached to ex %016llx phy%02d\n",
 				 SAS_ADDR(ex_phy->attached_sas_addr),
 				 ex_phy->attached_phy_id,
 				 SAS_ADDR(dev->sas_addr),
@@ -1126,7 +1131,7 @@ static int sas_ex_discover_dev(struct domain_device *dev, int phy_id)
 			    SAS_ADDR(child->sas_addr)) {
 				ex->ex_phy[i].phy_state= PHY_DEVICE_DISCOVERED;
 				if (sas_ex_join_wide_port(dev, i))
-					pr_debug("Attaching ex phy%d to wide port %016llx\n",
+					pr_debug("Attaching ex phy%02d to wide port %016llx\n",
 						 i, SAS_ADDR(ex->ex_phy[i].attached_sas_addr));
 			}
 		}
@@ -1151,7 +1156,7 @@ static int sas_find_sub_addr(struct domain_device *dev, u8 *sub_addr)
 		     phy->attached_dev_type == SAS_FANOUT_EXPANDER_DEVICE) &&
 		    phy->routing_attr == SUBTRACTIVE_ROUTING) {
 
-			memcpy(sub_addr, phy->attached_sas_addr,SAS_ADDR_SIZE);
+			memcpy(sub_addr, phy->attached_sas_addr, SAS_ADDR_SIZE);
 
 			return 1;
 		}
@@ -1163,7 +1168,7 @@ static int sas_check_level_subtractive_boundary(struct domain_device *dev)
 {
 	struct expander_device *ex = &dev->ex_dev;
 	struct domain_device *child;
-	u8 sub_addr[8] = {0, };
+	u8 sub_addr[SAS_ADDR_SIZE] = {0, };
 
 	list_for_each_entry(child, &ex->children, siblings) {
 		if (child->dev_type != SAS_EDGE_EXPANDER_DEVICE &&
@@ -1173,7 +1178,7 @@ static int sas_check_level_subtractive_boundary(struct domain_device *dev)
 			sas_find_sub_addr(child, sub_addr);
 			continue;
 		} else {
-			u8 s2[8];
+			u8 s2[SAS_ADDR_SIZE];
 
 			if (sas_find_sub_addr(child, s2) &&
 			    (SAS_ADDR(sub_addr) != SAS_ADDR(s2))) {
@@ -1261,7 +1266,7 @@ static int sas_check_ex_subtractive_boundary(struct domain_device *dev)
 			else if (SAS_ADDR(sub_sas_addr) !=
 				 SAS_ADDR(phy->attached_sas_addr)) {
 
-				pr_notice("ex %016llx phy 0x%x diverges(%016llx) on subtractive boundary(%016llx). Disabled\n",
+				pr_notice("ex %016llx phy%02d diverges(%016llx) on subtractive boundary(%016llx). Disabled\n",
 					  SAS_ADDR(dev->sas_addr), i,
 					  SAS_ADDR(phy->attached_sas_addr),
 					  SAS_ADDR(sub_sas_addr));
@@ -1282,7 +1287,7 @@ static void sas_print_parent_topology_bug(struct domain_device *child,
 	};
 	struct domain_device *parent = child->parent;
 
-	pr_notice("%s ex %016llx phy 0x%x <--> %s ex %016llx phy 0x%x has %c:%c routing link!\n",
+	pr_notice("%s ex %016llx phy%02d <--> %s ex %016llx phy%02d has %c:%c routing link!\n",
 		  ex_type[parent->dev_type],
 		  SAS_ADDR(parent->sas_addr),
 		  parent_phy->phy_id,
@@ -1304,7 +1309,7 @@ static int sas_check_eeds(struct domain_device *child,
 
 	if (SAS_ADDR(parent->port->disc.fanout_sas_addr) != 0) {
 		res = -ENODEV;
-		pr_warn("edge ex %016llx phy S:0x%x <--> edge ex %016llx phy S:0x%x, while there is a fanout ex %016llx\n",
+		pr_warn("edge ex %016llx phy S:%02d <--> edge ex %016llx phy S:%02d, while there is a fanout ex %016llx\n",
 			SAS_ADDR(parent->sas_addr),
 			parent_phy->phy_id,
 			SAS_ADDR(child->sas_addr),
@@ -1327,7 +1332,7 @@ static int sas_check_eeds(struct domain_device *child,
 		;
 	else {
 		res = -ENODEV;
-		pr_warn("edge ex %016llx phy 0x%x <--> edge ex %016llx phy 0x%x link forms a third EEDS!\n",
+		pr_warn("edge ex %016llx phy%02d <--> edge ex %016llx phy%02d link forms a third EEDS!\n",
 			SAS_ADDR(parent->sas_addr),
 			parent_phy->phy_id,
 			SAS_ADDR(child->sas_addr),
@@ -1445,11 +1450,11 @@ static int sas_configure_present(struct domain_device *dev, int phy_id,
 			goto out;
 		res = rri_resp[2];
 		if (res == SMP_RESP_NO_INDEX) {
-			pr_warn("overflow of indexes: dev %016llx phy 0x%x index 0x%x\n",
+			pr_warn("overflow of indexes: dev %016llx phy%02d index 0x%x\n",
 				SAS_ADDR(dev->sas_addr), phy_id, i);
 			goto out;
 		} else if (res != SMP_RESP_FUNC_ACC) {
-			pr_notice("%s: dev %016llx phy 0x%x index 0x%x result 0x%x\n",
+			pr_notice("%s: dev %016llx phy%02d index 0x%x result 0x%x\n",
 				  __func__, SAS_ADDR(dev->sas_addr), phy_id,
 				  i, res);
 			goto out;
@@ -1515,7 +1520,7 @@ static int sas_configure_set(struct domain_device *dev, int phy_id,
 		goto out;
 	res = cri_resp[2];
 	if (res == SMP_RESP_NO_INDEX) {
-		pr_warn("overflow of indexes: dev %016llx phy 0x%x index 0x%x\n",
+		pr_warn("overflow of indexes: dev %016llx phy%02d index 0x%x\n",
 			SAS_ADDR(dev->sas_addr), phy_id, index);
 	}
 out:
@@ -1760,10 +1765,11 @@ static int sas_get_phy_attached_dev(struct domain_device *dev, int phy_id,
 
 	res = sas_get_phy_discover(dev, phy_id, disc_resp);
 	if (res == 0) {
-		memcpy(sas_addr, disc_resp->disc.attached_sas_addr, 8);
+		memcpy(sas_addr, disc_resp->disc.attached_sas_addr,
+		       SAS_ADDR_SIZE);
 		*type = to_dev_type(dr);
 		if (*type == 0)
-			memset(sas_addr, 0, 8);
+			memset(sas_addr, 0, SAS_ADDR_SIZE);
 	}
 	kfree(disc_resp);
 	return res;
@@ -1870,10 +1876,12 @@ static int sas_find_bcast_dev(struct domain_device *dev,
 		if (phy_id != -1) {
 			*src_dev = dev;
 			ex->ex_change_count = ex_change_count;
-			pr_info("Expander phy change count has changed\n");
+			pr_info("ex %016llx phy%02d change count has changed\n",
+				SAS_ADDR(dev->sas_addr), phy_id);
 			return res;
 		} else
-			pr_info("Expander phys DID NOT change\n");
+			pr_info("ex %016llx phys DID NOT change\n",
+				SAS_ADDR(dev->sas_addr));
 	}
 	list_for_each_entry(ch, &ex->children, siblings) {
 		if (ch->dev_type == SAS_EDGE_EXPANDER_DEVICE || ch->dev_type == SAS_FANOUT_EXPANDER_DEVICE) {
@@ -1983,7 +1991,7 @@ static int sas_discover_new(struct domain_device *dev, int phy_id)
 	struct domain_device *child;
 	int res;
 
-	pr_debug("ex %016llx phy%d new device attached\n",
+	pr_debug("ex %016llx phy%02d new device attached\n",
 		 SAS_ADDR(dev->sas_addr), phy_id);
 	res = sas_ex_phy_discover(dev, phy_id);
 	if (res)
@@ -2022,15 +2030,23 @@ static bool dev_type_flutter(enum sas_device_type new, enum sas_device_type old)
 	return false;
 }
 
-static int sas_rediscover_dev(struct domain_device *dev, int phy_id, bool last)
+static int sas_rediscover_dev(struct domain_device *dev, int phy_id,
+			      bool last, int sibling)
 {
 	struct expander_device *ex = &dev->ex_dev;
 	struct ex_phy *phy = &ex->ex_phy[phy_id];
 	enum sas_device_type type = SAS_PHY_UNUSED;
-	u8 sas_addr[8];
+	u8 sas_addr[SAS_ADDR_SIZE];
+	char msg[80] = "";
 	int res;
 
-	memset(sas_addr, 0, 8);
+	if (!last)
+		sprintf(msg, ", part of a wide port with phy%02d", sibling);
+
+	pr_debug("ex %016llx rediscovering phy%02d%s\n",
+		 SAS_ADDR(dev->sas_addr), phy_id, msg);
+
+	memset(sas_addr, 0, SAS_ADDR_SIZE);
 	res = sas_get_phy_attached_dev(dev, phy_id, sas_addr, &type);
 	switch (res) {
 	case SMP_RESP_NO_PHY:
@@ -2052,6 +2068,11 @@ static int sas_rediscover_dev(struct domain_device *dev, int phy_id, bool last)
 	if ((SAS_ADDR(sas_addr) == 0) || (res == -ECOMM)) {
 		phy->phy_state = PHY_EMPTY;
 		sas_unregister_devs_sas_addr(dev, phy_id, last);
+		/*
+		 * Even though the PHY is empty, for convenience we discover
+		 * the PHY to update the PHY info, like negotiated linkrate.
+		 */
+		sas_ex_phy_discover(dev, phy_id);
 		return res;
 	} else if (SAS_ADDR(sas_addr) == SAS_ADDR(phy->attached_sas_addr) &&
 		   dev_type_flutter(type, phy->attached_dev_type)) {
@@ -2062,13 +2083,13 @@ static int sas_rediscover_dev(struct domain_device *dev, int phy_id, bool last)
 
 		if (ata_dev && phy->attached_dev_type == SAS_SATA_PENDING)
 			action = ", needs recovery";
-		pr_debug("ex %016llx phy 0x%x broadcast flutter%s\n",
+		pr_debug("ex %016llx phy%02d broadcast flutter%s\n",
 			 SAS_ADDR(dev->sas_addr), phy_id, action);
 		return res;
 	}
 
 	/* we always have to delete the old device when we went here */
-	pr_info("ex %016llx phy 0x%x replace %016llx\n",
+	pr_info("ex %016llx phy%02d replace %016llx\n",
 		SAS_ADDR(dev->sas_addr), phy_id,
 		SAS_ADDR(phy->attached_sas_addr));
 	sas_unregister_devs_sas_addr(dev, phy_id, last);
@@ -2098,7 +2119,7 @@ static int sas_rediscover(struct domain_device *dev, const int phy_id)
 	int i;
 	bool last = true;	/* is this the last phy of the port */
 
-	pr_debug("ex %016llx phy%d originated BROADCAST(CHANGE)\n",
+	pr_debug("ex %016llx phy%02d originated BROADCAST(CHANGE)\n",
 		 SAS_ADDR(dev->sas_addr), phy_id);
 
 	if (SAS_ADDR(changed_phy->attached_sas_addr) != 0) {
@@ -2109,13 +2130,11 @@ static int sas_rediscover(struct domain_device *dev, const int phy_id)
 				continue;
 			if (SAS_ADDR(phy->attached_sas_addr) ==
 			    SAS_ADDR(changed_phy->attached_sas_addr)) {
-				pr_debug("phy%d part of wide port with phy%d\n",
-					 phy_id, i);
 				last = false;
 				break;
 			}
 		}
-		res = sas_rediscover_dev(dev, phy_id, last);
+		res = sas_rediscover_dev(dev, phy_id, last, i);
 	} else
 		res = sas_discover_new(dev, phy_id);
 	return res;
diff --git a/drivers/scsi/libsas/sas_init.c b/drivers/scsi/libsas/sas_init.c
index 221340ee8651..28a460c36c0b 100644
--- a/drivers/scsi/libsas/sas_init.c
+++ b/drivers/scsi/libsas/sas_init.c
@@ -87,25 +87,27 @@ EXPORT_SYMBOL_GPL(sas_free_task);
 /*------------ SAS addr hash -----------*/
 void sas_hash_addr(u8 *hashed, const u8 *sas_addr)
 {
-        const u32 poly = 0x00DB2777;
-        u32     r = 0;
-        int     i;
-
-        for (i = 0; i < 8; i++) {
-                int b;
-                for (b = 7; b >= 0; b--) {
-                        r <<= 1;
-                        if ((1 << b) & sas_addr[i]) {
-                                if (!(r & 0x01000000))
-                                        r ^= poly;
-                        } else if (r & 0x01000000)
-                                r ^= poly;
-                }
-        }
-
-        hashed[0] = (r >> 16) & 0xFF;
-        hashed[1] = (r >> 8) & 0xFF ;
-        hashed[2] = r & 0xFF;
+	const u32 poly = 0x00DB2777;
+	u32 r = 0;
+	int i;
+
+	for (i = 0; i < SAS_ADDR_SIZE; i++) {
+		int b;
+
+		for (b = (SAS_ADDR_SIZE - 1); b >= 0; b--) {
+			r <<= 1;
+			if ((1 << b) & sas_addr[i]) {
+				if (!(r & 0x01000000))
+					r ^= poly;
+			} else if (r & 0x01000000) {
+				r ^= poly;
+			}
+		}
+	}
+
+	hashed[0] = (r >> 16) & 0xFF;
+	hashed[1] = (r >> 8) & 0xFF;
+	hashed[2] = r & 0xFF;
 }
 
 int sas_register_ha(struct sas_ha_struct *sas_ha)
@@ -623,7 +625,7 @@ struct asd_sas_event *sas_alloc_event(struct asd_sas_phy *phy)
 	if (atomic_read(&phy->event_nr) > phy->ha->event_thres) {
 		if (i->dft->lldd_control_phy) {
 			if (cmpxchg(&phy->in_shutdown, 0, 1) == 0) {
-				pr_notice("The phy%02d bursting events, shut it down.\n",
+				pr_notice("The phy%d bursting events, shut it down.\n",
 					  phy->id);
 				sas_notify_phy_event(phy, PHYE_SHUTDOWN);
 			}
diff --git a/drivers/scsi/libsas/sas_phy.c b/drivers/scsi/libsas/sas_phy.c
index 0374243c85d0..e030e1452136 100644
--- a/drivers/scsi/libsas/sas_phy.c
+++ b/drivers/scsi/libsas/sas_phy.c
@@ -122,11 +122,10 @@ static void sas_phye_shutdown(struct work_struct *work)
 		phy->enabled = 0;
 		ret = i->dft->lldd_control_phy(phy, PHY_FUNC_DISABLE, NULL);
 		if (ret)
-			pr_notice("lldd disable phy%02d returned %d\n",
-				  phy->id, ret);
+			pr_notice("lldd disable phy%d returned %d\n", phy->id,
+				  ret);
 	} else
-		pr_notice("phy%02d is not enabled, cannot shutdown\n",
-			  phy->id);
+		pr_notice("phy%d is not enabled, cannot shutdown\n", phy->id);
 }
 
 /* ---------- Phy class registration ---------- */
diff --git a/drivers/scsi/libsas/sas_port.c b/drivers/scsi/libsas/sas_port.c
index 03fe479359b6..38a10478605c 100644
--- a/drivers/scsi/libsas/sas_port.c
+++ b/drivers/scsi/libsas/sas_port.c
@@ -95,6 +95,7 @@ static void sas_form_port(struct asd_sas_phy *phy)
 	int i;
 	struct sas_ha_struct *sas_ha = phy->ha;
 	struct asd_sas_port *port = phy->port;
+	struct domain_device *port_dev;
 	struct sas_internal *si =
 		to_sas_internal(sas_ha->core.shost->transportt);
 	unsigned long flags;
@@ -153,8 +154,9 @@ static void sas_form_port(struct asd_sas_phy *phy)
 	}
 
 	/* add the phy to the port */
+	port_dev = port->port_dev;
 	list_add_tail(&phy->port_phy_el, &port->phy_list);
-	sas_phy_set_target(phy, port->port_dev);
+	sas_phy_set_target(phy, port_dev);
 	phy->port = port;
 	port->num_phys++;
 	port->phy_mask |= (1U << phy->id);
@@ -184,14 +186,21 @@ static void sas_form_port(struct asd_sas_phy *phy)
 		 port->phy_mask,
 		 SAS_ADDR(port->attached_sas_addr));
 
-	if (port->port_dev)
-		port->port_dev->pathways = port->num_phys;
+	if (port_dev)
+		port_dev->pathways = port->num_phys;
 
 	/* Tell the LLDD about this port formation. */
 	if (si->dft->lldd_port_formed)
 		si->dft->lldd_port_formed(phy);
 
 	sas_discover_event(phy->port, DISCE_DISCOVER_DOMAIN);
+	/* Only insert a revalidate event after initial discovery */
+	if (port_dev && sas_dev_type_is_expander(port_dev->dev_type)) {
+		struct expander_device *ex_dev = &port_dev->ex_dev;
+
+		ex_dev->ex_change_count = -1;
+		sas_discover_event(port, DISCE_REVALIDATE_DOMAIN);
+	}
 	flush_workqueue(sas_ha->disco_q);
 }
 
@@ -254,6 +263,15 @@ void sas_deform_port(struct asd_sas_phy *phy, int gone)
 	spin_unlock(&port->phy_list_lock);
 	spin_unlock_irqrestore(&sas_ha->phy_port_lock, flags);
 
+	/* Only insert revalidate event if the port still has members */
+	if (port->port && dev && sas_dev_type_is_expander(dev->dev_type)) {
+		struct expander_device *ex_dev = &dev->ex_dev;
+
+		ex_dev->ex_change_count = -1;
+		sas_discover_event(port, DISCE_REVALIDATE_DOMAIN);
+	}
+	flush_workqueue(sas_ha->disco_q);
+
 	return;
 }
 
diff --git a/drivers/scsi/lpfc/lpfc.h b/drivers/scsi/lpfc/lpfc.h
index 41d849f283f6..aafcffaa25f7 100644
--- a/drivers/scsi/lpfc/lpfc.h
+++ b/drivers/scsi/lpfc/lpfc.h
@@ -942,6 +942,7 @@ struct lpfc_hba {
 	int brd_no;			/* FC board number */
 	char SerialNumber[32];		/* adapter Serial Number */
 	char OptionROMVersion[32];	/* adapter BIOS / Fcode version */
+	char BIOSVersion[16];		/* Boot BIOS version */
 	char ModelDesc[256];		/* Model Description */
 	char ModelName[80];		/* Model Name */
 	char ProgramType[256];		/* Program Type */
diff --git a/drivers/scsi/lpfc/lpfc_attr.c b/drivers/scsi/lpfc/lpfc_attr.c
index ce3e541434dc..e9adb3f1961d 100644
--- a/drivers/scsi/lpfc/lpfc_attr.c
+++ b/drivers/scsi/lpfc/lpfc_attr.c
@@ -71,6 +71,23 @@
 #define LPFC_REG_WRITE_KEY_SIZE	4
 #define LPFC_REG_WRITE_KEY	"EMLX"
 
+const char *const trunk_errmsg[] = {	/* map errcode */
+	"",	/* There is no such error code at index 0*/
+	"link negotiated speed does not match existing"
+		" trunk - link was \"low\" speed",
+	"link negotiated speed does not match"
+		" existing trunk - link was \"middle\" speed",
+	"link negotiated speed does not match existing"
+		" trunk - link was \"high\" speed",
+	"Attached to non-trunking port - F_Port",
+	"Attached to non-trunking port - N_Port",
+	"FLOGI response timeout",
+	"non-FLOGI frame received",
+	"Invalid FLOGI response",
+	"Trunking initialization protocol",
+	"Trunk peer device mismatch",
+};
+
 /**
  * lpfc_jedec_to_ascii - Hex to ascii convertor according to JEDEC rules
  * @incr: integer to convert.
@@ -114,7 +131,7 @@ static ssize_t
 lpfc_drvr_version_show(struct device *dev, struct device_attribute *attr,
 		       char *buf)
 {
-	return snprintf(buf, PAGE_SIZE, LPFC_MODULE_DESC "\n");
+	return scnprintf(buf, PAGE_SIZE, LPFC_MODULE_DESC "\n");
 }
 
 /**
@@ -134,9 +151,9 @@ lpfc_enable_fip_show(struct device *dev, struct device_attribute *attr,
 	struct lpfc_hba   *phba = vport->phba;
 
 	if (phba->hba_flag & HBA_FIP_SUPPORT)
-		return snprintf(buf, PAGE_SIZE, "1\n");
+		return scnprintf(buf, PAGE_SIZE, "1\n");
 	else
-		return snprintf(buf, PAGE_SIZE, "0\n");
+		return scnprintf(buf, PAGE_SIZE, "0\n");
 }
 
 static ssize_t
@@ -564,14 +581,15 @@ lpfc_bg_info_show(struct device *dev, struct device_attribute *attr,
 	struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;
 	struct lpfc_hba   *phba = vport->phba;
 
-	if (phba->cfg_enable_bg)
+	if (phba->cfg_enable_bg) {
 		if (phba->sli3_options & LPFC_SLI3_BG_ENABLED)
-			return snprintf(buf, PAGE_SIZE, "BlockGuard Enabled\n");
+			return scnprintf(buf, PAGE_SIZE,
+					"BlockGuard Enabled\n");
 		else
-			return snprintf(buf, PAGE_SIZE,
+			return scnprintf(buf, PAGE_SIZE,
 					"BlockGuard Not Supported\n");
-	else
-			return snprintf(buf, PAGE_SIZE,
+	} else
+		return scnprintf(buf, PAGE_SIZE,
 					"BlockGuard Disabled\n");
 }
 
@@ -583,7 +601,7 @@ lpfc_bg_guard_err_show(struct device *dev, struct device_attribute *attr,
 	struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;
 	struct lpfc_hba   *phba = vport->phba;
 
-	return snprintf(buf, PAGE_SIZE, "%llu\n",
+	return scnprintf(buf, PAGE_SIZE, "%llu\n",
 			(unsigned long long)phba->bg_guard_err_cnt);
 }
 
@@ -595,7 +613,7 @@ lpfc_bg_apptag_err_show(struct device *dev, struct device_attribute *attr,
 	struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;
 	struct lpfc_hba   *phba = vport->phba;
 
-	return snprintf(buf, PAGE_SIZE, "%llu\n",
+	return scnprintf(buf, PAGE_SIZE, "%llu\n",
 			(unsigned long long)phba->bg_apptag_err_cnt);
 }
 
@@ -607,7 +625,7 @@ lpfc_bg_reftag_err_show(struct device *dev, struct device_attribute *attr,
 	struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;
 	struct lpfc_hba   *phba = vport->phba;
 
-	return snprintf(buf, PAGE_SIZE, "%llu\n",
+	return scnprintf(buf, PAGE_SIZE, "%llu\n",
 			(unsigned long long)phba->bg_reftag_err_cnt);
 }
 
@@ -625,7 +643,7 @@ lpfc_info_show(struct device *dev, struct device_attribute *attr,
 {
 	struct Scsi_Host *host = class_to_shost(dev);
 
-	return snprintf(buf, PAGE_SIZE, "%s\n",lpfc_info(host));
+	return scnprintf(buf, PAGE_SIZE, "%s\n", lpfc_info(host));
 }
 
 /**
@@ -644,7 +662,7 @@ lpfc_serialnum_show(struct device *dev, struct device_attribute *attr,
 	struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;
 	struct lpfc_hba   *phba = vport->phba;
 
-	return snprintf(buf, PAGE_SIZE, "%s\n",phba->SerialNumber);
+	return scnprintf(buf, PAGE_SIZE, "%s\n", phba->SerialNumber);
 }
 
 /**
@@ -666,7 +684,7 @@ lpfc_temp_sensor_show(struct device *dev, struct device_attribute *attr,
 	struct Scsi_Host *shost = class_to_shost(dev);
 	struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;
 	struct lpfc_hba   *phba = vport->phba;
-	return snprintf(buf, PAGE_SIZE, "%d\n",phba->temp_sensor_support);
+	return scnprintf(buf, PAGE_SIZE, "%d\n", phba->temp_sensor_support);
 }
 
 /**
@@ -685,7 +703,7 @@ lpfc_modeldesc_show(struct device *dev, struct device_attribute *attr,
 	struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;
 	struct lpfc_hba   *phba = vport->phba;
 
-	return snprintf(buf, PAGE_SIZE, "%s\n",phba->ModelDesc);
+	return scnprintf(buf, PAGE_SIZE, "%s\n", phba->ModelDesc);
 }
 
 /**
@@ -704,7 +722,7 @@ lpfc_modelname_show(struct device *dev, struct device_attribute *attr,
 	struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;
 	struct lpfc_hba   *phba = vport->phba;
 
-	return snprintf(buf, PAGE_SIZE, "%s\n",phba->ModelName);
+	return scnprintf(buf, PAGE_SIZE, "%s\n", phba->ModelName);
 }
 
 /**
@@ -723,7 +741,7 @@ lpfc_programtype_show(struct device *dev, struct device_attribute *attr,
 	struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;
 	struct lpfc_hba   *phba = vport->phba;
 
-	return snprintf(buf, PAGE_SIZE, "%s\n",phba->ProgramType);
+	return scnprintf(buf, PAGE_SIZE, "%s\n", phba->ProgramType);
 }
 
 /**
@@ -741,7 +759,7 @@ lpfc_mlomgmt_show(struct device *dev, struct device_attribute *attr, char *buf)
 	struct lpfc_vport *vport = (struct lpfc_vport *)shost->hostdata;
 	struct lpfc_hba   *phba = vport->phba;
 
-	return snprintf(buf, PAGE_SIZE, "%d\n",
+	return scnprintf(buf, PAGE_SIZE, "%d\n",
 		(phba->sli.sli_flag & LPFC_MENLO_MAINT));
 }
 
@@ -761,7 +779,7 @@ lpfc_vportnum_show(struct device *dev, struct device_attribute *attr,
 	struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;
 	struct lpfc_hba   *phba = vport->phba;
 
-	return snprintf(buf, PAGE_SIZE, "%s\n",phba->Port);
+	return scnprintf(buf, PAGE_SIZE, "%s\n", phba->Port);
 }
 
 /**
@@ -789,10 +807,10 @@ lpfc_fwrev_show(struct device *dev, struct device_attribute *attr,
 	sli_family = phba->sli4_hba.pc_sli4_params.sli_family;
 
 	if (phba->sli_rev < LPFC_SLI_REV4)
-		len = snprintf(buf, PAGE_SIZE, "%s, sli-%d\n",
+		len = scnprintf(buf, PAGE_SIZE, "%s, sli-%d\n",
 			       fwrev, phba->sli_rev);
 	else
-		len = snprintf(buf, PAGE_SIZE, "%s, sli-%d:%d:%x\n",
+		len = scnprintf(buf, PAGE_SIZE, "%s, sli-%d:%d:%x\n",
 			       fwrev, phba->sli_rev, if_type, sli_family);
 
 	return len;
@@ -816,7 +834,7 @@ lpfc_hdw_show(struct device *dev, struct device_attribute *attr, char *buf)
 	lpfc_vpd_t *vp = &phba->vpd;
 
 	lpfc_jedec_to_ascii(vp->rev.biuRev, hdw);
-	return snprintf(buf, PAGE_SIZE, "%s\n", hdw);
+	return scnprintf(buf, PAGE_SIZE, "%s\n", hdw);
 }
 
 /**
@@ -837,10 +855,11 @@ lpfc_option_rom_version_show(struct device *dev, struct device_attribute *attr,
 	char fwrev[FW_REV_STR_SIZE];
 
 	if (phba->sli_rev < LPFC_SLI_REV4)
-		return snprintf(buf, PAGE_SIZE, "%s\n", phba->OptionROMVersion);
+		return scnprintf(buf, PAGE_SIZE, "%s\n",
+				phba->OptionROMVersion);
 
 	lpfc_decode_firmware_rev(phba, fwrev, 1);
-	return snprintf(buf, PAGE_SIZE, "%s\n", fwrev);
+	return scnprintf(buf, PAGE_SIZE, "%s\n", fwrev);
 }
 
 /**
@@ -871,20 +890,20 @@ lpfc_link_state_show(struct device *dev, struct device_attribute *attr,
 	case LPFC_LINK_DOWN:
 	case LPFC_HBA_ERROR:
 		if (phba->hba_flag & LINK_DISABLED)
-			len += snprintf(buf + len, PAGE_SIZE-len,
+			len += scnprintf(buf + len, PAGE_SIZE-len,
 				"Link Down - User disabled\n");
 		else
-			len += snprintf(buf + len, PAGE_SIZE-len,
+			len += scnprintf(buf + len, PAGE_SIZE-len,
 				"Link Down\n");
 		break;
 	case LPFC_LINK_UP:
 	case LPFC_CLEAR_LA:
 	case LPFC_HBA_READY:
-		len += snprintf(buf + len, PAGE_SIZE-len, "Link Up - ");
+		len += scnprintf(buf + len, PAGE_SIZE-len, "Link Up - ");
 
 		switch (vport->port_state) {
 		case LPFC_LOCAL_CFG_LINK:
-			len += snprintf(buf + len, PAGE_SIZE-len,
+			len += scnprintf(buf + len, PAGE_SIZE-len,
 					"Configuring Link\n");
 			break;
 		case LPFC_FDISC:
@@ -894,38 +913,40 @@ lpfc_link_state_show(struct device *dev, struct device_attribute *attr,
 		case LPFC_NS_QRY:
 		case LPFC_BUILD_DISC_LIST:
 		case LPFC_DISC_AUTH:
-			len += snprintf(buf + len, PAGE_SIZE - len,
+			len += scnprintf(buf + len, PAGE_SIZE - len,
 					"Discovery\n");
 			break;
 		case LPFC_VPORT_READY:
-			len += snprintf(buf + len, PAGE_SIZE - len, "Ready\n");
+			len += scnprintf(buf + len, PAGE_SIZE - len,
+					"Ready\n");
 			break;
 
 		case LPFC_VPORT_FAILED:
-			len += snprintf(buf + len, PAGE_SIZE - len, "Failed\n");
+			len += scnprintf(buf + len, PAGE_SIZE - len,
+					"Failed\n");
 			break;
 
 		case LPFC_VPORT_UNKNOWN:
-			len += snprintf(buf + len, PAGE_SIZE - len,
+			len += scnprintf(buf + len, PAGE_SIZE - len,
 					"Unknown\n");
 			break;
 		}
 		if (phba->sli.sli_flag & LPFC_MENLO_MAINT)
-			len += snprintf(buf + len, PAGE_SIZE-len,
+			len += scnprintf(buf + len, PAGE_SIZE-len,
 					"   Menlo Maint Mode\n");
 		else if (phba->fc_topology == LPFC_TOPOLOGY_LOOP) {
 			if (vport->fc_flag & FC_PUBLIC_LOOP)
-				len += snprintf(buf + len, PAGE_SIZE-len,
+				len += scnprintf(buf + len, PAGE_SIZE-len,
 						"   Public Loop\n");
 			else
-				len += snprintf(buf + len, PAGE_SIZE-len,
+				len += scnprintf(buf + len, PAGE_SIZE-len,
 						"   Private Loop\n");
 		} else {
 			if (vport->fc_flag & FC_FABRIC)
-				len += snprintf(buf + len, PAGE_SIZE-len,
+				len += scnprintf(buf + len, PAGE_SIZE-len,
 						"   Fabric\n");
 			else
-				len += snprintf(buf + len, PAGE_SIZE-len,
+				len += scnprintf(buf + len, PAGE_SIZE-len,
 						"   Point-2-Point\n");
 		}
 	}
@@ -937,28 +958,28 @@ lpfc_link_state_show(struct device *dev, struct device_attribute *attr,
 		struct lpfc_trunk_link link = phba->trunk_link;
 
 		if (bf_get(lpfc_conf_trunk_port0, &phba->sli4_hba))
-			len += snprintf(buf + len, PAGE_SIZE - len,
+			len += scnprintf(buf + len, PAGE_SIZE - len,
 				"Trunk port 0: Link %s %s\n",
 				(link.link0.state == LPFC_LINK_UP) ?
 				 "Up" : "Down. ",
 				trunk_errmsg[link.link0.fault]);
 
 		if (bf_get(lpfc_conf_trunk_port1, &phba->sli4_hba))
-			len += snprintf(buf + len, PAGE_SIZE - len,
+			len += scnprintf(buf + len, PAGE_SIZE - len,
 				"Trunk port 1: Link %s %s\n",
 				(link.link1.state == LPFC_LINK_UP) ?
 				 "Up" : "Down. ",
 				trunk_errmsg[link.link1.fault]);
 
 		if (bf_get(lpfc_conf_trunk_port2, &phba->sli4_hba))
-			len += snprintf(buf + len, PAGE_SIZE - len,
+			len += scnprintf(buf + len, PAGE_SIZE - len,
 				"Trunk port 2: Link %s %s\n",
 				(link.link2.state == LPFC_LINK_UP) ?
 				 "Up" : "Down. ",
 				trunk_errmsg[link.link2.fault]);
 
 		if (bf_get(lpfc_conf_trunk_port3, &phba->sli4_hba))
-			len += snprintf(buf + len, PAGE_SIZE - len,
+			len += scnprintf(buf + len, PAGE_SIZE - len,
 				"Trunk port 3: Link %s %s\n",
 				(link.link3.state == LPFC_LINK_UP) ?
 				 "Up" : "Down. ",
@@ -986,15 +1007,15 @@ lpfc_sli4_protocol_show(struct device *dev, struct device_attribute *attr,
 	struct lpfc_hba *phba = vport->phba;
 
 	if (phba->sli_rev < LPFC_SLI_REV4)
-		return snprintf(buf, PAGE_SIZE, "fc\n");
+		return scnprintf(buf, PAGE_SIZE, "fc\n");
 
 	if (phba->sli4_hba.lnk_info.lnk_dv == LPFC_LNK_DAT_VAL) {
 		if (phba->sli4_hba.lnk_info.lnk_tp == LPFC_LNK_TYPE_GE)
-			return snprintf(buf, PAGE_SIZE, "fcoe\n");
+			return scnprintf(buf, PAGE_SIZE, "fcoe\n");
 		if (phba->sli4_hba.lnk_info.lnk_tp == LPFC_LNK_TYPE_FC)
-			return snprintf(buf, PAGE_SIZE, "fc\n");
+			return scnprintf(buf, PAGE_SIZE, "fc\n");
 	}
-	return snprintf(buf, PAGE_SIZE, "unknown\n");
+	return scnprintf(buf, PAGE_SIZE, "unknown\n");
 }
 
 /**
@@ -1014,7 +1035,7 @@ lpfc_oas_supported_show(struct device *dev, struct device_attribute *attr,
 	struct lpfc_vport *vport = (struct lpfc_vport *)shost->hostdata;
 	struct lpfc_hba *phba = vport->phba;
 
-	return snprintf(buf, PAGE_SIZE, "%d\n",
+	return scnprintf(buf, PAGE_SIZE, "%d\n",
 			phba->sli4_hba.pc_sli4_params.oas_supported);
 }
 
@@ -1072,7 +1093,7 @@ lpfc_num_discovered_ports_show(struct device *dev,
 	struct Scsi_Host  *shost = class_to_shost(dev);
 	struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;
 
-	return snprintf(buf, PAGE_SIZE, "%d\n",
+	return scnprintf(buf, PAGE_SIZE, "%d\n",
 			vport->fc_map_cnt + vport->fc_unmap_cnt);
 }
 
@@ -1204,6 +1225,20 @@ lpfc_do_offline(struct lpfc_hba *phba, uint32_t type)
 
 	psli = &phba->sli;
 
+	/*
+	 * If freeing the queues have already started, don't access them.
+	 * Otherwise set FREE_WAIT to indicate that queues are being used
+	 * to hold the freeing process until we finish.
+	 */
+	spin_lock_irq(&phba->hbalock);
+	if (!(psli->sli_flag & LPFC_QUEUE_FREE_INIT)) {
+		psli->sli_flag |= LPFC_QUEUE_FREE_WAIT;
+	} else {
+		spin_unlock_irq(&phba->hbalock);
+		goto skip_wait;
+	}
+	spin_unlock_irq(&phba->hbalock);
+
 	/* Wait a little for things to settle down, but not
 	 * long enough for dev loss timeout to expire.
 	 */
@@ -1225,6 +1260,11 @@ lpfc_do_offline(struct lpfc_hba *phba, uint32_t type)
 		}
 	}
 out:
+	spin_lock_irq(&phba->hbalock);
+	psli->sli_flag &= ~LPFC_QUEUE_FREE_WAIT;
+	spin_unlock_irq(&phba->hbalock);
+
+skip_wait:
 	init_completion(&online_compl);
 	rc = lpfc_workq_post_event(phba, &status, &online_compl, type);
 	if (rc == 0)
@@ -1258,7 +1298,7 @@ out:
  * -EBUSY,    port is not in offline state
  *      0,    successful
  */
-int
+static int
 lpfc_reset_pci_bus(struct lpfc_hba *phba)
 {
 	struct pci_dev *pdev = phba->pcidev;
@@ -1586,10 +1626,10 @@ lpfc_nport_evt_cnt_show(struct device *dev, struct device_attribute *attr,
 	struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;
 	struct lpfc_hba   *phba = vport->phba;
 
-	return snprintf(buf, PAGE_SIZE, "%d\n", phba->nport_event_cnt);
+	return scnprintf(buf, PAGE_SIZE, "%d\n", phba->nport_event_cnt);
 }
 
-int
+static int
 lpfc_set_trunking(struct lpfc_hba *phba, char *buff_out)
 {
 	LPFC_MBOXQ_t *mbox = NULL;
@@ -1675,7 +1715,7 @@ lpfc_board_mode_show(struct device *dev, struct device_attribute *attr,
 	else
 		state = "online";
 
-	return snprintf(buf, PAGE_SIZE, "%s\n", state);
+	return scnprintf(buf, PAGE_SIZE, "%s\n", state);
 }
 
 /**
@@ -1901,8 +1941,8 @@ lpfc_max_rpi_show(struct device *dev, struct device_attribute *attr,
 	uint32_t cnt;
 
 	if (lpfc_get_hba_info(phba, NULL, NULL, &cnt, NULL, NULL, NULL))
-		return snprintf(buf, PAGE_SIZE, "%d\n", cnt);
-	return snprintf(buf, PAGE_SIZE, "Unknown\n");
+		return scnprintf(buf, PAGE_SIZE, "%d\n", cnt);
+	return scnprintf(buf, PAGE_SIZE, "Unknown\n");
 }
 
 /**
@@ -1929,8 +1969,8 @@ lpfc_used_rpi_show(struct device *dev, struct device_attribute *attr,
 	uint32_t cnt, acnt;
 
 	if (lpfc_get_hba_info(phba, NULL, NULL, &cnt, &acnt, NULL, NULL))
-		return snprintf(buf, PAGE_SIZE, "%d\n", (cnt - acnt));
-	return snprintf(buf, PAGE_SIZE, "Unknown\n");
+		return scnprintf(buf, PAGE_SIZE, "%d\n", (cnt - acnt));
+	return scnprintf(buf, PAGE_SIZE, "Unknown\n");
 }
 
 /**
@@ -1957,8 +1997,8 @@ lpfc_max_xri_show(struct device *dev, struct device_attribute *attr,
 	uint32_t cnt;
 
 	if (lpfc_get_hba_info(phba, &cnt, NULL, NULL, NULL, NULL, NULL))
-		return snprintf(buf, PAGE_SIZE, "%d\n", cnt);
-	return snprintf(buf, PAGE_SIZE, "Unknown\n");
+		return scnprintf(buf, PAGE_SIZE, "%d\n", cnt);
+	return scnprintf(buf, PAGE_SIZE, "Unknown\n");
 }
 
 /**
@@ -1985,8 +2025,8 @@ lpfc_used_xri_show(struct device *dev, struct device_attribute *attr,
 	uint32_t cnt, acnt;
 
 	if (lpfc_get_hba_info(phba, &cnt, &acnt, NULL, NULL, NULL, NULL))
-		return snprintf(buf, PAGE_SIZE, "%d\n", (cnt - acnt));
-	return snprintf(buf, PAGE_SIZE, "Unknown\n");
+		return scnprintf(buf, PAGE_SIZE, "%d\n", (cnt - acnt));
+	return scnprintf(buf, PAGE_SIZE, "Unknown\n");
 }
 
 /**
@@ -2013,8 +2053,8 @@ lpfc_max_vpi_show(struct device *dev, struct device_attribute *attr,
 	uint32_t cnt;
 
 	if (lpfc_get_hba_info(phba, NULL, NULL, NULL, NULL, &cnt, NULL))
-		return snprintf(buf, PAGE_SIZE, "%d\n", cnt);
-	return snprintf(buf, PAGE_SIZE, "Unknown\n");
+		return scnprintf(buf, PAGE_SIZE, "%d\n", cnt);
+	return scnprintf(buf, PAGE_SIZE, "Unknown\n");
 }
 
 /**
@@ -2041,8 +2081,8 @@ lpfc_used_vpi_show(struct device *dev, struct device_attribute *attr,
 	uint32_t cnt, acnt;
 
 	if (lpfc_get_hba_info(phba, NULL, NULL, NULL, NULL, &cnt, &acnt))
-		return snprintf(buf, PAGE_SIZE, "%d\n", (cnt - acnt));
-	return snprintf(buf, PAGE_SIZE, "Unknown\n");
+		return scnprintf(buf, PAGE_SIZE, "%d\n", (cnt - acnt));
+	return scnprintf(buf, PAGE_SIZE, "Unknown\n");
 }
 
 /**
@@ -2067,10 +2107,10 @@ lpfc_npiv_info_show(struct device *dev, struct device_attribute *attr,
 	struct lpfc_hba   *phba = vport->phba;
 
 	if (!(phba->max_vpi))
-		return snprintf(buf, PAGE_SIZE, "NPIV Not Supported\n");
+		return scnprintf(buf, PAGE_SIZE, "NPIV Not Supported\n");
 	if (vport->port_type == LPFC_PHYSICAL_PORT)
-		return snprintf(buf, PAGE_SIZE, "NPIV Physical\n");
-	return snprintf(buf, PAGE_SIZE, "NPIV Virtual (VPI %d)\n", vport->vpi);
+		return scnprintf(buf, PAGE_SIZE, "NPIV Physical\n");
+	return scnprintf(buf, PAGE_SIZE, "NPIV Virtual (VPI %d)\n", vport->vpi);
 }
 
 /**
@@ -2092,7 +2132,7 @@ lpfc_poll_show(struct device *dev, struct device_attribute *attr,
 	struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;
 	struct lpfc_hba   *phba = vport->phba;
 
-	return snprintf(buf, PAGE_SIZE, "%#x\n", phba->cfg_poll);
+	return scnprintf(buf, PAGE_SIZE, "%#x\n", phba->cfg_poll);
 }
 
 /**
@@ -2196,7 +2236,7 @@ lpfc_fips_level_show(struct device *dev,  struct device_attribute *attr,
 	struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;
 	struct lpfc_hba   *phba = vport->phba;
 
-	return snprintf(buf, PAGE_SIZE, "%d\n", phba->fips_level);
+	return scnprintf(buf, PAGE_SIZE, "%d\n", phba->fips_level);
 }
 
 /**
@@ -2215,7 +2255,7 @@ lpfc_fips_rev_show(struct device *dev,  struct device_attribute *attr,
 	struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;
 	struct lpfc_hba   *phba = vport->phba;
 
-	return snprintf(buf, PAGE_SIZE, "%d\n", phba->fips_spec_rev);
+	return scnprintf(buf, PAGE_SIZE, "%d\n", phba->fips_spec_rev);
 }
 
 /**
@@ -2234,7 +2274,7 @@ lpfc_dss_show(struct device *dev, struct device_attribute *attr,
 	struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;
 	struct lpfc_hba   *phba = vport->phba;
 
-	return snprintf(buf, PAGE_SIZE, "%s - %sOperational\n",
+	return scnprintf(buf, PAGE_SIZE, "%s - %sOperational\n",
 			(phba->cfg_enable_dss) ? "Enabled" : "Disabled",
 			(phba->sli3_options & LPFC_SLI3_DSS_ENABLED) ?
 				"" : "Not ");
@@ -2263,7 +2303,7 @@ lpfc_sriov_hw_max_virtfn_show(struct device *dev,
 	uint16_t max_nr_virtfn;
 
 	max_nr_virtfn = lpfc_sli_sriov_nr_virtfn_get(phba);
-	return snprintf(buf, PAGE_SIZE, "%d\n", max_nr_virtfn);
+	return scnprintf(buf, PAGE_SIZE, "%d\n", max_nr_virtfn);
 }
 
 static inline bool lpfc_rangecheck(uint val, uint min, uint max)
@@ -2323,7 +2363,7 @@ lpfc_##attr##_show(struct device *dev, struct device_attribute *attr, \
 	struct Scsi_Host  *shost = class_to_shost(dev);\
 	struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;\
 	struct lpfc_hba   *phba = vport->phba;\
-	return snprintf(buf, PAGE_SIZE, "%d\n",\
+	return scnprintf(buf, PAGE_SIZE, "%d\n",\
 			phba->cfg_##attr);\
 }
 
@@ -2351,7 +2391,7 @@ lpfc_##attr##_show(struct device *dev, struct device_attribute *attr, \
 	struct lpfc_hba   *phba = vport->phba;\
 	uint val = 0;\
 	val = phba->cfg_##attr;\
-	return snprintf(buf, PAGE_SIZE, "%#x\n",\
+	return scnprintf(buf, PAGE_SIZE, "%#x\n",\
 			phba->cfg_##attr);\
 }
 
@@ -2487,7 +2527,7 @@ lpfc_##attr##_show(struct device *dev, struct device_attribute *attr, \
 { \
 	struct Scsi_Host  *shost = class_to_shost(dev);\
 	struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;\
-	return snprintf(buf, PAGE_SIZE, "%d\n", vport->cfg_##attr);\
+	return scnprintf(buf, PAGE_SIZE, "%d\n", vport->cfg_##attr);\
 }
 
 /**
@@ -2512,7 +2552,7 @@ lpfc_##attr##_show(struct device *dev, struct device_attribute *attr, \
 { \
 	struct Scsi_Host  *shost = class_to_shost(dev);\
 	struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;\
-	return snprintf(buf, PAGE_SIZE, "%#x\n", vport->cfg_##attr);\
+	return scnprintf(buf, PAGE_SIZE, "%#x\n", vport->cfg_##attr);\
 }
 
 /**
@@ -2784,7 +2824,7 @@ lpfc_soft_wwpn_show(struct device *dev, struct device_attribute *attr,
 	struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;
 	struct lpfc_hba   *phba = vport->phba;
 
-	return snprintf(buf, PAGE_SIZE, "0x%llx\n",
+	return scnprintf(buf, PAGE_SIZE, "0x%llx\n",
 			(unsigned long long)phba->cfg_soft_wwpn);
 }
 
@@ -2881,7 +2921,7 @@ lpfc_soft_wwnn_show(struct device *dev, struct device_attribute *attr,
 {
 	struct Scsi_Host *shost = class_to_shost(dev);
 	struct lpfc_hba *phba = ((struct lpfc_vport *)shost->hostdata)->phba;
-	return snprintf(buf, PAGE_SIZE, "0x%llx\n",
+	return scnprintf(buf, PAGE_SIZE, "0x%llx\n",
 			(unsigned long long)phba->cfg_soft_wwnn);
 }
 
@@ -2947,7 +2987,7 @@ lpfc_oas_tgt_show(struct device *dev, struct device_attribute *attr,
 	struct Scsi_Host *shost = class_to_shost(dev);
 	struct lpfc_hba *phba = ((struct lpfc_vport *)shost->hostdata)->phba;
 
-	return snprintf(buf, PAGE_SIZE, "0x%llx\n",
+	return scnprintf(buf, PAGE_SIZE, "0x%llx\n",
 			wwn_to_u64(phba->cfg_oas_tgt_wwpn));
 }
 
@@ -3015,7 +3055,7 @@ lpfc_oas_priority_show(struct device *dev, struct device_attribute *attr,
 	struct Scsi_Host *shost = class_to_shost(dev);
 	struct lpfc_hba *phba = ((struct lpfc_vport *)shost->hostdata)->phba;
 
-	return snprintf(buf, PAGE_SIZE, "%d\n", phba->cfg_oas_priority);
+	return scnprintf(buf, PAGE_SIZE, "%d\n", phba->cfg_oas_priority);
 }
 
 /**
@@ -3078,7 +3118,7 @@ lpfc_oas_vpt_show(struct device *dev, struct device_attribute *attr,
 	struct Scsi_Host *shost = class_to_shost(dev);
 	struct lpfc_hba *phba = ((struct lpfc_vport *)shost->hostdata)->phba;
 
-	return snprintf(buf, PAGE_SIZE, "0x%llx\n",
+	return scnprintf(buf, PAGE_SIZE, "0x%llx\n",
 			wwn_to_u64(phba->cfg_oas_vpt_wwpn));
 }
 
@@ -3149,7 +3189,7 @@ lpfc_oas_lun_state_show(struct device *dev, struct device_attribute *attr,
 	struct Scsi_Host *shost = class_to_shost(dev);
 	struct lpfc_hba *phba = ((struct lpfc_vport *)shost->hostdata)->phba;
 
-	return snprintf(buf, PAGE_SIZE, "%d\n", phba->cfg_oas_lun_state);
+	return scnprintf(buf, PAGE_SIZE, "%d\n", phba->cfg_oas_lun_state);
 }
 
 /**
@@ -3213,7 +3253,7 @@ lpfc_oas_lun_status_show(struct device *dev, struct device_attribute *attr,
 	if (!(phba->cfg_oas_flags & OAS_LUN_VALID))
 		return -EFAULT;
 
-	return snprintf(buf, PAGE_SIZE, "%d\n", phba->cfg_oas_lun_status);
+	return scnprintf(buf, PAGE_SIZE, "%d\n", phba->cfg_oas_lun_status);
 }
 static DEVICE_ATTR(lpfc_xlane_lun_status, S_IRUGO,
 		   lpfc_oas_lun_status_show, NULL);
@@ -3365,7 +3405,7 @@ lpfc_oas_lun_show(struct device *dev, struct device_attribute *attr,
 	if (oas_lun != NOT_OAS_ENABLED_LUN)
 		phba->cfg_oas_flags |= OAS_LUN_VALID;
 
-	len += snprintf(buf + len, PAGE_SIZE-len, "0x%llx", oas_lun);
+	len += scnprintf(buf + len, PAGE_SIZE-len, "0x%llx", oas_lun);
 
 	return len;
 }
@@ -3499,7 +3539,7 @@ lpfc_iocb_hw_show(struct device *dev, struct device_attribute *attr, char *buf)
 	struct Scsi_Host  *shost = class_to_shost(dev);
 	struct lpfc_hba   *phba = ((struct lpfc_vport *) shost->hostdata)->phba;
 
-	return snprintf(buf, PAGE_SIZE, "%d\n", phba->iocb_max);
+	return scnprintf(buf, PAGE_SIZE, "%d\n", phba->iocb_max);
 }
 
 static DEVICE_ATTR(iocb_hw, S_IRUGO,
@@ -3511,7 +3551,7 @@ lpfc_txq_hw_show(struct device *dev, struct device_attribute *attr, char *buf)
 	struct lpfc_hba   *phba = ((struct lpfc_vport *) shost->hostdata)->phba;
 	struct lpfc_sli_ring *pring = lpfc_phba_elsring(phba);
 
-	return snprintf(buf, PAGE_SIZE, "%d\n",
+	return scnprintf(buf, PAGE_SIZE, "%d\n",
 			pring ? pring->txq_max : 0);
 }
 
@@ -3525,7 +3565,7 @@ lpfc_txcmplq_hw_show(struct device *dev, struct device_attribute *attr,
 	struct lpfc_hba   *phba = ((struct lpfc_vport *) shost->hostdata)->phba;
 	struct lpfc_sli_ring *pring = lpfc_phba_elsring(phba);
 
-	return snprintf(buf, PAGE_SIZE, "%d\n",
+	return scnprintf(buf, PAGE_SIZE, "%d\n",
 			pring ? pring->txcmplq_max : 0);
 }
 
@@ -3561,7 +3601,7 @@ lpfc_nodev_tmo_show(struct device *dev, struct device_attribute *attr,
 	struct Scsi_Host  *shost = class_to_shost(dev);
 	struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;
 
-	return snprintf(buf, PAGE_SIZE, "%d\n",	vport->cfg_devloss_tmo);
+	return scnprintf(buf, PAGE_SIZE, "%d\n",	vport->cfg_devloss_tmo);
 }
 
 /**
@@ -4050,9 +4090,9 @@ lpfc_topology_store(struct device *dev, struct device_attribute *attr,
 		}
 		if ((phba->pcidev->device == PCI_DEVICE_ID_LANCER_G6_FC ||
 		     phba->pcidev->device == PCI_DEVICE_ID_LANCER_G7_FC) &&
-		    val == 4) {
+		    val != FLAGS_TOPOLOGY_MODE_PT_PT) {
 			lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT,
-				"3114 Loop mode not supported\n");
+				"3114 Only non-FC-AL mode is supported\n");
 			return -EINVAL;
 		}
 		phba->cfg_topology = val;
@@ -5169,12 +5209,12 @@ lpfc_fcp_cpu_map_show(struct device *dev, struct device_attribute *attr,
 
 	switch (phba->cfg_fcp_cpu_map) {
 	case 0:
-		len += snprintf(buf + len, PAGE_SIZE-len,
+		len += scnprintf(buf + len, PAGE_SIZE-len,
 				"fcp_cpu_map: No mapping (%d)\n",
 				phba->cfg_fcp_cpu_map);
 		return len;
 	case 1:
-		len += snprintf(buf + len, PAGE_SIZE-len,
+		len += scnprintf(buf + len, PAGE_SIZE-len,
 				"fcp_cpu_map: HBA centric mapping (%d): "
 				"%d of %d CPUs online from %d possible CPUs\n",
 				phba->cfg_fcp_cpu_map, num_online_cpus(),
@@ -5188,12 +5228,12 @@ lpfc_fcp_cpu_map_show(struct device *dev, struct device_attribute *attr,
 		cpup = &phba->sli4_hba.cpu_map[phba->sli4_hba.curr_disp_cpu];
 
 		if (!cpu_present(phba->sli4_hba.curr_disp_cpu))
-			len += snprintf(buf + len, PAGE_SIZE - len,
+			len += scnprintf(buf + len, PAGE_SIZE - len,
 					"CPU %02d not present\n",
 					phba->sli4_hba.curr_disp_cpu);
 		else if (cpup->irq == LPFC_VECTOR_MAP_EMPTY) {
 			if (cpup->hdwq == LPFC_VECTOR_MAP_EMPTY)
-				len += snprintf(
+				len += scnprintf(
 					buf + len, PAGE_SIZE - len,
 					"CPU %02d hdwq None "
 					"physid %d coreid %d ht %d\n",
@@ -5201,7 +5241,7 @@ lpfc_fcp_cpu_map_show(struct device *dev, struct device_attribute *attr,
 					cpup->phys_id,
 					cpup->core_id, cpup->hyper);
 			else
-				len += snprintf(
+				len += scnprintf(
 					buf + len, PAGE_SIZE - len,
 					"CPU %02d EQ %04d hdwq %04d "
 					"physid %d coreid %d ht %d\n",
@@ -5210,7 +5250,7 @@ lpfc_fcp_cpu_map_show(struct device *dev, struct device_attribute *attr,
 					cpup->core_id, cpup->hyper);
 		} else {
 			if (cpup->hdwq == LPFC_VECTOR_MAP_EMPTY)
-				len += snprintf(
+				len += scnprintf(
 					buf + len, PAGE_SIZE - len,
 					"CPU %02d hdwq None "
 					"physid %d coreid %d ht %d IRQ %d\n",
@@ -5218,7 +5258,7 @@ lpfc_fcp_cpu_map_show(struct device *dev, struct device_attribute *attr,
 					cpup->phys_id,
 					cpup->core_id, cpup->hyper, cpup->irq);
 			else
-				len += snprintf(
+				len += scnprintf(
 					buf + len, PAGE_SIZE - len,
 					"CPU %02d EQ %04d hdwq %04d "
 					"physid %d coreid %d ht %d IRQ %d\n",
@@ -5233,7 +5273,7 @@ lpfc_fcp_cpu_map_show(struct device *dev, struct device_attribute *attr,
 		if (phba->sli4_hba.curr_disp_cpu <
 				phba->sli4_hba.num_possible_cpu &&
 				(len >= (PAGE_SIZE - 64))) {
-			len += snprintf(buf + len,
+			len += scnprintf(buf + len,
 					PAGE_SIZE - len, "more...\n");
 			break;
 		}
@@ -5753,10 +5793,10 @@ lpfc_sg_seg_cnt_show(struct device *dev, struct device_attribute *attr,
 	struct lpfc_hba   *phba = vport->phba;
 	int len;
 
-	len = snprintf(buf, PAGE_SIZE, "SGL sz: %d  total SGEs: %d\n",
+	len = scnprintf(buf, PAGE_SIZE, "SGL sz: %d  total SGEs: %d\n",
 		       phba->cfg_sg_dma_buf_size, phba->cfg_total_seg_cnt);
 
-	len += snprintf(buf + len, PAGE_SIZE, "Cfg: %d  SCSI: %d  NVME: %d\n",
+	len += scnprintf(buf + len, PAGE_SIZE, "Cfg: %d  SCSI: %d  NVME: %d\n",
 			phba->cfg_sg_seg_cnt, phba->cfg_scsi_seg_cnt,
 			phba->cfg_nvme_seg_cnt);
 	return len;
@@ -6755,7 +6795,7 @@ lpfc_show_rport_##field (struct device *dev,				\
 {									\
 	struct fc_rport *rport = transport_class_to_rport(dev);		\
 	struct lpfc_rport_data *rdata = rport->hostdata;		\
-	return snprintf(buf, sz, format_string,				\
+	return scnprintf(buf, sz, format_string,			\
 		(rdata->target) ? cast rdata->target->field : 0);	\
 }
 
@@ -7003,6 +7043,7 @@ lpfc_get_cfgparam(struct lpfc_hba *phba)
 	if (phba->sli_rev != LPFC_SLI_REV4) {
 		/* NVME only supported on SLI4 */
 		phba->nvmet_support = 0;
+		phba->cfg_nvmet_mrq = 0;
 		phba->cfg_enable_fc4_type = LPFC_ENABLE_FCP;
 		phba->cfg_enable_bbcr = 0;
 		phba->cfg_xri_rebalancing = 0;
@@ -7104,7 +7145,7 @@ lpfc_nvme_mod_param_dep(struct lpfc_hba *phba)
 	} else {
 		/* Not NVME Target mode.  Turn off Target parameters. */
 		phba->nvmet_support = 0;
-		phba->cfg_nvmet_mrq = LPFC_NVMET_MRQ_OFF;
+		phba->cfg_nvmet_mrq = 0;
 		phba->cfg_nvmet_fb_size = 0;
 	}
 }
diff --git a/drivers/scsi/lpfc/lpfc_bsg.c b/drivers/scsi/lpfc/lpfc_bsg.c
index f2494d3b365c..b0202bc0aa62 100644
--- a/drivers/scsi/lpfc/lpfc_bsg.c
+++ b/drivers/scsi/lpfc/lpfc_bsg.c
@@ -1,7 +1,7 @@
 /*******************************************************************
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
- * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term *
+ * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term *
  * “Broadcom� refers to Broadcom Inc. and/or its subsidiaries.     *
  * Copyright (C) 2009-2015 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
@@ -1968,14 +1968,17 @@ link_diag_state_set_out:
 }
 
 /**
- * lpfc_sli4_bsg_set_internal_loopback - set sli4 internal loopback diagnostic
+ * lpfc_sli4_bsg_set_loopback_mode - set sli4 internal loopback diagnostic
  * @phba: Pointer to HBA context object.
+ * @mode: loopback mode to set
+ * @link_no: link number for loopback mode to set
  *
  * This function is responsible for issuing a sli4 mailbox command for setting
- * up internal loopback diagnostic.
+ * up loopback diagnostic for a link.
  */
 static int
-lpfc_sli4_bsg_set_internal_loopback(struct lpfc_hba *phba)
+lpfc_sli4_bsg_set_loopback_mode(struct lpfc_hba *phba, int mode,
+				uint32_t link_no)
 {
 	LPFC_MBOXQ_t *pmboxq;
 	uint32_t req_len, alloc_len;
@@ -1996,11 +1999,19 @@ lpfc_sli4_bsg_set_internal_loopback(struct lpfc_hba *phba)
 	}
 	link_diag_loopback = &pmboxq->u.mqe.un.link_diag_loopback;
 	bf_set(lpfc_mbx_set_diag_state_link_num,
-	       &link_diag_loopback->u.req, phba->sli4_hba.lnk_info.lnk_no);
-	bf_set(lpfc_mbx_set_diag_state_link_type,
-	       &link_diag_loopback->u.req, phba->sli4_hba.lnk_info.lnk_tp);
+	       &link_diag_loopback->u.req, link_no);
+
+	if (phba->sli4_hba.conf_trunk & (1 << link_no)) {
+		bf_set(lpfc_mbx_set_diag_state_link_type,
+		       &link_diag_loopback->u.req, LPFC_LNK_FC_TRUNKED);
+	} else {
+		bf_set(lpfc_mbx_set_diag_state_link_type,
+		       &link_diag_loopback->u.req,
+		       phba->sli4_hba.lnk_info.lnk_tp);
+	}
+
 	bf_set(lpfc_mbx_set_diag_lpbk_type, &link_diag_loopback->u.req,
-	       LPFC_DIAG_LOOPBACK_TYPE_INTERNAL);
+	       mode);
 
 	mbxstatus = lpfc_sli_issue_mbox_wait(phba, pmboxq, LPFC_MBOX_TMO);
 	if ((mbxstatus != MBX_SUCCESS) || (pmboxq->u.mb.mbxStatus)) {
@@ -2054,7 +2065,7 @@ lpfc_sli4_bsg_diag_loopback_mode(struct lpfc_hba *phba, struct bsg_job *job)
 	struct fc_bsg_request *bsg_request = job->request;
 	struct fc_bsg_reply *bsg_reply = job->reply;
 	struct diag_mode_set *loopback_mode;
-	uint32_t link_flags, timeout;
+	uint32_t link_flags, timeout, link_no;
 	int i, rc = 0;
 
 	/* no data to return just the return code */
@@ -2069,12 +2080,39 @@ lpfc_sli4_bsg_diag_loopback_mode(struct lpfc_hba *phba, struct bsg_job *job)
 				(int)(sizeof(struct fc_bsg_request) +
 				sizeof(struct diag_mode_set)));
 		rc = -EINVAL;
-		goto job_error;
+		goto job_done;
+	}
+
+	loopback_mode = (struct diag_mode_set *)
+		bsg_request->rqst_data.h_vendor.vendor_cmd;
+	link_flags = loopback_mode->type;
+	timeout = loopback_mode->timeout * 100;
+
+	if (loopback_mode->physical_link == -1)
+		link_no = phba->sli4_hba.lnk_info.lnk_no;
+	else
+		link_no = loopback_mode->physical_link;
+
+	if (link_flags == DISABLE_LOOP_BACK) {
+		rc = lpfc_sli4_bsg_set_loopback_mode(phba,
+					LPFC_DIAG_LOOPBACK_TYPE_DISABLE,
+					link_no);
+		if (!rc) {
+			/* Unset the need disable bit */
+			phba->sli4_hba.conf_trunk &= ~((1 << link_no) << 4);
+		}
+		goto job_done;
+	} else {
+		/* Check if we need to disable the loopback state */
+		if (phba->sli4_hba.conf_trunk & ((1 << link_no) << 4)) {
+			rc = -EPERM;
+			goto job_done;
+		}
 	}
 
 	rc = lpfc_bsg_diag_mode_enter(phba);
 	if (rc)
-		goto job_error;
+		goto job_done;
 
 	/* indicate we are in loobpack diagnostic mode */
 	spin_lock_irq(&phba->hbalock);
@@ -2084,15 +2122,11 @@ lpfc_sli4_bsg_diag_loopback_mode(struct lpfc_hba *phba, struct bsg_job *job)
 	/* reset port to start frome scratch */
 	rc = lpfc_selective_reset(phba);
 	if (rc)
-		goto job_error;
+		goto job_done;
 
 	/* bring the link to diagnostic mode */
 	lpfc_printf_log(phba, KERN_INFO, LOG_LIBDFC,
 			"3129 Bring link to diagnostic state.\n");
-	loopback_mode = (struct diag_mode_set *)
-		bsg_request->rqst_data.h_vendor.vendor_cmd;
-	link_flags = loopback_mode->type;
-	timeout = loopback_mode->timeout * 100;
 
 	rc = lpfc_sli4_bsg_set_link_diag_state(phba, 1);
 	if (rc) {
@@ -2120,13 +2154,54 @@ lpfc_sli4_bsg_diag_loopback_mode(struct lpfc_hba *phba, struct bsg_job *job)
 	lpfc_printf_log(phba, KERN_INFO, LOG_LIBDFC,
 			"3132 Set up loopback mode:x%x\n", link_flags);
 
-	if (link_flags == INTERNAL_LOOP_BACK)
-		rc = lpfc_sli4_bsg_set_internal_loopback(phba);
-	else if (link_flags == EXTERNAL_LOOP_BACK)
-		rc = lpfc_hba_init_link_fc_topology(phba,
-						    FLAGS_TOPOLOGY_MODE_PT_PT,
-						    MBX_NOWAIT);
-	else {
+	switch (link_flags) {
+	case INTERNAL_LOOP_BACK:
+		if (phba->sli4_hba.conf_trunk & (1 << link_no)) {
+			rc = lpfc_sli4_bsg_set_loopback_mode(phba,
+					LPFC_DIAG_LOOPBACK_TYPE_INTERNAL,
+					link_no);
+		} else {
+			/* Trunk is configured, but link is not in this trunk */
+			if (phba->sli4_hba.conf_trunk) {
+				rc = -ELNRNG;
+				goto loopback_mode_exit;
+			}
+
+			rc = lpfc_sli4_bsg_set_loopback_mode(phba,
+					LPFC_DIAG_LOOPBACK_TYPE_INTERNAL,
+					link_no);
+		}
+
+		if (!rc) {
+			/* Set the need disable bit */
+			phba->sli4_hba.conf_trunk |= (1 << link_no) << 4;
+		}
+
+		break;
+	case EXTERNAL_LOOP_BACK:
+		if (phba->sli4_hba.conf_trunk & (1 << link_no)) {
+			rc = lpfc_sli4_bsg_set_loopback_mode(phba,
+				LPFC_DIAG_LOOPBACK_TYPE_EXTERNAL_TRUNKED,
+				link_no);
+		} else {
+			/* Trunk is configured, but link is not in this trunk */
+			if (phba->sli4_hba.conf_trunk) {
+				rc = -ELNRNG;
+				goto loopback_mode_exit;
+			}
+
+			rc = lpfc_sli4_bsg_set_loopback_mode(phba,
+						LPFC_DIAG_LOOPBACK_TYPE_SERDES,
+						link_no);
+		}
+
+		if (!rc) {
+			/* Set the need disable bit */
+			phba->sli4_hba.conf_trunk |= (1 << link_no) << 4;
+		}
+
+		break;
+	default:
 		rc = -EINVAL;
 		lpfc_printf_log(phba, KERN_ERR, LOG_LIBDFC,
 				"3141 Loopback mode:x%x not supported\n",
@@ -2185,7 +2260,7 @@ loopback_mode_exit:
 	}
 	lpfc_bsg_diag_mode_exit(phba);
 
-job_error:
+job_done:
 	/* make error code available to userspace */
 	bsg_reply->result = rc;
 	/* complete the job back to userspace if no error */
diff --git a/drivers/scsi/lpfc/lpfc_bsg.h b/drivers/scsi/lpfc/lpfc_bsg.h
index 9151824beea4..d1708133fd54 100644
--- a/drivers/scsi/lpfc/lpfc_bsg.h
+++ b/drivers/scsi/lpfc/lpfc_bsg.h
@@ -1,7 +1,7 @@
 /*******************************************************************
  * This file is part of the Emulex Linux Device Driver for         *
  * Fibre Channel Host Bus Adapters.                                *
- * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term *
+ * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term *
  * “Broadcom� refers to Broadcom Inc. and/or its subsidiaries.     *
  * Copyright (C) 2010-2015 Emulex.  All rights reserved.           *
  * EMULEX and SLI are trademarks of Emulex.                        *
@@ -68,6 +68,7 @@ struct send_mgmt_resp {
 };
 
 
+#define DISABLE_LOOP_BACK  0x0 /* disables loop back */
 #define INTERNAL_LOOP_BACK 0x1 /* adapter short cuts the loop internally */
 #define EXTERNAL_LOOP_BACK 0x2 /* requires an external loopback plug */
 
@@ -75,6 +76,7 @@ struct diag_mode_set {
 	uint32_t command;
 	uint32_t type;
 	uint32_t timeout;
+	uint32_t physical_link;
 };
 
 struct sli4_link_diag {
diff --git a/drivers/scsi/lpfc/lpfc_ct.c b/drivers/scsi/lpfc/lpfc_ct.c
index 44f426347d4f..4812bbbf43cc 100644
--- a/drivers/scsi/lpfc/lpfc_ct.c
+++ b/drivers/scsi/lpfc/lpfc_ct.c
@@ -886,7 +886,7 @@ lpfc_cmpl_ct_cmd_gid_pt(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 	}
 	if (lpfc_error_lost_link(irsp)) {
 		lpfc_printf_vlog(vport, KERN_INFO, LOG_DISCOVERY,
-				 "4101 NS query failed due to link event\n");
+				 "4166 NS query failed due to link event\n");
 		if (vport->fc_flag & FC_RSCN_MODE)
 			lpfc_els_flush_rscn(vport);
 		goto out;
@@ -907,7 +907,7 @@ lpfc_cmpl_ct_cmd_gid_pt(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 		 * Re-issue the NS cmd
 		 */
 		lpfc_printf_vlog(vport, KERN_INFO, LOG_ELS,
-				 "4102 Process Deferred RSCN Data: x%x x%x\n",
+				 "4167 Process Deferred RSCN Data: x%x x%x\n",
 				 vport->fc_flag, vport->fc_rscn_id_cnt);
 		lpfc_els_handle_rscn(vport);
 
@@ -1430,7 +1430,7 @@ lpfc_vport_symbolic_port_name(struct lpfc_vport *vport, char *symbol,
 	 * Name object.  NPIV is not in play so this integer
 	 * value is sufficient and unique per FC-ID.
 	 */
-	n = snprintf(symbol, size, "%d", vport->phba->brd_no);
+	n = scnprintf(symbol, size, "%d", vport->phba->brd_no);
 	return n;
 }
 
@@ -1444,26 +1444,26 @@ lpfc_vport_symbolic_node_name(struct lpfc_vport *vport, char *symbol,
 
 	lpfc_decode_firmware_rev(vport->phba, fwrev, 0);
 
-	n = snprintf(symbol, size, "Emulex %s", vport->phba->ModelName);
+	n = scnprintf(symbol, size, "Emulex %s", vport->phba->ModelName);
 	if (size < n)
 		return n;
 
-	n += snprintf(symbol + n, size - n, " FV%s", fwrev);
+	n += scnprintf(symbol + n, size - n, " FV%s", fwrev);
 	if (size < n)
 		return n;
 
-	n += snprintf(symbol + n, size - n, " DV%s.",
+	n += scnprintf(symbol + n, size - n, " DV%s.",
 		      lpfc_release_version);
 	if (size < n)
 		return n;
 
-	n += snprintf(symbol + n, size - n, " HN:%s.",
+	n += scnprintf(symbol + n, size - n, " HN:%s.",
 		      init_utsname()->nodename);
 	if (size < n)
 		return n;
 
 	/* Note :- OS name is "Linux" */
-	n += snprintf(symbol + n, size - n, " OS:%s\n",
+	n += scnprintf(symbol + n, size - n, " OS:%s",
 		      init_utsname()->sysname);
 	return n;
 }
@@ -2005,8 +2005,11 @@ lpfc_fdmi_hba_attr_manufacturer(struct lpfc_vport *vport,
 	ae = (struct lpfc_fdmi_attr_entry *)&ad->AttrValue;
 	memset(ae, 0, 256);
 
+	/* This string MUST be consistent with other FC platforms
+	 * supported by Broadcom.
+	 */
 	strncpy(ae->un.AttrString,
-		"Broadcom Inc.",
+		"Emulex Corporation",
 		       sizeof(ae->un.AttrString));
 	len = strnlen(ae->un.AttrString,
 			  sizeof(ae->un.AttrString));
@@ -2301,7 +2304,8 @@ lpfc_fdmi_hba_attr_bios_ver(struct lpfc_vport *vport,
 	ae = (struct lpfc_fdmi_attr_entry *)&ad->AttrValue;
 	memset(ae, 0, 256);
 
-	lpfc_decode_firmware_rev(phba, ae->un.AttrString, 1);
+	strlcat(ae->un.AttrString, phba->BIOSVersion,
+		sizeof(ae->un.AttrString));
 	len = strnlen(ae->un.AttrString,
 			  sizeof(ae->un.AttrString));
 	len += (len & 3) ? (4 - (len & 3)) : 4;
@@ -2360,10 +2364,11 @@ lpfc_fdmi_port_attr_fc4type(struct lpfc_vport *vport,
 	ae = (struct lpfc_fdmi_attr_entry *)&ad->AttrValue;
 	memset(ae, 0, 32);
 
-	ae->un.AttrTypes[3] = 0x02; /* Type 1 - ELS */
-	ae->un.AttrTypes[2] = 0x01; /* Type 8 - FCP */
-	ae->un.AttrTypes[6] = 0x01; /* Type 40 - NVME */
-	ae->un.AttrTypes[7] = 0x01; /* Type 32 - CT */
+	ae->un.AttrTypes[3] = 0x02; /* Type 0x1 - ELS */
+	ae->un.AttrTypes[2] = 0x01; /* Type 0x8 - FCP */
+	if (vport->nvmei_support || vport->phba->nvmet_support)
+		ae->un.AttrTypes[6] = 0x01; /* Type 0x28 - NVME */
+	ae->un.AttrTypes[7] = 0x01; /* Type 0x20 - CT */
 	size = FOURBYTES + 32;
 	ad->AttrLen = cpu_to_be16(size);
 	ad->AttrType = cpu_to_be16(RPRT_SUPPORTED_FC4_TYPES);
@@ -2673,9 +2678,11 @@ lpfc_fdmi_port_attr_active_fc4type(struct lpfc_vport *vport,
 	ae = (struct lpfc_fdmi_attr_entry *)&ad->AttrValue;
 	memset(ae, 0, 32);
 
-	ae->un.AttrTypes[3] = 0x02; /* Type 1 - ELS */
-	ae->un.AttrTypes[2] = 0x01; /* Type 8 - FCP */
-	ae->un.AttrTypes[7] = 0x01; /* Type 32 - CT */
+	ae->un.AttrTypes[3] = 0x02; /* Type 0x1 - ELS */
+	ae->un.AttrTypes[2] = 0x01; /* Type 0x8 - FCP */
+	if (vport->phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME)
+		ae->un.AttrTypes[6] = 0x1; /* Type 0x28 - NVME */
+	ae->un.AttrTypes[7] = 0x01; /* Type 0x20 - CT */
 	size = FOURBYTES + 32;
 	ad->AttrLen = cpu_to_be16(size);
 	ad->AttrType = cpu_to_be16(RPRT_ACTIVE_FC4_TYPES);
diff --git a/drivers/scsi/lpfc/lpfc_debugfs.c b/drivers/scsi/lpfc/lpfc_debugfs.c
index 1215eaa530db..1ee857d9d165 100644
--- a/drivers/scsi/lpfc/lpfc_debugfs.c
+++ b/drivers/scsi/lpfc/lpfc_debugfs.c
@@ -170,7 +170,7 @@ lpfc_debugfs_disc_trc_data(struct lpfc_vport *vport, char *buf, int size)
 		snprintf(buffer,
 			LPFC_DEBUG_TRC_ENTRY_SIZE, "%010d:%010d ms:%s\n",
 			dtp->seq_cnt, ms, dtp->fmt);
-		len +=  snprintf(buf+len, size-len, buffer,
+		len +=  scnprintf(buf+len, size-len, buffer,
 			dtp->data1, dtp->data2, dtp->data3);
 	}
 	for (i = 0; i < index; i++) {
@@ -181,7 +181,7 @@ lpfc_debugfs_disc_trc_data(struct lpfc_vport *vport, char *buf, int size)
 		snprintf(buffer,
 			LPFC_DEBUG_TRC_ENTRY_SIZE, "%010d:%010d ms:%s\n",
 			dtp->seq_cnt, ms, dtp->fmt);
-		len +=  snprintf(buf+len, size-len, buffer,
+		len +=  scnprintf(buf+len, size-len, buffer,
 			dtp->data1, dtp->data2, dtp->data3);
 	}
 
@@ -236,7 +236,7 @@ lpfc_debugfs_slow_ring_trc_data(struct lpfc_hba *phba, char *buf, int size)
 		snprintf(buffer,
 			LPFC_DEBUG_TRC_ENTRY_SIZE, "%010d:%010d ms:%s\n",
 			dtp->seq_cnt, ms, dtp->fmt);
-		len +=  snprintf(buf+len, size-len, buffer,
+		len +=  scnprintf(buf+len, size-len, buffer,
 			dtp->data1, dtp->data2, dtp->data3);
 	}
 	for (i = 0; i < index; i++) {
@@ -247,7 +247,7 @@ lpfc_debugfs_slow_ring_trc_data(struct lpfc_hba *phba, char *buf, int size)
 		snprintf(buffer,
 			LPFC_DEBUG_TRC_ENTRY_SIZE, "%010d:%010d ms:%s\n",
 			dtp->seq_cnt, ms, dtp->fmt);
-		len +=  snprintf(buf+len, size-len, buffer,
+		len +=  scnprintf(buf+len, size-len, buffer,
 			dtp->data1, dtp->data2, dtp->data3);
 	}
 
@@ -307,7 +307,7 @@ lpfc_debugfs_hbqinfo_data(struct lpfc_hba *phba, char *buf, int size)
 
 	i = lpfc_debugfs_last_hbq;
 
-	len +=  snprintf(buf+len, size-len, "HBQ %d Info\n", i);
+	len +=  scnprintf(buf+len, size-len, "HBQ %d Info\n", i);
 
 	hbqs =  &phba->hbqs[i];
 	posted = 0;
@@ -315,21 +315,21 @@ lpfc_debugfs_hbqinfo_data(struct lpfc_hba *phba, char *buf, int size)
 		posted++;
 
 	hip =  lpfc_hbq_defs[i];
-	len +=  snprintf(buf+len, size-len,
+	len +=  scnprintf(buf+len, size-len,
 		"idx:%d prof:%d rn:%d bufcnt:%d icnt:%d acnt:%d posted %d\n",
 		hip->hbq_index, hip->profile, hip->rn,
 		hip->buffer_count, hip->init_count, hip->add_count, posted);
 
 	raw_index = phba->hbq_get[i];
 	getidx = le32_to_cpu(raw_index);
-	len +=  snprintf(buf+len, size-len,
+	len +=  scnprintf(buf+len, size-len,
 		"entries:%d bufcnt:%d Put:%d nPut:%d localGet:%d hbaGet:%d\n",
 		hbqs->entry_count, hbqs->buffer_count, hbqs->hbqPutIdx,
 		hbqs->next_hbqPutIdx, hbqs->local_hbqGetIdx, getidx);
 
 	hbqe = (struct lpfc_hbq_entry *) phba->hbqs[i].hbq_virt;
 	for (j=0; j<hbqs->entry_count; j++) {
-		len +=  snprintf(buf+len, size-len,
+		len +=  scnprintf(buf+len, size-len,
 			"%03d: %08x %04x %05x ", j,
 			le32_to_cpu(hbqe->bde.addrLow),
 			le32_to_cpu(hbqe->bde.tus.w),
@@ -341,14 +341,16 @@ lpfc_debugfs_hbqinfo_data(struct lpfc_hba *phba, char *buf, int size)
 		low = hbqs->hbqPutIdx - posted;
 		if (low >= 0) {
 			if ((j >= hbqs->hbqPutIdx) || (j < low)) {
-				len +=  snprintf(buf+len, size-len, "Unused\n");
+				len +=  scnprintf(buf + len, size - len,
+						"Unused\n");
 				goto skipit;
 			}
 		}
 		else {
 			if ((j >= hbqs->hbqPutIdx) &&
 				(j < (hbqs->entry_count+low))) {
-				len +=  snprintf(buf+len, size-len, "Unused\n");
+				len +=  scnprintf(buf + len, size - len,
+						"Unused\n");
 				goto skipit;
 			}
 		}
@@ -358,7 +360,7 @@ lpfc_debugfs_hbqinfo_data(struct lpfc_hba *phba, char *buf, int size)
 			hbq_buf = container_of(d_buf, struct hbq_dmabuf, dbuf);
 			phys = ((uint64_t)hbq_buf->dbuf.phys & 0xffffffff);
 			if (phys == le32_to_cpu(hbqe->bde.addrLow)) {
-				len +=  snprintf(buf+len, size-len,
+				len +=  scnprintf(buf+len, size-len,
 					"Buf%d: %p %06x\n", i,
 					hbq_buf->dbuf.virt, hbq_buf->tag);
 				found = 1;
@@ -367,7 +369,7 @@ lpfc_debugfs_hbqinfo_data(struct lpfc_hba *phba, char *buf, int size)
 			i++;
 		}
 		if (!found) {
-			len +=  snprintf(buf+len, size-len, "No DMAinfo?\n");
+			len +=  scnprintf(buf+len, size-len, "No DMAinfo?\n");
 		}
 skipit:
 		hbqe++;
@@ -413,14 +415,14 @@ lpfc_debugfs_commonxripools_data(struct lpfc_hba *phba, char *buf, int size)
 			break;
 		qp = &phba->sli4_hba.hdwq[lpfc_debugfs_last_xripool];
 
-		len +=  snprintf(buf + len, size - len, "HdwQ %d Info ", i);
+		len += scnprintf(buf + len, size - len, "HdwQ %d Info ", i);
 		spin_lock_irqsave(&qp->abts_scsi_buf_list_lock, iflag);
 		spin_lock(&qp->abts_nvme_buf_list_lock);
 		spin_lock(&qp->io_buf_list_get_lock);
 		spin_lock(&qp->io_buf_list_put_lock);
 		out = qp->total_io_bufs - (qp->get_io_bufs + qp->put_io_bufs +
 			qp->abts_scsi_io_bufs + qp->abts_nvme_io_bufs);
-		len +=  snprintf(buf + len, size - len,
+		len += scnprintf(buf + len, size - len,
 				 "tot:%d get:%d put:%d mt:%d "
 				 "ABTS scsi:%d nvme:%d Out:%d\n",
 			qp->total_io_bufs, qp->get_io_bufs, qp->put_io_bufs,
@@ -612,9 +614,9 @@ lpfc_debugfs_lockstat_data(struct lpfc_hba *phba, char *buf, int size)
 			break;
 		qp = &phba->sli4_hba.hdwq[lpfc_debugfs_last_lock];
 
-		len +=  snprintf(buf + len, size - len, "HdwQ %03d Lock ", i);
+		len += scnprintf(buf + len, size - len, "HdwQ %03d Lock ", i);
 		if (phba->cfg_xri_rebalancing) {
-			len +=  snprintf(buf + len, size - len,
+			len += scnprintf(buf + len, size - len,
 					 "get_pvt:%d mv_pvt:%d "
 					 "mv2pub:%d mv2pvt:%d "
 					 "put_pvt:%d put_pub:%d wq:%d\n",
@@ -626,7 +628,7 @@ lpfc_debugfs_lockstat_data(struct lpfc_hba *phba, char *buf, int size)
 					 qp->lock_conflict.free_pub_pool,
 					 qp->lock_conflict.wq_access);
 		} else {
-			len +=  snprintf(buf + len, size - len,
+			len += scnprintf(buf + len, size - len,
 					 "get:%d put:%d free:%d wq:%d\n",
 					 qp->lock_conflict.alloc_xri_get,
 					 qp->lock_conflict.alloc_xri_put,
@@ -678,7 +680,7 @@ lpfc_debugfs_dumpHBASlim_data(struct lpfc_hba *phba, char *buf, int size)
 	off = 0;
 	spin_lock_irq(&phba->hbalock);
 
-	len +=  snprintf(buf+len, size-len, "HBA SLIM\n");
+	len +=  scnprintf(buf+len, size-len, "HBA SLIM\n");
 	lpfc_memcpy_from_slim(buffer,
 		phba->MBslimaddr + lpfc_debugfs_last_hba_slim_off, 1024);
 
@@ -692,7 +694,7 @@ lpfc_debugfs_dumpHBASlim_data(struct lpfc_hba *phba, char *buf, int size)
 
 	i = 1024;
 	while (i > 0) {
-		len +=  snprintf(buf+len, size-len,
+		len +=  scnprintf(buf+len, size-len,
 		"%08x: %08x %08x %08x %08x %08x %08x %08x %08x\n",
 		off, *ptr, *(ptr+1), *(ptr+2), *(ptr+3), *(ptr+4),
 		*(ptr+5), *(ptr+6), *(ptr+7));
@@ -736,11 +738,11 @@ lpfc_debugfs_dumpHostSlim_data(struct lpfc_hba *phba, char *buf, int size)
 	off = 0;
 	spin_lock_irq(&phba->hbalock);
 
-	len +=  snprintf(buf+len, size-len, "SLIM Mailbox\n");
+	len +=  scnprintf(buf+len, size-len, "SLIM Mailbox\n");
 	ptr = (uint32_t *)phba->slim2p.virt;
 	i = sizeof(MAILBOX_t);
 	while (i > 0) {
-		len +=  snprintf(buf+len, size-len,
+		len +=  scnprintf(buf+len, size-len,
 		"%08x: %08x %08x %08x %08x %08x %08x %08x %08x\n",
 		off, *ptr, *(ptr+1), *(ptr+2), *(ptr+3), *(ptr+4),
 		*(ptr+5), *(ptr+6), *(ptr+7));
@@ -749,11 +751,11 @@ lpfc_debugfs_dumpHostSlim_data(struct lpfc_hba *phba, char *buf, int size)
 		off += (8 * sizeof(uint32_t));
 	}
 
-	len +=  snprintf(buf+len, size-len, "SLIM PCB\n");
+	len +=  scnprintf(buf+len, size-len, "SLIM PCB\n");
 	ptr = (uint32_t *)phba->pcb;
 	i = sizeof(PCB_t);
 	while (i > 0) {
-		len +=  snprintf(buf+len, size-len,
+		len +=  scnprintf(buf+len, size-len,
 		"%08x: %08x %08x %08x %08x %08x %08x %08x %08x\n",
 		off, *ptr, *(ptr+1), *(ptr+2), *(ptr+3), *(ptr+4),
 		*(ptr+5), *(ptr+6), *(ptr+7));
@@ -766,7 +768,7 @@ lpfc_debugfs_dumpHostSlim_data(struct lpfc_hba *phba, char *buf, int size)
 		for (i = 0; i < 4; i++) {
 			pgpp = &phba->port_gp[i];
 			pring = &psli->sli3_ring[i];
-			len +=  snprintf(buf+len, size-len,
+			len +=  scnprintf(buf+len, size-len,
 					 "Ring %d: CMD GetInx:%d "
 					 "(Max:%d Next:%d "
 					 "Local:%d flg:x%x)  "
@@ -783,7 +785,7 @@ lpfc_debugfs_dumpHostSlim_data(struct lpfc_hba *phba, char *buf, int size)
 		word1 = readl(phba->CAregaddr);
 		word2 = readl(phba->HSregaddr);
 		word3 = readl(phba->HCregaddr);
-		len +=  snprintf(buf+len, size-len, "HA:%08x CA:%08x HS:%08x "
+		len +=  scnprintf(buf+len, size-len, "HA:%08x CA:%08x HS:%08x "
 				 "HC:%08x\n", word0, word1, word2, word3);
 	}
 	spin_unlock_irq(&phba->hbalock);
@@ -821,12 +823,12 @@ lpfc_debugfs_nodelist_data(struct lpfc_vport *vport, char *buf, int size)
 	cnt = (LPFC_NODELIST_SIZE / LPFC_NODELIST_ENTRY_SIZE);
 	outio = 0;
 
-	len += snprintf(buf+len, size-len, "\nFCP Nodelist Entries ...\n");
+	len += scnprintf(buf+len, size-len, "\nFCP Nodelist Entries ...\n");
 	spin_lock_irq(shost->host_lock);
 	list_for_each_entry(ndlp, &vport->fc_nodes, nlp_listp) {
 		iocnt = 0;
 		if (!cnt) {
-			len +=  snprintf(buf+len, size-len,
+			len +=  scnprintf(buf+len, size-len,
 				"Missing Nodelist Entries\n");
 			break;
 		}
@@ -864,63 +866,63 @@ lpfc_debugfs_nodelist_data(struct lpfc_vport *vport, char *buf, int size)
 		default:
 			statep = "UNKNOWN";
 		}
-		len += snprintf(buf+len, size-len, "%s DID:x%06x ",
+		len += scnprintf(buf+len, size-len, "%s DID:x%06x ",
 				statep, ndlp->nlp_DID);
-		len += snprintf(buf+len, size-len,
+		len += scnprintf(buf+len, size-len,
 				"WWPN x%llx ",
 				wwn_to_u64(ndlp->nlp_portname.u.wwn));
-		len += snprintf(buf+len, size-len,
+		len += scnprintf(buf+len, size-len,
 				"WWNN x%llx ",
 				wwn_to_u64(ndlp->nlp_nodename.u.wwn));
 		if (ndlp->nlp_flag & NLP_RPI_REGISTERED)
-			len += snprintf(buf+len, size-len, "RPI:%03d ",
+			len += scnprintf(buf+len, size-len, "RPI:%03d ",
 					ndlp->nlp_rpi);
 		else
-			len += snprintf(buf+len, size-len, "RPI:none ");
-		len +=  snprintf(buf+len, size-len, "flag:x%08x ",
+			len += scnprintf(buf+len, size-len, "RPI:none ");
+		len +=  scnprintf(buf+len, size-len, "flag:x%08x ",
 			ndlp->nlp_flag);
 		if (!ndlp->nlp_type)
-			len += snprintf(buf+len, size-len, "UNKNOWN_TYPE ");
+			len += scnprintf(buf+len, size-len, "UNKNOWN_TYPE ");
 		if (ndlp->nlp_type & NLP_FC_NODE)
-			len += snprintf(buf+len, size-len, "FC_NODE ");
+			len += scnprintf(buf+len, size-len, "FC_NODE ");
 		if (ndlp->nlp_type & NLP_FABRIC) {
-			len += snprintf(buf+len, size-len, "FABRIC ");
+			len += scnprintf(buf+len, size-len, "FABRIC ");
 			iocnt = 0;
 		}
 		if (ndlp->nlp_type & NLP_FCP_TARGET)
-			len += snprintf(buf+len, size-len, "FCP_TGT sid:%d ",
+			len += scnprintf(buf+len, size-len, "FCP_TGT sid:%d ",
 				ndlp->nlp_sid);
 		if (ndlp->nlp_type & NLP_FCP_INITIATOR)
-			len += snprintf(buf+len, size-len, "FCP_INITIATOR ");
+			len += scnprintf(buf+len, size-len, "FCP_INITIATOR ");
 		if (ndlp->nlp_type & NLP_NVME_TARGET)
-			len += snprintf(buf + len,
+			len += scnprintf(buf + len,
 					size - len, "NVME_TGT sid:%d ",
 					NLP_NO_SID);
 		if (ndlp->nlp_type & NLP_NVME_INITIATOR)
-			len += snprintf(buf + len,
+			len += scnprintf(buf + len,
 					size - len, "NVME_INITIATOR ");
-		len += snprintf(buf+len, size-len, "usgmap:%x ",
+		len += scnprintf(buf+len, size-len, "usgmap:%x ",
 			ndlp->nlp_usg_map);
-		len += snprintf(buf+len, size-len, "refcnt:%x",
+		len += scnprintf(buf+len, size-len, "refcnt:%x",
 			kref_read(&ndlp->kref));
 		if (iocnt) {
 			i = atomic_read(&ndlp->cmd_pending);
-			len += snprintf(buf + len, size - len,
+			len += scnprintf(buf + len, size - len,
 					" OutIO:x%x Qdepth x%x",
 					i, ndlp->cmd_qdepth);
 			outio += i;
 		}
-		len += snprintf(buf + len, size - len, "defer:%x ",
+		len += scnprintf(buf + len, size - len, "defer:%x ",
 			ndlp->nlp_defer_did);
-		len +=  snprintf(buf+len, size-len, "\n");
+		len +=  scnprintf(buf+len, size-len, "\n");
 	}
 	spin_unlock_irq(shost->host_lock);
 
-	len += snprintf(buf + len, size - len,
+	len += scnprintf(buf + len, size - len,
 			"\nOutstanding IO x%x\n",  outio);
 
 	if (phba->nvmet_support && phba->targetport && (vport == phba->pport)) {
-		len += snprintf(buf + len, size - len,
+		len += scnprintf(buf + len, size - len,
 				"\nNVME Targetport Entry ...\n");
 
 		/* Port state is only one of two values for now. */
@@ -928,18 +930,18 @@ lpfc_debugfs_nodelist_data(struct lpfc_vport *vport, char *buf, int size)
 			statep = "REGISTERED";
 		else
 			statep = "INIT";
-		len += snprintf(buf + len, size - len,
+		len += scnprintf(buf + len, size - len,
 				"TGT WWNN x%llx WWPN x%llx State %s\n",
 				wwn_to_u64(vport->fc_nodename.u.wwn),
 				wwn_to_u64(vport->fc_portname.u.wwn),
 				statep);
-		len += snprintf(buf + len, size - len,
+		len += scnprintf(buf + len, size - len,
 				"    Targetport DID x%06x\n",
 				phba->targetport->port_id);
 		goto out_exit;
 	}
 
-	len += snprintf(buf + len, size - len,
+	len += scnprintf(buf + len, size - len,
 				"\nNVME Lport/Rport Entries ...\n");
 
 	localport = vport->localport;
@@ -954,11 +956,11 @@ lpfc_debugfs_nodelist_data(struct lpfc_vport *vport, char *buf, int size)
 	else
 		statep = "UNKNOWN ";
 
-	len += snprintf(buf + len, size - len,
+	len += scnprintf(buf + len, size - len,
 			"Lport DID x%06x PortState %s\n",
 			localport->port_id, statep);
 
-	len += snprintf(buf + len, size - len, "\tRport List:\n");
+	len += scnprintf(buf + len, size - len, "\tRport List:\n");
 	list_for_each_entry(ndlp, &vport->fc_nodes, nlp_listp) {
 		/* local short-hand pointer. */
 		spin_lock(&phba->hbalock);
@@ -985,32 +987,32 @@ lpfc_debugfs_nodelist_data(struct lpfc_vport *vport, char *buf, int size)
 		}
 
 		/* Tab in to show lport ownership. */
-		len += snprintf(buf + len, size - len,
+		len += scnprintf(buf + len, size - len,
 				"\t%s Port ID:x%06x ",
 				statep, nrport->port_id);
-		len += snprintf(buf + len, size - len, "WWPN x%llx ",
+		len += scnprintf(buf + len, size - len, "WWPN x%llx ",
 				nrport->port_name);
-		len += snprintf(buf + len, size - len, "WWNN x%llx ",
+		len += scnprintf(buf + len, size - len, "WWNN x%llx ",
 				nrport->node_name);
 
 		/* An NVME rport can have multiple roles. */
 		if (nrport->port_role & FC_PORT_ROLE_NVME_INITIATOR)
-			len +=  snprintf(buf + len, size - len,
+			len +=  scnprintf(buf + len, size - len,
 					 "INITIATOR ");
 		if (nrport->port_role & FC_PORT_ROLE_NVME_TARGET)
-			len +=  snprintf(buf + len, size - len,
+			len +=  scnprintf(buf + len, size - len,
 					 "TARGET ");
 		if (nrport->port_role & FC_PORT_ROLE_NVME_DISCOVERY)
-			len +=  snprintf(buf + len, size - len,
+			len +=  scnprintf(buf + len, size - len,
 					 "DISCSRVC ");
 		if (nrport->port_role & ~(FC_PORT_ROLE_NVME_INITIATOR |
 					  FC_PORT_ROLE_NVME_TARGET |
 					  FC_PORT_ROLE_NVME_DISCOVERY))
-			len +=  snprintf(buf + len, size - len,
+			len +=  scnprintf(buf + len, size - len,
 					 "UNKNOWN ROLE x%x",
 					 nrport->port_role);
 		/* Terminate the string. */
-		len +=  snprintf(buf + len, size - len, "\n");
+		len +=  scnprintf(buf + len, size - len, "\n");
 	}
 
 	spin_unlock_irq(shost->host_lock);
@@ -1049,35 +1051,35 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size)
 		if (!phba->targetport)
 			return len;
 		tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
-		len += snprintf(buf + len, size - len,
+		len += scnprintf(buf + len, size - len,
 				"\nNVME Targetport Statistics\n");
 
-		len += snprintf(buf + len, size - len,
+		len += scnprintf(buf + len, size - len,
 				"LS: Rcv %08x Drop %08x Abort %08x\n",
 				atomic_read(&tgtp->rcv_ls_req_in),
 				atomic_read(&tgtp->rcv_ls_req_drop),
 				atomic_read(&tgtp->xmt_ls_abort));
 		if (atomic_read(&tgtp->rcv_ls_req_in) !=
 		    atomic_read(&tgtp->rcv_ls_req_out)) {
-			len += snprintf(buf + len, size - len,
+			len += scnprintf(buf + len, size - len,
 					"Rcv LS: in %08x != out %08x\n",
 					atomic_read(&tgtp->rcv_ls_req_in),
 					atomic_read(&tgtp->rcv_ls_req_out));
 		}
 
-		len += snprintf(buf + len, size - len,
+		len += scnprintf(buf + len, size - len,
 				"LS: Xmt %08x Drop %08x Cmpl %08x\n",
 				atomic_read(&tgtp->xmt_ls_rsp),
 				atomic_read(&tgtp->xmt_ls_drop),
 				atomic_read(&tgtp->xmt_ls_rsp_cmpl));
 
-		len += snprintf(buf + len, size - len,
+		len += scnprintf(buf + len, size - len,
 				"LS: RSP Abort %08x xb %08x Err %08x\n",
 				atomic_read(&tgtp->xmt_ls_rsp_aborted),
 				atomic_read(&tgtp->xmt_ls_rsp_xb_set),
 				atomic_read(&tgtp->xmt_ls_rsp_error));
 
-		len += snprintf(buf + len, size - len,
+		len += scnprintf(buf + len, size - len,
 				"FCP: Rcv %08x Defer %08x Release %08x "
 				"Drop %08x\n",
 				atomic_read(&tgtp->rcv_fcp_cmd_in),
@@ -1087,13 +1089,13 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size)
 
 		if (atomic_read(&tgtp->rcv_fcp_cmd_in) !=
 		    atomic_read(&tgtp->rcv_fcp_cmd_out)) {
-			len += snprintf(buf + len, size - len,
+			len += scnprintf(buf + len, size - len,
 					"Rcv FCP: in %08x != out %08x\n",
 					atomic_read(&tgtp->rcv_fcp_cmd_in),
 					atomic_read(&tgtp->rcv_fcp_cmd_out));
 		}
 
-		len += snprintf(buf + len, size - len,
+		len += scnprintf(buf + len, size - len,
 				"FCP Rsp: read %08x readrsp %08x "
 				"write %08x rsp %08x\n",
 				atomic_read(&tgtp->xmt_fcp_read),
@@ -1101,31 +1103,31 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size)
 				atomic_read(&tgtp->xmt_fcp_write),
 				atomic_read(&tgtp->xmt_fcp_rsp));
 
-		len += snprintf(buf + len, size - len,
+		len += scnprintf(buf + len, size - len,
 				"FCP Rsp Cmpl: %08x err %08x drop %08x\n",
 				atomic_read(&tgtp->xmt_fcp_rsp_cmpl),
 				atomic_read(&tgtp->xmt_fcp_rsp_error),
 				atomic_read(&tgtp->xmt_fcp_rsp_drop));
 
-		len += snprintf(buf + len, size - len,
+		len += scnprintf(buf + len, size - len,
 				"FCP Rsp Abort: %08x xb %08x xricqe  %08x\n",
 				atomic_read(&tgtp->xmt_fcp_rsp_aborted),
 				atomic_read(&tgtp->xmt_fcp_rsp_xb_set),
 				atomic_read(&tgtp->xmt_fcp_xri_abort_cqe));
 
-		len += snprintf(buf + len, size - len,
+		len += scnprintf(buf + len, size - len,
 				"ABORT: Xmt %08x Cmpl %08x\n",
 				atomic_read(&tgtp->xmt_fcp_abort),
 				atomic_read(&tgtp->xmt_fcp_abort_cmpl));
 
-		len += snprintf(buf + len, size - len,
+		len += scnprintf(buf + len, size - len,
 				"ABORT: Sol %08x  Usol %08x Err %08x Cmpl %08x",
 				atomic_read(&tgtp->xmt_abort_sol),
 				atomic_read(&tgtp->xmt_abort_unsol),
 				atomic_read(&tgtp->xmt_abort_rsp),
 				atomic_read(&tgtp->xmt_abort_rsp_error));
 
-		len +=  snprintf(buf + len, size - len, "\n");
+		len +=  scnprintf(buf + len, size - len, "\n");
 
 		cnt = 0;
 		spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock);
@@ -1136,7 +1138,7 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size)
 		}
 		spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock);
 		if (cnt) {
-			len += snprintf(buf + len, size - len,
+			len += scnprintf(buf + len, size - len,
 					"ABORT: %d ctx entries\n", cnt);
 			spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock);
 			list_for_each_entry_safe(ctxp, next_ctxp,
@@ -1144,7 +1146,7 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size)
 				    list) {
 				if (len >= (size - LPFC_DEBUG_OUT_LINE_SZ))
 					break;
-				len += snprintf(buf + len, size - len,
+				len += scnprintf(buf + len, size - len,
 						"Entry: oxid %x state %x "
 						"flag %x\n",
 						ctxp->oxid, ctxp->state,
@@ -1158,7 +1160,7 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size)
 		tot += atomic_read(&tgtp->xmt_fcp_release);
 		tot = atomic_read(&tgtp->rcv_fcp_cmd_in) - tot;
 
-		len += snprintf(buf + len, size - len,
+		len += scnprintf(buf + len, size - len,
 				"IO_CTX: %08x  WAIT: cur %08x tot %08x\n"
 				"CTX Outstanding %08llx\n",
 				phba->sli4_hba.nvmet_xri_cnt,
@@ -1176,10 +1178,10 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size)
 		if (!lport)
 			return len;
 
-		len += snprintf(buf + len, size - len,
+		len += scnprintf(buf + len, size - len,
 				"\nNVME HDWQ Statistics\n");
 
-		len += snprintf(buf + len, size - len,
+		len += scnprintf(buf + len, size - len,
 				"LS: Xmt %016x Cmpl %016x\n",
 				atomic_read(&lport->fc4NvmeLsRequests),
 				atomic_read(&lport->fc4NvmeLsCmpls));
@@ -1199,20 +1201,20 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size)
 			if (i >= 32)
 				continue;
 
-			len += snprintf(buf + len, PAGE_SIZE - len,
+			len += scnprintf(buf + len, PAGE_SIZE - len,
 					"HDWQ (%d): Rd %016llx Wr %016llx "
 					"IO %016llx ",
 					i, data1, data2, data3);
-			len += snprintf(buf + len, PAGE_SIZE - len,
+			len += scnprintf(buf + len, PAGE_SIZE - len,
 					"Cmpl %016llx OutIO %016llx\n",
 					tot, ((data1 + data2 + data3) - tot));
 		}
-		len += snprintf(buf + len, PAGE_SIZE - len,
+		len += scnprintf(buf + len, PAGE_SIZE - len,
 				"Total FCP Cmpl %016llx Issue %016llx "
 				"OutIO %016llx\n",
 				totin, totout, totout - totin);
 
-		len += snprintf(buf + len, size - len,
+		len += scnprintf(buf + len, size - len,
 				"LS Xmt Err: Abrt %08x Err %08x  "
 				"Cmpl Err: xb %08x Err %08x\n",
 				atomic_read(&lport->xmt_ls_abort),
@@ -1220,7 +1222,7 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size)
 				atomic_read(&lport->cmpl_ls_xb),
 				atomic_read(&lport->cmpl_ls_err));
 
-		len += snprintf(buf + len, size - len,
+		len += scnprintf(buf + len, size - len,
 				"FCP Xmt Err: noxri %06x nondlp %06x "
 				"qdepth %06x wqerr %06x err %06x Abrt %06x\n",
 				atomic_read(&lport->xmt_fcp_noxri),
@@ -1230,7 +1232,7 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size)
 				atomic_read(&lport->xmt_fcp_err),
 				atomic_read(&lport->xmt_fcp_abort));
 
-		len += snprintf(buf + len, size - len,
+		len += scnprintf(buf + len, size - len,
 				"FCP Cmpl Err: xb %08x Err %08x\n",
 				atomic_read(&lport->cmpl_fcp_xb),
 				atomic_read(&lport->cmpl_fcp_err));
@@ -1322,58 +1324,58 @@ lpfc_debugfs_nvmektime_data(struct lpfc_vport *vport, char *buf, int size)
 
 	if (phba->nvmet_support == 0) {
 		/* NVME Initiator */
-		len += snprintf(buf + len, PAGE_SIZE - len,
+		len += scnprintf(buf + len, PAGE_SIZE - len,
 				"ktime %s: Total Samples: %lld\n",
 				(phba->ktime_on ?  "Enabled" : "Disabled"),
 				phba->ktime_data_samples);
 		if (phba->ktime_data_samples == 0)
 			return len;
 
-		len += snprintf(
+		len += scnprintf(
 			buf + len, PAGE_SIZE - len,
 			"Segment 1: Last NVME Cmd cmpl "
 			"done -to- Start of next NVME cnd (in driver)\n");
-		len += snprintf(
+		len += scnprintf(
 			buf + len, PAGE_SIZE - len,
 			"avg:%08lld min:%08lld max %08lld\n",
 			div_u64(phba->ktime_seg1_total,
 				phba->ktime_data_samples),
 			phba->ktime_seg1_min,
 			phba->ktime_seg1_max);
-		len += snprintf(
+		len += scnprintf(
 			buf + len, PAGE_SIZE - len,
 			"Segment 2: Driver start of NVME cmd "
 			"-to- Firmware WQ doorbell\n");
-		len += snprintf(
+		len += scnprintf(
 			buf + len, PAGE_SIZE - len,
 			"avg:%08lld min:%08lld max %08lld\n",
 			div_u64(phba->ktime_seg2_total,
 				phba->ktime_data_samples),
 			phba->ktime_seg2_min,
 			phba->ktime_seg2_max);
-		len += snprintf(
+		len += scnprintf(
 			buf + len, PAGE_SIZE - len,
 			"Segment 3: Firmware WQ doorbell -to- "
 			"MSI-X ISR cmpl\n");
-		len += snprintf(
+		len += scnprintf(
 			buf + len, PAGE_SIZE - len,
 			"avg:%08lld min:%08lld max %08lld\n",
 			div_u64(phba->ktime_seg3_total,
 				phba->ktime_data_samples),
 			phba->ktime_seg3_min,
 			phba->ktime_seg3_max);
-		len += snprintf(
+		len += scnprintf(
 			buf + len, PAGE_SIZE - len,
 			"Segment 4: MSI-X ISR cmpl -to- "
 			"NVME cmpl done\n");
-		len += snprintf(
+		len += scnprintf(
 			buf + len, PAGE_SIZE - len,
 			"avg:%08lld min:%08lld max %08lld\n",
 			div_u64(phba->ktime_seg4_total,
 				phba->ktime_data_samples),
 			phba->ktime_seg4_min,
 			phba->ktime_seg4_max);
-		len += snprintf(
+		len += scnprintf(
 			buf + len, PAGE_SIZE - len,
 			"Total IO avg time: %08lld\n",
 			div_u64(phba->ktime_seg1_total +
@@ -1385,7 +1387,7 @@ lpfc_debugfs_nvmektime_data(struct lpfc_vport *vport, char *buf, int size)
 	}
 
 	/* NVME Target */
-	len += snprintf(buf + len, PAGE_SIZE-len,
+	len += scnprintf(buf + len, PAGE_SIZE-len,
 			"ktime %s: Total Samples: %lld %lld\n",
 			(phba->ktime_on ? "Enabled" : "Disabled"),
 			phba->ktime_data_samples,
@@ -1393,46 +1395,46 @@ lpfc_debugfs_nvmektime_data(struct lpfc_vport *vport, char *buf, int size)
 	if (phba->ktime_data_samples == 0)
 		return len;
 
-	len += snprintf(buf + len, PAGE_SIZE-len,
+	len += scnprintf(buf + len, PAGE_SIZE-len,
 			"Segment 1: MSI-X ISR Rcv cmd -to- "
 			"cmd pass to NVME Layer\n");
-	len += snprintf(buf + len, PAGE_SIZE-len,
+	len += scnprintf(buf + len, PAGE_SIZE-len,
 			"avg:%08lld min:%08lld max %08lld\n",
 			div_u64(phba->ktime_seg1_total,
 				phba->ktime_data_samples),
 			phba->ktime_seg1_min,
 			phba->ktime_seg1_max);
-	len += snprintf(buf + len, PAGE_SIZE-len,
+	len += scnprintf(buf + len, PAGE_SIZE-len,
 			"Segment 2: cmd pass to NVME Layer- "
 			"-to- Driver rcv cmd OP (action)\n");
-	len += snprintf(buf + len, PAGE_SIZE-len,
+	len += scnprintf(buf + len, PAGE_SIZE-len,
 			"avg:%08lld min:%08lld max %08lld\n",
 			div_u64(phba->ktime_seg2_total,
 				phba->ktime_data_samples),
 			phba->ktime_seg2_min,
 			phba->ktime_seg2_max);
-	len += snprintf(buf + len, PAGE_SIZE-len,
+	len += scnprintf(buf + len, PAGE_SIZE-len,
 			"Segment 3: Driver rcv cmd OP -to- "
 			"Firmware WQ doorbell: cmd\n");
-	len += snprintf(buf + len, PAGE_SIZE-len,
+	len += scnprintf(buf + len, PAGE_SIZE-len,
 			"avg:%08lld min:%08lld max %08lld\n",
 			div_u64(phba->ktime_seg3_total,
 				phba->ktime_data_samples),
 			phba->ktime_seg3_min,
 			phba->ktime_seg3_max);
-	len += snprintf(buf + len, PAGE_SIZE-len,
+	len += scnprintf(buf + len, PAGE_SIZE-len,
 			"Segment 4: Firmware WQ doorbell: cmd "
 			"-to- MSI-X ISR for cmd cmpl\n");
-	len += snprintf(buf + len, PAGE_SIZE-len,
+	len += scnprintf(buf + len, PAGE_SIZE-len,
 			"avg:%08lld min:%08lld max %08lld\n",
 			div_u64(phba->ktime_seg4_total,
 				phba->ktime_data_samples),
 			phba->ktime_seg4_min,
 			phba->ktime_seg4_max);
-	len += snprintf(buf + len, PAGE_SIZE-len,
+	len += scnprintf(buf + len, PAGE_SIZE-len,
 			"Segment 5: MSI-X ISR for cmd cmpl "
 			"-to- NVME layer passed cmd done\n");
-	len += snprintf(buf + len, PAGE_SIZE-len,
+	len += scnprintf(buf + len, PAGE_SIZE-len,
 			"avg:%08lld min:%08lld max %08lld\n",
 			div_u64(phba->ktime_seg5_total,
 				phba->ktime_data_samples),
@@ -1440,10 +1442,10 @@ lpfc_debugfs_nvmektime_data(struct lpfc_vport *vport, char *buf, int size)
 			phba->ktime_seg5_max);
 
 	if (phba->ktime_status_samples == 0) {
-		len += snprintf(buf + len, PAGE_SIZE-len,
+		len += scnprintf(buf + len, PAGE_SIZE-len,
 				"Total: cmd received by MSI-X ISR "
 				"-to- cmd completed on wire\n");
-		len += snprintf(buf + len, PAGE_SIZE-len,
+		len += scnprintf(buf + len, PAGE_SIZE-len,
 				"avg:%08lld min:%08lld "
 				"max %08lld\n",
 				div_u64(phba->ktime_seg10_total,
@@ -1453,46 +1455,46 @@ lpfc_debugfs_nvmektime_data(struct lpfc_vport *vport, char *buf, int size)
 		return len;
 	}
 
-	len += snprintf(buf + len, PAGE_SIZE-len,
+	len += scnprintf(buf + len, PAGE_SIZE-len,
 			"Segment 6: NVME layer passed cmd done "
 			"-to- Driver rcv rsp status OP\n");
-	len += snprintf(buf + len, PAGE_SIZE-len,
+	len += scnprintf(buf + len, PAGE_SIZE-len,
 			"avg:%08lld min:%08lld max %08lld\n",
 			div_u64(phba->ktime_seg6_total,
 				phba->ktime_status_samples),
 			phba->ktime_seg6_min,
 			phba->ktime_seg6_max);
-	len += snprintf(buf + len, PAGE_SIZE-len,
+	len += scnprintf(buf + len, PAGE_SIZE-len,
 			"Segment 7: Driver rcv rsp status OP "
 			"-to- Firmware WQ doorbell: status\n");
-	len += snprintf(buf + len, PAGE_SIZE-len,
+	len += scnprintf(buf + len, PAGE_SIZE-len,
 			"avg:%08lld min:%08lld max %08lld\n",
 			div_u64(phba->ktime_seg7_total,
 				phba->ktime_status_samples),
 			phba->ktime_seg7_min,
 			phba->ktime_seg7_max);
-	len += snprintf(buf + len, PAGE_SIZE-len,
+	len += scnprintf(buf + len, PAGE_SIZE-len,
 			"Segment 8: Firmware WQ doorbell: status"
 			" -to- MSI-X ISR for status cmpl\n");
-	len += snprintf(buf + len, PAGE_SIZE-len,
+	len += scnprintf(buf + len, PAGE_SIZE-len,
 			"avg:%08lld min:%08lld max %08lld\n",
 			div_u64(phba->ktime_seg8_total,
 				phba->ktime_status_samples),
 			phba->ktime_seg8_min,
 			phba->ktime_seg8_max);
-	len += snprintf(buf + len, PAGE_SIZE-len,
+	len += scnprintf(buf + len, PAGE_SIZE-len,
 			"Segment 9: MSI-X ISR for status cmpl  "
 			"-to- NVME layer passed status done\n");
-	len += snprintf(buf + len, PAGE_SIZE-len,
+	len += scnprintf(buf + len, PAGE_SIZE-len,
 			"avg:%08lld min:%08lld max %08lld\n",
 			div_u64(phba->ktime_seg9_total,
 				phba->ktime_status_samples),
 			phba->ktime_seg9_min,
 			phba->ktime_seg9_max);
-	len += snprintf(buf + len, PAGE_SIZE-len,
+	len += scnprintf(buf + len, PAGE_SIZE-len,
 			"Total: cmd received by MSI-X ISR -to- "
 			"cmd completed on wire\n");
-	len += snprintf(buf + len, PAGE_SIZE-len,
+	len += scnprintf(buf + len, PAGE_SIZE-len,
 			"avg:%08lld min:%08lld max %08lld\n",
 			div_u64(phba->ktime_seg10_total,
 				phba->ktime_status_samples),
@@ -1527,7 +1529,7 @@ lpfc_debugfs_nvmeio_trc_data(struct lpfc_hba *phba, char *buf, int size)
 		(phba->nvmeio_trc_size - 1);
 	skip = phba->nvmeio_trc_output_idx;
 
-	len += snprintf(buf + len, size - len,
+	len += scnprintf(buf + len, size - len,
 			"%s IO Trace %s: next_idx %d skip %d size %d\n",
 			(phba->nvmet_support ? "NVME" : "NVMET"),
 			(state ? "Enabled" : "Disabled"),
@@ -1549,18 +1551,18 @@ lpfc_debugfs_nvmeio_trc_data(struct lpfc_hba *phba, char *buf, int size)
 		if (!dtp->fmt)
 			continue;
 
-		len +=  snprintf(buf + len, size - len, dtp->fmt,
+		len +=  scnprintf(buf + len, size - len, dtp->fmt,
 			dtp->data1, dtp->data2, dtp->data3);
 
 		if (phba->nvmeio_trc_output_idx >= phba->nvmeio_trc_size) {
 			phba->nvmeio_trc_output_idx = 0;
-			len += snprintf(buf + len, size - len,
+			len += scnprintf(buf + len, size - len,
 					"Trace Complete\n");
 			goto out;
 		}
 
 		if (len >= (size - LPFC_DEBUG_OUT_LINE_SZ)) {
-			len += snprintf(buf + len, size - len,
+			len += scnprintf(buf + len, size - len,
 					"Trace Continue (%d of %d)\n",
 					phba->nvmeio_trc_output_idx,
 					phba->nvmeio_trc_size);
@@ -1578,18 +1580,18 @@ lpfc_debugfs_nvmeio_trc_data(struct lpfc_hba *phba, char *buf, int size)
 		if (!dtp->fmt)
 			continue;
 
-		len +=  snprintf(buf + len, size - len, dtp->fmt,
+		len +=  scnprintf(buf + len, size - len, dtp->fmt,
 			dtp->data1, dtp->data2, dtp->data3);
 
 		if (phba->nvmeio_trc_output_idx >= phba->nvmeio_trc_size) {
 			phba->nvmeio_trc_output_idx = 0;
-			len += snprintf(buf + len, size - len,
+			len += scnprintf(buf + len, size - len,
 					"Trace Complete\n");
 			goto out;
 		}
 
 		if (len >= (size - LPFC_DEBUG_OUT_LINE_SZ)) {
-			len += snprintf(buf + len, size - len,
+			len += scnprintf(buf + len, size - len,
 					"Trace Continue (%d of %d)\n",
 					phba->nvmeio_trc_output_idx,
 					phba->nvmeio_trc_size);
@@ -1597,7 +1599,7 @@ lpfc_debugfs_nvmeio_trc_data(struct lpfc_hba *phba, char *buf, int size)
 		}
 	}
 
-	len += snprintf(buf + len, size - len,
+	len += scnprintf(buf + len, size - len,
 			"Trace Done\n");
 out:
 	return len;
@@ -1627,17 +1629,17 @@ lpfc_debugfs_cpucheck_data(struct lpfc_vport *vport, char *buf, int size)
 	uint32_t tot_rcv;
 	uint32_t tot_cmpl;
 
-	len += snprintf(buf + len, PAGE_SIZE - len,
+	len += scnprintf(buf + len, PAGE_SIZE - len,
 			"CPUcheck %s ",
 			(phba->cpucheck_on & LPFC_CHECK_NVME_IO ?
 				"Enabled" : "Disabled"));
 	if (phba->nvmet_support) {
-		len += snprintf(buf + len, PAGE_SIZE - len,
+		len += scnprintf(buf + len, PAGE_SIZE - len,
 				"%s\n",
 				(phba->cpucheck_on & LPFC_CHECK_NVMET_RCV ?
 					"Rcv Enabled\n" : "Rcv Disabled\n"));
 	} else {
-		len += snprintf(buf + len, PAGE_SIZE - len, "\n");
+		len += scnprintf(buf + len, PAGE_SIZE - len, "\n");
 	}
 	max_cnt = size - LPFC_DEBUG_OUT_LINE_SZ;
 
@@ -1658,7 +1660,7 @@ lpfc_debugfs_cpucheck_data(struct lpfc_vport *vport, char *buf, int size)
 		if (!tot_xmt && !tot_cmpl && !tot_rcv)
 			continue;
 
-		len += snprintf(buf + len, PAGE_SIZE - len,
+		len += scnprintf(buf + len, PAGE_SIZE - len,
 				"HDWQ %03d: ", i);
 		for (j = 0; j < LPFC_CHECK_CPU_CNT; j++) {
 			/* Only display non-zero counters */
@@ -1667,22 +1669,22 @@ lpfc_debugfs_cpucheck_data(struct lpfc_vport *vport, char *buf, int size)
 			    !qp->cpucheck_rcv_io[j])
 				continue;
 			if (phba->nvmet_support) {
-				len += snprintf(buf + len, PAGE_SIZE - len,
+				len += scnprintf(buf + len, PAGE_SIZE - len,
 						"CPU %03d: %x/%x/%x ", j,
 						qp->cpucheck_rcv_io[j],
 						qp->cpucheck_xmt_io[j],
 						qp->cpucheck_cmpl_io[j]);
 			} else {
-				len += snprintf(buf + len, PAGE_SIZE - len,
+				len += scnprintf(buf + len, PAGE_SIZE - len,
 						"CPU %03d: %x/%x ", j,
 						qp->cpucheck_xmt_io[j],
 						qp->cpucheck_cmpl_io[j]);
 			}
 		}
-		len += snprintf(buf + len, PAGE_SIZE - len,
+		len += scnprintf(buf + len, PAGE_SIZE - len,
 				"Total: %x\n", tot_xmt);
 		if (len >= max_cnt) {
-			len += snprintf(buf + len, PAGE_SIZE - len,
+			len += scnprintf(buf + len, PAGE_SIZE - len,
 					"Truncated ...\n");
 			return len;
 		}
@@ -2258,28 +2260,29 @@ lpfc_debugfs_dif_err_read(struct file *file, char __user *buf,
 	int cnt = 0;
 
 	if (dent == phba->debug_writeGuard)
-		cnt = snprintf(cbuf, 32, "%u\n", phba->lpfc_injerr_wgrd_cnt);
+		cnt = scnprintf(cbuf, 32, "%u\n", phba->lpfc_injerr_wgrd_cnt);
 	else if (dent == phba->debug_writeApp)
-		cnt = snprintf(cbuf, 32, "%u\n", phba->lpfc_injerr_wapp_cnt);
+		cnt = scnprintf(cbuf, 32, "%u\n", phba->lpfc_injerr_wapp_cnt);
 	else if (dent == phba->debug_writeRef)
-		cnt = snprintf(cbuf, 32, "%u\n", phba->lpfc_injerr_wref_cnt);
+		cnt = scnprintf(cbuf, 32, "%u\n", phba->lpfc_injerr_wref_cnt);
 	else if (dent == phba->debug_readGuard)
-		cnt = snprintf(cbuf, 32, "%u\n", phba->lpfc_injerr_rgrd_cnt);
+		cnt = scnprintf(cbuf, 32, "%u\n", phba->lpfc_injerr_rgrd_cnt);
 	else if (dent == phba->debug_readApp)
-		cnt = snprintf(cbuf, 32, "%u\n", phba->lpfc_injerr_rapp_cnt);
+		cnt = scnprintf(cbuf, 32, "%u\n", phba->lpfc_injerr_rapp_cnt);
 	else if (dent == phba->debug_readRef)
-		cnt = snprintf(cbuf, 32, "%u\n", phba->lpfc_injerr_rref_cnt);
+		cnt = scnprintf(cbuf, 32, "%u\n", phba->lpfc_injerr_rref_cnt);
 	else if (dent == phba->debug_InjErrNPortID)
-		cnt = snprintf(cbuf, 32, "0x%06x\n", phba->lpfc_injerr_nportid);
+		cnt = scnprintf(cbuf, 32, "0x%06x\n",
+				phba->lpfc_injerr_nportid);
 	else if (dent == phba->debug_InjErrWWPN) {
 		memcpy(&tmp, &phba->lpfc_injerr_wwpn, sizeof(struct lpfc_name));
 		tmp = cpu_to_be64(tmp);
-		cnt = snprintf(cbuf, 32, "0x%016llx\n", tmp);
+		cnt = scnprintf(cbuf, 32, "0x%016llx\n", tmp);
 	} else if (dent == phba->debug_InjErrLBA) {
 		if (phba->lpfc_injerr_lba == (sector_t)(-1))
-			cnt = snprintf(cbuf, 32, "off\n");
+			cnt = scnprintf(cbuf, 32, "off\n");
 		else
-			cnt = snprintf(cbuf, 32, "0x%llx\n",
+			cnt = scnprintf(cbuf, 32, "0x%llx\n",
 				 (uint64_t) phba->lpfc_injerr_lba);
 	} else
 		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
@@ -3224,17 +3227,17 @@ lpfc_idiag_pcicfg_read(struct file *file, char __user *buf, size_t nbytes,
 	switch (count) {
 	case SIZE_U8: /* byte (8 bits) */
 		pci_read_config_byte(pdev, where, &u8val);
-		len += snprintf(pbuffer+len, LPFC_PCI_CFG_SIZE-len,
+		len += scnprintf(pbuffer+len, LPFC_PCI_CFG_SIZE-len,
 				"%03x: %02x\n", where, u8val);
 		break;
 	case SIZE_U16: /* word (16 bits) */
 		pci_read_config_word(pdev, where, &u16val);
-		len += snprintf(pbuffer+len, LPFC_PCI_CFG_SIZE-len,
+		len += scnprintf(pbuffer+len, LPFC_PCI_CFG_SIZE-len,
 				"%03x: %04x\n", where, u16val);
 		break;
 	case SIZE_U32: /* double word (32 bits) */
 		pci_read_config_dword(pdev, where, &u32val);
-		len += snprintf(pbuffer+len, LPFC_PCI_CFG_SIZE-len,
+		len += scnprintf(pbuffer+len, LPFC_PCI_CFG_SIZE-len,
 				"%03x: %08x\n", where, u32val);
 		break;
 	case LPFC_PCI_CFG_BROWSE: /* browse all */
@@ -3254,25 +3257,25 @@ pcicfg_browse:
 	offset = offset_label;
 
 	/* Read PCI config space */
-	len += snprintf(pbuffer+len, LPFC_PCI_CFG_SIZE-len,
+	len += scnprintf(pbuffer+len, LPFC_PCI_CFG_SIZE-len,
 			"%03x: ", offset_label);
 	while (index > 0) {
 		pci_read_config_dword(pdev, offset, &u32val);
-		len += snprintf(pbuffer+len, LPFC_PCI_CFG_SIZE-len,
+		len += scnprintf(pbuffer+len, LPFC_PCI_CFG_SIZE-len,
 				"%08x ", u32val);
 		offset += sizeof(uint32_t);
 		if (offset >= LPFC_PCI_CFG_SIZE) {
-			len += snprintf(pbuffer+len,
+			len += scnprintf(pbuffer+len,
 					LPFC_PCI_CFG_SIZE-len, "\n");
 			break;
 		}
 		index -= sizeof(uint32_t);
 		if (!index)
-			len += snprintf(pbuffer+len, LPFC_PCI_CFG_SIZE-len,
+			len += scnprintf(pbuffer+len, LPFC_PCI_CFG_SIZE-len,
 					"\n");
 		else if (!(index % (8 * sizeof(uint32_t)))) {
 			offset_label += (8 * sizeof(uint32_t));
-			len += snprintf(pbuffer+len, LPFC_PCI_CFG_SIZE-len,
+			len += scnprintf(pbuffer+len, LPFC_PCI_CFG_SIZE-len,
 					"\n%03x: ", offset_label);
 		}
 	}
@@ -3543,7 +3546,7 @@ lpfc_idiag_baracc_read(struct file *file, char __user *buf, size_t nbytes,
 	if (acc_range == SINGLE_WORD) {
 		offset_run = offset;
 		u32val = readl(mem_mapped_bar + offset_run);
-		len += snprintf(pbuffer+len, LPFC_PCI_BAR_RD_BUF_SIZE-len,
+		len += scnprintf(pbuffer+len, LPFC_PCI_BAR_RD_BUF_SIZE-len,
 				"%05x: %08x\n", offset_run, u32val);
 	} else
 		goto baracc_browse;
@@ -3557,35 +3560,35 @@ baracc_browse:
 	offset_run = offset_label;
 
 	/* Read PCI bar memory mapped space */
-	len += snprintf(pbuffer+len, LPFC_PCI_BAR_RD_BUF_SIZE-len,
+	len += scnprintf(pbuffer+len, LPFC_PCI_BAR_RD_BUF_SIZE-len,
 			"%05x: ", offset_label);
 	index = LPFC_PCI_BAR_RD_SIZE;
 	while (index > 0) {
 		u32val = readl(mem_mapped_bar + offset_run);
-		len += snprintf(pbuffer+len, LPFC_PCI_BAR_RD_BUF_SIZE-len,
+		len += scnprintf(pbuffer+len, LPFC_PCI_BAR_RD_BUF_SIZE-len,
 				"%08x ", u32val);
 		offset_run += sizeof(uint32_t);
 		if (acc_range == LPFC_PCI_BAR_BROWSE) {
 			if (offset_run >= bar_size) {
-				len += snprintf(pbuffer+len,
+				len += scnprintf(pbuffer+len,
 					LPFC_PCI_BAR_RD_BUF_SIZE-len, "\n");
 				break;
 			}
 		} else {
 			if (offset_run >= offset +
 			    (acc_range * sizeof(uint32_t))) {
-				len += snprintf(pbuffer+len,
+				len += scnprintf(pbuffer+len,
 					LPFC_PCI_BAR_RD_BUF_SIZE-len, "\n");
 				break;
 			}
 		}
 		index -= sizeof(uint32_t);
 		if (!index)
-			len += snprintf(pbuffer+len,
+			len += scnprintf(pbuffer+len,
 					LPFC_PCI_BAR_RD_BUF_SIZE-len, "\n");
 		else if (!(index % (8 * sizeof(uint32_t)))) {
 			offset_label += (8 * sizeof(uint32_t));
-			len += snprintf(pbuffer+len,
+			len += scnprintf(pbuffer+len,
 					LPFC_PCI_BAR_RD_BUF_SIZE-len,
 					"\n%05x: ", offset_label);
 		}
@@ -3758,19 +3761,19 @@ __lpfc_idiag_print_wq(struct lpfc_queue *qp, char *wqtype,
 	if (!qp)
 		return len;
 
-	len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
+	len += scnprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
 			"\t\t%s WQ info: ", wqtype);
-	len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
+	len += scnprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
 			"AssocCQID[%04d]: WQ-STAT[oflow:x%x posted:x%llx]\n",
 			qp->assoc_qid, qp->q_cnt_1,
 			(unsigned long long)qp->q_cnt_4);
-	len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
+	len += scnprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
 			"\t\tWQID[%02d], QE-CNT[%04d], QE-SZ[%04d], "
 			"HST-IDX[%04d], PRT-IDX[%04d], NTFI[%03d]",
 			qp->queue_id, qp->entry_count,
 			qp->entry_size, qp->host_index,
 			qp->hba_index, qp->notify_interval);
-	len +=  snprintf(pbuffer + len,
+	len +=  scnprintf(pbuffer + len,
 			LPFC_QUE_INFO_GET_BUF_SIZE - len, "\n");
 	return len;
 }
@@ -3810,21 +3813,22 @@ __lpfc_idiag_print_cq(struct lpfc_queue *qp, char *cqtype,
 	if (!qp)
 		return len;
 
-	len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
+	len += scnprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
 			"\t%s CQ info: ", cqtype);
-	len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
+	len += scnprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
 			"AssocEQID[%02d]: CQ STAT[max:x%x relw:x%x "
 			"xabt:x%x wq:x%llx]\n",
 			qp->assoc_qid, qp->q_cnt_1, qp->q_cnt_2,
 			qp->q_cnt_3, (unsigned long long)qp->q_cnt_4);
-	len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
+	len += scnprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
 			"\tCQID[%02d], QE-CNT[%04d], QE-SZ[%04d], "
 			"HST-IDX[%04d], NTFI[%03d], PLMT[%03d]",
 			qp->queue_id, qp->entry_count,
 			qp->entry_size, qp->host_index,
 			qp->notify_interval, qp->max_proc_limit);
 
-	len +=  snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len, "\n");
+	len +=  scnprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
+			"\n");
 
 	return len;
 }
@@ -3836,19 +3840,19 @@ __lpfc_idiag_print_rqpair(struct lpfc_queue *qp, struct lpfc_queue *datqp,
 	if (!qp || !datqp)
 		return len;
 
-	len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
+	len += scnprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
 			"\t\t%s RQ info: ", rqtype);
-	len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
+	len += scnprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
 			"AssocCQID[%02d]: RQ-STAT[nopost:x%x nobuf:x%x "
 			"posted:x%x rcv:x%llx]\n",
 			qp->assoc_qid, qp->q_cnt_1, qp->q_cnt_2,
 			qp->q_cnt_3, (unsigned long long)qp->q_cnt_4);
-	len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
+	len += scnprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
 			"\t\tHQID[%02d], QE-CNT[%04d], QE-SZ[%04d], "
 			"HST-IDX[%04d], PRT-IDX[%04d], NTFI[%03d]\n",
 			qp->queue_id, qp->entry_count, qp->entry_size,
 			qp->host_index, qp->hba_index, qp->notify_interval);
-	len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
+	len += scnprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
 			"\t\tDQID[%02d], QE-CNT[%04d], QE-SZ[%04d], "
 			"HST-IDX[%04d], PRT-IDX[%04d], NTFI[%03d]\n",
 			datqp->queue_id, datqp->entry_count,
@@ -3927,18 +3931,19 @@ __lpfc_idiag_print_eq(struct lpfc_queue *qp, char *eqtype,
 	if (!qp)
 		return len;
 
-	len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
+	len += scnprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
 			"\n%s EQ info: EQ-STAT[max:x%x noE:x%x "
 			"cqe_proc:x%x eqe_proc:x%llx eqd %d]\n",
 			eqtype, qp->q_cnt_1, qp->q_cnt_2, qp->q_cnt_3,
 			(unsigned long long)qp->q_cnt_4, qp->q_mode);
-	len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
+	len += scnprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
 			"EQID[%02d], QE-CNT[%04d], QE-SZ[%04d], "
 			"HST-IDX[%04d], NTFI[%03d], PLMT[%03d], AFFIN[%03d]",
 			qp->queue_id, qp->entry_count, qp->entry_size,
 			qp->host_index, qp->notify_interval,
 			qp->max_proc_limit, qp->chann);
-	len +=  snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len, "\n");
+	len +=  scnprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
+			"\n");
 
 	return len;
 }
@@ -3991,9 +3996,10 @@ lpfc_idiag_queinfo_read(struct file *file, char __user *buf, size_t nbytes,
 		if (phba->lpfc_idiag_last_eq >= phba->cfg_hdw_queue)
 			phba->lpfc_idiag_last_eq = 0;
 
-		len += snprintf(pbuffer + len, LPFC_QUE_INFO_GET_BUF_SIZE - len,
-					"HDWQ %d out of %d HBA HDWQs\n",
-					x, phba->cfg_hdw_queue);
+		len += scnprintf(pbuffer + len,
+				 LPFC_QUE_INFO_GET_BUF_SIZE - len,
+				 "HDWQ %d out of %d HBA HDWQs\n",
+				 x, phba->cfg_hdw_queue);
 
 		/* Fast-path EQ */
 		qp = phba->sli4_hba.hdwq[x].hba_eq;
@@ -4075,7 +4081,7 @@ lpfc_idiag_queinfo_read(struct file *file, char __user *buf, size_t nbytes,
 	return simple_read_from_buffer(buf, nbytes, ppos, pbuffer, len);
 
 too_big:
-	len +=  snprintf(pbuffer + len,
+	len +=  scnprintf(pbuffer + len,
 		LPFC_QUE_INFO_GET_BUF_SIZE - len, "Truncated ...\n");
 out:
 	spin_unlock_irq(&phba->hbalock);
@@ -4131,22 +4137,22 @@ lpfc_idiag_queacc_read_qe(char *pbuffer, int len, struct lpfc_queue *pque,
 		return 0;
 
 	esize = pque->entry_size;
-	len += snprintf(pbuffer+len, LPFC_QUE_ACC_BUF_SIZE-len,
+	len += scnprintf(pbuffer+len, LPFC_QUE_ACC_BUF_SIZE-len,
 			"QE-INDEX[%04d]:\n", index);
 
 	offset = 0;
-	pentry = pque->qe[index].address;
+	pentry = lpfc_sli4_qe(pque, index);
 	while (esize > 0) {
-		len += snprintf(pbuffer+len, LPFC_QUE_ACC_BUF_SIZE-len,
+		len += scnprintf(pbuffer+len, LPFC_QUE_ACC_BUF_SIZE-len,
 				"%08x ", *pentry);
 		pentry++;
 		offset += sizeof(uint32_t);
 		esize -= sizeof(uint32_t);
 		if (esize > 0 && !(offset % (4 * sizeof(uint32_t))))
-			len += snprintf(pbuffer+len,
+			len += scnprintf(pbuffer+len,
 					LPFC_QUE_ACC_BUF_SIZE-len, "\n");
 	}
-	len += snprintf(pbuffer+len, LPFC_QUE_ACC_BUF_SIZE-len, "\n");
+	len += scnprintf(pbuffer+len, LPFC_QUE_ACC_BUF_SIZE-len, "\n");
 
 	return len;
 }
@@ -4485,7 +4491,7 @@ pass_check:
 		pque = (struct lpfc_queue *)idiag.ptr_private;
 		if (offset > pque->entry_size/sizeof(uint32_t) - 1)
 			goto error_out;
-		pentry = pque->qe[index].address;
+		pentry = lpfc_sli4_qe(pque, index);
 		pentry += offset;
 		if (idiag.cmd.opcode == LPFC_IDIAG_CMD_QUEACC_WR)
 			*pentry = value;
@@ -4506,7 +4512,7 @@ error_out:
  * lpfc_idiag_drbacc_read_reg - idiag debugfs read a doorbell register
  * @phba: The pointer to hba structure.
  * @pbuffer: The pointer to the buffer to copy the data to.
- * @len: The lenght of bytes to copied.
+ * @len: The length of bytes to copied.
  * @drbregid: The id to doorbell registers.
  *
  * Description:
@@ -4526,27 +4532,27 @@ lpfc_idiag_drbacc_read_reg(struct lpfc_hba *phba, char *pbuffer,
 
 	switch (drbregid) {
 	case LPFC_DRB_EQ:
-		len += snprintf(pbuffer + len, LPFC_DRB_ACC_BUF_SIZE-len,
+		len += scnprintf(pbuffer + len, LPFC_DRB_ACC_BUF_SIZE-len,
 				"EQ-DRB-REG: 0x%08x\n",
 				readl(phba->sli4_hba.EQDBregaddr));
 		break;
 	case LPFC_DRB_CQ:
-		len += snprintf(pbuffer + len, LPFC_DRB_ACC_BUF_SIZE - len,
+		len += scnprintf(pbuffer + len, LPFC_DRB_ACC_BUF_SIZE - len,
 				"CQ-DRB-REG: 0x%08x\n",
 				readl(phba->sli4_hba.CQDBregaddr));
 		break;
 	case LPFC_DRB_MQ:
-		len += snprintf(pbuffer+len, LPFC_DRB_ACC_BUF_SIZE-len,
+		len += scnprintf(pbuffer+len, LPFC_DRB_ACC_BUF_SIZE-len,
 				"MQ-DRB-REG:   0x%08x\n",
 				readl(phba->sli4_hba.MQDBregaddr));
 		break;
 	case LPFC_DRB_WQ:
-		len += snprintf(pbuffer+len, LPFC_DRB_ACC_BUF_SIZE-len,
+		len += scnprintf(pbuffer+len, LPFC_DRB_ACC_BUF_SIZE-len,
 				"WQ-DRB-REG:   0x%08x\n",
 				readl(phba->sli4_hba.WQDBregaddr));
 		break;
 	case LPFC_DRB_RQ:
-		len += snprintf(pbuffer+len, LPFC_DRB_ACC_BUF_SIZE-len,
+		len += scnprintf(pbuffer+len, LPFC_DRB_ACC_BUF_SIZE-len,
 				"RQ-DRB-REG:   0x%08x\n",
 				readl(phba->sli4_hba.RQDBregaddr));
 		break;
@@ -4716,7 +4722,7 @@ error_out:
  * lpfc_idiag_ctlacc_read_reg - idiag debugfs read a control registers
  * @phba: The pointer to hba structure.
  * @pbuffer: The pointer to the buffer to copy the data to.
- * @len: The lenght of bytes to copied.
+ * @len: The length of bytes to copied.
  * @drbregid: The id to doorbell registers.
  *
  * Description:
@@ -4736,37 +4742,37 @@ lpfc_idiag_ctlacc_read_reg(struct lpfc_hba *phba, char *pbuffer,
 
 	switch (ctlregid) {
 	case LPFC_CTL_PORT_SEM:
-		len += snprintf(pbuffer+len, LPFC_CTL_ACC_BUF_SIZE-len,
+		len += scnprintf(pbuffer+len, LPFC_CTL_ACC_BUF_SIZE-len,
 				"Port SemReg:   0x%08x\n",
 				readl(phba->sli4_hba.conf_regs_memmap_p +
 				      LPFC_CTL_PORT_SEM_OFFSET));
 		break;
 	case LPFC_CTL_PORT_STA:
-		len += snprintf(pbuffer+len, LPFC_CTL_ACC_BUF_SIZE-len,
+		len += scnprintf(pbuffer+len, LPFC_CTL_ACC_BUF_SIZE-len,
 				"Port StaReg:   0x%08x\n",
 				readl(phba->sli4_hba.conf_regs_memmap_p +
 				      LPFC_CTL_PORT_STA_OFFSET));
 		break;
 	case LPFC_CTL_PORT_CTL:
-		len += snprintf(pbuffer+len, LPFC_CTL_ACC_BUF_SIZE-len,
+		len += scnprintf(pbuffer+len, LPFC_CTL_ACC_BUF_SIZE-len,
 				"Port CtlReg:   0x%08x\n",
 				readl(phba->sli4_hba.conf_regs_memmap_p +
 				      LPFC_CTL_PORT_CTL_OFFSET));
 		break;
 	case LPFC_CTL_PORT_ER1:
-		len += snprintf(pbuffer+len, LPFC_CTL_ACC_BUF_SIZE-len,
+		len += scnprintf(pbuffer+len, LPFC_CTL_ACC_BUF_SIZE-len,
 				"Port Er1Reg:   0x%08x\n",
 				readl(phba->sli4_hba.conf_regs_memmap_p +
 				      LPFC_CTL_PORT_ER1_OFFSET));
 		break;
 	case LPFC_CTL_PORT_ER2:
-		len += snprintf(pbuffer+len, LPFC_CTL_ACC_BUF_SIZE-len,
+		len += scnprintf(pbuffer+len, LPFC_CTL_ACC_BUF_SIZE-len,
 				"Port Er2Reg:   0x%08x\n",
 				readl(phba->sli4_hba.conf_regs_memmap_p +
 				      LPFC_CTL_PORT_ER2_OFFSET));
 		break;
 	case LPFC_CTL_PDEV_CTL:
-		len += snprintf(pbuffer+len, LPFC_CTL_ACC_BUF_SIZE-len,
+		len += scnprintf(pbuffer+len, LPFC_CTL_ACC_BUF_SIZE-len,
 				"PDev CtlReg:   0x%08x\n",
 				readl(phba->sli4_hba.conf_regs_memmap_p +
 				      LPFC_CTL_PDEV_CTL_OFFSET));
@@ -4959,13 +4965,13 @@ lpfc_idiag_mbxacc_get_setup(struct lpfc_hba *phba, char *pbuffer)
 	mbx_dump_cnt = idiag.cmd.data[IDIAG_MBXACC_DPCNT_INDX];
 	mbx_word_cnt = idiag.cmd.data[IDIAG_MBXACC_WDCNT_INDX];
 
-	len += snprintf(pbuffer+len, LPFC_MBX_ACC_BUF_SIZE-len,
+	len += scnprintf(pbuffer+len, LPFC_MBX_ACC_BUF_SIZE-len,
 			"mbx_dump_map: 0x%08x\n", mbx_dump_map);
-	len += snprintf(pbuffer+len, LPFC_MBX_ACC_BUF_SIZE-len,
+	len += scnprintf(pbuffer+len, LPFC_MBX_ACC_BUF_SIZE-len,
 			"mbx_dump_cnt: %04d\n", mbx_dump_cnt);
-	len += snprintf(pbuffer+len, LPFC_MBX_ACC_BUF_SIZE-len,
+	len += scnprintf(pbuffer+len, LPFC_MBX_ACC_BUF_SIZE-len,
 			"mbx_word_cnt: %04d\n", mbx_word_cnt);
-	len += snprintf(pbuffer+len, LPFC_MBX_ACC_BUF_SIZE-len,
+	len += scnprintf(pbuffer+len, LPFC_MBX_ACC_BUF_SIZE-len,
 			"mbx_mbox_cmd: 0x%02x\n", mbx_mbox_cmd);
 
 	return len;
@@ -5114,35 +5120,35 @@ lpfc_idiag_extacc_avail_get(struct lpfc_hba *phba, char *pbuffer, int len)
 {
 	uint16_t ext_cnt, ext_size;
 
-	len += snprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
+	len += scnprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
 			"\nAvailable Extents Information:\n");
 
-	len += snprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
+	len += scnprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
 			"\tPort Available VPI extents: ");
 	lpfc_sli4_get_avail_extnt_rsrc(phba, LPFC_RSC_TYPE_FCOE_VPI,
 				       &ext_cnt, &ext_size);
-	len += snprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
+	len += scnprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
 			"Count %3d, Size %3d\n", ext_cnt, ext_size);
 
-	len += snprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
+	len += scnprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
 			"\tPort Available VFI extents: ");
 	lpfc_sli4_get_avail_extnt_rsrc(phba, LPFC_RSC_TYPE_FCOE_VFI,
 				       &ext_cnt, &ext_size);
-	len += snprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
+	len += scnprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
 			"Count %3d, Size %3d\n", ext_cnt, ext_size);
 
-	len += snprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
+	len += scnprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
 			"\tPort Available RPI extents: ");
 	lpfc_sli4_get_avail_extnt_rsrc(phba, LPFC_RSC_TYPE_FCOE_RPI,
 				       &ext_cnt, &ext_size);
-	len += snprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
+	len += scnprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
 			"Count %3d, Size %3d\n", ext_cnt, ext_size);
 
-	len += snprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
+	len += scnprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
 			"\tPort Available XRI extents: ");
 	lpfc_sli4_get_avail_extnt_rsrc(phba, LPFC_RSC_TYPE_FCOE_XRI,
 				       &ext_cnt, &ext_size);
-	len += snprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
+	len += scnprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
 			"Count %3d, Size %3d\n", ext_cnt, ext_size);
 
 	return len;
@@ -5166,55 +5172,55 @@ lpfc_idiag_extacc_alloc_get(struct lpfc_hba *phba, char *pbuffer, int len)
 	uint16_t ext_cnt, ext_size;
 	int rc;
 
-	len += snprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
+	len += scnprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
 			"\nAllocated Extents Information:\n");
 
-	len += snprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
+	len += scnprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
 			"\tHost Allocated VPI extents: ");
 	rc = lpfc_sli4_get_allocated_extnts(phba, LPFC_RSC_TYPE_FCOE_VPI,
 					    &ext_cnt, &ext_size);
 	if (!rc)
-		len += snprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
+		len += scnprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
 				"Port %d Extent %3d, Size %3d\n",
 				phba->brd_no, ext_cnt, ext_size);
 	else
-		len += snprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
+		len += scnprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
 				"N/A\n");
 
-	len += snprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
+	len += scnprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
 			"\tHost Allocated VFI extents: ");
 	rc = lpfc_sli4_get_allocated_extnts(phba, LPFC_RSC_TYPE_FCOE_VFI,
 					    &ext_cnt, &ext_size);
 	if (!rc)
-		len += snprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
+		len += scnprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
 				"Port %d Extent %3d, Size %3d\n",
 				phba->brd_no, ext_cnt, ext_size);
 	else
-		len += snprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
+		len += scnprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
 				"N/A\n");
 
-	len += snprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
+	len += scnprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
 			"\tHost Allocated RPI extents: ");
 	rc = lpfc_sli4_get_allocated_extnts(phba, LPFC_RSC_TYPE_FCOE_RPI,
 					    &ext_cnt, &ext_size);
 	if (!rc)
-		len += snprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
+		len += scnprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
 				"Port %d Extent %3d, Size %3d\n",
 				phba->brd_no, ext_cnt, ext_size);
 	else
-		len += snprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
+		len += scnprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
 				"N/A\n");
 
-	len += snprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
+	len += scnprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
 			"\tHost Allocated XRI extents: ");
 	rc = lpfc_sli4_get_allocated_extnts(phba, LPFC_RSC_TYPE_FCOE_XRI,
 					    &ext_cnt, &ext_size);
 	if (!rc)
-		len += snprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
+		len += scnprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
 				"Port %d Extent %3d, Size %3d\n",
 				phba->brd_no, ext_cnt, ext_size);
 	else
-		len += snprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
+		len += scnprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
 				"N/A\n");
 
 	return len;
@@ -5238,49 +5244,49 @@ lpfc_idiag_extacc_drivr_get(struct lpfc_hba *phba, char *pbuffer, int len)
 	struct lpfc_rsrc_blks *rsrc_blks;
 	int index;
 
-	len += snprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
+	len += scnprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
 			"\nDriver Extents Information:\n");
 
-	len += snprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
+	len += scnprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
 			"\tVPI extents:\n");
 	index = 0;
 	list_for_each_entry(rsrc_blks, &phba->lpfc_vpi_blk_list, list) {
-		len += snprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
+		len += scnprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
 				"\t\tBlock %3d: Start %4d, Count %4d\n",
 				index, rsrc_blks->rsrc_start,
 				rsrc_blks->rsrc_size);
 		index++;
 	}
-	len += snprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
+	len += scnprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
 			"\tVFI extents:\n");
 	index = 0;
 	list_for_each_entry(rsrc_blks, &phba->sli4_hba.lpfc_vfi_blk_list,
 			    list) {
-		len += snprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
+		len += scnprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
 				"\t\tBlock %3d: Start %4d, Count %4d\n",
 				index, rsrc_blks->rsrc_start,
 				rsrc_blks->rsrc_size);
 		index++;
 	}
 
-	len += snprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
+	len += scnprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
 			"\tRPI extents:\n");
 	index = 0;
 	list_for_each_entry(rsrc_blks, &phba->sli4_hba.lpfc_rpi_blk_list,
 			    list) {
-		len += snprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
+		len += scnprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
 				"\t\tBlock %3d: Start %4d, Count %4d\n",
 				index, rsrc_blks->rsrc_start,
 				rsrc_blks->rsrc_size);
 		index++;
 	}
 
-	len += snprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
+	len += scnprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
 			"\tXRI extents:\n");
 	index = 0;
 	list_for_each_entry(rsrc_blks, &phba->sli4_hba.lpfc_xri_blk_list,
 			    list) {
-		len += snprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
+		len += scnprintf(pbuffer+len, LPFC_EXT_ACC_BUF_SIZE-len,
 				"\t\tBlock %3d: Start %4d, Count %4d\n",
 				index, rsrc_blks->rsrc_start,
 				rsrc_blks->rsrc_size);
@@ -5706,11 +5712,11 @@ lpfc_idiag_mbxacc_dump_bsg_mbox(struct lpfc_hba *phba, enum nemb_type nemb_tp,
 				if (i != 0)
 					pr_err("%s\n", line_buf);
 				len = 0;
-				len += snprintf(line_buf+len,
+				len += scnprintf(line_buf+len,
 						LPFC_MBX_ACC_LBUF_SZ-len,
 						"%03d: ", i);
 			}
-			len += snprintf(line_buf+len, LPFC_MBX_ACC_LBUF_SZ-len,
+			len += scnprintf(line_buf+len, LPFC_MBX_ACC_LBUF_SZ-len,
 					"%08x ", (uint32_t)*pword);
 			pword++;
 		}
@@ -5773,11 +5779,11 @@ lpfc_idiag_mbxacc_dump_issue_mbox(struct lpfc_hba *phba, MAILBOX_t *pmbox)
 					pr_err("%s\n", line_buf);
 				len = 0;
 				memset(line_buf, 0, LPFC_MBX_ACC_LBUF_SZ);
-				len += snprintf(line_buf+len,
+				len += scnprintf(line_buf+len,
 						LPFC_MBX_ACC_LBUF_SZ-len,
 						"%03d: ", i);
 			}
-			len += snprintf(line_buf+len, LPFC_MBX_ACC_LBUF_SZ-len,
+			len += scnprintf(line_buf+len, LPFC_MBX_ACC_LBUF_SZ-len,
 					"%08x ",
 					((uint32_t)*pword) & 0xffffffff);
 			pword++;
@@ -5796,18 +5802,18 @@ lpfc_idiag_mbxacc_dump_issue_mbox(struct lpfc_hba *phba, MAILBOX_t *pmbox)
 					pr_err("%s\n", line_buf);
 				len = 0;
 				memset(line_buf, 0, LPFC_MBX_ACC_LBUF_SZ);
-				len += snprintf(line_buf+len,
+				len += scnprintf(line_buf+len,
 						LPFC_MBX_ACC_LBUF_SZ-len,
 						"%03d: ", i);
 			}
 			for (j = 0; j < 4; j++) {
-				len += snprintf(line_buf+len,
+				len += scnprintf(line_buf+len,
 						LPFC_MBX_ACC_LBUF_SZ-len,
 						"%02x",
 						((uint8_t)*pbyte) & 0xff);
 				pbyte++;
 			}
-			len += snprintf(line_buf+len,
+			len += scnprintf(line_buf+len,
 					LPFC_MBX_ACC_LBUF_SZ-len, " ");
 		}
 		if ((i - 1) % 8)
@@ -5891,7 +5897,7 @@ lpfc_debugfs_initialize(struct lpfc_vport *vport)
 					    phba, &lpfc_debugfs_op_lockstat);
 		if (!phba->debug_lockstat) {
 			lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT,
-					 "0913 Cant create debugfs lockstat\n");
+					 "4610 Cant create debugfs lockstat\n");
 			goto debug_failed;
 		}
 #endif
@@ -6134,7 +6140,7 @@ nvmeio_off:
 				    vport, &lpfc_debugfs_op_scsistat);
 	if (!vport->debug_scsistat) {
 		lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT,
-				 "0914 Cannot create debugfs scsistat\n");
+				 "4611 Cannot create debugfs scsistat\n");
 		goto debug_failed;
 	}
 
diff --git a/drivers/scsi/lpfc/lpfc_debugfs.h b/drivers/scsi/lpfc/lpfc_debugfs.h
index 93ab7dfb8ee0..2322ddb085c0 100644
--- a/drivers/scsi/lpfc/lpfc_debugfs.h
+++ b/drivers/scsi/lpfc/lpfc_debugfs.h
@@ -345,10 +345,10 @@ lpfc_debug_dump_qe(struct lpfc_queue *q, uint32_t idx)
 
 	esize = q->entry_size;
 	qe_word_cnt = esize / sizeof(uint32_t);
-	pword = q->qe[idx].address;
+	pword = lpfc_sli4_qe(q, idx);
 
 	len = 0;
-	len += snprintf(line_buf+len, LPFC_LBUF_SZ-len, "QE[%04d]: ", idx);
+	len += scnprintf(line_buf+len, LPFC_LBUF_SZ-len, "QE[%04d]: ", idx);
 	if (qe_word_cnt > 8)
 		printk(KERN_ERR "%s\n", line_buf);
 
@@ -359,11 +359,11 @@ lpfc_debug_dump_qe(struct lpfc_queue *q, uint32_t idx)
 			if (qe_word_cnt > 8) {
 				len = 0;
 				memset(line_buf, 0, LPFC_LBUF_SZ);
-				len += snprintf(line_buf+len, LPFC_LBUF_SZ-len,
+				len += scnprintf(line_buf+len, LPFC_LBUF_SZ-len,
 						"%03d: ", i);
 			}
 		}
-		len += snprintf(line_buf+len, LPFC_LBUF_SZ-len, "%08x ",
+		len += scnprintf(line_buf+len, LPFC_LBUF_SZ-len, "%08x ",
 				((uint32_t)*pword) & 0xffffffff);
 		pword++;
 	}
diff --git a/drivers/scsi/lpfc/lpfc_els.c b/drivers/scsi/lpfc/lpfc_els.c
index 7b0755e3527d..c8fb0b455f2a 100644
--- a/drivers/scsi/lpfc/lpfc_els.c
+++ b/drivers/scsi/lpfc/lpfc_els.c
@@ -1961,7 +1961,7 @@ lpfc_cmpl_els_plogi(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 	IOCB_t *irsp;
 	struct lpfc_nodelist *ndlp;
 	struct lpfc_dmabuf *prsp;
-	int disc, rc;
+	int disc;
 
 	/* we pass cmdiocb to state machine which needs rspiocb as well */
 	cmdiocb->context_un.rsp_iocb = rspiocb;
@@ -1990,7 +1990,6 @@ lpfc_cmpl_els_plogi(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 	disc = (ndlp->nlp_flag & NLP_NPR_2B_DISC);
 	ndlp->nlp_flag &= ~NLP_NPR_2B_DISC;
 	spin_unlock_irq(shost->host_lock);
-	rc = 0;
 
 	/* PLOGI completes to NPort <nlp_DID> */
 	lpfc_printf_vlog(vport, KERN_INFO, LOG_ELS,
@@ -2029,18 +2028,16 @@ lpfc_cmpl_els_plogi(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
 				 ndlp->nlp_DID, irsp->ulpStatus,
 				 irsp->un.ulpWord[4]);
 		/* Do not call DSM for lpfc_els_abort'ed ELS cmds */
-		if (lpfc_error_lost_link(irsp))
-			rc = NLP_STE_FREED_NODE;
-		else
-			rc = lpfc_disc_state_machine(vport, ndlp, cmdiocb,
-						     NLP_EVT_CMPL_PLOGI);
+		if (!lpfc_error_lost_link(irsp))
+			lpfc_disc_state_machine(vport, ndlp, cmdiocb,
+						NLP_EVT_CMPL_PLOGI);
 	} else {
 		/* Good status, call state machine */
 		prsp = list_entry(((struct lpfc_dmabuf *)
 				   cmdiocb->context2)->list.next,
 				  struct lpfc_dmabuf, list);
 		ndlp = lpfc_plogi_confirm_nport(phba, prsp->virt, ndlp);
-		rc = lpfc_disc_state_machine(vport, ndlp, cmdiocb,
+		lpfc_disc_state_machine(vport, ndlp, cmdiocb,
 					     NLP_EVT_CMPL_PLOGI);
 	}
 
@@ -6744,12 +6741,11 @@ lpfc_els_rcv_rnid(struct lpfc_vport *vport, struct lpfc_iocbq *cmdiocb,
 	uint32_t *lp;
 	RNID *rn;
 	struct ls_rjt stat;
-	uint32_t cmd;
 
 	pcmd = (struct lpfc_dmabuf *) cmdiocb->context2;
 	lp = (uint32_t *) pcmd->virt;
 
-	cmd = *lp++;
+	lp++;
 	rn = (RNID *) lp;
 
 	/* RNID received */
@@ -7508,14 +7504,14 @@ lpfc_els_rcv_farp(struct lpfc_vport *vport, struct lpfc_iocbq *cmdiocb,
 	uint32_t *lp;
 	IOCB_t *icmd;
 	FARP *fp;
-	uint32_t cmd, cnt, did;
+	uint32_t cnt, did;
 
 	icmd = &cmdiocb->iocb;
 	did = icmd->un.elsreq64.remoteID;
 	pcmd = (struct lpfc_dmabuf *) cmdiocb->context2;
 	lp = (uint32_t *) pcmd->virt;
 
-	cmd = *lp++;
+	lp++;
 	fp = (FARP *) lp;
 	/* FARP-REQ received from DID <did> */
 	lpfc_printf_vlog(vport, KERN_INFO, LOG_ELS,
@@ -7580,14 +7576,14 @@ lpfc_els_rcv_farpr(struct lpfc_vport *vport, struct lpfc_iocbq *cmdiocb,
 	struct lpfc_dmabuf *pcmd;
 	uint32_t *lp;
 	IOCB_t *icmd;
-	uint32_t cmd, did;
+	uint32_t did;
 
 	icmd = &cmdiocb->iocb;
 	did = icmd->un.elsreq64.remoteID;
 	pcmd = (struct lpfc_dmabuf *) cmdiocb->context2;
 	lp = (uint32_t *) pcmd->virt;
 
-	cmd = *lp++;
+	lp++;
 	/* FARP-RSP received from DID <did> */
 	lpfc_printf_vlog(vport, KERN_INFO, LOG_ELS,
 			 "0600 FARP-RSP received from DID x%x\n", did);
@@ -8454,6 +8450,14 @@ lpfc_els_unsol_buffer(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
 		rjt_err = LSRJT_UNABLE_TPC;
 		rjt_exp = LSEXP_INVALID_OX_RX;
 		break;
+	case ELS_CMD_FPIN:
+		/*
+		 * Received FPIN from fabric - pass it to the
+		 * transport FPIN handler.
+		 */
+		fc_host_fpin_rcv(shost, elsiocb->iocb.unsli3.rcvsli3.acc_len,
+				(char *)payload);
+		break;
 	default:
 		lpfc_debugfs_disc_trc(vport, LPFC_DISC_TRC_ELS_UNSOL,
 			"RCV ELS cmd:     cmd:x%x did:x%x/ste:x%x",
@@ -8776,7 +8780,6 @@ lpfc_cmpl_reg_new_vport(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb)
 				return;
 			}
 			/* fall through */
-
 		default:
 			/* Try to recover from this error */
 			if (phba->sli_rev == LPFC_SLI_REV4)
diff --git a/drivers/scsi/lpfc/lpfc_hbadisc.c b/drivers/scsi/lpfc/lpfc_hbadisc.c
index 14fffbebbbb5..c43852f97f25 100644
--- a/drivers/scsi/lpfc/lpfc_hbadisc.c
+++ b/drivers/scsi/lpfc/lpfc_hbadisc.c
@@ -885,15 +885,9 @@ lpfc_linkdown(struct lpfc_hba *phba)
 	LPFC_MBOXQ_t          *mb;
 	int i;
 
-	if (phba->link_state == LPFC_LINK_DOWN) {
-		if (phba->sli4_hba.conf_trunk) {
-			phba->trunk_link.link0.state = 0;
-			phba->trunk_link.link1.state = 0;
-			phba->trunk_link.link2.state = 0;
-			phba->trunk_link.link3.state = 0;
-		}
+	if (phba->link_state == LPFC_LINK_DOWN)
 		return 0;
-	}
+
 	/* Block all SCSI stack I/Os */
 	lpfc_scsi_dev_block(phba);
 
@@ -932,7 +926,11 @@ lpfc_linkdown(struct lpfc_hba *phba)
 		}
 	}
 	lpfc_destroy_vport_work_array(phba, vports);
-	/* Clean up any firmware default rpi's */
+
+	/* Clean up any SLI3 firmware default rpi's */
+	if (phba->sli_rev > LPFC_SLI_REV3)
+		goto skip_unreg_did;
+
 	mb = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
 	if (mb) {
 		lpfc_unreg_did(phba, 0xffff, LPFC_UNREG_ALL_DFLT_RPIS, mb);
@@ -944,6 +942,7 @@ lpfc_linkdown(struct lpfc_hba *phba)
 		}
 	}
 
+ skip_unreg_did:
 	/* Setup myDID for link up if we are in pt2pt mode */
 	if (phba->pport->fc_flag & FC_PT2PT) {
 		mb = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
@@ -4147,9 +4146,15 @@ lpfc_register_remote_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
 	rdata->pnode = lpfc_nlp_get(ndlp);
 
 	if (ndlp->nlp_type & NLP_FCP_TARGET)
-		rport_ids.roles |= FC_RPORT_ROLE_FCP_TARGET;
+		rport_ids.roles |= FC_PORT_ROLE_FCP_TARGET;
 	if (ndlp->nlp_type & NLP_FCP_INITIATOR)
-		rport_ids.roles |= FC_RPORT_ROLE_FCP_INITIATOR;
+		rport_ids.roles |= FC_PORT_ROLE_FCP_INITIATOR;
+	if (ndlp->nlp_type & NLP_NVME_INITIATOR)
+		rport_ids.roles |= FC_PORT_ROLE_NVME_INITIATOR;
+	if (ndlp->nlp_type & NLP_NVME_TARGET)
+		rport_ids.roles |= FC_PORT_ROLE_NVME_TARGET;
+	if (ndlp->nlp_type & NLP_NVME_DISCOVERY)
+		rport_ids.roles |= FC_PORT_ROLE_NVME_DISCOVERY;
 
 	if (rport_ids.roles !=  FC_RPORT_ROLE_UNKNOWN)
 		fc_remote_port_rolechg(rport, rport_ids.roles);
@@ -4675,6 +4680,7 @@ lpfc_check_sli_ndlp(struct lpfc_hba *phba,
 		case CMD_XMIT_ELS_RSP64_CX:
 			if (iocb->context1 == (uint8_t *) ndlp)
 				return 1;
+			/* fall through */
 		}
 	} else if (pring->ringno == LPFC_FCP_RING) {
 		/* Skip match check if waiting to relogin to FCP target */
@@ -4870,6 +4876,10 @@ lpfc_unreg_rpi(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
 					 * accept PLOGIs after unreg_rpi_cmpl
 					 */
 					acc_plogi = 0;
+				} else if (vport->load_flag & FC_UNLOADING) {
+					mbox->ctx_ndlp = NULL;
+					mbox->mbox_cmpl =
+						lpfc_sli_def_mbox_cmpl;
 				} else {
 					mbox->ctx_ndlp = ndlp;
 					mbox->mbox_cmpl =
@@ -4981,6 +4991,10 @@ lpfc_unreg_default_rpis(struct lpfc_vport *vport)
 	LPFC_MBOXQ_t     *mbox;
 	int rc;
 
+	/* Unreg DID is an SLI3 operation. */
+	if (phba->sli_rev > LPFC_SLI_REV3)
+		return;
+
 	mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
 	if (mbox) {
 		lpfc_unreg_did(phba, vport->vpi, LPFC_UNREG_ALL_DFLT_RPIS,
diff --git a/drivers/scsi/lpfc/lpfc_hw.h b/drivers/scsi/lpfc/lpfc_hw.h
index ec1227018913..edd8f3982023 100644
--- a/drivers/scsi/lpfc/lpfc_hw.h
+++ b/drivers/scsi/lpfc/lpfc_hw.h
@@ -560,6 +560,8 @@ struct fc_vft_header {
 #define fc_vft_hdr_hopct_WORD		word1
 };
 
+#include <uapi/scsi/fc/fc_els.h>
+
 /*
  *  Extended Link Service LS_COMMAND codes (Payload Word 0)
  */
@@ -603,6 +605,7 @@ struct fc_vft_header {
 #define ELS_CMD_RNID      0x78000000
 #define ELS_CMD_LIRR      0x7A000000
 #define ELS_CMD_LCB	  0x81000000
+#define ELS_CMD_FPIN	  0x16000000
 #else	/*  __LITTLE_ENDIAN_BITFIELD */
 #define ELS_CMD_MASK      0xffff
 #define ELS_RSP_MASK      0xff
@@ -643,6 +646,7 @@ struct fc_vft_header {
 #define ELS_CMD_RNID      0x78
 #define ELS_CMD_LIRR      0x7A
 #define ELS_CMD_LCB	  0x81
+#define ELS_CMD_FPIN	  ELS_FPIN
 #endif
 
 /*
diff --git a/drivers/scsi/lpfc/lpfc_hw4.h b/drivers/scsi/lpfc/lpfc_hw4.h
index ff875b833192..77f9a55a3f54 100644
--- a/drivers/scsi/lpfc/lpfc_hw4.h
+++ b/drivers/scsi/lpfc/lpfc_hw4.h
@@ -1894,18 +1894,19 @@ struct lpfc_mbx_set_link_diag_loopback {
 	union {
 		struct {
 			uint32_t word0;
-#define lpfc_mbx_set_diag_lpbk_type_SHIFT	0
-#define lpfc_mbx_set_diag_lpbk_type_MASK	0x00000003
-#define lpfc_mbx_set_diag_lpbk_type_WORD	word0
-#define LPFC_DIAG_LOOPBACK_TYPE_DISABLE		0x0
-#define LPFC_DIAG_LOOPBACK_TYPE_INTERNAL	0x1
-#define LPFC_DIAG_LOOPBACK_TYPE_SERDES		0x2
-#define lpfc_mbx_set_diag_lpbk_link_num_SHIFT	16
-#define lpfc_mbx_set_diag_lpbk_link_num_MASK	0x0000003F
-#define lpfc_mbx_set_diag_lpbk_link_num_WORD	word0
-#define lpfc_mbx_set_diag_lpbk_link_type_SHIFT	22
-#define lpfc_mbx_set_diag_lpbk_link_type_MASK	0x00000003
-#define lpfc_mbx_set_diag_lpbk_link_type_WORD	word0
+#define lpfc_mbx_set_diag_lpbk_type_SHIFT		0
+#define lpfc_mbx_set_diag_lpbk_type_MASK		0x00000003
+#define lpfc_mbx_set_diag_lpbk_type_WORD		word0
+#define LPFC_DIAG_LOOPBACK_TYPE_DISABLE			0x0
+#define LPFC_DIAG_LOOPBACK_TYPE_INTERNAL		0x1
+#define LPFC_DIAG_LOOPBACK_TYPE_SERDES			0x2
+#define LPFC_DIAG_LOOPBACK_TYPE_EXTERNAL_TRUNKED	0x3
+#define lpfc_mbx_set_diag_lpbk_link_num_SHIFT		16
+#define lpfc_mbx_set_diag_lpbk_link_num_MASK		0x0000003F
+#define lpfc_mbx_set_diag_lpbk_link_num_WORD		word0
+#define lpfc_mbx_set_diag_lpbk_link_type_SHIFT		22
+#define lpfc_mbx_set_diag_lpbk_link_type_MASK		0x00000003
+#define lpfc_mbx_set_diag_lpbk_link_type_WORD		word0
 		} req;
 		struct {
 			uint32_t word0;
@@ -4083,22 +4084,7 @@ struct lpfc_acqe_grp5 {
 	uint32_t trailer;
 };
 
-static char *const trunk_errmsg[] = {	/* map errcode */
-	"",	/* There is no such error code at index 0*/
-	"link negotiated speed does not match existing"
-		" trunk - link was \"low\" speed",
-	"link negotiated speed does not match"
-		" existing trunk - link was \"middle\" speed",
-	"link negotiated speed does not match existing"
-		" trunk - link was \"high\" speed",
-	"Attached to non-trunking port - F_Port",
-	"Attached to non-trunking port - N_Port",
-	"FLOGI response timeout",
-	"non-FLOGI frame received",
-	"Invalid FLOGI response",
-	"Trunking initialization protocol",
-	"Trunk peer device mismatch",
-};
+extern const char *const trunk_errmsg[];
 
 struct lpfc_acqe_fc_la {
 	uint32_t word0;
diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c
index 7fcdaed3fa94..eaaef682de25 100644
--- a/drivers/scsi/lpfc/lpfc_init.c
+++ b/drivers/scsi/lpfc/lpfc_init.c
@@ -1117,19 +1117,19 @@ lpfc_hba_down_post_s4(struct lpfc_hba *phba)
 
 		}
 	}
+	spin_unlock_irq(&phba->hbalock);
 
 	if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) {
-		spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock);
+		spin_lock_irq(&phba->sli4_hba.abts_nvmet_buf_list_lock);
 		list_splice_init(&phba->sli4_hba.lpfc_abts_nvmet_ctx_list,
 				 &nvmet_aborts);
-		spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock);
+		spin_unlock_irq(&phba->sli4_hba.abts_nvmet_buf_list_lock);
 		list_for_each_entry_safe(ctxp, ctxp_next, &nvmet_aborts, list) {
 			ctxp->flag &= ~(LPFC_NVMET_XBUSY | LPFC_NVMET_ABORT_OP);
 			lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf);
 		}
 	}
 
-	spin_unlock_irq(&phba->hbalock);
 	lpfc_sli4_free_sp_events(phba);
 	return cnt;
 }
@@ -1844,8 +1844,12 @@ lpfc_handle_eratt_s4(struct lpfc_hba *phba)
 	/* If the pci channel is offline, ignore possible errors, since
 	 * we cannot communicate with the pci card anyway.
 	 */
-	if (pci_channel_offline(phba->pcidev))
+	if (pci_channel_offline(phba->pcidev)) {
+		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+				"3166 pci channel is offline\n");
+		lpfc_sli4_offline_eratt(phba);
 		return;
+	}
 
 	memset(&portsmphr_reg, 0, sizeof(portsmphr_reg));
 	if_type = bf_get(lpfc_sli_intf_if_type, &phba->sli4_hba.sli_intf);
@@ -1922,6 +1926,7 @@ lpfc_handle_eratt_s4(struct lpfc_hba *phba)
 			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 				"3151 PCI bus read access failure: x%x\n",
 				readl(phba->sli4_hba.u.if_type2.STATUSregaddr));
+			lpfc_sli4_offline_eratt(phba);
 			return;
 		}
 		reg_err1 = readl(phba->sli4_hba.u.if_type2.ERR1regaddr);
@@ -3075,7 +3080,7 @@ lpfc_sli4_node_prep(struct lpfc_hba *phba)
  * This routine moves a batch of XRIs from lpfc_io_buf_list_put of HWQ 0
  * to expedite pool. Mark them as expedite.
  **/
-void lpfc_create_expedite_pool(struct lpfc_hba *phba)
+static void lpfc_create_expedite_pool(struct lpfc_hba *phba)
 {
 	struct lpfc_sli4_hdw_queue *qp;
 	struct lpfc_io_buf *lpfc_ncmd;
@@ -3110,7 +3115,7 @@ void lpfc_create_expedite_pool(struct lpfc_hba *phba)
  * This routine returns XRIs from expedite pool to lpfc_io_buf_list_put
  * of HWQ 0. Clear the mark.
  **/
-void lpfc_destroy_expedite_pool(struct lpfc_hba *phba)
+static void lpfc_destroy_expedite_pool(struct lpfc_hba *phba)
 {
 	struct lpfc_sli4_hdw_queue *qp;
 	struct lpfc_io_buf *lpfc_ncmd;
@@ -3230,7 +3235,7 @@ void lpfc_create_multixri_pools(struct lpfc_hba *phba)
  *
  * This routine returns XRIs from public/private to lpfc_io_buf_list_put.
  **/
-void lpfc_destroy_multixri_pools(struct lpfc_hba *phba)
+static void lpfc_destroy_multixri_pools(struct lpfc_hba *phba)
 {
 	u32 i;
 	u32 hwq_count;
@@ -3245,6 +3250,13 @@ void lpfc_destroy_multixri_pools(struct lpfc_hba *phba)
 	if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME)
 		lpfc_destroy_expedite_pool(phba);
 
+	if (!(phba->pport->load_flag & FC_UNLOADING)) {
+		lpfc_sli_flush_fcp_rings(phba);
+
+		if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME)
+			lpfc_sli_flush_nvme_rings(phba);
+	}
+
 	hwq_count = phba->cfg_hdw_queue;
 
 	for (i = 0; i < hwq_count; i++) {
@@ -3611,8 +3623,6 @@ lpfc_io_free(struct lpfc_hba *phba)
 	struct lpfc_sli4_hdw_queue *qp;
 	int idx;
 
-	spin_lock_irq(&phba->hbalock);
-
 	for (idx = 0; idx < phba->cfg_hdw_queue; idx++) {
 		qp = &phba->sli4_hba.hdwq[idx];
 		/* Release all the lpfc_nvme_bufs maintained by this host. */
@@ -3642,8 +3652,6 @@ lpfc_io_free(struct lpfc_hba *phba)
 		}
 		spin_unlock(&qp->io_buf_list_get_lock);
 	}
-
-	spin_unlock_irq(&phba->hbalock);
 }
 
 /**
@@ -4457,7 +4465,7 @@ finished:
 	return stat;
 }
 
-void lpfc_host_supported_speeds_set(struct Scsi_Host *shost)
+static void lpfc_host_supported_speeds_set(struct Scsi_Host *shost)
 {
 	struct lpfc_vport *vport = (struct lpfc_vport *)shost->hostdata;
 	struct lpfc_hba   *phba = vport->phba;
@@ -8603,9 +8611,9 @@ lpfc_sli4_queue_verify(struct lpfc_hba *phba)
 	if (phba->nvmet_support) {
 		if (phba->cfg_irq_chann < phba->cfg_nvmet_mrq)
 			phba->cfg_nvmet_mrq = phba->cfg_irq_chann;
+		if (phba->cfg_nvmet_mrq > LPFC_NVMET_MRQ_MAX)
+			phba->cfg_nvmet_mrq = LPFC_NVMET_MRQ_MAX;
 	}
-	if (phba->cfg_nvmet_mrq > LPFC_NVMET_MRQ_MAX)
-		phba->cfg_nvmet_mrq = LPFC_NVMET_MRQ_MAX;
 
 	lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 			"2574 IO channels: hdwQ %d IRQ %d MRQ: %d\n",
@@ -8626,10 +8634,12 @@ static int
 lpfc_alloc_nvme_wq_cq(struct lpfc_hba *phba, int wqidx)
 {
 	struct lpfc_queue *qdesc;
+	int cpu;
 
+	cpu = lpfc_find_cpu_handle(phba, wqidx, LPFC_FIND_BY_HDWQ);
 	qdesc = lpfc_sli4_queue_alloc(phba, LPFC_EXPANDED_PAGE_SIZE,
 				      phba->sli4_hba.cq_esize,
-				      LPFC_CQE_EXP_COUNT);
+				      LPFC_CQE_EXP_COUNT, cpu);
 	if (!qdesc) {
 		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 				"0508 Failed allocate fast-path NVME CQ (%d)\n",
@@ -8638,11 +8648,12 @@ lpfc_alloc_nvme_wq_cq(struct lpfc_hba *phba, int wqidx)
 	}
 	qdesc->qe_valid = 1;
 	qdesc->hdwq = wqidx;
-	qdesc->chann = lpfc_find_cpu_handle(phba, wqidx, LPFC_FIND_BY_HDWQ);
+	qdesc->chann = cpu;
 	phba->sli4_hba.hdwq[wqidx].nvme_cq = qdesc;
 
 	qdesc = lpfc_sli4_queue_alloc(phba, LPFC_EXPANDED_PAGE_SIZE,
-				      LPFC_WQE128_SIZE, LPFC_WQE_EXP_COUNT);
+				      LPFC_WQE128_SIZE, LPFC_WQE_EXP_COUNT,
+				      cpu);
 	if (!qdesc) {
 		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 				"0509 Failed allocate fast-path NVME WQ (%d)\n",
@@ -8661,18 +8672,20 @@ lpfc_alloc_fcp_wq_cq(struct lpfc_hba *phba, int wqidx)
 {
 	struct lpfc_queue *qdesc;
 	uint32_t wqesize;
+	int cpu;
 
+	cpu = lpfc_find_cpu_handle(phba, wqidx, LPFC_FIND_BY_HDWQ);
 	/* Create Fast Path FCP CQs */
 	if (phba->enab_exp_wqcq_pages)
 		/* Increase the CQ size when WQEs contain an embedded cdb */
 		qdesc = lpfc_sli4_queue_alloc(phba, LPFC_EXPANDED_PAGE_SIZE,
 					      phba->sli4_hba.cq_esize,
-					      LPFC_CQE_EXP_COUNT);
+					      LPFC_CQE_EXP_COUNT, cpu);
 
 	else
 		qdesc = lpfc_sli4_queue_alloc(phba, LPFC_DEFAULT_PAGE_SIZE,
 					      phba->sli4_hba.cq_esize,
-					      phba->sli4_hba.cq_ecount);
+					      phba->sli4_hba.cq_ecount, cpu);
 	if (!qdesc) {
 		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 			"0499 Failed allocate fast-path FCP CQ (%d)\n", wqidx);
@@ -8680,7 +8693,7 @@ lpfc_alloc_fcp_wq_cq(struct lpfc_hba *phba, int wqidx)
 	}
 	qdesc->qe_valid = 1;
 	qdesc->hdwq = wqidx;
-	qdesc->chann = lpfc_find_cpu_handle(phba, wqidx, LPFC_FIND_BY_HDWQ);
+	qdesc->chann = cpu;
 	phba->sli4_hba.hdwq[wqidx].fcp_cq = qdesc;
 
 	/* Create Fast Path FCP WQs */
@@ -8690,11 +8703,11 @@ lpfc_alloc_fcp_wq_cq(struct lpfc_hba *phba, int wqidx)
 			LPFC_WQE128_SIZE : phba->sli4_hba.wq_esize;
 		qdesc = lpfc_sli4_queue_alloc(phba, LPFC_EXPANDED_PAGE_SIZE,
 					      wqesize,
-					      LPFC_WQE_EXP_COUNT);
+					      LPFC_WQE_EXP_COUNT, cpu);
 	} else
 		qdesc = lpfc_sli4_queue_alloc(phba, LPFC_DEFAULT_PAGE_SIZE,
 					      phba->sli4_hba.wq_esize,
-					      phba->sli4_hba.wq_ecount);
+					      phba->sli4_hba.wq_ecount, cpu);
 
 	if (!qdesc) {
 		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
@@ -8727,7 +8740,7 @@ int
 lpfc_sli4_queue_create(struct lpfc_hba *phba)
 {
 	struct lpfc_queue *qdesc;
-	int idx, eqidx;
+	int idx, eqidx, cpu;
 	struct lpfc_sli4_hdw_queue *qp;
 	struct lpfc_eq_intr_info *eqi;
 
@@ -8814,13 +8827,15 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba)
 
 	/* Create HBA Event Queues (EQs) */
 	for (idx = 0; idx < phba->cfg_hdw_queue; idx++) {
+		/* determine EQ affinity */
+		eqidx = lpfc_find_eq_handle(phba, idx);
+		cpu = lpfc_find_cpu_handle(phba, eqidx, LPFC_FIND_BY_EQ);
 		/*
 		 * If there are more Hardware Queues than available
-		 * CQs, multiple Hardware Queues may share a common EQ.
+		 * EQs, multiple Hardware Queues may share a common EQ.
 		 */
 		if (idx >= phba->cfg_irq_chann) {
 			/* Share an existing EQ */
-			eqidx = lpfc_find_eq_handle(phba, idx);
 			phba->sli4_hba.hdwq[idx].hba_eq =
 				phba->sli4_hba.hdwq[eqidx].hba_eq;
 			continue;
@@ -8828,7 +8843,7 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba)
 		/* Create an EQ */
 		qdesc = lpfc_sli4_queue_alloc(phba, LPFC_DEFAULT_PAGE_SIZE,
 					      phba->sli4_hba.eq_esize,
-					      phba->sli4_hba.eq_ecount);
+					      phba->sli4_hba.eq_ecount, cpu);
 		if (!qdesc) {
 			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 					"0497 Failed allocate EQ (%d)\n", idx);
@@ -8838,9 +8853,7 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba)
 		qdesc->hdwq = idx;
 
 		/* Save the CPU this EQ is affinitised to */
-		eqidx = lpfc_find_eq_handle(phba, idx);
-		qdesc->chann = lpfc_find_cpu_handle(phba, eqidx,
-						    LPFC_FIND_BY_EQ);
+		qdesc->chann = cpu;
 		phba->sli4_hba.hdwq[idx].hba_eq = qdesc;
 		qdesc->last_cpu = qdesc->chann;
 		eqi = per_cpu_ptr(phba->sli4_hba.eq_info, qdesc->last_cpu);
@@ -8863,11 +8876,14 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba)
 
 		if (phba->nvmet_support) {
 			for (idx = 0; idx < phba->cfg_nvmet_mrq; idx++) {
+				cpu = lpfc_find_cpu_handle(phba, idx,
+							   LPFC_FIND_BY_HDWQ);
 				qdesc = lpfc_sli4_queue_alloc(
 						      phba,
 						      LPFC_DEFAULT_PAGE_SIZE,
 						      phba->sli4_hba.cq_esize,
-						      phba->sli4_hba.cq_ecount);
+						      phba->sli4_hba.cq_ecount,
+						      cpu);
 				if (!qdesc) {
 					lpfc_printf_log(
 						phba, KERN_ERR, LOG_INIT,
@@ -8877,7 +8893,7 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba)
 				}
 				qdesc->qe_valid = 1;
 				qdesc->hdwq = idx;
-				qdesc->chann = idx;
+				qdesc->chann = cpu;
 				phba->sli4_hba.nvmet_cqset[idx] = qdesc;
 			}
 		}
@@ -8887,10 +8903,11 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba)
 	 * Create Slow Path Completion Queues (CQs)
 	 */
 
+	cpu = lpfc_find_cpu_handle(phba, 0, LPFC_FIND_BY_EQ);
 	/* Create slow-path Mailbox Command Complete Queue */
 	qdesc = lpfc_sli4_queue_alloc(phba, LPFC_DEFAULT_PAGE_SIZE,
 				      phba->sli4_hba.cq_esize,
-				      phba->sli4_hba.cq_ecount);
+				      phba->sli4_hba.cq_ecount, cpu);
 	if (!qdesc) {
 		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 				"0500 Failed allocate slow-path mailbox CQ\n");
@@ -8902,7 +8919,7 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba)
 	/* Create slow-path ELS Complete Queue */
 	qdesc = lpfc_sli4_queue_alloc(phba, LPFC_DEFAULT_PAGE_SIZE,
 				      phba->sli4_hba.cq_esize,
-				      phba->sli4_hba.cq_ecount);
+				      phba->sli4_hba.cq_ecount, cpu);
 	if (!qdesc) {
 		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 				"0501 Failed allocate slow-path ELS CQ\n");
@@ -8921,7 +8938,7 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba)
 
 	qdesc = lpfc_sli4_queue_alloc(phba, LPFC_DEFAULT_PAGE_SIZE,
 				      phba->sli4_hba.mq_esize,
-				      phba->sli4_hba.mq_ecount);
+				      phba->sli4_hba.mq_ecount, cpu);
 	if (!qdesc) {
 		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 				"0505 Failed allocate slow-path MQ\n");
@@ -8937,7 +8954,7 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba)
 	/* Create slow-path ELS Work Queue */
 	qdesc = lpfc_sli4_queue_alloc(phba, LPFC_DEFAULT_PAGE_SIZE,
 				      phba->sli4_hba.wq_esize,
-				      phba->sli4_hba.wq_ecount);
+				      phba->sli4_hba.wq_ecount, cpu);
 	if (!qdesc) {
 		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 				"0504 Failed allocate slow-path ELS WQ\n");
@@ -8951,7 +8968,7 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba)
 		/* Create NVME LS Complete Queue */
 		qdesc = lpfc_sli4_queue_alloc(phba, LPFC_DEFAULT_PAGE_SIZE,
 					      phba->sli4_hba.cq_esize,
-					      phba->sli4_hba.cq_ecount);
+					      phba->sli4_hba.cq_ecount, cpu);
 		if (!qdesc) {
 			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 					"6079 Failed allocate NVME LS CQ\n");
@@ -8964,7 +8981,7 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba)
 		/* Create NVME LS Work Queue */
 		qdesc = lpfc_sli4_queue_alloc(phba, LPFC_DEFAULT_PAGE_SIZE,
 					      phba->sli4_hba.wq_esize,
-					      phba->sli4_hba.wq_ecount);
+					      phba->sli4_hba.wq_ecount, cpu);
 		if (!qdesc) {
 			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 					"6080 Failed allocate NVME LS WQ\n");
@@ -8982,7 +8999,7 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba)
 	/* Create Receive Queue for header */
 	qdesc = lpfc_sli4_queue_alloc(phba, LPFC_DEFAULT_PAGE_SIZE,
 				      phba->sli4_hba.rq_esize,
-				      phba->sli4_hba.rq_ecount);
+				      phba->sli4_hba.rq_ecount, cpu);
 	if (!qdesc) {
 		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 				"0506 Failed allocate receive HRQ\n");
@@ -8993,7 +9010,7 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba)
 	/* Create Receive Queue for data */
 	qdesc = lpfc_sli4_queue_alloc(phba, LPFC_DEFAULT_PAGE_SIZE,
 				      phba->sli4_hba.rq_esize,
-				      phba->sli4_hba.rq_ecount);
+				      phba->sli4_hba.rq_ecount, cpu);
 	if (!qdesc) {
 		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 				"0507 Failed allocate receive DRQ\n");
@@ -9004,11 +9021,14 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba)
 	if ((phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) &&
 	    phba->nvmet_support) {
 		for (idx = 0; idx < phba->cfg_nvmet_mrq; idx++) {
+			cpu = lpfc_find_cpu_handle(phba, idx,
+						   LPFC_FIND_BY_HDWQ);
 			/* Create NVMET Receive Queue for header */
 			qdesc = lpfc_sli4_queue_alloc(phba,
 						      LPFC_DEFAULT_PAGE_SIZE,
 						      phba->sli4_hba.rq_esize,
-						      LPFC_NVMET_RQE_DEF_COUNT);
+						      LPFC_NVMET_RQE_DEF_COUNT,
+						      cpu);
 			if (!qdesc) {
 				lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 						"3146 Failed allocate "
@@ -9019,8 +9039,9 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba)
 			phba->sli4_hba.nvmet_mrq_hdr[idx] = qdesc;
 
 			/* Only needed for header of RQ pair */
-			qdesc->rqbp = kzalloc(sizeof(struct lpfc_rqb),
-					      GFP_KERNEL);
+			qdesc->rqbp = kzalloc_node(sizeof(*qdesc->rqbp),
+						   GFP_KERNEL,
+						   cpu_to_node(cpu));
 			if (qdesc->rqbp == NULL) {
 				lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 						"6131 Failed allocate "
@@ -9035,7 +9056,8 @@ lpfc_sli4_queue_create(struct lpfc_hba *phba)
 			qdesc = lpfc_sli4_queue_alloc(phba,
 						      LPFC_DEFAULT_PAGE_SIZE,
 						      phba->sli4_hba.rq_esize,
-						      LPFC_NVMET_RQE_DEF_COUNT);
+						      LPFC_NVMET_RQE_DEF_COUNT,
+						      cpu);
 			if (!qdesc) {
 				lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
 						"3156 Failed allocate "
@@ -9134,6 +9156,20 @@ lpfc_sli4_release_hdwq(struct lpfc_hba *phba)
 void
 lpfc_sli4_queue_destroy(struct lpfc_hba *phba)
 {
+	/*
+	 * Set FREE_INIT before beginning to free the queues.
+	 * Wait until the users of queues to acknowledge to
+	 * release queues by clearing FREE_WAIT.
+	 */
+	spin_lock_irq(&phba->hbalock);
+	phba->sli.sli_flag |= LPFC_QUEUE_FREE_INIT;
+	while (phba->sli.sli_flag & LPFC_QUEUE_FREE_WAIT) {
+		spin_unlock_irq(&phba->hbalock);
+		msleep(20);
+		spin_lock_irq(&phba->hbalock);
+	}
+	spin_unlock_irq(&phba->hbalock);
+
 	/* Release HBA eqs */
 	if (phba->sli4_hba.hdwq)
 		lpfc_sli4_release_hdwq(phba);
@@ -9172,6 +9208,11 @@ lpfc_sli4_queue_destroy(struct lpfc_hba *phba)
 
 	/* Everything on this list has been freed */
 	INIT_LIST_HEAD(&phba->sli4_hba.lpfc_wq_list);
+
+	/* Done with freeing the queues */
+	spin_lock_irq(&phba->hbalock);
+	phba->sli.sli_flag &= ~LPFC_QUEUE_FREE_INIT;
+	spin_unlock_irq(&phba->hbalock);
 }
 
 int
@@ -9231,7 +9272,7 @@ lpfc_create_wq_cq(struct lpfc_hba *phba, struct lpfc_queue *eq,
 		rc = lpfc_wq_create(phba, wq, cq, qtype);
 		if (rc) {
 			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
-				"6123 Fail setup fastpath WQ (%d), rc = 0x%x\n",
+				"4618 Fail setup fastpath WQ (%d), rc = 0x%x\n",
 				qidx, (uint32_t)rc);
 			/* no need to tear down cq - caller will do so */
 			return rc;
@@ -9271,7 +9312,7 @@ lpfc_create_wq_cq(struct lpfc_hba *phba, struct lpfc_queue *eq,
  * This routine will populate the cq_lookup table by all
  * available CQ queue_id's.
  **/
-void
+static void
 lpfc_setup_cq_lookup(struct lpfc_hba *phba)
 {
 	struct lpfc_queue *eq, *childq;
@@ -10740,7 +10781,7 @@ lpfc_sli4_enable_msix(struct lpfc_hba *phba)
 				phba->cfg_irq_chann, vectors);
 		if (phba->cfg_irq_chann > vectors)
 			phba->cfg_irq_chann = vectors;
-		if (phba->cfg_nvmet_mrq > vectors)
+		if (phba->nvmet_support && (phba->cfg_nvmet_mrq > vectors))
 			phba->cfg_nvmet_mrq = vectors;
 	}
 
@@ -11297,7 +11338,7 @@ lpfc_get_sli4_parameters(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq)
 	    !phba->nvme_support) {
 		phba->nvme_support = 0;
 		phba->nvmet_support = 0;
-		phba->cfg_nvmet_mrq = LPFC_NVMET_MRQ_OFF;
+		phba->cfg_nvmet_mrq = 0;
 		lpfc_printf_log(phba, KERN_ERR, LOG_INIT | LOG_NVME,
 				"6101 Disabling NVME support: "
 				"Not supported by firmware: %d %d\n",
@@ -13046,7 +13087,7 @@ lpfc_io_resume(struct pci_dev *pdev)
  * is destroyed.
  *
  **/
-void
+static void
 lpfc_sli4_oas_verify(struct lpfc_hba *phba)
 {
 
diff --git a/drivers/scsi/lpfc/lpfc_nportdisc.c b/drivers/scsi/lpfc/lpfc_nportdisc.c
index 11d284c5486e..59252bfca14e 100644
--- a/drivers/scsi/lpfc/lpfc_nportdisc.c
+++ b/drivers/scsi/lpfc/lpfc_nportdisc.c
@@ -871,7 +871,7 @@ lpfc_disc_set_adisc(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
  * This function will send a unreg_login mailbox command to the firmware
  * to release a rpi.
  **/
-void
+static void
 lpfc_release_rpi(struct lpfc_hba *phba, struct lpfc_vport *vport,
 		 struct lpfc_nodelist *ndlp, uint16_t rpi)
 {
@@ -1733,7 +1733,6 @@ lpfc_cmpl_reglogin_reglogin_issue(struct lpfc_vport *vport,
 	LPFC_MBOXQ_t *pmb = (LPFC_MBOXQ_t *) arg;
 	MAILBOX_t *mb = &pmb->u.mb;
 	uint32_t did  = mb->un.varWords[1];
-	int rc = 0;
 
 	if (mb->mbxStatus) {
 		/* RegLogin failed */
@@ -1806,8 +1805,8 @@ lpfc_cmpl_reglogin_reglogin_issue(struct lpfc_vport *vport,
 			 * GFT_ID to determine if remote port supports NVME.
 			 */
 			if (vport->cfg_enable_fc4_type != LPFC_ENABLE_FCP) {
-				rc = lpfc_ns_cmd(vport, SLI_CTNS_GFT_ID,
-						 0, ndlp->nlp_DID);
+				lpfc_ns_cmd(vport, SLI_CTNS_GFT_ID, 0,
+					    ndlp->nlp_DID);
 				return ndlp->nlp_state;
 			}
 			ndlp->nlp_fc4_type = NLP_FC4_FCP;
diff --git a/drivers/scsi/lpfc/lpfc_nvme.c b/drivers/scsi/lpfc/lpfc_nvme.c
index d16ca413110d..9d99cb915390 100644
--- a/drivers/scsi/lpfc/lpfc_nvme.c
+++ b/drivers/scsi/lpfc/lpfc_nvme.c
@@ -229,7 +229,7 @@ lpfc_nvme_create_queue(struct nvme_fc_local_port *pnvme_lport,
 	if (qhandle == NULL)
 		return -ENOMEM;
 
-	qhandle->cpu_id = smp_processor_id();
+	qhandle->cpu_id = raw_smp_processor_id();
 	qhandle->qidx = qidx;
 	/*
 	 * NVME qidx == 0 is the admin queue, so both admin queue
@@ -312,7 +312,7 @@ lpfc_nvme_localport_delete(struct nvme_fc_local_port *localport)
  * Return value :
  * None
  */
-void
+static void
 lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port *remoteport)
 {
 	struct lpfc_nvme_rport *rport = remoteport->private;
@@ -1111,9 +1111,11 @@ lpfc_nvme_io_cmd_wqe_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn,
 out_err:
 			lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
 					 "6072 NVME Completion Error: xri %x "
-					 "status x%x result x%x placed x%x\n",
+					 "status x%x result x%x [x%x] "
+					 "placed x%x\n",
 					 lpfc_ncmd->cur_iocbq.sli4_xritag,
 					 lpfc_ncmd->status, lpfc_ncmd->result,
+					 wcqe->parameter,
 					 wcqe->total_data_placed);
 			nCmd->transferred_length = 0;
 			nCmd->rcv_rsplen = 0;
@@ -1141,7 +1143,7 @@ out_err:
 	if (phba->cpucheck_on & LPFC_CHECK_NVME_IO) {
 		uint32_t cpu;
 		idx = lpfc_ncmd->cur_iocbq.hba_wqidx;
-		cpu = smp_processor_id();
+		cpu = raw_smp_processor_id();
 		if (cpu < LPFC_CHECK_CPU_CNT) {
 			if (lpfc_ncmd->cpu != cpu)
 				lpfc_printf_vlog(vport,
@@ -1559,7 +1561,7 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
 	if (phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_HDWQ) {
 		idx = lpfc_queue_info->index;
 	} else {
-		cpu = smp_processor_id();
+		cpu = raw_smp_processor_id();
 		idx = phba->sli4_hba.cpu_map[cpu].hdwq;
 	}
 
@@ -1639,7 +1641,7 @@ lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
 		lpfc_ncmd->ts_cmd_wqput = ktime_get_ns();
 
 	if (phba->cpucheck_on & LPFC_CHECK_NVME_IO) {
-		cpu = smp_processor_id();
+		cpu = raw_smp_processor_id();
 		if (cpu < LPFC_CHECK_CPU_CNT) {
 			lpfc_ncmd->cpu = cpu;
 			if (idx != cpu)
@@ -2081,15 +2083,15 @@ lpfc_nvme_create_localport(struct lpfc_vport *vport)
 		lpfc_nvme_template.max_hw_queues =
 			phba->sli4_hba.num_present_cpu;
 
+	if (!IS_ENABLED(CONFIG_NVME_FC))
+		return ret;
+
 	/* localport is allocated from the stack, but the registration
 	 * call allocates heap memory as well as the private area.
 	 */
-#if (IS_ENABLED(CONFIG_NVME_FC))
+
 	ret = nvme_fc_register_localport(&nfcp_info, &lpfc_nvme_template,
 					 &vport->phba->pcidev->dev, &localport);
-#else
-	ret = -ENOMEM;
-#endif
 	if (!ret) {
 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME | LOG_NVME_DISC,
 				 "6005 Successfully registered local "
@@ -2124,6 +2126,7 @@ lpfc_nvme_create_localport(struct lpfc_vport *vport)
 	return ret;
 }
 
+#if (IS_ENABLED(CONFIG_NVME_FC))
 /* lpfc_nvme_lport_unreg_wait - Wait for the host to complete an lport unreg.
  *
  * The driver has to wait for the host nvme transport to callback
@@ -2134,12 +2137,11 @@ lpfc_nvme_create_localport(struct lpfc_vport *vport)
  * An uninterruptible wait is used because of the risk of transport-to-
  * driver state mismatch.
  */
-void
+static void
 lpfc_nvme_lport_unreg_wait(struct lpfc_vport *vport,
 			   struct lpfc_nvme_lport *lport,
 			   struct completion *lport_unreg_cmp)
 {
-#if (IS_ENABLED(CONFIG_NVME_FC))
 	u32 wait_tmo;
 	int ret;
 
@@ -2162,8 +2164,8 @@ lpfc_nvme_lport_unreg_wait(struct lpfc_vport *vport,
 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
 			 "6177 Lport %p Localport %p Complete Success\n",
 			 lport, vport->localport);
-#endif
 }
+#endif
 
 /**
  * lpfc_nvme_destroy_localport - Destroy lpfc_nvme bound to nvme transport.
diff --git a/drivers/scsi/lpfc/lpfc_nvmet.c b/drivers/scsi/lpfc/lpfc_nvmet.c
index 361e2b103648..d74bfd264495 100644
--- a/drivers/scsi/lpfc/lpfc_nvmet.c
+++ b/drivers/scsi/lpfc/lpfc_nvmet.c
@@ -220,7 +220,7 @@ lpfc_nvmet_cmd_template(void)
 	/* Word 12, 13, 14, 15 - is zero */
 }
 
-void
+static void
 lpfc_nvmet_defer_release(struct lpfc_hba *phba, struct lpfc_nvmet_rcv_ctx *ctxp)
 {
 	lockdep_assert_held(&ctxp->ctxlock);
@@ -325,7 +325,6 @@ lpfc_nvmet_ctxbuf_post(struct lpfc_hba *phba, struct lpfc_nvmet_ctxbuf *ctx_buf)
 	struct fc_frame_header *fc_hdr;
 	struct rqb_dmabuf *nvmebuf;
 	struct lpfc_nvmet_ctx_info *infop;
-	uint32_t *payload;
 	uint32_t size, oxid, sid;
 	int cpu;
 	unsigned long iflag;
@@ -370,7 +369,6 @@ lpfc_nvmet_ctxbuf_post(struct lpfc_hba *phba, struct lpfc_nvmet_ctxbuf *ctx_buf)
 		fc_hdr = (struct fc_frame_header *)(nvmebuf->hbuf.virt);
 		oxid = be16_to_cpu(fc_hdr->fh_ox_id);
 		tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
-		payload = (uint32_t *)(nvmebuf->dbuf.virt);
 		size = nvmebuf->bytes_recv;
 		sid = sli4_sid_from_fc_hdr(fc_hdr);
 
@@ -435,7 +433,7 @@ lpfc_nvmet_ctxbuf_post(struct lpfc_hba *phba, struct lpfc_nvmet_ctxbuf *ctx_buf)
 	 * Use the CPU context list, from the MRQ the IO was received on
 	 * (ctxp->idx), to save context structure.
 	 */
-	cpu = smp_processor_id();
+	cpu = raw_smp_processor_id();
 	infop = lpfc_get_ctx_list(phba, cpu, ctxp->idx);
 	spin_lock_irqsave(&infop->nvmet_ctx_list_lock, iflag);
 	list_add_tail(&ctx_buf->list, &infop->nvmet_ctx_list);
@@ -765,7 +763,7 @@ lpfc_nvmet_xmt_fcp_op_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
 	}
 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
 	if (phba->cpucheck_on & LPFC_CHECK_NVMET_IO) {
-		id = smp_processor_id();
+		id = raw_smp_processor_id();
 		if (id < LPFC_CHECK_CPU_CNT) {
 			if (ctxp->cpu != id)
 				lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,
@@ -906,7 +904,7 @@ lpfc_nvmet_xmt_fcp_op(struct nvmet_fc_target_port *tgtport,
 		ctxp->hdwq = &phba->sli4_hba.hdwq[rsp->hwqid];
 
 	if (phba->cpucheck_on & LPFC_CHECK_NVMET_IO) {
-		int id = smp_processor_id();
+		int id = raw_smp_processor_id();
 		if (id < LPFC_CHECK_CPU_CNT) {
 			if (rsp->hwqid != id)
 				lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,
@@ -1120,7 +1118,7 @@ lpfc_nvmet_defer_rcv(struct nvmet_fc_target_port *tgtport,
 
 
 	lpfc_nvmeio_data(phba, "NVMET DEFERRCV: xri x%x sz %d CPU %02x\n",
-			 ctxp->oxid, ctxp->size, smp_processor_id());
+			 ctxp->oxid, ctxp->size, raw_smp_processor_id());
 
 	if (!nvmebuf) {
 		lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,
@@ -1596,7 +1594,7 @@ lpfc_nvmet_rcv_unsol_abort(struct lpfc_vport *vport,
 
 		lpfc_nvmeio_data(phba,
 			"NVMET ABTS RCV: xri x%x CPU %02x rjt %d\n",
-			xri, smp_processor_id(), 0);
+			xri, raw_smp_processor_id(), 0);
 
 		lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
 				"6319 NVMET Rcv ABTS:acc xri x%x\n", xri);
@@ -1612,7 +1610,7 @@ lpfc_nvmet_rcv_unsol_abort(struct lpfc_vport *vport,
 	spin_unlock_irqrestore(&phba->hbalock, iflag);
 
 	lpfc_nvmeio_data(phba, "NVMET ABTS RCV: xri x%x CPU %02x rjt %d\n",
-			 xri, smp_processor_id(), 1);
+			 xri, raw_smp_processor_id(), 1);
 
 	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
 			"6320 NVMET Rcv ABTS:rjt xri x%x\n", xri);
@@ -1725,7 +1723,11 @@ lpfc_nvmet_destroy_targetport(struct lpfc_hba *phba)
 		}
 		tgtp->tport_unreg_cmp = &tport_unreg_cmp;
 		nvmet_fc_unregister_targetport(phba->targetport);
-		wait_for_completion_timeout(&tport_unreg_cmp, 5);
+		if (!wait_for_completion_timeout(tgtp->tport_unreg_cmp,
+					msecs_to_jiffies(LPFC_NVMET_WAIT_TMO)))
+			lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
+					"6179 Unreg targetport %p timeout "
+					"reached.\n", phba->targetport);
 		lpfc_nvmet_cleanup_io_context(phba);
 	}
 	phba->targetport = NULL;
@@ -1843,7 +1845,7 @@ lpfc_nvmet_process_rcv_fcp_req(struct lpfc_nvmet_ctxbuf *ctx_buf)
 	struct lpfc_hba *phba = ctxp->phba;
 	struct rqb_dmabuf *nvmebuf = ctxp->rqb_buffer;
 	struct lpfc_nvmet_tgtport *tgtp;
-	uint32_t *payload;
+	uint32_t *payload, qno;
 	uint32_t rc;
 	unsigned long iflags;
 
@@ -1876,6 +1878,15 @@ lpfc_nvmet_process_rcv_fcp_req(struct lpfc_nvmet_ctxbuf *ctx_buf)
 	/* Process FCP command */
 	if (rc == 0) {
 		atomic_inc(&tgtp->rcv_fcp_cmd_out);
+		spin_lock_irqsave(&ctxp->ctxlock, iflags);
+		if ((ctxp->flag & LPFC_NVMET_CTX_REUSE_WQ) ||
+		    (nvmebuf != ctxp->rqb_buffer)) {
+			spin_unlock_irqrestore(&ctxp->ctxlock, iflags);
+			return;
+		}
+		ctxp->rqb_buffer = NULL;
+		spin_unlock_irqrestore(&ctxp->ctxlock, iflags);
+		lpfc_rq_buf_free(phba, &nvmebuf->hbuf); /* repost */
 		return;
 	}
 
@@ -1886,6 +1897,20 @@ lpfc_nvmet_process_rcv_fcp_req(struct lpfc_nvmet_ctxbuf *ctx_buf)
 				 ctxp->oxid, ctxp->size, ctxp->sid);
 		atomic_inc(&tgtp->rcv_fcp_cmd_out);
 		atomic_inc(&tgtp->defer_fod);
+		spin_lock_irqsave(&ctxp->ctxlock, iflags);
+		if (ctxp->flag & LPFC_NVMET_CTX_REUSE_WQ) {
+			spin_unlock_irqrestore(&ctxp->ctxlock, iflags);
+			return;
+		}
+		spin_unlock_irqrestore(&ctxp->ctxlock, iflags);
+		/*
+		 * Post a replacement DMA buffer to RQ and defer
+		 * freeing rcv buffer till .defer_rcv callback
+		 */
+		qno = nvmebuf->idx;
+		lpfc_post_rq_buffer(
+			phba, phba->sli4_hba.nvmet_mrq_hdr[qno],
+			phba->sli4_hba.nvmet_mrq_data[qno], 1, qno);
 		return;
 	}
 	atomic_inc(&tgtp->rcv_fcp_cmd_drop);
@@ -1996,7 +2021,6 @@ lpfc_nvmet_unsol_fcp_buffer(struct lpfc_hba *phba,
 	struct fc_frame_header *fc_hdr;
 	struct lpfc_nvmet_ctxbuf *ctx_buf;
 	struct lpfc_nvmet_ctx_info *current_infop;
-	uint32_t *payload;
 	uint32_t size, oxid, sid, qno;
 	unsigned long iflag;
 	int current_cpu;
@@ -2020,7 +2044,7 @@ lpfc_nvmet_unsol_fcp_buffer(struct lpfc_hba *phba,
 	 * be empty, thus it would need to be replenished with the
 	 * context list from another CPU for this MRQ.
 	 */
-	current_cpu = smp_processor_id();
+	current_cpu = raw_smp_processor_id();
 	current_infop = lpfc_get_ctx_list(phba, current_cpu, idx);
 	spin_lock_irqsave(&current_infop->nvmet_ctx_list_lock, iflag);
 	if (current_infop->nvmet_ctx_list_cnt) {
@@ -2050,7 +2074,7 @@ lpfc_nvmet_unsol_fcp_buffer(struct lpfc_hba *phba,
 #endif
 
 	lpfc_nvmeio_data(phba, "NVMET FCP  RCV: xri x%x sz %d CPU %02x\n",
-			 oxid, size, smp_processor_id());
+			 oxid, size, raw_smp_processor_id());
 
 	tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
 
@@ -2074,7 +2098,6 @@ lpfc_nvmet_unsol_fcp_buffer(struct lpfc_hba *phba,
 		return;
 	}
 
-	payload = (uint32_t *)(nvmebuf->dbuf.virt);
 	sid = sli4_sid_from_fc_hdr(fc_hdr);
 
 	ctxp = (struct lpfc_nvmet_rcv_ctx *)ctx_buf->context;
@@ -2690,12 +2713,11 @@ lpfc_nvmet_sol_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
 {
 	struct lpfc_nvmet_rcv_ctx *ctxp;
 	struct lpfc_nvmet_tgtport *tgtp;
-	uint32_t status, result;
+	uint32_t result;
 	unsigned long flags;
 	bool released = false;
 
 	ctxp = cmdwqe->context2;
-	status = bf_get(lpfc_wcqe_c_status, wcqe);
 	result = wcqe->parameter;
 
 	tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
@@ -2761,11 +2783,10 @@ lpfc_nvmet_unsol_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
 	struct lpfc_nvmet_rcv_ctx *ctxp;
 	struct lpfc_nvmet_tgtport *tgtp;
 	unsigned long flags;
-	uint32_t status, result;
+	uint32_t result;
 	bool released = false;
 
 	ctxp = cmdwqe->context2;
-	status = bf_get(lpfc_wcqe_c_status, wcqe);
 	result = wcqe->parameter;
 
 	if (!ctxp) {
@@ -2842,10 +2863,9 @@ lpfc_nvmet_xmt_ls_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
 {
 	struct lpfc_nvmet_rcv_ctx *ctxp;
 	struct lpfc_nvmet_tgtport *tgtp;
-	uint32_t status, result;
+	uint32_t result;
 
 	ctxp = cmdwqe->context2;
-	status = bf_get(lpfc_wcqe_c_status, wcqe);
 	result = wcqe->parameter;
 
 	tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
@@ -3200,7 +3220,6 @@ lpfc_nvmet_unsol_ls_issue_abort(struct lpfc_hba *phba,
 {
 	struct lpfc_nvmet_tgtport *tgtp;
 	struct lpfc_iocbq *abts_wqeq;
-	union lpfc_wqe128 *wqe_abts;
 	unsigned long flags;
 	int rc;
 
@@ -3230,7 +3249,6 @@ lpfc_nvmet_unsol_ls_issue_abort(struct lpfc_hba *phba,
 		}
 	}
 	abts_wqeq = ctxp->wqeq;
-	wqe_abts = &abts_wqeq->wqe;
 
 	if (lpfc_nvmet_unsol_issue_abort(phba, ctxp, sid, xri) == 0) {
 		rc = WQE_BUSY;
diff --git a/drivers/scsi/lpfc/lpfc_nvmet.h b/drivers/scsi/lpfc/lpfc_nvmet.h
index 368deea2bcf8..2f3f603d94c4 100644
--- a/drivers/scsi/lpfc/lpfc_nvmet.h
+++ b/drivers/scsi/lpfc/lpfc_nvmet.h
@@ -27,10 +27,11 @@
 #define LPFC_NVMET_RQE_DEF_COUNT	2048
 #define LPFC_NVMET_SUCCESS_LEN		12
 
-#define LPFC_NVMET_MRQ_OFF		0xffff
 #define LPFC_NVMET_MRQ_AUTO		0
 #define LPFC_NVMET_MRQ_MAX		16
 
+#define LPFC_NVMET_WAIT_TMO		(5 * MSEC_PER_SEC)
+
 /* Used for NVME Target */
 struct lpfc_nvmet_tgtport {
 	struct lpfc_hba *phba;
diff --git a/drivers/scsi/lpfc/lpfc_scsi.c b/drivers/scsi/lpfc/lpfc_scsi.c
index ff3c5e0f4e2b..ba996fbde89b 100644
--- a/drivers/scsi/lpfc/lpfc_scsi.c
+++ b/drivers/scsi/lpfc/lpfc_scsi.c
@@ -688,7 +688,7 @@ lpfc_get_scsi_buf_s4(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
 	uint32_t sgl_size, cpu, idx;
 	int tag;
 
-	cpu = smp_processor_id();
+	cpu = raw_smp_processor_id();
 	if (cmnd && phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_HDWQ) {
 		tag = blk_mq_unique_tag(cmnd->request);
 		idx = blk_mq_unique_tag_to_hwq(tag);
@@ -3669,8 +3669,8 @@ lpfc_scsi_cmd_iocb_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pIocbIn,
 
 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
 	if (phba->cpucheck_on & LPFC_CHECK_SCSI_IO) {
-		cpu = smp_processor_id();
-		if (cpu < LPFC_CHECK_CPU_CNT)
+		cpu = raw_smp_processor_id();
+		if (cpu < LPFC_CHECK_CPU_CNT && phba->sli4_hba.hdwq)
 			phba->sli4_hba.hdwq[idx].cpucheck_cmpl_io[cpu]++;
 	}
 #endif
@@ -4463,7 +4463,7 @@ lpfc_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *cmnd)
 
 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
 	if (phba->cpucheck_on & LPFC_CHECK_SCSI_IO) {
-		cpu = smp_processor_id();
+		cpu = raw_smp_processor_id();
 		if (cpu < LPFC_CHECK_CPU_CNT) {
 			struct lpfc_sli4_hdw_queue *hdwq =
 					&phba->sli4_hba.hdwq[lpfc_cmd->hdwq_no];
@@ -5048,7 +5048,7 @@ lpfc_device_reset_handler(struct scsi_cmnd *cmnd)
 	rdata = lpfc_rport_data_from_scsi_device(cmnd->device);
 	if (!rdata || !rdata->pnode) {
 		lpfc_printf_vlog(vport, KERN_ERR, LOG_FCP,
-				 "0798 Device Reset rport failure: rdata x%p\n",
+				 "0798 Device Reset rdata failure: rdata x%p\n",
 				 rdata);
 		return FAILED;
 	}
@@ -5117,9 +5117,10 @@ lpfc_target_reset_handler(struct scsi_cmnd *cmnd)
 	int status;
 
 	rdata = lpfc_rport_data_from_scsi_device(cmnd->device);
-	if (!rdata) {
+	if (!rdata || !rdata->pnode) {
 		lpfc_printf_vlog(vport, KERN_ERR, LOG_FCP,
-			"0799 Target Reset rport failure: rdata x%p\n", rdata);
+				 "0799 Target Reset rdata failure: rdata x%p\n",
+				 rdata);
 		return FAILED;
 	}
 	pnode = rdata->pnode;
diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c
index 57b4a463b589..2acda188b0dc 100644
--- a/drivers/scsi/lpfc/lpfc_sli.c
+++ b/drivers/scsi/lpfc/lpfc_sli.c
@@ -87,9 +87,6 @@ static void lpfc_sli4_hba_handle_eqe(struct lpfc_hba *phba,
 				     struct lpfc_eqe *eqe);
 static bool lpfc_sli4_mbox_completions_pending(struct lpfc_hba *phba);
 static bool lpfc_sli4_process_missed_mbox_completions(struct lpfc_hba *phba);
-static int lpfc_sli4_abort_nvme_io(struct lpfc_hba *phba,
-				   struct lpfc_sli_ring *pring,
-				   struct lpfc_iocbq *cmdiocb);
 
 static IOCB_t *
 lpfc_get_iocb_from_iocbq(struct lpfc_iocbq *iocbq)
@@ -151,7 +148,7 @@ lpfc_sli4_wq_put(struct lpfc_queue *q, union lpfc_wqe128 *wqe)
 	/* sanity check on queue memory */
 	if (unlikely(!q))
 		return -ENOMEM;
-	temp_wqe = q->qe[q->host_index].wqe;
+	temp_wqe = lpfc_sli4_qe(q, q->host_index);
 
 	/* If the host has not yet processed the next entry then we are done */
 	idx = ((q->host_index + 1) % q->entry_count);
@@ -271,7 +268,7 @@ lpfc_sli4_mq_put(struct lpfc_queue *q, struct lpfc_mqe *mqe)
 	/* sanity check on queue memory */
 	if (unlikely(!q))
 		return -ENOMEM;
-	temp_mqe = q->qe[q->host_index].mqe;
+	temp_mqe = lpfc_sli4_qe(q, q->host_index);
 
 	/* If the host has not yet processed the next entry then we are done */
 	if (((q->host_index + 1) % q->entry_count) == q->hba_index)
@@ -331,7 +328,7 @@ lpfc_sli4_eq_get(struct lpfc_queue *q)
 	/* sanity check on queue memory */
 	if (unlikely(!q))
 		return NULL;
-	eqe = q->qe[q->host_index].eqe;
+	eqe = lpfc_sli4_qe(q, q->host_index);
 
 	/* If the next EQE is not valid then we are done */
 	if (bf_get_le32(lpfc_eqe_valid, eqe) != q->qe_valid)
@@ -355,7 +352,7 @@ lpfc_sli4_eq_get(struct lpfc_queue *q)
  * @q: The Event Queue to disable interrupts
  *
  **/
-inline void
+void
 lpfc_sli4_eq_clr_intr(struct lpfc_queue *q)
 {
 	struct lpfc_register doorbell;
@@ -374,7 +371,7 @@ lpfc_sli4_eq_clr_intr(struct lpfc_queue *q)
  * @q: The Event Queue to disable interrupts
  *
  **/
-inline void
+void
 lpfc_sli4_if6_eq_clr_intr(struct lpfc_queue *q)
 {
 	struct lpfc_register doorbell;
@@ -545,7 +542,7 @@ lpfc_sli4_cq_get(struct lpfc_queue *q)
 	/* sanity check on queue memory */
 	if (unlikely(!q))
 		return NULL;
-	cqe = q->qe[q->host_index].cqe;
+	cqe = lpfc_sli4_qe(q, q->host_index);
 
 	/* If the next CQE is not valid then we are done */
 	if (bf_get_le32(lpfc_cqe_valid, cqe) != q->qe_valid)
@@ -667,8 +664,8 @@ lpfc_sli4_rq_put(struct lpfc_queue *hq, struct lpfc_queue *dq,
 		return -ENOMEM;
 	hq_put_index = hq->host_index;
 	dq_put_index = dq->host_index;
-	temp_hrqe = hq->qe[hq_put_index].rqe;
-	temp_drqe = dq->qe[dq_put_index].rqe;
+	temp_hrqe = lpfc_sli4_qe(hq, hq_put_index);
+	temp_drqe = lpfc_sli4_qe(dq, dq_put_index);
 
 	if (hq->type != LPFC_HRQ || dq->type != LPFC_DRQ)
 		return -EINVAL;
@@ -907,10 +904,10 @@ lpfc_handle_rrq_active(struct lpfc_hba *phba)
 		mod_timer(&phba->rrq_tmr, next_time);
 	list_for_each_entry_safe(rrq, nextrrq, &send_rrq, list) {
 		list_del(&rrq->list);
-		if (!rrq->send_rrq)
+		if (!rrq->send_rrq) {
 			/* this call will free the rrq */
-		lpfc_clr_rrq_active(phba, rrq->xritag, rrq);
-		else if (lpfc_send_rrq(phba, rrq)) {
+			lpfc_clr_rrq_active(phba, rrq->xritag, rrq);
+		} else if (lpfc_send_rrq(phba, rrq)) {
 			/* if we send the rrq then the completion handler
 			*  will clear the bit in the xribitmap.
 			*/
@@ -2502,8 +2499,8 @@ lpfc_sli_def_mbox_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb)
 			} else {
 				ndlp->nlp_flag &= ~NLP_UNREG_INP;
 			}
+			pmb->ctx_ndlp = NULL;
 		}
-		pmb->ctx_ndlp = NULL;
 	}
 
 	/* Check security permission status on INIT_LINK mailbox command */
@@ -3922,33 +3919,6 @@ lpfc_sli_abort_iocb_ring(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
 }
 
 /**
- * lpfc_sli_abort_wqe_ring - Abort all iocbs in the ring
- * @phba: Pointer to HBA context object.
- * @pring: Pointer to driver SLI ring object.
- *
- * This function aborts all iocbs in the given ring and frees all the iocb
- * objects in txq. This function issues an abort iocb for all the iocb commands
- * in txcmplq. The iocbs in the txcmplq is not guaranteed to complete before
- * the return of this function. The caller is not required to hold any locks.
- **/
-void
-lpfc_sli_abort_wqe_ring(struct lpfc_hba *phba, struct lpfc_sli_ring *pring)
-{
-	LIST_HEAD(completions);
-	struct lpfc_iocbq *iocb, *next_iocb;
-
-	if (pring->ringno == LPFC_ELS_RING)
-		lpfc_fabric_abort_hba(phba);
-
-	spin_lock_irq(&phba->hbalock);
-	/* Next issue ABTS for everything on the txcmplq */
-	list_for_each_entry_safe(iocb, next_iocb, &pring->txcmplq, list)
-		lpfc_sli4_abort_nvme_io(phba, pring, iocb);
-	spin_unlock_irq(&phba->hbalock);
-}
-
-
-/**
  * lpfc_sli_abort_fcp_rings - Abort all iocbs in all FCP rings
  * @phba: Pointer to HBA context object.
  * @pring: Pointer to driver SLI ring object.
@@ -3978,33 +3948,6 @@ lpfc_sli_abort_fcp_rings(struct lpfc_hba *phba)
 }
 
 /**
- * lpfc_sli_abort_nvme_rings - Abort all wqes in all NVME rings
- * @phba: Pointer to HBA context object.
- *
- * This function aborts all wqes in NVME rings. This function issues an
- * abort wqe for all the outstanding IO commands in txcmplq. The iocbs in
- * the txcmplq is not guaranteed to complete before the return of this
- * function. The caller is not required to hold any locks.
- **/
-void
-lpfc_sli_abort_nvme_rings(struct lpfc_hba *phba)
-{
-	struct lpfc_sli_ring  *pring;
-	uint32_t i;
-
-	if ((phba->sli_rev < LPFC_SLI_REV4) ||
-	    !(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME))
-		return;
-
-	/* Abort all IO on each NVME ring. */
-	for (i = 0; i < phba->cfg_hdw_queue; i++) {
-		pring = phba->sli4_hba.hdwq[i].nvme_wq->pring;
-		lpfc_sli_abort_wqe_ring(phba, pring);
-	}
-}
-
-
-/**
  * lpfc_sli_flush_fcp_rings - flush all iocbs in the fcp ring
  * @phba: Pointer to HBA context object.
  *
@@ -4487,7 +4430,9 @@ lpfc_sli_brdreset(struct lpfc_hba *phba)
 	}
 
 	/* Turn off parity checking and serr during the physical reset */
-	pci_read_config_word(phba->pcidev, PCI_COMMAND, &cfg_value);
+	if (pci_read_config_word(phba->pcidev, PCI_COMMAND, &cfg_value))
+		return -EIO;
+
 	pci_write_config_word(phba->pcidev, PCI_COMMAND,
 			      (cfg_value &
 			       ~(PCI_COMMAND_PARITY | PCI_COMMAND_SERR)));
@@ -4564,7 +4509,12 @@ lpfc_sli4_brdreset(struct lpfc_hba *phba)
 			"0389 Performing PCI function reset!\n");
 
 	/* Turn off parity checking and serr during the physical reset */
-	pci_read_config_word(phba->pcidev, PCI_COMMAND, &cfg_value);
+	if (pci_read_config_word(phba->pcidev, PCI_COMMAND, &cfg_value)) {
+		lpfc_printf_log(phba, KERN_INFO, LOG_INIT,
+				"3205 PCI read Config failed\n");
+		return -EIO;
+	}
+
 	pci_write_config_word(phba->pcidev, PCI_COMMAND, (cfg_value &
 			      ~(PCI_COMMAND_PARITY | PCI_COMMAND_SERR)));
 
@@ -5395,7 +5345,7 @@ lpfc_sli4_read_rev(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq,
 }
 
 /**
- * lpfc_sli4_retrieve_pport_name - Retrieve SLI4 device physical port name
+ * lpfc_sli4_get_ctl_attr - Retrieve SLI4 device controller attributes
  * @phba: pointer to lpfc hba data structure.
  *
  * This routine retrieves SLI4 device physical port name this PCI function
@@ -5403,40 +5353,30 @@ lpfc_sli4_read_rev(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq,
  *
  * Return codes
  *      0 - successful
- *      otherwise - failed to retrieve physical port name
+ *      otherwise - failed to retrieve controller attributes
  **/
 static int
-lpfc_sli4_retrieve_pport_name(struct lpfc_hba *phba)
+lpfc_sli4_get_ctl_attr(struct lpfc_hba *phba)
 {
 	LPFC_MBOXQ_t *mboxq;
 	struct lpfc_mbx_get_cntl_attributes *mbx_cntl_attr;
 	struct lpfc_controller_attribute *cntl_attr;
-	struct lpfc_mbx_get_port_name *get_port_name;
 	void *virtaddr = NULL;
 	uint32_t alloclen, reqlen;
 	uint32_t shdr_status, shdr_add_status;
 	union lpfc_sli4_cfg_shdr *shdr;
-	char cport_name = 0;
 	int rc;
 
-	/* We assume nothing at this point */
-	phba->sli4_hba.lnk_info.lnk_dv = LPFC_LNK_DAT_INVAL;
-	phba->sli4_hba.pport_name_sta = LPFC_SLI4_PPNAME_NON;
-
 	mboxq = (LPFC_MBOXQ_t *)mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
 	if (!mboxq)
 		return -ENOMEM;
-	/* obtain link type and link number via READ_CONFIG */
-	phba->sli4_hba.lnk_info.lnk_dv = LPFC_LNK_DAT_INVAL;
-	lpfc_sli4_read_config(phba);
-	if (phba->sli4_hba.lnk_info.lnk_dv == LPFC_LNK_DAT_VAL)
-		goto retrieve_ppname;
 
-	/* obtain link type and link number via COMMON_GET_CNTL_ATTRIBUTES */
+	/* Send COMMON_GET_CNTL_ATTRIBUTES mbox cmd */
 	reqlen = sizeof(struct lpfc_mbx_get_cntl_attributes);
 	alloclen = lpfc_sli4_config(phba, mboxq, LPFC_MBOX_SUBSYSTEM_COMMON,
 			LPFC_MBOX_OPCODE_GET_CNTL_ATTRIBUTES, reqlen,
 			LPFC_SLI4_MBX_NEMBED);
+
 	if (alloclen < reqlen) {
 		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
 				"3084 Allocated DMA memory size (%d) is "
@@ -5462,16 +5402,71 @@ lpfc_sli4_retrieve_pport_name(struct lpfc_hba *phba)
 		rc = -ENXIO;
 		goto out_free_mboxq;
 	}
+
 	cntl_attr = &mbx_cntl_attr->cntl_attr;
 	phba->sli4_hba.lnk_info.lnk_dv = LPFC_LNK_DAT_VAL;
 	phba->sli4_hba.lnk_info.lnk_tp =
 		bf_get(lpfc_cntl_attr_lnk_type, cntl_attr);
 	phba->sli4_hba.lnk_info.lnk_no =
 		bf_get(lpfc_cntl_attr_lnk_numb, cntl_attr);
+
+	memset(phba->BIOSVersion, 0, sizeof(phba->BIOSVersion));
+	strlcat(phba->BIOSVersion, (char *)cntl_attr->bios_ver_str,
+		sizeof(phba->BIOSVersion));
+
 	lpfc_printf_log(phba, KERN_INFO, LOG_SLI,
-			"3086 lnk_type:%d, lnk_numb:%d\n",
+			"3086 lnk_type:%d, lnk_numb:%d, bios_ver:%s\n",
 			phba->sli4_hba.lnk_info.lnk_tp,
-			phba->sli4_hba.lnk_info.lnk_no);
+			phba->sli4_hba.lnk_info.lnk_no,
+			phba->BIOSVersion);
+out_free_mboxq:
+	if (rc != MBX_TIMEOUT) {
+		if (bf_get(lpfc_mqe_command, &mboxq->u.mqe) == MBX_SLI4_CONFIG)
+			lpfc_sli4_mbox_cmd_free(phba, mboxq);
+		else
+			mempool_free(mboxq, phba->mbox_mem_pool);
+	}
+	return rc;
+}
+
+/**
+ * lpfc_sli4_retrieve_pport_name - Retrieve SLI4 device physical port name
+ * @phba: pointer to lpfc hba data structure.
+ *
+ * This routine retrieves SLI4 device physical port name this PCI function
+ * is attached to.
+ *
+ * Return codes
+ *      0 - successful
+ *      otherwise - failed to retrieve physical port name
+ **/
+static int
+lpfc_sli4_retrieve_pport_name(struct lpfc_hba *phba)
+{
+	LPFC_MBOXQ_t *mboxq;
+	struct lpfc_mbx_get_port_name *get_port_name;
+	uint32_t shdr_status, shdr_add_status;
+	union lpfc_sli4_cfg_shdr *shdr;
+	char cport_name = 0;
+	int rc;
+
+	/* We assume nothing at this point */
+	phba->sli4_hba.lnk_info.lnk_dv = LPFC_LNK_DAT_INVAL;
+	phba->sli4_hba.pport_name_sta = LPFC_SLI4_PPNAME_NON;
+
+	mboxq = (LPFC_MBOXQ_t *)mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL);
+	if (!mboxq)
+		return -ENOMEM;
+	/* obtain link type and link number via READ_CONFIG */
+	phba->sli4_hba.lnk_info.lnk_dv = LPFC_LNK_DAT_INVAL;
+	lpfc_sli4_read_config(phba);
+	if (phba->sli4_hba.lnk_info.lnk_dv == LPFC_LNK_DAT_VAL)
+		goto retrieve_ppname;
+
+	/* obtain link type and link number via COMMON_GET_CNTL_ATTRIBUTES */
+	rc = lpfc_sli4_get_ctl_attr(phba);
+	if (rc)
+		goto out_free_mboxq;
 
 retrieve_ppname:
 	lpfc_sli4_config(phba, mboxq, LPFC_MBOX_SUBSYSTEM_COMMON,
@@ -7047,7 +7042,7 @@ lpfc_sli4_repost_sgl_list(struct lpfc_hba *phba,
  *
  * Returns: 0 = success, non-zero failure.
  **/
-int
+static int
 lpfc_sli4_repost_io_sgl_list(struct lpfc_hba *phba)
 {
 	LIST_HEAD(post_nblist);
@@ -7067,7 +7062,7 @@ lpfc_sli4_repost_io_sgl_list(struct lpfc_hba *phba)
 	return rc;
 }
 
-void
+static void
 lpfc_set_host_data(struct lpfc_hba *phba, LPFC_MBOXQ_t *mbox)
 {
 	uint32_t len;
@@ -7250,6 +7245,12 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba)
 				"3080 Successful retrieving SLI4 device "
 				"physical port name: %s.\n", phba->Port);
 
+	rc = lpfc_sli4_get_ctl_attr(phba);
+	if (!rc)
+		lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI,
+				"8351 Successful retrieving SLI4 device "
+				"CTL ATTR\n");
+
 	/*
 	 * Evaluate the read rev and vpd data. Populate the driver
 	 * state with the results. If this routine fails, the failure
@@ -7652,12 +7653,6 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba)
 		phba->cfg_xri_rebalancing = 0;
 	}
 
-	/* Arm the CQs and then EQs on device */
-	lpfc_sli4_arm_cqeq_intr(phba);
-
-	/* Indicate device interrupt mode */
-	phba->sli4_hba.intr_enable = 1;
-
 	/* Allow asynchronous mailbox command to go through */
 	spin_lock_irq(&phba->hbalock);
 	phba->sli.sli_flag &= ~LPFC_SLI_ASYNC_MBX_BLK;
@@ -7726,6 +7721,12 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba)
 		phba->trunk_link.link3.state = LPFC_LINK_DOWN;
 	spin_unlock_irq(&phba->hbalock);
 
+	/* Arm the CQs and then EQs on device */
+	lpfc_sli4_arm_cqeq_intr(phba);
+
+	/* Indicate device interrupt mode */
+	phba->sli4_hba.intr_enable = 1;
+
 	if (!(phba->hba_flag & HBA_FCOE_MODE) &&
 	    (phba->hba_flag & LINK_DISABLED)) {
 		lpfc_printf_log(phba, KERN_ERR, LOG_INIT | LOG_SLI,
@@ -7820,8 +7821,9 @@ lpfc_sli4_mbox_completions_pending(struct lpfc_hba *phba)
 	mcq = phba->sli4_hba.mbx_cq;
 	idx = mcq->hba_index;
 	qe_valid = mcq->qe_valid;
-	while (bf_get_le32(lpfc_cqe_valid, mcq->qe[idx].cqe) == qe_valid) {
-		mcqe = (struct lpfc_mcqe *)mcq->qe[idx].cqe;
+	while (bf_get_le32(lpfc_cqe_valid,
+	       (struct lpfc_cqe *)lpfc_sli4_qe(mcq, idx)) == qe_valid) {
+		mcqe = (struct lpfc_mcqe *)(lpfc_sli4_qe(mcq, idx));
 		if (bf_get_le32(lpfc_trailer_completed, mcqe) &&
 		    (!bf_get_le32(lpfc_trailer_async, mcqe))) {
 			pending_completions = true;
@@ -8500,7 +8502,7 @@ lpfc_sli4_wait_bmbx_ready(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq)
 		bmbx_reg.word0 = readl(phba->sli4_hba.BMBXregaddr);
 		db_ready = bf_get(lpfc_bmbx_rdy, &bmbx_reg);
 		if (!db_ready)
-			msleep(2);
+			mdelay(2);
 
 		if (time_after(jiffies, timeout))
 			return MBXERR_ERROR;
@@ -11264,102 +11266,6 @@ abort_iotag_exit:
 }
 
 /**
- * lpfc_sli4_abort_nvme_io - Issue abort for a command iocb
- * @phba: Pointer to HBA context object.
- * @pring: Pointer to driver SLI ring object.
- * @cmdiocb: Pointer to driver command iocb object.
- *
- * This function issues an abort iocb for the provided command iocb down to
- * the port. Other than the case the outstanding command iocb is an abort
- * request, this function issues abort out unconditionally. This function is
- * called with hbalock held. The function returns 0 when it fails due to
- * memory allocation failure or when the command iocb is an abort request.
- **/
-static int
-lpfc_sli4_abort_nvme_io(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
-			struct lpfc_iocbq *cmdiocb)
-{
-	struct lpfc_vport *vport = cmdiocb->vport;
-	struct lpfc_iocbq *abtsiocbp;
-	union lpfc_wqe128 *abts_wqe;
-	int retval;
-	int idx = cmdiocb->hba_wqidx;
-
-	/*
-	 * There are certain command types we don't want to abort.  And we
-	 * don't want to abort commands that are already in the process of
-	 * being aborted.
-	 */
-	if (cmdiocb->iocb.ulpCommand == CMD_ABORT_XRI_CN ||
-	    cmdiocb->iocb.ulpCommand == CMD_CLOSE_XRI_CN ||
-	    (cmdiocb->iocb_flag & LPFC_DRIVER_ABORTED) != 0)
-		return 0;
-
-	/* issue ABTS for this io based on iotag */
-	abtsiocbp = __lpfc_sli_get_iocbq(phba);
-	if (abtsiocbp == NULL)
-		return 0;
-
-	/* This signals the response to set the correct status
-	 * before calling the completion handler
-	 */
-	cmdiocb->iocb_flag |= LPFC_DRIVER_ABORTED;
-
-	/* Complete prepping the abort wqe and issue to the FW. */
-	abts_wqe = &abtsiocbp->wqe;
-
-	/* Clear any stale WQE contents */
-	memset(abts_wqe, 0, sizeof(union lpfc_wqe));
-	bf_set(abort_cmd_criteria, &abts_wqe->abort_cmd, T_XRI_TAG);
-
-	/* word 7 */
-	bf_set(wqe_cmnd, &abts_wqe->abort_cmd.wqe_com, CMD_ABORT_XRI_CX);
-	bf_set(wqe_class, &abts_wqe->abort_cmd.wqe_com,
-	       cmdiocb->iocb.ulpClass);
-
-	/* word 8 - tell the FW to abort the IO associated with this
-	 * outstanding exchange ID.
-	 */
-	abts_wqe->abort_cmd.wqe_com.abort_tag = cmdiocb->sli4_xritag;
-
-	/* word 9 - this is the iotag for the abts_wqe completion. */
-	bf_set(wqe_reqtag, &abts_wqe->abort_cmd.wqe_com,
-	       abtsiocbp->iotag);
-
-	/* word 10 */
-	bf_set(wqe_qosd, &abts_wqe->abort_cmd.wqe_com, 1);
-	bf_set(wqe_lenloc, &abts_wqe->abort_cmd.wqe_com, LPFC_WQE_LENLOC_NONE);
-
-	/* word 11 */
-	bf_set(wqe_cmd_type, &abts_wqe->abort_cmd.wqe_com, OTHER_COMMAND);
-	bf_set(wqe_wqec, &abts_wqe->abort_cmd.wqe_com, 1);
-	bf_set(wqe_cqid, &abts_wqe->abort_cmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
-
-	/* ABTS WQE must go to the same WQ as the WQE to be aborted */
-	abtsiocbp->iocb_flag |= LPFC_IO_NVME;
-	abtsiocbp->vport = vport;
-	abtsiocbp->wqe_cmpl = lpfc_nvme_abort_fcreq_cmpl;
-	retval = lpfc_sli4_issue_wqe(phba, &phba->sli4_hba.hdwq[idx],
-				     abtsiocbp);
-	if (retval) {
-		lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME,
-				 "6147 Failed abts issue_wqe with status x%x "
-				 "for oxid x%x\n",
-				 retval, cmdiocb->sli4_xritag);
-		lpfc_sli_release_iocbq(phba, abtsiocbp);
-		return retval;
-	}
-
-	lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME,
-			 "6148 Drv Abort NVME Request Issued for "
-			 "ox_id x%x on reqtag x%x\n",
-			 cmdiocb->sli4_xritag,
-			 abtsiocbp->iotag);
-
-	return retval;
-}
-
-/**
  * lpfc_sli_hba_iocb_abort - Abort all iocbs to an hba.
  * @phba: pointer to lpfc HBA data structure.
  *
@@ -13636,7 +13542,7 @@ lpfc_sli4_sp_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe,
 		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
 				"0390 Cannot schedule soft IRQ "
 				"for CQ eqcqid=%d, cqid=%d on CPU %d\n",
-				cqid, cq->queue_id, smp_processor_id());
+				cqid, cq->queue_id, raw_smp_processor_id());
 }
 
 /**
@@ -14019,7 +13925,7 @@ lpfc_sli4_nvmet_handle_rcqe(struct lpfc_hba *phba, struct lpfc_queue *cq,
 			return false;
 		}
 drop:
-		lpfc_in_buf_free(phba, &dma_buf->dbuf);
+		lpfc_rq_buf_free(phba, &dma_buf->hbuf);
 		break;
 	case FC_STATUS_INSUFF_BUF_FRM_DISC:
 		if (phba->nvmet_support) {
@@ -14185,7 +14091,7 @@ work_cq:
 		lpfc_printf_log(phba, KERN_ERR, LOG_SLI,
 				"0363 Cannot schedule soft IRQ "
 				"for CQ eqcqid=%d, cqid=%d on CPU %d\n",
-				cqid, cq->queue_id, smp_processor_id());
+				cqid, cq->queue_id, raw_smp_processor_id());
 }
 
 /**
@@ -14324,7 +14230,7 @@ lpfc_sli4_hba_intr_handler(int irq, void *dev_id)
 
 	eqi = phba->sli4_hba.eq_info;
 	icnt = this_cpu_inc_return(eqi->icnt);
-	fpeq->last_cpu = smp_processor_id();
+	fpeq->last_cpu = raw_smp_processor_id();
 
 	if (icnt > LPFC_EQD_ISR_TRIGGER &&
 	    phba->cfg_irq_chann == 1 &&
@@ -14410,6 +14316,9 @@ lpfc_sli4_queue_free(struct lpfc_queue *queue)
 	if (!queue)
 		return;
 
+	if (!list_empty(&queue->wq_list))
+		list_del(&queue->wq_list);
+
 	while (!list_empty(&queue->page_list)) {
 		list_remove_head(&queue->page_list, dmabuf, struct lpfc_dmabuf,
 				 list);
@@ -14425,9 +14334,6 @@ lpfc_sli4_queue_free(struct lpfc_queue *queue)
 	if (!list_empty(&queue->cpu_list))
 		list_del(&queue->cpu_list);
 
-	if (!list_empty(&queue->wq_list))
-		list_del(&queue->wq_list);
-
 	kfree(queue);
 	return;
 }
@@ -14438,6 +14344,7 @@ lpfc_sli4_queue_free(struct lpfc_queue *queue)
  * @page_size: The size of a queue page
  * @entry_size: The size of each queue entry for this queue.
  * @entry count: The number of entries that this queue will handle.
+ * @cpu: The cpu that will primarily utilize this queue.
  *
  * This function allocates a queue structure and the DMAable memory used for
  * the host resident queue. This function must be called before creating the
@@ -14445,28 +14352,26 @@ lpfc_sli4_queue_free(struct lpfc_queue *queue)
  **/
 struct lpfc_queue *
 lpfc_sli4_queue_alloc(struct lpfc_hba *phba, uint32_t page_size,
-		      uint32_t entry_size, uint32_t entry_count)
+		      uint32_t entry_size, uint32_t entry_count, int cpu)
 {
 	struct lpfc_queue *queue;
 	struct lpfc_dmabuf *dmabuf;
-	int x, total_qe_count;
-	void *dma_pointer;
 	uint32_t hw_page_size = phba->sli4_hba.pc_sli4_params.if_page_sz;
+	uint16_t x, pgcnt;
 
 	if (!phba->sli4_hba.pc_sli4_params.supported)
 		hw_page_size = page_size;
 
-	queue = kzalloc(sizeof(struct lpfc_queue) +
-			(sizeof(union sli4_qe) * entry_count), GFP_KERNEL);
-	if (!queue)
-		return NULL;
-	queue->page_count = (ALIGN(entry_size * entry_count,
-			hw_page_size))/hw_page_size;
+	pgcnt = ALIGN(entry_size * entry_count, hw_page_size) / hw_page_size;
 
 	/* If needed, Adjust page count to match the max the adapter supports */
-	if (phba->sli4_hba.pc_sli4_params.wqpcnt &&
-	    (queue->page_count > phba->sli4_hba.pc_sli4_params.wqpcnt))
-		queue->page_count = phba->sli4_hba.pc_sli4_params.wqpcnt;
+	if (pgcnt > phba->sli4_hba.pc_sli4_params.wqpcnt)
+		pgcnt = phba->sli4_hba.pc_sli4_params.wqpcnt;
+
+	queue = kzalloc_node(sizeof(*queue) + (sizeof(void *) * pgcnt),
+			     GFP_KERNEL, cpu_to_node(cpu));
+	if (!queue)
+		return NULL;
 
 	INIT_LIST_HEAD(&queue->list);
 	INIT_LIST_HEAD(&queue->wq_list);
@@ -14478,13 +14383,17 @@ lpfc_sli4_queue_alloc(struct lpfc_hba *phba, uint32_t page_size,
 	/* Set queue parameters now.  If the system cannot provide memory
 	 * resources, the free routine needs to know what was allocated.
 	 */
+	queue->page_count = pgcnt;
+	queue->q_pgs = (void **)&queue[1];
+	queue->entry_cnt_per_pg = hw_page_size / entry_size;
 	queue->entry_size = entry_size;
 	queue->entry_count = entry_count;
 	queue->page_size = hw_page_size;
 	queue->phba = phba;
 
-	for (x = 0, total_qe_count = 0; x < queue->page_count; x++) {
-		dmabuf = kzalloc(sizeof(struct lpfc_dmabuf), GFP_KERNEL);
+	for (x = 0; x < queue->page_count; x++) {
+		dmabuf = kzalloc_node(sizeof(*dmabuf), GFP_KERNEL,
+				      dev_to_node(&phba->pcidev->dev));
 		if (!dmabuf)
 			goto out_fail;
 		dmabuf->virt = dma_alloc_coherent(&phba->pcidev->dev,
@@ -14496,13 +14405,8 @@ lpfc_sli4_queue_alloc(struct lpfc_hba *phba, uint32_t page_size,
 		}
 		dmabuf->buffer_tag = x;
 		list_add_tail(&dmabuf->list, &queue->page_list);
-		/* initialize queue's entry array */
-		dma_pointer = dmabuf->virt;
-		for (; total_qe_count < entry_count &&
-		     dma_pointer < (hw_page_size + dmabuf->virt);
-		     total_qe_count++, dma_pointer += entry_size) {
-			queue->qe[total_qe_count].address = dma_pointer;
-		}
+		/* use lpfc_sli4_qe to index a paritcular entry in this page */
+		queue->q_pgs[x] = dmabuf->virt;
 	}
 	INIT_WORK(&queue->irqwork, lpfc_sli4_hba_process_cq);
 	INIT_WORK(&queue->spwork, lpfc_sli4_sp_process_cq);
diff --git a/drivers/scsi/lpfc/lpfc_sli.h b/drivers/scsi/lpfc/lpfc_sli.h
index 7a1a761efdd6..467b8270f7fd 100644
--- a/drivers/scsi/lpfc/lpfc_sli.h
+++ b/drivers/scsi/lpfc/lpfc_sli.h
@@ -327,6 +327,10 @@ struct lpfc_sli {
 #define LPFC_SLI_ASYNC_MBX_BLK    0x2000 /* Async mailbox is blocked */
 #define LPFC_SLI_SUPPRESS_RSP     0x4000 /* Suppress RSP feature is supported */
 #define LPFC_SLI_USE_EQDR         0x8000 /* EQ Delay Register is supported */
+#define LPFC_QUEUE_FREE_INIT	  0x10000 /* Queue freeing is in progress */
+#define LPFC_QUEUE_FREE_WAIT	  0x20000 /* Hold Queue free as it is being
+					   * used outside worker thread
+					   */
 
 	struct lpfc_sli_ring *sli3_ring;
 
@@ -427,14 +431,13 @@ struct lpfc_io_buf {
 		struct {
 			struct nvmefc_fcp_req *nvmeCmd;
 			uint16_t qidx;
-
-#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
-			uint64_t ts_cmd_start;
-			uint64_t ts_last_cmd;
-			uint64_t ts_cmd_wqput;
-			uint64_t ts_isr_cmpl;
-			uint64_t ts_data_nvme;
-#endif
 		};
 	};
+#ifdef CONFIG_SCSI_LPFC_DEBUG_FS
+	uint64_t ts_cmd_start;
+	uint64_t ts_last_cmd;
+	uint64_t ts_cmd_wqput;
+	uint64_t ts_isr_cmpl;
+	uint64_t ts_data_nvme;
+#endif
 };
diff --git a/drivers/scsi/lpfc/lpfc_sli4.h b/drivers/scsi/lpfc/lpfc_sli4.h
index 40c85091c805..8e4fd1a98023 100644
--- a/drivers/scsi/lpfc/lpfc_sli4.h
+++ b/drivers/scsi/lpfc/lpfc_sli4.h
@@ -117,21 +117,6 @@ enum lpfc_sli4_queue_subtype {
 	LPFC_USOL
 };
 
-union sli4_qe {
-	void *address;
-	struct lpfc_eqe *eqe;
-	struct lpfc_cqe *cqe;
-	struct lpfc_mcqe *mcqe;
-	struct lpfc_wcqe_complete *wcqe_complete;
-	struct lpfc_wcqe_release *wcqe_release;
-	struct sli4_wcqe_xri_aborted *wcqe_xri_aborted;
-	struct lpfc_rcqe_complete *rcqe_complete;
-	struct lpfc_mqe *mqe;
-	union  lpfc_wqe *wqe;
-	union  lpfc_wqe128 *wqe128;
-	struct lpfc_rqe *rqe;
-};
-
 /* RQ buffer list */
 struct lpfc_rqb {
 	uint16_t entry_count;	  /* Current number of RQ slots */
@@ -157,6 +142,7 @@ struct lpfc_queue {
 	struct list_head cpu_list;
 	uint32_t entry_count;	/* Number of entries to support on the queue */
 	uint32_t entry_size;	/* Size of each queue entry. */
+	uint32_t entry_cnt_per_pg;
 	uint32_t notify_interval; /* Queue Notification Interval
 				   * For chip->host queues (EQ, CQ, RQ):
 				   *  specifies the interval (number of
@@ -254,17 +240,17 @@ struct lpfc_queue {
 	uint16_t last_cpu;	/* most recent cpu */
 	uint8_t	qe_valid;
 	struct lpfc_queue *assoc_qp;
-	union sli4_qe qe[1];	/* array to index entries (must be last) */
+	void **q_pgs;	/* array to index entries per page */
 };
 
 struct lpfc_sli4_link {
-	uint16_t speed;
+	uint32_t speed;
 	uint8_t duplex;
 	uint8_t status;
 	uint8_t type;
 	uint8_t number;
 	uint8_t fault;
-	uint16_t logical_speed;
+	uint32_t logical_speed;
 	uint16_t topology;
 };
 
@@ -543,8 +529,9 @@ struct lpfc_sli4_lnk_info {
 #define LPFC_LNK_DAT_INVAL	0
 #define LPFC_LNK_DAT_VAL	1
 	uint8_t lnk_tp;
-#define LPFC_LNK_GE	0x0 /* FCoE */
-#define LPFC_LNK_FC	0x1 /* FC   */
+#define LPFC_LNK_GE		0x0 /* FCoE */
+#define LPFC_LNK_FC		0x1 /* FC */
+#define LPFC_LNK_FC_TRUNKED	0x2 /* FC_Trunked */
 	uint8_t lnk_no;
 	uint8_t optic_state;
 };
@@ -907,6 +894,18 @@ struct lpfc_sli4_hba {
 #define lpfc_conf_trunk_port3_WORD	conf_trunk
 #define lpfc_conf_trunk_port3_SHIFT	3
 #define lpfc_conf_trunk_port3_MASK	0x1
+#define lpfc_conf_trunk_port0_nd_WORD	conf_trunk
+#define lpfc_conf_trunk_port0_nd_SHIFT	4
+#define lpfc_conf_trunk_port0_nd_MASK	0x1
+#define lpfc_conf_trunk_port1_nd_WORD	conf_trunk
+#define lpfc_conf_trunk_port1_nd_SHIFT	5
+#define lpfc_conf_trunk_port1_nd_MASK	0x1
+#define lpfc_conf_trunk_port2_nd_WORD	conf_trunk
+#define lpfc_conf_trunk_port2_nd_SHIFT	6
+#define lpfc_conf_trunk_port2_nd_MASK	0x1
+#define lpfc_conf_trunk_port3_nd_WORD	conf_trunk
+#define lpfc_conf_trunk_port3_nd_SHIFT	7
+#define lpfc_conf_trunk_port3_nd_MASK	0x1
 };
 
 enum lpfc_sge_type {
@@ -990,8 +989,10 @@ int lpfc_sli4_mbx_read_fcf_rec(struct lpfc_hba *, struct lpfcMboxq *,
 			       uint16_t);
 
 void lpfc_sli4_hba_reset(struct lpfc_hba *);
-struct lpfc_queue *lpfc_sli4_queue_alloc(struct lpfc_hba *, uint32_t,
-					 uint32_t, uint32_t);
+struct lpfc_queue *lpfc_sli4_queue_alloc(struct lpfc_hba *phba,
+					 uint32_t page_size,
+					 uint32_t entry_size,
+					 uint32_t entry_count, int cpu);
 void lpfc_sli4_queue_free(struct lpfc_queue *);
 int lpfc_eq_create(struct lpfc_hba *, struct lpfc_queue *, uint32_t);
 void lpfc_modify_hba_eq_delay(struct lpfc_hba *phba, uint32_t startq,
@@ -1057,12 +1058,12 @@ void lpfc_sli_remove_dflt_fcf(struct lpfc_hba *);
 int lpfc_sli4_get_els_iocb_cnt(struct lpfc_hba *);
 int lpfc_sli4_get_iocb_cnt(struct lpfc_hba *phba);
 int lpfc_sli4_init_vpi(struct lpfc_vport *);
-inline void lpfc_sli4_eq_clr_intr(struct lpfc_queue *);
+void lpfc_sli4_eq_clr_intr(struct lpfc_queue *);
 void lpfc_sli4_write_cq_db(struct lpfc_hba *phba, struct lpfc_queue *q,
 			   uint32_t count, bool arm);
 void lpfc_sli4_write_eq_db(struct lpfc_hba *phba, struct lpfc_queue *q,
 			   uint32_t count, bool arm);
-inline void lpfc_sli4_if6_eq_clr_intr(struct lpfc_queue *q);
+void lpfc_sli4_if6_eq_clr_intr(struct lpfc_queue *q);
 void lpfc_sli4_if6_write_cq_db(struct lpfc_hba *phba, struct lpfc_queue *q,
 			       uint32_t count, bool arm);
 void lpfc_sli4_if6_write_eq_db(struct lpfc_hba *phba, struct lpfc_queue *q,
@@ -1079,3 +1080,8 @@ int lpfc_sli4_post_status_check(struct lpfc_hba *);
 uint8_t lpfc_sli_config_mbox_subsys_get(struct lpfc_hba *, LPFC_MBOXQ_t *);
 uint8_t lpfc_sli_config_mbox_opcode_get(struct lpfc_hba *, LPFC_MBOXQ_t *);
 void lpfc_sli4_ras_dma_free(struct lpfc_hba *phba);
+static inline void *lpfc_sli4_qe(struct lpfc_queue *q, uint16_t idx)
+{
+	return q->q_pgs[idx / q->entry_cnt_per_pg] +
+		(q->entry_size * (idx % q->entry_cnt_per_pg));
+}
diff --git a/drivers/scsi/lpfc/lpfc_version.h b/drivers/scsi/lpfc/lpfc_version.h
index 43fd693cf042..f7d9ef428417 100644
--- a/drivers/scsi/lpfc/lpfc_version.h
+++ b/drivers/scsi/lpfc/lpfc_version.h
@@ -20,7 +20,7 @@
  * included with this package.                                     *
  *******************************************************************/
 
-#define LPFC_DRIVER_VERSION "12.2.0.0"
+#define LPFC_DRIVER_VERSION "12.2.0.1"
 #define LPFC_DRIVER_NAME		"lpfc"
 
 /* Used for SLI 2/3 */
@@ -32,6 +32,6 @@
 
 #define LPFC_MODULE_DESC "Emulex LightPulse Fibre Channel SCSI driver " \
 		LPFC_DRIVER_VERSION
-#define LPFC_COPYRIGHT "Copyright (C) 2017-2018 Broadcom. All Rights " \
+#define LPFC_COPYRIGHT "Copyright (C) 2017-2019 Broadcom. All Rights " \
 		"Reserved. The term \"Broadcom\" refers to Broadcom Inc. " \
 		"and/or its subsidiaries."
diff --git a/drivers/scsi/megaraid/megaraid_sas_base.c b/drivers/scsi/megaraid/megaraid_sas_base.c
index 59a6546fd602..473a120eb75d 100644
--- a/drivers/scsi/megaraid/megaraid_sas_base.c
+++ b/drivers/scsi/megaraid/megaraid_sas_base.c
@@ -2724,7 +2724,7 @@ static int megasas_wait_for_outstanding(struct megasas_instance *instance)
 	do {
 		if ((fw_state == MFI_STATE_FAULT) || atomic_read(&instance->fw_outstanding)) {
 			dev_info(&instance->pdev->dev,
-				"%s:%d waiting_for_outstanding: before issue OCR. FW state = 0x%x, oustanding 0x%x\n",
+				"%s:%d waiting_for_outstanding: before issue OCR. FW state = 0x%x, outstanding 0x%x\n",
 				__func__, __LINE__, fw_state, atomic_read(&instance->fw_outstanding));
 			if (i == 3)
 				goto kill_hba_and_failed;
@@ -4647,7 +4647,7 @@ megasas_ld_list_query(struct megasas_instance *instance, u8 query_type)
  * Return:			0 if DCMD succeeded
  *				 non-zero if failed
  */
-int
+static int
 megasas_host_device_list_query(struct megasas_instance *instance,
 			       bool is_probe)
 {
diff --git a/drivers/scsi/megaraid/megaraid_sas_fusion.c b/drivers/scsi/megaraid/megaraid_sas_fusion.c
index e35c2b64c145..6129399c1942 100644
--- a/drivers/scsi/megaraid/megaraid_sas_fusion.c
+++ b/drivers/scsi/megaraid/megaraid_sas_fusion.c
@@ -4418,7 +4418,7 @@ int megasas_task_abort_fusion(struct scsi_cmnd *scmd)
 	if (!smid) {
 		ret = SUCCESS;
 		scmd_printk(KERN_NOTICE, scmd, "Command for which abort is"
-			" issued is not found in oustanding commands\n");
+			" issued is not found in outstanding commands\n");
 		mutex_unlock(&instance->reset_mutex);
 		goto out;
 	}
diff --git a/drivers/scsi/mpt3sas/Kconfig b/drivers/scsi/mpt3sas/Kconfig
index b736dbc80485..a072187875df 100644
--- a/drivers/scsi/mpt3sas/Kconfig
+++ b/drivers/scsi/mpt3sas/Kconfig
@@ -45,6 +45,7 @@ config SCSI_MPT3SAS
 	depends on PCI && SCSI
 	select SCSI_SAS_ATTRS
 	select RAID_ATTRS
+	select IRQ_POLL
 	---help---
 	This driver supports PCI-Express SAS 12Gb/s Host Adapters.
 
diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.c b/drivers/scsi/mpt3sas/mpt3sas_base.c
index f60b9e0a6ca6..8aacbd1e7db2 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_base.c
+++ b/drivers/scsi/mpt3sas/mpt3sas_base.c
@@ -94,6 +94,11 @@ module_param(max_msix_vectors, int, 0);
 MODULE_PARM_DESC(max_msix_vectors,
 	" max msix vectors");
 
+static int irqpoll_weight = -1;
+module_param(irqpoll_weight, int, 0);
+MODULE_PARM_DESC(irqpoll_weight,
+	"irq poll weight (default= one fourth of HBA queue depth)");
+
 static int mpt3sas_fwfault_debug;
 MODULE_PARM_DESC(mpt3sas_fwfault_debug,
 	" enable detection of firmware fault and halt firmware - (default=0)");
@@ -1382,20 +1387,30 @@ union reply_descriptor {
 	} u;
 };
 
+static u32 base_mod64(u64 dividend, u32 divisor)
+{
+	u32 remainder;
+
+	if (!divisor)
+		pr_err("mpt3sas: DIVISOR is zero, in div fn\n");
+	remainder = do_div(dividend, divisor);
+	return remainder;
+}
+
 /**
- * _base_interrupt - MPT adapter (IOC) specific interrupt handler.
- * @irq: irq number (not used)
- * @bus_id: bus identifier cookie == pointer to MPT_ADAPTER structure
+ * _base_process_reply_queue - Process reply descriptors from reply
+ *		descriptor post queue.
+ * @reply_q: per IRQ's reply queue object.
  *
- * Return: IRQ_HANDLED if processed, else IRQ_NONE.
+ * Return: number of reply descriptors processed from reply
+ *		descriptor queue.
  */
-static irqreturn_t
-_base_interrupt(int irq, void *bus_id)
+static int
+_base_process_reply_queue(struct adapter_reply_queue *reply_q)
 {
-	struct adapter_reply_queue *reply_q = bus_id;
 	union reply_descriptor rd;
-	u32 completed_cmds;
-	u8 request_desript_type;
+	u64 completed_cmds;
+	u8 request_descript_type;
 	u16 smid;
 	u8 cb_idx;
 	u32 reply;
@@ -1404,21 +1419,18 @@ _base_interrupt(int irq, void *bus_id)
 	Mpi2ReplyDescriptorsUnion_t *rpf;
 	u8 rc;
 
-	if (ioc->mask_interrupts)
-		return IRQ_NONE;
-
+	completed_cmds = 0;
 	if (!atomic_add_unless(&reply_q->busy, 1, 1))
-		return IRQ_NONE;
+		return completed_cmds;
 
 	rpf = &reply_q->reply_post_free[reply_q->reply_post_host_index];
-	request_desript_type = rpf->Default.ReplyFlags
+	request_descript_type = rpf->Default.ReplyFlags
 	     & MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK;
-	if (request_desript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED) {
+	if (request_descript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED) {
 		atomic_dec(&reply_q->busy);
-		return IRQ_NONE;
+		return completed_cmds;
 	}
 
-	completed_cmds = 0;
 	cb_idx = 0xFF;
 	do {
 		rd.word = le64_to_cpu(rpf->Words);
@@ -1426,11 +1438,11 @@ _base_interrupt(int irq, void *bus_id)
 			goto out;
 		reply = 0;
 		smid = le16_to_cpu(rpf->Default.DescriptorTypeDependent1);
-		if (request_desript_type ==
+		if (request_descript_type ==
 		    MPI25_RPY_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO_SUCCESS ||
-		    request_desript_type ==
+		    request_descript_type ==
 		    MPI2_RPY_DESCRIPT_FLAGS_SCSI_IO_SUCCESS ||
-		    request_desript_type ==
+		    request_descript_type ==
 		    MPI26_RPY_DESCRIPT_FLAGS_PCIE_ENCAPSULATED_SUCCESS) {
 			cb_idx = _base_get_cb_idx(ioc, smid);
 			if ((likely(cb_idx < MPT_MAX_CALLBACKS)) &&
@@ -1440,7 +1452,7 @@ _base_interrupt(int irq, void *bus_id)
 				if (rc)
 					mpt3sas_base_free_smid(ioc, smid);
 			}
-		} else if (request_desript_type ==
+		} else if (request_descript_type ==
 		    MPI2_RPY_DESCRIPT_FLAGS_ADDRESS_REPLY) {
 			reply = le32_to_cpu(
 			    rpf->AddressReply.ReplyFrameAddress);
@@ -1486,7 +1498,7 @@ _base_interrupt(int irq, void *bus_id)
 		    (reply_q->reply_post_host_index ==
 		    (ioc->reply_post_queue_depth - 1)) ? 0 :
 		    reply_q->reply_post_host_index + 1;
-		request_desript_type =
+		request_descript_type =
 		    reply_q->reply_post_free[reply_q->reply_post_host_index].
 		    Default.ReplyFlags & MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK;
 		completed_cmds++;
@@ -1495,7 +1507,7 @@ _base_interrupt(int irq, void *bus_id)
 		 * So that FW can find enough entries to post the Reply
 		 * Descriptors in the reply descriptor post queue.
 		 */
-		if (completed_cmds > ioc->hba_queue_depth/3) {
+		if (!base_mod64(completed_cmds, ioc->thresh_hold)) {
 			if (ioc->combined_reply_queue) {
 				writel(reply_q->reply_post_host_index |
 						((msix_index  & 7) <<
@@ -1507,9 +1519,14 @@ _base_interrupt(int irq, void *bus_id)
 						 MPI2_RPHI_MSIX_INDEX_SHIFT),
 						&ioc->chip->ReplyPostHostIndex);
 			}
-			completed_cmds = 1;
+			if (!reply_q->irq_poll_scheduled) {
+				reply_q->irq_poll_scheduled = true;
+				irq_poll_sched(&reply_q->irqpoll);
+			}
+			atomic_dec(&reply_q->busy);
+			return completed_cmds;
 		}
-		if (request_desript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED)
+		if (request_descript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED)
 			goto out;
 		if (!reply_q->reply_post_host_index)
 			rpf = reply_q->reply_post_free;
@@ -1521,14 +1538,14 @@ _base_interrupt(int irq, void *bus_id)
 
 	if (!completed_cmds) {
 		atomic_dec(&reply_q->busy);
-		return IRQ_NONE;
+		return completed_cmds;
 	}
 
 	if (ioc->is_warpdrive) {
 		writel(reply_q->reply_post_host_index,
 		ioc->reply_post_host_index[msix_index]);
 		atomic_dec(&reply_q->busy);
-		return IRQ_HANDLED;
+		return completed_cmds;
 	}
 
 	/* Update Reply Post Host Index.
@@ -1555,7 +1572,82 @@ _base_interrupt(int irq, void *bus_id)
 			MPI2_RPHI_MSIX_INDEX_SHIFT),
 			&ioc->chip->ReplyPostHostIndex);
 	atomic_dec(&reply_q->busy);
-	return IRQ_HANDLED;
+	return completed_cmds;
+}
+
+/**
+ * _base_interrupt - MPT adapter (IOC) specific interrupt handler.
+ * @irq: irq number (not used)
+ * @bus_id: bus identifier cookie == pointer to MPT_ADAPTER structure
+ *
+ * Return: IRQ_HANDLED if processed, else IRQ_NONE.
+ */
+static irqreturn_t
+_base_interrupt(int irq, void *bus_id)
+{
+	struct adapter_reply_queue *reply_q = bus_id;
+	struct MPT3SAS_ADAPTER *ioc = reply_q->ioc;
+
+	if (ioc->mask_interrupts)
+		return IRQ_NONE;
+	if (reply_q->irq_poll_scheduled)
+		return IRQ_HANDLED;
+	return ((_base_process_reply_queue(reply_q) > 0) ?
+			IRQ_HANDLED : IRQ_NONE);
+}
+
+/**
+ * _base_irqpoll - IRQ poll callback handler
+ * @irqpoll - irq_poll object
+ * @budget - irq poll weight
+ *
+ * returns number of reply descriptors processed
+ */
+static int
+_base_irqpoll(struct irq_poll *irqpoll, int budget)
+{
+	struct adapter_reply_queue *reply_q;
+	int num_entries = 0;
+
+	reply_q = container_of(irqpoll, struct adapter_reply_queue,
+			irqpoll);
+	if (reply_q->irq_line_enable) {
+		disable_irq(reply_q->os_irq);
+		reply_q->irq_line_enable = false;
+	}
+	num_entries = _base_process_reply_queue(reply_q);
+	if (num_entries < budget) {
+		irq_poll_complete(irqpoll);
+		reply_q->irq_poll_scheduled = false;
+		reply_q->irq_line_enable = true;
+		enable_irq(reply_q->os_irq);
+	}
+
+	return num_entries;
+}
+
+/**
+ * _base_init_irqpolls - initliaze IRQ polls
+ * @ioc: per adapter object
+ *
+ * returns nothing
+ */
+static void
+_base_init_irqpolls(struct MPT3SAS_ADAPTER *ioc)
+{
+	struct adapter_reply_queue *reply_q, *next;
+
+	if (list_empty(&ioc->reply_queue_list))
+		return;
+
+	list_for_each_entry_safe(reply_q, next, &ioc->reply_queue_list, list) {
+		irq_poll_init(&reply_q->irqpoll,
+			ioc->hba_queue_depth/4, _base_irqpoll);
+		reply_q->irq_poll_scheduled = false;
+		reply_q->irq_line_enable = true;
+		reply_q->os_irq = pci_irq_vector(ioc->pdev,
+		    reply_q->msix_index);
+	}
 }
 
 /**
@@ -1596,6 +1688,17 @@ mpt3sas_base_sync_reply_irqs(struct MPT3SAS_ADAPTER *ioc)
 		/* TMs are on msix_index == 0 */
 		if (reply_q->msix_index == 0)
 			continue;
+		if (reply_q->irq_poll_scheduled) {
+			/* Calling irq_poll_disable will wait for any pending
+			 * callbacks to have completed.
+			 */
+			irq_poll_disable(&reply_q->irqpoll);
+			irq_poll_enable(&reply_q->irqpoll);
+			reply_q->irq_poll_scheduled = false;
+			reply_q->irq_line_enable = true;
+			enable_irq(reply_q->os_irq);
+			continue;
+		}
 		synchronize_irq(pci_irq_vector(ioc->pdev, reply_q->msix_index));
 	}
 }
@@ -2757,6 +2860,11 @@ _base_assign_reply_queues(struct MPT3SAS_ADAPTER *ioc)
 
 	if (!_base_is_controller_msix_enabled(ioc))
 		return;
+	ioc->msix_load_balance = false;
+	if (ioc->reply_queue_count < num_online_cpus()) {
+		ioc->msix_load_balance = true;
+		return;
+	}
 
 	memset(ioc->cpu_msix_table, 0, ioc->cpu_msix_table_sz);
 
@@ -3015,6 +3123,8 @@ mpt3sas_base_map_resources(struct MPT3SAS_ADAPTER *ioc)
 	if (r)
 		goto out_fail;
 
+	if (!ioc->is_driver_loading)
+		_base_init_irqpolls(ioc);
 	/* Use the Combined reply queue feature only for SAS3 C0 & higher
 	 * revision HBAs and also only when reply queue count is greater than 8
 	 */
@@ -3158,6 +3268,12 @@ mpt3sas_base_get_reply_virt_addr(struct MPT3SAS_ADAPTER *ioc, u32 phys_addr)
 static inline u8
 _base_get_msix_index(struct MPT3SAS_ADAPTER *ioc)
 {
+	/* Enables reply_queue load balancing */
+	if (ioc->msix_load_balance)
+		return ioc->reply_queue_count ?
+		    base_mod64(atomic64_add_return(1,
+		    &ioc->total_io_cnt), ioc->reply_queue_count) : 0;
+
 	return ioc->cpu_msix_table[raw_smp_processor_id()];
 }
 
@@ -6506,6 +6622,12 @@ mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc)
 	if (r)
 		goto out_free_resources;
 
+	if (irqpoll_weight > 0)
+		ioc->thresh_hold = irqpoll_weight;
+	else
+		ioc->thresh_hold = ioc->hba_queue_depth/4;
+
+	_base_init_irqpolls(ioc);
 	init_waitqueue_head(&ioc->reset_wq);
 
 	/* allocate memory pd handle bitmask list */
diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.h b/drivers/scsi/mpt3sas/mpt3sas_base.h
index 19158cb59101..480219f0efc5 100644
--- a/drivers/scsi/mpt3sas/mpt3sas_base.h
+++ b/drivers/scsi/mpt3sas/mpt3sas_base.h
@@ -67,6 +67,7 @@
 #include <scsi/scsi_eh.h>
 #include <linux/pci.h>
 #include <linux/poll.h>
+#include <linux/irq_poll.h>
 
 #include "mpt3sas_debug.h"
 #include "mpt3sas_trigger_diag.h"
@@ -75,9 +76,9 @@
 #define MPT3SAS_DRIVER_NAME		"mpt3sas"
 #define MPT3SAS_AUTHOR "Avago Technologies <MPT-FusionLinux.pdl@avagotech.com>"
 #define MPT3SAS_DESCRIPTION	"LSI MPT Fusion SAS 3.0 Device Driver"
-#define MPT3SAS_DRIVER_VERSION		"27.102.00.00"
-#define MPT3SAS_MAJOR_VERSION		27
-#define MPT3SAS_MINOR_VERSION		102
+#define MPT3SAS_DRIVER_VERSION		"28.100.00.00"
+#define MPT3SAS_MAJOR_VERSION		28
+#define MPT3SAS_MINOR_VERSION		100
 #define MPT3SAS_BUILD_VERSION		0
 #define MPT3SAS_RELEASE_VERSION	00
 
@@ -882,6 +883,9 @@ struct _event_ack_list {
  * @reply_post_free: reply post base virt address
  * @name: the name registered to request_irq()
  * @busy: isr is actively processing replies on another cpu
+ * @os_irq: irq number
+ * @irqpoll: irq_poll object
+ * @irq_poll_scheduled: Tells whether irq poll is scheduled or not
  * @list: this list
 */
 struct adapter_reply_queue {
@@ -891,6 +895,10 @@ struct adapter_reply_queue {
 	Mpi2ReplyDescriptorsUnion_t *reply_post_free;
 	char			name[MPT_NAME_LENGTH];
 	atomic_t		busy;
+	u32			os_irq;
+	struct irq_poll         irqpoll;
+	bool			irq_poll_scheduled;
+	bool			irq_line_enable;
 	struct list_head	list;
 };
 
@@ -1016,7 +1024,12 @@ typedef void (*MPT3SAS_FLUSH_RUNNING_CMDS)(struct MPT3SAS_ADAPTER *ioc);
  * @msix_vector_count: number msix vectors
  * @cpu_msix_table: table for mapping cpus to msix index
  * @cpu_msix_table_sz: table size
+ * @total_io_cnt: Gives total IO count, used to load balance the interrupts
+ * @msix_load_balance: Enables load balancing of interrupts across
+ * the multiple MSIXs
  * @schedule_dead_ioc_flush_running_cmds: callback to flush pending commands
+ * @thresh_hold: Max number of reply descriptors processed
+ *				before updating Host Index
  * @scsi_io_cb_idx: shost generated commands
  * @tm_cb_idx: task management commands
  * @scsih_cb_idx: scsih internal commands
@@ -1192,6 +1205,9 @@ struct MPT3SAS_ADAPTER {
 	u32		ioc_reset_count;
 	MPT3SAS_FLUSH_RUNNING_CMDS schedule_dead_ioc_flush_running_cmds;
 	u32             non_operational_loop;
+	atomic64_t      total_io_cnt;
+	bool            msix_load_balance;
+	u16		thresh_hold;
 
 	/* internal commands, callback index */
 	u8		scsi_io_cb_idx;
diff --git a/drivers/scsi/mvsas/mv_64xx.c b/drivers/scsi/mvsas/mv_64xx.c
index b757d389e32f..2c699bc785a8 100644
--- a/drivers/scsi/mvsas/mv_64xx.c
+++ b/drivers/scsi/mvsas/mv_64xx.c
@@ -678,7 +678,8 @@ static u32 mvs_64xx_spi_read_data(struct mvs_info *mvi)
 static void mvs_64xx_spi_write_data(struct mvs_info *mvi, u32 data)
 {
 	void __iomem *regs = mvi->regs_ex;
-	 iow32(SPI_DATA_REG_64XX, data);
+
+	iow32(SPI_DATA_REG_64XX, data);
 }
 
 
diff --git a/drivers/scsi/mvsas/mv_94xx.c b/drivers/scsi/mvsas/mv_94xx.c
index eb5471bc7263..68b5b5f39a03 100644
--- a/drivers/scsi/mvsas/mv_94xx.c
+++ b/drivers/scsi/mvsas/mv_94xx.c
@@ -946,7 +946,8 @@ static u32 mvs_94xx_spi_read_data(struct mvs_info *mvi)
 static void mvs_94xx_spi_write_data(struct mvs_info *mvi, u32 data)
 {
 	void __iomem *regs = mvi->regs_ex - 0x10200;
-	 mw32(SPI_RD_DATA_REG_94XX, data);
+
+	mw32(SPI_RD_DATA_REG_94XX, data);
 }
 
 
diff --git a/drivers/scsi/mvsas/mv_sas.c b/drivers/scsi/mvsas/mv_sas.c
index 311d23c727ce..e933c65d8e0b 100644
--- a/drivers/scsi/mvsas/mv_sas.c
+++ b/drivers/scsi/mvsas/mv_sas.c
@@ -1422,7 +1422,7 @@ int mvs_I_T_nexus_reset(struct domain_device *dev)
 {
 	unsigned long flags;
 	int rc = TMF_RESP_FUNC_FAILED;
-    struct mvs_device * mvi_dev = (struct mvs_device *)dev->lldd_dev;
+	struct mvs_device *mvi_dev = (struct mvs_device *)dev->lldd_dev;
 	struct mvs_info *mvi = mvi_dev->mvi_info;
 
 	if (mvi_dev->dev_status != MVS_DEV_EH)
diff --git a/drivers/scsi/mvumi.c b/drivers/scsi/mvumi.c
index 3df02691a092..a5410615edac 100644
--- a/drivers/scsi/mvumi.c
+++ b/drivers/scsi/mvumi.c
@@ -752,7 +752,7 @@ static int mvumi_issue_blocked_cmd(struct mvumi_hba *mhba,
 		spin_lock_irqsave(mhba->shost->host_lock, flags);
 		atomic_dec(&cmd->sync_cmd);
 		if (mhba->tag_cmd[cmd->frame->tag]) {
-			mhba->tag_cmd[cmd->frame->tag] = 0;
+			mhba->tag_cmd[cmd->frame->tag] = NULL;
 			dev_warn(&mhba->pdev->dev, "TIMEOUT:release tag [%d]\n",
 							cmd->frame->tag);
 			tag_release_one(mhba, &mhba->tag_pool, cmd->frame->tag);
@@ -1794,7 +1794,7 @@ static void mvumi_handle_clob(struct mvumi_hba *mhba)
 		cmd = mhba->tag_cmd[ob_frame->tag];
 
 		atomic_dec(&mhba->fw_outstanding);
-		mhba->tag_cmd[ob_frame->tag] = 0;
+		mhba->tag_cmd[ob_frame->tag] = NULL;
 		tag_release_one(mhba, &mhba->tag_pool, ob_frame->tag);
 		if (cmd->scmd)
 			mvumi_complete_cmd(mhba, cmd, ob_frame);
@@ -2139,7 +2139,7 @@ static enum blk_eh_timer_return mvumi_timed_out(struct scsi_cmnd *scmd)
 	spin_lock_irqsave(mhba->shost->host_lock, flags);
 
 	if (mhba->tag_cmd[cmd->frame->tag]) {
-		mhba->tag_cmd[cmd->frame->tag] = 0;
+		mhba->tag_cmd[cmd->frame->tag] = NULL;
 		tag_release_one(mhba, &mhba->tag_pool, cmd->frame->tag);
 	}
 	if (!list_empty(&cmd->queue_pointer))
diff --git a/drivers/scsi/pm8001/pm8001_hwi.c b/drivers/scsi/pm8001/pm8001_hwi.c
index d0bb357034d8..109effd3557d 100644
--- a/drivers/scsi/pm8001/pm8001_hwi.c
+++ b/drivers/scsi/pm8001/pm8001_hwi.c
@@ -960,9 +960,9 @@ pm8001_chip_soft_rst(struct pm8001_hba_info *pm8001_ha)
 		return -1;
 	}
 	regVal = pm8001_cr32(pm8001_ha, 2, GPIO_GPIO_0_0UTPUT_CTL_OFFSET);
-		PM8001_INIT_DBG(pm8001_ha,
-				pm8001_printk("GPIO Output Control Register:"
-				" = 0x%x\n", regVal));
+	PM8001_INIT_DBG(pm8001_ha,
+			pm8001_printk("GPIO Output Control Register:"
+			" = 0x%x\n", regVal));
 	/* set GPIO-0 output control to tri-state */
 	regVal &= 0xFFFFFFFC;
 	pm8001_cw32(pm8001_ha, 2, GPIO_GPIO_0_0UTPUT_CTL_OFFSET, regVal);
@@ -1204,6 +1204,7 @@ void pm8001_chip_iounmap(struct pm8001_hba_info *pm8001_ha)
 	}
 }
 
+#ifndef PM8001_USE_MSIX
 /**
  * pm8001_chip_interrupt_enable - enable PM8001 chip interrupt
  * @pm8001_ha: our hba card information
@@ -1225,6 +1226,8 @@ pm8001_chip_intx_interrupt_disable(struct pm8001_hba_info *pm8001_ha)
 	pm8001_cw32(pm8001_ha, 0, MSGU_ODMR, ODMR_MASK_ALL);
 }
 
+#else
+
 /**
  * pm8001_chip_msix_interrupt_enable - enable PM8001 chip interrupt
  * @pm8001_ha: our hba card information
@@ -1256,6 +1259,7 @@ pm8001_chip_msix_interrupt_disable(struct pm8001_hba_info *pm8001_ha,
 	msi_index += MSIX_TABLE_BASE;
 	pm8001_cw32(pm8001_ha, 0,  msi_index, MSIX_INTERRUPT_DISABLE);
 }
+#endif
 
 /**
  * pm8001_chip_interrupt_enable - enable PM8001 chip interrupt
@@ -1266,10 +1270,9 @@ pm8001_chip_interrupt_enable(struct pm8001_hba_info *pm8001_ha, u8 vec)
 {
 #ifdef PM8001_USE_MSIX
 	pm8001_chip_msix_interrupt_enable(pm8001_ha, 0);
-	return;
-#endif
+#else
 	pm8001_chip_intx_interrupt_enable(pm8001_ha);
-
+#endif
 }
 
 /**
@@ -1281,10 +1284,9 @@ pm8001_chip_interrupt_disable(struct pm8001_hba_info *pm8001_ha, u8 vec)
 {
 #ifdef PM8001_USE_MSIX
 	pm8001_chip_msix_interrupt_disable(pm8001_ha, 0);
-	return;
-#endif
+#else
 	pm8001_chip_intx_interrupt_disable(pm8001_ha);
-
+#endif
 }
 
 /**
@@ -2898,7 +2900,6 @@ static void mpi_sata_event(struct pm8001_hba_info *pm8001_ha , void *piomb)
 static void
 mpi_smp_completion(struct pm8001_hba_info *pm8001_ha, void *piomb)
 {
-	u32 param;
 	struct sas_task *t;
 	struct pm8001_ccb_info *ccb;
 	unsigned long flags;
@@ -2913,7 +2914,6 @@ mpi_smp_completion(struct pm8001_hba_info *pm8001_ha, void *piomb)
 	tag = le32_to_cpu(psmpPayload->tag);
 
 	ccb = &pm8001_ha->ccb_info[tag];
-	param = le32_to_cpu(psmpPayload->param);
 	t = ccb->task;
 	ts = &t->task_status;
 	pm8001_dev = ccb->device;
@@ -2928,7 +2928,7 @@ mpi_smp_completion(struct pm8001_hba_info *pm8001_ha, void *piomb)
 		PM8001_IO_DBG(pm8001_ha, pm8001_printk("IO_SUCCESS\n"));
 		ts->resp = SAS_TASK_COMPLETE;
 		ts->stat = SAM_STAT_GOOD;
-	if (pm8001_dev)
+		if (pm8001_dev)
 			pm8001_dev->running_req--;
 		break;
 	case IO_ABORTED:
@@ -3244,11 +3244,9 @@ void pm8001_bytes_dmaed(struct pm8001_hba_info *pm8001_ha, int i)
 {
 	struct pm8001_phy *phy = &pm8001_ha->phy[i];
 	struct asd_sas_phy *sas_phy = &phy->sas_phy;
-	struct sas_ha_struct *sas_ha;
 	if (!phy->phy_attached)
 		return;
 
-	sas_ha = pm8001_ha->sas;
 	if (sas_phy->phy) {
 		struct sas_phy *sphy = sas_phy->phy;
 		sphy->negotiated_linkrate = sas_phy->linkrate;
@@ -4627,17 +4625,18 @@ static int pm8001_chip_phy_ctl_req(struct pm8001_hba_info *pm8001_ha,
 	return ret;
 }
 
-static u32 pm8001_chip_is_our_interupt(struct pm8001_hba_info *pm8001_ha)
+static u32 pm8001_chip_is_our_interrupt(struct pm8001_hba_info *pm8001_ha)
 {
-	u32 value;
 #ifdef PM8001_USE_MSIX
 	return 1;
-#endif
+#else
+	u32 value;
+
 	value = pm8001_cr32(pm8001_ha, 0, MSGU_ODR);
 	if (value)
 		return 1;
 	return 0;
-
+#endif
 }
 
 /**
@@ -5123,7 +5122,7 @@ const struct pm8001_dispatch pm8001_8001_dispatch = {
 	.chip_rst		= pm8001_hw_chip_rst,
 	.chip_iounmap		= pm8001_chip_iounmap,
 	.isr			= pm8001_chip_isr,
-	.is_our_interupt	= pm8001_chip_is_our_interupt,
+	.is_our_interrupt	= pm8001_chip_is_our_interrupt,
 	.isr_process_oq		= process_oq,
 	.interrupt_enable 	= pm8001_chip_interrupt_enable,
 	.interrupt_disable	= pm8001_chip_interrupt_disable,
diff --git a/drivers/scsi/pm8001/pm8001_init.c b/drivers/scsi/pm8001/pm8001_init.c
index a36060c23b37..3374f553c617 100644
--- a/drivers/scsi/pm8001/pm8001_init.c
+++ b/drivers/scsi/pm8001/pm8001_init.c
@@ -201,7 +201,7 @@ static irqreturn_t pm8001_interrupt_handler_msix(int irq, void *opaque)
 
 	if (unlikely(!pm8001_ha))
 		return IRQ_NONE;
-	if (!PM8001_CHIP_DISP->is_our_interupt(pm8001_ha))
+	if (!PM8001_CHIP_DISP->is_our_interrupt(pm8001_ha))
 		return IRQ_NONE;
 #ifdef PM8001_USE_TASKLET
 	tasklet_schedule(&pm8001_ha->tasklet[irq_vector->irq_id]);
@@ -224,7 +224,7 @@ static irqreturn_t pm8001_interrupt_handler_intx(int irq, void *dev_id)
 	pm8001_ha = sha->lldd_ha;
 	if (unlikely(!pm8001_ha))
 		return IRQ_NONE;
-	if (!PM8001_CHIP_DISP->is_our_interupt(pm8001_ha))
+	if (!PM8001_CHIP_DISP->is_our_interrupt(pm8001_ha))
 		return IRQ_NONE;
 
 #ifdef PM8001_USE_TASKLET
diff --git a/drivers/scsi/pm8001/pm8001_sas.c b/drivers/scsi/pm8001/pm8001_sas.c
index 084f2fcced0a..88eef3b18e41 100644
--- a/drivers/scsi/pm8001/pm8001_sas.c
+++ b/drivers/scsi/pm8001/pm8001_sas.c
@@ -740,8 +740,8 @@ static int pm8001_exec_internal_tmf_task(struct domain_device *dev,
 		wait_for_completion(&task->slow_task->completion);
 		if (pm8001_ha->chip_id != chip_8001) {
 			pm8001_dev->setds_completion = &completion_setstate;
-				PM8001_CHIP_DISP->set_dev_state_req(pm8001_ha,
-					pm8001_dev, 0x01);
+			PM8001_CHIP_DISP->set_dev_state_req(pm8001_ha,
+				pm8001_dev, 0x01);
 			wait_for_completion(&completion_setstate);
 		}
 		res = -TMF_RESP_FUNC_FAILED;
diff --git a/drivers/scsi/pm8001/pm8001_sas.h b/drivers/scsi/pm8001/pm8001_sas.h
index f88b0d33c385..ac6d8e3f22de 100644
--- a/drivers/scsi/pm8001/pm8001_sas.h
+++ b/drivers/scsi/pm8001/pm8001_sas.h
@@ -197,7 +197,7 @@ struct pm8001_dispatch {
 	int (*chip_ioremap)(struct pm8001_hba_info *pm8001_ha);
 	void (*chip_iounmap)(struct pm8001_hba_info *pm8001_ha);
 	irqreturn_t (*isr)(struct pm8001_hba_info *pm8001_ha, u8 vec);
-	u32 (*is_our_interupt)(struct pm8001_hba_info *pm8001_ha);
+	u32 (*is_our_interrupt)(struct pm8001_hba_info *pm8001_ha);
 	int (*isr_process_oq)(struct pm8001_hba_info *pm8001_ha, u8 vec);
 	void (*interrupt_enable)(struct pm8001_hba_info *pm8001_ha, u8 vec);
 	void (*interrupt_disable)(struct pm8001_hba_info *pm8001_ha, u8 vec);
diff --git a/drivers/scsi/pm8001/pm80xx_hwi.c b/drivers/scsi/pm8001/pm80xx_hwi.c
index 63e4f7d34d6c..301de40eb708 100644
--- a/drivers/scsi/pm8001/pm80xx_hwi.c
+++ b/drivers/scsi/pm8001/pm80xx_hwi.c
@@ -1316,7 +1316,7 @@ pm80xx_chip_soft_rst(struct pm8001_hba_info *pm8001_ha)
 
 static void pm80xx_hw_chip_rst(struct pm8001_hba_info *pm8001_ha)
 {
-	 u32 i;
+	u32 i;
 
 	PM8001_INIT_DBG(pm8001_ha,
 		pm8001_printk("chip reset start\n"));
@@ -4381,27 +4381,27 @@ static int pm80xx_chip_sata_req(struct pm8001_hba_info *pm8001_ha,
 			sata_cmd.len = cpu_to_le32(task->total_xfer_len);
 			sata_cmd.esgl = 0;
 		}
-			/* scsi cdb */
-			sata_cmd.atapi_scsi_cdb[0] =
-				cpu_to_le32(((task->ata_task.atapi_packet[0]) |
-				(task->ata_task.atapi_packet[1] << 8) |
-				(task->ata_task.atapi_packet[2] << 16) |
-				(task->ata_task.atapi_packet[3] << 24)));
-			sata_cmd.atapi_scsi_cdb[1] =
-				cpu_to_le32(((task->ata_task.atapi_packet[4]) |
-				(task->ata_task.atapi_packet[5] << 8) |
-				(task->ata_task.atapi_packet[6] << 16) |
-				(task->ata_task.atapi_packet[7] << 24)));
-			sata_cmd.atapi_scsi_cdb[2] =
-				cpu_to_le32(((task->ata_task.atapi_packet[8]) |
-				(task->ata_task.atapi_packet[9] << 8) |
-				(task->ata_task.atapi_packet[10] << 16) |
-				(task->ata_task.atapi_packet[11] << 24)));
-			sata_cmd.atapi_scsi_cdb[3] =
-				cpu_to_le32(((task->ata_task.atapi_packet[12]) |
-				(task->ata_task.atapi_packet[13] << 8) |
-				(task->ata_task.atapi_packet[14] << 16) |
-				(task->ata_task.atapi_packet[15] << 24)));
+		/* scsi cdb */
+		sata_cmd.atapi_scsi_cdb[0] =
+			cpu_to_le32(((task->ata_task.atapi_packet[0]) |
+			(task->ata_task.atapi_packet[1] << 8) |
+			(task->ata_task.atapi_packet[2] << 16) |
+			(task->ata_task.atapi_packet[3] << 24)));
+		sata_cmd.atapi_scsi_cdb[1] =
+			cpu_to_le32(((task->ata_task.atapi_packet[4]) |
+			(task->ata_task.atapi_packet[5] << 8) |
+			(task->ata_task.atapi_packet[6] << 16) |
+			(task->ata_task.atapi_packet[7] << 24)));
+		sata_cmd.atapi_scsi_cdb[2] =
+			cpu_to_le32(((task->ata_task.atapi_packet[8]) |
+			(task->ata_task.atapi_packet[9] << 8) |
+			(task->ata_task.atapi_packet[10] << 16) |
+			(task->ata_task.atapi_packet[11] << 24)));
+		sata_cmd.atapi_scsi_cdb[3] =
+			cpu_to_le32(((task->ata_task.atapi_packet[12]) |
+			(task->ata_task.atapi_packet[13] << 8) |
+			(task->ata_task.atapi_packet[14] << 16) |
+			(task->ata_task.atapi_packet[15] << 24)));
 	}
 
 	/* Check for read log for failed drive and return */
@@ -4617,17 +4617,18 @@ static int pm80xx_chip_phy_ctl_req(struct pm8001_hba_info *pm8001_ha,
 	return pm8001_mpi_build_cmd(pm8001_ha, circularQ, opc, &payload, 0);
 }
 
-static u32 pm80xx_chip_is_our_interupt(struct pm8001_hba_info *pm8001_ha)
+static u32 pm80xx_chip_is_our_interrupt(struct pm8001_hba_info *pm8001_ha)
 {
-	u32 value;
 #ifdef PM8001_USE_MSIX
 	return 1;
-#endif
+#else
+	u32 value;
+
 	value = pm8001_cr32(pm8001_ha, 0, MSGU_ODR);
 	if (value)
 		return 1;
 	return 0;
-
+#endif
 }
 
 /**
@@ -4724,7 +4725,7 @@ const struct pm8001_dispatch pm8001_80xx_dispatch = {
 	.chip_rst		= pm80xx_hw_chip_rst,
 	.chip_iounmap		= pm8001_chip_iounmap,
 	.isr			= pm80xx_chip_isr,
-	.is_our_interupt	= pm80xx_chip_is_our_interupt,
+	.is_our_interrupt	= pm80xx_chip_is_our_interrupt,
 	.isr_process_oq		= process_oq,
 	.interrupt_enable	= pm80xx_chip_interrupt_enable,
 	.interrupt_disable	= pm80xx_chip_interrupt_disable,
diff --git a/drivers/scsi/qedf/qedf.h b/drivers/scsi/qedf/qedf.h
index 2c78d8fb9122..2c08f6f4db42 100644
--- a/drivers/scsi/qedf/qedf.h
+++ b/drivers/scsi/qedf/qedf.h
@@ -35,9 +35,6 @@
 #define QEDF_DESCR "QLogic FCoE Offload Driver"
 #define QEDF_MODULE_NAME "qedf"
 
-#define QEDF_MIN_XID		0
-#define QEDF_MAX_SCSI_XID	(NUM_TASKS_PER_CONNECTION - 1)
-#define QEDF_MAX_ELS_XID	4095
 #define QEDF_FLOGI_RETRY_CNT	3
 #define QEDF_RPORT_RETRY_CNT	255
 #define QEDF_MAX_SESSIONS	1024
@@ -52,8 +49,8 @@
 	sizeof(struct fc_frame_header))
 #define QEDF_MAX_NPIV		64
 #define QEDF_TM_TIMEOUT		10
-#define QEDF_ABORT_TIMEOUT	10
-#define QEDF_CLEANUP_TIMEOUT	10
+#define QEDF_ABORT_TIMEOUT	(10 * 1000)
+#define QEDF_CLEANUP_TIMEOUT	1
 #define QEDF_MAX_CDB_LEN	16
 
 #define UPSTREAM_REMOVE		1
@@ -85,6 +82,7 @@ struct qedf_els_cb_arg {
 };
 
 enum qedf_ioreq_event {
+	QEDF_IOREQ_EV_NONE,
 	QEDF_IOREQ_EV_ABORT_SUCCESS,
 	QEDF_IOREQ_EV_ABORT_FAILED,
 	QEDF_IOREQ_EV_SEND_RRQ,
@@ -105,7 +103,6 @@ struct qedf_ioreq {
 	struct list_head link;
 	uint16_t xid;
 	struct scsi_cmnd *sc_cmd;
-	bool use_slowpath; /* Use slow SGL for this I/O */
 #define QEDF_SCSI_CMD		1
 #define QEDF_TASK_MGMT_CMD	2
 #define QEDF_ABTS		3
@@ -117,22 +114,43 @@ struct qedf_ioreq {
 #define QEDF_CMD_IN_ABORT		0x1
 #define QEDF_CMD_IN_CLEANUP		0x2
 #define QEDF_CMD_SRR_SENT		0x3
+#define QEDF_CMD_DIRTY			0x4
+#define QEDF_CMD_ERR_SCSI_DONE		0x5
 	u8 io_req_flags;
 	uint8_t tm_flags;
 	struct qedf_rport *fcport;
+#define	QEDF_CMD_ST_INACTIVE		0
+#define	QEDFC_CMD_ST_IO_ACTIVE		1
+#define	QEDFC_CMD_ST_ABORT_ACTIVE	2
+#define	QEDFC_CMD_ST_ABORT_ACTIVE_EH	3
+#define	QEDFC_CMD_ST_CLEANUP_ACTIVE	4
+#define	QEDFC_CMD_ST_CLEANUP_ACTIVE_EH	5
+#define	QEDFC_CMD_ST_RRQ_ACTIVE		6
+#define	QEDFC_CMD_ST_RRQ_WAIT		7
+#define	QEDFC_CMD_ST_OXID_RETIRE_WAIT	8
+#define	QEDFC_CMD_ST_TMF_ACTIVE		9
+#define	QEDFC_CMD_ST_DRAIN_ACTIVE	10
+#define	QEDFC_CMD_ST_CLEANED		11
+#define	QEDFC_CMD_ST_ELS_ACTIVE		12
+	atomic_t state;
 	unsigned long flags;
 	enum qedf_ioreq_event event;
 	size_t data_xfer_len;
+	/* ID: 001: Alloc cmd (qedf_alloc_cmd) */
+	/* ID: 002: Initiate ABTS (qedf_initiate_abts) */
+	/* ID: 003: For RRQ (qedf_process_abts_compl) */
 	struct kref refcount;
 	struct qedf_cmd_mgr *cmd_mgr;
 	struct io_bdt *bd_tbl;
 	struct delayed_work timeout_work;
 	struct completion tm_done;
 	struct completion abts_done;
+	struct completion cleanup_done;
 	struct e4_fcoe_task_context *task;
 	struct fcoe_task_params *task_params;
 	struct scsi_sgl_task_params *sgl_task_params;
 	int idx;
+	int lun;
 /*
  * Need to allocate enough room for both sense data and FCP response data
  * which has a max length of 8 bytes according to spec.
@@ -155,9 +173,9 @@ struct qedf_ioreq {
 	int fp_idx;
 	unsigned int cpu;
 	unsigned int int_cpu;
-#define QEDF_IOREQ_SLOW_SGE		0
-#define QEDF_IOREQ_SINGLE_SGE		1
-#define QEDF_IOREQ_FAST_SGE		2
+#define QEDF_IOREQ_UNKNOWN_SGE		1
+#define QEDF_IOREQ_SLOW_SGE		2
+#define QEDF_IOREQ_FAST_SGE		3
 	u8 sge_type;
 	struct delayed_work rrq_work;
 
@@ -172,6 +190,8 @@ struct qedf_ioreq {
 	 * during some form of error processing.
 	 */
 	bool return_scsi_cmd_on_abts;
+
+	unsigned int alloc;
 };
 
 extern struct workqueue_struct *qedf_io_wq;
@@ -181,7 +201,10 @@ struct qedf_rport {
 #define QEDF_RPORT_SESSION_READY 1
 #define QEDF_RPORT_UPLOADING_CONNECTION	2
 #define QEDF_RPORT_IN_RESET 3
+#define QEDF_RPORT_IN_LUN_RESET 4
+#define QEDF_RPORT_IN_TARGET_RESET 5
 	unsigned long flags;
+	int lun_reset_lun;
 	unsigned long retry_delay_timestamp;
 	struct fc_rport *rport;
 	struct fc_rport_priv *rdata;
@@ -191,6 +214,7 @@ struct qedf_rport {
 	void __iomem *p_doorbell;
 	/* Send queue management */
 	atomic_t free_sqes;
+	atomic_t ios_to_queue;
 	atomic_t num_active_ios;
 	struct fcoe_wqe *sq;
 	dma_addr_t sq_dma;
@@ -295,8 +319,6 @@ struct qedf_ctx {
 #define QEDF_DCBX_PENDING	0
 #define QEDF_DCBX_DONE		1
 	atomic_t dcbx;
-	uint16_t max_scsi_xid;
-	uint16_t max_els_xid;
 #define QEDF_NULL_VLAN_ID	-1
 #define QEDF_FALLBACK_VLAN	1002
 #define QEDF_DEFAULT_PRIO	3
@@ -371,7 +393,6 @@ struct qedf_ctx {
 
 	u32 slow_sge_ios;
 	u32 fast_sge_ios;
-	u32 single_sge_ios;
 
 	uint8_t	*grcdump;
 	uint32_t grcdump_size;
@@ -396,6 +417,8 @@ struct qedf_ctx {
 	u8 target_resets;
 	u8 task_set_fulls;
 	u8 busy;
+	/* Used for flush routine */
+	struct mutex flush_mutex;
 };
 
 struct io_bdt {
@@ -435,6 +458,12 @@ static inline void qedf_stop_all_io(struct qedf_ctx *qedf)
 /*
  * Externs
  */
+
+/*
+ * (QEDF_LOG_NPIV | QEDF_LOG_SESS | QEDF_LOG_LPORT | QEDF_LOG_ELS | QEDF_LOG_MQ
+ * | QEDF_LOG_IO | QEDF_LOG_UNSOL | QEDF_LOG_SCSI_TM | QEDF_LOG_MP_REQ |
+ * QEDF_LOG_EVT | QEDF_LOG_CONN | QEDF_LOG_DISC | QEDF_LOG_INFO)
+ */
 #define QEDF_DEFAULT_LOG_MASK		0x3CFB6
 extern const struct qed_fcoe_ops *qed_ops;
 extern uint qedf_dump_frames;
@@ -494,7 +523,7 @@ extern void qedf_set_vlan_id(struct qedf_ctx *qedf, int vlan_id);
 extern void qedf_create_sysfs_ctx_attr(struct qedf_ctx *qedf);
 extern void qedf_remove_sysfs_ctx_attr(struct qedf_ctx *qedf);
 extern void qedf_capture_grc_dump(struct qedf_ctx *qedf);
-extern void qedf_wait_for_upload(struct qedf_ctx *qedf);
+bool qedf_wait_for_upload(struct qedf_ctx *qedf);
 extern void qedf_process_unsol_compl(struct qedf_ctx *qedf, uint16_t que_idx,
 	struct fcoe_cqe *cqe);
 extern void qedf_restart_rport(struct qedf_rport *fcport);
@@ -508,6 +537,8 @@ extern void qedf_get_protocol_tlv_data(void *dev, void *data);
 extern void qedf_fp_io_handler(struct work_struct *work);
 extern void qedf_get_generic_tlv_data(void *dev, struct qed_generic_tlvs *data);
 extern void qedf_wq_grcdump(struct work_struct *work);
+void qedf_stag_change_work(struct work_struct *work);
+void qedf_ctx_soft_reset(struct fc_lport *lport);
 
 #define FCOE_WORD_TO_BYTE  4
 #define QEDF_MAX_TASK_NUM	0xFFFF
diff --git a/drivers/scsi/qedf/qedf_dbg.c b/drivers/scsi/qedf/qedf_dbg.c
index f2397ee9ba69..f7d170bffc82 100644
--- a/drivers/scsi/qedf/qedf_dbg.c
+++ b/drivers/scsi/qedf/qedf_dbg.c
@@ -15,10 +15,6 @@ qedf_dbg_err(struct qedf_dbg_ctx *qedf, const char *func, u32 line,
 {
 	va_list va;
 	struct va_format vaf;
-	char nfunc[32];
-
-	memset(nfunc, 0, sizeof(nfunc));
-	memcpy(nfunc, func, sizeof(nfunc) - 1);
 
 	va_start(va, fmt);
 
@@ -27,9 +23,9 @@ qedf_dbg_err(struct qedf_dbg_ctx *qedf, const char *func, u32 line,
 
 	if (likely(qedf) && likely(qedf->pdev))
 		pr_err("[%s]:[%s:%d]:%d: %pV", dev_name(&(qedf->pdev->dev)),
-			nfunc, line, qedf->host_no, &vaf);
+			func, line, qedf->host_no, &vaf);
 	else
-		pr_err("[0000:00:00.0]:[%s:%d]: %pV", nfunc, line, &vaf);
+		pr_err("[0000:00:00.0]:[%s:%d]: %pV", func, line, &vaf);
 
 	va_end(va);
 }
@@ -40,10 +36,6 @@ qedf_dbg_warn(struct qedf_dbg_ctx *qedf, const char *func, u32 line,
 {
 	va_list va;
 	struct va_format vaf;
-	char nfunc[32];
-
-	memset(nfunc, 0, sizeof(nfunc));
-	memcpy(nfunc, func, sizeof(nfunc) - 1);
 
 	va_start(va, fmt);
 
@@ -55,9 +47,9 @@ qedf_dbg_warn(struct qedf_dbg_ctx *qedf, const char *func, u32 line,
 
 	if (likely(qedf) && likely(qedf->pdev))
 		pr_warn("[%s]:[%s:%d]:%d: %pV", dev_name(&(qedf->pdev->dev)),
-			nfunc, line, qedf->host_no, &vaf);
+			func, line, qedf->host_no, &vaf);
 	else
-		pr_warn("[0000:00:00.0]:[%s:%d]: %pV", nfunc, line, &vaf);
+		pr_warn("[0000:00:00.0]:[%s:%d]: %pV", func, line, &vaf);
 
 ret:
 	va_end(va);
@@ -69,10 +61,6 @@ qedf_dbg_notice(struct qedf_dbg_ctx *qedf, const char *func, u32 line,
 {
 	va_list va;
 	struct va_format vaf;
-	char nfunc[32];
-
-	memset(nfunc, 0, sizeof(nfunc));
-	memcpy(nfunc, func, sizeof(nfunc) - 1);
 
 	va_start(va, fmt);
 
@@ -84,10 +72,10 @@ qedf_dbg_notice(struct qedf_dbg_ctx *qedf, const char *func, u32 line,
 
 	if (likely(qedf) && likely(qedf->pdev))
 		pr_notice("[%s]:[%s:%d]:%d: %pV",
-			  dev_name(&(qedf->pdev->dev)), nfunc, line,
+			  dev_name(&(qedf->pdev->dev)), func, line,
 			  qedf->host_no, &vaf);
 	else
-		pr_notice("[0000:00:00.0]:[%s:%d]: %pV", nfunc, line, &vaf);
+		pr_notice("[0000:00:00.0]:[%s:%d]: %pV", func, line, &vaf);
 
 ret:
 	va_end(va);
@@ -99,10 +87,6 @@ qedf_dbg_info(struct qedf_dbg_ctx *qedf, const char *func, u32 line,
 {
 	va_list va;
 	struct va_format vaf;
-	char nfunc[32];
-
-	memset(nfunc, 0, sizeof(nfunc));
-	memcpy(nfunc, func, sizeof(nfunc) - 1);
 
 	va_start(va, fmt);
 
@@ -114,9 +98,9 @@ qedf_dbg_info(struct qedf_dbg_ctx *qedf, const char *func, u32 line,
 
 	if (likely(qedf) && likely(qedf->pdev))
 		pr_info("[%s]:[%s:%d]:%d: %pV", dev_name(&(qedf->pdev->dev)),
-			nfunc, line, qedf->host_no, &vaf);
+			func, line, qedf->host_no, &vaf);
 	else
-		pr_info("[0000:00:00.0]:[%s:%d]: %pV", nfunc, line, &vaf);
+		pr_info("[0000:00:00.0]:[%s:%d]: %pV", func, line, &vaf);
 
 ret:
 	va_end(va);
diff --git a/drivers/scsi/qedf/qedf_debugfs.c b/drivers/scsi/qedf/qedf_debugfs.c
index a32d8ee4666e..235389209689 100644
--- a/drivers/scsi/qedf/qedf_debugfs.c
+++ b/drivers/scsi/qedf/qedf_debugfs.c
@@ -293,6 +293,33 @@ qedf_dbg_io_trace_open(struct inode *inode, struct file *file)
 	return single_open(file, qedf_io_trace_show, qedf);
 }
 
+/* Based on fip_state enum from libfcoe.h */
+static char *fip_state_names[] = {
+	"FIP_ST_DISABLED",
+	"FIP_ST_LINK_WAIT",
+	"FIP_ST_AUTO",
+	"FIP_ST_NON_FIP",
+	"FIP_ST_ENABLED",
+	"FIP_ST_VNMP_START",
+	"FIP_ST_VNMP_PROBE1",
+	"FIP_ST_VNMP_PROBE2",
+	"FIP_ST_VNMP_CLAIM",
+	"FIP_ST_VNMP_UP",
+};
+
+/* Based on fc_rport_state enum from libfc.h */
+static char *fc_rport_state_names[] = {
+	"RPORT_ST_INIT",
+	"RPORT_ST_FLOGI",
+	"RPORT_ST_PLOGI_WAIT",
+	"RPORT_ST_PLOGI",
+	"RPORT_ST_PRLI",
+	"RPORT_ST_RTV",
+	"RPORT_ST_READY",
+	"RPORT_ST_ADISC",
+	"RPORT_ST_DELETE",
+};
+
 static int
 qedf_driver_stats_show(struct seq_file *s, void *unused)
 {
@@ -300,10 +327,28 @@ qedf_driver_stats_show(struct seq_file *s, void *unused)
 	struct qedf_rport *fcport;
 	struct fc_rport_priv *rdata;
 
+	seq_printf(s, "Host WWNN/WWPN: %016llx/%016llx\n",
+		   qedf->wwnn, qedf->wwpn);
+	seq_printf(s, "Host NPortID: %06x\n", qedf->lport->port_id);
+	seq_printf(s, "Link State: %s\n", atomic_read(&qedf->link_state) ?
+	    "Up" : "Down");
+	seq_printf(s, "Logical Link State: %s\n", qedf->lport->link_up ?
+	    "Up" : "Down");
+	seq_printf(s, "FIP state: %s\n", fip_state_names[qedf->ctlr.state]);
+	seq_printf(s, "FIP VLAN ID: %d\n", qedf->vlan_id & 0xfff);
+	seq_printf(s, "FIP 802.1Q Priority: %d\n", qedf->prio);
+	if (qedf->ctlr.sel_fcf) {
+		seq_printf(s, "FCF WWPN: %016llx\n",
+			   qedf->ctlr.sel_fcf->switch_name);
+		seq_printf(s, "FCF MAC: %pM\n", qedf->ctlr.sel_fcf->fcf_mac);
+	} else {
+		seq_puts(s, "FCF not selected\n");
+	}
+
+	seq_puts(s, "\nSGE stats:\n\n");
 	seq_printf(s, "cmg_mgr free io_reqs: %d\n",
 	    atomic_read(&qedf->cmd_mgr->free_list_cnt));
 	seq_printf(s, "slow SGEs: %d\n", qedf->slow_sge_ios);
-	seq_printf(s, "single SGEs: %d\n", qedf->single_sge_ios);
 	seq_printf(s, "fast SGEs: %d\n\n", qedf->fast_sge_ios);
 
 	seq_puts(s, "Offloaded ports:\n\n");
@@ -313,9 +358,12 @@ qedf_driver_stats_show(struct seq_file *s, void *unused)
 		rdata = fcport->rdata;
 		if (rdata == NULL)
 			continue;
-		seq_printf(s, "%06x: free_sqes: %d, num_active_ios: %d\n",
-		    rdata->ids.port_id, atomic_read(&fcport->free_sqes),
-		    atomic_read(&fcport->num_active_ios));
+		seq_printf(s, "%016llx/%016llx/%06x: state=%s, free_sqes=%d, num_active_ios=%d\n",
+			   rdata->rport->node_name, rdata->rport->port_name,
+			   rdata->ids.port_id,
+			   fc_rport_state_names[rdata->rp_state],
+			   atomic_read(&fcport->free_sqes),
+			   atomic_read(&fcport->num_active_ios));
 	}
 	rcu_read_unlock();
 
@@ -361,7 +409,6 @@ qedf_dbg_clear_stats_cmd_write(struct file *filp,
 
 	/* Clear stat counters exposed by 'stats' node */
 	qedf->slow_sge_ios = 0;
-	qedf->single_sge_ios = 0;
 	qedf->fast_sge_ios = 0;
 
 	return count;
diff --git a/drivers/scsi/qedf/qedf_els.c b/drivers/scsi/qedf/qedf_els.c
index 04f0c4d2e256..d900c89e8049 100644
--- a/drivers/scsi/qedf/qedf_els.c
+++ b/drivers/scsi/qedf/qedf_els.c
@@ -23,8 +23,6 @@ static int qedf_initiate_els(struct qedf_rport *fcport, unsigned int op,
 	int rc = 0;
 	uint32_t did, sid;
 	uint16_t xid;
-	uint32_t start_time = jiffies / HZ;
-	uint32_t current_time;
 	struct fcoe_wqe *sqe;
 	unsigned long flags;
 	u16 sqe_idx;
@@ -59,18 +57,12 @@ static int qedf_initiate_els(struct qedf_rport *fcport, unsigned int op,
 		goto els_err;
 	}
 
-retry_els:
 	els_req = qedf_alloc_cmd(fcport, QEDF_ELS);
 	if (!els_req) {
-		current_time = jiffies / HZ;
-		if ((current_time - start_time) > 10) {
-			QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_ELS,
-				   "els: Failed els 0x%x\n", op);
-			rc = -ENOMEM;
-			goto els_err;
-		}
-		mdelay(20 * USEC_PER_MSEC);
-		goto retry_els;
+		QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_ELS,
+			  "Failed to alloc ELS request 0x%x\n", op);
+		rc = -ENOMEM;
+		goto els_err;
 	}
 
 	QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_ELS, "initiate_els els_req = "
@@ -143,6 +135,8 @@ retry_els:
 	QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_ELS, "Ringing doorbell for ELS "
 		   "req\n");
 	qedf_ring_doorbell(fcport);
+	set_bit(QEDF_CMD_OUTSTANDING, &els_req->flags);
+
 	spin_unlock_irqrestore(&fcport->rport_lock, flags);
 els_err:
 	return rc;
@@ -151,21 +145,16 @@ els_err:
 void qedf_process_els_compl(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
 	struct qedf_ioreq *els_req)
 {
-	struct fcoe_task_context *task_ctx;
-	struct scsi_cmnd *sc_cmd;
-	uint16_t xid;
 	struct fcoe_cqe_midpath_info *mp_info;
 
 	QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_ELS, "Entered with xid = 0x%x"
 		   " cmd_type = %d.\n", els_req->xid, els_req->cmd_type);
 
+	clear_bit(QEDF_CMD_OUTSTANDING, &els_req->flags);
+
 	/* Kill the ELS timer */
 	cancel_delayed_work(&els_req->timeout_work);
 
-	xid = els_req->xid;
-	task_ctx = qedf_get_task_mem(&qedf->tasks, xid);
-	sc_cmd = els_req->sc_cmd;
-
 	/* Get ELS response length from CQE */
 	mp_info = &cqe->cqe_info.midpath_info;
 	els_req->mp_req.resp_len = mp_info->data_placement_size;
@@ -205,8 +194,12 @@ static void qedf_rrq_compl(struct qedf_els_cb_arg *cb_arg)
 		   " orig xid = 0x%x, rrq_xid = 0x%x, refcount=%d\n",
 		   orig_io_req, orig_io_req->xid, rrq_req->xid, refcount);
 
-	/* This should return the aborted io_req to the command pool */
-	if (orig_io_req)
+	/*
+	 * This should return the aborted io_req to the command pool. Note that
+	 * we need to check the refcound in case the original request was
+	 * flushed but we get a completion on this xid.
+	 */
+	if (orig_io_req && refcount > 0)
 		kref_put(&orig_io_req->refcount, qedf_release_cmd);
 
 out_free:
@@ -233,6 +226,7 @@ int qedf_send_rrq(struct qedf_ioreq *aborted_io_req)
 	uint32_t sid;
 	uint32_t r_a_tov;
 	int rc;
+	int refcount;
 
 	if (!aborted_io_req) {
 		QEDF_ERR(NULL, "abort_io_req is NULL.\n");
@@ -241,6 +235,15 @@ int qedf_send_rrq(struct qedf_ioreq *aborted_io_req)
 
 	fcport = aborted_io_req->fcport;
 
+	if (!fcport) {
+		refcount = kref_read(&aborted_io_req->refcount);
+		QEDF_ERR(NULL,
+			 "RRQ work was queued prior to a flush xid=0x%x, refcount=%d.\n",
+			 aborted_io_req->xid, refcount);
+		kref_put(&aborted_io_req->refcount, qedf_release_cmd);
+		return -EINVAL;
+	}
+
 	/* Check that fcport is still offloaded */
 	if (!test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags)) {
 		QEDF_ERR(NULL, "fcport is no longer offloaded.\n");
@@ -253,6 +256,19 @@ int qedf_send_rrq(struct qedf_ioreq *aborted_io_req)
 	}
 
 	qedf = fcport->qedf;
+
+	/*
+	 * Sanity check that we can send a RRQ to make sure that refcount isn't
+	 * 0
+	 */
+	refcount = kref_read(&aborted_io_req->refcount);
+	if (refcount != 1) {
+		QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_ELS,
+			  "refcount for xid=%x io_req=%p refcount=%d is not 1.\n",
+			  aborted_io_req->xid, aborted_io_req, refcount);
+		return -EINVAL;
+	}
+
 	lport = qedf->lport;
 	sid = fcport->sid;
 	r_a_tov = lport->r_a_tov;
@@ -335,32 +351,49 @@ void qedf_restart_rport(struct qedf_rport *fcport)
 	struct fc_lport *lport;
 	struct fc_rport_priv *rdata;
 	u32 port_id;
+	unsigned long flags;
 
 	if (!fcport)
 		return;
 
+	spin_lock_irqsave(&fcport->rport_lock, flags);
 	if (test_bit(QEDF_RPORT_IN_RESET, &fcport->flags) ||
 	    !test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags) ||
 	    test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags)) {
 		QEDF_ERR(&(fcport->qedf->dbg_ctx), "fcport %p already in reset or not offloaded.\n",
 		    fcport);
+		spin_unlock_irqrestore(&fcport->rport_lock, flags);
 		return;
 	}
 
 	/* Set that we are now in reset */
 	set_bit(QEDF_RPORT_IN_RESET, &fcport->flags);
+	spin_unlock_irqrestore(&fcport->rport_lock, flags);
 
 	rdata = fcport->rdata;
-	if (rdata) {
+	if (rdata && !kref_get_unless_zero(&rdata->kref)) {
+		fcport->rdata = NULL;
+		rdata = NULL;
+	}
+
+	if (rdata && rdata->rp_state == RPORT_ST_READY) {
 		lport = fcport->qedf->lport;
 		port_id = rdata->ids.port_id;
 		QEDF_ERR(&(fcport->qedf->dbg_ctx),
 		    "LOGO port_id=%x.\n", port_id);
 		fc_rport_logoff(rdata);
+		kref_put(&rdata->kref, fc_rport_destroy);
+		mutex_lock(&lport->disc.disc_mutex);
 		/* Recreate the rport and log back in */
 		rdata = fc_rport_create(lport, port_id);
-		if (rdata)
+		if (rdata) {
+			mutex_unlock(&lport->disc.disc_mutex);
 			fc_rport_login(rdata);
+			fcport->rdata = rdata;
+		} else {
+			mutex_unlock(&lport->disc.disc_mutex);
+			fcport->rdata = NULL;
+		}
 	}
 	clear_bit(QEDF_RPORT_IN_RESET, &fcport->flags);
 }
@@ -569,7 +602,7 @@ static int qedf_send_srr(struct qedf_ioreq *orig_io_req, u32 offset, u8 r_ctl)
 	struct qedf_rport *fcport;
 	struct fc_lport *lport;
 	struct qedf_els_cb_arg *cb_arg = NULL;
-	u32 sid, r_a_tov;
+	u32 r_a_tov;
 	int rc;
 
 	if (!orig_io_req) {
@@ -595,7 +628,6 @@ static int qedf_send_srr(struct qedf_ioreq *orig_io_req, u32 offset, u8 r_ctl)
 
 	qedf = fcport->qedf;
 	lport = qedf->lport;
-	sid = fcport->sid;
 	r_a_tov = lport->r_a_tov;
 
 	QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_ELS, "Sending SRR orig_io=%p, "
diff --git a/drivers/scsi/qedf/qedf_fip.c b/drivers/scsi/qedf/qedf_fip.c
index 3fd3af799b3d..d4741f8dcb41 100644
--- a/drivers/scsi/qedf/qedf_fip.c
+++ b/drivers/scsi/qedf/qedf_fip.c
@@ -19,17 +19,16 @@ void qedf_fcoe_send_vlan_req(struct qedf_ctx *qedf)
 {
 	struct sk_buff *skb;
 	char *eth_fr;
-	int fr_len;
 	struct fip_vlan *vlan;
 #define MY_FIP_ALL_FCF_MACS        ((__u8[6]) { 1, 0x10, 0x18, 1, 0, 2 })
 	static u8 my_fcoe_all_fcfs[ETH_ALEN] = MY_FIP_ALL_FCF_MACS;
 	unsigned long flags = 0;
+	int rc = -1;
 
 	skb = dev_alloc_skb(sizeof(struct fip_vlan));
 	if (!skb)
 		return;
 
-	fr_len = sizeof(*vlan);
 	eth_fr = (char *)skb->data;
 	vlan = (struct fip_vlan *)eth_fr;
 
@@ -68,7 +67,13 @@ void qedf_fcoe_send_vlan_req(struct qedf_ctx *qedf)
 	}
 
 	set_bit(QED_LL2_XMIT_FLAGS_FIP_DISCOVERY, &flags);
-	qed_ops->ll2->start_xmit(qedf->cdev, skb, flags);
+	rc = qed_ops->ll2->start_xmit(qedf->cdev, skb, flags);
+	if (rc) {
+		QEDF_ERR(&qedf->dbg_ctx, "start_xmit failed rc = %d.\n", rc);
+		kfree_skb(skb);
+		return;
+	}
+
 }
 
 static void qedf_fcoe_process_vlan_resp(struct qedf_ctx *qedf,
@@ -95,6 +100,12 @@ static void qedf_fcoe_process_vlan_resp(struct qedf_ctx *qedf,
 		rlen -= dlen;
 	}
 
+	if (atomic_read(&qedf->link_state) == QEDF_LINK_DOWN) {
+		QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_DISC,
+			  "Dropping VLAN response as link is down.\n");
+		return;
+	}
+
 	QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC, "VLAN response, "
 		   "vid=0x%x.\n", vid);
 
@@ -114,6 +125,7 @@ void qedf_fip_send(struct fcoe_ctlr *fip, struct sk_buff *skb)
 	struct fip_header *fiph;
 	u16 op, vlan_tci = 0;
 	u8 sub;
+	int rc = -1;
 
 	if (!test_bit(QEDF_LL2_STARTED, &qedf->flags)) {
 		QEDF_WARN(&(qedf->dbg_ctx), "LL2 not started\n");
@@ -142,9 +154,16 @@ void qedf_fip_send(struct fcoe_ctlr *fip, struct sk_buff *skb)
 		print_hex_dump(KERN_WARNING, "fip ", DUMP_PREFIX_OFFSET, 16, 1,
 		    skb->data, skb->len, false);
 
-	qed_ops->ll2->start_xmit(qedf->cdev, skb, 0);
+	rc = qed_ops->ll2->start_xmit(qedf->cdev, skb, 0);
+	if (rc) {
+		QEDF_ERR(&qedf->dbg_ctx, "start_xmit failed rc = %d.\n", rc);
+		kfree_skb(skb);
+		return;
+	}
 }
 
+static u8 fcoe_all_enode[ETH_ALEN] = FIP_ALL_ENODE_MACS;
+
 /* Process incoming FIP frames. */
 void qedf_fip_recv(struct qedf_ctx *qedf, struct sk_buff *skb)
 {
@@ -157,20 +176,37 @@ void qedf_fip_recv(struct qedf_ctx *qedf, struct sk_buff *skb)
 	size_t rlen, dlen;
 	u16 op;
 	u8 sub;
-	bool do_reset = false;
+	bool fcf_valid = false;
+	/* Default is to handle CVL regardless of fabric id descriptor */
+	bool fabric_id_valid = true;
+	bool fc_wwpn_valid = false;
+	u64 switch_name;
+	u16 vlan = 0;
 
 	eth_hdr = (struct ethhdr *)skb_mac_header(skb);
 	fiph = (struct fip_header *) ((void *)skb->data + 2 * ETH_ALEN + 2);
 	op = ntohs(fiph->fip_op);
 	sub = fiph->fip_subcode;
 
-	QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_LL2, "FIP frame received: "
-	    "skb=%p fiph=%p source=%pM op=%x sub=%x", skb, fiph,
-	    eth_hdr->h_source, op, sub);
+	QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_LL2,
+		  "FIP frame received: skb=%p fiph=%p source=%pM destn=%pM op=%x sub=%x vlan=%04x",
+		  skb, fiph, eth_hdr->h_source, eth_hdr->h_dest, op,
+		  sub, vlan);
 	if (qedf_dump_frames)
 		print_hex_dump(KERN_WARNING, "fip ", DUMP_PREFIX_OFFSET, 16, 1,
 		    skb->data, skb->len, false);
 
+	if (!ether_addr_equal(eth_hdr->h_dest, qedf->mac) &&
+	    !ether_addr_equal(eth_hdr->h_dest, fcoe_all_enode) &&
+		!ether_addr_equal(eth_hdr->h_dest, qedf->data_src_addr)) {
+		QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_LL2,
+			  "Dropping FIP type 0x%x pkt due to destination MAC mismatch dest_mac=%pM ctlr.dest_addr=%pM data_src_addr=%pM.\n",
+			  op, eth_hdr->h_dest, qedf->mac,
+			  qedf->data_src_addr);
+		kfree_skb(skb);
+		return;
+	}
+
 	/* Handle FIP VLAN resp in the driver */
 	if (op == FIP_OP_VLAN && sub == FIP_SC_VL_NOTE) {
 		qedf_fcoe_process_vlan_resp(qedf, skb);
@@ -199,25 +235,36 @@ void qedf_fip_recv(struct qedf_ctx *qedf, struct sk_buff *skb)
 			switch (desc->fip_dtype) {
 			case FIP_DT_MAC:
 				mp = (struct fip_mac_desc *)desc;
-				QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_LL2,
-				    "fd_mac=%pM\n", mp->fd_mac);
+				QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_DISC,
+					  "Switch fd_mac=%pM.\n", mp->fd_mac);
 				if (ether_addr_equal(mp->fd_mac,
 				    qedf->ctlr.sel_fcf->fcf_mac))
-					do_reset = true;
+					fcf_valid = true;
 				break;
 			case FIP_DT_NAME:
 				wp = (struct fip_wwn_desc *)desc;
-				QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_LL2,
-				    "fc_wwpn=%016llx.\n",
-				    get_unaligned_be64(&wp->fd_wwn));
+				switch_name = get_unaligned_be64(&wp->fd_wwn);
+				QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_DISC,
+					  "Switch fd_wwn=%016llx fcf_switch_name=%016llx.\n",
+					  switch_name,
+					  qedf->ctlr.sel_fcf->switch_name);
+				if (switch_name ==
+				    qedf->ctlr.sel_fcf->switch_name)
+					fc_wwpn_valid = true;
 				break;
 			case FIP_DT_VN_ID:
 				vp = (struct fip_vn_desc *)desc;
-				QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_LL2,
-				    "fd_fc_id=%x.\n", ntoh24(vp->fd_fc_id));
-				if (ntoh24(vp->fd_fc_id) ==
+				QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_DISC,
+					  "vx_port fd_fc_id=%x fd_mac=%pM.\n",
+					  ntoh24(vp->fd_fc_id), vp->fd_mac);
+				/* Check vx_port fabric ID */
+				if (ntoh24(vp->fd_fc_id) !=
 				    qedf->lport->port_id)
-					do_reset = true;
+					fabric_id_valid = false;
+				/* Check vx_port MAC */
+				if (!ether_addr_equal(vp->fd_mac,
+						      qedf->data_src_addr))
+					fabric_id_valid = false;
 				break;
 			default:
 				/* Ignore anything else */
@@ -227,13 +274,11 @@ void qedf_fip_recv(struct qedf_ctx *qedf, struct sk_buff *skb)
 			rlen -= dlen;
 		}
 
-		QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_LL2,
-		    "do_reset=%d.\n", do_reset);
-		if (do_reset) {
-			fcoe_ctlr_link_down(&qedf->ctlr);
-			qedf_wait_for_upload(qedf);
-			fcoe_ctlr_link_up(&qedf->ctlr);
-		}
+		QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_DISC,
+			  "fcf_valid=%d fabric_id_valid=%d fc_wwpn_valid=%d.\n",
+			  fcf_valid, fabric_id_valid, fc_wwpn_valid);
+		if (fcf_valid && fabric_id_valid && fc_wwpn_valid)
+			qedf_ctx_soft_reset(qedf->lport);
 		kfree_skb(skb);
 	} else {
 		/* Everything else is handled by libfcoe */
diff --git a/drivers/scsi/qedf/qedf_io.c b/drivers/scsi/qedf/qedf_io.c
index 53e8221f6816..42f9f2a9d8ea 100644
--- a/drivers/scsi/qedf/qedf_io.c
+++ b/drivers/scsi/qedf/qedf_io.c
@@ -43,8 +43,9 @@ static void qedf_cmd_timeout(struct work_struct *work)
 	switch (io_req->cmd_type) {
 	case QEDF_ABTS:
 		if (qedf == NULL) {
-			QEDF_INFO(NULL, QEDF_LOG_IO, "qedf is NULL for xid=0x%x.\n",
-			    io_req->xid);
+			QEDF_INFO(NULL, QEDF_LOG_IO,
+				  "qedf is NULL for ABTS xid=0x%x.\n",
+				  io_req->xid);
 			return;
 		}
 
@@ -61,6 +62,9 @@ static void qedf_cmd_timeout(struct work_struct *work)
 		 */
 		kref_put(&io_req->refcount, qedf_release_cmd);
 
+		/* Clear in abort bit now that we're done with the command */
+		clear_bit(QEDF_CMD_IN_ABORT, &io_req->flags);
+
 		/*
 		 * Now that the original I/O and the ABTS are complete see
 		 * if we need to reconnect to the target.
@@ -68,6 +72,15 @@ static void qedf_cmd_timeout(struct work_struct *work)
 		qedf_restart_rport(fcport);
 		break;
 	case QEDF_ELS:
+		if (!qedf) {
+			QEDF_INFO(NULL, QEDF_LOG_IO,
+				  "qedf is NULL for ELS xid=0x%x.\n",
+				  io_req->xid);
+			return;
+		}
+		/* ELS request no longer outstanding since it timed out */
+		clear_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
+
 		kref_get(&io_req->refcount);
 		/*
 		 * Don't attempt to clean an ELS timeout as any subseqeunt
@@ -103,7 +116,7 @@ void qedf_cmd_mgr_free(struct qedf_cmd_mgr *cmgr)
 	struct io_bdt *bdt_info;
 	struct qedf_ctx *qedf = cmgr->qedf;
 	size_t bd_tbl_sz;
-	u16 min_xid = QEDF_MIN_XID;
+	u16 min_xid = 0;
 	u16 max_xid = (FCOE_PARAMS_NUM_TASKS - 1);
 	int num_ios;
 	int i;
@@ -157,6 +170,7 @@ static void qedf_handle_rrq(struct work_struct *work)
 	struct qedf_ioreq *io_req =
 	    container_of(work, struct qedf_ioreq, rrq_work.work);
 
+	atomic_set(&io_req->state, QEDFC_CMD_ST_RRQ_ACTIVE);
 	qedf_send_rrq(io_req);
 
 }
@@ -169,7 +183,7 @@ struct qedf_cmd_mgr *qedf_cmd_mgr_alloc(struct qedf_ctx *qedf)
 	u16 xid;
 	int i;
 	int num_ios;
-	u16 min_xid = QEDF_MIN_XID;
+	u16 min_xid = 0;
 	u16 max_xid = (FCOE_PARAMS_NUM_TASKS - 1);
 
 	/* Make sure num_queues is already set before calling this function */
@@ -201,7 +215,7 @@ struct qedf_cmd_mgr *qedf_cmd_mgr_alloc(struct qedf_ctx *qedf)
 	/*
 	 * Initialize I/O request fields.
 	 */
-	xid = QEDF_MIN_XID;
+	xid = 0;
 
 	for (i = 0; i < num_ios; i++) {
 		io_req = &cmgr->cmds[i];
@@ -329,7 +343,7 @@ struct qedf_ioreq *qedf_alloc_cmd(struct qedf_rport *fcport, u8 cmd_type)
 			cmd_mgr->idx = 0;
 
 		/* Check to make sure command was previously freed */
-		if (!test_bit(QEDF_CMD_OUTSTANDING, &io_req->flags))
+		if (!io_req->alloc)
 			break;
 	}
 
@@ -338,7 +352,14 @@ struct qedf_ioreq *qedf_alloc_cmd(struct qedf_rport *fcport, u8 cmd_type)
 		goto out_failed;
 	}
 
-	set_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
+	if (test_bit(QEDF_CMD_DIRTY, &io_req->flags))
+		QEDF_ERR(&qedf->dbg_ctx,
+			 "io_req found to be dirty ox_id = 0x%x.\n",
+			 io_req->xid);
+
+	/* Clear any flags now that we've reallocated the xid */
+	io_req->flags = 0;
+	io_req->alloc = 1;
 	spin_unlock_irqrestore(&cmd_mgr->lock, flags);
 
 	atomic_inc(&fcport->num_active_ios);
@@ -349,8 +370,13 @@ struct qedf_ioreq *qedf_alloc_cmd(struct qedf_rport *fcport, u8 cmd_type)
 	io_req->cmd_mgr = cmd_mgr;
 	io_req->fcport = fcport;
 
+	/* Clear any stale sc_cmd back pointer */
+	io_req->sc_cmd = NULL;
+	io_req->lun = -1;
+
 	/* Hold the io_req against deletion */
-	kref_init(&io_req->refcount);
+	kref_init(&io_req->refcount);	/* ID: 001 */
+	atomic_set(&io_req->state, QEDFC_CMD_ST_IO_ACTIVE);
 
 	/* Bind io_bdt for this io_req */
 	/* Have a static link between io_req and io_bdt_pool */
@@ -412,6 +438,10 @@ void qedf_release_cmd(struct kref *ref)
 	    container_of(ref, struct qedf_ioreq, refcount);
 	struct qedf_cmd_mgr *cmd_mgr = io_req->cmd_mgr;
 	struct qedf_rport *fcport = io_req->fcport;
+	unsigned long flags;
+
+	if (io_req->cmd_type == QEDF_SCSI_CMD)
+		WARN_ON(io_req->sc_cmd);
 
 	if (io_req->cmd_type == QEDF_ELS ||
 	    io_req->cmd_type == QEDF_TASK_MGMT_CMD)
@@ -419,36 +449,20 @@ void qedf_release_cmd(struct kref *ref)
 
 	atomic_inc(&cmd_mgr->free_list_cnt);
 	atomic_dec(&fcport->num_active_ios);
+	atomic_set(&io_req->state, QEDF_CMD_ST_INACTIVE);
 	if (atomic_read(&fcport->num_active_ios) < 0)
 		QEDF_WARN(&(fcport->qedf->dbg_ctx), "active_ios < 0.\n");
 
 	/* Increment task retry identifier now that the request is released */
 	io_req->task_retry_identifier++;
+	io_req->fcport = NULL;
 
-	clear_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
-}
-
-static int qedf_split_bd(struct qedf_ioreq *io_req, u64 addr, int sg_len,
-	int bd_index)
-{
-	struct scsi_sge *bd = io_req->bd_tbl->bd_tbl;
-	int frag_size, sg_frags;
-
-	sg_frags = 0;
-	while (sg_len) {
-		if (sg_len > QEDF_BD_SPLIT_SZ)
-			frag_size = QEDF_BD_SPLIT_SZ;
-		else
-			frag_size = sg_len;
-		bd[bd_index + sg_frags].sge_addr.lo = U64_LO(addr);
-		bd[bd_index + sg_frags].sge_addr.hi = U64_HI(addr);
-		bd[bd_index + sg_frags].sge_len = (uint16_t)frag_size;
-
-		addr += (u64)frag_size;
-		sg_frags++;
-		sg_len -= frag_size;
-	}
-	return sg_frags;
+	clear_bit(QEDF_CMD_DIRTY, &io_req->flags);
+	io_req->cpu = 0;
+	spin_lock_irqsave(&cmd_mgr->lock, flags);
+	io_req->fcport = NULL;
+	io_req->alloc = 0;
+	spin_unlock_irqrestore(&cmd_mgr->lock, flags);
 }
 
 static int qedf_map_sg(struct qedf_ioreq *io_req)
@@ -462,75 +476,45 @@ static int qedf_map_sg(struct qedf_ioreq *io_req)
 	int byte_count = 0;
 	int sg_count = 0;
 	int bd_count = 0;
-	int sg_frags;
-	unsigned int sg_len;
+	u32 sg_len;
 	u64 addr, end_addr;
-	int i;
+	int i = 0;
 
 	sg_count = dma_map_sg(&qedf->pdev->dev, scsi_sglist(sc),
 	    scsi_sg_count(sc), sc->sc_data_direction);
-
 	sg = scsi_sglist(sc);
 
-	/*
-	 * New condition to send single SGE as cached-SGL with length less
-	 * than 64k.
-	 */
-	if ((sg_count == 1) && (sg_dma_len(sg) <=
-	    QEDF_MAX_SGLEN_FOR_CACHESGL)) {
-		sg_len = sg_dma_len(sg);
-		addr = (u64)sg_dma_address(sg);
-
-		bd[bd_count].sge_addr.lo = (addr & 0xffffffff);
-		bd[bd_count].sge_addr.hi = (addr >> 32);
-		bd[bd_count].sge_len = (u16)sg_len;
+	io_req->sge_type = QEDF_IOREQ_UNKNOWN_SGE;
 
-		return ++bd_count;
-	}
+	if (sg_count <= 8 || io_req->io_req_flags == QEDF_READ)
+		io_req->sge_type = QEDF_IOREQ_FAST_SGE;
 
 	scsi_for_each_sg(sc, sg, sg_count, i) {
-		sg_len = sg_dma_len(sg);
+		sg_len = (u32)sg_dma_len(sg);
 		addr = (u64)sg_dma_address(sg);
 		end_addr = (u64)(addr + sg_len);
 
 		/*
-		 * First s/g element in the list so check if the end_addr
-		 * is paged aligned. Also check to make sure the length is
-		 * at least page size.
-		 */
-		if ((i == 0) && (sg_count > 1) &&
-		    ((end_addr % QEDF_PAGE_SIZE) ||
-		    sg_len < QEDF_PAGE_SIZE))
-			io_req->use_slowpath = true;
-		/*
-		 * Last s/g element so check if the start address is paged
-		 * aligned.
-		 */
-		else if ((i == (sg_count - 1)) && (sg_count > 1) &&
-		    (addr % QEDF_PAGE_SIZE))
-			io_req->use_slowpath = true;
-		/*
 		 * Intermediate s/g element so check if start and end address
-		 * is page aligned.
+		 * is page aligned.  Only required for writes and only if the
+		 * number of scatter/gather elements is 8 or more.
 		 */
-		else if ((i != 0) && (i != (sg_count - 1)) &&
-		    ((addr % QEDF_PAGE_SIZE) || (end_addr % QEDF_PAGE_SIZE)))
-			io_req->use_slowpath = true;
+		if (io_req->sge_type == QEDF_IOREQ_UNKNOWN_SGE && (i) &&
+		    (i != (sg_count - 1)) && sg_len < QEDF_PAGE_SIZE)
+			io_req->sge_type = QEDF_IOREQ_SLOW_SGE;
 
-		if (sg_len > QEDF_MAX_BD_LEN) {
-			sg_frags = qedf_split_bd(io_req, addr, sg_len,
-			    bd_count);
-		} else {
-			sg_frags = 1;
-			bd[bd_count].sge_addr.lo = U64_LO(addr);
-			bd[bd_count].sge_addr.hi  = U64_HI(addr);
-			bd[bd_count].sge_len = (uint16_t)sg_len;
-		}
+		bd[bd_count].sge_addr.lo = cpu_to_le32(U64_LO(addr));
+		bd[bd_count].sge_addr.hi  = cpu_to_le32(U64_HI(addr));
+		bd[bd_count].sge_len = cpu_to_le32(sg_len);
 
-		bd_count += sg_frags;
+		bd_count++;
 		byte_count += sg_len;
 	}
 
+	/* To catch a case where FAST and SLOW nothing is set, set FAST */
+	if (io_req->sge_type == QEDF_IOREQ_UNKNOWN_SGE)
+		io_req->sge_type = QEDF_IOREQ_FAST_SGE;
+
 	if (byte_count != scsi_bufflen(sc))
 		QEDF_ERR(&(qedf->dbg_ctx), "byte_count = %d != "
 			  "scsi_bufflen = %d, task_id = 0x%x.\n", byte_count,
@@ -655,8 +639,10 @@ static void  qedf_init_task(struct qedf_rport *fcport, struct fc_lport *lport,
 		io_req->sgl_task_params->num_sges = bd_count;
 		io_req->sgl_task_params->total_buffer_size =
 		    scsi_bufflen(io_req->sc_cmd);
-		io_req->sgl_task_params->small_mid_sge =
-			io_req->use_slowpath;
+		if (io_req->sge_type == QEDF_IOREQ_SLOW_SGE)
+			io_req->sgl_task_params->small_mid_sge = 1;
+		else
+			io_req->sgl_task_params->small_mid_sge = 0;
 	}
 
 	/* Fill in physical address of sense buffer */
@@ -679,16 +665,10 @@ static void  qedf_init_task(struct qedf_rport *fcport, struct fc_lport *lport,
 				    io_req->task_retry_identifier, fcp_cmnd);
 
 	/* Increment SGL type counters */
-	if (bd_count == 1) {
-		qedf->single_sge_ios++;
-		io_req->sge_type = QEDF_IOREQ_SINGLE_SGE;
-	} else if (io_req->use_slowpath) {
+	if (io_req->sge_type == QEDF_IOREQ_SLOW_SGE)
 		qedf->slow_sge_ios++;
-		io_req->sge_type = QEDF_IOREQ_SLOW_SGE;
-	} else {
+	else
 		qedf->fast_sge_ios++;
-		io_req->sge_type = QEDF_IOREQ_FAST_SGE;
-	}
 }
 
 void qedf_init_mp_task(struct qedf_ioreq *io_req,
@@ -770,9 +750,6 @@ void qedf_init_mp_task(struct qedf_ioreq *io_req,
 						     &task_fc_hdr,
 						     &tx_sgl_task_params,
 						     &rx_sgl_task_params, 0);
-
-	/* Midpath requests always consume 1 SGE */
-	qedf->single_sge_ios++;
 }
 
 /* Presumed that fcport->rport_lock is held */
@@ -804,8 +781,17 @@ void qedf_ring_doorbell(struct qedf_rport *fcport)
 	    FCOE_DB_DATA_AGG_VAL_SEL_SHIFT;
 
 	dbell.sq_prod = fcport->fw_sq_prod_idx;
+	/* wmb makes sure that the BDs data is updated before updating the
+	 * producer, otherwise FW may read old data from the BDs.
+	 */
+	wmb();
+	barrier();
 	writel(*(u32 *)&dbell, fcport->p_doorbell);
-	/* Make sure SQ index is updated so f/w prcesses requests in order */
+	/*
+	 * Fence required to flush the write combined buffer, since another
+	 * CPU may write to the same doorbell address and data may be lost
+	 * due to relaxed order nature of write combined bar.
+	 */
 	wmb();
 }
 
@@ -871,7 +857,7 @@ int qedf_post_io_req(struct qedf_rport *fcport, struct qedf_ioreq *io_req)
 	/* Initialize rest of io_req fileds */
 	io_req->data_xfer_len = scsi_bufflen(sc_cmd);
 	sc_cmd->SCp.ptr = (char *)io_req;
-	io_req->use_slowpath = false; /* Assume fast SGL by default */
+	io_req->sge_type = QEDF_IOREQ_FAST_SGE; /* Assume fast SGL by default */
 
 	/* Record which cpu this request is associated with */
 	io_req->cpu = smp_processor_id();
@@ -894,15 +880,24 @@ int qedf_post_io_req(struct qedf_rport *fcport, struct qedf_ioreq *io_req)
 	/* Build buffer descriptor list for firmware from sg list */
 	if (qedf_build_bd_list_from_sg(io_req)) {
 		QEDF_ERR(&(qedf->dbg_ctx), "BD list creation failed.\n");
+		/* Release cmd will release io_req, but sc_cmd is assigned */
+		io_req->sc_cmd = NULL;
 		kref_put(&io_req->refcount, qedf_release_cmd);
 		return -EAGAIN;
 	}
 
-	if (!test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags)) {
+	if (!test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags) ||
+	    test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags)) {
 		QEDF_ERR(&(qedf->dbg_ctx), "Session not offloaded yet.\n");
+		/* Release cmd will release io_req, but sc_cmd is assigned */
+		io_req->sc_cmd = NULL;
 		kref_put(&io_req->refcount, qedf_release_cmd);
+		return -EINVAL;
 	}
 
+	/* Record LUN number for later use if we neeed them */
+	io_req->lun = (int)sc_cmd->device->lun;
+
 	/* Obtain free SQE */
 	sqe_idx = qedf_get_sqe_idx(fcport);
 	sqe = &fcport->sq[sqe_idx];
@@ -913,6 +908,8 @@ int qedf_post_io_req(struct qedf_rport *fcport, struct qedf_ioreq *io_req)
 	if (!task_ctx) {
 		QEDF_WARN(&(qedf->dbg_ctx), "task_ctx is NULL, xid=%d.\n",
 			   xid);
+		/* Release cmd will release io_req, but sc_cmd is assigned */
+		io_req->sc_cmd = NULL;
 		kref_put(&io_req->refcount, qedf_release_cmd);
 		return -EINVAL;
 	}
@@ -922,6 +919,9 @@ int qedf_post_io_req(struct qedf_rport *fcport, struct qedf_ioreq *io_req)
 	/* Ring doorbell */
 	qedf_ring_doorbell(fcport);
 
+	/* Set that command is with the firmware now */
+	set_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
+
 	if (qedf_io_tracing && io_req->sc_cmd)
 		qedf_trace_io(fcport, io_req, QEDF_IO_TRACE_REQ);
 
@@ -940,7 +940,17 @@ qedf_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *sc_cmd)
 	int rc = 0;
 	int rval;
 	unsigned long flags = 0;
-
+	int num_sgs = 0;
+
+	num_sgs = scsi_sg_count(sc_cmd);
+	if (scsi_sg_count(sc_cmd) > QEDF_MAX_BDS_PER_CMD) {
+		QEDF_ERR(&qedf->dbg_ctx,
+			 "Number of SG elements %d exceeds what hardware limitation of %d.\n",
+			 num_sgs, QEDF_MAX_BDS_PER_CMD);
+		sc_cmd->result = DID_ERROR;
+		sc_cmd->scsi_done(sc_cmd);
+		return 0;
+	}
 
 	if (test_bit(QEDF_UNLOADING, &qedf->flags) ||
 	    test_bit(QEDF_DBG_STOP_IO, &qedf->flags)) {
@@ -980,7 +990,8 @@ qedf_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *sc_cmd)
 	/* rport and tgt are allocated together, so tgt should be non-NULL */
 	fcport = (struct qedf_rport *)&rp[1];
 
-	if (!test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags)) {
+	if (!test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags) ||
+	    test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags)) {
 		/*
 		 * Session is not offloaded yet. Let SCSI-ml retry
 		 * the command.
@@ -988,12 +999,16 @@ qedf_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *sc_cmd)
 		rc = SCSI_MLQUEUE_TARGET_BUSY;
 		goto exit_qcmd;
 	}
+
+	atomic_inc(&fcport->ios_to_queue);
+
 	if (fcport->retry_delay_timestamp) {
 		if (time_after(jiffies, fcport->retry_delay_timestamp)) {
 			fcport->retry_delay_timestamp = 0;
 		} else {
 			/* If retry_delay timer is active, flow off the ML */
 			rc = SCSI_MLQUEUE_TARGET_BUSY;
+			atomic_dec(&fcport->ios_to_queue);
 			goto exit_qcmd;
 		}
 	}
@@ -1001,6 +1016,7 @@ qedf_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *sc_cmd)
 	io_req = qedf_alloc_cmd(fcport, QEDF_SCSI_CMD);
 	if (!io_req) {
 		rc = SCSI_MLQUEUE_HOST_BUSY;
+		atomic_dec(&fcport->ios_to_queue);
 		goto exit_qcmd;
 	}
 
@@ -1015,6 +1031,7 @@ qedf_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *sc_cmd)
 		rc = SCSI_MLQUEUE_HOST_BUSY;
 	}
 	spin_unlock_irqrestore(&fcport->rport_lock, flags);
+	atomic_dec(&fcport->ios_to_queue);
 
 exit_qcmd:
 	return rc;
@@ -1091,7 +1108,7 @@ static void qedf_unmap_sg_list(struct qedf_ctx *qedf, struct qedf_ioreq *io_req)
 void qedf_scsi_completion(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
 	struct qedf_ioreq *io_req)
 {
-	u16 xid, rval;
+	u16 xid;
 	struct e4_fcoe_task_context *task_ctx;
 	struct scsi_cmnd *sc_cmd;
 	struct fcoe_cqe_rsp_info *fcp_rsp;
@@ -1105,6 +1122,15 @@ void qedf_scsi_completion(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
 	if (!cqe)
 		return;
 
+	if (!test_bit(QEDF_CMD_OUTSTANDING, &io_req->flags) ||
+	    test_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags) ||
+	    test_bit(QEDF_CMD_IN_ABORT, &io_req->flags)) {
+		QEDF_ERR(&qedf->dbg_ctx,
+			 "io_req xid=0x%x already in cleanup or abort processing or already completed.\n",
+			 io_req->xid);
+		return;
+	}
+
 	xid = io_req->xid;
 	task_ctx = qedf_get_task_mem(&qedf->tasks, xid);
 	sc_cmd = io_req->sc_cmd;
@@ -1121,6 +1147,12 @@ void qedf_scsi_completion(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
 		return;
 	}
 
+	if (!sc_cmd->device) {
+		QEDF_ERR(&qedf->dbg_ctx,
+			 "Device for sc_cmd %p is NULL.\n", sc_cmd);
+		return;
+	}
+
 	if (!sc_cmd->request) {
 		QEDF_WARN(&(qedf->dbg_ctx), "sc_cmd->request is NULL, "
 		    "sc_cmd=%p.\n", sc_cmd);
@@ -1135,6 +1167,19 @@ void qedf_scsi_completion(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
 
 	fcport = io_req->fcport;
 
+	/*
+	 * When flush is active, let the cmds be completed from the cleanup
+	 * context
+	 */
+	if (test_bit(QEDF_RPORT_IN_TARGET_RESET, &fcport->flags) ||
+	    (test_bit(QEDF_RPORT_IN_LUN_RESET, &fcport->flags) &&
+	     sc_cmd->device->lun == (u64)fcport->lun_reset_lun)) {
+		QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
+			  "Dropping good completion xid=0x%x as fcport is flushing",
+			  io_req->xid);
+		return;
+	}
+
 	qedf_parse_fcp_rsp(io_req, fcp_rsp);
 
 	qedf_unmap_sg_list(qedf, io_req);
@@ -1152,25 +1197,18 @@ void qedf_scsi_completion(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
 	fw_residual_flag = GET_FIELD(cqe->cqe_info.rsp_info.fw_error_flags,
 	    FCOE_CQE_RSP_INFO_FW_UNDERRUN);
 	if (fw_residual_flag) {
-		QEDF_ERR(&(qedf->dbg_ctx),
-		    "Firmware detected underrun: xid=0x%x fcp_rsp.flags=0x%02x "
-		    "fcp_resid=%d fw_residual=0x%x.\n", io_req->xid,
-		    fcp_rsp->rsp_flags.flags, io_req->fcp_resid,
-		    cqe->cqe_info.rsp_info.fw_residual);
+		QEDF_ERR(&qedf->dbg_ctx,
+			 "Firmware detected underrun: xid=0x%x fcp_rsp.flags=0x%02x fcp_resid=%d fw_residual=0x%x lba=%02x%02x%02x%02x.\n",
+			 io_req->xid, fcp_rsp->rsp_flags.flags,
+			 io_req->fcp_resid,
+			 cqe->cqe_info.rsp_info.fw_residual, sc_cmd->cmnd[2],
+			 sc_cmd->cmnd[3], sc_cmd->cmnd[4], sc_cmd->cmnd[5]);
 
 		if (io_req->cdb_status == 0)
 			sc_cmd->result = (DID_ERROR << 16) | io_req->cdb_status;
 		else
 			sc_cmd->result = (DID_OK << 16) | io_req->cdb_status;
 
-		/* Abort the command since we did not get all the data */
-		init_completion(&io_req->abts_done);
-		rval = qedf_initiate_abts(io_req, true);
-		if (rval) {
-			QEDF_ERR(&(qedf->dbg_ctx), "Failed to queue ABTS.\n");
-			sc_cmd->result = (DID_ERROR << 16) | io_req->cdb_status;
-		}
-
 		/*
 		 * Set resid to the whole buffer length so we won't try to resue
 		 * any previously data.
@@ -1242,6 +1280,12 @@ out:
 	if (qedf_io_tracing)
 		qedf_trace_io(fcport, io_req, QEDF_IO_TRACE_RSP);
 
+	/*
+	 * We wait till the end of the function to clear the
+	 * outstanding bit in case we need to send an abort
+	 */
+	clear_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
+
 	io_req->sc_cmd = NULL;
 	sc_cmd->SCp.ptr =  NULL;
 	sc_cmd->scsi_done(sc_cmd);
@@ -1259,6 +1303,19 @@ void qedf_scsi_done(struct qedf_ctx *qedf, struct qedf_ioreq *io_req,
 	if (!io_req)
 		return;
 
+	if (test_and_set_bit(QEDF_CMD_ERR_SCSI_DONE, &io_req->flags)) {
+		QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
+			  "io_req:%p scsi_done handling already done\n",
+			  io_req);
+		return;
+	}
+
+	/*
+	 * We will be done with this command after this call so clear the
+	 * outstanding bit.
+	 */
+	clear_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
+
 	xid = io_req->xid;
 	sc_cmd = io_req->sc_cmd;
 
@@ -1267,12 +1324,50 @@ void qedf_scsi_done(struct qedf_ctx *qedf, struct qedf_ioreq *io_req,
 		return;
 	}
 
+	if (!virt_addr_valid(sc_cmd)) {
+		QEDF_ERR(&qedf->dbg_ctx, "sc_cmd=%p is not valid.", sc_cmd);
+		goto bad_scsi_ptr;
+	}
+
 	if (!sc_cmd->SCp.ptr) {
 		QEDF_WARN(&(qedf->dbg_ctx), "SCp.ptr is NULL, returned in "
 		    "another context.\n");
 		return;
 	}
 
+	if (!sc_cmd->device) {
+		QEDF_ERR(&qedf->dbg_ctx, "Device for sc_cmd %p is NULL.\n",
+			 sc_cmd);
+		goto bad_scsi_ptr;
+	}
+
+	if (!virt_addr_valid(sc_cmd->device)) {
+		QEDF_ERR(&qedf->dbg_ctx,
+			 "Device pointer for sc_cmd %p is bad.\n", sc_cmd);
+		goto bad_scsi_ptr;
+	}
+
+	if (!sc_cmd->sense_buffer) {
+		QEDF_ERR(&qedf->dbg_ctx,
+			 "sc_cmd->sense_buffer for sc_cmd %p is NULL.\n",
+			 sc_cmd);
+		goto bad_scsi_ptr;
+	}
+
+	if (!virt_addr_valid(sc_cmd->sense_buffer)) {
+		QEDF_ERR(&qedf->dbg_ctx,
+			 "sc_cmd->sense_buffer for sc_cmd %p is bad.\n",
+			 sc_cmd);
+		goto bad_scsi_ptr;
+	}
+
+	if (!sc_cmd->scsi_done) {
+		QEDF_ERR(&qedf->dbg_ctx,
+			 "sc_cmd->scsi_done for sc_cmd %p is NULL.\n",
+			 sc_cmd);
+		goto bad_scsi_ptr;
+	}
+
 	qedf_unmap_sg_list(qedf, io_req);
 
 	sc_cmd->result = result << 16;
@@ -1299,6 +1394,15 @@ void qedf_scsi_done(struct qedf_ctx *qedf, struct qedf_ioreq *io_req,
 	sc_cmd->SCp.ptr = NULL;
 	sc_cmd->scsi_done(sc_cmd);
 	kref_put(&io_req->refcount, qedf_release_cmd);
+	return;
+
+bad_scsi_ptr:
+	/*
+	 * Clear the io_req->sc_cmd backpointer so we don't try to process
+	 * this again
+	 */
+	io_req->sc_cmd = NULL;
+	kref_put(&io_req->refcount, qedf_release_cmd);  /* ID: 001 */
 }
 
 /*
@@ -1437,6 +1541,10 @@ void qedf_flush_active_ios(struct qedf_rport *fcport, int lun)
 	struct qedf_ctx *qedf;
 	struct qedf_cmd_mgr *cmd_mgr;
 	int i, rc;
+	unsigned long flags;
+	int flush_cnt = 0;
+	int wait_cnt = 100;
+	int refcount = 0;
 
 	if (!fcport)
 		return;
@@ -1448,18 +1556,102 @@ void qedf_flush_active_ios(struct qedf_rport *fcport, int lun)
 	}
 
 	qedf = fcport->qedf;
+
+	if (!qedf) {
+		QEDF_ERR(NULL, "qedf is NULL.\n");
+		return;
+	}
+
+	/* Only wait for all commands to be queued in the Upload context */
+	if (test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags) &&
+	    (lun == -1)) {
+		while (atomic_read(&fcport->ios_to_queue)) {
+			QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
+				  "Waiting for %d I/Os to be queued\n",
+				  atomic_read(&fcport->ios_to_queue));
+			if (wait_cnt == 0) {
+				QEDF_ERR(NULL,
+					 "%d IOs request could not be queued\n",
+					 atomic_read(&fcport->ios_to_queue));
+			}
+			msleep(20);
+			wait_cnt--;
+		}
+	}
+
 	cmd_mgr = qedf->cmd_mgr;
 
-	QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO, "Flush active i/o's.\n");
+	QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
+		  "Flush active i/o's num=0x%x fcport=0x%p port_id=0x%06x scsi_id=%d.\n",
+		  atomic_read(&fcport->num_active_ios), fcport,
+		  fcport->rdata->ids.port_id, fcport->rport->scsi_target_id);
+	QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO, "Locking flush mutex.\n");
+
+	mutex_lock(&qedf->flush_mutex);
+	if (lun == -1) {
+		set_bit(QEDF_RPORT_IN_TARGET_RESET, &fcport->flags);
+	} else {
+		set_bit(QEDF_RPORT_IN_LUN_RESET, &fcport->flags);
+		fcport->lun_reset_lun = lun;
+	}
 
 	for (i = 0; i < FCOE_PARAMS_NUM_TASKS; i++) {
 		io_req = &cmd_mgr->cmds[i];
 
 		if (!io_req)
 			continue;
+		if (!io_req->fcport)
+			continue;
+
+		spin_lock_irqsave(&cmd_mgr->lock, flags);
+
+		if (io_req->alloc) {
+			if (!test_bit(QEDF_CMD_OUTSTANDING, &io_req->flags)) {
+				if (io_req->cmd_type == QEDF_SCSI_CMD)
+					QEDF_ERR(&qedf->dbg_ctx,
+						 "Allocated but not queued, xid=0x%x\n",
+						 io_req->xid);
+			}
+			spin_unlock_irqrestore(&cmd_mgr->lock, flags);
+		} else {
+			spin_unlock_irqrestore(&cmd_mgr->lock, flags);
+			continue;
+		}
+
 		if (io_req->fcport != fcport)
 			continue;
-		if (io_req->cmd_type == QEDF_ELS) {
+
+		/* In case of ABTS, CMD_OUTSTANDING is cleared on ABTS response,
+		 * but RRQ is still pending.
+		 * Workaround: Within qedf_send_rrq, we check if the fcport is
+		 * NULL, and we drop the ref on the io_req to clean it up.
+		 */
+		if (!test_bit(QEDF_CMD_OUTSTANDING, &io_req->flags)) {
+			refcount = kref_read(&io_req->refcount);
+			QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
+				  "Not outstanding, xid=0x%x, cmd_type=%d refcount=%d.\n",
+				  io_req->xid, io_req->cmd_type, refcount);
+			/* If RRQ work has been queue, try to cancel it and
+			 * free the io_req
+			 */
+			if (atomic_read(&io_req->state) ==
+			    QEDFC_CMD_ST_RRQ_WAIT) {
+				if (cancel_delayed_work_sync
+				    (&io_req->rrq_work)) {
+					QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
+						  "Putting reference for pending RRQ work xid=0x%x.\n",
+						  io_req->xid);
+					/* ID: 003 */
+					kref_put(&io_req->refcount,
+						 qedf_release_cmd);
+				}
+			}
+			continue;
+		}
+
+		/* Only consider flushing ELS during target reset */
+		if (io_req->cmd_type == QEDF_ELS &&
+		    lun == -1) {
 			rc = kref_get_unless_zero(&io_req->refcount);
 			if (!rc) {
 				QEDF_ERR(&(qedf->dbg_ctx),
@@ -1467,6 +1659,7 @@ void qedf_flush_active_ios(struct qedf_rport *fcport, int lun)
 				    io_req, io_req->xid);
 				continue;
 			}
+			flush_cnt++;
 			qedf_flush_els_req(qedf, io_req);
 			/*
 			 * Release the kref and go back to the top of the
@@ -1476,6 +1669,7 @@ void qedf_flush_active_ios(struct qedf_rport *fcport, int lun)
 		}
 
 		if (io_req->cmd_type == QEDF_ABTS) {
+			/* ID: 004 */
 			rc = kref_get_unless_zero(&io_req->refcount);
 			if (!rc) {
 				QEDF_ERR(&(qedf->dbg_ctx),
@@ -1483,28 +1677,50 @@ void qedf_flush_active_ios(struct qedf_rport *fcport, int lun)
 				    io_req, io_req->xid);
 				continue;
 			}
+			if (lun != -1 && io_req->lun != lun)
+				goto free_cmd;
+
 			QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
 			    "Flushing abort xid=0x%x.\n", io_req->xid);
 
-			clear_bit(QEDF_CMD_IN_ABORT, &io_req->flags);
-
-			if (io_req->sc_cmd) {
-				if (io_req->return_scsi_cmd_on_abts)
-					qedf_scsi_done(qedf, io_req, DID_ERROR);
+			if (cancel_delayed_work_sync(&io_req->rrq_work)) {
+				QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
+					  "Putting ref for cancelled RRQ work xid=0x%x.\n",
+					  io_req->xid);
+				kref_put(&io_req->refcount, qedf_release_cmd);
 			}
 
-			/* Notify eh_abort handler that ABTS is complete */
-			complete(&io_req->abts_done);
-			kref_put(&io_req->refcount, qedf_release_cmd);
-
+			if (cancel_delayed_work_sync(&io_req->timeout_work)) {
+				QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
+					  "Putting ref for cancelled tmo work xid=0x%x.\n",
+					  io_req->xid);
+				qedf_initiate_cleanup(io_req, true);
+				/* Notify eh_abort handler that ABTS is
+				 * complete
+				 */
+				complete(&io_req->abts_done);
+				clear_bit(QEDF_CMD_IN_ABORT, &io_req->flags);
+				/* ID: 002 */
+				kref_put(&io_req->refcount, qedf_release_cmd);
+			}
+			flush_cnt++;
 			goto free_cmd;
 		}
 
 		if (!io_req->sc_cmd)
 			continue;
-		if (lun > 0) {
-			if (io_req->sc_cmd->device->lun !=
-			    (u64)lun)
+		if (!io_req->sc_cmd->device) {
+			QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
+				  "Device backpointer NULL for sc_cmd=%p.\n",
+				  io_req->sc_cmd);
+			/* Put reference for non-existent scsi_cmnd */
+			io_req->sc_cmd = NULL;
+			qedf_initiate_cleanup(io_req, false);
+			kref_put(&io_req->refcount, qedf_release_cmd);
+			continue;
+		}
+		if (lun > -1) {
+			if (io_req->lun != lun)
 				continue;
 		}
 
@@ -1518,15 +1734,65 @@ void qedf_flush_active_ios(struct qedf_rport *fcport, int lun)
 			    "io_req=0x%p xid=0x%x\n", io_req, io_req->xid);
 			continue;
 		}
+
 		QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO,
 		    "Cleanup xid=0x%x.\n", io_req->xid);
+		flush_cnt++;
 
 		/* Cleanup task and return I/O mid-layer */
 		qedf_initiate_cleanup(io_req, true);
 
 free_cmd:
-		kref_put(&io_req->refcount, qedf_release_cmd);
+		kref_put(&io_req->refcount, qedf_release_cmd);	/* ID: 004 */
+	}
+
+	wait_cnt = 60;
+	QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
+		  "Flushed 0x%x I/Os, active=0x%x.\n",
+		  flush_cnt, atomic_read(&fcport->num_active_ios));
+	/* Only wait for all commands to complete in the Upload context */
+	if (test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags) &&
+	    (lun == -1)) {
+		while (atomic_read(&fcport->num_active_ios)) {
+			QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
+				  "Flushed 0x%x I/Os, active=0x%x cnt=%d.\n",
+				  flush_cnt,
+				  atomic_read(&fcport->num_active_ios),
+				  wait_cnt);
+			if (wait_cnt == 0) {
+				QEDF_ERR(&qedf->dbg_ctx,
+					 "Flushed %d I/Os, active=%d.\n",
+					 flush_cnt,
+					 atomic_read(&fcport->num_active_ios));
+				for (i = 0; i < FCOE_PARAMS_NUM_TASKS; i++) {
+					io_req = &cmd_mgr->cmds[i];
+					if (io_req->fcport &&
+					    io_req->fcport == fcport) {
+						refcount =
+						kref_read(&io_req->refcount);
+						set_bit(QEDF_CMD_DIRTY,
+							&io_req->flags);
+						QEDF_ERR(&qedf->dbg_ctx,
+							 "Outstanding io_req =%p xid=0x%x flags=0x%lx, sc_cmd=%p refcount=%d cmd_type=%d.\n",
+							 io_req, io_req->xid,
+							 io_req->flags,
+							 io_req->sc_cmd,
+							 refcount,
+							 io_req->cmd_type);
+					}
+				}
+				WARN_ON(1);
+				break;
+			}
+			msleep(500);
+			wait_cnt--;
+		}
 	}
+
+	clear_bit(QEDF_RPORT_IN_LUN_RESET, &fcport->flags);
+	clear_bit(QEDF_RPORT_IN_TARGET_RESET, &fcport->flags);
+	QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO, "Unlocking flush mutex.\n");
+	mutex_unlock(&qedf->flush_mutex);
 }
 
 /*
@@ -1545,52 +1811,60 @@ int qedf_initiate_abts(struct qedf_ioreq *io_req, bool return_scsi_cmd_on_abts)
 	unsigned long flags;
 	struct fcoe_wqe *sqe;
 	u16 sqe_idx;
+	int refcount = 0;
 
 	/* Sanity check qedf_rport before dereferencing any pointers */
 	if (!test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags)) {
 		QEDF_ERR(NULL, "tgt not offloaded\n");
 		rc = 1;
-		goto abts_err;
+		goto out;
 	}
 
+	qedf = fcport->qedf;
 	rdata = fcport->rdata;
+
+	if (!rdata || !kref_get_unless_zero(&rdata->kref)) {
+		QEDF_ERR(&qedf->dbg_ctx, "stale rport\n");
+		rc = 1;
+		goto out;
+	}
+
 	r_a_tov = rdata->r_a_tov;
-	qedf = fcport->qedf;
 	lport = qedf->lport;
 
 	if (lport->state != LPORT_ST_READY || !(lport->link_up)) {
 		QEDF_ERR(&(qedf->dbg_ctx), "link is not ready\n");
 		rc = 1;
-		goto abts_err;
+		goto drop_rdata_kref;
 	}
 
 	if (atomic_read(&qedf->link_down_tmo_valid) > 0) {
 		QEDF_ERR(&(qedf->dbg_ctx), "link_down_tmo active.\n");
 		rc = 1;
-		goto abts_err;
+		goto drop_rdata_kref;
 	}
 
 	/* Ensure room on SQ */
 	if (!atomic_read(&fcport->free_sqes)) {
 		QEDF_ERR(&(qedf->dbg_ctx), "No SQ entries available\n");
 		rc = 1;
-		goto abts_err;
+		goto drop_rdata_kref;
 	}
 
 	if (test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags)) {
 		QEDF_ERR(&qedf->dbg_ctx, "fcport is uploading.\n");
 		rc = 1;
-		goto out;
+		goto drop_rdata_kref;
 	}
 
 	if (!test_bit(QEDF_CMD_OUTSTANDING, &io_req->flags) ||
 	    test_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags) ||
 	    test_bit(QEDF_CMD_IN_ABORT, &io_req->flags)) {
-		QEDF_ERR(&(qedf->dbg_ctx), "io_req xid=0x%x already in "
-			  "cleanup or abort processing or already "
-			  "completed.\n", io_req->xid);
+		QEDF_ERR(&qedf->dbg_ctx,
+			 "io_req xid=0x%x sc_cmd=%p already in cleanup or abort processing or already completed.\n",
+			 io_req->xid, io_req->sc_cmd);
 		rc = 1;
-		goto out;
+		goto drop_rdata_kref;
 	}
 
 	kref_get(&io_req->refcount);
@@ -1599,18 +1873,17 @@ int qedf_initiate_abts(struct qedf_ioreq *io_req, bool return_scsi_cmd_on_abts)
 	qedf->control_requests++;
 	qedf->packet_aborts++;
 
-	/* Set the return CPU to be the same as the request one */
-	io_req->cpu = smp_processor_id();
-
 	/* Set the command type to abort */
 	io_req->cmd_type = QEDF_ABTS;
 	io_req->return_scsi_cmd_on_abts = return_scsi_cmd_on_abts;
 
 	set_bit(QEDF_CMD_IN_ABORT, &io_req->flags);
-	QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_SCSI_TM, "ABTS io_req xid = "
-		   "0x%x\n", xid);
+	refcount = kref_read(&io_req->refcount);
+	QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_SCSI_TM,
+		  "ABTS io_req xid = 0x%x refcount=%d\n",
+		  xid, refcount);
 
-	qedf_cmd_timer_set(qedf, io_req, QEDF_ABORT_TIMEOUT * HZ);
+	qedf_cmd_timer_set(qedf, io_req, QEDF_ABORT_TIMEOUT);
 
 	spin_lock_irqsave(&fcport->rport_lock, flags);
 
@@ -1624,13 +1897,8 @@ int qedf_initiate_abts(struct qedf_ioreq *io_req, bool return_scsi_cmd_on_abts)
 
 	spin_unlock_irqrestore(&fcport->rport_lock, flags);
 
-	return rc;
-abts_err:
-	/*
-	 * If the ABTS task fails to queue then we need to cleanup the
-	 * task at the firmware.
-	 */
-	qedf_initiate_cleanup(io_req, return_scsi_cmd_on_abts);
+drop_rdata_kref:
+	kref_put(&rdata->kref, fc_rport_destroy);
 out:
 	return rc;
 }
@@ -1640,27 +1908,62 @@ void qedf_process_abts_compl(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
 {
 	uint32_t r_ctl;
 	uint16_t xid;
+	int rc;
+	struct qedf_rport *fcport = io_req->fcport;
 
 	QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_SCSI_TM, "Entered with xid = "
 		   "0x%x cmd_type = %d\n", io_req->xid, io_req->cmd_type);
 
-	cancel_delayed_work(&io_req->timeout_work);
-
 	xid = io_req->xid;
 	r_ctl = cqe->cqe_info.abts_info.r_ctl;
 
+	/* This was added at a point when we were scheduling abts_compl &
+	 * cleanup_compl on different CPUs and there was a possibility of
+	 * the io_req to be freed from the other context before we got here.
+	 */
+	if (!fcport) {
+		QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
+			  "Dropping ABTS completion xid=0x%x as fcport is NULL",
+			  io_req->xid);
+		return;
+	}
+
+	/*
+	 * When flush is active, let the cmds be completed from the cleanup
+	 * context
+	 */
+	if (test_bit(QEDF_RPORT_IN_TARGET_RESET, &fcport->flags) ||
+	    test_bit(QEDF_RPORT_IN_LUN_RESET, &fcport->flags)) {
+		QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
+			  "Dropping ABTS completion xid=0x%x as fcport is flushing",
+			  io_req->xid);
+		return;
+	}
+
+	if (!cancel_delayed_work(&io_req->timeout_work)) {
+		QEDF_ERR(&qedf->dbg_ctx,
+			 "Wasn't able to cancel abts timeout work.\n");
+	}
+
 	switch (r_ctl) {
 	case FC_RCTL_BA_ACC:
 		QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_SCSI_TM,
 		    "ABTS response - ACC Send RRQ after R_A_TOV\n");
 		io_req->event = QEDF_IOREQ_EV_ABORT_SUCCESS;
+		rc = kref_get_unless_zero(&io_req->refcount);	/* ID: 003 */
+		if (!rc) {
+			QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_SCSI_TM,
+				  "kref is already zero so ABTS was already completed or flushed xid=0x%x.\n",
+				  io_req->xid);
+			return;
+		}
 		/*
 		 * Dont release this cmd yet. It will be relesed
 		 * after we get RRQ response
 		 */
-		kref_get(&io_req->refcount);
 		queue_delayed_work(qedf->dpc_wq, &io_req->rrq_work,
 		    msecs_to_jiffies(qedf->lport->r_a_tov));
+		atomic_set(&io_req->state, QEDFC_CMD_ST_RRQ_WAIT);
 		break;
 	/* For error cases let the cleanup return the command */
 	case FC_RCTL_BA_RJT:
@@ -1802,6 +2105,7 @@ int qedf_initiate_cleanup(struct qedf_ioreq *io_req,
 	unsigned long flags;
 	struct fcoe_wqe *sqe;
 	u16 sqe_idx;
+	int refcount = 0;
 
 	fcport = io_req->fcport;
 	if (!fcport) {
@@ -1823,36 +2127,45 @@ int qedf_initiate_cleanup(struct qedf_ioreq *io_req,
 	}
 
 	if (!test_bit(QEDF_CMD_OUTSTANDING, &io_req->flags) ||
-	    test_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags)) {
+	    test_and_set_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags)) {
 		QEDF_ERR(&(qedf->dbg_ctx), "io_req xid=0x%x already in "
 			  "cleanup processing or already completed.\n",
 			  io_req->xid);
 		return SUCCESS;
 	}
+	set_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags);
 
 	/* Ensure room on SQ */
 	if (!atomic_read(&fcport->free_sqes)) {
 		QEDF_ERR(&(qedf->dbg_ctx), "No SQ entries available\n");
+		/* Need to make sure we clear the flag since it was set */
+		clear_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags);
 		return FAILED;
 	}
 
+	if (io_req->cmd_type == QEDF_CLEANUP) {
+		QEDF_ERR(&qedf->dbg_ctx,
+			 "io_req=0x%x is already a cleanup command cmd_type=%d.\n",
+			 io_req->xid, io_req->cmd_type);
+		clear_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags);
+		return SUCCESS;
+	}
 
-	QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_IO, "Entered xid=0x%x\n",
-	    io_req->xid);
+	refcount = kref_read(&io_req->refcount);
+
+	QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_IO,
+		  "Entered xid=0x%x sc_cmd=%p cmd_type=%d flags=0x%lx refcount=%d fcport=%p port_id=0x%06x\n",
+		  io_req->xid, io_req->sc_cmd, io_req->cmd_type, io_req->flags,
+		  refcount, fcport, fcport->rdata->ids.port_id);
 
 	/* Cleanup cmds re-use the same TID as the original I/O */
 	xid = io_req->xid;
 	io_req->cmd_type = QEDF_CLEANUP;
 	io_req->return_scsi_cmd_on_abts = return_scsi_cmd_on_abts;
 
-	/* Set the return CPU to be the same as the request one */
-	io_req->cpu = smp_processor_id();
-
-	set_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags);
-
 	task = qedf_get_task_mem(&qedf->tasks, xid);
 
-	init_completion(&io_req->tm_done);
+	init_completion(&io_req->cleanup_done);
 
 	spin_lock_irqsave(&fcport->rport_lock, flags);
 
@@ -1866,8 +2179,8 @@ int qedf_initiate_cleanup(struct qedf_ioreq *io_req,
 
 	spin_unlock_irqrestore(&fcport->rport_lock, flags);
 
-	tmo = wait_for_completion_timeout(&io_req->tm_done,
-	    QEDF_CLEANUP_TIMEOUT * HZ);
+	tmo = wait_for_completion_timeout(&io_req->cleanup_done,
+					  QEDF_CLEANUP_TIMEOUT * HZ);
 
 	if (!tmo) {
 		rc = FAILED;
@@ -1880,6 +2193,16 @@ int qedf_initiate_cleanup(struct qedf_ioreq *io_req,
 		qedf_drain_request(qedf);
 	}
 
+	/* If it TASK MGMT handle it, reference will be decreased
+	 * in qedf_execute_tmf
+	 */
+	if (io_req->tm_flags  == FCP_TMF_LUN_RESET ||
+	    io_req->tm_flags == FCP_TMF_TGT_RESET) {
+		clear_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
+		io_req->sc_cmd = NULL;
+		complete(&io_req->tm_done);
+	}
+
 	if (io_req->sc_cmd) {
 		if (io_req->return_scsi_cmd_on_abts)
 			qedf_scsi_done(qedf, io_req, DID_ERROR);
@@ -1902,7 +2225,7 @@ void qedf_process_cleanup_compl(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
 	clear_bit(QEDF_CMD_IN_CLEANUP, &io_req->flags);
 
 	/* Complete so we can finish cleaning up the I/O */
-	complete(&io_req->tm_done);
+	complete(&io_req->cleanup_done);
 }
 
 static int qedf_execute_tmf(struct qedf_rport *fcport, struct scsi_cmnd *sc_cmd,
@@ -1915,6 +2238,7 @@ static int qedf_execute_tmf(struct qedf_rport *fcport, struct scsi_cmnd *sc_cmd,
 	int rc = 0;
 	uint16_t xid;
 	int tmo = 0;
+	int lun = 0;
 	unsigned long flags;
 	struct fcoe_wqe *sqe;
 	u16 sqe_idx;
@@ -1924,20 +2248,18 @@ static int qedf_execute_tmf(struct qedf_rport *fcport, struct scsi_cmnd *sc_cmd,
 		return FAILED;
 	}
 
+	lun = (int)sc_cmd->device->lun;
 	if (!test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags)) {
 		QEDF_ERR(&(qedf->dbg_ctx), "fcport not offloaded\n");
 		rc = FAILED;
-		return FAILED;
+		goto no_flush;
 	}
 
-	QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_SCSI_TM, "portid = 0x%x "
-		   "tm_flags = %d\n", fcport->rdata->ids.port_id, tm_flags);
-
 	io_req = qedf_alloc_cmd(fcport, QEDF_TASK_MGMT_CMD);
 	if (!io_req) {
 		QEDF_ERR(&(qedf->dbg_ctx), "Failed TMF");
 		rc = -EAGAIN;
-		goto reset_tmf_err;
+		goto no_flush;
 	}
 
 	if (tm_flags == FCP_TMF_LUN_RESET)
@@ -1950,7 +2272,7 @@ static int qedf_execute_tmf(struct qedf_rport *fcport, struct scsi_cmnd *sc_cmd,
 	io_req->fcport = fcport;
 	io_req->cmd_type = QEDF_TASK_MGMT_CMD;
 
-	/* Set the return CPU to be the same as the request one */
+	/* Record which cpu this request is associated with */
 	io_req->cpu = smp_processor_id();
 
 	/* Set TM flags */
@@ -1959,7 +2281,7 @@ static int qedf_execute_tmf(struct qedf_rport *fcport, struct scsi_cmnd *sc_cmd,
 	io_req->tm_flags = tm_flags;
 
 	/* Default is to return a SCSI command when an error occurs */
-	io_req->return_scsi_cmd_on_abts = true;
+	io_req->return_scsi_cmd_on_abts = false;
 
 	/* Obtain exchange id */
 	xid = io_req->xid;
@@ -1983,12 +2305,16 @@ static int qedf_execute_tmf(struct qedf_rport *fcport, struct scsi_cmnd *sc_cmd,
 
 	spin_unlock_irqrestore(&fcport->rport_lock, flags);
 
+	set_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
 	tmo = wait_for_completion_timeout(&io_req->tm_done,
 	    QEDF_TM_TIMEOUT * HZ);
 
 	if (!tmo) {
 		rc = FAILED;
 		QEDF_ERR(&(qedf->dbg_ctx), "wait for tm_cmpl timeout!\n");
+		/* Clear outstanding bit since command timed out */
+		clear_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
+		io_req->sc_cmd = NULL;
 	} else {
 		/* Check TMF response code */
 		if (io_req->fcp_rsp_code == 0)
@@ -1996,14 +2322,25 @@ static int qedf_execute_tmf(struct qedf_rport *fcport, struct scsi_cmnd *sc_cmd,
 		else
 			rc = FAILED;
 	}
+	/*
+	 * Double check that fcport has not gone into an uploading state before
+	 * executing the command flush for the LUN/target.
+	 */
+	if (test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags)) {
+		QEDF_ERR(&qedf->dbg_ctx,
+			 "fcport is uploading, not executing flush.\n");
+		goto no_flush;
+	}
+	/* We do not need this io_req any more */
+	kref_put(&io_req->refcount, qedf_release_cmd);
+
 
 	if (tm_flags == FCP_TMF_LUN_RESET)
-		qedf_flush_active_ios(fcport, (int)sc_cmd->device->lun);
+		qedf_flush_active_ios(fcport, lun);
 	else
 		qedf_flush_active_ios(fcport, -1);
 
-	kref_put(&io_req->refcount, qedf_release_cmd);
-
+no_flush:
 	if (rc != SUCCESS) {
 		QEDF_ERR(&(qedf->dbg_ctx), "task mgmt command failed...\n");
 		rc = FAILED;
@@ -2011,7 +2348,6 @@ static int qedf_execute_tmf(struct qedf_rport *fcport, struct scsi_cmnd *sc_cmd,
 		QEDF_ERR(&(qedf->dbg_ctx), "task mgmt command success...\n");
 		rc = SUCCESS;
 	}
-reset_tmf_err:
 	return rc;
 }
 
@@ -2021,26 +2357,65 @@ int qedf_initiate_tmf(struct scsi_cmnd *sc_cmd, u8 tm_flags)
 	struct fc_rport_libfc_priv *rp = rport->dd_data;
 	struct qedf_rport *fcport = (struct qedf_rport *)&rp[1];
 	struct qedf_ctx *qedf;
-	struct fc_lport *lport;
+	struct fc_lport *lport = shost_priv(sc_cmd->device->host);
 	int rc = SUCCESS;
 	int rval;
+	struct qedf_ioreq *io_req = NULL;
+	int ref_cnt = 0;
+	struct fc_rport_priv *rdata = fcport->rdata;
 
-	rval = fc_remote_port_chkready(rport);
+	QEDF_ERR(NULL,
+		 "tm_flags 0x%x sc_cmd %p op = 0x%02x target_id = 0x%x lun=%d\n",
+		 tm_flags, sc_cmd, sc_cmd->cmnd[0], rport->scsi_target_id,
+		 (int)sc_cmd->device->lun);
+
+	if (!rdata || !kref_get_unless_zero(&rdata->kref)) {
+		QEDF_ERR(NULL, "stale rport\n");
+		return FAILED;
+	}
+
+	QEDF_ERR(NULL, "portid=%06x tm_flags =%s\n", rdata->ids.port_id,
+		 (tm_flags == FCP_TMF_TGT_RESET) ? "TARGET RESET" :
+		 "LUN RESET");
+
+	if (sc_cmd->SCp.ptr) {
+		io_req = (struct qedf_ioreq *)sc_cmd->SCp.ptr;
+		ref_cnt = kref_read(&io_req->refcount);
+		QEDF_ERR(NULL,
+			 "orig io_req = %p xid = 0x%x ref_cnt = %d.\n",
+			 io_req, io_req->xid, ref_cnt);
+	}
 
+	rval = fc_remote_port_chkready(rport);
 	if (rval) {
 		QEDF_ERR(NULL, "device_reset rport not ready\n");
 		rc = FAILED;
 		goto tmf_err;
 	}
 
-	if (fcport == NULL) {
+	rc = fc_block_scsi_eh(sc_cmd);
+	if (rc)
+		goto tmf_err;
+
+	if (!fcport) {
 		QEDF_ERR(NULL, "device_reset: rport is NULL\n");
 		rc = FAILED;
 		goto tmf_err;
 	}
 
 	qedf = fcport->qedf;
-	lport = qedf->lport;
+
+	if (!qedf) {
+		QEDF_ERR(NULL, "qedf is NULL.\n");
+		rc = FAILED;
+		goto tmf_err;
+	}
+
+	if (test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags)) {
+		QEDF_ERR(&qedf->dbg_ctx, "Connection is getting uploaded.\n");
+		rc = SUCCESS;
+		goto tmf_err;
+	}
 
 	if (test_bit(QEDF_UNLOADING, &qedf->flags) ||
 	    test_bit(QEDF_DBG_STOP_IO, &qedf->flags)) {
@@ -2054,9 +2429,22 @@ int qedf_initiate_tmf(struct scsi_cmnd *sc_cmd, u8 tm_flags)
 		goto tmf_err;
 	}
 
+	if (test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags)) {
+		if (!fcport->rdata)
+			QEDF_ERR(&qedf->dbg_ctx, "fcport %p is uploading.\n",
+				 fcport);
+		else
+			QEDF_ERR(&qedf->dbg_ctx,
+				 "fcport %p port_id=%06x is uploading.\n",
+				 fcport, fcport->rdata->ids.port_id);
+		rc = FAILED;
+		goto tmf_err;
+	}
+
 	rc = qedf_execute_tmf(fcport, sc_cmd, tm_flags);
 
 tmf_err:
+	kref_put(&rdata->kref, fc_rport_destroy);
 	return rc;
 }
 
@@ -2065,6 +2453,8 @@ void qedf_process_tmf_compl(struct qedf_ctx *qedf, struct fcoe_cqe *cqe,
 {
 	struct fcoe_cqe_rsp_info *fcp_rsp;
 
+	clear_bit(QEDF_CMD_OUTSTANDING, &io_req->flags);
+
 	fcp_rsp = &cqe->cqe_info.rsp_info;
 	qedf_parse_fcp_rsp(io_req, fcp_rsp);
 
diff --git a/drivers/scsi/qedf/qedf_main.c b/drivers/scsi/qedf/qedf_main.c
index 9f9431a4cc0e..5b07235497c6 100644
--- a/drivers/scsi/qedf/qedf_main.c
+++ b/drivers/scsi/qedf/qedf_main.c
@@ -124,21 +124,24 @@ static bool qedf_initiate_fipvlan_req(struct qedf_ctx *qedf)
 {
 	int rc;
 
-	if (atomic_read(&qedf->link_state) != QEDF_LINK_UP) {
-		QEDF_ERR(&(qedf->dbg_ctx), "Link not up.\n");
-		return  false;
-	}
-
 	while (qedf->fipvlan_retries--) {
+		/* This is to catch if link goes down during fipvlan retries */
+		if (atomic_read(&qedf->link_state) == QEDF_LINK_DOWN) {
+			QEDF_ERR(&qedf->dbg_ctx, "Link not up.\n");
+			return false;
+		}
+
 		if (qedf->vlan_id > 0)
 			return true;
+
 		QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,
 			   "Retry %d.\n", qedf->fipvlan_retries);
 		init_completion(&qedf->fipvlan_compl);
 		qedf_fcoe_send_vlan_req(qedf);
 		rc = wait_for_completion_timeout(&qedf->fipvlan_compl,
 		    1 * HZ);
-		if (rc > 0) {
+		if (rc > 0 &&
+		    (atomic_read(&qedf->link_state) == QEDF_LINK_UP)) {
 			fcoe_ctlr_link_up(&qedf->ctlr);
 			return true;
 		}
@@ -153,12 +156,19 @@ static void qedf_handle_link_update(struct work_struct *work)
 	    container_of(work, struct qedf_ctx, link_update.work);
 	int rc;
 
-	QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC, "Entered.\n");
+	QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_DISC, "Entered. link_state=%d.\n",
+		  atomic_read(&qedf->link_state));
 
 	if (atomic_read(&qedf->link_state) == QEDF_LINK_UP) {
 		rc = qedf_initiate_fipvlan_req(qedf);
 		if (rc)
 			return;
+
+		if (atomic_read(&qedf->link_state) != QEDF_LINK_UP) {
+			qedf->vlan_id = 0;
+			return;
+		}
+
 		/*
 		 * If we get here then we never received a repsonse to our
 		 * fip vlan request so set the vlan_id to the default and
@@ -185,7 +195,9 @@ static void qedf_handle_link_update(struct work_struct *work)
 		QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,
 		    "Calling fcoe_ctlr_link_down().\n");
 		fcoe_ctlr_link_down(&qedf->ctlr);
-		qedf_wait_for_upload(qedf);
+		if (qedf_wait_for_upload(qedf) == false)
+			QEDF_ERR(&qedf->dbg_ctx,
+				 "Could not upload all sessions.\n");
 		/* Reset the number of FIP VLAN retries */
 		qedf->fipvlan_retries = qedf_fipvlan_retries;
 	}
@@ -615,50 +627,113 @@ static struct scsi_transport_template *qedf_fc_vport_transport_template;
 static int qedf_eh_abort(struct scsi_cmnd *sc_cmd)
 {
 	struct fc_rport *rport = starget_to_rport(scsi_target(sc_cmd->device));
-	struct fc_rport_libfc_priv *rp = rport->dd_data;
-	struct qedf_rport *fcport;
 	struct fc_lport *lport;
 	struct qedf_ctx *qedf;
 	struct qedf_ioreq *io_req;
+	struct fc_rport_libfc_priv *rp = rport->dd_data;
+	struct fc_rport_priv *rdata;
+	struct qedf_rport *fcport = NULL;
 	int rc = FAILED;
+	int wait_count = 100;
+	int refcount = 0;
 	int rval;
-
-	if (fc_remote_port_chkready(rport)) {
-		QEDF_ERR(NULL, "rport not ready\n");
-		goto out;
-	}
+	int got_ref = 0;
 
 	lport = shost_priv(sc_cmd->device->host);
 	qedf = (struct qedf_ctx *)lport_priv(lport);
 
-	if ((lport->state != LPORT_ST_READY) || !(lport->link_up)) {
-		QEDF_ERR(&(qedf->dbg_ctx), "link not ready.\n");
+	/* rport and tgt are allocated together, so tgt should be non-NULL */
+	fcport = (struct qedf_rport *)&rp[1];
+	rdata = fcport->rdata;
+	if (!rdata || !kref_get_unless_zero(&rdata->kref)) {
+		QEDF_ERR(&qedf->dbg_ctx, "stale rport, sc_cmd=%p\n", sc_cmd);
+		rc = 1;
 		goto out;
 	}
 
-	fcport = (struct qedf_rport *)&rp[1];
 
 	io_req = (struct qedf_ioreq *)sc_cmd->SCp.ptr;
 	if (!io_req) {
-		QEDF_ERR(&(qedf->dbg_ctx), "io_req is NULL.\n");
+		QEDF_ERR(&qedf->dbg_ctx,
+			 "sc_cmd not queued with lld, sc_cmd=%p op=0x%02x, port_id=%06x\n",
+			 sc_cmd, sc_cmd->cmnd[0],
+			 rdata->ids.port_id);
 		rc = SUCCESS;
-		goto out;
+		goto drop_rdata_kref;
+	}
+
+	rval = kref_get_unless_zero(&io_req->refcount);	/* ID: 005 */
+	if (rval)
+		got_ref = 1;
+
+	/* If we got a valid io_req, confirm it belongs to this sc_cmd. */
+	if (!rval || io_req->sc_cmd != sc_cmd) {
+		QEDF_ERR(&qedf->dbg_ctx,
+			 "Freed/Incorrect io_req, io_req->sc_cmd=%p, sc_cmd=%p, port_id=%06x, bailing out.\n",
+			 io_req->sc_cmd, sc_cmd, rdata->ids.port_id);
+
+		goto drop_rdata_kref;
+	}
+
+	if (fc_remote_port_chkready(rport)) {
+		refcount = kref_read(&io_req->refcount);
+		QEDF_ERR(&qedf->dbg_ctx,
+			 "rport not ready, io_req=%p, xid=0x%x sc_cmd=%p op=0x%02x, refcount=%d, port_id=%06x\n",
+			 io_req, io_req->xid, sc_cmd, sc_cmd->cmnd[0],
+			 refcount, rdata->ids.port_id);
+
+		goto drop_rdata_kref;
+	}
+
+	rc = fc_block_scsi_eh(sc_cmd);
+	if (rc)
+		goto drop_rdata_kref;
+
+	if (test_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags)) {
+		QEDF_ERR(&qedf->dbg_ctx,
+			 "Connection uploading, xid=0x%x., port_id=%06x\n",
+			 io_req->xid, rdata->ids.port_id);
+		while (io_req->sc_cmd && (wait_count != 0)) {
+			msleep(100);
+			wait_count--;
+		}
+		if (wait_count) {
+			QEDF_ERR(&qedf->dbg_ctx, "ABTS succeeded\n");
+			rc = SUCCESS;
+		} else {
+			QEDF_ERR(&qedf->dbg_ctx, "ABTS failed\n");
+			rc = FAILED;
+		}
+		goto drop_rdata_kref;
 	}
 
-	QEDF_ERR(&(qedf->dbg_ctx), "Aborting io_req sc_cmd=%p xid=0x%x "
-		  "fp_idx=%d.\n", sc_cmd, io_req->xid, io_req->fp_idx);
+	if (lport->state != LPORT_ST_READY || !(lport->link_up)) {
+		QEDF_ERR(&qedf->dbg_ctx, "link not ready.\n");
+		goto drop_rdata_kref;
+	}
+
+	QEDF_ERR(&qedf->dbg_ctx,
+		 "Aborting io_req=%p sc_cmd=%p xid=0x%x fp_idx=%d, port_id=%06x.\n",
+		 io_req, sc_cmd, io_req->xid, io_req->fp_idx,
+		 rdata->ids.port_id);
 
 	if (qedf->stop_io_on_error) {
 		qedf_stop_all_io(qedf);
 		rc = SUCCESS;
-		goto out;
+		goto drop_rdata_kref;
 	}
 
 	init_completion(&io_req->abts_done);
 	rval = qedf_initiate_abts(io_req, true);
 	if (rval) {
 		QEDF_ERR(&(qedf->dbg_ctx), "Failed to queue ABTS.\n");
-		goto out;
+		/*
+		 * If we fail to queue the ABTS then return this command to
+		 * the SCSI layer as it will own and free the xid
+		 */
+		rc = SUCCESS;
+		qedf_scsi_done(qedf, io_req, DID_ERROR);
+		goto drop_rdata_kref;
 	}
 
 	wait_for_completion(&io_req->abts_done);
@@ -684,38 +759,68 @@ static int qedf_eh_abort(struct scsi_cmnd *sc_cmd)
 		QEDF_ERR(&(qedf->dbg_ctx), "ABTS failed, xid=0x%x.\n",
 			  io_req->xid);
 
+drop_rdata_kref:
+	kref_put(&rdata->kref, fc_rport_destroy);
 out:
+	if (got_ref)
+		kref_put(&io_req->refcount, qedf_release_cmd);
 	return rc;
 }
 
 static int qedf_eh_target_reset(struct scsi_cmnd *sc_cmd)
 {
-	QEDF_ERR(NULL, "TARGET RESET Issued...");
+	QEDF_ERR(NULL, "%d:0:%d:%lld: TARGET RESET Issued...",
+		 sc_cmd->device->host->host_no, sc_cmd->device->id,
+		 sc_cmd->device->lun);
 	return qedf_initiate_tmf(sc_cmd, FCP_TMF_TGT_RESET);
 }
 
 static int qedf_eh_device_reset(struct scsi_cmnd *sc_cmd)
 {
-	QEDF_ERR(NULL, "LUN RESET Issued...\n");
+	QEDF_ERR(NULL, "%d:0:%d:%lld: LUN RESET Issued... ",
+		 sc_cmd->device->host->host_no, sc_cmd->device->id,
+		 sc_cmd->device->lun);
 	return qedf_initiate_tmf(sc_cmd, FCP_TMF_LUN_RESET);
 }
 
-void qedf_wait_for_upload(struct qedf_ctx *qedf)
+bool qedf_wait_for_upload(struct qedf_ctx *qedf)
 {
-	while (1) {
+	struct qedf_rport *fcport = NULL;
+	int wait_cnt = 120;
+
+	while (wait_cnt--) {
 		if (atomic_read(&qedf->num_offloads))
-			QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_DISC,
-			    "Waiting for all uploads to complete.\n");
+			QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_DISC,
+				  "Waiting for all uploads to complete num_offloads = 0x%x.\n",
+				  atomic_read(&qedf->num_offloads));
 		else
-			break;
+			return true;
 		msleep(500);
 	}
+
+	rcu_read_lock();
+	list_for_each_entry_rcu(fcport, &qedf->fcports, peers) {
+		if (fcport && test_bit(QEDF_RPORT_SESSION_READY,
+				       &fcport->flags)) {
+			if (fcport->rdata)
+				QEDF_ERR(&qedf->dbg_ctx,
+					 "Waiting for fcport %p portid=%06x.\n",
+					 fcport, fcport->rdata->ids.port_id);
+			} else {
+				QEDF_ERR(&qedf->dbg_ctx,
+					 "Waiting for fcport %p.\n", fcport);
+			}
+	}
+	rcu_read_unlock();
+	return false;
+
 }
 
 /* Performs soft reset of qedf_ctx by simulating a link down/up */
-static void qedf_ctx_soft_reset(struct fc_lport *lport)
+void qedf_ctx_soft_reset(struct fc_lport *lport)
 {
 	struct qedf_ctx *qedf;
+	struct qed_link_output if_link;
 
 	if (lport->vport) {
 		QEDF_ERR(NULL, "Cannot issue host reset on NPIV port.\n");
@@ -726,11 +831,32 @@ static void qedf_ctx_soft_reset(struct fc_lport *lport)
 
 	/* For host reset, essentially do a soft link up/down */
 	atomic_set(&qedf->link_state, QEDF_LINK_DOWN);
+	QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_DISC,
+		  "Queuing link down work.\n");
 	queue_delayed_work(qedf->link_update_wq, &qedf->link_update,
 	    0);
-	qedf_wait_for_upload(qedf);
+
+	if (qedf_wait_for_upload(qedf) == false) {
+		QEDF_ERR(&qedf->dbg_ctx, "Could not upload all sessions.\n");
+		WARN_ON(atomic_read(&qedf->num_offloads));
+	}
+
+	/* Before setting link up query physical link state */
+	qed_ops->common->get_link(qedf->cdev, &if_link);
+	/* Bail if the physical link is not up */
+	if (!if_link.link_up) {
+		QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_DISC,
+			  "Physical link is not up.\n");
+		return;
+	}
+	/* Flush and wait to make sure link down is processed */
+	flush_delayed_work(&qedf->link_update);
+	msleep(500);
+
 	atomic_set(&qedf->link_state, QEDF_LINK_UP);
 	qedf->vlan_id  = 0;
+	QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_DISC,
+		  "Queue link up work.\n");
 	queue_delayed_work(qedf->link_update_wq, &qedf->link_update,
 	    0);
 }
@@ -740,22 +866,6 @@ static int qedf_eh_host_reset(struct scsi_cmnd *sc_cmd)
 {
 	struct fc_lport *lport;
 	struct qedf_ctx *qedf;
-	struct fc_rport *rport = starget_to_rport(scsi_target(sc_cmd->device));
-	struct fc_rport_libfc_priv *rp = rport->dd_data;
-	struct qedf_rport *fcport = (struct qedf_rport *)&rp[1];
-	int rval;
-
-	rval = fc_remote_port_chkready(rport);
-
-	if (rval) {
-		QEDF_ERR(NULL, "device_reset rport not ready\n");
-		return FAILED;
-	}
-
-	if (fcport == NULL) {
-		QEDF_ERR(NULL, "device_reset: rport is NULL\n");
-		return FAILED;
-	}
 
 	lport = shost_priv(sc_cmd->device->host);
 	qedf = lport_priv(lport);
@@ -907,8 +1017,10 @@ static int qedf_xmit(struct fc_lport *lport, struct fc_frame *fp)
 		    "Dropping FCoE frame to %06x.\n", ntoh24(fh->fh_d_id));
 		kfree_skb(skb);
 		rdata = fc_rport_lookup(lport, ntoh24(fh->fh_d_id));
-		if (rdata)
+		if (rdata) {
 			rdata->retries = lport->max_rport_retry_count;
+			kref_put(&rdata->kref, fc_rport_destroy);
+		}
 		return -EINVAL;
 	}
 	/* End NPIV filtering */
@@ -1031,7 +1143,12 @@ static int qedf_xmit(struct fc_lport *lport, struct fc_frame *fp)
 	if (qedf_dump_frames)
 		print_hex_dump(KERN_WARNING, "fcoe: ", DUMP_PREFIX_OFFSET, 16,
 		    1, skb->data, skb->len, false);
-	qed_ops->ll2->start_xmit(qedf->cdev, skb, 0);
+	rc = qed_ops->ll2->start_xmit(qedf->cdev, skb, 0);
+	if (rc) {
+		QEDF_ERR(&qedf->dbg_ctx, "start_xmit failed rc = %d.\n", rc);
+		kfree_skb(skb);
+		return rc;
+	}
 
 	return 0;
 }
@@ -1224,6 +1341,8 @@ static void qedf_upload_connection(struct qedf_ctx *qedf,
 static void qedf_cleanup_fcport(struct qedf_ctx *qedf,
 	struct qedf_rport *fcport)
 {
+	struct fc_rport_priv *rdata = fcport->rdata;
+
 	QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_CONN, "Cleaning up portid=%06x.\n",
 	    fcport->rdata->ids.port_id);
 
@@ -1235,6 +1354,7 @@ static void qedf_cleanup_fcport(struct qedf_ctx *qedf,
 	qedf_free_sq(qedf, fcport);
 	fcport->rdata = NULL;
 	fcport->qedf = NULL;
+	kref_put(&rdata->kref, fc_rport_destroy);
 }
 
 /**
@@ -1310,6 +1430,8 @@ static void qedf_rport_event_handler(struct fc_lport *lport,
 			break;
 		}
 
+		/* Initial reference held on entry, so this can't fail */
+		kref_get(&rdata->kref);
 		fcport->rdata = rdata;
 		fcport->rport = rport;
 
@@ -1369,11 +1491,15 @@ static void qedf_rport_event_handler(struct fc_lport *lport,
 		 */
 		fcport = (struct qedf_rport *)&rp[1];
 
+		spin_lock_irqsave(&fcport->rport_lock, flags);
 		/* Only free this fcport if it is offloaded already */
-		if (test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags)) {
-			set_bit(QEDF_RPORT_UPLOADING_CONNECTION, &fcport->flags);
+		if (test_bit(QEDF_RPORT_SESSION_READY, &fcport->flags) &&
+		    !test_bit(QEDF_RPORT_UPLOADING_CONNECTION,
+		    &fcport->flags)) {
+			set_bit(QEDF_RPORT_UPLOADING_CONNECTION,
+				&fcport->flags);
+			spin_unlock_irqrestore(&fcport->rport_lock, flags);
 			qedf_cleanup_fcport(qedf, fcport);
-
 			/*
 			 * Remove fcport to list of qedf_ctx list of offloaded
 			 * ports
@@ -1385,8 +1511,9 @@ static void qedf_rport_event_handler(struct fc_lport *lport,
 			clear_bit(QEDF_RPORT_UPLOADING_CONNECTION,
 			    &fcport->flags);
 			atomic_dec(&qedf->num_offloads);
+		} else {
+			spin_unlock_irqrestore(&fcport->rport_lock, flags);
 		}
-
 		break;
 
 	case RPORT_EV_NONE:
@@ -1498,11 +1625,15 @@ static int qedf_lport_setup(struct qedf_ctx *qedf)
 	fc_set_wwnn(lport, qedf->wwnn);
 	fc_set_wwpn(lport, qedf->wwpn);
 
-	fcoe_libfc_config(lport, &qedf->ctlr, &qedf_lport_template, 0);
+	if (fcoe_libfc_config(lport, &qedf->ctlr, &qedf_lport_template, 0)) {
+		QEDF_ERR(&qedf->dbg_ctx,
+			 "fcoe_libfc_config failed.\n");
+		return -ENOMEM;
+	}
 
 	/* Allocate the exchange manager */
-	fc_exch_mgr_alloc(lport, FC_CLASS_3, qedf->max_scsi_xid + 1,
-	    qedf->max_els_xid, NULL);
+	fc_exch_mgr_alloc(lport, FC_CLASS_3, FCOE_PARAMS_NUM_TASKS,
+			  0xfffe, NULL);
 
 	if (fc_lport_init_stats(lport))
 		return -ENOMEM;
@@ -1625,14 +1756,15 @@ static int qedf_vport_create(struct fc_vport *vport, bool disabled)
 	vport_qedf->wwpn = vn_port->wwpn;
 
 	vn_port->host->transportt = qedf_fc_vport_transport_template;
-	vn_port->host->can_queue = QEDF_MAX_ELS_XID;
+	vn_port->host->can_queue = FCOE_PARAMS_NUM_TASKS;
 	vn_port->host->max_lun = qedf_max_lun;
 	vn_port->host->sg_tablesize = QEDF_MAX_BDS_PER_CMD;
 	vn_port->host->max_cmd_len = QEDF_MAX_CDB_LEN;
 
 	rc = scsi_add_host(vn_port->host, &vport->dev);
 	if (rc) {
-		QEDF_WARN(&(base_qedf->dbg_ctx), "Error adding Scsi_Host.\n");
+		QEDF_WARN(&base_qedf->dbg_ctx,
+			  "Error adding Scsi_Host rc=0x%x.\n", rc);
 		goto err2;
 	}
 
@@ -2155,7 +2287,8 @@ static int qedf_setup_int(struct qedf_ctx *qedf)
 	    QEDF_SIMD_HANDLER_NUM, qedf_simd_int_handler);
 	qedf->int_info.used_cnt = 1;
 
-	QEDF_ERR(&qedf->dbg_ctx, "Only MSI-X supported. Failing probe.\n");
+	QEDF_ERR(&qedf->dbg_ctx,
+		 "Cannot load driver due to a lack of MSI-X vectors.\n");
 	return -EINVAL;
 }
 
@@ -2356,6 +2489,13 @@ static int qedf_ll2_rx(void *cookie, struct sk_buff *skb,
 	struct qedf_ctx *qedf = (struct qedf_ctx *)cookie;
 	struct qedf_skb_work *skb_work;
 
+	if (atomic_read(&qedf->link_state) == QEDF_LINK_DOWN) {
+		QEDF_INFO(&qedf->dbg_ctx, QEDF_LOG_LL2,
+			  "Dropping frame as link state is down.\n");
+		kfree_skb(skb);
+		return 0;
+	}
+
 	skb_work = kzalloc(sizeof(struct qedf_skb_work), GFP_ATOMIC);
 	if (!skb_work) {
 		QEDF_WARN(&(qedf->dbg_ctx), "Could not allocate skb_work so "
@@ -2990,6 +3130,8 @@ static int __qedf_probe(struct pci_dev *pdev, int mode)
 			goto err0;
 		}
 
+		fc_disc_init(lport);
+
 		/* Initialize qedf_ctx */
 		qedf = lport_priv(lport);
 		qedf->lport = lport;
@@ -3005,6 +3147,7 @@ static int __qedf_probe(struct pci_dev *pdev, int mode)
 		pci_set_drvdata(pdev, qedf);
 		init_completion(&qedf->fipvlan_compl);
 		mutex_init(&qedf->stats_mutex);
+		mutex_init(&qedf->flush_mutex);
 
 		QEDF_INFO(&(qedf->dbg_ctx), QEDF_LOG_INFO,
 		   "QLogic FastLinQ FCoE Module qedf %s, "
@@ -3181,11 +3324,6 @@ static int __qedf_probe(struct pci_dev *pdev, int mode)
 	sprintf(host_buf, "host_%d", host->host_no);
 	qed_ops->common->set_name(qedf->cdev, host_buf);
 
-
-	/* Set xid max values */
-	qedf->max_scsi_xid = QEDF_MAX_SCSI_XID;
-	qedf->max_els_xid = QEDF_MAX_ELS_XID;
-
 	/* Allocate cmd mgr */
 	qedf->cmd_mgr = qedf_cmd_mgr_alloc(qedf);
 	if (!qedf->cmd_mgr) {
@@ -3196,12 +3334,15 @@ static int __qedf_probe(struct pci_dev *pdev, int mode)
 
 	if (mode != QEDF_MODE_RECOVERY) {
 		host->transportt = qedf_fc_transport_template;
-		host->can_queue = QEDF_MAX_ELS_XID;
 		host->max_lun = qedf_max_lun;
 		host->max_cmd_len = QEDF_MAX_CDB_LEN;
+		host->can_queue = FCOE_PARAMS_NUM_TASKS;
 		rc = scsi_add_host(host, &pdev->dev);
-		if (rc)
+		if (rc) {
+			QEDF_WARN(&qedf->dbg_ctx,
+				  "Error adding Scsi_Host rc=0x%x.\n", rc);
 			goto err6;
+		}
 	}
 
 	memset(&params, 0, sizeof(params));
@@ -3377,7 +3518,9 @@ static void __qedf_remove(struct pci_dev *pdev, int mode)
 		fcoe_ctlr_link_down(&qedf->ctlr);
 	else
 		fc_fabric_logoff(qedf->lport);
-	qedf_wait_for_upload(qedf);
+
+	if (qedf_wait_for_upload(qedf) == false)
+		QEDF_ERR(&qedf->dbg_ctx, "Could not upload all sessions.\n");
 
 #ifdef CONFIG_DEBUG_FS
 	qedf_dbg_host_exit(&(qedf->dbg_ctx));
diff --git a/drivers/scsi/qedf/qedf_version.h b/drivers/scsi/qedf/qedf_version.h
index 9455faacd5de..334a9cdf346a 100644
--- a/drivers/scsi/qedf/qedf_version.h
+++ b/drivers/scsi/qedf/qedf_version.h
@@ -7,9 +7,9 @@
  *  this source tree.
  */
 
-#define QEDF_VERSION		"8.33.16.20"
+#define QEDF_VERSION		"8.37.25.20"
 #define QEDF_DRIVER_MAJOR_VER		8
-#define QEDF_DRIVER_MINOR_VER		33
-#define QEDF_DRIVER_REV_VER		16
+#define QEDF_DRIVER_MINOR_VER		37
+#define QEDF_DRIVER_REV_VER		25
 #define QEDF_DRIVER_ENG_VER		20
 
diff --git a/drivers/scsi/qedi/qedi_fw.c b/drivers/scsi/qedi/qedi_fw.c
index f8f86774f77f..bd81bbee61e6 100644
--- a/drivers/scsi/qedi/qedi_fw.c
+++ b/drivers/scsi/qedi/qedi_fw.c
@@ -155,12 +155,10 @@ static void qedi_tmf_resp_work(struct work_struct *work)
 	struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data;
 	struct iscsi_session *session = conn->session;
 	struct iscsi_tm_rsp *resp_hdr_ptr;
-	struct iscsi_cls_session *cls_sess;
 	int rval = 0;
 
 	set_bit(QEDI_CONN_FW_CLEANUP, &qedi_conn->flags);
 	resp_hdr_ptr =  (struct iscsi_tm_rsp *)qedi_cmd->tmf_resp_buf;
-	cls_sess = iscsi_conn_to_session(qedi_conn->cls_conn);
 
 	iscsi_block_session(session->cls_session);
 	rval = qedi_cleanup_all_io(qedi, qedi_conn, qedi_cmd->task, true);
@@ -1366,7 +1364,6 @@ static void qedi_tmf_work(struct work_struct *work)
 	struct qedi_conn *qedi_conn = qedi_cmd->conn;
 	struct qedi_ctx *qedi = qedi_conn->qedi;
 	struct iscsi_conn *conn = qedi_conn->cls_conn->dd_data;
-	struct iscsi_cls_session *cls_sess;
 	struct qedi_work_map *list_work = NULL;
 	struct iscsi_task *mtask;
 	struct qedi_cmd *cmd;
@@ -1377,7 +1374,6 @@ static void qedi_tmf_work(struct work_struct *work)
 
 	mtask = qedi_cmd->task;
 	tmf_hdr = (struct iscsi_tm *)mtask->hdr;
-	cls_sess = iscsi_conn_to_session(qedi_conn->cls_conn);
 	set_bit(QEDI_CONN_FW_CLEANUP, &qedi_conn->flags);
 
 	ctask = iscsi_itt_to_task(conn, tmf_hdr->rtt);
diff --git a/drivers/scsi/qedi/qedi_iscsi.c b/drivers/scsi/qedi/qedi_iscsi.c
index 6d6d6013e35b..615cea4fad56 100644
--- a/drivers/scsi/qedi/qedi_iscsi.c
+++ b/drivers/scsi/qedi/qedi_iscsi.c
@@ -579,7 +579,7 @@ static int qedi_conn_start(struct iscsi_cls_conn *cls_conn)
 	rval = qedi_iscsi_update_conn(qedi, qedi_conn);
 	if (rval) {
 		iscsi_conn_printk(KERN_ALERT, conn,
-				  "conn_start: FW oflload conn failed.\n");
+				  "conn_start: FW offload conn failed.\n");
 		rval = -EINVAL;
 		goto start_err;
 	}
@@ -590,7 +590,7 @@ static int qedi_conn_start(struct iscsi_cls_conn *cls_conn)
 	rval = iscsi_conn_start(cls_conn);
 	if (rval) {
 		iscsi_conn_printk(KERN_ALERT, conn,
-				  "iscsi_conn_start: FW oflload conn failed!!\n");
+				  "iscsi_conn_start: FW offload conn failed!!\n");
 	}
 
 start_err:
@@ -993,13 +993,17 @@ static void qedi_ep_disconnect(struct iscsi_endpoint *ep)
 	struct iscsi_conn *conn = NULL;
 	struct qedi_ctx *qedi;
 	int ret = 0;
-	int wait_delay = 20 * HZ;
+	int wait_delay;
 	int abrt_conn = 0;
 	int count = 10;
 
+	wait_delay = 60 * HZ + DEF_MAX_RT_TIME;
 	qedi_ep = ep->dd_data;
 	qedi = qedi_ep->qedi;
 
+	if (qedi_ep->state == EP_STATE_OFLDCONN_START)
+		goto ep_exit_recover;
+
 	flush_work(&qedi_ep->offload_work);
 
 	if (qedi_ep->conn) {
@@ -1163,7 +1167,7 @@ static void qedi_offload_work(struct work_struct *work)
 	struct qedi_endpoint *qedi_ep =
 		container_of(work, struct qedi_endpoint, offload_work);
 	struct qedi_ctx *qedi;
-	int wait_delay = 20 * HZ;
+	int wait_delay = 5 * HZ;
 	int ret;
 
 	qedi = qedi_ep->qedi;
diff --git a/drivers/scsi/qla2xxx/qla_attr.c b/drivers/scsi/qla2xxx/qla_attr.c
index f928c4d3a1ef..8d560c562e9c 100644
--- a/drivers/scsi/qla2xxx/qla_attr.c
+++ b/drivers/scsi/qla2xxx/qla_attr.c
@@ -29,24 +29,27 @@ qla2x00_sysfs_read_fw_dump(struct file *filp, struct kobject *kobj,
 	if (!(ha->fw_dump_reading || ha->mctp_dump_reading))
 		return 0;
 
+	mutex_lock(&ha->optrom_mutex);
 	if (IS_P3P_TYPE(ha)) {
 		if (off < ha->md_template_size) {
 			rval = memory_read_from_buffer(buf, count,
 			    &off, ha->md_tmplt_hdr, ha->md_template_size);
-			return rval;
+		} else {
+			off -= ha->md_template_size;
+			rval = memory_read_from_buffer(buf, count,
+			    &off, ha->md_dump, ha->md_dump_size);
 		}
-		off -= ha->md_template_size;
-		rval = memory_read_from_buffer(buf, count,
-		    &off, ha->md_dump, ha->md_dump_size);
-		return rval;
-	} else if (ha->mctp_dumped && ha->mctp_dump_reading)
-		return memory_read_from_buffer(buf, count, &off, ha->mctp_dump,
+	} else if (ha->mctp_dumped && ha->mctp_dump_reading) {
+		rval = memory_read_from_buffer(buf, count, &off, ha->mctp_dump,
 		    MCTP_DUMP_SIZE);
-	else if (ha->fw_dump_reading)
-		return memory_read_from_buffer(buf, count, &off, ha->fw_dump,
+	} else if (ha->fw_dump_reading) {
+		rval = memory_read_from_buffer(buf, count, &off, ha->fw_dump,
 					ha->fw_dump_len);
-	else
-		return 0;
+	} else {
+		rval = 0;
+	}
+	mutex_unlock(&ha->optrom_mutex);
+	return rval;
 }
 
 static ssize_t
@@ -154,6 +157,8 @@ qla2x00_sysfs_read_nvram(struct file *filp, struct kobject *kobj,
 	struct scsi_qla_host *vha = shost_priv(dev_to_shost(container_of(kobj,
 	    struct device, kobj)));
 	struct qla_hw_data *ha = vha->hw;
+	uint32_t faddr;
+	struct active_regions active_regions = { };
 
 	if (!capable(CAP_SYS_ADMIN))
 		return 0;
@@ -164,11 +169,21 @@ qla2x00_sysfs_read_nvram(struct file *filp, struct kobject *kobj,
 		return -EAGAIN;
 	}
 
-	if (IS_NOCACHE_VPD_TYPE(ha))
-		ha->isp_ops->read_optrom(vha, ha->nvram, ha->flt_region_nvram << 2,
-		    ha->nvram_size);
+	if (!IS_NOCACHE_VPD_TYPE(ha)) {
+		mutex_unlock(&ha->optrom_mutex);
+		goto skip;
+	}
+
+	faddr = ha->flt_region_nvram;
+	if (IS_QLA28XX(ha)) {
+		if (active_regions.aux.vpd_nvram == QLA27XX_SECONDARY_IMAGE)
+			faddr = ha->flt_region_nvram_sec;
+	}
+	ha->isp_ops->read_optrom(vha, ha->nvram, faddr << 2, ha->nvram_size);
+
 	mutex_unlock(&ha->optrom_mutex);
 
+skip:
 	return memory_read_from_buffer(buf, count, &off, ha->nvram,
 					ha->nvram_size);
 }
@@ -223,9 +238,9 @@ qla2x00_sysfs_write_nvram(struct file *filp, struct kobject *kobj,
 	}
 
 	/* Write NVRAM. */
-	ha->isp_ops->write_nvram(vha, (uint8_t *)buf, ha->nvram_base, count);
-	ha->isp_ops->read_nvram(vha, (uint8_t *)ha->nvram, ha->nvram_base,
-	     count);
+	ha->isp_ops->write_nvram(vha, buf, ha->nvram_base, count);
+	ha->isp_ops->read_nvram(vha, ha->nvram, ha->nvram_base,
+	    count);
 	mutex_unlock(&ha->optrom_mutex);
 
 	ql_dbg(ql_dbg_user, vha, 0x7060,
@@ -364,7 +379,7 @@ qla2x00_sysfs_write_optrom_ctl(struct file *filp, struct kobject *kobj,
 		}
 
 		ha->optrom_region_start = start;
-		ha->optrom_region_size = start + size;
+		ha->optrom_region_size = size;
 
 		ha->optrom_state = QLA_SREADING;
 		ha->optrom_buffer = vmalloc(ha->optrom_region_size);
@@ -418,6 +433,10 @@ qla2x00_sysfs_write_optrom_ctl(struct file *filp, struct kobject *kobj,
 		 * 	0x000000 -> 0x07ffff -- Boot code.
 		 * 	0x080000 -> 0x0fffff -- Firmware.
 		 * 	0x120000 -> 0x12ffff -- VPD and HBA parameters.
+		 *
+		 * > ISP25xx type boards:
+		 *
+		 *      None -- should go through BSG.
 		 */
 		valid = 0;
 		if (ha->optrom_size == OPTROM_SIZE_2300 && start == 0)
@@ -425,9 +444,7 @@ qla2x00_sysfs_write_optrom_ctl(struct file *filp, struct kobject *kobj,
 		else if (start == (ha->flt_region_boot * 4) ||
 		    start == (ha->flt_region_fw * 4))
 			valid = 1;
-		else if (IS_QLA24XX_TYPE(ha) || IS_QLA25XX(ha)
-			|| IS_CNA_CAPABLE(ha) || IS_QLA2031(ha)
-			|| IS_QLA27XX(ha))
+		else if (IS_QLA24XX_TYPE(ha) || IS_QLA25XX(ha))
 			valid = 1;
 		if (!valid) {
 			ql_log(ql_log_warn, vha, 0x7065,
@@ -437,7 +454,7 @@ qla2x00_sysfs_write_optrom_ctl(struct file *filp, struct kobject *kobj,
 		}
 
 		ha->optrom_region_start = start;
-		ha->optrom_region_size = start + size;
+		ha->optrom_region_size = size;
 
 		ha->optrom_state = QLA_SWRITING;
 		ha->optrom_buffer = vmalloc(ha->optrom_region_size);
@@ -504,6 +521,7 @@ qla2x00_sysfs_read_vpd(struct file *filp, struct kobject *kobj,
 	    struct device, kobj)));
 	struct qla_hw_data *ha = vha->hw;
 	uint32_t faddr;
+	struct active_regions active_regions = { };
 
 	if (unlikely(pci_channel_offline(ha->pdev)))
 		return -EAGAIN;
@@ -511,22 +529,33 @@ qla2x00_sysfs_read_vpd(struct file *filp, struct kobject *kobj,
 	if (!capable(CAP_SYS_ADMIN))
 		return -EINVAL;
 
-	if (IS_NOCACHE_VPD_TYPE(ha)) {
-		faddr = ha->flt_region_vpd << 2;
+	if (IS_NOCACHE_VPD_TYPE(ha))
+		goto skip;
+
+	faddr = ha->flt_region_vpd << 2;
 
-		if (IS_QLA27XX(ha) &&
-		    qla27xx_find_valid_image(vha) == QLA27XX_SECONDARY_IMAGE)
+	if (IS_QLA28XX(ha)) {
+		qla28xx_get_aux_images(vha, &active_regions);
+		if (active_regions.aux.vpd_nvram == QLA27XX_SECONDARY_IMAGE)
 			faddr = ha->flt_region_vpd_sec << 2;
 
-		mutex_lock(&ha->optrom_mutex);
-		if (qla2x00_chip_is_down(vha)) {
-			mutex_unlock(&ha->optrom_mutex);
-			return -EAGAIN;
-		}
-		ha->isp_ops->read_optrom(vha, ha->vpd, faddr,
-		    ha->vpd_size);
+		ql_dbg(ql_dbg_init, vha, 0x7070,
+		    "Loading %s nvram image.\n",
+		    active_regions.aux.vpd_nvram == QLA27XX_PRIMARY_IMAGE ?
+		    "primary" : "secondary");
+	}
+
+	mutex_lock(&ha->optrom_mutex);
+	if (qla2x00_chip_is_down(vha)) {
 		mutex_unlock(&ha->optrom_mutex);
+		return -EAGAIN;
 	}
+
+	ha->isp_ops->read_optrom(vha, ha->vpd, faddr, ha->vpd_size);
+	mutex_unlock(&ha->optrom_mutex);
+
+	ha->isp_ops->read_optrom(vha, ha->vpd, faddr, ha->vpd_size);
+skip:
 	return memory_read_from_buffer(buf, count, &off, ha->vpd, ha->vpd_size);
 }
 
@@ -563,8 +592,8 @@ qla2x00_sysfs_write_vpd(struct file *filp, struct kobject *kobj,
 	}
 
 	/* Write NVRAM. */
-	ha->isp_ops->write_nvram(vha, (uint8_t *)buf, ha->vpd_base, count);
-	ha->isp_ops->read_nvram(vha, (uint8_t *)ha->vpd, ha->vpd_base, count);
+	ha->isp_ops->write_nvram(vha, buf, ha->vpd_base, count);
+	ha->isp_ops->read_nvram(vha, ha->vpd, ha->vpd_base, count);
 
 	/* Update flash version information for 4Gb & above. */
 	if (!IS_FWI2_CAPABLE(ha)) {
@@ -645,6 +674,7 @@ qla2x00_sysfs_write_reset(struct file *filp, struct kobject *kobj,
 	int type;
 	uint32_t idc_control;
 	uint8_t *tmp_data = NULL;
+
 	if (off != 0)
 		return -EINVAL;
 
@@ -682,7 +712,7 @@ qla2x00_sysfs_write_reset(struct file *filp, struct kobject *kobj,
 		ql_log(ql_log_info, vha, 0x706f,
 		    "Issuing MPI reset.\n");
 
-		if (IS_QLA83XX(ha) || IS_QLA27XX(ha)) {
+		if (IS_QLA83XX(ha) || IS_QLA27XX(ha) || IS_QLA28XX(ha)) {
 			uint32_t idc_control;
 
 			qla83xx_idc_lock(vha, 0);
@@ -858,7 +888,7 @@ do_read:
 		count = 0;
 	}
 
-	count = actual_size > count ? count: actual_size;
+	count = actual_size > count ? count : actual_size;
 	memcpy(buf, ha->xgmac_data, count);
 
 	return count;
@@ -934,7 +964,7 @@ static struct bin_attribute sysfs_dcbx_tlv_attr = {
 static struct sysfs_entry {
 	char *name;
 	struct bin_attribute *attr;
-	int is4GBp_only;
+	int type;
 } bin_file_entries[] = {
 	{ "fw_dump", &sysfs_fw_dump_attr, },
 	{ "nvram", &sysfs_nvram_attr, },
@@ -957,11 +987,11 @@ qla2x00_alloc_sysfs_attr(scsi_qla_host_t *vha)
 	int ret;
 
 	for (iter = bin_file_entries; iter->name; iter++) {
-		if (iter->is4GBp_only && !IS_FWI2_CAPABLE(vha->hw))
+		if (iter->type && !IS_FWI2_CAPABLE(vha->hw))
 			continue;
-		if (iter->is4GBp_only == 2 && !IS_QLA25XX(vha->hw))
+		if (iter->type == 2 && !IS_QLA25XX(vha->hw))
 			continue;
-		if (iter->is4GBp_only == 3 && !(IS_CNA_CAPABLE(vha->hw)))
+		if (iter->type == 3 && !(IS_CNA_CAPABLE(vha->hw)))
 			continue;
 
 		ret = sysfs_create_bin_file(&host->shost_gendev.kobj,
@@ -985,13 +1015,14 @@ qla2x00_free_sysfs_attr(scsi_qla_host_t *vha, bool stop_beacon)
 	struct qla_hw_data *ha = vha->hw;
 
 	for (iter = bin_file_entries; iter->name; iter++) {
-		if (iter->is4GBp_only && !IS_FWI2_CAPABLE(ha))
+		if (iter->type && !IS_FWI2_CAPABLE(ha))
 			continue;
-		if (iter->is4GBp_only == 2 && !IS_QLA25XX(ha))
+		if (iter->type == 2 && !IS_QLA25XX(ha))
 			continue;
-		if (iter->is4GBp_only == 3 && !(IS_CNA_CAPABLE(vha->hw)))
+		if (iter->type == 3 && !(IS_CNA_CAPABLE(ha)))
 			continue;
-		if (iter->is4GBp_only == 0x27 && !IS_QLA27XX(vha->hw))
+		if (iter->type == 0x27 &&
+		    (!IS_QLA27XX(ha) || !IS_QLA28XX(ha)))
 			continue;
 
 		sysfs_remove_bin_file(&host->shost_gendev.kobj,
@@ -1049,6 +1080,7 @@ qla2x00_isp_name_show(struct device *dev, struct device_attribute *attr,
 		      char *buf)
 {
 	scsi_qla_host_t *vha = shost_priv(class_to_shost(dev));
+
 	return scnprintf(buf, PAGE_SIZE, "ISP%04X\n", vha->hw->pdev->device);
 }
 
@@ -1082,6 +1114,7 @@ qla2x00_model_desc_show(struct device *dev, struct device_attribute *attr,
 			char *buf)
 {
 	scsi_qla_host_t *vha = shost_priv(class_to_shost(dev));
+
 	return scnprintf(buf, PAGE_SIZE, "%s\n", vha->hw->model_desc);
 }
 
@@ -1294,6 +1327,7 @@ qla2x00_optrom_bios_version_show(struct device *dev,
 {
 	scsi_qla_host_t *vha = shost_priv(class_to_shost(dev));
 	struct qla_hw_data *ha = vha->hw;
+
 	return scnprintf(buf, PAGE_SIZE, "%d.%02d\n", ha->bios_revision[1],
 	    ha->bios_revision[0]);
 }
@@ -1304,6 +1338,7 @@ qla2x00_optrom_efi_version_show(struct device *dev,
 {
 	scsi_qla_host_t *vha = shost_priv(class_to_shost(dev));
 	struct qla_hw_data *ha = vha->hw;
+
 	return scnprintf(buf, PAGE_SIZE, "%d.%02d\n", ha->efi_revision[1],
 	    ha->efi_revision[0]);
 }
@@ -1314,6 +1349,7 @@ qla2x00_optrom_fcode_version_show(struct device *dev,
 {
 	scsi_qla_host_t *vha = shost_priv(class_to_shost(dev));
 	struct qla_hw_data *ha = vha->hw;
+
 	return scnprintf(buf, PAGE_SIZE, "%d.%02d\n", ha->fcode_revision[1],
 	    ha->fcode_revision[0]);
 }
@@ -1324,6 +1360,7 @@ qla2x00_optrom_fw_version_show(struct device *dev,
 {
 	scsi_qla_host_t *vha = shost_priv(class_to_shost(dev));
 	struct qla_hw_data *ha = vha->hw;
+
 	return scnprintf(buf, PAGE_SIZE, "%d.%02d.%02d %d\n",
 	    ha->fw_revision[0], ha->fw_revision[1], ha->fw_revision[2],
 	    ha->fw_revision[3]);
@@ -1336,7 +1373,8 @@ qla2x00_optrom_gold_fw_version_show(struct device *dev,
 	scsi_qla_host_t *vha = shost_priv(class_to_shost(dev));
 	struct qla_hw_data *ha = vha->hw;
 
-	if (!IS_QLA81XX(ha) && !IS_QLA83XX(ha) && !IS_QLA27XX(ha))
+	if (!IS_QLA81XX(ha) && !IS_QLA83XX(ha) &&
+	    !IS_QLA27XX(ha) && !IS_QLA28XX(ha))
 		return scnprintf(buf, PAGE_SIZE, "\n");
 
 	return scnprintf(buf, PAGE_SIZE, "%d.%02d.%02d (%d)\n",
@@ -1349,6 +1387,7 @@ qla2x00_total_isp_aborts_show(struct device *dev,
 			      struct device_attribute *attr, char *buf)
 {
 	scsi_qla_host_t *vha = shost_priv(class_to_shost(dev));
+
 	return scnprintf(buf, PAGE_SIZE, "%d\n",
 	    vha->qla_stats.total_isp_aborts);
 }
@@ -1358,24 +1397,40 @@ qla24xx_84xx_fw_version_show(struct device *dev,
 	struct device_attribute *attr, char *buf)
 {
 	int rval = QLA_SUCCESS;
-	uint16_t status[2] = {0, 0};
+	uint16_t status[2] = { 0 };
 	scsi_qla_host_t *vha = shost_priv(class_to_shost(dev));
 	struct qla_hw_data *ha = vha->hw;
 
 	if (!IS_QLA84XX(ha))
 		return scnprintf(buf, PAGE_SIZE, "\n");
 
-	if (ha->cs84xx->op_fw_version == 0)
+	if (!ha->cs84xx->op_fw_version) {
 		rval = qla84xx_verify_chip(vha, status);
 
-	if ((rval == QLA_SUCCESS) && (status[0] == 0))
-		return scnprintf(buf, PAGE_SIZE, "%u\n",
-			(uint32_t)ha->cs84xx->op_fw_version);
+		if (!rval && !status[0])
+			return scnprintf(buf, PAGE_SIZE, "%u\n",
+			    (uint32_t)ha->cs84xx->op_fw_version);
+	}
 
 	return scnprintf(buf, PAGE_SIZE, "\n");
 }
 
 static ssize_t
+qla2x00_serdes_version_show(struct device *dev, struct device_attribute *attr,
+    char *buf)
+{
+	scsi_qla_host_t *vha = shost_priv(class_to_shost(dev));
+	struct qla_hw_data *ha = vha->hw;
+
+	if (!IS_QLA27XX(ha) && !IS_QLA28XX(ha))
+		return scnprintf(buf, PAGE_SIZE, "\n");
+
+	return scnprintf(buf, PAGE_SIZE, "%d.%02d.%02d\n",
+	    ha->serdes_version[0], ha->serdes_version[1],
+	    ha->serdes_version[2]);
+}
+
+static ssize_t
 qla2x00_mpi_version_show(struct device *dev, struct device_attribute *attr,
     char *buf)
 {
@@ -1383,7 +1438,7 @@ qla2x00_mpi_version_show(struct device *dev, struct device_attribute *attr,
 	struct qla_hw_data *ha = vha->hw;
 
 	if (!IS_QLA81XX(ha) && !IS_QLA8031(ha) && !IS_QLA8044(ha) &&
-	    !IS_QLA27XX(ha))
+	    !IS_QLA27XX(ha) && !IS_QLA28XX(ha))
 		return scnprintf(buf, PAGE_SIZE, "\n");
 
 	return scnprintf(buf, PAGE_SIZE, "%d.%02d.%02d (%x)\n",
@@ -1596,7 +1651,7 @@ qla2x00_pep_version_show(struct device *dev, struct device_attribute *attr,
 	scsi_qla_host_t *vha = shost_priv(class_to_shost(dev));
 	struct qla_hw_data *ha = vha->hw;
 
-	if (!IS_QLA27XX(ha))
+	if (!IS_QLA27XX(ha) && !IS_QLA28XX(ha))
 		return scnprintf(buf, PAGE_SIZE, "\n");
 
 	return scnprintf(buf, PAGE_SIZE, "%d.%02d.%02d\n",
@@ -1604,35 +1659,38 @@ qla2x00_pep_version_show(struct device *dev, struct device_attribute *attr,
 }
 
 static ssize_t
-qla2x00_min_link_speed_show(struct device *dev, struct device_attribute *attr,
-    char *buf)
+qla2x00_min_supported_speed_show(struct device *dev,
+    struct device_attribute *attr, char *buf)
 {
 	scsi_qla_host_t *vha = shost_priv(class_to_shost(dev));
 	struct qla_hw_data *ha = vha->hw;
 
-	if (!IS_QLA27XX(ha))
+	if (!IS_QLA27XX(ha) && !IS_QLA28XX(ha))
 		return scnprintf(buf, PAGE_SIZE, "\n");
 
 	return scnprintf(buf, PAGE_SIZE, "%s\n",
-	    ha->min_link_speed == 5 ? "32Gps" :
-	    ha->min_link_speed == 4 ? "16Gps" :
-	    ha->min_link_speed == 3 ? "8Gps" :
-	    ha->min_link_speed == 2 ? "4Gps" :
-	    ha->min_link_speed != 0 ? "unknown" : "");
+	    ha->min_supported_speed == 6 ? "64Gps" :
+	    ha->min_supported_speed == 5 ? "32Gps" :
+	    ha->min_supported_speed == 4 ? "16Gps" :
+	    ha->min_supported_speed == 3 ? "8Gps" :
+	    ha->min_supported_speed == 2 ? "4Gps" :
+	    ha->min_supported_speed != 0 ? "unknown" : "");
 }
 
 static ssize_t
-qla2x00_max_speed_sup_show(struct device *dev, struct device_attribute *attr,
-    char *buf)
+qla2x00_max_supported_speed_show(struct device *dev,
+    struct device_attribute *attr, char *buf)
 {
 	scsi_qla_host_t *vha = shost_priv(class_to_shost(dev));
 	struct qla_hw_data *ha = vha->hw;
 
-	if (!IS_QLA27XX(ha))
+	if (!IS_QLA27XX(ha) && !IS_QLA28XX(ha))
 		return scnprintf(buf, PAGE_SIZE, "\n");
 
 	return scnprintf(buf, PAGE_SIZE, "%s\n",
-	    ha->max_speed_sup ? "32Gps" : "16Gps");
+	    ha->max_supported_speed  == 2 ? "64Gps" :
+	    ha->max_supported_speed  == 1 ? "32Gps" :
+	    ha->max_supported_speed  == 0 ? "16Gps" : "unknown");
 }
 
 static ssize_t
@@ -1645,7 +1703,7 @@ qla2x00_port_speed_store(struct device *dev, struct device_attribute *attr,
 	int mode = QLA_SET_DATA_RATE_LR;
 	struct qla_hw_data *ha = vha->hw;
 
-	if (!IS_QLA27XX(vha->hw)) {
+	if (!IS_QLA27XX(ha) && !IS_QLA28XX(ha)) {
 		ql_log(ql_log_warn, vha, 0x70d8,
 		    "Speed setting not supported \n");
 		return -EINVAL;
@@ -2164,6 +2222,32 @@ qla2x00_dif_bundle_statistics_show(struct device *dev,
 	    ha->dif_bundle_dma_allocs, ha->pool.unusable.count);
 }
 
+static ssize_t
+qla2x00_fw_attr_show(struct device *dev,
+    struct device_attribute *attr, char *buf)
+{
+	scsi_qla_host_t *vha = shost_priv(class_to_shost(dev));
+	struct qla_hw_data *ha = vha->hw;
+
+	if (!IS_QLA27XX(ha) && !IS_QLA28XX(ha))
+		return scnprintf(buf, PAGE_SIZE, "\n");
+
+	return scnprintf(buf, PAGE_SIZE, "%llx\n",
+	    (uint64_t)ha->fw_attributes_ext[1] << 48 |
+	    (uint64_t)ha->fw_attributes_ext[0] << 32 |
+	    (uint64_t)ha->fw_attributes_h << 16 |
+	    (uint64_t)ha->fw_attributes);
+}
+
+static ssize_t
+qla2x00_port_no_show(struct device *dev, struct device_attribute *attr,
+    char *buf)
+{
+	scsi_qla_host_t *vha = shost_priv(class_to_shost(dev));
+
+	return scnprintf(buf, PAGE_SIZE, "%u\n", vha->hw->port_no);
+}
+
 static DEVICE_ATTR(driver_version, S_IRUGO, qla2x00_driver_version_show, NULL);
 static DEVICE_ATTR(fw_version, S_IRUGO, qla2x00_fw_version_show, NULL);
 static DEVICE_ATTR(serial_num, S_IRUGO, qla2x00_serial_num_show, NULL);
@@ -2192,6 +2276,7 @@ static DEVICE_ATTR(84xx_fw_version, S_IRUGO, qla24xx_84xx_fw_version_show,
 		   NULL);
 static DEVICE_ATTR(total_isp_aborts, S_IRUGO, qla2x00_total_isp_aborts_show,
 		   NULL);
+static DEVICE_ATTR(serdes_version, 0444, qla2x00_serdes_version_show, NULL);
 static DEVICE_ATTR(mpi_version, S_IRUGO, qla2x00_mpi_version_show, NULL);
 static DEVICE_ATTR(phy_version, S_IRUGO, qla2x00_phy_version_show, NULL);
 static DEVICE_ATTR(flash_block_size, S_IRUGO, qla2x00_flash_block_size_show,
@@ -2209,8 +2294,10 @@ static DEVICE_ATTR(allow_cna_fw_dump, S_IRUGO | S_IWUSR,
 		   qla2x00_allow_cna_fw_dump_show,
 		   qla2x00_allow_cna_fw_dump_store);
 static DEVICE_ATTR(pep_version, S_IRUGO, qla2x00_pep_version_show, NULL);
-static DEVICE_ATTR(min_link_speed, S_IRUGO, qla2x00_min_link_speed_show, NULL);
-static DEVICE_ATTR(max_speed_sup, S_IRUGO, qla2x00_max_speed_sup_show, NULL);
+static DEVICE_ATTR(min_supported_speed, 0444,
+		   qla2x00_min_supported_speed_show, NULL);
+static DEVICE_ATTR(max_supported_speed, 0444,
+		   qla2x00_max_supported_speed_show, NULL);
 static DEVICE_ATTR(zio_threshold, 0644,
     qla_zio_threshold_show,
     qla_zio_threshold_store);
@@ -2221,6 +2308,8 @@ static DEVICE_ATTR(dif_bundle_statistics, 0444,
     qla2x00_dif_bundle_statistics_show, NULL);
 static DEVICE_ATTR(port_speed, 0644, qla2x00_port_speed_show,
     qla2x00_port_speed_store);
+static DEVICE_ATTR(port_no, 0444, qla2x00_port_no_show, NULL);
+static DEVICE_ATTR(fw_attr, 0444, qla2x00_fw_attr_show, NULL);
 
 
 struct device_attribute *qla2x00_host_attrs[] = {
@@ -2242,6 +2331,7 @@ struct device_attribute *qla2x00_host_attrs[] = {
 	&dev_attr_optrom_fw_version,
 	&dev_attr_84xx_fw_version,
 	&dev_attr_total_isp_aborts,
+	&dev_attr_serdes_version,
 	&dev_attr_mpi_version,
 	&dev_attr_phy_version,
 	&dev_attr_flash_block_size,
@@ -2256,11 +2346,13 @@ struct device_attribute *qla2x00_host_attrs[] = {
 	&dev_attr_fw_dump_size,
 	&dev_attr_allow_cna_fw_dump,
 	&dev_attr_pep_version,
-	&dev_attr_min_link_speed,
-	&dev_attr_max_speed_sup,
+	&dev_attr_min_supported_speed,
+	&dev_attr_max_supported_speed,
 	&dev_attr_zio_threshold,
 	&dev_attr_dif_bundle_statistics,
 	&dev_attr_port_speed,
+	&dev_attr_port_no,
+	&dev_attr_fw_attr,
 	NULL, /* reserve for qlini_mode */
 	NULL, /* reserve for ql2xiniexchg */
 	NULL, /* reserve for ql2xexchoffld */
@@ -2296,16 +2388,15 @@ qla2x00_get_host_port_id(struct Scsi_Host *shost)
 static void
 qla2x00_get_host_speed(struct Scsi_Host *shost)
 {
-	struct qla_hw_data *ha = ((struct scsi_qla_host *)
-					(shost_priv(shost)))->hw;
-	u32 speed = FC_PORTSPEED_UNKNOWN;
+	scsi_qla_host_t *vha = shost_priv(shost);
+	u32 speed;
 
-	if (IS_QLAFX00(ha)) {
+	if (IS_QLAFX00(vha->hw)) {
 		qlafx00_get_host_speed(shost);
 		return;
 	}
 
-	switch (ha->link_data_rate) {
+	switch (vha->hw->link_data_rate) {
 	case PORT_SPEED_1GB:
 		speed = FC_PORTSPEED_1GBIT;
 		break;
@@ -2327,7 +2418,14 @@ qla2x00_get_host_speed(struct Scsi_Host *shost)
 	case PORT_SPEED_32GB:
 		speed = FC_PORTSPEED_32GBIT;
 		break;
+	case PORT_SPEED_64GB:
+		speed = FC_PORTSPEED_64GBIT;
+		break;
+	default:
+		speed = FC_PORTSPEED_UNKNOWN;
+		break;
 	}
+
 	fc_host_speed(shost) = speed;
 }
 
@@ -2335,7 +2433,7 @@ static void
 qla2x00_get_host_port_type(struct Scsi_Host *shost)
 {
 	scsi_qla_host_t *vha = shost_priv(shost);
-	uint32_t port_type = FC_PORTTYPE_UNKNOWN;
+	uint32_t port_type;
 
 	if (vha->vp_idx) {
 		fc_host_port_type(shost) = FC_PORTTYPE_NPIV;
@@ -2354,7 +2452,11 @@ qla2x00_get_host_port_type(struct Scsi_Host *shost)
 	case ISP_CFG_F:
 		port_type = FC_PORTTYPE_NPORT;
 		break;
+	default:
+		port_type = FC_PORTTYPE_UNKNOWN;
+		break;
 	}
+
 	fc_host_port_type(shost) = port_type;
 }
 
@@ -2416,13 +2518,10 @@ qla2x00_get_starget_port_id(struct scsi_target *starget)
 	fc_starget_port_id(starget) = port_id;
 }
 
-static void
+static inline void
 qla2x00_set_rport_loss_tmo(struct fc_rport *rport, uint32_t timeout)
 {
-	if (timeout)
-		rport->dev_loss_tmo = timeout;
-	else
-		rport->dev_loss_tmo = 1;
+	rport->dev_loss_tmo = timeout ? timeout : 1;
 }
 
 static void
@@ -2632,8 +2731,9 @@ static void
 qla2x00_get_host_fabric_name(struct Scsi_Host *shost)
 {
 	scsi_qla_host_t *vha = shost_priv(shost);
-	uint8_t node_name[WWN_SIZE] = { 0xFF, 0xFF, 0xFF, 0xFF, \
-		0xFF, 0xFF, 0xFF, 0xFF};
+	static const uint8_t node_name[WWN_SIZE] = {
+		0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF
+	};
 	u64 fabric_name = wwn_to_u64(node_name);
 
 	if (vha->device_flags & SWITCH_FOUND)
@@ -2711,8 +2811,8 @@ qla24xx_vport_create(struct fc_vport *fc_vport, bool disable)
 
 	/* initialized vport states */
 	atomic_set(&vha->loop_state, LOOP_DOWN);
-	vha->vp_err_state=  VP_ERR_PORTDWN;
-	vha->vp_prev_err_state=  VP_ERR_UNKWN;
+	vha->vp_err_state = VP_ERR_PORTDWN;
+	vha->vp_prev_err_state = VP_ERR_UNKWN;
 	/* Check if physical ha port is Up */
 	if (atomic_read(&base_vha->loop_state) == LOOP_DOWN ||
 	    atomic_read(&base_vha->loop_state) == LOOP_DEAD) {
@@ -2727,6 +2827,7 @@ qla24xx_vport_create(struct fc_vport *fc_vport, bool disable)
 	if (IS_T10_PI_CAPABLE(ha) && ql2xenabledif) {
 		if (ha->fw_attributes & BIT_4) {
 			int prot = 0, guard;
+
 			vha->flags.difdix_supported = 1;
 			ql_dbg(ql_dbg_user, vha, 0x7082,
 			    "Registered for DIF/DIX type 1 and 3 protection.\n");
@@ -2977,7 +3078,7 @@ void
 qla2x00_init_host_attr(scsi_qla_host_t *vha)
 {
 	struct qla_hw_data *ha = vha->hw;
-	u32 speed = FC_PORTSPEED_UNKNOWN;
+	u32 speeds = FC_PORTSPEED_UNKNOWN;
 
 	fc_host_dev_loss_tmo(vha->host) = ha->port_down_retry_count;
 	fc_host_node_name(vha->host) = wwn_to_u64(vha->node_name);
@@ -2988,25 +3089,45 @@ qla2x00_init_host_attr(scsi_qla_host_t *vha)
 	fc_host_npiv_vports_inuse(vha->host) = ha->cur_vport_count;
 
 	if (IS_CNA_CAPABLE(ha))
-		speed = FC_PORTSPEED_10GBIT;
-	else if (IS_QLA2031(ha))
-		speed = FC_PORTSPEED_16GBIT | FC_PORTSPEED_8GBIT |
-		    FC_PORTSPEED_4GBIT;
-	else if (IS_QLA25XX(ha))
-		speed = FC_PORTSPEED_8GBIT | FC_PORTSPEED_4GBIT |
-		    FC_PORTSPEED_2GBIT | FC_PORTSPEED_1GBIT;
+		speeds = FC_PORTSPEED_10GBIT;
+	else if (IS_QLA28XX(ha) || IS_QLA27XX(ha)) {
+		if (ha->max_supported_speed == 2) {
+			if (ha->min_supported_speed <= 6)
+				speeds |= FC_PORTSPEED_64GBIT;
+		}
+		if (ha->max_supported_speed == 2 ||
+		    ha->max_supported_speed == 1) {
+			if (ha->min_supported_speed <= 5)
+				speeds |= FC_PORTSPEED_32GBIT;
+		}
+		if (ha->max_supported_speed == 2 ||
+		    ha->max_supported_speed == 1 ||
+		    ha->max_supported_speed == 0) {
+			if (ha->min_supported_speed <= 4)
+				speeds |= FC_PORTSPEED_16GBIT;
+		}
+		if (ha->max_supported_speed == 1 ||
+		    ha->max_supported_speed == 0) {
+			if (ha->min_supported_speed <= 3)
+				speeds |= FC_PORTSPEED_8GBIT;
+		}
+		if (ha->max_supported_speed == 0) {
+			if (ha->min_supported_speed <= 2)
+				speeds |= FC_PORTSPEED_4GBIT;
+		}
+	} else if (IS_QLA2031(ha))
+		speeds = FC_PORTSPEED_16GBIT|FC_PORTSPEED_8GBIT|
+			FC_PORTSPEED_4GBIT;
+	else if (IS_QLA25XX(ha) || IS_QLAFX00(ha))
+		speeds = FC_PORTSPEED_8GBIT|FC_PORTSPEED_4GBIT|
+			FC_PORTSPEED_2GBIT|FC_PORTSPEED_1GBIT;
 	else if (IS_QLA24XX_TYPE(ha))
-		speed = FC_PORTSPEED_4GBIT | FC_PORTSPEED_2GBIT |
-		    FC_PORTSPEED_1GBIT;
+		speeds = FC_PORTSPEED_4GBIT|FC_PORTSPEED_2GBIT|
+			FC_PORTSPEED_1GBIT;
 	else if (IS_QLA23XX(ha))
-		speed = FC_PORTSPEED_2GBIT | FC_PORTSPEED_1GBIT;
-	else if (IS_QLAFX00(ha))
-		speed = FC_PORTSPEED_8GBIT | FC_PORTSPEED_4GBIT |
-		    FC_PORTSPEED_2GBIT | FC_PORTSPEED_1GBIT;
-	else if (IS_QLA27XX(ha))
-		speed = FC_PORTSPEED_32GBIT | FC_PORTSPEED_16GBIT |
-		    FC_PORTSPEED_8GBIT;
+		speeds = FC_PORTSPEED_2GBIT|FC_PORTSPEED_1GBIT;
 	else
-		speed = FC_PORTSPEED_1GBIT;
-	fc_host_supported_speeds(vha->host) = speed;
+		speeds = FC_PORTSPEED_1GBIT;
+
+	fc_host_supported_speeds(vha->host) = speeds;
 }
diff --git a/drivers/scsi/qla2xxx/qla_bsg.c b/drivers/scsi/qla2xxx/qla_bsg.c
index 17d42658ad9a..5441557b424b 100644
--- a/drivers/scsi/qla2xxx/qla_bsg.c
+++ b/drivers/scsi/qla2xxx/qla_bsg.c
@@ -1,4 +1,4 @@
-	/*
+/*
  * QLogic Fibre Channel HBA Driver
  * Copyright (c)  2003-2014 QLogic Corporation
  *
@@ -84,8 +84,7 @@ qla24xx_fcp_prio_cfg_valid(scsi_qla_host_t *vha,
 		return 0;
 	}
 
-	if (bcode[0] != 'H' || bcode[1] != 'Q' || bcode[2] != 'O' ||
-			bcode[3] != 'S') {
+	if (memcmp(bcode, "HQOS", 4)) {
 		/* Invalid FCP priority data header*/
 		ql_dbg(ql_dbg_user, vha, 0x7052,
 		    "Invalid FCP Priority data header. bcode=0x%x.\n",
@@ -1044,7 +1043,7 @@ qla84xx_updatefw(struct bsg_job *bsg_job)
 	}
 
 	flag = bsg_request->rqst_data.h_vendor.vendor_cmd[1];
-	fw_ver = le32_to_cpu(*((uint32_t *)((uint32_t *)fw_buf + 2)));
+	fw_ver = get_unaligned_le32((uint32_t *)fw_buf + 2);
 
 	mn->entry_type = VERIFY_CHIP_IOCB_TYPE;
 	mn->entry_count = 1;
@@ -1057,9 +1056,8 @@ qla84xx_updatefw(struct bsg_job *bsg_job)
 	mn->fw_ver =  cpu_to_le32(fw_ver);
 	mn->fw_size =  cpu_to_le32(data_len);
 	mn->fw_seq_size =  cpu_to_le32(data_len);
-	mn->dseg_address[0] = cpu_to_le32(LSD(fw_dma));
-	mn->dseg_address[1] = cpu_to_le32(MSD(fw_dma));
-	mn->dseg_length = cpu_to_le32(data_len);
+	put_unaligned_le64(fw_dma, &mn->dsd.address);
+	mn->dsd.length = cpu_to_le32(data_len);
 	mn->data_seg_cnt = cpu_to_le16(1);
 
 	rval = qla2x00_issue_iocb_timeout(vha, mn, mn_dma, 0, 120);
@@ -1238,9 +1236,8 @@ qla84xx_mgmt_cmd(struct bsg_job *bsg_job)
 	if (ql84_mgmt->mgmt.cmd != QLA84_MGMT_CHNG_CONFIG) {
 		mn->total_byte_cnt = cpu_to_le32(ql84_mgmt->mgmt.len);
 		mn->dseg_count = cpu_to_le16(1);
-		mn->dseg_address[0] = cpu_to_le32(LSD(mgmt_dma));
-		mn->dseg_address[1] = cpu_to_le32(MSD(mgmt_dma));
-		mn->dseg_length = cpu_to_le32(ql84_mgmt->mgmt.len);
+		put_unaligned_le64(mgmt_dma, &mn->dsd.address);
+		mn->dsd.length = cpu_to_le32(ql84_mgmt->mgmt.len);
 	}
 
 	rval = qla2x00_issue_iocb(vha, mn, mn_dma, 0);
@@ -1354,7 +1351,7 @@ qla24xx_iidma(struct bsg_job *bsg_job)
 
 	if (rval) {
 		ql_log(ql_log_warn, vha, 0x704c,
-		    "iIDMA cmd failed for %8phN -- "
+		    "iiDMA cmd failed for %8phN -- "
 		    "%04x %x %04x %04x.\n", fcport->port_name,
 		    rval, fcport->fp_speed, mb[0], mb[1]);
 		rval = (DID_ERROR << 16);
@@ -1412,7 +1409,8 @@ qla2x00_optrom_setup(struct bsg_job *bsg_job, scsi_qla_host_t *vha,
 		    start == (ha->flt_region_fw * 4))
 			valid = 1;
 		else if (IS_QLA24XX_TYPE(ha) || IS_QLA25XX(ha) ||
-		    IS_CNA_CAPABLE(ha) || IS_QLA2031(ha) || IS_QLA27XX(ha))
+		    IS_CNA_CAPABLE(ha) || IS_QLA2031(ha) || IS_QLA27XX(ha) ||
+		    IS_QLA28XX(ha))
 			valid = 1;
 		if (!valid) {
 			ql_log(ql_log_warn, vha, 0x7058,
@@ -1534,6 +1532,7 @@ qla2x00_update_fru_versions(struct bsg_job *bsg_job)
 	uint32_t count;
 	dma_addr_t sfp_dma;
 	void *sfp = dma_pool_alloc(ha->s_dma_pool, GFP_KERNEL, &sfp_dma);
+
 	if (!sfp) {
 		bsg_reply->reply_data.vendor_reply.vendor_rsp[0] =
 		    EXT_STATUS_NO_MEMORY;
@@ -1584,6 +1583,7 @@ qla2x00_read_fru_status(struct bsg_job *bsg_job)
 	struct qla_status_reg *sr = (void *)bsg;
 	dma_addr_t sfp_dma;
 	uint8_t *sfp = dma_pool_alloc(ha->s_dma_pool, GFP_KERNEL, &sfp_dma);
+
 	if (!sfp) {
 		bsg_reply->reply_data.vendor_reply.vendor_rsp[0] =
 		    EXT_STATUS_NO_MEMORY;
@@ -1634,6 +1634,7 @@ qla2x00_write_fru_status(struct bsg_job *bsg_job)
 	struct qla_status_reg *sr = (void *)bsg;
 	dma_addr_t sfp_dma;
 	uint8_t *sfp = dma_pool_alloc(ha->s_dma_pool, GFP_KERNEL, &sfp_dma);
+
 	if (!sfp) {
 		bsg_reply->reply_data.vendor_reply.vendor_rsp[0] =
 		    EXT_STATUS_NO_MEMORY;
@@ -1680,6 +1681,7 @@ qla2x00_write_i2c(struct bsg_job *bsg_job)
 	struct qla_i2c_access *i2c = (void *)bsg;
 	dma_addr_t sfp_dma;
 	uint8_t *sfp = dma_pool_alloc(ha->s_dma_pool, GFP_KERNEL, &sfp_dma);
+
 	if (!sfp) {
 		bsg_reply->reply_data.vendor_reply.vendor_rsp[0] =
 		    EXT_STATUS_NO_MEMORY;
@@ -1725,6 +1727,7 @@ qla2x00_read_i2c(struct bsg_job *bsg_job)
 	struct qla_i2c_access *i2c = (void *)bsg;
 	dma_addr_t sfp_dma;
 	uint8_t *sfp = dma_pool_alloc(ha->s_dma_pool, GFP_KERNEL, &sfp_dma);
+
 	if (!sfp) {
 		bsg_reply->reply_data.vendor_reply.vendor_rsp[0] =
 		    EXT_STATUS_NO_MEMORY;
@@ -1961,7 +1964,7 @@ qlafx00_mgmt_cmd(struct bsg_job *bsg_job)
 
 	/* Dump the vendor information */
 	ql_dump_buffer(ql_dbg_user + ql_dbg_verbose , vha, 0x70cf,
-	    (uint8_t *)piocb_rqst, sizeof(struct qla_mt_iocb_rqst_fx00));
+	    piocb_rqst, sizeof(*piocb_rqst));
 
 	if (!vha->flags.online) {
 		ql_log(ql_log_warn, vha, 0x70d0,
@@ -2157,7 +2160,7 @@ qla27xx_get_flash_upd_cap(struct bsg_job *bsg_job)
 	struct qla_hw_data *ha = vha->hw;
 	struct qla_flash_update_caps cap;
 
-	if (!(IS_QLA27XX(ha)))
+	if (!(IS_QLA27XX(ha)) && !IS_QLA28XX(ha))
 		return -EPERM;
 
 	memset(&cap, 0, sizeof(cap));
@@ -2190,7 +2193,7 @@ qla27xx_set_flash_upd_cap(struct bsg_job *bsg_job)
 	uint64_t online_fw_attr = 0;
 	struct qla_flash_update_caps cap;
 
-	if (!(IS_QLA27XX(ha)))
+	if (!IS_QLA27XX(ha) && !IS_QLA28XX(ha))
 		return -EPERM;
 
 	memset(&cap, 0, sizeof(cap));
@@ -2238,7 +2241,7 @@ qla27xx_get_bbcr_data(struct bsg_job *bsg_job)
 	uint8_t domain, area, al_pa, state;
 	int rval;
 
-	if (!(IS_QLA27XX(ha)))
+	if (!IS_QLA27XX(ha) && !IS_QLA28XX(ha))
 		return -EPERM;
 
 	memset(&bbcr, 0, sizeof(bbcr));
@@ -2323,8 +2326,8 @@ qla2x00_get_priv_stats(struct bsg_job *bsg_job)
 	rval = qla24xx_get_isp_stats(base_vha, stats, stats_dma, options);
 
 	if (rval == QLA_SUCCESS) {
-		ql_dump_buffer(ql_dbg_user + ql_dbg_verbose, vha, 0x70e3,
-		    (uint8_t *)stats, sizeof(*stats));
+		ql_dump_buffer(ql_dbg_user + ql_dbg_verbose, vha, 0x70e5,
+			stats, sizeof(*stats));
 		sg_copy_from_buffer(bsg_job->reply_payload.sg_list,
 			bsg_job->reply_payload.sg_cnt, stats, sizeof(*stats));
 	}
@@ -2353,7 +2356,8 @@ qla2x00_do_dport_diagnostics(struct bsg_job *bsg_job)
 	int rval;
 	struct qla_dport_diag *dd;
 
-	if (!IS_QLA83XX(vha->hw) && !IS_QLA27XX(vha->hw))
+	if (!IS_QLA83XX(vha->hw) && !IS_QLA27XX(vha->hw) &&
+	    !IS_QLA28XX(vha->hw))
 		return -EPERM;
 
 	dd = kmalloc(sizeof(*dd), GFP_KERNEL);
@@ -2388,6 +2392,45 @@ qla2x00_do_dport_diagnostics(struct bsg_job *bsg_job)
 }
 
 static int
+qla2x00_get_flash_image_status(struct bsg_job *bsg_job)
+{
+	scsi_qla_host_t *vha = shost_priv(fc_bsg_to_shost(bsg_job));
+	struct fc_bsg_reply *bsg_reply = bsg_job->reply;
+	struct qla_hw_data *ha = vha->hw;
+	struct qla_active_regions regions = { };
+	struct active_regions active_regions = { };
+
+	qla28xx_get_aux_images(vha, &active_regions);
+	regions.global_image = active_regions.global;
+
+	if (IS_QLA28XX(ha)) {
+		qla27xx_get_active_image(vha, &active_regions);
+		regions.board_config = active_regions.aux.board_config;
+		regions.vpd_nvram = active_regions.aux.vpd_nvram;
+		regions.npiv_config_0_1 = active_regions.aux.npiv_config_0_1;
+		regions.npiv_config_2_3 = active_regions.aux.npiv_config_2_3;
+	}
+
+	ql_dbg(ql_dbg_user, vha, 0x70e1,
+	    "%s(%lu): FW=%u BCFG=%u VPDNVR=%u NPIV01=%u NPIV02=%u\n",
+	    __func__, vha->host_no, regions.global_image,
+	    regions.board_config, regions.vpd_nvram,
+	    regions.npiv_config_0_1, regions.npiv_config_2_3);
+
+	sg_copy_from_buffer(bsg_job->reply_payload.sg_list,
+	    bsg_job->reply_payload.sg_cnt, &regions, sizeof(regions));
+
+	bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = EXT_STATUS_OK;
+	bsg_reply->reply_payload_rcv_len = sizeof(regions);
+	bsg_reply->result = DID_OK << 16;
+	bsg_job->reply_len = sizeof(struct fc_bsg_reply);
+	bsg_job_done(bsg_job, bsg_reply->result,
+	    bsg_reply->reply_payload_rcv_len);
+
+	return 0;
+}
+
+static int
 qla2x00_process_vendor_specific(struct bsg_job *bsg_job)
 {
 	struct fc_bsg_request *bsg_request = bsg_job->request;
@@ -2460,6 +2503,9 @@ qla2x00_process_vendor_specific(struct bsg_job *bsg_job)
 	case QL_VND_DPORT_DIAGNOSTICS:
 		return qla2x00_do_dport_diagnostics(bsg_job);
 
+	case QL_VND_SS_GET_FLASH_IMAGE_STATUS:
+		return qla2x00_get_flash_image_status(bsg_job);
+
 	default:
 		return -ENOSYS;
 	}
diff --git a/drivers/scsi/qla2xxx/qla_bsg.h b/drivers/scsi/qla2xxx/qla_bsg.h
index d97dfd521356..7594fad7b5b5 100644
--- a/drivers/scsi/qla2xxx/qla_bsg.h
+++ b/drivers/scsi/qla2xxx/qla_bsg.h
@@ -31,6 +31,7 @@
 #define QL_VND_GET_PRIV_STATS	0x18
 #define QL_VND_DPORT_DIAGNOSTICS	0x19
 #define QL_VND_GET_PRIV_STATS_EX	0x1A
+#define QL_VND_SS_GET_FLASH_IMAGE_STATUS	0x1E
 
 /* BSG Vendor specific subcode returns */
 #define EXT_STATUS_OK			0
@@ -279,4 +280,14 @@ struct qla_dport_diag {
 #define QLA_DPORT_RESULT	0x0
 #define QLA_DPORT_START		0x2
 
+/* active images in flash */
+struct qla_active_regions {
+	uint8_t global_image;
+	uint8_t board_config;
+	uint8_t vpd_nvram;
+	uint8_t npiv_config_0_1;
+	uint8_t npiv_config_2_3;
+	uint8_t reserved[32];
+} __packed;
+
 #endif
diff --git a/drivers/scsi/qla2xxx/qla_dbg.c b/drivers/scsi/qla2xxx/qla_dbg.c
index c7533fa7f46e..9e80646722e2 100644
--- a/drivers/scsi/qla2xxx/qla_dbg.c
+++ b/drivers/scsi/qla2xxx/qla_dbg.c
@@ -111,30 +111,25 @@ int
 qla27xx_dump_mpi_ram(struct qla_hw_data *ha, uint32_t addr, uint32_t *ram,
 	uint32_t ram_dwords, void **nxt)
 {
-	int rval;
-	uint32_t cnt, stat, timer, dwords, idx;
-	uint16_t mb0;
 	struct device_reg_24xx __iomem *reg = &ha->iobase->isp24;
 	dma_addr_t dump_dma = ha->gid_list_dma;
-	uint32_t *dump = (uint32_t *)ha->gid_list;
-
-	rval = QLA_SUCCESS;
-	mb0 = 0;
+	uint32_t *chunk = (void *)ha->gid_list;
+	uint32_t dwords = qla2x00_gid_list_size(ha) / 4;
+	uint32_t stat;
+	ulong i, j, timer = 6000000;
+	int rval = QLA_FUNCTION_FAILED;
 
-	WRT_REG_WORD(&reg->mailbox0, MBC_LOAD_DUMP_MPI_RAM);
 	clear_bit(MBX_INTERRUPT, &ha->mbx_cmd_flags);
+	for (i = 0; i < ram_dwords; i += dwords, addr += dwords) {
+		if (i + dwords > ram_dwords)
+			dwords = ram_dwords - i;
 
-	dwords = qla2x00_gid_list_size(ha) / 4;
-	for (cnt = 0; cnt < ram_dwords && rval == QLA_SUCCESS;
-	    cnt += dwords, addr += dwords) {
-		if (cnt + dwords > ram_dwords)
-			dwords = ram_dwords - cnt;
-
+		WRT_REG_WORD(&reg->mailbox0, MBC_LOAD_DUMP_MPI_RAM);
 		WRT_REG_WORD(&reg->mailbox1, LSW(addr));
 		WRT_REG_WORD(&reg->mailbox8, MSW(addr));
 
-		WRT_REG_WORD(&reg->mailbox2, MSW(dump_dma));
-		WRT_REG_WORD(&reg->mailbox3, LSW(dump_dma));
+		WRT_REG_WORD(&reg->mailbox2, MSW(LSD(dump_dma)));
+		WRT_REG_WORD(&reg->mailbox3, LSW(LSD(dump_dma)));
 		WRT_REG_WORD(&reg->mailbox6, MSW(MSD(dump_dma)));
 		WRT_REG_WORD(&reg->mailbox7, LSW(MSD(dump_dma)));
 
@@ -145,76 +140,76 @@ qla27xx_dump_mpi_ram(struct qla_hw_data *ha, uint32_t addr, uint32_t *ram,
 		WRT_REG_DWORD(&reg->hccr, HCCRX_SET_HOST_INT);
 
 		ha->flags.mbox_int = 0;
-		for (timer = 6000000; timer; timer--) {
-			/* Check for pending interrupts. */
-			stat = RD_REG_DWORD(&reg->host_status);
-			if (stat & HSRX_RISC_INT) {
-				stat &= 0xff;
-
-				if (stat == 0x1 || stat == 0x2 ||
-				    stat == 0x10 || stat == 0x11) {
-					set_bit(MBX_INTERRUPT,
-					    &ha->mbx_cmd_flags);
+		while (timer--) {
+			udelay(5);
 
-					mb0 = RD_REG_WORD(&reg->mailbox0);
-					RD_REG_WORD(&reg->mailbox1);
+			stat = RD_REG_DWORD(&reg->host_status);
+			/* Check for pending interrupts. */
+			if (!(stat & HSRX_RISC_INT))
+				continue;
 
-					WRT_REG_DWORD(&reg->hccr,
-					    HCCRX_CLR_RISC_INT);
-					RD_REG_DWORD(&reg->hccr);
-					break;
-				}
+			stat &= 0xff;
+			if (stat != 0x1 && stat != 0x2 &&
+			    stat != 0x10 && stat != 0x11) {
 
 				/* Clear this intr; it wasn't a mailbox intr */
 				WRT_REG_DWORD(&reg->hccr, HCCRX_CLR_RISC_INT);
 				RD_REG_DWORD(&reg->hccr);
+				continue;
 			}
-			udelay(5);
+
+			set_bit(MBX_INTERRUPT, &ha->mbx_cmd_flags);
+			rval = RD_REG_WORD(&reg->mailbox0) & MBS_MASK;
+			WRT_REG_DWORD(&reg->hccr, HCCRX_CLR_RISC_INT);
+			RD_REG_DWORD(&reg->hccr);
+			break;
 		}
 		ha->flags.mbox_int = 1;
+		*nxt = ram + i;
 
-		if (test_and_clear_bit(MBX_INTERRUPT, &ha->mbx_cmd_flags)) {
-			rval = mb0 & MBS_MASK;
-			for (idx = 0; idx < dwords; idx++)
-				ram[cnt + idx] = IS_QLA27XX(ha) ?
-				    le32_to_cpu(dump[idx]) : swab32(dump[idx]);
-		} else {
-			rval = QLA_FUNCTION_FAILED;
+		if (!test_and_clear_bit(MBX_INTERRUPT, &ha->mbx_cmd_flags)) {
+			/* no interrupt, timed out*/
+			return rval;
+		}
+		if (rval) {
+			/* error completion status */
+			return rval;
+		}
+		for (j = 0; j < dwords; j++) {
+			ram[i + j] =
+			    (IS_QLA27XX(ha) || IS_QLA28XX(ha)) ?
+			    chunk[j] : swab32(chunk[j]);
 		}
 	}
 
-	*nxt = rval == QLA_SUCCESS ? &ram[cnt] : NULL;
-	return rval;
+	*nxt = ram + i;
+	return QLA_SUCCESS;
 }
 
 int
 qla24xx_dump_ram(struct qla_hw_data *ha, uint32_t addr, uint32_t *ram,
     uint32_t ram_dwords, void **nxt)
 {
-	int rval;
-	uint32_t cnt, stat, timer, dwords, idx;
-	uint16_t mb0;
+	int rval = QLA_FUNCTION_FAILED;
 	struct device_reg_24xx __iomem *reg = &ha->iobase->isp24;
 	dma_addr_t dump_dma = ha->gid_list_dma;
-	uint32_t *dump = (uint32_t *)ha->gid_list;
+	uint32_t *chunk = (void *)ha->gid_list;
+	uint32_t dwords = qla2x00_gid_list_size(ha) / 4;
+	uint32_t stat;
+	ulong i, j, timer = 6000000;
 
-	rval = QLA_SUCCESS;
-	mb0 = 0;
-
-	WRT_REG_WORD(&reg->mailbox0, MBC_DUMP_RISC_RAM_EXTENDED);
 	clear_bit(MBX_INTERRUPT, &ha->mbx_cmd_flags);
 
-	dwords = qla2x00_gid_list_size(ha) / 4;
-	for (cnt = 0; cnt < ram_dwords && rval == QLA_SUCCESS;
-	    cnt += dwords, addr += dwords) {
-		if (cnt + dwords > ram_dwords)
-			dwords = ram_dwords - cnt;
+	for (i = 0; i < ram_dwords; i += dwords, addr += dwords) {
+		if (i + dwords > ram_dwords)
+			dwords = ram_dwords - i;
 
+		WRT_REG_WORD(&reg->mailbox0, MBC_DUMP_RISC_RAM_EXTENDED);
 		WRT_REG_WORD(&reg->mailbox1, LSW(addr));
 		WRT_REG_WORD(&reg->mailbox8, MSW(addr));
 
-		WRT_REG_WORD(&reg->mailbox2, MSW(dump_dma));
-		WRT_REG_WORD(&reg->mailbox3, LSW(dump_dma));
+		WRT_REG_WORD(&reg->mailbox2, MSW(LSD(dump_dma)));
+		WRT_REG_WORD(&reg->mailbox3, LSW(LSD(dump_dma)));
 		WRT_REG_WORD(&reg->mailbox6, MSW(MSD(dump_dma)));
 		WRT_REG_WORD(&reg->mailbox7, LSW(MSD(dump_dma)));
 
@@ -223,45 +218,48 @@ qla24xx_dump_ram(struct qla_hw_data *ha, uint32_t addr, uint32_t *ram,
 		WRT_REG_DWORD(&reg->hccr, HCCRX_SET_HOST_INT);
 
 		ha->flags.mbox_int = 0;
-		for (timer = 6000000; timer; timer--) {
-			/* Check for pending interrupts. */
+		while (timer--) {
+			udelay(5);
 			stat = RD_REG_DWORD(&reg->host_status);
-			if (stat & HSRX_RISC_INT) {
-				stat &= 0xff;
 
-				if (stat == 0x1 || stat == 0x2 ||
-				    stat == 0x10 || stat == 0x11) {
-					set_bit(MBX_INTERRUPT,
-					    &ha->mbx_cmd_flags);
-
-					mb0 = RD_REG_WORD(&reg->mailbox0);
-
-					WRT_REG_DWORD(&reg->hccr,
-					    HCCRX_CLR_RISC_INT);
-					RD_REG_DWORD(&reg->hccr);
-					break;
-				}
+			/* Check for pending interrupts. */
+			if (!(stat & HSRX_RISC_INT))
+				continue;
 
-				/* Clear this intr; it wasn't a mailbox intr */
+			stat &= 0xff;
+			if (stat != 0x1 && stat != 0x2 &&
+			    stat != 0x10 && stat != 0x11) {
 				WRT_REG_DWORD(&reg->hccr, HCCRX_CLR_RISC_INT);
 				RD_REG_DWORD(&reg->hccr);
+				continue;
 			}
-			udelay(5);
+
+			set_bit(MBX_INTERRUPT, &ha->mbx_cmd_flags);
+			rval = RD_REG_WORD(&reg->mailbox0) & MBS_MASK;
+			WRT_REG_DWORD(&reg->hccr, HCCRX_CLR_RISC_INT);
+			RD_REG_DWORD(&reg->hccr);
+			break;
 		}
 		ha->flags.mbox_int = 1;
+		*nxt = ram + i;
 
-		if (test_and_clear_bit(MBX_INTERRUPT, &ha->mbx_cmd_flags)) {
-			rval = mb0 & MBS_MASK;
-			for (idx = 0; idx < dwords; idx++)
-				ram[cnt + idx] = IS_QLA27XX(ha) ?
-				    le32_to_cpu(dump[idx]) : swab32(dump[idx]);
-		} else {
-			rval = QLA_FUNCTION_FAILED;
+		if (!test_and_clear_bit(MBX_INTERRUPT, &ha->mbx_cmd_flags)) {
+			/* no interrupt, timed out*/
+			return rval;
+		}
+		if (rval) {
+			/* error completion status */
+			return rval;
+		}
+		for (j = 0; j < dwords; j++) {
+			ram[i + j] =
+			    (IS_QLA27XX(ha) || IS_QLA28XX(ha)) ?
+			    chunk[j] : swab32(chunk[j]);
 		}
 	}
 
-	*nxt = rval == QLA_SUCCESS ? &ram[cnt]: NULL;
-	return rval;
+	*nxt = ram + i;
+	return QLA_SUCCESS;
 }
 
 static int
@@ -447,7 +445,7 @@ qla2xxx_dump_ram(struct qla_hw_data *ha, uint32_t addr, uint16_t *ram,
 		}
 	}
 
-	*nxt = rval == QLA_SUCCESS ? &ram[cnt]: NULL;
+	*nxt = rval == QLA_SUCCESS ? &ram[cnt] : NULL;
 	return rval;
 }
 
@@ -669,7 +667,8 @@ qla25xx_copy_mq(struct qla_hw_data *ha, void *ptr, uint32_t **last_chain)
 	struct qla2xxx_mq_chain *mq = ptr;
 	device_reg_t *reg;
 
-	if (!ha->mqenable || IS_QLA83XX(ha) || IS_QLA27XX(ha))
+	if (!ha->mqenable || IS_QLA83XX(ha) || IS_QLA27XX(ha) ||
+	    IS_QLA28XX(ha))
 		return ptr;
 
 	mq = ptr;
@@ -2521,7 +2520,7 @@ qla83xx_fw_dump_failed:
 /****************************************************************************/
 
 static inline int
-ql_mask_match(uint32_t level)
+ql_mask_match(uint level)
 {
 	return (level & ql2xextended_error_logging) == level;
 }
@@ -2540,7 +2539,7 @@ ql_mask_match(uint32_t level)
  * msg:   The message to be displayed.
  */
 void
-ql_dbg(uint32_t level, scsi_qla_host_t *vha, int32_t id, const char *fmt, ...)
+ql_dbg(uint level, scsi_qla_host_t *vha, uint id, const char *fmt, ...)
 {
 	va_list va;
 	struct va_format vaf;
@@ -2583,8 +2582,7 @@ ql_dbg(uint32_t level, scsi_qla_host_t *vha, int32_t id, const char *fmt, ...)
  * msg:   The message to be displayed.
  */
 void
-ql_dbg_pci(uint32_t level, struct pci_dev *pdev, int32_t id,
-	   const char *fmt, ...)
+ql_dbg_pci(uint level, struct pci_dev *pdev, uint id, const char *fmt, ...)
 {
 	va_list va;
 	struct va_format vaf;
@@ -2620,7 +2618,7 @@ ql_dbg_pci(uint32_t level, struct pci_dev *pdev, int32_t id,
  * msg:   The message to be displayed.
  */
 void
-ql_log(uint32_t level, scsi_qla_host_t *vha, int32_t id, const char *fmt, ...)
+ql_log(uint level, scsi_qla_host_t *vha, uint id, const char *fmt, ...)
 {
 	va_list va;
 	struct va_format vaf;
@@ -2678,8 +2676,7 @@ ql_log(uint32_t level, scsi_qla_host_t *vha, int32_t id, const char *fmt, ...)
  * msg:   The message to be displayed.
  */
 void
-ql_log_pci(uint32_t level, struct pci_dev *pdev, int32_t id,
-	   const char *fmt, ...)
+ql_log_pci(uint level, struct pci_dev *pdev, uint id, const char *fmt, ...)
 {
 	va_list va;
 	struct va_format vaf;
@@ -2719,7 +2716,7 @@ ql_log_pci(uint32_t level, struct pci_dev *pdev, int32_t id,
 }
 
 void
-ql_dump_regs(uint32_t level, scsi_qla_host_t *vha, int32_t id)
+ql_dump_regs(uint level, scsi_qla_host_t *vha, uint id)
 {
 	int i;
 	struct qla_hw_data *ha = vha->hw;
@@ -2741,13 +2738,12 @@ ql_dump_regs(uint32_t level, scsi_qla_host_t *vha, int32_t id)
 	ql_dbg(level, vha, id, "Mailbox registers:\n");
 	for (i = 0; i < 6; i++, mbx_reg++)
 		ql_dbg(level, vha, id,
-		    "mbox[%d] 0x%04x\n", i, RD_REG_WORD(mbx_reg));
+		    "mbox[%d] %#04x\n", i, RD_REG_WORD(mbx_reg));
 }
 
 
 void
-ql_dump_buffer(uint32_t level, scsi_qla_host_t *vha, int32_t id,
-	uint8_t *buf, uint size)
+ql_dump_buffer(uint level, scsi_qla_host_t *vha, uint id, void *buf, uint size)
 {
 	uint cnt;
 
diff --git a/drivers/scsi/qla2xxx/qla_dbg.h b/drivers/scsi/qla2xxx/qla_dbg.h
index 8877aa97d829..bb01b680ce9f 100644
--- a/drivers/scsi/qla2xxx/qla_dbg.h
+++ b/drivers/scsi/qla2xxx/qla_dbg.h
@@ -318,20 +318,20 @@ struct qla2xxx_fw_dump {
 				   * as compared to other log levels.
 				   */
 
-extern int ql_errlev;
+extern uint ql_errlev;
 
 void __attribute__((format (printf, 4, 5)))
-ql_dbg(uint32_t, scsi_qla_host_t *vha, int32_t, const char *fmt, ...);
+ql_dbg(uint, scsi_qla_host_t *vha, uint, const char *fmt, ...);
 void __attribute__((format (printf, 4, 5)))
-ql_dbg_pci(uint32_t, struct pci_dev *pdev, int32_t, const char *fmt, ...);
+ql_dbg_pci(uint, struct pci_dev *pdev, uint, const char *fmt, ...);
 void __attribute__((format (printf, 4, 5)))
 ql_dbg_qp(uint32_t, struct qla_qpair *, int32_t, const char *fmt, ...);
 
 
 void __attribute__((format (printf, 4, 5)))
-ql_log(uint32_t, scsi_qla_host_t *vha, int32_t, const char *fmt, ...);
+ql_log(uint, scsi_qla_host_t *vha, uint, const char *fmt, ...);
 void __attribute__((format (printf, 4, 5)))
-ql_log_pci(uint32_t, struct pci_dev *pdev, int32_t, const char *fmt, ...);
+ql_log_pci(uint, struct pci_dev *pdev, uint, const char *fmt, ...);
 
 void __attribute__((format (printf, 4, 5)))
 ql_log_qp(uint32_t, struct qla_qpair *, int32_t, const char *fmt, ...);
diff --git a/drivers/scsi/qla2xxx/qla_def.h b/drivers/scsi/qla2xxx/qla_def.h
index 3d46975a5e5c..1a4095c56eee 100644
--- a/drivers/scsi/qla2xxx/qla_def.h
+++ b/drivers/scsi/qla2xxx/qla_def.h
@@ -35,6 +35,7 @@
 #include <scsi/scsi_bsg_fc.h>
 
 #include "qla_bsg.h"
+#include "qla_dsd.h"
 #include "qla_nx.h"
 #include "qla_nx2.h"
 #include "qla_nvme.h"
@@ -545,7 +546,7 @@ typedef struct srb {
 	u32 gen2;	/* scratch */
 	int rc;
 	int retry_count;
-	struct completion comp;
+	struct completion *comp;
 	union {
 		struct srb_iocb iocb_cmd;
 		struct bsg_job *bsg_job;
@@ -1033,6 +1034,7 @@ struct mbx_cmd_32 {
 #define MBC_GET_FIRMWARE_VERSION	8	/* Get firmware revision. */
 #define MBC_LOAD_RISC_RAM		9	/* Load RAM command. */
 #define MBC_DUMP_RISC_RAM		0xa	/* Dump RAM command. */
+#define MBC_SECURE_FLASH_UPDATE		0xa	/* Secure Flash Update(28xx) */
 #define MBC_LOAD_RISC_RAM_EXTENDED	0xb	/* Load RAM extended. */
 #define MBC_DUMP_RISC_RAM_EXTENDED	0xc	/* Dump RAM extended. */
 #define MBC_WRITE_RAM_WORD_EXTENDED	0xd	/* Write RAM word extended */
@@ -1203,6 +1205,10 @@ struct mbx_cmd_32 {
 #define QLA27XX_IMG_STATUS_VER_MAJOR   0x01
 #define QLA27XX_IMG_STATUS_VER_MINOR    0x00
 #define QLA27XX_IMG_STATUS_SIGN   0xFACEFADE
+#define QLA28XX_IMG_STATUS_SIGN    0xFACEFADF
+#define QLA28XX_IMG_STATUS_SIGN		0xFACEFADF
+#define QLA28XX_AUX_IMG_STATUS_SIGN	0xFACEFAED
+#define QLA27XX_DEFAULT_IMAGE		0
 #define QLA27XX_PRIMARY_IMAGE  1
 #define QLA27XX_SECONDARY_IMAGE    2
 
@@ -1323,8 +1329,8 @@ typedef struct {
 	uint16_t response_q_inpointer;
 	uint16_t request_q_length;
 	uint16_t response_q_length;
-	uint32_t request_q_address[2];
-	uint32_t response_q_address[2];
+	__le64   request_q_address __packed;
+	__le64   response_q_address __packed;
 
 	uint16_t lun_enables;
 	uint8_t  command_resource_count;
@@ -1749,12 +1755,10 @@ typedef struct {
 	uint16_t dseg_count;		/* Data segment count. */
 	uint8_t scsi_cdb[MAX_CMDSZ]; 	/* SCSI command words. */
 	uint32_t byte_count;		/* Total byte count. */
-	uint32_t dseg_0_address;	/* Data segment 0 address. */
-	uint32_t dseg_0_length;		/* Data segment 0 length. */
-	uint32_t dseg_1_address;	/* Data segment 1 address. */
-	uint32_t dseg_1_length;		/* Data segment 1 length. */
-	uint32_t dseg_2_address;	/* Data segment 2 address. */
-	uint32_t dseg_2_length;		/* Data segment 2 length. */
+	union {
+		struct dsd32 dsd32[3];
+		struct dsd64 dsd64[2];
+	};
 } cmd_entry_t;
 
 /*
@@ -1775,10 +1779,7 @@ typedef struct {
 	uint16_t dseg_count;		/* Data segment count. */
 	uint8_t scsi_cdb[MAX_CMDSZ];	/* SCSI command words. */
 	uint32_t byte_count;		/* Total byte count. */
-	uint32_t dseg_0_address[2];	/* Data segment 0 address. */
-	uint32_t dseg_0_length;		/* Data segment 0 length. */
-	uint32_t dseg_1_address[2];	/* Data segment 1 address. */
-	uint32_t dseg_1_length;		/* Data segment 1 length. */
+	struct dsd64 dsd[2];
 } cmd_a64_entry_t, request_t;
 
 /*
@@ -1791,20 +1792,7 @@ typedef struct {
 	uint8_t sys_define;		/* System defined. */
 	uint8_t entry_status;		/* Entry Status. */
 	uint32_t reserved;
-	uint32_t dseg_0_address;	/* Data segment 0 address. */
-	uint32_t dseg_0_length;		/* Data segment 0 length. */
-	uint32_t dseg_1_address;	/* Data segment 1 address. */
-	uint32_t dseg_1_length;		/* Data segment 1 length. */
-	uint32_t dseg_2_address;	/* Data segment 2 address. */
-	uint32_t dseg_2_length;		/* Data segment 2 length. */
-	uint32_t dseg_3_address;	/* Data segment 3 address. */
-	uint32_t dseg_3_length;		/* Data segment 3 length. */
-	uint32_t dseg_4_address;	/* Data segment 4 address. */
-	uint32_t dseg_4_length;		/* Data segment 4 length. */
-	uint32_t dseg_5_address;	/* Data segment 5 address. */
-	uint32_t dseg_5_length;		/* Data segment 5 length. */
-	uint32_t dseg_6_address;	/* Data segment 6 address. */
-	uint32_t dseg_6_length;		/* Data segment 6 length. */
+	struct dsd32 dsd[7];
 } cont_entry_t;
 
 /*
@@ -1816,16 +1804,7 @@ typedef struct {
 	uint8_t entry_count;		/* Entry count. */
 	uint8_t sys_define;		/* System defined. */
 	uint8_t entry_status;		/* Entry Status. */
-	uint32_t dseg_0_address[2];	/* Data segment 0 address. */
-	uint32_t dseg_0_length;		/* Data segment 0 length. */
-	uint32_t dseg_1_address[2];	/* Data segment 1 address. */
-	uint32_t dseg_1_length;		/* Data segment 1 length. */
-	uint32_t dseg_2_address	[2];	/* Data segment 2 address. */
-	uint32_t dseg_2_length;		/* Data segment 2 length. */
-	uint32_t dseg_3_address[2];	/* Data segment 3 address. */
-	uint32_t dseg_3_length;		/* Data segment 3 length. */
-	uint32_t dseg_4_address[2];	/* Data segment 4 address. */
-	uint32_t dseg_4_length;		/* Data segment 4 length. */
+	struct dsd64 dsd[5];
 } cont_a64_entry_t;
 
 #define PO_MODE_DIF_INSERT	0
@@ -1869,8 +1848,7 @@ struct crc_context {
 			uint16_t	reserved_2;
 			uint16_t	reserved_3;
 			uint32_t	reserved_4;
-			uint32_t	data_address[2];
-			uint32_t	data_length;
+			struct dsd64	data_dsd;
 			uint32_t	reserved_5[2];
 			uint32_t	reserved_6;
 		} nobundling;
@@ -1880,11 +1858,8 @@ struct crc_context {
 			uint16_t	reserved_1;
 			__le16	dseg_count;	/* Data segment count */
 			uint32_t	reserved_2;
-			uint32_t	data_address[2];
-			uint32_t	data_length;
-			uint32_t	dif_address[2];
-			uint32_t	dif_length;	/* Data segment 0
-							 * length */
+			struct dsd64	data_dsd;
+			struct dsd64	dif_dsd;
 		} bundling;
 	} u;
 
@@ -2083,10 +2058,8 @@ typedef struct {
 	uint32_t handle2;
 	uint32_t rsp_bytecount;
 	uint32_t req_bytecount;
-	uint32_t dseg_req_address[2];	/* Data segment 0 address. */
-	uint32_t dseg_req_length;	/* Data segment 0 length. */
-	uint32_t dseg_rsp_address[2];	/* Data segment 1 address. */
-	uint32_t dseg_rsp_length;	/* Data segment 1 length. */
+	struct dsd64 req_dsd;
+	struct dsd64 rsp_dsd;
 } ms_iocb_entry_t;
 
 
@@ -2258,7 +2231,10 @@ typedef enum {
 	FCT_BROADCAST,
 	FCT_INITIATOR,
 	FCT_TARGET,
-	FCT_NVME
+	FCT_NVME_INITIATOR = 0x10,
+	FCT_NVME_TARGET = 0x20,
+	FCT_NVME_DISCOVERY = 0x40,
+	FCT_NVME = 0xf0,
 } fc_port_type_t;
 
 enum qla_sess_deletion {
@@ -2463,13 +2439,7 @@ struct event_arg {
 #define FCS_DEVICE_LOST		3
 #define FCS_ONLINE		4
 
-static const char * const port_state_str[] = {
-	"Unknown",
-	"UNCONFIGURED",
-	"DEAD",
-	"LOST",
-	"ONLINE"
-};
+extern const char *const port_state_str[5];
 
 /*
  * FC port flags.
@@ -2672,6 +2642,7 @@ struct ct_fdmiv2_hba_attributes {
 #define FDMI_PORT_SPEED_8GB		0x10
 #define FDMI_PORT_SPEED_16GB		0x20
 #define FDMI_PORT_SPEED_32GB		0x40
+#define FDMI_PORT_SPEED_64GB		0x80
 #define FDMI_PORT_SPEED_UNKNOWN		0x8000
 
 #define FC_CLASS_2	0x04
@@ -3060,7 +3031,7 @@ struct sns_cmd_pkt {
 		struct {
 			uint16_t buffer_length;
 			uint16_t reserved_1;
-			uint32_t buffer_address[2];
+			__le64	 buffer_address __packed;
 			uint16_t subcommand_length;
 			uint16_t reserved_2;
 			uint16_t subcommand;
@@ -3130,10 +3101,10 @@ struct rsp_que;
 struct isp_operations {
 
 	int (*pci_config) (struct scsi_qla_host *);
-	void (*reset_chip) (struct scsi_qla_host *);
+	int (*reset_chip)(struct scsi_qla_host *);
 	int (*chip_diag) (struct scsi_qla_host *);
 	void (*config_rings) (struct scsi_qla_host *);
-	void (*reset_adapter) (struct scsi_qla_host *);
+	int (*reset_adapter)(struct scsi_qla_host *);
 	int (*nvram_config) (struct scsi_qla_host *);
 	void (*update_fw_options) (struct scsi_qla_host *);
 	int (*load_risc) (struct scsi_qla_host *, uint32_t *);
@@ -3159,9 +3130,9 @@ struct isp_operations {
 	void *(*prep_ms_fdmi_iocb) (struct scsi_qla_host *, uint32_t,
 	    uint32_t);
 
-	uint8_t *(*read_nvram) (struct scsi_qla_host *, uint8_t *,
+	uint8_t *(*read_nvram)(struct scsi_qla_host *, void *,
 		uint32_t, uint32_t);
-	int (*write_nvram) (struct scsi_qla_host *, uint8_t *, uint32_t,
+	int (*write_nvram)(struct scsi_qla_host *, void *, uint32_t,
 		uint32_t);
 
 	void (*fw_dump) (struct scsi_qla_host *, int);
@@ -3170,16 +3141,16 @@ struct isp_operations {
 	int (*beacon_off) (struct scsi_qla_host *);
 	void (*beacon_blink) (struct scsi_qla_host *);
 
-	uint8_t * (*read_optrom) (struct scsi_qla_host *, uint8_t *,
+	void *(*read_optrom)(struct scsi_qla_host *, void *,
 		uint32_t, uint32_t);
-	int (*write_optrom) (struct scsi_qla_host *, uint8_t *, uint32_t,
+	int (*write_optrom)(struct scsi_qla_host *, void *, uint32_t,
 		uint32_t);
 
 	int (*get_flash_version) (struct scsi_qla_host *, void *);
 	int (*start_scsi) (srb_t *);
 	int (*start_scsi_mq) (srb_t *);
 	int (*abort_isp) (struct scsi_qla_host *);
-	int (*iospace_config)(struct qla_hw_data*);
+	int (*iospace_config)(struct qla_hw_data *);
 	int (*initialize_adapter)(struct scsi_qla_host *);
 };
 
@@ -3368,7 +3339,8 @@ struct qla_tc_param {
 #define QLA_MQ_SIZE 32
 #define QLA_MAX_QUEUES 256
 #define ISP_QUE_REG(ha, id) \
-	((ha->mqenable || IS_QLA83XX(ha) || IS_QLA27XX(ha)) ? \
+	((ha->mqenable || IS_QLA83XX(ha) || \
+	  IS_QLA27XX(ha) || IS_QLA28XX(ha)) ? \
 	 ((void __iomem *)ha->mqiobase + (QLA_QUE_PAGE * id)) :\
 	 ((void __iomem *)ha->iobase))
 #define QLA_REQ_QUE_ID(tag) \
@@ -3621,6 +3593,8 @@ struct qla_hw_data {
 		uint32_t	rida_fmt2:1;
 		uint32_t	purge_mbox:1;
 		uint32_t        n2n_bigger:1;
+		uint32_t	secure_adapter:1;
+		uint32_t	secure_fw:1;
 	} flags;
 
 	uint16_t max_exchg;
@@ -3703,6 +3677,7 @@ struct qla_hw_data {
 #define PORT_SPEED_8GB  0x04
 #define PORT_SPEED_16GB 0x05
 #define PORT_SPEED_32GB 0x06
+#define PORT_SPEED_64GB 0x07
 #define PORT_SPEED_10GB	0x13
 	uint16_t	link_data_rate;         /* F/W operating speed */
 	uint16_t	set_data_rate;		/* Set by user */
@@ -3729,6 +3704,11 @@ struct qla_hw_data {
 #define PCI_DEVICE_ID_QLOGIC_ISP2071	0x2071
 #define PCI_DEVICE_ID_QLOGIC_ISP2271	0x2271
 #define PCI_DEVICE_ID_QLOGIC_ISP2261	0x2261
+#define PCI_DEVICE_ID_QLOGIC_ISP2061	0x2061
+#define PCI_DEVICE_ID_QLOGIC_ISP2081	0x2081
+#define PCI_DEVICE_ID_QLOGIC_ISP2089	0x2089
+#define PCI_DEVICE_ID_QLOGIC_ISP2281	0x2281
+#define PCI_DEVICE_ID_QLOGIC_ISP2289	0x2289
 
 	uint32_t	isp_type;
 #define DT_ISP2100                      BIT_0
@@ -3753,7 +3733,12 @@ struct qla_hw_data {
 #define DT_ISP2071			BIT_19
 #define DT_ISP2271			BIT_20
 #define DT_ISP2261			BIT_21
-#define DT_ISP_LAST			(DT_ISP2261 << 1)
+#define DT_ISP2061			BIT_22
+#define DT_ISP2081			BIT_23
+#define DT_ISP2089			BIT_24
+#define DT_ISP2281			BIT_25
+#define DT_ISP2289			BIT_26
+#define DT_ISP_LAST			(DT_ISP2289 << 1)
 
 	uint32_t	device_type;
 #define DT_T10_PI                       BIT_25
@@ -3788,6 +3773,8 @@ struct qla_hw_data {
 #define IS_QLA2071(ha)	(DT_MASK(ha) & DT_ISP2071)
 #define IS_QLA2271(ha)	(DT_MASK(ha) & DT_ISP2271)
 #define IS_QLA2261(ha)	(DT_MASK(ha) & DT_ISP2261)
+#define IS_QLA2081(ha)	(DT_MASK(ha) & DT_ISP2081)
+#define IS_QLA2281(ha)	(DT_MASK(ha) & DT_ISP2281)
 
 #define IS_QLA23XX(ha)  (IS_QLA2300(ha) || IS_QLA2312(ha) || IS_QLA2322(ha) || \
 			IS_QLA6312(ha) || IS_QLA6322(ha))
@@ -3797,6 +3784,7 @@ struct qla_hw_data {
 #define IS_QLA83XX(ha)	(IS_QLA2031(ha) || IS_QLA8031(ha))
 #define IS_QLA84XX(ha)  (IS_QLA8432(ha))
 #define IS_QLA27XX(ha)  (IS_QLA2071(ha) || IS_QLA2271(ha) || IS_QLA2261(ha))
+#define IS_QLA28XX(ha)	(IS_QLA2081(ha) || IS_QLA2281(ha))
 #define IS_QLA24XX_TYPE(ha)     (IS_QLA24XX(ha) || IS_QLA54XX(ha) || \
 				IS_QLA84XX(ha))
 #define IS_CNA_CAPABLE(ha)	(IS_QLA81XX(ha) || IS_QLA82XX(ha) || \
@@ -3805,14 +3793,15 @@ struct qla_hw_data {
 #define IS_QLA2XXX_MIDTYPE(ha)	(IS_QLA24XX(ha) || IS_QLA84XX(ha) || \
 				IS_QLA25XX(ha) || IS_QLA81XX(ha) || \
 				IS_QLA82XX(ha) || IS_QLA83XX(ha) || \
-				IS_QLA8044(ha) || IS_QLA27XX(ha))
+				IS_QLA8044(ha) || IS_QLA27XX(ha) || \
+				IS_QLA28XX(ha))
 #define IS_MSIX_NACK_CAPABLE(ha) (IS_QLA81XX(ha) || IS_QLA83XX(ha) || \
-				IS_QLA27XX(ha))
+				IS_QLA27XX(ha) || IS_QLA28XX(ha))
 #define IS_NOPOLLING_TYPE(ha)	(IS_QLA81XX(ha) && (ha)->flags.msix_enabled)
 #define IS_FAC_REQUIRED(ha)	(IS_QLA81XX(ha) || IS_QLA83XX(ha) || \
-				IS_QLA27XX(ha))
+				IS_QLA27XX(ha) || IS_QLA28XX(ha))
 #define IS_NOCACHE_VPD_TYPE(ha)	(IS_QLA81XX(ha) || IS_QLA83XX(ha) || \
-				IS_QLA27XX(ha))
+				IS_QLA27XX(ha) || IS_QLA28XX(ha))
 #define IS_ALOGIO_CAPABLE(ha)	(IS_QLA23XX(ha) || IS_FWI2_CAPABLE(ha))
 
 #define IS_T10_PI_CAPABLE(ha)   ((ha)->device_type & DT_T10_PI)
@@ -3823,28 +3812,34 @@ struct qla_hw_data {
 #define HAS_EXTENDED_IDS(ha)    ((ha)->device_type & DT_EXTENDED_IDS)
 #define IS_CT6_SUPPORTED(ha)	((ha)->device_type & DT_CT6_SUPPORTED)
 #define IS_MQUE_CAPABLE(ha)	((ha)->mqenable || IS_QLA83XX(ha) || \
-				IS_QLA27XX(ha))
-#define IS_BIDI_CAPABLE(ha)	((IS_QLA25XX(ha) || IS_QLA2031(ha)))
+				IS_QLA27XX(ha) || IS_QLA28XX(ha))
+#define IS_BIDI_CAPABLE(ha) \
+    (IS_QLA25XX(ha) || IS_QLA2031(ha) || IS_QLA27XX(ha) || IS_QLA28XX(ha))
 /* Bit 21 of fw_attributes decides the MCTP capabilities */
 #define IS_MCTP_CAPABLE(ha)	(IS_QLA2031(ha) && \
 				((ha)->fw_attributes_ext[0] & BIT_0))
 #define IS_PI_UNINIT_CAPABLE(ha)	(IS_QLA83XX(ha) || IS_QLA27XX(ha))
 #define IS_PI_IPGUARD_CAPABLE(ha)	(IS_QLA83XX(ha) || IS_QLA27XX(ha))
 #define IS_PI_DIFB_DIX0_CAPABLE(ha)	(0)
-#define IS_PI_SPLIT_DET_CAPABLE_HBA(ha)	(IS_QLA83XX(ha) || IS_QLA27XX(ha))
+#define IS_PI_SPLIT_DET_CAPABLE_HBA(ha)	(IS_QLA83XX(ha) || IS_QLA27XX(ha) || \
+					IS_QLA28XX(ha))
 #define IS_PI_SPLIT_DET_CAPABLE(ha)	(IS_PI_SPLIT_DET_CAPABLE_HBA(ha) && \
     (((ha)->fw_attributes_h << 16 | (ha)->fw_attributes) & BIT_22))
-#define IS_ATIO_MSIX_CAPABLE(ha) (IS_QLA83XX(ha) || IS_QLA27XX(ha))
+#define IS_ATIO_MSIX_CAPABLE(ha) (IS_QLA83XX(ha) || IS_QLA27XX(ha) || \
+				IS_QLA28XX(ha))
 #define IS_TGT_MODE_CAPABLE(ha)	(ha->tgt.atio_q_length)
-#define IS_SHADOW_REG_CAPABLE(ha)  (IS_QLA27XX(ha))
-#define IS_DPORT_CAPABLE(ha)  (IS_QLA83XX(ha) || IS_QLA27XX(ha))
-#define IS_FAWWN_CAPABLE(ha)	(IS_QLA83XX(ha) || IS_QLA27XX(ha))
+#define IS_SHADOW_REG_CAPABLE(ha)  (IS_QLA27XX(ha) || IS_QLA28XX(ha))
+#define IS_DPORT_CAPABLE(ha)  (IS_QLA83XX(ha) || IS_QLA27XX(ha) || \
+				IS_QLA28XX(ha))
+#define IS_FAWWN_CAPABLE(ha)	(IS_QLA83XX(ha) || IS_QLA27XX(ha) || \
+				IS_QLA28XX(ha))
 #define IS_EXCHG_OFFLD_CAPABLE(ha) \
-	(IS_QLA81XX(ha) || IS_QLA83XX(ha) || IS_QLA27XX(ha))
+	(IS_QLA81XX(ha) || IS_QLA83XX(ha) || IS_QLA27XX(ha) || IS_QLA28XX(ha))
 #define IS_EXLOGIN_OFFLD_CAPABLE(ha) \
-	(IS_QLA25XX(ha) || IS_QLA81XX(ha) || IS_QLA83XX(ha) || IS_QLA27XX(ha))
+	(IS_QLA25XX(ha) || IS_QLA81XX(ha) || IS_QLA83XX(ha) || \
+	 IS_QLA27XX(ha) || IS_QLA28XX(ha))
 #define USE_ASYNC_SCAN(ha) (IS_QLA25XX(ha) || IS_QLA81XX(ha) ||\
-	IS_QLA83XX(ha) || IS_QLA27XX(ha))
+	IS_QLA83XX(ha) || IS_QLA27XX(ha) || IS_QLA28XX(ha))
 
 	/* HBA serial number */
 	uint8_t		serial0;
@@ -3888,6 +3883,9 @@ struct qla_hw_data {
 	void		*sfp_data;
 	dma_addr_t	sfp_data_dma;
 
+	void		*flt;
+	dma_addr_t	flt_dma;
+
 #define XGMAC_DATA_SIZE	4096
 	void		*xgmac_data;
 	dma_addr_t	xgmac_data_dma;
@@ -3999,18 +3997,23 @@ struct qla_hw_data {
 	uint8_t		fw_seriallink_options[4];
 	uint16_t	fw_seriallink_options24[4];
 
+	uint8_t		serdes_version[3];
 	uint8_t		mpi_version[3];
 	uint32_t	mpi_capabilities;
 	uint8_t		phy_version[3];
 	uint8_t		pep_version[3];
 
 	/* Firmware dump template */
-	void		*fw_dump_template;
-	uint32_t	fw_dump_template_len;
-	/* Firmware dump information. */
+	struct fwdt {
+		void *template;
+		ulong length;
+		ulong dump_size;
+	} fwdt[2];
 	struct qla2xxx_fw_dump *fw_dump;
 	uint32_t	fw_dump_len;
-	int		fw_dumped;
+	u32		fw_dump_alloc_len;
+	bool		fw_dumped;
+	bool		fw_dump_mpi;
 	unsigned long	fw_dump_cap_flags;
 #define RISC_PAUSE_CMPL		0
 #define DMA_SHUTDOWN_CMPL	1
@@ -4049,7 +4052,6 @@ struct qla_hw_data {
 	uint16_t	product_id[4];
 
 	uint8_t		model_number[16+1];
-#define BINZERO		"\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"
 	char		model_desc[80];
 	uint8_t		adapter_id[16+1];
 
@@ -4089,22 +4091,28 @@ struct qla_hw_data {
 	uint32_t	fdt_protect_sec_cmd;
 	uint32_t	fdt_wrt_sts_reg_cmd;
 
-	uint32_t        flt_region_flt;
-	uint32_t        flt_region_fdt;
-	uint32_t        flt_region_boot;
-	uint32_t        flt_region_boot_sec;
-	uint32_t        flt_region_fw;
-	uint32_t        flt_region_fw_sec;
-	uint32_t        flt_region_vpd_nvram;
-	uint32_t        flt_region_vpd;
-	uint32_t        flt_region_vpd_sec;
-	uint32_t        flt_region_nvram;
-	uint32_t        flt_region_npiv_conf;
-	uint32_t	flt_region_gold_fw;
-	uint32_t	flt_region_fcp_prio;
-	uint32_t	flt_region_bootload;
-	uint32_t	flt_region_img_status_pri;
-	uint32_t	flt_region_img_status_sec;
+	struct {
+		uint32_t	flt_region_flt;
+		uint32_t	flt_region_fdt;
+		uint32_t	flt_region_boot;
+		uint32_t	flt_region_boot_sec;
+		uint32_t	flt_region_fw;
+		uint32_t	flt_region_fw_sec;
+		uint32_t	flt_region_vpd_nvram;
+		uint32_t	flt_region_vpd_nvram_sec;
+		uint32_t	flt_region_vpd;
+		uint32_t	flt_region_vpd_sec;
+		uint32_t	flt_region_nvram;
+		uint32_t	flt_region_nvram_sec;
+		uint32_t	flt_region_npiv_conf;
+		uint32_t	flt_region_gold_fw;
+		uint32_t	flt_region_fcp_prio;
+		uint32_t	flt_region_bootload;
+		uint32_t	flt_region_img_status_pri;
+		uint32_t	flt_region_img_status_sec;
+		uint32_t	flt_region_aux_img_status_pri;
+		uint32_t	flt_region_aux_img_status_sec;
+	};
 	uint8_t         active_image;
 
 	/* Needed for BEACON */
@@ -4197,8 +4205,8 @@ struct qla_hw_data {
 	struct qlt_hw_data tgt;
 	int	allow_cna_fw_dump;
 	uint32_t fw_ability_mask;
-	uint16_t min_link_speed;
-	uint16_t max_speed_sup;
+	uint16_t min_supported_speed;
+	uint16_t max_supported_speed;
 
 	/* DMA pool for the DIF bundling buffers */
 	struct dma_pool *dif_bundl_pool;
@@ -4225,9 +4233,20 @@ struct qla_hw_data {
 
 	atomic_t zio_threshold;
 	uint16_t last_zio_threshold;
+
 #define DEFAULT_ZIO_THRESHOLD 5
 };
 
+struct active_regions {
+	uint8_t global;
+	struct {
+		uint8_t board_config;
+		uint8_t vpd_nvram;
+		uint8_t npiv_config_0_1;
+		uint8_t npiv_config_2_3;
+	} aux;
+};
+
 #define FW_ABILITY_MAX_SPEED_MASK	0xFUL
 #define FW_ABILITY_MAX_SPEED_16G	0x0
 #define FW_ABILITY_MAX_SPEED_32G	0x1
@@ -4315,6 +4334,7 @@ typedef struct scsi_qla_host {
 #define N2N_LOGIN_NEEDED	30
 #define IOCB_WORK_ACTIVE	31
 #define SET_ZIO_THRESHOLD_NEEDED 32
+#define ISP_ABORT_TO_ROM	33
 
 	unsigned long	pci_flags;
 #define PFLG_DISCONNECTED	0	/* PCI device removed */
@@ -4429,7 +4449,7 @@ typedef struct scsi_qla_host {
 	int fcport_count;
 	wait_queue_head_t fcport_waitQ;
 	wait_queue_head_t vref_waitq;
-	uint8_t min_link_speed_feat;
+	uint8_t min_supported_speed;
 	uint8_t n2n_node_name[WWN_SIZE];
 	uint8_t n2n_port_name[WWN_SIZE];
 	uint16_t	n2n_id;
@@ -4441,14 +4461,21 @@ typedef struct scsi_qla_host {
 
 struct qla27xx_image_status {
 	uint8_t image_status_mask;
-	uint16_t generation_number;
-	uint8_t reserved[3];
-	uint8_t ver_minor;
+	uint16_t generation;
 	uint8_t ver_major;
+	uint8_t ver_minor;
+	uint8_t bitmap;		/* 28xx only */
+	uint8_t reserved[2];
 	uint32_t checksum;
 	uint32_t signature;
 } __packed;
 
+/* 28xx aux image status bimap values */
+#define QLA28XX_AUX_IMG_BOARD_CONFIG		BIT_0
+#define QLA28XX_AUX_IMG_VPD_NVRAM		BIT_1
+#define QLA28XX_AUX_IMG_NPIV_CONFIG_0_1		BIT_2
+#define QLA28XX_AUX_IMG_NPIV_CONFIG_2_3		BIT_3
+
 #define SET_VP_IDX	1
 #define SET_AL_PA	2
 #define RESET_VP_IDX	3
@@ -4495,6 +4522,24 @@ struct qla2_sgx {
 	}					\
 }
 
+
+#define SFUB_CHECKSUM_SIZE	4
+
+struct secure_flash_update_block {
+	uint32_t	block_info;
+	uint32_t	signature_lo;
+	uint32_t	signature_hi;
+	uint32_t	signature_upper[0x3e];
+};
+
+struct secure_flash_update_block_pk {
+	uint32_t	block_info;
+	uint32_t	signature_lo;
+	uint32_t	signature_hi;
+	uint32_t	signature_upper[0x3e];
+	uint32_t	public_key[0x41];
+};
+
 /*
  * Macros to help code, maintain, etc.
  */
@@ -4595,6 +4640,7 @@ struct qla2_sgx {
 #define OPTROM_SIZE_81XX	0x400000
 #define OPTROM_SIZE_82XX	0x800000
 #define OPTROM_SIZE_83XX	0x1000000
+#define OPTROM_SIZE_28XX	0x2000000
 
 #define OPTROM_BURST_SIZE	0x1000
 #define OPTROM_BURST_DWORDS	(OPTROM_BURST_SIZE / 4)
@@ -4691,10 +4737,13 @@ struct sff_8247_a0 {
 #define AUTO_DETECT_SFP_SUPPORT(_vha)\
 	(ql2xautodetectsfp && !_vha->vp_idx &&		\
 	(IS_QLA25XX(_vha->hw) || IS_QLA81XX(_vha->hw) ||\
-	IS_QLA83XX(_vha->hw) || IS_QLA27XX(_vha->hw)))
+	IS_QLA83XX(_vha->hw) || IS_QLA27XX(_vha->hw) || \
+	 IS_QLA28XX(_vha->hw)))
+
+#define FLASH_SEMAPHORE_REGISTER_ADDR   0x00101016
 
 #define USER_CTRL_IRQ(_ha) (ql2xuctrlirq && QLA_TGT_MODE_ENABLED() && \
-	(IS_QLA27XX(_ha) || IS_QLA83XX(_ha)))
+	(IS_QLA27XX(_ha) || IS_QLA28XX(_ha) || IS_QLA83XX(_ha)))
 
 #define SAVE_TOPO(_ha) { \
 	if (_ha->current_topology)				\
diff --git a/drivers/scsi/qla2xxx/qla_dfs.c b/drivers/scsi/qla2xxx/qla_dfs.c
index 5819a45ac5ef..a432caebefec 100644
--- a/drivers/scsi/qla2xxx/qla_dfs.c
+++ b/drivers/scsi/qla2xxx/qla_dfs.c
@@ -41,6 +41,7 @@ static int
 qla2x00_dfs_tgt_sess_open(struct inode *inode, struct file *file)
 {
 	scsi_qla_host_t *vha = inode->i_private;
+
 	return single_open(file, qla2x00_dfs_tgt_sess_show, vha);
 }
 
@@ -161,6 +162,7 @@ static int
 qla_dfs_fw_resource_cnt_open(struct inode *inode, struct file *file)
 {
 	struct scsi_qla_host *vha = inode->i_private;
+
 	return single_open(file, qla_dfs_fw_resource_cnt_show, vha);
 }
 
@@ -250,6 +252,7 @@ static int
 qla_dfs_tgt_counters_open(struct inode *inode, struct file *file)
 {
 	struct scsi_qla_host *vha = inode->i_private;
+
 	return single_open(file, qla_dfs_tgt_counters_show, vha);
 }
 
@@ -386,7 +389,7 @@ qla_dfs_naqp_write(struct file *file, const char __user *buffer,
 	int rc = 0;
 	unsigned long num_act_qp;
 
-	if (!(IS_QLA27XX(ha) || IS_QLA83XX(ha))) {
+	if (!(IS_QLA27XX(ha) || IS_QLA83XX(ha) || IS_QLA28XX(ha))) {
 		pr_err("host%ld: this adapter does not support Multi Q.",
 		    vha->host_no);
 		return -EINVAL;
@@ -438,7 +441,7 @@ qla2x00_dfs_setup(scsi_qla_host_t *vha)
 	struct qla_hw_data *ha = vha->hw;
 
 	if (!IS_QLA25XX(ha) && !IS_QLA81XX(ha) && !IS_QLA83XX(ha) &&
-	    !IS_QLA27XX(ha))
+	    !IS_QLA27XX(ha) && !IS_QLA28XX(ha))
 		goto out;
 	if (!ha->fce)
 		goto out;
@@ -474,7 +477,7 @@ create_nodes:
 	ha->tgt.dfs_tgt_sess = debugfs_create_file("tgt_sess",
 		S_IRUSR, ha->dfs_dir, vha, &dfs_tgt_sess_ops);
 
-	if (IS_QLA27XX(ha) || IS_QLA83XX(ha))
+	if (IS_QLA27XX(ha) || IS_QLA83XX(ha) || IS_QLA28XX(ha))
 		ha->tgt.dfs_naqp = debugfs_create_file("naqp",
 		    0400, ha->dfs_dir, vha, &dfs_naqp_ops);
 out:
diff --git a/drivers/scsi/qla2xxx/qla_dsd.h b/drivers/scsi/qla2xxx/qla_dsd.h
new file mode 100644
index 000000000000..7479924ba422
--- /dev/null
+++ b/drivers/scsi/qla2xxx/qla_dsd.h
@@ -0,0 +1,30 @@
+#ifndef _QLA_DSD_H_
+#define _QLA_DSD_H_
+
+/* 32-bit data segment descriptor (8 bytes) */
+struct dsd32 {
+	__le32 address;
+	__le32 length;
+};
+
+static inline void append_dsd32(struct dsd32 **dsd, struct scatterlist *sg)
+{
+	put_unaligned_le32(sg_dma_address(sg), &(*dsd)->address);
+	put_unaligned_le32(sg_dma_len(sg),     &(*dsd)->length);
+	(*dsd)++;
+}
+
+/* 64-bit data segment descriptor (12 bytes) */
+struct dsd64 {
+	__le64 address;
+	__le32 length;
+} __packed;
+
+static inline void append_dsd64(struct dsd64 **dsd, struct scatterlist *sg)
+{
+	put_unaligned_le64(sg_dma_address(sg), &(*dsd)->address);
+	put_unaligned_le32(sg_dma_len(sg),     &(*dsd)->length);
+	(*dsd)++;
+}
+
+#endif
diff --git a/drivers/scsi/qla2xxx/qla_fw.h b/drivers/scsi/qla2xxx/qla_fw.h
index 50c1e6c62e31..df079a8c2b33 100644
--- a/drivers/scsi/qla2xxx/qla_fw.h
+++ b/drivers/scsi/qla2xxx/qla_fw.h
@@ -10,6 +10,8 @@
 #include <linux/nvme.h>
 #include <linux/nvme-fc.h>
 
+#include "qla_dsd.h"
+
 #define MBS_CHECKSUM_ERROR	0x4010
 #define MBS_INVALID_PRODUCT_KEY	0x4020
 
@@ -339,9 +341,9 @@ struct init_cb_24xx {
 
 	uint16_t prio_request_q_length;
 
-	uint32_t request_q_address[2];
-	uint32_t response_q_address[2];
-	uint32_t prio_request_q_address[2];
+	__le64	 request_q_address __packed;
+	__le64	 response_q_address __packed;
+	__le64	 prio_request_q_address __packed;
 
 	uint16_t msix;
 	uint16_t msix_atio;
@@ -349,7 +351,7 @@ struct init_cb_24xx {
 
 	uint16_t atio_q_inpointer;
 	uint16_t atio_q_length;
-	uint32_t atio_q_address[2];
+	__le64	 atio_q_address __packed;
 
 	uint16_t interrupt_delay_timer;		/* 100us increments. */
 	uint16_t login_timeout;
@@ -453,7 +455,7 @@ struct cmd_bidir {
 #define BD_WRITE_DATA			BIT_0
 
 	uint16_t fcp_cmnd_dseg_len;		/* Data segment length. */
-	uint32_t fcp_cmnd_dseg_address[2];	/* Data segment address. */
+	__le64	 fcp_cmnd_dseg_address __packed;/* Data segment address. */
 
 	uint16_t reserved[2];			/* Reserved */
 
@@ -463,8 +465,7 @@ struct cmd_bidir {
 	uint8_t port_id[3];			/* PortID of destination port.*/
 	uint8_t vp_index;
 
-	uint32_t fcp_data_dseg_address[2];	/* Data segment address. */
-	uint16_t fcp_data_dseg_len;		/* Data segment length. */
+	struct dsd64 fcp_dsd;
 };
 
 #define COMMAND_TYPE_6	0x48		/* Command Type 6 entry */
@@ -491,18 +492,18 @@ struct cmd_type_6 {
 #define CF_READ_DATA			BIT_1
 #define CF_WRITE_DATA			BIT_0
 
-	uint16_t fcp_cmnd_dseg_len;		/* Data segment length. */
-	uint32_t fcp_cmnd_dseg_address[2];	/* Data segment address. */
-
-	uint32_t fcp_rsp_dseg_address[2];	/* Data segment address. */
+	uint16_t fcp_cmnd_dseg_len;	/* Data segment length. */
+					/* Data segment address. */
+	__le64	 fcp_cmnd_dseg_address __packed;
+					/* Data segment address. */
+	__le64	 fcp_rsp_dseg_address __packed;
 
 	uint32_t byte_count;		/* Total byte count. */
 
 	uint8_t port_id[3];		/* PortID of destination port. */
 	uint8_t vp_index;
 
-	uint32_t fcp_data_dseg_address[2];	/* Data segment address. */
-	uint32_t fcp_data_dseg_len;		/* Data segment length. */
+	struct dsd64 fcp_dsd;
 };
 
 #define COMMAND_TYPE_7	0x18		/* Command Type 7 entry */
@@ -548,8 +549,7 @@ struct cmd_type_7 {
 	uint8_t port_id[3];		/* PortID of destination port. */
 	uint8_t vp_index;
 
-	uint32_t dseg_0_address[2];	/* Data segment 0 address. */
-	uint32_t dseg_0_len;		/* Data segment 0 length. */
+	struct dsd64 dsd;
 };
 
 #define COMMAND_TYPE_CRC_2	0x6A	/* Command Type CRC_2 (Type 6)
@@ -573,17 +573,17 @@ struct cmd_type_crc_2 {
 
 	uint16_t control_flags;		/* Control flags. */
 
-	uint16_t fcp_cmnd_dseg_len;		/* Data segment length. */
-	uint32_t fcp_cmnd_dseg_address[2];	/* Data segment address. */
-
-	uint32_t fcp_rsp_dseg_address[2];	/* Data segment address. */
+	uint16_t fcp_cmnd_dseg_len;	/* Data segment length. */
+	__le64	 fcp_cmnd_dseg_address __packed;
+					/* Data segment address. */
+	__le64	 fcp_rsp_dseg_address __packed;
 
 	uint32_t byte_count;		/* Total byte count. */
 
 	uint8_t port_id[3];		/* PortID of destination port. */
 	uint8_t vp_index;
 
-	uint32_t crc_context_address[2];	/* Data segment address. */
+	__le64	 crc_context_address __packed;	/* Data segment address. */
 	uint16_t crc_context_len;		/* Data segment length. */
 	uint16_t reserved_1;			/* MUST be set to 0. */
 };
@@ -717,10 +717,7 @@ struct ct_entry_24xx {
 	uint32_t rsp_byte_count;
 	uint32_t cmd_byte_count;
 
-	uint32_t dseg_0_address[2];	/* Data segment 0 address. */
-	uint32_t dseg_0_len;		/* Data segment 0 length. */
-	uint32_t dseg_1_address[2];	/* Data segment 1 address. */
-	uint32_t dseg_1_len;		/* Data segment 1 length. */
+	struct dsd64 dsd[2];
 };
 
 /*
@@ -767,9 +764,9 @@ struct els_entry_24xx {
 	uint32_t rx_byte_count;
 	uint32_t tx_byte_count;
 
-	uint32_t tx_address[2];		/* Data segment 0 address. */
+	__le64	 tx_address __packed;	/* Data segment 0 address. */
 	uint32_t tx_len;		/* Data segment 0 length. */
-	uint32_t rx_address[2];		/* Data segment 1 address. */
+	__le64	 rx_address __packed;	/* Data segment 1 address. */
 	uint32_t rx_len;		/* Data segment 1 length. */
 };
 
@@ -1422,9 +1419,9 @@ struct vf_evfp_entry_24xx {
         uint16_t control_flags;
         uint32_t io_parameter_0;
         uint32_t io_parameter_1;
-        uint32_t tx_address[2];         /* Data segment 0 address. */
+	__le64	 tx_address __packed;	/* Data segment 0 address. */
         uint32_t tx_len;                /* Data segment 0 length. */
-        uint32_t rx_address[2];         /* Data segment 1 address. */
+	__le64	 rx_address __packed;	/* Data segment 1 address. */
         uint32_t rx_len;                /* Data segment 1 length. */
 };
 
@@ -1515,13 +1512,31 @@ struct qla_flt_header {
 #define FLT_REG_VPD_SEC_27XX_2	0xD8
 #define FLT_REG_VPD_SEC_27XX_3	0xDA
 
+/* 28xx */
+#define FLT_REG_AUX_IMG_PRI_28XX	0x125
+#define FLT_REG_AUX_IMG_SEC_28XX	0x126
+#define FLT_REG_VPD_SEC_28XX_0		0x10C
+#define FLT_REG_VPD_SEC_28XX_1		0x10E
+#define FLT_REG_VPD_SEC_28XX_2		0x110
+#define FLT_REG_VPD_SEC_28XX_3		0x112
+#define FLT_REG_NVRAM_SEC_28XX_0	0x10D
+#define FLT_REG_NVRAM_SEC_28XX_1	0x10F
+#define FLT_REG_NVRAM_SEC_28XX_2	0x111
+#define FLT_REG_NVRAM_SEC_28XX_3	0x113
+
 struct qla_flt_region {
-	uint32_t code;
+	uint16_t code;
+	uint8_t attribute;
+	uint8_t reserved;
 	uint32_t size;
 	uint32_t start;
 	uint32_t end;
 };
 
+#define FLT_REGION_SIZE		16
+#define FLT_MAX_REGIONS		0xFF
+#define FLT_REGIONS_SIZE	(FLT_REGION_SIZE * FLT_MAX_REGIONS)
+
 /* Flash NPIV Configuration Table ********************************************/
 
 struct qla_npiv_header {
@@ -1588,8 +1603,7 @@ struct verify_chip_entry_84xx {
 	uint32_t fw_seq_size;
 	uint32_t relative_offset;
 
-	uint32_t dseg_address[2];
-	uint32_t dseg_length;
+	struct dsd64 dsd;
 };
 
 struct verify_chip_rsp_84xx {
@@ -1646,8 +1660,7 @@ struct access_chip_84xx {
 	uint32_t total_byte_cnt;
 	uint32_t reserved4;
 
-	uint32_t dseg_address[2];
-	uint32_t dseg_length;
+	struct dsd64 dsd;
 };
 
 struct access_chip_rsp_84xx {
@@ -1711,6 +1724,10 @@ struct access_chip_rsp_84xx {
 #define LR_DIST_FW_SHIFT	(LR_DIST_FW_POS - LR_DIST_NV_POS)
 #define LR_DIST_FW_FIELD(x)	((x) << LR_DIST_FW_SHIFT & 0xf000)
 
+/* FAC semaphore defines */
+#define FAC_SEMAPHORE_UNLOCK    0
+#define FAC_SEMAPHORE_LOCK      1
+
 struct nvram_81xx {
 	/* NVRAM header. */
 	uint8_t id[4];
@@ -1757,7 +1774,7 @@ struct nvram_81xx {
 	uint16_t reserved_6_3[14];
 
 	/* Offset 192. */
-	uint8_t min_link_speed;
+	uint8_t min_supported_speed;
 	uint8_t reserved_7_0;
 	uint16_t reserved_7[31];
 
@@ -1911,15 +1928,15 @@ struct init_cb_81xx {
 
 	uint16_t prio_request_q_length;
 
-	uint32_t request_q_address[2];
-	uint32_t response_q_address[2];
-	uint32_t prio_request_q_address[2];
+	__le64	 request_q_address __packed;
+	__le64	 response_q_address __packed;
+	__le64	 prio_request_q_address __packed;
 
 	uint8_t reserved_4[8];
 
 	uint16_t atio_q_inpointer;
 	uint16_t atio_q_length;
-	uint32_t atio_q_address[2];
+	__le64	 atio_q_address __packed;
 
 	uint16_t interrupt_delay_timer;		/* 100us increments. */
 	uint16_t login_timeout;
@@ -2005,6 +2022,8 @@ struct ex_init_cb_81xx {
 
 #define FARX_ACCESS_FLASH_CONF_81XX	0x7FFD0000
 #define FARX_ACCESS_FLASH_DATA_81XX	0x7F800000
+#define FARX_ACCESS_FLASH_CONF_28XX	0x7FFD0000
+#define FARX_ACCESS_FLASH_DATA_28XX	0x7F7D0000
 
 /* FCP priority config defines *************************************/
 /* operations */
@@ -2079,6 +2098,7 @@ struct qla_fcp_prio_cfg {
 #define FA_NPIV_CONF1_ADDR_81	0xD2000
 
 /* 83XX Flash locations -- occupies second 8MB region. */
-#define FA_FLASH_LAYOUT_ADDR_83	0xFC400
+#define FA_FLASH_LAYOUT_ADDR_83	(0x3F1000/4)
+#define FA_FLASH_LAYOUT_ADDR_28	(0x11000/4)
 
 #endif
diff --git a/drivers/scsi/qla2xxx/qla_gbl.h b/drivers/scsi/qla2xxx/qla_gbl.h
index 4eefe69ca807..bbe69ab5cf3f 100644
--- a/drivers/scsi/qla2xxx/qla_gbl.h
+++ b/drivers/scsi/qla2xxx/qla_gbl.h
@@ -18,14 +18,14 @@ extern int qla2100_pci_config(struct scsi_qla_host *);
 extern int qla2300_pci_config(struct scsi_qla_host *);
 extern int qla24xx_pci_config(scsi_qla_host_t *);
 extern int qla25xx_pci_config(scsi_qla_host_t *);
-extern void qla2x00_reset_chip(struct scsi_qla_host *);
-extern void qla24xx_reset_chip(struct scsi_qla_host *);
+extern int qla2x00_reset_chip(struct scsi_qla_host *);
+extern int qla24xx_reset_chip(struct scsi_qla_host *);
 extern int qla2x00_chip_diag(struct scsi_qla_host *);
 extern int qla24xx_chip_diag(struct scsi_qla_host *);
 extern void qla2x00_config_rings(struct scsi_qla_host *);
 extern void qla24xx_config_rings(struct scsi_qla_host *);
-extern void qla2x00_reset_adapter(struct scsi_qla_host *);
-extern void qla24xx_reset_adapter(struct scsi_qla_host *);
+extern int qla2x00_reset_adapter(struct scsi_qla_host *);
+extern int qla24xx_reset_adapter(struct scsi_qla_host *);
 extern int qla2x00_nvram_config(struct scsi_qla_host *);
 extern int qla24xx_nvram_config(struct scsi_qla_host *);
 extern int qla81xx_nvram_config(struct scsi_qla_host *);
@@ -38,8 +38,7 @@ extern int qla81xx_load_risc(scsi_qla_host_t *, uint32_t *);
 
 extern int qla2x00_perform_loop_resync(scsi_qla_host_t *);
 extern int qla2x00_loop_resync(scsi_qla_host_t *);
-
-extern int qla2x00_find_new_loop_id(scsi_qla_host_t *, fc_port_t *);
+extern void qla2x00_clear_loop_id(fc_port_t *fcport);
 
 extern int qla2x00_fabric_login(scsi_qla_host_t *, fc_port_t *, uint16_t *);
 extern int qla2x00_local_device_login(scsi_qla_host_t *, fc_port_t *);
@@ -80,6 +79,7 @@ int qla2x00_post_work(struct scsi_qla_host *vha, struct qla_work_evt *e);
 extern void *qla2x00_alloc_iocbs_ready(struct qla_qpair *, srb_t *);
 extern int qla24xx_update_fcport_fcp_prio(scsi_qla_host_t *, fc_port_t *);
 
+extern void qla2x00_set_fcport_state(fc_port_t *fcport, int state);
 extern fc_port_t *
 qla2x00_alloc_fcport(scsi_qla_host_t *, gfp_t );
 
@@ -93,7 +93,6 @@ extern int qla2xxx_mctp_dump(scsi_qla_host_t *);
 extern int
 qla2x00_alloc_outstanding_cmds(struct qla_hw_data *, struct req_que *);
 extern int qla2x00_init_rings(scsi_qla_host_t *);
-extern uint8_t qla27xx_find_valid_image(struct scsi_qla_host *);
 extern struct qla_qpair *qla2xxx_create_qpair(struct scsi_qla_host *,
 	int, int, bool);
 extern int qla2xxx_delete_qpair(struct scsi_qla_host *, struct qla_qpair *);
@@ -108,6 +107,11 @@ int qla24xx_fcport_handle_login(struct scsi_qla_host *, fc_port_t *);
 int qla24xx_detect_sfp(scsi_qla_host_t *vha);
 int qla24xx_post_gpdb_work(struct scsi_qla_host *, fc_port_t *, u8);
 
+extern void qla28xx_get_aux_images(struct scsi_qla_host *,
+    struct active_regions *);
+extern void qla27xx_get_active_image(struct scsi_qla_host *,
+    struct active_regions *);
+
 void qla2x00_async_prlo_done(struct scsi_qla_host *, fc_port_t *,
     uint16_t *);
 extern int qla2x00_post_async_prlo_work(struct scsi_qla_host *, fc_port_t *,
@@ -118,6 +122,7 @@ int qla_post_iidma_work(struct scsi_qla_host *vha, fc_port_t *fcport);
 void qla_do_iidma_work(struct scsi_qla_host *vha, fc_port_t *fcport);
 int qla2x00_reserve_mgmt_server_loop_id(scsi_qla_host_t *);
 void qla_rscn_replay(fc_port_t *fcport);
+extern bool qla24xx_risc_firmware_invalid(uint32_t *);
 
 /*
  * Global Data in qla_os.c source file.
@@ -215,7 +220,6 @@ extern void qla24xx_sched_upd_fcport(fc_port_t *);
 void qla2x00_handle_login_done_event(struct scsi_qla_host *, fc_port_t *,
 	uint16_t *);
 int qla24xx_post_gnl_work(struct scsi_qla_host *, fc_port_t *);
-int qla24xx_async_abort_cmd(srb_t *, bool);
 int qla24xx_post_relogin_work(struct scsi_qla_host *vha);
 void qla2x00_wait_for_sess_deletion(scsi_qla_host_t *);
 
@@ -238,7 +242,7 @@ extern void qla24xx_report_id_acquisition(scsi_qla_host_t *,
     struct vp_rpt_id_entry_24xx *);
 extern void qla2x00_do_dpc_all_vps(scsi_qla_host_t *);
 extern int qla24xx_vport_create_req_sanity_check(struct fc_vport *);
-extern scsi_qla_host_t * qla24xx_create_vhost(struct fc_vport *);
+extern scsi_qla_host_t *qla24xx_create_vhost(struct fc_vport *);
 
 extern void qla2x00_sp_free_dma(void *);
 extern char *qla2x00_get_fw_version_str(struct scsi_qla_host *, char *);
@@ -276,21 +280,20 @@ extern int qla2x00_start_sp(srb_t *);
 extern int qla24xx_dif_start_scsi(srb_t *);
 extern int qla2x00_start_bidir(srb_t *, struct scsi_qla_host *, uint32_t);
 extern int qla2xxx_dif_start_scsi_mq(srb_t *);
+extern void qla2x00_init_timer(srb_t *sp, unsigned long tmo);
 extern unsigned long qla2x00_get_async_timeout(struct scsi_qla_host *);
 
 extern void *qla2x00_alloc_iocbs(struct scsi_qla_host *, srb_t *);
 extern void *__qla2x00_alloc_iocbs(struct qla_qpair *, srb_t *);
 extern int qla2x00_issue_marker(scsi_qla_host_t *, int);
 extern int qla24xx_walk_and_build_sglist_no_difb(struct qla_hw_data *, srb_t *,
-	uint32_t *, uint16_t, struct qla_tc_param *);
+	struct dsd64 *, uint16_t, struct qla_tc_param *);
 extern int qla24xx_walk_and_build_sglist(struct qla_hw_data *, srb_t *,
-	uint32_t *, uint16_t, struct qla_tc_param *);
+	struct dsd64 *, uint16_t, struct qla_tc_param *);
 extern int qla24xx_walk_and_build_prot_sglist(struct qla_hw_data *, srb_t *,
-	uint32_t *, uint16_t, struct qla_tgt_cmd *);
+	struct dsd64 *, uint16_t, struct qla_tgt_cmd *);
 extern int qla24xx_get_one_block_sg(uint32_t, struct qla2_sgx *, uint32_t *);
 extern int qla24xx_configure_prot_mode(srb_t *, uint16_t *);
-extern int qla24xx_build_scsi_crc_2_iocbs(srb_t *,
-	struct cmd_type_crc_2 *, uint16_t, uint16_t, uint16_t);
 
 /*
  * Global Function Prototypes in qla_mbx.c source file.
@@ -466,6 +469,8 @@ qla81xx_fac_do_write_enable(scsi_qla_host_t *, int);
 extern int
 qla81xx_fac_erase_sector(scsi_qla_host_t *, uint32_t, uint32_t);
 
+extern int qla81xx_fac_semaphore_access(scsi_qla_host_t *, int);
+
 extern int
 qla2x00_get_xgmac_stats(scsi_qla_host_t *, dma_addr_t, uint16_t, uint16_t *);
 
@@ -511,6 +516,14 @@ extern int qla27xx_get_zio_threshold(scsi_qla_host_t *, uint16_t *);
 extern int qla27xx_set_zio_threshold(scsi_qla_host_t *, uint16_t);
 int qla24xx_res_count_wait(struct scsi_qla_host *, uint16_t *, int);
 
+extern int qla28xx_secure_flash_update(scsi_qla_host_t *, uint16_t, uint16_t,
+    uint32_t, dma_addr_t, uint32_t);
+
+extern int qla2xxx_read_remote_register(scsi_qla_host_t *, uint32_t,
+    uint32_t *);
+extern int qla2xxx_write_remote_register(scsi_qla_host_t *, uint32_t,
+    uint32_t);
+
 /*
  * Global Function Prototypes in qla_isr.c source file.
  */
@@ -542,19 +555,20 @@ fc_port_t *qla2x00_find_fcport_by_nportid(scsi_qla_host_t *, port_id_t *, u8);
  */
 extern void qla2x00_release_nvram_protection(scsi_qla_host_t *);
 extern uint32_t *qla24xx_read_flash_data(scsi_qla_host_t *, uint32_t *,
-					 uint32_t, uint32_t);
-extern uint8_t *qla2x00_read_nvram_data(scsi_qla_host_t *, uint8_t *, uint32_t,
-					uint32_t);
-extern uint8_t *qla24xx_read_nvram_data(scsi_qla_host_t *, uint8_t *, uint32_t,
-					uint32_t);
-extern int qla2x00_write_nvram_data(scsi_qla_host_t *, uint8_t *, uint32_t,
-				    uint32_t);
-extern int qla24xx_write_nvram_data(scsi_qla_host_t *, uint8_t *, uint32_t,
-				    uint32_t);
-extern uint8_t *qla25xx_read_nvram_data(scsi_qla_host_t *, uint8_t *, uint32_t,
-					uint32_t);
-extern int qla25xx_write_nvram_data(scsi_qla_host_t *, uint8_t *, uint32_t,
-				    uint32_t);
+    uint32_t, uint32_t);
+extern uint8_t *qla2x00_read_nvram_data(scsi_qla_host_t *, void *, uint32_t,
+    uint32_t);
+extern uint8_t *qla24xx_read_nvram_data(scsi_qla_host_t *, void *, uint32_t,
+    uint32_t);
+extern int qla2x00_write_nvram_data(scsi_qla_host_t *, void *, uint32_t,
+    uint32_t);
+extern int qla24xx_write_nvram_data(scsi_qla_host_t *, void *, uint32_t,
+    uint32_t);
+extern uint8_t *qla25xx_read_nvram_data(scsi_qla_host_t *, void *, uint32_t,
+    uint32_t);
+extern int qla25xx_write_nvram_data(scsi_qla_host_t *, void *, uint32_t,
+    uint32_t);
+
 extern int qla2x00_is_a_vp_did(scsi_qla_host_t *, uint32_t);
 bool qla2x00_check_reg32_for_disconnect(scsi_qla_host_t *, uint32_t);
 bool qla2x00_check_reg16_for_disconnect(scsi_qla_host_t *, uint16_t);
@@ -574,18 +588,18 @@ extern int qla83xx_restart_nic_firmware(scsi_qla_host_t *);
 extern int qla83xx_access_control(scsi_qla_host_t *, uint16_t, uint32_t,
 				  uint32_t, uint16_t *);
 
-extern uint8_t *qla2x00_read_optrom_data(struct scsi_qla_host *, uint8_t *,
+extern void *qla2x00_read_optrom_data(struct scsi_qla_host *, void *,
 					 uint32_t, uint32_t);
-extern int qla2x00_write_optrom_data(struct scsi_qla_host *, uint8_t *,
+extern int qla2x00_write_optrom_data(struct scsi_qla_host *, void *,
 				     uint32_t, uint32_t);
-extern uint8_t *qla24xx_read_optrom_data(struct scsi_qla_host *, uint8_t *,
+extern void *qla24xx_read_optrom_data(struct scsi_qla_host *, void *,
 					 uint32_t, uint32_t);
-extern int qla24xx_write_optrom_data(struct scsi_qla_host *, uint8_t *,
+extern int qla24xx_write_optrom_data(struct scsi_qla_host *, void *,
 				     uint32_t, uint32_t);
-extern uint8_t *qla25xx_read_optrom_data(struct scsi_qla_host *, uint8_t *,
+extern void *qla25xx_read_optrom_data(struct scsi_qla_host *, void *,
 					 uint32_t, uint32_t);
-extern uint8_t *qla8044_read_optrom_data(struct scsi_qla_host *,
-					 uint8_t *, uint32_t, uint32_t);
+extern void *qla8044_read_optrom_data(struct scsi_qla_host *,
+					 void *, uint32_t, uint32_t);
 extern void qla8044_watchdog(struct scsi_qla_host *vha);
 
 extern int qla2x00_get_flash_version(scsi_qla_host_t *, void *);
@@ -610,20 +624,13 @@ extern void qla82xx_fw_dump(scsi_qla_host_t *, int);
 extern void qla8044_fw_dump(scsi_qla_host_t *, int);
 
 extern void qla27xx_fwdump(scsi_qla_host_t *, int);
-extern ulong qla27xx_fwdt_calculate_dump_size(struct scsi_qla_host *);
+extern ulong qla27xx_fwdt_calculate_dump_size(struct scsi_qla_host *, void *);
 extern int qla27xx_fwdt_template_valid(void *);
 extern ulong qla27xx_fwdt_template_size(void *);
-extern const void *qla27xx_fwdt_template_default(void);
-extern ulong qla27xx_fwdt_template_default_size(void);
-
-extern void qla2x00_dump_regs(scsi_qla_host_t *);
-extern void qla2x00_dump_buffer(uint8_t *, uint32_t);
-extern void qla2x00_dump_buffer_zipped(uint8_t *, uint32_t);
-extern void ql_dump_regs(uint32_t, scsi_qla_host_t *, int32_t);
-extern void ql_dump_buffer(uint32_t, scsi_qla_host_t *, int32_t,
-			   uint8_t *, uint32_t);
-extern void qla2xxx_dump_post_process(scsi_qla_host_t *, int);
 
+extern void qla2xxx_dump_post_process(scsi_qla_host_t *, int);
+extern void ql_dump_regs(uint, scsi_qla_host_t *, uint);
+extern void ql_dump_buffer(uint, scsi_qla_host_t *, uint, void *, uint);
 /*
  * Global Function Prototypes in qla_gs.c source file.
  */
@@ -722,7 +729,7 @@ extern void qla24xx_wrt_rsp_reg(struct qla_hw_data *, uint16_t, uint16_t);
 /* qlafx00 related functions */
 extern int qlafx00_pci_config(struct scsi_qla_host *);
 extern int qlafx00_initialize_adapter(struct scsi_qla_host *);
-extern void qlafx00_soft_reset(scsi_qla_host_t *);
+extern int qlafx00_soft_reset(scsi_qla_host_t *);
 extern int qlafx00_chip_diag(scsi_qla_host_t *);
 extern void qlafx00_config_rings(struct scsi_qla_host *);
 extern char *qlafx00_pci_info_str(struct scsi_qla_host *, char *);
@@ -765,16 +772,16 @@ extern int qla82xx_pci_region_offset(struct pci_dev *, int);
 extern int qla82xx_iospace_config(struct qla_hw_data *);
 
 /* Initialization related functions */
-extern void qla82xx_reset_chip(struct scsi_qla_host *);
+extern int qla82xx_reset_chip(struct scsi_qla_host *);
 extern void qla82xx_config_rings(struct scsi_qla_host *);
 extern void qla82xx_watchdog(scsi_qla_host_t *);
 extern int qla82xx_start_firmware(scsi_qla_host_t *);
 
 /* Firmware and flash related functions */
 extern int qla82xx_load_risc(scsi_qla_host_t *, uint32_t *);
-extern uint8_t *qla82xx_read_optrom_data(struct scsi_qla_host *, uint8_t *,
+extern void *qla82xx_read_optrom_data(struct scsi_qla_host *, void *,
 					 uint32_t, uint32_t);
-extern int qla82xx_write_optrom_data(struct scsi_qla_host *, uint8_t *,
+extern int qla82xx_write_optrom_data(struct scsi_qla_host *, void *,
 				     uint32_t, uint32_t);
 
 /* Mailbox related functions */
@@ -870,7 +877,7 @@ extern void qla8044_clear_drv_active(struct qla_hw_data *);
 void qla8044_get_minidump(struct scsi_qla_host *vha);
 int qla8044_collect_md_data(struct scsi_qla_host *vha);
 extern int qla8044_md_get_template(scsi_qla_host_t *);
-extern int qla8044_write_optrom_data(struct scsi_qla_host *, uint8_t *,
+extern int qla8044_write_optrom_data(struct scsi_qla_host *, void *,
 				     uint32_t, uint32_t);
 extern irqreturn_t qla8044_intr_handler(int, void *);
 extern void qla82xx_mbx_completion(scsi_qla_host_t *, uint16_t);
diff --git a/drivers/scsi/qla2xxx/qla_gs.c b/drivers/scsi/qla2xxx/qla_gs.c
index c6fdad12428e..9f58e591666d 100644
--- a/drivers/scsi/qla2xxx/qla_gs.c
+++ b/drivers/scsi/qla2xxx/qla_gs.c
@@ -45,13 +45,11 @@ qla2x00_prep_ms_iocb(scsi_qla_host_t *vha, struct ct_arg *arg)
 	ms_pkt->rsp_bytecount = cpu_to_le32(arg->rsp_size);
 	ms_pkt->req_bytecount = cpu_to_le32(arg->req_size);
 
-	ms_pkt->dseg_req_address[0] = cpu_to_le32(LSD(arg->req_dma));
-	ms_pkt->dseg_req_address[1] = cpu_to_le32(MSD(arg->req_dma));
-	ms_pkt->dseg_req_length = ms_pkt->req_bytecount;
+	put_unaligned_le64(arg->req_dma, &ms_pkt->req_dsd.address);
+	ms_pkt->req_dsd.length = ms_pkt->req_bytecount;
 
-	ms_pkt->dseg_rsp_address[0] = cpu_to_le32(LSD(arg->rsp_dma));
-	ms_pkt->dseg_rsp_address[1] = cpu_to_le32(MSD(arg->rsp_dma));
-	ms_pkt->dseg_rsp_length = ms_pkt->rsp_bytecount;
+	put_unaligned_le64(arg->rsp_dma, &ms_pkt->rsp_dsd.address);
+	ms_pkt->rsp_dsd.length = ms_pkt->rsp_bytecount;
 
 	vha->qla_stats.control_requests++;
 
@@ -83,13 +81,11 @@ qla24xx_prep_ms_iocb(scsi_qla_host_t *vha, struct ct_arg *arg)
 	ct_pkt->rsp_byte_count = cpu_to_le32(arg->rsp_size);
 	ct_pkt->cmd_byte_count = cpu_to_le32(arg->req_size);
 
-	ct_pkt->dseg_0_address[0] = cpu_to_le32(LSD(arg->req_dma));
-	ct_pkt->dseg_0_address[1] = cpu_to_le32(MSD(arg->req_dma));
-	ct_pkt->dseg_0_len = ct_pkt->cmd_byte_count;
+	put_unaligned_le64(arg->req_dma, &ct_pkt->dsd[0].address);
+	ct_pkt->dsd[0].length = ct_pkt->cmd_byte_count;
 
-	ct_pkt->dseg_1_address[0] = cpu_to_le32(LSD(arg->rsp_dma));
-	ct_pkt->dseg_1_address[1] = cpu_to_le32(MSD(arg->rsp_dma));
-	ct_pkt->dseg_1_len = ct_pkt->rsp_byte_count;
+	put_unaligned_le64(arg->rsp_dma, &ct_pkt->dsd[1].address);
+	ct_pkt->dsd[1].length = ct_pkt->rsp_byte_count;
 	ct_pkt->vp_index = vha->vp_idx;
 
 	vha->qla_stats.control_requests++;
@@ -152,8 +148,8 @@ qla2x00_chk_ms_status(scsi_qla_host_t *vha, ms_iocb_entry_t *ms_pkt,
 				    vha->d_id.b.area, vha->d_id.b.al_pa,
 				    comp_status, ct_rsp->header.response);
 				ql_dump_buffer(ql_dbg_disc + ql_dbg_buffer, vha,
-				    0x2078, (uint8_t *)&ct_rsp->header,
-				    sizeof(struct ct_rsp_hdr));
+				    0x2078, ct_rsp,
+				    offsetof(typeof(*ct_rsp), rsp));
 				rval = QLA_INVALID_COMMAND;
 			} else
 				rval = QLA_SUCCESS;
@@ -1000,8 +996,7 @@ qla2x00_prep_sns_cmd(scsi_qla_host_t *vha, uint16_t cmd, uint16_t scmd_len,
 	memset(sns_cmd, 0, sizeof(struct sns_cmd_pkt));
 	wc = data_size / 2;			/* Size in 16bit words. */
 	sns_cmd->p.cmd.buffer_length = cpu_to_le16(wc);
-	sns_cmd->p.cmd.buffer_address[0] = cpu_to_le32(LSD(ha->sns_cmd_dma));
-	sns_cmd->p.cmd.buffer_address[1] = cpu_to_le32(MSD(ha->sns_cmd_dma));
+	put_unaligned_le64(ha->sns_cmd_dma, &sns_cmd->p.cmd.buffer_address);
 	sns_cmd->p.cmd.subcommand_length = cpu_to_le16(scmd_len);
 	sns_cmd->p.cmd.subcommand = cpu_to_le16(cmd);
 	wc = (data_size - 16) / 4;		/* Size in 32bit words. */
@@ -1385,6 +1380,7 @@ qla2x00_mgmt_svr_login(scsi_qla_host_t *vha)
 	int ret, rval;
 	uint16_t mb[MAILBOX_REGISTER_COUNT];
 	struct qla_hw_data *ha = vha->hw;
+
 	ret = QLA_SUCCESS;
 	if (vha->flags.management_server_logged_in)
 		return ret;
@@ -1423,6 +1419,7 @@ qla2x00_prep_ms_fdmi_iocb(scsi_qla_host_t *vha, uint32_t req_size,
 {
 	ms_iocb_entry_t *ms_pkt;
 	struct qla_hw_data *ha = vha->hw;
+
 	ms_pkt = ha->ms_iocb;
 	memset(ms_pkt, 0, sizeof(ms_iocb_entry_t));
 
@@ -1436,13 +1433,11 @@ qla2x00_prep_ms_fdmi_iocb(scsi_qla_host_t *vha, uint32_t req_size,
 	ms_pkt->rsp_bytecount = cpu_to_le32(rsp_size);
 	ms_pkt->req_bytecount = cpu_to_le32(req_size);
 
-	ms_pkt->dseg_req_address[0] = cpu_to_le32(LSD(ha->ct_sns_dma));
-	ms_pkt->dseg_req_address[1] = cpu_to_le32(MSD(ha->ct_sns_dma));
-	ms_pkt->dseg_req_length = ms_pkt->req_bytecount;
+	put_unaligned_le64(ha->ct_sns_dma, &ms_pkt->req_dsd.address);
+	ms_pkt->req_dsd.length = ms_pkt->req_bytecount;
 
-	ms_pkt->dseg_rsp_address[0] = cpu_to_le32(LSD(ha->ct_sns_dma));
-	ms_pkt->dseg_rsp_address[1] = cpu_to_le32(MSD(ha->ct_sns_dma));
-	ms_pkt->dseg_rsp_length = ms_pkt->rsp_bytecount;
+	put_unaligned_le64(ha->ct_sns_dma, &ms_pkt->rsp_dsd.address);
+	ms_pkt->rsp_dsd.length = ms_pkt->rsp_bytecount;
 
 	return ms_pkt;
 }
@@ -1474,13 +1469,11 @@ qla24xx_prep_ms_fdmi_iocb(scsi_qla_host_t *vha, uint32_t req_size,
 	ct_pkt->rsp_byte_count = cpu_to_le32(rsp_size);
 	ct_pkt->cmd_byte_count = cpu_to_le32(req_size);
 
-	ct_pkt->dseg_0_address[0] = cpu_to_le32(LSD(ha->ct_sns_dma));
-	ct_pkt->dseg_0_address[1] = cpu_to_le32(MSD(ha->ct_sns_dma));
-	ct_pkt->dseg_0_len = ct_pkt->cmd_byte_count;
+	put_unaligned_le64(ha->ct_sns_dma, &ct_pkt->dsd[0].address);
+	ct_pkt->dsd[0].length = ct_pkt->cmd_byte_count;
 
-	ct_pkt->dseg_1_address[0] = cpu_to_le32(LSD(ha->ct_sns_dma));
-	ct_pkt->dseg_1_address[1] = cpu_to_le32(MSD(ha->ct_sns_dma));
-	ct_pkt->dseg_1_len = ct_pkt->rsp_byte_count;
+	put_unaligned_le64(ha->ct_sns_dma, &ct_pkt->dsd[1].address);
+	ct_pkt->dsd[1].length = ct_pkt->rsp_byte_count;
 	ct_pkt->vp_index = vha->vp_idx;
 
 	return ct_pkt;
@@ -1495,10 +1488,10 @@ qla2x00_update_ms_fdmi_iocb(scsi_qla_host_t *vha, uint32_t req_size)
 
 	if (IS_FWI2_CAPABLE(ha)) {
 		ct_pkt->cmd_byte_count = cpu_to_le32(req_size);
-		ct_pkt->dseg_0_len = ct_pkt->cmd_byte_count;
+		ct_pkt->dsd[0].length = ct_pkt->cmd_byte_count;
 	} else {
 		ms_pkt->req_bytecount = cpu_to_le32(req_size);
-		ms_pkt->dseg_req_length = ms_pkt->req_bytecount;
+		ms_pkt->req_dsd.length = ms_pkt->req_bytecount;
 	}
 
 	return ms_pkt;
@@ -1794,7 +1787,7 @@ qla2x00_fdmi_rpa(scsi_qla_host_t *vha)
 	if (IS_CNA_CAPABLE(ha))
 		eiter->a.sup_speed = cpu_to_be32(
 		    FDMI_PORT_SPEED_10GB);
-	else if (IS_QLA27XX(ha))
+	else if (IS_QLA27XX(ha) || IS_QLA28XX(ha))
 		eiter->a.sup_speed = cpu_to_be32(
 		    FDMI_PORT_SPEED_32GB|
 		    FDMI_PORT_SPEED_16GB|
@@ -2373,7 +2366,7 @@ qla2x00_fdmiv2_rpa(scsi_qla_host_t *vha)
 	if (IS_CNA_CAPABLE(ha))
 		eiter->a.sup_speed = cpu_to_be32(
 		    FDMI_PORT_SPEED_10GB);
-	else if (IS_QLA27XX(ha))
+	else if (IS_QLA27XX(ha) || IS_QLA28XX(ha))
 		eiter->a.sup_speed = cpu_to_be32(
 		    FDMI_PORT_SPEED_32GB|
 		    FDMI_PORT_SPEED_16GB|
@@ -2446,7 +2439,7 @@ qla2x00_fdmiv2_rpa(scsi_qla_host_t *vha)
 	eiter->type = cpu_to_be16(FDMI_PORT_MAX_FRAME_SIZE);
 	eiter->len = cpu_to_be16(4 + 4);
 	eiter->a.max_frame_size = IS_FWI2_CAPABLE(ha) ?
-	    le16_to_cpu(icb24->frame_payload_size):
+	    le16_to_cpu(icb24->frame_payload_size) :
 	    le16_to_cpu(ha->init_cb->frame_payload_size);
 	eiter->a.max_frame_size = cpu_to_be32(eiter->a.max_frame_size);
 	size += 4 + 4;
@@ -2783,6 +2776,31 @@ qla24xx_prep_ct_fm_req(struct ct_sns_pkt *p, uint16_t cmd,
 	return &p->p.req;
 }
 
+static uint16_t
+qla2x00_port_speed_capability(uint16_t speed)
+{
+	switch (speed) {
+	case BIT_15:
+		return PORT_SPEED_1GB;
+	case BIT_14:
+		return PORT_SPEED_2GB;
+	case BIT_13:
+		return PORT_SPEED_4GB;
+	case BIT_12:
+		return PORT_SPEED_10GB;
+	case BIT_11:
+		return PORT_SPEED_8GB;
+	case BIT_10:
+		return PORT_SPEED_16GB;
+	case BIT_8:
+		return PORT_SPEED_32GB;
+	case BIT_7:
+		return PORT_SPEED_64GB;
+	default:
+		return PORT_SPEED_UNKNOWN;
+	}
+}
+
 /**
  * qla2x00_gpsc() - FCS Get Port Speed Capabilities (GPSC) query.
  * @vha: HA context
@@ -2855,31 +2873,8 @@ qla2x00_gpsc(scsi_qla_host_t *vha, sw_info_t *list)
 			}
 			rval = QLA_FUNCTION_FAILED;
 		} else {
-			/* Save port-speed */
-			switch (be16_to_cpu(ct_rsp->rsp.gpsc.speed)) {
-			case BIT_15:
-				list[i].fp_speed = PORT_SPEED_1GB;
-				break;
-			case BIT_14:
-				list[i].fp_speed = PORT_SPEED_2GB;
-				break;
-			case BIT_13:
-				list[i].fp_speed = PORT_SPEED_4GB;
-				break;
-			case BIT_12:
-				list[i].fp_speed = PORT_SPEED_10GB;
-				break;
-			case BIT_11:
-				list[i].fp_speed = PORT_SPEED_8GB;
-				break;
-			case BIT_10:
-				list[i].fp_speed = PORT_SPEED_16GB;
-				break;
-			case BIT_8:
-				list[i].fp_speed = PORT_SPEED_32GB;
-				break;
-			}
-
+			list->fp_speed = qla2x00_port_speed_capability(
+			    be16_to_cpu(ct_rsp->rsp.gpsc.speed));
 			ql_dbg(ql_dbg_disc, vha, 0x205b,
 			    "GPSC ext entry - fpn "
 			    "%8phN speeds=%04x speed=%04x.\n",
@@ -3031,6 +3026,8 @@ static void qla24xx_async_gpsc_sp_done(void *s, int res)
 	    "Async done-%s res %x, WWPN %8phC \n",
 	    sp->name, res, fcport->port_name);
 
+	fcport->flags &= ~(FCF_ASYNC_SENT | FCF_ASYNC_ACTIVE);
+
 	if (res == QLA_FUNCTION_TIMEOUT)
 		return;
 
@@ -3048,29 +3045,8 @@ static void qla24xx_async_gpsc_sp_done(void *s, int res)
 			goto done;
 		}
 	} else {
-		switch (be16_to_cpu(ct_rsp->rsp.gpsc.speed)) {
-		case BIT_15:
-			fcport->fp_speed = PORT_SPEED_1GB;
-			break;
-		case BIT_14:
-			fcport->fp_speed = PORT_SPEED_2GB;
-			break;
-		case BIT_13:
-			fcport->fp_speed = PORT_SPEED_4GB;
-			break;
-		case BIT_12:
-			fcport->fp_speed = PORT_SPEED_10GB;
-			break;
-		case BIT_11:
-			fcport->fp_speed = PORT_SPEED_8GB;
-			break;
-		case BIT_10:
-			fcport->fp_speed = PORT_SPEED_16GB;
-			break;
-		case BIT_8:
-			fcport->fp_speed = PORT_SPEED_32GB;
-			break;
-		}
+		fcport->fp_speed = qla2x00_port_speed_capability(
+		    be16_to_cpu(ct_rsp->rsp.gpsc.speed));
 
 		ql_dbg(ql_dbg_disc, vha, 0x2054,
 		    "Async-%s OUT WWPN %8phC speeds=%04x speed=%04x.\n",
@@ -4370,6 +4346,7 @@ int qla24xx_async_gnnid(scsi_qla_host_t *vha, fc_port_t *fcport)
 
 done_free_sp:
 	sp->free(sp);
+	fcport->flags &= ~FCF_ASYNC_SENT;
 done:
 	return rval;
 }
diff --git a/drivers/scsi/qla2xxx/qla_init.c b/drivers/scsi/qla2xxx/qla_init.c
index 0c700b140ce7..54772d4c377f 100644
--- a/drivers/scsi/qla2xxx/qla_init.c
+++ b/drivers/scsi/qla2xxx/qla_init.c
@@ -95,6 +95,79 @@ qla2x00_get_async_timeout(struct scsi_qla_host *vha)
 	return tmo;
 }
 
+static void qla24xx_abort_iocb_timeout(void *data)
+{
+	srb_t *sp = data;
+	struct srb_iocb *abt = &sp->u.iocb_cmd;
+
+	abt->u.abt.comp_status = CS_TIMEOUT;
+	sp->done(sp, QLA_FUNCTION_TIMEOUT);
+}
+
+static void qla24xx_abort_sp_done(void *ptr, int res)
+{
+	srb_t *sp = ptr;
+	struct srb_iocb *abt = &sp->u.iocb_cmd;
+
+	if (del_timer(&sp->u.iocb_cmd.timer)) {
+		if (sp->flags & SRB_WAKEUP_ON_COMP)
+			complete(&abt->u.abt.comp);
+		else
+			sp->free(sp);
+	}
+}
+
+static int qla24xx_async_abort_cmd(srb_t *cmd_sp, bool wait)
+{
+	scsi_qla_host_t *vha = cmd_sp->vha;
+	struct srb_iocb *abt_iocb;
+	srb_t *sp;
+	int rval = QLA_FUNCTION_FAILED;
+
+	sp = qla2xxx_get_qpair_sp(cmd_sp->vha, cmd_sp->qpair, cmd_sp->fcport,
+				  GFP_ATOMIC);
+	if (!sp)
+		goto done;
+
+	abt_iocb = &sp->u.iocb_cmd;
+	sp->type = SRB_ABT_CMD;
+	sp->name = "abort";
+	sp->qpair = cmd_sp->qpair;
+	if (wait)
+		sp->flags = SRB_WAKEUP_ON_COMP;
+
+	abt_iocb->timeout = qla24xx_abort_iocb_timeout;
+	init_completion(&abt_iocb->u.abt.comp);
+	/* FW can send 2 x ABTS's timeout/20s */
+	qla2x00_init_timer(sp, 42);
+
+	abt_iocb->u.abt.cmd_hndl = cmd_sp->handle;
+	abt_iocb->u.abt.req_que_no = cpu_to_le16(cmd_sp->qpair->req->id);
+
+	sp->done = qla24xx_abort_sp_done;
+
+	ql_dbg(ql_dbg_async, vha, 0x507c,
+	       "Abort command issued - hdl=%x, type=%x\n", cmd_sp->handle,
+	       cmd_sp->type);
+
+	rval = qla2x00_start_sp(sp);
+	if (rval != QLA_SUCCESS)
+		goto done_free_sp;
+
+	if (wait) {
+		wait_for_completion(&abt_iocb->u.abt.comp);
+		rval = abt_iocb->u.abt.comp_status == CS_COMPLETE ?
+			QLA_SUCCESS : QLA_FUNCTION_FAILED;
+	} else {
+		goto done;
+	}
+
+done_free_sp:
+	sp->free(sp);
+done:
+	return rval;
+}
+
 void
 qla2x00_async_iocb_timeout(void *data)
 {
@@ -514,6 +587,72 @@ done:
 	return rval;
 }
 
+static bool qla2x00_is_reserved_id(scsi_qla_host_t *vha, uint16_t loop_id)
+{
+	struct qla_hw_data *ha = vha->hw;
+
+	if (IS_FWI2_CAPABLE(ha))
+		return loop_id > NPH_LAST_HANDLE;
+
+	return (loop_id > ha->max_loop_id && loop_id < SNS_FIRST_LOOP_ID) ||
+		loop_id == MANAGEMENT_SERVER || loop_id == BROADCAST;
+}
+
+/**
+ * qla2x00_find_new_loop_id - scan through our port list and find a new usable loop ID
+ * @vha: adapter state pointer.
+ * @dev: port structure pointer.
+ *
+ * Returns:
+ *	qla2x00 local function return status code.
+ *
+ * Context:
+ *	Kernel context.
+ */
+static int qla2x00_find_new_loop_id(scsi_qla_host_t *vha, fc_port_t *dev)
+{
+	int	rval;
+	struct qla_hw_data *ha = vha->hw;
+	unsigned long flags = 0;
+
+	rval = QLA_SUCCESS;
+
+	spin_lock_irqsave(&ha->vport_slock, flags);
+
+	dev->loop_id = find_first_zero_bit(ha->loop_id_map, LOOPID_MAP_SIZE);
+	if (dev->loop_id >= LOOPID_MAP_SIZE ||
+	    qla2x00_is_reserved_id(vha, dev->loop_id)) {
+		dev->loop_id = FC_NO_LOOP_ID;
+		rval = QLA_FUNCTION_FAILED;
+	} else {
+		set_bit(dev->loop_id, ha->loop_id_map);
+	}
+	spin_unlock_irqrestore(&ha->vport_slock, flags);
+
+	if (rval == QLA_SUCCESS)
+		ql_dbg(ql_dbg_disc, dev->vha, 0x2086,
+		       "Assigning new loopid=%x, portid=%x.\n",
+		       dev->loop_id, dev->d_id.b24);
+	else
+		ql_log(ql_log_warn, dev->vha, 0x2087,
+		       "No loop_id's available, portid=%x.\n",
+		       dev->d_id.b24);
+
+	return rval;
+}
+
+void qla2x00_clear_loop_id(fc_port_t *fcport)
+{
+	struct qla_hw_data *ha = fcport->vha->hw;
+
+	if (fcport->loop_id == FC_NO_LOOP_ID ||
+	    qla2x00_is_reserved_id(fcport->vha, fcport->loop_id))
+		return;
+
+	clear_bit(fcport->loop_id, ha->loop_id_map);
+	fcport->loop_id = FC_NO_LOOP_ID;
+}
+
 static void qla24xx_handle_gnl_done_event(scsi_qla_host_t *vha,
 	struct event_arg *ea)
 {
@@ -1482,6 +1621,7 @@ int qla24xx_post_newsess_work(struct scsi_qla_host *vha, port_id_t *id,
     u8 *port_name, u8 *node_name, void *pla, u8 fc4_type)
 {
 	struct qla_work_evt *e;
+
 	e = qla2x00_alloc_work(vha, QLA_EVT_NEW_SESS);
 	if (!e)
 		return QLA_FUNCTION_FAILED;
@@ -1558,6 +1698,7 @@ void qla2x00_fcport_event_handler(scsi_qla_host_t *vha, struct event_arg *ea)
 			return;
 		{
 			unsigned long flags;
+
 			fcport = qla2x00_find_fcport_by_nportid
 				(vha, &ea->id, 1);
 			if (fcport) {
@@ -1620,21 +1761,21 @@ void qla2x00_fcport_event_handler(scsi_qla_host_t *vha, struct event_arg *ea)
  */
 void qla_rscn_replay(fc_port_t *fcport)
 {
-       struct event_arg ea;
+	struct event_arg ea;
 
-       switch (fcport->disc_state) {
-       case DSC_DELETE_PEND:
-               return;
-       default:
-               break;
-       }
+	switch (fcport->disc_state) {
+	case DSC_DELETE_PEND:
+		return;
+	default:
+		break;
+	}
 
-       if (fcport->scan_needed) {
-               memset(&ea, 0, sizeof(ea));
-               ea.event = FCME_RSCN;
-               ea.id = fcport->d_id;
-               ea.id.b.rsvd_1 = RSCN_PORT_ADDR;
-               qla2x00_fcport_event_handler(fcport->vha, &ea);
+	if (fcport->scan_needed) {
+		memset(&ea, 0, sizeof(ea));
+		ea.event = FCME_RSCN;
+		ea.id = fcport->d_id;
+		ea.id.b.rsvd_1 = RSCN_PORT_ADDR;
+		qla2x00_fcport_event_handler(fcport->vha, &ea);
 	}
 }
 
@@ -1717,82 +1858,6 @@ done:
 	return rval;
 }
 
-static void
-qla24xx_abort_iocb_timeout(void *data)
-{
-	srb_t *sp = data;
-	struct srb_iocb *abt = &sp->u.iocb_cmd;
-
-	abt->u.abt.comp_status = CS_TIMEOUT;
-	sp->done(sp, QLA_FUNCTION_TIMEOUT);
-}
-
-static void
-qla24xx_abort_sp_done(void *ptr, int res)
-{
-	srb_t *sp = ptr;
-	struct srb_iocb *abt = &sp->u.iocb_cmd;
-
-	if (del_timer(&sp->u.iocb_cmd.timer)) {
-		if (sp->flags & SRB_WAKEUP_ON_COMP)
-			complete(&abt->u.abt.comp);
-		else
-			sp->free(sp);
-	}
-}
-
-int
-qla24xx_async_abort_cmd(srb_t *cmd_sp, bool wait)
-{
-	scsi_qla_host_t *vha = cmd_sp->vha;
-	struct srb_iocb *abt_iocb;
-	srb_t *sp;
-	int rval = QLA_FUNCTION_FAILED;
-
-	sp = qla2xxx_get_qpair_sp(cmd_sp->vha, cmd_sp->qpair, cmd_sp->fcport,
-	    GFP_ATOMIC);
-	if (!sp)
-		goto done;
-
-	abt_iocb = &sp->u.iocb_cmd;
-	sp->type = SRB_ABT_CMD;
-	sp->name = "abort";
-	sp->qpair = cmd_sp->qpair;
-	if (wait)
-		sp->flags = SRB_WAKEUP_ON_COMP;
-
-	abt_iocb->timeout = qla24xx_abort_iocb_timeout;
-	init_completion(&abt_iocb->u.abt.comp);
-	/* FW can send 2 x ABTS's timeout/20s */
-	qla2x00_init_timer(sp, 42);
-
-	abt_iocb->u.abt.cmd_hndl = cmd_sp->handle;
-	abt_iocb->u.abt.req_que_no = cpu_to_le16(cmd_sp->qpair->req->id);
-
-	sp->done = qla24xx_abort_sp_done;
-
-	ql_dbg(ql_dbg_async, vha, 0x507c,
-	    "Abort command issued - hdl=%x, type=%x\n",
-	    cmd_sp->handle, cmd_sp->type);
-
-	rval = qla2x00_start_sp(sp);
-	if (rval != QLA_SUCCESS)
-		goto done_free_sp;
-
-	if (wait) {
-		wait_for_completion(&abt_iocb->u.abt.comp);
-		rval = abt_iocb->u.abt.comp_status == CS_COMPLETE ?
-			QLA_SUCCESS : QLA_FUNCTION_FAILED;
-	} else {
-		goto done;
-	}
-
-done_free_sp:
-	sp->free(sp);
-done:
-	return rval;
-}
-
 int
 qla24xx_async_abort_command(srb_t *sp)
 {
@@ -2102,6 +2167,7 @@ qla2x00_initialize_adapter(scsi_qla_host_t *vha)
 	int	rval;
 	struct qla_hw_data *ha = vha->hw;
 	struct req_que *req = ha->req_q_map[0];
+	struct device_reg_24xx __iomem *reg = &ha->iobase->isp24;
 
 	memset(&vha->qla_stats, 0, sizeof(vha->qla_stats));
 	memset(&vha->fc_host_stat, 0, sizeof(vha->fc_host_stat));
@@ -2136,6 +2202,15 @@ qla2x00_initialize_adapter(scsi_qla_host_t *vha)
 
 	ha->isp_ops->reset_chip(vha);
 
+	/* Check for secure flash support */
+	if (IS_QLA28XX(ha)) {
+		if (RD_REG_DWORD(&reg->mailbox12) & BIT_0) {
+			ql_log(ql_log_info, vha, 0xffff, "Adapter is Secure\n");
+			ha->flags.secure_adapter = 1;
+		}
+	}
+
+
 	rval = qla2xxx_get_flash_info(vha);
 	if (rval) {
 		ql_log(ql_log_fatal, vha, 0x004f,
@@ -2452,7 +2527,7 @@ qla2x00_isp_firmware(scsi_qla_host_t *vha)
  *
  * Returns 0 on success.
  */
-void
+int
 qla2x00_reset_chip(scsi_qla_host_t *vha)
 {
 	unsigned long   flags = 0;
@@ -2460,9 +2535,10 @@ qla2x00_reset_chip(scsi_qla_host_t *vha)
 	struct device_reg_2xxx __iomem *reg = &ha->iobase->isp;
 	uint32_t	cnt;
 	uint16_t	cmd;
+	int rval = QLA_FUNCTION_FAILED;
 
 	if (unlikely(pci_channel_offline(ha->pdev)))
-		return;
+		return rval;
 
 	ha->isp_ops->disable_intrs(ha);
 
@@ -2588,6 +2664,8 @@ qla2x00_reset_chip(scsi_qla_host_t *vha)
 	}
 
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
+
+	return QLA_SUCCESS;
 }
 
 /**
@@ -2828,14 +2906,15 @@ acquired:
  *
  * Returns 0 on success.
  */
-void
+int
 qla24xx_reset_chip(scsi_qla_host_t *vha)
 {
 	struct qla_hw_data *ha = vha->hw;
+	int rval = QLA_FUNCTION_FAILED;
 
 	if (pci_channel_offline(ha->pdev) &&
 	    ha->flags.pci_channel_io_perm_failure) {
-		return;
+		return rval;
 	}
 
 	ha->isp_ops->disable_intrs(ha);
@@ -2843,7 +2922,9 @@ qla24xx_reset_chip(scsi_qla_host_t *vha)
 	qla25xx_manipulate_risc_semaphore(vha);
 
 	/* Perform RISC reset. */
-	qla24xx_reset_risc(vha);
+	rval = qla24xx_reset_risc(vha);
+
+	return rval;
 }
 
 /**
@@ -3018,7 +3099,7 @@ qla2x00_alloc_offload_mem(scsi_qla_host_t *vha)
 	if (IS_FWI2_CAPABLE(ha)) {
 		/* Allocate memory for Fibre Channel Event Buffer. */
 		if (!IS_QLA25XX(ha) && !IS_QLA81XX(ha) && !IS_QLA83XX(ha) &&
-		    !IS_QLA27XX(ha))
+		    !IS_QLA27XX(ha) && !IS_QLA28XX(ha))
 			goto try_eft;
 
 		if (ha->fce)
@@ -3089,12 +3170,15 @@ eft_err:
 void
 qla2x00_alloc_fw_dump(scsi_qla_host_t *vha)
 {
+	int rval;
 	uint32_t dump_size, fixed_size, mem_size, req_q_size, rsp_q_size,
 	    eft_size, fce_size, mq_size;
 	struct qla_hw_data *ha = vha->hw;
 	struct req_que *req = ha->req_q_map[0];
 	struct rsp_que *rsp = ha->rsp_q_map[0];
 	struct qla2xxx_fw_dump *fw_dump;
+	dma_addr_t tc_dma;
+	void *tc;
 
 	dump_size = fixed_size = mem_size = eft_size = fce_size = mq_size = 0;
 	req_q_size = rsp_q_size = 0;
@@ -3106,7 +3190,7 @@ qla2x00_alloc_fw_dump(scsi_qla_host_t *vha)
 		mem_size = (ha->fw_memory_size - 0x11000 + 1) *
 		    sizeof(uint16_t);
 	} else if (IS_FWI2_CAPABLE(ha)) {
-		if (IS_QLA83XX(ha) || IS_QLA27XX(ha))
+		if (IS_QLA83XX(ha) || IS_QLA27XX(ha) || IS_QLA28XX(ha))
 			fixed_size = offsetof(struct qla83xx_fw_dump, ext_mem);
 		else if (IS_QLA81XX(ha))
 			fixed_size = offsetof(struct qla81xx_fw_dump, ext_mem);
@@ -3118,40 +3202,72 @@ qla2x00_alloc_fw_dump(scsi_qla_host_t *vha)
 		mem_size = (ha->fw_memory_size - 0x100000 + 1) *
 		    sizeof(uint32_t);
 		if (ha->mqenable) {
-			if (!IS_QLA83XX(ha) && !IS_QLA27XX(ha))
+			if (!IS_QLA83XX(ha) && !IS_QLA27XX(ha) &&
+			    !IS_QLA28XX(ha))
 				mq_size = sizeof(struct qla2xxx_mq_chain);
 			/*
-			 * Allocate maximum buffer size for all queues.
+			 * Allocate maximum buffer size for all queues - Q0.
 			 * Resizing must be done at end-of-dump processing.
 			 */
-			mq_size += ha->max_req_queues *
+			mq_size += (ha->max_req_queues - 1) *
 			    (req->length * sizeof(request_t));
-			mq_size += ha->max_rsp_queues *
+			mq_size += (ha->max_rsp_queues - 1) *
 			    (rsp->length * sizeof(response_t));
 		}
 		if (ha->tgt.atio_ring)
 			mq_size += ha->tgt.atio_q_length * sizeof(request_t);
 		/* Allocate memory for Fibre Channel Event Buffer. */
 		if (!IS_QLA25XX(ha) && !IS_QLA81XX(ha) && !IS_QLA83XX(ha) &&
-		    !IS_QLA27XX(ha))
+		    !IS_QLA27XX(ha) && !IS_QLA28XX(ha))
 			goto try_eft;
 
 		fce_size = sizeof(struct qla2xxx_fce_chain) + FCE_SIZE;
 try_eft:
+		if (ha->eft)
+			dma_free_coherent(&ha->pdev->dev,
+			    EFT_SIZE, ha->eft, ha->eft_dma);
+
+		/* Allocate memory for Extended Trace Buffer. */
+		tc = dma_alloc_coherent(&ha->pdev->dev, EFT_SIZE, &tc_dma,
+					 GFP_KERNEL);
+		if (!tc) {
+			ql_log(ql_log_warn, vha, 0x00c1,
+			    "Unable to allocate (%d KB) for EFT.\n",
+			    EFT_SIZE / 1024);
+			goto allocate;
+		}
+
+		rval = qla2x00_enable_eft_trace(vha, tc_dma, EFT_NUM_BUFFERS);
+		if (rval) {
+			ql_log(ql_log_warn, vha, 0x00c2,
+			    "Unable to initialize EFT (%d).\n", rval);
+			dma_free_coherent(&ha->pdev->dev, EFT_SIZE, tc,
+			    tc_dma);
+		}
 		ql_dbg(ql_dbg_init, vha, 0x00c3,
 		    "Allocated (%d KB) EFT ...\n", EFT_SIZE / 1024);
 		eft_size = EFT_SIZE;
 	}
 
-	if (IS_QLA27XX(ha)) {
-		if (!ha->fw_dump_template) {
-			ql_log(ql_log_warn, vha, 0x00ba,
-			    "Failed missing fwdump template\n");
-			return;
+	if (IS_QLA27XX(ha) || IS_QLA28XX(ha)) {
+		struct fwdt *fwdt = ha->fwdt;
+		uint j;
+
+		for (j = 0; j < 2; j++, fwdt++) {
+			if (!fwdt->template) {
+				ql_log(ql_log_warn, vha, 0x00ba,
+				    "-> fwdt%u no template\n", j);
+				continue;
+			}
+			ql_dbg(ql_dbg_init, vha, 0x00fa,
+			    "-> fwdt%u calculating fwdump size...\n", j);
+			fwdt->dump_size = qla27xx_fwdt_calculate_dump_size(
+			    vha, fwdt->template);
+			ql_dbg(ql_dbg_init, vha, 0x00fa,
+			    "-> fwdt%u calculated fwdump size = %#lx bytes\n",
+			    j, fwdt->dump_size);
+			dump_size += fwdt->dump_size;
 		}
-		dump_size = qla27xx_fwdt_calculate_dump_size(vha);
-		ql_dbg(ql_dbg_init, vha, 0x00fa,
-		    "-> allocating fwdump (%x bytes)...\n", dump_size);
 		goto allocate;
 	}
 
@@ -3170,42 +3286,66 @@ try_eft:
 			ha->exlogin_size;
 
 allocate:
-	if (!ha->fw_dump_len || dump_size != ha->fw_dump_len) {
+	if (!ha->fw_dump_len || dump_size > ha->fw_dump_alloc_len) {
+
+		ql_dbg(ql_dbg_init, vha, 0x00c5,
+		    "%s dump_size %d fw_dump_len %d fw_dump_alloc_len %d\n",
+		    __func__, dump_size, ha->fw_dump_len,
+		    ha->fw_dump_alloc_len);
+
 		fw_dump = vmalloc(dump_size);
 		if (!fw_dump) {
 			ql_log(ql_log_warn, vha, 0x00c4,
 			    "Unable to allocate (%d KB) for firmware dump.\n",
 			    dump_size / 1024);
 		} else {
-			if (ha->fw_dump)
+			mutex_lock(&ha->optrom_mutex);
+			if (ha->fw_dumped) {
+				memcpy(fw_dump, ha->fw_dump, ha->fw_dump_len);
 				vfree(ha->fw_dump);
-			ha->fw_dump = fw_dump;
-
-			ha->fw_dump_len = dump_size;
-			ql_dbg(ql_dbg_init, vha, 0x00c5,
-			    "Allocated (%d KB) for firmware dump.\n",
-			    dump_size / 1024);
-
-			if (IS_QLA27XX(ha))
-				return;
-
-			ha->fw_dump->signature[0] = 'Q';
-			ha->fw_dump->signature[1] = 'L';
-			ha->fw_dump->signature[2] = 'G';
-			ha->fw_dump->signature[3] = 'C';
-			ha->fw_dump->version = htonl(1);
-
-			ha->fw_dump->fixed_size = htonl(fixed_size);
-			ha->fw_dump->mem_size = htonl(mem_size);
-			ha->fw_dump->req_q_size = htonl(req_q_size);
-			ha->fw_dump->rsp_q_size = htonl(rsp_q_size);
-
-			ha->fw_dump->eft_size = htonl(eft_size);
-			ha->fw_dump->eft_addr_l = htonl(LSD(ha->eft_dma));
-			ha->fw_dump->eft_addr_h = htonl(MSD(ha->eft_dma));
+				ha->fw_dump = fw_dump;
+				ha->fw_dump_alloc_len =  dump_size;
+				ql_dbg(ql_dbg_init, vha, 0x00c5,
+				    "Re-Allocated (%d KB) and save firmware dump.\n",
+				    dump_size / 1024);
+			} else {
+				if (ha->fw_dump)
+					vfree(ha->fw_dump);
+				ha->fw_dump = fw_dump;
+
+				ha->fw_dump_len = ha->fw_dump_alloc_len =
+				    dump_size;
+				ql_dbg(ql_dbg_init, vha, 0x00c5,
+				    "Allocated (%d KB) for firmware dump.\n",
+				    dump_size / 1024);
+
+				if (IS_QLA27XX(ha) || IS_QLA28XX(ha)) {
+					mutex_unlock(&ha->optrom_mutex);
+					return;
+				}
 
-			ha->fw_dump->header_size =
-				htonl(offsetof(struct qla2xxx_fw_dump, isp));
+				ha->fw_dump->signature[0] = 'Q';
+				ha->fw_dump->signature[1] = 'L';
+				ha->fw_dump->signature[2] = 'G';
+				ha->fw_dump->signature[3] = 'C';
+				ha->fw_dump->version = htonl(1);
+
+				ha->fw_dump->fixed_size = htonl(fixed_size);
+				ha->fw_dump->mem_size = htonl(mem_size);
+				ha->fw_dump->req_q_size = htonl(req_q_size);
+				ha->fw_dump->rsp_q_size = htonl(rsp_q_size);
+
+				ha->fw_dump->eft_size = htonl(eft_size);
+				ha->fw_dump->eft_addr_l =
+				    htonl(LSD(ha->eft_dma));
+				ha->fw_dump->eft_addr_h =
+				    htonl(MSD(ha->eft_dma));
+
+				ha->fw_dump->header_size =
+					htonl(offsetof
+					    (struct qla2xxx_fw_dump, isp));
+			}
+			mutex_unlock(&ha->optrom_mutex);
 		}
 	}
 }
@@ -3498,7 +3638,8 @@ qla2x00_setup_chip(scsi_qla_host_t *vha)
 			if (rval == QLA_SUCCESS) {
 				qla24xx_detect_sfp(vha);
 
-				if ((IS_QLA83XX(ha) || IS_QLA27XX(ha)) &&
+				if ((IS_QLA83XX(ha) || IS_QLA27XX(ha) ||
+				    IS_QLA28XX(ha)) &&
 				    (ha->zio_mode == QLA_ZIO_MODE_6))
 					qla27xx_set_zio_threshold(vha,
 					    ha->last_zio_threshold);
@@ -3570,7 +3711,7 @@ enable_82xx_npiv:
 		spin_unlock_irqrestore(&ha->hardware_lock, flags);
 	}
 
-	if (IS_QLA27XX(ha))
+	if (IS_QLA27XX(ha) || IS_QLA28XX(ha))
 		ha->flags.fac_supported = 1;
 	else if (rval == QLA_SUCCESS && IS_FAC_REQUIRED(ha)) {
 		uint32_t size;
@@ -3585,7 +3726,8 @@ enable_82xx_npiv:
 			    ha->fw_major_version, ha->fw_minor_version,
 			    ha->fw_subminor_version);
 
-			if (IS_QLA83XX(ha) || IS_QLA27XX(ha)) {
+			if (IS_QLA83XX(ha) || IS_QLA27XX(ha) ||
+			    IS_QLA28XX(ha)) {
 				ha->flags.fac_supported = 0;
 				rval = QLA_SUCCESS;
 			}
@@ -3647,8 +3789,7 @@ qla2x00_update_fw_options(scsi_qla_host_t *vha)
 	ql_dbg(ql_dbg_init + ql_dbg_buffer, vha, 0x0115,
 	    "Serial link options.\n");
 	ql_dump_buffer(ql_dbg_init + ql_dbg_buffer, vha, 0x0109,
-	    (uint8_t *)&ha->fw_seriallink_options,
-	    sizeof(ha->fw_seriallink_options));
+	    ha->fw_seriallink_options, sizeof(ha->fw_seriallink_options));
 
 	ha->fw_options[1] &= ~FO1_SET_EMPHASIS_SWING;
 	if (ha->fw_seriallink_options[3] & BIT_2) {
@@ -3738,7 +3879,7 @@ qla24xx_update_fw_options(scsi_qla_host_t *vha)
 
 	/* Move PUREX, ABTS RX & RIDA to ATIOQ */
 	if (ql2xmvasynctoatio &&
-	    (IS_QLA83XX(ha) || IS_QLA27XX(ha))) {
+	    (IS_QLA83XX(ha) || IS_QLA27XX(ha) || IS_QLA28XX(ha))) {
 		if (qla_tgt_mode_enabled(vha) ||
 		    qla_dual_mode_enabled(vha))
 			ha->fw_options[2] |= BIT_11;
@@ -3746,7 +3887,8 @@ qla24xx_update_fw_options(scsi_qla_host_t *vha)
 			ha->fw_options[2] &= ~BIT_11;
 	}
 
-	if (IS_QLA25XX(ha) || IS_QLA83XX(ha) || IS_QLA27XX(ha)) {
+	if (IS_QLA25XX(ha) || IS_QLA83XX(ha) || IS_QLA27XX(ha) ||
+	    IS_QLA28XX(ha)) {
 		/*
 		 * Tell FW to track each exchange to prevent
 		 * driver from using stale exchange.
@@ -3799,10 +3941,8 @@ qla2x00_config_rings(struct scsi_qla_host *vha)
 	ha->init_cb->response_q_inpointer = cpu_to_le16(0);
 	ha->init_cb->request_q_length = cpu_to_le16(req->length);
 	ha->init_cb->response_q_length = cpu_to_le16(rsp->length);
-	ha->init_cb->request_q_address[0] = cpu_to_le32(LSD(req->dma));
-	ha->init_cb->request_q_address[1] = cpu_to_le32(MSD(req->dma));
-	ha->init_cb->response_q_address[0] = cpu_to_le32(LSD(rsp->dma));
-	ha->init_cb->response_q_address[1] = cpu_to_le32(MSD(rsp->dma));
+	put_unaligned_le64(req->dma, &ha->init_cb->request_q_address);
+	put_unaligned_le64(rsp->dma, &ha->init_cb->response_q_address);
 
 	WRT_REG_WORD(ISP_REQ_Q_IN(ha, reg), 0);
 	WRT_REG_WORD(ISP_REQ_Q_OUT(ha, reg), 0);
@@ -3829,21 +3969,19 @@ qla24xx_config_rings(struct scsi_qla_host *vha)
 	icb->response_q_inpointer = cpu_to_le16(0);
 	icb->request_q_length = cpu_to_le16(req->length);
 	icb->response_q_length = cpu_to_le16(rsp->length);
-	icb->request_q_address[0] = cpu_to_le32(LSD(req->dma));
-	icb->request_q_address[1] = cpu_to_le32(MSD(req->dma));
-	icb->response_q_address[0] = cpu_to_le32(LSD(rsp->dma));
-	icb->response_q_address[1] = cpu_to_le32(MSD(rsp->dma));
+	put_unaligned_le64(req->dma, &icb->request_q_address);
+	put_unaligned_le64(rsp->dma, &icb->response_q_address);
 
 	/* Setup ATIO queue dma pointers for target mode */
 	icb->atio_q_inpointer = cpu_to_le16(0);
 	icb->atio_q_length = cpu_to_le16(ha->tgt.atio_q_length);
-	icb->atio_q_address[0] = cpu_to_le32(LSD(ha->tgt.atio_dma));
-	icb->atio_q_address[1] = cpu_to_le32(MSD(ha->tgt.atio_dma));
+	put_unaligned_le64(ha->tgt.atio_dma, &icb->atio_q_address);
 
 	if (IS_SHADOW_REG_CAPABLE(ha))
 		icb->firmware_options_2 |= cpu_to_le32(BIT_30|BIT_29);
 
-	if (ha->mqenable || IS_QLA83XX(ha) || IS_QLA27XX(ha)) {
+	if (ha->mqenable || IS_QLA83XX(ha) || IS_QLA27XX(ha) ||
+	    IS_QLA28XX(ha)) {
 		icb->qos = cpu_to_le16(QLA_DEFAULT_QUE_QOS);
 		icb->rid = cpu_to_le16(rid);
 		if (ha->flags.msix_enabled) {
@@ -4266,11 +4404,14 @@ qla2x00_set_model_info(scsi_qla_host_t *vha, uint8_t *model, size_t len,
 {
 	char *st, *en;
 	uint16_t index;
+	uint64_t zero[2] = { 0 };
 	struct qla_hw_data *ha = vha->hw;
 	int use_tbl = !IS_QLA24XX_TYPE(ha) && !IS_QLA25XX(ha) &&
 	    !IS_CNA_CAPABLE(ha) && !IS_QLA2031(ha);
 
-	if (memcmp(model, BINZERO, len) != 0) {
+	if (len > sizeof(zero))
+		len = sizeof(zero);
+	if (memcmp(model, &zero, len) != 0) {
 		strncpy(ha->model_number, model, len);
 		st = en = ha->model_number;
 		en += len - 1;
@@ -4357,7 +4498,7 @@ qla2x00_nvram_config(scsi_qla_host_t *vha)
 	rval = QLA_SUCCESS;
 
 	/* Determine NVRAM starting address. */
-	ha->nvram_size = sizeof(nvram_t);
+	ha->nvram_size = sizeof(*nv);
 	ha->nvram_base = 0;
 	if (!IS_QLA2100(ha) && !IS_QLA2200(ha) && !IS_QLA2300(ha))
 		if ((RD_REG_WORD(&reg->ctrl_status) >> 14) == 1)
@@ -4371,16 +4512,15 @@ qla2x00_nvram_config(scsi_qla_host_t *vha)
 	ql_dbg(ql_dbg_init + ql_dbg_buffer, vha, 0x010f,
 	    "Contents of NVRAM.\n");
 	ql_dump_buffer(ql_dbg_init + ql_dbg_buffer, vha, 0x0110,
-	    (uint8_t *)nv, ha->nvram_size);
+	    nv, ha->nvram_size);
 
 	/* Bad NVRAM data, set defaults parameters. */
-	if (chksum || nv->id[0] != 'I' || nv->id[1] != 'S' ||
-	    nv->id[2] != 'P' || nv->id[3] != ' ' || nv->nvram_version < 1) {
+	if (chksum || memcmp("ISP ", nv->id, sizeof(nv->id)) ||
+	    nv->nvram_version < 1) {
 		/* Reset NVRAM data. */
 		ql_log(ql_log_warn, vha, 0x0064,
-		    "Inconsistent NVRAM "
-		    "detected: checksum=0x%x id=%c version=0x%x.\n",
-		    chksum, nv->id[0], nv->nvram_version);
+		    "Inconsistent NVRAM detected: checksum=%#x id=%.4s version=%#x.\n",
+		    chksum, nv->id, nv->nvram_version);
 		ql_log(ql_log_warn, vha, 0x0065,
 		    "Falling back to "
 		    "functioning (yet invalid -- WWPN) defaults.\n");
@@ -4629,7 +4769,7 @@ qla2x00_nvram_config(scsi_qla_host_t *vha)
 			ha->zio_mode = icb->add_firmware_options[0] &
 			    (BIT_3 | BIT_2 | BIT_1 | BIT_0);
 			ha->zio_timer = icb->interrupt_delay_timer ?
-			    icb->interrupt_delay_timer: 2;
+			    icb->interrupt_delay_timer : 2;
 		}
 		icb->add_firmware_options[0] &=
 		    ~(BIT_3 | BIT_2 | BIT_1 | BIT_0);
@@ -4662,7 +4802,7 @@ qla2x00_rport_del(void *data)
 	unsigned long flags;
 
 	spin_lock_irqsave(fcport->vha->host->host_lock, flags);
-	rport = fcport->drport ? fcport->drport: fcport->rport;
+	rport = fcport->drport ? fcport->drport : fcport->rport;
 	fcport->drport = NULL;
 	spin_unlock_irqrestore(fcport->vha->host->host_lock, flags);
 	if (rport) {
@@ -4675,6 +4815,23 @@ qla2x00_rport_del(void *data)
 	}
 }
 
+void qla2x00_set_fcport_state(fc_port_t *fcport, int state)
+{
+	int old_state;
+
+	old_state = atomic_read(&fcport->state);
+	atomic_set(&fcport->state, state);
+
+	/* Don't print state transitions during initial allocation of fcport */
+	if (old_state && old_state != state) {
+		ql_dbg(ql_dbg_disc, fcport->vha, 0x207d,
+		       "FCPort %8phC state transitioned from %s to %s - portid=%02x%02x%02x.\n",
+		       fcport->port_name, port_state_str[old_state],
+		       port_state_str[state], fcport->d_id.b.domain,
+		       fcport->d_id.b.area, fcport->d_id.b.al_pa);
+	}
+}
+
 /**
  * qla2x00_alloc_fcport() - Allocate a generic fcport.
  * @vha: HA context
@@ -4741,6 +4898,8 @@ qla2x00_free_fcport(fc_port_t *fcport)
 
 		fcport->ct_desc.ct_sns = NULL;
 	}
+	list_del(&fcport->list);
+	qla2x00_clear_loop_id(fcport);
 	kfree(fcport);
 }
 
@@ -4762,6 +4921,7 @@ qla2x00_configure_loop(scsi_qla_host_t *vha)
 	int  rval;
 	unsigned long flags, save_flags;
 	struct qla_hw_data *ha = vha->hw;
+
 	rval = QLA_SUCCESS;
 
 	/* Get Initiator ID */
@@ -4943,8 +5103,7 @@ qla2x00_configure_local_loop(scsi_qla_host_t *vha)
 	ql_dbg(ql_dbg_disc, vha, 0x2011,
 	    "Entries in ID list (%d).\n", entries);
 	ql_dump_buffer(ql_dbg_disc + ql_dbg_buffer, vha, 0x2075,
-	    (uint8_t *)ha->gid_list,
-	    entries * sizeof(struct gid_list_info));
+	    ha->gid_list, entries * sizeof(*ha->gid_list));
 
 	if (entries == 0) {
 		spin_lock_irqsave(&vha->work_lock, flags);
@@ -5194,16 +5353,23 @@ qla2x00_reg_remote_port(scsi_qla_host_t *vha, fc_port_t *fcport)
 
 	rport->supported_classes = fcport->supported_classes;
 
-	rport_ids.roles = FC_RPORT_ROLE_UNKNOWN;
+	rport_ids.roles = FC_PORT_ROLE_UNKNOWN;
 	if (fcport->port_type == FCT_INITIATOR)
-		rport_ids.roles |= FC_RPORT_ROLE_FCP_INITIATOR;
+		rport_ids.roles |= FC_PORT_ROLE_FCP_INITIATOR;
 	if (fcport->port_type == FCT_TARGET)
-		rport_ids.roles |= FC_RPORT_ROLE_FCP_TARGET;
+		rport_ids.roles |= FC_PORT_ROLE_FCP_TARGET;
+	if (fcport->port_type & FCT_NVME_INITIATOR)
+		rport_ids.roles |= FC_PORT_ROLE_NVME_INITIATOR;
+	if (fcport->port_type & FCT_NVME_TARGET)
+		rport_ids.roles |= FC_PORT_ROLE_NVME_TARGET;
+	if (fcport->port_type & FCT_NVME_DISCOVERY)
+		rport_ids.roles |= FC_PORT_ROLE_NVME_DISCOVERY;
 
 	ql_dbg(ql_dbg_disc, vha, 0x20ee,
 	    "%s %8phN. rport %p is %s mode\n",
 	    __func__, fcport->port_name, rport,
-	    (fcport->port_type == FCT_TARGET) ? "tgt" : "ini");
+	    (fcport->port_type == FCT_TARGET) ? "tgt" :
+	    ((fcport->port_type & FCT_NVME) ? "nvme" :"ini"));
 
 	fc_remote_port_rolechg(rport, rport_ids.roles);
 }
@@ -5778,55 +5944,6 @@ qla2x00_find_all_fabric_devs(scsi_qla_host_t *vha)
 	return (rval);
 }
 
-/*
- * qla2x00_find_new_loop_id
- *	Scan through our port list and find a new usable loop ID.
- *
- * Input:
- *	ha:	adapter state pointer.
- *	dev:	port structure pointer.
- *
- * Returns:
- *	qla2x00 local function return status code.
- *
- * Context:
- *	Kernel context.
- */
-int
-qla2x00_find_new_loop_id(scsi_qla_host_t *vha, fc_port_t *dev)
-{
-	int	rval;
-	struct qla_hw_data *ha = vha->hw;
-	unsigned long flags = 0;
-
-	rval = QLA_SUCCESS;
-
-	spin_lock_irqsave(&ha->vport_slock, flags);
-
-	dev->loop_id = find_first_zero_bit(ha->loop_id_map,
-	    LOOPID_MAP_SIZE);
-	if (dev->loop_id >= LOOPID_MAP_SIZE ||
-	    qla2x00_is_reserved_id(vha, dev->loop_id)) {
-		dev->loop_id = FC_NO_LOOP_ID;
-		rval = QLA_FUNCTION_FAILED;
-	} else
-		set_bit(dev->loop_id, ha->loop_id_map);
-
-	spin_unlock_irqrestore(&ha->vport_slock, flags);
-
-	if (rval == QLA_SUCCESS)
-		ql_dbg(ql_dbg_disc, dev->vha, 0x2086,
-		    "Assigning new loopid=%x, portid=%x.\n",
-		    dev->loop_id, dev->d_id.b24);
-	else
-		ql_log(ql_log_warn, dev->vha, 0x2087,
-		    "No loop_id's available, portid=%x.\n",
-		    dev->d_id.b24);
-
-	return (rval);
-}
-
-
 /* FW does not set aside Loop id for MGMT Server/FFFFFAh */
 int
 qla2x00_reserve_mgmt_server_loop_id(scsi_qla_host_t *vha)
@@ -6318,6 +6435,7 @@ qla83xx_initiating_reset(scsi_qla_host_t *vha)
 		qla83xx_idc_audit(vha, IDC_AUDIT_TIMESTAMP);
 	} else {
 		const char *state = qla83xx_dev_state_to_string(dev_state);
+
 		ql_log(ql_log_info, vha, 0xb057, "HW State: %s.\n", state);
 
 		/* SV: XXX: Is timeout required here? */
@@ -6639,6 +6757,14 @@ qla2x00_abort_isp(scsi_qla_host_t *vha)
 	if (vha->flags.online) {
 		qla2x00_abort_isp_cleanup(vha);
 
+		if (test_and_clear_bit(ISP_ABORT_TO_ROM, &vha->dpc_flags)) {
+			ha->flags.chip_reset_done = 1;
+			vha->flags.online = 1;
+			status = 0;
+			clear_bit(ISP_ABORT_RETRY, &vha->dpc_flags);
+			return status;
+		}
+
 		if (IS_QLA8031(ha)) {
 			ql_dbg(ql_dbg_p3p, vha, 0xb05c,
 			    "Clearing fcoe driver presence.\n");
@@ -6879,7 +7005,7 @@ qla25xx_init_queues(struct qla_hw_data *ha)
 * Input:
 *      ha = adapter block pointer.
 */
-void
+int
 qla2x00_reset_adapter(scsi_qla_host_t *vha)
 {
 	unsigned long flags = 0;
@@ -6895,17 +7021,20 @@ qla2x00_reset_adapter(scsi_qla_host_t *vha)
 	WRT_REG_WORD(&reg->hccr, HCCR_RELEASE_RISC);
 	RD_REG_WORD(&reg->hccr);			/* PCI Posting. */
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
+
+	return QLA_SUCCESS;
 }
 
-void
+int
 qla24xx_reset_adapter(scsi_qla_host_t *vha)
 {
 	unsigned long flags = 0;
 	struct qla_hw_data *ha = vha->hw;
 	struct device_reg_24xx __iomem *reg = &ha->iobase->isp24;
+	int rval = QLA_SUCCESS;
 
 	if (IS_P3P_TYPE(ha))
-		return;
+		return rval;
 
 	vha->flags.online = 0;
 	ha->isp_ops->disable_intrs(ha);
@@ -6919,6 +7048,8 @@ qla24xx_reset_adapter(scsi_qla_host_t *vha)
 
 	if (IS_NOPOLLING_TYPE(ha))
 		ha->isp_ops->enable_intrs(ha);
+
+	return rval;
 }
 
 /* On sparc systems, obtain port and node WWN from firmware
@@ -6969,34 +7100,33 @@ qla24xx_nvram_config(scsi_qla_host_t *vha)
 		ha->vpd_base = FA_NVRAM_VPD1_ADDR;
 	}
 
-	ha->nvram_size = sizeof(struct nvram_24xx);
+	ha->nvram_size = sizeof(*nv);
 	ha->vpd_size = FA_NVRAM_VPD_SIZE;
 
 	/* Get VPD data into cache */
 	ha->vpd = ha->nvram + VPD_OFFSET;
-	ha->isp_ops->read_nvram(vha, (uint8_t *)ha->vpd,
+	ha->isp_ops->read_nvram(vha, ha->vpd,
 	    ha->nvram_base - FA_NVRAM_FUNC0_ADDR, FA_NVRAM_VPD_SIZE * 4);
 
 	/* Get NVRAM data into cache and calculate checksum. */
 	dptr = (uint32_t *)nv;
-	ha->isp_ops->read_nvram(vha, (uint8_t *)dptr, ha->nvram_base,
-	    ha->nvram_size);
+	ha->isp_ops->read_nvram(vha, dptr, ha->nvram_base, ha->nvram_size);
 	for (cnt = 0, chksum = 0; cnt < ha->nvram_size >> 2; cnt++, dptr++)
 		chksum += le32_to_cpu(*dptr);
 
 	ql_dbg(ql_dbg_init + ql_dbg_buffer, vha, 0x006a,
 	    "Contents of NVRAM\n");
 	ql_dump_buffer(ql_dbg_init + ql_dbg_buffer, vha, 0x010d,
-	    (uint8_t *)nv, ha->nvram_size);
+	    nv, ha->nvram_size);
 
 	/* Bad NVRAM data, set defaults parameters. */
-	if (chksum || nv->id[0] != 'I' || nv->id[1] != 'S' || nv->id[2] != 'P'
-	    || nv->id[3] != ' ' ||
-	    nv->nvram_version < cpu_to_le16(ICB_VERSION)) {
+	if (chksum || memcmp("ISP ", nv->id, sizeof(nv->id)) ||
+	    le16_to_cpu(nv->nvram_version) < ICB_VERSION) {
 		/* Reset NVRAM data. */
 		ql_log(ql_log_warn, vha, 0x006b,
-		    "Inconsistent NVRAM detected: checksum=0x%x id=%c "
-		    "version=0x%x.\n", chksum, nv->id[0], nv->nvram_version);
+		    "Inconsistent NVRAM checksum=%#x id=%.4s version=%#x.\n",
+		    chksum, nv->id, nv->nvram_version);
+		ql_dump_buffer(ql_dbg_init, vha, 0x006b, nv, sizeof(*nv));
 		ql_log(ql_log_warn, vha, 0x006c,
 		    "Falling back to functioning (yet invalid -- WWPN) "
 		    "defaults.\n");
@@ -7104,11 +7234,11 @@ qla24xx_nvram_config(scsi_qla_host_t *vha)
 	ha->flags.disable_risc_code_load = 0;
 	ha->flags.enable_lip_reset = 0;
 	ha->flags.enable_lip_full_login =
-	    le32_to_cpu(nv->host_p) & BIT_10 ? 1: 0;
+	    le32_to_cpu(nv->host_p) & BIT_10 ? 1 : 0;
 	ha->flags.enable_target_reset =
-	    le32_to_cpu(nv->host_p) & BIT_11 ? 1: 0;
+	    le32_to_cpu(nv->host_p) & BIT_11 ? 1 : 0;
 	ha->flags.enable_led_scheme = 0;
-	ha->flags.disable_serdes = le32_to_cpu(nv->host_p) & BIT_5 ? 1: 0;
+	ha->flags.disable_serdes = le32_to_cpu(nv->host_p) & BIT_5 ? 1 : 0;
 
 	ha->operating_mode = (le32_to_cpu(icb->firmware_options_2) &
 	    (BIT_6 | BIT_5 | BIT_4)) >> 4;
@@ -7182,7 +7312,7 @@ qla24xx_nvram_config(scsi_qla_host_t *vha)
 		ha->zio_mode = le32_to_cpu(icb->firmware_options_2) &
 		    (BIT_3 | BIT_2 | BIT_1 | BIT_0);
 		ha->zio_timer = le16_to_cpu(icb->interrupt_delay_timer) ?
-		    le16_to_cpu(icb->interrupt_delay_timer): 2;
+		    le16_to_cpu(icb->interrupt_delay_timer) : 2;
 	}
 	icb->firmware_options_2 &= cpu_to_le32(
 	    ~(BIT_3 | BIT_2 | BIT_1 | BIT_0));
@@ -7205,128 +7335,311 @@ qla24xx_nvram_config(scsi_qla_host_t *vha)
 	return (rval);
 }
 
-uint8_t qla27xx_find_valid_image(struct scsi_qla_host *vha)
+static void
+qla27xx_print_image(struct scsi_qla_host *vha, char *name,
+    struct qla27xx_image_status *image_status)
+{
+	ql_dbg(ql_dbg_init, vha, 0x018b,
+	    "%s %s: mask=%#02x gen=%#04x ver=%u.%u map=%#01x sum=%#08x sig=%#08x\n",
+	    name, "status",
+	    image_status->image_status_mask,
+	    le16_to_cpu(image_status->generation),
+	    image_status->ver_major,
+	    image_status->ver_minor,
+	    image_status->bitmap,
+	    le32_to_cpu(image_status->checksum),
+	    le32_to_cpu(image_status->signature));
+}
+
+static bool
+qla28xx_check_aux_image_status_signature(
+    struct qla27xx_image_status *image_status)
+{
+	ulong signature = le32_to_cpu(image_status->signature);
+
+	return signature != QLA28XX_AUX_IMG_STATUS_SIGN;
+}
+
+static bool
+qla27xx_check_image_status_signature(struct qla27xx_image_status *image_status)
+{
+	ulong signature = le32_to_cpu(image_status->signature);
+
+	return
+	    signature != QLA27XX_IMG_STATUS_SIGN &&
+	    signature != QLA28XX_IMG_STATUS_SIGN;
+}
+
+static ulong
+qla27xx_image_status_checksum(struct qla27xx_image_status *image_status)
+{
+	uint32_t *p = (void *)image_status;
+	uint n = sizeof(*image_status) / sizeof(*p);
+	uint32_t sum = 0;
+
+	for ( ; n--; p++)
+		sum += le32_to_cpup(p);
+
+	return sum;
+}
+
+static inline uint
+qla28xx_component_bitmask(struct qla27xx_image_status *aux, uint bitmask)
+{
+	return aux->bitmap & bitmask ?
+	    QLA27XX_SECONDARY_IMAGE : QLA27XX_PRIMARY_IMAGE;
+}
+
+static void
+qla28xx_component_status(
+    struct active_regions *active_regions, struct qla27xx_image_status *aux)
+{
+	active_regions->aux.board_config =
+	    qla28xx_component_bitmask(aux, QLA28XX_AUX_IMG_BOARD_CONFIG);
+
+	active_regions->aux.vpd_nvram =
+	    qla28xx_component_bitmask(aux, QLA28XX_AUX_IMG_VPD_NVRAM);
+
+	active_regions->aux.npiv_config_0_1 =
+	    qla28xx_component_bitmask(aux, QLA28XX_AUX_IMG_NPIV_CONFIG_0_1);
+
+	active_regions->aux.npiv_config_2_3 =
+	    qla28xx_component_bitmask(aux, QLA28XX_AUX_IMG_NPIV_CONFIG_2_3);
+}
+
+static int
+qla27xx_compare_image_generation(
+    struct qla27xx_image_status *pri_image_status,
+    struct qla27xx_image_status *sec_image_status)
+{
+	/* calculate generation delta as uint16 (this accounts for wrap) */
+	int16_t delta =
+	    le16_to_cpu(pri_image_status->generation) -
+	    le16_to_cpu(sec_image_status->generation);
+
+	ql_dbg(ql_dbg_init, NULL, 0x0180, "generation delta = %d\n", delta);
+
+	return delta;
+}
+
+void
+qla28xx_get_aux_images(
+	struct scsi_qla_host *vha, struct active_regions *active_regions)
 {
-	struct qla27xx_image_status pri_image_status, sec_image_status;
-	uint8_t valid_pri_image, valid_sec_image;
-	uint32_t *wptr;
-	uint32_t cnt, chksum, size;
 	struct qla_hw_data *ha = vha->hw;
+	struct qla27xx_image_status pri_aux_image_status, sec_aux_image_status;
+	bool valid_pri_image = false, valid_sec_image = false;
+	bool active_pri_image = false, active_sec_image = false;
+
+	if (!ha->flt_region_aux_img_status_pri) {
+		ql_dbg(ql_dbg_init, vha, 0x018a, "Primary aux image not addressed\n");
+		goto check_sec_image;
+	}
 
-	valid_pri_image = valid_sec_image = 1;
-	ha->active_image = 0;
-	size = sizeof(struct qla27xx_image_status) / sizeof(uint32_t);
+	qla24xx_read_flash_data(vha, (void *)&pri_aux_image_status,
+	    ha->flt_region_aux_img_status_pri,
+	    sizeof(pri_aux_image_status) >> 2);
+	qla27xx_print_image(vha, "Primary aux image", &pri_aux_image_status);
 
-	if (!ha->flt_region_img_status_pri) {
-		valid_pri_image = 0;
+	if (qla28xx_check_aux_image_status_signature(&pri_aux_image_status)) {
+		ql_dbg(ql_dbg_init, vha, 0x018b,
+		    "Primary aux image signature (%#x) not valid\n",
+		    le32_to_cpu(pri_aux_image_status.signature));
 		goto check_sec_image;
 	}
 
-	qla24xx_read_flash_data(vha, (uint32_t *)(&pri_image_status),
-	    ha->flt_region_img_status_pri, size);
+	if (qla27xx_image_status_checksum(&pri_aux_image_status)) {
+		ql_dbg(ql_dbg_init, vha, 0x018c,
+		    "Primary aux image checksum failed\n");
+		goto check_sec_image;
+	}
+
+	valid_pri_image = true;
+
+	if (pri_aux_image_status.image_status_mask & 1) {
+		ql_dbg(ql_dbg_init, vha, 0x018d,
+		    "Primary aux image is active\n");
+		active_pri_image = true;
+	}
+
+check_sec_image:
+	if (!ha->flt_region_aux_img_status_sec) {
+		ql_dbg(ql_dbg_init, vha, 0x018a,
+		    "Secondary aux image not addressed\n");
+		goto check_valid_image;
+	}
+
+	qla24xx_read_flash_data(vha, (void *)&sec_aux_image_status,
+	    ha->flt_region_aux_img_status_sec,
+	    sizeof(sec_aux_image_status) >> 2);
+	qla27xx_print_image(vha, "Secondary aux image", &sec_aux_image_status);
 
-	if (pri_image_status.signature != QLA27XX_IMG_STATUS_SIGN) {
+	if (qla28xx_check_aux_image_status_signature(&sec_aux_image_status)) {
 		ql_dbg(ql_dbg_init, vha, 0x018b,
-		    "Primary image signature (0x%x) not valid\n",
-		    pri_image_status.signature);
-		valid_pri_image = 0;
+		    "Secondary aux image signature (%#x) not valid\n",
+		    le32_to_cpu(sec_aux_image_status.signature));
+		goto check_valid_image;
+	}
+
+	if (qla27xx_image_status_checksum(&sec_aux_image_status)) {
+		ql_dbg(ql_dbg_init, vha, 0x018c,
+		    "Secondary aux image checksum failed\n");
+		goto check_valid_image;
+	}
+
+	valid_sec_image = true;
+
+	if (sec_aux_image_status.image_status_mask & 1) {
+		ql_dbg(ql_dbg_init, vha, 0x018d,
+		    "Secondary aux image is active\n");
+		active_sec_image = true;
+	}
+
+check_valid_image:
+	if (valid_pri_image && active_pri_image &&
+	    valid_sec_image && active_sec_image) {
+		if (qla27xx_compare_image_generation(&pri_aux_image_status,
+		    &sec_aux_image_status) >= 0) {
+			qla28xx_component_status(active_regions,
+			    &pri_aux_image_status);
+		} else {
+			qla28xx_component_status(active_regions,
+			    &sec_aux_image_status);
+		}
+	} else if (valid_pri_image && active_pri_image) {
+		qla28xx_component_status(active_regions, &pri_aux_image_status);
+	} else if (valid_sec_image && active_sec_image) {
+		qla28xx_component_status(active_regions, &sec_aux_image_status);
+	}
+
+	ql_dbg(ql_dbg_init, vha, 0x018f,
+	    "aux images active: BCFG=%u VPD/NVR=%u NPIV0/1=%u NPIV2/3=%u\n",
+	    active_regions->aux.board_config,
+	    active_regions->aux.vpd_nvram,
+	    active_regions->aux.npiv_config_0_1,
+	    active_regions->aux.npiv_config_2_3);
+}
+
+void
+qla27xx_get_active_image(struct scsi_qla_host *vha,
+    struct active_regions *active_regions)
+{
+	struct qla_hw_data *ha = vha->hw;
+	struct qla27xx_image_status pri_image_status, sec_image_status;
+	bool valid_pri_image = false, valid_sec_image = false;
+	bool active_pri_image = false, active_sec_image = false;
+
+	if (!ha->flt_region_img_status_pri) {
+		ql_dbg(ql_dbg_init, vha, 0x018a, "Primary image not addressed\n");
 		goto check_sec_image;
 	}
 
-	wptr = (uint32_t *)(&pri_image_status);
-	cnt = size;
+	qla24xx_read_flash_data(vha, (void *)(&pri_image_status),
+	    ha->flt_region_img_status_pri, sizeof(pri_image_status) >> 2);
+	qla27xx_print_image(vha, "Primary image", &pri_image_status);
 
-	for (chksum = 0; cnt--; wptr++)
-		chksum += le32_to_cpu(*wptr);
+	if (qla27xx_check_image_status_signature(&pri_image_status)) {
+		ql_dbg(ql_dbg_init, vha, 0x018b,
+		    "Primary image signature (%#x) not valid\n",
+		    le32_to_cpu(pri_image_status.signature));
+		goto check_sec_image;
+	}
 
-	if (chksum) {
+	if (qla27xx_image_status_checksum(&pri_image_status)) {
 		ql_dbg(ql_dbg_init, vha, 0x018c,
-		    "Checksum validation failed for primary image (0x%x)\n",
-		    chksum);
-		valid_pri_image = 0;
+		    "Primary image checksum failed\n");
+		goto check_sec_image;
+	}
+
+	valid_pri_image = true;
+
+	if (pri_image_status.image_status_mask & 1) {
+		ql_dbg(ql_dbg_init, vha, 0x018d,
+		    "Primary image is active\n");
+		active_pri_image = true;
 	}
 
 check_sec_image:
 	if (!ha->flt_region_img_status_sec) {
-		valid_sec_image = 0;
+		ql_dbg(ql_dbg_init, vha, 0x018a, "Secondary image not addressed\n");
 		goto check_valid_image;
 	}
 
 	qla24xx_read_flash_data(vha, (uint32_t *)(&sec_image_status),
-	    ha->flt_region_img_status_sec, size);
+	    ha->flt_region_img_status_sec, sizeof(sec_image_status) >> 2);
+	qla27xx_print_image(vha, "Secondary image", &sec_image_status);
 
-	if (sec_image_status.signature != QLA27XX_IMG_STATUS_SIGN) {
-		ql_dbg(ql_dbg_init, vha, 0x018d,
-		    "Secondary image signature(0x%x) not valid\n",
-		    sec_image_status.signature);
-		valid_sec_image = 0;
+	if (qla27xx_check_image_status_signature(&sec_image_status)) {
+		ql_dbg(ql_dbg_init, vha, 0x018b,
+		    "Secondary image signature (%#x) not valid\n",
+		    le32_to_cpu(sec_image_status.signature));
 		goto check_valid_image;
 	}
 
-	wptr = (uint32_t *)(&sec_image_status);
-	cnt = size;
-	for (chksum = 0; cnt--; wptr++)
-		chksum += le32_to_cpu(*wptr);
-	if (chksum) {
-		ql_dbg(ql_dbg_init, vha, 0x018e,
-		    "Checksum validation failed for secondary image (0x%x)\n",
-		    chksum);
-		valid_sec_image = 0;
+	if (qla27xx_image_status_checksum(&sec_image_status)) {
+		ql_dbg(ql_dbg_init, vha, 0x018c,
+		    "Secondary image checksum failed\n");
+		goto check_valid_image;
+	}
+
+	valid_sec_image = true;
+
+	if (sec_image_status.image_status_mask & 1) {
+		ql_dbg(ql_dbg_init, vha, 0x018d,
+		    "Secondary image is active\n");
+		active_sec_image = true;
 	}
 
 check_valid_image:
-	if (valid_pri_image && (pri_image_status.image_status_mask & 0x1))
-		ha->active_image = QLA27XX_PRIMARY_IMAGE;
-	if (valid_sec_image && (sec_image_status.image_status_mask & 0x1)) {
-		if (!ha->active_image ||
-		    pri_image_status.generation_number <
-		    sec_image_status.generation_number)
-			ha->active_image = QLA27XX_SECONDARY_IMAGE;
+	if (valid_pri_image && active_pri_image)
+		active_regions->global = QLA27XX_PRIMARY_IMAGE;
+
+	if (valid_sec_image && active_sec_image) {
+		if (!active_regions->global ||
+		    qla27xx_compare_image_generation(
+			&pri_image_status, &sec_image_status) < 0) {
+			active_regions->global = QLA27XX_SECONDARY_IMAGE;
+		}
 	}
 
-	ql_dbg(ql_dbg_init + ql_dbg_verbose, vha, 0x018f, "%s image\n",
-	    ha->active_image == 0 ? "default bootld and fw" :
-	    ha->active_image == 1 ? "primary" :
-	    ha->active_image == 2 ? "secondary" :
-	    "Invalid");
+	ql_dbg(ql_dbg_init, vha, 0x018f, "active image %s (%u)\n",
+	    active_regions->global == QLA27XX_DEFAULT_IMAGE ?
+		"default (boot/fw)" :
+	    active_regions->global == QLA27XX_PRIMARY_IMAGE ?
+		"primary" :
+	    active_regions->global == QLA27XX_SECONDARY_IMAGE ?
+		"secondary" : "invalid",
+	    active_regions->global);
+}
 
-	return ha->active_image;
+bool qla24xx_risc_firmware_invalid(uint32_t *dword)
+{
+	return
+	    !(dword[4] | dword[5] | dword[6] | dword[7]) ||
+	    !(~dword[4] | ~dword[5] | ~dword[6] | ~dword[7]);
 }
 
 static int
 qla24xx_load_risc_flash(scsi_qla_host_t *vha, uint32_t *srisc_addr,
     uint32_t faddr)
 {
-	int	rval = QLA_SUCCESS;
-	int	segments, fragment;
-	uint32_t *dcode, dlen;
-	uint32_t risc_addr;
-	uint32_t risc_size;
-	uint32_t i;
+	int rval;
+	uint templates, segments, fragment;
+	ulong i;
+	uint j;
+	ulong dlen;
+	uint32_t *dcode;
+	uint32_t risc_addr, risc_size, risc_attr = 0;
 	struct qla_hw_data *ha = vha->hw;
 	struct req_que *req = ha->req_q_map[0];
+	struct fwdt *fwdt = ha->fwdt;
 
 	ql_dbg(ql_dbg_init, vha, 0x008b,
 	    "FW: Loading firmware from flash (%x).\n", faddr);
 
-	rval = QLA_SUCCESS;
-
-	segments = FA_RISC_CODE_SEGMENTS;
-	dcode = (uint32_t *)req->ring;
-	*srisc_addr = 0;
-
-	if (IS_QLA27XX(ha) &&
-	    qla27xx_find_valid_image(vha) == QLA27XX_SECONDARY_IMAGE)
-		faddr = ha->flt_region_fw_sec;
-
-	/* Validate firmware image by checking version. */
-	qla24xx_read_flash_data(vha, dcode, faddr + 4, 4);
-	for (i = 0; i < 4; i++)
-		dcode[i] = be32_to_cpu(dcode[i]);
-	if ((dcode[0] == 0xffffffff && dcode[1] == 0xffffffff &&
-	    dcode[2] == 0xffffffff && dcode[3] == 0xffffffff) ||
-	    (dcode[0] == 0 && dcode[1] == 0 && dcode[2] == 0 &&
-		dcode[3] == 0)) {
+	dcode = (void *)req->ring;
+	qla24xx_read_flash_data(vha, dcode, faddr, 8);
+	if (qla24xx_risc_firmware_invalid(dcode)) {
 		ql_log(ql_log_fatal, vha, 0x008c,
 		    "Unable to verify the integrity of flash firmware "
 		    "image.\n");
@@ -7337,34 +7650,36 @@ qla24xx_load_risc_flash(scsi_qla_host_t *vha, uint32_t *srisc_addr,
 		return QLA_FUNCTION_FAILED;
 	}
 
-	while (segments && rval == QLA_SUCCESS) {
-		/* Read segment's load information. */
-		qla24xx_read_flash_data(vha, dcode, faddr, 4);
-
+	dcode = (void *)req->ring;
+	*srisc_addr = 0;
+	segments = FA_RISC_CODE_SEGMENTS;
+	for (j = 0; j < segments; j++) {
+		ql_dbg(ql_dbg_init, vha, 0x008d,
+		    "-> Loading segment %u...\n", j);
+		qla24xx_read_flash_data(vha, dcode, faddr, 10);
 		risc_addr = be32_to_cpu(dcode[2]);
-		*srisc_addr = *srisc_addr == 0 ? risc_addr : *srisc_addr;
 		risc_size = be32_to_cpu(dcode[3]);
+		if (!*srisc_addr) {
+			*srisc_addr = risc_addr;
+			risc_attr = be32_to_cpu(dcode[9]);
+		}
 
-		fragment = 0;
-		while (risc_size > 0 && rval == QLA_SUCCESS) {
-			dlen = (uint32_t)(ha->fw_transfer_size >> 2);
+		dlen = ha->fw_transfer_size >> 2;
+		for (fragment = 0; risc_size; fragment++) {
 			if (dlen > risc_size)
 				dlen = risc_size;
 
 			ql_dbg(ql_dbg_init, vha, 0x008e,
-			    "Loading risc segment@ risc addr %x "
-			    "number of dwords 0x%x offset 0x%x.\n",
-			    risc_addr, dlen, faddr);
-
+			    "-> Loading fragment %u: %#x <- %#x (%#lx dwords)...\n",
+			    fragment, risc_addr, faddr, dlen);
 			qla24xx_read_flash_data(vha, dcode, faddr, dlen);
 			for (i = 0; i < dlen; i++)
 				dcode[i] = swab32(dcode[i]);
 
-			rval = qla2x00_load_ram(vha, req->dma, risc_addr,
-			    dlen);
+			rval = qla2x00_load_ram(vha, req->dma, risc_addr, dlen);
 			if (rval) {
 				ql_log(ql_log_fatal, vha, 0x008f,
-				    "Failed to load segment %d of firmware.\n",
+				    "-> Failed load firmware fragment %u.\n",
 				    fragment);
 				return QLA_FUNCTION_FAILED;
 			}
@@ -7372,107 +7687,82 @@ qla24xx_load_risc_flash(scsi_qla_host_t *vha, uint32_t *srisc_addr,
 			faddr += dlen;
 			risc_addr += dlen;
 			risc_size -= dlen;
-			fragment++;
 		}
-
-		/* Next segment. */
-		segments--;
 	}
 
-	if (!IS_QLA27XX(ha))
-		return rval;
+	if (!IS_QLA27XX(ha) && !IS_QLA28XX(ha))
+		return QLA_SUCCESS;
 
-	if (ha->fw_dump_template)
-		vfree(ha->fw_dump_template);
-	ha->fw_dump_template = NULL;
-	ha->fw_dump_template_len = 0;
-
-	ql_dbg(ql_dbg_init, vha, 0x0161,
-	    "Loading fwdump template from %x\n", faddr);
-	qla24xx_read_flash_data(vha, dcode, faddr, 7);
-	risc_size = be32_to_cpu(dcode[2]);
-	ql_dbg(ql_dbg_init, vha, 0x0162,
-	    "-> array size %x dwords\n", risc_size);
-	if (risc_size == 0 || risc_size == ~0)
-		goto default_template;
-
-	dlen = (risc_size - 8) * sizeof(*dcode);
-	ql_dbg(ql_dbg_init, vha, 0x0163,
-	    "-> template allocating %x bytes...\n", dlen);
-	ha->fw_dump_template = vmalloc(dlen);
-	if (!ha->fw_dump_template) {
-		ql_log(ql_log_warn, vha, 0x0164,
-		    "Failed fwdump template allocate %x bytes.\n", risc_size);
-		goto default_template;
-	}
-
-	faddr += 7;
-	risc_size -= 8;
-	dcode = ha->fw_dump_template;
-	qla24xx_read_flash_data(vha, dcode, faddr, risc_size);
-	for (i = 0; i < risc_size; i++)
-		dcode[i] = le32_to_cpu(dcode[i]);
-
-	if (!qla27xx_fwdt_template_valid(dcode)) {
-		ql_log(ql_log_warn, vha, 0x0165,
-		    "Failed fwdump template validate\n");
-		goto default_template;
-	}
-
-	dlen = qla27xx_fwdt_template_size(dcode);
-	ql_dbg(ql_dbg_init, vha, 0x0166,
-	    "-> template size %x bytes\n", dlen);
-	if (dlen > risc_size * sizeof(*dcode)) {
-		ql_log(ql_log_warn, vha, 0x0167,
-		    "Failed fwdump template exceeds array by %zx bytes\n",
-		    (size_t)(dlen - risc_size * sizeof(*dcode)));
-		goto default_template;
-	}
-	ha->fw_dump_template_len = dlen;
-	return rval;
+	templates = (risc_attr & BIT_9) ? 2 : 1;
+	ql_dbg(ql_dbg_init, vha, 0x0160, "-> templates = %u\n", templates);
+	for (j = 0; j < templates; j++, fwdt++) {
+		if (fwdt->template)
+			vfree(fwdt->template);
+		fwdt->template = NULL;
+		fwdt->length = 0;
+
+		dcode = (void *)req->ring;
+		qla24xx_read_flash_data(vha, dcode, faddr, 7);
+		risc_size = be32_to_cpu(dcode[2]);
+		ql_dbg(ql_dbg_init, vha, 0x0161,
+		    "-> fwdt%u template array at %#x (%#x dwords)\n",
+		    j, faddr, risc_size);
+		if (!risc_size || !~risc_size) {
+			ql_dbg(ql_dbg_init, vha, 0x0162,
+			    "-> fwdt%u failed to read array\n", j);
+			goto failed;
+		}
 
-default_template:
-	ql_log(ql_log_warn, vha, 0x0168, "Using default fwdump template\n");
-	if (ha->fw_dump_template)
-		vfree(ha->fw_dump_template);
-	ha->fw_dump_template = NULL;
-	ha->fw_dump_template_len = 0;
-
-	dlen = qla27xx_fwdt_template_default_size();
-	ql_dbg(ql_dbg_init, vha, 0x0169,
-	    "-> template allocating %x bytes...\n", dlen);
-	ha->fw_dump_template = vmalloc(dlen);
-	if (!ha->fw_dump_template) {
-		ql_log(ql_log_warn, vha, 0x016a,
-		    "Failed fwdump template allocate %x bytes.\n", risc_size);
-		goto failed_template;
-	}
-
-	dcode = ha->fw_dump_template;
-	risc_size = dlen / sizeof(*dcode);
-	memcpy(dcode, qla27xx_fwdt_template_default(), dlen);
-	for (i = 0; i < risc_size; i++)
-		dcode[i] = be32_to_cpu(dcode[i]);
-
-	if (!qla27xx_fwdt_template_valid(ha->fw_dump_template)) {
-		ql_log(ql_log_warn, vha, 0x016b,
-		    "Failed fwdump template validate\n");
-		goto failed_template;
-	}
-
-	dlen = qla27xx_fwdt_template_size(ha->fw_dump_template);
-	ql_dbg(ql_dbg_init, vha, 0x016c,
-	    "-> template size %x bytes\n", dlen);
-	ha->fw_dump_template_len = dlen;
-	return rval;
+		/* skip header and ignore checksum */
+		faddr += 7;
+		risc_size -= 8;
+
+		ql_dbg(ql_dbg_init, vha, 0x0163,
+		    "-> fwdt%u template allocate template %#x words...\n",
+		    j, risc_size);
+		fwdt->template = vmalloc(risc_size * sizeof(*dcode));
+		if (!fwdt->template) {
+			ql_log(ql_log_warn, vha, 0x0164,
+			    "-> fwdt%u failed allocate template.\n", j);
+			goto failed;
+		}
 
-failed_template:
-	ql_log(ql_log_warn, vha, 0x016d, "Failed default fwdump template\n");
-	if (ha->fw_dump_template)
-		vfree(ha->fw_dump_template);
-	ha->fw_dump_template = NULL;
-	ha->fw_dump_template_len = 0;
-	return rval;
+		dcode = fwdt->template;
+		qla24xx_read_flash_data(vha, dcode, faddr, risc_size);
+
+		if (!qla27xx_fwdt_template_valid(dcode)) {
+			ql_log(ql_log_warn, vha, 0x0165,
+			    "-> fwdt%u failed template validate\n", j);
+			goto failed;
+		}
+
+		dlen = qla27xx_fwdt_template_size(dcode);
+		ql_dbg(ql_dbg_init, vha, 0x0166,
+		    "-> fwdt%u template size %#lx bytes (%#lx words)\n",
+		    j, dlen, dlen / sizeof(*dcode));
+		if (dlen > risc_size * sizeof(*dcode)) {
+			ql_log(ql_log_warn, vha, 0x0167,
+			    "-> fwdt%u template exceeds array (%-lu bytes)\n",
+			    j, dlen - risc_size * sizeof(*dcode));
+			goto failed;
+		}
+
+		fwdt->length = dlen;
+		ql_dbg(ql_dbg_init, vha, 0x0168,
+		    "-> fwdt%u loaded template ok\n", j);
+
+		faddr += risc_size + 1;
+	}
+
+	return QLA_SUCCESS;
+
+failed:
+	if (fwdt->template)
+		vfree(fwdt->template);
+	fwdt->template = NULL;
+	fwdt->length = 0;
+
+	return QLA_SUCCESS;
 }
 
 #define QLA_FW_URL "http://ldriver.qlogic.com/firmware/"
@@ -7580,94 +7870,73 @@ static int
 qla24xx_load_risc_blob(scsi_qla_host_t *vha, uint32_t *srisc_addr)
 {
 	int	rval;
-	int	segments, fragment;
-	uint32_t *dcode, dlen;
-	uint32_t risc_addr;
-	uint32_t risc_size;
-	uint32_t i;
+	uint templates, segments, fragment;
+	uint32_t *dcode;
+	ulong dlen;
+	uint32_t risc_addr, risc_size, risc_attr = 0;
+	ulong i;
+	uint j;
 	struct fw_blob *blob;
-	const uint32_t *fwcode;
-	uint32_t fwclen;
+	uint32_t *fwcode;
 	struct qla_hw_data *ha = vha->hw;
 	struct req_que *req = ha->req_q_map[0];
+	struct fwdt *fwdt = ha->fwdt;
+
+	ql_dbg(ql_dbg_init, vha, 0x0090,
+	    "-> FW: Loading via request-firmware.\n");
 
-	/* Load firmware blob. */
 	blob = qla2x00_request_firmware(vha);
 	if (!blob) {
-		ql_log(ql_log_warn, vha, 0x0090,
-		    "Firmware image unavailable.\n");
-		ql_log(ql_log_warn, vha, 0x0091,
-		    "Firmware images can be retrieved from: "
-		    QLA_FW_URL ".\n");
+		ql_log(ql_log_warn, vha, 0x0092,
+		    "-> Firmware file not found.\n");
 
 		return QLA_FUNCTION_FAILED;
 	}
 
-	ql_dbg(ql_dbg_init, vha, 0x0092,
-	    "FW: Loading via request-firmware.\n");
-
-	rval = QLA_SUCCESS;
-
-	segments = FA_RISC_CODE_SEGMENTS;
-	dcode = (uint32_t *)req->ring;
-	*srisc_addr = 0;
-	fwcode = (uint32_t *)blob->fw->data;
-	fwclen = 0;
-
-	/* Validate firmware image by checking version. */
-	if (blob->fw->size < 8 * sizeof(uint32_t)) {
+	fwcode = (void *)blob->fw->data;
+	dcode = fwcode;
+	if (qla24xx_risc_firmware_invalid(dcode)) {
 		ql_log(ql_log_fatal, vha, 0x0093,
 		    "Unable to verify integrity of firmware image (%zd).\n",
 		    blob->fw->size);
-		return QLA_FUNCTION_FAILED;
-	}
-	for (i = 0; i < 4; i++)
-		dcode[i] = be32_to_cpu(fwcode[i + 4]);
-	if ((dcode[0] == 0xffffffff && dcode[1] == 0xffffffff &&
-	    dcode[2] == 0xffffffff && dcode[3] == 0xffffffff) ||
-	    (dcode[0] == 0 && dcode[1] == 0 && dcode[2] == 0 &&
-		dcode[3] == 0)) {
-		ql_log(ql_log_fatal, vha, 0x0094,
-		    "Unable to verify integrity of firmware image (%zd).\n",
-		    blob->fw->size);
 		ql_log(ql_log_fatal, vha, 0x0095,
 		    "Firmware data: %08x %08x %08x %08x.\n",
 		    dcode[0], dcode[1], dcode[2], dcode[3]);
 		return QLA_FUNCTION_FAILED;
 	}
 
-	while (segments && rval == QLA_SUCCESS) {
+	dcode = (void *)req->ring;
+	*srisc_addr = 0;
+	segments = FA_RISC_CODE_SEGMENTS;
+	for (j = 0; j < segments; j++) {
+		ql_dbg(ql_dbg_init, vha, 0x0096,
+		    "-> Loading segment %u...\n", j);
 		risc_addr = be32_to_cpu(fwcode[2]);
-		*srisc_addr = *srisc_addr == 0 ? risc_addr : *srisc_addr;
 		risc_size = be32_to_cpu(fwcode[3]);
 
-		/* Validate firmware image size. */
-		fwclen += risc_size * sizeof(uint32_t);
-		if (blob->fw->size < fwclen) {
-			ql_log(ql_log_fatal, vha, 0x0096,
-			    "Unable to verify integrity of firmware image "
-			    "(%zd).\n", blob->fw->size);
-			return QLA_FUNCTION_FAILED;
+		if (!*srisc_addr) {
+			*srisc_addr = risc_addr;
+			risc_attr = be32_to_cpu(fwcode[9]);
 		}
 
-		fragment = 0;
-		while (risc_size > 0 && rval == QLA_SUCCESS) {
-			dlen = (uint32_t)(ha->fw_transfer_size >> 2);
+		dlen = ha->fw_transfer_size >> 2;
+		for (fragment = 0; risc_size; fragment++) {
 			if (dlen > risc_size)
 				dlen = risc_size;
 
 			ql_dbg(ql_dbg_init, vha, 0x0097,
-			    "Loading risc segment@ risc addr %x "
-			    "number of dwords 0x%x.\n", risc_addr, dlen);
+			    "-> Loading fragment %u: %#x <- %#x (%#lx words)...\n",
+			    fragment, risc_addr,
+			    (uint32_t)(fwcode - (typeof(fwcode))blob->fw->data),
+			    dlen);
 
 			for (i = 0; i < dlen; i++)
 				dcode[i] = swab32(fwcode[i]);
 
-			rval = qla2x00_load_ram(vha, req->dma, risc_addr,
-			    dlen);
+			rval = qla2x00_load_ram(vha, req->dma, risc_addr, dlen);
 			if (rval) {
 				ql_log(ql_log_fatal, vha, 0x0098,
-				    "Failed to load segment %d of firmware.\n",
+				    "-> Failed load firmware fragment %u.\n",
 				    fragment);
 				return QLA_FUNCTION_FAILED;
 			}
@@ -7675,106 +7944,82 @@ qla24xx_load_risc_blob(scsi_qla_host_t *vha, uint32_t *srisc_addr)
 			fwcode += dlen;
 			risc_addr += dlen;
 			risc_size -= dlen;
-			fragment++;
 		}
-
-		/* Next segment. */
-		segments--;
 	}
 
-	if (!IS_QLA27XX(ha))
-		return rval;
+	if (!IS_QLA27XX(ha) && !IS_QLA28XX(ha))
+		return QLA_SUCCESS;
 
-	if (ha->fw_dump_template)
-		vfree(ha->fw_dump_template);
-	ha->fw_dump_template = NULL;
-	ha->fw_dump_template_len = 0;
-
-	ql_dbg(ql_dbg_init, vha, 0x171,
-	    "Loading fwdump template from %x\n",
-	    (uint32_t)((void *)fwcode - (void *)blob->fw->data));
-	risc_size = be32_to_cpu(fwcode[2]);
-	ql_dbg(ql_dbg_init, vha, 0x172,
-	    "-> array size %x dwords\n", risc_size);
-	if (risc_size == 0 || risc_size == ~0)
-		goto default_template;
-
-	dlen = (risc_size - 8) * sizeof(*fwcode);
-	ql_dbg(ql_dbg_init, vha, 0x0173,
-	    "-> template allocating %x bytes...\n", dlen);
-	ha->fw_dump_template = vmalloc(dlen);
-	if (!ha->fw_dump_template) {
-		ql_log(ql_log_warn, vha, 0x0174,
-		    "Failed fwdump template allocate %x bytes.\n", risc_size);
-		goto default_template;
-	}
-
-	fwcode += 7;
-	risc_size -= 8;
-	dcode = ha->fw_dump_template;
-	for (i = 0; i < risc_size; i++)
-		dcode[i] = le32_to_cpu(fwcode[i]);
-
-	if (!qla27xx_fwdt_template_valid(dcode)) {
-		ql_log(ql_log_warn, vha, 0x0175,
-		    "Failed fwdump template validate\n");
-		goto default_template;
-	}
-
-	dlen = qla27xx_fwdt_template_size(dcode);
-	ql_dbg(ql_dbg_init, vha, 0x0176,
-	    "-> template size %x bytes\n", dlen);
-	if (dlen > risc_size * sizeof(*fwcode)) {
-		ql_log(ql_log_warn, vha, 0x0177,
-		    "Failed fwdump template exceeds array by %zx bytes\n",
-		    (size_t)(dlen - risc_size * sizeof(*fwcode)));
-		goto default_template;
-	}
-	ha->fw_dump_template_len = dlen;
-	return rval;
+	templates = (risc_attr & BIT_9) ? 2 : 1;
+	ql_dbg(ql_dbg_init, vha, 0x0170, "-> templates = %u\n", templates);
+	for (j = 0; j < templates; j++, fwdt++) {
+		if (fwdt->template)
+			vfree(fwdt->template);
+		fwdt->template = NULL;
+		fwdt->length = 0;
+
+		risc_size = be32_to_cpu(fwcode[2]);
+		ql_dbg(ql_dbg_init, vha, 0x0171,
+		    "-> fwdt%u template array at %#x (%#x dwords)\n",
+		    j, (uint32_t)((void *)fwcode - (void *)blob->fw->data),
+		    risc_size);
+		if (!risc_size || !~risc_size) {
+			ql_dbg(ql_dbg_init, vha, 0x0172,
+			    "-> fwdt%u failed to read array\n", j);
+			goto failed;
+		}
 
-default_template:
-	ql_log(ql_log_warn, vha, 0x0178, "Using default fwdump template\n");
-	if (ha->fw_dump_template)
-		vfree(ha->fw_dump_template);
-	ha->fw_dump_template = NULL;
-	ha->fw_dump_template_len = 0;
-
-	dlen = qla27xx_fwdt_template_default_size();
-	ql_dbg(ql_dbg_init, vha, 0x0179,
-	    "-> template allocating %x bytes...\n", dlen);
-	ha->fw_dump_template = vmalloc(dlen);
-	if (!ha->fw_dump_template) {
-		ql_log(ql_log_warn, vha, 0x017a,
-		    "Failed fwdump template allocate %x bytes.\n", risc_size);
-		goto failed_template;
-	}
-
-	dcode = ha->fw_dump_template;
-	risc_size = dlen / sizeof(*fwcode);
-	fwcode = qla27xx_fwdt_template_default();
-	for (i = 0; i < risc_size; i++)
-		dcode[i] = be32_to_cpu(fwcode[i]);
-
-	if (!qla27xx_fwdt_template_valid(ha->fw_dump_template)) {
-		ql_log(ql_log_warn, vha, 0x017b,
-		    "Failed fwdump template validate\n");
-		goto failed_template;
-	}
-
-	dlen = qla27xx_fwdt_template_size(ha->fw_dump_template);
-	ql_dbg(ql_dbg_init, vha, 0x017c,
-	    "-> template size %x bytes\n", dlen);
-	ha->fw_dump_template_len = dlen;
-	return rval;
+		/* skip header and ignore checksum */
+		fwcode += 7;
+		risc_size -= 8;
+
+		ql_dbg(ql_dbg_init, vha, 0x0173,
+		    "-> fwdt%u template allocate template %#x words...\n",
+		    j, risc_size);
+		fwdt->template = vmalloc(risc_size * sizeof(*dcode));
+		if (!fwdt->template) {
+			ql_log(ql_log_warn, vha, 0x0174,
+			    "-> fwdt%u failed allocate template.\n", j);
+			goto failed;
+		}
 
-failed_template:
-	ql_log(ql_log_warn, vha, 0x017d, "Failed default fwdump template\n");
-	if (ha->fw_dump_template)
-		vfree(ha->fw_dump_template);
-	ha->fw_dump_template = NULL;
-	ha->fw_dump_template_len = 0;
-	return rval;
+		dcode = fwdt->template;
+		for (i = 0; i < risc_size; i++)
+			dcode[i] = fwcode[i];
+
+		if (!qla27xx_fwdt_template_valid(dcode)) {
+			ql_log(ql_log_warn, vha, 0x0175,
+			    "-> fwdt%u failed template validate\n", j);
+			goto failed;
+		}
+
+		dlen = qla27xx_fwdt_template_size(dcode);
+		ql_dbg(ql_dbg_init, vha, 0x0176,
+		    "-> fwdt%u template size %#lx bytes (%#lx words)\n",
+		    j, dlen, dlen / sizeof(*dcode));
+		if (dlen > risc_size * sizeof(*dcode)) {
+			ql_log(ql_log_warn, vha, 0x0177,
+			    "-> fwdt%u template exceeds array (%-lu bytes)\n",
+			    j, dlen - risc_size * sizeof(*dcode));
+			goto failed;
+		}
+
+		fwdt->length = dlen;
+		ql_dbg(ql_dbg_init, vha, 0x0178,
+		    "-> fwdt%u loaded template ok\n", j);
+
+		fwcode += risc_size + 1;
+	}
+
+	return QLA_SUCCESS;
+
+failed:
+	if (fwdt->template)
+		vfree(fwdt->template);
+	fwdt->template = NULL;
+	fwdt->length = 0;
+
+	return QLA_SUCCESS;
 }
 
 int
@@ -7803,32 +8048,50 @@ qla81xx_load_risc(scsi_qla_host_t *vha, uint32_t *srisc_addr)
 {
 	int rval;
 	struct qla_hw_data *ha = vha->hw;
+	struct active_regions active_regions = { };
 
 	if (ql2xfwloadbin == 2)
 		goto try_blob_fw;
 
-	/*
-	 * FW Load priority:
+	/* FW Load priority:
 	 * 1) Firmware residing in flash.
 	 * 2) Firmware via request-firmware interface (.bin file).
-	 * 3) Golden-Firmware residing in flash -- limited operation.
+	 * 3) Golden-Firmware residing in flash -- (limited operation).
 	 */
+
+	if (!IS_QLA27XX(ha) && !IS_QLA28XX(ha))
+		goto try_primary_fw;
+
+	qla27xx_get_active_image(vha, &active_regions);
+
+	if (active_regions.global != QLA27XX_SECONDARY_IMAGE)
+		goto try_primary_fw;
+
+	ql_dbg(ql_dbg_init, vha, 0x008b,
+	    "Loading secondary firmware image.\n");
+	rval = qla24xx_load_risc_flash(vha, srisc_addr, ha->flt_region_fw_sec);
+	if (!rval)
+		return rval;
+
+try_primary_fw:
+	ql_dbg(ql_dbg_init, vha, 0x008b,
+	    "Loading primary firmware image.\n");
 	rval = qla24xx_load_risc_flash(vha, srisc_addr, ha->flt_region_fw);
-	if (rval == QLA_SUCCESS)
+	if (!rval)
 		return rval;
 
 try_blob_fw:
 	rval = qla24xx_load_risc_blob(vha, srisc_addr);
-	if (rval == QLA_SUCCESS || !ha->flt_region_gold_fw)
+	if (!rval || !ha->flt_region_gold_fw)
 		return rval;
 
 	ql_log(ql_log_info, vha, 0x0099,
 	    "Attempting to fallback to golden firmware.\n");
 	rval = qla24xx_load_risc_flash(vha, srisc_addr, ha->flt_region_gold_fw);
-	if (rval != QLA_SUCCESS)
+	if (rval)
 		return rval;
 
-	ql_log(ql_log_info, vha, 0x009a, "Update operational firmware.\n");
+	ql_log(ql_log_info, vha, 0x009a, "Need firmware flash update.\n");
 	ha->flags.running_gold_fw = 1;
 	return rval;
 }
@@ -7963,6 +8226,7 @@ void
 qla84xx_put_chip(struct scsi_qla_host *vha)
 {
 	struct qla_hw_data *ha = vha->hw;
+
 	if (ha->cs84xx)
 		kref_put(&ha->cs84xx->kref, __qla84xx_chip_release);
 }
@@ -7980,7 +8244,7 @@ qla84xx_init_chip(scsi_qla_host_t *vha)
 
 	mutex_unlock(&ha->cs84xx->fw_update_mutex);
 
-	return rval != QLA_SUCCESS || status[0] ? QLA_FUNCTION_FAILED:
+	return rval != QLA_SUCCESS || status[0] ? QLA_FUNCTION_FAILED :
 	    QLA_SUCCESS;
 }
 
@@ -7997,25 +8261,48 @@ qla81xx_nvram_config(scsi_qla_host_t *vha)
 	uint32_t chksum;
 	uint16_t cnt;
 	struct qla_hw_data *ha = vha->hw;
+	uint32_t faddr;
+	struct active_regions active_regions = { };
 
 	rval = QLA_SUCCESS;
 	icb = (struct init_cb_81xx *)ha->init_cb;
 	nv = ha->nvram;
 
 	/* Determine NVRAM starting address. */
-	ha->nvram_size = sizeof(struct nvram_81xx);
+	ha->nvram_size = sizeof(*nv);
 	ha->vpd_size = FA_NVRAM_VPD_SIZE;
 	if (IS_P3P_TYPE(ha) || IS_QLA8031(ha))
 		ha->vpd_size = FA_VPD_SIZE_82XX;
 
+	if (IS_QLA28XX(ha) || IS_QLA27XX(ha))
+		qla28xx_get_aux_images(vha, &active_regions);
+
 	/* Get VPD data into cache */
 	ha->vpd = ha->nvram + VPD_OFFSET;
-	ha->isp_ops->read_optrom(vha, ha->vpd, ha->flt_region_vpd << 2,
-	    ha->vpd_size);
+
+	faddr = ha->flt_region_vpd;
+	if (IS_QLA28XX(ha)) {
+		if (active_regions.aux.vpd_nvram == QLA27XX_SECONDARY_IMAGE)
+			faddr = ha->flt_region_vpd_sec;
+		ql_dbg(ql_dbg_init, vha, 0x0110,
+		    "Loading %s nvram image.\n",
+		    active_regions.aux.vpd_nvram == QLA27XX_PRIMARY_IMAGE ?
+		    "primary" : "secondary");
+	}
+	qla24xx_read_flash_data(vha, ha->vpd, faddr, ha->vpd_size >> 2);
 
 	/* Get NVRAM data into cache and calculate checksum. */
-	ha->isp_ops->read_optrom(vha, ha->nvram, ha->flt_region_nvram << 2,
-	    ha->nvram_size);
+	faddr = ha->flt_region_nvram;
+	if (IS_QLA28XX(ha)) {
+		if (active_regions.aux.vpd_nvram == QLA27XX_SECONDARY_IMAGE)
+			faddr = ha->flt_region_nvram_sec;
+	}
+	ql_dbg(ql_dbg_init, vha, 0x0110,
+	    "Loading %s nvram image.\n",
+	    active_regions.aux.vpd_nvram == QLA27XX_PRIMARY_IMAGE ?
+	    "primary" : "secondary");
+	qla24xx_read_flash_data(vha, ha->nvram, faddr, ha->nvram_size >> 2);
+
 	dptr = (uint32_t *)nv;
 	for (cnt = 0, chksum = 0; cnt < ha->nvram_size >> 2; cnt++, dptr++)
 		chksum += le32_to_cpu(*dptr);
@@ -8023,17 +8310,16 @@ qla81xx_nvram_config(scsi_qla_host_t *vha)
 	ql_dbg(ql_dbg_init + ql_dbg_buffer, vha, 0x0111,
 	    "Contents of NVRAM:\n");
 	ql_dump_buffer(ql_dbg_init + ql_dbg_buffer, vha, 0x0112,
-	    (uint8_t *)nv, ha->nvram_size);
+	    nv, ha->nvram_size);
 
 	/* Bad NVRAM data, set defaults parameters. */
-	if (chksum || nv->id[0] != 'I' || nv->id[1] != 'S' || nv->id[2] != 'P'
-	    || nv->id[3] != ' ' ||
-	    nv->nvram_version < cpu_to_le16(ICB_VERSION)) {
+	if (chksum || memcmp("ISP ", nv->id, sizeof(nv->id)) ||
+	    le16_to_cpu(nv->nvram_version) < ICB_VERSION) {
 		/* Reset NVRAM data. */
 		ql_log(ql_log_info, vha, 0x0073,
-		    "Inconsistent NVRAM detected: checksum=0x%x id=%c "
-		    "version=0x%x.\n", chksum, nv->id[0],
-		    le16_to_cpu(nv->nvram_version));
+		    "Inconsistent NVRAM checksum=%#x id=%.4s version=%#x.\n",
+		    chksum, nv->id, le16_to_cpu(nv->nvram_version));
+		ql_dump_buffer(ql_dbg_init, vha, 0x0073, nv, sizeof(*nv));
 		ql_log(ql_log_info, vha, 0x0074,
 		    "Falling back to functioning (yet invalid -- WWPN) "
 		    "defaults.\n");
@@ -8154,11 +8440,11 @@ qla81xx_nvram_config(scsi_qla_host_t *vha)
 	ha->flags.disable_risc_code_load = 0;
 	ha->flags.enable_lip_reset = 0;
 	ha->flags.enable_lip_full_login =
-	    le32_to_cpu(nv->host_p) & BIT_10 ? 1: 0;
+	    le32_to_cpu(nv->host_p) & BIT_10 ? 1 : 0;
 	ha->flags.enable_target_reset =
-	    le32_to_cpu(nv->host_p) & BIT_11 ? 1: 0;
+	    le32_to_cpu(nv->host_p) & BIT_11 ? 1 : 0;
 	ha->flags.enable_led_scheme = 0;
-	ha->flags.disable_serdes = le32_to_cpu(nv->host_p) & BIT_5 ? 1: 0;
+	ha->flags.disable_serdes = le32_to_cpu(nv->host_p) & BIT_5 ? 1 : 0;
 
 	ha->operating_mode = (le32_to_cpu(icb->firmware_options_2) &
 	    (BIT_6 | BIT_5 | BIT_4)) >> 4;
@@ -8222,7 +8508,8 @@ qla81xx_nvram_config(scsi_qla_host_t *vha)
 		ha->login_retry_count = ql2xloginretrycount;
 
 	/* if not running MSI-X we need handshaking on interrupts */
-	if (!vha->hw->flags.msix_enabled && (IS_QLA83XX(ha) || IS_QLA27XX(ha)))
+	if (!vha->hw->flags.msix_enabled &&
+	    (IS_QLA83XX(ha) || IS_QLA27XX(ha) || IS_QLA28XX(ha)))
 		icb->firmware_options_2 |= cpu_to_le32(BIT_22);
 
 	/* Enable ZIO. */
@@ -8230,7 +8517,7 @@ qla81xx_nvram_config(scsi_qla_host_t *vha)
 		ha->zio_mode = le32_to_cpu(icb->firmware_options_2) &
 		    (BIT_3 | BIT_2 | BIT_1 | BIT_0);
 		ha->zio_timer = le16_to_cpu(icb->interrupt_delay_timer) ?
-		    le16_to_cpu(icb->interrupt_delay_timer): 2;
+		    le16_to_cpu(icb->interrupt_delay_timer) : 2;
 	}
 	icb->firmware_options_2 &= cpu_to_le32(
 	    ~(BIT_3 | BIT_2 | BIT_1 | BIT_0));
@@ -8255,12 +8542,6 @@ qla81xx_nvram_config(scsi_qla_host_t *vha)
 	/* N2N: driver will initiate Login instead of FW */
 	icb->firmware_options_3 |= BIT_8;
 
-	if (IS_QLA27XX(ha)) {
-		icb->firmware_options_3 |= BIT_8;
-		ql_dbg(ql_log_info, vha, 0x0075,
-		    "Enabling direct connection.\n");
-	}
-
 	if (rval) {
 		ql_log(ql_log_warn, vha, 0x0076,
 		    "NVRAM configuration failed.\n");
@@ -8621,7 +8902,6 @@ struct qla_qpair *qla2xxx_create_qpair(struct scsi_qla_host *vha, int qos,
 			    "Failed to allocate memory for queue pair.\n");
 			return NULL;
 		}
-		memset(qpair, 0, sizeof(struct qla_qpair));
 
 		qpair->hw = vha->hw;
 		qpair->vha = vha;
@@ -8668,7 +8948,7 @@ struct qla_qpair *qla2xxx_create_qpair(struct scsi_qla_host *vha, int qos,
 		qpair->msix->in_use = 1;
 		list_add_tail(&qpair->qp_list_elem, &vha->qp_list);
 		qpair->pdev = ha->pdev;
-		if (IS_QLA27XX(ha) || IS_QLA83XX(ha))
+		if (IS_QLA27XX(ha) || IS_QLA83XX(ha) || IS_QLA28XX(ha))
 			qpair->reqq_start_iocbs = qla_83xx_start_iocbs;
 
 		mutex_unlock(&ha->mq_lock);
diff --git a/drivers/scsi/qla2xxx/qla_inline.h b/drivers/scsi/qla2xxx/qla_inline.h
index 512c3c37b447..bf063c664352 100644
--- a/drivers/scsi/qla2xxx/qla_inline.h
+++ b/drivers/scsi/qla2xxx/qla_inline.h
@@ -91,43 +91,6 @@ host_to_adap(uint8_t *src, uint8_t *dst, uint32_t bsize)
 }
 
 static inline void
-qla2x00_set_reserved_loop_ids(struct qla_hw_data *ha)
-{
-	int i;
-
-	if (IS_FWI2_CAPABLE(ha))
-		return;
-
-	for (i = 0; i < SNS_FIRST_LOOP_ID; i++)
-		set_bit(i, ha->loop_id_map);
-	set_bit(MANAGEMENT_SERVER, ha->loop_id_map);
-	set_bit(BROADCAST, ha->loop_id_map);
-}
-
-static inline int
-qla2x00_is_reserved_id(scsi_qla_host_t *vha, uint16_t loop_id)
-{
-	struct qla_hw_data *ha = vha->hw;
-	if (IS_FWI2_CAPABLE(ha))
-		return (loop_id > NPH_LAST_HANDLE);
-
-	return ((loop_id > ha->max_loop_id && loop_id < SNS_FIRST_LOOP_ID) ||
-	    loop_id == MANAGEMENT_SERVER || loop_id == BROADCAST);
-}
-
-static inline void
-qla2x00_clear_loop_id(fc_port_t *fcport) {
-	struct qla_hw_data *ha = fcport->vha->hw;
-
-	if (fcport->loop_id == FC_NO_LOOP_ID ||
-	    qla2x00_is_reserved_id(fcport->vha, fcport->loop_id))
-		return;
-
-	clear_bit(fcport->loop_id, ha->loop_id_map);
-	fcport->loop_id = FC_NO_LOOP_ID;
-}
-
-static inline void
 qla2x00_clean_dsd_pool(struct qla_hw_data *ha, struct crc_context *ctx)
 {
 	struct dsd_dma *dsd, *tdsd;
@@ -142,25 +105,6 @@ qla2x00_clean_dsd_pool(struct qla_hw_data *ha, struct crc_context *ctx)
 	INIT_LIST_HEAD(&ctx->dsd_list);
 }
 
-static inline void
-qla2x00_set_fcport_state(fc_port_t *fcport, int state)
-{
-	int old_state;
-
-	old_state = atomic_read(&fcport->state);
-	atomic_set(&fcport->state, state);
-
-	/* Don't print state transitions during initial allocation of fcport */
-	if (old_state && old_state != state) {
-		ql_dbg(ql_dbg_disc, fcport->vha, 0x207d,
-		    "FCPort %8phC state transitioned from %s to %s - "
-			"portid=%02x%02x%02x.\n", fcport->port_name,
-		    port_state_str[old_state], port_state_str[state],
-		    fcport->d_id.b.domain, fcport->d_id.b.area,
-		    fcport->d_id.b.al_pa);
-	}
-}
-
 static inline int
 qla2x00_hba_err_chk_enabled(srb_t *sp)
 {
@@ -240,6 +184,7 @@ done:
 static inline void
 qla2xxx_rel_qpair_sp(struct qla_qpair *qpair, srb_t *sp)
 {
+	sp->qpair = NULL;
 	mempool_free(sp, qpair->srb_mempool);
 	QLA_QPAIR_MARK_NOT_BUSY(qpair);
 }
@@ -274,18 +219,6 @@ qla2x00_rel_sp(srb_t *sp)
 	qla2xxx_rel_qpair_sp(sp->qpair, sp);
 }
 
-static inline void
-qla2x00_init_timer(srb_t *sp, unsigned long tmo)
-{
-	timer_setup(&sp->u.iocb_cmd.timer, qla2x00_sp_timeout, 0);
-	sp->u.iocb_cmd.timer.expires = jiffies + tmo * HZ;
-	sp->free = qla2x00_sp_free;
-	init_completion(&sp->comp);
-	if (IS_QLAFX00(sp->vha->hw) && (sp->type == SRB_FXIOCB_DCMD))
-		init_completion(&sp->u.iocb_cmd.u.fxiocb.fxiocb_comp);
-	add_timer(&sp->u.iocb_cmd.timer);
-}
-
 static inline int
 qla2x00_gid_list_size(struct qla_hw_data *ha)
 {
diff --git a/drivers/scsi/qla2xxx/qla_iocb.c b/drivers/scsi/qla2xxx/qla_iocb.c
index 456a41d2e2c6..9312b19ed708 100644
--- a/drivers/scsi/qla2xxx/qla_iocb.c
+++ b/drivers/scsi/qla2xxx/qla_iocb.c
@@ -107,7 +107,7 @@ qla2x00_prep_cont_type0_iocb(struct scsi_qla_host *vha)
 	cont_pkt = (cont_entry_t *)req->ring_ptr;
 
 	/* Load packet defaults. */
-	*((uint32_t *)(&cont_pkt->entry_type)) = cpu_to_le32(CONTINUE_TYPE);
+	put_unaligned_le32(CONTINUE_TYPE, &cont_pkt->entry_type);
 
 	return (cont_pkt);
 }
@@ -136,9 +136,8 @@ qla2x00_prep_cont_type1_iocb(scsi_qla_host_t *vha, struct req_que *req)
 	cont_pkt = (cont_a64_entry_t *)req->ring_ptr;
 
 	/* Load packet defaults. */
-	*((uint32_t *)(&cont_pkt->entry_type)) = IS_QLAFX00(vha->hw) ?
-	    cpu_to_le32(CONTINUE_A64_TYPE_FX00) :
-	    cpu_to_le32(CONTINUE_A64_TYPE);
+	put_unaligned_le32(IS_QLAFX00(vha->hw) ? CONTINUE_A64_TYPE_FX00 :
+			   CONTINUE_A64_TYPE, &cont_pkt->entry_type);
 
 	return (cont_pkt);
 }
@@ -193,7 +192,7 @@ void qla2x00_build_scsi_iocbs_32(srb_t *sp, cmd_entry_t *cmd_pkt,
     uint16_t tot_dsds)
 {
 	uint16_t	avail_dsds;
-	uint32_t	*cur_dsd;
+	struct dsd32	*cur_dsd;
 	scsi_qla_host_t	*vha;
 	struct scsi_cmnd *cmd;
 	struct scatterlist *sg;
@@ -202,8 +201,7 @@ void qla2x00_build_scsi_iocbs_32(srb_t *sp, cmd_entry_t *cmd_pkt,
 	cmd = GET_CMD_SP(sp);
 
 	/* Update entry type to indicate Command Type 2 IOCB */
-	*((uint32_t *)(&cmd_pkt->entry_type)) =
-	    cpu_to_le32(COMMAND_TYPE);
+	put_unaligned_le32(COMMAND_TYPE, &cmd_pkt->entry_type);
 
 	/* No data transfer */
 	if (!scsi_bufflen(cmd) || cmd->sc_data_direction == DMA_NONE) {
@@ -215,8 +213,8 @@ void qla2x00_build_scsi_iocbs_32(srb_t *sp, cmd_entry_t *cmd_pkt,
 	cmd_pkt->control_flags |= cpu_to_le16(qla2x00_get_cmd_direction(sp));
 
 	/* Three DSDs are available in the Command Type 2 IOCB */
-	avail_dsds = 3;
-	cur_dsd = (uint32_t *)&cmd_pkt->dseg_0_address;
+	avail_dsds = ARRAY_SIZE(cmd_pkt->dsd32);
+	cur_dsd = cmd_pkt->dsd32;
 
 	/* Load data segments */
 	scsi_for_each_sg(cmd, sg, tot_dsds, i) {
@@ -229,12 +227,11 @@ void qla2x00_build_scsi_iocbs_32(srb_t *sp, cmd_entry_t *cmd_pkt,
 			 * Type 0 IOCB.
 			 */
 			cont_pkt = qla2x00_prep_cont_type0_iocb(vha);
-			cur_dsd = (uint32_t *)&cont_pkt->dseg_0_address;
-			avail_dsds = 7;
+			cur_dsd = cont_pkt->dsd;
+			avail_dsds = ARRAY_SIZE(cont_pkt->dsd);
 		}
 
-		*cur_dsd++ = cpu_to_le32(sg_dma_address(sg));
-		*cur_dsd++ = cpu_to_le32(sg_dma_len(sg));
+		append_dsd32(&cur_dsd, sg);
 		avail_dsds--;
 	}
 }
@@ -251,7 +248,7 @@ void qla2x00_build_scsi_iocbs_64(srb_t *sp, cmd_entry_t *cmd_pkt,
     uint16_t tot_dsds)
 {
 	uint16_t	avail_dsds;
-	uint32_t	*cur_dsd;
+	struct dsd64	*cur_dsd;
 	scsi_qla_host_t	*vha;
 	struct scsi_cmnd *cmd;
 	struct scatterlist *sg;
@@ -260,7 +257,7 @@ void qla2x00_build_scsi_iocbs_64(srb_t *sp, cmd_entry_t *cmd_pkt,
 	cmd = GET_CMD_SP(sp);
 
 	/* Update entry type to indicate Command Type 3 IOCB */
-	*((uint32_t *)(&cmd_pkt->entry_type)) = cpu_to_le32(COMMAND_A64_TYPE);
+	put_unaligned_le32(COMMAND_A64_TYPE, &cmd_pkt->entry_type);
 
 	/* No data transfer */
 	if (!scsi_bufflen(cmd) || cmd->sc_data_direction == DMA_NONE) {
@@ -272,12 +269,11 @@ void qla2x00_build_scsi_iocbs_64(srb_t *sp, cmd_entry_t *cmd_pkt,
 	cmd_pkt->control_flags |= cpu_to_le16(qla2x00_get_cmd_direction(sp));
 
 	/* Two DSDs are available in the Command Type 3 IOCB */
-	avail_dsds = 2;
-	cur_dsd = (uint32_t *)&cmd_pkt->dseg_0_address;
+	avail_dsds = ARRAY_SIZE(cmd_pkt->dsd64);
+	cur_dsd = cmd_pkt->dsd64;
 
 	/* Load data segments */
 	scsi_for_each_sg(cmd, sg, tot_dsds, i) {
-		dma_addr_t	sle_dma;
 		cont_a64_entry_t *cont_pkt;
 
 		/* Allocate additional continuation packets? */
@@ -287,14 +283,11 @@ void qla2x00_build_scsi_iocbs_64(srb_t *sp, cmd_entry_t *cmd_pkt,
 			 * Type 1 IOCB.
 			 */
 			cont_pkt = qla2x00_prep_cont_type1_iocb(vha, vha->req);
-			cur_dsd = (uint32_t *)cont_pkt->dseg_0_address;
-			avail_dsds = 5;
+			cur_dsd = cont_pkt->dsd;
+			avail_dsds = ARRAY_SIZE(cont_pkt->dsd);
 		}
 
-		sle_dma = sg_dma_address(sg);
-		*cur_dsd++ = cpu_to_le32(LSD(sle_dma));
-		*cur_dsd++ = cpu_to_le32(MSD(sle_dma));
-		*cur_dsd++ = cpu_to_le32(sg_dma_len(sg));
+		append_dsd64(&cur_dsd, sg);
 		avail_dsds--;
 	}
 }
@@ -467,7 +460,7 @@ qla2x00_start_iocbs(struct scsi_qla_host *vha, struct req_que *req)
 			req->ring_ptr++;
 
 		/* Set chip new ring index. */
-		if (ha->mqenable || IS_QLA27XX(ha)) {
+		if (ha->mqenable || IS_QLA27XX(ha) || IS_QLA28XX(ha)) {
 			WRT_REG_DWORD(req->req_q_in, req->ring_index);
 		} else if (IS_QLA83XX(ha)) {
 			WRT_REG_DWORD(req->req_q_in, req->ring_index);
@@ -580,13 +573,11 @@ static inline int
 qla24xx_build_scsi_type_6_iocbs(srb_t *sp, struct cmd_type_6 *cmd_pkt,
 	uint16_t tot_dsds)
 {
-	uint32_t *cur_dsd = NULL;
+	struct dsd64 *cur_dsd = NULL, *next_dsd;
 	scsi_qla_host_t	*vha;
 	struct qla_hw_data *ha;
 	struct scsi_cmnd *cmd;
 	struct	scatterlist *cur_seg;
-	uint32_t *dsd_seg;
-	void *next_dsd;
 	uint8_t avail_dsds;
 	uint8_t first_iocb = 1;
 	uint32_t dsd_list_len;
@@ -596,7 +587,7 @@ qla24xx_build_scsi_type_6_iocbs(srb_t *sp, struct cmd_type_6 *cmd_pkt,
 	cmd = GET_CMD_SP(sp);
 
 	/* Update entry type to indicate Command Type 3 IOCB */
-	*((uint32_t *)(&cmd_pkt->entry_type)) = cpu_to_le32(COMMAND_TYPE_6);
+	put_unaligned_le32(COMMAND_TYPE_6, &cmd_pkt->entry_type);
 
 	/* No data transfer */
 	if (!scsi_bufflen(cmd) || cmd->sc_data_direction == DMA_NONE) {
@@ -638,32 +629,27 @@ qla24xx_build_scsi_type_6_iocbs(srb_t *sp, struct cmd_type_6 *cmd_pkt,
 
 		if (first_iocb) {
 			first_iocb = 0;
-			dsd_seg = (uint32_t *)&cmd_pkt->fcp_data_dseg_address;
-			*dsd_seg++ = cpu_to_le32(LSD(dsd_ptr->dsd_list_dma));
-			*dsd_seg++ = cpu_to_le32(MSD(dsd_ptr->dsd_list_dma));
-			cmd_pkt->fcp_data_dseg_len = cpu_to_le32(dsd_list_len);
+			put_unaligned_le64(dsd_ptr->dsd_list_dma,
+					   &cmd_pkt->fcp_dsd.address);
+			cmd_pkt->fcp_dsd.length = cpu_to_le32(dsd_list_len);
 		} else {
-			*cur_dsd++ = cpu_to_le32(LSD(dsd_ptr->dsd_list_dma));
-			*cur_dsd++ = cpu_to_le32(MSD(dsd_ptr->dsd_list_dma));
-			*cur_dsd++ = cpu_to_le32(dsd_list_len);
+			put_unaligned_le64(dsd_ptr->dsd_list_dma,
+					   &cur_dsd->address);
+			cur_dsd->length = cpu_to_le32(dsd_list_len);
+			cur_dsd++;
 		}
-		cur_dsd = (uint32_t *)next_dsd;
+		cur_dsd = next_dsd;
 		while (avail_dsds) {
-			dma_addr_t	sle_dma;
-
-			sle_dma = sg_dma_address(cur_seg);
-			*cur_dsd++ = cpu_to_le32(LSD(sle_dma));
-			*cur_dsd++ = cpu_to_le32(MSD(sle_dma));
-			*cur_dsd++ = cpu_to_le32(sg_dma_len(cur_seg));
+			append_dsd64(&cur_dsd, cur_seg);
 			cur_seg = sg_next(cur_seg);
 			avail_dsds--;
 		}
 	}
 
 	/* Null termination */
-	*cur_dsd++ =  0;
-	*cur_dsd++ = 0;
-	*cur_dsd++ = 0;
+	cur_dsd->address = 0;
+	cur_dsd->length = 0;
+	cur_dsd++;
 	cmd_pkt->control_flags |= CF_DATA_SEG_DESCR_ENABLE;
 	return 0;
 }
@@ -702,7 +688,7 @@ qla24xx_build_scsi_iocbs(srb_t *sp, struct cmd_type_7 *cmd_pkt,
 	uint16_t tot_dsds, struct req_que *req)
 {
 	uint16_t	avail_dsds;
-	uint32_t	*cur_dsd;
+	struct dsd64	*cur_dsd;
 	scsi_qla_host_t	*vha;
 	struct scsi_cmnd *cmd;
 	struct scatterlist *sg;
@@ -711,7 +697,7 @@ qla24xx_build_scsi_iocbs(srb_t *sp, struct cmd_type_7 *cmd_pkt,
 	cmd = GET_CMD_SP(sp);
 
 	/* Update entry type to indicate Command Type 3 IOCB */
-	*((uint32_t *)(&cmd_pkt->entry_type)) = cpu_to_le32(COMMAND_TYPE_7);
+	put_unaligned_le32(COMMAND_TYPE_7, &cmd_pkt->entry_type);
 
 	/* No data transfer */
 	if (!scsi_bufflen(cmd) || cmd->sc_data_direction == DMA_NONE) {
@@ -734,12 +720,11 @@ qla24xx_build_scsi_iocbs(srb_t *sp, struct cmd_type_7 *cmd_pkt,
 
 	/* One DSD is available in the Command Type 3 IOCB */
 	avail_dsds = 1;
-	cur_dsd = (uint32_t *)&cmd_pkt->dseg_0_address;
+	cur_dsd = &cmd_pkt->dsd;
 
 	/* Load data segments */
 
 	scsi_for_each_sg(cmd, sg, tot_dsds, i) {
-		dma_addr_t	sle_dma;
 		cont_a64_entry_t *cont_pkt;
 
 		/* Allocate additional continuation packets? */
@@ -749,14 +734,11 @@ qla24xx_build_scsi_iocbs(srb_t *sp, struct cmd_type_7 *cmd_pkt,
 			 * Type 1 IOCB.
 			 */
 			cont_pkt = qla2x00_prep_cont_type1_iocb(vha, req);
-			cur_dsd = (uint32_t *)cont_pkt->dseg_0_address;
-			avail_dsds = 5;
+			cur_dsd = cont_pkt->dsd;
+			avail_dsds = ARRAY_SIZE(cont_pkt->dsd);
 		}
 
-		sle_dma = sg_dma_address(sg);
-		*cur_dsd++ = cpu_to_le32(LSD(sle_dma));
-		*cur_dsd++ = cpu_to_le32(MSD(sle_dma));
-		*cur_dsd++ = cpu_to_le32(sg_dma_len(sg));
+		append_dsd64(&cur_dsd, sg);
 		avail_dsds--;
 	}
 }
@@ -892,14 +874,14 @@ qla24xx_get_one_block_sg(uint32_t blk_sz, struct qla2_sgx *sgx,
 
 int
 qla24xx_walk_and_build_sglist_no_difb(struct qla_hw_data *ha, srb_t *sp,
-	uint32_t *dsd, uint16_t tot_dsds, struct qla_tc_param *tc)
+	struct dsd64 *dsd, uint16_t tot_dsds, struct qla_tc_param *tc)
 {
 	void *next_dsd;
 	uint8_t avail_dsds = 0;
 	uint32_t dsd_list_len;
 	struct dsd_dma *dsd_ptr;
 	struct scatterlist *sg_prot;
-	uint32_t *cur_dsd = dsd;
+	struct dsd64 *cur_dsd = dsd;
 	uint16_t	used_dsds = tot_dsds;
 	uint32_t	prot_int; /* protection interval */
 	uint32_t	partial;
@@ -973,14 +955,14 @@ alloc_and_fill:
 
 
 			/* add new list to cmd iocb or last list */
-			*cur_dsd++ = cpu_to_le32(LSD(dsd_ptr->dsd_list_dma));
-			*cur_dsd++ = cpu_to_le32(MSD(dsd_ptr->dsd_list_dma));
-			*cur_dsd++ = dsd_list_len;
-			cur_dsd = (uint32_t *)next_dsd;
+			put_unaligned_le64(dsd_ptr->dsd_list_dma,
+					   &cur_dsd->address);
+			cur_dsd->length = cpu_to_le32(dsd_list_len);
+			cur_dsd = next_dsd;
 		}
-		*cur_dsd++ = cpu_to_le32(LSD(sle_dma));
-		*cur_dsd++ = cpu_to_le32(MSD(sle_dma));
-		*cur_dsd++ = cpu_to_le32(sle_dma_len);
+		put_unaligned_le64(sle_dma, &cur_dsd->address);
+		cur_dsd->length = cpu_to_le32(sle_dma_len);
+		cur_dsd++;
 		avail_dsds--;
 
 		if (partial == 0) {
@@ -999,22 +981,22 @@ alloc_and_fill:
 		}
 	}
 	/* Null termination */
-	*cur_dsd++ = 0;
-	*cur_dsd++ = 0;
-	*cur_dsd++ = 0;
+	cur_dsd->address = 0;
+	cur_dsd->length = 0;
+	cur_dsd++;
 	return 0;
 }
 
 int
-qla24xx_walk_and_build_sglist(struct qla_hw_data *ha, srb_t *sp, uint32_t *dsd,
-	uint16_t tot_dsds, struct qla_tc_param *tc)
+qla24xx_walk_and_build_sglist(struct qla_hw_data *ha, srb_t *sp,
+	struct dsd64 *dsd, uint16_t tot_dsds, struct qla_tc_param *tc)
 {
 	void *next_dsd;
 	uint8_t avail_dsds = 0;
 	uint32_t dsd_list_len;
 	struct dsd_dma *dsd_ptr;
 	struct scatterlist *sg, *sgl;
-	uint32_t *cur_dsd = dsd;
+	struct dsd64 *cur_dsd = dsd;
 	int	i;
 	uint16_t	used_dsds = tot_dsds;
 	struct scsi_cmnd *cmd;
@@ -1031,8 +1013,6 @@ qla24xx_walk_and_build_sglist(struct qla_hw_data *ha, srb_t *sp, uint32_t *dsd,
 
 
 	for_each_sg(sgl, sg, tot_dsds, i) {
-		dma_addr_t	sle_dma;
-
 		/* Allocate additional continuation packets? */
 		if (avail_dsds == 0) {
 			avail_dsds = (used_dsds > QLA_DSDS_PER_IOCB) ?
@@ -1072,29 +1052,25 @@ qla24xx_walk_and_build_sglist(struct qla_hw_data *ha, srb_t *sp, uint32_t *dsd,
 			}
 
 			/* add new list to cmd iocb or last list */
-			*cur_dsd++ = cpu_to_le32(LSD(dsd_ptr->dsd_list_dma));
-			*cur_dsd++ = cpu_to_le32(MSD(dsd_ptr->dsd_list_dma));
-			*cur_dsd++ = dsd_list_len;
-			cur_dsd = (uint32_t *)next_dsd;
+			put_unaligned_le64(dsd_ptr->dsd_list_dma,
+					   &cur_dsd->address);
+			cur_dsd->length = cpu_to_le32(dsd_list_len);
+			cur_dsd = next_dsd;
 		}
-		sle_dma = sg_dma_address(sg);
-
-		*cur_dsd++ = cpu_to_le32(LSD(sle_dma));
-		*cur_dsd++ = cpu_to_le32(MSD(sle_dma));
-		*cur_dsd++ = cpu_to_le32(sg_dma_len(sg));
+		append_dsd64(&cur_dsd, sg);
 		avail_dsds--;
 
 	}
 	/* Null termination */
-	*cur_dsd++ = 0;
-	*cur_dsd++ = 0;
-	*cur_dsd++ = 0;
+	cur_dsd->address = 0;
+	cur_dsd->length = 0;
+	cur_dsd++;
 	return 0;
 }
 
 int
 qla24xx_walk_and_build_prot_sglist(struct qla_hw_data *ha, srb_t *sp,
-    uint32_t *cur_dsd, uint16_t tot_dsds, struct qla_tgt_cmd *tc)
+	struct dsd64 *cur_dsd, uint16_t tot_dsds, struct qla_tgt_cmd *tc)
 {
 	struct dsd_dma *dsd_ptr = NULL, *dif_dsd, *nxt_dsd;
 	struct scatterlist *sg, *sgl;
@@ -1109,6 +1085,7 @@ qla24xx_walk_and_build_prot_sglist(struct qla_hw_data *ha, srb_t *sp,
 
 	if (sp) {
 		struct scsi_cmnd *cmd = GET_CMD_SP(sp);
+
 		sgl = scsi_prot_sglist(cmd);
 		vha = sp->vha;
 		difctx = sp->u.scmd.ctx;
@@ -1314,16 +1291,15 @@ qla24xx_walk_and_build_prot_sglist(struct qla_hw_data *ha, srb_t *sp,
 				}
 
 				/* add new list to cmd iocb or last list */
-				*cur_dsd++ =
-				    cpu_to_le32(LSD(dsd_ptr->dsd_list_dma));
-				*cur_dsd++ =
-				    cpu_to_le32(MSD(dsd_ptr->dsd_list_dma));
-				*cur_dsd++ = dsd_list_len;
+				put_unaligned_le64(dsd_ptr->dsd_list_dma,
+						   &cur_dsd->address);
+				cur_dsd->length = cpu_to_le32(dsd_list_len);
 				cur_dsd = dsd_ptr->dsd_addr;
 			}
-			*cur_dsd++ = cpu_to_le32(LSD(dif_dsd->dsd_list_dma));
-			*cur_dsd++ = cpu_to_le32(MSD(dif_dsd->dsd_list_dma));
-			*cur_dsd++ = cpu_to_le32(sglen);
+			put_unaligned_le64(dif_dsd->dsd_list_dma,
+					   &cur_dsd->address);
+			cur_dsd->length = cpu_to_le32(sglen);
+			cur_dsd++;
 			avail_dsds--;
 			difctx->dif_bundl_len -= sglen;
 			track_difbundl_buf--;
@@ -1334,8 +1310,6 @@ qla24xx_walk_and_build_prot_sglist(struct qla_hw_data *ha, srb_t *sp,
 			difctx->no_ldif_dsd, difctx->no_dif_bundl);
 	} else {
 		for_each_sg(sgl, sg, tot_dsds, i) {
-			dma_addr_t sle_dma;
-
 			/* Allocate additional continuation packets? */
 			if (avail_dsds == 0) {
 				avail_dsds = (used_dsds > QLA_DSDS_PER_IOCB) ?
@@ -1375,24 +1349,19 @@ qla24xx_walk_and_build_prot_sglist(struct qla_hw_data *ha, srb_t *sp,
 				}
 
 				/* add new list to cmd iocb or last list */
-				*cur_dsd++ =
-				    cpu_to_le32(LSD(dsd_ptr->dsd_list_dma));
-				*cur_dsd++ =
-				    cpu_to_le32(MSD(dsd_ptr->dsd_list_dma));
-				*cur_dsd++ = dsd_list_len;
+				put_unaligned_le64(dsd_ptr->dsd_list_dma,
+						   &cur_dsd->address);
+				cur_dsd->length = cpu_to_le32(dsd_list_len);
 				cur_dsd = dsd_ptr->dsd_addr;
 			}
-			sle_dma = sg_dma_address(sg);
-			*cur_dsd++ = cpu_to_le32(LSD(sle_dma));
-			*cur_dsd++ = cpu_to_le32(MSD(sle_dma));
-			*cur_dsd++ = cpu_to_le32(sg_dma_len(sg));
+			append_dsd64(&cur_dsd, sg);
 			avail_dsds--;
 		}
 	}
 	/* Null termination */
-	*cur_dsd++ = 0;
-	*cur_dsd++ = 0;
-	*cur_dsd++ = 0;
+	cur_dsd->address = 0;
+	cur_dsd->length = 0;
+	cur_dsd++;
 	return 0;
 }
 /**
@@ -1405,11 +1374,12 @@ qla24xx_walk_and_build_prot_sglist(struct qla_hw_data *ha, srb_t *sp,
  * @tot_prot_dsds: Total number of segments with protection information
  * @fw_prot_opts: Protection options to be passed to firmware
  */
-inline int
+static inline int
 qla24xx_build_scsi_crc_2_iocbs(srb_t *sp, struct cmd_type_crc_2 *cmd_pkt,
     uint16_t tot_dsds, uint16_t tot_prot_dsds, uint16_t fw_prot_opts)
 {
-	uint32_t		*cur_dsd, *fcp_dl;
+	struct dsd64		*cur_dsd;
+	uint32_t		*fcp_dl;
 	scsi_qla_host_t		*vha;
 	struct scsi_cmnd	*cmd;
 	uint32_t		total_bytes = 0;
@@ -1427,7 +1397,7 @@ qla24xx_build_scsi_crc_2_iocbs(srb_t *sp, struct cmd_type_crc_2 *cmd_pkt,
 	cmd = GET_CMD_SP(sp);
 
 	/* Update entry type to indicate Command Type CRC_2 IOCB */
-	*((uint32_t *)(&cmd_pkt->entry_type)) = cpu_to_le32(COMMAND_TYPE_CRC_2);
+	put_unaligned_le32(COMMAND_TYPE_CRC_2, &cmd_pkt->entry_type);
 
 	vha = sp->vha;
 	ha = vha->hw;
@@ -1475,8 +1445,7 @@ qla24xx_build_scsi_crc_2_iocbs(srb_t *sp, struct cmd_type_crc_2 *cmd_pkt,
 	qla24xx_set_t10dif_tags(sp, (struct fw_dif_context *)
 	    &crc_ctx_pkt->ref_tag, tot_prot_dsds);
 
-	cmd_pkt->crc_context_address[0] = cpu_to_le32(LSD(crc_ctx_dma));
-	cmd_pkt->crc_context_address[1] = cpu_to_le32(MSD(crc_ctx_dma));
+	put_unaligned_le64(crc_ctx_dma, &cmd_pkt->crc_context_address);
 	cmd_pkt->crc_context_len = CRC_CONTEXT_LEN_FW;
 
 	/* Determine SCSI command length -- align to 4 byte boundary */
@@ -1503,10 +1472,8 @@ qla24xx_build_scsi_crc_2_iocbs(srb_t *sp, struct cmd_type_crc_2 *cmd_pkt,
 	int_to_scsilun(cmd->device->lun, &fcp_cmnd->lun);
 	memcpy(fcp_cmnd->cdb, cmd->cmnd, cmd->cmd_len);
 	cmd_pkt->fcp_cmnd_dseg_len = cpu_to_le16(fcp_cmnd_len);
-	cmd_pkt->fcp_cmnd_dseg_address[0] = cpu_to_le32(
-	    LSD(crc_ctx_dma + CRC_CONTEXT_FCPCMND_OFF));
-	cmd_pkt->fcp_cmnd_dseg_address[1] = cpu_to_le32(
-	    MSD(crc_ctx_dma + CRC_CONTEXT_FCPCMND_OFF));
+	put_unaligned_le64(crc_ctx_dma + CRC_CONTEXT_FCPCMND_OFF,
+			   &cmd_pkt->fcp_cmnd_dseg_address);
 	fcp_cmnd->task_management = 0;
 	fcp_cmnd->task_attribute = TSK_SIMPLE;
 
@@ -1520,18 +1487,18 @@ qla24xx_build_scsi_crc_2_iocbs(srb_t *sp, struct cmd_type_crc_2 *cmd_pkt,
 	switch (scsi_get_prot_op(GET_CMD_SP(sp))) {
 	case SCSI_PROT_READ_INSERT:
 	case SCSI_PROT_WRITE_STRIP:
-	    total_bytes = data_bytes;
-	    data_bytes += dif_bytes;
-	    break;
+		total_bytes = data_bytes;
+		data_bytes += dif_bytes;
+		break;
 
 	case SCSI_PROT_READ_STRIP:
 	case SCSI_PROT_WRITE_INSERT:
 	case SCSI_PROT_READ_PASS:
 	case SCSI_PROT_WRITE_PASS:
-	    total_bytes = data_bytes + dif_bytes;
-	    break;
+		total_bytes = data_bytes + dif_bytes;
+		break;
 	default:
-	    BUG();
+		BUG();
 	}
 
 	if (!qla2x00_hba_err_chk_enabled(sp))
@@ -1548,7 +1515,7 @@ qla24xx_build_scsi_crc_2_iocbs(srb_t *sp, struct cmd_type_crc_2 *cmd_pkt,
 	}
 
 	if (!bundling) {
-		cur_dsd = (uint32_t *) &crc_ctx_pkt->u.nobundling.data_address;
+		cur_dsd = &crc_ctx_pkt->u.nobundling.data_dsd;
 	} else {
 		/*
 		 * Configure Bundling if we need to fetch interlaving
@@ -1558,7 +1525,7 @@ qla24xx_build_scsi_crc_2_iocbs(srb_t *sp, struct cmd_type_crc_2 *cmd_pkt,
 		crc_ctx_pkt->u.bundling.dif_byte_count = cpu_to_le32(dif_bytes);
 		crc_ctx_pkt->u.bundling.dseg_count = cpu_to_le16(tot_dsds -
 							tot_prot_dsds);
-		cur_dsd = (uint32_t *) &crc_ctx_pkt->u.bundling.data_address;
+		cur_dsd = &crc_ctx_pkt->u.bundling.data_dsd;
 	}
 
 	/* Finish the common fields of CRC pkt */
@@ -1591,7 +1558,7 @@ qla24xx_build_scsi_crc_2_iocbs(srb_t *sp, struct cmd_type_crc_2 *cmd_pkt,
 	if (bundling && tot_prot_dsds) {
 		/* Walks dif segments */
 		cmd_pkt->control_flags |= cpu_to_le16(CF_DIF_SEG_DESCR_ENABLE);
-		cur_dsd = (uint32_t *) &crc_ctx_pkt->u.bundling.dif_address;
+		cur_dsd = &crc_ctx_pkt->u.bundling.dif_dsd;
 		if (qla24xx_walk_and_build_prot_sglist(ha, sp, cur_dsd,
 				tot_prot_dsds, NULL))
 			goto crc_queuing_error;
@@ -2325,7 +2292,8 @@ __qla2x00_alloc_iocbs(struct qla_qpair *qpair, srb_t *sp)
 	if (req->cnt < req_cnt + 2) {
 		if (qpair->use_shadow_reg)
 			cnt = *req->out_ptr;
-		else if (ha->mqenable || IS_QLA83XX(ha) || IS_QLA27XX(ha))
+		else if (ha->mqenable || IS_QLA83XX(ha) || IS_QLA27XX(ha) ||
+		    IS_QLA28XX(ha))
 			cnt = RD_REG_DWORD(&reg->isp25mq.req_q_out);
 		else if (IS_P3P_TYPE(ha))
 			cnt = RD_REG_DWORD(&reg->isp82.req_q_out);
@@ -2494,7 +2462,7 @@ qla2x00_logout_iocb(srb_t *sp, struct mbx_entry *mbx)
 	SET_TARGET_ID(ha, mbx->loop_id, sp->fcport->loop_id);
 	mbx->mb0 = cpu_to_le16(MBC_LOGOUT_FABRIC_PORT);
 	mbx->mb1 = HAS_EXTENDED_IDS(ha) ?
-	    cpu_to_le16(sp->fcport->loop_id):
+	    cpu_to_le16(sp->fcport->loop_id) :
 	    cpu_to_le16(sp->fcport->loop_id << 8);
 	mbx->mb2 = cpu_to_le16(sp->fcport->d_id.b.domain);
 	mbx->mb3 = cpu_to_le16(sp->fcport->d_id.b.area << 8 |
@@ -2565,6 +2533,16 @@ qla24xx_tm_iocb(srb_t *sp, struct tsk_mgmt_entry *tsk)
 	}
 }
 
+void qla2x00_init_timer(srb_t *sp, unsigned long tmo)
+{
+	timer_setup(&sp->u.iocb_cmd.timer, qla2x00_sp_timeout, 0);
+	sp->u.iocb_cmd.timer.expires = jiffies + tmo * HZ;
+	sp->free = qla2x00_sp_free;
+	if (IS_QLAFX00(sp->vha->hw) && sp->type == SRB_FXIOCB_DCMD)
+		init_completion(&sp->u.iocb_cmd.u.fxiocb.fxiocb_comp);
+	add_timer(&sp->u.iocb_cmd.timer);
+}
+
 static void
 qla2x00_els_dcmd_sp_free(void *data)
 {
@@ -2726,18 +2704,13 @@ qla24xx_els_logo_iocb(srb_t *sp, struct els_entry_24xx *els_iocb)
 	if (elsio->u.els_logo.els_cmd == ELS_DCMD_PLOGI) {
 		els_iocb->tx_byte_count = els_iocb->tx_len =
 			sizeof(struct els_plogi_payload);
-		els_iocb->tx_address[0] =
-			cpu_to_le32(LSD(elsio->u.els_plogi.els_plogi_pyld_dma));
-		els_iocb->tx_address[1] =
-			cpu_to_le32(MSD(elsio->u.els_plogi.els_plogi_pyld_dma));
-
+		put_unaligned_le64(elsio->u.els_plogi.els_plogi_pyld_dma,
+				   &els_iocb->tx_address);
 		els_iocb->rx_dsd_count = 1;
 		els_iocb->rx_byte_count = els_iocb->rx_len =
 			sizeof(struct els_plogi_payload);
-		els_iocb->rx_address[0] =
-			cpu_to_le32(LSD(elsio->u.els_plogi.els_resp_pyld_dma));
-		els_iocb->rx_address[1] =
-			cpu_to_le32(MSD(elsio->u.els_plogi.els_resp_pyld_dma));
+		put_unaligned_le64(elsio->u.els_plogi.els_resp_pyld_dma,
+				   &els_iocb->rx_address);
 
 		ql_dbg(ql_dbg_io + ql_dbg_buffer, vha, 0x3073,
 		    "PLOGI ELS IOCB:\n");
@@ -2745,15 +2718,12 @@ qla24xx_els_logo_iocb(srb_t *sp, struct els_entry_24xx *els_iocb)
 		    (uint8_t *)els_iocb, 0x70);
 	} else {
 		els_iocb->tx_byte_count = sizeof(struct els_logo_payload);
-		els_iocb->tx_address[0] =
-		    cpu_to_le32(LSD(elsio->u.els_logo.els_logo_pyld_dma));
-		els_iocb->tx_address[1] =
-		    cpu_to_le32(MSD(elsio->u.els_logo.els_logo_pyld_dma));
+		put_unaligned_le64(elsio->u.els_logo.els_logo_pyld_dma,
+				   &els_iocb->tx_address);
 		els_iocb->tx_len = cpu_to_le32(sizeof(struct els_logo_payload));
 
 		els_iocb->rx_byte_count = 0;
-		els_iocb->rx_address[0] = 0;
-		els_iocb->rx_address[1] = 0;
+		els_iocb->rx_address = 0;
 		els_iocb->rx_len = 0;
 	}
 
@@ -2976,17 +2946,13 @@ qla24xx_els_iocb(srb_t *sp, struct els_entry_24xx *els_iocb)
         els_iocb->tx_byte_count =
             cpu_to_le32(bsg_job->request_payload.payload_len);
 
-        els_iocb->tx_address[0] = cpu_to_le32(LSD(sg_dma_address
-            (bsg_job->request_payload.sg_list)));
-        els_iocb->tx_address[1] = cpu_to_le32(MSD(sg_dma_address
-            (bsg_job->request_payload.sg_list)));
+	put_unaligned_le64(sg_dma_address(bsg_job->request_payload.sg_list),
+			   &els_iocb->tx_address);
         els_iocb->tx_len = cpu_to_le32(sg_dma_len
             (bsg_job->request_payload.sg_list));
 
-        els_iocb->rx_address[0] = cpu_to_le32(LSD(sg_dma_address
-            (bsg_job->reply_payload.sg_list)));
-        els_iocb->rx_address[1] = cpu_to_le32(MSD(sg_dma_address
-            (bsg_job->reply_payload.sg_list)));
+	put_unaligned_le64(sg_dma_address(bsg_job->reply_payload.sg_list),
+			   &els_iocb->rx_address);
         els_iocb->rx_len = cpu_to_le32(sg_dma_len
             (bsg_job->reply_payload.sg_list));
 
@@ -2997,14 +2963,13 @@ static void
 qla2x00_ct_iocb(srb_t *sp, ms_iocb_entry_t *ct_iocb)
 {
 	uint16_t        avail_dsds;
-	uint32_t        *cur_dsd;
+	struct dsd64	*cur_dsd;
 	struct scatterlist *sg;
 	int index;
 	uint16_t tot_dsds;
 	scsi_qla_host_t *vha = sp->vha;
 	struct qla_hw_data *ha = vha->hw;
 	struct bsg_job *bsg_job = sp->u.bsg_job;
-	int loop_iterartion = 0;
 	int entry_count = 1;
 
 	memset(ct_iocb, 0, sizeof(ms_iocb_entry_t));
@@ -3024,25 +2989,20 @@ qla2x00_ct_iocb(srb_t *sp, ms_iocb_entry_t *ct_iocb)
 	ct_iocb->rsp_bytecount =
 	    cpu_to_le32(bsg_job->reply_payload.payload_len);
 
-	ct_iocb->dseg_req_address[0] = cpu_to_le32(LSD(sg_dma_address
-	    (bsg_job->request_payload.sg_list)));
-	ct_iocb->dseg_req_address[1] = cpu_to_le32(MSD(sg_dma_address
-	    (bsg_job->request_payload.sg_list)));
-	ct_iocb->dseg_req_length = ct_iocb->req_bytecount;
+	put_unaligned_le64(sg_dma_address(bsg_job->request_payload.sg_list),
+			   &ct_iocb->req_dsd.address);
+	ct_iocb->req_dsd.length = ct_iocb->req_bytecount;
 
-	ct_iocb->dseg_rsp_address[0] = cpu_to_le32(LSD(sg_dma_address
-	    (bsg_job->reply_payload.sg_list)));
-	ct_iocb->dseg_rsp_address[1] = cpu_to_le32(MSD(sg_dma_address
-	    (bsg_job->reply_payload.sg_list)));
-	ct_iocb->dseg_rsp_length = ct_iocb->rsp_bytecount;
+	put_unaligned_le64(sg_dma_address(bsg_job->reply_payload.sg_list),
+			   &ct_iocb->rsp_dsd.address);
+	ct_iocb->rsp_dsd.length = ct_iocb->rsp_bytecount;
 
 	avail_dsds = 1;
-	cur_dsd = (uint32_t *)ct_iocb->dseg_rsp_address;
+	cur_dsd = &ct_iocb->rsp_dsd;
 	index = 0;
 	tot_dsds = bsg_job->reply_payload.sg_cnt;
 
 	for_each_sg(bsg_job->reply_payload.sg_list, sg, tot_dsds, index) {
-		dma_addr_t       sle_dma;
 		cont_a64_entry_t *cont_pkt;
 
 		/* Allocate additional continuation packets? */
@@ -3053,16 +3013,12 @@ qla2x00_ct_iocb(srb_t *sp, ms_iocb_entry_t *ct_iocb)
 			       */
 			cont_pkt = qla2x00_prep_cont_type1_iocb(vha,
 			    vha->hw->req_q_map[0]);
-			cur_dsd = (uint32_t *) cont_pkt->dseg_0_address;
+			cur_dsd = cont_pkt->dsd;
 			avail_dsds = 5;
 			entry_count++;
 		}
 
-		sle_dma = sg_dma_address(sg);
-		*cur_dsd++   = cpu_to_le32(LSD(sle_dma));
-		*cur_dsd++   = cpu_to_le32(MSD(sle_dma));
-		*cur_dsd++   = cpu_to_le32(sg_dma_len(sg));
-		loop_iterartion++;
+		append_dsd64(&cur_dsd, sg);
 		avail_dsds--;
 	}
 	ct_iocb->entry_count = entry_count;
@@ -3074,7 +3030,7 @@ static void
 qla24xx_ct_iocb(srb_t *sp, struct ct_entry_24xx *ct_iocb)
 {
 	uint16_t        avail_dsds;
-	uint32_t        *cur_dsd;
+	struct dsd64	*cur_dsd;
 	struct scatterlist *sg;
 	int index;
 	uint16_t cmd_dsds, rsp_dsds;
@@ -3103,12 +3059,10 @@ qla24xx_ct_iocb(srb_t *sp, struct ct_entry_24xx *ct_iocb)
             cpu_to_le32(bsg_job->request_payload.payload_len);
 
 	avail_dsds = 2;
-	cur_dsd = (uint32_t *)ct_iocb->dseg_0_address;
+	cur_dsd = ct_iocb->dsd;
 	index = 0;
 
 	for_each_sg(bsg_job->request_payload.sg_list, sg, cmd_dsds, index) {
-		dma_addr_t       sle_dma;
-
 		/* Allocate additional continuation packets? */
 		if (avail_dsds == 0) {
 			/*
@@ -3117,23 +3071,18 @@ qla24xx_ct_iocb(srb_t *sp, struct ct_entry_24xx *ct_iocb)
 			 */
 			cont_pkt = qla2x00_prep_cont_type1_iocb(
 			    vha, ha->req_q_map[0]);
-			cur_dsd = (uint32_t *) cont_pkt->dseg_0_address;
+			cur_dsd = cont_pkt->dsd;
 			avail_dsds = 5;
 			entry_count++;
 		}
 
-		sle_dma = sg_dma_address(sg);
-		*cur_dsd++   = cpu_to_le32(LSD(sle_dma));
-		*cur_dsd++   = cpu_to_le32(MSD(sle_dma));
-		*cur_dsd++   = cpu_to_le32(sg_dma_len(sg));
+		append_dsd64(&cur_dsd, sg);
 		avail_dsds--;
 	}
 
 	index = 0;
 
 	for_each_sg(bsg_job->reply_payload.sg_list, sg, rsp_dsds, index) {
-		dma_addr_t       sle_dma;
-
 		/* Allocate additional continuation packets? */
 		if (avail_dsds == 0) {
 			/*
@@ -3142,15 +3091,12 @@ qla24xx_ct_iocb(srb_t *sp, struct ct_entry_24xx *ct_iocb)
 			       */
 			cont_pkt = qla2x00_prep_cont_type1_iocb(vha,
 			    ha->req_q_map[0]);
-			cur_dsd = (uint32_t *) cont_pkt->dseg_0_address;
+			cur_dsd = cont_pkt->dsd;
 			avail_dsds = 5;
 			entry_count++;
 		}
 
-		sle_dma = sg_dma_address(sg);
-		*cur_dsd++   = cpu_to_le32(LSD(sle_dma));
-		*cur_dsd++   = cpu_to_le32(MSD(sle_dma));
-		*cur_dsd++   = cpu_to_le32(sg_dma_len(sg));
+		append_dsd64(&cur_dsd, sg);
 		avail_dsds--;
 	}
         ct_iocb->entry_count = entry_count;
@@ -3371,10 +3317,8 @@ sufficient_dsds:
 		*fcp_dl = htonl((uint32_t)scsi_bufflen(cmd));
 
 		cmd_pkt->fcp_cmnd_dseg_len = cpu_to_le16(ctx->fcp_cmnd_len);
-		cmd_pkt->fcp_cmnd_dseg_address[0] =
-		    cpu_to_le32(LSD(ctx->fcp_cmnd_dma));
-		cmd_pkt->fcp_cmnd_dseg_address[1] =
-		    cpu_to_le32(MSD(ctx->fcp_cmnd_dma));
+		put_unaligned_le64(ctx->fcp_cmnd_dma,
+				   &cmd_pkt->fcp_cmnd_dseg_address);
 
 		sp->flags |= SRB_FCP_CMND_DMA_VALID;
 		cmd_pkt->byte_count = cpu_to_le32((uint32_t)scsi_bufflen(cmd));
@@ -3386,6 +3330,7 @@ sufficient_dsds:
 		cmd_pkt->entry_status = (uint8_t) rsp->id;
 	} else {
 		struct cmd_type_7 *cmd_pkt;
+
 		req_cnt = qla24xx_calc_iocbs(vha, tot_dsds);
 		if (req->cnt < (req_cnt + 2)) {
 			cnt = (uint16_t)RD_REG_DWORD_RELAXED(
@@ -3590,15 +3535,13 @@ qla_nvme_ls(srb_t *sp, struct pt_ls4_request *cmd_pkt)
 
 	cmd_pkt->tx_dseg_count = 1;
 	cmd_pkt->tx_byte_count = nvme->u.nvme.cmd_len;
-	cmd_pkt->dseg0_len = nvme->u.nvme.cmd_len;
-	cmd_pkt->dseg0_address[0] = cpu_to_le32(LSD(nvme->u.nvme.cmd_dma));
-	cmd_pkt->dseg0_address[1] = cpu_to_le32(MSD(nvme->u.nvme.cmd_dma));
+	cmd_pkt->dsd[0].length = nvme->u.nvme.cmd_len;
+	put_unaligned_le64(nvme->u.nvme.cmd_dma, &cmd_pkt->dsd[0].address);
 
 	cmd_pkt->rx_dseg_count = 1;
 	cmd_pkt->rx_byte_count = nvme->u.nvme.rsp_len;
-	cmd_pkt->dseg1_len  = nvme->u.nvme.rsp_len;
-	cmd_pkt->dseg1_address[0] =  cpu_to_le32(LSD(nvme->u.nvme.rsp_dma));
-	cmd_pkt->dseg1_address[1] =  cpu_to_le32(MSD(nvme->u.nvme.rsp_dma));
+	cmd_pkt->dsd[1].length  = nvme->u.nvme.rsp_len;
+	put_unaligned_le64(nvme->u.nvme.rsp_dma, &cmd_pkt->dsd[1].address);
 
 	return rval;
 }
@@ -3737,7 +3680,7 @@ qla25xx_build_bidir_iocb(srb_t *sp, struct scsi_qla_host *vha,
 				struct cmd_bidir *cmd_pkt, uint32_t tot_dsds)
 {
 	uint16_t avail_dsds;
-	uint32_t *cur_dsd;
+	struct dsd64 *cur_dsd;
 	uint32_t req_data_len = 0;
 	uint32_t rsp_data_len = 0;
 	struct scatterlist *sg;
@@ -3746,8 +3689,7 @@ qla25xx_build_bidir_iocb(srb_t *sp, struct scsi_qla_host *vha,
 	struct bsg_job *bsg_job = sp->u.bsg_job;
 
 	/*Update entry type to indicate bidir command */
-	*((uint32_t *)(&cmd_pkt->entry_type)) =
-		cpu_to_le32(COMMAND_BIDIRECTIONAL);
+	put_unaligned_le32(COMMAND_BIDIRECTIONAL, &cmd_pkt->entry_type);
 
 	/* Set the transfer direction, in this set both flags
 	 * Also set the BD_WRAP_BACK flag, firmware will take care
@@ -3773,13 +3715,12 @@ qla25xx_build_bidir_iocb(srb_t *sp, struct scsi_qla_host *vha,
 	 * are bundled in continuation iocb
 	 */
 	avail_dsds = 1;
-	cur_dsd = (uint32_t *)&cmd_pkt->fcp_data_dseg_address;
+	cur_dsd = &cmd_pkt->fcp_dsd;
 
 	index = 0;
 
 	for_each_sg(bsg_job->request_payload.sg_list, sg,
 				bsg_job->request_payload.sg_cnt, index) {
-		dma_addr_t sle_dma;
 		cont_a64_entry_t *cont_pkt;
 
 		/* Allocate additional continuation packets */
@@ -3788,14 +3729,11 @@ qla25xx_build_bidir_iocb(srb_t *sp, struct scsi_qla_host *vha,
 			 * 5 DSDS
 			 */
 			cont_pkt = qla2x00_prep_cont_type1_iocb(vha, vha->req);
-			cur_dsd = (uint32_t *) cont_pkt->dseg_0_address;
+			cur_dsd = cont_pkt->dsd;
 			avail_dsds = 5;
 			entry_count++;
 		}
-		sle_dma = sg_dma_address(sg);
-		*cur_dsd++   = cpu_to_le32(LSD(sle_dma));
-		*cur_dsd++   = cpu_to_le32(MSD(sle_dma));
-		*cur_dsd++   = cpu_to_le32(sg_dma_len(sg));
+		append_dsd64(&cur_dsd, sg);
 		avail_dsds--;
 	}
 	/* For read request DSD will always goes to continuation IOCB
@@ -3805,7 +3743,6 @@ qla25xx_build_bidir_iocb(srb_t *sp, struct scsi_qla_host *vha,
 	 */
 	for_each_sg(bsg_job->reply_payload.sg_list, sg,
 				bsg_job->reply_payload.sg_cnt, index) {
-		dma_addr_t sle_dma;
 		cont_a64_entry_t *cont_pkt;
 
 		/* Allocate additional continuation packets */
@@ -3814,14 +3751,11 @@ qla25xx_build_bidir_iocb(srb_t *sp, struct scsi_qla_host *vha,
 			 * 5 DSDS
 			 */
 			cont_pkt = qla2x00_prep_cont_type1_iocb(vha, vha->req);
-			cur_dsd = (uint32_t *) cont_pkt->dseg_0_address;
+			cur_dsd = cont_pkt->dsd;
 			avail_dsds = 5;
 			entry_count++;
 		}
-		sle_dma = sg_dma_address(sg);
-		*cur_dsd++   = cpu_to_le32(LSD(sle_dma));
-		*cur_dsd++   = cpu_to_le32(MSD(sle_dma));
-		*cur_dsd++   = cpu_to_le32(sg_dma_len(sg));
+		append_dsd64(&cur_dsd, sg);
 		avail_dsds--;
 	}
 	/* This value should be same as number of IOCB required for this cmd */
diff --git a/drivers/scsi/qla2xxx/qla_isr.c b/drivers/scsi/qla2xxx/qla_isr.c
index 69bbea9239cc..78aec50abe0f 100644
--- a/drivers/scsi/qla2xxx/qla_isr.c
+++ b/drivers/scsi/qla2xxx/qla_isr.c
@@ -23,6 +23,14 @@ static void qla2x00_status_cont_entry(struct rsp_que *, sts_cont_entry_t *);
 static int qla2x00_error_entry(scsi_qla_host_t *, struct rsp_que *,
 	sts_entry_t *);
 
+const char *const port_state_str[] = {
+	"Unknown",
+	"UNCONFIGURED",
+	"DEAD",
+	"LOST",
+	"ONLINE"
+};
+
 /**
  * qla2100_intr_handler() - Process interrupts for the ISP2100 and ISP2200.
  * @irq: interrupt number
@@ -41,7 +49,7 @@ qla2100_intr_handler(int irq, void *dev_id)
 	int		status;
 	unsigned long	iter;
 	uint16_t	hccr;
-	uint16_t	mb[4];
+	uint16_t	mb[8];
 	struct rsp_que *rsp;
 	unsigned long	flags;
 
@@ -160,7 +168,7 @@ qla2300_intr_handler(int irq, void *dev_id)
 	unsigned long	iter;
 	uint32_t	stat;
 	uint16_t	hccr;
-	uint16_t	mb[4];
+	uint16_t	mb[8];
 	struct rsp_que *rsp;
 	struct qla_hw_data *ha;
 	unsigned long	flags;
@@ -366,7 +374,7 @@ qla2x00_get_link_speed_str(struct qla_hw_data *ha, uint16_t speed)
 	static const char *const link_speeds[] = {
 		"1", "2", "?", "4", "8", "16", "32", "10"
 	};
-#define	QLA_LAST_SPEED	7
+#define	QLA_LAST_SPEED (ARRAY_SIZE(link_speeds) - 1)
 
 	if (IS_QLA2100(ha) || IS_QLA2200(ha))
 		return link_speeds[0];
@@ -708,12 +716,15 @@ skip_rio:
 		break;
 
 	case MBA_SYSTEM_ERR:		/* System Error */
-		mbx = (IS_QLA81XX(ha) || IS_QLA83XX(ha) || IS_QLA27XX(ha)) ?
+		mbx = (IS_QLA81XX(ha) || IS_QLA83XX(ha) || IS_QLA27XX(ha) ||
+		    IS_QLA28XX(ha)) ?
 			RD_REG_WORD(&reg24->mailbox7) : 0;
 		ql_log(ql_log_warn, vha, 0x5003,
 		    "ISP System Error - mbx1=%xh mbx2=%xh mbx3=%xh "
 		    "mbx7=%xh.\n", mb[1], mb[2], mb[3], mbx);
-
+		ha->fw_dump_mpi =
+		    (IS_QLA27XX(ha) || IS_QLA28XX(ha)) &&
+		    RD_REG_WORD(&reg24->mailbox7) & BIT_8;
 		ha->isp_ops->fw_dump(vha, 1);
 		ha->flags.fw_init_done = 0;
 		QLA_FW_STOPPED(ha);
@@ -837,6 +848,7 @@ skip_rio:
 				if (ha->flags.fawwpn_enabled &&
 				    (ha->current_topology == ISP_CFG_F)) {
 					void *wwpn = ha->init_cb->port_name;
+
 					memcpy(vha->port_name, wwpn, WWN_SIZE);
 					fc_host_port_name(vha->host) =
 					    wwn_to_u64(vha->port_name);
@@ -1372,7 +1384,7 @@ qla2x00_mbx_iocb_entry(scsi_qla_host_t *vha, struct req_que *req,
 		    le16_to_cpu(mbx->status_flags));
 
 		ql_dump_buffer(ql_dbg_async + ql_dbg_buffer, vha, 0x5029,
-		    (uint8_t *)mbx, sizeof(*mbx));
+		    mbx, sizeof(*mbx));
 
 		goto logio_done;
 	}
@@ -1516,7 +1528,7 @@ qla2x00_ct_entry(scsi_qla_host_t *vha, struct req_que *req,
 			    bsg_reply->reply_payload_rcv_len = 0;
 		    }
 		    ql_dump_buffer(ql_dbg_async + ql_dbg_buffer, vha, 0x5035,
-			(uint8_t *)pkt, sizeof(*pkt));
+			pkt, sizeof(*pkt));
 	    } else {
 		    res = DID_OK << 16;
 		    bsg_reply->reply_payload_rcv_len =
@@ -1591,8 +1603,8 @@ qla24xx_els_ct_entry(scsi_qla_host_t *vha, struct req_que *req,
 	}
 
 	comp_status = fw_status[0] = le16_to_cpu(pkt->comp_status);
-	fw_status[1] = le16_to_cpu(((struct els_sts_entry_24xx*)pkt)->error_subcode_1);
-	fw_status[2] = le16_to_cpu(((struct els_sts_entry_24xx*)pkt)->error_subcode_2);
+	fw_status[1] = le16_to_cpu(((struct els_sts_entry_24xx *)pkt)->error_subcode_1);
+	fw_status[2] = le16_to_cpu(((struct els_sts_entry_24xx *)pkt)->error_subcode_2);
 
 	if (iocb_type == ELS_IOCB_TYPE) {
 		els = &sp->u.iocb_cmd;
@@ -1613,7 +1625,7 @@ qla24xx_els_ct_entry(scsi_qla_host_t *vha, struct req_que *req,
 				res = DID_ERROR << 16;
 			}
 		}
-		ql_log(ql_log_info, vha, 0x503f,
+		ql_dbg(ql_dbg_user, vha, 0x503f,
 		    "ELS IOCB Done -%s error hdl=%x comp_status=0x%x error subcode 1=0x%x error subcode 2=0x%x total_byte=0x%x\n",
 		    type, sp->handle, comp_status, fw_status[1], fw_status[2],
 		    le16_to_cpu(((struct els_sts_entry_24xx *)
@@ -1656,7 +1668,7 @@ qla24xx_els_ct_entry(scsi_qla_host_t *vha, struct req_que *req,
 		memcpy(bsg_job->reply + sizeof(struct fc_bsg_reply),
 		       fw_status, sizeof(fw_status));
 		ql_dump_buffer(ql_dbg_user + ql_dbg_buffer, vha, 0x5056,
-				(uint8_t *)pkt, sizeof(*pkt));
+		    pkt, sizeof(*pkt));
 	}
 	else {
 		res =  DID_OK << 16;
@@ -1700,7 +1712,7 @@ qla24xx_logio_entry(scsi_qla_host_t *vha, struct req_que *req,
 		    fcport->d_id.b.area, fcport->d_id.b.al_pa,
 		    logio->entry_status);
 		ql_dump_buffer(ql_dbg_async + ql_dbg_buffer, vha, 0x504d,
-		    (uint8_t *)logio, sizeof(*logio));
+		    logio, sizeof(*logio));
 
 		goto logio_done;
 	}
@@ -1846,8 +1858,8 @@ qla24xx_tm_iocb_entry(scsi_qla_host_t *vha, struct req_que *req, void *tsk)
 	}
 
 	if (iocb->u.tmf.data != QLA_SUCCESS)
-		ql_dump_buffer(ql_dbg_async + ql_dbg_buffer, vha, 0x5055,
-		    (uint8_t *)sts, sizeof(*sts));
+		ql_dump_buffer(ql_dbg_async + ql_dbg_buffer, sp->vha, 0x5055,
+		    sts, sizeof(*sts));
 
 	sp->done(sp, 0);
 }
@@ -1969,6 +1981,52 @@ static void qla_ctrlvp_completed(scsi_qla_host_t *vha, struct req_que *req,
 	sp->done(sp, rval);
 }
 
+/* Process a single response queue entry. */
+static void qla2x00_process_response_entry(struct scsi_qla_host *vha,
+					   struct rsp_que *rsp,
+					   sts_entry_t *pkt)
+{
+	sts21_entry_t *sts21_entry;
+	sts22_entry_t *sts22_entry;
+	uint16_t handle_cnt;
+	uint16_t cnt;
+
+	switch (pkt->entry_type) {
+	case STATUS_TYPE:
+		qla2x00_status_entry(vha, rsp, pkt);
+		break;
+	case STATUS_TYPE_21:
+		sts21_entry = (sts21_entry_t *)pkt;
+		handle_cnt = sts21_entry->handle_count;
+		for (cnt = 0; cnt < handle_cnt; cnt++)
+			qla2x00_process_completed_request(vha, rsp->req,
+						sts21_entry->handle[cnt]);
+		break;
+	case STATUS_TYPE_22:
+		sts22_entry = (sts22_entry_t *)pkt;
+		handle_cnt = sts22_entry->handle_count;
+		for (cnt = 0; cnt < handle_cnt; cnt++)
+			qla2x00_process_completed_request(vha, rsp->req,
+						sts22_entry->handle[cnt]);
+		break;
+	case STATUS_CONT_TYPE:
+		qla2x00_status_cont_entry(rsp, (sts_cont_entry_t *)pkt);
+		break;
+	case MBX_IOCB_TYPE:
+		qla2x00_mbx_iocb_entry(vha, rsp->req, (struct mbx_entry *)pkt);
+		break;
+	case CT_IOCB_TYPE:
+		qla2x00_ct_entry(vha, rsp->req, pkt, CT_IOCB_TYPE);
+		break;
+	default:
+		/* Type Not Supported. */
+		ql_log(ql_log_warn, vha, 0x504a,
+		       "Received unknown response pkt type %x entry status=%x.\n",
+		       pkt->entry_type, pkt->entry_status);
+		break;
+	}
+}
+
 /**
  * qla2x00_process_response_queue() - Process response queue entries.
  * @rsp: response queue
@@ -1980,8 +2038,6 @@ qla2x00_process_response_queue(struct rsp_que *rsp)
 	struct qla_hw_data *ha = rsp->hw;
 	struct device_reg_2xxx __iomem *reg = &ha->iobase->isp;
 	sts_entry_t	*pkt;
-	uint16_t        handle_cnt;
-	uint16_t        cnt;
 
 	vha = pci_get_drvdata(ha->pdev);
 
@@ -2006,42 +2062,7 @@ qla2x00_process_response_queue(struct rsp_que *rsp)
 			continue;
 		}
 
-		switch (pkt->entry_type) {
-		case STATUS_TYPE:
-			qla2x00_status_entry(vha, rsp, pkt);
-			break;
-		case STATUS_TYPE_21:
-			handle_cnt = ((sts21_entry_t *)pkt)->handle_count;
-			for (cnt = 0; cnt < handle_cnt; cnt++) {
-				qla2x00_process_completed_request(vha, rsp->req,
-				    ((sts21_entry_t *)pkt)->handle[cnt]);
-			}
-			break;
-		case STATUS_TYPE_22:
-			handle_cnt = ((sts22_entry_t *)pkt)->handle_count;
-			for (cnt = 0; cnt < handle_cnt; cnt++) {
-				qla2x00_process_completed_request(vha, rsp->req,
-				    ((sts22_entry_t *)pkt)->handle[cnt]);
-			}
-			break;
-		case STATUS_CONT_TYPE:
-			qla2x00_status_cont_entry(rsp, (sts_cont_entry_t *)pkt);
-			break;
-		case MBX_IOCB_TYPE:
-			qla2x00_mbx_iocb_entry(vha, rsp->req,
-			    (struct mbx_entry *)pkt);
-			break;
-		case CT_IOCB_TYPE:
-			qla2x00_ct_entry(vha, rsp->req, pkt, CT_IOCB_TYPE);
-			break;
-		default:
-			/* Type Not Supported. */
-			ql_log(ql_log_warn, vha, 0x504a,
-			    "Received unknown response pkt type %x "
-			    "entry status=%x.\n",
-			    pkt->entry_type, pkt->entry_status);
-			break;
-		}
+		qla2x00_process_response_entry(vha, rsp, pkt);
 		((response_t *)pkt)->signature = RESPONSE_PROCESSED;
 		wmb();
 	}
@@ -2238,6 +2259,7 @@ qla25xx_process_bidir_status_iocb(scsi_qla_host_t *vha, void *pkt,
 	struct fc_bsg_reply *bsg_reply;
 	sts_entry_t *sts;
 	struct sts_entry_24xx *sts24;
+
 	sts = (sts_entry_t *) pkt;
 	sts24 = (struct sts_entry_24xx *) pkt;
 
@@ -3014,7 +3036,8 @@ process_err:
 			qla24xx_els_ct_entry(vha, rsp->req, pkt, ELS_IOCB_TYPE);
 			break;
 		case ABTS_RECV_24XX:
-			if (IS_QLA83XX(ha) || IS_QLA27XX(ha)) {
+			if (IS_QLA83XX(ha) || IS_QLA27XX(ha) ||
+			    IS_QLA28XX(ha)) {
 				/* ensure that the ATIO queue is empty */
 				qlt_handle_abts_recv(vha, rsp,
 				    (response_t *)pkt);
@@ -3072,6 +3095,7 @@ process_err:
 	/* Adjust ring index */
 	if (IS_P3P_TYPE(ha)) {
 		struct device_reg_82xx __iomem *reg = &ha->iobase->isp82;
+
 		WRT_REG_DWORD(&reg->rsp_q_out[0], rsp->ring_index);
 	} else {
 		WRT_REG_DWORD(rsp->rsp_q_out, rsp->ring_index);
@@ -3087,7 +3111,7 @@ qla2xxx_check_risc_status(scsi_qla_host_t *vha)
 	struct device_reg_24xx __iomem *reg = &ha->iobase->isp24;
 
 	if (!IS_QLA25XX(ha) && !IS_QLA81XX(ha) && !IS_QLA83XX(ha) &&
-	    !IS_QLA27XX(ha))
+	    !IS_QLA27XX(ha) && !IS_QLA28XX(ha))
 		return;
 
 	rval = QLA_SUCCESS;
@@ -3475,7 +3499,7 @@ qla24xx_enable_msix(struct qla_hw_data *ha, struct rsp_que *rsp)
 		ql_log(ql_log_fatal, vha, 0x00c8,
 		    "Failed to allocate memory for ha->msix_entries.\n");
 		ret = -ENOMEM;
-		goto msix_out;
+		goto free_irqs;
 	}
 	ha->flags.msix_enabled = 1;
 
@@ -3539,7 +3563,7 @@ msix_register_fail:
 	}
 
 	/* Enable MSI-X vector for response queue update for queue 0 */
-	if (IS_QLA83XX(ha) || IS_QLA27XX(ha)) {
+	if (IS_QLA83XX(ha) || IS_QLA27XX(ha) || IS_QLA28XX(ha)) {
 		if (ha->msixbase && ha->mqiobase &&
 		    (ha->max_rsp_queues > 1 || ha->max_req_queues > 1 ||
 		     ql2xmqsupport))
@@ -3558,6 +3582,10 @@ msix_register_fail:
 
 msix_out:
 	return ret;
+
+free_irqs:
+	pci_free_irq_vectors(ha->pdev);
+	goto msix_out;
 }
 
 int
@@ -3570,7 +3598,7 @@ qla2x00_request_irqs(struct qla_hw_data *ha, struct rsp_que *rsp)
 	/* If possible, enable MSI-X. */
 	if (ql2xenablemsix == 0 || (!IS_QLA2432(ha) && !IS_QLA2532(ha) &&
 	    !IS_QLA8432(ha) && !IS_CNA_CAPABLE(ha) && !IS_QLA2031(ha) &&
-	    !IS_QLAFX00(ha) && !IS_QLA27XX(ha)))
+	    !IS_QLAFX00(ha) && !IS_QLA27XX(ha) && !IS_QLA28XX(ha)))
 		goto skip_msi;
 
 	if (ql2xenablemsix == 2)
@@ -3609,7 +3637,7 @@ skip_msix:
 
 	if (!IS_QLA24XX(ha) && !IS_QLA2532(ha) && !IS_QLA8432(ha) &&
 	    !IS_QLA8001(ha) && !IS_P3P_TYPE(ha) && !IS_QLAFX00(ha) &&
-	    !IS_QLA27XX(ha))
+	    !IS_QLA27XX(ha) && !IS_QLA28XX(ha))
 		goto skip_msi;
 
 	ret = pci_alloc_irq_vectors(ha->pdev, 1, 1, PCI_IRQ_MSI);
diff --git a/drivers/scsi/qla2xxx/qla_mbx.c b/drivers/scsi/qla2xxx/qla_mbx.c
index 5400696e1f6b..133f5f6270ff 100644
--- a/drivers/scsi/qla2xxx/qla_mbx.c
+++ b/drivers/scsi/qla2xxx/qla_mbx.c
@@ -567,9 +567,9 @@ mbx_done:
 		    mcp->mb[0]);
 	} else if (rval) {
 		if (ql2xextended_error_logging & (ql_dbg_disc|ql_dbg_mbx)) {
-			pr_warn("%s [%s]-%04x:%ld: **** Failed", QL_MSGHDR,
+			pr_warn("%s [%s]-%04x:%ld: **** Failed=%x", QL_MSGHDR,
 			    dev_name(&ha->pdev->dev), 0x1020+0x800,
-			    vha->host_no);
+			    vha->host_no, rval);
 			mboxes = mcp->in_mb;
 			cnt = 4;
 			for (i = 0; i < ha->mbx_count && cnt; i++, mboxes >>= 1)
@@ -634,14 +634,15 @@ qla2x00_load_ram(scsi_qla_host_t *vha, dma_addr_t req_dma, uint32_t risc_addr,
 		mcp->out_mb |= MBX_4;
 	}
 
-	mcp->in_mb = MBX_0;
+	mcp->in_mb = MBX_1|MBX_0;
 	mcp->tov = MBX_TOV_SECONDS;
 	mcp->flags = 0;
 	rval = qla2x00_mailbox_command(vha, mcp);
 
 	if (rval != QLA_SUCCESS) {
 		ql_dbg(ql_dbg_mbx, vha, 0x1023,
-		    "Failed=%x mb[0]=%x.\n", rval, mcp->mb[0]);
+		    "Failed=%x mb[0]=%x mb[1]=%x.\n",
+		    rval, mcp->mb[0], mcp->mb[1]);
 	} else {
 		ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x1024,
 		    "Done %s.\n", __func__);
@@ -656,7 +657,7 @@ static inline uint16_t qla25xx_set_sfp_lr_dist(struct qla_hw_data *ha)
 {
 	uint16_t mb4 = BIT_0;
 
-	if (IS_QLA83XX(ha) || IS_QLA27XX(ha))
+	if (IS_QLA83XX(ha) || IS_QLA27XX(ha) || IS_QLA28XX(ha))
 		mb4 |= ha->long_range_distance << LR_DIST_FW_POS;
 
 	return mb4;
@@ -666,7 +667,7 @@ static inline uint16_t qla25xx_set_nvr_lr_dist(struct qla_hw_data *ha)
 {
 	uint16_t mb4 = BIT_0;
 
-	if (IS_QLA83XX(ha) || IS_QLA27XX(ha)) {
+	if (IS_QLA83XX(ha) || IS_QLA27XX(ha) || IS_QLA28XX(ha)) {
 		struct nvram_81xx *nv = ha->nvram;
 
 		mb4 |= LR_DIST_FW_FIELD(nv->enhanced_features);
@@ -711,7 +712,7 @@ qla2x00_execute_fw(scsi_qla_host_t *vha, uint32_t risc_addr)
 		mcp->mb[4] = 0;
 		ha->flags.using_lr_setting = 0;
 		if (IS_QLA25XX(ha) || IS_QLA81XX(ha) || IS_QLA83XX(ha) ||
-		    IS_QLA27XX(ha)) {
+		    IS_QLA27XX(ha) || IS_QLA28XX(ha)) {
 			if (ql2xautodetectsfp) {
 				if (ha->flags.detected_lr_sfp) {
 					mcp->mb[4] |=
@@ -730,19 +731,20 @@ qla2x00_execute_fw(scsi_qla_host_t *vha, uint32_t risc_addr)
 			}
 		}
 
-		if (ql2xnvmeenable && IS_QLA27XX(ha))
+		if (ql2xnvmeenable && (IS_QLA27XX(ha) || IS_QLA28XX(ha)))
 			mcp->mb[4] |= NVME_ENABLE_FLAG;
 
-		if (IS_QLA83XX(ha) || IS_QLA27XX(ha)) {
+		if (IS_QLA83XX(ha) || IS_QLA27XX(ha) || IS_QLA28XX(ha)) {
 			struct nvram_81xx *nv = ha->nvram;
 			/* set minimum speed if specified in nvram */
-			if (nv->min_link_speed >= 2 &&
-			    nv->min_link_speed <= 5) {
+			if (nv->min_supported_speed >= 2 &&
+			    nv->min_supported_speed <= 5) {
 				mcp->mb[4] |= BIT_4;
-				mcp->mb[11] = nv->min_link_speed;
+				mcp->mb[11] |= nv->min_supported_speed & 0xF;
 				mcp->out_mb |= MBX_11;
 				mcp->in_mb |= BIT_5;
-				vha->min_link_speed_feat = nv->min_link_speed;
+				vha->min_supported_speed =
+				    nv->min_supported_speed;
 			}
 		}
 
@@ -770,34 +772,39 @@ qla2x00_execute_fw(scsi_qla_host_t *vha, uint32_t risc_addr)
 	if (rval != QLA_SUCCESS) {
 		ql_dbg(ql_dbg_mbx, vha, 0x1026,
 		    "Failed=%x mb[0]=%x.\n", rval, mcp->mb[0]);
-	} else {
-		if (IS_FWI2_CAPABLE(ha)) {
-			ha->fw_ability_mask = mcp->mb[3] << 16 | mcp->mb[2];
-			ql_dbg(ql_dbg_mbx, vha, 0x119a,
-			    "fw_ability_mask=%x.\n", ha->fw_ability_mask);
-			ql_dbg(ql_dbg_mbx, vha, 0x1027,
-			    "exchanges=%x.\n", mcp->mb[1]);
-			if (IS_QLA83XX(ha) || IS_QLA27XX(ha)) {
-				ha->max_speed_sup = mcp->mb[2] & BIT_0;
-				ql_dbg(ql_dbg_mbx, vha, 0x119b,
-				    "Maximum speed supported=%s.\n",
-				    ha->max_speed_sup ? "32Gps" : "16Gps");
-				if (vha->min_link_speed_feat) {
-					ha->min_link_speed = mcp->mb[5];
-					ql_dbg(ql_dbg_mbx, vha, 0x119c,
-					    "Minimum speed set=%s.\n",
-					    mcp->mb[5] == 5 ? "32Gps" :
-					    mcp->mb[5] == 4 ? "16Gps" :
-					    mcp->mb[5] == 3 ? "8Gps" :
-					    mcp->mb[5] == 2 ? "4Gps" :
-						"unknown");
-				}
-			}
+		return rval;
+	}
+
+	if (!IS_FWI2_CAPABLE(ha))
+		goto done;
+
+	ha->fw_ability_mask = mcp->mb[3] << 16 | mcp->mb[2];
+	ql_dbg(ql_dbg_mbx, vha, 0x119a,
+	    "fw_ability_mask=%x.\n", ha->fw_ability_mask);
+	ql_dbg(ql_dbg_mbx, vha, 0x1027, "exchanges=%x.\n", mcp->mb[1]);
+	if (IS_QLA27XX(ha) || IS_QLA28XX(ha)) {
+		ha->max_supported_speed = mcp->mb[2] & (BIT_0|BIT_1);
+		ql_dbg(ql_dbg_mbx, vha, 0x119b, "max_supported_speed=%s.\n",
+		    ha->max_supported_speed == 0 ? "16Gps" :
+		    ha->max_supported_speed == 1 ? "32Gps" :
+		    ha->max_supported_speed == 2 ? "64Gps" : "unknown");
+		if (vha->min_supported_speed) {
+			ha->min_supported_speed = mcp->mb[5] &
+			    (BIT_0 | BIT_1 | BIT_2);
+			ql_dbg(ql_dbg_mbx, vha, 0x119c,
+			    "min_supported_speed=%s.\n",
+			    ha->min_supported_speed == 6 ? "64Gps" :
+			    ha->min_supported_speed == 5 ? "32Gps" :
+			    ha->min_supported_speed == 4 ? "16Gps" :
+			    ha->min_supported_speed == 3 ? "8Gps" :
+			    ha->min_supported_speed == 2 ? "4Gps" : "unknown");
 		}
-		ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x1028,
-		    "Done.\n");
 	}
 
+done:
+	ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x1028,
+	    "Done %s.\n", __func__);
+
 	return rval;
 }
 
@@ -1053,10 +1060,10 @@ qla2x00_get_fw_version(scsi_qla_host_t *vha)
 		mcp->in_mb |= MBX_13|MBX_12|MBX_11|MBX_10|MBX_9|MBX_8;
 	if (IS_FWI2_CAPABLE(ha))
 		mcp->in_mb |= MBX_17|MBX_16|MBX_15;
-	if (IS_QLA27XX(ha))
+	if (IS_QLA27XX(ha) || IS_QLA28XX(ha))
 		mcp->in_mb |=
 		    MBX_25|MBX_24|MBX_23|MBX_22|MBX_21|MBX_20|MBX_19|MBX_18|
-		    MBX_14|MBX_13|MBX_11|MBX_10|MBX_9|MBX_8;
+		    MBX_14|MBX_13|MBX_11|MBX_10|MBX_9|MBX_8|MBX_7;
 
 	mcp->flags = 0;
 	mcp->tov = MBX_TOV_SECONDS;
@@ -1122,7 +1129,10 @@ qla2x00_get_fw_version(scsi_qla_host_t *vha)
 		}
 	}
 
-	if (IS_QLA27XX(ha)) {
+	if (IS_QLA27XX(ha) || IS_QLA28XX(ha)) {
+		ha->serdes_version[0] = mcp->mb[7] & 0xff;
+		ha->serdes_version[1] = mcp->mb[8] >> 8;
+		ha->serdes_version[2] = mcp->mb[8] & 0xff;
 		ha->mpi_version[0] = mcp->mb[10] & 0xff;
 		ha->mpi_version[1] = mcp->mb[11] >> 8;
 		ha->mpi_version[2] = mcp->mb[11] & 0xff;
@@ -1133,6 +1143,13 @@ qla2x00_get_fw_version(scsi_qla_host_t *vha)
 		ha->fw_shared_ram_end = (mcp->mb[21] << 16) | mcp->mb[20];
 		ha->fw_ddr_ram_start = (mcp->mb[23] << 16) | mcp->mb[22];
 		ha->fw_ddr_ram_end = (mcp->mb[25] << 16) | mcp->mb[24];
+		if (IS_QLA28XX(ha)) {
+			if (mcp->mb[16] & BIT_10) {
+				ql_log(ql_log_info, vha, 0xffff,
+				    "FW support secure flash updates\n");
+				ha->flags.secure_fw = 1;
+			}
+		}
 	}
 
 failed:
@@ -1638,7 +1655,7 @@ qla2x00_get_adapter_id(scsi_qla_host_t *vha, uint16_t *id, uint8_t *al_pa,
 		mcp->in_mb |= MBX_13|MBX_12|MBX_11|MBX_10;
 	if (IS_FWI2_CAPABLE(vha->hw))
 		mcp->in_mb |= MBX_19|MBX_18|MBX_17|MBX_16;
-	if (IS_QLA27XX(vha->hw))
+	if (IS_QLA27XX(vha->hw) || IS_QLA28XX(vha->hw))
 		mcp->in_mb |= MBX_15;
 	mcp->tov = MBX_TOV_SECONDS;
 	mcp->flags = 0;
@@ -1692,7 +1709,7 @@ qla2x00_get_adapter_id(scsi_qla_host_t *vha, uint16_t *id, uint8_t *al_pa,
 			}
 		}
 
-		if (IS_QLA27XX(vha->hw))
+		if (IS_QLA27XX(vha->hw) || IS_QLA28XX(vha->hw))
 			vha->bbcr = mcp->mb[15];
 	}
 
@@ -1808,7 +1825,7 @@ qla2x00_init_firmware(scsi_qla_host_t *vha, uint16_t size)
 	}
 	/* 1 and 2 should normally be captured. */
 	mcp->in_mb = MBX_2|MBX_1|MBX_0;
-	if (IS_QLA83XX(ha) || IS_QLA27XX(ha))
+	if (IS_QLA83XX(ha) || IS_QLA27XX(ha) || IS_QLA28XX(ha))
 		/* mb3 is additional info about the installed SFP. */
 		mcp->in_mb  |= MBX_3;
 	mcp->buf_size = size;
@@ -1819,10 +1836,20 @@ qla2x00_init_firmware(scsi_qla_host_t *vha, uint16_t size)
 	if (rval != QLA_SUCCESS) {
 		/*EMPTY*/
 		ql_dbg(ql_dbg_mbx, vha, 0x104d,
-		    "Failed=%x mb[0]=%x, mb[1]=%x, mb[2]=%x, mb[3]=%x,.\n",
+		    "Failed=%x mb[0]=%x, mb[1]=%x, mb[2]=%x, mb[3]=%x.\n",
 		    rval, mcp->mb[0], mcp->mb[1], mcp->mb[2], mcp->mb[3]);
+		if (ha->init_cb) {
+			ql_dbg(ql_dbg_mbx, vha, 0x104d, "init_cb:\n");
+			ql_dump_buffer(ql_dbg_init + ql_dbg_verbose, vha,
+			    0x0104d, ha->init_cb, sizeof(*ha->init_cb));
+		}
+		if (ha->ex_init_cb && ha->ex_init_cb->ex_version) {
+			ql_dbg(ql_dbg_mbx, vha, 0x104d, "ex_init_cb:\n");
+			ql_dump_buffer(ql_dbg_init + ql_dbg_verbose, vha,
+			    0x0104d, ha->ex_init_cb, sizeof(*ha->ex_init_cb));
+		}
 	} else {
-		if (IS_QLA27XX(ha)) {
+		if (IS_QLA27XX(ha) || IS_QLA28XX(ha)) {
 			if (mcp->mb[2] == 6 || mcp->mb[3] == 2)
 				ql_dbg(ql_dbg_mbx, vha, 0x119d,
 				    "Invalid SFP/Validation Failed\n");
@@ -2006,7 +2033,7 @@ qla2x00_get_port_database(scsi_qla_host_t *vha, fc_port_t *fcport, uint8_t opt)
 
 		/* Passback COS information. */
 		fcport->supported_classes = (pd->options & BIT_4) ?
-		    FC_COS_CLASS2: FC_COS_CLASS3;
+		    FC_COS_CLASS2 : FC_COS_CLASS3;
 	}
 
 gpd_error_out:
@@ -2076,7 +2103,7 @@ qla2x00_get_firmware_state(scsi_qla_host_t *vha, uint16_t *states)
 		/*EMPTY*/
 		ql_dbg(ql_dbg_mbx, vha, 0x1055, "Failed=%x.\n", rval);
 	} else {
-		if (IS_QLA27XX(ha)) {
+		if (IS_QLA27XX(ha) || IS_QLA28XX(ha)) {
 			if (mcp->mb[2] == 6 || mcp->mb[3] == 2)
 				ql_dbg(ql_dbg_mbx, vha, 0x119e,
 				    "Invalid SFP/Validation Failed\n");
@@ -2859,7 +2886,8 @@ qla2x00_get_resource_cnts(scsi_qla_host_t *vha)
 	mcp->mb[0] = MBC_GET_RESOURCE_COUNTS;
 	mcp->out_mb = MBX_0;
 	mcp->in_mb = MBX_11|MBX_10|MBX_7|MBX_6|MBX_3|MBX_2|MBX_1|MBX_0;
-	if (IS_QLA81XX(vha->hw) || IS_QLA83XX(vha->hw) || IS_QLA27XX(vha->hw))
+	if (IS_QLA81XX(ha) || IS_QLA83XX(ha) ||
+	    IS_QLA27XX(ha) || IS_QLA28XX(ha))
 		mcp->in_mb |= MBX_12;
 	mcp->tov = MBX_TOV_SECONDS;
 	mcp->flags = 0;
@@ -2884,7 +2912,8 @@ qla2x00_get_resource_cnts(scsi_qla_host_t *vha)
 		ha->orig_fw_iocb_count = mcp->mb[10];
 		if (ha->flags.npiv_supported)
 			ha->max_npiv_vports = mcp->mb[11];
-		if (IS_QLA81XX(ha) || IS_QLA83XX(ha) || IS_QLA27XX(ha))
+		if (IS_QLA81XX(ha) || IS_QLA83XX(ha) || IS_QLA27XX(ha) ||
+		    IS_QLA28XX(ha))
 			ha->fw_max_fcf_count = mcp->mb[12];
 	}
 
@@ -3248,7 +3277,7 @@ __qla24xx_issue_tmf(char *name, uint32_t type, struct fc_port *fcport,
 
 	/* Issue marker IOCB. */
 	rval2 = qla2x00_marker(vha, ha->base_qpair, fcport->loop_id, l,
-	    type == TCF_LUN_RESET ? MK_SYNC_ID_LUN: MK_SYNC_ID);
+	    type == TCF_LUN_RESET ? MK_SYNC_ID_LUN : MK_SYNC_ID);
 	if (rval2 != QLA_SUCCESS) {
 		ql_dbg(ql_dbg_mbx, vha, 0x1099,
 		    "Failed to issue marker IOCB (%x).\n", rval2);
@@ -3323,7 +3352,7 @@ qla2x00_write_serdes_word(scsi_qla_host_t *vha, uint16_t addr, uint16_t data)
 	mbx_cmd_t *mcp = &mc;
 
 	if (!IS_QLA25XX(vha->hw) && !IS_QLA2031(vha->hw) &&
-	    !IS_QLA27XX(vha->hw))
+	    !IS_QLA27XX(vha->hw) && !IS_QLA28XX(vha->hw))
 		return QLA_FUNCTION_FAILED;
 
 	ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x1182,
@@ -3362,7 +3391,7 @@ qla2x00_read_serdes_word(scsi_qla_host_t *vha, uint16_t addr, uint16_t *data)
 	mbx_cmd_t *mcp = &mc;
 
 	if (!IS_QLA25XX(vha->hw) && !IS_QLA2031(vha->hw) &&
-	    !IS_QLA27XX(vha->hw))
+	    !IS_QLA27XX(vha->hw) && !IS_QLA28XX(vha->hw))
 		return QLA_FUNCTION_FAILED;
 
 	ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x1185,
@@ -3631,7 +3660,8 @@ qla2x00_enable_fce_trace(scsi_qla_host_t *vha, dma_addr_t fce_dma,
 	    "Entered %s.\n", __func__);
 
 	if (!IS_QLA25XX(vha->hw) && !IS_QLA81XX(vha->hw) &&
-	    !IS_QLA83XX(vha->hw) && !IS_QLA27XX(vha->hw))
+	    !IS_QLA83XX(vha->hw) && !IS_QLA27XX(vha->hw) &&
+	    !IS_QLA28XX(vha->hw))
 		return QLA_FUNCTION_FAILED;
 
 	if (unlikely(pci_channel_offline(vha->hw->pdev)))
@@ -3744,7 +3774,7 @@ qla2x00_get_idma_speed(scsi_qla_host_t *vha, uint16_t loop_id,
 	rval = qla2x00_mailbox_command(vha, mcp);
 
 	/* Return mailbox statuses. */
-	if (mb != NULL) {
+	if (mb) {
 		mb[0] = mcp->mb[0];
 		mb[1] = mcp->mb[1];
 		mb[3] = mcp->mb[3];
@@ -3779,7 +3809,7 @@ qla2x00_set_idma_speed(scsi_qla_host_t *vha, uint16_t loop_id,
 	mcp->mb[0] = MBC_PORT_PARAMS;
 	mcp->mb[1] = loop_id;
 	mcp->mb[2] = BIT_0;
-	mcp->mb[3] = port_speed & (BIT_5|BIT_4|BIT_3|BIT_2|BIT_1|BIT_0);
+	mcp->mb[3] = port_speed & 0x3F;
 	mcp->mb[9] = vha->vp_idx;
 	mcp->out_mb = MBX_9|MBX_3|MBX_2|MBX_1|MBX_0;
 	mcp->in_mb = MBX_3|MBX_1|MBX_0;
@@ -3788,7 +3818,7 @@ qla2x00_set_idma_speed(scsi_qla_host_t *vha, uint16_t loop_id,
 	rval = qla2x00_mailbox_command(vha, mcp);
 
 	/* Return mailbox statuses. */
-	if (mb != NULL) {
+	if (mb) {
 		mb[0] = mcp->mb[0];
 		mb[1] = mcp->mb[1];
 		mb[3] = mcp->mb[3];
@@ -4230,7 +4260,7 @@ qla84xx_verify_chip(struct scsi_qla_host *vha, uint16_t *status)
 		ql_dbg(ql_dbg_mbx + ql_dbg_buffer, vha, 0x111c,
 		    "Dump of Verify Request.\n");
 		ql_dump_buffer(ql_dbg_mbx + ql_dbg_buffer, vha, 0x111e,
-		    (uint8_t *)mn, sizeof(*mn));
+		    mn, sizeof(*mn));
 
 		rval = qla2x00_issue_iocb_timeout(vha, mn, mn_dma, 0, 120);
 		if (rval != QLA_SUCCESS) {
@@ -4242,7 +4272,7 @@ qla84xx_verify_chip(struct scsi_qla_host *vha, uint16_t *status)
 		ql_dbg(ql_dbg_mbx + ql_dbg_buffer, vha, 0x1110,
 		    "Dump of Verify Response.\n");
 		ql_dump_buffer(ql_dbg_mbx + ql_dbg_buffer, vha, 0x1118,
-		    (uint8_t *)mn, sizeof(*mn));
+		    mn, sizeof(*mn));
 
 		status[0] = le16_to_cpu(mn->p.rsp.comp_status);
 		status[1] = status[0] == CS_VCS_CHIP_FAILURE ?
@@ -4318,7 +4348,7 @@ qla25xx_init_req_que(struct scsi_qla_host *vha, struct req_que *req)
 	mcp->mb[12] = req->qos;
 	mcp->mb[11] = req->vp_idx;
 	mcp->mb[13] = req->rid;
-	if (IS_QLA83XX(ha) || IS_QLA27XX(ha))
+	if (IS_QLA83XX(ha) || IS_QLA27XX(ha) || IS_QLA28XX(ha))
 		mcp->mb[15] = 0;
 
 	mcp->mb[4] = req->id;
@@ -4332,9 +4362,10 @@ qla25xx_init_req_que(struct scsi_qla_host *vha, struct req_que *req)
 	mcp->flags = MBX_DMA_OUT;
 	mcp->tov = MBX_TOV_SECONDS * 2;
 
-	if (IS_QLA81XX(ha) || IS_QLA83XX(ha) || IS_QLA27XX(ha))
+	if (IS_QLA81XX(ha) || IS_QLA83XX(ha) || IS_QLA27XX(ha) ||
+	    IS_QLA28XX(ha))
 		mcp->in_mb |= MBX_1;
-	if (IS_QLA83XX(ha) || IS_QLA27XX(ha)) {
+	if (IS_QLA83XX(ha) || IS_QLA27XX(ha) || IS_QLA28XX(ha)) {
 		mcp->out_mb |= MBX_15;
 		/* debug q create issue in SR-IOV */
 		mcp->in_mb |= MBX_9 | MBX_8 | MBX_7;
@@ -4343,7 +4374,7 @@ qla25xx_init_req_que(struct scsi_qla_host *vha, struct req_que *req)
 	spin_lock_irqsave(&ha->hardware_lock, flags);
 	if (!(req->options & BIT_0)) {
 		WRT_REG_DWORD(req->req_q_in, 0);
-		if (!IS_QLA83XX(ha) && !IS_QLA27XX(ha))
+		if (!IS_QLA83XX(ha) && !IS_QLA27XX(ha) && !IS_QLA28XX(ha))
 			WRT_REG_DWORD(req->req_q_out, 0);
 	}
 	spin_unlock_irqrestore(&ha->hardware_lock, flags);
@@ -4387,7 +4418,7 @@ qla25xx_init_rsp_que(struct scsi_qla_host *vha, struct rsp_que *rsp)
 	mcp->mb[5] = rsp->length;
 	mcp->mb[14] = rsp->msix->entry;
 	mcp->mb[13] = rsp->rid;
-	if (IS_QLA83XX(ha) || IS_QLA27XX(ha))
+	if (IS_QLA83XX(ha) || IS_QLA27XX(ha) || IS_QLA28XX(ha))
 		mcp->mb[15] = 0;
 
 	mcp->mb[4] = rsp->id;
@@ -4404,7 +4435,7 @@ qla25xx_init_rsp_que(struct scsi_qla_host *vha, struct rsp_que *rsp)
 	if (IS_QLA81XX(ha)) {
 		mcp->out_mb |= MBX_12|MBX_11|MBX_10;
 		mcp->in_mb |= MBX_1;
-	} else if (IS_QLA83XX(ha) || IS_QLA27XX(ha)) {
+	} else if (IS_QLA83XX(ha) || IS_QLA27XX(ha) || IS_QLA28XX(ha)) {
 		mcp->out_mb |= MBX_15|MBX_12|MBX_11|MBX_10;
 		mcp->in_mb |= MBX_1;
 		/* debug q create issue in SR-IOV */
@@ -4414,7 +4445,7 @@ qla25xx_init_rsp_que(struct scsi_qla_host *vha, struct rsp_que *rsp)
 	spin_lock_irqsave(&ha->hardware_lock, flags);
 	if (!(rsp->options & BIT_0)) {
 		WRT_REG_DWORD(rsp->rsp_q_out, 0);
-		if (!IS_QLA83XX(ha) && !IS_QLA27XX(ha))
+		if (!IS_QLA83XX(ha) && !IS_QLA27XX(ha) && !IS_QLA28XX(ha))
 			WRT_REG_DWORD(rsp->rsp_q_in, 0);
 	}
 
@@ -4472,7 +4503,7 @@ qla81xx_fac_get_sector_size(scsi_qla_host_t *vha, uint32_t *sector_size)
 	    "Entered %s.\n", __func__);
 
 	if (!IS_QLA81XX(vha->hw) && !IS_QLA83XX(vha->hw) &&
-	    !IS_QLA27XX(vha->hw))
+	    !IS_QLA27XX(vha->hw) && !IS_QLA28XX(vha->hw))
 		return QLA_FUNCTION_FAILED;
 
 	mcp->mb[0] = MBC_FLASH_ACCESS_CTRL;
@@ -4504,7 +4535,7 @@ qla81xx_fac_do_write_enable(scsi_qla_host_t *vha, int enable)
 	mbx_cmd_t *mcp = &mc;
 
 	if (!IS_QLA81XX(vha->hw) && !IS_QLA83XX(vha->hw) &&
-	    !IS_QLA27XX(vha->hw))
+	    !IS_QLA27XX(vha->hw) && !IS_QLA28XX(vha->hw))
 		return QLA_FUNCTION_FAILED;
 
 	ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x10df,
@@ -4539,7 +4570,7 @@ qla81xx_fac_erase_sector(scsi_qla_host_t *vha, uint32_t start, uint32_t finish)
 	mbx_cmd_t *mcp = &mc;
 
 	if (!IS_QLA81XX(vha->hw) && !IS_QLA83XX(vha->hw) &&
-	    !IS_QLA27XX(vha->hw))
+	    !IS_QLA27XX(vha->hw) && !IS_QLA28XX(vha->hw))
 		return QLA_FUNCTION_FAILED;
 
 	ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x10e2,
@@ -4570,6 +4601,42 @@ qla81xx_fac_erase_sector(scsi_qla_host_t *vha, uint32_t start, uint32_t finish)
 }
 
 int
+qla81xx_fac_semaphore_access(scsi_qla_host_t *vha, int lock)
+{
+	int rval = QLA_SUCCESS;
+	mbx_cmd_t mc;
+	mbx_cmd_t *mcp = &mc;
+	struct qla_hw_data *ha = vha->hw;
+
+	if (!IS_QLA81XX(ha) && !IS_QLA83XX(ha) &&
+	    !IS_QLA27XX(ha) && !IS_QLA28XX(ha))
+		return rval;
+
+	ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x10e2,
+	    "Entered %s.\n", __func__);
+
+	mcp->mb[0] = MBC_FLASH_ACCESS_CTRL;
+	mcp->mb[1] = (lock ? FAC_OPT_CMD_LOCK_SEMAPHORE :
+	    FAC_OPT_CMD_UNLOCK_SEMAPHORE);
+	mcp->out_mb = MBX_1|MBX_0;
+	mcp->in_mb = MBX_1|MBX_0;
+	mcp->tov = MBX_TOV_SECONDS;
+	mcp->flags = 0;
+	rval = qla2x00_mailbox_command(vha, mcp);
+
+	if (rval != QLA_SUCCESS) {
+		ql_dbg(ql_dbg_mbx, vha, 0x10e3,
+		    "Failed=%x mb[0]=%x mb[1]=%x mb[2]=%x.\n",
+		    rval, mcp->mb[0], mcp->mb[1], mcp->mb[2]);
+	} else {
+		ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x10e4,
+		    "Done %s.\n", __func__);
+	}
+
+	return rval;
+}
+
+int
 qla81xx_restart_mpi_firmware(scsi_qla_host_t *vha)
 {
 	int rval = 0;
@@ -4818,10 +4885,10 @@ qla2x00_read_sfp(scsi_qla_host_t *vha, dma_addr_t sfp_dma, uint8_t *sfp,
 	if (rval != QLA_SUCCESS) {
 		ql_dbg(ql_dbg_mbx, vha, 0x10e9,
 		    "Failed=%x mb[0]=%x.\n", rval, mcp->mb[0]);
-		if (mcp->mb[0] == MBS_COMMAND_ERROR &&
-		    mcp->mb[1] == 0x22)
+		if (mcp->mb[0] == MBS_COMMAND_ERROR && mcp->mb[1] == 0x22) {
 			/* sfp is not there */
 			rval = QLA_INTERFACE_ERROR;
+		}
 	} else {
 		ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x10ea,
 		    "Done %s.\n", __func__);
@@ -5161,13 +5228,14 @@ qla2x00_write_ram_word(scsi_qla_host_t *vha, uint32_t risc_addr, uint32_t data)
 	mcp->mb[3] = MSW(data);
 	mcp->mb[8] = MSW(risc_addr);
 	mcp->out_mb = MBX_8|MBX_3|MBX_2|MBX_1|MBX_0;
-	mcp->in_mb = MBX_0;
+	mcp->in_mb = MBX_1|MBX_0;
 	mcp->tov = 30;
 	mcp->flags = 0;
 	rval = qla2x00_mailbox_command(vha, mcp);
 	if (rval != QLA_SUCCESS) {
 		ql_dbg(ql_dbg_mbx, vha, 0x1101,
-		    "Failed=%x mb[0]=%x.\n", rval, mcp->mb[0]);
+		    "Failed=%x mb[0]=%x mb[1]=%x.\n",
+		    rval, mcp->mb[0], mcp->mb[1]);
 	} else {
 		ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x1102,
 		    "Done %s.\n", __func__);
@@ -5278,7 +5346,7 @@ qla2x00_set_data_rate(scsi_qla_host_t *vha, uint16_t mode)
 
 	mcp->out_mb = MBX_2|MBX_1|MBX_0;
 	mcp->in_mb = MBX_2|MBX_1|MBX_0;
-	if (IS_QLA83XX(ha) || IS_QLA27XX(ha))
+	if (IS_QLA83XX(ha) || IS_QLA27XX(ha) || IS_QLA28XX(ha))
 		mcp->in_mb |= MBX_4|MBX_3;
 	mcp->tov = MBX_TOV_SECONDS;
 	mcp->flags = 0;
@@ -5316,7 +5384,7 @@ qla2x00_get_data_rate(scsi_qla_host_t *vha)
 	mcp->mb[1] = QLA_GET_DATA_RATE;
 	mcp->out_mb = MBX_1|MBX_0;
 	mcp->in_mb = MBX_2|MBX_1|MBX_0;
-	if (IS_QLA83XX(ha) || IS_QLA27XX(ha))
+	if (IS_QLA83XX(ha) || IS_QLA27XX(ha) || IS_QLA28XX(ha))
 		mcp->in_mb |= MBX_3;
 	mcp->tov = MBX_TOV_SECONDS;
 	mcp->flags = 0;
@@ -5346,7 +5414,7 @@ qla81xx_get_port_config(scsi_qla_host_t *vha, uint16_t *mb)
 	    "Entered %s.\n", __func__);
 
 	if (!IS_QLA81XX(ha) && !IS_QLA83XX(ha) && !IS_QLA8044(ha) &&
-	    !IS_QLA27XX(ha))
+	    !IS_QLA27XX(ha) && !IS_QLA28XX(ha))
 		return QLA_FUNCTION_FAILED;
 	mcp->mb[0] = MBC_GET_PORT_CONFIG;
 	mcp->out_mb = MBX_0;
@@ -5662,6 +5730,7 @@ qla8044_md_get_template(scsi_qla_host_t *vha)
 	mbx_cmd_t *mcp = &mc;
 	int rval = QLA_FUNCTION_FAILED;
 	int offset = 0, size = MINIDUMP_SIZE_36K;
+
 	ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0xb11f,
 	    "Entered %s.\n", __func__);
 
@@ -5842,7 +5911,7 @@ qla83xx_wr_reg(scsi_qla_host_t *vha, uint32_t reg, uint32_t data)
 	mbx_cmd_t mc;
 	mbx_cmd_t *mcp = &mc;
 
-	if (!IS_QLA83XX(ha) && !IS_QLA27XX(ha))
+	if (!IS_QLA83XX(ha) && !IS_QLA27XX(ha) && !IS_QLA28XX(ha))
 		return QLA_FUNCTION_FAILED;
 
 	ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x1130,
@@ -5917,7 +5986,7 @@ qla83xx_rd_reg(scsi_qla_host_t *vha, uint32_t reg, uint32_t *data)
 	struct qla_hw_data *ha = vha->hw;
 	unsigned long retry_max_time = jiffies + (2 * HZ);
 
-	if (!IS_QLA83XX(ha) && !IS_QLA27XX(ha))
+	if (!IS_QLA83XX(ha) && !IS_QLA27XX(ha) && !IS_QLA28XX(ha))
 		return QLA_FUNCTION_FAILED;
 
 	ql_dbg(ql_dbg_mbx, vha, 0x114b, "Entered %s.\n", __func__);
@@ -5967,7 +6036,7 @@ qla83xx_restart_nic_firmware(scsi_qla_host_t *vha)
 	mbx_cmd_t *mcp = &mc;
 	struct qla_hw_data *ha = vha->hw;
 
-	if (!IS_QLA83XX(ha) && !IS_QLA27XX(ha))
+	if (!IS_QLA83XX(ha))
 		return QLA_FUNCTION_FAILED;
 
 	ql_dbg(ql_dbg_mbx, vha, 0x1143, "Entered %s.\n", __func__);
@@ -6101,7 +6170,8 @@ qla26xx_dport_diagnostics(scsi_qla_host_t *vha,
 	mbx_cmd_t *mcp = &mc;
 	dma_addr_t dd_dma;
 
-	if (!IS_QLA83XX(vha->hw) && !IS_QLA27XX(vha->hw))
+	if (!IS_QLA83XX(vha->hw) && !IS_QLA27XX(vha->hw) &&
+	    !IS_QLA28XX(vha->hw))
 		return QLA_FUNCTION_FAILED;
 
 	ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x119f,
@@ -6318,7 +6388,13 @@ int __qla24xx_parse_gpdb(struct scsi_qla_host *vha, fc_port_t *fcport,
 	fcport->d_id.b.rsvd_1 = 0;
 
 	if (fcport->fc4f_nvme) {
-		fcport->port_type = FCT_NVME;
+		fcport->port_type = 0;
+		if ((pd->prli_svc_param_word_3[0] & BIT_5) == 0)
+			fcport->port_type |= FCT_NVME_INITIATOR;
+		if ((pd->prli_svc_param_word_3[0] & BIT_4) == 0)
+			fcport->port_type |= FCT_NVME_TARGET;
+		if ((pd->prli_svc_param_word_3[0] & BIT_3) == 0)
+			fcport->port_type |= FCT_NVME_DISCOVERY;
 	} else {
 		/* If not target must be initiator or unknown type. */
 		if ((pd->prli_svc_param_word_3[0] & BIT_4) == 0)
@@ -6507,3 +6583,101 @@ int qla24xx_res_count_wait(struct scsi_qla_host *vha,
 done:
 	return rval;
 }
+
+int qla28xx_secure_flash_update(scsi_qla_host_t *vha, uint16_t opts,
+    uint16_t region, uint32_t len, dma_addr_t sfub_dma_addr,
+    uint32_t sfub_len)
+{
+	int		rval;
+	mbx_cmd_t mc;
+	mbx_cmd_t *mcp = &mc;
+
+	mcp->mb[0] = MBC_SECURE_FLASH_UPDATE;
+	mcp->mb[1] = opts;
+	mcp->mb[2] = region;
+	mcp->mb[3] = MSW(len);
+	mcp->mb[4] = LSW(len);
+	mcp->mb[5] = MSW(sfub_dma_addr);
+	mcp->mb[6] = LSW(sfub_dma_addr);
+	mcp->mb[7] = MSW(MSD(sfub_dma_addr));
+	mcp->mb[8] = LSW(MSD(sfub_dma_addr));
+	mcp->mb[9] = sfub_len;
+	mcp->out_mb =
+	    MBX_9|MBX_8|MBX_7|MBX_6|MBX_5|MBX_4|MBX_3|MBX_2|MBX_1|MBX_0;
+	mcp->in_mb = MBX_2|MBX_1|MBX_0;
+	mcp->tov = MBX_TOV_SECONDS;
+	mcp->flags = 0;
+	rval = qla2x00_mailbox_command(vha, mcp);
+
+	if (rval != QLA_SUCCESS) {
+		ql_dbg(ql_dbg_mbx, vha, 0xffff, "%s(%ld): failed rval 0x%x, %x %x %x",
+			__func__, vha->host_no, rval, mcp->mb[0], mcp->mb[1],
+			mcp->mb[2]);
+	}
+
+	return rval;
+}
+
+int qla2xxx_write_remote_register(scsi_qla_host_t *vha, uint32_t addr,
+    uint32_t data)
+{
+	int rval;
+	mbx_cmd_t mc;
+	mbx_cmd_t *mcp = &mc;
+
+	ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x10e8,
+	    "Entered %s.\n", __func__);
+
+	mcp->mb[0] = MBC_WRITE_REMOTE_REG;
+	mcp->mb[1] = LSW(addr);
+	mcp->mb[2] = MSW(addr);
+	mcp->mb[3] = LSW(data);
+	mcp->mb[4] = MSW(data);
+	mcp->out_mb = MBX_4|MBX_3|MBX_2|MBX_1|MBX_0;
+	mcp->in_mb = MBX_1|MBX_0;
+	mcp->tov = MBX_TOV_SECONDS;
+	mcp->flags = 0;
+	rval = qla2x00_mailbox_command(vha, mcp);
+
+	if (rval != QLA_SUCCESS) {
+		ql_dbg(ql_dbg_mbx, vha, 0x10e9,
+		    "Failed=%x mb[0]=%x.\n", rval, mcp->mb[0]);
+	} else {
+		ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x10ea,
+		    "Done %s.\n", __func__);
+	}
+
+	return rval;
+}
+
+int qla2xxx_read_remote_register(scsi_qla_host_t *vha, uint32_t addr,
+    uint32_t *data)
+{
+	int rval;
+	mbx_cmd_t mc;
+	mbx_cmd_t *mcp = &mc;
+
+	ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x10e8,
+	    "Entered %s.\n", __func__);
+
+	mcp->mb[0] = MBC_READ_REMOTE_REG;
+	mcp->mb[1] = LSW(addr);
+	mcp->mb[2] = MSW(addr);
+	mcp->out_mb = MBX_2|MBX_1|MBX_0;
+	mcp->in_mb = MBX_4|MBX_3|MBX_2|MBX_1|MBX_0;
+	mcp->tov = MBX_TOV_SECONDS;
+	mcp->flags = 0;
+	rval = qla2x00_mailbox_command(vha, mcp);
+
+	*data = (uint32_t)((((uint32_t)mcp->mb[4]) << 16) | mcp->mb[3]);
+
+	if (rval != QLA_SUCCESS) {
+		ql_dbg(ql_dbg_mbx, vha, 0x10e9,
+		    "Failed=%x mb[0]=%x.\n", rval, mcp->mb[0]);
+	} else {
+		ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x10ea,
+		    "Done %s.\n", __func__);
+	}
+
+	return rval;
+}
diff --git a/drivers/scsi/qla2xxx/qla_mid.c b/drivers/scsi/qla2xxx/qla_mid.c
index 099d8e9851cb..b2977e49356b 100644
--- a/drivers/scsi/qla2xxx/qla_mid.c
+++ b/drivers/scsi/qla2xxx/qla_mid.c
@@ -905,7 +905,8 @@ static void qla_ctrlvp_sp_done(void *s, int res)
 {
 	struct srb *sp = s;
 
-	complete(&sp->comp);
+	if (sp->comp)
+		complete(sp->comp);
 	/* don't free sp here. Let the caller do the free */
 }
 
@@ -922,6 +923,7 @@ int qla24xx_control_vp(scsi_qla_host_t *vha, int cmd)
 	struct qla_hw_data *ha = vha->hw;
 	int	vp_index = vha->vp_idx;
 	struct scsi_qla_host *base_vha = pci_get_drvdata(ha->pdev);
+	DECLARE_COMPLETION_ONSTACK(comp);
 	srb_t *sp;
 
 	ql_dbg(ql_dbg_vport, vha, 0x10c1,
@@ -936,6 +938,7 @@ int qla24xx_control_vp(scsi_qla_host_t *vha, int cmd)
 
 	sp->type = SRB_CTRL_VP;
 	sp->name = "ctrl_vp";
+	sp->comp = &comp;
 	sp->done = qla_ctrlvp_sp_done;
 	sp->u.iocb_cmd.timeout = qla2x00_async_iocb_timeout;
 	qla2x00_init_timer(sp, qla2x00_get_async_timeout(vha) + 2);
@@ -953,7 +956,9 @@ int qla24xx_control_vp(scsi_qla_host_t *vha, int cmd)
 	ql_dbg(ql_dbg_vport, vha, 0x113f, "%s hndl %x submitted\n",
 	    sp->name, sp->handle);
 
-	wait_for_completion(&sp->comp);
+	wait_for_completion(&comp);
+	sp->comp = NULL;
+
 	rval = sp->rc;
 	switch (rval) {
 	case QLA_FUNCTION_TIMEOUT:
diff --git a/drivers/scsi/qla2xxx/qla_mr.c b/drivers/scsi/qla2xxx/qla_mr.c
index 60f964c53c01..942ee13b96a4 100644
--- a/drivers/scsi/qla2xxx/qla_mr.c
+++ b/drivers/scsi/qla2xxx/qla_mr.c
@@ -273,9 +273,9 @@ premature_exit:
 
 	if (rval) {
 		ql_log(ql_log_warn, base_vha, 0x1163,
-		    "**** Failed mbx[0]=%x, mb[1]=%x, mb[2]=%x, "
-		    "mb[3]=%x, cmd=%x ****.\n",
-		    mcp->mb[0], mcp->mb[1], mcp->mb[2], mcp->mb[3], command);
+		       "**** Failed=%x mbx[0]=%x, mb[1]=%x, mb[2]=%x, mb[3]=%x, cmd=%x ****.\n",
+		       rval, mcp->mb[0], mcp->mb[1], mcp->mb[2], mcp->mb[3],
+		       command);
 	} else {
 		ql_dbg(ql_dbg_mbx, base_vha, 0x1164, "Done %s.\n", __func__);
 	}
@@ -629,17 +629,20 @@ qlafx00_soc_cpu_reset(scsi_qla_host_t *vha)
  *
  * Returns 0 on success.
  */
-void
+int
 qlafx00_soft_reset(scsi_qla_host_t *vha)
 {
 	struct qla_hw_data *ha = vha->hw;
+	int rval = QLA_FUNCTION_FAILED;
 
 	if (unlikely(pci_channel_offline(ha->pdev) &&
 	    ha->flags.pci_channel_io_perm_failure))
-		return;
+		return rval;
 
 	ha->isp_ops->disable_intrs(ha);
 	qlafx00_soc_cpu_reset(vha);
+
+	return QLA_SUCCESS;
 }
 
 /**
@@ -1138,8 +1141,8 @@ qlafx00_find_all_targets(scsi_qla_host_t *vha,
 
 	ql_dbg(ql_dbg_disc + ql_dbg_init, vha, 0x2088,
 	    "Listing Target bit map...\n");
-	ql_dump_buffer(ql_dbg_disc + ql_dbg_init, vha,
-	    0x2089, (uint8_t *)ha->gid_list, 32);
+	ql_dump_buffer(ql_dbg_disc + ql_dbg_init, vha, 0x2089,
+	    ha->gid_list, 32);
 
 	/* Allocate temporary rmtport for any new rmtports discovered. */
 	new_fcport = qla2x00_alloc_fcport(vha, GFP_KERNEL);
@@ -1320,6 +1323,7 @@ qlafx00_configure_devices(scsi_qla_host_t *vha)
 {
 	int  rval;
 	unsigned long flags;
+
 	rval = QLA_SUCCESS;
 
 	flags = vha->dpc_flags;
@@ -1913,8 +1917,7 @@ qlafx00_fx_disc(scsi_qla_host_t *vha, fc_port_t *fcport, uint16_t fx_type)
 			    phost_info->domainname,
 			    phost_info->hostdriver);
 			ql_dump_buffer(ql_dbg_init + ql_dbg_disc, vha, 0x014d,
-			    (uint8_t *)phost_info,
-			    sizeof(struct host_system_info));
+			    phost_info, sizeof(*phost_info));
 		}
 	}
 
@@ -1968,7 +1971,7 @@ qlafx00_fx_disc(scsi_qla_host_t *vha, fc_port_t *fcport, uint16_t fx_type)
 		vha->d_id.b.al_pa = pinfo->port_id[2];
 		qlafx00_update_host_attr(vha, pinfo);
 		ql_dump_buffer(ql_dbg_init + ql_dbg_buffer, vha, 0x0141,
-		    (uint8_t *)pinfo, 16);
+		    pinfo, 16);
 	} else if (fx_type == FXDISC_GET_TGT_NODE_INFO) {
 		struct qlafx00_tgt_node_info *pinfo =
 		    (struct qlafx00_tgt_node_info *) fdisc->u.fxiocb.rsp_addr;
@@ -1976,12 +1979,12 @@ qlafx00_fx_disc(scsi_qla_host_t *vha, fc_port_t *fcport, uint16_t fx_type)
 		memcpy(fcport->port_name, pinfo->tgt_node_wwpn, WWN_SIZE);
 		fcport->port_type = FCT_TARGET;
 		ql_dump_buffer(ql_dbg_init + ql_dbg_buffer, vha, 0x0144,
-		    (uint8_t *)pinfo, 16);
+		    pinfo, 16);
 	} else if (fx_type == FXDISC_GET_TGT_NODE_LIST) {
 		struct qlafx00_tgt_node_info *pinfo =
 		    (struct qlafx00_tgt_node_info *) fdisc->u.fxiocb.rsp_addr;
 		ql_dump_buffer(ql_dbg_init + ql_dbg_buffer, vha, 0x0146,
-		    (uint8_t *)pinfo, 16);
+		    pinfo, 16);
 		memcpy(vha->hw->gid_list, pinfo, QLAFX00_TGT_NODE_LIST_SIZE);
 	} else if (fx_type == FXDISC_ABORT_IOCTL)
 		fdisc->u.fxiocb.result =
@@ -2248,18 +2251,16 @@ qlafx00_ioctl_iosb_entry(scsi_qla_host_t *vha, struct req_que *req,
 
 		fw_sts_ptr = bsg_job->reply + sizeof(struct fc_bsg_reply);
 
-		memcpy(fw_sts_ptr, (uint8_t *)&fstatus,
-		    sizeof(struct qla_mt_iocb_rsp_fx00));
+		memcpy(fw_sts_ptr, &fstatus, sizeof(fstatus));
 		bsg_job->reply_len = sizeof(struct fc_bsg_reply) +
 			sizeof(struct qla_mt_iocb_rsp_fx00) + sizeof(uint8_t);
 
 		ql_dump_buffer(ql_dbg_user + ql_dbg_verbose,
-		    sp->fcport->vha, 0x5080,
-		    (uint8_t *)pkt, sizeof(struct ioctl_iocb_entry_fx00));
+		    sp->vha, 0x5080, pkt, sizeof(*pkt));
 
 		ql_dump_buffer(ql_dbg_user + ql_dbg_verbose,
-		    sp->fcport->vha, 0x5074,
-		    (uint8_t *)fw_sts_ptr, sizeof(struct qla_mt_iocb_rsp_fx00));
+		    sp->vha, 0x5074,
+		    fw_sts_ptr, sizeof(fstatus));
 
 		res = bsg_reply->result = DID_OK << 16;
 		bsg_reply->reply_payload_rcv_len =
@@ -2597,7 +2598,7 @@ qlafx00_status_cont_entry(struct rsp_que *rsp, sts_cont_entry_t *pkt)
 
 		/* Move sense data. */
 		ql_dump_buffer(ql_dbg_io + ql_dbg_buffer, vha, 0x304e,
-		    (uint8_t *)pkt, sizeof(sts_cont_entry_t));
+		    pkt, sizeof(*pkt));
 		memcpy(sense_ptr, pkt->data, sense_sz);
 		ql_dump_buffer(ql_dbg_io + ql_dbg_buffer, vha, 0x304a,
 		    sense_ptr, sense_sz);
@@ -2992,7 +2993,7 @@ qlafx00_build_scsi_iocbs(srb_t *sp, struct cmd_type_7_fx00 *cmd_pkt,
 			 uint16_t tot_dsds, struct cmd_type_7_fx00 *lcmd_pkt)
 {
 	uint16_t	avail_dsds;
-	__le32 *cur_dsd;
+	struct dsd64	*cur_dsd;
 	scsi_qla_host_t	*vha;
 	struct scsi_cmnd *cmd;
 	struct scatterlist *sg;
@@ -3028,12 +3029,10 @@ qlafx00_build_scsi_iocbs(srb_t *sp, struct cmd_type_7_fx00 *cmd_pkt,
 
 	/* One DSD is available in the Command Type 3 IOCB */
 	avail_dsds = 1;
-	cur_dsd = (__le32 *)&lcmd_pkt->dseg_0_address;
+	cur_dsd = &lcmd_pkt->dsd;
 
 	/* Load data segments */
 	scsi_for_each_sg(cmd, sg, tot_dsds, i) {
-		dma_addr_t	sle_dma;
-
 		/* Allocate additional continuation packets? */
 		if (avail_dsds == 0) {
 			/*
@@ -3043,26 +3042,23 @@ qlafx00_build_scsi_iocbs(srb_t *sp, struct cmd_type_7_fx00 *cmd_pkt,
 			memset(&lcont_pkt, 0, REQUEST_ENTRY_SIZE);
 			cont_pkt =
 			    qlafx00_prep_cont_type1_iocb(req, &lcont_pkt);
-			cur_dsd = (__le32 *)lcont_pkt.dseg_0_address;
+			cur_dsd = lcont_pkt.dsd;
 			avail_dsds = 5;
 			cont = 1;
 		}
 
-		sle_dma = sg_dma_address(sg);
-		*cur_dsd++ = cpu_to_le32(LSD(sle_dma));
-		*cur_dsd++ = cpu_to_le32(MSD(sle_dma));
-		*cur_dsd++ = cpu_to_le32(sg_dma_len(sg));
+		append_dsd64(&cur_dsd, sg);
 		avail_dsds--;
 		if (avail_dsds == 0 && cont == 1) {
 			cont = 0;
 			memcpy_toio((void __iomem *)cont_pkt, &lcont_pkt,
-			    REQUEST_ENTRY_SIZE);
+			    sizeof(lcont_pkt));
 		}
 
 	}
 	if (avail_dsds != 0 && cont == 1) {
 		memcpy_toio((void __iomem *)cont_pkt, &lcont_pkt,
-		    REQUEST_ENTRY_SIZE);
+		    sizeof(lcont_pkt));
 	}
 }
 
@@ -3172,9 +3168,9 @@ qlafx00_start_scsi(srb_t *sp)
 	lcmd_pkt.entry_status = (uint8_t) rsp->id;
 
 	ql_dump_buffer(ql_dbg_io + ql_dbg_buffer, vha, 0x302e,
-	    (uint8_t *)cmd->cmnd, cmd->cmd_len);
+	    cmd->cmnd, cmd->cmd_len);
 	ql_dump_buffer(ql_dbg_io + ql_dbg_buffer, vha, 0x3032,
-	    (uint8_t *)&lcmd_pkt, REQUEST_ENTRY_SIZE);
+	    &lcmd_pkt, sizeof(lcmd_pkt));
 
 	memcpy_toio((void __iomem *)cmd_pkt, &lcmd_pkt, REQUEST_ENTRY_SIZE);
 	wmb();
@@ -3282,11 +3278,9 @@ qlafx00_fxdisc_iocb(srb_t *sp, struct fxdisc_entry_fx00 *pfxiocb)
 			fx_iocb.req_dsdcnt = cpu_to_le16(1);
 			fx_iocb.req_xfrcnt =
 			    cpu_to_le16(fxio->u.fxiocb.req_len);
-			fx_iocb.dseg_rq_address[0] =
-			    cpu_to_le32(LSD(fxio->u.fxiocb.req_dma_handle));
-			fx_iocb.dseg_rq_address[1] =
-			    cpu_to_le32(MSD(fxio->u.fxiocb.req_dma_handle));
-			fx_iocb.dseg_rq_len =
+			put_unaligned_le64(fxio->u.fxiocb.req_dma_handle,
+					   &fx_iocb.dseg_rq.address);
+			fx_iocb.dseg_rq.length =
 			    cpu_to_le32(fxio->u.fxiocb.req_len);
 		}
 
@@ -3294,11 +3288,9 @@ qlafx00_fxdisc_iocb(srb_t *sp, struct fxdisc_entry_fx00 *pfxiocb)
 			fx_iocb.rsp_dsdcnt = cpu_to_le16(1);
 			fx_iocb.rsp_xfrcnt =
 			    cpu_to_le16(fxio->u.fxiocb.rsp_len);
-			fx_iocb.dseg_rsp_address[0] =
-			    cpu_to_le32(LSD(fxio->u.fxiocb.rsp_dma_handle));
-			fx_iocb.dseg_rsp_address[1] =
-			    cpu_to_le32(MSD(fxio->u.fxiocb.rsp_dma_handle));
-			fx_iocb.dseg_rsp_len =
+			put_unaligned_le64(fxio->u.fxiocb.rsp_dma_handle,
+					   &fx_iocb.dseg_rsp.address);
+			fx_iocb.dseg_rsp.length =
 			    cpu_to_le32(fxio->u.fxiocb.rsp_len);
 		}
 
@@ -3308,6 +3300,7 @@ qlafx00_fxdisc_iocb(srb_t *sp, struct fxdisc_entry_fx00 *pfxiocb)
 		fx_iocb.flags = fxio->u.fxiocb.flags;
 	} else {
 		struct scatterlist *sg;
+
 		bsg_job = sp->u.bsg_job;
 		bsg_request = bsg_job->request;
 		piocb_rqst = (struct qla_mt_iocb_rqst_fx00 *)
@@ -3327,19 +3320,17 @@ qlafx00_fxdisc_iocb(srb_t *sp, struct fxdisc_entry_fx00 *pfxiocb)
 			int avail_dsds, tot_dsds;
 			cont_a64_entry_t lcont_pkt;
 			cont_a64_entry_t *cont_pkt = NULL;
-			__le32 *cur_dsd;
+			struct dsd64 *cur_dsd;
 			int index = 0, cont = 0;
 
 			fx_iocb.req_dsdcnt =
 			    cpu_to_le16(bsg_job->request_payload.sg_cnt);
 			tot_dsds =
 			    bsg_job->request_payload.sg_cnt;
-			cur_dsd = (__le32 *)&fx_iocb.dseg_rq_address[0];
+			cur_dsd = &fx_iocb.dseg_rq;
 			avail_dsds = 1;
 			for_each_sg(bsg_job->request_payload.sg_list, sg,
 			    tot_dsds, index) {
-				dma_addr_t sle_dma;
-
 				/* Allocate additional continuation packets? */
 				if (avail_dsds == 0) {
 					/*
@@ -3351,17 +3342,13 @@ qlafx00_fxdisc_iocb(srb_t *sp, struct fxdisc_entry_fx00 *pfxiocb)
 					cont_pkt =
 					    qlafx00_prep_cont_type1_iocb(
 						sp->vha->req, &lcont_pkt);
-					cur_dsd = (__le32 *)
-					    lcont_pkt.dseg_0_address;
+					cur_dsd = lcont_pkt.dsd;
 					avail_dsds = 5;
 					cont = 1;
 					entry_cnt++;
 				}
 
-				sle_dma = sg_dma_address(sg);
-				*cur_dsd++   = cpu_to_le32(LSD(sle_dma));
-				*cur_dsd++   = cpu_to_le32(MSD(sle_dma));
-				*cur_dsd++   = cpu_to_le32(sg_dma_len(sg));
+				append_dsd64(&cur_dsd, sg);
 				avail_dsds--;
 
 				if (avail_dsds == 0 && cont == 1) {
@@ -3389,19 +3376,17 @@ qlafx00_fxdisc_iocb(srb_t *sp, struct fxdisc_entry_fx00 *pfxiocb)
 			int avail_dsds, tot_dsds;
 			cont_a64_entry_t lcont_pkt;
 			cont_a64_entry_t *cont_pkt = NULL;
-			__le32 *cur_dsd;
+			struct dsd64 *cur_dsd;
 			int index = 0, cont = 0;
 
 			fx_iocb.rsp_dsdcnt =
 			   cpu_to_le16(bsg_job->reply_payload.sg_cnt);
 			tot_dsds = bsg_job->reply_payload.sg_cnt;
-			cur_dsd = (__le32 *)&fx_iocb.dseg_rsp_address[0];
+			cur_dsd = &fx_iocb.dseg_rsp;
 			avail_dsds = 1;
 
 			for_each_sg(bsg_job->reply_payload.sg_list, sg,
 			    tot_dsds, index) {
-				dma_addr_t sle_dma;
-
 				/* Allocate additional continuation packets? */
 				if (avail_dsds == 0) {
 					/*
@@ -3413,17 +3398,13 @@ qlafx00_fxdisc_iocb(srb_t *sp, struct fxdisc_entry_fx00 *pfxiocb)
 					cont_pkt =
 					    qlafx00_prep_cont_type1_iocb(
 						sp->vha->req, &lcont_pkt);
-					cur_dsd = (__le32 *)
-					    lcont_pkt.dseg_0_address;
+					cur_dsd = lcont_pkt.dsd;
 					avail_dsds = 5;
 					cont = 1;
 					entry_cnt++;
 				}
 
-				sle_dma = sg_dma_address(sg);
-				*cur_dsd++   = cpu_to_le32(LSD(sle_dma));
-				*cur_dsd++   = cpu_to_le32(MSD(sle_dma));
-				*cur_dsd++   = cpu_to_le32(sg_dma_len(sg));
+				append_dsd64(&cur_dsd, sg);
 				avail_dsds--;
 
 				if (avail_dsds == 0 && cont == 1) {
@@ -3454,10 +3435,8 @@ qlafx00_fxdisc_iocb(srb_t *sp, struct fxdisc_entry_fx00 *pfxiocb)
 	}
 
 	ql_dump_buffer(ql_dbg_user + ql_dbg_verbose,
-	    sp->vha, 0x3047,
-	    (uint8_t *)&fx_iocb, sizeof(struct fxdisc_entry_fx00));
+	    sp->vha, 0x3047, &fx_iocb, sizeof(fx_iocb));
 
-	memcpy_toio((void __iomem *)pfxiocb, &fx_iocb,
-	    sizeof(struct fxdisc_entry_fx00));
+	memcpy_toio((void __iomem *)pfxiocb, &fx_iocb, sizeof(fx_iocb));
 	wmb();
 }
diff --git a/drivers/scsi/qla2xxx/qla_mr.h b/drivers/scsi/qla2xxx/qla_mr.h
index aeaa1b40b1fc..4567f0c42486 100644
--- a/drivers/scsi/qla2xxx/qla_mr.h
+++ b/drivers/scsi/qla2xxx/qla_mr.h
@@ -7,6 +7,8 @@
 #ifndef __QLA_MR_H
 #define __QLA_MR_H
 
+#include "qla_dsd.h"
+
 /*
  * The PCI VendorID and DeviceID for our board.
  */
@@ -46,8 +48,7 @@ struct cmd_type_7_fx00 {
 	uint8_t fcp_cdb[MAX_CMDSZ];	/* SCSI command words. */
 	__le32 byte_count;		/* Total byte count. */
 
-	uint32_t dseg_0_address[2];	/* Data segment 0 address. */
-	uint32_t dseg_0_len;		/* Data segment 0 length. */
+	struct dsd64 dsd;
 };
 
 #define	STATUS_TYPE_FX00	0x01		/* Status entry. */
@@ -176,10 +177,8 @@ struct fxdisc_entry_fx00 {
 	uint8_t flags;
 	uint8_t reserved_1;
 
-	__le32 dseg_rq_address[2];	/* Data segment 0 address. */
-	__le32 dseg_rq_len;		/* Data segment 0 length. */
-	__le32 dseg_rsp_address[2];	/* Data segment 1 address. */
-	__le32 dseg_rsp_len;		/* Data segment 1 length. */
+	struct dsd64 dseg_rq;
+	struct dsd64 dseg_rsp;
 
 	__le32 dataword;
 	__le32 adapid;
diff --git a/drivers/scsi/qla2xxx/qla_nvme.c b/drivers/scsi/qla2xxx/qla_nvme.c
index 41c85da3ab32..22e3fba28e51 100644
--- a/drivers/scsi/qla2xxx/qla_nvme.c
+++ b/drivers/scsi/qla2xxx/qla_nvme.c
@@ -131,14 +131,10 @@ static void qla_nvme_sp_ls_done(void *ptr, int res)
 	struct nvmefc_ls_req   *fd;
 	struct nvme_private *priv;
 
-	if (atomic_read(&sp->ref_count) == 0) {
-		ql_log(ql_log_warn, sp->fcport->vha, 0x2123,
-		    "SP reference-count to ZERO on LS_done -- sp=%p.\n", sp);
+	if (WARN_ON_ONCE(atomic_read(&sp->ref_count) == 0))
 		return;
-	}
 
-	if (!atomic_dec_and_test(&sp->ref_count))
-		return;
+	atomic_dec(&sp->ref_count);
 
 	if (res)
 		res = -EINVAL;
@@ -161,15 +157,18 @@ static void qla_nvme_sp_done(void *ptr, int res)
 	nvme = &sp->u.iocb_cmd;
 	fd = nvme->u.nvme.desc;
 
-	if (!atomic_dec_and_test(&sp->ref_count))
+	if (WARN_ON_ONCE(atomic_read(&sp->ref_count) == 0))
 		return;
 
-	if (res == QLA_SUCCESS)
-		fd->status = 0;
-	else
-		fd->status = NVME_SC_INTERNAL;
+	atomic_dec(&sp->ref_count);
 
-	fd->rcv_rsplen = nvme->u.nvme.rsp_pyld_len;
+	if (res == QLA_SUCCESS) {
+		fd->rcv_rsplen = nvme->u.nvme.rsp_pyld_len;
+	} else {
+		fd->rcv_rsplen = 0;
+		fd->transferred_length = 0;
+	}
+	fd->status = 0;
 	fd->done(fd);
 	qla2xxx_rel_qpair_sp(sp->qpair, sp);
 
@@ -185,14 +184,24 @@ static void qla_nvme_abort_work(struct work_struct *work)
 	struct qla_hw_data *ha = fcport->vha->hw;
 	int rval;
 
-	if (fcport)
-		ql_dbg(ql_dbg_io, fcport->vha, 0xffff,
-		    "%s called for sp=%p, hndl=%x on fcport=%p deleted=%d\n",
-		    __func__, sp, sp->handle, fcport, fcport->deleted);
+	ql_dbg(ql_dbg_io, fcport->vha, 0xffff,
+	       "%s called for sp=%p, hndl=%x on fcport=%p deleted=%d\n",
+	       __func__, sp, sp->handle, fcport, fcport->deleted);
 
 	if (!ha->flags.fw_started && (fcport && fcport->deleted))
 		return;
 
+	if (ha->flags.host_shutting_down) {
+		ql_log(ql_log_info, sp->fcport->vha, 0xffff,
+		    "%s Calling done on sp: %p, type: 0x%x, sp->ref_count: 0x%x\n",
+		    __func__, sp, sp->type, atomic_read(&sp->ref_count));
+		sp->done(sp, 0);
+		return;
+	}
+
+	if (WARN_ON_ONCE(atomic_read(&sp->ref_count) == 0))
+		return;
+
 	rval = ha->isp_ops->abort_command(sp);
 
 	ql_dbg(ql_dbg_io, fcport->vha, 0x212b,
@@ -291,7 +300,7 @@ static inline int qla2x00_start_nvme_mq(srb_t *sp)
 	uint16_t        req_cnt;
 	uint16_t        tot_dsds;
 	uint16_t	avail_dsds;
-	uint32_t	*cur_dsd;
+	struct dsd64	*cur_dsd;
 	struct req_que *req = NULL;
 	struct scsi_qla_host *vha = sp->fcport->vha;
 	struct qla_hw_data *ha = vha->hw;
@@ -340,6 +349,7 @@ static inline int qla2x00_start_nvme_mq(srb_t *sp)
 
 	if (unlikely(!fd->sqid)) {
 		struct nvme_fc_cmd_iu *cmd = fd->cmdaddr;
+
 		if (cmd->sqe.common.opcode == nvme_admin_async_event) {
 			nvme->u.nvme.aen_op = 1;
 			atomic_inc(&ha->nvme_active_aen_cnt);
@@ -395,25 +405,22 @@ static inline int qla2x00_start_nvme_mq(srb_t *sp)
 
 	/* NVME RSP IU */
 	cmd_pkt->nvme_rsp_dsd_len = cpu_to_le16(fd->rsplen);
-	cmd_pkt->nvme_rsp_dseg_address[0] = cpu_to_le32(LSD(fd->rspdma));
-	cmd_pkt->nvme_rsp_dseg_address[1] = cpu_to_le32(MSD(fd->rspdma));
+	put_unaligned_le64(fd->rspdma, &cmd_pkt->nvme_rsp_dseg_address);
 
 	/* NVME CNMD IU */
 	cmd_pkt->nvme_cmnd_dseg_len = cpu_to_le16(fd->cmdlen);
-	cmd_pkt->nvme_cmnd_dseg_address[0] = cpu_to_le32(LSD(fd->cmddma));
-	cmd_pkt->nvme_cmnd_dseg_address[1] = cpu_to_le32(MSD(fd->cmddma));
+	cmd_pkt->nvme_cmnd_dseg_address = cpu_to_le64(fd->cmddma);
 
 	cmd_pkt->dseg_count = cpu_to_le16(tot_dsds);
 	cmd_pkt->byte_count = cpu_to_le32(fd->payload_length);
 
 	/* One DSD is available in the Command Type NVME IOCB */
 	avail_dsds = 1;
-	cur_dsd = (uint32_t *)&cmd_pkt->nvme_data_dseg_address[0];
+	cur_dsd = &cmd_pkt->nvme_dsd;
 	sgl = fd->first_sgl;
 
 	/* Load data segments */
 	for_each_sg(sgl, sg, tot_dsds, i) {
-		dma_addr_t      sle_dma;
 		cont_a64_entry_t *cont_pkt;
 
 		/* Allocate additional continuation packets? */
@@ -432,17 +439,14 @@ static inline int qla2x00_start_nvme_mq(srb_t *sp)
 				req->ring_ptr++;
 			}
 			cont_pkt = (cont_a64_entry_t *)req->ring_ptr;
-			*((uint32_t *)(&cont_pkt->entry_type)) =
-			    cpu_to_le32(CONTINUE_A64_TYPE);
+			put_unaligned_le32(CONTINUE_A64_TYPE,
+					   &cont_pkt->entry_type);
 
-			cur_dsd = (uint32_t *)cont_pkt->dseg_0_address;
-			avail_dsds = 5;
+			cur_dsd = cont_pkt->dsd;
+			avail_dsds = ARRAY_SIZE(cont_pkt->dsd);
 		}
 
-		sle_dma = sg_dma_address(sg);
-		*cur_dsd++ = cpu_to_le32(LSD(sle_dma));
-		*cur_dsd++ = cpu_to_le32(MSD(sle_dma));
-		*cur_dsd++ = cpu_to_le32(sg_dma_len(sg));
+		append_dsd64(&cur_dsd, sg);
 		avail_dsds--;
 	}
 
@@ -573,7 +577,7 @@ static struct nvme_fc_port_template qla_nvme_fc_transport = {
 	.fcp_io		= qla_nvme_post_cmd,
 	.fcp_abort	= qla_nvme_fcp_abort,
 	.max_hw_queues  = 8,
-	.max_sgl_segments = 128,
+	.max_sgl_segments = 1024,
 	.max_dif_sgl_segments = 64,
 	.dma_boundary = 0xFFFFFFFF,
 	.local_priv_sz  = 8,
@@ -582,40 +586,11 @@ static struct nvme_fc_port_template qla_nvme_fc_transport = {
 	.fcprqst_priv_sz = sizeof(struct nvme_private),
 };
 
-#define NVME_ABORT_POLLING_PERIOD    2
-static int qla_nvme_wait_on_command(srb_t *sp)
-{
-	int ret = QLA_SUCCESS;
-
-	wait_event_timeout(sp->nvme_ls_waitq, (atomic_read(&sp->ref_count) > 1),
-	    NVME_ABORT_POLLING_PERIOD*HZ);
-
-	if (atomic_read(&sp->ref_count) > 1)
-		ret = QLA_FUNCTION_FAILED;
-
-	return ret;
-}
-
-void qla_nvme_abort(struct qla_hw_data *ha, struct srb *sp, int res)
-{
-	int rval;
-
-	if (ha->flags.fw_started) {
-		rval = ha->isp_ops->abort_command(sp);
-		if (!rval && !qla_nvme_wait_on_command(sp))
-			ql_log(ql_log_warn, NULL, 0x2112,
-			    "timed out waiting on sp=%p\n", sp);
-	} else {
-		sp->done(sp, res);
-	}
-}
-
 static void qla_nvme_unregister_remote_port(struct work_struct *work)
 {
 	struct fc_port *fcport = container_of(work, struct fc_port,
 	    nvme_del_work);
 	struct qla_nvme_rport *qla_rport, *trport;
-	scsi_qla_host_t *base_vha;
 
 	if (!IS_ENABLED(CONFIG_NVME_FC))
 		return;
@@ -623,23 +598,19 @@ static void qla_nvme_unregister_remote_port(struct work_struct *work)
 	ql_log(ql_log_warn, NULL, 0x2112,
 	    "%s: unregister remoteport on %p\n",__func__, fcport);
 
-	base_vha = pci_get_drvdata(fcport->vha->hw->pdev);
-	if (test_bit(PFLG_DRIVER_REMOVING, &base_vha->pci_flags)) {
-		ql_dbg(ql_dbg_disc, fcport->vha, 0x2114,
-		    "%s: Notify FC-NVMe transport, set devloss=0\n",
-		    __func__);
-
-		nvme_fc_set_remoteport_devloss(fcport->nvme_remote_port, 0);
-	}
-
 	list_for_each_entry_safe(qla_rport, trport,
 	    &fcport->vha->nvme_rport_list, list) {
 		if (qla_rport->fcport == fcport) {
 			ql_log(ql_log_info, fcport->vha, 0x2113,
 			    "%s: fcport=%p\n", __func__, fcport);
+			nvme_fc_set_remoteport_devloss
+				(fcport->nvme_remote_port, 0);
 			init_completion(&fcport->nvme_del_done);
-			nvme_fc_unregister_remoteport(
-			    fcport->nvme_remote_port);
+			if (nvme_fc_unregister_remoteport
+			    (fcport->nvme_remote_port))
+				ql_log(ql_log_info, fcport->vha, 0x2114,
+				    "%s: Failed to unregister nvme_remote_port\n",
+				    __func__);
 			wait_for_completion(&fcport->nvme_del_done);
 			break;
 		}
diff --git a/drivers/scsi/qla2xxx/qla_nvme.h b/drivers/scsi/qla2xxx/qla_nvme.h
index da8dad5ad693..d3b8a6440113 100644
--- a/drivers/scsi/qla2xxx/qla_nvme.h
+++ b/drivers/scsi/qla2xxx/qla_nvme.h
@@ -13,6 +13,7 @@
 #include <linux/nvme-fc-driver.h>
 
 #include "qla_def.h"
+#include "qla_dsd.h"
 
 /* default dev loss time (seconds) before transport tears down ctrl */
 #define NVME_FC_DEV_LOSS_TMO  30
@@ -64,16 +65,15 @@ struct cmd_nvme {
 #define CF_WRITE_DATA                   BIT_0
 
 	uint16_t nvme_cmnd_dseg_len;             /* Data segment length. */
-	uint32_t nvme_cmnd_dseg_address[2];      /* Data segment address. */
-	uint32_t nvme_rsp_dseg_address[2];       /* Data segment address. */
+	__le64	 nvme_cmnd_dseg_address __packed;/* Data segment address. */
+	__le64	 nvme_rsp_dseg_address __packed; /* Data segment address. */
 
 	uint32_t byte_count;            /* Total byte count. */
 
 	uint8_t port_id[3];             /* PortID of destination port. */
 	uint8_t vp_index;
 
-	uint32_t nvme_data_dseg_address[2];      /* Data segment address. */
-	uint32_t nvme_data_dseg_len;             /* Data segment length. */
+	struct dsd64 nvme_dsd;
 };
 
 #define PT_LS4_REQUEST 0x89	/* Link Service pass-through IOCB (request) */
@@ -101,10 +101,7 @@ struct pt_ls4_request {
 	uint32_t rsvd3;
 	uint32_t rx_byte_count;
 	uint32_t tx_byte_count;
-	uint32_t dseg0_address[2];
-	uint32_t dseg0_len;
-	uint32_t dseg1_address[2];
-	uint32_t dseg1_len;
+	struct dsd64 dsd[2];
 };
 
 #define PT_LS4_UNSOL 0x56	/* pass-up unsolicited rec FC-NVMe request */
@@ -145,7 +142,6 @@ struct pt_ls4_rx_unsol {
 int qla_nvme_register_hba(struct scsi_qla_host *);
 int  qla_nvme_register_remote(struct scsi_qla_host *, struct fc_port *);
 void qla_nvme_delete(struct scsi_qla_host *);
-void qla_nvme_abort(struct qla_hw_data *, struct srb *sp, int res);
 void qla24xx_nvme_ls4_iocb(struct scsi_qla_host *, struct pt_ls4_request *,
     struct req_que *);
 void qla24xx_async_gffid_sp_done(void *, int);
diff --git a/drivers/scsi/qla2xxx/qla_nx.c b/drivers/scsi/qla2xxx/qla_nx.c
index f2f54806f4da..c760ae354174 100644
--- a/drivers/scsi/qla2xxx/qla_nx.c
+++ b/drivers/scsi/qla2xxx/qla_nx.c
@@ -6,6 +6,7 @@
  */
 #include "qla_def.h"
 #include <linux/delay.h>
+#include <linux/io-64-nonatomic-lo-hi.h>
 #include <linux/pci.h>
 #include <linux/ratelimit.h>
 #include <linux/vmalloc.h>
@@ -608,6 +609,7 @@ qla82xx_pci_set_window(struct qla_hw_data *ha, unsigned long long addr)
 	} else if (addr_in_range(addr, QLA82XX_ADDR_OCM0,
 		QLA82XX_ADDR_OCM0_MAX)) {
 		unsigned int temp1;
+
 		if ((addr & 0x00ff800) == 0xff800) {
 			ql_log(ql_log_warn, vha, 0xb004,
 			    "%s: QM access not handled.\n", __func__);
@@ -990,6 +992,7 @@ static int
 qla82xx_read_status_reg(struct qla_hw_data *ha, uint32_t *val)
 {
 	scsi_qla_host_t *vha = pci_get_drvdata(ha->pdev);
+
 	qla82xx_wr_32(ha, QLA82XX_ROMUSB_ROM_INSTR_OPCODE, M25P_INSTR_RDSR);
 	qla82xx_wait_rom_busy(ha);
 	if (qla82xx_wait_rom_done(ha)) {
@@ -1030,6 +1033,7 @@ static int
 qla82xx_flash_set_write_enable(struct qla_hw_data *ha)
 {
 	uint32_t val;
+
 	qla82xx_wait_rom_busy(ha);
 	qla82xx_wr_32(ha, QLA82XX_ROMUSB_ROM_ABYTE_CNT, 0);
 	qla82xx_wr_32(ha, QLA82XX_ROMUSB_ROM_INSTR_OPCODE, M25P_INSTR_WREN);
@@ -1047,6 +1051,7 @@ static int
 qla82xx_write_status_reg(struct qla_hw_data *ha, uint32_t val)
 {
 	scsi_qla_host_t *vha = pci_get_drvdata(ha->pdev);
+
 	if (qla82xx_flash_set_write_enable(ha))
 		return -1;
 	qla82xx_wr_32(ha, QLA82XX_ROMUSB_ROM_WDATA, val);
@@ -1063,6 +1068,7 @@ static int
 qla82xx_write_disable_flash(struct qla_hw_data *ha)
 {
 	scsi_qla_host_t *vha = pci_get_drvdata(ha->pdev);
+
 	qla82xx_wr_32(ha, QLA82XX_ROMUSB_ROM_INSTR_OPCODE, M25P_INSTR_WRDI);
 	if (qla82xx_wait_rom_done(ha)) {
 		ql_log(ql_log_warn, vha, 0xb00f,
@@ -1435,6 +1441,7 @@ qla82xx_fw_load_from_flash(struct qla_hw_data *ha)
 	long memaddr = BOOTLD_START;
 	u64 data;
 	u32 high, low;
+
 	size = (IMAGE_START - BOOTLD_START) / 8;
 
 	for (i = 0; i < size; i++) {
@@ -1757,11 +1764,14 @@ qla82xx_pci_config(scsi_qla_host_t *vha)
  *
  * Returns 0 on success.
  */
-void
+int
 qla82xx_reset_chip(scsi_qla_host_t *vha)
 {
 	struct qla_hw_data *ha = vha->hw;
+
 	ha->isp_ops->disable_intrs(ha);
+
+	return QLA_SUCCESS;
 }
 
 void qla82xx_config_rings(struct scsi_qla_host *vha)
@@ -1778,10 +1788,8 @@ void qla82xx_config_rings(struct scsi_qla_host *vha)
 	icb->response_q_inpointer = cpu_to_le16(0);
 	icb->request_q_length = cpu_to_le16(req->length);
 	icb->response_q_length = cpu_to_le16(rsp->length);
-	icb->request_q_address[0] = cpu_to_le32(LSD(req->dma));
-	icb->request_q_address[1] = cpu_to_le32(MSD(req->dma));
-	icb->response_q_address[0] = cpu_to_le32(LSD(rsp->dma));
-	icb->response_q_address[1] = cpu_to_le32(MSD(rsp->dma));
+	put_unaligned_le64(req->dma, &icb->request_q_address);
+	put_unaligned_le64(rsp->dma, &icb->response_q_address);
 
 	WRT_REG_DWORD(&reg->req_q_out[0], 0);
 	WRT_REG_DWORD(&reg->rsp_q_in[0], 0);
@@ -1992,6 +2000,7 @@ qla82xx_mbx_completion(scsi_qla_host_t *vha, uint16_t mb0)
 	uint16_t __iomem *wptr;
 	struct qla_hw_data *ha = vha->hw;
 	struct device_reg_82xx __iomem *reg = &ha->iobase->isp82;
+
 	wptr = (uint16_t __iomem *)&reg->mailbox_out[1];
 
 	/* Load return mailbox registers. */
@@ -2028,7 +2037,7 @@ qla82xx_intr_handler(int irq, void *dev_id)
 	unsigned long	flags;
 	unsigned long	iter;
 	uint32_t	stat = 0;
-	uint16_t	mb[4];
+	uint16_t	mb[8];
 
 	rsp = (struct rsp_que *) dev_id;
 	if (!rsp) {
@@ -2112,7 +2121,7 @@ qla82xx_msix_default(int irq, void *dev_id)
 	unsigned long flags;
 	uint32_t stat = 0;
 	uint32_t host_int = 0;
-	uint16_t mb[4];
+	uint16_t mb[8];
 
 	rsp = (struct rsp_que *) dev_id;
 	if (!rsp) {
@@ -2208,7 +2217,7 @@ qla82xx_poll(int irq, void *dev_id)
 	int status = 0;
 	uint32_t stat;
 	uint32_t host_int = 0;
-	uint16_t mb[4];
+	uint16_t mb[8];
 	unsigned long flags;
 
 	rsp = (struct rsp_que *) dev_id;
@@ -2262,6 +2271,7 @@ void
 qla82xx_enable_intrs(struct qla_hw_data *ha)
 {
 	scsi_qla_host_t *vha = pci_get_drvdata(ha->pdev);
+
 	qla82xx_mbx_intr_enable(vha);
 	spin_lock_irq(&ha->hardware_lock);
 	if (IS_QLA8044(ha))
@@ -2276,6 +2286,7 @@ void
 qla82xx_disable_intrs(struct qla_hw_data *ha)
 {
 	scsi_qla_host_t *vha = pci_get_drvdata(ha->pdev);
+
 	qla82xx_mbx_intr_disable(vha);
 	spin_lock_irq(&ha->hardware_lock);
 	if (IS_QLA8044(ha))
@@ -2658,8 +2669,8 @@ done:
 /*
  * Address and length are byte address
  */
-uint8_t *
-qla82xx_read_optrom_data(struct scsi_qla_host *vha, uint8_t *buf,
+void *
+qla82xx_read_optrom_data(struct scsi_qla_host *vha, void *buf,
 	uint32_t offset, uint32_t length)
 {
 	scsi_block_requests(vha->host);
@@ -2767,15 +2778,14 @@ write_done:
 }
 
 int
-qla82xx_write_optrom_data(struct scsi_qla_host *vha, uint8_t *buf,
+qla82xx_write_optrom_data(struct scsi_qla_host *vha, void *buf,
 	uint32_t offset, uint32_t length)
 {
 	int rval;
 
 	/* Suspend HBA. */
 	scsi_block_requests(vha->host);
-	rval = qla82xx_write_flash_data(vha, (uint32_t *)buf, offset,
-		length >> 2);
+	rval = qla82xx_write_flash_data(vha, buf, offset, length >> 2);
 	scsi_unblock_requests(vha->host);
 
 	/* Convert return ISP82xx to generic */
@@ -4464,6 +4474,7 @@ qla82xx_beacon_on(struct scsi_qla_host *vha)
 
 	int rval;
 	struct qla_hw_data *ha = vha->hw;
+
 	qla82xx_idc_lock(ha);
 	rval = qla82xx_mbx_beacon_ctl(vha, 1);
 
@@ -4484,6 +4495,7 @@ qla82xx_beacon_off(struct scsi_qla_host *vha)
 
 	int rval;
 	struct qla_hw_data *ha = vha->hw;
+
 	qla82xx_idc_lock(ha);
 	rval = qla82xx_mbx_beacon_ctl(vha, 0);
 
diff --git a/drivers/scsi/qla2xxx/qla_nx.h b/drivers/scsi/qla2xxx/qla_nx.h
index 71a41093530e..3c7beef92c35 100644
--- a/drivers/scsi/qla2xxx/qla_nx.h
+++ b/drivers/scsi/qla2xxx/qla_nx.h
@@ -7,7 +7,7 @@
 #ifndef __QLA_NX_H
 #define __QLA_NX_H
 
-#include <linux/io-64-nonatomic-lo-hi.h>
+#include <scsi/scsi.h>
 
 /*
  * Following are the states of the Phantom. Phantom will set them and
diff --git a/drivers/scsi/qla2xxx/qla_nx2.c b/drivers/scsi/qla2xxx/qla_nx2.c
index fe856b602e03..369ac04d0454 100644
--- a/drivers/scsi/qla2xxx/qla_nx2.c
+++ b/drivers/scsi/qla2xxx/qla_nx2.c
@@ -559,12 +559,12 @@ exit_lock_error:
 /*
  * Address and length are byte address
  */
-uint8_t *
-qla8044_read_optrom_data(struct scsi_qla_host *vha, uint8_t *buf,
+void *
+qla8044_read_optrom_data(struct scsi_qla_host *vha, void *buf,
 	uint32_t offset, uint32_t length)
 {
 	scsi_block_requests(vha->host);
-	if (qla8044_read_flash_data(vha, (uint8_t *)buf, offset, length / 4)
+	if (qla8044_read_flash_data(vha, buf, offset, length / 4)
 	    != QLA_SUCCESS) {
 		ql_log(ql_log_warn, vha,  0xb08d,
 		    "%s: Failed to read from flash\n",
@@ -3007,10 +3007,9 @@ qla8044_minidump_process_rddfe(struct scsi_qla_host *vha,
 	uint16_t count;
 	uint32_t poll, mask, modify_mask;
 	uint32_t wait_count = 0;
-
 	uint32_t *data_ptr = *d_ptr;
-
 	struct qla8044_minidump_entry_rddfe *rddfe;
+
 	rddfe = (struct qla8044_minidump_entry_rddfe *) entry_hdr;
 
 	addr1 = rddfe->addr_1;
@@ -3797,7 +3796,7 @@ qla8044_write_flash_dword_mode(scsi_qla_host_t *vha, uint32_t *dwptr,
 }
 
 int
-qla8044_write_optrom_data(struct scsi_qla_host *vha, uint8_t *buf,
+qla8044_write_optrom_data(struct scsi_qla_host *vha, void *buf,
 			  uint32_t offset, uint32_t length)
 {
 	int rval = QLA_FUNCTION_FAILED, i, burst_iter_count;
@@ -3896,7 +3895,7 @@ qla8044_intr_handler(int irq, void *dev_id)
 	unsigned long	flags;
 	unsigned long	iter;
 	uint32_t	stat;
-	uint16_t	mb[4];
+	uint16_t	mb[8];
 	uint32_t leg_int_ptr = 0, pf_bit;
 
 	rsp = (struct rsp_que *) dev_id;
diff --git a/drivers/scsi/qla2xxx/qla_os.c b/drivers/scsi/qla2xxx/qla_os.c
index 91f576d743fe..e1c82a0a9745 100644
--- a/drivers/scsi/qla2xxx/qla_os.c
+++ b/drivers/scsi/qla2xxx/qla_os.c
@@ -42,7 +42,7 @@ static struct kmem_cache *ctx_cachep;
 /*
  * error level for logging
  */
-int ql_errlev = ql_log_all;
+uint ql_errlev = 0x8001;
 
 static int ql2xenableclass2;
 module_param(ql2xenableclass2, int, S_IRUGO|S_IRUSR);
@@ -108,7 +108,7 @@ MODULE_PARM_DESC(ql2xshiftctondsd,
 		"Set to control shifting of command type processing "
 		"based on total number of SG elements.");
 
-int ql2xfdmienable=1;
+int ql2xfdmienable = 1;
 module_param(ql2xfdmienable, int, S_IRUGO|S_IWUSR);
 module_param_named(fdmi, ql2xfdmienable, int, S_IRUGO|S_IWUSR);
 MODULE_PARM_DESC(ql2xfdmienable,
@@ -154,7 +154,7 @@ MODULE_PARM_DESC(ql2xenablehba_err_chk,
 		"  1 -- Error isolation enabled only for DIX Type 0\n"
 		"  2 -- Error isolation enabled for all Types\n");
 
-int ql2xiidmaenable=1;
+int ql2xiidmaenable = 1;
 module_param(ql2xiidmaenable, int, S_IRUGO);
 MODULE_PARM_DESC(ql2xiidmaenable,
 		"Enables iIDMA settings "
@@ -285,14 +285,14 @@ MODULE_PARM_DESC(qla2xuseresexchforels,
 		 "Reserve 1/2 of emergency exchanges for ELS.\n"
 		 " 0 (default): disabled");
 
-int ql2xprotmask;
+static int ql2xprotmask;
 module_param(ql2xprotmask, int, 0644);
 MODULE_PARM_DESC(ql2xprotmask,
 		 "Override DIF/DIX protection capabilities mask\n"
 		 "Default is 0 which sets protection mask based on "
 		 "capabilities reported by HBA firmware.\n");
 
-int ql2xprotguard;
+static int ql2xprotguard;
 module_param(ql2xprotguard, int, 0644);
 MODULE_PARM_DESC(ql2xprotguard, "Override choice of DIX checksum\n"
 		 "  0 -- Let HBA firmware decide\n"
@@ -306,58 +306,12 @@ MODULE_PARM_DESC(ql2xdifbundlinginternalbuffers,
     "0 (Default). Based on check.\n"
     "1 Force using internal buffers\n");
 
-/*
- * SCSI host template entry points
- */
-static int qla2xxx_slave_configure(struct scsi_device * device);
-static int qla2xxx_slave_alloc(struct scsi_device *);
-static int qla2xxx_scan_finished(struct Scsi_Host *, unsigned long time);
-static void qla2xxx_scan_start(struct Scsi_Host *);
-static void qla2xxx_slave_destroy(struct scsi_device *);
-static int qla2xxx_queuecommand(struct Scsi_Host *h, struct scsi_cmnd *cmd);
-static int qla2xxx_eh_abort(struct scsi_cmnd *);
-static int qla2xxx_eh_device_reset(struct scsi_cmnd *);
-static int qla2xxx_eh_target_reset(struct scsi_cmnd *);
-static int qla2xxx_eh_bus_reset(struct scsi_cmnd *);
-static int qla2xxx_eh_host_reset(struct scsi_cmnd *);
-
 static void qla2x00_clear_drv_active(struct qla_hw_data *);
 static void qla2x00_free_device(scsi_qla_host_t *);
 static int qla2xxx_map_queues(struct Scsi_Host *shost);
 static void qla2x00_destroy_deferred_work(struct qla_hw_data *);
 
 
-struct scsi_host_template qla2xxx_driver_template = {
-	.module			= THIS_MODULE,
-	.name			= QLA2XXX_DRIVER_NAME,
-	.queuecommand		= qla2xxx_queuecommand,
-
-	.eh_timed_out		= fc_eh_timed_out,
-	.eh_abort_handler	= qla2xxx_eh_abort,
-	.eh_device_reset_handler = qla2xxx_eh_device_reset,
-	.eh_target_reset_handler = qla2xxx_eh_target_reset,
-	.eh_bus_reset_handler	= qla2xxx_eh_bus_reset,
-	.eh_host_reset_handler	= qla2xxx_eh_host_reset,
-
-	.slave_configure	= qla2xxx_slave_configure,
-
-	.slave_alloc		= qla2xxx_slave_alloc,
-	.slave_destroy		= qla2xxx_slave_destroy,
-	.scan_finished		= qla2xxx_scan_finished,
-	.scan_start		= qla2xxx_scan_start,
-	.change_queue_depth	= scsi_change_queue_depth,
-	.map_queues             = qla2xxx_map_queues,
-	.this_id		= -1,
-	.cmd_per_lun		= 3,
-	.sg_tablesize		= SG_ALL,
-
-	.max_sectors		= 0xFFFF,
-	.shost_attrs		= qla2x00_host_attrs,
-
-	.supported_mode		= MODE_INITIATOR,
-	.track_queue_depth	= 1,
-};
-
 static struct scsi_transport_template *qla2xxx_transport_template = NULL;
 struct scsi_transport_template *qla2xxx_transport_vport_template = NULL;
 
@@ -411,6 +365,7 @@ static void qla_init_base_qpair(struct scsi_qla_host *vha, struct req_que *req,
     struct rsp_que *rsp)
 {
 	struct qla_hw_data *ha = vha->hw;
+
 	rsp->qpair = ha->base_qpair;
 	rsp->req = req;
 	ha->base_qpair->hw = ha;
@@ -427,7 +382,7 @@ static void qla_init_base_qpair(struct scsi_qla_host *vha, struct req_que *req,
 	qla_cpu_update(rsp->qpair, raw_smp_processor_id());
 	ha->base_qpair->pdev = ha->pdev;
 
-	if (IS_QLA27XX(ha) || IS_QLA83XX(ha))
+	if (IS_QLA27XX(ha) || IS_QLA83XX(ha) || IS_QLA28XX(ha))
 		ha->base_qpair->reqq_start_iocbs = qla_83xx_start_iocbs;
 }
 
@@ -435,6 +390,7 @@ static int qla2x00_alloc_queues(struct qla_hw_data *ha, struct req_que *req,
 				struct rsp_que *rsp)
 {
 	scsi_qla_host_t *vha = pci_get_drvdata(ha->pdev);
+
 	ha->req_q_map = kcalloc(ha->max_req_queues, sizeof(struct req_que *),
 				GFP_KERNEL);
 	if (!ha->req_q_map) {
@@ -726,7 +682,7 @@ qla2x00_sp_free_dma(void *ptr)
 	}
 
 	if (!ctx)
-		goto end;
+		return;
 
 	if (sp->flags & SRB_CRC_CTX_DSD_VALID) {
 		/* List assured to be having elements */
@@ -751,12 +707,6 @@ qla2x00_sp_free_dma(void *ptr)
 		ha->gbl_dsd_avail += ctx1->dsd_use_cnt;
 		mempool_free(ctx1, ha->ctx_mempool);
 	}
-
-end:
-	if (sp->type != SRB_NVME_CMD && sp->type != SRB_NVME_LS) {
-		CMD_SP(cmd) = NULL;
-		qla2x00_rel_sp(sp);
-	}
 }
 
 void
@@ -764,22 +714,20 @@ qla2x00_sp_compl(void *ptr, int res)
 {
 	srb_t *sp = ptr;
 	struct scsi_cmnd *cmd = GET_CMD_SP(sp);
+	struct completion *comp = sp->comp;
 
-	cmd->result = res;
-
-	if (atomic_read(&sp->ref_count) == 0) {
-		ql_dbg(ql_dbg_io, sp->vha, 0x3015,
-		    "SP reference-count to ZERO -- sp=%p cmd=%p.\n",
-		    sp, GET_CMD_SP(sp));
-		if (ql2xextended_error_logging & ql_dbg_io)
-			WARN_ON(atomic_read(&sp->ref_count) == 0);
-		return;
-	}
-	if (!atomic_dec_and_test(&sp->ref_count))
+	if (WARN_ON_ONCE(atomic_read(&sp->ref_count) == 0))
 		return;
 
+	atomic_dec(&sp->ref_count);
+
 	sp->free(sp);
+	cmd->result = res;
+	CMD_SP(cmd) = NULL;
 	cmd->scsi_done(cmd);
+	if (comp)
+		complete(comp);
+	qla2x00_rel_sp(sp);
 }
 
 void
@@ -802,7 +750,7 @@ qla2xxx_qpair_sp_free_dma(void *ptr)
 	}
 
 	if (!ctx)
-		goto end;
+		return;
 
 	if (sp->flags & SRB_CRC_CTX_DSD_VALID) {
 		/* List assured to be having elements */
@@ -810,25 +758,8 @@ qla2xxx_qpair_sp_free_dma(void *ptr)
 		sp->flags &= ~SRB_CRC_CTX_DSD_VALID;
 	}
 
-	if (sp->flags & SRB_CRC_CTX_DMA_VALID) {
-		struct crc_context *ctx0 = ctx;
-
-		dma_pool_free(ha->dl_dma_pool, ctx, ctx0->crc_ctx_dma);
-		sp->flags &= ~SRB_CRC_CTX_DMA_VALID;
-	}
-
-	if (sp->flags & SRB_FCP_CMND_DMA_VALID) {
-		struct ct6_dsd *ctx1 = ctx;
-		dma_pool_free(ha->fcp_cmnd_dma_pool, ctx1->fcp_cmnd,
-		    ctx1->fcp_cmnd_dma);
-		list_splice(&ctx1->dsd_list, &ha->gbl_dsd_list);
-		ha->gbl_dsd_inuse -= ctx1->dsd_use_cnt;
-		ha->gbl_dsd_avail += ctx1->dsd_use_cnt;
-		mempool_free(ctx1, ha->ctx_mempool);
-		sp->flags &= ~SRB_FCP_CMND_DMA_VALID;
-	}
 	if (sp->flags & SRB_DIF_BUNDL_DMA_VALID) {
-		struct crc_context *difctx = sp->u.scmd.ctx;
+		struct crc_context *difctx = ctx;
 		struct dsd_dma *dif_dsd, *nxt_dsd;
 
 		list_for_each_entry_safe(dif_dsd, nxt_dsd,
@@ -863,9 +794,24 @@ qla2xxx_qpair_sp_free_dma(void *ptr)
 		sp->flags &= ~SRB_DIF_BUNDL_DMA_VALID;
 	}
 
-end:
-	CMD_SP(cmd) = NULL;
-	qla2xxx_rel_qpair_sp(sp->qpair, sp);
+	if (sp->flags & SRB_FCP_CMND_DMA_VALID) {
+		struct ct6_dsd *ctx1 = ctx;
+
+		dma_pool_free(ha->fcp_cmnd_dma_pool, ctx1->fcp_cmnd,
+		    ctx1->fcp_cmnd_dma);
+		list_splice(&ctx1->dsd_list, &ha->gbl_dsd_list);
+		ha->gbl_dsd_inuse -= ctx1->dsd_use_cnt;
+		ha->gbl_dsd_avail += ctx1->dsd_use_cnt;
+		mempool_free(ctx1, ha->ctx_mempool);
+		sp->flags &= ~SRB_FCP_CMND_DMA_VALID;
+	}
+
+	if (sp->flags & SRB_CRC_CTX_DMA_VALID) {
+		struct crc_context *ctx0 = ctx;
+
+		dma_pool_free(ha->dl_dma_pool, ctx, ctx0->crc_ctx_dma);
+		sp->flags &= ~SRB_CRC_CTX_DMA_VALID;
+	}
 }
 
 void
@@ -873,27 +819,22 @@ qla2xxx_qpair_sp_compl(void *ptr, int res)
 {
 	srb_t *sp = ptr;
 	struct scsi_cmnd *cmd = GET_CMD_SP(sp);
+	struct completion *comp = sp->comp;
 
-	cmd->result = res;
-
-	if (atomic_read(&sp->ref_count) == 0) {
-		ql_dbg(ql_dbg_io, sp->fcport->vha, 0x3079,
-		    "SP reference-count to ZERO -- sp=%p cmd=%p.\n",
-		    sp, GET_CMD_SP(sp));
-		if (ql2xextended_error_logging & ql_dbg_io)
-			WARN_ON(atomic_read(&sp->ref_count) == 0);
-		return;
-	}
-	if (!atomic_dec_and_test(&sp->ref_count))
+	if (WARN_ON_ONCE(atomic_read(&sp->ref_count) == 0))
 		return;
 
+	atomic_dec(&sp->ref_count);
+
 	sp->free(sp);
+	cmd->result = res;
+	CMD_SP(cmd) = NULL;
 	cmd->scsi_done(cmd);
+	if (comp)
+		complete(comp);
+	qla2xxx_rel_qpair_sp(sp->qpair, sp);
 }
 
-/* If we are SP1 here, we need to still take and release the host_lock as SP1
- * does not have the changes necessary to avoid taking host->host_lock.
- */
 static int
 qla2xxx_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
 {
@@ -908,7 +849,8 @@ qla2xxx_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
 	uint32_t tag;
 	uint16_t hwq;
 
-	if (unlikely(test_bit(UNLOADING, &base_vha->dpc_flags))) {
+	if (unlikely(test_bit(UNLOADING, &base_vha->dpc_flags)) ||
+	    WARN_ON_ONCE(!rport)) {
 		cmd->result = DID_NO_CONNECT << 16;
 		goto qc24_fail_command;
 	}
@@ -1031,7 +973,7 @@ qla2xxx_mqueuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd,
 	srb_t *sp;
 	int rval;
 
-	rval = fc_remote_port_chkready(rport);
+	rval = rport ? fc_remote_port_chkready(rport) : FC_PORTSTATE_OFFLINE;
 	if (rval) {
 		cmd->result = rval;
 		ql_dbg(ql_dbg_io + ql_dbg_verbose, vha, 0x3076,
@@ -1272,7 +1214,7 @@ qla2x00_wait_for_chip_reset(scsi_qla_host_t *vha)
 static int
 sp_get(struct srb *sp)
 {
-	if (!refcount_inc_not_zero((refcount_t*)&sp->ref_count))
+	if (!refcount_inc_not_zero((refcount_t *)&sp->ref_count))
 		/* kref get fail */
 		return ENXIO;
 	else
@@ -1332,7 +1274,7 @@ qla2xxx_eh_abort(struct scsi_cmnd *cmd)
 	unsigned int id;
 	uint64_t lun;
 	unsigned long flags;
-	int rval, wait = 0;
+	int rval;
 	struct qla_hw_data *ha = vha->hw;
 	struct qla_qpair *qpair;
 
@@ -1345,7 +1287,6 @@ qla2xxx_eh_abort(struct scsi_cmnd *cmd)
 	ret = fc_block_scsi_eh(cmd);
 	if (ret != 0)
 		return ret;
-	ret = SUCCESS;
 
 	sp = (srb_t *) CMD_SP(cmd);
 	if (!sp)
@@ -1356,7 +1297,7 @@ qla2xxx_eh_abort(struct scsi_cmnd *cmd)
 		return SUCCESS;
 
 	spin_lock_irqsave(qpair->qp_lock_ptr, flags);
-	if (!CMD_SP(cmd)) {
+	if (sp->type != SRB_SCSI_CMD || GET_CMD_SP(sp) != cmd) {
 		/* there's a chance an interrupt could clear
 		   the ptr as part of done & free */
 		spin_unlock_irqrestore(qpair->qp_lock_ptr, flags);
@@ -1377,58 +1318,31 @@ qla2xxx_eh_abort(struct scsi_cmnd *cmd)
 	    "Aborting from RISC nexus=%ld:%d:%llu sp=%p cmd=%p handle=%x\n",
 	    vha->host_no, id, lun, sp, cmd, sp->handle);
 
-	/* Get a reference to the sp and drop the lock.*/
-
 	rval = ha->isp_ops->abort_command(sp);
-	if (rval) {
-		if (rval == QLA_FUNCTION_PARAMETER_ERROR)
-			ret = SUCCESS;
-		else
-			ret = FAILED;
-
-		ql_dbg(ql_dbg_taskm, vha, 0x8003,
-		    "Abort command mbx failed cmd=%p, rval=%x.\n", cmd, rval);
-	} else {
-		ql_dbg(ql_dbg_taskm, vha, 0x8004,
-		    "Abort command mbx success cmd=%p.\n", cmd);
-		wait = 1;
-	}
-
-	spin_lock_irqsave(qpair->qp_lock_ptr, flags);
-	/*
-	 * Clear the slot in the oustanding_cmds array if we can't find the
-	 * command to reclaim the resources.
-	 */
-	if (rval == QLA_FUNCTION_PARAMETER_ERROR)
-		vha->req->outstanding_cmds[sp->handle] = NULL;
+	ql_dbg(ql_dbg_taskm, vha, 0x8003,
+	       "Abort command mbx cmd=%p, rval=%x.\n", cmd, rval);
 
-	/*
-	 * sp->done will do ref_count--
-	 * sp_get() took an extra count above
-	 */
-	sp->done(sp, DID_RESET << 16);
-
-	/* Did the command return during mailbox execution? */
-	if (ret == FAILED && !CMD_SP(cmd))
+	switch (rval) {
+	case QLA_SUCCESS:
+		/*
+		 * The command has been aborted. That means that the firmware
+		 * won't report a completion.
+		 */
+		sp->done(sp, DID_ABORT << 16);
 		ret = SUCCESS;
-
-	if (!CMD_SP(cmd))
-		wait = 0;
-
-	spin_unlock_irqrestore(qpair->qp_lock_ptr, flags);
-
-	/* Wait for the command to be returned. */
-	if (wait) {
-		if (qla2x00_eh_wait_on_command(cmd) != QLA_SUCCESS) {
-			ql_log(ql_log_warn, vha, 0x8006,
-			    "Abort handler timed out cmd=%p.\n", cmd);
-			ret = FAILED;
-		}
+		break;
+	default:
+		/*
+		 * Either abort failed or abort and completion raced. Let
+		 * the SCSI core retry the abort in the former case.
+		 */
+		ret = FAILED;
+		break;
 	}
 
 	ql_log(ql_log_info, vha, 0x801c,
-	    "Abort command issued nexus=%ld:%d:%llu --  %d %x.\n",
-	    vha->host_no, id, lun, wait, ret);
+	    "Abort command issued nexus=%ld:%d:%llu -- %x.\n",
+	    vha->host_no, id, lun, ret);
 
 	return ret;
 }
@@ -1804,42 +1718,34 @@ static void qla2x00_abort_srb(struct qla_qpair *qp, srb_t *sp, const int res,
 	__releases(qp->qp_lock_ptr)
 	__acquires(qp->qp_lock_ptr)
 {
+	DECLARE_COMPLETION_ONSTACK(comp);
 	scsi_qla_host_t *vha = qp->vha;
 	struct qla_hw_data *ha = vha->hw;
+	int rval;
 
-	if (sp->type == SRB_NVME_CMD || sp->type == SRB_NVME_LS) {
-		if (!sp_get(sp)) {
-			/* got sp */
-			spin_unlock_irqrestore(qp->qp_lock_ptr, *flags);
-			qla_nvme_abort(ha, sp, res);
-			spin_lock_irqsave(qp->qp_lock_ptr, *flags);
-		}
-	} else if (GET_CMD_SP(sp) && !ha->flags.eeh_busy &&
-		   !test_bit(ABORT_ISP_ACTIVE, &vha->dpc_flags) &&
-		   !qla2x00_isp_reg_stat(ha) && sp->type == SRB_SCSI_CMD) {
-		/*
-		 * Don't abort commands in adapter during EEH recovery as it's
-		 * not accessible/responding.
-		 *
-		 * Get a reference to the sp and drop the lock. The reference
-		 * ensures this sp->done() call and not the call in
-		 * qla2xxx_eh_abort() ends the SCSI cmd (with result 'res').
-		 */
-		if (!sp_get(sp)) {
-			int status;
+	if (sp_get(sp))
+		return;
 
-			spin_unlock_irqrestore(qp->qp_lock_ptr, *flags);
-			status = qla2xxx_eh_abort(GET_CMD_SP(sp));
-			spin_lock_irqsave(qp->qp_lock_ptr, *flags);
-			/*
-			 * Get rid of extra reference caused
-			 * by early exit from qla2xxx_eh_abort
-			 */
-			if (status == FAST_IO_FAIL)
-				atomic_dec(&sp->ref_count);
+	if (sp->type == SRB_NVME_CMD || sp->type == SRB_NVME_LS ||
+	    (sp->type == SRB_SCSI_CMD && !ha->flags.eeh_busy &&
+	     !test_bit(ABORT_ISP_ACTIVE, &vha->dpc_flags) &&
+	     !qla2x00_isp_reg_stat(ha))) {
+		sp->comp = &comp;
+		rval = ha->isp_ops->abort_command(sp);
+		spin_unlock_irqrestore(qp->qp_lock_ptr, *flags);
+
+		switch (rval) {
+		case QLA_SUCCESS:
+			sp->done(sp, res);
+			break;
+		case QLA_FUNCTION_PARAMETER_ERROR:
+			wait_for_completion(&comp);
+			break;
 		}
+
+		spin_lock_irqsave(qp->qp_lock_ptr, *flags);
+		sp->comp = NULL;
 	}
-	sp->done(sp, res);
 }
 
 static void
@@ -1875,15 +1781,10 @@ __qla2x00_abort_all_cmds(struct qla_qpair *qp, int res)
 					continue;
 				}
 				cmd = (struct qla_tgt_cmd *)sp;
-				qlt_abort_cmd_on_host_reset(cmd->vha, cmd);
+				cmd->aborted = 1;
 				break;
 			case TYPE_TGT_TMCMD:
-				/*
-				 * Currently, only ABTS response gets on the
-				 * outstanding_cmds[]
-				 */
-				ha->tgt.tgt_ops->free_mcmd(
-				   (struct qla_tgt_mgmt_cmd *)sp);
+				/* Skip task management functions. */
 				break;
 			default:
 				break;
@@ -2753,6 +2654,24 @@ qla2x00_set_isp_flags(struct qla_hw_data *ha)
 		ha->device_type |= DT_T10_PI;
 		ha->fw_srisc_address = RISC_START_ADDRESS_2400;
 		break;
+	case PCI_DEVICE_ID_QLOGIC_ISP2081:
+	case PCI_DEVICE_ID_QLOGIC_ISP2089:
+		ha->isp_type |= DT_ISP2081;
+		ha->device_type |= DT_ZIO_SUPPORTED;
+		ha->device_type |= DT_FWI2;
+		ha->device_type |= DT_IIDMA;
+		ha->device_type |= DT_T10_PI;
+		ha->fw_srisc_address = RISC_START_ADDRESS_2400;
+		break;
+	case PCI_DEVICE_ID_QLOGIC_ISP2281:
+	case PCI_DEVICE_ID_QLOGIC_ISP2289:
+		ha->isp_type |= DT_ISP2281;
+		ha->device_type |= DT_ZIO_SUPPORTED;
+		ha->device_type |= DT_FWI2;
+		ha->device_type |= DT_IIDMA;
+		ha->device_type |= DT_T10_PI;
+		ha->fw_srisc_address = RISC_START_ADDRESS_2400;
+		break;
 	}
 
 	if (IS_QLA82XX(ha))
@@ -2760,7 +2679,8 @@ qla2x00_set_isp_flags(struct qla_hw_data *ha)
 	else {
 		/* Get adapter physical port no from interrupt pin register. */
 		pci_read_config_byte(ha->pdev, PCI_INTERRUPT_PIN, &ha->port_no);
-		if (IS_QLA27XX(ha))
+		if (IS_QLA25XX(ha) || IS_QLA2031(ha) ||
+		    IS_QLA27XX(ha) || IS_QLA28XX(ha))
 			ha->port_no--;
 		else
 			ha->port_no = !(ha->port_no & 1);
@@ -2857,7 +2777,11 @@ qla2x00_probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
 	    pdev->device == PCI_DEVICE_ID_QLOGIC_ISP8044 ||
 	    pdev->device == PCI_DEVICE_ID_QLOGIC_ISP2071 ||
 	    pdev->device == PCI_DEVICE_ID_QLOGIC_ISP2271 ||
-	    pdev->device == PCI_DEVICE_ID_QLOGIC_ISP2261) {
+	    pdev->device == PCI_DEVICE_ID_QLOGIC_ISP2261 ||
+	    pdev->device == PCI_DEVICE_ID_QLOGIC_ISP2081 ||
+	    pdev->device == PCI_DEVICE_ID_QLOGIC_ISP2281 ||
+	    pdev->device == PCI_DEVICE_ID_QLOGIC_ISP2089 ||
+	    pdev->device == PCI_DEVICE_ID_QLOGIC_ISP2289) {
 		bars = pci_select_bars(pdev, IORESOURCE_MEM);
 		mem_only = 1;
 		ql_dbg_pci(ql_dbg_init, pdev, 0x0007,
@@ -2877,6 +2801,10 @@ qla2x00_probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
 	/* This may fail but that's ok */
 	pci_enable_pcie_error_reporting(pdev);
 
+	/* Turn off T10-DIF when FC-NVMe is enabled */
+	if (ql2xnvmeenable)
+		ql2xenabledif = 0;
+
 	ha = kzalloc(sizeof(struct qla_hw_data), GFP_KERNEL);
 	if (!ha) {
 		ql_log_pci(ql_log_fatal, pdev, 0x0009,
@@ -2906,7 +2834,7 @@ qla2x00_probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
 
 	/* Set EEH reset type to fundamental if required by hba */
 	if (IS_QLA24XX(ha) || IS_QLA25XX(ha) || IS_QLA81XX(ha) ||
-	    IS_QLA83XX(ha) || IS_QLA27XX(ha))
+	    IS_QLA83XX(ha) || IS_QLA27XX(ha) || IS_QLA28XX(ha))
 		pdev->needs_freset = 1;
 
 	ha->prev_topology = 0;
@@ -3085,6 +3013,23 @@ qla2x00_probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
 		ha->flash_data_off = FARX_ACCESS_FLASH_DATA_81XX;
 		ha->nvram_conf_off = ~0;
 		ha->nvram_data_off = ~0;
+	} else if (IS_QLA28XX(ha)) {
+		ha->portnum = PCI_FUNC(ha->pdev->devfn);
+		ha->max_fibre_devices = MAX_FIBRE_DEVICES_2400;
+		ha->mbx_count = MAILBOX_REGISTER_COUNT;
+		req_length = REQUEST_ENTRY_CNT_24XX;
+		rsp_length = RESPONSE_ENTRY_CNT_2300;
+		ha->tgt.atio_q_length = ATIO_ENTRY_CNT_24XX;
+		ha->max_loop_id = SNS_LAST_LOOP_ID_2300;
+		ha->init_cb_size = sizeof(struct mid_init_cb_81xx);
+		ha->gid_list_info_size = 8;
+		ha->optrom_size = OPTROM_SIZE_28XX;
+		ha->nvram_npiv_size = QLA_MAX_VPORTS_QLA25XX;
+		ha->isp_ops = &qla27xx_isp_ops;
+		ha->flash_conf_off = FARX_ACCESS_FLASH_CONF_28XX;
+		ha->flash_data_off = FARX_ACCESS_FLASH_DATA_28XX;
+		ha->nvram_conf_off = ~0;
+		ha->nvram_data_off = ~0;
 	}
 
 	ql_dbg_pci(ql_dbg_init, pdev, 0x001e,
@@ -3250,7 +3195,8 @@ qla2x00_probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
 	req->req_q_out = &ha->iobase->isp24.req_q_out;
 	rsp->rsp_q_in = &ha->iobase->isp24.rsp_q_in;
 	rsp->rsp_q_out = &ha->iobase->isp24.rsp_q_out;
-	if (ha->mqenable || IS_QLA83XX(ha) || IS_QLA27XX(ha)) {
+	if (ha->mqenable || IS_QLA83XX(ha) || IS_QLA27XX(ha) ||
+	    IS_QLA28XX(ha)) {
 		req->req_q_in = &ha->mqiobase->isp25mq.req_q_in;
 		req->req_q_out = &ha->mqiobase->isp25mq.req_q_out;
 		rsp->rsp_q_in = &ha->mqiobase->isp25mq.rsp_q_in;
@@ -3395,6 +3341,7 @@ skip_dpc:
 	if (IS_T10_PI_CAPABLE(ha) && ql2xenabledif) {
 		if (ha->fw_attributes & BIT_4) {
 			int prot = 0, guard;
+
 			base_vha->flags.difdix_supported = 1;
 			ql_dbg(ql_dbg_init, base_vha, 0x00f1,
 			    "Registering for DIF/DIX type 1 and 3 protection.\n");
@@ -3576,7 +3523,8 @@ qla2x00_shutdown(struct pci_dev *pdev)
 	if (ha->eft)
 		qla2x00_disable_eft_trace(vha);
 
-	if (IS_QLA25XX(ha) ||  IS_QLA2031(ha) || IS_QLA27XX(ha)) {
+	if (IS_QLA25XX(ha) ||  IS_QLA2031(ha) || IS_QLA27XX(ha) ||
+	    IS_QLA28XX(ha)) {
 		if (ha->flags.fw_started)
 			qla2x00_abort_isp_cleanup(vha);
 	} else {
@@ -3681,7 +3629,8 @@ qla2x00_unmap_iobases(struct qla_hw_data *ha)
 		if (ha->mqiobase)
 			iounmap(ha->mqiobase);
 
-		if ((IS_QLA83XX(ha) || IS_QLA27XX(ha)) && ha->msixbase)
+		if ((IS_QLA83XX(ha) || IS_QLA27XX(ha) || IS_QLA28XX(ha)) &&
+		    ha->msixbase)
 			iounmap(ha->msixbase);
 	}
 }
@@ -3732,7 +3681,8 @@ qla2x00_remove_one(struct pci_dev *pdev)
 	}
 	qla2x00_wait_for_hba_ready(base_vha);
 
-	if (IS_QLA25XX(ha) || IS_QLA2031(ha) || IS_QLA27XX(ha)) {
+	if (IS_QLA25XX(ha) || IS_QLA2031(ha) || IS_QLA27XX(ha) ||
+	    IS_QLA28XX(ha)) {
 		if (ha->flags.fw_started)
 			qla2x00_abort_isp_cleanup(base_vha);
 	} else if (!IS_QLAFX00(ha)) {
@@ -3770,8 +3720,6 @@ qla2x00_remove_one(struct pci_dev *pdev)
 
 	qla2x00_delete_all_vps(ha, base_vha);
 
-	qla2x00_abort_all_cmds(base_vha, DID_NO_CONNECT << 16);
-
 	qla2x00_dfs_remove(base_vha);
 
 	qla84xx_put_chip(base_vha);
@@ -3860,11 +3808,8 @@ void qla2x00_free_fcports(struct scsi_qla_host *vha)
 {
 	fc_port_t *fcport, *tfcport;
 
-	list_for_each_entry_safe(fcport, tfcport, &vha->vp_fcports, list) {
-		list_del(&fcport->list);
-		qla2x00_clear_loop_id(fcport);
-		kfree(fcport);
-	}
+	list_for_each_entry_safe(fcport, tfcport, &vha->vp_fcports, list)
+		qla2x00_free_fcport(fcport);
 }
 
 static inline void
@@ -3889,6 +3834,7 @@ qla2x00_schedule_rport_del(struct scsi_qla_host *vha, fc_port_t *fcport,
 		qla2xxx_wake_dpc(base_vha);
 	} else {
 		int now;
+
 		if (rport) {
 			ql_dbg(ql_dbg_disc, fcport->vha, 0x2109,
 			    "%s %8phN. rport %p roles %x\n",
@@ -3980,6 +3926,19 @@ qla2x00_mark_all_devices_lost(scsi_qla_host_t *vha, int defer)
 	}
 }
 
+static void qla2x00_set_reserved_loop_ids(struct qla_hw_data *ha)
+{
+	int i;
+
+	if (IS_FWI2_CAPABLE(ha))
+		return;
+
+	for (i = 0; i < SNS_FIRST_LOOP_ID; i++)
+		set_bit(i, ha->loop_id_map);
+	set_bit(MANAGEMENT_SERVER, ha->loop_id_map);
+	set_bit(BROADCAST, ha->loop_id_map);
+}
+
 /*
 * qla2x00_mem_alloc
 *      Allocates adapter memory.
@@ -4222,7 +4181,8 @@ qla2x00_mem_alloc(struct qla_hw_data *ha, uint16_t req_len, uint16_t rsp_len,
 		ha->npiv_info = NULL;
 
 	/* Get consistent memory allocated for EX-INIT-CB. */
-	if (IS_CNA_CAPABLE(ha) || IS_QLA2031(ha) || IS_QLA27XX(ha)) {
+	if (IS_CNA_CAPABLE(ha) || IS_QLA2031(ha) || IS_QLA27XX(ha) ||
+	    IS_QLA28XX(ha)) {
 		ha->ex_init_cb = dma_pool_alloc(ha->s_dma_pool, GFP_KERNEL,
 		    &ha->ex_init_cb_dma);
 		if (!ha->ex_init_cb)
@@ -4265,8 +4225,20 @@ qla2x00_mem_alloc(struct qla_hw_data *ha, uint16_t req_len, uint16_t rsp_len,
 		goto fail_sfp_data;
 	}
 
+	ha->flt = dma_alloc_coherent(&ha->pdev->dev,
+	    sizeof(struct qla_flt_header) + FLT_REGIONS_SIZE, &ha->flt_dma,
+	    GFP_KERNEL);
+	if (!ha->flt) {
+		ql_dbg_pci(ql_dbg_init, ha->pdev, 0x011b,
+		    "Unable to allocate memory for FLT.\n");
+		goto fail_flt_buffer;
+	}
+
 	return 0;
 
+fail_flt_buffer:
+	dma_free_coherent(&ha->pdev->dev, SFP_DEV_SIZE,
+	    ha->sfp_data, ha->sfp_data_dma);
 fail_sfp_data:
 	kfree(ha->loop_id_map);
 fail_loop_id_map:
@@ -4602,6 +4574,9 @@ qla2x00_free_exchoffld_buffer(struct qla_hw_data *ha)
 static void
 qla2x00_free_fw_dump(struct qla_hw_data *ha)
 {
+	struct fwdt *fwdt = ha->fwdt;
+	uint j;
+
 	if (ha->fce)
 		dma_free_coherent(&ha->pdev->dev,
 		    FCE_SIZE, ha->fce, ha->fce_dma);
@@ -4612,8 +4587,6 @@ qla2x00_free_fw_dump(struct qla_hw_data *ha)
 
 	if (ha->fw_dump)
 		vfree(ha->fw_dump);
-	if (ha->fw_dump_template)
-		vfree(ha->fw_dump_template);
 
 	ha->fce = NULL;
 	ha->fce_dma = 0;
@@ -4624,8 +4597,13 @@ qla2x00_free_fw_dump(struct qla_hw_data *ha)
 	ha->fw_dump_reading = 0;
 	ha->fw_dump = NULL;
 	ha->fw_dump_len = 0;
-	ha->fw_dump_template = NULL;
-	ha->fw_dump_template_len = 0;
+
+	for (j = 0; j < 2; j++, fwdt++) {
+		if (fwdt->template)
+			vfree(fwdt->template);
+		fwdt->template = NULL;
+		fwdt->length = 0;
+	}
 }
 
 /*
@@ -4643,44 +4621,68 @@ qla2x00_mem_free(struct qla_hw_data *ha)
 	if (ha->mctp_dump)
 		dma_free_coherent(&ha->pdev->dev, MCTP_DUMP_SIZE, ha->mctp_dump,
 		    ha->mctp_dump_dma);
+	ha->mctp_dump = NULL;
 
 	mempool_destroy(ha->srb_mempool);
+	ha->srb_mempool = NULL;
 
 	if (ha->dcbx_tlv)
 		dma_free_coherent(&ha->pdev->dev, DCBX_TLV_DATA_SIZE,
 		    ha->dcbx_tlv, ha->dcbx_tlv_dma);
+	ha->dcbx_tlv = NULL;
 
 	if (ha->xgmac_data)
 		dma_free_coherent(&ha->pdev->dev, XGMAC_DATA_SIZE,
 		    ha->xgmac_data, ha->xgmac_data_dma);
+	ha->xgmac_data = NULL;
 
 	if (ha->sns_cmd)
 		dma_free_coherent(&ha->pdev->dev, sizeof(struct sns_cmd_pkt),
 		ha->sns_cmd, ha->sns_cmd_dma);
+	ha->sns_cmd = NULL;
+	ha->sns_cmd_dma = 0;
 
 	if (ha->ct_sns)
 		dma_free_coherent(&ha->pdev->dev, sizeof(struct ct_sns_pkt),
 		ha->ct_sns, ha->ct_sns_dma);
+	ha->ct_sns = NULL;
+	ha->ct_sns_dma = 0;
 
 	if (ha->sfp_data)
 		dma_free_coherent(&ha->pdev->dev, SFP_DEV_SIZE, ha->sfp_data,
 		    ha->sfp_data_dma);
+	ha->sfp_data = NULL;
+
+	if (ha->flt)
+		dma_free_coherent(&ha->pdev->dev, SFP_DEV_SIZE,
+		    ha->flt, ha->flt_dma);
+	ha->flt = NULL;
+	ha->flt_dma = 0;
 
 	if (ha->ms_iocb)
 		dma_pool_free(ha->s_dma_pool, ha->ms_iocb, ha->ms_iocb_dma);
+	ha->ms_iocb = NULL;
+	ha->ms_iocb_dma = 0;
 
 	if (ha->ex_init_cb)
 		dma_pool_free(ha->s_dma_pool,
 			ha->ex_init_cb, ha->ex_init_cb_dma);
+	ha->ex_init_cb = NULL;
+	ha->ex_init_cb_dma = 0;
 
 	if (ha->async_pd)
 		dma_pool_free(ha->s_dma_pool, ha->async_pd, ha->async_pd_dma);
+	ha->async_pd = NULL;
+	ha->async_pd_dma = 0;
 
 	dma_pool_destroy(ha->s_dma_pool);
+	ha->s_dma_pool = NULL;
 
 	if (ha->gid_list)
 		dma_free_coherent(&ha->pdev->dev, qla2x00_gid_list_size(ha),
 		ha->gid_list, ha->gid_list_dma);
+	ha->gid_list = NULL;
+	ha->gid_list_dma = 0;
 
 	if (IS_QLA82XX(ha)) {
 		if (!list_empty(&ha->gbl_dsd_list)) {
@@ -4698,10 +4700,13 @@ qla2x00_mem_free(struct qla_hw_data *ha)
 	}
 
 	dma_pool_destroy(ha->dl_dma_pool);
+	ha->dl_dma_pool = NULL;
 
 	dma_pool_destroy(ha->fcp_cmnd_dma_pool);
+	ha->fcp_cmnd_dma_pool = NULL;
 
 	mempool_destroy(ha->ctx_mempool);
+	ha->ctx_mempool = NULL;
 
 	if (ql2xenabledif) {
 		struct dsd_dma *dsd, *nxt;
@@ -4728,53 +4733,26 @@ qla2x00_mem_free(struct qla_hw_data *ha)
 
 	if (ha->dif_bundl_pool)
 		dma_pool_destroy(ha->dif_bundl_pool);
+	ha->dif_bundl_pool = NULL;
 
 	qlt_mem_free(ha);
 
 	if (ha->init_cb)
 		dma_free_coherent(&ha->pdev->dev, ha->init_cb_size,
 			ha->init_cb, ha->init_cb_dma);
+	ha->init_cb = NULL;
+	ha->init_cb_dma = 0;
 
 	vfree(ha->optrom_buffer);
+	ha->optrom_buffer = NULL;
 	kfree(ha->nvram);
+	ha->nvram = NULL;
 	kfree(ha->npiv_info);
+	ha->npiv_info = NULL;
 	kfree(ha->swl);
+	ha->swl = NULL;
 	kfree(ha->loop_id_map);
-
-	ha->srb_mempool = NULL;
-	ha->ctx_mempool = NULL;
-	ha->sns_cmd = NULL;
-	ha->sns_cmd_dma = 0;
-	ha->ct_sns = NULL;
-	ha->ct_sns_dma = 0;
-	ha->ms_iocb = NULL;
-	ha->ms_iocb_dma = 0;
-	ha->init_cb = NULL;
-	ha->init_cb_dma = 0;
-	ha->ex_init_cb = NULL;
-	ha->ex_init_cb_dma = 0;
-	ha->async_pd = NULL;
-	ha->async_pd_dma = 0;
 	ha->loop_id_map = NULL;
-	ha->npiv_info = NULL;
-	ha->optrom_buffer = NULL;
-	ha->swl = NULL;
-	ha->nvram = NULL;
-	ha->mctp_dump = NULL;
-	ha->dcbx_tlv = NULL;
-	ha->xgmac_data = NULL;
-	ha->sfp_data = NULL;
-
-	ha->s_dma_pool = NULL;
-	ha->dl_dma_pool = NULL;
-	ha->fcp_cmnd_dma_pool = NULL;
-
-	ha->gid_list = NULL;
-	ha->gid_list_dma = 0;
-
-	ha->tgt.atio_ring = NULL;
-	ha->tgt.atio_dma = 0;
-	ha->tgt.tgt_vp_map = NULL;
 }
 
 struct scsi_qla_host *qla2x00_create_host(struct scsi_host_template *sht,
@@ -5608,6 +5586,7 @@ qla83xx_force_lock_recovery(scsi_qla_host_t *base_vha)
 	uint32_t idc_lck_rcvry_stage_mask = 0x3;
 	uint32_t idc_lck_rcvry_owner_mask = 0x3c;
 	struct qla_hw_data *ha = base_vha->hw;
+
 	ql_dbg(ql_dbg_p3p, base_vha, 0xb086,
 	    "Trying force recovery of the IDC lock.\n");
 
@@ -6677,8 +6656,10 @@ qla2x00_timer(struct timer_list *t)
 	 * FC-NVME
 	 * see if the active AEN count has changed from what was last reported.
 	 */
-	if (!vha->vp_idx && (atomic_read(&ha->nvme_active_aen_cnt) !=
-	    ha->nvme_last_rptd_aen) && ha->zio_mode == QLA_ZIO_MODE_6) {
+	if (!vha->vp_idx &&
+	    (atomic_read(&ha->nvme_active_aen_cnt) != ha->nvme_last_rptd_aen) &&
+	    ha->zio_mode == QLA_ZIO_MODE_6 &&
+	    !ha->flags.host_shutting_down) {
 		ql_log(ql_log_info, vha, 0x3002,
 		    "nvme: Sched: Set ZIO exchange threshold to %d.\n",
 		    ha->nvme_last_rptd_aen);
@@ -6690,7 +6671,7 @@ qla2x00_timer(struct timer_list *t)
 	if (!vha->vp_idx &&
 	    (atomic_read(&ha->zio_threshold) != ha->last_zio_threshold) &&
 	    (ha->zio_mode == QLA_ZIO_MODE_6) &&
-	    (IS_QLA83XX(ha) || IS_QLA27XX(ha))) {
+	    (IS_QLA83XX(ha) || IS_QLA27XX(ha) || IS_QLA28XX(ha))) {
 		ql_log(ql_log_info, vha, 0x3002,
 		    "Sched: Set ZIO exchange threshold to %d.\n",
 		    ha->last_zio_threshold);
@@ -6736,7 +6717,6 @@ qla2x00_timer(struct timer_list *t)
 
 /* Firmware interface routines. */
 
-#define FW_BLOBS	11
 #define FW_ISP21XX	0
 #define FW_ISP22XX	1
 #define FW_ISP2300	2
@@ -6748,6 +6728,7 @@ qla2x00_timer(struct timer_list *t)
 #define FW_ISP2031	8
 #define FW_ISP8031	9
 #define FW_ISP27XX	10
+#define FW_ISP28XX	11
 
 #define FW_FILE_ISP21XX	"ql2100_fw.bin"
 #define FW_FILE_ISP22XX	"ql2200_fw.bin"
@@ -6760,11 +6741,12 @@ qla2x00_timer(struct timer_list *t)
 #define FW_FILE_ISP2031	"ql2600_fw.bin"
 #define FW_FILE_ISP8031	"ql8300_fw.bin"
 #define FW_FILE_ISP27XX	"ql2700_fw.bin"
+#define FW_FILE_ISP28XX	"ql2800_fw.bin"
 
 
 static DEFINE_MUTEX(qla_fw_lock);
 
-static struct fw_blob qla_fw_blobs[FW_BLOBS] = {
+static struct fw_blob qla_fw_blobs[] = {
 	{ .name = FW_FILE_ISP21XX, .segs = { 0x1000, 0 }, },
 	{ .name = FW_FILE_ISP22XX, .segs = { 0x1000, 0 }, },
 	{ .name = FW_FILE_ISP2300, .segs = { 0x800, 0 }, },
@@ -6776,6 +6758,8 @@ static struct fw_blob qla_fw_blobs[FW_BLOBS] = {
 	{ .name = FW_FILE_ISP2031, },
 	{ .name = FW_FILE_ISP8031, },
 	{ .name = FW_FILE_ISP27XX, },
+	{ .name = FW_FILE_ISP28XX, },
+	{ .name = NULL, },
 };
 
 struct fw_blob *
@@ -6806,10 +6790,15 @@ qla2x00_request_firmware(scsi_qla_host_t *vha)
 		blob = &qla_fw_blobs[FW_ISP8031];
 	} else if (IS_QLA27XX(ha)) {
 		blob = &qla_fw_blobs[FW_ISP27XX];
+	} else if (IS_QLA28XX(ha)) {
+		blob = &qla_fw_blobs[FW_ISP28XX];
 	} else {
 		return NULL;
 	}
 
+	if (!blob->name)
+		return NULL;
+
 	mutex_lock(&qla_fw_lock);
 	if (blob->fw)
 		goto out;
@@ -6819,7 +6808,6 @@ qla2x00_request_firmware(scsi_qla_host_t *vha)
 		    "Failed to load firmware image (%s).\n", blob->name);
 		blob->fw = NULL;
 		blob = NULL;
-		goto out;
 	}
 
 out:
@@ -6830,11 +6818,11 @@ out:
 static void
 qla2x00_release_firmware(void)
 {
-	int idx;
+	struct fw_blob *blob;
 
 	mutex_lock(&qla_fw_lock);
-	for (idx = 0; idx < FW_BLOBS; idx++)
-		release_firmware(qla_fw_blobs[idx].fw);
+	for (blob = qla_fw_blobs; blob->name; blob++)
+		release_firmware(blob->fw);
 	mutex_unlock(&qla_fw_lock);
 }
 
@@ -7179,7 +7167,7 @@ static int qla2xxx_map_queues(struct Scsi_Host *shost)
 {
 	int rc;
 	scsi_qla_host_t *vha = (scsi_qla_host_t *)shost->hostdata;
-	struct blk_mq_queue_map *qmap = &shost->tag_set.map[0];
+	struct blk_mq_queue_map *qmap = &shost->tag_set.map[HCTX_TYPE_DEFAULT];
 
 	if (USER_CTRL_IRQ(vha->hw) || !vha->hw->mqiobase)
 		rc = blk_mq_map_queues(qmap);
@@ -7188,6 +7176,37 @@ static int qla2xxx_map_queues(struct Scsi_Host *shost)
 	return rc;
 }
 
+struct scsi_host_template qla2xxx_driver_template = {
+	.module			= THIS_MODULE,
+	.name			= QLA2XXX_DRIVER_NAME,
+	.queuecommand		= qla2xxx_queuecommand,
+
+	.eh_timed_out		= fc_eh_timed_out,
+	.eh_abort_handler	= qla2xxx_eh_abort,
+	.eh_device_reset_handler = qla2xxx_eh_device_reset,
+	.eh_target_reset_handler = qla2xxx_eh_target_reset,
+	.eh_bus_reset_handler	= qla2xxx_eh_bus_reset,
+	.eh_host_reset_handler	= qla2xxx_eh_host_reset,
+
+	.slave_configure	= qla2xxx_slave_configure,
+
+	.slave_alloc		= qla2xxx_slave_alloc,
+	.slave_destroy		= qla2xxx_slave_destroy,
+	.scan_finished		= qla2xxx_scan_finished,
+	.scan_start		= qla2xxx_scan_start,
+	.change_queue_depth	= scsi_change_queue_depth,
+	.map_queues             = qla2xxx_map_queues,
+	.this_id		= -1,
+	.cmd_per_lun		= 3,
+	.sg_tablesize		= SG_ALL,
+
+	.max_sectors		= 0xFFFF,
+	.shost_attrs		= qla2x00_host_attrs,
+
+	.supported_mode		= MODE_INITIATOR,
+	.track_queue_depth	= 1,
+};
+
 static const struct pci_error_handlers qla2xxx_err_handler = {
 	.error_detected = qla2xxx_pci_error_detected,
 	.mmio_enabled = qla2xxx_pci_mmio_enabled,
@@ -7220,6 +7239,11 @@ static struct pci_device_id qla2xxx_pci_tbl[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, PCI_DEVICE_ID_QLOGIC_ISP2071) },
 	{ PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, PCI_DEVICE_ID_QLOGIC_ISP2271) },
 	{ PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, PCI_DEVICE_ID_QLOGIC_ISP2261) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, PCI_DEVICE_ID_QLOGIC_ISP2061) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, PCI_DEVICE_ID_QLOGIC_ISP2081) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, PCI_DEVICE_ID_QLOGIC_ISP2281) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, PCI_DEVICE_ID_QLOGIC_ISP2089) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, PCI_DEVICE_ID_QLOGIC_ISP2289) },
 	{ 0 },
 };
 MODULE_DEVICE_TABLE(pci, qla2xxx_pci_tbl);
@@ -7249,6 +7273,30 @@ qla2x00_module_init(void)
 {
 	int ret = 0;
 
+	BUILD_BUG_ON(sizeof(cmd_entry_t) != 64);
+	BUILD_BUG_ON(sizeof(cont_a64_entry_t) != 64);
+	BUILD_BUG_ON(sizeof(cont_entry_t) != 64);
+	BUILD_BUG_ON(sizeof(init_cb_t) != 96);
+	BUILD_BUG_ON(sizeof(ms_iocb_entry_t) != 64);
+	BUILD_BUG_ON(sizeof(request_t) != 64);
+	BUILD_BUG_ON(sizeof(struct access_chip_84xx) != 64);
+	BUILD_BUG_ON(sizeof(struct cmd_bidir) != 64);
+	BUILD_BUG_ON(sizeof(struct cmd_nvme) != 64);
+	BUILD_BUG_ON(sizeof(struct cmd_type_6) != 64);
+	BUILD_BUG_ON(sizeof(struct cmd_type_7) != 64);
+	BUILD_BUG_ON(sizeof(struct cmd_type_7_fx00) != 64);
+	BUILD_BUG_ON(sizeof(struct cmd_type_crc_2) != 64);
+	BUILD_BUG_ON(sizeof(struct ct_entry_24xx) != 64);
+	BUILD_BUG_ON(sizeof(struct ctio_crc2_to_fw) != 64);
+	BUILD_BUG_ON(sizeof(struct els_entry_24xx) != 64);
+	BUILD_BUG_ON(sizeof(struct fxdisc_entry_fx00) != 64);
+	BUILD_BUG_ON(sizeof(struct init_cb_24xx) != 128);
+	BUILD_BUG_ON(sizeof(struct init_cb_81xx) != 128);
+	BUILD_BUG_ON(sizeof(struct pt_ls4_request) != 64);
+	BUILD_BUG_ON(sizeof(struct sns_cmd_pkt) != 2064);
+	BUILD_BUG_ON(sizeof(struct verify_chip_entry_84xx) != 64);
+	BUILD_BUG_ON(sizeof(struct vf_evfp_entry_24xx) != 56);
+
 	/* Allocate cache for SRBs. */
 	srb_cachep = kmem_cache_create("qla2xxx_srbs", sizeof(srb_t), 0,
 	    SLAB_HWCACHE_ALIGN, NULL);
@@ -7261,8 +7309,7 @@ qla2x00_module_init(void)
 	/* Initialize target kmem_cache and mem_pools */
 	ret = qlt_init();
 	if (ret < 0) {
-		kmem_cache_destroy(srb_cachep);
-		return ret;
+		goto destroy_cache;
 	} else if (ret > 0) {
 		/*
 		 * If initiator mode is explictly disabled by qlt_init(),
@@ -7286,11 +7333,10 @@ qla2x00_module_init(void)
 	qla2xxx_transport_template =
 	    fc_attach_transport(&qla2xxx_transport_functions);
 	if (!qla2xxx_transport_template) {
-		kmem_cache_destroy(srb_cachep);
 		ql_log(ql_log_fatal, NULL, 0x0002,
 		    "fc_attach_transport failed...Failing load!.\n");
-		qlt_exit();
-		return -ENODEV;
+		ret = -ENODEV;
+		goto qlt_exit;
 	}
 
 	apidev_major = register_chrdev(0, QLA2XXX_APIDEV, &apidev_fops);
@@ -7302,27 +7348,37 @@ qla2x00_module_init(void)
 	qla2xxx_transport_vport_template =
 	    fc_attach_transport(&qla2xxx_transport_vport_functions);
 	if (!qla2xxx_transport_vport_template) {
-		kmem_cache_destroy(srb_cachep);
-		qlt_exit();
-		fc_release_transport(qla2xxx_transport_template);
 		ql_log(ql_log_fatal, NULL, 0x0004,
 		    "fc_attach_transport vport failed...Failing load!.\n");
-		return -ENODEV;
+		ret = -ENODEV;
+		goto unreg_chrdev;
 	}
 	ql_log(ql_log_info, NULL, 0x0005,
 	    "QLogic Fibre Channel HBA Driver: %s.\n",
 	    qla2x00_version_str);
 	ret = pci_register_driver(&qla2xxx_pci_driver);
 	if (ret) {
-		kmem_cache_destroy(srb_cachep);
-		qlt_exit();
-		fc_release_transport(qla2xxx_transport_template);
-		fc_release_transport(qla2xxx_transport_vport_template);
 		ql_log(ql_log_fatal, NULL, 0x0006,
 		    "pci_register_driver failed...ret=%d Failing load!.\n",
 		    ret);
+		goto release_vport_transport;
 	}
 	return ret;
+
+release_vport_transport:
+	fc_release_transport(qla2xxx_transport_vport_template);
+
+unreg_chrdev:
+	if (apidev_major >= 0)
+		unregister_chrdev(apidev_major, QLA2XXX_APIDEV);
+	fc_release_transport(qla2xxx_transport_template);
+
+qlt_exit:
+	qlt_exit();
+
+destroy_cache:
+	kmem_cache_destroy(srb_cachep);
+	return ret;
 }
 
 /**
@@ -7331,14 +7387,15 @@ qla2x00_module_init(void)
 static void __exit
 qla2x00_module_exit(void)
 {
-	unregister_chrdev(apidev_major, QLA2XXX_APIDEV);
 	pci_unregister_driver(&qla2xxx_pci_driver);
 	qla2x00_release_firmware();
-	kmem_cache_destroy(srb_cachep);
-	qlt_exit();
 	kmem_cache_destroy(ctx_cachep);
-	fc_release_transport(qla2xxx_transport_template);
 	fc_release_transport(qla2xxx_transport_vport_template);
+	if (apidev_major >= 0)
+		unregister_chrdev(apidev_major, QLA2XXX_APIDEV);
+	fc_release_transport(qla2xxx_transport_template);
+	qlt_exit();
+	kmem_cache_destroy(srb_cachep);
 }
 
 module_init(qla2x00_module_init);
diff --git a/drivers/scsi/qla2xxx/qla_sup.c b/drivers/scsi/qla2xxx/qla_sup.c
index 2a3055c799fb..1eb82384d933 100644
--- a/drivers/scsi/qla2xxx/qla_sup.c
+++ b/drivers/scsi/qla2xxx/qla_sup.c
@@ -429,66 +429,64 @@ qla2x00_set_nvram_protection(struct qla_hw_data *ha, int stat)
 static inline uint32_t
 flash_conf_addr(struct qla_hw_data *ha, uint32_t faddr)
 {
-	return ha->flash_conf_off | faddr;
+	return ha->flash_conf_off + faddr;
 }
 
 static inline uint32_t
 flash_data_addr(struct qla_hw_data *ha, uint32_t faddr)
 {
-	return ha->flash_data_off | faddr;
+	return ha->flash_data_off + faddr;
 }
 
 static inline uint32_t
 nvram_conf_addr(struct qla_hw_data *ha, uint32_t naddr)
 {
-	return ha->nvram_conf_off | naddr;
+	return ha->nvram_conf_off + naddr;
 }
 
 static inline uint32_t
 nvram_data_addr(struct qla_hw_data *ha, uint32_t naddr)
 {
-	return ha->nvram_data_off | naddr;
+	return ha->nvram_data_off + naddr;
 }
 
-static uint32_t
-qla24xx_read_flash_dword(struct qla_hw_data *ha, uint32_t addr)
+static int
+qla24xx_read_flash_dword(struct qla_hw_data *ha, uint32_t addr, uint32_t *data)
 {
-	int rval;
-	uint32_t cnt, data;
 	struct device_reg_24xx __iomem *reg = &ha->iobase->isp24;
+	ulong cnt = 30000;
 
 	WRT_REG_DWORD(&reg->flash_addr, addr & ~FARX_DATA_FLAG);
-	/* Wait for READ cycle to complete. */
-	rval = QLA_SUCCESS;
-	for (cnt = 3000;
-	    (RD_REG_DWORD(&reg->flash_addr) & FARX_DATA_FLAG) == 0 &&
-	    rval == QLA_SUCCESS; cnt--) {
-		if (cnt)
-			udelay(10);
-		else
-			rval = QLA_FUNCTION_TIMEOUT;
+
+	while (cnt--) {
+		if (RD_REG_DWORD(&reg->flash_addr) & FARX_DATA_FLAG) {
+			*data = RD_REG_DWORD(&reg->flash_data);
+			return QLA_SUCCESS;
+		}
+		udelay(10);
 		cond_resched();
 	}
 
-	/* TODO: What happens if we time out? */
-	data = 0xDEADDEAD;
-	if (rval == QLA_SUCCESS)
-		data = RD_REG_DWORD(&reg->flash_data);
-
-	return data;
+	ql_log(ql_log_warn, pci_get_drvdata(ha->pdev), 0x7090,
+	    "Flash read dword at %x timeout.\n", addr);
+	*data = 0xDEADDEAD;
+	return QLA_FUNCTION_TIMEOUT;
 }
 
 uint32_t *
 qla24xx_read_flash_data(scsi_qla_host_t *vha, uint32_t *dwptr, uint32_t faddr,
     uint32_t dwords)
 {
-	uint32_t i;
+	ulong i;
 	struct qla_hw_data *ha = vha->hw;
 
 	/* Dword reads to flash. */
-	for (i = 0; i < dwords; i++, faddr++)
-		dwptr[i] = cpu_to_le32(qla24xx_read_flash_dword(ha,
-		    flash_data_addr(ha, faddr)));
+	faddr =  flash_data_addr(ha, faddr);
+	for (i = 0; i < dwords; i++, faddr++, dwptr++) {
+		if (qla24xx_read_flash_dword(ha, faddr, dwptr))
+			break;
+		cpu_to_le32s(dwptr);
+	}
 
 	return dwptr;
 }
@@ -496,35 +494,37 @@ qla24xx_read_flash_data(scsi_qla_host_t *vha, uint32_t *dwptr, uint32_t faddr,
 static int
 qla24xx_write_flash_dword(struct qla_hw_data *ha, uint32_t addr, uint32_t data)
 {
-	int rval;
-	uint32_t cnt;
 	struct device_reg_24xx __iomem *reg = &ha->iobase->isp24;
+	ulong cnt = 500000;
 
 	WRT_REG_DWORD(&reg->flash_data, data);
-	RD_REG_DWORD(&reg->flash_data);		/* PCI Posting. */
 	WRT_REG_DWORD(&reg->flash_addr, addr | FARX_DATA_FLAG);
-	/* Wait for Write cycle to complete. */
-	rval = QLA_SUCCESS;
-	for (cnt = 500000; (RD_REG_DWORD(&reg->flash_addr) & FARX_DATA_FLAG) &&
-	    rval == QLA_SUCCESS; cnt--) {
-		if (cnt)
-			udelay(10);
-		else
-			rval = QLA_FUNCTION_TIMEOUT;
+
+	while (cnt--) {
+		if (!(RD_REG_DWORD(&reg->flash_addr) & FARX_DATA_FLAG))
+			return QLA_SUCCESS;
+		udelay(10);
 		cond_resched();
 	}
-	return rval;
+
+	ql_log(ql_log_warn, pci_get_drvdata(ha->pdev), 0x7090,
+	    "Flash write dword at %x timeout.\n", addr);
+	return QLA_FUNCTION_TIMEOUT;
 }
 
 static void
 qla24xx_get_flash_manufacturer(struct qla_hw_data *ha, uint8_t *man_id,
     uint8_t *flash_id)
 {
-	uint32_t ids;
+	uint32_t faddr, ids = 0;
 
-	ids = qla24xx_read_flash_dword(ha, flash_conf_addr(ha, 0x03ab));
-	*man_id = LSB(ids);
-	*flash_id = MSB(ids);
+	*man_id = *flash_id = 0;
+
+	faddr = flash_conf_addr(ha, 0x03ab);
+	if (!qla24xx_read_flash_dword(ha, faddr, &ids)) {
+		*man_id = LSB(ids);
+		*flash_id = MSB(ids);
+	}
 
 	/* Check if man_id and flash_id are valid. */
 	if (ids != 0xDEADDEAD && (*man_id == 0 || *flash_id == 0)) {
@@ -534,9 +534,11 @@ qla24xx_get_flash_manufacturer(struct qla_hw_data *ha, uint8_t *man_id,
 		 * Example: ATMEL 0x00 01 45 1F
 		 * Extract MFG and Dev ID from last two bytes.
 		 */
-		ids = qla24xx_read_flash_dword(ha, flash_conf_addr(ha, 0x009f));
-		*man_id = LSB(ids);
-		*flash_id = MSB(ids);
+		faddr = flash_conf_addr(ha, 0x009f);
+		if (!qla24xx_read_flash_dword(ha, faddr, &ids)) {
+			*man_id = LSB(ids);
+			*flash_id = MSB(ids);
+		}
 	}
 }
 
@@ -545,12 +547,12 @@ qla2xxx_find_flt_start(scsi_qla_host_t *vha, uint32_t *start)
 {
 	const char *loc, *locations[] = { "DEF", "PCI" };
 	uint32_t pcihdr, pcids;
-	uint32_t *dcode;
-	uint8_t *buf, *bcode, last_image;
 	uint16_t cnt, chksum, *wptr;
-	struct qla_flt_location *fltl;
 	struct qla_hw_data *ha = vha->hw;
 	struct req_que *req = ha->req_q_map[0];
+	struct qla_flt_location *fltl = (void *)req->ring;
+	uint32_t *dcode = (void *)req->ring;
+	uint8_t *buf = (void *)req->ring, *bcode,  last_image;
 
 	/*
 	 * FLT-location structure resides after the last PCI region.
@@ -571,12 +573,13 @@ qla2xxx_find_flt_start(scsi_qla_host_t *vha, uint32_t *start)
 	} else if (IS_QLA83XX(ha) || IS_QLA27XX(ha)) {
 		*start = FA_FLASH_LAYOUT_ADDR_83;
 		goto end;
+	} else if (IS_QLA28XX(ha)) {
+		*start = FA_FLASH_LAYOUT_ADDR_28;
+		goto end;
 	}
+
 	/* Begin with first PCI expansion ROM header. */
-	buf = (uint8_t *)req->ring;
-	dcode = (uint32_t *)req->ring;
 	pcihdr = 0;
-	last_image = 1;
 	do {
 		/* Verify PCI expansion ROM header. */
 		qla24xx_read_flash_data(vha, dcode, pcihdr >> 2, 0x20);
@@ -601,22 +604,19 @@ qla2xxx_find_flt_start(scsi_qla_host_t *vha, uint32_t *start)
 	} while (!last_image);
 
 	/* Now verify FLT-location structure. */
-	fltl = (struct qla_flt_location *)req->ring;
-	qla24xx_read_flash_data(vha, dcode, pcihdr >> 2,
-	    sizeof(struct qla_flt_location) >> 2);
-	if (fltl->sig[0] != 'Q' || fltl->sig[1] != 'F' ||
-	    fltl->sig[2] != 'L' || fltl->sig[3] != 'T')
+	qla24xx_read_flash_data(vha, dcode, pcihdr >> 2, sizeof(*fltl) >> 2);
+	if (memcmp(fltl->sig, "QFLT", 4))
 		goto end;
 
-	wptr = (uint16_t *)req->ring;
-	cnt = sizeof(struct qla_flt_location) >> 1;
+	wptr = (void *)req->ring;
+	cnt = sizeof(*fltl) / sizeof(*wptr);
 	for (chksum = 0; cnt--; wptr++)
 		chksum += le16_to_cpu(*wptr);
 	if (chksum) {
 		ql_log(ql_log_fatal, vha, 0x0045,
 		    "Inconsistent FLTL detected: checksum=0x%x.\n", chksum);
 		ql_dump_buffer(ql_dbg_init + ql_dbg_buffer, vha, 0x010e,
-		    buf, sizeof(struct qla_flt_location));
+		    fltl, sizeof(*fltl));
 		return QLA_FUNCTION_FAILED;
 	}
 
@@ -634,7 +634,7 @@ end:
 static void
 qla2xxx_get_flt_info(scsi_qla_host_t *vha, uint32_t flt_addr)
 {
-	const char *loc, *locations[] = { "DEF", "FLT" };
+	const char *locations[] = { "DEF", "FLT" }, *loc = locations[1];
 	const uint32_t def_fw[] =
 		{ FA_RISC_CODE_ADDR, FA_RISC_CODE_ADDR, FA_RISC_CODE_ADDR_81 };
 	const uint32_t def_boot[] =
@@ -664,20 +664,13 @@ qla2xxx_get_flt_info(scsi_qla_host_t *vha, uint32_t flt_addr)
 	const uint32_t fcp_prio_cfg1[] =
 		{ FA_FCP_PRIO1_ADDR, FA_FCP_PRIO1_ADDR_25,
 			0 };
-	uint32_t def;
-	uint16_t *wptr;
-	uint16_t cnt, chksum;
-	uint32_t start;
-	struct qla_flt_header *flt;
-	struct qla_flt_region *region;
-	struct qla_hw_data *ha = vha->hw;
-	struct req_que *req = ha->req_q_map[0];
 
-	def = 0;
-	if (IS_QLA25XX(ha))
-		def = 1;
-	else if (IS_QLA81XX(ha))
-		def = 2;
+	struct qla_hw_data *ha = vha->hw;
+	uint32_t def = IS_QLA81XX(ha) ? 2 : IS_QLA25XX(ha) ? 1 : 0;
+	struct qla_flt_header *flt = (void *)ha->flt;
+	struct qla_flt_region *region = (void *)&flt[1];
+	uint16_t *wptr, cnt, chksum;
+	uint32_t start;
 
 	/* Assign FCP prio region since older adapters may not have FLT, or
 	   FCP prio region in it's FLT.
@@ -686,12 +679,11 @@ qla2xxx_get_flt_info(scsi_qla_host_t *vha, uint32_t flt_addr)
 	    fcp_prio_cfg0[def] : fcp_prio_cfg1[def];
 
 	ha->flt_region_flt = flt_addr;
-	wptr = (uint16_t *)req->ring;
-	flt = (struct qla_flt_header *)req->ring;
-	region = (struct qla_flt_region *)&flt[1];
-	ha->isp_ops->read_optrom(vha, (uint8_t *)req->ring,
-	    flt_addr << 2, OPTROM_BURST_SIZE);
-	if (*wptr == cpu_to_le16(0xffff))
+	wptr = (uint16_t *)ha->flt;
+	qla24xx_read_flash_data(vha, (void *)flt, flt_addr,
+	    (sizeof(struct qla_flt_header) + FLT_REGIONS_SIZE) >> 2);
+
+	if (le16_to_cpu(*wptr) == 0xffff)
 		goto no_flash_data;
 	if (flt->version != cpu_to_le16(1)) {
 		ql_log(ql_log_warn, vha, 0x0047,
@@ -701,7 +693,7 @@ qla2xxx_get_flt_info(scsi_qla_host_t *vha, uint32_t flt_addr)
 		goto no_flash_data;
 	}
 
-	cnt = (sizeof(struct qla_flt_header) + le16_to_cpu(flt->length)) >> 1;
+	cnt = (sizeof(*flt) + le16_to_cpu(flt->length)) / sizeof(*wptr);
 	for (chksum = 0; cnt--; wptr++)
 		chksum += le16_to_cpu(*wptr);
 	if (chksum) {
@@ -712,18 +704,20 @@ qla2xxx_get_flt_info(scsi_qla_host_t *vha, uint32_t flt_addr)
 		goto no_flash_data;
 	}
 
-	loc = locations[1];
-	cnt = le16_to_cpu(flt->length) / sizeof(struct qla_flt_region);
+	cnt = le16_to_cpu(flt->length) / sizeof(*region);
 	for ( ; cnt; cnt--, region++) {
 		/* Store addresses as DWORD offsets. */
 		start = le32_to_cpu(region->start) >> 2;
 		ql_dbg(ql_dbg_init, vha, 0x0049,
-		    "FLT[%02x]: start=0x%x "
-		    "end=0x%x size=0x%x.\n", le32_to_cpu(region->code) & 0xff,
-		    start, le32_to_cpu(region->end) >> 2,
-		    le32_to_cpu(region->size));
-
-		switch (le32_to_cpu(region->code) & 0xff) {
+		    "FLT[%#x]: start=%#x end=%#x size=%#x.\n",
+		    le16_to_cpu(region->code), start,
+		    le32_to_cpu(region->end) >> 2,
+		    le32_to_cpu(region->size) >> 2);
+		if (region->attribute)
+			ql_log(ql_dbg_init, vha, 0xffff,
+			    "Region %x is secure\n", region->code);
+
+		switch (le16_to_cpu(region->code)) {
 		case FLT_REG_FCOE_FW:
 			if (!IS_QLA8031(ha))
 				break;
@@ -753,13 +747,13 @@ qla2xxx_get_flt_info(scsi_qla_host_t *vha, uint32_t flt_addr)
 				ha->flt_region_vpd = start;
 			break;
 		case FLT_REG_VPD_2:
-			if (!IS_QLA27XX(ha))
+			if (!IS_QLA27XX(ha) && !IS_QLA28XX(ha))
 				break;
 			if (ha->port_no == 2)
 				ha->flt_region_vpd = start;
 			break;
 		case FLT_REG_VPD_3:
-			if (!IS_QLA27XX(ha))
+			if (!IS_QLA27XX(ha) && !IS_QLA28XX(ha))
 				break;
 			if (ha->port_no == 3)
 				ha->flt_region_vpd = start;
@@ -777,13 +771,13 @@ qla2xxx_get_flt_info(scsi_qla_host_t *vha, uint32_t flt_addr)
 				ha->flt_region_nvram = start;
 			break;
 		case FLT_REG_NVRAM_2:
-			if (!IS_QLA27XX(ha))
+			if (!IS_QLA27XX(ha) && !IS_QLA28XX(ha))
 				break;
 			if (ha->port_no == 2)
 				ha->flt_region_nvram = start;
 			break;
 		case FLT_REG_NVRAM_3:
-			if (!IS_QLA27XX(ha))
+			if (!IS_QLA27XX(ha) && !IS_QLA28XX(ha))
 				break;
 			if (ha->port_no == 3)
 				ha->flt_region_nvram = start;
@@ -847,36 +841,74 @@ qla2xxx_get_flt_info(scsi_qla_host_t *vha, uint32_t flt_addr)
 				ha->flt_region_nvram = start;
 			break;
 		case FLT_REG_IMG_PRI_27XX:
-			if (IS_QLA27XX(ha))
+			if (IS_QLA27XX(ha) && !IS_QLA28XX(ha))
 				ha->flt_region_img_status_pri = start;
 			break;
 		case FLT_REG_IMG_SEC_27XX:
-			if (IS_QLA27XX(ha))
+			if (IS_QLA27XX(ha) && !IS_QLA28XX(ha))
 				ha->flt_region_img_status_sec = start;
 			break;
 		case FLT_REG_FW_SEC_27XX:
-			if (IS_QLA27XX(ha))
+			if (IS_QLA27XX(ha) && !IS_QLA28XX(ha))
 				ha->flt_region_fw_sec = start;
 			break;
 		case FLT_REG_BOOTLOAD_SEC_27XX:
-			if (IS_QLA27XX(ha))
+			if (IS_QLA27XX(ha) && !IS_QLA28XX(ha))
 				ha->flt_region_boot_sec = start;
 			break;
+		case FLT_REG_AUX_IMG_PRI_28XX:
+			if (IS_QLA27XX(ha) || IS_QLA28XX(ha))
+				ha->flt_region_aux_img_status_pri = start;
+			break;
+		case FLT_REG_AUX_IMG_SEC_28XX:
+			if (IS_QLA27XX(ha) || IS_QLA28XX(ha))
+				ha->flt_region_aux_img_status_sec = start;
+			break;
+		case FLT_REG_NVRAM_SEC_28XX_0:
+			if (IS_QLA27XX(ha) || IS_QLA28XX(ha))
+				if (ha->port_no == 0)
+					ha->flt_region_nvram_sec = start;
+			break;
+		case FLT_REG_NVRAM_SEC_28XX_1:
+			if (IS_QLA27XX(ha) || IS_QLA28XX(ha))
+				if (ha->port_no == 1)
+					ha->flt_region_nvram_sec = start;
+			break;
+		case FLT_REG_NVRAM_SEC_28XX_2:
+			if (IS_QLA27XX(ha) || IS_QLA28XX(ha))
+				if (ha->port_no == 2)
+					ha->flt_region_nvram_sec = start;
+			break;
+		case FLT_REG_NVRAM_SEC_28XX_3:
+			if (IS_QLA27XX(ha) || IS_QLA28XX(ha))
+				if (ha->port_no == 3)
+					ha->flt_region_nvram_sec = start;
+			break;
 		case FLT_REG_VPD_SEC_27XX_0:
-			if (IS_QLA27XX(ha))
-				ha->flt_region_vpd_sec = start;
+		case FLT_REG_VPD_SEC_28XX_0:
+			if (IS_QLA27XX(ha) || IS_QLA28XX(ha)) {
+				ha->flt_region_vpd_nvram_sec = start;
+				if (ha->port_no == 0)
+					ha->flt_region_vpd_sec = start;
+			}
 			break;
 		case FLT_REG_VPD_SEC_27XX_1:
-			if (IS_QLA27XX(ha))
-				ha->flt_region_vpd_sec = start;
+		case FLT_REG_VPD_SEC_28XX_1:
+			if (IS_QLA27XX(ha) || IS_QLA28XX(ha))
+				if (ha->port_no == 1)
+					ha->flt_region_vpd_sec = start;
 			break;
 		case FLT_REG_VPD_SEC_27XX_2:
-			if (IS_QLA27XX(ha))
-				ha->flt_region_vpd_sec = start;
+		case FLT_REG_VPD_SEC_28XX_2:
+			if (IS_QLA27XX(ha) || IS_QLA28XX(ha))
+				if (ha->port_no == 2)
+					ha->flt_region_vpd_sec = start;
 			break;
 		case FLT_REG_VPD_SEC_27XX_3:
-			if (IS_QLA27XX(ha))
-				ha->flt_region_vpd_sec = start;
+		case FLT_REG_VPD_SEC_28XX_3:
+			if (IS_QLA27XX(ha) || IS_QLA28XX(ha))
+				if (ha->port_no == 3)
+					ha->flt_region_vpd_sec = start;
 			break;
 		}
 	}
@@ -912,22 +944,19 @@ qla2xxx_get_fdt_info(scsi_qla_host_t *vha)
 #define FLASH_BLK_SIZE_32K	0x8000
 #define FLASH_BLK_SIZE_64K	0x10000
 	const char *loc, *locations[] = { "MID", "FDT" };
+	struct qla_hw_data *ha = vha->hw;
+	struct req_que *req = ha->req_q_map[0];
 	uint16_t cnt, chksum;
-	uint16_t *wptr;
-	struct qla_fdt_layout *fdt;
+	uint16_t *wptr = (void *)req->ring;
+	struct qla_fdt_layout *fdt = (void *)req->ring;
 	uint8_t	man_id, flash_id;
 	uint16_t mid = 0, fid = 0;
-	struct qla_hw_data *ha = vha->hw;
-	struct req_que *req = ha->req_q_map[0];
 
-	wptr = (uint16_t *)req->ring;
-	fdt = (struct qla_fdt_layout *)req->ring;
-	ha->isp_ops->read_optrom(vha, (uint8_t *)req->ring,
-	    ha->flt_region_fdt << 2, OPTROM_BURST_SIZE);
-	if (*wptr == cpu_to_le16(0xffff))
+	qla24xx_read_flash_data(vha, (void *)fdt, ha->flt_region_fdt,
+	    OPTROM_BURST_DWORDS);
+	if (le16_to_cpu(*wptr) == 0xffff)
 		goto no_flash_data;
-	if (fdt->sig[0] != 'Q' || fdt->sig[1] != 'L' || fdt->sig[2] != 'I' ||
-	    fdt->sig[3] != 'D')
+	if (memcmp(fdt->sig, "QLID", 4))
 		goto no_flash_data;
 
 	for (cnt = 0, chksum = 0; cnt < sizeof(*fdt) >> 1; cnt++, wptr++)
@@ -938,7 +967,7 @@ qla2xxx_get_fdt_info(scsi_qla_host_t *vha)
 		    " checksum=0x%x id=%c version0x%x.\n", chksum,
 		    fdt->sig[0], le16_to_cpu(fdt->version));
 		ql_dump_buffer(ql_dbg_init + ql_dbg_buffer, vha, 0x0113,
-		    (uint8_t *)fdt, sizeof(*fdt));
+		    fdt, sizeof(*fdt));
 		goto no_flash_data;
 	}
 
@@ -958,7 +987,7 @@ qla2xxx_get_fdt_info(scsi_qla_host_t *vha)
 		ha->fdt_unprotect_sec_cmd = flash_conf_addr(ha, 0x0300 |
 		    fdt->unprotect_sec_cmd);
 		ha->fdt_protect_sec_cmd = fdt->protect_sec_cmd ?
-		    flash_conf_addr(ha, 0x0300 | fdt->protect_sec_cmd):
+		    flash_conf_addr(ha, 0x0300 | fdt->protect_sec_cmd) :
 		    flash_conf_addr(ha, 0x0336);
 	}
 	goto done;
@@ -1019,8 +1048,7 @@ qla2xxx_get_idc_param(scsi_qla_host_t *vha)
 		return;
 
 	wptr = (uint32_t *)req->ring;
-	ha->isp_ops->read_optrom(vha, (uint8_t *)req->ring,
-		QLA82XX_IDC_PARAM_ADDR , 8);
+	ha->isp_ops->read_optrom(vha, req->ring, QLA82XX_IDC_PARAM_ADDR, 8);
 
 	if (*wptr == cpu_to_le32(0xffffffff)) {
 		ha->fcoe_dev_init_timeout = QLA82XX_ROM_DEV_INIT_TIMEOUT;
@@ -1045,7 +1073,8 @@ qla2xxx_get_flash_info(scsi_qla_host_t *vha)
 	struct qla_hw_data *ha = vha->hw;
 
 	if (!IS_QLA24XX_TYPE(ha) && !IS_QLA25XX(ha) &&
-	    !IS_CNA_CAPABLE(ha) && !IS_QLA2031(ha) && !IS_QLA27XX(ha))
+	    !IS_CNA_CAPABLE(ha) && !IS_QLA2031(ha) &&
+	    !IS_QLA27XX(ha) && !IS_QLA28XX(ha))
 		return QLA_SUCCESS;
 
 	ret = qla2xxx_find_flt_start(vha, &flt_addr);
@@ -1081,8 +1110,8 @@ qla2xxx_flash_npiv_conf(scsi_qla_host_t *vha)
 	if (IS_QLA8044(ha))
 		return;
 
-	ha->isp_ops->read_optrom(vha, (uint8_t *)&hdr,
-	    ha->flt_region_npiv_conf << 2, sizeof(struct qla_npiv_header));
+	ha->isp_ops->read_optrom(vha, &hdr, ha->flt_region_npiv_conf << 2,
+	    sizeof(struct qla_npiv_header));
 	if (hdr.version == cpu_to_le16(0xffff))
 		return;
 	if (hdr.version != cpu_to_le16(1)) {
@@ -1101,8 +1130,8 @@ qla2xxx_flash_npiv_conf(scsi_qla_host_t *vha)
 		return;
 	}
 
-	ha->isp_ops->read_optrom(vha, (uint8_t *)data,
-	    ha->flt_region_npiv_conf << 2, NPIV_CONFIG_SIZE);
+	ha->isp_ops->read_optrom(vha, data, ha->flt_region_npiv_conf << 2,
+	    NPIV_CONFIG_SIZE);
 
 	cnt = (sizeof(hdr) + le16_to_cpu(hdr.entries) * sizeof(*entry)) >> 1;
 	for (wptr = data, chksum = 0; cnt--; wptr++)
@@ -1139,10 +1168,8 @@ qla2xxx_flash_npiv_conf(scsi_qla_host_t *vha)
 		vid.node_name = wwn_to_u64(entry->node_name);
 
 		ql_dbg(ql_dbg_user, vha, 0x7093,
-		    "NPIV[%02x]: wwpn=%llx "
-		    "wwnn=%llx vf_id=0x%x Q_qos=0x%x F_qos=0x%x.\n", cnt,
-		    (unsigned long long)vid.port_name,
-		    (unsigned long long)vid.node_name,
+		    "NPIV[%02x]: wwpn=%llx wwnn=%llx vf_id=%#x Q_qos=%#x F_qos=%#x.\n",
+		    cnt, vid.port_name, vid.node_name,
 		    le16_to_cpu(entry->vf_id),
 		    entry->q_qos, entry->f_qos);
 
@@ -1150,10 +1177,8 @@ qla2xxx_flash_npiv_conf(scsi_qla_host_t *vha)
 			vport = fc_vport_create(vha->host, 0, &vid);
 			if (!vport)
 				ql_log(ql_log_warn, vha, 0x7094,
-				    "NPIV-Config Failed to create vport [%02x]: "
-				    "wwpn=%llx wwnn=%llx.\n", cnt,
-				    (unsigned long long)vid.port_name,
-				    (unsigned long long)vid.node_name);
+				    "NPIV-Config Failed to create vport [%02x]: wwpn=%llx wwnn=%llx.\n",
+				    cnt, vid.port_name, vid.node_name);
 		}
 	}
 done:
@@ -1188,9 +1213,10 @@ done:
 static int
 qla24xx_protect_flash(scsi_qla_host_t *vha)
 {
-	uint32_t cnt;
 	struct qla_hw_data *ha = vha->hw;
 	struct device_reg_24xx __iomem *reg = &ha->iobase->isp24;
+	ulong cnt = 300;
+	uint32_t faddr, dword;
 
 	if (ha->flags.fac_supported)
 		return qla81xx_fac_do_write_enable(vha, 0);
@@ -1199,11 +1225,14 @@ qla24xx_protect_flash(scsi_qla_host_t *vha)
 		goto skip_wrt_protect;
 
 	/* Enable flash write-protection and wait for completion. */
-	qla24xx_write_flash_dword(ha, flash_conf_addr(ha, 0x101),
-	    ha->fdt_wrt_disable);
-	for (cnt = 300; cnt &&
-	    qla24xx_read_flash_dword(ha, flash_conf_addr(ha, 0x005)) & BIT_0;
-	    cnt--) {
+	faddr = flash_conf_addr(ha, 0x101);
+	qla24xx_write_flash_dword(ha, faddr, ha->fdt_wrt_disable);
+	faddr = flash_conf_addr(ha, 0x5);
+	while (cnt--) {
+		if (!qla24xx_read_flash_dword(ha, faddr, &dword)) {
+			if (!(dword & BIT_0))
+				break;
+		}
 		udelay(10);
 	}
 
@@ -1211,7 +1240,6 @@ skip_wrt_protect:
 	/* Disable flash write. */
 	WRT_REG_DWORD(&reg->ctrl_status,
 	    RD_REG_DWORD(&reg->ctrl_status) & ~CSRX_FLASH_ENABLE);
-	RD_REG_DWORD(&reg->ctrl_status);	/* PCI Posting. */
 
 	return QLA_SUCCESS;
 }
@@ -1239,107 +1267,103 @@ qla24xx_write_flash_data(scsi_qla_host_t *vha, uint32_t *dwptr, uint32_t faddr,
     uint32_t dwords)
 {
 	int ret;
-	uint32_t liter;
-	uint32_t sec_mask, rest_addr;
-	uint32_t fdata;
+	ulong liter;
+	ulong dburst = OPTROM_BURST_DWORDS; /* burst size in dwords */
+	uint32_t sec_mask, rest_addr, fdata;
 	dma_addr_t optrom_dma;
 	void *optrom = NULL;
 	struct qla_hw_data *ha = vha->hw;
 
-	/* Prepare burst-capable write on supported ISPs. */
-	if ((IS_QLA25XX(ha) || IS_QLA81XX(ha) || IS_QLA83XX(ha) ||
-	    IS_QLA27XX(ha)) &&
-	    !(faddr & 0xfff) && dwords > OPTROM_BURST_DWORDS) {
-		optrom = dma_alloc_coherent(&ha->pdev->dev, OPTROM_BURST_SIZE,
-		    &optrom_dma, GFP_KERNEL);
-		if (!optrom) {
-			ql_log(ql_log_warn, vha, 0x7095,
-			    "Unable to allocate "
-			    "memory for optrom burst write (%x KB).\n",
-			    OPTROM_BURST_SIZE / 1024);
-		}
-	}
+	if (!IS_QLA25XX(ha) && !IS_QLA81XX(ha) && !IS_QLA83XX(ha) &&
+	    !IS_QLA27XX(ha) && !IS_QLA28XX(ha))
+		goto next;
 
-	rest_addr = (ha->fdt_block_size >> 2) - 1;
-	sec_mask = ~rest_addr;
+	/* Allocate dma buffer for burst write */
+	optrom = dma_alloc_coherent(&ha->pdev->dev, OPTROM_BURST_SIZE,
+	    &optrom_dma, GFP_KERNEL);
+	if (!optrom) {
+		ql_log(ql_log_warn, vha, 0x7095,
+		    "Failed allocate burst (%x bytes)\n", OPTROM_BURST_SIZE);
+	}
 
+next:
+	ql_log(ql_log_warn + ql_dbg_verbose, vha, 0x7095,
+	    "Unprotect flash...\n");
 	ret = qla24xx_unprotect_flash(vha);
-	if (ret != QLA_SUCCESS) {
+	if (ret) {
 		ql_log(ql_log_warn, vha, 0x7096,
-		    "Unable to unprotect flash for update.\n");
+		    "Failed to unprotect flash.\n");
 		goto done;
 	}
 
+	rest_addr = (ha->fdt_block_size >> 2) - 1;
+	sec_mask = ~rest_addr;
 	for (liter = 0; liter < dwords; liter++, faddr++, dwptr++) {
 		fdata = (faddr & sec_mask) << 2;
 
 		/* Are we at the beginning of a sector? */
-		if ((faddr & rest_addr) == 0) {
-			/* Do sector unprotect. */
-			if (ha->fdt_unprotect_sec_cmd)
-				qla24xx_write_flash_dword(ha,
-				    ha->fdt_unprotect_sec_cmd,
-				    (fdata & 0xff00) | ((fdata << 16) &
-				    0xff0000) | ((fdata >> 16) & 0xff));
+		if (!(faddr & rest_addr)) {
+			ql_log(ql_log_warn + ql_dbg_verbose, vha, 0x7095,
+			    "Erase sector %#x...\n", faddr);
+
 			ret = qla24xx_erase_sector(vha, fdata);
-			if (ret != QLA_SUCCESS) {
+			if (ret) {
 				ql_dbg(ql_dbg_user, vha, 0x7007,
-				    "Unable to erase erase sector: address=%x.\n",
-				    faddr);
+				    "Failed to erase sector %x.\n", faddr);
 				break;
 			}
 		}
 
-		/* Go with burst-write. */
-		if (optrom && (liter + OPTROM_BURST_DWORDS) <= dwords) {
-			/* Copy data to DMA'ble buffer. */
-			memcpy(optrom, dwptr, OPTROM_BURST_SIZE);
+		if (optrom) {
+			/* If smaller than a burst remaining */
+			if (dwords - liter < dburst)
+				dburst = dwords - liter;
 
+			/* Copy to dma buffer */
+			memcpy(optrom, dwptr, dburst << 2);
+
+			/* Burst write */
+			ql_log(ql_log_warn + ql_dbg_verbose, vha, 0x7095,
+			    "Write burst (%#lx dwords)...\n", dburst);
 			ret = qla2x00_load_ram(vha, optrom_dma,
-			    flash_data_addr(ha, faddr),
-			    OPTROM_BURST_DWORDS);
-			if (ret != QLA_SUCCESS) {
-				ql_log(ql_log_warn, vha, 0x7097,
-				    "Unable to burst-write optrom segment "
-				    "(%x/%x/%llx).\n", ret,
-				    flash_data_addr(ha, faddr),
-				    (unsigned long long)optrom_dma);
-				ql_log(ql_log_warn, vha, 0x7098,
-				    "Reverting to slow-write.\n");
-
-				dma_free_coherent(&ha->pdev->dev,
-				    OPTROM_BURST_SIZE, optrom, optrom_dma);
-				optrom = NULL;
-			} else {
-				liter += OPTROM_BURST_DWORDS - 1;
-				faddr += OPTROM_BURST_DWORDS - 1;
-				dwptr += OPTROM_BURST_DWORDS - 1;
+			    flash_data_addr(ha, faddr), dburst);
+			if (!ret) {
+				liter += dburst - 1;
+				faddr += dburst - 1;
+				dwptr += dburst - 1;
 				continue;
 			}
+
+			ql_log(ql_log_warn, vha, 0x7097,
+			    "Failed burst-write at %x (%p/%#llx)....\n",
+			    flash_data_addr(ha, faddr), optrom,
+			    (u64)optrom_dma);
+
+			dma_free_coherent(&ha->pdev->dev,
+			    OPTROM_BURST_SIZE, optrom, optrom_dma);
+			optrom = NULL;
+			if (IS_QLA27XX(ha) || IS_QLA28XX(ha))
+				break;
+			ql_log(ql_log_warn, vha, 0x7098,
+			    "Reverting to slow write...\n");
 		}
 
+		/* Slow write */
 		ret = qla24xx_write_flash_dword(ha,
 		    flash_data_addr(ha, faddr), cpu_to_le32(*dwptr));
-		if (ret != QLA_SUCCESS) {
+		if (ret) {
 			ql_dbg(ql_dbg_user, vha, 0x7006,
-			    "Unable to program flash address=%x data=%x.\n",
-			    faddr, *dwptr);
+			    "Failed slopw write %x (%x)\n", faddr, *dwptr);
 			break;
 		}
-
-		/* Do sector protect. */
-		if (ha->fdt_unprotect_sec_cmd &&
-		    ((faddr & rest_addr) == rest_addr))
-			qla24xx_write_flash_dword(ha,
-			    ha->fdt_protect_sec_cmd,
-			    (fdata & 0xff00) | ((fdata << 16) &
-			    0xff0000) | ((fdata >> 16) & 0xff));
 	}
 
+	ql_log(ql_log_warn + ql_dbg_verbose, vha, 0x7095,
+	    "Protect flash...\n");
 	ret = qla24xx_protect_flash(vha);
-	if (ret != QLA_SUCCESS)
+	if (ret)
 		ql_log(ql_log_warn, vha, 0x7099,
-		    "Unable to protect flash after update.\n");
+		    "Failed to protect flash\n");
 done:
 	if (optrom)
 		dma_free_coherent(&ha->pdev->dev,
@@ -1349,7 +1373,7 @@ done:
 }
 
 uint8_t *
-qla2x00_read_nvram_data(scsi_qla_host_t *vha, uint8_t *buf, uint32_t naddr,
+qla2x00_read_nvram_data(scsi_qla_host_t *vha, void *buf, uint32_t naddr,
     uint32_t bytes)
 {
 	uint32_t i;
@@ -1368,27 +1392,30 @@ qla2x00_read_nvram_data(scsi_qla_host_t *vha, uint8_t *buf, uint32_t naddr,
 }
 
 uint8_t *
-qla24xx_read_nvram_data(scsi_qla_host_t *vha, uint8_t *buf, uint32_t naddr,
+qla24xx_read_nvram_data(scsi_qla_host_t *vha, void *buf, uint32_t naddr,
     uint32_t bytes)
 {
-	uint32_t i;
-	uint32_t *dwptr;
 	struct qla_hw_data *ha = vha->hw;
+	uint32_t *dwptr = buf;
+	uint32_t i;
 
 	if (IS_P3P_TYPE(ha))
 		return  buf;
 
 	/* Dword reads to flash. */
-	dwptr = (uint32_t *)buf;
-	for (i = 0; i < bytes >> 2; i++, naddr++)
-		dwptr[i] = cpu_to_le32(qla24xx_read_flash_dword(ha,
-		    nvram_data_addr(ha, naddr)));
+	naddr = nvram_data_addr(ha, naddr);
+	bytes >>= 2;
+	for (i = 0; i < bytes; i++, naddr++, dwptr++) {
+		if (qla24xx_read_flash_dword(ha, naddr, dwptr))
+			break;
+		cpu_to_le32s(dwptr);
+	}
 
 	return buf;
 }
 
 int
-qla2x00_write_nvram_data(scsi_qla_host_t *vha, uint8_t *buf, uint32_t naddr,
+qla2x00_write_nvram_data(scsi_qla_host_t *vha, void *buf, uint32_t naddr,
     uint32_t bytes)
 {
 	int ret, stat;
@@ -1422,14 +1449,14 @@ qla2x00_write_nvram_data(scsi_qla_host_t *vha, uint8_t *buf, uint32_t naddr,
 }
 
 int
-qla24xx_write_nvram_data(scsi_qla_host_t *vha, uint8_t *buf, uint32_t naddr,
+qla24xx_write_nvram_data(scsi_qla_host_t *vha, void *buf, uint32_t naddr,
     uint32_t bytes)
 {
-	int ret;
-	uint32_t i;
-	uint32_t *dwptr;
 	struct qla_hw_data *ha = vha->hw;
 	struct device_reg_24xx __iomem *reg = &ha->iobase->isp24;
+	uint32_t *dwptr = buf;
+	uint32_t i;
+	int ret;
 
 	ret = QLA_SUCCESS;
 
@@ -1446,11 +1473,10 @@ qla24xx_write_nvram_data(scsi_qla_host_t *vha, uint8_t *buf, uint32_t naddr,
 	qla24xx_write_flash_dword(ha, nvram_conf_addr(ha, 0x101), 0);
 
 	/* Dword writes to flash. */
-	dwptr = (uint32_t *)buf;
-	for (i = 0; i < bytes >> 2; i++, naddr++, dwptr++) {
-		ret = qla24xx_write_flash_dword(ha,
-		    nvram_data_addr(ha, naddr), cpu_to_le32(*dwptr));
-		if (ret != QLA_SUCCESS) {
+	naddr = nvram_data_addr(ha, naddr);
+	bytes >>= 2;
+	for (i = 0; i < bytes; i++, naddr++, dwptr++) {
+		if (qla24xx_write_flash_dword(ha, naddr, cpu_to_le32(*dwptr))) {
 			ql_dbg(ql_dbg_user, vha, 0x709a,
 			    "Unable to program nvram address=%x data=%x.\n",
 			    naddr, *dwptr);
@@ -1470,31 +1496,34 @@ qla24xx_write_nvram_data(scsi_qla_host_t *vha, uint8_t *buf, uint32_t naddr,
 }
 
 uint8_t *
-qla25xx_read_nvram_data(scsi_qla_host_t *vha, uint8_t *buf, uint32_t naddr,
+qla25xx_read_nvram_data(scsi_qla_host_t *vha, void *buf, uint32_t naddr,
     uint32_t bytes)
 {
-	uint32_t i;
-	uint32_t *dwptr;
 	struct qla_hw_data *ha = vha->hw;
+	uint32_t *dwptr = buf;
+	uint32_t i;
 
 	/* Dword reads to flash. */
-	dwptr = (uint32_t *)buf;
-	for (i = 0; i < bytes >> 2; i++, naddr++)
-		dwptr[i] = cpu_to_le32(qla24xx_read_flash_dword(ha,
-		    flash_data_addr(ha, ha->flt_region_vpd_nvram | naddr)));
+	naddr = flash_data_addr(ha, ha->flt_region_vpd_nvram | naddr);
+	bytes >>= 2;
+	for (i = 0; i < bytes; i++, naddr++, dwptr++) {
+		if (qla24xx_read_flash_dword(ha, naddr, dwptr))
+			break;
+
+		cpu_to_le32s(dwptr);
+	}
 
 	return buf;
 }
 
+#define RMW_BUFFER_SIZE	(64 * 1024)
 int
-qla25xx_write_nvram_data(scsi_qla_host_t *vha, uint8_t *buf, uint32_t naddr,
+qla25xx_write_nvram_data(scsi_qla_host_t *vha, void *buf, uint32_t naddr,
     uint32_t bytes)
 {
 	struct qla_hw_data *ha = vha->hw;
-#define RMW_BUFFER_SIZE	(64 * 1024)
-	uint8_t *dbuf;
+	uint8_t *dbuf = vmalloc(RMW_BUFFER_SIZE);
 
-	dbuf = vmalloc(RMW_BUFFER_SIZE);
 	if (!dbuf)
 		return QLA_MEMORY_ALLOC_FAILED;
 	ha->isp_ops->read_optrom(vha, dbuf, ha->flt_region_vpd_nvram << 2,
@@ -1728,7 +1757,7 @@ qla83xx_select_led_port(struct qla_hw_data *ha)
 {
 	uint32_t led_select_value = 0;
 
-	if (!IS_QLA83XX(ha) && !IS_QLA27XX(ha))
+	if (!IS_QLA83XX(ha) && !IS_QLA27XX(ha) && !IS_QLA28XX(ha))
 		goto out;
 
 	if (ha->port_no == 0)
@@ -1749,13 +1778,14 @@ qla83xx_beacon_blink(struct scsi_qla_host *vha)
 	uint16_t orig_led_cfg[6];
 	uint32_t led_10_value, led_43_value;
 
-	if (!IS_QLA83XX(ha) && !IS_QLA81XX(ha) && !IS_QLA27XX(ha))
+	if (!IS_QLA83XX(ha) && !IS_QLA81XX(ha) && !IS_QLA27XX(ha) &&
+	    !IS_QLA28XX(ha))
 		return;
 
 	if (!ha->beacon_blink_led)
 		return;
 
-	if (IS_QLA27XX(ha)) {
+	if (IS_QLA27XX(ha) || IS_QLA28XX(ha)) {
 		qla2x00_write_ram_word(vha, 0x1003, 0x40000230);
 		qla2x00_write_ram_word(vha, 0x1004, 0x40000230);
 	} else if (IS_QLA2031(ha)) {
@@ -1845,7 +1875,7 @@ qla24xx_beacon_on(struct scsi_qla_host *vha)
 			return QLA_FUNCTION_FAILED;
 		}
 
-		if (IS_QLA2031(ha) || IS_QLA27XX(ha))
+		if (IS_QLA2031(ha) || IS_QLA27XX(ha) || IS_QLA28XX(ha))
 			goto skip_gpio;
 
 		spin_lock_irqsave(&ha->hardware_lock, flags);
@@ -1885,7 +1915,7 @@ qla24xx_beacon_off(struct scsi_qla_host *vha)
 
 	ha->beacon_blink_led = 0;
 
-	if (IS_QLA2031(ha) || IS_QLA27XX(ha))
+	if (IS_QLA2031(ha) || IS_QLA27XX(ha) || IS_QLA28XX(ha))
 		goto set_fw_options;
 
 	if (IS_QLA8031(ha) || IS_QLA81XX(ha))
@@ -2314,8 +2344,8 @@ qla2x00_resume_hba(struct scsi_qla_host *vha)
 	scsi_unblock_requests(vha->host);
 }
 
-uint8_t *
-qla2x00_read_optrom_data(struct scsi_qla_host *vha, uint8_t *buf,
+void *
+qla2x00_read_optrom_data(struct scsi_qla_host *vha, void *buf,
     uint32_t offset, uint32_t length)
 {
 	uint32_t addr, midpoint;
@@ -2349,12 +2379,12 @@ qla2x00_read_optrom_data(struct scsi_qla_host *vha, uint8_t *buf,
 }
 
 int
-qla2x00_write_optrom_data(struct scsi_qla_host *vha, uint8_t *buf,
+qla2x00_write_optrom_data(struct scsi_qla_host *vha, void *buf,
     uint32_t offset, uint32_t length)
 {
 
 	int rval;
-	uint8_t man_id, flash_id, sec_number, data;
+	uint8_t man_id, flash_id, sec_number, *data;
 	uint16_t wd;
 	uint32_t addr, liter, sec_mask, rest_addr;
 	struct qla_hw_data *ha = vha->hw;
@@ -2483,7 +2513,7 @@ update_flash:
 
 		for (addr = offset, liter = 0; liter < length; liter++,
 		    addr++) {
-			data = buf[liter];
+			data = buf + liter;
 			/* Are we at the beginning of a sector? */
 			if ((addr & rest_addr) == 0) {
 				if (IS_QLA2322(ha) || IS_QLA6322(ha)) {
@@ -2551,7 +2581,7 @@ update_flash:
 				}
 			}
 
-			if (qla2x00_program_flash_address(ha, addr, data,
+			if (qla2x00_program_flash_address(ha, addr, *data,
 			    man_id, flash_id)) {
 				rval = QLA_FUNCTION_FAILED;
 				break;
@@ -2567,8 +2597,8 @@ update_flash:
 	return rval;
 }
 
-uint8_t *
-qla24xx_read_optrom_data(struct scsi_qla_host *vha, uint8_t *buf,
+void *
+qla24xx_read_optrom_data(struct scsi_qla_host *vha, void *buf,
     uint32_t offset, uint32_t length)
 {
 	struct qla_hw_data *ha = vha->hw;
@@ -2578,7 +2608,7 @@ qla24xx_read_optrom_data(struct scsi_qla_host *vha, uint8_t *buf,
 	set_bit(MBX_UPDATE_FLASH_ACTIVE, &ha->mbx_cmd_flags);
 
 	/* Go with read. */
-	qla24xx_read_flash_data(vha, (uint32_t *)buf, offset >> 2, length >> 2);
+	qla24xx_read_flash_data(vha, (void *)buf, offset >> 2, length >> 2);
 
 	/* Resume HBA. */
 	clear_bit(MBX_UPDATE_FLASH_ACTIVE, &ha->mbx_cmd_flags);
@@ -2587,8 +2617,340 @@ qla24xx_read_optrom_data(struct scsi_qla_host *vha, uint8_t *buf,
 	return buf;
 }
 
+static int
+qla28xx_extract_sfub_and_verify(struct scsi_qla_host *vha, uint32_t *buf,
+    uint32_t len, uint32_t buf_size_without_sfub, uint8_t *sfub_buf)
+{
+	uint32_t *p, check_sum = 0;
+	int i;
+
+	p = buf + buf_size_without_sfub;
+
+	/* Extract SFUB from end of file */
+	memcpy(sfub_buf, (uint8_t *)p,
+	    sizeof(struct secure_flash_update_block));
+
+	for (i = 0; i < (sizeof(struct secure_flash_update_block) >> 2); i++)
+		check_sum += p[i];
+
+	check_sum = (~check_sum) + 1;
+
+	if (check_sum != p[i]) {
+		ql_log(ql_log_warn, vha, 0x7097,
+		    "SFUB checksum failed, 0x%x, 0x%x\n",
+		    check_sum, p[i]);
+		return QLA_COMMAND_ERROR;
+	}
+
+	return QLA_SUCCESS;
+}
+
+static int
+qla28xx_get_flash_region(struct scsi_qla_host *vha, uint32_t start,
+    struct qla_flt_region *region)
+{
+	struct qla_hw_data *ha = vha->hw;
+	struct qla_flt_header *flt;
+	struct qla_flt_region *flt_reg;
+	uint16_t cnt;
+	int rval = QLA_FUNCTION_FAILED;
+
+	if (!ha->flt)
+		return QLA_FUNCTION_FAILED;
+
+	flt = (struct qla_flt_header *)ha->flt;
+	flt_reg = (struct qla_flt_region *)&flt[1];
+	cnt = le16_to_cpu(flt->length) / sizeof(struct qla_flt_region);
+
+	for (; cnt; cnt--, flt_reg++) {
+		if (flt_reg->start == start) {
+			memcpy((uint8_t *)region, flt_reg,
+			    sizeof(struct qla_flt_region));
+			rval = QLA_SUCCESS;
+			break;
+		}
+	}
+
+	return rval;
+}
+
+static int
+qla28xx_write_flash_data(scsi_qla_host_t *vha, uint32_t *dwptr, uint32_t faddr,
+    uint32_t dwords)
+{
+	struct qla_hw_data *ha = vha->hw;
+	ulong liter;
+	ulong dburst = OPTROM_BURST_DWORDS; /* burst size in dwords */
+	uint32_t sec_mask, rest_addr, fdata;
+	void *optrom = NULL;
+	dma_addr_t optrom_dma;
+	int rval;
+	struct secure_flash_update_block *sfub;
+	dma_addr_t sfub_dma;
+	uint32_t offset = faddr << 2;
+	uint32_t buf_size_without_sfub = 0;
+	struct qla_flt_region region;
+	bool reset_to_rom = false;
+	uint32_t risc_size, risc_attr = 0;
+	uint32_t *fw_array = NULL;
+
+	/* Retrieve region info - must be a start address passed in */
+	rval = qla28xx_get_flash_region(vha, offset, &region);
+
+	if (rval != QLA_SUCCESS) {
+		ql_log(ql_log_warn, vha, 0xffff,
+		    "Invalid address %x - not a region start address\n",
+		    offset);
+		goto done;
+	}
+
+	/* Allocate dma buffer for burst write */
+	optrom = dma_alloc_coherent(&ha->pdev->dev, OPTROM_BURST_SIZE,
+	    &optrom_dma, GFP_KERNEL);
+	if (!optrom) {
+		ql_log(ql_log_warn, vha, 0x7095,
+		    "Failed allocate burst (%x bytes)\n", OPTROM_BURST_SIZE);
+		rval = QLA_COMMAND_ERROR;
+		goto done;
+	}
+
+	/*
+	 * If adapter supports secure flash and region is secure
+	 * extract secure flash update block (SFUB) and verify
+	 */
+	if (ha->flags.secure_adapter && region.attribute) {
+
+		ql_log(ql_log_warn + ql_dbg_verbose, vha, 0xffff,
+		    "Region %x is secure\n", region.code);
+
+		if (region.code == FLT_REG_FW ||
+		    region.code == FLT_REG_FW_SEC_27XX) {
+			fw_array = dwptr;
+
+			/* 1st fw array */
+			risc_size = be32_to_cpu(fw_array[3]);
+			risc_attr = be32_to_cpu(fw_array[9]);
+
+			buf_size_without_sfub = risc_size;
+			fw_array += risc_size;
+
+			/* 2nd fw array */
+			risc_size = be32_to_cpu(fw_array[3]);
+
+			buf_size_without_sfub += risc_size;
+			fw_array += risc_size;
+
+			/* 1st dump template */
+			risc_size = be32_to_cpu(fw_array[2]);
+
+			/* skip header and ignore checksum */
+			buf_size_without_sfub += risc_size;
+			fw_array += risc_size;
+
+			if (risc_attr & BIT_9) {
+				/* 2nd dump template */
+				risc_size = be32_to_cpu(fw_array[2]);
+
+				/* skip header and ignore checksum */
+				buf_size_without_sfub += risc_size;
+				fw_array += risc_size;
+			}
+		} else {
+			ql_log(ql_log_warn + ql_dbg_verbose, vha, 0xffff,
+			    "Secure region %x not supported\n",
+			    region.code);
+			rval = QLA_COMMAND_ERROR;
+			goto done;
+		}
+
+		sfub = dma_alloc_coherent(&ha->pdev->dev,
+			sizeof(struct secure_flash_update_block), &sfub_dma,
+			GFP_KERNEL);
+		if (!sfub) {
+			ql_log(ql_log_warn, vha, 0xffff,
+			    "Unable to allocate memory for SFUB\n");
+			rval = QLA_COMMAND_ERROR;
+			goto done;
+		}
+
+		rval = qla28xx_extract_sfub_and_verify(vha, dwptr, dwords,
+			buf_size_without_sfub, (uint8_t *)sfub);
+
+		if (rval != QLA_SUCCESS)
+			goto done;
+
+		ql_log(ql_log_warn + ql_dbg_verbose, vha, 0xffff,
+		    "SFUB extract and verify successful\n");
+	}
+
+	rest_addr = (ha->fdt_block_size >> 2) - 1;
+	sec_mask = ~rest_addr;
+
+	/* Lock semaphore */
+	rval = qla81xx_fac_semaphore_access(vha, FAC_SEMAPHORE_LOCK);
+	if (rval != QLA_SUCCESS) {
+		ql_log(ql_log_warn, vha, 0xffff,
+		    "Unable to lock flash semaphore.");
+		goto done;
+	}
+
+	ql_log(ql_log_warn + ql_dbg_verbose, vha, 0x7095,
+	    "Unprotect flash...\n");
+	rval = qla24xx_unprotect_flash(vha);
+	if (rval) {
+		qla81xx_fac_semaphore_access(vha, FAC_SEMAPHORE_UNLOCK);
+		ql_log(ql_log_warn, vha, 0x7096, "Failed unprotect flash\n");
+		goto done;
+	}
+
+	for (liter = 0; liter < dwords; liter++, faddr++) {
+		fdata = (faddr & sec_mask) << 2;
+
+		/* If start of sector */
+		if (!(faddr & rest_addr)) {
+			ql_log(ql_log_warn + ql_dbg_verbose, vha, 0x7095,
+			    "Erase sector %#x...\n", faddr);
+			rval = qla24xx_erase_sector(vha, fdata);
+			if (rval) {
+				ql_dbg(ql_dbg_user, vha, 0x7007,
+				    "Failed erase sector %#x\n", faddr);
+				goto write_protect;
+			}
+		}
+	}
+
+	if (ha->flags.secure_adapter) {
+		/*
+		 * If adapter supports secure flash but FW doesn't,
+		 * disable write protect, release semaphore and reset
+		 * chip to execute ROM code in order to update region securely
+		 */
+		if (!ha->flags.secure_fw) {
+			ql_log(ql_log_warn + ql_dbg_verbose, vha, 0xffff,
+			    "Disable Write and Release Semaphore.");
+			rval = qla24xx_protect_flash(vha);
+			if (rval != QLA_SUCCESS) {
+				qla81xx_fac_semaphore_access(vha,
+					FAC_SEMAPHORE_UNLOCK);
+				ql_log(ql_log_warn, vha, 0xffff,
+				    "Unable to protect flash.");
+				goto done;
+			}
+
+			ql_log(ql_log_warn + ql_dbg_verbose, vha, 0xffff,
+			    "Reset chip to ROM.");
+			set_bit(ISP_ABORT_NEEDED, &vha->dpc_flags);
+			set_bit(ISP_ABORT_TO_ROM, &vha->dpc_flags);
+			qla2xxx_wake_dpc(vha);
+			rval = qla2x00_wait_for_chip_reset(vha);
+			if (rval != QLA_SUCCESS) {
+				ql_log(ql_log_warn, vha, 0xffff,
+				    "Unable to reset to ROM code.");
+				goto done;
+			}
+			reset_to_rom = true;
+			ha->flags.fac_supported = 0;
+
+			ql_log(ql_log_warn + ql_dbg_verbose, vha, 0xffff,
+			    "Lock Semaphore");
+			rval = qla2xxx_write_remote_register(vha,
+			    FLASH_SEMAPHORE_REGISTER_ADDR, 0x00020002);
+			if (rval != QLA_SUCCESS) {
+				ql_log(ql_log_warn, vha, 0xffff,
+				    "Unable to lock flash semaphore.");
+				goto done;
+			}
+
+			/* Unprotect flash */
+			ql_log(ql_log_warn + ql_dbg_verbose, vha, 0xffff,
+			    "Enable Write.");
+			rval = qla2x00_write_ram_word(vha, 0x7ffd0101, 0);
+			if (rval) {
+				ql_log(ql_log_warn, vha, 0x7096,
+				    "Failed unprotect flash\n");
+				goto done;
+			}
+		}
+
+		/* If region is secure, send Secure Flash MB Cmd */
+		if (region.attribute && buf_size_without_sfub) {
+			ql_log(ql_log_warn + ql_dbg_verbose, vha, 0xffff,
+			    "Sending Secure Flash MB Cmd\n");
+			rval = qla28xx_secure_flash_update(vha, 0, region.code,
+				buf_size_without_sfub, sfub_dma,
+				sizeof(struct secure_flash_update_block));
+			if (rval != QLA_SUCCESS) {
+				ql_log(ql_log_warn, vha, 0xffff,
+				    "Secure Flash MB Cmd failed %x.", rval);
+				goto write_protect;
+			}
+		}
+
+	}
+
+	/* re-init flash offset */
+	faddr = offset >> 2;
+
+	for (liter = 0; liter < dwords; liter++, faddr++, dwptr++) {
+		fdata = (faddr & sec_mask) << 2;
+
+		/* If smaller than a burst remaining */
+		if (dwords - liter < dburst)
+			dburst = dwords - liter;
+
+		/* Copy to dma buffer */
+		memcpy(optrom, dwptr, dburst << 2);
+
+		/* Burst write */
+		ql_log(ql_log_warn + ql_dbg_verbose, vha, 0x7095,
+		    "Write burst (%#lx dwords)...\n", dburst);
+		rval = qla2x00_load_ram(vha, optrom_dma,
+		    flash_data_addr(ha, faddr), dburst);
+		if (rval != QLA_SUCCESS) {
+			ql_log(ql_log_warn, vha, 0x7097,
+			    "Failed burst write at %x (%p/%#llx)...\n",
+			    flash_data_addr(ha, faddr), optrom,
+			    (u64)optrom_dma);
+			break;
+		}
+
+		liter += dburst - 1;
+		faddr += dburst - 1;
+		dwptr += dburst - 1;
+		continue;
+	}
+
+write_protect:
+	ql_log(ql_log_warn + ql_dbg_verbose, vha, 0x7095,
+	    "Protect flash...\n");
+	rval = qla24xx_protect_flash(vha);
+	if (rval) {
+		qla81xx_fac_semaphore_access(vha, FAC_SEMAPHORE_UNLOCK);
+		ql_log(ql_log_warn, vha, 0x7099,
+		    "Failed protect flash\n");
+	}
+
+	if (reset_to_rom == true) {
+		/* Schedule DPC to restart the RISC */
+		set_bit(ISP_ABORT_NEEDED, &vha->dpc_flags);
+		qla2xxx_wake_dpc(vha);
+
+		rval = qla2x00_wait_for_hba_online(vha);
+		if (rval != QLA_SUCCESS)
+			ql_log(ql_log_warn, vha, 0xffff,
+			    "Adapter did not come out of reset\n");
+	}
+
+done:
+	if (optrom)
+		dma_free_coherent(&ha->pdev->dev,
+		    OPTROM_BURST_SIZE, optrom, optrom_dma);
+
+	return rval;
+}
+
 int
-qla24xx_write_optrom_data(struct scsi_qla_host *vha, uint8_t *buf,
+qla24xx_write_optrom_data(struct scsi_qla_host *vha, void *buf,
     uint32_t offset, uint32_t length)
 {
 	int rval;
@@ -2599,8 +2961,12 @@ qla24xx_write_optrom_data(struct scsi_qla_host *vha, uint8_t *buf,
 	set_bit(MBX_UPDATE_FLASH_ACTIVE, &ha->mbx_cmd_flags);
 
 	/* Go with write. */
-	rval = qla24xx_write_flash_data(vha, (uint32_t *)buf, offset >> 2,
-	    length >> 2);
+	if (IS_QLA28XX(ha))
+		rval = qla28xx_write_flash_data(vha, (uint32_t *)buf,
+		    offset >> 2, length >> 2);
+	else
+		rval = qla24xx_write_flash_data(vha, (uint32_t *)buf,
+		    offset >> 2, length >> 2);
 
 	clear_bit(MBX_UPDATE_FLASH_ACTIVE, &ha->mbx_cmd_flags);
 	scsi_unblock_requests(vha->host);
@@ -2608,8 +2974,8 @@ qla24xx_write_optrom_data(struct scsi_qla_host *vha, uint8_t *buf,
 	return rval;
 }
 
-uint8_t *
-qla25xx_read_optrom_data(struct scsi_qla_host *vha, uint8_t *buf,
+void *
+qla25xx_read_optrom_data(struct scsi_qla_host *vha, void *buf,
     uint32_t offset, uint32_t length)
 {
 	int rval;
@@ -2620,7 +2986,7 @@ qla25xx_read_optrom_data(struct scsi_qla_host *vha, uint8_t *buf,
 	struct qla_hw_data *ha = vha->hw;
 
 	if (IS_QLA25XX(ha) || IS_QLA81XX(ha) || IS_QLA83XX(ha) ||
-	    IS_QLA27XX(ha))
+	    IS_QLA27XX(ha) || IS_QLA28XX(ha))
 		goto try_fast;
 	if (offset & 0xfff)
 		goto slow_read;
@@ -2628,6 +2994,8 @@ qla25xx_read_optrom_data(struct scsi_qla_host *vha, uint8_t *buf,
 		goto slow_read;
 
 try_fast:
+	if (offset & 0xff)
+		goto slow_read;
 	optrom = dma_alloc_coherent(&ha->pdev->dev, OPTROM_BURST_SIZE,
 	    &optrom_dma, GFP_KERNEL);
 	if (!optrom) {
@@ -2874,7 +3242,7 @@ qla2x00_get_flash_version(scsi_qla_host_t *vha, void *mbuf)
 		    "Dumping fw "
 		    "ver from flash:.\n");
 		ql_dump_buffer(ql_dbg_init + ql_dbg_buffer, vha, 0x010b,
-		    (uint8_t *)dbyte, 8);
+		    dbyte, 32);
 
 		if ((dcode[0] == 0xffff && dcode[1] == 0xffff &&
 		    dcode[2] == 0xffff && dcode[3] == 0xffff) ||
@@ -2905,8 +3273,8 @@ qla82xx_get_flash_version(scsi_qla_host_t *vha, void *mbuf)
 {
 	int ret = QLA_SUCCESS;
 	uint32_t pcihdr, pcids;
-	uint32_t *dcode;
-	uint8_t *bcode;
+	uint32_t *dcode = mbuf;
+	uint8_t *bcode = mbuf;
 	uint8_t code_type, last_image;
 	struct qla_hw_data *ha = vha->hw;
 
@@ -2918,17 +3286,14 @@ qla82xx_get_flash_version(scsi_qla_host_t *vha, void *mbuf)
 	memset(ha->fcode_revision, 0, sizeof(ha->fcode_revision));
 	memset(ha->fw_revision, 0, sizeof(ha->fw_revision));
 
-	dcode = mbuf;
-
 	/* Begin with first PCI expansion ROM header. */
 	pcihdr = ha->flt_region_boot << 2;
 	last_image = 1;
 	do {
 		/* Verify PCI expansion ROM header. */
-		ha->isp_ops->read_optrom(vha, (uint8_t *)dcode, pcihdr,
-		    0x20 * 4);
+		ha->isp_ops->read_optrom(vha, dcode, pcihdr, 0x20 * 4);
 		bcode = mbuf + (pcihdr % 4);
-		if (bcode[0x0] != 0x55 || bcode[0x1] != 0xaa) {
+		if (memcmp(bcode, "\x55\xaa", 2)) {
 			/* No signature */
 			ql_log(ql_log_fatal, vha, 0x0154,
 			    "No matching ROM signature.\n");
@@ -2939,13 +3304,11 @@ qla82xx_get_flash_version(scsi_qla_host_t *vha, void *mbuf)
 		/* Locate PCI data structure. */
 		pcids = pcihdr + ((bcode[0x19] << 8) | bcode[0x18]);
 
-		ha->isp_ops->read_optrom(vha, (uint8_t *)dcode, pcids,
-		    0x20 * 4);
+		ha->isp_ops->read_optrom(vha, dcode, pcids, 0x20 * 4);
 		bcode = mbuf + (pcihdr % 4);
 
 		/* Validate signature of PCI data structure. */
-		if (bcode[0x0] != 'P' || bcode[0x1] != 'C' ||
-		    bcode[0x2] != 'I' || bcode[0x3] != 'R') {
+		if (memcmp(bcode, "PCIR", 4)) {
 			/* Incorrect header. */
 			ql_log(ql_log_fatal, vha, 0x0155,
 			    "PCI data struct not found pcir_adr=%x.\n", pcids);
@@ -2996,8 +3359,7 @@ qla82xx_get_flash_version(scsi_qla_host_t *vha, void *mbuf)
 	/* Read firmware image information. */
 	memset(ha->fw_revision, 0, sizeof(ha->fw_revision));
 	dcode = mbuf;
-	ha->isp_ops->read_optrom(vha, (uint8_t *)dcode, ha->flt_region_fw << 2,
-	    0x20);
+	ha->isp_ops->read_optrom(vha, dcode, ha->flt_region_fw << 2, 0x20);
 	bcode = mbuf + (pcihdr % 4);
 
 	/* Validate signature of PCI data structure. */
@@ -3019,15 +3381,14 @@ int
 qla24xx_get_flash_version(scsi_qla_host_t *vha, void *mbuf)
 {
 	int ret = QLA_SUCCESS;
-	uint32_t pcihdr, pcids;
-	uint32_t *dcode;
-	uint8_t *bcode;
+	uint32_t pcihdr = 0, pcids = 0;
+	uint32_t *dcode = mbuf;
+	uint8_t *bcode = mbuf;
 	uint8_t code_type, last_image;
 	int i;
 	struct qla_hw_data *ha = vha->hw;
 	uint32_t faddr = 0;
-
-	pcihdr = pcids = 0;
+	struct active_regions active_regions = { };
 
 	if (IS_P3P_TYPE(ha))
 		return ret;
@@ -3040,18 +3401,19 @@ qla24xx_get_flash_version(scsi_qla_host_t *vha, void *mbuf)
 	memset(ha->fcode_revision, 0, sizeof(ha->fcode_revision));
 	memset(ha->fw_revision, 0, sizeof(ha->fw_revision));
 
-	dcode = mbuf;
 	pcihdr = ha->flt_region_boot << 2;
-	if (IS_QLA27XX(ha) &&
-	    qla27xx_find_valid_image(vha) == QLA27XX_SECONDARY_IMAGE)
-		pcihdr = ha->flt_region_boot_sec << 2;
+	if (IS_QLA27XX(ha) || IS_QLA28XX(ha)) {
+		qla27xx_get_active_image(vha, &active_regions);
+		if (active_regions.global == QLA27XX_SECONDARY_IMAGE) {
+			pcihdr = ha->flt_region_boot_sec << 2;
+		}
+	}
 
-	last_image = 1;
 	do {
 		/* Verify PCI expansion ROM header. */
 		qla24xx_read_flash_data(vha, dcode, pcihdr >> 2, 0x20);
 		bcode = mbuf + (pcihdr % 4);
-		if (bcode[0x0] != 0x55 || bcode[0x1] != 0xaa) {
+		if (memcmp(bcode, "\x55\xaa", 2)) {
 			/* No signature */
 			ql_log(ql_log_fatal, vha, 0x0059,
 			    "No matching ROM signature.\n");
@@ -3066,11 +3428,11 @@ qla24xx_get_flash_version(scsi_qla_host_t *vha, void *mbuf)
 		bcode = mbuf + (pcihdr % 4);
 
 		/* Validate signature of PCI data structure. */
-		if (bcode[0x0] != 'P' || bcode[0x1] != 'C' ||
-		    bcode[0x2] != 'I' || bcode[0x3] != 'R') {
+		if (memcmp(bcode, "PCIR", 4)) {
 			/* Incorrect header. */
 			ql_log(ql_log_fatal, vha, 0x005a,
 			    "PCI data struct not found pcir_adr=%x.\n", pcids);
+			ql_dump_buffer(ql_dbg_init, vha, 0x0059, dcode, 32);
 			ret = QLA_FUNCTION_FAILED;
 			break;
 		}
@@ -3117,30 +3479,24 @@ qla24xx_get_flash_version(scsi_qla_host_t *vha, void *mbuf)
 
 	/* Read firmware image information. */
 	memset(ha->fw_revision, 0, sizeof(ha->fw_revision));
-	dcode = mbuf;
 	faddr = ha->flt_region_fw;
-	if (IS_QLA27XX(ha) &&
-	    qla27xx_find_valid_image(vha) == QLA27XX_SECONDARY_IMAGE)
-		faddr = ha->flt_region_fw_sec;
-
-	qla24xx_read_flash_data(vha, dcode, faddr + 4, 4);
-	for (i = 0; i < 4; i++)
-		dcode[i] = be32_to_cpu(dcode[i]);
+	if (IS_QLA27XX(ha) || IS_QLA28XX(ha)) {
+		qla27xx_get_active_image(vha, &active_regions);
+		if (active_regions.global == QLA27XX_SECONDARY_IMAGE)
+			faddr = ha->flt_region_fw_sec;
+	}
 
-	if ((dcode[0] == 0xffffffff && dcode[1] == 0xffffffff &&
-	    dcode[2] == 0xffffffff && dcode[3] == 0xffffffff) ||
-	    (dcode[0] == 0 && dcode[1] == 0 && dcode[2] == 0 &&
-	    dcode[3] == 0)) {
+	qla24xx_read_flash_data(vha, dcode, faddr, 8);
+	if (qla24xx_risc_firmware_invalid(dcode)) {
 		ql_log(ql_log_warn, vha, 0x005f,
 		    "Unrecognized fw revision at %x.\n",
 		    ha->flt_region_fw * 4);
+		ql_dump_buffer(ql_dbg_init, vha, 0x005f, dcode, 32);
 	} else {
-		ha->fw_revision[0] = dcode[0];
-		ha->fw_revision[1] = dcode[1];
-		ha->fw_revision[2] = dcode[2];
-		ha->fw_revision[3] = dcode[3];
+		for (i = 0; i < 4; i++)
+			ha->fw_revision[i] = be32_to_cpu(dcode[4+i]);
 		ql_dbg(ql_dbg_init, vha, 0x0060,
-		    "Firmware revision %d.%d.%d (%x).\n",
+		    "Firmware revision (flash) %u.%u.%u (%x).\n",
 		    ha->fw_revision[0], ha->fw_revision[1],
 		    ha->fw_revision[2], ha->fw_revision[3]);
 	}
@@ -3152,20 +3508,17 @@ qla24xx_get_flash_version(scsi_qla_host_t *vha, void *mbuf)
 	}
 
 	memset(ha->gold_fw_version, 0, sizeof(ha->gold_fw_version));
-	dcode = mbuf;
-	ha->isp_ops->read_optrom(vha, (uint8_t *)dcode,
-	    ha->flt_region_gold_fw << 2, 32);
-
-	if (dcode[4] == 0xFFFFFFFF && dcode[5] == 0xFFFFFFFF &&
-	    dcode[6] == 0xFFFFFFFF && dcode[7] == 0xFFFFFFFF) {
+	faddr = ha->flt_region_gold_fw;
+	qla24xx_read_flash_data(vha, (void *)dcode, ha->flt_region_gold_fw, 8);
+	if (qla24xx_risc_firmware_invalid(dcode)) {
 		ql_log(ql_log_warn, vha, 0x0056,
-		    "Unrecognized golden fw at 0x%x.\n",
-		    ha->flt_region_gold_fw * 4);
+		    "Unrecognized golden fw at %#x.\n", faddr);
+		ql_dump_buffer(ql_dbg_init, vha, 0x0056, dcode, 32);
 		return ret;
 	}
 
-	for (i = 4; i < 8; i++)
-		ha->gold_fw_version[i-4] = be32_to_cpu(dcode[i]);
+	for (i = 0; i < 4; i++)
+		ha->gold_fw_version[i] = be32_to_cpu(dcode[4+i]);
 
 	return ret;
 }
@@ -3237,7 +3590,7 @@ qla24xx_read_fcp_prio_cfg(scsi_qla_host_t *vha)
 	fcp_prio_addr = ha->flt_region_fcp_prio;
 
 	/* first read the fcp priority data header from flash */
-	ha->isp_ops->read_optrom(vha, (uint8_t *)ha->fcp_prio_cfg,
+	ha->isp_ops->read_optrom(vha, ha->fcp_prio_cfg,
 			fcp_prio_addr << 2, FCP_PRIO_CFG_HDR_SIZE);
 
 	if (!qla24xx_fcp_prio_cfg_valid(vha, ha->fcp_prio_cfg, 0))
@@ -3248,7 +3601,7 @@ qla24xx_read_fcp_prio_cfg(scsi_qla_host_t *vha)
 	len = ha->fcp_prio_cfg->num_entries * FCP_PRIO_CFG_ENTRY_SIZE;
 	max_len = FCP_PRIO_CFG_SIZE - FCP_PRIO_CFG_HDR_SIZE;
 
-	ha->isp_ops->read_optrom(vha, (uint8_t *)&ha->fcp_prio_cfg->entry[0],
+	ha->isp_ops->read_optrom(vha, &ha->fcp_prio_cfg->entry[0],
 			fcp_prio_addr << 2, (len < max_len ? len : max_len));
 
 	/* revalidate the entire FCP priority config data, including entries */
diff --git a/drivers/scsi/qla2xxx/qla_target.c b/drivers/scsi/qla2xxx/qla_target.c
index 582d1663f971..3eeae72793bc 100644
--- a/drivers/scsi/qla2xxx/qla_target.c
+++ b/drivers/scsi/qla2xxx/qla_target.c
@@ -184,6 +184,7 @@ static inline int qlt_issue_marker(struct scsi_qla_host *vha, int vha_locked)
 	/* Send marker if required */
 	if (unlikely(vha->marker_needed != 0)) {
 		int rc = qla2x00_issue_marker(vha, vha_locked);
+
 		if (rc != QLA_SUCCESS) {
 			ql_dbg(ql_dbg_tgt, vha, 0xe03d,
 			    "qla_target(%d): issue_marker() failed\n",
@@ -557,6 +558,7 @@ static int qla24xx_post_nack_work(struct scsi_qla_host *vha, fc_port_t *fcport,
 	struct imm_ntfy_from_isp *ntfy, int type)
 {
 	struct qla_work_evt *e;
+
 	e = qla2x00_alloc_work(vha, QLA_EVT_NACK);
 	if (!e)
 		return QLA_FUNCTION_FAILED;
@@ -680,7 +682,6 @@ done:
 void qla24xx_do_nack_work(struct scsi_qla_host *vha, struct qla_work_evt *e)
 {
 	fc_port_t *t;
-	unsigned long flags;
 
 	switch (e->u.nack.type) {
 	case SRB_NACK_PRLI:
@@ -693,24 +694,19 @@ void qla24xx_do_nack_work(struct scsi_qla_host *vha, struct qla_work_evt *e)
 		if (t) {
 			ql_log(ql_log_info, vha, 0xd034,
 			    "%s create sess success %p", __func__, t);
-			spin_lock_irqsave(&vha->hw->tgt.sess_lock, flags);
 			/* create sess has an extra kref */
 			vha->hw->tgt.tgt_ops->put_sess(e->u.nack.fcport);
-			spin_unlock_irqrestore(&vha->hw->tgt.sess_lock, flags);
 		}
 		break;
 	}
 	qla24xx_async_notify_ack(vha, e->u.nack.fcport,
-	    (struct imm_ntfy_from_isp*)e->u.nack.iocb, e->u.nack.type);
+	    (struct imm_ntfy_from_isp *)e->u.nack.iocb, e->u.nack.type);
 }
 
 void qla24xx_delete_sess_fn(struct work_struct *work)
 {
 	fc_port_t *fcport = container_of(work, struct fc_port, del_work);
 	struct qla_hw_data *ha = fcport->vha->hw;
-	unsigned long flags;
-
-	spin_lock_irqsave(&ha->tgt.sess_lock, flags);
 
 	if (fcport->se_sess) {
 		ha->tgt.tgt_ops->shutdown_sess(fcport);
@@ -718,7 +714,6 @@ void qla24xx_delete_sess_fn(struct work_struct *work)
 	} else {
 		qlt_unreg_sess(fcport);
 	}
-	spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
 }
 
 /*
@@ -787,8 +782,9 @@ void qlt_fc_port_added(struct scsi_qla_host *vha, fc_port_t *fcport)
 		    fcport->port_name, sess->loop_id);
 		sess->local = 0;
 	}
-	ha->tgt.tgt_ops->put_sess(sess);
 	spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
+
+	ha->tgt.tgt_ops->put_sess(sess);
 }
 
 /*
@@ -980,6 +976,8 @@ void qlt_free_session_done(struct work_struct *work)
 		sess->send_els_logo);
 
 	if (!IS_SW_RESV_ADDR(sess->d_id)) {
+		qla2x00_mark_device_lost(vha, sess, 0, 0);
+
 		if (sess->send_els_logo) {
 			qlt_port_logo_t logo;
 
@@ -1076,6 +1074,7 @@ void qlt_free_session_done(struct work_struct *work)
 		struct qlt_plogi_ack_t *con =
 		    sess->plogi_link[QLT_PLOGI_LINK_CONFLICT];
 		struct imm_ntfy_from_isp *iocb;
+
 		own = sess->plogi_link[QLT_PLOGI_LINK_SAME_WWN];
 
 		if (con) {
@@ -1160,8 +1159,6 @@ void qlt_unreg_sess(struct fc_port *sess)
 	if (sess->se_sess)
 		vha->hw->tgt.tgt_ops->clear_nacl_from_fcport_map(sess);
 
-	qla2x00_mark_device_lost(vha, sess, 0, 0);
-
 	sess->deleted = QLA_SESS_DELETION_IN_PROGRESS;
 	sess->disc_state = DSC_DELETE_PEND;
 	sess->last_rscn_gen = sess->rscn_gen;
@@ -1329,6 +1326,7 @@ static int qla24xx_get_loop_id(struct scsi_qla_host *vha, const uint8_t *s_id,
 	res = -ENOENT;
 	for (i = 0; i < entries; i++) {
 		struct gid_list_info *gid = (struct gid_list_info *)id_iter;
+
 		if ((gid->al_pa == s_id[2]) &&
 		    (gid->area == s_id[1]) &&
 		    (gid->domain == s_id[0])) {
@@ -2331,14 +2329,14 @@ void qlt_send_resp_ctio(struct qla_qpair *qpair, struct qla_tgt_cmd *cmd,
 		ctio->u.status1.scsi_status |=
 		    cpu_to_le16(SS_RESIDUAL_UNDER);
 
-	/* Response code and sense key */
-	put_unaligned_le32(((0x70 << 24) | (sense_key << 8)),
-	    (&ctio->u.status1.sense_data)[0]);
+	/* Fixed format sense data. */
+	ctio->u.status1.sense_data[0] = 0x70;
+	ctio->u.status1.sense_data[2] = sense_key;
 	/* Additional sense length */
-	put_unaligned_le32(0x0a, (&ctio->u.status1.sense_data)[1]);
+	ctio->u.status1.sense_data[7] = 0xa;
 	/* ASC and ASCQ */
-	put_unaligned_le32(((asc << 24) | (ascq << 16)),
-	    (&ctio->u.status1.sense_data)[3]);
+	ctio->u.status1.sense_data[12] = asc;
+	ctio->u.status1.sense_data[13] = ascq;
 
 	/* Memory Barrier */
 	wmb();
@@ -2387,7 +2385,7 @@ void qlt_xmit_tm_rsp(struct qla_tgt_mgmt_cmd *mcmd)
 		case ELS_PRLO:
 		case ELS_TPRLO:
 			ql_dbg(ql_dbg_disc, vha, 0x2106,
-			    "TM response logo %phC status %#x state %#x",
+			    "TM response logo %8phC status %#x state %#x",
 			    mcmd->sess->port_name, mcmd->fc_tm_rsp,
 			    mcmd->flags);
 			qlt_schedule_sess_for_deletion(mcmd->sess);
@@ -2485,6 +2483,7 @@ static void qlt_unmap_sg(struct scsi_qla_host *vha, struct qla_tgt_cmd *cmd)
 {
 	struct qla_hw_data *ha;
 	struct qla_qpair *qpair;
+
 	if (!cmd->sg_mapped)
 		return;
 
@@ -2635,7 +2634,7 @@ static int qlt_24xx_build_ctio_pkt(struct qla_qpair *qpair,
 static void qlt_load_cont_data_segments(struct qla_tgt_prm *prm)
 {
 	int cnt;
-	uint32_t *dword_ptr;
+	struct dsd64 *cur_dsd;
 
 	/* Build continuation packets */
 	while (prm->seg_cnt > 0) {
@@ -2656,19 +2655,13 @@ static void qlt_load_cont_data_segments(struct qla_tgt_prm *prm)
 		cont_pkt64->sys_define = 0;
 
 		cont_pkt64->entry_type = CONTINUE_A64_TYPE;
-		dword_ptr = (uint32_t *)&cont_pkt64->dseg_0_address;
+		cur_dsd = cont_pkt64->dsd;
 
 		/* Load continuation entry data segments */
 		for (cnt = 0;
 		    cnt < QLA_TGT_DATASEGS_PER_CONT_24XX && prm->seg_cnt;
 		    cnt++, prm->seg_cnt--) {
-			*dword_ptr++ =
-			    cpu_to_le32(lower_32_bits
-				(sg_dma_address(prm->sg)));
-			*dword_ptr++ = cpu_to_le32(upper_32_bits
-			    (sg_dma_address(prm->sg)));
-			*dword_ptr++ = cpu_to_le32(sg_dma_len(prm->sg));
-
+			append_dsd64(&cur_dsd, prm->sg);
 			prm->sg = sg_next(prm->sg);
 		}
 	}
@@ -2681,13 +2674,13 @@ static void qlt_load_cont_data_segments(struct qla_tgt_prm *prm)
 static void qlt_load_data_segments(struct qla_tgt_prm *prm)
 {
 	int cnt;
-	uint32_t *dword_ptr;
+	struct dsd64 *cur_dsd;
 	struct ctio7_to_24xx *pkt24 = (struct ctio7_to_24xx *)prm->pkt;
 
 	pkt24->u.status0.transfer_length = cpu_to_le32(prm->cmd->bufflen);
 
 	/* Setup packet address segment pointer */
-	dword_ptr = pkt24->u.status0.dseg_0_address;
+	cur_dsd = &pkt24->u.status0.dsd;
 
 	/* Set total data segment count */
 	if (prm->seg_cnt)
@@ -2695,8 +2688,8 @@ static void qlt_load_data_segments(struct qla_tgt_prm *prm)
 
 	if (prm->seg_cnt == 0) {
 		/* No data transfer */
-		*dword_ptr++ = 0;
-		*dword_ptr = 0;
+		cur_dsd->address = 0;
+		cur_dsd->length = 0;
 		return;
 	}
 
@@ -2706,14 +2699,7 @@ static void qlt_load_data_segments(struct qla_tgt_prm *prm)
 	for (cnt = 0;
 	    (cnt < QLA_TGT_DATASEGS_PER_CMD_24XX) && prm->seg_cnt;
 	    cnt++, prm->seg_cnt--) {
-		*dword_ptr++ =
-		    cpu_to_le32(lower_32_bits(sg_dma_address(prm->sg)));
-
-		*dword_ptr++ = cpu_to_le32(upper_32_bits(
-			sg_dma_address(prm->sg)));
-
-		*dword_ptr++ = cpu_to_le32(sg_dma_len(prm->sg));
-
+		append_dsd64(&cur_dsd, prm->sg);
 		prm->sg = sg_next(prm->sg);
 	}
 
@@ -3037,7 +3023,7 @@ qla_tgt_set_dif_tags(struct qla_tgt_cmd *cmd, struct crc_context *ctx,
 static inline int
 qlt_build_ctio_crc2_pkt(struct qla_qpair *qpair, struct qla_tgt_prm *prm)
 {
-	uint32_t		*cur_dsd;
+	struct dsd64		*cur_dsd;
 	uint32_t		transfer_length = 0;
 	uint32_t		data_bytes;
 	uint32_t		dif_bytes;
@@ -3183,12 +3169,11 @@ qlt_build_ctio_crc2_pkt(struct qla_qpair *qpair, struct qla_tgt_prm *prm)
 
 	qla_tgt_set_dif_tags(cmd, crc_ctx_pkt, &fw_prot_opts);
 
-	pkt->crc_context_address[0] = cpu_to_le32(LSD(crc_ctx_dma));
-	pkt->crc_context_address[1] = cpu_to_le32(MSD(crc_ctx_dma));
+	put_unaligned_le64(crc_ctx_dma, &pkt->crc_context_address);
 	pkt->crc_context_len = CRC_CONTEXT_LEN_FW;
 
 	if (!bundling) {
-		cur_dsd = (uint32_t *) &crc_ctx_pkt->u.nobundling.data_address;
+		cur_dsd = &crc_ctx_pkt->u.nobundling.data_dsd;
 	} else {
 		/*
 		 * Configure Bundling if we need to fetch interlaving
@@ -3198,7 +3183,7 @@ qlt_build_ctio_crc2_pkt(struct qla_qpair *qpair, struct qla_tgt_prm *prm)
 		crc_ctx_pkt->u.bundling.dif_byte_count = cpu_to_le32(dif_bytes);
 		crc_ctx_pkt->u.bundling.dseg_count =
 			cpu_to_le16(prm->tot_dsds - prm->prot_seg_cnt);
-		cur_dsd = (uint32_t *) &crc_ctx_pkt->u.bundling.data_address;
+		cur_dsd = &crc_ctx_pkt->u.bundling.data_dsd;
 	}
 
 	/* Finish the common fields of CRC pkt */
@@ -3231,7 +3216,7 @@ qlt_build_ctio_crc2_pkt(struct qla_qpair *qpair, struct qla_tgt_prm *prm)
 		/* Walks dif segments */
 		pkt->add_flags |= CTIO_CRC2_AF_DIF_DSD_ENA;
 
-		cur_dsd = (uint32_t *) &crc_ctx_pkt->u.bundling.dif_address;
+		cur_dsd = &crc_ctx_pkt->u.bundling.dif_dsd;
 		if (qla24xx_walk_and_build_prot_sglist(ha, NULL, cur_dsd,
 			prm->prot_seg_cnt, cmd))
 			goto crc_queuing_error;
@@ -3263,7 +3248,6 @@ int qlt_xmit_response(struct qla_tgt_cmd *cmd, int xmit_type,
 	if (!qpair->fw_started || (cmd->reset_count != qpair->chip_reset) ||
 	    (cmd->sess && cmd->sess->deleted)) {
 		cmd->state = QLA_TGT_STATE_PROCESSED;
-		qlt_abort_cmd_on_host_reset(cmd->vha, cmd);
 		return 0;
 	}
 
@@ -3292,7 +3276,6 @@ int qlt_xmit_response(struct qla_tgt_cmd *cmd, int xmit_type,
 		 * previous life, just abort the processing.
 		 */
 		cmd->state = QLA_TGT_STATE_PROCESSED;
-		qlt_abort_cmd_on_host_reset(cmd->vha, cmd);
 		ql_dbg_qp(ql_dbg_async, qpair, 0xe101,
 			"RESET-RSP online/active/old-count/new-count = %d/%d/%d/%d.\n",
 			vha->flags.online, qla2x00_reset_active(vha),
@@ -3384,9 +3367,7 @@ int qlt_xmit_response(struct qla_tgt_cmd *cmd, int xmit_type,
 
 
 	cmd->state = QLA_TGT_STATE_PROCESSED; /* Mid-level is done processing */
-	spin_lock(&cmd->cmd_lock);
 	cmd->cmd_sent_to_fw = 1;
-	spin_unlock(&cmd->cmd_lock);
 	cmd->ctio_flags = le16_to_cpu(pkt->u.status0.flags);
 
 	/* Memory Barrier */
@@ -3433,8 +3414,10 @@ int qlt_rdy_to_xfer(struct qla_tgt_cmd *cmd)
 		 * Either the port is not online or this request was from
 		 * previous life, just abort the processing.
 		 */
-		cmd->state = QLA_TGT_STATE_NEED_DATA;
-		qlt_abort_cmd_on_host_reset(cmd->vha, cmd);
+		cmd->aborted = 1;
+		cmd->write_data_transferred = 0;
+		cmd->state = QLA_TGT_STATE_DATA_IN;
+		vha->hw->tgt.tgt_ops->handle_data(cmd);
 		ql_dbg_qp(ql_dbg_async, qpair, 0xe102,
 			"RESET-XFR online/active/old-count/new-count = %d/%d/%d/%d.\n",
 			vha->flags.online, qla2x00_reset_active(vha),
@@ -3465,9 +3448,7 @@ int qlt_rdy_to_xfer(struct qla_tgt_cmd *cmd)
 		qlt_load_data_segments(&prm);
 
 	cmd->state = QLA_TGT_STATE_NEED_DATA;
-	spin_lock(&cmd->cmd_lock);
 	cmd->cmd_sent_to_fw = 1;
-	spin_unlock(&cmd->cmd_lock);
 	cmd->ctio_flags = le16_to_cpu(pkt->u.status0.flags);
 
 	/* Memory Barrier */
@@ -3646,33 +3627,11 @@ static int __qlt_send_term_imm_notif(struct scsi_qla_host *vha,
 static void qlt_send_term_imm_notif(struct scsi_qla_host *vha,
 	struct imm_ntfy_from_isp *imm, int ha_locked)
 {
-	unsigned long flags = 0;
 	int rc;
 
-	if (ha_locked) {
-		rc = __qlt_send_term_imm_notif(vha, imm);
-
-#if 0	/* Todo  */
-		if (rc == -ENOMEM)
-			qlt_alloc_qfull_cmd(vha, imm, 0, 0);
-#else
-		if (rc) {
-		}
-#endif
-		goto done;
-	}
-
-	spin_lock_irqsave(&vha->hw->hardware_lock, flags);
+	WARN_ON_ONCE(!ha_locked);
 	rc = __qlt_send_term_imm_notif(vha, imm);
-
-#if 0	/* Todo */
-	if (rc == -ENOMEM)
-		qlt_alloc_qfull_cmd(vha, imm, 0, 0);
-#endif
-
-done:
-	if (!ha_locked)
-		spin_unlock_irqrestore(&vha->hw->hardware_lock, flags);
+	pr_debug("rc = %d\n", rc);
 }
 
 /*
@@ -3913,6 +3872,7 @@ static int qlt_term_ctio_exchange(struct qla_qpair *qpair, void *ctio,
 
 	if (ctio != NULL) {
 		struct ctio7_from_24xx *c = (struct ctio7_from_24xx *)ctio;
+
 		term = !(c->flags &
 		    cpu_to_le16(OF_TERM_EXCH));
 	} else
@@ -3977,39 +3937,6 @@ static void *qlt_ctio_to_cmd(struct scsi_qla_host *vha,
 	return cmd;
 }
 
-/* hardware_lock should be held by caller. */
-void
-qlt_abort_cmd_on_host_reset(struct scsi_qla_host *vha, struct qla_tgt_cmd *cmd)
-{
-	struct qla_hw_data *ha = vha->hw;
-
-	if (cmd->sg_mapped)
-		qlt_unmap_sg(vha, cmd);
-
-	/* TODO: fix debug message type and ids. */
-	if (cmd->state == QLA_TGT_STATE_PROCESSED) {
-		ql_dbg(ql_dbg_io, vha, 0xff00,
-		    "HOST-ABORT: state=PROCESSED.\n");
-	} else if (cmd->state == QLA_TGT_STATE_NEED_DATA) {
-		cmd->write_data_transferred = 0;
-		cmd->state = QLA_TGT_STATE_DATA_IN;
-
-		ql_dbg(ql_dbg_io, vha, 0xff01,
-		    "HOST-ABORT: state=DATA_IN.\n");
-
-		ha->tgt.tgt_ops->handle_data(cmd);
-		return;
-	} else {
-		ql_dbg(ql_dbg_io, vha, 0xff03,
-		    "HOST-ABORT: state=BAD(%d).\n",
-		    cmd->state);
-		dump_stack();
-	}
-
-	cmd->trc_flags |= TRC_FLUSH;
-	ha->tgt.tgt_ops->free_cmd(cmd);
-}
-
 /*
  * ha->hardware_lock supposed to be held on entry. Might drop it, then reaquire
  */
@@ -4031,7 +3958,7 @@ static void qlt_do_ctio_completion(struct scsi_qla_host *vha,
 		return;
 	}
 
-	cmd = (struct qla_tgt_cmd *)qlt_ctio_to_cmd(vha, rsp, handle, ctio);
+	cmd = qlt_ctio_to_cmd(vha, rsp, handle, ctio);
 	if (cmd == NULL)
 		return;
 
@@ -4240,11 +4167,9 @@ static void __qlt_do_work(struct qla_tgt_cmd *cmd)
 	if (ret != 0)
 		goto out_term;
 	/*
-	 * Drop extra session reference from qla_tgt_handle_cmd_for_atio*(
+	 * Drop extra session reference from qlt_handle_cmd_for_atio().
 	 */
-	spin_lock_irqsave(&ha->tgt.sess_lock, flags);
 	ha->tgt.tgt_ops->put_sess(sess);
-	spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
 	return;
 
 out_term:
@@ -4261,9 +4186,7 @@ out_term:
 	target_free_tag(sess->se_sess, &cmd->se_cmd);
 	spin_unlock_irqrestore(qpair->qp_lock_ptr, flags);
 
-	spin_lock_irqsave(&ha->tgt.sess_lock, flags);
 	ha->tgt.tgt_ops->put_sess(sess);
-	spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
 }
 
 static void qlt_do_work(struct work_struct *work)
@@ -4472,9 +4395,7 @@ static int qlt_handle_cmd_for_atio(struct scsi_qla_host *vha,
 	if (!cmd) {
 		ql_dbg(ql_dbg_io, vha, 0x3062,
 		    "qla_target(%d): Allocation of cmd failed\n", vha->vp_idx);
-		spin_lock_irqsave(&ha->tgt.sess_lock, flags);
 		ha->tgt.tgt_ops->put_sess(sess);
-		spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
 		return -EBUSY;
 	}
 
@@ -4773,6 +4694,7 @@ static int abort_cmds_for_s_id(struct scsi_qla_host *vha, port_id_t *s_id)
 
 	list_for_each_entry(op, &vha->unknown_atio_list, cmd_list) {
 		uint32_t op_key = sid_to_key(op->atio.u.isp24.fcp_hdr.s_id);
+
 		if (op_key == key) {
 			op->aborted = true;
 			count++;
@@ -4781,6 +4703,7 @@ static int abort_cmds_for_s_id(struct scsi_qla_host *vha, port_id_t *s_id)
 
 	list_for_each_entry(cmd, &vha->qla_cmd_list, cmd_list) {
 		uint32_t cmd_key = sid_to_key(cmd->atio.u.isp24.fcp_hdr.s_id);
+
 		if (cmd_key == key) {
 			cmd->aborted = 1;
 			count++;
@@ -5051,6 +4974,7 @@ static int qlt_24xx_handle_els(struct scsi_qla_host *vha,
 		if (sess != NULL) {
 			bool delete = false;
 			int sec;
+
 			spin_lock_irqsave(&tgt->ha->tgt.sess_lock, flags);
 			switch (sess->fw_login_state) {
 			case DSC_LS_PLOGI_PEND:
@@ -5203,6 +5127,7 @@ static int qlt_24xx_handle_els(struct scsi_qla_host *vha,
 	case ELS_ADISC:
 	{
 		struct qla_tgt *tgt = vha->vha_tgt.qla_tgt;
+
 		if (tgt->link_reinit_iocb_pending) {
 			qlt_send_notify_ack(ha->base_qpair,
 			    &tgt->link_reinit_iocb, 0, 0, 0, 0, 0, 0);
@@ -5266,6 +5191,7 @@ static void qlt_handle_imm_notify(struct scsi_qla_host *vha,
 	case IMM_NTFY_LIP_LINK_REINIT:
 	{
 		struct qla_tgt *tgt = vha->vha_tgt.qla_tgt;
+
 		ql_dbg(ql_dbg_tgt_mgt, vha, 0xf033,
 		    "qla_target(%d): LINK REINIT (loop %#x, "
 		    "subcode %x)\n", vha->vp_idx,
@@ -5492,11 +5418,7 @@ qlt_alloc_qfull_cmd(struct scsi_qla_host *vha,
 	se_sess = sess->se_sess;
 
 	tag = sbitmap_queue_get(&se_sess->sess_tag_pool, &cpu);
-	if (tag < 0)
-		return;
-
-	cmd = &((struct qla_tgt_cmd *)se_sess->sess_cmd_map)[tag];
-	if (!cmd) {
+	if (tag < 0) {
 		ql_dbg(ql_dbg_io, vha, 0x3009,
 			"qla_target(%d): %s: Allocation of cmd failed\n",
 			vha->vp_idx, __func__);
@@ -5511,6 +5433,7 @@ qlt_alloc_qfull_cmd(struct scsi_qla_host *vha,
 		return;
 	}
 
+	cmd = &((struct qla_tgt_cmd *)se_sess->sess_cmd_map)[tag];
 	memset(cmd, 0, sizeof(struct qla_tgt_cmd));
 
 	qlt_incr_num_pend_cmds(vha);
@@ -5820,8 +5743,7 @@ static void qlt_handle_abts_completion(struct scsi_qla_host *vha,
 	struct qla_tgt_mgmt_cmd *mcmd;
 	struct qla_hw_data *ha = vha->hw;
 
-	mcmd = (struct qla_tgt_mgmt_cmd *)qlt_ctio_to_cmd(vha, rsp,
-	    pkt->handle, pkt);
+	mcmd = qlt_ctio_to_cmd(vha, rsp, pkt->handle, pkt);
 	if (mcmd == NULL && h != QLA_TGT_SKIP_HANDLE) {
 		ql_dbg(ql_dbg_async, vha, 0xe064,
 		    "qla_target(%d): ABTS Comp without mcmd\n",
@@ -5883,6 +5805,7 @@ static void qlt_response_pkt(struct scsi_qla_host *vha,
 	case CTIO_TYPE7:
 	{
 		struct ctio7_from_24xx *entry = (struct ctio7_from_24xx *)pkt;
+
 		qlt_do_ctio_completion(vha, rsp, entry->handle,
 		    le16_to_cpu(entry->status)|(pkt->entry_status << 16),
 		    entry);
@@ -5893,6 +5816,7 @@ static void qlt_response_pkt(struct scsi_qla_host *vha,
 	{
 		struct atio_from_isp *atio = (struct atio_from_isp *)pkt;
 		int rc;
+
 		if (atio->u.isp2x.status !=
 		    cpu_to_le16(ATIO_CDB_VALID)) {
 			ql_dbg(ql_dbg_tgt, vha, 0xe05e,
@@ -5941,6 +5865,7 @@ static void qlt_response_pkt(struct scsi_qla_host *vha,
 	case CONTINUE_TGT_IO_TYPE:
 	{
 		struct ctio_to_2xxx *entry = (struct ctio_to_2xxx *)pkt;
+
 		qlt_do_ctio_completion(vha, rsp, entry->handle,
 		    le16_to_cpu(entry->status)|(pkt->entry_status << 16),
 		    entry);
@@ -5950,6 +5875,7 @@ static void qlt_response_pkt(struct scsi_qla_host *vha,
 	case CTIO_A64_TYPE:
 	{
 		struct ctio_to_2xxx *entry = (struct ctio_to_2xxx *)pkt;
+
 		qlt_do_ctio_completion(vha, rsp, entry->handle,
 		    le16_to_cpu(entry->status)|(pkt->entry_status << 16),
 		    entry);
@@ -5964,6 +5890,7 @@ static void qlt_response_pkt(struct scsi_qla_host *vha,
 	case NOTIFY_ACK_TYPE:
 		if (tgt->notify_ack_expected > 0) {
 			struct nack_to_isp *entry = (struct nack_to_isp *)pkt;
+
 			ql_dbg(ql_dbg_tgt, vha, 0xe036,
 			    "NOTIFY_ACK seq %08x status %x\n",
 			    le16_to_cpu(entry->u.isp2x.seq_id),
@@ -6239,6 +6166,7 @@ retry:
 
 		if (rc == -ENOENT) {
 			qlt_port_logo_t logo;
+
 			sid_to_portid(s_id, &logo.id);
 			logo.cmd_count = 1;
 			qlt_send_first_logo(vha, &logo);
@@ -6318,17 +6246,19 @@ static void qlt_abort_work(struct qla_tgt *tgt,
 	}
 
 	rc = __qlt_24xx_handle_abts(vha, &prm->abts, sess);
-	ha->tgt.tgt_ops->put_sess(sess);
 	spin_unlock_irqrestore(&ha->tgt.sess_lock, flags2);
 
+	ha->tgt.tgt_ops->put_sess(sess);
+
 	if (rc != 0)
 		goto out_term;
 	return;
 
 out_term2:
+	spin_unlock_irqrestore(&ha->tgt.sess_lock, flags2);
+
 	if (sess)
 		ha->tgt.tgt_ops->put_sess(sess);
-	spin_unlock_irqrestore(&ha->tgt.sess_lock, flags2);
 
 out_term:
 	spin_lock_irqsave(&ha->hardware_lock, flags);
@@ -6386,9 +6316,10 @@ static void qlt_tmr_work(struct qla_tgt *tgt,
 	    scsilun_to_int((struct scsi_lun *)&a->u.isp24.fcp_cmnd.lun);
 
 	rc = qlt_issue_task_mgmt(sess, unpacked_lun, fn, iocb, 0);
-	ha->tgt.tgt_ops->put_sess(sess);
 	spin_unlock_irqrestore(&ha->tgt.sess_lock, flags);
 
+	ha->tgt.tgt_ops->put_sess(sess);
+
 	if (rc != 0)
 		goto out_term;
 	return;
@@ -6499,6 +6430,7 @@ int qlt_add_target(struct qla_hw_data *ha, struct scsi_qla_host *base_vha)
 		unsigned long flags;
 
 		struct qla_qpair *qpair = ha->queue_pair_map[i];
+
 		h = &tgt->qphints[i + 1];
 		INIT_LIST_HEAD(&h->hint_elem);
 		if (qpair) {
@@ -6937,7 +6869,7 @@ qlt_24xx_config_rings(struct scsi_qla_host *vha)
 	RD_REG_DWORD(ISP_ATIO_Q_OUT(vha));
 
 	if (ha->flags.msix_enabled) {
-		if (IS_QLA83XX(ha) || IS_QLA27XX(ha)) {
+		if (IS_QLA83XX(ha) || IS_QLA27XX(ha) || IS_QLA28XX(ha)) {
 			if (IS_QLA2071(ha)) {
 				/* 4 ports Baker: Enable Interrupt Handshake */
 				icb->msix_atio = 0;
@@ -6952,7 +6884,7 @@ qlt_24xx_config_rings(struct scsi_qla_host *vha)
 		}
 	} else {
 		/* INTx|MSI */
-		if (IS_QLA83XX(ha) || IS_QLA27XX(ha)) {
+		if (IS_QLA83XX(ha) || IS_QLA27XX(ha) || IS_QLA28XX(ha)) {
 			icb->msix_atio = 0;
 			icb->firmware_options_2 |= BIT_26;
 			ql_dbg(ql_dbg_init, vha, 0xf072,
@@ -7201,7 +7133,8 @@ qlt_probe_one_stage1(struct scsi_qla_host *base_vha, struct qla_hw_data *ha)
 	if (!QLA_TGT_MODE_ENABLED())
 		return;
 
-	if  ((ql2xenablemsix == 0) || IS_QLA83XX(ha) || IS_QLA27XX(ha)) {
+	if  ((ql2xenablemsix == 0) || IS_QLA83XX(ha) || IS_QLA27XX(ha) ||
+	    IS_QLA28XX(ha)) {
 		ISP_ATIO_Q_IN(base_vha) = &ha->mqiobase->isp25mq.atio_q_in;
 		ISP_ATIO_Q_OUT(base_vha) = &ha->mqiobase->isp25mq.atio_q_out;
 	} else {
@@ -7329,7 +7262,10 @@ qlt_mem_free(struct qla_hw_data *ha)
 		    sizeof(struct atio_from_isp), ha->tgt.atio_ring,
 		    ha->tgt.atio_dma);
 	}
+	ha->tgt.atio_ring = NULL;
+	ha->tgt.atio_dma = 0;
 	kfree(ha->tgt.tgt_vp_map);
+	ha->tgt.tgt_vp_map = NULL;
 }
 
 /* vport_slock to be held by the caller */
@@ -7413,6 +7349,9 @@ int __init qlt_init(void)
 {
 	int ret;
 
+	BUILD_BUG_ON(sizeof(struct ctio7_to_24xx) != 64);
+	BUILD_BUG_ON(sizeof(struct ctio_to_2xxx) != 64);
+
 	if (!qlt_parse_ini_mode()) {
 		ql_log(ql_log_fatal, NULL, 0xe06b,
 		    "qlt_parse_ini_mode() failed\n");
diff --git a/drivers/scsi/qla2xxx/qla_target.h b/drivers/scsi/qla2xxx/qla_target.h
index f3de75000a08..89ceffa7d4fd 100644
--- a/drivers/scsi/qla2xxx/qla_target.h
+++ b/drivers/scsi/qla2xxx/qla_target.h
@@ -29,6 +29,7 @@
 #define __QLA_TARGET_H
 
 #include "qla_def.h"
+#include "qla_dsd.h"
 
 /*
  * Must be changed on any change in any initiator visible interfaces or
@@ -224,12 +225,7 @@ struct ctio_to_2xxx {
 	uint16_t reserved_1[3];
 	uint16_t scsi_status;
 	uint32_t transfer_length;
-	uint32_t dseg_0_address;	/* Data segment 0 address. */
-	uint32_t dseg_0_length;		/* Data segment 0 length. */
-	uint32_t dseg_1_address;	/* Data segment 1 address. */
-	uint32_t dseg_1_length;		/* Data segment 1 length. */
-	uint32_t dseg_2_address;	/* Data segment 2 address. */
-	uint32_t dseg_2_length;		/* Data segment 2 length. */
+	struct dsd32 dsd[3];
 } __packed;
 #define ATIO_PATH_INVALID       0x07
 #define ATIO_CANT_PROV_CAP      0x16
@@ -429,10 +425,7 @@ struct ctio7_to_24xx {
 			uint32_t reserved2;
 			uint32_t transfer_length;
 			uint32_t reserved3;
-			/* Data segment 0 address. */
-			uint32_t dseg_0_address[2];
-			/* Data segment 0 length. */
-			uint32_t dseg_0_length;
+			struct dsd64 dsd;
 		} status0;
 		struct {
 			uint16_t sense_length;
@@ -526,10 +519,10 @@ struct ctio_crc2_to_fw {
 	uint32_t reserved5;
 	__le32 transfer_length;		/* total fc transfer length */
 	uint32_t reserved6;
-	__le32 crc_context_address[2];/* Data segment address. */
+	__le64	 crc_context_address __packed; /* Data segment address. */
 	uint16_t crc_context_len;	/* Data segment length. */
 	uint16_t reserved_1;		/* MUST be set to 0. */
-} __packed;
+};
 
 /* CTIO Type CRC_x Status IOCB */
 struct ctio_crc_from_fw {
@@ -855,7 +848,7 @@ enum trace_flags {
 	TRC_CTIO_ERR = BIT_11,
 	TRC_CTIO_DONE = BIT_12,
 	TRC_CTIO_ABORTED =  BIT_13,
-	TRC_CTIO_STRANGE= BIT_14,
+	TRC_CTIO_STRANGE = BIT_14,
 	TRC_CMD_DONE = BIT_15,
 	TRC_CMD_CHK_STOP = BIT_16,
 	TRC_CMD_FREE = BIT_17,
@@ -889,10 +882,14 @@ struct qla_tgt_cmd {
 	unsigned int term_exchg:1;
 	unsigned int cmd_sent_to_fw:1;
 	unsigned int cmd_in_wq:1;
-	unsigned int aborted:1;
-	unsigned int data_work:1;
-	unsigned int data_work_free:1;
-	unsigned int released:1;
+
+	/*
+	 * This variable may be set from outside the LIO and I/O completion
+	 * callback functions. Do not declare this member variable as a
+	 * bitfield to avoid a read-modify-write operation when this variable
+	 * is set.
+	 */
+	unsigned int aborted;
 
 	struct scatterlist *sg;	/* cmd data buffer SG vector */
 	int sg_cnt;		/* SG segments count */
@@ -1103,7 +1100,5 @@ extern void qlt_do_generation_tick(struct scsi_qla_host *, int *);
 
 void qlt_send_resp_ctio(struct qla_qpair *, struct qla_tgt_cmd *, uint8_t,
     uint8_t, uint8_t, uint8_t);
-extern void qlt_abort_cmd_on_host_reset(struct scsi_qla_host *,
-    struct qla_tgt_cmd *);
 
 #endif /* __QLA_TARGET_H */
diff --git a/drivers/scsi/qla2xxx/qla_tmpl.c b/drivers/scsi/qla2xxx/qla_tmpl.c
index 9e52500caff0..de696a07532e 100644
--- a/drivers/scsi/qla2xxx/qla_tmpl.c
+++ b/drivers/scsi/qla2xxx/qla_tmpl.c
@@ -7,103 +7,9 @@
 #include "qla_def.h"
 #include "qla_tmpl.h"
 
-/* note default template is in big endian */
-static const uint32_t ql27xx_fwdt_default_template[] = {
-	0x63000000, 0xa4000000, 0x7c050000, 0x00000000,
-	0x30000000, 0x01000000, 0x00000000, 0xc0406eb4,
-	0x00000000, 0x00000000, 0x00000000, 0x00000000,
-	0x00000000, 0x00000000, 0x00000000, 0x00000000,
-	0x00000000, 0x00000000, 0x00000000, 0x00000000,
-	0x00000000, 0x00000000, 0x00000000, 0x00000000,
-	0x00000000, 0x00000000, 0x00000000, 0x00000000,
-	0x00000000, 0x00000000, 0x00000000, 0x00000000,
-	0x00000000, 0x00000000, 0x00000000, 0x00000000,
-	0x00000000, 0x00000000, 0x00000000, 0x00000000,
-	0x00000000, 0x04010000, 0x14000000, 0x00000000,
-	0x02000000, 0x44000000, 0x09010000, 0x10000000,
-	0x00000000, 0x02000000, 0x01010000, 0x1c000000,
-	0x00000000, 0x02000000, 0x00600000, 0x00000000,
-	0xc0000000, 0x01010000, 0x1c000000, 0x00000000,
-	0x02000000, 0x00600000, 0x00000000, 0xcc000000,
-	0x01010000, 0x1c000000, 0x00000000, 0x02000000,
-	0x10600000, 0x00000000, 0xd4000000, 0x01010000,
-	0x1c000000, 0x00000000, 0x02000000, 0x700f0000,
-	0x00000060, 0xf0000000, 0x00010000, 0x18000000,
-	0x00000000, 0x02000000, 0x00700000, 0x041000c0,
-	0x00010000, 0x18000000, 0x00000000, 0x02000000,
-	0x10700000, 0x041000c0, 0x00010000, 0x18000000,
-	0x00000000, 0x02000000, 0x40700000, 0x041000c0,
-	0x01010000, 0x1c000000, 0x00000000, 0x02000000,
-	0x007c0000, 0x01000000, 0xc0000000, 0x00010000,
-	0x18000000, 0x00000000, 0x02000000, 0x007c0000,
-	0x040300c4, 0x00010000, 0x18000000, 0x00000000,
-	0x02000000, 0x007c0000, 0x040100c0, 0x01010000,
-	0x1c000000, 0x00000000, 0x02000000, 0x007c0000,
-	0x00000000, 0xc0000000, 0x00010000, 0x18000000,
-	0x00000000, 0x02000000, 0x007c0000, 0x04200000,
-	0x0b010000, 0x18000000, 0x00000000, 0x02000000,
-	0x0c000000, 0x00000000, 0x02010000, 0x20000000,
-	0x00000000, 0x02000000, 0x700f0000, 0x040100fc,
-	0xf0000000, 0x000000b0, 0x02010000, 0x20000000,
-	0x00000000, 0x02000000, 0x700f0000, 0x040100fc,
-	0xf0000000, 0x000010b0, 0x02010000, 0x20000000,
-	0x00000000, 0x02000000, 0x700f0000, 0x040100fc,
-	0xf0000000, 0x000020b0, 0x02010000, 0x20000000,
-	0x00000000, 0x02000000, 0x700f0000, 0x040100fc,
-	0xf0000000, 0x000030b0, 0x02010000, 0x20000000,
-	0x00000000, 0x02000000, 0x700f0000, 0x040100fc,
-	0xf0000000, 0x000040b0, 0x02010000, 0x20000000,
-	0x00000000, 0x02000000, 0x700f0000, 0x040100fc,
-	0xf0000000, 0x000050b0, 0x02010000, 0x20000000,
-	0x00000000, 0x02000000, 0x700f0000, 0x040100fc,
-	0xf0000000, 0x000060b0, 0x02010000, 0x20000000,
-	0x00000000, 0x02000000, 0x700f0000, 0x040100fc,
-	0xf0000000, 0x000070b0, 0x02010000, 0x20000000,
-	0x00000000, 0x02000000, 0x700f0000, 0x040100fc,
-	0xf0000000, 0x000080b0, 0x02010000, 0x20000000,
-	0x00000000, 0x02000000, 0x700f0000, 0x040100fc,
-	0xf0000000, 0x000090b0, 0x02010000, 0x20000000,
-	0x00000000, 0x02000000, 0x700f0000, 0x040100fc,
-	0xf0000000, 0x0000a0b0, 0x00010000, 0x18000000,
-	0x00000000, 0x02000000, 0x0a000000, 0x040100c0,
-	0x00010000, 0x18000000, 0x00000000, 0x02000000,
-	0x0a000000, 0x04200080, 0x00010000, 0x18000000,
-	0x00000000, 0x02000000, 0x00be0000, 0x041000c0,
-	0x00010000, 0x18000000, 0x00000000, 0x02000000,
-	0x10be0000, 0x041000c0, 0x00010000, 0x18000000,
-	0x00000000, 0x02000000, 0x20be0000, 0x041000c0,
-	0x00010000, 0x18000000, 0x00000000, 0x02000000,
-	0x30be0000, 0x041000c0, 0x00010000, 0x18000000,
-	0x00000000, 0x02000000, 0x00b00000, 0x041000c0,
-	0x00010000, 0x18000000, 0x00000000, 0x02000000,
-	0x10b00000, 0x041000c0, 0x00010000, 0x18000000,
-	0x00000000, 0x02000000, 0x20b00000, 0x041000c0,
-	0x00010000, 0x18000000, 0x00000000, 0x02000000,
-	0x30b00000, 0x041000c0, 0x00010000, 0x18000000,
-	0x00000000, 0x02000000, 0x00300000, 0x041000c0,
-	0x00010000, 0x18000000, 0x00000000, 0x02000000,
-	0x10300000, 0x041000c0, 0x00010000, 0x18000000,
-	0x00000000, 0x02000000, 0x20300000, 0x041000c0,
-	0x00010000, 0x18000000, 0x00000000, 0x02000000,
-	0x30300000, 0x041000c0, 0x0a010000, 0x10000000,
-	0x00000000, 0x02000000, 0x06010000, 0x1c000000,
-	0x00000000, 0x02000000, 0x01000000, 0x00000200,
-	0xff230200, 0x06010000, 0x1c000000, 0x00000000,
-	0x02000000, 0x02000000, 0x00001000, 0x00000000,
-	0x07010000, 0x18000000, 0x00000000, 0x02000000,
-	0x00000000, 0x01000000, 0x07010000, 0x18000000,
-	0x00000000, 0x02000000, 0x00000000, 0x02000000,
-	0x07010000, 0x18000000, 0x00000000, 0x02000000,
-	0x00000000, 0x03000000, 0x0d010000, 0x14000000,
-	0x00000000, 0x02000000, 0x00000000, 0xff000000,
-	0x10000000, 0x00000000, 0x00000080,
-};
-
-static inline void __iomem *
-qla27xx_isp_reg(struct scsi_qla_host *vha)
-{
-	return &vha->hw->iobase->isp24;
-}
+#define ISPREG(vha)	(&(vha)->hw->iobase->isp24)
+#define IOBAR(reg)	offsetof(typeof(*(reg)), iobase_addr)
+#define IOBASE(vha)	IOBAR(ISPREG(vha))
 
 static inline void
 qla27xx_insert16(uint16_t value, void *buf, ulong *len)
@@ -128,7 +34,6 @@ qla27xx_insert32(uint32_t value, void *buf, ulong *len)
 static inline void
 qla27xx_insertbuf(void *mem, ulong size, void *buf, ulong *len)
 {
-
 	if (buf && mem && size) {
 		buf += *len;
 		memcpy(buf, mem, size);
@@ -190,9 +95,9 @@ static inline void
 qla27xx_write_reg(__iomem struct device_reg_24xx *reg,
 	uint offset, uint32_t data, void *buf)
 {
-	__iomem void *window = (void __iomem *)reg + offset;
-
 	if (buf) {
+		void __iomem *window = (void __iomem *)reg + offset;
+
 		WRT_REG_DWORD(window, data);
 	}
 }
@@ -205,7 +110,7 @@ qla27xx_read_window(__iomem struct device_reg_24xx *reg,
 	void __iomem *window = (void __iomem *)reg + offset;
 	void (*readn)(void __iomem*, void *, ulong *) = qla27xx_read_vector(width);
 
-	qla27xx_write_reg(reg, IOBASE_ADDR, addr, buf);
+	qla27xx_write_reg(reg, IOBAR(reg), addr, buf);
 	while (count--) {
 		qla27xx_insert32(addr, buf, len);
 		readn(window, buf, len);
@@ -224,7 +129,7 @@ qla27xx_skip_entry(struct qla27xx_fwdt_entry *ent, void *buf)
 static inline struct qla27xx_fwdt_entry *
 qla27xx_next_entry(struct qla27xx_fwdt_entry *ent)
 {
-	return (void *)ent + ent->hdr.size;
+	return (void *)ent + le32_to_cpu(ent->hdr.size);
 }
 
 static struct qla27xx_fwdt_entry *
@@ -254,12 +159,14 @@ static struct qla27xx_fwdt_entry *
 qla27xx_fwdt_entry_t256(struct scsi_qla_host *vha,
 	struct qla27xx_fwdt_entry *ent, void *buf, ulong *len)
 {
-	struct device_reg_24xx __iomem *reg = qla27xx_isp_reg(vha);
+	ulong addr = le32_to_cpu(ent->t256.base_addr);
+	uint offset = ent->t256.pci_offset;
+	ulong count = le16_to_cpu(ent->t256.reg_count);
+	uint width = ent->t256.reg_width;
 
 	ql_dbg(ql_dbg_misc, vha, 0xd200,
 	    "%s: rdio t1 [%lx]\n", __func__, *len);
-	qla27xx_read_window(reg, ent->t256.base_addr, ent->t256.pci_offset,
-	    ent->t256.reg_count, ent->t256.reg_width, buf, len);
+	qla27xx_read_window(ISPREG(vha), addr, offset, count, width, buf, len);
 
 	return qla27xx_next_entry(ent);
 }
@@ -268,12 +175,14 @@ static struct qla27xx_fwdt_entry *
 qla27xx_fwdt_entry_t257(struct scsi_qla_host *vha,
 	struct qla27xx_fwdt_entry *ent, void *buf, ulong *len)
 {
-	struct device_reg_24xx __iomem *reg = qla27xx_isp_reg(vha);
+	ulong addr = le32_to_cpu(ent->t257.base_addr);
+	uint offset = ent->t257.pci_offset;
+	ulong data = le32_to_cpu(ent->t257.write_data);
 
 	ql_dbg(ql_dbg_misc, vha, 0xd201,
 	    "%s: wrio t1 [%lx]\n", __func__, *len);
-	qla27xx_write_reg(reg, IOBASE_ADDR, ent->t257.base_addr, buf);
-	qla27xx_write_reg(reg, ent->t257.pci_offset, ent->t257.write_data, buf);
+	qla27xx_write_reg(ISPREG(vha), IOBASE(vha), addr, buf);
+	qla27xx_write_reg(ISPREG(vha), offset, data, buf);
 
 	return qla27xx_next_entry(ent);
 }
@@ -282,13 +191,17 @@ static struct qla27xx_fwdt_entry *
 qla27xx_fwdt_entry_t258(struct scsi_qla_host *vha,
 	struct qla27xx_fwdt_entry *ent, void *buf, ulong *len)
 {
-	struct device_reg_24xx __iomem *reg = qla27xx_isp_reg(vha);
+	uint banksel = ent->t258.banksel_offset;
+	ulong bank = le32_to_cpu(ent->t258.bank);
+	ulong addr = le32_to_cpu(ent->t258.base_addr);
+	uint offset = ent->t258.pci_offset;
+	uint count = le16_to_cpu(ent->t258.reg_count);
+	uint width = ent->t258.reg_width;
 
 	ql_dbg(ql_dbg_misc, vha, 0xd202,
 	    "%s: rdio t2 [%lx]\n", __func__, *len);
-	qla27xx_write_reg(reg, ent->t258.banksel_offset, ent->t258.bank, buf);
-	qla27xx_read_window(reg, ent->t258.base_addr, ent->t258.pci_offset,
-	    ent->t258.reg_count, ent->t258.reg_width, buf, len);
+	qla27xx_write_reg(ISPREG(vha), banksel, bank, buf);
+	qla27xx_read_window(ISPREG(vha), addr, offset, count, width, buf, len);
 
 	return qla27xx_next_entry(ent);
 }
@@ -297,13 +210,17 @@ static struct qla27xx_fwdt_entry *
 qla27xx_fwdt_entry_t259(struct scsi_qla_host *vha,
 	struct qla27xx_fwdt_entry *ent, void *buf, ulong *len)
 {
-	struct device_reg_24xx __iomem *reg = qla27xx_isp_reg(vha);
+	ulong addr = le32_to_cpu(ent->t259.base_addr);
+	uint banksel = ent->t259.banksel_offset;
+	ulong bank = le32_to_cpu(ent->t259.bank);
+	uint offset = ent->t259.pci_offset;
+	ulong data = le32_to_cpu(ent->t259.write_data);
 
 	ql_dbg(ql_dbg_misc, vha, 0xd203,
 	    "%s: wrio t2 [%lx]\n", __func__, *len);
-	qla27xx_write_reg(reg, IOBASE_ADDR, ent->t259.base_addr, buf);
-	qla27xx_write_reg(reg, ent->t259.banksel_offset, ent->t259.bank, buf);
-	qla27xx_write_reg(reg, ent->t259.pci_offset, ent->t259.write_data, buf);
+	qla27xx_write_reg(ISPREG(vha), IOBASE(vha), addr, buf);
+	qla27xx_write_reg(ISPREG(vha), banksel, bank, buf);
+	qla27xx_write_reg(ISPREG(vha), offset, data, buf);
 
 	return qla27xx_next_entry(ent);
 }
@@ -312,12 +229,12 @@ static struct qla27xx_fwdt_entry *
 qla27xx_fwdt_entry_t260(struct scsi_qla_host *vha,
 	struct qla27xx_fwdt_entry *ent, void *buf, ulong *len)
 {
-	struct device_reg_24xx __iomem *reg = qla27xx_isp_reg(vha);
+	uint offset = ent->t260.pci_offset;
 
 	ql_dbg(ql_dbg_misc, vha, 0xd204,
 	    "%s: rdpci [%lx]\n", __func__, *len);
-	qla27xx_insert32(ent->t260.pci_offset, buf, len);
-	qla27xx_read_reg(reg, ent->t260.pci_offset, buf, len);
+	qla27xx_insert32(offset, buf, len);
+	qla27xx_read_reg(ISPREG(vha), offset, buf, len);
 
 	return qla27xx_next_entry(ent);
 }
@@ -326,11 +243,12 @@ static struct qla27xx_fwdt_entry *
 qla27xx_fwdt_entry_t261(struct scsi_qla_host *vha,
 	struct qla27xx_fwdt_entry *ent, void *buf, ulong *len)
 {
-	struct device_reg_24xx __iomem *reg = qla27xx_isp_reg(vha);
+	uint offset = ent->t261.pci_offset;
+	ulong data = le32_to_cpu(ent->t261.write_data);
 
 	ql_dbg(ql_dbg_misc, vha, 0xd205,
 	    "%s: wrpci [%lx]\n", __func__, *len);
-	qla27xx_write_reg(reg, ent->t261.pci_offset, ent->t261.write_data, buf);
+	qla27xx_write_reg(ISPREG(vha), offset, data, buf);
 
 	return qla27xx_next_entry(ent);
 }
@@ -339,51 +257,50 @@ static struct qla27xx_fwdt_entry *
 qla27xx_fwdt_entry_t262(struct scsi_qla_host *vha,
 	struct qla27xx_fwdt_entry *ent, void *buf, ulong *len)
 {
+	uint area = ent->t262.ram_area;
+	ulong start = le32_to_cpu(ent->t262.start_addr);
+	ulong end = le32_to_cpu(ent->t262.end_addr);
 	ulong dwords;
-	ulong start;
-	ulong end;
 
 	ql_dbg(ql_dbg_misc, vha, 0xd206,
 	    "%s: rdram(%x) [%lx]\n", __func__, ent->t262.ram_area, *len);
-	start = ent->t262.start_addr;
-	end = ent->t262.end_addr;
 
-	if (ent->t262.ram_area == T262_RAM_AREA_CRITICAL_RAM) {
+	if (area == T262_RAM_AREA_CRITICAL_RAM) {
 		;
-	} else if (ent->t262.ram_area == T262_RAM_AREA_EXTERNAL_RAM) {
+	} else if (area == T262_RAM_AREA_EXTERNAL_RAM) {
 		end = vha->hw->fw_memory_size;
 		if (buf)
-			ent->t262.end_addr = end;
-	} else if (ent->t262.ram_area == T262_RAM_AREA_SHARED_RAM) {
+			ent->t262.end_addr = cpu_to_le32(end);
+	} else if (area == T262_RAM_AREA_SHARED_RAM) {
 		start = vha->hw->fw_shared_ram_start;
 		end = vha->hw->fw_shared_ram_end;
 		if (buf) {
-			ent->t262.start_addr = start;
-			ent->t262.end_addr = end;
+			ent->t262.start_addr = cpu_to_le32(start);
+			ent->t262.end_addr = cpu_to_le32(end);
 		}
-	} else if (ent->t262.ram_area == T262_RAM_AREA_DDR_RAM) {
+	} else if (area == T262_RAM_AREA_DDR_RAM) {
 		start = vha->hw->fw_ddr_ram_start;
 		end = vha->hw->fw_ddr_ram_end;
 		if (buf) {
-			ent->t262.start_addr = start;
-			ent->t262.end_addr = end;
+			ent->t262.start_addr = cpu_to_le32(start);
+			ent->t262.end_addr = cpu_to_le32(end);
 		}
-	} else if (ent->t262.ram_area == T262_RAM_AREA_MISC) {
+	} else if (area == T262_RAM_AREA_MISC) {
 		if (buf) {
-			ent->t262.start_addr = start;
-			ent->t262.end_addr = end;
+			ent->t262.start_addr = cpu_to_le32(start);
+			ent->t262.end_addr = cpu_to_le32(end);
 		}
 	} else {
 		ql_dbg(ql_dbg_misc, vha, 0xd022,
-		    "%s: unknown area %x\n", __func__, ent->t262.ram_area);
+		    "%s: unknown area %x\n", __func__, area);
 		qla27xx_skip_entry(ent, buf);
 		goto done;
 	}
 
 	if (end < start || start == 0 || end == 0) {
 		ql_dbg(ql_dbg_misc, vha, 0xd023,
-		    "%s: unusable range (start=%x end=%x)\n", __func__,
-		    ent->t262.end_addr, ent->t262.start_addr);
+		    "%s: unusable range (start=%lx end=%lx)\n",
+		    __func__, start, end);
 		qla27xx_skip_entry(ent, buf);
 		goto done;
 	}
@@ -402,13 +319,14 @@ static struct qla27xx_fwdt_entry *
 qla27xx_fwdt_entry_t263(struct scsi_qla_host *vha,
 	struct qla27xx_fwdt_entry *ent, void *buf, ulong *len)
 {
+	uint type = ent->t263.queue_type;
 	uint count = 0;
 	uint i;
 	uint length;
 
-	ql_dbg(ql_dbg_misc, vha, 0xd207,
-	    "%s: getq(%x) [%lx]\n", __func__, ent->t263.queue_type, *len);
-	if (ent->t263.queue_type == T263_QUEUE_TYPE_REQ) {
+	ql_dbg(ql_dbg_misc + ql_dbg_verbose, vha, 0xd207,
+	    "%s: getq(%x) [%lx]\n", __func__, type, *len);
+	if (type == T263_QUEUE_TYPE_REQ) {
 		for (i = 0; i < vha->hw->max_req_queues; i++) {
 			struct req_que *req = vha->hw->req_q_map[i];
 
@@ -422,7 +340,7 @@ qla27xx_fwdt_entry_t263(struct scsi_qla_host *vha,
 				count++;
 			}
 		}
-	} else if (ent->t263.queue_type == T263_QUEUE_TYPE_RSP) {
+	} else if (type == T263_QUEUE_TYPE_RSP) {
 		for (i = 0; i < vha->hw->max_rsp_queues; i++) {
 			struct rsp_que *rsp = vha->hw->rsp_q_map[i];
 
@@ -450,7 +368,7 @@ qla27xx_fwdt_entry_t263(struct scsi_qla_host *vha,
 		}
 	} else {
 		ql_dbg(ql_dbg_misc, vha, 0xd026,
-		    "%s: unknown queue %x\n", __func__, ent->t263.queue_type);
+		    "%s: unknown queue %x\n", __func__, type);
 		qla27xx_skip_entry(ent, buf);
 	}
 
@@ -496,12 +414,10 @@ static struct qla27xx_fwdt_entry *
 qla27xx_fwdt_entry_t265(struct scsi_qla_host *vha,
 	struct qla27xx_fwdt_entry *ent, void *buf, ulong *len)
 {
-	struct device_reg_24xx __iomem *reg = qla27xx_isp_reg(vha);
-
-	ql_dbg(ql_dbg_misc, vha, 0xd209,
+	ql_dbg(ql_dbg_misc + ql_dbg_verbose, vha, 0xd209,
 	    "%s: pause risc [%lx]\n", __func__, *len);
 	if (buf)
-		qla24xx_pause_risc(reg, vha->hw);
+		qla24xx_pause_risc(ISPREG(vha), vha->hw);
 
 	return qla27xx_next_entry(ent);
 }
@@ -522,11 +438,12 @@ static struct qla27xx_fwdt_entry *
 qla27xx_fwdt_entry_t267(struct scsi_qla_host *vha,
 	struct qla27xx_fwdt_entry *ent, void *buf, ulong *len)
 {
-	struct device_reg_24xx __iomem *reg = qla27xx_isp_reg(vha);
+	uint offset = ent->t267.pci_offset;
+	ulong data = le32_to_cpu(ent->t267.data);
 
 	ql_dbg(ql_dbg_misc, vha, 0xd20b,
 	    "%s: dis intr [%lx]\n", __func__, *len);
-	qla27xx_write_reg(reg, ent->t267.pci_offset, ent->t267.data, buf);
+	qla27xx_write_reg(ISPREG(vha), offset, data, buf);
 
 	return qla27xx_next_entry(ent);
 }
@@ -622,17 +539,16 @@ static struct qla27xx_fwdt_entry *
 qla27xx_fwdt_entry_t270(struct scsi_qla_host *vha,
 	struct qla27xx_fwdt_entry *ent, void *buf, ulong *len)
 {
-	struct device_reg_24xx __iomem *reg = qla27xx_isp_reg(vha);
-	ulong dwords = ent->t270.count;
-	ulong addr = ent->t270.addr;
+	ulong addr = le32_to_cpu(ent->t270.addr);
+	ulong dwords = le32_to_cpu(ent->t270.count);
 
 	ql_dbg(ql_dbg_misc, vha, 0xd20e,
 	    "%s: rdremreg [%lx]\n", __func__, *len);
-	qla27xx_write_reg(reg, IOBASE_ADDR, 0x40, buf);
+	qla27xx_write_reg(ISPREG(vha), IOBASE_ADDR, 0x40, buf);
 	while (dwords--) {
-		qla27xx_write_reg(reg, 0xc0, addr|0x80000000, buf);
+		qla27xx_write_reg(ISPREG(vha), 0xc0, addr|0x80000000, buf);
 		qla27xx_insert32(addr, buf, len);
-		qla27xx_read_reg(reg, 0xc4, buf, len);
+		qla27xx_read_reg(ISPREG(vha), 0xc4, buf, len);
 		addr += sizeof(uint32_t);
 	}
 
@@ -643,15 +559,14 @@ static struct qla27xx_fwdt_entry *
 qla27xx_fwdt_entry_t271(struct scsi_qla_host *vha,
 	struct qla27xx_fwdt_entry *ent, void *buf, ulong *len)
 {
-	struct device_reg_24xx __iomem *reg = qla27xx_isp_reg(vha);
-	ulong addr = ent->t271.addr;
-	ulong data = ent->t271.data;
+	ulong addr = le32_to_cpu(ent->t271.addr);
+	ulong data = le32_to_cpu(ent->t271.data);
 
 	ql_dbg(ql_dbg_misc, vha, 0xd20f,
 	    "%s: wrremreg [%lx]\n", __func__, *len);
-	qla27xx_write_reg(reg, IOBASE_ADDR, 0x40, buf);
-	qla27xx_write_reg(reg, 0xc4, data, buf);
-	qla27xx_write_reg(reg, 0xc0, addr, buf);
+	qla27xx_write_reg(ISPREG(vha), IOBASE(vha), 0x40, buf);
+	qla27xx_write_reg(ISPREG(vha), 0xc4, data, buf);
+	qla27xx_write_reg(ISPREG(vha), 0xc0, addr, buf);
 
 	return qla27xx_next_entry(ent);
 }
@@ -660,8 +575,8 @@ static struct qla27xx_fwdt_entry *
 qla27xx_fwdt_entry_t272(struct scsi_qla_host *vha,
 	struct qla27xx_fwdt_entry *ent, void *buf, ulong *len)
 {
-	ulong dwords = ent->t272.count;
-	ulong start = ent->t272.addr;
+	ulong dwords = le32_to_cpu(ent->t272.count);
+	ulong start = le32_to_cpu(ent->t272.addr);
 
 	ql_dbg(ql_dbg_misc, vha, 0xd210,
 	    "%s: rdremram [%lx]\n", __func__, *len);
@@ -680,8 +595,8 @@ static struct qla27xx_fwdt_entry *
 qla27xx_fwdt_entry_t273(struct scsi_qla_host *vha,
 	struct qla27xx_fwdt_entry *ent, void *buf, ulong *len)
 {
-	ulong dwords = ent->t273.count;
-	ulong addr = ent->t273.addr;
+	ulong dwords = le32_to_cpu(ent->t273.count);
+	ulong addr = le32_to_cpu(ent->t273.addr);
 	uint32_t value;
 
 	ql_dbg(ql_dbg_misc, vha, 0xd211,
@@ -703,12 +618,13 @@ static struct qla27xx_fwdt_entry *
 qla27xx_fwdt_entry_t274(struct scsi_qla_host *vha,
 	struct qla27xx_fwdt_entry *ent, void *buf, ulong *len)
 {
+	ulong type = ent->t274.queue_type;
 	uint count = 0;
 	uint i;
 
-	ql_dbg(ql_dbg_misc, vha, 0xd212,
-	    "%s: getqsh(%x) [%lx]\n", __func__, ent->t274.queue_type, *len);
-	if (ent->t274.queue_type == T274_QUEUE_TYPE_REQ_SHAD) {
+	ql_dbg(ql_dbg_misc + ql_dbg_verbose, vha, 0xd212,
+	    "%s: getqsh(%lx) [%lx]\n", __func__, type, *len);
+	if (type == T274_QUEUE_TYPE_REQ_SHAD) {
 		for (i = 0; i < vha->hw->max_req_queues; i++) {
 			struct req_que *req = vha->hw->req_q_map[i];
 
@@ -720,7 +636,7 @@ qla27xx_fwdt_entry_t274(struct scsi_qla_host *vha,
 				count++;
 			}
 		}
-	} else if (ent->t274.queue_type == T274_QUEUE_TYPE_RSP_SHAD) {
+	} else if (type == T274_QUEUE_TYPE_RSP_SHAD) {
 		for (i = 0; i < vha->hw->max_rsp_queues; i++) {
 			struct rsp_que *rsp = vha->hw->rsp_q_map[i];
 
@@ -746,7 +662,7 @@ qla27xx_fwdt_entry_t274(struct scsi_qla_host *vha,
 		}
 	} else {
 		ql_dbg(ql_dbg_misc, vha, 0xd02f,
-		    "%s: unknown queue %x\n", __func__, ent->t274.queue_type);
+		    "%s: unknown queue %lx\n", __func__, type);
 		qla27xx_skip_entry(ent, buf);
 	}
 
@@ -765,23 +681,26 @@ qla27xx_fwdt_entry_t275(struct scsi_qla_host *vha,
 	struct qla27xx_fwdt_entry *ent, void *buf, ulong *len)
 {
 	ulong offset = offsetof(typeof(*ent), t275.buffer);
+	ulong length = le32_to_cpu(ent->t275.length);
+	ulong size = le32_to_cpu(ent->hdr.size);
+	void *buffer = ent->t275.buffer;
 
-	ql_dbg(ql_dbg_misc, vha, 0xd213,
-	    "%s: buffer(%x) [%lx]\n", __func__, ent->t275.length, *len);
-	if (!ent->t275.length) {
+	ql_dbg(ql_dbg_misc + ql_dbg_verbose, vha, 0xd213,
+	    "%s: buffer(%lx) [%lx]\n", __func__, length, *len);
+	if (!length) {
 		ql_dbg(ql_dbg_misc, vha, 0xd020,
 		    "%s: buffer zero length\n", __func__);
 		qla27xx_skip_entry(ent, buf);
 		goto done;
 	}
-	if (offset + ent->t275.length > ent->hdr.size) {
+	if (offset + length > size) {
+		length = size - offset;
 		ql_dbg(ql_dbg_misc, vha, 0xd030,
-		    "%s: buffer overflow\n", __func__);
-		qla27xx_skip_entry(ent, buf);
-		goto done;
+		    "%s: buffer overflow, truncate [%lx]\n", __func__, length);
+		ent->t275.length = cpu_to_le32(length);
 	}
 
-	qla27xx_insertbuf(ent->t275.buffer, ent->t275.length, buf, len);
+	qla27xx_insertbuf(buffer, length, buf, len);
 done:
 	return qla27xx_next_entry(ent);
 }
@@ -790,15 +709,22 @@ static struct qla27xx_fwdt_entry *
 qla27xx_fwdt_entry_t276(struct scsi_qla_host *vha,
     struct qla27xx_fwdt_entry *ent, void *buf, ulong *len)
 {
-	uint type = vha->hw->pdev->device >> 4 & 0xf;
-	uint func = vha->hw->port_no & 0x3;
-
 	ql_dbg(ql_dbg_misc + ql_dbg_verbose, vha, 0xd214,
 	    "%s: cond [%lx]\n", __func__, *len);
 
-	if (type != ent->t276.cond1 || func != ent->t276.cond2) {
-		ent = qla27xx_next_entry(ent);
-		qla27xx_skip_entry(ent, buf);
+	if (buf) {
+		ulong cond1 = le32_to_cpu(ent->t276.cond1);
+		ulong cond2 = le32_to_cpu(ent->t276.cond2);
+		uint type = vha->hw->pdev->device >> 4 & 0xf;
+		uint func = vha->hw->port_no & 0x3;
+
+		if (type != cond1 || func != cond2) {
+			struct qla27xx_fwdt_template *tmp = buf;
+
+			tmp->count--;
+			ent = qla27xx_next_entry(ent);
+			qla27xx_skip_entry(ent, buf);
+		}
 	}
 
 	return qla27xx_next_entry(ent);
@@ -808,13 +734,15 @@ static struct qla27xx_fwdt_entry *
 qla27xx_fwdt_entry_t277(struct scsi_qla_host *vha,
     struct qla27xx_fwdt_entry *ent, void *buf, ulong *len)
 {
-	struct device_reg_24xx __iomem *reg = qla27xx_isp_reg(vha);
+	ulong cmd_addr = le32_to_cpu(ent->t277.cmd_addr);
+	ulong wr_cmd_data = le32_to_cpu(ent->t277.wr_cmd_data);
+	ulong data_addr = le32_to_cpu(ent->t277.data_addr);
 
 	ql_dbg(ql_dbg_misc + ql_dbg_verbose, vha, 0xd215,
 	    "%s: rdpep [%lx]\n", __func__, *len);
-	qla27xx_insert32(ent->t277.wr_cmd_data, buf, len);
-	qla27xx_write_reg(reg, ent->t277.cmd_addr, ent->t277.wr_cmd_data, buf);
-	qla27xx_read_reg(reg, ent->t277.data_addr, buf, len);
+	qla27xx_insert32(wr_cmd_data, buf, len);
+	qla27xx_write_reg(ISPREG(vha), cmd_addr, wr_cmd_data, buf);
+	qla27xx_read_reg(ISPREG(vha), data_addr, buf, len);
 
 	return qla27xx_next_entry(ent);
 }
@@ -823,12 +751,15 @@ static struct qla27xx_fwdt_entry *
 qla27xx_fwdt_entry_t278(struct scsi_qla_host *vha,
     struct qla27xx_fwdt_entry *ent, void *buf, ulong *len)
 {
-	struct device_reg_24xx __iomem *reg = qla27xx_isp_reg(vha);
+	ulong cmd_addr = le32_to_cpu(ent->t278.cmd_addr);
+	ulong wr_cmd_data = le32_to_cpu(ent->t278.wr_cmd_data);
+	ulong data_addr = le32_to_cpu(ent->t278.data_addr);
+	ulong wr_data = le32_to_cpu(ent->t278.wr_data);
 
 	ql_dbg(ql_dbg_misc + ql_dbg_verbose, vha, 0xd216,
 	    "%s: wrpep [%lx]\n", __func__, *len);
-	qla27xx_write_reg(reg, ent->t278.data_addr, ent->t278.wr_data, buf);
-	qla27xx_write_reg(reg, ent->t278.cmd_addr, ent->t278.wr_cmd_data, buf);
+	qla27xx_write_reg(ISPREG(vha), data_addr, wr_data, buf);
+	qla27xx_write_reg(ISPREG(vha), cmd_addr, wr_cmd_data, buf);
 
 	return qla27xx_next_entry(ent);
 }
@@ -837,8 +768,10 @@ static struct qla27xx_fwdt_entry *
 qla27xx_fwdt_entry_other(struct scsi_qla_host *vha,
 	struct qla27xx_fwdt_entry *ent, void *buf, ulong *len)
 {
+	ulong type = le32_to_cpu(ent->hdr.type);
+
 	ql_dbg(ql_dbg_misc, vha, 0xd2ff,
-	    "%s: type %x [%lx]\n", __func__, ent->hdr.type, *len);
+	    "%s: other %lx [%lx]\n", __func__, type, *len);
 	qla27xx_skip_entry(ent, buf);
 
 	return qla27xx_next_entry(ent);
@@ -893,36 +826,27 @@ static void
 qla27xx_walk_template(struct scsi_qla_host *vha,
 	struct qla27xx_fwdt_template *tmp, void *buf, ulong *len)
 {
-	struct qla27xx_fwdt_entry *ent = (void *)tmp + tmp->entry_offset;
-	ulong count = tmp->entry_count;
+	struct qla27xx_fwdt_entry *ent = (void *)tmp +
+	    le32_to_cpu(tmp->entry_offset);
+	ulong type;
 
+	tmp->count = le32_to_cpu(tmp->entry_count);
 	ql_dbg(ql_dbg_misc, vha, 0xd01a,
-	    "%s: entry count %lx\n", __func__, count);
-	while (count--) {
-		ent = qla27xx_find_entry(ent->hdr.type)(vha, ent, buf, len);
+	    "%s: entry count %u\n", __func__, tmp->count);
+	while (ent && tmp->count--) {
+		type = le32_to_cpu(ent->hdr.type);
+		ent = qla27xx_find_entry(type)(vha, ent, buf, len);
 		if (!ent)
 			break;
 	}
 
-	if (count)
+	if (tmp->count)
 		ql_dbg(ql_dbg_misc, vha, 0xd018,
-		    "%s: entry residual count (%lx)\n", __func__, count);
+		    "%s: entry count residual=+%u\n", __func__, tmp->count);
 
 	if (ent)
 		ql_dbg(ql_dbg_misc, vha, 0xd019,
-		    "%s: missing end entry (%lx)\n", __func__, count);
-
-	if (buf && *len != vha->hw->fw_dump_len)
-		ql_dbg(ql_dbg_misc, vha, 0xd01b,
-		    "%s: length=%#lx residual=%+ld\n",
-		    __func__, *len, vha->hw->fw_dump_len - *len);
-
-	if (buf) {
-		ql_log(ql_log_warn, vha, 0xd015,
-		    "Firmware dump saved to temp buffer (%lu/%p)\n",
-		    vha->host_no, vha->hw->fw_dump);
-		qla2x00_post_uevent_work(vha, QLA_UEVENT_CODE_FW_DUMP);
-	}
+		    "%s: missing end entry\n", __func__);
 }
 
 static void
@@ -945,8 +869,8 @@ qla27xx_driver_info(struct qla27xx_fwdt_template *tmp)
 }
 
 static void
-qla27xx_firmware_info(struct qla27xx_fwdt_template *tmp,
-	struct scsi_qla_host *vha)
+qla27xx_firmware_info(struct scsi_qla_host *vha,
+    struct qla27xx_fwdt_template *tmp)
 {
 	tmp->firmware_version[0] = vha->hw->fw_major_version;
 	tmp->firmware_version[1] = vha->hw->fw_minor_version;
@@ -963,19 +887,19 @@ ql27xx_edit_template(struct scsi_qla_host *vha,
 {
 	qla27xx_time_stamp(tmp);
 	qla27xx_driver_info(tmp);
-	qla27xx_firmware_info(tmp, vha);
+	qla27xx_firmware_info(vha, tmp);
 }
 
 static inline uint32_t
 qla27xx_template_checksum(void *p, ulong size)
 {
-	uint32_t *buf = p;
+	__le32 *buf = p;
 	uint64_t sum = 0;
 
 	size /= sizeof(*buf);
 
-	while (size--)
-		sum += *buf++;
+	for ( ; size--; buf++)
+		sum += le32_to_cpu(*buf);
 
 	sum = (sum & 0xffffffff) + (sum >> 32);
 
@@ -991,29 +915,29 @@ qla27xx_verify_template_checksum(struct qla27xx_fwdt_template *tmp)
 static inline int
 qla27xx_verify_template_header(struct qla27xx_fwdt_template *tmp)
 {
-	return tmp->template_type == TEMPLATE_TYPE_FWDUMP;
+	return le32_to_cpu(tmp->template_type) == TEMPLATE_TYPE_FWDUMP;
 }
 
-static void
-qla27xx_execute_fwdt_template(struct scsi_qla_host *vha)
+static ulong
+qla27xx_execute_fwdt_template(struct scsi_qla_host *vha,
+    struct qla27xx_fwdt_template *tmp, void *buf)
 {
-	struct qla27xx_fwdt_template *tmp = vha->hw->fw_dump_template;
-	ulong len;
+	ulong len = 0;
 
 	if (qla27xx_fwdt_template_valid(tmp)) {
 		len = tmp->template_size;
-		tmp = memcpy(vha->hw->fw_dump, tmp, len);
+		tmp = memcpy(buf, tmp, len);
 		ql27xx_edit_template(vha, tmp);
-		qla27xx_walk_template(vha, tmp, tmp, &len);
-		vha->hw->fw_dump_len = len;
-		vha->hw->fw_dumped = 1;
+		qla27xx_walk_template(vha, tmp, buf, &len);
 	}
+
+	return len;
 }
 
 ulong
-qla27xx_fwdt_calculate_dump_size(struct scsi_qla_host *vha)
+qla27xx_fwdt_calculate_dump_size(struct scsi_qla_host *vha, void *p)
 {
-	struct qla27xx_fwdt_template *tmp = vha->hw->fw_dump_template;
+	struct qla27xx_fwdt_template *tmp = p;
 	ulong len = 0;
 
 	if (qla27xx_fwdt_template_valid(tmp)) {
@@ -1032,18 +956,6 @@ qla27xx_fwdt_template_size(void *p)
 	return tmp->template_size;
 }
 
-ulong
-qla27xx_fwdt_template_default_size(void)
-{
-	return sizeof(ql27xx_fwdt_default_template);
-}
-
-const void *
-qla27xx_fwdt_template_default(void)
-{
-	return ql27xx_fwdt_default_template;
-}
-
 int
 qla27xx_fwdt_template_valid(void *p)
 {
@@ -1051,7 +963,8 @@ qla27xx_fwdt_template_valid(void *p)
 
 	if (!qla27xx_verify_template_header(tmp)) {
 		ql_log(ql_log_warn, NULL, 0xd01c,
-		    "%s: template type %x\n", __func__, tmp->template_type);
+		    "%s: template type %x\n", __func__,
+		    le32_to_cpu(tmp->template_type));
 		return false;
 	}
 
@@ -1074,17 +987,41 @@ qla27xx_fwdump(scsi_qla_host_t *vha, int hardware_locked)
 		spin_lock_irqsave(&vha->hw->hardware_lock, flags);
 #endif
 
-	if (!vha->hw->fw_dump)
-		ql_log(ql_log_warn, vha, 0xd01e, "fwdump buffer missing.\n");
-	else if (!vha->hw->fw_dump_template)
-		ql_log(ql_log_warn, vha, 0xd01f, "fwdump template missing.\n");
-	else if (vha->hw->fw_dumped)
-		ql_log(ql_log_warn, vha, 0xd300,
-		    "Firmware has been previously dumped (%p),"
-		    " -- ignoring request\n", vha->hw->fw_dump);
-	else {
-		QLA_FW_STOPPED(vha->hw);
-		qla27xx_execute_fwdt_template(vha);
+	if (!vha->hw->fw_dump) {
+		ql_log(ql_log_warn, vha, 0xd01e, "-> fwdump no buffer\n");
+	} else if (vha->hw->fw_dumped) {
+		ql_log(ql_log_warn, vha, 0xd01f,
+		    "-> Firmware already dumped (%p) -- ignoring request\n",
+		    vha->hw->fw_dump);
+	} else {
+		struct fwdt *fwdt = vha->hw->fwdt;
+		uint j;
+		ulong len;
+		void *buf = vha->hw->fw_dump;
+
+		for (j = 0; j < 2; j++, fwdt++, buf += len) {
+			ql_log(ql_log_warn, vha, 0xd011,
+			    "-> fwdt%u running...\n", j);
+			if (!fwdt->template) {
+				ql_log(ql_log_warn, vha, 0xd012,
+				    "-> fwdt%u no template\n", j);
+				break;
+			}
+			len = qla27xx_execute_fwdt_template(vha,
+			    fwdt->template, buf);
+			if (len != fwdt->dump_size) {
+				ql_log(ql_log_warn, vha, 0xd013,
+				    "-> fwdt%u fwdump residual=%+ld\n",
+				    j, fwdt->dump_size - len);
+			}
+		}
+		vha->hw->fw_dump_len = buf - (void *)vha->hw->fw_dump;
+		vha->hw->fw_dumped = 1;
+
+		ql_log(ql_log_warn, vha, 0xd015,
+		    "-> Firmware dump saved to buffer (%lu/%p) <%lx>\n",
+		    vha->host_no, vha->hw->fw_dump, vha->hw->fw_dump_cap_flags);
+		qla2x00_post_uevent_work(vha, QLA_UEVENT_CODE_FW_DUMP);
 	}
 
 #ifndef __CHECKER__
diff --git a/drivers/scsi/qla2xxx/qla_tmpl.h b/drivers/scsi/qla2xxx/qla_tmpl.h
index 5c2c2a8a19c4..d2a0014e8b21 100644
--- a/drivers/scsi/qla2xxx/qla_tmpl.h
+++ b/drivers/scsi/qla2xxx/qla_tmpl.h
@@ -11,12 +11,12 @@
 #define IOBASE_ADDR	offsetof(struct device_reg_24xx, iobase_addr)
 
 struct __packed qla27xx_fwdt_template {
-	uint32_t template_type;
-	uint32_t entry_offset;
+	__le32 template_type;
+	__le32 entry_offset;
 	uint32_t template_size;
-	uint32_t reserved_1;
+	uint32_t count;		/* borrow field for running/residual count */
 
-	uint32_t entry_count;
+	__le32 entry_count;
 	uint32_t template_version;
 	uint32_t capture_timestamp;
 	uint32_t template_checksum;
@@ -65,8 +65,8 @@ struct __packed qla27xx_fwdt_template {
 
 struct __packed qla27xx_fwdt_entry {
 	struct __packed {
-		uint32_t type;
-		uint32_t size;
+		__le32 type;
+		__le32 size;
 		uint32_t reserved_1;
 
 		uint8_t  capture_flags;
@@ -81,36 +81,36 @@ struct __packed qla27xx_fwdt_entry {
 		} t255;
 
 		struct __packed {
-			uint32_t base_addr;
+			__le32 base_addr;
 			uint8_t  reg_width;
-			uint16_t reg_count;
+			__le16 reg_count;
 			uint8_t  pci_offset;
 		} t256;
 
 		struct __packed {
-			uint32_t base_addr;
-			uint32_t write_data;
+			__le32 base_addr;
+			__le32 write_data;
 			uint8_t  pci_offset;
 			uint8_t  reserved[3];
 		} t257;
 
 		struct __packed {
-			uint32_t base_addr;
+			__le32 base_addr;
 			uint8_t  reg_width;
-			uint16_t reg_count;
+			__le16 reg_count;
 			uint8_t  pci_offset;
 			uint8_t  banksel_offset;
 			uint8_t  reserved[3];
-			uint32_t bank;
+			__le32 bank;
 		} t258;
 
 		struct __packed {
-			uint32_t base_addr;
-			uint32_t write_data;
+			__le32 base_addr;
+			__le32 write_data;
 			uint8_t  reserved[2];
 			uint8_t  pci_offset;
 			uint8_t  banksel_offset;
-			uint32_t bank;
+			__le32 bank;
 		} t259;
 
 		struct __packed {
@@ -121,14 +121,14 @@ struct __packed qla27xx_fwdt_entry {
 		struct __packed {
 			uint8_t pci_offset;
 			uint8_t reserved[3];
-			uint32_t write_data;
+			__le32 write_data;
 		} t261;
 
 		struct __packed {
 			uint8_t  ram_area;
 			uint8_t  reserved[3];
-			uint32_t start_addr;
-			uint32_t end_addr;
+			__le32 start_addr;
+			__le32 end_addr;
 		} t262;
 
 		struct __packed {
@@ -158,7 +158,7 @@ struct __packed qla27xx_fwdt_entry {
 		struct __packed {
 			uint8_t  pci_offset;
 			uint8_t  reserved[3];
-			uint32_t data;
+			__le32 data;
 		} t267;
 
 		struct __packed {
@@ -173,23 +173,23 @@ struct __packed qla27xx_fwdt_entry {
 		} t269;
 
 		struct __packed {
-			uint32_t addr;
-			uint32_t count;
+			__le32 addr;
+			__le32 count;
 		} t270;
 
 		struct __packed {
-			uint32_t addr;
-			uint32_t data;
+			__le32 addr;
+			__le32 data;
 		} t271;
 
 		struct __packed {
-			uint32_t addr;
-			uint32_t count;
+			__le32 addr;
+			__le32 count;
 		} t272;
 
 		struct __packed {
-			uint32_t addr;
-			uint32_t count;
+			__le32 addr;
+			__le32 count;
 		} t273;
 
 		struct __packed {
@@ -199,26 +199,26 @@ struct __packed qla27xx_fwdt_entry {
 		} t274;
 
 		struct __packed {
-			uint32_t length;
+			__le32 length;
 			uint8_t  buffer[];
 		} t275;
 
 		struct __packed {
-			uint32_t cond1;
-			uint32_t cond2;
+			__le32 cond1;
+			__le32 cond2;
 		} t276;
 
 		struct __packed {
-			uint32_t cmd_addr;
-			uint32_t wr_cmd_data;
-			uint32_t data_addr;
+			__le32 cmd_addr;
+			__le32 wr_cmd_data;
+			__le32 data_addr;
 		} t277;
 
 		struct __packed {
-			uint32_t cmd_addr;
-			uint32_t wr_cmd_data;
-			uint32_t data_addr;
-			uint32_t wr_data;
+			__le32 cmd_addr;
+			__le32 wr_cmd_data;
+			__le32 data_addr;
+			__le32 wr_data;
 		} t278;
 	};
 };
diff --git a/drivers/scsi/qla2xxx/qla_version.h b/drivers/scsi/qla2xxx/qla_version.h
index 0690dac24081..cd6bdf71e533 100644
--- a/drivers/scsi/qla2xxx/qla_version.h
+++ b/drivers/scsi/qla2xxx/qla_version.h
@@ -7,9 +7,9 @@
 /*
  * Driver version
  */
-#define QLA2XXX_VERSION      "10.00.00.14-k"
+#define QLA2XXX_VERSION      "10.01.00.16-k"
 
 #define QLA_DRIVER_MAJOR_VER	10
-#define QLA_DRIVER_MINOR_VER	0
+#define QLA_DRIVER_MINOR_VER	1
 #define QLA_DRIVER_PATCH_VER	0
 #define QLA_DRIVER_BETA_VER	0
diff --git a/drivers/scsi/qla2xxx/tcm_qla2xxx.c b/drivers/scsi/qla2xxx/tcm_qla2xxx.c
index 8a3075d17c63..ec9f1996b417 100644
--- a/drivers/scsi/qla2xxx/tcm_qla2xxx.c
+++ b/drivers/scsi/qla2xxx/tcm_qla2xxx.c
@@ -24,22 +24,16 @@
 
 
 #include <linux/module.h>
-#include <linux/moduleparam.h>
 #include <linux/utsname.h>
 #include <linux/vmalloc.h>
-#include <linux/init.h>
 #include <linux/list.h>
 #include <linux/slab.h>
-#include <linux/kthread.h>
 #include <linux/types.h>
 #include <linux/string.h>
 #include <linux/configfs.h>
 #include <linux/ctype.h>
 #include <asm/unaligned.h>
-#include <scsi/scsi.h>
 #include <scsi/scsi_host.h>
-#include <scsi/scsi_device.h>
-#include <scsi/scsi_cmnd.h>
 #include <target/target_core_base.h>
 #include <target/target_core_fabric.h>
 
@@ -267,25 +261,17 @@ static void tcm_qla2xxx_free_mcmd(struct qla_tgt_mgmt_cmd *mcmd)
 static void tcm_qla2xxx_complete_free(struct work_struct *work)
 {
 	struct qla_tgt_cmd *cmd = container_of(work, struct qla_tgt_cmd, work);
-	bool released = false;
-	unsigned long flags;
 
 	cmd->cmd_in_wq = 0;
 
 	WARN_ON(cmd->trc_flags & TRC_CMD_FREE);
 
-	spin_lock_irqsave(&cmd->cmd_lock, flags);
+	/* To do: protect all tgt_counters manipulations with proper locking. */
 	cmd->qpair->tgt_counters.qla_core_ret_sta_ctio++;
 	cmd->trc_flags |= TRC_CMD_FREE;
 	cmd->cmd_sent_to_fw = 0;
-	if (cmd->released)
-		released = true;
-	spin_unlock_irqrestore(&cmd->cmd_lock, flags);
 
-	if (released)
-		qlt_free_cmd(cmd);
-	else
-		transport_generic_free_cmd(&cmd->se_cmd, 0);
+	transport_generic_free_cmd(&cmd->se_cmd, 0);
 }
 
 /*
@@ -326,7 +312,6 @@ static int tcm_qla2xxx_check_stop_free(struct se_cmd *se_cmd)
 static void tcm_qla2xxx_release_cmd(struct se_cmd *se_cmd)
 {
 	struct qla_tgt_cmd *cmd;
-	unsigned long flags;
 
 	if (se_cmd->se_cmd_flags & SCF_SCSI_TMR_CDB) {
 		struct qla_tgt_mgmt_cmd *mcmd = container_of(se_cmd,
@@ -336,14 +321,10 @@ static void tcm_qla2xxx_release_cmd(struct se_cmd *se_cmd)
 	}
 	cmd = container_of(se_cmd, struct qla_tgt_cmd, se_cmd);
 
-	spin_lock_irqsave(&cmd->cmd_lock, flags);
-	if (cmd->cmd_sent_to_fw) {
-		cmd->released = 1;
-		spin_unlock_irqrestore(&cmd->cmd_lock, flags);
-	} else {
-		spin_unlock_irqrestore(&cmd->cmd_lock, flags);
-		qlt_free_cmd(cmd);
-	}
+	if (WARN_ON(cmd->cmd_sent_to_fw))
+		return;
+
+	qlt_free_cmd(cmd);
 }
 
 static void tcm_qla2xxx_release_session(struct kref *kref)
@@ -359,7 +340,6 @@ static void tcm_qla2xxx_put_sess(struct fc_port *sess)
 	if (!sess)
 		return;
 
-	assert_spin_locked(&sess->vha->hw->tgt.sess_lock);
 	kref_put(&sess->sess_kref, tcm_qla2xxx_release_session);
 }
 
@@ -374,8 +354,9 @@ static void tcm_qla2xxx_close_session(struct se_session *se_sess)
 
 	spin_lock_irqsave(&vha->hw->tgt.sess_lock, flags);
 	target_sess_cmd_list_set_waiting(se_sess);
-	tcm_qla2xxx_put_sess(sess);
 	spin_unlock_irqrestore(&vha->hw->tgt.sess_lock, flags);
+
+	tcm_qla2xxx_put_sess(sess);
 }
 
 static u32 tcm_qla2xxx_sess_get_index(struct se_session *se_sess)
@@ -399,6 +380,8 @@ static int tcm_qla2xxx_write_pending(struct se_cmd *se_cmd)
 			cmd->se_cmd.transport_state,
 			cmd->se_cmd.t_state,
 			cmd->se_cmd.se_cmd_flags);
+		transport_generic_request_failure(&cmd->se_cmd,
+			TCM_CHECK_CONDITION_ABORT_CMD);
 		return 0;
 	}
 	cmd->trc_flags |= TRC_XFR_RDY;
@@ -488,32 +471,18 @@ static int tcm_qla2xxx_handle_cmd(scsi_qla_host_t *vha, struct qla_tgt_cmd *cmd,
 static void tcm_qla2xxx_handle_data_work(struct work_struct *work)
 {
 	struct qla_tgt_cmd *cmd = container_of(work, struct qla_tgt_cmd, work);
-	unsigned long flags;
 
 	/*
 	 * Ensure that the complete FCP WRITE payload has been received.
 	 * Otherwise return an exception via CHECK_CONDITION status.
 	 */
 	cmd->cmd_in_wq = 0;
-
-	spin_lock_irqsave(&cmd->cmd_lock, flags);
 	cmd->cmd_sent_to_fw = 0;
-
-	if (cmd->released) {
-		spin_unlock_irqrestore(&cmd->cmd_lock, flags);
-		qlt_free_cmd(cmd);
-		return;
-	}
-
-	cmd->data_work = 1;
 	if (cmd->aborted) {
-		cmd->data_work_free = 1;
-		spin_unlock_irqrestore(&cmd->cmd_lock, flags);
-
-		tcm_qla2xxx_free_cmd(cmd);
+		transport_generic_request_failure(&cmd->se_cmd,
+			TCM_CHECK_CONDITION_ABORT_CMD);
 		return;
 	}
-	spin_unlock_irqrestore(&cmd->cmd_lock, flags);
 
 	cmd->qpair->tgt_counters.qla_core_ret_ctio++;
 	if (!cmd->write_data_transferred) {
@@ -829,7 +798,6 @@ static void tcm_qla2xxx_clear_nacl_from_fcport_map(struct fc_port *sess)
 
 static void tcm_qla2xxx_shutdown_sess(struct fc_port *sess)
 {
-	assert_spin_locked(&sess->vha->hw->tgt.sess_lock);
 	target_sess_cmd_list_set_waiting(sess->se_sess);
 }
 
@@ -1489,7 +1457,7 @@ static int tcm_qla2xxx_check_initiator_node_acl(
 	 */
 	tpg = lport->tpg_1;
 	if (!tpg) {
-		pr_err("Unable to lcoate struct tcm_qla2xxx_lport->tpg_1\n");
+		pr_err("Unable to locate struct tcm_qla2xxx_lport->tpg_1\n");
 		return -EINVAL;
 	}
 	/*
diff --git a/drivers/scsi/qla4xxx/ql4_os.c b/drivers/scsi/qla4xxx/ql4_os.c
index 6e4f4931ae17..8c674eca09f1 100644
--- a/drivers/scsi/qla4xxx/ql4_os.c
+++ b/drivers/scsi/qla4xxx/ql4_os.c
@@ -5930,7 +5930,7 @@ static int get_fw_boot_info(struct scsi_qla_host *ha, uint16_t ddb_index[])
 		val = rd_nvram_byte(ha, sec_addr);
 		if (val & BIT_7)
 			ddb_index[1] = (val & 0x7f);
-
+		goto exit_boot_info;
 	} else if (is_qla80XX(ha)) {
 		buf = dma_alloc_coherent(&ha->pdev->dev, size,
 					 &buf_dma, GFP_KERNEL);
diff --git a/drivers/scsi/qlogicfas408.c b/drivers/scsi/qlogicfas408.c
index 8b471a925b43..136681ad18a5 100644
--- a/drivers/scsi/qlogicfas408.c
+++ b/drivers/scsi/qlogicfas408.c
@@ -139,7 +139,7 @@ static int ql_pdma(struct qlogicfas408_priv *priv, int phase, char *request, int
 	} else {		/* out */
 #if QL_TURBO_PDMA
 		rtrc(4)
-		    if (reqlen >= 128 && inb(qbase + 8) & 0x10) {	/* empty */
+		if (reqlen >= 128 && inb(qbase + 8) & 0x10) {	/* empty */
 			outsl(qbase + 4, request, 32);
 			reqlen -= 128;
 			request += 128;
@@ -240,7 +240,7 @@ static void ql_icmd(struct scsi_cmnd *cmd)
 	outb(0x40 | qlcfg8 | priv->qinitid, qbase + 8);
 	outb(qlcfg7, qbase + 7);
 	outb(qlcfg6, qbase + 6);
-	 /**/ outb(qlcfg5, qbase + 5);	/* select timer */
+	outb(qlcfg5, qbase + 5);	/* select timer */
 	outb(qlcfg9 & 7, qbase + 9);	/* prescaler */
 /*	outb(0x99, qbase + 5);	*/
 	outb(scmd_id(cmd), qbase + 4);
diff --git a/drivers/scsi/scsi_error.c b/drivers/scsi/scsi_error.c
index 1b8378f36139..8e9680572b9f 100644
--- a/drivers/scsi/scsi_error.c
+++ b/drivers/scsi/scsi_error.c
@@ -2393,7 +2393,6 @@ out_put_autopm_host:
 	scsi_autopm_put_host(shost);
 	return error;
 }
-EXPORT_SYMBOL(scsi_ioctl_reset);
 
 bool scsi_command_normalize_sense(const struct scsi_cmnd *cmd,
 				  struct scsi_sense_hdr *sshdr)
diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c
index 07dfc17d4824..0916bd6d22b0 100644
--- a/drivers/scsi/scsi_lib.c
+++ b/drivers/scsi/scsi_lib.c
@@ -141,8 +141,6 @@ scsi_set_blocked(struct scsi_cmnd *cmd, int reason)
 
 static void scsi_mq_requeue_cmd(struct scsi_cmnd *cmd)
 {
-	struct scsi_device *sdev = cmd->device;
-
 	if (cmd->request->rq_flags & RQF_DONTPREP) {
 		cmd->request->rq_flags &= ~RQF_DONTPREP;
 		scsi_mq_uninit_cmd(cmd);
@@ -150,7 +148,6 @@ static void scsi_mq_requeue_cmd(struct scsi_cmnd *cmd)
 		WARN_ON_ONCE(true);
 	}
 	blk_mq_requeue_request(cmd->request, true);
-	put_device(&sdev->sdev_gendev);
 }
 
 /**
@@ -189,19 +186,7 @@ static void __scsi_queue_insert(struct scsi_cmnd *cmd, int reason, bool unbusy)
 	 */
 	cmd->result = 0;
 
-	/*
-	 * Before a SCSI command is dispatched,
-	 * get_device(&sdev->sdev_gendev) is called and the host,
-	 * target and device busy counters are increased. Since
-	 * requeuing a request causes these actions to be repeated and
-	 * since scsi_device_unbusy() has already been called,
-	 * put_device(&device->sdev_gendev) must still be called. Call
-	 * put_device() after blk_mq_requeue_request() to avoid that
-	 * removal of the SCSI device can start before requeueing has
-	 * happened.
-	 */
 	blk_mq_requeue_request(cmd->request, true);
-	put_device(&device->sdev_gendev);
 }
 
 /*
@@ -619,7 +604,6 @@ static bool scsi_end_request(struct request *req, blk_status_t error,
 		blk_mq_run_hw_queues(q, true);
 
 	percpu_ref_put(&q->q_usage_counter);
-	put_device(&sdev->sdev_gendev);
 	return false;
 }
 
@@ -1613,7 +1597,6 @@ static void scsi_mq_put_budget(struct blk_mq_hw_ctx *hctx)
 	struct scsi_device *sdev = q->queuedata;
 
 	atomic_dec(&sdev->device_busy);
-	put_device(&sdev->sdev_gendev);
 }
 
 static bool scsi_mq_get_budget(struct blk_mq_hw_ctx *hctx)
@@ -1621,16 +1604,9 @@ static bool scsi_mq_get_budget(struct blk_mq_hw_ctx *hctx)
 	struct request_queue *q = hctx->queue;
 	struct scsi_device *sdev = q->queuedata;
 
-	if (!get_device(&sdev->sdev_gendev))
-		goto out;
-	if (!scsi_dev_queue_ready(q, sdev))
-		goto out_put_device;
-
-	return true;
+	if (scsi_dev_queue_ready(q, sdev))
+		return true;
 
-out_put_device:
-	put_device(&sdev->sdev_gendev);
-out:
 	if (atomic_read(&sdev->device_busy) == 0 && !scsi_device_blocked(sdev))
 		blk_mq_delay_run_hw_queue(hctx, SCSI_QUEUE_DELAY);
 	return false;
@@ -1770,7 +1746,7 @@ static int scsi_map_queues(struct blk_mq_tag_set *set)
 
 	if (shost->hostt->map_queues)
 		return shost->hostt->map_queues(shost);
-	return blk_mq_map_queues(&set->map[0]);
+	return blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]);
 }
 
 void __scsi_init_queue(struct Scsi_Host *shost, struct request_queue *q)
diff --git a/drivers/scsi/scsi_scan.c b/drivers/scsi/scsi_scan.c
index 53380e07b40e..058079f915f1 100644
--- a/drivers/scsi/scsi_scan.c
+++ b/drivers/scsi/scsi_scan.c
@@ -1129,7 +1129,8 @@ static int scsi_probe_and_add_lun(struct scsi_target *starget,
 	 * that no LUN is present, so don't add sdev in these cases.
 	 * Two specific examples are:
 	 * 1) NetApp targets: return PQ=1, PDT=0x1f
-	 * 2) USB UFI: returns PDT=0x1f, with the PQ bits being "reserved"
+	 * 2) IBM/2145 targets: return PQ=1, PDT=0
+	 * 3) USB UFI: returns PDT=0x1f, with the PQ bits being "reserved"
 	 *    in the UFI 1.0 spec (we cannot rely on reserved bits).
 	 *
 	 * References:
@@ -1143,8 +1144,8 @@ static int scsi_probe_and_add_lun(struct scsi_target *starget,
 	 * PDT=00h Direct-access device (floppy)
 	 * PDT=1Fh none (no FDD connected to the requested logical unit)
 	 */
-	if (((result[0] >> 5) == 1 || starget->pdt_1f_for_no_lun) &&
-	    (result[0] & 0x1f) == 0x1f &&
+	if (((result[0] >> 5) == 1 ||
+	    (starget->pdt_1f_for_no_lun && (result[0] & 0x1f) == 0x1f)) &&
 	    !scsi_is_wlun(lun)) {
 		SCSI_LOG_SCAN_BUS(3, sdev_printk(KERN_INFO, sdev,
 					"scsi scan: peripheral device type"
diff --git a/drivers/scsi/scsi_transport_fc.c b/drivers/scsi/scsi_transport_fc.c
index d7035270d274..d9e3cf3721f6 100644
--- a/drivers/scsi/scsi_transport_fc.c
+++ b/drivers/scsi/scsi_transport_fc.c
@@ -147,6 +147,7 @@ static const struct {
 	{ FCH_EVT_PORT_OFFLINE,		"port_offline" },
 	{ FCH_EVT_PORT_FABRIC,		"port_fabric" },
 	{ FCH_EVT_LINK_UNKNOWN,		"link_unknown" },
+	{ FCH_EVT_LINK_FPIN,		"link_FPIN" },
 	{ FCH_EVT_VENDOR_UNIQUE,	"vendor_unique" },
 };
 fc_enum_name_search(host_event_code, fc_host_event_code,
@@ -295,6 +296,9 @@ static const struct {
 	{ FC_PORT_ROLE_FCP_INITIATOR,		"FCP Initiator" },
 	{ FC_PORT_ROLE_IP_PORT,			"IP Port" },
 	{ FC_PORT_ROLE_FCP_DUMMY_INITIATOR,	"FCP Dummy Initiator" },
+	{ FC_PORT_ROLE_NVME_INITIATOR,		"NVMe Initiator" },
+	{ FC_PORT_ROLE_NVME_TARGET,		"NVMe Target" },
+	{ FC_PORT_ROLE_NVME_DISCOVERY,		"NVMe Discovery" },
 };
 fc_bitfield_name_search(port_roles, fc_port_role_names)
 
@@ -523,20 +527,23 @@ fc_get_event_number(void)
 }
 EXPORT_SYMBOL(fc_get_event_number);
 
-
 /**
- * fc_host_post_event - called to post an even on an fc_host.
+ * fc_host_post_fc_event - routine to do the work of posting an event
+ *                      on an fc_host.
  * @shost:		host the event occurred on
  * @event_number:	fc event number obtained from get_fc_event_number()
  * @event_code:		fc_host event being posted
- * @event_data:		32bits of data for the event being posted
+ * @data_len:		amount, in bytes, of event data
+ * @data_buf:		pointer to event data
+ * @vendor_id:          value for Vendor id
  *
  * Notes:
  *	This routine assumes no locks are held on entry.
  */
 void
-fc_host_post_event(struct Scsi_Host *shost, u32 event_number,
-		enum fc_host_event_code event_code, u32 event_data)
+fc_host_post_fc_event(struct Scsi_Host *shost, u32 event_number,
+		enum fc_host_event_code event_code,
+		u32 data_len, char *data_buf, u64 vendor_id)
 {
 	struct sk_buff *skb;
 	struct nlmsghdr	*nlh;
@@ -545,12 +552,15 @@ fc_host_post_event(struct Scsi_Host *shost, u32 event_number,
 	u32 len;
 	int err;
 
+	if (!data_buf || data_len < 4)
+		data_len = 0;
+
 	if (!scsi_nl_sock) {
 		err = -ENOENT;
 		goto send_fail;
 	}
 
-	len = FC_NL_MSGALIGN(sizeof(*event));
+	len = FC_NL_MSGALIGN(sizeof(*event) + data_len);
 
 	skb = nlmsg_new(len, GFP_KERNEL);
 	if (!skb) {
@@ -568,12 +578,13 @@ fc_host_post_event(struct Scsi_Host *shost, u32 event_number,
 	INIT_SCSI_NL_HDR(&event->snlh, SCSI_NL_TRANSPORT_FC,
 				FC_NL_ASYNC_EVENT, len);
 	event->seconds = ktime_get_real_seconds();
-	event->vendor_id = 0;
+	event->vendor_id = vendor_id;
 	event->host_no = shost->host_no;
-	event->event_datalen = sizeof(u32);	/* bytes */
+	event->event_datalen = data_len;	/* bytes */
 	event->event_num = event_number;
 	event->event_code = event_code;
-	event->event_data = event_data;
+	if (data_len)
+		memcpy(&event->event_data, data_buf, data_len);
 
 	nlmsg_multicast(scsi_nl_sock, skb, 0, SCSI_NL_GRP_FC_EVENTS,
 			GFP_KERNEL);
@@ -586,14 +597,35 @@ send_fail:
 	printk(KERN_WARNING
 		"%s: Dropped Event : host %d %s data 0x%08x - err %d\n",
 		__func__, shost->host_no,
-		(name) ? name : "<unknown>", event_data, err);
+		(name) ? name : "<unknown>",
+		(data_len) ? *((u32 *)data_buf) : 0xFFFFFFFF, err);
 	return;
 }
+EXPORT_SYMBOL(fc_host_post_fc_event);
+
+/**
+ * fc_host_post_event - called to post an even on an fc_host.
+ * @shost:		host the event occurred on
+ * @event_number:	fc event number obtained from get_fc_event_number()
+ * @event_code:		fc_host event being posted
+ * @event_data:		32bits of data for the event being posted
+ *
+ * Notes:
+ *	This routine assumes no locks are held on entry.
+ */
+void
+fc_host_post_event(struct Scsi_Host *shost, u32 event_number,
+		enum fc_host_event_code event_code, u32 event_data)
+{
+	fc_host_post_fc_event(shost, event_number, event_code,
+		(u32)sizeof(u32), (char *)&event_data, 0);
+}
 EXPORT_SYMBOL(fc_host_post_event);
 
 
 /**
- * fc_host_post_vendor_event - called to post a vendor unique event on an fc_host
+ * fc_host_post_vendor_event - called to post a vendor unique event
+ *                      on an fc_host
  * @shost:		host the event occurred on
  * @event_number:	fc event number obtained from get_fc_event_number()
  * @data_len:		amount, in bytes, of vendor unique data
@@ -607,56 +639,27 @@ void
 fc_host_post_vendor_event(struct Scsi_Host *shost, u32 event_number,
 		u32 data_len, char * data_buf, u64 vendor_id)
 {
-	struct sk_buff *skb;
-	struct nlmsghdr	*nlh;
-	struct fc_nl_event *event;
-	u32 len;
-	int err;
-
-	if (!scsi_nl_sock) {
-		err = -ENOENT;
-		goto send_vendor_fail;
-	}
-
-	len = FC_NL_MSGALIGN(sizeof(*event) + data_len);
-
-	skb = nlmsg_new(len, GFP_KERNEL);
-	if (!skb) {
-		err = -ENOBUFS;
-		goto send_vendor_fail;
-	}
-
-	nlh = nlmsg_put(skb, 0, 0, SCSI_TRANSPORT_MSG, len, 0);
-	if (!nlh) {
-		err = -ENOBUFS;
-		goto send_vendor_fail_skb;
-	}
-	event = nlmsg_data(nlh);
-
-	INIT_SCSI_NL_HDR(&event->snlh, SCSI_NL_TRANSPORT_FC,
-				FC_NL_ASYNC_EVENT, len);
-	event->seconds = ktime_get_real_seconds();
-	event->vendor_id = vendor_id;
-	event->host_no = shost->host_no;
-	event->event_datalen = data_len;	/* bytes */
-	event->event_num = event_number;
-	event->event_code = FCH_EVT_VENDOR_UNIQUE;
-	memcpy(&event->event_data, data_buf, data_len);
-
-	nlmsg_multicast(scsi_nl_sock, skb, 0, SCSI_NL_GRP_FC_EVENTS,
-			GFP_KERNEL);
-	return;
-
-send_vendor_fail_skb:
-	kfree_skb(skb);
-send_vendor_fail:
-	printk(KERN_WARNING
-		"%s: Dropped Event : host %d vendor_unique - err %d\n",
-		__func__, shost->host_no, err);
-	return;
+	fc_host_post_fc_event(shost, event_number, FCH_EVT_VENDOR_UNIQUE,
+		data_len, data_buf, vendor_id);
 }
 EXPORT_SYMBOL(fc_host_post_vendor_event);
 
+/**
+ * fc_host_rcv_fpin - routine to process a received FPIN.
+ * @shost:		host the FPIN was received on
+ * @fpin_len:		length of FPIN payload, in bytes
+ * @fpin_buf:		pointer to FPIN payload
+ *
+ * Notes:
+ *	This routine assumes no locks are held on entry.
+ */
+void
+fc_host_fpin_rcv(struct Scsi_Host *shost, u32 fpin_len, char *fpin_buf)
+{
+	fc_host_post_fc_event(shost, fc_get_event_number(),
+				FCH_EVT_LINK_FPIN, fpin_len, fpin_buf, 0);
+}
+EXPORT_SYMBOL(fc_host_fpin_rcv);
 
 
 static __init int fc_transport_init(void)
diff --git a/drivers/scsi/smartpqi/Makefile b/drivers/scsi/smartpqi/Makefile
index a03a6edb0060..28985e508b5c 100644
--- a/drivers/scsi/smartpqi/Makefile
+++ b/drivers/scsi/smartpqi/Makefile
@@ -1,2 +1,3 @@
+# SPDX-License-Identifier: GPL-2.0
 obj-$(CONFIG_SCSI_SMARTPQI) += smartpqi.o
 smartpqi-objs := smartpqi_init.o smartpqi_sis.o smartpqi_sas_transport.o
diff --git a/drivers/scsi/smartpqi/smartpqi.h b/drivers/scsi/smartpqi/smartpqi.h
index af962368818b..e8e768849c70 100644
--- a/drivers/scsi/smartpqi/smartpqi.h
+++ b/drivers/scsi/smartpqi/smartpqi.h
@@ -1,18 +1,11 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  *    driver for Microsemi PQI-based storage controllers
- *    Copyright (c) 2016-2017 Microsemi Corporation
+ *    Copyright (c) 2019 Microchip Technology Inc. and its subsidiaries
+ *    Copyright (c) 2016-2018 Microsemi Corporation
  *    Copyright (c) 2016 PMC-Sierra, Inc.
  *
- *    This program is free software; you can redistribute it and/or modify
- *    it under the terms of the GNU General Public License as published by
- *    the Free Software Foundation; version 2 of the License.
- *
- *    This program is distributed in the hope that it will be useful,
- *    but WITHOUT ANY WARRANTY; without even the implied warranty of
- *    MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
- *    NON INFRINGEMENT.  See the GNU General Public License for more details.
- *
- *    Questions/Comments/Bugfixes to esc.storagedev@microsemi.com
+ *    Questions/Comments/Bugfixes to storagedev@microchip.com
  *
  */
 
diff --git a/drivers/scsi/smartpqi/smartpqi_init.c b/drivers/scsi/smartpqi/smartpqi_init.c
index 75ec43aa8df3..c26cac819f9e 100644
--- a/drivers/scsi/smartpqi/smartpqi_init.c
+++ b/drivers/scsi/smartpqi/smartpqi_init.c
@@ -1,18 +1,11 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  *    driver for Microsemi PQI-based storage controllers
- *    Copyright (c) 2016-2017 Microsemi Corporation
+ *    Copyright (c) 2019 Microchip Technology Inc. and its subsidiaries
+ *    Copyright (c) 2016-2018 Microsemi Corporation
  *    Copyright (c) 2016 PMC-Sierra, Inc.
  *
- *    This program is free software; you can redistribute it and/or modify
- *    it under the terms of the GNU General Public License as published by
- *    the Free Software Foundation; version 2 of the License.
- *
- *    This program is distributed in the hope that it will be useful,
- *    but WITHOUT ANY WARRANTY; without even the implied warranty of
- *    MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
- *    NON INFRINGEMENT.  See the GNU General Public License for more details.
- *
- *    Questions/Comments/Bugfixes to esc.storagedev@microsemi.com
+ *    Questions/Comments/Bugfixes to storagedev@microchip.com
  *
  */
 
@@ -40,11 +33,11 @@
 #define BUILD_TIMESTAMP
 #endif
 
-#define DRIVER_VERSION		"1.2.4-070"
+#define DRIVER_VERSION		"1.2.6-015"
 #define DRIVER_MAJOR		1
 #define DRIVER_MINOR		2
-#define DRIVER_RELEASE		4
-#define DRIVER_REVISION		70
+#define DRIVER_RELEASE		6
+#define DRIVER_REVISION		15
 
 #define DRIVER_NAME		"Microsemi PQI Driver (v" \
 				DRIVER_VERSION BUILD_TIMESTAMP ")"
@@ -5660,9 +5653,11 @@ static int pqi_lun_reset(struct pqi_ctrl_info *ctrl_info,
 	return rc;
 }
 
+/* Performs a reset at the LUN level. */
+
 #define PQI_LUN_RESET_RETRIES			3
 #define PQI_LUN_RESET_RETRY_INTERVAL_MSECS	10000
-/* Performs a reset at the LUN level. */
+#define PQI_LUN_RESET_PENDING_IO_TIMEOUT_SECS	120
 
 static int _pqi_device_reset(struct pqi_ctrl_info *ctrl_info,
 	struct pqi_scsi_dev *device)
@@ -5673,12 +5668,12 @@ static int _pqi_device_reset(struct pqi_ctrl_info *ctrl_info,
 
 	for (retries = 0;;) {
 		rc = pqi_lun_reset(ctrl_info, device);
-		if (rc != -EAGAIN ||
-		    ++retries > PQI_LUN_RESET_RETRIES)
+		if (rc != -EAGAIN || ++retries > PQI_LUN_RESET_RETRIES)
 			break;
 		msleep(PQI_LUN_RESET_RETRY_INTERVAL_MSECS);
 	}
-	timeout_secs = rc ? PQI_LUN_RESET_TIMEOUT_SECS : NO_TIMEOUT;
+
+	timeout_secs = rc ? PQI_LUN_RESET_PENDING_IO_TIMEOUT_SECS : NO_TIMEOUT;
 
 	rc |= pqi_device_wait_for_pending_io(ctrl_info, device, timeout_secs);
 
@@ -5707,6 +5702,7 @@ static int pqi_device_reset(struct pqi_ctrl_info *ctrl_info,
 	pqi_device_reset_done(device);
 
 	mutex_unlock(&ctrl_info->lun_reset_mutex);
+
 	return rc;
 }
 
@@ -5737,6 +5733,7 @@ static int pqi_eh_device_reset_handler(struct scsi_cmnd *scmd)
 	pqi_wait_until_ofa_finished(ctrl_info);
 
 	rc = pqi_device_reset(ctrl_info, device);
+
 out:
 	dev_err(&ctrl_info->pci_dev->dev,
 		"reset of scsi %d:%d:%d:%d: %s\n",
@@ -5795,7 +5792,7 @@ static int pqi_map_queues(struct Scsi_Host *shost)
 {
 	struct pqi_ctrl_info *ctrl_info = shost_to_hba(shost);
 
-	return blk_mq_pci_map_queues(&shost->tag_set.map[0],
+	return blk_mq_pci_map_queues(&shost->tag_set.map[HCTX_TYPE_DEFAULT],
 					ctrl_info->pci_dev, 0);
 }
 
@@ -7948,6 +7945,22 @@ static const struct pci_device_id pqi_pci_id_table[] = {
 	},
 	{
 		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       0x193d, 0x1104)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       0x193d, 0x1105)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       0x193d, 0x1106)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
+			       0x193d, 0x1107)
+	},
+	{
+		PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
 			       0x193d, 0x8460)
 	},
 	{
diff --git a/drivers/scsi/smartpqi/smartpqi_sas_transport.c b/drivers/scsi/smartpqi/smartpqi_sas_transport.c
index 0e4ef215115f..5cca1b9ef1f1 100644
--- a/drivers/scsi/smartpqi/smartpqi_sas_transport.c
+++ b/drivers/scsi/smartpqi/smartpqi_sas_transport.c
@@ -1,18 +1,11 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  *    driver for Microsemi PQI-based storage controllers
- *    Copyright (c) 2016-2017 Microsemi Corporation
+ *    Copyright (c) 2019 Microchip Technology Inc. and its subsidiaries
+ *    Copyright (c) 2016-2018 Microsemi Corporation
  *    Copyright (c) 2016 PMC-Sierra, Inc.
  *
- *    This program is free software; you can redistribute it and/or modify
- *    it under the terms of the GNU General Public License as published by
- *    the Free Software Foundation; version 2 of the License.
- *
- *    This program is distributed in the hope that it will be useful,
- *    but WITHOUT ANY WARRANTY; without even the implied warranty of
- *    MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
- *    NON INFRINGEMENT.  See the GNU General Public License for more details.
- *
- *    Questions/Comments/Bugfixes to esc.storagedev@microsemi.com
+ *    Questions/Comments/Bugfixes to storagedev@microchip.com
  *
  */
 
diff --git a/drivers/scsi/smartpqi/smartpqi_sis.c b/drivers/scsi/smartpqi/smartpqi_sis.c
index dcd11c6418cc..f0d6e88ba2c1 100644
--- a/drivers/scsi/smartpqi/smartpqi_sis.c
+++ b/drivers/scsi/smartpqi/smartpqi_sis.c
@@ -1,18 +1,11 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  *    driver for Microsemi PQI-based storage controllers
- *    Copyright (c) 2016-2017 Microsemi Corporation
+ *    Copyright (c) 2019 Microchip Technology Inc. and its subsidiaries
+ *    Copyright (c) 2016-2018 Microsemi Corporation
  *    Copyright (c) 2016 PMC-Sierra, Inc.
  *
- *    This program is free software; you can redistribute it and/or modify
- *    it under the terms of the GNU General Public License as published by
- *    the Free Software Foundation; version 2 of the License.
- *
- *    This program is distributed in the hope that it will be useful,
- *    but WITHOUT ANY WARRANTY; without even the implied warranty of
- *    MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
- *    NON INFRINGEMENT.  See the GNU General Public License for more details.
- *
- *    Questions/Comments/Bugfixes to esc.storagedev@microsemi.com
+ *    Questions/Comments/Bugfixes to storagedev@microchip.com
  *
  */
 
diff --git a/drivers/scsi/smartpqi/smartpqi_sis.h b/drivers/scsi/smartpqi/smartpqi_sis.h
index d018cb9c3f82..86b0e484d921 100644
--- a/drivers/scsi/smartpqi/smartpqi_sis.h
+++ b/drivers/scsi/smartpqi/smartpqi_sis.h
@@ -1,18 +1,11 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  *    driver for Microsemi PQI-based storage controllers
- *    Copyright (c) 2016-2017 Microsemi Corporation
+ *    Copyright (c) 2019 Microchip Technology Inc. and its subsidiaries
+ *    Copyright (c) 2016-2018 Microsemi Corporation
  *    Copyright (c) 2016 PMC-Sierra, Inc.
  *
- *    This program is free software; you can redistribute it and/or modify
- *    it under the terms of the GNU General Public License as published by
- *    the Free Software Foundation; version 2 of the License.
- *
- *    This program is distributed in the hope that it will be useful,
- *    but WITHOUT ANY WARRANTY; without even the implied warranty of
- *    MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
- *    NON INFRINGEMENT.  See the GNU General Public License for more details.
- *
- *    Questions/Comments/Bugfixes to esc.storagedev@microsemi.com
+ *    Questions/Comments/Bugfixes to storagedev@microchip.com
  *
  */
 
diff --git a/drivers/scsi/ufs/Kconfig b/drivers/scsi/ufs/Kconfig
index 6db37cf306b0..0b845ab7c3bf 100644
--- a/drivers/scsi/ufs/Kconfig
+++ b/drivers/scsi/ufs/Kconfig
@@ -99,6 +99,7 @@ config SCSI_UFS_DWC_TC_PLATFORM
 config SCSI_UFS_QCOM
 	tristate "QCOM specific hooks to UFS controller platform driver"
 	depends on SCSI_UFSHCD_PLATFORM && ARCH_QCOM
+	select RESET_CONTROLLER
 	help
 	  This selects the QCOM specific additions to UFSHCD platform driver.
 	  UFS host on QCOM needs some vendor specific configuration before
@@ -108,6 +109,20 @@ config SCSI_UFS_QCOM
 	  Select this if you have UFS controller on QCOM chipset.
 	  If unsure, say N.
 
+config SCSI_UFS_MEDIATEK
+	tristate "Mediatek specific hooks to UFS controller platform driver"
+	depends on SCSI_UFSHCD_PLATFORM && ARCH_MEDIATEK
+	select PHY_MTK_UFS
+	help
+	  This selects the Mediatek specific additions to UFSHCD platform driver.
+	  UFS host on Mediatek needs some vendor specific configuration before
+	  accessing the hardware which includes PHY configuration and vendor
+	  specific registers.
+
+	  Select this if you have UFS controller on Mediatek chipset.
+
+	  If unsure, say N.
+
 config SCSI_UFS_HISI
 	tristate "Hisilicon specific hooks to UFS controller platform driver"
 	depends on (ARCH_HISI || COMPILE_TEST) && SCSI_UFSHCD_PLATFORM
diff --git a/drivers/scsi/ufs/Makefile b/drivers/scsi/ufs/Makefile
index a3bd70c3652c..2a9097939bcb 100644
--- a/drivers/scsi/ufs/Makefile
+++ b/drivers/scsi/ufs/Makefile
@@ -10,3 +10,4 @@ ufshcd-core-$(CONFIG_SCSI_UFS_BSG)	+= ufs_bsg.o
 obj-$(CONFIG_SCSI_UFSHCD_PCI) += ufshcd-pci.o
 obj-$(CONFIG_SCSI_UFSHCD_PLATFORM) += ufshcd-pltfrm.o
 obj-$(CONFIG_SCSI_UFS_HISI) += ufs-hisi.o
+obj-$(CONFIG_SCSI_UFS_MEDIATEK) += ufs-mediatek.o
diff --git a/drivers/scsi/ufs/cdns-pltfrm.c b/drivers/scsi/ufs/cdns-pltfrm.c
index 4a37b4f57164..86dbb723f3ac 100644
--- a/drivers/scsi/ufs/cdns-pltfrm.c
+++ b/drivers/scsi/ufs/cdns-pltfrm.c
@@ -17,7 +17,8 @@
 
 #include "ufshcd-pltfrm.h"
 
-#define CDNS_UFS_REG_HCLKDIV 0xFC
+#define CDNS_UFS_REG_HCLKDIV	0xFC
+#define CDNS_UFS_REG_PHY_XCFGD1	0x113C
 
 /**
  * Sets HCLKDIV register value based on the core_clk
@@ -77,11 +78,66 @@ static int cdns_ufs_setup_clocks(struct ufs_hba *hba, bool on,
 	return cdns_ufs_set_hclkdiv(hba);
 }
 
-static struct ufs_hba_variant_ops cdns_pltfm_hba_vops = {
+/**
+ * cdns_ufs_init - performs additional ufs initialization
+ * @hba: host controller instance
+ *
+ * Returns status of initialization
+ */
+static int cdns_ufs_init(struct ufs_hba *hba)
+{
+	int status = 0;
+
+	if (hba->vops && hba->vops->phy_initialization)
+		status = hba->vops->phy_initialization(hba);
+
+	return status;
+}
+
+/**
+ * cdns_ufs_m31_16nm_phy_initialization - performs m31 phy initialization
+ * @hba: host controller instance
+ *
+ * Always returns 0
+ */
+static int cdns_ufs_m31_16nm_phy_initialization(struct ufs_hba *hba)
+{
+	u32 data;
+
+	/* Increase RX_Advanced_Min_ActivateTime_Capability */
+	data = ufshcd_readl(hba, CDNS_UFS_REG_PHY_XCFGD1);
+	data |= BIT(24);
+	ufshcd_writel(hba, data, CDNS_UFS_REG_PHY_XCFGD1);
+
+	return 0;
+}
+
+static const struct ufs_hba_variant_ops cdns_ufs_pltfm_hba_vops = {
+	.name = "cdns-ufs-pltfm",
+	.setup_clocks = cdns_ufs_setup_clocks,
+};
+
+static const struct ufs_hba_variant_ops cdns_ufs_m31_16nm_pltfm_hba_vops = {
 	.name = "cdns-ufs-pltfm",
+	.init = cdns_ufs_init,
 	.setup_clocks = cdns_ufs_setup_clocks,
+	.phy_initialization = cdns_ufs_m31_16nm_phy_initialization,
+};
+
+static const struct of_device_id cdns_ufs_of_match[] = {
+	{
+		.compatible = "cdns,ufshc",
+		.data =  &cdns_ufs_pltfm_hba_vops,
+	},
+	{
+		.compatible = "cdns,ufshc-m31-16nm",
+		.data =  &cdns_ufs_m31_16nm_pltfm_hba_vops,
+	},
+	{ },
 };
 
+MODULE_DEVICE_TABLE(of, cdns_ufs_of_match);
+
 /**
  * cdns_ufs_pltfrm_probe - probe routine of the driver
  * @pdev: pointer to platform device handle
@@ -91,10 +147,15 @@ static struct ufs_hba_variant_ops cdns_pltfm_hba_vops = {
 static int cdns_ufs_pltfrm_probe(struct platform_device *pdev)
 {
 	int err;
+	const struct of_device_id *of_id;
+	struct ufs_hba_variant_ops *vops;
 	struct device *dev = &pdev->dev;
 
+	of_id = of_match_node(cdns_ufs_of_match, dev->of_node);
+	vops = (struct ufs_hba_variant_ops *)of_id->data;
+
 	/* Perform generic probe */
-	err = ufshcd_pltfrm_init(pdev, &cdns_pltfm_hba_vops);
+	err = ufshcd_pltfrm_init(pdev, vops);
 	if (err)
 		dev_err(dev, "ufshcd_pltfrm_init() failed %d\n", err);
 
@@ -115,13 +176,6 @@ static int cdns_ufs_pltfrm_remove(struct platform_device *pdev)
 	return 0;
 }
 
-static const struct of_device_id cdns_ufs_of_match[] = {
-	{ .compatible = "cdns,ufshc" },
-	{},
-};
-
-MODULE_DEVICE_TABLE(of, cdns_ufs_of_match);
-
 static const struct dev_pm_ops cdns_ufs_dev_pm_ops = {
 	.suspend         = ufshcd_pltfrm_suspend,
 	.resume          = ufshcd_pltfrm_resume,
diff --git a/drivers/scsi/ufs/ufs-hisi.c b/drivers/scsi/ufs/ufs-hisi.c
index 0e855b5afe82..7aed0a1a794e 100644
--- a/drivers/scsi/ufs/ufs-hisi.c
+++ b/drivers/scsi/ufs/ufs-hisi.c
@@ -293,108 +293,7 @@ static int ufs_hisi_link_startup_notify(struct ufs_hba *hba,
 	return err;
 }
 
-struct ufs_hisi_dev_params {
-	u32 pwm_rx_gear; /* pwm rx gear to work in */
-	u32 pwm_tx_gear; /* pwm tx gear to work in */
-	u32 hs_rx_gear;  /* hs rx gear to work in */
-	u32 hs_tx_gear;  /* hs tx gear to work in */
-	u32 rx_lanes;    /* number of rx lanes */
-	u32 tx_lanes;    /* number of tx lanes */
-	u32 rx_pwr_pwm;  /* rx pwm working pwr */
-	u32 tx_pwr_pwm;  /* tx pwm working pwr */
-	u32 rx_pwr_hs;   /* rx hs working pwr */
-	u32 tx_pwr_hs;   /* tx hs working pwr */
-	u32 hs_rate;     /* rate A/B to work in HS */
-	u32 desired_working_mode;
-};
-
-static int ufs_hisi_get_pwr_dev_param(
-				    struct ufs_hisi_dev_params *hisi_param,
-				    struct ufs_pa_layer_attr *dev_max,
-				    struct ufs_pa_layer_attr *agreed_pwr)
-{
-	int min_hisi_gear;
-	int min_dev_gear;
-	bool is_dev_sup_hs = false;
-	bool is_hisi_max_hs = false;
-
-	if (dev_max->pwr_rx == FASTAUTO_MODE || dev_max->pwr_rx == FAST_MODE)
-		is_dev_sup_hs = true;
-
-	if (hisi_param->desired_working_mode == FAST) {
-		is_hisi_max_hs = true;
-		min_hisi_gear = min_t(u32, hisi_param->hs_rx_gear,
-				       hisi_param->hs_tx_gear);
-	} else {
-		min_hisi_gear = min_t(u32, hisi_param->pwm_rx_gear,
-				       hisi_param->pwm_tx_gear);
-	}
-
-	/*
-	 * device doesn't support HS but
-	 * hisi_param->desired_working_mode is HS,
-	 * thus device and hisi_param don't agree
-	 */
-	if (!is_dev_sup_hs && is_hisi_max_hs) {
-		pr_err("%s: device not support HS\n", __func__);
-		return -ENOTSUPP;
-	} else if (is_dev_sup_hs && is_hisi_max_hs) {
-		/*
-		 * since device supports HS, it supports FAST_MODE.
-		 * since hisi_param->desired_working_mode is also HS
-		 * then final decision (FAST/FASTAUTO) is done according
-		 * to hisi_params as it is the restricting factor
-		 */
-		agreed_pwr->pwr_rx = agreed_pwr->pwr_tx =
-			hisi_param->rx_pwr_hs;
-	} else {
-		/*
-		 * here hisi_param->desired_working_mode is PWM.
-		 * it doesn't matter whether device supports HS or PWM,
-		 * in both cases hisi_param->desired_working_mode will
-		 * determine the mode
-		 */
-		agreed_pwr->pwr_rx = agreed_pwr->pwr_tx =
-			hisi_param->rx_pwr_pwm;
-	}
-
-	/*
-	 * we would like tx to work in the minimum number of lanes
-	 * between device capability and vendor preferences.
-	 * the same decision will be made for rx
-	 */
-	agreed_pwr->lane_tx =
-		min_t(u32, dev_max->lane_tx, hisi_param->tx_lanes);
-	agreed_pwr->lane_rx =
-		min_t(u32, dev_max->lane_rx, hisi_param->rx_lanes);
-
-	/* device maximum gear is the minimum between device rx and tx gears */
-	min_dev_gear = min_t(u32, dev_max->gear_rx, dev_max->gear_tx);
-
-	/*
-	 * if both device capabilities and vendor pre-defined preferences are
-	 * both HS or both PWM then set the minimum gear to be the chosen
-	 * working gear.
-	 * if one is PWM and one is HS then the one that is PWM get to decide
-	 * what is the gear, as it is the one that also decided previously what
-	 * pwr the device will be configured to.
-	 */
-	if ((is_dev_sup_hs && is_hisi_max_hs) ||
-	    (!is_dev_sup_hs && !is_hisi_max_hs))
-		agreed_pwr->gear_rx = agreed_pwr->gear_tx =
-			min_t(u32, min_dev_gear, min_hisi_gear);
-	else
-		agreed_pwr->gear_rx = agreed_pwr->gear_tx = min_hisi_gear;
-
-	agreed_pwr->hs_rate = hisi_param->hs_rate;
-
-	pr_info("ufs final power mode: gear = %d, lane = %d, pwr = %d, rate = %d\n",
-		agreed_pwr->gear_rx, agreed_pwr->lane_rx, agreed_pwr->pwr_rx,
-		agreed_pwr->hs_rate);
-	return 0;
-}
-
-static void ufs_hisi_set_dev_cap(struct ufs_hisi_dev_params *hisi_param)
+static void ufs_hisi_set_dev_cap(struct ufs_dev_params *hisi_param)
 {
 	hisi_param->rx_lanes = UFS_HISI_LIMIT_NUM_LANES_RX;
 	hisi_param->tx_lanes = UFS_HISI_LIMIT_NUM_LANES_TX;
@@ -477,7 +376,7 @@ static int ufs_hisi_pwr_change_notify(struct ufs_hba *hba,
 				       struct ufs_pa_layer_attr *dev_max_params,
 				       struct ufs_pa_layer_attr *dev_req_params)
 {
-	struct ufs_hisi_dev_params ufs_hisi_cap;
+	struct ufs_dev_params ufs_hisi_cap;
 	int ret = 0;
 
 	if (!dev_req_params) {
@@ -490,8 +389,8 @@ static int ufs_hisi_pwr_change_notify(struct ufs_hba *hba,
 	switch (status) {
 	case PRE_CHANGE:
 		ufs_hisi_set_dev_cap(&ufs_hisi_cap);
-		ret = ufs_hisi_get_pwr_dev_param(
-			&ufs_hisi_cap, dev_max_params, dev_req_params);
+		ret = ufshcd_get_pwr_dev_param(&ufs_hisi_cap,
+					       dev_max_params, dev_req_params);
 		if (ret) {
 			dev_err(hba->dev,
 			    "%s: failed to determine capabilities\n", __func__);
@@ -587,6 +486,10 @@ static int ufs_hisi_init_common(struct ufs_hba *hba)
 	ufshcd_set_variant(hba, host);
 
 	host->rst  = devm_reset_control_get(dev, "rst");
+	if (IS_ERR(host->rst)) {
+		dev_err(dev, "%s: failed to get reset control\n", __func__);
+		return PTR_ERR(host->rst);
+	}
 
 	ufs_hisi_set_pm_lvl(hba);
 
diff --git a/drivers/scsi/ufs/ufs-mediatek.c b/drivers/scsi/ufs/ufs-mediatek.c
new file mode 100644
index 000000000000..0f6ff33ce52e
--- /dev/null
+++ b/drivers/scsi/ufs/ufs-mediatek.c
@@ -0,0 +1,368 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2019 MediaTek Inc.
+ * Authors:
+ *	Stanley Chu <stanley.chu@mediatek.com>
+ *	Peter Wang <peter.wang@mediatek.com>
+ */
+
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+
+#include "ufshcd.h"
+#include "ufshcd-pltfrm.h"
+#include "unipro.h"
+#include "ufs-mediatek.h"
+
+static void ufs_mtk_cfg_unipro_cg(struct ufs_hba *hba, bool enable)
+{
+	u32 tmp;
+
+	if (enable) {
+		ufshcd_dme_get(hba,
+			       UIC_ARG_MIB(VS_SAVEPOWERCONTROL), &tmp);
+		tmp = tmp |
+		      (1 << RX_SYMBOL_CLK_GATE_EN) |
+		      (1 << SYS_CLK_GATE_EN) |
+		      (1 << TX_CLK_GATE_EN);
+		ufshcd_dme_set(hba,
+			       UIC_ARG_MIB(VS_SAVEPOWERCONTROL), tmp);
+
+		ufshcd_dme_get(hba,
+			       UIC_ARG_MIB(VS_DEBUGCLOCKENABLE), &tmp);
+		tmp = tmp & ~(1 << TX_SYMBOL_CLK_REQ_FORCE);
+		ufshcd_dme_set(hba,
+			       UIC_ARG_MIB(VS_DEBUGCLOCKENABLE), tmp);
+	} else {
+		ufshcd_dme_get(hba,
+			       UIC_ARG_MIB(VS_SAVEPOWERCONTROL), &tmp);
+		tmp = tmp & ~((1 << RX_SYMBOL_CLK_GATE_EN) |
+			      (1 << SYS_CLK_GATE_EN) |
+			      (1 << TX_CLK_GATE_EN));
+		ufshcd_dme_set(hba,
+			       UIC_ARG_MIB(VS_SAVEPOWERCONTROL), tmp);
+
+		ufshcd_dme_get(hba,
+			       UIC_ARG_MIB(VS_DEBUGCLOCKENABLE), &tmp);
+		tmp = tmp | (1 << TX_SYMBOL_CLK_REQ_FORCE);
+		ufshcd_dme_set(hba,
+			       UIC_ARG_MIB(VS_DEBUGCLOCKENABLE), tmp);
+	}
+}
+
+static int ufs_mtk_bind_mphy(struct ufs_hba *hba)
+{
+	struct ufs_mtk_host *host = ufshcd_get_variant(hba);
+	struct device *dev = hba->dev;
+	struct device_node *np = dev->of_node;
+	int err = 0;
+
+	host->mphy = devm_of_phy_get_by_index(dev, np, 0);
+
+	if (host->mphy == ERR_PTR(-EPROBE_DEFER)) {
+		/*
+		 * UFS driver might be probed before the phy driver does.
+		 * In that case we would like to return EPROBE_DEFER code.
+		 */
+		err = -EPROBE_DEFER;
+		dev_info(dev,
+			 "%s: required phy hasn't probed yet. err = %d\n",
+			__func__, err);
+	} else if (IS_ERR(host->mphy)) {
+		err = PTR_ERR(host->mphy);
+		dev_info(dev, "%s: PHY get failed %d\n", __func__, err);
+	}
+
+	if (err)
+		host->mphy = NULL;
+
+	return err;
+}
+
+/**
+ * ufs_mtk_setup_clocks - enables/disable clocks
+ * @hba: host controller instance
+ * @on: If true, enable clocks else disable them.
+ * @status: PRE_CHANGE or POST_CHANGE notify
+ *
+ * Returns 0 on success, non-zero on failure.
+ */
+static int ufs_mtk_setup_clocks(struct ufs_hba *hba, bool on,
+				enum ufs_notify_change_status status)
+{
+	struct ufs_mtk_host *host = ufshcd_get_variant(hba);
+	int ret = -EINVAL;
+
+	/*
+	 * In case ufs_mtk_init() is not yet done, simply ignore.
+	 * This ufs_mtk_setup_clocks() shall be called from
+	 * ufs_mtk_init() after init is done.
+	 */
+	if (!host)
+		return 0;
+
+	switch (status) {
+	case PRE_CHANGE:
+		if (!on)
+			ret = phy_power_off(host->mphy);
+		break;
+	case POST_CHANGE:
+		if (on)
+			ret = phy_power_on(host->mphy);
+		break;
+	}
+
+	return ret;
+}
+
+/**
+ * ufs_mtk_init - find other essential mmio bases
+ * @hba: host controller instance
+ *
+ * Binds PHY with controller and powers up PHY enabling clocks
+ * and regulators.
+ *
+ * Returns -EPROBE_DEFER if binding fails, returns negative error
+ * on phy power up failure and returns zero on success.
+ */
+static int ufs_mtk_init(struct ufs_hba *hba)
+{
+	struct ufs_mtk_host *host;
+	struct device *dev = hba->dev;
+	int err = 0;
+
+	host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL);
+	if (!host) {
+		err = -ENOMEM;
+		dev_info(dev, "%s: no memory for mtk ufs host\n", __func__);
+		goto out;
+	}
+
+	host->hba = hba;
+	ufshcd_set_variant(hba, host);
+
+	err = ufs_mtk_bind_mphy(hba);
+	if (err)
+		goto out_variant_clear;
+
+	/*
+	 * ufshcd_vops_init() is invoked after
+	 * ufshcd_setup_clock(true) in ufshcd_hba_init() thus
+	 * phy clock setup is skipped.
+	 *
+	 * Enable phy clocks specifically here.
+	 */
+	ufs_mtk_setup_clocks(hba, true, POST_CHANGE);
+
+	goto out;
+
+out_variant_clear:
+	ufshcd_set_variant(hba, NULL);
+out:
+	return err;
+}
+
+static int ufs_mtk_pre_pwr_change(struct ufs_hba *hba,
+				  struct ufs_pa_layer_attr *dev_max_params,
+				  struct ufs_pa_layer_attr *dev_req_params)
+{
+	struct ufs_dev_params host_cap;
+	int ret;
+
+	host_cap.tx_lanes = UFS_MTK_LIMIT_NUM_LANES_TX;
+	host_cap.rx_lanes = UFS_MTK_LIMIT_NUM_LANES_RX;
+	host_cap.hs_rx_gear = UFS_MTK_LIMIT_HSGEAR_RX;
+	host_cap.hs_tx_gear = UFS_MTK_LIMIT_HSGEAR_TX;
+	host_cap.pwm_rx_gear = UFS_MTK_LIMIT_PWMGEAR_RX;
+	host_cap.pwm_tx_gear = UFS_MTK_LIMIT_PWMGEAR_TX;
+	host_cap.rx_pwr_pwm = UFS_MTK_LIMIT_RX_PWR_PWM;
+	host_cap.tx_pwr_pwm = UFS_MTK_LIMIT_TX_PWR_PWM;
+	host_cap.rx_pwr_hs = UFS_MTK_LIMIT_RX_PWR_HS;
+	host_cap.tx_pwr_hs = UFS_MTK_LIMIT_TX_PWR_HS;
+	host_cap.hs_rate = UFS_MTK_LIMIT_HS_RATE;
+	host_cap.desired_working_mode =
+				UFS_MTK_LIMIT_DESIRED_MODE;
+
+	ret = ufshcd_get_pwr_dev_param(&host_cap,
+				       dev_max_params,
+				       dev_req_params);
+	if (ret) {
+		pr_info("%s: failed to determine capabilities\n",
+			__func__);
+	}
+
+	return ret;
+}
+
+static int ufs_mtk_pwr_change_notify(struct ufs_hba *hba,
+				     enum ufs_notify_change_status stage,
+				     struct ufs_pa_layer_attr *dev_max_params,
+				     struct ufs_pa_layer_attr *dev_req_params)
+{
+	int ret = 0;
+
+	switch (stage) {
+	case PRE_CHANGE:
+		ret = ufs_mtk_pre_pwr_change(hba, dev_max_params,
+					     dev_req_params);
+		break;
+	case POST_CHANGE:
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	return ret;
+}
+
+static int ufs_mtk_pre_link(struct ufs_hba *hba)
+{
+	int ret;
+	u32 tmp;
+
+	/* disable deep stall */
+	ret = ufshcd_dme_get(hba, UIC_ARG_MIB(VS_SAVEPOWERCONTROL), &tmp);
+	if (ret)
+		return ret;
+
+	tmp &= ~(1 << 6);
+
+	ret = ufshcd_dme_set(hba, UIC_ARG_MIB(VS_SAVEPOWERCONTROL), tmp);
+
+	return ret;
+}
+
+static int ufs_mtk_post_link(struct ufs_hba *hba)
+{
+	/* disable device LCC */
+	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_LOCAL_TX_LCC_ENABLE), 0);
+
+	/* enable unipro clock gating feature */
+	ufs_mtk_cfg_unipro_cg(hba, true);
+
+	return 0;
+}
+
+static int ufs_mtk_link_startup_notify(struct ufs_hba *hba,
+				       enum ufs_notify_change_status stage)
+{
+	int ret = 0;
+
+	switch (stage) {
+	case PRE_CHANGE:
+		ret = ufs_mtk_pre_link(hba);
+		break;
+	case POST_CHANGE:
+		ret = ufs_mtk_post_link(hba);
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	return ret;
+}
+
+static int ufs_mtk_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)
+{
+	struct ufs_mtk_host *host = ufshcd_get_variant(hba);
+
+	if (ufshcd_is_link_hibern8(hba))
+		phy_power_off(host->mphy);
+
+	return 0;
+}
+
+static int ufs_mtk_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
+{
+	struct ufs_mtk_host *host = ufshcd_get_variant(hba);
+
+	if (ufshcd_is_link_hibern8(hba))
+		phy_power_on(host->mphy);
+
+	return 0;
+}
+
+/**
+ * struct ufs_hba_mtk_vops - UFS MTK specific variant operations
+ *
+ * The variant operations configure the necessary controller and PHY
+ * handshake during initialization.
+ */
+static struct ufs_hba_variant_ops ufs_hba_mtk_vops = {
+	.name                = "mediatek.ufshci",
+	.init                = ufs_mtk_init,
+	.setup_clocks        = ufs_mtk_setup_clocks,
+	.link_startup_notify = ufs_mtk_link_startup_notify,
+	.pwr_change_notify   = ufs_mtk_pwr_change_notify,
+	.suspend             = ufs_mtk_suspend,
+	.resume              = ufs_mtk_resume,
+};
+
+/**
+ * ufs_mtk_probe - probe routine of the driver
+ * @pdev: pointer to Platform device handle
+ *
+ * Return zero for success and non-zero for failure
+ */
+static int ufs_mtk_probe(struct platform_device *pdev)
+{
+	int err;
+	struct device *dev = &pdev->dev;
+
+	/* perform generic probe */
+	err = ufshcd_pltfrm_init(pdev, &ufs_hba_mtk_vops);
+	if (err)
+		dev_info(dev, "probe failed %d\n", err);
+
+	return err;
+}
+
+/**
+ * ufs_mtk_remove - set driver_data of the device to NULL
+ * @pdev: pointer to platform device handle
+ *
+ * Always return 0
+ */
+static int ufs_mtk_remove(struct platform_device *pdev)
+{
+	struct ufs_hba *hba =  platform_get_drvdata(pdev);
+
+	pm_runtime_get_sync(&(pdev)->dev);
+	ufshcd_remove(hba);
+	return 0;
+}
+
+static const struct of_device_id ufs_mtk_of_match[] = {
+	{ .compatible = "mediatek,mt8183-ufshci"},
+	{},
+};
+
+static const struct dev_pm_ops ufs_mtk_pm_ops = {
+	.suspend         = ufshcd_pltfrm_suspend,
+	.resume          = ufshcd_pltfrm_resume,
+	.runtime_suspend = ufshcd_pltfrm_runtime_suspend,
+	.runtime_resume  = ufshcd_pltfrm_runtime_resume,
+	.runtime_idle    = ufshcd_pltfrm_runtime_idle,
+};
+
+static struct platform_driver ufs_mtk_pltform = {
+	.probe      = ufs_mtk_probe,
+	.remove     = ufs_mtk_remove,
+	.shutdown   = ufshcd_pltfrm_shutdown,
+	.driver = {
+		.name   = "ufshcd-mtk",
+		.pm     = &ufs_mtk_pm_ops,
+		.of_match_table = ufs_mtk_of_match,
+	},
+};
+
+MODULE_AUTHOR("Stanley Chu <stanley.chu@mediatek.com>");
+MODULE_AUTHOR("Peter Wang <peter.wang@mediatek.com>");
+MODULE_DESCRIPTION("MediaTek UFS Host Driver");
+MODULE_LICENSE("GPL v2");
+
+module_platform_driver(ufs_mtk_pltform);
diff --git a/drivers/scsi/ufs/ufs-mediatek.h b/drivers/scsi/ufs/ufs-mediatek.h
new file mode 100644
index 000000000000..19f8c42fe06f
--- /dev/null
+++ b/drivers/scsi/ufs/ufs-mediatek.h
@@ -0,0 +1,53 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2019 MediaTek Inc.
+ */
+
+#ifndef _UFS_MEDIATEK_H
+#define _UFS_MEDIATEK_H
+
+/*
+ * Vendor specific pre-defined parameters
+ */
+#define UFS_MTK_LIMIT_NUM_LANES_RX  1
+#define UFS_MTK_LIMIT_NUM_LANES_TX  1
+#define UFS_MTK_LIMIT_HSGEAR_RX     UFS_HS_G3
+#define UFS_MTK_LIMIT_HSGEAR_TX     UFS_HS_G3
+#define UFS_MTK_LIMIT_PWMGEAR_RX    UFS_PWM_G4
+#define UFS_MTK_LIMIT_PWMGEAR_TX    UFS_PWM_G4
+#define UFS_MTK_LIMIT_RX_PWR_PWM    SLOW_MODE
+#define UFS_MTK_LIMIT_TX_PWR_PWM    SLOW_MODE
+#define UFS_MTK_LIMIT_RX_PWR_HS     FAST_MODE
+#define UFS_MTK_LIMIT_TX_PWR_HS     FAST_MODE
+#define UFS_MTK_LIMIT_HS_RATE       PA_HS_MODE_B
+#define UFS_MTK_LIMIT_DESIRED_MODE  UFS_HS_MODE
+
+/*
+ * Other attributes
+ */
+#define VS_DEBUGCLOCKENABLE         0xD0A1
+#define VS_SAVEPOWERCONTROL         0xD0A6
+#define VS_UNIPROPOWERDOWNCONTROL   0xD0A8
+
+/*
+ * VS_DEBUGCLOCKENABLE
+ */
+enum {
+	TX_SYMBOL_CLK_REQ_FORCE = 5,
+};
+
+/*
+ * VS_SAVEPOWERCONTROL
+ */
+enum {
+	RX_SYMBOL_CLK_GATE_EN   = 0,
+	SYS_CLK_GATE_EN         = 2,
+	TX_CLK_GATE_EN          = 3,
+};
+
+struct ufs_mtk_host {
+	struct ufs_hba *hba;
+	struct phy *mphy;
+};
+
+#endif /* !_UFS_MEDIATEK_H */
diff --git a/drivers/scsi/ufs/ufs-qcom.c b/drivers/scsi/ufs/ufs-qcom.c
index 3aeadb14aae1..ea7219407309 100644
--- a/drivers/scsi/ufs/ufs-qcom.c
+++ b/drivers/scsi/ufs/ufs-qcom.c
@@ -16,6 +16,7 @@
 #include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/phy/phy.h>
+#include <linux/reset-controller.h>
 
 #include "ufshcd.h"
 #include "ufshcd-pltfrm.h"
@@ -49,6 +50,11 @@ static void ufs_qcom_get_default_testbus_cfg(struct ufs_qcom_host *host);
 static int ufs_qcom_set_dme_vs_core_clk_ctrl_clear_div(struct ufs_hba *hba,
 						       u32 clk_cycles);
 
+static struct ufs_qcom_host *rcdev_to_ufs_host(struct reset_controller_dev *rcd)
+{
+	return container_of(rcd, struct ufs_qcom_host, rcdev);
+}
+
 static void ufs_qcom_dump_regs_wrapper(struct ufs_hba *hba, int offset, int len,
 				       const char *prefix, void *priv)
 {
@@ -255,11 +261,6 @@ static int ufs_qcom_power_up_sequence(struct ufs_hba *hba)
 	if (is_rate_B)
 		phy_set_mode(phy, PHY_MODE_UFS_HS_B);
 
-	/* Assert PHY reset and apply PHY calibration values */
-	ufs_qcom_assert_reset(hba);
-	/* provide 1ms delay to let the reset pulse propagate */
-	usleep_range(1000, 1100);
-
 	/* phy initialization - calibrate the phy */
 	ret = phy_init(phy);
 	if (ret) {
@@ -268,15 +269,6 @@ static int ufs_qcom_power_up_sequence(struct ufs_hba *hba)
 		goto out;
 	}
 
-	/* De-assert PHY reset and start serdes */
-	ufs_qcom_deassert_reset(hba);
-
-	/*
-	 * after reset deassertion, phy will need all ref clocks,
-	 * voltage, current to settle down before starting serdes.
-	 */
-	usleep_range(1000, 1100);
-
 	/* power on phy - start serdes and phy's power and clocks */
 	ret = phy_power_on(phy);
 	if (ret) {
@@ -290,7 +282,6 @@ static int ufs_qcom_power_up_sequence(struct ufs_hba *hba)
 	return 0;
 
 out_disable_phy:
-	ufs_qcom_assert_reset(hba);
 	phy_exit(phy);
 out:
 	return ret;
@@ -554,21 +545,10 @@ static int ufs_qcom_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)
 		ufs_qcom_disable_lane_clks(host);
 		phy_power_off(phy);
 
-		/* Assert PHY soft reset */
-		ufs_qcom_assert_reset(hba);
-		goto out;
-	}
-
-	/*
-	 * If UniPro link is not active, PHY ref_clk, main PHY analog power
-	 * rail and low noise analog power rail for PLL can be switched off.
-	 */
-	if (!ufs_qcom_is_link_active(hba)) {
+	} else if (!ufs_qcom_is_link_active(hba)) {
 		ufs_qcom_disable_lane_clks(host);
-		phy_power_off(phy);
 	}
 
-out:
 	return ret;
 }
 
@@ -578,118 +558,25 @@ static int ufs_qcom_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
 	struct phy *phy = host->generic_phy;
 	int err;
 
-	err = phy_power_on(phy);
-	if (err) {
-		dev_err(hba->dev, "%s: failed enabling regs, err = %d\n",
-			__func__, err);
-		goto out;
-	}
-
-	err = ufs_qcom_enable_lane_clks(host);
-	if (err)
-		goto out;
-
-	hba->is_sys_suspended = false;
-
-out:
-	return err;
-}
-
-struct ufs_qcom_dev_params {
-	u32 pwm_rx_gear;	/* pwm rx gear to work in */
-	u32 pwm_tx_gear;	/* pwm tx gear to work in */
-	u32 hs_rx_gear;		/* hs rx gear to work in */
-	u32 hs_tx_gear;		/* hs tx gear to work in */
-	u32 rx_lanes;		/* number of rx lanes */
-	u32 tx_lanes;		/* number of tx lanes */
-	u32 rx_pwr_pwm;		/* rx pwm working pwr */
-	u32 tx_pwr_pwm;		/* tx pwm working pwr */
-	u32 rx_pwr_hs;		/* rx hs working pwr */
-	u32 tx_pwr_hs;		/* tx hs working pwr */
-	u32 hs_rate;		/* rate A/B to work in HS */
-	u32 desired_working_mode;
-};
+	if (ufs_qcom_is_link_off(hba)) {
+		err = phy_power_on(phy);
+		if (err) {
+			dev_err(hba->dev, "%s: failed PHY power on: %d\n",
+				__func__, err);
+			return err;
+		}
 
-static int ufs_qcom_get_pwr_dev_param(struct ufs_qcom_dev_params *qcom_param,
-				      struct ufs_pa_layer_attr *dev_max,
-				      struct ufs_pa_layer_attr *agreed_pwr)
-{
-	int min_qcom_gear;
-	int min_dev_gear;
-	bool is_dev_sup_hs = false;
-	bool is_qcom_max_hs = false;
-
-	if (dev_max->pwr_rx == FAST_MODE)
-		is_dev_sup_hs = true;
-
-	if (qcom_param->desired_working_mode == FAST) {
-		is_qcom_max_hs = true;
-		min_qcom_gear = min_t(u32, qcom_param->hs_rx_gear,
-				      qcom_param->hs_tx_gear);
-	} else {
-		min_qcom_gear = min_t(u32, qcom_param->pwm_rx_gear,
-				      qcom_param->pwm_tx_gear);
-	}
+		err = ufs_qcom_enable_lane_clks(host);
+		if (err)
+			return err;
 
-	/*
-	 * device doesn't support HS but qcom_param->desired_working_mode is
-	 * HS, thus device and qcom_param don't agree
-	 */
-	if (!is_dev_sup_hs && is_qcom_max_hs) {
-		pr_err("%s: failed to agree on power mode (device doesn't support HS but requested power is HS)\n",
-			__func__);
-		return -ENOTSUPP;
-	} else if (is_dev_sup_hs && is_qcom_max_hs) {
-		/*
-		 * since device supports HS, it supports FAST_MODE.
-		 * since qcom_param->desired_working_mode is also HS
-		 * then final decision (FAST/FASTAUTO) is done according
-		 * to qcom_params as it is the restricting factor
-		 */
-		agreed_pwr->pwr_rx = agreed_pwr->pwr_tx =
-						qcom_param->rx_pwr_hs;
-	} else {
-		/*
-		 * here qcom_param->desired_working_mode is PWM.
-		 * it doesn't matter whether device supports HS or PWM,
-		 * in both cases qcom_param->desired_working_mode will
-		 * determine the mode
-		 */
-		 agreed_pwr->pwr_rx = agreed_pwr->pwr_tx =
-						qcom_param->rx_pwr_pwm;
+	} else if (!ufs_qcom_is_link_active(hba)) {
+		err = ufs_qcom_enable_lane_clks(host);
+		if (err)
+			return err;
 	}
 
-	/*
-	 * we would like tx to work in the minimum number of lanes
-	 * between device capability and vendor preferences.
-	 * the same decision will be made for rx
-	 */
-	agreed_pwr->lane_tx = min_t(u32, dev_max->lane_tx,
-						qcom_param->tx_lanes);
-	agreed_pwr->lane_rx = min_t(u32, dev_max->lane_rx,
-						qcom_param->rx_lanes);
-
-	/* device maximum gear is the minimum between device rx and tx gears */
-	min_dev_gear = min_t(u32, dev_max->gear_rx, dev_max->gear_tx);
-
-	/*
-	 * if both device capabilities and vendor pre-defined preferences are
-	 * both HS or both PWM then set the minimum gear to be the chosen
-	 * working gear.
-	 * if one is PWM and one is HS then the one that is PWM get to decide
-	 * what is the gear, as it is the one that also decided previously what
-	 * pwr the device will be configured to.
-	 */
-	if ((is_dev_sup_hs && is_qcom_max_hs) ||
-	    (!is_dev_sup_hs && !is_qcom_max_hs))
-		agreed_pwr->gear_rx = agreed_pwr->gear_tx =
-			min_t(u32, min_dev_gear, min_qcom_gear);
-	else if (!is_dev_sup_hs)
-		agreed_pwr->gear_rx = agreed_pwr->gear_tx = min_dev_gear;
-	else
-		agreed_pwr->gear_rx = agreed_pwr->gear_tx = min_qcom_gear;
-
-	agreed_pwr->hs_rate = qcom_param->hs_rate;
+	hba->is_sys_suspended = false;
 	return 0;
 }
 
@@ -920,7 +807,7 @@ static int ufs_qcom_pwr_change_notify(struct ufs_hba *hba,
 {
 	u32 val;
 	struct ufs_qcom_host *host = ufshcd_get_variant(hba);
-	struct ufs_qcom_dev_params ufs_qcom_cap;
+	struct ufs_dev_params ufs_qcom_cap;
 	int ret = 0;
 
 	if (!dev_req_params) {
@@ -959,9 +846,9 @@ static int ufs_qcom_pwr_change_notify(struct ufs_hba *hba,
 				ufs_qcom_cap.hs_rx_gear = UFS_HS_G2;
 		}
 
-		ret = ufs_qcom_get_pwr_dev_param(&ufs_qcom_cap,
-						 dev_max_params,
-						 dev_req_params);
+		ret = ufshcd_get_pwr_dev_param(&ufs_qcom_cap,
+					       dev_max_params,
+					       dev_req_params);
 		if (ret) {
 			pr_err("%s: failed to determine capabilities\n",
 					__func__);
@@ -1118,8 +1005,6 @@ static int ufs_qcom_setup_clocks(struct ufs_hba *hba, bool on,
 		return 0;
 
 	if (on && (status == POST_CHANGE)) {
-		phy_power_on(host->generic_phy);
-
 		/* enable the device ref clock for HS mode*/
 		if (ufshcd_is_hs_mode(&hba->pwr_info))
 			ufs_qcom_dev_ref_clk_ctrl(host, true);
@@ -1131,9 +1016,6 @@ static int ufs_qcom_setup_clocks(struct ufs_hba *hba, bool on,
 		if (!ufs_qcom_is_link_active(hba)) {
 			/* disable device ref_clk */
 			ufs_qcom_dev_ref_clk_ctrl(host, false);
-
-			/* powering off PHY during aggressive clk gating */
-			phy_power_off(host->generic_phy);
 		}
 
 		vote = host->bus_vote.min_bw_vote;
@@ -1147,6 +1029,41 @@ static int ufs_qcom_setup_clocks(struct ufs_hba *hba, bool on,
 	return err;
 }
 
+static int
+ufs_qcom_reset_assert(struct reset_controller_dev *rcdev, unsigned long id)
+{
+	struct ufs_qcom_host *host = rcdev_to_ufs_host(rcdev);
+
+	/* Currently this code only knows about a single reset. */
+	WARN_ON(id);
+	ufs_qcom_assert_reset(host->hba);
+	/* provide 1ms delay to let the reset pulse propagate. */
+	usleep_range(1000, 1100);
+	return 0;
+}
+
+static int
+ufs_qcom_reset_deassert(struct reset_controller_dev *rcdev, unsigned long id)
+{
+	struct ufs_qcom_host *host = rcdev_to_ufs_host(rcdev);
+
+	/* Currently this code only knows about a single reset. */
+	WARN_ON(id);
+	ufs_qcom_deassert_reset(host->hba);
+
+	/*
+	 * after reset deassertion, phy will need all ref clocks,
+	 * voltage, current to settle down before starting serdes.
+	 */
+	usleep_range(1000, 1100);
+	return 0;
+}
+
+static const struct reset_control_ops ufs_qcom_reset_ops = {
+	.assert = ufs_qcom_reset_assert,
+	.deassert = ufs_qcom_reset_deassert,
+};
+
 #define	ANDROID_BOOT_DEV_MAX	30
 static char android_boot_dev[ANDROID_BOOT_DEV_MAX];
 
@@ -1191,6 +1108,17 @@ static int ufs_qcom_init(struct ufs_hba *hba)
 	host->hba = hba;
 	ufshcd_set_variant(hba, host);
 
+	/* Fire up the reset controller. Failure here is non-fatal. */
+	host->rcdev.of_node = dev->of_node;
+	host->rcdev.ops = &ufs_qcom_reset_ops;
+	host->rcdev.owner = dev->driver->owner;
+	host->rcdev.nr_resets = 1;
+	err = devm_reset_controller_register(dev, &host->rcdev);
+	if (err) {
+		dev_warn(dev, "Failed to register reset controller\n");
+		err = 0;
+	}
+
 	/*
 	 * voting/devoting device ref_clk source is time consuming hence
 	 * skip devoting it during aggressive clock gating. This clock
diff --git a/drivers/scsi/ufs/ufs-qcom.h b/drivers/scsi/ufs/ufs-qcom.h
index c114826316eb..68a880185752 100644
--- a/drivers/scsi/ufs/ufs-qcom.h
+++ b/drivers/scsi/ufs/ufs-qcom.h
@@ -14,6 +14,8 @@
 #ifndef UFS_QCOM_H_
 #define UFS_QCOM_H_
 
+#include <linux/reset-controller.h>
+
 #define MAX_UFS_QCOM_HOSTS	1
 #define MAX_U32                 (~(u32)0)
 #define MPHY_TX_FSM_STATE       0x41
@@ -237,6 +239,8 @@ struct ufs_qcom_host {
 	/* Bitmask for enabling debug prints */
 	u32 dbg_print_en;
 	struct ufs_qcom_testbus testbus;
+
+	struct reset_controller_dev rcdev;
 };
 
 static inline u32
diff --git a/drivers/scsi/ufs/ufs.h b/drivers/scsi/ufs/ufs.h
index 21e4ccb5ba6e..99a9c4d16f6b 100644
--- a/drivers/scsi/ufs/ufs.h
+++ b/drivers/scsi/ufs/ufs.h
@@ -516,7 +516,6 @@ struct ufs_vreg {
 	bool enabled;
 	int min_uV;
 	int max_uV;
-	int min_uA;
 	int max_uA;
 };
 
diff --git a/drivers/scsi/ufs/ufshcd-pltfrm.c b/drivers/scsi/ufs/ufshcd-pltfrm.c
index 27213676329c..8a74ec30c3d2 100644
--- a/drivers/scsi/ufs/ufshcd-pltfrm.c
+++ b/drivers/scsi/ufs/ufshcd-pltfrm.c
@@ -39,6 +39,7 @@
 
 #include "ufshcd.h"
 #include "ufshcd-pltfrm.h"
+#include "unipro.h"
 
 #define UFSHCD_DEFAULT_LANES_PER_DIRECTION		2
 
@@ -151,20 +152,12 @@ static int ufshcd_populate_vreg(struct device *dev, const char *name,
 
 	vreg->name = kstrdup(name, GFP_KERNEL);
 
-	/* if fixed regulator no need further initialization */
-	snprintf(prop_name, MAX_PROP_SIZE, "%s-fixed-regulator", name);
-	if (of_property_read_bool(np, prop_name))
-		goto out;
-
 	snprintf(prop_name, MAX_PROP_SIZE, "%s-max-microamp", name);
-	ret = of_property_read_u32(np, prop_name, &vreg->max_uA);
-	if (ret) {
-		dev_err(dev, "%s: unable to find %s err %d\n",
-				__func__, prop_name, ret);
-		goto out;
+	if (of_property_read_u32(np, prop_name, &vreg->max_uA)) {
+		dev_info(dev, "%s: unable to find %s\n", __func__, prop_name);
+		vreg->max_uA = 0;
 	}
 
-	vreg->min_uA = 0;
 	if (!strcmp(name, "vcc")) {
 		if (of_property_read_bool(np, "vcc-supply-1p8")) {
 			vreg->min_uV = UFS_VREG_VCC_1P8_MIN_UV;
@@ -290,6 +283,103 @@ static void ufshcd_init_lanes_per_dir(struct ufs_hba *hba)
 }
 
 /**
+ * ufshcd_get_pwr_dev_param - get finally agreed attributes for
+ *                            power mode change
+ * @pltfrm_param: pointer to platform parameters
+ * @dev_max: pointer to device attributes
+ * @agreed_pwr: returned agreed attributes
+ *
+ * Returns 0 on success, non-zero value on failure
+ */
+int ufshcd_get_pwr_dev_param(struct ufs_dev_params *pltfrm_param,
+			     struct ufs_pa_layer_attr *dev_max,
+			     struct ufs_pa_layer_attr *agreed_pwr)
+{
+	int min_pltfrm_gear;
+	int min_dev_gear;
+	bool is_dev_sup_hs = false;
+	bool is_pltfrm_max_hs = false;
+
+	if (dev_max->pwr_rx == FAST_MODE)
+		is_dev_sup_hs = true;
+
+	if (pltfrm_param->desired_working_mode == UFS_HS_MODE) {
+		is_pltfrm_max_hs = true;
+		min_pltfrm_gear = min_t(u32, pltfrm_param->hs_rx_gear,
+					pltfrm_param->hs_tx_gear);
+	} else {
+		min_pltfrm_gear = min_t(u32, pltfrm_param->pwm_rx_gear,
+					pltfrm_param->pwm_tx_gear);
+	}
+
+	/*
+	 * device doesn't support HS but
+	 * pltfrm_param->desired_working_mode is HS,
+	 * thus device and pltfrm_param don't agree
+	 */
+	if (!is_dev_sup_hs && is_pltfrm_max_hs) {
+		pr_info("%s: device doesn't support HS\n",
+			__func__);
+		return -ENOTSUPP;
+	} else if (is_dev_sup_hs && is_pltfrm_max_hs) {
+		/*
+		 * since device supports HS, it supports FAST_MODE.
+		 * since pltfrm_param->desired_working_mode is also HS
+		 * then final decision (FAST/FASTAUTO) is done according
+		 * to pltfrm_params as it is the restricting factor
+		 */
+		agreed_pwr->pwr_rx = pltfrm_param->rx_pwr_hs;
+		agreed_pwr->pwr_tx = agreed_pwr->pwr_rx;
+	} else {
+		/*
+		 * here pltfrm_param->desired_working_mode is PWM.
+		 * it doesn't matter whether device supports HS or PWM,
+		 * in both cases pltfrm_param->desired_working_mode will
+		 * determine the mode
+		 */
+		agreed_pwr->pwr_rx = pltfrm_param->rx_pwr_pwm;
+		agreed_pwr->pwr_tx = agreed_pwr->pwr_rx;
+	}
+
+	/*
+	 * we would like tx to work in the minimum number of lanes
+	 * between device capability and vendor preferences.
+	 * the same decision will be made for rx
+	 */
+	agreed_pwr->lane_tx = min_t(u32, dev_max->lane_tx,
+				    pltfrm_param->tx_lanes);
+	agreed_pwr->lane_rx = min_t(u32, dev_max->lane_rx,
+				    pltfrm_param->rx_lanes);
+
+	/* device maximum gear is the minimum between device rx and tx gears */
+	min_dev_gear = min_t(u32, dev_max->gear_rx, dev_max->gear_tx);
+
+	/*
+	 * if both device capabilities and vendor pre-defined preferences are
+	 * both HS or both PWM then set the minimum gear to be the chosen
+	 * working gear.
+	 * if one is PWM and one is HS then the one that is PWM get to decide
+	 * what is the gear, as it is the one that also decided previously what
+	 * pwr the device will be configured to.
+	 */
+	if ((is_dev_sup_hs && is_pltfrm_max_hs) ||
+	    (!is_dev_sup_hs && !is_pltfrm_max_hs)) {
+		agreed_pwr->gear_rx =
+			min_t(u32, min_dev_gear, min_pltfrm_gear);
+	} else if (!is_dev_sup_hs) {
+		agreed_pwr->gear_rx = min_dev_gear;
+	} else {
+		agreed_pwr->gear_rx = min_pltfrm_gear;
+	}
+	agreed_pwr->gear_tx = agreed_pwr->gear_rx;
+
+	agreed_pwr->hs_rate = pltfrm_param->hs_rate;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(ufshcd_get_pwr_dev_param);
+
+/**
  * ufshcd_pltfrm_init - probe routine of the driver
  * @pdev: pointer to Platform device handle
  * @vops: pointer to variant ops
diff --git a/drivers/scsi/ufs/ufshcd-pltfrm.h b/drivers/scsi/ufs/ufshcd-pltfrm.h
index 1f29e1fd6d52..0919ebba182b 100644
--- a/drivers/scsi/ufs/ufshcd-pltfrm.h
+++ b/drivers/scsi/ufs/ufshcd-pltfrm.h
@@ -16,6 +16,27 @@
 
 #include "ufshcd.h"
 
+#define UFS_PWM_MODE 1
+#define UFS_HS_MODE  2
+
+struct ufs_dev_params {
+	u32 pwm_rx_gear;        /* pwm rx gear to work in */
+	u32 pwm_tx_gear;        /* pwm tx gear to work in */
+	u32 hs_rx_gear;         /* hs rx gear to work in */
+	u32 hs_tx_gear;         /* hs tx gear to work in */
+	u32 rx_lanes;           /* number of rx lanes */
+	u32 tx_lanes;           /* number of tx lanes */
+	u32 rx_pwr_pwm;         /* rx pwm working pwr */
+	u32 tx_pwr_pwm;         /* tx pwm working pwr */
+	u32 rx_pwr_hs;          /* rx hs working pwr */
+	u32 tx_pwr_hs;          /* tx hs working pwr */
+	u32 hs_rate;            /* rate A/B to work in HS */
+	u32 desired_working_mode;
+};
+
+int ufshcd_get_pwr_dev_param(struct ufs_dev_params *dev_param,
+			     struct ufs_pa_layer_attr *dev_max,
+			     struct ufs_pa_layer_attr *agreed_pwr);
 int ufshcd_pltfrm_init(struct platform_device *pdev,
 		       const struct ufs_hba_variant_ops *vops);
 void ufshcd_pltfrm_shutdown(struct platform_device *pdev);
diff --git a/drivers/scsi/ufs/ufshcd.c b/drivers/scsi/ufs/ufshcd.c
index e040f9dd9ff3..8c1c551f2b42 100644
--- a/drivers/scsi/ufs/ufshcd.c
+++ b/drivers/scsi/ufs/ufshcd.c
@@ -4704,10 +4704,10 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
 				"Reject UPIU not fully implemented\n");
 			break;
 		default:
-			result = DID_ERROR << 16;
 			dev_err(hba->dev,
 				"Unexpected request response code = %x\n",
 				result);
+			result = DID_ERROR << 16;
 			break;
 		}
 		break;
@@ -6294,19 +6294,19 @@ static u32 ufshcd_find_max_sup_active_icc_level(struct ufs_hba *hba,
 		goto out;
 	}
 
-	if (hba->vreg_info.vcc)
+	if (hba->vreg_info.vcc && hba->vreg_info.vcc->max_uA)
 		icc_level = ufshcd_get_max_icc_level(
 				hba->vreg_info.vcc->max_uA,
 				POWER_DESC_MAX_ACTV_ICC_LVLS - 1,
 				&desc_buf[PWR_DESC_ACTIVE_LVLS_VCC_0]);
 
-	if (hba->vreg_info.vccq)
+	if (hba->vreg_info.vccq && hba->vreg_info.vccq->max_uA)
 		icc_level = ufshcd_get_max_icc_level(
 				hba->vreg_info.vccq->max_uA,
 				icc_level,
 				&desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ_0]);
 
-	if (hba->vreg_info.vccq2)
+	if (hba->vreg_info.vccq2 && hba->vreg_info.vccq2->max_uA)
 		icc_level = ufshcd_get_max_icc_level(
 				hba->vreg_info.vccq2->max_uA,
 				icc_level,
@@ -7004,6 +7004,15 @@ static int ufshcd_config_vreg_load(struct device *dev, struct ufs_vreg *vreg,
 	if (!vreg)
 		return 0;
 
+	/*
+	 * "set_load" operation shall be required on those regulators
+	 * which specifically configured current limitation. Otherwise
+	 * zero max_uA may cause unexpected behavior when regulator is
+	 * enabled or set as high power mode.
+	 */
+	if (!vreg->max_uA)
+		return 0;
+
 	ret = regulator_set_load(vreg->reg, ua);
 	if (ret < 0) {
 		dev_err(dev, "%s: %s set load (ua=%d) failed, err=%d\n",
@@ -7039,12 +7048,15 @@ static int ufshcd_config_vreg(struct device *dev,
 	name = vreg->name;
 
 	if (regulator_count_voltages(reg) > 0) {
-		min_uV = on ? vreg->min_uV : 0;
-		ret = regulator_set_voltage(reg, min_uV, vreg->max_uV);
-		if (ret) {
-			dev_err(dev, "%s: %s set voltage failed, err=%d\n",
+		if (vreg->min_uV && vreg->max_uV) {
+			min_uV = on ? vreg->min_uV : 0;
+			ret = regulator_set_voltage(reg, min_uV, vreg->max_uV);
+			if (ret) {
+				dev_err(dev,
+					"%s: %s set voltage failed, err=%d\n",
 					__func__, name, ret);
-			goto out;
+				goto out;
+			}
 		}
 
 		uA_load = on ? vreg->max_uA : 0;
@@ -7103,9 +7115,6 @@ static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on)
 	struct device *dev = hba->dev;
 	struct ufs_vreg_info *info = &hba->vreg_info;
 
-	if (!info)
-		goto out;
-
 	ret = ufshcd_toggle_vreg(dev, info->vcc, on);
 	if (ret)
 		goto out;
@@ -7131,10 +7140,7 @@ static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on)
 {
 	struct ufs_vreg_info *info = &hba->vreg_info;
 
-	if (info)
-		return ufshcd_toggle_vreg(hba->dev, info->vdd_hba, on);
-
-	return 0;
+	return ufshcd_toggle_vreg(hba->dev, info->vdd_hba, on);
 }
 
 static int ufshcd_get_vreg(struct device *dev, struct ufs_vreg *vreg)
@@ -7160,9 +7166,6 @@ static int ufshcd_init_vreg(struct ufs_hba *hba)
 	struct device *dev = hba->dev;
 	struct ufs_vreg_info *info = &hba->vreg_info;
 
-	if (!info)
-		goto out;
-
 	ret = ufshcd_get_vreg(dev, info->vcc);
 	if (ret)
 		goto out;
diff --git a/drivers/scsi/ufs/unipro.h b/drivers/scsi/ufs/unipro.h
index 23129d7b2678..c77e36526447 100644
--- a/drivers/scsi/ufs/unipro.h
+++ b/drivers/scsi/ufs/unipro.h
@@ -52,7 +52,7 @@
 #define RX_HS_UNTERMINATED_ENABLE		0x00A6
 #define RX_ENTER_HIBERN8			0x00A7
 #define RX_BYPASS_8B10B_ENABLE			0x00A8
-#define RX_TERMINATION_FORCE_ENABLE		0x0089
+#define RX_TERMINATION_FORCE_ENABLE		0x00A9
 #define RX_MIN_ACTIVATETIME_CAPABILITY		0x008F
 #define RX_HIBERN8TIME_CAPABILITY		0x0092
 #define RX_REFCLKFREQ				0x00EB
diff --git a/drivers/scsi/virtio_scsi.c b/drivers/scsi/virtio_scsi.c
index f8cb7c23305b..c47d38bca948 100644
--- a/drivers/scsi/virtio_scsi.c
+++ b/drivers/scsi/virtio_scsi.c
@@ -659,7 +659,7 @@ static int virtscsi_abort(struct scsi_cmnd *sc)
 static int virtscsi_map_queues(struct Scsi_Host *shost)
 {
 	struct virtio_scsi *vscsi = shost_priv(shost);
-	struct blk_mq_queue_map *qmap = &shost->tag_set.map[0];
+	struct blk_mq_queue_map *qmap = &shost->tag_set.map[HCTX_TYPE_DEFAULT];
 
 	return blk_mq_virtio_map_queues(qmap, vscsi->vdev, 2);
 }
diff --git a/drivers/soc/sunxi/Kconfig b/drivers/soc/sunxi/Kconfig
index 353b07e40176..e84eb4e59f58 100644
--- a/drivers/soc/sunxi/Kconfig
+++ b/drivers/soc/sunxi/Kconfig
@@ -4,6 +4,7 @@
 config SUNXI_SRAM
 	bool
 	default ARCH_SUNXI
+	select REGMAP_MMIO
 	help
 	  Say y here to enable the SRAM controller support. This
 	  device is responsible on mapping the SRAM in the sunXi SoCs
diff --git a/drivers/spi/spi-pxa2xx.c b/drivers/spi/spi-pxa2xx.c
index e59c8b27b155..298a0bec29d1 100644
--- a/drivers/spi/spi-pxa2xx.c
+++ b/drivers/spi/spi-pxa2xx.c
@@ -1498,12 +1498,7 @@ static int pxa2xx_spi_get_port_id(struct acpi_device *adev)
 
 static bool pxa2xx_spi_idma_filter(struct dma_chan *chan, void *param)
 {
-	struct device *dev = param;
-
-	if (dev != chan->device->dev->parent)
-		return false;
-
-	return true;
+	return param == chan->device->dev;
 }
 
 #endif /* CONFIG_PCI */
diff --git a/drivers/staging/Kconfig b/drivers/staging/Kconfig
index 86001a9f67e0..d5f771fafc21 100644
--- a/drivers/staging/Kconfig
+++ b/drivers/staging/Kconfig
@@ -94,8 +94,6 @@ source "drivers/staging/greybus/Kconfig"
 
 source "drivers/staging/vc04_services/Kconfig"
 
-source "drivers/staging/vboxvideo/Kconfig"
-
 source "drivers/staging/pi433/Kconfig"
 
 source "drivers/staging/mt7621-pci/Kconfig"
diff --git a/drivers/staging/Makefile b/drivers/staging/Makefile
index dc3da72b3ff9..0da0d3f0b5e4 100644
--- a/drivers/staging/Makefile
+++ b/drivers/staging/Makefile
@@ -37,7 +37,6 @@ obj-$(CONFIG_MOST)		+= most/
 obj-$(CONFIG_KS7010)		+= ks7010/
 obj-$(CONFIG_GREYBUS)		+= greybus/
 obj-$(CONFIG_BCM2835_VCHIQ)	+= vc04_services/
-obj-$(CONFIG_DRM_VBOXVIDEO)	+= vboxvideo/
 obj-$(CONFIG_PI433)		+= pi433/
 obj-$(CONFIG_PCI_MT7621)	+= mt7621-pci/
 obj-$(CONFIG_PCI_MT7621_PHY)	+= mt7621-pci-phy/
diff --git a/drivers/staging/media/Kconfig b/drivers/staging/media/Kconfig
index 44280b6c021a..f77f5eee7fc2 100644
--- a/drivers/staging/media/Kconfig
+++ b/drivers/staging/media/Kconfig
@@ -2,7 +2,7 @@
 menuconfig STAGING_MEDIA
 	bool "Media staging drivers"
 	default n
-	---help---
+	help
 	  This option allows you to select a number of media drivers that
 	  don't have the "normal" Linux kernel quality level.
 	  Most of them don't follow properly the V4L, DVB and/or RC API's,
@@ -34,8 +34,6 @@ source "drivers/staging/media/sunxi/Kconfig"
 
 source "drivers/staging/media/tegra-vde/Kconfig"
 
-source "drivers/staging/media/zoran/Kconfig"
-
 source "drivers/staging/media/ipu3/Kconfig"
 
 source "drivers/staging/media/soc_camera/Kconfig"
diff --git a/drivers/staging/media/Makefile b/drivers/staging/media/Makefile
index 0355e3030504..99218bfc997f 100644
--- a/drivers/staging/media/Makefile
+++ b/drivers/staging/media/Makefile
@@ -5,7 +5,6 @@ obj-$(CONFIG_VIDEO_DM365_VPFE)	+= davinci_vpfe/
 obj-$(CONFIG_VIDEO_OMAP4)	+= omap4iss/
 obj-$(CONFIG_VIDEO_SUNXI)	+= sunxi/
 obj-$(CONFIG_TEGRA_VDE)		+= tegra-vde/
-obj-$(CONFIG_VIDEO_ZORAN)	+= zoran/
 obj-$(CONFIG_VIDEO_ROCKCHIP_VPU) += rockchip/vpu/
 obj-$(CONFIG_VIDEO_IPU3_IMGU)	+= ipu3/
 obj-$(CONFIG_SOC_CAMERA)	+= soc_camera/
diff --git a/drivers/staging/media/bcm2048/Kconfig b/drivers/staging/media/bcm2048/Kconfig
index c93a0a848c90..ab2d50cac140 100644
--- a/drivers/staging/media/bcm2048/Kconfig
+++ b/drivers/staging/media/bcm2048/Kconfig
@@ -6,7 +6,7 @@
 config I2C_BCM2048
 	tristate "Broadcom BCM2048 FM Radio Receiver support"
 	depends on I2C && VIDEO_V4L2 && RADIO_ADAPTERS
-	---help---
+	help
 	  Say Y here if you want support to BCM2048 FM Radio Receiver.
 	  This device driver supports only i2c bus.
 
diff --git a/drivers/staging/media/bcm2048/radio-bcm2048.c b/drivers/staging/media/bcm2048/radio-bcm2048.c
index d9b02ff66259..09903ffb13ba 100644
--- a/drivers/staging/media/bcm2048/radio-bcm2048.c
+++ b/drivers/staging/media/bcm2048/radio-bcm2048.c
@@ -2404,7 +2404,7 @@ static int bcm2048_vidioc_g_audio(struct file *file, void *priv,
 	if (audio->index > 1)
 		return -EINVAL;
 
-	strncpy(audio->name, "Radio", 32);
+	strscpy(audio->name, "Radio", sizeof(audio->name));
 	audio->capability = V4L2_AUDCAP_STEREO;
 
 	return 0;
@@ -2432,7 +2432,7 @@ static int bcm2048_vidioc_g_tuner(struct file *file, void *priv,
 	if (tuner->index > 0)
 		return -EINVAL;
 
-	strncpy(tuner->name, "FM Receiver", 32);
+	strscpy(tuner->name, "FM Receiver", sizeof(tuner->name));
 	tuner->type = V4L2_TUNER_RADIO;
 	tuner->rangelow =
 		dev_to_v4l2(bcm2048_get_region_bottom_frequency(bdev));
diff --git a/drivers/staging/media/davinci_vpfe/Kconfig b/drivers/staging/media/davinci_vpfe/Kconfig
index eb61141b29e8..94bf6746c03f 100644
--- a/drivers/staging/media/davinci_vpfe/Kconfig
+++ b/drivers/staging/media/davinci_vpfe/Kconfig
@@ -2,7 +2,7 @@
 config VIDEO_DM365_VPFE
 	tristate "DM365 VPFE Media Controller Capture Driver"
 	depends on VIDEO_V4L2
-	depends on (ARCH_DAVINCI_DM365 && !VIDEO_DM365_ISIF) || COMPILE_TEST
+	depends on (ARCH_DAVINCI_DM365 && !VIDEO_DM365_ISIF) || (COMPILE_TEST && !ARCH_OMAP1)
 	depends on VIDEO_V4L2_SUBDEV_API
 	depends on VIDEO_DAVINCI_VPBE_DISPLAY
 	select VIDEOBUF2_DMA_CONTIG
diff --git a/drivers/staging/media/davinci_vpfe/dm365_isif.c b/drivers/staging/media/davinci_vpfe/dm365_isif.c
index 0a6d038fcec9..46fd8184fc77 100644
--- a/drivers/staging/media/davinci_vpfe/dm365_isif.c
+++ b/drivers/staging/media/davinci_vpfe/dm365_isif.c
@@ -433,9 +433,9 @@ static int isif_get_params(struct v4l2_subdev *sd, void *params)
 	return 0;
 }
 
-static int isif_validate_df_csc_params(struct vpfe_isif_df_csc *df_csc)
+static int isif_validate_df_csc_params(const struct vpfe_isif_df_csc *df_csc)
 {
-	struct vpfe_isif_color_space_conv *csc;
+	const struct vpfe_isif_color_space_conv *csc;
 	int err = -EINVAL;
 	int i;
 
@@ -481,7 +481,7 @@ static int isif_validate_df_csc_params(struct vpfe_isif_df_csc *df_csc)
 #define DM365_ISIF_MAX_DFCMEM0		0x1fff
 #define DM365_ISIF_MAX_DFCMEM1		0x1fff
 
-static int isif_validate_dfc_params(struct vpfe_isif_dfc *dfc)
+static int isif_validate_dfc_params(const struct vpfe_isif_dfc *dfc)
 {
 	int err = -EINVAL;
 	int i;
@@ -532,7 +532,7 @@ static int isif_validate_dfc_params(struct vpfe_isif_dfc *dfc)
 #define DM365_ISIF_MAX_CLVSV			0x1fff
 #define DM365_ISIF_MAX_HEIGHT_BLACK_REGION	0x1fff
 
-static int isif_validate_bclamp_params(struct vpfe_isif_black_clamp *bclamp)
+static int isif_validate_bclamp_params(const struct vpfe_isif_black_clamp *bclamp)
 {
 	int err = -EINVAL;
 
@@ -580,7 +580,7 @@ static int isif_validate_bclamp_params(struct vpfe_isif_black_clamp *bclamp)
 }
 
 static int
-isif_validate_raw_params(struct vpfe_isif_raw_config *params)
+isif_validate_raw_params(const struct vpfe_isif_raw_config *params)
 {
 	int ret;
 
@@ -593,20 +593,18 @@ isif_validate_raw_params(struct vpfe_isif_raw_config *params)
 	return isif_validate_bclamp_params(&params->bclamp);
 }
 
-static int isif_set_params(struct v4l2_subdev *sd, void *params)
+static int isif_set_params(struct v4l2_subdev *sd, const struct vpfe_isif_raw_config *params)
 {
 	struct vpfe_isif_device *isif = v4l2_get_subdevdata(sd);
-	struct vpfe_isif_raw_config isif_raw_params;
 	int ret = -EINVAL;
 
 	/* only raw module parameters can be set through the IOCTL */
 	if (isif->formats[ISIF_PAD_SINK].code != MEDIA_BUS_FMT_SGRBG12_1X12)
 		return ret;
 
-	memcpy(&isif_raw_params, params, sizeof(isif_raw_params));
-	if (!isif_validate_raw_params(&isif_raw_params)) {
-		memcpy(&isif->isif_cfg.bayer.config_params, &isif_raw_params,
-			sizeof(isif_raw_params));
+	if (!isif_validate_raw_params(params)) {
+		memcpy(&isif->isif_cfg.bayer.config_params, params,
+			sizeof(*params));
 		ret = 0;
 	}
 	return ret;
diff --git a/drivers/staging/media/davinci_vpfe/dm365_resizer.c b/drivers/staging/media/davinci_vpfe/dm365_resizer.c
index d460963106fa..7adf1fae43f6 100644
--- a/drivers/staging/media/davinci_vpfe/dm365_resizer.c
+++ b/drivers/staging/media/davinci_vpfe/dm365_resizer.c
@@ -1881,7 +1881,7 @@ int vpfe_resizer_init(struct vpfe_resizer_device *vpfe_rsz,
 	struct v4l2_subdev *sd = &vpfe_rsz->crop_resizer.subdev;
 	struct media_pad *pads = &vpfe_rsz->crop_resizer.pads[0];
 	struct media_entity *me = &sd->entity;
-	static resource_size_t  res_len;
+	resource_size_t res_len;
 	struct resource *res;
 	int ret;
 
diff --git a/drivers/staging/media/imx/Kconfig b/drivers/staging/media/imx/Kconfig
index 6c8b69491ef0..4c726345dc25 100644
--- a/drivers/staging/media/imx/Kconfig
+++ b/drivers/staging/media/imx/Kconfig
@@ -7,7 +7,7 @@ config VIDEO_IMX_MEDIA
 	depends on HAS_DMA
 	select VIDEOBUF2_DMA_CONTIG
 	select V4L2_FWNODE
-	---help---
+	help
 	  Say yes here to enable support for video4linux media controller
 	  driver for the i.MX5/6 SOC.
 
@@ -18,7 +18,7 @@ config VIDEO_IMX_CSI
 	tristate "i.MX5/6 Camera Sensor Interface driver"
 	depends on VIDEO_IMX_MEDIA && VIDEO_DEV && I2C
 	default y
-	---help---
+	help
 	  A video4linux camera sensor interface driver for i.MX5/6.
 
 config VIDEO_IMX7_CSI
diff --git a/drivers/staging/media/imx/imx-ic-common.c b/drivers/staging/media/imx/imx-ic-common.c
index 7e2455097315..18cd4cb92431 100644
--- a/drivers/staging/media/imx/imx-ic-common.c
+++ b/drivers/staging/media/imx/imx-ic-common.c
@@ -22,7 +22,7 @@ static struct imx_ic_ops *ic_ops[IC_NUM_OPS] = {
 
 static int imx_ic_probe(struct platform_device *pdev)
 {
-	struct imx_media_internal_sd_platformdata *pdata;
+	struct imx_media_ipu_internal_sd_pdata *pdata;
 	struct imx_ic_priv *priv;
 	int ret;
 
@@ -59,7 +59,7 @@ static int imx_ic_probe(struct platform_device *pdev)
 	priv->sd.owner = THIS_MODULE;
 	priv->sd.flags = V4L2_SUBDEV_FL_HAS_DEVNODE | V4L2_SUBDEV_FL_HAS_EVENTS;
 	priv->sd.grp_id = pdata->grp_id;
-	strncpy(priv->sd.name, pdata->sd_name, sizeof(priv->sd.name));
+	strscpy(priv->sd.name, pdata->sd_name, sizeof(priv->sd.name));
 
 	ret = ic_ops[priv->task_id]->init(priv);
 	if (ret)
diff --git a/drivers/staging/media/imx/imx-media-csi.c b/drivers/staging/media/imx/imx-media-csi.c
index edc0e9a2a90d..28fe66052cc7 100644
--- a/drivers/staging/media/imx/imx-media-csi.c
+++ b/drivers/staging/media/imx/imx-media-csi.c
@@ -150,9 +150,10 @@ static inline bool requires_passthrough(struct v4l2_fwnode_endpoint *ep,
 /*
  * Parses the fwnode endpoint from the source pad of the entity
  * connected to this CSI. This will either be the entity directly
- * upstream from the CSI-2 receiver, or directly upstream from the
- * video mux. The endpoint is needed to determine the bus type and
- * bus config coming into the CSI.
+ * upstream from the CSI-2 receiver, directly upstream from the
+ * video mux, or directly upstream from the CSI itself. The endpoint
+ * is needed to determine the bus type and bus config coming into
+ * the CSI.
  */
 static int csi_get_upstream_endpoint(struct csi_priv *priv,
 				     struct v4l2_fwnode_endpoint *ep)
@@ -168,7 +169,8 @@ static int csi_get_upstream_endpoint(struct csi_priv *priv,
 	if (!priv->src_sd)
 		return -EPIPE;
 
-	src = &priv->src_sd->entity;
+	sd = priv->src_sd;
+	src = &sd->entity;
 
 	if (src->function == MEDIA_ENT_F_VID_MUX) {
 		/*
@@ -182,6 +184,14 @@ static int csi_get_upstream_endpoint(struct csi_priv *priv,
 			src = &sd->entity;
 	}
 
+	/*
+	 * If the source is neither the video mux nor the CSI-2 receiver,
+	 * get the source pad directly upstream from CSI itself.
+	 */
+	if (src->function != MEDIA_ENT_F_VID_MUX &&
+	    sd->grp_id != IMX_MEDIA_GRP_ID_CSI2)
+		src = &priv->sd.entity;
+
 	/* get source pad of entity directly upstream from src */
 	pad = imx_media_find_upstream_pad(priv->md, src, 0);
 	if (IS_ERR(pad))
diff --git a/drivers/staging/media/imx/imx-media-dev-common.c b/drivers/staging/media/imx/imx-media-dev-common.c
index 910594125889..6cd93419b81d 100644
--- a/drivers/staging/media/imx/imx-media-dev-common.c
+++ b/drivers/staging/media/imx/imx-media-dev-common.c
@@ -30,7 +30,7 @@ struct imx_media_dev *imx_media_dev_init(struct device *dev)
 
 	dev_set_drvdata(dev, imxmd);
 
-	strlcpy(imxmd->md.model, "imx-media", sizeof(imxmd->md.model));
+	strscpy(imxmd->md.model, "imx-media", sizeof(imxmd->md.model));
 	imxmd->md.ops = &imx_media_md_ops;
 	imxmd->md.dev = dev;
 
@@ -38,7 +38,7 @@ struct imx_media_dev *imx_media_dev_init(struct device *dev)
 
 	imxmd->v4l2_dev.mdev = &imxmd->md;
 	imxmd->v4l2_dev.notify = imx_media_notify;
-	strlcpy(imxmd->v4l2_dev.name, "imx-media",
+	strscpy(imxmd->v4l2_dev.name, "imx-media",
 		sizeof(imxmd->v4l2_dev.name));
 
 	media_device_init(&imxmd->md);
diff --git a/drivers/staging/media/imx/imx-media-dev.c b/drivers/staging/media/imx/imx-media-dev.c
index bd4ddea488f1..6be95584006d 100644
--- a/drivers/staging/media/imx/imx-media-dev.c
+++ b/drivers/staging/media/imx/imx-media-dev.c
@@ -46,12 +46,14 @@ int imx_media_add_async_subdev(struct imx_media_dev *imxmd,
 	int ret;
 
 	if (fwnode) {
-		asd = v4l2_async_notifier_add_fwnode_subdev(
-			&imxmd->notifier, fwnode, sizeof(*imxasd));
+		asd = v4l2_async_notifier_add_fwnode_subdev(&imxmd->notifier,
+							    fwnode,
+							    sizeof(*imxasd));
 	} else {
 		devname = dev_name(&pdev->dev);
-		asd = v4l2_async_notifier_add_devname_subdev(
-			&imxmd->notifier, devname, sizeof(*imxasd));
+		asd = v4l2_async_notifier_add_devname_subdev(&imxmd->notifier,
+							     devname,
+							     sizeof(*imxasd));
 	}
 
 	if (IS_ERR(asd)) {
@@ -262,10 +264,9 @@ static int imx_media_alloc_pad_vdev_lists(struct imx_media_dev *imxmd)
 
 	list_for_each_entry(sd, &imxmd->v4l2_dev.subdevs, list) {
 		entity = &sd->entity;
-		vdev_lists = devm_kcalloc(
-			imxmd->md.dev,
-			entity->num_pads, sizeof(*vdev_lists),
-			GFP_KERNEL);
+		vdev_lists = devm_kcalloc(imxmd->md.dev,
+					  entity->num_pads, sizeof(*vdev_lists),
+					  GFP_KERNEL);
 		if (!vdev_lists)
 			return -ENOMEM;
 
@@ -473,13 +474,6 @@ static int imx_media_probe(struct platform_device *pdev)
 		goto cleanup;
 	}
 
-	ret = imx_media_add_internal_subdevs(imxmd);
-	if (ret) {
-		v4l2_err(&imxmd->v4l2_dev,
-			 "add_internal_subdevs failed with %d\n", ret);
-		goto cleanup;
-	}
-
 	ret = imx_media_dev_notifier_register(imxmd);
 	if (ret)
 		goto del_int;
@@ -487,7 +481,7 @@ static int imx_media_probe(struct platform_device *pdev)
 	return 0;
 
 del_int:
-	imx_media_remove_internal_subdevs(imxmd);
+	imx_media_remove_ipu_internal_subdevs(imxmd);
 cleanup:
 	v4l2_async_notifier_cleanup(&imxmd->notifier);
 	v4l2_device_unregister(&imxmd->v4l2_dev);
@@ -504,7 +498,7 @@ static int imx_media_remove(struct platform_device *pdev)
 	v4l2_info(&imxmd->v4l2_dev, "Removing imx-media\n");
 
 	v4l2_async_notifier_unregister(&imxmd->notifier);
-	imx_media_remove_internal_subdevs(imxmd);
+	imx_media_remove_ipu_internal_subdevs(imxmd);
 	v4l2_async_notifier_cleanup(&imxmd->notifier);
 	media_device_unregister(&imxmd->md);
 	v4l2_device_unregister(&imxmd->v4l2_dev);
diff --git a/drivers/staging/media/imx/imx-media-internal-sd.c b/drivers/staging/media/imx/imx-media-internal-sd.c
index c547280a309e..df49ebfbe98a 100644
--- a/drivers/staging/media/imx/imx-media-internal-sd.c
+++ b/drivers/staging/media/imx/imx-media-internal-sd.c
@@ -2,7 +2,7 @@
 /*
  * Media driver for Freescale i.MX5/6 SOC
  *
- * Adds the internal subdevices and the media links between them.
+ * Adds the IPU internal subdevices and the media links between them.
  *
  * Copyright (c) 2016 Mentor Graphics Inc.
  */
@@ -188,7 +188,7 @@ static struct v4l2_subdev *find_sink(struct imx_media_dev *imxmd,
 
 	/*
 	 * retrieve IPU id from subdev name, note: can't get this from
-	 * struct imx_media_internal_sd_platformdata because if src is
+	 * struct imx_media_ipu_internal_sd_pdata because if src is
 	 * a CSI, it has different struct ipu_client_platformdata which
 	 * does not contain IPU id.
 	 */
@@ -266,7 +266,7 @@ static int add_internal_subdev(struct imx_media_dev *imxmd,
 			       const struct internal_subdev *isd,
 			       int ipu_id)
 {
-	struct imx_media_internal_sd_platformdata pdata;
+	struct imx_media_ipu_internal_sd_pdata pdata;
 	struct platform_device_info pdevinfo = {};
 	struct platform_device *pdev;
 
@@ -294,13 +294,14 @@ static int add_internal_subdev(struct imx_media_dev *imxmd,
 }
 
 /* adds the internal subdevs in one ipu */
-static int add_ipu_internal_subdevs(struct imx_media_dev *imxmd, int ipu_id)
+int imx_media_add_ipu_internal_subdevs(struct imx_media_dev *imxmd,
+				       int ipu_id)
 {
 	enum isd_enum i;
+	int ret;
 
 	for (i = 0; i < num_isd; i++) {
 		const struct internal_subdev *isd = &int_subdev[i];
-		int ret;
 
 		/*
 		 * the CSIs are represented in the device-tree, so those
@@ -318,32 +319,17 @@ static int add_ipu_internal_subdevs(struct imx_media_dev *imxmd, int ipu_id)
 		}
 
 		if (ret)
-			return ret;
+			goto remove;
 	}
 
 	return 0;
-}
-
-int imx_media_add_internal_subdevs(struct imx_media_dev *imxmd)
-{
-	int ret;
-
-	ret = add_ipu_internal_subdevs(imxmd, 0);
-	if (ret)
-		goto remove;
-
-	ret = add_ipu_internal_subdevs(imxmd, 1);
-	if (ret)
-		goto remove;
-
-	return 0;
 
 remove:
-	imx_media_remove_internal_subdevs(imxmd);
+	imx_media_remove_ipu_internal_subdevs(imxmd);
 	return ret;
 }
 
-void imx_media_remove_internal_subdevs(struct imx_media_dev *imxmd)
+void imx_media_remove_ipu_internal_subdevs(struct imx_media_dev *imxmd)
 {
 	struct imx_media_async_subdev *imxasd;
 	struct v4l2_async_subdev *asd;
diff --git a/drivers/staging/media/imx/imx-media-of.c b/drivers/staging/media/imx/imx-media-of.c
index 09580d83c685..990e82aa8e42 100644
--- a/drivers/staging/media/imx/imx-media-of.c
+++ b/drivers/staging/media/imx/imx-media-of.c
@@ -19,36 +19,25 @@
 int imx_media_of_add_csi(struct imx_media_dev *imxmd,
 			 struct device_node *csi_np)
 {
-	int ret;
-
 	if (!of_device_is_available(csi_np)) {
 		dev_dbg(imxmd->md.dev, "%s: %pOFn not enabled\n", __func__,
 			csi_np);
-		/* unavailable is not an error */
-		return 0;
+		return -ENODEV;
 	}
 
 	/* add CSI fwnode to async notifier */
-	ret = imx_media_add_async_subdev(imxmd, of_fwnode_handle(csi_np), NULL);
-	if (ret) {
-		if (ret == -EEXIST) {
-			/* already added, everything is fine */
-			return 0;
-		}
-
-		/* other error, can't continue */
-		return ret;
-	}
-
-	return 0;
+	return imx_media_add_async_subdev(imxmd, of_fwnode_handle(csi_np),
+					  NULL);
 }
 EXPORT_SYMBOL_GPL(imx_media_of_add_csi);
 
 int imx_media_add_of_subdevs(struct imx_media_dev *imxmd,
 			     struct device_node *np)
 {
+	bool ipu_found[2] = {false, false};
 	struct device_node *csi_np;
 	int i, ret;
+	u32 ipu_id;
 
 	for (i = 0; ; i++) {
 		csi_np = of_parse_phandle(np, "ports", i);
@@ -56,12 +45,43 @@ int imx_media_add_of_subdevs(struct imx_media_dev *imxmd,
 			break;
 
 		ret = imx_media_of_add_csi(imxmd, csi_np);
-		of_node_put(csi_np);
-		if (ret)
-			return ret;
+		if (ret) {
+			/* unavailable or already added is not an error */
+			if (ret == -ENODEV || ret == -EEXIST) {
+				of_node_put(csi_np);
+				continue;
+			}
+
+			/* other error, can't continue */
+			goto err_out;
+		}
+
+		ret = of_alias_get_id(csi_np->parent, "ipu");
+		if (ret < 0)
+			goto err_out;
+		if (ret > 1) {
+			ret = -EINVAL;
+			goto err_out;
+		}
+
+		ipu_id = ret;
+
+		if (!ipu_found[ipu_id]) {
+			ret = imx_media_add_ipu_internal_subdevs(imxmd,
+								 ipu_id);
+			if (ret)
+				goto err_out;
+		}
+
+		ipu_found[ipu_id] = true;
 	}
 
 	return 0;
+
+err_out:
+	imx_media_remove_ipu_internal_subdevs(imxmd);
+	of_node_put(csi_np);
+	return ret;
 }
 
 /*
@@ -141,15 +161,18 @@ int imx_media_create_csi_of_links(struct imx_media_dev *imxmd,
 				  struct v4l2_subdev *csi)
 {
 	struct device_node *csi_np = csi->dev->of_node;
-	struct fwnode_handle *fwnode, *csi_ep;
-	struct v4l2_fwnode_link link;
 	struct device_node *ep;
-	int ret;
-
-	link.local_node = of_fwnode_handle(csi_np);
-	link.local_port = CSI_SINK_PAD;
 
 	for_each_child_of_node(csi_np, ep) {
+		struct fwnode_handle *fwnode, *csi_ep;
+		struct v4l2_fwnode_link link;
+		int ret;
+
+		memset(&link, 0, sizeof(link));
+
+		link.local_node = of_fwnode_handle(csi_np);
+		link.local_port = CSI_SINK_PAD;
+
 		csi_ep = of_fwnode_handle(ep);
 
 		fwnode = fwnode_graph_get_remote_endpoint(csi_ep);
diff --git a/drivers/staging/media/imx/imx-media-vdic.c b/drivers/staging/media/imx/imx-media-vdic.c
index 3f4b5e9080d4..4487374c9435 100644
--- a/drivers/staging/media/imx/imx-media-vdic.c
+++ b/drivers/staging/media/imx/imx-media-vdic.c
@@ -227,6 +227,12 @@ static void __maybe_unused prepare_vdi_in_buffers(struct vdic_priv *priv,
 		curr_phys = vb2_dma_contig_plane_dma_addr(curr_vb, 0);
 		next_phys = vb2_dma_contig_plane_dma_addr(curr_vb, 0) + is;
 		break;
+	default:
+		/*
+		 * can't get here, priv->fieldtype can only be one of
+		 * the above. This is to quiet smatch errors.
+		 */
+		return;
 	}
 
 	ipu_cpmem_set_buffer(priv->vdi_in_ch_p, 0, prev_phys);
@@ -740,7 +746,7 @@ static int vdic_link_setup(struct media_entity *entity,
 		remote_sd = media_entity_to_v4l2_subdev(remote->entity);
 
 		/* direct pad must connect to a CSI */
-		if (!(remote_sd->grp_id & IMX_MEDIA_GRP_ID_CSI) ||
+		if (!(remote_sd->grp_id & IMX_MEDIA_GRP_ID_IPU_CSI) ||
 		    remote->index != CSI_SRC_PAD_DIRECT) {
 			ret = -EINVAL;
 			goto out;
@@ -930,7 +936,7 @@ static const struct v4l2_subdev_internal_ops vdic_internal_ops = {
 
 static int imx_vdic_probe(struct platform_device *pdev)
 {
-	struct imx_media_internal_sd_platformdata *pdata;
+	struct imx_media_ipu_internal_sd_pdata *pdata;
 	struct vdic_priv *priv;
 	int ret;
 
@@ -954,7 +960,7 @@ static int imx_vdic_probe(struct platform_device *pdev)
 	priv->sd.flags = V4L2_SUBDEV_FL_HAS_DEVNODE;
 	/* get our group id */
 	priv->sd.grp_id = pdata->grp_id;
-	strncpy(priv->sd.name, pdata->sd_name, sizeof(priv->sd.name));
+	strscpy(priv->sd.name, pdata->sd_name, sizeof(priv->sd.name));
 
 	mutex_init(&priv->lock);
 
diff --git a/drivers/staging/media/imx/imx-media.h b/drivers/staging/media/imx/imx-media.h
index 1f7501d527fb..eb59ba0c3b62 100644
--- a/drivers/staging/media/imx/imx-media.h
+++ b/drivers/staging/media/imx/imx-media.h
@@ -111,7 +111,7 @@ struct imx_media_pad_vdev {
 	struct list_head list;
 };
 
-struct imx_media_internal_sd_platformdata {
+struct imx_media_ipu_internal_sd_pdata {
 	char sd_name[V4L2_SUBDEV_NAME_SIZE];
 	u32 grp_id;
 	int ipu_id;
@@ -248,10 +248,11 @@ struct imx_media_fim *imx_media_fim_init(struct v4l2_subdev *sd);
 void imx_media_fim_free(struct imx_media_fim *fim);
 
 /* imx-media-internal-sd.c */
-int imx_media_add_internal_subdevs(struct imx_media_dev *imxmd);
+int imx_media_add_ipu_internal_subdevs(struct imx_media_dev *imxmd,
+				       int ipu_id);
 int imx_media_create_ipu_internal_links(struct imx_media_dev *imxmd,
 					struct v4l2_subdev *sd);
-void imx_media_remove_internal_subdevs(struct imx_media_dev *imxmd);
+void imx_media_remove_ipu_internal_subdevs(struct imx_media_dev *imxmd);
 
 /* imx-media-of.c */
 int imx_media_add_of_subdevs(struct imx_media_dev *dev,
diff --git a/drivers/staging/media/imx/imx7-media-csi.c b/drivers/staging/media/imx/imx7-media-csi.c
index 3fba7c27c0ec..18eb5d3ecf10 100644
--- a/drivers/staging/media/imx/imx7-media-csi.c
+++ b/drivers/staging/media/imx/imx7-media-csi.c
@@ -1051,7 +1051,9 @@ static int imx7_csi_set_fmt(struct v4l2_subdev *sd,
 		goto out_unlock;
 	}
 
-	imx7_csi_try_fmt(csi, cfg, sdformat, &cc);
+	ret = imx7_csi_try_fmt(csi, cfg, sdformat, &cc);
+	if (ret < 0)
+		goto out_unlock;
 
 	fmt = imx7_csi_get_format(csi, cfg, sdformat->pad, sdformat->which);
 	if (!fmt) {
@@ -1271,7 +1273,7 @@ static int imx7_csi_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, &csi->sd);
 
 	ret = imx_media_of_add_csi(imxmd, node);
-	if (ret < 0)
+	if (ret < 0 && ret != -ENODEV && ret != -EEXIST)
 		goto cleanup;
 
 	ret = imx_media_dev_notifier_register(imxmd);
diff --git a/drivers/staging/media/imx/imx7-mipi-csis.c b/drivers/staging/media/imx/imx7-mipi-csis.c
index 2ddcc42ab8ff..19455f425416 100644
--- a/drivers/staging/media/imx/imx7-mipi-csis.c
+++ b/drivers/staging/media/imx/imx7-mipi-csis.c
@@ -9,6 +9,7 @@
  */
 
 #include <linux/clk.h>
+#include <linux/debugfs.h>
 #include <linux/delay.h>
 #include <linux/errno.h>
 #include <linux/interrupt.h>
@@ -491,7 +492,7 @@ static int mipi_csis_clk_get(struct csi_state *state)
 
 	state->wrap_clk = devm_clk_get(dev, "wrap");
 	if (IS_ERR(state->wrap_clk))
-		return IS_ERR(state->wrap_clk);
+		return PTR_ERR(state->wrap_clk);
 
 	/* Set clock rate */
 	ret = clk_set_rate(state->wrap_clk, state->clk_frequency);
@@ -706,7 +707,7 @@ static int mipi_csis_set_fmt(struct v4l2_subdev *mipi_sd,
 	fmt = mipi_csis_get_format(state, cfg, sdformat->which, sdformat->pad);
 
 	mutex_lock(&state->lock);
-	if (fmt && sdformat->pad == CSIS_PAD_SOURCE) {
+	if (sdformat->pad == CSIS_PAD_SOURCE) {
 		sdformat->format = *fmt;
 		goto unlock;
 	}
@@ -889,8 +890,6 @@ static int mipi_csis_subdev_init(struct v4l2_subdev *mipi_sd,
 	return ret;
 }
 
-#ifdef CONFIG_DEBUG_FS
-#include <linux/debugfs.h>
 
 static int mipi_csis_dump_regs_show(struct seq_file *m, void *private)
 {
@@ -900,7 +899,7 @@ static int mipi_csis_dump_regs_show(struct seq_file *m, void *private)
 }
 DEFINE_SHOW_ATTRIBUTE(mipi_csis_dump_regs);
 
-static int __init_or_module mipi_csis_debugfs_init(struct csi_state *state)
+static int mipi_csis_debugfs_init(struct csi_state *state)
 {
 	struct dentry *d;
 
@@ -934,17 +933,6 @@ static void mipi_csis_debugfs_exit(struct csi_state *state)
 	debugfs_remove_recursive(state->debugfs_root);
 }
 
-#else
-static int mipi_csis_debugfs_init(struct csi_state *state __maybe_unused)
-{
-	return 0;
-}
-
-static void mipi_csis_debugfs_exit(struct csi_state *state __maybe_unused)
-{
-}
-#endif
-
 static int mipi_csis_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
@@ -1039,8 +1027,7 @@ disable_clock:
 
 static int mipi_csis_pm_suspend(struct device *dev, bool runtime)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct v4l2_subdev *mipi_sd = platform_get_drvdata(pdev);
+	struct v4l2_subdev *mipi_sd = dev_get_drvdata(dev);
 	struct csi_state *state = mipi_sd_to_csis_state(mipi_sd);
 	int ret = 0;
 
@@ -1064,8 +1051,7 @@ unlock:
 
 static int mipi_csis_pm_resume(struct device *dev, bool runtime)
 {
-	struct platform_device *pdev = to_platform_device(dev);
-	struct v4l2_subdev *mipi_sd = platform_get_drvdata(pdev);
+	struct v4l2_subdev *mipi_sd = dev_get_drvdata(dev);
 	struct csi_state *state = mipi_sd_to_csis_state(mipi_sd);
 	int ret = 0;
 
diff --git a/drivers/staging/media/ipu3/Kconfig b/drivers/staging/media/ipu3/Kconfig
index f80f3e35f431..4b51c67eac88 100644
--- a/drivers/staging/media/ipu3/Kconfig
+++ b/drivers/staging/media/ipu3/Kconfig
@@ -6,7 +6,7 @@ config VIDEO_IPU3_IMGU
 	depends on X86
 	select IOMMU_IOVA
 	select VIDEOBUF2_DMA_SG
-	---help---
+	help
 	  This is the Video4Linux2 driver for Intel IPU3 image processing unit,
 	  found in Intel Skylake and Kaby Lake SoCs and used for processing
 	  images and video.
diff --git a/drivers/staging/media/ipu3/ipu3-css.c b/drivers/staging/media/ipu3/ipu3-css.c
index 15ab77e4b766..23cf5b2cfe8b 100644
--- a/drivers/staging/media/ipu3/ipu3-css.c
+++ b/drivers/staging/media/ipu3/ipu3-css.c
@@ -3,6 +3,7 @@
 
 #include <linux/device.h>
 #include <linux/iopoll.h>
+#include <linux/slab.h>
 
 #include "ipu3-css.h"
 #include "ipu3-css-fw.h"
@@ -1744,15 +1745,18 @@ int imgu_css_fmt_try(struct imgu_css *css,
 	struct v4l2_rect *const bds = &r[IPU3_CSS_RECT_BDS];
 	struct v4l2_rect *const env = &r[IPU3_CSS_RECT_ENVELOPE];
 	struct v4l2_rect *const gdc = &r[IPU3_CSS_RECT_GDC];
-	struct imgu_css_queue q[IPU3_CSS_QUEUES];
-	struct v4l2_pix_format_mplane *const in =
-					&q[IPU3_CSS_QUEUE_IN].fmt.mpix;
-	struct v4l2_pix_format_mplane *const out =
-					&q[IPU3_CSS_QUEUE_OUT].fmt.mpix;
-	struct v4l2_pix_format_mplane *const vf =
-					&q[IPU3_CSS_QUEUE_VF].fmt.mpix;
+	struct imgu_css_queue *q;
+	struct v4l2_pix_format_mplane *in, *out, *vf;
 	int i, s, ret;
 
+	q = kcalloc(IPU3_CSS_QUEUES, sizeof(struct imgu_css_queue), GFP_KERNEL);
+	if (!q)
+		return -ENOMEM;
+
+	in  = &q[IPU3_CSS_QUEUE_IN].fmt.mpix;
+	out = &q[IPU3_CSS_QUEUE_OUT].fmt.mpix;
+	vf  = &q[IPU3_CSS_QUEUE_VF].fmt.mpix;
+
 	/* Adjust all formats, get statistics buffer sizes and formats */
 	for (i = 0; i < IPU3_CSS_QUEUES; i++) {
 		if (fmts[i])
@@ -1766,7 +1770,8 @@ int imgu_css_fmt_try(struct imgu_css *css,
 					IPU3_CSS_QUEUE_TO_FLAGS(i))) {
 			dev_notice(css->dev, "can not initialize queue %s\n",
 				   qnames[i]);
-			return -EINVAL;
+			ret = -EINVAL;
+			goto out;
 		}
 	}
 	for (i = 0; i < IPU3_CSS_RECTS; i++) {
@@ -1788,7 +1793,8 @@ int imgu_css_fmt_try(struct imgu_css *css,
 	if (!imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_IN]) ||
 	    !imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_OUT])) {
 		dev_warn(css->dev, "required queues are disabled\n");
-		return -EINVAL;
+		ret = -EINVAL;
+		goto out;
 	}
 
 	if (!imgu_css_queue_enabled(&q[IPU3_CSS_QUEUE_OUT])) {
@@ -1829,7 +1835,8 @@ int imgu_css_fmt_try(struct imgu_css *css,
 	ret = imgu_css_find_binary(css, pipe, q, r);
 	if (ret < 0) {
 		dev_err(css->dev, "failed to find suitable binary\n");
-		return -EINVAL;
+		ret = -EINVAL;
+		goto out;
 	}
 	css->pipes[pipe].bindex = ret;
 
@@ -1843,7 +1850,8 @@ int imgu_css_fmt_try(struct imgu_css *css,
 						IPU3_CSS_QUEUE_TO_FLAGS(i))) {
 				dev_err(css->dev,
 					"final resolution adjustment failed\n");
-				return -EINVAL;
+				ret = -EINVAL;
+				goto out;
 			}
 			*fmts[i] = q[i].fmt.mpix;
 		}
@@ -1859,7 +1867,10 @@ int imgu_css_fmt_try(struct imgu_css *css,
 		 bds->width, bds->height, gdc->width, gdc->height,
 		 out->width, out->height, vf->width, vf->height);
 
-	return 0;
+	ret = 0;
+out:
+	kfree(q);
+	return ret;
 }
 
 int imgu_css_fmt_set(struct imgu_css *css,
@@ -2160,11 +2171,6 @@ int imgu_css_set_parameters(struct imgu_css *css, unsigned int pipe,
 	obgrid_size = imgu_css_fw_obgrid_size(bi);
 	stripes = bi->info.isp.sp.iterator.num_stripes ? : 1;
 
-	/*
-	 * TODO(b/118782861): If userspace queues more than 4 buffers, the
-	 * parameters from previous buffers will be overwritten. Fix the driver
-	 * not to allow this.
-	 */
 	imgu_css_pool_get(&css_pipe->pool.parameter_set_info);
 	param_set = imgu_css_pool_last(&css_pipe->pool.parameter_set_info,
 				       0)->vaddr;
diff --git a/drivers/staging/media/ipu3/ipu3-v4l2.c b/drivers/staging/media/ipu3/ipu3-v4l2.c
index 9c0352b193a7..a7bc22040ed8 100644
--- a/drivers/staging/media/ipu3/ipu3-v4l2.c
+++ b/drivers/staging/media/ipu3/ipu3-v4l2.c
@@ -66,7 +66,7 @@ static int imgu_subdev_s_stream(struct v4l2_subdev *sd, int enable)
 	struct imgu_css_pipe *css_pipe = &imgu->css.pipes[pipe];
 	struct imgu_media_pipe *imgu_pipe = &imgu->imgu_pipe[pipe];
 
-	dev_dbg(dev, "%s %d for pipe %d", __func__, enable, pipe);
+	dev_dbg(dev, "%s %d for pipe %u", __func__, enable, pipe);
 	/* grab ctrl after streamon and return after off */
 	v4l2_ctrl_grab(imgu_sd->ctrl, enable);
 
@@ -101,7 +101,7 @@ static int imgu_subdev_s_stream(struct v4l2_subdev *sd, int enable)
 	else
 		css_pipe->pipe_id = IPU3_CSS_PIPE_ID_CAPTURE;
 
-	dev_dbg(dev, "IPU3 pipe %d pipe_id %d", pipe, css_pipe->pipe_id);
+	dev_dbg(dev, "IPU3 pipe %u pipe_id %u", pipe, css_pipe->pipe_id);
 
 	rects[IPU3_CSS_RECT_EFFECTIVE] = &imgu_sd->rect.eff;
 	rects[IPU3_CSS_RECT_BDS] = &imgu_sd->rect.bds;
@@ -109,7 +109,7 @@ static int imgu_subdev_s_stream(struct v4l2_subdev *sd, int enable)
 
 	r = imgu_css_fmt_set(&imgu->css, fmts, rects, pipe);
 	if (r) {
-		dev_err(dev, "failed to set initial formats pipe %d with (%d)",
+		dev_err(dev, "failed to set initial formats pipe %u with (%d)",
 			pipe, r);
 		return r;
 	}
@@ -157,7 +157,7 @@ static int imgu_subdev_set_fmt(struct v4l2_subdev *sd,
 	u32 pad = fmt->pad;
 	unsigned int pipe = imgu_sd->pipe;
 
-	dev_dbg(&imgu->pci_dev->dev, "set subdev %d pad %d fmt to [%dx%d]",
+	dev_dbg(&imgu->pci_dev->dev, "set subdev %u pad %u fmt to [%ux%u]",
 		pipe, pad, fmt->format.width, fmt->format.height);
 
 	imgu_pipe = &imgu->imgu_pipe[pipe];
@@ -233,7 +233,7 @@ static int imgu_subdev_set_selection(struct v4l2_subdev *sd,
 	struct v4l2_rect *rect, *try_sel;
 
 	dev_dbg(&imgu->pci_dev->dev,
-		 "set subdev %d sel which %d target 0x%4x rect [%dx%d]",
+		 "set subdev %u sel which %u target 0x%4x rect [%ux%u]",
 		 imgu_sd->pipe, sel->which, sel->target,
 		 sel->r.width, sel->r.height);
 
@@ -279,7 +279,7 @@ static int imgu_link_setup(struct media_entity *entity,
 
 	WARN_ON(pad >= IMGU_NODE_NUM);
 
-	dev_dbg(&imgu->pci_dev->dev, "pipe %d pad %d is %s", pipe, pad,
+	dev_dbg(&imgu->pci_dev->dev, "pipe %u pad %u is %s", pipe, pad,
 		 flags & MEDIA_LNK_FL_ENABLED ? "enabled" : "disabled");
 
 	imgu_pipe = &imgu->imgu_pipe[pipe];
@@ -294,7 +294,7 @@ static int imgu_link_setup(struct media_entity *entity,
 	else
 		__clear_bit(pipe, imgu->css.enabled_pipes);
 
-	dev_dbg(&imgu->pci_dev->dev, "pipe %d is %s", pipe,
+	dev_dbg(&imgu->pci_dev->dev, "pipe %u is %s", pipe,
 		 flags & MEDIA_LNK_FL_ENABLED ? "enabled" : "disabled");
 
 	return 0;
@@ -341,8 +341,10 @@ static void imgu_vb2_buf_queue(struct vb2_buffer *vb)
 	struct imgu_video_device *node =
 		container_of(vb->vb2_queue, struct imgu_video_device, vbq);
 	unsigned int queue = imgu_node_to_queue(node->id);
+	struct imgu_buffer *buf = container_of(vb, struct imgu_buffer,
+					       vid_buf.vbb.vb2_buf);
 	unsigned long need_bytes;
-	unsigned int pipe = node->pipe;
+	unsigned long payload = vb2_get_plane_payload(vb, 0);
 
 	if (vb->vb2_queue->type == V4L2_BUF_TYPE_META_CAPTURE ||
 	    vb->vb2_queue->type == V4L2_BUF_TYPE_META_OUTPUT)
@@ -350,42 +352,26 @@ static void imgu_vb2_buf_queue(struct vb2_buffer *vb)
 	else
 		need_bytes = node->vdev_fmt.fmt.pix_mp.plane_fmt[0].sizeimage;
 
-	if (queue == IPU3_CSS_QUEUE_PARAMS) {
-		unsigned long payload = vb2_get_plane_payload(vb, 0);
-		struct vb2_v4l2_buffer *buf =
-			container_of(vb, struct vb2_v4l2_buffer, vb2_buf);
-		int r = -EINVAL;
-
-		if (payload == 0) {
-			payload = need_bytes;
-			vb2_set_plane_payload(vb, 0, payload);
-		}
-		if (payload >= need_bytes)
-			r = imgu_css_set_parameters(&imgu->css, pipe,
-						    vb2_plane_vaddr(vb, 0));
-		buf->flags = V4L2_BUF_FLAG_DONE;
-		vb2_buffer_done(vb, r == 0 ? VB2_BUF_STATE_DONE
-					   : VB2_BUF_STATE_ERROR);
-
-	} else {
-		struct imgu_buffer *buf = container_of(vb, struct imgu_buffer,
-						       vid_buf.vbb.vb2_buf);
+	if (queue == IPU3_CSS_QUEUE_PARAMS && payload && payload < need_bytes) {
+		dev_err(&imgu->pci_dev->dev, "invalid data size for params.");
+		vb2_buffer_done(vb, VB2_BUF_STATE_ERROR);
+		return;
+	}
 
-		mutex_lock(&imgu->lock);
+	mutex_lock(&imgu->lock);
+	if (queue != IPU3_CSS_QUEUE_PARAMS)
 		imgu_css_buf_init(&buf->css_buf, queue, buf->map.daddr);
-		list_add_tail(&buf->vid_buf.list,
-			      &node->buffers);
-		mutex_unlock(&imgu->lock);
 
-		vb2_set_plane_payload(&buf->vid_buf.vbb.vb2_buf, 0, need_bytes);
+	list_add_tail(&buf->vid_buf.list, &node->buffers);
+	mutex_unlock(&imgu->lock);
 
-		if (imgu->streaming)
-			imgu_queue_buffers(imgu, false, pipe);
-	}
+	vb2_set_plane_payload(vb, 0, need_bytes);
 
-	dev_dbg(&imgu->pci_dev->dev, "%s for pipe %d node %d", __func__,
-		node->pipe, node->id);
+	if (imgu->streaming)
+		imgu_queue_buffers(imgu, false, node->pipe);
 
+	dev_dbg(&imgu->pci_dev->dev, "%s for pipe %u node %u", __func__,
+		node->pipe, node->id);
 }
 
 static int imgu_vb2_queue_setup(struct vb2_queue *vq,
@@ -435,7 +421,7 @@ static bool imgu_all_nodes_streaming(struct imgu_device *imgu,
 	pipe = except->pipe;
 	if (!test_bit(pipe, imgu->css.enabled_pipes)) {
 		dev_warn(&imgu->pci_dev->dev,
-			 "pipe %d link is not ready yet", pipe);
+			 "pipe %u link is not ready yet", pipe);
 		return false;
 	}
 
@@ -479,7 +465,7 @@ static int imgu_vb2_start_streaming(struct vb2_queue *vq, unsigned int count)
 	int r;
 	unsigned int pipe;
 
-	dev_dbg(dev, "%s node name %s pipe %d id %u", __func__,
+	dev_dbg(dev, "%s node name %s pipe %u id %u", __func__,
 		node->name, node->pipe, node->id);
 
 	if (imgu->streaming) {
@@ -539,7 +525,7 @@ static void imgu_vb2_stop_streaming(struct vb2_queue *vq)
 	WARN_ON(!node->enabled);
 
 	pipe = node->pipe;
-	dev_dbg(dev, "Try to stream off node [%d][%d]", pipe, node->id);
+	dev_dbg(dev, "Try to stream off node [%u][%u]", pipe, node->id);
 	imgu_pipe = &imgu->imgu_pipe[pipe];
 	r = v4l2_subdev_call(&imgu_pipe->imgu_sd.subdev, video, s_stream, 0);
 	if (r)
@@ -664,20 +650,19 @@ static int imgu_fmt(struct imgu_device *imgu, unsigned int pipe, int node,
 		    struct v4l2_format *f, bool try)
 {
 	struct device *dev = &imgu->pci_dev->dev;
-	struct v4l2_pix_format_mplane try_fmts[IPU3_CSS_QUEUES];
 	struct v4l2_pix_format_mplane *fmts[IPU3_CSS_QUEUES] = { NULL };
 	struct v4l2_rect *rects[IPU3_CSS_RECTS] = { NULL };
 	struct v4l2_mbus_framefmt pad_fmt;
 	unsigned int i, css_q;
-	int r;
+	int ret;
 	struct imgu_css_pipe *css_pipe = &imgu->css.pipes[pipe];
 	struct imgu_media_pipe *imgu_pipe = &imgu->imgu_pipe[pipe];
 	struct imgu_v4l2_subdev *imgu_sd = &imgu_pipe->imgu_sd;
 
-	dev_dbg(dev, "set fmt node [%u][%u](try = %d)", pipe, node, try);
+	dev_dbg(dev, "set fmt node [%u][%u](try = %u)", pipe, node, try);
 
 	for (i = 0; i < IMGU_NODE_NUM; i++)
-		dev_dbg(dev, "IMGU pipe %d node %d enabled = %d",
+		dev_dbg(dev, "IMGU pipe %u node %u enabled = %u",
 			pipe, i, imgu_pipe->nodes[i].enabled);
 
 	if (imgu_pipe->nodes[IMGU_NODE_VF].enabled)
@@ -688,7 +673,7 @@ static int imgu_fmt(struct imgu_device *imgu, unsigned int pipe, int node,
 	else
 		css_pipe->pipe_id = IPU3_CSS_PIPE_ID_CAPTURE;
 
-	dev_dbg(dev, "IPU3 pipe %d pipe_id = %d", pipe, css_pipe->pipe_id);
+	dev_dbg(dev, "IPU3 pipe %u pipe_id = %u", pipe, css_pipe->pipe_id);
 
 	for (i = 0; i < IPU3_CSS_QUEUES; i++) {
 		unsigned int inode = imgu_map_node(imgu, i);
@@ -698,9 +683,13 @@ static int imgu_fmt(struct imgu_device *imgu, unsigned int pipe, int node,
 			continue;
 
 		if (try) {
-			try_fmts[i] =
-				imgu_pipe->nodes[inode].vdev_fmt.fmt.pix_mp;
-			fmts[i] = &try_fmts[i];
+			fmts[i] = kmemdup(&imgu_pipe->nodes[inode].vdev_fmt.fmt.pix_mp,
+					  sizeof(struct v4l2_pix_format_mplane),
+					  GFP_KERNEL);
+			if (!fmts[i]) {
+				ret = -ENOMEM;
+				goto out;
+			}
 		} else {
 			fmts[i] = &imgu_pipe->nodes[inode].vdev_fmt.fmt.pix_mp;
 		}
@@ -730,26 +719,33 @@ static int imgu_fmt(struct imgu_device *imgu, unsigned int pipe, int node,
 	 * before we return success from this function, so set it here.
 	 */
 	css_q = imgu_node_to_queue(node);
-	if (fmts[css_q])
-		*fmts[css_q] = f->fmt.pix_mp;
-	else
-		return -EINVAL;
+	if (!fmts[css_q]) {
+		ret = -EINVAL;
+		goto out;
+	}
+	*fmts[css_q] = f->fmt.pix_mp;
 
 	if (try)
-		r = imgu_css_fmt_try(&imgu->css, fmts, rects, pipe);
+		ret = imgu_css_fmt_try(&imgu->css, fmts, rects, pipe);
 	else
-		r = imgu_css_fmt_set(&imgu->css, fmts, rects, pipe);
+		ret = imgu_css_fmt_set(&imgu->css, fmts, rects, pipe);
 
-	/* r is the binary number in the firmware blob */
-	if (r < 0)
-		return r;
+	/* ret is the binary number in the firmware blob */
+	if (ret < 0)
+		goto out;
 
 	if (try)
 		f->fmt.pix_mp = *fmts[css_q];
 	else
 		f->fmt = imgu_pipe->nodes[node].vdev_fmt.fmt;
 
-	return 0;
+out:
+	if (try) {
+		for (i = 0; i < IPU3_CSS_QUEUES; i++)
+			kfree(fmts[i]);
+	}
+
+	return ret;
 }
 
 static int imgu_try_fmt(struct file *file, void *fh, struct v4l2_format *f)
@@ -781,7 +777,7 @@ static int imgu_vidioc_try_fmt(struct file *file, void *fh,
 	struct v4l2_pix_format_mplane *pix_mp = &f->fmt.pix_mp;
 	int r;
 
-	dev_dbg(dev, "%s [%ux%u] for node %d\n", __func__,
+	dev_dbg(dev, "%s [%ux%u] for node %u\n", __func__,
 		pix_mp->width, pix_mp->height, node->id);
 
 	r = imgu_try_fmt(file, fh, f);
@@ -799,7 +795,7 @@ static int imgu_vidioc_s_fmt(struct file *file, void *fh, struct v4l2_format *f)
 	struct v4l2_pix_format_mplane *pix_mp = &f->fmt.pix_mp;
 	int r;
 
-	dev_dbg(dev, "%s [%ux%u] for node %d\n", __func__,
+	dev_dbg(dev, "%s [%ux%u] for node %u\n", __func__,
 		pix_mp->width, pix_mp->height, node->id);
 
 	r = imgu_try_fmt(file, fh, f);
@@ -1022,7 +1018,7 @@ static int imgu_sd_s_ctrl(struct v4l2_ctrl *ctrl)
 	struct imgu_device *imgu = v4l2_get_subdevdata(&imgu_sd->subdev);
 	struct device *dev = &imgu->pci_dev->dev;
 
-	dev_dbg(dev, "set val %d to ctrl 0x%8x for subdev %d",
+	dev_dbg(dev, "set val %d to ctrl 0x%8x for subdev %u",
 		ctrl->val, ctrl->id, imgu_sd->pipe);
 
 	switch (ctrl->id) {
@@ -1122,7 +1118,7 @@ static int imgu_v4l2_subdev_register(struct imgu_device *imgu,
 	imgu_sd->subdev.flags = V4L2_SUBDEV_FL_HAS_DEVNODE |
 				V4L2_SUBDEV_FL_HAS_EVENTS;
 	snprintf(imgu_sd->subdev.name, sizeof(imgu_sd->subdev.name),
-		 "%s %d", IMGU_NAME, pipe);
+		 "%s %u", IMGU_NAME, pipe);
 	v4l2_set_subdevdata(&imgu_sd->subdev, imgu);
 	atomic_set(&imgu_sd->running_mode, IPU3_RUNNING_MODE_VIDEO);
 	v4l2_ctrl_handler_init(hdl, 1);
@@ -1240,7 +1236,7 @@ static int imgu_v4l2_node_setup(struct imgu_device *imgu, unsigned int pipe,
 	}
 
 	/* Initialize vdev */
-	snprintf(vdev->name, sizeof(vdev->name), "%s %d %s",
+	snprintf(vdev->name, sizeof(vdev->name), "%s %u %s",
 		 IMGU_NAME, pipe, node->name);
 	vdev->release = video_device_release_empty;
 	vdev->fops = &imgu_v4l2_fops;
@@ -1335,7 +1331,7 @@ static int imgu_v4l2_register_pipes(struct imgu_device *imgu)
 		r = imgu_v4l2_subdev_register(imgu, &imgu_pipe->imgu_sd, i);
 		if (r) {
 			dev_err(&imgu->pci_dev->dev,
-				"failed to register subdev%d ret (%d)\n", i, r);
+				"failed to register subdev%u ret (%d)\n", i, r);
 			goto pipes_cleanup;
 		}
 		r = imgu_v4l2_nodes_setup_pipe(imgu, i);
diff --git a/drivers/staging/media/ipu3/ipu3.c b/drivers/staging/media/ipu3/ipu3.c
index e0bbdad7cf5b..a7372395a101 100644
--- a/drivers/staging/media/ipu3/ipu3.c
+++ b/drivers/staging/media/ipu3/ipu3.c
@@ -236,6 +236,11 @@ int imgu_queue_buffers(struct imgu_device *imgu, bool initial, unsigned int pipe
 	dev_dbg(&imgu->pci_dev->dev, "Queue buffers to pipe %d", pipe);
 	mutex_lock(&imgu->lock);
 
+	if (!imgu_css_pipe_queue_empty(&imgu->css, pipe)) {
+		mutex_unlock(&imgu->lock);
+		return 0;
+	}
+
 	/* Buffer set is queued to FW only when input buffer is ready */
 	for (node = IMGU_NODE_NUM - 1;
 	     imgu_queue_getbuf(imgu, IMGU_NODE_IN, pipe);
@@ -245,6 +250,31 @@ int imgu_queue_buffers(struct imgu_device *imgu, bool initial, unsigned int pipe
 			dev_warn(&imgu->pci_dev->dev,
 				 "Vf not enabled, ignore queue");
 			continue;
+		} else if (node == IMGU_NODE_PARAMS &&
+			   imgu_pipe->nodes[node].enabled) {
+			struct vb2_buffer *vb;
+			struct imgu_vb2_buffer *ivb;
+
+			/* No parameters for this frame */
+			if (list_empty(&imgu_pipe->nodes[node].buffers))
+				continue;
+
+			ivb = list_first_entry(&imgu_pipe->nodes[node].buffers,
+					       struct imgu_vb2_buffer, list);
+			vb = &ivb->vbb.vb2_buf;
+			r = imgu_css_set_parameters(&imgu->css, pipe,
+						    vb2_plane_vaddr(vb, 0));
+			if (r) {
+				vb2_buffer_done(vb, VB2_BUF_STATE_ERROR);
+				dev_warn(&imgu->pci_dev->dev,
+					 "set parameters failed.");
+				continue;
+			}
+
+			vb2_buffer_done(vb, VB2_BUF_STATE_DONE);
+			dev_dbg(&imgu->pci_dev->dev,
+				"queue user parameters %d to css.", vb->index);
+			list_del(&ivb->list);
 		} else if (imgu_pipe->queue_enabled[node]) {
 			struct imgu_css_buffer *buf =
 				imgu_queue_getbuf(imgu, node, pipe);
@@ -790,7 +820,7 @@ out:
  * PCI rpm framework checks the existence of driver rpm callbacks.
  * Place a dummy callback here to avoid rpm going into error state.
  */
-static int imgu_rpm_dummy_cb(struct device *dev)
+static __maybe_unused int imgu_rpm_dummy_cb(struct device *dev)
 {
 	return 0;
 }
diff --git a/drivers/staging/media/mt9t031/Kconfig b/drivers/staging/media/mt9t031/Kconfig
deleted file mode 100644
index 232f0cdca68b..000000000000
--- a/drivers/staging/media/mt9t031/Kconfig
+++ /dev/null
@@ -1,6 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-config SOC_CAMERA_MT9T031
-	tristate "mt9t031 support (DEPRECATED)"
-	depends on SOC_CAMERA && I2C
-	help
-	  This driver supports MT9T031 cameras from Micron.
diff --git a/drivers/staging/media/mt9t031/Makefile b/drivers/staging/media/mt9t031/Makefile
deleted file mode 100644
index f663f73a5474..000000000000
--- a/drivers/staging/media/mt9t031/Makefile
+++ /dev/null
@@ -1,2 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-obj-$(CONFIG_SOC_CAMERA_MT9T031)		+= mt9t031.o
diff --git a/drivers/staging/media/mt9t031/TODO b/drivers/staging/media/mt9t031/TODO
deleted file mode 100644
index 15580a4f950c..000000000000
--- a/drivers/staging/media/mt9t031/TODO
+++ /dev/null
@@ -1,5 +0,0 @@
-This sensor driver needs to be converted to a regular
-v4l2 subdev driver. The soc_camera framework is deprecated and
-will be removed in the future. Unless someone does this work this
-sensor driver will be deleted when the soc_camera framework is
-deleted.
diff --git a/drivers/staging/media/omap4iss/Kconfig b/drivers/staging/media/omap4iss/Kconfig
index 841cc0b3ce13..4dcbc5065821 100644
--- a/drivers/staging/media/omap4iss/Kconfig
+++ b/drivers/staging/media/omap4iss/Kconfig
@@ -6,5 +6,5 @@ config VIDEO_OMAP4
 	depends on ARCH_OMAP4 || COMPILE_TEST
 	select MFD_SYSCON
 	select VIDEOBUF2_DMA_CONTIG
-	---help---
+	help
 	  Driver for an OMAP 4 ISS controller.
diff --git a/drivers/staging/media/rockchip/vpu/rockchip_vpu_drv.c b/drivers/staging/media/rockchip/vpu/rockchip_vpu_drv.c
index 962412c79b91..58721c46fba4 100644
--- a/drivers/staging/media/rockchip/vpu/rockchip_vpu_drv.c
+++ b/drivers/staging/media/rockchip/vpu/rockchip_vpu_drv.c
@@ -22,7 +22,6 @@
 #include <media/v4l2-event.h>
 #include <media/v4l2-mem2mem.h>
 #include <media/videobuf2-core.h>
-#include <media/videobuf2-core.h>
 #include <media/videobuf2-vmalloc.h>
 
 #include "rockchip_vpu_common.h"
@@ -463,7 +462,7 @@ static int rockchip_vpu_probe(struct platform_device *pdev)
 	}
 
 	vpu->mdev.dev = vpu->dev;
-	strlcpy(vpu->mdev.model, DRIVER_NAME, sizeof(vpu->mdev.model));
+	strscpy(vpu->mdev.model, DRIVER_NAME, sizeof(vpu->mdev.model));
 	media_device_init(&vpu->mdev);
 	vpu->v4l2_dev.mdev = &vpu->mdev;
 
diff --git a/drivers/staging/media/rockchip/vpu/rockchip_vpu_enc.c b/drivers/staging/media/rockchip/vpu/rockchip_vpu_enc.c
index ab0fb2053620..fb5e36aedd8c 100644
--- a/drivers/staging/media/rockchip/vpu/rockchip_vpu_enc.c
+++ b/drivers/staging/media/rockchip/vpu/rockchip_vpu_enc.c
@@ -41,7 +41,7 @@
  * @is_compressed: Is it a compressed format?
  * @multiplanar: Is it a multiplanar variant format? (e.g. NV12M)
  */
-struct v4l2_format_info {
+struct rockchip_vpu_v4l2_format_info {
 	u32 format;
 	u32 header_size;
 	u8 num_planes;
@@ -52,10 +52,10 @@ struct v4l2_format_info {
 	u8 multiplanar;
 };
 
-static const struct v4l2_format_info *
-v4l2_format_info(u32 format)
+static const struct rockchip_vpu_v4l2_format_info *
+rockchip_vpu_v4l2_format_info(u32 format)
 {
-	static const struct v4l2_format_info formats[] = {
+	static const struct rockchip_vpu_v4l2_format_info formats[] = {
 		{ .format = V4L2_PIX_FMT_YUV420M,	.num_planes = 3, .cpp = { 1, 1, 1 }, .hsub = 2, .vsub = 2, .multiplanar = 1 },
 		{ .format = V4L2_PIX_FMT_NV12M,		.num_planes = 2, .cpp = { 1, 2, 0 }, .hsub = 2, .vsub = 2, .multiplanar = 1 },
 		{ .format = V4L2_PIX_FMT_YUYV,		.num_planes = 1, .cpp = { 2, 0, 0 }, .hsub = 2, .vsub = 1 },
@@ -76,11 +76,11 @@ static void
 fill_pixfmt_mp(struct v4l2_pix_format_mplane *pixfmt,
 	       int pixelformat, int width, int height)
 {
-	const struct v4l2_format_info *info;
+	const struct rockchip_vpu_v4l2_format_info *info;
 	struct v4l2_plane_pix_format *plane;
 	int i;
 
-	info = v4l2_format_info(pixelformat);
+	info = rockchip_vpu_v4l2_format_info(pixelformat);
 	if (!info)
 		return;
 
diff --git a/drivers/staging/media/soc_camera/TODO b/drivers/staging/media/soc_camera/TODO
new file mode 100644
index 000000000000..932af6443b67
--- /dev/null
+++ b/drivers/staging/media/soc_camera/TODO
@@ -0,0 +1,4 @@
+The SoC camera framework is obsolete and scheduled for removal in the near
+future. Developers are encouraged to convert the drivers to use the
+regular V4L2 API if these drivers are still needed (and if someone has the
+hardware).
diff --git a/drivers/staging/media/sunxi/cedrus/cedrus.c b/drivers/staging/media/sunxi/cedrus/cedrus.c
index ff11cbeba205..d0429c0e6b6b 100644
--- a/drivers/staging/media/sunxi/cedrus/cedrus.c
+++ b/drivers/staging/media/sunxi/cedrus/cedrus.c
@@ -300,7 +300,7 @@ static int cedrus_probe(struct platform_device *pdev)
 			 "Failed to initialize V4L2 M2M device\n");
 		ret = PTR_ERR(dev->m2m_dev);
 
-		goto err_video;
+		goto err_v4l2;
 	}
 
 	dev->mdev.dev = &pdev->dev;
@@ -310,23 +310,23 @@ static int cedrus_probe(struct platform_device *pdev)
 	dev->mdev.ops = &cedrus_m2m_media_ops;
 	dev->v4l2_dev.mdev = &dev->mdev;
 
-	ret = v4l2_m2m_register_media_controller(dev->m2m_dev, vfd,
-						 MEDIA_ENT_F_PROC_VIDEO_DECODER);
-	if (ret) {
-		v4l2_err(&dev->v4l2_dev,
-			 "Failed to initialize V4L2 M2M media controller\n");
-		goto err_m2m;
-	}
-
 	ret = video_register_device(vfd, VFL_TYPE_GRABBER, 0);
 	if (ret) {
 		v4l2_err(&dev->v4l2_dev, "Failed to register video device\n");
-		goto err_v4l2;
+		goto err_m2m;
 	}
 
 	v4l2_info(&dev->v4l2_dev,
 		  "Device registered as /dev/video%d\n", vfd->num);
 
+	ret = v4l2_m2m_register_media_controller(dev->m2m_dev, vfd,
+						 MEDIA_ENT_F_PROC_VIDEO_DECODER);
+	if (ret) {
+		v4l2_err(&dev->v4l2_dev,
+			 "Failed to initialize V4L2 M2M media controller\n");
+		goto err_video;
+	}
+
 	ret = media_device_register(&dev->mdev);
 	if (ret) {
 		v4l2_err(&dev->v4l2_dev, "Failed to register media device\n");
@@ -339,10 +339,10 @@ static int cedrus_probe(struct platform_device *pdev)
 
 err_m2m_mc:
 	v4l2_m2m_unregister_media_controller(dev->m2m_dev);
-err_m2m:
-	v4l2_m2m_release(dev->m2m_dev);
 err_video:
 	video_unregister_device(&dev->vfd);
+err_m2m:
+	v4l2_m2m_release(dev->m2m_dev);
 err_v4l2:
 	v4l2_device_unregister(&dev->v4l2_dev);
 
@@ -396,6 +396,11 @@ static const struct cedrus_variant sun50i_h5_cedrus_variant = {
 	.capabilities	= CEDRUS_CAPABILITY_UNTILED,
 };
 
+static const struct cedrus_variant sun50i_h6_cedrus_variant = {
+	.capabilities	= CEDRUS_CAPABILITY_UNTILED,
+	.quirks		= CEDRUS_QUIRK_NO_DMA_OFFSET,
+};
+
 static const struct of_device_id cedrus_dt_match[] = {
 	{
 		.compatible = "allwinner,sun4i-a10-video-engine",
@@ -425,6 +430,10 @@ static const struct of_device_id cedrus_dt_match[] = {
 		.compatible = "allwinner,sun50i-h5-video-engine",
 		.data = &sun50i_h5_cedrus_variant,
 	},
+	{
+		.compatible = "allwinner,sun50i-h6-video-engine",
+		.data = &sun50i_h6_cedrus_variant,
+	},
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, cedrus_dt_match);
diff --git a/drivers/staging/media/sunxi/cedrus/cedrus.h b/drivers/staging/media/sunxi/cedrus/cedrus.h
index 4aedd24a9848..c57c04b41d2e 100644
--- a/drivers/staging/media/sunxi/cedrus/cedrus.h
+++ b/drivers/staging/media/sunxi/cedrus/cedrus.h
@@ -28,6 +28,8 @@
 
 #define CEDRUS_CAPABILITY_UNTILED	BIT(0)
 
+#define CEDRUS_QUIRK_NO_DMA_OFFSET	BIT(0)
+
 enum cedrus_codec {
 	CEDRUS_CODEC_MPEG2,
 
@@ -91,6 +93,7 @@ struct cedrus_dec_ops {
 
 struct cedrus_variant {
 	unsigned int	capabilities;
+	unsigned int	quirks;
 };
 
 struct cedrus_dev {
diff --git a/drivers/staging/media/sunxi/cedrus/cedrus_hw.c b/drivers/staging/media/sunxi/cedrus/cedrus_hw.c
index 0acf219a8c91..fbfff7c1c771 100644
--- a/drivers/staging/media/sunxi/cedrus/cedrus_hw.c
+++ b/drivers/staging/media/sunxi/cedrus/cedrus_hw.c
@@ -177,7 +177,8 @@ int cedrus_hw_probe(struct cedrus_dev *dev)
 	 */
 
 #ifdef PHYS_PFN_OFFSET
-	dev->dev->dma_pfn_offset = PHYS_PFN_OFFSET;
+	if (!(variant->quirks & CEDRUS_QUIRK_NO_DMA_OFFSET))
+		dev->dev->dma_pfn_offset = PHYS_PFN_OFFSET;
 #endif
 
 	ret = of_reserved_mem_device_init(dev->dev);
diff --git a/drivers/staging/media/sunxi/cedrus/cedrus_video.c b/drivers/staging/media/sunxi/cedrus/cedrus_video.c
index b47854b3bce4..9673874ece10 100644
--- a/drivers/staging/media/sunxi/cedrus/cedrus_video.c
+++ b/drivers/staging/media/sunxi/cedrus/cedrus_video.c
@@ -536,6 +536,7 @@ int cedrus_queue_init(void *priv, struct vb2_queue *src_vq,
 	src_vq->lock = &ctx->dev->dev_mutex;
 	src_vq->dev = ctx->dev->dev;
 	src_vq->supports_requests = true;
+	src_vq->requires_requests = true;
 
 	ret = vb2_queue_init(src_vq);
 	if (ret)
diff --git a/drivers/staging/media/zoran/Kconfig b/drivers/staging/media/zoran/Kconfig
deleted file mode 100644
index 84502b0fe48a..000000000000
--- a/drivers/staging/media/zoran/Kconfig
+++ /dev/null
@@ -1,76 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-config VIDEO_ZORAN
-	tristate "Zoran ZR36057/36067 Video For Linux (Deprecated)"
-	depends on PCI && I2C_ALGOBIT && VIDEO_V4L2 && VIRT_TO_BUS
-	depends on !ALPHA
-	help
-	  Say Y for support for MJPEG capture cards based on the Zoran
-	  36057/36067 PCI controller chipset. This includes the Iomega
-	  Buz, Pinnacle DC10+ and the Linux Media Labs LML33. There is
-	  a driver homepage at <http://mjpeg.sf.net/driver-zoran/>. For
-	  more information, check <file:Documentation/media/v4l-drivers/zoran.rst>.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called zr36067.
-
-config VIDEO_ZORAN_DC30
-	tristate "Pinnacle/Miro DC30(+) support"
-	depends on VIDEO_ZORAN
-	select VIDEO_ADV7175 if MEDIA_SUBDRV_AUTOSELECT
-	select VIDEO_VPX3220 if MEDIA_SUBDRV_AUTOSELECT
-	help
-	  Support for the Pinnacle/Miro DC30(+) MJPEG capture/playback
-	  card. This also supports really old DC10 cards based on the
-	  zr36050 MJPEG codec and zr36016 VFE.
-
-config VIDEO_ZORAN_ZR36060
-	tristate "Zoran ZR36060"
-	depends on VIDEO_ZORAN
-	help
-	  Say Y to support Zoran boards based on 36060 chips.
-	  This includes Iomega Buz, Pinnacle DC10, Linux media Labs 33
-	  and 33 R10 and AverMedia 6 boards.
-
-config VIDEO_ZORAN_BUZ
-	tristate "Iomega Buz support"
-	depends on VIDEO_ZORAN_ZR36060
-	select VIDEO_SAA711X if MEDIA_SUBDRV_AUTOSELECT
-	select VIDEO_SAA7185 if MEDIA_SUBDRV_AUTOSELECT
-	help
-	  Support for the Iomega Buz MJPEG capture/playback card.
-
-config VIDEO_ZORAN_DC10
-	tristate "Pinnacle/Miro DC10(+) support"
-	depends on VIDEO_ZORAN_ZR36060
-	select VIDEO_SAA7110 if MEDIA_SUBDRV_AUTOSELECT
-	select VIDEO_ADV7175 if MEDIA_SUBDRV_AUTOSELECT
-	help
-	  Support for the Pinnacle/Miro DC10(+) MJPEG capture/playback
-	  card.
-
-config VIDEO_ZORAN_LML33
-	tristate "Linux Media Labs LML33 support"
-	depends on VIDEO_ZORAN_ZR36060
-	select VIDEO_BT819 if MEDIA_SUBDRV_AUTOSELECT
-	select VIDEO_BT856 if MEDIA_SUBDRV_AUTOSELECT
-	help
-	  Support for the Linux Media Labs LML33 MJPEG capture/playback
-	  card.
-
-config VIDEO_ZORAN_LML33R10
-	tristate "Linux Media Labs LML33R10 support"
-	depends on VIDEO_ZORAN_ZR36060
-	select VIDEO_SAA711X if MEDIA_SUBDRV_AUTOSELECT
-	select VIDEO_ADV7170 if MEDIA_SUBDRV_AUTOSELECT
-	help
-	  support for the Linux Media Labs LML33R10 MJPEG capture/playback
-	  card.
-
-config VIDEO_ZORAN_AVS6EYES
-	tristate "AverMedia 6 Eyes support"
-	depends on VIDEO_ZORAN_ZR36060
-	select VIDEO_BT856 if MEDIA_SUBDRV_AUTOSELECT
-	select VIDEO_BT866 if MEDIA_SUBDRV_AUTOSELECT
-	select VIDEO_KS0127 if MEDIA_SUBDRV_AUTOSELECT
-	help
-	  Support for the AverMedia 6 Eyes video surveillance card.
diff --git a/drivers/staging/media/zoran/Makefile b/drivers/staging/media/zoran/Makefile
deleted file mode 100644
index 21ac29a71458..000000000000
--- a/drivers/staging/media/zoran/Makefile
+++ /dev/null
@@ -1,7 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-zr36067-objs	:=	zoran_procfs.o zoran_device.o \
-			zoran_driver.o zoran_card.o
-
-obj-$(CONFIG_VIDEO_ZORAN) += zr36067.o videocodec.o
-obj-$(CONFIG_VIDEO_ZORAN_DC30) += zr36050.o zr36016.o
-obj-$(CONFIG_VIDEO_ZORAN_ZR36060) += zr36060.o
diff --git a/drivers/staging/media/zoran/TODO b/drivers/staging/media/zoran/TODO
deleted file mode 100644
index 54464095d0d7..000000000000
--- a/drivers/staging/media/zoran/TODO
+++ /dev/null
@@ -1,4 +0,0 @@
-The zoran driver is marked deprecated. It will be removed
-around May 2019 unless someone is willing to update this
-driver to the latest V4L2 frameworks (especially the vb2
-framework).
diff --git a/drivers/staging/media/zoran/videocodec.c b/drivers/staging/media/zoran/videocodec.c
deleted file mode 100644
index c1ee5cb7e66b..000000000000
--- a/drivers/staging/media/zoran/videocodec.c
+++ /dev/null
@@ -1,379 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0+
-/*
- * VIDEO MOTION CODECs internal API for video devices
- *
- * Interface for MJPEG (and maybe later MPEG/WAVELETS) codec's
- * bound to a master device.
- *
- * (c) 2002 Wolfgang Scherr <scherr@net4you.at>
- *
- * $Id: videocodec.c,v 1.1.2.8 2003/03/29 07:16:04 rbultje Exp $
- * ------------------------------------------------------------------------
- */
-
-#define VIDEOCODEC_VERSION "v0.2"
-
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/init.h>
-#include <linux/types.h>
-#include <linux/slab.h>
-
-// kernel config is here (procfs flag)
-
-#ifdef CONFIG_PROC_FS
-#include <linux/proc_fs.h>
-#include <linux/seq_file.h>
-#include <linux/uaccess.h>
-#endif
-
-#include "videocodec.h"
-
-static int debug;
-module_param(debug, int, 0);
-MODULE_PARM_DESC(debug, "Debug level (0-4)");
-
-#define dprintk(num, format, args...) \
-	do { \
-		if (debug >= num) \
-			printk(format, ##args); \
-	} while (0)
-
-struct attached_list {
-	struct videocodec *codec;
-	struct attached_list *next;
-};
-
-struct codec_list {
-	const struct videocodec *codec;
-	int attached;
-	struct attached_list *list;
-	struct codec_list *next;
-};
-
-static struct codec_list *codeclist_top = NULL;
-
-/* ================================================= */
-/* function prototypes of the master/slave interface */
-/* ================================================= */
-
-struct videocodec *
-videocodec_attach (struct videocodec_master *master)
-{
-	struct codec_list *h = codeclist_top;
-	struct attached_list *a, *ptr;
-	struct videocodec *codec;
-	int res;
-
-	if (!master) {
-		dprintk(1, KERN_ERR "videocodec_attach: no data\n");
-		return NULL;
-	}
-
-	dprintk(2,
-		"videocodec_attach: '%s', flags %lx, magic %lx\n",
-		master->name, master->flags, master->magic);
-
-	if (!h) {
-		dprintk(1,
-			KERN_ERR
-			"videocodec_attach: no device available\n");
-		return NULL;
-	}
-
-	while (h) {
-		// attach only if the slave has at least the flags
-		// expected by the master
-		if ((master->flags & h->codec->flags) == master->flags) {
-			dprintk(4, "videocodec_attach: try '%s'\n",
-				h->codec->name);
-
-			if (!try_module_get(h->codec->owner))
-				return NULL;
-
-			codec = kmemdup(h->codec, sizeof(struct videocodec),
-					GFP_KERNEL);
-			if (!codec) {
-				dprintk(1,
-					KERN_ERR
-					"videocodec_attach: no mem\n");
-				goto out_module_put;
-			}
-
-			res = strlen(codec->name);
-			snprintf(codec->name + res, sizeof(codec->name) - res,
-				 "[%d]", h->attached);
-			codec->master_data = master;
-			res = codec->setup(codec);
-			if (res == 0) {
-				dprintk(3, "videocodec_attach '%s'\n",
-					codec->name);
-				ptr = kzalloc(sizeof(struct attached_list), GFP_KERNEL);
-				if (!ptr) {
-					dprintk(1,
-						KERN_ERR
-						"videocodec_attach: no memory\n");
-					goto out_kfree;
-				}
-				ptr->codec = codec;
-
-				a = h->list;
-				if (!a) {
-					h->list = ptr;
-					dprintk(4,
-						"videocodec: first element\n");
-				} else {
-					while (a->next)
-						a = a->next;	// find end
-					a->next = ptr;
-					dprintk(4,
-						"videocodec: in after '%s'\n",
-						h->codec->name);
-				}
-
-				h->attached += 1;
-				return codec;
-			} else {
-				kfree(codec);
-			}
-		}
-		h = h->next;
-	}
-
-	dprintk(1, KERN_ERR "videocodec_attach: no codec found!\n");
-	return NULL;
-
- out_module_put:
-	module_put(h->codec->owner);
- out_kfree:
-	kfree(codec);
-	return NULL;
-}
-
-int
-videocodec_detach (struct videocodec *codec)
-{
-	struct codec_list *h = codeclist_top;
-	struct attached_list *a, *prev;
-	int res;
-
-	if (!codec) {
-		dprintk(1, KERN_ERR "videocodec_detach: no data\n");
-		return -EINVAL;
-	}
-
-	dprintk(2,
-		"videocodec_detach: '%s', type: %x, flags %lx, magic %lx\n",
-		codec->name, codec->type, codec->flags, codec->magic);
-
-	if (!h) {
-		dprintk(1,
-			KERN_ERR "videocodec_detach: no device left...\n");
-		return -ENXIO;
-	}
-
-	while (h) {
-		a = h->list;
-		prev = NULL;
-		while (a) {
-			if (codec == a->codec) {
-				res = a->codec->unset(a->codec);
-				if (res >= 0) {
-					dprintk(3,
-						"videocodec_detach: '%s'\n",
-						a->codec->name);
-					a->codec->master_data = NULL;
-				} else {
-					dprintk(1,
-						KERN_ERR
-						"videocodec_detach: '%s'\n",
-						a->codec->name);
-					a->codec->master_data = NULL;
-				}
-				if (prev == NULL) {
-					h->list = a->next;
-					dprintk(4,
-						"videocodec: delete first\n");
-				} else {
-					prev->next = a->next;
-					dprintk(4,
-						"videocodec: delete middle\n");
-				}
-				module_put(a->codec->owner);
-				kfree(a->codec);
-				kfree(a);
-				h->attached -= 1;
-				return 0;
-			}
-			prev = a;
-			a = a->next;
-		}
-		h = h->next;
-	}
-
-	dprintk(1, KERN_ERR "videocodec_detach: given codec not found!\n");
-	return -EINVAL;
-}
-
-int
-videocodec_register (const struct videocodec *codec)
-{
-	struct codec_list *ptr, *h = codeclist_top;
-
-	if (!codec) {
-		dprintk(1, KERN_ERR "videocodec_register: no data!\n");
-		return -EINVAL;
-	}
-
-	dprintk(2,
-		"videocodec: register '%s', type: %x, flags %lx, magic %lx\n",
-		codec->name, codec->type, codec->flags, codec->magic);
-
-	ptr = kzalloc(sizeof(struct codec_list), GFP_KERNEL);
-	if (!ptr) {
-		dprintk(1, KERN_ERR "videocodec_register: no memory\n");
-		return -ENOMEM;
-	}
-	ptr->codec = codec;
-
-	if (!h) {
-		codeclist_top = ptr;
-		dprintk(4, "videocodec: hooked in as first element\n");
-	} else {
-		while (h->next)
-			h = h->next;	// find the end
-		h->next = ptr;
-		dprintk(4, "videocodec: hooked in after '%s'\n",
-			h->codec->name);
-	}
-
-	return 0;
-}
-
-int
-videocodec_unregister (const struct videocodec *codec)
-{
-	struct codec_list *prev = NULL, *h = codeclist_top;
-
-	if (!codec) {
-		dprintk(1, KERN_ERR "videocodec_unregister: no data!\n");
-		return -EINVAL;
-	}
-
-	dprintk(2,
-		"videocodec: unregister '%s', type: %x, flags %lx, magic %lx\n",
-		codec->name, codec->type, codec->flags, codec->magic);
-
-	if (!h) {
-		dprintk(1,
-			KERN_ERR
-			"videocodec_unregister: no device left...\n");
-		return -ENXIO;
-	}
-
-	while (h) {
-		if (codec == h->codec) {
-			if (h->attached) {
-				dprintk(1,
-					KERN_ERR
-					"videocodec: '%s' is used\n",
-					h->codec->name);
-				return -EBUSY;
-			}
-			dprintk(3, "videocodec: unregister '%s' is ok.\n",
-				h->codec->name);
-			if (prev == NULL) {
-				codeclist_top = h->next;
-				dprintk(4,
-					"videocodec: delete first element\n");
-			} else {
-				prev->next = h->next;
-				dprintk(4,
-					"videocodec: delete middle element\n");
-			}
-			kfree(h);
-			return 0;
-		}
-		prev = h;
-		h = h->next;
-	}
-
-	dprintk(1,
-		KERN_ERR
-		"videocodec_unregister: given codec not found!\n");
-	return -EINVAL;
-}
-
-#ifdef CONFIG_PROC_FS
-static int proc_videocodecs_show(struct seq_file *m, void *v)
-{
-	struct codec_list *h = codeclist_top;
-	struct attached_list *a;
-
-	seq_printf(m, "<S>lave or attached <M>aster name  type flags    magic    ");
-	seq_printf(m, "(connected as)\n");
-
-	while (h) {
-		seq_printf(m, "S %32s %04x %08lx %08lx (TEMPLATE)\n",
-			      h->codec->name, h->codec->type,
-			      h->codec->flags, h->codec->magic);
-		a = h->list;
-		while (a) {
-			seq_printf(m, "M %32s %04x %08lx %08lx (%s)\n",
-				      a->codec->master_data->name,
-				      a->codec->master_data->type,
-				      a->codec->master_data->flags,
-				      a->codec->master_data->magic,
-				      a->codec->name);
-			a = a->next;
-		}
-		h = h->next;
-	}
-
-	return 0;
-}
-#endif
-
-/* ===================== */
-/* hook in driver module */
-/* ===================== */
-static int __init
-videocodec_init (void)
-{
-#ifdef CONFIG_PROC_FS
-	static struct proc_dir_entry *videocodec_proc_entry;
-#endif
-
-	printk(KERN_INFO "Linux video codec intermediate layer: %s\n",
-	       VIDEOCODEC_VERSION);
-
-#ifdef CONFIG_PROC_FS
-	videocodec_proc_entry = proc_create_single("videocodecs", 0, NULL,
-			proc_videocodecs_show);
-	if (!videocodec_proc_entry) {
-		dprintk(1, KERN_ERR "videocodec: can't init procfs.\n");
-	}
-#endif
-	return 0;
-}
-
-static void __exit
-videocodec_exit (void)
-{
-#ifdef CONFIG_PROC_FS
-	remove_proc_entry("videocodecs", NULL);
-#endif
-}
-
-EXPORT_SYMBOL(videocodec_attach);
-EXPORT_SYMBOL(videocodec_detach);
-EXPORT_SYMBOL(videocodec_register);
-EXPORT_SYMBOL(videocodec_unregister);
-
-module_init(videocodec_init);
-module_exit(videocodec_exit);
-
-MODULE_AUTHOR("Wolfgang Scherr <scherr@net4you.at>");
-MODULE_DESCRIPTION("Intermediate API module for video codecs "
-		   VIDEOCODEC_VERSION);
-MODULE_LICENSE("GPL");
diff --git a/drivers/staging/media/zoran/videocodec.h b/drivers/staging/media/zoran/videocodec.h
deleted file mode 100644
index 4946791fce0d..000000000000
--- a/drivers/staging/media/zoran/videocodec.h
+++ /dev/null
@@ -1,334 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0+ */
-/*
- * VIDEO MOTION CODECs internal API for video devices
- *
- * Interface for MJPEG (and maybe later MPEG/WAVELETS) codec's
- * bound to a master device.
- *
- * (c) 2002 Wolfgang Scherr <scherr@net4you.at>
- */
-
-/* =================== */
-/* general description */
-/* =================== */
-
-/* Should ease the (re-)usage of drivers supporting cards with (different)
-   video codecs. The codecs register to this module their functionality,
-   and the processors (masters) can attach to them if they fit.
-
-   The codecs are typically have a "strong" binding to their master - so I
-   don't think it makes sense to have a full blown interfacing as with e.g.
-   i2c. If you have an other opinion, let's discuss & implement it :-)))
-
-   Usage:
-
-   The slave has just to setup the videocodec structure and use two functions:
-   videocodec_register(codecdata);
-   videocodec_unregister(codecdata);
-   The best is just calling them at module (de-)initialisation.
-
-   The master sets up the structure videocodec_master and calls:
-   codecdata=videocodec_attach(master_codecdata);
-   videocodec_detach(codecdata);
-
-   The slave is called during attach/detach via functions setup previously
-   during register. At that time, the master_data pointer is set up
-   and the slave can access any io registers of the master device (in the case
-   the slave is bound to it). Otherwise it doesn't need this functions and
-   therfor they may not be initialized.
-
-   The other functions are just for convenience, as they are for sure used by
-   most/all of the codecs. The last ones may be omitted, too.
-
-   See the structure declaration below for more information and which data has
-   to be set up for the master and the slave.
-
-   ----------------------------------------------------------------------------
-   The master should have "knowledge" of the slave and vice versa.  So the data
-   structures sent to/from slave via set_data/get_data set_image/get_image are
-   device dependent and vary between MJPEG/MPEG/WAVELET/... devices. (!!!!)
-   ----------------------------------------------------------------------------
-*/
-
-
-/* ========================================== */
-/* description of the videocodec_io structure */
-/* ========================================== */
-
-/*
-   ==== master setup ====
-   name -> name of the device structure for reference and debugging
-   master_data ->  data ref. for the master (e.g. the zr36055,57,67)
-   readreg -> ref. to read-fn from register (setup by master, used by slave)
-   writereg -> ref. to write-fn to register (setup by master, used by slave)
-	       this two functions do the lowlevel I/O job
-
-   ==== slave functionality setup ====
-   slave_data -> data ref. for the slave (e.g. the zr36050,60)
-   check -> fn-ref. checks availability of an device, returns -EIO on failure or
-	    the type on success
-	    this makes espcecially sense if a driver module supports more than
-	    one codec which may be quite similar to access, nevertheless it
-	    is good for a first functionality check
-
-   -- main functions you always need for compression/decompression --
-
-   set_mode -> this fn-ref. resets the entire codec, and sets up the mode
-	       with the last defined norm/size (or device default if not
-	       available) - it returns 0 if the mode is possible
-   set_size -> this fn-ref. sets the norm and image size for
-	       compression/decompression (returns 0 on success)
-	       the norm param is defined in videodev2.h (V4L2_STD_*)
-
-   additional setup may be available, too - but the codec should work with
-   some default values even without this
-
-   set_data -> sets device-specific data (tables, quality etc.)
-   get_data -> query device-specific data (tables, quality etc.)
-
-   if the device delivers interrupts, they may be setup/handled here
-   setup_interrupt -> codec irq setup (not needed for 36050/60)
-   handle_interrupt -> codec irq handling (not needed for 36050/60)
-
-   if the device delivers pictures, they may be handled here
-   put_image -> puts image data to the codec (not needed for 36050/60)
-   get_image -> gets image data from the codec (not needed for 36050/60)
-		the calls include frame numbers and flags (even/odd/...)
-		if needed and a flag which allows blocking until its ready
-*/
-
-/* ============== */
-/* user interface */
-/* ============== */
-
-/*
-   Currently there is only a information display planned, as the layer
-   is not visible for the user space at all.
-
-   Information is available via procfs. The current entry is "/proc/videocodecs"
-   but it makes sense to "hide" it in the /proc/video tree of v4l(2) --TODO--.
-
-A example for such an output is:
-
-<S>lave or attached <M>aster name  type flags    magic    (connected as)
-S                          zr36050 0002 0000d001 00000000 (TEMPLATE)
-M                       zr36055[0] 0001 0000c001 00000000 (zr36050[0])
-M                       zr36055[1] 0001 0000c001 00000000 (zr36050[1])
-
-*/
-
-
-/* =============================================== */
-/* special defines for the videocodec_io structure */
-/* =============================================== */
-
-#ifndef __LINUX_VIDEOCODEC_H
-#define __LINUX_VIDEOCODEC_H
-
-#include <linux/videodev2.h>
-
-#define CODEC_DO_COMPRESSION 0
-#define CODEC_DO_EXPANSION   1
-
-/* this are the current codec flags I think they are needed */
-/*  -> type value in structure */
-#define CODEC_FLAG_JPEG      0x00000001L	// JPEG codec
-#define CODEC_FLAG_MPEG      0x00000002L	// MPEG1/2/4 codec
-#define CODEC_FLAG_DIVX      0x00000004L	// DIVX codec
-#define CODEC_FLAG_WAVELET   0x00000008L	// WAVELET codec
-					  // room for other types
-
-#define CODEC_FLAG_MAGIC     0x00000800L	// magic key must match
-#define CODEC_FLAG_HARDWARE  0x00001000L	// is a hardware codec
-#define CODEC_FLAG_VFE       0x00002000L	// has direct video frontend
-#define CODEC_FLAG_ENCODER   0x00004000L	// compression capability
-#define CODEC_FLAG_DECODER   0x00008000L	// decompression capability
-#define CODEC_FLAG_NEEDIRQ   0x00010000L	// needs irq handling
-#define CODEC_FLAG_RDWRPIC   0x00020000L	// handles picture I/O
-
-/* a list of modes, some are just examples (is there any HW?) */
-#define CODEC_MODE_BJPG      0x0001	// Baseline JPEG
-#define CODEC_MODE_LJPG      0x0002	// Lossless JPEG
-#define CODEC_MODE_MPEG1     0x0003	// MPEG 1
-#define CODEC_MODE_MPEG2     0x0004	// MPEG 2
-#define CODEC_MODE_MPEG4     0x0005	// MPEG 4
-#define CODEC_MODE_MSDIVX    0x0006	// MS DivX
-#define CODEC_MODE_ODIVX     0x0007	// Open DivX
-#define CODEC_MODE_WAVELET   0x0008	// Wavelet
-
-/* this are the current codec types I want to implement */
-/*  -> type value in structure */
-#define CODEC_TYPE_NONE    0
-#define CODEC_TYPE_L64702  1
-#define CODEC_TYPE_ZR36050 2
-#define CODEC_TYPE_ZR36016 3
-#define CODEC_TYPE_ZR36060 4
-
-/* the type of data may be enhanced by future implementations (data-fn.'s) */
-/*  -> used in command                                                     */
-#define CODEC_G_STATUS         0x0000	/* codec status (query only) */
-#define CODEC_S_CODEC_MODE     0x0001	/* codec mode (baseline JPEG, MPEG1,... */
-#define CODEC_G_CODEC_MODE     0x8001
-#define CODEC_S_VFE            0x0002	/* additional video frontend setup */
-#define CODEC_G_VFE            0x8002
-#define CODEC_S_MMAP           0x0003	/* MMAP setup (if available) */
-
-#define CODEC_S_JPEG_TDS_BYTE  0x0010	/* target data size in bytes */
-#define CODEC_G_JPEG_TDS_BYTE  0x8010
-#define CODEC_S_JPEG_SCALE     0x0011	/* scaling factor for quant. tables */
-#define CODEC_G_JPEG_SCALE     0x8011
-#define CODEC_S_JPEG_HDT_DATA  0x0018	/* huffman-tables */
-#define CODEC_G_JPEG_HDT_DATA  0x8018
-#define CODEC_S_JPEG_QDT_DATA  0x0019	/* quantizing-tables */
-#define CODEC_G_JPEG_QDT_DATA  0x8019
-#define CODEC_S_JPEG_APP_DATA  0x001A	/* APP marker */
-#define CODEC_G_JPEG_APP_DATA  0x801A
-#define CODEC_S_JPEG_COM_DATA  0x001B	/* COM marker */
-#define CODEC_G_JPEG_COM_DATA  0x801B
-
-#define CODEC_S_PRIVATE        0x1000	/* "private" commands start here */
-#define CODEC_G_PRIVATE        0x9000
-
-#define CODEC_G_FLAG           0x8000	/* this is how 'get' is detected */
-
-/* types of transfer, directly user space or a kernel buffer (image-fn.'s) */
-/*  -> used in get_image, put_image                                        */
-#define CODEC_TRANSFER_KERNEL 0	/* use "memcopy" */
-#define CODEC_TRANSFER_USER   1	/* use "to/from_user" */
-
-
-/* ========================= */
-/* the structures itself ... */
-/* ========================= */
-
-struct vfe_polarity {
-	unsigned int vsync_pol:1;
-	unsigned int hsync_pol:1;
-	unsigned int field_pol:1;
-	unsigned int blank_pol:1;
-	unsigned int subimg_pol:1;
-	unsigned int poe_pol:1;
-	unsigned int pvalid_pol:1;
-	unsigned int vclk_pol:1;
-};
-
-struct vfe_settings {
-	__u32 x, y;		/* Offsets into image */
-	__u32 width, height;	/* Area to capture */
-	__u16 decimation;	/* Decimation divider */
-	__u16 flags;		/* Flags for capture */
-	__u16 quality;		/* quality of the video */
-};
-
-struct tvnorm {
-	u16 Wt, Wa, HStart, HSyncStart, Ht, Ha, VStart;
-};
-
-struct jpeg_com_marker {
-	int len; /* number of usable bytes in data */
-	char data[60];
-};
-
-struct jpeg_app_marker {
-	int appn; /* number app segment */
-	int len; /* number of usable bytes in data */
-	char data[60];
-};
-
-struct videocodec {
-	struct module *owner;
-	/* -- filled in by slave device during register -- */
-	char name[32];
-	unsigned long magic;	/* may be used for client<->master attaching */
-	unsigned long flags;	/* functionality flags */
-	unsigned int type;	/* codec type */
-
-	/* -- these is filled in later during master device attach -- */
-
-	struct videocodec_master *master_data;
-
-	/* -- these are filled in by the slave device during register -- */
-
-	void *data;		/* private slave data */
-
-	/* attach/detach client functions (indirect call) */
-	int (*setup) (struct videocodec * codec);
-	int (*unset) (struct videocodec * codec);
-
-	/* main functions, every client needs them for sure! */
-	// set compression or decompression (or freeze, stop, standby, etc)
-	int (*set_mode) (struct videocodec * codec,
-			 int mode);
-	// setup picture size and norm (for the codec's video frontend)
-	int (*set_video) (struct videocodec * codec,
-			  struct tvnorm * norm,
-			  struct vfe_settings * cap,
-			  struct vfe_polarity * pol);
-	// other control commands, also mmap setup etc.
-	int (*control) (struct videocodec * codec,
-			int type,
-			int size,
-			void *data);
-
-	/* additional setup/query/processing (may be NULL pointer) */
-	// interrupt setup / handling (for irq's delivered by master)
-	int (*setup_interrupt) (struct videocodec * codec,
-				long mode);
-	int (*handle_interrupt) (struct videocodec * codec,
-				 int source,
-				 long flag);
-	// picture interface (if any)
-	long (*put_image) (struct videocodec * codec,
-			   int tr_type,
-			   int block,
-			   long *fr_num,
-			   long *flag,
-			   long size,
-			   void *buf);
-	long (*get_image) (struct videocodec * codec,
-			   int tr_type,
-			   int block,
-			   long *fr_num,
-			   long *flag,
-			   long size,
-			   void *buf);
-};
-
-struct videocodec_master {
-	/* -- filled in by master device for registration -- */
-	char name[32];
-	unsigned long magic;	/* may be used for client<->master attaching */
-	unsigned long flags;	/* functionality flags */
-	unsigned int type;	/* master type */
-
-	void *data;		/* private master data */
-
-	 __u32(*readreg) (struct videocodec * codec,
-			  __u16 reg);
-	void (*writereg) (struct videocodec * codec,
-			  __u16 reg,
-			  __u32 value);
-};
-
-
-/* ================================================= */
-/* function prototypes of the master/slave interface */
-/* ================================================= */
-
-/* attach and detach commands for the master */
-// * master structure needs to be kmalloc'ed before calling attach
-//   and free'd after calling detach
-// * returns pointer on success, NULL on failure
-extern struct videocodec *videocodec_attach(struct videocodec_master *);
-// * 0 on success, <0 (errno) on failure
-extern int videocodec_detach(struct videocodec *);
-
-/* register and unregister commands for the slaves */
-// * 0 on success, <0 (errno) on failure
-extern int videocodec_register(const struct videocodec *);
-// * 0 on success, <0 (errno) on failure
-extern int videocodec_unregister(const struct videocodec *);
-
-/* the other calls are directly done via the videocodec structure! */
-
-#endif				/*ifndef __LINUX_VIDEOCODEC_H */
diff --git a/drivers/staging/media/zoran/zoran.h b/drivers/staging/media/zoran/zoran.h
deleted file mode 100644
index 1b2e1fb3555f..000000000000
--- a/drivers/staging/media/zoran/zoran.h
+++ /dev/null
@@ -1,392 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0+ */
-/*
- * zoran - Iomega Buz driver
- *
- * Copyright (C) 1999 Rainer Johanni <Rainer@Johanni.de>
- *
- * based on
- *
- * zoran.0.0.3 Copyright (C) 1998 Dave Perks <dperks@ibm.net>
- *
- * and
- *
- * bttv - Bt848 frame grabber driver
- * Copyright (C) 1996,97,98 Ralph  Metzler (rjkm@thp.uni-koeln.de)
- *                        & Marcus Metzler (mocm@thp.uni-koeln.de)
- */
-#ifndef _BUZ_H_
-#define _BUZ_H_
-
-#include <media/v4l2-device.h>
-#include <media/v4l2-ctrls.h>
-#include <media/v4l2-fh.h>
-
-struct zoran_sync {
-	unsigned long frame;	/* number of buffer that has been free'd */
-	unsigned long length;	/* number of code bytes in buffer (capture only) */
-	unsigned long seq;	/* frame sequence number */
-	u64 ts;			/* timestamp */
-};
-
-
-#define ZORAN_NAME    "ZORAN"	/* name of the device */
-
-#define ZR_DEVNAME(zr) ((zr)->name)
-
-#define   BUZ_MAX_WIDTH   (zr->timing->Wa)
-#define   BUZ_MAX_HEIGHT  (zr->timing->Ha)
-#define   BUZ_MIN_WIDTH    32	/* never display less than 32 pixels */
-#define   BUZ_MIN_HEIGHT   24	/* never display less than 24 rows */
-
-#define BUZ_NUM_STAT_COM    4
-#define BUZ_MASK_STAT_COM   3
-
-#define BUZ_MAX_FRAME     256	/* Must be a power of 2 */
-#define BUZ_MASK_FRAME    255	/* Must be BUZ_MAX_FRAME-1 */
-
-#define BUZ_MAX_INPUT       16
-
-#if VIDEO_MAX_FRAME <= 32
-#   define   V4L_MAX_FRAME   32
-#elif VIDEO_MAX_FRAME <= 64
-#   define   V4L_MAX_FRAME   64
-#else
-#   error   "Too many video frame buffers to handle"
-#endif
-#define   V4L_MASK_FRAME   (V4L_MAX_FRAME - 1)
-
-#define MAX_FRAME (BUZ_MAX_FRAME > VIDEO_MAX_FRAME ? BUZ_MAX_FRAME : VIDEO_MAX_FRAME)
-
-#include "zr36057.h"
-
-enum card_type {
-	UNKNOWN = -1,
-
-	/* Pinnacle/Miro */
-	DC10_old,		/* DC30 like */
-	DC10_new,		/* DC10plus like */
-	DC10plus,
-	DC30,
-	DC30plus,
-
-	/* Linux Media Labs */
-	LML33,
-	LML33R10,
-
-	/* Iomega */
-	BUZ,
-
-	/* AverMedia */
-	AVS6EYES,
-
-	/* total number of cards */
-	NUM_CARDS
-};
-
-enum zoran_codec_mode {
-	BUZ_MODE_IDLE,		/* nothing going on */
-	BUZ_MODE_MOTION_COMPRESS,	/* grabbing frames */
-	BUZ_MODE_MOTION_DECOMPRESS,	/* playing frames */
-	BUZ_MODE_STILL_COMPRESS,	/* still frame conversion */
-	BUZ_MODE_STILL_DECOMPRESS	/* still frame conversion */
-};
-
-enum zoran_buffer_state {
-	BUZ_STATE_USER,		/* buffer is owned by application */
-	BUZ_STATE_PEND,		/* buffer is queued in pend[] ready to feed to I/O */
-	BUZ_STATE_DMA,		/* buffer is queued in dma[] for I/O */
-	BUZ_STATE_DONE		/* buffer is ready to return to application */
-};
-
-enum zoran_map_mode {
-	ZORAN_MAP_MODE_RAW,
-	ZORAN_MAP_MODE_JPG_REC,
-#define ZORAN_MAP_MODE_JPG ZORAN_MAP_MODE_JPG_REC
-	ZORAN_MAP_MODE_JPG_PLAY,
-};
-
-enum gpio_type {
-	ZR_GPIO_JPEG_SLEEP = 0,
-	ZR_GPIO_JPEG_RESET,
-	ZR_GPIO_JPEG_FRAME,
-	ZR_GPIO_VID_DIR,
-	ZR_GPIO_VID_EN,
-	ZR_GPIO_VID_RESET,
-	ZR_GPIO_CLK_SEL1,
-	ZR_GPIO_CLK_SEL2,
-	ZR_GPIO_MAX,
-};
-
-enum gpcs_type {
-	GPCS_JPEG_RESET = 0,
-	GPCS_JPEG_START,
-	GPCS_MAX,
-};
-
-struct zoran_format {
-	char *name;
-	__u32 fourcc;
-	int colorspace;
-	int depth;
-	__u32 flags;
-	__u32 vfespfr;
-};
-/* flags */
-#define ZORAN_FORMAT_COMPRESSED 1<<0
-#define ZORAN_FORMAT_OVERLAY    1<<1
-#define ZORAN_FORMAT_CAPTURE	1<<2
-#define ZORAN_FORMAT_PLAYBACK	1<<3
-
-/* overlay-settings */
-struct zoran_overlay_settings {
-	int is_set;
-	int x, y, width, height;	/* position */
-	int clipcount;		/* position and number of clips */
-	const struct zoran_format *format;	/* overlay format */
-};
-
-/* v4l-capture settings */
-struct zoran_v4l_settings {
-	int width, height, bytesperline;	/* capture size */
-	const struct zoran_format *format;	/* capture format */
-};
-
-/* jpg-capture/-playback settings */
-struct zoran_jpg_settings {
-	int decimation;		/* this bit is used to set everything to default */
-	int HorDcm, VerDcm, TmpDcm;	/* capture decimation settings (TmpDcm=1 means both fields) */
-	int field_per_buff, odd_even;	/* field-settings (odd_even=1 (+TmpDcm=1) means top-field-first) */
-	int img_x, img_y, img_width, img_height;	/* crop settings (subframe capture) */
-	struct v4l2_jpegcompression jpg_comp;	/* JPEG-specific capture settings */
-};
-
-struct zoran_fh;
-
-struct zoran_mapping {
-	struct zoran_fh *fh;
-	atomic_t count;
-};
-
-struct zoran_buffer {
-	struct zoran_mapping *map;
-	enum zoran_buffer_state state;	/* state: unused/pending/dma/done */
-	struct zoran_sync bs;		/* DONE: info to return to application */
-	union {
-		struct {
-			__le32 *frag_tab;	/* addresses of frag table */
-			u32 frag_tab_bus;	/* same value cached to save time in ISR */
-		} jpg;
-		struct {
-			char *fbuffer;		/* virtual address of frame buffer */
-			unsigned long fbuffer_phys;/* physical address of frame buffer */
-			unsigned long fbuffer_bus;/* bus address of frame buffer */
-		} v4l;
-	};
-};
-
-enum zoran_lock_activity {
-	ZORAN_FREE,		/* free for use */
-	ZORAN_ACTIVE,		/* active but unlocked */
-	ZORAN_LOCKED,		/* locked */
-};
-
-/* buffer collections */
-struct zoran_buffer_col {
-	enum zoran_lock_activity active;	/* feature currently in use? */
-	unsigned int num_buffers, buffer_size;
-	struct zoran_buffer buffer[MAX_FRAME];	/* buffers */
-	u8 allocated;		/* Flag if buffers are allocated  */
-	u8 need_contiguous;	/* Flag if contiguous buffers are needed */
-	/* only applies to jpg buffers, raw buffers are always contiguous */
-};
-
-struct zoran;
-
-/* zoran_fh contains per-open() settings */
-struct zoran_fh {
-	struct v4l2_fh fh;
-	struct zoran *zr;
-
-	enum zoran_map_mode map_mode;		/* Flag which bufferset will map by next mmap() */
-
-	struct zoran_overlay_settings overlay_settings;
-	u32 *overlay_mask;			/* overlay mask */
-	enum zoran_lock_activity overlay_active;/* feature currently in use? */
-
-	struct zoran_buffer_col buffers;	/* buffers' info */
-
-	struct zoran_v4l_settings v4l_settings;	/* structure with a lot of things to play with */
-	struct zoran_jpg_settings jpg_settings;	/* structure with a lot of things to play with */
-};
-
-struct card_info {
-	enum card_type type;
-	char name[32];
-	const char *i2c_decoder;	/* i2c decoder device */
-	const unsigned short *addrs_decoder;
-	const char *i2c_encoder;	/* i2c encoder device */
-	const unsigned short *addrs_encoder;
-	u16 video_vfe, video_codec;			/* videocodec types */
-	u16 audio_chip;					/* audio type */
-
-	int inputs;		/* number of video inputs */
-	struct input {
-		int muxsel;
-		char name[32];
-	} input[BUZ_MAX_INPUT];
-
-	v4l2_std_id norms;
-	struct tvnorm *tvn[3];	/* supported TV norms */
-
-	u32 jpeg_int;		/* JPEG interrupt */
-	u32 vsync_int;		/* VSYNC interrupt */
-	s8 gpio[ZR_GPIO_MAX];
-	u8 gpcs[GPCS_MAX];
-
-	struct vfe_polarity vfe_pol;
-	u8 gpio_pol[ZR_GPIO_MAX];
-
-	/* is the /GWS line connected? */
-	u8 gws_not_connected;
-
-	/* avs6eyes mux setting */
-	u8 input_mux;
-
-	void (*init) (struct zoran * zr);
-};
-
-struct zoran {
-	struct v4l2_device v4l2_dev;
-	struct v4l2_ctrl_handler hdl;
-	struct video_device *video_dev;
-
-	struct i2c_adapter i2c_adapter;	/* */
-	struct i2c_algo_bit_data i2c_algo;	/* */
-	u32 i2cbr;
-
-	struct v4l2_subdev *decoder;	/* video decoder sub-device */
-	struct v4l2_subdev *encoder;	/* video encoder sub-device */
-
-	struct videocodec *codec;	/* video codec */
-	struct videocodec *vfe;	/* video front end */
-
-	struct mutex lock;	/* file ops serialize lock */
-
-	u8 initialized;		/* flag if zoran has been correctly initialized */
-	int user;		/* number of current users */
-	struct card_info card;
-	struct tvnorm *timing;
-
-	unsigned short id;	/* number of this device */
-	char name[32];		/* name of this device */
-	struct pci_dev *pci_dev;	/* PCI device */
-	unsigned char revision;	/* revision of zr36057 */
-	unsigned char __iomem *zr36057_mem;/* pointer to mapped IO memory */
-
-	spinlock_t spinlock;	/* Spinlock */
-
-	/* Video for Linux parameters */
-	int input;	/* card's norm and input */
-	v4l2_std_id norm;
-
-	/* Current buffer params */
-	void    *vbuf_base;
-	int     vbuf_height, vbuf_width;
-	int     vbuf_depth;
-	int     vbuf_bytesperline;
-
-	struct zoran_overlay_settings overlay_settings;
-	u32 *overlay_mask;	/* overlay mask */
-	enum zoran_lock_activity overlay_active;	/* feature currently in use? */
-
-	wait_queue_head_t v4l_capq;
-
-	int v4l_overlay_active;	/* Overlay grab is activated */
-	int v4l_memgrab_active;	/* Memory grab is activated */
-
-	int v4l_grab_frame;	/* Frame number being currently grabbed */
-#define NO_GRAB_ACTIVE (-1)
-	unsigned long v4l_grab_seq;	/* Number of frames grabbed */
-	struct zoran_v4l_settings v4l_settings;	/* structure with a lot of things to play with */
-
-	/* V4L grab queue of frames pending */
-	unsigned long v4l_pend_head;
-	unsigned long v4l_pend_tail;
-	unsigned long v4l_sync_tail;
-	int v4l_pend[V4L_MAX_FRAME];
-	struct zoran_buffer_col v4l_buffers;	/* V4L buffers' info */
-
-	/* Buz MJPEG parameters */
-	enum zoran_codec_mode codec_mode;	/* status of codec */
-	struct zoran_jpg_settings jpg_settings;	/* structure with a lot of things to play with */
-
-	wait_queue_head_t jpg_capq;	/* wait here for grab to finish */
-
-	/* grab queue counts/indices, mask with BUZ_MASK_STAT_COM before using as index */
-	/* (dma_head - dma_tail) is number active in DMA, must be <= BUZ_NUM_STAT_COM */
-	/* (value & BUZ_MASK_STAT_COM) corresponds to index in stat_com table */
-	unsigned long jpg_que_head;	/* Index where to put next buffer which is queued */
-	unsigned long jpg_dma_head;	/* Index of next buffer which goes into stat_com  */
-	unsigned long jpg_dma_tail;	/* Index of last buffer in stat_com               */
-	unsigned long jpg_que_tail;	/* Index of last buffer in queue                  */
-	unsigned long jpg_seq_num;	/* count of frames since grab/play started        */
-	unsigned long jpg_err_seq;	/* last seq_num before error                      */
-	unsigned long jpg_err_shift;
-	unsigned long jpg_queued_num;	/* count of frames queued since grab/play started */
-
-	/* zr36057's code buffer table */
-	__le32 *stat_com;		/* stat_com[i] is indexed by dma_head/tail & BUZ_MASK_STAT_COM */
-
-	/* (value & BUZ_MASK_FRAME) corresponds to index in pend[] queue */
-	int jpg_pend[BUZ_MAX_FRAME];
-
-	/* array indexed by frame number */
-	struct zoran_buffer_col jpg_buffers;	/* MJPEG buffers' info */
-
-	/* Additional stuff for testing */
-#ifdef CONFIG_PROC_FS
-	struct proc_dir_entry *zoran_proc;
-#else
-	void *zoran_proc;
-#endif
-	int testing;
-	int jpeg_error;
-	int intr_counter_GIRQ1;
-	int intr_counter_GIRQ0;
-	int intr_counter_CodRepIRQ;
-	int intr_counter_JPEGRepIRQ;
-	int field_counter;
-	int IRQ1_in;
-	int IRQ1_out;
-	int JPEG_in;
-	int JPEG_out;
-	int JPEG_0;
-	int JPEG_1;
-	int END_event_missed;
-	int JPEG_missed;
-	int JPEG_error;
-	int num_errors;
-	int JPEG_max_missed;
-	int JPEG_min_missed;
-
-	u32 last_isr;
-	unsigned long frame_num;
-
-	wait_queue_head_t test_q;
-};
-
-static inline struct zoran *to_zoran(struct v4l2_device *v4l2_dev)
-{
-	return container_of(v4l2_dev, struct zoran, v4l2_dev);
-}
-
-/* There was something called _ALPHA_BUZ that used the PCI address instead of
- * the kernel iomapped address for btread/btwrite.  */
-#define btwrite(dat,adr)    writel((dat), zr->zr36057_mem+(adr))
-#define btread(adr)         readl(zr->zr36057_mem+(adr))
-
-#define btand(dat,adr)      btwrite((dat) & btread(adr), adr)
-#define btor(dat,adr)       btwrite((dat) | btread(adr), adr)
-#define btaor(dat,mask,adr) btwrite((dat) | ((mask) & btread(adr)), adr)
-
-#endif
diff --git a/drivers/staging/media/zoran/zoran_card.c b/drivers/staging/media/zoran/zoran_card.c
deleted file mode 100644
index 1d8cd7946bd8..000000000000
--- a/drivers/staging/media/zoran/zoran_card.c
+++ /dev/null
@@ -1,1511 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0+
-/*
- * Zoran zr36057/zr36067 PCI controller driver, for the
- * Pinnacle/Miro DC10/DC10+/DC30/DC30+, Iomega Buz, Linux
- * Media Labs LML33/LML33R10.
- *
- * This part handles card-specific data and detection
- *
- * Copyright (C) 2000 Serguei Miridonov <mirsev@cicese.mx>
- *
- * Currently maintained by:
- *   Ronald Bultje    <rbultje@ronald.bitfreak.net>
- *   Laurent Pinchart <laurent.pinchart@skynet.be>
- *   Mailinglist      <mjpeg-users@lists.sf.net>
- */
-#include <linux/delay.h>
-
-#include <linux/types.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/init.h>
-#include <linux/vmalloc.h>
-#include <linux/slab.h>
-
-#include <linux/proc_fs.h>
-#include <linux/i2c.h>
-#include <linux/i2c-algo-bit.h>
-#include <linux/videodev2.h>
-#include <linux/spinlock.h>
-#include <linux/sem.h>
-#include <linux/kmod.h>
-#include <linux/wait.h>
-
-#include <linux/pci.h>
-#include <linux/interrupt.h>
-#include <linux/mutex.h>
-#include <linux/io.h>
-#include <media/v4l2-common.h>
-#include <media/i2c/bt819.h>
-
-#include "videocodec.h"
-#include "zoran.h"
-#include "zoran_card.h"
-#include "zoran_device.h"
-#include "zoran_procfs.h"
-
-extern const struct zoran_format zoran_formats[];
-
-static int card[BUZ_MAX] = { [0 ... (BUZ_MAX-1)] = -1 };
-module_param_array(card, int, NULL, 0444);
-MODULE_PARM_DESC(card, "Card type");
-
-/*
-   The video mem address of the video card.
-   The driver has a little database for some videocards
-   to determine it from there. If your video card is not in there
-   you have either to give it to the driver as a parameter
-   or set in in a VIDIOCSFBUF ioctl
- */
-
-static unsigned long vidmem;	/* default = 0 - Video memory base address */
-module_param_hw(vidmem, ulong, iomem, 0444);
-MODULE_PARM_DESC(vidmem, "Default video memory base address");
-
-/*
-   Default input and video norm at startup of the driver.
-*/
-
-static unsigned int default_input;	/* default 0 = Composite, 1 = S-Video */
-module_param(default_input, uint, 0444);
-MODULE_PARM_DESC(default_input,
-		 "Default input (0=Composite, 1=S-Video, 2=Internal)");
-
-static int default_mux = 1;	/* 6 Eyes input selection */
-module_param(default_mux, int, 0644);
-MODULE_PARM_DESC(default_mux,
-		 "Default 6 Eyes mux setting (Input selection)");
-
-static int default_norm;	/* default 0 = PAL, 1 = NTSC 2 = SECAM */
-module_param(default_norm, int, 0444);
-MODULE_PARM_DESC(default_norm, "Default norm (0=PAL, 1=NTSC, 2=SECAM)");
-
-/* /dev/videoN, -1 for autodetect */
-static int video_nr[BUZ_MAX] = { [0 ... (BUZ_MAX-1)] = -1 };
-module_param_array(video_nr, int, NULL, 0444);
-MODULE_PARM_DESC(video_nr, "Video device number (-1=Auto)");
-
-int v4l_nbufs = 4;
-int v4l_bufsize = 864;		/* Everybody should be able to work with this setting */
-module_param(v4l_nbufs, int, 0644);
-MODULE_PARM_DESC(v4l_nbufs, "Maximum number of V4L buffers to use");
-module_param(v4l_bufsize, int, 0644);
-MODULE_PARM_DESC(v4l_bufsize, "Maximum size per V4L buffer (in kB)");
-
-int jpg_nbufs = 32;
-int jpg_bufsize = 512;		/* max size for 100% quality full-PAL frame */
-module_param(jpg_nbufs, int, 0644);
-MODULE_PARM_DESC(jpg_nbufs, "Maximum number of JPG buffers to use");
-module_param(jpg_bufsize, int, 0644);
-MODULE_PARM_DESC(jpg_bufsize, "Maximum size per JPG buffer (in kB)");
-
-int pass_through = 0;		/* 1=Pass through TV signal when device is not used */
-				/* 0=Show color bar when device is not used (LML33: only if lml33dpath=1) */
-module_param(pass_through, int, 0644);
-MODULE_PARM_DESC(pass_through,
-		 "Pass TV signal through to TV-out when idling");
-
-int zr36067_debug = 1;
-module_param_named(debug, zr36067_debug, int, 0644);
-MODULE_PARM_DESC(debug, "Debug level (0-5)");
-
-#define ZORAN_VERSION "0.10.1"
-
-MODULE_DESCRIPTION("Zoran-36057/36067 JPEG codec driver");
-MODULE_AUTHOR("Serguei Miridonov");
-MODULE_LICENSE("GPL");
-MODULE_VERSION(ZORAN_VERSION);
-
-#define ZR_DEVICE(subven, subdev, data)	{ \
-	.vendor = PCI_VENDOR_ID_ZORAN, .device = PCI_DEVICE_ID_ZORAN_36057, \
-	.subvendor = (subven), .subdevice = (subdev), .driver_data = (data) }
-
-static const struct pci_device_id zr36067_pci_tbl[] = {
-	ZR_DEVICE(PCI_VENDOR_ID_MIRO, PCI_DEVICE_ID_MIRO_DC10PLUS, DC10plus),
-	ZR_DEVICE(PCI_VENDOR_ID_MIRO, PCI_DEVICE_ID_MIRO_DC30PLUS, DC30plus),
-	ZR_DEVICE(PCI_VENDOR_ID_ELECTRONICDESIGNGMBH, PCI_DEVICE_ID_LML_33R10, LML33R10),
-	ZR_DEVICE(PCI_VENDOR_ID_IOMEGA, PCI_DEVICE_ID_IOMEGA_BUZ, BUZ),
-	ZR_DEVICE(PCI_ANY_ID, PCI_ANY_ID, NUM_CARDS),
-	{0}
-};
-MODULE_DEVICE_TABLE(pci, zr36067_pci_tbl);
-
-static unsigned int zoran_num;		/* number of cards found */
-
-/* videocodec bus functions ZR36060 */
-static u32
-zr36060_read (struct videocodec *codec,
-	      u16                reg)
-{
-	struct zoran *zr = (struct zoran *) codec->master_data->data;
-	__u32 data;
-
-	if (post_office_wait(zr)
-	    || post_office_write(zr, 0, 1, reg >> 8)
-	    || post_office_write(zr, 0, 2, reg & 0xff)) {
-		return -1;
-	}
-
-	data = post_office_read(zr, 0, 3) & 0xff;
-	return data;
-}
-
-static void
-zr36060_write (struct videocodec *codec,
-	       u16                reg,
-	       u32                val)
-{
-	struct zoran *zr = (struct zoran *) codec->master_data->data;
-
-	if (post_office_wait(zr)
-	    || post_office_write(zr, 0, 1, reg >> 8)
-	    || post_office_write(zr, 0, 2, reg & 0xff)) {
-		return;
-	}
-
-	post_office_write(zr, 0, 3, val & 0xff);
-}
-
-/* videocodec bus functions ZR36050 */
-static u32
-zr36050_read (struct videocodec *codec,
-	      u16                reg)
-{
-	struct zoran *zr = (struct zoran *) codec->master_data->data;
-	__u32 data;
-
-	if (post_office_wait(zr)
-	    || post_office_write(zr, 1, 0, reg >> 2)) {	// reg. HIGHBYTES
-		return -1;
-	}
-
-	data = post_office_read(zr, 0, reg & 0x03) & 0xff;	// reg. LOWBYTES + read
-	return data;
-}
-
-static void
-zr36050_write (struct videocodec *codec,
-	       u16                reg,
-	       u32                val)
-{
-	struct zoran *zr = (struct zoran *) codec->master_data->data;
-
-	if (post_office_wait(zr)
-	    || post_office_write(zr, 1, 0, reg >> 2)) {	// reg. HIGHBYTES
-		return;
-	}
-
-	post_office_write(zr, 0, reg & 0x03, val & 0xff);	// reg. LOWBYTES + wr. data
-}
-
-/* videocodec bus functions ZR36016 */
-static u32
-zr36016_read (struct videocodec *codec,
-	      u16                reg)
-{
-	struct zoran *zr = (struct zoran *) codec->master_data->data;
-	__u32 data;
-
-	if (post_office_wait(zr)) {
-		return -1;
-	}
-
-	data = post_office_read(zr, 2, reg & 0x03) & 0xff;	// read
-	return data;
-}
-
-/* hack for in zoran_device.c */
-void
-zr36016_write (struct videocodec *codec,
-	       u16                reg,
-	       u32                val)
-{
-	struct zoran *zr = (struct zoran *) codec->master_data->data;
-
-	if (post_office_wait(zr)) {
-		return;
-	}
-
-	post_office_write(zr, 2, reg & 0x03, val & 0x0ff);	// wr. data
-}
-
-/*
- * Board specific information
- */
-
-static void
-dc10_init (struct zoran *zr)
-{
-	dprintk(3, KERN_DEBUG "%s: %s\n", ZR_DEVNAME(zr), __func__);
-
-	/* Pixel clock selection */
-	GPIO(zr, 4, 0);
-	GPIO(zr, 5, 1);
-	/* Enable the video bus sync signals */
-	GPIO(zr, 7, 0);
-}
-
-static void
-dc10plus_init (struct zoran *zr)
-{
-	dprintk(3, KERN_DEBUG "%s: %s\n", ZR_DEVNAME(zr), __func__);
-}
-
-static void
-buz_init (struct zoran *zr)
-{
-	dprintk(3, KERN_DEBUG "%s: %s\n", ZR_DEVNAME(zr), __func__);
-
-	/* some stuff from Iomega */
-	pci_write_config_dword(zr->pci_dev, 0xfc, 0x90680f15);
-	pci_write_config_dword(zr->pci_dev, 0x0c, 0x00012020);
-	pci_write_config_dword(zr->pci_dev, 0xe8, 0xc0200000);
-}
-
-static void
-lml33_init (struct zoran *zr)
-{
-	dprintk(3, KERN_DEBUG "%s: %s\n", ZR_DEVNAME(zr), __func__);
-
-	GPIO(zr, 2, 1);		// Set Composite input/output
-}
-
-static void
-avs6eyes_init (struct zoran *zr)
-{
-	// AverMedia 6-Eyes original driver by Christer Weinigel
-
-	// Lifted straight from Christer's old driver and
-	// modified slightly by Martin Samuelsson.
-
-	int mux = default_mux; /* 1 = BT866, 7 = VID1 */
-
-	GPIO(zr, 4, 1); /* Bt866 SLEEP on */
-	udelay(2);
-
-	GPIO(zr, 0, 1); /* ZR36060 /RESET on */
-	GPIO(zr, 1, 0); /* ZR36060 /SLEEP on */
-	GPIO(zr, 2, mux & 1);   /* MUX S0 */
-	GPIO(zr, 3, 0); /* /FRAME on */
-	GPIO(zr, 4, 0); /* Bt866 SLEEP off */
-	GPIO(zr, 5, mux & 2);   /* MUX S1 */
-	GPIO(zr, 6, 0); /* ? */
-	GPIO(zr, 7, mux & 4);   /* MUX S2 */
-
-}
-
-static char *
-codecid_to_modulename (u16 codecid)
-{
-	char *name = NULL;
-
-	switch (codecid) {
-	case CODEC_TYPE_ZR36060:
-		name = "zr36060";
-		break;
-	case CODEC_TYPE_ZR36050:
-		name = "zr36050";
-		break;
-	case CODEC_TYPE_ZR36016:
-		name = "zr36016";
-		break;
-	}
-
-	return name;
-}
-
-// struct tvnorm {
-//      u16 Wt, Wa, HStart, HSyncStart, Ht, Ha, VStart;
-// };
-
-static struct tvnorm f50sqpixel = { 944, 768, 83, 880, 625, 576, 16 };
-static struct tvnorm f60sqpixel = { 780, 640, 51, 716, 525, 480, 12 };
-static struct tvnorm f50ccir601 = { 864, 720, 75, 804, 625, 576, 18 };
-static struct tvnorm f60ccir601 = { 858, 720, 57, 788, 525, 480, 16 };
-
-static struct tvnorm f50ccir601_lml33 = { 864, 720, 75+34, 804, 625, 576, 18 };
-static struct tvnorm f60ccir601_lml33 = { 858, 720, 57+34, 788, 525, 480, 16 };
-
-/* The DC10 (57/16/50) uses VActive as HSync, so HStart must be 0 */
-static struct tvnorm f50sqpixel_dc10 = { 944, 768, 0, 880, 625, 576, 0 };
-static struct tvnorm f60sqpixel_dc10 = { 780, 640, 0, 716, 525, 480, 12 };
-
-/* FIXME: I cannot swap U and V in saa7114, so i do one
- * pixel left shift in zoran (75 -> 74)
- * (Maxim Yevtyushkin <max@linuxmedialabs.com>) */
-static struct tvnorm f50ccir601_lm33r10 = { 864, 720, 74+54, 804, 625, 576, 18 };
-static struct tvnorm f60ccir601_lm33r10 = { 858, 720, 56+54, 788, 525, 480, 16 };
-
-/* FIXME: The ks0127 seem incapable of swapping U and V, too, which is why I
- * copy Maxim's left shift hack for the 6 Eyes.
- *
- * Christer's driver used the unshifted norms, though...
- * /Sam  */
-static struct tvnorm f50ccir601_avs6eyes = { 864, 720, 74, 804, 625, 576, 18 };
-static struct tvnorm f60ccir601_avs6eyes = { 858, 720, 56, 788, 525, 480, 16 };
-
-static const unsigned short vpx3220_addrs[] = { 0x43, 0x47, I2C_CLIENT_END };
-static const unsigned short saa7110_addrs[] = { 0x4e, 0x4f, I2C_CLIENT_END };
-static const unsigned short saa7111_addrs[] = { 0x25, 0x24, I2C_CLIENT_END };
-static const unsigned short saa7114_addrs[] = { 0x21, 0x20, I2C_CLIENT_END };
-static const unsigned short adv717x_addrs[] = { 0x6a, 0x6b, 0x2a, 0x2b, I2C_CLIENT_END };
-static const unsigned short ks0127_addrs[] = { 0x6c, 0x6d, I2C_CLIENT_END };
-static const unsigned short saa7185_addrs[] = { 0x44, I2C_CLIENT_END };
-static const unsigned short bt819_addrs[] = { 0x45, I2C_CLIENT_END };
-static const unsigned short bt856_addrs[] = { 0x44, I2C_CLIENT_END };
-static const unsigned short bt866_addrs[] = { 0x44, I2C_CLIENT_END };
-
-static struct card_info zoran_cards[NUM_CARDS] = {
-	{
-		.type = DC10_old,
-		.name = "DC10(old)",
-		.i2c_decoder = "vpx3220a",
-		.addrs_decoder = vpx3220_addrs,
-		.video_codec = CODEC_TYPE_ZR36050,
-		.video_vfe = CODEC_TYPE_ZR36016,
-
-		.inputs = 3,
-		.input = {
-			{ 1, "Composite" },
-			{ 2, "S-Video" },
-			{ 0, "Internal/comp" }
-		},
-		.norms = V4L2_STD_NTSC|V4L2_STD_PAL|V4L2_STD_SECAM,
-		.tvn = {
-			&f50sqpixel_dc10,
-			&f60sqpixel_dc10,
-			&f50sqpixel_dc10
-		},
-		.jpeg_int = 0,
-		.vsync_int = ZR36057_ISR_GIRQ1,
-		.gpio = { 2, 1, -1, 3, 7, 0, 4, 5 },
-		.gpio_pol = { 0, 0, 0, 1, 0, 0, 0, 0 },
-		.gpcs = { -1, 0 },
-		.vfe_pol = { 0, 0, 0, 0, 0, 0, 0, 0 },
-		.gws_not_connected = 0,
-		.input_mux = 0,
-		.init = &dc10_init,
-	}, {
-		.type = DC10_new,
-		.name = "DC10(new)",
-		.i2c_decoder = "saa7110",
-		.addrs_decoder = saa7110_addrs,
-		.i2c_encoder = "adv7175",
-		.addrs_encoder = adv717x_addrs,
-		.video_codec = CODEC_TYPE_ZR36060,
-
-		.inputs = 3,
-		.input = {
-				{ 0, "Composite" },
-				{ 7, "S-Video" },
-				{ 5, "Internal/comp" }
-			},
-		.norms = V4L2_STD_NTSC|V4L2_STD_PAL|V4L2_STD_SECAM,
-		.tvn = {
-				&f50sqpixel,
-				&f60sqpixel,
-				&f50sqpixel},
-		.jpeg_int = ZR36057_ISR_GIRQ0,
-		.vsync_int = ZR36057_ISR_GIRQ1,
-		.gpio = { 3, 0, 6, 1, 2, -1, 4, 5 },
-		.gpio_pol = { 0, 0, 0, 0, 0, 0, 0, 0 },
-		.gpcs = { -1, 1},
-		.vfe_pol = { 1, 1, 1, 1, 0, 0, 0, 0 },
-		.gws_not_connected = 0,
-		.input_mux = 0,
-		.init = &dc10plus_init,
-	}, {
-		.type = DC10plus,
-		.name = "DC10plus",
-		.i2c_decoder = "saa7110",
-		.addrs_decoder = saa7110_addrs,
-		.i2c_encoder = "adv7175",
-		.addrs_encoder = adv717x_addrs,
-		.video_codec = CODEC_TYPE_ZR36060,
-
-		.inputs = 3,
-		.input = {
-			{ 0, "Composite" },
-			{ 7, "S-Video" },
-			{ 5, "Internal/comp" }
-		},
-		.norms = V4L2_STD_NTSC|V4L2_STD_PAL|V4L2_STD_SECAM,
-		.tvn = {
-			&f50sqpixel,
-			&f60sqpixel,
-			&f50sqpixel
-		},
-		.jpeg_int = ZR36057_ISR_GIRQ0,
-		.vsync_int = ZR36057_ISR_GIRQ1,
-		.gpio = { 3, 0, 6, 1, 2, -1, 4, 5 },
-		.gpio_pol = { 0, 0, 0, 0, 0, 0, 0, 0 },
-		.gpcs = { -1, 1 },
-		.vfe_pol = { 1, 1, 1, 1, 0, 0, 0, 0 },
-		.gws_not_connected = 0,
-		.input_mux = 0,
-		.init = &dc10plus_init,
-	}, {
-		.type = DC30,
-		.name = "DC30",
-		.i2c_decoder = "vpx3220a",
-		.addrs_decoder = vpx3220_addrs,
-		.i2c_encoder = "adv7175",
-		.addrs_encoder = adv717x_addrs,
-		.video_codec = CODEC_TYPE_ZR36050,
-		.video_vfe = CODEC_TYPE_ZR36016,
-
-		.inputs = 3,
-		.input = {
-			{ 1, "Composite" },
-			{ 2, "S-Video" },
-			{ 0, "Internal/comp" }
-		},
-		.norms = V4L2_STD_NTSC|V4L2_STD_PAL|V4L2_STD_SECAM,
-		.tvn = {
-			&f50sqpixel_dc10,
-			&f60sqpixel_dc10,
-			&f50sqpixel_dc10
-		},
-		.jpeg_int = 0,
-		.vsync_int = ZR36057_ISR_GIRQ1,
-		.gpio = { 2, 1, -1, 3, 7, 0, 4, 5 },
-		.gpio_pol = { 0, 0, 0, 1, 0, 0, 0, 0 },
-		.gpcs = { -1, 0 },
-		.vfe_pol = { 0, 0, 0, 0, 0, 0, 0, 0 },
-		.gws_not_connected = 0,
-		.input_mux = 0,
-		.init = &dc10_init,
-	}, {
-		.type = DC30plus,
-		.name = "DC30plus",
-		.i2c_decoder = "vpx3220a",
-		.addrs_decoder = vpx3220_addrs,
-		.i2c_encoder = "adv7175",
-		.addrs_encoder = adv717x_addrs,
-		.video_codec = CODEC_TYPE_ZR36050,
-		.video_vfe = CODEC_TYPE_ZR36016,
-
-		.inputs = 3,
-		.input = {
-			{ 1, "Composite" },
-			{ 2, "S-Video" },
-			{ 0, "Internal/comp" }
-		},
-		.norms = V4L2_STD_NTSC|V4L2_STD_PAL|V4L2_STD_SECAM,
-		.tvn = {
-			&f50sqpixel_dc10,
-			&f60sqpixel_dc10,
-			&f50sqpixel_dc10
-		},
-		.jpeg_int = 0,
-		.vsync_int = ZR36057_ISR_GIRQ1,
-		.gpio = { 2, 1, -1, 3, 7, 0, 4, 5 },
-		.gpio_pol = { 0, 0, 0, 1, 0, 0, 0, 0 },
-		.gpcs = { -1, 0 },
-		.vfe_pol = { 0, 0, 0, 0, 0, 0, 0, 0 },
-		.gws_not_connected = 0,
-		.input_mux = 0,
-		.init = &dc10_init,
-	}, {
-		.type = LML33,
-		.name = "LML33",
-		.i2c_decoder = "bt819a",
-		.addrs_decoder = bt819_addrs,
-		.i2c_encoder = "bt856",
-		.addrs_encoder = bt856_addrs,
-		.video_codec = CODEC_TYPE_ZR36060,
-
-		.inputs = 2,
-		.input = {
-			{ 0, "Composite" },
-			{ 7, "S-Video" }
-		},
-		.norms = V4L2_STD_NTSC|V4L2_STD_PAL,
-		.tvn = {
-			&f50ccir601_lml33,
-			&f60ccir601_lml33,
-			NULL
-		},
-		.jpeg_int = ZR36057_ISR_GIRQ1,
-		.vsync_int = ZR36057_ISR_GIRQ0,
-		.gpio = { 1, -1, 3, 5, 7, -1, -1, -1 },
-		.gpio_pol = { 0, 0, 0, 0, 1, 0, 0, 0 },
-		.gpcs = { 3, 1 },
-		.vfe_pol = { 1, 1, 0, 0, 0, 1, 0, 0 },
-		.gws_not_connected = 1,
-		.input_mux = 0,
-		.init = &lml33_init,
-	}, {
-		.type = LML33R10,
-		.name = "LML33R10",
-		.i2c_decoder = "saa7114",
-		.addrs_decoder = saa7114_addrs,
-		.i2c_encoder = "adv7170",
-		.addrs_encoder = adv717x_addrs,
-		.video_codec = CODEC_TYPE_ZR36060,
-
-		.inputs = 2,
-		.input = {
-			{ 0, "Composite" },
-			{ 7, "S-Video" }
-		},
-		.norms = V4L2_STD_NTSC|V4L2_STD_PAL,
-		.tvn = {
-			&f50ccir601_lm33r10,
-			&f60ccir601_lm33r10,
-			NULL
-		},
-		.jpeg_int = ZR36057_ISR_GIRQ1,
-		.vsync_int = ZR36057_ISR_GIRQ0,
-		.gpio = { 1, -1, 3, 5, 7, -1, -1, -1 },
-		.gpio_pol = { 0, 0, 0, 0, 1, 0, 0, 0 },
-		.gpcs = { 3, 1 },
-		.vfe_pol = { 1, 1, 0, 0, 0, 1, 0, 0 },
-		.gws_not_connected = 1,
-		.input_mux = 0,
-		.init = &lml33_init,
-	}, {
-		.type = BUZ,
-		.name = "Buz",
-		.i2c_decoder = "saa7111",
-		.addrs_decoder = saa7111_addrs,
-		.i2c_encoder = "saa7185",
-		.addrs_encoder = saa7185_addrs,
-		.video_codec = CODEC_TYPE_ZR36060,
-
-		.inputs = 2,
-		.input = {
-			{ 3, "Composite" },
-			{ 7, "S-Video" }
-		},
-		.norms = V4L2_STD_NTSC|V4L2_STD_PAL|V4L2_STD_SECAM,
-		.tvn = {
-			&f50ccir601,
-			&f60ccir601,
-			&f50ccir601
-		},
-		.jpeg_int = ZR36057_ISR_GIRQ1,
-		.vsync_int = ZR36057_ISR_GIRQ0,
-		.gpio = { 1, -1, 3, -1, -1, -1, -1, -1 },
-		.gpio_pol = { 0, 0, 0, 0, 0, 0, 0, 0 },
-		.gpcs = { 3, 1 },
-		.vfe_pol = { 1, 1, 0, 0, 0, 1, 0, 0 },
-		.gws_not_connected = 1,
-		.input_mux = 0,
-		.init = &buz_init,
-	}, {
-		.type = AVS6EYES,
-		.name = "6-Eyes",
-		/* AverMedia chose not to brand the 6-Eyes. Thus it
-		   can't be autodetected, and requires card=x. */
-		.i2c_decoder = "ks0127",
-		.addrs_decoder = ks0127_addrs,
-		.i2c_encoder = "bt866",
-		.addrs_encoder = bt866_addrs,
-		.video_codec = CODEC_TYPE_ZR36060,
-
-		.inputs = 10,
-		.input = {
-			{ 0, "Composite 1" },
-			{ 1, "Composite 2" },
-			{ 2, "Composite 3" },
-			{ 4, "Composite 4" },
-			{ 5, "Composite 5" },
-			{ 6, "Composite 6" },
-			{ 8, "S-Video 1" },
-			{ 9, "S-Video 2" },
-			{10, "S-Video 3" },
-			{15, "YCbCr" }
-		},
-		.norms = V4L2_STD_NTSC|V4L2_STD_PAL,
-		.tvn = {
-			&f50ccir601_avs6eyes,
-			&f60ccir601_avs6eyes,
-			NULL
-		},
-		.jpeg_int = ZR36057_ISR_GIRQ1,
-		.vsync_int = ZR36057_ISR_GIRQ0,
-		.gpio = { 1, 0, 3, -1, -1, -1, -1, -1 },// Validity unknown /Sam
-		.gpio_pol = { 0, 0, 0, 0, 0, 0, 0, 0 }, // Validity unknown /Sam
-		.gpcs = { 3, 1 },			// Validity unknown /Sam
-		.vfe_pol = { 1, 0, 0, 0, 0, 1, 0, 0 },  // Validity unknown /Sam
-		.gws_not_connected = 1,
-		.input_mux = 1,
-		.init = &avs6eyes_init,
-	}
-
-};
-
-/*
- * I2C functions
- */
-/* software I2C functions */
-static int
-zoran_i2c_getsda (void *data)
-{
-	struct zoran *zr = (struct zoran *) data;
-
-	return (btread(ZR36057_I2CBR) >> 1) & 1;
-}
-
-static int
-zoran_i2c_getscl (void *data)
-{
-	struct zoran *zr = (struct zoran *) data;
-
-	return btread(ZR36057_I2CBR) & 1;
-}
-
-static void
-zoran_i2c_setsda (void *data,
-		  int   state)
-{
-	struct zoran *zr = (struct zoran *) data;
-
-	if (state)
-		zr->i2cbr |= 2;
-	else
-		zr->i2cbr &= ~2;
-	btwrite(zr->i2cbr, ZR36057_I2CBR);
-}
-
-static void
-zoran_i2c_setscl (void *data,
-		  int   state)
-{
-	struct zoran *zr = (struct zoran *) data;
-
-	if (state)
-		zr->i2cbr |= 1;
-	else
-		zr->i2cbr &= ~1;
-	btwrite(zr->i2cbr, ZR36057_I2CBR);
-}
-
-static const struct i2c_algo_bit_data zoran_i2c_bit_data_template = {
-	.setsda = zoran_i2c_setsda,
-	.setscl = zoran_i2c_setscl,
-	.getsda = zoran_i2c_getsda,
-	.getscl = zoran_i2c_getscl,
-	.udelay = 10,
-	.timeout = 100,
-};
-
-static int
-zoran_register_i2c (struct zoran *zr)
-{
-	zr->i2c_algo = zoran_i2c_bit_data_template;
-	zr->i2c_algo.data = zr;
-	strscpy(zr->i2c_adapter.name, ZR_DEVNAME(zr),
-		sizeof(zr->i2c_adapter.name));
-	i2c_set_adapdata(&zr->i2c_adapter, &zr->v4l2_dev);
-	zr->i2c_adapter.algo_data = &zr->i2c_algo;
-	zr->i2c_adapter.dev.parent = &zr->pci_dev->dev;
-	return i2c_bit_add_bus(&zr->i2c_adapter);
-}
-
-static void
-zoran_unregister_i2c (struct zoran *zr)
-{
-	i2c_del_adapter(&zr->i2c_adapter);
-}
-
-/* Check a zoran_params struct for correctness, insert default params */
-
-int
-zoran_check_jpg_settings (struct zoran              *zr,
-			  struct zoran_jpg_settings *settings,
-			  int try)
-{
-	int err = 0, err0 = 0;
-
-	dprintk(4,
-		KERN_DEBUG
-		"%s: %s - dec: %d, Hdcm: %d, Vdcm: %d, Tdcm: %d\n",
-		ZR_DEVNAME(zr), __func__, settings->decimation, settings->HorDcm,
-		settings->VerDcm, settings->TmpDcm);
-	dprintk(4,
-		KERN_DEBUG
-		"%s: %s - x: %d, y: %d, w: %d, y: %d\n",
-		ZR_DEVNAME(zr), __func__, settings->img_x, settings->img_y,
-		settings->img_width, settings->img_height);
-	/* Check decimation, set default values for decimation = 1, 2, 4 */
-	switch (settings->decimation) {
-	case 1:
-
-		settings->HorDcm = 1;
-		settings->VerDcm = 1;
-		settings->TmpDcm = 1;
-		settings->field_per_buff = 2;
-		settings->img_x = 0;
-		settings->img_y = 0;
-		settings->img_width = BUZ_MAX_WIDTH;
-		settings->img_height = BUZ_MAX_HEIGHT / 2;
-		break;
-	case 2:
-
-		settings->HorDcm = 2;
-		settings->VerDcm = 1;
-		settings->TmpDcm = 2;
-		settings->field_per_buff = 1;
-		settings->img_x = (BUZ_MAX_WIDTH == 720) ? 8 : 0;
-		settings->img_y = 0;
-		settings->img_width =
-		    (BUZ_MAX_WIDTH == 720) ? 704 : BUZ_MAX_WIDTH;
-		settings->img_height = BUZ_MAX_HEIGHT / 2;
-		break;
-	case 4:
-
-		if (zr->card.type == DC10_new) {
-			dprintk(1,
-				KERN_DEBUG
-				"%s: %s - HDec by 4 is not supported on the DC10\n",
-				ZR_DEVNAME(zr), __func__);
-			err0++;
-			break;
-		}
-
-		settings->HorDcm = 4;
-		settings->VerDcm = 2;
-		settings->TmpDcm = 2;
-		settings->field_per_buff = 1;
-		settings->img_x = (BUZ_MAX_WIDTH == 720) ? 8 : 0;
-		settings->img_y = 0;
-		settings->img_width =
-		    (BUZ_MAX_WIDTH == 720) ? 704 : BUZ_MAX_WIDTH;
-		settings->img_height = BUZ_MAX_HEIGHT / 2;
-		break;
-	case 0:
-
-		/* We have to check the data the user has set */
-
-		if (settings->HorDcm != 1 && settings->HorDcm != 2 &&
-		    (zr->card.type == DC10_new || settings->HorDcm != 4)) {
-			settings->HorDcm = clamp(settings->HorDcm, 1, 2);
-			err0++;
-		}
-		if (settings->VerDcm != 1 && settings->VerDcm != 2) {
-			settings->VerDcm = clamp(settings->VerDcm, 1, 2);
-			err0++;
-		}
-		if (settings->TmpDcm != 1 && settings->TmpDcm != 2) {
-			settings->TmpDcm = clamp(settings->TmpDcm, 1, 2);
-			err0++;
-		}
-		if (settings->field_per_buff != 1 &&
-		    settings->field_per_buff != 2) {
-			settings->field_per_buff = clamp(settings->field_per_buff, 1, 2);
-			err0++;
-		}
-		if (settings->img_x < 0) {
-			settings->img_x = 0;
-			err0++;
-		}
-		if (settings->img_y < 0) {
-			settings->img_y = 0;
-			err0++;
-		}
-		if (settings->img_width < 0 || settings->img_width > BUZ_MAX_WIDTH) {
-			settings->img_width = clamp(settings->img_width, 0, (int)BUZ_MAX_WIDTH);
-			err0++;
-		}
-		if (settings->img_height < 0 || settings->img_height > BUZ_MAX_HEIGHT / 2) {
-			settings->img_height = clamp(settings->img_height, 0, BUZ_MAX_HEIGHT / 2);
-			err0++;
-		}
-		if (settings->img_x + settings->img_width > BUZ_MAX_WIDTH) {
-			settings->img_x = BUZ_MAX_WIDTH - settings->img_width;
-			err0++;
-		}
-		if (settings->img_y + settings->img_height > BUZ_MAX_HEIGHT / 2) {
-			settings->img_y = BUZ_MAX_HEIGHT / 2 - settings->img_height;
-			err0++;
-		}
-		if (settings->img_width % (16 * settings->HorDcm) != 0) {
-			settings->img_width -= settings->img_width % (16 * settings->HorDcm);
-			if (settings->img_width == 0)
-				settings->img_width = 16 * settings->HorDcm;
-			err0++;
-		}
-		if (settings->img_height % (8 * settings->VerDcm) != 0) {
-			settings->img_height -= settings->img_height % (8 * settings->VerDcm);
-			if (settings->img_height == 0)
-				settings->img_height = 8 * settings->VerDcm;
-			err0++;
-		}
-
-		if (!try && err0) {
-			dprintk(1,
-				KERN_ERR
-				"%s: %s - error in params for decimation = 0\n",
-				ZR_DEVNAME(zr), __func__);
-			err++;
-		}
-		break;
-	default:
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - decimation = %d, must be 0, 1, 2 or 4\n",
-			ZR_DEVNAME(zr), __func__, settings->decimation);
-		err++;
-		break;
-	}
-
-	if (settings->jpg_comp.quality > 100)
-		settings->jpg_comp.quality = 100;
-	if (settings->jpg_comp.quality < 5)
-		settings->jpg_comp.quality = 5;
-	if (settings->jpg_comp.APPn < 0)
-		settings->jpg_comp.APPn = 0;
-	if (settings->jpg_comp.APPn > 15)
-		settings->jpg_comp.APPn = 15;
-	if (settings->jpg_comp.APP_len < 0)
-		settings->jpg_comp.APP_len = 0;
-	if (settings->jpg_comp.APP_len > 60)
-		settings->jpg_comp.APP_len = 60;
-	if (settings->jpg_comp.COM_len < 0)
-		settings->jpg_comp.COM_len = 0;
-	if (settings->jpg_comp.COM_len > 60)
-		settings->jpg_comp.COM_len = 60;
-	if (err)
-		return -EINVAL;
-	return 0;
-}
-
-void
-zoran_open_init_params (struct zoran *zr)
-{
-	int i;
-
-	/* User must explicitly set a window */
-	zr->overlay_settings.is_set = 0;
-	zr->overlay_mask = NULL;
-	zr->overlay_active = ZORAN_FREE;
-
-	zr->v4l_memgrab_active = 0;
-	zr->v4l_overlay_active = 0;
-	zr->v4l_grab_frame = NO_GRAB_ACTIVE;
-	zr->v4l_grab_seq = 0;
-	zr->v4l_settings.width = 192;
-	zr->v4l_settings.height = 144;
-	zr->v4l_settings.format = &zoran_formats[7];	/* YUY2 - YUV-4:2:2 packed */
-	zr->v4l_settings.bytesperline =
-	    zr->v4l_settings.width *
-	    ((zr->v4l_settings.format->depth + 7) / 8);
-
-	/* DMA ring stuff for V4L */
-	zr->v4l_pend_tail = 0;
-	zr->v4l_pend_head = 0;
-	zr->v4l_sync_tail = 0;
-	zr->v4l_buffers.active = ZORAN_FREE;
-	for (i = 0; i < VIDEO_MAX_FRAME; i++) {
-		zr->v4l_buffers.buffer[i].state = BUZ_STATE_USER;	/* nothing going on */
-	}
-	zr->v4l_buffers.allocated = 0;
-
-	for (i = 0; i < BUZ_MAX_FRAME; i++) {
-		zr->jpg_buffers.buffer[i].state = BUZ_STATE_USER;	/* nothing going on */
-	}
-	zr->jpg_buffers.active = ZORAN_FREE;
-	zr->jpg_buffers.allocated = 0;
-	/* Set necessary params and call zoran_check_jpg_settings to set the defaults */
-	zr->jpg_settings.decimation = 1;
-	zr->jpg_settings.jpg_comp.quality = 50;	/* default compression factor 8 */
-	if (zr->card.type != BUZ)
-		zr->jpg_settings.odd_even = 1;
-	else
-		zr->jpg_settings.odd_even = 0;
-	zr->jpg_settings.jpg_comp.APPn = 0;
-	zr->jpg_settings.jpg_comp.APP_len = 0;	/* No APPn marker */
-	memset(zr->jpg_settings.jpg_comp.APP_data, 0,
-	       sizeof(zr->jpg_settings.jpg_comp.APP_data));
-	zr->jpg_settings.jpg_comp.COM_len = 0;	/* No COM marker */
-	memset(zr->jpg_settings.jpg_comp.COM_data, 0,
-	       sizeof(zr->jpg_settings.jpg_comp.COM_data));
-	zr->jpg_settings.jpg_comp.jpeg_markers =
-	    V4L2_JPEG_MARKER_DHT | V4L2_JPEG_MARKER_DQT;
-	i = zoran_check_jpg_settings(zr, &zr->jpg_settings, 0);
-	if (i)
-		dprintk(1, KERN_ERR "%s: %s internal error\n",
-			ZR_DEVNAME(zr), __func__);
-
-	clear_interrupt_counters(zr);
-	zr->testing = 0;
-}
-
-static void test_interrupts (struct zoran *zr)
-{
-	DEFINE_WAIT(wait);
-	int timeout, icr;
-
-	clear_interrupt_counters(zr);
-
-	zr->testing = 1;
-	icr = btread(ZR36057_ICR);
-	btwrite(0x78000000 | ZR36057_ICR_IntPinEn, ZR36057_ICR);
-	prepare_to_wait(&zr->test_q, &wait, TASK_INTERRUPTIBLE);
-	timeout = schedule_timeout(HZ);
-	finish_wait(&zr->test_q, &wait);
-	btwrite(0, ZR36057_ICR);
-	btwrite(0x78000000, ZR36057_ISR);
-	zr->testing = 0;
-	dprintk(5, KERN_INFO "%s: Testing interrupts...\n", ZR_DEVNAME(zr));
-	if (timeout) {
-		dprintk(1, ": time spent: %d\n", 1 * HZ - timeout);
-	}
-	if (zr36067_debug > 1)
-		print_interrupts(zr);
-	btwrite(icr, ZR36057_ICR);
-}
-
-static int zr36057_init (struct zoran *zr)
-{
-	int j, err;
-
-	dprintk(1,
-		KERN_INFO
-		"%s: %s - initializing card[%d], zr=%p\n",
-		ZR_DEVNAME(zr), __func__, zr->id, zr);
-
-	/* default setup of all parameters which will persist between opens */
-	zr->user = 0;
-
-	init_waitqueue_head(&zr->v4l_capq);
-	init_waitqueue_head(&zr->jpg_capq);
-	init_waitqueue_head(&zr->test_q);
-	zr->jpg_buffers.allocated = 0;
-	zr->v4l_buffers.allocated = 0;
-
-	zr->vbuf_base = (void *) vidmem;
-	zr->vbuf_width = 0;
-	zr->vbuf_height = 0;
-	zr->vbuf_depth = 0;
-	zr->vbuf_bytesperline = 0;
-
-	/* Avoid nonsense settings from user for default input/norm */
-	if (default_norm < 0 || default_norm > 2)
-		default_norm = 0;
-	if (default_norm == 0) {
-		zr->norm = V4L2_STD_PAL;
-		zr->timing = zr->card.tvn[0];
-	} else if (default_norm == 1) {
-		zr->norm = V4L2_STD_NTSC;
-		zr->timing = zr->card.tvn[1];
-	} else {
-		zr->norm = V4L2_STD_SECAM;
-		zr->timing = zr->card.tvn[2];
-	}
-	if (zr->timing == NULL) {
-		dprintk(1,
-			KERN_WARNING
-			"%s: %s - default TV standard not supported by hardware. PAL will be used.\n",
-			ZR_DEVNAME(zr), __func__);
-		zr->norm = V4L2_STD_PAL;
-		zr->timing = zr->card.tvn[0];
-	}
-
-	if (default_input > zr->card.inputs-1) {
-		dprintk(1,
-			KERN_WARNING
-			"%s: default_input value %d out of range (0-%d)\n",
-			ZR_DEVNAME(zr), default_input, zr->card.inputs-1);
-		default_input = 0;
-	}
-	zr->input = default_input;
-
-	/* default setup (will be repeated at every open) */
-	zoran_open_init_params(zr);
-
-	/* allocate memory *before* doing anything to the hardware
-	 * in case allocation fails */
-	zr->stat_com = kzalloc(BUZ_NUM_STAT_COM * 4, GFP_KERNEL);
-	zr->video_dev = video_device_alloc();
-	if (!zr->stat_com || !zr->video_dev) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - kmalloc (STAT_COM) failed\n",
-			ZR_DEVNAME(zr), __func__);
-		err = -ENOMEM;
-		goto exit_free;
-	}
-	for (j = 0; j < BUZ_NUM_STAT_COM; j++) {
-		zr->stat_com[j] = cpu_to_le32(1); /* mark as unavailable to zr36057 */
-	}
-
-	/*
-	 *   Now add the template and register the device unit.
-	 */
-	*zr->video_dev = zoran_template;
-	zr->video_dev->v4l2_dev = &zr->v4l2_dev;
-	zr->video_dev->lock = &zr->lock;
-	strscpy(zr->video_dev->name, ZR_DEVNAME(zr), sizeof(zr->video_dev->name));
-	/* It's not a mem2mem device, but you can both capture and output from
-	   one and the same device. This should really be split up into two
-	   device nodes, but that's a job for another day. */
-	zr->video_dev->vfl_dir = VFL_DIR_M2M;
-	err = video_register_device(zr->video_dev, VFL_TYPE_GRABBER, video_nr[zr->id]);
-	if (err < 0)
-		goto exit_free;
-	video_set_drvdata(zr->video_dev, zr);
-
-	zoran_init_hardware(zr);
-	if (zr36067_debug > 2)
-		detect_guest_activity(zr);
-	test_interrupts(zr);
-	if (!pass_through) {
-		decoder_call(zr, video, s_stream, 0);
-		encoder_call(zr, video, s_routing, 2, 0, 0);
-	}
-
-	zr->zoran_proc = NULL;
-	zr->initialized = 1;
-	return 0;
-
-exit_free:
-	kfree(zr->stat_com);
-	kfree(zr->video_dev);
-	return err;
-}
-
-static void zoran_remove(struct pci_dev *pdev)
-{
-	struct v4l2_device *v4l2_dev = dev_get_drvdata(&pdev->dev);
-	struct zoran *zr = to_zoran(v4l2_dev);
-
-	if (!zr->initialized)
-		goto exit_free;
-
-	/* unregister videocodec bus */
-	if (zr->codec) {
-		struct videocodec_master *master = zr->codec->master_data;
-
-		videocodec_detach(zr->codec);
-		kfree(master);
-	}
-	if (zr->vfe) {
-		struct videocodec_master *master = zr->vfe->master_data;
-
-		videocodec_detach(zr->vfe);
-		kfree(master);
-	}
-
-	/* unregister i2c bus */
-	zoran_unregister_i2c(zr);
-	/* disable PCI bus-mastering */
-	zoran_set_pci_master(zr, 0);
-	/* put chip into reset */
-	btwrite(0, ZR36057_SPGPPCR);
-	free_irq(zr->pci_dev->irq, zr);
-	/* unmap and free memory */
-	kfree(zr->stat_com);
-	zoran_proc_cleanup(zr);
-	iounmap(zr->zr36057_mem);
-	pci_disable_device(zr->pci_dev);
-	video_unregister_device(zr->video_dev);
-exit_free:
-	v4l2_ctrl_handler_free(&zr->hdl);
-	v4l2_device_unregister(&zr->v4l2_dev);
-	kfree(zr);
-}
-
-void
-zoran_vdev_release (struct video_device *vdev)
-{
-	kfree(vdev);
-}
-
-static struct videocodec_master *zoran_setup_videocodec(struct zoran *zr,
-							int type)
-{
-	struct videocodec_master *m = NULL;
-
-	m = kmalloc(sizeof(struct videocodec_master), GFP_KERNEL);
-	if (!m) {
-		return m;
-	}
-
-	/* magic and type are unused for master struct. Makes sense only at
-	   codec structs.
-	   In the past, .type were initialized to the old V4L1 .hardware
-	   value, as VID_HARDWARE_ZR36067
-	 */
-	m->magic = 0L;
-	m->type = 0;
-
-	m->flags = CODEC_FLAG_ENCODER | CODEC_FLAG_DECODER;
-	strscpy(m->name, ZR_DEVNAME(zr), sizeof(m->name));
-	m->data = zr;
-
-	switch (type) {
-	case CODEC_TYPE_ZR36060:
-		m->readreg = zr36060_read;
-		m->writereg = zr36060_write;
-		m->flags |= CODEC_FLAG_JPEG | CODEC_FLAG_VFE;
-		break;
-	case CODEC_TYPE_ZR36050:
-		m->readreg = zr36050_read;
-		m->writereg = zr36050_write;
-		m->flags |= CODEC_FLAG_JPEG;
-		break;
-	case CODEC_TYPE_ZR36016:
-		m->readreg = zr36016_read;
-		m->writereg = zr36016_write;
-		m->flags |= CODEC_FLAG_VFE;
-		break;
-	}
-
-	return m;
-}
-
-static void zoran_subdev_notify(struct v4l2_subdev *sd, unsigned int cmd, void *arg)
-{
-	struct zoran *zr = to_zoran(sd->v4l2_dev);
-
-	/* Bt819 needs to reset its FIFO buffer using #FRST pin and
-	   LML33 card uses GPIO(7) for that. */
-	if (cmd == BT819_FIFO_RESET_LOW)
-		GPIO(zr, 7, 0);
-	else if (cmd == BT819_FIFO_RESET_HIGH)
-		GPIO(zr, 7, 1);
-}
-
-/*
- *   Scan for a Buz card (actually for the PCI controller ZR36057),
- *   request the irq and map the io memory
- */
-static int zoran_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
-{
-	unsigned char latency, need_latency;
-	struct zoran *zr;
-	int result;
-	struct videocodec_master *master_vfe = NULL;
-	struct videocodec_master *master_codec = NULL;
-	int card_num;
-	char *codec_name, *vfe_name;
-	unsigned int nr;
-
-
-	nr = zoran_num++;
-	if (nr >= BUZ_MAX) {
-		dprintk(1, KERN_ERR "%s: driver limited to %d card(s) maximum\n",
-			ZORAN_NAME, BUZ_MAX);
-		return -ENOENT;
-	}
-
-	zr = kzalloc(sizeof(struct zoran), GFP_KERNEL);
-	if (!zr) {
-		dprintk(1, KERN_ERR "%s: %s - kzalloc failed\n",
-			ZORAN_NAME, __func__);
-		return -ENOMEM;
-	}
-	zr->v4l2_dev.notify = zoran_subdev_notify;
-	if (v4l2_device_register(&pdev->dev, &zr->v4l2_dev))
-		goto zr_free_mem;
-	zr->pci_dev = pdev;
-	zr->id = nr;
-	snprintf(ZR_DEVNAME(zr), sizeof(ZR_DEVNAME(zr)), "MJPEG[%u]", zr->id);
-	if (v4l2_ctrl_handler_init(&zr->hdl, 10))
-		goto zr_unreg;
-	zr->v4l2_dev.ctrl_handler = &zr->hdl;
-	spin_lock_init(&zr->spinlock);
-	mutex_init(&zr->lock);
-	if (pci_enable_device(pdev))
-		goto zr_unreg;
-	zr->revision = zr->pci_dev->revision;
-
-	dprintk(1,
-		KERN_INFO
-		"%s: Zoran ZR360%c7 (rev %d), irq: %d, memory: 0x%08llx\n",
-		ZR_DEVNAME(zr), zr->revision < 2 ? '5' : '6', zr->revision,
-		zr->pci_dev->irq, (uint64_t)pci_resource_start(zr->pci_dev, 0));
-	if (zr->revision >= 2) {
-		dprintk(1,
-			KERN_INFO
-			"%s: Subsystem vendor=0x%04x id=0x%04x\n",
-			ZR_DEVNAME(zr), zr->pci_dev->subsystem_vendor,
-			zr->pci_dev->subsystem_device);
-	}
-
-	/* Use auto-detected card type? */
-	if (card[nr] == -1) {
-		if (zr->revision < 2) {
-			dprintk(1,
-				KERN_ERR
-				"%s: No card type specified, please use the card=X module parameter\n",
-				ZR_DEVNAME(zr));
-			dprintk(1,
-				KERN_ERR
-				"%s: It is not possible to auto-detect ZR36057 based cards\n",
-				ZR_DEVNAME(zr));
-			goto zr_unreg;
-		}
-
-		card_num = ent->driver_data;
-		if (card_num >= NUM_CARDS) {
-			dprintk(1,
-				KERN_ERR
-				"%s: Unknown card, try specifying card=X module parameter\n",
-				ZR_DEVNAME(zr));
-			goto zr_unreg;
-		}
-		dprintk(3,
-			KERN_DEBUG
-			"%s: %s() - card %s detected\n",
-			ZR_DEVNAME(zr), __func__, zoran_cards[card_num].name);
-	} else {
-		card_num = card[nr];
-		if (card_num >= NUM_CARDS || card_num < 0) {
-			dprintk(1,
-				KERN_ERR
-				"%s: User specified card type %d out of range (0 .. %d)\n",
-				ZR_DEVNAME(zr), card_num, NUM_CARDS - 1);
-			goto zr_unreg;
-		}
-	}
-
-	/* even though we make this a non pointer and thus
-	 * theoretically allow for making changes to this struct
-	 * on a per-individual card basis at runtime, this is
-	 * strongly discouraged. This structure is intended to
-	 * keep general card information, no settings or anything */
-	zr->card = zoran_cards[card_num];
-	snprintf(ZR_DEVNAME(zr), sizeof(ZR_DEVNAME(zr)),
-		 "%s[%u]", zr->card.name, zr->id);
-
-	zr->zr36057_mem = pci_ioremap_bar(zr->pci_dev, 0);
-	if (!zr->zr36057_mem) {
-		dprintk(1, KERN_ERR "%s: %s() - ioremap failed\n",
-			ZR_DEVNAME(zr), __func__);
-		goto zr_unreg;
-	}
-
-	result = request_irq(zr->pci_dev->irq, zoran_irq,
-			     IRQF_SHARED, ZR_DEVNAME(zr), zr);
-	if (result < 0) {
-		if (result == -EINVAL) {
-			dprintk(1,
-				KERN_ERR
-				"%s: %s - bad irq number or handler\n",
-				ZR_DEVNAME(zr), __func__);
-		} else if (result == -EBUSY) {
-			dprintk(1,
-				KERN_ERR
-				"%s: %s - IRQ %d busy, change your PnP config in BIOS\n",
-				ZR_DEVNAME(zr), __func__, zr->pci_dev->irq);
-		} else {
-			dprintk(1,
-				KERN_ERR
-				"%s: %s - can't assign irq, error code %d\n",
-				ZR_DEVNAME(zr), __func__, result);
-		}
-		goto zr_unmap;
-	}
-
-	/* set PCI latency timer */
-	pci_read_config_byte(zr->pci_dev, PCI_LATENCY_TIMER,
-			     &latency);
-	need_latency = zr->revision > 1 ? 32 : 48;
-	if (latency != need_latency) {
-		dprintk(2, KERN_INFO "%s: Changing PCI latency from %d to %d\n",
-			ZR_DEVNAME(zr), latency, need_latency);
-		pci_write_config_byte(zr->pci_dev, PCI_LATENCY_TIMER,
-				      need_latency);
-	}
-
-	zr36057_restart(zr);
-	/* i2c */
-	dprintk(2, KERN_INFO "%s: Initializing i2c bus...\n",
-		ZR_DEVNAME(zr));
-
-	if (zoran_register_i2c(zr) < 0) {
-		dprintk(1, KERN_ERR "%s: %s - can't initialize i2c bus\n",
-			ZR_DEVNAME(zr), __func__);
-		goto zr_free_irq;
-	}
-
-	zr->decoder = v4l2_i2c_new_subdev(&zr->v4l2_dev,
-		&zr->i2c_adapter, zr->card.i2c_decoder,
-		0, zr->card.addrs_decoder);
-
-	if (zr->card.i2c_encoder)
-		zr->encoder = v4l2_i2c_new_subdev(&zr->v4l2_dev,
-			&zr->i2c_adapter, zr->card.i2c_encoder,
-			0, zr->card.addrs_encoder);
-
-	dprintk(2,
-		KERN_INFO "%s: Initializing videocodec bus...\n",
-		ZR_DEVNAME(zr));
-
-	if (zr->card.video_codec) {
-		codec_name = codecid_to_modulename(zr->card.video_codec);
-		if (codec_name) {
-			result = request_module(codec_name);
-			if (result) {
-				dprintk(1,
-					KERN_ERR
-					"%s: failed to load modules %s: %d\n",
-					ZR_DEVNAME(zr), codec_name, result);
-			}
-		}
-	}
-	if (zr->card.video_vfe) {
-		vfe_name = codecid_to_modulename(zr->card.video_vfe);
-		if (vfe_name) {
-			result = request_module(vfe_name);
-			if (result < 0) {
-				dprintk(1,
-					KERN_ERR
-					"%s: failed to load modules %s: %d\n",
-					ZR_DEVNAME(zr), vfe_name, result);
-			}
-		}
-	}
-
-	/* reset JPEG codec */
-	jpeg_codec_sleep(zr, 1);
-	jpeg_codec_reset(zr);
-	/* video bus enabled */
-	/* display codec revision */
-	if (zr->card.video_codec != 0) {
-		master_codec = zoran_setup_videocodec(zr, zr->card.video_codec);
-		if (!master_codec)
-			goto zr_unreg_i2c;
-		zr->codec = videocodec_attach(master_codec);
-		if (!zr->codec) {
-			dprintk(1, KERN_ERR "%s: %s - no codec found\n",
-				ZR_DEVNAME(zr), __func__);
-			goto zr_free_codec;
-		}
-		if (zr->codec->type != zr->card.video_codec) {
-			dprintk(1, KERN_ERR "%s: %s - wrong codec\n",
-				ZR_DEVNAME(zr), __func__);
-			goto zr_detach_codec;
-		}
-	}
-	if (zr->card.video_vfe != 0) {
-		master_vfe = zoran_setup_videocodec(zr, zr->card.video_vfe);
-		if (!master_vfe)
-			goto zr_detach_codec;
-		zr->vfe = videocodec_attach(master_vfe);
-		if (!zr->vfe) {
-			dprintk(1, KERN_ERR "%s: %s - no VFE found\n",
-				ZR_DEVNAME(zr), __func__);
-			goto zr_free_vfe;
-		}
-		if (zr->vfe->type != zr->card.video_vfe) {
-			dprintk(1, KERN_ERR "%s: %s = wrong VFE\n",
-				ZR_DEVNAME(zr), __func__);
-			goto zr_detach_vfe;
-		}
-	}
-
-	/* take care of Natoma chipset and a revision 1 zr36057 */
-	if ((pci_pci_problems & PCIPCI_NATOMA) && zr->revision <= 1) {
-		zr->jpg_buffers.need_contiguous = 1;
-		dprintk(1, KERN_INFO
-			"%s: ZR36057/Natoma bug, max. buffer size is 128K\n",
-			ZR_DEVNAME(zr));
-	}
-
-	if (zr36057_init(zr) < 0)
-		goto zr_detach_vfe;
-
-	zoran_proc_init(zr);
-
-	return 0;
-
-zr_detach_vfe:
-	videocodec_detach(zr->vfe);
-zr_free_vfe:
-	kfree(master_vfe);
-zr_detach_codec:
-	videocodec_detach(zr->codec);
-zr_free_codec:
-	kfree(master_codec);
-zr_unreg_i2c:
-	zoran_unregister_i2c(zr);
-zr_free_irq:
-	btwrite(0, ZR36057_SPGPPCR);
-	free_irq(zr->pci_dev->irq, zr);
-zr_unmap:
-	iounmap(zr->zr36057_mem);
-zr_unreg:
-	v4l2_ctrl_handler_free(&zr->hdl);
-	v4l2_device_unregister(&zr->v4l2_dev);
-zr_free_mem:
-	kfree(zr);
-
-	return -ENODEV;
-}
-
-static struct pci_driver zoran_driver = {
-	.name = "zr36067",
-	.id_table = zr36067_pci_tbl,
-	.probe = zoran_probe,
-	.remove = zoran_remove,
-};
-
-static int __init zoran_init(void)
-{
-	int res;
-
-	printk(KERN_INFO "Zoran MJPEG board driver version %s\n",
-	       ZORAN_VERSION);
-
-	/* check the parameters we have been given, adjust if necessary */
-	if (v4l_nbufs < 2)
-		v4l_nbufs = 2;
-	if (v4l_nbufs > VIDEO_MAX_FRAME)
-		v4l_nbufs = VIDEO_MAX_FRAME;
-	/* The user specifies the in KB, we want them in byte
-	 * (and page aligned) */
-	v4l_bufsize = PAGE_ALIGN(v4l_bufsize * 1024);
-	if (v4l_bufsize < 32768)
-		v4l_bufsize = 32768;
-	/* 2 MB is arbitrary but sufficient for the maximum possible images */
-	if (v4l_bufsize > 2048 * 1024)
-		v4l_bufsize = 2048 * 1024;
-	if (jpg_nbufs < 4)
-		jpg_nbufs = 4;
-	if (jpg_nbufs > BUZ_MAX_FRAME)
-		jpg_nbufs = BUZ_MAX_FRAME;
-	jpg_bufsize = PAGE_ALIGN(jpg_bufsize * 1024);
-	if (jpg_bufsize < 8192)
-		jpg_bufsize = 8192;
-	if (jpg_bufsize > (512 * 1024))
-		jpg_bufsize = 512 * 1024;
-	/* Use parameter for vidmem or try to find a video card */
-	if (vidmem) {
-		dprintk(1,
-			KERN_INFO
-			"%s: Using supplied video memory base address @ 0x%lx\n",
-			ZORAN_NAME, vidmem);
-	}
-
-	/* some mainboards might not do PCI-PCI data transfer well */
-	if (pci_pci_problems & (PCIPCI_FAIL|PCIAGP_FAIL|PCIPCI_ALIMAGIK)) {
-		dprintk(1,
-			KERN_WARNING
-			"%s: chipset does not support reliable PCI-PCI DMA\n",
-			ZORAN_NAME);
-	}
-
-	res = pci_register_driver(&zoran_driver);
-	if (res) {
-		dprintk(1,
-			KERN_ERR
-			"%s: Unable to register ZR36057 driver\n",
-			ZORAN_NAME);
-		return res;
-	}
-
-	return 0;
-}
-
-static void __exit zoran_exit(void)
-{
-	pci_unregister_driver(&zoran_driver);
-}
-
-module_init(zoran_init);
-module_exit(zoran_exit);
diff --git a/drivers/staging/media/zoran/zoran_card.h b/drivers/staging/media/zoran/zoran_card.h
deleted file mode 100644
index 600b9a3f320c..000000000000
--- a/drivers/staging/media/zoran/zoran_card.h
+++ /dev/null
@@ -1,40 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0+ */
-/*
- * Zoran zr36057/zr36067 PCI controller driver, for the
- * Pinnacle/Miro DC10/DC10+/DC30/DC30+, Iomega Buz, Linux
- * Media Labs LML33/LML33R10.
- *
- * This part handles card-specific data and detection
- *
- * Copyright (C) 2000 Serguei Miridonov <mirsev@cicese.mx>
- *
- * Currently maintained by:
- *   Ronald Bultje    <rbultje@ronald.bitfreak.net>
- *   Laurent Pinchart <laurent.pinchart@skynet.be>
- *   Mailinglist      <mjpeg-users@lists.sf.net>
- */
-#ifndef __ZORAN_CARD_H__
-#define __ZORAN_CARD_H__
-
-extern int zr36067_debug;
-
-#define dprintk(num, format, args...) \
-	do { \
-		if (zr36067_debug >= num) \
-			printk(format, ##args); \
-	} while (0)
-
-/* Anybody who uses more than four? */
-#define BUZ_MAX 4
-
-extern const struct video_device zoran_template;
-
-extern int zoran_check_jpg_settings(struct zoran *zr,
-				    struct zoran_jpg_settings *settings,
-				    int try);
-extern void zoran_open_init_params(struct zoran *zr);
-extern void zoran_vdev_release(struct video_device *vdev);
-
-void zr36016_write(struct videocodec *codec, u16 reg, u32 val);
-
-#endif				/* __ZORAN_CARD_H__ */
diff --git a/drivers/staging/media/zoran/zoran_device.c b/drivers/staging/media/zoran/zoran_device.c
deleted file mode 100644
index 2191fe69b0f9..000000000000
--- a/drivers/staging/media/zoran/zoran_device.c
+++ /dev/null
@@ -1,1609 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0+
-/*
- * Zoran zr36057/zr36067 PCI controller driver, for the
- * Pinnacle/Miro DC10/DC10+/DC30/DC30+, Iomega Buz, Linux
- * Media Labs LML33/LML33R10.
- *
- * This part handles device access (PCI/I2C/codec/...)
- *
- * Copyright (C) 2000 Serguei Miridonov <mirsev@cicese.mx>
- *
- * Currently maintained by:
- *   Ronald Bultje    <rbultje@ronald.bitfreak.net>
- *   Laurent Pinchart <laurent.pinchart@skynet.be>
- *   Mailinglist      <mjpeg-users@lists.sf.net>
- */
-#include <linux/types.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/vmalloc.h>
-#include <linux/ktime.h>
-#include <linux/sched/signal.h>
-
-#include <linux/interrupt.h>
-#include <linux/proc_fs.h>
-#include <linux/i2c.h>
-#include <linux/i2c-algo-bit.h>
-#include <linux/videodev2.h>
-#include <media/v4l2-common.h>
-#include <linux/spinlock.h>
-#include <linux/sem.h>
-
-#include <linux/pci.h>
-#include <linux/delay.h>
-#include <linux/wait.h>
-
-#include <asm/byteorder.h>
-#include <asm/io.h>
-
-#include "videocodec.h"
-#include "zoran.h"
-#include "zoran_device.h"
-#include "zoran_card.h"
-
-#define IRQ_MASK ( ZR36057_ISR_GIRQ0 | \
-		   ZR36057_ISR_GIRQ1 | \
-		   ZR36057_ISR_JPEGRepIRQ )
-
-static bool lml33dpath;		/* default = 0
-				 * 1 will use digital path in capture
-				 * mode instead of analog. It can be
-				 * used for picture adjustments using
-				 * tool like xawtv while watching image
-				 * on TV monitor connected to the output.
-				 * However, due to absence of 75 Ohm
-				 * load on Bt819 input, there will be
-				 * some image imperfections */
-
-module_param(lml33dpath, bool, 0644);
-MODULE_PARM_DESC(lml33dpath,
-		 "Use digital path capture mode (on LML33 cards)");
-
-static void
-zr36057_init_vfe (struct zoran *zr);
-
-/*
- * General Purpose I/O and Guest bus access
- */
-
-/*
- * This is a bit tricky. When a board lacks a GPIO function, the corresponding
- * GPIO bit number in the card_info structure is set to 0.
- */
-
-void
-GPIO (struct zoran *zr,
-      int           bit,
-      unsigned int  value)
-{
-	u32 reg;
-	u32 mask;
-
-	/* Make sure the bit number is legal
-	 * A bit number of -1 (lacking) gives a mask of 0,
-	 * making it harmless */
-	mask = (1 << (24 + bit)) & 0xff000000;
-	reg = btread(ZR36057_GPPGCR1) & ~mask;
-	if (value) {
-		reg |= mask;
-	}
-	btwrite(reg, ZR36057_GPPGCR1);
-	udelay(1);
-}
-
-/*
- * Wait til post office is no longer busy
- */
-
-int
-post_office_wait (struct zoran *zr)
-{
-	u32 por;
-
-//      while (((por = btread(ZR36057_POR)) & (ZR36057_POR_POPen | ZR36057_POR_POTime)) == ZR36057_POR_POPen) {
-	while ((por = btread(ZR36057_POR)) & ZR36057_POR_POPen) {
-		/* wait for something to happen */
-	}
-	if ((por & ZR36057_POR_POTime) && !zr->card.gws_not_connected) {
-		/* In LML33/BUZ \GWS line is not connected, so it has always timeout set */
-		dprintk(1, KERN_INFO "%s: pop timeout %08x\n", ZR_DEVNAME(zr),
-			por);
-		return -1;
-	}
-
-	return 0;
-}
-
-int
-post_office_write (struct zoran *zr,
-		   unsigned int  guest,
-		   unsigned int  reg,
-		   unsigned int  value)
-{
-	u32 por;
-
-	por =
-	    ZR36057_POR_PODir | ZR36057_POR_POTime | ((guest & 7) << 20) |
-	    ((reg & 7) << 16) | (value & 0xFF);
-	btwrite(por, ZR36057_POR);
-
-	return post_office_wait(zr);
-}
-
-int
-post_office_read (struct zoran *zr,
-		  unsigned int  guest,
-		  unsigned int  reg)
-{
-	u32 por;
-
-	por = ZR36057_POR_POTime | ((guest & 7) << 20) | ((reg & 7) << 16);
-	btwrite(por, ZR36057_POR);
-	if (post_office_wait(zr) < 0) {
-		return -1;
-	}
-
-	return btread(ZR36057_POR) & 0xFF;
-}
-
-/*
- * detect guests
- */
-
-static void
-dump_guests (struct zoran *zr)
-{
-	if (zr36067_debug > 2) {
-		int i, guest[8];
-
-		for (i = 1; i < 8; i++) {	// Don't read jpeg codec here
-			guest[i] = post_office_read(zr, i, 0);
-		}
-
-		printk(KERN_INFO "%s: Guests: %*ph\n",
-		       ZR_DEVNAME(zr), 8, guest);
-	}
-}
-
-void
-detect_guest_activity (struct zoran *zr)
-{
-	int timeout, i, j, res, guest[8], guest0[8], change[8][3];
-	ktime_t t0, t1;
-
-	dump_guests(zr);
-	printk(KERN_INFO "%s: Detecting guests activity, please wait...\n",
-	       ZR_DEVNAME(zr));
-	for (i = 1; i < 8; i++) {	// Don't read jpeg codec here
-		guest0[i] = guest[i] = post_office_read(zr, i, 0);
-	}
-
-	timeout = 0;
-	j = 0;
-	t0 = ktime_get();
-	while (timeout < 10000) {
-		udelay(10);
-		timeout++;
-		for (i = 1; (i < 8) && (j < 8); i++) {
-			res = post_office_read(zr, i, 0);
-			if (res != guest[i]) {
-				t1 = ktime_get();
-				change[j][0] = ktime_to_us(ktime_sub(t1, t0));
-				t0 = t1;
-				change[j][1] = i;
-				change[j][2] = res;
-				j++;
-				guest[i] = res;
-			}
-		}
-		if (j >= 8)
-			break;
-	}
-
-	printk(KERN_INFO "%s: Guests: %*ph\n", ZR_DEVNAME(zr), 8, guest0);
-
-	if (j == 0) {
-		printk(KERN_INFO "%s: No activity detected.\n", ZR_DEVNAME(zr));
-		return;
-	}
-	for (i = 0; i < j; i++) {
-		printk(KERN_INFO "%s: %6d: %d => 0x%02x\n", ZR_DEVNAME(zr),
-		       change[i][0], change[i][1], change[i][2]);
-	}
-}
-
-/*
- * JPEG Codec access
- */
-
-void
-jpeg_codec_sleep (struct zoran *zr,
-		  int           sleep)
-{
-	GPIO(zr, zr->card.gpio[ZR_GPIO_JPEG_SLEEP], !sleep);
-	if (!sleep) {
-		dprintk(3,
-			KERN_DEBUG
-			"%s: jpeg_codec_sleep() - wake GPIO=0x%08x\n",
-			ZR_DEVNAME(zr), btread(ZR36057_GPPGCR1));
-		udelay(500);
-	} else {
-		dprintk(3,
-			KERN_DEBUG
-			"%s: jpeg_codec_sleep() - sleep GPIO=0x%08x\n",
-			ZR_DEVNAME(zr), btread(ZR36057_GPPGCR1));
-		udelay(2);
-	}
-}
-
-int
-jpeg_codec_reset (struct zoran *zr)
-{
-	/* Take the codec out of sleep */
-	jpeg_codec_sleep(zr, 0);
-
-	if (zr->card.gpcs[GPCS_JPEG_RESET] != 0xff) {
-		post_office_write(zr, zr->card.gpcs[GPCS_JPEG_RESET], 0,
-				  0);
-		udelay(2);
-	} else {
-		GPIO(zr, zr->card.gpio[ZR_GPIO_JPEG_RESET], 0);
-		udelay(2);
-		GPIO(zr, zr->card.gpio[ZR_GPIO_JPEG_RESET], 1);
-		udelay(2);
-	}
-
-	return 0;
-}
-
-/*
- *   Set the registers for the size we have specified. Don't bother
- *   trying to understand this without the ZR36057 manual in front of
- *   you [AC].
- *
- *   PS: The manual is free for download in .pdf format from
- *   www.zoran.com - nicely done those folks.
- */
-
-static void
-zr36057_adjust_vfe (struct zoran          *zr,
-		    enum zoran_codec_mode  mode)
-{
-	u32 reg;
-
-	switch (mode) {
-	case BUZ_MODE_MOTION_DECOMPRESS:
-		btand(~ZR36057_VFESPFR_ExtFl, ZR36057_VFESPFR);
-		reg = btread(ZR36057_VFEHCR);
-		if ((reg & (1 << 10)) && zr->card.type != LML33R10) {
-			reg += ((1 << 10) | 1);
-		}
-		btwrite(reg, ZR36057_VFEHCR);
-		break;
-	case BUZ_MODE_MOTION_COMPRESS:
-	case BUZ_MODE_IDLE:
-	default:
-		if ((zr->norm & V4L2_STD_NTSC) ||
-		    (zr->card.type == LML33R10 &&
-		     (zr->norm & V4L2_STD_PAL)))
-			btand(~ZR36057_VFESPFR_ExtFl, ZR36057_VFESPFR);
-		else
-			btor(ZR36057_VFESPFR_ExtFl, ZR36057_VFESPFR);
-		reg = btread(ZR36057_VFEHCR);
-		if (!(reg & (1 << 10)) && zr->card.type != LML33R10) {
-			reg -= ((1 << 10) | 1);
-		}
-		btwrite(reg, ZR36057_VFEHCR);
-		break;
-	}
-}
-
-/*
- * set geometry
- */
-
-static void
-zr36057_set_vfe (struct zoran              *zr,
-		 int                        video_width,
-		 int                        video_height,
-		 const struct zoran_format *format)
-{
-	struct tvnorm *tvn;
-	unsigned HStart, HEnd, VStart, VEnd;
-	unsigned DispMode;
-	unsigned VidWinWid, VidWinHt;
-	unsigned hcrop1, hcrop2, vcrop1, vcrop2;
-	unsigned Wa, We, Ha, He;
-	unsigned X, Y, HorDcm, VerDcm;
-	u32 reg;
-	unsigned mask_line_size;
-
-	tvn = zr->timing;
-
-	Wa = tvn->Wa;
-	Ha = tvn->Ha;
-
-	dprintk(2, KERN_INFO "%s: set_vfe() - width = %d, height = %d\n",
-		ZR_DEVNAME(zr), video_width, video_height);
-
-	if (video_width < BUZ_MIN_WIDTH ||
-	    video_height < BUZ_MIN_HEIGHT ||
-	    video_width > Wa || video_height > Ha) {
-		dprintk(1, KERN_ERR "%s: set_vfe: w=%d h=%d not valid\n",
-			ZR_DEVNAME(zr), video_width, video_height);
-		return;
-	}
-
-	/**** zr36057 ****/
-
-	/* horizontal */
-	VidWinWid = video_width;
-	X = DIV_ROUND_UP(VidWinWid * 64, tvn->Wa);
-	We = (VidWinWid * 64) / X;
-	HorDcm = 64 - X;
-	hcrop1 = 2 * ((tvn->Wa - We) / 4);
-	hcrop2 = tvn->Wa - We - hcrop1;
-	HStart = tvn->HStart ? tvn->HStart : 1;
-	/* (Ronald) Original comment:
-	 * "| 1 Doesn't have any effect, tested on both a DC10 and a DC10+"
-	 * this is false. It inverses chroma values on the LML33R10 (so Cr
-	 * suddenly is shown as Cb and reverse, really cool effect if you
-	 * want to see blue faces, not useful otherwise). So don't use |1.
-	 * However, the DC10 has '0' as HStart, but does need |1, so we
-	 * use a dirty check...
-	 */
-	HEnd = HStart + tvn->Wa - 1;
-	HStart += hcrop1;
-	HEnd -= hcrop2;
-	reg = ((HStart & ZR36057_VFEHCR_Hmask) << ZR36057_VFEHCR_HStart)
-	    | ((HEnd & ZR36057_VFEHCR_Hmask) << ZR36057_VFEHCR_HEnd);
-	if (zr->card.vfe_pol.hsync_pol)
-		reg |= ZR36057_VFEHCR_HSPol;
-	btwrite(reg, ZR36057_VFEHCR);
-
-	/* Vertical */
-	DispMode = !(video_height > BUZ_MAX_HEIGHT / 2);
-	VidWinHt = DispMode ? video_height : video_height / 2;
-	Y = DIV_ROUND_UP(VidWinHt * 64 * 2, tvn->Ha);
-	He = (VidWinHt * 64) / Y;
-	VerDcm = 64 - Y;
-	vcrop1 = (tvn->Ha / 2 - He) / 2;
-	vcrop2 = tvn->Ha / 2 - He - vcrop1;
-	VStart = tvn->VStart;
-	VEnd = VStart + tvn->Ha / 2;	// - 1; FIXME SnapShot times out with -1 in 768*576 on the DC10 - LP
-	VStart += vcrop1;
-	VEnd -= vcrop2;
-	reg = ((VStart & ZR36057_VFEVCR_Vmask) << ZR36057_VFEVCR_VStart)
-	    | ((VEnd & ZR36057_VFEVCR_Vmask) << ZR36057_VFEVCR_VEnd);
-	if (zr->card.vfe_pol.vsync_pol)
-		reg |= ZR36057_VFEVCR_VSPol;
-	btwrite(reg, ZR36057_VFEVCR);
-
-	/* scaler and pixel format */
-	reg = 0;
-	reg |= (HorDcm << ZR36057_VFESPFR_HorDcm);
-	reg |= (VerDcm << ZR36057_VFESPFR_VerDcm);
-	reg |= (DispMode << ZR36057_VFESPFR_DispMode);
-	/* RJ: I don't know, why the following has to be the opposite
-	 * of the corresponding ZR36060 setting, but only this way
-	 * we get the correct colors when uncompressing to the screen  */
-	//reg |= ZR36057_VFESPFR_VCLKPol; /**/
-	/* RJ: Don't know if that is needed for NTSC also */
-	if (!(zr->norm & V4L2_STD_NTSC))
-		reg |= ZR36057_VFESPFR_ExtFl;	// NEEDED!!!!!!! Wolfgang
-	reg |= ZR36057_VFESPFR_TopField;
-	if (HorDcm >= 48) {
-		reg |= 3 << ZR36057_VFESPFR_HFilter;	/* 5 tap filter */
-	} else if (HorDcm >= 32) {
-		reg |= 2 << ZR36057_VFESPFR_HFilter;	/* 4 tap filter */
-	} else if (HorDcm >= 16) {
-		reg |= 1 << ZR36057_VFESPFR_HFilter;	/* 3 tap filter */
-	}
-	reg |= format->vfespfr;
-	btwrite(reg, ZR36057_VFESPFR);
-
-	/* display configuration */
-	reg = (16 << ZR36057_VDCR_MinPix)
-	    | (VidWinHt << ZR36057_VDCR_VidWinHt)
-	    | (VidWinWid << ZR36057_VDCR_VidWinWid);
-	if (pci_pci_problems & PCIPCI_TRITON)
-		// || zr->revision < 1) // Revision 1 has also Triton support
-		reg &= ~ZR36057_VDCR_Triton;
-	else
-		reg |= ZR36057_VDCR_Triton;
-	btwrite(reg, ZR36057_VDCR);
-
-	/* (Ronald) don't write this if overlay_mask = NULL */
-	if (zr->overlay_mask) {
-		/* Write overlay clipping mask data, but don't enable overlay clipping */
-		/* RJ: since this makes only sense on the screen, we use
-		 * zr->overlay_settings.width instead of video_width */
-
-		mask_line_size = (BUZ_MAX_WIDTH + 31) / 32;
-		reg = virt_to_bus(zr->overlay_mask);
-		btwrite(reg, ZR36057_MMTR);
-		reg = virt_to_bus(zr->overlay_mask + mask_line_size);
-		btwrite(reg, ZR36057_MMBR);
-		reg =
-		    mask_line_size - (zr->overlay_settings.width +
-				      31) / 32;
-		if (DispMode == 0)
-			reg += mask_line_size;
-		reg <<= ZR36057_OCR_MaskStride;
-		btwrite(reg, ZR36057_OCR);
-	}
-
-	zr36057_adjust_vfe(zr, zr->codec_mode);
-}
-
-/*
- * Switch overlay on or off
- */
-
-void
-zr36057_overlay (struct zoran *zr,
-		 int           on)
-{
-	u32 reg;
-
-	if (on) {
-		/* do the necessary settings ... */
-		btand(~ZR36057_VDCR_VidEn, ZR36057_VDCR);	/* switch it off first */
-
-		zr36057_set_vfe(zr,
-				zr->overlay_settings.width,
-				zr->overlay_settings.height,
-				zr->overlay_settings.format);
-
-		/* Start and length of each line MUST be 4-byte aligned.
-		 * This should be already checked before the call to this routine.
-		 * All error messages are internal driver checking only! */
-
-		/* video display top and bottom registers */
-		reg = (long) zr->vbuf_base +
-		    zr->overlay_settings.x *
-		    ((zr->overlay_settings.format->depth + 7) / 8) +
-		    zr->overlay_settings.y *
-		    zr->vbuf_bytesperline;
-		btwrite(reg, ZR36057_VDTR);
-		if (reg & 3)
-			dprintk(1,
-				KERN_ERR
-				"%s: zr36057_overlay() - video_address not aligned\n",
-				ZR_DEVNAME(zr));
-		if (zr->overlay_settings.height > BUZ_MAX_HEIGHT / 2)
-			reg += zr->vbuf_bytesperline;
-		btwrite(reg, ZR36057_VDBR);
-
-		/* video stride, status, and frame grab register */
-		reg = zr->vbuf_bytesperline -
-		    zr->overlay_settings.width *
-		    ((zr->overlay_settings.format->depth + 7) / 8);
-		if (zr->overlay_settings.height > BUZ_MAX_HEIGHT / 2)
-			reg += zr->vbuf_bytesperline;
-		if (reg & 3)
-			dprintk(1,
-				KERN_ERR
-				"%s: zr36057_overlay() - video_stride not aligned\n",
-				ZR_DEVNAME(zr));
-		reg = reg << ZR36057_VSSFGR_DispStride;
-		reg |= ZR36057_VSSFGR_VidOvf;	/* clear overflow status */
-		btwrite(reg, ZR36057_VSSFGR);
-
-		/* Set overlay clipping */
-		if (zr->overlay_settings.clipcount > 0)
-			btor(ZR36057_OCR_OvlEnable, ZR36057_OCR);
-
-		/* ... and switch it on */
-		btor(ZR36057_VDCR_VidEn, ZR36057_VDCR);
-	} else {
-		/* Switch it off */
-		btand(~ZR36057_VDCR_VidEn, ZR36057_VDCR);
-	}
-}
-
-/*
- * The overlay mask has one bit for each pixel on a scan line,
- *  and the maximum window size is BUZ_MAX_WIDTH * BUZ_MAX_HEIGHT pixels.
- */
-
-void write_overlay_mask(struct zoran_fh *fh, struct v4l2_clip *vp, int count)
-{
-	struct zoran *zr = fh->zr;
-	unsigned mask_line_size = (BUZ_MAX_WIDTH + 31) / 32;
-	u32 *mask;
-	int x, y, width, height;
-	unsigned i, j, k;
-
-	/* fill mask with one bits */
-	memset(fh->overlay_mask, ~0, mask_line_size * 4 * BUZ_MAX_HEIGHT);
-
-	for (i = 0; i < count; ++i) {
-		/* pick up local copy of clip */
-		x = vp[i].c.left;
-		y = vp[i].c.top;
-		width = vp[i].c.width;
-		height = vp[i].c.height;
-
-		/* trim clips that extend beyond the window */
-		if (x < 0) {
-			width += x;
-			x = 0;
-		}
-		if (y < 0) {
-			height += y;
-			y = 0;
-		}
-		if (x + width > fh->overlay_settings.width) {
-			width = fh->overlay_settings.width - x;
-		}
-		if (y + height > fh->overlay_settings.height) {
-			height = fh->overlay_settings.height - y;
-		}
-
-		/* ignore degenerate clips */
-		if (height <= 0) {
-			continue;
-		}
-		if (width <= 0) {
-			continue;
-		}
-
-		/* apply clip for each scan line */
-		for (j = 0; j < height; ++j) {
-			/* reset bit for each pixel */
-			/* this can be optimized later if need be */
-			mask = fh->overlay_mask + (y + j) * mask_line_size;
-			for (k = 0; k < width; ++k) {
-				mask[(x + k) / 32] &=
-				    ~((u32) 1 << (x + k) % 32);
-			}
-		}
-	}
-}
-
-/* Enable/Disable uncompressed memory grabbing of the 36057 */
-
-void
-zr36057_set_memgrab (struct zoran *zr,
-		     int           mode)
-{
-	if (mode) {
-		/* We only check SnapShot and not FrameGrab here.  SnapShot==1
-		 * means a capture is already in progress, but FrameGrab==1
-		 * doesn't necessary mean that.  It's more correct to say a 1
-		 * to 0 transition indicates a capture completed.  If a
-		 * capture is pending when capturing is tuned off, FrameGrab
-		 * will be stuck at 1 until capturing is turned back on.
-		 */
-		if (btread(ZR36057_VSSFGR) & ZR36057_VSSFGR_SnapShot)
-			dprintk(1,
-				KERN_WARNING
-				"%s: zr36057_set_memgrab(1) with SnapShot on!?\n",
-				ZR_DEVNAME(zr));
-
-		/* switch on VSync interrupts */
-		btwrite(IRQ_MASK, ZR36057_ISR);	// Clear Interrupts
-		btor(zr->card.vsync_int, ZR36057_ICR);	// SW
-
-		/* enable SnapShot */
-		btor(ZR36057_VSSFGR_SnapShot, ZR36057_VSSFGR);
-
-		/* Set zr36057 video front end  and enable video */
-		zr36057_set_vfe(zr, zr->v4l_settings.width,
-				zr->v4l_settings.height,
-				zr->v4l_settings.format);
-
-		zr->v4l_memgrab_active = 1;
-	} else {
-		/* switch off VSync interrupts */
-		btand(~zr->card.vsync_int, ZR36057_ICR);	// SW
-
-		zr->v4l_memgrab_active = 0;
-		zr->v4l_grab_frame = NO_GRAB_ACTIVE;
-
-		/* re-enable grabbing to screen if it was running */
-		if (zr->v4l_overlay_active) {
-			zr36057_overlay(zr, 1);
-		} else {
-			btand(~ZR36057_VDCR_VidEn, ZR36057_VDCR);
-			btand(~ZR36057_VSSFGR_SnapShot, ZR36057_VSSFGR);
-		}
-	}
-}
-
-int
-wait_grab_pending (struct zoran *zr)
-{
-	unsigned long flags;
-
-	/* wait until all pending grabs are finished */
-
-	if (!zr->v4l_memgrab_active)
-		return 0;
-
-	wait_event_interruptible(zr->v4l_capq,
-			(zr->v4l_pend_tail == zr->v4l_pend_head));
-	if (signal_pending(current))
-		return -ERESTARTSYS;
-
-	spin_lock_irqsave(&zr->spinlock, flags);
-	zr36057_set_memgrab(zr, 0);
-	spin_unlock_irqrestore(&zr->spinlock, flags);
-
-	return 0;
-}
-
-/*****************************************************************************
- *                                                                           *
- *  Set up the Buz-specific MJPEG part                                       *
- *                                                                           *
- *****************************************************************************/
-
-static inline void
-set_frame (struct zoran *zr,
-	   int           val)
-{
-	GPIO(zr, zr->card.gpio[ZR_GPIO_JPEG_FRAME], val);
-}
-
-static void
-set_videobus_dir (struct zoran *zr,
-		  int           val)
-{
-	switch (zr->card.type) {
-	case LML33:
-	case LML33R10:
-		if (!lml33dpath)
-			GPIO(zr, 5, val);
-		else
-			GPIO(zr, 5, 1);
-		break;
-	default:
-		GPIO(zr, zr->card.gpio[ZR_GPIO_VID_DIR],
-		     zr->card.gpio_pol[ZR_GPIO_VID_DIR] ? !val : val);
-		break;
-	}
-}
-
-static void
-init_jpeg_queue (struct zoran *zr)
-{
-	int i;
-
-	/* re-initialize DMA ring stuff */
-	zr->jpg_que_head = 0;
-	zr->jpg_dma_head = 0;
-	zr->jpg_dma_tail = 0;
-	zr->jpg_que_tail = 0;
-	zr->jpg_seq_num = 0;
-	zr->JPEG_error = 0;
-	zr->num_errors = 0;
-	zr->jpg_err_seq = 0;
-	zr->jpg_err_shift = 0;
-	zr->jpg_queued_num = 0;
-	for (i = 0; i < zr->jpg_buffers.num_buffers; i++) {
-		zr->jpg_buffers.buffer[i].state = BUZ_STATE_USER;	/* nothing going on */
-	}
-	for (i = 0; i < BUZ_NUM_STAT_COM; i++) {
-		zr->stat_com[i] = cpu_to_le32(1);	/* mark as unavailable to zr36057 */
-	}
-}
-
-static void
-zr36057_set_jpg (struct zoran          *zr,
-		 enum zoran_codec_mode  mode)
-{
-	struct tvnorm *tvn;
-	u32 reg;
-
-	tvn = zr->timing;
-
-	/* assert P_Reset, disable code transfer, deassert Active */
-	btwrite(0, ZR36057_JPC);
-
-	/* MJPEG compression mode */
-	switch (mode) {
-
-	case BUZ_MODE_MOTION_COMPRESS:
-	default:
-		reg = ZR36057_JMC_MJPGCmpMode;
-		break;
-
-	case BUZ_MODE_MOTION_DECOMPRESS:
-		reg = ZR36057_JMC_MJPGExpMode;
-		reg |= ZR36057_JMC_SyncMstr;
-		/* RJ: The following is experimental - improves the output to screen */
-		//if(zr->jpg_settings.VFIFO_FB) reg |= ZR36057_JMC_VFIFO_FB; // No, it doesn't. SM
-		break;
-
-	case BUZ_MODE_STILL_COMPRESS:
-		reg = ZR36057_JMC_JPGCmpMode;
-		break;
-
-	case BUZ_MODE_STILL_DECOMPRESS:
-		reg = ZR36057_JMC_JPGExpMode;
-		break;
-
-	}
-	reg |= ZR36057_JMC_JPG;
-	if (zr->jpg_settings.field_per_buff == 1)
-		reg |= ZR36057_JMC_Fld_per_buff;
-	btwrite(reg, ZR36057_JMC);
-
-	/* vertical */
-	btor(ZR36057_VFEVCR_VSPol, ZR36057_VFEVCR);
-	reg = (6 << ZR36057_VSP_VsyncSize) |
-	      (tvn->Ht << ZR36057_VSP_FrmTot);
-	btwrite(reg, ZR36057_VSP);
-	reg = ((zr->jpg_settings.img_y + tvn->VStart) << ZR36057_FVAP_NAY) |
-	      (zr->jpg_settings.img_height << ZR36057_FVAP_PAY);
-	btwrite(reg, ZR36057_FVAP);
-
-	/* horizontal */
-	if (zr->card.vfe_pol.hsync_pol)
-		btor(ZR36057_VFEHCR_HSPol, ZR36057_VFEHCR);
-	else
-		btand(~ZR36057_VFEHCR_HSPol, ZR36057_VFEHCR);
-	reg = ((tvn->HSyncStart) << ZR36057_HSP_HsyncStart) |
-	      (tvn->Wt << ZR36057_HSP_LineTot);
-	btwrite(reg, ZR36057_HSP);
-	reg = ((zr->jpg_settings.img_x +
-		tvn->HStart + 4) << ZR36057_FHAP_NAX) |
-	      (zr->jpg_settings.img_width << ZR36057_FHAP_PAX);
-	btwrite(reg, ZR36057_FHAP);
-
-	/* field process parameters */
-	if (zr->jpg_settings.odd_even)
-		reg = ZR36057_FPP_Odd_Even;
-	else
-		reg = 0;
-
-	btwrite(reg, ZR36057_FPP);
-
-	/* Set proper VCLK Polarity, else colors will be wrong during playback */
-	//btor(ZR36057_VFESPFR_VCLKPol, ZR36057_VFESPFR);
-
-	/* code base address */
-	reg = virt_to_bus(zr->stat_com);
-	btwrite(reg, ZR36057_JCBA);
-
-	/* FIFO threshold (FIFO is 160. double words) */
-	/* NOTE: decimal values here */
-	switch (mode) {
-
-	case BUZ_MODE_STILL_COMPRESS:
-	case BUZ_MODE_MOTION_COMPRESS:
-		if (zr->card.type != BUZ)
-			reg = 140;
-		else
-			reg = 60;
-		break;
-
-	case BUZ_MODE_STILL_DECOMPRESS:
-	case BUZ_MODE_MOTION_DECOMPRESS:
-		reg = 20;
-		break;
-
-	default:
-		reg = 80;
-		break;
-
-	}
-	btwrite(reg, ZR36057_JCFT);
-	zr36057_adjust_vfe(zr, mode);
-
-}
-
-void
-print_interrupts (struct zoran *zr)
-{
-	int res, noerr = 0;
-
-	printk(KERN_INFO "%s: interrupts received:", ZR_DEVNAME(zr));
-	if ((res = zr->field_counter) < -1 || res > 1) {
-		printk(KERN_CONT " FD:%d", res);
-	}
-	if ((res = zr->intr_counter_GIRQ1) != 0) {
-		printk(KERN_CONT " GIRQ1:%d", res);
-		noerr++;
-	}
-	if ((res = zr->intr_counter_GIRQ0) != 0) {
-		printk(KERN_CONT " GIRQ0:%d", res);
-		noerr++;
-	}
-	if ((res = zr->intr_counter_CodRepIRQ) != 0) {
-		printk(KERN_CONT " CodRepIRQ:%d", res);
-		noerr++;
-	}
-	if ((res = zr->intr_counter_JPEGRepIRQ) != 0) {
-		printk(KERN_CONT " JPEGRepIRQ:%d", res);
-		noerr++;
-	}
-	if (zr->JPEG_max_missed) {
-		printk(KERN_CONT " JPEG delays: max=%d min=%d", zr->JPEG_max_missed,
-		       zr->JPEG_min_missed);
-	}
-	if (zr->END_event_missed) {
-		printk(KERN_CONT " ENDs missed: %d", zr->END_event_missed);
-	}
-	//if (zr->jpg_queued_num) {
-	printk(KERN_CONT " queue_state=%ld/%ld/%ld/%ld", zr->jpg_que_tail,
-	       zr->jpg_dma_tail, zr->jpg_dma_head, zr->jpg_que_head);
-	//}
-	if (!noerr) {
-		printk(KERN_CONT ": no interrupts detected.");
-	}
-	printk(KERN_CONT "\n");
-}
-
-void
-clear_interrupt_counters (struct zoran *zr)
-{
-	zr->intr_counter_GIRQ1 = 0;
-	zr->intr_counter_GIRQ0 = 0;
-	zr->intr_counter_CodRepIRQ = 0;
-	zr->intr_counter_JPEGRepIRQ = 0;
-	zr->field_counter = 0;
-	zr->IRQ1_in = 0;
-	zr->IRQ1_out = 0;
-	zr->JPEG_in = 0;
-	zr->JPEG_out = 0;
-	zr->JPEG_0 = 0;
-	zr->JPEG_1 = 0;
-	zr->END_event_missed = 0;
-	zr->JPEG_missed = 0;
-	zr->JPEG_max_missed = 0;
-	zr->JPEG_min_missed = 0x7fffffff;
-}
-
-static u32
-count_reset_interrupt (struct zoran *zr)
-{
-	u32 isr;
-
-	if ((isr = btread(ZR36057_ISR) & 0x78000000)) {
-		if (isr & ZR36057_ISR_GIRQ1) {
-			btwrite(ZR36057_ISR_GIRQ1, ZR36057_ISR);
-			zr->intr_counter_GIRQ1++;
-		}
-		if (isr & ZR36057_ISR_GIRQ0) {
-			btwrite(ZR36057_ISR_GIRQ0, ZR36057_ISR);
-			zr->intr_counter_GIRQ0++;
-		}
-		if (isr & ZR36057_ISR_CodRepIRQ) {
-			btwrite(ZR36057_ISR_CodRepIRQ, ZR36057_ISR);
-			zr->intr_counter_CodRepIRQ++;
-		}
-		if (isr & ZR36057_ISR_JPEGRepIRQ) {
-			btwrite(ZR36057_ISR_JPEGRepIRQ, ZR36057_ISR);
-			zr->intr_counter_JPEGRepIRQ++;
-		}
-	}
-	return isr;
-}
-
-void
-jpeg_start (struct zoran *zr)
-{
-	int reg;
-
-	zr->frame_num = 0;
-
-	/* deassert P_reset, disable code transfer, deassert Active */
-	btwrite(ZR36057_JPC_P_Reset, ZR36057_JPC);
-	/* stop flushing the internal code buffer */
-	btand(~ZR36057_MCTCR_CFlush, ZR36057_MCTCR);
-	/* enable code transfer */
-	btor(ZR36057_JPC_CodTrnsEn, ZR36057_JPC);
-
-	/* clear IRQs */
-	btwrite(IRQ_MASK, ZR36057_ISR);
-	/* enable the JPEG IRQs */
-	btwrite(zr->card.jpeg_int |
-			ZR36057_ICR_JPEGRepIRQ |
-			ZR36057_ICR_IntPinEn,
-		ZR36057_ICR);
-
-	set_frame(zr, 0);	// \FRAME
-
-	/* set the JPEG codec guest ID */
-	reg = (zr->card.gpcs[1] << ZR36057_JCGI_JPEGuestID) |
-	       (0 << ZR36057_JCGI_JPEGuestReg);
-	btwrite(reg, ZR36057_JCGI);
-
-	if (zr->card.video_vfe == CODEC_TYPE_ZR36016 &&
-	    zr->card.video_codec == CODEC_TYPE_ZR36050) {
-		/* Enable processing on the ZR36016 */
-		if (zr->vfe)
-			zr36016_write(zr->vfe, 0, 1);
-
-		/* load the address of the GO register in the ZR36050 latch */
-		post_office_write(zr, 0, 0, 0);
-	}
-
-	/* assert Active */
-	btor(ZR36057_JPC_Active, ZR36057_JPC);
-
-	/* enable the Go generation */
-	btor(ZR36057_JMC_Go_en, ZR36057_JMC);
-	udelay(30);
-
-	set_frame(zr, 1);	// /FRAME
-
-	dprintk(3, KERN_DEBUG "%s: jpeg_start\n", ZR_DEVNAME(zr));
-}
-
-void
-zr36057_enable_jpg (struct zoran          *zr,
-		    enum zoran_codec_mode  mode)
-{
-	struct vfe_settings cap;
-	int field_size =
-	    zr->jpg_buffers.buffer_size / zr->jpg_settings.field_per_buff;
-
-	zr->codec_mode = mode;
-
-	cap.x = zr->jpg_settings.img_x;
-	cap.y = zr->jpg_settings.img_y;
-	cap.width = zr->jpg_settings.img_width;
-	cap.height = zr->jpg_settings.img_height;
-	cap.decimation =
-	    zr->jpg_settings.HorDcm | (zr->jpg_settings.VerDcm << 8);
-	cap.quality = zr->jpg_settings.jpg_comp.quality;
-
-	switch (mode) {
-
-	case BUZ_MODE_MOTION_COMPRESS: {
-		struct jpeg_app_marker app;
-		struct jpeg_com_marker com;
-
-		/* In motion compress mode, the decoder output must be enabled, and
-		 * the video bus direction set to input.
-		 */
-		set_videobus_dir(zr, 0);
-		decoder_call(zr, video, s_stream, 1);
-		encoder_call(zr, video, s_routing, 0, 0, 0);
-
-		/* Take the JPEG codec and the VFE out of sleep */
-		jpeg_codec_sleep(zr, 0);
-
-		/* set JPEG app/com marker */
-		app.appn = zr->jpg_settings.jpg_comp.APPn;
-		app.len = zr->jpg_settings.jpg_comp.APP_len;
-		memcpy(app.data, zr->jpg_settings.jpg_comp.APP_data, 60);
-		zr->codec->control(zr->codec, CODEC_S_JPEG_APP_DATA,
-				   sizeof(struct jpeg_app_marker), &app);
-
-		com.len = zr->jpg_settings.jpg_comp.COM_len;
-		memcpy(com.data, zr->jpg_settings.jpg_comp.COM_data, 60);
-		zr->codec->control(zr->codec, CODEC_S_JPEG_COM_DATA,
-				   sizeof(struct jpeg_com_marker), &com);
-
-		/* Setup the JPEG codec */
-		zr->codec->control(zr->codec, CODEC_S_JPEG_TDS_BYTE,
-				   sizeof(int), &field_size);
-		zr->codec->set_video(zr->codec, zr->timing, &cap,
-				     &zr->card.vfe_pol);
-		zr->codec->set_mode(zr->codec, CODEC_DO_COMPRESSION);
-
-		/* Setup the VFE */
-		if (zr->vfe) {
-			zr->vfe->control(zr->vfe, CODEC_S_JPEG_TDS_BYTE,
-					 sizeof(int), &field_size);
-			zr->vfe->set_video(zr->vfe, zr->timing, &cap,
-					   &zr->card.vfe_pol);
-			zr->vfe->set_mode(zr->vfe, CODEC_DO_COMPRESSION);
-		}
-
-		init_jpeg_queue(zr);
-		zr36057_set_jpg(zr, mode);	// \P_Reset, ... Video param, FIFO
-
-		clear_interrupt_counters(zr);
-		dprintk(2, KERN_INFO "%s: enable_jpg(MOTION_COMPRESS)\n",
-			ZR_DEVNAME(zr));
-		break;
-	}
-
-	case BUZ_MODE_MOTION_DECOMPRESS:
-		/* In motion decompression mode, the decoder output must be disabled, and
-		 * the video bus direction set to output.
-		 */
-		decoder_call(zr, video, s_stream, 0);
-		set_videobus_dir(zr, 1);
-		encoder_call(zr, video, s_routing, 1, 0, 0);
-
-		/* Take the JPEG codec and the VFE out of sleep */
-		jpeg_codec_sleep(zr, 0);
-		/* Setup the VFE */
-		if (zr->vfe) {
-			zr->vfe->set_video(zr->vfe, zr->timing, &cap,
-					   &zr->card.vfe_pol);
-			zr->vfe->set_mode(zr->vfe, CODEC_DO_EXPANSION);
-		}
-		/* Setup the JPEG codec */
-		zr->codec->set_video(zr->codec, zr->timing, &cap,
-				     &zr->card.vfe_pol);
-		zr->codec->set_mode(zr->codec, CODEC_DO_EXPANSION);
-
-		init_jpeg_queue(zr);
-		zr36057_set_jpg(zr, mode);	// \P_Reset, ... Video param, FIFO
-
-		clear_interrupt_counters(zr);
-		dprintk(2, KERN_INFO "%s: enable_jpg(MOTION_DECOMPRESS)\n",
-			ZR_DEVNAME(zr));
-		break;
-
-	case BUZ_MODE_IDLE:
-	default:
-		/* shut down processing */
-		btand(~(zr->card.jpeg_int | ZR36057_ICR_JPEGRepIRQ),
-		      ZR36057_ICR);
-		btwrite(zr->card.jpeg_int | ZR36057_ICR_JPEGRepIRQ,
-			ZR36057_ISR);
-		btand(~ZR36057_JMC_Go_en, ZR36057_JMC);	// \Go_en
-
-		msleep(50);
-
-		set_videobus_dir(zr, 0);
-		set_frame(zr, 1);	// /FRAME
-		btor(ZR36057_MCTCR_CFlush, ZR36057_MCTCR);	// /CFlush
-		btwrite(0, ZR36057_JPC);	// \P_Reset,\CodTrnsEn,\Active
-		btand(~ZR36057_JMC_VFIFO_FB, ZR36057_JMC);
-		btand(~ZR36057_JMC_SyncMstr, ZR36057_JMC);
-		jpeg_codec_reset(zr);
-		jpeg_codec_sleep(zr, 1);
-		zr36057_adjust_vfe(zr, mode);
-
-		decoder_call(zr, video, s_stream, 1);
-		encoder_call(zr, video, s_routing, 0, 0, 0);
-
-		dprintk(2, KERN_INFO "%s: enable_jpg(IDLE)\n", ZR_DEVNAME(zr));
-		break;
-
-	}
-}
-
-/* when this is called the spinlock must be held */
-void
-zoran_feed_stat_com (struct zoran *zr)
-{
-	/* move frames from pending queue to DMA */
-
-	int frame, i, max_stat_com;
-
-	max_stat_com =
-	    (zr->jpg_settings.TmpDcm ==
-	     1) ? BUZ_NUM_STAT_COM : (BUZ_NUM_STAT_COM >> 1);
-
-	while ((zr->jpg_dma_head - zr->jpg_dma_tail) < max_stat_com &&
-	       zr->jpg_dma_head < zr->jpg_que_head) {
-
-		frame = zr->jpg_pend[zr->jpg_dma_head & BUZ_MASK_FRAME];
-		if (zr->jpg_settings.TmpDcm == 1) {
-			/* fill 1 stat_com entry */
-			i = (zr->jpg_dma_head -
-			     zr->jpg_err_shift) & BUZ_MASK_STAT_COM;
-			if (!(zr->stat_com[i] & cpu_to_le32(1)))
-				break;
-			zr->stat_com[i] =
-			    cpu_to_le32(zr->jpg_buffers.buffer[frame].jpg.frag_tab_bus);
-		} else {
-			/* fill 2 stat_com entries */
-			i = ((zr->jpg_dma_head -
-			      zr->jpg_err_shift) & 1) * 2;
-			if (!(zr->stat_com[i] & cpu_to_le32(1)))
-				break;
-			zr->stat_com[i] =
-			    cpu_to_le32(zr->jpg_buffers.buffer[frame].jpg.frag_tab_bus);
-			zr->stat_com[i + 1] =
-			    cpu_to_le32(zr->jpg_buffers.buffer[frame].jpg.frag_tab_bus);
-		}
-		zr->jpg_buffers.buffer[frame].state = BUZ_STATE_DMA;
-		zr->jpg_dma_head++;
-
-	}
-	if (zr->codec_mode == BUZ_MODE_MOTION_DECOMPRESS)
-		zr->jpg_queued_num++;
-}
-
-/* when this is called the spinlock must be held */
-static void
-zoran_reap_stat_com (struct zoran *zr)
-{
-	/* move frames from DMA queue to done queue */
-
-	int i;
-	u32 stat_com;
-	unsigned int seq;
-	unsigned int dif;
-	struct zoran_buffer *buffer;
-	int frame;
-
-	/* In motion decompress we don't have a hardware frame counter,
-	 * we just count the interrupts here */
-
-	if (zr->codec_mode == BUZ_MODE_MOTION_DECOMPRESS) {
-		zr->jpg_seq_num++;
-	}
-	while (zr->jpg_dma_tail < zr->jpg_dma_head) {
-		if (zr->jpg_settings.TmpDcm == 1)
-			i = (zr->jpg_dma_tail -
-			     zr->jpg_err_shift) & BUZ_MASK_STAT_COM;
-		else
-			i = ((zr->jpg_dma_tail -
-			      zr->jpg_err_shift) & 1) * 2 + 1;
-
-		stat_com = le32_to_cpu(zr->stat_com[i]);
-
-		if ((stat_com & 1) == 0) {
-			return;
-		}
-		frame = zr->jpg_pend[zr->jpg_dma_tail & BUZ_MASK_FRAME];
-		buffer = &zr->jpg_buffers.buffer[frame];
-		buffer->bs.ts = ktime_get_ns();
-
-		if (zr->codec_mode == BUZ_MODE_MOTION_COMPRESS) {
-			buffer->bs.length = (stat_com & 0x7fffff) >> 1;
-
-			/* update sequence number with the help of the counter in stat_com */
-
-			seq = ((stat_com >> 24) + zr->jpg_err_seq) & 0xff;
-			dif = (seq - zr->jpg_seq_num) & 0xff;
-			zr->jpg_seq_num += dif;
-		} else {
-			buffer->bs.length = 0;
-		}
-		buffer->bs.seq =
-		    zr->jpg_settings.TmpDcm ==
-		    2 ? (zr->jpg_seq_num >> 1) : zr->jpg_seq_num;
-		buffer->state = BUZ_STATE_DONE;
-
-		zr->jpg_dma_tail++;
-	}
-}
-
-static void zoran_restart(struct zoran *zr)
-{
-	/* Now the stat_comm buffer is ready for restart */
-	unsigned int status = 0;
-	int mode;
-
-	if (zr->codec_mode == BUZ_MODE_MOTION_COMPRESS) {
-		decoder_call(zr, video, g_input_status, &status);
-		mode = CODEC_DO_COMPRESSION;
-	} else {
-		status = V4L2_IN_ST_NO_SIGNAL;
-		mode = CODEC_DO_EXPANSION;
-	}
-	if (zr->codec_mode == BUZ_MODE_MOTION_DECOMPRESS ||
-	    !(status & V4L2_IN_ST_NO_SIGNAL)) {
-		/********** RESTART code *************/
-		jpeg_codec_reset(zr);
-		zr->codec->set_mode(zr->codec, mode);
-		zr36057_set_jpg(zr, zr->codec_mode);
-		jpeg_start(zr);
-
-		if (zr->num_errors <= 8)
-			dprintk(2, KERN_INFO "%s: Restart\n",
-				ZR_DEVNAME(zr));
-
-		zr->JPEG_missed = 0;
-		zr->JPEG_error = 2;
-		/********** End RESTART code ***********/
-	}
-}
-
-static void
-error_handler (struct zoran *zr,
-	       u32           astat,
-	       u32           stat)
-{
-	int i;
-
-	/* This is JPEG error handling part */
-	if (zr->codec_mode != BUZ_MODE_MOTION_COMPRESS &&
-	    zr->codec_mode != BUZ_MODE_MOTION_DECOMPRESS) {
-		return;
-	}
-
-	if ((stat & 1) == 0 &&
-	    zr->codec_mode == BUZ_MODE_MOTION_COMPRESS &&
-	    zr->jpg_dma_tail - zr->jpg_que_tail >= zr->jpg_buffers.num_buffers) {
-		/* No free buffers... */
-		zoran_reap_stat_com(zr);
-		zoran_feed_stat_com(zr);
-		wake_up_interruptible(&zr->jpg_capq);
-		zr->JPEG_missed = 0;
-		return;
-	}
-
-	if (zr->JPEG_error == 1) {
-		zoran_restart(zr);
-		return;
-	}
-
-	/*
-	 * First entry: error just happened during normal operation
-	 *
-	 * In BUZ_MODE_MOTION_COMPRESS:
-	 *
-	 * Possible glitch in TV signal. In this case we should
-	 * stop the codec and wait for good quality signal before
-	 * restarting it to avoid further problems
-	 *
-	 * In BUZ_MODE_MOTION_DECOMPRESS:
-	 *
-	 * Bad JPEG frame: we have to mark it as processed (codec crashed
-	 * and was not able to do it itself), and to remove it from queue.
-	 */
-	btand(~ZR36057_JMC_Go_en, ZR36057_JMC);
-	udelay(1);
-	stat = stat | (post_office_read(zr, 7, 0) & 3) << 8;
-	btwrite(0, ZR36057_JPC);
-	btor(ZR36057_MCTCR_CFlush, ZR36057_MCTCR);
-	jpeg_codec_reset(zr);
-	jpeg_codec_sleep(zr, 1);
-	zr->JPEG_error = 1;
-	zr->num_errors++;
-
-	/* Report error */
-	if (zr36067_debug > 1 && zr->num_errors <= 8) {
-		long frame;
-		int j;
-
-		frame = zr->jpg_pend[zr->jpg_dma_tail & BUZ_MASK_FRAME];
-		printk(KERN_ERR
-		       "%s: JPEG error stat=0x%08x(0x%08x) queue_state=%ld/%ld/%ld/%ld seq=%ld frame=%ld. Codec stopped. ",
-		       ZR_DEVNAME(zr), stat, zr->last_isr,
-		       zr->jpg_que_tail, zr->jpg_dma_tail,
-		       zr->jpg_dma_head, zr->jpg_que_head,
-		       zr->jpg_seq_num, frame);
-		printk(KERN_INFO "stat_com frames:");
-		for (j = 0; j < BUZ_NUM_STAT_COM; j++) {
-			for (i = 0; i < zr->jpg_buffers.num_buffers; i++) {
-				if (le32_to_cpu(zr->stat_com[j]) == zr->jpg_buffers.buffer[i].jpg.frag_tab_bus)
-					printk(KERN_CONT "% d->%d", j, i);
-			}
-		}
-		printk(KERN_CONT "\n");
-	}
-	/* Find an entry in stat_com and rotate contents */
-	if (zr->jpg_settings.TmpDcm == 1)
-		i = (zr->jpg_dma_tail - zr->jpg_err_shift) & BUZ_MASK_STAT_COM;
-	else
-		i = ((zr->jpg_dma_tail - zr->jpg_err_shift) & 1) * 2;
-	if (zr->codec_mode == BUZ_MODE_MOTION_DECOMPRESS) {
-		/* Mimic zr36067 operation */
-		zr->stat_com[i] |= cpu_to_le32(1);
-		if (zr->jpg_settings.TmpDcm != 1)
-			zr->stat_com[i + 1] |= cpu_to_le32(1);
-		/* Refill */
-		zoran_reap_stat_com(zr);
-		zoran_feed_stat_com(zr);
-		wake_up_interruptible(&zr->jpg_capq);
-		/* Find an entry in stat_com again after refill */
-		if (zr->jpg_settings.TmpDcm == 1)
-			i = (zr->jpg_dma_tail - zr->jpg_err_shift) & BUZ_MASK_STAT_COM;
-		else
-			i = ((zr->jpg_dma_tail - zr->jpg_err_shift) & 1) * 2;
-	}
-	if (i) {
-		/* Rotate stat_comm entries to make current entry first */
-		int j;
-		__le32 bus_addr[BUZ_NUM_STAT_COM];
-
-		/* Here we are copying the stat_com array, which
-		 * is already in little endian format, so
-		 * no endian conversions here
-		 */
-		memcpy(bus_addr, zr->stat_com, sizeof(bus_addr));
-
-		for (j = 0; j < BUZ_NUM_STAT_COM; j++)
-			zr->stat_com[j] = bus_addr[(i + j) & BUZ_MASK_STAT_COM];
-
-		zr->jpg_err_shift += i;
-		zr->jpg_err_shift &= BUZ_MASK_STAT_COM;
-	}
-	if (zr->codec_mode == BUZ_MODE_MOTION_COMPRESS)
-		zr->jpg_err_seq = zr->jpg_seq_num;	/* + 1; */
-	zoran_restart(zr);
-}
-
-irqreturn_t
-zoran_irq (int             irq,
-	   void           *dev_id)
-{
-	u32 stat, astat;
-	int count;
-	struct zoran *zr;
-	unsigned long flags;
-
-	zr = dev_id;
-	count = 0;
-
-	if (zr->testing) {
-		/* Testing interrupts */
-		spin_lock_irqsave(&zr->spinlock, flags);
-		while ((stat = count_reset_interrupt(zr))) {
-			if (count++ > 100) {
-				btand(~ZR36057_ICR_IntPinEn, ZR36057_ICR);
-				dprintk(1,
-					KERN_ERR
-					"%s: IRQ lockup while testing, isr=0x%08x, cleared int mask\n",
-					ZR_DEVNAME(zr), stat);
-				wake_up_interruptible(&zr->test_q);
-			}
-		}
-		zr->last_isr = stat;
-		spin_unlock_irqrestore(&zr->spinlock, flags);
-		return IRQ_HANDLED;
-	}
-
-	spin_lock_irqsave(&zr->spinlock, flags);
-	while (1) {
-		/* get/clear interrupt status bits */
-		stat = count_reset_interrupt(zr);
-		astat = stat & IRQ_MASK;
-		if (!astat) {
-			break;
-		}
-		dprintk(4,
-			KERN_DEBUG
-			"zoran_irq: astat: 0x%08x, mask: 0x%08x\n",
-			astat, btread(ZR36057_ICR));
-		if (astat & zr->card.vsync_int) {	// SW
-
-			if (zr->codec_mode == BUZ_MODE_MOTION_DECOMPRESS ||
-			    zr->codec_mode == BUZ_MODE_MOTION_COMPRESS) {
-				/* count missed interrupts */
-				zr->JPEG_missed++;
-			}
-			//post_office_read(zr,1,0);
-			/* Interrupts may still happen when
-			 * zr->v4l_memgrab_active is switched off.
-			 * We simply ignore them */
-
-			if (zr->v4l_memgrab_active) {
-				/* A lot more checks should be here ... */
-				if ((btread(ZR36057_VSSFGR) & ZR36057_VSSFGR_SnapShot) == 0)
-					dprintk(1,
-						KERN_WARNING
-						"%s: BuzIRQ with SnapShot off ???\n",
-						ZR_DEVNAME(zr));
-
-				if (zr->v4l_grab_frame != NO_GRAB_ACTIVE) {
-					/* There is a grab on a frame going on, check if it has finished */
-					if ((btread(ZR36057_VSSFGR) & ZR36057_VSSFGR_FrameGrab) == 0) {
-						/* it is finished, notify the user */
-
-						zr->v4l_buffers.buffer[zr->v4l_grab_frame].state = BUZ_STATE_DONE;
-						zr->v4l_buffers.buffer[zr->v4l_grab_frame].bs.seq = zr->v4l_grab_seq;
-						zr->v4l_buffers.buffer[zr->v4l_grab_frame].bs.ts = ktime_get_ns();
-						zr->v4l_grab_frame = NO_GRAB_ACTIVE;
-						zr->v4l_pend_tail++;
-					}
-				}
-
-				if (zr->v4l_grab_frame == NO_GRAB_ACTIVE)
-					wake_up_interruptible(&zr->v4l_capq);
-
-				/* Check if there is another grab queued */
-
-				if (zr->v4l_grab_frame == NO_GRAB_ACTIVE &&
-				    zr->v4l_pend_tail != zr->v4l_pend_head) {
-					int frame = zr->v4l_pend[zr->v4l_pend_tail & V4L_MASK_FRAME];
-					u32 reg;
-
-					zr->v4l_grab_frame = frame;
-
-					/* Set zr36057 video front end and enable video */
-
-					/* Buffer address */
-
-					reg = zr->v4l_buffers.buffer[frame].v4l.fbuffer_bus;
-					btwrite(reg, ZR36057_VDTR);
-					if (zr->v4l_settings.height > BUZ_MAX_HEIGHT / 2)
-						reg += zr->v4l_settings.bytesperline;
-					btwrite(reg, ZR36057_VDBR);
-
-					/* video stride, status, and frame grab register */
-					reg = 0;
-					if (zr->v4l_settings.height > BUZ_MAX_HEIGHT / 2)
-						reg += zr->v4l_settings.bytesperline;
-					reg = reg << ZR36057_VSSFGR_DispStride;
-					reg |= ZR36057_VSSFGR_VidOvf;
-					reg |= ZR36057_VSSFGR_SnapShot;
-					reg |= ZR36057_VSSFGR_FrameGrab;
-					btwrite(reg, ZR36057_VSSFGR);
-
-					btor(ZR36057_VDCR_VidEn,
-					     ZR36057_VDCR);
-				}
-			}
-
-			/* even if we don't grab, we do want to increment
-			 * the sequence counter to see lost frames */
-			zr->v4l_grab_seq++;
-		}
-#if (IRQ_MASK & ZR36057_ISR_CodRepIRQ)
-		if (astat & ZR36057_ISR_CodRepIRQ) {
-			zr->intr_counter_CodRepIRQ++;
-			IDEBUG(printk(KERN_DEBUG "%s: ZR36057_ISR_CodRepIRQ\n",
-				ZR_DEVNAME(zr)));
-			btand(~ZR36057_ICR_CodRepIRQ, ZR36057_ICR);
-		}
-#endif				/* (IRQ_MASK & ZR36057_ISR_CodRepIRQ) */
-
-#if (IRQ_MASK & ZR36057_ISR_JPEGRepIRQ)
-		if ((astat & ZR36057_ISR_JPEGRepIRQ) &&
-		    (zr->codec_mode == BUZ_MODE_MOTION_DECOMPRESS ||
-		     zr->codec_mode == BUZ_MODE_MOTION_COMPRESS)) {
-			if (zr36067_debug > 1 && (!zr->frame_num || zr->JPEG_error)) {
-				char sv[BUZ_NUM_STAT_COM + 1];
-				int i;
-
-				printk(KERN_INFO
-				       "%s: first frame ready: state=0x%08x odd_even=%d field_per_buff=%d delay=%d\n",
-				       ZR_DEVNAME(zr), stat,
-				       zr->jpg_settings.odd_even,
-				       zr->jpg_settings.field_per_buff,
-				       zr->JPEG_missed);
-
-				for (i = 0; i < BUZ_NUM_STAT_COM; i++)
-					sv[i] = le32_to_cpu(zr->stat_com[i]) & 1 ? '1' : '0';
-				sv[BUZ_NUM_STAT_COM] = 0;
-				printk(KERN_INFO
-				       "%s: stat_com=%s queue_state=%ld/%ld/%ld/%ld\n",
-				       ZR_DEVNAME(zr), sv,
-				       zr->jpg_que_tail,
-				       zr->jpg_dma_tail,
-				       zr->jpg_dma_head,
-				       zr->jpg_que_head);
-			} else {
-				/* Get statistics */
-				if (zr->JPEG_missed > zr->JPEG_max_missed)
-					zr->JPEG_max_missed = zr->JPEG_missed;
-				if (zr->JPEG_missed < zr->JPEG_min_missed)
-					zr->JPEG_min_missed = zr->JPEG_missed;
-			}
-
-			if (zr36067_debug > 2 && zr->frame_num < 6) {
-				int i;
-
-				printk(KERN_INFO "%s: seq=%ld stat_com:",
-				       ZR_DEVNAME(zr), zr->jpg_seq_num);
-				for (i = 0; i < 4; i++) {
-					printk(KERN_CONT " %08x",
-					       le32_to_cpu(zr->stat_com[i]));
-				}
-				printk(KERN_CONT "\n");
-			}
-			zr->frame_num++;
-			zr->JPEG_missed = 0;
-			zr->JPEG_error = 0;
-			zoran_reap_stat_com(zr);
-			zoran_feed_stat_com(zr);
-			wake_up_interruptible(&zr->jpg_capq);
-		}
-#endif				/* (IRQ_MASK & ZR36057_ISR_JPEGRepIRQ) */
-
-		/* DATERR, too many fields missed, error processing */
-		if ((astat & zr->card.jpeg_int) ||
-		    zr->JPEG_missed > 25 ||
-		    zr->JPEG_error == 1	||
-		    ((zr->codec_mode == BUZ_MODE_MOTION_DECOMPRESS) &&
-		     (zr->frame_num && (zr->JPEG_missed > zr->jpg_settings.field_per_buff)))) {
-			error_handler(zr, astat, stat);
-		}
-
-		count++;
-		if (count > 10) {
-			dprintk(2, KERN_WARNING "%s: irq loop %d\n",
-				ZR_DEVNAME(zr), count);
-			if (count > 20) {
-				btand(~ZR36057_ICR_IntPinEn, ZR36057_ICR);
-				dprintk(2,
-					KERN_ERR
-					"%s: IRQ lockup, cleared int mask\n",
-					ZR_DEVNAME(zr));
-				break;
-			}
-		}
-		zr->last_isr = stat;
-	}
-	spin_unlock_irqrestore(&zr->spinlock, flags);
-
-	return IRQ_HANDLED;
-}
-
-void
-zoran_set_pci_master (struct zoran *zr,
-		      int           set_master)
-{
-	if (set_master) {
-		pci_set_master(zr->pci_dev);
-	} else {
-		u16 command;
-
-		pci_read_config_word(zr->pci_dev, PCI_COMMAND, &command);
-		command &= ~PCI_COMMAND_MASTER;
-		pci_write_config_word(zr->pci_dev, PCI_COMMAND, command);
-	}
-}
-
-void
-zoran_init_hardware (struct zoran *zr)
-{
-	/* Enable bus-mastering */
-	zoran_set_pci_master(zr, 1);
-
-	/* Initialize the board */
-	if (zr->card.init) {
-		zr->card.init(zr);
-	}
-
-	decoder_call(zr, core, init, 0);
-	decoder_call(zr, video, s_std, zr->norm);
-	decoder_call(zr, video, s_routing,
-		zr->card.input[zr->input].muxsel, 0, 0);
-
-	encoder_call(zr, core, init, 0);
-	encoder_call(zr, video, s_std_output, zr->norm);
-	encoder_call(zr, video, s_routing, 0, 0, 0);
-
-	/* toggle JPEG codec sleep to sync PLL */
-	jpeg_codec_sleep(zr, 1);
-	jpeg_codec_sleep(zr, 0);
-
-	/*
-	 * set individual interrupt enables (without GIRQ1)
-	 * but don't global enable until zoran_open()
-	 */
-	zr36057_init_vfe(zr);
-
-	zr36057_enable_jpg(zr, BUZ_MODE_IDLE);
-
-	btwrite(IRQ_MASK, ZR36057_ISR);	// Clears interrupts
-}
-
-void
-zr36057_restart (struct zoran *zr)
-{
-	btwrite(0, ZR36057_SPGPPCR);
-	mdelay(1);
-	btor(ZR36057_SPGPPCR_SoftReset, ZR36057_SPGPPCR);
-	mdelay(1);
-
-	/* assert P_Reset */
-	btwrite(0, ZR36057_JPC);
-	/* set up GPIO direction - all output */
-	btwrite(ZR36057_SPGPPCR_SoftReset | 0, ZR36057_SPGPPCR);
-
-	/* set up GPIO pins and guest bus timing */
-	btwrite((0x81 << 24) | 0x8888, ZR36057_GPPGCR1);
-}
-
-/*
- * initialize video front end
- */
-
-static void
-zr36057_init_vfe (struct zoran *zr)
-{
-	u32 reg;
-
-	reg = btread(ZR36057_VFESPFR);
-	reg |= ZR36057_VFESPFR_LittleEndian;
-	reg &= ~ZR36057_VFESPFR_VCLKPol;
-	reg |= ZR36057_VFESPFR_ExtFl;
-	reg |= ZR36057_VFESPFR_TopField;
-	btwrite(reg, ZR36057_VFESPFR);
-	reg = btread(ZR36057_VDCR);
-	if (pci_pci_problems & PCIPCI_TRITON)
-		// || zr->revision < 1) // Revision 1 has also Triton support
-		reg &= ~ZR36057_VDCR_Triton;
-	else
-		reg |= ZR36057_VDCR_Triton;
-	btwrite(reg, ZR36057_VDCR);
-}
diff --git a/drivers/staging/media/zoran/zoran_device.h b/drivers/staging/media/zoran/zoran_device.h
deleted file mode 100644
index b3b8a03f2004..000000000000
--- a/drivers/staging/media/zoran/zoran_device.h
+++ /dev/null
@@ -1,81 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0+ */
-/*
- * Zoran zr36057/zr36067 PCI controller driver, for the
- * Pinnacle/Miro DC10/DC10+/DC30/DC30+, Iomega Buz, Linux
- * Media Labs LML33/LML33R10.
- *
- * This part handles card-specific data and detection
- *
- * Copyright (C) 2000 Serguei Miridonov <mirsev@cicese.mx>
- *
- * Currently maintained by:
- *   Ronald Bultje    <rbultje@ronald.bitfreak.net>
- *   Laurent Pinchart <laurent.pinchart@skynet.be>
- *   Mailinglist      <mjpeg-users@lists.sf.net>
- */
-#ifndef __ZORAN_DEVICE_H__
-#define __ZORAN_DEVICE_H__
-
-/* general purpose I/O */
-extern void GPIO(struct zoran *zr,
-		 int bit,
-		 unsigned int value);
-
-/* codec (or actually: guest bus) access */
-extern int post_office_wait(struct zoran *zr);
-extern int post_office_write(struct zoran *zr,
-			     unsigned int guest,
-			     unsigned int reg,
-			     unsigned int value);
-extern int post_office_read(struct zoran *zr,
-			    unsigned int guest,
-			    unsigned int reg);
-
-extern void detect_guest_activity(struct zoran *zr);
-
-extern void jpeg_codec_sleep(struct zoran *zr,
-			     int sleep);
-extern int jpeg_codec_reset(struct zoran *zr);
-
-/* zr360x7 access to raw capture */
-extern void zr36057_overlay(struct zoran *zr,
-			    int on);
-extern void write_overlay_mask(struct zoran_fh *fh,
-			       struct v4l2_clip *vp,
-			       int count);
-extern void zr36057_set_memgrab(struct zoran *zr,
-				int mode);
-extern int wait_grab_pending(struct zoran *zr);
-
-/* interrupts */
-extern void print_interrupts(struct zoran *zr);
-extern void clear_interrupt_counters(struct zoran *zr);
-extern irqreturn_t zoran_irq(int irq, void *dev_id);
-
-/* JPEG codec access */
-extern void jpeg_start(struct zoran *zr);
-extern void zr36057_enable_jpg(struct zoran *zr,
-			       enum zoran_codec_mode mode);
-extern void zoran_feed_stat_com(struct zoran *zr);
-
-/* general */
-extern void zoran_set_pci_master(struct zoran *zr,
-				 int set_master);
-extern void zoran_init_hardware(struct zoran *zr);
-extern void zr36057_restart(struct zoran *zr);
-
-extern const struct zoran_format zoran_formats[];
-
-extern int v4l_nbufs;
-extern int v4l_bufsize;
-extern int jpg_nbufs;
-extern int jpg_bufsize;
-extern int pass_through;
-
-/* i2c */
-#define decoder_call(zr, o, f, args...) \
-	v4l2_subdev_call(zr->decoder, o, f, ##args)
-#define encoder_call(zr, o, f, args...) \
-	v4l2_subdev_call(zr->encoder, o, f, ##args)
-
-#endif				/* __ZORAN_DEVICE_H__ */
diff --git a/drivers/staging/media/zoran/zoran_driver.c b/drivers/staging/media/zoran/zoran_driver.c
deleted file mode 100644
index 03bbfb723b43..000000000000
--- a/drivers/staging/media/zoran/zoran_driver.c
+++ /dev/null
@@ -1,2800 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0+
-/*
- * Zoran zr36057/zr36067 PCI controller driver, for the
- * Pinnacle/Miro DC10/DC10+/DC30/DC30+, Iomega Buz, Linux
- * Media Labs LML33/LML33R10.
- *
- * Copyright (C) 2000 Serguei Miridonov <mirsev@cicese.mx>
- *
- * Changes for BUZ by Wolfgang Scherr <scherr@net4you.net>
- *
- * Changes for DC10/DC30 by Laurent Pinchart <laurent.pinchart@skynet.be>
- *
- * Changes for LML33R10 by Maxim Yevtyushkin <max@linuxmedialabs.com>
- *
- * Changes for videodev2/v4l2 by Ronald Bultje <rbultje@ronald.bitfreak.net>
- *
- * Based on
- *
- * Miro DC10 driver
- * Copyright (C) 1999 Wolfgang Scherr <scherr@net4you.net>
- *
- * Iomega Buz driver version 1.0
- * Copyright (C) 1999 Rainer Johanni <Rainer@Johanni.de>
- *
- * buz.0.0.3
- * Copyright (C) 1998 Dave Perks <dperks@ibm.net>
- *
- * bttv - Bt848 frame grabber driver
- * Copyright (C) 1996,97,98 Ralph  Metzler (rjkm@thp.uni-koeln.de)
- *                        & Marcus Metzler (mocm@thp.uni-koeln.de)
- */
-#include <linux/init.h>
-#include <linux/module.h>
-#include <linux/delay.h>
-#include <linux/slab.h>
-#include <linux/pci.h>
-#include <linux/vmalloc.h>
-#include <linux/wait.h>
-
-#include <linux/interrupt.h>
-#include <linux/i2c.h>
-#include <linux/i2c-algo-bit.h>
-
-#include <linux/spinlock.h>
-
-#include <linux/videodev2.h>
-#include <media/v4l2-common.h>
-#include <media/v4l2-ioctl.h>
-#include <media/v4l2-event.h>
-#include "videocodec.h"
-
-#include <asm/byteorder.h>
-#include <asm/io.h>
-#include <linux/uaccess.h>
-#include <linux/proc_fs.h>
-
-#include <linux/mutex.h>
-#include "zoran.h"
-#include "zoran_device.h"
-#include "zoran_card.h"
-
-
-const struct zoran_format zoran_formats[] = {
-	{
-		.name = "15-bit RGB LE",
-		.fourcc = V4L2_PIX_FMT_RGB555,
-		.colorspace = V4L2_COLORSPACE_SRGB,
-		.depth = 15,
-		.flags = ZORAN_FORMAT_CAPTURE |
-			 ZORAN_FORMAT_OVERLAY,
-		.vfespfr = ZR36057_VFESPFR_RGB555|ZR36057_VFESPFR_ErrDif|
-			   ZR36057_VFESPFR_LittleEndian,
-	}, {
-		.name = "15-bit RGB BE",
-		.fourcc = V4L2_PIX_FMT_RGB555X,
-		.colorspace = V4L2_COLORSPACE_SRGB,
-		.depth = 15,
-		.flags = ZORAN_FORMAT_CAPTURE |
-			 ZORAN_FORMAT_OVERLAY,
-		.vfespfr = ZR36057_VFESPFR_RGB555|ZR36057_VFESPFR_ErrDif,
-	}, {
-		.name = "16-bit RGB LE",
-		.fourcc = V4L2_PIX_FMT_RGB565,
-		.colorspace = V4L2_COLORSPACE_SRGB,
-		.depth = 16,
-		.flags = ZORAN_FORMAT_CAPTURE |
-			 ZORAN_FORMAT_OVERLAY,
-		.vfespfr = ZR36057_VFESPFR_RGB565|ZR36057_VFESPFR_ErrDif|
-			   ZR36057_VFESPFR_LittleEndian,
-	}, {
-		.name = "16-bit RGB BE",
-		.fourcc = V4L2_PIX_FMT_RGB565X,
-		.colorspace = V4L2_COLORSPACE_SRGB,
-		.depth = 16,
-		.flags = ZORAN_FORMAT_CAPTURE |
-			 ZORAN_FORMAT_OVERLAY,
-		.vfespfr = ZR36057_VFESPFR_RGB565|ZR36057_VFESPFR_ErrDif,
-	}, {
-		.name = "24-bit RGB",
-		.fourcc = V4L2_PIX_FMT_BGR24,
-		.colorspace = V4L2_COLORSPACE_SRGB,
-		.depth = 24,
-		.flags = ZORAN_FORMAT_CAPTURE |
-			 ZORAN_FORMAT_OVERLAY,
-		.vfespfr = ZR36057_VFESPFR_RGB888|ZR36057_VFESPFR_Pack24,
-	}, {
-		.name = "32-bit RGB LE",
-		.fourcc = V4L2_PIX_FMT_BGR32,
-		.colorspace = V4L2_COLORSPACE_SRGB,
-		.depth = 32,
-		.flags = ZORAN_FORMAT_CAPTURE |
-			 ZORAN_FORMAT_OVERLAY,
-		.vfespfr = ZR36057_VFESPFR_RGB888|ZR36057_VFESPFR_LittleEndian,
-	}, {
-		.name = "32-bit RGB BE",
-		.fourcc = V4L2_PIX_FMT_RGB32,
-		.colorspace = V4L2_COLORSPACE_SRGB,
-		.depth = 32,
-		.flags = ZORAN_FORMAT_CAPTURE |
-			 ZORAN_FORMAT_OVERLAY,
-		.vfespfr = ZR36057_VFESPFR_RGB888,
-	}, {
-		.name = "4:2:2, packed, YUYV",
-		.fourcc = V4L2_PIX_FMT_YUYV,
-		.colorspace = V4L2_COLORSPACE_SMPTE170M,
-		.depth = 16,
-		.flags = ZORAN_FORMAT_CAPTURE |
-			 ZORAN_FORMAT_OVERLAY,
-		.vfespfr = ZR36057_VFESPFR_YUV422,
-	}, {
-		.name = "4:2:2, packed, UYVY",
-		.fourcc = V4L2_PIX_FMT_UYVY,
-		.colorspace = V4L2_COLORSPACE_SMPTE170M,
-		.depth = 16,
-		.flags = ZORAN_FORMAT_CAPTURE |
-			 ZORAN_FORMAT_OVERLAY,
-		.vfespfr = ZR36057_VFESPFR_YUV422|ZR36057_VFESPFR_LittleEndian,
-	}, {
-		.name = "Hardware-encoded Motion-JPEG",
-		.fourcc = V4L2_PIX_FMT_MJPEG,
-		.colorspace = V4L2_COLORSPACE_SMPTE170M,
-		.depth = 0,
-		.flags = ZORAN_FORMAT_CAPTURE |
-			 ZORAN_FORMAT_PLAYBACK |
-			 ZORAN_FORMAT_COMPRESSED,
-	}
-};
-#define NUM_FORMATS ARRAY_SIZE(zoran_formats)
-
-	/* small helper function for calculating buffersizes for v4l2
-	 * we calculate the nearest higher power-of-two, which
-	 * will be the recommended buffersize */
-static __u32
-zoran_v4l2_calc_bufsize (struct zoran_jpg_settings *settings)
-{
-	__u8 div = settings->VerDcm * settings->HorDcm * settings->TmpDcm;
-	__u32 num = (1024 * 512) / (div);
-	__u32 result = 2;
-
-	num--;
-	while (num) {
-		num >>= 1;
-		result <<= 1;
-	}
-
-	if (result > jpg_bufsize)
-		return jpg_bufsize;
-	if (result < 8192)
-		return 8192;
-	return result;
-}
-
-/* forward references */
-static void v4l_fbuffer_free(struct zoran_fh *fh);
-static void jpg_fbuffer_free(struct zoran_fh *fh);
-
-/* Set mapping mode */
-static void map_mode_raw(struct zoran_fh *fh)
-{
-	fh->map_mode = ZORAN_MAP_MODE_RAW;
-	fh->buffers.buffer_size = v4l_bufsize;
-	fh->buffers.num_buffers = v4l_nbufs;
-}
-static void map_mode_jpg(struct zoran_fh *fh, int play)
-{
-	fh->map_mode = play ? ZORAN_MAP_MODE_JPG_PLAY : ZORAN_MAP_MODE_JPG_REC;
-	fh->buffers.buffer_size = jpg_bufsize;
-	fh->buffers.num_buffers = jpg_nbufs;
-}
-static inline const char *mode_name(enum zoran_map_mode mode)
-{
-	return mode == ZORAN_MAP_MODE_RAW ? "V4L" : "JPG";
-}
-
-/*
- *   Allocate the V4L grab buffers
- *
- *   These have to be pysically contiguous.
- */
-
-static int v4l_fbuffer_alloc(struct zoran_fh *fh)
-{
-	struct zoran *zr = fh->zr;
-	int i, off;
-	unsigned char *mem;
-
-	for (i = 0; i < fh->buffers.num_buffers; i++) {
-		if (fh->buffers.buffer[i].v4l.fbuffer)
-			dprintk(2,
-				KERN_WARNING
-				"%s: %s - buffer %d already allocated!?\n",
-				ZR_DEVNAME(zr), __func__, i);
-
-		//udelay(20);
-		mem = kmalloc(fh->buffers.buffer_size,
-			      GFP_KERNEL | __GFP_NOWARN);
-		if (!mem) {
-			v4l_fbuffer_free(fh);
-			return -ENOBUFS;
-		}
-		fh->buffers.buffer[i].v4l.fbuffer = mem;
-		fh->buffers.buffer[i].v4l.fbuffer_phys = virt_to_phys(mem);
-		fh->buffers.buffer[i].v4l.fbuffer_bus = virt_to_bus(mem);
-		for (off = 0; off < fh->buffers.buffer_size;
-		     off += PAGE_SIZE)
-			SetPageReserved(virt_to_page(mem + off));
-		dprintk(4,
-			KERN_INFO
-			"%s: %s - V4L frame %d mem %p (bus: 0x%llx)\n",
-			ZR_DEVNAME(zr), __func__, i, mem,
-			(unsigned long long)virt_to_bus(mem));
-	}
-
-	fh->buffers.allocated = 1;
-
-	return 0;
-}
-
-/* free the V4L grab buffers */
-static void v4l_fbuffer_free(struct zoran_fh *fh)
-{
-	struct zoran *zr = fh->zr;
-	int i, off;
-	unsigned char *mem;
-
-	dprintk(4, KERN_INFO "%s: %s\n", ZR_DEVNAME(zr), __func__);
-
-	for (i = 0; i < fh->buffers.num_buffers; i++) {
-		if (!fh->buffers.buffer[i].v4l.fbuffer)
-			continue;
-
-		mem = fh->buffers.buffer[i].v4l.fbuffer;
-		for (off = 0; off < fh->buffers.buffer_size;
-		     off += PAGE_SIZE)
-			ClearPageReserved(virt_to_page(mem + off));
-		kfree(fh->buffers.buffer[i].v4l.fbuffer);
-		fh->buffers.buffer[i].v4l.fbuffer = NULL;
-	}
-
-	fh->buffers.allocated = 0;
-}
-
-/*
- *   Allocate the MJPEG grab buffers.
- *
- *   If a Natoma chipset is present and this is a revision 1 zr36057,
- *   each MJPEG buffer needs to be physically contiguous.
- *   (RJ: This statement is from Dave Perks' original driver,
- *   I could never check it because I have a zr36067)
- *
- *   RJ: The contents grab buffers needs never be accessed in the driver.
- *       Therefore there is no need to allocate them with vmalloc in order
- *       to get a contiguous virtual memory space.
- *       I don't understand why many other drivers first allocate them with
- *       vmalloc (which uses internally also get_zeroed_page, but delivers you
- *       virtual addresses) and then again have to make a lot of efforts
- *       to get the physical address.
- *
- *   Ben Capper:
- *       On big-endian architectures (such as ppc) some extra steps
- *       are needed. When reading and writing to the stat_com array
- *       and fragment buffers, the device expects to see little-
- *       endian values. The use of cpu_to_le32() and le32_to_cpu()
- *       in this function (and one or two others in zoran_device.c)
- *       ensure that these values are always stored in little-endian
- *       form, regardless of architecture. The zr36057 does Very Bad
- *       Things on big endian architectures if the stat_com array
- *       and fragment buffers are not little-endian.
- */
-
-static int jpg_fbuffer_alloc(struct zoran_fh *fh)
-{
-	struct zoran *zr = fh->zr;
-	int i, j, off;
-	u8 *mem;
-
-	for (i = 0; i < fh->buffers.num_buffers; i++) {
-		if (fh->buffers.buffer[i].jpg.frag_tab)
-			dprintk(2,
-				KERN_WARNING
-				"%s: %s - buffer %d already allocated!?\n",
-				ZR_DEVNAME(zr), __func__, i);
-
-		/* Allocate fragment table for this buffer */
-
-		mem = (void *)get_zeroed_page(GFP_KERNEL);
-		if (!mem) {
-			dprintk(1,
-				KERN_ERR
-				"%s: %s - get_zeroed_page (frag_tab) failed for buffer %d\n",
-				ZR_DEVNAME(zr), __func__, i);
-			jpg_fbuffer_free(fh);
-			return -ENOBUFS;
-		}
-		fh->buffers.buffer[i].jpg.frag_tab = (__le32 *)mem;
-		fh->buffers.buffer[i].jpg.frag_tab_bus = virt_to_bus(mem);
-
-		if (fh->buffers.need_contiguous) {
-			mem = kmalloc(fh->buffers.buffer_size, GFP_KERNEL);
-			if (!mem) {
-				dprintk(1,
-					KERN_ERR
-					"%s: %s - kmalloc failed for buffer %d\n",
-					ZR_DEVNAME(zr), __func__, i);
-				jpg_fbuffer_free(fh);
-				return -ENOBUFS;
-			}
-			fh->buffers.buffer[i].jpg.frag_tab[0] =
-				cpu_to_le32(virt_to_bus(mem));
-			fh->buffers.buffer[i].jpg.frag_tab[1] =
-				cpu_to_le32((fh->buffers.buffer_size >> 1) | 1);
-			for (off = 0; off < fh->buffers.buffer_size; off += PAGE_SIZE)
-				SetPageReserved(virt_to_page(mem + off));
-		} else {
-			/* jpg_bufsize is already page aligned */
-			for (j = 0; j < fh->buffers.buffer_size / PAGE_SIZE; j++) {
-				mem = (void *)get_zeroed_page(GFP_KERNEL);
-				if (mem == NULL) {
-					dprintk(1,
-						KERN_ERR
-						"%s: %s - get_zeroed_page failed for buffer %d\n",
-						ZR_DEVNAME(zr), __func__, i);
-					jpg_fbuffer_free(fh);
-					return -ENOBUFS;
-				}
-
-				fh->buffers.buffer[i].jpg.frag_tab[2 * j] =
-					cpu_to_le32(virt_to_bus(mem));
-				fh->buffers.buffer[i].jpg.frag_tab[2 * j + 1] =
-					cpu_to_le32((PAGE_SIZE >> 2) << 1);
-				SetPageReserved(virt_to_page(mem));
-			}
-
-			fh->buffers.buffer[i].jpg.frag_tab[2 * j - 1] |= cpu_to_le32(1);
-		}
-	}
-
-	dprintk(4,
-		KERN_DEBUG "%s: %s - %d KB allocated\n",
-		ZR_DEVNAME(zr), __func__,
-		(fh->buffers.num_buffers * fh->buffers.buffer_size) >> 10);
-
-	fh->buffers.allocated = 1;
-
-	return 0;
-}
-
-/* free the MJPEG grab buffers */
-static void jpg_fbuffer_free(struct zoran_fh *fh)
-{
-	struct zoran *zr = fh->zr;
-	int i, j, off;
-	unsigned char *mem;
-	__le32 frag_tab;
-	struct zoran_buffer *buffer;
-
-	dprintk(4, KERN_DEBUG "%s: %s\n", ZR_DEVNAME(zr), __func__);
-
-	for (i = 0, buffer = &fh->buffers.buffer[0];
-	     i < fh->buffers.num_buffers; i++, buffer++) {
-		if (!buffer->jpg.frag_tab)
-			continue;
-
-		if (fh->buffers.need_contiguous) {
-			frag_tab = buffer->jpg.frag_tab[0];
-
-			if (frag_tab) {
-				mem = bus_to_virt(le32_to_cpu(frag_tab));
-				for (off = 0; off < fh->buffers.buffer_size; off += PAGE_SIZE)
-					ClearPageReserved(virt_to_page(mem + off));
-				kfree(mem);
-				buffer->jpg.frag_tab[0] = 0;
-				buffer->jpg.frag_tab[1] = 0;
-			}
-		} else {
-			for (j = 0; j < fh->buffers.buffer_size / PAGE_SIZE; j++) {
-				frag_tab = buffer->jpg.frag_tab[2 * j];
-
-				if (!frag_tab)
-					break;
-				ClearPageReserved(virt_to_page(bus_to_virt(le32_to_cpu(frag_tab))));
-				free_page((unsigned long)bus_to_virt(le32_to_cpu(frag_tab)));
-				buffer->jpg.frag_tab[2 * j] = 0;
-				buffer->jpg.frag_tab[2 * j + 1] = 0;
-			}
-		}
-
-		free_page((unsigned long)buffer->jpg.frag_tab);
-		buffer->jpg.frag_tab = NULL;
-	}
-
-	fh->buffers.allocated = 0;
-}
-
-/*
- *   V4L Buffer grabbing
- */
-
-static int
-zoran_v4l_set_format (struct zoran_fh           *fh,
-		      int                        width,
-		      int                        height,
-		      const struct zoran_format *format)
-{
-	struct zoran *zr = fh->zr;
-	int bpp;
-
-	/* Check size and format of the grab wanted */
-
-	if (height < BUZ_MIN_HEIGHT || width < BUZ_MIN_WIDTH ||
-	    height > BUZ_MAX_HEIGHT || width > BUZ_MAX_WIDTH) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - wrong frame size (%dx%d)\n",
-			ZR_DEVNAME(zr), __func__, width, height);
-		return -EINVAL;
-	}
-
-	bpp = (format->depth + 7) / 8;
-
-	/* Check against available buffer size */
-	if (height * width * bpp > fh->buffers.buffer_size) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - video buffer size (%d kB) is too small\n",
-			ZR_DEVNAME(zr), __func__, fh->buffers.buffer_size >> 10);
-		return -EINVAL;
-	}
-
-	/* The video front end needs 4-byte alinged line sizes */
-
-	if ((bpp == 2 && (width & 1)) || (bpp == 3 && (width & 3))) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - wrong frame alignment\n",
-			ZR_DEVNAME(zr), __func__);
-		return -EINVAL;
-	}
-
-	fh->v4l_settings.width = width;
-	fh->v4l_settings.height = height;
-	fh->v4l_settings.format = format;
-	fh->v4l_settings.bytesperline = bpp * fh->v4l_settings.width;
-
-	return 0;
-}
-
-static int zoran_v4l_queue_frame(struct zoran_fh *fh, int num)
-{
-	struct zoran *zr = fh->zr;
-	unsigned long flags;
-	int res = 0;
-
-	if (!fh->buffers.allocated) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - buffers not yet allocated\n",
-			ZR_DEVNAME(zr), __func__);
-		res = -ENOMEM;
-	}
-
-	/* No grabbing outside the buffer range! */
-	if (num >= fh->buffers.num_buffers || num < 0) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - buffer %d is out of range\n",
-			ZR_DEVNAME(zr), __func__, num);
-		res = -EINVAL;
-	}
-
-	spin_lock_irqsave(&zr->spinlock, flags);
-
-	if (fh->buffers.active == ZORAN_FREE) {
-		if (zr->v4l_buffers.active == ZORAN_FREE) {
-			zr->v4l_buffers = fh->buffers;
-			fh->buffers.active = ZORAN_ACTIVE;
-		} else {
-			dprintk(1,
-				KERN_ERR
-				"%s: %s - another session is already capturing\n",
-				ZR_DEVNAME(zr), __func__);
-			res = -EBUSY;
-		}
-	}
-
-	/* make sure a grab isn't going on currently with this buffer */
-	if (!res) {
-		switch (zr->v4l_buffers.buffer[num].state) {
-		default:
-		case BUZ_STATE_PEND:
-			if (zr->v4l_buffers.active == ZORAN_FREE) {
-				fh->buffers.active = ZORAN_FREE;
-				zr->v4l_buffers.allocated = 0;
-			}
-			res = -EBUSY;	/* what are you doing? */
-			break;
-		case BUZ_STATE_DONE:
-			dprintk(2,
-				KERN_WARNING
-				"%s: %s - queueing buffer %d in state DONE!?\n",
-				ZR_DEVNAME(zr), __func__, num);
-			/* fall through */
-		case BUZ_STATE_USER:
-			/* since there is at least one unused buffer there's room for at least
-			 * one more pend[] entry */
-			zr->v4l_pend[zr->v4l_pend_head++ & V4L_MASK_FRAME] = num;
-			zr->v4l_buffers.buffer[num].state = BUZ_STATE_PEND;
-			zr->v4l_buffers.buffer[num].bs.length =
-			    fh->v4l_settings.bytesperline *
-			    zr->v4l_settings.height;
-			fh->buffers.buffer[num] = zr->v4l_buffers.buffer[num];
-			break;
-		}
-	}
-
-	spin_unlock_irqrestore(&zr->spinlock, flags);
-
-	if (!res && zr->v4l_buffers.active == ZORAN_FREE)
-		zr->v4l_buffers.active = fh->buffers.active;
-
-	return res;
-}
-
-/*
- * Sync on a V4L buffer
- */
-
-static int v4l_sync(struct zoran_fh *fh, int frame)
-{
-	struct zoran *zr = fh->zr;
-	unsigned long flags;
-
-	if (fh->buffers.active == ZORAN_FREE) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - no grab active for this session\n",
-			ZR_DEVNAME(zr), __func__);
-		return -EINVAL;
-	}
-
-	/* check passed-in frame number */
-	if (frame >= fh->buffers.num_buffers || frame < 0) {
-		dprintk(1,
-			KERN_ERR "%s: %s - frame %d is invalid\n",
-			ZR_DEVNAME(zr), __func__, frame);
-		return -EINVAL;
-	}
-
-	/* Check if is buffer was queued at all */
-	if (zr->v4l_buffers.buffer[frame].state == BUZ_STATE_USER) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - attempt to sync on a buffer which was not queued?\n",
-			ZR_DEVNAME(zr), __func__);
-		return -EPROTO;
-	}
-
-	mutex_unlock(&zr->lock);
-	/* wait on this buffer to get ready */
-	if (!wait_event_interruptible_timeout(zr->v4l_capq,
-		(zr->v4l_buffers.buffer[frame].state != BUZ_STATE_PEND), 10*HZ)) {
-		mutex_lock(&zr->lock);
-		return -ETIME;
-	}
-	mutex_lock(&zr->lock);
-	if (signal_pending(current))
-		return -ERESTARTSYS;
-
-	/* buffer should now be in BUZ_STATE_DONE */
-	if (zr->v4l_buffers.buffer[frame].state != BUZ_STATE_DONE)
-		dprintk(2,
-			KERN_ERR "%s: %s - internal state error\n",
-			ZR_DEVNAME(zr), __func__);
-
-	zr->v4l_buffers.buffer[frame].state = BUZ_STATE_USER;
-	fh->buffers.buffer[frame] = zr->v4l_buffers.buffer[frame];
-
-	spin_lock_irqsave(&zr->spinlock, flags);
-
-	/* Check if streaming capture has finished */
-	if (zr->v4l_pend_tail == zr->v4l_pend_head) {
-		zr36057_set_memgrab(zr, 0);
-		if (zr->v4l_buffers.active == ZORAN_ACTIVE) {
-			fh->buffers.active = zr->v4l_buffers.active = ZORAN_FREE;
-			zr->v4l_buffers.allocated = 0;
-		}
-	}
-
-	spin_unlock_irqrestore(&zr->spinlock, flags);
-
-	return 0;
-}
-
-/*
- *   Queue a MJPEG buffer for capture/playback
- */
-
-static int zoran_jpg_queue_frame(struct zoran_fh *fh, int num,
-				 enum zoran_codec_mode mode)
-{
-	struct zoran *zr = fh->zr;
-	unsigned long flags;
-	int res = 0;
-
-	/* Check if buffers are allocated */
-	if (!fh->buffers.allocated) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - buffers not yet allocated\n",
-			ZR_DEVNAME(zr), __func__);
-		return -ENOMEM;
-	}
-
-	/* No grabbing outside the buffer range! */
-	if (num >= fh->buffers.num_buffers || num < 0) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - buffer %d out of range\n",
-			ZR_DEVNAME(zr), __func__, num);
-		return -EINVAL;
-	}
-
-	/* what is the codec mode right now? */
-	if (zr->codec_mode == BUZ_MODE_IDLE) {
-		zr->jpg_settings = fh->jpg_settings;
-	} else if (zr->codec_mode != mode) {
-		/* wrong codec mode active - invalid */
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - codec in wrong mode\n",
-			ZR_DEVNAME(zr), __func__);
-		return -EINVAL;
-	}
-
-	if (fh->buffers.active == ZORAN_FREE) {
-		if (zr->jpg_buffers.active == ZORAN_FREE) {
-			zr->jpg_buffers = fh->buffers;
-			fh->buffers.active = ZORAN_ACTIVE;
-		} else {
-			dprintk(1,
-				KERN_ERR
-				"%s: %s - another session is already capturing\n",
-				ZR_DEVNAME(zr), __func__);
-			res = -EBUSY;
-		}
-	}
-
-	if (!res && zr->codec_mode == BUZ_MODE_IDLE) {
-		/* Ok load up the jpeg codec */
-		zr36057_enable_jpg(zr, mode);
-	}
-
-	spin_lock_irqsave(&zr->spinlock, flags);
-
-	if (!res) {
-		switch (zr->jpg_buffers.buffer[num].state) {
-		case BUZ_STATE_DONE:
-			dprintk(2,
-				KERN_WARNING
-				"%s: %s - queuing frame in BUZ_STATE_DONE state!?\n",
-				ZR_DEVNAME(zr), __func__);
-			/* fall through */
-		case BUZ_STATE_USER:
-			/* since there is at least one unused buffer there's room for at
-			 *least one more pend[] entry */
-			zr->jpg_pend[zr->jpg_que_head++ & BUZ_MASK_FRAME] = num;
-			zr->jpg_buffers.buffer[num].state = BUZ_STATE_PEND;
-			fh->buffers.buffer[num] = zr->jpg_buffers.buffer[num];
-			zoran_feed_stat_com(zr);
-			break;
-		default:
-		case BUZ_STATE_DMA:
-		case BUZ_STATE_PEND:
-			if (zr->jpg_buffers.active == ZORAN_FREE) {
-				fh->buffers.active = ZORAN_FREE;
-				zr->jpg_buffers.allocated = 0;
-			}
-			res = -EBUSY;	/* what are you doing? */
-			break;
-		}
-	}
-
-	spin_unlock_irqrestore(&zr->spinlock, flags);
-
-	if (!res && zr->jpg_buffers.active == ZORAN_FREE)
-		zr->jpg_buffers.active = fh->buffers.active;
-
-	return res;
-}
-
-static int jpg_qbuf(struct zoran_fh *fh, int frame, enum zoran_codec_mode mode)
-{
-	struct zoran *zr = fh->zr;
-	int res = 0;
-
-	/* Does the user want to stop streaming? */
-	if (frame < 0) {
-		if (zr->codec_mode == mode) {
-			if (fh->buffers.active == ZORAN_FREE) {
-				dprintk(1,
-					KERN_ERR
-					"%s: %s(-1) - session not active\n",
-					ZR_DEVNAME(zr), __func__);
-				return -EINVAL;
-			}
-			fh->buffers.active = zr->jpg_buffers.active = ZORAN_FREE;
-			zr->jpg_buffers.allocated = 0;
-			zr36057_enable_jpg(zr, BUZ_MODE_IDLE);
-			return 0;
-		} else {
-			dprintk(1,
-				KERN_ERR
-				"%s: %s - stop streaming but not in streaming mode\n",
-				ZR_DEVNAME(zr), __func__);
-			return -EINVAL;
-		}
-	}
-
-	if ((res = zoran_jpg_queue_frame(fh, frame, mode)))
-		return res;
-
-	/* Start the jpeg codec when the first frame is queued  */
-	if (!res && zr->jpg_que_head == 1)
-		jpeg_start(zr);
-
-	return res;
-}
-
-/*
- *   Sync on a MJPEG buffer
- */
-
-static int jpg_sync(struct zoran_fh *fh, struct zoran_sync *bs)
-{
-	struct zoran *zr = fh->zr;
-	unsigned long flags;
-	int frame;
-
-	if (fh->buffers.active == ZORAN_FREE) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - capture is not currently active\n",
-			ZR_DEVNAME(zr), __func__);
-		return -EINVAL;
-	}
-	if (zr->codec_mode != BUZ_MODE_MOTION_DECOMPRESS &&
-	    zr->codec_mode != BUZ_MODE_MOTION_COMPRESS) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - codec not in streaming mode\n",
-			ZR_DEVNAME(zr), __func__);
-		return -EINVAL;
-	}
-	mutex_unlock(&zr->lock);
-	if (!wait_event_interruptible_timeout(zr->jpg_capq,
-			(zr->jpg_que_tail != zr->jpg_dma_tail ||
-			 zr->jpg_dma_tail == zr->jpg_dma_head),
-			10*HZ)) {
-		int isr;
-
-		btand(~ZR36057_JMC_Go_en, ZR36057_JMC);
-		udelay(1);
-		zr->codec->control(zr->codec, CODEC_G_STATUS,
-					   sizeof(isr), &isr);
-		mutex_lock(&zr->lock);
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - timeout: codec isr=0x%02x\n",
-			ZR_DEVNAME(zr), __func__, isr);
-
-		return -ETIME;
-
-	}
-	mutex_lock(&zr->lock);
-	if (signal_pending(current))
-		return -ERESTARTSYS;
-
-	spin_lock_irqsave(&zr->spinlock, flags);
-
-	if (zr->jpg_dma_tail != zr->jpg_dma_head)
-		frame = zr->jpg_pend[zr->jpg_que_tail++ & BUZ_MASK_FRAME];
-	else
-		frame = zr->jpg_pend[zr->jpg_que_tail & BUZ_MASK_FRAME];
-
-	/* buffer should now be in BUZ_STATE_DONE */
-	if (zr->jpg_buffers.buffer[frame].state != BUZ_STATE_DONE)
-		dprintk(2,
-			KERN_ERR "%s: %s - internal state error\n",
-			ZR_DEVNAME(zr), __func__);
-
-	*bs = zr->jpg_buffers.buffer[frame].bs;
-	bs->frame = frame;
-	zr->jpg_buffers.buffer[frame].state = BUZ_STATE_USER;
-	fh->buffers.buffer[frame] = zr->jpg_buffers.buffer[frame];
-
-	spin_unlock_irqrestore(&zr->spinlock, flags);
-
-	return 0;
-}
-
-static void zoran_open_init_session(struct zoran_fh *fh)
-{
-	int i;
-	struct zoran *zr = fh->zr;
-
-	/* Per default, map the V4L Buffers */
-	map_mode_raw(fh);
-
-	/* take over the card's current settings */
-	fh->overlay_settings = zr->overlay_settings;
-	fh->overlay_settings.is_set = 0;
-	fh->overlay_settings.format = zr->overlay_settings.format;
-	fh->overlay_active = ZORAN_FREE;
-
-	/* v4l settings */
-	fh->v4l_settings = zr->v4l_settings;
-	/* jpg settings */
-	fh->jpg_settings = zr->jpg_settings;
-
-	/* buffers */
-	memset(&fh->buffers, 0, sizeof(fh->buffers));
-	for (i = 0; i < MAX_FRAME; i++) {
-		fh->buffers.buffer[i].state = BUZ_STATE_USER;	/* nothing going on */
-		fh->buffers.buffer[i].bs.frame = i;
-	}
-	fh->buffers.allocated = 0;
-	fh->buffers.active = ZORAN_FREE;
-}
-
-static void zoran_close_end_session(struct zoran_fh *fh)
-{
-	struct zoran *zr = fh->zr;
-
-	/* overlay */
-	if (fh->overlay_active != ZORAN_FREE) {
-		fh->overlay_active = zr->overlay_active = ZORAN_FREE;
-		zr->v4l_overlay_active = 0;
-		if (!zr->v4l_memgrab_active)
-			zr36057_overlay(zr, 0);
-		zr->overlay_mask = NULL;
-	}
-
-	if (fh->map_mode == ZORAN_MAP_MODE_RAW) {
-		/* v4l capture */
-		if (fh->buffers.active != ZORAN_FREE) {
-			unsigned long flags;
-
-			spin_lock_irqsave(&zr->spinlock, flags);
-			zr36057_set_memgrab(zr, 0);
-			zr->v4l_buffers.allocated = 0;
-			zr->v4l_buffers.active = fh->buffers.active = ZORAN_FREE;
-			spin_unlock_irqrestore(&zr->spinlock, flags);
-		}
-
-		/* v4l buffers */
-		if (fh->buffers.allocated)
-			v4l_fbuffer_free(fh);
-	} else {
-		/* jpg capture */
-		if (fh->buffers.active != ZORAN_FREE) {
-			zr36057_enable_jpg(zr, BUZ_MODE_IDLE);
-			zr->jpg_buffers.allocated = 0;
-			zr->jpg_buffers.active = fh->buffers.active = ZORAN_FREE;
-		}
-
-		/* jpg buffers */
-		if (fh->buffers.allocated)
-			jpg_fbuffer_free(fh);
-	}
-}
-
-/*
- *   Open a zoran card. Right now the flags stuff is just playing
- */
-
-static int zoran_open(struct file *file)
-{
-	struct zoran *zr = video_drvdata(file);
-	struct zoran_fh *fh;
-	int res, first_open = 0;
-
-	dprintk(2, KERN_INFO "%s: %s(%s, pid=[%d]), users(-)=%d\n",
-		ZR_DEVNAME(zr), __func__, current->comm, task_pid_nr(current), zr->user + 1);
-
-	mutex_lock(&zr->lock);
-
-	if (zr->user >= 2048) {
-		dprintk(1, KERN_ERR "%s: too many users (%d) on device\n",
-			ZR_DEVNAME(zr), zr->user);
-		res = -EBUSY;
-		goto fail_unlock;
-	}
-
-	/* now, create the open()-specific file_ops struct */
-	fh = kzalloc(sizeof(struct zoran_fh), GFP_KERNEL);
-	if (!fh) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - allocation of zoran_fh failed\n",
-			ZR_DEVNAME(zr), __func__);
-		res = -ENOMEM;
-		goto fail_unlock;
-	}
-	v4l2_fh_init(&fh->fh, video_devdata(file));
-
-	/* used to be BUZ_MAX_WIDTH/HEIGHT, but that gives overflows
-	 * on norm-change! */
-	fh->overlay_mask =
-	    kmalloc(array3_size((768 + 31) / 32, 576, 4), GFP_KERNEL);
-	if (!fh->overlay_mask) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - allocation of overlay_mask failed\n",
-			ZR_DEVNAME(zr), __func__);
-		res = -ENOMEM;
-		goto fail_fh;
-	}
-
-	if (zr->user++ == 0)
-		first_open = 1;
-
-	/* default setup - TODO: look at flags */
-	if (first_open) {	/* First device open */
-		zr36057_restart(zr);
-		zoran_open_init_params(zr);
-		zoran_init_hardware(zr);
-
-		btor(ZR36057_ICR_IntPinEn, ZR36057_ICR);
-	}
-
-	/* set file_ops stuff */
-	file->private_data = fh;
-	fh->zr = zr;
-	zoran_open_init_session(fh);
-	v4l2_fh_add(&fh->fh);
-	mutex_unlock(&zr->lock);
-
-	return 0;
-
-fail_fh:
-	v4l2_fh_exit(&fh->fh);
-	kfree(fh);
-fail_unlock:
-	mutex_unlock(&zr->lock);
-
-	dprintk(2, KERN_INFO "%s: open failed (%d), users(-)=%d\n",
-		ZR_DEVNAME(zr), res, zr->user);
-
-	return res;
-}
-
-static int
-zoran_close(struct file  *file)
-{
-	struct zoran_fh *fh = file->private_data;
-	struct zoran *zr = fh->zr;
-
-	dprintk(2, KERN_INFO "%s: %s(%s, pid=[%d]), users(+)=%d\n",
-		ZR_DEVNAME(zr), __func__, current->comm, task_pid_nr(current), zr->user - 1);
-
-	/* kernel locks (fs/device.c), so don't do that ourselves
-	 * (prevents deadlocks) */
-	mutex_lock(&zr->lock);
-
-	zoran_close_end_session(fh);
-
-	if (zr->user-- == 1) {	/* Last process */
-		/* Clean up JPEG process */
-		wake_up_interruptible(&zr->jpg_capq);
-		zr36057_enable_jpg(zr, BUZ_MODE_IDLE);
-		zr->jpg_buffers.allocated = 0;
-		zr->jpg_buffers.active = ZORAN_FREE;
-
-		/* disable interrupts */
-		btand(~ZR36057_ICR_IntPinEn, ZR36057_ICR);
-
-		if (zr36067_debug > 1)
-			print_interrupts(zr);
-
-		/* Overlay off */
-		zr->v4l_overlay_active = 0;
-		zr36057_overlay(zr, 0);
-		zr->overlay_mask = NULL;
-
-		/* capture off */
-		wake_up_interruptible(&zr->v4l_capq);
-		zr36057_set_memgrab(zr, 0);
-		zr->v4l_buffers.allocated = 0;
-		zr->v4l_buffers.active = ZORAN_FREE;
-		zoran_set_pci_master(zr, 0);
-
-		if (!pass_through) {	/* Switch to color bar */
-			decoder_call(zr, video, s_stream, 0);
-			encoder_call(zr, video, s_routing, 2, 0, 0);
-		}
-	}
-	mutex_unlock(&zr->lock);
-
-	v4l2_fh_del(&fh->fh);
-	v4l2_fh_exit(&fh->fh);
-	kfree(fh->overlay_mask);
-	kfree(fh);
-
-	dprintk(4, KERN_INFO "%s: %s done\n", ZR_DEVNAME(zr), __func__);
-
-	return 0;
-}
-
-static int setup_fbuffer(struct zoran_fh *fh,
-	       void                      *base,
-	       const struct zoran_format *fmt,
-	       int                        width,
-	       int                        height,
-	       int                        bytesperline)
-{
-	struct zoran *zr = fh->zr;
-
-	/* (Ronald) v4l/v4l2 guidelines */
-	if (!capable(CAP_SYS_ADMIN) && !capable(CAP_SYS_RAWIO))
-		return -EPERM;
-
-	/* Don't allow frame buffer overlay if PCI or AGP is buggy, or on
-	   ALi Magik (that needs very low latency while the card needs a
-	   higher value always) */
-
-	if (pci_pci_problems & (PCIPCI_FAIL | PCIAGP_FAIL | PCIPCI_ALIMAGIK))
-		return -ENXIO;
-
-	/* we need a bytesperline value, even if not given */
-	if (!bytesperline)
-		bytesperline = width * ((fmt->depth + 7) & ~7) / 8;
-
-#if 0
-	if (zr->overlay_active) {
-		/* dzjee... stupid users... don't even bother to turn off
-		 * overlay before changing the memory location...
-		 * normally, we would return errors here. However, one of
-		 * the tools that does this is... xawtv! and since xawtv
-		 * is used by +/- 99% of the users, we'd rather be user-
-		 * friendly and silently do as if nothing went wrong */
-		dprintk(3,
-			KERN_ERR
-			"%s: %s - forced overlay turnoff because framebuffer changed\n",
-			ZR_DEVNAME(zr), __func__);
-		zr36057_overlay(zr, 0);
-	}
-#endif
-
-	if (!(fmt->flags & ZORAN_FORMAT_OVERLAY)) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - no valid overlay format given\n",
-			ZR_DEVNAME(zr), __func__);
-		return -EINVAL;
-	}
-	if (height <= 0 || width <= 0 || bytesperline <= 0) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - invalid height/width/bpl value (%d|%d|%d)\n",
-			ZR_DEVNAME(zr), __func__, width, height, bytesperline);
-		return -EINVAL;
-	}
-	if (bytesperline & 3) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - bytesperline (%d) must be 4-byte aligned\n",
-			ZR_DEVNAME(zr), __func__, bytesperline);
-		return -EINVAL;
-	}
-
-	zr->vbuf_base = (void *) ((unsigned long) base & ~3);
-	zr->vbuf_height = height;
-	zr->vbuf_width = width;
-	zr->vbuf_depth = fmt->depth;
-	zr->overlay_settings.format = fmt;
-	zr->vbuf_bytesperline = bytesperline;
-
-	/* The user should set new window parameters */
-	zr->overlay_settings.is_set = 0;
-
-	return 0;
-}
-
-
-static int setup_window(struct zoran_fh *fh,
-			int x,
-			int y,
-			int width,
-			int height,
-			struct v4l2_clip __user *clips,
-			unsigned int clipcount,
-			void __user *bitmap)
-{
-	struct zoran *zr = fh->zr;
-	struct v4l2_clip *vcp = NULL;
-	int on, end;
-
-
-	if (!zr->vbuf_base) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - frame buffer has to be set first\n",
-			ZR_DEVNAME(zr), __func__);
-		return -EINVAL;
-	}
-
-	if (!fh->overlay_settings.format) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - no overlay format set\n",
-			ZR_DEVNAME(zr), __func__);
-		return -EINVAL;
-	}
-
-	if (clipcount > 2048) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - invalid clipcount\n",
-			 ZR_DEVNAME(zr), __func__);
-		return -EINVAL;
-	}
-
-	/*
-	 * The video front end needs 4-byte alinged line sizes, we correct that
-	 * silently here if necessary
-	 */
-	if (zr->vbuf_depth == 15 || zr->vbuf_depth == 16) {
-		end = (x + width) & ~1;	/* round down */
-		x = (x + 1) & ~1;	/* round up */
-		width = end - x;
-	}
-
-	if (zr->vbuf_depth == 24) {
-		end = (x + width) & ~3;	/* round down */
-		x = (x + 3) & ~3;	/* round up */
-		width = end - x;
-	}
-
-	if (width > BUZ_MAX_WIDTH)
-		width = BUZ_MAX_WIDTH;
-	if (height > BUZ_MAX_HEIGHT)
-		height = BUZ_MAX_HEIGHT;
-
-	/* Check for invalid parameters */
-	if (width < BUZ_MIN_WIDTH || height < BUZ_MIN_HEIGHT ||
-	    width > BUZ_MAX_WIDTH || height > BUZ_MAX_HEIGHT) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - width = %d or height = %d invalid\n",
-			ZR_DEVNAME(zr), __func__, width, height);
-		return -EINVAL;
-	}
-
-	fh->overlay_settings.x = x;
-	fh->overlay_settings.y = y;
-	fh->overlay_settings.width = width;
-	fh->overlay_settings.height = height;
-	fh->overlay_settings.clipcount = clipcount;
-
-	/*
-	 * If an overlay is running, we have to switch it off
-	 * and switch it on again in order to get the new settings in effect.
-	 *
-	 * We also want to avoid that the overlay mask is written
-	 * when an overlay is running.
-	 */
-
-	on = zr->v4l_overlay_active && !zr->v4l_memgrab_active &&
-	    zr->overlay_active != ZORAN_FREE &&
-	    fh->overlay_active != ZORAN_FREE;
-	if (on)
-		zr36057_overlay(zr, 0);
-
-	/*
-	 *   Write the overlay mask if clips are wanted.
-	 *   We prefer a bitmap.
-	 */
-	if (bitmap) {
-		/* fake value - it just means we want clips */
-		fh->overlay_settings.clipcount = 1;
-
-		if (copy_from_user(fh->overlay_mask, bitmap,
-				   (width * height + 7) / 8)) {
-			return -EFAULT;
-		}
-	} else if (clipcount) {
-		/* write our own bitmap from the clips */
-		vcp = vmalloc(array_size(sizeof(struct v4l2_clip),
-					 clipcount + 4));
-		if (vcp == NULL) {
-			dprintk(1,
-				KERN_ERR
-				"%s: %s - Alloc of clip mask failed\n",
-				ZR_DEVNAME(zr), __func__);
-			return -ENOMEM;
-		}
-		if (copy_from_user
-		    (vcp, clips, sizeof(struct v4l2_clip) * clipcount)) {
-			vfree(vcp);
-			return -EFAULT;
-		}
-		write_overlay_mask(fh, vcp, clipcount);
-		vfree(vcp);
-	}
-
-	fh->overlay_settings.is_set = 1;
-	if (fh->overlay_active != ZORAN_FREE &&
-	    zr->overlay_active != ZORAN_FREE)
-		zr->overlay_settings = fh->overlay_settings;
-
-	if (on)
-		zr36057_overlay(zr, 1);
-
-	/* Make sure the changes come into effect */
-	return wait_grab_pending(zr);
-}
-
-static int setup_overlay(struct zoran_fh *fh, int on)
-{
-	struct zoran *zr = fh->zr;
-
-	/* If there is nothing to do, return immediately */
-	if ((on && fh->overlay_active != ZORAN_FREE) ||
-	    (!on && fh->overlay_active == ZORAN_FREE))
-		return 0;
-
-	/* check whether we're touching someone else's overlay */
-	if (on && zr->overlay_active != ZORAN_FREE &&
-	    fh->overlay_active == ZORAN_FREE) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - overlay is already active for another session\n",
-			ZR_DEVNAME(zr), __func__);
-		return -EBUSY;
-	}
-	if (!on && zr->overlay_active != ZORAN_FREE &&
-	    fh->overlay_active == ZORAN_FREE) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - you cannot cancel someone else's session\n",
-			ZR_DEVNAME(zr), __func__);
-		return -EPERM;
-	}
-
-	if (on == 0) {
-		zr->overlay_active = fh->overlay_active = ZORAN_FREE;
-		zr->v4l_overlay_active = 0;
-		/* When a grab is running, the video simply
-		 * won't be switched on any more */
-		if (!zr->v4l_memgrab_active)
-			zr36057_overlay(zr, 0);
-		zr->overlay_mask = NULL;
-	} else {
-		if (!zr->vbuf_base || !fh->overlay_settings.is_set) {
-			dprintk(1,
-				KERN_ERR
-				"%s: %s - buffer or window not set\n",
-				ZR_DEVNAME(zr), __func__);
-			return -EINVAL;
-		}
-		if (!fh->overlay_settings.format) {
-			dprintk(1,
-				KERN_ERR
-				"%s: %s - no overlay format set\n",
-				ZR_DEVNAME(zr), __func__);
-			return -EINVAL;
-		}
-		zr->overlay_active = fh->overlay_active = ZORAN_LOCKED;
-		zr->v4l_overlay_active = 1;
-		zr->overlay_mask = fh->overlay_mask;
-		zr->overlay_settings = fh->overlay_settings;
-		if (!zr->v4l_memgrab_active)
-			zr36057_overlay(zr, 1);
-		/* When a grab is running, the video will be
-		 * switched on when grab is finished */
-	}
-
-	/* Make sure the changes come into effect */
-	return wait_grab_pending(zr);
-}
-
-/* get the status of a buffer in the clients buffer queue */
-static int zoran_v4l2_buffer_status(struct zoran_fh *fh,
-				    struct v4l2_buffer *buf, int num)
-{
-	struct zoran *zr = fh->zr;
-	unsigned long flags;
-
-	buf->flags = V4L2_BUF_FLAG_MAPPED | V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
-
-	switch (fh->map_mode) {
-	case ZORAN_MAP_MODE_RAW:
-		/* check range */
-		if (num < 0 || num >= fh->buffers.num_buffers ||
-		    !fh->buffers.allocated) {
-			dprintk(1,
-				KERN_ERR
-				"%s: %s - wrong number or buffers not allocated\n",
-				ZR_DEVNAME(zr), __func__);
-			return -EINVAL;
-		}
-
-		spin_lock_irqsave(&zr->spinlock, flags);
-		dprintk(3,
-			KERN_DEBUG
-			"%s: %s() - raw active=%c, buffer %d: state=%c, map=%c\n",
-			ZR_DEVNAME(zr), __func__,
-			"FAL"[fh->buffers.active], num,
-			"UPMD"[zr->v4l_buffers.buffer[num].state],
-			fh->buffers.buffer[num].map ? 'Y' : 'N');
-		spin_unlock_irqrestore(&zr->spinlock, flags);
-
-		buf->type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
-		buf->length = fh->buffers.buffer_size;
-
-		/* get buffer */
-		buf->bytesused = fh->buffers.buffer[num].bs.length;
-		if (fh->buffers.buffer[num].state == BUZ_STATE_DONE ||
-		    fh->buffers.buffer[num].state == BUZ_STATE_USER) {
-			buf->sequence = fh->buffers.buffer[num].bs.seq;
-			buf->flags |= V4L2_BUF_FLAG_DONE;
-			buf->timestamp = ns_to_timeval(fh->buffers.buffer[num].bs.ts);
-		} else {
-			buf->flags |= V4L2_BUF_FLAG_QUEUED;
-		}
-
-		if (fh->v4l_settings.height <= BUZ_MAX_HEIGHT / 2)
-			buf->field = V4L2_FIELD_TOP;
-		else
-			buf->field = V4L2_FIELD_INTERLACED;
-
-		break;
-
-	case ZORAN_MAP_MODE_JPG_REC:
-	case ZORAN_MAP_MODE_JPG_PLAY:
-
-		/* check range */
-		if (num < 0 || num >= fh->buffers.num_buffers ||
-		    !fh->buffers.allocated) {
-			dprintk(1,
-				KERN_ERR
-				"%s: %s - wrong number or buffers not allocated\n",
-				ZR_DEVNAME(zr), __func__);
-			return -EINVAL;
-		}
-
-		buf->type = (fh->map_mode == ZORAN_MAP_MODE_JPG_REC) ?
-			      V4L2_BUF_TYPE_VIDEO_CAPTURE :
-			      V4L2_BUF_TYPE_VIDEO_OUTPUT;
-		buf->length = fh->buffers.buffer_size;
-
-		/* these variables are only written after frame has been captured */
-		if (fh->buffers.buffer[num].state == BUZ_STATE_DONE ||
-		    fh->buffers.buffer[num].state == BUZ_STATE_USER) {
-			buf->sequence = fh->buffers.buffer[num].bs.seq;
-			buf->timestamp = ns_to_timeval(fh->buffers.buffer[num].bs.ts);
-			buf->bytesused = fh->buffers.buffer[num].bs.length;
-			buf->flags |= V4L2_BUF_FLAG_DONE;
-		} else {
-			buf->flags |= V4L2_BUF_FLAG_QUEUED;
-		}
-
-		/* which fields are these? */
-		if (fh->jpg_settings.TmpDcm != 1)
-			buf->field = fh->jpg_settings.odd_even ?
-				V4L2_FIELD_TOP : V4L2_FIELD_BOTTOM;
-		else
-			buf->field = fh->jpg_settings.odd_even ?
-				V4L2_FIELD_SEQ_TB : V4L2_FIELD_SEQ_BT;
-
-		break;
-
-	default:
-
-		dprintk(5,
-			KERN_ERR
-			"%s: %s - invalid buffer type|map_mode (%d|%d)\n",
-			ZR_DEVNAME(zr), __func__, buf->type, fh->map_mode);
-		return -EINVAL;
-	}
-
-	buf->memory = V4L2_MEMORY_MMAP;
-	buf->index = num;
-	buf->m.offset = buf->length * num;
-
-	return 0;
-}
-
-static int
-zoran_set_norm (struct zoran *zr,
-		v4l2_std_id norm)
-{
-	int on;
-
-	if (zr->v4l_buffers.active != ZORAN_FREE ||
-	    zr->jpg_buffers.active != ZORAN_FREE) {
-		dprintk(1,
-			KERN_WARNING
-			"%s: %s called while in playback/capture mode\n",
-			ZR_DEVNAME(zr), __func__);
-		return -EBUSY;
-	}
-
-	if (!(norm & zr->card.norms)) {
-		dprintk(1,
-			KERN_ERR "%s: %s - unsupported norm %llx\n",
-			ZR_DEVNAME(zr), __func__, norm);
-		return -EINVAL;
-	}
-
-	if (norm & V4L2_STD_SECAM)
-		zr->timing = zr->card.tvn[2];
-	else if (norm & V4L2_STD_NTSC)
-		zr->timing = zr->card.tvn[1];
-	else
-		zr->timing = zr->card.tvn[0];
-
-	/* We switch overlay off and on since a change in the
-	 * norm needs different VFE settings */
-	on = zr->overlay_active && !zr->v4l_memgrab_active;
-	if (on)
-		zr36057_overlay(zr, 0);
-
-	decoder_call(zr, video, s_std, norm);
-	encoder_call(zr, video, s_std_output, norm);
-
-	if (on)
-		zr36057_overlay(zr, 1);
-
-	/* Make sure the changes come into effect */
-	zr->norm = norm;
-
-	return 0;
-}
-
-static int
-zoran_set_input (struct zoran *zr,
-		 int           input)
-{
-	if (input == zr->input) {
-		return 0;
-	}
-
-	if (zr->v4l_buffers.active != ZORAN_FREE ||
-	    zr->jpg_buffers.active != ZORAN_FREE) {
-		dprintk(1,
-			KERN_WARNING
-			"%s: %s called while in playback/capture mode\n",
-			ZR_DEVNAME(zr), __func__);
-		return -EBUSY;
-	}
-
-	if (input < 0 || input >= zr->card.inputs) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - unsupported input %d\n",
-			ZR_DEVNAME(zr), __func__, input);
-		return -EINVAL;
-	}
-
-	zr->input = input;
-
-	decoder_call(zr, video, s_routing,
-			zr->card.input[input].muxsel, 0, 0);
-
-	return 0;
-}
-
-/*
- *   ioctl routine
- */
-
-static int zoran_querycap(struct file *file, void *__fh, struct v4l2_capability *cap)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-
-	strscpy(cap->card, ZR_DEVNAME(zr), sizeof(cap->card));
-	strscpy(cap->driver, "zoran", sizeof(cap->driver));
-	snprintf(cap->bus_info, sizeof(cap->bus_info), "PCI:%s",
-		 pci_name(zr->pci_dev));
-	cap->device_caps = V4L2_CAP_STREAMING | V4L2_CAP_VIDEO_CAPTURE |
-			   V4L2_CAP_VIDEO_OUTPUT | V4L2_CAP_VIDEO_OVERLAY;
-	cap->capabilities = cap->device_caps | V4L2_CAP_DEVICE_CAPS;
-	return 0;
-}
-
-static int zoran_enum_fmt(struct zoran *zr, struct v4l2_fmtdesc *fmt, int flag)
-{
-	unsigned int num, i;
-
-	for (num = i = 0; i < NUM_FORMATS; i++) {
-		if (zoran_formats[i].flags & flag && num++ == fmt->index) {
-			strncpy(fmt->description, zoran_formats[i].name,
-				sizeof(fmt->description) - 1);
-			/* fmt struct pre-zeroed, so adding '\0' not needed */
-			fmt->pixelformat = zoran_formats[i].fourcc;
-			if (zoran_formats[i].flags & ZORAN_FORMAT_COMPRESSED)
-				fmt->flags |= V4L2_FMT_FLAG_COMPRESSED;
-			return 0;
-		}
-	}
-	return -EINVAL;
-}
-
-static int zoran_enum_fmt_vid_cap(struct file *file, void *__fh,
-					    struct v4l2_fmtdesc *f)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-
-	return zoran_enum_fmt(zr, f, ZORAN_FORMAT_CAPTURE);
-}
-
-static int zoran_enum_fmt_vid_out(struct file *file, void *__fh,
-					    struct v4l2_fmtdesc *f)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-
-	return zoran_enum_fmt(zr, f, ZORAN_FORMAT_PLAYBACK);
-}
-
-static int zoran_enum_fmt_vid_overlay(struct file *file, void *__fh,
-					    struct v4l2_fmtdesc *f)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-
-	return zoran_enum_fmt(zr, f, ZORAN_FORMAT_OVERLAY);
-}
-
-static int zoran_g_fmt_vid_out(struct file *file, void *__fh,
-					struct v4l2_format *fmt)
-{
-	struct zoran_fh *fh = __fh;
-
-	fmt->fmt.pix.width = fh->jpg_settings.img_width / fh->jpg_settings.HorDcm;
-	fmt->fmt.pix.height = fh->jpg_settings.img_height * 2 /
-		(fh->jpg_settings.VerDcm * fh->jpg_settings.TmpDcm);
-	fmt->fmt.pix.sizeimage = zoran_v4l2_calc_bufsize(&fh->jpg_settings);
-	fmt->fmt.pix.pixelformat = V4L2_PIX_FMT_MJPEG;
-	if (fh->jpg_settings.TmpDcm == 1)
-		fmt->fmt.pix.field = (fh->jpg_settings.odd_even ?
-				V4L2_FIELD_SEQ_TB : V4L2_FIELD_SEQ_BT);
-	else
-		fmt->fmt.pix.field = (fh->jpg_settings.odd_even ?
-				V4L2_FIELD_TOP : V4L2_FIELD_BOTTOM);
-	fmt->fmt.pix.bytesperline = 0;
-	fmt->fmt.pix.colorspace = V4L2_COLORSPACE_SMPTE170M;
-
-	return 0;
-}
-
-static int zoran_g_fmt_vid_cap(struct file *file, void *__fh,
-					struct v4l2_format *fmt)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-
-	if (fh->map_mode != ZORAN_MAP_MODE_RAW)
-		return zoran_g_fmt_vid_out(file, fh, fmt);
-
-	fmt->fmt.pix.width = fh->v4l_settings.width;
-	fmt->fmt.pix.height = fh->v4l_settings.height;
-	fmt->fmt.pix.sizeimage = fh->v4l_settings.bytesperline *
-					fh->v4l_settings.height;
-	fmt->fmt.pix.pixelformat = fh->v4l_settings.format->fourcc;
-	fmt->fmt.pix.colorspace = fh->v4l_settings.format->colorspace;
-	fmt->fmt.pix.bytesperline = fh->v4l_settings.bytesperline;
-	if (BUZ_MAX_HEIGHT < (fh->v4l_settings.height * 2))
-		fmt->fmt.pix.field = V4L2_FIELD_INTERLACED;
-	else
-		fmt->fmt.pix.field = V4L2_FIELD_TOP;
-	return 0;
-}
-
-static int zoran_g_fmt_vid_overlay(struct file *file, void *__fh,
-					struct v4l2_format *fmt)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-
-	fmt->fmt.win.w.left = fh->overlay_settings.x;
-	fmt->fmt.win.w.top = fh->overlay_settings.y;
-	fmt->fmt.win.w.width = fh->overlay_settings.width;
-	fmt->fmt.win.w.height = fh->overlay_settings.height;
-	if (fh->overlay_settings.width * 2 > BUZ_MAX_HEIGHT)
-		fmt->fmt.win.field = V4L2_FIELD_INTERLACED;
-	else
-		fmt->fmt.win.field = V4L2_FIELD_TOP;
-
-	return 0;
-}
-
-static int zoran_try_fmt_vid_overlay(struct file *file, void *__fh,
-					struct v4l2_format *fmt)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-
-	if (fmt->fmt.win.w.width > BUZ_MAX_WIDTH)
-		fmt->fmt.win.w.width = BUZ_MAX_WIDTH;
-	if (fmt->fmt.win.w.width < BUZ_MIN_WIDTH)
-		fmt->fmt.win.w.width = BUZ_MIN_WIDTH;
-	if (fmt->fmt.win.w.height > BUZ_MAX_HEIGHT)
-		fmt->fmt.win.w.height = BUZ_MAX_HEIGHT;
-	if (fmt->fmt.win.w.height < BUZ_MIN_HEIGHT)
-		fmt->fmt.win.w.height = BUZ_MIN_HEIGHT;
-
-	return 0;
-}
-
-static int zoran_try_fmt_vid_out(struct file *file, void *__fh,
-					struct v4l2_format *fmt)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-	struct zoran_jpg_settings settings;
-	int res = 0;
-
-	if (fmt->fmt.pix.pixelformat != V4L2_PIX_FMT_MJPEG)
-		return -EINVAL;
-
-	settings = fh->jpg_settings;
-
-	/* we actually need to set 'real' parameters now */
-	if ((fmt->fmt.pix.height * 2) > BUZ_MAX_HEIGHT)
-		settings.TmpDcm = 1;
-	else
-		settings.TmpDcm = 2;
-	settings.decimation = 0;
-	if (fmt->fmt.pix.height <= fh->jpg_settings.img_height / 2)
-		settings.VerDcm = 2;
-	else
-		settings.VerDcm = 1;
-	if (fmt->fmt.pix.width <= fh->jpg_settings.img_width / 4)
-		settings.HorDcm = 4;
-	else if (fmt->fmt.pix.width <= fh->jpg_settings.img_width / 2)
-		settings.HorDcm = 2;
-	else
-		settings.HorDcm = 1;
-	if (settings.TmpDcm == 1)
-		settings.field_per_buff = 2;
-	else
-		settings.field_per_buff = 1;
-
-	if (settings.HorDcm > 1) {
-		settings.img_x = (BUZ_MAX_WIDTH == 720) ? 8 : 0;
-		settings.img_width = (BUZ_MAX_WIDTH == 720) ? 704 : BUZ_MAX_WIDTH;
-	} else {
-		settings.img_x = 0;
-		settings.img_width = BUZ_MAX_WIDTH;
-	}
-
-	/* check */
-	res = zoran_check_jpg_settings(zr, &settings, 1);
-	if (res)
-		return res;
-
-	/* tell the user what we actually did */
-	fmt->fmt.pix.width = settings.img_width / settings.HorDcm;
-	fmt->fmt.pix.height = settings.img_height * 2 /
-		(settings.TmpDcm * settings.VerDcm);
-	if (settings.TmpDcm == 1)
-		fmt->fmt.pix.field = (fh->jpg_settings.odd_even ?
-				V4L2_FIELD_SEQ_TB : V4L2_FIELD_SEQ_BT);
-	else
-		fmt->fmt.pix.field = (fh->jpg_settings.odd_even ?
-				V4L2_FIELD_TOP : V4L2_FIELD_BOTTOM);
-
-	fmt->fmt.pix.sizeimage = zoran_v4l2_calc_bufsize(&settings);
-	fmt->fmt.pix.bytesperline = 0;
-	fmt->fmt.pix.colorspace = V4L2_COLORSPACE_SMPTE170M;
-	return res;
-}
-
-static int zoran_try_fmt_vid_cap(struct file *file, void *__fh,
-					struct v4l2_format *fmt)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-	int bpp;
-	int i;
-
-	if (fmt->fmt.pix.pixelformat == V4L2_PIX_FMT_MJPEG)
-		return zoran_try_fmt_vid_out(file, fh, fmt);
-
-	for (i = 0; i < NUM_FORMATS; i++)
-		if (zoran_formats[i].fourcc == fmt->fmt.pix.pixelformat)
-			break;
-
-	if (i == NUM_FORMATS)
-		return -EINVAL;
-
-	bpp = DIV_ROUND_UP(zoran_formats[i].depth, 8);
-	v4l_bound_align_image(
-		&fmt->fmt.pix.width, BUZ_MIN_WIDTH, BUZ_MAX_WIDTH, bpp == 2 ? 1 : 2,
-		&fmt->fmt.pix.height, BUZ_MIN_HEIGHT, BUZ_MAX_HEIGHT, 0, 0);
-	return 0;
-}
-
-static int zoran_s_fmt_vid_overlay(struct file *file, void *__fh,
-					struct v4l2_format *fmt)
-{
-	struct zoran_fh *fh = __fh;
-
-	dprintk(3, "x=%d, y=%d, w=%d, h=%d, cnt=%d, map=0x%p\n",
-			fmt->fmt.win.w.left, fmt->fmt.win.w.top,
-			fmt->fmt.win.w.width,
-			fmt->fmt.win.w.height,
-			fmt->fmt.win.clipcount,
-			fmt->fmt.win.bitmap);
-	return setup_window(fh, fmt->fmt.win.w.left, fmt->fmt.win.w.top,
-			   fmt->fmt.win.w.width, fmt->fmt.win.w.height,
-			   (struct v4l2_clip __user *)fmt->fmt.win.clips,
-			   fmt->fmt.win.clipcount, fmt->fmt.win.bitmap);
-}
-
-static int zoran_s_fmt_vid_out(struct file *file, void *__fh,
-					struct v4l2_format *fmt)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-	__le32 printformat = __cpu_to_le32(fmt->fmt.pix.pixelformat);
-	struct zoran_jpg_settings settings;
-	int res = 0;
-
-	dprintk(3, "size=%dx%d, fmt=0x%x (%4.4s)\n",
-			fmt->fmt.pix.width, fmt->fmt.pix.height,
-			fmt->fmt.pix.pixelformat,
-			(char *) &printformat);
-	if (fmt->fmt.pix.pixelformat != V4L2_PIX_FMT_MJPEG)
-		return -EINVAL;
-
-	if (fh->buffers.allocated) {
-		dprintk(1, KERN_ERR "%s: VIDIOC_S_FMT - cannot change capture mode\n",
-			ZR_DEVNAME(zr));
-		return -EBUSY;
-	}
-
-	settings = fh->jpg_settings;
-
-	/* we actually need to set 'real' parameters now */
-	if (fmt->fmt.pix.height * 2 > BUZ_MAX_HEIGHT)
-		settings.TmpDcm = 1;
-	else
-		settings.TmpDcm = 2;
-	settings.decimation = 0;
-	if (fmt->fmt.pix.height <= fh->jpg_settings.img_height / 2)
-		settings.VerDcm = 2;
-	else
-		settings.VerDcm = 1;
-	if (fmt->fmt.pix.width <= fh->jpg_settings.img_width / 4)
-		settings.HorDcm = 4;
-	else if (fmt->fmt.pix.width <= fh->jpg_settings.img_width / 2)
-		settings.HorDcm = 2;
-	else
-		settings.HorDcm = 1;
-	if (settings.TmpDcm == 1)
-		settings.field_per_buff = 2;
-	else
-		settings.field_per_buff = 1;
-
-	if (settings.HorDcm > 1) {
-		settings.img_x = (BUZ_MAX_WIDTH == 720) ? 8 : 0;
-		settings.img_width = (BUZ_MAX_WIDTH == 720) ? 704 : BUZ_MAX_WIDTH;
-	} else {
-		settings.img_x = 0;
-		settings.img_width = BUZ_MAX_WIDTH;
-	}
-
-	/* check */
-	res = zoran_check_jpg_settings(zr, &settings, 0);
-	if (res)
-		return res;
-
-	/* it's ok, so set them */
-	fh->jpg_settings = settings;
-
-	map_mode_jpg(fh, fmt->type == V4L2_BUF_TYPE_VIDEO_OUTPUT);
-	fh->buffers.buffer_size = zoran_v4l2_calc_bufsize(&fh->jpg_settings);
-
-	/* tell the user what we actually did */
-	fmt->fmt.pix.width = settings.img_width / settings.HorDcm;
-	fmt->fmt.pix.height = settings.img_height * 2 /
-		(settings.TmpDcm * settings.VerDcm);
-	if (settings.TmpDcm == 1)
-		fmt->fmt.pix.field = (fh->jpg_settings.odd_even ?
-				V4L2_FIELD_SEQ_TB : V4L2_FIELD_SEQ_BT);
-	else
-		fmt->fmt.pix.field = (fh->jpg_settings.odd_even ?
-				V4L2_FIELD_TOP : V4L2_FIELD_BOTTOM);
-	fmt->fmt.pix.bytesperline = 0;
-	fmt->fmt.pix.sizeimage = fh->buffers.buffer_size;
-	fmt->fmt.pix.colorspace = V4L2_COLORSPACE_SMPTE170M;
-	return res;
-}
-
-static int zoran_s_fmt_vid_cap(struct file *file, void *__fh,
-					struct v4l2_format *fmt)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-	int i;
-	int res = 0;
-
-	if (fmt->fmt.pix.pixelformat == V4L2_PIX_FMT_MJPEG)
-		return zoran_s_fmt_vid_out(file, fh, fmt);
-
-	for (i = 0; i < NUM_FORMATS; i++)
-		if (fmt->fmt.pix.pixelformat == zoran_formats[i].fourcc)
-			break;
-	if (i == NUM_FORMATS) {
-		dprintk(1, KERN_ERR "%s: VIDIOC_S_FMT - unknown/unsupported format 0x%x\n",
-			ZR_DEVNAME(zr), fmt->fmt.pix.pixelformat);
-		return -EINVAL;
-	}
-
-	if ((fh->map_mode != ZORAN_MAP_MODE_RAW && fh->buffers.allocated) ||
-	    fh->buffers.active != ZORAN_FREE) {
-		dprintk(1, KERN_ERR "%s: VIDIOC_S_FMT - cannot change capture mode\n",
-				ZR_DEVNAME(zr));
-		return -EBUSY;
-	}
-	if (fmt->fmt.pix.height > BUZ_MAX_HEIGHT)
-		fmt->fmt.pix.height = BUZ_MAX_HEIGHT;
-	if (fmt->fmt.pix.width > BUZ_MAX_WIDTH)
-		fmt->fmt.pix.width = BUZ_MAX_WIDTH;
-
-	map_mode_raw(fh);
-
-	res = zoran_v4l_set_format(fh, fmt->fmt.pix.width, fmt->fmt.pix.height,
-				   &zoran_formats[i]);
-	if (res)
-		return res;
-
-	/* tell the user the results/missing stuff */
-	fmt->fmt.pix.bytesperline = fh->v4l_settings.bytesperline;
-	fmt->fmt.pix.sizeimage = fh->v4l_settings.height * fh->v4l_settings.bytesperline;
-	fmt->fmt.pix.colorspace = fh->v4l_settings.format->colorspace;
-	if (BUZ_MAX_HEIGHT < (fh->v4l_settings.height * 2))
-		fmt->fmt.pix.field = V4L2_FIELD_INTERLACED;
-	else
-		fmt->fmt.pix.field = V4L2_FIELD_TOP;
-	return res;
-}
-
-static int zoran_g_fbuf(struct file *file, void *__fh,
-		struct v4l2_framebuffer *fb)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-
-	memset(fb, 0, sizeof(*fb));
-	fb->base = zr->vbuf_base;
-	fb->fmt.width = zr->vbuf_width;
-	fb->fmt.height = zr->vbuf_height;
-	if (zr->overlay_settings.format)
-		fb->fmt.pixelformat = fh->overlay_settings.format->fourcc;
-	fb->fmt.bytesperline = zr->vbuf_bytesperline;
-	fb->fmt.colorspace = V4L2_COLORSPACE_SRGB;
-	fb->fmt.field = V4L2_FIELD_INTERLACED;
-	fb->capability = V4L2_FBUF_CAP_LIST_CLIPPING;
-
-	return 0;
-}
-
-static int zoran_s_fbuf(struct file *file, void *__fh,
-		const struct v4l2_framebuffer *fb)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-	int i;
-	__le32 printformat = __cpu_to_le32(fb->fmt.pixelformat);
-
-	for (i = 0; i < NUM_FORMATS; i++)
-		if (zoran_formats[i].fourcc == fb->fmt.pixelformat)
-			break;
-	if (i == NUM_FORMATS) {
-		dprintk(1, KERN_ERR "%s: VIDIOC_S_FBUF - format=0x%x (%4.4s) not allowed\n",
-			ZR_DEVNAME(zr), fb->fmt.pixelformat,
-			(char *)&printformat);
-		return -EINVAL;
-	}
-
-	return setup_fbuffer(fh, fb->base, &zoran_formats[i], fb->fmt.width,
-			    fb->fmt.height, fb->fmt.bytesperline);
-}
-
-static int zoran_overlay(struct file *file, void *__fh, unsigned int on)
-{
-	struct zoran_fh *fh = __fh;
-
-	return setup_overlay(fh, on);
-}
-
-static int zoran_streamoff(struct file *file, void *__fh, enum v4l2_buf_type type);
-
-static int zoran_reqbufs(struct file *file, void *__fh, struct v4l2_requestbuffers *req)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-	int res = 0;
-
-	if (req->memory != V4L2_MEMORY_MMAP) {
-		dprintk(2,
-				KERN_ERR
-				"%s: only MEMORY_MMAP capture is supported, not %d\n",
-				ZR_DEVNAME(zr), req->memory);
-		return -EINVAL;
-	}
-
-	if (req->count == 0)
-		return zoran_streamoff(file, fh, req->type);
-
-	if (fh->buffers.allocated) {
-		dprintk(2,
-				KERN_ERR
-				"%s: VIDIOC_REQBUFS - buffers already allocated\n",
-				ZR_DEVNAME(zr));
-		return -EBUSY;
-	}
-
-	if (fh->map_mode == ZORAN_MAP_MODE_RAW &&
-	    req->type == V4L2_BUF_TYPE_VIDEO_CAPTURE) {
-		/* control user input */
-		if (req->count < 2)
-			req->count = 2;
-		if (req->count > v4l_nbufs)
-			req->count = v4l_nbufs;
-
-		/* The next mmap will map the V4L buffers */
-		map_mode_raw(fh);
-		fh->buffers.num_buffers = req->count;
-
-		if (v4l_fbuffer_alloc(fh)) {
-			return -ENOMEM;
-		}
-	} else if (fh->map_mode == ZORAN_MAP_MODE_JPG_REC ||
-		   fh->map_mode == ZORAN_MAP_MODE_JPG_PLAY) {
-		/* we need to calculate size ourselves now */
-		if (req->count < 4)
-			req->count = 4;
-		if (req->count > jpg_nbufs)
-			req->count = jpg_nbufs;
-
-		/* The next mmap will map the MJPEG buffers */
-		map_mode_jpg(fh, req->type == V4L2_BUF_TYPE_VIDEO_OUTPUT);
-		fh->buffers.num_buffers = req->count;
-		fh->buffers.buffer_size = zoran_v4l2_calc_bufsize(&fh->jpg_settings);
-
-		if (jpg_fbuffer_alloc(fh)) {
-			return -ENOMEM;
-		}
-	} else {
-		dprintk(1,
-				KERN_ERR
-				"%s: VIDIOC_REQBUFS - unknown type %d\n",
-				ZR_DEVNAME(zr), req->type);
-		return -EINVAL;
-	}
-	return res;
-}
-
-static int zoran_querybuf(struct file *file, void *__fh, struct v4l2_buffer *buf)
-{
-	struct zoran_fh *fh = __fh;
-
-	return zoran_v4l2_buffer_status(fh, buf, buf->index);
-}
-
-static int zoran_qbuf(struct file *file, void *__fh, struct v4l2_buffer *buf)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-	int res = 0, codec_mode, buf_type;
-
-	switch (fh->map_mode) {
-	case ZORAN_MAP_MODE_RAW:
-		if (buf->type != V4L2_BUF_TYPE_VIDEO_CAPTURE) {
-			dprintk(1, KERN_ERR
-				"%s: VIDIOC_QBUF - invalid buf->type=%d for map_mode=%d\n",
-				ZR_DEVNAME(zr), buf->type, fh->map_mode);
-			return -EINVAL;
-		}
-
-		res = zoran_v4l_queue_frame(fh, buf->index);
-		if (res)
-			return res;
-		if (!zr->v4l_memgrab_active && fh->buffers.active == ZORAN_LOCKED)
-			zr36057_set_memgrab(zr, 1);
-		break;
-
-	case ZORAN_MAP_MODE_JPG_REC:
-	case ZORAN_MAP_MODE_JPG_PLAY:
-		if (fh->map_mode == ZORAN_MAP_MODE_JPG_PLAY) {
-			buf_type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
-			codec_mode = BUZ_MODE_MOTION_DECOMPRESS;
-		} else {
-			buf_type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
-			codec_mode = BUZ_MODE_MOTION_COMPRESS;
-		}
-
-		if (buf->type != buf_type) {
-			dprintk(1, KERN_ERR
-				"%s: VIDIOC_QBUF - invalid buf->type=%d for map_mode=%d\n",
-				ZR_DEVNAME(zr), buf->type, fh->map_mode);
-			return -EINVAL;
-		}
-
-		res = zoran_jpg_queue_frame(fh, buf->index, codec_mode);
-		if (res != 0)
-			return res;
-		if (zr->codec_mode == BUZ_MODE_IDLE &&
-		    fh->buffers.active == ZORAN_LOCKED)
-			zr36057_enable_jpg(zr, codec_mode);
-
-		break;
-
-	default:
-		dprintk(1, KERN_ERR
-			"%s: VIDIOC_QBUF - unsupported type %d\n",
-			ZR_DEVNAME(zr), buf->type);
-		res = -EINVAL;
-		break;
-	}
-	return res;
-}
-
-static int zoran_dqbuf(struct file *file, void *__fh, struct v4l2_buffer *buf)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-	int res = 0, buf_type, num = -1;	/* compiler borks here (?) */
-
-	switch (fh->map_mode) {
-	case ZORAN_MAP_MODE_RAW:
-		if (buf->type != V4L2_BUF_TYPE_VIDEO_CAPTURE) {
-			dprintk(1, KERN_ERR
-				"%s: VIDIOC_QBUF - invalid buf->type=%d for map_mode=%d\n",
-				ZR_DEVNAME(zr), buf->type, fh->map_mode);
-			return -EINVAL;
-		}
-
-		num = zr->v4l_pend[zr->v4l_sync_tail & V4L_MASK_FRAME];
-		if (file->f_flags & O_NONBLOCK &&
-		    zr->v4l_buffers.buffer[num].state != BUZ_STATE_DONE) {
-			return -EAGAIN;
-		}
-		res = v4l_sync(fh, num);
-		if (res)
-			return res;
-		zr->v4l_sync_tail++;
-		res = zoran_v4l2_buffer_status(fh, buf, num);
-		break;
-
-	case ZORAN_MAP_MODE_JPG_REC:
-	case ZORAN_MAP_MODE_JPG_PLAY:
-	{
-		struct zoran_sync bs;
-
-		if (fh->map_mode == ZORAN_MAP_MODE_JPG_PLAY)
-			buf_type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
-		else
-			buf_type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
-
-		if (buf->type != buf_type) {
-			dprintk(1, KERN_ERR
-				"%s: VIDIOC_QBUF - invalid buf->type=%d for map_mode=%d\n",
-				ZR_DEVNAME(zr), buf->type, fh->map_mode);
-			return -EINVAL;
-		}
-
-		num = zr->jpg_pend[zr->jpg_que_tail & BUZ_MASK_FRAME];
-
-		if (file->f_flags & O_NONBLOCK &&
-		    zr->jpg_buffers.buffer[num].state != BUZ_STATE_DONE) {
-			return -EAGAIN;
-		}
-		bs.frame = 0; /* suppress compiler warning */
-		res = jpg_sync(fh, &bs);
-		if (res)
-			return res;
-		res = zoran_v4l2_buffer_status(fh, buf, bs.frame);
-		break;
-	}
-
-	default:
-		dprintk(1, KERN_ERR
-			"%s: VIDIOC_DQBUF - unsupported type %d\n",
-			ZR_DEVNAME(zr), buf->type);
-		res = -EINVAL;
-		break;
-	}
-	return res;
-}
-
-static int zoran_streamon(struct file *file, void *__fh, enum v4l2_buf_type type)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-	int res = 0;
-
-	switch (fh->map_mode) {
-	case ZORAN_MAP_MODE_RAW:	/* raw capture */
-		if (zr->v4l_buffers.active != ZORAN_ACTIVE ||
-		    fh->buffers.active != ZORAN_ACTIVE) {
-			return -EBUSY;
-		}
-
-		zr->v4l_buffers.active = fh->buffers.active = ZORAN_LOCKED;
-		zr->v4l_settings = fh->v4l_settings;
-
-		zr->v4l_sync_tail = zr->v4l_pend_tail;
-		if (!zr->v4l_memgrab_active &&
-		    zr->v4l_pend_head != zr->v4l_pend_tail) {
-			zr36057_set_memgrab(zr, 1);
-		}
-		break;
-
-	case ZORAN_MAP_MODE_JPG_REC:
-	case ZORAN_MAP_MODE_JPG_PLAY:
-		/* what is the codec mode right now? */
-		if (zr->jpg_buffers.active != ZORAN_ACTIVE ||
-		    fh->buffers.active != ZORAN_ACTIVE) {
-			return -EBUSY;
-		}
-
-		zr->jpg_buffers.active = fh->buffers.active = ZORAN_LOCKED;
-
-		if (zr->jpg_que_head != zr->jpg_que_tail) {
-			/* Start the jpeg codec when the first frame is queued  */
-			jpeg_start(zr);
-		}
-		break;
-
-	default:
-		dprintk(1,
-			KERN_ERR
-			"%s: VIDIOC_STREAMON - invalid map mode %d\n",
-			ZR_DEVNAME(zr), fh->map_mode);
-		res = -EINVAL;
-		break;
-	}
-	return res;
-}
-
-static int zoran_streamoff(struct file *file, void *__fh, enum v4l2_buf_type type)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-	int i, res = 0;
-	unsigned long flags;
-
-	switch (fh->map_mode) {
-	case ZORAN_MAP_MODE_RAW:	/* raw capture */
-		if (fh->buffers.active == ZORAN_FREE &&
-		    zr->v4l_buffers.active != ZORAN_FREE) {
-			return -EPERM;	/* stay off other's settings! */
-		}
-		if (zr->v4l_buffers.active == ZORAN_FREE)
-			return res;
-
-		spin_lock_irqsave(&zr->spinlock, flags);
-		/* unload capture */
-		if (zr->v4l_memgrab_active) {
-
-			zr36057_set_memgrab(zr, 0);
-		}
-
-		for (i = 0; i < fh->buffers.num_buffers; i++)
-			zr->v4l_buffers.buffer[i].state = BUZ_STATE_USER;
-		fh->buffers = zr->v4l_buffers;
-
-		zr->v4l_buffers.active = fh->buffers.active = ZORAN_FREE;
-
-		zr->v4l_grab_seq = 0;
-		zr->v4l_pend_head = zr->v4l_pend_tail = 0;
-		zr->v4l_sync_tail = 0;
-
-		spin_unlock_irqrestore(&zr->spinlock, flags);
-
-		break;
-
-	case ZORAN_MAP_MODE_JPG_REC:
-	case ZORAN_MAP_MODE_JPG_PLAY:
-		if (fh->buffers.active == ZORAN_FREE &&
-		    zr->jpg_buffers.active != ZORAN_FREE) {
-			return -EPERM;	/* stay off other's settings! */
-		}
-		if (zr->jpg_buffers.active == ZORAN_FREE)
-			return res;
-
-		res = jpg_qbuf(fh, -1,
-			     (fh->map_mode == ZORAN_MAP_MODE_JPG_REC) ?
-			     BUZ_MODE_MOTION_COMPRESS :
-			     BUZ_MODE_MOTION_DECOMPRESS);
-		if (res)
-			return res;
-		break;
-	default:
-		dprintk(1, KERN_ERR
-			"%s: VIDIOC_STREAMOFF - invalid map mode %d\n",
-			ZR_DEVNAME(zr), fh->map_mode);
-		res = -EINVAL;
-		break;
-	}
-	return res;
-}
-static int zoran_g_std(struct file *file, void *__fh, v4l2_std_id *std)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-
-	*std = zr->norm;
-	return 0;
-}
-
-static int zoran_s_std(struct file *file, void *__fh, v4l2_std_id std)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-	int res = 0;
-
-	res = zoran_set_norm(zr, std);
-	if (res)
-		return res;
-
-	return wait_grab_pending(zr);
-}
-
-static int zoran_enum_input(struct file *file, void *__fh,
-				 struct v4l2_input *inp)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-
-	if (inp->index >= zr->card.inputs)
-		return -EINVAL;
-
-	strncpy(inp->name, zr->card.input[inp->index].name,
-		sizeof(inp->name) - 1);
-	inp->type = V4L2_INPUT_TYPE_CAMERA;
-	inp->std = V4L2_STD_ALL;
-
-	/* Get status of video decoder */
-	decoder_call(zr, video, g_input_status, &inp->status);
-	return 0;
-}
-
-static int zoran_g_input(struct file *file, void *__fh, unsigned int *input)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-
-	*input = zr->input;
-
-	return 0;
-}
-
-static int zoran_s_input(struct file *file, void *__fh, unsigned int input)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-	int res;
-
-	res = zoran_set_input(zr, input);
-	if (res)
-		return res;
-
-	/* Make sure the changes come into effect */
-	return wait_grab_pending(zr);
-}
-
-static int zoran_enum_output(struct file *file, void *__fh,
-				  struct v4l2_output *outp)
-{
-	if (outp->index != 0)
-		return -EINVAL;
-
-	outp->index = 0;
-	outp->type = V4L2_OUTPUT_TYPE_ANALOGVGAOVERLAY;
-	strncpy(outp->name, "Autodetect", sizeof(outp->name)-1);
-
-	return 0;
-}
-
-static int zoran_g_output(struct file *file, void *__fh, unsigned int *output)
-{
-	*output = 0;
-
-	return 0;
-}
-
-static int zoran_s_output(struct file *file, void *__fh, unsigned int output)
-{
-	if (output != 0)
-		return -EINVAL;
-
-	return 0;
-}
-
-/* cropping (sub-frame capture) */
-static int zoran_g_selection(struct file *file, void *__fh, struct v4l2_selection *sel)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-
-	if (sel->type != V4L2_BUF_TYPE_VIDEO_OUTPUT &&
-	    sel->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
-		return -EINVAL;
-
-	if (fh->map_mode == ZORAN_MAP_MODE_RAW) {
-		dprintk(1, KERN_ERR
-			"%s: VIDIOC_G_SELECTION - subcapture only supported for compressed capture\n",
-			ZR_DEVNAME(zr));
-		return -EINVAL;
-	}
-
-	switch (sel->target) {
-	case V4L2_SEL_TGT_CROP:
-		sel->r.top = fh->jpg_settings.img_y;
-		sel->r.left = fh->jpg_settings.img_x;
-		sel->r.width = fh->jpg_settings.img_width;
-		sel->r.height = fh->jpg_settings.img_height;
-		break;
-	case V4L2_SEL_TGT_CROP_DEFAULT:
-		sel->r.top = sel->r.left = 0;
-		sel->r.width = BUZ_MIN_WIDTH;
-		sel->r.height = BUZ_MIN_HEIGHT;
-		break;
-	case V4L2_SEL_TGT_CROP_BOUNDS:
-		sel->r.top = sel->r.left = 0;
-		sel->r.width = BUZ_MAX_WIDTH;
-		sel->r.height = BUZ_MAX_HEIGHT;
-		break;
-	default:
-		return -EINVAL;
-	}
-	return 0;
-}
-
-static int zoran_s_selection(struct file *file, void *__fh, struct v4l2_selection *sel)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-	struct zoran_jpg_settings settings;
-	int res;
-
-	if (sel->type != V4L2_BUF_TYPE_VIDEO_OUTPUT &&
-	    sel->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
-		return -EINVAL;
-
-	if (sel->target != V4L2_SEL_TGT_CROP)
-		return -EINVAL;
-
-	if (fh->map_mode == ZORAN_MAP_MODE_RAW) {
-		dprintk(1, KERN_ERR
-			"%s: VIDIOC_S_SELECTION - subcapture only supported for compressed capture\n",
-			ZR_DEVNAME(zr));
-		return -EINVAL;
-	}
-
-	settings = fh->jpg_settings;
-
-	if (fh->buffers.allocated) {
-		dprintk(1, KERN_ERR
-			"%s: VIDIOC_S_SELECTION - cannot change settings while active\n",
-			ZR_DEVNAME(zr));
-		return -EBUSY;
-	}
-
-	/* move into a form that we understand */
-	settings.img_x = sel->r.left;
-	settings.img_y = sel->r.top;
-	settings.img_width = sel->r.width;
-	settings.img_height = sel->r.height;
-
-	/* check validity */
-	res = zoran_check_jpg_settings(zr, &settings, 0);
-	if (res)
-		return res;
-
-	/* accept */
-	fh->jpg_settings = settings;
-	return res;
-}
-
-static int zoran_g_jpegcomp(struct file *file, void *__fh,
-					struct v4l2_jpegcompression *params)
-{
-	struct zoran_fh *fh = __fh;
-	memset(params, 0, sizeof(*params));
-
-	params->quality = fh->jpg_settings.jpg_comp.quality;
-	params->APPn = fh->jpg_settings.jpg_comp.APPn;
-	memcpy(params->APP_data,
-	       fh->jpg_settings.jpg_comp.APP_data,
-	       fh->jpg_settings.jpg_comp.APP_len);
-	params->APP_len = fh->jpg_settings.jpg_comp.APP_len;
-	memcpy(params->COM_data,
-	       fh->jpg_settings.jpg_comp.COM_data,
-	       fh->jpg_settings.jpg_comp.COM_len);
-	params->COM_len = fh->jpg_settings.jpg_comp.COM_len;
-	params->jpeg_markers =
-	    fh->jpg_settings.jpg_comp.jpeg_markers;
-
-	return 0;
-}
-
-static int zoran_s_jpegcomp(struct file *file, void *__fh,
-					const struct v4l2_jpegcompression *params)
-{
-	struct zoran_fh *fh = __fh;
-	struct zoran *zr = fh->zr;
-	int res = 0;
-	struct zoran_jpg_settings settings;
-
-	settings = fh->jpg_settings;
-
-	settings.jpg_comp = *params;
-
-	if (fh->buffers.active != ZORAN_FREE) {
-		dprintk(1, KERN_WARNING
-			"%s: VIDIOC_S_JPEGCOMP called while in playback/capture mode\n",
-			ZR_DEVNAME(zr));
-		return -EBUSY;
-	}
-
-	res = zoran_check_jpg_settings(zr, &settings, 0);
-	if (res)
-		return res;
-	if (!fh->buffers.allocated)
-		fh->buffers.buffer_size =
-			zoran_v4l2_calc_bufsize(&fh->jpg_settings);
-	fh->jpg_settings.jpg_comp = settings.jpg_comp;
-	return res;
-}
-
-static __poll_t
-zoran_poll (struct file *file,
-	    poll_table  *wait)
-{
-	struct zoran_fh *fh = file->private_data;
-	struct zoran *zr = fh->zr;
-	__poll_t res = v4l2_ctrl_poll(file, wait);
-	int frame;
-	unsigned long flags;
-
-	/* we should check whether buffers are ready to be synced on
-	 * (w/o waits - O_NONBLOCK) here
-	 * if ready for read (sync), return EPOLLIN|EPOLLRDNORM,
-	 * if ready for write (sync), return EPOLLOUT|EPOLLWRNORM,
-	 * if error, return EPOLLERR,
-	 * if no buffers queued or so, return EPOLLNVAL
-	 */
-
-	switch (fh->map_mode) {
-	case ZORAN_MAP_MODE_RAW:
-		poll_wait(file, &zr->v4l_capq, wait);
-		frame = zr->v4l_pend[zr->v4l_sync_tail & V4L_MASK_FRAME];
-
-		spin_lock_irqsave(&zr->spinlock, flags);
-		dprintk(3,
-			KERN_DEBUG
-			"%s: %s() raw - active=%c, sync_tail=%lu/%c, pend_tail=%lu, pend_head=%lu\n",
-			ZR_DEVNAME(zr), __func__,
-			"FAL"[fh->buffers.active], zr->v4l_sync_tail,
-			"UPMD"[zr->v4l_buffers.buffer[frame].state],
-			zr->v4l_pend_tail, zr->v4l_pend_head);
-		/* Process is the one capturing? */
-		if (fh->buffers.active != ZORAN_FREE &&
-		    /* Buffer ready to DQBUF? */
-		    zr->v4l_buffers.buffer[frame].state == BUZ_STATE_DONE)
-			res |= EPOLLIN | EPOLLRDNORM;
-		spin_unlock_irqrestore(&zr->spinlock, flags);
-
-		break;
-
-	case ZORAN_MAP_MODE_JPG_REC:
-	case ZORAN_MAP_MODE_JPG_PLAY:
-		poll_wait(file, &zr->jpg_capq, wait);
-		frame = zr->jpg_pend[zr->jpg_que_tail & BUZ_MASK_FRAME];
-
-		spin_lock_irqsave(&zr->spinlock, flags);
-		dprintk(3,
-			KERN_DEBUG
-			"%s: %s() jpg - active=%c, que_tail=%lu/%c, que_head=%lu, dma=%lu/%lu\n",
-			ZR_DEVNAME(zr), __func__,
-			"FAL"[fh->buffers.active], zr->jpg_que_tail,
-			"UPMD"[zr->jpg_buffers.buffer[frame].state],
-			zr->jpg_que_head, zr->jpg_dma_tail, zr->jpg_dma_head);
-		if (fh->buffers.active != ZORAN_FREE &&
-		    zr->jpg_buffers.buffer[frame].state == BUZ_STATE_DONE) {
-			if (fh->map_mode == ZORAN_MAP_MODE_JPG_REC)
-				res |= EPOLLIN | EPOLLRDNORM;
-			else
-				res |= EPOLLOUT | EPOLLWRNORM;
-		}
-		spin_unlock_irqrestore(&zr->spinlock, flags);
-
-		break;
-
-	default:
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - internal error, unknown map_mode=%d\n",
-			ZR_DEVNAME(zr), __func__, fh->map_mode);
-		res |= EPOLLERR;
-	}
-
-	return res;
-}
-
-
-/*
- * This maps the buffers to user space.
- *
- * Depending on the state of fh->map_mode
- * the V4L or the MJPEG buffers are mapped
- * per buffer or all together
- *
- * Note that we need to connect to some
- * unmap signal event to unmap the de-allocate
- * the buffer accordingly (zoran_vm_close())
- */
-
-static void
-zoran_vm_open (struct vm_area_struct *vma)
-{
-	struct zoran_mapping *map = vma->vm_private_data;
-	atomic_inc(&map->count);
-}
-
-static void
-zoran_vm_close (struct vm_area_struct *vma)
-{
-	struct zoran_mapping *map = vma->vm_private_data;
-	struct zoran_fh *fh = map->fh;
-	struct zoran *zr = fh->zr;
-	int i;
-
-	dprintk(3, KERN_INFO "%s: %s - munmap(%s)\n", ZR_DEVNAME(zr),
-		__func__, mode_name(fh->map_mode));
-
-	for (i = 0; i < fh->buffers.num_buffers; i++) {
-		if (fh->buffers.buffer[i].map == map)
-			fh->buffers.buffer[i].map = NULL;
-	}
-	kfree(map);
-
-	/* Any buffers still mapped? */
-	for (i = 0; i < fh->buffers.num_buffers; i++) {
-		if (fh->buffers.buffer[i].map) {
-			return;
-		}
-	}
-
-	dprintk(3, KERN_INFO "%s: %s - free %s buffers\n", ZR_DEVNAME(zr),
-		__func__, mode_name(fh->map_mode));
-
-	if (fh->map_mode == ZORAN_MAP_MODE_RAW) {
-		if (fh->buffers.active != ZORAN_FREE) {
-			unsigned long flags;
-
-			spin_lock_irqsave(&zr->spinlock, flags);
-			zr36057_set_memgrab(zr, 0);
-			zr->v4l_buffers.allocated = 0;
-			zr->v4l_buffers.active = fh->buffers.active = ZORAN_FREE;
-			spin_unlock_irqrestore(&zr->spinlock, flags);
-		}
-		v4l_fbuffer_free(fh);
-	} else {
-		if (fh->buffers.active != ZORAN_FREE) {
-			jpg_qbuf(fh, -1, zr->codec_mode);
-			zr->jpg_buffers.allocated = 0;
-			zr->jpg_buffers.active = fh->buffers.active = ZORAN_FREE;
-		}
-		jpg_fbuffer_free(fh);
-	}
-}
-
-static const struct vm_operations_struct zoran_vm_ops = {
-	.open = zoran_vm_open,
-	.close = zoran_vm_close,
-};
-
-static int
-zoran_mmap (struct file           *file,
-	    struct vm_area_struct *vma)
-{
-	struct zoran_fh *fh = file->private_data;
-	struct zoran *zr = fh->zr;
-	unsigned long size = (vma->vm_end - vma->vm_start);
-	unsigned long offset = vma->vm_pgoff << PAGE_SHIFT;
-	int i, j;
-	unsigned long page, start = vma->vm_start, todo, pos, fraglen;
-	int first, last;
-	struct zoran_mapping *map;
-	int res = 0;
-
-	dprintk(3,
-		KERN_INFO "%s: %s(%s) of 0x%08lx-0x%08lx (size=%lu)\n",
-		ZR_DEVNAME(zr), __func__,
-		mode_name(fh->map_mode), vma->vm_start, vma->vm_end, size);
-
-	if (!(vma->vm_flags & VM_SHARED) || !(vma->vm_flags & VM_READ) ||
-	    !(vma->vm_flags & VM_WRITE)) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s - no MAP_SHARED/PROT_{READ,WRITE} given\n",
-			ZR_DEVNAME(zr), __func__);
-		return -EINVAL;
-	}
-
-	if (!fh->buffers.allocated) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s(%s) - buffers not yet allocated\n",
-			ZR_DEVNAME(zr), __func__, mode_name(fh->map_mode));
-		return -ENOMEM;
-	}
-
-	first = offset / fh->buffers.buffer_size;
-	last = first - 1 + size / fh->buffers.buffer_size;
-	if (offset % fh->buffers.buffer_size != 0 ||
-	    size % fh->buffers.buffer_size != 0 || first < 0 ||
-	    last < 0 || first >= fh->buffers.num_buffers ||
-	    last >= fh->buffers.buffer_size) {
-		dprintk(1,
-			KERN_ERR
-			"%s: %s(%s) - offset=%lu or size=%lu invalid for bufsize=%d and numbufs=%d\n",
-			ZR_DEVNAME(zr), __func__, mode_name(fh->map_mode), offset, size,
-			fh->buffers.buffer_size,
-			fh->buffers.num_buffers);
-		return -EINVAL;
-	}
-
-	/* Check if any buffers are already mapped */
-	for (i = first; i <= last; i++) {
-		if (fh->buffers.buffer[i].map) {
-			dprintk(1,
-				KERN_ERR
-				"%s: %s(%s) - buffer %d already mapped\n",
-				ZR_DEVNAME(zr), __func__, mode_name(fh->map_mode), i);
-			return -EBUSY;
-		}
-	}
-
-	/* map these buffers */
-	map = kmalloc(sizeof(struct zoran_mapping), GFP_KERNEL);
-	if (!map) {
-		return -ENOMEM;
-	}
-	map->fh = fh;
-	atomic_set(&map->count, 1);
-
-	vma->vm_ops = &zoran_vm_ops;
-	vma->vm_flags |= VM_DONTEXPAND;
-	vma->vm_private_data = map;
-
-	if (fh->map_mode == ZORAN_MAP_MODE_RAW) {
-		for (i = first; i <= last; i++) {
-			todo = size;
-			if (todo > fh->buffers.buffer_size)
-				todo = fh->buffers.buffer_size;
-			page = fh->buffers.buffer[i].v4l.fbuffer_phys;
-			if (remap_pfn_range(vma, start, page >> PAGE_SHIFT,
-							todo, PAGE_SHARED)) {
-				dprintk(1,
-					KERN_ERR
-					"%s: %s(V4L) - remap_pfn_range failed\n",
-					ZR_DEVNAME(zr), __func__);
-				return -EAGAIN;
-			}
-			size -= todo;
-			start += todo;
-			fh->buffers.buffer[i].map = map;
-			if (size == 0)
-				break;
-		}
-	} else {
-		for (i = first; i <= last; i++) {
-			for (j = 0;
-			     j < fh->buffers.buffer_size / PAGE_SIZE;
-			     j++) {
-				fraglen =
-				    (le32_to_cpu(fh->buffers.buffer[i].jpg.
-				     frag_tab[2 * j + 1]) & ~1) << 1;
-				todo = size;
-				if (todo > fraglen)
-					todo = fraglen;
-				pos =
-				    le32_to_cpu(fh->buffers.
-				    buffer[i].jpg.frag_tab[2 * j]);
-				/* should just be pos on i386 */
-				page = virt_to_phys(bus_to_virt(pos))
-								>> PAGE_SHIFT;
-				if (remap_pfn_range(vma, start, page,
-							todo, PAGE_SHARED)) {
-					dprintk(1,
-						KERN_ERR
-						"%s: %s(V4L) - remap_pfn_range failed\n",
-						ZR_DEVNAME(zr), __func__);
-					return -EAGAIN;
-				}
-				size -= todo;
-				start += todo;
-				if (size == 0)
-					break;
-				if (le32_to_cpu(fh->buffers.buffer[i].jpg.
-				    frag_tab[2 * j + 1]) & 1)
-					break;	/* was last fragment */
-			}
-			fh->buffers.buffer[i].map = map;
-			if (size == 0)
-				break;
-
-		}
-	}
-	return res;
-}
-
-static const struct v4l2_ioctl_ops zoran_ioctl_ops = {
-	.vidioc_querycap		    = zoran_querycap,
-	.vidioc_s_selection		    = zoran_s_selection,
-	.vidioc_g_selection		    = zoran_g_selection,
-	.vidioc_enum_input		    = zoran_enum_input,
-	.vidioc_g_input			    = zoran_g_input,
-	.vidioc_s_input			    = zoran_s_input,
-	.vidioc_enum_output		    = zoran_enum_output,
-	.vidioc_g_output		    = zoran_g_output,
-	.vidioc_s_output		    = zoran_s_output,
-	.vidioc_g_fbuf			    = zoran_g_fbuf,
-	.vidioc_s_fbuf			    = zoran_s_fbuf,
-	.vidioc_g_std			    = zoran_g_std,
-	.vidioc_s_std			    = zoran_s_std,
-	.vidioc_g_jpegcomp		    = zoran_g_jpegcomp,
-	.vidioc_s_jpegcomp		    = zoran_s_jpegcomp,
-	.vidioc_overlay			    = zoran_overlay,
-	.vidioc_reqbufs			    = zoran_reqbufs,
-	.vidioc_querybuf		    = zoran_querybuf,
-	.vidioc_qbuf			    = zoran_qbuf,
-	.vidioc_dqbuf			    = zoran_dqbuf,
-	.vidioc_streamon		    = zoran_streamon,
-	.vidioc_streamoff		    = zoran_streamoff,
-	.vidioc_enum_fmt_vid_cap	    = zoran_enum_fmt_vid_cap,
-	.vidioc_enum_fmt_vid_out	    = zoran_enum_fmt_vid_out,
-	.vidioc_enum_fmt_vid_overlay	    = zoran_enum_fmt_vid_overlay,
-	.vidioc_g_fmt_vid_cap		    = zoran_g_fmt_vid_cap,
-	.vidioc_g_fmt_vid_out               = zoran_g_fmt_vid_out,
-	.vidioc_g_fmt_vid_overlay           = zoran_g_fmt_vid_overlay,
-	.vidioc_s_fmt_vid_cap		    = zoran_s_fmt_vid_cap,
-	.vidioc_s_fmt_vid_out               = zoran_s_fmt_vid_out,
-	.vidioc_s_fmt_vid_overlay           = zoran_s_fmt_vid_overlay,
-	.vidioc_try_fmt_vid_cap		    = zoran_try_fmt_vid_cap,
-	.vidioc_try_fmt_vid_out		    = zoran_try_fmt_vid_out,
-	.vidioc_try_fmt_vid_overlay	    = zoran_try_fmt_vid_overlay,
-	.vidioc_subscribe_event             = v4l2_ctrl_subscribe_event,
-	.vidioc_unsubscribe_event           = v4l2_event_unsubscribe,
-};
-
-static const struct v4l2_file_operations zoran_fops = {
-	.owner = THIS_MODULE,
-	.open = zoran_open,
-	.release = zoran_close,
-	.unlocked_ioctl = video_ioctl2,
-	.mmap = zoran_mmap,
-	.poll = zoran_poll,
-};
-
-const struct video_device zoran_template = {
-	.name = ZORAN_NAME,
-	.fops = &zoran_fops,
-	.ioctl_ops = &zoran_ioctl_ops,
-	.release = &zoran_vdev_release,
-	.tvnorms = V4L2_STD_NTSC | V4L2_STD_PAL | V4L2_STD_SECAM,
-};
-
diff --git a/drivers/staging/media/zoran/zoran_procfs.c b/drivers/staging/media/zoran/zoran_procfs.c
deleted file mode 100644
index 941f73fe323b..000000000000
--- a/drivers/staging/media/zoran/zoran_procfs.c
+++ /dev/null
@@ -1,211 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0+
-/*
- * Zoran zr36057/zr36067 PCI controller driver, for the
- * Pinnacle/Miro DC10/DC10+/DC30/DC30+, Iomega Buz, Linux
- * Media Labs LML33/LML33R10.
- *
- * This part handles the procFS entries (/proc/ZORAN[%d])
- *
- * Copyright (C) 2000 Serguei Miridonov <mirsev@cicese.mx>
- *
- * Currently maintained by:
- *   Ronald Bultje    <rbultje@ronald.bitfreak.net>
- *   Laurent Pinchart <laurent.pinchart@skynet.be>
- *   Mailinglist      <mjpeg-users@lists.sf.net>
- */
-#include <linux/types.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/vmalloc.h>
-
-#include <linux/proc_fs.h>
-#include <linux/pci.h>
-#include <linux/i2c.h>
-#include <linux/i2c-algo-bit.h>
-#include <linux/videodev2.h>
-#include <linux/spinlock.h>
-#include <linux/sem.h>
-#include <linux/seq_file.h>
-
-#include <linux/ctype.h>
-#include <linux/poll.h>
-#include <asm/io.h>
-
-#include "videocodec.h"
-#include "zoran.h"
-#include "zoran_procfs.h"
-#include "zoran_card.h"
-
-#ifdef CONFIG_PROC_FS
-struct procfs_params_zr36067 {
-	char *name;
-	short reg;
-	u32 mask;
-	short bit;
-};
-
-static const struct procfs_params_zr36067 zr67[] = {
-	{"HSPol", 0x000, 1, 30},
-	{"HStart", 0x000, 0x3ff, 10},
-	{"HEnd", 0x000, 0x3ff, 0},
-
-	{"VSPol", 0x004, 1, 30},
-	{"VStart", 0x004, 0x3ff, 10},
-	{"VEnd", 0x004, 0x3ff, 0},
-
-	{"ExtFl", 0x008, 1, 26},
-	{"TopField", 0x008, 1, 25},
-	{"VCLKPol", 0x008, 1, 24},
-	{"DupFld", 0x008, 1, 20},
-	{"LittleEndian", 0x008, 1, 0},
-
-	{"HsyncStart", 0x10c, 0xffff, 16},
-	{"LineTot", 0x10c, 0xffff, 0},
-
-	{"NAX", 0x110, 0xffff, 16},
-	{"PAX", 0x110, 0xffff, 0},
-
-	{"NAY", 0x114, 0xffff, 16},
-	{"PAY", 0x114, 0xffff, 0},
-
-	/* {"",,,}, */
-
-	{NULL, 0, 0, 0},
-};
-
-static void
-setparam (struct zoran *zr,
-	  char         *name,
-	  char         *sval)
-{
-	int i = 0, reg0, reg, val;
-
-	while (zr67[i].name != NULL) {
-		if (!strncmp(name, zr67[i].name, strlen(zr67[i].name))) {
-			reg = reg0 = btread(zr67[i].reg);
-			reg &= ~(zr67[i].mask << zr67[i].bit);
-			if (!isdigit(sval[0]))
-				break;
-			val = simple_strtoul(sval, NULL, 0);
-			if ((val & ~zr67[i].mask))
-				break;
-			reg |= (val & zr67[i].mask) << zr67[i].bit;
-			dprintk(4,
-				KERN_INFO
-				"%s: setparam: setting ZR36067 register 0x%03x: 0x%08x=>0x%08x %s=%d\n",
-				ZR_DEVNAME(zr), zr67[i].reg, reg0, reg,
-				zr67[i].name, val);
-			btwrite(reg, zr67[i].reg);
-			break;
-		}
-		i++;
-	}
-}
-
-static int zoran_show(struct seq_file *p, void *v)
-{
-	struct zoran *zr = p->private;
-	int i;
-
-	seq_printf(p, "ZR36067 registers:\n");
-	for (i = 0; i < 0x130; i += 16)
-		seq_printf(p, "%03X %08X  %08X  %08X  %08X \n", i,
-			   btread(i), btread(i+4), btread(i+8), btread(i+12));
-	return 0;
-}
-
-static int zoran_open(struct inode *inode, struct file *file)
-{
-	struct zoran *data = PDE_DATA(inode);
-	return single_open(file, zoran_show, data);
-}
-
-static ssize_t zoran_write(struct file *file, const char __user *buffer,
-			size_t count, loff_t *ppos)
-{
-	struct zoran *zr = PDE_DATA(file_inode(file));
-	char *string, *sp;
-	char *line, *ldelim, *varname, *svar, *tdelim;
-
-	if (count > 32768)	/* Stupidity filter */
-		return -EINVAL;
-
-	string = sp = vmalloc(count + 1);
-	if (!string) {
-		dprintk(1,
-			KERN_ERR
-			"%s: write_proc: can not allocate memory\n",
-			ZR_DEVNAME(zr));
-		return -ENOMEM;
-	}
-	if (copy_from_user(string, buffer, count)) {
-		vfree (string);
-		return -EFAULT;
-	}
-	string[count] = 0;
-	dprintk(4, KERN_INFO "%s: write_proc: name=%pD count=%zu zr=%p\n",
-		ZR_DEVNAME(zr), file, count, zr);
-	ldelim = " \t\n";
-	tdelim = "=";
-	line = strpbrk(sp, ldelim);
-	while (line) {
-		*line = 0;
-		svar = strpbrk(sp, tdelim);
-		if (svar) {
-			*svar = 0;
-			varname = sp;
-			svar++;
-			setparam(zr, varname, svar);
-		}
-		sp = line + 1;
-		line = strpbrk(sp, ldelim);
-	}
-	vfree(string);
-
-	return count;
-}
-
-static const struct file_operations zoran_operations = {
-	.owner		= THIS_MODULE,
-	.open		= zoran_open,
-	.read		= seq_read,
-	.write		= zoran_write,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
-#endif
-
-int
-zoran_proc_init (struct zoran *zr)
-{
-#ifdef CONFIG_PROC_FS
-	char name[8];
-
-	snprintf(name, 7, "zoran%d", zr->id);
-	zr->zoran_proc = proc_create_data(name, 0, NULL, &zoran_operations, zr);
-	if (zr->zoran_proc != NULL) {
-		dprintk(2,
-			KERN_INFO
-			"%s: procfs entry /proc/%s allocated. data=%p\n",
-			ZR_DEVNAME(zr), name, zr);
-	} else {
-		dprintk(1, KERN_ERR "%s: Unable to initialise /proc/%s\n",
-			ZR_DEVNAME(zr), name);
-		return 1;
-	}
-#endif
-	return 0;
-}
-
-void
-zoran_proc_cleanup (struct zoran *zr)
-{
-#ifdef CONFIG_PROC_FS
-	char name[8];
-
-	snprintf(name, 7, "zoran%d", zr->id);
-	if (zr->zoran_proc)
-		remove_proc_entry(name, NULL);
-	zr->zoran_proc = NULL;
-#endif
-}
diff --git a/drivers/staging/media/zoran/zoran_procfs.h b/drivers/staging/media/zoran/zoran_procfs.h
deleted file mode 100644
index db9f642c851a..000000000000
--- a/drivers/staging/media/zoran/zoran_procfs.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0+ */
-/*
- * Zoran zr36057/zr36067 PCI controller driver, for the
- * Pinnacle/Miro DC10/DC10+/DC30/DC30+, Iomega Buz, Linux
- * Media Labs LML33/LML33R10.
- *
- * This part handles card-specific data and detection
- *
- * Copyright (C) 2000 Serguei Miridonov <mirsev@cicese.mx>
- *
- * Currently maintained by:
- *   Ronald Bultje    <rbultje@ronald.bitfreak.net>
- *   Laurent Pinchart <laurent.pinchart@skynet.be>
- *   Mailinglist      <mjpeg-users@lists.sf.net>
- */
-#ifndef __ZORAN_PROCFS_H__
-#define __ZORAN_PROCFS_H__
-
-extern int zoran_proc_init(struct zoran *zr);
-extern void zoran_proc_cleanup(struct zoran *zr);
-
-#endif				/* __ZORAN_PROCFS_H__ */
diff --git a/drivers/staging/media/zoran/zr36016.c b/drivers/staging/media/zoran/zr36016.c
deleted file mode 100644
index b300a0abe95f..000000000000
--- a/drivers/staging/media/zoran/zr36016.c
+++ /dev/null
@@ -1,500 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0+
-/*
- * Zoran ZR36016 basic configuration functions
- *
- * Copyright (C) 2001 Wolfgang Scherr <scherr@net4you.at>
- */
-#define ZR016_VERSION "v0.7"
-
-#include <linux/module.h>
-#include <linux/init.h>
-#include <linux/slab.h>
-#include <linux/delay.h>
-
-#include <linux/types.h>
-#include <linux/wait.h>
-
-/* I/O commands, error codes */
-#include <asm/io.h>
-
-/* v4l  API */
-
-/* headerfile of this module */
-#include "zr36016.h"
-
-/* codec io API */
-#include "videocodec.h"
-
-/* it doesn't make sense to have more than 20 or so,
-  just to prevent some unwanted loops */
-#define MAX_CODECS 20
-
-/* amount of chips attached via this driver */
-static int zr36016_codecs;
-
-/* debugging is available via module parameter */
-static int debug;
-module_param(debug, int, 0);
-MODULE_PARM_DESC(debug, "Debug level (0-4)");
-
-#define dprintk(num, format, args...) \
-	do { \
-		if (debug >= num) \
-			printk(format, ##args); \
-	} while (0)
-
-/* =========================================================================
-   Local hardware I/O functions:
-
-   read/write via codec layer (registers are located in the master device)
-   ========================================================================= */
-
-/* read and write functions */
-static u8
-zr36016_read (struct zr36016 *ptr,
-	      u16             reg)
-{
-	u8 value = 0;
-
-	// just in case something is wrong...
-	if (ptr->codec->master_data->readreg)
-		value =
-		    (ptr->codec->master_data->
-		     readreg(ptr->codec, reg)) & 0xFF;
-	else
-		dprintk(1,
-			KERN_ERR "%s: invalid I/O setup, nothing read!\n",
-			ptr->name);
-
-	dprintk(4, "%s: reading from 0x%04x: %02x\n", ptr->name, reg,
-		value);
-
-	return value;
-}
-
-static void
-zr36016_write (struct zr36016 *ptr,
-	       u16             reg,
-	       u8              value)
-{
-	dprintk(4, "%s: writing 0x%02x to 0x%04x\n", ptr->name, value,
-		reg);
-
-	// just in case something is wrong...
-	if (ptr->codec->master_data->writereg) {
-		ptr->codec->master_data->writereg(ptr->codec, reg, value);
-	} else
-		dprintk(1,
-			KERN_ERR
-			"%s: invalid I/O setup, nothing written!\n",
-			ptr->name);
-}
-
-/* indirect read and write functions */
-/* the 016 supports auto-addr-increment, but
- * writing it all time cost not much and is safer... */
-static u8
-zr36016_readi (struct zr36016 *ptr,
-	       u16             reg)
-{
-	u8 value = 0;
-
-	// just in case something is wrong...
-	if ((ptr->codec->master_data->writereg) &&
-	    (ptr->codec->master_data->readreg)) {
-		ptr->codec->master_data->writereg(ptr->codec, ZR016_IADDR, reg & 0x0F);	// ADDR
-		value = (ptr->codec->master_data->readreg(ptr->codec, ZR016_IDATA)) & 0xFF;	// DATA
-	} else
-		dprintk(1,
-			KERN_ERR
-			"%s: invalid I/O setup, nothing read (i)!\n",
-			ptr->name);
-
-	dprintk(4, "%s: reading indirect from 0x%04x: %02x\n", ptr->name,
-		reg, value);
-	return value;
-}
-
-static void
-zr36016_writei (struct zr36016 *ptr,
-		u16             reg,
-		u8              value)
-{
-	dprintk(4, "%s: writing indirect 0x%02x to 0x%04x\n", ptr->name,
-		value, reg);
-
-	// just in case something is wrong...
-	if (ptr->codec->master_data->writereg) {
-		ptr->codec->master_data->writereg(ptr->codec, ZR016_IADDR, reg & 0x0F);	// ADDR
-		ptr->codec->master_data->writereg(ptr->codec, ZR016_IDATA, value & 0x0FF);	// DATA
-	} else
-		dprintk(1,
-			KERN_ERR
-			"%s: invalid I/O setup, nothing written (i)!\n",
-			ptr->name);
-}
-
-/* =========================================================================
-   Local helper function:
-
-   version read
-   ========================================================================= */
-
-/* version kept in datastructure */
-static u8
-zr36016_read_version (struct zr36016 *ptr)
-{
-	ptr->version = zr36016_read(ptr, 0) >> 4;
-	return ptr->version;
-}
-
-/* =========================================================================
-   Local helper function:
-
-   basic test of "connectivity", writes/reads to/from PAX-Lo register
-   ========================================================================= */
-
-static int
-zr36016_basic_test (struct zr36016 *ptr)
-{
-	if (debug) {
-		int i;
-		zr36016_writei(ptr, ZR016I_PAX_LO, 0x55);
-		dprintk(1, KERN_INFO "%s: registers: ", ptr->name);
-		for (i = 0; i <= 0x0b; i++)
-			dprintk(1, "%02x ", zr36016_readi(ptr, i));
-		dprintk(1, "\n");
-	}
-	// for testing just write 0, then the default value to a register and read
-	// it back in both cases
-	zr36016_writei(ptr, ZR016I_PAX_LO, 0x00);
-	if (zr36016_readi(ptr, ZR016I_PAX_LO) != 0x0) {
-		dprintk(1,
-			KERN_ERR
-			"%s: attach failed, can't connect to vfe processor!\n",
-			ptr->name);
-		return -ENXIO;
-	}
-	zr36016_writei(ptr, ZR016I_PAX_LO, 0x0d0);
-	if (zr36016_readi(ptr, ZR016I_PAX_LO) != 0x0d0) {
-		dprintk(1,
-			KERN_ERR
-			"%s: attach failed, can't connect to vfe processor!\n",
-			ptr->name);
-		return -ENXIO;
-	}
-	// we allow version numbers from 0-3, should be enough, though
-	zr36016_read_version(ptr);
-	if (ptr->version & 0x0c) {
-		dprintk(1,
-			KERN_ERR
-			"%s: attach failed, suspicious version %d found...\n",
-			ptr->name, ptr->version);
-		return -ENXIO;
-	}
-
-	return 0;		/* looks good! */
-}
-
-/* =========================================================================
-   Local helper function:
-
-   simple loop for pushing the init datasets - NO USE --
-   ========================================================================= */
-
-#if 0
-static int zr36016_pushit (struct zr36016 *ptr,
-			   u16             startreg,
-			   u16             len,
-			   const char     *data)
-{
-	int i=0;
-
-	dprintk(4, "%s: write data block to 0x%04x (len=%d)\n",
-		ptr->name, startreg,len);
-	while (i<len) {
-		zr36016_writei(ptr, startreg++,  data[i++]);
-	}
-
-	return i;
-}
-#endif
-
-/* =========================================================================
-   Basic datasets & init:
-
-   //TODO//
-   ========================================================================= */
-
-static void
-zr36016_init (struct zr36016 *ptr)
-{
-	// stop any processing
-	zr36016_write(ptr, ZR016_GOSTOP, 0);
-
-	// mode setup (yuv422 in and out, compression/expansuon due to mode)
-	zr36016_write(ptr, ZR016_MODE,
-		      ZR016_YUV422 | ZR016_YUV422_YUV422 |
-		      (ptr->mode == CODEC_DO_COMPRESSION ?
-		       ZR016_COMPRESSION : ZR016_EXPANSION));
-
-	// misc setup
-	zr36016_writei(ptr, ZR016I_SETUP1,
-		       (ptr->xdec ? (ZR016_HRFL | ZR016_HORZ) : 0) |
-		       (ptr->ydec ? ZR016_VERT : 0) | ZR016_CNTI);
-	zr36016_writei(ptr, ZR016I_SETUP2, ZR016_CCIR);
-
-	// Window setup
-	// (no extra offset for now, norm defines offset, default width height)
-	zr36016_writei(ptr, ZR016I_PAX_HI, ptr->width >> 8);
-	zr36016_writei(ptr, ZR016I_PAX_LO, ptr->width & 0xFF);
-	zr36016_writei(ptr, ZR016I_PAY_HI, ptr->height >> 8);
-	zr36016_writei(ptr, ZR016I_PAY_LO, ptr->height & 0xFF);
-	zr36016_writei(ptr, ZR016I_NAX_HI, ptr->xoff >> 8);
-	zr36016_writei(ptr, ZR016I_NAX_LO, ptr->xoff & 0xFF);
-	zr36016_writei(ptr, ZR016I_NAY_HI, ptr->yoff >> 8);
-	zr36016_writei(ptr, ZR016I_NAY_LO, ptr->yoff & 0xFF);
-
-	/* shall we continue now, please? */
-	zr36016_write(ptr, ZR016_GOSTOP, 1);
-}
-
-/* =========================================================================
-   CODEC API FUNCTIONS
-
-   this functions are accessed by the master via the API structure
-   ========================================================================= */
-
-/* set compression/expansion mode and launches codec -
-   this should be the last call from the master before starting processing */
-static int
-zr36016_set_mode (struct videocodec *codec,
-		  int                mode)
-{
-	struct zr36016 *ptr = (struct zr36016 *) codec->data;
-
-	dprintk(2, "%s: set_mode %d call\n", ptr->name, mode);
-
-	if ((mode != CODEC_DO_EXPANSION) && (mode != CODEC_DO_COMPRESSION))
-		return -EINVAL;
-
-	ptr->mode = mode;
-	zr36016_init(ptr);
-
-	return 0;
-}
-
-/* set picture size */
-static int
-zr36016_set_video (struct videocodec   *codec,
-		   struct tvnorm       *norm,
-		   struct vfe_settings *cap,
-		   struct vfe_polarity *pol)
-{
-	struct zr36016 *ptr = (struct zr36016 *) codec->data;
-
-	dprintk(2, "%s: set_video %d.%d, %d/%d-%dx%d (0x%x) call\n",
-		ptr->name, norm->HStart, norm->VStart,
-		cap->x, cap->y, cap->width, cap->height,
-		cap->decimation);
-
-	/* if () return -EINVAL;
-	 * trust the master driver that it knows what it does - so
-	 * we allow invalid startx/y for now ... */
-	ptr->width = cap->width;
-	ptr->height = cap->height;
-	/* (Ronald) This is ugly. zoran_device.c, line 387
-	 * already mentions what happens if HStart is even
-	 * (blue faces, etc., cr/cb inversed). There's probably
-	 * some good reason why HStart is 0 instead of 1, so I'm
-	 * leaving it to this for now, but really... This can be
-	 * done a lot simpler */
-	ptr->xoff = (norm->HStart ? norm->HStart : 1) + cap->x;
-	/* Something to note here (I don't understand it), setting
-	 * VStart too high will cause the codec to 'not work'. I
-	 * really don't get it. values of 16 (VStart) already break
-	 * it here. Just '0' seems to work. More testing needed! */
-	ptr->yoff = norm->VStart + cap->y;
-	/* (Ronald) dzjeeh, can't this thing do hor_decimation = 4? */
-	ptr->xdec = ((cap->decimation & 0xff) == 1) ? 0 : 1;
-	ptr->ydec = (((cap->decimation >> 8) & 0xff) == 1) ? 0 : 1;
-
-	return 0;
-}
-
-/* additional control functions */
-static int
-zr36016_control (struct videocodec *codec,
-		 int                type,
-		 int                size,
-		 void              *data)
-{
-	struct zr36016 *ptr = (struct zr36016 *) codec->data;
-	int *ival = (int *) data;
-
-	dprintk(2, "%s: control %d call with %d byte\n", ptr->name, type,
-		size);
-
-	switch (type) {
-	case CODEC_G_STATUS:	/* get last status - we don't know it ... */
-		if (size != sizeof(int))
-			return -EFAULT;
-		*ival = 0;
-		break;
-
-	case CODEC_G_CODEC_MODE:
-		if (size != sizeof(int))
-			return -EFAULT;
-		*ival = 0;
-		break;
-
-	case CODEC_S_CODEC_MODE:
-		if (size != sizeof(int))
-			return -EFAULT;
-		if (*ival != 0)
-			return -EINVAL;
-		/* not needed, do nothing */
-		return 0;
-
-	case CODEC_G_VFE:
-	case CODEC_S_VFE:
-		return 0;
-
-	case CODEC_S_MMAP:
-		/* not available, give an error */
-		return -ENXIO;
-
-	default:
-		return -EINVAL;
-	}
-
-	return size;
-}
-
-/* =========================================================================
-   Exit and unregister function:
-
-   Deinitializes Zoran's JPEG processor
-   ========================================================================= */
-
-static int
-zr36016_unset (struct videocodec *codec)
-{
-	struct zr36016 *ptr = codec->data;
-
-	if (ptr) {
-		/* do wee need some codec deinit here, too ???? */
-
-		dprintk(1, "%s: finished codec #%d\n", ptr->name,
-			ptr->num);
-		kfree(ptr);
-		codec->data = NULL;
-
-		zr36016_codecs--;
-		return 0;
-	}
-
-	return -EFAULT;
-}
-
-/* =========================================================================
-   Setup and registry function:
-
-   Initializes Zoran's JPEG processor
-
-   Also sets pixel size, average code size, mode (compr./decompr.)
-   (the given size is determined by the processor with the video interface)
-   ========================================================================= */
-
-static int
-zr36016_setup (struct videocodec *codec)
-{
-	struct zr36016 *ptr;
-	int res;
-
-	dprintk(2, "zr36016: initializing VFE subsystem #%d.\n",
-		zr36016_codecs);
-
-	if (zr36016_codecs == MAX_CODECS) {
-		dprintk(1,
-			KERN_ERR "zr36016: Can't attach more codecs!\n");
-		return -ENOSPC;
-	}
-	//mem structure init
-	codec->data = ptr = kzalloc(sizeof(struct zr36016), GFP_KERNEL);
-	if (NULL == ptr) {
-		dprintk(1, KERN_ERR "zr36016: Can't get enough memory!\n");
-		return -ENOMEM;
-	}
-
-	snprintf(ptr->name, sizeof(ptr->name), "zr36016[%d]",
-		 zr36016_codecs);
-	ptr->num = zr36016_codecs++;
-	ptr->codec = codec;
-
-	//testing
-	res = zr36016_basic_test(ptr);
-	if (res < 0) {
-		zr36016_unset(codec);
-		return res;
-	}
-	//final setup
-	ptr->mode = CODEC_DO_COMPRESSION;
-	ptr->width = 768;
-	ptr->height = 288;
-	ptr->xdec = 1;
-	ptr->ydec = 0;
-	zr36016_init(ptr);
-
-	dprintk(1, KERN_INFO "%s: codec v%d attached and running\n",
-		ptr->name, ptr->version);
-
-	return 0;
-}
-
-static const struct videocodec zr36016_codec = {
-	.owner = THIS_MODULE,
-	.name = "zr36016",
-	.magic = 0L,		// magic not used
-	.flags =
-	    CODEC_FLAG_HARDWARE | CODEC_FLAG_VFE | CODEC_FLAG_ENCODER |
-	    CODEC_FLAG_DECODER,
-	.type = CODEC_TYPE_ZR36016,
-	.setup = zr36016_setup,	// functionality
-	.unset = zr36016_unset,
-	.set_mode = zr36016_set_mode,
-	.set_video = zr36016_set_video,
-	.control = zr36016_control,
-	// others are not used
-};
-
-/* =========================================================================
-   HOOK IN DRIVER AS KERNEL MODULE
-   ========================================================================= */
-
-static int __init
-zr36016_init_module (void)
-{
-	//dprintk(1, "ZR36016 driver %s\n",ZR016_VERSION);
-	zr36016_codecs = 0;
-	return videocodec_register(&zr36016_codec);
-}
-
-static void __exit
-zr36016_cleanup_module (void)
-{
-	if (zr36016_codecs) {
-		dprintk(1,
-			"zr36016: something's wrong - %d codecs left somehow.\n",
-			zr36016_codecs);
-	}
-	videocodec_unregister(&zr36016_codec);
-}
-
-module_init(zr36016_init_module);
-module_exit(zr36016_cleanup_module);
-
-MODULE_AUTHOR("Wolfgang Scherr <scherr@net4you.at>");
-MODULE_DESCRIPTION("Driver module for ZR36016 video frontends "
-		   ZR016_VERSION);
-MODULE_LICENSE("GPL");
diff --git a/drivers/staging/media/zoran/zr36016.h b/drivers/staging/media/zoran/zr36016.h
deleted file mode 100644
index 6e66581c27b2..000000000000
--- a/drivers/staging/media/zoran/zr36016.h
+++ /dev/null
@@ -1,91 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0+ */
-/*
- * Zoran ZR36016 basic configuration functions - header file
- *
- * Copyright (C) 2001 Wolfgang Scherr <scherr@net4you.at>
- */
-#ifndef ZR36016_H
-#define ZR36016_H
-
-/* data stored for each zoran jpeg codec chip */
-struct zr36016 {
-	char name[32];
-	int num;
-	/* io datastructure */
-	struct videocodec *codec;
-	// coder status
-	__u8 version;
-	// actual coder setup
-	int mode;
-
-	__u16 xoff;
-	__u16 yoff;
-	__u16 width;
-	__u16 height;
-	__u16 xdec;
-	__u16 ydec;
-};
-
-/* direct  register addresses */
-#define ZR016_GOSTOP      0x00
-#define ZR016_MODE        0x01
-#define ZR016_IADDR       0x02
-#define ZR016_IDATA       0x03
-
-/* indirect  register addresses */
-#define ZR016I_SETUP1     0x00
-#define ZR016I_SETUP2     0x01
-#define ZR016I_NAX_LO     0x02
-#define ZR016I_NAX_HI     0x03
-#define ZR016I_PAX_LO     0x04
-#define ZR016I_PAX_HI     0x05
-#define ZR016I_NAY_LO     0x06
-#define ZR016I_NAY_HI     0x07
-#define ZR016I_PAY_LO     0x08
-#define ZR016I_PAY_HI     0x09
-#define ZR016I_NOL_LO     0x0a
-#define ZR016I_NOL_HI     0x0b
-
-/* possible values for mode register */
-#define ZR016_RGB444_YUV444  0x00
-#define ZR016_RGB444_YUV422  0x01
-#define ZR016_RGB444_YUV411  0x02
-#define ZR016_RGB444_Y400    0x03
-#define ZR016_RGB444_RGB444  0x04
-#define ZR016_YUV444_YUV444  0x08
-#define ZR016_YUV444_YUV422  0x09
-#define ZR016_YUV444_YUV411  0x0a
-#define ZR016_YUV444_Y400    0x0b
-#define ZR016_YUV444_RGB444  0x0c
-#define ZR016_YUV422_YUV422  0x11
-#define ZR016_YUV422_YUV411  0x12
-#define ZR016_YUV422_Y400    0x13
-#define ZR016_YUV411_YUV411  0x16
-#define ZR016_YUV411_Y400    0x17
-#define ZR016_4444_4444      0x19
-#define ZR016_100_100        0x1b
-
-#define ZR016_RGB444         0x00
-#define ZR016_YUV444         0x20
-#define ZR016_YUV422         0x40
-
-#define ZR016_COMPRESSION    0x80
-#define ZR016_EXPANSION      0x80
-
-/* possible values for setup 1 register */
-#define ZR016_CKRT           0x80
-#define ZR016_VERT           0x40
-#define ZR016_HORZ           0x20
-#define ZR016_HRFL           0x10
-#define ZR016_DSFL           0x08
-#define ZR016_SBFL           0x04
-#define ZR016_RSTR           0x02
-#define ZR016_CNTI           0x01
-
-/* possible values for setup 2 register */
-#define ZR016_SYEN           0x40
-#define ZR016_CCIR           0x04
-#define ZR016_SIGN           0x02
-#define ZR016_YMCS           0x01
-
-#endif				/*fndef ZR36016_H */
diff --git a/drivers/staging/media/zoran/zr36050.c b/drivers/staging/media/zoran/zr36050.c
deleted file mode 100644
index cd58307af378..000000000000
--- a/drivers/staging/media/zoran/zr36050.c
+++ /dev/null
@@ -1,880 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0+
-/*
- * Zoran ZR36050 basic configuration functions
- *
- * Copyright (C) 2001 Wolfgang Scherr <scherr@net4you.at>
- */
-#define ZR050_VERSION "v0.7.1"
-
-#include <linux/module.h>
-#include <linux/init.h>
-#include <linux/slab.h>
-#include <linux/delay.h>
-
-#include <linux/types.h>
-#include <linux/wait.h>
-
-/* I/O commands, error codes */
-#include <asm/io.h>
-
-/* headerfile of this module */
-#include "zr36050.h"
-
-/* codec io API */
-#include "videocodec.h"
-
-/* it doesn't make sense to have more than 20 or so,
-  just to prevent some unwanted loops */
-#define MAX_CODECS 20
-
-/* amount of chips attached via this driver */
-static int zr36050_codecs;
-
-/* debugging is available via module parameter */
-static int debug;
-module_param(debug, int, 0);
-MODULE_PARM_DESC(debug, "Debug level (0-4)");
-
-#define dprintk(num, format, args...) \
-	do { \
-		if (debug >= num) \
-			printk(format, ##args); \
-	} while (0)
-
-/* =========================================================================
-   Local hardware I/O functions:
-
-   read/write via codec layer (registers are located in the master device)
-   ========================================================================= */
-
-/* read and write functions */
-static u8
-zr36050_read (struct zr36050 *ptr,
-	      u16             reg)
-{
-	u8 value = 0;
-
-	// just in case something is wrong...
-	if (ptr->codec->master_data->readreg)
-		value = (ptr->codec->master_data->readreg(ptr->codec,
-							  reg)) & 0xFF;
-	else
-		dprintk(1,
-			KERN_ERR "%s: invalid I/O setup, nothing read!\n",
-			ptr->name);
-
-	dprintk(4, "%s: reading from 0x%04x: %02x\n", ptr->name, reg,
-		value);
-
-	return value;
-}
-
-static void
-zr36050_write (struct zr36050 *ptr,
-	       u16             reg,
-	       u8              value)
-{
-	dprintk(4, "%s: writing 0x%02x to 0x%04x\n", ptr->name, value,
-		reg);
-
-	// just in case something is wrong...
-	if (ptr->codec->master_data->writereg)
-		ptr->codec->master_data->writereg(ptr->codec, reg, value);
-	else
-		dprintk(1,
-			KERN_ERR
-			"%s: invalid I/O setup, nothing written!\n",
-			ptr->name);
-}
-
-/* =========================================================================
-   Local helper function:
-
-   status read
-   ========================================================================= */
-
-/* status is kept in datastructure */
-static u8
-zr36050_read_status1 (struct zr36050 *ptr)
-{
-	ptr->status1 = zr36050_read(ptr, ZR050_STATUS_1);
-
-	zr36050_read(ptr, 0);
-	return ptr->status1;
-}
-
-/* =========================================================================
-   Local helper function:
-
-   scale factor read
-   ========================================================================= */
-
-/* scale factor is kept in datastructure */
-static u16
-zr36050_read_scalefactor (struct zr36050 *ptr)
-{
-	ptr->scalefact = (zr36050_read(ptr, ZR050_SF_HI) << 8) |
-			 (zr36050_read(ptr, ZR050_SF_LO) & 0xFF);
-
-	/* leave 0 selected for an eventually GO from master */
-	zr36050_read(ptr, 0);
-	return ptr->scalefact;
-}
-
-/* =========================================================================
-   Local helper function:
-
-   wait if codec is ready to proceed (end of processing) or time is over
-   ========================================================================= */
-
-static void
-zr36050_wait_end (struct zr36050 *ptr)
-{
-	int i = 0;
-
-	while (!(zr36050_read_status1(ptr) & 0x4)) {
-		udelay(1);
-		if (i++ > 200000) {	// 200ms, there is for sure something wrong!!!
-			dprintk(1,
-				"%s: timeout at wait_end (last status: 0x%02x)\n",
-				ptr->name, ptr->status1);
-			break;
-		}
-	}
-}
-
-/* =========================================================================
-   Local helper function:
-
-   basic test of "connectivity", writes/reads to/from memory the SOF marker
-   ========================================================================= */
-
-static int
-zr36050_basic_test (struct zr36050 *ptr)
-{
-	zr36050_write(ptr, ZR050_SOF_IDX, 0x00);
-	zr36050_write(ptr, ZR050_SOF_IDX + 1, 0x00);
-	if ((zr36050_read(ptr, ZR050_SOF_IDX) |
-	     zr36050_read(ptr, ZR050_SOF_IDX + 1)) != 0x0000) {
-		dprintk(1,
-			KERN_ERR
-			"%s: attach failed, can't connect to jpeg processor!\n",
-			ptr->name);
-		return -ENXIO;
-	}
-	zr36050_write(ptr, ZR050_SOF_IDX, 0xff);
-	zr36050_write(ptr, ZR050_SOF_IDX + 1, 0xc0);
-	if (((zr36050_read(ptr, ZR050_SOF_IDX) << 8) |
-	     zr36050_read(ptr, ZR050_SOF_IDX + 1)) != 0xffc0) {
-		dprintk(1,
-			KERN_ERR
-			"%s: attach failed, can't connect to jpeg processor!\n",
-			ptr->name);
-		return -ENXIO;
-	}
-
-	zr36050_wait_end(ptr);
-	if ((ptr->status1 & 0x4) == 0) {
-		dprintk(1,
-			KERN_ERR
-			"%s: attach failed, jpeg processor failed (end flag)!\n",
-			ptr->name);
-		return -EBUSY;
-	}
-
-	return 0;		/* looks good! */
-}
-
-/* =========================================================================
-   Local helper function:
-
-   simple loop for pushing the init datasets
-   ========================================================================= */
-
-static int
-zr36050_pushit (struct zr36050 *ptr,
-		u16             startreg,
-		u16             len,
-		const char     *data)
-{
-	int i = 0;
-
-	dprintk(4, "%s: write data block to 0x%04x (len=%d)\n", ptr->name,
-		startreg, len);
-	while (i < len) {
-		zr36050_write(ptr, startreg++, data[i++]);
-	}
-
-	return i;
-}
-
-/* =========================================================================
-   Basic datasets:
-
-   jpeg baseline setup data (you find it on lots places in internet, or just
-   extract it from any regular .jpg image...)
-
-   Could be variable, but until it's not needed it they are just fixed to save
-   memory. Otherwise expand zr36050 structure with arrays, push the values to
-   it and initialize from there, as e.g. the linux zr36057/60 driver does it.
-   ========================================================================= */
-
-static const char zr36050_dqt[0x86] = {
-	0xff, 0xdb,		//Marker: DQT
-	0x00, 0x84,		//Length: 2*65+2
-	0x00,			//Pq,Tq first table
-	0x10, 0x0b, 0x0c, 0x0e, 0x0c, 0x0a, 0x10, 0x0e,
-	0x0d, 0x0e, 0x12, 0x11, 0x10, 0x13, 0x18, 0x28,
-	0x1a, 0x18, 0x16, 0x16, 0x18, 0x31, 0x23, 0x25,
-	0x1d, 0x28, 0x3a, 0x33, 0x3d, 0x3c, 0x39, 0x33,
-	0x38, 0x37, 0x40, 0x48, 0x5c, 0x4e, 0x40, 0x44,
-	0x57, 0x45, 0x37, 0x38, 0x50, 0x6d, 0x51, 0x57,
-	0x5f, 0x62, 0x67, 0x68, 0x67, 0x3e, 0x4d, 0x71,
-	0x79, 0x70, 0x64, 0x78, 0x5c, 0x65, 0x67, 0x63,
-	0x01,			//Pq,Tq second table
-	0x11, 0x12, 0x12, 0x18, 0x15, 0x18, 0x2f, 0x1a,
-	0x1a, 0x2f, 0x63, 0x42, 0x38, 0x42, 0x63, 0x63,
-	0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
-	0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
-	0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
-	0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
-	0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
-	0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63
-};
-
-static const char zr36050_dht[0x1a4] = {
-	0xff, 0xc4,		//Marker: DHT
-	0x01, 0xa2,		//Length: 2*AC, 2*DC
-	0x00,			//DC first table
-	0x00, 0x01, 0x05, 0x01, 0x01, 0x01, 0x01, 0x01,
-	0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,
-	0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B,
-	0x01,			//DC second table
-	0x00, 0x03, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-	0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,
-	0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B,
-	0x10,			//AC first table
-	0x00, 0x02, 0x01, 0x03, 0x03, 0x02, 0x04, 0x03,
-	0x05, 0x05, 0x04, 0x04, 0x00, 0x00,
-	0x01, 0x7D, 0x01, 0x02, 0x03, 0x00, 0x04, 0x11,
-	0x05, 0x12, 0x21, 0x31, 0x41, 0x06, 0x13, 0x51, 0x61,
-	0x07, 0x22, 0x71, 0x14, 0x32, 0x81, 0x91, 0xA1,
-	0x08, 0x23, 0x42, 0xB1, 0xC1, 0x15, 0x52, 0xD1, 0xF0, 0x24,
-	0x33, 0x62, 0x72, 0x82, 0x09, 0x0A, 0x16, 0x17,
-	0x18, 0x19, 0x1A, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x34,
-	0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x43, 0x44,
-	0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x53, 0x54, 0x55, 0x56,
-	0x57, 0x58, 0x59, 0x5A, 0x63, 0x64, 0x65, 0x66,
-	0x67, 0x68, 0x69, 0x6A, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78,
-	0x79, 0x7A, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88,
-	0x89, 0x8A, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99,
-	0x9A, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8,
-	0xA9, 0xAA, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9,
-	0xBA, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8,
-	0xC9, 0xCA, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8, 0xD9,
-	0xDA, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
-	0xE8, 0xE9, 0xEA, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
-	0xF8, 0xF9, 0xFA,
-	0x11,			//AC second table
-	0x00, 0x02, 0x01, 0x02, 0x04, 0x04, 0x03, 0x04,
-	0x07, 0x05, 0x04, 0x04, 0x00, 0x01,
-	0x02, 0x77, 0x00, 0x01, 0x02, 0x03, 0x11, 0x04,
-	0x05, 0x21, 0x31, 0x06, 0x12, 0x41, 0x51, 0x07, 0x61, 0x71,
-	0x13, 0x22, 0x32, 0x81, 0x08, 0x14, 0x42, 0x91,
-	0xA1, 0xB1, 0xC1, 0x09, 0x23, 0x33, 0x52, 0xF0, 0x15, 0x62,
-	0x72, 0xD1, 0x0A, 0x16, 0x24, 0x34, 0xE1, 0x25,
-	0xF1, 0x17, 0x18, 0x19, 0x1A, 0x26, 0x27, 0x28, 0x29, 0x2A,
-	0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x43, 0x44,
-	0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x53, 0x54, 0x55, 0x56,
-	0x57, 0x58, 0x59, 0x5A, 0x63, 0x64, 0x65, 0x66,
-	0x67, 0x68, 0x69, 0x6A, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78,
-	0x79, 0x7A, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
-	0x88, 0x89, 0x8A, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98,
-	0x99, 0x9A, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7,
-	0xA8, 0xA9, 0xAA, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8,
-	0xB9, 0xBA, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7,
-	0xC8, 0xC9, 0xCA, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8,
-	0xD9, 0xDA, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
-	0xE8, 0xE9, 0xEA, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8,
-	0xF9, 0xFA
-};
-
-/* jpeg baseline setup, this is just fixed in this driver (YUV pictures) */
-#define NO_OF_COMPONENTS          0x3	//Y,U,V
-#define BASELINE_PRECISION        0x8	//MCU size (?)
-static const char zr36050_tq[8] = { 0, 1, 1, 0, 0, 0, 0, 0 };	//table idx's QT
-static const char zr36050_td[8] = { 0, 1, 1, 0, 0, 0, 0, 0 };	//table idx's DC
-static const char zr36050_ta[8] = { 0, 1, 1, 0, 0, 0, 0, 0 };	//table idx's AC
-
-/* horizontal 422 decimation setup (maybe we support 411 or so later, too) */
-static const char zr36050_decimation_h[8] = { 2, 1, 1, 0, 0, 0, 0, 0 };
-static const char zr36050_decimation_v[8] = { 1, 1, 1, 0, 0, 0, 0, 0 };
-
-/* =========================================================================
-   Local helper functions:
-
-   calculation and setup of parameter-dependent JPEG baseline segments
-   (needed for compression only)
-   ========================================================================= */
-
-/* ------------------------------------------------------------------------- */
-
-/* SOF (start of frame) segment depends on width, height and sampling ratio
-			 of each color component */
-
-static int
-zr36050_set_sof (struct zr36050 *ptr)
-{
-	char sof_data[34];	// max. size of register set
-	int i;
-
-	dprintk(3, "%s: write SOF (%dx%d, %d components)\n", ptr->name,
-		ptr->width, ptr->height, NO_OF_COMPONENTS);
-	sof_data[0] = 0xff;
-	sof_data[1] = 0xc0;
-	sof_data[2] = 0x00;
-	sof_data[3] = (3 * NO_OF_COMPONENTS) + 8;
-	sof_data[4] = BASELINE_PRECISION;	// only '8' possible with zr36050
-	sof_data[5] = (ptr->height) >> 8;
-	sof_data[6] = (ptr->height) & 0xff;
-	sof_data[7] = (ptr->width) >> 8;
-	sof_data[8] = (ptr->width) & 0xff;
-	sof_data[9] = NO_OF_COMPONENTS;
-	for (i = 0; i < NO_OF_COMPONENTS; i++) {
-		sof_data[10 + (i * 3)] = i;	// index identifier
-		sof_data[11 + (i * 3)] = (ptr->h_samp_ratio[i] << 4) | (ptr->v_samp_ratio[i]);	// sampling ratios
-		sof_data[12 + (i * 3)] = zr36050_tq[i];	// Q table selection
-	}
-	return zr36050_pushit(ptr, ZR050_SOF_IDX,
-			      (3 * NO_OF_COMPONENTS) + 10, sof_data);
-}
-
-/* ------------------------------------------------------------------------- */
-
-/* SOS (start of scan) segment depends on the used scan components
-			of each color component */
-
-static int
-zr36050_set_sos (struct zr36050 *ptr)
-{
-	char sos_data[16];	// max. size of register set
-	int i;
-
-	dprintk(3, "%s: write SOS\n", ptr->name);
-	sos_data[0] = 0xff;
-	sos_data[1] = 0xda;
-	sos_data[2] = 0x00;
-	sos_data[3] = 2 + 1 + (2 * NO_OF_COMPONENTS) + 3;
-	sos_data[4] = NO_OF_COMPONENTS;
-	for (i = 0; i < NO_OF_COMPONENTS; i++) {
-		sos_data[5 + (i * 2)] = i;	// index
-		sos_data[6 + (i * 2)] = (zr36050_td[i] << 4) | zr36050_ta[i];	// AC/DC tbl.sel.
-	}
-	sos_data[2 + 1 + (2 * NO_OF_COMPONENTS) + 2] = 00;	// scan start
-	sos_data[2 + 1 + (2 * NO_OF_COMPONENTS) + 3] = 0x3F;
-	sos_data[2 + 1 + (2 * NO_OF_COMPONENTS) + 4] = 00;
-	return zr36050_pushit(ptr, ZR050_SOS1_IDX,
-			      4 + 1 + (2 * NO_OF_COMPONENTS) + 3,
-			      sos_data);
-}
-
-/* ------------------------------------------------------------------------- */
-
-/* DRI (define restart interval) */
-
-static int
-zr36050_set_dri (struct zr36050 *ptr)
-{
-	char dri_data[6];	// max. size of register set
-
-	dprintk(3, "%s: write DRI\n", ptr->name);
-	dri_data[0] = 0xff;
-	dri_data[1] = 0xdd;
-	dri_data[2] = 0x00;
-	dri_data[3] = 0x04;
-	dri_data[4] = ptr->dri >> 8;
-	dri_data[5] = ptr->dri & 0xff;
-	return zr36050_pushit(ptr, ZR050_DRI_IDX, 6, dri_data);
-}
-
-/* =========================================================================
-   Setup function:
-
-   Setup compression/decompression of Zoran's JPEG processor
-   ( see also zoran 36050 manual )
-
-   ... sorry for the spaghetti code ...
-   ========================================================================= */
-static void
-zr36050_init (struct zr36050 *ptr)
-{
-	int sum = 0;
-	long bitcnt, tmp;
-
-	if (ptr->mode == CODEC_DO_COMPRESSION) {
-		dprintk(2, "%s: COMPRESSION SETUP\n", ptr->name);
-
-		/* 050 communicates with 057 in master mode */
-		zr36050_write(ptr, ZR050_HARDWARE, ZR050_HW_MSTR);
-
-		/* encoding table preload for compression */
-		zr36050_write(ptr, ZR050_MODE,
-			      ZR050_MO_COMP | ZR050_MO_TLM);
-		zr36050_write(ptr, ZR050_OPTIONS, 0);
-
-		/* disable all IRQs */
-		zr36050_write(ptr, ZR050_INT_REQ_0, 0);
-		zr36050_write(ptr, ZR050_INT_REQ_1, 3);	// low 2 bits always 1
-
-		/* volume control settings */
-		/*zr36050_write(ptr, ZR050_MBCV, ptr->max_block_vol);*/
-		zr36050_write(ptr, ZR050_SF_HI, ptr->scalefact >> 8);
-		zr36050_write(ptr, ZR050_SF_LO, ptr->scalefact & 0xff);
-
-		zr36050_write(ptr, ZR050_AF_HI, 0xff);
-		zr36050_write(ptr, ZR050_AF_M, 0xff);
-		zr36050_write(ptr, ZR050_AF_LO, 0xff);
-
-		/* setup the variable jpeg tables */
-		sum += zr36050_set_sof(ptr);
-		sum += zr36050_set_sos(ptr);
-		sum += zr36050_set_dri(ptr);
-
-		/* setup the fixed jpeg tables - maybe variable, though -
-		 * (see table init section above) */
-		dprintk(3, "%s: write DQT, DHT, APP\n", ptr->name);
-		sum += zr36050_pushit(ptr, ZR050_DQT_IDX,
-				      sizeof(zr36050_dqt), zr36050_dqt);
-		sum += zr36050_pushit(ptr, ZR050_DHT_IDX,
-				      sizeof(zr36050_dht), zr36050_dht);
-		zr36050_write(ptr, ZR050_APP_IDX, 0xff);
-		zr36050_write(ptr, ZR050_APP_IDX + 1, 0xe0 + ptr->app.appn);
-		zr36050_write(ptr, ZR050_APP_IDX + 2, 0x00);
-		zr36050_write(ptr, ZR050_APP_IDX + 3, ptr->app.len + 2);
-		sum += zr36050_pushit(ptr, ZR050_APP_IDX + 4, 60,
-				      ptr->app.data) + 4;
-		zr36050_write(ptr, ZR050_COM_IDX, 0xff);
-		zr36050_write(ptr, ZR050_COM_IDX + 1, 0xfe);
-		zr36050_write(ptr, ZR050_COM_IDX + 2, 0x00);
-		zr36050_write(ptr, ZR050_COM_IDX + 3, ptr->com.len + 2);
-		sum += zr36050_pushit(ptr, ZR050_COM_IDX + 4, 60,
-				      ptr->com.data) + 4;
-
-		/* do the internal huffman table preload */
-		zr36050_write(ptr, ZR050_MARKERS_EN, ZR050_ME_DHTI);
-
-		zr36050_write(ptr, ZR050_GO, 1);	// launch codec
-		zr36050_wait_end(ptr);
-		dprintk(2, "%s: Status after table preload: 0x%02x\n",
-			ptr->name, ptr->status1);
-
-		if ((ptr->status1 & 0x4) == 0) {
-			dprintk(1, KERN_ERR "%s: init aborted!\n",
-				ptr->name);
-			return;	// something is wrong, its timed out!!!!
-		}
-
-		/* setup misc. data for compression (target code sizes) */
-
-		/* size of compressed code to reach without header data */
-		sum = ptr->real_code_vol - sum;
-		bitcnt = sum << 3;	/* need the size in bits */
-
-		tmp = bitcnt >> 16;
-		dprintk(3,
-			"%s: code: csize=%d, tot=%d, bit=%ld, highbits=%ld\n",
-			ptr->name, sum, ptr->real_code_vol, bitcnt, tmp);
-		zr36050_write(ptr, ZR050_TCV_NET_HI, tmp >> 8);
-		zr36050_write(ptr, ZR050_TCV_NET_MH, tmp & 0xff);
-		tmp = bitcnt & 0xffff;
-		zr36050_write(ptr, ZR050_TCV_NET_ML, tmp >> 8);
-		zr36050_write(ptr, ZR050_TCV_NET_LO, tmp & 0xff);
-
-		bitcnt -= bitcnt >> 7;	// bits without stuffing
-		bitcnt -= ((bitcnt * 5) >> 6);	// bits without eob
-
-		tmp = bitcnt >> 16;
-		dprintk(3, "%s: code: nettobit=%ld, highnettobits=%ld\n",
-			ptr->name, bitcnt, tmp);
-		zr36050_write(ptr, ZR050_TCV_DATA_HI, tmp >> 8);
-		zr36050_write(ptr, ZR050_TCV_DATA_MH, tmp & 0xff);
-		tmp = bitcnt & 0xffff;
-		zr36050_write(ptr, ZR050_TCV_DATA_ML, tmp >> 8);
-		zr36050_write(ptr, ZR050_TCV_DATA_LO, tmp & 0xff);
-
-		/* compression setup with or without bitrate control */
-		zr36050_write(ptr, ZR050_MODE,
-			      ZR050_MO_COMP | ZR050_MO_PASS2 |
-			      (ptr->bitrate_ctrl ? ZR050_MO_BRC : 0));
-
-		/* this headers seem to deliver "valid AVI" jpeg frames */
-		zr36050_write(ptr, ZR050_MARKERS_EN,
-			      ZR050_ME_DQT | ZR050_ME_DHT |
-			      ((ptr->app.len > 0) ? ZR050_ME_APP : 0) |
-			      ((ptr->com.len > 0) ? ZR050_ME_COM : 0));
-	} else {
-		dprintk(2, "%s: EXPANSION SETUP\n", ptr->name);
-
-		/* 050 communicates with 055 in master mode */
-		zr36050_write(ptr, ZR050_HARDWARE,
-			      ZR050_HW_MSTR | ZR050_HW_CFIS_2_CLK);
-
-		/* encoding table preload */
-		zr36050_write(ptr, ZR050_MODE, ZR050_MO_TLM);
-
-		/* disable all IRQs */
-		zr36050_write(ptr, ZR050_INT_REQ_0, 0);
-		zr36050_write(ptr, ZR050_INT_REQ_1, 3);	// low 2 bits always 1
-
-		dprintk(3, "%s: write DHT\n", ptr->name);
-		zr36050_pushit(ptr, ZR050_DHT_IDX, sizeof(zr36050_dht),
-			       zr36050_dht);
-
-		/* do the internal huffman table preload */
-		zr36050_write(ptr, ZR050_MARKERS_EN, ZR050_ME_DHTI);
-
-		zr36050_write(ptr, ZR050_GO, 1);	// launch codec
-		zr36050_wait_end(ptr);
-		dprintk(2, "%s: Status after table preload: 0x%02x\n",
-			ptr->name, ptr->status1);
-
-		if ((ptr->status1 & 0x4) == 0) {
-			dprintk(1, KERN_ERR "%s: init aborted!\n",
-				ptr->name);
-			return;	// something is wrong, its timed out!!!!
-		}
-
-		/* setup misc. data for expansion */
-		zr36050_write(ptr, ZR050_MODE, 0);
-		zr36050_write(ptr, ZR050_MARKERS_EN, 0);
-	}
-
-	/* adr on selected, to allow GO from master */
-	zr36050_read(ptr, 0);
-}
-
-/* =========================================================================
-   CODEC API FUNCTIONS
-
-   this functions are accessed by the master via the API structure
-   ========================================================================= */
-
-/* set compression/expansion mode and launches codec -
-   this should be the last call from the master before starting processing */
-static int
-zr36050_set_mode (struct videocodec *codec,
-		  int                mode)
-{
-	struct zr36050 *ptr = (struct zr36050 *) codec->data;
-
-	dprintk(2, "%s: set_mode %d call\n", ptr->name, mode);
-
-	if ((mode != CODEC_DO_EXPANSION) && (mode != CODEC_DO_COMPRESSION))
-		return -EINVAL;
-
-	ptr->mode = mode;
-	zr36050_init(ptr);
-
-	return 0;
-}
-
-/* set picture size (norm is ignored as the codec doesn't know about it) */
-static int
-zr36050_set_video (struct videocodec   *codec,
-		   struct tvnorm       *norm,
-		   struct vfe_settings *cap,
-		   struct vfe_polarity *pol)
-{
-	struct zr36050 *ptr = (struct zr36050 *) codec->data;
-	int size;
-
-	dprintk(2, "%s: set_video %d.%d, %d/%d-%dx%d (0x%x) q%d call\n",
-		ptr->name, norm->HStart, norm->VStart,
-		cap->x, cap->y, cap->width, cap->height,
-		cap->decimation, cap->quality);
-	/* if () return -EINVAL;
-	 * trust the master driver that it knows what it does - so
-	 * we allow invalid startx/y and norm for now ... */
-	ptr->width = cap->width / (cap->decimation & 0xff);
-	ptr->height = cap->height / ((cap->decimation >> 8) & 0xff);
-
-	/* (KM) JPEG quality */
-	size = ptr->width * ptr->height;
-	size *= 16; /* size in bits */
-	/* apply quality setting */
-	size = size * cap->quality / 200;
-
-	/* Minimum: 1kb */
-	if (size < 8192)
-		size = 8192;
-	/* Maximum: 7/8 of code buffer */
-	if (size > ptr->total_code_vol * 7)
-		size = ptr->total_code_vol * 7;
-
-	ptr->real_code_vol = size >> 3; /* in bytes */
-
-	/* Set max_block_vol here (previously in zr36050_init, moved
-	 * here for consistency with zr36060 code */
-	zr36050_write(ptr, ZR050_MBCV, ptr->max_block_vol);
-
-	return 0;
-}
-
-/* additional control functions */
-static int
-zr36050_control (struct videocodec *codec,
-		 int                type,
-		 int                size,
-		 void              *data)
-{
-	struct zr36050 *ptr = (struct zr36050 *) codec->data;
-	int *ival = (int *) data;
-
-	dprintk(2, "%s: control %d call with %d byte\n", ptr->name, type,
-		size);
-
-	switch (type) {
-	case CODEC_G_STATUS:	/* get last status */
-		if (size != sizeof(int))
-			return -EFAULT;
-		zr36050_read_status1(ptr);
-		*ival = ptr->status1;
-		break;
-
-	case CODEC_G_CODEC_MODE:
-		if (size != sizeof(int))
-			return -EFAULT;
-		*ival = CODEC_MODE_BJPG;
-		break;
-
-	case CODEC_S_CODEC_MODE:
-		if (size != sizeof(int))
-			return -EFAULT;
-		if (*ival != CODEC_MODE_BJPG)
-			return -EINVAL;
-		/* not needed, do nothing */
-		return 0;
-
-	case CODEC_G_VFE:
-	case CODEC_S_VFE:
-		/* not needed, do nothing */
-		return 0;
-
-	case CODEC_S_MMAP:
-		/* not available, give an error */
-		return -ENXIO;
-
-	case CODEC_G_JPEG_TDS_BYTE:	/* get target volume in byte */
-		if (size != sizeof(int))
-			return -EFAULT;
-		*ival = ptr->total_code_vol;
-		break;
-
-	case CODEC_S_JPEG_TDS_BYTE:	/* get target volume in byte */
-		if (size != sizeof(int))
-			return -EFAULT;
-		ptr->total_code_vol = *ival;
-		/* (Kieran Morrissey)
-		 * code copied from zr36060.c to ensure proper bitrate */
-		ptr->real_code_vol = (ptr->total_code_vol * 6) >> 3;
-		break;
-
-	case CODEC_G_JPEG_SCALE:	/* get scaling factor */
-		if (size != sizeof(int))
-			return -EFAULT;
-		*ival = zr36050_read_scalefactor(ptr);
-		break;
-
-	case CODEC_S_JPEG_SCALE:	/* set scaling factor */
-		if (size != sizeof(int))
-			return -EFAULT;
-		ptr->scalefact = *ival;
-		break;
-
-	case CODEC_G_JPEG_APP_DATA: {	/* get appn marker data */
-		struct jpeg_app_marker *app = data;
-
-		if (size != sizeof(struct jpeg_app_marker))
-			return -EFAULT;
-
-		*app = ptr->app;
-		break;
-	}
-
-	case CODEC_S_JPEG_APP_DATA: {	 /* set appn marker data */
-		struct jpeg_app_marker *app = data;
-
-		if (size != sizeof(struct jpeg_app_marker))
-			return -EFAULT;
-
-		ptr->app = *app;
-		break;
-	}
-
-	case CODEC_G_JPEG_COM_DATA: {	/* get comment marker data */
-		struct jpeg_com_marker *com = data;
-
-		if (size != sizeof(struct jpeg_com_marker))
-			return -EFAULT;
-
-		*com = ptr->com;
-		break;
-	}
-
-	case CODEC_S_JPEG_COM_DATA: {	/* set comment marker data */
-		struct jpeg_com_marker *com = data;
-
-		if (size != sizeof(struct jpeg_com_marker))
-			return -EFAULT;
-
-		ptr->com = *com;
-		break;
-	}
-
-	default:
-		return -EINVAL;
-	}
-
-	return size;
-}
-
-/* =========================================================================
-   Exit and unregister function:
-
-   Deinitializes Zoran's JPEG processor
-   ========================================================================= */
-
-static int
-zr36050_unset (struct videocodec *codec)
-{
-	struct zr36050 *ptr = codec->data;
-
-	if (ptr) {
-		/* do wee need some codec deinit here, too ???? */
-
-		dprintk(1, "%s: finished codec #%d\n", ptr->name,
-			ptr->num);
-		kfree(ptr);
-		codec->data = NULL;
-
-		zr36050_codecs--;
-		return 0;
-	}
-
-	return -EFAULT;
-}
-
-/* =========================================================================
-   Setup and registry function:
-
-   Initializes Zoran's JPEG processor
-
-   Also sets pixel size, average code size, mode (compr./decompr.)
-   (the given size is determined by the processor with the video interface)
-   ========================================================================= */
-
-static int
-zr36050_setup (struct videocodec *codec)
-{
-	struct zr36050 *ptr;
-	int res;
-
-	dprintk(2, "zr36050: initializing MJPEG subsystem #%d.\n",
-		zr36050_codecs);
-
-	if (zr36050_codecs == MAX_CODECS) {
-		dprintk(1,
-			KERN_ERR "zr36050: Can't attach more codecs!\n");
-		return -ENOSPC;
-	}
-	//mem structure init
-	codec->data = ptr = kzalloc(sizeof(struct zr36050), GFP_KERNEL);
-	if (NULL == ptr) {
-		dprintk(1, KERN_ERR "zr36050: Can't get enough memory!\n");
-		return -ENOMEM;
-	}
-
-	snprintf(ptr->name, sizeof(ptr->name), "zr36050[%d]",
-		 zr36050_codecs);
-	ptr->num = zr36050_codecs++;
-	ptr->codec = codec;
-
-	//testing
-	res = zr36050_basic_test(ptr);
-	if (res < 0) {
-		zr36050_unset(codec);
-		return res;
-	}
-	//final setup
-	memcpy(ptr->h_samp_ratio, zr36050_decimation_h, 8);
-	memcpy(ptr->v_samp_ratio, zr36050_decimation_v, 8);
-
-	ptr->bitrate_ctrl = 0;	/* 0 or 1 - fixed file size flag
-				 * (what is the difference?) */
-	ptr->mode = CODEC_DO_COMPRESSION;
-	ptr->width = 384;
-	ptr->height = 288;
-	ptr->total_code_vol = 16000;
-	ptr->max_block_vol = 240;
-	ptr->scalefact = 0x100;
-	ptr->dri = 1;
-
-	/* no app/com marker by default */
-	ptr->app.appn = 0;
-	ptr->app.len = 0;
-	ptr->com.len = 0;
-
-	zr36050_init(ptr);
-
-	dprintk(1, KERN_INFO "%s: codec attached and running\n",
-		ptr->name);
-
-	return 0;
-}
-
-static const struct videocodec zr36050_codec = {
-	.owner = THIS_MODULE,
-	.name = "zr36050",
-	.magic = 0L,		// magic not used
-	.flags =
-	    CODEC_FLAG_JPEG | CODEC_FLAG_HARDWARE | CODEC_FLAG_ENCODER |
-	    CODEC_FLAG_DECODER,
-	.type = CODEC_TYPE_ZR36050,
-	.setup = zr36050_setup,	// functionality
-	.unset = zr36050_unset,
-	.set_mode = zr36050_set_mode,
-	.set_video = zr36050_set_video,
-	.control = zr36050_control,
-	// others are not used
-};
-
-/* =========================================================================
-   HOOK IN DRIVER AS KERNEL MODULE
-   ========================================================================= */
-
-static int __init
-zr36050_init_module (void)
-{
-	//dprintk(1, "ZR36050 driver %s\n",ZR050_VERSION);
-	zr36050_codecs = 0;
-	return videocodec_register(&zr36050_codec);
-}
-
-static void __exit
-zr36050_cleanup_module (void)
-{
-	if (zr36050_codecs) {
-		dprintk(1,
-			"zr36050: something's wrong - %d codecs left somehow.\n",
-			zr36050_codecs);
-	}
-	videocodec_unregister(&zr36050_codec);
-}
-
-module_init(zr36050_init_module);
-module_exit(zr36050_cleanup_module);
-
-MODULE_AUTHOR("Wolfgang Scherr <scherr@net4you.at>");
-MODULE_DESCRIPTION("Driver module for ZR36050 jpeg processors "
-		   ZR050_VERSION);
-MODULE_LICENSE("GPL");
diff --git a/drivers/staging/media/zoran/zr36050.h b/drivers/staging/media/zoran/zr36050.h
deleted file mode 100644
index c485913dc820..000000000000
--- a/drivers/staging/media/zoran/zr36050.h
+++ /dev/null
@@ -1,163 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0+ */
-/*
- * Zoran ZR36050 basic configuration functions - header file
- *
- * Copyright (C) 2001 Wolfgang Scherr <scherr@net4you.at>
- */
-#ifndef ZR36050_H
-#define ZR36050_H
-
-#include "videocodec.h"
-
-/* data stored for each zoran jpeg codec chip */
-struct zr36050 {
-	char name[32];
-	int num;
-	/* io datastructure */
-	struct videocodec *codec;
-	// last coder status
-	__u8 status1;
-	// actual coder setup
-	int mode;
-
-	__u16 width;
-	__u16 height;
-
-	__u16 bitrate_ctrl;
-
-	__u32 total_code_vol;
-	__u32 real_code_vol;
-	__u16 max_block_vol;
-
-	__u8 h_samp_ratio[8];
-	__u8 v_samp_ratio[8];
-	__u16 scalefact;
-	__u16 dri;
-
-	/* com/app marker */
-	struct jpeg_com_marker com;
-	struct jpeg_app_marker app;
-};
-
-/* zr36050 register addresses */
-#define ZR050_GO                  0x000
-#define ZR050_HARDWARE            0x002
-#define ZR050_MODE                0x003
-#define ZR050_OPTIONS             0x004
-#define ZR050_MBCV                0x005
-#define ZR050_MARKERS_EN          0x006
-#define ZR050_INT_REQ_0           0x007
-#define ZR050_INT_REQ_1           0x008
-#define ZR050_TCV_NET_HI          0x009
-#define ZR050_TCV_NET_MH          0x00a
-#define ZR050_TCV_NET_ML          0x00b
-#define ZR050_TCV_NET_LO          0x00c
-#define ZR050_TCV_DATA_HI         0x00d
-#define ZR050_TCV_DATA_MH         0x00e
-#define ZR050_TCV_DATA_ML         0x00f
-#define ZR050_TCV_DATA_LO         0x010
-#define ZR050_SF_HI               0x011
-#define ZR050_SF_LO               0x012
-#define ZR050_AF_HI               0x013
-#define ZR050_AF_M                0x014
-#define ZR050_AF_LO               0x015
-#define ZR050_ACV_HI              0x016
-#define ZR050_ACV_MH              0x017
-#define ZR050_ACV_ML              0x018
-#define ZR050_ACV_LO              0x019
-#define ZR050_ACT_HI              0x01a
-#define ZR050_ACT_MH              0x01b
-#define ZR050_ACT_ML              0x01c
-#define ZR050_ACT_LO              0x01d
-#define ZR050_ACV_TRUN_HI         0x01e
-#define ZR050_ACV_TRUN_MH         0x01f
-#define ZR050_ACV_TRUN_ML         0x020
-#define ZR050_ACV_TRUN_LO         0x021
-#define ZR050_STATUS_0            0x02e
-#define ZR050_STATUS_1            0x02f
-
-#define ZR050_SOF_IDX             0x040
-#define ZR050_SOS1_IDX            0x07a
-#define ZR050_SOS2_IDX            0x08a
-#define ZR050_SOS3_IDX            0x09a
-#define ZR050_SOS4_IDX            0x0aa
-#define ZR050_DRI_IDX             0x0c0
-#define ZR050_DNL_IDX             0x0c6
-#define ZR050_DQT_IDX             0x0cc
-#define ZR050_DHT_IDX             0x1d4
-#define ZR050_APP_IDX             0x380
-#define ZR050_COM_IDX             0x3c0
-
-/* zr36050 hardware register bits */
-
-#define ZR050_HW_BSWD                0x80
-#define ZR050_HW_MSTR                0x40
-#define ZR050_HW_DMA                 0x20
-#define ZR050_HW_CFIS_1_CLK          0x00
-#define ZR050_HW_CFIS_2_CLK          0x04
-#define ZR050_HW_CFIS_3_CLK          0x08
-#define ZR050_HW_CFIS_4_CLK          0x0C
-#define ZR050_HW_CFIS_5_CLK          0x10
-#define ZR050_HW_CFIS_6_CLK          0x14
-#define ZR050_HW_CFIS_7_CLK          0x18
-#define ZR050_HW_CFIS_8_CLK          0x1C
-#define ZR050_HW_BELE                0x01
-
-/* zr36050 mode register bits */
-
-#define ZR050_MO_COMP                0x80
-#define ZR050_MO_ATP                 0x40
-#define ZR050_MO_PASS2               0x20
-#define ZR050_MO_TLM                 0x10
-#define ZR050_MO_DCONLY              0x08
-#define ZR050_MO_BRC                 0x04
-
-#define ZR050_MO_ATP                 0x40
-#define ZR050_MO_PASS2               0x20
-#define ZR050_MO_TLM                 0x10
-#define ZR050_MO_DCONLY              0x08
-
-/* zr36050 option register bits */
-
-#define ZR050_OP_NSCN_1              0x00
-#define ZR050_OP_NSCN_2              0x20
-#define ZR050_OP_NSCN_3              0x40
-#define ZR050_OP_NSCN_4              0x60
-#define ZR050_OP_NSCN_5              0x80
-#define ZR050_OP_NSCN_6              0xA0
-#define ZR050_OP_NSCN_7              0xC0
-#define ZR050_OP_NSCN_8              0xE0
-#define ZR050_OP_OVF                 0x10
-
-
-/* zr36050 markers-enable register bits */
-
-#define ZR050_ME_APP                 0x80
-#define ZR050_ME_COM                 0x40
-#define ZR050_ME_DRI                 0x20
-#define ZR050_ME_DQT                 0x10
-#define ZR050_ME_DHT                 0x08
-#define ZR050_ME_DNL                 0x04
-#define ZR050_ME_DQTI                0x02
-#define ZR050_ME_DHTI                0x01
-
-/* zr36050 status0/1 register bit masks */
-
-#define ZR050_ST_RST_MASK            0x20
-#define ZR050_ST_SOF_MASK            0x02
-#define ZR050_ST_SOS_MASK            0x02
-#define ZR050_ST_DATRDY_MASK         0x80
-#define ZR050_ST_MRKDET_MASK         0x40
-#define ZR050_ST_RFM_MASK            0x10
-#define ZR050_ST_RFD_MASK            0x08
-#define ZR050_ST_END_MASK            0x04
-#define ZR050_ST_TCVOVF_MASK         0x02
-#define ZR050_ST_DATOVF_MASK         0x01
-
-/* pixel component idx */
-
-#define ZR050_Y_COMPONENT         0
-#define ZR050_U_COMPONENT         1
-#define ZR050_V_COMPONENT         2
-
-#endif				/*fndef ZR36050_H */
diff --git a/drivers/staging/media/zoran/zr36057.h b/drivers/staging/media/zoran/zr36057.h
deleted file mode 100644
index c5138cef4ef5..000000000000
--- a/drivers/staging/media/zoran/zr36057.h
+++ /dev/null
@@ -1,154 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0+ */
-/*
- * zr36057.h - zr36057 register offsets
- *
- * Copyright (C) 1998 Dave Perks <dperks@ibm.net>
- */
-#ifndef _ZR36057_H_
-#define _ZR36057_H_
-
-
-/* Zoran ZR36057 registers */
-
-#define ZR36057_VFEHCR          0x000	/* Video Front End, Horizontal Configuration Register */
-#define ZR36057_VFEHCR_HSPol            (1<<30)
-#define ZR36057_VFEHCR_HStart           10
-#define ZR36057_VFEHCR_HEnd		0
-#define ZR36057_VFEHCR_Hmask		0x3ff
-
-#define ZR36057_VFEVCR          0x004	/* Video Front End, Vertical Configuration Register */
-#define ZR36057_VFEVCR_VSPol            (1<<30)
-#define ZR36057_VFEVCR_VStart           10
-#define ZR36057_VFEVCR_VEnd		0
-#define ZR36057_VFEVCR_Vmask		0x3ff
-
-#define ZR36057_VFESPFR         0x008	/* Video Front End, Scaler and Pixel Format Register */
-#define ZR36057_VFESPFR_ExtFl           (1<<26)
-#define ZR36057_VFESPFR_TopField        (1<<25)
-#define ZR36057_VFESPFR_VCLKPol         (1<<24)
-#define ZR36057_VFESPFR_HFilter         21
-#define ZR36057_VFESPFR_HorDcm          14
-#define ZR36057_VFESPFR_VerDcm          8
-#define ZR36057_VFESPFR_DispMode        6
-#define ZR36057_VFESPFR_YUV422          (0<<3)
-#define ZR36057_VFESPFR_RGB888          (1<<3)
-#define ZR36057_VFESPFR_RGB565          (2<<3)
-#define ZR36057_VFESPFR_RGB555          (3<<3)
-#define ZR36057_VFESPFR_ErrDif          (1<<2)
-#define ZR36057_VFESPFR_Pack24          (1<<1)
-#define ZR36057_VFESPFR_LittleEndian    (1<<0)
-
-#define ZR36057_VDTR            0x00c	/* Video Display "Top" Register */
-
-#define ZR36057_VDBR            0x010	/* Video Display "Bottom" Register */
-
-#define ZR36057_VSSFGR          0x014	/* Video Stride, Status, and Frame Grab Register */
-#define ZR36057_VSSFGR_DispStride       16
-#define ZR36057_VSSFGR_VidOvf           (1<<8)
-#define ZR36057_VSSFGR_SnapShot         (1<<1)
-#define ZR36057_VSSFGR_FrameGrab        (1<<0)
-
-#define ZR36057_VDCR            0x018	/* Video Display Configuration Register */
-#define ZR36057_VDCR_VidEn              (1<<31)
-#define ZR36057_VDCR_MinPix             24
-#define ZR36057_VDCR_Triton             (1<<24)
-#define ZR36057_VDCR_VidWinHt           12
-#define ZR36057_VDCR_VidWinWid          0
-
-#define ZR36057_MMTR            0x01c	/* Masking Map "Top" Register */
-
-#define ZR36057_MMBR            0x020	/* Masking Map "Bottom" Register */
-
-#define ZR36057_OCR             0x024	/* Overlay Control Register */
-#define ZR36057_OCR_OvlEnable           (1 << 15)
-#define ZR36057_OCR_MaskStride          0
-
-#define ZR36057_SPGPPCR         0x028	/* System, PCI, and General Purpose Pins Control Register */
-#define ZR36057_SPGPPCR_SoftReset	(1<<24)
-
-#define ZR36057_GPPGCR1         0x02c	/* General Purpose Pins and GuestBus Control Register (1) */
-
-#define ZR36057_MCSAR           0x030	/* MPEG Code Source Address Register */
-
-#define ZR36057_MCTCR           0x034	/* MPEG Code Transfer Control Register */
-#define ZR36057_MCTCR_CodTime           (1 << 30)
-#define ZR36057_MCTCR_CEmpty            (1 << 29)
-#define ZR36057_MCTCR_CFlush            (1 << 28)
-#define ZR36057_MCTCR_CodGuestID	20
-#define ZR36057_MCTCR_CodGuestReg	16
-
-#define ZR36057_MCMPR           0x038	/* MPEG Code Memory Pointer Register */
-
-#define ZR36057_ISR             0x03c	/* Interrupt Status Register */
-#define ZR36057_ISR_GIRQ1               (1<<30)
-#define ZR36057_ISR_GIRQ0               (1<<29)
-#define ZR36057_ISR_CodRepIRQ           (1<<28)
-#define ZR36057_ISR_JPEGRepIRQ          (1<<27)
-
-#define ZR36057_ICR             0x040	/* Interrupt Control Register */
-#define ZR36057_ICR_GIRQ1               (1<<30)
-#define ZR36057_ICR_GIRQ0               (1<<29)
-#define ZR36057_ICR_CodRepIRQ           (1<<28)
-#define ZR36057_ICR_JPEGRepIRQ          (1<<27)
-#define ZR36057_ICR_IntPinEn            (1<<24)
-
-#define ZR36057_I2CBR           0x044	/* I2C Bus Register */
-#define ZR36057_I2CBR_SDA		(1<<1)
-#define ZR36057_I2CBR_SCL		(1<<0)
-
-#define ZR36057_JMC             0x100	/* JPEG Mode and Control */
-#define ZR36057_JMC_JPG                 (1 << 31)
-#define ZR36057_JMC_JPGExpMode          (0 << 29)
-#define ZR36057_JMC_JPGCmpMode          (1 << 29)
-#define ZR36057_JMC_MJPGExpMode         (2 << 29)
-#define ZR36057_JMC_MJPGCmpMode         (3 << 29)
-#define ZR36057_JMC_RTBUSY_FB           (1 << 6)
-#define ZR36057_JMC_Go_en               (1 << 5)
-#define ZR36057_JMC_SyncMstr            (1 << 4)
-#define ZR36057_JMC_Fld_per_buff        (1 << 3)
-#define ZR36057_JMC_VFIFO_FB            (1 << 2)
-#define ZR36057_JMC_CFIFO_FB            (1 << 1)
-#define ZR36057_JMC_Stll_LitEndian      (1 << 0)
-
-#define ZR36057_JPC             0x104	/* JPEG Process Control */
-#define ZR36057_JPC_P_Reset             (1 << 7)
-#define ZR36057_JPC_CodTrnsEn           (1 << 5)
-#define ZR36057_JPC_Active              (1 << 0)
-
-#define ZR36057_VSP             0x108	/* Vertical Sync Parameters */
-#define ZR36057_VSP_VsyncSize           16
-#define ZR36057_VSP_FrmTot              0
-
-#define ZR36057_HSP             0x10c	/* Horizontal Sync Parameters */
-#define ZR36057_HSP_HsyncStart          16
-#define ZR36057_HSP_LineTot             0
-
-#define ZR36057_FHAP            0x110	/* Field Horizontal Active Portion */
-#define ZR36057_FHAP_NAX                16
-#define ZR36057_FHAP_PAX                0
-
-#define ZR36057_FVAP            0x114	/* Field Vertical Active Portion */
-#define ZR36057_FVAP_NAY                16
-#define ZR36057_FVAP_PAY                0
-
-#define ZR36057_FPP             0x118	/* Field Process Parameters */
-#define ZR36057_FPP_Odd_Even            (1 << 0)
-
-#define ZR36057_JCBA            0x11c	/* JPEG Code Base Address */
-
-#define ZR36057_JCFT            0x120	/* JPEG Code FIFO Threshold */
-
-#define ZR36057_JCGI            0x124	/* JPEG Codec Guest ID */
-#define ZR36057_JCGI_JPEGuestID         4
-#define ZR36057_JCGI_JPEGuestReg        0
-
-#define ZR36057_GCR2            0x12c	/* GuestBus Control Register (2) */
-
-#define ZR36057_POR             0x200	/* Post Office Register */
-#define ZR36057_POR_POPen               (1<<25)
-#define ZR36057_POR_POTime              (1<<24)
-#define ZR36057_POR_PODir               (1<<23)
-
-#define ZR36057_STR             0x300	/* "Still" Transfer Register */
-
-#endif
diff --git a/drivers/staging/media/zoran/zr36060.c b/drivers/staging/media/zoran/zr36060.c
deleted file mode 100644
index a3c817fd5c07..000000000000
--- a/drivers/staging/media/zoran/zr36060.c
+++ /dev/null
@@ -1,994 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0+
-/*
- * Zoran ZR36060 basic configuration functions
- *
- * Copyright (C) 2002 Laurent Pinchart <laurent.pinchart@skynet.be>
- */
-#define ZR060_VERSION "v0.7"
-
-#include <linux/module.h>
-#include <linux/init.h>
-#include <linux/slab.h>
-#include <linux/delay.h>
-
-#include <linux/types.h>
-#include <linux/wait.h>
-
-/* I/O commands, error codes */
-#include <asm/io.h>
-
-/* headerfile of this module */
-#include "zr36060.h"
-
-/* codec io API */
-#include "videocodec.h"
-
-/* it doesn't make sense to have more than 20 or so,
- * just to prevent some unwanted loops
- */
-#define MAX_CODECS 20
-
-/* amount of chips attached via this driver */
-static int zr36060_codecs;
-
-static bool low_bitrate;
-module_param(low_bitrate, bool, 0);
-MODULE_PARM_DESC(low_bitrate, "Buz compatibility option, halves bitrate");
-
-/* debugging is available via module parameter */
-static int debug;
-module_param(debug, int, 0);
-MODULE_PARM_DESC(debug, "Debug level (0-4)");
-
-#define dprintk(num, format, args...) \
-	do { \
-		if (debug >= num) \
-			printk(format, ##args); \
-	} while (0)
-
-/* =========================================================================
-   Local hardware I/O functions:
-
-   read/write via codec layer (registers are located in the master device)
-   ========================================================================= */
-
-/* read and write functions */
-static u8
-zr36060_read (struct zr36060 *ptr,
-	      u16             reg)
-{
-	u8 value = 0;
-
-	// just in case something is wrong...
-	if (ptr->codec->master_data->readreg)
-		value = (ptr->codec->master_data->readreg(ptr->codec,
-							  reg)) & 0xff;
-	else
-		dprintk(1,
-			KERN_ERR "%s: invalid I/O setup, nothing read!\n",
-			ptr->name);
-
-	//dprintk(4, "%s: reading from 0x%04x: %02x\n",ptr->name,reg,value);
-
-	return value;
-}
-
-static void
-zr36060_write(struct zr36060 *ptr,
-	      u16             reg,
-	      u8              value)
-{
-	//dprintk(4, "%s: writing 0x%02x to 0x%04x\n",ptr->name,value,reg);
-	dprintk(4, "0x%02x @0x%04x\n", value, reg);
-
-	// just in case something is wrong...
-	if (ptr->codec->master_data->writereg)
-		ptr->codec->master_data->writereg(ptr->codec, reg, value);
-	else
-		dprintk(1,
-			KERN_ERR
-			"%s: invalid I/O setup, nothing written!\n",
-			ptr->name);
-}
-
-/* =========================================================================
-   Local helper function:
-
-   status read
-   ========================================================================= */
-
-/* status is kept in datastructure */
-static u8
-zr36060_read_status (struct zr36060 *ptr)
-{
-	ptr->status = zr36060_read(ptr, ZR060_CFSR);
-
-	zr36060_read(ptr, 0);
-	return ptr->status;
-}
-
-/* =========================================================================
-   Local helper function:
-
-   scale factor read
-   ========================================================================= */
-
-/* scale factor is kept in datastructure */
-static u16
-zr36060_read_scalefactor (struct zr36060 *ptr)
-{
-	ptr->scalefact = (zr36060_read(ptr, ZR060_SF_HI) << 8) |
-			 (zr36060_read(ptr, ZR060_SF_LO) & 0xFF);
-
-	/* leave 0 selected for an eventually GO from master */
-	zr36060_read(ptr, 0);
-	return ptr->scalefact;
-}
-
-/* =========================================================================
-   Local helper function:
-
-   wait if codec is ready to proceed (end of processing) or time is over
-   ========================================================================= */
-
-static void
-zr36060_wait_end (struct zr36060 *ptr)
-{
-	int i = 0;
-
-	while (zr36060_read_status(ptr) & ZR060_CFSR_Busy) {
-		udelay(1);
-		if (i++ > 200000) {	// 200ms, there is for sure something wrong!!!
-			dprintk(1,
-				"%s: timeout at wait_end (last status: 0x%02x)\n",
-				ptr->name, ptr->status);
-			break;
-		}
-	}
-}
-
-/* =========================================================================
-   Local helper function:
-
-   basic test of "connectivity", writes/reads to/from memory the SOF marker
-   ========================================================================= */
-
-static int
-zr36060_basic_test (struct zr36060 *ptr)
-{
-	if ((zr36060_read(ptr, ZR060_IDR_DEV) != 0x33) &&
-	    (zr36060_read(ptr, ZR060_IDR_REV) != 0x01)) {
-		dprintk(1,
-			KERN_ERR
-			"%s: attach failed, can't connect to jpeg processor!\n",
-			ptr->name);
-		return -ENXIO;
-	}
-
-	zr36060_wait_end(ptr);
-	if (ptr->status & ZR060_CFSR_Busy) {
-		dprintk(1,
-			KERN_ERR
-			"%s: attach failed, jpeg processor failed (end flag)!\n",
-			ptr->name);
-		return -EBUSY;
-	}
-
-	return 0;		/* looks good! */
-}
-
-/* =========================================================================
-   Local helper function:
-
-   simple loop for pushing the init datasets
-   ========================================================================= */
-
-static int
-zr36060_pushit (struct zr36060 *ptr,
-		u16             startreg,
-		u16             len,
-		const char     *data)
-{
-	int i = 0;
-
-	dprintk(4, "%s: write data block to 0x%04x (len=%d)\n", ptr->name,
-		startreg, len);
-	while (i < len) {
-		zr36060_write(ptr, startreg++, data[i++]);
-	}
-
-	return i;
-}
-
-/* =========================================================================
-   Basic datasets:
-
-   jpeg baseline setup data (you find it on lots places in internet, or just
-   extract it from any regular .jpg image...)
-
-   Could be variable, but until it's not needed it they are just fixed to save
-   memory. Otherwise expand zr36060 structure with arrays, push the values to
-   it and initialize from there, as e.g. the linux zr36057/60 driver does it.
-   ========================================================================= */
-
-static const char zr36060_dqt[0x86] = {
-	0xff, 0xdb,		//Marker: DQT
-	0x00, 0x84,		//Length: 2*65+2
-	0x00,			//Pq,Tq first table
-	0x10, 0x0b, 0x0c, 0x0e, 0x0c, 0x0a, 0x10, 0x0e,
-	0x0d, 0x0e, 0x12, 0x11, 0x10, 0x13, 0x18, 0x28,
-	0x1a, 0x18, 0x16, 0x16, 0x18, 0x31, 0x23, 0x25,
-	0x1d, 0x28, 0x3a, 0x33, 0x3d, 0x3c, 0x39, 0x33,
-	0x38, 0x37, 0x40, 0x48, 0x5c, 0x4e, 0x40, 0x44,
-	0x57, 0x45, 0x37, 0x38, 0x50, 0x6d, 0x51, 0x57,
-	0x5f, 0x62, 0x67, 0x68, 0x67, 0x3e, 0x4d, 0x71,
-	0x79, 0x70, 0x64, 0x78, 0x5c, 0x65, 0x67, 0x63,
-	0x01,			//Pq,Tq second table
-	0x11, 0x12, 0x12, 0x18, 0x15, 0x18, 0x2f, 0x1a,
-	0x1a, 0x2f, 0x63, 0x42, 0x38, 0x42, 0x63, 0x63,
-	0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
-	0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
-	0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
-	0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
-	0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63,
-	0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63, 0x63
-};
-
-static const char zr36060_dht[0x1a4] = {
-	0xff, 0xc4,		//Marker: DHT
-	0x01, 0xa2,		//Length: 2*AC, 2*DC
-	0x00,			//DC first table
-	0x00, 0x01, 0x05, 0x01, 0x01, 0x01, 0x01, 0x01,
-	0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,
-	0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B,
-	0x01,			//DC second table
-	0x00, 0x03, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
-	0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,
-	0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B,
-	0x10,			//AC first table
-	0x00, 0x02, 0x01, 0x03, 0x03, 0x02, 0x04, 0x03,
-	0x05, 0x05, 0x04, 0x04, 0x00, 0x00,
-	0x01, 0x7D, 0x01, 0x02, 0x03, 0x00, 0x04, 0x11,
-	0x05, 0x12, 0x21, 0x31, 0x41, 0x06, 0x13, 0x51, 0x61,
-	0x07, 0x22, 0x71, 0x14, 0x32, 0x81, 0x91, 0xA1,
-	0x08, 0x23, 0x42, 0xB1, 0xC1, 0x15, 0x52, 0xD1, 0xF0, 0x24,
-	0x33, 0x62, 0x72, 0x82, 0x09, 0x0A, 0x16, 0x17,
-	0x18, 0x19, 0x1A, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x34,
-	0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x43, 0x44,
-	0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x53, 0x54, 0x55, 0x56,
-	0x57, 0x58, 0x59, 0x5A, 0x63, 0x64, 0x65, 0x66,
-	0x67, 0x68, 0x69, 0x6A, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78,
-	0x79, 0x7A, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88,
-	0x89, 0x8A, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99,
-	0x9A, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8,
-	0xA9, 0xAA, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9,
-	0xBA, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8,
-	0xC9, 0xCA, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8, 0xD9,
-	0xDA, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
-	0xE8, 0xE9, 0xEA, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
-	0xF8, 0xF9, 0xFA,
-	0x11,			//AC second table
-	0x00, 0x02, 0x01, 0x02, 0x04, 0x04, 0x03, 0x04,
-	0x07, 0x05, 0x04, 0x04, 0x00, 0x01,
-	0x02, 0x77, 0x00, 0x01, 0x02, 0x03, 0x11, 0x04,
-	0x05, 0x21, 0x31, 0x06, 0x12, 0x41, 0x51, 0x07, 0x61, 0x71,
-	0x13, 0x22, 0x32, 0x81, 0x08, 0x14, 0x42, 0x91,
-	0xA1, 0xB1, 0xC1, 0x09, 0x23, 0x33, 0x52, 0xF0, 0x15, 0x62,
-	0x72, 0xD1, 0x0A, 0x16, 0x24, 0x34, 0xE1, 0x25,
-	0xF1, 0x17, 0x18, 0x19, 0x1A, 0x26, 0x27, 0x28, 0x29, 0x2A,
-	0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x43, 0x44,
-	0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x53, 0x54, 0x55, 0x56,
-	0x57, 0x58, 0x59, 0x5A, 0x63, 0x64, 0x65, 0x66,
-	0x67, 0x68, 0x69, 0x6A, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78,
-	0x79, 0x7A, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
-	0x88, 0x89, 0x8A, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98,
-	0x99, 0x9A, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7,
-	0xA8, 0xA9, 0xAA, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8,
-	0xB9, 0xBA, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7,
-	0xC8, 0xC9, 0xCA, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8,
-	0xD9, 0xDA, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
-	0xE8, 0xE9, 0xEA, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8,
-	0xF9, 0xFA
-};
-
-/* jpeg baseline setup, this is just fixed in this driver (YUV pictures) */
-#define NO_OF_COMPONENTS          0x3	//Y,U,V
-#define BASELINE_PRECISION        0x8	//MCU size (?)
-static const char zr36060_tq[8] = { 0, 1, 1, 0, 0, 0, 0, 0 };	//table idx's QT
-static const char zr36060_td[8] = { 0, 1, 1, 0, 0, 0, 0, 0 };	//table idx's DC
-static const char zr36060_ta[8] = { 0, 1, 1, 0, 0, 0, 0, 0 };	//table idx's AC
-
-/* horizontal 422 decimation setup (maybe we support 411 or so later, too) */
-static const char zr36060_decimation_h[8] = { 2, 1, 1, 0, 0, 0, 0, 0 };
-static const char zr36060_decimation_v[8] = { 1, 1, 1, 0, 0, 0, 0, 0 };
-
-/* =========================================================================
-   Local helper functions:
-
-   calculation and setup of parameter-dependent JPEG baseline segments
-   (needed for compression only)
-   ========================================================================= */
-
-/* ------------------------------------------------------------------------- */
-
-/* SOF (start of frame) segment depends on width, height and sampling ratio
-			 of each color component */
-
-static int
-zr36060_set_sof (struct zr36060 *ptr)
-{
-	char sof_data[34];	// max. size of register set
-	int i;
-
-	dprintk(3, "%s: write SOF (%dx%d, %d components)\n", ptr->name,
-		ptr->width, ptr->height, NO_OF_COMPONENTS);
-	sof_data[0] = 0xff;
-	sof_data[1] = 0xc0;
-	sof_data[2] = 0x00;
-	sof_data[3] = (3 * NO_OF_COMPONENTS) + 8;
-	sof_data[4] = BASELINE_PRECISION;	// only '8' possible with zr36060
-	sof_data[5] = (ptr->height) >> 8;
-	sof_data[6] = (ptr->height) & 0xff;
-	sof_data[7] = (ptr->width) >> 8;
-	sof_data[8] = (ptr->width) & 0xff;
-	sof_data[9] = NO_OF_COMPONENTS;
-	for (i = 0; i < NO_OF_COMPONENTS; i++) {
-		sof_data[10 + (i * 3)] = i;	// index identifier
-		sof_data[11 + (i * 3)] = (ptr->h_samp_ratio[i] << 4) |
-					 (ptr->v_samp_ratio[i]); // sampling ratios
-		sof_data[12 + (i * 3)] = zr36060_tq[i];	// Q table selection
-	}
-	return zr36060_pushit(ptr, ZR060_SOF_IDX,
-			      (3 * NO_OF_COMPONENTS) + 10, sof_data);
-}
-
-/* ------------------------------------------------------------------------- */
-
-/* SOS (start of scan) segment depends on the used scan components
-			of each color component */
-
-static int
-zr36060_set_sos (struct zr36060 *ptr)
-{
-	char sos_data[16];	// max. size of register set
-	int i;
-
-	dprintk(3, "%s: write SOS\n", ptr->name);
-	sos_data[0] = 0xff;
-	sos_data[1] = 0xda;
-	sos_data[2] = 0x00;
-	sos_data[3] = 2 + 1 + (2 * NO_OF_COMPONENTS) + 3;
-	sos_data[4] = NO_OF_COMPONENTS;
-	for (i = 0; i < NO_OF_COMPONENTS; i++) {
-		sos_data[5 + (i * 2)] = i;	// index
-		sos_data[6 + (i * 2)] = (zr36060_td[i] << 4) |
-					zr36060_ta[i]; // AC/DC tbl.sel.
-	}
-	sos_data[2 + 1 + (2 * NO_OF_COMPONENTS) + 2] = 00;	// scan start
-	sos_data[2 + 1 + (2 * NO_OF_COMPONENTS) + 3] = 0x3f;
-	sos_data[2 + 1 + (2 * NO_OF_COMPONENTS) + 4] = 00;
-	return zr36060_pushit(ptr, ZR060_SOS_IDX,
-			      4 + 1 + (2 * NO_OF_COMPONENTS) + 3,
-			      sos_data);
-}
-
-/* ------------------------------------------------------------------------- */
-
-/* DRI (define restart interval) */
-
-static int
-zr36060_set_dri (struct zr36060 *ptr)
-{
-	char dri_data[6];	// max. size of register set
-
-	dprintk(3, "%s: write DRI\n", ptr->name);
-	dri_data[0] = 0xff;
-	dri_data[1] = 0xdd;
-	dri_data[2] = 0x00;
-	dri_data[3] = 0x04;
-	dri_data[4] = (ptr->dri) >> 8;
-	dri_data[5] = (ptr->dri) & 0xff;
-	return zr36060_pushit(ptr, ZR060_DRI_IDX, 6, dri_data);
-}
-
-/* =========================================================================
-   Setup function:
-
-   Setup compression/decompression of Zoran's JPEG processor
-   ( see also zoran 36060 manual )
-
-   ... sorry for the spaghetti code ...
-   ========================================================================= */
-static void
-zr36060_init (struct zr36060 *ptr)
-{
-	int sum = 0;
-	long bitcnt, tmp;
-
-	if (ptr->mode == CODEC_DO_COMPRESSION) {
-		dprintk(2, "%s: COMPRESSION SETUP\n", ptr->name);
-
-		zr36060_write(ptr, ZR060_LOAD, ZR060_LOAD_SyncRst);
-
-		/* 060 communicates with 067 in master mode */
-		zr36060_write(ptr, ZR060_CIR, ZR060_CIR_CodeMstr);
-
-		/* Compression with or without variable scale factor */
-		/*FIXME: What about ptr->bitrate_ctrl? */
-		zr36060_write(ptr, ZR060_CMR,
-			      ZR060_CMR_Comp | ZR060_CMR_Pass2 |
-			      ZR060_CMR_BRB);
-
-		/* Must be zero */
-		zr36060_write(ptr, ZR060_MBZ, 0x00);
-		zr36060_write(ptr, ZR060_TCR_HI, 0x00);
-		zr36060_write(ptr, ZR060_TCR_LO, 0x00);
-
-		/* Disable all IRQs - no DataErr means autoreset */
-		zr36060_write(ptr, ZR060_IMR, 0);
-
-		/* volume control settings */
-		zr36060_write(ptr, ZR060_SF_HI, ptr->scalefact >> 8);
-		zr36060_write(ptr, ZR060_SF_LO, ptr->scalefact & 0xff);
-
-		zr36060_write(ptr, ZR060_AF_HI, 0xff);
-		zr36060_write(ptr, ZR060_AF_M, 0xff);
-		zr36060_write(ptr, ZR060_AF_LO, 0xff);
-
-		/* setup the variable jpeg tables */
-		sum += zr36060_set_sof(ptr);
-		sum += zr36060_set_sos(ptr);
-		sum += zr36060_set_dri(ptr);
-
-		/* setup the fixed jpeg tables - maybe variable, though -
-		 * (see table init section above) */
-		sum +=
-		    zr36060_pushit(ptr, ZR060_DQT_IDX, sizeof(zr36060_dqt),
-				   zr36060_dqt);
-		sum +=
-		    zr36060_pushit(ptr, ZR060_DHT_IDX, sizeof(zr36060_dht),
-				   zr36060_dht);
-		zr36060_write(ptr, ZR060_APP_IDX, 0xff);
-		zr36060_write(ptr, ZR060_APP_IDX + 1, 0xe0 + ptr->app.appn);
-		zr36060_write(ptr, ZR060_APP_IDX + 2, 0x00);
-		zr36060_write(ptr, ZR060_APP_IDX + 3, ptr->app.len + 2);
-		sum += zr36060_pushit(ptr, ZR060_APP_IDX + 4, 60,
-				      ptr->app.data) + 4;
-		zr36060_write(ptr, ZR060_COM_IDX, 0xff);
-		zr36060_write(ptr, ZR060_COM_IDX + 1, 0xfe);
-		zr36060_write(ptr, ZR060_COM_IDX + 2, 0x00);
-		zr36060_write(ptr, ZR060_COM_IDX + 3, ptr->com.len + 2);
-		sum += zr36060_pushit(ptr, ZR060_COM_IDX + 4, 60,
-				      ptr->com.data) + 4;
-
-		/* setup misc. data for compression (target code sizes) */
-
-		/* size of compressed code to reach without header data */
-		sum = ptr->real_code_vol - sum;
-		bitcnt = sum << 3;	/* need the size in bits */
-
-		tmp = bitcnt >> 16;
-		dprintk(3,
-			"%s: code: csize=%d, tot=%d, bit=%ld, highbits=%ld\n",
-			ptr->name, sum, ptr->real_code_vol, bitcnt, tmp);
-		zr36060_write(ptr, ZR060_TCV_NET_HI, tmp >> 8);
-		zr36060_write(ptr, ZR060_TCV_NET_MH, tmp & 0xff);
-		tmp = bitcnt & 0xffff;
-		zr36060_write(ptr, ZR060_TCV_NET_ML, tmp >> 8);
-		zr36060_write(ptr, ZR060_TCV_NET_LO, tmp & 0xff);
-
-		bitcnt -= bitcnt >> 7;	// bits without stuffing
-		bitcnt -= ((bitcnt * 5) >> 6);	// bits without eob
-
-		tmp = bitcnt >> 16;
-		dprintk(3, "%s: code: nettobit=%ld, highnettobits=%ld\n",
-			ptr->name, bitcnt, tmp);
-		zr36060_write(ptr, ZR060_TCV_DATA_HI, tmp >> 8);
-		zr36060_write(ptr, ZR060_TCV_DATA_MH, tmp & 0xff);
-		tmp = bitcnt & 0xffff;
-		zr36060_write(ptr, ZR060_TCV_DATA_ML, tmp >> 8);
-		zr36060_write(ptr, ZR060_TCV_DATA_LO, tmp & 0xff);
-
-		/* JPEG markers to be included in the compressed stream */
-		zr36060_write(ptr, ZR060_MER,
-			      ZR060_MER_DQT | ZR060_MER_DHT |
-			      ((ptr->com.len > 0) ? ZR060_MER_Com : 0) |
-			      ((ptr->app.len > 0) ? ZR060_MER_App : 0));
-
-		/* Setup the Video Frontend */
-		/* Limit pixel range to 16..235 as per CCIR-601 */
-		zr36060_write(ptr, ZR060_VCR, ZR060_VCR_Range);
-
-	} else {
-		dprintk(2, "%s: EXPANSION SETUP\n", ptr->name);
-
-		zr36060_write(ptr, ZR060_LOAD, ZR060_LOAD_SyncRst);
-
-		/* 060 communicates with 067 in master mode */
-		zr36060_write(ptr, ZR060_CIR, ZR060_CIR_CodeMstr);
-
-		/* Decompression */
-		zr36060_write(ptr, ZR060_CMR, 0);
-
-		/* Must be zero */
-		zr36060_write(ptr, ZR060_MBZ, 0x00);
-		zr36060_write(ptr, ZR060_TCR_HI, 0x00);
-		zr36060_write(ptr, ZR060_TCR_LO, 0x00);
-
-		/* Disable all IRQs - no DataErr means autoreset */
-		zr36060_write(ptr, ZR060_IMR, 0);
-
-		/* setup misc. data for expansion */
-		zr36060_write(ptr, ZR060_MER, 0);
-
-		/* setup the fixed jpeg tables - maybe variable, though -
-		 * (see table init section above) */
-		zr36060_pushit(ptr, ZR060_DHT_IDX, sizeof(zr36060_dht),
-			       zr36060_dht);
-
-		/* Setup the Video Frontend */
-		//zr36060_write(ptr, ZR060_VCR, ZR060_VCR_FIExt);
-		//this doesn't seem right and doesn't work...
-		zr36060_write(ptr, ZR060_VCR, ZR060_VCR_Range);
-	}
-
-	/* Load the tables */
-	zr36060_write(ptr, ZR060_LOAD,
-		      ZR060_LOAD_SyncRst | ZR060_LOAD_Load);
-	zr36060_wait_end(ptr);
-	dprintk(2, "%s: Status after table preload: 0x%02x\n", ptr->name,
-		ptr->status);
-
-	if (ptr->status & ZR060_CFSR_Busy) {
-		dprintk(1, KERN_ERR "%s: init aborted!\n", ptr->name);
-		return;		// something is wrong, its timed out!!!!
-	}
-}
-
-/* =========================================================================
-   CODEC API FUNCTIONS
-
-   this functions are accessed by the master via the API structure
-   ========================================================================= */
-
-/* set compression/expansion mode and launches codec -
-   this should be the last call from the master before starting processing */
-static int
-zr36060_set_mode (struct videocodec *codec,
-		  int                mode)
-{
-	struct zr36060 *ptr = (struct zr36060 *) codec->data;
-
-	dprintk(2, "%s: set_mode %d call\n", ptr->name, mode);
-
-	if ((mode != CODEC_DO_EXPANSION) && (mode != CODEC_DO_COMPRESSION))
-		return -EINVAL;
-
-	ptr->mode = mode;
-	zr36060_init(ptr);
-
-	return 0;
-}
-
-/* set picture size (norm is ignored as the codec doesn't know about it) */
-static int
-zr36060_set_video (struct videocodec   *codec,
-		   struct tvnorm       *norm,
-		   struct vfe_settings *cap,
-		   struct vfe_polarity *pol)
-{
-	struct zr36060 *ptr = (struct zr36060 *) codec->data;
-	u32 reg;
-	int size;
-
-	dprintk(2, "%s: set_video %d/%d-%dx%d (%%%d) call\n", ptr->name,
-		cap->x, cap->y, cap->width, cap->height, cap->decimation);
-
-	/* if () return -EINVAL;
-	 * trust the master driver that it knows what it does - so
-	 * we allow invalid startx/y and norm for now ... */
-	ptr->width = cap->width / (cap->decimation & 0xff);
-	ptr->height = cap->height / (cap->decimation >> 8);
-
-	zr36060_write(ptr, ZR060_LOAD, ZR060_LOAD_SyncRst);
-
-	/* Note that VSPol/HSPol bits in zr36060 have the opposite
-	 * meaning of their zr360x7 counterparts with the same names
-	 * N.b. for VSPol this is only true if FIVEdge = 0 (default,
-	 * left unchanged here - in accordance with datasheet).
-	*/
-	reg = (!pol->vsync_pol ? ZR060_VPR_VSPol : 0)
-	    | (!pol->hsync_pol ? ZR060_VPR_HSPol : 0)
-	    | (pol->field_pol ? ZR060_VPR_FIPol : 0)
-	    | (pol->blank_pol ? ZR060_VPR_BLPol : 0)
-	    | (pol->subimg_pol ? ZR060_VPR_SImgPol : 0)
-	    | (pol->poe_pol ? ZR060_VPR_PoePol : 0)
-	    | (pol->pvalid_pol ? ZR060_VPR_PValPol : 0)
-	    | (pol->vclk_pol ? ZR060_VPR_VCLKPol : 0);
-	zr36060_write(ptr, ZR060_VPR, reg);
-
-	reg = 0;
-	switch (cap->decimation & 0xff) {
-	default:
-	case 1:
-		break;
-
-	case 2:
-		reg |= ZR060_SR_HScale2;
-		break;
-
-	case 4:
-		reg |= ZR060_SR_HScale4;
-		break;
-	}
-
-	switch (cap->decimation >> 8) {
-	default:
-	case 1:
-		break;
-
-	case 2:
-		reg |= ZR060_SR_VScale;
-		break;
-	}
-	zr36060_write(ptr, ZR060_SR, reg);
-
-	zr36060_write(ptr, ZR060_BCR_Y, 0x00);
-	zr36060_write(ptr, ZR060_BCR_U, 0x80);
-	zr36060_write(ptr, ZR060_BCR_V, 0x80);
-
-	/* sync generator */
-
-	reg = norm->Ht - 1;	/* Vtotal */
-	zr36060_write(ptr, ZR060_SGR_VTOTAL_HI, (reg >> 8) & 0xff);
-	zr36060_write(ptr, ZR060_SGR_VTOTAL_LO, (reg >> 0) & 0xff);
-
-	reg = norm->Wt - 1;	/* Htotal */
-	zr36060_write(ptr, ZR060_SGR_HTOTAL_HI, (reg >> 8) & 0xff);
-	zr36060_write(ptr, ZR060_SGR_HTOTAL_LO, (reg >> 0) & 0xff);
-
-	reg = 6 - 1;		/* VsyncSize */
-	zr36060_write(ptr, ZR060_SGR_VSYNC, reg);
-
-	//reg   = 30 - 1;               /* HsyncSize */
-///*CP*/        reg = (zr->params.norm == 1 ? 57 : 68);
-	reg = 68;
-	zr36060_write(ptr, ZR060_SGR_HSYNC, reg);
-
-	reg = norm->VStart - 1;	/* BVstart */
-	zr36060_write(ptr, ZR060_SGR_BVSTART, reg);
-
-	reg += norm->Ha / 2;	/* BVend */
-	zr36060_write(ptr, ZR060_SGR_BVEND_HI, (reg >> 8) & 0xff);
-	zr36060_write(ptr, ZR060_SGR_BVEND_LO, (reg >> 0) & 0xff);
-
-	reg = norm->HStart - 1;	/* BHstart */
-	zr36060_write(ptr, ZR060_SGR_BHSTART, reg);
-
-	reg += norm->Wa;	/* BHend */
-	zr36060_write(ptr, ZR060_SGR_BHEND_HI, (reg >> 8) & 0xff);
-	zr36060_write(ptr, ZR060_SGR_BHEND_LO, (reg >> 0) & 0xff);
-
-	/* active area */
-	reg = cap->y + norm->VStart;	/* Vstart */
-	zr36060_write(ptr, ZR060_AAR_VSTART_HI, (reg >> 8) & 0xff);
-	zr36060_write(ptr, ZR060_AAR_VSTART_LO, (reg >> 0) & 0xff);
-
-	reg += cap->height;	/* Vend */
-	zr36060_write(ptr, ZR060_AAR_VEND_HI, (reg >> 8) & 0xff);
-	zr36060_write(ptr, ZR060_AAR_VEND_LO, (reg >> 0) & 0xff);
-
-	reg = cap->x + norm->HStart;	/* Hstart */
-	zr36060_write(ptr, ZR060_AAR_HSTART_HI, (reg >> 8) & 0xff);
-	zr36060_write(ptr, ZR060_AAR_HSTART_LO, (reg >> 0) & 0xff);
-
-	reg += cap->width;	/* Hend */
-	zr36060_write(ptr, ZR060_AAR_HEND_HI, (reg >> 8) & 0xff);
-	zr36060_write(ptr, ZR060_AAR_HEND_LO, (reg >> 0) & 0xff);
-
-	/* subimage area */
-	reg = norm->VStart - 4;	/* SVstart */
-	zr36060_write(ptr, ZR060_SWR_VSTART_HI, (reg >> 8) & 0xff);
-	zr36060_write(ptr, ZR060_SWR_VSTART_LO, (reg >> 0) & 0xff);
-
-	reg += norm->Ha / 2 + 8;	/* SVend */
-	zr36060_write(ptr, ZR060_SWR_VEND_HI, (reg >> 8) & 0xff);
-	zr36060_write(ptr, ZR060_SWR_VEND_LO, (reg >> 0) & 0xff);
-
-	reg = norm->HStart /*+ 64 */  - 4;	/* SHstart */
-	zr36060_write(ptr, ZR060_SWR_HSTART_HI, (reg >> 8) & 0xff);
-	zr36060_write(ptr, ZR060_SWR_HSTART_LO, (reg >> 0) & 0xff);
-
-	reg += norm->Wa + 8;	/* SHend */
-	zr36060_write(ptr, ZR060_SWR_HEND_HI, (reg >> 8) & 0xff);
-	zr36060_write(ptr, ZR060_SWR_HEND_LO, (reg >> 0) & 0xff);
-
-	size = ptr->width * ptr->height;
-	/* Target compressed field size in bits: */
-	size = size * 16;	/* uncompressed size in bits */
-	/* (Ronald) by default, quality = 100 is a compression
-	 * ratio 1:2. Setting low_bitrate (insmod option) sets
-	 * it to 1:4 (instead of 1:2, zr36060 max) as limit because the
-	 * buz can't handle more at decimation=1... Use low_bitrate if
-	 * you have a Buz, unless you know what you're doing
-	 */
-	size = size * cap->quality / (low_bitrate ? 400 : 200);
-	/* Lower limit (arbitrary, 1 KB) */
-	if (size < 8192)
-		size = 8192;
-	/* Upper limit: 7/8 of the code buffers */
-	if (size > ptr->total_code_vol * 7)
-		size = ptr->total_code_vol * 7;
-
-	ptr->real_code_vol = size >> 3;	/* in bytes */
-
-	/* the MBCVR is the *maximum* block volume, according to the
-	 * JPEG ISO specs, this shouldn't be used, since that allows
-	 * for the best encoding quality. So set it to it's max value
-	 */
-	reg = ptr->max_block_vol;
-	zr36060_write(ptr, ZR060_MBCVR, reg);
-
-	return 0;
-}
-
-/* additional control functions */
-static int
-zr36060_control (struct videocodec *codec,
-		 int                type,
-		 int                size,
-		 void              *data)
-{
-	struct zr36060 *ptr = (struct zr36060 *) codec->data;
-	int *ival = (int *) data;
-
-	dprintk(2, "%s: control %d call with %d byte\n", ptr->name, type,
-		size);
-
-	switch (type) {
-	case CODEC_G_STATUS:	/* get last status */
-		if (size != sizeof(int))
-			return -EFAULT;
-		zr36060_read_status(ptr);
-		*ival = ptr->status;
-		break;
-
-	case CODEC_G_CODEC_MODE:
-		if (size != sizeof(int))
-			return -EFAULT;
-		*ival = CODEC_MODE_BJPG;
-		break;
-
-	case CODEC_S_CODEC_MODE:
-		if (size != sizeof(int))
-			return -EFAULT;
-		if (*ival != CODEC_MODE_BJPG)
-			return -EINVAL;
-		/* not needed, do nothing */
-		return 0;
-
-	case CODEC_G_VFE:
-	case CODEC_S_VFE:
-		/* not needed, do nothing */
-		return 0;
-
-	case CODEC_S_MMAP:
-		/* not available, give an error */
-		return -ENXIO;
-
-	case CODEC_G_JPEG_TDS_BYTE:	/* get target volume in byte */
-		if (size != sizeof(int))
-			return -EFAULT;
-		*ival = ptr->total_code_vol;
-		break;
-
-	case CODEC_S_JPEG_TDS_BYTE:	/* get target volume in byte */
-		if (size != sizeof(int))
-			return -EFAULT;
-		ptr->total_code_vol = *ival;
-		ptr->real_code_vol = (ptr->total_code_vol * 6) >> 3;
-		break;
-
-	case CODEC_G_JPEG_SCALE:	/* get scaling factor */
-		if (size != sizeof(int))
-			return -EFAULT;
-		*ival = zr36060_read_scalefactor(ptr);
-		break;
-
-	case CODEC_S_JPEG_SCALE:	/* set scaling factor */
-		if (size != sizeof(int))
-			return -EFAULT;
-		ptr->scalefact = *ival;
-		break;
-
-	case CODEC_G_JPEG_APP_DATA: {	/* get appn marker data */
-		struct jpeg_app_marker *app = data;
-
-		if (size != sizeof(struct jpeg_app_marker))
-			return -EFAULT;
-
-		*app = ptr->app;
-		break;
-	}
-
-	case CODEC_S_JPEG_APP_DATA: {	/* set appn marker data */
-		struct jpeg_app_marker *app = data;
-
-		if (size != sizeof(struct jpeg_app_marker))
-			return -EFAULT;
-
-		ptr->app = *app;
-		break;
-	}
-
-	case CODEC_G_JPEG_COM_DATA: {	/* get comment marker data */
-		struct jpeg_com_marker *com = data;
-
-		if (size != sizeof(struct jpeg_com_marker))
-			return -EFAULT;
-
-		*com = ptr->com;
-		break;
-	}
-
-	case CODEC_S_JPEG_COM_DATA: {	/* set comment marker data */
-		struct jpeg_com_marker *com = data;
-
-		if (size != sizeof(struct jpeg_com_marker))
-			return -EFAULT;
-
-		ptr->com = *com;
-		break;
-	}
-
-	default:
-		return -EINVAL;
-	}
-
-	return size;
-}
-
-/* =========================================================================
-   Exit and unregister function:
-
-   Deinitializes Zoran's JPEG processor
-   ========================================================================= */
-
-static int
-zr36060_unset (struct videocodec *codec)
-{
-	struct zr36060 *ptr = codec->data;
-
-	if (ptr) {
-		/* do wee need some codec deinit here, too ???? */
-
-		dprintk(1, "%s: finished codec #%d\n", ptr->name,
-			ptr->num);
-		kfree(ptr);
-		codec->data = NULL;
-
-		zr36060_codecs--;
-		return 0;
-	}
-
-	return -EFAULT;
-}
-
-/* =========================================================================
-   Setup and registry function:
-
-   Initializes Zoran's JPEG processor
-
-   Also sets pixel size, average code size, mode (compr./decompr.)
-   (the given size is determined by the processor with the video interface)
-   ========================================================================= */
-
-static int
-zr36060_setup (struct videocodec *codec)
-{
-	struct zr36060 *ptr;
-	int res;
-
-	dprintk(2, "zr36060: initializing MJPEG subsystem #%d.\n",
-		zr36060_codecs);
-
-	if (zr36060_codecs == MAX_CODECS) {
-		dprintk(1,
-			KERN_ERR "zr36060: Can't attach more codecs!\n");
-		return -ENOSPC;
-	}
-	//mem structure init
-	codec->data = ptr = kzalloc(sizeof(struct zr36060), GFP_KERNEL);
-	if (NULL == ptr) {
-		dprintk(1, KERN_ERR "zr36060: Can't get enough memory!\n");
-		return -ENOMEM;
-	}
-
-	snprintf(ptr->name, sizeof(ptr->name), "zr36060[%d]",
-		 zr36060_codecs);
-	ptr->num = zr36060_codecs++;
-	ptr->codec = codec;
-
-	//testing
-	res = zr36060_basic_test(ptr);
-	if (res < 0) {
-		zr36060_unset(codec);
-		return res;
-	}
-	//final setup
-	memcpy(ptr->h_samp_ratio, zr36060_decimation_h, 8);
-	memcpy(ptr->v_samp_ratio, zr36060_decimation_v, 8);
-
-	ptr->bitrate_ctrl = 0;	/* 0 or 1 - fixed file size flag
-				 * (what is the difference?)
-				 */
-	ptr->mode = CODEC_DO_COMPRESSION;
-	ptr->width = 384;
-	ptr->height = 288;
-	ptr->total_code_vol = 16000;	/* CHECKME */
-	ptr->real_code_vol = (ptr->total_code_vol * 6) >> 3;
-	ptr->max_block_vol = 240;	/* CHECKME, was 120 is 240 */
-	ptr->scalefact = 0x100;
-	ptr->dri = 1;		/* CHECKME, was 8 is 1 */
-
-	/* by default, no COM or APP markers - app should set those */
-	ptr->com.len = 0;
-	ptr->app.appn = 0;
-	ptr->app.len = 0;
-
-	zr36060_init(ptr);
-
-	dprintk(1, KERN_INFO "%s: codec attached and running\n",
-		ptr->name);
-
-	return 0;
-}
-
-static const struct videocodec zr36060_codec = {
-	.owner = THIS_MODULE,
-	.name = "zr36060",
-	.magic = 0L,		// magic not used
-	.flags =
-	    CODEC_FLAG_JPEG | CODEC_FLAG_HARDWARE | CODEC_FLAG_ENCODER |
-	    CODEC_FLAG_DECODER | CODEC_FLAG_VFE,
-	.type = CODEC_TYPE_ZR36060,
-	.setup = zr36060_setup,	// functionality
-	.unset = zr36060_unset,
-	.set_mode = zr36060_set_mode,
-	.set_video = zr36060_set_video,
-	.control = zr36060_control,
-	// others are not used
-};
-
-/* =========================================================================
-   HOOK IN DRIVER AS KERNEL MODULE
-   ========================================================================= */
-
-static int __init
-zr36060_init_module (void)
-{
-	//dprintk(1, "zr36060 driver %s\n",ZR060_VERSION);
-	zr36060_codecs = 0;
-	return videocodec_register(&zr36060_codec);
-}
-
-static void __exit
-zr36060_cleanup_module (void)
-{
-	if (zr36060_codecs) {
-		dprintk(1,
-			"zr36060: something's wrong - %d codecs left somehow.\n",
-			zr36060_codecs);
-	}
-
-	/* however, we can't just stay alive */
-	videocodec_unregister(&zr36060_codec);
-}
-
-module_init(zr36060_init_module);
-module_exit(zr36060_cleanup_module);
-
-MODULE_AUTHOR("Laurent Pinchart <laurent.pinchart@skynet.be>");
-MODULE_DESCRIPTION("Driver module for ZR36060 jpeg processors "
-		   ZR060_VERSION);
-MODULE_LICENSE("GPL");
diff --git a/drivers/staging/media/zoran/zr36060.h b/drivers/staging/media/zoran/zr36060.h
deleted file mode 100644
index 9fa553dc475f..000000000000
--- a/drivers/staging/media/zoran/zr36060.h
+++ /dev/null
@@ -1,200 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0+ */
-/*
- * Zoran ZR36060 basic configuration functions - header file
- *
- * Copyright (C) 2002 Laurent Pinchart <laurent.pinchart@skynet.be>
- */
-#ifndef ZR36060_H
-#define ZR36060_H
-
-#include "videocodec.h"
-
-/* data stored for each zoran jpeg codec chip */
-struct zr36060 {
-	char name[32];
-	int num;
-	/* io datastructure */
-	struct videocodec *codec;
-	// last coder status
-	__u8 status;
-	// actual coder setup
-	int mode;
-
-	__u16 width;
-	__u16 height;
-
-	__u16 bitrate_ctrl;
-
-	__u32 total_code_vol;
-	__u32 real_code_vol;
-	__u16 max_block_vol;
-
-	__u8 h_samp_ratio[8];
-	__u8 v_samp_ratio[8];
-	__u16 scalefact;
-	__u16 dri;
-
-	/* app/com marker data */
-	struct jpeg_app_marker app;
-	struct jpeg_com_marker com;
-};
-
-/* ZR36060 register addresses */
-#define ZR060_LOAD			0x000
-#define ZR060_CFSR			0x001
-#define ZR060_CIR			0x002
-#define ZR060_CMR			0x003
-#define ZR060_MBZ			0x004
-#define ZR060_MBCVR			0x005
-#define ZR060_MER			0x006
-#define ZR060_IMR			0x007
-#define ZR060_ISR			0x008
-#define ZR060_TCV_NET_HI		0x009
-#define ZR060_TCV_NET_MH		0x00a
-#define ZR060_TCV_NET_ML		0x00b
-#define ZR060_TCV_NET_LO		0x00c
-#define ZR060_TCV_DATA_HI		0x00d
-#define ZR060_TCV_DATA_MH		0x00e
-#define ZR060_TCV_DATA_ML		0x00f
-#define ZR060_TCV_DATA_LO		0x010
-#define ZR060_SF_HI			0x011
-#define ZR060_SF_LO			0x012
-#define ZR060_AF_HI			0x013
-#define ZR060_AF_M			0x014
-#define ZR060_AF_LO			0x015
-#define ZR060_ACV_HI			0x016
-#define ZR060_ACV_MH			0x017
-#define ZR060_ACV_ML			0x018
-#define ZR060_ACV_LO			0x019
-#define ZR060_ACT_HI			0x01a
-#define ZR060_ACT_MH			0x01b
-#define ZR060_ACT_ML			0x01c
-#define ZR060_ACT_LO			0x01d
-#define ZR060_ACV_TRUN_HI		0x01e
-#define ZR060_ACV_TRUN_MH		0x01f
-#define ZR060_ACV_TRUN_ML		0x020
-#define ZR060_ACV_TRUN_LO		0x021
-#define ZR060_IDR_DEV			0x022
-#define ZR060_IDR_REV			0x023
-#define ZR060_TCR_HI			0x024
-#define ZR060_TCR_LO			0x025
-#define ZR060_VCR			0x030
-#define ZR060_VPR			0x031
-#define ZR060_SR			0x032
-#define ZR060_BCR_Y			0x033
-#define ZR060_BCR_U			0x034
-#define ZR060_BCR_V			0x035
-#define ZR060_SGR_VTOTAL_HI		0x036
-#define ZR060_SGR_VTOTAL_LO		0x037
-#define ZR060_SGR_HTOTAL_HI		0x038
-#define ZR060_SGR_HTOTAL_LO		0x039
-#define ZR060_SGR_VSYNC			0x03a
-#define ZR060_SGR_HSYNC			0x03b
-#define ZR060_SGR_BVSTART		0x03c
-#define ZR060_SGR_BHSTART		0x03d
-#define ZR060_SGR_BVEND_HI		0x03e
-#define ZR060_SGR_BVEND_LO		0x03f
-#define ZR060_SGR_BHEND_HI		0x040
-#define ZR060_SGR_BHEND_LO		0x041
-#define ZR060_AAR_VSTART_HI		0x042
-#define ZR060_AAR_VSTART_LO		0x043
-#define ZR060_AAR_VEND_HI		0x044
-#define ZR060_AAR_VEND_LO		0x045
-#define ZR060_AAR_HSTART_HI		0x046
-#define ZR060_AAR_HSTART_LO		0x047
-#define ZR060_AAR_HEND_HI		0x048
-#define ZR060_AAR_HEND_LO		0x049
-#define ZR060_SWR_VSTART_HI		0x04a
-#define ZR060_SWR_VSTART_LO		0x04b
-#define ZR060_SWR_VEND_HI		0x04c
-#define ZR060_SWR_VEND_LO		0x04d
-#define ZR060_SWR_HSTART_HI		0x04e
-#define ZR060_SWR_HSTART_LO		0x04f
-#define ZR060_SWR_HEND_HI		0x050
-#define ZR060_SWR_HEND_LO		0x051
-
-#define ZR060_SOF_IDX			0x060
-#define ZR060_SOS_IDX			0x07a
-#define ZR060_DRI_IDX			0x0c0
-#define ZR060_DQT_IDX			0x0cc
-#define ZR060_DHT_IDX			0x1d4
-#define ZR060_APP_IDX			0x380
-#define ZR060_COM_IDX			0x3c0
-
-/* ZR36060 LOAD register bits */
-
-#define ZR060_LOAD_Load			(1 << 7)
-#define ZR060_LOAD_SyncRst		(1 << 0)
-
-/* ZR36060 Code FIFO Status register bits */
-
-#define ZR060_CFSR_Busy			(1 << 7)
-#define ZR060_CFSR_CBusy		(1 << 2)
-#define ZR060_CFSR_CFIFO		(3 << 0)
-
-/* ZR36060 Code Interface register */
-
-#define ZR060_CIR_Code16		(1 << 7)
-#define ZR060_CIR_Endian		(1 << 6)
-#define ZR060_CIR_CFIS			(1 << 2)
-#define ZR060_CIR_CodeMstr		(1 << 0)
-
-/* ZR36060 Codec Mode register */
-
-#define ZR060_CMR_Comp			(1 << 7)
-#define ZR060_CMR_ATP			(1 << 6)
-#define ZR060_CMR_Pass2			(1 << 5)
-#define ZR060_CMR_TLM			(1 << 4)
-#define ZR060_CMR_BRB			(1 << 2)
-#define ZR060_CMR_FSF			(1 << 1)
-
-/* ZR36060 Markers Enable register */
-
-#define ZR060_MER_App			(1 << 7)
-#define ZR060_MER_Com			(1 << 6)
-#define ZR060_MER_DRI			(1 << 5)
-#define ZR060_MER_DQT			(1 << 4)
-#define ZR060_MER_DHT			(1 << 3)
-
-/* ZR36060 Interrupt Mask register */
-
-#define ZR060_IMR_EOAV			(1 << 3)
-#define ZR060_IMR_EOI			(1 << 2)
-#define ZR060_IMR_End			(1 << 1)
-#define ZR060_IMR_DataErr		(1 << 0)
-
-/* ZR36060 Interrupt Status register */
-
-#define ZR060_ISR_ProCnt		(3 << 6)
-#define ZR060_ISR_EOAV			(1 << 3)
-#define ZR060_ISR_EOI			(1 << 2)
-#define ZR060_ISR_End			(1 << 1)
-#define ZR060_ISR_DataErr		(1 << 0)
-
-/* ZR36060 Video Control register */
-
-#define ZR060_VCR_Video8		(1 << 7)
-#define ZR060_VCR_Range			(1 << 6)
-#define ZR060_VCR_FIDet			(1 << 3)
-#define ZR060_VCR_FIVedge		(1 << 2)
-#define ZR060_VCR_FIExt			(1 << 1)
-#define ZR060_VCR_SyncMstr		(1 << 0)
-
-/* ZR36060 Video Polarity register */
-
-#define ZR060_VPR_VCLKPol		(1 << 7)
-#define ZR060_VPR_PValPol		(1 << 6)
-#define ZR060_VPR_PoePol		(1 << 5)
-#define ZR060_VPR_SImgPol		(1 << 4)
-#define ZR060_VPR_BLPol			(1 << 3)
-#define ZR060_VPR_FIPol			(1 << 2)
-#define ZR060_VPR_HSPol			(1 << 1)
-#define ZR060_VPR_VSPol			(1 << 0)
-
-/* ZR36060 Scaling register */
-
-#define ZR060_SR_VScale			(1 << 2)
-#define ZR060_SR_HScale2		(1 << 0)
-#define ZR060_SR_HScale4		(2 << 0)
-
-#endif				/*fndef ZR36060_H */
diff --git a/drivers/staging/vboxvideo/TODO b/drivers/staging/vboxvideo/TODO
deleted file mode 100644
index 7f97c47a4042..000000000000
--- a/drivers/staging/vboxvideo/TODO
+++ /dev/null
@@ -1,10 +0,0 @@
-TODO:
--Get a full review from the drm-maintainers on dri-devel done on this driver
--Drop all the logic around initial_mode_queried, the master_set and
- master_drop callbacks and everything related to this. kms clients can handle
- hotplugs.
--Extend this TODO with the results of that review
-
-Please send any patches to Greg Kroah-Hartman <gregkh@linuxfoundation.org>,
-Hans de Goede <hdegoede@redhat.com>, Michael Thayer <michael.thayer@oracle.com>
-and dri-devel@lists.freedesktop.org .
diff --git a/drivers/target/iscsi/iscsi_target.c b/drivers/target/iscsi/iscsi_target.c
index 5ce6e2a40e00..59d32453b891 100644
--- a/drivers/target/iscsi/iscsi_target.c
+++ b/drivers/target/iscsi/iscsi_target.c
@@ -573,7 +573,8 @@ iscsit_xmit_nondatain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 	return 0;
 }
 
-static int iscsit_map_iovec(struct iscsi_cmd *, struct kvec *, u32, u32);
+static int iscsit_map_iovec(struct iscsi_cmd *cmd, struct kvec *iov, int nvec,
+			    u32 data_offset, u32 data_length);
 static void iscsit_unmap_iovec(struct iscsi_cmd *);
 static u32 iscsit_do_crypto_hash_sg(struct ahash_request *, struct iscsi_cmd *,
 				    u32, u32, u32, u8 *);
@@ -604,7 +605,8 @@ iscsit_xmit_datain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 			 *header_digest);
 	}
 
-	iov_ret = iscsit_map_iovec(cmd, &cmd->iov_data[1],
+	iov_ret = iscsit_map_iovec(cmd, &cmd->iov_data[iov_count],
+				   cmd->orig_iov_data_count - (iov_count + 2),
 				   datain->offset, datain->length);
 	if (iov_ret < 0)
 		return -1;
@@ -886,13 +888,10 @@ EXPORT_SYMBOL(iscsit_reject_cmd);
  * Map some portion of the allocated scatterlist to an iovec, suitable for
  * kernel sockets to copy data in/out.
  */
-static int iscsit_map_iovec(
-	struct iscsi_cmd *cmd,
-	struct kvec *iov,
-	u32 data_offset,
-	u32 data_length)
+static int iscsit_map_iovec(struct iscsi_cmd *cmd, struct kvec *iov, int nvec,
+			    u32 data_offset, u32 data_length)
 {
-	u32 i = 0;
+	u32 i = 0, orig_data_length = data_length;
 	struct scatterlist *sg;
 	unsigned int page_off;
 
@@ -901,9 +900,12 @@ static int iscsit_map_iovec(
 	 */
 	u32 ent = data_offset / PAGE_SIZE;
 
+	if (!data_length)
+		return 0;
+
 	if (ent >= cmd->se_cmd.t_data_nents) {
 		pr_err("Initial page entry out-of-bounds\n");
-		return -1;
+		goto overflow;
 	}
 
 	sg = &cmd->se_cmd.t_data_sg[ent];
@@ -913,7 +915,12 @@ static int iscsit_map_iovec(
 	cmd->first_data_sg_off = page_off;
 
 	while (data_length) {
-		u32 cur_len = min_t(u32, data_length, sg->length - page_off);
+		u32 cur_len;
+
+		if (WARN_ON_ONCE(!sg || i >= nvec))
+			goto overflow;
+
+		cur_len = min_t(u32, data_length, sg->length - page_off);
 
 		iov[i].iov_base = kmap(sg_page(sg)) + sg->offset + page_off;
 		iov[i].iov_len = cur_len;
@@ -927,6 +934,16 @@ static int iscsit_map_iovec(
 	cmd->kmapped_nents = i;
 
 	return i;
+
+overflow:
+	pr_err("offset %d + length %d overflow; %d/%d; sg-list:\n",
+	       data_offset, orig_data_length, i, nvec);
+	for_each_sg(cmd->se_cmd.t_data_sg, sg,
+		    cmd->se_cmd.t_data_nents, i) {
+		pr_err("[%d] off %d len %d\n",
+		       i, sg->offset, sg->length);
+	}
+	return -1;
 }
 
 static void iscsit_unmap_iovec(struct iscsi_cmd *cmd)
@@ -1268,27 +1285,27 @@ iscsit_get_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr,
 			  bool dump_payload)
 {
 	int cmdsn_ret = 0, immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION;
+	int rc;
+
 	/*
 	 * Special case for Unsupported SAM WRITE Opcodes and ImmediateData=Yes.
 	 */
-	if (dump_payload)
-		goto after_immediate_data;
-	/*
-	 * Check for underflow case where both EDTL and immediate data payload
-	 * exceeds what is presented by CDB's TRANSFER LENGTH, and what has
-	 * already been set in target_cmd_size_check() as se_cmd->data_length.
-	 *
-	 * For this special case, fail the command and dump the immediate data
-	 * payload.
-	 */
-	if (cmd->first_burst_len > cmd->se_cmd.data_length) {
-		cmd->sense_reason = TCM_INVALID_CDB_FIELD;
-		goto after_immediate_data;
+	if (dump_payload) {
+		u32 length = min(cmd->se_cmd.data_length - cmd->write_data_done,
+				 cmd->first_burst_len);
+
+		pr_debug("Dumping min(%d - %d, %d) = %d bytes of immediate data\n",
+			 cmd->se_cmd.data_length, cmd->write_data_done,
+			 cmd->first_burst_len, length);
+		rc = iscsit_dump_data_payload(cmd->conn, length, 1);
+		pr_debug("Finished dumping immediate data\n");
+		if (rc < 0)
+			immed_ret = IMMEDIATE_DATA_CANNOT_RECOVER;
+	} else {
+		immed_ret = iscsit_handle_immediate_data(cmd, hdr,
+							 cmd->first_burst_len);
 	}
 
-	immed_ret = iscsit_handle_immediate_data(cmd, hdr,
-					cmd->first_burst_len);
-after_immediate_data:
 	if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) {
 		/*
 		 * A PDU/CmdSN carrying Immediate Data passed
@@ -1301,12 +1318,9 @@ after_immediate_data:
 			return -1;
 
 		if (cmd->sense_reason || cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
-			int rc;
-
-			rc = iscsit_dump_data_payload(cmd->conn,
-						      cmd->first_burst_len, 1);
 			target_put_sess_cmd(&cmd->se_cmd);
-			return rc;
+
+			return 0;
 		} else if (cmd->unsolicited_data)
 			iscsit_set_unsolicited_dataout(cmd);
 
@@ -1568,14 +1582,16 @@ iscsit_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 {
 	struct kvec *iov;
 	u32 checksum, iov_count = 0, padding = 0, rx_got = 0, rx_size = 0;
-	u32 payload_length = ntoh24(hdr->dlength);
+	u32 payload_length;
 	int iov_ret, data_crc_failed = 0;
 
+	payload_length = min_t(u32, cmd->se_cmd.data_length,
+			       ntoh24(hdr->dlength));
 	rx_size += payload_length;
 	iov = &cmd->iov_data[0];
 
-	iov_ret = iscsit_map_iovec(cmd, iov, be32_to_cpu(hdr->offset),
-				   payload_length);
+	iov_ret = iscsit_map_iovec(cmd, iov, cmd->orig_iov_data_count - 2,
+				   be32_to_cpu(hdr->offset), payload_length);
 	if (iov_ret < 0)
 		return -1;
 
@@ -1595,6 +1611,7 @@ iscsit_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 		rx_size += ISCSI_CRC_LEN;
 	}
 
+	WARN_ON_ONCE(iov_count > cmd->orig_iov_data_count);
 	rx_got = rx_data(conn, &cmd->iov_data[0], iov_count, rx_size);
 
 	iscsit_unmap_iovec(cmd);
@@ -1860,6 +1877,7 @@ static int iscsit_handle_nop_out(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 			rx_size += ISCSI_CRC_LEN;
 		}
 
+		WARN_ON_ONCE(niov > ARRAY_SIZE(cmd->iov_misc));
 		rx_got = rx_data(conn, &cmd->iov_misc[0], niov, rx_size);
 		if (rx_got != rx_size) {
 			ret = -1;
@@ -2265,6 +2283,7 @@ iscsit_handle_text_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
 			rx_size += ISCSI_CRC_LEN;
 		}
 
+		WARN_ON_ONCE(niov > ARRAY_SIZE(iov));
 		rx_got = rx_data(conn, &iov[0], niov, rx_size);
 		if (rx_got != rx_size)
 			goto reject;
@@ -2575,14 +2594,34 @@ static int iscsit_handle_immediate_data(
 	u32 checksum, iov_count = 0, padding = 0;
 	struct iscsi_conn *conn = cmd->conn;
 	struct kvec *iov;
+	void *overflow_buf = NULL;
 
-	iov_ret = iscsit_map_iovec(cmd, cmd->iov_data, cmd->write_data_done, length);
+	BUG_ON(cmd->write_data_done > cmd->se_cmd.data_length);
+	rx_size = min(cmd->se_cmd.data_length - cmd->write_data_done, length);
+	iov_ret = iscsit_map_iovec(cmd, cmd->iov_data,
+				   cmd->orig_iov_data_count - 2,
+				   cmd->write_data_done, rx_size);
 	if (iov_ret < 0)
 		return IMMEDIATE_DATA_CANNOT_RECOVER;
 
-	rx_size = length;
 	iov_count = iov_ret;
 	iov = &cmd->iov_data[0];
+	if (rx_size < length) {
+		/*
+		 * Special case: length of immediate data exceeds the data
+		 * buffer size derived from the CDB.
+		 */
+		overflow_buf = kmalloc(length - rx_size, GFP_KERNEL);
+		if (!overflow_buf) {
+			iscsit_unmap_iovec(cmd);
+			return IMMEDIATE_DATA_CANNOT_RECOVER;
+		}
+		cmd->overflow_buf = overflow_buf;
+		iov[iov_count].iov_base = overflow_buf;
+		iov[iov_count].iov_len = length - rx_size;
+		iov_count++;
+		rx_size = length;
+	}
 
 	padding = ((-length) & 3);
 	if (padding != 0) {
@@ -2597,6 +2636,7 @@ static int iscsit_handle_immediate_data(
 		rx_size += ISCSI_CRC_LEN;
 	}
 
+	WARN_ON_ONCE(iov_count > cmd->orig_iov_data_count);
 	rx_got = rx_data(conn, &cmd->iov_data[0], iov_count, rx_size);
 
 	iscsit_unmap_iovec(cmd);
@@ -3121,6 +3161,12 @@ int iscsit_build_r2ts_for_cmd(
 				else
 					xfer_len = conn->sess->sess_ops->MaxBurstLength;
 			}
+
+			if ((s32)xfer_len < 0) {
+				cmd->cmd_flags |= ICF_SENT_LAST_R2T;
+				break;
+			}
+
 			cmd->r2t_offset += xfer_len;
 
 			if (cmd->r2t_offset == cmd->se_cmd.data_length)
diff --git a/drivers/target/iscsi/iscsi_target_login.c b/drivers/target/iscsi/iscsi_target_login.c
index ae3209efd0e0..683d04580eb3 100644
--- a/drivers/target/iscsi/iscsi_target_login.c
+++ b/drivers/target/iscsi/iscsi_target_login.c
@@ -883,9 +883,6 @@ int iscsit_setup_np(
 		return -EINVAL;
 	}
 
-	np->np_ip_proto = IPPROTO_TCP;
-	np->np_sock_type = SOCK_STREAM;
-
 	ret = sock_create(sockaddr->ss_family, np->np_sock_type,
 			np->np_ip_proto, &sock);
 	if (ret < 0) {
@@ -1159,13 +1156,13 @@ static struct iscsi_conn *iscsit_alloc_conn(struct iscsi_np *np)
 
 	if (!zalloc_cpumask_var(&conn->conn_cpumask, GFP_KERNEL)) {
 		pr_err("Unable to allocate conn->conn_cpumask\n");
-		goto free_mask;
+		goto free_conn_ops;
 	}
 
 	return conn;
 
-free_mask:
-	free_cpumask_var(conn->conn_cpumask);
+free_conn_ops:
+	kfree(conn->conn_ops);
 put_transport:
 	iscsit_put_transport(conn->conn_transport);
 free_conn:
diff --git a/drivers/target/iscsi/iscsi_target_util.c b/drivers/target/iscsi/iscsi_target_util.c
index 3ac494f63a0b..fae85bfd790e 100644
--- a/drivers/target/iscsi/iscsi_target_util.c
+++ b/drivers/target/iscsi/iscsi_target_util.c
@@ -67,6 +67,8 @@ int iscsit_add_r2t_to_list(
 
 	lockdep_assert_held(&cmd->r2t_lock);
 
+	WARN_ON_ONCE((s32)xfer_len < 0);
+
 	r2t = kmem_cache_zalloc(lio_r2t_cache, GFP_ATOMIC);
 	if (!r2t) {
 		pr_err("Unable to allocate memory for struct iscsi_r2t.\n");
@@ -735,6 +737,7 @@ void iscsit_release_cmd(struct iscsi_cmd *cmd)
 	kfree(cmd->pdu_list);
 	kfree(cmd->seq_list);
 	kfree(cmd->tmr_req);
+	kfree(cmd->overflow_buf);
 	kfree(cmd->iov_data);
 	kfree(cmd->text_in_ptr);
 
@@ -769,6 +772,8 @@ void iscsit_free_cmd(struct iscsi_cmd *cmd, bool shutdown)
 	struct se_cmd *se_cmd = cmd->se_cmd.se_tfo ? &cmd->se_cmd : NULL;
 	int rc;
 
+	WARN_ON(!list_empty(&cmd->i_conn_node));
+
 	__iscsit_free_cmd(cmd, shutdown);
 	if (se_cmd) {
 		rc = transport_generic_free_cmd(se_cmd, shutdown);
diff --git a/drivers/target/target_core_alua.c b/drivers/target/target_core_alua.c
index e09f0cf86bed..893f1fe8e373 100644
--- a/drivers/target/target_core_alua.c
+++ b/drivers/target/target_core_alua.c
@@ -1760,8 +1760,10 @@ void core_alua_free_tg_pt_gp(
 	 * can be made while we are releasing struct t10_alua_tg_pt_gp.
 	 */
 	spin_lock(&dev->t10_alua.tg_pt_gps_lock);
-	list_del(&tg_pt_gp->tg_pt_gp_list);
-	dev->t10_alua.alua_tg_pt_gps_counter--;
+	if (tg_pt_gp->tg_pt_gp_valid_id) {
+		list_del(&tg_pt_gp->tg_pt_gp_list);
+		dev->t10_alua.alua_tg_pt_gps_count--;
+	}
 	spin_unlock(&dev->t10_alua.tg_pt_gps_lock);
 
 	/*
diff --git a/drivers/target/target_core_configfs.c b/drivers/target/target_core_configfs.c
index fc5ef31f5ba8..db2558fe8d46 100644
--- a/drivers/target/target_core_configfs.c
+++ b/drivers/target/target_core_configfs.c
@@ -1227,6 +1227,29 @@ static struct t10_wwn *to_t10_wwn(struct config_item *item)
 	return container_of(to_config_group(item), struct t10_wwn, t10_wwn_group);
 }
 
+static ssize_t target_check_inquiry_data(char *buf)
+{
+	size_t len;
+	int i;
+
+	len = strlen(buf);
+
+	/*
+	 * SPC 4.3.1:
+	 * ASCII data fields shall contain only ASCII printable characters
+	 * (i.e., code values 20h to 7Eh) and may be terminated with one or
+	 * more ASCII null (00h) characters.
+	 */
+	for (i = 0; i < len; i++) {
+		if (buf[i] < 0x20 || buf[i] > 0x7E) {
+			pr_err("Emulated T10 Inquiry Data contains non-ASCII-printable characters\n");
+			return -EINVAL;
+		}
+	}
+
+	return len;
+}
+
 /*
  * STANDARD and VPD page 0x83 T10 Vendor Identification
  */
@@ -1245,7 +1268,7 @@ static ssize_t target_wwn_vendor_id_store(struct config_item *item,
 	unsigned char buf[INQUIRY_VENDOR_LEN + 2];
 	char *stripped = NULL;
 	size_t len;
-	int i;
+	ssize_t ret;
 
 	len = strlcpy(buf, page, sizeof(buf));
 	if (len < sizeof(buf)) {
@@ -1260,19 +1283,10 @@ static ssize_t target_wwn_vendor_id_store(struct config_item *item,
 		return -EOVERFLOW;
 	}
 
-	/*
-	 * SPC 4.3.1:
-	 * ASCII data fields shall contain only ASCII printable characters (i.e.,
-	 * code values 20h to 7Eh) and may be terminated with one or more ASCII
-	 * null (00h) characters.
-	 */
-	for (i = 0; i < len; i++) {
-		if ((stripped[i] < 0x20) || (stripped[i] > 0x7E)) {
-			pr_err("Emulated T10 Vendor Identification contains"
-				" non-ASCII-printable characters\n");
-			return -EINVAL;
-		}
-	}
+	ret = target_check_inquiry_data(stripped);
+
+	if (ret < 0)
+		return ret;
 
 	/*
 	 * Check to see if any active exports exist.  If they do exist, fail
@@ -1295,6 +1309,118 @@ static ssize_t target_wwn_vendor_id_store(struct config_item *item,
 	return count;
 }
 
+static ssize_t target_wwn_product_id_show(struct config_item *item,
+		char *page)
+{
+	return sprintf(page, "%s\n", &to_t10_wwn(item)->model[0]);
+}
+
+static ssize_t target_wwn_product_id_store(struct config_item *item,
+		const char *page, size_t count)
+{
+	struct t10_wwn *t10_wwn = to_t10_wwn(item);
+	struct se_device *dev = t10_wwn->t10_dev;
+	/* +2 to allow for a trailing (stripped) '\n' and null-terminator */
+	unsigned char buf[INQUIRY_MODEL_LEN + 2];
+	char *stripped = NULL;
+	size_t len;
+	ssize_t ret;
+
+	len = strlcpy(buf, page, sizeof(buf));
+	if (len < sizeof(buf)) {
+		/* Strip any newline added from userspace. */
+		stripped = strstrip(buf);
+		len = strlen(stripped);
+	}
+	if (len > INQUIRY_MODEL_LEN) {
+		pr_err("Emulated T10 Vendor exceeds INQUIRY_MODEL_LEN: "
+			 __stringify(INQUIRY_MODEL_LEN)
+			"\n");
+		return -EOVERFLOW;
+	}
+
+	ret = target_check_inquiry_data(stripped);
+
+	if (ret < 0)
+		return ret;
+
+	/*
+	 * Check to see if any active exports exist.  If they do exist, fail
+	 * here as changing this information on the fly (underneath the
+	 * initiator side OS dependent multipath code) could cause negative
+	 * effects.
+	 */
+	if (dev->export_count) {
+		pr_err("Unable to set T10 Model while active %d exports exist\n",
+			dev->export_count);
+		return -EINVAL;
+	}
+
+	BUILD_BUG_ON(sizeof(dev->t10_wwn.model) != INQUIRY_MODEL_LEN + 1);
+	strlcpy(dev->t10_wwn.model, stripped, sizeof(dev->t10_wwn.model));
+
+	pr_debug("Target_Core_ConfigFS: Set emulated T10 Model Identification: %s\n",
+		 dev->t10_wwn.model);
+
+	return count;
+}
+
+static ssize_t target_wwn_revision_show(struct config_item *item,
+		char *page)
+{
+	return sprintf(page, "%s\n", &to_t10_wwn(item)->revision[0]);
+}
+
+static ssize_t target_wwn_revision_store(struct config_item *item,
+		const char *page, size_t count)
+{
+	struct t10_wwn *t10_wwn = to_t10_wwn(item);
+	struct se_device *dev = t10_wwn->t10_dev;
+	/* +2 to allow for a trailing (stripped) '\n' and null-terminator */
+	unsigned char buf[INQUIRY_REVISION_LEN + 2];
+	char *stripped = NULL;
+	size_t len;
+	ssize_t ret;
+
+	len = strlcpy(buf, page, sizeof(buf));
+	if (len < sizeof(buf)) {
+		/* Strip any newline added from userspace. */
+		stripped = strstrip(buf);
+		len = strlen(stripped);
+	}
+	if (len > INQUIRY_REVISION_LEN) {
+		pr_err("Emulated T10 Revision exceeds INQUIRY_REVISION_LEN: "
+			 __stringify(INQUIRY_REVISION_LEN)
+			"\n");
+		return -EOVERFLOW;
+	}
+
+	ret = target_check_inquiry_data(stripped);
+
+	if (ret < 0)
+		return ret;
+
+	/*
+	 * Check to see if any active exports exist.  If they do exist, fail
+	 * here as changing this information on the fly (underneath the
+	 * initiator side OS dependent multipath code) could cause negative
+	 * effects.
+	 */
+	if (dev->export_count) {
+		pr_err("Unable to set T10 Revision while active %d exports exist\n",
+			dev->export_count);
+		return -EINVAL;
+	}
+
+	BUILD_BUG_ON(sizeof(dev->t10_wwn.revision) != INQUIRY_REVISION_LEN + 1);
+	strlcpy(dev->t10_wwn.revision, stripped, sizeof(dev->t10_wwn.revision));
+
+	pr_debug("Target_Core_ConfigFS: Set emulated T10 Revision: %s\n",
+		 dev->t10_wwn.revision);
+
+	return count;
+}
+
 /*
  * VPD page 0x80 Unit serial
  */
@@ -1442,6 +1568,8 @@ DEF_DEV_WWN_ASSOC_SHOW(vpd_assoc_target_port, 0x10);
 DEF_DEV_WWN_ASSOC_SHOW(vpd_assoc_scsi_target_device, 0x20);
 
 CONFIGFS_ATTR(target_wwn_, vendor_id);
+CONFIGFS_ATTR(target_wwn_, product_id);
+CONFIGFS_ATTR(target_wwn_, revision);
 CONFIGFS_ATTR(target_wwn_, vpd_unit_serial);
 CONFIGFS_ATTR_RO(target_wwn_, vpd_protocol_identifier);
 CONFIGFS_ATTR_RO(target_wwn_, vpd_assoc_logical_unit);
@@ -1450,6 +1578,8 @@ CONFIGFS_ATTR_RO(target_wwn_, vpd_assoc_scsi_target_device);
 
 static struct configfs_attribute *target_core_dev_wwn_attrs[] = {
 	&target_wwn_attr_vendor_id,
+	&target_wwn_attr_product_id,
+	&target_wwn_attr_revision,
 	&target_wwn_attr_vpd_unit_serial,
 	&target_wwn_attr_vpd_protocol_identifier,
 	&target_wwn_attr_vpd_assoc_logical_unit,
@@ -1494,11 +1624,12 @@ static ssize_t target_core_dev_pr_show_spc3_res(struct se_device *dev,
 static ssize_t target_core_dev_pr_show_spc2_res(struct se_device *dev,
 		char *page)
 {
+	struct se_session *sess = dev->reservation_holder;
 	struct se_node_acl *se_nacl;
 	ssize_t len;
 
-	se_nacl = dev->dev_reserved_node_acl;
-	if (se_nacl) {
+	if (sess) {
+		se_nacl = sess->se_node_acl;
 		len = sprintf(page,
 			      "SPC-2 Reservation: %s Initiator: %s\n",
 			      se_nacl->se_tpg->se_tpg_tfo->fabric_name,
diff --git a/drivers/target/target_core_device.c b/drivers/target/target_core_device.c
index 1f8482b6473b..7eae1c823c4e 100644
--- a/drivers/target/target_core_device.c
+++ b/drivers/target/target_core_device.c
@@ -85,7 +85,7 @@ transport_lookup_cmd_lun(struct se_cmd *se_cmd, u64 unpacked_lun)
 			goto out_unlock;
 		}
 
-		se_cmd->se_lun = rcu_dereference(deve->se_lun);
+		se_cmd->se_lun = se_lun;
 		se_cmd->pr_res_key = deve->pr_res_key;
 		se_cmd->orig_fe_lun = unpacked_lun;
 		se_cmd->se_cmd_flags |= SCF_SE_LUN_CMD;
@@ -176,7 +176,7 @@ int transport_lookup_tmr_lun(struct se_cmd *se_cmd, u64 unpacked_lun)
 			goto out_unlock;
 		}
 
-		se_cmd->se_lun = rcu_dereference(deve->se_lun);
+		se_cmd->se_lun = se_lun;
 		se_cmd->pr_res_key = deve->pr_res_key;
 		se_cmd->orig_fe_lun = unpacked_lun;
 		se_cmd->se_cmd_flags |= SCF_SE_LUN_CMD;
diff --git a/drivers/target/target_core_pr.c b/drivers/target/target_core_pr.c
index 1597a9ebadca..03767693f580 100644
--- a/drivers/target/target_core_pr.c
+++ b/drivers/target/target_core_pr.c
@@ -111,10 +111,10 @@ target_scsi2_reservation_check(struct se_cmd *cmd)
 		break;
 	}
 
-	if (!dev->dev_reserved_node_acl || !sess)
+	if (!dev->reservation_holder || !sess)
 		return 0;
 
-	if (dev->dev_reserved_node_acl != sess->se_node_acl)
+	if (dev->reservation_holder->se_node_acl != sess->se_node_acl)
 		return TCM_RESERVATION_CONFLICT;
 
 	if (dev->dev_reservation_flags & DRF_SPC2_RESERVATIONS_WITH_ISID) {
@@ -200,6 +200,16 @@ static int target_check_scsi2_reservation_conflict(struct se_cmd *cmd)
 	return 0;
 }
 
+void target_release_reservation(struct se_device *dev)
+{
+	dev->reservation_holder = NULL;
+	dev->dev_reservation_flags &= ~DRF_SPC2_RESERVATIONS;
+	if (dev->dev_reservation_flags & DRF_SPC2_RESERVATIONS_WITH_ISID) {
+		dev->dev_res_bin_isid = 0;
+		dev->dev_reservation_flags &= ~DRF_SPC2_RESERVATIONS_WITH_ISID;
+	}
+}
+
 sense_reason_t
 target_scsi2_reservation_release(struct se_cmd *cmd)
 {
@@ -217,21 +227,16 @@ target_scsi2_reservation_release(struct se_cmd *cmd)
 		return TCM_RESERVATION_CONFLICT;
 
 	spin_lock(&dev->dev_reservation_lock);
-	if (!dev->dev_reserved_node_acl || !sess)
+	if (!dev->reservation_holder || !sess)
 		goto out_unlock;
 
-	if (dev->dev_reserved_node_acl != sess->se_node_acl)
+	if (dev->reservation_holder->se_node_acl != sess->se_node_acl)
 		goto out_unlock;
 
 	if (dev->dev_res_bin_isid != sess->sess_bin_isid)
 		goto out_unlock;
 
-	dev->dev_reserved_node_acl = NULL;
-	dev->dev_reservation_flags &= ~DRF_SPC2_RESERVATIONS;
-	if (dev->dev_reservation_flags & DRF_SPC2_RESERVATIONS_WITH_ISID) {
-		dev->dev_res_bin_isid = 0;
-		dev->dev_reservation_flags &= ~DRF_SPC2_RESERVATIONS_WITH_ISID;
-	}
+	target_release_reservation(dev);
 	tpg = sess->se_tpg;
 	pr_debug("SCSI-2 Released reservation for %s LUN: %llu ->"
 		" MAPPED LUN: %llu for %s\n",
@@ -275,13 +280,13 @@ target_scsi2_reservation_reserve(struct se_cmd *cmd)
 
 	tpg = sess->se_tpg;
 	spin_lock(&dev->dev_reservation_lock);
-	if (dev->dev_reserved_node_acl &&
-	   (dev->dev_reserved_node_acl != sess->se_node_acl)) {
+	if (dev->reservation_holder &&
+	    dev->reservation_holder->se_node_acl != sess->se_node_acl) {
 		pr_err("SCSI-2 RESERVATION CONFLIFT for %s fabric\n",
 			tpg->se_tpg_tfo->fabric_name);
 		pr_err("Original reserver LUN: %llu %s\n",
 			cmd->se_lun->unpacked_lun,
-			dev->dev_reserved_node_acl->initiatorname);
+			dev->reservation_holder->se_node_acl->initiatorname);
 		pr_err("Current attempt - LUN: %llu -> MAPPED LUN: %llu"
 			" from %s \n", cmd->se_lun->unpacked_lun,
 			cmd->orig_fe_lun,
@@ -290,7 +295,7 @@ target_scsi2_reservation_reserve(struct se_cmd *cmd)
 		goto out_unlock;
 	}
 
-	dev->dev_reserved_node_acl = sess->se_node_acl;
+	dev->reservation_holder = sess;
 	dev->dev_reservation_flags |= DRF_SPC2_RESERVATIONS;
 	if (sess->sess_bin_isid != 0) {
 		dev->dev_res_bin_isid = sess->sess_bin_isid;
diff --git a/drivers/target/target_core_pr.h b/drivers/target/target_core_pr.h
index 198fad5c89dc..a31c93e4e19c 100644
--- a/drivers/target/target_core_pr.h
+++ b/drivers/target/target_core_pr.h
@@ -58,6 +58,7 @@ extern struct kmem_cache *t10_pr_reg_cache;
 
 extern void core_pr_dump_initiator_port(struct t10_pr_registration *,
 			char *, u32);
+extern void target_release_reservation(struct se_device *dev);
 extern sense_reason_t target_scsi2_reservation_release(struct se_cmd *);
 extern sense_reason_t target_scsi2_reservation_reserve(struct se_cmd *);
 extern int core_scsi3_alloc_aptpl_registration(
diff --git a/drivers/target/target_core_tmr.c b/drivers/target/target_core_tmr.c
index 3a1bb799a9ab..344df737f3a3 100644
--- a/drivers/target/target_core_tmr.c
+++ b/drivers/target/target_core_tmr.c
@@ -390,7 +390,7 @@ int core_tmr_lun_reset(
 	if (!preempt_and_abort_list &&
 	     (dev->dev_reservation_flags & DRF_SPC2_RESERVATIONS)) {
 		spin_lock(&dev->dev_reservation_lock);
-		dev->dev_reserved_node_acl = NULL;
+		dev->reservation_holder = NULL;
 		dev->dev_reservation_flags &= ~DRF_SPC2_RESERVATIONS;
 		spin_unlock(&dev->dev_reservation_lock);
 		pr_debug("LUN_RESET: SCSI-2 Released reservation\n");
diff --git a/drivers/target/target_core_xcopy.c b/drivers/target/target_core_xcopy.c
index 9be1418e919f..e59a896ac58a 100644
--- a/drivers/target/target_core_xcopy.c
+++ b/drivers/target/target_core_xcopy.c
@@ -389,7 +389,6 @@ out:
  */
 
 struct xcopy_pt_cmd {
-	bool remote_port;
 	struct se_cmd se_cmd;
 	struct completion xpt_passthrough_sem;
 	unsigned char sense_buffer[TRANSPORT_SENSE_BUFFER];
@@ -506,72 +505,20 @@ void target_xcopy_release_pt(void)
 		destroy_workqueue(xcopy_wq);
 }
 
-static void target_xcopy_setup_pt_port(
-	struct xcopy_pt_cmd *xpt_cmd,
-	struct xcopy_op *xop,
-	bool remote_port)
-{
-	struct se_cmd *ec_cmd = xop->xop_se_cmd;
-	struct se_cmd *pt_cmd = &xpt_cmd->se_cmd;
-
-	if (xop->op_origin == XCOL_SOURCE_RECV_OP) {
-		/*
-		 * Honor destination port reservations for X-COPY PUSH emulation
-		 * when CDB is received on local source port, and READs blocks to
-		 * WRITE on remote destination port.
-		 */
-		if (remote_port) {
-			xpt_cmd->remote_port = remote_port;
-		} else {
-			pt_cmd->se_lun = ec_cmd->se_lun;
-			pt_cmd->se_dev = ec_cmd->se_dev;
-
-			pr_debug("Honoring local SRC port from ec_cmd->se_dev:"
-				" %p\n", pt_cmd->se_dev);
-			pt_cmd->se_lun = ec_cmd->se_lun;
-			pr_debug("Honoring local SRC port from ec_cmd->se_lun: %p\n",
-				pt_cmd->se_lun);
-		}
-	} else {
-		/*
-		 * Honor source port reservation for X-COPY PULL emulation
-		 * when CDB is received on local desintation port, and READs
-		 * blocks from the remote source port to WRITE on local
-		 * destination port.
-		 */
-		if (remote_port) {
-			xpt_cmd->remote_port = remote_port;
-		} else {
-			pt_cmd->se_lun = ec_cmd->se_lun;
-			pt_cmd->se_dev = ec_cmd->se_dev;
-
-			pr_debug("Honoring local DST port from ec_cmd->se_dev:"
-				" %p\n", pt_cmd->se_dev);
-			pt_cmd->se_lun = ec_cmd->se_lun;
-			pr_debug("Honoring local DST port from ec_cmd->se_lun: %p\n",
-				pt_cmd->se_lun);
-		}
-	}
-}
-
-static void target_xcopy_init_pt_lun(struct se_device *se_dev,
-		struct se_cmd *pt_cmd, bool remote_port)
-{
-	/*
-	 * Don't allocate + init an pt_cmd->se_lun if honoring local port for
-	 * reservations.  The pt_cmd->se_lun pointer will be setup from within
-	 * target_xcopy_setup_pt_port()
-	 */
-	if (remote_port) {
-		pr_debug("Setup emulated se_dev: %p from se_dev\n",
-			pt_cmd->se_dev);
-		pt_cmd->se_lun = &se_dev->xcopy_lun;
-		pt_cmd->se_dev = se_dev;
-	}
-
-	pt_cmd->se_cmd_flags |= SCF_SE_LUN_CMD;
-}
-
+/*
+ * target_xcopy_setup_pt_cmd - set up a pass-through command
+ * @xpt_cmd:	 Data structure to initialize.
+ * @xop:	 Describes the XCOPY operation received from an initiator.
+ * @se_dev:	 Backend device to associate with @xpt_cmd if
+ *		 @remote_port == true.
+ * @cdb:	 SCSI CDB to be copied into @xpt_cmd.
+ * @remote_port: If false, use the LUN through which the XCOPY command has
+ *		 been received. If true, use @se_dev->xcopy_lun.
+ * @alloc_mem:	 Whether or not to allocate an SGL list.
+ *
+ * Set up a SCSI command (READ or WRITE) that will be used to execute an
+ * XCOPY command.
+ */
 static int target_xcopy_setup_pt_cmd(
 	struct xcopy_pt_cmd *xpt_cmd,
 	struct xcopy_op *xop,
@@ -583,12 +530,19 @@ static int target_xcopy_setup_pt_cmd(
 	struct se_cmd *cmd = &xpt_cmd->se_cmd;
 	sense_reason_t sense_rc;
 	int ret = 0, rc;
+
 	/*
 	 * Setup LUN+port to honor reservations based upon xop->op_origin for
 	 * X-COPY PUSH or X-COPY PULL based upon where the CDB was received.
 	 */
-	target_xcopy_init_pt_lun(se_dev, cmd, remote_port);
-	target_xcopy_setup_pt_port(xpt_cmd, xop, remote_port);
+	if (remote_port) {
+		cmd->se_lun = &se_dev->xcopy_lun;
+		cmd->se_dev = se_dev;
+	} else {
+		cmd->se_lun = xop->xop_se_cmd->se_lun;
+		cmd->se_dev = xop->xop_se_cmd->se_dev;
+	}
+	cmd->se_cmd_flags |= SCF_SE_LUN_CMD;
 
 	cmd->tag = 0;
 	sense_rc = target_setup_cmd_from_cdb(cmd, cdb);
diff --git a/drivers/tty/Kconfig b/drivers/tty/Kconfig
index e0a04bfc873e..3b1d312bb175 100644
--- a/drivers/tty/Kconfig
+++ b/drivers/tty/Kconfig
@@ -1,3 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
 config TTY
 	bool "Enable TTY" if EXPERT
 	default y
@@ -83,7 +84,6 @@ config HW_CONSOLE
 config VT_HW_CONSOLE_BINDING
        bool "Support for binding and unbinding console drivers"
        depends on HW_CONSOLE
-       default n
        ---help---
          The virtual terminal is the device that interacts with the physical
          terminal through console drivers. On these systems, at least one
@@ -175,7 +175,7 @@ config ROCKETPORT
 	  This driver supports Comtrol RocketPort and RocketModem PCI boards.   
           These boards provide 2, 4, 8, 16, or 32 high-speed serial ports or
           modems.  For information about the RocketPort/RocketModem  boards
-          and this driver read <file:Documentation/serial/rocket.txt>.
+          and this driver read <file:Documentation/serial/rocket.rst>.
 
 	  To compile this driver as a module, choose M here: the
 	  module will be called rocket.
@@ -193,7 +193,7 @@ config CYCLADES
 	  your Linux box, for instance in order to become a dial-in server.
 
 	  For information about the Cyclades-Z card, read
-	  <file:Documentation/serial/README.cycladesZ>.
+	  <file:Documentation/serial/cyclades_z.rst>.
 
 	  To compile this driver as a module, choose M here: the
 	  module will be called cyclades.
@@ -312,7 +312,6 @@ config N_GSM
 config TRACE_ROUTER
 	tristate "Trace data router for MIPI P1149.7 cJTAG standard"
 	depends on TRACE_SINK
-	default n
 	help
 	  The trace router uses the Linux tty line discipline framework to
 	  route trace data coming from a tty port (say UART for example) to
@@ -328,7 +327,6 @@ config TRACE_ROUTER
 
 config TRACE_SINK
 	tristate "Trace data sink for MIPI P1149.7 cJTAG standard"
-	default n
 	help
 	  The trace sink uses the Linux line discipline framework to receive
 	  trace data coming from the trace router line discipline driver
@@ -376,6 +374,20 @@ config PPC_EARLY_DEBUG_EHV_BC_HANDLE
 	  there simply will be no early console output.  This is true also
 	  if you don't boot under a hypervisor at all.
 
+config NULL_TTY
+	tristate "NULL TTY driver"
+	help
+	  Say Y here if you want a NULL TTY which simply discards messages.
+
+	  This is useful to allow userspace applications which expect a console
+	  device to work without modifications even when no console is
+	  available or desired.
+
+	  In order to use this driver, you should redirect the console to this
+	  TTY, or boot the kernel with console=ttynull.
+
+	  If unsure, say N.
+
 config GOLDFISH_TTY
 	tristate "Goldfish TTY Driver"
 	depends on GOLDFISH
diff --git a/drivers/tty/Makefile b/drivers/tty/Makefile
index c72cafdf32b4..020b1cd9294f 100644
--- a/drivers/tty/Makefile
+++ b/drivers/tty/Makefile
@@ -25,6 +25,7 @@ obj-$(CONFIG_ISI)		+= isicom.o
 obj-$(CONFIG_MOXA_INTELLIO)	+= moxa.o
 obj-$(CONFIG_MOXA_SMARTIO)	+= mxser.o
 obj-$(CONFIG_NOZOMI)		+= nozomi.o
+obj-$(CONFIG_NULL_TTY)	        += ttynull.o
 obj-$(CONFIG_ROCKETPORT)	+= rocket.o
 obj-$(CONFIG_SYNCLINK_GT)	+= synclink_gt.o
 obj-$(CONFIG_SYNCLINKMP)	+= synclinkmp.o
diff --git a/drivers/tty/hvc/Kconfig b/drivers/tty/hvc/Kconfig
index 4293c172e120..4d22b911111f 100644
--- a/drivers/tty/hvc/Kconfig
+++ b/drivers/tty/hvc/Kconfig
@@ -1,3 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
 if TTY
 
 config HVC_DRIVER
@@ -24,7 +25,6 @@ config HVC_CONSOLE
 config HVC_OLD_HVSI
 	bool "Old driver for pSeries serial port (/dev/hvsi*)"
 	depends on HVC_CONSOLE
-	default n
 
 config HVC_OPAL
 	bool "OPAL Console support"
@@ -73,7 +73,6 @@ config HVC_UDBG
        bool "udbg based fake hypervisor console"
        depends on PPC
        select HVC_DRIVER
-       default n
        help
          This is meant to be used during HW bring up or debugging when
 	 no other console mechanism exist but udbg, to get you a quick
diff --git a/drivers/tty/ipwireless/Makefile b/drivers/tty/ipwireless/Makefile
index fe2e1730986b..a665d021e24d 100644
--- a/drivers/tty/ipwireless/Makefile
+++ b/drivers/tty/ipwireless/Makefile
@@ -1,3 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
 #
 # Makefile for the IPWireless driver
 #
diff --git a/drivers/tty/ipwireless/main.c b/drivers/tty/ipwireless/main.c
index 3475e841ef5c..4c18bbfe1a92 100644
--- a/drivers/tty/ipwireless/main.c
+++ b/drivers/tty/ipwireless/main.c
@@ -114,6 +114,10 @@ static int ipwireless_probe(struct pcmcia_device *p_dev, void *priv_data)
 
 	ipw->common_memory = ioremap(p_dev->resource[2]->start,
 				resource_size(p_dev->resource[2]));
+	if (!ipw->common_memory) {
+		ret = -ENOMEM;
+		goto exit1;
+	}
 	if (!request_mem_region(p_dev->resource[2]->start,
 				resource_size(p_dev->resource[2]),
 				IPWIRELESS_PCCARD_NAME)) {
@@ -134,6 +138,10 @@ static int ipwireless_probe(struct pcmcia_device *p_dev, void *priv_data)
 
 	ipw->attr_memory = ioremap(p_dev->resource[3]->start,
 				resource_size(p_dev->resource[3]));
+	if (!ipw->attr_memory) {
+		ret = -ENOMEM;
+		goto exit3;
+	}
 	if (!request_mem_region(p_dev->resource[3]->start,
 				resource_size(p_dev->resource[3]),
 				IPWIRELESS_PCCARD_NAME)) {
diff --git a/drivers/tty/n_tty.c b/drivers/tty/n_tty.c
index 9cdb0fa3c4bf..f9c584244f72 100644
--- a/drivers/tty/n_tty.c
+++ b/drivers/tty/n_tty.c
@@ -550,9 +550,9 @@ static ssize_t process_output_block(struct tty_struct *tty,
 	mutex_lock(&ldata->output_lock);
 
 	space = tty_write_room(tty);
-	if (!space) {
+	if (space <= 0) {
 		mutex_unlock(&ldata->output_lock);
-		return 0;
+		return space;
 	}
 	if (nr > space)
 		nr = space;
diff --git a/drivers/tty/rocket.c b/drivers/tty/rocket.c
index 27aeca30eeae..5ba6816ebf81 100644
--- a/drivers/tty/rocket.c
+++ b/drivers/tty/rocket.c
@@ -1283,23 +1283,29 @@ static int rp_ioctl(struct tty_struct *tty,
 		return -ENXIO;
 
 	switch (cmd) {
-	case RCKP_GET_STRUCT:
-		if (copy_to_user(argp, info, sizeof (struct r_port)))
-			ret = -EFAULT;
-		break;
 	case RCKP_GET_CONFIG:
+		dev_warn_ratelimited(tty->dev,
+					"RCKP_GET_CONFIG option is deprecated\n");
 		ret = get_config(info, argp);
 		break;
 	case RCKP_SET_CONFIG:
+		dev_warn_ratelimited(tty->dev,
+					"RCKP_SET_CONFIG option is deprecated\n");
 		ret = set_config(tty, info, argp);
 		break;
 	case RCKP_GET_PORTS:
+		dev_warn_ratelimited(tty->dev,
+					"RCKP_GET_PORTS option is deprecated\n");
 		ret = get_ports(info, argp);
 		break;
 	case RCKP_RESET_RM2:
+		dev_warn_ratelimited(tty->dev,
+					"RCKP_RESET_RM2 option is deprecated\n");
 		ret = reset_rm2(info, argp);
 		break;
 	case RCKP_GET_VERSION:
+		dev_warn_ratelimited(tty->dev,
+					"RCKP_GET_VERSION option is deprecated\n");
 		ret = get_version(info, argp);
 		break;
 	default:
diff --git a/drivers/tty/rocket.h b/drivers/tty/rocket.h
index d0560203f215..d62ed6587f32 100644
--- a/drivers/tty/rocket.h
+++ b/drivers/tty/rocket.h
@@ -71,7 +71,6 @@ struct rocket_version {
 /*
  * Rocketport ioctls -- "RP"
  */
-#define RCKP_GET_STRUCT		0x00525001
 #define RCKP_GET_CONFIG		0x00525002
 #define RCKP_SET_CONFIG		0x00525003
 #define RCKP_GET_PORTS		0x00525004
diff --git a/drivers/tty/serdev/Kconfig b/drivers/tty/serdev/Kconfig
index 1dbc8352e027..46ae732bfc68 100644
--- a/drivers/tty/serdev/Kconfig
+++ b/drivers/tty/serdev/Kconfig
@@ -1,3 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
 #
 # Serial bus device driver configuration
 #
diff --git a/drivers/tty/serdev/Makefile b/drivers/tty/serdev/Makefile
index 0cbdb9444d9d..078417e5b068 100644
--- a/drivers/tty/serdev/Makefile
+++ b/drivers/tty/serdev/Makefile
@@ -1,3 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
 serdev-objs := core.o
 
 obj-$(CONFIG_SERIAL_DEV_BUS) += serdev.o
diff --git a/drivers/tty/serial/8250/8250_dw.c b/drivers/tty/serial/8250/8250_dw.c
index d31b975dd3fd..284e8d052fc3 100644
--- a/drivers/tty/serial/8250/8250_dw.c
+++ b/drivers/tty/serial/8250/8250_dw.c
@@ -365,7 +365,7 @@ static bool dw8250_fallback_dma_filter(struct dma_chan *chan, void *param)
 
 static bool dw8250_idma_filter(struct dma_chan *chan, void *param)
 {
-	return param == chan->device->dev->parent;
+	return param == chan->device->dev;
 }
 
 /*
@@ -434,7 +434,7 @@ static void dw8250_quirks(struct uart_port *p, struct dw8250_data *data)
 		data->uart_16550_compatible = true;
 	}
 
-	/* Platforms with iDMA */
+	/* Platforms with iDMA 64-bit */
 	if (platform_get_resource_byname(to_platform_device(p->dev),
 					 IORESOURCE_MEM, "lpss_priv")) {
 		data->dma.rx_param = p->dev->parent;
diff --git a/drivers/tty/serial/8250/8250_exar.c b/drivers/tty/serial/8250/8250_exar.c
index 0089aa305ef9..edd6dfe055bf 100644
--- a/drivers/tty/serial/8250/8250_exar.c
+++ b/drivers/tty/serial/8250/8250_exar.c
@@ -361,12 +361,15 @@ static const struct exar8250_platform iot2040_platform = {
 	.register_gpio = iot2040_register_gpio,
 };
 
+/*
+ * For SIMATIC IOT2000, only IOT2040 and its variants have the Exar device,
+ * IOT2020 doesn't have. Therefore it is sufficient to match on the common
+ * board name after the device was found.
+ */
 static const struct dmi_system_id exar_platforms[] = {
 	{
 		.matches = {
 			DMI_EXACT_MATCH(DMI_BOARD_NAME, "SIMATIC IOT2000"),
-			DMI_EXACT_MATCH(DMI_BOARD_ASSET_TAG,
-					"6ES7647-0AA00-1YA2"),
 		},
 		.driver_data = (void *)&iot2040_platform,
 	},
diff --git a/drivers/tty/serial/8250/8250_fintek.c b/drivers/tty/serial/8250/8250_fintek.c
index 79a4958b3f5c..31c91c2f8c6e 100644
--- a/drivers/tty/serial/8250/8250_fintek.c
+++ b/drivers/tty/serial/8250/8250_fintek.c
@@ -303,8 +303,9 @@ static void fintek_8250_goto_highspeed(struct uart_8250_port *uart,
 	}
 }
 
-void fintek_8250_set_termios(struct uart_port *port, struct ktermios *termios,
-			struct ktermios *old)
+static void fintek_8250_set_termios(struct uart_port *port,
+				    struct ktermios *termios,
+				    struct ktermios *old)
 {
 	struct fintek_8250 *pdata = port->private_data;
 	unsigned int baud = tty_termios_baud_rate(termios);
diff --git a/drivers/tty/serial/8250/8250_men_mcb.c b/drivers/tty/serial/8250/8250_men_mcb.c
index 127017cc41d9..02c5aff58a74 100644
--- a/drivers/tty/serial/8250/8250_men_mcb.c
+++ b/drivers/tty/serial/8250/8250_men_mcb.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 #include <linux/device.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
diff --git a/drivers/tty/serial/8250/8250_mtk.c b/drivers/tty/serial/8250/8250_mtk.c
index c1fdbc0b6840..417c7c810df9 100644
--- a/drivers/tty/serial/8250/8250_mtk.c
+++ b/drivers/tty/serial/8250/8250_mtk.c
@@ -21,15 +21,32 @@
 
 #include "8250.h"
 
-#define UART_MTK_HIGHS		0x09	/* Highspeed register */
-#define UART_MTK_SAMPLE_COUNT	0x0a	/* Sample count register */
-#define UART_MTK_SAMPLE_POINT	0x0b	/* Sample point register */
+#define MTK_UART_HIGHS		0x09	/* Highspeed register */
+#define MTK_UART_SAMPLE_COUNT	0x0a	/* Sample count register */
+#define MTK_UART_SAMPLE_POINT	0x0b	/* Sample point register */
 #define MTK_UART_RATE_FIX	0x0d	/* UART Rate Fix Register */
-
+#define MTK_UART_ESCAPE_DAT	0x10	/* Escape Character register */
+#define MTK_UART_ESCAPE_EN	0x11	/* Escape Enable register */
 #define MTK_UART_DMA_EN		0x13	/* DMA Enable register */
+#define MTK_UART_RXTRI_AD	0x14	/* RX Trigger address */
+#define MTK_UART_FRACDIV_L	0x15	/* Fractional divider LSB address */
+#define MTK_UART_FRACDIV_M	0x16	/* Fractional divider MSB address */
+#define MTK_UART_IER_XOFFI	0x20	/* Enable XOFF character interrupt */
+#define MTK_UART_IER_RTSI	0x40	/* Enable RTS Modem status interrupt */
+#define MTK_UART_IER_CTSI	0x80	/* Enable CTS Modem status interrupt */
+
+#define MTK_UART_EFR_EN		0x10	/* Enable enhancement feature */
+#define MTK_UART_EFR_RTS	0x40	/* Enable hardware rx flow control */
+#define MTK_UART_EFR_CTS	0x80	/* Enable hardware tx flow control */
+#define MTK_UART_EFR_NO_SW_FC	0x0	/* no sw flow control */
+#define MTK_UART_EFR_XON1_XOFF1	0xa	/* XON1/XOFF1 as sw flow control */
+#define MTK_UART_EFR_XON2_XOFF2	0x5	/* XON2/XOFF2 as sw flow control */
+#define MTK_UART_EFR_SW_FC_MASK	0xf	/* Enable CTS Modem status interrupt */
+#define MTK_UART_EFR_HW_FC	(MTK_UART_EFR_RTS | MTK_UART_EFR_CTS)
 #define MTK_UART_DMA_EN_TX	0x2
 #define MTK_UART_DMA_EN_RX	0x5
 
+#define MTK_UART_ESCAPE_CHAR	0x77	/* Escape char added under sw fc */
 #define MTK_UART_TX_SIZE	UART_XMIT_SIZE
 #define MTK_UART_RX_SIZE	0x8000
 #define MTK_UART_TX_TRIGGER	1
@@ -46,6 +63,7 @@ enum dma_rx_status {
 struct mtk8250_data {
 	int			line;
 	unsigned int		rx_pos;
+	unsigned int		clk_count;
 	struct clk		*uart_clk;
 	struct clk		*bus_clk;
 	struct uart_8250_dma	*dma;
@@ -54,6 +72,13 @@ struct mtk8250_data {
 #endif
 };
 
+/* flow control mode */
+enum {
+	MTK_UART_FC_NONE,
+	MTK_UART_FC_SW,
+	MTK_UART_FC_HW,
+};
+
 #ifdef CONFIG_SERIAL_8250_DMA
 static void mtk8250_rx_dma(struct uart_8250_port *up);
 
@@ -192,13 +217,89 @@ static void mtk8250_shutdown(struct uart_port *port)
 	return serial8250_do_shutdown(port);
 }
 
+static void mtk8250_disable_intrs(struct uart_8250_port *up, int mask)
+{
+	serial_out(up, UART_IER, serial_in(up, UART_IER) & (~mask));
+}
+
+static void mtk8250_enable_intrs(struct uart_8250_port *up, int mask)
+{
+	serial_out(up, UART_IER, serial_in(up, UART_IER) | mask);
+}
+
+static void mtk8250_set_flow_ctrl(struct uart_8250_port *up, int mode)
+{
+	struct uart_port *port = &up->port;
+	int lcr = serial_in(up, UART_LCR);
+
+	serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B);
+	serial_out(up, UART_EFR, UART_EFR_ECB);
+	serial_out(up, UART_LCR, lcr);
+	lcr = serial_in(up, UART_LCR);
+
+	switch (mode) {
+	case MTK_UART_FC_NONE:
+		serial_out(up, MTK_UART_ESCAPE_DAT, MTK_UART_ESCAPE_CHAR);
+		serial_out(up, MTK_UART_ESCAPE_EN, 0x00);
+		serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B);
+		serial_out(up, UART_EFR, serial_in(up, UART_EFR) &
+			(~(MTK_UART_EFR_HW_FC | MTK_UART_EFR_SW_FC_MASK)));
+		serial_out(up, UART_LCR, lcr);
+		mtk8250_disable_intrs(up, MTK_UART_IER_XOFFI |
+			MTK_UART_IER_RTSI | MTK_UART_IER_CTSI);
+		break;
+
+	case MTK_UART_FC_HW:
+		serial_out(up, MTK_UART_ESCAPE_DAT, MTK_UART_ESCAPE_CHAR);
+		serial_out(up, MTK_UART_ESCAPE_EN, 0x00);
+		serial_out(up, UART_MCR, UART_MCR_RTS);
+		serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B);
+
+		/*enable hw flow control*/
+		serial_out(up, UART_EFR, MTK_UART_EFR_HW_FC |
+			(serial_in(up, UART_EFR) &
+			(~(MTK_UART_EFR_HW_FC | MTK_UART_EFR_SW_FC_MASK))));
+
+		serial_out(up, UART_LCR, lcr);
+		mtk8250_disable_intrs(up, MTK_UART_IER_XOFFI);
+		mtk8250_enable_intrs(up, MTK_UART_IER_CTSI | MTK_UART_IER_RTSI);
+		break;
+
+	case MTK_UART_FC_SW:	/*MTK software flow control */
+		serial_out(up, MTK_UART_ESCAPE_DAT, MTK_UART_ESCAPE_CHAR);
+		serial_out(up, MTK_UART_ESCAPE_EN, 0x01);
+		serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B);
+
+		/*enable sw flow control */
+		serial_out(up, UART_EFR, MTK_UART_EFR_XON1_XOFF1 |
+			(serial_in(up, UART_EFR) &
+			(~(MTK_UART_EFR_HW_FC | MTK_UART_EFR_SW_FC_MASK))));
+
+		serial_out(up, UART_XON1, START_CHAR(port->state->port.tty));
+		serial_out(up, UART_XOFF1, STOP_CHAR(port->state->port.tty));
+		serial_out(up, UART_LCR, lcr);
+		mtk8250_disable_intrs(up, MTK_UART_IER_CTSI|MTK_UART_IER_RTSI);
+		mtk8250_enable_intrs(up, MTK_UART_IER_XOFFI);
+		break;
+	default:
+		break;
+	}
+}
+
 static void
 mtk8250_set_termios(struct uart_port *port, struct ktermios *termios,
 			struct ktermios *old)
 {
+	unsigned short fraction_L_mapping[] = {
+		0, 1, 0x5, 0x15, 0x55, 0x57, 0x57, 0x77, 0x7F, 0xFF, 0xFF
+	};
+	unsigned short fraction_M_mapping[] = {
+		0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 3
+	};
 	struct uart_8250_port *up = up_to_u8250p(port);
+	unsigned int baud, quot, fraction;
 	unsigned long flags;
-	unsigned int baud, quot;
+	int mode;
 
 #ifdef CONFIG_SERIAL_8250_DMA
 	if (up->dma) {
@@ -214,7 +315,7 @@ mtk8250_set_termios(struct uart_port *port, struct ktermios *termios,
 	serial8250_do_set_termios(port, termios, old);
 
 	/*
-	 * Mediatek UARTs use an extra highspeed register (UART_MTK_HIGHS)
+	 * Mediatek UARTs use an extra highspeed register (MTK_UART_HIGHS)
 	 *
 	 * We need to recalcualte the quot register, as the claculation depends
 	 * on the vaule in the highspeed register.
@@ -230,18 +331,11 @@ mtk8250_set_termios(struct uart_port *port, struct ktermios *termios,
 				  port->uartclk / 16 / UART_DIV_MAX,
 				  port->uartclk);
 
-	if (baud <= 115200) {
-		serial_port_out(port, UART_MTK_HIGHS, 0x0);
+	if (baud < 115200) {
+		serial_port_out(port, MTK_UART_HIGHS, 0x0);
 		quot = uart_get_divisor(port, baud);
-	} else if (baud <= 576000) {
-		serial_port_out(port, UART_MTK_HIGHS, 0x2);
-
-		/* Set to next lower baudrate supported */
-		if ((baud == 500000) || (baud == 576000))
-			baud = 460800;
-		quot = DIV_ROUND_UP(port->uartclk, 4 * baud);
 	} else {
-		serial_port_out(port, UART_MTK_HIGHS, 0x3);
+		serial_port_out(port, MTK_UART_HIGHS, 0x3);
 		quot = DIV_ROUND_UP(port->uartclk, 256 * baud);
 	}
 
@@ -258,18 +352,40 @@ mtk8250_set_termios(struct uart_port *port, struct ktermios *termios,
 	/* reset DLAB */
 	serial_port_out(port, UART_LCR, up->lcr);
 
-	if (baud > 460800) {
+	if (baud >= 115200) {
 		unsigned int tmp;
 
-		tmp = DIV_ROUND_CLOSEST(port->uartclk, quot * baud);
-		serial_port_out(port, UART_MTK_SAMPLE_COUNT, tmp - 1);
-		serial_port_out(port, UART_MTK_SAMPLE_POINT,
-					(tmp - 2) >> 1);
+		tmp = (port->uartclk / (baud *  quot)) - 1;
+		serial_port_out(port, MTK_UART_SAMPLE_COUNT, tmp);
+		serial_port_out(port, MTK_UART_SAMPLE_POINT,
+					(tmp >> 1) - 1);
+
+		/*count fraction to set fractoin register */
+		fraction = ((port->uartclk  * 100) / baud / quot) % 100;
+		fraction = DIV_ROUND_CLOSEST(fraction, 10);
+		serial_port_out(port, MTK_UART_FRACDIV_L,
+						fraction_L_mapping[fraction]);
+		serial_port_out(port, MTK_UART_FRACDIV_M,
+						fraction_M_mapping[fraction]);
 	} else {
-		serial_port_out(port, UART_MTK_SAMPLE_COUNT, 0x00);
-		serial_port_out(port, UART_MTK_SAMPLE_POINT, 0xff);
+		serial_port_out(port, MTK_UART_SAMPLE_COUNT, 0x00);
+		serial_port_out(port, MTK_UART_SAMPLE_POINT, 0xff);
+		serial_port_out(port, MTK_UART_FRACDIV_L, 0x00);
+		serial_port_out(port, MTK_UART_FRACDIV_M, 0x00);
 	}
 
+	if ((termios->c_cflag & CRTSCTS) && (!(termios->c_iflag & CRTSCTS)))
+		mode = MTK_UART_FC_HW;
+	else if (termios->c_iflag & CRTSCTS)
+		mode = MTK_UART_FC_SW;
+	else
+		mode = MTK_UART_FC_NONE;
+
+	mtk8250_set_flow_ctrl(up, mode);
+
+	if (uart_console(port))
+		up->port.cons->cflag = termios->c_cflag;
+
 	spin_unlock_irqrestore(&port->lock, flags);
 	/* Don't rewrite B0 */
 	if (tty_termios_baud_rate(termios))
diff --git a/drivers/tty/serial/8250/Kconfig b/drivers/tty/serial/8250/Kconfig
index 15c2c5463835..296115f6a4d8 100644
--- a/drivers/tty/serial/8250/Kconfig
+++ b/drivers/tty/serial/8250/Kconfig
@@ -1,3 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
 #
 # The 8250/16550 serial drivers.  You shouldn't be in this list unless
 # you somehow have an implicit or explicit dependency on SERIAL_8250.
diff --git a/drivers/tty/serial/Kconfig b/drivers/tty/serial/Kconfig
index 72966bc0ac76..0d31251e04cc 100644
--- a/drivers/tty/serial/Kconfig
+++ b/drivers/tty/serial/Kconfig
@@ -1,3 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
 #
 # Serial device configuration
 #
@@ -369,7 +370,6 @@ config SERIAL_MAX310X
 	depends on SPI_MASTER
 	select SERIAL_CORE
 	select REGMAP_SPI if SPI_MASTER
-	default n
 	help
 	  This selects support for an advanced UART from Maxim (Dallas).
 	  Supported ICs are MAX3107, MAX3108, MAX3109, MAX14830.
@@ -652,7 +652,6 @@ config SERIAL_MUX_CONSOLE
 config PDC_CONSOLE
 	bool "PDC software console support"
 	depends on PARISC && !SERIAL_MUX && VT
-	default n
 	help
 	  Saying Y here will enable the software based PDC console to be 
 	  used as the system console.  This is useful for machines in 
@@ -1095,6 +1094,30 @@ config SERIAL_OMAP_CONSOLE
 	  your boot loader about how to pass options to the kernel at
 	  boot time.)
 
+config SERIAL_SIFIVE
+	tristate "SiFive UART support"
+	depends on OF
+	select SERIAL_CORE
+	help
+	  Select this option if you are building a kernel for a device that
+	  contains a SiFive UART IP block.  This type of UART is present on
+	  SiFive FU540 SoCs, among others.
+
+config SERIAL_SIFIVE_CONSOLE
+	bool "Console on SiFive UART"
+	depends on SERIAL_SIFIVE=y
+	select SERIAL_CORE_CONSOLE
+	help
+	  Select this option if you would like to use a SiFive UART as the
+	  system console.
+
+	  Even if you say Y here, the currently visible virtual console
+	  (/dev/tty0) will still be used as the system console by default, but
+	  you can alter that using a kernel command line option such as
+	  "console=ttySIFx". (Try "man bootparam" or see the documentation of
+	  your boot loader about how to pass options to the kernel at
+	  boot time.)
+
 config SERIAL_LANTIQ
 	bool "Lantiq serial driver"
 	depends on LANTIQ
@@ -1109,7 +1132,6 @@ config SERIAL_QE
 	depends on QUICC_ENGINE
 	select SERIAL_CORE
 	select FW_LOADER
-	default n
 	help
 	  This driver supports the QE serial ports on Freescale embedded
 	  PowerPC that contain a QUICC Engine.
@@ -1582,6 +1604,32 @@ config SERIAL_RDA_CONSOLE
 	  Say 'Y' here if you wish to use the RDA8810PL UART as the system
 	  console. Only earlycon is implemented currently.
 
+config SERIAL_MILBEAUT_USIO
+	tristate "Milbeaut USIO/UART serial port support"
+	depends on ARCH_MILBEAUT || (COMPILE_TEST && OF)
+	default ARCH_MILBEAUT
+	select SERIAL_CORE
+	help
+	  This selects the USIO/UART IP found in Socionext Milbeaut SoCs.
+
+config SERIAL_MILBEAUT_USIO_PORTS
+	int "Maximum number of CSIO/UART ports (1-8)"
+	range 1 8
+	depends on SERIAL_MILBEAUT_USIO
+	default "4"
+
+config SERIAL_MILBEAUT_USIO_CONSOLE
+	bool "Support for console on MILBEAUT USIO/UART serial port"
+	depends on SERIAL_MILBEAUT_USIO=y
+	default y
+	select SERIAL_CORE_CONSOLE
+	select SERIAL_EARLYCON
+	help
+	  Say 'Y' here if you wish to use a USIO/UART of Socionext Milbeaut
+	  SoCs as the system console (the system console is the device which
+	  receives all kernel messages and warnings and which allows logins in
+	  single user mode).
+
 endmenu
 
 config SERIAL_MCTRL_GPIO
diff --git a/drivers/tty/serial/Makefile b/drivers/tty/serial/Makefile
index 40b702aaa85e..79c3d513db7e 100644
--- a/drivers/tty/serial/Makefile
+++ b/drivers/tty/serial/Makefile
@@ -92,6 +92,8 @@ obj-$(CONFIG_SERIAL_PIC32)	+= pic32_uart.o
 obj-$(CONFIG_SERIAL_MPS2_UART)	+= mps2-uart.o
 obj-$(CONFIG_SERIAL_OWL)	+= owl-uart.o
 obj-$(CONFIG_SERIAL_RDA)	+= rda-uart.o
+obj-$(CONFIG_SERIAL_MILBEAUT_USIO) += milbeaut_usio.o
+obj-$(CONFIG_SERIAL_SIFIVE)	+= sifive.o
 
 # GPIOLIB helpers for modem control lines
 obj-$(CONFIG_SERIAL_MCTRL_GPIO)	+= serial_mctrl_gpio.o
diff --git a/drivers/tty/serial/cpm_uart/Makefile b/drivers/tty/serial/cpm_uart/Makefile
index 896a5d57881c..3f3a6ed02ed4 100644
--- a/drivers/tty/serial/cpm_uart/Makefile
+++ b/drivers/tty/serial/cpm_uart/Makefile
@@ -1,3 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
 #
 # Makefile for the Motorola 8xx FEC ethernet controller
 #
diff --git a/drivers/tty/serial/jsm/Makefile b/drivers/tty/serial/jsm/Makefile
index 705d1ff6fdeb..4f2dbada7741 100644
--- a/drivers/tty/serial/jsm/Makefile
+++ b/drivers/tty/serial/jsm/Makefile
@@ -1,3 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
 #
 # Makefile for Jasmine adapter
 #
diff --git a/drivers/tty/serial/milbeaut_usio.c b/drivers/tty/serial/milbeaut_usio.c
new file mode 100644
index 000000000000..949ab7efc4fc
--- /dev/null
+++ b/drivers/tty/serial/milbeaut_usio.c
@@ -0,0 +1,614 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2018 Socionext Inc.
+ */
+
+#if defined(CONFIG_SERIAL_MILBEAUT_USIO_CONSOLE) && defined(CONFIG_MAGIC_SYSRQ)
+#define SUPPORT_SYSRQ
+#endif
+
+#include <linux/clk.h>
+#include <linux/console.h>
+#include <linux/module.h>
+#include <linux/of_irq.h>
+#include <linux/platform_device.h>
+#include <linux/serial_core.h>
+#include <linux/tty.h>
+#include <linux/tty_flip.h>
+
+#define USIO_NAME		"mlb-usio-uart"
+#define USIO_UART_DEV_NAME	"ttyUSI"
+
+static struct uart_port mlb_usio_ports[CONFIG_SERIAL_MILBEAUT_USIO_PORTS];
+
+#define RX	0
+#define TX	1
+static int mlb_usio_irq[CONFIG_SERIAL_MILBEAUT_USIO_PORTS][2];
+
+#define MLB_USIO_REG_SMR		0
+#define MLB_USIO_REG_SCR		1
+#define MLB_USIO_REG_ESCR		2
+#define MLB_USIO_REG_SSR		3
+#define MLB_USIO_REG_DR			4
+#define MLB_USIO_REG_BGR		6
+#define MLB_USIO_REG_FCR		12
+#define MLB_USIO_REG_FBYTE		14
+
+#define MLB_USIO_SMR_SOE		BIT(0)
+#define MLB_USIO_SMR_SBL		BIT(3)
+#define MLB_USIO_SCR_TXE		BIT(0)
+#define MLB_USIO_SCR_RXE		BIT(1)
+#define MLB_USIO_SCR_TBIE		BIT(2)
+#define MLB_USIO_SCR_TIE		BIT(3)
+#define MLB_USIO_SCR_RIE		BIT(4)
+#define MLB_USIO_SCR_UPCL		BIT(7)
+#define MLB_USIO_ESCR_L_8BIT		0
+#define MLB_USIO_ESCR_L_5BIT		1
+#define MLB_USIO_ESCR_L_6BIT		2
+#define MLB_USIO_ESCR_L_7BIT		3
+#define MLB_USIO_ESCR_P			BIT(3)
+#define MLB_USIO_ESCR_PEN		BIT(4)
+#define MLB_USIO_ESCR_FLWEN		BIT(7)
+#define MLB_USIO_SSR_TBI		BIT(0)
+#define MLB_USIO_SSR_TDRE		BIT(1)
+#define MLB_USIO_SSR_RDRF		BIT(2)
+#define MLB_USIO_SSR_ORE		BIT(3)
+#define MLB_USIO_SSR_FRE		BIT(4)
+#define MLB_USIO_SSR_PE			BIT(5)
+#define MLB_USIO_SSR_REC		BIT(7)
+#define MLB_USIO_SSR_BRK		BIT(8)
+#define MLB_USIO_FCR_FE1		BIT(0)
+#define MLB_USIO_FCR_FE2		BIT(1)
+#define MLB_USIO_FCR_FCL1		BIT(2)
+#define MLB_USIO_FCR_FCL2		BIT(3)
+#define MLB_USIO_FCR_FSET		BIT(4)
+#define MLB_USIO_FCR_FTIE		BIT(9)
+#define MLB_USIO_FCR_FDRQ		BIT(10)
+#define MLB_USIO_FCR_FRIIE		BIT(11)
+
+static void mlb_usio_stop_tx(struct uart_port *port)
+{
+	writew(readw(port->membase + MLB_USIO_REG_FCR) & ~MLB_USIO_FCR_FTIE,
+	       port->membase + MLB_USIO_REG_FCR);
+	writeb(readb(port->membase + MLB_USIO_REG_SCR) & ~MLB_USIO_SCR_TBIE,
+	       port->membase + MLB_USIO_REG_SCR);
+}
+
+static void mlb_usio_tx_chars(struct uart_port *port)
+{
+	struct circ_buf *xmit = &port->state->xmit;
+	int count;
+
+	writew(readw(port->membase + MLB_USIO_REG_FCR) & ~MLB_USIO_FCR_FTIE,
+	       port->membase + MLB_USIO_REG_FCR);
+	writeb(readb(port->membase + MLB_USIO_REG_SCR) &
+	       ~(MLB_USIO_SCR_TIE | MLB_USIO_SCR_TBIE),
+	       port->membase + MLB_USIO_REG_SCR);
+
+	if (port->x_char) {
+		writew(port->x_char, port->membase + MLB_USIO_REG_DR);
+		port->icount.tx++;
+		port->x_char = 0;
+		return;
+	}
+	if (uart_circ_empty(xmit) || uart_tx_stopped(port)) {
+		mlb_usio_stop_tx(port);
+		return;
+	}
+
+	count = port->fifosize -
+		(readw(port->membase + MLB_USIO_REG_FBYTE) & 0xff);
+
+	do {
+		writew(xmit->buf[xmit->tail], port->membase + MLB_USIO_REG_DR);
+
+		xmit->tail = (xmit->tail + 1) & (UART_XMIT_SIZE - 1);
+		port->icount.tx++;
+		if (uart_circ_empty(xmit))
+			break;
+
+	} while (--count > 0);
+
+	writew(readw(port->membase + MLB_USIO_REG_FCR) & ~MLB_USIO_FCR_FDRQ,
+	       port->membase + MLB_USIO_REG_FCR);
+
+	writeb(readb(port->membase + MLB_USIO_REG_SCR) | MLB_USIO_SCR_TBIE,
+	       port->membase + MLB_USIO_REG_SCR);
+
+	if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
+		uart_write_wakeup(port);
+
+	if (uart_circ_empty(xmit))
+		mlb_usio_stop_tx(port);
+}
+
+static void mlb_usio_start_tx(struct uart_port *port)
+{
+	u16 fcr = readw(port->membase + MLB_USIO_REG_FCR);
+
+	writew(fcr | MLB_USIO_FCR_FTIE, port->membase + MLB_USIO_REG_FCR);
+	if (!(fcr & MLB_USIO_FCR_FDRQ))
+		return;
+
+	writeb(readb(port->membase + MLB_USIO_REG_SCR) | MLB_USIO_SCR_TBIE,
+	       port->membase + MLB_USIO_REG_SCR);
+
+	if (readb(port->membase + MLB_USIO_REG_SSR) & MLB_USIO_SSR_TBI)
+		mlb_usio_tx_chars(port);
+}
+
+static void mlb_usio_stop_rx(struct uart_port *port)
+{
+	writeb(readb(port->membase + MLB_USIO_REG_SCR) & ~MLB_USIO_SCR_RIE,
+	       port->membase + MLB_USIO_REG_SCR);
+}
+
+static void mlb_usio_enable_ms(struct uart_port *port)
+{
+	writeb(readb(port->membase + MLB_USIO_REG_SCR) |
+	       MLB_USIO_SCR_RIE | MLB_USIO_SCR_RXE,
+	       port->membase + MLB_USIO_REG_SCR);
+}
+
+static void mlb_usio_rx_chars(struct uart_port *port)
+{
+	struct tty_port *ttyport = &port->state->port;
+	unsigned long flag = 0;
+	char ch = 0;
+	u8 status;
+	int max_count = 2;
+
+	while (max_count--) {
+		status = readb(port->membase + MLB_USIO_REG_SSR);
+
+		if (!(status & MLB_USIO_SSR_RDRF))
+			break;
+
+		if (!(status & (MLB_USIO_SSR_ORE | MLB_USIO_SSR_FRE |
+				MLB_USIO_SSR_PE))) {
+			ch = readw(port->membase + MLB_USIO_REG_DR);
+			flag = TTY_NORMAL;
+			port->icount.rx++;
+			if (uart_handle_sysrq_char(port, ch))
+				continue;
+			uart_insert_char(port, status, MLB_USIO_SSR_ORE,
+					 ch, flag);
+			continue;
+		}
+		if (status & MLB_USIO_SSR_PE)
+			port->icount.parity++;
+		if (status & MLB_USIO_SSR_ORE)
+			port->icount.overrun++;
+		status &= port->read_status_mask;
+		if (status & MLB_USIO_SSR_BRK) {
+			flag = TTY_BREAK;
+			ch = 0;
+		} else
+			if (status & MLB_USIO_SSR_PE) {
+				flag = TTY_PARITY;
+				ch = 0;
+			} else
+				if (status & MLB_USIO_SSR_FRE) {
+					flag = TTY_FRAME;
+					ch = 0;
+				}
+		if (flag)
+			uart_insert_char(port, status, MLB_USIO_SSR_ORE,
+					 ch, flag);
+
+		writeb(readb(port->membase + MLB_USIO_REG_SSR) |
+				MLB_USIO_SSR_REC,
+				port->membase + MLB_USIO_REG_SSR);
+
+		max_count = readw(port->membase + MLB_USIO_REG_FBYTE) >> 8;
+		writew(readw(port->membase + MLB_USIO_REG_FCR) |
+		       MLB_USIO_FCR_FE2 | MLB_USIO_FCR_FRIIE,
+		port->membase + MLB_USIO_REG_FCR);
+	}
+
+	tty_flip_buffer_push(ttyport);
+}
+
+static irqreturn_t mlb_usio_rx_irq(int irq, void *dev_id)
+{
+	struct uart_port *port = dev_id;
+
+	spin_lock(&port->lock);
+	mlb_usio_rx_chars(port);
+	spin_unlock(&port->lock);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t mlb_usio_tx_irq(int irq, void *dev_id)
+{
+	struct uart_port *port = dev_id;
+
+	spin_lock(&port->lock);
+	if (readb(port->membase + MLB_USIO_REG_SSR) & MLB_USIO_SSR_TBI)
+		mlb_usio_tx_chars(port);
+	spin_unlock(&port->lock);
+
+	return IRQ_HANDLED;
+}
+
+static unsigned int mlb_usio_tx_empty(struct uart_port *port)
+{
+	return (readb(port->membase + MLB_USIO_REG_SSR) & MLB_USIO_SSR_TBI) ?
+		TIOCSER_TEMT : 0;
+}
+
+static void mlb_usio_set_mctrl(struct uart_port *port, unsigned int mctrl)
+{
+}
+
+static unsigned int mlb_usio_get_mctrl(struct uart_port *port)
+{
+	return TIOCM_CAR | TIOCM_DSR | TIOCM_CTS;
+
+}
+
+static void mlb_usio_break_ctl(struct uart_port *port, int break_state)
+{
+}
+
+static int mlb_usio_startup(struct uart_port *port)
+{
+	const char *portname = to_platform_device(port->dev)->name;
+	unsigned long flags;
+	int ret, index = port->line;
+	unsigned char  escr;
+
+	ret = request_irq(mlb_usio_irq[index][RX], mlb_usio_rx_irq,
+				0, portname, port);
+	if (ret)
+		return ret;
+	ret = request_irq(mlb_usio_irq[index][TX], mlb_usio_tx_irq,
+				0, portname, port);
+	if (ret) {
+		free_irq(mlb_usio_irq[index][RX], port);
+		return ret;
+	}
+
+	escr = readb(port->membase + MLB_USIO_REG_ESCR);
+	if (of_property_read_bool(port->dev->of_node, "auto-flow-control"))
+		escr |= MLB_USIO_ESCR_FLWEN;
+	spin_lock_irqsave(&port->lock, flags);
+	writeb(0, port->membase + MLB_USIO_REG_SCR);
+	writeb(escr, port->membase + MLB_USIO_REG_ESCR);
+	writeb(MLB_USIO_SCR_UPCL, port->membase + MLB_USIO_REG_SCR);
+	writeb(MLB_USIO_SSR_REC, port->membase + MLB_USIO_REG_SSR);
+	writew(0, port->membase + MLB_USIO_REG_FCR);
+	writew(MLB_USIO_FCR_FCL1 | MLB_USIO_FCR_FCL2,
+	       port->membase + MLB_USIO_REG_FCR);
+	writew(MLB_USIO_FCR_FE1 | MLB_USIO_FCR_FE2 | MLB_USIO_FCR_FRIIE,
+	       port->membase + MLB_USIO_REG_FCR);
+	writew(0, port->membase + MLB_USIO_REG_FBYTE);
+	writew(BIT(12), port->membase + MLB_USIO_REG_FBYTE);
+
+	writeb(MLB_USIO_SCR_TXE  | MLB_USIO_SCR_RIE | MLB_USIO_SCR_TBIE |
+	       MLB_USIO_SCR_RXE, port->membase + MLB_USIO_REG_SCR);
+	spin_unlock_irqrestore(&port->lock, flags);
+
+	return 0;
+}
+
+static void mlb_usio_shutdown(struct uart_port *port)
+{
+	int index = port->line;
+
+	free_irq(mlb_usio_irq[index][RX], port);
+	free_irq(mlb_usio_irq[index][TX], port);
+}
+
+static void mlb_usio_set_termios(struct uart_port *port,
+			struct ktermios *termios, struct ktermios *old)
+{
+	unsigned int escr, smr = MLB_USIO_SMR_SOE;
+	unsigned long flags, baud, quot;
+
+	switch (termios->c_cflag & CSIZE) {
+	case CS5:
+		escr = MLB_USIO_ESCR_L_5BIT;
+		break;
+	case CS6:
+		escr = MLB_USIO_ESCR_L_6BIT;
+		break;
+	case CS7:
+		escr = MLB_USIO_ESCR_L_7BIT;
+		break;
+	case CS8:
+	default:
+		escr = MLB_USIO_ESCR_L_8BIT;
+		break;
+	}
+
+	if (termios->c_cflag & CSTOPB)
+		smr |= MLB_USIO_SMR_SBL;
+
+	if (termios->c_cflag & PARENB) {
+		escr |= MLB_USIO_ESCR_PEN;
+		if (termios->c_cflag & PARODD)
+			escr |= MLB_USIO_ESCR_P;
+	}
+	/* Set hard flow control */
+	if (of_property_read_bool(port->dev->of_node, "auto-flow-control") ||
+			(termios->c_cflag & CRTSCTS))
+		escr |= MLB_USIO_ESCR_FLWEN;
+
+	baud = uart_get_baud_rate(port, termios, old, 0, port->uartclk);
+	if (baud > 1)
+		quot = port->uartclk / baud - 1;
+	else
+		quot = 0;
+
+	spin_lock_irqsave(&port->lock, flags);
+	uart_update_timeout(port, termios->c_cflag, baud);
+	port->read_status_mask = MLB_USIO_SSR_ORE | MLB_USIO_SSR_RDRF |
+				 MLB_USIO_SSR_TDRE;
+	if (termios->c_iflag & INPCK)
+		port->read_status_mask |= MLB_USIO_SSR_FRE | MLB_USIO_SSR_PE;
+
+	port->ignore_status_mask = 0;
+	if (termios->c_iflag & IGNPAR)
+		port->ignore_status_mask |= MLB_USIO_SSR_FRE | MLB_USIO_SSR_PE;
+	if ((termios->c_iflag & IGNBRK) && (termios->c_iflag & IGNPAR))
+		port->ignore_status_mask |= MLB_USIO_SSR_ORE;
+	if ((termios->c_cflag & CREAD) == 0)
+		port->ignore_status_mask |= MLB_USIO_SSR_RDRF;
+
+	writeb(0, port->membase + MLB_USIO_REG_SCR);
+	writeb(MLB_USIO_SCR_UPCL, port->membase + MLB_USIO_REG_SCR);
+	writeb(MLB_USIO_SSR_REC, port->membase + MLB_USIO_REG_SSR);
+	writew(0, port->membase + MLB_USIO_REG_FCR);
+	writeb(smr, port->membase + MLB_USIO_REG_SMR);
+	writeb(escr, port->membase + MLB_USIO_REG_ESCR);
+	writew(quot, port->membase + MLB_USIO_REG_BGR);
+	writew(0, port->membase + MLB_USIO_REG_FCR);
+	writew(MLB_USIO_FCR_FCL1 | MLB_USIO_FCR_FCL2 | MLB_USIO_FCR_FE1 |
+	       MLB_USIO_FCR_FE2 | MLB_USIO_FCR_FRIIE,
+	       port->membase + MLB_USIO_REG_FCR);
+	writew(0, port->membase + MLB_USIO_REG_FBYTE);
+	writew(BIT(12), port->membase + MLB_USIO_REG_FBYTE);
+	writeb(MLB_USIO_SCR_RIE | MLB_USIO_SCR_RXE | MLB_USIO_SCR_TBIE |
+	       MLB_USIO_SCR_TXE, port->membase + MLB_USIO_REG_SCR);
+	spin_unlock_irqrestore(&port->lock, flags);
+}
+
+static const char *mlb_usio_type(struct uart_port *port)
+{
+	return ((port->type == PORT_MLB_USIO) ? USIO_NAME : NULL);
+}
+
+static void mlb_usio_config_port(struct uart_port *port, int flags)
+{
+	if (flags & UART_CONFIG_TYPE)
+		port->type = PORT_MLB_USIO;
+}
+
+static const struct uart_ops mlb_usio_ops = {
+	.tx_empty	= mlb_usio_tx_empty,
+	.set_mctrl	= mlb_usio_set_mctrl,
+	.get_mctrl	= mlb_usio_get_mctrl,
+	.stop_tx	= mlb_usio_stop_tx,
+	.start_tx	= mlb_usio_start_tx,
+	.stop_rx	= mlb_usio_stop_rx,
+	.enable_ms	= mlb_usio_enable_ms,
+	.break_ctl	= mlb_usio_break_ctl,
+	.startup	= mlb_usio_startup,
+	.shutdown	= mlb_usio_shutdown,
+	.set_termios	= mlb_usio_set_termios,
+	.type		= mlb_usio_type,
+	.config_port	= mlb_usio_config_port,
+};
+
+#ifdef CONFIG_SERIAL_MILBEAUT_USIO_CONSOLE
+
+static void mlb_usio_console_putchar(struct uart_port *port, int c)
+{
+	while (!(readb(port->membase + MLB_USIO_REG_SSR) & MLB_USIO_SSR_TDRE))
+		cpu_relax();
+
+	writew(c, port->membase + MLB_USIO_REG_DR);
+}
+
+static void mlb_usio_console_write(struct console *co, const char *s,
+			       unsigned int count)
+{
+	struct uart_port *port = &mlb_usio_ports[co->index];
+
+	uart_console_write(port, s, count, mlb_usio_console_putchar);
+}
+
+static int __init mlb_usio_console_setup(struct console *co, char *options)
+{
+	struct uart_port *port;
+	int baud = 115200;
+	int parity = 'n';
+	int flow = 'n';
+	int bits = 8;
+
+	if (co->index >= CONFIG_SERIAL_MILBEAUT_USIO_PORTS)
+		return -ENODEV;
+
+	port = &mlb_usio_ports[co->index];
+	if (!port->membase)
+		return -ENODEV;
+
+
+	if (options)
+		uart_parse_options(options, &baud, &parity, &bits, &flow);
+
+	if (of_property_read_bool(port->dev->of_node, "auto-flow-control"))
+		flow = 'r';
+
+	return uart_set_options(port, co, baud, parity, bits, flow);
+}
+
+
+static struct uart_driver mlb_usio_uart_driver;
+static struct console mlb_usio_console = {
+	.name   = USIO_UART_DEV_NAME,
+	.write  = mlb_usio_console_write,
+	.device = uart_console_device,
+	.setup  = mlb_usio_console_setup,
+	.flags  = CON_PRINTBUFFER,
+	.index  = -1,
+	.data   = &mlb_usio_uart_driver,
+};
+
+static int __init mlb_usio_console_init(void)
+{
+	register_console(&mlb_usio_console);
+	return 0;
+}
+console_initcall(mlb_usio_console_init);
+
+
+static void mlb_usio_early_console_write(struct console *co, const char *s,
+					u_int count)
+{
+	struct earlycon_device *dev = co->data;
+
+	uart_console_write(&dev->port, s, count, mlb_usio_console_putchar);
+}
+
+static int __init mlb_usio_early_console_setup(struct earlycon_device *device,
+						const char *opt)
+{
+	if (!device->port.membase)
+		return -ENODEV;
+	device->con->write = mlb_usio_early_console_write;
+	return 0;
+}
+
+OF_EARLYCON_DECLARE(mlb_usio, "socionext,milbeaut-usio-uart",
+			mlb_usio_early_console_setup);
+
+#define USIO_CONSOLE	(&mlb_usio_console)
+#else
+#define USIO_CONSOLE	NULL
+#endif
+
+static struct  uart_driver mlb_usio_uart_driver = {
+	.owner		= THIS_MODULE,
+	.driver_name	= USIO_NAME,
+	.dev_name	= USIO_UART_DEV_NAME,
+	.cons           = USIO_CONSOLE,
+	.nr		= CONFIG_SERIAL_MILBEAUT_USIO_PORTS,
+};
+
+static int mlb_usio_probe(struct platform_device *pdev)
+{
+	struct clk *clk = devm_clk_get(&pdev->dev, NULL);
+	struct uart_port *port;
+	struct resource *res;
+	int index = 0;
+	int ret;
+
+	if (IS_ERR(clk)) {
+		dev_err(&pdev->dev, "Missing clock\n");
+		return PTR_ERR(clk);
+	}
+	ret = clk_prepare_enable(clk);
+	if (ret) {
+		dev_err(&pdev->dev, "Clock enable failed: %d\n", ret);
+		return ret;
+	}
+	of_property_read_u32(pdev->dev.of_node, "index", &index);
+	port = &mlb_usio_ports[index];
+
+	port->private_data = (void *)clk;
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (res == NULL) {
+		dev_err(&pdev->dev, "Missing regs\n");
+		ret = -ENODEV;
+		goto failed;
+	}
+	port->membase = devm_ioremap(&pdev->dev, res->start,
+				resource_size(res));
+
+	ret = platform_get_irq_byname(pdev, "rx");
+	mlb_usio_irq[index][RX] = ret;
+
+	ret = platform_get_irq_byname(pdev, "tx");
+	mlb_usio_irq[index][TX] = ret;
+
+	port->irq = mlb_usio_irq[index][RX];
+	port->uartclk = clk_get_rate(clk);
+	port->fifosize = 128;
+	port->iotype = UPIO_MEM32;
+	port->flags = UPF_BOOT_AUTOCONF | UPF_SPD_VHI;
+	port->line = index;
+	port->ops = &mlb_usio_ops;
+	port->dev = &pdev->dev;
+
+	ret = uart_add_one_port(&mlb_usio_uart_driver, port);
+	if (ret) {
+		dev_err(&pdev->dev, "Adding port failed: %d\n", ret);
+		goto failed;
+	}
+	return 0;
+
+failed:
+	clk_disable_unprepare(clk);
+
+	return ret;
+}
+
+static int mlb_usio_remove(struct platform_device *pdev)
+{
+	struct uart_port *port = &mlb_usio_ports[pdev->id];
+	struct clk *clk = port->private_data;
+
+	uart_remove_one_port(&mlb_usio_uart_driver, port);
+	clk_disable_unprepare(clk);
+
+	return 0;
+}
+
+static const struct of_device_id mlb_usio_dt_ids[] = {
+	{ .compatible = "socionext,milbeaut-usio-uart" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, mlb_usio_dt_ids);
+
+static struct platform_driver mlb_usio_driver = {
+	.probe          = mlb_usio_probe,
+	.remove         = mlb_usio_remove,
+	.driver         = {
+		.name   = USIO_NAME,
+		.of_match_table = mlb_usio_dt_ids,
+	},
+};
+
+static int __init mlb_usio_init(void)
+{
+	int ret = uart_register_driver(&mlb_usio_uart_driver);
+
+	if (ret) {
+		pr_err("%s: uart registration failed: %d\n", __func__, ret);
+		return ret;
+	}
+	ret = platform_driver_register(&mlb_usio_driver);
+	if (ret) {
+		uart_unregister_driver(&mlb_usio_uart_driver);
+		pr_err("%s: drv registration failed: %d\n", __func__, ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void __exit mlb_usio_exit(void)
+{
+	platform_driver_unregister(&mlb_usio_driver);
+	uart_unregister_driver(&mlb_usio_uart_driver);
+}
+
+module_init(mlb_usio_init);
+module_exit(mlb_usio_exit);
+
+MODULE_AUTHOR("SOCIONEXT");
+MODULE_DESCRIPTION("MILBEAUT_USIO/UART Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/tty/serial/sc16is7xx.c b/drivers/tty/serial/sc16is7xx.c
index a31db15cd7c0..7d3ae31cc720 100644
--- a/drivers/tty/serial/sc16is7xx.c
+++ b/drivers/tty/serial/sc16is7xx.c
@@ -14,9 +14,9 @@
 #include <linux/device.h>
 #include <linux/gpio/driver.h>
 #include <linux/i2c.h>
+#include <linux/mod_devicetable.h>
 #include <linux/module.h>
-#include <linux/of.h>
-#include <linux/of_device.h>
+#include <linux/property.h>
 #include <linux/regmap.h>
 #include <linux/serial_core.h>
 #include <linux/serial.h>
@@ -1179,7 +1179,8 @@ static int sc16is7xx_probe(struct device *dev,
 			   struct regmap *regmap, int irq, unsigned long flags)
 {
 	struct sched_param sched_param = { .sched_priority = MAX_RT_PRIO / 2 };
-	unsigned long freq, *pfreq = dev_get_platdata(dev);
+	unsigned long freq = 0, *pfreq = dev_get_platdata(dev);
+	u32 uartclk = 0;
 	int i, ret;
 	struct sc16is7xx_port *s;
 
@@ -1193,10 +1194,17 @@ static int sc16is7xx_probe(struct device *dev,
 		return -ENOMEM;
 	}
 
+	/* Always ask for fixed clock rate from a property. */
+	device_property_read_u32(dev, "clock-frequency", &uartclk);
+
 	s->clk = devm_clk_get(dev, NULL);
 	if (IS_ERR(s->clk)) {
+		if (uartclk)
+			freq = uartclk;
 		if (pfreq)
 			freq = *pfreq;
+		if (freq)
+			dev_dbg(dev, "Clock frequency: %luHz\n", freq);
 		else
 			return PTR_ERR(s->clk);
 	} else {
@@ -1384,13 +1392,9 @@ static int sc16is7xx_spi_probe(struct spi_device *spi)
 		return ret;
 
 	if (spi->dev.of_node) {
-		const struct of_device_id *of_id =
-			of_match_device(sc16is7xx_dt_ids, &spi->dev);
-
-		if (!of_id)
+		devtype = device_get_match_data(&spi->dev);
+		if (!devtype)
 			return -ENODEV;
-
-		devtype = (struct sc16is7xx_devtype *)of_id->data;
 	} else {
 		const struct spi_device_id *id_entry = spi_get_device_id(spi);
 
@@ -1426,7 +1430,7 @@ MODULE_DEVICE_TABLE(spi, sc16is7xx_spi_id_table);
 static struct spi_driver sc16is7xx_spi_uart_driver = {
 	.driver = {
 		.name		= SC16IS7XX_NAME,
-		.of_match_table	= of_match_ptr(sc16is7xx_dt_ids),
+		.of_match_table	= sc16is7xx_dt_ids,
 	},
 	.probe		= sc16is7xx_spi_probe,
 	.remove		= sc16is7xx_spi_remove,
@@ -1445,13 +1449,9 @@ static int sc16is7xx_i2c_probe(struct i2c_client *i2c,
 	struct regmap *regmap;
 
 	if (i2c->dev.of_node) {
-		const struct of_device_id *of_id =
-				of_match_device(sc16is7xx_dt_ids, &i2c->dev);
-
-		if (!of_id)
+		devtype = device_get_match_data(&i2c->dev);
+		if (!devtype)
 			return -ENODEV;
-
-		devtype = (struct sc16is7xx_devtype *)of_id->data;
 	} else {
 		devtype = (struct sc16is7xx_devtype *)id->driver_data;
 		flags = IRQF_TRIGGER_FALLING;
@@ -1484,7 +1484,7 @@ MODULE_DEVICE_TABLE(i2c, sc16is7xx_i2c_id_table);
 static struct i2c_driver sc16is7xx_i2c_uart_driver = {
 	.driver = {
 		.name		= SC16IS7XX_NAME,
-		.of_match_table	= of_match_ptr(sc16is7xx_dt_ids),
+		.of_match_table	= sc16is7xx_dt_ids,
 	},
 	.probe		= sc16is7xx_i2c_probe,
 	.remove		= sc16is7xx_i2c_remove,
diff --git a/drivers/tty/serial/serial_core.c b/drivers/tty/serial/serial_core.c
index 351843f847c0..83f4dd0bfd74 100644
--- a/drivers/tty/serial/serial_core.c
+++ b/drivers/tty/serial/serial_core.c
@@ -130,9 +130,6 @@ static void uart_start(struct tty_struct *tty)
 	struct uart_port *port;
 	unsigned long flags;
 
-	if (!state)
-		return;
-
 	port = uart_port_lock(state, flags);
 	__uart_start(tty);
 	uart_port_unlock(port, flags);
@@ -730,9 +727,6 @@ static void uart_unthrottle(struct tty_struct *tty)
 	upstat_t mask = UPSTAT_SYNC_FIFO;
 	struct uart_port *port;
 
-	if (!state)
-		return;
-
 	port = uart_port_ref(state);
 	if (!port)
 		return;
@@ -1514,7 +1508,7 @@ static void uart_set_termios(struct tty_struct *tty,
 	}
 
 	uart_change_speed(tty, state, old_termios);
-	/* reload cflag from termios; port driver may have overriden flags */
+	/* reload cflag from termios; port driver may have overridden flags */
 	cflag = tty->termios.c_cflag;
 
 	/* Handle transition to B0 status */
@@ -1747,6 +1741,16 @@ static void uart_dtr_rts(struct tty_port *port, int raise)
 	uart_port_deref(uport);
 }
 
+static int uart_install(struct tty_driver *driver, struct tty_struct *tty)
+{
+	struct uart_driver *drv = driver->driver_state;
+	struct uart_state *state = drv->state + tty->index;
+
+	tty->driver_data = state;
+
+	return tty_standard_install(driver, tty);
+}
+
 /*
  * Calls to uart_open are serialised by the tty_lock in
  *   drivers/tty/tty_io.c:tty_open()
@@ -1759,11 +1763,8 @@ static void uart_dtr_rts(struct tty_port *port, int raise)
  */
 static int uart_open(struct tty_struct *tty, struct file *filp)
 {
-	struct uart_driver *drv = tty->driver->driver_state;
-	int retval, line = tty->index;
-	struct uart_state *state = drv->state + line;
-
-	tty->driver_data = state;
+	struct uart_state *state = tty->driver_data;
+	int retval;
 
 	retval = tty_port_open(&state->port, tty, filp);
 	if (retval > 0)
@@ -2448,6 +2449,7 @@ static void uart_poll_put_char(struct tty_driver *driver, int line, char ch)
 #endif
 
 static const struct tty_operations uart_ops = {
+	.install	= uart_install,
 	.open		= uart_open,
 	.close		= uart_close,
 	.write		= uart_write,
@@ -2505,7 +2507,7 @@ static const struct tty_port_operations uart_port_ops = {
 int uart_register_driver(struct uart_driver *drv)
 {
 	struct tty_driver *normal;
-	int i, retval;
+	int i, retval = -ENOMEM;
 
 	BUG_ON(drv->state);
 
@@ -2557,7 +2559,7 @@ int uart_register_driver(struct uart_driver *drv)
 out_kfree:
 	kfree(drv->state);
 out:
-	return -ENOMEM;
+	return retval;
 }
 
 /**
diff --git a/drivers/tty/serial/sifive.c b/drivers/tty/serial/sifive.c
new file mode 100644
index 000000000000..be4687814353
--- /dev/null
+++ b/drivers/tty/serial/sifive.c
@@ -0,0 +1,1056 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * SiFive UART driver
+ * Copyright (C) 2018 Paul Walmsley <paul@pwsan.com>
+ * Copyright (C) 2018-2019 SiFive
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * Based partially on:
+ * - drivers/tty/serial/pxa.c
+ * - drivers/tty/serial/amba-pl011.c
+ * - drivers/tty/serial/uartlite.c
+ * - drivers/tty/serial/omap-serial.c
+ * - drivers/pwm/pwm-sifive.c
+ *
+ * See the following sources for further documentation:
+ * - Chapter 19 "Universal Asynchronous Receiver/Transmitter (UART)" of
+ *   SiFive FE310-G000 v2p3
+ * - The tree/master/src/main/scala/devices/uart directory of
+ *   https://github.com/sifive/sifive-blocks/
+ *
+ * The SiFive UART design is not 8250-compatible.  The following common
+ * features are not supported:
+ * - Word lengths other than 8 bits
+ * - Break handling
+ * - Parity
+ * - Flow control
+ * - Modem signals (DSR, RI, etc.)
+ * On the other hand, the design is free from the baggage of the 8250
+ * programming model.
+ */
+
+#include <linux/clk.h>
+#include <linux/console.h>
+#include <linux/delay.h>
+#include <linux/init.h>
+#include <linux/io.h>
+#include <linux/irq.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_irq.h>
+#include <linux/platform_device.h>
+#include <linux/serial_core.h>
+#include <linux/serial_reg.h>
+#include <linux/slab.h>
+#include <linux/tty.h>
+#include <linux/tty_flip.h>
+
+/*
+ * Register offsets
+ */
+
+/* TXDATA */
+#define SIFIVE_SERIAL_TXDATA_OFFS		0x0
+#define SIFIVE_SERIAL_TXDATA_FULL_SHIFT		31
+#define SIFIVE_SERIAL_TXDATA_FULL_MASK		(1 << SIFIVE_SERIAL_TXDATA_FULL_SHIFT)
+#define SIFIVE_SERIAL_TXDATA_DATA_SHIFT		0
+#define SIFIVE_SERIAL_TXDATA_DATA_MASK		(0xff << SIFIVE_SERIAL_TXDATA_DATA_SHIFT)
+
+/* RXDATA */
+#define SIFIVE_SERIAL_RXDATA_OFFS		0x4
+#define SIFIVE_SERIAL_RXDATA_EMPTY_SHIFT	31
+#define SIFIVE_SERIAL_RXDATA_EMPTY_MASK		(1 << SIFIVE_SERIAL_RXDATA_EMPTY_SHIFT)
+#define SIFIVE_SERIAL_RXDATA_DATA_SHIFT		0
+#define SIFIVE_SERIAL_RXDATA_DATA_MASK		(0xff << SIFIVE_SERIAL_RXDATA_DATA_SHIFT)
+
+/* TXCTRL */
+#define SIFIVE_SERIAL_TXCTRL_OFFS		0x8
+#define SIFIVE_SERIAL_TXCTRL_TXCNT_SHIFT	16
+#define SIFIVE_SERIAL_TXCTRL_TXCNT_MASK		(0x7 << SIFIVE_SERIAL_TXCTRL_TXCNT_SHIFT)
+#define SIFIVE_SERIAL_TXCTRL_NSTOP_SHIFT	1
+#define SIFIVE_SERIAL_TXCTRL_NSTOP_MASK		(1 << SIFIVE_SERIAL_TXCTRL_NSTOP_SHIFT)
+#define SIFIVE_SERIAL_TXCTRL_TXEN_SHIFT		0
+#define SIFIVE_SERIAL_TXCTRL_TXEN_MASK		(1 << SIFIVE_SERIAL_TXCTRL_TXEN_SHIFT)
+
+/* RXCTRL */
+#define SIFIVE_SERIAL_RXCTRL_OFFS		0xC
+#define SIFIVE_SERIAL_RXCTRL_RXCNT_SHIFT	16
+#define SIFIVE_SERIAL_RXCTRL_RXCNT_MASK		(0x7 << SIFIVE_SERIAL_TXCTRL_TXCNT_SHIFT)
+#define SIFIVE_SERIAL_RXCTRL_RXEN_SHIFT		0
+#define SIFIVE_SERIAL_RXCTRL_RXEN_MASK		(1 << SIFIVE_SERIAL_RXCTRL_RXEN_SHIFT)
+
+/* IE */
+#define SIFIVE_SERIAL_IE_OFFS			0x10
+#define SIFIVE_SERIAL_IE_RXWM_SHIFT		1
+#define SIFIVE_SERIAL_IE_RXWM_MASK		(1 << SIFIVE_SERIAL_IE_RXWM_SHIFT)
+#define SIFIVE_SERIAL_IE_TXWM_SHIFT		0
+#define SIFIVE_SERIAL_IE_TXWM_MASK		(1 << SIFIVE_SERIAL_IE_TXWM_SHIFT)
+
+/* IP */
+#define SIFIVE_SERIAL_IP_OFFS			0x14
+#define SIFIVE_SERIAL_IP_RXWM_SHIFT		1
+#define SIFIVE_SERIAL_IP_RXWM_MASK		(1 << SIFIVE_SERIAL_IP_RXWM_SHIFT)
+#define SIFIVE_SERIAL_IP_TXWM_SHIFT		0
+#define SIFIVE_SERIAL_IP_TXWM_MASK		(1 << SIFIVE_SERIAL_IP_TXWM_SHIFT)
+
+/* DIV */
+#define SIFIVE_SERIAL_DIV_OFFS			0x18
+#define SIFIVE_SERIAL_DIV_DIV_SHIFT		0
+#define SIFIVE_SERIAL_DIV_DIV_MASK		(0xffff << SIFIVE_SERIAL_IP_DIV_SHIFT)
+
+/*
+ * Config macros
+ */
+
+/*
+ * SIFIVE_SERIAL_MAX_PORTS: maximum number of UARTs on a device that can
+ *                          host a serial console
+ */
+#define SIFIVE_SERIAL_MAX_PORTS			8
+
+/*
+ * SIFIVE_DEFAULT_BAUD_RATE: default baud rate that the driver should
+ *                           configure itself to use
+ */
+#define SIFIVE_DEFAULT_BAUD_RATE		115200
+
+/* SIFIVE_SERIAL_NAME: our driver's name that we pass to the operating system */
+#define SIFIVE_SERIAL_NAME			"sifive-serial"
+
+/* SIFIVE_TTY_PREFIX: tty name prefix for SiFive serial ports */
+#define SIFIVE_TTY_PREFIX			"ttySIF"
+
+/* SIFIVE_TX_FIFO_DEPTH: depth of the TX FIFO (in bytes) */
+#define SIFIVE_TX_FIFO_DEPTH			8
+
+/* SIFIVE_RX_FIFO_DEPTH: depth of the TX FIFO (in bytes) */
+#define SIFIVE_RX_FIFO_DEPTH			8
+
+#if (SIFIVE_TX_FIFO_DEPTH != SIFIVE_RX_FIFO_DEPTH)
+#error Driver does not support configurations with different TX, RX FIFO sizes
+#endif
+
+/*
+ *
+ */
+
+/**
+ * sifive_serial_port - driver-specific data extension to struct uart_port
+ * @port: struct uart_port embedded in this struct
+ * @dev: struct device *
+ * @ier: shadowed copy of the interrupt enable register
+ * @clkin_rate: input clock to the UART IP block.
+ * @baud_rate: UART serial line rate (e.g., 115200 baud)
+ * @clk_notifier: clock rate change notifier for upstream clock changes
+ *
+ * Configuration data specific to this SiFive UART.
+ */
+struct sifive_serial_port {
+	struct uart_port	port;
+	struct device		*dev;
+	unsigned char		ier;
+	unsigned long		clkin_rate;
+	unsigned long		baud_rate;
+	struct clk		*clk;
+	struct notifier_block	clk_notifier;
+};
+
+/*
+ * Structure container-of macros
+ */
+
+#define port_to_sifive_serial_port(p) (container_of((p), \
+						    struct sifive_serial_port, \
+						    port))
+
+#define notifier_to_sifive_serial_port(nb) (container_of((nb), \
+							 struct sifive_serial_port, \
+							 clk_notifier))
+
+/*
+ * Forward declarations
+ */
+static void sifive_serial_stop_tx(struct uart_port *port);
+
+/*
+ * Internal functions
+ */
+
+/**
+ * __ssp_early_writel() - write to a SiFive serial port register (early)
+ * @port: pointer to a struct uart_port record
+ * @offs: register address offset from the IP block base address
+ * @v: value to write to the register
+ *
+ * Given a pointer @port to a struct uart_port record, write the value
+ * @v to the IP block register address offset @offs.  This function is
+ * intended for early console use.
+ *
+ * Context: Intended to be used only by the earlyconsole code.
+ */
+static void __ssp_early_writel(u32 v, u16 offs, struct uart_port *port)
+{
+	writel_relaxed(v, port->membase + offs);
+}
+
+/**
+ * __ssp_early_readl() - read from a SiFive serial port register (early)
+ * @port: pointer to a struct uart_port record
+ * @offs: register address offset from the IP block base address
+ *
+ * Given a pointer @port to a struct uart_port record, read the
+ * contents of the IP block register located at offset @offs from the
+ * IP block base and return it.  This function is intended for early
+ * console use.
+ *
+ * Context: Intended to be called only by the earlyconsole code or by
+ *          __ssp_readl() or __ssp_writel() (in this driver)
+ *
+ * Returns: the register value read from the UART.
+ */
+static u32 __ssp_early_readl(struct uart_port *port, u16 offs)
+{
+	return readl_relaxed(port->membase + offs);
+}
+
+/**
+ * __ssp_writel() - write to a SiFive serial port register
+ * @v: value to write to the register
+ * @offs: register address offset from the IP block base address
+ * @ssp: pointer to a struct sifive_serial_port record
+ *
+ * Write the value @v to the IP block register located at offset @offs from the
+ * IP block base, given a pointer @ssp to a struct sifive_serial_port record.
+ *
+ * Context: Any context.
+ */
+static void __ssp_writel(u32 v, u16 offs, struct sifive_serial_port *ssp)
+{
+	__ssp_early_writel(v, offs, &ssp->port);
+}
+
+/**
+ * __ssp_readl() - read from a SiFive serial port register
+ * @ssp: pointer to a struct sifive_serial_port record
+ * @offs: register address offset from the IP block base address
+ *
+ * Read the contents of the IP block register located at offset @offs from the
+ * IP block base, given a pointer @ssp to a struct sifive_serial_port record.
+ *
+ * Context: Any context.
+ *
+ * Returns: the value of the UART register
+ */
+static u32 __ssp_readl(struct sifive_serial_port *ssp, u16 offs)
+{
+	return __ssp_early_readl(&ssp->port, offs);
+}
+
+/**
+ * sifive_serial_is_txfifo_full() - is the TXFIFO full?
+ * @ssp: pointer to a struct sifive_serial_port
+ *
+ * Read the transmit FIFO "full" bit, returning a non-zero value if the
+ * TX FIFO is full, or zero if space remains.  Intended to be used to prevent
+ * writes to the TX FIFO when it's full.
+ *
+ * Returns: SIFIVE_SERIAL_TXDATA_FULL_MASK (non-zero) if the transmit FIFO
+ * is full, or 0 if space remains.
+ */
+static int sifive_serial_is_txfifo_full(struct sifive_serial_port *ssp)
+{
+	return __ssp_readl(ssp, SIFIVE_SERIAL_TXDATA_OFFS) &
+		SIFIVE_SERIAL_TXDATA_FULL_MASK;
+}
+
+/**
+ * __ssp_transmit_char() - enqueue a byte to transmit onto the TX FIFO
+ * @ssp: pointer to a struct sifive_serial_port
+ * @ch: character to transmit
+ *
+ * Enqueue a byte @ch onto the transmit FIFO, given a pointer @ssp to the
+ * struct sifive_serial_port * to transmit on.  Caller should first check to
+ * ensure that the TXFIFO has space; see sifive_serial_is_txfifo_full().
+ *
+ * Context: Any context.
+ */
+static void __ssp_transmit_char(struct sifive_serial_port *ssp, int ch)
+{
+	__ssp_writel(ch, SIFIVE_SERIAL_TXDATA_OFFS, ssp);
+}
+
+/**
+ * __ssp_transmit_chars() - enqueue multiple bytes onto the TX FIFO
+ * @ssp: pointer to a struct sifive_serial_port
+ *
+ * Transfer up to a TX FIFO size's worth of characters from the Linux serial
+ * transmit buffer to the SiFive UART TX FIFO.
+ *
+ * Context: Any context.  Expects @ssp->port.lock to be held by caller.
+ */
+static void __ssp_transmit_chars(struct sifive_serial_port *ssp)
+{
+	struct circ_buf *xmit = &ssp->port.state->xmit;
+	int count;
+
+	if (ssp->port.x_char) {
+		__ssp_transmit_char(ssp, ssp->port.x_char);
+		ssp->port.icount.tx++;
+		ssp->port.x_char = 0;
+		return;
+	}
+	if (uart_circ_empty(xmit) || uart_tx_stopped(&ssp->port)) {
+		sifive_serial_stop_tx(&ssp->port);
+		return;
+	}
+	count = SIFIVE_TX_FIFO_DEPTH;
+	do {
+		__ssp_transmit_char(ssp, xmit->buf[xmit->tail]);
+		xmit->tail = (xmit->tail + 1) & (UART_XMIT_SIZE - 1);
+		ssp->port.icount.tx++;
+		if (uart_circ_empty(xmit))
+			break;
+	} while (--count > 0);
+
+	if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
+		uart_write_wakeup(&ssp->port);
+
+	if (uart_circ_empty(xmit))
+		sifive_serial_stop_tx(&ssp->port);
+}
+
+/**
+ * __ssp_enable_txwm() - enable transmit watermark interrupts
+ * @ssp: pointer to a struct sifive_serial_port
+ *
+ * Enable interrupt generation when the transmit FIFO watermark is reached
+ * on the SiFive UART referred to by @ssp.
+ */
+static void __ssp_enable_txwm(struct sifive_serial_port *ssp)
+{
+	if (ssp->ier & SIFIVE_SERIAL_IE_TXWM_MASK)
+		return;
+
+	ssp->ier |= SIFIVE_SERIAL_IE_TXWM_MASK;
+	__ssp_writel(ssp->ier, SIFIVE_SERIAL_IE_OFFS, ssp);
+}
+
+/**
+ * __ssp_enable_rxwm() - enable receive watermark interrupts
+ * @ssp: pointer to a struct sifive_serial_port
+ *
+ * Enable interrupt generation when the receive FIFO watermark is reached
+ * on the SiFive UART referred to by @ssp.
+ */
+static void __ssp_enable_rxwm(struct sifive_serial_port *ssp)
+{
+	if (ssp->ier & SIFIVE_SERIAL_IE_RXWM_MASK)
+		return;
+
+	ssp->ier |= SIFIVE_SERIAL_IE_RXWM_MASK;
+	__ssp_writel(ssp->ier, SIFIVE_SERIAL_IE_OFFS, ssp);
+}
+
+/**
+ * __ssp_disable_txwm() - disable transmit watermark interrupts
+ * @ssp: pointer to a struct sifive_serial_port
+ *
+ * Disable interrupt generation when the transmit FIFO watermark is reached
+ * on the UART referred to by @ssp.
+ */
+static void __ssp_disable_txwm(struct sifive_serial_port *ssp)
+{
+	if (!(ssp->ier & SIFIVE_SERIAL_IE_TXWM_MASK))
+		return;
+
+	ssp->ier &= ~SIFIVE_SERIAL_IE_TXWM_MASK;
+	__ssp_writel(ssp->ier, SIFIVE_SERIAL_IE_OFFS, ssp);
+}
+
+/**
+ * __ssp_disable_rxwm() - disable receive watermark interrupts
+ * @ssp: pointer to a struct sifive_serial_port
+ *
+ * Disable interrupt generation when the receive FIFO watermark is reached
+ * on the UART referred to by @ssp.
+ */
+static void __ssp_disable_rxwm(struct sifive_serial_port *ssp)
+{
+	if (!(ssp->ier & SIFIVE_SERIAL_IE_RXWM_MASK))
+		return;
+
+	ssp->ier &= ~SIFIVE_SERIAL_IE_RXWM_MASK;
+	__ssp_writel(ssp->ier, SIFIVE_SERIAL_IE_OFFS, ssp);
+}
+
+/**
+ * __ssp_receive_char() - receive a byte from the UART
+ * @ssp: pointer to a struct sifive_serial_port
+ * @is_empty: char pointer to return whether the RX FIFO is empty
+ *
+ * Try to read a byte from the SiFive UART RX FIFO, referenced by
+ * @ssp, and to return it.  Also returns the RX FIFO empty bit in
+ * the char pointed to by @ch.  The caller must pass the byte back to the
+ * Linux serial layer if needed.
+ *
+ * Returns: the byte read from the UART RX FIFO.
+ */
+static char __ssp_receive_char(struct sifive_serial_port *ssp, char *is_empty)
+{
+	u32 v;
+	u8 ch;
+
+	v = __ssp_readl(ssp, SIFIVE_SERIAL_RXDATA_OFFS);
+
+	if (!is_empty)
+		WARN_ON(1);
+	else
+		*is_empty = (v & SIFIVE_SERIAL_RXDATA_EMPTY_MASK) >>
+			SIFIVE_SERIAL_RXDATA_EMPTY_SHIFT;
+
+	ch = (v & SIFIVE_SERIAL_RXDATA_DATA_MASK) >>
+		SIFIVE_SERIAL_RXDATA_DATA_SHIFT;
+
+	return ch;
+}
+
+/**
+ * __ssp_receive_chars() - receive multiple bytes from the UART
+ * @ssp: pointer to a struct sifive_serial_port
+ *
+ * Receive up to an RX FIFO's worth of bytes from the SiFive UART referred
+ * to by @ssp and pass them up to the Linux serial layer.
+ *
+ * Context: Expects ssp->port.lock to be held by caller.
+ */
+static void __ssp_receive_chars(struct sifive_serial_port *ssp)
+{
+	unsigned char ch;
+	char is_empty;
+	int c;
+
+	for (c = SIFIVE_RX_FIFO_DEPTH; c > 0; --c) {
+		ch = __ssp_receive_char(ssp, &is_empty);
+		if (is_empty)
+			break;
+
+		ssp->port.icount.rx++;
+		uart_insert_char(&ssp->port, 0, 0, ch, TTY_NORMAL);
+	}
+
+	spin_unlock(&ssp->port.lock);
+	tty_flip_buffer_push(&ssp->port.state->port);
+	spin_lock(&ssp->port.lock);
+}
+
+/**
+ * __ssp_update_div() - calculate the divisor setting by the line rate
+ * @ssp: pointer to a struct sifive_serial_port
+ *
+ * Calculate the appropriate value of the clock divisor for the UART
+ * and target line rate referred to by @ssp and write it into the
+ * hardware.
+ */
+static void __ssp_update_div(struct sifive_serial_port *ssp)
+{
+	u16 div;
+
+	div = DIV_ROUND_UP(ssp->clkin_rate, ssp->baud_rate) - 1;
+
+	__ssp_writel(div, SIFIVE_SERIAL_DIV_OFFS, ssp);
+}
+
+/**
+ * __ssp_update_baud_rate() - set the UART "baud rate"
+ * @ssp: pointer to a struct sifive_serial_port
+ * @rate: new target bit rate
+ *
+ * Calculate the UART divisor value for the target bit rate @rate for the
+ * SiFive UART described by @ssp and program it into the UART.  There may
+ * be some error between the target bit rate and the actual bit rate implemented
+ * by the UART due to clock ratio granularity.
+ */
+static void __ssp_update_baud_rate(struct sifive_serial_port *ssp,
+				   unsigned int rate)
+{
+	if (ssp->baud_rate == rate)
+		return;
+
+	ssp->baud_rate = rate;
+	__ssp_update_div(ssp);
+}
+
+/**
+ * __ssp_set_stop_bits() - set the number of stop bits
+ * @ssp: pointer to a struct sifive_serial_port
+ * @nstop: 1 or 2 (stop bits)
+ *
+ * Program the SiFive UART referred to by @ssp to use @nstop stop bits.
+ */
+static void __ssp_set_stop_bits(struct sifive_serial_port *ssp, char nstop)
+{
+	u32 v;
+
+	if (nstop < 1 || nstop > 2) {
+		WARN_ON(1);
+		return;
+	}
+
+	v = __ssp_readl(ssp, SIFIVE_SERIAL_TXCTRL_OFFS);
+	v &= ~SIFIVE_SERIAL_TXCTRL_NSTOP_MASK;
+	v |= (nstop - 1) << SIFIVE_SERIAL_TXCTRL_NSTOP_SHIFT;
+	__ssp_writel(v, SIFIVE_SERIAL_TXCTRL_OFFS, ssp);
+}
+
+/**
+ * __ssp_wait_for_xmitr() - wait for an empty slot on the TX FIFO
+ * @ssp: pointer to a struct sifive_serial_port
+ *
+ * Delay while the UART TX FIFO referred to by @ssp is marked as full.
+ *
+ * Context: Any context.
+ */
+static void __maybe_unused __ssp_wait_for_xmitr(struct sifive_serial_port *ssp)
+{
+	while (sifive_serial_is_txfifo_full(ssp))
+		udelay(1); /* XXX Could probably be more intelligent here */
+}
+
+/*
+ * Linux serial API functions
+ */
+
+static void sifive_serial_stop_tx(struct uart_port *port)
+{
+	struct sifive_serial_port *ssp = port_to_sifive_serial_port(port);
+
+	__ssp_disable_txwm(ssp);
+}
+
+static void sifive_serial_stop_rx(struct uart_port *port)
+{
+	struct sifive_serial_port *ssp = port_to_sifive_serial_port(port);
+
+	__ssp_disable_rxwm(ssp);
+}
+
+static void sifive_serial_start_tx(struct uart_port *port)
+{
+	struct sifive_serial_port *ssp = port_to_sifive_serial_port(port);
+
+	__ssp_enable_txwm(ssp);
+}
+
+static irqreturn_t sifive_serial_irq(int irq, void *dev_id)
+{
+	struct sifive_serial_port *ssp = dev_id;
+	u32 ip;
+
+	spin_lock(&ssp->port.lock);
+
+	ip = __ssp_readl(ssp, SIFIVE_SERIAL_IP_OFFS);
+	if (!ip) {
+		spin_unlock(&ssp->port.lock);
+		return IRQ_NONE;
+	}
+
+	if (ip & SIFIVE_SERIAL_IP_RXWM_MASK)
+		__ssp_receive_chars(ssp);
+	if (ip & SIFIVE_SERIAL_IP_TXWM_MASK)
+		__ssp_transmit_chars(ssp);
+
+	spin_unlock(&ssp->port.lock);
+
+	return IRQ_HANDLED;
+}
+
+static unsigned int sifive_serial_tx_empty(struct uart_port *port)
+{
+	return TIOCSER_TEMT;
+}
+
+static unsigned int sifive_serial_get_mctrl(struct uart_port *port)
+{
+	return TIOCM_CAR | TIOCM_CTS | TIOCM_DSR;
+}
+
+static void sifive_serial_set_mctrl(struct uart_port *port, unsigned int mctrl)
+{
+	/* IP block does not support these signals */
+}
+
+static void sifive_serial_break_ctl(struct uart_port *port, int break_state)
+{
+	/* IP block does not support sending a break */
+}
+
+static int sifive_serial_startup(struct uart_port *port)
+{
+	struct sifive_serial_port *ssp = port_to_sifive_serial_port(port);
+
+	__ssp_enable_rxwm(ssp);
+
+	return 0;
+}
+
+static void sifive_serial_shutdown(struct uart_port *port)
+{
+	struct sifive_serial_port *ssp = port_to_sifive_serial_port(port);
+
+	__ssp_disable_rxwm(ssp);
+	__ssp_disable_txwm(ssp);
+}
+
+/**
+ * sifive_serial_clk_notifier() - clock post-rate-change notifier
+ * @nb: pointer to the struct notifier_block, from the notifier code
+ * @event: event mask from the notifier code
+ * @data: pointer to the struct clk_notifier_data from the notifier code
+ *
+ * On the V0 SoC, the UART IP block is derived from the CPU clock source
+ * after a synchronous divide-by-two divider, so any CPU clock rate change
+ * requires the UART baud rate to be updated.  This presumably could corrupt any
+ * serial word currently being transmitted or received.  It would probably
+ * be better to stop receives and transmits, then complete the baud rate
+ * change, then re-enable them.
+ */
+static int sifive_serial_clk_notifier(struct notifier_block *nb,
+				      unsigned long event, void *data)
+{
+	struct clk_notifier_data *cnd = data;
+	struct sifive_serial_port *ssp = notifier_to_sifive_serial_port(nb);
+
+	if (event == POST_RATE_CHANGE && ssp->clkin_rate != cnd->new_rate) {
+		ssp->clkin_rate = cnd->new_rate;
+		__ssp_update_div(ssp);
+	}
+
+	return NOTIFY_OK;
+}
+
+static void sifive_serial_set_termios(struct uart_port *port,
+				      struct ktermios *termios,
+				      struct ktermios *old)
+{
+	struct sifive_serial_port *ssp = port_to_sifive_serial_port(port);
+	unsigned long flags;
+	u32 v, old_v;
+	int rate;
+	char nstop;
+
+	if ((termios->c_cflag & CSIZE) != CS8)
+		dev_err_once(ssp->port.dev, "only 8-bit words supported\n");
+	if (termios->c_iflag & (INPCK | PARMRK))
+		dev_err_once(ssp->port.dev, "parity checking not supported\n");
+	if (termios->c_iflag & BRKINT)
+		dev_err_once(ssp->port.dev, "BREAK detection not supported\n");
+
+	/* Set number of stop bits */
+	nstop = (termios->c_cflag & CSTOPB) ? 2 : 1;
+	__ssp_set_stop_bits(ssp, nstop);
+
+	/* Set line rate */
+	rate = uart_get_baud_rate(port, termios, old, 0, ssp->clkin_rate / 16);
+	__ssp_update_baud_rate(ssp, rate);
+
+	spin_lock_irqsave(&ssp->port.lock, flags);
+
+	/* Update the per-port timeout */
+	uart_update_timeout(port, termios->c_cflag, rate);
+
+	ssp->port.read_status_mask = 0;
+
+	/* Ignore all characters if CREAD is not set */
+	v = __ssp_readl(ssp, SIFIVE_SERIAL_RXCTRL_OFFS);
+	old_v = v;
+	if ((termios->c_cflag & CREAD) == 0)
+		v &= SIFIVE_SERIAL_RXCTRL_RXEN_MASK;
+	else
+		v |= SIFIVE_SERIAL_RXCTRL_RXEN_MASK;
+	if (v != old_v)
+		__ssp_writel(v, SIFIVE_SERIAL_RXCTRL_OFFS, ssp);
+
+	spin_unlock_irqrestore(&ssp->port.lock, flags);
+}
+
+static void sifive_serial_release_port(struct uart_port *port)
+{
+}
+
+static int sifive_serial_request_port(struct uart_port *port)
+{
+	return 0;
+}
+
+static void sifive_serial_config_port(struct uart_port *port, int flags)
+{
+	struct sifive_serial_port *ssp = port_to_sifive_serial_port(port);
+
+	ssp->port.type = PORT_SIFIVE_V0;
+}
+
+static int sifive_serial_verify_port(struct uart_port *port,
+				     struct serial_struct *ser)
+{
+	return -EINVAL;
+}
+
+static const char *sifive_serial_type(struct uart_port *port)
+{
+	return port->type == PORT_SIFIVE_V0 ? "SiFive UART v0" : NULL;
+}
+
+/*
+ * Early console support
+ */
+
+#ifdef CONFIG_SERIAL_EARLYCON
+static void early_sifive_serial_putc(struct uart_port *port, int c)
+{
+	while (__ssp_early_readl(port, SIFIVE_SERIAL_TXDATA_OFFS) &
+	       SIFIVE_SERIAL_TXDATA_FULL_MASK)
+		cpu_relax();
+
+	__ssp_early_writel(c, SIFIVE_SERIAL_TXDATA_OFFS, port);
+}
+
+static void early_sifive_serial_write(struct console *con, const char *s,
+				      unsigned int n)
+{
+	struct earlycon_device *dev = con->data;
+	struct uart_port *port = &dev->port;
+
+	uart_console_write(port, s, n, early_sifive_serial_putc);
+}
+
+static int __init early_sifive_serial_setup(struct earlycon_device *dev,
+					    const char *options)
+{
+	struct uart_port *port = &dev->port;
+
+	if (!port->membase)
+		return -ENODEV;
+
+	dev->con->write = early_sifive_serial_write;
+
+	return 0;
+}
+
+OF_EARLYCON_DECLARE(sifive, "sifive,uart0", early_sifive_serial_setup);
+OF_EARLYCON_DECLARE(sifive, "sifive,fu540-c000-uart0",
+		    early_sifive_serial_setup);
+#endif /* CONFIG_SERIAL_EARLYCON */
+
+/*
+ * Linux console interface
+ */
+
+#ifdef CONFIG_SERIAL_SIFIVE_CONSOLE
+
+static struct sifive_serial_port *sifive_serial_console_ports[SIFIVE_SERIAL_MAX_PORTS];
+
+static void sifive_serial_console_putchar(struct uart_port *port, int ch)
+{
+	struct sifive_serial_port *ssp = port_to_sifive_serial_port(port);
+
+	__ssp_wait_for_xmitr(ssp);
+	__ssp_transmit_char(ssp, ch);
+}
+
+static void sifive_serial_console_write(struct console *co, const char *s,
+					unsigned int count)
+{
+	struct sifive_serial_port *ssp = sifive_serial_console_ports[co->index];
+	unsigned long flags;
+	unsigned int ier;
+	int locked = 1;
+
+	if (!ssp)
+		return;
+
+	local_irq_save(flags);
+	if (ssp->port.sysrq)
+		locked = 0;
+	else if (oops_in_progress)
+		locked = spin_trylock(&ssp->port.lock);
+	else
+		spin_lock(&ssp->port.lock);
+
+	ier = __ssp_readl(ssp, SIFIVE_SERIAL_IE_OFFS);
+	__ssp_writel(0, SIFIVE_SERIAL_IE_OFFS, ssp);
+
+	uart_console_write(&ssp->port, s, count, sifive_serial_console_putchar);
+
+	__ssp_writel(ier, SIFIVE_SERIAL_IE_OFFS, ssp);
+
+	if (locked)
+		spin_unlock(&ssp->port.lock);
+	local_irq_restore(flags);
+}
+
+static int __init sifive_serial_console_setup(struct console *co, char *options)
+{
+	struct sifive_serial_port *ssp;
+	int baud = SIFIVE_DEFAULT_BAUD_RATE;
+	int bits = 8;
+	int parity = 'n';
+	int flow = 'n';
+
+	if (co->index < 0 || co->index >= SIFIVE_SERIAL_MAX_PORTS)
+		return -ENODEV;
+
+	ssp = sifive_serial_console_ports[co->index];
+	if (!ssp)
+		return -ENODEV;
+
+	if (options)
+		uart_parse_options(options, &baud, &parity, &bits, &flow);
+
+	return uart_set_options(&ssp->port, co, baud, parity, bits, flow);
+}
+
+static struct uart_driver sifive_serial_uart_driver;
+
+static struct console sifive_serial_console = {
+	.name		= SIFIVE_TTY_PREFIX,
+	.write		= sifive_serial_console_write,
+	.device		= uart_console_device,
+	.setup		= sifive_serial_console_setup,
+	.flags		= CON_PRINTBUFFER,
+	.index		= -1,
+	.data		= &sifive_serial_uart_driver,
+};
+
+static int __init sifive_console_init(void)
+{
+	register_console(&sifive_serial_console);
+	return 0;
+}
+
+console_initcall(sifive_console_init);
+
+static void __ssp_add_console_port(struct sifive_serial_port *ssp)
+{
+	sifive_serial_console_ports[ssp->port.line] = ssp;
+}
+
+static void __ssp_remove_console_port(struct sifive_serial_port *ssp)
+{
+	sifive_serial_console_ports[ssp->port.line] = 0;
+}
+
+#define SIFIVE_SERIAL_CONSOLE	(&sifive_serial_console)
+
+#else
+
+#define SIFIVE_SERIAL_CONSOLE	NULL
+
+static void __ssp_add_console_port(struct sifive_serial_port *ssp)
+{}
+static void __ssp_remove_console_port(struct sifive_serial_port *ssp)
+{}
+
+#endif
+
+static const struct uart_ops sifive_serial_uops = {
+	.tx_empty	= sifive_serial_tx_empty,
+	.set_mctrl	= sifive_serial_set_mctrl,
+	.get_mctrl	= sifive_serial_get_mctrl,
+	.stop_tx	= sifive_serial_stop_tx,
+	.start_tx	= sifive_serial_start_tx,
+	.stop_rx	= sifive_serial_stop_rx,
+	.break_ctl	= sifive_serial_break_ctl,
+	.startup	= sifive_serial_startup,
+	.shutdown	= sifive_serial_shutdown,
+	.set_termios	= sifive_serial_set_termios,
+	.type		= sifive_serial_type,
+	.release_port	= sifive_serial_release_port,
+	.request_port	= sifive_serial_request_port,
+	.config_port	= sifive_serial_config_port,
+	.verify_port	= sifive_serial_verify_port,
+};
+
+static struct uart_driver sifive_serial_uart_driver = {
+	.owner		= THIS_MODULE,
+	.driver_name	= SIFIVE_SERIAL_NAME,
+	.dev_name	= SIFIVE_TTY_PREFIX,
+	.nr		= SIFIVE_SERIAL_MAX_PORTS,
+	.cons		= SIFIVE_SERIAL_CONSOLE,
+};
+
+static int sifive_serial_probe(struct platform_device *pdev)
+{
+	struct sifive_serial_port *ssp;
+	struct resource *mem;
+	struct clk *clk;
+	void __iomem *base;
+	int irq, id, r;
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0) {
+		dev_err(&pdev->dev, "could not acquire interrupt\n");
+		return -EPROBE_DEFER;
+	}
+
+	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	base = devm_ioremap_resource(&pdev->dev, mem);
+	if (IS_ERR(base)) {
+		dev_err(&pdev->dev, "could not acquire device memory\n");
+		return PTR_ERR(base);
+	}
+
+	clk = devm_clk_get(&pdev->dev, NULL);
+	if (IS_ERR(clk)) {
+		dev_err(&pdev->dev, "unable to find controller clock\n");
+		return PTR_ERR(clk);
+	}
+
+	id = of_alias_get_id(pdev->dev.of_node, "serial");
+	if (id < 0) {
+		dev_err(&pdev->dev, "missing aliases entry\n");
+		return id;
+	}
+
+#ifdef CONFIG_SERIAL_SIFIVE_CONSOLE
+	if (id > SIFIVE_SERIAL_MAX_PORTS) {
+		dev_err(&pdev->dev, "too many UARTs (%d)\n", id);
+		return -EINVAL;
+	}
+#endif
+
+	ssp = devm_kzalloc(&pdev->dev, sizeof(*ssp), GFP_KERNEL);
+	if (!ssp)
+		return -ENOMEM;
+
+	ssp->port.dev = &pdev->dev;
+	ssp->port.type = PORT_SIFIVE_V0;
+	ssp->port.iotype = UPIO_MEM;
+	ssp->port.irq = irq;
+	ssp->port.fifosize = SIFIVE_TX_FIFO_DEPTH;
+	ssp->port.ops = &sifive_serial_uops;
+	ssp->port.line = id;
+	ssp->port.mapbase = mem->start;
+	ssp->port.membase = base;
+	ssp->dev = &pdev->dev;
+	ssp->clk = clk;
+	ssp->clk_notifier.notifier_call = sifive_serial_clk_notifier;
+
+	r = clk_notifier_register(ssp->clk, &ssp->clk_notifier);
+	if (r) {
+		dev_err(&pdev->dev, "could not register clock notifier: %d\n",
+			r);
+		goto probe_out1;
+	}
+
+	/* Set up clock divider */
+	ssp->clkin_rate = clk_get_rate(ssp->clk);
+	ssp->baud_rate = SIFIVE_DEFAULT_BAUD_RATE;
+	__ssp_update_div(ssp);
+
+	platform_set_drvdata(pdev, ssp);
+
+	/* Enable transmits and set the watermark level to 1 */
+	__ssp_writel((1 << SIFIVE_SERIAL_TXCTRL_TXCNT_SHIFT) |
+		     SIFIVE_SERIAL_TXCTRL_TXEN_MASK,
+		     SIFIVE_SERIAL_TXCTRL_OFFS, ssp);
+
+	/* Enable receives and set the watermark level to 0 */
+	__ssp_writel((0 << SIFIVE_SERIAL_RXCTRL_RXCNT_SHIFT) |
+		     SIFIVE_SERIAL_RXCTRL_RXEN_MASK,
+		     SIFIVE_SERIAL_RXCTRL_OFFS, ssp);
+
+	r = request_irq(ssp->port.irq, sifive_serial_irq, ssp->port.irqflags,
+			dev_name(&pdev->dev), ssp);
+	if (r) {
+		dev_err(&pdev->dev, "could not attach interrupt: %d\n", r);
+		goto probe_out2;
+	}
+
+	__ssp_add_console_port(ssp);
+
+	r = uart_add_one_port(&sifive_serial_uart_driver, &ssp->port);
+	if (r != 0) {
+		dev_err(&pdev->dev, "could not add uart: %d\n", r);
+		goto probe_out3;
+	}
+
+	return 0;
+
+probe_out3:
+	__ssp_remove_console_port(ssp);
+	free_irq(ssp->port.irq, ssp);
+probe_out2:
+	clk_notifier_unregister(ssp->clk, &ssp->clk_notifier);
+probe_out1:
+	return r;
+}
+
+static int sifive_serial_remove(struct platform_device *dev)
+{
+	struct sifive_serial_port *ssp = platform_get_drvdata(dev);
+
+	__ssp_remove_console_port(ssp);
+	uart_remove_one_port(&sifive_serial_uart_driver, &ssp->port);
+	free_irq(ssp->port.irq, ssp);
+	clk_notifier_unregister(ssp->clk, &ssp->clk_notifier);
+
+	return 0;
+}
+
+static const struct of_device_id sifive_serial_of_match[] = {
+	{ .compatible = "sifive,fu540-c000-uart0" },
+	{ .compatible = "sifive,uart0" },
+	{},
+};
+MODULE_DEVICE_TABLE(of, sifive_serial_of_match);
+
+static struct platform_driver sifive_serial_platform_driver = {
+	.probe		= sifive_serial_probe,
+	.remove		= sifive_serial_remove,
+	.driver		= {
+		.name	= SIFIVE_SERIAL_NAME,
+		.of_match_table = of_match_ptr(sifive_serial_of_match),
+	},
+};
+
+static int __init sifive_serial_init(void)
+{
+	int r;
+
+	r = uart_register_driver(&sifive_serial_uart_driver);
+	if (r)
+		goto init_out1;
+
+	r = platform_driver_register(&sifive_serial_platform_driver);
+	if (r)
+		goto init_out2;
+
+	return 0;
+
+init_out2:
+	uart_unregister_driver(&sifive_serial_uart_driver);
+init_out1:
+	return r;
+}
+
+static void __exit sifive_serial_exit(void)
+{
+	platform_driver_unregister(&sifive_serial_platform_driver);
+	uart_unregister_driver(&sifive_serial_uart_driver);
+}
+
+module_init(sifive_serial_init);
+module_exit(sifive_serial_exit);
+
+MODULE_DESCRIPTION("SiFive UART serial driver");
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Paul Walmsley <paul@pwsan.com>");
diff --git a/drivers/tty/serial/sn_console.c b/drivers/tty/serial/sn_console.c
index fe9170731c16..283493358a62 100644
--- a/drivers/tty/serial/sn_console.c
+++ b/drivers/tty/serial/sn_console.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * C-Brick Serial Port (and console) driver for SGI Altix machines.
  *
diff --git a/drivers/tty/serial/sprd_serial.c b/drivers/tty/serial/sprd_serial.c
index 1891a45ac05d..73d71a4e6c0c 100644
--- a/drivers/tty/serial/sprd_serial.c
+++ b/drivers/tty/serial/sprd_serial.c
@@ -10,6 +10,9 @@
 #include <linux/clk.h>
 #include <linux/console.h>
 #include <linux/delay.h>
+#include <linux/dmaengine.h>
+#include <linux/dma-mapping.h>
+#include <linux/dma/sprd-dma.h>
 #include <linux/io.h>
 #include <linux/ioport.h>
 #include <linux/kernel.h>
@@ -75,6 +78,7 @@
 
 /* control register 1 */
 #define SPRD_CTL1		0x001C
+#define SPRD_DMA_EN		BIT(15)
 #define RX_HW_FLOW_CTL_THLD	BIT(6)
 #define RX_HW_FLOW_CTL_EN	BIT(7)
 #define TX_HW_FLOW_CTL_EN	BIT(8)
@@ -86,6 +90,7 @@
 #define THLD_TX_EMPTY		0x40
 #define THLD_TX_EMPTY_SHIFT	8
 #define THLD_RX_FULL		0x40
+#define THLD_RX_FULL_MASK	GENMASK(6, 0)
 
 /* config baud rate register */
 #define SPRD_CLKD0		0x0024
@@ -100,15 +105,38 @@
 #define SPRD_IMSR_TX_FIFO_EMPTY	BIT(1)
 #define SPRD_IMSR_BREAK_DETECT	BIT(7)
 #define SPRD_IMSR_TIMEOUT	BIT(13)
+#define SPRD_DEFAULT_SOURCE_CLK	26000000
+
+#define SPRD_RX_DMA_STEP	1
+#define SPRD_RX_FIFO_FULL	1
+#define SPRD_TX_FIFO_FULL	0x20
+#define SPRD_UART_RX_SIZE	(UART_XMIT_SIZE / 4)
+
+struct sprd_uart_dma {
+	struct dma_chan *chn;
+	unsigned char *virt;
+	dma_addr_t phys_addr;
+	dma_cookie_t cookie;
+	u32 trans_len;
+	bool enable;
+};
 
 struct sprd_uart_port {
 	struct uart_port port;
 	char name[16];
+	struct clk *clk;
+	struct sprd_uart_dma tx_dma;
+	struct sprd_uart_dma rx_dma;
+	dma_addr_t pos;
+	unsigned char *rx_buf_tail;
 };
 
 static struct sprd_uart_port *sprd_port[UART_NR_MAX];
 static int sprd_ports_num;
 
+static int sprd_start_dma_rx(struct uart_port *port);
+static int sprd_tx_dma_config(struct uart_port *port);
+
 static inline unsigned int serial_in(struct uart_port *port,
 				     unsigned int offset)
 {
@@ -139,45 +167,389 @@ static void sprd_set_mctrl(struct uart_port *port, unsigned int mctrl)
 	/* nothing to do */
 }
 
-static void sprd_stop_tx(struct uart_port *port)
+static void sprd_stop_rx(struct uart_port *port)
 {
+	struct sprd_uart_port *sp =
+		container_of(port, struct sprd_uart_port, port);
 	unsigned int ien, iclr;
 
+	if (sp->rx_dma.enable)
+		dmaengine_terminate_all(sp->rx_dma.chn);
+
 	iclr = serial_in(port, SPRD_ICLR);
 	ien = serial_in(port, SPRD_IEN);
 
-	iclr |= SPRD_IEN_TX_EMPTY;
-	ien &= ~SPRD_IEN_TX_EMPTY;
+	ien &= ~(SPRD_IEN_RX_FULL | SPRD_IEN_BREAK_DETECT);
+	iclr |= SPRD_IEN_RX_FULL | SPRD_IEN_BREAK_DETECT;
 
-	serial_out(port, SPRD_ICLR, iclr);
 	serial_out(port, SPRD_IEN, ien);
+	serial_out(port, SPRD_ICLR, iclr);
 }
 
-static void sprd_start_tx(struct uart_port *port)
+static void sprd_uart_dma_enable(struct uart_port *port, bool enable)
 {
-	unsigned int ien;
+	u32 val = serial_in(port, SPRD_CTL1);
 
-	ien = serial_in(port, SPRD_IEN);
-	if (!(ien & SPRD_IEN_TX_EMPTY)) {
-		ien |= SPRD_IEN_TX_EMPTY;
-		serial_out(port, SPRD_IEN, ien);
+	if (enable)
+		val |= SPRD_DMA_EN;
+	else
+		val &= ~SPRD_DMA_EN;
+
+	serial_out(port, SPRD_CTL1, val);
+}
+
+static void sprd_stop_tx_dma(struct uart_port *port)
+{
+	struct sprd_uart_port *sp =
+		container_of(port, struct sprd_uart_port, port);
+	struct circ_buf *xmit = &port->state->xmit;
+	struct dma_tx_state state;
+	u32 trans_len;
+
+	dmaengine_pause(sp->tx_dma.chn);
+
+	dmaengine_tx_status(sp->tx_dma.chn, sp->tx_dma.cookie, &state);
+	if (state.residue) {
+		trans_len = state.residue - sp->tx_dma.phys_addr;
+		xmit->tail = (xmit->tail + trans_len) & (UART_XMIT_SIZE - 1);
+		port->icount.tx += trans_len;
+		dma_unmap_single(port->dev, sp->tx_dma.phys_addr,
+				 sp->tx_dma.trans_len, DMA_TO_DEVICE);
 	}
+
+	dmaengine_terminate_all(sp->tx_dma.chn);
+	sp->tx_dma.trans_len = 0;
 }
 
-static void sprd_stop_rx(struct uart_port *port)
+static int sprd_tx_buf_remap(struct uart_port *port)
+{
+	struct sprd_uart_port *sp =
+		container_of(port, struct sprd_uart_port, port);
+	struct circ_buf *xmit = &port->state->xmit;
+
+	sp->tx_dma.trans_len =
+		CIRC_CNT_TO_END(xmit->head, xmit->tail, UART_XMIT_SIZE);
+
+	sp->tx_dma.phys_addr = dma_map_single(port->dev,
+					      (void *)&(xmit->buf[xmit->tail]),
+					      sp->tx_dma.trans_len,
+					      DMA_TO_DEVICE);
+	return dma_mapping_error(port->dev, sp->tx_dma.phys_addr);
+}
+
+static void sprd_complete_tx_dma(void *data)
+{
+	struct uart_port *port = (struct uart_port *)data;
+	struct sprd_uart_port *sp =
+		container_of(port, struct sprd_uart_port, port);
+	struct circ_buf *xmit = &port->state->xmit;
+	unsigned long flags;
+
+	spin_lock_irqsave(&port->lock, flags);
+	dma_unmap_single(port->dev, sp->tx_dma.phys_addr,
+			 sp->tx_dma.trans_len, DMA_TO_DEVICE);
+
+	xmit->tail = (xmit->tail + sp->tx_dma.trans_len) & (UART_XMIT_SIZE - 1);
+	port->icount.tx += sp->tx_dma.trans_len;
+
+	if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
+		uart_write_wakeup(port);
+
+	if (uart_circ_empty(xmit) || sprd_tx_buf_remap(port) ||
+	    sprd_tx_dma_config(port))
+		sp->tx_dma.trans_len = 0;
+
+	spin_unlock_irqrestore(&port->lock, flags);
+}
+
+static int sprd_uart_dma_submit(struct uart_port *port,
+				struct sprd_uart_dma *ud, u32 trans_len,
+				enum dma_transfer_direction direction,
+				dma_async_tx_callback callback)
+{
+	struct dma_async_tx_descriptor *dma_des;
+	unsigned long flags;
+
+	flags = SPRD_DMA_FLAGS(SPRD_DMA_CHN_MODE_NONE,
+			       SPRD_DMA_NO_TRG,
+			       SPRD_DMA_FRAG_REQ,
+			       SPRD_DMA_TRANS_INT);
+
+	dma_des = dmaengine_prep_slave_single(ud->chn, ud->phys_addr, trans_len,
+					      direction, flags);
+	if (!dma_des)
+		return -ENODEV;
+
+	dma_des->callback = callback;
+	dma_des->callback_param = port;
+
+	ud->cookie = dmaengine_submit(dma_des);
+	if (dma_submit_error(ud->cookie))
+		return dma_submit_error(ud->cookie);
+
+	dma_async_issue_pending(ud->chn);
+
+	return 0;
+}
+
+static int sprd_tx_dma_config(struct uart_port *port)
+{
+	struct sprd_uart_port *sp =
+		container_of(port, struct sprd_uart_port, port);
+	u32 burst = sp->tx_dma.trans_len > SPRD_TX_FIFO_FULL ?
+		SPRD_TX_FIFO_FULL : sp->tx_dma.trans_len;
+	int ret;
+	struct dma_slave_config cfg = {
+		.dst_addr = port->mapbase + SPRD_TXD,
+		.src_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE,
+		.dst_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE,
+		.src_maxburst = burst,
+	};
+
+	ret = dmaengine_slave_config(sp->tx_dma.chn, &cfg);
+	if (ret < 0)
+		return ret;
+
+	return sprd_uart_dma_submit(port, &sp->tx_dma, sp->tx_dma.trans_len,
+				    DMA_MEM_TO_DEV, sprd_complete_tx_dma);
+}
+
+static void sprd_start_tx_dma(struct uart_port *port)
+{
+	struct sprd_uart_port *sp =
+		container_of(port, struct sprd_uart_port, port);
+	struct circ_buf *xmit = &port->state->xmit;
+
+	if (port->x_char) {
+		serial_out(port, SPRD_TXD, port->x_char);
+		port->icount.tx++;
+		port->x_char = 0;
+		return;
+	}
+
+	if (uart_circ_empty(xmit) || uart_tx_stopped(port)) {
+		sprd_stop_tx_dma(port);
+		return;
+	}
+
+	if (sp->tx_dma.trans_len)
+		return;
+
+	if (sprd_tx_buf_remap(port) || sprd_tx_dma_config(port))
+		sp->tx_dma.trans_len = 0;
+}
+
+static void sprd_rx_full_thld(struct uart_port *port, u32 thld)
+{
+	u32 val = serial_in(port, SPRD_CTL2);
+
+	val &= ~THLD_RX_FULL_MASK;
+	val |= thld & THLD_RX_FULL_MASK;
+	serial_out(port, SPRD_CTL2, val);
+}
+
+static int sprd_rx_alloc_buf(struct sprd_uart_port *sp)
+{
+	sp->rx_dma.virt = dma_alloc_coherent(sp->port.dev, SPRD_UART_RX_SIZE,
+					     &sp->rx_dma.phys_addr, GFP_KERNEL);
+	if (!sp->rx_dma.virt)
+		return -ENOMEM;
+
+	return 0;
+}
+
+static void sprd_rx_free_buf(struct sprd_uart_port *sp)
+{
+	if (sp->rx_dma.virt)
+		dma_free_coherent(sp->port.dev, SPRD_UART_RX_SIZE,
+				  sp->rx_dma.virt, sp->rx_dma.phys_addr);
+
+}
+
+static int sprd_rx_dma_config(struct uart_port *port, u32 burst)
+{
+	struct sprd_uart_port *sp =
+		container_of(port, struct sprd_uart_port, port);
+	struct dma_slave_config cfg = {
+		.src_addr = port->mapbase + SPRD_RXD,
+		.src_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE,
+		.dst_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE,
+		.src_maxburst = burst,
+	};
+
+	return dmaengine_slave_config(sp->rx_dma.chn, &cfg);
+}
+
+static void sprd_uart_dma_rx(struct uart_port *port)
+{
+	struct sprd_uart_port *sp =
+		container_of(port, struct sprd_uart_port, port);
+	struct tty_port *tty = &port->state->port;
+
+	port->icount.rx += sp->rx_dma.trans_len;
+	tty_insert_flip_string(tty, sp->rx_buf_tail, sp->rx_dma.trans_len);
+	tty_flip_buffer_push(tty);
+}
+
+static void sprd_uart_dma_irq(struct uart_port *port)
+{
+	struct sprd_uart_port *sp =
+		container_of(port, struct sprd_uart_port, port);
+	struct dma_tx_state state;
+	enum dma_status status;
+
+	status = dmaengine_tx_status(sp->rx_dma.chn,
+				     sp->rx_dma.cookie, &state);
+	if (status == DMA_ERROR)
+		sprd_stop_rx(port);
+
+	if (!state.residue && sp->pos == sp->rx_dma.phys_addr)
+		return;
+
+	if (!state.residue) {
+		sp->rx_dma.trans_len = SPRD_UART_RX_SIZE +
+			sp->rx_dma.phys_addr - sp->pos;
+		sp->pos = sp->rx_dma.phys_addr;
+	} else {
+		sp->rx_dma.trans_len = state.residue - sp->pos;
+		sp->pos = state.residue;
+	}
+
+	sprd_uart_dma_rx(port);
+	sp->rx_buf_tail += sp->rx_dma.trans_len;
+}
+
+static void sprd_complete_rx_dma(void *data)
+{
+	struct uart_port *port = (struct uart_port *)data;
+	struct sprd_uart_port *sp =
+		container_of(port, struct sprd_uart_port, port);
+	struct dma_tx_state state;
+	enum dma_status status;
+	unsigned long flags;
+
+	spin_lock_irqsave(&port->lock, flags);
+
+	status = dmaengine_tx_status(sp->rx_dma.chn,
+				     sp->rx_dma.cookie, &state);
+	if (status != DMA_COMPLETE) {
+		sprd_stop_rx(port);
+		spin_unlock_irqrestore(&port->lock, flags);
+		return;
+	}
+
+	if (sp->pos != sp->rx_dma.phys_addr) {
+		sp->rx_dma.trans_len =  SPRD_UART_RX_SIZE +
+			sp->rx_dma.phys_addr - sp->pos;
+		sprd_uart_dma_rx(port);
+		sp->rx_buf_tail += sp->rx_dma.trans_len;
+	}
+
+	if (sprd_start_dma_rx(port))
+		sprd_stop_rx(port);
+
+	spin_unlock_irqrestore(&port->lock, flags);
+}
+
+static int sprd_start_dma_rx(struct uart_port *port)
+{
+	struct sprd_uart_port *sp =
+		container_of(port, struct sprd_uart_port, port);
+	int ret;
+
+	if (!sp->rx_dma.enable)
+		return 0;
+
+	sp->pos = sp->rx_dma.phys_addr;
+	sp->rx_buf_tail = sp->rx_dma.virt;
+	sprd_rx_full_thld(port, SPRD_RX_FIFO_FULL);
+	ret = sprd_rx_dma_config(port, SPRD_RX_DMA_STEP);
+	if (ret)
+		return ret;
+
+	return sprd_uart_dma_submit(port, &sp->rx_dma, SPRD_UART_RX_SIZE,
+				    DMA_DEV_TO_MEM, sprd_complete_rx_dma);
+}
+
+static void sprd_release_dma(struct uart_port *port)
+{
+	struct sprd_uart_port *sp =
+		container_of(port, struct sprd_uart_port, port);
+
+	sprd_uart_dma_enable(port, false);
+
+	if (sp->rx_dma.enable)
+		dma_release_channel(sp->rx_dma.chn);
+
+	if (sp->tx_dma.enable)
+		dma_release_channel(sp->tx_dma.chn);
+
+	sp->tx_dma.enable = false;
+	sp->rx_dma.enable = false;
+}
+
+static void sprd_request_dma(struct uart_port *port)
 {
+	struct sprd_uart_port *sp =
+		container_of(port, struct sprd_uart_port, port);
+
+	sp->tx_dma.enable = true;
+	sp->rx_dma.enable = true;
+
+	sp->tx_dma.chn = dma_request_chan(port->dev, "tx");
+	if (IS_ERR(sp->tx_dma.chn)) {
+		dev_err(port->dev, "request TX DMA channel failed, ret = %ld\n",
+			PTR_ERR(sp->tx_dma.chn));
+		sp->tx_dma.enable = false;
+	}
+
+	sp->rx_dma.chn = dma_request_chan(port->dev, "rx");
+	if (IS_ERR(sp->rx_dma.chn)) {
+		dev_err(port->dev, "request RX DMA channel failed, ret = %ld\n",
+			PTR_ERR(sp->rx_dma.chn));
+		sp->rx_dma.enable = false;
+	}
+}
+
+static void sprd_stop_tx(struct uart_port *port)
+{
+	struct sprd_uart_port *sp = container_of(port, struct sprd_uart_port,
+						 port);
 	unsigned int ien, iclr;
 
+	if (sp->tx_dma.enable) {
+		sprd_stop_tx_dma(port);
+		return;
+	}
+
 	iclr = serial_in(port, SPRD_ICLR);
 	ien = serial_in(port, SPRD_IEN);
 
-	ien &= ~(SPRD_IEN_RX_FULL | SPRD_IEN_BREAK_DETECT);
-	iclr |= SPRD_IEN_RX_FULL | SPRD_IEN_BREAK_DETECT;
+	iclr |= SPRD_IEN_TX_EMPTY;
+	ien &= ~SPRD_IEN_TX_EMPTY;
 
 	serial_out(port, SPRD_IEN, ien);
 	serial_out(port, SPRD_ICLR, iclr);
 }
 
+static void sprd_start_tx(struct uart_port *port)
+{
+	struct sprd_uart_port *sp = container_of(port, struct sprd_uart_port,
+						 port);
+	unsigned int ien;
+
+	if (sp->tx_dma.enable) {
+		sprd_start_tx_dma(port);
+		return;
+	}
+
+	ien = serial_in(port, SPRD_IEN);
+	if (!(ien & SPRD_IEN_TX_EMPTY)) {
+		ien |= SPRD_IEN_TX_EMPTY;
+		serial_out(port, SPRD_IEN, ien);
+	}
+}
+
 /* The Sprd serial does not support this function. */
 static void sprd_break_ctl(struct uart_port *port, int break_state)
 {
@@ -218,9 +590,16 @@ static int handle_lsr_errors(struct uart_port *port,
 
 static inline void sprd_rx(struct uart_port *port)
 {
+	struct sprd_uart_port *sp = container_of(port, struct sprd_uart_port,
+						 port);
 	struct tty_port *tty = &port->state->port;
 	unsigned int ch, flag, lsr, max_count = SPRD_TIMEOUT;
 
+	if (sp->rx_dma.enable) {
+		sprd_uart_dma_irq(port);
+		return;
+	}
+
 	while ((serial_in(port, SPRD_STS1) & SPRD_RX_FIFO_CNT_MASK) &&
 	       max_count--) {
 		lsr = serial_in(port, SPRD_LSR);
@@ -304,6 +683,25 @@ static irqreturn_t sprd_handle_irq(int irq, void *dev_id)
 	return IRQ_HANDLED;
 }
 
+static void sprd_uart_dma_startup(struct uart_port *port,
+				  struct sprd_uart_port *sp)
+{
+	int ret;
+
+	sprd_request_dma(port);
+	if (!(sp->rx_dma.enable || sp->tx_dma.enable))
+		return;
+
+	ret = sprd_start_dma_rx(port);
+	if (ret) {
+		sp->rx_dma.enable = false;
+		dma_release_channel(sp->rx_dma.chn);
+		dev_warn(port->dev, "fail to start RX dma mode\n");
+	}
+
+	sprd_uart_dma_enable(port, true);
+}
+
 static int sprd_startup(struct uart_port *port)
 {
 	int ret = 0;
@@ -332,6 +730,9 @@ static int sprd_startup(struct uart_port *port)
 	/* allocate irq */
 	sp = container_of(port, struct sprd_uart_port, port);
 	snprintf(sp->name, sizeof(sp->name), "sprd_serial%d", port->line);
+
+	sprd_uart_dma_startup(port, sp);
+
 	ret = devm_request_irq(port->dev, port->irq, sprd_handle_irq,
 			       IRQF_SHARED, sp->name, port);
 	if (ret) {
@@ -346,7 +747,9 @@ static int sprd_startup(struct uart_port *port)
 	/* enable interrupt */
 	spin_lock_irqsave(&port->lock, flags);
 	ien = serial_in(port, SPRD_IEN);
-	ien |= SPRD_IEN_RX_FULL | SPRD_IEN_BREAK_DETECT | SPRD_IEN_TIMEOUT;
+	ien |= SPRD_IEN_BREAK_DETECT | SPRD_IEN_TIMEOUT;
+	if (!sp->rx_dma.enable)
+		ien |= SPRD_IEN_RX_FULL;
 	serial_out(port, SPRD_IEN, ien);
 	spin_unlock_irqrestore(&port->lock, flags);
 
@@ -355,6 +758,7 @@ static int sprd_startup(struct uart_port *port)
 
 static void sprd_shutdown(struct uart_port *port)
 {
+	sprd_release_dma(port);
 	serial_out(port, SPRD_IEN, 0);
 	serial_out(port, SPRD_ICLR, ~0);
 	devm_free_irq(port->dev, port->irq, port);
@@ -491,6 +895,22 @@ static int sprd_verify_port(struct uart_port *port, struct serial_struct *ser)
 	return 0;
 }
 
+static void sprd_pm(struct uart_port *port, unsigned int state,
+		unsigned int oldstate)
+{
+	struct sprd_uart_port *sup =
+		container_of(port, struct sprd_uart_port, port);
+
+	switch (state) {
+	case UART_PM_STATE_ON:
+		clk_prepare_enable(sup->clk);
+		break;
+	case UART_PM_STATE_OFF:
+		clk_disable_unprepare(sup->clk);
+		break;
+	}
+}
+
 static const struct uart_ops serial_sprd_ops = {
 	.tx_empty = sprd_tx_empty,
 	.get_mctrl = sprd_get_mctrl,
@@ -507,6 +927,7 @@ static const struct uart_ops serial_sprd_ops = {
 	.request_port = sprd_request_port,
 	.config_port = sprd_config_port,
 	.verify_port = sprd_verify_port,
+	.pm = sprd_pm,
 };
 
 #ifdef CONFIG_SERIAL_SPRD_CONSOLE
@@ -668,6 +1089,43 @@ static int sprd_remove(struct platform_device *dev)
 	if (!sprd_ports_num)
 		uart_unregister_driver(&sprd_uart_driver);
 
+	sprd_rx_free_buf(sup);
+
+	return 0;
+}
+
+static int sprd_clk_init(struct uart_port *uport)
+{
+	struct clk *clk_uart, *clk_parent;
+	struct sprd_uart_port *u = sprd_port[uport->line];
+
+	clk_uart = devm_clk_get(uport->dev, "uart");
+	if (IS_ERR(clk_uart)) {
+		dev_warn(uport->dev, "uart%d can't get uart clock\n",
+			 uport->line);
+		clk_uart = NULL;
+	}
+
+	clk_parent = devm_clk_get(uport->dev, "source");
+	if (IS_ERR(clk_parent)) {
+		dev_warn(uport->dev, "uart%d can't get source clock\n",
+			 uport->line);
+		clk_parent = NULL;
+	}
+
+	if (!clk_uart || clk_set_parent(clk_uart, clk_parent))
+		uport->uartclk = SPRD_DEFAULT_SOURCE_CLK;
+	else
+		uport->uartclk = clk_get_rate(clk_uart);
+
+	u->clk = devm_clk_get(uport->dev, "enable");
+	if (IS_ERR(u->clk)) {
+		if (PTR_ERR(u->clk) != -EPROBE_DEFER)
+			dev_err(uport->dev, "uart%d can't get enable clock\n",
+				uport->line);
+		return PTR_ERR(u->clk);
+	}
+
 	return 0;
 }
 
@@ -675,7 +1133,6 @@ static int sprd_probe(struct platform_device *pdev)
 {
 	struct resource *res;
 	struct uart_port *up;
-	struct clk *clk;
 	int irq;
 	int index;
 	int ret;
@@ -704,9 +1161,9 @@ static int sprd_probe(struct platform_device *pdev)
 	up->ops = &serial_sprd_ops;
 	up->flags = UPF_BOOT_AUTOCONF;
 
-	clk = devm_clk_get(&pdev->dev, NULL);
-	if (!IS_ERR_OR_NULL(clk))
-		up->uartclk = clk_get_rate(clk);
+	ret = sprd_clk_init(up);
+	if (ret)
+		return ret;
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	up->membase = devm_ioremap_resource(&pdev->dev, res);
@@ -722,6 +1179,14 @@ static int sprd_probe(struct platform_device *pdev)
 	}
 	up->irq = irq;
 
+	/*
+	 * Allocate one dma buffer to prepare for receive transfer, in case
+	 * memory allocation failure at runtime.
+	 */
+	ret = sprd_rx_alloc_buf(sprd_port[index]);
+	if (ret)
+		return ret;
+
 	if (!sprd_ports_num) {
 		ret = uart_register_driver(&sprd_uart_driver);
 		if (ret < 0) {
diff --git a/drivers/tty/serial/ucc_uart.c b/drivers/tty/serial/ucc_uart.c
index 2b6376e6e5ad..6e3c66ab0e62 100644
--- a/drivers/tty/serial/ucc_uart.c
+++ b/drivers/tty/serial/ucc_uart.c
@@ -1081,7 +1081,7 @@ static int qe_uart_verify_port(struct uart_port *port,
 }
 /* UART operations
  *
- * Details on these functions can be found in Documentation/serial/driver
+ * Details on these functions can be found in Documentation/serial/driver.rst
  */
 static const struct uart_ops qe_uart_pops = {
 	.tx_empty       = qe_uart_tx_empty,
diff --git a/drivers/tty/serial/xilinx_uartps.c b/drivers/tty/serial/xilinx_uartps.c
index 74089f5e5b53..605354fd60b1 100644
--- a/drivers/tty/serial/xilinx_uartps.c
+++ b/drivers/tty/serial/xilinx_uartps.c
@@ -193,6 +193,7 @@ struct cdns_uart {
 	int			id;
 	struct notifier_block	clk_rate_change_nb;
 	u32			quirks;
+	bool cts_override;
 };
 struct cdns_platform_data {
 	u32 quirks;
@@ -1000,6 +1001,11 @@ static void cdns_uart_config_port(struct uart_port *port, int flags)
  */
 static unsigned int cdns_uart_get_mctrl(struct uart_port *port)
 {
+	struct cdns_uart *cdns_uart_data = port->private_data;
+
+	if (cdns_uart_data->cts_override)
+		return 0;
+
 	return TIOCM_CTS | TIOCM_DSR | TIOCM_CAR;
 }
 
@@ -1007,6 +1013,10 @@ static void cdns_uart_set_mctrl(struct uart_port *port, unsigned int mctrl)
 {
 	u32 val;
 	u32 mode_reg;
+	struct cdns_uart *cdns_uart_data = port->private_data;
+
+	if (cdns_uart_data->cts_override)
+		return;
 
 	val = readl(port->membase + CDNS_UART_MODEMCR);
 	mode_reg = readl(port->membase + CDNS_UART_MR);
@@ -1665,6 +1675,8 @@ static int cdns_uart_probe(struct platform_device *pdev)
 		console_port = NULL;
 #endif
 
+	cdns_uart_data->cts_override = of_property_read_bool(pdev->dev.of_node,
+							     "cts-override");
 	return 0;
 
 err_out_pm_disable:
diff --git a/drivers/tty/sysrq.c b/drivers/tty/sysrq.c
index fa0ce7dd9e24..59e82e6d776d 100644
--- a/drivers/tty/sysrq.c
+++ b/drivers/tty/sysrq.c
@@ -208,7 +208,7 @@ static struct sysrq_key_op sysrq_showlocks_op = {
 #endif
 
 #ifdef CONFIG_SMP
-static DEFINE_SPINLOCK(show_lock);
+static DEFINE_RAW_SPINLOCK(show_lock);
 
 static void showacpu(void *dummy)
 {
@@ -218,10 +218,10 @@ static void showacpu(void *dummy)
 	if (idle_cpu(smp_processor_id()))
 		return;
 
-	spin_lock_irqsave(&show_lock, flags);
+	raw_spin_lock_irqsave(&show_lock, flags);
 	pr_info("CPU%d:\n", smp_processor_id());
 	show_stack(NULL, NULL);
-	spin_unlock_irqrestore(&show_lock, flags);
+	raw_spin_unlock_irqrestore(&show_lock, flags);
 }
 
 static void sysrq_showregs_othercpus(struct work_struct *dummy)
diff --git a/drivers/tty/tty_io.c b/drivers/tty/tty_io.c
index 5fa250157025..033ac7e6a70d 100644
--- a/drivers/tty/tty_io.c
+++ b/drivers/tty/tty_io.c
@@ -1173,7 +1173,7 @@ static struct tty_struct *tty_driver_lookup_tty(struct tty_driver *driver,
  *	tty_init_termios	-  helper for termios setup
  *	@tty: the tty to set up
  *
- *	Initialise the termios structures for this tty. Thus runs under
+ *	Initialise the termios structure for this tty. This runs under
  *	the tty_mutex currently so we can be relaxed about ordering.
  */
 
diff --git a/drivers/tty/tty_jobctrl.c b/drivers/tty/tty_jobctrl.c
index c4ecd66fafef..f8ed50a16848 100644
--- a/drivers/tty/tty_jobctrl.c
+++ b/drivers/tty/tty_jobctrl.c
@@ -44,7 +44,7 @@ int __tty_check_change(struct tty_struct *tty, int sig)
 	tty_pgrp = tty->pgrp;
 	spin_unlock_irqrestore(&tty->ctrl_lock, flags);
 
-	if (tty_pgrp && pgrp != tty->pgrp) {
+	if (tty_pgrp && pgrp != tty_pgrp) {
 		if (is_ignored(sig)) {
 			if (sig == SIGTTIN)
 				ret = -EIO;
@@ -313,7 +313,7 @@ void disassociate_ctty(int on_exit)
 	read_unlock(&tasklist_lock);
 }
 
-/**
+/*
  *
  *	no_tty	- Ensure the current process does not have a controlling tty
  */
diff --git a/drivers/tty/tty_port.c b/drivers/tty/tty_port.c
index a9e12b3bc31d..044c3cbdcfa4 100644
--- a/drivers/tty/tty_port.c
+++ b/drivers/tty/tty_port.c
@@ -325,7 +325,7 @@ static void tty_port_shutdown(struct tty_port *port, struct tty_struct *tty)
 		if (tty && C_HUPCL(tty))
 			tty_port_lower_dtr_rts(port);
 
-		if (port->ops && port->ops->shutdown)
+		if (port->ops->shutdown)
 			port->ops->shutdown(port);
 	}
 out:
@@ -398,7 +398,7 @@ EXPORT_SYMBOL_GPL(tty_port_tty_wakeup);
  */
 int tty_port_carrier_raised(struct tty_port *port)
 {
-	if (!port->ops || !port->ops->carrier_raised)
+	if (port->ops->carrier_raised == NULL)
 		return 1;
 	return port->ops->carrier_raised(port);
 }
@@ -414,7 +414,7 @@ EXPORT_SYMBOL(tty_port_carrier_raised);
  */
 void tty_port_raise_dtr_rts(struct tty_port *port)
 {
-	if (port->ops && port->ops->dtr_rts)
+	if (port->ops->dtr_rts)
 		port->ops->dtr_rts(port, 1);
 }
 EXPORT_SYMBOL(tty_port_raise_dtr_rts);
@@ -429,7 +429,7 @@ EXPORT_SYMBOL(tty_port_raise_dtr_rts);
  */
 void tty_port_lower_dtr_rts(struct tty_port *port)
 {
-	if (port->ops && port->ops->dtr_rts)
+	if (port->ops->dtr_rts)
 		port->ops->dtr_rts(port, 0);
 }
 EXPORT_SYMBOL(tty_port_lower_dtr_rts);
@@ -684,7 +684,7 @@ int tty_port_open(struct tty_port *port, struct tty_struct *tty,
 
 	if (!tty_port_initialized(port)) {
 		clear_bit(TTY_IO_ERROR, &tty->flags);
-		if (port->ops && port->ops->activate) {
+		if (port->ops->activate) {
 			int retval = port->ops->activate(port, tty);
 			if (retval) {
 				mutex_unlock(&port->mutex);
diff --git a/drivers/tty/ttynull.c b/drivers/tty/ttynull.c
new file mode 100644
index 000000000000..17f05b7eb6d3
--- /dev/null
+++ b/drivers/tty/ttynull.c
@@ -0,0 +1,109 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2019 Axis Communications AB
+ *
+ * Based on ttyprintk.c:
+ *  Copyright (C) 2010 Samo Pogacnik
+ */
+
+#include <linux/console.h>
+#include <linux/module.h>
+#include <linux/tty.h>
+
+static const struct tty_port_operations ttynull_port_ops;
+static struct tty_driver *ttynull_driver;
+static struct tty_port ttynull_port;
+
+static int ttynull_open(struct tty_struct *tty, struct file *filp)
+{
+	return tty_port_open(&ttynull_port, tty, filp);
+}
+
+static void ttynull_close(struct tty_struct *tty, struct file *filp)
+{
+	tty_port_close(&ttynull_port, tty, filp);
+}
+
+static void ttynull_hangup(struct tty_struct *tty)
+{
+	tty_port_hangup(&ttynull_port);
+}
+
+static int ttynull_write(struct tty_struct *tty, const unsigned char *buf,
+			 int count)
+{
+	return count;
+}
+
+static int ttynull_write_room(struct tty_struct *tty)
+{
+	return 65536;
+}
+
+static const struct tty_operations ttynull_ops = {
+	.open = ttynull_open,
+	.close = ttynull_close,
+	.hangup = ttynull_hangup,
+	.write = ttynull_write,
+	.write_room = ttynull_write_room,
+};
+
+static struct tty_driver *ttynull_device(struct console *c, int *index)
+{
+	*index = 0;
+	return ttynull_driver;
+}
+
+static struct console ttynull_console = {
+	.name = "ttynull",
+	.device = ttynull_device,
+};
+
+static int __init ttynull_init(void)
+{
+	struct tty_driver *driver;
+	int ret;
+
+	driver = tty_alloc_driver(1,
+		TTY_DRIVER_RESET_TERMIOS |
+		TTY_DRIVER_REAL_RAW |
+		TTY_DRIVER_UNNUMBERED_NODE);
+	if (IS_ERR(driver))
+		return PTR_ERR(driver);
+
+	tty_port_init(&ttynull_port);
+	ttynull_port.ops = &ttynull_port_ops;
+
+	driver->driver_name = "ttynull";
+	driver->name = "ttynull";
+	driver->type = TTY_DRIVER_TYPE_CONSOLE;
+	driver->init_termios = tty_std_termios;
+	driver->init_termios.c_oflag = OPOST | OCRNL | ONOCR | ONLRET;
+	tty_set_operations(driver, &ttynull_ops);
+	tty_port_link_device(&ttynull_port, driver, 0);
+
+	ret = tty_register_driver(driver);
+	if (ret < 0) {
+		put_tty_driver(driver);
+		tty_port_destroy(&ttynull_port);
+		return ret;
+	}
+
+	ttynull_driver = driver;
+	register_console(&ttynull_console);
+
+	return 0;
+}
+
+static void __exit ttynull_exit(void)
+{
+	unregister_console(&ttynull_console);
+	tty_unregister_driver(ttynull_driver);
+	put_tty_driver(ttynull_driver);
+	tty_port_destroy(&ttynull_port);
+}
+
+module_init(ttynull_init);
+module_exit(ttynull_exit);
+
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/tty/vcc.c b/drivers/tty/vcc.c
index 58b454c34560..d2a1e1228c82 100644
--- a/drivers/tty/vcc.c
+++ b/drivers/tty/vcc.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /* vcc.c: sun4v virtual channel concentrator
  *
  * Copyright (C) 2017 Oracle. All rights reserved.
diff --git a/drivers/tty/vt/.gitignore b/drivers/tty/vt/.gitignore
index 83683a2d8e6a..9b38b85f9d9a 100644
--- a/drivers/tty/vt/.gitignore
+++ b/drivers/tty/vt/.gitignore
@@ -1,2 +1,3 @@
+# SPDX-License-Identifier: GPL-2.0
 consolemap_deftbl.c
 defkeymap.c
diff --git a/drivers/tty/vt/consolemap.c b/drivers/tty/vt/consolemap.c
index 7c7ada0b3ea0..b28aa0d289f8 100644
--- a/drivers/tty/vt/consolemap.c
+++ b/drivers/tty/vt/consolemap.c
@@ -542,7 +542,7 @@ int con_set_unimap(struct vc_data *vc, ushort ct, struct unipair __user *list)
 	if (!ct)
 		return 0;
 
-	unilist = memdup_user(list, ct * sizeof(struct unipair));
+	unilist = vmemdup_user(list, ct * sizeof(struct unipair));
 	if (IS_ERR(unilist))
 		return PTR_ERR(unilist);
 
@@ -641,7 +641,7 @@ int con_set_unimap(struct vc_data *vc, ushort ct, struct unipair __user *list)
 
 out_unlock:
 	console_unlock();
-	kfree(unilist);
+	kvfree(unilist);
 	return err;
 }
 
@@ -743,7 +743,7 @@ int con_get_unimap(struct vc_data *vc, ushort ct, ushort __user *uct, struct uni
 	struct uni_pagedir *p;
 	struct unipair *unilist;
 
-	unilist = kmalloc_array(ct, sizeof(struct unipair), GFP_KERNEL);
+	unilist = kvmalloc_array(ct, sizeof(struct unipair), GFP_KERNEL);
 	if (!unilist)
 		return -ENOMEM;
 
@@ -775,7 +775,7 @@ int con_get_unimap(struct vc_data *vc, ushort ct, ushort __user *uct, struct uni
 	if (copy_to_user(list, unilist, min(ect, ct) * sizeof(struct unipair)))
 		ret = -EFAULT;
 	put_user(ect, uct);
-	kfree(unilist);
+	kvfree(unilist);
 	return ret ? ret : (ect <= ct) ? 0 : -ENOMEM;
 }
 
diff --git a/drivers/tty/vt/cp437.uni b/drivers/tty/vt/cp437.uni
index bc6163484f62..a1991904c559 100644
--- a/drivers/tty/vt/cp437.uni
+++ b/drivers/tty/vt/cp437.uni
@@ -1,3 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
 #
 # Unicode table for IBM Codepage 437.  Note that there are many more
 # substitutions that could be conceived (for example, thick-line
diff --git a/drivers/tty/vt/defkeymap.c_shipped b/drivers/tty/vt/defkeymap.c_shipped
index d2208dfe3f67..c7095fb7d2d1 100644
--- a/drivers/tty/vt/defkeymap.c_shipped
+++ b/drivers/tty/vt/defkeymap.c_shipped
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /* Do not edit this file! It was automatically generated by   */
 /*    loadkeys --mktable defkeymap.map > defkeymap.c          */
 
diff --git a/drivers/tty/vt/defkeymap.map b/drivers/tty/vt/defkeymap.map
index 50b30cace261..37f1ac6ddfb9 100644
--- a/drivers/tty/vt/defkeymap.map
+++ b/drivers/tty/vt/defkeymap.map
@@ -1,3 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
 # Default kernel keymap. This uses 7 modifier combinations.
 keymaps 0-2,4-5,8,12
 # Change the above line into
diff --git a/drivers/tty/vt/keyboard.c b/drivers/tty/vt/keyboard.c
index 88312c6c92cc..515fc095e3b4 100644
--- a/drivers/tty/vt/keyboard.c
+++ b/drivers/tty/vt/keyboard.c
@@ -123,6 +123,7 @@ static const int NR_TYPES = ARRAY_SIZE(max_vals);
 static struct input_handler kbd_handler;
 static DEFINE_SPINLOCK(kbd_event_lock);
 static DEFINE_SPINLOCK(led_lock);
+static DEFINE_SPINLOCK(func_buf_lock); /* guard 'func_buf'  and friends */
 static unsigned long key_down[BITS_TO_LONGS(KEY_CNT)];	/* keyboard key bitmap */
 static unsigned char shift_down[NR_SHIFT];		/* shift state counters.. */
 static bool dead_key_next;
@@ -1449,7 +1450,7 @@ static void kbd_keycode(unsigned int keycode, int down, int hw_raw)
 						KBD_UNICODE, &param);
 		if (rc != NOTIFY_STOP)
 			if (down && !raw_mode)
-				to_utf8(vc, keysym);
+				k_unicode(vc, keysym, !down);
 		return;
 	}
 
@@ -1990,11 +1991,12 @@ int vt_do_kdgkb_ioctl(int cmd, struct kbsentry __user *user_kdgkb, int perm)
 	char *p;
 	u_char *q;
 	u_char __user *up;
-	int sz;
+	int sz, fnw_sz;
 	int delta;
 	char *first_free, *fj, *fnw;
 	int i, j, k;
 	int ret;
+	unsigned long flags;
 
 	if (!capable(CAP_SYS_TTY_CONFIG))
 		perm = 0;
@@ -2037,7 +2039,14 @@ int vt_do_kdgkb_ioctl(int cmd, struct kbsentry __user *user_kdgkb, int perm)
 			goto reterr;
 		}
 
+		fnw = NULL;
+		fnw_sz = 0;
+		/* race aginst other writers */
+		again:
+		spin_lock_irqsave(&func_buf_lock, flags);
 		q = func_table[i];
+
+		/* fj pointer to next entry after 'q' */
 		first_free = funcbufptr + (funcbufsize - funcbufleft);
 		for (j = i+1; j < MAX_NR_FUNC && !func_table[j]; j++)
 			;
@@ -2045,10 +2054,12 @@ int vt_do_kdgkb_ioctl(int cmd, struct kbsentry __user *user_kdgkb, int perm)
 			fj = func_table[j];
 		else
 			fj = first_free;
-
+		/* buffer usage increase by new entry */
 		delta = (q ? -strlen(q) : 1) + strlen(kbs->kb_string);
+
 		if (delta <= funcbufleft) { 	/* it fits in current buf */
 		    if (j < MAX_NR_FUNC) {
+			/* make enough space for new entry at 'fj' */
 			memmove(fj + delta, fj, first_free - fj);
 			for (k = j; k < MAX_NR_FUNC; k++)
 			    if (func_table[k])
@@ -2061,20 +2072,28 @@ int vt_do_kdgkb_ioctl(int cmd, struct kbsentry __user *user_kdgkb, int perm)
 		    sz = 256;
 		    while (sz < funcbufsize - funcbufleft + delta)
 		      sz <<= 1;
-		    fnw = kmalloc(sz, GFP_KERNEL);
-		    if(!fnw) {
-		      ret = -ENOMEM;
-		      goto reterr;
+		    if (fnw_sz != sz) {
+		      spin_unlock_irqrestore(&func_buf_lock, flags);
+		      kfree(fnw);
+		      fnw = kmalloc(sz, GFP_KERNEL);
+		      fnw_sz = sz;
+		      if (!fnw) {
+			ret = -ENOMEM;
+			goto reterr;
+		      }
+		      goto again;
 		    }
 
 		    if (!q)
 		      func_table[i] = fj;
+		    /* copy data before insertion point to new location */
 		    if (fj > funcbufptr)
 			memmove(fnw, funcbufptr, fj - funcbufptr);
 		    for (k = 0; k < j; k++)
 		      if (func_table[k])
 			func_table[k] = fnw + (func_table[k] - funcbufptr);
 
+		    /* copy data after insertion point to new location */
 		    if (first_free > fj) {
 			memmove(fnw + (fj - funcbufptr) + delta, fj, first_free - fj);
 			for (k = j; k < MAX_NR_FUNC; k++)
@@ -2087,7 +2106,9 @@ int vt_do_kdgkb_ioctl(int cmd, struct kbsentry __user *user_kdgkb, int perm)
 		    funcbufleft = funcbufleft - delta + sz - funcbufsize;
 		    funcbufsize = sz;
 		}
+		/* finally insert item itself */
 		strcpy(func_table[i], kbs->kb_string);
+		spin_unlock_irqrestore(&func_buf_lock, flags);
 		break;
 	}
 	ret = 0;
diff --git a/drivers/tty/vt/vc_screen.c b/drivers/tty/vt/vc_screen.c
index 160f46115aaa..1f042346e722 100644
--- a/drivers/tty/vt/vc_screen.c
+++ b/drivers/tty/vt/vc_screen.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
  * Provide access to virtual console memory.
- * /dev/vcs0: the screen as it is being viewed right now (possibly scrolled)
+ * /dev/vcs: the screen as it is being viewed right now (possibly scrolled)
  * /dev/vcsN: the screen of /dev/ttyN (1 <= N <= 63)
  *            [minor: N]
  *
diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c
index fb6f1c93e5c2..fdd12f8c3deb 100644
--- a/drivers/tty/vt/vt.c
+++ b/drivers/tty/vt/vt.c
@@ -4180,8 +4180,6 @@ void do_blank_screen(int entering_gfx)
 		return;
 	}
 
-	if (blank_state != blank_normal_wait)
-		return;
 	blank_state = blank_off;
 
 	/* don't blank graphics */
diff --git a/drivers/usb/chipidea/ci_hdrc_msm.c b/drivers/usb/chipidea/ci_hdrc_msm.c
index 880009987460..b8b3caad889c 100644
--- a/drivers/usb/chipidea/ci_hdrc_msm.c
+++ b/drivers/usb/chipidea/ci_hdrc_msm.c
@@ -205,12 +205,9 @@ static int ci_hdrc_msm_probe(struct platform_device *pdev)
 	if (IS_ERR(clk))
 		return PTR_ERR(clk);
 
-	ci->fs_clk = clk = devm_clk_get(&pdev->dev, "fs");
-	if (IS_ERR(clk)) {
-		if (PTR_ERR(clk) == -EPROBE_DEFER)
-			return -EPROBE_DEFER;
-		ci->fs_clk = NULL;
-	}
+	ci->fs_clk = clk = devm_clk_get_optional(&pdev->dev, "fs");
+	if (IS_ERR(clk))
+		return PTR_ERR(clk);
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
 	ci->base = devm_ioremap_resource(&pdev->dev, res);
diff --git a/drivers/usb/class/cdc-acm.c b/drivers/usb/class/cdc-acm.c
index ec666eb4b7b4..183b41753c98 100644
--- a/drivers/usb/class/cdc-acm.c
+++ b/drivers/usb/class/cdc-acm.c
@@ -468,14 +468,13 @@ static void acm_read_bulk_callback(struct urb *urb)
 {
 	struct acm_rb *rb = urb->context;
 	struct acm *acm = rb->instance;
-	unsigned long flags;
 	int status = urb->status;
+	bool stopped = false;
+	bool stalled = false;
 
 	dev_vdbg(&acm->data->dev, "got urb %d, len %d, status %d\n",
 		rb->index, urb->actual_length, status);
 
-	set_bit(rb->index, &acm->read_urbs_free);
-
 	if (!acm->dev) {
 		dev_dbg(&acm->data->dev, "%s - disconnected\n", __func__);
 		return;
@@ -488,15 +487,16 @@ static void acm_read_bulk_callback(struct urb *urb)
 		break;
 	case -EPIPE:
 		set_bit(EVENT_RX_STALL, &acm->flags);
-		schedule_work(&acm->work);
-		return;
+		stalled = true;
+		break;
 	case -ENOENT:
 	case -ECONNRESET:
 	case -ESHUTDOWN:
 		dev_dbg(&acm->data->dev,
 			"%s - urb shutting down with status: %d\n",
 			__func__, status);
-		return;
+		stopped = true;
+		break;
 	default:
 		dev_dbg(&acm->data->dev,
 			"%s - nonzero urb status received: %d\n",
@@ -505,20 +505,29 @@ static void acm_read_bulk_callback(struct urb *urb)
 	}
 
 	/*
-	 * Unthrottle may run on another CPU which needs to see events
-	 * in the same order. Submission has an implict barrier
+	 * Make sure URB processing is done before marking as free to avoid
+	 * racing with unthrottle() on another CPU. Matches the barriers
+	 * implied by the test_and_clear_bit() in acm_submit_read_urb().
 	 */
 	smp_mb__before_atomic();
+	set_bit(rb->index, &acm->read_urbs_free);
+	/*
+	 * Make sure URB is marked as free before checking the throttled flag
+	 * to avoid racing with unthrottle() on another CPU. Matches the
+	 * smp_mb() in unthrottle().
+	 */
+	smp_mb__after_atomic();
 
-	/* throttle device if requested by tty */
-	spin_lock_irqsave(&acm->read_lock, flags);
-	acm->throttled = acm->throttle_req;
-	if (!acm->throttled) {
-		spin_unlock_irqrestore(&acm->read_lock, flags);
-		acm_submit_read_urb(acm, rb->index, GFP_ATOMIC);
-	} else {
-		spin_unlock_irqrestore(&acm->read_lock, flags);
+	if (stopped || stalled) {
+		if (stalled)
+			schedule_work(&acm->work);
+		return;
 	}
+
+	if (test_bit(ACM_THROTTLED, &acm->flags))
+		return;
+
+	acm_submit_read_urb(acm, rb->index, GFP_ATOMIC);
 }
 
 /* data interface wrote those outgoing bytes */
@@ -655,10 +664,7 @@ static int acm_port_activate(struct tty_port *port, struct tty_struct *tty)
 	/*
 	 * Unthrottle device in case the TTY was closed while throttled.
 	 */
-	spin_lock_irq(&acm->read_lock);
-	acm->throttled = 0;
-	acm->throttle_req = 0;
-	spin_unlock_irq(&acm->read_lock);
+	clear_bit(ACM_THROTTLED, &acm->flags);
 
 	retval = acm_submit_read_urbs(acm, GFP_KERNEL);
 	if (retval)
@@ -826,24 +832,19 @@ static void acm_tty_throttle(struct tty_struct *tty)
 {
 	struct acm *acm = tty->driver_data;
 
-	spin_lock_irq(&acm->read_lock);
-	acm->throttle_req = 1;
-	spin_unlock_irq(&acm->read_lock);
+	set_bit(ACM_THROTTLED, &acm->flags);
 }
 
 static void acm_tty_unthrottle(struct tty_struct *tty)
 {
 	struct acm *acm = tty->driver_data;
-	unsigned int was_throttled;
 
-	spin_lock_irq(&acm->read_lock);
-	was_throttled = acm->throttled;
-	acm->throttled = 0;
-	acm->throttle_req = 0;
-	spin_unlock_irq(&acm->read_lock);
+	clear_bit(ACM_THROTTLED, &acm->flags);
+
+	/* Matches the smp_mb__after_atomic() in acm_read_bulk_callback(). */
+	smp_mb();
 
-	if (was_throttled)
-		acm_submit_read_urbs(acm, GFP_KERNEL);
+	acm_submit_read_urbs(acm, GFP_KERNEL);
 }
 
 static int acm_tty_break_ctl(struct tty_struct *tty, int state)
diff --git a/drivers/usb/class/cdc-acm.h b/drivers/usb/class/cdc-acm.h
index 515aad0847ee..ca1c026382c2 100644
--- a/drivers/usb/class/cdc-acm.h
+++ b/drivers/usb/class/cdc-acm.h
@@ -108,6 +108,7 @@ struct acm {
 	unsigned long flags;
 #		define EVENT_TTY_WAKEUP	0
 #		define EVENT_RX_STALL	1
+#		define ACM_THROTTLED	2
 	struct usb_cdc_line_coding line;		/* bits, stop, parity */
 	struct work_struct work;			/* work queue entry for line discipline waking up */
 	unsigned int ctrlin;				/* input control lines (DCD, DSR, RI, break, overruns) */
@@ -122,8 +123,6 @@ struct acm {
 	unsigned int ctrl_caps;				/* control capabilities from the class specific header */
 	unsigned int susp_count;			/* number of suspended interfaces */
 	unsigned int combined_interfaces:1;		/* control and data collapsed */
-	unsigned int throttled:1;			/* actually throttled */
-	unsigned int throttle_req:1;			/* throttle requested */
 	u8 bInterval;
 	struct usb_anchor delayed;			/* writes queued for a device about to be woken */
 	unsigned long quirks;
diff --git a/drivers/usb/common/common.c b/drivers/usb/common/common.c
index 73c8e6591746..18f5dcf58b0d 100644
--- a/drivers/usb/common/common.c
+++ b/drivers/usb/common/common.c
@@ -16,6 +16,22 @@
 #include <linux/usb/otg.h>
 #include <linux/of_platform.h>
 
+static const char *const ep_type_names[] = {
+	[USB_ENDPOINT_XFER_CONTROL] = "ctrl",
+	[USB_ENDPOINT_XFER_ISOC] = "isoc",
+	[USB_ENDPOINT_XFER_BULK] = "bulk",
+	[USB_ENDPOINT_XFER_INT] = "intr",
+};
+
+const char *usb_ep_type_string(int ep_type)
+{
+	if (ep_type < 0 || ep_type >= ARRAY_SIZE(ep_type_names))
+		return "unknown";
+
+	return ep_type_names[ep_type];
+}
+EXPORT_SYMBOL_GPL(usb_ep_type_string);
+
 const char *usb_otg_state_string(enum usb_otg_state state)
 {
 	static const char *const names[] = {
diff --git a/drivers/usb/core/hcd.c b/drivers/usb/core/hcd.c
index 975d7c1288e3..94d22551fc1b 100644
--- a/drivers/usb/core/hcd.c
+++ b/drivers/usb/core/hcd.c
@@ -1878,23 +1878,10 @@ rescan:
 		/* kick hcd */
 		unlink1(hcd, urb, -ESHUTDOWN);
 		dev_dbg (hcd->self.controller,
-			"shutdown urb %pK ep%d%s%s\n",
+			"shutdown urb %pK ep%d%s-%s\n",
 			urb, usb_endpoint_num(&ep->desc),
 			is_in ? "in" : "out",
-			({	char *s;
-
-				 switch (usb_endpoint_type(&ep->desc)) {
-				 case USB_ENDPOINT_XFER_CONTROL:
-					s = ""; break;
-				 case USB_ENDPOINT_XFER_BULK:
-					s = "-bulk"; break;
-				 case USB_ENDPOINT_XFER_INT:
-					s = "-intr"; break;
-				 default:
-					s = "-iso"; break;
-				};
-				s;
-			}));
+			usb_ep_type_string(usb_endpoint_type(&ep->desc)));
 		usb_put_urb (urb);
 
 		/* list contents may have changed */
@@ -2448,6 +2435,19 @@ EXPORT_SYMBOL_GPL(usb_hcd_irq);
 
 /*-------------------------------------------------------------------------*/
 
+/* Workqueue routine for when the root-hub has died. */
+static void hcd_died_work(struct work_struct *work)
+{
+	struct usb_hcd *hcd = container_of(work, struct usb_hcd, died_work);
+	static char *env[] = {
+		"ERROR=DEAD",
+		NULL
+	};
+
+	/* Notify user space that the host controller has died */
+	kobject_uevent_env(&hcd->self.root_hub->dev.kobj, KOBJ_OFFLINE, env);
+}
+
 /**
  * usb_hc_died - report abnormal shutdown of a host controller (bus glue)
  * @hcd: pointer to the HCD representing the controller
@@ -2488,6 +2488,13 @@ void usb_hc_died (struct usb_hcd *hcd)
 			usb_kick_hub_wq(hcd->self.root_hub);
 		}
 	}
+
+	/* Handle the case where this function gets called with a shared HCD */
+	if (usb_hcd_is_primary_hcd(hcd))
+		schedule_work(&hcd->died_work);
+	else
+		schedule_work(&hcd->primary_hcd->died_work);
+
 	spin_unlock_irqrestore (&hcd_root_hub_lock, flags);
 	/* Make sure that the other roothub is also deallocated. */
 }
@@ -2555,6 +2562,8 @@ struct usb_hcd *__usb_create_hcd(const struct hc_driver *driver,
 	INIT_WORK(&hcd->wakeup_work, hcd_resume_work);
 #endif
 
+	INIT_WORK(&hcd->died_work, hcd_died_work);
+
 	hcd->driver = driver;
 	hcd->speed = driver->flags & HCD_MASK;
 	hcd->product_desc = (driver->product_desc) ? driver->product_desc :
@@ -2908,6 +2917,7 @@ error_create_attr_group:
 #ifdef CONFIG_PM
 	cancel_work_sync(&hcd->wakeup_work);
 #endif
+	cancel_work_sync(&hcd->died_work);
 	mutex_lock(&usb_bus_idr_lock);
 	usb_disconnect(&rhdev);		/* Sets rhdev to NULL */
 	mutex_unlock(&usb_bus_idr_lock);
@@ -2968,6 +2978,7 @@ void usb_remove_hcd(struct usb_hcd *hcd)
 #ifdef CONFIG_PM
 	cancel_work_sync(&hcd->wakeup_work);
 #endif
+	cancel_work_sync(&hcd->died_work);
 
 	mutex_lock(&usb_bus_idr_lock);
 	usb_disconnect(&rhdev);		/* Sets rhdev to NULL */
@@ -3020,6 +3031,9 @@ usb_hcd_platform_shutdown(struct platform_device *dev)
 {
 	struct usb_hcd *hcd = platform_get_drvdata(dev);
 
+	/* No need for pm_runtime_put(), we're shutting down */
+	pm_runtime_get_sync(&dev->dev);
+
 	if (hcd->driver->shutdown)
 		hcd->driver->shutdown(hcd);
 }
diff --git a/drivers/usb/core/hub.c b/drivers/usb/core/hub.c
index 8d4631c81b9f..2f94568ba385 100644
--- a/drivers/usb/core/hub.c
+++ b/drivers/usb/core/hub.c
@@ -3174,13 +3174,14 @@ static int usb_disable_remote_wakeup(struct usb_device *udev)
 }
 
 /* Count of wakeup-enabled devices at or below udev */
-static unsigned wakeup_enabled_descendants(struct usb_device *udev)
+unsigned usb_wakeup_enabled_descendants(struct usb_device *udev)
 {
 	struct usb_hub *hub = usb_hub_to_struct_hub(udev);
 
 	return udev->do_remote_wakeup +
 			(hub ? hub->wakeup_enabled_descendants : 0);
 }
+EXPORT_SYMBOL_GPL(usb_wakeup_enabled_descendants);
 
 /*
  * usb_port_suspend - suspend a usb device's upstream port
@@ -3282,7 +3283,7 @@ int usb_port_suspend(struct usb_device *udev, pm_message_t msg)
 	 * Therefore we will turn on the suspend feature if udev or any of its
 	 * descendants is enabled for remote wakeup.
 	 */
-	else if (PMSG_IS_AUTO(msg) || wakeup_enabled_descendants(udev) > 0)
+	else if (PMSG_IS_AUTO(msg) || usb_wakeup_enabled_descendants(udev) > 0)
 		status = set_port_feature(hub->hdev, port1,
 				USB_PORT_FEAT_SUSPEND);
 	else {
@@ -3668,7 +3669,6 @@ static int hub_suspend(struct usb_interface *intf, pm_message_t msg)
 	struct usb_hub		*hub = usb_get_intfdata(intf);
 	struct usb_device	*hdev = hub->hdev;
 	unsigned		port1;
-	int			status;
 
 	/*
 	 * Warn if children aren't already suspended.
@@ -3687,7 +3687,7 @@ static int hub_suspend(struct usb_interface *intf, pm_message_t msg)
 		}
 		if (udev)
 			hub->wakeup_enabled_descendants +=
-					wakeup_enabled_descendants(udev);
+					usb_wakeup_enabled_descendants(udev);
 	}
 
 	if (hdev->do_remote_wakeup && hub->quirk_check_port_auto_suspend) {
@@ -3702,12 +3702,12 @@ static int hub_suspend(struct usb_interface *intf, pm_message_t msg)
 	if (hub_is_superspeed(hdev) && hdev->do_remote_wakeup) {
 		/* Enable hub to send remote wakeup for all ports. */
 		for (port1 = 1; port1 <= hdev->maxchild; port1++) {
-			status = set_port_feature(hdev,
-					port1 |
-					USB_PORT_FEAT_REMOTE_WAKE_CONNECT |
-					USB_PORT_FEAT_REMOTE_WAKE_DISCONNECT |
-					USB_PORT_FEAT_REMOTE_WAKE_OVER_CURRENT,
-					USB_PORT_FEAT_REMOTE_WAKE_MASK);
+			set_port_feature(hdev,
+					 port1 |
+					 USB_PORT_FEAT_REMOTE_WAKE_CONNECT |
+					 USB_PORT_FEAT_REMOTE_WAKE_DISCONNECT |
+					 USB_PORT_FEAT_REMOTE_WAKE_OVER_CURRENT,
+					 USB_PORT_FEAT_REMOTE_WAKE_MASK);
 		}
 	}
 
@@ -5902,7 +5902,10 @@ int usb_reset_device(struct usb_device *udev)
 					cintf->needs_binding = 1;
 			}
 		}
-		usb_unbind_and_rebind_marked_interfaces(udev);
+
+		/* If the reset failed, hub_wq will unbind drivers later */
+		if (ret == 0)
+			usb_unbind_and_rebind_marked_interfaces(udev);
 	}
 
 	usb_autosuspend_device(udev);
diff --git a/drivers/usb/dwc2/core.c b/drivers/usb/dwc2/core.c
index 55d5ae2a7ec7..8b499d643461 100644
--- a/drivers/usb/dwc2/core.c
+++ b/drivers/usb/dwc2/core.c
@@ -1020,6 +1020,205 @@ int dwc2_hsotg_wait_bit_clear(struct dwc2_hsotg *hsotg, u32 offset, u32 mask,
 	return -ETIMEDOUT;
 }
 
+/*
+ * Initializes the FSLSPClkSel field of the HCFG register depending on the
+ * PHY type
+ */
+void dwc2_init_fs_ls_pclk_sel(struct dwc2_hsotg *hsotg)
+{
+	u32 hcfg, val;
+
+	if ((hsotg->hw_params.hs_phy_type == GHWCFG2_HS_PHY_TYPE_ULPI &&
+	     hsotg->hw_params.fs_phy_type == GHWCFG2_FS_PHY_TYPE_DEDICATED &&
+	     hsotg->params.ulpi_fs_ls) ||
+	    hsotg->params.phy_type == DWC2_PHY_TYPE_PARAM_FS) {
+		/* Full speed PHY */
+		val = HCFG_FSLSPCLKSEL_48_MHZ;
+	} else {
+		/* High speed PHY running at full speed or high speed */
+		val = HCFG_FSLSPCLKSEL_30_60_MHZ;
+	}
+
+	dev_dbg(hsotg->dev, "Initializing HCFG.FSLSPClkSel to %08x\n", val);
+	hcfg = dwc2_readl(hsotg, HCFG);
+	hcfg &= ~HCFG_FSLSPCLKSEL_MASK;
+	hcfg |= val << HCFG_FSLSPCLKSEL_SHIFT;
+	dwc2_writel(hsotg, hcfg, HCFG);
+}
+
+static int dwc2_fs_phy_init(struct dwc2_hsotg *hsotg, bool select_phy)
+{
+	u32 usbcfg, ggpio, i2cctl;
+	int retval = 0;
+
+	/*
+	 * core_init() is now called on every switch so only call the
+	 * following for the first time through
+	 */
+	if (select_phy) {
+		dev_dbg(hsotg->dev, "FS PHY selected\n");
+
+		usbcfg = dwc2_readl(hsotg, GUSBCFG);
+		if (!(usbcfg & GUSBCFG_PHYSEL)) {
+			usbcfg |= GUSBCFG_PHYSEL;
+			dwc2_writel(hsotg, usbcfg, GUSBCFG);
+
+			/* Reset after a PHY select */
+			retval = dwc2_core_reset(hsotg, false);
+
+			if (retval) {
+				dev_err(hsotg->dev,
+					"%s: Reset failed, aborting", __func__);
+				return retval;
+			}
+		}
+
+		if (hsotg->params.activate_stm_fs_transceiver) {
+			ggpio = dwc2_readl(hsotg, GGPIO);
+			if (!(ggpio & GGPIO_STM32_OTG_GCCFG_PWRDWN)) {
+				dev_dbg(hsotg->dev, "Activating transceiver\n");
+				/*
+				 * STM32F4x9 uses the GGPIO register as general
+				 * core configuration register.
+				 */
+				ggpio |= GGPIO_STM32_OTG_GCCFG_PWRDWN;
+				dwc2_writel(hsotg, ggpio, GGPIO);
+			}
+		}
+	}
+
+	/*
+	 * Program DCFG.DevSpd or HCFG.FSLSPclkSel to 48Mhz in FS. Also
+	 * do this on HNP Dev/Host mode switches (done in dev_init and
+	 * host_init).
+	 */
+	if (dwc2_is_host_mode(hsotg))
+		dwc2_init_fs_ls_pclk_sel(hsotg);
+
+	if (hsotg->params.i2c_enable) {
+		dev_dbg(hsotg->dev, "FS PHY enabling I2C\n");
+
+		/* Program GUSBCFG.OtgUtmiFsSel to I2C */
+		usbcfg = dwc2_readl(hsotg, GUSBCFG);
+		usbcfg |= GUSBCFG_OTG_UTMI_FS_SEL;
+		dwc2_writel(hsotg, usbcfg, GUSBCFG);
+
+		/* Program GI2CCTL.I2CEn */
+		i2cctl = dwc2_readl(hsotg, GI2CCTL);
+		i2cctl &= ~GI2CCTL_I2CDEVADDR_MASK;
+		i2cctl |= 1 << GI2CCTL_I2CDEVADDR_SHIFT;
+		i2cctl &= ~GI2CCTL_I2CEN;
+		dwc2_writel(hsotg, i2cctl, GI2CCTL);
+		i2cctl |= GI2CCTL_I2CEN;
+		dwc2_writel(hsotg, i2cctl, GI2CCTL);
+	}
+
+	return retval;
+}
+
+static int dwc2_hs_phy_init(struct dwc2_hsotg *hsotg, bool select_phy)
+{
+	u32 usbcfg, usbcfg_old;
+	int retval = 0;
+
+	if (!select_phy)
+		return 0;
+
+	usbcfg = dwc2_readl(hsotg, GUSBCFG);
+	usbcfg_old = usbcfg;
+
+	/*
+	 * HS PHY parameters. These parameters are preserved during soft reset
+	 * so only program the first time. Do a soft reset immediately after
+	 * setting phyif.
+	 */
+	switch (hsotg->params.phy_type) {
+	case DWC2_PHY_TYPE_PARAM_ULPI:
+		/* ULPI interface */
+		dev_dbg(hsotg->dev, "HS ULPI PHY selected\n");
+		usbcfg |= GUSBCFG_ULPI_UTMI_SEL;
+		usbcfg &= ~(GUSBCFG_PHYIF16 | GUSBCFG_DDRSEL);
+		if (hsotg->params.phy_ulpi_ddr)
+			usbcfg |= GUSBCFG_DDRSEL;
+
+		/* Set external VBUS indicator as needed. */
+		if (hsotg->params.oc_disable)
+			usbcfg |= (GUSBCFG_ULPI_INT_VBUS_IND |
+				   GUSBCFG_INDICATORPASSTHROUGH);
+		break;
+	case DWC2_PHY_TYPE_PARAM_UTMI:
+		/* UTMI+ interface */
+		dev_dbg(hsotg->dev, "HS UTMI+ PHY selected\n");
+		usbcfg &= ~(GUSBCFG_ULPI_UTMI_SEL | GUSBCFG_PHYIF16);
+		if (hsotg->params.phy_utmi_width == 16)
+			usbcfg |= GUSBCFG_PHYIF16;
+
+		/* Set turnaround time */
+		if (dwc2_is_device_mode(hsotg)) {
+			usbcfg &= ~GUSBCFG_USBTRDTIM_MASK;
+			if (hsotg->params.phy_utmi_width == 16)
+				usbcfg |= 5 << GUSBCFG_USBTRDTIM_SHIFT;
+			else
+				usbcfg |= 9 << GUSBCFG_USBTRDTIM_SHIFT;
+		}
+		break;
+	default:
+		dev_err(hsotg->dev, "FS PHY selected at HS!\n");
+		break;
+	}
+
+	if (usbcfg != usbcfg_old) {
+		dwc2_writel(hsotg, usbcfg, GUSBCFG);
+
+		/* Reset after setting the PHY parameters */
+		retval = dwc2_core_reset(hsotg, false);
+		if (retval) {
+			dev_err(hsotg->dev,
+				"%s: Reset failed, aborting", __func__);
+			return retval;
+		}
+	}
+
+	return retval;
+}
+
+int dwc2_phy_init(struct dwc2_hsotg *hsotg, bool select_phy)
+{
+	u32 usbcfg;
+	int retval = 0;
+
+	if ((hsotg->params.speed == DWC2_SPEED_PARAM_FULL ||
+	     hsotg->params.speed == DWC2_SPEED_PARAM_LOW) &&
+	    hsotg->params.phy_type == DWC2_PHY_TYPE_PARAM_FS) {
+		/* If FS/LS mode with FS/LS PHY */
+		retval = dwc2_fs_phy_init(hsotg, select_phy);
+		if (retval)
+			return retval;
+	} else {
+		/* High speed PHY */
+		retval = dwc2_hs_phy_init(hsotg, select_phy);
+		if (retval)
+			return retval;
+	}
+
+	if (hsotg->hw_params.hs_phy_type == GHWCFG2_HS_PHY_TYPE_ULPI &&
+	    hsotg->hw_params.fs_phy_type == GHWCFG2_FS_PHY_TYPE_DEDICATED &&
+	    hsotg->params.ulpi_fs_ls) {
+		dev_dbg(hsotg->dev, "Setting ULPI FSLS\n");
+		usbcfg = dwc2_readl(hsotg, GUSBCFG);
+		usbcfg |= GUSBCFG_ULPI_FS_LS;
+		usbcfg |= GUSBCFG_ULPI_CLK_SUSP_M;
+		dwc2_writel(hsotg, usbcfg, GUSBCFG);
+	} else {
+		usbcfg = dwc2_readl(hsotg, GUSBCFG);
+		usbcfg &= ~GUSBCFG_ULPI_FS_LS;
+		usbcfg &= ~GUSBCFG_ULPI_CLK_SUSP_M;
+		dwc2_writel(hsotg, usbcfg, GUSBCFG);
+	}
+
+	return retval;
+}
+
 MODULE_DESCRIPTION("DESIGNWARE HS OTG Core");
 MODULE_AUTHOR("Synopsys, Inc.");
 MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/usb/dwc2/core.h b/drivers/usb/dwc2/core.h
index 30bab8463c96..152ac41dfb2d 100644
--- a/drivers/usb/dwc2/core.h
+++ b/drivers/usb/dwc2/core.h
@@ -859,6 +859,8 @@ struct dwc2_hregs_backup {
  * @gadget_enabled:	Peripheral mode sub-driver initialization indicator.
  * @ll_hw_enabled:	Status of low-level hardware resources.
  * @hibernated:		True if core is hibernated
+ * @reset_phy_on_wake:	Quirk saying that we should assert PHY reset on a
+ *			remote wakeup.
  * @frame_number:       Frame number read from the core. For both device
  *			and host modes. The value ranges are from 0
  *			to HFNUM_MAX_FRNUM.
@@ -869,7 +871,6 @@ struct dwc2_hregs_backup {
  *                      removed once all SoCs support usb transceiver.
  * @supplies:           Definition of USB power supplies
  * @vbus_supply:        Regulator supplying vbus.
- * @phyif:              PHY interface width
  * @lock:		Spinlock that protects all the driver data structures
  * @priv:		Stores a pointer to the struct usb_hcd
  * @queuing_high_bandwidth: True if multiple packets of a high-bandwidth
@@ -972,6 +973,7 @@ struct dwc2_hregs_backup {
  * @status_buf_dma:     DMA address for status_buf
  * @start_work:         Delayed work for handling host A-cable connection
  * @reset_work:         Delayed work for handling a port reset
+ * @phy_reset_work:     Work structure for doing a PHY reset
  * @otg_port:           OTG port number
  * @frame_list:         Frame list
  * @frame_list_dma:     Frame list DMA address
@@ -991,6 +993,7 @@ struct dwc2_hregs_backup {
  * @ctrl_buff:          Buffer for EP0 control requests.
  * @ctrl_req:           Request for EP0 control packets.
  * @ep0_state:          EP0 control transfers state
+ * @delayed_status:		true when gadget driver asks for delayed status
  * @test_mode:          USB test mode requested by the host
  * @remote_wakeup_allowed: True if device is allowed to wake-up host by
  *                      remote-wakeup signalling
@@ -1045,6 +1048,7 @@ struct dwc2_hsotg {
 	unsigned int gadget_enabled:1;
 	unsigned int ll_hw_enabled:1;
 	unsigned int hibernated:1;
+	unsigned int reset_phy_on_wake:1;
 	u16 frame_number;
 
 	struct phy *phy;
@@ -1052,7 +1056,6 @@ struct dwc2_hsotg {
 	struct dwc2_hsotg_plat *plat;
 	struct regulator_bulk_data supplies[DWC2_NUM_SUPPLIES];
 	struct regulator *vbus_supply;
-	u32 phyif;
 
 	spinlock_t lock;
 	void *priv;
@@ -1147,6 +1150,7 @@ struct dwc2_hsotg {
 
 	struct delayed_work start_work;
 	struct delayed_work reset_work;
+	struct work_struct phy_reset_work;
 	u8 otg_port;
 	u32 *frame_list;
 	dma_addr_t frame_list_dma;
@@ -1172,6 +1176,7 @@ struct dwc2_hsotg {
 	void *ep0_buff;
 	void *ctrl_buff;
 	enum dwc2_ep0_state ep0_state;
+	unsigned delayed_status : 1;
 	u8 test_mode;
 
 	dma_addr_t setup_desc_dma[2];
@@ -1283,6 +1288,8 @@ int dwc2_exit_partial_power_down(struct dwc2_hsotg *hsotg, bool restore);
 int dwc2_enter_hibernation(struct dwc2_hsotg *hsotg, int is_host);
 int dwc2_exit_hibernation(struct dwc2_hsotg *hsotg, int rem_wakeup,
 		int reset, int is_host);
+void dwc2_init_fs_ls_pclk_sel(struct dwc2_hsotg *hsotg);
+int dwc2_phy_init(struct dwc2_hsotg *hsotg, bool select_phy);
 
 void dwc2_force_mode(struct dwc2_hsotg *hsotg, bool host);
 void dwc2_force_dr_mode(struct dwc2_hsotg *hsotg);
@@ -1431,6 +1438,8 @@ int dwc2_restore_host_registers(struct dwc2_hsotg *hsotg);
 int dwc2_host_enter_hibernation(struct dwc2_hsotg *hsotg);
 int dwc2_host_exit_hibernation(struct dwc2_hsotg *hsotg,
 			       int rem_wakeup, int reset);
+static inline void dwc2_host_schedule_phy_reset(struct dwc2_hsotg *hsotg)
+{ schedule_work(&hsotg->phy_reset_work); }
 #else
 static inline int dwc2_hcd_get_frame_number(struct dwc2_hsotg *hsotg)
 { return 0; }
@@ -1454,6 +1463,7 @@ static inline int dwc2_host_enter_hibernation(struct dwc2_hsotg *hsotg)
 static inline int dwc2_host_exit_hibernation(struct dwc2_hsotg *hsotg,
 					     int rem_wakeup, int reset)
 { return 0; }
+static inline void dwc2_host_schedule_phy_reset(struct dwc2_hsotg *hsotg) {}
 
 #endif
 
diff --git a/drivers/usb/dwc2/core_intr.c b/drivers/usb/dwc2/core_intr.c
index 19ae2595f1c3..6af6add3d4c0 100644
--- a/drivers/usb/dwc2/core_intr.c
+++ b/drivers/usb/dwc2/core_intr.c
@@ -435,6 +435,18 @@ static void dwc2_handle_wakeup_detected_intr(struct dwc2_hsotg *hsotg)
 			/* Restart the Phy Clock */
 			pcgcctl &= ~PCGCTL_STOPPCLK;
 			dwc2_writel(hsotg, pcgcctl, PCGCTL);
+
+			/*
+			 * If we've got this quirk then the PHY is stuck upon
+			 * wakeup.  Assert reset.  This will propagate out and
+			 * eventually we'll re-enumerate the device.  Not great
+			 * but the best we can do.  We can't call phy_reset()
+			 * at interrupt time but there's no hurry, so we'll
+			 * schedule it for later.
+			 */
+			if (hsotg->reset_phy_on_wake)
+				dwc2_host_schedule_phy_reset(hsotg);
+
 			mod_timer(&hsotg->wkp_timer,
 				  jiffies + msecs_to_jiffies(71));
 		} else {
diff --git a/drivers/usb/dwc2/gadget.c b/drivers/usb/dwc2/gadget.c
index 6812a8a3a98b..16ffd9fd9361 100644
--- a/drivers/usb/dwc2/gadget.c
+++ b/drivers/usb/dwc2/gadget.c
@@ -27,6 +27,8 @@
 #include <linux/usb/ch9.h>
 #include <linux/usb/gadget.h>
 #include <linux/usb/phy.h>
+#include <linux/usb/composite.h>
+
 
 #include "core.h"
 #include "hw.h"
@@ -714,13 +716,11 @@ static unsigned int dwc2_gadget_get_chain_limit(struct dwc2_hsotg_ep *hs_ep)
 	unsigned int maxsize;
 
 	if (is_isoc)
-		maxsize = hs_ep->dir_in ? DEV_DMA_ISOC_TX_NBYTES_LIMIT :
-					   DEV_DMA_ISOC_RX_NBYTES_LIMIT;
+		maxsize = (hs_ep->dir_in ? DEV_DMA_ISOC_TX_NBYTES_LIMIT :
+					   DEV_DMA_ISOC_RX_NBYTES_LIMIT) *
+					   MAX_DMA_DESC_NUM_HS_ISOC;
 	else
-		maxsize = DEV_DMA_NBYTES_LIMIT;
-
-	/* Above size of one descriptor was chosen, multiple it */
-	maxsize *= MAX_DMA_DESC_NUM_GENERIC;
+		maxsize = DEV_DMA_NBYTES_LIMIT * MAX_DMA_DESC_NUM_GENERIC;
 
 	return maxsize;
 }
@@ -932,7 +932,7 @@ static int dwc2_gadget_fill_isoc_desc(struct dwc2_hsotg_ep *hs_ep,
 
 	/* Update index of last configured entry in the chain */
 	hs_ep->next_desc++;
-	if (hs_ep->next_desc >= MAX_DMA_DESC_NUM_GENERIC)
+	if (hs_ep->next_desc >= MAX_DMA_DESC_NUM_HS_ISOC)
 		hs_ep->next_desc = 0;
 
 	return 0;
@@ -964,7 +964,7 @@ static void dwc2_gadget_start_isoc_ddma(struct dwc2_hsotg_ep *hs_ep)
 	}
 
 	/* Initialize descriptor chain by Host Busy status */
-	for (i = 0; i < MAX_DMA_DESC_NUM_GENERIC; i++) {
+	for (i = 0; i < MAX_DMA_DESC_NUM_HS_ISOC; i++) {
 		desc = &hs_ep->desc_list[i];
 		desc->status = 0;
 		desc->status |= (DEV_DMA_BUFF_STS_HBUSY
@@ -1446,6 +1446,11 @@ static int dwc2_hsotg_ep_queue(struct usb_ep *ep, struct usb_request *req,
 		return 0;
 	}
 
+	/* Change EP direction if status phase request is after data out */
+	if (!hs_ep->index && !req->length && !hs_ep->dir_in &&
+	    hs->ep0_state == DWC2_EP0_DATA_OUT)
+		hs_ep->dir_in = 1;
+
 	if (first) {
 		if (!hs_ep->isochronous) {
 			dwc2_hsotg_start_req(hs, hs_ep, hs_req, false);
@@ -1938,6 +1943,10 @@ static void dwc2_hsotg_process_control(struct dwc2_hsotg *hsotg,
 			dev_dbg(hsotg->dev, "driver->setup() ret %d\n", ret);
 	}
 
+	hsotg->delayed_status = false;
+	if (ret == USB_GADGET_DELAYED_STATUS)
+		hsotg->delayed_status = true;
+
 	/*
 	 * the request is either unhandlable, or is not formatted correctly
 	 * so respond with a STALL for the status stage to indicate failure.
@@ -2157,12 +2166,17 @@ static void dwc2_gadget_complete_isoc_request_ddma(struct dwc2_hsotg_ep *hs_ep)
 			 */
 			if (!hs_ep->dir_in && ureq->length & 0x3)
 				ureq->actual += 4 - (ureq->length & 0x3);
+
+			/* Set actual frame number for completed transfers */
+			ureq->frame_number =
+				(desc_sts & DEV_DMA_ISOC_FRNUM_MASK) >>
+				DEV_DMA_ISOC_FRNUM_SHIFT;
 		}
 
 		dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0);
 
 		hs_ep->compl_desc++;
-		if (hs_ep->compl_desc > (MAX_DMA_DESC_NUM_GENERIC - 1))
+		if (hs_ep->compl_desc > (MAX_DMA_DESC_NUM_HS_ISOC - 1))
 			hs_ep->compl_desc = 0;
 		desc_sts = hs_ep->desc_list[hs_ep->compl_desc].status;
 	}
@@ -2311,6 +2325,7 @@ static unsigned int dwc2_gadget_get_xfersize_ddma(struct dwc2_hsotg_ep *hs_ep)
 		if (status & DEV_DMA_STS_MASK)
 			dev_err(hsotg->dev, "descriptor %d closed with %x\n",
 				i, status & DEV_DMA_STS_MASK);
+		desc++;
 	}
 
 	return bytes_rem;
@@ -2387,8 +2402,8 @@ static void dwc2_hsotg_handle_outdone(struct dwc2_hsotg *hsotg, int epnum)
 	if (!using_desc_dma(hsotg) && epnum == 0 &&
 	    hsotg->ep0_state == DWC2_EP0_DATA_OUT) {
 		/* Move to STATUS IN */
-		dwc2_hsotg_ep0_zlp(hsotg, true);
-		return;
+		if (!hsotg->delayed_status)
+			dwc2_hsotg_ep0_zlp(hsotg, true);
 	}
 
 	/*
@@ -3053,8 +3068,20 @@ static void dwc2_hsotg_epint(struct dwc2_hsotg *hsotg, unsigned int idx,
 		/* Safety check EP0 state when STSPHSERCVD asserted */
 		if (hsotg->ep0_state == DWC2_EP0_DATA_OUT) {
 			/* Move to STATUS IN for DDMA */
-			if (using_desc_dma(hsotg))
-				dwc2_hsotg_ep0_zlp(hsotg, true);
+			if (using_desc_dma(hsotg)) {
+				if (!hsotg->delayed_status)
+					dwc2_hsotg_ep0_zlp(hsotg, true);
+				else
+				/* In case of 3 stage Control Write with delayed
+				 * status, when Status IN transfer started
+				 * before STSPHSERCVD asserted, NAKSTS bit not
+				 * cleared by CNAK in dwc2_hsotg_start_req()
+				 * function. Clear now NAKSTS to allow complete
+				 * transfer.
+				 */
+					dwc2_set_bit(hsotg, DIEPCTL(0),
+						     DXEPCTL_CNAK);
+			}
 		}
 
 	}
@@ -3314,21 +3341,14 @@ void dwc2_hsotg_core_init_disconnected(struct dwc2_hsotg *hsotg,
 
 	/* keep other bits untouched (so e.g. forced modes are not lost) */
 	usbcfg = dwc2_readl(hsotg, GUSBCFG);
-	usbcfg &= ~(GUSBCFG_TOUTCAL_MASK | GUSBCFG_PHYIF16 | GUSBCFG_SRPCAP |
-		GUSBCFG_HNPCAP | GUSBCFG_USBTRDTIM_MASK);
-
-	if (hsotg->params.phy_type == DWC2_PHY_TYPE_PARAM_FS &&
-	    (hsotg->params.speed == DWC2_SPEED_PARAM_FULL ||
-	     hsotg->params.speed == DWC2_SPEED_PARAM_LOW)) {
-		/* FS/LS Dedicated Transceiver Interface */
-		usbcfg |= GUSBCFG_PHYSEL;
-	} else {
-		/* set the PLL on, remove the HNP/SRP and set the PHY */
-		val = (hsotg->phyif == GUSBCFG_PHYIF8) ? 9 : 5;
-		usbcfg |= hsotg->phyif | GUSBCFG_TOUTCAL(7) |
-			(val << GUSBCFG_USBTRDTIM_SHIFT);
-	}
-	dwc2_writel(hsotg, usbcfg, GUSBCFG);
+	usbcfg &= ~GUSBCFG_TOUTCAL_MASK;
+	usbcfg |= GUSBCFG_TOUTCAL(7);
+
+	/* remove the HNP/SRP and set the PHY */
+	usbcfg &= ~(GUSBCFG_SRPCAP | GUSBCFG_HNPCAP);
+        dwc2_writel(hsotg, usbcfg, GUSBCFG);
+
+	dwc2_phy_init(hsotg, true);
 
 	dwc2_hsotg_init_fifo(hsotg);
 
@@ -3899,6 +3919,7 @@ static int dwc2_hsotg_ep_enable(struct usb_ep *ep,
 	unsigned int i, val, size;
 	int ret = 0;
 	unsigned char ep_type;
+	int desc_num;
 
 	dev_dbg(hsotg->dev,
 		"%s: ep %s: a 0x%02x, attr 0x%02x, mps 0x%04x, intr %d\n",
@@ -3945,11 +3966,15 @@ static int dwc2_hsotg_ep_enable(struct usb_ep *ep,
 	dev_dbg(hsotg->dev, "%s: read DxEPCTL=0x%08x from 0x%08x\n",
 		__func__, epctrl, epctrl_reg);
 
+	if (using_desc_dma(hsotg) && ep_type == USB_ENDPOINT_XFER_ISOC)
+		desc_num = MAX_DMA_DESC_NUM_HS_ISOC;
+	else
+		desc_num = MAX_DMA_DESC_NUM_GENERIC;
+
 	/* Allocate DMA descriptor chain for non-ctrl endpoints */
 	if (using_desc_dma(hsotg) && !hs_ep->desc_list) {
 		hs_ep->desc_list = dmam_alloc_coherent(hsotg->dev,
-			MAX_DMA_DESC_NUM_GENERIC *
-			sizeof(struct dwc2_dma_desc),
+			desc_num * sizeof(struct dwc2_dma_desc),
 			&hs_ep->desc_list_dma, GFP_ATOMIC);
 		if (!hs_ep->desc_list) {
 			ret = -ENOMEM;
@@ -4092,7 +4117,7 @@ error1:
 
 error2:
 	if (ret && using_desc_dma(hsotg) && hs_ep->desc_list) {
-		dmam_free_coherent(hsotg->dev, MAX_DMA_DESC_NUM_GENERIC *
+		dmam_free_coherent(hsotg->dev, desc_num *
 			sizeof(struct dwc2_dma_desc),
 			hs_ep->desc_list, hs_ep->desc_list_dma);
 		hs_ep->desc_list = NULL;
@@ -4328,8 +4353,6 @@ static const struct usb_ep_ops dwc2_hsotg_ep_ops = {
  */
 static void dwc2_hsotg_init(struct dwc2_hsotg *hsotg)
 {
-	u32 trdtim;
-	u32 usbcfg;
 	/* unmask subset of endpoint interrupts */
 
 	dwc2_writel(hsotg, DIEPMSK_TIMEOUTMSK | DIEPMSK_AHBERRMSK |
@@ -4353,17 +4376,6 @@ static void dwc2_hsotg_init(struct dwc2_hsotg *hsotg)
 
 	dwc2_hsotg_init_fifo(hsotg);
 
-	/* keep other bits untouched (so e.g. forced modes are not lost) */
-	usbcfg = dwc2_readl(hsotg, GUSBCFG);
-	usbcfg &= ~(GUSBCFG_TOUTCAL_MASK | GUSBCFG_PHYIF16 | GUSBCFG_SRPCAP |
-		GUSBCFG_HNPCAP | GUSBCFG_USBTRDTIM_MASK);
-
-	/* set the PLL on, remove the HNP/SRP and set the PHY */
-	trdtim = (hsotg->phyif == GUSBCFG_PHYIF8) ? 9 : 5;
-	usbcfg |= hsotg->phyif | GUSBCFG_TOUTCAL(7) |
-		(trdtim << GUSBCFG_USBTRDTIM_SHIFT);
-	dwc2_writel(hsotg, usbcfg, GUSBCFG);
-
 	if (using_dma(hsotg))
 		dwc2_set_bit(hsotg, GAHBCFG, GAHBCFG_DMA_EN);
 }
@@ -5073,6 +5085,7 @@ void dwc2_gadget_init_lpm(struct dwc2_hsotg *hsotg)
 	val |= hsotg->params.lpm_clock_gating ? GLPMCFG_ENBLSLPM : 0;
 	val |= hsotg->params.hird_threshold << GLPMCFG_HIRD_THRES_SHIFT;
 	val |= hsotg->params.besl ? GLPMCFG_ENBESL : 0;
+	val |= GLPMCFG_LPM_REJECT_CTRL_CONTROL;
 	val |= GLPMCFG_LPM_ACCEPT_CTRL_ISOC;
 	dwc2_writel(hsotg, val, GLPMCFG);
 	dev_dbg(hsotg->dev, "GLPMCFG=0x%08x\n", dwc2_readl(hsotg, GLPMCFG));
diff --git a/drivers/usb/dwc2/hcd.c b/drivers/usb/dwc2/hcd.c
index 3f087962f498..b50ec3714fd8 100644
--- a/drivers/usb/dwc2/hcd.c
+++ b/drivers/usb/dwc2/hcd.c
@@ -97,196 +97,6 @@ static void dwc2_enable_common_interrupts(struct dwc2_hsotg *hsotg)
 	dwc2_writel(hsotg, intmsk, GINTMSK);
 }
 
-/*
- * Initializes the FSLSPClkSel field of the HCFG register depending on the
- * PHY type
- */
-static void dwc2_init_fs_ls_pclk_sel(struct dwc2_hsotg *hsotg)
-{
-	u32 hcfg, val;
-
-	if ((hsotg->hw_params.hs_phy_type == GHWCFG2_HS_PHY_TYPE_ULPI &&
-	     hsotg->hw_params.fs_phy_type == GHWCFG2_FS_PHY_TYPE_DEDICATED &&
-	     hsotg->params.ulpi_fs_ls) ||
-	    hsotg->params.phy_type == DWC2_PHY_TYPE_PARAM_FS) {
-		/* Full speed PHY */
-		val = HCFG_FSLSPCLKSEL_48_MHZ;
-	} else {
-		/* High speed PHY running at full speed or high speed */
-		val = HCFG_FSLSPCLKSEL_30_60_MHZ;
-	}
-
-	dev_dbg(hsotg->dev, "Initializing HCFG.FSLSPClkSel to %08x\n", val);
-	hcfg = dwc2_readl(hsotg, HCFG);
-	hcfg &= ~HCFG_FSLSPCLKSEL_MASK;
-	hcfg |= val << HCFG_FSLSPCLKSEL_SHIFT;
-	dwc2_writel(hsotg, hcfg, HCFG);
-}
-
-static int dwc2_fs_phy_init(struct dwc2_hsotg *hsotg, bool select_phy)
-{
-	u32 usbcfg, ggpio, i2cctl;
-	int retval = 0;
-
-	/*
-	 * core_init() is now called on every switch so only call the
-	 * following for the first time through
-	 */
-	if (select_phy) {
-		dev_dbg(hsotg->dev, "FS PHY selected\n");
-
-		usbcfg = dwc2_readl(hsotg, GUSBCFG);
-		if (!(usbcfg & GUSBCFG_PHYSEL)) {
-			usbcfg |= GUSBCFG_PHYSEL;
-			dwc2_writel(hsotg, usbcfg, GUSBCFG);
-
-			/* Reset after a PHY select */
-			retval = dwc2_core_reset(hsotg, false);
-
-			if (retval) {
-				dev_err(hsotg->dev,
-					"%s: Reset failed, aborting", __func__);
-				return retval;
-			}
-		}
-
-		if (hsotg->params.activate_stm_fs_transceiver) {
-			ggpio = dwc2_readl(hsotg, GGPIO);
-			if (!(ggpio & GGPIO_STM32_OTG_GCCFG_PWRDWN)) {
-				dev_dbg(hsotg->dev, "Activating transceiver\n");
-				/*
-				 * STM32F4x9 uses the GGPIO register as general
-				 * core configuration register.
-				 */
-				ggpio |= GGPIO_STM32_OTG_GCCFG_PWRDWN;
-				dwc2_writel(hsotg, ggpio, GGPIO);
-			}
-		}
-	}
-
-	/*
-	 * Program DCFG.DevSpd or HCFG.FSLSPclkSel to 48Mhz in FS. Also
-	 * do this on HNP Dev/Host mode switches (done in dev_init and
-	 * host_init).
-	 */
-	if (dwc2_is_host_mode(hsotg))
-		dwc2_init_fs_ls_pclk_sel(hsotg);
-
-	if (hsotg->params.i2c_enable) {
-		dev_dbg(hsotg->dev, "FS PHY enabling I2C\n");
-
-		/* Program GUSBCFG.OtgUtmiFsSel to I2C */
-		usbcfg = dwc2_readl(hsotg, GUSBCFG);
-		usbcfg |= GUSBCFG_OTG_UTMI_FS_SEL;
-		dwc2_writel(hsotg, usbcfg, GUSBCFG);
-
-		/* Program GI2CCTL.I2CEn */
-		i2cctl = dwc2_readl(hsotg, GI2CCTL);
-		i2cctl &= ~GI2CCTL_I2CDEVADDR_MASK;
-		i2cctl |= 1 << GI2CCTL_I2CDEVADDR_SHIFT;
-		i2cctl &= ~GI2CCTL_I2CEN;
-		dwc2_writel(hsotg, i2cctl, GI2CCTL);
-		i2cctl |= GI2CCTL_I2CEN;
-		dwc2_writel(hsotg, i2cctl, GI2CCTL);
-	}
-
-	return retval;
-}
-
-static int dwc2_hs_phy_init(struct dwc2_hsotg *hsotg, bool select_phy)
-{
-	u32 usbcfg, usbcfg_old;
-	int retval = 0;
-
-	if (!select_phy)
-		return 0;
-
-	usbcfg = dwc2_readl(hsotg, GUSBCFG);
-	usbcfg_old = usbcfg;
-
-	/*
-	 * HS PHY parameters. These parameters are preserved during soft reset
-	 * so only program the first time. Do a soft reset immediately after
-	 * setting phyif.
-	 */
-	switch (hsotg->params.phy_type) {
-	case DWC2_PHY_TYPE_PARAM_ULPI:
-		/* ULPI interface */
-		dev_dbg(hsotg->dev, "HS ULPI PHY selected\n");
-		usbcfg |= GUSBCFG_ULPI_UTMI_SEL;
-		usbcfg &= ~(GUSBCFG_PHYIF16 | GUSBCFG_DDRSEL);
-		if (hsotg->params.phy_ulpi_ddr)
-			usbcfg |= GUSBCFG_DDRSEL;
-
-		/* Set external VBUS indicator as needed. */
-		if (hsotg->params.oc_disable)
-			usbcfg |= (GUSBCFG_ULPI_INT_VBUS_IND |
-				   GUSBCFG_INDICATORPASSTHROUGH);
-		break;
-	case DWC2_PHY_TYPE_PARAM_UTMI:
-		/* UTMI+ interface */
-		dev_dbg(hsotg->dev, "HS UTMI+ PHY selected\n");
-		usbcfg &= ~(GUSBCFG_ULPI_UTMI_SEL | GUSBCFG_PHYIF16);
-		if (hsotg->params.phy_utmi_width == 16)
-			usbcfg |= GUSBCFG_PHYIF16;
-		break;
-	default:
-		dev_err(hsotg->dev, "FS PHY selected at HS!\n");
-		break;
-	}
-
-	if (usbcfg != usbcfg_old) {
-		dwc2_writel(hsotg, usbcfg, GUSBCFG);
-
-		/* Reset after setting the PHY parameters */
-		retval = dwc2_core_reset(hsotg, false);
-		if (retval) {
-			dev_err(hsotg->dev,
-				"%s: Reset failed, aborting", __func__);
-			return retval;
-		}
-	}
-
-	return retval;
-}
-
-static int dwc2_phy_init(struct dwc2_hsotg *hsotg, bool select_phy)
-{
-	u32 usbcfg;
-	int retval = 0;
-
-	if ((hsotg->params.speed == DWC2_SPEED_PARAM_FULL ||
-	     hsotg->params.speed == DWC2_SPEED_PARAM_LOW) &&
-	    hsotg->params.phy_type == DWC2_PHY_TYPE_PARAM_FS) {
-		/* If FS/LS mode with FS/LS PHY */
-		retval = dwc2_fs_phy_init(hsotg, select_phy);
-		if (retval)
-			return retval;
-	} else {
-		/* High speed PHY */
-		retval = dwc2_hs_phy_init(hsotg, select_phy);
-		if (retval)
-			return retval;
-	}
-
-	if (hsotg->hw_params.hs_phy_type == GHWCFG2_HS_PHY_TYPE_ULPI &&
-	    hsotg->hw_params.fs_phy_type == GHWCFG2_FS_PHY_TYPE_DEDICATED &&
-	    hsotg->params.ulpi_fs_ls) {
-		dev_dbg(hsotg->dev, "Setting ULPI FSLS\n");
-		usbcfg = dwc2_readl(hsotg, GUSBCFG);
-		usbcfg |= GUSBCFG_ULPI_FS_LS;
-		usbcfg |= GUSBCFG_ULPI_CLK_SUSP_M;
-		dwc2_writel(hsotg, usbcfg, GUSBCFG);
-	} else {
-		usbcfg = dwc2_readl(hsotg, GUSBCFG);
-		usbcfg &= ~GUSBCFG_ULPI_FS_LS;
-		usbcfg &= ~GUSBCFG_ULPI_CLK_SUSP_M;
-		dwc2_writel(hsotg, usbcfg, GUSBCFG);
-	}
-
-	return retval;
-}
-
 static int dwc2_gahbcfg_init(struct dwc2_hsotg *hsotg)
 {
 	u32 ahbcfg = dwc2_readl(hsotg, GAHBCFG);
@@ -2437,25 +2247,31 @@ static void dwc2_core_host_init(struct dwc2_hsotg *hsotg)
 		num_channels = hsotg->params.host_channels;
 		for (i = 0; i < num_channels; i++) {
 			hcchar = dwc2_readl(hsotg, HCCHAR(i));
-			hcchar &= ~HCCHAR_CHENA;
-			hcchar |= HCCHAR_CHDIS;
-			hcchar &= ~HCCHAR_EPDIR;
-			dwc2_writel(hsotg, hcchar, HCCHAR(i));
+			if (hcchar & HCCHAR_CHENA) {
+				hcchar &= ~HCCHAR_CHENA;
+				hcchar |= HCCHAR_CHDIS;
+				hcchar &= ~HCCHAR_EPDIR;
+				dwc2_writel(hsotg, hcchar, HCCHAR(i));
+			}
 		}
 
 		/* Halt all channels to put them into a known state */
 		for (i = 0; i < num_channels; i++) {
 			hcchar = dwc2_readl(hsotg, HCCHAR(i));
-			hcchar |= HCCHAR_CHENA | HCCHAR_CHDIS;
-			hcchar &= ~HCCHAR_EPDIR;
-			dwc2_writel(hsotg, hcchar, HCCHAR(i));
-			dev_dbg(hsotg->dev, "%s: Halt channel %d\n",
-				__func__, i);
-
-			if (dwc2_hsotg_wait_bit_clear(hsotg, HCCHAR(i),
-						      HCCHAR_CHENA, 1000)) {
-				dev_warn(hsotg->dev, "Unable to clear enable on channel %d\n",
-					 i);
+			if (hcchar & HCCHAR_CHENA) {
+				hcchar |= HCCHAR_CHENA | HCCHAR_CHDIS;
+				hcchar &= ~HCCHAR_EPDIR;
+				dwc2_writel(hsotg, hcchar, HCCHAR(i));
+				dev_dbg(hsotg->dev, "%s: Halt channel %d\n",
+					__func__, i);
+
+				if (dwc2_hsotg_wait_bit_clear(hsotg, HCCHAR(i),
+							      HCCHAR_CHENA,
+							      1000)) {
+					dev_warn(hsotg->dev,
+						 "Unable to clear enable on channel %d\n",
+						 i);
+				}
 			}
 		}
 	}
@@ -4376,6 +4192,17 @@ static void dwc2_hcd_reset_func(struct work_struct *work)
 	spin_unlock_irqrestore(&hsotg->lock, flags);
 }
 
+static void dwc2_hcd_phy_reset_func(struct work_struct *work)
+{
+	struct dwc2_hsotg *hsotg = container_of(work, struct dwc2_hsotg,
+						phy_reset_work);
+	int ret;
+
+	ret = phy_reset(hsotg->phy);
+	if (ret)
+		dev_warn(hsotg->dev, "PHY reset failed\n");
+}
+
 /*
  * =========================================================================
  *  Linux HC Driver Functions
@@ -4471,6 +4298,7 @@ static int _dwc2_hcd_suspend(struct usb_hcd *hcd)
 	unsigned long flags;
 	int ret = 0;
 	u32 hprt0;
+	u32 pcgctl;
 
 	spin_lock_irqsave(&hsotg->lock, flags);
 
@@ -4486,7 +4314,7 @@ static int _dwc2_hcd_suspend(struct usb_hcd *hcd)
 	if (hsotg->op_state == OTG_STATE_B_PERIPHERAL)
 		goto unlock;
 
-	if (hsotg->params.power_down != DWC2_POWER_DOWN_PARAM_PARTIAL)
+	if (hsotg->params.power_down > DWC2_POWER_DOWN_PARAM_PARTIAL)
 		goto skip_power_saving;
 
 	/*
@@ -4495,21 +4323,35 @@ static int _dwc2_hcd_suspend(struct usb_hcd *hcd)
 	 */
 	if (!hsotg->bus_suspended) {
 		hprt0 = dwc2_read_hprt0(hsotg);
-		hprt0 |= HPRT0_SUSP;
-		hprt0 &= ~HPRT0_PWR;
-		dwc2_writel(hsotg, hprt0, HPRT0);
-		spin_unlock_irqrestore(&hsotg->lock, flags);
-		dwc2_vbus_supply_exit(hsotg);
-		spin_lock_irqsave(&hsotg->lock, flags);
+		if (hprt0 & HPRT0_CONNSTS) {
+			hprt0 |= HPRT0_SUSP;
+			if (hsotg->params.power_down == DWC2_POWER_DOWN_PARAM_PARTIAL)
+				hprt0 &= ~HPRT0_PWR;
+			dwc2_writel(hsotg, hprt0, HPRT0);
+		}
+		if (hsotg->params.power_down == DWC2_POWER_DOWN_PARAM_PARTIAL) {
+			spin_unlock_irqrestore(&hsotg->lock, flags);
+			dwc2_vbus_supply_exit(hsotg);
+			spin_lock_irqsave(&hsotg->lock, flags);
+		} else {
+			pcgctl = readl(hsotg->regs + PCGCTL);
+			pcgctl |= PCGCTL_STOPPCLK;
+			writel(pcgctl, hsotg->regs + PCGCTL);
+		}
 	}
 
-	/* Enter partial_power_down */
-	ret = dwc2_enter_partial_power_down(hsotg);
-	if (ret) {
-		if (ret != -ENOTSUPP)
-			dev_err(hsotg->dev,
-				"enter partial_power_down failed\n");
-		goto skip_power_saving;
+	if (hsotg->params.power_down == DWC2_POWER_DOWN_PARAM_PARTIAL) {
+		/* Enter partial_power_down */
+		ret = dwc2_enter_partial_power_down(hsotg);
+		if (ret) {
+			if (ret != -ENOTSUPP)
+				dev_err(hsotg->dev,
+					"enter partial_power_down failed\n");
+			goto skip_power_saving;
+		}
+
+		/* After entering partial_power_down, hardware is no more accessible */
+		clear_bit(HCD_FLAG_HW_ACCESSIBLE, &hcd->flags);
 	}
 
 	/* Ask phy to be suspended */
@@ -4519,9 +4361,6 @@ static int _dwc2_hcd_suspend(struct usb_hcd *hcd)
 		spin_lock_irqsave(&hsotg->lock, flags);
 	}
 
-	/* After entering partial_power_down, hardware is no more accessible */
-	clear_bit(HCD_FLAG_HW_ACCESSIBLE, &hcd->flags);
-
 skip_power_saving:
 	hsotg->lx_state = DWC2_L2;
 unlock:
@@ -4534,6 +4373,7 @@ static int _dwc2_hcd_resume(struct usb_hcd *hcd)
 {
 	struct dwc2_hsotg *hsotg = dwc2_hcd_to_hsotg(hcd);
 	unsigned long flags;
+	u32 pcgctl;
 	int ret = 0;
 
 	spin_lock_irqsave(&hsotg->lock, flags);
@@ -4544,18 +4384,12 @@ static int _dwc2_hcd_resume(struct usb_hcd *hcd)
 	if (hsotg->lx_state != DWC2_L2)
 		goto unlock;
 
-	if (hsotg->params.power_down != DWC2_POWER_DOWN_PARAM_PARTIAL) {
+	if (hsotg->params.power_down > DWC2_POWER_DOWN_PARAM_PARTIAL) {
 		hsotg->lx_state = DWC2_L0;
 		goto unlock;
 	}
 
 	/*
-	 * Set HW accessible bit before powering on the controller
-	 * since an interrupt may rise.
-	 */
-	set_bit(HCD_FLAG_HW_ACCESSIBLE, &hcd->flags);
-
-	/*
 	 * Enable power if not already done.
 	 * This must not be spinlocked since duration
 	 * of this call is unknown.
@@ -4566,10 +4400,23 @@ static int _dwc2_hcd_resume(struct usb_hcd *hcd)
 		spin_lock_irqsave(&hsotg->lock, flags);
 	}
 
-	/* Exit partial_power_down */
-	ret = dwc2_exit_partial_power_down(hsotg, true);
-	if (ret && (ret != -ENOTSUPP))
-		dev_err(hsotg->dev, "exit partial_power_down failed\n");
+	if (hsotg->params.power_down == DWC2_POWER_DOWN_PARAM_PARTIAL) {
+		/*
+		 * Set HW accessible bit before powering on the controller
+		 * since an interrupt may rise.
+		 */
+		set_bit(HCD_FLAG_HW_ACCESSIBLE, &hcd->flags);
+
+
+		/* Exit partial_power_down */
+		ret = dwc2_exit_partial_power_down(hsotg, true);
+		if (ret && (ret != -ENOTSUPP))
+			dev_err(hsotg->dev, "exit partial_power_down failed\n");
+	} else {
+		pcgctl = readl(hsotg->regs + PCGCTL);
+		pcgctl &= ~PCGCTL_STOPPCLK;
+		writel(pcgctl, hsotg->regs + PCGCTL);
+	}
 
 	hsotg->lx_state = DWC2_L0;
 
@@ -4581,10 +4428,12 @@ static int _dwc2_hcd_resume(struct usb_hcd *hcd)
 		spin_unlock_irqrestore(&hsotg->lock, flags);
 		dwc2_port_resume(hsotg);
 	} else {
-		dwc2_vbus_supply_init(hsotg);
+		if (hsotg->params.power_down == DWC2_POWER_DOWN_PARAM_PARTIAL) {
+			dwc2_vbus_supply_init(hsotg);
 
-		/* Wait for controller to correctly update D+/D- level */
-		usleep_range(3000, 5000);
+			/* Wait for controller to correctly update D+/D- level */
+			usleep_range(3000, 5000);
+		}
 
 		/*
 		 * Clear Port Enable and Port Status changes.
@@ -5130,6 +4979,8 @@ static void dwc2_hcd_free(struct dwc2_hsotg *hsotg)
 		destroy_workqueue(hsotg->wq_otg);
 	}
 
+	cancel_work_sync(&hsotg->phy_reset_work);
+
 	del_timer(&hsotg->wkp_timer);
 }
 
@@ -5271,11 +5122,10 @@ int dwc2_hcd_init(struct dwc2_hsotg *hsotg)
 		hsotg->hc_ptr_array[i] = channel;
 	}
 
-	/* Initialize hsotg start work */
+	/* Initialize work */
 	INIT_DELAYED_WORK(&hsotg->start_work, dwc2_hcd_start_func);
-
-	/* Initialize port reset work */
 	INIT_DELAYED_WORK(&hsotg->reset_work, dwc2_hcd_reset_func);
+	INIT_WORK(&hsotg->phy_reset_work, dwc2_hcd_phy_reset_func);
 
 	/*
 	 * Allocate space for storing data on status transactions. Normally no
diff --git a/drivers/usb/dwc2/hw.h b/drivers/usb/dwc2/hw.h
index 98af924a9a5c..510e87ec0be8 100644
--- a/drivers/usb/dwc2/hw.h
+++ b/drivers/usb/dwc2/hw.h
@@ -310,12 +310,12 @@
 #define GHWCFG4_NUM_DEV_MODE_CTRL_EP_SHIFT	16
 #define GHWCFG4_UTMI_PHY_DATA_WIDTH_MASK	(0x3 << 14)
 #define GHWCFG4_UTMI_PHY_DATA_WIDTH_SHIFT	14
-#define GHWCFG4_ACG_SUPPORTED			BIT(12)
-#define GHWCFG4_IPG_ISOC_SUPPORTED		BIT(11)
-#define GHWCFG4_SERVICE_INTERVAL_SUPPORTED      BIT(10)
 #define GHWCFG4_UTMI_PHY_DATA_WIDTH_8		0
 #define GHWCFG4_UTMI_PHY_DATA_WIDTH_16		1
 #define GHWCFG4_UTMI_PHY_DATA_WIDTH_8_OR_16	2
+#define GHWCFG4_ACG_SUPPORTED			BIT(12)
+#define GHWCFG4_IPG_ISOC_SUPPORTED		BIT(11)
+#define GHWCFG4_SERVICE_INTERVAL_SUPPORTED      BIT(10)
 #define GHWCFG4_XHIBER				BIT(7)
 #define GHWCFG4_HIBER				BIT(6)
 #define GHWCFG4_MIN_AHB_FREQ			BIT(5)
@@ -333,7 +333,7 @@
 #define GLPMCFG_SNDLPM			BIT(24)
 #define GLPMCFG_RETRY_CNT_MASK		(0x7 << 21)
 #define GLPMCFG_RETRY_CNT_SHIFT		21
-#define GLPMCFG_LPM_ACCEPT_CTRL_CONTROL	BIT(21)
+#define GLPMCFG_LPM_REJECT_CTRL_CONTROL	BIT(21)
 #define GLPMCFG_LPM_ACCEPT_CTRL_ISOC	BIT(22)
 #define GLPMCFG_LPM_CHNL_INDX_MASK	(0xf << 17)
 #define GLPMCFG_LPM_CHNL_INDX_SHIFT	17
diff --git a/drivers/usb/dwc2/params.c b/drivers/usb/dwc2/params.c
index 24ff5f21cb25..6900eea57526 100644
--- a/drivers/usb/dwc2/params.c
+++ b/drivers/usb/dwc2/params.c
@@ -121,6 +121,16 @@ static void dwc2_set_amlogic_params(struct dwc2_hsotg *hsotg)
 	p->power_down = DWC2_POWER_DOWN_PARAM_NONE;
 }
 
+static void dwc2_set_amlogic_g12a_params(struct dwc2_hsotg *hsotg)
+{
+	struct dwc2_core_params *p = &hsotg->params;
+
+	p->lpm = false;
+	p->lpm_clock_gating = false;
+	p->besl = false;
+	p->hird_threshold_en = false;
+}
+
 static void dwc2_set_amcc_params(struct dwc2_hsotg *hsotg)
 {
 	struct dwc2_core_params *p = &hsotg->params;
@@ -167,6 +177,8 @@ const struct of_device_id dwc2_of_match_table[] = {
 	  .data = dwc2_set_amlogic_params },
 	{ .compatible = "amlogic,meson-gxbb-usb",
 	  .data = dwc2_set_amlogic_params },
+	{ .compatible = "amlogic,meson-g12a-usb",
+	  .data = dwc2_set_amlogic_g12a_params },
 	{ .compatible = "amcc,dwc-otg", .data = dwc2_set_amcc_params },
 	{ .compatible = "st,stm32f4x9-fsotg",
 	  .data = dwc2_set_stm32f4x9_fsotg_params },
@@ -273,6 +285,23 @@ static void dwc2_set_param_power_down(struct dwc2_hsotg *hsotg)
 	hsotg->params.power_down = val;
 }
 
+static void dwc2_set_param_lpm(struct dwc2_hsotg *hsotg)
+{
+	struct dwc2_core_params *p = &hsotg->params;
+
+	p->lpm = hsotg->hw_params.lpm_mode;
+	if (p->lpm) {
+		p->lpm_clock_gating = true;
+		p->besl = true;
+		p->hird_threshold_en = true;
+		p->hird_threshold = 4;
+	} else {
+		p->lpm_clock_gating = false;
+		p->besl = false;
+		p->hird_threshold_en = false;
+	}
+}
+
 /**
  * dwc2_set_default_params() - Set all core parameters to their
  * auto-detected default values.
@@ -291,6 +320,7 @@ static void dwc2_set_default_params(struct dwc2_hsotg *hsotg)
 	dwc2_set_param_speed(hsotg);
 	dwc2_set_param_phy_utmi_width(hsotg);
 	dwc2_set_param_power_down(hsotg);
+	dwc2_set_param_lpm(hsotg);
 	p->phy_ulpi_ddr = false;
 	p->phy_ulpi_ext_vbus = false;
 
@@ -303,11 +333,6 @@ static void dwc2_set_default_params(struct dwc2_hsotg *hsotg)
 	p->reload_ctl = (hw->snpsid >= DWC2_CORE_REV_2_92a);
 	p->uframe_sched = true;
 	p->external_id_pin_ctl = false;
-	p->lpm = true;
-	p->lpm_clock_gating = true;
-	p->besl = true;
-	p->hird_threshold_en = true;
-	p->hird_threshold = 4;
 	p->ipg_isoc_en = false;
 	p->service_interval = false;
 	p->max_packet_count = hw->max_packet_count;
diff --git a/drivers/usb/dwc2/platform.c b/drivers/usb/dwc2/platform.c
index c0b64d483552..d10a7f8daec3 100644
--- a/drivers/usb/dwc2/platform.c
+++ b/drivers/usb/dwc2/platform.c
@@ -230,9 +230,6 @@ static int dwc2_lowlevel_hw_init(struct dwc2_hsotg *hsotg)
 
 	reset_control_deassert(hsotg->reset_ecc);
 
-	/* Set default UTMI width */
-	hsotg->phyif = GUSBCFG_PHYIF16;
-
 	/*
 	 * Attempt to find a generic PHY, then look for an old style
 	 * USB PHY and then fall back to pdata
@@ -280,14 +277,14 @@ static int dwc2_lowlevel_hw_init(struct dwc2_hsotg *hsotg)
 		 * width is 8-bit and set the phyif appropriately.
 		 */
 		if (phy_get_bus_width(hsotg->phy) == 8)
-			hsotg->phyif = GUSBCFG_PHYIF8;
+			hsotg->params.phy_utmi_width = 8;
 	}
 
 	/* Clock */
-	hsotg->clk = devm_clk_get(hsotg->dev, "otg");
+	hsotg->clk = devm_clk_get_optional(hsotg->dev, "otg");
 	if (IS_ERR(hsotg->clk)) {
-		hsotg->clk = NULL;
-		dev_dbg(hsotg->dev, "cannot get otg clock\n");
+		dev_err(hsotg->dev, "cannot get otg clock\n");
+		return PTR_ERR(hsotg->clk);
 	}
 
 	/* Regulators */
@@ -481,6 +478,15 @@ static int dwc2_driver_probe(struct platform_device *dev)
 		hsotg->gadget_enabled = 1;
 	}
 
+	hsotg->reset_phy_on_wake =
+		of_property_read_bool(dev->dev.of_node,
+				      "snps,reset-phy-on-wake");
+	if (hsotg->reset_phy_on_wake && !hsotg->phy) {
+		dev_warn(hsotg->dev,
+			 "Quirk reset-phy-on-wake only supports generic PHYs\n");
+		hsotg->reset_phy_on_wake = false;
+	}
+
 	if (hsotg->dr_mode != USB_DR_MODE_PERIPHERAL) {
 		retval = dwc2_hcd_init(hsotg);
 		if (retval) {
diff --git a/drivers/usb/dwc3/Kconfig b/drivers/usb/dwc3/Kconfig
index 2b1494460d0c..4a62045cc812 100644
--- a/drivers/usb/dwc3/Kconfig
+++ b/drivers/usb/dwc3/Kconfig
@@ -54,7 +54,8 @@ comment "Platform Glue Driver Support"
 
 config USB_DWC3_OMAP
 	tristate "Texas Instruments OMAP5 and similar Platforms"
-	depends on EXTCON && (ARCH_OMAP2PLUS || COMPILE_TEST)
+	depends on ARCH_OMAP2PLUS || COMPILE_TEST
+	depends on EXTCON || !EXTCON
 	depends on OF
 	default USB_DWC3
 	help
@@ -95,6 +96,16 @@ config USB_DWC3_KEYSTONE
 	  Support of USB2/3 functionality in TI Keystone2 and AM654 platforms.
 	  Say 'Y' or 'M' here if you have one such device
 
+config USB_DWC3_MESON_G12A
+       tristate "Amlogic Meson G12A Platforms"
+       depends on OF && COMMON_CLK
+       depends on ARCH_MESON || COMPILE_TEST
+       default USB_DWC3
+       select USB_ROLE_SWITCH
+       help
+         Support USB2/3 functionality in Amlogic G12A platforms.
+	 Say 'Y' or 'M' if you have one such device.
+
 config USB_DWC3_OF_SIMPLE
        tristate "Generic OF Simple Glue Layer"
        depends on OF && COMMON_CLK
@@ -115,7 +126,8 @@ config USB_DWC3_ST
 
 config USB_DWC3_QCOM
 	tristate "Qualcomm Platform"
-	depends on EXTCON && (ARCH_QCOM || COMPILE_TEST)
+	depends on ARCH_QCOM || COMPILE_TEST
+	depends on EXTCON || !EXTCON
 	depends on OF
 	default USB_DWC3
 	help
diff --git a/drivers/usb/dwc3/Makefile b/drivers/usb/dwc3/Makefile
index 6e3ef6144e5d..ae86da0dc5bd 100644
--- a/drivers/usb/dwc3/Makefile
+++ b/drivers/usb/dwc3/Makefile
@@ -47,6 +47,7 @@ obj-$(CONFIG_USB_DWC3_EXYNOS)		+= dwc3-exynos.o
 obj-$(CONFIG_USB_DWC3_PCI)		+= dwc3-pci.o
 obj-$(CONFIG_USB_DWC3_HAPS)		+= dwc3-haps.o
 obj-$(CONFIG_USB_DWC3_KEYSTONE)		+= dwc3-keystone.o
+obj-$(CONFIG_USB_DWC3_MESON_G12A)	+= dwc3-meson-g12a.o
 obj-$(CONFIG_USB_DWC3_OF_SIMPLE)	+= dwc3-of-simple.o
 obj-$(CONFIG_USB_DWC3_ST)		+= dwc3-st.o
 obj-$(CONFIG_USB_DWC3_QCOM)		+= dwc3-qcom.o
diff --git a/drivers/usb/dwc3/core.c b/drivers/usb/dwc3/core.c
index a1b126f90261..4aff1d8dbc4f 100644
--- a/drivers/usb/dwc3/core.c
+++ b/drivers/usb/dwc3/core.c
@@ -828,6 +828,7 @@ static void dwc3_set_incr_burst_type(struct dwc3 *dwc)
 	ret = device_property_read_u32_array(dev,
 			"snps,incr-burst-type-adjustment", vals, ntype);
 	if (ret) {
+		kfree(vals);
 		dev_err(dev, "Error to get property\n");
 		return;
 	}
@@ -846,6 +847,8 @@ static void dwc3_set_incr_burst_type(struct dwc3 *dwc)
 		incrx_mode = INCRX_BURST_MODE;
 	}
 
+	kfree(vals);
+
 	/* Enable Undefined Length INCR Burst and Enable INCRx Burst */
 	cfg &= ~DWC3_GSBUSCFG0_INCRBRST_MASK;
 	if (incrx_mode)
@@ -893,12 +896,6 @@ static int dwc3_core_init(struct dwc3 *dwc)
 	u32			reg;
 	int			ret;
 
-	if (!dwc3_core_is_valid(dwc)) {
-		dev_err(dwc->dev, "this is not a DesignWare USB3 DRD Core\n");
-		ret = -ENODEV;
-		goto err0;
-	}
-
 	/*
 	 * Write Linux Version Code to our GUID register so it's easy to figure
 	 * out which kernel version a bug was found.
@@ -1218,7 +1215,7 @@ static void dwc3_get_properties(struct dwc3 *dwc)
 	u8			tx_max_burst_prd;
 
 	/* default to highest possible threshold */
-	lpm_nyet_threshold = 0xff;
+	lpm_nyet_threshold = 0xf;
 
 	/* default to -3.5dB de-emphasis */
 	tx_de_emphasis = 1;
@@ -1426,6 +1423,11 @@ static int dwc3_probe(struct platform_device *pdev)
 	dwc->regs	= regs;
 	dwc->regs_size	= resource_size(&dwc_res);
 
+	if (!dwc3_core_is_valid(dwc)) {
+		dev_err(dwc->dev, "this is not a DesignWare USB3 DRD Core\n");
+		return -ENODEV;
+	}
+
 	dwc3_get_properties(dwc);
 
 	dwc->reset = devm_reset_control_get_optional_shared(dev, NULL);
@@ -1600,6 +1602,7 @@ static int dwc3_suspend_common(struct dwc3 *dwc, pm_message_t msg)
 		spin_lock_irqsave(&dwc->lock, flags);
 		dwc3_gadget_suspend(dwc);
 		spin_unlock_irqrestore(&dwc->lock, flags);
+		synchronize_irq(dwc->irq_gadget);
 		dwc3_core_exit(dwc);
 		break;
 	case DWC3_GCTL_PRTCAP_HOST:
@@ -1632,6 +1635,7 @@ static int dwc3_suspend_common(struct dwc3 *dwc, pm_message_t msg)
 			spin_lock_irqsave(&dwc->lock, flags);
 			dwc3_gadget_suspend(dwc);
 			spin_unlock_irqrestore(&dwc->lock, flags);
+			synchronize_irq(dwc->irq_gadget);
 		}
 
 		dwc3_otg_exit(dwc);
diff --git a/drivers/usb/dwc3/core.h b/drivers/usb/dwc3/core.h
index 1528d395b156..f19cbeb01087 100644
--- a/drivers/usb/dwc3/core.h
+++ b/drivers/usb/dwc3/core.h
@@ -406,8 +406,7 @@
 #define DWC3_DCTL_TRGTULST_SS_INACT	(DWC3_DCTL_TRGTULST(6))
 
 /* These apply for core versions 1.94a and later */
-#define DWC3_DCTL_LPM_ERRATA_MASK	DWC3_DCTL_LPM_ERRATA(0xf)
-#define DWC3_DCTL_LPM_ERRATA(n)		((n) << 20)
+#define DWC3_DCTL_NYET_THRES(n)		(((n) & 0xf) << 20)
 
 #define DWC3_DCTL_KEEP_CONNECT		BIT(19)
 #define DWC3_DCTL_L1_HIBER_EN		BIT(18)
diff --git a/drivers/usb/dwc3/debug.h b/drivers/usb/dwc3/debug.h
index 6759a7efd8d5..068259fdfb0c 100644
--- a/drivers/usb/dwc3/debug.h
+++ b/drivers/usb/dwc3/debug.h
@@ -250,6 +250,9 @@ static inline void dwc3_decode_get_status(__u8 t, __u16 i, __u16 l, char *str,
 		size_t size)
 {
 	switch (t & USB_RECIP_MASK) {
+	case USB_RECIP_DEVICE:
+		snprintf(str, size, "Get Device Status(Length = %d)", l);
+		break;
 	case USB_RECIP_INTERFACE:
 		snprintf(str, size, "Get Interface Status(Intf = %d, Length = %d)",
 				i, l);
diff --git a/drivers/usb/dwc3/dwc3-meson-g12a.c b/drivers/usb/dwc3/dwc3-meson-g12a.c
new file mode 100644
index 000000000000..2aec31a2eacb
--- /dev/null
+++ b/drivers/usb/dwc3/dwc3-meson-g12a.c
@@ -0,0 +1,604 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * USB Glue for Amlogic G12A SoCs
+ *
+ * Copyright (c) 2019 BayLibre, SAS
+ * Author: Neil Armstrong <narmstrong@baylibre.com>
+ */
+
+/*
+ * The USB is organized with a glue around the DWC3 Controller IP as :
+ * - Control registers for each USB2 Ports
+ * - Control registers for the USB PHY layer
+ * - SuperSpeed PHY can be enabled only if port is used
+ *
+ * TOFIX:
+ * - Add dynamic OTG switching with ID change interrupt
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/platform_device.h>
+#include <linux/clk.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/pm_runtime.h>
+#include <linux/regmap.h>
+#include <linux/bitfield.h>
+#include <linux/bitops.h>
+#include <linux/reset.h>
+#include <linux/phy/phy.h>
+#include <linux/usb/otg.h>
+#include <linux/usb/role.h>
+#include <linux/regulator/consumer.h>
+
+/* USB2 Ports Control Registers */
+
+#define U2P_REG_SIZE						0x20
+
+#define U2P_R0							0x0
+	#define U2P_R0_HOST_DEVICE				BIT(0)
+	#define U2P_R0_POWER_OK					BIT(1)
+	#define U2P_R0_HAST_MODE				BIT(2)
+	#define U2P_R0_POWER_ON_RESET				BIT(3)
+	#define U2P_R0_ID_PULLUP				BIT(4)
+	#define U2P_R0_DRV_VBUS					BIT(5)
+
+#define U2P_R1							0x4
+	#define U2P_R1_PHY_READY				BIT(0)
+	#define U2P_R1_ID_DIG					BIT(1)
+	#define U2P_R1_OTG_SESSION_VALID			BIT(2)
+	#define U2P_R1_VBUS_VALID				BIT(3)
+
+/* USB Glue Control Registers */
+
+#define USB_R0							0x80
+	#define USB_R0_P30_LANE0_TX2RX_LOOPBACK			BIT(17)
+	#define USB_R0_P30_LANE0_EXT_PCLK_REQ			BIT(18)
+	#define USB_R0_P30_PCS_RX_LOS_MASK_VAL_MASK		GENMASK(28, 19)
+	#define USB_R0_U2D_SS_SCALEDOWN_MODE_MASK		GENMASK(30, 29)
+	#define USB_R0_U2D_ACT					BIT(31)
+
+#define USB_R1							0x84
+	#define USB_R1_U3H_BIGENDIAN_GS				BIT(0)
+	#define USB_R1_U3H_PME_ENABLE				BIT(1)
+	#define USB_R1_U3H_HUB_PORT_OVERCURRENT_MASK		GENMASK(4, 2)
+	#define USB_R1_U3H_HUB_PORT_PERM_ATTACH_MASK		GENMASK(9, 7)
+	#define USB_R1_U3H_HOST_U2_PORT_DISABLE_MASK		GENMASK(13, 12)
+	#define USB_R1_U3H_HOST_U3_PORT_DISABLE			BIT(16)
+	#define USB_R1_U3H_HOST_PORT_POWER_CONTROL_PRESENT	BIT(17)
+	#define USB_R1_U3H_HOST_MSI_ENABLE			BIT(18)
+	#define USB_R1_U3H_FLADJ_30MHZ_REG_MASK			GENMASK(24, 19)
+	#define USB_R1_P30_PCS_TX_SWING_FULL_MASK		GENMASK(31, 25)
+
+#define USB_R2							0x88
+	#define USB_R2_P30_PCS_TX_DEEMPH_3P5DB_MASK		GENMASK(25, 20)
+	#define USB_R2_P30_PCS_TX_DEEMPH_6DB_MASK		GENMASK(31, 26)
+
+#define USB_R3							0x8c
+	#define USB_R3_P30_SSC_ENABLE				BIT(0)
+	#define USB_R3_P30_SSC_RANGE_MASK			GENMASK(3, 1)
+	#define USB_R3_P30_SSC_REF_CLK_SEL_MASK			GENMASK(12, 4)
+	#define USB_R3_P30_REF_SSP_EN				BIT(13)
+
+#define USB_R4							0x90
+	#define USB_R4_P21_PORT_RESET_0				BIT(0)
+	#define USB_R4_P21_SLEEP_M0				BIT(1)
+	#define USB_R4_MEM_PD_MASK				GENMASK(3, 2)
+	#define USB_R4_P21_ONLY					BIT(4)
+
+#define USB_R5							0x94
+	#define USB_R5_ID_DIG_SYNC				BIT(0)
+	#define USB_R5_ID_DIG_REG				BIT(1)
+	#define USB_R5_ID_DIG_CFG_MASK				GENMASK(3, 2)
+	#define USB_R5_ID_DIG_EN_0				BIT(4)
+	#define USB_R5_ID_DIG_EN_1				BIT(5)
+	#define USB_R5_ID_DIG_CURR				BIT(6)
+	#define USB_R5_ID_DIG_IRQ				BIT(7)
+	#define USB_R5_ID_DIG_TH_MASK				GENMASK(15, 8)
+	#define USB_R5_ID_DIG_CNT_MASK				GENMASK(23, 16)
+
+enum {
+	USB2_HOST_PHY = 0,
+	USB2_OTG_PHY,
+	USB3_HOST_PHY,
+	PHY_COUNT,
+};
+
+static const char *phy_names[PHY_COUNT] = {
+	"usb2-phy0", "usb2-phy1", "usb3-phy0",
+};
+
+struct dwc3_meson_g12a {
+	struct device		*dev;
+	struct regmap		*regmap;
+	struct clk		*clk;
+	struct reset_control	*reset;
+	struct phy		*phys[PHY_COUNT];
+	enum usb_dr_mode	otg_mode;
+	enum phy_mode		otg_phy_mode;
+	unsigned int		usb2_ports;
+	unsigned int		usb3_ports;
+	struct regulator	*vbus;
+	struct usb_role_switch_desc switch_desc;
+	struct usb_role_switch	*role_switch;
+};
+
+static void dwc3_meson_g12a_usb2_set_mode(struct dwc3_meson_g12a *priv,
+					  int i, enum phy_mode mode)
+{
+	if (mode == PHY_MODE_USB_HOST)
+		regmap_update_bits(priv->regmap, U2P_R0 + (U2P_REG_SIZE * i),
+				U2P_R0_HOST_DEVICE,
+				U2P_R0_HOST_DEVICE);
+	else
+		regmap_update_bits(priv->regmap, U2P_R0 + (U2P_REG_SIZE * i),
+				U2P_R0_HOST_DEVICE, 0);
+}
+
+static int dwc3_meson_g12a_usb2_init(struct dwc3_meson_g12a *priv)
+{
+	int i;
+
+	if (priv->otg_mode == USB_DR_MODE_PERIPHERAL)
+		priv->otg_phy_mode = PHY_MODE_USB_DEVICE;
+	else
+		priv->otg_phy_mode = PHY_MODE_USB_HOST;
+
+	for (i = 0 ; i < USB3_HOST_PHY ; ++i) {
+		if (!priv->phys[i])
+			continue;
+
+		regmap_update_bits(priv->regmap, U2P_R0 + (U2P_REG_SIZE * i),
+				   U2P_R0_POWER_ON_RESET,
+				   U2P_R0_POWER_ON_RESET);
+
+		if (i == USB2_OTG_PHY) {
+			regmap_update_bits(priv->regmap,
+				U2P_R0 + (U2P_REG_SIZE * i),
+				U2P_R0_ID_PULLUP | U2P_R0_DRV_VBUS,
+				U2P_R0_ID_PULLUP | U2P_R0_DRV_VBUS);
+
+			dwc3_meson_g12a_usb2_set_mode(priv, i,
+						      priv->otg_phy_mode);
+		} else
+			dwc3_meson_g12a_usb2_set_mode(priv, i,
+						      PHY_MODE_USB_HOST);
+
+		regmap_update_bits(priv->regmap, U2P_R0 + (U2P_REG_SIZE * i),
+				   U2P_R0_POWER_ON_RESET, 0);
+	}
+
+	return 0;
+}
+
+static void dwc3_meson_g12a_usb3_init(struct dwc3_meson_g12a *priv)
+{
+	regmap_update_bits(priv->regmap, USB_R3,
+			USB_R3_P30_SSC_RANGE_MASK |
+			USB_R3_P30_REF_SSP_EN,
+			USB_R3_P30_SSC_ENABLE |
+			FIELD_PREP(USB_R3_P30_SSC_RANGE_MASK, 2) |
+			USB_R3_P30_REF_SSP_EN);
+	udelay(2);
+
+	regmap_update_bits(priv->regmap, USB_R2,
+			USB_R2_P30_PCS_TX_DEEMPH_3P5DB_MASK,
+			FIELD_PREP(USB_R2_P30_PCS_TX_DEEMPH_3P5DB_MASK, 0x15));
+
+	regmap_update_bits(priv->regmap, USB_R2,
+			USB_R2_P30_PCS_TX_DEEMPH_6DB_MASK,
+			FIELD_PREP(USB_R2_P30_PCS_TX_DEEMPH_6DB_MASK, 0x20));
+
+	udelay(2);
+
+	regmap_update_bits(priv->regmap, USB_R1,
+			USB_R1_U3H_HOST_PORT_POWER_CONTROL_PRESENT,
+			USB_R1_U3H_HOST_PORT_POWER_CONTROL_PRESENT);
+
+	regmap_update_bits(priv->regmap, USB_R1,
+			USB_R1_P30_PCS_TX_SWING_FULL_MASK,
+			FIELD_PREP(USB_R1_P30_PCS_TX_SWING_FULL_MASK, 127));
+}
+
+static void dwc3_meson_g12a_usb_otg_apply_mode(struct dwc3_meson_g12a *priv)
+{
+	if (priv->otg_phy_mode == PHY_MODE_USB_DEVICE) {
+		regmap_update_bits(priv->regmap, USB_R0,
+				USB_R0_U2D_ACT, USB_R0_U2D_ACT);
+		regmap_update_bits(priv->regmap, USB_R0,
+				USB_R0_U2D_SS_SCALEDOWN_MODE_MASK, 0);
+		regmap_update_bits(priv->regmap, USB_R4,
+				USB_R4_P21_SLEEP_M0, USB_R4_P21_SLEEP_M0);
+	} else {
+		regmap_update_bits(priv->regmap, USB_R0,
+				USB_R0_U2D_ACT, 0);
+		regmap_update_bits(priv->regmap, USB_R4,
+				USB_R4_P21_SLEEP_M0, 0);
+	}
+}
+
+static int dwc3_meson_g12a_usb_init(struct dwc3_meson_g12a *priv)
+{
+	int ret;
+
+	ret = dwc3_meson_g12a_usb2_init(priv);
+	if (ret)
+		return ret;
+
+	regmap_update_bits(priv->regmap, USB_R1,
+			USB_R1_U3H_FLADJ_30MHZ_REG_MASK,
+			FIELD_PREP(USB_R1_U3H_FLADJ_30MHZ_REG_MASK, 0x20));
+
+	regmap_update_bits(priv->regmap, USB_R5,
+			USB_R5_ID_DIG_EN_0,
+			USB_R5_ID_DIG_EN_0);
+	regmap_update_bits(priv->regmap, USB_R5,
+			USB_R5_ID_DIG_EN_1,
+			USB_R5_ID_DIG_EN_1);
+	regmap_update_bits(priv->regmap, USB_R5,
+			USB_R5_ID_DIG_TH_MASK,
+			FIELD_PREP(USB_R5_ID_DIG_TH_MASK, 0xff));
+
+	/* If we have an actual SuperSpeed port, initialize it */
+	if (priv->usb3_ports)
+		dwc3_meson_g12a_usb3_init(priv);
+
+	dwc3_meson_g12a_usb_otg_apply_mode(priv);
+
+	return 0;
+}
+
+static const struct regmap_config phy_meson_g12a_usb3_regmap_conf = {
+	.reg_bits = 8,
+	.val_bits = 32,
+	.reg_stride = 4,
+	.max_register = USB_R5,
+};
+
+static int dwc3_meson_g12a_get_phys(struct dwc3_meson_g12a *priv)
+{
+	int i;
+
+	for (i = 0 ; i < PHY_COUNT ; ++i) {
+		priv->phys[i] = devm_phy_optional_get(priv->dev, phy_names[i]);
+		if (!priv->phys[i])
+			continue;
+
+		if (IS_ERR(priv->phys[i]))
+			return PTR_ERR(priv->phys[i]);
+
+		if (i == USB3_HOST_PHY)
+			priv->usb3_ports++;
+		else
+			priv->usb2_ports++;
+	}
+
+	dev_info(priv->dev, "USB2 ports: %d\n", priv->usb2_ports);
+	dev_info(priv->dev, "USB3 ports: %d\n", priv->usb3_ports);
+
+	return 0;
+}
+
+static enum phy_mode dwc3_meson_g12a_get_id(struct dwc3_meson_g12a *priv)
+{
+	u32 reg;
+
+	regmap_read(priv->regmap, USB_R5, &reg);
+
+	if (reg & (USB_R5_ID_DIG_SYNC | USB_R5_ID_DIG_REG))
+		return PHY_MODE_USB_DEVICE;
+
+	return PHY_MODE_USB_HOST;
+}
+
+static int dwc3_meson_g12a_otg_mode_set(struct dwc3_meson_g12a *priv,
+					enum phy_mode mode)
+{
+	int ret;
+
+	if (!priv->phys[USB2_OTG_PHY])
+		return -EINVAL;
+
+	if (mode == PHY_MODE_USB_HOST)
+		dev_info(priv->dev, "switching to Host Mode\n");
+	else
+		dev_info(priv->dev, "switching to Device Mode\n");
+
+	if (priv->vbus) {
+		if (mode == PHY_MODE_USB_DEVICE)
+			ret = regulator_disable(priv->vbus);
+		else
+			ret = regulator_enable(priv->vbus);
+		if (ret)
+			return ret;
+	}
+
+	priv->otg_phy_mode = mode;
+
+	dwc3_meson_g12a_usb2_set_mode(priv, USB2_OTG_PHY, mode);
+
+	dwc3_meson_g12a_usb_otg_apply_mode(priv);
+
+	return 0;
+}
+
+static int dwc3_meson_g12a_role_set(struct device *dev, enum usb_role role)
+{
+	struct dwc3_meson_g12a *priv = dev_get_drvdata(dev);
+	enum phy_mode mode;
+
+	if (role == USB_ROLE_NONE)
+		return 0;
+
+	mode = (role == USB_ROLE_HOST) ? PHY_MODE_USB_HOST
+				       : PHY_MODE_USB_DEVICE;
+
+	if (mode == priv->otg_phy_mode)
+		return 0;
+
+	return dwc3_meson_g12a_otg_mode_set(priv, mode);
+}
+
+static enum usb_role dwc3_meson_g12a_role_get(struct device *dev)
+{
+	struct dwc3_meson_g12a *priv = dev_get_drvdata(dev);
+
+	return priv->otg_phy_mode == PHY_MODE_USB_HOST ?
+		USB_ROLE_HOST : USB_ROLE_DEVICE;
+}
+
+static struct device *dwc3_meson_g12_find_child(struct device *dev,
+						const char *compatible)
+{
+	struct platform_device *pdev;
+	struct device_node *np;
+
+	np = of_get_compatible_child(dev->of_node, compatible);
+	if (!np)
+		return NULL;
+
+	pdev = of_find_device_by_node(np);
+	of_node_put(np);
+	if (!pdev)
+		return NULL;
+
+	return &pdev->dev;
+}
+
+static int dwc3_meson_g12a_probe(struct platform_device *pdev)
+{
+	struct dwc3_meson_g12a	*priv;
+	struct device		*dev = &pdev->dev;
+	struct device_node	*np = dev->of_node;
+	void __iomem *base;
+	struct resource *res;
+	enum phy_mode otg_id;
+	int ret, i;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	base = devm_ioremap_resource(dev, res);
+	if (IS_ERR(base))
+		return PTR_ERR(base);
+
+	priv->regmap = devm_regmap_init_mmio(dev, base,
+					     &phy_meson_g12a_usb3_regmap_conf);
+	if (IS_ERR(priv->regmap))
+		return PTR_ERR(priv->regmap);
+
+	priv->vbus = devm_regulator_get_optional(dev, "vbus");
+	if (IS_ERR(priv->vbus)) {
+		if (PTR_ERR(priv->vbus) == -EPROBE_DEFER)
+			return PTR_ERR(priv->vbus);
+		priv->vbus = NULL;
+	}
+
+	priv->clk = devm_clk_get(dev, NULL);
+	if (IS_ERR(priv->clk))
+		return PTR_ERR(priv->clk);
+
+	ret = clk_prepare_enable(priv->clk);
+	if (ret)
+		return ret;
+
+	devm_add_action_or_reset(dev,
+				 (void(*)(void *))clk_disable_unprepare,
+				 priv->clk);
+
+	platform_set_drvdata(pdev, priv);
+	priv->dev = dev;
+
+	priv->reset = devm_reset_control_get(dev, NULL);
+	if (IS_ERR(priv->reset)) {
+		ret = PTR_ERR(priv->reset);
+		dev_err(dev, "failed to get device reset, err=%d\n", ret);
+		return ret;
+	}
+
+	ret = reset_control_reset(priv->reset);
+	if (ret)
+		return ret;
+
+	ret = dwc3_meson_g12a_get_phys(priv);
+	if (ret)
+		return ret;
+
+	if (priv->vbus) {
+		ret = regulator_enable(priv->vbus);
+		if (ret)
+			return ret;
+	}
+
+	/* Get dr_mode */
+	priv->otg_mode = usb_get_dr_mode(dev);
+
+	dwc3_meson_g12a_usb_init(priv);
+
+	/* Init PHYs */
+	for (i = 0 ; i < PHY_COUNT ; ++i) {
+		ret = phy_init(priv->phys[i]);
+		if (ret)
+			return ret;
+	}
+
+	/* Set PHY Power */
+	for (i = 0 ; i < PHY_COUNT ; ++i) {
+		ret = phy_power_on(priv->phys[i]);
+		if (ret)
+			goto err_phys_exit;
+	}
+
+	ret = of_platform_populate(np, NULL, NULL, dev);
+	if (ret) {
+		clk_disable_unprepare(priv->clk);
+		goto err_phys_power;
+	}
+
+	/* Setup OTG mode corresponding to the ID pin */
+	if (priv->otg_mode == USB_DR_MODE_OTG) {
+		/* TOFIX Handle ID mode toggling via IRQ */
+		otg_id = dwc3_meson_g12a_get_id(priv);
+		if (otg_id != priv->otg_phy_mode) {
+			if (dwc3_meson_g12a_otg_mode_set(priv, otg_id))
+				dev_warn(dev, "Failed to switch OTG mode\n");
+		}
+	}
+
+	/* Setup role switcher */
+	priv->switch_desc.usb2_port = dwc3_meson_g12_find_child(dev,
+								"snps,dwc3");
+	priv->switch_desc.udc = dwc3_meson_g12_find_child(dev, "snps,dwc2");
+	priv->switch_desc.allow_userspace_control = true;
+	priv->switch_desc.set = dwc3_meson_g12a_role_set;
+	priv->switch_desc.get = dwc3_meson_g12a_role_get;
+
+	priv->role_switch = usb_role_switch_register(dev, &priv->switch_desc);
+	if (IS_ERR(priv->role_switch))
+		dev_warn(dev, "Unable to register Role Switch\n");
+
+	pm_runtime_set_active(dev);
+	pm_runtime_enable(dev);
+	pm_runtime_get_sync(dev);
+
+	return 0;
+
+err_phys_power:
+	for (i = 0 ; i < PHY_COUNT ; ++i)
+		phy_power_off(priv->phys[i]);
+
+err_phys_exit:
+	for (i = 0 ; i < PHY_COUNT ; ++i)
+		phy_exit(priv->phys[i]);
+
+	return ret;
+}
+
+static int dwc3_meson_g12a_remove(struct platform_device *pdev)
+{
+	struct dwc3_meson_g12a *priv = platform_get_drvdata(pdev);
+	struct device *dev = &pdev->dev;
+	int i;
+
+	usb_role_switch_unregister(priv->role_switch);
+
+	of_platform_depopulate(dev);
+
+	for (i = 0 ; i < PHY_COUNT ; ++i) {
+		phy_power_off(priv->phys[i]);
+		phy_exit(priv->phys[i]);
+	}
+
+	pm_runtime_disable(dev);
+	pm_runtime_put_noidle(dev);
+	pm_runtime_set_suspended(dev);
+
+	return 0;
+}
+
+static int __maybe_unused dwc3_meson_g12a_runtime_suspend(struct device *dev)
+{
+	struct dwc3_meson_g12a	*priv = dev_get_drvdata(dev);
+
+	clk_disable(priv->clk);
+
+	return 0;
+}
+
+static int __maybe_unused dwc3_meson_g12a_runtime_resume(struct device *dev)
+{
+	struct dwc3_meson_g12a	*priv = dev_get_drvdata(dev);
+
+	return clk_enable(priv->clk);
+}
+
+static int __maybe_unused dwc3_meson_g12a_suspend(struct device *dev)
+{
+	struct dwc3_meson_g12a *priv = dev_get_drvdata(dev);
+	int i;
+
+	for (i = 0 ; i < PHY_COUNT ; ++i) {
+		phy_power_off(priv->phys[i]);
+		phy_exit(priv->phys[i]);
+	}
+
+	reset_control_assert(priv->reset);
+
+	return 0;
+}
+
+static int __maybe_unused dwc3_meson_g12a_resume(struct device *dev)
+{
+	struct dwc3_meson_g12a *priv = dev_get_drvdata(dev);
+	int i, ret;
+
+	reset_control_deassert(priv->reset);
+
+	dwc3_meson_g12a_usb_init(priv);
+
+	/* Init PHYs */
+	for (i = 0 ; i < PHY_COUNT ; ++i) {
+		ret = phy_init(priv->phys[i]);
+		if (ret)
+			return ret;
+	}
+
+	/* Set PHY Power */
+	for (i = 0 ; i < PHY_COUNT ; ++i) {
+		ret = phy_power_on(priv->phys[i]);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static const struct dev_pm_ops dwc3_meson_g12a_dev_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(dwc3_meson_g12a_suspend, dwc3_meson_g12a_resume)
+	SET_RUNTIME_PM_OPS(dwc3_meson_g12a_runtime_suspend,
+			   dwc3_meson_g12a_runtime_resume, NULL)
+};
+
+static const struct of_device_id dwc3_meson_g12a_match[] = {
+	{ .compatible = "amlogic,meson-g12a-usb-ctrl" },
+	{ /* Sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, dwc3_meson_g12a_match);
+
+static struct platform_driver dwc3_meson_g12a_driver = {
+	.probe		= dwc3_meson_g12a_probe,
+	.remove		= dwc3_meson_g12a_remove,
+	.driver		= {
+		.name	= "dwc3-meson-g12a",
+		.of_match_table = dwc3_meson_g12a_match,
+		.pm	= &dwc3_meson_g12a_dev_pm_ops,
+	},
+};
+
+module_platform_driver(dwc3_meson_g12a_driver);
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("Amlogic Meson G12A USB Glue Layer");
+MODULE_AUTHOR("Neil Armstrong <narmstrong@baylibre.com>");
diff --git a/drivers/usb/dwc3/dwc3-of-simple.c b/drivers/usb/dwc3/dwc3-of-simple.c
index 4c2771c5e727..bdac3e7d7b18 100644
--- a/drivers/usb/dwc3/dwc3-of-simple.c
+++ b/drivers/usb/dwc3/dwc3-of-simple.c
@@ -24,59 +24,13 @@
 
 struct dwc3_of_simple {
 	struct device		*dev;
-	struct clk		**clks;
+	struct clk_bulk_data	*clks;
 	int			num_clocks;
 	struct reset_control	*resets;
 	bool			pulse_resets;
 	bool			need_reset;
 };
 
-static int dwc3_of_simple_clk_init(struct dwc3_of_simple *simple, int count)
-{
-	struct device		*dev = simple->dev;
-	struct device_node	*np = dev->of_node;
-	int			i;
-
-	simple->num_clocks = count;
-
-	if (!count)
-		return 0;
-
-	simple->clks = devm_kcalloc(dev, simple->num_clocks,
-			sizeof(struct clk *), GFP_KERNEL);
-	if (!simple->clks)
-		return -ENOMEM;
-
-	for (i = 0; i < simple->num_clocks; i++) {
-		struct clk	*clk;
-		int		ret;
-
-		clk = of_clk_get(np, i);
-		if (IS_ERR(clk)) {
-			while (--i >= 0) {
-				clk_disable_unprepare(simple->clks[i]);
-				clk_put(simple->clks[i]);
-			}
-			return PTR_ERR(clk);
-		}
-
-		ret = clk_prepare_enable(clk);
-		if (ret < 0) {
-			while (--i >= 0) {
-				clk_disable_unprepare(simple->clks[i]);
-				clk_put(simple->clks[i]);
-			}
-			clk_put(clk);
-
-			return ret;
-		}
-
-		simple->clks[i] = clk;
-	}
-
-	return 0;
-}
-
 static int dwc3_of_simple_probe(struct platform_device *pdev)
 {
 	struct dwc3_of_simple	*simple;
@@ -84,7 +38,6 @@ static int dwc3_of_simple_probe(struct platform_device *pdev)
 	struct device_node	*np = dev->of_node;
 
 	int			ret;
-	int			i;
 	bool			shared_resets = false;
 
 	simple = devm_kzalloc(dev, sizeof(*simple), GFP_KERNEL);
@@ -107,7 +60,8 @@ static int dwc3_of_simple_probe(struct platform_device *pdev)
 		simple->pulse_resets = true;
 	}
 
-	simple->resets = of_reset_control_array_get(np, shared_resets, true);
+	simple->resets = of_reset_control_array_get(np, shared_resets, true,
+						    true);
 	if (IS_ERR(simple->resets)) {
 		ret = PTR_ERR(simple->resets);
 		dev_err(dev, "failed to get device resets, err=%d\n", ret);
@@ -124,20 +78,18 @@ static int dwc3_of_simple_probe(struct platform_device *pdev)
 			goto err_resetc_put;
 	}
 
-	ret = dwc3_of_simple_clk_init(simple, of_count_phandle_with_args(np,
-						"clocks", "#clock-cells"));
+	ret = clk_bulk_get_all(simple->dev, &simple->clks);
+	if (ret < 0)
+		goto err_resetc_assert;
+
+	simple->num_clocks = ret;
+	ret = clk_bulk_prepare_enable(simple->num_clocks, simple->clks);
 	if (ret)
 		goto err_resetc_assert;
 
 	ret = of_platform_populate(np, NULL, NULL, dev);
-	if (ret) {
-		for (i = 0; i < simple->num_clocks; i++) {
-			clk_disable_unprepare(simple->clks[i]);
-			clk_put(simple->clks[i]);
-		}
-
-		goto err_resetc_assert;
-	}
+	if (ret)
+		goto err_clk_put;
 
 	pm_runtime_set_active(dev);
 	pm_runtime_enable(dev);
@@ -145,6 +97,10 @@ static int dwc3_of_simple_probe(struct platform_device *pdev)
 
 	return 0;
 
+err_clk_put:
+	clk_bulk_disable_unprepare(simple->num_clocks, simple->clks);
+	clk_bulk_put_all(simple->num_clocks, simple->clks);
+
 err_resetc_assert:
 	if (!simple->pulse_resets)
 		reset_control_assert(simple->resets);
@@ -158,14 +114,11 @@ static int dwc3_of_simple_remove(struct platform_device *pdev)
 {
 	struct dwc3_of_simple	*simple = platform_get_drvdata(pdev);
 	struct device		*dev = &pdev->dev;
-	int			i;
 
 	of_platform_depopulate(dev);
 
-	for (i = 0; i < simple->num_clocks; i++) {
-		clk_disable_unprepare(simple->clks[i]);
-		clk_put(simple->clks[i]);
-	}
+	clk_bulk_disable_unprepare(simple->num_clocks, simple->clks);
+	clk_bulk_put_all(simple->num_clocks, simple->clks);
 	simple->num_clocks = 0;
 
 	if (!simple->pulse_resets)
@@ -183,10 +136,8 @@ static int dwc3_of_simple_remove(struct platform_device *pdev)
 static int __maybe_unused dwc3_of_simple_runtime_suspend(struct device *dev)
 {
 	struct dwc3_of_simple	*simple = dev_get_drvdata(dev);
-	int			i;
 
-	for (i = 0; i < simple->num_clocks; i++)
-		clk_disable(simple->clks[i]);
+	clk_bulk_disable(simple->num_clocks, simple->clks);
 
 	return 0;
 }
@@ -194,19 +145,8 @@ static int __maybe_unused dwc3_of_simple_runtime_suspend(struct device *dev)
 static int __maybe_unused dwc3_of_simple_runtime_resume(struct device *dev)
 {
 	struct dwc3_of_simple	*simple = dev_get_drvdata(dev);
-	int			ret;
-	int			i;
-
-	for (i = 0; i < simple->num_clocks; i++) {
-		ret = clk_enable(simple->clks[i]);
-		if (ret < 0) {
-			while (--i >= 0)
-				clk_disable(simple->clks[i]);
-			return ret;
-		}
-	}
 
-	return 0;
+	return clk_bulk_enable(simple->num_clocks, simple->clks);
 }
 
 static int __maybe_unused dwc3_of_simple_suspend(struct device *dev)
diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c
index e293400cc6e9..d67655384eb2 100644
--- a/drivers/usb/dwc3/gadget.c
+++ b/drivers/usb/dwc3/gadget.c
@@ -2863,7 +2863,7 @@ static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
 				"LPM Erratum not available on dwc3 revisions < 2.40a\n");
 
 		if (dwc->has_lpm_erratum && dwc->revision >= DWC3_REVISION_240A)
-			reg |= DWC3_DCTL_LPM_ERRATA(dwc->lpm_nyet_threshold);
+			reg |= DWC3_DCTL_NYET_THRES(dwc->lpm_nyet_threshold);
 
 		dwc3_writel(dwc->regs, DWC3_DCTL, reg);
 	} else {
@@ -3301,6 +3301,7 @@ int dwc3_gadget_init(struct dwc3 *dwc)
 	dwc->gadget.sg_supported	= true;
 	dwc->gadget.name		= "dwc3-gadget";
 	dwc->gadget.is_otg		= dwc->dr_mode == USB_DR_MODE_OTG;
+	dwc->gadget.lpm_capable		= true;
 
 	/*
 	 * FIXME We might be setting max_speed to <SUPER, however versions
@@ -3384,8 +3385,6 @@ int dwc3_gadget_suspend(struct dwc3 *dwc)
 	dwc3_disconnect_gadget(dwc);
 	__dwc3_gadget_stop(dwc);
 
-	synchronize_irq(dwc->irq_gadget);
-
 	return 0;
 }
 
diff --git a/drivers/usb/gadget/function/f_fs.c b/drivers/usb/gadget/function/f_fs.c
index 20413c276c61..47be961f1bf3 100644
--- a/drivers/usb/gadget/function/f_fs.c
+++ b/drivers/usb/gadget/function/f_fs.c
@@ -1133,7 +1133,8 @@ error_lock:
 error_mutex:
 	mutex_unlock(&epfile->mutex);
 error:
-	ffs_free_buffer(io_data);
+	if (ret != -EIOCBQUEUED) /* don't free if there is iocb queued */
+		ffs_free_buffer(io_data);
 	return ret;
 }
 
diff --git a/drivers/usb/gadget/function/f_ncm.c b/drivers/usb/gadget/function/f_ncm.c
index 5780fba620ab..2d6e76e4cffa 100644
--- a/drivers/usb/gadget/function/f_ncm.c
+++ b/drivers/usb/gadget/function/f_ncm.c
@@ -23,6 +23,7 @@
 #include "u_ether.h"
 #include "u_ether_configfs.h"
 #include "u_ncm.h"
+#include "configfs.h"
 
 /*
  * This function is a "CDC Network Control Model" (CDC NCM) Ethernet link.
@@ -35,9 +36,7 @@
 
 /* to trigger crc/non-crc ndp signature */
 
-#define NCM_NDP_HDR_CRC_MASK	0x01000000
 #define NCM_NDP_HDR_CRC		0x01000000
-#define NCM_NDP_HDR_NOCRC	0x00000000
 
 enum ncm_notify_state {
 	NCM_NOTIFY_NONE,		/* don't notify */
@@ -526,6 +525,7 @@ static inline void ncm_reset_values(struct f_ncm *ncm)
 {
 	ncm->parser_opts = &ndp16_opts;
 	ncm->is_crc = false;
+	ncm->ndp_sign = ncm->parser_opts->ndp_sign;
 	ncm->port.cdc_filter = DEFAULT_FILTER;
 
 	/* doesn't make sense for ncm, fixed size used */
@@ -805,25 +805,20 @@ static int ncm_setup(struct usb_function *f, const struct usb_ctrlrequest *ctrl)
 	case ((USB_DIR_OUT | USB_TYPE_CLASS | USB_RECIP_INTERFACE) << 8)
 		| USB_CDC_SET_CRC_MODE:
 	{
-		int ndp_hdr_crc = 0;
-
 		if (w_length != 0 || w_index != ncm->ctrl_id)
 			goto invalid;
 		switch (w_value) {
 		case 0x0000:
 			ncm->is_crc = false;
-			ndp_hdr_crc = NCM_NDP_HDR_NOCRC;
 			DBG(cdev, "non-CRC mode selected\n");
 			break;
 		case 0x0001:
 			ncm->is_crc = true;
-			ndp_hdr_crc = NCM_NDP_HDR_CRC;
 			DBG(cdev, "CRC mode selected\n");
 			break;
 		default:
 			goto invalid;
 		}
-		ncm->ndp_sign = ncm->parser_opts->ndp_sign | ndp_hdr_crc;
 		value = 0;
 		break;
 	}
@@ -840,6 +835,8 @@ invalid:
 			ctrl->bRequestType, ctrl->bRequest,
 			w_value, w_index, w_length);
 	}
+	ncm->ndp_sign = ncm->parser_opts->ndp_sign |
+		(ncm->is_crc ? NCM_NDP_HDR_CRC : 0);
 
 	/* respond with data transfer or status phase? */
 	if (value >= 0) {
@@ -1395,6 +1392,16 @@ static int ncm_bind(struct usb_configuration *c, struct usb_function *f)
 		return -EINVAL;
 
 	ncm_opts = container_of(f->fi, struct f_ncm_opts, func_inst);
+
+	if (cdev->use_os_string) {
+		f->os_desc_table = kzalloc(sizeof(*f->os_desc_table),
+					   GFP_KERNEL);
+		if (!f->os_desc_table)
+			return -ENOMEM;
+		f->os_desc_n = 1;
+		f->os_desc_table[0].os_desc = &ncm_opts->ncm_os_desc;
+	}
+
 	/*
 	 * in drivers/usb/gadget/configfs.c:configfs_composite_bind()
 	 * configurations are bound in sequence with list_for_each_entry,
@@ -1408,13 +1415,15 @@ static int ncm_bind(struct usb_configuration *c, struct usb_function *f)
 		status = gether_register_netdev(ncm_opts->net);
 		mutex_unlock(&ncm_opts->lock);
 		if (status)
-			return status;
+			goto fail;
 		ncm_opts->bound = true;
 	}
 	us = usb_gstrings_attach(cdev, ncm_strings,
 				 ARRAY_SIZE(ncm_string_defs));
-	if (IS_ERR(us))
-		return PTR_ERR(us);
+	if (IS_ERR(us)) {
+		status = PTR_ERR(us);
+		goto fail;
+	}
 	ncm_control_intf.iInterface = us[STRING_CTRL_IDX].id;
 	ncm_data_nop_intf.iInterface = us[STRING_DATA_IDX].id;
 	ncm_data_intf.iInterface = us[STRING_DATA_IDX].id;
@@ -1431,6 +1440,10 @@ static int ncm_bind(struct usb_configuration *c, struct usb_function *f)
 	ncm_control_intf.bInterfaceNumber = status;
 	ncm_union_desc.bMasterInterface0 = status;
 
+	if (cdev->use_os_string)
+		f->os_desc_table[0].if_id =
+			ncm_iad_desc.bFirstInterface;
+
 	status = usb_interface_id(c, f);
 	if (status < 0)
 		goto fail;
@@ -1510,6 +1523,9 @@ static int ncm_bind(struct usb_configuration *c, struct usb_function *f)
 	return 0;
 
 fail:
+	kfree(f->os_desc_table);
+	f->os_desc_n = 0;
+
 	if (ncm->notify_req) {
 		kfree(ncm->notify_req->buf);
 		usb_ep_free_request(ncm->notify, ncm->notify_req);
@@ -1564,16 +1580,22 @@ static void ncm_free_inst(struct usb_function_instance *f)
 		gether_cleanup(netdev_priv(opts->net));
 	else
 		free_netdev(opts->net);
+	kfree(opts->ncm_interf_group);
 	kfree(opts);
 }
 
 static struct usb_function_instance *ncm_alloc_inst(void)
 {
 	struct f_ncm_opts *opts;
+	struct usb_os_desc *descs[1];
+	char *names[1];
+	struct config_group *ncm_interf_group;
 
 	opts = kzalloc(sizeof(*opts), GFP_KERNEL);
 	if (!opts)
 		return ERR_PTR(-ENOMEM);
+	opts->ncm_os_desc.ext_compat_id = opts->ncm_ext_compat_id;
+
 	mutex_init(&opts->lock);
 	opts->func_inst.free_func_inst = ncm_free_inst;
 	opts->net = gether_setup_default();
@@ -1582,8 +1604,20 @@ static struct usb_function_instance *ncm_alloc_inst(void)
 		kfree(opts);
 		return ERR_CAST(net);
 	}
+	INIT_LIST_HEAD(&opts->ncm_os_desc.ext_prop);
+
+	descs[0] = &opts->ncm_os_desc;
+	names[0] = "ncm";
 
 	config_group_init_type_name(&opts->func_inst.group, "", &ncm_func_type);
+	ncm_interf_group =
+		usb_os_desc_prepare_interf_dir(&opts->func_inst.group, 1, descs,
+					       names, THIS_MODULE);
+	if (IS_ERR(ncm_interf_group)) {
+		ncm_free_inst(&opts->func_inst);
+		return ERR_CAST(ncm_interf_group);
+	}
+	opts->ncm_interf_group = ncm_interf_group;
 
 	return &opts->func_inst;
 }
@@ -1609,6 +1643,9 @@ static void ncm_unbind(struct usb_configuration *c, struct usb_function *f)
 
 	hrtimer_cancel(&ncm->task_timer);
 
+	kfree(f->os_desc_table);
+	f->os_desc_n = 0;
+
 	ncm_string_defs[0].id = 0;
 	usb_free_all_descriptors(f);
 
diff --git a/drivers/usb/gadget/function/f_uac1_legacy.c b/drivers/usb/gadget/function/f_uac1_legacy.c
index 24c086bcdeaa..6677ae932de0 100644
--- a/drivers/usb/gadget/function/f_uac1_legacy.c
+++ b/drivers/usb/gadget/function/f_uac1_legacy.c
@@ -54,8 +54,8 @@ static struct uac1_ac_header_descriptor_1 ac_header_desc = {
 	.bLength =		UAC_DT_AC_HEADER_LENGTH,
 	.bDescriptorType =	USB_DT_CS_INTERFACE,
 	.bDescriptorSubtype =	UAC_HEADER,
-	.bcdADC =		__constant_cpu_to_le16(0x0100),
-	.wTotalLength =		__constant_cpu_to_le16(UAC_DT_TOTAL_LENGTH),
+	.bcdADC =		cpu_to_le16(0x0100),
+	.wTotalLength =		cpu_to_le16(UAC_DT_TOTAL_LENGTH),
 	.bInCollection =	F_AUDIO_NUM_INTERFACES,
 	.baInterfaceNr = {
 	/* Interface number of the first AudioStream interface */
@@ -183,7 +183,7 @@ static struct uac_iso_endpoint_descriptor as_iso_out_desc = {
 	.bDescriptorSubtype =	UAC_EP_GENERAL,
 	.bmAttributes = 	1,
 	.bLockDelayUnits =	1,
-	.wLockDelay =		__constant_cpu_to_le16(1),
+	.wLockDelay =		cpu_to_le16(1),
 };
 
 static struct usb_descriptor_header *f_audio_desc[] = {
diff --git a/drivers/usb/gadget/function/u_ncm.h b/drivers/usb/gadget/function/u_ncm.h
index d483e45c0f77..70da3201a1d0 100644
--- a/drivers/usb/gadget/function/u_ncm.h
+++ b/drivers/usb/gadget/function/u_ncm.h
@@ -20,6 +20,9 @@ struct f_ncm_opts {
 	struct net_device		*net;
 	bool				bound;
 
+	struct config_group		*ncm_interf_group;
+	struct usb_os_desc		ncm_os_desc;
+	char				ncm_ext_compat_id[16];
 	/*
 	 * Read/write access to configfs attributes is handled by configfs.
 	 *
diff --git a/drivers/usb/gadget/udc/aspeed-vhub/epn.c b/drivers/usb/gadget/udc/aspeed-vhub/epn.c
index 83340f4fdc6e..35941dc125f9 100644
--- a/drivers/usb/gadget/udc/aspeed-vhub/epn.c
+++ b/drivers/usb/gadget/udc/aspeed-vhub/epn.c
@@ -593,10 +593,6 @@ static int ast_vhub_epn_disable(struct usb_ep* u_ep)
 static int ast_vhub_epn_enable(struct usb_ep* u_ep,
 			       const struct usb_endpoint_descriptor *desc)
 {
-	static const char *ep_type_string[] __maybe_unused = { "ctrl",
-							       "isoc",
-							       "bulk",
-							       "intr" };
 	struct ast_vhub_ep *ep = to_ast_ep(u_ep);
 	struct ast_vhub_dev *dev;
 	struct ast_vhub *vhub;
@@ -646,7 +642,7 @@ static int ast_vhub_epn_enable(struct usb_ep* u_ep,
 	ep->epn.wedged = false;
 
 	EPDBG(ep, "Enabling [%s] %s num %d maxpacket=%d\n",
-	      ep->epn.is_in ? "in" : "out", ep_type_string[type],
+	      ep->epn.is_in ? "in" : "out", usb_ep_type_string(type),
 	      usb_endpoint_num(desc), maxpacket);
 
 	/* Can we use DMA descriptor mode ? */
diff --git a/drivers/usb/gadget/udc/atmel_usba_udc.c b/drivers/usb/gadget/udc/atmel_usba_udc.c
index 660712e0bf98..503d275bc4c4 100644
--- a/drivers/usb/gadget/udc/atmel_usba_udc.c
+++ b/drivers/usb/gadget/udc/atmel_usba_udc.c
@@ -358,8 +358,20 @@ static inline u32 usba_int_enb_get(struct usba_udc *udc)
 	return udc->int_enb_cache;
 }
 
-static inline void usba_int_enb_set(struct usba_udc *udc, u32 val)
+static inline void usba_int_enb_set(struct usba_udc *udc, u32 mask)
 {
+	u32 val;
+
+	val = udc->int_enb_cache | mask;
+	usba_writel(udc, INT_ENB, val);
+	udc->int_enb_cache = val;
+}
+
+static inline void usba_int_enb_clear(struct usba_udc *udc, u32 mask)
+{
+	u32 val;
+
+	val = udc->int_enb_cache & ~mask;
 	usba_writel(udc, INT_ENB, val);
 	udc->int_enb_cache = val;
 }
@@ -629,14 +641,12 @@ usba_ep_enable(struct usb_ep *_ep, const struct usb_endpoint_descriptor *desc)
 	if (ep->can_dma) {
 		u32 ctrl;
 
-		usba_int_enb_set(udc, usba_int_enb_get(udc) |
-				      USBA_BF(EPT_INT, 1 << ep->index) |
+		usba_int_enb_set(udc, USBA_BF(EPT_INT, 1 << ep->index) |
 				      USBA_BF(DMA_INT, 1 << ep->index));
 		ctrl = USBA_AUTO_VALID | USBA_INTDIS_DMA;
 		usba_ep_writel(ep, CTL_ENB, ctrl);
 	} else {
-		usba_int_enb_set(udc, usba_int_enb_get(udc) |
-				      USBA_BF(EPT_INT, 1 << ep->index));
+		usba_int_enb_set(udc, USBA_BF(EPT_INT, 1 << ep->index));
 	}
 
 	spin_unlock_irqrestore(&udc->lock, flags);
@@ -680,8 +690,7 @@ static int usba_ep_disable(struct usb_ep *_ep)
 		usba_dma_readl(ep, STATUS);
 	}
 	usba_ep_writel(ep, CTL_DIS, USBA_EPT_ENABLE);
-	usba_int_enb_set(udc, usba_int_enb_get(udc) &
-			      ~USBA_BF(EPT_INT, 1 << ep->index));
+	usba_int_enb_clear(udc, USBA_BF(EPT_INT, 1 << ep->index));
 
 	request_complete_list(ep, &req_list, -ESHUTDOWN);
 
@@ -1694,6 +1703,9 @@ static void usba_dma_irq(struct usba_udc *udc, struct usba_ep *ep)
 	}
 }
 
+static int start_clock(struct usba_udc *udc);
+static void stop_clock(struct usba_udc *udc);
+
 static irqreturn_t usba_udc_irq(int irq, void *devid)
 {
 	struct usba_udc *udc = devid;
@@ -1708,10 +1720,13 @@ static irqreturn_t usba_udc_irq(int irq, void *devid)
 	DBG(DBG_INT, "irq, status=%#08x\n", status);
 
 	if (status & USBA_DET_SUSPEND) {
+		usba_writel(udc, INT_CLR, USBA_DET_SUSPEND|USBA_WAKE_UP);
+		usba_int_enb_set(udc, USBA_WAKE_UP);
+		usba_int_enb_clear(udc, USBA_DET_SUSPEND);
+		udc->suspended = true;
 		toggle_bias(udc, 0);
-		usba_writel(udc, INT_CLR, USBA_DET_SUSPEND);
-		usba_int_enb_set(udc, int_enb | USBA_WAKE_UP);
 		udc->bias_pulse_needed = true;
+		stop_clock(udc);
 		DBG(DBG_BUS, "Suspend detected\n");
 		if (udc->gadget.speed != USB_SPEED_UNKNOWN
 				&& udc->driver && udc->driver->suspend) {
@@ -1722,14 +1737,17 @@ static irqreturn_t usba_udc_irq(int irq, void *devid)
 	}
 
 	if (status & USBA_WAKE_UP) {
+		start_clock(udc);
 		toggle_bias(udc, 1);
 		usba_writel(udc, INT_CLR, USBA_WAKE_UP);
-		usba_int_enb_set(udc, int_enb & ~USBA_WAKE_UP);
 		DBG(DBG_BUS, "Wake Up CPU detected\n");
 	}
 
 	if (status & USBA_END_OF_RESUME) {
+		udc->suspended = false;
 		usba_writel(udc, INT_CLR, USBA_END_OF_RESUME);
+		usba_int_enb_clear(udc, USBA_WAKE_UP);
+		usba_int_enb_set(udc, USBA_DET_SUSPEND);
 		generate_bias_pulse(udc);
 		DBG(DBG_BUS, "Resume detected\n");
 		if (udc->gadget.speed != USB_SPEED_UNKNOWN
@@ -1744,6 +1762,8 @@ static irqreturn_t usba_udc_irq(int irq, void *devid)
 	if (dma_status) {
 		int i;
 
+		usba_int_enb_set(udc, USBA_DET_SUSPEND);
+
 		for (i = 1; i <= USBA_NR_DMAS; i++)
 			if (dma_status & (1 << i))
 				usba_dma_irq(udc, &udc->usba_ep[i]);
@@ -1753,6 +1773,8 @@ static irqreturn_t usba_udc_irq(int irq, void *devid)
 	if (ep_status) {
 		int i;
 
+		usba_int_enb_set(udc, USBA_DET_SUSPEND);
+
 		for (i = 0; i < udc->num_ep; i++)
 			if (ep_status & (1 << i)) {
 				if (ep_is_control(&udc->usba_ep[i]))
@@ -1766,7 +1788,9 @@ static irqreturn_t usba_udc_irq(int irq, void *devid)
 		struct usba_ep *ep0, *ep;
 		int i, n;
 
-		usba_writel(udc, INT_CLR, USBA_END_OF_RESET);
+		usba_writel(udc, INT_CLR,
+			USBA_END_OF_RESET|USBA_END_OF_RESUME
+			|USBA_DET_SUSPEND|USBA_WAKE_UP);
 		generate_bias_pulse(udc);
 		reset_all_endpoints(udc);
 
@@ -1793,7 +1817,12 @@ static irqreturn_t usba_udc_irq(int irq, void *devid)
 				| USBA_BF(BK_NUMBER, USBA_BK_NUMBER_ONE)));
 		usba_ep_writel(ep0, CTL_ENB,
 				USBA_EPT_ENABLE | USBA_RX_SETUP);
-		usba_int_enb_set(udc, int_enb | USBA_BF(EPT_INT, 1) |
+
+		/* If we get reset while suspended... */
+		udc->suspended = false;
+		usba_int_enb_clear(udc, USBA_WAKE_UP);
+
+		usba_int_enb_set(udc, USBA_BF(EPT_INT, 1) |
 				      USBA_DET_SUSPEND | USBA_END_OF_RESUME);
 
 		/*
@@ -1827,6 +1856,8 @@ static int start_clock(struct usba_udc *udc)
 	if (udc->clocked)
 		return 0;
 
+	pm_stay_awake(&udc->pdev->dev);
+
 	ret = clk_prepare_enable(udc->pclk);
 	if (ret)
 		return ret;
@@ -1849,6 +1880,8 @@ static void stop_clock(struct usba_udc *udc)
 	clk_disable_unprepare(udc->pclk);
 
 	udc->clocked = false;
+
+	pm_relax(&udc->pdev->dev);
 }
 
 static int usba_start(struct usba_udc *udc)
@@ -1860,9 +1893,19 @@ static int usba_start(struct usba_udc *udc)
 	if (ret)
 		return ret;
 
+	if (udc->suspended)
+		return 0;
+
 	spin_lock_irqsave(&udc->lock, flags);
 	toggle_bias(udc, 1);
 	usba_writel(udc, CTRL, USBA_ENABLE_MASK);
+	/* Clear all requested and pending interrupts... */
+	usba_writel(udc, INT_ENB, 0);
+	udc->int_enb_cache = 0;
+	usba_writel(udc, INT_CLR,
+		USBA_END_OF_RESET|USBA_END_OF_RESUME
+		|USBA_DET_SUSPEND|USBA_WAKE_UP);
+	/* ...and enable just 'reset' IRQ to get us started */
 	usba_int_enb_set(udc, USBA_END_OF_RESET);
 	spin_unlock_irqrestore(&udc->lock, flags);
 
@@ -1873,6 +1916,9 @@ static void usba_stop(struct usba_udc *udc)
 {
 	unsigned long flags;
 
+	if (udc->suspended)
+		return;
+
 	spin_lock_irqsave(&udc->lock, flags);
 	udc->gadget.speed = USB_SPEED_UNKNOWN;
 	reset_all_endpoints(udc);
@@ -1900,6 +1946,7 @@ static irqreturn_t usba_vbus_irq_thread(int irq, void *devid)
 		if (vbus) {
 			usba_start(udc);
 		} else {
+			udc->suspended = false;
 			usba_stop(udc);
 
 			if (udc->driver->disconnect)
@@ -1963,6 +2010,7 @@ static int atmel_usba_stop(struct usb_gadget *gadget)
 	if (fifo_mode == 0)
 		udc->configured_ep = 1;
 
+	udc->suspended = false;
 	usba_stop(udc);
 
 	udc->driver = NULL;
@@ -2276,6 +2324,7 @@ static int usba_udc_suspend(struct device *dev)
 	mutex_lock(&udc->vbus_mutex);
 
 	if (!device_may_wakeup(dev)) {
+		udc->suspended = false;
 		usba_stop(udc);
 		goto out;
 	}
@@ -2285,10 +2334,13 @@ static int usba_udc_suspend(struct device *dev)
 	 * to request vbus irq, assuming always on.
 	 */
 	if (udc->vbus_pin) {
+		/* FIXME: right to stop here...??? */
 		usba_stop(udc);
 		enable_irq_wake(gpiod_to_irq(udc->vbus_pin));
 	}
 
+	enable_irq_wake(udc->irq);
+
 out:
 	mutex_unlock(&udc->vbus_mutex);
 	return 0;
@@ -2302,8 +2354,12 @@ static int usba_udc_resume(struct device *dev)
 	if (!udc->driver)
 		return 0;
 
-	if (device_may_wakeup(dev) && udc->vbus_pin)
-		disable_irq_wake(gpiod_to_irq(udc->vbus_pin));
+	if (device_may_wakeup(dev)) {
+		if (udc->vbus_pin)
+			disable_irq_wake(gpiod_to_irq(udc->vbus_pin));
+
+		disable_irq_wake(udc->irq);
+	}
 
 	/* If Vbus is present, enable the controller and wait for reset */
 	mutex_lock(&udc->vbus_mutex);
diff --git a/drivers/usb/gadget/udc/atmel_usba_udc.h b/drivers/usb/gadget/udc/atmel_usba_udc.h
index 030bf797cd25..a0225e4543d4 100644
--- a/drivers/usb/gadget/udc/atmel_usba_udc.h
+++ b/drivers/usb/gadget/udc/atmel_usba_udc.h
@@ -331,6 +331,7 @@ struct usba_udc {
 	struct usba_ep *usba_ep;
 	bool bias_pulse_needed;
 	bool clocked;
+	bool suspended;
 
 	u16 devstatus;
 
diff --git a/drivers/usb/gadget/udc/dummy_hcd.c b/drivers/usb/gadget/udc/dummy_hcd.c
index 213b52508621..8414fac74493 100644
--- a/drivers/usb/gadget/udc/dummy_hcd.c
+++ b/drivers/usb/gadget/udc/dummy_hcd.c
@@ -617,21 +617,7 @@ static int dummy_enable(struct usb_ep *_ep,
 		_ep->name,
 		desc->bEndpointAddress & 0x0f,
 		(desc->bEndpointAddress & USB_DIR_IN) ? "in" : "out",
-		({ char *val;
-		 switch (usb_endpoint_type(desc)) {
-		 case USB_ENDPOINT_XFER_BULK:
-			 val = "bulk";
-			 break;
-		 case USB_ENDPOINT_XFER_ISOC:
-			 val = "iso";
-			 break;
-		 case USB_ENDPOINT_XFER_INT:
-			 val = "intr";
-			 break;
-		 default:
-			 val = "ctrl";
-			 break;
-		 } val; }),
+		usb_ep_type_string(usb_endpoint_type(desc)),
 		max, ep->stream_en ? "enabled" : "disabled");
 
 	/* at this point real hardware should be NAKing transfers
diff --git a/drivers/usb/gadget/udc/lpc32xx_udc.c b/drivers/usb/gadget/udc/lpc32xx_udc.c
index b0781771704e..d8f1c60793ed 100644
--- a/drivers/usb/gadget/udc/lpc32xx_udc.c
+++ b/drivers/usb/gadget/udc/lpc32xx_udc.c
@@ -115,6 +115,11 @@ struct lpc32xx_ep {
 	bool			wedge;
 };
 
+enum atx_type {
+	ISP1301,
+	STOTG04,
+};
+
 /*
  * Common UDC structure
  */
@@ -129,8 +134,6 @@ struct lpc32xx_udc {
 
 	/* Board and device specific */
 	struct lpc32xx_usbd_cfg	*board;
-	u32			io_p_start;
-	u32			io_p_size;
 	void __iomem		*udp_baseaddr;
 	int			udp_irq[4];
 	struct clk		*usb_slv_clk;
@@ -151,10 +154,10 @@ struct lpc32xx_udc {
 	u8			last_vbus;
 	int			pullup;
 	int			poweron;
+	enum atx_type		atx;
 
 	/* Work queues related to I2C support */
 	struct work_struct	pullup_job;
-	struct work_struct	vbus_job;
 	struct work_struct	power_job;
 
 	/* USB device peripheral - various */
@@ -553,6 +556,15 @@ static inline void remove_debug_file(struct lpc32xx_udc *udc) {}
 /* Primary initialization sequence for the ISP1301 transceiver */
 static void isp1301_udc_configure(struct lpc32xx_udc *udc)
 {
+	u8 value;
+	s32 vendor, product;
+
+	vendor = i2c_smbus_read_word_data(udc->isp1301_i2c_client, 0x00);
+	product = i2c_smbus_read_word_data(udc->isp1301_i2c_client, 0x02);
+
+	if (vendor == 0x0483 && product == 0xa0c4)
+		udc->atx = STOTG04;
+
 	/* LPC32XX only supports DAT_SE0 USB mode */
 	/* This sequence is important */
 
@@ -572,8 +584,12 @@ static void isp1301_udc_configure(struct lpc32xx_udc *udc)
 	 */
 	i2c_smbus_write_byte_data(udc->isp1301_i2c_client,
 		(ISP1301_I2C_MODE_CONTROL_2 | ISP1301_I2C_REG_CLEAR_ADDR), ~0);
+
+	value = MC2_BI_DI;
+	if (udc->atx != STOTG04)
+		value |= MC2_SPD_SUSP_CTRL;
 	i2c_smbus_write_byte_data(udc->isp1301_i2c_client,
-		ISP1301_I2C_MODE_CONTROL_2, (MC2_BI_DI | MC2_SPD_SUSP_CTRL));
+		ISP1301_I2C_MODE_CONTROL_2, value);
 
 	/* Driver VBUS_DRV high or low depending on board setup */
 	if (udc->board->vbus_drv_pol != 0)
@@ -601,24 +617,19 @@ static void isp1301_udc_configure(struct lpc32xx_udc *udc)
 		(ISP1301_I2C_OTG_CONTROL_1 | ISP1301_I2C_REG_CLEAR_ADDR),
 		OTG1_VBUS_DISCHRG);
 
-	/* Clear and enable VBUS high edge interrupt */
 	i2c_smbus_write_byte_data(udc->isp1301_i2c_client,
 		ISP1301_I2C_INTERRUPT_LATCH | ISP1301_I2C_REG_CLEAR_ADDR, ~0);
+
 	i2c_smbus_write_byte_data(udc->isp1301_i2c_client,
 		ISP1301_I2C_INTERRUPT_FALLING | ISP1301_I2C_REG_CLEAR_ADDR, ~0);
 	i2c_smbus_write_byte_data(udc->isp1301_i2c_client,
-		ISP1301_I2C_INTERRUPT_FALLING, INT_VBUS_VLD);
-	i2c_smbus_write_byte_data(udc->isp1301_i2c_client,
 		ISP1301_I2C_INTERRUPT_RISING | ISP1301_I2C_REG_CLEAR_ADDR, ~0);
-	i2c_smbus_write_byte_data(udc->isp1301_i2c_client,
-		ISP1301_I2C_INTERRUPT_RISING, INT_VBUS_VLD);
 
-	dev_info(udc->dev, "ISP1301 Vendor ID  : 0x%04x\n",
-		 i2c_smbus_read_word_data(udc->isp1301_i2c_client, 0x00));
-	dev_info(udc->dev, "ISP1301 Product ID : 0x%04x\n",
-		 i2c_smbus_read_word_data(udc->isp1301_i2c_client, 0x02));
+	dev_info(udc->dev, "ISP1301 Vendor ID  : 0x%04x\n", vendor);
+	dev_info(udc->dev, "ISP1301 Product ID : 0x%04x\n", product);
 	dev_info(udc->dev, "ISP1301 Version ID : 0x%04x\n",
 		 i2c_smbus_read_word_data(udc->isp1301_i2c_client, 0x14));
+
 }
 
 /* Enables or disables the USB device pullup via the ISP1301 transceiver */
@@ -661,6 +672,10 @@ static void isp1301_pullup_enable(struct lpc32xx_udc *udc, int en_pullup,
 /* Powers up or down the ISP1301 transceiver */
 static void isp1301_set_powerstate(struct lpc32xx_udc *udc, int enable)
 {
+	/* There is no "global power down" register for stotg04 */
+	if (udc->atx == STOTG04)
+		return;
+
 	if (enable != 0)
 		/* Power up ISP1301 - this ISP1301 will automatically wakeup
 		   when VBUS is detected */
@@ -2830,11 +2845,9 @@ static irqreturn_t lpc32xx_usb_devdma_irq(int irq, void *_udc)
  * VBUS detection, pullup handler, and Gadget cable state notification
  *
  */
-static void vbus_work(struct work_struct *work)
+static void vbus_work(struct lpc32xx_udc *udc)
 {
 	u8 value;
-	struct lpc32xx_udc *udc = container_of(work, struct lpc32xx_udc,
-					       vbus_job);
 
 	if (udc->enabled != 0) {
 		/* Discharge VBUS real quick */
@@ -2870,18 +2883,13 @@ static void vbus_work(struct work_struct *work)
 			lpc32xx_vbus_session(&udc->gadget, udc->vbus);
 		}
 	}
-
-	/* Re-enable after completion */
-	enable_irq(udc->udp_irq[IRQ_USB_ATX]);
 }
 
 static irqreturn_t lpc32xx_usb_vbus_irq(int irq, void *_udc)
 {
 	struct lpc32xx_udc *udc = _udc;
 
-	/* Defer handling of VBUS IRQ to work queue */
-	disable_irq_nosync(udc->udp_irq[IRQ_USB_ATX]);
-	schedule_work(&udc->vbus_job);
+	vbus_work(udc);
 
 	return IRQ_HANDLED;
 }
@@ -2890,7 +2898,6 @@ static int lpc32xx_start(struct usb_gadget *gadget,
 			 struct usb_gadget_driver *driver)
 {
 	struct lpc32xx_udc *udc = to_udc(gadget);
-	int i;
 
 	if (!driver || driver->max_speed < USB_SPEED_FULL || !driver->setup) {
 		dev_err(udc->dev, "bad parameter.\n");
@@ -2910,22 +2917,25 @@ static int lpc32xx_start(struct usb_gadget *gadget,
 
 	/* Force VBUS process once to check for cable insertion */
 	udc->last_vbus = udc->vbus = 0;
-	schedule_work(&udc->vbus_job);
+	vbus_work(udc);
 
-	/* Do not re-enable ATX IRQ (3) */
-	for (i = IRQ_USB_LP; i < IRQ_USB_ATX; i++)
-		enable_irq(udc->udp_irq[i]);
+	/* enable interrupts */
+	i2c_smbus_write_byte_data(udc->isp1301_i2c_client,
+		ISP1301_I2C_INTERRUPT_FALLING, INT_SESS_VLD | INT_VBUS_VLD);
+	i2c_smbus_write_byte_data(udc->isp1301_i2c_client,
+		ISP1301_I2C_INTERRUPT_RISING, INT_SESS_VLD | INT_VBUS_VLD);
 
 	return 0;
 }
 
 static int lpc32xx_stop(struct usb_gadget *gadget)
 {
-	int i;
 	struct lpc32xx_udc *udc = to_udc(gadget);
 
-	for (i = IRQ_USB_LP; i <= IRQ_USB_ATX; i++)
-		disable_irq(udc->udp_irq[i]);
+	i2c_smbus_write_byte_data(udc->isp1301_i2c_client,
+		ISP1301_I2C_INTERRUPT_FALLING | ISP1301_I2C_REG_CLEAR_ADDR, ~0);
+	i2c_smbus_write_byte_data(udc->isp1301_i2c_client,
+		ISP1301_I2C_INTERRUPT_RISING | ISP1301_I2C_REG_CLEAR_ADDR, ~0);
 
 	if (udc->clocked) {
 		spin_lock(&udc->lock);
@@ -2999,7 +3009,7 @@ static int lpc32xx_udc_probe(struct platform_device *pdev)
 	dma_addr_t dma_handle;
 	struct device_node *isp1301_node;
 
-	udc = kmemdup(&controller_template, sizeof(*udc), GFP_KERNEL);
+	udc = devm_kmemdup(dev, &controller_template, sizeof(*udc), GFP_KERNEL);
 	if (!udc)
 		return -ENOMEM;
 
@@ -3022,8 +3032,7 @@ static int lpc32xx_udc_probe(struct platform_device *pdev)
 
 	udc->isp1301_i2c_client = isp1301_get_client(isp1301_node);
 	if (!udc->isp1301_i2c_client) {
-		retval = -EPROBE_DEFER;
-		goto phy_fail;
+		return -EPROBE_DEFER;
 	}
 
 	dev_info(udc->dev, "ISP1301 I2C device at address 0x%x\n",
@@ -3032,7 +3041,7 @@ static int lpc32xx_udc_probe(struct platform_device *pdev)
 	pdev->dev.dma_mask = &lpc32xx_usbd_dmamask;
 	retval = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
 	if (retval)
-		goto resource_fail;
+		return retval;
 
 	udc->board = &lpc32xx_usbddata;
 
@@ -3045,10 +3054,8 @@ static int lpc32xx_udc_probe(struct platform_device *pdev)
 	 *  IORESOURCE_IRQ, USB transceiver interrupt number
 	 */
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!res) {
-		retval = -ENXIO;
-		goto resource_fail;
-	}
+	if (!res)
+		return -ENXIO;
 
 	spin_lock_init(&udc->lock);
 
@@ -3058,45 +3065,33 @@ static int lpc32xx_udc_probe(struct platform_device *pdev)
 		if (udc->udp_irq[i] < 0) {
 			dev_err(udc->dev,
 				"irq resource %d not available!\n", i);
-			retval = udc->udp_irq[i];
-			goto irq_fail;
+			return udc->udp_irq[i];
 		}
 	}
 
-	udc->io_p_start = res->start;
-	udc->io_p_size = resource_size(res);
-	if (!request_mem_region(udc->io_p_start, udc->io_p_size, driver_name)) {
-		dev_err(udc->dev, "someone's using UDC memory\n");
-		retval = -EBUSY;
-		goto request_mem_region_fail;
-	}
-
-	udc->udp_baseaddr = ioremap(udc->io_p_start, udc->io_p_size);
+	udc->udp_baseaddr = devm_ioremap_resource(dev, res);
 	if (!udc->udp_baseaddr) {
-		retval = -ENOMEM;
 		dev_err(udc->dev, "IO map failure\n");
-		goto io_map_fail;
+		return -ENOMEM;
 	}
 
 	/* Get USB device clock */
-	udc->usb_slv_clk = clk_get(&pdev->dev, NULL);
+	udc->usb_slv_clk = devm_clk_get(&pdev->dev, NULL);
 	if (IS_ERR(udc->usb_slv_clk)) {
 		dev_err(udc->dev, "failed to acquire USB device clock\n");
-		retval = PTR_ERR(udc->usb_slv_clk);
-		goto usb_clk_get_fail;
+		return PTR_ERR(udc->usb_slv_clk);
 	}
 
 	/* Enable USB device clock */
 	retval = clk_prepare_enable(udc->usb_slv_clk);
 	if (retval < 0) {
 		dev_err(udc->dev, "failed to start USB device clock\n");
-		goto usb_clk_enable_fail;
+		return retval;
 	}
 
 	/* Setup deferred workqueue data */
 	udc->poweron = udc->pullup = 0;
 	INIT_WORK(&udc->pullup_job, pullup_work);
-	INIT_WORK(&udc->vbus_job, vbus_work);
 #ifdef CONFIG_PM
 	INIT_WORK(&udc->power_job, power_work);
 #endif
@@ -3134,47 +3129,44 @@ static int lpc32xx_udc_probe(struct platform_device *pdev)
 
 	/* Request IRQs - low and high priority USB device IRQs are routed to
 	 * the same handler, while the DMA interrupt is routed elsewhere */
-	retval = request_irq(udc->udp_irq[IRQ_USB_LP], lpc32xx_usb_lp_irq,
-			     0, "udc_lp", udc);
+	retval = devm_request_irq(dev, udc->udp_irq[IRQ_USB_LP],
+				  lpc32xx_usb_lp_irq, 0, "udc_lp", udc);
 	if (retval < 0) {
 		dev_err(udc->dev, "LP request irq %d failed\n",
 			udc->udp_irq[IRQ_USB_LP]);
-		goto irq_lp_fail;
+		goto irq_req_fail;
 	}
-	retval = request_irq(udc->udp_irq[IRQ_USB_HP], lpc32xx_usb_hp_irq,
-			     0, "udc_hp", udc);
+	retval = devm_request_irq(dev, udc->udp_irq[IRQ_USB_HP],
+				  lpc32xx_usb_hp_irq, 0, "udc_hp", udc);
 	if (retval < 0) {
 		dev_err(udc->dev, "HP request irq %d failed\n",
 			udc->udp_irq[IRQ_USB_HP]);
-		goto irq_hp_fail;
+		goto irq_req_fail;
 	}
 
-	retval = request_irq(udc->udp_irq[IRQ_USB_DEVDMA],
-			     lpc32xx_usb_devdma_irq, 0, "udc_dma", udc);
+	retval = devm_request_irq(dev, udc->udp_irq[IRQ_USB_DEVDMA],
+				  lpc32xx_usb_devdma_irq, 0, "udc_dma", udc);
 	if (retval < 0) {
 		dev_err(udc->dev, "DEV request irq %d failed\n",
 			udc->udp_irq[IRQ_USB_DEVDMA]);
-		goto irq_dev_fail;
+		goto irq_req_fail;
 	}
 
 	/* The transceiver interrupt is used for VBUS detection and will
 	   kick off the VBUS handler function */
-	retval = request_irq(udc->udp_irq[IRQ_USB_ATX], lpc32xx_usb_vbus_irq,
-			     0, "udc_otg", udc);
+	retval = devm_request_threaded_irq(dev, udc->udp_irq[IRQ_USB_ATX], NULL,
+					   lpc32xx_usb_vbus_irq, IRQF_ONESHOT,
+					   "udc_otg", udc);
 	if (retval < 0) {
 		dev_err(udc->dev, "VBUS request irq %d failed\n",
 			udc->udp_irq[IRQ_USB_ATX]);
-		goto irq_xcvr_fail;
+		goto irq_req_fail;
 	}
 
 	/* Initialize wait queue */
 	init_waitqueue_head(&udc->ep_disable_wait_queue);
 	atomic_set(&udc->enabled_ep_cnt, 0);
 
-	/* Keep all IRQs disabled until GadgetFS starts up */
-	for (i = IRQ_USB_LP; i <= IRQ_USB_ATX; i++)
-		disable_irq(udc->udp_irq[i]);
-
 	retval = usb_add_gadget_udc(dev, &udc->gadget);
 	if (retval < 0)
 		goto add_gadget_fail;
@@ -3190,32 +3182,15 @@ static int lpc32xx_udc_probe(struct platform_device *pdev)
 	return 0;
 
 add_gadget_fail:
-	free_irq(udc->udp_irq[IRQ_USB_ATX], udc);
-irq_xcvr_fail:
-	free_irq(udc->udp_irq[IRQ_USB_DEVDMA], udc);
-irq_dev_fail:
-	free_irq(udc->udp_irq[IRQ_USB_HP], udc);
-irq_hp_fail:
-	free_irq(udc->udp_irq[IRQ_USB_LP], udc);
-irq_lp_fail:
+irq_req_fail:
 	dma_pool_destroy(udc->dd_cache);
 dma_alloc_fail:
 	dma_free_coherent(&pdev->dev, UDCA_BUFF_SIZE,
 			  udc->udca_v_base, udc->udca_p_base);
 i2c_fail:
 	clk_disable_unprepare(udc->usb_slv_clk);
-usb_clk_enable_fail:
-	clk_put(udc->usb_slv_clk);
-usb_clk_get_fail:
-	iounmap(udc->udp_baseaddr);
-io_map_fail:
-	release_mem_region(udc->io_p_start, udc->io_p_size);
 	dev_err(udc->dev, "%s probe failed, %d\n", driver_name, retval);
-request_mem_region_fail:
-irq_fail:
-resource_fail:
-phy_fail:
-	kfree(udc);
+
 	return retval;
 }
 
@@ -3231,24 +3206,14 @@ static int lpc32xx_udc_remove(struct platform_device *pdev)
 	udc_disable(udc);
 	pullup(udc, 0);
 
-	free_irq(udc->udp_irq[IRQ_USB_ATX], udc);
-
 	device_init_wakeup(&pdev->dev, 0);
 	remove_debug_file(udc);
 
 	dma_pool_destroy(udc->dd_cache);
 	dma_free_coherent(&pdev->dev, UDCA_BUFF_SIZE,
 			  udc->udca_v_base, udc->udca_p_base);
-	free_irq(udc->udp_irq[IRQ_USB_DEVDMA], udc);
-	free_irq(udc->udp_irq[IRQ_USB_HP], udc);
-	free_irq(udc->udp_irq[IRQ_USB_LP], udc);
 
 	clk_disable_unprepare(udc->usb_slv_clk);
-	clk_put(udc->usb_slv_clk);
-
-	iounmap(udc->udp_baseaddr);
-	release_mem_region(udc->io_p_start, udc->io_p_size);
-	kfree(udc);
 
 	return 0;
 }
diff --git a/drivers/usb/gadget/udc/net2272.c b/drivers/usb/gadget/udc/net2272.c
index c2011cd7df8c..564aeee1a1fe 100644
--- a/drivers/usb/gadget/udc/net2272.c
+++ b/drivers/usb/gadget/udc/net2272.c
@@ -573,8 +573,7 @@ net2272_read_fifo(struct net2272_ep *ep, struct net2272_request *req)
 
 		/* completion */
 		if (unlikely(cleanup || is_short ||
-				((req->req.actual == req->req.length)
-				 && !req->req.zero))) {
+				req->req.actual == req->req.length)) {
 
 			if (cleanup) {
 				net2272_out_flush(ep);
diff --git a/drivers/usb/gadget/udc/net2280.c b/drivers/usb/gadget/udc/net2280.c
index 898339e5df10..b6bbe2e448ba 100644
--- a/drivers/usb/gadget/udc/net2280.c
+++ b/drivers/usb/gadget/udc/net2280.c
@@ -789,8 +789,7 @@ static int read_fifo(struct net2280_ep *ep, struct net2280_request *req)
 		(void) readl(&ep->regs->ep_rsp);
 	}
 
-	return is_short || ((req->req.actual == req->req.length) &&
-			!req->req.zero);
+	return is_short || req->req.actual == req->req.length;
 }
 
 /* fill out dma descriptor to match a given request */
@@ -1058,7 +1057,7 @@ net2280_queue(struct usb_ep *_ep, struct usb_request *_req, gfp_t gfp_flags)
 			/* PIO ... stuff the fifo, or unblock it.  */
 			if (ep->is_in)
 				write_fifo(ep, _req);
-			else if (list_empty(&ep->queue)) {
+			else {
 				u32	s;
 
 				/* OUT FIFO might have packet(s) buffered */
diff --git a/drivers/usb/host/fhci-sched.c b/drivers/usb/host/fhci-sched.c
index 3d12cdd5f999..3235d5307403 100644
--- a/drivers/usb/host/fhci-sched.c
+++ b/drivers/usb/host/fhci-sched.c
@@ -727,8 +727,7 @@ void fhci_queue_urb(struct fhci_hcd *fhci, struct urb *urb)
 		}
 		ed->speed = (urb->dev->speed == USB_SPEED_LOW) ?
 			FHCI_LOW_SPEED : FHCI_FULL_SPEED;
-		ed->max_pkt_size = usb_maxpacket(urb->dev,
-			urb->pipe, usb_pipeout(urb->pipe));
+		ed->max_pkt_size = usb_endpoint_maxp(&urb->ep->desc);
 		urb->ep->hcpriv = ed;
 		fhci_dbg(fhci, "new ep speed=%d max_pkt_size=%d\n",
 			 ed->speed, ed->max_pkt_size);
@@ -768,8 +767,7 @@ void fhci_queue_urb(struct fhci_hcd *fhci, struct urb *urb)
 		if (urb->transfer_flags & URB_ZERO_PACKET &&
 				urb->transfer_buffer_length > 0 &&
 				((urb->transfer_buffer_length %
-				usb_maxpacket(urb->dev, urb->pipe,
-				usb_pipeout(urb->pipe))) == 0))
+				usb_endpoint_maxp(&urb->ep->desc)) == 0))
 			urb_state = US_BULK0;
 		while (data_len > 4096) {
 			td = fhci_td_fill(fhci, urb, urb_priv, ed, cnt,
@@ -807,8 +805,8 @@ void fhci_queue_urb(struct fhci_hcd *fhci, struct urb *urb)
 		break;
 	case FHCI_TF_CTRL:
 		ed->dev_addr = usb_pipedevice(urb->pipe);
-		ed->max_pkt_size = usb_maxpacket(urb->dev, urb->pipe,
-			usb_pipeout(urb->pipe));
+		ed->max_pkt_size = usb_endpoint_maxp(&urb->ep->desc);
+
 		/* setup stage */
 		td = fhci_td_fill(fhci, urb, urb_priv, ed, cnt++, FHCI_TA_SETUP,
 			USB_TD_TOGGLE_DATA0, urb->setup_packet, 8, 0, 0, true);
diff --git a/drivers/usb/host/u132-hcd.c b/drivers/usb/host/u132-hcd.c
index 6343fbacd244..4a5c9b599c57 100644
--- a/drivers/usb/host/u132-hcd.c
+++ b/drivers/usb/host/u132-hcd.c
@@ -3203,6 +3203,8 @@ static int __init u132_hcd_init(void)
 		return -ENODEV;
 	printk(KERN_INFO "driver %s\n", hcd_name);
 	workqueue = create_singlethread_workqueue("u132");
+	if (!workqueue)
+		return -ENOMEM;
 	retval = platform_driver_register(&u132_platform_driver);
 	if (retval)
 		destroy_workqueue(workqueue);
diff --git a/drivers/usb/host/xhci-hub.c b/drivers/usb/host/xhci-hub.c
index 96a740543183..3abe70ff1b1e 100644
--- a/drivers/usb/host/xhci-hub.c
+++ b/drivers/usb/host/xhci-hub.c
@@ -487,8 +487,8 @@ static void xhci_disable_port(struct usb_hcd *hcd, struct xhci_hcd *xhci,
 	/* Write 1 to disable the port */
 	writel(port_status | PORT_PE, addr);
 	port_status = readl(addr);
-	xhci_dbg(xhci, "disable port, actual port %d status  = 0x%x\n",
-			wIndex, port_status);
+	xhci_dbg(xhci, "disable port %d-%d, portsc: 0x%x\n",
+		 hcd->self.busnum, wIndex + 1, port_status);
 }
 
 static void xhci_clear_port_change_bit(struct xhci_hcd *xhci, u16 wValue,
@@ -537,8 +537,9 @@ static void xhci_clear_port_change_bit(struct xhci_hcd *xhci, u16 wValue,
 	/* Change bits are all write 1 to clear */
 	writel(port_status | status, addr);
 	port_status = readl(addr);
-	xhci_dbg(xhci, "clear port %s change, actual port %d status  = 0x%x\n",
-			port_change_bit, wIndex, port_status);
+
+	xhci_dbg(xhci, "clear port%d %s change, portsc: 0x%x\n",
+		 wIndex + 1, port_change_bit, port_status);
 }
 
 struct xhci_hub *xhci_get_rhub(struct usb_hcd *hcd)
@@ -565,13 +566,16 @@ static void xhci_set_port_power(struct xhci_hcd *xhci, struct usb_hcd *hcd,
 	rhub = xhci_get_rhub(hcd);
 	port = rhub->ports[index];
 	temp = readl(port->addr);
+
+	xhci_dbg(xhci, "set port power %d-%d %s, portsc: 0x%x\n",
+		 hcd->self.busnum, index + 1, on ? "ON" : "OFF", temp);
+
 	temp = xhci_port_state_to_neutral(temp);
+
 	if (on) {
 		/* Power on */
 		writel(temp | PORT_POWER, port->addr);
-		temp = readl(port->addr);
-		xhci_dbg(xhci, "set port power, actual port %d status  = 0x%x\n",
-						index, temp);
+		readl(port->addr);
 	} else {
 		/* Power off */
 		writel(temp & ~PORT_POWER, port->addr);
@@ -666,12 +670,17 @@ void xhci_set_link_state(struct xhci_hcd *xhci, struct xhci_port *port,
 			 u32 link_state)
 {
 	u32 temp;
+	u32 portsc;
 
-	temp = readl(port->addr);
-	temp = xhci_port_state_to_neutral(temp);
+	portsc = readl(port->addr);
+	temp = xhci_port_state_to_neutral(portsc);
 	temp &= ~PORT_PLS_MASK;
 	temp |= PORT_LINK_STROBE | link_state;
 	writel(temp, port->addr);
+
+	xhci_dbg(xhci, "Set port %d-%d link state, portsc: 0x%x, write 0x%x",
+		 port->rhub->hcd->self.busnum, port->hcd_portnum + 1,
+		 portsc, temp);
 }
 
 static void xhci_set_remote_wake_mask(struct xhci_hcd *xhci,
@@ -840,7 +849,9 @@ static int xhci_handle_usb2_port_link_resume(struct xhci_port *port,
 	} else if (time_after_eq(jiffies, bus_state->resume_done[wIndex])) {
 		int time_left;
 
-		xhci_dbg(xhci, "Resume USB2 port %d\n", wIndex + 1);
+		xhci_dbg(xhci, "resume USB2 port %d-%d\n",
+			 hcd->self.busnum, wIndex + 1);
+
 		bus_state->resume_done[wIndex] = 0;
 		clear_bit(wIndex, &bus_state->resuming_ports);
 
@@ -867,9 +878,8 @@ static int xhci_handle_usb2_port_link_resume(struct xhci_port *port,
 		} else {
 			int port_status = readl(port->addr);
 
-			xhci_warn(xhci, "Port resume %i msec timed out, portsc = 0x%x\n",
-				  XHCI_MAX_REXIT_TIMEOUT_MS,
-				  port_status);
+			xhci_warn(xhci, "Port resume timed out, port %d-%d: 0x%x\n",
+				  hcd->self.busnum, wIndex + 1, port_status);
 			*status |= USB_PORT_STAT_SUSPEND;
 			clear_bit(wIndex, &bus_state->rexit_ports);
 		}
@@ -1124,9 +1134,8 @@ int xhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue,
 		if (status == 0xffffffff)
 			goto error;
 
-		xhci_dbg(xhci, "get port status, actual port %d status  = 0x%x\n",
-				wIndex, temp);
-		xhci_dbg(xhci, "Get port status returned 0x%x\n", status);
+		xhci_dbg(xhci, "Get port status %d-%d read: 0x%x, return 0x%x",
+			 hcd->self.busnum, wIndex + 1, temp, status);
 
 		put_unaligned(cpu_to_le32(status), (__le32 *) buf);
 		/* if USB 3.1 extended port status return additional 4 bytes */
@@ -1182,7 +1191,8 @@ int xhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue,
 			temp = readl(ports[wIndex]->addr);
 			if ((temp & PORT_PE) == 0 || (temp & PORT_RESET)
 				|| (temp & PORT_PLS_MASK) >= XDEV_U3) {
-				xhci_warn(xhci, "USB core suspending device not in U0/U1/U2.\n");
+				xhci_warn(xhci, "USB core suspending port %d-%d not in U0/U1/U2\n",
+					  hcd->self.busnum, wIndex + 1);
 				goto error;
 			}
 
diff --git a/drivers/usb/host/xhci-mtk.c b/drivers/usb/host/xhci-mtk.c
index 60987c787e44..026fe18972d3 100644
--- a/drivers/usb/host/xhci-mtk.c
+++ b/drivers/usb/host/xhci-mtk.c
@@ -206,19 +206,6 @@ static int xhci_mtk_ssusb_config(struct xhci_hcd_mtk *mtk)
 	return xhci_mtk_host_enable(mtk);
 }
 
-/* ignore the error if the clock does not exist */
-static struct clk *optional_clk_get(struct device *dev, const char *id)
-{
-	struct clk *opt_clk;
-
-	opt_clk = devm_clk_get(dev, id);
-	/* ignore error number except EPROBE_DEFER */
-	if (IS_ERR(opt_clk) && (PTR_ERR(opt_clk) != -EPROBE_DEFER))
-		opt_clk = NULL;
-
-	return opt_clk;
-}
-
 static int xhci_mtk_clks_get(struct xhci_hcd_mtk *mtk)
 {
 	struct device *dev = mtk->dev;
@@ -229,15 +216,15 @@ static int xhci_mtk_clks_get(struct xhci_hcd_mtk *mtk)
 		return PTR_ERR(mtk->sys_clk);
 	}
 
-	mtk->ref_clk = optional_clk_get(dev, "ref_ck");
+	mtk->ref_clk = devm_clk_get_optional(dev, "ref_ck");
 	if (IS_ERR(mtk->ref_clk))
 		return PTR_ERR(mtk->ref_clk);
 
-	mtk->mcu_clk = optional_clk_get(dev, "mcu_ck");
+	mtk->mcu_clk = devm_clk_get_optional(dev, "mcu_ck");
 	if (IS_ERR(mtk->mcu_clk))
 		return PTR_ERR(mtk->mcu_clk);
 
-	mtk->dma_clk = optional_clk_get(dev, "dma_ck");
+	mtk->dma_clk = devm_clk_get_optional(dev, "dma_ck");
 	return PTR_ERR_OR_ZERO(mtk->dma_clk);
 }
 
diff --git a/drivers/usb/host/xhci-plat.c b/drivers/usb/host/xhci-plat.c
index 0ac4ec975547..998241f5fce3 100644
--- a/drivers/usb/host/xhci-plat.c
+++ b/drivers/usb/host/xhci-plat.c
@@ -165,8 +165,6 @@ static int xhci_plat_probe(struct platform_device *pdev)
 	struct xhci_hcd		*xhci;
 	struct resource         *res;
 	struct usb_hcd		*hcd;
-	struct clk              *clk;
-	struct clk              *reg_clk;
 	int			ret;
 	int			irq;
 
@@ -235,31 +233,32 @@ static int xhci_plat_probe(struct platform_device *pdev)
 	hcd->rsrc_start = res->start;
 	hcd->rsrc_len = resource_size(res);
 
+	xhci = hcd_to_xhci(hcd);
+
 	/*
 	 * Not all platforms have clks so it is not an error if the
 	 * clock do not exist.
 	 */
-	reg_clk = devm_clk_get(&pdev->dev, "reg");
-	if (!IS_ERR(reg_clk)) {
-		ret = clk_prepare_enable(reg_clk);
-		if (ret)
-			goto put_hcd;
-	} else if (PTR_ERR(reg_clk) == -EPROBE_DEFER) {
-		ret = -EPROBE_DEFER;
+	xhci->reg_clk = devm_clk_get_optional(&pdev->dev, "reg");
+	if (IS_ERR(xhci->reg_clk)) {
+		ret = PTR_ERR(xhci->reg_clk);
 		goto put_hcd;
 	}
 
-	clk = devm_clk_get(&pdev->dev, NULL);
-	if (!IS_ERR(clk)) {
-		ret = clk_prepare_enable(clk);
-		if (ret)
-			goto disable_reg_clk;
-	} else if (PTR_ERR(clk) == -EPROBE_DEFER) {
-		ret = -EPROBE_DEFER;
+	ret = clk_prepare_enable(xhci->reg_clk);
+	if (ret)
+		goto put_hcd;
+
+	xhci->clk = devm_clk_get_optional(&pdev->dev, NULL);
+	if (IS_ERR(xhci->clk)) {
+		ret = PTR_ERR(xhci->clk);
 		goto disable_reg_clk;
 	}
 
-	xhci = hcd_to_xhci(hcd);
+	ret = clk_prepare_enable(xhci->clk);
+	if (ret)
+		goto disable_reg_clk;
+
 	priv_match = of_device_get_match_data(&pdev->dev);
 	if (priv_match) {
 		struct xhci_plat_priv *priv = hcd_to_xhci_priv(hcd);
@@ -271,8 +270,6 @@ static int xhci_plat_probe(struct platform_device *pdev)
 
 	device_wakeup_enable(hcd->self.controller);
 
-	xhci->clk = clk;
-	xhci->reg_clk = reg_clk;
 	xhci->main_hcd = hcd;
 	xhci->shared_hcd = __usb_create_hcd(driver, sysdev, &pdev->dev,
 			dev_name(&pdev->dev), hcd);
@@ -348,10 +345,10 @@ put_usb3_hcd:
 	usb_put_hcd(xhci->shared_hcd);
 
 disable_clk:
-	clk_disable_unprepare(clk);
+	clk_disable_unprepare(xhci->clk);
 
 disable_reg_clk:
-	clk_disable_unprepare(reg_clk);
+	clk_disable_unprepare(xhci->reg_clk);
 
 put_hcd:
 	usb_put_hcd(hcd);
diff --git a/drivers/usb/host/xhci-ring.c b/drivers/usb/host/xhci-ring.c
index 9215a28dad40..fed3385aeac0 100644
--- a/drivers/usb/host/xhci-ring.c
+++ b/drivers/usb/host/xhci-ring.c
@@ -1569,18 +1569,19 @@ static void handle_port_status(struct xhci_hcd *xhci,
 			  "WARN: xHC returned failed port status event\n");
 
 	port_id = GET_PORT_ID(le32_to_cpu(event->generic.field[0]));
-	xhci_dbg(xhci, "Port Status Change Event for port %d\n", port_id);
-
 	max_ports = HCS_MAX_PORTS(xhci->hcs_params1);
+
 	if ((port_id <= 0) || (port_id > max_ports)) {
-		xhci_warn(xhci, "Invalid port id %d\n", port_id);
+		xhci_warn(xhci, "Port change event with invalid port ID %d\n",
+			  port_id);
 		inc_deq(xhci, xhci->event_ring);
 		return;
 	}
 
 	port = &xhci->hw_ports[port_id - 1];
 	if (!port || !port->rhub || port->hcd_portnum == DUPLICATE_ENTRY) {
-		xhci_warn(xhci, "Event for invalid port %u\n", port_id);
+		xhci_warn(xhci, "Port change event, no port for port ID %u\n",
+			  port_id);
 		bogus_port_status = true;
 		goto cleanup;
 	}
@@ -1597,6 +1598,9 @@ static void handle_port_status(struct xhci_hcd *xhci,
 	hcd_portnum = port->hcd_portnum;
 	portsc = readl(port->addr);
 
+	xhci_dbg(xhci, "Port change event, %d-%d, id %d, portsc: 0x%x\n",
+		 hcd->self.busnum, hcd_portnum + 1, port_id, portsc);
+
 	trace_xhci_handle_port_status(hcd_portnum, portsc);
 
 	if (hcd->state == HC_STATE_SUSPENDED) {
@@ -3275,6 +3279,12 @@ int xhci_queue_bulk_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
 			field |= TRB_IOC;
 			more_trbs_coming = false;
 			td->last_trb = ring->enqueue;
+
+			if (xhci_urb_suitable_for_idt(urb)) {
+				memcpy(&send_addr, urb->transfer_buffer,
+				       trb_buff_len);
+				field |= TRB_IDT;
+			}
 		}
 
 		/* Only set interrupt on short packet for IN endpoints */
@@ -3414,6 +3424,12 @@ int xhci_queue_ctrl_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
 	if (urb->transfer_buffer_length > 0) {
 		u32 length_field, remainder;
 
+		if (xhci_urb_suitable_for_idt(urb)) {
+			memcpy(&urb->transfer_dma, urb->transfer_buffer,
+			       urb->transfer_buffer_length);
+			field |= TRB_IDT;
+		}
+
 		remainder = xhci_td_remainder(xhci, 0,
 				urb->transfer_buffer_length,
 				urb->transfer_buffer_length,
diff --git a/drivers/usb/host/xhci-tegra.c b/drivers/usb/host/xhci-tegra.c
index efb0cad8710e..294158113d62 100644
--- a/drivers/usb/host/xhci-tegra.c
+++ b/drivers/usb/host/xhci-tegra.c
@@ -161,6 +161,7 @@ struct tegra_xusb_soc {
 	} ports;
 
 	bool scale_ss_clock;
+	bool has_ipfs;
 };
 
 struct tegra_xusb {
@@ -637,16 +638,18 @@ static irqreturn_t tegra_xusb_mbox_thread(int irq, void *data)
 	return IRQ_HANDLED;
 }
 
-static void tegra_xusb_ipfs_config(struct tegra_xusb *tegra,
-				   struct resource *regs)
+static void tegra_xusb_config(struct tegra_xusb *tegra,
+			      struct resource *regs)
 {
 	u32 value;
 
-	value = ipfs_readl(tegra, IPFS_XUSB_HOST_CONFIGURATION_0);
-	value |= IPFS_EN_FPCI;
-	ipfs_writel(tegra, value, IPFS_XUSB_HOST_CONFIGURATION_0);
+	if (tegra->soc->has_ipfs) {
+		value = ipfs_readl(tegra, IPFS_XUSB_HOST_CONFIGURATION_0);
+		value |= IPFS_EN_FPCI;
+		ipfs_writel(tegra, value, IPFS_XUSB_HOST_CONFIGURATION_0);
 
-	usleep_range(10, 20);
+		usleep_range(10, 20);
+	}
 
 	/* Program BAR0 space */
 	value = fpci_readl(tegra, XUSB_CFG_4);
@@ -661,13 +664,15 @@ static void tegra_xusb_ipfs_config(struct tegra_xusb *tegra,
 	value |= XUSB_IO_SPACE_EN | XUSB_MEM_SPACE_EN | XUSB_BUS_MASTER_EN;
 	fpci_writel(tegra, value, XUSB_CFG_1);
 
-	/* Enable interrupt assertion */
-	value = ipfs_readl(tegra, IPFS_XUSB_HOST_INTR_MASK_0);
-	value |= IPFS_IP_INT_MASK;
-	ipfs_writel(tegra, value, IPFS_XUSB_HOST_INTR_MASK_0);
+	if (tegra->soc->has_ipfs) {
+		/* Enable interrupt assertion */
+		value = ipfs_readl(tegra, IPFS_XUSB_HOST_INTR_MASK_0);
+		value |= IPFS_IP_INT_MASK;
+		ipfs_writel(tegra, value, IPFS_XUSB_HOST_INTR_MASK_0);
 
-	/* Set hysteresis */
-	ipfs_writel(tegra, 0x80, IPFS_XUSB_HOST_CLKGATE_HYSTERESIS_0);
+		/* Set hysteresis */
+		ipfs_writel(tegra, 0x80, IPFS_XUSB_HOST_CLKGATE_HYSTERESIS_0);
+	}
 }
 
 static int tegra_xusb_clk_enable(struct tegra_xusb *tegra)
@@ -1015,10 +1020,12 @@ static int tegra_xusb_probe(struct platform_device *pdev)
 	if (IS_ERR(tegra->fpci_base))
 		return PTR_ERR(tegra->fpci_base);
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 2);
-	tegra->ipfs_base = devm_ioremap_resource(&pdev->dev, res);
-	if (IS_ERR(tegra->ipfs_base))
-		return PTR_ERR(tegra->ipfs_base);
+	if (tegra->soc->has_ipfs) {
+		res = platform_get_resource(pdev, IORESOURCE_MEM, 2);
+		tegra->ipfs_base = devm_ioremap_resource(&pdev->dev, res);
+		if (IS_ERR(tegra->ipfs_base))
+			return PTR_ERR(tegra->ipfs_base);
+	}
 
 	tegra->xhci_irq = platform_get_irq(pdev, 0);
 	if (tegra->xhci_irq < 0)
@@ -1208,7 +1215,7 @@ static int tegra_xusb_probe(struct platform_device *pdev)
 		goto disable_rpm;
 	}
 
-	tegra_xusb_ipfs_config(tegra, regs);
+	tegra_xusb_config(tegra, regs);
 
 	err = tegra_xusb_load_firmware(tegra);
 	if (err < 0) {
@@ -1380,6 +1387,7 @@ static const struct tegra_xusb_soc tegra124_soc = {
 		.usb3 = { .offset = 0, .count = 2, },
 	},
 	.scale_ss_clock = true,
+	.has_ipfs = true,
 };
 MODULE_FIRMWARE("nvidia/tegra124/xusb.bin");
 
@@ -1411,12 +1419,38 @@ static const struct tegra_xusb_soc tegra210_soc = {
 		.usb3 = { .offset = 0, .count = 4, },
 	},
 	.scale_ss_clock = false,
+	.has_ipfs = true,
 };
 MODULE_FIRMWARE("nvidia/tegra210/xusb.bin");
 
+static const char * const tegra186_supply_names[] = {
+};
+
+static const struct tegra_xusb_phy_type tegra186_phy_types[] = {
+	{ .name = "usb3", .num = 3, },
+	{ .name = "usb2", .num = 3, },
+	{ .name = "hsic", .num = 1, },
+};
+
+static const struct tegra_xusb_soc tegra186_soc = {
+	.firmware = "nvidia/tegra186/xusb.bin",
+	.supply_names = tegra186_supply_names,
+	.num_supplies = ARRAY_SIZE(tegra186_supply_names),
+	.phy_types = tegra186_phy_types,
+	.num_types = ARRAY_SIZE(tegra186_phy_types),
+	.ports = {
+		.usb3 = { .offset = 0, .count = 3, },
+		.usb2 = { .offset = 3, .count = 3, },
+		.hsic = { .offset = 6, .count = 1, },
+	},
+	.scale_ss_clock = false,
+	.has_ipfs = false,
+};
+
 static const struct of_device_id tegra_xusb_of_match[] = {
 	{ .compatible = "nvidia,tegra124-xusb", .data = &tegra124_soc },
 	{ .compatible = "nvidia,tegra210-xusb", .data = &tegra210_soc },
+	{ .compatible = "nvidia,tegra186-xusb", .data = &tegra186_soc },
 	{ },
 };
 MODULE_DEVICE_TABLE(of, tegra_xusb_of_match);
diff --git a/drivers/usb/host/xhci-trace.h b/drivers/usb/host/xhci-trace.h
index 88b427434bd8..052a269d86f2 100644
--- a/drivers/usb/host/xhci-trace.h
+++ b/drivers/usb/host/xhci-trace.h
@@ -366,6 +366,11 @@ DEFINE_EVENT(xhci_log_ep_ctx, xhci_handle_cmd_config_ep,
 	TP_ARGS(ctx)
 );
 
+DEFINE_EVENT(xhci_log_ep_ctx, xhci_add_endpoint,
+	TP_PROTO(struct xhci_ep_ctx *ctx),
+	TP_ARGS(ctx)
+);
+
 DECLARE_EVENT_CLASS(xhci_log_slot_ctx,
 	TP_PROTO(struct xhci_slot_ctx *ctx),
 	TP_ARGS(ctx),
@@ -432,6 +437,31 @@ DEFINE_EVENT(xhci_log_slot_ctx, xhci_configure_endpoint,
 	TP_ARGS(ctx)
 );
 
+DECLARE_EVENT_CLASS(xhci_log_ctrl_ctx,
+	TP_PROTO(struct xhci_input_control_ctx *ctrl_ctx),
+	TP_ARGS(ctrl_ctx),
+	TP_STRUCT__entry(
+		__field(u32, drop)
+		__field(u32, add)
+	),
+	TP_fast_assign(
+		__entry->drop = le32_to_cpu(ctrl_ctx->drop_flags);
+		__entry->add = le32_to_cpu(ctrl_ctx->add_flags);
+	),
+	TP_printk("%s", xhci_decode_ctrl_ctx(__entry->drop, __entry->add)
+	)
+);
+
+DEFINE_EVENT(xhci_log_ctrl_ctx, xhci_address_ctrl_ctx,
+	TP_PROTO(struct xhci_input_control_ctx *ctrl_ctx),
+	TP_ARGS(ctrl_ctx)
+);
+
+DEFINE_EVENT(xhci_log_ctrl_ctx, xhci_configure_endpoint_ctrl_ctx,
+	TP_PROTO(struct xhci_input_control_ctx *ctrl_ctx),
+	TP_ARGS(ctrl_ctx)
+);
+
 DECLARE_EVENT_CLASS(xhci_log_ring,
 	TP_PROTO(struct xhci_ring *ring),
 	TP_ARGS(ring),
diff --git a/drivers/usb/host/xhci.c b/drivers/usb/host/xhci.c
index 7fa58c99f126..a9bb796794e3 100644
--- a/drivers/usb/host/xhci.c
+++ b/drivers/usb/host/xhci.c
@@ -893,7 +893,7 @@ static void xhci_disable_port_wake_on_bits(struct xhci_hcd *xhci)
 	struct xhci_port **ports;
 	int port_index;
 	unsigned long flags;
-	u32 t1, t2;
+	u32 t1, t2, portsc;
 
 	spin_lock_irqsave(&xhci->lock, flags);
 
@@ -902,10 +902,15 @@ static void xhci_disable_port_wake_on_bits(struct xhci_hcd *xhci)
 	ports = xhci->usb3_rhub.ports;
 	while (port_index--) {
 		t1 = readl(ports[port_index]->addr);
+		portsc = t1;
 		t1 = xhci_port_state_to_neutral(t1);
 		t2 = t1 & ~PORT_WAKE_BITS;
-		if (t1 != t2)
+		if (t1 != t2) {
 			writel(t2, ports[port_index]->addr);
+			xhci_dbg(xhci, "disable wake bits port %d-%d, portsc: 0x%x, write: 0x%x\n",
+				 xhci->usb3_rhub.hcd->self.busnum,
+				 port_index + 1, portsc, t2);
+		}
 	}
 
 	/* disable usb2 ports Wake bits */
@@ -913,12 +918,16 @@ static void xhci_disable_port_wake_on_bits(struct xhci_hcd *xhci)
 	ports = xhci->usb2_rhub.ports;
 	while (port_index--) {
 		t1 = readl(ports[port_index]->addr);
+		portsc = t1;
 		t1 = xhci_port_state_to_neutral(t1);
 		t2 = t1 & ~PORT_WAKE_BITS;
-		if (t1 != t2)
+		if (t1 != t2) {
 			writel(t2, ports[port_index]->addr);
+			xhci_dbg(xhci, "disable wake bits port %d-%d, portsc: 0x%x, write: 0x%x\n",
+				 xhci->usb2_rhub.hcd->self.busnum,
+				 port_index + 1, portsc, t2);
+		}
 	}
-
 	spin_unlock_irqrestore(&xhci->lock, flags);
 }
 
@@ -1238,6 +1247,21 @@ EXPORT_SYMBOL_GPL(xhci_resume);
 
 /*-------------------------------------------------------------------------*/
 
+/*
+ * Bypass the DMA mapping if URB is suitable for Immediate Transfer (IDT),
+ * we'll copy the actual data into the TRB address register. This is limited to
+ * transfers up to 8 bytes on output endpoints of any kind with wMaxPacketSize
+ * >= 8 bytes. If suitable for IDT only one Transfer TRB per TD is allowed.
+ */
+static int xhci_map_urb_for_dma(struct usb_hcd *hcd, struct urb *urb,
+				gfp_t mem_flags)
+{
+	if (xhci_urb_suitable_for_idt(urb))
+		return 0;
+
+	return usb_hcd_map_urb_for_dma(hcd, urb, mem_flags);
+}
+
 /**
  * xhci_get_endpoint_index - Used for passing endpoint bitmasks between the core and
  * HCDs.  Find the index for an endpoint given its descriptor.  Use the return
@@ -1783,6 +1807,7 @@ static int xhci_add_endpoint(struct usb_hcd *hcd, struct usb_device *udev,
 	struct xhci_container_ctx *in_ctx;
 	unsigned int ep_index;
 	struct xhci_input_control_ctx *ctrl_ctx;
+	struct xhci_ep_ctx *ep_ctx;
 	u32 added_ctxs;
 	u32 new_add_flags, new_drop_flags;
 	struct xhci_virt_device *virt_dev;
@@ -1873,6 +1898,9 @@ static int xhci_add_endpoint(struct usb_hcd *hcd, struct usb_device *udev,
 	/* Store the usb_device pointer for later use */
 	ep->hcpriv = udev;
 
+	ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, ep_index);
+	trace_xhci_add_endpoint(ep_ctx);
+
 	xhci_debugfs_create_endpoint(xhci, virt_dev, ep_index);
 
 	xhci_dbg(xhci, "add ep 0x%x, slot id %d, new drop flags = %#x, new add flags = %#x\n",
@@ -2747,6 +2775,8 @@ static int xhci_configure_endpoint(struct xhci_hcd *xhci,
 	}
 
 	slot_ctx = xhci_get_slot_ctx(xhci, command->in_ctx);
+
+	trace_xhci_configure_endpoint_ctrl_ctx(ctrl_ctx);
 	trace_xhci_configure_endpoint(slot_ctx);
 
 	if (!ctx_change)
@@ -4012,6 +4042,7 @@ static int xhci_setup_device(struct usb_hcd *hcd, struct usb_device *udev,
 	trace_xhci_address_ctx(xhci, virt_dev->in_ctx,
 				le32_to_cpu(slot_ctx->dev_info) >> 27);
 
+	trace_xhci_address_ctrl_ctx(ctrl_ctx);
 	spin_lock_irqsave(&xhci->lock, flags);
 	trace_xhci_setup_device(virt_dev);
 	ret = xhci_queue_address_device(xhci, command, virt_dev->in_ctx->dma,
@@ -5154,6 +5185,7 @@ static const struct hc_driver xhci_hc_driver = {
 	/*
 	 * managing i/o requests and associated device resources
 	 */
+	.map_urb_for_dma =      xhci_map_urb_for_dma,
 	.urb_enqueue =		xhci_urb_enqueue,
 	.urb_dequeue =		xhci_urb_dequeue,
 	.alloc_dev =		xhci_alloc_dev,
diff --git a/drivers/usb/host/xhci.h b/drivers/usb/host/xhci.h
index 9334cdee382a..a450a99e90eb 100644
--- a/drivers/usb/host/xhci.h
+++ b/drivers/usb/host/xhci.h
@@ -1303,6 +1303,8 @@ enum xhci_setup_dev {
 #define TRB_IOC			(1<<5)
 /* The buffer pointer contains immediate data */
 #define TRB_IDT			(1<<6)
+/* TDs smaller than this might use IDT */
+#define TRB_IDT_MAX_SIZE	8
 
 /* Block Event Interrupt */
 #define	TRB_BEI			(1<<9)
@@ -2149,6 +2151,21 @@ static inline struct xhci_ring *xhci_urb_to_transfer_ring(struct xhci_hcd *xhci,
 					urb->stream_id);
 }
 
+/*
+ * TODO: As per spec Isochronous IDT transmissions are supported. We bypass
+ * them anyways as we where unable to find a device that matches the
+ * constraints.
+ */
+static inline bool xhci_urb_suitable_for_idt(struct urb *urb)
+{
+	if (!usb_endpoint_xfer_isoc(&urb->ep->desc) && usb_urb_dir_out(urb) &&
+	    usb_endpoint_maxp(&urb->ep->desc) >= TRB_IDT_MAX_SIZE &&
+	    urb->transfer_buffer_length <= TRB_IDT_MAX_SIZE)
+		return true;
+
+	return false;
+}
+
 static inline char *xhci_slot_state_string(u32 state)
 {
 	switch (state) {
@@ -2384,6 +2401,35 @@ static inline const char *xhci_decode_trb(u32 field0, u32 field1, u32 field2,
 	return str;
 }
 
+static inline const char *xhci_decode_ctrl_ctx(unsigned long drop,
+					       unsigned long add)
+{
+	static char	str[1024];
+	unsigned int	bit;
+	int		ret = 0;
+
+	if (drop) {
+		ret = sprintf(str, "Drop:");
+		for_each_set_bit(bit, &drop, 32)
+			ret += sprintf(str + ret, " %d%s",
+				       bit / 2,
+				       bit % 2 ? "in":"out");
+		ret += sprintf(str + ret, ", ");
+	}
+
+	if (add) {
+		ret += sprintf(str + ret, "Add:%s%s",
+			       (add & SLOT_FLAG) ? " slot":"",
+			       (add & EP0_FLAG) ? " ep0":"");
+		add &= ~(SLOT_FLAG | EP0_FLAG);
+		for_each_set_bit(bit, &add, 32)
+			ret += sprintf(str + ret, " %d%s",
+				       bit / 2,
+				       bit % 2 ? "in":"out");
+	}
+	return str;
+}
+
 static inline const char *xhci_decode_slot_context(u32 info, u32 info2,
 		u32 tt_info, u32 state)
 {
diff --git a/drivers/usb/isp1760/isp1760-hcd.c b/drivers/usb/isp1760/isp1760-hcd.c
index 8142c6b4c4cf..320fc4739835 100644
--- a/drivers/usb/isp1760/isp1760-hcd.c
+++ b/drivers/usb/isp1760/isp1760-hcd.c
@@ -788,11 +788,11 @@ static void collect_qtds(struct usb_hcd *hcd, struct isp1760_qh *qh,
 					mem_reads8(hcd->regs, qtd->payload_addr,
 							qtd->data_buffer,
 							qtd->actual_length);
-					/* Fall through (?) */
+					/* Fall through */
 				case OUT_PID:
 					qtd->urb->actual_length +=
 							qtd->actual_length;
-					/* Fall through ... */
+					/* Fall through */
 				case SETUP_PID:
 					break;
 				}
diff --git a/drivers/usb/misc/usb251xb.c b/drivers/usb/misc/usb251xb.c
index 04684849d683..4d6ae3795a88 100644
--- a/drivers/usb/misc/usb251xb.c
+++ b/drivers/usb/misc/usb251xb.c
@@ -12,6 +12,7 @@
 
 #include <linux/delay.h>
 #include <linux/gpio/consumer.h>
+#include <linux/gpio/driver.h>
 #include <linux/i2c.h>
 #include <linux/module.h>
 #include <linux/nls.h>
@@ -222,11 +223,51 @@ static const struct usb251xb_data usb2517i_data = {
 	.product_str = "USB2517i",
 };
 
+#ifdef CONFIG_GPIOLIB
+static int usb251xb_check_dev_children(struct device *dev, void *child)
+{
+	if (dev->type == &i2c_adapter_type) {
+		return device_for_each_child(dev, child,
+					     usb251xb_check_dev_children);
+	}
+
+	return (dev == child);
+}
+
+static int usb251x_check_gpio_chip(struct usb251xb *hub)
+{
+	struct gpio_chip *gc = gpiod_to_chip(hub->gpio_reset);
+	struct i2c_adapter *adap = hub->i2c->adapter;
+	int ret;
+
+	if (!hub->gpio_reset)
+		return 0;
+
+	if (!gc)
+		return -EINVAL;
+
+	ret = usb251xb_check_dev_children(&adap->dev, gc->parent);
+	if (ret) {
+		dev_err(hub->dev, "Reset GPIO chip is at the same i2c-bus\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+#else
+static int usb251x_check_gpio_chip(struct usb251xb *hub)
+{
+	return 0;
+}
+#endif
+
 static void usb251xb_reset(struct usb251xb *hub, int state)
 {
 	if (!hub->gpio_reset)
 		return;
 
+	i2c_lock_bus(hub->i2c->adapter, I2C_LOCK_SEGMENT);
+
 	gpiod_set_value_cansleep(hub->gpio_reset, state);
 
 	/* wait for hub recovery/stabilization */
@@ -234,6 +275,8 @@ static void usb251xb_reset(struct usb251xb *hub, int state)
 		usleep_range(500, 750);	/* >=500us at power on */
 	else
 		usleep_range(1, 10);	/* >=1us at power down */
+
+	i2c_unlock_bus(hub->i2c->adapter, I2C_LOCK_SEGMENT);
 }
 
 static int usb251xb_connect(struct usb251xb *hub)
@@ -331,18 +374,31 @@ out_err:
 }
 
 #ifdef CONFIG_OF
+static void usb251xb_get_ports_field(struct usb251xb *hub,
+				    const char *prop_name, u8 port_cnt, u8 *fld)
+{
+	struct device *dev = hub->dev;
+	struct property *prop;
+	const __be32 *p;
+	u32 port;
+
+	of_property_for_each_u32(dev->of_node, prop_name, prop, p, port) {
+		if ((port >= 1) && (port <= port_cnt))
+			*fld |= BIT(port);
+		else
+			dev_warn(dev, "port %u doesn't exist\n", port);
+	}
+}
+
 static int usb251xb_get_ofdata(struct usb251xb *hub,
 			       struct usb251xb_data *data)
 {
 	struct device *dev = hub->dev;
 	struct device_node *np = dev->of_node;
-	int len, err, i;
-	u32 port, property_u32 = 0;
-	const u32 *cproperty_u32;
+	int len, err;
+	u32 property_u32 = 0;
 	const char *cproperty_char;
 	char str[USB251XB_STRING_BUFSIZE / 2];
-	struct property *prop;
-	const __be32 *p;
 
 	if (!np) {
 		dev_err(dev, "failed to get ofdata\n");
@@ -444,46 +500,16 @@ static int usb251xb_get_ofdata(struct usb251xb *hub,
 		hub->conf_data3 |= BIT(0);
 
 	hub->non_rem_dev = USB251XB_DEF_NON_REMOVABLE_DEVICES;
-	cproperty_u32 = of_get_property(np, "non-removable-ports", &len);
-	if (cproperty_u32 && (len / sizeof(u32)) > 0) {
-		for (i = 0; i < len / sizeof(u32); i++) {
-			u32 port = be32_to_cpu(cproperty_u32[i]);
-
-			if ((port >= 1) && (port <= data->port_cnt))
-				hub->non_rem_dev |= BIT(port);
-			else
-				dev_warn(dev, "NRD port %u doesn't exist\n",
-					port);
-		}
-	}
+	usb251xb_get_ports_field(hub, "non-removable-ports", data->port_cnt,
+				 &hub->non_rem_dev);
 
 	hub->port_disable_sp = USB251XB_DEF_PORT_DISABLE_SELF;
-	cproperty_u32 = of_get_property(np, "sp-disabled-ports", &len);
-	if (cproperty_u32 && (len / sizeof(u32)) > 0) {
-		for (i = 0; i < len / sizeof(u32); i++) {
-			u32 port = be32_to_cpu(cproperty_u32[i]);
-
-			if ((port >= 1) && (port <= data->port_cnt))
-				hub->port_disable_sp |= BIT(port);
-			else
-				dev_warn(dev, "PDS port %u doesn't exist\n",
-					port);
-		}
-	}
+	usb251xb_get_ports_field(hub, "sp-disabled-ports", data->port_cnt,
+				 &hub->port_disable_sp);
 
 	hub->port_disable_bp = USB251XB_DEF_PORT_DISABLE_BUS;
-	cproperty_u32 = of_get_property(np, "bp-disabled-ports", &len);
-	if (cproperty_u32 && (len / sizeof(u32)) > 0) {
-		for (i = 0; i < len / sizeof(u32); i++) {
-			u32 port = be32_to_cpu(cproperty_u32[i]);
-
-			if ((port >= 1) && (port <= data->port_cnt))
-				hub->port_disable_bp |= BIT(port);
-			else
-				dev_warn(dev, "PDB port %u doesn't exist\n",
-					port);
-		}
-	}
+	usb251xb_get_ports_field(hub, "bp-disabled-ports", data->port_cnt,
+				 &hub->port_disable_bp);
 
 	hub->max_power_sp = USB251XB_DEF_MAX_POWER_SELF;
 	if (!of_property_read_u32(np, "sp-max-total-current-microamp",
@@ -546,10 +572,10 @@ static int usb251xb_get_ofdata(struct usb251xb *hub,
 	 * register controls the USB DP/DM signal swapping for each port.
 	 */
 	hub->port_swap = USB251XB_DEF_PORT_SWAP;
-	of_property_for_each_u32(np, "swap-dx-lanes", prop, p, port) {
-		if (port <= data->port_cnt)
-			hub->port_swap |= BIT(port);
-	}
+	usb251xb_get_ports_field(hub, "swap-dx-lanes", data->port_cnt,
+				 &hub->port_swap);
+	if (of_get_property(np, "swap-us-lanes", NULL))
+		hub->port_swap |= BIT(0);
 
 	/* The following parameters are currently not exposed to devicetree, but
 	 * may be as soon as needed.
@@ -621,6 +647,25 @@ static int usb251xb_probe(struct usb251xb *hub)
 		}
 	}
 
+	/*
+	 * usb251x SMBus-slave SCL lane is muxed with CFG_SEL0 pin. So if anyone
+	 * tries to work with the bus at the moment the hub reset is released,
+	 * it may cause an invalid config being latched by usb251x. Particularly
+	 * one of the config modes makes the hub loading a default registers
+	 * value without SMBus-slave interface activation. If the hub
+	 * accidentally gets this mode, this will cause the driver SMBus-
+	 * functions failure. Normally we could just lock the SMBus-segment the
+	 * hub i2c-interface resides for the device-specific reset timing. But
+	 * the GPIO controller, which is used to handle the hub reset, might be
+	 * placed at the same i2c-bus segment. In this case an error should be
+	 * returned since we can't safely use the GPIO controller to clear the
+	 * reset state (it may affect the hub configuration) and we can't lock
+	 * the i2c-bus segment (it will cause a deadlock).
+	 */
+	err = usb251x_check_gpio_chip(hub);
+	if (err)
+		return err;
+
 	err = usb251xb_connect(hub);
 	if (err) {
 		dev_err(dev, "Failed to connect hub (%d)\n", err);
diff --git a/drivers/usb/misc/usb3503.c b/drivers/usb/misc/usb3503.c
index d5141aa79dd4..72f39a9751b5 100644
--- a/drivers/usb/misc/usb3503.c
+++ b/drivers/usb/misc/usb3503.c
@@ -172,7 +172,6 @@ static int usb3503_probe(struct usb3503 *hub)
 		hub->gpio_reset		= pdata->gpio_reset;
 		hub->mode		= pdata->initial_mode;
 	} else if (np) {
-		struct clk *clk;
 		u32 rate = 0;
 		hub->port_off_mask = 0;
 
@@ -198,34 +197,29 @@ static int usb3503_probe(struct usb3503 *hub)
 			}
 		}
 
-		clk = devm_clk_get(dev, "refclk");
-		if (IS_ERR(clk) && PTR_ERR(clk) != -ENOENT) {
+		hub->clk = devm_clk_get_optional(dev, "refclk");
+		if (IS_ERR(hub->clk)) {
 			dev_err(dev, "unable to request refclk (%ld)\n",
-					PTR_ERR(clk));
-			return PTR_ERR(clk);
+					PTR_ERR(hub->clk));
+			return PTR_ERR(hub->clk);
 		}
 
-		if (!IS_ERR(clk)) {
-			hub->clk = clk;
-
-			if (rate != 0) {
-				err = clk_set_rate(hub->clk, rate);
-				if (err) {
-					dev_err(dev,
-						"unable to set reference clock rate to %d\n",
-						(int) rate);
-					return err;
-				}
-			}
-
-			err = clk_prepare_enable(hub->clk);
+		if (rate != 0) {
+			err = clk_set_rate(hub->clk, rate);
 			if (err) {
 				dev_err(dev,
-					"unable to enable reference clock\n");
+					"unable to set reference clock rate to %d\n",
+					(int)rate);
 				return err;
 			}
 		}
 
+		err = clk_prepare_enable(hub->clk);
+		if (err) {
+			dev_err(dev, "unable to enable reference clock\n");
+			return err;
+		}
+
 		property = of_get_property(np, "disabled-ports", &len);
 		if (property && (len / sizeof(u32)) > 0) {
 			int i;
@@ -324,8 +318,7 @@ static int usb3503_i2c_remove(struct i2c_client *i2c)
 	struct usb3503 *hub;
 
 	hub = i2c_get_clientdata(i2c);
-	if (hub->clk)
-		clk_disable_unprepare(hub->clk);
+	clk_disable_unprepare(hub->clk);
 
 	return 0;
 }
@@ -348,8 +341,7 @@ static int usb3503_platform_remove(struct platform_device *pdev)
 	struct usb3503 *hub;
 
 	hub = platform_get_drvdata(pdev);
-	if (hub->clk)
-		clk_disable_unprepare(hub->clk);
+	clk_disable_unprepare(hub->clk);
 
 	return 0;
 }
@@ -358,18 +350,14 @@ static int usb3503_platform_remove(struct platform_device *pdev)
 static int usb3503_suspend(struct usb3503 *hub)
 {
 	usb3503_switch_mode(hub, USB3503_MODE_STANDBY);
-
-	if (hub->clk)
-		clk_disable_unprepare(hub->clk);
+	clk_disable_unprepare(hub->clk);
 
 	return 0;
 }
 
 static int usb3503_resume(struct usb3503 *hub)
 {
-	if (hub->clk)
-		clk_prepare_enable(hub->clk);
-
+	clk_prepare_enable(hub->clk);
 	usb3503_switch_mode(hub, hub->mode);
 
 	return 0;
diff --git a/drivers/usb/mtu3/Makefile b/drivers/usb/mtu3/Makefile
index 4a9715812bf9..3bf8cbcc1add 100644
--- a/drivers/usb/mtu3/Makefile
+++ b/drivers/usb/mtu3/Makefile
@@ -2,10 +2,17 @@
 
 ccflags-$(CONFIG_USB_MTU3_DEBUG)	+= -DDEBUG
 
+# define_trace.h needs to know how to find our header
+CFLAGS_mtu3_trace.o	:= -I$(src)
+
 obj-$(CONFIG_USB_MTU3)	+= mtu3.o
 
 mtu3-y	:= mtu3_plat.o
 
+ifneq ($(CONFIG_TRACING),)
+	mtu3-y	+= mtu3_trace.o
+endif
+
 ifneq ($(filter y,$(CONFIG_USB_MTU3_HOST) $(CONFIG_USB_MTU3_DUAL_ROLE)),)
 	mtu3-y	+= mtu3_host.o
 endif
@@ -17,3 +24,7 @@ endif
 ifneq ($(CONFIG_USB_MTU3_DUAL_ROLE),)
 	mtu3-y	+= mtu3_dr.o
 endif
+
+ifneq ($(CONFIG_DEBUG_FS),)
+	mtu3-y	+= mtu3_debugfs.o
+endif
diff --git a/drivers/usb/mtu3/mtu3.h b/drivers/usb/mtu3/mtu3.h
index 87823ac0d120..76ecf12fdf62 100644
--- a/drivers/usb/mtu3/mtu3.h
+++ b/drivers/usb/mtu3/mtu3.h
@@ -63,6 +63,15 @@ struct mtu3_request;
 #define MTU3_U2_IP_SLOT_DEFAULT 1
 
 /**
+ * IP TRUNK version
+ * from 0x1003 version, USB3 Gen2 is supported, two changes affect driver:
+ * 1. MAXPKT and MULTI bits layout of TXCSR1 and RXCSR1 are adjusted,
+ *    but not backward compatible
+ * 2. QMU extend buffer length supported
+ */
+#define MTU3_TRUNK_VERS_1003	0x1003
+
+/**
  * Normally the device works on HS or SS, to simplify fifo management,
  * devide fifo into some 512B parts, use bitmap to manage it; And
  * 128 bits size of bitmap is large enough, that means it can manage
@@ -135,45 +144,33 @@ struct mtu3_fifo_info {
  *	The format of TX GPD is a little different from RX one.
  *	And the size of GPD is 16 bytes.
  *
- * @flag:
+ * @dw0_info:
  *	bit0: Hardware Own (HWO)
  *	bit1: Buffer Descriptor Present (BDP), always 0, BD is not supported
  *	bit2: Bypass (BPS), 1: HW skips this GPD if HWO = 1
+ *	bit6: [EL] Zero Length Packet (ZLP), moved from @dw3_info[29]
  *	bit7: Interrupt On Completion (IOC)
- * @chksum: This is used to validate the contents of this GPD;
- *	If TXQ_CS_EN / RXQ_CS_EN bit is set, an interrupt is issued
- *	when checksum validation fails;
- *	Checksum value is calculated over the 16 bytes of the GPD by default;
- * @data_buf_len (RX ONLY): This value indicates the length of
- *	the assigned data buffer
- * @tx_ext_addr (TX ONLY): [3:0] are 4 extension bits of @buffer,
- *	[7:4] are 4 extension bits of @next_gpd
+ *	bit[31:16]: ([EL] bit[31:12]) allow data buffer length (RX ONLY),
+ *		the buffer length of the data to receive
+ *	bit[23:16]: ([EL] bit[31:24]) extension address (TX ONLY),
+ *		lower 4 bits are extension bits of @buffer,
+ *		upper 4 bits are extension bits of @next_gpd
  * @next_gpd: Physical address of the next GPD
  * @buffer: Physical address of the data buffer
- * @buf_len:
- *	(TX): This value indicates the length of the assigned data buffer
- *	(RX): The total length of data received
- * @ext_len: reserved
- * @rx_ext_addr(RX ONLY): [3:0] are 4 extension bits of @buffer,
- *	[7:4] are 4 extension bits of @next_gpd
- * @ext_flag:
- *	bit5 (TX ONLY): Zero Length Packet (ZLP),
+ * @dw3_info:
+ *	bit[15:0]: ([EL] bit[19:0]) data buffer length,
+ *		(TX): the buffer length of the data to transmit
+ *		(RX): The total length of data received
+ *	bit[23:16]: ([EL] bit[31:24]) extension address (RX ONLY),
+ *		lower 4 bits are extension bits of @buffer,
+ *		upper 4 bits are extension bits of @next_gpd
+ *	bit29: ([EL] abandoned) Zero Length Packet (ZLP) (TX ONLY)
  */
 struct qmu_gpd {
-	__u8 flag;
-	__u8 chksum;
-	union {
-		__le16 data_buf_len;
-		__le16 tx_ext_addr;
-	};
+	__le32 dw0_info;
 	__le32 next_gpd;
 	__le32 buffer;
-	__le16 buf_len;
-	union {
-		__u8 ext_len;
-		__u8 rx_ext_addr;
-	};
-	__u8 ext_flag;
+	__le32 dw3_info;
 } __packed;
 
 /**
@@ -316,6 +313,7 @@ static inline struct ssusb_mtk *dev_to_ssusb(struct device *dev)
  * @may_wakeup: means device's remote wakeup is enabled
  * @is_self_powered: is reported in device status and the config descriptor
  * @delayed_status: true when function drivers ask for delayed status
+ * @gen2cp: compatible with USB3 Gen2 IP
  * @ep0_req: dummy request used while handling standard USB requests
  *		for GET_STATUS and SET_SEL
  * @setup_buf: ep0 response buffer for GET_STATUS and SET_SEL requests
@@ -356,6 +354,7 @@ struct mtu3 {
 	unsigned u2_enable:1;
 	unsigned is_u3_ip:1;
 	unsigned delayed_status:1;
+	unsigned gen2cp:1;
 
 	u8 address;
 	u8 test_mode_nr;
diff --git a/drivers/usb/mtu3/mtu3_core.c b/drivers/usb/mtu3/mtu3_core.c
index 4fee200795a5..f8bd1d57e795 100644
--- a/drivers/usb/mtu3/mtu3_core.c
+++ b/drivers/usb/mtu3/mtu3_core.c
@@ -16,6 +16,8 @@
 #include <linux/platform_device.h>
 
 #include "mtu3.h"
+#include "mtu3_debug.h"
+#include "mtu3_trace.h"
 
 static int ep_fifo_alloc(struct mtu3_ep *mep, u32 seg_size)
 {
@@ -299,6 +301,7 @@ int mtu3_config_ep(struct mtu3 *mtu, struct mtu3_ep *mep,
 			int interval, int burst, int mult)
 {
 	void __iomem *mbase = mtu->mac_base;
+	bool gen2cp = mtu->gen2cp;
 	int epnum = mep->epnum;
 	u32 csr0, csr1, csr2;
 	int fifo_sgsz, fifo_addr;
@@ -319,7 +322,7 @@ int mtu3_config_ep(struct mtu3 *mtu, struct mtu3_ep *mep,
 
 		num_pkts = (burst + 1) * (mult + 1) - 1;
 		csr1 = TX_SS_BURST(burst) | TX_SLOT(mep->slot);
-		csr1 |= TX_MAX_PKT(num_pkts) | TX_MULT(mult);
+		csr1 |= TX_MAX_PKT(gen2cp, num_pkts) | TX_MULT(gen2cp, mult);
 
 		csr2 = TX_FIFOADDR(fifo_addr >> 4);
 		csr2 |= TX_FIFOSEGSIZE(fifo_sgsz);
@@ -355,7 +358,7 @@ int mtu3_config_ep(struct mtu3 *mtu, struct mtu3_ep *mep,
 
 		num_pkts = (burst + 1) * (mult + 1) - 1;
 		csr1 = RX_SS_BURST(burst) | RX_SLOT(mep->slot);
-		csr1 |= RX_MAX_PKT(num_pkts) | RX_MULT(mult);
+		csr1 |= RX_MAX_PKT(gen2cp, num_pkts) | RX_MULT(gen2cp, mult);
 
 		csr2 = RX_FIFOADDR(fifo_addr >> 4);
 		csr2 |= RX_FIFOSEGSIZE(fifo_sgsz);
@@ -600,6 +603,10 @@ static void mtu3_regs_init(struct mtu3 *mtu)
 	mtu3_clrbits(mbase, U3D_MISC_CTRL, VBUS_FRC_EN | VBUS_ON);
 	/* enable automatical HWRW from L1 */
 	mtu3_setbits(mbase, U3D_POWER_MANAGEMENT, LPM_HRWE);
+
+	/* use new QMU format when HW version >= 0x1003 */
+	if (mtu->gen2cp)
+		mtu3_writel(mbase, U3D_QFCR, ~0x0);
 }
 
 static irqreturn_t mtu3_link_isr(struct mtu3 *mtu)
@@ -650,6 +657,8 @@ static irqreturn_t mtu3_link_isr(struct mtu3 *mtu)
 		break;
 	}
 	dev_dbg(mtu->dev, "%s: %s\n", __func__, usb_speed_string(udev_speed));
+	mtu3_dbg_trace(mtu->dev, "link speed %s",
+		       usb_speed_string(udev_speed));
 
 	mtu->g.speed = udev_speed;
 	mtu->g.ep0->maxpacket = maxpkt;
@@ -672,6 +681,7 @@ static irqreturn_t mtu3_u3_ltssm_isr(struct mtu3 *mtu)
 	ltssm &= mtu3_readl(mbase, U3D_LTSSM_INTR_ENABLE);
 	mtu3_writel(mbase, U3D_LTSSM_INTR, ltssm); /* W1C */
 	dev_dbg(mtu->dev, "=== LTSSM[%x] ===\n", ltssm);
+	trace_mtu3_u3_ltssm_isr(ltssm);
 
 	if (ltssm & (HOT_RST_INTR | WARM_RST_INTR))
 		mtu3_gadget_reset(mtu);
@@ -702,6 +712,7 @@ static irqreturn_t mtu3_u2_common_isr(struct mtu3 *mtu)
 	u2comm &= mtu3_readl(mbase, U3D_COMMON_USB_INTR_ENABLE);
 	mtu3_writel(mbase, U3D_COMMON_USB_INTR, u2comm); /* W1C */
 	dev_dbg(mtu->dev, "=== U2COMM[%x] ===\n", u2comm);
+	trace_mtu3_u2_common_isr(u2comm);
 
 	if (u2comm & SUSPEND_INTR)
 		mtu3_gadget_suspend(mtu);
@@ -749,13 +760,15 @@ static irqreturn_t mtu3_irq(int irq, void *data)
 
 static int mtu3_hw_init(struct mtu3 *mtu)
 {
-	u32 cap_dev;
+	u32 value;
 	int ret;
 
-	mtu->hw_version = mtu3_readl(mtu->ippc_base, U3D_SSUSB_HW_ID);
+	value = mtu3_readl(mtu->ippc_base, U3D_SSUSB_IP_TRUNK_VERS);
+	mtu->hw_version = IP_TRUNK_VERS(value);
+	mtu->gen2cp = !!(mtu->hw_version >= MTU3_TRUNK_VERS_1003);
 
-	cap_dev = mtu3_readl(mtu->ippc_base, U3D_SSUSB_IP_DEV_CAP);
-	mtu->is_u3_ip = !!SSUSB_IP_DEV_U3_PORT_NUM(cap_dev);
+	value = mtu3_readl(mtu->ippc_base, U3D_SSUSB_IP_DEV_CAP);
+	mtu->is_u3_ip = !!SSUSB_IP_DEV_U3_PORT_NUM(value);
 
 	dev_info(mtu->dev, "IP version 0x%x(%s IP)\n", mtu->hw_version,
 		mtu->is_u3_ip ? "U3" : "U2");
@@ -893,6 +906,8 @@ int ssusb_gadget_init(struct ssusb_mtk *ssusb)
 	if (mtu->ssusb->dr_mode == USB_DR_MODE_OTG)
 		mtu3_stop(mtu);
 
+	ssusb_dev_debugfs_init(ssusb);
+
 	dev_dbg(dev, " %s() done...\n", __func__);
 
 	return 0;
diff --git a/drivers/usb/mtu3/mtu3_debug.h b/drivers/usb/mtu3/mtu3_debug.h
new file mode 100644
index 000000000000..e96a69234d05
--- /dev/null
+++ b/drivers/usb/mtu3/mtu3_debug.h
@@ -0,0 +1,50 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * mtu3_debug.h - debug header
+ *
+ * Copyright (C) 2019 MediaTek Inc.
+ *
+ * Author: Chunfeng Yun <chunfeng.yun@mediatek.com>
+ */
+
+#ifndef __MTU3_DEBUG_H__
+#define __MTU3_DEBUG_H__
+
+#include <linux/debugfs.h>
+
+#define MTU3_DEBUGFS_NAME_LEN 32
+
+struct mtu3_regset {
+	char name[MTU3_DEBUGFS_NAME_LEN];
+	struct debugfs_regset32 regset;
+	size_t nregs;
+};
+
+struct mtu3_file_map {
+	const char *name;
+	int (*show)(struct seq_file *s, void *unused);
+};
+
+#if IS_ENABLED(CONFIG_DEBUG_FS)
+void ssusb_dev_debugfs_init(struct ssusb_mtk *ssusb);
+void ssusb_dr_debugfs_init(struct ssusb_mtk *ssusb);
+void ssusb_debugfs_create_root(struct ssusb_mtk *ssusb);
+void ssusb_debugfs_remove_root(struct ssusb_mtk *ssusb);
+
+#else
+static inline void ssusb_dev_debugfs_init(struct ssusb_mtk *ssusb) {}
+static inline void ssusb_dr_debugfs_init(struct ssusb_mtk *ssusb) {}
+static inline void ssusb_debugfs_create_root(struct ssusb_mtk *ssusb) {}
+static inline void ssusb_debugfs_remove_root(struct ssusb_mtk *ssusb) {}
+
+#endif /* CONFIG_DEBUG_FS */
+
+#if IS_ENABLED(CONFIG_TRACING)
+void mtu3_dbg_trace(struct device *dev, const char *fmt, ...);
+
+#else
+static inline void mtu3_dbg_trace(struct device *dev, const char *fmt, ...) {}
+
+#endif /* CONFIG_TRACING */
+
+#endif /* __MTU3_DEBUG_H__ */
diff --git a/drivers/usb/mtu3/mtu3_debugfs.c b/drivers/usb/mtu3/mtu3_debugfs.c
new file mode 100644
index 000000000000..62c57ddc554e
--- /dev/null
+++ b/drivers/usb/mtu3/mtu3_debugfs.c
@@ -0,0 +1,539 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * mtu3_debugfs.c - debugfs interface
+ *
+ * Copyright (C) 2019 MediaTek Inc.
+ *
+ * Author: Chunfeng Yun <chunfeng.yun@mediatek.com>
+ */
+
+#include <linux/uaccess.h>
+
+#include "mtu3.h"
+#include "mtu3_dr.h"
+#include "mtu3_debug.h"
+
+#define dump_register(nm)		\
+{					\
+	.name = __stringify(nm),	\
+	.offset = U3D_ ##nm,		\
+}
+
+#define dump_prb_reg(nm, os)	\
+{				\
+	.name = nm,		\
+	.offset = os,		\
+}
+
+static const struct debugfs_reg32 mtu3_ippc_regs[] = {
+	dump_register(SSUSB_IP_PW_CTRL0),
+	dump_register(SSUSB_IP_PW_CTRL1),
+	dump_register(SSUSB_IP_PW_CTRL2),
+	dump_register(SSUSB_IP_PW_CTRL3),
+	dump_register(SSUSB_OTG_STS),
+	dump_register(SSUSB_IP_XHCI_CAP),
+	dump_register(SSUSB_IP_DEV_CAP),
+	dump_register(SSUSB_U3_CTRL_0P),
+	dump_register(SSUSB_U2_CTRL_0P),
+	dump_register(SSUSB_HW_ID),
+	dump_register(SSUSB_HW_SUB_ID),
+	dump_register(SSUSB_IP_SPARE0),
+};
+
+static const struct debugfs_reg32 mtu3_dev_regs[] = {
+	dump_register(LV1ISR),
+	dump_register(LV1IER),
+	dump_register(EPISR),
+	dump_register(EPIER),
+	dump_register(EP0CSR),
+	dump_register(RXCOUNT0),
+	dump_register(QISAR0),
+	dump_register(QIER0),
+	dump_register(QISAR1),
+	dump_register(QIER1),
+	dump_register(CAP_EPNTXFFSZ),
+	dump_register(CAP_EPNRXFFSZ),
+	dump_register(CAP_EPINFO),
+	dump_register(MISC_CTRL),
+};
+
+static const struct debugfs_reg32 mtu3_csr_regs[] = {
+	dump_register(DEVICE_CONF),
+	dump_register(DEV_LINK_INTR_ENABLE),
+	dump_register(DEV_LINK_INTR),
+	dump_register(LTSSM_CTRL),
+	dump_register(USB3_CONFIG),
+	dump_register(LINK_STATE_MACHINE),
+	dump_register(LTSSM_INTR_ENABLE),
+	dump_register(LTSSM_INTR),
+	dump_register(U3U2_SWITCH_CTRL),
+	dump_register(POWER_MANAGEMENT),
+	dump_register(DEVICE_CONTROL),
+	dump_register(COMMON_USB_INTR_ENABLE),
+	dump_register(COMMON_USB_INTR),
+	dump_register(USB20_MISC_CONTROL),
+	dump_register(USB20_OPSTATE),
+};
+
+static int mtu3_link_state_show(struct seq_file *sf, void *unused)
+{
+	struct mtu3 *mtu = sf->private;
+	void __iomem *mbase = mtu->mac_base;
+
+	seq_printf(sf, "opstate: %#x, ltssm: %#x\n",
+		   mtu3_readl(mbase, U3D_USB20_OPSTATE),
+		   LTSSM_STATE(mtu3_readl(mbase, U3D_LINK_STATE_MACHINE)));
+
+	return 0;
+}
+
+static int mtu3_ep_used_show(struct seq_file *sf, void *unused)
+{
+	struct mtu3 *mtu = sf->private;
+	struct mtu3_ep *mep;
+	unsigned long flags;
+	int used = 0;
+	int i;
+
+	spin_lock_irqsave(&mtu->lock, flags);
+
+	for (i = 0; i < mtu->num_eps; i++) {
+		mep = mtu->in_eps + i;
+		if (mep->flags & MTU3_EP_ENABLED) {
+			seq_printf(sf, "%s - type: %d\n", mep->name, mep->type);
+			used++;
+		}
+
+		mep = mtu->out_eps + i;
+		if (mep->flags & MTU3_EP_ENABLED) {
+			seq_printf(sf, "%s - type: %d\n", mep->name, mep->type);
+			used++;
+		}
+	}
+	seq_printf(sf, "total used: %d eps\n", used);
+
+	spin_unlock_irqrestore(&mtu->lock, flags);
+
+	return 0;
+}
+
+DEFINE_SHOW_ATTRIBUTE(mtu3_link_state);
+DEFINE_SHOW_ATTRIBUTE(mtu3_ep_used);
+
+static void mtu3_debugfs_regset(struct mtu3 *mtu, void __iomem *base,
+				const struct debugfs_reg32 *regs, size_t nregs,
+				const char *name, struct dentry *parent)
+{
+	struct debugfs_regset32 *regset;
+	struct mtu3_regset *mregs;
+
+	mregs = devm_kzalloc(mtu->dev, sizeof(*regset), GFP_KERNEL);
+	if (!mregs)
+		return;
+
+	sprintf(mregs->name, "%s", name);
+	regset = &mregs->regset;
+	regset->regs = regs;
+	regset->nregs = nregs;
+	regset->base = base;
+
+	debugfs_create_regset32(mregs->name, 0444, parent, regset);
+}
+
+static void mtu3_debugfs_ep_regset(struct mtu3 *mtu, struct mtu3_ep *mep,
+				   struct dentry *parent)
+{
+	struct debugfs_reg32 *regs;
+	int epnum = mep->epnum;
+	int in = mep->is_in;
+
+	regs = devm_kcalloc(mtu->dev, 7, sizeof(*regs), GFP_KERNEL);
+	if (!regs)
+		return;
+
+	regs[0].name = in ? "TCR0" : "RCR0";
+	regs[0].offset = in ? MU3D_EP_TXCR0(epnum) : MU3D_EP_RXCR0(epnum);
+	regs[1].name = in ? "TCR1" : "RCR1";
+	regs[1].offset = in ? MU3D_EP_TXCR1(epnum) : MU3D_EP_RXCR1(epnum);
+	regs[2].name = in ? "TCR2" : "RCR2";
+	regs[2].offset = in ? MU3D_EP_TXCR2(epnum) : MU3D_EP_RXCR2(epnum);
+	regs[3].name = in ? "TQHIAR" : "RQHIAR";
+	regs[3].offset = in ? USB_QMU_TQHIAR(epnum) : USB_QMU_RQHIAR(epnum);
+	regs[4].name = in ? "TQCSR" : "RQCSR";
+	regs[4].offset = in ? USB_QMU_TQCSR(epnum) : USB_QMU_RQCSR(epnum);
+	regs[5].name = in ? "TQSAR" : "RQSAR";
+	regs[5].offset = in ? USB_QMU_TQSAR(epnum) : USB_QMU_RQSAR(epnum);
+	regs[6].name = in ? "TQCPR" : "RQCPR";
+	regs[6].offset = in ? USB_QMU_TQCPR(epnum) : USB_QMU_RQCPR(epnum);
+
+	mtu3_debugfs_regset(mtu, mtu->mac_base, regs, 7, "ep-regs", parent);
+}
+
+static int mtu3_ep_info_show(struct seq_file *sf, void *unused)
+{
+	struct mtu3_ep *mep = sf->private;
+	struct mtu3 *mtu = mep->mtu;
+	unsigned long flags;
+
+	spin_lock_irqsave(&mtu->lock, flags);
+	seq_printf(sf, "ep - type:%d, maxp:%d, slot:%d, flags:%x\n",
+		   mep->type, mep->maxp, mep->slot, mep->flags);
+	spin_unlock_irqrestore(&mtu->lock, flags);
+
+	return 0;
+}
+
+static int mtu3_fifo_show(struct seq_file *sf, void *unused)
+{
+	struct mtu3_ep *mep = sf->private;
+	struct mtu3 *mtu = mep->mtu;
+	unsigned long flags;
+
+	spin_lock_irqsave(&mtu->lock, flags);
+	seq_printf(sf, "fifo - seg_size:%d, addr:%d, size:%d\n",
+		   mep->fifo_seg_size, mep->fifo_addr, mep->fifo_size);
+	spin_unlock_irqrestore(&mtu->lock, flags);
+
+	return 0;
+}
+
+static int mtu3_qmu_ring_show(struct seq_file *sf, void *unused)
+{
+	struct mtu3_ep *mep = sf->private;
+	struct mtu3 *mtu = mep->mtu;
+	struct mtu3_gpd_ring *ring;
+	unsigned long flags;
+
+	ring = &mep->gpd_ring;
+	spin_lock_irqsave(&mtu->lock, flags);
+	seq_printf(sf,
+		   "qmu-ring - dma:%pad, start:%p, end:%p, enq:%p, dep:%p\n",
+		   &ring->dma, ring->start, ring->end,
+		   ring->enqueue, ring->dequeue);
+	spin_unlock_irqrestore(&mtu->lock, flags);
+
+	return 0;
+}
+
+static int mtu3_qmu_gpd_show(struct seq_file *sf, void *unused)
+{
+	struct mtu3_ep *mep = sf->private;
+	struct mtu3 *mtu = mep->mtu;
+	struct mtu3_gpd_ring *ring;
+	struct qmu_gpd *gpd;
+	dma_addr_t dma;
+	unsigned long flags;
+	int i;
+
+	spin_lock_irqsave(&mtu->lock, flags);
+	ring = &mep->gpd_ring;
+	gpd = ring->start;
+	if (!gpd || !(mep->flags & MTU3_EP_ENABLED)) {
+		seq_puts(sf, "empty!\n");
+		goto out;
+	}
+
+	for (i = 0; i < MAX_GPD_NUM; i++, gpd++) {
+		dma = ring->dma + i * sizeof(*gpd);
+		seq_printf(sf, "gpd.%03d -> %pad, %p: %08x %08x %08x %08x\n",
+			   i, &dma, gpd, gpd->dw0_info, gpd->next_gpd,
+			   gpd->buffer, gpd->dw3_info);
+	}
+
+out:
+	spin_unlock_irqrestore(&mtu->lock, flags);
+
+	return 0;
+}
+
+static const struct mtu3_file_map mtu3_ep_files[] = {
+	{"ep-info", mtu3_ep_info_show, },
+	{"fifo", mtu3_fifo_show, },
+	{"qmu-ring", mtu3_qmu_ring_show, },
+	{"qmu-gpd", mtu3_qmu_gpd_show, },
+};
+
+static int mtu3_ep_open(struct inode *inode, struct file *file)
+{
+	const char *file_name = file_dentry(file)->d_iname;
+	const struct mtu3_file_map *f_map;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(mtu3_ep_files); i++) {
+		f_map = &mtu3_ep_files[i];
+
+		if (strcmp(f_map->name, file_name) == 0)
+			break;
+	}
+
+	return single_open(file, f_map->show, inode->i_private);
+}
+
+static const struct file_operations mtu3_ep_fops = {
+	.open = mtu3_ep_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+};
+
+static struct debugfs_reg32 mtu3_prb_regs[] = {
+	dump_prb_reg("enable", U3D_SSUSB_PRB_CTRL0),
+	dump_prb_reg("byte-sell", U3D_SSUSB_PRB_CTRL1),
+	dump_prb_reg("byte-selh", U3D_SSUSB_PRB_CTRL2),
+	dump_prb_reg("module-sel", U3D_SSUSB_PRB_CTRL3),
+	dump_prb_reg("sw-out", U3D_SSUSB_PRB_CTRL4),
+	dump_prb_reg("data", U3D_SSUSB_PRB_CTRL5),
+};
+
+static int mtu3_probe_show(struct seq_file *sf, void *unused)
+{
+	const char *file_name = file_dentry(sf->file)->d_iname;
+	struct mtu3 *mtu = sf->private;
+	const struct debugfs_reg32 *regs;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(mtu3_prb_regs); i++) {
+		regs = &mtu3_prb_regs[i];
+
+		if (strcmp(regs->name, file_name) == 0)
+			break;
+	}
+
+	seq_printf(sf, "0x%04x - 0x%08x\n", (u32)regs->offset,
+		   mtu3_readl(mtu->ippc_base, (u32)regs->offset));
+
+	return 0;
+}
+
+static int mtu3_probe_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, mtu3_probe_show, inode->i_private);
+}
+
+static ssize_t mtu3_probe_write(struct file *file, const char __user *ubuf,
+				size_t count, loff_t *ppos)
+{
+	const char *file_name = file_dentry(file)->d_iname;
+	struct seq_file *sf = file->private_data;
+	struct mtu3 *mtu = sf->private;
+	const struct debugfs_reg32 *regs;
+	char buf[32];
+	u32 val;
+	int i;
+
+	if (copy_from_user(&buf, ubuf, min_t(size_t, sizeof(buf) - 1, count)))
+		return -EFAULT;
+
+	if (kstrtou32(buf, 0, &val))
+		return -EINVAL;
+
+	for (i = 0; i < ARRAY_SIZE(mtu3_prb_regs); i++) {
+		regs = &mtu3_prb_regs[i];
+
+		if (strcmp(regs->name, file_name) == 0)
+			break;
+	}
+	mtu3_writel(mtu->ippc_base, (u32)regs->offset, val);
+
+	return count;
+}
+
+static const struct file_operations mtu3_probe_fops = {
+	.open = mtu3_probe_open,
+	.write = mtu3_probe_write,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+};
+
+static void mtu3_debugfs_create_prb_files(struct mtu3 *mtu)
+{
+	struct ssusb_mtk *ssusb = mtu->ssusb;
+	struct debugfs_reg32 *regs;
+	struct dentry *dir_prb;
+	int i;
+
+	dir_prb = debugfs_create_dir("probe", ssusb->dbgfs_root);
+
+	for (i = 0; i < ARRAY_SIZE(mtu3_prb_regs); i++) {
+		regs = &mtu3_prb_regs[i];
+		debugfs_create_file(regs->name, 0644, dir_prb,
+				    mtu, &mtu3_probe_fops);
+	}
+
+	mtu3_debugfs_regset(mtu, mtu->ippc_base, mtu3_prb_regs,
+			    ARRAY_SIZE(mtu3_prb_regs), "regs", dir_prb);
+}
+
+static void mtu3_debugfs_create_ep_dir(struct mtu3_ep *mep,
+				       struct dentry *parent)
+{
+	const struct mtu3_file_map *files;
+	struct dentry *dir_ep;
+	int i;
+
+	dir_ep = debugfs_create_dir(mep->name, parent);
+	mtu3_debugfs_ep_regset(mep->mtu, mep, dir_ep);
+
+	for (i = 0; i < ARRAY_SIZE(mtu3_ep_files); i++) {
+		files = &mtu3_ep_files[i];
+
+		debugfs_create_file(files->name, 0444, dir_ep,
+				    mep, &mtu3_ep_fops);
+	}
+}
+
+static void mtu3_debugfs_create_ep_dirs(struct mtu3 *mtu)
+{
+	struct ssusb_mtk *ssusb = mtu->ssusb;
+	struct dentry *dir_eps;
+	int i;
+
+	dir_eps = debugfs_create_dir("eps", ssusb->dbgfs_root);
+
+	for (i = 1; i < mtu->num_eps; i++) {
+		mtu3_debugfs_create_ep_dir(mtu->in_eps + i, dir_eps);
+		mtu3_debugfs_create_ep_dir(mtu->out_eps + i, dir_eps);
+	}
+}
+
+void ssusb_dev_debugfs_init(struct ssusb_mtk *ssusb)
+{
+	struct mtu3 *mtu = ssusb->u3d;
+	struct dentry *dir_regs;
+
+	dir_regs = debugfs_create_dir("regs", ssusb->dbgfs_root);
+
+	mtu3_debugfs_regset(mtu, mtu->ippc_base,
+			    mtu3_ippc_regs, ARRAY_SIZE(mtu3_ippc_regs),
+			    "reg-ippc", dir_regs);
+
+	mtu3_debugfs_regset(mtu, mtu->mac_base,
+			    mtu3_dev_regs, ARRAY_SIZE(mtu3_dev_regs),
+			    "reg-dev", dir_regs);
+
+	mtu3_debugfs_regset(mtu, mtu->mac_base,
+			    mtu3_csr_regs, ARRAY_SIZE(mtu3_csr_regs),
+			    "reg-csr", dir_regs);
+
+	mtu3_debugfs_create_ep_dirs(mtu);
+
+	mtu3_debugfs_create_prb_files(mtu);
+
+	debugfs_create_file("link-state", 0444, ssusb->dbgfs_root,
+			    mtu, &mtu3_link_state_fops);
+	debugfs_create_file("ep-used", 0444, ssusb->dbgfs_root,
+			    mtu, &mtu3_ep_used_fops);
+}
+
+static int ssusb_mode_show(struct seq_file *sf, void *unused)
+{
+	struct ssusb_mtk *ssusb = sf->private;
+
+	seq_printf(sf, "current mode: %s(%s drd)\n(echo device/host)\n",
+		   ssusb->is_host ? "host" : "device",
+		   ssusb->otg_switch.manual_drd_enabled ? "manual" : "auto");
+
+	return 0;
+}
+
+static int ssusb_mode_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, ssusb_mode_show, inode->i_private);
+}
+
+static ssize_t ssusb_mode_write(struct file *file, const char __user *ubuf,
+				size_t count, loff_t *ppos)
+{
+	struct seq_file *sf = file->private_data;
+	struct ssusb_mtk *ssusb = sf->private;
+	char buf[16];
+
+	if (copy_from_user(&buf, ubuf, min_t(size_t, sizeof(buf) - 1, count)))
+		return -EFAULT;
+
+	if (!strncmp(buf, "host", 4) && !ssusb->is_host) {
+		ssusb_mode_manual_switch(ssusb, 1);
+	} else if (!strncmp(buf, "device", 6) && ssusb->is_host) {
+		ssusb_mode_manual_switch(ssusb, 0);
+	} else {
+		dev_err(ssusb->dev, "wrong or duplicated setting\n");
+		return -EINVAL;
+	}
+
+	return count;
+}
+
+static const struct file_operations ssusb_mode_fops = {
+	.open = ssusb_mode_open,
+	.write = ssusb_mode_write,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+};
+
+static int ssusb_vbus_show(struct seq_file *sf, void *unused)
+{
+	struct ssusb_mtk *ssusb = sf->private;
+	struct otg_switch_mtk *otg_sx = &ssusb->otg_switch;
+
+	seq_printf(sf, "vbus state: %s\n(echo on/off)\n",
+		   regulator_is_enabled(otg_sx->vbus) ? "on" : "off");
+
+	return 0;
+}
+
+static int ssusb_vbus_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, ssusb_vbus_show, inode->i_private);
+}
+
+static ssize_t ssusb_vbus_write(struct file *file, const char __user *ubuf,
+				size_t count, loff_t *ppos)
+{
+	struct seq_file *sf = file->private_data;
+	struct ssusb_mtk *ssusb = sf->private;
+	struct otg_switch_mtk *otg_sx = &ssusb->otg_switch;
+	char buf[16];
+	bool enable;
+
+	if (copy_from_user(&buf, ubuf, min_t(size_t, sizeof(buf) - 1, count)))
+		return -EFAULT;
+
+	if (kstrtobool(buf, &enable)) {
+		dev_err(ssusb->dev, "wrong setting\n");
+		return -EINVAL;
+	}
+
+	ssusb_set_vbus(otg_sx, enable);
+
+	return count;
+}
+
+static const struct file_operations ssusb_vbus_fops = {
+	.open = ssusb_vbus_open,
+	.write = ssusb_vbus_write,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+};
+
+void ssusb_dr_debugfs_init(struct ssusb_mtk *ssusb)
+{
+	struct dentry *root = ssusb->dbgfs_root;
+
+	debugfs_create_file("mode", 0644, root, ssusb, &ssusb_mode_fops);
+	debugfs_create_file("vbus", 0644, root, ssusb, &ssusb_vbus_fops);
+}
+
+void ssusb_debugfs_create_root(struct ssusb_mtk *ssusb)
+{
+	ssusb->dbgfs_root =
+		debugfs_create_dir(dev_name(ssusb->dev), usb_debug_root);
+}
+
+void ssusb_debugfs_remove_root(struct ssusb_mtk *ssusb)
+{
+	debugfs_remove_recursive(ssusb->dbgfs_root);
+	ssusb->dbgfs_root = NULL;
+}
diff --git a/drivers/usb/mtu3/mtu3_dr.c b/drivers/usb/mtu3/mtu3_dr.c
index ac60e9c8564e..5fcb71af875a 100644
--- a/drivers/usb/mtu3/mtu3_dr.c
+++ b/drivers/usb/mtu3/mtu3_dr.c
@@ -7,16 +7,9 @@
  * Author: Chunfeng Yun <chunfeng.yun@mediatek.com>
  */
 
-#include <linux/debugfs.h>
-#include <linux/irq.h>
-#include <linux/kernel.h>
-#include <linux/of_device.h>
-#include <linux/pinctrl/consumer.h>
-#include <linux/seq_file.h>
-#include <linux/uaccess.h>
-
 #include "mtu3.h"
 #include "mtu3_dr.h"
+#include "mtu3_debug.h"
 
 #define USB2_PORT 2
 #define USB3_PORT 3
@@ -28,6 +21,22 @@ enum mtu3_vbus_id_state {
 	MTU3_VBUS_VALID,
 };
 
+static char *mailbox_state_string(enum mtu3_vbus_id_state state)
+{
+	switch (state) {
+	case MTU3_ID_FLOAT:
+		return "ID_FLOAT";
+	case MTU3_ID_GROUND:
+		return "ID_GROUND";
+	case MTU3_VBUS_OFF:
+		return "VBUS_OFF";
+	case MTU3_VBUS_VALID:
+		return "VBUS_VALID";
+	default:
+		return "UNKNOWN";
+	}
+}
+
 static void toggle_opstate(struct ssusb_mtk *ssusb)
 {
 	if (!ssusb->otg_switch.is_u3_drd) {
@@ -147,7 +156,8 @@ static void ssusb_set_mailbox(struct otg_switch_mtk *otg_sx,
 		container_of(otg_sx, struct ssusb_mtk, otg_switch);
 	struct mtu3 *mtu = ssusb->u3d;
 
-	dev_dbg(ssusb->dev, "mailbox state(%d)\n", status);
+	dev_dbg(ssusb->dev, "mailbox %s\n", mailbox_state_string(status));
+	mtu3_dbg_trace(ssusb->dev, "mailbox %s", mailbox_state_string(status));
 
 	switch (status) {
 	case MTU3_ID_GROUND:
@@ -238,14 +248,18 @@ static int ssusb_extcon_register(struct otg_switch_mtk *otg_sx)
 	otg_sx->vbus_nb.notifier_call = ssusb_vbus_notifier;
 	ret = devm_extcon_register_notifier(ssusb->dev, edev, EXTCON_USB,
 					&otg_sx->vbus_nb);
-	if (ret < 0)
+	if (ret < 0) {
 		dev_err(ssusb->dev, "failed to register notifier for USB\n");
+		return ret;
+	}
 
 	otg_sx->id_nb.notifier_call = ssusb_id_notifier;
 	ret = devm_extcon_register_notifier(ssusb->dev, edev, EXTCON_USB_HOST,
 					&otg_sx->id_nb);
-	if (ret < 0)
+	if (ret < 0) {
 		dev_err(ssusb->dev, "failed to register notifier for USB-HOST\n");
+		return ret;
+	}
 
 	dev_dbg(ssusb->dev, "EXTCON_USB: %d, EXTCON_USB_HOST: %d\n",
 		extcon_get_state(edev, EXTCON_USB),
@@ -266,7 +280,7 @@ static int ssusb_extcon_register(struct otg_switch_mtk *otg_sx)
  * This is useful in special cases, such as uses TYPE-A receptacle but also
  * wants to support dual-role mode.
  */
-static void ssusb_mode_manual_switch(struct ssusb_mtk *ssusb, int to_host)
+void ssusb_mode_manual_switch(struct ssusb_mtk *ssusb, int to_host)
 {
 	struct otg_switch_mtk *otg_sx = &ssusb->otg_switch;
 
@@ -281,114 +295,6 @@ static void ssusb_mode_manual_switch(struct ssusb_mtk *ssusb, int to_host)
 	}
 }
 
-static int ssusb_mode_show(struct seq_file *sf, void *unused)
-{
-	struct ssusb_mtk *ssusb = sf->private;
-
-	seq_printf(sf, "current mode: %s(%s drd)\n(echo device/host)\n",
-		ssusb->is_host ? "host" : "device",
-		ssusb->otg_switch.manual_drd_enabled ? "manual" : "auto");
-
-	return 0;
-}
-
-static int ssusb_mode_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, ssusb_mode_show, inode->i_private);
-}
-
-static ssize_t ssusb_mode_write(struct file *file,
-	const char __user *ubuf, size_t count, loff_t *ppos)
-{
-	struct seq_file *sf = file->private_data;
-	struct ssusb_mtk *ssusb = sf->private;
-	char buf[16];
-
-	if (copy_from_user(&buf, ubuf, min_t(size_t, sizeof(buf) - 1, count)))
-		return -EFAULT;
-
-	if (!strncmp(buf, "host", 4) && !ssusb->is_host) {
-		ssusb_mode_manual_switch(ssusb, 1);
-	} else if (!strncmp(buf, "device", 6) && ssusb->is_host) {
-		ssusb_mode_manual_switch(ssusb, 0);
-	} else {
-		dev_err(ssusb->dev, "wrong or duplicated setting\n");
-		return -EINVAL;
-	}
-
-	return count;
-}
-
-static const struct file_operations ssusb_mode_fops = {
-	.open = ssusb_mode_open,
-	.write = ssusb_mode_write,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-};
-
-static int ssusb_vbus_show(struct seq_file *sf, void *unused)
-{
-	struct ssusb_mtk *ssusb = sf->private;
-	struct otg_switch_mtk *otg_sx = &ssusb->otg_switch;
-
-	seq_printf(sf, "vbus state: %s\n(echo on/off)\n",
-		regulator_is_enabled(otg_sx->vbus) ? "on" : "off");
-
-	return 0;
-}
-
-static int ssusb_vbus_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, ssusb_vbus_show, inode->i_private);
-}
-
-static ssize_t ssusb_vbus_write(struct file *file,
-	const char __user *ubuf, size_t count, loff_t *ppos)
-{
-	struct seq_file *sf = file->private_data;
-	struct ssusb_mtk *ssusb = sf->private;
-	struct otg_switch_mtk *otg_sx = &ssusb->otg_switch;
-	char buf[16];
-	bool enable;
-
-	if (copy_from_user(&buf, ubuf, min_t(size_t, sizeof(buf) - 1, count)))
-		return -EFAULT;
-
-	if (kstrtobool(buf, &enable)) {
-		dev_err(ssusb->dev, "wrong setting\n");
-		return -EINVAL;
-	}
-
-	ssusb_set_vbus(otg_sx, enable);
-
-	return count;
-}
-
-static const struct file_operations ssusb_vbus_fops = {
-	.open = ssusb_vbus_open,
-	.write = ssusb_vbus_write,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-};
-
-static void ssusb_debugfs_init(struct ssusb_mtk *ssusb)
-{
-	struct dentry *root;
-
-	root = debugfs_create_dir(dev_name(ssusb->dev), usb_debug_root);
-	ssusb->dbgfs_root = root;
-
-	debugfs_create_file("mode", 0644, root, ssusb, &ssusb_mode_fops);
-	debugfs_create_file("vbus", 0644, root, ssusb, &ssusb_vbus_fops);
-}
-
-static void ssusb_debugfs_exit(struct ssusb_mtk *ssusb)
-{
-	debugfs_remove_recursive(ssusb->dbgfs_root);
-}
-
 void ssusb_set_force_mode(struct ssusb_mtk *ssusb,
 			  enum mtu3_dr_force_mode mode)
 {
@@ -415,25 +321,23 @@ void ssusb_set_force_mode(struct ssusb_mtk *ssusb,
 int ssusb_otg_switch_init(struct ssusb_mtk *ssusb)
 {
 	struct otg_switch_mtk *otg_sx = &ssusb->otg_switch;
+	int ret = 0;
 
 	INIT_WORK(&otg_sx->id_work, ssusb_id_work);
 	INIT_WORK(&otg_sx->vbus_work, ssusb_vbus_work);
 
 	if (otg_sx->manual_drd_enabled)
-		ssusb_debugfs_init(ssusb);
+		ssusb_dr_debugfs_init(ssusb);
 	else
-		ssusb_extcon_register(otg_sx);
+		ret = ssusb_extcon_register(otg_sx);
 
-	return 0;
+	return ret;
 }
 
 void ssusb_otg_switch_exit(struct ssusb_mtk *ssusb)
 {
 	struct otg_switch_mtk *otg_sx = &ssusb->otg_switch;
 
-	if (otg_sx->manual_drd_enabled)
-		ssusb_debugfs_exit(ssusb);
-
 	cancel_work_sync(&otg_sx->id_work);
 	cancel_work_sync(&otg_sx->vbus_work);
 }
diff --git a/drivers/usb/mtu3/mtu3_dr.h b/drivers/usb/mtu3/mtu3_dr.h
index 50702fdcde28..ba6fe357ce29 100644
--- a/drivers/usb/mtu3/mtu3_dr.h
+++ b/drivers/usb/mtu3/mtu3_dr.h
@@ -71,6 +71,7 @@ static inline void ssusb_gadget_exit(struct ssusb_mtk *ssusb)
 #if IS_ENABLED(CONFIG_USB_MTU3_DUAL_ROLE)
 int ssusb_otg_switch_init(struct ssusb_mtk *ssusb);
 void ssusb_otg_switch_exit(struct ssusb_mtk *ssusb);
+void ssusb_mode_manual_switch(struct ssusb_mtk *ssusb, int to_host);
 int ssusb_set_vbus(struct otg_switch_mtk *otg_sx, int is_on);
 void ssusb_set_force_mode(struct ssusb_mtk *ssusb,
 			  enum mtu3_dr_force_mode mode);
@@ -85,6 +86,9 @@ static inline int ssusb_otg_switch_init(struct ssusb_mtk *ssusb)
 static inline void ssusb_otg_switch_exit(struct ssusb_mtk *ssusb)
 {}
 
+static inline void
+ssusb_mode_manual_switch(struct ssusb_mtk *ssusb, int to_host) {}
+
 static inline int ssusb_set_vbus(struct otg_switch_mtk *otg_sx, int is_on)
 {
 	return 0;
diff --git a/drivers/usb/mtu3/mtu3_gadget.c b/drivers/usb/mtu3/mtu3_gadget.c
index bbcd3332471d..f93732e53fd8 100644
--- a/drivers/usb/mtu3/mtu3_gadget.c
+++ b/drivers/usb/mtu3/mtu3_gadget.c
@@ -8,6 +8,7 @@
  */
 
 #include "mtu3.h"
+#include "mtu3_trace.h"
 
 void mtu3_req_complete(struct mtu3_ep *mep,
 		     struct usb_request *req, int status)
@@ -25,6 +26,8 @@ __acquires(mep->mtu->lock)
 
 	mtu = mreq->mtu;
 	mep->busy = 1;
+
+	trace_mtu3_req_complete(mreq);
 	spin_unlock(&mtu->lock);
 
 	/* ep0 makes use of PIO, needn't unmap it */
@@ -201,6 +204,7 @@ error:
 	spin_unlock_irqrestore(&mtu->lock, flags);
 
 	dev_dbg(mtu->dev, "%s active_ep=%d\n", __func__, mtu->active_ep);
+	trace_mtu3_gadget_ep_enable(mep);
 
 	return ret;
 }
@@ -212,6 +216,7 @@ static int mtu3_gadget_ep_disable(struct usb_ep *ep)
 	unsigned long flags;
 
 	dev_dbg(mtu->dev, "%s %s\n", __func__, mep->name);
+	trace_mtu3_gadget_ep_disable(mep);
 
 	if (!(mep->flags & MTU3_EP_ENABLED)) {
 		dev_warn(mtu->dev, "%s is already disabled\n", mep->name);
@@ -242,13 +247,17 @@ struct usb_request *mtu3_alloc_request(struct usb_ep *ep, gfp_t gfp_flags)
 	mreq->request.dma = DMA_ADDR_INVALID;
 	mreq->epnum = mep->epnum;
 	mreq->mep = mep;
+	trace_mtu3_alloc_request(mreq);
 
 	return &mreq->request;
 }
 
 void mtu3_free_request(struct usb_ep *ep, struct usb_request *req)
 {
-	kfree(to_mtu3_request(req));
+	struct mtu3_request *mreq = to_mtu3_request(req);
+
+	trace_mtu3_free_request(mreq);
+	kfree(mreq);
 }
 
 static int mtu3_gadget_queue(struct usb_ep *ep,
@@ -278,10 +287,12 @@ static int mtu3_gadget_queue(struct usb_ep *ep,
 		__func__, mep->is_in ? "TX" : "RX", mreq->epnum, ep->name,
 		mreq, ep->maxpacket, mreq->request.length);
 
-	if (req->length > GPD_BUF_SIZE) {
+	if (req->length > GPD_BUF_SIZE ||
+	    (mtu->gen2cp && req->length > GPD_BUF_SIZE_EL)) {
 		dev_warn(mtu->dev,
 			"req length > supported MAX:%d requested:%d\n",
-			GPD_BUF_SIZE, req->length);
+			mtu->gen2cp ? GPD_BUF_SIZE_EL : GPD_BUF_SIZE,
+			req->length);
 		return -EOPNOTSUPP;
 	}
 
@@ -314,6 +325,7 @@ static int mtu3_gadget_queue(struct usb_ep *ep,
 
 error:
 	spin_unlock_irqrestore(&mtu->lock, flags);
+	trace_mtu3_gadget_queue(mreq);
 
 	return ret;
 }
@@ -331,6 +343,7 @@ static int mtu3_gadget_dequeue(struct usb_ep *ep, struct usb_request *req)
 		return -EINVAL;
 
 	dev_dbg(mtu->dev, "%s : req=%p\n", __func__, req);
+	trace_mtu3_gadget_dequeue(mreq);
 
 	spin_lock_irqsave(&mtu->lock, flags);
 
@@ -401,6 +414,7 @@ static int mtu3_gadget_ep_set_halt(struct usb_ep *ep, int value)
 
 done:
 	spin_unlock_irqrestore(&mtu->lock, flags);
+	trace_mtu3_gadget_ep_set_halt(mep);
 
 	return ret;
 }
diff --git a/drivers/usb/mtu3/mtu3_gadget_ep0.c b/drivers/usb/mtu3/mtu3_gadget_ep0.c
index 7cb7ac980446..4da216c99726 100644
--- a/drivers/usb/mtu3/mtu3_gadget_ep0.c
+++ b/drivers/usb/mtu3/mtu3_gadget_ep0.c
@@ -11,6 +11,8 @@
 #include <linux/usb/composite.h>
 
 #include "mtu3.h"
+#include "mtu3_debug.h"
+#include "mtu3_trace.h"
 
 /* ep0 is always mtu3->in_eps[0] */
 #define	next_ep0_request(mtu)	next_request((mtu)->ep0)
@@ -634,6 +636,7 @@ __acquires(mtu->lock)
 	int handled = 0;
 
 	ep0_read_setup(mtu, &setup);
+	trace_mtu3_handle_setup(&setup);
 
 	if ((setup.bRequestType & USB_TYPE_MASK) == USB_TYPE_STANDARD)
 		handled = handle_standard_request(mtu, &setup);
@@ -710,6 +713,7 @@ irqreturn_t mtu3_ep0_isr(struct mtu3 *mtu)
 		ret = IRQ_HANDLED;
 	}
 	dev_dbg(mtu->dev, "ep0_state: %s\n", decode_ep0_state(mtu));
+	mtu3_dbg_trace(mtu->dev, "ep0_state %s", decode_ep0_state(mtu));
 
 	switch (mtu->ep0_state) {
 	case MU3D_EP0_STATE_TX:
diff --git a/drivers/usb/mtu3/mtu3_hw_regs.h b/drivers/usb/mtu3/mtu3_hw_regs.h
index 1d65b7476f23..8382d066749e 100644
--- a/drivers/usb/mtu3/mtu3_hw_regs.h
+++ b/drivers/usb/mtu3/mtu3_hw_regs.h
@@ -49,6 +49,7 @@
 #define U3D_QCR1		(SSUSB_DEV_BASE + 0x0404)
 #define U3D_QCR2		(SSUSB_DEV_BASE + 0x0408)
 #define U3D_QCR3		(SSUSB_DEV_BASE + 0x040C)
+#define U3D_QFCR		(SSUSB_DEV_BASE + 0x0428)
 #define U3D_TXQHIAR1		(SSUSB_DEV_BASE + 0x0484)
 #define U3D_RXQHIAR1		(SSUSB_DEV_BASE + 0x04C4)
 
@@ -133,11 +134,23 @@
 #define TX_W1C_BITS		(~(TX_SENTSTALL))
 
 /* U3D_TX1CSR1 */
-#define TX_MULT(x)		(((x) & 0x3) << 22)
-#define TX_MAX_PKT(x)		(((x) & 0x3f) << 16)
+#define TX_MAX_PKT_G2(x)	(((x) & 0x7f) << 24)
+#define TX_MULT_G2(x)		(((x) & 0x7) << 21)
+#define TX_MULT_OG(x)		(((x) & 0x3) << 22)
+#define TX_MAX_PKT_OG(x)	(((x) & 0x3f) << 16)
 #define TX_SLOT(x)		(((x) & 0x3f) << 8)
 #define TX_TYPE(x)		(((x) & 0x3) << 4)
 #define TX_SS_BURST(x)		(((x) & 0xf) << 0)
+#define TX_MULT(g2c, x)		\
+({				\
+	typeof(x) x_ = (x);	\
+	(g2c) ? TX_MULT_G2(x_) : TX_MULT_OG(x_);	\
+})
+#define TX_MAX_PKT(g2c, x)	\
+({				\
+	typeof(x) x_ = (x);	\
+	(g2c) ? TX_MAX_PKT_G2(x_) : TX_MAX_PKT_OG(x_);	\
+})
 
 /* for TX_TYPE & RX_TYPE */
 #define TYPE_BULK		(0x0)
@@ -160,11 +173,23 @@
 #define RX_W1C_BITS		(~(RX_SENTSTALL | RX_RXPKTRDY))
 
 /* U3D_RX1CSR1 */
-#define RX_MULT(x)		(((x) & 0x3) << 22)
-#define RX_MAX_PKT(x)		(((x) & 0x3f) << 16)
+#define RX_MAX_PKT_G2(x)	(((x) & 0x7f) << 24)
+#define RX_MULT_G2(x)		(((x) & 0x7) << 21)
+#define RX_MULT_OG(x)		(((x) & 0x3) << 22)
+#define RX_MAX_PKT_OG(x)	(((x) & 0x3f) << 16)
 #define RX_SLOT(x)		(((x) & 0x3f) << 8)
 #define RX_TYPE(x)		(((x) & 0x3) << 4)
 #define RX_SS_BURST(x)		(((x) & 0xf) << 0)
+#define RX_MULT(g2c, x)		\
+({				\
+	typeof(x) x_ = (x);	\
+	(g2c) ? RX_MULT_G2(x_) : RX_MULT_OG(x_);	\
+})
+#define RX_MAX_PKT(g2c, x)	\
+({				\
+	typeof(x) x_ = (x);	\
+	(g2c) ? RX_MAX_PKT_G2(x_) : RX_MAX_PKT_OG(x_);	\
+})
 
 /* U3D_RX1CSR2 */
 #define RX_BINTERVAL(x)		(((x) & 0xff) << 24)
@@ -265,6 +290,7 @@
 #define U3D_LTSSM_CTRL		(SSUSB_USB3_MAC_CSR_BASE + 0x0010)
 #define U3D_USB3_CONFIG		(SSUSB_USB3_MAC_CSR_BASE + 0x001C)
 
+#define U3D_LINK_STATE_MACHINE	(SSUSB_USB3_MAC_CSR_BASE + 0x0134)
 #define U3D_LTSSM_INTR_ENABLE	(SSUSB_USB3_MAC_CSR_BASE + 0x013C)
 #define U3D_LTSSM_INTR		(SSUSB_USB3_MAC_CSR_BASE + 0x0140)
 
@@ -282,6 +308,9 @@
 /* U3D_USB3_CONFIG */
 #define USB3_EN			BIT(0)
 
+/* U3D_LINK_STATE_MACHINE */
+#define LTSSM_STATE(x)	((x) & 0x1f)
+
 /* U3D_LTSSM_INTR_ENABLE */
 /* U3D_LTSSM_INTR */
 #define U3_RESUME_INTR		BIT(18)
@@ -347,6 +376,7 @@
 #define U3D_USB20_FRAME_NUM		(SSUSB_USB2_CSR_BASE + 0x003C)
 #define U3D_USB20_LPM_PARAMETER		(SSUSB_USB2_CSR_BASE + 0x0044)
 #define U3D_USB20_MISC_CONTROL		(SSUSB_USB2_CSR_BASE + 0x004C)
+#define U3D_USB20_OPSTATE		(SSUSB_USB2_CSR_BASE + 0x0060)
 
 /*---------------- SSUSB_USB2_CSR FIELD DEFINITION ----------------*/
 
@@ -419,6 +449,13 @@
 #define U3D_SSUSB_DEV_RST_CTRL	(SSUSB_SIFSLV_IPPC_BASE + 0x0098)
 #define U3D_SSUSB_HW_ID		(SSUSB_SIFSLV_IPPC_BASE + 0x00A0)
 #define U3D_SSUSB_HW_SUB_ID	(SSUSB_SIFSLV_IPPC_BASE + 0x00A4)
+#define U3D_SSUSB_IP_TRUNK_VERS	(U3D_SSUSB_HW_SUB_ID)
+#define U3D_SSUSB_PRB_CTRL0	(SSUSB_SIFSLV_IPPC_BASE + 0x00B0)
+#define U3D_SSUSB_PRB_CTRL1	(SSUSB_SIFSLV_IPPC_BASE + 0x00B4)
+#define U3D_SSUSB_PRB_CTRL2	(SSUSB_SIFSLV_IPPC_BASE + 0x00B8)
+#define U3D_SSUSB_PRB_CTRL3	(SSUSB_SIFSLV_IPPC_BASE + 0x00BC)
+#define U3D_SSUSB_PRB_CTRL4	(SSUSB_SIFSLV_IPPC_BASE + 0x00C0)
+#define U3D_SSUSB_PRB_CTRL5	(SSUSB_SIFSLV_IPPC_BASE + 0x00C4)
 #define U3D_SSUSB_IP_SPARE0	(SSUSB_SIFSLV_IPPC_BASE + 0x00C8)
 
 /*---------------- SSUSB_SIFSLV_IPPC FIELD DEFINITION ----------------*/
@@ -483,4 +520,7 @@
 /* U3D_SSUSB_DEV_RST_CTRL */
 #define SSUSB_DEV_SW_RST		BIT(0)
 
+/* U3D_SSUSB_IP_TRUNK_VERS */
+#define IP_TRUNK_VERS(x)		(((x) >> 16) & 0xffff)
+
 #endif	/* _SSUSB_HW_REGS_H_ */
diff --git a/drivers/usb/mtu3/mtu3_plat.c b/drivers/usb/mtu3/mtu3_plat.c
index e086630e41a9..fd0f6c5dfbc1 100644
--- a/drivers/usb/mtu3/mtu3_plat.c
+++ b/drivers/usb/mtu3/mtu3_plat.c
@@ -16,6 +16,7 @@
 
 #include "mtu3.h"
 #include "mtu3_dr.h"
+#include "mtu3_debug.h"
 
 /* u2-port0 should be powered on and enabled; */
 int ssusb_check_clocks(struct ssusb_mtk *ssusb, u32 ex_clks)
@@ -210,30 +211,16 @@ static void ssusb_ip_sw_reset(struct ssusb_mtk *ssusb)
 	mtu3_setbits(ssusb->ippc_base, U3D_SSUSB_IP_PW_CTRL2, SSUSB_IP_DEV_PDN);
 }
 
-/* ignore the error if the clock does not exist */
-static struct clk *get_optional_clk(struct device *dev, const char *id)
-{
-	struct clk *opt_clk;
-
-	opt_clk = devm_clk_get(dev, id);
-	/* ignore error number except EPROBE_DEFER */
-	if (IS_ERR(opt_clk) && (PTR_ERR(opt_clk) != -EPROBE_DEFER))
-		opt_clk = NULL;
-
-	return opt_clk;
-}
-
 static int get_ssusb_rscs(struct platform_device *pdev, struct ssusb_mtk *ssusb)
 {
 	struct device_node *node = pdev->dev.of_node;
 	struct otg_switch_mtk *otg_sx = &ssusb->otg_switch;
 	struct device *dev = &pdev->dev;
-	struct regulator *vbus;
 	struct resource *res;
 	int i;
 	int ret;
 
-	ssusb->vusb33 = devm_regulator_get(&pdev->dev, "vusb33");
+	ssusb->vusb33 = devm_regulator_get(dev, "vusb33");
 	if (IS_ERR(ssusb->vusb33)) {
 		dev_err(dev, "failed to get vusb33\n");
 		return PTR_ERR(ssusb->vusb33);
@@ -245,15 +232,15 @@ static int get_ssusb_rscs(struct platform_device *pdev, struct ssusb_mtk *ssusb)
 		return PTR_ERR(ssusb->sys_clk);
 	}
 
-	ssusb->ref_clk = get_optional_clk(dev, "ref_ck");
+	ssusb->ref_clk = devm_clk_get_optional(dev, "ref_ck");
 	if (IS_ERR(ssusb->ref_clk))
 		return PTR_ERR(ssusb->ref_clk);
 
-	ssusb->mcu_clk = get_optional_clk(dev, "mcu_ck");
+	ssusb->mcu_clk = devm_clk_get_optional(dev, "mcu_ck");
 	if (IS_ERR(ssusb->mcu_clk))
 		return PTR_ERR(ssusb->mcu_clk);
 
-	ssusb->dma_clk = get_optional_clk(dev, "dma_ck");
+	ssusb->dma_clk = devm_clk_get_optional(dev, "dma_ck");
 	if (IS_ERR(ssusb->dma_clk))
 		return PTR_ERR(ssusb->dma_clk);
 
@@ -286,7 +273,7 @@ static int get_ssusb_rscs(struct platform_device *pdev, struct ssusb_mtk *ssusb)
 		ssusb->dr_mode = USB_DR_MODE_OTG;
 
 	if (ssusb->dr_mode == USB_DR_MODE_PERIPHERAL)
-		return 0;
+		goto out;
 
 	/* if host role is supported */
 	ret = ssusb_wakeup_of_property_parse(ssusb, node);
@@ -299,15 +286,14 @@ static int get_ssusb_rscs(struct platform_device *pdev, struct ssusb_mtk *ssusb)
 	of_property_read_u32(node, "mediatek,u3p-dis-msk",
 			     &ssusb->u3p_dis_msk);
 
-	vbus = devm_regulator_get(&pdev->dev, "vbus");
-	if (IS_ERR(vbus)) {
+	otg_sx->vbus = devm_regulator_get(dev, "vbus");
+	if (IS_ERR(otg_sx->vbus)) {
 		dev_err(dev, "failed to get vbus\n");
-		return PTR_ERR(vbus);
+		return PTR_ERR(otg_sx->vbus);
 	}
-	otg_sx->vbus = vbus;
 
 	if (ssusb->dr_mode == USB_DR_MODE_HOST)
-		return 0;
+		goto out;
 
 	/* if dual-role mode is supported */
 	otg_sx->is_u3_drd = of_property_read_bool(node, "mediatek,usb3-drd");
@@ -322,6 +308,7 @@ static int get_ssusb_rscs(struct platform_device *pdev, struct ssusb_mtk *ssusb)
 		}
 	}
 
+out:
 	dev_info(dev, "dr_mode: %d, is_u3_dr: %d, u3p_dis_msk: %x, drd: %s\n",
 		ssusb->dr_mode, otg_sx->is_u3_drd, ssusb->u3p_dis_msk,
 		otg_sx->manual_drd_enabled ? "manual" : "auto");
@@ -354,6 +341,8 @@ static int mtu3_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
+	ssusb_debugfs_create_root(ssusb);
+
 	/* enable power domain */
 	pm_runtime_enable(dev);
 	pm_runtime_get_sync(dev);
@@ -401,7 +390,11 @@ static int mtu3_probe(struct platform_device *pdev)
 			goto gadget_exit;
 		}
 
-		ssusb_otg_switch_init(ssusb);
+		ret = ssusb_otg_switch_init(ssusb);
+		if (ret) {
+			dev_err(dev, "failed to initialize switch\n");
+			goto host_exit;
+		}
 		break;
 	default:
 		dev_err(dev, "unsupported mode: %d\n", ssusb->dr_mode);
@@ -411,6 +404,8 @@ static int mtu3_probe(struct platform_device *pdev)
 
 	return 0;
 
+host_exit:
+	ssusb_host_exit(ssusb);
 gadget_exit:
 	ssusb_gadget_exit(ssusb);
 comm_exit:
@@ -418,6 +413,7 @@ comm_exit:
 comm_init_err:
 	pm_runtime_put_sync(dev);
 	pm_runtime_disable(dev);
+	ssusb_debugfs_remove_root(ssusb);
 
 	return ret;
 }
@@ -445,6 +441,7 @@ static int mtu3_remove(struct platform_device *pdev)
 	ssusb_rscs_exit(ssusb);
 	pm_runtime_put_sync(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
+	ssusb_debugfs_remove_root(ssusb);
 
 	return 0;
 }
diff --git a/drivers/usb/mtu3/mtu3_qmu.c b/drivers/usb/mtu3/mtu3_qmu.c
index 09f19f70fe8f..3f414f91b589 100644
--- a/drivers/usb/mtu3/mtu3_qmu.c
+++ b/drivers/usb/mtu3/mtu3_qmu.c
@@ -22,17 +22,49 @@
 #include <linux/iopoll.h>
 
 #include "mtu3.h"
+#include "mtu3_trace.h"
 
 #define QMU_CHECKSUM_LEN	16
 
 #define GPD_FLAGS_HWO	BIT(0)
 #define GPD_FLAGS_BDP	BIT(1)
 #define GPD_FLAGS_BPS	BIT(2)
+#define GPD_FLAGS_ZLP	BIT(6)
 #define GPD_FLAGS_IOC	BIT(7)
-
-#define GPD_EXT_FLAG_ZLP	BIT(5)
-#define GPD_EXT_NGP(x)		(((x) & 0xf) << 4)
-#define GPD_EXT_BUF(x)		(((x) & 0xf) << 0)
+#define GET_GPD_HWO(gpd)	(le32_to_cpu((gpd)->dw0_info) & GPD_FLAGS_HWO)
+
+#define GPD_RX_BUF_LEN_OG(x)	(((x) & 0xffff) << 16)
+#define GPD_RX_BUF_LEN_EL(x)	(((x) & 0xfffff) << 12)
+#define GPD_RX_BUF_LEN(mtu, x)	\
+({				\
+	typeof(x) x_ = (x);	\
+	((mtu)->gen2cp) ? GPD_RX_BUF_LEN_EL(x_) : GPD_RX_BUF_LEN_OG(x_); \
+})
+
+#define GPD_DATA_LEN_OG(x)	((x) & 0xffff)
+#define GPD_DATA_LEN_EL(x)	((x) & 0xfffff)
+#define GPD_DATA_LEN(mtu, x)	\
+({				\
+	typeof(x) x_ = (x);	\
+	((mtu)->gen2cp) ? GPD_DATA_LEN_EL(x_) : GPD_DATA_LEN_OG(x_); \
+})
+
+#define GPD_EXT_FLAG_ZLP	BIT(29)
+#define GPD_EXT_NGP_OG(x)	(((x) & 0xf) << 20)
+#define GPD_EXT_BUF_OG(x)	(((x) & 0xf) << 16)
+#define GPD_EXT_NGP_EL(x)	(((x) & 0xf) << 28)
+#define GPD_EXT_BUF_EL(x)	(((x) & 0xf) << 24)
+#define GPD_EXT_NGP(mtu, x)	\
+({				\
+	typeof(x) x_ = (x);	\
+	((mtu)->gen2cp) ? GPD_EXT_NGP_EL(x_) : GPD_EXT_NGP_OG(x_); \
+})
+
+#define GPD_EXT_BUF(mtu, x)	\
+({				\
+	typeof(x) x_ = (x);	\
+	((mtu)->gen2cp) ? GPD_EXT_BUF_EL(x_) : GPD_EXT_BUF_OG(x_); \
+})
 
 #define HILO_GEN64(hi, lo) (((u64)(hi) << 32) + (lo))
 #define HILO_DMA(hi, lo)	\
@@ -125,7 +157,7 @@ static void reset_gpd_list(struct mtu3_ep *mep)
 	struct qmu_gpd *gpd = ring->start;
 
 	if (gpd) {
-		gpd->flag &= ~GPD_FLAGS_HWO;
+		gpd->dw0_info &= cpu_to_le32(~GPD_FLAGS_HWO);
 		gpd_ring_init(ring, gpd);
 	}
 }
@@ -214,16 +246,14 @@ static int mtu3_prepare_tx_gpd(struct mtu3_ep *mep, struct mtu3_request *mreq)
 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;
 	struct qmu_gpd *gpd = ring->enqueue;
 	struct usb_request *req = &mreq->request;
+	struct mtu3 *mtu = mep->mtu;
 	dma_addr_t enq_dma;
-	u16 ext_addr;
-
-	/* set all fields to zero as default value */
-	memset(gpd, 0, sizeof(*gpd));
+	u32 ext_addr;
 
+	gpd->dw0_info = 0;	/* SW own it */
 	gpd->buffer = cpu_to_le32(lower_32_bits(req->dma));
-	ext_addr = GPD_EXT_BUF(upper_32_bits(req->dma));
-	gpd->buf_len = cpu_to_le16(req->length);
-	gpd->flag |= GPD_FLAGS_IOC;
+	ext_addr = GPD_EXT_BUF(mtu, upper_32_bits(req->dma));
+	gpd->dw3_info = cpu_to_le32(GPD_DATA_LEN(mtu, req->length));
 
 	/* get the next GPD */
 	enq = advance_enq_gpd(ring);
@@ -231,17 +261,22 @@ static int mtu3_prepare_tx_gpd(struct mtu3_ep *mep, struct mtu3_request *mreq)
 	dev_dbg(mep->mtu->dev, "TX-EP%d queue gpd=%p, enq=%p, qdma=%pad\n",
 		mep->epnum, gpd, enq, &enq_dma);
 
-	enq->flag &= ~GPD_FLAGS_HWO;
+	enq->dw0_info &= cpu_to_le32(~GPD_FLAGS_HWO);
 	gpd->next_gpd = cpu_to_le32(lower_32_bits(enq_dma));
-	ext_addr |= GPD_EXT_NGP(upper_32_bits(enq_dma));
-	gpd->tx_ext_addr = cpu_to_le16(ext_addr);
-
-	if (req->zero)
-		gpd->ext_flag |= GPD_EXT_FLAG_ZLP;
+	ext_addr |= GPD_EXT_NGP(mtu, upper_32_bits(enq_dma));
+	gpd->dw0_info = cpu_to_le32(ext_addr);
+
+	if (req->zero) {
+		if (mtu->gen2cp)
+			gpd->dw0_info |= cpu_to_le32(GPD_FLAGS_ZLP);
+		else
+			gpd->dw3_info |= cpu_to_le32(GPD_EXT_FLAG_ZLP);
+	}
 
-	gpd->flag |= GPD_FLAGS_HWO;
+	gpd->dw0_info |= cpu_to_le32(GPD_FLAGS_IOC | GPD_FLAGS_HWO);
 
 	mreq->gpd = gpd;
+	trace_mtu3_prepare_gpd(mep, gpd);
 
 	return 0;
 }
@@ -252,16 +287,14 @@ static int mtu3_prepare_rx_gpd(struct mtu3_ep *mep, struct mtu3_request *mreq)
 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;
 	struct qmu_gpd *gpd = ring->enqueue;
 	struct usb_request *req = &mreq->request;
+	struct mtu3 *mtu = mep->mtu;
 	dma_addr_t enq_dma;
-	u16 ext_addr;
-
-	/* set all fields to zero as default value */
-	memset(gpd, 0, sizeof(*gpd));
+	u32 ext_addr;
 
+	gpd->dw0_info = 0;	/* SW own it */
 	gpd->buffer = cpu_to_le32(lower_32_bits(req->dma));
-	ext_addr = GPD_EXT_BUF(upper_32_bits(req->dma));
-	gpd->data_buf_len = cpu_to_le16(req->length);
-	gpd->flag |= GPD_FLAGS_IOC;
+	ext_addr = GPD_EXT_BUF(mtu, upper_32_bits(req->dma));
+	gpd->dw0_info = cpu_to_le32(GPD_RX_BUF_LEN(mtu, req->length));
 
 	/* get the next GPD */
 	enq = advance_enq_gpd(ring);
@@ -269,13 +302,14 @@ static int mtu3_prepare_rx_gpd(struct mtu3_ep *mep, struct mtu3_request *mreq)
 	dev_dbg(mep->mtu->dev, "RX-EP%d queue gpd=%p, enq=%p, qdma=%pad\n",
 		mep->epnum, gpd, enq, &enq_dma);
 
-	enq->flag &= ~GPD_FLAGS_HWO;
+	enq->dw0_info &= cpu_to_le32(~GPD_FLAGS_HWO);
 	gpd->next_gpd = cpu_to_le32(lower_32_bits(enq_dma));
-	ext_addr |= GPD_EXT_NGP(upper_32_bits(enq_dma));
-	gpd->rx_ext_addr = cpu_to_le16(ext_addr);
-	gpd->flag |= GPD_FLAGS_HWO;
+	ext_addr |= GPD_EXT_NGP(mtu, upper_32_bits(enq_dma));
+	gpd->dw3_info = cpu_to_le32(ext_addr);
+	gpd->dw0_info |= cpu_to_le32(GPD_FLAGS_IOC | GPD_FLAGS_HWO);
 
 	mreq->gpd = gpd;
+	trace_mtu3_prepare_gpd(mep, gpd);
 
 	return 0;
 }
@@ -382,27 +416,25 @@ static void qmu_tx_zlp_error_handler(struct mtu3 *mtu, u8 epnum)
 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;
 	void __iomem *mbase = mtu->mac_base;
 	struct qmu_gpd *gpd_current = NULL;
-	struct usb_request *req = NULL;
 	struct mtu3_request *mreq;
 	dma_addr_t cur_gpd_dma;
 	u32 txcsr = 0;
 	int ret;
 
 	mreq = next_request(mep);
-	if (mreq && mreq->request.length == 0)
-		req = &mreq->request;
-	else
+	if (mreq && mreq->request.length != 0)
 		return;
 
 	cur_gpd_dma = read_txq_cur_addr(mbase, epnum);
 	gpd_current = gpd_dma_to_virt(ring, cur_gpd_dma);
 
-	if (le16_to_cpu(gpd_current->buf_len) != 0) {
+	if (GPD_DATA_LEN(mtu, le32_to_cpu(gpd_current->dw3_info)) != 0) {
 		dev_err(mtu->dev, "TX EP%d buffer length error(!=0)\n", epnum);
 		return;
 	}
 
-	dev_dbg(mtu->dev, "%s send ZLP for req=%p\n", __func__, req);
+	dev_dbg(mtu->dev, "%s send ZLP for req=%p\n", __func__, mreq);
+	trace_mtu3_zlp_exp_gpd(mep, gpd_current);
 
 	mtu3_clrbits(mbase, MU3D_EP_TXCR0(mep->epnum), TX_DMAREQEN);
 
@@ -415,8 +447,7 @@ static void qmu_tx_zlp_error_handler(struct mtu3 *mtu, u8 epnum)
 	mtu3_setbits(mbase, MU3D_EP_TXCR0(mep->epnum), TX_TXPKTRDY);
 
 	/* by pass the current GDP */
-	gpd_current->flag |= GPD_FLAGS_BPS;
-	gpd_current->flag |= GPD_FLAGS_HWO;
+	gpd_current->dw0_info |= cpu_to_le32(GPD_FLAGS_BPS | GPD_FLAGS_HWO);
 
 	/*enable DMAREQEN, switch back to QMU mode */
 	mtu3_setbits(mbase, MU3D_EP_TXCR0(mep->epnum), TX_DMAREQEN);
@@ -448,7 +479,7 @@ static void qmu_done_tx(struct mtu3 *mtu, u8 epnum)
 	dev_dbg(mtu->dev, "%s EP%d, last=%p, current=%p, enq=%p\n",
 		__func__, epnum, gpd, gpd_current, ring->enqueue);
 
-	while (gpd != gpd_current && !(gpd->flag & GPD_FLAGS_HWO)) {
+	while (gpd != gpd_current && !GET_GPD_HWO(gpd)) {
 
 		mreq = next_request(mep);
 
@@ -458,7 +489,8 @@ static void qmu_done_tx(struct mtu3 *mtu, u8 epnum)
 		}
 
 		request = &mreq->request;
-		request->actual = le16_to_cpu(gpd->buf_len);
+		request->actual = GPD_DATA_LEN(mtu, le32_to_cpu(gpd->dw3_info));
+		trace_mtu3_complete_gpd(mep, gpd);
 		mtu3_req_complete(mep, request, 0);
 
 		gpd = advance_deq_gpd(ring);
@@ -486,7 +518,7 @@ static void qmu_done_rx(struct mtu3 *mtu, u8 epnum)
 	dev_dbg(mtu->dev, "%s EP%d, last=%p, current=%p, enq=%p\n",
 		__func__, epnum, gpd, gpd_current, ring->enqueue);
 
-	while (gpd != gpd_current && !(gpd->flag & GPD_FLAGS_HWO)) {
+	while (gpd != gpd_current && !GET_GPD_HWO(gpd)) {
 
 		mreq = next_request(mep);
 
@@ -496,7 +528,8 @@ static void qmu_done_rx(struct mtu3 *mtu, u8 epnum)
 		}
 		req = &mreq->request;
 
-		req->actual = le16_to_cpu(gpd->buf_len);
+		req->actual = GPD_DATA_LEN(mtu, le32_to_cpu(gpd->dw3_info));
+		trace_mtu3_complete_gpd(mep, gpd);
 		mtu3_req_complete(mep, req, 0);
 
 		gpd = advance_deq_gpd(ring);
@@ -574,6 +607,7 @@ irqreturn_t mtu3_qmu_isr(struct mtu3 *mtu)
 	dev_dbg(mtu->dev, "=== QMUdone[tx=%x, rx=%x] QMUexp[%x] ===\n",
 		(qmu_done_status & 0xFFFF), qmu_done_status >> 16,
 		qmu_status);
+	trace_mtu3_qmu_isr(qmu_done_status, qmu_status);
 
 	if (qmu_done_status)
 		qmu_done_isr(mtu, qmu_done_status);
diff --git a/drivers/usb/mtu3/mtu3_qmu.h b/drivers/usb/mtu3/mtu3_qmu.h
index 81f5151a55ed..9cfde201db63 100644
--- a/drivers/usb/mtu3/mtu3_qmu.h
+++ b/drivers/usb/mtu3/mtu3_qmu.h
@@ -15,6 +15,7 @@
 #define QMU_GPD_RING_SIZE	(MAX_GPD_NUM * QMU_GPD_SIZE)
 
 #define GPD_BUF_SIZE		65532
+#define GPD_BUF_SIZE_EL		1048572
 
 void mtu3_qmu_stop(struct mtu3_ep *mep);
 int mtu3_qmu_start(struct mtu3_ep *mep);
diff --git a/drivers/usb/mtu3/mtu3_trace.c b/drivers/usb/mtu3/mtu3_trace.c
new file mode 100644
index 000000000000..4f5e7857ec31
--- /dev/null
+++ b/drivers/usb/mtu3/mtu3_trace.c
@@ -0,0 +1,23 @@
+// SPDX-License-Identifier: GPL-2.0
+/**
+ * mtu3_trace.c - trace support
+ *
+ * Copyright (C) 2019 MediaTek Inc.
+ *
+ * Author: Chunfeng Yun <chunfeng.yun@mediatek.com>
+ */
+
+#define CREATE_TRACE_POINTS
+#include "mtu3_trace.h"
+
+void mtu3_dbg_trace(struct device *dev, const char *fmt, ...)
+{
+	struct va_format vaf;
+	va_list args;
+
+	va_start(args, fmt);
+	vaf.fmt = fmt;
+	vaf.va = &args;
+	trace_mtu3_log(dev, &vaf);
+	va_end(args);
+}
diff --git a/drivers/usb/mtu3/mtu3_trace.h b/drivers/usb/mtu3/mtu3_trace.h
new file mode 100644
index 000000000000..050e30f0fbd4
--- /dev/null
+++ b/drivers/usb/mtu3/mtu3_trace.h
@@ -0,0 +1,279 @@
+// SPDX-License-Identifier: GPL-2.0
+/**
+ * mtu3_trace.h - trace support
+ *
+ * Copyright (C) 2019 MediaTek Inc.
+ *
+ * Author: Chunfeng Yun <chunfeng.yun@mediatek.com>
+ */
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM mtu3
+
+#if !defined(__MTU3_TRACE_H__) || defined(TRACE_HEADER_MULTI_READ)
+#define __MTU3_TRACE_H__
+
+#include <linux/types.h>
+#include <linux/tracepoint.h>
+
+#include "mtu3.h"
+
+#define MTU3_MSG_MAX	256
+
+TRACE_EVENT(mtu3_log,
+	TP_PROTO(struct device *dev, struct va_format *vaf),
+	TP_ARGS(dev, vaf),
+	TP_STRUCT__entry(
+		__string(name, dev_name(dev))
+		__dynamic_array(char, msg, MTU3_MSG_MAX)
+	),
+	TP_fast_assign(
+		__assign_str(name, dev_name(dev));
+		vsnprintf(__get_str(msg), MTU3_MSG_MAX, vaf->fmt, *vaf->va);
+	),
+	TP_printk("%s: %s", __get_str(name), __get_str(msg))
+);
+
+TRACE_EVENT(mtu3_u3_ltssm_isr,
+	TP_PROTO(u32 intr),
+	TP_ARGS(intr),
+	TP_STRUCT__entry(
+		__field(u32, intr)
+	),
+	TP_fast_assign(
+		__entry->intr = intr;
+	),
+	TP_printk("(%08x) %s %s %s %s %s %s", __entry->intr,
+		__entry->intr & HOT_RST_INTR ? "HOT_RST" : "",
+		__entry->intr & WARM_RST_INTR ? "WARM_RST" : "",
+		__entry->intr & ENTER_U3_INTR ? "ENT_U3" : "",
+		__entry->intr & EXIT_U3_INTR ? "EXIT_U3" : "",
+		__entry->intr & VBUS_RISE_INTR ? "VBUS_RISE" : "",
+		__entry->intr & VBUS_FALL_INTR ? "VBUS_FALL" : ""
+	)
+);
+
+TRACE_EVENT(mtu3_u2_common_isr,
+	TP_PROTO(u32 intr),
+	TP_ARGS(intr),
+	TP_STRUCT__entry(
+		__field(u32, intr)
+	),
+	TP_fast_assign(
+		__entry->intr = intr;
+	),
+	TP_printk("(%08x) %s %s %s", __entry->intr,
+		__entry->intr & SUSPEND_INTR ? "SUSPEND" : "",
+		__entry->intr & RESUME_INTR ? "RESUME" : "",
+		__entry->intr & RESET_INTR ? "RESET" : ""
+	)
+);
+
+TRACE_EVENT(mtu3_qmu_isr,
+	TP_PROTO(u32 done_intr, u32 exp_intr),
+	TP_ARGS(done_intr, exp_intr),
+	TP_STRUCT__entry(
+		__field(u32, done_intr)
+		__field(u32, exp_intr)
+	),
+	TP_fast_assign(
+		__entry->done_intr = done_intr;
+		__entry->exp_intr = exp_intr;
+	),
+	TP_printk("done (tx %04x, rx %04x), exp (%08x)",
+		__entry->done_intr & 0xffff,
+		__entry->done_intr >> 16,
+		__entry->exp_intr
+	)
+);
+
+DECLARE_EVENT_CLASS(mtu3_log_setup,
+	TP_PROTO(struct usb_ctrlrequest *setup),
+	TP_ARGS(setup),
+	TP_STRUCT__entry(
+		__field(__u8, bRequestType)
+		__field(__u8, bRequest)
+		__field(__u16, wValue)
+		__field(__u16, wIndex)
+		__field(__u16, wLength)
+	),
+	TP_fast_assign(
+		__entry->bRequestType = setup->bRequestType;
+		__entry->bRequest = setup->bRequest;
+		__entry->wValue = le16_to_cpu(setup->wValue);
+		__entry->wIndex = le16_to_cpu(setup->wIndex);
+		__entry->wLength = le16_to_cpu(setup->wLength);
+	),
+	TP_printk("setup - %02x %02x %04x %04x %04x",
+		__entry->bRequestType, __entry->bRequest,
+		__entry->wValue, __entry->wIndex, __entry->wLength
+	)
+);
+
+DEFINE_EVENT(mtu3_log_setup, mtu3_handle_setup,
+	TP_PROTO(struct usb_ctrlrequest *setup),
+	TP_ARGS(setup)
+);
+
+DECLARE_EVENT_CLASS(mtu3_log_request,
+	TP_PROTO(struct mtu3_request *mreq),
+	TP_ARGS(mreq),
+	TP_STRUCT__entry(
+		__string(name, mreq->mep->name)
+		__field(struct mtu3_request *, mreq)
+		__field(struct qmu_gpd *, gpd)
+		__field(unsigned int, actual)
+		__field(unsigned int, length)
+		__field(int, status)
+		__field(int, zero)
+		__field(int, no_interrupt)
+	),
+	TP_fast_assign(
+		__assign_str(name, mreq->mep->name);
+		__entry->mreq = mreq;
+		__entry->gpd = mreq->gpd;
+		__entry->actual = mreq->request.actual;
+		__entry->length = mreq->request.length;
+		__entry->status = mreq->request.status;
+		__entry->zero = mreq->request.zero;
+		__entry->no_interrupt = mreq->request.no_interrupt;
+	),
+	TP_printk("%s: req %p gpd %p len %u/%u %s%s --> %d",
+		__get_str(name), __entry->mreq, __entry->gpd,
+		__entry->actual, __entry->length,
+		__entry->zero ? "Z" : "z",
+		__entry->no_interrupt ? "i" : "I",
+		__entry->status
+	)
+);
+
+DEFINE_EVENT(mtu3_log_request, mtu3_alloc_request,
+	TP_PROTO(struct mtu3_request *req),
+	TP_ARGS(req)
+);
+
+DEFINE_EVENT(mtu3_log_request, mtu3_free_request,
+	TP_PROTO(struct mtu3_request *req),
+	TP_ARGS(req)
+);
+
+DEFINE_EVENT(mtu3_log_request, mtu3_gadget_queue,
+	TP_PROTO(struct mtu3_request *req),
+	TP_ARGS(req)
+);
+
+DEFINE_EVENT(mtu3_log_request, mtu3_gadget_dequeue,
+	TP_PROTO(struct mtu3_request *req),
+	TP_ARGS(req)
+);
+
+DEFINE_EVENT(mtu3_log_request, mtu3_req_complete,
+	TP_PROTO(struct mtu3_request *req),
+	TP_ARGS(req)
+);
+
+DECLARE_EVENT_CLASS(mtu3_log_gpd,
+	TP_PROTO(struct mtu3_ep *mep, struct qmu_gpd *gpd),
+	TP_ARGS(mep, gpd),
+	TP_STRUCT__entry(
+		__string(name, mep->name)
+		__field(struct qmu_gpd *, gpd)
+		__field(u32, dw0)
+		__field(u32, dw1)
+		__field(u32, dw2)
+		__field(u32, dw3)
+	),
+	TP_fast_assign(
+		__assign_str(name, mep->name);
+		__entry->gpd = gpd;
+		__entry->dw0 = le32_to_cpu(gpd->dw0_info);
+		__entry->dw1 = le32_to_cpu(gpd->next_gpd);
+		__entry->dw2 = le32_to_cpu(gpd->buffer);
+		__entry->dw3 = le32_to_cpu(gpd->dw3_info);
+	),
+	TP_printk("%s: gpd %p - %08x %08x %08x %08x",
+		__get_str(name), __entry->gpd,
+		__entry->dw0, __entry->dw1,
+		__entry->dw2, __entry->dw3
+	)
+);
+
+DEFINE_EVENT(mtu3_log_gpd, mtu3_prepare_gpd,
+	TP_PROTO(struct mtu3_ep *mep, struct qmu_gpd *gpd),
+	TP_ARGS(mep, gpd)
+);
+
+DEFINE_EVENT(mtu3_log_gpd, mtu3_complete_gpd,
+	TP_PROTO(struct mtu3_ep *mep, struct qmu_gpd *gpd),
+	TP_ARGS(mep, gpd)
+);
+
+DEFINE_EVENT(mtu3_log_gpd, mtu3_zlp_exp_gpd,
+	TP_PROTO(struct mtu3_ep *mep, struct qmu_gpd *gpd),
+	TP_ARGS(mep, gpd)
+);
+
+DECLARE_EVENT_CLASS(mtu3_log_ep,
+	TP_PROTO(struct mtu3_ep *mep),
+	TP_ARGS(mep),
+	TP_STRUCT__entry(
+		__string(name, mep->name)
+		__field(unsigned int, type)
+		__field(unsigned int, slot)
+		__field(unsigned int, maxp)
+		__field(unsigned int, mult)
+		__field(unsigned int, maxburst)
+		__field(unsigned int, flags)
+		__field(unsigned int, direction)
+		__field(struct mtu3_gpd_ring *, gpd_ring)
+	),
+	TP_fast_assign(
+		__assign_str(name, mep->name);
+		__entry->type = mep->type;
+		__entry->slot = mep->slot;
+		__entry->maxp = mep->ep.maxpacket;
+		__entry->mult = mep->ep.mult;
+		__entry->maxburst = mep->ep.maxburst;
+		__entry->flags = mep->flags;
+		__entry->direction = mep->is_in;
+		__entry->gpd_ring = &mep->gpd_ring;
+	),
+	TP_printk("%s: type %d maxp %d slot %d mult %d burst %d ring %p/%pad flags %c:%c%c%c:%c",
+		__get_str(name), __entry->type,
+		__entry->maxp, __entry->slot,
+		__entry->mult, __entry->maxburst,
+		__entry->gpd_ring, &__entry->gpd_ring->dma,
+		__entry->flags & MTU3_EP_ENABLED ? 'E' : 'e',
+		__entry->flags & MTU3_EP_STALL ? 'S' : 's',
+		__entry->flags & MTU3_EP_WEDGE ? 'W' : 'w',
+		__entry->flags & MTU3_EP_BUSY ? 'B' : 'b',
+		__entry->direction ? '<' : '>'
+	)
+);
+
+DEFINE_EVENT(mtu3_log_ep, mtu3_gadget_ep_enable,
+	TP_PROTO(struct mtu3_ep *mep),
+	TP_ARGS(mep)
+);
+
+DEFINE_EVENT(mtu3_log_ep, mtu3_gadget_ep_disable,
+	TP_PROTO(struct mtu3_ep *mep),
+	TP_ARGS(mep)
+);
+
+DEFINE_EVENT(mtu3_log_ep, mtu3_gadget_ep_set_halt,
+	TP_PROTO(struct mtu3_ep *mep),
+	TP_ARGS(mep)
+);
+
+#endif /* __MTU3_TRACE_H__ */
+
+/* this part has to be here */
+
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_FILE mtu3_trace
+
+#include <trace/define_trace.h>
diff --git a/drivers/usb/musb/Kconfig b/drivers/usb/musb/Kconfig
index f742fddc5e2c..52f8e2b57ad5 100644
--- a/drivers/usb/musb/Kconfig
+++ b/drivers/usb/musb/Kconfig
@@ -67,7 +67,7 @@ config USB_MUSB_SUNXI
 	depends on NOP_USB_XCEIV
 	depends on PHY_SUN4I_USB
 	depends on EXTCON
-	depends on GENERIC_PHY
+	select GENERIC_PHY
 	select SUNXI_SRAM
 
 config USB_MUSB_DAVINCI
diff --git a/drivers/usb/musb/jz4740.c b/drivers/usb/musb/jz4740.c
index a60627bf7be3..5261f8dfedec 100644
--- a/drivers/usb/musb/jz4740.c
+++ b/drivers/usb/musb/jz4740.c
@@ -74,10 +74,14 @@ static struct musb_hdrc_platform_data jz4740_musb_platform_data = {
 
 static int jz4740_musb_init(struct musb *musb)
 {
-	usb_phy_generic_register();
-	musb->xceiv = usb_get_phy(USB_PHY_TYPE_USB2);
+	struct device *dev = musb->controller->parent;
+
+	if (dev->of_node)
+		musb->xceiv = devm_usb_get_phy_by_phandle(dev, "phys", 0);
+	else
+		musb->xceiv = devm_usb_get_phy(dev, USB_PHY_TYPE_USB2);
 	if (IS_ERR(musb->xceiv)) {
-		pr_err("HS UDC: no transceiver configured\n");
+		dev_err(dev, "No transceiver configured\n");
 		return PTR_ERR(musb->xceiv);
 	}
 
@@ -91,13 +95,6 @@ static int jz4740_musb_init(struct musb *musb)
 	return 0;
 }
 
-static int jz4740_musb_exit(struct musb *musb)
-{
-	usb_put_phy(musb->xceiv);
-
-	return 0;
-}
-
 /*
  * DMA has not been confirmed to work with CONFIG_USB_INVENTRA_DMA,
  * so let's not set up the dma function pointers yet.
@@ -106,7 +103,6 @@ static const struct musb_platform_ops jz4740_musb_ops = {
 	.quirks		= MUSB_DMA_INVENTRA | MUSB_INDEXED_EP,
 	.fifo_mode	= 2,
 	.init		= jz4740_musb_init,
-	.exit		= jz4740_musb_exit,
 };
 
 static int jz4740_probe(struct platform_device *pdev)
@@ -183,7 +179,6 @@ static int jz4740_remove(struct platform_device *pdev)
 	struct jz4740_glue	*glue = platform_get_drvdata(pdev);
 
 	platform_device_unregister(glue->musb);
-	usb_phy_generic_unregister(pdev);
 	clk_disable_unprepare(glue->clk);
 
 	return 0;
diff --git a/drivers/usb/musb/musb_core.c b/drivers/usb/musb/musb_core.c
index b7d56272f9d1..9f5a4819a744 100644
--- a/drivers/usb/musb/musb_core.c
+++ b/drivers/usb/musb/musb_core.c
@@ -1497,10 +1497,11 @@ static int musb_core_init(u16 musb_type, struct musb *musb)
 	} else {
 		musb->is_multipoint = 0;
 		type = "";
-#ifndef	CONFIG_USB_OTG_BLACKLIST_HUB
-		pr_err("%s: kernel must blacklist external hubs\n",
-		       musb_driver_name);
-#endif
+		if (IS_ENABLED(CONFIG_USB) &&
+		    !IS_ENABLED(CONFIG_USB_OTG_BLACKLIST_HUB)) {
+			pr_err("%s: kernel must blacklist external hubs\n",
+			       musb_driver_name);
+		}
 	}
 
 	/* log release info */
diff --git a/drivers/usb/musb/musb_dsps.c b/drivers/usb/musb/musb_dsps.c
index 403eb97915f8..327d4f7baaf7 100644
--- a/drivers/usb/musb/musb_dsps.c
+++ b/drivers/usb/musb/musb_dsps.c
@@ -168,8 +168,7 @@ static void dsps_mod_timer_optional(struct dsps_glue *glue)
 static void dsps_musb_enable(struct musb *musb)
 {
 	struct device *dev = musb->controller;
-	struct platform_device *pdev = to_platform_device(dev->parent);
-	struct dsps_glue *glue = platform_get_drvdata(pdev);
+	struct dsps_glue *glue = dev_get_drvdata(dev->parent);
 	const struct dsps_musb_wrapper *wrp = glue->wrp;
 	void __iomem *reg_base = musb->ctrl_base;
 	u32 epmask, coremask;
@@ -195,8 +194,7 @@ static void dsps_musb_enable(struct musb *musb)
 static void dsps_musb_disable(struct musb *musb)
 {
 	struct device *dev = musb->controller;
-	struct platform_device *pdev = to_platform_device(dev->parent);
-	struct dsps_glue *glue = platform_get_drvdata(pdev);
+	struct dsps_glue *glue = dev_get_drvdata(dev->parent);
 	const struct dsps_musb_wrapper *wrp = glue->wrp;
 	void __iomem *reg_base = musb->ctrl_base;
 
diff --git a/drivers/usb/musb/omap2430.c b/drivers/usb/musb/omap2430.c
index b1dd81fb5f55..a3d2fef67746 100644
--- a/drivers/usb/musb/omap2430.c
+++ b/drivers/usb/musb/omap2430.c
@@ -531,6 +531,9 @@ static int omap2430_runtime_suspend(struct device *dev)
 
 	omap2430_low_level_exit(musb);
 
+	phy_power_off(musb->phy);
+	phy_exit(musb->phy);
+
 	return 0;
 }
 
@@ -542,6 +545,9 @@ static int omap2430_runtime_resume(struct device *dev)
 	if (!musb)
 		return 0;
 
+	phy_init(musb->phy);
+	phy_power_on(musb->phy);
+
 	omap2430_low_level_init(musb);
 	musb_writel(musb->mregs, OTG_INTERFSEL,
 		    musb->context.otg_interfsel);
diff --git a/drivers/usb/phy/Kconfig b/drivers/usb/phy/Kconfig
index 8c509b060c09..24b4f091acb8 100644
--- a/drivers/usb/phy/Kconfig
+++ b/drivers/usb/phy/Kconfig
@@ -21,7 +21,7 @@ config AB8500_USB
 	  in host mode, low speed.
 
 config FSL_USB2_OTG
-	bool "Freescale USB OTG Transceiver Driver"
+	tristate "Freescale USB OTG Transceiver Driver"
 	depends on USB_EHCI_FSL && USB_FSL_USB2 && USB_OTG_FSM=y && PM
 	depends on USB_GADGET || !USB_GADGET # if USB_GADGET=m, this can't be 'y'
 	select USB_PHY
diff --git a/drivers/usb/serial/ark3116.c b/drivers/usb/serial/ark3116.c
index ff38aa8963cf..71a9206ea1e2 100644
--- a/drivers/usb/serial/ark3116.c
+++ b/drivers/usb/serial/ark3116.c
@@ -189,16 +189,6 @@ static int ark3116_port_remove(struct usb_serial_port *port)
 	return 0;
 }
 
-static void ark3116_init_termios(struct tty_struct *tty)
-{
-	struct ktermios *termios = &tty->termios;
-	*termios = tty_std_termios;
-	termios->c_cflag = B9600 | CS8
-				      | CREAD | HUPCL | CLOCAL;
-	termios->c_ispeed = 9600;
-	termios->c_ospeed = 9600;
-}
-
 static void ark3116_set_termios(struct tty_struct *tty,
 				struct usb_serial_port *port,
 				struct ktermios *old_termios)
@@ -645,7 +635,6 @@ static struct usb_serial_driver ark3116_device = {
 	.port_probe =		ark3116_port_probe,
 	.port_remove =		ark3116_port_remove,
 	.set_termios =		ark3116_set_termios,
-	.init_termios =		ark3116_init_termios,
 	.get_serial =		ark3116_get_serial_info,
 	.tiocmget =		ark3116_tiocmget,
 	.tiocmset =		ark3116_tiocmset,
diff --git a/drivers/usb/serial/cypress_m8.c b/drivers/usb/serial/cypress_m8.c
index ed51bc48eea6..72d3ae1ebc64 100644
--- a/drivers/usb/serial/cypress_m8.c
+++ b/drivers/usb/serial/cypress_m8.c
@@ -98,7 +98,6 @@ struct cypress_private {
 	int write_urb_interval;            /* interval to use for write urb */
 	int read_urb_interval;             /* interval to use for read urb */
 	int comm_is_ok;                    /* true if communication is (still) ok */
-	int termios_initialized;
 	__u8 line_control;	   	   /* holds dtr / rts value */
 	__u8 current_status;	   	   /* received from last read - info on dsr,cts,cd,ri,etc */
 	__u8 current_config;	   	   /* stores the current configuration byte */
@@ -107,11 +106,7 @@ struct cypress_private {
 	int get_cfg_unsafe;		   /* If true, the CYPRESS_GET_CONFIG is unsafe */
 	int baud_rate;			   /* stores current baud rate in
 					      integer form */
-	int isthrottled;		   /* if throttled, discard reads */
 	char prev_status;		   /* used for TIOCMIWAIT */
-	/* we pass a pointer to this as the argument sent to
-	   cypress_set_termios old_termios */
-	struct ktermios tmp_termios; 	   /* stores the old termios settings */
 };
 
 /* function prototypes for the Cypress USB to serial device */
@@ -126,6 +121,7 @@ static int  cypress_write(struct tty_struct *tty, struct usb_serial_port *port,
 			const unsigned char *buf, int count);
 static void cypress_send(struct usb_serial_port *port);
 static int  cypress_write_room(struct tty_struct *tty);
+static void cypress_earthmate_init_termios(struct tty_struct *tty);
 static void cypress_set_termios(struct tty_struct *tty,
 			struct usb_serial_port *port, struct ktermios *old);
 static int  cypress_tiocmget(struct tty_struct *tty);
@@ -153,6 +149,7 @@ static struct usb_serial_driver cypress_earthmate_device = {
 	.dtr_rts =			cypress_dtr_rts,
 	.write =			cypress_write,
 	.write_room =			cypress_write_room,
+	.init_termios =			cypress_earthmate_init_termios,
 	.set_termios =			cypress_set_termios,
 	.tiocmget =			cypress_tiocmget,
 	.tiocmset =			cypress_tiocmset,
@@ -467,7 +464,6 @@ static int cypress_generic_port_probe(struct usb_serial_port *port)
 
 	priv->cmd_ctrl = 0;
 	priv->line_control = 0;
-	priv->termios_initialized = 0;
 	priv->rx_flags = 0;
 	/* Default packet format setting is determined by packet size.
 	   Anything with a size larger then 9 must have a separate
@@ -604,7 +600,7 @@ static int cypress_open(struct tty_struct *tty, struct usb_serial_port *port)
 	cypress_send(port);
 
 	if (tty)
-		cypress_set_termios(tty, port, &priv->tmp_termios);
+		cypress_set_termios(tty, port, NULL);
 
 	/* setup the port and start reading from the device */
 	usb_fill_int_urb(port->interrupt_in_urb, serial->dev,
@@ -857,6 +853,11 @@ static int cypress_tiocmset(struct tty_struct *tty,
 	return cypress_write(tty, port, NULL, 0);
 }
 
+static void cypress_earthmate_init_termios(struct tty_struct *tty)
+{
+	tty_encode_baud_rate(tty, 4800, 4800);
+}
+
 static void cypress_set_termios(struct tty_struct *tty,
 	struct usb_serial_port *port, struct ktermios *old_termios)
 {
@@ -868,45 +869,11 @@ static void cypress_set_termios(struct tty_struct *tty,
 	__u8 oldlines;
 	int linechange = 0;
 
-	spin_lock_irqsave(&priv->lock, flags);
-	/* We can't clean this one up as we don't know the device type
-	   early enough */
-	if (!priv->termios_initialized) {
-		if (priv->chiptype == CT_EARTHMATE) {
-			tty->termios = tty_std_termios;
-			tty->termios.c_cflag = B4800 | CS8 | CREAD | HUPCL |
-				CLOCAL;
-			tty->termios.c_ispeed = 4800;
-			tty->termios.c_ospeed = 4800;
-		} else if (priv->chiptype == CT_CYPHIDCOM) {
-			tty->termios = tty_std_termios;
-			tty->termios.c_cflag = B9600 | CS8 | CREAD | HUPCL |
-				CLOCAL;
-			tty->termios.c_ispeed = 9600;
-			tty->termios.c_ospeed = 9600;
-		} else if (priv->chiptype == CT_CA42V2) {
-			tty->termios = tty_std_termios;
-			tty->termios.c_cflag = B9600 | CS8 | CREAD | HUPCL |
-				CLOCAL;
-			tty->termios.c_ispeed = 9600;
-			tty->termios.c_ospeed = 9600;
-		}
-		priv->termios_initialized = 1;
-	}
-	spin_unlock_irqrestore(&priv->lock, flags);
-
 	/* Unsupported features need clearing */
 	tty->termios.c_cflag &= ~(CMSPAR|CRTSCTS);
 
 	cflag = tty->termios.c_cflag;
 
-	/* check if there are new settings */
-	if (old_termios) {
-		spin_lock_irqsave(&priv->lock, flags);
-		priv->tmp_termios = tty->termios;
-		spin_unlock_irqrestore(&priv->lock, flags);
-	}
-
 	/* set number of data bits, parity, stop bits */
 	/* when parity is disabled the parity type bit is ignored */
 
diff --git a/drivers/usb/serial/digi_acceleport.c b/drivers/usb/serial/digi_acceleport.c
index e7f244cf2c07..578ebdd86520 100644
--- a/drivers/usb/serial/digi_acceleport.c
+++ b/drivers/usb/serial/digi_acceleport.c
@@ -569,9 +569,9 @@ static int digi_set_modem_signals(struct usb_serial_port *port,
 	ret = usb_submit_urb(oob_port->write_urb, GFP_ATOMIC);
 	if (ret == 0) {
 		oob_priv->dp_write_urb_in_use = 1;
-		port_priv->dp_modem_signals =
-			(port_priv->dp_modem_signals&~(TIOCM_DTR|TIOCM_RTS))
-			| (modem_signals&(TIOCM_DTR|TIOCM_RTS));
+		port_priv->dp_modem_signals &= ~(TIOCM_DTR | TIOCM_RTS);
+		port_priv->dp_modem_signals |=
+				modem_signals & (TIOCM_DTR | TIOCM_RTS);
 	}
 	spin_unlock(&port_priv->dp_port_lock);
 	spin_unlock_irqrestore(&oob_priv->dp_port_lock, flags);
@@ -740,9 +740,9 @@ static void digi_set_termios(struct tty_struct *tty,
 	/* set parity */
 	tty->termios.c_cflag &= ~CMSPAR;
 
-	if ((cflag&(PARENB|PARODD)) != (old_cflag&(PARENB|PARODD))) {
-		if (cflag&PARENB) {
-			if (cflag&PARODD)
+	if ((cflag & (PARENB | PARODD)) != (old_cflag & (PARENB | PARODD))) {
+		if (cflag & PARENB) {
+			if (cflag & PARODD)
 				arg = DIGI_PARITY_ODD;
 			else
 				arg = DIGI_PARITY_EVEN;
@@ -755,9 +755,9 @@ static void digi_set_termios(struct tty_struct *tty,
 		buf[i++] = 0;
 	}
 	/* set word size */
-	if ((cflag&CSIZE) != (old_cflag&CSIZE)) {
+	if ((cflag & CSIZE) != (old_cflag & CSIZE)) {
 		arg = -1;
-		switch (cflag&CSIZE) {
+		switch (cflag & CSIZE) {
 		case CS5: arg = DIGI_WORD_SIZE_5; break;
 		case CS6: arg = DIGI_WORD_SIZE_6; break;
 		case CS7: arg = DIGI_WORD_SIZE_7; break;
@@ -765,7 +765,7 @@ static void digi_set_termios(struct tty_struct *tty,
 		default:
 			dev_dbg(dev,
 				"digi_set_termios: can't handle word size %d\n",
-				(cflag&CSIZE));
+				cflag & CSIZE);
 			break;
 		}
 
@@ -779,9 +779,9 @@ static void digi_set_termios(struct tty_struct *tty,
 	}
 
 	/* set stop bits */
-	if ((cflag&CSTOPB) != (old_cflag&CSTOPB)) {
+	if ((cflag & CSTOPB) != (old_cflag & CSTOPB)) {
 
-		if ((cflag&CSTOPB))
+		if ((cflag & CSTOPB))
 			arg = DIGI_STOP_BITS_2;
 		else
 			arg = DIGI_STOP_BITS_1;
@@ -794,15 +794,15 @@ static void digi_set_termios(struct tty_struct *tty,
 	}
 
 	/* set input flow control */
-	if ((iflag&IXOFF) != (old_iflag&IXOFF)
-	    || (cflag&CRTSCTS) != (old_cflag&CRTSCTS)) {
+	if ((iflag & IXOFF) != (old_iflag & IXOFF) ||
+			(cflag & CRTSCTS) != (old_cflag & CRTSCTS)) {
 		arg = 0;
-		if (iflag&IXOFF)
+		if (iflag & IXOFF)
 			arg |= DIGI_INPUT_FLOW_CONTROL_XON_XOFF;
 		else
 			arg &= ~DIGI_INPUT_FLOW_CONTROL_XON_XOFF;
 
-		if (cflag&CRTSCTS) {
+		if (cflag & CRTSCTS) {
 			arg |= DIGI_INPUT_FLOW_CONTROL_RTS;
 
 			/* On USB-4 it is necessary to assert RTS prior */
@@ -822,19 +822,18 @@ static void digi_set_termios(struct tty_struct *tty,
 	}
 
 	/* set output flow control */
-	if ((iflag & IXON) != (old_iflag & IXON)
-	    || (cflag & CRTSCTS) != (old_cflag & CRTSCTS)) {
+	if ((iflag & IXON) != (old_iflag & IXON) ||
+			(cflag & CRTSCTS) != (old_cflag & CRTSCTS)) {
 		arg = 0;
 		if (iflag & IXON)
 			arg |= DIGI_OUTPUT_FLOW_CONTROL_XON_XOFF;
 		else
 			arg &= ~DIGI_OUTPUT_FLOW_CONTROL_XON_XOFF;
 
-		if (cflag & CRTSCTS) {
+		if (cflag & CRTSCTS)
 			arg |= DIGI_OUTPUT_FLOW_CONTROL_CTS;
-		} else {
+		else
 			arg &= ~DIGI_OUTPUT_FLOW_CONTROL_CTS;
-		}
 
 		buf[i++] = DIGI_CMD_SET_OUTPUT_FLOW_CONTROL;
 		buf[i++] = priv->dp_port_num;
@@ -1084,7 +1083,7 @@ static int digi_chars_in_buffer(struct tty_struct *tty)
 static void digi_dtr_rts(struct usb_serial_port *port, int on)
 {
 	/* Adjust DTR and RTS */
-	digi_set_modem_signals(port, on * (TIOCM_DTR|TIOCM_RTS), 1);
+	digi_set_modem_signals(port, on * (TIOCM_DTR | TIOCM_RTS), 1);
 }
 
 static int digi_open(struct tty_struct *tty, struct usb_serial_port *port)
diff --git a/drivers/usb/serial/f81232.c b/drivers/usb/serial/f81232.c
index 0dcdcb4b2cde..43fa1f0716b7 100644
--- a/drivers/usb/serial/f81232.c
+++ b/drivers/usb/serial/f81232.c
@@ -28,7 +28,8 @@ static const struct usb_device_id id_table[] = {
 MODULE_DEVICE_TABLE(usb, id_table);
 
 /* Maximum baudrate for F81232 */
-#define F81232_MAX_BAUDRATE		115200
+#define F81232_MAX_BAUDRATE		1500000
+#define F81232_DEF_BAUDRATE		9600
 
 /* USB Control EP parameter */
 #define F81232_REGISTER_REQUEST		0xa0
@@ -41,19 +42,46 @@ MODULE_DEVICE_TABLE(usb, id_table);
 #define FIFO_CONTROL_REGISTER		(0x02 + SERIAL_BASE_ADDRESS)
 #define LINE_CONTROL_REGISTER		(0x03 + SERIAL_BASE_ADDRESS)
 #define MODEM_CONTROL_REGISTER		(0x04 + SERIAL_BASE_ADDRESS)
+#define LINE_STATUS_REGISTER		(0x05 + SERIAL_BASE_ADDRESS)
 #define MODEM_STATUS_REGISTER		(0x06 + SERIAL_BASE_ADDRESS)
 
+/*
+ * F81232 Clock registers (106h)
+ *
+ * Bit1-0:	Clock source selector
+ *			00: 1.846MHz.
+ *			01: 18.46MHz.
+ *			10: 24MHz.
+ *			11: 14.77MHz.
+ */
+#define F81232_CLK_REGISTER		0x106
+#define F81232_CLK_1_846_MHZ		0
+#define F81232_CLK_18_46_MHZ		BIT(0)
+#define F81232_CLK_24_MHZ		BIT(1)
+#define F81232_CLK_14_77_MHZ		(BIT(1) | BIT(0))
+#define F81232_CLK_MASK			GENMASK(1, 0)
+
 struct f81232_private {
 	struct mutex lock;
 	u8 modem_control;
 	u8 modem_status;
+	u8 shadow_lcr;
+	speed_t baud_base;
+	struct work_struct lsr_work;
 	struct work_struct interrupt_work;
 	struct usb_serial_port *port;
 };
 
-static int calc_baud_divisor(speed_t baudrate)
+static u32 const baudrate_table[] = { 115200, 921600, 1152000, 1500000 };
+static u8 const clock_table[] = { F81232_CLK_1_846_MHZ, F81232_CLK_14_77_MHZ,
+				F81232_CLK_18_46_MHZ, F81232_CLK_24_MHZ };
+
+static int calc_baud_divisor(speed_t baudrate, speed_t clockrate)
 {
-	return DIV_ROUND_CLOSEST(F81232_MAX_BAUDRATE, baudrate);
+	if (!baudrate)
+		return 0;
+
+	return DIV_ROUND_CLOSEST(clockrate, baudrate);
 }
 
 static int f81232_get_register(struct usb_serial_port *port, u16 reg, u8 *val)
@@ -127,6 +155,21 @@ static int f81232_set_register(struct usb_serial_port *port, u16 reg, u8 val)
 	return status;
 }
 
+static int f81232_set_mask_register(struct usb_serial_port *port, u16 reg,
+					u8 mask, u8 val)
+{
+	int status;
+	u8 tmp;
+
+	status = f81232_get_register(port, reg, &tmp);
+	if (status)
+		return status;
+
+	tmp = (tmp & ~mask) | (val & mask);
+
+	return f81232_set_register(port, reg, tmp);
+}
+
 static void f81232_read_msr(struct usb_serial_port *port)
 {
 	int status;
@@ -282,6 +325,7 @@ exit:
 static void f81232_process_read_urb(struct urb *urb)
 {
 	struct usb_serial_port *port = urb->context;
+	struct f81232_private *priv = usb_get_serial_port_data(port);
 	unsigned char *data = urb->transfer_buffer;
 	char tty_flag;
 	unsigned int i;
@@ -315,6 +359,7 @@ static void f81232_process_read_urb(struct urb *urb)
 
 			if (lsr & UART_LSR_OE) {
 				port->icount.overrun++;
+				schedule_work(&priv->lsr_work);
 				tty_insert_flip_char(&port->port, 0,
 						TTY_OVERRUN);
 			}
@@ -333,22 +378,72 @@ static void f81232_process_read_urb(struct urb *urb)
 
 static void f81232_break_ctl(struct tty_struct *tty, int break_state)
 {
-	/* FIXME - Stubbed out for now */
+	struct usb_serial_port *port = tty->driver_data;
+	struct f81232_private *priv = usb_get_serial_port_data(port);
+	int status;
 
-	/*
-	 * break_state = -1 to turn on break, and 0 to turn off break
-	 * see drivers/char/tty_io.c to see it used.
-	 * last_set_data_urb_value NEVER has the break bit set in it.
-	 */
+	mutex_lock(&priv->lock);
+
+	if (break_state)
+		priv->shadow_lcr |= UART_LCR_SBC;
+	else
+		priv->shadow_lcr &= ~UART_LCR_SBC;
+
+	status = f81232_set_register(port, LINE_CONTROL_REGISTER,
+					priv->shadow_lcr);
+	if (status)
+		dev_err(&port->dev, "set break failed: %d\n", status);
+
+	mutex_unlock(&priv->lock);
 }
 
-static void f81232_set_baudrate(struct usb_serial_port *port, speed_t baudrate)
+static int f81232_find_clk(speed_t baudrate)
 {
+	int idx;
+
+	for (idx = 0; idx < ARRAY_SIZE(baudrate_table); ++idx) {
+		if (baudrate <= baudrate_table[idx] &&
+				baudrate_table[idx] % baudrate == 0)
+			return idx;
+	}
+
+	return -EINVAL;
+}
+
+static void f81232_set_baudrate(struct tty_struct *tty,
+				struct usb_serial_port *port, speed_t baudrate,
+				speed_t old_baudrate)
+{
+	struct f81232_private *priv = usb_get_serial_port_data(port);
 	u8 lcr;
 	int divisor;
 	int status = 0;
+	int i;
+	int idx;
+	speed_t baud_list[] = { baudrate, old_baudrate, F81232_DEF_BAUDRATE };
+
+	for (i = 0; i < ARRAY_SIZE(baud_list); ++i) {
+		idx = f81232_find_clk(baud_list[i]);
+		if (idx >= 0) {
+			baudrate = baud_list[i];
+			tty_encode_baud_rate(tty, baudrate, baudrate);
+			break;
+		}
+	}
 
-	divisor = calc_baud_divisor(baudrate);
+	if (idx < 0)
+		return;
+
+	priv->baud_base = baudrate_table[idx];
+	divisor = calc_baud_divisor(baudrate, priv->baud_base);
+
+	status = f81232_set_mask_register(port, F81232_CLK_REGISTER,
+			F81232_CLK_MASK, clock_table[idx]);
+	if (status) {
+		dev_err(&port->dev, "%s failed to set CLK_REG: %d\n",
+			__func__, status);
+		return;
+	}
 
 	status = f81232_get_register(port, LINE_CONTROL_REGISTER,
 			 &lcr); /* get LCR */
@@ -435,9 +530,11 @@ static int f81232_port_disable(struct usb_serial_port *port)
 static void f81232_set_termios(struct tty_struct *tty,
 		struct usb_serial_port *port, struct ktermios *old_termios)
 {
+	struct f81232_private *priv = usb_get_serial_port_data(port);
 	u8 new_lcr = 0;
 	int status = 0;
 	speed_t baudrate;
+	speed_t old_baud;
 
 	/* Don't change anything if nothing has changed */
 	if (old_termios && !tty_termios_hw_change(&tty->termios, old_termios))
@@ -450,11 +547,12 @@ static void f81232_set_termios(struct tty_struct *tty,
 
 	baudrate = tty_get_baud_rate(tty);
 	if (baudrate > 0) {
-		if (baudrate > F81232_MAX_BAUDRATE) {
-			baudrate = F81232_MAX_BAUDRATE;
-			tty_encode_baud_rate(tty, baudrate, baudrate);
-		}
-		f81232_set_baudrate(port, baudrate);
+		if (old_termios)
+			old_baud = tty_termios_baud_rate(old_termios);
+		else
+			old_baud = F81232_DEF_BAUDRATE;
+
+		f81232_set_baudrate(tty, port, baudrate, old_baud);
 	}
 
 	if (C_PARENB(tty)) {
@@ -486,11 +584,18 @@ static void f81232_set_termios(struct tty_struct *tty,
 		break;
 	}
 
+	mutex_lock(&priv->lock);
+
+	new_lcr |= (priv->shadow_lcr & UART_LCR_SBC);
 	status = f81232_set_register(port, LINE_CONTROL_REGISTER, new_lcr);
 	if (status) {
 		dev_err(&port->dev, "%s failed to set LCR: %d\n",
 			__func__, status);
 	}
+
+	priv->shadow_lcr = new_lcr;
+
+	mutex_unlock(&priv->lock);
 }
 
 static int f81232_tiocmget(struct tty_struct *tty)
@@ -556,9 +661,13 @@ static int f81232_open(struct tty_struct *tty, struct usb_serial_port *port)
 
 static void f81232_close(struct usb_serial_port *port)
 {
+	struct f81232_private *port_priv = usb_get_serial_port_data(port);
+
 	f81232_port_disable(port);
 	usb_serial_generic_close(port);
 	usb_kill_urb(port->interrupt_in_urb);
+	flush_work(&port_priv->interrupt_work);
+	flush_work(&port_priv->lsr_work);
 }
 
 static void f81232_dtr_rts(struct usb_serial_port *port, int on)
@@ -587,11 +696,12 @@ static int f81232_get_serial_info(struct tty_struct *tty,
 		struct serial_struct *ss)
 {
 	struct usb_serial_port *port = tty->driver_data;
+	struct f81232_private *priv = usb_get_serial_port_data(port);
 
 	ss->type = PORT_16550A;
 	ss->line = port->minor;
 	ss->port = port->port_number;
-	ss->baud_base = F81232_MAX_BAUDRATE;
+	ss->baud_base = priv->baud_base;
 	return 0;
 }
 
@@ -603,6 +713,21 @@ static void  f81232_interrupt_work(struct work_struct *work)
 	f81232_read_msr(priv->port);
 }
 
+static void f81232_lsr_worker(struct work_struct *work)
+{
+	struct f81232_private *priv;
+	struct usb_serial_port *port;
+	int status;
+	u8 tmp;
+
+	priv = container_of(work, struct f81232_private, lsr_work);
+	port = priv->port;
+
+	status = f81232_get_register(port, LINE_STATUS_REGISTER, &tmp);
+	if (status)
+		dev_warn(&port->dev, "read LSR failed: %d\n", status);
+}
+
 static int f81232_port_probe(struct usb_serial_port *port)
 {
 	struct f81232_private *priv;
@@ -613,6 +738,7 @@ static int f81232_port_probe(struct usb_serial_port *port)
 
 	mutex_init(&priv->lock);
 	INIT_WORK(&priv->interrupt_work,  f81232_interrupt_work);
+	INIT_WORK(&priv->lsr_work, f81232_lsr_worker);
 
 	usb_set_serial_port_data(port, priv);
 
@@ -632,6 +758,42 @@ static int f81232_port_remove(struct usb_serial_port *port)
 	return 0;
 }
 
+static int f81232_suspend(struct usb_serial *serial, pm_message_t message)
+{
+	struct usb_serial_port *port = serial->port[0];
+	struct f81232_private *port_priv = usb_get_serial_port_data(port);
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(port->read_urbs); ++i)
+		usb_kill_urb(port->read_urbs[i]);
+
+	usb_kill_urb(port->interrupt_in_urb);
+
+	if (port_priv) {
+		flush_work(&port_priv->interrupt_work);
+		flush_work(&port_priv->lsr_work);
+	}
+
+	return 0;
+}
+
+static int f81232_resume(struct usb_serial *serial)
+{
+	struct usb_serial_port *port = serial->port[0];
+	int result;
+
+	if (tty_port_initialized(&port->port)) {
+		result = usb_submit_urb(port->interrupt_in_urb, GFP_NOIO);
+		if (result) {
+			dev_err(&port->dev, "submit interrupt urb failed: %d\n",
+					result);
+			return result;
+		}
+	}
+
+	return usb_serial_generic_resume(serial);
+}
+
 static struct usb_serial_driver f81232_device = {
 	.driver = {
 		.owner =	THIS_MODULE,
@@ -655,6 +817,8 @@ static struct usb_serial_driver f81232_device = {
 	.read_int_callback =	f81232_read_int_callback,
 	.port_probe =		f81232_port_probe,
 	.port_remove =		f81232_port_remove,
+	.suspend =		f81232_suspend,
+	.resume =		f81232_resume,
 };
 
 static struct usb_serial_driver * const serial_drivers[] = {
diff --git a/drivers/usb/serial/generic.c b/drivers/usb/serial/generic.c
index 2274d9625f63..1be8bea372a2 100644
--- a/drivers/usb/serial/generic.c
+++ b/drivers/usb/serial/generic.c
@@ -106,12 +106,8 @@ void usb_serial_generic_deregister(void)
 int usb_serial_generic_open(struct tty_struct *tty, struct usb_serial_port *port)
 {
 	int result = 0;
-	unsigned long flags;
 
-	spin_lock_irqsave(&port->lock, flags);
-	port->throttled = 0;
-	port->throttle_req = 0;
-	spin_unlock_irqrestore(&port->lock, flags);
+	clear_bit(USB_SERIAL_THROTTLED, &port->flags);
 
 	if (port->bulk_in_size)
 		result = usb_serial_generic_submit_read_urbs(port, GFP_KERNEL);
@@ -375,7 +371,7 @@ void usb_serial_generic_read_bulk_callback(struct urb *urb)
 {
 	struct usb_serial_port *port = urb->context;
 	unsigned char *data = urb->transfer_buffer;
-	unsigned long flags;
+	bool stopped = false;
 	int status = urb->status;
 	int i;
 
@@ -383,42 +379,55 @@ void usb_serial_generic_read_bulk_callback(struct urb *urb)
 		if (urb == port->read_urbs[i])
 			break;
 	}
-	set_bit(i, &port->read_urbs_free);
 
 	dev_dbg(&port->dev, "%s - urb %d, len %d\n", __func__, i,
 							urb->actual_length);
 	switch (status) {
 	case 0:
+		usb_serial_debug_data(&port->dev, __func__, urb->actual_length,
+							data);
+		port->serial->type->process_read_urb(urb);
 		break;
 	case -ENOENT:
 	case -ECONNRESET:
 	case -ESHUTDOWN:
 		dev_dbg(&port->dev, "%s - urb stopped: %d\n",
 							__func__, status);
-		return;
+		stopped = true;
+		break;
 	case -EPIPE:
 		dev_err(&port->dev, "%s - urb stopped: %d\n",
 							__func__, status);
-		return;
+		stopped = true;
+		break;
 	default:
 		dev_dbg(&port->dev, "%s - nonzero urb status: %d\n",
 							__func__, status);
-		goto resubmit;
+		break;
 	}
 
-	usb_serial_debug_data(&port->dev, __func__, urb->actual_length, data);
-	port->serial->type->process_read_urb(urb);
+	/*
+	 * Make sure URB processing is done before marking as free to avoid
+	 * racing with unthrottle() on another CPU. Matches the barriers
+	 * implied by the test_and_clear_bit() in
+	 * usb_serial_generic_submit_read_urb().
+	 */
+	smp_mb__before_atomic();
+	set_bit(i, &port->read_urbs_free);
+	/*
+	 * Make sure URB is marked as free before checking the throttled flag
+	 * to avoid racing with unthrottle() on another CPU. Matches the
+	 * smp_mb() in unthrottle().
+	 */
+	smp_mb__after_atomic();
 
-resubmit:
-	/* Throttle the device if requested by tty */
-	spin_lock_irqsave(&port->lock, flags);
-	port->throttled = port->throttle_req;
-	if (!port->throttled) {
-		spin_unlock_irqrestore(&port->lock, flags);
-		usb_serial_generic_submit_read_urb(port, i, GFP_ATOMIC);
-	} else {
-		spin_unlock_irqrestore(&port->lock, flags);
-	}
+	if (stopped)
+		return;
+
+	if (test_bit(USB_SERIAL_THROTTLED, &port->flags))
+		return;
+
+	usb_serial_generic_submit_read_urb(port, i, GFP_ATOMIC);
 }
 EXPORT_SYMBOL_GPL(usb_serial_generic_read_bulk_callback);
 
@@ -454,10 +463,9 @@ void usb_serial_generic_write_bulk_callback(struct urb *urb)
 	default:
 		dev_err_console(port, "%s - nonzero urb status: %d\n",
 							__func__, status);
-		goto resubmit;
+		break;
 	}
 
-resubmit:
 	usb_serial_generic_write_start(port, GFP_ATOMIC);
 	usb_serial_port_softint(port);
 }
@@ -466,26 +474,24 @@ EXPORT_SYMBOL_GPL(usb_serial_generic_write_bulk_callback);
 void usb_serial_generic_throttle(struct tty_struct *tty)
 {
 	struct usb_serial_port *port = tty->driver_data;
-	unsigned long flags;
 
-	spin_lock_irqsave(&port->lock, flags);
-	port->throttle_req = 1;
-	spin_unlock_irqrestore(&port->lock, flags);
+	set_bit(USB_SERIAL_THROTTLED, &port->flags);
 }
 EXPORT_SYMBOL_GPL(usb_serial_generic_throttle);
 
 void usb_serial_generic_unthrottle(struct tty_struct *tty)
 {
 	struct usb_serial_port *port = tty->driver_data;
-	int was_throttled;
 
-	spin_lock_irq(&port->lock);
-	was_throttled = port->throttled;
-	port->throttled = port->throttle_req = 0;
-	spin_unlock_irq(&port->lock);
+	clear_bit(USB_SERIAL_THROTTLED, &port->flags);
+
+	/*
+	 * Matches the smp_mb__after_atomic() in
+	 * usb_serial_generic_read_bulk_callback().
+	 */
+	smp_mb();
 
-	if (was_throttled)
-		usb_serial_generic_submit_read_urbs(port, GFP_KERNEL);
+	usb_serial_generic_submit_read_urbs(port, GFP_KERNEL);
 }
 EXPORT_SYMBOL_GPL(usb_serial_generic_unthrottle);
 
diff --git a/drivers/usb/serial/io_edgeport.c b/drivers/usb/serial/io_edgeport.c
index 4ca31c0e4174..48a439298a68 100644
--- a/drivers/usb/serial/io_edgeport.c
+++ b/drivers/usb/serial/io_edgeport.c
@@ -1751,7 +1751,7 @@ static void process_rcvd_data(struct edgeport_serial *edge_serial,
 				edge_serial->rxState = EXPECT_HDR2;
 				break;
 			}
-			/* otherwise, drop on through */
+			/* Fall through */
 		case EXPECT_HDR2:
 			edge_serial->rxHeader2 = *buffer;
 			++buffer;
@@ -1790,29 +1790,20 @@ static void process_rcvd_data(struct edgeport_serial *edge_serial,
 						edge_serial->rxHeader2, 0);
 				edge_serial->rxState = EXPECT_HDR1;
 				break;
-			} else {
-				edge_serial->rxPort =
-				    IOSP_GET_HDR_PORT(edge_serial->rxHeader1);
-				edge_serial->rxBytesRemaining =
-				    IOSP_GET_HDR_DATA_LEN(
-						edge_serial->rxHeader1,
-						edge_serial->rxHeader2);
-				dev_dbg(dev, "%s - Data for Port %u Len %u\n",
-					__func__,
-					edge_serial->rxPort,
-					edge_serial->rxBytesRemaining);
-
-				/* ASSERT(DevExt->RxPort < DevExt->NumPorts);
-				 * ASSERT(DevExt->RxBytesRemaining <
-				 *		IOSP_MAX_DATA_LENGTH);
-				 */
-
-				if (bufferLength == 0) {
-					edge_serial->rxState = EXPECT_DATA;
-					break;
-				}
-				/* Else, drop through */
 			}
+
+			edge_serial->rxPort = IOSP_GET_HDR_PORT(edge_serial->rxHeader1);
+			edge_serial->rxBytesRemaining = IOSP_GET_HDR_DATA_LEN(edge_serial->rxHeader1,
+									      edge_serial->rxHeader2);
+			dev_dbg(dev, "%s - Data for Port %u Len %u\n", __func__,
+				edge_serial->rxPort,
+				edge_serial->rxBytesRemaining);
+
+			if (bufferLength == 0) {
+				edge_serial->rxState = EXPECT_DATA;
+				break;
+			}
+			/* Fall through */
 		case EXPECT_DATA: /* Expect data */
 			if (bufferLength < edge_serial->rxBytesRemaining) {
 				rxLen = bufferLength;
diff --git a/drivers/usb/serial/iuu_phoenix.c b/drivers/usb/serial/iuu_phoenix.c
index 449e89db9cea..d5bff69b1769 100644
--- a/drivers/usb/serial/iuu_phoenix.c
+++ b/drivers/usb/serial/iuu_phoenix.c
@@ -942,9 +942,7 @@ static void iuu_close(struct usb_serial_port *port)
 
 static void iuu_init_termios(struct tty_struct *tty)
 {
-	tty->termios = tty_std_termios;
-	tty->termios.c_cflag = CLOCAL | CREAD | CS8 | B9600
-				| TIOCM_CTS | CSTOPB | PARENB;
+	tty->termios.c_cflag = B9600 | CS8 | CSTOPB | CREAD | PARENB | CLOCAL;
 	tty->termios.c_ispeed = 9600;
 	tty->termios.c_ospeed = 9600;
 	tty->termios.c_lflag = 0;
diff --git a/drivers/usb/serial/oti6858.c b/drivers/usb/serial/oti6858.c
index ae9cb15ee02d..38ae0fc826cc 100644
--- a/drivers/usb/serial/oti6858.c
+++ b/drivers/usb/serial/oti6858.c
@@ -393,10 +393,7 @@ static int oti6858_chars_in_buffer(struct tty_struct *tty)
 
 static void oti6858_init_termios(struct tty_struct *tty)
 {
-	tty->termios = tty_std_termios;
-	tty->termios.c_cflag = B38400 | CS8 | CREAD | HUPCL | CLOCAL;
-	tty->termios.c_ispeed = 38400;
-	tty->termios.c_ospeed = 38400;
+	tty_encode_baud_rate(tty, 38400, 38400);
 }
 
 static void oti6858_set_termios(struct tty_struct *tty,
diff --git a/drivers/usb/serial/pl2303.c b/drivers/usb/serial/pl2303.c
index bb3f9aa4a909..55122ac84518 100644
--- a/drivers/usb/serial/pl2303.c
+++ b/drivers/usb/serial/pl2303.c
@@ -145,6 +145,8 @@ MODULE_DEVICE_TABLE(usb, id_table);
 #define UART_OVERRUN_ERROR		0x40
 #define UART_CTS			0x80
 
+#define PL2303_FLOWCTRL_MASK		0xf0
+
 static void pl2303_set_break(struct usb_serial_port *port, bool enable);
 
 enum pl2303_type {
@@ -156,6 +158,7 @@ enum pl2303_type {
 struct pl2303_type_data {
 	speed_t max_baud_rate;
 	unsigned long quirks;
+	unsigned int no_autoxonxoff:1;
 };
 
 struct pl2303_serial_private {
@@ -173,11 +176,12 @@ struct pl2303_private {
 
 static const struct pl2303_type_data pl2303_type_data[TYPE_COUNT] = {
 	[TYPE_01] = {
-		.max_baud_rate =	1228800,
-		.quirks =		PL2303_QUIRK_LEGACY,
+		.max_baud_rate		= 1228800,
+		.quirks			= PL2303_QUIRK_LEGACY,
+		.no_autoxonxoff		= true,
 	},
 	[TYPE_HX] = {
-		.max_baud_rate =	12000000,
+		.max_baud_rate		= 12000000,
 	},
 };
 
@@ -223,6 +227,29 @@ static int pl2303_vendor_write(struct usb_serial *serial, u16 value, u16 index)
 	return 0;
 }
 
+static int pl2303_update_reg(struct usb_serial *serial, u8 reg, u8 mask, u8 val)
+{
+	int ret = 0;
+	u8 *buf;
+
+	buf = kmalloc(1, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	ret = pl2303_vendor_read(serial, reg | 0x80, buf);
+	if (ret)
+		goto out_free;
+
+	*buf &= ~mask;
+	*buf |= val & mask;
+
+	ret = pl2303_vendor_write(serial, reg, *buf);
+out_free:
+	kfree(buf);
+
+	return ret;
+}
+
 static int pl2303_probe(struct usb_serial *serial,
 					const struct usb_device_id *id)
 {
@@ -552,6 +579,20 @@ static bool pl2303_termios_change(const struct ktermios *a, const struct ktermio
 	return tty_termios_hw_change(a, b) || ixon_change;
 }
 
+static bool pl2303_enable_xonxoff(struct tty_struct *tty, const struct pl2303_type_data *type)
+{
+	if (!I_IXON(tty) || I_IXANY(tty))
+		return false;
+
+	if (START_CHAR(tty) != 0x11 || STOP_CHAR(tty) != 0x13)
+		return false;
+
+	if (type->no_autoxonxoff)
+		return false;
+
+	return true;
+}
+
 static void pl2303_set_termios(struct tty_struct *tty,
 		struct usb_serial_port *port, struct ktermios *old_termios)
 {
@@ -678,14 +719,13 @@ static void pl2303_set_termios(struct tty_struct *tty,
 
 	if (C_CRTSCTS(tty)) {
 		if (spriv->quirks & PL2303_QUIRK_LEGACY)
-			pl2303_vendor_write(serial, 0x0, 0x41);
+			pl2303_update_reg(serial, 0, PL2303_FLOWCTRL_MASK, 0x40);
 		else
-			pl2303_vendor_write(serial, 0x0, 0x61);
-	} else if (I_IXON(tty) && !I_IXANY(tty) && START_CHAR(tty) == 0x11 &&
-			STOP_CHAR(tty) == 0x13) {
-		pl2303_vendor_write(serial, 0x0, 0xc0);
+			pl2303_update_reg(serial, 0, PL2303_FLOWCTRL_MASK, 0x60);
+	} else if (pl2303_enable_xonxoff(tty, spriv->type)) {
+		pl2303_update_reg(serial, 0, PL2303_FLOWCTRL_MASK, 0xc0);
 	} else {
-		pl2303_vendor_write(serial, 0x0, 0x0);
+		pl2303_update_reg(serial, 0, PL2303_FLOWCTRL_MASK, 0);
 	}
 
 	kfree(buf);
diff --git a/drivers/usb/serial/spcp8x5.c b/drivers/usb/serial/spcp8x5.c
index b42714855364..3bac55bd9bd9 100644
--- a/drivers/usb/serial/spcp8x5.c
+++ b/drivers/usb/serial/spcp8x5.c
@@ -281,10 +281,7 @@ static void spcp8x5_dtr_rts(struct usb_serial_port *port, int on)
 
 static void spcp8x5_init_termios(struct tty_struct *tty)
 {
-	tty->termios = tty_std_termios;
-	tty->termios.c_cflag = B115200 | CS8 | CREAD | HUPCL | CLOCAL;
-	tty->termios.c_ispeed = 115200;
-	tty->termios.c_ospeed = 115200;
+	tty_encode_baud_rate(tty, 115200, 115200);
 }
 
 static void spcp8x5_set_termios(struct tty_struct *tty,
diff --git a/drivers/usb/serial/usb-serial.c b/drivers/usb/serial/usb-serial.c
index 7e89efbf2c28..676c296103a2 100644
--- a/drivers/usb/serial/usb-serial.c
+++ b/drivers/usb/serial/usb-serial.c
@@ -164,9 +164,9 @@ void usb_serial_put(struct usb_serial *serial)
  * @driver: the driver (USB in our case)
  * @tty: the tty being created
  *
- * Create the termios objects for this tty.  We use the default
+ * Initialise the termios structure for this tty.  We use the default
  * USB serial settings but permit them to be overridden by
- * serial->type->init_termios.
+ * serial->type->init_termios on first open.
  *
  * This is the first place a new tty gets used.  Hence this is where we
  * acquire references to the usb_serial structure and the driver module,
@@ -178,6 +178,7 @@ static int serial_install(struct tty_driver *driver, struct tty_struct *tty)
 	int idx = tty->index;
 	struct usb_serial *serial;
 	struct usb_serial_port *port;
+	bool init_termios;
 	int retval = -ENODEV;
 
 	port = usb_serial_port_get_by_minor(idx);
@@ -192,14 +193,16 @@ static int serial_install(struct tty_driver *driver, struct tty_struct *tty)
 	if (retval)
 		goto error_get_interface;
 
+	init_termios = (driver->termios[idx] == NULL);
+
 	retval = tty_standard_install(driver, tty);
 	if (retval)
 		goto error_init_termios;
 
 	mutex_unlock(&serial->disc_mutex);
 
-	/* allow the driver to update the settings */
-	if (serial->type->init_termios)
+	/* allow the driver to update the initial settings */
+	if (init_termios && serial->type->init_termios)
 		serial->type->init_termios(tty);
 
 	tty->driver_data = port;
diff --git a/drivers/usb/storage/scsiglue.c b/drivers/usb/storage/scsiglue.c
index a73ea495d5a7..59190d88fa9f 100644
--- a/drivers/usb/storage/scsiglue.c
+++ b/drivers/usb/storage/scsiglue.c
@@ -65,6 +65,7 @@ static const char* host_info(struct Scsi_Host *host)
 static int slave_alloc (struct scsi_device *sdev)
 {
 	struct us_data *us = host_to_us(sdev->host);
+	int maxp;
 
 	/*
 	 * Set the INQUIRY transfer length to 36.  We don't use any of
@@ -74,20 +75,17 @@ static int slave_alloc (struct scsi_device *sdev)
 	sdev->inquiry_len = 36;
 
 	/*
-	 * USB has unusual DMA-alignment requirements: Although the
-	 * starting address of each scatter-gather element doesn't matter,
-	 * the length of each element except the last must be divisible
-	 * by the Bulk maxpacket value.  There's currently no way to
-	 * express this by block-layer constraints, so we'll cop out
-	 * and simply require addresses to be aligned at 512-byte
-	 * boundaries.  This is okay since most block I/O involves
-	 * hardware sectors that are multiples of 512 bytes in length,
-	 * and since host controllers up through USB 2.0 have maxpacket
-	 * values no larger than 512.
-	 *
-	 * But it doesn't suffice for Wireless USB, where Bulk maxpacket
-	 * values can be as large as 2048.  To make that work properly
-	 * will require changes to the block layer.
+	 * USB has unusual scatter-gather requirements: the length of each
+	 * scatterlist element except the last must be divisible by the
+	 * Bulk maxpacket value.  Fortunately this value is always a
+	 * power of 2.  Inform the block layer about this requirement.
+	 */
+	maxp = usb_maxpacket(us->pusb_dev, us->recv_bulk_pipe, 0);
+	blk_queue_virt_boundary(sdev->request_queue, maxp - 1);
+
+	/*
+	 * Some host controllers may have alignment requirements.
+	 * We'll play it safe by requiring 512-byte alignment always.
 	 */
 	blk_queue_update_dma_alignment(sdev->request_queue, (512 - 1));
 
diff --git a/drivers/usb/storage/sierra_ms.c b/drivers/usb/storage/sierra_ms.c
index 6ac60abd2e15..e605cbc3d8bf 100644
--- a/drivers/usb/storage/sierra_ms.c
+++ b/drivers/usb/storage/sierra_ms.c
@@ -194,8 +194,6 @@ int sierra_ms_init(struct us_data *us)
 		kfree(swocInfo);
 	}
 complete:
-	result = device_create_file(&us->pusb_intf->dev, &dev_attr_truinst);
-
-	return 0;
+	return device_create_file(&us->pusb_intf->dev, &dev_attr_truinst);
 }
 
diff --git a/drivers/usb/storage/uas.c b/drivers/usb/storage/uas.c
index a6d68191c861..047c5922618f 100644
--- a/drivers/usb/storage/uas.c
+++ b/drivers/usb/storage/uas.c
@@ -789,24 +789,33 @@ static int uas_slave_alloc(struct scsi_device *sdev)
 {
 	struct uas_dev_info *devinfo =
 		(struct uas_dev_info *)sdev->host->hostdata;
+	int maxp;
 
 	sdev->hostdata = devinfo;
 
 	/*
-	 * USB has unusual DMA-alignment requirements: Although the
-	 * starting address of each scatter-gather element doesn't matter,
-	 * the length of each element except the last must be divisible
-	 * by the Bulk maxpacket value.  There's currently no way to
-	 * express this by block-layer constraints, so we'll cop out
-	 * and simply require addresses to be aligned at 512-byte
-	 * boundaries.  This is okay since most block I/O involves
-	 * hardware sectors that are multiples of 512 bytes in length,
-	 * and since host controllers up through USB 2.0 have maxpacket
-	 * values no larger than 512.
+	 * We have two requirements here. We must satisfy the requirements
+	 * of the physical HC and the demands of the protocol, as we
+	 * definitely want no additional memory allocation in this path
+	 * ruling out using bounce buffers.
 	 *
-	 * But it doesn't suffice for Wireless USB, where Bulk maxpacket
-	 * values can be as large as 2048.  To make that work properly
-	 * will require changes to the block layer.
+	 * For a transmission on USB to continue we must never send
+	 * a package that is smaller than maxpacket. Hence the length of each
+         * scatterlist element except the last must be divisible by the
+         * Bulk maxpacket value.
+	 * If the HC does not ensure that through SG,
+	 * the upper layer must do that. We must assume nothing
+	 * about the capabilities off the HC, so we use the most
+	 * pessimistic requirement.
+	 */
+
+	maxp = usb_maxpacket(devinfo->udev, devinfo->data_in_pipe, 0);
+	blk_queue_virt_boundary(sdev->request_queue, maxp - 1);
+
+	/*
+	 * The protocol has no requirements on alignment in the strict sense.
+	 * Controllers may or may not have alignment restrictions.
+	 * As this is not exported, we use an extremely conservative guess.
 	 */
 	blk_queue_update_dma_alignment(sdev->request_queue, (512 - 1));
 
diff --git a/drivers/usb/typec/altmodes/Kconfig b/drivers/usb/typec/altmodes/Kconfig
index ef2226eb7a33..187690fd1a5b 100644
--- a/drivers/usb/typec/altmodes/Kconfig
+++ b/drivers/usb/typec/altmodes/Kconfig
@@ -12,4 +12,14 @@ config TYPEC_DP_ALTMODE
 	  To compile this driver as a module, choose M here: the
 	  module will be called typec_displayport.
 
+config TYPEC_NVIDIA_ALTMODE
+	tristate "NVIDIA Alternate Mode driver"
+	depends on TYPEC_DP_ALTMODE
+	help
+	  Latest NVIDIA GPUs support VirtualLink devices. Select this
+	  to enable support for VirtualLink devices with NVIDIA GPUs.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called typec_displayport.
+
 endmenu
diff --git a/drivers/usb/typec/altmodes/Makefile b/drivers/usb/typec/altmodes/Makefile
index eda8456f1c92..45717548b396 100644
--- a/drivers/usb/typec/altmodes/Makefile
+++ b/drivers/usb/typec/altmodes/Makefile
@@ -2,3 +2,5 @@
 
 obj-$(CONFIG_TYPEC_DP_ALTMODE)		+= typec_displayport.o
 typec_displayport-y			:= displayport.o
+obj-$(CONFIG_TYPEC_NVIDIA_ALTMODE)	+= typec_nvidia.o
+typec_nvidia-y				:= nvidia.o
diff --git a/drivers/usb/typec/altmodes/displayport.c b/drivers/usb/typec/altmodes/displayport.c
index 610d790bc9be..4092248a5936 100644
--- a/drivers/usb/typec/altmodes/displayport.c
+++ b/drivers/usb/typec/altmodes/displayport.c
@@ -14,7 +14,7 @@
 #include <linux/usb/pd_vdo.h>
 #include <linux/usb/typec_dp.h>
 
-#define DP_HEADER(cmd)			(VDO(USB_TYPEC_DP_SID, 1, cmd) | \
+#define DP_HEADER(_dp, cmd)		(VDO((_dp)->alt->svid, 1, cmd) | \
 					 VDO_OPOS(USB_TYPEC_DP_MODE))
 
 enum {
@@ -100,7 +100,7 @@ static int dp_altmode_configure(struct dp_altmode *dp, u8 con)
 		if (dp->data.status & DP_STATUS_PREFER_MULTI_FUNC &&
 		    pin_assign & DP_PIN_ASSIGN_MULTI_FUNC_MASK)
 			pin_assign &= DP_PIN_ASSIGN_MULTI_FUNC_MASK;
-		else
+		else if (pin_assign & DP_PIN_ASSIGN_DP_ONLY_MASK)
 			pin_assign &= DP_PIN_ASSIGN_DP_ONLY_MASK;
 
 		if (!pin_assign)
@@ -155,7 +155,7 @@ static int dp_altmode_configured(struct dp_altmode *dp)
 
 static int dp_altmode_configure_vdm(struct dp_altmode *dp, u32 conf)
 {
-	u32 header = DP_HEADER(DP_CMD_CONFIGURE);
+	u32 header = DP_HEADER(dp, DP_CMD_CONFIGURE);
 	int ret;
 
 	ret = typec_altmode_notify(dp->alt, TYPEC_STATE_SAFE, &dp->data);
@@ -193,7 +193,7 @@ static void dp_altmode_work(struct work_struct *work)
 			dev_err(&dp->alt->dev, "failed to enter mode\n");
 		break;
 	case DP_STATE_UPDATE:
-		header = DP_HEADER(DP_CMD_STATUS_UPDATE);
+		header = DP_HEADER(dp, DP_CMD_STATUS_UPDATE);
 		vdo = 1;
 		ret = typec_altmode_vdm(dp->alt, header, &vdo, 2);
 		if (ret)
@@ -507,7 +507,7 @@ static const struct attribute_group dp_altmode_group = {
 	.attrs = dp_altmode_attrs,
 };
 
-static int dp_altmode_probe(struct typec_altmode *alt)
+int dp_altmode_probe(struct typec_altmode *alt)
 {
 	const struct typec_altmode *port = typec_altmode_get_partner(alt);
 	struct dp_altmode *dp;
@@ -545,14 +545,16 @@ static int dp_altmode_probe(struct typec_altmode *alt)
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(dp_altmode_probe);
 
-static void dp_altmode_remove(struct typec_altmode *alt)
+void dp_altmode_remove(struct typec_altmode *alt)
 {
 	struct dp_altmode *dp = typec_altmode_get_drvdata(alt);
 
 	sysfs_remove_group(&alt->dev.kobj, &dp_altmode_group);
 	cancel_work_sync(&dp->work);
 }
+EXPORT_SYMBOL_GPL(dp_altmode_remove);
 
 static const struct typec_device_id dp_typec_id[] = {
 	{ USB_TYPEC_DP_SID, USB_TYPEC_DP_MODE },
diff --git a/drivers/usb/typec/altmodes/displayport.h b/drivers/usb/typec/altmodes/displayport.h
new file mode 100644
index 000000000000..e120364da9fd
--- /dev/null
+++ b/drivers/usb/typec/altmodes/displayport.h
@@ -0,0 +1,8 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#if IS_ENABLED(CONFIG_TYPEC_DP_ALTMODE)
+int dp_altmode_probe(struct typec_altmode *alt);
+void dp_altmode_remove(struct typec_altmode *alt);
+#else
+int dp_altmode_probe(struct typec_altmode *alt) { return -ENOTSUPP; }
+void dp_altmode_remove(struct typec_altmode *alt) { }
+#endif /* CONFIG_TYPEC_DP_ALTMODE */
diff --git a/drivers/usb/typec/altmodes/nvidia.c b/drivers/usb/typec/altmodes/nvidia.c
new file mode 100644
index 000000000000..c36769736405
--- /dev/null
+++ b/drivers/usb/typec/altmodes/nvidia.c
@@ -0,0 +1,44 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2019 NVIDIA Corporation. All rights reserved.
+ *
+ * NVIDIA USB Type-C Alt Mode Driver
+ */
+#include <linux/module.h>
+#include <linux/usb/typec_altmode.h>
+#include <linux/usb/typec_dp.h>
+#include "displayport.h"
+
+static int nvidia_altmode_probe(struct typec_altmode *alt)
+{
+	if (alt->svid == USB_TYPEC_NVIDIA_VLINK_SID)
+		return dp_altmode_probe(alt);
+	else
+		return -ENOTSUPP;
+}
+
+static void nvidia_altmode_remove(struct typec_altmode *alt)
+{
+	if (alt->svid == USB_TYPEC_NVIDIA_VLINK_SID)
+		dp_altmode_remove(alt);
+}
+
+static const struct typec_device_id nvidia_typec_id[] = {
+	{ USB_TYPEC_NVIDIA_VLINK_SID, TYPEC_ANY_MODE },
+	{ },
+};
+MODULE_DEVICE_TABLE(typec, nvidia_typec_id);
+
+static struct typec_altmode_driver nvidia_altmode_driver = {
+	.id_table = nvidia_typec_id,
+	.probe = nvidia_altmode_probe,
+	.remove = nvidia_altmode_remove,
+	.driver = {
+		.name = "typec_nvidia",
+		.owner = THIS_MODULE,
+	},
+};
+module_typec_altmode_driver(nvidia_altmode_driver);
+
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("NVIDIA USB Type-C Alt Mode Driver");
diff --git a/drivers/usb/typec/mux/pi3usb30532.c b/drivers/usb/typec/mux/pi3usb30532.c
index 64eb5983e17a..9294e85fd34b 100644
--- a/drivers/usb/typec/mux/pi3usb30532.c
+++ b/drivers/usb/typec/mux/pi3usb30532.c
@@ -84,7 +84,8 @@ static int pi3usb30532_mux_set(struct typec_mux *mux, int state)
 
 	switch (state) {
 	case TYPEC_STATE_SAFE:
-		new_conf = PI3USB30532_CONF_OPEN;
+		new_conf = (new_conf & PI3USB30532_CONF_SWAP) |
+			   PI3USB30532_CONF_OPEN;
 		break;
 	case TYPEC_STATE_USB:
 		new_conf = (new_conf & PI3USB30532_CONF_SWAP) |
diff --git a/drivers/usb/typec/tcpm/fusb302.c b/drivers/usb/typec/tcpm/fusb302.c
index e9344997329c..7302f7501ec9 100644
--- a/drivers/usb/typec/tcpm/fusb302.c
+++ b/drivers/usb/typec/tcpm/fusb302.c
@@ -23,6 +23,7 @@
 #include <linux/sched/clock.h>
 #include <linux/seq_file.h>
 #include <linux/slab.h>
+#include <linux/spinlock.h>
 #include <linux/string.h>
 #include <linux/types.h>
 #include <linux/usb/typec.h>
@@ -78,6 +79,10 @@ struct fusb302_chip {
 
 	struct regulator *vbus;
 
+	spinlock_t irq_lock;
+	struct work_struct irq_work;
+	bool irq_suspended;
+	bool irq_while_suspended;
 	int gpio_int_n;
 	int gpio_int_n_irq;
 	struct extcon_dev *extcon;
@@ -85,9 +90,6 @@ struct fusb302_chip {
 	struct workqueue_struct *wq;
 	struct delayed_work bc_lvl_handler;
 
-	atomic_t pm_suspend;
-	atomic_t i2c_busy;
-
 	/* lock for sharing chip states */
 	struct mutex lock;
 
@@ -99,7 +101,6 @@ struct fusb302_chip {
 	bool intr_comp_chng;
 
 	/* port status */
-	bool pull_up;
 	bool vconn_on;
 	bool vbus_on;
 	bool charge_on;
@@ -124,13 +125,13 @@ struct fusb302_chip {
  */
 
 #ifdef CONFIG_DEBUG_FS
-
 static bool fusb302_log_full(struct fusb302_chip *chip)
 {
 	return chip->logbuffer_tail ==
 		(chip->logbuffer_head + 1) % LOG_BUFFER_ENTRIES;
 }
 
+__printf(2, 0)
 static void _fusb302_log(struct fusb302_chip *chip, const char *fmt,
 			 va_list args)
 {
@@ -234,43 +235,15 @@ static void fusb302_debugfs_exit(const struct fusb302_chip *chip) { }
 
 #endif
 
-#define FUSB302_RESUME_RETRY 10
-#define FUSB302_RESUME_RETRY_SLEEP 50
-
-static bool fusb302_is_suspended(struct fusb302_chip *chip)
-{
-	int retry_cnt;
-
-	for (retry_cnt = 0; retry_cnt < FUSB302_RESUME_RETRY; retry_cnt++) {
-		if (atomic_read(&chip->pm_suspend)) {
-			dev_err(chip->dev, "i2c: pm suspend, retry %d/%d\n",
-				retry_cnt + 1, FUSB302_RESUME_RETRY);
-			msleep(FUSB302_RESUME_RETRY_SLEEP);
-		} else {
-			return false;
-		}
-	}
-
-	return true;
-}
-
 static int fusb302_i2c_write(struct fusb302_chip *chip,
 			     u8 address, u8 data)
 {
 	int ret = 0;
 
-	atomic_set(&chip->i2c_busy, 1);
-
-	if (fusb302_is_suspended(chip)) {
-		atomic_set(&chip->i2c_busy, 0);
-		return -ETIMEDOUT;
-	}
-
 	ret = i2c_smbus_write_byte_data(chip->i2c_client, address, data);
 	if (ret < 0)
 		fusb302_log(chip, "cannot write 0x%02x to 0x%02x, ret=%d",
 			    data, address, ret);
-	atomic_set(&chip->i2c_busy, 0);
 
 	return ret;
 }
@@ -282,19 +255,12 @@ static int fusb302_i2c_block_write(struct fusb302_chip *chip, u8 address,
 
 	if (length <= 0)
 		return ret;
-	atomic_set(&chip->i2c_busy, 1);
-
-	if (fusb302_is_suspended(chip)) {
-		atomic_set(&chip->i2c_busy, 0);
-		return -ETIMEDOUT;
-	}
 
 	ret = i2c_smbus_write_i2c_block_data(chip->i2c_client, address,
 					     length, data);
 	if (ret < 0)
 		fusb302_log(chip, "cannot block write 0x%02x, len=%d, ret=%d",
 			    address, length, ret);
-	atomic_set(&chip->i2c_busy, 0);
 
 	return ret;
 }
@@ -304,18 +270,10 @@ static int fusb302_i2c_read(struct fusb302_chip *chip,
 {
 	int ret = 0;
 
-	atomic_set(&chip->i2c_busy, 1);
-
-	if (fusb302_is_suspended(chip)) {
-		atomic_set(&chip->i2c_busy, 0);
-		return -ETIMEDOUT;
-	}
-
 	ret = i2c_smbus_read_byte_data(chip->i2c_client, address);
 	*data = (u8)ret;
 	if (ret < 0)
 		fusb302_log(chip, "cannot read %02x, ret=%d", address, ret);
-	atomic_set(&chip->i2c_busy, 0);
 
 	return ret;
 }
@@ -327,12 +285,6 @@ static int fusb302_i2c_block_read(struct fusb302_chip *chip, u8 address,
 
 	if (length <= 0)
 		return ret;
-	atomic_set(&chip->i2c_busy, 1);
-
-	if (fusb302_is_suspended(chip)) {
-		atomic_set(&chip->i2c_busy, 0);
-		return -ETIMEDOUT;
-	}
 
 	ret = i2c_smbus_read_i2c_block_data(chip->i2c_client, address,
 					    length, data);
@@ -348,8 +300,6 @@ static int fusb302_i2c_block_read(struct fusb302_chip *chip, u8 address,
 	}
 
 done:
-	atomic_set(&chip->i2c_busy, 0);
-
 	return ret;
 }
 
@@ -519,32 +469,6 @@ static int tcpm_get_current_limit(struct tcpc_dev *dev)
 	return current_limit;
 }
 
-static int fusb302_set_cc_pull(struct fusb302_chip *chip,
-			       bool pull_up, bool pull_down)
-{
-	int ret = 0;
-	u8 data = 0x00;
-	u8 mask = FUSB_REG_SWITCHES0_CC1_PU_EN |
-		  FUSB_REG_SWITCHES0_CC2_PU_EN |
-		  FUSB_REG_SWITCHES0_CC1_PD_EN |
-		  FUSB_REG_SWITCHES0_CC2_PD_EN;
-
-	if (pull_up)
-		data |= (chip->cc_polarity == TYPEC_POLARITY_CC1) ?
-			FUSB_REG_SWITCHES0_CC1_PU_EN :
-			FUSB_REG_SWITCHES0_CC2_PU_EN;
-	if (pull_down)
-		data |= FUSB_REG_SWITCHES0_CC1_PD_EN |
-			FUSB_REG_SWITCHES0_CC2_PD_EN;
-	ret = fusb302_i2c_mask_write(chip, FUSB_REG_SWITCHES0,
-				     mask, data);
-	if (ret < 0)
-		return ret;
-	chip->pull_up = pull_up;
-
-	return ret;
-}
-
 static int fusb302_set_src_current(struct fusb302_chip *chip,
 				   enum src_current_status status)
 {
@@ -634,6 +558,8 @@ static int fusb302_set_toggling(struct fusb302_chip *chip,
 			return ret;
 		chip->intr_togdone = false;
 	} else {
+		/* Datasheet says vconn MUST be off when toggling */
+		WARN(chip->vconn_on, "Vconn is on during toggle start");
 		/* unmask TOGDONE interrupt */
 		ret = fusb302_i2c_clear_bits(chip, FUSB_REG_MASKA,
 					     FUSB_REG_MASKA_TOGDONE);
@@ -676,26 +602,27 @@ static int tcpm_set_cc(struct tcpc_dev *dev, enum typec_cc_status cc)
 {
 	struct fusb302_chip *chip = container_of(dev, struct fusb302_chip,
 						 tcpc_dev);
+	u8 switches0_mask = FUSB_REG_SWITCHES0_CC1_PU_EN |
+			    FUSB_REG_SWITCHES0_CC2_PU_EN |
+			    FUSB_REG_SWITCHES0_CC1_PD_EN |
+			    FUSB_REG_SWITCHES0_CC2_PD_EN;
+	u8 rd_mda, switches0_data = 0x00;
 	int ret = 0;
-	bool pull_up, pull_down;
-	u8 rd_mda;
-	enum toggling_mode mode;
 
 	mutex_lock(&chip->lock);
 	switch (cc) {
 	case TYPEC_CC_OPEN:
-		pull_up = false;
-		pull_down = false;
 		break;
 	case TYPEC_CC_RD:
-		pull_up = false;
-		pull_down = true;
+		switches0_data |= FUSB_REG_SWITCHES0_CC1_PD_EN |
+				  FUSB_REG_SWITCHES0_CC2_PD_EN;
 		break;
 	case TYPEC_CC_RP_DEF:
 	case TYPEC_CC_RP_1_5:
 	case TYPEC_CC_RP_3_0:
-		pull_up = true;
-		pull_down = false;
+		switches0_data |= (chip->cc_polarity == TYPEC_POLARITY_CC1) ?
+				  FUSB_REG_SWITCHES0_CC1_PU_EN :
+				  FUSB_REG_SWITCHES0_CC2_PU_EN;
 		break;
 	default:
 		fusb302_log(chip, "unsupported cc value %s",
@@ -703,34 +630,38 @@ static int tcpm_set_cc(struct tcpc_dev *dev, enum typec_cc_status cc)
 		ret = -EINVAL;
 		goto done;
 	}
+
+	fusb302_log(chip, "cc := %s", typec_cc_status_name[cc]);
+
 	ret = fusb302_set_toggling(chip, TOGGLING_MODE_OFF);
 	if (ret < 0) {
-		fusb302_log(chip, "cannot stop toggling, ret=%d", ret);
+		fusb302_log(chip, "cannot set toggling mode, ret=%d", ret);
 		goto done;
 	}
-	ret = fusb302_set_cc_pull(chip, pull_up, pull_down);
+
+	ret = fusb302_i2c_mask_write(chip, FUSB_REG_SWITCHES0,
+				     switches0_mask, switches0_data);
 	if (ret < 0) {
-		fusb302_log(chip,
-			    "cannot set cc pulling up %s, down %s, ret = %d",
-			    pull_up ? "True" : "False",
-			    pull_down ? "True" : "False",
-			    ret);
+		fusb302_log(chip, "cannot set pull-up/-down, ret = %d", ret);
 		goto done;
 	}
 	/* reset the cc status */
 	chip->cc1 = TYPEC_CC_OPEN;
 	chip->cc2 = TYPEC_CC_OPEN;
+
 	/* adjust current for SRC */
-	if (pull_up) {
-		ret = fusb302_set_src_current(chip, cc_src_current[cc]);
-		if (ret < 0) {
-			fusb302_log(chip, "cannot set src current %s, ret=%d",
-				    typec_cc_status_name[cc], ret);
-			goto done;
-		}
+	ret = fusb302_set_src_current(chip, cc_src_current[cc]);
+	if (ret < 0) {
+		fusb302_log(chip, "cannot set src current %s, ret=%d",
+			    typec_cc_status_name[cc], ret);
+		goto done;
 	}
+
 	/* enable/disable interrupts, BC_LVL for SNK and COMP_CHNG for SRC */
-	if (pull_up) {
+	switch (cc) {
+	case TYPEC_CC_RP_DEF:
+	case TYPEC_CC_RP_1_5:
+	case TYPEC_CC_RP_3_0:
 		rd_mda = rd_mda_value[cc_src_current[cc]];
 		ret = fusb302_i2c_write(chip, FUSB_REG_MEASURE, rd_mda);
 		if (ret < 0) {
@@ -748,10 +679,9 @@ static int tcpm_set_cc(struct tcpc_dev *dev, enum typec_cc_status cc)
 				    ret);
 			goto done;
 		}
-		chip->intr_bc_lvl = false;
 		chip->intr_comp_chng = true;
-	}
-	if (pull_down) {
+		break;
+	case TYPEC_CC_RD:
 		ret = fusb302_i2c_mask_write(chip, FUSB_REG_MASK,
 					     FUSB_REG_MASK_BC_LVL |
 					     FUSB_REG_MASK_COMP_CHNG,
@@ -762,32 +692,10 @@ static int tcpm_set_cc(struct tcpc_dev *dev, enum typec_cc_status cc)
 			goto done;
 		}
 		chip->intr_bc_lvl = true;
-		chip->intr_comp_chng = false;
-	}
-	fusb302_log(chip, "cc := %s", typec_cc_status_name[cc]);
-
-	/* Enable detection for fixed SNK or SRC only roles */
-	switch (cc) {
-	case TYPEC_CC_RD:
-		mode = TOGGLING_MODE_SNK;
-		break;
-	case TYPEC_CC_RP_DEF:
-	case TYPEC_CC_RP_1_5:
-	case TYPEC_CC_RP_3_0:
-		mode = TOGGLING_MODE_SRC;
 		break;
 	default:
-		mode = TOGGLING_MODE_OFF;
 		break;
 	}
-
-	if (mode != TOGGLING_MODE_OFF) {
-		ret = fusb302_set_toggling(chip, mode);
-		if (ret < 0)
-			fusb302_log(chip,
-				    "cannot set fixed role toggling mode, ret=%d",
-				    ret);
-	}
 done:
 	mutex_unlock(&chip->lock);
 
@@ -1005,13 +913,27 @@ done:
 	return ret;
 }
 
-static int tcpm_start_drp_toggling(struct tcpc_dev *dev,
-				   enum typec_cc_status cc)
+static int tcpm_start_toggling(struct tcpc_dev *dev,
+			       enum typec_port_type port_type,
+			       enum typec_cc_status cc)
 {
 	struct fusb302_chip *chip = container_of(dev, struct fusb302_chip,
 						 tcpc_dev);
+	enum toggling_mode mode = TOGGLING_MODE_OFF;
 	int ret = 0;
 
+	switch (port_type) {
+	case TYPEC_PORT_SRC:
+		mode = TOGGLING_MODE_SRC;
+		break;
+	case TYPEC_PORT_SNK:
+		mode = TOGGLING_MODE_SNK;
+		break;
+	case TYPEC_PORT_DRP:
+		mode = TOGGLING_MODE_DRP;
+		break;
+	}
+
 	mutex_lock(&chip->lock);
 	ret = fusb302_set_src_current(chip, cc_src_current[cc]);
 	if (ret < 0) {
@@ -1019,7 +941,7 @@ static int tcpm_start_drp_toggling(struct tcpc_dev *dev,
 			    typec_cc_status_name[cc], ret);
 		goto done;
 	}
-	ret = fusb302_set_toggling(chip, TOGGLING_MODE_DRP);
+	ret = fusb302_set_toggling(chip, mode);
 	if (ret < 0) {
 		fusb302_log(chip,
 			    "unable to start drp toggling, ret=%d", ret);
@@ -1217,7 +1139,7 @@ static void init_tcpc_dev(struct tcpc_dev *fusb302_tcpc_dev)
 	fusb302_tcpc_dev->set_vbus = tcpm_set_vbus;
 	fusb302_tcpc_dev->set_pd_rx = tcpm_set_pd_rx;
 	fusb302_tcpc_dev->set_roles = tcpm_set_roles;
-	fusb302_tcpc_dev->start_drp_toggling = tcpm_start_drp_toggling;
+	fusb302_tcpc_dev->start_toggling = tcpm_start_toggling;
 	fusb302_tcpc_dev->pd_transmit = tcpm_pd_transmit;
 }
 
@@ -1226,38 +1148,36 @@ static const char * const cc_polarity_name[] = {
 	[TYPEC_POLARITY_CC2]	= "Polarity_CC2",
 };
 
-static int fusb302_set_cc_polarity(struct fusb302_chip *chip,
-				   enum typec_cc_polarity cc_polarity)
+static int fusb302_set_cc_polarity_and_pull(struct fusb302_chip *chip,
+					    enum typec_cc_polarity cc_polarity,
+					    bool pull_up, bool pull_down)
 {
 	int ret = 0;
-	u8 switches0_mask = FUSB_REG_SWITCHES0_CC1_PU_EN |
-			    FUSB_REG_SWITCHES0_CC2_PU_EN |
-			    FUSB_REG_SWITCHES0_VCONN_CC1 |
-			    FUSB_REG_SWITCHES0_VCONN_CC2 |
-			    FUSB_REG_SWITCHES0_MEAS_CC1 |
-			    FUSB_REG_SWITCHES0_MEAS_CC2;
 	u8 switches0_data = 0x00;
 	u8 switches1_mask = FUSB_REG_SWITCHES1_TXCC1_EN |
 			    FUSB_REG_SWITCHES1_TXCC2_EN;
 	u8 switches1_data = 0x00;
 
+	if (pull_down)
+		switches0_data |= FUSB_REG_SWITCHES0_CC1_PD_EN |
+				  FUSB_REG_SWITCHES0_CC2_PD_EN;
+
 	if (cc_polarity == TYPEC_POLARITY_CC1) {
-		switches0_data = FUSB_REG_SWITCHES0_MEAS_CC1;
+		switches0_data |= FUSB_REG_SWITCHES0_MEAS_CC1;
 		if (chip->vconn_on)
 			switches0_data |= FUSB_REG_SWITCHES0_VCONN_CC2;
-		if (chip->pull_up)
+		if (pull_up)
 			switches0_data |= FUSB_REG_SWITCHES0_CC1_PU_EN;
 		switches1_data = FUSB_REG_SWITCHES1_TXCC1_EN;
 	} else {
-		switches0_data = FUSB_REG_SWITCHES0_MEAS_CC2;
+		switches0_data |= FUSB_REG_SWITCHES0_MEAS_CC2;
 		if (chip->vconn_on)
 			switches0_data |= FUSB_REG_SWITCHES0_VCONN_CC1;
-		if (chip->pull_up)
+		if (pull_up)
 			switches0_data |= FUSB_REG_SWITCHES0_CC2_PU_EN;
 		switches1_data = FUSB_REG_SWITCHES1_TXCC2_EN;
 	}
-	ret = fusb302_i2c_mask_write(chip, FUSB_REG_SWITCHES0,
-				     switches0_mask, switches0_data);
+	ret = fusb302_i2c_write(chip, FUSB_REG_SWITCHES0, switches0_data);
 	if (ret < 0)
 		return ret;
 	ret = fusb302_i2c_mask_write(chip, FUSB_REG_SWITCHES1,
@@ -1278,16 +1198,10 @@ static int fusb302_handle_togdone_snk(struct fusb302_chip *chip,
 	enum typec_cc_polarity cc_polarity;
 	enum typec_cc_status cc_status_active, cc1, cc2;
 
-	/* set pull_up, pull_down */
-	ret = fusb302_set_cc_pull(chip, false, true);
-	if (ret < 0) {
-		fusb302_log(chip, "cannot set cc to pull down, ret=%d", ret);
-		return ret;
-	}
-	/* set polarity */
+	/* set polarity and pull_up, pull_down */
 	cc_polarity = (togdone_result == FUSB_REG_STATUS1A_TOGSS_SNK1) ?
 		      TYPEC_POLARITY_CC1 : TYPEC_POLARITY_CC2;
-	ret = fusb302_set_cc_polarity(chip, cc_polarity);
+	ret = fusb302_set_cc_polarity_and_pull(chip, cc_polarity, false, true);
 	if (ret < 0) {
 		fusb302_log(chip, "cannot set cc polarity %s, ret=%d",
 			    cc_polarity_name[cc_polarity], ret);
@@ -1337,6 +1251,62 @@ static int fusb302_handle_togdone_snk(struct fusb302_chip *chip,
 	return ret;
 }
 
+/* On error returns < 0, otherwise a typec_cc_status value */
+static int fusb302_get_src_cc_status(struct fusb302_chip *chip,
+				     enum typec_cc_polarity cc_polarity,
+				     enum typec_cc_status *cc)
+{
+	u8 ra_mda = ra_mda_value[chip->src_current_status];
+	u8 rd_mda = rd_mda_value[chip->src_current_status];
+	u8 switches0_data, status0;
+	int ret;
+
+	/* Step 1: Set switches so that we measure the right CC pin */
+	switches0_data = (cc_polarity == TYPEC_POLARITY_CC1) ?
+		FUSB_REG_SWITCHES0_CC1_PU_EN | FUSB_REG_SWITCHES0_MEAS_CC1 :
+		FUSB_REG_SWITCHES0_CC2_PU_EN | FUSB_REG_SWITCHES0_MEAS_CC2;
+	ret = fusb302_i2c_write(chip, FUSB_REG_SWITCHES0, switches0_data);
+	if (ret < 0)
+		return ret;
+
+	fusb302_i2c_read(chip, FUSB_REG_SWITCHES0, &status0);
+	fusb302_log(chip, "get_src_cc_status switches: 0x%0x", status0);
+
+	/* Step 2: Set compararator volt to differentiate between Open and Rd */
+	ret = fusb302_i2c_write(chip, FUSB_REG_MEASURE, rd_mda);
+	if (ret < 0)
+		return ret;
+
+	usleep_range(50, 100);
+	ret = fusb302_i2c_read(chip, FUSB_REG_STATUS0, &status0);
+	if (ret < 0)
+		return ret;
+
+	fusb302_log(chip, "get_src_cc_status rd_mda status0: 0x%0x", status0);
+	if (status0 & FUSB_REG_STATUS0_COMP) {
+		*cc = TYPEC_CC_OPEN;
+		return 0;
+	}
+
+	/* Step 3: Set compararator input to differentiate between Rd and Ra. */
+	ret = fusb302_i2c_write(chip, FUSB_REG_MEASURE, ra_mda);
+	if (ret < 0)
+		return ret;
+
+	usleep_range(50, 100);
+	ret = fusb302_i2c_read(chip, FUSB_REG_STATUS0, &status0);
+	if (ret < 0)
+		return ret;
+
+	fusb302_log(chip, "get_src_cc_status ra_mda status0: 0x%0x", status0);
+	if (status0 & FUSB_REG_STATUS0_COMP)
+		*cc = TYPEC_CC_RD;
+	else
+		*cc = TYPEC_CC_RA;
+
+	return 0;
+}
+
 static int fusb302_handle_togdone_src(struct fusb302_chip *chip,
 				      u8 togdone_result)
 {
@@ -1347,77 +1317,62 @@ static int fusb302_handle_togdone_src(struct fusb302_chip *chip,
 	 * - set I_COMP interrupt on
 	 */
 	int ret = 0;
-	u8 status0;
-	u8 ra_mda = ra_mda_value[chip->src_current_status];
 	u8 rd_mda = rd_mda_value[chip->src_current_status];
-	bool ra_comp, rd_comp;
+	enum toggling_mode toggling_mode = chip->toggling_mode;
 	enum typec_cc_polarity cc_polarity;
-	enum typec_cc_status cc_status_active, cc1, cc2;
+	enum typec_cc_status cc1, cc2;
 
-	/* set pull_up, pull_down */
-	ret = fusb302_set_cc_pull(chip, true, false);
-	if (ret < 0) {
-		fusb302_log(chip, "cannot set cc to pull up, ret=%d", ret);
+	/*
+	 * The toggle-engine will stop in a src state if it sees either Ra or
+	 * Rd. Determine the status for both CC pins, starting with the one
+	 * where toggling stopped, as that is where the switches point now.
+	 */
+	if (togdone_result == FUSB_REG_STATUS1A_TOGSS_SRC1)
+		ret = fusb302_get_src_cc_status(chip, TYPEC_POLARITY_CC1, &cc1);
+	else
+		ret = fusb302_get_src_cc_status(chip, TYPEC_POLARITY_CC2, &cc2);
+	if (ret < 0)
 		return ret;
-	}
-	/* set polarity */
-	cc_polarity = (togdone_result == FUSB_REG_STATUS1A_TOGSS_SRC1) ?
-		      TYPEC_POLARITY_CC1 : TYPEC_POLARITY_CC2;
-	ret = fusb302_set_cc_polarity(chip, cc_polarity);
+	/* we must turn off toggling before we can measure the other pin */
+	ret = fusb302_set_toggling(chip, TOGGLING_MODE_OFF);
 	if (ret < 0) {
-		fusb302_log(chip, "cannot set cc polarity %s, ret=%d",
-			    cc_polarity_name[cc_polarity], ret);
+		fusb302_log(chip, "cannot set toggling mode off, ret=%d", ret);
 		return ret;
 	}
-	/* fusb302_set_cc_polarity() has set the correct measure block */
-	ret = fusb302_i2c_write(chip, FUSB_REG_MEASURE, rd_mda);
-	if (ret < 0)
-		return ret;
-	usleep_range(50, 100);
-	ret = fusb302_i2c_read(chip, FUSB_REG_STATUS0, &status0);
+	/* get the status of the other pin */
+	if (togdone_result == FUSB_REG_STATUS1A_TOGSS_SRC1)
+		ret = fusb302_get_src_cc_status(chip, TYPEC_POLARITY_CC2, &cc2);
+	else
+		ret = fusb302_get_src_cc_status(chip, TYPEC_POLARITY_CC1, &cc1);
 	if (ret < 0)
 		return ret;
-	rd_comp = !!(status0 & FUSB_REG_STATUS0_COMP);
-	if (!rd_comp) {
-		ret = fusb302_i2c_write(chip, FUSB_REG_MEASURE, ra_mda);
-		if (ret < 0)
-			return ret;
-		usleep_range(50, 100);
-		ret = fusb302_i2c_read(chip, FUSB_REG_STATUS0, &status0);
-		if (ret < 0)
-			return ret;
-		ra_comp = !!(status0 & FUSB_REG_STATUS0_COMP);
+
+	/* determine polarity based on the status of both pins */
+	if (cc1 == TYPEC_CC_RD &&
+			(cc2 == TYPEC_CC_OPEN || cc2 == TYPEC_CC_RA)) {
+		cc_polarity = TYPEC_POLARITY_CC1;
+	} else if (cc2 == TYPEC_CC_RD &&
+		    (cc1 == TYPEC_CC_OPEN || cc1 == TYPEC_CC_RA)) {
+		cc_polarity = TYPEC_POLARITY_CC2;
+	} else {
+		fusb302_log(chip, "unexpected CC status cc1=%s, cc2=%s, restarting toggling",
+			    typec_cc_status_name[cc1],
+			    typec_cc_status_name[cc2]);
+		return fusb302_set_toggling(chip, toggling_mode);
 	}
-	if (rd_comp)
-		cc_status_active = TYPEC_CC_OPEN;
-	else if (ra_comp)
-		cc_status_active = TYPEC_CC_RD;
-	else
-		/* Ra is not supported, report as Open */
-		cc_status_active = TYPEC_CC_OPEN;
-	/* restart toggling if the cc status on the active line is OPEN */
-	if (cc_status_active == TYPEC_CC_OPEN) {
-		fusb302_log(chip, "restart toggling as CC_OPEN detected");
-		ret = fusb302_set_toggling(chip, chip->toggling_mode);
+	/* set polarity and pull_up, pull_down */
+	ret = fusb302_set_cc_polarity_and_pull(chip, cc_polarity, true, false);
+	if (ret < 0) {
+		fusb302_log(chip, "cannot set cc polarity %s, ret=%d",
+			    cc_polarity_name[cc_polarity], ret);
 		return ret;
 	}
 	/* update tcpm with the new cc value */
-	cc1 = (cc_polarity == TYPEC_POLARITY_CC1) ?
-	      cc_status_active : TYPEC_CC_OPEN;
-	cc2 = (cc_polarity == TYPEC_POLARITY_CC2) ?
-	      cc_status_active : TYPEC_CC_OPEN;
 	if ((chip->cc1 != cc1) || (chip->cc2 != cc2)) {
 		chip->cc1 = cc1;
 		chip->cc2 = cc2;
 		tcpm_cc_change(chip->tcpm_port);
 	}
-	/* turn off toggling */
-	ret = fusb302_set_toggling(chip, TOGGLING_MODE_OFF);
-	if (ret < 0) {
-		fusb302_log(chip,
-			    "cannot set toggling mode off, ret=%d", ret);
-		return ret;
-	}
 	/* set MDAC to Rd threshold, and unmask I_COMP for unplug detection */
 	ret = fusb302_i2c_write(chip, FUSB_REG_MEASURE, rd_mda);
 	if (ret < 0)
@@ -1427,7 +1382,7 @@ static int fusb302_handle_togdone_src(struct fusb302_chip *chip,
 				     FUSB_REG_MASK_COMP_CHNG);
 	if (ret < 0) {
 		fusb302_log(chip,
-			    "cannot unmask bc_lcl interrupt, ret=%d", ret);
+			    "cannot unmask comp_chng interrupt, ret=%d", ret);
 		return ret;
 	}
 	chip->intr_comp_chng = true;
@@ -1532,6 +1487,25 @@ static int fusb302_pd_read_message(struct fusb302_chip *chip,
 static irqreturn_t fusb302_irq_intn(int irq, void *dev_id)
 {
 	struct fusb302_chip *chip = dev_id;
+	unsigned long flags;
+
+	/* Disable our level triggered IRQ until our irq_work has cleared it */
+	disable_irq_nosync(chip->gpio_int_n_irq);
+
+	spin_lock_irqsave(&chip->irq_lock, flags);
+	if (chip->irq_suspended)
+		chip->irq_while_suspended = true;
+	else
+		schedule_work(&chip->irq_work);
+	spin_unlock_irqrestore(&chip->irq_lock, flags);
+
+	return IRQ_HANDLED;
+}
+
+static void fusb302_irq_work(struct work_struct *work)
+{
+	struct fusb302_chip *chip = container_of(work, struct fusb302_chip,
+						 irq_work);
 	int ret = 0;
 	u8 interrupt;
 	u8 interrupta;
@@ -1602,11 +1576,9 @@ static irqreturn_t fusb302_irq_intn(int irq, void *dev_id)
 		fusb302_log(chip, "IRQ: COMP_CHNG, comp=%s",
 			    comp_result ? "true" : "false");
 		if (comp_result) {
-			/* cc level > Rd_threashold, detach */
-			if (chip->cc_polarity == TYPEC_POLARITY_CC1)
-				chip->cc1 = TYPEC_CC_OPEN;
-			else
-				chip->cc2 = TYPEC_CC_OPEN;
+			/* cc level > Rd_threshold, detach */
+			chip->cc1 = TYPEC_CC_OPEN;
+			chip->cc2 = TYPEC_CC_OPEN;
 			tcpm_cc_change(chip->tcpm_port);
 		}
 	}
@@ -1662,8 +1634,7 @@ static irqreturn_t fusb302_irq_intn(int irq, void *dev_id)
 	}
 done:
 	mutex_unlock(&chip->lock);
-
-	return IRQ_HANDLED;
+	enable_irq(chip->gpio_int_n_irq);
 }
 
 static int init_gpio(struct fusb302_chip *chip)
@@ -1779,6 +1750,8 @@ static int fusb302_probe(struct i2c_client *client,
 	if (!chip->wq)
 		return -ENOMEM;
 
+	spin_lock_init(&chip->irq_lock);
+	INIT_WORK(&chip->irq_work, fusb302_irq_work);
 	INIT_DELAYED_WORK(&chip->bc_lvl_handler, fusb302_bc_lvl_handler_work);
 	init_tcpc_dev(&chip->tcpc_dev);
 
@@ -1798,10 +1771,9 @@ static int fusb302_probe(struct i2c_client *client,
 		goto destroy_workqueue;
 	}
 
-	ret = devm_request_threaded_irq(chip->dev, chip->gpio_int_n_irq,
-					NULL, fusb302_irq_intn,
-					IRQF_ONESHOT | IRQF_TRIGGER_LOW,
-					"fsc_interrupt_int_n", chip);
+	ret = request_irq(chip->gpio_int_n_irq, fusb302_irq_intn,
+			  IRQF_ONESHOT | IRQF_TRIGGER_LOW,
+			  "fsc_interrupt_int_n", chip);
 	if (ret < 0) {
 		dev_err(dev, "cannot request IRQ for GPIO Int_N, ret=%d", ret);
 		goto tcpm_unregister_port;
@@ -1824,6 +1796,10 @@ static int fusb302_remove(struct i2c_client *client)
 {
 	struct fusb302_chip *chip = i2c_get_clientdata(client);
 
+	disable_irq_wake(chip->gpio_int_n_irq);
+	free_irq(chip->gpio_int_n_irq, chip);
+	cancel_work_sync(&chip->irq_work);
+	cancel_delayed_work_sync(&chip->bc_lvl_handler);
 	tcpm_unregister_port(chip->tcpm_port);
 	destroy_workqueue(chip->wq);
 	fusb302_debugfs_exit(chip);
@@ -1834,19 +1810,29 @@ static int fusb302_remove(struct i2c_client *client)
 static int fusb302_pm_suspend(struct device *dev)
 {
 	struct fusb302_chip *chip = dev->driver_data;
+	unsigned long flags;
 
-	if (atomic_read(&chip->i2c_busy))
-		return -EBUSY;
-	atomic_set(&chip->pm_suspend, 1);
+	spin_lock_irqsave(&chip->irq_lock, flags);
+	chip->irq_suspended = true;
+	spin_unlock_irqrestore(&chip->irq_lock, flags);
 
+	/* Make sure any pending irq_work is finished before the bus suspends */
+	flush_work(&chip->irq_work);
 	return 0;
 }
 
 static int fusb302_pm_resume(struct device *dev)
 {
 	struct fusb302_chip *chip = dev->driver_data;
+	unsigned long flags;
 
-	atomic_set(&chip->pm_suspend, 0);
+	spin_lock_irqsave(&chip->irq_lock, flags);
+	if (chip->irq_while_suspended) {
+		schedule_work(&chip->irq_work);
+		chip->irq_while_suspended = false;
+	}
+	chip->irq_suspended = false;
+	spin_unlock_irqrestore(&chip->irq_lock, flags);
 
 	return 0;
 }
diff --git a/drivers/usb/typec/tcpm/tcpci.c b/drivers/usb/typec/tcpm/tcpci.c
index ac6b418b15f1..c1f7073a56de 100644
--- a/drivers/usb/typec/tcpm/tcpci.c
+++ b/drivers/usb/typec/tcpm/tcpci.c
@@ -100,13 +100,17 @@ static int tcpci_set_cc(struct tcpc_dev *tcpc, enum typec_cc_status cc)
 	return 0;
 }
 
-static int tcpci_start_drp_toggling(struct tcpc_dev *tcpc,
-				    enum typec_cc_status cc)
+static int tcpci_start_toggling(struct tcpc_dev *tcpc,
+				enum typec_port_type port_type,
+				enum typec_cc_status cc)
 {
 	int ret;
 	struct tcpci *tcpci = tcpc_to_tcpci(tcpc);
 	unsigned int reg = TCPC_ROLE_CTRL_DRP;
 
+	if (port_type != TYPEC_PORT_DRP)
+		return -EOPNOTSUPP;
+
 	/* Handle vendor drp toggling */
 	if (tcpci->data->start_drp_toggling) {
 		ret = tcpci->data->start_drp_toggling(tcpci, tcpci->data, cc);
@@ -511,7 +515,7 @@ struct tcpci *tcpci_register_port(struct device *dev, struct tcpci_data *data)
 	tcpci->tcpc.get_cc = tcpci_get_cc;
 	tcpci->tcpc.set_polarity = tcpci_set_polarity;
 	tcpci->tcpc.set_vconn = tcpci_set_vconn;
-	tcpci->tcpc.start_drp_toggling = tcpci_start_drp_toggling;
+	tcpci->tcpc.start_toggling = tcpci_start_toggling;
 
 	tcpci->tcpc.set_pd_rx = tcpci_set_pd_rx;
 	tcpci->tcpc.set_roles = tcpci_set_roles;
diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c
index a2233d72ae7c..fba32d84e578 100644
--- a/drivers/usb/typec/tcpm/tcpm.c
+++ b/drivers/usb/typec/tcpm/tcpm.c
@@ -31,7 +31,7 @@
 
 #define FOREACH_STATE(S)			\
 	S(INVALID_STATE),			\
-	S(DRP_TOGGLING),			\
+	S(TOGGLING),			\
 	S(SRC_UNATTACHED),			\
 	S(SRC_ATTACH_WAIT),			\
 	S(SRC_ATTACHED),			\
@@ -472,7 +472,7 @@ static void tcpm_log(struct tcpm_port *port, const char *fmt, ...)
 	/* Do not log while disconnected and unattached */
 	if (tcpm_port_is_disconnected(port) &&
 	    (port->state == SRC_UNATTACHED || port->state == SNK_UNATTACHED ||
-	     port->state == DRP_TOGGLING))
+	     port->state == TOGGLING))
 		return;
 
 	va_start(args, fmt);
@@ -2540,20 +2540,16 @@ static int tcpm_set_charge(struct tcpm_port *port, bool charge)
 	return 0;
 }
 
-static bool tcpm_start_drp_toggling(struct tcpm_port *port,
-				    enum typec_cc_status cc)
+static bool tcpm_start_toggling(struct tcpm_port *port, enum typec_cc_status cc)
 {
 	int ret;
 
-	if (port->tcpc->start_drp_toggling &&
-	    port->port_type == TYPEC_PORT_DRP) {
-		tcpm_log_force(port, "Start DRP toggling");
-		ret = port->tcpc->start_drp_toggling(port->tcpc, cc);
-		if (!ret)
-			return true;
-	}
+	if (!port->tcpc->start_toggling)
+		return false;
 
-	return false;
+	tcpm_log_force(port, "Start toggling");
+	ret = port->tcpc->start_toggling(port->tcpc, port->port_type, cc);
+	return ret == 0;
 }
 
 static void tcpm_set_cc(struct tcpm_port *port, enum typec_cc_status cc)
@@ -2847,15 +2843,15 @@ static void run_state_machine(struct tcpm_port *port)
 
 	port->enter_state = port->state;
 	switch (port->state) {
-	case DRP_TOGGLING:
+	case TOGGLING:
 		break;
 	/* SRC states */
 	case SRC_UNATTACHED:
 		if (!port->non_pd_role_swap)
 			tcpm_swap_complete(port, -ENOTCONN);
 		tcpm_src_detach(port);
-		if (tcpm_start_drp_toggling(port, tcpm_rp_cc(port))) {
-			tcpm_set_state(port, DRP_TOGGLING, 0);
+		if (tcpm_start_toggling(port, tcpm_rp_cc(port))) {
+			tcpm_set_state(port, TOGGLING, 0);
 			break;
 		}
 		tcpm_set_cc(port, tcpm_rp_cc(port));
@@ -3053,8 +3049,8 @@ static void run_state_machine(struct tcpm_port *port)
 			tcpm_swap_complete(port, -ENOTCONN);
 		tcpm_pps_complete(port, -ENOTCONN);
 		tcpm_snk_detach(port);
-		if (tcpm_start_drp_toggling(port, TYPEC_CC_RD)) {
-			tcpm_set_state(port, DRP_TOGGLING, 0);
+		if (tcpm_start_toggling(port, TYPEC_CC_RD)) {
+			tcpm_set_state(port, TOGGLING, 0);
 			break;
 		}
 		tcpm_set_cc(port, TYPEC_CC_RD);
@@ -3621,7 +3617,7 @@ static void _tcpm_cc_change(struct tcpm_port *port, enum typec_cc_status cc1,
 						       : "connected");
 
 	switch (port->state) {
-	case DRP_TOGGLING:
+	case TOGGLING:
 		if (tcpm_port_is_debug(port) || tcpm_port_is_audio(port) ||
 		    tcpm_port_is_source(port))
 			tcpm_set_state(port, SRC_ATTACH_WAIT, 0);
diff --git a/drivers/usb/typec/tcpm/wcove.c b/drivers/usb/typec/tcpm/wcove.c
index 6770afd40765..6b317c150bdd 100644
--- a/drivers/usb/typec/tcpm/wcove.c
+++ b/drivers/usb/typec/tcpm/wcove.c
@@ -416,12 +416,16 @@ static int wcove_pd_transmit(struct tcpc_dev *tcpc,
 	return regmap_write(wcove->regmap, USBC_TXCMD, cmd | USBC_TXCMD_START);
 }
 
-static int wcove_start_drp_toggling(struct tcpc_dev *tcpc,
-				    enum typec_cc_status cc)
+static int wcove_start_toggling(struct tcpc_dev *tcpc,
+				enum typec_port_type port_type,
+				enum typec_cc_status cc)
 {
 	struct wcove_typec *wcove = tcpc_to_wcove(tcpc);
 	unsigned int usbc_ctrl;
 
+	if (port_type != TYPEC_PORT_DRP)
+		return -EOPNOTSUPP;
+
 	usbc_ctrl = USBC_CONTROL1_MODE_DRP | USBC_CONTROL1_DRPTOGGLE_RANDOM;
 
 	switch (cc) {
@@ -587,17 +591,14 @@ static const u32 snk_pdo[] = {
 	PDO_VAR(5000, 12000, 3000),
 };
 
-static struct tcpc_config wcove_typec_config = {
-	.src_pdo = src_pdo,
-	.nr_src_pdo = ARRAY_SIZE(src_pdo),
-	.snk_pdo = snk_pdo,
-	.nr_snk_pdo = ARRAY_SIZE(snk_pdo),
-
-	.operating_snk_mw = 15000,
-
-	.type = TYPEC_PORT_DRP,
-	.data = TYPEC_PORT_DRD,
-	.default_role = TYPEC_SINK,
+static const struct property_entry wcove_props[] = {
+	PROPERTY_ENTRY_STRING("data-role", "dual"),
+	PROPERTY_ENTRY_STRING("power-role", "dual"),
+	PROPERTY_ENTRY_STRING("try-power-role", "sink"),
+	PROPERTY_ENTRY_U32_ARRAY("source-pdos", src_pdo),
+	PROPERTY_ENTRY_U32_ARRAY("sink-pdos", snk_pdo),
+	PROPERTY_ENTRY_U32("op-sink-microwatt", 15000),
+	{ }
 };
 
 static int wcove_typec_probe(struct platform_device *pdev)
@@ -642,23 +643,28 @@ static int wcove_typec_probe(struct platform_device *pdev)
 	wcove->tcpc.set_polarity = wcove_set_polarity;
 	wcove->tcpc.set_vconn = wcove_set_vconn;
 	wcove->tcpc.set_current_limit = wcove_set_current_limit;
-	wcove->tcpc.start_drp_toggling = wcove_start_drp_toggling;
+	wcove->tcpc.start_toggling = wcove_start_toggling;
 
 	wcove->tcpc.set_pd_rx = wcove_set_pd_rx;
 	wcove->tcpc.set_roles = wcove_set_roles;
 	wcove->tcpc.pd_transmit = wcove_pd_transmit;
 
-	wcove->tcpc.config = &wcove_typec_config;
+	wcove->tcpc.fwnode = fwnode_create_software_node(wcove_props, NULL);
+	if (IS_ERR(wcove->tcpc.fwnode))
+		return PTR_ERR(wcove->tcpc.fwnode);
 
 	wcove->tcpm = tcpm_register_port(wcove->dev, &wcove->tcpc);
-	if (IS_ERR(wcove->tcpm))
+	if (IS_ERR(wcove->tcpm)) {
+		fwnode_remove_software_node(wcove->tcpc.fwnode);
 		return PTR_ERR(wcove->tcpm);
+	}
 
 	ret = devm_request_threaded_irq(&pdev->dev, irq, NULL,
 					wcove_typec_irq, IRQF_ONESHOT,
 					"wcove_typec", wcove);
 	if (ret) {
 		tcpm_unregister_port(wcove->tcpm);
+		fwnode_remove_software_node(wcove->tcpc.fwnode);
 		return ret;
 	}
 
@@ -678,6 +684,7 @@ static int wcove_typec_remove(struct platform_device *pdev)
 	regmap_write(wcove->regmap, USBC_IRQMASK2, val | USBC_IRQMASK2_ALL);
 
 	tcpm_unregister_port(wcove->tcpm);
+	fwnode_remove_software_node(wcove->tcpc.fwnode);
 
 	return 0;
 }
diff --git a/drivers/usb/typec/ucsi/Makefile b/drivers/usb/typec/ucsi/Makefile
index 2f4900b26210..b35e15a1f02c 100644
--- a/drivers/usb/typec/ucsi/Makefile
+++ b/drivers/usb/typec/ucsi/Makefile
@@ -1,12 +1,15 @@
 # SPDX-License-Identifier: GPL-2.0
-CFLAGS_trace.o			:= -I$(src)
+CFLAGS_trace.o				:= -I$(src)
 
-obj-$(CONFIG_TYPEC_UCSI)	+= typec_ucsi.o
+obj-$(CONFIG_TYPEC_UCSI)		+= typec_ucsi.o
 
-typec_ucsi-y			:= ucsi.o
+typec_ucsi-y				:= ucsi.o
 
-typec_ucsi-$(CONFIG_TRACING)	+= trace.o
+typec_ucsi-$(CONFIG_TRACING)		+= trace.o
 
-obj-$(CONFIG_UCSI_ACPI)		+= ucsi_acpi.o
+ifneq ($(CONFIG_TYPEC_DP_ALTMODE),)
+	typec_ucsi-y			+= displayport.o
+endif
 
-obj-$(CONFIG_UCSI_CCG)		+= ucsi_ccg.o
+obj-$(CONFIG_UCSI_ACPI)			+= ucsi_acpi.o
+obj-$(CONFIG_UCSI_CCG)			+= ucsi_ccg.o
diff --git a/drivers/usb/typec/ucsi/displayport.c b/drivers/usb/typec/ucsi/displayport.c
new file mode 100644
index 000000000000..6c103697c582
--- /dev/null
+++ b/drivers/usb/typec/ucsi/displayport.c
@@ -0,0 +1,315 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * UCSI DisplayPort Alternate Mode Support
+ *
+ * Copyright (C) 2018, Intel Corporation
+ * Author: Heikki Krogerus <heikki.krogerus@linux.intel.com>
+ */
+
+#include <linux/usb/typec_dp.h>
+#include <linux/usb/pd_vdo.h>
+
+#include "ucsi.h"
+
+#define UCSI_CMD_SET_NEW_CAM(_con_num_, _enter_, _cam_, _am_)		\
+	 (UCSI_SET_NEW_CAM | ((_con_num_) << 16) | ((_enter_) << 23) |	\
+	  ((_cam_) << 24) | ((u64)(_am_) << 32))
+
+struct ucsi_dp {
+	struct typec_displayport_data data;
+	struct ucsi_connector *con;
+	struct typec_altmode *alt;
+	struct work_struct work;
+	int offset;
+
+	bool override;
+	bool initialized;
+
+	u32 header;
+	u32 *vdo_data;
+	u8 vdo_size;
+};
+
+/*
+ * Note. Alternate mode control is optional feature in UCSI. It means that even
+ * if the system supports alternate modes, the OS may not be aware of them.
+ *
+ * In most cases however, the OS will be able to see the supported alternate
+ * modes, but it may still not be able to configure them, not even enter or exit
+ * them. That is because UCSI defines alt mode details and alt mode "overriding"
+ * as separate options.
+ *
+ * In case alt mode details are supported, but overriding is not, the driver
+ * will still display the supported pin assignments and configuration, but any
+ * changes the user attempts to do will lead into failure with return value of
+ * -EOPNOTSUPP.
+ */
+
+static int ucsi_displayport_enter(struct typec_altmode *alt)
+{
+	struct ucsi_dp *dp = typec_altmode_get_drvdata(alt);
+	struct ucsi_control ctrl;
+	u8 cur = 0;
+	int ret;
+
+	mutex_lock(&dp->con->lock);
+
+	if (!dp->override && dp->initialized) {
+		const struct typec_altmode *p = typec_altmode_get_partner(alt);
+
+		dev_warn(&p->dev,
+			 "firmware doesn't support alternate mode overriding\n");
+		mutex_unlock(&dp->con->lock);
+		return -EOPNOTSUPP;
+	}
+
+	UCSI_CMD_GET_CURRENT_CAM(ctrl, dp->con->num);
+	ret = ucsi_send_command(dp->con->ucsi, &ctrl, &cur, sizeof(cur));
+	if (ret < 0) {
+		if (dp->con->ucsi->ppm->data->version > 0x0100) {
+			mutex_unlock(&dp->con->lock);
+			return ret;
+		}
+		cur = 0xff;
+	}
+
+	if (cur != 0xff) {
+		mutex_unlock(&dp->con->lock);
+		return -EBUSY;
+	}
+
+	/*
+	 * We can't send the New CAM command yet to the PPM as it needs the
+	 * configuration value as well. Pretending that we have now entered the
+	 * mode, and letting the alt mode driver continue.
+	 */
+
+	dp->header = VDO(USB_TYPEC_DP_SID, 1, CMD_ENTER_MODE);
+	dp->header |= VDO_OPOS(USB_TYPEC_DP_MODE);
+	dp->header |= VDO_CMDT(CMDT_RSP_ACK);
+
+	dp->vdo_data = NULL;
+	dp->vdo_size = 1;
+
+	schedule_work(&dp->work);
+
+	mutex_unlock(&dp->con->lock);
+
+	return 0;
+}
+
+static int ucsi_displayport_exit(struct typec_altmode *alt)
+{
+	struct ucsi_dp *dp = typec_altmode_get_drvdata(alt);
+	struct ucsi_control ctrl;
+	int ret = 0;
+
+	mutex_lock(&dp->con->lock);
+
+	if (!dp->override) {
+		const struct typec_altmode *p = typec_altmode_get_partner(alt);
+
+		dev_warn(&p->dev,
+			 "firmware doesn't support alternate mode overriding\n");
+		ret = -EOPNOTSUPP;
+		goto out_unlock;
+	}
+
+	ctrl.raw_cmd = UCSI_CMD_SET_NEW_CAM(dp->con->num, 0, dp->offset, 0);
+	ret = ucsi_send_command(dp->con->ucsi, &ctrl, NULL, 0);
+	if (ret < 0)
+		goto out_unlock;
+
+	dp->header = VDO(USB_TYPEC_DP_SID, 1, CMD_EXIT_MODE);
+	dp->header |= VDO_OPOS(USB_TYPEC_DP_MODE);
+	dp->header |= VDO_CMDT(CMDT_RSP_ACK);
+
+	dp->vdo_data = NULL;
+	dp->vdo_size = 1;
+
+	schedule_work(&dp->work);
+
+out_unlock:
+	mutex_unlock(&dp->con->lock);
+
+	return ret;
+}
+
+/*
+ * We do not actually have access to the Status Update VDO, so we have to guess
+ * things.
+ */
+static int ucsi_displayport_status_update(struct ucsi_dp *dp)
+{
+	u32 cap = dp->alt->vdo;
+
+	dp->data.status = DP_STATUS_ENABLED;
+
+	/*
+	 * If pin assignement D is supported, claiming always
+	 * that Multi-function is preferred.
+	 */
+	if (DP_CAP_CAPABILITY(cap) & DP_CAP_UFP_D) {
+		dp->data.status |= DP_STATUS_CON_UFP_D;
+
+		if (DP_CAP_UFP_D_PIN_ASSIGN(cap) & BIT(DP_PIN_ASSIGN_D))
+			dp->data.status |= DP_STATUS_PREFER_MULTI_FUNC;
+	} else {
+		dp->data.status |= DP_STATUS_CON_DFP_D;
+
+		if (DP_CAP_DFP_D_PIN_ASSIGN(cap) & BIT(DP_PIN_ASSIGN_D))
+			dp->data.status |= DP_STATUS_PREFER_MULTI_FUNC;
+	}
+
+	dp->vdo_data = &dp->data.status;
+	dp->vdo_size = 2;
+
+	return 0;
+}
+
+static int ucsi_displayport_configure(struct ucsi_dp *dp)
+{
+	u32 pins = DP_CONF_GET_PIN_ASSIGN(dp->data.conf);
+	struct ucsi_control ctrl;
+
+	if (!dp->override)
+		return 0;
+
+	ctrl.raw_cmd = UCSI_CMD_SET_NEW_CAM(dp->con->num, 1, dp->offset, pins);
+
+	return ucsi_send_command(dp->con->ucsi, &ctrl, NULL, 0);
+}
+
+static int ucsi_displayport_vdm(struct typec_altmode *alt,
+				u32 header, const u32 *data, int count)
+{
+	struct ucsi_dp *dp = typec_altmode_get_drvdata(alt);
+	int cmd_type = PD_VDO_CMDT(header);
+	int cmd = PD_VDO_CMD(header);
+
+	mutex_lock(&dp->con->lock);
+
+	if (!dp->override && dp->initialized) {
+		const struct typec_altmode *p = typec_altmode_get_partner(alt);
+
+		dev_warn(&p->dev,
+			 "firmware doesn't support alternate mode overriding\n");
+		mutex_unlock(&dp->con->lock);
+		return -EOPNOTSUPP;
+	}
+
+	switch (cmd_type) {
+	case CMDT_INIT:
+		dp->header = VDO(USB_TYPEC_DP_SID, 1, cmd);
+		dp->header |= VDO_OPOS(USB_TYPEC_DP_MODE);
+
+		switch (cmd) {
+		case DP_CMD_STATUS_UPDATE:
+			if (ucsi_displayport_status_update(dp))
+				dp->header |= VDO_CMDT(CMDT_RSP_NAK);
+			else
+				dp->header |= VDO_CMDT(CMDT_RSP_ACK);
+			break;
+		case DP_CMD_CONFIGURE:
+			dp->data.conf = *data;
+			if (ucsi_displayport_configure(dp)) {
+				dp->header |= VDO_CMDT(CMDT_RSP_NAK);
+			} else {
+				dp->header |= VDO_CMDT(CMDT_RSP_ACK);
+				if (dp->initialized)
+					ucsi_altmode_update_active(dp->con);
+				else
+					dp->initialized = true;
+			}
+			break;
+		default:
+			dp->header |= VDO_CMDT(CMDT_RSP_ACK);
+			break;
+		}
+
+		schedule_work(&dp->work);
+		break;
+	default:
+		break;
+	}
+
+	mutex_unlock(&dp->con->lock);
+
+	return 0;
+}
+
+static const struct typec_altmode_ops ucsi_displayport_ops = {
+	.enter = ucsi_displayport_enter,
+	.exit = ucsi_displayport_exit,
+	.vdm = ucsi_displayport_vdm,
+};
+
+static void ucsi_displayport_work(struct work_struct *work)
+{
+	struct ucsi_dp *dp = container_of(work, struct ucsi_dp, work);
+	int ret;
+
+	mutex_lock(&dp->con->lock);
+
+	ret = typec_altmode_vdm(dp->alt, dp->header,
+				dp->vdo_data, dp->vdo_size);
+	if (ret)
+		dev_err(&dp->alt->dev, "VDM 0x%x failed\n", dp->header);
+
+	dp->vdo_data = NULL;
+	dp->vdo_size = 0;
+	dp->header = 0;
+
+	mutex_unlock(&dp->con->lock);
+}
+
+void ucsi_displayport_remove_partner(struct typec_altmode *alt)
+{
+	struct ucsi_dp *dp;
+
+	if (!alt)
+		return;
+
+	dp = typec_altmode_get_drvdata(alt);
+	dp->data.conf = 0;
+	dp->data.status = 0;
+	dp->initialized = false;
+}
+
+struct typec_altmode *ucsi_register_displayport(struct ucsi_connector *con,
+						bool override, int offset,
+						struct typec_altmode_desc *desc)
+{
+	u8 all_assignments = BIT(DP_PIN_ASSIGN_C) | BIT(DP_PIN_ASSIGN_D) |
+			     BIT(DP_PIN_ASSIGN_E);
+	struct typec_altmode *alt;
+	struct ucsi_dp *dp;
+
+	/* We can't rely on the firmware with the capabilities. */
+	desc->vdo |= DP_CAP_DP_SIGNALING | DP_CAP_RECEPTACLE;
+
+	/* Claiming that we support all pin assignments */
+	desc->vdo |= all_assignments << 8;
+	desc->vdo |= all_assignments << 16;
+
+	alt = typec_port_register_altmode(con->port, desc);
+	if (IS_ERR(alt))
+		return alt;
+
+	dp = devm_kzalloc(&alt->dev, sizeof(*dp), GFP_KERNEL);
+	if (!dp) {
+		typec_unregister_altmode(alt);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	INIT_WORK(&dp->work, ucsi_displayport_work);
+	dp->override = override;
+	dp->offset = offset;
+	dp->con = con;
+	dp->alt = alt;
+
+	alt->ops = &ucsi_displayport_ops;
+	typec_altmode_set_drvdata(alt, dp);
+
+	return alt;
+}
diff --git a/drivers/usb/typec/ucsi/trace.c b/drivers/usb/typec/ucsi/trace.c
index ffa3b4c3f338..1dabafb74320 100644
--- a/drivers/usb/typec/ucsi/trace.c
+++ b/drivers/usb/typec/ucsi/trace.c
@@ -60,3 +60,15 @@ const char *ucsi_cci_str(u32 cci)
 
 	return "";
 }
+
+static const char * const ucsi_recipient_strs[] = {
+	[UCSI_RECIPIENT_CON]		= "port",
+	[UCSI_RECIPIENT_SOP]		= "partner",
+	[UCSI_RECIPIENT_SOP_P]		= "plug (prime)",
+	[UCSI_RECIPIENT_SOP_PP]		= "plug (double prime)",
+};
+
+const char *ucsi_recipient_str(u8 recipient)
+{
+	return ucsi_recipient_strs[recipient];
+}
diff --git a/drivers/usb/typec/ucsi/trace.h b/drivers/usb/typec/ucsi/trace.h
index 5e2906df2db7..783ec9c72055 100644
--- a/drivers/usb/typec/ucsi/trace.h
+++ b/drivers/usb/typec/ucsi/trace.h
@@ -7,6 +7,7 @@
 #define __UCSI_TRACE_H
 
 #include <linux/tracepoint.h>
+#include <linux/usb/typec_altmode.h>
 
 const char *ucsi_cmd_str(u64 raw_cmd);
 const char *ucsi_ack_str(u8 ack);
@@ -134,6 +135,31 @@ DEFINE_EVENT(ucsi_log_connector_status, ucsi_register_port,
 	TP_ARGS(port, status)
 );
 
+DECLARE_EVENT_CLASS(ucsi_log_register_altmode,
+	TP_PROTO(u8 recipient, struct typec_altmode *alt),
+	TP_ARGS(recipient, alt),
+	TP_STRUCT__entry(
+		__field(u8, recipient)
+		__field(u16, svid)
+		__field(u8, mode)
+		__field(u32, vdo)
+	),
+	TP_fast_assign(
+		__entry->recipient = recipient;
+		__entry->svid = alt->svid;
+		__entry->mode = alt->mode;
+		__entry->vdo = alt->vdo;
+	),
+	TP_printk("%s alt mode: svid %04x, mode %d vdo %x",
+		  ucsi_recipient_str(__entry->recipient), __entry->svid,
+		  __entry->mode, __entry->vdo)
+);
+
+DEFINE_EVENT(ucsi_log_register_altmode, ucsi_register_altmode,
+	TP_PROTO(u8 recipient, struct typec_altmode *alt),
+	TP_ARGS(recipient, alt)
+);
+
 #endif /* __UCSI_TRACE_H */
 
 /* This part must be outside protection */
diff --git a/drivers/usb/typec/ucsi/ucsi.c b/drivers/usb/typec/ucsi/ucsi.c
index 8d0a6fe748bd..7850b851cecd 100644
--- a/drivers/usb/typec/ucsi/ucsi.c
+++ b/drivers/usb/typec/ucsi/ucsi.c
@@ -12,7 +12,7 @@
 #include <linux/module.h>
 #include <linux/delay.h>
 #include <linux/slab.h>
-#include <linux/usb/typec.h>
+#include <linux/usb/typec_dp.h>
 
 #include "ucsi.h"
 #include "trace.h"
@@ -39,49 +39,6 @@
  */
 #define UCSI_SWAP_TIMEOUT_MS	5000
 
-enum ucsi_status {
-	UCSI_IDLE = 0,
-	UCSI_BUSY,
-	UCSI_ERROR,
-};
-
-struct ucsi_connector {
-	int num;
-
-	struct ucsi *ucsi;
-	struct work_struct work;
-	struct completion complete;
-
-	struct typec_port *port;
-	struct typec_partner *partner;
-
-	struct typec_capability typec_cap;
-
-	struct ucsi_connector_status status;
-	struct ucsi_connector_capability cap;
-};
-
-struct ucsi {
-	struct device *dev;
-	struct ucsi_ppm *ppm;
-
-	enum ucsi_status status;
-	struct completion complete;
-	struct ucsi_capability cap;
-	struct ucsi_connector *connector;
-
-	struct work_struct work;
-
-	/* PPM Communication lock */
-	struct mutex ppm_lock;
-
-	/* PPM communication flags */
-	unsigned long flags;
-#define EVENT_PENDING	0
-#define COMMAND_PENDING	1
-#define ACK_PENDING	2
-};
-
 static inline int ucsi_sync(struct ucsi *ucsi)
 {
 	if (ucsi->ppm && ucsi->ppm->sync)
@@ -238,8 +195,226 @@ err:
 	return ret;
 }
 
+int ucsi_send_command(struct ucsi *ucsi, struct ucsi_control *ctrl,
+		      void *retval, size_t size)
+{
+	int ret;
+
+	mutex_lock(&ucsi->ppm_lock);
+	ret = ucsi_run_command(ucsi, ctrl, retval, size);
+	mutex_unlock(&ucsi->ppm_lock);
+
+	return ret;
+}
+
 /* -------------------------------------------------------------------------- */
 
+void ucsi_altmode_update_active(struct ucsi_connector *con)
+{
+	const struct typec_altmode *altmode = NULL;
+	struct ucsi_control ctrl;
+	int ret;
+	u8 cur;
+	int i;
+
+	UCSI_CMD_GET_CURRENT_CAM(ctrl, con->num);
+	ret = ucsi_run_command(con->ucsi, &ctrl, &cur, sizeof(cur));
+	if (ret < 0) {
+		if (con->ucsi->ppm->data->version > 0x0100) {
+			dev_err(con->ucsi->dev,
+				"GET_CURRENT_CAM command failed\n");
+			return;
+		}
+		cur = 0xff;
+	}
+
+	if (cur < UCSI_MAX_ALTMODES)
+		altmode = typec_altmode_get_partner(con->port_altmode[cur]);
+
+	for (i = 0; con->partner_altmode[i]; i++)
+		typec_altmode_update_active(con->partner_altmode[i],
+					    con->partner_altmode[i] == altmode);
+}
+
+static u8 ucsi_altmode_next_mode(struct typec_altmode **alt, u16 svid)
+{
+	u8 mode = 1;
+	int i;
+
+	for (i = 0; alt[i]; i++)
+		if (alt[i]->svid == svid)
+			mode++;
+
+	return mode;
+}
+
+static int ucsi_next_altmode(struct typec_altmode **alt)
+{
+	int i = 0;
+
+	for (i = 0; i < UCSI_MAX_ALTMODES; i++)
+		if (!alt[i])
+			return i;
+
+	return -ENOENT;
+}
+
+static int ucsi_register_altmode(struct ucsi_connector *con,
+				 struct typec_altmode_desc *desc,
+				 u8 recipient)
+{
+	struct typec_altmode *alt;
+	bool override;
+	int ret;
+	int i;
+
+	override = !!(con->ucsi->cap.features & UCSI_CAP_ALT_MODE_OVERRIDE);
+
+	switch (recipient) {
+	case UCSI_RECIPIENT_CON:
+		i = ucsi_next_altmode(con->port_altmode);
+		if (i < 0) {
+			ret = i;
+			goto err;
+		}
+
+		desc->mode = ucsi_altmode_next_mode(con->port_altmode,
+						    desc->svid);
+
+		switch (desc->svid) {
+		case USB_TYPEC_DP_SID:
+		case USB_TYPEC_NVIDIA_VLINK_SID:
+			alt = ucsi_register_displayport(con, override, i, desc);
+			break;
+		default:
+			alt = typec_port_register_altmode(con->port, desc);
+			break;
+		}
+
+		if (IS_ERR(alt)) {
+			ret = PTR_ERR(alt);
+			goto err;
+		}
+
+		con->port_altmode[i] = alt;
+		break;
+	case UCSI_RECIPIENT_SOP:
+		i = ucsi_next_altmode(con->partner_altmode);
+		if (i < 0) {
+			ret = i;
+			goto err;
+		}
+
+		desc->mode = ucsi_altmode_next_mode(con->partner_altmode,
+						    desc->svid);
+
+		alt = typec_partner_register_altmode(con->partner, desc);
+		if (IS_ERR(alt)) {
+			ret = PTR_ERR(alt);
+			goto err;
+		}
+
+		con->partner_altmode[i] = alt;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	trace_ucsi_register_altmode(recipient, alt);
+
+	return 0;
+
+err:
+	dev_err(con->ucsi->dev, "failed to registers svid 0x%04x mode %d\n",
+		desc->svid, desc->mode);
+
+	return ret;
+}
+
+static int ucsi_register_altmodes(struct ucsi_connector *con, u8 recipient)
+{
+	int max_altmodes = UCSI_MAX_ALTMODES;
+	struct typec_altmode_desc desc;
+	struct ucsi_altmode alt[2];
+	struct ucsi_control ctrl;
+	int num = 1;
+	int ret;
+	int len;
+	int j;
+	int i;
+
+	if (!(con->ucsi->cap.features & UCSI_CAP_ALT_MODE_DETAILS))
+		return 0;
+
+	if (recipient == UCSI_RECIPIENT_SOP && con->partner_altmode[0])
+		return 0;
+
+	if (recipient == UCSI_RECIPIENT_CON)
+		max_altmodes = con->ucsi->cap.num_alt_modes;
+
+	for (i = 0; i < max_altmodes;) {
+		memset(alt, 0, sizeof(alt));
+		UCSI_CMD_GET_ALTERNATE_MODES(ctrl, recipient, con->num, i, 1);
+		len = ucsi_run_command(con->ucsi, &ctrl, alt, sizeof(alt));
+		if (len <= 0)
+			return len;
+
+		/*
+		 * This code is requesting one alt mode at a time, but some PPMs
+		 * may still return two. If that happens both alt modes need be
+		 * registered and the offset for the next alt mode has to be
+		 * incremented.
+		 */
+		num = len / sizeof(alt[0]);
+		i += num;
+
+		for (j = 0; j < num; j++) {
+			if (!alt[j].svid)
+				return 0;
+
+			memset(&desc, 0, sizeof(desc));
+			desc.vdo = alt[j].mid;
+			desc.svid = alt[j].svid;
+			desc.roles = TYPEC_PORT_DRD;
+
+			ret = ucsi_register_altmode(con, &desc, recipient);
+			if (ret)
+				return ret;
+		}
+	}
+
+	return 0;
+}
+
+static void ucsi_unregister_altmodes(struct ucsi_connector *con, u8 recipient)
+{
+	const struct typec_altmode *pdev;
+	struct typec_altmode **adev;
+	int i = 0;
+
+	switch (recipient) {
+	case UCSI_RECIPIENT_CON:
+		adev = con->port_altmode;
+		break;
+	case UCSI_RECIPIENT_SOP:
+		adev = con->partner_altmode;
+		break;
+	default:
+		return;
+	}
+
+	while (adev[i]) {
+		if (recipient == UCSI_RECIPIENT_SOP &&
+		    (adev[i]->svid == USB_TYPEC_DP_SID ||
+			adev[i]->svid == USB_TYPEC_NVIDIA_VLINK_SID)) {
+			pdev = typec_altmode_get_partner(adev[i]);
+			ucsi_displayport_remove_partner((void *)pdev);
+		}
+		typec_unregister_altmode(adev[i]);
+		adev[i++] = NULL;
+	}
+}
+
 static void ucsi_pwr_opmode_change(struct ucsi_connector *con)
 {
 	switch (con->status.pwr_op_mode) {
@@ -299,10 +474,43 @@ static void ucsi_unregister_partner(struct ucsi_connector *con)
 	if (!con->partner)
 		return;
 
+	ucsi_unregister_altmodes(con, UCSI_RECIPIENT_SOP);
 	typec_unregister_partner(con->partner);
 	con->partner = NULL;
 }
 
+static void ucsi_partner_change(struct ucsi_connector *con)
+{
+	int ret;
+
+	if (!con->partner)
+		return;
+
+	switch (con->status.partner_type) {
+	case UCSI_CONSTAT_PARTNER_TYPE_UFP:
+		typec_set_data_role(con->port, TYPEC_HOST);
+		break;
+	case UCSI_CONSTAT_PARTNER_TYPE_DFP:
+		typec_set_data_role(con->port, TYPEC_DEVICE);
+		break;
+	default:
+		break;
+	}
+
+	/* Complete pending data role swap */
+	if (!completion_done(&con->complete))
+		complete(&con->complete);
+
+	/* Can't rely on Partner Flags field. Always checking the alt modes. */
+	ret = ucsi_register_altmodes(con, UCSI_RECIPIENT_SOP);
+	if (ret)
+		dev_err(con->ucsi->dev,
+			"con%d: failed to register partner alternate modes\n",
+			con->num);
+	else
+		ucsi_altmode_update_active(con);
+}
+
 static void ucsi_connector_change(struct work_struct *work)
 {
 	struct ucsi_connector *con = container_of(work, struct ucsi_connector,
@@ -311,10 +519,10 @@ static void ucsi_connector_change(struct work_struct *work)
 	struct ucsi_control ctrl;
 	int ret;
 
-	mutex_lock(&ucsi->ppm_lock);
+	mutex_lock(&con->lock);
 
 	UCSI_CMD_GET_CONNECTOR_STATUS(ctrl, con->num);
-	ret = ucsi_run_command(ucsi, &ctrl, &con->status, sizeof(con->status));
+	ret = ucsi_send_command(ucsi, &ctrl, &con->status, sizeof(con->status));
 	if (ret < 0) {
 		dev_err(ucsi->dev, "%s: GET_CONNECTOR_STATUS failed (%d)\n",
 			__func__, ret);
@@ -332,23 +540,6 @@ static void ucsi_connector_change(struct work_struct *work)
 			complete(&con->complete);
 	}
 
-	if (con->status.change & UCSI_CONSTAT_PARTNER_CHANGE) {
-		switch (con->status.partner_type) {
-		case UCSI_CONSTAT_PARTNER_TYPE_UFP:
-			typec_set_data_role(con->port, TYPEC_HOST);
-			break;
-		case UCSI_CONSTAT_PARTNER_TYPE_DFP:
-			typec_set_data_role(con->port, TYPEC_DEVICE);
-			break;
-		default:
-			break;
-		}
-
-		/* Complete pending data role swap */
-		if (!completion_done(&con->complete))
-			complete(&con->complete);
-	}
-
 	if (con->status.change & UCSI_CONSTAT_CONNECT_CHANGE) {
 		typec_set_pwr_role(con->port, con->status.pwr_dir);
 
@@ -369,6 +560,19 @@ static void ucsi_connector_change(struct work_struct *work)
 			ucsi_unregister_partner(con);
 	}
 
+	if (con->status.change & UCSI_CONSTAT_CAM_CHANGE) {
+		/*
+		 * We don't need to know the currently supported alt modes here.
+		 * Running GET_CAM_SUPPORTED command just to make sure the PPM
+		 * does not get stuck in case it assumes we do so.
+		 */
+		UCSI_CMD_GET_CAM_SUPPORTED(ctrl, con->num);
+		ucsi_run_command(con->ucsi, &ctrl, NULL, 0);
+	}
+
+	if (con->status.change & UCSI_CONSTAT_PARTNER_CHANGE)
+		ucsi_partner_change(con);
+
 	ret = ucsi_ack(ucsi, UCSI_ACK_EVENT);
 	if (ret)
 		dev_err(ucsi->dev, "%s: ACK failed (%d)", __func__, ret);
@@ -377,7 +581,7 @@ static void ucsi_connector_change(struct work_struct *work)
 
 out_unlock:
 	clear_bit(EVENT_PENDING, &ucsi->flags);
-	mutex_unlock(&ucsi->ppm_lock);
+	mutex_unlock(&con->lock);
 }
 
 /**
@@ -427,7 +631,7 @@ static int ucsi_reset_connector(struct ucsi_connector *con, bool hard)
 
 	UCSI_CMD_CONNECTOR_RESET(ctrl, con, hard);
 
-	return ucsi_run_command(con->ucsi, &ctrl, NULL, 0);
+	return ucsi_send_command(con->ucsi, &ctrl, NULL, 0);
 }
 
 static int ucsi_reset_ppm(struct ucsi *ucsi)
@@ -481,15 +685,17 @@ static int ucsi_role_cmd(struct ucsi_connector *con, struct ucsi_control *ctrl)
 {
 	int ret;
 
-	ret = ucsi_run_command(con->ucsi, ctrl, NULL, 0);
+	ret = ucsi_send_command(con->ucsi, ctrl, NULL, 0);
 	if (ret == -ETIMEDOUT) {
 		struct ucsi_control c;
 
 		/* PPM most likely stopped responding. Resetting everything. */
+		mutex_lock(&con->ucsi->ppm_lock);
 		ucsi_reset_ppm(con->ucsi);
+		mutex_unlock(&con->ucsi->ppm_lock);
 
 		UCSI_CMD_SET_NTFY_ENABLE(c, UCSI_ENABLE_NTFY_ALL);
-		ucsi_run_command(con->ucsi, &c, NULL, 0);
+		ucsi_send_command(con->ucsi, &c, NULL, 0);
 
 		ucsi_reset_connector(con, true);
 	}
@@ -504,10 +710,12 @@ ucsi_dr_swap(const struct typec_capability *cap, enum typec_data_role role)
 	struct ucsi_control ctrl;
 	int ret = 0;
 
-	if (!con->partner)
-		return -ENOTCONN;
+	mutex_lock(&con->lock);
 
-	mutex_lock(&con->ucsi->ppm_lock);
+	if (!con->partner) {
+		ret = -ENOTCONN;
+		goto out_unlock;
+	}
 
 	if ((con->status.partner_type == UCSI_CONSTAT_PARTNER_TYPE_DFP &&
 	     role == TYPEC_DEVICE) ||
@@ -520,18 +728,14 @@ ucsi_dr_swap(const struct typec_capability *cap, enum typec_data_role role)
 	if (ret < 0)
 		goto out_unlock;
 
-	mutex_unlock(&con->ucsi->ppm_lock);
-
 	if (!wait_for_completion_timeout(&con->complete,
 					msecs_to_jiffies(UCSI_SWAP_TIMEOUT_MS)))
-		return -ETIMEDOUT;
-
-	return 0;
+		ret = -ETIMEDOUT;
 
 out_unlock:
-	mutex_unlock(&con->ucsi->ppm_lock);
+	mutex_unlock(&con->lock);
 
-	return ret;
+	return ret < 0 ? ret : 0;
 }
 
 static int
@@ -541,10 +745,12 @@ ucsi_pr_swap(const struct typec_capability *cap, enum typec_role role)
 	struct ucsi_control ctrl;
 	int ret = 0;
 
-	if (!con->partner)
-		return -ENOTCONN;
+	mutex_lock(&con->lock);
 
-	mutex_lock(&con->ucsi->ppm_lock);
+	if (!con->partner) {
+		ret = -ENOTCONN;
+		goto out_unlock;
+	}
 
 	if (con->status.pwr_dir == role)
 		goto out_unlock;
@@ -554,13 +760,11 @@ ucsi_pr_swap(const struct typec_capability *cap, enum typec_role role)
 	if (ret < 0)
 		goto out_unlock;
 
-	mutex_unlock(&con->ucsi->ppm_lock);
-
 	if (!wait_for_completion_timeout(&con->complete,
-					msecs_to_jiffies(UCSI_SWAP_TIMEOUT_MS)))
-		return -ETIMEDOUT;
-
-	mutex_lock(&con->ucsi->ppm_lock);
+				msecs_to_jiffies(UCSI_SWAP_TIMEOUT_MS))) {
+		ret = -ETIMEDOUT;
+		goto out_unlock;
+	}
 
 	/* Something has gone wrong while swapping the role */
 	if (con->status.pwr_op_mode != UCSI_CONSTAT_PWR_OPMODE_PD) {
@@ -569,7 +773,7 @@ ucsi_pr_swap(const struct typec_capability *cap, enum typec_role role)
 	}
 
 out_unlock:
-	mutex_unlock(&con->ucsi->ppm_lock);
+	mutex_unlock(&con->lock);
 
 	return ret;
 }
@@ -595,6 +799,7 @@ static int ucsi_register_port(struct ucsi *ucsi, int index)
 
 	INIT_WORK(&con->work, ucsi_connector_change);
 	init_completion(&con->complete);
+	mutex_init(&con->lock);
 	con->num = index + 1;
 	con->ucsi = ucsi;
 
@@ -636,6 +841,12 @@ static int ucsi_register_port(struct ucsi *ucsi, int index)
 	if (IS_ERR(con->port))
 		return PTR_ERR(con->port);
 
+	/* Alternate modes */
+	ret = ucsi_register_altmodes(con, UCSI_RECIPIENT_CON);
+	if (ret)
+		dev_err(ucsi->dev, "con%d: failed to register alt modes\n",
+			con->num);
+
 	/* Get the status */
 	UCSI_CMD_GET_CONNECTOR_STATUS(ctrl, con->num);
 	ret = ucsi_run_command(ucsi, &ctrl, &con->status, sizeof(con->status));
@@ -662,6 +873,16 @@ static int ucsi_register_port(struct ucsi *ucsi, int index)
 	if (con->status.connected)
 		ucsi_register_partner(con);
 
+	if (con->partner) {
+		ret = ucsi_register_altmodes(con, UCSI_RECIPIENT_SOP);
+		if (ret)
+			dev_err(ucsi->dev,
+				"con%d: failed to register alternate modes\n",
+				con->num);
+		else
+			ucsi_altmode_update_active(con);
+	}
+
 	trace_ucsi_register_port(con->num, &con->status);
 
 	return 0;
@@ -730,6 +951,7 @@ static void ucsi_init(struct work_struct *work)
 err_unregister:
 	for (con = ucsi->connector; con->port; con++) {
 		ucsi_unregister_partner(con);
+		ucsi_unregister_altmodes(con, UCSI_RECIPIENT_CON);
 		typec_unregister_port(con->port);
 		con->port = NULL;
 	}
@@ -788,17 +1010,15 @@ void ucsi_unregister_ppm(struct ucsi *ucsi)
 	/* Make sure that we are not in the middle of driver initialization */
 	cancel_work_sync(&ucsi->work);
 
-	mutex_lock(&ucsi->ppm_lock);
-
 	/* Disable everything except command complete notification */
 	UCSI_CMD_SET_NTFY_ENABLE(ctrl, UCSI_ENABLE_NTFY_CMD_COMPLETE)
-	ucsi_run_command(ucsi, &ctrl, NULL, 0);
-
-	mutex_unlock(&ucsi->ppm_lock);
+	ucsi_send_command(ucsi, &ctrl, NULL, 0);
 
 	for (i = 0; i < ucsi->cap.num_connectors; i++) {
 		cancel_work_sync(&ucsi->connector[i].work);
 		ucsi_unregister_partner(&ucsi->connector[i]);
+		ucsi_unregister_altmodes(&ucsi->connector[i],
+					 UCSI_RECIPIENT_CON);
 		typec_unregister_port(ucsi->connector[i].port);
 	}
 
diff --git a/drivers/usb/typec/ucsi/ucsi.h b/drivers/usb/typec/ucsi/ucsi.h
index 53b80f40a908..1e2981aef629 100644
--- a/drivers/usb/typec/ucsi/ucsi.h
+++ b/drivers/usb/typec/ucsi/ucsi.h
@@ -6,6 +6,7 @@
 #include <linux/bitops.h>
 #include <linux/device.h>
 #include <linux/types.h>
+#include <linux/usb/typec.h>
 
 /* -------------------------------------------------------------------------- */
 
@@ -60,6 +61,20 @@ struct ucsi_uor_cmd {
 	u16:6; /* reserved */
 } __packed;
 
+/* Get Alternate Modes Command structure */
+struct ucsi_altmode_cmd {
+	u8 cmd;
+	u8 length;
+	u8 recipient;
+#define UCSI_RECIPIENT_CON			0
+#define UCSI_RECIPIENT_SOP			1
+#define UCSI_RECIPIENT_SOP_P			2
+#define UCSI_RECIPIENT_SOP_PP			3
+	u8 con_num;
+	u8 offset;
+	u8 num_altmodes;
+} __packed;
+
 struct ucsi_control {
 	union {
 		u64 raw_cmd;
@@ -67,6 +82,7 @@ struct ucsi_control {
 		struct ucsi_uor_cmd uor;
 		struct ucsi_ack_cmd ack;
 		struct ucsi_con_rst con_rst;
+		struct ucsi_altmode_cmd alt;
 	};
 };
 
@@ -112,6 +128,30 @@ struct ucsi_control {
 	(_ctrl_).cmd.data = _con_;					\
 }
 
+/* Helper for preparing ucsi_control for GET_ALTERNATE_MODES command. */
+#define UCSI_CMD_GET_ALTERNATE_MODES(_ctrl_, _r_, _con_num_, _o_, _num_)\
+{									\
+	__UCSI_CMD((_ctrl_), UCSI_GET_ALTERNATE_MODES)			\
+	_ctrl_.alt.recipient = (_r_);					\
+	_ctrl_.alt.con_num = (_con_num_);				\
+	_ctrl_.alt.offset = (_o_);					\
+	_ctrl_.alt.num_altmodes = (_num_) - 1;				\
+}
+
+/* Helper for preparing ucsi_control for GET_CAM_SUPPORTED command. */
+#define UCSI_CMD_GET_CAM_SUPPORTED(_ctrl_, _con_)			\
+{									\
+	__UCSI_CMD((_ctrl_), UCSI_GET_CAM_SUPPORTED)			\
+	_ctrl_.cmd.data = (_con_);					\
+}
+
+/* Helper for preparing ucsi_control for GET_CAM_SUPPORTED command. */
+#define UCSI_CMD_GET_CURRENT_CAM(_ctrl_, _con_)			\
+{									\
+	__UCSI_CMD((_ctrl_), UCSI_GET_CURRENT_CAM)			\
+	_ctrl_.cmd.data = (_con_);					\
+}
+
 /* Helper for preparing ucsi_control for GET_CONNECTOR_STATUS command. */
 #define UCSI_CMD_GET_CONNECTOR_STATUS(_ctrl_, _con_)			\
 {									\
@@ -334,4 +374,82 @@ struct ucsi *ucsi_register_ppm(struct device *dev, struct ucsi_ppm *ppm);
 void ucsi_unregister_ppm(struct ucsi *ucsi);
 void ucsi_notify(struct ucsi *ucsi);
 
+/* -------------------------------------------------------------------------- */
+
+enum ucsi_status {
+	UCSI_IDLE = 0,
+	UCSI_BUSY,
+	UCSI_ERROR,
+};
+
+struct ucsi {
+	struct device *dev;
+	struct ucsi_ppm *ppm;
+
+	enum ucsi_status status;
+	struct completion complete;
+	struct ucsi_capability cap;
+	struct ucsi_connector *connector;
+
+	struct work_struct work;
+
+	/* PPM Communication lock */
+	struct mutex ppm_lock;
+
+	/* PPM communication flags */
+	unsigned long flags;
+#define EVENT_PENDING	0
+#define COMMAND_PENDING	1
+#define ACK_PENDING	2
+};
+
+#define UCSI_MAX_SVID		5
+#define UCSI_MAX_ALTMODES	(UCSI_MAX_SVID * 6)
+
+struct ucsi_connector {
+	int num;
+
+	struct ucsi *ucsi;
+	struct mutex lock; /* port lock */
+	struct work_struct work;
+	struct completion complete;
+
+	struct typec_port *port;
+	struct typec_partner *partner;
+
+	struct typec_altmode *port_altmode[UCSI_MAX_ALTMODES];
+	struct typec_altmode *partner_altmode[UCSI_MAX_ALTMODES];
+
+	struct typec_capability typec_cap;
+
+	struct ucsi_connector_status status;
+	struct ucsi_connector_capability cap;
+};
+
+int ucsi_send_command(struct ucsi *ucsi, struct ucsi_control *ctrl,
+		      void *retval, size_t size);
+
+void ucsi_altmode_update_active(struct ucsi_connector *con);
+
+#if IS_ENABLED(CONFIG_TYPEC_DP_ALTMODE)
+struct typec_altmode *
+ucsi_register_displayport(struct ucsi_connector *con,
+			  bool override, int offset,
+			  struct typec_altmode_desc *desc);
+
+void ucsi_displayport_remove_partner(struct typec_altmode *adev);
+
+#else
+static inline struct typec_altmode *
+ucsi_register_displayport(struct ucsi_connector *con,
+			  bool override, int offset,
+			  struct typec_altmode_desc *desc)
+{
+	return NULL;
+}
+
+static inline void
+ucsi_displayport_remove_partner(struct typec_altmode *adev) { }
+#endif /* CONFIG_TYPEC_DP_ALTMODE */
+
 #endif /* __DRIVER_USB_TYPEC_UCSI_H */
diff --git a/drivers/usb/typec/ucsi/ucsi_ccg.c b/drivers/usb/typec/ucsi/ucsi_ccg.c
index de8a43bdff68..9d46aa9e4e35 100644
--- a/drivers/usb/typec/ucsi/ucsi_ccg.c
+++ b/drivers/usb/typec/ucsi/ucsi_ccg.c
@@ -9,6 +9,7 @@
  */
 #include <linux/acpi.h>
 #include <linux/delay.h>
+#include <linux/firmware.h>
 #include <linux/i2c.h>
 #include <linux/module.h>
 #include <linux/pci.h>
@@ -17,18 +18,172 @@
 #include <asm/unaligned.h>
 #include "ucsi.h"
 
+enum enum_fw_mode {
+	BOOT,   /* bootloader */
+	FW1,    /* FW partition-1 (contains secondary fw) */
+	FW2,    /* FW partition-2 (contains primary fw) */
+	FW_INVALID,
+};
+
+#define CCGX_RAB_DEVICE_MODE			0x0000
+#define CCGX_RAB_INTR_REG			0x0006
+#define  DEV_INT				BIT(0)
+#define  PORT0_INT				BIT(1)
+#define  PORT1_INT				BIT(2)
+#define  UCSI_READ_INT				BIT(7)
+#define CCGX_RAB_JUMP_TO_BOOT			0x0007
+#define  TO_BOOT				'J'
+#define  TO_ALT_FW				'A'
+#define CCGX_RAB_RESET_REQ			0x0008
+#define  RESET_SIG				'R'
+#define  CMD_RESET_I2C				0x0
+#define  CMD_RESET_DEV				0x1
+#define CCGX_RAB_ENTER_FLASHING			0x000A
+#define  FLASH_ENTER_SIG			'P'
+#define CCGX_RAB_VALIDATE_FW			0x000B
+#define CCGX_RAB_FLASH_ROW_RW			0x000C
+#define  FLASH_SIG				'F'
+#define  FLASH_RD_CMD				0x0
+#define  FLASH_WR_CMD				0x1
+#define  FLASH_FWCT1_WR_CMD			0x2
+#define  FLASH_FWCT2_WR_CMD			0x3
+#define  FLASH_FWCT_SIG_WR_CMD			0x4
+#define CCGX_RAB_READ_ALL_VER			0x0010
+#define CCGX_RAB_READ_FW2_VER			0x0020
+#define CCGX_RAB_UCSI_CONTROL			0x0039
+#define CCGX_RAB_UCSI_CONTROL_START		BIT(0)
+#define CCGX_RAB_UCSI_CONTROL_STOP		BIT(1)
+#define CCGX_RAB_UCSI_DATA_BLOCK(offset)	(0xf000 | ((offset) & 0xff))
+#define REG_FLASH_RW_MEM        0x0200
+#define DEV_REG_IDX				CCGX_RAB_DEVICE_MODE
+#define CCGX_RAB_PDPORT_ENABLE			0x002C
+#define  PDPORT_1		BIT(0)
+#define  PDPORT_2		BIT(1)
+#define CCGX_RAB_RESPONSE			0x007E
+#define  ASYNC_EVENT				BIT(7)
+
+/* CCGx events & async msg codes */
+#define RESET_COMPLETE		0x80
+#define EVENT_INDEX		RESET_COMPLETE
+#define PORT_CONNECT_DET	0x84
+#define PORT_DISCONNECT_DET	0x85
+#define ROLE_SWAP_COMPELETE	0x87
+
+/* ccg firmware */
+#define CYACD_LINE_SIZE         527
+#define CCG4_ROW_SIZE           256
+#define FW1_METADATA_ROW        0x1FF
+#define FW2_METADATA_ROW        0x1FE
+#define FW_CFG_TABLE_SIG_SIZE	256
+
+static int secondary_fw_min_ver = 41;
+
+enum enum_flash_mode {
+	SECONDARY_BL,	/* update secondary using bootloader */
+	PRIMARY,	/* update primary using secondary */
+	SECONDARY,	/* update secondary using primary */
+	FLASH_NOT_NEEDED,	/* update not required */
+	FLASH_INVALID,
+};
+
+static const char * const ccg_fw_names[] = {
+	"ccg_boot.cyacd",
+	"ccg_primary.cyacd",
+	"ccg_secondary.cyacd"
+};
+
+struct ccg_dev_info {
+#define CCG_DEVINFO_FWMODE_SHIFT (0)
+#define CCG_DEVINFO_FWMODE_MASK (0x3 << CCG_DEVINFO_FWMODE_SHIFT)
+#define CCG_DEVINFO_PDPORTS_SHIFT (2)
+#define CCG_DEVINFO_PDPORTS_MASK (0x3 << CCG_DEVINFO_PDPORTS_SHIFT)
+	u8 mode;
+	u8 bl_mode;
+	__le16 silicon_id;
+	__le16 bl_last_row;
+} __packed;
+
+struct version_format {
+	__le16 build;
+	u8 patch;
+	u8 ver;
+#define CCG_VERSION_MIN_SHIFT (0)
+#define CCG_VERSION_MIN_MASK (0xf << CCG_VERSION_MIN_SHIFT)
+#define CCG_VERSION_MAJ_SHIFT (4)
+#define CCG_VERSION_MAJ_MASK (0xf << CCG_VERSION_MAJ_SHIFT)
+} __packed;
+
+struct version_info {
+	struct version_format base;
+	struct version_format app;
+};
+
+struct fw_config_table {
+	u32 identity;
+	u16 table_size;
+	u8 fwct_version;
+	u8 is_key_change;
+	u8 guid[16];
+	struct version_format base;
+	struct version_format app;
+	u8 primary_fw_digest[32];
+	u32 key_exp_length;
+	u8 key_modulus[256];
+	u8 key_exp[4];
+};
+
+/* CCGx response codes */
+enum ccg_resp_code {
+	CMD_NO_RESP             = 0x00,
+	CMD_SUCCESS             = 0x02,
+	FLASH_DATA_AVAILABLE    = 0x03,
+	CMD_INVALID             = 0x05,
+	FLASH_UPDATE_FAIL       = 0x07,
+	INVALID_FW              = 0x08,
+	INVALID_ARG             = 0x09,
+	CMD_NOT_SUPPORT         = 0x0A,
+	TRANSACTION_FAIL        = 0x0C,
+	PD_CMD_FAIL             = 0x0D,
+	UNDEF_ERROR             = 0x0F,
+	INVALID_RESP		= 0x10,
+};
+
+#define CCG_EVENT_MAX	(EVENT_INDEX + 43)
+
+struct ccg_cmd {
+	u16 reg;
+	u32 data;
+	int len;
+	u32 delay; /* ms delay for cmd timeout  */
+};
+
+struct ccg_resp {
+	u8 code;
+	u8 length;
+};
+
 struct ucsi_ccg {
 	struct device *dev;
 	struct ucsi *ucsi;
 	struct ucsi_ppm ppm;
 	struct i2c_client *client;
-};
+	struct ccg_dev_info info;
+	/* version info for boot, primary and secondary */
+	struct version_info version[FW2 + 1];
+	/* CCG HPI communication flags */
+	unsigned long flags;
+#define RESET_PENDING	0
+#define DEV_CMD_PENDING	1
+	struct ccg_resp dev_resp;
+	u8 cmd_resp;
+	int port_num;
+	int irq;
+	struct work_struct work;
+	struct mutex lock; /* to sync between user and driver thread */
 
-#define CCGX_RAB_INTR_REG			0x06
-#define CCGX_RAB_UCSI_CONTROL			0x39
-#define CCGX_RAB_UCSI_CONTROL_START		BIT(0)
-#define CCGX_RAB_UCSI_CONTROL_STOP		BIT(1)
-#define CCGX_RAB_UCSI_DATA_BLOCK(offset)	(0xf000 | ((offset) & 0xff))
+	/* fw build with vendor information */
+	u16 fw_build;
+};
 
 static int ccg_read(struct ucsi_ccg *uc, u16 rab, u8 *data, u32 len)
 {
@@ -220,6 +375,687 @@ static irqreturn_t ccg_irq_handler(int irq, void *data)
 	return IRQ_HANDLED;
 }
 
+static int get_fw_info(struct ucsi_ccg *uc)
+{
+	int err;
+
+	err = ccg_read(uc, CCGX_RAB_READ_ALL_VER, (u8 *)(&uc->version),
+		       sizeof(uc->version));
+	if (err < 0)
+		return err;
+
+	err = ccg_read(uc, CCGX_RAB_DEVICE_MODE, (u8 *)(&uc->info),
+		       sizeof(uc->info));
+	if (err < 0)
+		return err;
+
+	return 0;
+}
+
+static inline bool invalid_async_evt(int code)
+{
+	return (code >= CCG_EVENT_MAX) || (code < EVENT_INDEX);
+}
+
+static void ccg_process_response(struct ucsi_ccg *uc)
+{
+	struct device *dev = uc->dev;
+
+	if (uc->dev_resp.code & ASYNC_EVENT) {
+		if (uc->dev_resp.code == RESET_COMPLETE) {
+			if (test_bit(RESET_PENDING, &uc->flags))
+				uc->cmd_resp = uc->dev_resp.code;
+			get_fw_info(uc);
+		}
+		if (invalid_async_evt(uc->dev_resp.code))
+			dev_err(dev, "invalid async evt %d\n",
+				uc->dev_resp.code);
+	} else {
+		if (test_bit(DEV_CMD_PENDING, &uc->flags)) {
+			uc->cmd_resp = uc->dev_resp.code;
+			clear_bit(DEV_CMD_PENDING, &uc->flags);
+		} else {
+			dev_err(dev, "dev resp 0x%04x but no cmd pending\n",
+				uc->dev_resp.code);
+		}
+	}
+}
+
+static int ccg_read_response(struct ucsi_ccg *uc)
+{
+	unsigned long target = jiffies + msecs_to_jiffies(1000);
+	struct device *dev = uc->dev;
+	u8 intval;
+	int status;
+
+	/* wait for interrupt status to get updated */
+	do {
+		status = ccg_read(uc, CCGX_RAB_INTR_REG, &intval,
+				  sizeof(intval));
+		if (status < 0)
+			return status;
+
+		if (intval & DEV_INT)
+			break;
+		usleep_range(500, 600);
+	} while (time_is_after_jiffies(target));
+
+	if (time_is_before_jiffies(target)) {
+		dev_err(dev, "response timeout error\n");
+		return -ETIME;
+	}
+
+	status = ccg_read(uc, CCGX_RAB_RESPONSE, (u8 *)&uc->dev_resp,
+			  sizeof(uc->dev_resp));
+	if (status < 0)
+		return status;
+
+	status = ccg_write(uc, CCGX_RAB_INTR_REG, &intval, sizeof(intval));
+	if (status < 0)
+		return status;
+
+	return 0;
+}
+
+/* Caller must hold uc->lock */
+static int ccg_send_command(struct ucsi_ccg *uc, struct ccg_cmd *cmd)
+{
+	struct device *dev = uc->dev;
+	int ret;
+
+	switch (cmd->reg & 0xF000) {
+	case DEV_REG_IDX:
+		set_bit(DEV_CMD_PENDING, &uc->flags);
+		break;
+	default:
+		dev_err(dev, "invalid cmd register\n");
+		break;
+	}
+
+	ret = ccg_write(uc, cmd->reg, (u8 *)&cmd->data, cmd->len);
+	if (ret < 0)
+		return ret;
+
+	msleep(cmd->delay);
+
+	ret = ccg_read_response(uc);
+	if (ret < 0) {
+		dev_err(dev, "response read error\n");
+		switch (cmd->reg & 0xF000) {
+		case DEV_REG_IDX:
+			clear_bit(DEV_CMD_PENDING, &uc->flags);
+			break;
+		default:
+			dev_err(dev, "invalid cmd register\n");
+			break;
+		}
+		return -EIO;
+	}
+	ccg_process_response(uc);
+
+	return uc->cmd_resp;
+}
+
+static int ccg_cmd_enter_flashing(struct ucsi_ccg *uc)
+{
+	struct ccg_cmd cmd;
+	int ret;
+
+	cmd.reg = CCGX_RAB_ENTER_FLASHING;
+	cmd.data = FLASH_ENTER_SIG;
+	cmd.len = 1;
+	cmd.delay = 50;
+
+	mutex_lock(&uc->lock);
+
+	ret = ccg_send_command(uc, &cmd);
+
+	mutex_unlock(&uc->lock);
+
+	if (ret != CMD_SUCCESS) {
+		dev_err(uc->dev, "enter flashing failed ret=%d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int ccg_cmd_reset(struct ucsi_ccg *uc)
+{
+	struct ccg_cmd cmd;
+	u8 *p;
+	int ret;
+
+	p = (u8 *)&cmd.data;
+	cmd.reg = CCGX_RAB_RESET_REQ;
+	p[0] = RESET_SIG;
+	p[1] = CMD_RESET_DEV;
+	cmd.len = 2;
+	cmd.delay = 5000;
+
+	mutex_lock(&uc->lock);
+
+	set_bit(RESET_PENDING, &uc->flags);
+
+	ret = ccg_send_command(uc, &cmd);
+	if (ret != RESET_COMPLETE)
+		goto err_clear_flag;
+
+	ret = 0;
+
+err_clear_flag:
+	clear_bit(RESET_PENDING, &uc->flags);
+
+	mutex_unlock(&uc->lock);
+
+	return ret;
+}
+
+static int ccg_cmd_port_control(struct ucsi_ccg *uc, bool enable)
+{
+	struct ccg_cmd cmd;
+	int ret;
+
+	cmd.reg = CCGX_RAB_PDPORT_ENABLE;
+	if (enable)
+		cmd.data = (uc->port_num == 1) ?
+			    PDPORT_1 : (PDPORT_1 | PDPORT_2);
+	else
+		cmd.data = 0x0;
+	cmd.len = 1;
+	cmd.delay = 10;
+
+	mutex_lock(&uc->lock);
+
+	ret = ccg_send_command(uc, &cmd);
+
+	mutex_unlock(&uc->lock);
+
+	if (ret != CMD_SUCCESS) {
+		dev_err(uc->dev, "port control failed ret=%d\n", ret);
+		return ret;
+	}
+	return 0;
+}
+
+static int ccg_cmd_jump_boot_mode(struct ucsi_ccg *uc, int bl_mode)
+{
+	struct ccg_cmd cmd;
+	int ret;
+
+	cmd.reg = CCGX_RAB_JUMP_TO_BOOT;
+
+	if (bl_mode)
+		cmd.data = TO_BOOT;
+	else
+		cmd.data = TO_ALT_FW;
+
+	cmd.len = 1;
+	cmd.delay = 100;
+
+	mutex_lock(&uc->lock);
+
+	set_bit(RESET_PENDING, &uc->flags);
+
+	ret = ccg_send_command(uc, &cmd);
+	if (ret != RESET_COMPLETE)
+		goto err_clear_flag;
+
+	ret = 0;
+
+err_clear_flag:
+	clear_bit(RESET_PENDING, &uc->flags);
+
+	mutex_unlock(&uc->lock);
+
+	return ret;
+}
+
+static int
+ccg_cmd_write_flash_row(struct ucsi_ccg *uc, u16 row,
+			const void *data, u8 fcmd)
+{
+	struct i2c_client *client = uc->client;
+	struct ccg_cmd cmd;
+	u8 buf[CCG4_ROW_SIZE + 2];
+	u8 *p;
+	int ret;
+
+	/* Copy the data into the flash read/write memory. */
+	put_unaligned_le16(REG_FLASH_RW_MEM, buf);
+
+	memcpy(buf + 2, data, CCG4_ROW_SIZE);
+
+	mutex_lock(&uc->lock);
+
+	ret = i2c_master_send(client, buf, CCG4_ROW_SIZE + 2);
+	if (ret != CCG4_ROW_SIZE + 2) {
+		dev_err(uc->dev, "REG_FLASH_RW_MEM write fail %d\n", ret);
+		mutex_unlock(&uc->lock);
+		return ret < 0 ? ret : -EIO;
+	}
+
+	/* Use the FLASH_ROW_READ_WRITE register to trigger */
+	/* writing of data to the desired flash row */
+	p = (u8 *)&cmd.data;
+	cmd.reg = CCGX_RAB_FLASH_ROW_RW;
+	p[0] = FLASH_SIG;
+	p[1] = fcmd;
+	put_unaligned_le16(row, &p[2]);
+	cmd.len = 4;
+	cmd.delay = 50;
+	if (fcmd == FLASH_FWCT_SIG_WR_CMD)
+		cmd.delay += 400;
+	if (row == 510)
+		cmd.delay += 220;
+	ret = ccg_send_command(uc, &cmd);
+
+	mutex_unlock(&uc->lock);
+
+	if (ret != CMD_SUCCESS) {
+		dev_err(uc->dev, "write flash row failed ret=%d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int ccg_cmd_validate_fw(struct ucsi_ccg *uc, unsigned int fwid)
+{
+	struct ccg_cmd cmd;
+	int ret;
+
+	cmd.reg = CCGX_RAB_VALIDATE_FW;
+	cmd.data = fwid;
+	cmd.len = 1;
+	cmd.delay = 500;
+
+	mutex_lock(&uc->lock);
+
+	ret = ccg_send_command(uc, &cmd);
+
+	mutex_unlock(&uc->lock);
+
+	if (ret != CMD_SUCCESS)
+		return ret;
+
+	return 0;
+}
+
+static bool ccg_check_vendor_version(struct ucsi_ccg *uc,
+				     struct version_format *app,
+				     struct fw_config_table *fw_cfg)
+{
+	struct device *dev = uc->dev;
+
+	/* Check if the fw build is for supported vendors */
+	if (le16_to_cpu(app->build) != uc->fw_build) {
+		dev_info(dev, "current fw is not from supported vendor\n");
+		return false;
+	}
+
+	/* Check if the new fw build is for supported vendors */
+	if (le16_to_cpu(fw_cfg->app.build) != uc->fw_build) {
+		dev_info(dev, "new fw is not from supported vendor\n");
+		return false;
+	}
+	return true;
+}
+
+static bool ccg_check_fw_version(struct ucsi_ccg *uc, const char *fw_name,
+				 struct version_format *app)
+{
+	const struct firmware *fw = NULL;
+	struct device *dev = uc->dev;
+	struct fw_config_table fw_cfg;
+	u32 cur_version, new_version;
+	bool is_later = false;
+
+	if (request_firmware(&fw, fw_name, dev) != 0) {
+		dev_err(dev, "error: Failed to open cyacd file %s\n", fw_name);
+		return false;
+	}
+
+	/*
+	 * check if signed fw
+	 * last part of fw image is fw cfg table and signature
+	 */
+	if (fw->size < sizeof(fw_cfg) + FW_CFG_TABLE_SIG_SIZE)
+		goto out_release_firmware;
+
+	memcpy((uint8_t *)&fw_cfg, fw->data + fw->size -
+	       sizeof(fw_cfg) - FW_CFG_TABLE_SIG_SIZE, sizeof(fw_cfg));
+
+	if (fw_cfg.identity != ('F' | 'W' << 8 | 'C' << 16 | 'T' << 24)) {
+		dev_info(dev, "not a signed image\n");
+		goto out_release_firmware;
+	}
+
+	/* compare input version with FWCT version */
+	cur_version = le16_to_cpu(app->build) | app->patch << 16 |
+			app->ver << 24;
+
+	new_version = le16_to_cpu(fw_cfg.app.build) | fw_cfg.app.patch << 16 |
+			fw_cfg.app.ver << 24;
+
+	if (!ccg_check_vendor_version(uc, app, &fw_cfg))
+		goto out_release_firmware;
+
+	if (new_version > cur_version)
+		is_later = true;
+
+out_release_firmware:
+	release_firmware(fw);
+	return is_later;
+}
+
+static int ccg_fw_update_needed(struct ucsi_ccg *uc,
+				enum enum_flash_mode *mode)
+{
+	struct device *dev = uc->dev;
+	int err;
+	struct version_info version[3];
+
+	err = ccg_read(uc, CCGX_RAB_DEVICE_MODE, (u8 *)(&uc->info),
+		       sizeof(uc->info));
+	if (err) {
+		dev_err(dev, "read device mode failed\n");
+		return err;
+	}
+
+	err = ccg_read(uc, CCGX_RAB_READ_ALL_VER, (u8 *)version,
+		       sizeof(version));
+	if (err) {
+		dev_err(dev, "read device mode failed\n");
+		return err;
+	}
+
+	if (memcmp(&version[FW1], "\0\0\0\0\0\0\0\0",
+		   sizeof(struct version_info)) == 0) {
+		dev_info(dev, "secondary fw is not flashed\n");
+		*mode = SECONDARY_BL;
+	} else if (le16_to_cpu(version[FW1].base.build) <
+		secondary_fw_min_ver) {
+		dev_info(dev, "secondary fw version is too low (< %d)\n",
+			 secondary_fw_min_ver);
+		*mode = SECONDARY;
+	} else if (memcmp(&version[FW2], "\0\0\0\0\0\0\0\0",
+		   sizeof(struct version_info)) == 0) {
+		dev_info(dev, "primary fw is not flashed\n");
+		*mode = PRIMARY;
+	} else if (ccg_check_fw_version(uc, ccg_fw_names[PRIMARY],
+		   &version[FW2].app)) {
+		dev_info(dev, "found primary fw with later version\n");
+		*mode = PRIMARY;
+	} else {
+		dev_info(dev, "secondary and primary fw are the latest\n");
+		*mode = FLASH_NOT_NEEDED;
+	}
+	return 0;
+}
+
+static int do_flash(struct ucsi_ccg *uc, enum enum_flash_mode mode)
+{
+	struct device *dev = uc->dev;
+	const struct firmware *fw = NULL;
+	const char *p, *s;
+	const char *eof;
+	int err, row, len, line_sz, line_cnt = 0;
+	unsigned long start_time = jiffies;
+	struct fw_config_table  fw_cfg;
+	u8 fw_cfg_sig[FW_CFG_TABLE_SIG_SIZE];
+	u8 *wr_buf;
+
+	err = request_firmware(&fw, ccg_fw_names[mode], dev);
+	if (err) {
+		dev_err(dev, "request %s failed err=%d\n",
+			ccg_fw_names[mode], err);
+		return err;
+	}
+
+	if (((uc->info.mode & CCG_DEVINFO_FWMODE_MASK) >>
+			CCG_DEVINFO_FWMODE_SHIFT) == FW2) {
+		err = ccg_cmd_port_control(uc, false);
+		if (err < 0)
+			goto release_fw;
+		err = ccg_cmd_jump_boot_mode(uc, 0);
+		if (err < 0)
+			goto release_fw;
+	}
+
+	eof = fw->data + fw->size;
+
+	/*
+	 * check if signed fw
+	 * last part of fw image is fw cfg table and signature
+	 */
+	if (fw->size < sizeof(fw_cfg) + sizeof(fw_cfg_sig))
+		goto not_signed_fw;
+
+	memcpy((uint8_t *)&fw_cfg, fw->data + fw->size -
+	       sizeof(fw_cfg) - sizeof(fw_cfg_sig), sizeof(fw_cfg));
+
+	if (fw_cfg.identity != ('F' | ('W' << 8) | ('C' << 16) | ('T' << 24))) {
+		dev_info(dev, "not a signed image\n");
+		goto not_signed_fw;
+	}
+	eof = fw->data + fw->size - sizeof(fw_cfg) - sizeof(fw_cfg_sig);
+
+	memcpy((uint8_t *)&fw_cfg_sig,
+	       fw->data + fw->size - sizeof(fw_cfg_sig), sizeof(fw_cfg_sig));
+
+	/* flash fw config table and signature first */
+	err = ccg_cmd_write_flash_row(uc, 0, (u8 *)&fw_cfg,
+				      FLASH_FWCT1_WR_CMD);
+	if (err)
+		goto release_fw;
+
+	err = ccg_cmd_write_flash_row(uc, 0, (u8 *)&fw_cfg + CCG4_ROW_SIZE,
+				      FLASH_FWCT2_WR_CMD);
+	if (err)
+		goto release_fw;
+
+	err = ccg_cmd_write_flash_row(uc, 0, &fw_cfg_sig,
+				      FLASH_FWCT_SIG_WR_CMD);
+	if (err)
+		goto release_fw;
+
+not_signed_fw:
+	wr_buf = kzalloc(CCG4_ROW_SIZE + 4, GFP_KERNEL);
+	if (!wr_buf)
+		return -ENOMEM;
+
+	err = ccg_cmd_enter_flashing(uc);
+	if (err)
+		goto release_mem;
+
+	/*****************************************************************
+	 * CCG firmware image (.cyacd) file line format
+	 *
+	 * :00rrrrllll[dd....]cc/r/n
+	 *
+	 * :00   header
+	 * rrrr is row number to flash				(4 char)
+	 * llll is data len to flash				(4 char)
+	 * dd   is a data field represents one byte of data	(512 char)
+	 * cc   is checksum					(2 char)
+	 * \r\n newline
+	 *
+	 * Total length: 3 + 4 + 4 + 512 + 2 + 2 = 527
+	 *
+	 *****************************************************************/
+
+	p = strnchr(fw->data, fw->size, ':');
+	while (p < eof) {
+		s = strnchr(p + 1, eof - p - 1, ':');
+
+		if (!s)
+			s = eof;
+
+		line_sz = s - p;
+
+		if (line_sz != CYACD_LINE_SIZE) {
+			dev_err(dev, "Bad FW format line_sz=%d\n", line_sz);
+			err =  -EINVAL;
+			goto release_mem;
+		}
+
+		if (hex2bin(wr_buf, p + 3, CCG4_ROW_SIZE + 4)) {
+			err =  -EINVAL;
+			goto release_mem;
+		}
+
+		row = get_unaligned_be16(wr_buf);
+		len = get_unaligned_be16(&wr_buf[2]);
+
+		if (len != CCG4_ROW_SIZE) {
+			err =  -EINVAL;
+			goto release_mem;
+		}
+
+		err = ccg_cmd_write_flash_row(uc, row, wr_buf + 4,
+					      FLASH_WR_CMD);
+		if (err)
+			goto release_mem;
+
+		line_cnt++;
+		p = s;
+	}
+
+	dev_info(dev, "total %d row flashed. time: %dms\n",
+		 line_cnt, jiffies_to_msecs(jiffies - start_time));
+
+	err = ccg_cmd_validate_fw(uc, (mode == PRIMARY) ? FW2 :  FW1);
+	if (err)
+		dev_err(dev, "%s validation failed err=%d\n",
+			(mode == PRIMARY) ? "FW2" :  "FW1", err);
+	else
+		dev_info(dev, "%s validated\n",
+			 (mode == PRIMARY) ? "FW2" :  "FW1");
+
+	err = ccg_cmd_port_control(uc, false);
+	if (err < 0)
+		goto release_mem;
+
+	err = ccg_cmd_reset(uc);
+	if (err < 0)
+		goto release_mem;
+
+	err = ccg_cmd_port_control(uc, true);
+	if (err < 0)
+		goto release_mem;
+
+release_mem:
+	kfree(wr_buf);
+
+release_fw:
+	release_firmware(fw);
+	return err;
+}
+
+/*******************************************************************************
+ * CCG4 has two copies of the firmware in addition to the bootloader.
+ * If the device is running FW1, FW2 can be updated with the new version.
+ * Dual firmware mode allows the CCG device to stay in a PD contract and support
+ * USB PD and Type-C functionality while a firmware update is in progress.
+ ******************************************************************************/
+static int ccg_fw_update(struct ucsi_ccg *uc, enum enum_flash_mode flash_mode)
+{
+	int err;
+
+	while (flash_mode != FLASH_NOT_NEEDED) {
+		err = do_flash(uc, flash_mode);
+		if (err < 0)
+			return err;
+		err = ccg_fw_update_needed(uc, &flash_mode);
+		if (err < 0)
+			return err;
+	}
+	dev_info(uc->dev, "CCG FW update successful\n");
+
+	return err;
+}
+
+static int ccg_restart(struct ucsi_ccg *uc)
+{
+	struct device *dev = uc->dev;
+	int status;
+
+	status = ucsi_ccg_init(uc);
+	if (status < 0) {
+		dev_err(dev, "ucsi_ccg_start fail, err=%d\n", status);
+		return status;
+	}
+
+	status = request_threaded_irq(uc->irq, NULL, ccg_irq_handler,
+				      IRQF_ONESHOT | IRQF_TRIGGER_HIGH,
+				      dev_name(dev), uc);
+	if (status < 0) {
+		dev_err(dev, "request_threaded_irq failed - %d\n", status);
+		return status;
+	}
+
+	uc->ucsi = ucsi_register_ppm(dev, &uc->ppm);
+	if (IS_ERR(uc->ucsi)) {
+		dev_err(uc->dev, "ucsi_register_ppm failed\n");
+		return PTR_ERR(uc->ucsi);
+	}
+
+	return 0;
+}
+
+static void ccg_update_firmware(struct work_struct *work)
+{
+	struct ucsi_ccg *uc = container_of(work, struct ucsi_ccg, work);
+	enum enum_flash_mode flash_mode;
+	int status;
+
+	status = ccg_fw_update_needed(uc, &flash_mode);
+	if (status < 0)
+		return;
+
+	if (flash_mode != FLASH_NOT_NEEDED) {
+		ucsi_unregister_ppm(uc->ucsi);
+		free_irq(uc->irq, uc);
+
+		ccg_fw_update(uc, flash_mode);
+		ccg_restart(uc);
+	}
+}
+
+static ssize_t do_flash_store(struct device *dev,
+			      struct device_attribute *attr,
+			      const char *buf, size_t n)
+{
+	struct ucsi_ccg *uc = i2c_get_clientdata(to_i2c_client(dev));
+	bool flash;
+
+	if (kstrtobool(buf, &flash))
+		return -EINVAL;
+
+	if (!flash)
+		return n;
+
+	if (uc->fw_build == 0x0) {
+		dev_err(dev, "fail to flash FW due to missing FW build info\n");
+		return -EINVAL;
+	}
+
+	schedule_work(&uc->work);
+	return n;
+}
+
+static DEVICE_ATTR_WO(do_flash);
+
+static struct attribute *ucsi_ccg_sysfs_attrs[] = {
+	&dev_attr_do_flash.attr,
+	NULL,
+};
+
+static struct attribute_group ucsi_ccg_attr_group = {
+	.attrs = ucsi_ccg_sysfs_attrs,
+};
+
 static int ucsi_ccg_probe(struct i2c_client *client,
 			  const struct i2c_device_id *id)
 {
@@ -240,6 +1076,14 @@ static int ucsi_ccg_probe(struct i2c_client *client,
 	uc->ppm.sync = ucsi_ccg_sync;
 	uc->dev = dev;
 	uc->client = client;
+	mutex_init(&uc->lock);
+	INIT_WORK(&uc->work, ccg_update_firmware);
+
+	/* Only fail FW flashing when FW build information is not provided */
+	status = device_property_read_u16(dev, "ccgx,firmware-build",
+					  &uc->fw_build);
+	if (status)
+		dev_err(uc->dev, "failed to get FW build information\n");
 
 	/* reset ccg device and initialize ucsi */
 	status = ucsi_ccg_init(uc);
@@ -248,15 +1092,27 @@ static int ucsi_ccg_probe(struct i2c_client *client,
 		return status;
 	}
 
-	status = devm_request_threaded_irq(dev, client->irq, NULL,
-					   ccg_irq_handler,
-					   IRQF_ONESHOT | IRQF_TRIGGER_HIGH,
-					   dev_name(dev), uc);
+	status = get_fw_info(uc);
+	if (status < 0) {
+		dev_err(uc->dev, "get_fw_info failed - %d\n", status);
+		return status;
+	}
+
+	uc->port_num = 1;
+
+	if (uc->info.mode & CCG_DEVINFO_PDPORTS_MASK)
+		uc->port_num++;
+
+	status = request_threaded_irq(client->irq, NULL, ccg_irq_handler,
+				      IRQF_ONESHOT | IRQF_TRIGGER_HIGH,
+				      dev_name(dev), uc);
 	if (status < 0) {
 		dev_err(uc->dev, "request_threaded_irq failed - %d\n", status);
 		return status;
 	}
 
+	uc->irq = client->irq;
+
 	uc->ucsi = ucsi_register_ppm(dev, &uc->ppm);
 	if (IS_ERR(uc->ucsi)) {
 		dev_err(uc->dev, "ucsi_register_ppm failed\n");
@@ -273,6 +1129,11 @@ static int ucsi_ccg_probe(struct i2c_client *client,
 	}
 
 	i2c_set_clientdata(client, uc);
+
+	status = sysfs_create_group(&uc->dev->kobj, &ucsi_ccg_attr_group);
+	if (status)
+		dev_err(uc->dev, "cannot create sysfs group: %d\n", status);
+
 	return 0;
 }
 
@@ -280,7 +1141,10 @@ static int ucsi_ccg_remove(struct i2c_client *client)
 {
 	struct ucsi_ccg *uc = i2c_get_clientdata(client);
 
+	cancel_work_sync(&uc->work);
 	ucsi_unregister_ppm(uc->ucsi);
+	free_irq(uc->irq, uc);
+	sysfs_remove_group(&uc->dev->kobj, &ucsi_ccg_attr_group);
 
 	return 0;
 }
diff --git a/drivers/usb/usbip/stub_rx.c b/drivers/usb/usbip/stub_rx.c
index dbfb2f24d71e..b0a855acafa3 100644
--- a/drivers/usb/usbip/stub_rx.c
+++ b/drivers/usb/usbip/stub_rx.c
@@ -17,9 +17,9 @@ static int is_clear_halt_cmd(struct urb *urb)
 
 	req = (struct usb_ctrlrequest *) urb->setup_packet;
 
-	 return (req->bRequest == USB_REQ_CLEAR_FEATURE) &&
-		 (req->bRequestType == USB_RECIP_ENDPOINT) &&
-		 (req->wValue == USB_ENDPOINT_HALT);
+	return (req->bRequest == USB_REQ_CLEAR_FEATURE) &&
+	       (req->bRequestType == USB_RECIP_ENDPOINT) &&
+	       (req->wValue == USB_ENDPOINT_HALT);
 }
 
 static int is_set_interface_cmd(struct urb *urb)
diff --git a/drivers/usb/usbip/vhci_hcd.c b/drivers/usb/usbip/vhci_hcd.c
index f46ee1fefe02..000ab7225717 100644
--- a/drivers/usb/usbip/vhci_hcd.c
+++ b/drivers/usb/usbip/vhci_hcd.c
@@ -508,6 +508,7 @@ static int vhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue,
 		case USB_PORT_FEAT_U1_TIMEOUT:
 			usbip_dbg_vhci_rh(
 				" SetPortFeature: USB_PORT_FEAT_U1_TIMEOUT\n");
+			/* Fall through */
 		case USB_PORT_FEAT_U2_TIMEOUT:
 			usbip_dbg_vhci_rh(
 				" SetPortFeature: USB_PORT_FEAT_U2_TIMEOUT\n");
@@ -654,15 +655,9 @@ error:
 static void vhci_tx_urb(struct urb *urb, struct vhci_device *vdev)
 {
 	struct vhci_priv *priv;
-	struct vhci_hcd *vhci_hcd;
+	struct vhci_hcd *vhci_hcd = vdev_to_vhci_hcd(vdev);
 	unsigned long flags;
 
-	if (!vdev) {
-		pr_err("could not get virtual device");
-		return;
-	}
-	vhci_hcd = vdev_to_vhci_hcd(vdev);
-
 	priv = kzalloc(sizeof(struct vhci_priv), GFP_ATOMIC);
 	if (!priv) {
 		usbip_event_add(&vdev->ud, VDEV_EVENT_ERROR_MALLOC);
diff --git a/drivers/video/fbdev/da8xx-fb.c b/drivers/video/fbdev/da8xx-fb.c
index 43f2a4816860..ec62274b914b 100644
--- a/drivers/video/fbdev/da8xx-fb.c
+++ b/drivers/video/fbdev/da8xx-fb.c
@@ -1097,9 +1097,9 @@ static int fb_remove(struct platform_device *dev)
 
 		unregister_framebuffer(info);
 		fb_dealloc_cmap(&info->cmap);
-		dma_free_coherent(NULL, PALETTE_SIZE, par->v_palette_base,
+		dma_free_coherent(par->dev, PALETTE_SIZE, par->v_palette_base,
 				  par->p_palette_base);
-		dma_free_coherent(NULL, par->vram_size, par->vram_virt,
+		dma_free_coherent(par->dev, par->vram_size, par->vram_virt,
 				  par->vram_phys);
 		pm_runtime_put_sync(&dev->dev);
 		pm_runtime_disable(&dev->dev);
@@ -1425,7 +1425,7 @@ static int fb_probe(struct platform_device *device)
 	par->vram_size = roundup(par->vram_size/8, ulcm);
 	par->vram_size = par->vram_size * LCD_NUM_BUFFERS;
 
-	par->vram_virt = dma_alloc_coherent(NULL,
+	par->vram_virt = dma_alloc_coherent(par->dev,
 					    par->vram_size,
 					    &par->vram_phys,
 					    GFP_KERNEL | GFP_DMA);
@@ -1446,7 +1446,7 @@ static int fb_probe(struct platform_device *device)
 		da8xx_fb_fix.line_length - 1;
 
 	/* allocate palette buffer */
-	par->v_palette_base = dma_alloc_coherent(NULL, PALETTE_SIZE,
+	par->v_palette_base = dma_alloc_coherent(par->dev, PALETTE_SIZE,
 						 &par->p_palette_base,
 						 GFP_KERNEL | GFP_DMA);
 	if (!par->v_palette_base) {
@@ -1532,11 +1532,12 @@ err_dealloc_cmap:
 	fb_dealloc_cmap(&da8xx_fb_info->cmap);
 
 err_release_pl_mem:
-	dma_free_coherent(NULL, PALETTE_SIZE, par->v_palette_base,
+	dma_free_coherent(par->dev, PALETTE_SIZE, par->v_palette_base,
 			  par->p_palette_base);
 
 err_release_fb_mem:
-	dma_free_coherent(NULL, par->vram_size, par->vram_virt, par->vram_phys);
+	dma_free_coherent(par->dev, par->vram_size, par->vram_virt,
+		          par->vram_phys);
 
 err_release_fb:
 	framebuffer_release(da8xx_fb_info);
diff --git a/drivers/video/fbdev/gbefb.c b/drivers/video/fbdev/gbefb.c
index 1a242b1338e9..3fcb33232ba3 100644
--- a/drivers/video/fbdev/gbefb.c
+++ b/drivers/video/fbdev/gbefb.c
@@ -1162,9 +1162,9 @@ static int gbefb_probe(struct platform_device *p_dev)
 	}
 	gbe_revision = gbe->ctrlstat & 15;
 
-	gbe_tiles.cpu =
-		dma_alloc_coherent(NULL, GBE_TLB_SIZE * sizeof(uint16_t),
-				   &gbe_tiles.dma, GFP_KERNEL);
+	gbe_tiles.cpu = dmam_alloc_coherent(&p_dev->dev,
+				GBE_TLB_SIZE * sizeof(uint16_t),
+				&gbe_tiles.dma, GFP_KERNEL);
 	if (!gbe_tiles.cpu) {
 		printk(KERN_ERR "gbefb: couldn't allocate tiles table\n");
 		ret = -ENOMEM;
@@ -1178,19 +1178,20 @@ static int gbefb_probe(struct platform_device *p_dev)
 		if (!gbe_mem) {
 			printk(KERN_ERR "gbefb: couldn't map framebuffer\n");
 			ret = -ENOMEM;
-			goto out_tiles_free;
+			goto out_release_mem_region;
 		}
 
 		gbe_dma_addr = 0;
 	} else {
 		/* try to allocate memory with the classical allocator
 		 * this has high chance to fail on low memory machines */
-		gbe_mem = dma_alloc_wc(NULL, gbe_mem_size, &gbe_dma_addr,
-				       GFP_KERNEL);
+		gbe_mem = dmam_alloc_attrs(&p_dev->dev, gbe_mem_size,
+				&gbe_dma_addr, GFP_KERNEL,
+				DMA_ATTR_WRITE_COMBINE);
 		if (!gbe_mem) {
 			printk(KERN_ERR "gbefb: couldn't allocate framebuffer memory\n");
 			ret = -ENOMEM;
-			goto out_tiles_free;
+			goto out_release_mem_region;
 		}
 
 		gbe_mem_phys = (unsigned long) gbe_dma_addr;
@@ -1237,11 +1238,6 @@ static int gbefb_probe(struct platform_device *p_dev)
 
 out_gbe_unmap:
 	arch_phys_wc_del(par->wc_cookie);
-	if (gbe_dma_addr)
-		dma_free_wc(NULL, gbe_mem_size, gbe_mem, gbe_mem_phys);
-out_tiles_free:
-	dma_free_coherent(NULL, GBE_TLB_SIZE * sizeof(uint16_t),
-			  (void *)gbe_tiles.cpu, gbe_tiles.dma);
 out_release_mem_region:
 	release_mem_region(GBE_BASE, sizeof(struct sgi_gbe));
 out_release_framebuffer:
@@ -1258,10 +1254,6 @@ static int gbefb_remove(struct platform_device* p_dev)
 	unregister_framebuffer(info);
 	gbe_turn_off();
 	arch_phys_wc_del(par->wc_cookie);
-	if (gbe_dma_addr)
-		dma_free_wc(NULL, gbe_mem_size, gbe_mem, gbe_mem_phys);
-	dma_free_coherent(NULL, GBE_TLB_SIZE * sizeof(uint16_t),
-			  (void *)gbe_tiles.cpu, gbe_tiles.dma);
 	release_mem_region(GBE_BASE, sizeof(struct sgi_gbe));
 	gbefb_remove_sysfs(&p_dev->dev);
 	framebuffer_release(info);
diff --git a/drivers/video/fbdev/pxa3xx-gcu.c b/drivers/video/fbdev/pxa3xx-gcu.c
index 69cfb337c857..047a2fa4b87e 100644
--- a/drivers/video/fbdev/pxa3xx-gcu.c
+++ b/drivers/video/fbdev/pxa3xx-gcu.c
@@ -96,6 +96,7 @@ struct pxa3xx_gcu_batch {
 };
 
 struct pxa3xx_gcu_priv {
+	struct device		 *dev;
 	void __iomem		 *mmio_base;
 	struct clk		 *clk;
 	struct pxa3xx_gcu_shared *shared;
@@ -493,7 +494,7 @@ pxa3xx_gcu_mmap(struct file *file, struct vm_area_struct *vma)
 		if (size != SHARED_SIZE)
 			return -EINVAL;
 
-		return dma_mmap_coherent(NULL, vma,
+		return dma_mmap_coherent(priv->dev, vma,
 			priv->shared, priv->shared_phys, size);
 
 	case SHARED_SIZE >> PAGE_SHIFT:
@@ -670,6 +671,7 @@ static int pxa3xx_gcu_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, priv);
 	priv->resource_mem = r;
+	priv->dev = dev;
 	pxa3xx_gcu_reset(priv);
 	pxa3xx_gcu_init_debug_timer(priv);
 
diff --git a/fs/cifs/cifs_debug.c b/fs/cifs/cifs_debug.c
index 13c1288b04a7..6a69f11aacf7 100644
--- a/fs/cifs/cifs_debug.c
+++ b/fs/cifs/cifs_debug.c
@@ -312,12 +312,10 @@ static int cifs_debug_data_proc_show(struct seq_file *m, void *v)
 			atomic_read(&server->smbd_conn->send_credits),
 			atomic_read(&server->smbd_conn->receive_credits),
 			server->smbd_conn->receive_credit_target);
-		seq_printf(m, "\nPending send_pending: %x send_payload_pending:"
-			" %x smbd_send_pending: %x smbd_recv_pending: %x",
+		seq_printf(m, "\nPending send_pending: %x "
+			"send_payload_pending: %x",
 			atomic_read(&server->smbd_conn->send_pending),
-			atomic_read(&server->smbd_conn->send_payload_pending),
-			server->smbd_conn->smbd_send_pending,
-			server->smbd_conn->smbd_recv_pending);
+			atomic_read(&server->smbd_conn->send_payload_pending));
 		seq_printf(m, "\nReceive buffers count_receive_queue: %x "
 			"count_empty_packet_queue: %x",
 			server->smbd_conn->count_receive_queue,
@@ -334,6 +332,12 @@ skip_rdma:
 #endif
 		seq_printf(m, "\nNumber of credits: %d Dialect 0x%x",
 			server->credits,  server->dialect);
+		if (server->compress_algorithm == SMB3_COMPRESS_LZNT1)
+			seq_printf(m, " COMPRESS_LZNT1");
+		else if (server->compress_algorithm == SMB3_COMPRESS_LZ77)
+			seq_printf(m, " COMPRESS_LZ77");
+		else if (server->compress_algorithm == SMB3_COMPRESS_LZ77_HUFF)
+			seq_printf(m, " COMPRESS_LZ77_HUFF");
 		if (server->sign)
 			seq_printf(m, " signed");
 		if (server->posix_ext_supported)
@@ -462,8 +466,13 @@ static ssize_t cifs_stats_proc_write(struct file *file,
 			server = list_entry(tmp1, struct TCP_Server_Info,
 					    tcp_ses_list);
 #ifdef CONFIG_CIFS_STATS2
-			for (i = 0; i < NUMBER_OF_SMB2_COMMANDS; i++)
+			for (i = 0; i < NUMBER_OF_SMB2_COMMANDS; i++) {
+				atomic_set(&server->num_cmds[i], 0);
 				atomic_set(&server->smb2slowcmd[i], 0);
+				server->time_per_cmd[i] = 0;
+				server->slowest_cmd[i] = 0;
+				server->fastest_cmd[0] = 0;
+			}
 #endif /* CONFIG_CIFS_STATS2 */
 			list_for_each(tmp2, &server->smb_ses_list) {
 				ses = list_entry(tmp2, struct cifs_ses,
@@ -531,9 +540,19 @@ static int cifs_stats_proc_show(struct seq_file *m, void *v)
 		server = list_entry(tmp1, struct TCP_Server_Info,
 				    tcp_ses_list);
 #ifdef CONFIG_CIFS_STATS2
+		seq_puts(m, "\nTotal time spent processing by command. Time ");
+		seq_printf(m, "units are jiffies (%d per second)\n", HZ);
+		seq_puts(m, "  SMB3 CMD\tNumber\tTotal Time\tFastest\tSlowest\n");
+		seq_puts(m, "  --------\t------\t----------\t-------\t-------\n");
+		for (j = 0; j < NUMBER_OF_SMB2_COMMANDS; j++)
+			seq_printf(m, "  %d\t\t%d\t%llu\t\t%u\t%u\n", j,
+				atomic_read(&server->num_cmds[j]),
+				server->time_per_cmd[j],
+				server->fastest_cmd[j],
+				server->slowest_cmd[j]);
 		for (j = 0; j < NUMBER_OF_SMB2_COMMANDS; j++)
 			if (atomic_read(&server->smb2slowcmd[j]))
-				seq_printf(m, "%d slow responses from %s for command %d\n",
+				seq_printf(m, "  %d slow responses from %s for command %d\n",
 					atomic_read(&server->smb2slowcmd[j]),
 					server->hostname, j);
 #endif /* STATS2 */
diff --git a/fs/cifs/cifsfs.c b/fs/cifs/cifsfs.c
index 877174761efb..b1a5fcfa3ce1 100644
--- a/fs/cifs/cifsfs.c
+++ b/fs/cifs/cifsfs.c
@@ -483,6 +483,8 @@ cifs_show_options(struct seq_file *s, struct dentry *root)
 		seq_puts(s, ",seal");
 	if (tcon->nocase)
 		seq_puts(s, ",nocase");
+	if (tcon->local_lease)
+		seq_puts(s, ",locallease");
 	if (tcon->retry)
 		seq_puts(s, ",hard");
 	else
@@ -984,6 +986,7 @@ const struct inode_operations cifs_file_inode_ops = {
 	.getattr = cifs_getattr,
 	.permission = cifs_permission,
 	.listxattr = cifs_listxattr,
+	.fiemap = cifs_fiemap,
 };
 
 const struct inode_operations cifs_symlink_inode_ops = {
diff --git a/fs/cifs/cifsfs.h b/fs/cifs/cifsfs.h
index 5c0298b9998f..aea005703785 100644
--- a/fs/cifs/cifsfs.h
+++ b/fs/cifs/cifsfs.h
@@ -84,6 +84,8 @@ extern int cifs_revalidate_mapping(struct inode *inode);
 extern int cifs_zap_mapping(struct inode *inode);
 extern int cifs_getattr(const struct path *, struct kstat *, u32, unsigned int);
 extern int cifs_setattr(struct dentry *, struct iattr *);
+extern int cifs_fiemap(struct inode *, struct fiemap_extent_info *, u64 start,
+		       u64 len);
 
 extern const struct inode_operations cifs_file_inode_ops;
 extern const struct inode_operations cifs_symlink_inode_ops;
@@ -150,5 +152,5 @@ extern long cifs_ioctl(struct file *filep, unsigned int cmd, unsigned long arg);
 extern const struct export_operations cifs_export_ops;
 #endif /* CONFIG_CIFS_NFSD_EXPORT */
 
-#define CIFS_VERSION   "2.19"
+#define CIFS_VERSION   "2.20"
 #endif				/* _CIFSFS_H */
diff --git a/fs/cifs/cifsglob.h b/fs/cifs/cifsglob.h
index 585ad3207cb1..33c251b408aa 100644
--- a/fs/cifs/cifsglob.h
+++ b/fs/cifs/cifsglob.h
@@ -355,7 +355,8 @@ struct smb_version_operations {
 			       struct cifs_sb_info *);
 	/* query symlink target */
 	int (*query_symlink)(const unsigned int, struct cifs_tcon *,
-			     const char *, char **, struct cifs_sb_info *);
+			     struct cifs_sb_info *, const char *,
+			     char **, bool);
 	/* open a file for non-posix mounts */
 	int (*open)(const unsigned int, struct cifs_open_parms *,
 		    __u32 *, FILE_ALL_INFO *);
@@ -493,6 +494,9 @@ struct smb_version_operations {
 			 char *full_path,
 			 umode_t mode,
 			 dev_t device_number);
+	/* version specific fiemap implementation */
+	int (*fiemap)(struct cifs_tcon *tcon, struct cifsFileInfo *,
+		      struct fiemap_extent_info *, u64, u64);
 };
 
 struct smb_version_values {
@@ -596,6 +600,10 @@ struct smb_vol {
 	unsigned int max_credits; /* smb3 max_credits 10 < credits < 60000 */
 };
 
+/**
+ * CIFS superblock mount flags (mnt_cifs_flags) to consider when
+ * trying to reuse existing superblock for a new mount
+ */
 #define CIFS_MOUNT_MASK (CIFS_MOUNT_NO_PERM | CIFS_MOUNT_SET_UID | \
 			 CIFS_MOUNT_SERVER_INUM | CIFS_MOUNT_DIRECT_IO | \
 			 CIFS_MOUNT_NO_XATTR | CIFS_MOUNT_MAP_SPECIAL_CHR | \
@@ -606,8 +614,13 @@ struct smb_vol {
 			 CIFS_MOUNT_NOPOSIXBRL | CIFS_MOUNT_NOSSYNC | \
 			 CIFS_MOUNT_FSCACHE | CIFS_MOUNT_MF_SYMLINKS | \
 			 CIFS_MOUNT_MULTIUSER | CIFS_MOUNT_STRICT_IO | \
-			 CIFS_MOUNT_CIFS_BACKUPUID | CIFS_MOUNT_CIFS_BACKUPGID)
+			 CIFS_MOUNT_CIFS_BACKUPUID | CIFS_MOUNT_CIFS_BACKUPGID | \
+			 CIFS_MOUNT_NO_DFS)
 
+/**
+ * Generic VFS superblock mount flags (s_flags) to consider when
+ * trying to reuse existing superblock for a new mount
+ */
 #define CIFS_MS_MASK (SB_RDONLY | SB_MANDLOCK | SB_NOEXEC | SB_NOSUID | \
 		      SB_NODEV | SB_SYNCHRONOUS)
 
@@ -714,10 +727,15 @@ struct TCP_Server_Info {
 #ifdef CONFIG_CIFS_STATS2
 	atomic_t in_send; /* requests trying to send */
 	atomic_t num_waiters;   /* blocked waiting to get in sendrecv */
+	atomic_t num_cmds[NUMBER_OF_SMB2_COMMANDS]; /* total requests by cmd */
 	atomic_t smb2slowcmd[NUMBER_OF_SMB2_COMMANDS]; /* count resps > 1 sec */
+	__u64 time_per_cmd[NUMBER_OF_SMB2_COMMANDS]; /* total time per cmd */
+	__u32 slowest_cmd[NUMBER_OF_SMB2_COMMANDS];
+	__u32 fastest_cmd[NUMBER_OF_SMB2_COMMANDS];
 #endif /* STATS2 */
 	unsigned int	max_read;
 	unsigned int	max_write;
+	__le16	compress_algorithm;
 	__le16	cipher_type;
 	 /* save initital negprot hash */
 	__u8	preauth_sha_hash[SMB2_PREAUTH_HASH_SIZE];
@@ -1673,11 +1691,11 @@ static inline bool is_retryable_error(int error)
 
 /* Type of Request to SendReceive2 */
 #define   CIFS_BLOCKING_OP      1    /* operation can block */
-#define   CIFS_ASYNC_OP         2    /* do not wait for response */
+#define   CIFS_NON_BLOCKING     2    /* do not block waiting for credits */
 #define   CIFS_TIMEOUT_MASK 0x003    /* only one of above set in req */
 #define   CIFS_LOG_ERROR    0x010    /* log NT STATUS if non-zero */
 #define   CIFS_LARGE_BUF_OP 0x020    /* large request buffer */
-#define   CIFS_NO_RESP      0x040    /* no response buffer required */
+#define   CIFS_NO_RSP_BUF   0x040    /* no response buffer required */
 
 /* Type of request operation */
 #define   CIFS_ECHO_OP      0x080    /* echo request */
@@ -1687,6 +1705,7 @@ static inline bool is_retryable_error(int error)
 
 #define   CIFS_HAS_CREDITS 0x0400    /* already has credits */
 #define   CIFS_TRANSFORM_REQ 0x0800    /* transform request before sending */
+#define   CIFS_NO_SRV_RSP    0x1000    /* there is no server response */
 
 /* Security Flags: indicate type of session setup needed */
 #define   CIFSSEC_MAY_SIGN	0x00001
diff --git a/fs/cifs/cifsproto.h b/fs/cifs/cifsproto.h
index 4f96b3b00a7a..e23234207fc2 100644
--- a/fs/cifs/cifsproto.h
+++ b/fs/cifs/cifsproto.h
@@ -526,12 +526,21 @@ extern int E_md4hash(const unsigned char *passwd, unsigned char *p16,
 			const struct nls_table *codepage);
 extern int SMBencrypt(unsigned char *passwd, const unsigned char *c8,
 			unsigned char *p24);
+
+extern int
+cifs_setup_volume_info(struct smb_vol *volume_info, char *mount_data,
+		       const char *devname, bool is_smb3);
 extern void
 cifs_cleanup_volume_info_contents(struct smb_vol *volume_info);
 
 extern struct TCP_Server_Info *
 cifs_find_tcp_session(struct smb_vol *vol);
 
+extern void cifs_put_smb_ses(struct cifs_ses *ses);
+
+extern struct cifs_ses *
+cifs_get_smb_ses(struct TCP_Server_Info *server, struct smb_vol *volume_info);
+
 void cifs_readdata_release(struct kref *refcount);
 int cifs_async_readv(struct cifs_readdata *rdata);
 int cifs_readv_receive(struct TCP_Server_Info *server, struct mid_q_entry *mid);
diff --git a/fs/cifs/cifssmb.c b/fs/cifs/cifssmb.c
index f43747c062a7..1fbd92843a73 100644
--- a/fs/cifs/cifssmb.c
+++ b/fs/cifs/cifssmb.c
@@ -860,7 +860,7 @@ CIFSSMBEcho(struct TCP_Server_Info *server)
 	iov[1].iov_base = (char *)smb + 4;
 
 	rc = cifs_call_async(server, &rqst, NULL, cifs_echo_callback, NULL,
-			     server, CIFS_ASYNC_OP | CIFS_ECHO_OP, NULL);
+			     server, CIFS_NON_BLOCKING | CIFS_ECHO_OP, NULL);
 	if (rc)
 		cifs_dbg(FYI, "Echo request failed: %d\n", rc);
 
@@ -2508,8 +2508,8 @@ int cifs_lockv(const unsigned int xid, struct cifs_tcon *tcon,
 	iov[1].iov_len = (num_unlock + num_lock) * sizeof(LOCKING_ANDX_RANGE);
 
 	cifs_stats_inc(&tcon->stats.cifs_stats.num_locks);
-	rc = SendReceive2(xid, tcon->ses, iov, 2, &resp_buf_type, CIFS_NO_RESP,
-			  &rsp_iov);
+	rc = SendReceive2(xid, tcon->ses, iov, 2, &resp_buf_type,
+			  CIFS_NO_RSP_BUF, &rsp_iov);
 	cifs_small_buf_release(pSMB);
 	if (rc)
 		cifs_dbg(FYI, "Send error in cifs_lockv = %d\n", rc);
@@ -2540,7 +2540,7 @@ CIFSSMBLock(const unsigned int xid, struct cifs_tcon *tcon,
 
 	if (lockType == LOCKING_ANDX_OPLOCK_RELEASE) {
 		/* no response expected */
-		flags = CIFS_ASYNC_OP | CIFS_OBREAK_OP;
+		flags = CIFS_NO_SRV_RSP | CIFS_NON_BLOCKING | CIFS_OBREAK_OP;
 		pSMB->Timeout = 0;
 	} else if (waitFlag) {
 		flags = CIFS_BLOCKING_OP; /* blocking operation, no timeout */
@@ -6567,93 +6567,3 @@ SetEARetry:
 	return rc;
 }
 #endif
-
-#ifdef CONFIG_CIFS_DNOTIFY_EXPERIMENTAL /* BB unused temporarily */
-/*
- *	Years ago the kernel added a "dnotify" function for Samba server,
- *	to allow network clients (such as Windows) to display updated
- *	lists of files in directory listings automatically when
- *	files are added by one user when another user has the
- *	same directory open on their desktop.  The Linux cifs kernel
- *	client hooked into the kernel side of this interface for
- *	the same reason, but ironically when the VFS moved from
- *	"dnotify" to "inotify" it became harder to plug in Linux
- *	network file system clients (the most obvious use case
- *	for notify interfaces is when multiple users can update
- *	the contents of the same directory - exactly what network
- *	file systems can do) although the server (Samba) could
- *	still use it.  For the short term we leave the worker
- *	function ifdeffed out (below) until inotify is fixed
- *	in the VFS to make it easier to plug in network file
- *	system clients.  If inotify turns out to be permanently
- *	incompatible for network fs clients, we could instead simply
- *	expose this config flag by adding a future cifs (and smb2) notify ioctl.
- */
-int CIFSSMBNotify(const unsigned int xid, struct cifs_tcon *tcon,
-		  const int notify_subdirs, const __u16 netfid,
-		  __u32 filter, struct file *pfile, int multishot,
-		  const struct nls_table *nls_codepage)
-{
-	int rc = 0;
-	struct smb_com_transaction_change_notify_req *pSMB = NULL;
-	struct smb_com_ntransaction_change_notify_rsp *pSMBr = NULL;
-	struct dir_notify_req *dnotify_req;
-	int bytes_returned;
-
-	cifs_dbg(FYI, "In CIFSSMBNotify for file handle %d\n", (int)netfid);
-	rc = smb_init(SMB_COM_NT_TRANSACT, 23, tcon, (void **) &pSMB,
-		      (void **) &pSMBr);
-	if (rc)
-		return rc;
-
-	pSMB->TotalParameterCount = 0 ;
-	pSMB->TotalDataCount = 0;
-	pSMB->MaxParameterCount = cpu_to_le32(2);
-	pSMB->MaxDataCount = cpu_to_le32(CIFSMaxBufSize & 0xFFFFFF00);
-	pSMB->MaxSetupCount = 4;
-	pSMB->Reserved = 0;
-	pSMB->ParameterOffset = 0;
-	pSMB->DataCount = 0;
-	pSMB->DataOffset = 0;
-	pSMB->SetupCount = 4; /* single byte does not need le conversion */
-	pSMB->SubCommand = cpu_to_le16(NT_TRANSACT_NOTIFY_CHANGE);
-	pSMB->ParameterCount = pSMB->TotalParameterCount;
-	if (notify_subdirs)
-		pSMB->WatchTree = 1; /* one byte - no le conversion needed */
-	pSMB->Reserved2 = 0;
-	pSMB->CompletionFilter = cpu_to_le32(filter);
-	pSMB->Fid = netfid; /* file handle always le */
-	pSMB->ByteCount = 0;
-
-	rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB,
-			 (struct smb_hdr *)pSMBr, &bytes_returned,
-			 CIFS_ASYNC_OP);
-	if (rc) {
-		cifs_dbg(FYI, "Error in Notify = %d\n", rc);
-	} else {
-		/* Add file to outstanding requests */
-		/* BB change to kmem cache alloc */
-		dnotify_req = kmalloc(
-						sizeof(struct dir_notify_req),
-						 GFP_KERNEL);
-		if (dnotify_req) {
-			dnotify_req->Pid = pSMB->hdr.Pid;
-			dnotify_req->PidHigh = pSMB->hdr.PidHigh;
-			dnotify_req->Mid = pSMB->hdr.Mid;
-			dnotify_req->Tid = pSMB->hdr.Tid;
-			dnotify_req->Uid = pSMB->hdr.Uid;
-			dnotify_req->netfid = netfid;
-			dnotify_req->pfile = pfile;
-			dnotify_req->filter = filter;
-			dnotify_req->multishot = multishot;
-			spin_lock(&GlobalMid_Lock);
-			list_add_tail(&dnotify_req->lhead,
-					&GlobalDnotifyReqList);
-			spin_unlock(&GlobalMid_Lock);
-		} else
-			rc = -ENOMEM;
-	}
-	cifs_buf_release(pSMB);
-	return rc;
-}
-#endif /* was needed for dnotify, and will be needed for inotify when VFS fix */
diff --git a/fs/cifs/connect.c b/fs/cifs/connect.c
index 4c0e44489f21..084756cfdaee 100644
--- a/fs/cifs/connect.c
+++ b/fs/cifs/connect.c
@@ -323,8 +323,6 @@ static int ip_connect(struct TCP_Server_Info *server);
 static int generic_ip_connect(struct TCP_Server_Info *server);
 static void tlink_rb_insert(struct rb_root *root, struct tcon_link *new_tlink);
 static void cifs_prune_tlinks(struct work_struct *work);
-static int cifs_setup_volume_info(struct smb_vol *volume_info, char *mount_data,
-					const char *devname, bool is_smb3);
 static char *extract_hostname(const char *unc);
 
 /*
@@ -530,21 +528,6 @@ cifs_reconnect(struct TCP_Server_Info *server)
 	/* do not want to be sending data on a socket we are freeing */
 	cifs_dbg(FYI, "%s: tearing down socket\n", __func__);
 	mutex_lock(&server->srv_mutex);
-	if (server->ssocket) {
-		cifs_dbg(FYI, "State: 0x%x Flags: 0x%lx\n",
-			 server->ssocket->state, server->ssocket->flags);
-		kernel_sock_shutdown(server->ssocket, SHUT_WR);
-		cifs_dbg(FYI, "Post shutdown state: 0x%x Flags: 0x%lx\n",
-			 server->ssocket->state, server->ssocket->flags);
-		sock_release(server->ssocket);
-		server->ssocket = NULL;
-	}
-	server->sequence_number = 0;
-	server->session_estab = false;
-	kfree(server->session_key.response);
-	server->session_key.response = NULL;
-	server->session_key.len = 0;
-	server->lstrp = jiffies;
 
 	/* mark submitted MIDs for retry and issue callback */
 	INIT_LIST_HEAD(&retry_list);
@@ -557,7 +540,6 @@ cifs_reconnect(struct TCP_Server_Info *server)
 		list_move(&mid_entry->qhead, &retry_list);
 	}
 	spin_unlock(&GlobalMid_Lock);
-	mutex_unlock(&server->srv_mutex);
 
 	cifs_dbg(FYI, "%s: issuing mid callbacks\n", __func__);
 	list_for_each_safe(tmp, tmp2, &retry_list) {
@@ -566,6 +548,25 @@ cifs_reconnect(struct TCP_Server_Info *server)
 		mid_entry->callback(mid_entry);
 	}
 
+	if (server->ssocket) {
+		cifs_dbg(FYI, "State: 0x%x Flags: 0x%lx\n",
+			 server->ssocket->state, server->ssocket->flags);
+		kernel_sock_shutdown(server->ssocket, SHUT_WR);
+		cifs_dbg(FYI, "Post shutdown state: 0x%x Flags: 0x%lx\n",
+			 server->ssocket->state, server->ssocket->flags);
+		sock_release(server->ssocket);
+		server->ssocket = NULL;
+	} else if (cifs_rdma_enabled(server))
+		smbd_destroy(server);
+	server->sequence_number = 0;
+	server->session_estab = false;
+	kfree(server->session_key.response);
+	server->session_key.response = NULL;
+	server->session_key.len = 0;
+	server->lstrp = jiffies;
+
+	mutex_unlock(&server->srv_mutex);
+
 	do {
 		try_to_freeze();
 
@@ -931,10 +932,8 @@ static void clean_demultiplex_info(struct TCP_Server_Info *server)
 	wake_up_all(&server->request_q);
 	/* give those requests time to exit */
 	msleep(125);
-	if (cifs_rdma_enabled(server) && server->smbd_conn) {
-		smbd_destroy(server->smbd_conn);
-		server->smbd_conn = NULL;
-	}
+	if (cifs_rdma_enabled(server))
+		smbd_destroy(server);
 	if (server->ssocket) {
 		sock_release(server->ssocket);
 		server->ssocket = NULL;
@@ -2904,8 +2903,7 @@ cifs_find_smb_ses(struct TCP_Server_Info *server, struct smb_vol *vol)
 	return NULL;
 }
 
-static void
-cifs_put_smb_ses(struct cifs_ses *ses)
+void cifs_put_smb_ses(struct cifs_ses *ses)
 {
 	unsigned int rc, xid;
 	struct TCP_Server_Info *server = ses->server;
@@ -3082,7 +3080,7 @@ cifs_set_cifscreds(struct smb_vol *vol __attribute__((unused)),
  * already got a server reference (server refcount +1). See
  * cifs_get_tcon() for refcount explanations.
  */
-static struct cifs_ses *
+struct cifs_ses *
 cifs_get_smb_ses(struct TCP_Server_Info *server, struct smb_vol *volume_info)
 {
 	int rc = -ENOMEM;
@@ -4389,7 +4387,7 @@ static int mount_do_dfs_failover(const char *path,
 }
 #endif
 
-static int
+int
 cifs_setup_volume_info(struct smb_vol *volume_info, char *mount_data,
 			const char *devname, bool is_smb3)
 {
@@ -4543,7 +4541,7 @@ int cifs_mount(struct cifs_sb_info *cifs_sb, struct smb_vol *vol)
 	struct cifs_tcon *tcon = NULL;
 	struct TCP_Server_Info *server;
 	char *root_path = NULL, *full_path = NULL;
-	char *old_mountdata;
+	char *old_mountdata, *origin_mountdata = NULL;
 	int count;
 
 	rc = mount_get_conns(vol, cifs_sb, &xid, &server, &ses, &tcon);
@@ -4602,6 +4600,14 @@ int cifs_mount(struct cifs_sb_info *cifs_sb, struct smb_vol *vol)
 		goto error;
 	}
 
+	/* Save DFS root volume information for DFS refresh worker */
+	origin_mountdata = kstrndup(cifs_sb->mountdata,
+				    strlen(cifs_sb->mountdata), GFP_KERNEL);
+	if (!origin_mountdata) {
+		rc = -ENOMEM;
+		goto error;
+	}
+
 	if (cifs_sb->mountdata != old_mountdata) {
 		/* If we were redirected, reconnect to new target server */
 		mount_put_conns(cifs_sb, xid, server, ses, tcon);
@@ -4710,7 +4716,7 @@ int cifs_mount(struct cifs_sb_info *cifs_sb, struct smb_vol *vol)
 	}
 	spin_unlock(&cifs_tcp_ses_lock);
 
-	rc = dfs_cache_add_vol(vol, cifs_sb->origin_fullpath);
+	rc = dfs_cache_add_vol(origin_mountdata, vol, cifs_sb->origin_fullpath);
 	if (rc) {
 		kfree(cifs_sb->origin_fullpath);
 		goto error;
@@ -4728,6 +4734,7 @@ out:
 error:
 	kfree(full_path);
 	kfree(root_path);
+	kfree(origin_mountdata);
 	mount_put_conns(cifs_sb, xid, server, ses, tcon);
 	return rc;
 }
diff --git a/fs/cifs/dfs_cache.c b/fs/cifs/dfs_cache.c
index 09b7d0d4f6e4..85dc89d3a203 100644
--- a/fs/cifs/dfs_cache.c
+++ b/fs/cifs/dfs_cache.c
@@ -2,7 +2,7 @@
 /*
  * DFS referral cache routines
  *
- * Copyright (c) 2018 Paulo Alcantara <palcantara@suse.de>
+ * Copyright (c) 2018-2019 Paulo Alcantara <palcantara@suse.de>
  */
 
 #include <linux/rcupdate.h>
@@ -52,6 +52,7 @@ static struct kmem_cache *dfs_cache_slab __read_mostly;
 struct dfs_cache_vol_info {
 	char *vi_fullpath;
 	struct smb_vol vi_vol;
+	char *vi_mntdata;
 	struct list_head vi_list;
 };
 
@@ -529,6 +530,7 @@ static inline void free_vol(struct dfs_cache_vol_info *vi)
 {
 	list_del(&vi->vi_list);
 	kfree(vi->vi_fullpath);
+	kfree(vi->vi_mntdata);
 	cifs_cleanup_volume_info_contents(&vi->vi_vol);
 	kfree(vi);
 }
@@ -1139,17 +1141,18 @@ err_free_username:
  * dfs_cache_add_vol - add a cifs volume during mount() that will be handled by
  * DFS cache refresh worker.
  *
+ * @mntdata: mount data.
  * @vol: cifs volume.
  * @fullpath: origin full path.
  *
  * Return zero if volume was set up correctly, otherwise non-zero.
  */
-int dfs_cache_add_vol(struct smb_vol *vol, const char *fullpath)
+int dfs_cache_add_vol(char *mntdata, struct smb_vol *vol, const char *fullpath)
 {
 	int rc;
 	struct dfs_cache_vol_info *vi;
 
-	if (!vol || !fullpath)
+	if (!vol || !fullpath || !mntdata)
 		return -EINVAL;
 
 	cifs_dbg(FYI, "%s: fullpath: %s\n", __func__, fullpath);
@@ -1168,6 +1171,8 @@ int dfs_cache_add_vol(struct smb_vol *vol, const char *fullpath)
 	if (rc)
 		goto err_free_fullpath;
 
+	vi->vi_mntdata = mntdata;
+
 	mutex_lock(&dfs_cache.dc_lock);
 	list_add_tail(&vi->vi_list, &dfs_cache.dc_vol_list);
 	mutex_unlock(&dfs_cache.dc_lock);
@@ -1275,8 +1280,102 @@ static void get_tcons(struct TCP_Server_Info *server, struct list_head *head)
 	spin_unlock(&cifs_tcp_ses_lock);
 }
 
+static inline bool is_dfs_link(const char *path)
+{
+	char *s;
+
+	s = strchr(path + 1, '\\');
+	if (!s)
+		return false;
+	return !!strchr(s + 1, '\\');
+}
+
+static inline char *get_dfs_root(const char *path)
+{
+	char *s, *npath;
+
+	s = strchr(path + 1, '\\');
+	if (!s)
+		return ERR_PTR(-EINVAL);
+
+	s = strchr(s + 1, '\\');
+	if (!s)
+		return ERR_PTR(-EINVAL);
+
+	npath = kstrndup(path, s - path, GFP_KERNEL);
+	if (!npath)
+		return ERR_PTR(-ENOMEM);
+
+	return npath;
+}
+
+/* Find root SMB session out of a DFS link path */
+static struct cifs_ses *find_root_ses(struct dfs_cache_vol_info *vi,
+				      struct cifs_tcon *tcon, const char *path)
+{
+	char *rpath;
+	int rc;
+	struct dfs_info3_param ref = {0};
+	char *mdata = NULL, *devname = NULL;
+	bool is_smb3 = tcon->ses->server->vals->header_preamble_size == 0;
+	struct TCP_Server_Info *server;
+	struct cifs_ses *ses;
+	struct smb_vol vol;
+
+	rpath = get_dfs_root(path);
+	if (IS_ERR(rpath))
+		return ERR_CAST(rpath);
+
+	memset(&vol, 0, sizeof(vol));
+
+	rc = dfs_cache_noreq_find(rpath, &ref, NULL);
+	if (rc) {
+		ses = ERR_PTR(rc);
+		goto out;
+	}
+
+	mdata = cifs_compose_mount_options(vi->vi_mntdata, rpath, &ref,
+					   &devname);
+	free_dfs_info_param(&ref);
+
+	if (IS_ERR(mdata)) {
+		ses = ERR_CAST(mdata);
+		mdata = NULL;
+		goto out;
+	}
+
+	rc = cifs_setup_volume_info(&vol, mdata, devname, is_smb3);
+	kfree(devname);
+
+	if (rc) {
+		ses = ERR_PTR(rc);
+		goto out;
+	}
+
+	server = cifs_find_tcp_session(&vol);
+	if (IS_ERR_OR_NULL(server)) {
+		ses = ERR_PTR(-EHOSTDOWN);
+		goto out;
+	}
+	if (server->tcpStatus != CifsGood) {
+		cifs_put_tcp_session(server, 0);
+		ses = ERR_PTR(-EHOSTDOWN);
+		goto out;
+	}
+
+	ses = cifs_get_smb_ses(server, &vol);
+
+out:
+	cifs_cleanup_volume_info_contents(&vol);
+	kfree(mdata);
+	kfree(rpath);
+
+	return ses;
+}
+
 /* Refresh DFS cache entry from a given tcon */
-static void do_refresh_tcon(struct dfs_cache *dc, struct cifs_tcon *tcon)
+static void do_refresh_tcon(struct dfs_cache *dc, struct dfs_cache_vol_info *vi,
+			    struct cifs_tcon *tcon)
 {
 	int rc = 0;
 	unsigned int xid;
@@ -1285,6 +1384,7 @@ static void do_refresh_tcon(struct dfs_cache *dc, struct cifs_tcon *tcon)
 	struct dfs_cache_entry *ce;
 	struct dfs_info3_param *refs = NULL;
 	int numrefs = 0;
+	struct cifs_ses *root_ses = NULL, *ses;
 
 	xid = get_xid();
 
@@ -1306,13 +1406,24 @@ static void do_refresh_tcon(struct dfs_cache *dc, struct cifs_tcon *tcon)
 	if (!cache_entry_expired(ce))
 		goto out;
 
-	if (unlikely(!tcon->ses->server->ops->get_dfs_refer)) {
+	/* If it's a DFS Link, then use root SMB session for refreshing it */
+	if (is_dfs_link(npath)) {
+		ses = root_ses = find_root_ses(vi, tcon, npath);
+		if (IS_ERR(ses)) {
+			rc = PTR_ERR(ses);
+			root_ses = NULL;
+			goto out;
+		}
+	} else {
+		ses = tcon->ses;
+	}
+
+	if (unlikely(!ses->server->ops->get_dfs_refer)) {
 		rc = -EOPNOTSUPP;
 	} else {
-		rc = tcon->ses->server->ops->get_dfs_refer(xid, tcon->ses, path,
-							   &refs, &numrefs,
-							   dc->dc_nlsc,
-							   tcon->remap);
+		rc = ses->server->ops->get_dfs_refer(xid, ses, path, &refs,
+						     &numrefs, dc->dc_nlsc,
+						     tcon->remap);
 		if (!rc) {
 			mutex_lock(&dfs_cache_list_lock);
 			ce = __update_cache_entry(npath, refs, numrefs);
@@ -1323,9 +1434,11 @@ static void do_refresh_tcon(struct dfs_cache *dc, struct cifs_tcon *tcon)
 				rc = PTR_ERR(ce);
 		}
 	}
-	if (rc)
-		cifs_dbg(FYI, "%s: failed to update expired entry\n", __func__);
+
 out:
+	if (root_ses)
+		cifs_put_smb_ses(root_ses);
+
 	free_xid(xid);
 	free_normalized_path(path, npath);
 }
@@ -1333,9 +1446,6 @@ out:
 /*
  * Worker that will refresh DFS cache based on lowest TTL value from a DFS
  * referral.
- *
- * FIXME: ensure that all requests are sent to DFS root for refreshing the
- * cache.
  */
 static void refresh_cache_worker(struct work_struct *work)
 {
@@ -1356,7 +1466,7 @@ static void refresh_cache_worker(struct work_struct *work)
 			goto next;
 		get_tcons(server, &list);
 		list_for_each_entry_safe(tcon, ntcon, &list, ulist) {
-			do_refresh_tcon(dc, tcon);
+			do_refresh_tcon(dc, vi, tcon);
 			list_del_init(&tcon->ulist);
 			cifs_put_tcon(tcon);
 		}
diff --git a/fs/cifs/dfs_cache.h b/fs/cifs/dfs_cache.h
index 22f366514f3a..76c732943f5f 100644
--- a/fs/cifs/dfs_cache.h
+++ b/fs/cifs/dfs_cache.h
@@ -2,7 +2,7 @@
 /*
  * DFS referral cache routines
  *
- * Copyright (c) 2018 Paulo Alcantara <palcantara@suse.de>
+ * Copyright (c) 2018-2019 Paulo Alcantara <palcantara@suse.de>
  */
 
 #ifndef _CIFS_DFS_CACHE_H
@@ -43,7 +43,8 @@ dfs_cache_noreq_update_tgthint(const char *path,
 extern int dfs_cache_get_tgt_referral(const char *path,
 				      const struct dfs_cache_tgt_iterator *it,
 				      struct dfs_info3_param *ref);
-extern int dfs_cache_add_vol(struct smb_vol *vol, const char *fullpath);
+extern int dfs_cache_add_vol(char *mntdata, struct smb_vol *vol,
+			const char *fullpath);
 extern int dfs_cache_update_vol(const char *fullpath,
 				struct TCP_Server_Info *server);
 extern void dfs_cache_del_vol(const char *fullpath);
diff --git a/fs/cifs/file.c b/fs/cifs/file.c
index 7037a137fa53..ce9a5be11df5 100644
--- a/fs/cifs/file.c
+++ b/fs/cifs/file.c
@@ -2443,7 +2443,6 @@ int cifs_strict_fsync(struct file *file, loff_t start, loff_t end,
 	rc = file_write_and_wait_range(file, start, end);
 	if (rc)
 		return rc;
-	inode_lock(inode);
 
 	xid = get_xid();
 
@@ -2468,7 +2467,6 @@ int cifs_strict_fsync(struct file *file, loff_t start, loff_t end,
 	}
 
 	free_xid(xid);
-	inode_unlock(inode);
 	return rc;
 }
 
@@ -2480,12 +2478,10 @@ int cifs_fsync(struct file *file, loff_t start, loff_t end, int datasync)
 	struct TCP_Server_Info *server;
 	struct cifsFileInfo *smbfile = file->private_data;
 	struct cifs_sb_info *cifs_sb = CIFS_FILE_SB(file);
-	struct inode *inode = file->f_mapping->host;
 
 	rc = file_write_and_wait_range(file, start, end);
 	if (rc)
 		return rc;
-	inode_lock(inode);
 
 	xid = get_xid();
 
@@ -2502,7 +2498,6 @@ int cifs_fsync(struct file *file, loff_t start, loff_t end, int datasync)
 	}
 
 	free_xid(xid);
-	inode_unlock(inode);
 	return rc;
 }
 
diff --git a/fs/cifs/inode.c b/fs/cifs/inode.c
index 538fd7d807e4..d7cc62252634 100644
--- a/fs/cifs/inode.c
+++ b/fs/cifs/inode.c
@@ -2116,6 +2116,43 @@ int cifs_getattr(const struct path *path, struct kstat *stat,
 	return rc;
 }
 
+int cifs_fiemap(struct inode *inode, struct fiemap_extent_info *fei, u64 start,
+		u64 len)
+{
+	struct cifsInodeInfo *cifs_i = CIFS_I(inode);
+	struct cifs_sb_info *cifs_sb = CIFS_SB(cifs_i->vfs_inode.i_sb);
+	struct cifs_tcon *tcon = cifs_sb_master_tcon(cifs_sb);
+	struct TCP_Server_Info *server = tcon->ses->server;
+	struct cifsFileInfo *cfile;
+	int rc;
+
+	/*
+	 * We need to be sure that all dirty pages are written as they
+	 * might fill holes on the server.
+	 */
+	if (!CIFS_CACHE_READ(CIFS_I(inode)) && inode->i_mapping &&
+	    inode->i_mapping->nrpages != 0) {
+		rc = filemap_fdatawait(inode->i_mapping);
+		if (rc) {
+			mapping_set_error(inode->i_mapping, rc);
+			return rc;
+		}
+	}
+
+	cfile = find_readable_file(cifs_i, false);
+	if (cfile == NULL)
+		return -EINVAL;
+
+	if (server->ops->fiemap) {
+		rc = server->ops->fiemap(tcon, cfile, fei, start, len);
+		cifsFileInfo_put(cfile);
+		return rc;
+	}
+
+	cifsFileInfo_put(cfile);
+	return -ENOTSUPP;
+}
+
 static int cifs_truncate_page(struct address_space *mapping, loff_t from)
 {
 	pgoff_t index = from >> PAGE_SHIFT;
diff --git a/fs/cifs/link.c b/fs/cifs/link.c
index 62216dc8f9f5..b736acd3917b 100644
--- a/fs/cifs/link.c
+++ b/fs/cifs/link.c
@@ -648,9 +648,16 @@ cifs_get_link(struct dentry *direntry, struct inode *inode,
 		rc = query_mf_symlink(xid, tcon, cifs_sb, full_path,
 				      &target_path);
 
-	if (rc != 0 && server->ops->query_symlink)
-		rc = server->ops->query_symlink(xid, tcon, full_path,
-						&target_path, cifs_sb);
+	if (rc != 0 && server->ops->query_symlink) {
+		struct cifsInodeInfo *cifsi = CIFS_I(inode);
+		bool reparse_point = false;
+
+		if (cifsi->cifsAttrs & ATTR_REPARSE)
+			reparse_point = true;
+
+		rc = server->ops->query_symlink(xid, tcon, cifs_sb, full_path,
+						&target_path, reparse_point);
+	}
 
 	kfree(full_path);
 	free_xid(xid);
diff --git a/fs/cifs/smb1ops.c b/fs/cifs/smb1ops.c
index c711f1f39bf2..c4e75afa3258 100644
--- a/fs/cifs/smb1ops.c
+++ b/fs/cifs/smb1ops.c
@@ -950,8 +950,8 @@ cifs_unix_dfs_readlink(const unsigned int xid, struct cifs_tcon *tcon,
 
 static int
 cifs_query_symlink(const unsigned int xid, struct cifs_tcon *tcon,
-		   const char *full_path, char **target_path,
-		   struct cifs_sb_info *cifs_sb)
+		   struct cifs_sb_info *cifs_sb, const char *full_path,
+		   char **target_path, bool is_reparse_point)
 {
 	int rc;
 	int oplock = 0;
@@ -960,6 +960,11 @@ cifs_query_symlink(const unsigned int xid, struct cifs_tcon *tcon,
 
 	cifs_dbg(FYI, "%s: path: %s\n", __func__, full_path);
 
+	if (is_reparse_point) {
+		cifs_dbg(VFS, "reparse points not handled for SMB1 symlinks\n");
+		return -EOPNOTSUPP;
+	}
+
 	/* Check for unix extensions */
 	if (cap_unix(tcon->ses)) {
 		rc = CIFSSMBUnixQuerySymLink(xid, tcon, full_path, target_path,
diff --git a/fs/cifs/smb2ops.c b/fs/cifs/smb2ops.c
index c36ff0d1fe2a..a930c8965e5c 100644
--- a/fs/cifs/smb2ops.c
+++ b/fs/cifs/smb2ops.c
@@ -1382,6 +1382,26 @@ smb2_ioctl_query_info(const unsigned int xid,
 	oparms.fid = &fid;
 	oparms.reconnect = false;
 
+	/*
+	 * FSCTL codes encode the special access they need in the fsctl code.
+	 */
+	if (qi.flags & PASSTHRU_FSCTL) {
+		switch (qi.info_type & FSCTL_DEVICE_ACCESS_MASK) {
+		case FSCTL_DEVICE_ACCESS_FILE_READ_WRITE_ACCESS:
+			oparms.desired_access = FILE_READ_DATA | FILE_WRITE_DATA | FILE_READ_ATTRIBUTES | SYNCHRONIZE;
+			break;
+		case FSCTL_DEVICE_ACCESS_FILE_ANY_ACCESS:
+			oparms.desired_access = GENERIC_ALL;
+			break;
+		case FSCTL_DEVICE_ACCESS_FILE_READ_ACCESS:
+			oparms.desired_access = GENERIC_READ;
+			break;
+		case FSCTL_DEVICE_ACCESS_FILE_WRITE_ACCESS:
+			oparms.desired_access = GENERIC_WRITE;
+			break;
+		}
+	}
+
 	rc = SMB2_open_init(tcon, &rqst[0], &oplock, &oparms, path);
 	if (rc)
 		goto iqinf_exit;
@@ -1399,8 +1419,9 @@ smb2_ioctl_query_info(const unsigned int xid,
 
 			rc = SMB2_ioctl_init(tcon, &rqst[1],
 					     COMPOUND_FID, COMPOUND_FID,
-					     qi.info_type, true, NULL,
-					     0, CIFSMaxBufSize);
+					     qi.info_type, true, buffer,
+					     qi.output_buffer_length,
+					     CIFSMaxBufSize);
 		}
 	} else if (qi.flags == PASSTHRU_QUERY_INFO) {
 		memset(&qi_iov, 0, sizeof(qi_iov));
@@ -1441,12 +1462,19 @@ smb2_ioctl_query_info(const unsigned int xid,
 		io_rsp = (struct smb2_ioctl_rsp *)rsp_iov[1].iov_base;
 		if (le32_to_cpu(io_rsp->OutputCount) < qi.input_buffer_length)
 			qi.input_buffer_length = le32_to_cpu(io_rsp->OutputCount);
+		if (qi.input_buffer_length > 0 &&
+		    le32_to_cpu(io_rsp->OutputOffset) + qi.input_buffer_length > rsp_iov[1].iov_len) {
+			rc = -EFAULT;
+			goto iqinf_exit;
+		}
 		if (copy_to_user(&pqi->input_buffer_length, &qi.input_buffer_length,
 				 sizeof(qi.input_buffer_length))) {
 			rc = -EFAULT;
 			goto iqinf_exit;
 		}
-		if (copy_to_user(pqi + 1, &io_rsp[1], qi.input_buffer_length)) {
+		if (copy_to_user((void __user *)pqi + sizeof(struct smb_query_info),
+				 (const void *)io_rsp + le32_to_cpu(io_rsp->OutputOffset),
+				 qi.input_buffer_length)) {
 			rc = -EFAULT;
 			goto iqinf_exit;
 		}
@@ -1821,6 +1849,14 @@ smb3_enum_snapshots(const unsigned int xid, struct cifs_tcon *tcon,
 	u32 max_response_size;
 	struct smb_snapshot_array snapshot_in;
 
+	/*
+	 * On the first query to enumerate the list of snapshots available
+	 * for this volume the buffer begins with 0 (number of snapshots
+	 * which can be returned is zero since at that point we do not know
+	 * how big the buffer needs to be). On the second query,
+	 * it (ret_data_len) is set to number of snapshots so we can
+	 * know to set the maximum response size larger (see below).
+	 */
 	if (get_user(ret_data_len, (unsigned int __user *)ioc_buf))
 		return -EFAULT;
 
@@ -2354,46 +2390,129 @@ smb2_get_dfs_refer(const unsigned int xid, struct cifs_ses *ses,
 
 static int
 smb2_query_symlink(const unsigned int xid, struct cifs_tcon *tcon,
-		   const char *full_path, char **target_path,
-		   struct cifs_sb_info *cifs_sb)
+		   struct cifs_sb_info *cifs_sb, const char *full_path,
+		   char **target_path, bool is_reparse_point)
 {
 	int rc;
-	__le16 *utf16_path;
+	__le16 *utf16_path = NULL;
 	__u8 oplock = SMB2_OPLOCK_LEVEL_NONE;
 	struct cifs_open_parms oparms;
 	struct cifs_fid fid;
 	struct kvec err_iov = {NULL, 0};
 	struct smb2_err_rsp *err_buf = NULL;
-	int resp_buftype;
 	struct smb2_symlink_err_rsp *symlink;
 	unsigned int sub_len;
 	unsigned int sub_offset;
 	unsigned int print_len;
 	unsigned int print_offset;
+	int flags = 0;
+	struct smb_rqst rqst[3];
+	int resp_buftype[3];
+	struct kvec rsp_iov[3];
+	struct kvec open_iov[SMB2_CREATE_IOV_SIZE];
+	struct kvec io_iov[SMB2_IOCTL_IOV_SIZE];
+	struct kvec close_iov[1];
+	struct smb2_create_rsp *create_rsp;
+	struct smb2_ioctl_rsp *ioctl_rsp;
+	char *ioctl_buf;
+	u32 plen;
 
 	cifs_dbg(FYI, "%s: path: %s\n", __func__, full_path);
 
+	if (smb3_encryption_required(tcon))
+		flags |= CIFS_TRANSFORM_REQ;
+
+	memset(rqst, 0, sizeof(rqst));
+	resp_buftype[0] = resp_buftype[1] = resp_buftype[2] = CIFS_NO_BUFFER;
+	memset(rsp_iov, 0, sizeof(rsp_iov));
+
 	utf16_path = cifs_convert_path_to_utf16(full_path, cifs_sb);
 	if (!utf16_path)
 		return -ENOMEM;
 
+	/* Open */
+	memset(&open_iov, 0, sizeof(open_iov));
+	rqst[0].rq_iov = open_iov;
+	rqst[0].rq_nvec = SMB2_CREATE_IOV_SIZE;
+
+	memset(&oparms, 0, sizeof(oparms));
 	oparms.tcon = tcon;
 	oparms.desired_access = FILE_READ_ATTRIBUTES;
 	oparms.disposition = FILE_OPEN;
+
 	if (backup_cred(cifs_sb))
 		oparms.create_options = CREATE_OPEN_BACKUP_INTENT;
 	else
 		oparms.create_options = 0;
+	if (is_reparse_point)
+		oparms.create_options = OPEN_REPARSE_POINT;
+
 	oparms.fid = &fid;
 	oparms.reconnect = false;
 
-	rc = SMB2_open(xid, &oparms, utf16_path, &oplock, NULL, &err_iov,
-		       &resp_buftype);
-	if (!rc)
-		SMB2_close(xid, tcon, fid.persistent_fid, fid.volatile_fid);
+	rc = SMB2_open_init(tcon, &rqst[0], &oplock, &oparms, utf16_path);
+	if (rc)
+		goto querty_exit;
+	smb2_set_next_command(tcon, &rqst[0]);
+
+
+	/* IOCTL */
+	memset(&io_iov, 0, sizeof(io_iov));
+	rqst[1].rq_iov = io_iov;
+	rqst[1].rq_nvec = SMB2_IOCTL_IOV_SIZE;
+
+	rc = SMB2_ioctl_init(tcon, &rqst[1], fid.persistent_fid,
+			     fid.volatile_fid, FSCTL_GET_REPARSE_POINT,
+			     true /* is_fctl */, NULL, 0, CIFSMaxBufSize);
+	if (rc)
+		goto querty_exit;
+
+	smb2_set_next_command(tcon, &rqst[1]);
+	smb2_set_related(&rqst[1]);
+
+
+	/* Close */
+	memset(&close_iov, 0, sizeof(close_iov));
+	rqst[2].rq_iov = close_iov;
+	rqst[2].rq_nvec = 1;
+
+	rc = SMB2_close_init(tcon, &rqst[2], COMPOUND_FID, COMPOUND_FID);
+	if (rc)
+		goto querty_exit;
+
+	smb2_set_related(&rqst[2]);
+
+	rc = compound_send_recv(xid, tcon->ses, flags, 3, rqst,
+				resp_buftype, rsp_iov);
+
+	create_rsp = rsp_iov[0].iov_base;
+	if (create_rsp && create_rsp->sync_hdr.Status)
+		err_iov = rsp_iov[0];
+	ioctl_rsp = rsp_iov[1].iov_base;
+
+	/*
+	 * Open was successful and we got an ioctl response.
+	 */
+	if ((rc == 0) && (is_reparse_point)) {
+		/* See MS-FSCC 2.3.23 */
+
+		ioctl_buf = (char *)ioctl_rsp + le32_to_cpu(ioctl_rsp->OutputOffset);
+		plen = le32_to_cpu(ioctl_rsp->OutputCount);
+
+		if (plen + le32_to_cpu(ioctl_rsp->OutputOffset) >
+		    rsp_iov[1].iov_len) {
+			cifs_dbg(VFS, "srv returned invalid ioctl length: %d\n", plen);
+			rc = -EIO;
+			goto querty_exit;
+		}
+
+		/* Do stuff with ioctl_buf/plen */
+		goto querty_exit;
+	}
+
 	if (!rc || !err_iov.iov_base) {
 		rc = -ENOENT;
-		goto free_path;
+		goto querty_exit;
 	}
 
 	err_buf = err_iov.iov_base;
@@ -2433,9 +2552,14 @@ smb2_query_symlink(const unsigned int xid, struct cifs_tcon *tcon,
 	cifs_dbg(FYI, "%s: target path: %s\n", __func__, *target_path);
 
  querty_exit:
-	free_rsp_buf(resp_buftype, err_buf);
- free_path:
+	cifs_dbg(FYI, "query symlink rc %d\n", rc);
 	kfree(utf16_path);
+	SMB2_open_free(&rqst[0]);
+	SMB2_ioctl_free(&rqst[1]);
+	SMB2_close_free(&rqst[2]);
+	free_rsp_buf(resp_buftype[0], rsp_iov[0].iov_base);
+	free_rsp_buf(resp_buftype[1], rsp_iov[1].iov_base);
+	free_rsp_buf(resp_buftype[2], rsp_iov[2].iov_base);
 	return rc;
 }
 
@@ -2612,16 +2736,8 @@ static long smb3_zero_range(struct file *file, struct cifs_tcon *tcon,
 	struct cifsInodeInfo *cifsi;
 	struct cifsFileInfo *cfile = file->private_data;
 	struct file_zero_data_information fsctl_buf;
-	struct smb_rqst rqst[2];
-	int resp_buftype[2];
-	struct kvec rsp_iov[2];
-	struct kvec io_iov[SMB2_IOCTL_IOV_SIZE];
-	struct kvec si_iov[1];
-	unsigned int size[1];
-	void *data[1];
 	long rc;
 	unsigned int xid;
-	int num = 0, flags = 0;
 	__le64 eof;
 
 	xid = get_xid();
@@ -2643,39 +2759,16 @@ static long smb3_zero_range(struct file *file, struct cifs_tcon *tcon,
 			return rc;
 		}
 
-	/*
-	 * Must check if file sparse since fallocate -z (zero range) assumes
-	 * non-sparse allocation
-	 */
-	if (!(cifsi->cifsAttrs & FILE_ATTRIBUTE_SPARSE_FILE)) {
-		rc = -EOPNOTSUPP;
-		trace_smb3_zero_err(xid, cfile->fid.persistent_fid, tcon->tid,
-			      ses->Suid, offset, len, rc);
-		free_xid(xid);
-		return rc;
-	}
-
 	cifs_dbg(FYI, "offset %lld len %lld", offset, len);
 
 	fsctl_buf.FileOffset = cpu_to_le64(offset);
 	fsctl_buf.BeyondFinalZero = cpu_to_le64(offset + len);
 
-	if (smb3_encryption_required(tcon))
-		flags |= CIFS_TRANSFORM_REQ;
-
-	memset(rqst, 0, sizeof(rqst));
-	resp_buftype[0] = resp_buftype[1] = CIFS_NO_BUFFER;
-	memset(rsp_iov, 0, sizeof(rsp_iov));
-
-
-	memset(&io_iov, 0, sizeof(io_iov));
-	rqst[num].rq_iov = io_iov;
-	rqst[num].rq_nvec = SMB2_IOCTL_IOV_SIZE;
-	rc = SMB2_ioctl_init(tcon, &rqst[num++], cfile->fid.persistent_fid,
-			     cfile->fid.volatile_fid, FSCTL_SET_ZERO_DATA,
-			     true /* is_fctl */, (char *)&fsctl_buf,
-			     sizeof(struct file_zero_data_information),
-			     CIFSMaxBufSize);
+	rc = SMB2_ioctl(xid, tcon, cfile->fid.persistent_fid,
+			cfile->fid.volatile_fid, FSCTL_SET_ZERO_DATA, true,
+			(char *)&fsctl_buf,
+			sizeof(struct file_zero_data_information),
+			0, NULL, NULL);
 	if (rc)
 		goto zero_range_exit;
 
@@ -2683,33 +2776,12 @@ static long smb3_zero_range(struct file *file, struct cifs_tcon *tcon,
 	 * do we also need to change the size of the file?
 	 */
 	if (keep_size == false && i_size_read(inode) < offset + len) {
-		smb2_set_next_command(tcon, &rqst[0]);
-
-		memset(&si_iov, 0, sizeof(si_iov));
-		rqst[num].rq_iov = si_iov;
-		rqst[num].rq_nvec = 1;
-
 		eof = cpu_to_le64(offset + len);
-		size[0] = 8; /* sizeof __le64 */
-		data[0] = &eof;
-
-		rc = SMB2_set_info_init(tcon, &rqst[num++],
-					cfile->fid.persistent_fid,
-					cfile->fid.persistent_fid,
-					current->tgid,
-					FILE_END_OF_FILE_INFORMATION,
-					SMB2_O_INFO_FILE, 0, data, size);
-		smb2_set_related(&rqst[1]);
+		rc = SMB2_set_eof(xid, tcon, cfile->fid.persistent_fid,
+				  cfile->fid.volatile_fid, cfile->pid, &eof);
 	}
 
-	rc = compound_send_recv(xid, ses, flags, num, rqst,
-				resp_buftype, rsp_iov);
-
  zero_range_exit:
-	SMB2_ioctl_free(&rqst[0]);
-	SMB2_set_info_free(&rqst[1]);
-	free_rsp_buf(resp_buftype[0], rsp_iov[0].iov_base);
-	free_rsp_buf(resp_buftype[1], rsp_iov[1].iov_base);
 	free_xid(xid);
 	if (rc)
 		trace_smb3_zero_err(xid, cfile->fid.persistent_fid, tcon->tid,
@@ -2850,6 +2922,79 @@ static long smb3_simple_falloc(struct file *file, struct cifs_tcon *tcon,
 	return rc;
 }
 
+static int smb3_fiemap(struct cifs_tcon *tcon,
+		       struct cifsFileInfo *cfile,
+		       struct fiemap_extent_info *fei, u64 start, u64 len)
+{
+	unsigned int xid;
+	struct file_allocated_range_buffer in_data, *out_data;
+	u32 out_data_len;
+	int i, num, rc, flags, last_blob;
+	u64 next;
+
+	if (fiemap_check_flags(fei, FIEMAP_FLAG_SYNC))
+		return -EBADR;
+
+	xid = get_xid();
+ again:
+	in_data.file_offset = cpu_to_le64(start);
+	in_data.length = cpu_to_le64(len);
+
+	rc = SMB2_ioctl(xid, tcon, cfile->fid.persistent_fid,
+			cfile->fid.volatile_fid,
+			FSCTL_QUERY_ALLOCATED_RANGES, true,
+			(char *)&in_data, sizeof(in_data),
+			1024 * sizeof(struct file_allocated_range_buffer),
+			(char **)&out_data, &out_data_len);
+	if (rc == -E2BIG) {
+		last_blob = 0;
+		rc = 0;
+	} else
+		last_blob = 1;
+	if (rc)
+		goto out;
+
+	if (out_data_len < sizeof(struct file_allocated_range_buffer)) {
+		rc = -EINVAL;
+		goto out;
+	}
+	if (out_data_len % sizeof(struct file_allocated_range_buffer)) {
+		rc = -EINVAL;
+		goto out;
+	}
+
+	num = out_data_len / sizeof(struct file_allocated_range_buffer);
+	for (i = 0; i < num; i++) {
+		flags = 0;
+		if (i == num - 1 && last_blob)
+			flags |= FIEMAP_EXTENT_LAST;
+
+		rc = fiemap_fill_next_extent(fei,
+				le64_to_cpu(out_data[i].file_offset),
+				le64_to_cpu(out_data[i].file_offset),
+				le64_to_cpu(out_data[i].length),
+				flags);
+		if (rc < 0)
+			goto out;
+		if (rc == 1) {
+			rc = 0;
+			goto out;
+		}
+	}
+
+	if (!last_blob) {
+		next = le64_to_cpu(out_data[num - 1].file_offset) +
+		  le64_to_cpu(out_data[num - 1].length);
+		len = len - (next - start);
+		start = next;
+		goto again;
+	}
+
+ out:
+	free_xid(xid);
+	kfree(out_data);
+	return rc;
+}
 
 static long smb3_fallocate(struct file *file, struct cifs_tcon *tcon, int mode,
 			   loff_t off, loff_t len)
@@ -2917,26 +3062,28 @@ smb21_set_oplock_level(struct cifsInodeInfo *cinode, __u32 oplock,
 		       unsigned int epoch, bool *purge_cache)
 {
 	char message[5] = {0};
+	unsigned int new_oplock = 0;
 
 	oplock &= 0xFF;
 	if (oplock == SMB2_OPLOCK_LEVEL_NOCHANGE)
 		return;
 
-	cinode->oplock = 0;
 	if (oplock & SMB2_LEASE_READ_CACHING_HE) {
-		cinode->oplock |= CIFS_CACHE_READ_FLG;
+		new_oplock |= CIFS_CACHE_READ_FLG;
 		strcat(message, "R");
 	}
 	if (oplock & SMB2_LEASE_HANDLE_CACHING_HE) {
-		cinode->oplock |= CIFS_CACHE_HANDLE_FLG;
+		new_oplock |= CIFS_CACHE_HANDLE_FLG;
 		strcat(message, "H");
 	}
 	if (oplock & SMB2_LEASE_WRITE_CACHING_HE) {
-		cinode->oplock |= CIFS_CACHE_WRITE_FLG;
+		new_oplock |= CIFS_CACHE_WRITE_FLG;
 		strcat(message, "W");
 	}
-	if (!cinode->oplock)
-		strcat(message, "None");
+	if (!new_oplock)
+		strncpy(message, "None", sizeof(message));
+
+	cinode->oplock = new_oplock;
 	cifs_dbg(FYI, "%s Lease granted on inode %p\n", message,
 		 &cinode->vfs_inode);
 }
@@ -4018,6 +4165,7 @@ struct smb_version_operations smb20_operations = {
 	.next_header = smb2_next_header,
 	.ioctl_query_info = smb2_ioctl_query_info,
 	.make_node = smb2_make_node,
+	.fiemap = smb3_fiemap,
 };
 
 struct smb_version_operations smb21_operations = {
@@ -4117,6 +4265,7 @@ struct smb_version_operations smb21_operations = {
 	.next_header = smb2_next_header,
 	.ioctl_query_info = smb2_ioctl_query_info,
 	.make_node = smb2_make_node,
+	.fiemap = smb3_fiemap,
 };
 
 struct smb_version_operations smb30_operations = {
@@ -4225,6 +4374,7 @@ struct smb_version_operations smb30_operations = {
 	.next_header = smb2_next_header,
 	.ioctl_query_info = smb2_ioctl_query_info,
 	.make_node = smb2_make_node,
+	.fiemap = smb3_fiemap,
 };
 
 struct smb_version_operations smb311_operations = {
@@ -4334,6 +4484,7 @@ struct smb_version_operations smb311_operations = {
 	.next_header = smb2_next_header,
 	.ioctl_query_info = smb2_ioctl_query_info,
 	.make_node = smb2_make_node,
+	.fiemap = smb3_fiemap,
 };
 
 struct smb_version_values smb20_values = {
diff --git a/fs/cifs/smb2pdu.c b/fs/cifs/smb2pdu.c
index a37774a55f3a..29f011d8d8e2 100644
--- a/fs/cifs/smb2pdu.c
+++ b/fs/cifs/smb2pdu.c
@@ -459,10 +459,7 @@ smb2_plain_req_init(__le16 smb2_command, struct cifs_tcon *tcon,
 	return rc;
 }
 
-
-#define SMB2_PREAUTH_INTEGRITY_CAPABILITIES	cpu_to_le16(1)
-#define SMB2_ENCRYPTION_CAPABILITIES		cpu_to_le16(2)
-#define SMB2_POSIX_EXTENSIONS_AVAILABLE		cpu_to_le16(0x100)
+/* For explanation of negotiate contexts see MS-SMB2 section 2.2.3.1 */
 
 static void
 build_preauth_ctxt(struct smb2_preauth_neg_context *pneg_ctxt)
@@ -476,6 +473,19 @@ build_preauth_ctxt(struct smb2_preauth_neg_context *pneg_ctxt)
 }
 
 static void
+build_compression_ctxt(struct smb2_compression_capabilities_context *pneg_ctxt)
+{
+	pneg_ctxt->ContextType = SMB2_COMPRESSION_CAPABILITIES;
+	pneg_ctxt->DataLength =
+		cpu_to_le16(sizeof(struct smb2_compression_capabilities_context)
+			  - sizeof(struct smb2_neg_context));
+	pneg_ctxt->CompressionAlgorithmCount = cpu_to_le16(3);
+	pneg_ctxt->CompressionAlgorithms[0] = SMB3_COMPRESS_LZ77;
+	pneg_ctxt->CompressionAlgorithms[1] = SMB3_COMPRESS_LZ77_HUFF;
+	pneg_ctxt->CompressionAlgorithms[2] = SMB3_COMPRESS_LZNT1;
+}
+
+static void
 build_encrypt_ctxt(struct smb2_encryption_neg_context *pneg_ctxt)
 {
 	pneg_ctxt->ContextType = SMB2_ENCRYPTION_CAPABILITIES;
@@ -541,10 +551,17 @@ assemble_neg_contexts(struct smb2_negotiate_req *req,
 	*total_len += ctxt_len;
 	pneg_ctxt += ctxt_len;
 
+	build_compression_ctxt((struct smb2_compression_capabilities_context *)
+				pneg_ctxt);
+	ctxt_len = DIV_ROUND_UP(
+		sizeof(struct smb2_compression_capabilities_context), 8) * 8;
+	*total_len += ctxt_len;
+	pneg_ctxt += ctxt_len;
+
 	build_posix_ctxt((struct smb2_posix_neg_context *)pneg_ctxt);
 	*total_len += sizeof(struct smb2_posix_neg_context);
 
-	req->NegotiateContextCount = cpu_to_le16(3);
+	req->NegotiateContextCount = cpu_to_le16(4);
 }
 
 static void decode_preauth_context(struct smb2_preauth_neg_context *ctxt)
@@ -562,6 +579,27 @@ static void decode_preauth_context(struct smb2_preauth_neg_context *ctxt)
 		printk_once(KERN_WARNING "unknown SMB3 hash algorithm\n");
 }
 
+static void decode_compress_ctx(struct TCP_Server_Info *server,
+			 struct smb2_compression_capabilities_context *ctxt)
+{
+	unsigned int len = le16_to_cpu(ctxt->DataLength);
+
+	/* sizeof compress context is a one element compression capbility struct */
+	if (len < 10) {
+		printk_once(KERN_WARNING "server sent bad compression cntxt\n");
+		return;
+	}
+	if (le16_to_cpu(ctxt->CompressionAlgorithmCount) != 1) {
+		printk_once(KERN_WARNING "illegal SMB3 compress algorithm count\n");
+		return;
+	}
+	if (le16_to_cpu(ctxt->CompressionAlgorithms[0]) > 3) {
+		printk_once(KERN_WARNING "unknown compression algorithm\n");
+		return;
+	}
+	server->compress_algorithm = ctxt->CompressionAlgorithms[0];
+}
+
 static int decode_encrypt_ctx(struct TCP_Server_Info *server,
 			      struct smb2_encryption_neg_context *ctxt)
 {
@@ -626,6 +664,9 @@ static int smb311_decode_neg_context(struct smb2_negotiate_rsp *rsp,
 		else if (pctx->ContextType == SMB2_ENCRYPTION_CAPABILITIES)
 			rc = decode_encrypt_ctx(server,
 				(struct smb2_encryption_neg_context *)pctx);
+		else if (pctx->ContextType == SMB2_COMPRESSION_CAPABILITIES)
+			decode_compress_ctx(server,
+				(struct smb2_compression_capabilities_context *)pctx);
 		else if (pctx->ContextType == SMB2_POSIX_EXTENSIONS_AVAILABLE)
 			server->posix_ext_supported = true;
 		else
@@ -1541,7 +1582,7 @@ SMB2_logoff(const unsigned int xid, struct cifs_ses *ses)
 	else if (server->sign)
 		req->sync_hdr.Flags |= SMB2_FLAGS_SIGNED;
 
-	flags |= CIFS_NO_RESP;
+	flags |= CIFS_NO_RSP_BUF;
 
 	iov[0].iov_base = (char *)req;
 	iov[0].iov_len = total_len;
@@ -1742,7 +1783,7 @@ SMB2_tdis(const unsigned int xid, struct cifs_tcon *tcon)
 	if (smb3_encryption_required(tcon))
 		flags |= CIFS_TRANSFORM_REQ;
 
-	flags |= CIFS_NO_RESP;
+	flags |= CIFS_NO_RSP_BUF;
 
 	iov[0].iov_base = (char *)req;
 	iov[0].iov_len = total_len;
@@ -2625,7 +2666,7 @@ SMB2_ioctl(const unsigned int xid, struct cifs_tcon *tcon, u64 persistent_fid,
 		trace_smb3_fsctl_err(xid, persistent_fid, tcon->tid,
 				ses->Suid, 0, opcode, rc);
 
-	if ((rc != 0) && (rc != -EINVAL)) {
+	if ((rc != 0) && (rc != -EINVAL) && (rc != -E2BIG)) {
 		cifs_stats_fail_inc(tcon, SMB2_IOCTL_HE);
 		goto ioctl_exit;
 	} else if (rc == -EINVAL) {
@@ -2634,6 +2675,11 @@ SMB2_ioctl(const unsigned int xid, struct cifs_tcon *tcon, u64 persistent_fid,
 			cifs_stats_fail_inc(tcon, SMB2_IOCTL_HE);
 			goto ioctl_exit;
 		}
+	} else if (rc == -E2BIG) {
+		if (opcode != FSCTL_QUERY_ALLOCATED_RANGES) {
+			cifs_stats_fail_inc(tcon, SMB2_IOCTL_HE);
+			goto ioctl_exit;
+		}
 	}
 
 	/* check if caller wants to look at return data or just return rc */
@@ -3223,7 +3269,7 @@ smb2_new_read_req(void **buf, unsigned int *total_len,
 				rdata->nr_pages, rdata->page_offset,
 				rdata->tailsz, true, need_invalidate);
 		if (!rdata->mr)
-			return -ENOBUFS;
+			return -EAGAIN;
 
 		req->Channel = SMB2_CHANNEL_RDMA_V1_INVALIDATE;
 		if (need_invalidate)
@@ -3628,7 +3674,7 @@ smb2_async_writev(struct cifs_writedata *wdata,
 				wdata->nr_pages, wdata->page_offset,
 				wdata->tailsz, false, need_invalidate);
 		if (!wdata->mr) {
-			rc = -ENOBUFS;
+			rc = -EAGAIN;
 			goto async_writev_out;
 		}
 		req->Length = 0;
@@ -4164,7 +4210,7 @@ SMB2_oplock_break(const unsigned int xid, struct cifs_tcon *tcon,
 	req->OplockLevel = oplock_level;
 	req->sync_hdr.CreditRequest = cpu_to_le16(1);
 
-	flags |= CIFS_NO_RESP;
+	flags |= CIFS_NO_RSP_BUF;
 
 	iov[0].iov_base = (char *)req;
 	iov[0].iov_len = total_len;
@@ -4438,7 +4484,7 @@ smb2_lockv(const unsigned int xid, struct cifs_tcon *tcon,
 	struct kvec rsp_iov;
 	int resp_buf_type;
 	unsigned int count;
-	int flags = CIFS_NO_RESP;
+	int flags = CIFS_NO_RSP_BUF;
 	unsigned int total_len;
 
 	cifs_dbg(FYI, "smb2_lockv num lock %d\n", num_lock);
@@ -4531,7 +4577,7 @@ SMB2_lease_break(const unsigned int xid, struct cifs_tcon *tcon,
 	memcpy(req->LeaseKey, lease_key, 16);
 	req->LeaseState = lease_state;
 
-	flags |= CIFS_NO_RESP;
+	flags |= CIFS_NO_RSP_BUF;
 
 	iov[0].iov_base = (char *)req;
 	iov[0].iov_len = total_len;
diff --git a/fs/cifs/smb2pdu.h b/fs/cifs/smb2pdu.h
index ee8977688e21..c7d5813bebd8 100644
--- a/fs/cifs/smb2pdu.h
+++ b/fs/cifs/smb2pdu.h
@@ -251,6 +251,14 @@ struct smb2_negotiate_req {
 #define SMB2_NT_FIND			0x00100000
 #define SMB2_LARGE_FILES		0x00200000
 
+
+/* Negotiate Contexts - ContextTypes. See MS-SMB2 section 2.2.3.1 for details */
+#define SMB2_PREAUTH_INTEGRITY_CAPABILITIES	cpu_to_le16(1)
+#define SMB2_ENCRYPTION_CAPABILITIES		cpu_to_le16(2)
+#define SMB2_COMPRESSION_CAPABILITIES		cpu_to_le16(3)
+#define SMB2_NETNAME_NEGOTIATE_CONTEXT_ID	cpu_to_le16(5)
+#define SMB2_POSIX_EXTENSIONS_AVAILABLE		cpu_to_le16(0x100)
+
 struct smb2_neg_context {
 	__le16	ContextType;
 	__le16	DataLength;
@@ -288,6 +296,27 @@ struct smb2_encryption_neg_context {
 	__le16	Ciphers[1]; /* Ciphers[0] since only one used now */
 } __packed;
 
+/* See MS-SMB2 2.2.3.1.3 */
+#define SMB3_COMPRESS_NONE	cpu_to_le16(0x0000)
+#define SMB3_COMPRESS_LZNT1	cpu_to_le16(0x0001)
+#define SMB3_COMPRESS_LZ77	cpu_to_le16(0x0002)
+#define SMB3_COMPRESS_LZ77_HUFF	cpu_to_le16(0x0003)
+
+struct smb2_compression_capabilities_context {
+	__le16	ContextType; /* 3 */
+	__le16  DataLength;
+	__u32	Reserved;
+	__le16	CompressionAlgorithmCount;
+	__u16	Padding;
+	__u32	Reserved1;
+	__le16	CompressionAlgorithms[3];
+} __packed;
+
+/*
+ * For smb2_netname_negotiate_context_id See MS-SMB2 2.2.3.1.4.
+ * Its struct simply contains NetName, an array of Unicode characters
+ */
+
 #define POSIX_CTXT_DATA_LEN	16
 struct smb2_posix_neg_context {
 	__le16	ContextType; /* 0x100 */
@@ -842,6 +871,11 @@ struct fsctl_get_integrity_information_rsp {
 	__le32	ClusterSizeInBytes;
 } __packed;
 
+struct file_allocated_range_buffer {
+	__le64	file_offset;
+	__le64	length;
+} __packed;
+
 /* Integrity ChecksumAlgorithm choices for above */
 #define	CHECKSUM_TYPE_NONE	0x0000
 #define	CHECKSUM_TYPE_CRC64	0x0002
@@ -1047,6 +1081,7 @@ struct smb2_flush_rsp {
 
 /* For read request Flags field below, following flag is defined for SMB3.02 */
 #define SMB2_READFLAG_READ_UNBUFFERED	0x01
+#define SMB2_READFLAG_REQUEST_COMPRESSED 0x02 /* See MS-SMB2 2.2.19 */
 
 /* Channel field for read and write: exactly one of following flags can be set*/
 #define SMB2_CHANNEL_NONE	cpu_to_le32(0x00000000)
@@ -1113,6 +1148,42 @@ struct smb2_write_rsp {
 	__u8   Buffer[1];
 } __packed;
 
+/* notify flags */
+#define SMB2_WATCH_TREE			0x0001
+
+/* notify completion filter flags. See MS-FSCC 2.6 and MS-SMB2 2.2.35 */
+#define FILE_NOTIFY_CHANGE_FILE_NAME		0x00000001
+#define FILE_NOTIFY_CHANGE_DIR_NAME		0x00000002
+#define FILE_NOTIFY_CHANGE_ATTRIBUTES		0x00000004
+#define FILE_NOTIFY_CHANGE_SIZE			0x00000008
+#define FILE_NOTIFY_CHANGE_LAST_WRITE		0x00000010
+#define FILE_NOTIFY_CHANGE_LAST_ACCESS		0x00000020
+#define FILE_NOTIFY_CHANGE_CREATION		0x00000040
+#define FILE_NOTIFY_CHANGE_EA			0x00000080
+#define FILE_NOTIFY_CHANGE_SECURITY		0x00000100
+#define FILE_NOTIFY_CHANGE_STREAM_NAME		0x00000200
+#define FILE_NOTIFY_CHANGE_STREAM_SIZE		0x00000400
+#define FILE_NOTIFY_CHANGE_STREAM_WRITE		0x00000800
+
+struct smb2_change_notify_req {
+	struct smb2_sync_hdr sync_hdr;
+	__le16	StructureSize;
+	__le16	Flags;
+	__le32	OutputBufferLength;
+	__u64	PersistentFileId; /* opaque endianness */
+	__u64	VolatileFileId; /* opaque endianness */
+	__le32	CompletionFilter;
+	__u32	Reserved;
+} __packed;
+
+struct smb2_change_notify_rsp {
+	struct smb2_sync_hdr sync_hdr;
+	__le16	StructureSize;  /* Must be 9 */
+	__le16	OutputBufferOffset;
+	__le32	OutputBufferLength;
+	__u8	Buffer[1]; /* array of file notify structs */
+} __packed;
+
 #define SMB2_LOCKFLAG_SHARED_LOCK	0x0001
 #define SMB2_LOCKFLAG_EXCLUSIVE_LOCK	0x0002
 #define SMB2_LOCKFLAG_UNLOCK		0x0004
diff --git a/fs/cifs/smb2status.h b/fs/cifs/smb2status.h
index 447c0c6e4c64..7505056e9580 100644
--- a/fs/cifs/smb2status.h
+++ b/fs/cifs/smb2status.h
@@ -40,1743 +40,1743 @@ struct ntstatus {
 	__le32 Code;
 };
 
-#define STATUS_SUCCESS __constant_cpu_to_le32(0x00000000)
-#define STATUS_WAIT_0 __constant_cpu_to_le32(0x00000000)
-#define STATUS_WAIT_1 __constant_cpu_to_le32(0x00000001)
-#define STATUS_WAIT_2 __constant_cpu_to_le32(0x00000002)
-#define STATUS_WAIT_3 __constant_cpu_to_le32(0x00000003)
-#define STATUS_WAIT_63 __constant_cpu_to_le32(0x0000003F)
-#define STATUS_ABANDONED __constant_cpu_to_le32(0x00000080)
-#define STATUS_ABANDONED_WAIT_0 __constant_cpu_to_le32(0x00000080)
-#define STATUS_ABANDONED_WAIT_63 __constant_cpu_to_le32(0x000000BF)
-#define STATUS_USER_APC __constant_cpu_to_le32(0x000000C0)
-#define STATUS_KERNEL_APC __constant_cpu_to_le32(0x00000100)
-#define STATUS_ALERTED __constant_cpu_to_le32(0x00000101)
-#define STATUS_TIMEOUT __constant_cpu_to_le32(0x00000102)
-#define STATUS_PENDING __constant_cpu_to_le32(0x00000103)
-#define STATUS_REPARSE __constant_cpu_to_le32(0x00000104)
-#define STATUS_MORE_ENTRIES __constant_cpu_to_le32(0x00000105)
-#define STATUS_NOT_ALL_ASSIGNED __constant_cpu_to_le32(0x00000106)
-#define STATUS_SOME_NOT_MAPPED __constant_cpu_to_le32(0x00000107)
-#define STATUS_OPLOCK_BREAK_IN_PROGRESS __constant_cpu_to_le32(0x00000108)
-#define STATUS_VOLUME_MOUNTED __constant_cpu_to_le32(0x00000109)
-#define STATUS_RXACT_COMMITTED __constant_cpu_to_le32(0x0000010A)
-#define STATUS_NOTIFY_CLEANUP __constant_cpu_to_le32(0x0000010B)
-#define STATUS_NOTIFY_ENUM_DIR __constant_cpu_to_le32(0x0000010C)
-#define STATUS_NO_QUOTAS_FOR_ACCOUNT __constant_cpu_to_le32(0x0000010D)
-#define STATUS_PRIMARY_TRANSPORT_CONNECT_FAILED __constant_cpu_to_le32(0x0000010E)
-#define STATUS_PAGE_FAULT_TRANSITION __constant_cpu_to_le32(0x00000110)
-#define STATUS_PAGE_FAULT_DEMAND_ZERO __constant_cpu_to_le32(0x00000111)
-#define STATUS_PAGE_FAULT_COPY_ON_WRITE __constant_cpu_to_le32(0x00000112)
-#define STATUS_PAGE_FAULT_GUARD_PAGE __constant_cpu_to_le32(0x00000113)
-#define STATUS_PAGE_FAULT_PAGING_FILE __constant_cpu_to_le32(0x00000114)
-#define STATUS_CACHE_PAGE_LOCKED __constant_cpu_to_le32(0x00000115)
-#define STATUS_CRASH_DUMP __constant_cpu_to_le32(0x00000116)
-#define STATUS_BUFFER_ALL_ZEROS __constant_cpu_to_le32(0x00000117)
-#define STATUS_REPARSE_OBJECT __constant_cpu_to_le32(0x00000118)
-#define STATUS_RESOURCE_REQUIREMENTS_CHANGED __constant_cpu_to_le32(0x00000119)
-#define STATUS_TRANSLATION_COMPLETE __constant_cpu_to_le32(0x00000120)
-#define STATUS_DS_MEMBERSHIP_EVALUATED_LOCALLY __constant_cpu_to_le32(0x00000121)
-#define STATUS_NOTHING_TO_TERMINATE __constant_cpu_to_le32(0x00000122)
-#define STATUS_PROCESS_NOT_IN_JOB __constant_cpu_to_le32(0x00000123)
-#define STATUS_PROCESS_IN_JOB __constant_cpu_to_le32(0x00000124)
-#define STATUS_VOLSNAP_HIBERNATE_READY __constant_cpu_to_le32(0x00000125)
-#define STATUS_FSFILTER_OP_COMPLETED_SUCCESSFULLY __constant_cpu_to_le32(0x00000126)
-#define STATUS_INTERRUPT_VECTOR_ALREADY_CONNECTED __constant_cpu_to_le32(0x00000127)
-#define STATUS_INTERRUPT_STILL_CONNECTED __constant_cpu_to_le32(0x00000128)
-#define STATUS_PROCESS_CLONED __constant_cpu_to_le32(0x00000129)
-#define STATUS_FILE_LOCKED_WITH_ONLY_READERS __constant_cpu_to_le32(0x0000012A)
-#define STATUS_FILE_LOCKED_WITH_WRITERS __constant_cpu_to_le32(0x0000012B)
-#define STATUS_RESOURCEMANAGER_READ_ONLY __constant_cpu_to_le32(0x00000202)
-#define STATUS_WAIT_FOR_OPLOCK __constant_cpu_to_le32(0x00000367)
-#define DBG_EXCEPTION_HANDLED __constant_cpu_to_le32(0x00010001)
-#define DBG_CONTINUE __constant_cpu_to_le32(0x00010002)
-#define STATUS_FLT_IO_COMPLETE __constant_cpu_to_le32(0x001C0001)
-#define STATUS_OBJECT_NAME_EXISTS __constant_cpu_to_le32(0x40000000)
-#define STATUS_THREAD_WAS_SUSPENDED __constant_cpu_to_le32(0x40000001)
-#define STATUS_WORKING_SET_LIMIT_RANGE __constant_cpu_to_le32(0x40000002)
-#define STATUS_IMAGE_NOT_AT_BASE __constant_cpu_to_le32(0x40000003)
-#define STATUS_RXACT_STATE_CREATED __constant_cpu_to_le32(0x40000004)
-#define STATUS_SEGMENT_NOTIFICATION __constant_cpu_to_le32(0x40000005)
-#define STATUS_LOCAL_USER_SESSION_KEY __constant_cpu_to_le32(0x40000006)
-#define STATUS_BAD_CURRENT_DIRECTORY __constant_cpu_to_le32(0x40000007)
-#define STATUS_SERIAL_MORE_WRITES __constant_cpu_to_le32(0x40000008)
-#define STATUS_REGISTRY_RECOVERED __constant_cpu_to_le32(0x40000009)
-#define STATUS_FT_READ_RECOVERY_FROM_BACKUP __constant_cpu_to_le32(0x4000000A)
-#define STATUS_FT_WRITE_RECOVERY __constant_cpu_to_le32(0x4000000B)
-#define STATUS_SERIAL_COUNTER_TIMEOUT __constant_cpu_to_le32(0x4000000C)
-#define STATUS_NULL_LM_PASSWORD __constant_cpu_to_le32(0x4000000D)
-#define STATUS_IMAGE_MACHINE_TYPE_MISMATCH __constant_cpu_to_le32(0x4000000E)
-#define STATUS_RECEIVE_PARTIAL __constant_cpu_to_le32(0x4000000F)
-#define STATUS_RECEIVE_EXPEDITED __constant_cpu_to_le32(0x40000010)
-#define STATUS_RECEIVE_PARTIAL_EXPEDITED __constant_cpu_to_le32(0x40000011)
-#define STATUS_EVENT_DONE __constant_cpu_to_le32(0x40000012)
-#define STATUS_EVENT_PENDING __constant_cpu_to_le32(0x40000013)
-#define STATUS_CHECKING_FILE_SYSTEM __constant_cpu_to_le32(0x40000014)
-#define STATUS_FATAL_APP_EXIT __constant_cpu_to_le32(0x40000015)
-#define STATUS_PREDEFINED_HANDLE __constant_cpu_to_le32(0x40000016)
-#define STATUS_WAS_UNLOCKED __constant_cpu_to_le32(0x40000017)
-#define STATUS_SERVICE_NOTIFICATION __constant_cpu_to_le32(0x40000018)
-#define STATUS_WAS_LOCKED __constant_cpu_to_le32(0x40000019)
-#define STATUS_LOG_HARD_ERROR __constant_cpu_to_le32(0x4000001A)
-#define STATUS_ALREADY_WIN32 __constant_cpu_to_le32(0x4000001B)
-#define STATUS_WX86_UNSIMULATE __constant_cpu_to_le32(0x4000001C)
-#define STATUS_WX86_CONTINUE __constant_cpu_to_le32(0x4000001D)
-#define STATUS_WX86_SINGLE_STEP __constant_cpu_to_le32(0x4000001E)
-#define STATUS_WX86_BREAKPOINT __constant_cpu_to_le32(0x4000001F)
-#define STATUS_WX86_EXCEPTION_CONTINUE __constant_cpu_to_le32(0x40000020)
-#define STATUS_WX86_EXCEPTION_LASTCHANCE __constant_cpu_to_le32(0x40000021)
-#define STATUS_WX86_EXCEPTION_CHAIN __constant_cpu_to_le32(0x40000022)
-#define STATUS_IMAGE_MACHINE_TYPE_MISMATCH_EXE __constant_cpu_to_le32(0x40000023)
-#define STATUS_NO_YIELD_PERFORMED __constant_cpu_to_le32(0x40000024)
-#define STATUS_TIMER_RESUME_IGNORED __constant_cpu_to_le32(0x40000025)
-#define STATUS_ARBITRATION_UNHANDLED __constant_cpu_to_le32(0x40000026)
-#define STATUS_CARDBUS_NOT_SUPPORTED __constant_cpu_to_le32(0x40000027)
-#define STATUS_WX86_CREATEWX86TIB __constant_cpu_to_le32(0x40000028)
-#define STATUS_MP_PROCESSOR_MISMATCH __constant_cpu_to_le32(0x40000029)
-#define STATUS_HIBERNATED __constant_cpu_to_le32(0x4000002A)
-#define STATUS_RESUME_HIBERNATION __constant_cpu_to_le32(0x4000002B)
-#define STATUS_FIRMWARE_UPDATED __constant_cpu_to_le32(0x4000002C)
-#define STATUS_DRIVERS_LEAKING_LOCKED_PAGES __constant_cpu_to_le32(0x4000002D)
-#define STATUS_MESSAGE_RETRIEVED __constant_cpu_to_le32(0x4000002E)
-#define STATUS_SYSTEM_POWERSTATE_TRANSITION __constant_cpu_to_le32(0x4000002F)
-#define STATUS_ALPC_CHECK_COMPLETION_LIST __constant_cpu_to_le32(0x40000030)
-#define STATUS_SYSTEM_POWERSTATE_COMPLEX_TRANSITION __constant_cpu_to_le32(0x40000031)
-#define STATUS_ACCESS_AUDIT_BY_POLICY __constant_cpu_to_le32(0x40000032)
-#define STATUS_ABANDON_HIBERFILE __constant_cpu_to_le32(0x40000033)
-#define STATUS_BIZRULES_NOT_ENABLED __constant_cpu_to_le32(0x40000034)
-#define STATUS_WAKE_SYSTEM __constant_cpu_to_le32(0x40000294)
-#define STATUS_DS_SHUTTING_DOWN __constant_cpu_to_le32(0x40000370)
-#define DBG_REPLY_LATER __constant_cpu_to_le32(0x40010001)
-#define DBG_UNABLE_TO_PROVIDE_HANDLE __constant_cpu_to_le32(0x40010002)
-#define DBG_TERMINATE_THREAD __constant_cpu_to_le32(0x40010003)
-#define DBG_TERMINATE_PROCESS __constant_cpu_to_le32(0x40010004)
-#define DBG_CONTROL_C __constant_cpu_to_le32(0x40010005)
-#define DBG_PRINTEXCEPTION_C __constant_cpu_to_le32(0x40010006)
-#define DBG_RIPEXCEPTION __constant_cpu_to_le32(0x40010007)
-#define DBG_CONTROL_BREAK __constant_cpu_to_le32(0x40010008)
-#define DBG_COMMAND_EXCEPTION __constant_cpu_to_le32(0x40010009)
-#define RPC_NT_UUID_LOCAL_ONLY __constant_cpu_to_le32(0x40020056)
-#define RPC_NT_SEND_INCOMPLETE __constant_cpu_to_le32(0x400200AF)
-#define STATUS_CTX_CDM_CONNECT __constant_cpu_to_le32(0x400A0004)
-#define STATUS_CTX_CDM_DISCONNECT __constant_cpu_to_le32(0x400A0005)
-#define STATUS_SXS_RELEASE_ACTIVATION_CONTEXT __constant_cpu_to_le32(0x4015000D)
-#define STATUS_RECOVERY_NOT_NEEDED __constant_cpu_to_le32(0x40190034)
-#define STATUS_RM_ALREADY_STARTED __constant_cpu_to_le32(0x40190035)
-#define STATUS_LOG_NO_RESTART __constant_cpu_to_le32(0x401A000C)
-#define STATUS_VIDEO_DRIVER_DEBUG_REPORT_REQUEST __constant_cpu_to_le32(0x401B00EC)
-#define STATUS_GRAPHICS_PARTIAL_DATA_POPULATED __constant_cpu_to_le32(0x401E000A)
-#define STATUS_GRAPHICS_DRIVER_MISMATCH __constant_cpu_to_le32(0x401E0117)
-#define STATUS_GRAPHICS_MODE_NOT_PINNED __constant_cpu_to_le32(0x401E0307)
-#define STATUS_GRAPHICS_NO_PREFERRED_MODE __constant_cpu_to_le32(0x401E031E)
-#define STATUS_GRAPHICS_DATASET_IS_EMPTY __constant_cpu_to_le32(0x401E034B)
-#define STATUS_GRAPHICS_NO_MORE_ELEMENTS_IN_DATASET __constant_cpu_to_le32(0x401E034C)
-#define STATUS_GRAPHICS_PATH_CONTENT_GEOMETRY_TRANSFORMATION_NOT_PINNED __constant_cpu_to_le32(0x401E0351)
-#define STATUS_GRAPHICS_UNKNOWN_CHILD_STATUS __constant_cpu_to_le32(0x401E042F)
-#define STATUS_GRAPHICS_LEADLINK_START_DEFERRED __constant_cpu_to_le32(0x401E0437)
-#define STATUS_GRAPHICS_POLLING_TOO_FREQUENTLY __constant_cpu_to_le32(0x401E0439)
-#define STATUS_GRAPHICS_START_DEFERRED __constant_cpu_to_le32(0x401E043A)
-#define STATUS_NDIS_INDICATION_REQUIRED __constant_cpu_to_le32(0x40230001)
-#define STATUS_GUARD_PAGE_VIOLATION __constant_cpu_to_le32(0x80000001)
-#define STATUS_DATATYPE_MISALIGNMENT __constant_cpu_to_le32(0x80000002)
-#define STATUS_BREAKPOINT __constant_cpu_to_le32(0x80000003)
-#define STATUS_SINGLE_STEP __constant_cpu_to_le32(0x80000004)
-#define STATUS_BUFFER_OVERFLOW __constant_cpu_to_le32(0x80000005)
-#define STATUS_NO_MORE_FILES __constant_cpu_to_le32(0x80000006)
-#define STATUS_WAKE_SYSTEM_DEBUGGER __constant_cpu_to_le32(0x80000007)
-#define STATUS_HANDLES_CLOSED __constant_cpu_to_le32(0x8000000A)
-#define STATUS_NO_INHERITANCE __constant_cpu_to_le32(0x8000000B)
-#define STATUS_GUID_SUBSTITUTION_MADE __constant_cpu_to_le32(0x8000000C)
-#define STATUS_PARTIAL_COPY __constant_cpu_to_le32(0x8000000D)
-#define STATUS_DEVICE_PAPER_EMPTY __constant_cpu_to_le32(0x8000000E)
-#define STATUS_DEVICE_POWERED_OFF __constant_cpu_to_le32(0x8000000F)
-#define STATUS_DEVICE_OFF_LINE __constant_cpu_to_le32(0x80000010)
-#define STATUS_DEVICE_BUSY __constant_cpu_to_le32(0x80000011)
-#define STATUS_NO_MORE_EAS __constant_cpu_to_le32(0x80000012)
-#define STATUS_INVALID_EA_NAME __constant_cpu_to_le32(0x80000013)
-#define STATUS_EA_LIST_INCONSISTENT __constant_cpu_to_le32(0x80000014)
-#define STATUS_INVALID_EA_FLAG __constant_cpu_to_le32(0x80000015)
-#define STATUS_VERIFY_REQUIRED __constant_cpu_to_le32(0x80000016)
-#define STATUS_EXTRANEOUS_INFORMATION __constant_cpu_to_le32(0x80000017)
-#define STATUS_RXACT_COMMIT_NECESSARY __constant_cpu_to_le32(0x80000018)
-#define STATUS_NO_MORE_ENTRIES __constant_cpu_to_le32(0x8000001A)
-#define STATUS_FILEMARK_DETECTED __constant_cpu_to_le32(0x8000001B)
-#define STATUS_MEDIA_CHANGED __constant_cpu_to_le32(0x8000001C)
-#define STATUS_BUS_RESET __constant_cpu_to_le32(0x8000001D)
-#define STATUS_END_OF_MEDIA __constant_cpu_to_le32(0x8000001E)
-#define STATUS_BEGINNING_OF_MEDIA __constant_cpu_to_le32(0x8000001F)
-#define STATUS_MEDIA_CHECK __constant_cpu_to_le32(0x80000020)
-#define STATUS_SETMARK_DETECTED __constant_cpu_to_le32(0x80000021)
-#define STATUS_NO_DATA_DETECTED __constant_cpu_to_le32(0x80000022)
-#define STATUS_REDIRECTOR_HAS_OPEN_HANDLES __constant_cpu_to_le32(0x80000023)
-#define STATUS_SERVER_HAS_OPEN_HANDLES __constant_cpu_to_le32(0x80000024)
-#define STATUS_ALREADY_DISCONNECTED __constant_cpu_to_le32(0x80000025)
-#define STATUS_LONGJUMP __constant_cpu_to_le32(0x80000026)
-#define STATUS_CLEANER_CARTRIDGE_INSTALLED __constant_cpu_to_le32(0x80000027)
-#define STATUS_PLUGPLAY_QUERY_VETOED __constant_cpu_to_le32(0x80000028)
-#define STATUS_UNWIND_CONSOLIDATE __constant_cpu_to_le32(0x80000029)
-#define STATUS_REGISTRY_HIVE_RECOVERED __constant_cpu_to_le32(0x8000002A)
-#define STATUS_DLL_MIGHT_BE_INSECURE __constant_cpu_to_le32(0x8000002B)
-#define STATUS_DLL_MIGHT_BE_INCOMPATIBLE __constant_cpu_to_le32(0x8000002C)
-#define STATUS_STOPPED_ON_SYMLINK __constant_cpu_to_le32(0x8000002D)
-#define STATUS_DEVICE_REQUIRES_CLEANING __constant_cpu_to_le32(0x80000288)
-#define STATUS_DEVICE_DOOR_OPEN __constant_cpu_to_le32(0x80000289)
-#define STATUS_DATA_LOST_REPAIR __constant_cpu_to_le32(0x80000803)
-#define DBG_EXCEPTION_NOT_HANDLED __constant_cpu_to_le32(0x80010001)
-#define STATUS_CLUSTER_NODE_ALREADY_UP __constant_cpu_to_le32(0x80130001)
-#define STATUS_CLUSTER_NODE_ALREADY_DOWN __constant_cpu_to_le32(0x80130002)
-#define STATUS_CLUSTER_NETWORK_ALREADY_ONLINE __constant_cpu_to_le32(0x80130003)
-#define STATUS_CLUSTER_NETWORK_ALREADY_OFFLINE __constant_cpu_to_le32(0x80130004)
-#define STATUS_CLUSTER_NODE_ALREADY_MEMBER __constant_cpu_to_le32(0x80130005)
-#define STATUS_COULD_NOT_RESIZE_LOG __constant_cpu_to_le32(0x80190009)
-#define STATUS_NO_TXF_METADATA __constant_cpu_to_le32(0x80190029)
-#define STATUS_CANT_RECOVER_WITH_HANDLE_OPEN __constant_cpu_to_le32(0x80190031)
-#define STATUS_TXF_METADATA_ALREADY_PRESENT __constant_cpu_to_le32(0x80190041)
-#define STATUS_TRANSACTION_SCOPE_CALLBACKS_NOT_SET __constant_cpu_to_le32(0x80190042)
-#define STATUS_VIDEO_HUNG_DISPLAY_DRIVER_THREAD_RECOVERED __constant_cpu_to_le32(0x801B00EB)
-#define STATUS_FLT_BUFFER_TOO_SMALL __constant_cpu_to_le32(0x801C0001)
-#define STATUS_FVE_PARTIAL_METADATA __constant_cpu_to_le32(0x80210001)
-#define STATUS_UNSUCCESSFUL __constant_cpu_to_le32(0xC0000001)
-#define STATUS_NOT_IMPLEMENTED __constant_cpu_to_le32(0xC0000002)
-#define STATUS_INVALID_INFO_CLASS __constant_cpu_to_le32(0xC0000003)
-#define STATUS_INFO_LENGTH_MISMATCH __constant_cpu_to_le32(0xC0000004)
-#define STATUS_ACCESS_VIOLATION __constant_cpu_to_le32(0xC0000005)
-#define STATUS_IN_PAGE_ERROR __constant_cpu_to_le32(0xC0000006)
-#define STATUS_PAGEFILE_QUOTA __constant_cpu_to_le32(0xC0000007)
-#define STATUS_INVALID_HANDLE __constant_cpu_to_le32(0xC0000008)
-#define STATUS_BAD_INITIAL_STACK __constant_cpu_to_le32(0xC0000009)
-#define STATUS_BAD_INITIAL_PC __constant_cpu_to_le32(0xC000000A)
-#define STATUS_INVALID_CID __constant_cpu_to_le32(0xC000000B)
-#define STATUS_TIMER_NOT_CANCELED __constant_cpu_to_le32(0xC000000C)
-#define STATUS_INVALID_PARAMETER __constant_cpu_to_le32(0xC000000D)
-#define STATUS_NO_SUCH_DEVICE __constant_cpu_to_le32(0xC000000E)
-#define STATUS_NO_SUCH_FILE __constant_cpu_to_le32(0xC000000F)
-#define STATUS_INVALID_DEVICE_REQUEST __constant_cpu_to_le32(0xC0000010)
-#define STATUS_END_OF_FILE __constant_cpu_to_le32(0xC0000011)
-#define STATUS_WRONG_VOLUME __constant_cpu_to_le32(0xC0000012)
-#define STATUS_NO_MEDIA_IN_DEVICE __constant_cpu_to_le32(0xC0000013)
-#define STATUS_UNRECOGNIZED_MEDIA __constant_cpu_to_le32(0xC0000014)
-#define STATUS_NONEXISTENT_SECTOR __constant_cpu_to_le32(0xC0000015)
-#define STATUS_MORE_PROCESSING_REQUIRED __constant_cpu_to_le32(0xC0000016)
-#define STATUS_NO_MEMORY __constant_cpu_to_le32(0xC0000017)
-#define STATUS_CONFLICTING_ADDRESSES __constant_cpu_to_le32(0xC0000018)
-#define STATUS_NOT_MAPPED_VIEW __constant_cpu_to_le32(0xC0000019)
-#define STATUS_UNABLE_TO_FREE_VM __constant_cpu_to_le32(0xC000001A)
-#define STATUS_UNABLE_TO_DELETE_SECTION __constant_cpu_to_le32(0xC000001B)
-#define STATUS_INVALID_SYSTEM_SERVICE __constant_cpu_to_le32(0xC000001C)
-#define STATUS_ILLEGAL_INSTRUCTION __constant_cpu_to_le32(0xC000001D)
-#define STATUS_INVALID_LOCK_SEQUENCE __constant_cpu_to_le32(0xC000001E)
-#define STATUS_INVALID_VIEW_SIZE __constant_cpu_to_le32(0xC000001F)
-#define STATUS_INVALID_FILE_FOR_SECTION __constant_cpu_to_le32(0xC0000020)
-#define STATUS_ALREADY_COMMITTED __constant_cpu_to_le32(0xC0000021)
-#define STATUS_ACCESS_DENIED __constant_cpu_to_le32(0xC0000022)
-#define STATUS_BUFFER_TOO_SMALL __constant_cpu_to_le32(0xC0000023)
-#define STATUS_OBJECT_TYPE_MISMATCH __constant_cpu_to_le32(0xC0000024)
-#define STATUS_NONCONTINUABLE_EXCEPTION __constant_cpu_to_le32(0xC0000025)
-#define STATUS_INVALID_DISPOSITION __constant_cpu_to_le32(0xC0000026)
-#define STATUS_UNWIND __constant_cpu_to_le32(0xC0000027)
-#define STATUS_BAD_STACK __constant_cpu_to_le32(0xC0000028)
-#define STATUS_INVALID_UNWIND_TARGET __constant_cpu_to_le32(0xC0000029)
-#define STATUS_NOT_LOCKED __constant_cpu_to_le32(0xC000002A)
-#define STATUS_PARITY_ERROR __constant_cpu_to_le32(0xC000002B)
-#define STATUS_UNABLE_TO_DECOMMIT_VM __constant_cpu_to_le32(0xC000002C)
-#define STATUS_NOT_COMMITTED __constant_cpu_to_le32(0xC000002D)
-#define STATUS_INVALID_PORT_ATTRIBUTES __constant_cpu_to_le32(0xC000002E)
-#define STATUS_PORT_MESSAGE_TOO_LONG __constant_cpu_to_le32(0xC000002F)
-#define STATUS_INVALID_PARAMETER_MIX __constant_cpu_to_le32(0xC0000030)
-#define STATUS_INVALID_QUOTA_LOWER __constant_cpu_to_le32(0xC0000031)
-#define STATUS_DISK_CORRUPT_ERROR __constant_cpu_to_le32(0xC0000032)
-#define STATUS_OBJECT_NAME_INVALID __constant_cpu_to_le32(0xC0000033)
-#define STATUS_OBJECT_NAME_NOT_FOUND __constant_cpu_to_le32(0xC0000034)
-#define STATUS_OBJECT_NAME_COLLISION __constant_cpu_to_le32(0xC0000035)
-#define STATUS_PORT_DISCONNECTED __constant_cpu_to_le32(0xC0000037)
-#define STATUS_DEVICE_ALREADY_ATTACHED __constant_cpu_to_le32(0xC0000038)
-#define STATUS_OBJECT_PATH_INVALID __constant_cpu_to_le32(0xC0000039)
-#define STATUS_OBJECT_PATH_NOT_FOUND __constant_cpu_to_le32(0xC000003A)
-#define STATUS_OBJECT_PATH_SYNTAX_BAD __constant_cpu_to_le32(0xC000003B)
-#define STATUS_DATA_OVERRUN __constant_cpu_to_le32(0xC000003C)
-#define STATUS_DATA_LATE_ERROR __constant_cpu_to_le32(0xC000003D)
-#define STATUS_DATA_ERROR __constant_cpu_to_le32(0xC000003E)
-#define STATUS_CRC_ERROR __constant_cpu_to_le32(0xC000003F)
-#define STATUS_SECTION_TOO_BIG __constant_cpu_to_le32(0xC0000040)
-#define STATUS_PORT_CONNECTION_REFUSED __constant_cpu_to_le32(0xC0000041)
-#define STATUS_INVALID_PORT_HANDLE __constant_cpu_to_le32(0xC0000042)
-#define STATUS_SHARING_VIOLATION __constant_cpu_to_le32(0xC0000043)
-#define STATUS_QUOTA_EXCEEDED __constant_cpu_to_le32(0xC0000044)
-#define STATUS_INVALID_PAGE_PROTECTION __constant_cpu_to_le32(0xC0000045)
-#define STATUS_MUTANT_NOT_OWNED __constant_cpu_to_le32(0xC0000046)
-#define STATUS_SEMAPHORE_LIMIT_EXCEEDED __constant_cpu_to_le32(0xC0000047)
-#define STATUS_PORT_ALREADY_SET __constant_cpu_to_le32(0xC0000048)
-#define STATUS_SECTION_NOT_IMAGE __constant_cpu_to_le32(0xC0000049)
-#define STATUS_SUSPEND_COUNT_EXCEEDED __constant_cpu_to_le32(0xC000004A)
-#define STATUS_THREAD_IS_TERMINATING __constant_cpu_to_le32(0xC000004B)
-#define STATUS_BAD_WORKING_SET_LIMIT __constant_cpu_to_le32(0xC000004C)
-#define STATUS_INCOMPATIBLE_FILE_MAP __constant_cpu_to_le32(0xC000004D)
-#define STATUS_SECTION_PROTECTION __constant_cpu_to_le32(0xC000004E)
-#define STATUS_EAS_NOT_SUPPORTED __constant_cpu_to_le32(0xC000004F)
-#define STATUS_EA_TOO_LARGE __constant_cpu_to_le32(0xC0000050)
-#define STATUS_NONEXISTENT_EA_ENTRY __constant_cpu_to_le32(0xC0000051)
-#define STATUS_NO_EAS_ON_FILE __constant_cpu_to_le32(0xC0000052)
-#define STATUS_EA_CORRUPT_ERROR __constant_cpu_to_le32(0xC0000053)
-#define STATUS_FILE_LOCK_CONFLICT __constant_cpu_to_le32(0xC0000054)
-#define STATUS_LOCK_NOT_GRANTED __constant_cpu_to_le32(0xC0000055)
-#define STATUS_DELETE_PENDING __constant_cpu_to_le32(0xC0000056)
-#define STATUS_CTL_FILE_NOT_SUPPORTED __constant_cpu_to_le32(0xC0000057)
-#define STATUS_UNKNOWN_REVISION __constant_cpu_to_le32(0xC0000058)
-#define STATUS_REVISION_MISMATCH __constant_cpu_to_le32(0xC0000059)
-#define STATUS_INVALID_OWNER __constant_cpu_to_le32(0xC000005A)
-#define STATUS_INVALID_PRIMARY_GROUP __constant_cpu_to_le32(0xC000005B)
-#define STATUS_NO_IMPERSONATION_TOKEN __constant_cpu_to_le32(0xC000005C)
-#define STATUS_CANT_DISABLE_MANDATORY __constant_cpu_to_le32(0xC000005D)
-#define STATUS_NO_LOGON_SERVERS __constant_cpu_to_le32(0xC000005E)
-#define STATUS_NO_SUCH_LOGON_SESSION __constant_cpu_to_le32(0xC000005F)
-#define STATUS_NO_SUCH_PRIVILEGE __constant_cpu_to_le32(0xC0000060)
-#define STATUS_PRIVILEGE_NOT_HELD __constant_cpu_to_le32(0xC0000061)
-#define STATUS_INVALID_ACCOUNT_NAME __constant_cpu_to_le32(0xC0000062)
-#define STATUS_USER_EXISTS __constant_cpu_to_le32(0xC0000063)
-#define STATUS_NO_SUCH_USER __constant_cpu_to_le32(0xC0000064)
-#define STATUS_GROUP_EXISTS __constant_cpu_to_le32(0xC0000065)
-#define STATUS_NO_SUCH_GROUP __constant_cpu_to_le32(0xC0000066)
-#define STATUS_MEMBER_IN_GROUP __constant_cpu_to_le32(0xC0000067)
-#define STATUS_MEMBER_NOT_IN_GROUP __constant_cpu_to_le32(0xC0000068)
-#define STATUS_LAST_ADMIN __constant_cpu_to_le32(0xC0000069)
-#define STATUS_WRONG_PASSWORD __constant_cpu_to_le32(0xC000006A)
-#define STATUS_ILL_FORMED_PASSWORD __constant_cpu_to_le32(0xC000006B)
-#define STATUS_PASSWORD_RESTRICTION __constant_cpu_to_le32(0xC000006C)
-#define STATUS_LOGON_FAILURE __constant_cpu_to_le32(0xC000006D)
-#define STATUS_ACCOUNT_RESTRICTION __constant_cpu_to_le32(0xC000006E)
-#define STATUS_INVALID_LOGON_HOURS __constant_cpu_to_le32(0xC000006F)
-#define STATUS_INVALID_WORKSTATION __constant_cpu_to_le32(0xC0000070)
-#define STATUS_PASSWORD_EXPIRED __constant_cpu_to_le32(0xC0000071)
-#define STATUS_ACCOUNT_DISABLED __constant_cpu_to_le32(0xC0000072)
-#define STATUS_NONE_MAPPED __constant_cpu_to_le32(0xC0000073)
-#define STATUS_TOO_MANY_LUIDS_REQUESTED __constant_cpu_to_le32(0xC0000074)
-#define STATUS_LUIDS_EXHAUSTED __constant_cpu_to_le32(0xC0000075)
-#define STATUS_INVALID_SUB_AUTHORITY __constant_cpu_to_le32(0xC0000076)
-#define STATUS_INVALID_ACL __constant_cpu_to_le32(0xC0000077)
-#define STATUS_INVALID_SID __constant_cpu_to_le32(0xC0000078)
-#define STATUS_INVALID_SECURITY_DESCR __constant_cpu_to_le32(0xC0000079)
-#define STATUS_PROCEDURE_NOT_FOUND __constant_cpu_to_le32(0xC000007A)
-#define STATUS_INVALID_IMAGE_FORMAT __constant_cpu_to_le32(0xC000007B)
-#define STATUS_NO_TOKEN __constant_cpu_to_le32(0xC000007C)
-#define STATUS_BAD_INHERITANCE_ACL __constant_cpu_to_le32(0xC000007D)
-#define STATUS_RANGE_NOT_LOCKED __constant_cpu_to_le32(0xC000007E)
-#define STATUS_DISK_FULL __constant_cpu_to_le32(0xC000007F)
-#define STATUS_SERVER_DISABLED __constant_cpu_to_le32(0xC0000080)
-#define STATUS_SERVER_NOT_DISABLED __constant_cpu_to_le32(0xC0000081)
-#define STATUS_TOO_MANY_GUIDS_REQUESTED __constant_cpu_to_le32(0xC0000082)
-#define STATUS_GUIDS_EXHAUSTED __constant_cpu_to_le32(0xC0000083)
-#define STATUS_INVALID_ID_AUTHORITY __constant_cpu_to_le32(0xC0000084)
-#define STATUS_AGENTS_EXHAUSTED __constant_cpu_to_le32(0xC0000085)
-#define STATUS_INVALID_VOLUME_LABEL __constant_cpu_to_le32(0xC0000086)
-#define STATUS_SECTION_NOT_EXTENDED __constant_cpu_to_le32(0xC0000087)
-#define STATUS_NOT_MAPPED_DATA __constant_cpu_to_le32(0xC0000088)
-#define STATUS_RESOURCE_DATA_NOT_FOUND __constant_cpu_to_le32(0xC0000089)
-#define STATUS_RESOURCE_TYPE_NOT_FOUND __constant_cpu_to_le32(0xC000008A)
-#define STATUS_RESOURCE_NAME_NOT_FOUND __constant_cpu_to_le32(0xC000008B)
-#define STATUS_ARRAY_BOUNDS_EXCEEDED __constant_cpu_to_le32(0xC000008C)
-#define STATUS_FLOAT_DENORMAL_OPERAND __constant_cpu_to_le32(0xC000008D)
-#define STATUS_FLOAT_DIVIDE_BY_ZERO __constant_cpu_to_le32(0xC000008E)
-#define STATUS_FLOAT_INEXACT_RESULT __constant_cpu_to_le32(0xC000008F)
-#define STATUS_FLOAT_INVALID_OPERATION __constant_cpu_to_le32(0xC0000090)
-#define STATUS_FLOAT_OVERFLOW __constant_cpu_to_le32(0xC0000091)
-#define STATUS_FLOAT_STACK_CHECK __constant_cpu_to_le32(0xC0000092)
-#define STATUS_FLOAT_UNDERFLOW __constant_cpu_to_le32(0xC0000093)
-#define STATUS_INTEGER_DIVIDE_BY_ZERO __constant_cpu_to_le32(0xC0000094)
-#define STATUS_INTEGER_OVERFLOW __constant_cpu_to_le32(0xC0000095)
-#define STATUS_PRIVILEGED_INSTRUCTION __constant_cpu_to_le32(0xC0000096)
-#define STATUS_TOO_MANY_PAGING_FILES __constant_cpu_to_le32(0xC0000097)
-#define STATUS_FILE_INVALID __constant_cpu_to_le32(0xC0000098)
-#define STATUS_ALLOTTED_SPACE_EXCEEDED __constant_cpu_to_le32(0xC0000099)
-#define STATUS_INSUFFICIENT_RESOURCES __constant_cpu_to_le32(0xC000009A)
-#define STATUS_DFS_EXIT_PATH_FOUND __constant_cpu_to_le32(0xC000009B)
-#define STATUS_DEVICE_DATA_ERROR __constant_cpu_to_le32(0xC000009C)
-#define STATUS_DEVICE_NOT_CONNECTED __constant_cpu_to_le32(0xC000009D)
-#define STATUS_DEVICE_POWER_FAILURE __constant_cpu_to_le32(0xC000009E)
-#define STATUS_FREE_VM_NOT_AT_BASE __constant_cpu_to_le32(0xC000009F)
-#define STATUS_MEMORY_NOT_ALLOCATED __constant_cpu_to_le32(0xC00000A0)
-#define STATUS_WORKING_SET_QUOTA __constant_cpu_to_le32(0xC00000A1)
-#define STATUS_MEDIA_WRITE_PROTECTED __constant_cpu_to_le32(0xC00000A2)
-#define STATUS_DEVICE_NOT_READY __constant_cpu_to_le32(0xC00000A3)
-#define STATUS_INVALID_GROUP_ATTRIBUTES __constant_cpu_to_le32(0xC00000A4)
-#define STATUS_BAD_IMPERSONATION_LEVEL __constant_cpu_to_le32(0xC00000A5)
-#define STATUS_CANT_OPEN_ANONYMOUS __constant_cpu_to_le32(0xC00000A6)
-#define STATUS_BAD_VALIDATION_CLASS __constant_cpu_to_le32(0xC00000A7)
-#define STATUS_BAD_TOKEN_TYPE __constant_cpu_to_le32(0xC00000A8)
-#define STATUS_BAD_MASTER_BOOT_RECORD __constant_cpu_to_le32(0xC00000A9)
-#define STATUS_INSTRUCTION_MISALIGNMENT __constant_cpu_to_le32(0xC00000AA)
-#define STATUS_INSTANCE_NOT_AVAILABLE __constant_cpu_to_le32(0xC00000AB)
-#define STATUS_PIPE_NOT_AVAILABLE __constant_cpu_to_le32(0xC00000AC)
-#define STATUS_INVALID_PIPE_STATE __constant_cpu_to_le32(0xC00000AD)
-#define STATUS_PIPE_BUSY __constant_cpu_to_le32(0xC00000AE)
-#define STATUS_ILLEGAL_FUNCTION __constant_cpu_to_le32(0xC00000AF)
-#define STATUS_PIPE_DISCONNECTED __constant_cpu_to_le32(0xC00000B0)
-#define STATUS_PIPE_CLOSING __constant_cpu_to_le32(0xC00000B1)
-#define STATUS_PIPE_CONNECTED __constant_cpu_to_le32(0xC00000B2)
-#define STATUS_PIPE_LISTENING __constant_cpu_to_le32(0xC00000B3)
-#define STATUS_INVALID_READ_MODE __constant_cpu_to_le32(0xC00000B4)
-#define STATUS_IO_TIMEOUT __constant_cpu_to_le32(0xC00000B5)
-#define STATUS_FILE_FORCED_CLOSED __constant_cpu_to_le32(0xC00000B6)
-#define STATUS_PROFILING_NOT_STARTED __constant_cpu_to_le32(0xC00000B7)
-#define STATUS_PROFILING_NOT_STOPPED __constant_cpu_to_le32(0xC00000B8)
-#define STATUS_COULD_NOT_INTERPRET __constant_cpu_to_le32(0xC00000B9)
-#define STATUS_FILE_IS_A_DIRECTORY __constant_cpu_to_le32(0xC00000BA)
-#define STATUS_NOT_SUPPORTED __constant_cpu_to_le32(0xC00000BB)
-#define STATUS_REMOTE_NOT_LISTENING __constant_cpu_to_le32(0xC00000BC)
-#define STATUS_DUPLICATE_NAME __constant_cpu_to_le32(0xC00000BD)
-#define STATUS_BAD_NETWORK_PATH __constant_cpu_to_le32(0xC00000BE)
-#define STATUS_NETWORK_BUSY __constant_cpu_to_le32(0xC00000BF)
-#define STATUS_DEVICE_DOES_NOT_EXIST __constant_cpu_to_le32(0xC00000C0)
-#define STATUS_TOO_MANY_COMMANDS __constant_cpu_to_le32(0xC00000C1)
-#define STATUS_ADAPTER_HARDWARE_ERROR __constant_cpu_to_le32(0xC00000C2)
-#define STATUS_INVALID_NETWORK_RESPONSE __constant_cpu_to_le32(0xC00000C3)
-#define STATUS_UNEXPECTED_NETWORK_ERROR __constant_cpu_to_le32(0xC00000C4)
-#define STATUS_BAD_REMOTE_ADAPTER __constant_cpu_to_le32(0xC00000C5)
-#define STATUS_PRINT_QUEUE_FULL __constant_cpu_to_le32(0xC00000C6)
-#define STATUS_NO_SPOOL_SPACE __constant_cpu_to_le32(0xC00000C7)
-#define STATUS_PRINT_CANCELLED __constant_cpu_to_le32(0xC00000C8)
-#define STATUS_NETWORK_NAME_DELETED __constant_cpu_to_le32(0xC00000C9)
-#define STATUS_NETWORK_ACCESS_DENIED __constant_cpu_to_le32(0xC00000CA)
-#define STATUS_BAD_DEVICE_TYPE __constant_cpu_to_le32(0xC00000CB)
-#define STATUS_BAD_NETWORK_NAME __constant_cpu_to_le32(0xC00000CC)
-#define STATUS_TOO_MANY_NAMES __constant_cpu_to_le32(0xC00000CD)
-#define STATUS_TOO_MANY_SESSIONS __constant_cpu_to_le32(0xC00000CE)
-#define STATUS_SHARING_PAUSED __constant_cpu_to_le32(0xC00000CF)
-#define STATUS_REQUEST_NOT_ACCEPTED __constant_cpu_to_le32(0xC00000D0)
-#define STATUS_REDIRECTOR_PAUSED __constant_cpu_to_le32(0xC00000D1)
-#define STATUS_NET_WRITE_FAULT __constant_cpu_to_le32(0xC00000D2)
-#define STATUS_PROFILING_AT_LIMIT __constant_cpu_to_le32(0xC00000D3)
-#define STATUS_NOT_SAME_DEVICE __constant_cpu_to_le32(0xC00000D4)
-#define STATUS_FILE_RENAMED __constant_cpu_to_le32(0xC00000D5)
-#define STATUS_VIRTUAL_CIRCUIT_CLOSED __constant_cpu_to_le32(0xC00000D6)
-#define STATUS_NO_SECURITY_ON_OBJECT __constant_cpu_to_le32(0xC00000D7)
-#define STATUS_CANT_WAIT __constant_cpu_to_le32(0xC00000D8)
-#define STATUS_PIPE_EMPTY __constant_cpu_to_le32(0xC00000D9)
-#define STATUS_CANT_ACCESS_DOMAIN_INFO __constant_cpu_to_le32(0xC00000DA)
-#define STATUS_CANT_TERMINATE_SELF __constant_cpu_to_le32(0xC00000DB)
-#define STATUS_INVALID_SERVER_STATE __constant_cpu_to_le32(0xC00000DC)
-#define STATUS_INVALID_DOMAIN_STATE __constant_cpu_to_le32(0xC00000DD)
-#define STATUS_INVALID_DOMAIN_ROLE __constant_cpu_to_le32(0xC00000DE)
-#define STATUS_NO_SUCH_DOMAIN __constant_cpu_to_le32(0xC00000DF)
-#define STATUS_DOMAIN_EXISTS __constant_cpu_to_le32(0xC00000E0)
-#define STATUS_DOMAIN_LIMIT_EXCEEDED __constant_cpu_to_le32(0xC00000E1)
-#define STATUS_OPLOCK_NOT_GRANTED __constant_cpu_to_le32(0xC00000E2)
-#define STATUS_INVALID_OPLOCK_PROTOCOL __constant_cpu_to_le32(0xC00000E3)
-#define STATUS_INTERNAL_DB_CORRUPTION __constant_cpu_to_le32(0xC00000E4)
-#define STATUS_INTERNAL_ERROR __constant_cpu_to_le32(0xC00000E5)
-#define STATUS_GENERIC_NOT_MAPPED __constant_cpu_to_le32(0xC00000E6)
-#define STATUS_BAD_DESCRIPTOR_FORMAT __constant_cpu_to_le32(0xC00000E7)
-#define STATUS_INVALID_USER_BUFFER __constant_cpu_to_le32(0xC00000E8)
-#define STATUS_UNEXPECTED_IO_ERROR __constant_cpu_to_le32(0xC00000E9)
-#define STATUS_UNEXPECTED_MM_CREATE_ERR __constant_cpu_to_le32(0xC00000EA)
-#define STATUS_UNEXPECTED_MM_MAP_ERROR __constant_cpu_to_le32(0xC00000EB)
-#define STATUS_UNEXPECTED_MM_EXTEND_ERR __constant_cpu_to_le32(0xC00000EC)
-#define STATUS_NOT_LOGON_PROCESS __constant_cpu_to_le32(0xC00000ED)
-#define STATUS_LOGON_SESSION_EXISTS __constant_cpu_to_le32(0xC00000EE)
-#define STATUS_INVALID_PARAMETER_1 __constant_cpu_to_le32(0xC00000EF)
-#define STATUS_INVALID_PARAMETER_2 __constant_cpu_to_le32(0xC00000F0)
-#define STATUS_INVALID_PARAMETER_3 __constant_cpu_to_le32(0xC00000F1)
-#define STATUS_INVALID_PARAMETER_4 __constant_cpu_to_le32(0xC00000F2)
-#define STATUS_INVALID_PARAMETER_5 __constant_cpu_to_le32(0xC00000F3)
-#define STATUS_INVALID_PARAMETER_6 __constant_cpu_to_le32(0xC00000F4)
-#define STATUS_INVALID_PARAMETER_7 __constant_cpu_to_le32(0xC00000F5)
-#define STATUS_INVALID_PARAMETER_8 __constant_cpu_to_le32(0xC00000F6)
-#define STATUS_INVALID_PARAMETER_9 __constant_cpu_to_le32(0xC00000F7)
-#define STATUS_INVALID_PARAMETER_10 __constant_cpu_to_le32(0xC00000F8)
-#define STATUS_INVALID_PARAMETER_11 __constant_cpu_to_le32(0xC00000F9)
-#define STATUS_INVALID_PARAMETER_12 __constant_cpu_to_le32(0xC00000FA)
-#define STATUS_REDIRECTOR_NOT_STARTED __constant_cpu_to_le32(0xC00000FB)
-#define STATUS_REDIRECTOR_STARTED __constant_cpu_to_le32(0xC00000FC)
-#define STATUS_STACK_OVERFLOW __constant_cpu_to_le32(0xC00000FD)
-#define STATUS_NO_SUCH_PACKAGE __constant_cpu_to_le32(0xC00000FE)
-#define STATUS_BAD_FUNCTION_TABLE __constant_cpu_to_le32(0xC00000FF)
-#define STATUS_VARIABLE_NOT_FOUND __constant_cpu_to_le32(0xC0000100)
-#define STATUS_DIRECTORY_NOT_EMPTY __constant_cpu_to_le32(0xC0000101)
-#define STATUS_FILE_CORRUPT_ERROR __constant_cpu_to_le32(0xC0000102)
-#define STATUS_NOT_A_DIRECTORY __constant_cpu_to_le32(0xC0000103)
-#define STATUS_BAD_LOGON_SESSION_STATE __constant_cpu_to_le32(0xC0000104)
-#define STATUS_LOGON_SESSION_COLLISION __constant_cpu_to_le32(0xC0000105)
-#define STATUS_NAME_TOO_LONG __constant_cpu_to_le32(0xC0000106)
-#define STATUS_FILES_OPEN __constant_cpu_to_le32(0xC0000107)
-#define STATUS_CONNECTION_IN_USE __constant_cpu_to_le32(0xC0000108)
-#define STATUS_MESSAGE_NOT_FOUND __constant_cpu_to_le32(0xC0000109)
-#define STATUS_PROCESS_IS_TERMINATING __constant_cpu_to_le32(0xC000010A)
-#define STATUS_INVALID_LOGON_TYPE __constant_cpu_to_le32(0xC000010B)
-#define STATUS_NO_GUID_TRANSLATION __constant_cpu_to_le32(0xC000010C)
-#define STATUS_CANNOT_IMPERSONATE __constant_cpu_to_le32(0xC000010D)
-#define STATUS_IMAGE_ALREADY_LOADED __constant_cpu_to_le32(0xC000010E)
-#define STATUS_ABIOS_NOT_PRESENT __constant_cpu_to_le32(0xC000010F)
-#define STATUS_ABIOS_LID_NOT_EXIST __constant_cpu_to_le32(0xC0000110)
-#define STATUS_ABIOS_LID_ALREADY_OWNED __constant_cpu_to_le32(0xC0000111)
-#define STATUS_ABIOS_NOT_LID_OWNER __constant_cpu_to_le32(0xC0000112)
-#define STATUS_ABIOS_INVALID_COMMAND __constant_cpu_to_le32(0xC0000113)
-#define STATUS_ABIOS_INVALID_LID __constant_cpu_to_le32(0xC0000114)
-#define STATUS_ABIOS_SELECTOR_NOT_AVAILABLE __constant_cpu_to_le32(0xC0000115)
-#define STATUS_ABIOS_INVALID_SELECTOR __constant_cpu_to_le32(0xC0000116)
-#define STATUS_NO_LDT __constant_cpu_to_le32(0xC0000117)
-#define STATUS_INVALID_LDT_SIZE __constant_cpu_to_le32(0xC0000118)
-#define STATUS_INVALID_LDT_OFFSET __constant_cpu_to_le32(0xC0000119)
-#define STATUS_INVALID_LDT_DESCRIPTOR __constant_cpu_to_le32(0xC000011A)
-#define STATUS_INVALID_IMAGE_NE_FORMAT __constant_cpu_to_le32(0xC000011B)
-#define STATUS_RXACT_INVALID_STATE __constant_cpu_to_le32(0xC000011C)
-#define STATUS_RXACT_COMMIT_FAILURE __constant_cpu_to_le32(0xC000011D)
-#define STATUS_MAPPED_FILE_SIZE_ZERO __constant_cpu_to_le32(0xC000011E)
-#define STATUS_TOO_MANY_OPENED_FILES __constant_cpu_to_le32(0xC000011F)
-#define STATUS_CANCELLED __constant_cpu_to_le32(0xC0000120)
-#define STATUS_CANNOT_DELETE __constant_cpu_to_le32(0xC0000121)
-#define STATUS_INVALID_COMPUTER_NAME __constant_cpu_to_le32(0xC0000122)
-#define STATUS_FILE_DELETED __constant_cpu_to_le32(0xC0000123)
-#define STATUS_SPECIAL_ACCOUNT __constant_cpu_to_le32(0xC0000124)
-#define STATUS_SPECIAL_GROUP __constant_cpu_to_le32(0xC0000125)
-#define STATUS_SPECIAL_USER __constant_cpu_to_le32(0xC0000126)
-#define STATUS_MEMBERS_PRIMARY_GROUP __constant_cpu_to_le32(0xC0000127)
-#define STATUS_FILE_CLOSED __constant_cpu_to_le32(0xC0000128)
-#define STATUS_TOO_MANY_THREADS __constant_cpu_to_le32(0xC0000129)
-#define STATUS_THREAD_NOT_IN_PROCESS __constant_cpu_to_le32(0xC000012A)
-#define STATUS_TOKEN_ALREADY_IN_USE __constant_cpu_to_le32(0xC000012B)
-#define STATUS_PAGEFILE_QUOTA_EXCEEDED __constant_cpu_to_le32(0xC000012C)
-#define STATUS_COMMITMENT_LIMIT __constant_cpu_to_le32(0xC000012D)
-#define STATUS_INVALID_IMAGE_LE_FORMAT __constant_cpu_to_le32(0xC000012E)
-#define STATUS_INVALID_IMAGE_NOT_MZ __constant_cpu_to_le32(0xC000012F)
-#define STATUS_INVALID_IMAGE_PROTECT __constant_cpu_to_le32(0xC0000130)
-#define STATUS_INVALID_IMAGE_WIN_16 __constant_cpu_to_le32(0xC0000131)
-#define STATUS_LOGON_SERVER_CONFLICT __constant_cpu_to_le32(0xC0000132)
-#define STATUS_TIME_DIFFERENCE_AT_DC __constant_cpu_to_le32(0xC0000133)
-#define STATUS_SYNCHRONIZATION_REQUIRED __constant_cpu_to_le32(0xC0000134)
-#define STATUS_DLL_NOT_FOUND __constant_cpu_to_le32(0xC0000135)
-#define STATUS_OPEN_FAILED __constant_cpu_to_le32(0xC0000136)
-#define STATUS_IO_PRIVILEGE_FAILED __constant_cpu_to_le32(0xC0000137)
-#define STATUS_ORDINAL_NOT_FOUND __constant_cpu_to_le32(0xC0000138)
-#define STATUS_ENTRYPOINT_NOT_FOUND __constant_cpu_to_le32(0xC0000139)
-#define STATUS_CONTROL_C_EXIT __constant_cpu_to_le32(0xC000013A)
-#define STATUS_LOCAL_DISCONNECT __constant_cpu_to_le32(0xC000013B)
-#define STATUS_REMOTE_DISCONNECT __constant_cpu_to_le32(0xC000013C)
-#define STATUS_REMOTE_RESOURCES __constant_cpu_to_le32(0xC000013D)
-#define STATUS_LINK_FAILED __constant_cpu_to_le32(0xC000013E)
-#define STATUS_LINK_TIMEOUT __constant_cpu_to_le32(0xC000013F)
-#define STATUS_INVALID_CONNECTION __constant_cpu_to_le32(0xC0000140)
-#define STATUS_INVALID_ADDRESS __constant_cpu_to_le32(0xC0000141)
-#define STATUS_DLL_INIT_FAILED __constant_cpu_to_le32(0xC0000142)
-#define STATUS_MISSING_SYSTEMFILE __constant_cpu_to_le32(0xC0000143)
-#define STATUS_UNHANDLED_EXCEPTION __constant_cpu_to_le32(0xC0000144)
-#define STATUS_APP_INIT_FAILURE __constant_cpu_to_le32(0xC0000145)
-#define STATUS_PAGEFILE_CREATE_FAILED __constant_cpu_to_le32(0xC0000146)
-#define STATUS_NO_PAGEFILE __constant_cpu_to_le32(0xC0000147)
-#define STATUS_INVALID_LEVEL __constant_cpu_to_le32(0xC0000148)
-#define STATUS_WRONG_PASSWORD_CORE __constant_cpu_to_le32(0xC0000149)
-#define STATUS_ILLEGAL_FLOAT_CONTEXT __constant_cpu_to_le32(0xC000014A)
-#define STATUS_PIPE_BROKEN __constant_cpu_to_le32(0xC000014B)
-#define STATUS_REGISTRY_CORRUPT __constant_cpu_to_le32(0xC000014C)
-#define STATUS_REGISTRY_IO_FAILED __constant_cpu_to_le32(0xC000014D)
-#define STATUS_NO_EVENT_PAIR __constant_cpu_to_le32(0xC000014E)
-#define STATUS_UNRECOGNIZED_VOLUME __constant_cpu_to_le32(0xC000014F)
-#define STATUS_SERIAL_NO_DEVICE_INITED __constant_cpu_to_le32(0xC0000150)
-#define STATUS_NO_SUCH_ALIAS __constant_cpu_to_le32(0xC0000151)
-#define STATUS_MEMBER_NOT_IN_ALIAS __constant_cpu_to_le32(0xC0000152)
-#define STATUS_MEMBER_IN_ALIAS __constant_cpu_to_le32(0xC0000153)
-#define STATUS_ALIAS_EXISTS __constant_cpu_to_le32(0xC0000154)
-#define STATUS_LOGON_NOT_GRANTED __constant_cpu_to_le32(0xC0000155)
-#define STATUS_TOO_MANY_SECRETS __constant_cpu_to_le32(0xC0000156)
-#define STATUS_SECRET_TOO_LONG __constant_cpu_to_le32(0xC0000157)
-#define STATUS_INTERNAL_DB_ERROR __constant_cpu_to_le32(0xC0000158)
-#define STATUS_FULLSCREEN_MODE __constant_cpu_to_le32(0xC0000159)
-#define STATUS_TOO_MANY_CONTEXT_IDS __constant_cpu_to_le32(0xC000015A)
-#define STATUS_LOGON_TYPE_NOT_GRANTED __constant_cpu_to_le32(0xC000015B)
-#define STATUS_NOT_REGISTRY_FILE __constant_cpu_to_le32(0xC000015C)
-#define STATUS_NT_CROSS_ENCRYPTION_REQUIRED __constant_cpu_to_le32(0xC000015D)
-#define STATUS_DOMAIN_CTRLR_CONFIG_ERROR __constant_cpu_to_le32(0xC000015E)
-#define STATUS_FT_MISSING_MEMBER __constant_cpu_to_le32(0xC000015F)
-#define STATUS_ILL_FORMED_SERVICE_ENTRY __constant_cpu_to_le32(0xC0000160)
-#define STATUS_ILLEGAL_CHARACTER __constant_cpu_to_le32(0xC0000161)
-#define STATUS_UNMAPPABLE_CHARACTER __constant_cpu_to_le32(0xC0000162)
-#define STATUS_UNDEFINED_CHARACTER __constant_cpu_to_le32(0xC0000163)
-#define STATUS_FLOPPY_VOLUME __constant_cpu_to_le32(0xC0000164)
-#define STATUS_FLOPPY_ID_MARK_NOT_FOUND __constant_cpu_to_le32(0xC0000165)
-#define STATUS_FLOPPY_WRONG_CYLINDER __constant_cpu_to_le32(0xC0000166)
-#define STATUS_FLOPPY_UNKNOWN_ERROR __constant_cpu_to_le32(0xC0000167)
-#define STATUS_FLOPPY_BAD_REGISTERS __constant_cpu_to_le32(0xC0000168)
-#define STATUS_DISK_RECALIBRATE_FAILED __constant_cpu_to_le32(0xC0000169)
-#define STATUS_DISK_OPERATION_FAILED __constant_cpu_to_le32(0xC000016A)
-#define STATUS_DISK_RESET_FAILED __constant_cpu_to_le32(0xC000016B)
-#define STATUS_SHARED_IRQ_BUSY __constant_cpu_to_le32(0xC000016C)
-#define STATUS_FT_ORPHANING __constant_cpu_to_le32(0xC000016D)
-#define STATUS_BIOS_FAILED_TO_CONNECT_INTERRUPT __constant_cpu_to_le32(0xC000016E)
-#define STATUS_PARTITION_FAILURE __constant_cpu_to_le32(0xC0000172)
-#define STATUS_INVALID_BLOCK_LENGTH __constant_cpu_to_le32(0xC0000173)
-#define STATUS_DEVICE_NOT_PARTITIONED __constant_cpu_to_le32(0xC0000174)
-#define STATUS_UNABLE_TO_LOCK_MEDIA __constant_cpu_to_le32(0xC0000175)
-#define STATUS_UNABLE_TO_UNLOAD_MEDIA __constant_cpu_to_le32(0xC0000176)
-#define STATUS_EOM_OVERFLOW __constant_cpu_to_le32(0xC0000177)
-#define STATUS_NO_MEDIA __constant_cpu_to_le32(0xC0000178)
-#define STATUS_NO_SUCH_MEMBER __constant_cpu_to_le32(0xC000017A)
-#define STATUS_INVALID_MEMBER __constant_cpu_to_le32(0xC000017B)
-#define STATUS_KEY_DELETED __constant_cpu_to_le32(0xC000017C)
-#define STATUS_NO_LOG_SPACE __constant_cpu_to_le32(0xC000017D)
-#define STATUS_TOO_MANY_SIDS __constant_cpu_to_le32(0xC000017E)
-#define STATUS_LM_CROSS_ENCRYPTION_REQUIRED __constant_cpu_to_le32(0xC000017F)
-#define STATUS_KEY_HAS_CHILDREN __constant_cpu_to_le32(0xC0000180)
-#define STATUS_CHILD_MUST_BE_VOLATILE __constant_cpu_to_le32(0xC0000181)
-#define STATUS_DEVICE_CONFIGURATION_ERROR __constant_cpu_to_le32(0xC0000182)
-#define STATUS_DRIVER_INTERNAL_ERROR __constant_cpu_to_le32(0xC0000183)
-#define STATUS_INVALID_DEVICE_STATE __constant_cpu_to_le32(0xC0000184)
-#define STATUS_IO_DEVICE_ERROR __constant_cpu_to_le32(0xC0000185)
-#define STATUS_DEVICE_PROTOCOL_ERROR __constant_cpu_to_le32(0xC0000186)
-#define STATUS_BACKUP_CONTROLLER __constant_cpu_to_le32(0xC0000187)
-#define STATUS_LOG_FILE_FULL __constant_cpu_to_le32(0xC0000188)
-#define STATUS_TOO_LATE __constant_cpu_to_le32(0xC0000189)
-#define STATUS_NO_TRUST_LSA_SECRET __constant_cpu_to_le32(0xC000018A)
-#define STATUS_NO_TRUST_SAM_ACCOUNT __constant_cpu_to_le32(0xC000018B)
-#define STATUS_TRUSTED_DOMAIN_FAILURE __constant_cpu_to_le32(0xC000018C)
-#define STATUS_TRUSTED_RELATIONSHIP_FAILURE __constant_cpu_to_le32(0xC000018D)
-#define STATUS_EVENTLOG_FILE_CORRUPT __constant_cpu_to_le32(0xC000018E)
-#define STATUS_EVENTLOG_CANT_START __constant_cpu_to_le32(0xC000018F)
-#define STATUS_TRUST_FAILURE __constant_cpu_to_le32(0xC0000190)
-#define STATUS_MUTANT_LIMIT_EXCEEDED __constant_cpu_to_le32(0xC0000191)
-#define STATUS_NETLOGON_NOT_STARTED __constant_cpu_to_le32(0xC0000192)
-#define STATUS_ACCOUNT_EXPIRED __constant_cpu_to_le32(0xC0000193)
-#define STATUS_POSSIBLE_DEADLOCK __constant_cpu_to_le32(0xC0000194)
-#define STATUS_NETWORK_CREDENTIAL_CONFLICT __constant_cpu_to_le32(0xC0000195)
-#define STATUS_REMOTE_SESSION_LIMIT __constant_cpu_to_le32(0xC0000196)
-#define STATUS_EVENTLOG_FILE_CHANGED __constant_cpu_to_le32(0xC0000197)
-#define STATUS_NOLOGON_INTERDOMAIN_TRUST_ACCOUNT __constant_cpu_to_le32(0xC0000198)
-#define STATUS_NOLOGON_WORKSTATION_TRUST_ACCOUNT __constant_cpu_to_le32(0xC0000199)
-#define STATUS_NOLOGON_SERVER_TRUST_ACCOUNT __constant_cpu_to_le32(0xC000019A)
-#define STATUS_DOMAIN_TRUST_INCONSISTENT __constant_cpu_to_le32(0xC000019B)
-#define STATUS_FS_DRIVER_REQUIRED __constant_cpu_to_le32(0xC000019C)
-#define STATUS_IMAGE_ALREADY_LOADED_AS_DLL __constant_cpu_to_le32(0xC000019D)
-#define STATUS_NETWORK_OPEN_RESTRICTION __constant_cpu_to_le32(0xC0000201)
-#define STATUS_NO_USER_SESSION_KEY __constant_cpu_to_le32(0xC0000202)
-#define STATUS_USER_SESSION_DELETED __constant_cpu_to_le32(0xC0000203)
-#define STATUS_RESOURCE_LANG_NOT_FOUND __constant_cpu_to_le32(0xC0000204)
-#define STATUS_INSUFF_SERVER_RESOURCES __constant_cpu_to_le32(0xC0000205)
-#define STATUS_INVALID_BUFFER_SIZE __constant_cpu_to_le32(0xC0000206)
-#define STATUS_INVALID_ADDRESS_COMPONENT __constant_cpu_to_le32(0xC0000207)
-#define STATUS_INVALID_ADDRESS_WILDCARD __constant_cpu_to_le32(0xC0000208)
-#define STATUS_TOO_MANY_ADDRESSES __constant_cpu_to_le32(0xC0000209)
-#define STATUS_ADDRESS_ALREADY_EXISTS __constant_cpu_to_le32(0xC000020A)
-#define STATUS_ADDRESS_CLOSED __constant_cpu_to_le32(0xC000020B)
-#define STATUS_CONNECTION_DISCONNECTED __constant_cpu_to_le32(0xC000020C)
-#define STATUS_CONNECTION_RESET __constant_cpu_to_le32(0xC000020D)
-#define STATUS_TOO_MANY_NODES __constant_cpu_to_le32(0xC000020E)
-#define STATUS_TRANSACTION_ABORTED __constant_cpu_to_le32(0xC000020F)
-#define STATUS_TRANSACTION_TIMED_OUT __constant_cpu_to_le32(0xC0000210)
-#define STATUS_TRANSACTION_NO_RELEASE __constant_cpu_to_le32(0xC0000211)
-#define STATUS_TRANSACTION_NO_MATCH __constant_cpu_to_le32(0xC0000212)
-#define STATUS_TRANSACTION_RESPONDED __constant_cpu_to_le32(0xC0000213)
-#define STATUS_TRANSACTION_INVALID_ID __constant_cpu_to_le32(0xC0000214)
-#define STATUS_TRANSACTION_INVALID_TYPE __constant_cpu_to_le32(0xC0000215)
-#define STATUS_NOT_SERVER_SESSION __constant_cpu_to_le32(0xC0000216)
-#define STATUS_NOT_CLIENT_SESSION __constant_cpu_to_le32(0xC0000217)
-#define STATUS_CANNOT_LOAD_REGISTRY_FILE __constant_cpu_to_le32(0xC0000218)
-#define STATUS_DEBUG_ATTACH_FAILED __constant_cpu_to_le32(0xC0000219)
-#define STATUS_SYSTEM_PROCESS_TERMINATED __constant_cpu_to_le32(0xC000021A)
-#define STATUS_DATA_NOT_ACCEPTED __constant_cpu_to_le32(0xC000021B)
-#define STATUS_NO_BROWSER_SERVERS_FOUND __constant_cpu_to_le32(0xC000021C)
-#define STATUS_VDM_HARD_ERROR __constant_cpu_to_le32(0xC000021D)
-#define STATUS_DRIVER_CANCEL_TIMEOUT __constant_cpu_to_le32(0xC000021E)
-#define STATUS_REPLY_MESSAGE_MISMATCH __constant_cpu_to_le32(0xC000021F)
-#define STATUS_MAPPED_ALIGNMENT __constant_cpu_to_le32(0xC0000220)
-#define STATUS_IMAGE_CHECKSUM_MISMATCH __constant_cpu_to_le32(0xC0000221)
-#define STATUS_LOST_WRITEBEHIND_DATA __constant_cpu_to_le32(0xC0000222)
-#define STATUS_CLIENT_SERVER_PARAMETERS_INVALID __constant_cpu_to_le32(0xC0000223)
-#define STATUS_PASSWORD_MUST_CHANGE __constant_cpu_to_le32(0xC0000224)
-#define STATUS_NOT_FOUND __constant_cpu_to_le32(0xC0000225)
-#define STATUS_NOT_TINY_STREAM __constant_cpu_to_le32(0xC0000226)
-#define STATUS_RECOVERY_FAILURE __constant_cpu_to_le32(0xC0000227)
-#define STATUS_STACK_OVERFLOW_READ __constant_cpu_to_le32(0xC0000228)
-#define STATUS_FAIL_CHECK __constant_cpu_to_le32(0xC0000229)
-#define STATUS_DUPLICATE_OBJECTID __constant_cpu_to_le32(0xC000022A)
-#define STATUS_OBJECTID_EXISTS __constant_cpu_to_le32(0xC000022B)
-#define STATUS_CONVERT_TO_LARGE __constant_cpu_to_le32(0xC000022C)
-#define STATUS_RETRY __constant_cpu_to_le32(0xC000022D)
-#define STATUS_FOUND_OUT_OF_SCOPE __constant_cpu_to_le32(0xC000022E)
-#define STATUS_ALLOCATE_BUCKET __constant_cpu_to_le32(0xC000022F)
-#define STATUS_PROPSET_NOT_FOUND __constant_cpu_to_le32(0xC0000230)
-#define STATUS_MARSHALL_OVERFLOW __constant_cpu_to_le32(0xC0000231)
-#define STATUS_INVALID_VARIANT __constant_cpu_to_le32(0xC0000232)
-#define STATUS_DOMAIN_CONTROLLER_NOT_FOUND __constant_cpu_to_le32(0xC0000233)
-#define STATUS_ACCOUNT_LOCKED_OUT __constant_cpu_to_le32(0xC0000234)
-#define STATUS_HANDLE_NOT_CLOSABLE __constant_cpu_to_le32(0xC0000235)
-#define STATUS_CONNECTION_REFUSED __constant_cpu_to_le32(0xC0000236)
-#define STATUS_GRACEFUL_DISCONNECT __constant_cpu_to_le32(0xC0000237)
-#define STATUS_ADDRESS_ALREADY_ASSOCIATED __constant_cpu_to_le32(0xC0000238)
-#define STATUS_ADDRESS_NOT_ASSOCIATED __constant_cpu_to_le32(0xC0000239)
-#define STATUS_CONNECTION_INVALID __constant_cpu_to_le32(0xC000023A)
-#define STATUS_CONNECTION_ACTIVE __constant_cpu_to_le32(0xC000023B)
-#define STATUS_NETWORK_UNREACHABLE __constant_cpu_to_le32(0xC000023C)
-#define STATUS_HOST_UNREACHABLE __constant_cpu_to_le32(0xC000023D)
-#define STATUS_PROTOCOL_UNREACHABLE __constant_cpu_to_le32(0xC000023E)
-#define STATUS_PORT_UNREACHABLE __constant_cpu_to_le32(0xC000023F)
-#define STATUS_REQUEST_ABORTED __constant_cpu_to_le32(0xC0000240)
-#define STATUS_CONNECTION_ABORTED __constant_cpu_to_le32(0xC0000241)
-#define STATUS_BAD_COMPRESSION_BUFFER __constant_cpu_to_le32(0xC0000242)
-#define STATUS_USER_MAPPED_FILE __constant_cpu_to_le32(0xC0000243)
-#define STATUS_AUDIT_FAILED __constant_cpu_to_le32(0xC0000244)
-#define STATUS_TIMER_RESOLUTION_NOT_SET __constant_cpu_to_le32(0xC0000245)
-#define STATUS_CONNECTION_COUNT_LIMIT __constant_cpu_to_le32(0xC0000246)
-#define STATUS_LOGIN_TIME_RESTRICTION __constant_cpu_to_le32(0xC0000247)
-#define STATUS_LOGIN_WKSTA_RESTRICTION __constant_cpu_to_le32(0xC0000248)
-#define STATUS_IMAGE_MP_UP_MISMATCH __constant_cpu_to_le32(0xC0000249)
-#define STATUS_INSUFFICIENT_LOGON_INFO __constant_cpu_to_le32(0xC0000250)
-#define STATUS_BAD_DLL_ENTRYPOINT __constant_cpu_to_le32(0xC0000251)
-#define STATUS_BAD_SERVICE_ENTRYPOINT __constant_cpu_to_le32(0xC0000252)
-#define STATUS_LPC_REPLY_LOST __constant_cpu_to_le32(0xC0000253)
-#define STATUS_IP_ADDRESS_CONFLICT1 __constant_cpu_to_le32(0xC0000254)
-#define STATUS_IP_ADDRESS_CONFLICT2 __constant_cpu_to_le32(0xC0000255)
-#define STATUS_REGISTRY_QUOTA_LIMIT __constant_cpu_to_le32(0xC0000256)
-#define STATUS_PATH_NOT_COVERED __constant_cpu_to_le32(0xC0000257)
-#define STATUS_NO_CALLBACK_ACTIVE __constant_cpu_to_le32(0xC0000258)
-#define STATUS_LICENSE_QUOTA_EXCEEDED __constant_cpu_to_le32(0xC0000259)
-#define STATUS_PWD_TOO_SHORT __constant_cpu_to_le32(0xC000025A)
-#define STATUS_PWD_TOO_RECENT __constant_cpu_to_le32(0xC000025B)
-#define STATUS_PWD_HISTORY_CONFLICT __constant_cpu_to_le32(0xC000025C)
-#define STATUS_PLUGPLAY_NO_DEVICE __constant_cpu_to_le32(0xC000025E)
-#define STATUS_UNSUPPORTED_COMPRESSION __constant_cpu_to_le32(0xC000025F)
-#define STATUS_INVALID_HW_PROFILE __constant_cpu_to_le32(0xC0000260)
-#define STATUS_INVALID_PLUGPLAY_DEVICE_PATH __constant_cpu_to_le32(0xC0000261)
-#define STATUS_DRIVER_ORDINAL_NOT_FOUND __constant_cpu_to_le32(0xC0000262)
-#define STATUS_DRIVER_ENTRYPOINT_NOT_FOUND __constant_cpu_to_le32(0xC0000263)
-#define STATUS_RESOURCE_NOT_OWNED __constant_cpu_to_le32(0xC0000264)
-#define STATUS_TOO_MANY_LINKS __constant_cpu_to_le32(0xC0000265)
-#define STATUS_QUOTA_LIST_INCONSISTENT __constant_cpu_to_le32(0xC0000266)
-#define STATUS_FILE_IS_OFFLINE __constant_cpu_to_le32(0xC0000267)
-#define STATUS_EVALUATION_EXPIRATION __constant_cpu_to_le32(0xC0000268)
-#define STATUS_ILLEGAL_DLL_RELOCATION __constant_cpu_to_le32(0xC0000269)
-#define STATUS_LICENSE_VIOLATION __constant_cpu_to_le32(0xC000026A)
-#define STATUS_DLL_INIT_FAILED_LOGOFF __constant_cpu_to_le32(0xC000026B)
-#define STATUS_DRIVER_UNABLE_TO_LOAD __constant_cpu_to_le32(0xC000026C)
-#define STATUS_DFS_UNAVAILABLE __constant_cpu_to_le32(0xC000026D)
-#define STATUS_VOLUME_DISMOUNTED __constant_cpu_to_le32(0xC000026E)
-#define STATUS_WX86_INTERNAL_ERROR __constant_cpu_to_le32(0xC000026F)
-#define STATUS_WX86_FLOAT_STACK_CHECK __constant_cpu_to_le32(0xC0000270)
-#define STATUS_VALIDATE_CONTINUE __constant_cpu_to_le32(0xC0000271)
-#define STATUS_NO_MATCH __constant_cpu_to_le32(0xC0000272)
-#define STATUS_NO_MORE_MATCHES __constant_cpu_to_le32(0xC0000273)
-#define STATUS_NOT_A_REPARSE_POINT __constant_cpu_to_le32(0xC0000275)
-#define STATUS_IO_REPARSE_TAG_INVALID __constant_cpu_to_le32(0xC0000276)
-#define STATUS_IO_REPARSE_TAG_MISMATCH __constant_cpu_to_le32(0xC0000277)
-#define STATUS_IO_REPARSE_DATA_INVALID __constant_cpu_to_le32(0xC0000278)
-#define STATUS_IO_REPARSE_TAG_NOT_HANDLED __constant_cpu_to_le32(0xC0000279)
-#define STATUS_REPARSE_POINT_NOT_RESOLVED __constant_cpu_to_le32(0xC0000280)
-#define STATUS_DIRECTORY_IS_A_REPARSE_POINT __constant_cpu_to_le32(0xC0000281)
-#define STATUS_RANGE_LIST_CONFLICT __constant_cpu_to_le32(0xC0000282)
-#define STATUS_SOURCE_ELEMENT_EMPTY __constant_cpu_to_le32(0xC0000283)
-#define STATUS_DESTINATION_ELEMENT_FULL __constant_cpu_to_le32(0xC0000284)
-#define STATUS_ILLEGAL_ELEMENT_ADDRESS __constant_cpu_to_le32(0xC0000285)
-#define STATUS_MAGAZINE_NOT_PRESENT __constant_cpu_to_le32(0xC0000286)
-#define STATUS_REINITIALIZATION_NEEDED __constant_cpu_to_le32(0xC0000287)
-#define STATUS_ENCRYPTION_FAILED __constant_cpu_to_le32(0xC000028A)
-#define STATUS_DECRYPTION_FAILED __constant_cpu_to_le32(0xC000028B)
-#define STATUS_RANGE_NOT_FOUND __constant_cpu_to_le32(0xC000028C)
-#define STATUS_NO_RECOVERY_POLICY __constant_cpu_to_le32(0xC000028D)
-#define STATUS_NO_EFS __constant_cpu_to_le32(0xC000028E)
-#define STATUS_WRONG_EFS __constant_cpu_to_le32(0xC000028F)
-#define STATUS_NO_USER_KEYS __constant_cpu_to_le32(0xC0000290)
-#define STATUS_FILE_NOT_ENCRYPTED __constant_cpu_to_le32(0xC0000291)
-#define STATUS_NOT_EXPORT_FORMAT __constant_cpu_to_le32(0xC0000292)
-#define STATUS_FILE_ENCRYPTED __constant_cpu_to_le32(0xC0000293)
-#define STATUS_WMI_GUID_NOT_FOUND __constant_cpu_to_le32(0xC0000295)
-#define STATUS_WMI_INSTANCE_NOT_FOUND __constant_cpu_to_le32(0xC0000296)
-#define STATUS_WMI_ITEMID_NOT_FOUND __constant_cpu_to_le32(0xC0000297)
-#define STATUS_WMI_TRY_AGAIN __constant_cpu_to_le32(0xC0000298)
-#define STATUS_SHARED_POLICY __constant_cpu_to_le32(0xC0000299)
-#define STATUS_POLICY_OBJECT_NOT_FOUND __constant_cpu_to_le32(0xC000029A)
-#define STATUS_POLICY_ONLY_IN_DS __constant_cpu_to_le32(0xC000029B)
-#define STATUS_VOLUME_NOT_UPGRADED __constant_cpu_to_le32(0xC000029C)
-#define STATUS_REMOTE_STORAGE_NOT_ACTIVE __constant_cpu_to_le32(0xC000029D)
-#define STATUS_REMOTE_STORAGE_MEDIA_ERROR __constant_cpu_to_le32(0xC000029E)
-#define STATUS_NO_TRACKING_SERVICE __constant_cpu_to_le32(0xC000029F)
-#define STATUS_SERVER_SID_MISMATCH __constant_cpu_to_le32(0xC00002A0)
-#define STATUS_DS_NO_ATTRIBUTE_OR_VALUE __constant_cpu_to_le32(0xC00002A1)
-#define STATUS_DS_INVALID_ATTRIBUTE_SYNTAX __constant_cpu_to_le32(0xC00002A2)
-#define STATUS_DS_ATTRIBUTE_TYPE_UNDEFINED __constant_cpu_to_le32(0xC00002A3)
-#define STATUS_DS_ATTRIBUTE_OR_VALUE_EXISTS __constant_cpu_to_le32(0xC00002A4)
-#define STATUS_DS_BUSY __constant_cpu_to_le32(0xC00002A5)
-#define STATUS_DS_UNAVAILABLE __constant_cpu_to_le32(0xC00002A6)
-#define STATUS_DS_NO_RIDS_ALLOCATED __constant_cpu_to_le32(0xC00002A7)
-#define STATUS_DS_NO_MORE_RIDS __constant_cpu_to_le32(0xC00002A8)
-#define STATUS_DS_INCORRECT_ROLE_OWNER __constant_cpu_to_le32(0xC00002A9)
-#define STATUS_DS_RIDMGR_INIT_ERROR __constant_cpu_to_le32(0xC00002AA)
-#define STATUS_DS_OBJ_CLASS_VIOLATION __constant_cpu_to_le32(0xC00002AB)
-#define STATUS_DS_CANT_ON_NON_LEAF __constant_cpu_to_le32(0xC00002AC)
-#define STATUS_DS_CANT_ON_RDN __constant_cpu_to_le32(0xC00002AD)
-#define STATUS_DS_CANT_MOD_OBJ_CLASS __constant_cpu_to_le32(0xC00002AE)
-#define STATUS_DS_CROSS_DOM_MOVE_FAILED __constant_cpu_to_le32(0xC00002AF)
-#define STATUS_DS_GC_NOT_AVAILABLE __constant_cpu_to_le32(0xC00002B0)
-#define STATUS_DIRECTORY_SERVICE_REQUIRED __constant_cpu_to_le32(0xC00002B1)
-#define STATUS_REPARSE_ATTRIBUTE_CONFLICT __constant_cpu_to_le32(0xC00002B2)
-#define STATUS_CANT_ENABLE_DENY_ONLY __constant_cpu_to_le32(0xC00002B3)
-#define STATUS_FLOAT_MULTIPLE_FAULTS __constant_cpu_to_le32(0xC00002B4)
-#define STATUS_FLOAT_MULTIPLE_TRAPS __constant_cpu_to_le32(0xC00002B5)
-#define STATUS_DEVICE_REMOVED __constant_cpu_to_le32(0xC00002B6)
-#define STATUS_JOURNAL_DELETE_IN_PROGRESS __constant_cpu_to_le32(0xC00002B7)
-#define STATUS_JOURNAL_NOT_ACTIVE __constant_cpu_to_le32(0xC00002B8)
-#define STATUS_NOINTERFACE __constant_cpu_to_le32(0xC00002B9)
-#define STATUS_DS_ADMIN_LIMIT_EXCEEDED __constant_cpu_to_le32(0xC00002C1)
-#define STATUS_DRIVER_FAILED_SLEEP __constant_cpu_to_le32(0xC00002C2)
-#define STATUS_MUTUAL_AUTHENTICATION_FAILED __constant_cpu_to_le32(0xC00002C3)
-#define STATUS_CORRUPT_SYSTEM_FILE __constant_cpu_to_le32(0xC00002C4)
-#define STATUS_DATATYPE_MISALIGNMENT_ERROR __constant_cpu_to_le32(0xC00002C5)
-#define STATUS_WMI_READ_ONLY __constant_cpu_to_le32(0xC00002C6)
-#define STATUS_WMI_SET_FAILURE __constant_cpu_to_le32(0xC00002C7)
-#define STATUS_COMMITMENT_MINIMUM __constant_cpu_to_le32(0xC00002C8)
-#define STATUS_REG_NAT_CONSUMPTION __constant_cpu_to_le32(0xC00002C9)
-#define STATUS_TRANSPORT_FULL __constant_cpu_to_le32(0xC00002CA)
-#define STATUS_DS_SAM_INIT_FAILURE __constant_cpu_to_le32(0xC00002CB)
-#define STATUS_ONLY_IF_CONNECTED __constant_cpu_to_le32(0xC00002CC)
-#define STATUS_DS_SENSITIVE_GROUP_VIOLATION __constant_cpu_to_le32(0xC00002CD)
-#define STATUS_PNP_RESTART_ENUMERATION __constant_cpu_to_le32(0xC00002CE)
-#define STATUS_JOURNAL_ENTRY_DELETED __constant_cpu_to_le32(0xC00002CF)
-#define STATUS_DS_CANT_MOD_PRIMARYGROUPID __constant_cpu_to_le32(0xC00002D0)
-#define STATUS_SYSTEM_IMAGE_BAD_SIGNATURE __constant_cpu_to_le32(0xC00002D1)
-#define STATUS_PNP_REBOOT_REQUIRED __constant_cpu_to_le32(0xC00002D2)
-#define STATUS_POWER_STATE_INVALID __constant_cpu_to_le32(0xC00002D3)
-#define STATUS_DS_INVALID_GROUP_TYPE __constant_cpu_to_le32(0xC00002D4)
-#define STATUS_DS_NO_NEST_GLOBALGROUP_IN_MIXEDDOMAIN __constant_cpu_to_le32(0xC00002D5)
-#define STATUS_DS_NO_NEST_LOCALGROUP_IN_MIXEDDOMAIN __constant_cpu_to_le32(0xC00002D6)
-#define STATUS_DS_GLOBAL_CANT_HAVE_LOCAL_MEMBER __constant_cpu_to_le32(0xC00002D7)
-#define STATUS_DS_GLOBAL_CANT_HAVE_UNIVERSAL_MEMBER __constant_cpu_to_le32(0xC00002D8)
-#define STATUS_DS_UNIVERSAL_CANT_HAVE_LOCAL_MEMBER __constant_cpu_to_le32(0xC00002D9)
-#define STATUS_DS_GLOBAL_CANT_HAVE_CROSSDOMAIN_MEMBER __constant_cpu_to_le32(0xC00002DA)
-#define STATUS_DS_LOCAL_CANT_HAVE_CROSSDOMAIN_LOCAL_MEMBER __constant_cpu_to_le32(0xC00002DB)
-#define STATUS_DS_HAVE_PRIMARY_MEMBERS __constant_cpu_to_le32(0xC00002DC)
-#define STATUS_WMI_NOT_SUPPORTED __constant_cpu_to_le32(0xC00002DD)
-#define STATUS_INSUFFICIENT_POWER __constant_cpu_to_le32(0xC00002DE)
-#define STATUS_SAM_NEED_BOOTKEY_PASSWORD __constant_cpu_to_le32(0xC00002DF)
-#define STATUS_SAM_NEED_BOOTKEY_FLOPPY __constant_cpu_to_le32(0xC00002E0)
-#define STATUS_DS_CANT_START __constant_cpu_to_le32(0xC00002E1)
-#define STATUS_DS_INIT_FAILURE __constant_cpu_to_le32(0xC00002E2)
-#define STATUS_SAM_INIT_FAILURE __constant_cpu_to_le32(0xC00002E3)
-#define STATUS_DS_GC_REQUIRED __constant_cpu_to_le32(0xC00002E4)
-#define STATUS_DS_LOCAL_MEMBER_OF_LOCAL_ONLY __constant_cpu_to_le32(0xC00002E5)
-#define STATUS_DS_NO_FPO_IN_UNIVERSAL_GROUPS __constant_cpu_to_le32(0xC00002E6)
-#define STATUS_DS_MACHINE_ACCOUNT_QUOTA_EXCEEDED __constant_cpu_to_le32(0xC00002E7)
-#define STATUS_MULTIPLE_FAULT_VIOLATION __constant_cpu_to_le32(0xC00002E8)
-#define STATUS_CURRENT_DOMAIN_NOT_ALLOWED __constant_cpu_to_le32(0xC00002E9)
-#define STATUS_CANNOT_MAKE __constant_cpu_to_le32(0xC00002EA)
-#define STATUS_SYSTEM_SHUTDOWN __constant_cpu_to_le32(0xC00002EB)
-#define STATUS_DS_INIT_FAILURE_CONSOLE __constant_cpu_to_le32(0xC00002EC)
-#define STATUS_DS_SAM_INIT_FAILURE_CONSOLE __constant_cpu_to_le32(0xC00002ED)
-#define STATUS_UNFINISHED_CONTEXT_DELETED __constant_cpu_to_le32(0xC00002EE)
-#define STATUS_NO_TGT_REPLY __constant_cpu_to_le32(0xC00002EF)
-#define STATUS_OBJECTID_NOT_FOUND __constant_cpu_to_le32(0xC00002F0)
-#define STATUS_NO_IP_ADDRESSES __constant_cpu_to_le32(0xC00002F1)
-#define STATUS_WRONG_CREDENTIAL_HANDLE __constant_cpu_to_le32(0xC00002F2)
-#define STATUS_CRYPTO_SYSTEM_INVALID __constant_cpu_to_le32(0xC00002F3)
-#define STATUS_MAX_REFERRALS_EXCEEDED __constant_cpu_to_le32(0xC00002F4)
-#define STATUS_MUST_BE_KDC __constant_cpu_to_le32(0xC00002F5)
-#define STATUS_STRONG_CRYPTO_NOT_SUPPORTED __constant_cpu_to_le32(0xC00002F6)
-#define STATUS_TOO_MANY_PRINCIPALS __constant_cpu_to_le32(0xC00002F7)
-#define STATUS_NO_PA_DATA __constant_cpu_to_le32(0xC00002F8)
-#define STATUS_PKINIT_NAME_MISMATCH __constant_cpu_to_le32(0xC00002F9)
-#define STATUS_SMARTCARD_LOGON_REQUIRED __constant_cpu_to_le32(0xC00002FA)
-#define STATUS_KDC_INVALID_REQUEST __constant_cpu_to_le32(0xC00002FB)
-#define STATUS_KDC_UNABLE_TO_REFER __constant_cpu_to_le32(0xC00002FC)
-#define STATUS_KDC_UNKNOWN_ETYPE __constant_cpu_to_le32(0xC00002FD)
-#define STATUS_SHUTDOWN_IN_PROGRESS __constant_cpu_to_le32(0xC00002FE)
-#define STATUS_SERVER_SHUTDOWN_IN_PROGRESS __constant_cpu_to_le32(0xC00002FF)
-#define STATUS_NOT_SUPPORTED_ON_SBS __constant_cpu_to_le32(0xC0000300)
-#define STATUS_WMI_GUID_DISCONNECTED __constant_cpu_to_le32(0xC0000301)
-#define STATUS_WMI_ALREADY_DISABLED __constant_cpu_to_le32(0xC0000302)
-#define STATUS_WMI_ALREADY_ENABLED __constant_cpu_to_le32(0xC0000303)
-#define STATUS_MFT_TOO_FRAGMENTED __constant_cpu_to_le32(0xC0000304)
-#define STATUS_COPY_PROTECTION_FAILURE __constant_cpu_to_le32(0xC0000305)
-#define STATUS_CSS_AUTHENTICATION_FAILURE __constant_cpu_to_le32(0xC0000306)
-#define STATUS_CSS_KEY_NOT_PRESENT __constant_cpu_to_le32(0xC0000307)
-#define STATUS_CSS_KEY_NOT_ESTABLISHED __constant_cpu_to_le32(0xC0000308)
-#define STATUS_CSS_SCRAMBLED_SECTOR __constant_cpu_to_le32(0xC0000309)
-#define STATUS_CSS_REGION_MISMATCH __constant_cpu_to_le32(0xC000030A)
-#define STATUS_CSS_RESETS_EXHAUSTED __constant_cpu_to_le32(0xC000030B)
-#define STATUS_PKINIT_FAILURE __constant_cpu_to_le32(0xC0000320)
-#define STATUS_SMARTCARD_SUBSYSTEM_FAILURE __constant_cpu_to_le32(0xC0000321)
-#define STATUS_NO_KERB_KEY __constant_cpu_to_le32(0xC0000322)
-#define STATUS_HOST_DOWN __constant_cpu_to_le32(0xC0000350)
-#define STATUS_UNSUPPORTED_PREAUTH __constant_cpu_to_le32(0xC0000351)
-#define STATUS_EFS_ALG_BLOB_TOO_BIG __constant_cpu_to_le32(0xC0000352)
-#define STATUS_PORT_NOT_SET __constant_cpu_to_le32(0xC0000353)
-#define STATUS_DEBUGGER_INACTIVE __constant_cpu_to_le32(0xC0000354)
-#define STATUS_DS_VERSION_CHECK_FAILURE __constant_cpu_to_le32(0xC0000355)
-#define STATUS_AUDITING_DISABLED __constant_cpu_to_le32(0xC0000356)
-#define STATUS_PRENT4_MACHINE_ACCOUNT __constant_cpu_to_le32(0xC0000357)
-#define STATUS_DS_AG_CANT_HAVE_UNIVERSAL_MEMBER __constant_cpu_to_le32(0xC0000358)
-#define STATUS_INVALID_IMAGE_WIN_32 __constant_cpu_to_le32(0xC0000359)
-#define STATUS_INVALID_IMAGE_WIN_64 __constant_cpu_to_le32(0xC000035A)
-#define STATUS_BAD_BINDINGS __constant_cpu_to_le32(0xC000035B)
-#define STATUS_NETWORK_SESSION_EXPIRED __constant_cpu_to_le32(0xC000035C)
-#define STATUS_APPHELP_BLOCK __constant_cpu_to_le32(0xC000035D)
-#define STATUS_ALL_SIDS_FILTERED __constant_cpu_to_le32(0xC000035E)
-#define STATUS_NOT_SAFE_MODE_DRIVER __constant_cpu_to_le32(0xC000035F)
-#define STATUS_ACCESS_DISABLED_BY_POLICY_DEFAULT __constant_cpu_to_le32(0xC0000361)
-#define STATUS_ACCESS_DISABLED_BY_POLICY_PATH __constant_cpu_to_le32(0xC0000362)
-#define STATUS_ACCESS_DISABLED_BY_POLICY_PUBLISHER __constant_cpu_to_le32(0xC0000363)
-#define STATUS_ACCESS_DISABLED_BY_POLICY_OTHER __constant_cpu_to_le32(0xC0000364)
-#define STATUS_FAILED_DRIVER_ENTRY __constant_cpu_to_le32(0xC0000365)
-#define STATUS_DEVICE_ENUMERATION_ERROR __constant_cpu_to_le32(0xC0000366)
-#define STATUS_MOUNT_POINT_NOT_RESOLVED __constant_cpu_to_le32(0xC0000368)
-#define STATUS_INVALID_DEVICE_OBJECT_PARAMETER __constant_cpu_to_le32(0xC0000369)
-#define STATUS_MCA_OCCURED __constant_cpu_to_le32(0xC000036A)
-#define STATUS_DRIVER_BLOCKED_CRITICAL __constant_cpu_to_le32(0xC000036B)
-#define STATUS_DRIVER_BLOCKED __constant_cpu_to_le32(0xC000036C)
-#define STATUS_DRIVER_DATABASE_ERROR __constant_cpu_to_le32(0xC000036D)
-#define STATUS_SYSTEM_HIVE_TOO_LARGE __constant_cpu_to_le32(0xC000036E)
-#define STATUS_INVALID_IMPORT_OF_NON_DLL __constant_cpu_to_le32(0xC000036F)
-#define STATUS_NO_SECRETS __constant_cpu_to_le32(0xC0000371)
-#define STATUS_ACCESS_DISABLED_NO_SAFER_UI_BY_POLICY __constant_cpu_to_le32(0xC0000372)
-#define STATUS_FAILED_STACK_SWITCH __constant_cpu_to_le32(0xC0000373)
-#define STATUS_HEAP_CORRUPTION __constant_cpu_to_le32(0xC0000374)
-#define STATUS_SMARTCARD_WRONG_PIN __constant_cpu_to_le32(0xC0000380)
-#define STATUS_SMARTCARD_CARD_BLOCKED __constant_cpu_to_le32(0xC0000381)
-#define STATUS_SMARTCARD_CARD_NOT_AUTHENTICATED __constant_cpu_to_le32(0xC0000382)
-#define STATUS_SMARTCARD_NO_CARD __constant_cpu_to_le32(0xC0000383)
-#define STATUS_SMARTCARD_NO_KEY_CONTAINER __constant_cpu_to_le32(0xC0000384)
-#define STATUS_SMARTCARD_NO_CERTIFICATE __constant_cpu_to_le32(0xC0000385)
-#define STATUS_SMARTCARD_NO_KEYSET __constant_cpu_to_le32(0xC0000386)
-#define STATUS_SMARTCARD_IO_ERROR __constant_cpu_to_le32(0xC0000387)
-#define STATUS_DOWNGRADE_DETECTED __constant_cpu_to_le32(0xC0000388)
-#define STATUS_SMARTCARD_CERT_REVOKED __constant_cpu_to_le32(0xC0000389)
-#define STATUS_ISSUING_CA_UNTRUSTED __constant_cpu_to_le32(0xC000038A)
-#define STATUS_REVOCATION_OFFLINE_C __constant_cpu_to_le32(0xC000038B)
-#define STATUS_PKINIT_CLIENT_FAILURE __constant_cpu_to_le32(0xC000038C)
-#define STATUS_SMARTCARD_CERT_EXPIRED __constant_cpu_to_le32(0xC000038D)
-#define STATUS_DRIVER_FAILED_PRIOR_UNLOAD __constant_cpu_to_le32(0xC000038E)
-#define STATUS_SMARTCARD_SILENT_CONTEXT __constant_cpu_to_le32(0xC000038F)
-#define STATUS_PER_USER_TRUST_QUOTA_EXCEEDED __constant_cpu_to_le32(0xC0000401)
-#define STATUS_ALL_USER_TRUST_QUOTA_EXCEEDED __constant_cpu_to_le32(0xC0000402)
-#define STATUS_USER_DELETE_TRUST_QUOTA_EXCEEDED __constant_cpu_to_le32(0xC0000403)
-#define STATUS_DS_NAME_NOT_UNIQUE __constant_cpu_to_le32(0xC0000404)
-#define STATUS_DS_DUPLICATE_ID_FOUND __constant_cpu_to_le32(0xC0000405)
-#define STATUS_DS_GROUP_CONVERSION_ERROR __constant_cpu_to_le32(0xC0000406)
-#define STATUS_VOLSNAP_PREPARE_HIBERNATE __constant_cpu_to_le32(0xC0000407)
-#define STATUS_USER2USER_REQUIRED __constant_cpu_to_le32(0xC0000408)
-#define STATUS_STACK_BUFFER_OVERRUN __constant_cpu_to_le32(0xC0000409)
-#define STATUS_NO_S4U_PROT_SUPPORT __constant_cpu_to_le32(0xC000040A)
-#define STATUS_CROSSREALM_DELEGATION_FAILURE __constant_cpu_to_le32(0xC000040B)
-#define STATUS_REVOCATION_OFFLINE_KDC __constant_cpu_to_le32(0xC000040C)
-#define STATUS_ISSUING_CA_UNTRUSTED_KDC __constant_cpu_to_le32(0xC000040D)
-#define STATUS_KDC_CERT_EXPIRED __constant_cpu_to_le32(0xC000040E)
-#define STATUS_KDC_CERT_REVOKED __constant_cpu_to_le32(0xC000040F)
-#define STATUS_PARAMETER_QUOTA_EXCEEDED __constant_cpu_to_le32(0xC0000410)
-#define STATUS_HIBERNATION_FAILURE __constant_cpu_to_le32(0xC0000411)
-#define STATUS_DELAY_LOAD_FAILED __constant_cpu_to_le32(0xC0000412)
-#define STATUS_AUTHENTICATION_FIREWALL_FAILED __constant_cpu_to_le32(0xC0000413)
-#define STATUS_VDM_DISALLOWED __constant_cpu_to_le32(0xC0000414)
-#define STATUS_HUNG_DISPLAY_DRIVER_THREAD __constant_cpu_to_le32(0xC0000415)
-#define STATUS_INSUFFICIENT_RESOURCE_FOR_SPECIFIED_SHARED_SECTION_SIZE __constant_cpu_to_le32(0xC0000416)
-#define STATUS_INVALID_CRUNTIME_PARAMETER __constant_cpu_to_le32(0xC0000417)
-#define STATUS_NTLM_BLOCKED __constant_cpu_to_le32(0xC0000418)
-#define STATUS_ASSERTION_FAILURE __constant_cpu_to_le32(0xC0000420)
-#define STATUS_VERIFIER_STOP __constant_cpu_to_le32(0xC0000421)
-#define STATUS_CALLBACK_POP_STACK __constant_cpu_to_le32(0xC0000423)
-#define STATUS_INCOMPATIBLE_DRIVER_BLOCKED __constant_cpu_to_le32(0xC0000424)
-#define STATUS_HIVE_UNLOADED __constant_cpu_to_le32(0xC0000425)
-#define STATUS_COMPRESSION_DISABLED __constant_cpu_to_le32(0xC0000426)
-#define STATUS_FILE_SYSTEM_LIMITATION __constant_cpu_to_le32(0xC0000427)
-#define STATUS_INVALID_IMAGE_HASH __constant_cpu_to_le32(0xC0000428)
-#define STATUS_NOT_CAPABLE __constant_cpu_to_le32(0xC0000429)
-#define STATUS_REQUEST_OUT_OF_SEQUENCE __constant_cpu_to_le32(0xC000042A)
-#define STATUS_IMPLEMENTATION_LIMIT __constant_cpu_to_le32(0xC000042B)
-#define STATUS_ELEVATION_REQUIRED __constant_cpu_to_le32(0xC000042C)
-#define STATUS_BEYOND_VDL __constant_cpu_to_le32(0xC0000432)
-#define STATUS_ENCOUNTERED_WRITE_IN_PROGRESS __constant_cpu_to_le32(0xC0000433)
-#define STATUS_PTE_CHANGED __constant_cpu_to_le32(0xC0000434)
-#define STATUS_PURGE_FAILED __constant_cpu_to_le32(0xC0000435)
-#define STATUS_CRED_REQUIRES_CONFIRMATION __constant_cpu_to_le32(0xC0000440)
-#define STATUS_CS_ENCRYPTION_INVALID_SERVER_RESPONSE __constant_cpu_to_le32(0xC0000441)
-#define STATUS_CS_ENCRYPTION_UNSUPPORTED_SERVER __constant_cpu_to_le32(0xC0000442)
-#define STATUS_CS_ENCRYPTION_EXISTING_ENCRYPTED_FILE __constant_cpu_to_le32(0xC0000443)
-#define STATUS_CS_ENCRYPTION_NEW_ENCRYPTED_FILE __constant_cpu_to_le32(0xC0000444)
-#define STATUS_CS_ENCRYPTION_FILE_NOT_CSE __constant_cpu_to_le32(0xC0000445)
-#define STATUS_INVALID_LABEL __constant_cpu_to_le32(0xC0000446)
-#define STATUS_DRIVER_PROCESS_TERMINATED __constant_cpu_to_le32(0xC0000450)
-#define STATUS_AMBIGUOUS_SYSTEM_DEVICE __constant_cpu_to_le32(0xC0000451)
-#define STATUS_SYSTEM_DEVICE_NOT_FOUND __constant_cpu_to_le32(0xC0000452)
-#define STATUS_RESTART_BOOT_APPLICATION __constant_cpu_to_le32(0xC0000453)
-#define STATUS_INVALID_TASK_NAME __constant_cpu_to_le32(0xC0000500)
-#define STATUS_INVALID_TASK_INDEX __constant_cpu_to_le32(0xC0000501)
-#define STATUS_THREAD_ALREADY_IN_TASK __constant_cpu_to_le32(0xC0000502)
-#define STATUS_CALLBACK_BYPASS __constant_cpu_to_le32(0xC0000503)
-#define STATUS_PORT_CLOSED __constant_cpu_to_le32(0xC0000700)
-#define STATUS_MESSAGE_LOST __constant_cpu_to_le32(0xC0000701)
-#define STATUS_INVALID_MESSAGE __constant_cpu_to_le32(0xC0000702)
-#define STATUS_REQUEST_CANCELED __constant_cpu_to_le32(0xC0000703)
-#define STATUS_RECURSIVE_DISPATCH __constant_cpu_to_le32(0xC0000704)
-#define STATUS_LPC_RECEIVE_BUFFER_EXPECTED __constant_cpu_to_le32(0xC0000705)
-#define STATUS_LPC_INVALID_CONNECTION_USAGE __constant_cpu_to_le32(0xC0000706)
-#define STATUS_LPC_REQUESTS_NOT_ALLOWED __constant_cpu_to_le32(0xC0000707)
-#define STATUS_RESOURCE_IN_USE __constant_cpu_to_le32(0xC0000708)
-#define STATUS_HARDWARE_MEMORY_ERROR __constant_cpu_to_le32(0xC0000709)
-#define STATUS_THREADPOOL_HANDLE_EXCEPTION __constant_cpu_to_le32(0xC000070A)
-#define STATUS_THREADPOOL_SET_EVENT_ON_COMPLETION_FAILED __constant_cpu_to_le32(0xC000070B)
-#define STATUS_THREADPOOL_RELEASE_SEMAPHORE_ON_COMPLETION_FAILED __constant_cpu_to_le32(0xC000070C)
-#define STATUS_THREADPOOL_RELEASE_MUTEX_ON_COMPLETION_FAILED __constant_cpu_to_le32(0xC000070D)
-#define STATUS_THREADPOOL_FREE_LIBRARY_ON_COMPLETION_FAILED __constant_cpu_to_le32(0xC000070E)
-#define STATUS_THREADPOOL_RELEASED_DURING_OPERATION __constant_cpu_to_le32(0xC000070F)
-#define STATUS_CALLBACK_RETURNED_WHILE_IMPERSONATING __constant_cpu_to_le32(0xC0000710)
-#define STATUS_APC_RETURNED_WHILE_IMPERSONATING __constant_cpu_to_le32(0xC0000711)
-#define STATUS_PROCESS_IS_PROTECTED __constant_cpu_to_le32(0xC0000712)
-#define STATUS_MCA_EXCEPTION __constant_cpu_to_le32(0xC0000713)
-#define STATUS_CERTIFICATE_MAPPING_NOT_UNIQUE __constant_cpu_to_le32(0xC0000714)
-#define STATUS_SYMLINK_CLASS_DISABLED __constant_cpu_to_le32(0xC0000715)
-#define STATUS_INVALID_IDN_NORMALIZATION __constant_cpu_to_le32(0xC0000716)
-#define STATUS_NO_UNICODE_TRANSLATION __constant_cpu_to_le32(0xC0000717)
-#define STATUS_ALREADY_REGISTERED __constant_cpu_to_le32(0xC0000718)
-#define STATUS_CONTEXT_MISMATCH __constant_cpu_to_le32(0xC0000719)
-#define STATUS_PORT_ALREADY_HAS_COMPLETION_LIST __constant_cpu_to_le32(0xC000071A)
-#define STATUS_CALLBACK_RETURNED_THREAD_PRIORITY __constant_cpu_to_le32(0xC000071B)
-#define STATUS_INVALID_THREAD __constant_cpu_to_le32(0xC000071C)
-#define STATUS_CALLBACK_RETURNED_TRANSACTION __constant_cpu_to_le32(0xC000071D)
-#define STATUS_CALLBACK_RETURNED_LDR_LOCK __constant_cpu_to_le32(0xC000071E)
-#define STATUS_CALLBACK_RETURNED_LANG __constant_cpu_to_le32(0xC000071F)
-#define STATUS_CALLBACK_RETURNED_PRI_BACK __constant_cpu_to_le32(0xC0000720)
-#define STATUS_CALLBACK_RETURNED_THREAD_AFFINITY __constant_cpu_to_le32(0xC0000721)
-#define STATUS_DISK_REPAIR_DISABLED __constant_cpu_to_le32(0xC0000800)
-#define STATUS_DS_DOMAIN_RENAME_IN_PROGRESS __constant_cpu_to_le32(0xC0000801)
-#define STATUS_DISK_QUOTA_EXCEEDED __constant_cpu_to_le32(0xC0000802)
-#define STATUS_CONTENT_BLOCKED __constant_cpu_to_le32(0xC0000804)
-#define STATUS_BAD_CLUSTERS __constant_cpu_to_le32(0xC0000805)
-#define STATUS_VOLUME_DIRTY __constant_cpu_to_le32(0xC0000806)
-#define STATUS_FILE_CHECKED_OUT __constant_cpu_to_le32(0xC0000901)
-#define STATUS_CHECKOUT_REQUIRED __constant_cpu_to_le32(0xC0000902)
-#define STATUS_BAD_FILE_TYPE __constant_cpu_to_le32(0xC0000903)
-#define STATUS_FILE_TOO_LARGE __constant_cpu_to_le32(0xC0000904)
-#define STATUS_FORMS_AUTH_REQUIRED __constant_cpu_to_le32(0xC0000905)
-#define STATUS_VIRUS_INFECTED __constant_cpu_to_le32(0xC0000906)
-#define STATUS_VIRUS_DELETED __constant_cpu_to_le32(0xC0000907)
-#define STATUS_BAD_MCFG_TABLE __constant_cpu_to_le32(0xC0000908)
-#define STATUS_WOW_ASSERTION __constant_cpu_to_le32(0xC0009898)
-#define STATUS_INVALID_SIGNATURE __constant_cpu_to_le32(0xC000A000)
-#define STATUS_HMAC_NOT_SUPPORTED __constant_cpu_to_le32(0xC000A001)
-#define STATUS_IPSEC_QUEUE_OVERFLOW __constant_cpu_to_le32(0xC000A010)
-#define STATUS_ND_QUEUE_OVERFLOW __constant_cpu_to_le32(0xC000A011)
-#define STATUS_HOPLIMIT_EXCEEDED __constant_cpu_to_le32(0xC000A012)
-#define STATUS_PROTOCOL_NOT_SUPPORTED __constant_cpu_to_le32(0xC000A013)
-#define STATUS_LOST_WRITEBEHIND_DATA_NETWORK_DISCONNECTED __constant_cpu_to_le32(0xC000A080)
-#define STATUS_LOST_WRITEBEHIND_DATA_NETWORK_SERVER_ERROR __constant_cpu_to_le32(0xC000A081)
-#define STATUS_LOST_WRITEBEHIND_DATA_LOCAL_DISK_ERROR __constant_cpu_to_le32(0xC000A082)
-#define STATUS_XML_PARSE_ERROR __constant_cpu_to_le32(0xC000A083)
-#define STATUS_XMLDSIG_ERROR __constant_cpu_to_le32(0xC000A084)
-#define STATUS_WRONG_COMPARTMENT __constant_cpu_to_le32(0xC000A085)
-#define STATUS_AUTHIP_FAILURE __constant_cpu_to_le32(0xC000A086)
-#define DBG_NO_STATE_CHANGE __constant_cpu_to_le32(0xC0010001)
-#define DBG_APP_NOT_IDLE __constant_cpu_to_le32(0xC0010002)
-#define RPC_NT_INVALID_STRING_BINDING __constant_cpu_to_le32(0xC0020001)
-#define RPC_NT_WRONG_KIND_OF_BINDING __constant_cpu_to_le32(0xC0020002)
-#define RPC_NT_INVALID_BINDING __constant_cpu_to_le32(0xC0020003)
-#define RPC_NT_PROTSEQ_NOT_SUPPORTED __constant_cpu_to_le32(0xC0020004)
-#define RPC_NT_INVALID_RPC_PROTSEQ __constant_cpu_to_le32(0xC0020005)
-#define RPC_NT_INVALID_STRING_UUID __constant_cpu_to_le32(0xC0020006)
-#define RPC_NT_INVALID_ENDPOINT_FORMAT __constant_cpu_to_le32(0xC0020007)
-#define RPC_NT_INVALID_NET_ADDR __constant_cpu_to_le32(0xC0020008)
-#define RPC_NT_NO_ENDPOINT_FOUND __constant_cpu_to_le32(0xC0020009)
-#define RPC_NT_INVALID_TIMEOUT __constant_cpu_to_le32(0xC002000A)
-#define RPC_NT_OBJECT_NOT_FOUND __constant_cpu_to_le32(0xC002000B)
-#define RPC_NT_ALREADY_REGISTERED __constant_cpu_to_le32(0xC002000C)
-#define RPC_NT_TYPE_ALREADY_REGISTERED __constant_cpu_to_le32(0xC002000D)
-#define RPC_NT_ALREADY_LISTENING __constant_cpu_to_le32(0xC002000E)
-#define RPC_NT_NO_PROTSEQS_REGISTERED __constant_cpu_to_le32(0xC002000F)
-#define RPC_NT_NOT_LISTENING __constant_cpu_to_le32(0xC0020010)
-#define RPC_NT_UNKNOWN_MGR_TYPE __constant_cpu_to_le32(0xC0020011)
-#define RPC_NT_UNKNOWN_IF __constant_cpu_to_le32(0xC0020012)
-#define RPC_NT_NO_BINDINGS __constant_cpu_to_le32(0xC0020013)
-#define RPC_NT_NO_PROTSEQS __constant_cpu_to_le32(0xC0020014)
-#define RPC_NT_CANT_CREATE_ENDPOINT __constant_cpu_to_le32(0xC0020015)
-#define RPC_NT_OUT_OF_RESOURCES __constant_cpu_to_le32(0xC0020016)
-#define RPC_NT_SERVER_UNAVAILABLE __constant_cpu_to_le32(0xC0020017)
-#define RPC_NT_SERVER_TOO_BUSY __constant_cpu_to_le32(0xC0020018)
-#define RPC_NT_INVALID_NETWORK_OPTIONS __constant_cpu_to_le32(0xC0020019)
-#define RPC_NT_NO_CALL_ACTIVE __constant_cpu_to_le32(0xC002001A)
-#define RPC_NT_CALL_FAILED __constant_cpu_to_le32(0xC002001B)
-#define RPC_NT_CALL_FAILED_DNE __constant_cpu_to_le32(0xC002001C)
-#define RPC_NT_PROTOCOL_ERROR __constant_cpu_to_le32(0xC002001D)
-#define RPC_NT_UNSUPPORTED_TRANS_SYN __constant_cpu_to_le32(0xC002001F)
-#define RPC_NT_UNSUPPORTED_TYPE __constant_cpu_to_le32(0xC0020021)
-#define RPC_NT_INVALID_TAG __constant_cpu_to_le32(0xC0020022)
-#define RPC_NT_INVALID_BOUND __constant_cpu_to_le32(0xC0020023)
-#define RPC_NT_NO_ENTRY_NAME __constant_cpu_to_le32(0xC0020024)
-#define RPC_NT_INVALID_NAME_SYNTAX __constant_cpu_to_le32(0xC0020025)
-#define RPC_NT_UNSUPPORTED_NAME_SYNTAX __constant_cpu_to_le32(0xC0020026)
-#define RPC_NT_UUID_NO_ADDRESS __constant_cpu_to_le32(0xC0020028)
-#define RPC_NT_DUPLICATE_ENDPOINT __constant_cpu_to_le32(0xC0020029)
-#define RPC_NT_UNKNOWN_AUTHN_TYPE __constant_cpu_to_le32(0xC002002A)
-#define RPC_NT_MAX_CALLS_TOO_SMALL __constant_cpu_to_le32(0xC002002B)
-#define RPC_NT_STRING_TOO_LONG __constant_cpu_to_le32(0xC002002C)
-#define RPC_NT_PROTSEQ_NOT_FOUND __constant_cpu_to_le32(0xC002002D)
-#define RPC_NT_PROCNUM_OUT_OF_RANGE __constant_cpu_to_le32(0xC002002E)
-#define RPC_NT_BINDING_HAS_NO_AUTH __constant_cpu_to_le32(0xC002002F)
-#define RPC_NT_UNKNOWN_AUTHN_SERVICE __constant_cpu_to_le32(0xC0020030)
-#define RPC_NT_UNKNOWN_AUTHN_LEVEL __constant_cpu_to_le32(0xC0020031)
-#define RPC_NT_INVALID_AUTH_IDENTITY __constant_cpu_to_le32(0xC0020032)
-#define RPC_NT_UNKNOWN_AUTHZ_SERVICE __constant_cpu_to_le32(0xC0020033)
-#define EPT_NT_INVALID_ENTRY __constant_cpu_to_le32(0xC0020034)
-#define EPT_NT_CANT_PERFORM_OP __constant_cpu_to_le32(0xC0020035)
-#define EPT_NT_NOT_REGISTERED __constant_cpu_to_le32(0xC0020036)
-#define RPC_NT_NOTHING_TO_EXPORT __constant_cpu_to_le32(0xC0020037)
-#define RPC_NT_INCOMPLETE_NAME __constant_cpu_to_le32(0xC0020038)
-#define RPC_NT_INVALID_VERS_OPTION __constant_cpu_to_le32(0xC0020039)
-#define RPC_NT_NO_MORE_MEMBERS __constant_cpu_to_le32(0xC002003A)
-#define RPC_NT_NOT_ALL_OBJS_UNEXPORTED __constant_cpu_to_le32(0xC002003B)
-#define RPC_NT_INTERFACE_NOT_FOUND __constant_cpu_to_le32(0xC002003C)
-#define RPC_NT_ENTRY_ALREADY_EXISTS __constant_cpu_to_le32(0xC002003D)
-#define RPC_NT_ENTRY_NOT_FOUND __constant_cpu_to_le32(0xC002003E)
-#define RPC_NT_NAME_SERVICE_UNAVAILABLE __constant_cpu_to_le32(0xC002003F)
-#define RPC_NT_INVALID_NAF_ID __constant_cpu_to_le32(0xC0020040)
-#define RPC_NT_CANNOT_SUPPORT __constant_cpu_to_le32(0xC0020041)
-#define RPC_NT_NO_CONTEXT_AVAILABLE __constant_cpu_to_le32(0xC0020042)
-#define RPC_NT_INTERNAL_ERROR __constant_cpu_to_le32(0xC0020043)
-#define RPC_NT_ZERO_DIVIDE __constant_cpu_to_le32(0xC0020044)
-#define RPC_NT_ADDRESS_ERROR __constant_cpu_to_le32(0xC0020045)
-#define RPC_NT_FP_DIV_ZERO __constant_cpu_to_le32(0xC0020046)
-#define RPC_NT_FP_UNDERFLOW __constant_cpu_to_le32(0xC0020047)
-#define RPC_NT_FP_OVERFLOW __constant_cpu_to_le32(0xC0020048)
-#define RPC_NT_CALL_IN_PROGRESS __constant_cpu_to_le32(0xC0020049)
-#define RPC_NT_NO_MORE_BINDINGS __constant_cpu_to_le32(0xC002004A)
-#define RPC_NT_GROUP_MEMBER_NOT_FOUND __constant_cpu_to_le32(0xC002004B)
-#define EPT_NT_CANT_CREATE __constant_cpu_to_le32(0xC002004C)
-#define RPC_NT_INVALID_OBJECT __constant_cpu_to_le32(0xC002004D)
-#define RPC_NT_NO_INTERFACES __constant_cpu_to_le32(0xC002004F)
-#define RPC_NT_CALL_CANCELLED __constant_cpu_to_le32(0xC0020050)
-#define RPC_NT_BINDING_INCOMPLETE __constant_cpu_to_le32(0xC0020051)
-#define RPC_NT_COMM_FAILURE __constant_cpu_to_le32(0xC0020052)
-#define RPC_NT_UNSUPPORTED_AUTHN_LEVEL __constant_cpu_to_le32(0xC0020053)
-#define RPC_NT_NO_PRINC_NAME __constant_cpu_to_le32(0xC0020054)
-#define RPC_NT_NOT_RPC_ERROR __constant_cpu_to_le32(0xC0020055)
-#define RPC_NT_SEC_PKG_ERROR __constant_cpu_to_le32(0xC0020057)
-#define RPC_NT_NOT_CANCELLED __constant_cpu_to_le32(0xC0020058)
-#define RPC_NT_INVALID_ASYNC_HANDLE __constant_cpu_to_le32(0xC0020062)
-#define RPC_NT_INVALID_ASYNC_CALL __constant_cpu_to_le32(0xC0020063)
-#define RPC_NT_PROXY_ACCESS_DENIED __constant_cpu_to_le32(0xC0020064)
-#define RPC_NT_NO_MORE_ENTRIES __constant_cpu_to_le32(0xC0030001)
-#define RPC_NT_SS_CHAR_TRANS_OPEN_FAIL __constant_cpu_to_le32(0xC0030002)
-#define RPC_NT_SS_CHAR_TRANS_SHORT_FILE __constant_cpu_to_le32(0xC0030003)
-#define RPC_NT_SS_IN_NULL_CONTEXT __constant_cpu_to_le32(0xC0030004)
-#define RPC_NT_SS_CONTEXT_MISMATCH __constant_cpu_to_le32(0xC0030005)
-#define RPC_NT_SS_CONTEXT_DAMAGED __constant_cpu_to_le32(0xC0030006)
-#define RPC_NT_SS_HANDLES_MISMATCH __constant_cpu_to_le32(0xC0030007)
-#define RPC_NT_SS_CANNOT_GET_CALL_HANDLE __constant_cpu_to_le32(0xC0030008)
-#define RPC_NT_NULL_REF_POINTER __constant_cpu_to_le32(0xC0030009)
-#define RPC_NT_ENUM_VALUE_OUT_OF_RANGE __constant_cpu_to_le32(0xC003000A)
-#define RPC_NT_BYTE_COUNT_TOO_SMALL __constant_cpu_to_le32(0xC003000B)
-#define RPC_NT_BAD_STUB_DATA __constant_cpu_to_le32(0xC003000C)
-#define RPC_NT_INVALID_ES_ACTION __constant_cpu_to_le32(0xC0030059)
-#define RPC_NT_WRONG_ES_VERSION __constant_cpu_to_le32(0xC003005A)
-#define RPC_NT_WRONG_STUB_VERSION __constant_cpu_to_le32(0xC003005B)
-#define RPC_NT_INVALID_PIPE_OBJECT __constant_cpu_to_le32(0xC003005C)
-#define RPC_NT_INVALID_PIPE_OPERATION __constant_cpu_to_le32(0xC003005D)
-#define RPC_NT_WRONG_PIPE_VERSION __constant_cpu_to_le32(0xC003005E)
-#define RPC_NT_PIPE_CLOSED __constant_cpu_to_le32(0xC003005F)
-#define RPC_NT_PIPE_DISCIPLINE_ERROR __constant_cpu_to_le32(0xC0030060)
-#define RPC_NT_PIPE_EMPTY __constant_cpu_to_le32(0xC0030061)
-#define STATUS_PNP_BAD_MPS_TABLE __constant_cpu_to_le32(0xC0040035)
-#define STATUS_PNP_TRANSLATION_FAILED __constant_cpu_to_le32(0xC0040036)
-#define STATUS_PNP_IRQ_TRANSLATION_FAILED __constant_cpu_to_le32(0xC0040037)
-#define STATUS_PNP_INVALID_ID __constant_cpu_to_le32(0xC0040038)
-#define STATUS_IO_REISSUE_AS_CACHED __constant_cpu_to_le32(0xC0040039)
-#define STATUS_CTX_WINSTATION_NAME_INVALID __constant_cpu_to_le32(0xC00A0001)
-#define STATUS_CTX_INVALID_PD __constant_cpu_to_le32(0xC00A0002)
-#define STATUS_CTX_PD_NOT_FOUND __constant_cpu_to_le32(0xC00A0003)
-#define STATUS_CTX_CLOSE_PENDING __constant_cpu_to_le32(0xC00A0006)
-#define STATUS_CTX_NO_OUTBUF __constant_cpu_to_le32(0xC00A0007)
-#define STATUS_CTX_MODEM_INF_NOT_FOUND __constant_cpu_to_le32(0xC00A0008)
-#define STATUS_CTX_INVALID_MODEMNAME __constant_cpu_to_le32(0xC00A0009)
-#define STATUS_CTX_RESPONSE_ERROR __constant_cpu_to_le32(0xC00A000A)
-#define STATUS_CTX_MODEM_RESPONSE_TIMEOUT __constant_cpu_to_le32(0xC00A000B)
-#define STATUS_CTX_MODEM_RESPONSE_NO_CARRIER __constant_cpu_to_le32(0xC00A000C)
-#define STATUS_CTX_MODEM_RESPONSE_NO_DIALTONE __constant_cpu_to_le32(0xC00A000D)
-#define STATUS_CTX_MODEM_RESPONSE_BUSY __constant_cpu_to_le32(0xC00A000E)
-#define STATUS_CTX_MODEM_RESPONSE_VOICE __constant_cpu_to_le32(0xC00A000F)
-#define STATUS_CTX_TD_ERROR __constant_cpu_to_le32(0xC00A0010)
-#define STATUS_CTX_LICENSE_CLIENT_INVALID __constant_cpu_to_le32(0xC00A0012)
-#define STATUS_CTX_LICENSE_NOT_AVAILABLE __constant_cpu_to_le32(0xC00A0013)
-#define STATUS_CTX_LICENSE_EXPIRED __constant_cpu_to_le32(0xC00A0014)
-#define STATUS_CTX_WINSTATION_NOT_FOUND __constant_cpu_to_le32(0xC00A0015)
-#define STATUS_CTX_WINSTATION_NAME_COLLISION __constant_cpu_to_le32(0xC00A0016)
-#define STATUS_CTX_WINSTATION_BUSY __constant_cpu_to_le32(0xC00A0017)
-#define STATUS_CTX_BAD_VIDEO_MODE __constant_cpu_to_le32(0xC00A0018)
-#define STATUS_CTX_GRAPHICS_INVALID __constant_cpu_to_le32(0xC00A0022)
-#define STATUS_CTX_NOT_CONSOLE __constant_cpu_to_le32(0xC00A0024)
-#define STATUS_CTX_CLIENT_QUERY_TIMEOUT __constant_cpu_to_le32(0xC00A0026)
-#define STATUS_CTX_CONSOLE_DISCONNECT __constant_cpu_to_le32(0xC00A0027)
-#define STATUS_CTX_CONSOLE_CONNECT __constant_cpu_to_le32(0xC00A0028)
-#define STATUS_CTX_SHADOW_DENIED __constant_cpu_to_le32(0xC00A002A)
-#define STATUS_CTX_WINSTATION_ACCESS_DENIED __constant_cpu_to_le32(0xC00A002B)
-#define STATUS_CTX_INVALID_WD __constant_cpu_to_le32(0xC00A002E)
-#define STATUS_CTX_WD_NOT_FOUND __constant_cpu_to_le32(0xC00A002F)
-#define STATUS_CTX_SHADOW_INVALID __constant_cpu_to_le32(0xC00A0030)
-#define STATUS_CTX_SHADOW_DISABLED __constant_cpu_to_le32(0xC00A0031)
-#define STATUS_RDP_PROTOCOL_ERROR __constant_cpu_to_le32(0xC00A0032)
-#define STATUS_CTX_CLIENT_LICENSE_NOT_SET __constant_cpu_to_le32(0xC00A0033)
-#define STATUS_CTX_CLIENT_LICENSE_IN_USE __constant_cpu_to_le32(0xC00A0034)
-#define STATUS_CTX_SHADOW_ENDED_BY_MODE_CHANGE __constant_cpu_to_le32(0xC00A0035)
-#define STATUS_CTX_SHADOW_NOT_RUNNING __constant_cpu_to_le32(0xC00A0036)
-#define STATUS_CTX_LOGON_DISABLED __constant_cpu_to_le32(0xC00A0037)
-#define STATUS_CTX_SECURITY_LAYER_ERROR __constant_cpu_to_le32(0xC00A0038)
-#define STATUS_TS_INCOMPATIBLE_SESSIONS __constant_cpu_to_le32(0xC00A0039)
-#define STATUS_MUI_FILE_NOT_FOUND __constant_cpu_to_le32(0xC00B0001)
-#define STATUS_MUI_INVALID_FILE __constant_cpu_to_le32(0xC00B0002)
-#define STATUS_MUI_INVALID_RC_CONFIG __constant_cpu_to_le32(0xC00B0003)
-#define STATUS_MUI_INVALID_LOCALE_NAME __constant_cpu_to_le32(0xC00B0004)
-#define STATUS_MUI_INVALID_ULTIMATEFALLBACK_NAME __constant_cpu_to_le32(0xC00B0005)
-#define STATUS_MUI_FILE_NOT_LOADED __constant_cpu_to_le32(0xC00B0006)
-#define STATUS_RESOURCE_ENUM_USER_STOP __constant_cpu_to_le32(0xC00B0007)
-#define STATUS_CLUSTER_INVALID_NODE __constant_cpu_to_le32(0xC0130001)
-#define STATUS_CLUSTER_NODE_EXISTS __constant_cpu_to_le32(0xC0130002)
-#define STATUS_CLUSTER_JOIN_IN_PROGRESS __constant_cpu_to_le32(0xC0130003)
-#define STATUS_CLUSTER_NODE_NOT_FOUND __constant_cpu_to_le32(0xC0130004)
-#define STATUS_CLUSTER_LOCAL_NODE_NOT_FOUND __constant_cpu_to_le32(0xC0130005)
-#define STATUS_CLUSTER_NETWORK_EXISTS __constant_cpu_to_le32(0xC0130006)
-#define STATUS_CLUSTER_NETWORK_NOT_FOUND __constant_cpu_to_le32(0xC0130007)
-#define STATUS_CLUSTER_NETINTERFACE_EXISTS __constant_cpu_to_le32(0xC0130008)
-#define STATUS_CLUSTER_NETINTERFACE_NOT_FOUND __constant_cpu_to_le32(0xC0130009)
-#define STATUS_CLUSTER_INVALID_REQUEST __constant_cpu_to_le32(0xC013000A)
-#define STATUS_CLUSTER_INVALID_NETWORK_PROVIDER __constant_cpu_to_le32(0xC013000B)
-#define STATUS_CLUSTER_NODE_DOWN __constant_cpu_to_le32(0xC013000C)
-#define STATUS_CLUSTER_NODE_UNREACHABLE __constant_cpu_to_le32(0xC013000D)
-#define STATUS_CLUSTER_NODE_NOT_MEMBER __constant_cpu_to_le32(0xC013000E)
-#define STATUS_CLUSTER_JOIN_NOT_IN_PROGRESS __constant_cpu_to_le32(0xC013000F)
-#define STATUS_CLUSTER_INVALID_NETWORK __constant_cpu_to_le32(0xC0130010)
-#define STATUS_CLUSTER_NO_NET_ADAPTERS __constant_cpu_to_le32(0xC0130011)
-#define STATUS_CLUSTER_NODE_UP __constant_cpu_to_le32(0xC0130012)
-#define STATUS_CLUSTER_NODE_PAUSED __constant_cpu_to_le32(0xC0130013)
-#define STATUS_CLUSTER_NODE_NOT_PAUSED __constant_cpu_to_le32(0xC0130014)
-#define STATUS_CLUSTER_NO_SECURITY_CONTEXT __constant_cpu_to_le32(0xC0130015)
-#define STATUS_CLUSTER_NETWORK_NOT_INTERNAL __constant_cpu_to_le32(0xC0130016)
-#define STATUS_CLUSTER_POISONED __constant_cpu_to_le32(0xC0130017)
-#define STATUS_ACPI_INVALID_OPCODE __constant_cpu_to_le32(0xC0140001)
-#define STATUS_ACPI_STACK_OVERFLOW __constant_cpu_to_le32(0xC0140002)
-#define STATUS_ACPI_ASSERT_FAILED __constant_cpu_to_le32(0xC0140003)
-#define STATUS_ACPI_INVALID_INDEX __constant_cpu_to_le32(0xC0140004)
-#define STATUS_ACPI_INVALID_ARGUMENT __constant_cpu_to_le32(0xC0140005)
-#define STATUS_ACPI_FATAL __constant_cpu_to_le32(0xC0140006)
-#define STATUS_ACPI_INVALID_SUPERNAME __constant_cpu_to_le32(0xC0140007)
-#define STATUS_ACPI_INVALID_ARGTYPE __constant_cpu_to_le32(0xC0140008)
-#define STATUS_ACPI_INVALID_OBJTYPE __constant_cpu_to_le32(0xC0140009)
-#define STATUS_ACPI_INVALID_TARGETTYPE __constant_cpu_to_le32(0xC014000A)
-#define STATUS_ACPI_INCORRECT_ARGUMENT_COUNT __constant_cpu_to_le32(0xC014000B)
-#define STATUS_ACPI_ADDRESS_NOT_MAPPED __constant_cpu_to_le32(0xC014000C)
-#define STATUS_ACPI_INVALID_EVENTTYPE __constant_cpu_to_le32(0xC014000D)
-#define STATUS_ACPI_HANDLER_COLLISION __constant_cpu_to_le32(0xC014000E)
-#define STATUS_ACPI_INVALID_DATA __constant_cpu_to_le32(0xC014000F)
-#define STATUS_ACPI_INVALID_REGION __constant_cpu_to_le32(0xC0140010)
-#define STATUS_ACPI_INVALID_ACCESS_SIZE __constant_cpu_to_le32(0xC0140011)
-#define STATUS_ACPI_ACQUIRE_GLOBAL_LOCK __constant_cpu_to_le32(0xC0140012)
-#define STATUS_ACPI_ALREADY_INITIALIZED __constant_cpu_to_le32(0xC0140013)
-#define STATUS_ACPI_NOT_INITIALIZED __constant_cpu_to_le32(0xC0140014)
-#define STATUS_ACPI_INVALID_MUTEX_LEVEL __constant_cpu_to_le32(0xC0140015)
-#define STATUS_ACPI_MUTEX_NOT_OWNED __constant_cpu_to_le32(0xC0140016)
-#define STATUS_ACPI_MUTEX_NOT_OWNER __constant_cpu_to_le32(0xC0140017)
-#define STATUS_ACPI_RS_ACCESS __constant_cpu_to_le32(0xC0140018)
-#define STATUS_ACPI_INVALID_TABLE __constant_cpu_to_le32(0xC0140019)
-#define STATUS_ACPI_REG_HANDLER_FAILED __constant_cpu_to_le32(0xC0140020)
-#define STATUS_ACPI_POWER_REQUEST_FAILED __constant_cpu_to_le32(0xC0140021)
-#define STATUS_SXS_SECTION_NOT_FOUND __constant_cpu_to_le32(0xC0150001)
-#define STATUS_SXS_CANT_GEN_ACTCTX __constant_cpu_to_le32(0xC0150002)
-#define STATUS_SXS_INVALID_ACTCTXDATA_FORMAT __constant_cpu_to_le32(0xC0150003)
-#define STATUS_SXS_ASSEMBLY_NOT_FOUND __constant_cpu_to_le32(0xC0150004)
-#define STATUS_SXS_MANIFEST_FORMAT_ERROR __constant_cpu_to_le32(0xC0150005)
-#define STATUS_SXS_MANIFEST_PARSE_ERROR __constant_cpu_to_le32(0xC0150006)
-#define STATUS_SXS_ACTIVATION_CONTEXT_DISABLED __constant_cpu_to_le32(0xC0150007)
-#define STATUS_SXS_KEY_NOT_FOUND __constant_cpu_to_le32(0xC0150008)
-#define STATUS_SXS_VERSION_CONFLICT __constant_cpu_to_le32(0xC0150009)
-#define STATUS_SXS_WRONG_SECTION_TYPE __constant_cpu_to_le32(0xC015000A)
-#define STATUS_SXS_THREAD_QUERIES_DISABLED __constant_cpu_to_le32(0xC015000B)
-#define STATUS_SXS_ASSEMBLY_MISSING __constant_cpu_to_le32(0xC015000C)
-#define STATUS_SXS_PROCESS_DEFAULT_ALREADY_SET __constant_cpu_to_le32(0xC015000E)
-#define STATUS_SXS_EARLY_DEACTIVATION __constant_cpu_to_le32(0xC015000F)
-#define STATUS_SXS_INVALID_DEACTIVATION __constant_cpu_to_le32(0xC0150010)
-#define STATUS_SXS_MULTIPLE_DEACTIVATION __constant_cpu_to_le32(0xC0150011)
-#define STATUS_SXS_SYSTEM_DEFAULT_ACTIVATION_CONTEXT_EMPTY __constant_cpu_to_le32(0xC0150012)
-#define STATUS_SXS_PROCESS_TERMINATION_REQUESTED __constant_cpu_to_le32(0xC0150013)
-#define STATUS_SXS_CORRUPT_ACTIVATION_STACK __constant_cpu_to_le32(0xC0150014)
-#define STATUS_SXS_CORRUPTION __constant_cpu_to_le32(0xC0150015)
-#define STATUS_SXS_INVALID_IDENTITY_ATTRIBUTE_VALUE __constant_cpu_to_le32(0xC0150016)
-#define STATUS_SXS_INVALID_IDENTITY_ATTRIBUTE_NAME __constant_cpu_to_le32(0xC0150017)
-#define STATUS_SXS_IDENTITY_DUPLICATE_ATTRIBUTE __constant_cpu_to_le32(0xC0150018)
-#define STATUS_SXS_IDENTITY_PARSE_ERROR __constant_cpu_to_le32(0xC0150019)
-#define STATUS_SXS_COMPONENT_STORE_CORRUPT __constant_cpu_to_le32(0xC015001A)
-#define STATUS_SXS_FILE_HASH_MISMATCH __constant_cpu_to_le32(0xC015001B)
-#define STATUS_SXS_MANIFEST_IDENTITY_SAME_BUT_CONTENTS_DIFFERENT __constant_cpu_to_le32(0xC015001C)
-#define STATUS_SXS_IDENTITIES_DIFFERENT __constant_cpu_to_le32(0xC015001D)
-#define STATUS_SXS_ASSEMBLY_IS_NOT_A_DEPLOYMENT __constant_cpu_to_le32(0xC015001E)
-#define STATUS_SXS_FILE_NOT_PART_OF_ASSEMBLY __constant_cpu_to_le32(0xC015001F)
-#define STATUS_ADVANCED_INSTALLER_FAILED __constant_cpu_to_le32(0xC0150020)
-#define STATUS_XML_ENCODING_MISMATCH __constant_cpu_to_le32(0xC0150021)
-#define STATUS_SXS_MANIFEST_TOO_BIG __constant_cpu_to_le32(0xC0150022)
-#define STATUS_SXS_SETTING_NOT_REGISTERED __constant_cpu_to_le32(0xC0150023)
-#define STATUS_SXS_TRANSACTION_CLOSURE_INCOMPLETE __constant_cpu_to_le32(0xC0150024)
-#define STATUS_SMI_PRIMITIVE_INSTALLER_FAILED __constant_cpu_to_le32(0xC0150025)
-#define STATUS_GENERIC_COMMAND_FAILED __constant_cpu_to_le32(0xC0150026)
-#define STATUS_SXS_FILE_HASH_MISSING __constant_cpu_to_le32(0xC0150027)
-#define STATUS_TRANSACTIONAL_CONFLICT __constant_cpu_to_le32(0xC0190001)
-#define STATUS_INVALID_TRANSACTION __constant_cpu_to_le32(0xC0190002)
-#define STATUS_TRANSACTION_NOT_ACTIVE __constant_cpu_to_le32(0xC0190003)
-#define STATUS_TM_INITIALIZATION_FAILED __constant_cpu_to_le32(0xC0190004)
-#define STATUS_RM_NOT_ACTIVE __constant_cpu_to_le32(0xC0190005)
-#define STATUS_RM_METADATA_CORRUPT __constant_cpu_to_le32(0xC0190006)
-#define STATUS_TRANSACTION_NOT_JOINED __constant_cpu_to_le32(0xC0190007)
-#define STATUS_DIRECTORY_NOT_RM __constant_cpu_to_le32(0xC0190008)
-#define STATUS_TRANSACTIONS_UNSUPPORTED_REMOTE __constant_cpu_to_le32(0xC019000A)
-#define STATUS_LOG_RESIZE_INVALID_SIZE __constant_cpu_to_le32(0xC019000B)
-#define STATUS_REMOTE_FILE_VERSION_MISMATCH __constant_cpu_to_le32(0xC019000C)
-#define STATUS_CRM_PROTOCOL_ALREADY_EXISTS __constant_cpu_to_le32(0xC019000F)
-#define STATUS_TRANSACTION_PROPAGATION_FAILED __constant_cpu_to_le32(0xC0190010)
-#define STATUS_CRM_PROTOCOL_NOT_FOUND __constant_cpu_to_le32(0xC0190011)
-#define STATUS_TRANSACTION_SUPERIOR_EXISTS __constant_cpu_to_le32(0xC0190012)
-#define STATUS_TRANSACTION_REQUEST_NOT_VALID __constant_cpu_to_le32(0xC0190013)
-#define STATUS_TRANSACTION_NOT_REQUESTED __constant_cpu_to_le32(0xC0190014)
-#define STATUS_TRANSACTION_ALREADY_ABORTED __constant_cpu_to_le32(0xC0190015)
-#define STATUS_TRANSACTION_ALREADY_COMMITTED __constant_cpu_to_le32(0xC0190016)
-#define STATUS_TRANSACTION_INVALID_MARSHALL_BUFFER __constant_cpu_to_le32(0xC0190017)
-#define STATUS_CURRENT_TRANSACTION_NOT_VALID __constant_cpu_to_le32(0xC0190018)
-#define STATUS_LOG_GROWTH_FAILED __constant_cpu_to_le32(0xC0190019)
-#define STATUS_OBJECT_NO_LONGER_EXISTS __constant_cpu_to_le32(0xC0190021)
-#define STATUS_STREAM_MINIVERSION_NOT_FOUND __constant_cpu_to_le32(0xC0190022)
-#define STATUS_STREAM_MINIVERSION_NOT_VALID __constant_cpu_to_le32(0xC0190023)
-#define STATUS_MINIVERSION_INACCESSIBLE_FROM_SPECIFIED_TRANSACTION __constant_cpu_to_le32(0xC0190024)
-#define STATUS_CANT_OPEN_MINIVERSION_WITH_MODIFY_INTENT __constant_cpu_to_le32(0xC0190025)
-#define STATUS_CANT_CREATE_MORE_STREAM_MINIVERSIONS __constant_cpu_to_le32(0xC0190026)
-#define STATUS_HANDLE_NO_LONGER_VALID __constant_cpu_to_le32(0xC0190028)
-#define STATUS_LOG_CORRUPTION_DETECTED __constant_cpu_to_le32(0xC0190030)
-#define STATUS_RM_DISCONNECTED __constant_cpu_to_le32(0xC0190032)
-#define STATUS_ENLISTMENT_NOT_SUPERIOR __constant_cpu_to_le32(0xC0190033)
-#define STATUS_FILE_IDENTITY_NOT_PERSISTENT __constant_cpu_to_le32(0xC0190036)
-#define STATUS_CANT_BREAK_TRANSACTIONAL_DEPENDENCY __constant_cpu_to_le32(0xC0190037)
-#define STATUS_CANT_CROSS_RM_BOUNDARY __constant_cpu_to_le32(0xC0190038)
-#define STATUS_TXF_DIR_NOT_EMPTY __constant_cpu_to_le32(0xC0190039)
-#define STATUS_INDOUBT_TRANSACTIONS_EXIST __constant_cpu_to_le32(0xC019003A)
-#define STATUS_TM_VOLATILE __constant_cpu_to_le32(0xC019003B)
-#define STATUS_ROLLBACK_TIMER_EXPIRED __constant_cpu_to_le32(0xC019003C)
-#define STATUS_TXF_ATTRIBUTE_CORRUPT __constant_cpu_to_le32(0xC019003D)
-#define STATUS_EFS_NOT_ALLOWED_IN_TRANSACTION __constant_cpu_to_le32(0xC019003E)
-#define STATUS_TRANSACTIONAL_OPEN_NOT_ALLOWED __constant_cpu_to_le32(0xC019003F)
-#define STATUS_TRANSACTED_MAPPING_UNSUPPORTED_REMOTE __constant_cpu_to_le32(0xC0190040)
-#define STATUS_TRANSACTION_REQUIRED_PROMOTION __constant_cpu_to_le32(0xC0190043)
-#define STATUS_CANNOT_EXECUTE_FILE_IN_TRANSACTION __constant_cpu_to_le32(0xC0190044)
-#define STATUS_TRANSACTIONS_NOT_FROZEN __constant_cpu_to_le32(0xC0190045)
-#define STATUS_TRANSACTION_FREEZE_IN_PROGRESS __constant_cpu_to_le32(0xC0190046)
-#define STATUS_NOT_SNAPSHOT_VOLUME __constant_cpu_to_le32(0xC0190047)
-#define STATUS_NO_SAVEPOINT_WITH_OPEN_FILES __constant_cpu_to_le32(0xC0190048)
-#define STATUS_SPARSE_NOT_ALLOWED_IN_TRANSACTION __constant_cpu_to_le32(0xC0190049)
-#define STATUS_TM_IDENTITY_MISMATCH __constant_cpu_to_le32(0xC019004A)
-#define STATUS_FLOATED_SECTION __constant_cpu_to_le32(0xC019004B)
-#define STATUS_CANNOT_ACCEPT_TRANSACTED_WORK __constant_cpu_to_le32(0xC019004C)
-#define STATUS_CANNOT_ABORT_TRANSACTIONS __constant_cpu_to_le32(0xC019004D)
-#define STATUS_TRANSACTION_NOT_FOUND __constant_cpu_to_le32(0xC019004E)
-#define STATUS_RESOURCEMANAGER_NOT_FOUND __constant_cpu_to_le32(0xC019004F)
-#define STATUS_ENLISTMENT_NOT_FOUND __constant_cpu_to_le32(0xC0190050)
-#define STATUS_TRANSACTIONMANAGER_NOT_FOUND __constant_cpu_to_le32(0xC0190051)
-#define STATUS_TRANSACTIONMANAGER_NOT_ONLINE __constant_cpu_to_le32(0xC0190052)
-#define STATUS_TRANSACTIONMANAGER_RECOVERY_NAME_COLLISION __constant_cpu_to_le32(0xC0190053)
-#define STATUS_TRANSACTION_NOT_ROOT __constant_cpu_to_le32(0xC0190054)
-#define STATUS_TRANSACTION_OBJECT_EXPIRED __constant_cpu_to_le32(0xC0190055)
-#define STATUS_COMPRESSION_NOT_ALLOWED_IN_TRANSACTION __constant_cpu_to_le32(0xC0190056)
-#define STATUS_TRANSACTION_RESPONSE_NOT_ENLISTED __constant_cpu_to_le32(0xC0190057)
-#define STATUS_TRANSACTION_RECORD_TOO_LONG __constant_cpu_to_le32(0xC0190058)
-#define STATUS_NO_LINK_TRACKING_IN_TRANSACTION __constant_cpu_to_le32(0xC0190059)
-#define STATUS_OPERATION_NOT_SUPPORTED_IN_TRANSACTION __constant_cpu_to_le32(0xC019005A)
-#define STATUS_TRANSACTION_INTEGRITY_VIOLATED __constant_cpu_to_le32(0xC019005B)
-#define STATUS_LOG_SECTOR_INVALID __constant_cpu_to_le32(0xC01A0001)
-#define STATUS_LOG_SECTOR_PARITY_INVALID __constant_cpu_to_le32(0xC01A0002)
-#define STATUS_LOG_SECTOR_REMAPPED __constant_cpu_to_le32(0xC01A0003)
-#define STATUS_LOG_BLOCK_INCOMPLETE __constant_cpu_to_le32(0xC01A0004)
-#define STATUS_LOG_INVALID_RANGE __constant_cpu_to_le32(0xC01A0005)
-#define STATUS_LOG_BLOCKS_EXHAUSTED __constant_cpu_to_le32(0xC01A0006)
-#define STATUS_LOG_READ_CONTEXT_INVALID __constant_cpu_to_le32(0xC01A0007)
-#define STATUS_LOG_RESTART_INVALID __constant_cpu_to_le32(0xC01A0008)
-#define STATUS_LOG_BLOCK_VERSION __constant_cpu_to_le32(0xC01A0009)
-#define STATUS_LOG_BLOCK_INVALID __constant_cpu_to_le32(0xC01A000A)
-#define STATUS_LOG_READ_MODE_INVALID __constant_cpu_to_le32(0xC01A000B)
-#define STATUS_LOG_METADATA_CORRUPT __constant_cpu_to_le32(0xC01A000D)
-#define STATUS_LOG_METADATA_INVALID __constant_cpu_to_le32(0xC01A000E)
-#define STATUS_LOG_METADATA_INCONSISTENT __constant_cpu_to_le32(0xC01A000F)
-#define STATUS_LOG_RESERVATION_INVALID __constant_cpu_to_le32(0xC01A0010)
-#define STATUS_LOG_CANT_DELETE __constant_cpu_to_le32(0xC01A0011)
-#define STATUS_LOG_CONTAINER_LIMIT_EXCEEDED __constant_cpu_to_le32(0xC01A0012)
-#define STATUS_LOG_START_OF_LOG __constant_cpu_to_le32(0xC01A0013)
-#define STATUS_LOG_POLICY_ALREADY_INSTALLED __constant_cpu_to_le32(0xC01A0014)
-#define STATUS_LOG_POLICY_NOT_INSTALLED __constant_cpu_to_le32(0xC01A0015)
-#define STATUS_LOG_POLICY_INVALID __constant_cpu_to_le32(0xC01A0016)
-#define STATUS_LOG_POLICY_CONFLICT __constant_cpu_to_le32(0xC01A0017)
-#define STATUS_LOG_PINNED_ARCHIVE_TAIL __constant_cpu_to_le32(0xC01A0018)
-#define STATUS_LOG_RECORD_NONEXISTENT __constant_cpu_to_le32(0xC01A0019)
-#define STATUS_LOG_RECORDS_RESERVED_INVALID __constant_cpu_to_le32(0xC01A001A)
-#define STATUS_LOG_SPACE_RESERVED_INVALID __constant_cpu_to_le32(0xC01A001B)
-#define STATUS_LOG_TAIL_INVALID __constant_cpu_to_le32(0xC01A001C)
-#define STATUS_LOG_FULL __constant_cpu_to_le32(0xC01A001D)
-#define STATUS_LOG_MULTIPLEXED __constant_cpu_to_le32(0xC01A001E)
-#define STATUS_LOG_DEDICATED __constant_cpu_to_le32(0xC01A001F)
-#define STATUS_LOG_ARCHIVE_NOT_IN_PROGRESS __constant_cpu_to_le32(0xC01A0020)
-#define STATUS_LOG_ARCHIVE_IN_PROGRESS __constant_cpu_to_le32(0xC01A0021)
-#define STATUS_LOG_EPHEMERAL __constant_cpu_to_le32(0xC01A0022)
-#define STATUS_LOG_NOT_ENOUGH_CONTAINERS __constant_cpu_to_le32(0xC01A0023)
-#define STATUS_LOG_CLIENT_ALREADY_REGISTERED __constant_cpu_to_le32(0xC01A0024)
-#define STATUS_LOG_CLIENT_NOT_REGISTERED __constant_cpu_to_le32(0xC01A0025)
-#define STATUS_LOG_FULL_HANDLER_IN_PROGRESS __constant_cpu_to_le32(0xC01A0026)
-#define STATUS_LOG_CONTAINER_READ_FAILED __constant_cpu_to_le32(0xC01A0027)
-#define STATUS_LOG_CONTAINER_WRITE_FAILED __constant_cpu_to_le32(0xC01A0028)
-#define STATUS_LOG_CONTAINER_OPEN_FAILED __constant_cpu_to_le32(0xC01A0029)
-#define STATUS_LOG_CONTAINER_STATE_INVALID __constant_cpu_to_le32(0xC01A002A)
-#define STATUS_LOG_STATE_INVALID __constant_cpu_to_le32(0xC01A002B)
-#define STATUS_LOG_PINNED __constant_cpu_to_le32(0xC01A002C)
-#define STATUS_LOG_METADATA_FLUSH_FAILED __constant_cpu_to_le32(0xC01A002D)
-#define STATUS_LOG_INCONSISTENT_SECURITY __constant_cpu_to_le32(0xC01A002E)
-#define STATUS_LOG_APPENDED_FLUSH_FAILED __constant_cpu_to_le32(0xC01A002F)
-#define STATUS_LOG_PINNED_RESERVATION __constant_cpu_to_le32(0xC01A0030)
-#define STATUS_VIDEO_HUNG_DISPLAY_DRIVER_THREAD __constant_cpu_to_le32(0xC01B00EA)
-#define STATUS_FLT_NO_HANDLER_DEFINED __constant_cpu_to_le32(0xC01C0001)
-#define STATUS_FLT_CONTEXT_ALREADY_DEFINED __constant_cpu_to_le32(0xC01C0002)
-#define STATUS_FLT_INVALID_ASYNCHRONOUS_REQUEST __constant_cpu_to_le32(0xC01C0003)
-#define STATUS_FLT_DISALLOW_FAST_IO __constant_cpu_to_le32(0xC01C0004)
-#define STATUS_FLT_INVALID_NAME_REQUEST __constant_cpu_to_le32(0xC01C0005)
-#define STATUS_FLT_NOT_SAFE_TO_POST_OPERATION __constant_cpu_to_le32(0xC01C0006)
-#define STATUS_FLT_NOT_INITIALIZED __constant_cpu_to_le32(0xC01C0007)
-#define STATUS_FLT_FILTER_NOT_READY __constant_cpu_to_le32(0xC01C0008)
-#define STATUS_FLT_POST_OPERATION_CLEANUP __constant_cpu_to_le32(0xC01C0009)
-#define STATUS_FLT_INTERNAL_ERROR __constant_cpu_to_le32(0xC01C000A)
-#define STATUS_FLT_DELETING_OBJECT __constant_cpu_to_le32(0xC01C000B)
-#define STATUS_FLT_MUST_BE_NONPAGED_POOL __constant_cpu_to_le32(0xC01C000C)
-#define STATUS_FLT_DUPLICATE_ENTRY __constant_cpu_to_le32(0xC01C000D)
-#define STATUS_FLT_CBDQ_DISABLED __constant_cpu_to_le32(0xC01C000E)
-#define STATUS_FLT_DO_NOT_ATTACH __constant_cpu_to_le32(0xC01C000F)
-#define STATUS_FLT_DO_NOT_DETACH __constant_cpu_to_le32(0xC01C0010)
-#define STATUS_FLT_INSTANCE_ALTITUDE_COLLISION __constant_cpu_to_le32(0xC01C0011)
-#define STATUS_FLT_INSTANCE_NAME_COLLISION __constant_cpu_to_le32(0xC01C0012)
-#define STATUS_FLT_FILTER_NOT_FOUND __constant_cpu_to_le32(0xC01C0013)
-#define STATUS_FLT_VOLUME_NOT_FOUND __constant_cpu_to_le32(0xC01C0014)
-#define STATUS_FLT_INSTANCE_NOT_FOUND __constant_cpu_to_le32(0xC01C0015)
-#define STATUS_FLT_CONTEXT_ALLOCATION_NOT_FOUND __constant_cpu_to_le32(0xC01C0016)
-#define STATUS_FLT_INVALID_CONTEXT_REGISTRATION __constant_cpu_to_le32(0xC01C0017)
-#define STATUS_FLT_NAME_CACHE_MISS __constant_cpu_to_le32(0xC01C0018)
-#define STATUS_FLT_NO_DEVICE_OBJECT __constant_cpu_to_le32(0xC01C0019)
-#define STATUS_FLT_VOLUME_ALREADY_MOUNTED __constant_cpu_to_le32(0xC01C001A)
-#define STATUS_FLT_ALREADY_ENLISTED __constant_cpu_to_le32(0xC01C001B)
-#define STATUS_FLT_CONTEXT_ALREADY_LINKED __constant_cpu_to_le32(0xC01C001C)
-#define STATUS_FLT_NO_WAITER_FOR_REPLY __constant_cpu_to_le32(0xC01C0020)
-#define STATUS_MONITOR_NO_DESCRIPTOR __constant_cpu_to_le32(0xC01D0001)
-#define STATUS_MONITOR_UNKNOWN_DESCRIPTOR_FORMAT __constant_cpu_to_le32(0xC01D0002)
-#define STATUS_MONITOR_INVALID_DESCRIPTOR_CHECKSUM __constant_cpu_to_le32(0xC01D0003)
-#define STATUS_MONITOR_INVALID_STANDARD_TIMING_BLOCK __constant_cpu_to_le32(0xC01D0004)
-#define STATUS_MONITOR_WMI_DATABLOCK_REGISTRATION_FAILED __constant_cpu_to_le32(0xC01D0005)
-#define STATUS_MONITOR_INVALID_SERIAL_NUMBER_MONDSC_BLOCK __constant_cpu_to_le32(0xC01D0006)
-#define STATUS_MONITOR_INVALID_USER_FRIENDLY_MONDSC_BLOCK __constant_cpu_to_le32(0xC01D0007)
-#define STATUS_MONITOR_NO_MORE_DESCRIPTOR_DATA __constant_cpu_to_le32(0xC01D0008)
-#define STATUS_MONITOR_INVALID_DETAILED_TIMING_BLOCK __constant_cpu_to_le32(0xC01D0009)
-#define STATUS_GRAPHICS_NOT_EXCLUSIVE_MODE_OWNER __constant_cpu_to_le32(0xC01E0000)
-#define STATUS_GRAPHICS_INSUFFICIENT_DMA_BUFFER __constant_cpu_to_le32(0xC01E0001)
-#define STATUS_GRAPHICS_INVALID_DISPLAY_ADAPTER __constant_cpu_to_le32(0xC01E0002)
-#define STATUS_GRAPHICS_ADAPTER_WAS_RESET __constant_cpu_to_le32(0xC01E0003)
-#define STATUS_GRAPHICS_INVALID_DRIVER_MODEL __constant_cpu_to_le32(0xC01E0004)
-#define STATUS_GRAPHICS_PRESENT_MODE_CHANGED __constant_cpu_to_le32(0xC01E0005)
-#define STATUS_GRAPHICS_PRESENT_OCCLUDED __constant_cpu_to_le32(0xC01E0006)
-#define STATUS_GRAPHICS_PRESENT_DENIED __constant_cpu_to_le32(0xC01E0007)
-#define STATUS_GRAPHICS_CANNOTCOLORCONVERT __constant_cpu_to_le32(0xC01E0008)
-#define STATUS_GRAPHICS_NO_VIDEO_MEMORY __constant_cpu_to_le32(0xC01E0100)
-#define STATUS_GRAPHICS_CANT_LOCK_MEMORY __constant_cpu_to_le32(0xC01E0101)
-#define STATUS_GRAPHICS_ALLOCATION_BUSY __constant_cpu_to_le32(0xC01E0102)
-#define STATUS_GRAPHICS_TOO_MANY_REFERENCES __constant_cpu_to_le32(0xC01E0103)
-#define STATUS_GRAPHICS_TRY_AGAIN_LATER __constant_cpu_to_le32(0xC01E0104)
-#define STATUS_GRAPHICS_TRY_AGAIN_NOW __constant_cpu_to_le32(0xC01E0105)
-#define STATUS_GRAPHICS_ALLOCATION_INVALID __constant_cpu_to_le32(0xC01E0106)
-#define STATUS_GRAPHICS_UNSWIZZLING_APERTURE_UNAVAILABLE __constant_cpu_to_le32(0xC01E0107)
-#define STATUS_GRAPHICS_UNSWIZZLING_APERTURE_UNSUPPORTED __constant_cpu_to_le32(0xC01E0108)
-#define STATUS_GRAPHICS_CANT_EVICT_PINNED_ALLOCATION __constant_cpu_to_le32(0xC01E0109)
-#define STATUS_GRAPHICS_INVALID_ALLOCATION_USAGE __constant_cpu_to_le32(0xC01E0110)
-#define STATUS_GRAPHICS_CANT_RENDER_LOCKED_ALLOCATION __constant_cpu_to_le32(0xC01E0111)
-#define STATUS_GRAPHICS_ALLOCATION_CLOSED __constant_cpu_to_le32(0xC01E0112)
-#define STATUS_GRAPHICS_INVALID_ALLOCATION_INSTANCE __constant_cpu_to_le32(0xC01E0113)
-#define STATUS_GRAPHICS_INVALID_ALLOCATION_HANDLE __constant_cpu_to_le32(0xC01E0114)
-#define STATUS_GRAPHICS_WRONG_ALLOCATION_DEVICE __constant_cpu_to_le32(0xC01E0115)
-#define STATUS_GRAPHICS_ALLOCATION_CONTENT_LOST __constant_cpu_to_le32(0xC01E0116)
-#define STATUS_GRAPHICS_GPU_EXCEPTION_ON_DEVICE __constant_cpu_to_le32(0xC01E0200)
-#define STATUS_GRAPHICS_INVALID_VIDPN_TOPOLOGY __constant_cpu_to_le32(0xC01E0300)
-#define STATUS_GRAPHICS_VIDPN_TOPOLOGY_NOT_SUPPORTED __constant_cpu_to_le32(0xC01E0301)
-#define STATUS_GRAPHICS_VIDPN_TOPOLOGY_CURRENTLY_NOT_SUPPORTED __constant_cpu_to_le32(0xC01E0302)
-#define STATUS_GRAPHICS_INVALID_VIDPN __constant_cpu_to_le32(0xC01E0303)
-#define STATUS_GRAPHICS_INVALID_VIDEO_PRESENT_SOURCE __constant_cpu_to_le32(0xC01E0304)
-#define STATUS_GRAPHICS_INVALID_VIDEO_PRESENT_TARGET __constant_cpu_to_le32(0xC01E0305)
-#define STATUS_GRAPHICS_VIDPN_MODALITY_NOT_SUPPORTED __constant_cpu_to_le32(0xC01E0306)
-#define STATUS_GRAPHICS_INVALID_VIDPN_SOURCEMODESET __constant_cpu_to_le32(0xC01E0308)
-#define STATUS_GRAPHICS_INVALID_VIDPN_TARGETMODESET __constant_cpu_to_le32(0xC01E0309)
-#define STATUS_GRAPHICS_INVALID_FREQUENCY __constant_cpu_to_le32(0xC01E030A)
-#define STATUS_GRAPHICS_INVALID_ACTIVE_REGION __constant_cpu_to_le32(0xC01E030B)
-#define STATUS_GRAPHICS_INVALID_TOTAL_REGION __constant_cpu_to_le32(0xC01E030C)
-#define STATUS_GRAPHICS_INVALID_VIDEO_PRESENT_SOURCE_MODE __constant_cpu_to_le32(0xC01E0310)
-#define STATUS_GRAPHICS_INVALID_VIDEO_PRESENT_TARGET_MODE __constant_cpu_to_le32(0xC01E0311)
-#define STATUS_GRAPHICS_PINNED_MODE_MUST_REMAIN_IN_SET __constant_cpu_to_le32(0xC01E0312)
-#define STATUS_GRAPHICS_PATH_ALREADY_IN_TOPOLOGY __constant_cpu_to_le32(0xC01E0313)
-#define STATUS_GRAPHICS_MODE_ALREADY_IN_MODESET __constant_cpu_to_le32(0xC01E0314)
-#define STATUS_GRAPHICS_INVALID_VIDEOPRESENTSOURCESET __constant_cpu_to_le32(0xC01E0315)
-#define STATUS_GRAPHICS_INVALID_VIDEOPRESENTTARGETSET __constant_cpu_to_le32(0xC01E0316)
-#define STATUS_GRAPHICS_SOURCE_ALREADY_IN_SET __constant_cpu_to_le32(0xC01E0317)
-#define STATUS_GRAPHICS_TARGET_ALREADY_IN_SET __constant_cpu_to_le32(0xC01E0318)
-#define STATUS_GRAPHICS_INVALID_VIDPN_PRESENT_PATH __constant_cpu_to_le32(0xC01E0319)
-#define STATUS_GRAPHICS_NO_RECOMMENDED_VIDPN_TOPOLOGY __constant_cpu_to_le32(0xC01E031A)
-#define STATUS_GRAPHICS_INVALID_MONITOR_FREQUENCYRANGESET __constant_cpu_to_le32(0xC01E031B)
-#define STATUS_GRAPHICS_INVALID_MONITOR_FREQUENCYRANGE __constant_cpu_to_le32(0xC01E031C)
-#define STATUS_GRAPHICS_FREQUENCYRANGE_NOT_IN_SET __constant_cpu_to_le32(0xC01E031D)
-#define STATUS_GRAPHICS_FREQUENCYRANGE_ALREADY_IN_SET __constant_cpu_to_le32(0xC01E031F)
-#define STATUS_GRAPHICS_STALE_MODESET __constant_cpu_to_le32(0xC01E0320)
-#define STATUS_GRAPHICS_INVALID_MONITOR_SOURCEMODESET __constant_cpu_to_le32(0xC01E0321)
-#define STATUS_GRAPHICS_INVALID_MONITOR_SOURCE_MODE __constant_cpu_to_le32(0xC01E0322)
-#define STATUS_GRAPHICS_NO_RECOMMENDED_FUNCTIONAL_VIDPN __constant_cpu_to_le32(0xC01E0323)
-#define STATUS_GRAPHICS_MODE_ID_MUST_BE_UNIQUE __constant_cpu_to_le32(0xC01E0324)
-#define STATUS_GRAPHICS_EMPTY_ADAPTER_MONITOR_MODE_SUPPORT_INTERSECTION __constant_cpu_to_le32(0xC01E0325)
-#define STATUS_GRAPHICS_VIDEO_PRESENT_TARGETS_LESS_THAN_SOURCES __constant_cpu_to_le32(0xC01E0326)
-#define STATUS_GRAPHICS_PATH_NOT_IN_TOPOLOGY __constant_cpu_to_le32(0xC01E0327)
-#define STATUS_GRAPHICS_ADAPTER_MUST_HAVE_AT_LEAST_ONE_SOURCE __constant_cpu_to_le32(0xC01E0328)
-#define STATUS_GRAPHICS_ADAPTER_MUST_HAVE_AT_LEAST_ONE_TARGET __constant_cpu_to_le32(0xC01E0329)
-#define STATUS_GRAPHICS_INVALID_MONITORDESCRIPTORSET __constant_cpu_to_le32(0xC01E032A)
-#define STATUS_GRAPHICS_INVALID_MONITORDESCRIPTOR __constant_cpu_to_le32(0xC01E032B)
-#define STATUS_GRAPHICS_MONITORDESCRIPTOR_NOT_IN_SET __constant_cpu_to_le32(0xC01E032C)
-#define STATUS_GRAPHICS_MONITORDESCRIPTOR_ALREADY_IN_SET __constant_cpu_to_le32(0xC01E032D)
-#define STATUS_GRAPHICS_MONITORDESCRIPTOR_ID_MUST_BE_UNIQUE __constant_cpu_to_le32(0xC01E032E)
-#define STATUS_GRAPHICS_INVALID_VIDPN_TARGET_SUBSET_TYPE __constant_cpu_to_le32(0xC01E032F)
-#define STATUS_GRAPHICS_RESOURCES_NOT_RELATED __constant_cpu_to_le32(0xC01E0330)
-#define STATUS_GRAPHICS_SOURCE_ID_MUST_BE_UNIQUE __constant_cpu_to_le32(0xC01E0331)
-#define STATUS_GRAPHICS_TARGET_ID_MUST_BE_UNIQUE __constant_cpu_to_le32(0xC01E0332)
-#define STATUS_GRAPHICS_NO_AVAILABLE_VIDPN_TARGET __constant_cpu_to_le32(0xC01E0333)
-#define STATUS_GRAPHICS_MONITOR_COULD_NOT_BE_ASSOCIATED_WITH_ADAPTER __constant_cpu_to_le32(0xC01E0334)
-#define STATUS_GRAPHICS_NO_VIDPNMGR __constant_cpu_to_le32(0xC01E0335)
-#define STATUS_GRAPHICS_NO_ACTIVE_VIDPN __constant_cpu_to_le32(0xC01E0336)
-#define STATUS_GRAPHICS_STALE_VIDPN_TOPOLOGY __constant_cpu_to_le32(0xC01E0337)
-#define STATUS_GRAPHICS_MONITOR_NOT_CONNECTED __constant_cpu_to_le32(0xC01E0338)
-#define STATUS_GRAPHICS_SOURCE_NOT_IN_TOPOLOGY __constant_cpu_to_le32(0xC01E0339)
-#define STATUS_GRAPHICS_INVALID_PRIMARYSURFACE_SIZE __constant_cpu_to_le32(0xC01E033A)
-#define STATUS_GRAPHICS_INVALID_VISIBLEREGION_SIZE __constant_cpu_to_le32(0xC01E033B)
-#define STATUS_GRAPHICS_INVALID_STRIDE __constant_cpu_to_le32(0xC01E033C)
-#define STATUS_GRAPHICS_INVALID_PIXELFORMAT __constant_cpu_to_le32(0xC01E033D)
-#define STATUS_GRAPHICS_INVALID_COLORBASIS __constant_cpu_to_le32(0xC01E033E)
-#define STATUS_GRAPHICS_INVALID_PIXELVALUEACCESSMODE __constant_cpu_to_le32(0xC01E033F)
-#define STATUS_GRAPHICS_TARGET_NOT_IN_TOPOLOGY __constant_cpu_to_le32(0xC01E0340)
-#define STATUS_GRAPHICS_NO_DISPLAY_MODE_MANAGEMENT_SUPPORT __constant_cpu_to_le32(0xC01E0341)
-#define STATUS_GRAPHICS_VIDPN_SOURCE_IN_USE __constant_cpu_to_le32(0xC01E0342)
-#define STATUS_GRAPHICS_CANT_ACCESS_ACTIVE_VIDPN __constant_cpu_to_le32(0xC01E0343)
-#define STATUS_GRAPHICS_INVALID_PATH_IMPORTANCE_ORDINAL __constant_cpu_to_le32(0xC01E0344)
-#define STATUS_GRAPHICS_INVALID_PATH_CONTENT_GEOMETRY_TRANSFORMATION __constant_cpu_to_le32(0xC01E0345)
-#define STATUS_GRAPHICS_PATH_CONTENT_GEOMETRY_TRANSFORMATION_NOT_SUPPORTED __constant_cpu_to_le32(0xC01E0346)
-#define STATUS_GRAPHICS_INVALID_GAMMA_RAMP __constant_cpu_to_le32(0xC01E0347)
-#define STATUS_GRAPHICS_GAMMA_RAMP_NOT_SUPPORTED __constant_cpu_to_le32(0xC01E0348)
-#define STATUS_GRAPHICS_MULTISAMPLING_NOT_SUPPORTED __constant_cpu_to_le32(0xC01E0349)
-#define STATUS_GRAPHICS_MODE_NOT_IN_MODESET __constant_cpu_to_le32(0xC01E034A)
-#define STATUS_GRAPHICS_INVALID_VIDPN_TOPOLOGY_RECOMMENDATION_REASON __constant_cpu_to_le32(0xC01E034D)
-#define STATUS_GRAPHICS_INVALID_PATH_CONTENT_TYPE __constant_cpu_to_le32(0xC01E034E)
-#define STATUS_GRAPHICS_INVALID_COPYPROTECTION_TYPE __constant_cpu_to_le32(0xC01E034F)
-#define STATUS_GRAPHICS_UNASSIGNED_MODESET_ALREADY_EXISTS __constant_cpu_to_le32(0xC01E0350)
-#define STATUS_GRAPHICS_INVALID_SCANLINE_ORDERING __constant_cpu_to_le32(0xC01E0352)
-#define STATUS_GRAPHICS_TOPOLOGY_CHANGES_NOT_ALLOWED __constant_cpu_to_le32(0xC01E0353)
-#define STATUS_GRAPHICS_NO_AVAILABLE_IMPORTANCE_ORDINALS __constant_cpu_to_le32(0xC01E0354)
-#define STATUS_GRAPHICS_INCOMPATIBLE_PRIVATE_FORMAT __constant_cpu_to_le32(0xC01E0355)
-#define STATUS_GRAPHICS_INVALID_MODE_PRUNING_ALGORITHM __constant_cpu_to_le32(0xC01E0356)
-#define STATUS_GRAPHICS_INVALID_MONITOR_CAPABILITY_ORIGIN __constant_cpu_to_le32(0xC01E0357)
-#define STATUS_GRAPHICS_INVALID_MONITOR_FREQUENCYRANGE_CONSTRAINT __constant_cpu_to_le32(0xC01E0358)
-#define STATUS_GRAPHICS_MAX_NUM_PATHS_REACHED __constant_cpu_to_le32(0xC01E0359)
-#define STATUS_GRAPHICS_CANCEL_VIDPN_TOPOLOGY_AUGMENTATION __constant_cpu_to_le32(0xC01E035A)
-#define STATUS_GRAPHICS_INVALID_CLIENT_TYPE __constant_cpu_to_le32(0xC01E035B)
-#define STATUS_GRAPHICS_CLIENTVIDPN_NOT_SET __constant_cpu_to_le32(0xC01E035C)
-#define STATUS_GRAPHICS_SPECIFIED_CHILD_ALREADY_CONNECTED __constant_cpu_to_le32(0xC01E0400)
-#define STATUS_GRAPHICS_CHILD_DESCRIPTOR_NOT_SUPPORTED __constant_cpu_to_le32(0xC01E0401)
-#define STATUS_GRAPHICS_NOT_A_LINKED_ADAPTER __constant_cpu_to_le32(0xC01E0430)
-#define STATUS_GRAPHICS_LEADLINK_NOT_ENUMERATED __constant_cpu_to_le32(0xC01E0431)
-#define STATUS_GRAPHICS_CHAINLINKS_NOT_ENUMERATED __constant_cpu_to_le32(0xC01E0432)
-#define STATUS_GRAPHICS_ADAPTER_CHAIN_NOT_READY __constant_cpu_to_le32(0xC01E0433)
-#define STATUS_GRAPHICS_CHAINLINKS_NOT_STARTED __constant_cpu_to_le32(0xC01E0434)
-#define STATUS_GRAPHICS_CHAINLINKS_NOT_POWERED_ON __constant_cpu_to_le32(0xC01E0435)
-#define STATUS_GRAPHICS_INCONSISTENT_DEVICE_LINK_STATE __constant_cpu_to_le32(0xC01E0436)
-#define STATUS_GRAPHICS_NOT_POST_DEVICE_DRIVER __constant_cpu_to_le32(0xC01E0438)
-#define STATUS_GRAPHICS_ADAPTER_ACCESS_NOT_EXCLUDED __constant_cpu_to_le32(0xC01E043B)
-#define STATUS_GRAPHICS_OPM_PROTECTED_OUTPUT_DOES_NOT_HAVE_COPP_SEMANTICS __constant_cpu_to_le32(0xC01E051C)
-#define STATUS_GRAPHICS_OPM_INVALID_INFORMATION_REQUEST __constant_cpu_to_le32(0xC01E051D)
-#define STATUS_GRAPHICS_OPM_DRIVER_INTERNAL_ERROR __constant_cpu_to_le32(0xC01E051E)
-#define STATUS_GRAPHICS_OPM_PROTECTED_OUTPUT_DOES_NOT_HAVE_OPM_SEMANTICS __constant_cpu_to_le32(0xC01E051F)
-#define STATUS_GRAPHICS_OPM_SIGNALING_NOT_SUPPORTED __constant_cpu_to_le32(0xC01E0520)
-#define STATUS_GRAPHICS_OPM_INVALID_CONFIGURATION_REQUEST __constant_cpu_to_le32(0xC01E0521)
-#define STATUS_GRAPHICS_OPM_NOT_SUPPORTED __constant_cpu_to_le32(0xC01E0500)
-#define STATUS_GRAPHICS_COPP_NOT_SUPPORTED __constant_cpu_to_le32(0xC01E0501)
-#define STATUS_GRAPHICS_UAB_NOT_SUPPORTED __constant_cpu_to_le32(0xC01E0502)
-#define STATUS_GRAPHICS_OPM_INVALID_ENCRYPTED_PARAMETERS __constant_cpu_to_le32(0xC01E0503)
-#define STATUS_GRAPHICS_OPM_PARAMETER_ARRAY_TOO_SMALL __constant_cpu_to_le32(0xC01E0504)
-#define STATUS_GRAPHICS_OPM_NO_PROTECTED_OUTPUTS_EXIST __constant_cpu_to_le32(0xC01E0505)
-#define STATUS_GRAPHICS_PVP_NO_DISPLAY_DEVICE_CORRESPONDS_TO_NAME __constant_cpu_to_le32(0xC01E0506)
-#define STATUS_GRAPHICS_PVP_DISPLAY_DEVICE_NOT_ATTACHED_TO_DESKTOP __constant_cpu_to_le32(0xC01E0507)
-#define STATUS_GRAPHICS_PVP_MIRRORING_DEVICES_NOT_SUPPORTED __constant_cpu_to_le32(0xC01E0508)
-#define STATUS_GRAPHICS_OPM_INVALID_POINTER __constant_cpu_to_le32(0xC01E050A)
-#define STATUS_GRAPHICS_OPM_INTERNAL_ERROR __constant_cpu_to_le32(0xC01E050B)
-#define STATUS_GRAPHICS_OPM_INVALID_HANDLE __constant_cpu_to_le32(0xC01E050C)
-#define STATUS_GRAPHICS_PVP_NO_MONITORS_CORRESPOND_TO_DISPLAY_DEVICE __constant_cpu_to_le32(0xC01E050D)
-#define STATUS_GRAPHICS_PVP_INVALID_CERTIFICATE_LENGTH __constant_cpu_to_le32(0xC01E050E)
-#define STATUS_GRAPHICS_OPM_SPANNING_MODE_ENABLED __constant_cpu_to_le32(0xC01E050F)
-#define STATUS_GRAPHICS_OPM_THEATER_MODE_ENABLED __constant_cpu_to_le32(0xC01E0510)
-#define STATUS_GRAPHICS_PVP_HFS_FAILED __constant_cpu_to_le32(0xC01E0511)
-#define STATUS_GRAPHICS_OPM_INVALID_SRM __constant_cpu_to_le32(0xC01E0512)
-#define STATUS_GRAPHICS_OPM_OUTPUT_DOES_NOT_SUPPORT_HDCP __constant_cpu_to_le32(0xC01E0513)
-#define STATUS_GRAPHICS_OPM_OUTPUT_DOES_NOT_SUPPORT_ACP __constant_cpu_to_le32(0xC01E0514)
-#define STATUS_GRAPHICS_OPM_OUTPUT_DOES_NOT_SUPPORT_CGMSA __constant_cpu_to_le32(0xC01E0515)
-#define STATUS_GRAPHICS_OPM_HDCP_SRM_NEVER_SET __constant_cpu_to_le32(0xC01E0516)
-#define STATUS_GRAPHICS_OPM_RESOLUTION_TOO_HIGH __constant_cpu_to_le32(0xC01E0517)
-#define STATUS_GRAPHICS_OPM_ALL_HDCP_HARDWARE_ALREADY_IN_USE __constant_cpu_to_le32(0xC01E0518)
-#define STATUS_GRAPHICS_OPM_PROTECTED_OUTPUT_NO_LONGER_EXISTS __constant_cpu_to_le32(0xC01E051A)
-#define STATUS_GRAPHICS_OPM_SESSION_TYPE_CHANGE_IN_PROGRESS __constant_cpu_to_le32(0xC01E051B)
-#define STATUS_GRAPHICS_I2C_NOT_SUPPORTED __constant_cpu_to_le32(0xC01E0580)
-#define STATUS_GRAPHICS_I2C_DEVICE_DOES_NOT_EXIST __constant_cpu_to_le32(0xC01E0581)
-#define STATUS_GRAPHICS_I2C_ERROR_TRANSMITTING_DATA __constant_cpu_to_le32(0xC01E0582)
-#define STATUS_GRAPHICS_I2C_ERROR_RECEIVING_DATA __constant_cpu_to_le32(0xC01E0583)
-#define STATUS_GRAPHICS_DDCCI_VCP_NOT_SUPPORTED __constant_cpu_to_le32(0xC01E0584)
-#define STATUS_GRAPHICS_DDCCI_INVALID_DATA __constant_cpu_to_le32(0xC01E0585)
-#define STATUS_GRAPHICS_DDCCI_MONITOR_RETURNED_INVALID_TIMING_STATUS_BYTE __constant_cpu_to_le32(0xC01E0586)
-#define STATUS_GRAPHICS_DDCCI_INVALID_CAPABILITIES_STRING __constant_cpu_to_le32(0xC01E0587)
-#define STATUS_GRAPHICS_MCA_INTERNAL_ERROR __constant_cpu_to_le32(0xC01E0588)
-#define STATUS_GRAPHICS_DDCCI_INVALID_MESSAGE_COMMAND __constant_cpu_to_le32(0xC01E0589)
-#define STATUS_GRAPHICS_DDCCI_INVALID_MESSAGE_LENGTH __constant_cpu_to_le32(0xC01E058A)
-#define STATUS_GRAPHICS_DDCCI_INVALID_MESSAGE_CHECKSUM __constant_cpu_to_le32(0xC01E058B)
-#define STATUS_GRAPHICS_INVALID_PHYSICAL_MONITOR_HANDLE __constant_cpu_to_le32(0xC01E058C)
-#define STATUS_GRAPHICS_MONITOR_NO_LONGER_EXISTS __constant_cpu_to_le32(0xC01E058D)
-#define STATUS_GRAPHICS_ONLY_CONSOLE_SESSION_SUPPORTED __constant_cpu_to_le32(0xC01E05E0)
-#define STATUS_GRAPHICS_NO_DISPLAY_DEVICE_CORRESPONDS_TO_NAME __constant_cpu_to_le32(0xC01E05E1)
-#define STATUS_GRAPHICS_DISPLAY_DEVICE_NOT_ATTACHED_TO_DESKTOP __constant_cpu_to_le32(0xC01E05E2)
-#define STATUS_GRAPHICS_MIRRORING_DEVICES_NOT_SUPPORTED __constant_cpu_to_le32(0xC01E05E3)
-#define STATUS_GRAPHICS_INVALID_POINTER __constant_cpu_to_le32(0xC01E05E4)
-#define STATUS_GRAPHICS_NO_MONITORS_CORRESPOND_TO_DISPLAY_DEVICE __constant_cpu_to_le32(0xC01E05E5)
-#define STATUS_GRAPHICS_PARAMETER_ARRAY_TOO_SMALL __constant_cpu_to_le32(0xC01E05E6)
-#define STATUS_GRAPHICS_INTERNAL_ERROR __constant_cpu_to_le32(0xC01E05E7)
-#define STATUS_GRAPHICS_SESSION_TYPE_CHANGE_IN_PROGRESS __constant_cpu_to_le32(0xC01E05E8)
-#define STATUS_FVE_LOCKED_VOLUME __constant_cpu_to_le32(0xC0210000)
-#define STATUS_FVE_NOT_ENCRYPTED __constant_cpu_to_le32(0xC0210001)
-#define STATUS_FVE_BAD_INFORMATION __constant_cpu_to_le32(0xC0210002)
-#define STATUS_FVE_TOO_SMALL __constant_cpu_to_le32(0xC0210003)
-#define STATUS_FVE_FAILED_WRONG_FS __constant_cpu_to_le32(0xC0210004)
-#define STATUS_FVE_FAILED_BAD_FS __constant_cpu_to_le32(0xC0210005)
-#define STATUS_FVE_FS_NOT_EXTENDED __constant_cpu_to_le32(0xC0210006)
-#define STATUS_FVE_FS_MOUNTED __constant_cpu_to_le32(0xC0210007)
-#define STATUS_FVE_NO_LICENSE __constant_cpu_to_le32(0xC0210008)
-#define STATUS_FVE_ACTION_NOT_ALLOWED __constant_cpu_to_le32(0xC0210009)
-#define STATUS_FVE_BAD_DATA __constant_cpu_to_le32(0xC021000A)
-#define STATUS_FVE_VOLUME_NOT_BOUND __constant_cpu_to_le32(0xC021000B)
-#define STATUS_FVE_NOT_DATA_VOLUME __constant_cpu_to_le32(0xC021000C)
-#define STATUS_FVE_CONV_READ_ERROR __constant_cpu_to_le32(0xC021000D)
-#define STATUS_FVE_CONV_WRITE_ERROR __constant_cpu_to_le32(0xC021000E)
-#define STATUS_FVE_OVERLAPPED_UPDATE __constant_cpu_to_le32(0xC021000F)
-#define STATUS_FVE_FAILED_SECTOR_SIZE __constant_cpu_to_le32(0xC0210010)
-#define STATUS_FVE_FAILED_AUTHENTICATION __constant_cpu_to_le32(0xC0210011)
-#define STATUS_FVE_NOT_OS_VOLUME __constant_cpu_to_le32(0xC0210012)
-#define STATUS_FVE_KEYFILE_NOT_FOUND __constant_cpu_to_le32(0xC0210013)
-#define STATUS_FVE_KEYFILE_INVALID __constant_cpu_to_le32(0xC0210014)
-#define STATUS_FVE_KEYFILE_NO_VMK __constant_cpu_to_le32(0xC0210015)
-#define STATUS_FVE_TPM_DISABLED __constant_cpu_to_le32(0xC0210016)
-#define STATUS_FVE_TPM_SRK_AUTH_NOT_ZERO __constant_cpu_to_le32(0xC0210017)
-#define STATUS_FVE_TPM_INVALID_PCR __constant_cpu_to_le32(0xC0210018)
-#define STATUS_FVE_TPM_NO_VMK __constant_cpu_to_le32(0xC0210019)
-#define STATUS_FVE_PIN_INVALID __constant_cpu_to_le32(0xC021001A)
-#define STATUS_FVE_AUTH_INVALID_APPLICATION __constant_cpu_to_le32(0xC021001B)
-#define STATUS_FVE_AUTH_INVALID_CONFIG __constant_cpu_to_le32(0xC021001C)
-#define STATUS_FVE_DEBUGGER_ENABLED __constant_cpu_to_le32(0xC021001D)
-#define STATUS_FVE_DRY_RUN_FAILED __constant_cpu_to_le32(0xC021001E)
-#define STATUS_FVE_BAD_METADATA_POINTER __constant_cpu_to_le32(0xC021001F)
-#define STATUS_FVE_OLD_METADATA_COPY __constant_cpu_to_le32(0xC0210020)
-#define STATUS_FVE_REBOOT_REQUIRED __constant_cpu_to_le32(0xC0210021)
-#define STATUS_FVE_RAW_ACCESS __constant_cpu_to_le32(0xC0210022)
-#define STATUS_FVE_RAW_BLOCKED __constant_cpu_to_le32(0xC0210023)
-#define STATUS_FWP_CALLOUT_NOT_FOUND __constant_cpu_to_le32(0xC0220001)
-#define STATUS_FWP_CONDITION_NOT_FOUND __constant_cpu_to_le32(0xC0220002)
-#define STATUS_FWP_FILTER_NOT_FOUND __constant_cpu_to_le32(0xC0220003)
-#define STATUS_FWP_LAYER_NOT_FOUND __constant_cpu_to_le32(0xC0220004)
-#define STATUS_FWP_PROVIDER_NOT_FOUND __constant_cpu_to_le32(0xC0220005)
-#define STATUS_FWP_PROVIDER_CONTEXT_NOT_FOUND __constant_cpu_to_le32(0xC0220006)
-#define STATUS_FWP_SUBLAYER_NOT_FOUND __constant_cpu_to_le32(0xC0220007)
-#define STATUS_FWP_NOT_FOUND __constant_cpu_to_le32(0xC0220008)
-#define STATUS_FWP_ALREADY_EXISTS __constant_cpu_to_le32(0xC0220009)
-#define STATUS_FWP_IN_USE __constant_cpu_to_le32(0xC022000A)
-#define STATUS_FWP_DYNAMIC_SESSION_IN_PROGRESS __constant_cpu_to_le32(0xC022000B)
-#define STATUS_FWP_WRONG_SESSION __constant_cpu_to_le32(0xC022000C)
-#define STATUS_FWP_NO_TXN_IN_PROGRESS __constant_cpu_to_le32(0xC022000D)
-#define STATUS_FWP_TXN_IN_PROGRESS __constant_cpu_to_le32(0xC022000E)
-#define STATUS_FWP_TXN_ABORTED __constant_cpu_to_le32(0xC022000F)
-#define STATUS_FWP_SESSION_ABORTED __constant_cpu_to_le32(0xC0220010)
-#define STATUS_FWP_INCOMPATIBLE_TXN __constant_cpu_to_le32(0xC0220011)
-#define STATUS_FWP_TIMEOUT __constant_cpu_to_le32(0xC0220012)
-#define STATUS_FWP_NET_EVENTS_DISABLED __constant_cpu_to_le32(0xC0220013)
-#define STATUS_FWP_INCOMPATIBLE_LAYER __constant_cpu_to_le32(0xC0220014)
-#define STATUS_FWP_KM_CLIENTS_ONLY __constant_cpu_to_le32(0xC0220015)
-#define STATUS_FWP_LIFETIME_MISMATCH __constant_cpu_to_le32(0xC0220016)
-#define STATUS_FWP_BUILTIN_OBJECT __constant_cpu_to_le32(0xC0220017)
-#define STATUS_FWP_TOO_MANY_BOOTTIME_FILTERS __constant_cpu_to_le32(0xC0220018)
-#define STATUS_FWP_TOO_MANY_CALLOUTS __constant_cpu_to_le32(0xC0220018)
-#define STATUS_FWP_NOTIFICATION_DROPPED __constant_cpu_to_le32(0xC0220019)
-#define STATUS_FWP_TRAFFIC_MISMATCH __constant_cpu_to_le32(0xC022001A)
-#define STATUS_FWP_INCOMPATIBLE_SA_STATE __constant_cpu_to_le32(0xC022001B)
-#define STATUS_FWP_NULL_POINTER __constant_cpu_to_le32(0xC022001C)
-#define STATUS_FWP_INVALID_ENUMERATOR __constant_cpu_to_le32(0xC022001D)
-#define STATUS_FWP_INVALID_FLAGS __constant_cpu_to_le32(0xC022001E)
-#define STATUS_FWP_INVALID_NET_MASK __constant_cpu_to_le32(0xC022001F)
-#define STATUS_FWP_INVALID_RANGE __constant_cpu_to_le32(0xC0220020)
-#define STATUS_FWP_INVALID_INTERVAL __constant_cpu_to_le32(0xC0220021)
-#define STATUS_FWP_ZERO_LENGTH_ARRAY __constant_cpu_to_le32(0xC0220022)
-#define STATUS_FWP_NULL_DISPLAY_NAME __constant_cpu_to_le32(0xC0220023)
-#define STATUS_FWP_INVALID_ACTION_TYPE __constant_cpu_to_le32(0xC0220024)
-#define STATUS_FWP_INVALID_WEIGHT __constant_cpu_to_le32(0xC0220025)
-#define STATUS_FWP_MATCH_TYPE_MISMATCH __constant_cpu_to_le32(0xC0220026)
-#define STATUS_FWP_TYPE_MISMATCH __constant_cpu_to_le32(0xC0220027)
-#define STATUS_FWP_OUT_OF_BOUNDS __constant_cpu_to_le32(0xC0220028)
-#define STATUS_FWP_RESERVED __constant_cpu_to_le32(0xC0220029)
-#define STATUS_FWP_DUPLICATE_CONDITION __constant_cpu_to_le32(0xC022002A)
-#define STATUS_FWP_DUPLICATE_KEYMOD __constant_cpu_to_le32(0xC022002B)
-#define STATUS_FWP_ACTION_INCOMPATIBLE_WITH_LAYER __constant_cpu_to_le32(0xC022002C)
-#define STATUS_FWP_ACTION_INCOMPATIBLE_WITH_SUBLAYER __constant_cpu_to_le32(0xC022002D)
-#define STATUS_FWP_CONTEXT_INCOMPATIBLE_WITH_LAYER __constant_cpu_to_le32(0xC022002E)
-#define STATUS_FWP_CONTEXT_INCOMPATIBLE_WITH_CALLOUT __constant_cpu_to_le32(0xC022002F)
-#define STATUS_FWP_INCOMPATIBLE_AUTH_METHOD __constant_cpu_to_le32(0xC0220030)
-#define STATUS_FWP_INCOMPATIBLE_DH_GROUP __constant_cpu_to_le32(0xC0220031)
-#define STATUS_FWP_EM_NOT_SUPPORTED __constant_cpu_to_le32(0xC0220032)
-#define STATUS_FWP_NEVER_MATCH __constant_cpu_to_le32(0xC0220033)
-#define STATUS_FWP_PROVIDER_CONTEXT_MISMATCH __constant_cpu_to_le32(0xC0220034)
-#define STATUS_FWP_INVALID_PARAMETER __constant_cpu_to_le32(0xC0220035)
-#define STATUS_FWP_TOO_MANY_SUBLAYERS __constant_cpu_to_le32(0xC0220036)
-#define STATUS_FWP_CALLOUT_NOTIFICATION_FAILED __constant_cpu_to_le32(0xC0220037)
-#define STATUS_FWP_INCOMPATIBLE_AUTH_CONFIG __constant_cpu_to_le32(0xC0220038)
-#define STATUS_FWP_INCOMPATIBLE_CIPHER_CONFIG __constant_cpu_to_le32(0xC0220039)
-#define STATUS_FWP_TCPIP_NOT_READY __constant_cpu_to_le32(0xC0220100)
-#define STATUS_FWP_INJECT_HANDLE_CLOSING __constant_cpu_to_le32(0xC0220101)
-#define STATUS_FWP_INJECT_HANDLE_STALE __constant_cpu_to_le32(0xC0220102)
-#define STATUS_FWP_CANNOT_PEND __constant_cpu_to_le32(0xC0220103)
-#define STATUS_NDIS_CLOSING __constant_cpu_to_le32(0xC0230002)
-#define STATUS_NDIS_BAD_VERSION __constant_cpu_to_le32(0xC0230004)
-#define STATUS_NDIS_BAD_CHARACTERISTICS __constant_cpu_to_le32(0xC0230005)
-#define STATUS_NDIS_ADAPTER_NOT_FOUND __constant_cpu_to_le32(0xC0230006)
-#define STATUS_NDIS_OPEN_FAILED __constant_cpu_to_le32(0xC0230007)
-#define STATUS_NDIS_DEVICE_FAILED __constant_cpu_to_le32(0xC0230008)
-#define STATUS_NDIS_MULTICAST_FULL __constant_cpu_to_le32(0xC0230009)
-#define STATUS_NDIS_MULTICAST_EXISTS __constant_cpu_to_le32(0xC023000A)
-#define STATUS_NDIS_MULTICAST_NOT_FOUND __constant_cpu_to_le32(0xC023000B)
-#define STATUS_NDIS_REQUEST_ABORTED __constant_cpu_to_le32(0xC023000C)
-#define STATUS_NDIS_RESET_IN_PROGRESS __constant_cpu_to_le32(0xC023000D)
-#define STATUS_NDIS_INVALID_PACKET __constant_cpu_to_le32(0xC023000F)
-#define STATUS_NDIS_INVALID_DEVICE_REQUEST __constant_cpu_to_le32(0xC0230010)
-#define STATUS_NDIS_ADAPTER_NOT_READY __constant_cpu_to_le32(0xC0230011)
-#define STATUS_NDIS_INVALID_LENGTH __constant_cpu_to_le32(0xC0230014)
-#define STATUS_NDIS_INVALID_DATA __constant_cpu_to_le32(0xC0230015)
-#define STATUS_NDIS_BUFFER_TOO_SHORT __constant_cpu_to_le32(0xC0230016)
-#define STATUS_NDIS_INVALID_OID __constant_cpu_to_le32(0xC0230017)
-#define STATUS_NDIS_ADAPTER_REMOVED __constant_cpu_to_le32(0xC0230018)
-#define STATUS_NDIS_UNSUPPORTED_MEDIA __constant_cpu_to_le32(0xC0230019)
-#define STATUS_NDIS_GROUP_ADDRESS_IN_USE __constant_cpu_to_le32(0xC023001A)
-#define STATUS_NDIS_FILE_NOT_FOUND __constant_cpu_to_le32(0xC023001B)
-#define STATUS_NDIS_ERROR_READING_FILE __constant_cpu_to_le32(0xC023001C)
-#define STATUS_NDIS_ALREADY_MAPPED __constant_cpu_to_le32(0xC023001D)
-#define STATUS_NDIS_RESOURCE_CONFLICT __constant_cpu_to_le32(0xC023001E)
-#define STATUS_NDIS_MEDIA_DISCONNECTED __constant_cpu_to_le32(0xC023001F)
-#define STATUS_NDIS_INVALID_ADDRESS __constant_cpu_to_le32(0xC0230022)
-#define STATUS_NDIS_PAUSED __constant_cpu_to_le32(0xC023002A)
-#define STATUS_NDIS_INTERFACE_NOT_FOUND __constant_cpu_to_le32(0xC023002B)
-#define STATUS_NDIS_UNSUPPORTED_REVISION __constant_cpu_to_le32(0xC023002C)
-#define STATUS_NDIS_INVALID_PORT __constant_cpu_to_le32(0xC023002D)
-#define STATUS_NDIS_INVALID_PORT_STATE __constant_cpu_to_le32(0xC023002E)
-#define STATUS_NDIS_LOW_POWER_STATE __constant_cpu_to_le32(0xC023002F)
-#define STATUS_NDIS_NOT_SUPPORTED __constant_cpu_to_le32(0xC02300BB)
-#define STATUS_NDIS_DOT11_AUTO_CONFIG_ENABLED __constant_cpu_to_le32(0xC0232000)
-#define STATUS_NDIS_DOT11_MEDIA_IN_USE __constant_cpu_to_le32(0xC0232001)
-#define STATUS_NDIS_DOT11_POWER_STATE_INVALID __constant_cpu_to_le32(0xC0232002)
-#define STATUS_IPSEC_BAD_SPI __constant_cpu_to_le32(0xC0360001)
-#define STATUS_IPSEC_SA_LIFETIME_EXPIRED __constant_cpu_to_le32(0xC0360002)
-#define STATUS_IPSEC_WRONG_SA __constant_cpu_to_le32(0xC0360003)
-#define STATUS_IPSEC_REPLAY_CHECK_FAILED __constant_cpu_to_le32(0xC0360004)
-#define STATUS_IPSEC_INVALID_PACKET __constant_cpu_to_le32(0xC0360005)
-#define STATUS_IPSEC_INTEGRITY_CHECK_FAILED __constant_cpu_to_le32(0xC0360006)
-#define STATUS_IPSEC_CLEAR_TEXT_DROP __constant_cpu_to_le32(0xC0360007)
+#define STATUS_SUCCESS cpu_to_le32(0x00000000)
+#define STATUS_WAIT_0 cpu_to_le32(0x00000000)
+#define STATUS_WAIT_1 cpu_to_le32(0x00000001)
+#define STATUS_WAIT_2 cpu_to_le32(0x00000002)
+#define STATUS_WAIT_3 cpu_to_le32(0x00000003)
+#define STATUS_WAIT_63 cpu_to_le32(0x0000003F)
+#define STATUS_ABANDONED cpu_to_le32(0x00000080)
+#define STATUS_ABANDONED_WAIT_0 cpu_to_le32(0x00000080)
+#define STATUS_ABANDONED_WAIT_63 cpu_to_le32(0x000000BF)
+#define STATUS_USER_APC cpu_to_le32(0x000000C0)
+#define STATUS_KERNEL_APC cpu_to_le32(0x00000100)
+#define STATUS_ALERTED cpu_to_le32(0x00000101)
+#define STATUS_TIMEOUT cpu_to_le32(0x00000102)
+#define STATUS_PENDING cpu_to_le32(0x00000103)
+#define STATUS_REPARSE cpu_to_le32(0x00000104)
+#define STATUS_MORE_ENTRIES cpu_to_le32(0x00000105)
+#define STATUS_NOT_ALL_ASSIGNED cpu_to_le32(0x00000106)
+#define STATUS_SOME_NOT_MAPPED cpu_to_le32(0x00000107)
+#define STATUS_OPLOCK_BREAK_IN_PROGRESS cpu_to_le32(0x00000108)
+#define STATUS_VOLUME_MOUNTED cpu_to_le32(0x00000109)
+#define STATUS_RXACT_COMMITTED cpu_to_le32(0x0000010A)
+#define STATUS_NOTIFY_CLEANUP cpu_to_le32(0x0000010B)
+#define STATUS_NOTIFY_ENUM_DIR cpu_to_le32(0x0000010C)
+#define STATUS_NO_QUOTAS_FOR_ACCOUNT cpu_to_le32(0x0000010D)
+#define STATUS_PRIMARY_TRANSPORT_CONNECT_FAILED cpu_to_le32(0x0000010E)
+#define STATUS_PAGE_FAULT_TRANSITION cpu_to_le32(0x00000110)
+#define STATUS_PAGE_FAULT_DEMAND_ZERO cpu_to_le32(0x00000111)
+#define STATUS_PAGE_FAULT_COPY_ON_WRITE cpu_to_le32(0x00000112)
+#define STATUS_PAGE_FAULT_GUARD_PAGE cpu_to_le32(0x00000113)
+#define STATUS_PAGE_FAULT_PAGING_FILE cpu_to_le32(0x00000114)
+#define STATUS_CACHE_PAGE_LOCKED cpu_to_le32(0x00000115)
+#define STATUS_CRASH_DUMP cpu_to_le32(0x00000116)
+#define STATUS_BUFFER_ALL_ZEROS cpu_to_le32(0x00000117)
+#define STATUS_REPARSE_OBJECT cpu_to_le32(0x00000118)
+#define STATUS_RESOURCE_REQUIREMENTS_CHANGED cpu_to_le32(0x00000119)
+#define STATUS_TRANSLATION_COMPLETE cpu_to_le32(0x00000120)
+#define STATUS_DS_MEMBERSHIP_EVALUATED_LOCALLY cpu_to_le32(0x00000121)
+#define STATUS_NOTHING_TO_TERMINATE cpu_to_le32(0x00000122)
+#define STATUS_PROCESS_NOT_IN_JOB cpu_to_le32(0x00000123)
+#define STATUS_PROCESS_IN_JOB cpu_to_le32(0x00000124)
+#define STATUS_VOLSNAP_HIBERNATE_READY cpu_to_le32(0x00000125)
+#define STATUS_FSFILTER_OP_COMPLETED_SUCCESSFULLY cpu_to_le32(0x00000126)
+#define STATUS_INTERRUPT_VECTOR_ALREADY_CONNECTED cpu_to_le32(0x00000127)
+#define STATUS_INTERRUPT_STILL_CONNECTED cpu_to_le32(0x00000128)
+#define STATUS_PROCESS_CLONED cpu_to_le32(0x00000129)
+#define STATUS_FILE_LOCKED_WITH_ONLY_READERS cpu_to_le32(0x0000012A)
+#define STATUS_FILE_LOCKED_WITH_WRITERS cpu_to_le32(0x0000012B)
+#define STATUS_RESOURCEMANAGER_READ_ONLY cpu_to_le32(0x00000202)
+#define STATUS_WAIT_FOR_OPLOCK cpu_to_le32(0x00000367)
+#define DBG_EXCEPTION_HANDLED cpu_to_le32(0x00010001)
+#define DBG_CONTINUE cpu_to_le32(0x00010002)
+#define STATUS_FLT_IO_COMPLETE cpu_to_le32(0x001C0001)
+#define STATUS_OBJECT_NAME_EXISTS cpu_to_le32(0x40000000)
+#define STATUS_THREAD_WAS_SUSPENDED cpu_to_le32(0x40000001)
+#define STATUS_WORKING_SET_LIMIT_RANGE cpu_to_le32(0x40000002)
+#define STATUS_IMAGE_NOT_AT_BASE cpu_to_le32(0x40000003)
+#define STATUS_RXACT_STATE_CREATED cpu_to_le32(0x40000004)
+#define STATUS_SEGMENT_NOTIFICATION cpu_to_le32(0x40000005)
+#define STATUS_LOCAL_USER_SESSION_KEY cpu_to_le32(0x40000006)
+#define STATUS_BAD_CURRENT_DIRECTORY cpu_to_le32(0x40000007)
+#define STATUS_SERIAL_MORE_WRITES cpu_to_le32(0x40000008)
+#define STATUS_REGISTRY_RECOVERED cpu_to_le32(0x40000009)
+#define STATUS_FT_READ_RECOVERY_FROM_BACKUP cpu_to_le32(0x4000000A)
+#define STATUS_FT_WRITE_RECOVERY cpu_to_le32(0x4000000B)
+#define STATUS_SERIAL_COUNTER_TIMEOUT cpu_to_le32(0x4000000C)
+#define STATUS_NULL_LM_PASSWORD cpu_to_le32(0x4000000D)
+#define STATUS_IMAGE_MACHINE_TYPE_MISMATCH cpu_to_le32(0x4000000E)
+#define STATUS_RECEIVE_PARTIAL cpu_to_le32(0x4000000F)
+#define STATUS_RECEIVE_EXPEDITED cpu_to_le32(0x40000010)
+#define STATUS_RECEIVE_PARTIAL_EXPEDITED cpu_to_le32(0x40000011)
+#define STATUS_EVENT_DONE cpu_to_le32(0x40000012)
+#define STATUS_EVENT_PENDING cpu_to_le32(0x40000013)
+#define STATUS_CHECKING_FILE_SYSTEM cpu_to_le32(0x40000014)
+#define STATUS_FATAL_APP_EXIT cpu_to_le32(0x40000015)
+#define STATUS_PREDEFINED_HANDLE cpu_to_le32(0x40000016)
+#define STATUS_WAS_UNLOCKED cpu_to_le32(0x40000017)
+#define STATUS_SERVICE_NOTIFICATION cpu_to_le32(0x40000018)
+#define STATUS_WAS_LOCKED cpu_to_le32(0x40000019)
+#define STATUS_LOG_HARD_ERROR cpu_to_le32(0x4000001A)
+#define STATUS_ALREADY_WIN32 cpu_to_le32(0x4000001B)
+#define STATUS_WX86_UNSIMULATE cpu_to_le32(0x4000001C)
+#define STATUS_WX86_CONTINUE cpu_to_le32(0x4000001D)
+#define STATUS_WX86_SINGLE_STEP cpu_to_le32(0x4000001E)
+#define STATUS_WX86_BREAKPOINT cpu_to_le32(0x4000001F)
+#define STATUS_WX86_EXCEPTION_CONTINUE cpu_to_le32(0x40000020)
+#define STATUS_WX86_EXCEPTION_LASTCHANCE cpu_to_le32(0x40000021)
+#define STATUS_WX86_EXCEPTION_CHAIN cpu_to_le32(0x40000022)
+#define STATUS_IMAGE_MACHINE_TYPE_MISMATCH_EXE cpu_to_le32(0x40000023)
+#define STATUS_NO_YIELD_PERFORMED cpu_to_le32(0x40000024)
+#define STATUS_TIMER_RESUME_IGNORED cpu_to_le32(0x40000025)
+#define STATUS_ARBITRATION_UNHANDLED cpu_to_le32(0x40000026)
+#define STATUS_CARDBUS_NOT_SUPPORTED cpu_to_le32(0x40000027)
+#define STATUS_WX86_CREATEWX86TIB cpu_to_le32(0x40000028)
+#define STATUS_MP_PROCESSOR_MISMATCH cpu_to_le32(0x40000029)
+#define STATUS_HIBERNATED cpu_to_le32(0x4000002A)
+#define STATUS_RESUME_HIBERNATION cpu_to_le32(0x4000002B)
+#define STATUS_FIRMWARE_UPDATED cpu_to_le32(0x4000002C)
+#define STATUS_DRIVERS_LEAKING_LOCKED_PAGES cpu_to_le32(0x4000002D)
+#define STATUS_MESSAGE_RETRIEVED cpu_to_le32(0x4000002E)
+#define STATUS_SYSTEM_POWERSTATE_TRANSITION cpu_to_le32(0x4000002F)
+#define STATUS_ALPC_CHECK_COMPLETION_LIST cpu_to_le32(0x40000030)
+#define STATUS_SYSTEM_POWERSTATE_COMPLEX_TRANSITION cpu_to_le32(0x40000031)
+#define STATUS_ACCESS_AUDIT_BY_POLICY cpu_to_le32(0x40000032)
+#define STATUS_ABANDON_HIBERFILE cpu_to_le32(0x40000033)
+#define STATUS_BIZRULES_NOT_ENABLED cpu_to_le32(0x40000034)
+#define STATUS_WAKE_SYSTEM cpu_to_le32(0x40000294)
+#define STATUS_DS_SHUTTING_DOWN cpu_to_le32(0x40000370)
+#define DBG_REPLY_LATER cpu_to_le32(0x40010001)
+#define DBG_UNABLE_TO_PROVIDE_HANDLE cpu_to_le32(0x40010002)
+#define DBG_TERMINATE_THREAD cpu_to_le32(0x40010003)
+#define DBG_TERMINATE_PROCESS cpu_to_le32(0x40010004)
+#define DBG_CONTROL_C cpu_to_le32(0x40010005)
+#define DBG_PRINTEXCEPTION_C cpu_to_le32(0x40010006)
+#define DBG_RIPEXCEPTION cpu_to_le32(0x40010007)
+#define DBG_CONTROL_BREAK cpu_to_le32(0x40010008)
+#define DBG_COMMAND_EXCEPTION cpu_to_le32(0x40010009)
+#define RPC_NT_UUID_LOCAL_ONLY cpu_to_le32(0x40020056)
+#define RPC_NT_SEND_INCOMPLETE cpu_to_le32(0x400200AF)
+#define STATUS_CTX_CDM_CONNECT cpu_to_le32(0x400A0004)
+#define STATUS_CTX_CDM_DISCONNECT cpu_to_le32(0x400A0005)
+#define STATUS_SXS_RELEASE_ACTIVATION_CONTEXT cpu_to_le32(0x4015000D)
+#define STATUS_RECOVERY_NOT_NEEDED cpu_to_le32(0x40190034)
+#define STATUS_RM_ALREADY_STARTED cpu_to_le32(0x40190035)
+#define STATUS_LOG_NO_RESTART cpu_to_le32(0x401A000C)
+#define STATUS_VIDEO_DRIVER_DEBUG_REPORT_REQUEST cpu_to_le32(0x401B00EC)
+#define STATUS_GRAPHICS_PARTIAL_DATA_POPULATED cpu_to_le32(0x401E000A)
+#define STATUS_GRAPHICS_DRIVER_MISMATCH cpu_to_le32(0x401E0117)
+#define STATUS_GRAPHICS_MODE_NOT_PINNED cpu_to_le32(0x401E0307)
+#define STATUS_GRAPHICS_NO_PREFERRED_MODE cpu_to_le32(0x401E031E)
+#define STATUS_GRAPHICS_DATASET_IS_EMPTY cpu_to_le32(0x401E034B)
+#define STATUS_GRAPHICS_NO_MORE_ELEMENTS_IN_DATASET cpu_to_le32(0x401E034C)
+#define STATUS_GRAPHICS_PATH_CONTENT_GEOMETRY_TRANSFORMATION_NOT_PINNED cpu_to_le32(0x401E0351)
+#define STATUS_GRAPHICS_UNKNOWN_CHILD_STATUS cpu_to_le32(0x401E042F)
+#define STATUS_GRAPHICS_LEADLINK_START_DEFERRED cpu_to_le32(0x401E0437)
+#define STATUS_GRAPHICS_POLLING_TOO_FREQUENTLY cpu_to_le32(0x401E0439)
+#define STATUS_GRAPHICS_START_DEFERRED cpu_to_le32(0x401E043A)
+#define STATUS_NDIS_INDICATION_REQUIRED cpu_to_le32(0x40230001)
+#define STATUS_GUARD_PAGE_VIOLATION cpu_to_le32(0x80000001)
+#define STATUS_DATATYPE_MISALIGNMENT cpu_to_le32(0x80000002)
+#define STATUS_BREAKPOINT cpu_to_le32(0x80000003)
+#define STATUS_SINGLE_STEP cpu_to_le32(0x80000004)
+#define STATUS_BUFFER_OVERFLOW cpu_to_le32(0x80000005)
+#define STATUS_NO_MORE_FILES cpu_to_le32(0x80000006)
+#define STATUS_WAKE_SYSTEM_DEBUGGER cpu_to_le32(0x80000007)
+#define STATUS_HANDLES_CLOSED cpu_to_le32(0x8000000A)
+#define STATUS_NO_INHERITANCE cpu_to_le32(0x8000000B)
+#define STATUS_GUID_SUBSTITUTION_MADE cpu_to_le32(0x8000000C)
+#define STATUS_PARTIAL_COPY cpu_to_le32(0x8000000D)
+#define STATUS_DEVICE_PAPER_EMPTY cpu_to_le32(0x8000000E)
+#define STATUS_DEVICE_POWERED_OFF cpu_to_le32(0x8000000F)
+#define STATUS_DEVICE_OFF_LINE cpu_to_le32(0x80000010)
+#define STATUS_DEVICE_BUSY cpu_to_le32(0x80000011)
+#define STATUS_NO_MORE_EAS cpu_to_le32(0x80000012)
+#define STATUS_INVALID_EA_NAME cpu_to_le32(0x80000013)
+#define STATUS_EA_LIST_INCONSISTENT cpu_to_le32(0x80000014)
+#define STATUS_INVALID_EA_FLAG cpu_to_le32(0x80000015)
+#define STATUS_VERIFY_REQUIRED cpu_to_le32(0x80000016)
+#define STATUS_EXTRANEOUS_INFORMATION cpu_to_le32(0x80000017)
+#define STATUS_RXACT_COMMIT_NECESSARY cpu_to_le32(0x80000018)
+#define STATUS_NO_MORE_ENTRIES cpu_to_le32(0x8000001A)
+#define STATUS_FILEMARK_DETECTED cpu_to_le32(0x8000001B)
+#define STATUS_MEDIA_CHANGED cpu_to_le32(0x8000001C)
+#define STATUS_BUS_RESET cpu_to_le32(0x8000001D)
+#define STATUS_END_OF_MEDIA cpu_to_le32(0x8000001E)
+#define STATUS_BEGINNING_OF_MEDIA cpu_to_le32(0x8000001F)
+#define STATUS_MEDIA_CHECK cpu_to_le32(0x80000020)
+#define STATUS_SETMARK_DETECTED cpu_to_le32(0x80000021)
+#define STATUS_NO_DATA_DETECTED cpu_to_le32(0x80000022)
+#define STATUS_REDIRECTOR_HAS_OPEN_HANDLES cpu_to_le32(0x80000023)
+#define STATUS_SERVER_HAS_OPEN_HANDLES cpu_to_le32(0x80000024)
+#define STATUS_ALREADY_DISCONNECTED cpu_to_le32(0x80000025)
+#define STATUS_LONGJUMP cpu_to_le32(0x80000026)
+#define STATUS_CLEANER_CARTRIDGE_INSTALLED cpu_to_le32(0x80000027)
+#define STATUS_PLUGPLAY_QUERY_VETOED cpu_to_le32(0x80000028)
+#define STATUS_UNWIND_CONSOLIDATE cpu_to_le32(0x80000029)
+#define STATUS_REGISTRY_HIVE_RECOVERED cpu_to_le32(0x8000002A)
+#define STATUS_DLL_MIGHT_BE_INSECURE cpu_to_le32(0x8000002B)
+#define STATUS_DLL_MIGHT_BE_INCOMPATIBLE cpu_to_le32(0x8000002C)
+#define STATUS_STOPPED_ON_SYMLINK cpu_to_le32(0x8000002D)
+#define STATUS_DEVICE_REQUIRES_CLEANING cpu_to_le32(0x80000288)
+#define STATUS_DEVICE_DOOR_OPEN cpu_to_le32(0x80000289)
+#define STATUS_DATA_LOST_REPAIR cpu_to_le32(0x80000803)
+#define DBG_EXCEPTION_NOT_HANDLED cpu_to_le32(0x80010001)
+#define STATUS_CLUSTER_NODE_ALREADY_UP cpu_to_le32(0x80130001)
+#define STATUS_CLUSTER_NODE_ALREADY_DOWN cpu_to_le32(0x80130002)
+#define STATUS_CLUSTER_NETWORK_ALREADY_ONLINE cpu_to_le32(0x80130003)
+#define STATUS_CLUSTER_NETWORK_ALREADY_OFFLINE cpu_to_le32(0x80130004)
+#define STATUS_CLUSTER_NODE_ALREADY_MEMBER cpu_to_le32(0x80130005)
+#define STATUS_COULD_NOT_RESIZE_LOG cpu_to_le32(0x80190009)
+#define STATUS_NO_TXF_METADATA cpu_to_le32(0x80190029)
+#define STATUS_CANT_RECOVER_WITH_HANDLE_OPEN cpu_to_le32(0x80190031)
+#define STATUS_TXF_METADATA_ALREADY_PRESENT cpu_to_le32(0x80190041)
+#define STATUS_TRANSACTION_SCOPE_CALLBACKS_NOT_SET cpu_to_le32(0x80190042)
+#define STATUS_VIDEO_HUNG_DISPLAY_DRIVER_THREAD_RECOVERED cpu_to_le32(0x801B00EB)
+#define STATUS_FLT_BUFFER_TOO_SMALL cpu_to_le32(0x801C0001)
+#define STATUS_FVE_PARTIAL_METADATA cpu_to_le32(0x80210001)
+#define STATUS_UNSUCCESSFUL cpu_to_le32(0xC0000001)
+#define STATUS_NOT_IMPLEMENTED cpu_to_le32(0xC0000002)
+#define STATUS_INVALID_INFO_CLASS cpu_to_le32(0xC0000003)
+#define STATUS_INFO_LENGTH_MISMATCH cpu_to_le32(0xC0000004)
+#define STATUS_ACCESS_VIOLATION cpu_to_le32(0xC0000005)
+#define STATUS_IN_PAGE_ERROR cpu_to_le32(0xC0000006)
+#define STATUS_PAGEFILE_QUOTA cpu_to_le32(0xC0000007)
+#define STATUS_INVALID_HANDLE cpu_to_le32(0xC0000008)
+#define STATUS_BAD_INITIAL_STACK cpu_to_le32(0xC0000009)
+#define STATUS_BAD_INITIAL_PC cpu_to_le32(0xC000000A)
+#define STATUS_INVALID_CID cpu_to_le32(0xC000000B)
+#define STATUS_TIMER_NOT_CANCELED cpu_to_le32(0xC000000C)
+#define STATUS_INVALID_PARAMETER cpu_to_le32(0xC000000D)
+#define STATUS_NO_SUCH_DEVICE cpu_to_le32(0xC000000E)
+#define STATUS_NO_SUCH_FILE cpu_to_le32(0xC000000F)
+#define STATUS_INVALID_DEVICE_REQUEST cpu_to_le32(0xC0000010)
+#define STATUS_END_OF_FILE cpu_to_le32(0xC0000011)
+#define STATUS_WRONG_VOLUME cpu_to_le32(0xC0000012)
+#define STATUS_NO_MEDIA_IN_DEVICE cpu_to_le32(0xC0000013)
+#define STATUS_UNRECOGNIZED_MEDIA cpu_to_le32(0xC0000014)
+#define STATUS_NONEXISTENT_SECTOR cpu_to_le32(0xC0000015)
+#define STATUS_MORE_PROCESSING_REQUIRED cpu_to_le32(0xC0000016)
+#define STATUS_NO_MEMORY cpu_to_le32(0xC0000017)
+#define STATUS_CONFLICTING_ADDRESSES cpu_to_le32(0xC0000018)
+#define STATUS_NOT_MAPPED_VIEW cpu_to_le32(0xC0000019)
+#define STATUS_UNABLE_TO_FREE_VM cpu_to_le32(0xC000001A)
+#define STATUS_UNABLE_TO_DELETE_SECTION cpu_to_le32(0xC000001B)
+#define STATUS_INVALID_SYSTEM_SERVICE cpu_to_le32(0xC000001C)
+#define STATUS_ILLEGAL_INSTRUCTION cpu_to_le32(0xC000001D)
+#define STATUS_INVALID_LOCK_SEQUENCE cpu_to_le32(0xC000001E)
+#define STATUS_INVALID_VIEW_SIZE cpu_to_le32(0xC000001F)
+#define STATUS_INVALID_FILE_FOR_SECTION cpu_to_le32(0xC0000020)
+#define STATUS_ALREADY_COMMITTED cpu_to_le32(0xC0000021)
+#define STATUS_ACCESS_DENIED cpu_to_le32(0xC0000022)
+#define STATUS_BUFFER_TOO_SMALL cpu_to_le32(0xC0000023)
+#define STATUS_OBJECT_TYPE_MISMATCH cpu_to_le32(0xC0000024)
+#define STATUS_NONCONTINUABLE_EXCEPTION cpu_to_le32(0xC0000025)
+#define STATUS_INVALID_DISPOSITION cpu_to_le32(0xC0000026)
+#define STATUS_UNWIND cpu_to_le32(0xC0000027)
+#define STATUS_BAD_STACK cpu_to_le32(0xC0000028)
+#define STATUS_INVALID_UNWIND_TARGET cpu_to_le32(0xC0000029)
+#define STATUS_NOT_LOCKED cpu_to_le32(0xC000002A)
+#define STATUS_PARITY_ERROR cpu_to_le32(0xC000002B)
+#define STATUS_UNABLE_TO_DECOMMIT_VM cpu_to_le32(0xC000002C)
+#define STATUS_NOT_COMMITTED cpu_to_le32(0xC000002D)
+#define STATUS_INVALID_PORT_ATTRIBUTES cpu_to_le32(0xC000002E)
+#define STATUS_PORT_MESSAGE_TOO_LONG cpu_to_le32(0xC000002F)
+#define STATUS_INVALID_PARAMETER_MIX cpu_to_le32(0xC0000030)
+#define STATUS_INVALID_QUOTA_LOWER cpu_to_le32(0xC0000031)
+#define STATUS_DISK_CORRUPT_ERROR cpu_to_le32(0xC0000032)
+#define STATUS_OBJECT_NAME_INVALID cpu_to_le32(0xC0000033)
+#define STATUS_OBJECT_NAME_NOT_FOUND cpu_to_le32(0xC0000034)
+#define STATUS_OBJECT_NAME_COLLISION cpu_to_le32(0xC0000035)
+#define STATUS_PORT_DISCONNECTED cpu_to_le32(0xC0000037)
+#define STATUS_DEVICE_ALREADY_ATTACHED cpu_to_le32(0xC0000038)
+#define STATUS_OBJECT_PATH_INVALID cpu_to_le32(0xC0000039)
+#define STATUS_OBJECT_PATH_NOT_FOUND cpu_to_le32(0xC000003A)
+#define STATUS_OBJECT_PATH_SYNTAX_BAD cpu_to_le32(0xC000003B)
+#define STATUS_DATA_OVERRUN cpu_to_le32(0xC000003C)
+#define STATUS_DATA_LATE_ERROR cpu_to_le32(0xC000003D)
+#define STATUS_DATA_ERROR cpu_to_le32(0xC000003E)
+#define STATUS_CRC_ERROR cpu_to_le32(0xC000003F)
+#define STATUS_SECTION_TOO_BIG cpu_to_le32(0xC0000040)
+#define STATUS_PORT_CONNECTION_REFUSED cpu_to_le32(0xC0000041)
+#define STATUS_INVALID_PORT_HANDLE cpu_to_le32(0xC0000042)
+#define STATUS_SHARING_VIOLATION cpu_to_le32(0xC0000043)
+#define STATUS_QUOTA_EXCEEDED cpu_to_le32(0xC0000044)
+#define STATUS_INVALID_PAGE_PROTECTION cpu_to_le32(0xC0000045)
+#define STATUS_MUTANT_NOT_OWNED cpu_to_le32(0xC0000046)
+#define STATUS_SEMAPHORE_LIMIT_EXCEEDED cpu_to_le32(0xC0000047)
+#define STATUS_PORT_ALREADY_SET cpu_to_le32(0xC0000048)
+#define STATUS_SECTION_NOT_IMAGE cpu_to_le32(0xC0000049)
+#define STATUS_SUSPEND_COUNT_EXCEEDED cpu_to_le32(0xC000004A)
+#define STATUS_THREAD_IS_TERMINATING cpu_to_le32(0xC000004B)
+#define STATUS_BAD_WORKING_SET_LIMIT cpu_to_le32(0xC000004C)
+#define STATUS_INCOMPATIBLE_FILE_MAP cpu_to_le32(0xC000004D)
+#define STATUS_SECTION_PROTECTION cpu_to_le32(0xC000004E)
+#define STATUS_EAS_NOT_SUPPORTED cpu_to_le32(0xC000004F)
+#define STATUS_EA_TOO_LARGE cpu_to_le32(0xC0000050)
+#define STATUS_NONEXISTENT_EA_ENTRY cpu_to_le32(0xC0000051)
+#define STATUS_NO_EAS_ON_FILE cpu_to_le32(0xC0000052)
+#define STATUS_EA_CORRUPT_ERROR cpu_to_le32(0xC0000053)
+#define STATUS_FILE_LOCK_CONFLICT cpu_to_le32(0xC0000054)
+#define STATUS_LOCK_NOT_GRANTED cpu_to_le32(0xC0000055)
+#define STATUS_DELETE_PENDING cpu_to_le32(0xC0000056)
+#define STATUS_CTL_FILE_NOT_SUPPORTED cpu_to_le32(0xC0000057)
+#define STATUS_UNKNOWN_REVISION cpu_to_le32(0xC0000058)
+#define STATUS_REVISION_MISMATCH cpu_to_le32(0xC0000059)
+#define STATUS_INVALID_OWNER cpu_to_le32(0xC000005A)
+#define STATUS_INVALID_PRIMARY_GROUP cpu_to_le32(0xC000005B)
+#define STATUS_NO_IMPERSONATION_TOKEN cpu_to_le32(0xC000005C)
+#define STATUS_CANT_DISABLE_MANDATORY cpu_to_le32(0xC000005D)
+#define STATUS_NO_LOGON_SERVERS cpu_to_le32(0xC000005E)
+#define STATUS_NO_SUCH_LOGON_SESSION cpu_to_le32(0xC000005F)
+#define STATUS_NO_SUCH_PRIVILEGE cpu_to_le32(0xC0000060)
+#define STATUS_PRIVILEGE_NOT_HELD cpu_to_le32(0xC0000061)
+#define STATUS_INVALID_ACCOUNT_NAME cpu_to_le32(0xC0000062)
+#define STATUS_USER_EXISTS cpu_to_le32(0xC0000063)
+#define STATUS_NO_SUCH_USER cpu_to_le32(0xC0000064)
+#define STATUS_GROUP_EXISTS cpu_to_le32(0xC0000065)
+#define STATUS_NO_SUCH_GROUP cpu_to_le32(0xC0000066)
+#define STATUS_MEMBER_IN_GROUP cpu_to_le32(0xC0000067)
+#define STATUS_MEMBER_NOT_IN_GROUP cpu_to_le32(0xC0000068)
+#define STATUS_LAST_ADMIN cpu_to_le32(0xC0000069)
+#define STATUS_WRONG_PASSWORD cpu_to_le32(0xC000006A)
+#define STATUS_ILL_FORMED_PASSWORD cpu_to_le32(0xC000006B)
+#define STATUS_PASSWORD_RESTRICTION cpu_to_le32(0xC000006C)
+#define STATUS_LOGON_FAILURE cpu_to_le32(0xC000006D)
+#define STATUS_ACCOUNT_RESTRICTION cpu_to_le32(0xC000006E)
+#define STATUS_INVALID_LOGON_HOURS cpu_to_le32(0xC000006F)
+#define STATUS_INVALID_WORKSTATION cpu_to_le32(0xC0000070)
+#define STATUS_PASSWORD_EXPIRED cpu_to_le32(0xC0000071)
+#define STATUS_ACCOUNT_DISABLED cpu_to_le32(0xC0000072)
+#define STATUS_NONE_MAPPED cpu_to_le32(0xC0000073)
+#define STATUS_TOO_MANY_LUIDS_REQUESTED cpu_to_le32(0xC0000074)
+#define STATUS_LUIDS_EXHAUSTED cpu_to_le32(0xC0000075)
+#define STATUS_INVALID_SUB_AUTHORITY cpu_to_le32(0xC0000076)
+#define STATUS_INVALID_ACL cpu_to_le32(0xC0000077)
+#define STATUS_INVALID_SID cpu_to_le32(0xC0000078)
+#define STATUS_INVALID_SECURITY_DESCR cpu_to_le32(0xC0000079)
+#define STATUS_PROCEDURE_NOT_FOUND cpu_to_le32(0xC000007A)
+#define STATUS_INVALID_IMAGE_FORMAT cpu_to_le32(0xC000007B)
+#define STATUS_NO_TOKEN cpu_to_le32(0xC000007C)
+#define STATUS_BAD_INHERITANCE_ACL cpu_to_le32(0xC000007D)
+#define STATUS_RANGE_NOT_LOCKED cpu_to_le32(0xC000007E)
+#define STATUS_DISK_FULL cpu_to_le32(0xC000007F)
+#define STATUS_SERVER_DISABLED cpu_to_le32(0xC0000080)
+#define STATUS_SERVER_NOT_DISABLED cpu_to_le32(0xC0000081)
+#define STATUS_TOO_MANY_GUIDS_REQUESTED cpu_to_le32(0xC0000082)
+#define STATUS_GUIDS_EXHAUSTED cpu_to_le32(0xC0000083)
+#define STATUS_INVALID_ID_AUTHORITY cpu_to_le32(0xC0000084)
+#define STATUS_AGENTS_EXHAUSTED cpu_to_le32(0xC0000085)
+#define STATUS_INVALID_VOLUME_LABEL cpu_to_le32(0xC0000086)
+#define STATUS_SECTION_NOT_EXTENDED cpu_to_le32(0xC0000087)
+#define STATUS_NOT_MAPPED_DATA cpu_to_le32(0xC0000088)
+#define STATUS_RESOURCE_DATA_NOT_FOUND cpu_to_le32(0xC0000089)
+#define STATUS_RESOURCE_TYPE_NOT_FOUND cpu_to_le32(0xC000008A)
+#define STATUS_RESOURCE_NAME_NOT_FOUND cpu_to_le32(0xC000008B)
+#define STATUS_ARRAY_BOUNDS_EXCEEDED cpu_to_le32(0xC000008C)
+#define STATUS_FLOAT_DENORMAL_OPERAND cpu_to_le32(0xC000008D)
+#define STATUS_FLOAT_DIVIDE_BY_ZERO cpu_to_le32(0xC000008E)
+#define STATUS_FLOAT_INEXACT_RESULT cpu_to_le32(0xC000008F)
+#define STATUS_FLOAT_INVALID_OPERATION cpu_to_le32(0xC0000090)
+#define STATUS_FLOAT_OVERFLOW cpu_to_le32(0xC0000091)
+#define STATUS_FLOAT_STACK_CHECK cpu_to_le32(0xC0000092)
+#define STATUS_FLOAT_UNDERFLOW cpu_to_le32(0xC0000093)
+#define STATUS_INTEGER_DIVIDE_BY_ZERO cpu_to_le32(0xC0000094)
+#define STATUS_INTEGER_OVERFLOW cpu_to_le32(0xC0000095)
+#define STATUS_PRIVILEGED_INSTRUCTION cpu_to_le32(0xC0000096)
+#define STATUS_TOO_MANY_PAGING_FILES cpu_to_le32(0xC0000097)
+#define STATUS_FILE_INVALID cpu_to_le32(0xC0000098)
+#define STATUS_ALLOTTED_SPACE_EXCEEDED cpu_to_le32(0xC0000099)
+#define STATUS_INSUFFICIENT_RESOURCES cpu_to_le32(0xC000009A)
+#define STATUS_DFS_EXIT_PATH_FOUND cpu_to_le32(0xC000009B)
+#define STATUS_DEVICE_DATA_ERROR cpu_to_le32(0xC000009C)
+#define STATUS_DEVICE_NOT_CONNECTED cpu_to_le32(0xC000009D)
+#define STATUS_DEVICE_POWER_FAILURE cpu_to_le32(0xC000009E)
+#define STATUS_FREE_VM_NOT_AT_BASE cpu_to_le32(0xC000009F)
+#define STATUS_MEMORY_NOT_ALLOCATED cpu_to_le32(0xC00000A0)
+#define STATUS_WORKING_SET_QUOTA cpu_to_le32(0xC00000A1)
+#define STATUS_MEDIA_WRITE_PROTECTED cpu_to_le32(0xC00000A2)
+#define STATUS_DEVICE_NOT_READY cpu_to_le32(0xC00000A3)
+#define STATUS_INVALID_GROUP_ATTRIBUTES cpu_to_le32(0xC00000A4)
+#define STATUS_BAD_IMPERSONATION_LEVEL cpu_to_le32(0xC00000A5)
+#define STATUS_CANT_OPEN_ANONYMOUS cpu_to_le32(0xC00000A6)
+#define STATUS_BAD_VALIDATION_CLASS cpu_to_le32(0xC00000A7)
+#define STATUS_BAD_TOKEN_TYPE cpu_to_le32(0xC00000A8)
+#define STATUS_BAD_MASTER_BOOT_RECORD cpu_to_le32(0xC00000A9)
+#define STATUS_INSTRUCTION_MISALIGNMENT cpu_to_le32(0xC00000AA)
+#define STATUS_INSTANCE_NOT_AVAILABLE cpu_to_le32(0xC00000AB)
+#define STATUS_PIPE_NOT_AVAILABLE cpu_to_le32(0xC00000AC)
+#define STATUS_INVALID_PIPE_STATE cpu_to_le32(0xC00000AD)
+#define STATUS_PIPE_BUSY cpu_to_le32(0xC00000AE)
+#define STATUS_ILLEGAL_FUNCTION cpu_to_le32(0xC00000AF)
+#define STATUS_PIPE_DISCONNECTED cpu_to_le32(0xC00000B0)
+#define STATUS_PIPE_CLOSING cpu_to_le32(0xC00000B1)
+#define STATUS_PIPE_CONNECTED cpu_to_le32(0xC00000B2)
+#define STATUS_PIPE_LISTENING cpu_to_le32(0xC00000B3)
+#define STATUS_INVALID_READ_MODE cpu_to_le32(0xC00000B4)
+#define STATUS_IO_TIMEOUT cpu_to_le32(0xC00000B5)
+#define STATUS_FILE_FORCED_CLOSED cpu_to_le32(0xC00000B6)
+#define STATUS_PROFILING_NOT_STARTED cpu_to_le32(0xC00000B7)
+#define STATUS_PROFILING_NOT_STOPPED cpu_to_le32(0xC00000B8)
+#define STATUS_COULD_NOT_INTERPRET cpu_to_le32(0xC00000B9)
+#define STATUS_FILE_IS_A_DIRECTORY cpu_to_le32(0xC00000BA)
+#define STATUS_NOT_SUPPORTED cpu_to_le32(0xC00000BB)
+#define STATUS_REMOTE_NOT_LISTENING cpu_to_le32(0xC00000BC)
+#define STATUS_DUPLICATE_NAME cpu_to_le32(0xC00000BD)
+#define STATUS_BAD_NETWORK_PATH cpu_to_le32(0xC00000BE)
+#define STATUS_NETWORK_BUSY cpu_to_le32(0xC00000BF)
+#define STATUS_DEVICE_DOES_NOT_EXIST cpu_to_le32(0xC00000C0)
+#define STATUS_TOO_MANY_COMMANDS cpu_to_le32(0xC00000C1)
+#define STATUS_ADAPTER_HARDWARE_ERROR cpu_to_le32(0xC00000C2)
+#define STATUS_INVALID_NETWORK_RESPONSE cpu_to_le32(0xC00000C3)
+#define STATUS_UNEXPECTED_NETWORK_ERROR cpu_to_le32(0xC00000C4)
+#define STATUS_BAD_REMOTE_ADAPTER cpu_to_le32(0xC00000C5)
+#define STATUS_PRINT_QUEUE_FULL cpu_to_le32(0xC00000C6)
+#define STATUS_NO_SPOOL_SPACE cpu_to_le32(0xC00000C7)
+#define STATUS_PRINT_CANCELLED cpu_to_le32(0xC00000C8)
+#define STATUS_NETWORK_NAME_DELETED cpu_to_le32(0xC00000C9)
+#define STATUS_NETWORK_ACCESS_DENIED cpu_to_le32(0xC00000CA)
+#define STATUS_BAD_DEVICE_TYPE cpu_to_le32(0xC00000CB)
+#define STATUS_BAD_NETWORK_NAME cpu_to_le32(0xC00000CC)
+#define STATUS_TOO_MANY_NAMES cpu_to_le32(0xC00000CD)
+#define STATUS_TOO_MANY_SESSIONS cpu_to_le32(0xC00000CE)
+#define STATUS_SHARING_PAUSED cpu_to_le32(0xC00000CF)
+#define STATUS_REQUEST_NOT_ACCEPTED cpu_to_le32(0xC00000D0)
+#define STATUS_REDIRECTOR_PAUSED cpu_to_le32(0xC00000D1)
+#define STATUS_NET_WRITE_FAULT cpu_to_le32(0xC00000D2)
+#define STATUS_PROFILING_AT_LIMIT cpu_to_le32(0xC00000D3)
+#define STATUS_NOT_SAME_DEVICE cpu_to_le32(0xC00000D4)
+#define STATUS_FILE_RENAMED cpu_to_le32(0xC00000D5)
+#define STATUS_VIRTUAL_CIRCUIT_CLOSED cpu_to_le32(0xC00000D6)
+#define STATUS_NO_SECURITY_ON_OBJECT cpu_to_le32(0xC00000D7)
+#define STATUS_CANT_WAIT cpu_to_le32(0xC00000D8)
+#define STATUS_PIPE_EMPTY cpu_to_le32(0xC00000D9)
+#define STATUS_CANT_ACCESS_DOMAIN_INFO cpu_to_le32(0xC00000DA)
+#define STATUS_CANT_TERMINATE_SELF cpu_to_le32(0xC00000DB)
+#define STATUS_INVALID_SERVER_STATE cpu_to_le32(0xC00000DC)
+#define STATUS_INVALID_DOMAIN_STATE cpu_to_le32(0xC00000DD)
+#define STATUS_INVALID_DOMAIN_ROLE cpu_to_le32(0xC00000DE)
+#define STATUS_NO_SUCH_DOMAIN cpu_to_le32(0xC00000DF)
+#define STATUS_DOMAIN_EXISTS cpu_to_le32(0xC00000E0)
+#define STATUS_DOMAIN_LIMIT_EXCEEDED cpu_to_le32(0xC00000E1)
+#define STATUS_OPLOCK_NOT_GRANTED cpu_to_le32(0xC00000E2)
+#define STATUS_INVALID_OPLOCK_PROTOCOL cpu_to_le32(0xC00000E3)
+#define STATUS_INTERNAL_DB_CORRUPTION cpu_to_le32(0xC00000E4)
+#define STATUS_INTERNAL_ERROR cpu_to_le32(0xC00000E5)
+#define STATUS_GENERIC_NOT_MAPPED cpu_to_le32(0xC00000E6)
+#define STATUS_BAD_DESCRIPTOR_FORMAT cpu_to_le32(0xC00000E7)
+#define STATUS_INVALID_USER_BUFFER cpu_to_le32(0xC00000E8)
+#define STATUS_UNEXPECTED_IO_ERROR cpu_to_le32(0xC00000E9)
+#define STATUS_UNEXPECTED_MM_CREATE_ERR cpu_to_le32(0xC00000EA)
+#define STATUS_UNEXPECTED_MM_MAP_ERROR cpu_to_le32(0xC00000EB)
+#define STATUS_UNEXPECTED_MM_EXTEND_ERR cpu_to_le32(0xC00000EC)
+#define STATUS_NOT_LOGON_PROCESS cpu_to_le32(0xC00000ED)
+#define STATUS_LOGON_SESSION_EXISTS cpu_to_le32(0xC00000EE)
+#define STATUS_INVALID_PARAMETER_1 cpu_to_le32(0xC00000EF)
+#define STATUS_INVALID_PARAMETER_2 cpu_to_le32(0xC00000F0)
+#define STATUS_INVALID_PARAMETER_3 cpu_to_le32(0xC00000F1)
+#define STATUS_INVALID_PARAMETER_4 cpu_to_le32(0xC00000F2)
+#define STATUS_INVALID_PARAMETER_5 cpu_to_le32(0xC00000F3)
+#define STATUS_INVALID_PARAMETER_6 cpu_to_le32(0xC00000F4)
+#define STATUS_INVALID_PARAMETER_7 cpu_to_le32(0xC00000F5)
+#define STATUS_INVALID_PARAMETER_8 cpu_to_le32(0xC00000F6)
+#define STATUS_INVALID_PARAMETER_9 cpu_to_le32(0xC00000F7)
+#define STATUS_INVALID_PARAMETER_10 cpu_to_le32(0xC00000F8)
+#define STATUS_INVALID_PARAMETER_11 cpu_to_le32(0xC00000F9)
+#define STATUS_INVALID_PARAMETER_12 cpu_to_le32(0xC00000FA)
+#define STATUS_REDIRECTOR_NOT_STARTED cpu_to_le32(0xC00000FB)
+#define STATUS_REDIRECTOR_STARTED cpu_to_le32(0xC00000FC)
+#define STATUS_STACK_OVERFLOW cpu_to_le32(0xC00000FD)
+#define STATUS_NO_SUCH_PACKAGE cpu_to_le32(0xC00000FE)
+#define STATUS_BAD_FUNCTION_TABLE cpu_to_le32(0xC00000FF)
+#define STATUS_VARIABLE_NOT_FOUND cpu_to_le32(0xC0000100)
+#define STATUS_DIRECTORY_NOT_EMPTY cpu_to_le32(0xC0000101)
+#define STATUS_FILE_CORRUPT_ERROR cpu_to_le32(0xC0000102)
+#define STATUS_NOT_A_DIRECTORY cpu_to_le32(0xC0000103)
+#define STATUS_BAD_LOGON_SESSION_STATE cpu_to_le32(0xC0000104)
+#define STATUS_LOGON_SESSION_COLLISION cpu_to_le32(0xC0000105)
+#define STATUS_NAME_TOO_LONG cpu_to_le32(0xC0000106)
+#define STATUS_FILES_OPEN cpu_to_le32(0xC0000107)
+#define STATUS_CONNECTION_IN_USE cpu_to_le32(0xC0000108)
+#define STATUS_MESSAGE_NOT_FOUND cpu_to_le32(0xC0000109)
+#define STATUS_PROCESS_IS_TERMINATING cpu_to_le32(0xC000010A)
+#define STATUS_INVALID_LOGON_TYPE cpu_to_le32(0xC000010B)
+#define STATUS_NO_GUID_TRANSLATION cpu_to_le32(0xC000010C)
+#define STATUS_CANNOT_IMPERSONATE cpu_to_le32(0xC000010D)
+#define STATUS_IMAGE_ALREADY_LOADED cpu_to_le32(0xC000010E)
+#define STATUS_ABIOS_NOT_PRESENT cpu_to_le32(0xC000010F)
+#define STATUS_ABIOS_LID_NOT_EXIST cpu_to_le32(0xC0000110)
+#define STATUS_ABIOS_LID_ALREADY_OWNED cpu_to_le32(0xC0000111)
+#define STATUS_ABIOS_NOT_LID_OWNER cpu_to_le32(0xC0000112)
+#define STATUS_ABIOS_INVALID_COMMAND cpu_to_le32(0xC0000113)
+#define STATUS_ABIOS_INVALID_LID cpu_to_le32(0xC0000114)
+#define STATUS_ABIOS_SELECTOR_NOT_AVAILABLE cpu_to_le32(0xC0000115)
+#define STATUS_ABIOS_INVALID_SELECTOR cpu_to_le32(0xC0000116)
+#define STATUS_NO_LDT cpu_to_le32(0xC0000117)
+#define STATUS_INVALID_LDT_SIZE cpu_to_le32(0xC0000118)
+#define STATUS_INVALID_LDT_OFFSET cpu_to_le32(0xC0000119)
+#define STATUS_INVALID_LDT_DESCRIPTOR cpu_to_le32(0xC000011A)
+#define STATUS_INVALID_IMAGE_NE_FORMAT cpu_to_le32(0xC000011B)
+#define STATUS_RXACT_INVALID_STATE cpu_to_le32(0xC000011C)
+#define STATUS_RXACT_COMMIT_FAILURE cpu_to_le32(0xC000011D)
+#define STATUS_MAPPED_FILE_SIZE_ZERO cpu_to_le32(0xC000011E)
+#define STATUS_TOO_MANY_OPENED_FILES cpu_to_le32(0xC000011F)
+#define STATUS_CANCELLED cpu_to_le32(0xC0000120)
+#define STATUS_CANNOT_DELETE cpu_to_le32(0xC0000121)
+#define STATUS_INVALID_COMPUTER_NAME cpu_to_le32(0xC0000122)
+#define STATUS_FILE_DELETED cpu_to_le32(0xC0000123)
+#define STATUS_SPECIAL_ACCOUNT cpu_to_le32(0xC0000124)
+#define STATUS_SPECIAL_GROUP cpu_to_le32(0xC0000125)
+#define STATUS_SPECIAL_USER cpu_to_le32(0xC0000126)
+#define STATUS_MEMBERS_PRIMARY_GROUP cpu_to_le32(0xC0000127)
+#define STATUS_FILE_CLOSED cpu_to_le32(0xC0000128)
+#define STATUS_TOO_MANY_THREADS cpu_to_le32(0xC0000129)
+#define STATUS_THREAD_NOT_IN_PROCESS cpu_to_le32(0xC000012A)
+#define STATUS_TOKEN_ALREADY_IN_USE cpu_to_le32(0xC000012B)
+#define STATUS_PAGEFILE_QUOTA_EXCEEDED cpu_to_le32(0xC000012C)
+#define STATUS_COMMITMENT_LIMIT cpu_to_le32(0xC000012D)
+#define STATUS_INVALID_IMAGE_LE_FORMAT cpu_to_le32(0xC000012E)
+#define STATUS_INVALID_IMAGE_NOT_MZ cpu_to_le32(0xC000012F)
+#define STATUS_INVALID_IMAGE_PROTECT cpu_to_le32(0xC0000130)
+#define STATUS_INVALID_IMAGE_WIN_16 cpu_to_le32(0xC0000131)
+#define STATUS_LOGON_SERVER_CONFLICT cpu_to_le32(0xC0000132)
+#define STATUS_TIME_DIFFERENCE_AT_DC cpu_to_le32(0xC0000133)
+#define STATUS_SYNCHRONIZATION_REQUIRED cpu_to_le32(0xC0000134)
+#define STATUS_DLL_NOT_FOUND cpu_to_le32(0xC0000135)
+#define STATUS_OPEN_FAILED cpu_to_le32(0xC0000136)
+#define STATUS_IO_PRIVILEGE_FAILED cpu_to_le32(0xC0000137)
+#define STATUS_ORDINAL_NOT_FOUND cpu_to_le32(0xC0000138)
+#define STATUS_ENTRYPOINT_NOT_FOUND cpu_to_le32(0xC0000139)
+#define STATUS_CONTROL_C_EXIT cpu_to_le32(0xC000013A)
+#define STATUS_LOCAL_DISCONNECT cpu_to_le32(0xC000013B)
+#define STATUS_REMOTE_DISCONNECT cpu_to_le32(0xC000013C)
+#define STATUS_REMOTE_RESOURCES cpu_to_le32(0xC000013D)
+#define STATUS_LINK_FAILED cpu_to_le32(0xC000013E)
+#define STATUS_LINK_TIMEOUT cpu_to_le32(0xC000013F)
+#define STATUS_INVALID_CONNECTION cpu_to_le32(0xC0000140)
+#define STATUS_INVALID_ADDRESS cpu_to_le32(0xC0000141)
+#define STATUS_DLL_INIT_FAILED cpu_to_le32(0xC0000142)
+#define STATUS_MISSING_SYSTEMFILE cpu_to_le32(0xC0000143)
+#define STATUS_UNHANDLED_EXCEPTION cpu_to_le32(0xC0000144)
+#define STATUS_APP_INIT_FAILURE cpu_to_le32(0xC0000145)
+#define STATUS_PAGEFILE_CREATE_FAILED cpu_to_le32(0xC0000146)
+#define STATUS_NO_PAGEFILE cpu_to_le32(0xC0000147)
+#define STATUS_INVALID_LEVEL cpu_to_le32(0xC0000148)
+#define STATUS_WRONG_PASSWORD_CORE cpu_to_le32(0xC0000149)
+#define STATUS_ILLEGAL_FLOAT_CONTEXT cpu_to_le32(0xC000014A)
+#define STATUS_PIPE_BROKEN cpu_to_le32(0xC000014B)
+#define STATUS_REGISTRY_CORRUPT cpu_to_le32(0xC000014C)
+#define STATUS_REGISTRY_IO_FAILED cpu_to_le32(0xC000014D)
+#define STATUS_NO_EVENT_PAIR cpu_to_le32(0xC000014E)
+#define STATUS_UNRECOGNIZED_VOLUME cpu_to_le32(0xC000014F)
+#define STATUS_SERIAL_NO_DEVICE_INITED cpu_to_le32(0xC0000150)
+#define STATUS_NO_SUCH_ALIAS cpu_to_le32(0xC0000151)
+#define STATUS_MEMBER_NOT_IN_ALIAS cpu_to_le32(0xC0000152)
+#define STATUS_MEMBER_IN_ALIAS cpu_to_le32(0xC0000153)
+#define STATUS_ALIAS_EXISTS cpu_to_le32(0xC0000154)
+#define STATUS_LOGON_NOT_GRANTED cpu_to_le32(0xC0000155)
+#define STATUS_TOO_MANY_SECRETS cpu_to_le32(0xC0000156)
+#define STATUS_SECRET_TOO_LONG cpu_to_le32(0xC0000157)
+#define STATUS_INTERNAL_DB_ERROR cpu_to_le32(0xC0000158)
+#define STATUS_FULLSCREEN_MODE cpu_to_le32(0xC0000159)
+#define STATUS_TOO_MANY_CONTEXT_IDS cpu_to_le32(0xC000015A)
+#define STATUS_LOGON_TYPE_NOT_GRANTED cpu_to_le32(0xC000015B)
+#define STATUS_NOT_REGISTRY_FILE cpu_to_le32(0xC000015C)
+#define STATUS_NT_CROSS_ENCRYPTION_REQUIRED cpu_to_le32(0xC000015D)
+#define STATUS_DOMAIN_CTRLR_CONFIG_ERROR cpu_to_le32(0xC000015E)
+#define STATUS_FT_MISSING_MEMBER cpu_to_le32(0xC000015F)
+#define STATUS_ILL_FORMED_SERVICE_ENTRY cpu_to_le32(0xC0000160)
+#define STATUS_ILLEGAL_CHARACTER cpu_to_le32(0xC0000161)
+#define STATUS_UNMAPPABLE_CHARACTER cpu_to_le32(0xC0000162)
+#define STATUS_UNDEFINED_CHARACTER cpu_to_le32(0xC0000163)
+#define STATUS_FLOPPY_VOLUME cpu_to_le32(0xC0000164)
+#define STATUS_FLOPPY_ID_MARK_NOT_FOUND cpu_to_le32(0xC0000165)
+#define STATUS_FLOPPY_WRONG_CYLINDER cpu_to_le32(0xC0000166)
+#define STATUS_FLOPPY_UNKNOWN_ERROR cpu_to_le32(0xC0000167)
+#define STATUS_FLOPPY_BAD_REGISTERS cpu_to_le32(0xC0000168)
+#define STATUS_DISK_RECALIBRATE_FAILED cpu_to_le32(0xC0000169)
+#define STATUS_DISK_OPERATION_FAILED cpu_to_le32(0xC000016A)
+#define STATUS_DISK_RESET_FAILED cpu_to_le32(0xC000016B)
+#define STATUS_SHARED_IRQ_BUSY cpu_to_le32(0xC000016C)
+#define STATUS_FT_ORPHANING cpu_to_le32(0xC000016D)
+#define STATUS_BIOS_FAILED_TO_CONNECT_INTERRUPT cpu_to_le32(0xC000016E)
+#define STATUS_PARTITION_FAILURE cpu_to_le32(0xC0000172)
+#define STATUS_INVALID_BLOCK_LENGTH cpu_to_le32(0xC0000173)
+#define STATUS_DEVICE_NOT_PARTITIONED cpu_to_le32(0xC0000174)
+#define STATUS_UNABLE_TO_LOCK_MEDIA cpu_to_le32(0xC0000175)
+#define STATUS_UNABLE_TO_UNLOAD_MEDIA cpu_to_le32(0xC0000176)
+#define STATUS_EOM_OVERFLOW cpu_to_le32(0xC0000177)
+#define STATUS_NO_MEDIA cpu_to_le32(0xC0000178)
+#define STATUS_NO_SUCH_MEMBER cpu_to_le32(0xC000017A)
+#define STATUS_INVALID_MEMBER cpu_to_le32(0xC000017B)
+#define STATUS_KEY_DELETED cpu_to_le32(0xC000017C)
+#define STATUS_NO_LOG_SPACE cpu_to_le32(0xC000017D)
+#define STATUS_TOO_MANY_SIDS cpu_to_le32(0xC000017E)
+#define STATUS_LM_CROSS_ENCRYPTION_REQUIRED cpu_to_le32(0xC000017F)
+#define STATUS_KEY_HAS_CHILDREN cpu_to_le32(0xC0000180)
+#define STATUS_CHILD_MUST_BE_VOLATILE cpu_to_le32(0xC0000181)
+#define STATUS_DEVICE_CONFIGURATION_ERROR cpu_to_le32(0xC0000182)
+#define STATUS_DRIVER_INTERNAL_ERROR cpu_to_le32(0xC0000183)
+#define STATUS_INVALID_DEVICE_STATE cpu_to_le32(0xC0000184)
+#define STATUS_IO_DEVICE_ERROR cpu_to_le32(0xC0000185)
+#define STATUS_DEVICE_PROTOCOL_ERROR cpu_to_le32(0xC0000186)
+#define STATUS_BACKUP_CONTROLLER cpu_to_le32(0xC0000187)
+#define STATUS_LOG_FILE_FULL cpu_to_le32(0xC0000188)
+#define STATUS_TOO_LATE cpu_to_le32(0xC0000189)
+#define STATUS_NO_TRUST_LSA_SECRET cpu_to_le32(0xC000018A)
+#define STATUS_NO_TRUST_SAM_ACCOUNT cpu_to_le32(0xC000018B)
+#define STATUS_TRUSTED_DOMAIN_FAILURE cpu_to_le32(0xC000018C)
+#define STATUS_TRUSTED_RELATIONSHIP_FAILURE cpu_to_le32(0xC000018D)
+#define STATUS_EVENTLOG_FILE_CORRUPT cpu_to_le32(0xC000018E)
+#define STATUS_EVENTLOG_CANT_START cpu_to_le32(0xC000018F)
+#define STATUS_TRUST_FAILURE cpu_to_le32(0xC0000190)
+#define STATUS_MUTANT_LIMIT_EXCEEDED cpu_to_le32(0xC0000191)
+#define STATUS_NETLOGON_NOT_STARTED cpu_to_le32(0xC0000192)
+#define STATUS_ACCOUNT_EXPIRED cpu_to_le32(0xC0000193)
+#define STATUS_POSSIBLE_DEADLOCK cpu_to_le32(0xC0000194)
+#define STATUS_NETWORK_CREDENTIAL_CONFLICT cpu_to_le32(0xC0000195)
+#define STATUS_REMOTE_SESSION_LIMIT cpu_to_le32(0xC0000196)
+#define STATUS_EVENTLOG_FILE_CHANGED cpu_to_le32(0xC0000197)
+#define STATUS_NOLOGON_INTERDOMAIN_TRUST_ACCOUNT cpu_to_le32(0xC0000198)
+#define STATUS_NOLOGON_WORKSTATION_TRUST_ACCOUNT cpu_to_le32(0xC0000199)
+#define STATUS_NOLOGON_SERVER_TRUST_ACCOUNT cpu_to_le32(0xC000019A)
+#define STATUS_DOMAIN_TRUST_INCONSISTENT cpu_to_le32(0xC000019B)
+#define STATUS_FS_DRIVER_REQUIRED cpu_to_le32(0xC000019C)
+#define STATUS_IMAGE_ALREADY_LOADED_AS_DLL cpu_to_le32(0xC000019D)
+#define STATUS_NETWORK_OPEN_RESTRICTION cpu_to_le32(0xC0000201)
+#define STATUS_NO_USER_SESSION_KEY cpu_to_le32(0xC0000202)
+#define STATUS_USER_SESSION_DELETED cpu_to_le32(0xC0000203)
+#define STATUS_RESOURCE_LANG_NOT_FOUND cpu_to_le32(0xC0000204)
+#define STATUS_INSUFF_SERVER_RESOURCES cpu_to_le32(0xC0000205)
+#define STATUS_INVALID_BUFFER_SIZE cpu_to_le32(0xC0000206)
+#define STATUS_INVALID_ADDRESS_COMPONENT cpu_to_le32(0xC0000207)
+#define STATUS_INVALID_ADDRESS_WILDCARD cpu_to_le32(0xC0000208)
+#define STATUS_TOO_MANY_ADDRESSES cpu_to_le32(0xC0000209)
+#define STATUS_ADDRESS_ALREADY_EXISTS cpu_to_le32(0xC000020A)
+#define STATUS_ADDRESS_CLOSED cpu_to_le32(0xC000020B)
+#define STATUS_CONNECTION_DISCONNECTED cpu_to_le32(0xC000020C)
+#define STATUS_CONNECTION_RESET cpu_to_le32(0xC000020D)
+#define STATUS_TOO_MANY_NODES cpu_to_le32(0xC000020E)
+#define STATUS_TRANSACTION_ABORTED cpu_to_le32(0xC000020F)
+#define STATUS_TRANSACTION_TIMED_OUT cpu_to_le32(0xC0000210)
+#define STATUS_TRANSACTION_NO_RELEASE cpu_to_le32(0xC0000211)
+#define STATUS_TRANSACTION_NO_MATCH cpu_to_le32(0xC0000212)
+#define STATUS_TRANSACTION_RESPONDED cpu_to_le32(0xC0000213)
+#define STATUS_TRANSACTION_INVALID_ID cpu_to_le32(0xC0000214)
+#define STATUS_TRANSACTION_INVALID_TYPE cpu_to_le32(0xC0000215)
+#define STATUS_NOT_SERVER_SESSION cpu_to_le32(0xC0000216)
+#define STATUS_NOT_CLIENT_SESSION cpu_to_le32(0xC0000217)
+#define STATUS_CANNOT_LOAD_REGISTRY_FILE cpu_to_le32(0xC0000218)
+#define STATUS_DEBUG_ATTACH_FAILED cpu_to_le32(0xC0000219)
+#define STATUS_SYSTEM_PROCESS_TERMINATED cpu_to_le32(0xC000021A)
+#define STATUS_DATA_NOT_ACCEPTED cpu_to_le32(0xC000021B)
+#define STATUS_NO_BROWSER_SERVERS_FOUND cpu_to_le32(0xC000021C)
+#define STATUS_VDM_HARD_ERROR cpu_to_le32(0xC000021D)
+#define STATUS_DRIVER_CANCEL_TIMEOUT cpu_to_le32(0xC000021E)
+#define STATUS_REPLY_MESSAGE_MISMATCH cpu_to_le32(0xC000021F)
+#define STATUS_MAPPED_ALIGNMENT cpu_to_le32(0xC0000220)
+#define STATUS_IMAGE_CHECKSUM_MISMATCH cpu_to_le32(0xC0000221)
+#define STATUS_LOST_WRITEBEHIND_DATA cpu_to_le32(0xC0000222)
+#define STATUS_CLIENT_SERVER_PARAMETERS_INVALID cpu_to_le32(0xC0000223)
+#define STATUS_PASSWORD_MUST_CHANGE cpu_to_le32(0xC0000224)
+#define STATUS_NOT_FOUND cpu_to_le32(0xC0000225)
+#define STATUS_NOT_TINY_STREAM cpu_to_le32(0xC0000226)
+#define STATUS_RECOVERY_FAILURE cpu_to_le32(0xC0000227)
+#define STATUS_STACK_OVERFLOW_READ cpu_to_le32(0xC0000228)
+#define STATUS_FAIL_CHECK cpu_to_le32(0xC0000229)
+#define STATUS_DUPLICATE_OBJECTID cpu_to_le32(0xC000022A)
+#define STATUS_OBJECTID_EXISTS cpu_to_le32(0xC000022B)
+#define STATUS_CONVERT_TO_LARGE cpu_to_le32(0xC000022C)
+#define STATUS_RETRY cpu_to_le32(0xC000022D)
+#define STATUS_FOUND_OUT_OF_SCOPE cpu_to_le32(0xC000022E)
+#define STATUS_ALLOCATE_BUCKET cpu_to_le32(0xC000022F)
+#define STATUS_PROPSET_NOT_FOUND cpu_to_le32(0xC0000230)
+#define STATUS_MARSHALL_OVERFLOW cpu_to_le32(0xC0000231)
+#define STATUS_INVALID_VARIANT cpu_to_le32(0xC0000232)
+#define STATUS_DOMAIN_CONTROLLER_NOT_FOUND cpu_to_le32(0xC0000233)
+#define STATUS_ACCOUNT_LOCKED_OUT cpu_to_le32(0xC0000234)
+#define STATUS_HANDLE_NOT_CLOSABLE cpu_to_le32(0xC0000235)
+#define STATUS_CONNECTION_REFUSED cpu_to_le32(0xC0000236)
+#define STATUS_GRACEFUL_DISCONNECT cpu_to_le32(0xC0000237)
+#define STATUS_ADDRESS_ALREADY_ASSOCIATED cpu_to_le32(0xC0000238)
+#define STATUS_ADDRESS_NOT_ASSOCIATED cpu_to_le32(0xC0000239)
+#define STATUS_CONNECTION_INVALID cpu_to_le32(0xC000023A)
+#define STATUS_CONNECTION_ACTIVE cpu_to_le32(0xC000023B)
+#define STATUS_NETWORK_UNREACHABLE cpu_to_le32(0xC000023C)
+#define STATUS_HOST_UNREACHABLE cpu_to_le32(0xC000023D)
+#define STATUS_PROTOCOL_UNREACHABLE cpu_to_le32(0xC000023E)
+#define STATUS_PORT_UNREACHABLE cpu_to_le32(0xC000023F)
+#define STATUS_REQUEST_ABORTED cpu_to_le32(0xC0000240)
+#define STATUS_CONNECTION_ABORTED cpu_to_le32(0xC0000241)
+#define STATUS_BAD_COMPRESSION_BUFFER cpu_to_le32(0xC0000242)
+#define STATUS_USER_MAPPED_FILE cpu_to_le32(0xC0000243)
+#define STATUS_AUDIT_FAILED cpu_to_le32(0xC0000244)
+#define STATUS_TIMER_RESOLUTION_NOT_SET cpu_to_le32(0xC0000245)
+#define STATUS_CONNECTION_COUNT_LIMIT cpu_to_le32(0xC0000246)
+#define STATUS_LOGIN_TIME_RESTRICTION cpu_to_le32(0xC0000247)
+#define STATUS_LOGIN_WKSTA_RESTRICTION cpu_to_le32(0xC0000248)
+#define STATUS_IMAGE_MP_UP_MISMATCH cpu_to_le32(0xC0000249)
+#define STATUS_INSUFFICIENT_LOGON_INFO cpu_to_le32(0xC0000250)
+#define STATUS_BAD_DLL_ENTRYPOINT cpu_to_le32(0xC0000251)
+#define STATUS_BAD_SERVICE_ENTRYPOINT cpu_to_le32(0xC0000252)
+#define STATUS_LPC_REPLY_LOST cpu_to_le32(0xC0000253)
+#define STATUS_IP_ADDRESS_CONFLICT1 cpu_to_le32(0xC0000254)
+#define STATUS_IP_ADDRESS_CONFLICT2 cpu_to_le32(0xC0000255)
+#define STATUS_REGISTRY_QUOTA_LIMIT cpu_to_le32(0xC0000256)
+#define STATUS_PATH_NOT_COVERED cpu_to_le32(0xC0000257)
+#define STATUS_NO_CALLBACK_ACTIVE cpu_to_le32(0xC0000258)
+#define STATUS_LICENSE_QUOTA_EXCEEDED cpu_to_le32(0xC0000259)
+#define STATUS_PWD_TOO_SHORT cpu_to_le32(0xC000025A)
+#define STATUS_PWD_TOO_RECENT cpu_to_le32(0xC000025B)
+#define STATUS_PWD_HISTORY_CONFLICT cpu_to_le32(0xC000025C)
+#define STATUS_PLUGPLAY_NO_DEVICE cpu_to_le32(0xC000025E)
+#define STATUS_UNSUPPORTED_COMPRESSION cpu_to_le32(0xC000025F)
+#define STATUS_INVALID_HW_PROFILE cpu_to_le32(0xC0000260)
+#define STATUS_INVALID_PLUGPLAY_DEVICE_PATH cpu_to_le32(0xC0000261)
+#define STATUS_DRIVER_ORDINAL_NOT_FOUND cpu_to_le32(0xC0000262)
+#define STATUS_DRIVER_ENTRYPOINT_NOT_FOUND cpu_to_le32(0xC0000263)
+#define STATUS_RESOURCE_NOT_OWNED cpu_to_le32(0xC0000264)
+#define STATUS_TOO_MANY_LINKS cpu_to_le32(0xC0000265)
+#define STATUS_QUOTA_LIST_INCONSISTENT cpu_to_le32(0xC0000266)
+#define STATUS_FILE_IS_OFFLINE cpu_to_le32(0xC0000267)
+#define STATUS_EVALUATION_EXPIRATION cpu_to_le32(0xC0000268)
+#define STATUS_ILLEGAL_DLL_RELOCATION cpu_to_le32(0xC0000269)
+#define STATUS_LICENSE_VIOLATION cpu_to_le32(0xC000026A)
+#define STATUS_DLL_INIT_FAILED_LOGOFF cpu_to_le32(0xC000026B)
+#define STATUS_DRIVER_UNABLE_TO_LOAD cpu_to_le32(0xC000026C)
+#define STATUS_DFS_UNAVAILABLE cpu_to_le32(0xC000026D)
+#define STATUS_VOLUME_DISMOUNTED cpu_to_le32(0xC000026E)
+#define STATUS_WX86_INTERNAL_ERROR cpu_to_le32(0xC000026F)
+#define STATUS_WX86_FLOAT_STACK_CHECK cpu_to_le32(0xC0000270)
+#define STATUS_VALIDATE_CONTINUE cpu_to_le32(0xC0000271)
+#define STATUS_NO_MATCH cpu_to_le32(0xC0000272)
+#define STATUS_NO_MORE_MATCHES cpu_to_le32(0xC0000273)
+#define STATUS_NOT_A_REPARSE_POINT cpu_to_le32(0xC0000275)
+#define STATUS_IO_REPARSE_TAG_INVALID cpu_to_le32(0xC0000276)
+#define STATUS_IO_REPARSE_TAG_MISMATCH cpu_to_le32(0xC0000277)
+#define STATUS_IO_REPARSE_DATA_INVALID cpu_to_le32(0xC0000278)
+#define STATUS_IO_REPARSE_TAG_NOT_HANDLED cpu_to_le32(0xC0000279)
+#define STATUS_REPARSE_POINT_NOT_RESOLVED cpu_to_le32(0xC0000280)
+#define STATUS_DIRECTORY_IS_A_REPARSE_POINT cpu_to_le32(0xC0000281)
+#define STATUS_RANGE_LIST_CONFLICT cpu_to_le32(0xC0000282)
+#define STATUS_SOURCE_ELEMENT_EMPTY cpu_to_le32(0xC0000283)
+#define STATUS_DESTINATION_ELEMENT_FULL cpu_to_le32(0xC0000284)
+#define STATUS_ILLEGAL_ELEMENT_ADDRESS cpu_to_le32(0xC0000285)
+#define STATUS_MAGAZINE_NOT_PRESENT cpu_to_le32(0xC0000286)
+#define STATUS_REINITIALIZATION_NEEDED cpu_to_le32(0xC0000287)
+#define STATUS_ENCRYPTION_FAILED cpu_to_le32(0xC000028A)
+#define STATUS_DECRYPTION_FAILED cpu_to_le32(0xC000028B)
+#define STATUS_RANGE_NOT_FOUND cpu_to_le32(0xC000028C)
+#define STATUS_NO_RECOVERY_POLICY cpu_to_le32(0xC000028D)
+#define STATUS_NO_EFS cpu_to_le32(0xC000028E)
+#define STATUS_WRONG_EFS cpu_to_le32(0xC000028F)
+#define STATUS_NO_USER_KEYS cpu_to_le32(0xC0000290)
+#define STATUS_FILE_NOT_ENCRYPTED cpu_to_le32(0xC0000291)
+#define STATUS_NOT_EXPORT_FORMAT cpu_to_le32(0xC0000292)
+#define STATUS_FILE_ENCRYPTED cpu_to_le32(0xC0000293)
+#define STATUS_WMI_GUID_NOT_FOUND cpu_to_le32(0xC0000295)
+#define STATUS_WMI_INSTANCE_NOT_FOUND cpu_to_le32(0xC0000296)
+#define STATUS_WMI_ITEMID_NOT_FOUND cpu_to_le32(0xC0000297)
+#define STATUS_WMI_TRY_AGAIN cpu_to_le32(0xC0000298)
+#define STATUS_SHARED_POLICY cpu_to_le32(0xC0000299)
+#define STATUS_POLICY_OBJECT_NOT_FOUND cpu_to_le32(0xC000029A)
+#define STATUS_POLICY_ONLY_IN_DS cpu_to_le32(0xC000029B)
+#define STATUS_VOLUME_NOT_UPGRADED cpu_to_le32(0xC000029C)
+#define STATUS_REMOTE_STORAGE_NOT_ACTIVE cpu_to_le32(0xC000029D)
+#define STATUS_REMOTE_STORAGE_MEDIA_ERROR cpu_to_le32(0xC000029E)
+#define STATUS_NO_TRACKING_SERVICE cpu_to_le32(0xC000029F)
+#define STATUS_SERVER_SID_MISMATCH cpu_to_le32(0xC00002A0)
+#define STATUS_DS_NO_ATTRIBUTE_OR_VALUE cpu_to_le32(0xC00002A1)
+#define STATUS_DS_INVALID_ATTRIBUTE_SYNTAX cpu_to_le32(0xC00002A2)
+#define STATUS_DS_ATTRIBUTE_TYPE_UNDEFINED cpu_to_le32(0xC00002A3)
+#define STATUS_DS_ATTRIBUTE_OR_VALUE_EXISTS cpu_to_le32(0xC00002A4)
+#define STATUS_DS_BUSY cpu_to_le32(0xC00002A5)
+#define STATUS_DS_UNAVAILABLE cpu_to_le32(0xC00002A6)
+#define STATUS_DS_NO_RIDS_ALLOCATED cpu_to_le32(0xC00002A7)
+#define STATUS_DS_NO_MORE_RIDS cpu_to_le32(0xC00002A8)
+#define STATUS_DS_INCORRECT_ROLE_OWNER cpu_to_le32(0xC00002A9)
+#define STATUS_DS_RIDMGR_INIT_ERROR cpu_to_le32(0xC00002AA)
+#define STATUS_DS_OBJ_CLASS_VIOLATION cpu_to_le32(0xC00002AB)
+#define STATUS_DS_CANT_ON_NON_LEAF cpu_to_le32(0xC00002AC)
+#define STATUS_DS_CANT_ON_RDN cpu_to_le32(0xC00002AD)
+#define STATUS_DS_CANT_MOD_OBJ_CLASS cpu_to_le32(0xC00002AE)
+#define STATUS_DS_CROSS_DOM_MOVE_FAILED cpu_to_le32(0xC00002AF)
+#define STATUS_DS_GC_NOT_AVAILABLE cpu_to_le32(0xC00002B0)
+#define STATUS_DIRECTORY_SERVICE_REQUIRED cpu_to_le32(0xC00002B1)
+#define STATUS_REPARSE_ATTRIBUTE_CONFLICT cpu_to_le32(0xC00002B2)
+#define STATUS_CANT_ENABLE_DENY_ONLY cpu_to_le32(0xC00002B3)
+#define STATUS_FLOAT_MULTIPLE_FAULTS cpu_to_le32(0xC00002B4)
+#define STATUS_FLOAT_MULTIPLE_TRAPS cpu_to_le32(0xC00002B5)
+#define STATUS_DEVICE_REMOVED cpu_to_le32(0xC00002B6)
+#define STATUS_JOURNAL_DELETE_IN_PROGRESS cpu_to_le32(0xC00002B7)
+#define STATUS_JOURNAL_NOT_ACTIVE cpu_to_le32(0xC00002B8)
+#define STATUS_NOINTERFACE cpu_to_le32(0xC00002B9)
+#define STATUS_DS_ADMIN_LIMIT_EXCEEDED cpu_to_le32(0xC00002C1)
+#define STATUS_DRIVER_FAILED_SLEEP cpu_to_le32(0xC00002C2)
+#define STATUS_MUTUAL_AUTHENTICATION_FAILED cpu_to_le32(0xC00002C3)
+#define STATUS_CORRUPT_SYSTEM_FILE cpu_to_le32(0xC00002C4)
+#define STATUS_DATATYPE_MISALIGNMENT_ERROR cpu_to_le32(0xC00002C5)
+#define STATUS_WMI_READ_ONLY cpu_to_le32(0xC00002C6)
+#define STATUS_WMI_SET_FAILURE cpu_to_le32(0xC00002C7)
+#define STATUS_COMMITMENT_MINIMUM cpu_to_le32(0xC00002C8)
+#define STATUS_REG_NAT_CONSUMPTION cpu_to_le32(0xC00002C9)
+#define STATUS_TRANSPORT_FULL cpu_to_le32(0xC00002CA)
+#define STATUS_DS_SAM_INIT_FAILURE cpu_to_le32(0xC00002CB)
+#define STATUS_ONLY_IF_CONNECTED cpu_to_le32(0xC00002CC)
+#define STATUS_DS_SENSITIVE_GROUP_VIOLATION cpu_to_le32(0xC00002CD)
+#define STATUS_PNP_RESTART_ENUMERATION cpu_to_le32(0xC00002CE)
+#define STATUS_JOURNAL_ENTRY_DELETED cpu_to_le32(0xC00002CF)
+#define STATUS_DS_CANT_MOD_PRIMARYGROUPID cpu_to_le32(0xC00002D0)
+#define STATUS_SYSTEM_IMAGE_BAD_SIGNATURE cpu_to_le32(0xC00002D1)
+#define STATUS_PNP_REBOOT_REQUIRED cpu_to_le32(0xC00002D2)
+#define STATUS_POWER_STATE_INVALID cpu_to_le32(0xC00002D3)
+#define STATUS_DS_INVALID_GROUP_TYPE cpu_to_le32(0xC00002D4)
+#define STATUS_DS_NO_NEST_GLOBALGROUP_IN_MIXEDDOMAIN cpu_to_le32(0xC00002D5)
+#define STATUS_DS_NO_NEST_LOCALGROUP_IN_MIXEDDOMAIN cpu_to_le32(0xC00002D6)
+#define STATUS_DS_GLOBAL_CANT_HAVE_LOCAL_MEMBER cpu_to_le32(0xC00002D7)
+#define STATUS_DS_GLOBAL_CANT_HAVE_UNIVERSAL_MEMBER cpu_to_le32(0xC00002D8)
+#define STATUS_DS_UNIVERSAL_CANT_HAVE_LOCAL_MEMBER cpu_to_le32(0xC00002D9)
+#define STATUS_DS_GLOBAL_CANT_HAVE_CROSSDOMAIN_MEMBER cpu_to_le32(0xC00002DA)
+#define STATUS_DS_LOCAL_CANT_HAVE_CROSSDOMAIN_LOCAL_MEMBER cpu_to_le32(0xC00002DB)
+#define STATUS_DS_HAVE_PRIMARY_MEMBERS cpu_to_le32(0xC00002DC)
+#define STATUS_WMI_NOT_SUPPORTED cpu_to_le32(0xC00002DD)
+#define STATUS_INSUFFICIENT_POWER cpu_to_le32(0xC00002DE)
+#define STATUS_SAM_NEED_BOOTKEY_PASSWORD cpu_to_le32(0xC00002DF)
+#define STATUS_SAM_NEED_BOOTKEY_FLOPPY cpu_to_le32(0xC00002E0)
+#define STATUS_DS_CANT_START cpu_to_le32(0xC00002E1)
+#define STATUS_DS_INIT_FAILURE cpu_to_le32(0xC00002E2)
+#define STATUS_SAM_INIT_FAILURE cpu_to_le32(0xC00002E3)
+#define STATUS_DS_GC_REQUIRED cpu_to_le32(0xC00002E4)
+#define STATUS_DS_LOCAL_MEMBER_OF_LOCAL_ONLY cpu_to_le32(0xC00002E5)
+#define STATUS_DS_NO_FPO_IN_UNIVERSAL_GROUPS cpu_to_le32(0xC00002E6)
+#define STATUS_DS_MACHINE_ACCOUNT_QUOTA_EXCEEDED cpu_to_le32(0xC00002E7)
+#define STATUS_MULTIPLE_FAULT_VIOLATION cpu_to_le32(0xC00002E8)
+#define STATUS_CURRENT_DOMAIN_NOT_ALLOWED cpu_to_le32(0xC00002E9)
+#define STATUS_CANNOT_MAKE cpu_to_le32(0xC00002EA)
+#define STATUS_SYSTEM_SHUTDOWN cpu_to_le32(0xC00002EB)
+#define STATUS_DS_INIT_FAILURE_CONSOLE cpu_to_le32(0xC00002EC)
+#define STATUS_DS_SAM_INIT_FAILURE_CONSOLE cpu_to_le32(0xC00002ED)
+#define STATUS_UNFINISHED_CONTEXT_DELETED cpu_to_le32(0xC00002EE)
+#define STATUS_NO_TGT_REPLY cpu_to_le32(0xC00002EF)
+#define STATUS_OBJECTID_NOT_FOUND cpu_to_le32(0xC00002F0)
+#define STATUS_NO_IP_ADDRESSES cpu_to_le32(0xC00002F1)
+#define STATUS_WRONG_CREDENTIAL_HANDLE cpu_to_le32(0xC00002F2)
+#define STATUS_CRYPTO_SYSTEM_INVALID cpu_to_le32(0xC00002F3)
+#define STATUS_MAX_REFERRALS_EXCEEDED cpu_to_le32(0xC00002F4)
+#define STATUS_MUST_BE_KDC cpu_to_le32(0xC00002F5)
+#define STATUS_STRONG_CRYPTO_NOT_SUPPORTED cpu_to_le32(0xC00002F6)
+#define STATUS_TOO_MANY_PRINCIPALS cpu_to_le32(0xC00002F7)
+#define STATUS_NO_PA_DATA cpu_to_le32(0xC00002F8)
+#define STATUS_PKINIT_NAME_MISMATCH cpu_to_le32(0xC00002F9)
+#define STATUS_SMARTCARD_LOGON_REQUIRED cpu_to_le32(0xC00002FA)
+#define STATUS_KDC_INVALID_REQUEST cpu_to_le32(0xC00002FB)
+#define STATUS_KDC_UNABLE_TO_REFER cpu_to_le32(0xC00002FC)
+#define STATUS_KDC_UNKNOWN_ETYPE cpu_to_le32(0xC00002FD)
+#define STATUS_SHUTDOWN_IN_PROGRESS cpu_to_le32(0xC00002FE)
+#define STATUS_SERVER_SHUTDOWN_IN_PROGRESS cpu_to_le32(0xC00002FF)
+#define STATUS_NOT_SUPPORTED_ON_SBS cpu_to_le32(0xC0000300)
+#define STATUS_WMI_GUID_DISCONNECTED cpu_to_le32(0xC0000301)
+#define STATUS_WMI_ALREADY_DISABLED cpu_to_le32(0xC0000302)
+#define STATUS_WMI_ALREADY_ENABLED cpu_to_le32(0xC0000303)
+#define STATUS_MFT_TOO_FRAGMENTED cpu_to_le32(0xC0000304)
+#define STATUS_COPY_PROTECTION_FAILURE cpu_to_le32(0xC0000305)
+#define STATUS_CSS_AUTHENTICATION_FAILURE cpu_to_le32(0xC0000306)
+#define STATUS_CSS_KEY_NOT_PRESENT cpu_to_le32(0xC0000307)
+#define STATUS_CSS_KEY_NOT_ESTABLISHED cpu_to_le32(0xC0000308)
+#define STATUS_CSS_SCRAMBLED_SECTOR cpu_to_le32(0xC0000309)
+#define STATUS_CSS_REGION_MISMATCH cpu_to_le32(0xC000030A)
+#define STATUS_CSS_RESETS_EXHAUSTED cpu_to_le32(0xC000030B)
+#define STATUS_PKINIT_FAILURE cpu_to_le32(0xC0000320)
+#define STATUS_SMARTCARD_SUBSYSTEM_FAILURE cpu_to_le32(0xC0000321)
+#define STATUS_NO_KERB_KEY cpu_to_le32(0xC0000322)
+#define STATUS_HOST_DOWN cpu_to_le32(0xC0000350)
+#define STATUS_UNSUPPORTED_PREAUTH cpu_to_le32(0xC0000351)
+#define STATUS_EFS_ALG_BLOB_TOO_BIG cpu_to_le32(0xC0000352)
+#define STATUS_PORT_NOT_SET cpu_to_le32(0xC0000353)
+#define STATUS_DEBUGGER_INACTIVE cpu_to_le32(0xC0000354)
+#define STATUS_DS_VERSION_CHECK_FAILURE cpu_to_le32(0xC0000355)
+#define STATUS_AUDITING_DISABLED cpu_to_le32(0xC0000356)
+#define STATUS_PRENT4_MACHINE_ACCOUNT cpu_to_le32(0xC0000357)
+#define STATUS_DS_AG_CANT_HAVE_UNIVERSAL_MEMBER cpu_to_le32(0xC0000358)
+#define STATUS_INVALID_IMAGE_WIN_32 cpu_to_le32(0xC0000359)
+#define STATUS_INVALID_IMAGE_WIN_64 cpu_to_le32(0xC000035A)
+#define STATUS_BAD_BINDINGS cpu_to_le32(0xC000035B)
+#define STATUS_NETWORK_SESSION_EXPIRED cpu_to_le32(0xC000035C)
+#define STATUS_APPHELP_BLOCK cpu_to_le32(0xC000035D)
+#define STATUS_ALL_SIDS_FILTERED cpu_to_le32(0xC000035E)
+#define STATUS_NOT_SAFE_MODE_DRIVER cpu_to_le32(0xC000035F)
+#define STATUS_ACCESS_DISABLED_BY_POLICY_DEFAULT cpu_to_le32(0xC0000361)
+#define STATUS_ACCESS_DISABLED_BY_POLICY_PATH cpu_to_le32(0xC0000362)
+#define STATUS_ACCESS_DISABLED_BY_POLICY_PUBLISHER cpu_to_le32(0xC0000363)
+#define STATUS_ACCESS_DISABLED_BY_POLICY_OTHER cpu_to_le32(0xC0000364)
+#define STATUS_FAILED_DRIVER_ENTRY cpu_to_le32(0xC0000365)
+#define STATUS_DEVICE_ENUMERATION_ERROR cpu_to_le32(0xC0000366)
+#define STATUS_MOUNT_POINT_NOT_RESOLVED cpu_to_le32(0xC0000368)
+#define STATUS_INVALID_DEVICE_OBJECT_PARAMETER cpu_to_le32(0xC0000369)
+#define STATUS_MCA_OCCURED cpu_to_le32(0xC000036A)
+#define STATUS_DRIVER_BLOCKED_CRITICAL cpu_to_le32(0xC000036B)
+#define STATUS_DRIVER_BLOCKED cpu_to_le32(0xC000036C)
+#define STATUS_DRIVER_DATABASE_ERROR cpu_to_le32(0xC000036D)
+#define STATUS_SYSTEM_HIVE_TOO_LARGE cpu_to_le32(0xC000036E)
+#define STATUS_INVALID_IMPORT_OF_NON_DLL cpu_to_le32(0xC000036F)
+#define STATUS_NO_SECRETS cpu_to_le32(0xC0000371)
+#define STATUS_ACCESS_DISABLED_NO_SAFER_UI_BY_POLICY cpu_to_le32(0xC0000372)
+#define STATUS_FAILED_STACK_SWITCH cpu_to_le32(0xC0000373)
+#define STATUS_HEAP_CORRUPTION cpu_to_le32(0xC0000374)
+#define STATUS_SMARTCARD_WRONG_PIN cpu_to_le32(0xC0000380)
+#define STATUS_SMARTCARD_CARD_BLOCKED cpu_to_le32(0xC0000381)
+#define STATUS_SMARTCARD_CARD_NOT_AUTHENTICATED cpu_to_le32(0xC0000382)
+#define STATUS_SMARTCARD_NO_CARD cpu_to_le32(0xC0000383)
+#define STATUS_SMARTCARD_NO_KEY_CONTAINER cpu_to_le32(0xC0000384)
+#define STATUS_SMARTCARD_NO_CERTIFICATE cpu_to_le32(0xC0000385)
+#define STATUS_SMARTCARD_NO_KEYSET cpu_to_le32(0xC0000386)
+#define STATUS_SMARTCARD_IO_ERROR cpu_to_le32(0xC0000387)
+#define STATUS_DOWNGRADE_DETECTED cpu_to_le32(0xC0000388)
+#define STATUS_SMARTCARD_CERT_REVOKED cpu_to_le32(0xC0000389)
+#define STATUS_ISSUING_CA_UNTRUSTED cpu_to_le32(0xC000038A)
+#define STATUS_REVOCATION_OFFLINE_C cpu_to_le32(0xC000038B)
+#define STATUS_PKINIT_CLIENT_FAILURE cpu_to_le32(0xC000038C)
+#define STATUS_SMARTCARD_CERT_EXPIRED cpu_to_le32(0xC000038D)
+#define STATUS_DRIVER_FAILED_PRIOR_UNLOAD cpu_to_le32(0xC000038E)
+#define STATUS_SMARTCARD_SILENT_CONTEXT cpu_to_le32(0xC000038F)
+#define STATUS_PER_USER_TRUST_QUOTA_EXCEEDED cpu_to_le32(0xC0000401)
+#define STATUS_ALL_USER_TRUST_QUOTA_EXCEEDED cpu_to_le32(0xC0000402)
+#define STATUS_USER_DELETE_TRUST_QUOTA_EXCEEDED cpu_to_le32(0xC0000403)
+#define STATUS_DS_NAME_NOT_UNIQUE cpu_to_le32(0xC0000404)
+#define STATUS_DS_DUPLICATE_ID_FOUND cpu_to_le32(0xC0000405)
+#define STATUS_DS_GROUP_CONVERSION_ERROR cpu_to_le32(0xC0000406)
+#define STATUS_VOLSNAP_PREPARE_HIBERNATE cpu_to_le32(0xC0000407)
+#define STATUS_USER2USER_REQUIRED cpu_to_le32(0xC0000408)
+#define STATUS_STACK_BUFFER_OVERRUN cpu_to_le32(0xC0000409)
+#define STATUS_NO_S4U_PROT_SUPPORT cpu_to_le32(0xC000040A)
+#define STATUS_CROSSREALM_DELEGATION_FAILURE cpu_to_le32(0xC000040B)
+#define STATUS_REVOCATION_OFFLINE_KDC cpu_to_le32(0xC000040C)
+#define STATUS_ISSUING_CA_UNTRUSTED_KDC cpu_to_le32(0xC000040D)
+#define STATUS_KDC_CERT_EXPIRED cpu_to_le32(0xC000040E)
+#define STATUS_KDC_CERT_REVOKED cpu_to_le32(0xC000040F)
+#define STATUS_PARAMETER_QUOTA_EXCEEDED cpu_to_le32(0xC0000410)
+#define STATUS_HIBERNATION_FAILURE cpu_to_le32(0xC0000411)
+#define STATUS_DELAY_LOAD_FAILED cpu_to_le32(0xC0000412)
+#define STATUS_AUTHENTICATION_FIREWALL_FAILED cpu_to_le32(0xC0000413)
+#define STATUS_VDM_DISALLOWED cpu_to_le32(0xC0000414)
+#define STATUS_HUNG_DISPLAY_DRIVER_THREAD cpu_to_le32(0xC0000415)
+#define STATUS_INSUFFICIENT_RESOURCE_FOR_SPECIFIED_SHARED_SECTION_SIZE cpu_to_le32(0xC0000416)
+#define STATUS_INVALID_CRUNTIME_PARAMETER cpu_to_le32(0xC0000417)
+#define STATUS_NTLM_BLOCKED cpu_to_le32(0xC0000418)
+#define STATUS_ASSERTION_FAILURE cpu_to_le32(0xC0000420)
+#define STATUS_VERIFIER_STOP cpu_to_le32(0xC0000421)
+#define STATUS_CALLBACK_POP_STACK cpu_to_le32(0xC0000423)
+#define STATUS_INCOMPATIBLE_DRIVER_BLOCKED cpu_to_le32(0xC0000424)
+#define STATUS_HIVE_UNLOADED cpu_to_le32(0xC0000425)
+#define STATUS_COMPRESSION_DISABLED cpu_to_le32(0xC0000426)
+#define STATUS_FILE_SYSTEM_LIMITATION cpu_to_le32(0xC0000427)
+#define STATUS_INVALID_IMAGE_HASH cpu_to_le32(0xC0000428)
+#define STATUS_NOT_CAPABLE cpu_to_le32(0xC0000429)
+#define STATUS_REQUEST_OUT_OF_SEQUENCE cpu_to_le32(0xC000042A)
+#define STATUS_IMPLEMENTATION_LIMIT cpu_to_le32(0xC000042B)
+#define STATUS_ELEVATION_REQUIRED cpu_to_le32(0xC000042C)
+#define STATUS_BEYOND_VDL cpu_to_le32(0xC0000432)
+#define STATUS_ENCOUNTERED_WRITE_IN_PROGRESS cpu_to_le32(0xC0000433)
+#define STATUS_PTE_CHANGED cpu_to_le32(0xC0000434)
+#define STATUS_PURGE_FAILED cpu_to_le32(0xC0000435)
+#define STATUS_CRED_REQUIRES_CONFIRMATION cpu_to_le32(0xC0000440)
+#define STATUS_CS_ENCRYPTION_INVALID_SERVER_RESPONSE cpu_to_le32(0xC0000441)
+#define STATUS_CS_ENCRYPTION_UNSUPPORTED_SERVER cpu_to_le32(0xC0000442)
+#define STATUS_CS_ENCRYPTION_EXISTING_ENCRYPTED_FILE cpu_to_le32(0xC0000443)
+#define STATUS_CS_ENCRYPTION_NEW_ENCRYPTED_FILE cpu_to_le32(0xC0000444)
+#define STATUS_CS_ENCRYPTION_FILE_NOT_CSE cpu_to_le32(0xC0000445)
+#define STATUS_INVALID_LABEL cpu_to_le32(0xC0000446)
+#define STATUS_DRIVER_PROCESS_TERMINATED cpu_to_le32(0xC0000450)
+#define STATUS_AMBIGUOUS_SYSTEM_DEVICE cpu_to_le32(0xC0000451)
+#define STATUS_SYSTEM_DEVICE_NOT_FOUND cpu_to_le32(0xC0000452)
+#define STATUS_RESTART_BOOT_APPLICATION cpu_to_le32(0xC0000453)
+#define STATUS_INVALID_TASK_NAME cpu_to_le32(0xC0000500)
+#define STATUS_INVALID_TASK_INDEX cpu_to_le32(0xC0000501)
+#define STATUS_THREAD_ALREADY_IN_TASK cpu_to_le32(0xC0000502)
+#define STATUS_CALLBACK_BYPASS cpu_to_le32(0xC0000503)
+#define STATUS_PORT_CLOSED cpu_to_le32(0xC0000700)
+#define STATUS_MESSAGE_LOST cpu_to_le32(0xC0000701)
+#define STATUS_INVALID_MESSAGE cpu_to_le32(0xC0000702)
+#define STATUS_REQUEST_CANCELED cpu_to_le32(0xC0000703)
+#define STATUS_RECURSIVE_DISPATCH cpu_to_le32(0xC0000704)
+#define STATUS_LPC_RECEIVE_BUFFER_EXPECTED cpu_to_le32(0xC0000705)
+#define STATUS_LPC_INVALID_CONNECTION_USAGE cpu_to_le32(0xC0000706)
+#define STATUS_LPC_REQUESTS_NOT_ALLOWED cpu_to_le32(0xC0000707)
+#define STATUS_RESOURCE_IN_USE cpu_to_le32(0xC0000708)
+#define STATUS_HARDWARE_MEMORY_ERROR cpu_to_le32(0xC0000709)
+#define STATUS_THREADPOOL_HANDLE_EXCEPTION cpu_to_le32(0xC000070A)
+#define STATUS_THREADPOOL_SET_EVENT_ON_COMPLETION_FAILED cpu_to_le32(0xC000070B)
+#define STATUS_THREADPOOL_RELEASE_SEMAPHORE_ON_COMPLETION_FAILED cpu_to_le32(0xC000070C)
+#define STATUS_THREADPOOL_RELEASE_MUTEX_ON_COMPLETION_FAILED cpu_to_le32(0xC000070D)
+#define STATUS_THREADPOOL_FREE_LIBRARY_ON_COMPLETION_FAILED cpu_to_le32(0xC000070E)
+#define STATUS_THREADPOOL_RELEASED_DURING_OPERATION cpu_to_le32(0xC000070F)
+#define STATUS_CALLBACK_RETURNED_WHILE_IMPERSONATING cpu_to_le32(0xC0000710)
+#define STATUS_APC_RETURNED_WHILE_IMPERSONATING cpu_to_le32(0xC0000711)
+#define STATUS_PROCESS_IS_PROTECTED cpu_to_le32(0xC0000712)
+#define STATUS_MCA_EXCEPTION cpu_to_le32(0xC0000713)
+#define STATUS_CERTIFICATE_MAPPING_NOT_UNIQUE cpu_to_le32(0xC0000714)
+#define STATUS_SYMLINK_CLASS_DISABLED cpu_to_le32(0xC0000715)
+#define STATUS_INVALID_IDN_NORMALIZATION cpu_to_le32(0xC0000716)
+#define STATUS_NO_UNICODE_TRANSLATION cpu_to_le32(0xC0000717)
+#define STATUS_ALREADY_REGISTERED cpu_to_le32(0xC0000718)
+#define STATUS_CONTEXT_MISMATCH cpu_to_le32(0xC0000719)
+#define STATUS_PORT_ALREADY_HAS_COMPLETION_LIST cpu_to_le32(0xC000071A)
+#define STATUS_CALLBACK_RETURNED_THREAD_PRIORITY cpu_to_le32(0xC000071B)
+#define STATUS_INVALID_THREAD cpu_to_le32(0xC000071C)
+#define STATUS_CALLBACK_RETURNED_TRANSACTION cpu_to_le32(0xC000071D)
+#define STATUS_CALLBACK_RETURNED_LDR_LOCK cpu_to_le32(0xC000071E)
+#define STATUS_CALLBACK_RETURNED_LANG cpu_to_le32(0xC000071F)
+#define STATUS_CALLBACK_RETURNED_PRI_BACK cpu_to_le32(0xC0000720)
+#define STATUS_CALLBACK_RETURNED_THREAD_AFFINITY cpu_to_le32(0xC0000721)
+#define STATUS_DISK_REPAIR_DISABLED cpu_to_le32(0xC0000800)
+#define STATUS_DS_DOMAIN_RENAME_IN_PROGRESS cpu_to_le32(0xC0000801)
+#define STATUS_DISK_QUOTA_EXCEEDED cpu_to_le32(0xC0000802)
+#define STATUS_CONTENT_BLOCKED cpu_to_le32(0xC0000804)
+#define STATUS_BAD_CLUSTERS cpu_to_le32(0xC0000805)
+#define STATUS_VOLUME_DIRTY cpu_to_le32(0xC0000806)
+#define STATUS_FILE_CHECKED_OUT cpu_to_le32(0xC0000901)
+#define STATUS_CHECKOUT_REQUIRED cpu_to_le32(0xC0000902)
+#define STATUS_BAD_FILE_TYPE cpu_to_le32(0xC0000903)
+#define STATUS_FILE_TOO_LARGE cpu_to_le32(0xC0000904)
+#define STATUS_FORMS_AUTH_REQUIRED cpu_to_le32(0xC0000905)
+#define STATUS_VIRUS_INFECTED cpu_to_le32(0xC0000906)
+#define STATUS_VIRUS_DELETED cpu_to_le32(0xC0000907)
+#define STATUS_BAD_MCFG_TABLE cpu_to_le32(0xC0000908)
+#define STATUS_WOW_ASSERTION cpu_to_le32(0xC0009898)
+#define STATUS_INVALID_SIGNATURE cpu_to_le32(0xC000A000)
+#define STATUS_HMAC_NOT_SUPPORTED cpu_to_le32(0xC000A001)
+#define STATUS_IPSEC_QUEUE_OVERFLOW cpu_to_le32(0xC000A010)
+#define STATUS_ND_QUEUE_OVERFLOW cpu_to_le32(0xC000A011)
+#define STATUS_HOPLIMIT_EXCEEDED cpu_to_le32(0xC000A012)
+#define STATUS_PROTOCOL_NOT_SUPPORTED cpu_to_le32(0xC000A013)
+#define STATUS_LOST_WRITEBEHIND_DATA_NETWORK_DISCONNECTED cpu_to_le32(0xC000A080)
+#define STATUS_LOST_WRITEBEHIND_DATA_NETWORK_SERVER_ERROR cpu_to_le32(0xC000A081)
+#define STATUS_LOST_WRITEBEHIND_DATA_LOCAL_DISK_ERROR cpu_to_le32(0xC000A082)
+#define STATUS_XML_PARSE_ERROR cpu_to_le32(0xC000A083)
+#define STATUS_XMLDSIG_ERROR cpu_to_le32(0xC000A084)
+#define STATUS_WRONG_COMPARTMENT cpu_to_le32(0xC000A085)
+#define STATUS_AUTHIP_FAILURE cpu_to_le32(0xC000A086)
+#define DBG_NO_STATE_CHANGE cpu_to_le32(0xC0010001)
+#define DBG_APP_NOT_IDLE cpu_to_le32(0xC0010002)
+#define RPC_NT_INVALID_STRING_BINDING cpu_to_le32(0xC0020001)
+#define RPC_NT_WRONG_KIND_OF_BINDING cpu_to_le32(0xC0020002)
+#define RPC_NT_INVALID_BINDING cpu_to_le32(0xC0020003)
+#define RPC_NT_PROTSEQ_NOT_SUPPORTED cpu_to_le32(0xC0020004)
+#define RPC_NT_INVALID_RPC_PROTSEQ cpu_to_le32(0xC0020005)
+#define RPC_NT_INVALID_STRING_UUID cpu_to_le32(0xC0020006)
+#define RPC_NT_INVALID_ENDPOINT_FORMAT cpu_to_le32(0xC0020007)
+#define RPC_NT_INVALID_NET_ADDR cpu_to_le32(0xC0020008)
+#define RPC_NT_NO_ENDPOINT_FOUND cpu_to_le32(0xC0020009)
+#define RPC_NT_INVALID_TIMEOUT cpu_to_le32(0xC002000A)
+#define RPC_NT_OBJECT_NOT_FOUND cpu_to_le32(0xC002000B)
+#define RPC_NT_ALREADY_REGISTERED cpu_to_le32(0xC002000C)
+#define RPC_NT_TYPE_ALREADY_REGISTERED cpu_to_le32(0xC002000D)
+#define RPC_NT_ALREADY_LISTENING cpu_to_le32(0xC002000E)
+#define RPC_NT_NO_PROTSEQS_REGISTERED cpu_to_le32(0xC002000F)
+#define RPC_NT_NOT_LISTENING cpu_to_le32(0xC0020010)
+#define RPC_NT_UNKNOWN_MGR_TYPE cpu_to_le32(0xC0020011)
+#define RPC_NT_UNKNOWN_IF cpu_to_le32(0xC0020012)
+#define RPC_NT_NO_BINDINGS cpu_to_le32(0xC0020013)
+#define RPC_NT_NO_PROTSEQS cpu_to_le32(0xC0020014)
+#define RPC_NT_CANT_CREATE_ENDPOINT cpu_to_le32(0xC0020015)
+#define RPC_NT_OUT_OF_RESOURCES cpu_to_le32(0xC0020016)
+#define RPC_NT_SERVER_UNAVAILABLE cpu_to_le32(0xC0020017)
+#define RPC_NT_SERVER_TOO_BUSY cpu_to_le32(0xC0020018)
+#define RPC_NT_INVALID_NETWORK_OPTIONS cpu_to_le32(0xC0020019)
+#define RPC_NT_NO_CALL_ACTIVE cpu_to_le32(0xC002001A)
+#define RPC_NT_CALL_FAILED cpu_to_le32(0xC002001B)
+#define RPC_NT_CALL_FAILED_DNE cpu_to_le32(0xC002001C)
+#define RPC_NT_PROTOCOL_ERROR cpu_to_le32(0xC002001D)
+#define RPC_NT_UNSUPPORTED_TRANS_SYN cpu_to_le32(0xC002001F)
+#define RPC_NT_UNSUPPORTED_TYPE cpu_to_le32(0xC0020021)
+#define RPC_NT_INVALID_TAG cpu_to_le32(0xC0020022)
+#define RPC_NT_INVALID_BOUND cpu_to_le32(0xC0020023)
+#define RPC_NT_NO_ENTRY_NAME cpu_to_le32(0xC0020024)
+#define RPC_NT_INVALID_NAME_SYNTAX cpu_to_le32(0xC0020025)
+#define RPC_NT_UNSUPPORTED_NAME_SYNTAX cpu_to_le32(0xC0020026)
+#define RPC_NT_UUID_NO_ADDRESS cpu_to_le32(0xC0020028)
+#define RPC_NT_DUPLICATE_ENDPOINT cpu_to_le32(0xC0020029)
+#define RPC_NT_UNKNOWN_AUTHN_TYPE cpu_to_le32(0xC002002A)
+#define RPC_NT_MAX_CALLS_TOO_SMALL cpu_to_le32(0xC002002B)
+#define RPC_NT_STRING_TOO_LONG cpu_to_le32(0xC002002C)
+#define RPC_NT_PROTSEQ_NOT_FOUND cpu_to_le32(0xC002002D)
+#define RPC_NT_PROCNUM_OUT_OF_RANGE cpu_to_le32(0xC002002E)
+#define RPC_NT_BINDING_HAS_NO_AUTH cpu_to_le32(0xC002002F)
+#define RPC_NT_UNKNOWN_AUTHN_SERVICE cpu_to_le32(0xC0020030)
+#define RPC_NT_UNKNOWN_AUTHN_LEVEL cpu_to_le32(0xC0020031)
+#define RPC_NT_INVALID_AUTH_IDENTITY cpu_to_le32(0xC0020032)
+#define RPC_NT_UNKNOWN_AUTHZ_SERVICE cpu_to_le32(0xC0020033)
+#define EPT_NT_INVALID_ENTRY cpu_to_le32(0xC0020034)
+#define EPT_NT_CANT_PERFORM_OP cpu_to_le32(0xC0020035)
+#define EPT_NT_NOT_REGISTERED cpu_to_le32(0xC0020036)
+#define RPC_NT_NOTHING_TO_EXPORT cpu_to_le32(0xC0020037)
+#define RPC_NT_INCOMPLETE_NAME cpu_to_le32(0xC0020038)
+#define RPC_NT_INVALID_VERS_OPTION cpu_to_le32(0xC0020039)
+#define RPC_NT_NO_MORE_MEMBERS cpu_to_le32(0xC002003A)
+#define RPC_NT_NOT_ALL_OBJS_UNEXPORTED cpu_to_le32(0xC002003B)
+#define RPC_NT_INTERFACE_NOT_FOUND cpu_to_le32(0xC002003C)
+#define RPC_NT_ENTRY_ALREADY_EXISTS cpu_to_le32(0xC002003D)
+#define RPC_NT_ENTRY_NOT_FOUND cpu_to_le32(0xC002003E)
+#define RPC_NT_NAME_SERVICE_UNAVAILABLE cpu_to_le32(0xC002003F)
+#define RPC_NT_INVALID_NAF_ID cpu_to_le32(0xC0020040)
+#define RPC_NT_CANNOT_SUPPORT cpu_to_le32(0xC0020041)
+#define RPC_NT_NO_CONTEXT_AVAILABLE cpu_to_le32(0xC0020042)
+#define RPC_NT_INTERNAL_ERROR cpu_to_le32(0xC0020043)
+#define RPC_NT_ZERO_DIVIDE cpu_to_le32(0xC0020044)
+#define RPC_NT_ADDRESS_ERROR cpu_to_le32(0xC0020045)
+#define RPC_NT_FP_DIV_ZERO cpu_to_le32(0xC0020046)
+#define RPC_NT_FP_UNDERFLOW cpu_to_le32(0xC0020047)
+#define RPC_NT_FP_OVERFLOW cpu_to_le32(0xC0020048)
+#define RPC_NT_CALL_IN_PROGRESS cpu_to_le32(0xC0020049)
+#define RPC_NT_NO_MORE_BINDINGS cpu_to_le32(0xC002004A)
+#define RPC_NT_GROUP_MEMBER_NOT_FOUND cpu_to_le32(0xC002004B)
+#define EPT_NT_CANT_CREATE cpu_to_le32(0xC002004C)
+#define RPC_NT_INVALID_OBJECT cpu_to_le32(0xC002004D)
+#define RPC_NT_NO_INTERFACES cpu_to_le32(0xC002004F)
+#define RPC_NT_CALL_CANCELLED cpu_to_le32(0xC0020050)
+#define RPC_NT_BINDING_INCOMPLETE cpu_to_le32(0xC0020051)
+#define RPC_NT_COMM_FAILURE cpu_to_le32(0xC0020052)
+#define RPC_NT_UNSUPPORTED_AUTHN_LEVEL cpu_to_le32(0xC0020053)
+#define RPC_NT_NO_PRINC_NAME cpu_to_le32(0xC0020054)
+#define RPC_NT_NOT_RPC_ERROR cpu_to_le32(0xC0020055)
+#define RPC_NT_SEC_PKG_ERROR cpu_to_le32(0xC0020057)
+#define RPC_NT_NOT_CANCELLED cpu_to_le32(0xC0020058)
+#define RPC_NT_INVALID_ASYNC_HANDLE cpu_to_le32(0xC0020062)
+#define RPC_NT_INVALID_ASYNC_CALL cpu_to_le32(0xC0020063)
+#define RPC_NT_PROXY_ACCESS_DENIED cpu_to_le32(0xC0020064)
+#define RPC_NT_NO_MORE_ENTRIES cpu_to_le32(0xC0030001)
+#define RPC_NT_SS_CHAR_TRANS_OPEN_FAIL cpu_to_le32(0xC0030002)
+#define RPC_NT_SS_CHAR_TRANS_SHORT_FILE cpu_to_le32(0xC0030003)
+#define RPC_NT_SS_IN_NULL_CONTEXT cpu_to_le32(0xC0030004)
+#define RPC_NT_SS_CONTEXT_MISMATCH cpu_to_le32(0xC0030005)
+#define RPC_NT_SS_CONTEXT_DAMAGED cpu_to_le32(0xC0030006)
+#define RPC_NT_SS_HANDLES_MISMATCH cpu_to_le32(0xC0030007)
+#define RPC_NT_SS_CANNOT_GET_CALL_HANDLE cpu_to_le32(0xC0030008)
+#define RPC_NT_NULL_REF_POINTER cpu_to_le32(0xC0030009)
+#define RPC_NT_ENUM_VALUE_OUT_OF_RANGE cpu_to_le32(0xC003000A)
+#define RPC_NT_BYTE_COUNT_TOO_SMALL cpu_to_le32(0xC003000B)
+#define RPC_NT_BAD_STUB_DATA cpu_to_le32(0xC003000C)
+#define RPC_NT_INVALID_ES_ACTION cpu_to_le32(0xC0030059)
+#define RPC_NT_WRONG_ES_VERSION cpu_to_le32(0xC003005A)
+#define RPC_NT_WRONG_STUB_VERSION cpu_to_le32(0xC003005B)
+#define RPC_NT_INVALID_PIPE_OBJECT cpu_to_le32(0xC003005C)
+#define RPC_NT_INVALID_PIPE_OPERATION cpu_to_le32(0xC003005D)
+#define RPC_NT_WRONG_PIPE_VERSION cpu_to_le32(0xC003005E)
+#define RPC_NT_PIPE_CLOSED cpu_to_le32(0xC003005F)
+#define RPC_NT_PIPE_DISCIPLINE_ERROR cpu_to_le32(0xC0030060)
+#define RPC_NT_PIPE_EMPTY cpu_to_le32(0xC0030061)
+#define STATUS_PNP_BAD_MPS_TABLE cpu_to_le32(0xC0040035)
+#define STATUS_PNP_TRANSLATION_FAILED cpu_to_le32(0xC0040036)
+#define STATUS_PNP_IRQ_TRANSLATION_FAILED cpu_to_le32(0xC0040037)
+#define STATUS_PNP_INVALID_ID cpu_to_le32(0xC0040038)
+#define STATUS_IO_REISSUE_AS_CACHED cpu_to_le32(0xC0040039)
+#define STATUS_CTX_WINSTATION_NAME_INVALID cpu_to_le32(0xC00A0001)
+#define STATUS_CTX_INVALID_PD cpu_to_le32(0xC00A0002)
+#define STATUS_CTX_PD_NOT_FOUND cpu_to_le32(0xC00A0003)
+#define STATUS_CTX_CLOSE_PENDING cpu_to_le32(0xC00A0006)
+#define STATUS_CTX_NO_OUTBUF cpu_to_le32(0xC00A0007)
+#define STATUS_CTX_MODEM_INF_NOT_FOUND cpu_to_le32(0xC00A0008)
+#define STATUS_CTX_INVALID_MODEMNAME cpu_to_le32(0xC00A0009)
+#define STATUS_CTX_RESPONSE_ERROR cpu_to_le32(0xC00A000A)
+#define STATUS_CTX_MODEM_RESPONSE_TIMEOUT cpu_to_le32(0xC00A000B)
+#define STATUS_CTX_MODEM_RESPONSE_NO_CARRIER cpu_to_le32(0xC00A000C)
+#define STATUS_CTX_MODEM_RESPONSE_NO_DIALTONE cpu_to_le32(0xC00A000D)
+#define STATUS_CTX_MODEM_RESPONSE_BUSY cpu_to_le32(0xC00A000E)
+#define STATUS_CTX_MODEM_RESPONSE_VOICE cpu_to_le32(0xC00A000F)
+#define STATUS_CTX_TD_ERROR cpu_to_le32(0xC00A0010)
+#define STATUS_CTX_LICENSE_CLIENT_INVALID cpu_to_le32(0xC00A0012)
+#define STATUS_CTX_LICENSE_NOT_AVAILABLE cpu_to_le32(0xC00A0013)
+#define STATUS_CTX_LICENSE_EXPIRED cpu_to_le32(0xC00A0014)
+#define STATUS_CTX_WINSTATION_NOT_FOUND cpu_to_le32(0xC00A0015)
+#define STATUS_CTX_WINSTATION_NAME_COLLISION cpu_to_le32(0xC00A0016)
+#define STATUS_CTX_WINSTATION_BUSY cpu_to_le32(0xC00A0017)
+#define STATUS_CTX_BAD_VIDEO_MODE cpu_to_le32(0xC00A0018)
+#define STATUS_CTX_GRAPHICS_INVALID cpu_to_le32(0xC00A0022)
+#define STATUS_CTX_NOT_CONSOLE cpu_to_le32(0xC00A0024)
+#define STATUS_CTX_CLIENT_QUERY_TIMEOUT cpu_to_le32(0xC00A0026)
+#define STATUS_CTX_CONSOLE_DISCONNECT cpu_to_le32(0xC00A0027)
+#define STATUS_CTX_CONSOLE_CONNECT cpu_to_le32(0xC00A0028)
+#define STATUS_CTX_SHADOW_DENIED cpu_to_le32(0xC00A002A)
+#define STATUS_CTX_WINSTATION_ACCESS_DENIED cpu_to_le32(0xC00A002B)
+#define STATUS_CTX_INVALID_WD cpu_to_le32(0xC00A002E)
+#define STATUS_CTX_WD_NOT_FOUND cpu_to_le32(0xC00A002F)
+#define STATUS_CTX_SHADOW_INVALID cpu_to_le32(0xC00A0030)
+#define STATUS_CTX_SHADOW_DISABLED cpu_to_le32(0xC00A0031)
+#define STATUS_RDP_PROTOCOL_ERROR cpu_to_le32(0xC00A0032)
+#define STATUS_CTX_CLIENT_LICENSE_NOT_SET cpu_to_le32(0xC00A0033)
+#define STATUS_CTX_CLIENT_LICENSE_IN_USE cpu_to_le32(0xC00A0034)
+#define STATUS_CTX_SHADOW_ENDED_BY_MODE_CHANGE cpu_to_le32(0xC00A0035)
+#define STATUS_CTX_SHADOW_NOT_RUNNING cpu_to_le32(0xC00A0036)
+#define STATUS_CTX_LOGON_DISABLED cpu_to_le32(0xC00A0037)
+#define STATUS_CTX_SECURITY_LAYER_ERROR cpu_to_le32(0xC00A0038)
+#define STATUS_TS_INCOMPATIBLE_SESSIONS cpu_to_le32(0xC00A0039)
+#define STATUS_MUI_FILE_NOT_FOUND cpu_to_le32(0xC00B0001)
+#define STATUS_MUI_INVALID_FILE cpu_to_le32(0xC00B0002)
+#define STATUS_MUI_INVALID_RC_CONFIG cpu_to_le32(0xC00B0003)
+#define STATUS_MUI_INVALID_LOCALE_NAME cpu_to_le32(0xC00B0004)
+#define STATUS_MUI_INVALID_ULTIMATEFALLBACK_NAME cpu_to_le32(0xC00B0005)
+#define STATUS_MUI_FILE_NOT_LOADED cpu_to_le32(0xC00B0006)
+#define STATUS_RESOURCE_ENUM_USER_STOP cpu_to_le32(0xC00B0007)
+#define STATUS_CLUSTER_INVALID_NODE cpu_to_le32(0xC0130001)
+#define STATUS_CLUSTER_NODE_EXISTS cpu_to_le32(0xC0130002)
+#define STATUS_CLUSTER_JOIN_IN_PROGRESS cpu_to_le32(0xC0130003)
+#define STATUS_CLUSTER_NODE_NOT_FOUND cpu_to_le32(0xC0130004)
+#define STATUS_CLUSTER_LOCAL_NODE_NOT_FOUND cpu_to_le32(0xC0130005)
+#define STATUS_CLUSTER_NETWORK_EXISTS cpu_to_le32(0xC0130006)
+#define STATUS_CLUSTER_NETWORK_NOT_FOUND cpu_to_le32(0xC0130007)
+#define STATUS_CLUSTER_NETINTERFACE_EXISTS cpu_to_le32(0xC0130008)
+#define STATUS_CLUSTER_NETINTERFACE_NOT_FOUND cpu_to_le32(0xC0130009)
+#define STATUS_CLUSTER_INVALID_REQUEST cpu_to_le32(0xC013000A)
+#define STATUS_CLUSTER_INVALID_NETWORK_PROVIDER cpu_to_le32(0xC013000B)
+#define STATUS_CLUSTER_NODE_DOWN cpu_to_le32(0xC013000C)
+#define STATUS_CLUSTER_NODE_UNREACHABLE cpu_to_le32(0xC013000D)
+#define STATUS_CLUSTER_NODE_NOT_MEMBER cpu_to_le32(0xC013000E)
+#define STATUS_CLUSTER_JOIN_NOT_IN_PROGRESS cpu_to_le32(0xC013000F)
+#define STATUS_CLUSTER_INVALID_NETWORK cpu_to_le32(0xC0130010)
+#define STATUS_CLUSTER_NO_NET_ADAPTERS cpu_to_le32(0xC0130011)
+#define STATUS_CLUSTER_NODE_UP cpu_to_le32(0xC0130012)
+#define STATUS_CLUSTER_NODE_PAUSED cpu_to_le32(0xC0130013)
+#define STATUS_CLUSTER_NODE_NOT_PAUSED cpu_to_le32(0xC0130014)
+#define STATUS_CLUSTER_NO_SECURITY_CONTEXT cpu_to_le32(0xC0130015)
+#define STATUS_CLUSTER_NETWORK_NOT_INTERNAL cpu_to_le32(0xC0130016)
+#define STATUS_CLUSTER_POISONED cpu_to_le32(0xC0130017)
+#define STATUS_ACPI_INVALID_OPCODE cpu_to_le32(0xC0140001)
+#define STATUS_ACPI_STACK_OVERFLOW cpu_to_le32(0xC0140002)
+#define STATUS_ACPI_ASSERT_FAILED cpu_to_le32(0xC0140003)
+#define STATUS_ACPI_INVALID_INDEX cpu_to_le32(0xC0140004)
+#define STATUS_ACPI_INVALID_ARGUMENT cpu_to_le32(0xC0140005)
+#define STATUS_ACPI_FATAL cpu_to_le32(0xC0140006)
+#define STATUS_ACPI_INVALID_SUPERNAME cpu_to_le32(0xC0140007)
+#define STATUS_ACPI_INVALID_ARGTYPE cpu_to_le32(0xC0140008)
+#define STATUS_ACPI_INVALID_OBJTYPE cpu_to_le32(0xC0140009)
+#define STATUS_ACPI_INVALID_TARGETTYPE cpu_to_le32(0xC014000A)
+#define STATUS_ACPI_INCORRECT_ARGUMENT_COUNT cpu_to_le32(0xC014000B)
+#define STATUS_ACPI_ADDRESS_NOT_MAPPED cpu_to_le32(0xC014000C)
+#define STATUS_ACPI_INVALID_EVENTTYPE cpu_to_le32(0xC014000D)
+#define STATUS_ACPI_HANDLER_COLLISION cpu_to_le32(0xC014000E)
+#define STATUS_ACPI_INVALID_DATA cpu_to_le32(0xC014000F)
+#define STATUS_ACPI_INVALID_REGION cpu_to_le32(0xC0140010)
+#define STATUS_ACPI_INVALID_ACCESS_SIZE cpu_to_le32(0xC0140011)
+#define STATUS_ACPI_ACQUIRE_GLOBAL_LOCK cpu_to_le32(0xC0140012)
+#define STATUS_ACPI_ALREADY_INITIALIZED cpu_to_le32(0xC0140013)
+#define STATUS_ACPI_NOT_INITIALIZED cpu_to_le32(0xC0140014)
+#define STATUS_ACPI_INVALID_MUTEX_LEVEL cpu_to_le32(0xC0140015)
+#define STATUS_ACPI_MUTEX_NOT_OWNED cpu_to_le32(0xC0140016)
+#define STATUS_ACPI_MUTEX_NOT_OWNER cpu_to_le32(0xC0140017)
+#define STATUS_ACPI_RS_ACCESS cpu_to_le32(0xC0140018)
+#define STATUS_ACPI_INVALID_TABLE cpu_to_le32(0xC0140019)
+#define STATUS_ACPI_REG_HANDLER_FAILED cpu_to_le32(0xC0140020)
+#define STATUS_ACPI_POWER_REQUEST_FAILED cpu_to_le32(0xC0140021)
+#define STATUS_SXS_SECTION_NOT_FOUND cpu_to_le32(0xC0150001)
+#define STATUS_SXS_CANT_GEN_ACTCTX cpu_to_le32(0xC0150002)
+#define STATUS_SXS_INVALID_ACTCTXDATA_FORMAT cpu_to_le32(0xC0150003)
+#define STATUS_SXS_ASSEMBLY_NOT_FOUND cpu_to_le32(0xC0150004)
+#define STATUS_SXS_MANIFEST_FORMAT_ERROR cpu_to_le32(0xC0150005)
+#define STATUS_SXS_MANIFEST_PARSE_ERROR cpu_to_le32(0xC0150006)
+#define STATUS_SXS_ACTIVATION_CONTEXT_DISABLED cpu_to_le32(0xC0150007)
+#define STATUS_SXS_KEY_NOT_FOUND cpu_to_le32(0xC0150008)
+#define STATUS_SXS_VERSION_CONFLICT cpu_to_le32(0xC0150009)
+#define STATUS_SXS_WRONG_SECTION_TYPE cpu_to_le32(0xC015000A)
+#define STATUS_SXS_THREAD_QUERIES_DISABLED cpu_to_le32(0xC015000B)
+#define STATUS_SXS_ASSEMBLY_MISSING cpu_to_le32(0xC015000C)
+#define STATUS_SXS_PROCESS_DEFAULT_ALREADY_SET cpu_to_le32(0xC015000E)
+#define STATUS_SXS_EARLY_DEACTIVATION cpu_to_le32(0xC015000F)
+#define STATUS_SXS_INVALID_DEACTIVATION cpu_to_le32(0xC0150010)
+#define STATUS_SXS_MULTIPLE_DEACTIVATION cpu_to_le32(0xC0150011)
+#define STATUS_SXS_SYSTEM_DEFAULT_ACTIVATION_CONTEXT_EMPTY cpu_to_le32(0xC0150012)
+#define STATUS_SXS_PROCESS_TERMINATION_REQUESTED cpu_to_le32(0xC0150013)
+#define STATUS_SXS_CORRUPT_ACTIVATION_STACK cpu_to_le32(0xC0150014)
+#define STATUS_SXS_CORRUPTION cpu_to_le32(0xC0150015)
+#define STATUS_SXS_INVALID_IDENTITY_ATTRIBUTE_VALUE cpu_to_le32(0xC0150016)
+#define STATUS_SXS_INVALID_IDENTITY_ATTRIBUTE_NAME cpu_to_le32(0xC0150017)
+#define STATUS_SXS_IDENTITY_DUPLICATE_ATTRIBUTE cpu_to_le32(0xC0150018)
+#define STATUS_SXS_IDENTITY_PARSE_ERROR cpu_to_le32(0xC0150019)
+#define STATUS_SXS_COMPONENT_STORE_CORRUPT cpu_to_le32(0xC015001A)
+#define STATUS_SXS_FILE_HASH_MISMATCH cpu_to_le32(0xC015001B)
+#define STATUS_SXS_MANIFEST_IDENTITY_SAME_BUT_CONTENTS_DIFFERENT cpu_to_le32(0xC015001C)
+#define STATUS_SXS_IDENTITIES_DIFFERENT cpu_to_le32(0xC015001D)
+#define STATUS_SXS_ASSEMBLY_IS_NOT_A_DEPLOYMENT cpu_to_le32(0xC015001E)
+#define STATUS_SXS_FILE_NOT_PART_OF_ASSEMBLY cpu_to_le32(0xC015001F)
+#define STATUS_ADVANCED_INSTALLER_FAILED cpu_to_le32(0xC0150020)
+#define STATUS_XML_ENCODING_MISMATCH cpu_to_le32(0xC0150021)
+#define STATUS_SXS_MANIFEST_TOO_BIG cpu_to_le32(0xC0150022)
+#define STATUS_SXS_SETTING_NOT_REGISTERED cpu_to_le32(0xC0150023)
+#define STATUS_SXS_TRANSACTION_CLOSURE_INCOMPLETE cpu_to_le32(0xC0150024)
+#define STATUS_SMI_PRIMITIVE_INSTALLER_FAILED cpu_to_le32(0xC0150025)
+#define STATUS_GENERIC_COMMAND_FAILED cpu_to_le32(0xC0150026)
+#define STATUS_SXS_FILE_HASH_MISSING cpu_to_le32(0xC0150027)
+#define STATUS_TRANSACTIONAL_CONFLICT cpu_to_le32(0xC0190001)
+#define STATUS_INVALID_TRANSACTION cpu_to_le32(0xC0190002)
+#define STATUS_TRANSACTION_NOT_ACTIVE cpu_to_le32(0xC0190003)
+#define STATUS_TM_INITIALIZATION_FAILED cpu_to_le32(0xC0190004)
+#define STATUS_RM_NOT_ACTIVE cpu_to_le32(0xC0190005)
+#define STATUS_RM_METADATA_CORRUPT cpu_to_le32(0xC0190006)
+#define STATUS_TRANSACTION_NOT_JOINED cpu_to_le32(0xC0190007)
+#define STATUS_DIRECTORY_NOT_RM cpu_to_le32(0xC0190008)
+#define STATUS_TRANSACTIONS_UNSUPPORTED_REMOTE cpu_to_le32(0xC019000A)
+#define STATUS_LOG_RESIZE_INVALID_SIZE cpu_to_le32(0xC019000B)
+#define STATUS_REMOTE_FILE_VERSION_MISMATCH cpu_to_le32(0xC019000C)
+#define STATUS_CRM_PROTOCOL_ALREADY_EXISTS cpu_to_le32(0xC019000F)
+#define STATUS_TRANSACTION_PROPAGATION_FAILED cpu_to_le32(0xC0190010)
+#define STATUS_CRM_PROTOCOL_NOT_FOUND cpu_to_le32(0xC0190011)
+#define STATUS_TRANSACTION_SUPERIOR_EXISTS cpu_to_le32(0xC0190012)
+#define STATUS_TRANSACTION_REQUEST_NOT_VALID cpu_to_le32(0xC0190013)
+#define STATUS_TRANSACTION_NOT_REQUESTED cpu_to_le32(0xC0190014)
+#define STATUS_TRANSACTION_ALREADY_ABORTED cpu_to_le32(0xC0190015)
+#define STATUS_TRANSACTION_ALREADY_COMMITTED cpu_to_le32(0xC0190016)
+#define STATUS_TRANSACTION_INVALID_MARSHALL_BUFFER cpu_to_le32(0xC0190017)
+#define STATUS_CURRENT_TRANSACTION_NOT_VALID cpu_to_le32(0xC0190018)
+#define STATUS_LOG_GROWTH_FAILED cpu_to_le32(0xC0190019)
+#define STATUS_OBJECT_NO_LONGER_EXISTS cpu_to_le32(0xC0190021)
+#define STATUS_STREAM_MINIVERSION_NOT_FOUND cpu_to_le32(0xC0190022)
+#define STATUS_STREAM_MINIVERSION_NOT_VALID cpu_to_le32(0xC0190023)
+#define STATUS_MINIVERSION_INACCESSIBLE_FROM_SPECIFIED_TRANSACTION cpu_to_le32(0xC0190024)
+#define STATUS_CANT_OPEN_MINIVERSION_WITH_MODIFY_INTENT cpu_to_le32(0xC0190025)
+#define STATUS_CANT_CREATE_MORE_STREAM_MINIVERSIONS cpu_to_le32(0xC0190026)
+#define STATUS_HANDLE_NO_LONGER_VALID cpu_to_le32(0xC0190028)
+#define STATUS_LOG_CORRUPTION_DETECTED cpu_to_le32(0xC0190030)
+#define STATUS_RM_DISCONNECTED cpu_to_le32(0xC0190032)
+#define STATUS_ENLISTMENT_NOT_SUPERIOR cpu_to_le32(0xC0190033)
+#define STATUS_FILE_IDENTITY_NOT_PERSISTENT cpu_to_le32(0xC0190036)
+#define STATUS_CANT_BREAK_TRANSACTIONAL_DEPENDENCY cpu_to_le32(0xC0190037)
+#define STATUS_CANT_CROSS_RM_BOUNDARY cpu_to_le32(0xC0190038)
+#define STATUS_TXF_DIR_NOT_EMPTY cpu_to_le32(0xC0190039)
+#define STATUS_INDOUBT_TRANSACTIONS_EXIST cpu_to_le32(0xC019003A)
+#define STATUS_TM_VOLATILE cpu_to_le32(0xC019003B)
+#define STATUS_ROLLBACK_TIMER_EXPIRED cpu_to_le32(0xC019003C)
+#define STATUS_TXF_ATTRIBUTE_CORRUPT cpu_to_le32(0xC019003D)
+#define STATUS_EFS_NOT_ALLOWED_IN_TRANSACTION cpu_to_le32(0xC019003E)
+#define STATUS_TRANSACTIONAL_OPEN_NOT_ALLOWED cpu_to_le32(0xC019003F)
+#define STATUS_TRANSACTED_MAPPING_UNSUPPORTED_REMOTE cpu_to_le32(0xC0190040)
+#define STATUS_TRANSACTION_REQUIRED_PROMOTION cpu_to_le32(0xC0190043)
+#define STATUS_CANNOT_EXECUTE_FILE_IN_TRANSACTION cpu_to_le32(0xC0190044)
+#define STATUS_TRANSACTIONS_NOT_FROZEN cpu_to_le32(0xC0190045)
+#define STATUS_TRANSACTION_FREEZE_IN_PROGRESS cpu_to_le32(0xC0190046)
+#define STATUS_NOT_SNAPSHOT_VOLUME cpu_to_le32(0xC0190047)
+#define STATUS_NO_SAVEPOINT_WITH_OPEN_FILES cpu_to_le32(0xC0190048)
+#define STATUS_SPARSE_NOT_ALLOWED_IN_TRANSACTION cpu_to_le32(0xC0190049)
+#define STATUS_TM_IDENTITY_MISMATCH cpu_to_le32(0xC019004A)
+#define STATUS_FLOATED_SECTION cpu_to_le32(0xC019004B)
+#define STATUS_CANNOT_ACCEPT_TRANSACTED_WORK cpu_to_le32(0xC019004C)
+#define STATUS_CANNOT_ABORT_TRANSACTIONS cpu_to_le32(0xC019004D)
+#define STATUS_TRANSACTION_NOT_FOUND cpu_to_le32(0xC019004E)
+#define STATUS_RESOURCEMANAGER_NOT_FOUND cpu_to_le32(0xC019004F)
+#define STATUS_ENLISTMENT_NOT_FOUND cpu_to_le32(0xC0190050)
+#define STATUS_TRANSACTIONMANAGER_NOT_FOUND cpu_to_le32(0xC0190051)
+#define STATUS_TRANSACTIONMANAGER_NOT_ONLINE cpu_to_le32(0xC0190052)
+#define STATUS_TRANSACTIONMANAGER_RECOVERY_NAME_COLLISION cpu_to_le32(0xC0190053)
+#define STATUS_TRANSACTION_NOT_ROOT cpu_to_le32(0xC0190054)
+#define STATUS_TRANSACTION_OBJECT_EXPIRED cpu_to_le32(0xC0190055)
+#define STATUS_COMPRESSION_NOT_ALLOWED_IN_TRANSACTION cpu_to_le32(0xC0190056)
+#define STATUS_TRANSACTION_RESPONSE_NOT_ENLISTED cpu_to_le32(0xC0190057)
+#define STATUS_TRANSACTION_RECORD_TOO_LONG cpu_to_le32(0xC0190058)
+#define STATUS_NO_LINK_TRACKING_IN_TRANSACTION cpu_to_le32(0xC0190059)
+#define STATUS_OPERATION_NOT_SUPPORTED_IN_TRANSACTION cpu_to_le32(0xC019005A)
+#define STATUS_TRANSACTION_INTEGRITY_VIOLATED cpu_to_le32(0xC019005B)
+#define STATUS_LOG_SECTOR_INVALID cpu_to_le32(0xC01A0001)
+#define STATUS_LOG_SECTOR_PARITY_INVALID cpu_to_le32(0xC01A0002)
+#define STATUS_LOG_SECTOR_REMAPPED cpu_to_le32(0xC01A0003)
+#define STATUS_LOG_BLOCK_INCOMPLETE cpu_to_le32(0xC01A0004)
+#define STATUS_LOG_INVALID_RANGE cpu_to_le32(0xC01A0005)
+#define STATUS_LOG_BLOCKS_EXHAUSTED cpu_to_le32(0xC01A0006)
+#define STATUS_LOG_READ_CONTEXT_INVALID cpu_to_le32(0xC01A0007)
+#define STATUS_LOG_RESTART_INVALID cpu_to_le32(0xC01A0008)
+#define STATUS_LOG_BLOCK_VERSION cpu_to_le32(0xC01A0009)
+#define STATUS_LOG_BLOCK_INVALID cpu_to_le32(0xC01A000A)
+#define STATUS_LOG_READ_MODE_INVALID cpu_to_le32(0xC01A000B)
+#define STATUS_LOG_METADATA_CORRUPT cpu_to_le32(0xC01A000D)
+#define STATUS_LOG_METADATA_INVALID cpu_to_le32(0xC01A000E)
+#define STATUS_LOG_METADATA_INCONSISTENT cpu_to_le32(0xC01A000F)
+#define STATUS_LOG_RESERVATION_INVALID cpu_to_le32(0xC01A0010)
+#define STATUS_LOG_CANT_DELETE cpu_to_le32(0xC01A0011)
+#define STATUS_LOG_CONTAINER_LIMIT_EXCEEDED cpu_to_le32(0xC01A0012)
+#define STATUS_LOG_START_OF_LOG cpu_to_le32(0xC01A0013)
+#define STATUS_LOG_POLICY_ALREADY_INSTALLED cpu_to_le32(0xC01A0014)
+#define STATUS_LOG_POLICY_NOT_INSTALLED cpu_to_le32(0xC01A0015)
+#define STATUS_LOG_POLICY_INVALID cpu_to_le32(0xC01A0016)
+#define STATUS_LOG_POLICY_CONFLICT cpu_to_le32(0xC01A0017)
+#define STATUS_LOG_PINNED_ARCHIVE_TAIL cpu_to_le32(0xC01A0018)
+#define STATUS_LOG_RECORD_NONEXISTENT cpu_to_le32(0xC01A0019)
+#define STATUS_LOG_RECORDS_RESERVED_INVALID cpu_to_le32(0xC01A001A)
+#define STATUS_LOG_SPACE_RESERVED_INVALID cpu_to_le32(0xC01A001B)
+#define STATUS_LOG_TAIL_INVALID cpu_to_le32(0xC01A001C)
+#define STATUS_LOG_FULL cpu_to_le32(0xC01A001D)
+#define STATUS_LOG_MULTIPLEXED cpu_to_le32(0xC01A001E)
+#define STATUS_LOG_DEDICATED cpu_to_le32(0xC01A001F)
+#define STATUS_LOG_ARCHIVE_NOT_IN_PROGRESS cpu_to_le32(0xC01A0020)
+#define STATUS_LOG_ARCHIVE_IN_PROGRESS cpu_to_le32(0xC01A0021)
+#define STATUS_LOG_EPHEMERAL cpu_to_le32(0xC01A0022)
+#define STATUS_LOG_NOT_ENOUGH_CONTAINERS cpu_to_le32(0xC01A0023)
+#define STATUS_LOG_CLIENT_ALREADY_REGISTERED cpu_to_le32(0xC01A0024)
+#define STATUS_LOG_CLIENT_NOT_REGISTERED cpu_to_le32(0xC01A0025)
+#define STATUS_LOG_FULL_HANDLER_IN_PROGRESS cpu_to_le32(0xC01A0026)
+#define STATUS_LOG_CONTAINER_READ_FAILED cpu_to_le32(0xC01A0027)
+#define STATUS_LOG_CONTAINER_WRITE_FAILED cpu_to_le32(0xC01A0028)
+#define STATUS_LOG_CONTAINER_OPEN_FAILED cpu_to_le32(0xC01A0029)
+#define STATUS_LOG_CONTAINER_STATE_INVALID cpu_to_le32(0xC01A002A)
+#define STATUS_LOG_STATE_INVALID cpu_to_le32(0xC01A002B)
+#define STATUS_LOG_PINNED cpu_to_le32(0xC01A002C)
+#define STATUS_LOG_METADATA_FLUSH_FAILED cpu_to_le32(0xC01A002D)
+#define STATUS_LOG_INCONSISTENT_SECURITY cpu_to_le32(0xC01A002E)
+#define STATUS_LOG_APPENDED_FLUSH_FAILED cpu_to_le32(0xC01A002F)
+#define STATUS_LOG_PINNED_RESERVATION cpu_to_le32(0xC01A0030)
+#define STATUS_VIDEO_HUNG_DISPLAY_DRIVER_THREAD cpu_to_le32(0xC01B00EA)
+#define STATUS_FLT_NO_HANDLER_DEFINED cpu_to_le32(0xC01C0001)
+#define STATUS_FLT_CONTEXT_ALREADY_DEFINED cpu_to_le32(0xC01C0002)
+#define STATUS_FLT_INVALID_ASYNCHRONOUS_REQUEST cpu_to_le32(0xC01C0003)
+#define STATUS_FLT_DISALLOW_FAST_IO cpu_to_le32(0xC01C0004)
+#define STATUS_FLT_INVALID_NAME_REQUEST cpu_to_le32(0xC01C0005)
+#define STATUS_FLT_NOT_SAFE_TO_POST_OPERATION cpu_to_le32(0xC01C0006)
+#define STATUS_FLT_NOT_INITIALIZED cpu_to_le32(0xC01C0007)
+#define STATUS_FLT_FILTER_NOT_READY cpu_to_le32(0xC01C0008)
+#define STATUS_FLT_POST_OPERATION_CLEANUP cpu_to_le32(0xC01C0009)
+#define STATUS_FLT_INTERNAL_ERROR cpu_to_le32(0xC01C000A)
+#define STATUS_FLT_DELETING_OBJECT cpu_to_le32(0xC01C000B)
+#define STATUS_FLT_MUST_BE_NONPAGED_POOL cpu_to_le32(0xC01C000C)
+#define STATUS_FLT_DUPLICATE_ENTRY cpu_to_le32(0xC01C000D)
+#define STATUS_FLT_CBDQ_DISABLED cpu_to_le32(0xC01C000E)
+#define STATUS_FLT_DO_NOT_ATTACH cpu_to_le32(0xC01C000F)
+#define STATUS_FLT_DO_NOT_DETACH cpu_to_le32(0xC01C0010)
+#define STATUS_FLT_INSTANCE_ALTITUDE_COLLISION cpu_to_le32(0xC01C0011)
+#define STATUS_FLT_INSTANCE_NAME_COLLISION cpu_to_le32(0xC01C0012)
+#define STATUS_FLT_FILTER_NOT_FOUND cpu_to_le32(0xC01C0013)
+#define STATUS_FLT_VOLUME_NOT_FOUND cpu_to_le32(0xC01C0014)
+#define STATUS_FLT_INSTANCE_NOT_FOUND cpu_to_le32(0xC01C0015)
+#define STATUS_FLT_CONTEXT_ALLOCATION_NOT_FOUND cpu_to_le32(0xC01C0016)
+#define STATUS_FLT_INVALID_CONTEXT_REGISTRATION cpu_to_le32(0xC01C0017)
+#define STATUS_FLT_NAME_CACHE_MISS cpu_to_le32(0xC01C0018)
+#define STATUS_FLT_NO_DEVICE_OBJECT cpu_to_le32(0xC01C0019)
+#define STATUS_FLT_VOLUME_ALREADY_MOUNTED cpu_to_le32(0xC01C001A)
+#define STATUS_FLT_ALREADY_ENLISTED cpu_to_le32(0xC01C001B)
+#define STATUS_FLT_CONTEXT_ALREADY_LINKED cpu_to_le32(0xC01C001C)
+#define STATUS_FLT_NO_WAITER_FOR_REPLY cpu_to_le32(0xC01C0020)
+#define STATUS_MONITOR_NO_DESCRIPTOR cpu_to_le32(0xC01D0001)
+#define STATUS_MONITOR_UNKNOWN_DESCRIPTOR_FORMAT cpu_to_le32(0xC01D0002)
+#define STATUS_MONITOR_INVALID_DESCRIPTOR_CHECKSUM cpu_to_le32(0xC01D0003)
+#define STATUS_MONITOR_INVALID_STANDARD_TIMING_BLOCK cpu_to_le32(0xC01D0004)
+#define STATUS_MONITOR_WMI_DATABLOCK_REGISTRATION_FAILED cpu_to_le32(0xC01D0005)
+#define STATUS_MONITOR_INVALID_SERIAL_NUMBER_MONDSC_BLOCK cpu_to_le32(0xC01D0006)
+#define STATUS_MONITOR_INVALID_USER_FRIENDLY_MONDSC_BLOCK cpu_to_le32(0xC01D0007)
+#define STATUS_MONITOR_NO_MORE_DESCRIPTOR_DATA cpu_to_le32(0xC01D0008)
+#define STATUS_MONITOR_INVALID_DETAILED_TIMING_BLOCK cpu_to_le32(0xC01D0009)
+#define STATUS_GRAPHICS_NOT_EXCLUSIVE_MODE_OWNER cpu_to_le32(0xC01E0000)
+#define STATUS_GRAPHICS_INSUFFICIENT_DMA_BUFFER cpu_to_le32(0xC01E0001)
+#define STATUS_GRAPHICS_INVALID_DISPLAY_ADAPTER cpu_to_le32(0xC01E0002)
+#define STATUS_GRAPHICS_ADAPTER_WAS_RESET cpu_to_le32(0xC01E0003)
+#define STATUS_GRAPHICS_INVALID_DRIVER_MODEL cpu_to_le32(0xC01E0004)
+#define STATUS_GRAPHICS_PRESENT_MODE_CHANGED cpu_to_le32(0xC01E0005)
+#define STATUS_GRAPHICS_PRESENT_OCCLUDED cpu_to_le32(0xC01E0006)
+#define STATUS_GRAPHICS_PRESENT_DENIED cpu_to_le32(0xC01E0007)
+#define STATUS_GRAPHICS_CANNOTCOLORCONVERT cpu_to_le32(0xC01E0008)
+#define STATUS_GRAPHICS_NO_VIDEO_MEMORY cpu_to_le32(0xC01E0100)
+#define STATUS_GRAPHICS_CANT_LOCK_MEMORY cpu_to_le32(0xC01E0101)
+#define STATUS_GRAPHICS_ALLOCATION_BUSY cpu_to_le32(0xC01E0102)
+#define STATUS_GRAPHICS_TOO_MANY_REFERENCES cpu_to_le32(0xC01E0103)
+#define STATUS_GRAPHICS_TRY_AGAIN_LATER cpu_to_le32(0xC01E0104)
+#define STATUS_GRAPHICS_TRY_AGAIN_NOW cpu_to_le32(0xC01E0105)
+#define STATUS_GRAPHICS_ALLOCATION_INVALID cpu_to_le32(0xC01E0106)
+#define STATUS_GRAPHICS_UNSWIZZLING_APERTURE_UNAVAILABLE cpu_to_le32(0xC01E0107)
+#define STATUS_GRAPHICS_UNSWIZZLING_APERTURE_UNSUPPORTED cpu_to_le32(0xC01E0108)
+#define STATUS_GRAPHICS_CANT_EVICT_PINNED_ALLOCATION cpu_to_le32(0xC01E0109)
+#define STATUS_GRAPHICS_INVALID_ALLOCATION_USAGE cpu_to_le32(0xC01E0110)
+#define STATUS_GRAPHICS_CANT_RENDER_LOCKED_ALLOCATION cpu_to_le32(0xC01E0111)
+#define STATUS_GRAPHICS_ALLOCATION_CLOSED cpu_to_le32(0xC01E0112)
+#define STATUS_GRAPHICS_INVALID_ALLOCATION_INSTANCE cpu_to_le32(0xC01E0113)
+#define STATUS_GRAPHICS_INVALID_ALLOCATION_HANDLE cpu_to_le32(0xC01E0114)
+#define STATUS_GRAPHICS_WRONG_ALLOCATION_DEVICE cpu_to_le32(0xC01E0115)
+#define STATUS_GRAPHICS_ALLOCATION_CONTENT_LOST cpu_to_le32(0xC01E0116)
+#define STATUS_GRAPHICS_GPU_EXCEPTION_ON_DEVICE cpu_to_le32(0xC01E0200)
+#define STATUS_GRAPHICS_INVALID_VIDPN_TOPOLOGY cpu_to_le32(0xC01E0300)
+#define STATUS_GRAPHICS_VIDPN_TOPOLOGY_NOT_SUPPORTED cpu_to_le32(0xC01E0301)
+#define STATUS_GRAPHICS_VIDPN_TOPOLOGY_CURRENTLY_NOT_SUPPORTED cpu_to_le32(0xC01E0302)
+#define STATUS_GRAPHICS_INVALID_VIDPN cpu_to_le32(0xC01E0303)
+#define STATUS_GRAPHICS_INVALID_VIDEO_PRESENT_SOURCE cpu_to_le32(0xC01E0304)
+#define STATUS_GRAPHICS_INVALID_VIDEO_PRESENT_TARGET cpu_to_le32(0xC01E0305)
+#define STATUS_GRAPHICS_VIDPN_MODALITY_NOT_SUPPORTED cpu_to_le32(0xC01E0306)
+#define STATUS_GRAPHICS_INVALID_VIDPN_SOURCEMODESET cpu_to_le32(0xC01E0308)
+#define STATUS_GRAPHICS_INVALID_VIDPN_TARGETMODESET cpu_to_le32(0xC01E0309)
+#define STATUS_GRAPHICS_INVALID_FREQUENCY cpu_to_le32(0xC01E030A)
+#define STATUS_GRAPHICS_INVALID_ACTIVE_REGION cpu_to_le32(0xC01E030B)
+#define STATUS_GRAPHICS_INVALID_TOTAL_REGION cpu_to_le32(0xC01E030C)
+#define STATUS_GRAPHICS_INVALID_VIDEO_PRESENT_SOURCE_MODE cpu_to_le32(0xC01E0310)
+#define STATUS_GRAPHICS_INVALID_VIDEO_PRESENT_TARGET_MODE cpu_to_le32(0xC01E0311)
+#define STATUS_GRAPHICS_PINNED_MODE_MUST_REMAIN_IN_SET cpu_to_le32(0xC01E0312)
+#define STATUS_GRAPHICS_PATH_ALREADY_IN_TOPOLOGY cpu_to_le32(0xC01E0313)
+#define STATUS_GRAPHICS_MODE_ALREADY_IN_MODESET cpu_to_le32(0xC01E0314)
+#define STATUS_GRAPHICS_INVALID_VIDEOPRESENTSOURCESET cpu_to_le32(0xC01E0315)
+#define STATUS_GRAPHICS_INVALID_VIDEOPRESENTTARGETSET cpu_to_le32(0xC01E0316)
+#define STATUS_GRAPHICS_SOURCE_ALREADY_IN_SET cpu_to_le32(0xC01E0317)
+#define STATUS_GRAPHICS_TARGET_ALREADY_IN_SET cpu_to_le32(0xC01E0318)
+#define STATUS_GRAPHICS_INVALID_VIDPN_PRESENT_PATH cpu_to_le32(0xC01E0319)
+#define STATUS_GRAPHICS_NO_RECOMMENDED_VIDPN_TOPOLOGY cpu_to_le32(0xC01E031A)
+#define STATUS_GRAPHICS_INVALID_MONITOR_FREQUENCYRANGESET cpu_to_le32(0xC01E031B)
+#define STATUS_GRAPHICS_INVALID_MONITOR_FREQUENCYRANGE cpu_to_le32(0xC01E031C)
+#define STATUS_GRAPHICS_FREQUENCYRANGE_NOT_IN_SET cpu_to_le32(0xC01E031D)
+#define STATUS_GRAPHICS_FREQUENCYRANGE_ALREADY_IN_SET cpu_to_le32(0xC01E031F)
+#define STATUS_GRAPHICS_STALE_MODESET cpu_to_le32(0xC01E0320)
+#define STATUS_GRAPHICS_INVALID_MONITOR_SOURCEMODESET cpu_to_le32(0xC01E0321)
+#define STATUS_GRAPHICS_INVALID_MONITOR_SOURCE_MODE cpu_to_le32(0xC01E0322)
+#define STATUS_GRAPHICS_NO_RECOMMENDED_FUNCTIONAL_VIDPN cpu_to_le32(0xC01E0323)
+#define STATUS_GRAPHICS_MODE_ID_MUST_BE_UNIQUE cpu_to_le32(0xC01E0324)
+#define STATUS_GRAPHICS_EMPTY_ADAPTER_MONITOR_MODE_SUPPORT_INTERSECTION cpu_to_le32(0xC01E0325)
+#define STATUS_GRAPHICS_VIDEO_PRESENT_TARGETS_LESS_THAN_SOURCES cpu_to_le32(0xC01E0326)
+#define STATUS_GRAPHICS_PATH_NOT_IN_TOPOLOGY cpu_to_le32(0xC01E0327)
+#define STATUS_GRAPHICS_ADAPTER_MUST_HAVE_AT_LEAST_ONE_SOURCE cpu_to_le32(0xC01E0328)
+#define STATUS_GRAPHICS_ADAPTER_MUST_HAVE_AT_LEAST_ONE_TARGET cpu_to_le32(0xC01E0329)
+#define STATUS_GRAPHICS_INVALID_MONITORDESCRIPTORSET cpu_to_le32(0xC01E032A)
+#define STATUS_GRAPHICS_INVALID_MONITORDESCRIPTOR cpu_to_le32(0xC01E032B)
+#define STATUS_GRAPHICS_MONITORDESCRIPTOR_NOT_IN_SET cpu_to_le32(0xC01E032C)
+#define STATUS_GRAPHICS_MONITORDESCRIPTOR_ALREADY_IN_SET cpu_to_le32(0xC01E032D)
+#define STATUS_GRAPHICS_MONITORDESCRIPTOR_ID_MUST_BE_UNIQUE cpu_to_le32(0xC01E032E)
+#define STATUS_GRAPHICS_INVALID_VIDPN_TARGET_SUBSET_TYPE cpu_to_le32(0xC01E032F)
+#define STATUS_GRAPHICS_RESOURCES_NOT_RELATED cpu_to_le32(0xC01E0330)
+#define STATUS_GRAPHICS_SOURCE_ID_MUST_BE_UNIQUE cpu_to_le32(0xC01E0331)
+#define STATUS_GRAPHICS_TARGET_ID_MUST_BE_UNIQUE cpu_to_le32(0xC01E0332)
+#define STATUS_GRAPHICS_NO_AVAILABLE_VIDPN_TARGET cpu_to_le32(0xC01E0333)
+#define STATUS_GRAPHICS_MONITOR_COULD_NOT_BE_ASSOCIATED_WITH_ADAPTER cpu_to_le32(0xC01E0334)
+#define STATUS_GRAPHICS_NO_VIDPNMGR cpu_to_le32(0xC01E0335)
+#define STATUS_GRAPHICS_NO_ACTIVE_VIDPN cpu_to_le32(0xC01E0336)
+#define STATUS_GRAPHICS_STALE_VIDPN_TOPOLOGY cpu_to_le32(0xC01E0337)
+#define STATUS_GRAPHICS_MONITOR_NOT_CONNECTED cpu_to_le32(0xC01E0338)
+#define STATUS_GRAPHICS_SOURCE_NOT_IN_TOPOLOGY cpu_to_le32(0xC01E0339)
+#define STATUS_GRAPHICS_INVALID_PRIMARYSURFACE_SIZE cpu_to_le32(0xC01E033A)
+#define STATUS_GRAPHICS_INVALID_VISIBLEREGION_SIZE cpu_to_le32(0xC01E033B)
+#define STATUS_GRAPHICS_INVALID_STRIDE cpu_to_le32(0xC01E033C)
+#define STATUS_GRAPHICS_INVALID_PIXELFORMAT cpu_to_le32(0xC01E033D)
+#define STATUS_GRAPHICS_INVALID_COLORBASIS cpu_to_le32(0xC01E033E)
+#define STATUS_GRAPHICS_INVALID_PIXELVALUEACCESSMODE cpu_to_le32(0xC01E033F)
+#define STATUS_GRAPHICS_TARGET_NOT_IN_TOPOLOGY cpu_to_le32(0xC01E0340)
+#define STATUS_GRAPHICS_NO_DISPLAY_MODE_MANAGEMENT_SUPPORT cpu_to_le32(0xC01E0341)
+#define STATUS_GRAPHICS_VIDPN_SOURCE_IN_USE cpu_to_le32(0xC01E0342)
+#define STATUS_GRAPHICS_CANT_ACCESS_ACTIVE_VIDPN cpu_to_le32(0xC01E0343)
+#define STATUS_GRAPHICS_INVALID_PATH_IMPORTANCE_ORDINAL cpu_to_le32(0xC01E0344)
+#define STATUS_GRAPHICS_INVALID_PATH_CONTENT_GEOMETRY_TRANSFORMATION cpu_to_le32(0xC01E0345)
+#define STATUS_GRAPHICS_PATH_CONTENT_GEOMETRY_TRANSFORMATION_NOT_SUPPORTED cpu_to_le32(0xC01E0346)
+#define STATUS_GRAPHICS_INVALID_GAMMA_RAMP cpu_to_le32(0xC01E0347)
+#define STATUS_GRAPHICS_GAMMA_RAMP_NOT_SUPPORTED cpu_to_le32(0xC01E0348)
+#define STATUS_GRAPHICS_MULTISAMPLING_NOT_SUPPORTED cpu_to_le32(0xC01E0349)
+#define STATUS_GRAPHICS_MODE_NOT_IN_MODESET cpu_to_le32(0xC01E034A)
+#define STATUS_GRAPHICS_INVALID_VIDPN_TOPOLOGY_RECOMMENDATION_REASON cpu_to_le32(0xC01E034D)
+#define STATUS_GRAPHICS_INVALID_PATH_CONTENT_TYPE cpu_to_le32(0xC01E034E)
+#define STATUS_GRAPHICS_INVALID_COPYPROTECTION_TYPE cpu_to_le32(0xC01E034F)
+#define STATUS_GRAPHICS_UNASSIGNED_MODESET_ALREADY_EXISTS cpu_to_le32(0xC01E0350)
+#define STATUS_GRAPHICS_INVALID_SCANLINE_ORDERING cpu_to_le32(0xC01E0352)
+#define STATUS_GRAPHICS_TOPOLOGY_CHANGES_NOT_ALLOWED cpu_to_le32(0xC01E0353)
+#define STATUS_GRAPHICS_NO_AVAILABLE_IMPORTANCE_ORDINALS cpu_to_le32(0xC01E0354)
+#define STATUS_GRAPHICS_INCOMPATIBLE_PRIVATE_FORMAT cpu_to_le32(0xC01E0355)
+#define STATUS_GRAPHICS_INVALID_MODE_PRUNING_ALGORITHM cpu_to_le32(0xC01E0356)
+#define STATUS_GRAPHICS_INVALID_MONITOR_CAPABILITY_ORIGIN cpu_to_le32(0xC01E0357)
+#define STATUS_GRAPHICS_INVALID_MONITOR_FREQUENCYRANGE_CONSTRAINT cpu_to_le32(0xC01E0358)
+#define STATUS_GRAPHICS_MAX_NUM_PATHS_REACHED cpu_to_le32(0xC01E0359)
+#define STATUS_GRAPHICS_CANCEL_VIDPN_TOPOLOGY_AUGMENTATION cpu_to_le32(0xC01E035A)
+#define STATUS_GRAPHICS_INVALID_CLIENT_TYPE cpu_to_le32(0xC01E035B)
+#define STATUS_GRAPHICS_CLIENTVIDPN_NOT_SET cpu_to_le32(0xC01E035C)
+#define STATUS_GRAPHICS_SPECIFIED_CHILD_ALREADY_CONNECTED cpu_to_le32(0xC01E0400)
+#define STATUS_GRAPHICS_CHILD_DESCRIPTOR_NOT_SUPPORTED cpu_to_le32(0xC01E0401)
+#define STATUS_GRAPHICS_NOT_A_LINKED_ADAPTER cpu_to_le32(0xC01E0430)
+#define STATUS_GRAPHICS_LEADLINK_NOT_ENUMERATED cpu_to_le32(0xC01E0431)
+#define STATUS_GRAPHICS_CHAINLINKS_NOT_ENUMERATED cpu_to_le32(0xC01E0432)
+#define STATUS_GRAPHICS_ADAPTER_CHAIN_NOT_READY cpu_to_le32(0xC01E0433)
+#define STATUS_GRAPHICS_CHAINLINKS_NOT_STARTED cpu_to_le32(0xC01E0434)
+#define STATUS_GRAPHICS_CHAINLINKS_NOT_POWERED_ON cpu_to_le32(0xC01E0435)
+#define STATUS_GRAPHICS_INCONSISTENT_DEVICE_LINK_STATE cpu_to_le32(0xC01E0436)
+#define STATUS_GRAPHICS_NOT_POST_DEVICE_DRIVER cpu_to_le32(0xC01E0438)
+#define STATUS_GRAPHICS_ADAPTER_ACCESS_NOT_EXCLUDED cpu_to_le32(0xC01E043B)
+#define STATUS_GRAPHICS_OPM_PROTECTED_OUTPUT_DOES_NOT_HAVE_COPP_SEMANTICS cpu_to_le32(0xC01E051C)
+#define STATUS_GRAPHICS_OPM_INVALID_INFORMATION_REQUEST cpu_to_le32(0xC01E051D)
+#define STATUS_GRAPHICS_OPM_DRIVER_INTERNAL_ERROR cpu_to_le32(0xC01E051E)
+#define STATUS_GRAPHICS_OPM_PROTECTED_OUTPUT_DOES_NOT_HAVE_OPM_SEMANTICS cpu_to_le32(0xC01E051F)
+#define STATUS_GRAPHICS_OPM_SIGNALING_NOT_SUPPORTED cpu_to_le32(0xC01E0520)
+#define STATUS_GRAPHICS_OPM_INVALID_CONFIGURATION_REQUEST cpu_to_le32(0xC01E0521)
+#define STATUS_GRAPHICS_OPM_NOT_SUPPORTED cpu_to_le32(0xC01E0500)
+#define STATUS_GRAPHICS_COPP_NOT_SUPPORTED cpu_to_le32(0xC01E0501)
+#define STATUS_GRAPHICS_UAB_NOT_SUPPORTED cpu_to_le32(0xC01E0502)
+#define STATUS_GRAPHICS_OPM_INVALID_ENCRYPTED_PARAMETERS cpu_to_le32(0xC01E0503)
+#define STATUS_GRAPHICS_OPM_PARAMETER_ARRAY_TOO_SMALL cpu_to_le32(0xC01E0504)
+#define STATUS_GRAPHICS_OPM_NO_PROTECTED_OUTPUTS_EXIST cpu_to_le32(0xC01E0505)
+#define STATUS_GRAPHICS_PVP_NO_DISPLAY_DEVICE_CORRESPONDS_TO_NAME cpu_to_le32(0xC01E0506)
+#define STATUS_GRAPHICS_PVP_DISPLAY_DEVICE_NOT_ATTACHED_TO_DESKTOP cpu_to_le32(0xC01E0507)
+#define STATUS_GRAPHICS_PVP_MIRRORING_DEVICES_NOT_SUPPORTED cpu_to_le32(0xC01E0508)
+#define STATUS_GRAPHICS_OPM_INVALID_POINTER cpu_to_le32(0xC01E050A)
+#define STATUS_GRAPHICS_OPM_INTERNAL_ERROR cpu_to_le32(0xC01E050B)
+#define STATUS_GRAPHICS_OPM_INVALID_HANDLE cpu_to_le32(0xC01E050C)
+#define STATUS_GRAPHICS_PVP_NO_MONITORS_CORRESPOND_TO_DISPLAY_DEVICE cpu_to_le32(0xC01E050D)
+#define STATUS_GRAPHICS_PVP_INVALID_CERTIFICATE_LENGTH cpu_to_le32(0xC01E050E)
+#define STATUS_GRAPHICS_OPM_SPANNING_MODE_ENABLED cpu_to_le32(0xC01E050F)
+#define STATUS_GRAPHICS_OPM_THEATER_MODE_ENABLED cpu_to_le32(0xC01E0510)
+#define STATUS_GRAPHICS_PVP_HFS_FAILED cpu_to_le32(0xC01E0511)
+#define STATUS_GRAPHICS_OPM_INVALID_SRM cpu_to_le32(0xC01E0512)
+#define STATUS_GRAPHICS_OPM_OUTPUT_DOES_NOT_SUPPORT_HDCP cpu_to_le32(0xC01E0513)
+#define STATUS_GRAPHICS_OPM_OUTPUT_DOES_NOT_SUPPORT_ACP cpu_to_le32(0xC01E0514)
+#define STATUS_GRAPHICS_OPM_OUTPUT_DOES_NOT_SUPPORT_CGMSA cpu_to_le32(0xC01E0515)
+#define STATUS_GRAPHICS_OPM_HDCP_SRM_NEVER_SET cpu_to_le32(0xC01E0516)
+#define STATUS_GRAPHICS_OPM_RESOLUTION_TOO_HIGH cpu_to_le32(0xC01E0517)
+#define STATUS_GRAPHICS_OPM_ALL_HDCP_HARDWARE_ALREADY_IN_USE cpu_to_le32(0xC01E0518)
+#define STATUS_GRAPHICS_OPM_PROTECTED_OUTPUT_NO_LONGER_EXISTS cpu_to_le32(0xC01E051A)
+#define STATUS_GRAPHICS_OPM_SESSION_TYPE_CHANGE_IN_PROGRESS cpu_to_le32(0xC01E051B)
+#define STATUS_GRAPHICS_I2C_NOT_SUPPORTED cpu_to_le32(0xC01E0580)
+#define STATUS_GRAPHICS_I2C_DEVICE_DOES_NOT_EXIST cpu_to_le32(0xC01E0581)
+#define STATUS_GRAPHICS_I2C_ERROR_TRANSMITTING_DATA cpu_to_le32(0xC01E0582)
+#define STATUS_GRAPHICS_I2C_ERROR_RECEIVING_DATA cpu_to_le32(0xC01E0583)
+#define STATUS_GRAPHICS_DDCCI_VCP_NOT_SUPPORTED cpu_to_le32(0xC01E0584)
+#define STATUS_GRAPHICS_DDCCI_INVALID_DATA cpu_to_le32(0xC01E0585)
+#define STATUS_GRAPHICS_DDCCI_MONITOR_RETURNED_INVALID_TIMING_STATUS_BYTE cpu_to_le32(0xC01E0586)
+#define STATUS_GRAPHICS_DDCCI_INVALID_CAPABILITIES_STRING cpu_to_le32(0xC01E0587)
+#define STATUS_GRAPHICS_MCA_INTERNAL_ERROR cpu_to_le32(0xC01E0588)
+#define STATUS_GRAPHICS_DDCCI_INVALID_MESSAGE_COMMAND cpu_to_le32(0xC01E0589)
+#define STATUS_GRAPHICS_DDCCI_INVALID_MESSAGE_LENGTH cpu_to_le32(0xC01E058A)
+#define STATUS_GRAPHICS_DDCCI_INVALID_MESSAGE_CHECKSUM cpu_to_le32(0xC01E058B)
+#define STATUS_GRAPHICS_INVALID_PHYSICAL_MONITOR_HANDLE cpu_to_le32(0xC01E058C)
+#define STATUS_GRAPHICS_MONITOR_NO_LONGER_EXISTS cpu_to_le32(0xC01E058D)
+#define STATUS_GRAPHICS_ONLY_CONSOLE_SESSION_SUPPORTED cpu_to_le32(0xC01E05E0)
+#define STATUS_GRAPHICS_NO_DISPLAY_DEVICE_CORRESPONDS_TO_NAME cpu_to_le32(0xC01E05E1)
+#define STATUS_GRAPHICS_DISPLAY_DEVICE_NOT_ATTACHED_TO_DESKTOP cpu_to_le32(0xC01E05E2)
+#define STATUS_GRAPHICS_MIRRORING_DEVICES_NOT_SUPPORTED cpu_to_le32(0xC01E05E3)
+#define STATUS_GRAPHICS_INVALID_POINTER cpu_to_le32(0xC01E05E4)
+#define STATUS_GRAPHICS_NO_MONITORS_CORRESPOND_TO_DISPLAY_DEVICE cpu_to_le32(0xC01E05E5)
+#define STATUS_GRAPHICS_PARAMETER_ARRAY_TOO_SMALL cpu_to_le32(0xC01E05E6)
+#define STATUS_GRAPHICS_INTERNAL_ERROR cpu_to_le32(0xC01E05E7)
+#define STATUS_GRAPHICS_SESSION_TYPE_CHANGE_IN_PROGRESS cpu_to_le32(0xC01E05E8)
+#define STATUS_FVE_LOCKED_VOLUME cpu_to_le32(0xC0210000)
+#define STATUS_FVE_NOT_ENCRYPTED cpu_to_le32(0xC0210001)
+#define STATUS_FVE_BAD_INFORMATION cpu_to_le32(0xC0210002)
+#define STATUS_FVE_TOO_SMALL cpu_to_le32(0xC0210003)
+#define STATUS_FVE_FAILED_WRONG_FS cpu_to_le32(0xC0210004)
+#define STATUS_FVE_FAILED_BAD_FS cpu_to_le32(0xC0210005)
+#define STATUS_FVE_FS_NOT_EXTENDED cpu_to_le32(0xC0210006)
+#define STATUS_FVE_FS_MOUNTED cpu_to_le32(0xC0210007)
+#define STATUS_FVE_NO_LICENSE cpu_to_le32(0xC0210008)
+#define STATUS_FVE_ACTION_NOT_ALLOWED cpu_to_le32(0xC0210009)
+#define STATUS_FVE_BAD_DATA cpu_to_le32(0xC021000A)
+#define STATUS_FVE_VOLUME_NOT_BOUND cpu_to_le32(0xC021000B)
+#define STATUS_FVE_NOT_DATA_VOLUME cpu_to_le32(0xC021000C)
+#define STATUS_FVE_CONV_READ_ERROR cpu_to_le32(0xC021000D)
+#define STATUS_FVE_CONV_WRITE_ERROR cpu_to_le32(0xC021000E)
+#define STATUS_FVE_OVERLAPPED_UPDATE cpu_to_le32(0xC021000F)
+#define STATUS_FVE_FAILED_SECTOR_SIZE cpu_to_le32(0xC0210010)
+#define STATUS_FVE_FAILED_AUTHENTICATION cpu_to_le32(0xC0210011)
+#define STATUS_FVE_NOT_OS_VOLUME cpu_to_le32(0xC0210012)
+#define STATUS_FVE_KEYFILE_NOT_FOUND cpu_to_le32(0xC0210013)
+#define STATUS_FVE_KEYFILE_INVALID cpu_to_le32(0xC0210014)
+#define STATUS_FVE_KEYFILE_NO_VMK cpu_to_le32(0xC0210015)
+#define STATUS_FVE_TPM_DISABLED cpu_to_le32(0xC0210016)
+#define STATUS_FVE_TPM_SRK_AUTH_NOT_ZERO cpu_to_le32(0xC0210017)
+#define STATUS_FVE_TPM_INVALID_PCR cpu_to_le32(0xC0210018)
+#define STATUS_FVE_TPM_NO_VMK cpu_to_le32(0xC0210019)
+#define STATUS_FVE_PIN_INVALID cpu_to_le32(0xC021001A)
+#define STATUS_FVE_AUTH_INVALID_APPLICATION cpu_to_le32(0xC021001B)
+#define STATUS_FVE_AUTH_INVALID_CONFIG cpu_to_le32(0xC021001C)
+#define STATUS_FVE_DEBUGGER_ENABLED cpu_to_le32(0xC021001D)
+#define STATUS_FVE_DRY_RUN_FAILED cpu_to_le32(0xC021001E)
+#define STATUS_FVE_BAD_METADATA_POINTER cpu_to_le32(0xC021001F)
+#define STATUS_FVE_OLD_METADATA_COPY cpu_to_le32(0xC0210020)
+#define STATUS_FVE_REBOOT_REQUIRED cpu_to_le32(0xC0210021)
+#define STATUS_FVE_RAW_ACCESS cpu_to_le32(0xC0210022)
+#define STATUS_FVE_RAW_BLOCKED cpu_to_le32(0xC0210023)
+#define STATUS_FWP_CALLOUT_NOT_FOUND cpu_to_le32(0xC0220001)
+#define STATUS_FWP_CONDITION_NOT_FOUND cpu_to_le32(0xC0220002)
+#define STATUS_FWP_FILTER_NOT_FOUND cpu_to_le32(0xC0220003)
+#define STATUS_FWP_LAYER_NOT_FOUND cpu_to_le32(0xC0220004)
+#define STATUS_FWP_PROVIDER_NOT_FOUND cpu_to_le32(0xC0220005)
+#define STATUS_FWP_PROVIDER_CONTEXT_NOT_FOUND cpu_to_le32(0xC0220006)
+#define STATUS_FWP_SUBLAYER_NOT_FOUND cpu_to_le32(0xC0220007)
+#define STATUS_FWP_NOT_FOUND cpu_to_le32(0xC0220008)
+#define STATUS_FWP_ALREADY_EXISTS cpu_to_le32(0xC0220009)
+#define STATUS_FWP_IN_USE cpu_to_le32(0xC022000A)
+#define STATUS_FWP_DYNAMIC_SESSION_IN_PROGRESS cpu_to_le32(0xC022000B)
+#define STATUS_FWP_WRONG_SESSION cpu_to_le32(0xC022000C)
+#define STATUS_FWP_NO_TXN_IN_PROGRESS cpu_to_le32(0xC022000D)
+#define STATUS_FWP_TXN_IN_PROGRESS cpu_to_le32(0xC022000E)
+#define STATUS_FWP_TXN_ABORTED cpu_to_le32(0xC022000F)
+#define STATUS_FWP_SESSION_ABORTED cpu_to_le32(0xC0220010)
+#define STATUS_FWP_INCOMPATIBLE_TXN cpu_to_le32(0xC0220011)
+#define STATUS_FWP_TIMEOUT cpu_to_le32(0xC0220012)
+#define STATUS_FWP_NET_EVENTS_DISABLED cpu_to_le32(0xC0220013)
+#define STATUS_FWP_INCOMPATIBLE_LAYER cpu_to_le32(0xC0220014)
+#define STATUS_FWP_KM_CLIENTS_ONLY cpu_to_le32(0xC0220015)
+#define STATUS_FWP_LIFETIME_MISMATCH cpu_to_le32(0xC0220016)
+#define STATUS_FWP_BUILTIN_OBJECT cpu_to_le32(0xC0220017)
+#define STATUS_FWP_TOO_MANY_BOOTTIME_FILTERS cpu_to_le32(0xC0220018)
+#define STATUS_FWP_TOO_MANY_CALLOUTS cpu_to_le32(0xC0220018)
+#define STATUS_FWP_NOTIFICATION_DROPPED cpu_to_le32(0xC0220019)
+#define STATUS_FWP_TRAFFIC_MISMATCH cpu_to_le32(0xC022001A)
+#define STATUS_FWP_INCOMPATIBLE_SA_STATE cpu_to_le32(0xC022001B)
+#define STATUS_FWP_NULL_POINTER cpu_to_le32(0xC022001C)
+#define STATUS_FWP_INVALID_ENUMERATOR cpu_to_le32(0xC022001D)
+#define STATUS_FWP_INVALID_FLAGS cpu_to_le32(0xC022001E)
+#define STATUS_FWP_INVALID_NET_MASK cpu_to_le32(0xC022001F)
+#define STATUS_FWP_INVALID_RANGE cpu_to_le32(0xC0220020)
+#define STATUS_FWP_INVALID_INTERVAL cpu_to_le32(0xC0220021)
+#define STATUS_FWP_ZERO_LENGTH_ARRAY cpu_to_le32(0xC0220022)
+#define STATUS_FWP_NULL_DISPLAY_NAME cpu_to_le32(0xC0220023)
+#define STATUS_FWP_INVALID_ACTION_TYPE cpu_to_le32(0xC0220024)
+#define STATUS_FWP_INVALID_WEIGHT cpu_to_le32(0xC0220025)
+#define STATUS_FWP_MATCH_TYPE_MISMATCH cpu_to_le32(0xC0220026)
+#define STATUS_FWP_TYPE_MISMATCH cpu_to_le32(0xC0220027)
+#define STATUS_FWP_OUT_OF_BOUNDS cpu_to_le32(0xC0220028)
+#define STATUS_FWP_RESERVED cpu_to_le32(0xC0220029)
+#define STATUS_FWP_DUPLICATE_CONDITION cpu_to_le32(0xC022002A)
+#define STATUS_FWP_DUPLICATE_KEYMOD cpu_to_le32(0xC022002B)
+#define STATUS_FWP_ACTION_INCOMPATIBLE_WITH_LAYER cpu_to_le32(0xC022002C)
+#define STATUS_FWP_ACTION_INCOMPATIBLE_WITH_SUBLAYER cpu_to_le32(0xC022002D)
+#define STATUS_FWP_CONTEXT_INCOMPATIBLE_WITH_LAYER cpu_to_le32(0xC022002E)
+#define STATUS_FWP_CONTEXT_INCOMPATIBLE_WITH_CALLOUT cpu_to_le32(0xC022002F)
+#define STATUS_FWP_INCOMPATIBLE_AUTH_METHOD cpu_to_le32(0xC0220030)
+#define STATUS_FWP_INCOMPATIBLE_DH_GROUP cpu_to_le32(0xC0220031)
+#define STATUS_FWP_EM_NOT_SUPPORTED cpu_to_le32(0xC0220032)
+#define STATUS_FWP_NEVER_MATCH cpu_to_le32(0xC0220033)
+#define STATUS_FWP_PROVIDER_CONTEXT_MISMATCH cpu_to_le32(0xC0220034)
+#define STATUS_FWP_INVALID_PARAMETER cpu_to_le32(0xC0220035)
+#define STATUS_FWP_TOO_MANY_SUBLAYERS cpu_to_le32(0xC0220036)
+#define STATUS_FWP_CALLOUT_NOTIFICATION_FAILED cpu_to_le32(0xC0220037)
+#define STATUS_FWP_INCOMPATIBLE_AUTH_CONFIG cpu_to_le32(0xC0220038)
+#define STATUS_FWP_INCOMPATIBLE_CIPHER_CONFIG cpu_to_le32(0xC0220039)
+#define STATUS_FWP_TCPIP_NOT_READY cpu_to_le32(0xC0220100)
+#define STATUS_FWP_INJECT_HANDLE_CLOSING cpu_to_le32(0xC0220101)
+#define STATUS_FWP_INJECT_HANDLE_STALE cpu_to_le32(0xC0220102)
+#define STATUS_FWP_CANNOT_PEND cpu_to_le32(0xC0220103)
+#define STATUS_NDIS_CLOSING cpu_to_le32(0xC0230002)
+#define STATUS_NDIS_BAD_VERSION cpu_to_le32(0xC0230004)
+#define STATUS_NDIS_BAD_CHARACTERISTICS cpu_to_le32(0xC0230005)
+#define STATUS_NDIS_ADAPTER_NOT_FOUND cpu_to_le32(0xC0230006)
+#define STATUS_NDIS_OPEN_FAILED cpu_to_le32(0xC0230007)
+#define STATUS_NDIS_DEVICE_FAILED cpu_to_le32(0xC0230008)
+#define STATUS_NDIS_MULTICAST_FULL cpu_to_le32(0xC0230009)
+#define STATUS_NDIS_MULTICAST_EXISTS cpu_to_le32(0xC023000A)
+#define STATUS_NDIS_MULTICAST_NOT_FOUND cpu_to_le32(0xC023000B)
+#define STATUS_NDIS_REQUEST_ABORTED cpu_to_le32(0xC023000C)
+#define STATUS_NDIS_RESET_IN_PROGRESS cpu_to_le32(0xC023000D)
+#define STATUS_NDIS_INVALID_PACKET cpu_to_le32(0xC023000F)
+#define STATUS_NDIS_INVALID_DEVICE_REQUEST cpu_to_le32(0xC0230010)
+#define STATUS_NDIS_ADAPTER_NOT_READY cpu_to_le32(0xC0230011)
+#define STATUS_NDIS_INVALID_LENGTH cpu_to_le32(0xC0230014)
+#define STATUS_NDIS_INVALID_DATA cpu_to_le32(0xC0230015)
+#define STATUS_NDIS_BUFFER_TOO_SHORT cpu_to_le32(0xC0230016)
+#define STATUS_NDIS_INVALID_OID cpu_to_le32(0xC0230017)
+#define STATUS_NDIS_ADAPTER_REMOVED cpu_to_le32(0xC0230018)
+#define STATUS_NDIS_UNSUPPORTED_MEDIA cpu_to_le32(0xC0230019)
+#define STATUS_NDIS_GROUP_ADDRESS_IN_USE cpu_to_le32(0xC023001A)
+#define STATUS_NDIS_FILE_NOT_FOUND cpu_to_le32(0xC023001B)
+#define STATUS_NDIS_ERROR_READING_FILE cpu_to_le32(0xC023001C)
+#define STATUS_NDIS_ALREADY_MAPPED cpu_to_le32(0xC023001D)
+#define STATUS_NDIS_RESOURCE_CONFLICT cpu_to_le32(0xC023001E)
+#define STATUS_NDIS_MEDIA_DISCONNECTED cpu_to_le32(0xC023001F)
+#define STATUS_NDIS_INVALID_ADDRESS cpu_to_le32(0xC0230022)
+#define STATUS_NDIS_PAUSED cpu_to_le32(0xC023002A)
+#define STATUS_NDIS_INTERFACE_NOT_FOUND cpu_to_le32(0xC023002B)
+#define STATUS_NDIS_UNSUPPORTED_REVISION cpu_to_le32(0xC023002C)
+#define STATUS_NDIS_INVALID_PORT cpu_to_le32(0xC023002D)
+#define STATUS_NDIS_INVALID_PORT_STATE cpu_to_le32(0xC023002E)
+#define STATUS_NDIS_LOW_POWER_STATE cpu_to_le32(0xC023002F)
+#define STATUS_NDIS_NOT_SUPPORTED cpu_to_le32(0xC02300BB)
+#define STATUS_NDIS_DOT11_AUTO_CONFIG_ENABLED cpu_to_le32(0xC0232000)
+#define STATUS_NDIS_DOT11_MEDIA_IN_USE cpu_to_le32(0xC0232001)
+#define STATUS_NDIS_DOT11_POWER_STATE_INVALID cpu_to_le32(0xC0232002)
+#define STATUS_IPSEC_BAD_SPI cpu_to_le32(0xC0360001)
+#define STATUS_IPSEC_SA_LIFETIME_EXPIRED cpu_to_le32(0xC0360002)
+#define STATUS_IPSEC_WRONG_SA cpu_to_le32(0xC0360003)
+#define STATUS_IPSEC_REPLAY_CHECK_FAILED cpu_to_le32(0xC0360004)
+#define STATUS_IPSEC_INVALID_PACKET cpu_to_le32(0xC0360005)
+#define STATUS_IPSEC_INTEGRITY_CHECK_FAILED cpu_to_le32(0xC0360006)
+#define STATUS_IPSEC_CLEAR_TEXT_DROP cpu_to_le32(0xC0360007)
diff --git a/fs/cifs/smbdirect.c b/fs/cifs/smbdirect.c
index b943b74cd246..251ef1223206 100644
--- a/fs/cifs/smbdirect.c
+++ b/fs/cifs/smbdirect.c
@@ -164,95 +164,6 @@ do {									\
 #define log_rdma_mr(level, fmt, args...) \
 		log_rdma(level, LOG_RDMA_MR, fmt, ##args)
 
-/*
- * Destroy the transport and related RDMA and memory resources
- * Need to go through all the pending counters and make sure on one is using
- * the transport while it is destroyed
- */
-static void smbd_destroy_rdma_work(struct work_struct *work)
-{
-	struct smbd_response *response;
-	struct smbd_connection *info =
-		container_of(work, struct smbd_connection, destroy_work);
-	unsigned long flags;
-
-	log_rdma_event(INFO, "destroying qp\n");
-	ib_drain_qp(info->id->qp);
-	rdma_destroy_qp(info->id);
-
-	/* Unblock all I/O waiting on the send queue */
-	wake_up_interruptible_all(&info->wait_send_queue);
-
-	log_rdma_event(INFO, "cancelling idle timer\n");
-	cancel_delayed_work_sync(&info->idle_timer_work);
-	log_rdma_event(INFO, "cancelling send immediate work\n");
-	cancel_delayed_work_sync(&info->send_immediate_work);
-
-	log_rdma_event(INFO, "wait for all send to finish\n");
-	wait_event(info->wait_smbd_send_pending,
-		info->smbd_send_pending == 0);
-
-	log_rdma_event(INFO, "wait for all recv to finish\n");
-	wake_up_interruptible(&info->wait_reassembly_queue);
-	wait_event(info->wait_smbd_recv_pending,
-		info->smbd_recv_pending == 0);
-
-	log_rdma_event(INFO, "wait for all send posted to IB to finish\n");
-	wait_event(info->wait_send_pending,
-		atomic_read(&info->send_pending) == 0);
-	wait_event(info->wait_send_payload_pending,
-		atomic_read(&info->send_payload_pending) == 0);
-
-	log_rdma_event(INFO, "freeing mr list\n");
-	wake_up_interruptible_all(&info->wait_mr);
-	wait_event(info->wait_for_mr_cleanup,
-		atomic_read(&info->mr_used_count) == 0);
-	destroy_mr_list(info);
-
-	/* It's not posssible for upper layer to get to reassembly */
-	log_rdma_event(INFO, "drain the reassembly queue\n");
-	do {
-		spin_lock_irqsave(&info->reassembly_queue_lock, flags);
-		response = _get_first_reassembly(info);
-		if (response) {
-			list_del(&response->list);
-			spin_unlock_irqrestore(
-				&info->reassembly_queue_lock, flags);
-			put_receive_buffer(info, response);
-		} else
-			spin_unlock_irqrestore(&info->reassembly_queue_lock, flags);
-	} while (response);
-
-	info->reassembly_data_length = 0;
-
-	log_rdma_event(INFO, "free receive buffers\n");
-	wait_event(info->wait_receive_queues,
-		info->count_receive_queue + info->count_empty_packet_queue
-			== info->receive_credit_max);
-	destroy_receive_buffers(info);
-
-	ib_free_cq(info->send_cq);
-	ib_free_cq(info->recv_cq);
-	ib_dealloc_pd(info->pd);
-	rdma_destroy_id(info->id);
-
-	/* free mempools */
-	mempool_destroy(info->request_mempool);
-	kmem_cache_destroy(info->request_cache);
-
-	mempool_destroy(info->response_mempool);
-	kmem_cache_destroy(info->response_cache);
-
-	info->transport_status = SMBD_DESTROYED;
-	wake_up_all(&info->wait_destroy);
-}
-
-static int smbd_process_disconnected(struct smbd_connection *info)
-{
-	schedule_work(&info->destroy_work);
-	return 0;
-}
-
 static void smbd_disconnect_rdma_work(struct work_struct *work)
 {
 	struct smbd_connection *info =
@@ -319,7 +230,9 @@ static int smbd_conn_upcall(
 		}
 
 		info->transport_status = SMBD_DISCONNECTED;
-		smbd_process_disconnected(info);
+		wake_up_interruptible(&info->disconn_wait);
+		wake_up_interruptible(&info->wait_reassembly_queue);
+		wake_up_interruptible_all(&info->wait_send_queue);
 		break;
 
 	default:
@@ -940,7 +853,7 @@ static int smbd_create_header(struct smbd_connection *info,
 
 	if (info->transport_status != SMBD_CONNECTED) {
 		log_outgoing(ERR, "disconnected not sending\n");
-		return -ENOENT;
+		return -EAGAIN;
 	}
 	atomic_dec(&info->send_credits);
 
@@ -1066,6 +979,7 @@ static int smbd_post_send(struct smbd_connection *info,
 				wake_up(&info->wait_send_pending);
 		}
 		smbd_disconnect_rdma_connection(info);
+		rc = -EAGAIN;
 	} else
 		/* Reset timer for idle connection after packet is sent */
 		mod_delayed_work(info->workqueue, &info->idle_timer_work,
@@ -1478,17 +1392,97 @@ static void idle_connection_timer(struct work_struct *work)
 			info->keep_alive_interval*HZ);
 }
 
-/* Destroy this SMBD connection, called from upper layer */
-void smbd_destroy(struct smbd_connection *info)
+/*
+ * Destroy the transport and related RDMA and memory resources
+ * Need to go through all the pending counters and make sure on one is using
+ * the transport while it is destroyed
+ */
+void smbd_destroy(struct TCP_Server_Info *server)
 {
+	struct smbd_connection *info = server->smbd_conn;
+	struct smbd_response *response;
+	unsigned long flags;
+
+	if (!info) {
+		log_rdma_event(INFO, "rdma session already destroyed\n");
+		return;
+	}
+
 	log_rdma_event(INFO, "destroying rdma session\n");
+	if (info->transport_status != SMBD_DISCONNECTED) {
+		rdma_disconnect(server->smbd_conn->id);
+		log_rdma_event(INFO, "wait for transport being disconnected\n");
+		wait_event_interruptible(
+			info->disconn_wait,
+			info->transport_status == SMBD_DISCONNECTED);
+	}
 
-	/* Kick off the disconnection process */
-	smbd_disconnect_rdma_connection(info);
+	log_rdma_event(INFO, "destroying qp\n");
+	ib_drain_qp(info->id->qp);
+	rdma_destroy_qp(info->id);
+
+	log_rdma_event(INFO, "cancelling idle timer\n");
+	cancel_delayed_work_sync(&info->idle_timer_work);
+	log_rdma_event(INFO, "cancelling send immediate work\n");
+	cancel_delayed_work_sync(&info->send_immediate_work);
+
+	log_rdma_event(INFO, "wait for all send posted to IB to finish\n");
+	wait_event(info->wait_send_pending,
+		atomic_read(&info->send_pending) == 0);
+	wait_event(info->wait_send_payload_pending,
+		atomic_read(&info->send_payload_pending) == 0);
+
+	/* It's not posssible for upper layer to get to reassembly */
+	log_rdma_event(INFO, "drain the reassembly queue\n");
+	do {
+		spin_lock_irqsave(&info->reassembly_queue_lock, flags);
+		response = _get_first_reassembly(info);
+		if (response) {
+			list_del(&response->list);
+			spin_unlock_irqrestore(
+				&info->reassembly_queue_lock, flags);
+			put_receive_buffer(info, response);
+		} else
+			spin_unlock_irqrestore(
+				&info->reassembly_queue_lock, flags);
+	} while (response);
+	info->reassembly_data_length = 0;
+
+	log_rdma_event(INFO, "free receive buffers\n");
+	wait_event(info->wait_receive_queues,
+		info->count_receive_queue + info->count_empty_packet_queue
+			== info->receive_credit_max);
+	destroy_receive_buffers(info);
+
+	/*
+	 * For performance reasons, memory registration and deregistration
+	 * are not locked by srv_mutex. It is possible some processes are
+	 * blocked on transport srv_mutex while holding memory registration.
+	 * Release the transport srv_mutex to allow them to hit the failure
+	 * path when sending data, and then release memory registartions.
+	 */
+	log_rdma_event(INFO, "freeing mr list\n");
+	wake_up_interruptible_all(&info->wait_mr);
+	while (atomic_read(&info->mr_used_count)) {
+		mutex_unlock(&server->srv_mutex);
+		msleep(1000);
+		mutex_lock(&server->srv_mutex);
+	}
+	destroy_mr_list(info);
+
+	ib_free_cq(info->send_cq);
+	ib_free_cq(info->recv_cq);
+	ib_dealloc_pd(info->pd);
+	rdma_destroy_id(info->id);
+
+	/* free mempools */
+	mempool_destroy(info->request_mempool);
+	kmem_cache_destroy(info->request_cache);
 
-	log_rdma_event(INFO, "wait for transport being destroyed\n");
-	wait_event(info->wait_destroy,
-		info->transport_status == SMBD_DESTROYED);
+	mempool_destroy(info->response_mempool);
+	kmem_cache_destroy(info->response_cache);
+
+	info->transport_status = SMBD_DESTROYED;
 
 	destroy_workqueue(info->workqueue);
 	kfree(info);
@@ -1513,17 +1507,9 @@ int smbd_reconnect(struct TCP_Server_Info *server)
 	 */
 	if (server->smbd_conn->transport_status == SMBD_CONNECTED) {
 		log_rdma_event(INFO, "disconnecting transport\n");
-		smbd_disconnect_rdma_connection(server->smbd_conn);
+		smbd_destroy(server);
 	}
 
-	/* wait until the transport is destroyed */
-	if (!wait_event_timeout(server->smbd_conn->wait_destroy,
-		server->smbd_conn->transport_status == SMBD_DESTROYED, 5*HZ))
-		return -EAGAIN;
-
-	destroy_workqueue(server->smbd_conn->workqueue);
-	kfree(server->smbd_conn);
-
 create_conn:
 	log_rdma_event(INFO, "creating rdma session\n");
 	server->smbd_conn = smbd_get_connection(
@@ -1739,12 +1725,13 @@ static struct smbd_connection *_smbd_get_connection(
 	conn_param.retry_count = SMBD_CM_RETRY;
 	conn_param.rnr_retry_count = SMBD_CM_RNR_RETRY;
 	conn_param.flow_control = 0;
-	init_waitqueue_head(&info->wait_destroy);
 
 	log_rdma_event(INFO, "connecting to IP %pI4 port %d\n",
 		&addr_in->sin_addr, port);
 
 	init_waitqueue_head(&info->conn_wait);
+	init_waitqueue_head(&info->disconn_wait);
+	init_waitqueue_head(&info->wait_reassembly_queue);
 	rc = rdma_connect(info->id, &conn_param);
 	if (rc) {
 		log_rdma_event(ERR, "rdma_connect() failed with %i\n", rc);
@@ -1768,19 +1755,11 @@ static struct smbd_connection *_smbd_get_connection(
 	}
 
 	init_waitqueue_head(&info->wait_send_queue);
-	init_waitqueue_head(&info->wait_reassembly_queue);
-
 	INIT_DELAYED_WORK(&info->idle_timer_work, idle_connection_timer);
 	INIT_DELAYED_WORK(&info->send_immediate_work, send_immediate_work);
 	queue_delayed_work(info->workqueue, &info->idle_timer_work,
 		info->keep_alive_interval*HZ);
 
-	init_waitqueue_head(&info->wait_smbd_send_pending);
-	info->smbd_send_pending = 0;
-
-	init_waitqueue_head(&info->wait_smbd_recv_pending);
-	info->smbd_recv_pending = 0;
-
 	init_waitqueue_head(&info->wait_send_pending);
 	atomic_set(&info->send_pending, 0);
 
@@ -1788,7 +1767,6 @@ static struct smbd_connection *_smbd_get_connection(
 	atomic_set(&info->send_payload_pending, 0);
 
 	INIT_WORK(&info->disconnect_work, smbd_disconnect_rdma_work);
-	INIT_WORK(&info->destroy_work, smbd_destroy_rdma_work);
 	INIT_WORK(&info->recv_done_work, smbd_recv_done_work);
 	INIT_WORK(&info->post_send_credits_work, smbd_post_send_credits);
 	info->new_credits_offered = 0;
@@ -1810,7 +1788,7 @@ static struct smbd_connection *_smbd_get_connection(
 
 allocate_mr_failed:
 	/* At this point, need to a full transport shutdown */
-	smbd_destroy(info);
+	smbd_destroy(server);
 	return NULL;
 
 negotiation_failed:
@@ -1882,11 +1860,6 @@ static int smbd_recv_buf(struct smbd_connection *info, char *buf,
 	int rc;
 
 again:
-	if (info->transport_status != SMBD_CONNECTED) {
-		log_read(ERR, "disconnected\n");
-		return -ENODEV;
-	}
-
 	/*
 	 * No need to hold the reassembly queue lock all the time as we are
 	 * the only one reading from the front of the queue. The transport
@@ -2000,7 +1973,12 @@ read_rfc1002_done:
 			info->transport_status != SMBD_CONNECTED);
 	/* Don't return any data if interrupted */
 	if (rc)
-		return -ENODEV;
+		return rc;
+
+	if (info->transport_status != SMBD_CONNECTED) {
+		log_read(ERR, "disconnected\n");
+		return 0;
+	}
 
 	goto again;
 }
@@ -2052,8 +2030,6 @@ int smbd_recv(struct smbd_connection *info, struct msghdr *msg)
 	unsigned int to_read, page_offset;
 	int rc;
 
-	info->smbd_recv_pending++;
-
 	if (iov_iter_rw(&msg->msg_iter) == WRITE) {
 		/* It's a bug in upper layer to get there */
 		cifs_dbg(VFS, "CIFS: invalid msg iter dir %u\n",
@@ -2084,9 +2060,6 @@ int smbd_recv(struct smbd_connection *info, struct msghdr *msg)
 	}
 
 out:
-	info->smbd_recv_pending--;
-	wake_up(&info->wait_smbd_recv_pending);
-
 	/* SMBDirect will read it all or nothing */
 	if (rc > 0)
 		msg->msg_iter.count = 0;
@@ -2099,7 +2072,8 @@ out:
  * rqst: the data to write
  * return value: 0 if successfully write, otherwise error code
  */
-int smbd_send(struct TCP_Server_Info *server, struct smb_rqst *rqst)
+int smbd_send(struct TCP_Server_Info *server,
+	int num_rqst, struct smb_rqst *rqst_array)
 {
 	struct smbd_connection *info = server->smbd_conn;
 	struct kvec vec;
@@ -2111,54 +2085,49 @@ int smbd_send(struct TCP_Server_Info *server, struct smb_rqst *rqst)
 		info->max_send_size - sizeof(struct smbd_data_transfer);
 	struct kvec *iov;
 	int rc;
+	struct smb_rqst *rqst;
+	int rqst_idx;
 
-	info->smbd_send_pending++;
 	if (info->transport_status != SMBD_CONNECTED) {
-		rc = -ENODEV;
+		rc = -EAGAIN;
 		goto done;
 	}
 
 	/*
-	 * Skip the RFC1002 length defined in MS-SMB2 section 2.1
-	 * It is used only for TCP transport in the iov[0]
-	 * In future we may want to add a transport layer under protocol
-	 * layer so this will only be issued to TCP transport
-	 */
-
-	if (rqst->rq_iov[0].iov_len != 4) {
-		log_write(ERR, "expected the pdu length in 1st iov, but got %zu\n", rqst->rq_iov[0].iov_len);
-		return -EINVAL;
-	}
-
-	/*
 	 * Add in the page array if there is one. The caller needs to set
 	 * rq_tailsz to PAGE_SIZE when the buffer has multiple pages and
 	 * ends at page boundary
 	 */
-	buflen = smb_rqst_len(server, rqst);
+	remaining_data_length = 0;
+	for (i = 0; i < num_rqst; i++)
+		remaining_data_length += smb_rqst_len(server, &rqst_array[i]);
 
-	if (buflen + sizeof(struct smbd_data_transfer) >
+	if (remaining_data_length + sizeof(struct smbd_data_transfer) >
 		info->max_fragmented_send_size) {
 		log_write(ERR, "payload size %d > max size %d\n",
-			buflen, info->max_fragmented_send_size);
+			remaining_data_length, info->max_fragmented_send_size);
 		rc = -EINVAL;
 		goto done;
 	}
 
-	iov = &rqst->rq_iov[1];
+	rqst_idx = 0;
+
+next_rqst:
+	rqst = &rqst_array[rqst_idx];
+	iov = rqst->rq_iov;
 
-	cifs_dbg(FYI, "Sending smb (RDMA): smb_len=%u\n", buflen);
-	for (i = 0; i < rqst->rq_nvec-1; i++)
+	cifs_dbg(FYI, "Sending smb (RDMA): idx=%d smb_len=%lu\n",
+		rqst_idx, smb_rqst_len(server, rqst));
+	for (i = 0; i < rqst->rq_nvec; i++)
 		dump_smb(iov[i].iov_base, iov[i].iov_len);
 
-	remaining_data_length = buflen;
 
-	log_write(INFO, "rqst->rq_nvec=%d rqst->rq_npages=%d rq_pagesz=%d "
-		"rq_tailsz=%d buflen=%d\n",
-		rqst->rq_nvec, rqst->rq_npages, rqst->rq_pagesz,
-		rqst->rq_tailsz, buflen);
+	log_write(INFO, "rqst_idx=%d nvec=%d rqst->rq_npages=%d rq_pagesz=%d "
+		"rq_tailsz=%d buflen=%lu\n",
+		rqst_idx, rqst->rq_nvec, rqst->rq_npages, rqst->rq_pagesz,
+		rqst->rq_tailsz, smb_rqst_len(server, rqst));
 
-	start = i = iov[0].iov_len ? 0 : 1;
+	start = i = 0;
 	buflen = 0;
 	while (true) {
 		buflen += iov[i].iov_len;
@@ -2206,14 +2175,14 @@ int smbd_send(struct TCP_Server_Info *server, struct smb_rqst *rqst)
 						goto done;
 				}
 				i++;
-				if (i == rqst->rq_nvec-1)
+				if (i == rqst->rq_nvec)
 					break;
 			}
 			start = i;
 			buflen = 0;
 		} else {
 			i++;
-			if (i == rqst->rq_nvec-1) {
+			if (i == rqst->rq_nvec) {
 				/* send out all remaining vecs */
 				remaining_data_length -= buflen;
 				log_write(INFO,
@@ -2257,6 +2226,10 @@ int smbd_send(struct TCP_Server_Info *server, struct smb_rqst *rqst)
 		}
 	}
 
+	rqst_idx++;
+	if (rqst_idx < num_rqst)
+		goto next_rqst;
+
 done:
 	/*
 	 * As an optimization, we don't wait for individual I/O to finish
@@ -2268,9 +2241,6 @@ done:
 	wait_event(info->wait_send_payload_pending,
 		atomic_read(&info->send_payload_pending) == 0);
 
-	info->smbd_send_pending--;
-	wake_up(&info->wait_smbd_send_pending);
-
 	return rc;
 }
 
diff --git a/fs/cifs/smbdirect.h b/fs/cifs/smbdirect.h
index a11096254f29..f6241b8bce5f 100644
--- a/fs/cifs/smbdirect.h
+++ b/fs/cifs/smbdirect.h
@@ -70,12 +70,11 @@ struct smbd_connection {
 	int ri_rc;
 	struct completion ri_done;
 	wait_queue_head_t conn_wait;
-	wait_queue_head_t wait_destroy;
+	wait_queue_head_t disconn_wait;
 
 	struct completion negotiate_completion;
 	bool negotiate_done;
 
-	struct work_struct destroy_work;
 	struct work_struct disconnect_work;
 	struct work_struct recv_done_work;
 	struct work_struct post_send_credits_work;
@@ -123,13 +122,6 @@ struct smbd_connection {
 	wait_queue_head_t wait_for_mr_cleanup;
 
 	/* Activity accoutning */
-	/* Pending reqeusts issued from upper layer */
-	int smbd_send_pending;
-	wait_queue_head_t wait_smbd_send_pending;
-
-	int smbd_recv_pending;
-	wait_queue_head_t wait_smbd_recv_pending;
-
 	atomic_t send_pending;
 	wait_queue_head_t wait_send_pending;
 	atomic_t send_payload_pending;
@@ -288,11 +280,12 @@ struct smbd_connection *smbd_get_connection(
 /* Reconnect SMBDirect session */
 int smbd_reconnect(struct TCP_Server_Info *server);
 /* Destroy SMBDirect session */
-void smbd_destroy(struct smbd_connection *info);
+void smbd_destroy(struct TCP_Server_Info *server);
 
 /* Interface for carrying upper layer I/O through send/recv */
 int smbd_recv(struct smbd_connection *info, struct msghdr *msg);
-int smbd_send(struct TCP_Server_Info *server, struct smb_rqst *rqst);
+int smbd_send(struct TCP_Server_Info *server,
+	int num_rqst, struct smb_rqst *rqst);
 
 enum mr_state {
 	MR_READY,
@@ -330,9 +323,9 @@ struct smbd_connection {};
 static inline void *smbd_get_connection(
 	struct TCP_Server_Info *server, struct sockaddr *dstaddr) {return NULL;}
 static inline int smbd_reconnect(struct TCP_Server_Info *server) {return -1; }
-static inline void smbd_destroy(struct smbd_connection *info) {}
+static inline void smbd_destroy(struct TCP_Server_Info *server) {}
 static inline int smbd_recv(struct smbd_connection *info, struct msghdr *msg) {return -1; }
-static inline int smbd_send(struct TCP_Server_Info *server, struct smb_rqst *rqst) {return -1; }
+static inline int smbd_send(struct TCP_Server_Info *server, int num_rqst, struct smb_rqst *rqst) {return -1; }
 #endif
 
 #endif
diff --git a/fs/cifs/smbfsctl.h b/fs/cifs/smbfsctl.h
index f996daeea271..08628e6a42ac 100644
--- a/fs/cifs/smbfsctl.h
+++ b/fs/cifs/smbfsctl.h
@@ -35,6 +35,33 @@
  * below). Additional detail on less common ones can be found in MS-FSCC
  * section 2.3.
  */
+
+/*
+ * FSCTL values are 32 bits and are constructed as
+ * <device 16bits> <access 2bits> <function 12bits> <method 2bits>
+ */
+/* Device */
+#define FSCTL_DEVICE_DFS                 (0x0006 << 16)
+#define FSCTL_DEVICE_FILE_SYSTEM         (0x0009 << 16)
+#define FSCTL_DEVICE_NAMED_PIPE          (0x0011 << 16)
+#define FSCTL_DEVICE_NETWORK_FILE_SYSTEM (0x0014 << 16)
+#define FSCTL_DEVICE_MASK                0xffff0000
+/* Access */
+#define FSCTL_DEVICE_ACCESS_FILE_ANY_ACCESS        (0x00 << 14)
+#define FSCTL_DEVICE_ACCESS_FILE_READ_ACCESS       (0x01 << 14)
+#define FSCTL_DEVICE_ACCESS_FILE_WRITE_ACCESS      (0x02 << 14)
+#define FSCTL_DEVICE_ACCESS_FILE_READ_WRITE_ACCESS (0x03 << 14)
+#define FSCTL_DEVICE_ACCESS_MASK                   0x0000c000
+/* Function */
+#define FSCTL_DEVICE_FUNCTION_MASK       0x00003ffc
+/* Method */
+#define FSCTL_DEVICE_METHOD_BUFFERED   0x00
+#define FSCTL_DEVICE_METHOD_IN_DIRECT  0x01
+#define FSCTL_DEVICE_METHOD_OUT_DIRECT 0x02
+#define FSCTL_DEVICE_METHOD_NEITHER    0x03
+#define FSCTL_DEVICE_METHOD_MASK       0x00000003
+
+
 #define FSCTL_DFS_GET_REFERRALS      0x00060194
 #define FSCTL_DFS_GET_REFERRALS_EX   0x000601B0
 #define FSCTL_REQUEST_OPLOCK_LEVEL_1 0x00090000
@@ -76,7 +103,7 @@
 #define FSCTL_SET_ZERO_ON_DEALLOC    0x00090194 /* BB add struct */
 #define FSCTL_SET_SHORT_NAME_BEHAVIOR 0x000901B4 /* BB add struct */
 #define FSCTL_GET_INTEGRITY_INFORMATION 0x0009027C
-#define FSCTL_QUERY_ALLOCATED_RANGES 0x000940CF /* BB add struct */
+#define FSCTL_QUERY_ALLOCATED_RANGES 0x000940CF
 #define FSCTL_SET_DEFECT_MANAGEMENT  0x00098134 /* BB add struct */
 #define FSCTL_FILE_LEVEL_TRIM        0x00098208 /* BB add struct */
 #define FSCTL_DUPLICATE_EXTENTS_TO_FILE 0x00098344
diff --git a/fs/cifs/transport.c b/fs/cifs/transport.c
index 1de8e996e566..9a16ff4b9f5e 100644
--- a/fs/cifs/transport.c
+++ b/fs/cifs/transport.c
@@ -104,7 +104,10 @@ DeleteMidQEntry(struct mid_q_entry *midEntry)
 {
 #ifdef CONFIG_CIFS_STATS2
 	__le16 command = midEntry->server->vals->lock_cmd;
+	__u16 smb_cmd = le16_to_cpu(midEntry->command);
 	unsigned long now;
+	unsigned long roundtrip_time;
+	struct TCP_Server_Info *server = midEntry->server;
 #endif
 	midEntry->mid_state = MID_FREE;
 	atomic_dec(&midCount);
@@ -114,6 +117,23 @@ DeleteMidQEntry(struct mid_q_entry *midEntry)
 		cifs_small_buf_release(midEntry->resp_buf);
 #ifdef CONFIG_CIFS_STATS2
 	now = jiffies;
+	if (now < midEntry->when_alloc)
+		cifs_dbg(VFS, "invalid mid allocation time\n");
+	roundtrip_time = now - midEntry->when_alloc;
+
+	if (smb_cmd < NUMBER_OF_SMB2_COMMANDS) {
+		if (atomic_read(&server->num_cmds[smb_cmd]) == 0) {
+			server->slowest_cmd[smb_cmd] = roundtrip_time;
+			server->fastest_cmd[smb_cmd] = roundtrip_time;
+		} else {
+			if (server->slowest_cmd[smb_cmd] < roundtrip_time)
+				server->slowest_cmd[smb_cmd] = roundtrip_time;
+			else if (server->fastest_cmd[smb_cmd] > roundtrip_time)
+				server->fastest_cmd[smb_cmd] = roundtrip_time;
+		}
+		cifs_stats_inc(&server->num_cmds[smb_cmd]);
+		server->time_per_cmd[smb_cmd] += roundtrip_time;
+	}
 	/*
 	 * commands taking longer than one second (default) can be indications
 	 * that something is wrong, unless it is quite a slow link or a very
@@ -131,11 +151,10 @@ DeleteMidQEntry(struct mid_q_entry *midEntry)
 		 * smb2slowcmd[NUMBER_OF_SMB2_COMMANDS] counts by command
 		 * NB: le16_to_cpu returns unsigned so can not be negative below
 		 */
-		if (le16_to_cpu(midEntry->command) < NUMBER_OF_SMB2_COMMANDS)
-			cifs_stats_inc(&midEntry->server->smb2slowcmd[le16_to_cpu(midEntry->command)]);
+		if (smb_cmd < NUMBER_OF_SMB2_COMMANDS)
+			cifs_stats_inc(&server->smb2slowcmd[smb_cmd]);
 
-		trace_smb3_slow_rsp(le16_to_cpu(midEntry->command),
-			       midEntry->mid, midEntry->pid,
+		trace_smb3_slow_rsp(smb_cmd, midEntry->mid, midEntry->pid,
 			       midEntry->when_sent, midEntry->when_received);
 		if (cifsFYI & CIFS_TIMER) {
 			pr_debug(" CIFS slow rsp: cmd %d mid %llu",
@@ -300,7 +319,7 @@ __smb_send_rqst(struct TCP_Server_Info *server, int num_rqst,
 	__be32 rfc1002_marker;
 
 	if (cifs_rdma_enabled(server) && server->smbd_conn) {
-		rc = smbd_send(server, rqst);
+		rc = smbd_send(server, num_rqst, rqst);
 		goto smbd_done;
 	}
 
@@ -510,7 +529,7 @@ wait_for_free_credits(struct TCP_Server_Info *server, const int num_credits,
 		return -EAGAIN;
 
 	spin_lock(&server->req_lock);
-	if ((flags & CIFS_TIMEOUT_MASK) == CIFS_ASYNC_OP) {
+	if ((flags & CIFS_TIMEOUT_MASK) == CIFS_NON_BLOCKING) {
 		/* oplock breaks must not be held up */
 		server->in_flight++;
 		*credits -= 1;
@@ -819,7 +838,7 @@ SendReceiveNoRsp(const unsigned int xid, struct cifs_ses *ses,
 
 	iov[0].iov_base = in_buf;
 	iov[0].iov_len = get_rfc1002_length(in_buf) + 4;
-	flags |= CIFS_NO_RESP;
+	flags |= CIFS_NO_RSP_BUF;
 	rc = SendReceive2(xid, ses, iov, 1, &resp_buf_type, flags, &rsp_iov);
 	cifs_dbg(NOISY, "SendRcvNoRsp flags %d rc %d\n", flags, rc);
 
@@ -1054,8 +1073,11 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses,
 
 	mutex_unlock(&ses->server->srv_mutex);
 
-	if (rc < 0) {
-		/* Sending failed for some reason - return credits back */
+	/*
+	 * If sending failed for some reason or it is an oplock break that we
+	 * will not receive a response to - return credits back
+	 */
+	if (rc < 0 || (flags & CIFS_NO_SRV_RSP)) {
 		for (i = 0; i < num_rqst; i++)
 			add_credits(ses->server, &credits[i], optype);
 		goto out;
@@ -1076,9 +1098,6 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses,
 		smb311_update_preauth_hash(ses, rqst[0].rq_iov,
 					   rqst[0].rq_nvec);
 
-	if ((flags & CIFS_TIMEOUT_MASK) == CIFS_ASYNC_OP)
-		goto out;
-
 	for (i = 0; i < num_rqst; i++) {
 		rc = wait_for_response(ses->server, midQ[i]);
 		if (rc != 0)
@@ -1132,7 +1151,7 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses,
 						     flags & CIFS_LOG_ERROR);
 
 		/* mark it so buf will not be freed by cifs_delete_mid */
-		if ((flags & CIFS_NO_RESP) == 0)
+		if ((flags & CIFS_NO_RSP_BUF) == 0)
 			midQ[i]->resp_buf = NULL;
 
 	}
@@ -1283,9 +1302,6 @@ SendReceive(const unsigned int xid, struct cifs_ses *ses,
 	if (rc < 0)
 		goto out;
 
-	if ((flags & CIFS_TIMEOUT_MASK) == CIFS_ASYNC_OP)
-		goto out;
-
 	rc = wait_for_response(ses->server, midQ);
 	if (rc != 0) {
 		send_cancel(ses->server, &rqst, midQ);
diff --git a/fs/gfs2/aops.c b/fs/gfs2/aops.c
index 05dd78f4b2b3..6210d4429d84 100644
--- a/fs/gfs2/aops.c
+++ b/fs/gfs2/aops.c
@@ -649,7 +649,7 @@ out_uninit:
  */
 void adjust_fs_space(struct inode *inode)
 {
-	struct gfs2_sbd *sdp = inode->i_sb->s_fs_info;
+	struct gfs2_sbd *sdp = GFS2_SB(inode);
 	struct gfs2_inode *m_ip = GFS2_I(sdp->sd_statfs_inode);
 	struct gfs2_inode *l_ip = GFS2_I(sdp->sd_sc_inode);
 	struct gfs2_statfs_change_host *m_sc = &sdp->sd_statfs_master;
@@ -657,10 +657,13 @@ void adjust_fs_space(struct inode *inode)
 	struct buffer_head *m_bh, *l_bh;
 	u64 fs_total, new_free;
 
+	if (gfs2_trans_begin(sdp, 2 * RES_STATFS, 0) != 0)
+		return;
+
 	/* Total up the file system space, according to the latest rindex. */
 	fs_total = gfs2_ri_total(sdp);
 	if (gfs2_meta_inode_buffer(m_ip, &m_bh) != 0)
-		return;
+		goto out;
 
 	spin_lock(&sdp->sd_statfs_spin);
 	gfs2_statfs_change_in(m_sc, m_bh->b_data +
@@ -675,11 +678,14 @@ void adjust_fs_space(struct inode *inode)
 	gfs2_statfs_change(sdp, new_free, new_free, 0);
 
 	if (gfs2_meta_inode_buffer(l_ip, &l_bh) != 0)
-		goto out;
+		goto out2;
 	update_statfs(sdp, m_bh, l_bh);
 	brelse(l_bh);
-out:
+out2:
 	brelse(m_bh);
+out:
+	sdp->sd_rindex_uptodate = 0;
+	gfs2_trans_end(sdp);
 }
 
 /**
diff --git a/fs/gfs2/bmap.c b/fs/gfs2/bmap.c
index 2f9290f69610..f42718dd292f 100644
--- a/fs/gfs2/bmap.c
+++ b/fs/gfs2/bmap.c
@@ -142,7 +142,7 @@ int gfs2_unstuff_dinode(struct gfs2_inode *ip, struct page *page)
 		if (error)
 			goto out_brelse;
 		if (isdir) {
-			gfs2_trans_add_unrevoke(GFS2_SB(&ip->i_inode), block, 1);
+			gfs2_trans_remove_revoke(GFS2_SB(&ip->i_inode), block, 1);
 			error = gfs2_dir_get_new_buffer(ip, block, &bh);
 			if (error)
 				goto out_brelse;
@@ -676,7 +676,7 @@ static int gfs2_iomap_alloc(struct inode *inode, struct iomap *iomap,
 			goto out;
 		alloced += n;
 		if (state != ALLOC_DATA || gfs2_is_jdata(ip))
-			gfs2_trans_add_unrevoke(sdp, bn, n);
+			gfs2_trans_remove_revoke(sdp, bn, n);
 		switch (state) {
 		/* Growing height of tree */
 		case ALLOC_GROW_HEIGHT:
@@ -925,6 +925,32 @@ do_alloc:
 	goto out;
 }
 
+/**
+ * gfs2_lblk_to_dblk - convert logical block to disk block
+ * @inode: the inode of the file we're mapping
+ * @lblock: the block relative to the start of the file
+ * @dblock: the returned dblock, if no error
+ *
+ * This function maps a single block from a file logical block (relative to
+ * the start of the file) to a file system absolute block using iomap.
+ *
+ * Returns: the absolute file system block, or an error
+ */
+int gfs2_lblk_to_dblk(struct inode *inode, u32 lblock, u64 *dblock)
+{
+	struct iomap iomap = { };
+	struct metapath mp = { .mp_aheight = 1, };
+	loff_t pos = (loff_t)lblock << inode->i_blkbits;
+	int ret;
+
+	ret = gfs2_iomap_get(inode, pos, i_blocksize(inode), 0, &iomap, &mp);
+	release_metapath(&mp);
+	if (ret == 0)
+		*dblock = iomap.addr >> inode->i_blkbits;
+
+	return ret;
+}
+
 static int gfs2_write_lock(struct inode *inode)
 {
 	struct gfs2_inode *ip = GFS2_I(inode);
@@ -965,17 +991,28 @@ static void gfs2_write_unlock(struct inode *inode)
 	gfs2_glock_dq_uninit(&ip->i_gh);
 }
 
+static int gfs2_iomap_page_prepare(struct inode *inode, loff_t pos,
+				   unsigned len, struct iomap *iomap)
+{
+	struct gfs2_sbd *sdp = GFS2_SB(inode);
+
+	return gfs2_trans_begin(sdp, RES_DINODE + (len >> inode->i_blkbits), 0);
+}
+
 static void gfs2_iomap_page_done(struct inode *inode, loff_t pos,
 				 unsigned copied, struct page *page,
 				 struct iomap *iomap)
 {
 	struct gfs2_inode *ip = GFS2_I(inode);
+	struct gfs2_sbd *sdp = GFS2_SB(inode);
 
-	if (page)
+	if (page && !gfs2_is_stuffed(ip))
 		gfs2_page_add_databufs(ip, page, offset_in_page(pos), copied);
+	gfs2_trans_end(sdp);
 }
 
 static const struct iomap_page_ops gfs2_iomap_page_ops = {
+	.page_prepare = gfs2_iomap_page_prepare,
 	.page_done = gfs2_iomap_page_done,
 };
 
@@ -1031,31 +1068,45 @@ static int gfs2_iomap_begin_write(struct inode *inode, loff_t pos,
 	if (alloc_required)
 		rblocks += gfs2_rg_blocks(ip, data_blocks + ind_blocks);
 
-	ret = gfs2_trans_begin(sdp, rblocks, iomap->length >> inode->i_blkbits);
-	if (ret)
-		goto out_trans_fail;
+	if (unstuff || iomap->type == IOMAP_HOLE) {
+		struct gfs2_trans *tr;
 
-	if (unstuff) {
-		ret = gfs2_unstuff_dinode(ip, NULL);
-		if (ret)
-			goto out_trans_end;
-		release_metapath(mp);
-		ret = gfs2_iomap_get(inode, iomap->offset, iomap->length,
-				     flags, iomap, mp);
+		ret = gfs2_trans_begin(sdp, rblocks,
+				       iomap->length >> inode->i_blkbits);
 		if (ret)
-			goto out_trans_end;
-	}
+			goto out_trans_fail;
 
-	if (iomap->type == IOMAP_HOLE) {
-		ret = gfs2_iomap_alloc(inode, iomap, flags, mp);
-		if (ret) {
-			gfs2_trans_end(sdp);
-			gfs2_inplace_release(ip);
-			punch_hole(ip, iomap->offset, iomap->length);
-			goto out_qunlock;
+		if (unstuff) {
+			ret = gfs2_unstuff_dinode(ip, NULL);
+			if (ret)
+				goto out_trans_end;
+			release_metapath(mp);
+			ret = gfs2_iomap_get(inode, iomap->offset,
+					     iomap->length, flags, iomap, mp);
+			if (ret)
+				goto out_trans_end;
+		}
+
+		if (iomap->type == IOMAP_HOLE) {
+			ret = gfs2_iomap_alloc(inode, iomap, flags, mp);
+			if (ret) {
+				gfs2_trans_end(sdp);
+				gfs2_inplace_release(ip);
+				punch_hole(ip, iomap->offset, iomap->length);
+				goto out_qunlock;
+			}
 		}
+
+		tr = current->journal_info;
+		if (tr->tr_num_buf_new)
+			__mark_inode_dirty(inode, I_DIRTY_DATASYNC);
+		else
+			gfs2_trans_add_meta(ip->i_gl, mp->mp_bh[0]);
+
+		gfs2_trans_end(sdp);
 	}
-	if (!gfs2_is_stuffed(ip) && gfs2_is_jdata(ip))
+
+	if (gfs2_is_stuffed(ip) || gfs2_is_jdata(ip))
 		iomap->page_ops = &gfs2_iomap_page_ops;
 	return 0;
 
@@ -1095,10 +1146,6 @@ static int gfs2_iomap_begin(struct inode *inode, loff_t pos, loff_t length,
 		    iomap->type != IOMAP_MAPPED)
 			ret = -ENOTBLK;
 	}
-	if (!ret) {
-		get_bh(mp.mp_bh[0]);
-		iomap->private = mp.mp_bh[0];
-	}
 	release_metapath(&mp);
 	trace_gfs2_iomap_end(ip, iomap, ret);
 	return ret;
@@ -1109,27 +1156,16 @@ static int gfs2_iomap_end(struct inode *inode, loff_t pos, loff_t length,
 {
 	struct gfs2_inode *ip = GFS2_I(inode);
 	struct gfs2_sbd *sdp = GFS2_SB(inode);
-	struct gfs2_trans *tr = current->journal_info;
-	struct buffer_head *dibh = iomap->private;
 
 	if ((flags & (IOMAP_WRITE | IOMAP_DIRECT)) != IOMAP_WRITE)
 		goto out;
 
-	if (iomap->type != IOMAP_INLINE) {
+	if (!gfs2_is_stuffed(ip))
 		gfs2_ordered_add_inode(ip);
 
-		if (tr->tr_num_buf_new)
-			__mark_inode_dirty(inode, I_DIRTY_DATASYNC);
-		else
-			gfs2_trans_add_meta(ip->i_gl, dibh);
-	}
-
-	if (inode == sdp->sd_rindex) {
+	if (inode == sdp->sd_rindex)
 		adjust_fs_space(inode);
-		sdp->sd_rindex_uptodate = 0;
-	}
 
-	gfs2_trans_end(sdp);
 	gfs2_inplace_release(ip);
 
 	if (length != written && (iomap->flags & IOMAP_F_NEW)) {
@@ -1149,8 +1185,6 @@ static int gfs2_iomap_end(struct inode *inode, loff_t pos, loff_t length,
 	gfs2_write_unlock(inode);
 
 out:
-	if (dibh)
-		brelse(dibh);
 	return 0;
 }
 
diff --git a/fs/gfs2/bmap.h b/fs/gfs2/bmap.h
index 6b18fb323f0a..19a1fd772c61 100644
--- a/fs/gfs2/bmap.h
+++ b/fs/gfs2/bmap.h
@@ -64,5 +64,6 @@ extern int gfs2_write_alloc_required(struct gfs2_inode *ip, u64 offset,
 extern int gfs2_map_journal_extents(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd);
 extern void gfs2_free_journal_extents(struct gfs2_jdesc *jd);
 extern int __gfs2_punch_hole(struct file *file, loff_t offset, loff_t length);
+extern int gfs2_lblk_to_dblk(struct inode *inode, u32 lblock, u64 *dblock);
 
 #endif /* __BMAP_DOT_H__ */
diff --git a/fs/gfs2/dir.c b/fs/gfs2/dir.c
index daa14ab4e31b..db9a05244a35 100644
--- a/fs/gfs2/dir.c
+++ b/fs/gfs2/dir.c
@@ -883,7 +883,7 @@ static struct gfs2_leaf *new_leaf(struct inode *inode, struct buffer_head **pbh,
 	if (!bh)
 		return NULL;
 
-	gfs2_trans_add_unrevoke(GFS2_SB(inode), bn, 1);
+	gfs2_trans_remove_revoke(GFS2_SB(inode), bn, 1);
 	gfs2_trans_add_meta(ip->i_gl, bh);
 	gfs2_metatype_set(bh, GFS2_METATYPE_LF, GFS2_FORMAT_LF);
 	leaf = (struct gfs2_leaf *)bh->b_data;
diff --git a/fs/gfs2/glock.c b/fs/gfs2/glock.c
index d32964cd1117..15c605cfcfc8 100644
--- a/fs/gfs2/glock.c
+++ b/fs/gfs2/glock.c
@@ -140,6 +140,7 @@ void gfs2_glock_free(struct gfs2_glock *gl)
 {
 	struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
 
+	BUG_ON(test_bit(GLF_REVOKES, &gl->gl_flags));
 	rhashtable_remove_fast(&gl_hash_table, &gl->gl_node, ht_parms);
 	smp_mb();
 	wake_up_glock(gl);
@@ -183,15 +184,19 @@ static int demote_ok(const struct gfs2_glock *gl)
 
 void gfs2_glock_add_to_lru(struct gfs2_glock *gl)
 {
+	if (!(gl->gl_ops->go_flags & GLOF_LRU))
+		return;
+
 	spin_lock(&lru_lock);
 
-	if (!list_empty(&gl->gl_lru))
-		list_del_init(&gl->gl_lru);
-	else
+	list_del(&gl->gl_lru);
+	list_add_tail(&gl->gl_lru, &lru_list);
+
+	if (!test_bit(GLF_LRU, &gl->gl_flags)) {
+		set_bit(GLF_LRU, &gl->gl_flags);
 		atomic_inc(&lru_count);
+	}
 
-	list_add_tail(&gl->gl_lru, &lru_list);
-	set_bit(GLF_LRU, &gl->gl_flags);
 	spin_unlock(&lru_lock);
 }
 
@@ -201,7 +206,7 @@ static void gfs2_glock_remove_from_lru(struct gfs2_glock *gl)
 		return;
 
 	spin_lock(&lru_lock);
-	if (!list_empty(&gl->gl_lru)) {
+	if (test_bit(GLF_LRU, &gl->gl_flags)) {
 		list_del_init(&gl->gl_lru);
 		atomic_dec(&lru_count);
 		clear_bit(GLF_LRU, &gl->gl_flags);
@@ -1159,8 +1164,7 @@ void gfs2_glock_dq(struct gfs2_holder *gh)
 		    !test_bit(GLF_DEMOTE, &gl->gl_flags))
 			fast_path = 1;
 	}
-	if (!test_bit(GLF_LFLUSH, &gl->gl_flags) && demote_ok(gl) &&
-	    (glops->go_flags & GLOF_LRU))
+	if (!test_bit(GLF_LFLUSH, &gl->gl_flags) && demote_ok(gl))
 		gfs2_glock_add_to_lru(gl);
 
 	trace_gfs2_glock_queue(gh, 0);
@@ -1456,6 +1460,7 @@ __acquires(&lru_lock)
 		if (!spin_trylock(&gl->gl_lockref.lock)) {
 add_back_to_lru:
 			list_add(&gl->gl_lru, &lru_list);
+			set_bit(GLF_LRU, &gl->gl_flags);
 			atomic_inc(&lru_count);
 			continue;
 		}
@@ -1463,7 +1468,6 @@ add_back_to_lru:
 			spin_unlock(&gl->gl_lockref.lock);
 			goto add_back_to_lru;
 		}
-		clear_bit(GLF_LRU, &gl->gl_flags);
 		gl->gl_lockref.count++;
 		if (demote_ok(gl))
 			handle_callback(gl, LM_ST_UNLOCKED, 0, false);
@@ -1498,6 +1502,7 @@ static long gfs2_scan_glock_lru(int nr)
 		if (!test_bit(GLF_LOCK, &gl->gl_flags)) {
 			list_move(&gl->gl_lru, &dispose);
 			atomic_dec(&lru_count);
+			clear_bit(GLF_LRU, &gl->gl_flags);
 			freed++;
 			continue;
 		}
@@ -1796,7 +1801,7 @@ void gfs2_dump_glock(struct seq_file *seq, struct gfs2_glock *gl)
 		  state2str(gl->gl_target),
 		  state2str(gl->gl_demote_state), dtime,
 		  atomic_read(&gl->gl_ail_count),
-		  atomic_read(&gl->gl_revokes),
+		  test_bit(GLF_REVOKES, &gl->gl_flags) ? 1 : 0,
 		  (int)gl->gl_lockref.count, gl->gl_hold_time);
 
 	list_for_each_entry(gh, &gl->gl_holders, gh_list)
diff --git a/fs/gfs2/glops.c b/fs/gfs2/glops.c
index 78510ab91835..24ada3ccc525 100644
--- a/fs/gfs2/glops.c
+++ b/fs/gfs2/glops.c
@@ -28,6 +28,7 @@
 #include "util.h"
 #include "trans.h"
 #include "dir.h"
+#include "lops.h"
 
 struct workqueue_struct *gfs2_freeze_wq;
 
@@ -531,7 +532,7 @@ static int freeze_go_xmote_bh(struct gfs2_glock *gl, struct gfs2_holder *gh)
 	if (test_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags)) {
 		j_gl->gl_ops->go_inval(j_gl, DIO_METADATA);
 
-		error = gfs2_find_jhead(sdp->sd_jdesc, &head);
+		error = gfs2_find_jhead(sdp->sd_jdesc, &head, false);
 		if (error)
 			gfs2_consist(sdp);
 		if (!(head.lh_flags & GFS2_LOG_HEAD_UNMOUNT))
diff --git a/fs/gfs2/incore.h b/fs/gfs2/incore.h
index cdf07b408f54..b15755068593 100644
--- a/fs/gfs2/incore.h
+++ b/fs/gfs2/incore.h
@@ -345,6 +345,7 @@ enum {
 	GLF_OBJECT			= 14, /* Used only for tracing */
 	GLF_BLOCKING			= 15,
 	GLF_INODE_CREATING		= 16, /* Inode creation occurring */
+	GLF_REVOKES			= 17, /* Glock has revokes in queue */
 };
 
 struct gfs2_glock {
@@ -374,7 +375,6 @@ struct gfs2_glock {
 	struct list_head gl_lru;
 	struct list_head gl_ail_list;
 	atomic_t gl_ail_count;
-	atomic_t gl_revokes;
 	struct delayed_work gl_work;
 	union {
 		/* For inode and iopen glocks only */
@@ -535,7 +535,7 @@ struct gfs2_jdesc {
 	unsigned long jd_flags;
 #define JDF_RECOVERY 1
 	unsigned int jd_jid;
-	unsigned int jd_blocks;
+	u32 jd_blocks;
 	int jd_recover_error;
 	/* Replay stuff */
 
@@ -621,6 +621,7 @@ enum {
 	SDF_SKIP_DLM_UNLOCK	= 8,
 	SDF_FORCE_AIL_FLUSH     = 9,
 	SDF_AIL1_IO_ERROR	= 10,
+	SDF_FS_FROZEN           = 11,
 };
 
 enum gfs2_freeze_state {
@@ -809,8 +810,8 @@ struct gfs2_sbd {
 	atomic_t sd_log_pinned;
 	unsigned int sd_log_num_revoke;
 
-	struct list_head sd_log_le_revoke;
-	struct list_head sd_log_le_ordered;
+	struct list_head sd_log_revokes;
+	struct list_head sd_log_ordered;
 	spinlock_t sd_ordered_lock;
 
 	atomic_t sd_log_thresh1;
diff --git a/fs/gfs2/log.c b/fs/gfs2/log.c
index b8830fda51e8..a2e1df488df0 100644
--- a/fs/gfs2/log.c
+++ b/fs/gfs2/log.c
@@ -551,9 +551,9 @@ static void gfs2_ordered_write(struct gfs2_sbd *sdp)
 	LIST_HEAD(written);
 
 	spin_lock(&sdp->sd_ordered_lock);
-	list_sort(NULL, &sdp->sd_log_le_ordered, &ip_cmp);
-	while (!list_empty(&sdp->sd_log_le_ordered)) {
-		ip = list_entry(sdp->sd_log_le_ordered.next, struct gfs2_inode, i_ordered);
+	list_sort(NULL, &sdp->sd_log_ordered, &ip_cmp);
+	while (!list_empty(&sdp->sd_log_ordered)) {
+		ip = list_entry(sdp->sd_log_ordered.next, struct gfs2_inode, i_ordered);
 		if (ip->i_inode.i_mapping->nrpages == 0) {
 			test_and_clear_bit(GIF_ORDERED, &ip->i_flags);
 			list_del(&ip->i_ordered);
@@ -564,7 +564,7 @@ static void gfs2_ordered_write(struct gfs2_sbd *sdp)
 		filemap_fdatawrite(ip->i_inode.i_mapping);
 		spin_lock(&sdp->sd_ordered_lock);
 	}
-	list_splice(&written, &sdp->sd_log_le_ordered);
+	list_splice(&written, &sdp->sd_log_ordered);
 	spin_unlock(&sdp->sd_ordered_lock);
 }
 
@@ -573,8 +573,8 @@ static void gfs2_ordered_wait(struct gfs2_sbd *sdp)
 	struct gfs2_inode *ip;
 
 	spin_lock(&sdp->sd_ordered_lock);
-	while (!list_empty(&sdp->sd_log_le_ordered)) {
-		ip = list_entry(sdp->sd_log_le_ordered.next, struct gfs2_inode, i_ordered);
+	while (!list_empty(&sdp->sd_log_ordered)) {
+		ip = list_entry(sdp->sd_log_ordered.next, struct gfs2_inode, i_ordered);
 		list_del(&ip->i_ordered);
 		WARN_ON(!test_and_clear_bit(GIF_ORDERED, &ip->i_flags));
 		if (ip->i_inode.i_mapping->nrpages == 0)
@@ -606,9 +606,12 @@ void gfs2_add_revoke(struct gfs2_sbd *sdp, struct gfs2_bufdata *bd)
 	gfs2_remove_from_ail(bd); /* drops ref on bh */
 	bd->bd_bh = NULL;
 	sdp->sd_log_num_revoke++;
-	atomic_inc(&gl->gl_revokes);
+	if (!test_bit(GLF_REVOKES, &gl->gl_flags)) {
+		set_bit(GLF_REVOKES, &gl->gl_flags);
+		gfs2_glock_hold(gl);
+	}
 	set_bit(GLF_LFLUSH, &gl->gl_flags);
-	list_add(&bd->bd_list, &sdp->sd_log_le_revoke);
+	list_add(&bd->bd_list, &sdp->sd_log_revokes);
 }
 
 void gfs2_write_revokes(struct gfs2_sbd *sdp)
@@ -666,11 +669,12 @@ out_of_blocks:
 }
 
 /**
- * write_log_header - Write a journal log header buffer at sd_log_flush_head
+ * gfs2_write_log_header - Write a journal log header buffer at lblock
  * @sdp: The GFS2 superblock
  * @jd: journal descriptor of the journal to which we are writing
  * @seq: sequence number
  * @tail: tail of the log
+ * @lblock: value for lh_blkno (block number relative to start of journal)
  * @flags: log header flags GFS2_LOG_HEAD_*
  * @op_flags: flags to pass to the bio
  *
@@ -678,7 +682,8 @@ out_of_blocks:
  */
 
 void gfs2_write_log_header(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd,
-			   u64 seq, u32 tail, u32 flags, int op_flags)
+			   u64 seq, u32 tail, u32 lblock, u32 flags,
+			   int op_flags)
 {
 	struct gfs2_log_header *lh;
 	u32 hash, crc;
@@ -686,7 +691,7 @@ void gfs2_write_log_header(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd,
 	struct gfs2_statfs_change_host *l_sc = &sdp->sd_statfs_local;
 	struct timespec64 tv;
 	struct super_block *sb = sdp->sd_vfs;
-	u64 addr;
+	u64 dblock;
 
 	lh = page_address(page);
 	clear_page(lh);
@@ -699,15 +704,21 @@ void gfs2_write_log_header(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd,
 	lh->lh_sequence = cpu_to_be64(seq);
 	lh->lh_flags = cpu_to_be32(flags);
 	lh->lh_tail = cpu_to_be32(tail);
-	lh->lh_blkno = cpu_to_be32(sdp->sd_log_flush_head);
+	lh->lh_blkno = cpu_to_be32(lblock);
 	hash = ~crc32(~0, lh, LH_V1_SIZE);
 	lh->lh_hash = cpu_to_be32(hash);
 
 	ktime_get_coarse_real_ts64(&tv);
 	lh->lh_nsec = cpu_to_be32(tv.tv_nsec);
 	lh->lh_sec = cpu_to_be64(tv.tv_sec);
-	addr = gfs2_log_bmap(sdp);
-	lh->lh_addr = cpu_to_be64(addr);
+	if (!list_empty(&jd->extent_list))
+		dblock = gfs2_log_bmap(sdp);
+	else {
+		int ret = gfs2_lblk_to_dblk(jd->jd_inode, lblock, &dblock);
+		if (gfs2_assert_withdraw(sdp, ret == 0))
+			return;
+	}
+	lh->lh_addr = cpu_to_be64(dblock);
 	lh->lh_jinode = cpu_to_be64(GFS2_I(jd->jd_inode)->i_no_addr);
 
 	/* We may only write local statfs, quota, etc., when writing to our
@@ -732,8 +743,8 @@ void gfs2_write_log_header(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd,
 		     sb->s_blocksize - LH_V1_SIZE - 4);
 	lh->lh_crc = cpu_to_be32(crc);
 
-	gfs2_log_write(sdp, page, sb->s_blocksize, 0, addr);
-	gfs2_log_submit_bio(&sdp->sd_log_bio, REQ_OP_WRITE, op_flags);
+	gfs2_log_write(sdp, page, sb->s_blocksize, 0, dblock);
+	gfs2_log_submit_bio(&sdp->sd_log_bio, REQ_OP_WRITE | op_flags);
 	log_flush_wait(sdp);
 }
 
@@ -761,7 +772,7 @@ static void log_write_header(struct gfs2_sbd *sdp, u32 flags)
 	}
 	sdp->sd_log_idle = (tail == sdp->sd_log_flush_head);
 	gfs2_write_log_header(sdp, sdp->sd_jdesc, sdp->sd_log_sequence++, tail,
-			      flags, op_flags);
+			      sdp->sd_log_flush_head, flags, op_flags);
 
 	if (sdp->sd_log_tail != tail)
 		log_pull_tail(sdp, tail);
@@ -810,7 +821,7 @@ void gfs2_log_flush(struct gfs2_sbd *sdp, struct gfs2_glock *gl, u32 flags)
 
 	gfs2_ordered_write(sdp);
 	lops_before_commit(sdp, tr);
-	gfs2_log_submit_bio(&sdp->sd_log_bio, REQ_OP_WRITE, 0);
+	gfs2_log_submit_bio(&sdp->sd_log_bio, REQ_OP_WRITE);
 
 	if (sdp->sd_log_head != sdp->sd_log_flush_head) {
 		log_flush_wait(sdp);
diff --git a/fs/gfs2/log.h b/fs/gfs2/log.h
index 1bc9bd444b28..7a34a3234266 100644
--- a/fs/gfs2/log.h
+++ b/fs/gfs2/log.h
@@ -59,7 +59,7 @@ static inline void gfs2_ordered_add_inode(struct gfs2_inode *ip)
 	if (!test_bit(GIF_ORDERED, &ip->i_flags)) {
 		spin_lock(&sdp->sd_ordered_lock);
 		if (!test_and_set_bit(GIF_ORDERED, &ip->i_flags))
-			list_add(&ip->i_ordered, &sdp->sd_log_le_ordered);
+			list_add(&ip->i_ordered, &sdp->sd_log_ordered);
 		spin_unlock(&sdp->sd_ordered_lock);
 	}
 }
@@ -70,7 +70,8 @@ extern unsigned int gfs2_struct2blk(struct gfs2_sbd *sdp, unsigned int nstruct,
 extern void gfs2_log_release(struct gfs2_sbd *sdp, unsigned int blks);
 extern int gfs2_log_reserve(struct gfs2_sbd *sdp, unsigned int blks);
 extern void gfs2_write_log_header(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd,
-				  u64 seq, u32 tail, u32 flags, int op_flags);
+				  u64 seq, u32 tail, u32 lblock, u32 flags,
+				  int op_flags);
 extern void gfs2_log_flush(struct gfs2_sbd *sdp, struct gfs2_glock *gl,
 			   u32 type);
 extern void gfs2_log_commit(struct gfs2_sbd *sdp, struct gfs2_trans *trans);
diff --git a/fs/gfs2/lops.c b/fs/gfs2/lops.c
index 6f09b5e3dd6e..33ab662c9aac 100644
--- a/fs/gfs2/lops.c
+++ b/fs/gfs2/lops.c
@@ -17,7 +17,9 @@
 #include <linux/bio.h>
 #include <linux/fs.h>
 #include <linux/list_sort.h>
+#include <linux/blkdev.h>
 
+#include "bmap.h"
 #include "dir.h"
 #include "gfs2.h"
 #include "incore.h"
@@ -194,7 +196,6 @@ static void gfs2_end_log_write_bh(struct gfs2_sbd *sdp,
 /**
  * gfs2_end_log_write - end of i/o to the log
  * @bio: The bio
- * @error: Status of i/o request
  *
  * Each bio_vec contains either data from the pagecache or data
  * relating to the log itself. Here we iterate over the bio_vec
@@ -231,20 +232,19 @@ static void gfs2_end_log_write(struct bio *bio)
 /**
  * gfs2_log_submit_bio - Submit any pending log bio
  * @biop: Address of the bio pointer
- * @op: REQ_OP
- * @op_flags: req_flag_bits
+ * @opf: REQ_OP | op_flags
  *
  * Submit any pending part-built or full bio to the block device. If
  * there is no pending bio, then this is a no-op.
  */
 
-void gfs2_log_submit_bio(struct bio **biop, int op, int op_flags)
+void gfs2_log_submit_bio(struct bio **biop, int opf)
 {
 	struct bio *bio = *biop;
 	if (bio) {
 		struct gfs2_sbd *sdp = bio->bi_private;
 		atomic_inc(&sdp->sd_log_in_flight);
-		bio_set_op_attrs(bio, op, op_flags);
+		bio->bi_opf = opf;
 		submit_bio(bio);
 		*biop = NULL;
 	}
@@ -305,7 +305,7 @@ static struct bio *gfs2_log_get_bio(struct gfs2_sbd *sdp, u64 blkno,
 		nblk >>= sdp->sd_fsb2bb_shift;
 		if (blkno == nblk && !flush)
 			return bio;
-		gfs2_log_submit_bio(biop, op, 0);
+		gfs2_log_submit_bio(biop, op);
 	}
 
 	*biop = gfs2_log_alloc_bio(sdp, blkno, end_io);
@@ -376,6 +376,205 @@ void gfs2_log_write_page(struct gfs2_sbd *sdp, struct page *page)
 		       gfs2_log_bmap(sdp));
 }
 
+/**
+ * gfs2_end_log_read - end I/O callback for reads from the log
+ * @bio: The bio
+ *
+ * Simply unlock the pages in the bio. The main thread will wait on them and
+ * process them in order as necessary.
+ */
+
+static void gfs2_end_log_read(struct bio *bio)
+{
+	struct page *page;
+	struct bio_vec *bvec;
+	struct bvec_iter_all iter_all;
+
+	bio_for_each_segment_all(bvec, bio, iter_all) {
+		page = bvec->bv_page;
+		if (bio->bi_status) {
+			int err = blk_status_to_errno(bio->bi_status);
+
+			SetPageError(page);
+			mapping_set_error(page->mapping, err);
+		}
+		unlock_page(page);
+	}
+
+	bio_put(bio);
+}
+
+/**
+ * gfs2_jhead_pg_srch - Look for the journal head in a given page.
+ * @jd: The journal descriptor
+ * @page: The page to look in
+ *
+ * Returns: 1 if found, 0 otherwise.
+ */
+
+static bool gfs2_jhead_pg_srch(struct gfs2_jdesc *jd,
+			      struct gfs2_log_header_host *head,
+			      struct page *page)
+{
+	struct gfs2_sbd *sdp = GFS2_SB(jd->jd_inode);
+	struct gfs2_log_header_host uninitialized_var(lh);
+	void *kaddr = kmap_atomic(page);
+	unsigned int offset;
+	bool ret = false;
+
+	for (offset = 0; offset < PAGE_SIZE; offset += sdp->sd_sb.sb_bsize) {
+		if (!__get_log_header(sdp, kaddr + offset, 0, &lh)) {
+			if (lh.lh_sequence > head->lh_sequence)
+				*head = lh;
+			else {
+				ret = true;
+				break;
+			}
+		}
+	}
+	kunmap_atomic(kaddr);
+	return ret;
+}
+
+/**
+ * gfs2_jhead_process_page - Search/cleanup a page
+ * @jd: The journal descriptor
+ * @index: Index of the page to look into
+ * @done: If set, perform only cleanup, else search and set if found.
+ *
+ * Find the page with 'index' in the journal's mapping. Search the page for
+ * the journal head if requested (cleanup == false). Release refs on the
+ * page so the page cache can reclaim it (put_page() twice). We grabbed a
+ * reference on this page two times, first when we did a find_or_create_page()
+ * to obtain the page to add it to the bio and second when we do a
+ * find_get_page() here to get the page to wait on while I/O on it is being
+ * completed.
+ * This function is also used to free up a page we might've grabbed but not
+ * used. Maybe we added it to a bio, but not submitted it for I/O. Or we
+ * submitted the I/O, but we already found the jhead so we only need to drop
+ * our references to the page.
+ */
+
+static void gfs2_jhead_process_page(struct gfs2_jdesc *jd, unsigned long index,
+				    struct gfs2_log_header_host *head,
+				    bool *done)
+{
+	struct page *page;
+
+	page = find_get_page(jd->jd_inode->i_mapping, index);
+	wait_on_page_locked(page);
+
+	if (PageError(page))
+		*done = true;
+
+	if (!*done)
+		*done = gfs2_jhead_pg_srch(jd, head, page);
+
+	put_page(page); /* Once for find_get_page */
+	put_page(page); /* Once more for find_or_create_page */
+}
+
+/**
+ * gfs2_find_jhead - find the head of a log
+ * @jd: The journal descriptor
+ * @head: The log descriptor for the head of the log is returned here
+ *
+ * Do a search of a journal by reading it in large chunks using bios and find
+ * the valid log entry with the highest sequence number.  (i.e. the log head)
+ *
+ * Returns: 0 on success, errno otherwise
+ */
+int gfs2_find_jhead(struct gfs2_jdesc *jd, struct gfs2_log_header_host *head,
+		    bool keep_cache)
+{
+	struct gfs2_sbd *sdp = GFS2_SB(jd->jd_inode);
+	struct address_space *mapping = jd->jd_inode->i_mapping;
+	unsigned int block = 0, blocks_submitted = 0, blocks_read = 0;
+	unsigned int bsize = sdp->sd_sb.sb_bsize;
+	unsigned int bsize_shift = sdp->sd_sb.sb_bsize_shift;
+	unsigned int shift = PAGE_SHIFT - bsize_shift;
+	unsigned int readhead_blocks = BIO_MAX_PAGES << shift;
+	struct gfs2_journal_extent *je;
+	int sz, ret = 0;
+	struct bio *bio = NULL;
+	struct page *page = NULL;
+	bool done = false;
+	errseq_t since;
+
+	memset(head, 0, sizeof(*head));
+	if (list_empty(&jd->extent_list))
+		gfs2_map_journal_extents(sdp, jd);
+
+	since = filemap_sample_wb_err(mapping);
+	list_for_each_entry(je, &jd->extent_list, list) {
+		for (; block < je->lblock + je->blocks; block++) {
+			u64 dblock;
+
+			if (!page) {
+				page = find_or_create_page(mapping,
+						block >> shift, GFP_NOFS);
+				if (!page) {
+					ret = -ENOMEM;
+					done = true;
+					goto out;
+				}
+			}
+
+			if (bio) {
+				unsigned int off;
+
+				off = (block << bsize_shift) & ~PAGE_MASK;
+				sz = bio_add_page(bio, page, bsize, off);
+				if (sz == bsize) { /* block added */
+					if (off + bsize == PAGE_SIZE) {
+						page = NULL;
+						goto page_added;
+					}
+					continue;
+				}
+				blocks_submitted = block + 1;
+				submit_bio(bio);
+				bio = NULL;
+			}
+
+			dblock = je->dblock + (block - je->lblock);
+			bio = gfs2_log_alloc_bio(sdp, dblock, gfs2_end_log_read);
+			bio->bi_opf = REQ_OP_READ;
+			sz = bio_add_page(bio, page, bsize, 0);
+			gfs2_assert_warn(sdp, sz == bsize);
+			if (bsize == PAGE_SIZE)
+				page = NULL;
+
+page_added:
+			if (blocks_submitted < blocks_read + readhead_blocks) {
+				/* Keep at least one bio in flight */
+				continue;
+			}
+
+			gfs2_jhead_process_page(jd, blocks_read >> shift, head, &done);
+			blocks_read += PAGE_SIZE >> bsize_shift;
+			if (done)
+				goto out;  /* found */
+		}
+	}
+
+out:
+	if (bio)
+		submit_bio(bio);
+	while (blocks_read < block) {
+		gfs2_jhead_process_page(jd, blocks_read >> shift, head, &done);
+		blocks_read += PAGE_SIZE >> bsize_shift;
+	}
+
+	if (!ret)
+		ret = filemap_check_wb_err(mapping, since);
+
+	if (!keep_cache)
+		truncate_inode_pages(mapping, 0);
+
+	return ret;
+}
+
 static struct page *gfs2_get_log_desc(struct gfs2_sbd *sdp, u32 ld_type,
 				      u32 ld_length, u32 ld_data1)
 {
@@ -529,7 +728,7 @@ static void buf_lo_before_scan(struct gfs2_jdesc *jd,
 	jd->jd_replayed_blocks = 0;
 }
 
-static int buf_lo_scan_elements(struct gfs2_jdesc *jd, unsigned int start,
+static int buf_lo_scan_elements(struct gfs2_jdesc *jd, u32 start,
 				struct gfs2_log_descriptor *ld, __be64 *ptr,
 				int pass)
 {
@@ -622,7 +821,7 @@ static void revoke_lo_before_commit(struct gfs2_sbd *sdp, struct gfs2_trans *tr)
 {
 	struct gfs2_meta_header *mh;
 	unsigned int offset;
-	struct list_head *head = &sdp->sd_log_le_revoke;
+	struct list_head *head = &sdp->sd_log_revokes;
 	struct gfs2_bufdata *bd;
 	struct page *page;
 	unsigned int length;
@@ -660,18 +859,35 @@ static void revoke_lo_before_commit(struct gfs2_sbd *sdp, struct gfs2_trans *tr)
 
 static void revoke_lo_after_commit(struct gfs2_sbd *sdp, struct gfs2_trans *tr)
 {
-	struct list_head *head = &sdp->sd_log_le_revoke;
-	struct gfs2_bufdata *bd;
-	struct gfs2_glock *gl;
+	struct list_head *head = &sdp->sd_log_revokes;
+	struct gfs2_bufdata *bd, *tmp;
 
-	while (!list_empty(head)) {
-		bd = list_entry(head->next, struct gfs2_bufdata, bd_list);
-		list_del_init(&bd->bd_list);
-		gl = bd->bd_gl;
-		atomic_dec(&gl->gl_revokes);
-		clear_bit(GLF_LFLUSH, &gl->gl_flags);
+	/*
+	 * Glocks can be referenced repeatedly on the revoke list, but the list
+	 * only holds one reference.  All glocks on the list will have the
+	 * GLF_REVOKES flag set initially.
+	 */
+
+	list_for_each_entry_safe(bd, tmp, head, bd_list) {
+		struct gfs2_glock *gl = bd->bd_gl;
+
+		if (test_bit(GLF_REVOKES, &gl->gl_flags)) {
+			/* Keep each glock on the list exactly once. */
+			clear_bit(GLF_REVOKES, &gl->gl_flags);
+			continue;
+		}
+		list_del(&bd->bd_list);
 		kmem_cache_free(gfs2_bufdata_cachep, bd);
 	}
+	list_for_each_entry_safe(bd, tmp, head, bd_list) {
+		struct gfs2_glock *gl = bd->bd_gl;
+
+		list_del(&bd->bd_list);
+		kmem_cache_free(gfs2_bufdata_cachep, bd);
+		clear_bit(GLF_LFLUSH, &gl->gl_flags);
+		gfs2_glock_queue_put(gl);
+	}
+	/* the list is empty now */
 }
 
 static void revoke_lo_before_scan(struct gfs2_jdesc *jd,
@@ -684,7 +900,7 @@ static void revoke_lo_before_scan(struct gfs2_jdesc *jd,
 	jd->jd_replay_tail = head->lh_tail;
 }
 
-static int revoke_lo_scan_elements(struct gfs2_jdesc *jd, unsigned int start,
+static int revoke_lo_scan_elements(struct gfs2_jdesc *jd, u32 start,
 				   struct gfs2_log_descriptor *ld, __be64 *ptr,
 				   int pass)
 {
@@ -766,7 +982,7 @@ static void databuf_lo_before_commit(struct gfs2_sbd *sdp, struct gfs2_trans *tr
 	gfs2_before_commit(sdp, limit, nbuf, &tr->tr_databuf, 1);
 }
 
-static int databuf_lo_scan_elements(struct gfs2_jdesc *jd, unsigned int start,
+static int databuf_lo_scan_elements(struct gfs2_jdesc *jd, u32 start,
 				    struct gfs2_log_descriptor *ld,
 				    __be64 *ptr, int pass)
 {
@@ -852,7 +1068,7 @@ static void databuf_lo_after_commit(struct gfs2_sbd *sdp, struct gfs2_trans *tr)
 }
 
 
-const struct gfs2_log_operations gfs2_buf_lops = {
+static const struct gfs2_log_operations gfs2_buf_lops = {
 	.lo_before_commit = buf_lo_before_commit,
 	.lo_after_commit = buf_lo_after_commit,
 	.lo_before_scan = buf_lo_before_scan,
@@ -861,7 +1077,7 @@ const struct gfs2_log_operations gfs2_buf_lops = {
 	.lo_name = "buf",
 };
 
-const struct gfs2_log_operations gfs2_revoke_lops = {
+static const struct gfs2_log_operations gfs2_revoke_lops = {
 	.lo_before_commit = revoke_lo_before_commit,
 	.lo_after_commit = revoke_lo_after_commit,
 	.lo_before_scan = revoke_lo_before_scan,
@@ -870,7 +1086,7 @@ const struct gfs2_log_operations gfs2_revoke_lops = {
 	.lo_name = "revoke",
 };
 
-const struct gfs2_log_operations gfs2_databuf_lops = {
+static const struct gfs2_log_operations gfs2_databuf_lops = {
 	.lo_before_commit = databuf_lo_before_commit,
 	.lo_after_commit = databuf_lo_after_commit,
 	.lo_scan_elements = databuf_lo_scan_elements,
diff --git a/fs/gfs2/lops.h b/fs/gfs2/lops.h
index 711c4d89c063..f195ffb435ac 100644
--- a/fs/gfs2/lops.h
+++ b/fs/gfs2/lops.h
@@ -20,18 +20,15 @@
 	((sizeof(struct gfs2_log_descriptor) + (2 * sizeof(__be64) - 1)) & \
 	 ~(2 * sizeof(__be64) - 1))
 
-extern const struct gfs2_log_operations gfs2_glock_lops;
-extern const struct gfs2_log_operations gfs2_buf_lops;
-extern const struct gfs2_log_operations gfs2_revoke_lops;
-extern const struct gfs2_log_operations gfs2_databuf_lops;
-
 extern const struct gfs2_log_operations *gfs2_log_ops[];
 extern u64 gfs2_log_bmap(struct gfs2_sbd *sdp);
 extern void gfs2_log_write(struct gfs2_sbd *sdp, struct page *page,
 			   unsigned size, unsigned offset, u64 blkno);
 extern void gfs2_log_write_page(struct gfs2_sbd *sdp, struct page *page);
-extern void gfs2_log_submit_bio(struct bio **biop, int op, int op_flags);
+extern void gfs2_log_submit_bio(struct bio **biop, int opf);
 extern void gfs2_pin(struct gfs2_sbd *sdp, struct buffer_head *bh);
+extern int gfs2_find_jhead(struct gfs2_jdesc *jd,
+			   struct gfs2_log_header_host *head, bool keep_cache);
 
 static inline unsigned int buf_limit(struct gfs2_sbd *sdp)
 {
@@ -77,7 +74,7 @@ static inline void lops_before_scan(struct gfs2_jdesc *jd,
 			gfs2_log_ops[x]->lo_before_scan(jd, head, pass);
 }
 
-static inline int lops_scan_elements(struct gfs2_jdesc *jd, unsigned int start,
+static inline int lops_scan_elements(struct gfs2_jdesc *jd, u32 start,
 				     struct gfs2_log_descriptor *ld,
 				     __be64 *ptr,
 				     unsigned int pass)
diff --git a/fs/gfs2/main.c b/fs/gfs2/main.c
index 136484ef35d3..c700738de1f7 100644
--- a/fs/gfs2/main.c
+++ b/fs/gfs2/main.c
@@ -59,7 +59,6 @@ static void gfs2_init_glock_once(void *foo)
 	INIT_LIST_HEAD(&gl->gl_lru);
 	INIT_LIST_HEAD(&gl->gl_ail_list);
 	atomic_set(&gl->gl_ail_count, 0);
-	atomic_set(&gl->gl_revokes, 0);
 }
 
 static void gfs2_init_gl_aspace_once(void *foo)
diff --git a/fs/gfs2/ops_fstype.c b/fs/gfs2/ops_fstype.c
index b041cb8ae383..46f6615eaf12 100644
--- a/fs/gfs2/ops_fstype.c
+++ b/fs/gfs2/ops_fstype.c
@@ -41,6 +41,7 @@
 #include "dir.h"
 #include "meta_io.h"
 #include "trace_gfs2.h"
+#include "lops.h"
 
 #define DO 0
 #define UNDO 1
@@ -117,8 +118,8 @@ static struct gfs2_sbd *init_sbd(struct super_block *sb)
 
 	spin_lock_init(&sdp->sd_log_lock);
 	atomic_set(&sdp->sd_log_pinned, 0);
-	INIT_LIST_HEAD(&sdp->sd_log_le_revoke);
-	INIT_LIST_HEAD(&sdp->sd_log_le_ordered);
+	INIT_LIST_HEAD(&sdp->sd_log_revokes);
+	INIT_LIST_HEAD(&sdp->sd_log_ordered);
 	spin_lock_init(&sdp->sd_ordered_lock);
 
 	init_waitqueue_head(&sdp->sd_log_waitq);
@@ -616,7 +617,7 @@ static int check_journal_clean(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd)
 		fs_err(sdp, "Error checking journal for spectator mount.\n");
 		goto out_unlock;
 	}
-	error = gfs2_find_jhead(jd, &head);
+	error = gfs2_find_jhead(jd, &head, false);
 	if (error) {
 		fs_err(sdp, "Error parsing journal for spectator mount.\n");
 		goto out_unlock;
diff --git a/fs/gfs2/recovery.c b/fs/gfs2/recovery.c
index 2dac43065382..389b3ef77e20 100644
--- a/fs/gfs2/recovery.c
+++ b/fs/gfs2/recovery.c
@@ -182,129 +182,6 @@ static int get_log_header(struct gfs2_jdesc *jd, unsigned int blk,
 }
 
 /**
- * find_good_lh - find a good log header
- * @jd: the journal
- * @blk: the segment to start searching from
- * @lh: the log header to fill in
- * @forward: if true search forward in the log, else search backward
- *
- * Call get_log_header() to get a log header for a segment, but if the
- * segment is bad, either scan forward or backward until we find a good one.
- *
- * Returns: errno
- */
-
-static int find_good_lh(struct gfs2_jdesc *jd, unsigned int *blk,
-			struct gfs2_log_header_host *head)
-{
-	unsigned int orig_blk = *blk;
-	int error;
-
-	for (;;) {
-		error = get_log_header(jd, *blk, head);
-		if (error <= 0)
-			return error;
-
-		if (++*blk == jd->jd_blocks)
-			*blk = 0;
-
-		if (*blk == orig_blk) {
-			gfs2_consist_inode(GFS2_I(jd->jd_inode));
-			return -EIO;
-		}
-	}
-}
-
-/**
- * jhead_scan - make sure we've found the head of the log
- * @jd: the journal
- * @head: this is filled in with the log descriptor of the head
- *
- * At this point, seg and lh should be either the head of the log or just
- * before.  Scan forward until we find the head.
- *
- * Returns: errno
- */
-
-static int jhead_scan(struct gfs2_jdesc *jd, struct gfs2_log_header_host *head)
-{
-	unsigned int blk = head->lh_blkno;
-	struct gfs2_log_header_host lh;
-	int error;
-
-	for (;;) {
-		if (++blk == jd->jd_blocks)
-			blk = 0;
-
-		error = get_log_header(jd, blk, &lh);
-		if (error < 0)
-			return error;
-		if (error == 1)
-			continue;
-
-		if (lh.lh_sequence == head->lh_sequence) {
-			gfs2_consist_inode(GFS2_I(jd->jd_inode));
-			return -EIO;
-		}
-		if (lh.lh_sequence < head->lh_sequence)
-			break;
-
-		*head = lh;
-	}
-
-	return 0;
-}
-
-/**
- * gfs2_find_jhead - find the head of a log
- * @jd: the journal
- * @head: the log descriptor for the head of the log is returned here
- *
- * Do a binary search of a journal and find the valid log entry with the
- * highest sequence number.  (i.e. the log head)
- *
- * Returns: errno
- */
-
-int gfs2_find_jhead(struct gfs2_jdesc *jd, struct gfs2_log_header_host *head)
-{
-	struct gfs2_log_header_host lh_1, lh_m;
-	u32 blk_1, blk_2, blk_m;
-	int error;
-
-	blk_1 = 0;
-	blk_2 = jd->jd_blocks - 1;
-
-	for (;;) {
-		blk_m = (blk_1 + blk_2) / 2;
-
-		error = find_good_lh(jd, &blk_1, &lh_1);
-		if (error)
-			return error;
-
-		error = find_good_lh(jd, &blk_m, &lh_m);
-		if (error)
-			return error;
-
-		if (blk_1 == blk_m || blk_m == blk_2)
-			break;
-
-		if (lh_1.lh_sequence <= lh_m.lh_sequence)
-			blk_1 = blk_m;
-		else
-			blk_2 = blk_m;
-	}
-
-	error = jhead_scan(jd, &lh_1);
-	if (error)
-		return error;
-
-	*head = lh_1;
-
-	return error;
-}
-
-/**
  * foreach_descriptor - go through the active part of the log
  * @jd: the journal
  * @start: the first log header in the active region
@@ -316,7 +193,7 @@ int gfs2_find_jhead(struct gfs2_jdesc *jd, struct gfs2_log_header_host *head)
  * Returns: errno
  */
 
-static int foreach_descriptor(struct gfs2_jdesc *jd, unsigned int start,
+static int foreach_descriptor(struct gfs2_jdesc *jd, u32 start,
 			      unsigned int end, int pass)
 {
 	struct gfs2_sbd *sdp = GFS2_SB(jd->jd_inode);
@@ -386,10 +263,12 @@ static void clean_journal(struct gfs2_jdesc *jd,
 			  struct gfs2_log_header_host *head)
 {
 	struct gfs2_sbd *sdp = GFS2_SB(jd->jd_inode);
+	u32 lblock = head->lh_blkno;
 
-	sdp->sd_log_flush_head = head->lh_blkno;
-	gfs2_replay_incr_blk(jd, &sdp->sd_log_flush_head);
-	gfs2_write_log_header(sdp, jd, head->lh_sequence + 1, 0,
+	gfs2_replay_incr_blk(jd, &lblock);
+	if (jd->jd_jid == sdp->sd_lockstruct.ls_jid)
+		sdp->sd_log_flush_head = lblock;
+	gfs2_write_log_header(sdp, jd, head->lh_sequence + 1, 0, lblock,
 			      GFS2_LOG_HEAD_UNMOUNT | GFS2_LOG_HEAD_RECOVERY,
 			      REQ_PREFLUSH | REQ_FUA | REQ_META | REQ_SYNC);
 }
@@ -467,7 +346,7 @@ void gfs2_recover_func(struct work_struct *work)
 	if (error)
 		goto fail_gunlock_ji;
 
-	error = gfs2_find_jhead(jd, &head);
+	error = gfs2_find_jhead(jd, &head, true);
 	if (error)
 		goto fail_gunlock_ji;
 	t_jhd = ktime_get();
diff --git a/fs/gfs2/recovery.h b/fs/gfs2/recovery.h
index 11d81248be85..1831a1974c8c 100644
--- a/fs/gfs2/recovery.h
+++ b/fs/gfs2/recovery.h
@@ -14,7 +14,7 @@
 
 extern struct workqueue_struct *gfs_recovery_wq;
 
-static inline void gfs2_replay_incr_blk(struct gfs2_jdesc *jd, unsigned int *blk)
+static inline void gfs2_replay_incr_blk(struct gfs2_jdesc *jd, u32 *blk)
 {
 	if (++*blk == jd->jd_blocks)
 	        *blk = 0;
@@ -27,8 +27,6 @@ extern int gfs2_revoke_add(struct gfs2_jdesc *jd, u64 blkno, unsigned int where)
 extern int gfs2_revoke_check(struct gfs2_jdesc *jd, u64 blkno, unsigned int where);
 extern void gfs2_revoke_clean(struct gfs2_jdesc *jd);
 
-extern int gfs2_find_jhead(struct gfs2_jdesc *jd,
-		    struct gfs2_log_header_host *head);
 extern int gfs2_recover_journal(struct gfs2_jdesc *gfs2_jd, bool wait);
 extern void gfs2_recover_func(struct work_struct *work);
 extern int __get_log_header(struct gfs2_sbd *sdp,
diff --git a/fs/gfs2/rgrp.c b/fs/gfs2/rgrp.c
index 17a8d3b43990..15d6e32de55f 100644
--- a/fs/gfs2/rgrp.c
+++ b/fs/gfs2/rgrp.c
@@ -1729,25 +1729,22 @@ fail:
 static int gfs2_rbm_find(struct gfs2_rbm *rbm, u8 state, u32 *minext,
 			 const struct gfs2_inode *ip, bool nowrap)
 {
+	bool scan_from_start = rbm->bii == 0 && rbm->offset == 0;
 	struct buffer_head *bh;
-	int initial_bii;
-	u32 initial_offset;
-	int first_bii = rbm->bii;
-	u32 first_offset = rbm->offset;
+	int last_bii;
 	u32 offset;
 	u8 *buffer;
-	int n = 0;
-	int iters = rbm->rgd->rd_length;
+	bool wrapped = false;
 	int ret;
 	struct gfs2_bitmap *bi;
 	struct gfs2_extent maxext = { .rbm.rgd = rbm->rgd, };
 
-	/* If we are not starting at the beginning of a bitmap, then we
-	 * need to add one to the bitmap count to ensure that we search
-	 * the starting bitmap twice.
+	/*
+	 * Determine the last bitmap to search.  If we're not starting at the
+	 * beginning of a bitmap, we need to search that bitmap twice to scan
+	 * the entire resource group.
 	 */
-	if (rbm->offset != 0)
-		iters++;
+	last_bii = rbm->bii - (rbm->offset == 0);
 
 	while(1) {
 		bi = rbm_bi(rbm);
@@ -1761,47 +1758,46 @@ static int gfs2_rbm_find(struct gfs2_rbm *rbm, u8 state, u32 *minext,
 		WARN_ON(!buffer_uptodate(bh));
 		if (state != GFS2_BLKST_UNLINKED && bi->bi_clone)
 			buffer = bi->bi_clone + bi->bi_offset;
-		initial_offset = rbm->offset;
 		offset = gfs2_bitfit(buffer, bi->bi_bytes, rbm->offset, state);
-		if (offset == BFITNOENT)
-			goto bitmap_full;
+		if (offset == BFITNOENT) {
+			if (state == GFS2_BLKST_FREE && rbm->offset == 0)
+				set_bit(GBF_FULL, &bi->bi_flags);
+			goto next_bitmap;
+		}
 		rbm->offset = offset;
 		if (ip == NULL)
 			return 0;
 
-		initial_bii = rbm->bii;
 		ret = gfs2_reservation_check_and_update(rbm, ip,
 							minext ? *minext : 0,
 							&maxext);
 		if (ret == 0)
 			return 0;
-		if (ret > 0) {
-			n += (rbm->bii - initial_bii);
+		if (ret > 0)
 			goto next_iter;
-		}
 		if (ret == -E2BIG) {
 			rbm->bii = 0;
 			rbm->offset = 0;
-			n += (rbm->bii - initial_bii);
 			goto res_covered_end_of_rgrp;
 		}
 		return ret;
 
-bitmap_full:	/* Mark bitmap as full and fall through */
-		if ((state == GFS2_BLKST_FREE) && initial_offset == 0)
-			set_bit(GBF_FULL, &bi->bi_flags);
-
 next_bitmap:	/* Find next bitmap in the rgrp */
 		rbm->offset = 0;
 		rbm->bii++;
 		if (rbm->bii == rbm->rgd->rd_length)
 			rbm->bii = 0;
 res_covered_end_of_rgrp:
-		if ((rbm->bii == 0) && nowrap)
-			break;
-		n++;
+		if (rbm->bii == 0) {
+			if (wrapped)
+				break;
+			wrapped = true;
+			if (nowrap)
+				break;
+		}
 next_iter:
-		if (n >= iters)
+		/* Have we scanned the entire resource group? */
+		if (wrapped && rbm->bii > last_bii)
 			break;
 	}
 
@@ -1811,8 +1807,8 @@ next_iter:
 	/* If the extent was too small, and it's smaller than the smallest
 	   to have failed before, remember for future reference that it's
 	   useless to search this rgrp again for this amount or more. */
-	if ((first_offset == 0) && (first_bii == 0) &&
-	    (*minext < rbm->rgd->rd_extfail_pt))
+	if (wrapped && (scan_from_start || rbm->bii > last_bii) &&
+	    *minext < rbm->rgd->rd_extfail_pt)
 		rbm->rgd->rd_extfail_pt = *minext;
 
 	/* If the maximum extent we found is big enough to fulfill the
@@ -2444,7 +2440,7 @@ int gfs2_alloc_blocks(struct gfs2_inode *ip, u64 *bn, unsigned int *nblocks,
 
 	gfs2_statfs_change(sdp, 0, -(s64)*nblocks, dinode ? 1 : 0);
 	if (dinode)
-		gfs2_trans_add_unrevoke(sdp, block, *nblocks);
+		gfs2_trans_remove_revoke(sdp, block, *nblocks);
 
 	gfs2_quota_change(ip, *nblocks, ip->i_inode.i_uid, ip->i_inode.i_gid);
 
diff --git a/fs/gfs2/super.c b/fs/gfs2/super.c
index 7b8d2306b3d3..fbf6b1fd330b 100644
--- a/fs/gfs2/super.c
+++ b/fs/gfs2/super.c
@@ -45,6 +45,7 @@
 #include "util.h"
 #include "sys.h"
 #include "xattr.h"
+#include "lops.h"
 
 #define args_neq(a1, a2, x) ((a1)->ar_##x != (a2)->ar_##x)
 
@@ -425,7 +426,7 @@ int gfs2_make_fs_rw(struct gfs2_sbd *sdp)
 
 	j_gl->gl_ops->go_inval(j_gl, DIO_METADATA);
 
-	error = gfs2_find_jhead(sdp->sd_jdesc, &head);
+	error = gfs2_find_jhead(sdp->sd_jdesc, &head, false);
 	if (error)
 		goto fail;
 
@@ -680,7 +681,7 @@ static int gfs2_lock_fs_check_clean(struct gfs2_sbd *sdp,
 		error = gfs2_jdesc_check(jd);
 		if (error)
 			break;
-		error = gfs2_find_jhead(jd, &lh);
+		error = gfs2_find_jhead(jd, &lh, false);
 		if (error)
 			break;
 		if (!(lh.lh_flags & GFS2_LOG_HEAD_UNMOUNT)) {
@@ -973,8 +974,7 @@ void gfs2_freeze_func(struct work_struct *work)
 	if (error) {
 		printk(KERN_INFO "GFS2: couldn't get freeze lock : %d\n", error);
 		gfs2_assert_withdraw(sdp, 0);
-	}
-	else {
+	} else {
 		atomic_set(&sdp->sd_freeze_state, SFS_UNFROZEN);
 		error = thaw_super(sb);
 		if (error) {
@@ -987,6 +987,8 @@ void gfs2_freeze_func(struct work_struct *work)
 		gfs2_glock_dq_uninit(&freeze_gh);
 	}
 	deactivate_super(sb);
+	clear_bit_unlock(SDF_FS_FROZEN, &sdp->sd_flags);
+	wake_up_bit(&sdp->sd_flags, SDF_FS_FROZEN);
 	return;
 }
 
@@ -1029,6 +1031,7 @@ static int gfs2_freeze(struct super_block *sb)
 		msleep(1000);
 	}
 	error = 0;
+	set_bit(SDF_FS_FROZEN, &sdp->sd_flags);
 out:
 	mutex_unlock(&sdp->sd_freeze_mutex);
 	return error;
@@ -1053,7 +1056,7 @@ static int gfs2_unfreeze(struct super_block *sb)
 
 	gfs2_glock_dq_uninit(&sdp->sd_freeze_gh);
 	mutex_unlock(&sdp->sd_freeze_mutex);
-	return 0;
+	return wait_on_bit(&sdp->sd_flags, SDF_FS_FROZEN, TASK_INTERRUPTIBLE);
 }
 
 /**
@@ -1474,7 +1477,7 @@ static void gfs2_final_release_pages(struct gfs2_inode *ip)
 	truncate_inode_pages(gfs2_glock2aspace(ip->i_gl), 0);
 	truncate_inode_pages(&inode->i_data, 0);
 
-	if (atomic_read(&gl->gl_revokes) == 0) {
+	if (!test_bit(GLF_REVOKES, &gl->gl_flags)) {
 		clear_bit(GLF_LFLUSH, &gl->gl_flags);
 		clear_bit(GLF_DIRTY, &gl->gl_flags);
 	}
@@ -1630,8 +1633,6 @@ alloc_failed:
 			goto out_truncate;
 	}
 
-	/* Case 1 starts here */
-
 	if (S_ISDIR(inode->i_mode) &&
 	    (ip->i_diskflags & GFS2_DIF_EXHASH)) {
 		error = gfs2_dir_exhash_dealloc(ip);
@@ -1670,7 +1671,6 @@ out_truncate:
 	write_inode_now(inode, 1);
 	gfs2_ail_flush(ip->i_gl, 0);
 
-	/* Case 2 starts here */
 	error = gfs2_trans_begin(sdp, 0, sdp->sd_jdesc->jd_blocks);
 	if (error)
 		goto out_unlock;
@@ -1680,7 +1680,6 @@ out_truncate:
 	gfs2_trans_end(sdp);
 
 out_unlock:
-	/* Error path for case 1 */
 	if (gfs2_rs_active(&ip->i_res))
 		gfs2_rs_deltree(&ip->i_res);
 
@@ -1699,7 +1698,6 @@ out_unlock:
 	if (error && error != GLR_TRYFAILED && error != -EROFS)
 		fs_warn(sdp, "gfs2_evict_inode: %d\n", error);
 out:
-	/* Case 3 starts here */
 	truncate_inode_pages_final(&inode->i_data);
 	gfs2_rsqa_delete(ip, NULL);
 	gfs2_ordered_del_inode(ip);
diff --git a/fs/gfs2/trans.c b/fs/gfs2/trans.c
index cd9a94a6b5bb..d328da7cde36 100644
--- a/fs/gfs2/trans.c
+++ b/fs/gfs2/trans.c
@@ -253,14 +253,14 @@ void gfs2_trans_add_revoke(struct gfs2_sbd *sdp, struct gfs2_bufdata *bd)
 	tr->tr_num_revoke++;
 }
 
-void gfs2_trans_add_unrevoke(struct gfs2_sbd *sdp, u64 blkno, unsigned int len)
+void gfs2_trans_remove_revoke(struct gfs2_sbd *sdp, u64 blkno, unsigned int len)
 {
 	struct gfs2_bufdata *bd, *tmp;
 	struct gfs2_trans *tr = current->journal_info;
 	unsigned int n = len;
 
 	gfs2_log_lock(sdp);
-	list_for_each_entry_safe(bd, tmp, &sdp->sd_log_le_revoke, bd_list) {
+	list_for_each_entry_safe(bd, tmp, &sdp->sd_log_revokes, bd_list) {
 		if ((bd->bd_blkno >= blkno) && (bd->bd_blkno < (blkno + len))) {
 			list_del_init(&bd->bd_list);
 			gfs2_assert_withdraw(sdp, sdp->sd_log_num_revoke);
diff --git a/fs/gfs2/trans.h b/fs/gfs2/trans.h
index ad70087d0597..1e39f056ccb7 100644
--- a/fs/gfs2/trans.h
+++ b/fs/gfs2/trans.h
@@ -44,6 +44,6 @@ extern void gfs2_trans_end(struct gfs2_sbd *sdp);
 extern void gfs2_trans_add_data(struct gfs2_glock *gl, struct buffer_head *bh);
 extern void gfs2_trans_add_meta(struct gfs2_glock *gl, struct buffer_head *bh);
 extern void gfs2_trans_add_revoke(struct gfs2_sbd *sdp, struct gfs2_bufdata *bd);
-extern void gfs2_trans_add_unrevoke(struct gfs2_sbd *sdp, u64 blkno, unsigned int len);
+extern void gfs2_trans_remove_revoke(struct gfs2_sbd *sdp, u64 blkno, unsigned int len);
 
 #endif /* __TRANS_DOT_H__ */
diff --git a/fs/gfs2/xattr.c b/fs/gfs2/xattr.c
index 996c915a9c97..675e704830df 100644
--- a/fs/gfs2/xattr.c
+++ b/fs/gfs2/xattr.c
@@ -631,7 +631,7 @@ static int ea_alloc_blk(struct gfs2_inode *ip, struct buffer_head **bhp)
 	error = gfs2_alloc_blocks(ip, &block, &n, 0, NULL);
 	if (error)
 		return error;
-	gfs2_trans_add_unrevoke(sdp, block, 1);
+	gfs2_trans_remove_revoke(sdp, block, 1);
 	*bhp = gfs2_meta_new(ip->i_gl, block);
 	gfs2_trans_add_meta(ip->i_gl, *bhp);
 	gfs2_metatype_set(*bhp, GFS2_METATYPE_EA, GFS2_FORMAT_EA);
@@ -693,7 +693,7 @@ static int ea_write(struct gfs2_inode *ip, struct gfs2_ea_header *ea,
 			error = gfs2_alloc_blocks(ip, &block, &n, 0, NULL);
 			if (error)
 				return error;
-			gfs2_trans_add_unrevoke(sdp, block, 1);
+			gfs2_trans_remove_revoke(sdp, block, 1);
 			bh = gfs2_meta_new(ip->i_gl, block);
 			gfs2_trans_add_meta(ip->i_gl, bh);
 			gfs2_metatype_set(bh, GFS2_METATYPE_ED, GFS2_FORMAT_ED);
@@ -997,7 +997,7 @@ static int ea_set_block(struct gfs2_inode *ip, struct gfs2_ea_request *er,
 		error = gfs2_alloc_blocks(ip, &blk, &n, 0, NULL);
 		if (error)
 			return error;
-		gfs2_trans_add_unrevoke(sdp, blk, 1);
+		gfs2_trans_remove_revoke(sdp, blk, 1);
 		indbh = gfs2_meta_new(ip->i_gl, blk);
 		gfs2_trans_add_meta(ip->i_gl, indbh);
 		gfs2_metatype_set(indbh, GFS2_METATYPE_IN, GFS2_FORMAT_IN);
diff --git a/fs/lockd/clntlock.c b/fs/lockd/clntlock.c
index c2a128678e6e..70f520b41a19 100644
--- a/fs/lockd/clntlock.c
+++ b/fs/lockd/clntlock.c
@@ -63,7 +63,7 @@ struct nlm_host *nlmclnt_init(const struct nlmclnt_initdata *nlm_init)
 	host = nlmclnt_lookup_host(nlm_init->address, nlm_init->addrlen,
 				   nlm_init->protocol, nlm_version,
 				   nlm_init->hostname, nlm_init->noresvport,
-				   nlm_init->net);
+				   nlm_init->net, nlm_init->cred);
 	if (host == NULL)
 		goto out_nohost;
 	if (host->h_rpcclnt == NULL && nlm_bind_host(host) == NULL)
diff --git a/fs/lockd/clntproc.c b/fs/lockd/clntproc.c
index e8a004097d18..d9c32d1a20c0 100644
--- a/fs/lockd/clntproc.c
+++ b/fs/lockd/clntproc.c
@@ -715,7 +715,7 @@ static void nlmclnt_unlock_callback(struct rpc_task *task, void *data)
 	struct nlm_rqst	*req = data;
 	u32 status = ntohl(req->a_res.status);
 
-	if (RPC_ASSASSINATED(task))
+	if (RPC_SIGNALLED(task))
 		goto die;
 
 	if (task->tk_status < 0) {
@@ -783,7 +783,7 @@ static void nlmclnt_cancel_callback(struct rpc_task *task, void *data)
 	struct nlm_rqst	*req = data;
 	u32 status = ntohl(req->a_res.status);
 
-	if (RPC_ASSASSINATED(task))
+	if (RPC_SIGNALLED(task))
 		goto die;
 
 	if (task->tk_status < 0) {
diff --git a/fs/lockd/host.c b/fs/lockd/host.c
index f0b5c987d6ae..7d46fafdbbe5 100644
--- a/fs/lockd/host.c
+++ b/fs/lockd/host.c
@@ -60,6 +60,7 @@ struct nlm_lookup_host_info {
 	const size_t		hostname_len;	/* it's length */
 	const int		noresvport;	/* use non-priv port */
 	struct net		*net;		/* network namespace to bind */
+	const struct cred	*cred;
 };
 
 /*
@@ -162,6 +163,7 @@ static struct nlm_host *nlm_alloc_host(struct nlm_lookup_host_info *ni,
 	host->h_nsmhandle  = nsm;
 	host->h_addrbuf    = nsm->sm_addrbuf;
 	host->net	   = ni->net;
+	host->h_cred	   = get_cred(ni->cred),
 	strlcpy(host->nodename, utsname()->nodename, sizeof(host->nodename));
 
 out:
@@ -188,6 +190,7 @@ static void nlm_destroy_host_locked(struct nlm_host *host)
 	clnt = host->h_rpcclnt;
 	if (clnt != NULL)
 		rpc_shutdown_client(clnt);
+	put_cred(host->h_cred);
 	kfree(host);
 
 	ln->nrhosts--;
@@ -202,6 +205,8 @@ static void nlm_destroy_host_locked(struct nlm_host *host)
  * @version: NLM protocol version
  * @hostname: '\0'-terminated hostname of server
  * @noresvport: 1 if non-privileged port should be used
+ * @net: pointer to net namespace
+ * @cred: pointer to cred
  *
  * Returns an nlm_host structure that matches the passed-in
  * [server address, transport protocol, NLM version, server hostname].
@@ -214,7 +219,8 @@ struct nlm_host *nlmclnt_lookup_host(const struct sockaddr *sap,
 				     const u32 version,
 				     const char *hostname,
 				     int noresvport,
-				     struct net *net)
+				     struct net *net,
+				     const struct cred *cred)
 {
 	struct nlm_lookup_host_info ni = {
 		.server		= 0,
@@ -226,6 +232,7 @@ struct nlm_host *nlmclnt_lookup_host(const struct sockaddr *sap,
 		.hostname_len	= strlen(hostname),
 		.noresvport	= noresvport,
 		.net		= net,
+		.cred		= cred,
 	};
 	struct hlist_head *chain;
 	struct nlm_host	*host;
@@ -458,6 +465,7 @@ nlm_bind_host(struct nlm_host *host)
 			.authflavor	= RPC_AUTH_UNIX,
 			.flags		= (RPC_CLNT_CREATE_NOPING |
 					   RPC_CLNT_CREATE_AUTOBIND),
+			.cred		= host->h_cred,
 		};
 
 		/*
diff --git a/fs/lockd/mon.c b/fs/lockd/mon.c
index 654594ef4f94..1eabd91870e6 100644
--- a/fs/lockd/mon.c
+++ b/fs/lockd/mon.c
@@ -82,6 +82,7 @@ static struct rpc_clnt *nsm_create(struct net *net, const char *nodename)
 		.version		= NSM_VERSION,
 		.authflavor		= RPC_AUTH_NULL,
 		.flags			= RPC_CLNT_CREATE_NOPING,
+		.cred			= current_cred(),
 	};
 
 	return rpc_create(&args);
diff --git a/fs/nfs/client.c b/fs/nfs/client.c
index 90d71fda65ce..da74c4c4a244 100644
--- a/fs/nfs/client.c
+++ b/fs/nfs/client.c
@@ -284,6 +284,7 @@ static struct nfs_client *nfs_match_client(const struct nfs_client_initdata *dat
 	struct nfs_client *clp;
 	const struct sockaddr *sap = data->addr;
 	struct nfs_net *nn = net_generic(data->net, nfs_net_id);
+	int error;
 
 again:
 	list_for_each_entry(clp, &nn->nfs_client_list, cl_share_link) {
@@ -296,9 +297,11 @@ again:
 		if (clp->cl_cons_state > NFS_CS_READY) {
 			refcount_inc(&clp->cl_count);
 			spin_unlock(&nn->nfs_client_lock);
-			nfs_wait_client_init_complete(clp);
+			error = nfs_wait_client_init_complete(clp);
 			nfs_put_client(clp);
 			spin_lock(&nn->nfs_client_lock);
+			if (error < 0)
+				return ERR_PTR(error);
 			goto again;
 		}
 
@@ -407,6 +410,8 @@ struct nfs_client *nfs_get_client(const struct nfs_client_initdata *cl_init)
 		clp = nfs_match_client(cl_init);
 		if (clp) {
 			spin_unlock(&nn->nfs_client_lock);
+			if (IS_ERR(clp))
+				return clp;
 			if (new)
 				new->rpc_ops->free_client(new);
 			return nfs_found_client(cl_init, clp);
@@ -500,6 +505,7 @@ int nfs_create_rpc_client(struct nfs_client *clp,
 		.program	= &nfs_program,
 		.version	= clp->rpc_ops->version,
 		.authflavor	= flavor,
+		.cred		= cl_init->cred,
 	};
 
 	if (test_bit(NFS_CS_DISCRTRY, &clp->cl_flags))
@@ -598,6 +604,8 @@ int nfs_init_server_rpcclient(struct nfs_server *server,
 			sizeof(server->client->cl_timeout_default));
 	server->client->cl_timeout = &server->client->cl_timeout_default;
 	server->client->cl_softrtry = 0;
+	if (server->flags & NFS_MOUNT_SOFTERR)
+		server->client->cl_softerr = 1;
 	if (server->flags & NFS_MOUNT_SOFT)
 		server->client->cl_softrtry = 1;
 
@@ -652,6 +660,7 @@ static int nfs_init_server(struct nfs_server *server,
 		.proto = data->nfs_server.protocol,
 		.net = data->net,
 		.timeparms = &timeparms,
+		.cred = server->cred,
 	};
 	struct nfs_client *clp;
 	int error;
@@ -920,6 +929,7 @@ void nfs_free_server(struct nfs_server *server)
 	ida_destroy(&server->lockowner_id);
 	ida_destroy(&server->openowner_id);
 	nfs_free_iostats(server->io_stats);
+	put_cred(server->cred);
 	kfree(server);
 	nfs_release_automount_timer();
 }
@@ -940,6 +950,8 @@ struct nfs_server *nfs_create_server(struct nfs_mount_info *mount_info,
 	if (!server)
 		return ERR_PTR(-ENOMEM);
 
+	server->cred = get_cred(current_cred());
+
 	error = -ENOMEM;
 	fattr = nfs_alloc_fattr();
 	if (fattr == NULL)
@@ -1006,6 +1018,8 @@ struct nfs_server *nfs_clone_server(struct nfs_server *source,
 	if (!server)
 		return ERR_PTR(-ENOMEM);
 
+	server->cred = get_cred(source->cred);
+
 	error = -ENOMEM;
 	fattr_fsinfo = nfs_alloc_fattr();
 	if (fattr_fsinfo == NULL)
diff --git a/fs/nfs/delegation.c b/fs/nfs/delegation.c
index 2f6b447cdd82..8b78274e3e56 100644
--- a/fs/nfs/delegation.c
+++ b/fs/nfs/delegation.c
@@ -1034,6 +1034,18 @@ void nfs_mark_test_expired_all_delegations(struct nfs_client *clp)
 }
 
 /**
+ * nfs_test_expired_all_delegations - test all delegations for a client
+ * @clp: nfs_client to process
+ *
+ * Helper for handling "recallable state revoked" status from server.
+ */
+void nfs_test_expired_all_delegations(struct nfs_client *clp)
+{
+	nfs_mark_test_expired_all_delegations(clp);
+	nfs4_schedule_state_manager(clp);
+}
+
+/**
  * nfs_reap_expired_delegations - reap expired delegations
  * @clp: nfs_client to process
  *
diff --git a/fs/nfs/delegation.h b/fs/nfs/delegation.h
index 35b4b02c1ae0..5799777df5ec 100644
--- a/fs/nfs/delegation.h
+++ b/fs/nfs/delegation.h
@@ -58,6 +58,7 @@ void nfs_delegation_mark_reclaim(struct nfs_client *clp);
 void nfs_delegation_reap_unclaimed(struct nfs_client *clp);
 
 void nfs_mark_test_expired_all_delegations(struct nfs_client *clp);
+void nfs_test_expired_all_delegations(struct nfs_client *clp);
 void nfs_reap_expired_delegations(struct nfs_client *clp);
 
 /* NFSv4 delegation-related procedures */
diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c
index a71d0b42d160..47d445bec8c9 100644
--- a/fs/nfs/dir.c
+++ b/fs/nfs/dir.c
@@ -714,8 +714,9 @@ out:
  * We only need to convert from xdr once so future lookups are much simpler
  */
 static
-int nfs_readdir_filler(nfs_readdir_descriptor_t *desc, struct page* page)
+int nfs_readdir_filler(void *data, struct page* page)
 {
+	nfs_readdir_descriptor_t *desc = data;
 	struct inode	*inode = file_inode(desc->file);
 	int ret;
 
@@ -762,8 +763,8 @@ void cache_page_release(nfs_readdir_descriptor_t *desc)
 static
 struct page *get_cache_page(nfs_readdir_descriptor_t *desc)
 {
-	return read_cache_page(desc->file->f_mapping,
-			desc->page_index, (filler_t *)nfs_readdir_filler, desc);
+	return read_cache_page(desc->file->f_mapping, desc->page_index,
+			nfs_readdir_filler, desc);
 }
 
 /*
diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c
index 0fd811ac08b5..2436bd92bc00 100644
--- a/fs/nfs/direct.c
+++ b/fs/nfs/direct.c
@@ -492,7 +492,7 @@ static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq,
 			struct nfs_page *req;
 			unsigned int req_len = min_t(size_t, bytes, PAGE_SIZE - pgbase);
 			/* XXX do we need to do the eof zeroing found in async_filler? */
-			req = nfs_create_request(dreq->ctx, pagevec[i], NULL,
+			req = nfs_create_request(dreq->ctx, pagevec[i],
 						 pgbase, req_len);
 			if (IS_ERR(req)) {
 				result = PTR_ERR(req);
@@ -663,6 +663,8 @@ static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq)
 	}
 
 	list_for_each_entry_safe(req, tmp, &reqs, wb_list) {
+		/* Bump the transmission count */
+		req->wb_nio++;
 		if (!nfs_pageio_add_request(&desc, req)) {
 			nfs_list_move_request(req, &failed);
 			spin_lock(&cinfo.inode->i_lock);
@@ -703,6 +705,11 @@ static void nfs_direct_commit_complete(struct nfs_commit_data *data)
 		req = nfs_list_entry(data->pages.next);
 		nfs_list_remove_request(req);
 		if (dreq->flags == NFS_ODIRECT_RESCHED_WRITES) {
+			/*
+			 * Despite the reboot, the write was successful,
+			 * so reset wb_nio.
+			 */
+			req->wb_nio = 0;
 			/* Note the rewrite will go through mds */
 			nfs_mark_request_commit(req, NULL, &cinfo, 0);
 		} else
@@ -899,7 +906,7 @@ static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq,
 			struct nfs_page *req;
 			unsigned int req_len = min_t(size_t, bytes, PAGE_SIZE - pgbase);
 
-			req = nfs_create_request(dreq->ctx, pagevec[i], NULL,
+			req = nfs_create_request(dreq->ctx, pagevec[i],
 						 pgbase, req_len);
 			if (IS_ERR(req)) {
 				result = PTR_ERR(req);
diff --git a/fs/nfs/file.c b/fs/nfs/file.c
index 4899b85f9b3c..144e183250c3 100644
--- a/fs/nfs/file.c
+++ b/fs/nfs/file.c
@@ -147,7 +147,7 @@ nfs_file_flush(struct file *file, fl_owner_t id)
 		return 0;
 
 	/* Flush writes to the server and return any errors */
-	return vfs_fsync(file, 0);
+	return nfs_wb_all(inode);
 }
 
 ssize_t
@@ -199,13 +199,6 @@ EXPORT_SYMBOL_GPL(nfs_file_mmap);
  * Flush any dirty pages for this process, and check for write errors.
  * The return status from this call provides a reliable indication of
  * whether any write errors occurred for this process.
- *
- * Notice that it clears the NFS_CONTEXT_ERROR_WRITE before synching to
- * disk, but it retrieves and clears ctx->error after synching, despite
- * the two being set at the same time in nfs_context_set_write_error().
- * This is because the former is used to notify the _next_ call to
- * nfs_file_write() that a write error occurred, and hence cause it to
- * fall back to doing a synchronous write.
  */
 static int
 nfs_file_fsync_commit(struct file *file, int datasync)
@@ -220,11 +213,8 @@ nfs_file_fsync_commit(struct file *file, int datasync)
 	nfs_inc_stats(inode, NFSIOS_VFSFSYNC);
 	do_resend = test_and_clear_bit(NFS_CONTEXT_RESEND_WRITES, &ctx->flags);
 	status = nfs_commit_inode(inode, FLUSH_SYNC);
-	if (test_bit(NFS_CONTEXT_ERROR_WRITE, &ctx->flags)) {
-		ret = xchg(&ctx->error, 0);
-		if (ret)
-			goto out;
-	}
+	if (status == 0)
+		status = file_check_and_advance_wb_err(file);
 	if (status < 0) {
 		ret = status;
 		goto out;
@@ -245,13 +235,7 @@ nfs_file_fsync(struct file *file, loff_t start, loff_t end, int datasync)
 	trace_nfs_fsync_enter(inode);
 
 	do {
-		struct nfs_open_context *ctx = nfs_file_open_context(file);
-		ret = filemap_write_and_wait_range(inode->i_mapping, start, end);
-		if (test_and_clear_bit(NFS_CONTEXT_ERROR_WRITE, &ctx->flags)) {
-			int ret2 = xchg(&ctx->error, 0);
-			if (ret2)
-				ret = ret2;
-		}
+		ret = file_write_and_wait_range(file, start, end);
 		if (ret != 0)
 			break;
 		ret = nfs_file_fsync_commit(file, datasync);
@@ -600,8 +584,7 @@ static int nfs_need_check_write(struct file *filp, struct inode *inode)
 	struct nfs_open_context *ctx;
 
 	ctx = nfs_file_open_context(filp);
-	if (test_bit(NFS_CONTEXT_ERROR_WRITE, &ctx->flags) ||
-	    nfs_ctx_key_to_expire(ctx, inode))
+	if (nfs_ctx_key_to_expire(ctx, inode))
 		return 1;
 	return 0;
 }
@@ -655,7 +638,7 @@ ssize_t nfs_file_write(struct kiocb *iocb, struct iov_iter *from)
 
 	/* Return error values */
 	if (nfs_need_check_write(file, inode)) {
-		int err = vfs_fsync(file, 0);
+		int err = nfs_wb_all(inode);
 		if (err < 0)
 			result = err;
 	}
@@ -709,7 +692,7 @@ do_unlk(struct file *filp, int cmd, struct file_lock *fl, int is_local)
 	 * Flush all pending writes before doing anything
 	 * with locks..
 	 */
-	vfs_fsync(filp, 0);
+	nfs_wb_all(inode);
 
 	l_ctx = nfs_get_lock_context(nfs_file_open_context(filp));
 	if (!IS_ERR(l_ctx)) {
diff --git a/fs/nfs/filelayout/filelayout.c b/fs/nfs/filelayout/filelayout.c
index 61f46facb39c..3cb073c50fa6 100644
--- a/fs/nfs/filelayout/filelayout.c
+++ b/fs/nfs/filelayout/filelayout.c
@@ -904,7 +904,7 @@ fl_pnfs_update_layout(struct inode *ino,
 	status = filelayout_check_deviceid(lo, fl, gfp_flags);
 	if (status) {
 		pnfs_put_lseg(lseg);
-		lseg = ERR_PTR(status);
+		lseg = NULL;
 	}
 out:
 	return lseg;
@@ -917,7 +917,7 @@ filelayout_pg_init_read(struct nfs_pageio_descriptor *pgio,
 	pnfs_generic_pg_check_layout(pgio);
 	if (!pgio->pg_lseg) {
 		pgio->pg_lseg = fl_pnfs_update_layout(pgio->pg_inode,
-						      req->wb_context,
+						      nfs_req_openctx(req),
 						      0,
 						      NFS4_MAX_UINT64,
 						      IOMODE_READ,
@@ -944,7 +944,7 @@ filelayout_pg_init_write(struct nfs_pageio_descriptor *pgio,
 	pnfs_generic_pg_check_layout(pgio);
 	if (!pgio->pg_lseg) {
 		pgio->pg_lseg = fl_pnfs_update_layout(pgio->pg_inode,
-						      req->wb_context,
+						      nfs_req_openctx(req),
 						      0,
 						      NFS4_MAX_UINT64,
 						      IOMODE_RW,
diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c
index 6673d4ff5a2a..9920c52bd0cd 100644
--- a/fs/nfs/flexfilelayout/flexfilelayout.c
+++ b/fs/nfs/flexfilelayout/flexfilelayout.c
@@ -28,6 +28,8 @@
 #define FF_LAYOUT_POLL_RETRY_MAX     (15*HZ)
 #define FF_LAYOUTRETURN_MAXERR 20
 
+static unsigned short io_maxretrans;
+
 static void ff_layout_read_record_layoutstats_done(struct rpc_task *task,
 		struct nfs_pgio_header *hdr);
 static int ff_layout_mirror_prepare_stats(struct pnfs_layout_hdr *lo,
@@ -871,7 +873,7 @@ ff_layout_pg_get_read(struct nfs_pageio_descriptor *pgio,
 {
 	pnfs_put_lseg(pgio->pg_lseg);
 	pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
-					   req->wb_context,
+					   nfs_req_openctx(req),
 					   0,
 					   NFS4_MAX_UINT64,
 					   IOMODE_READ,
@@ -925,6 +927,7 @@ retry:
 	pgm = &pgio->pg_mirrors[0];
 	pgm->pg_bsize = mirror->mirror_ds->ds_versions[0].rsize;
 
+	pgio->pg_maxretrans = io_maxretrans;
 	return;
 out_nolseg:
 	if (pgio->pg_error < 0)
@@ -950,7 +953,7 @@ retry:
 	pnfs_generic_pg_check_layout(pgio);
 	if (!pgio->pg_lseg) {
 		pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
-						   req->wb_context,
+						   nfs_req_openctx(req),
 						   0,
 						   NFS4_MAX_UINT64,
 						   IOMODE_RW,
@@ -992,6 +995,7 @@ retry:
 		pgm->pg_bsize = mirror->mirror_ds->ds_versions[0].wsize;
 	}
 
+	pgio->pg_maxretrans = io_maxretrans;
 	return;
 
 out_mds:
@@ -1006,7 +1010,7 @@ ff_layout_pg_get_mirror_count_write(struct nfs_pageio_descriptor *pgio,
 {
 	if (!pgio->pg_lseg) {
 		pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
-						   req->wb_context,
+						   nfs_req_openctx(req),
 						   0,
 						   NFS4_MAX_UINT64,
 						   IOMODE_RW,
@@ -2515,3 +2519,7 @@ MODULE_DESCRIPTION("The NFSv4 flexfile layout driver");
 
 module_init(nfs4flexfilelayout_init);
 module_exit(nfs4flexfilelayout_exit);
+
+module_param(io_maxretrans, ushort, 0644);
+MODULE_PARM_DESC(io_maxretrans, "The  number of times the NFSv4.1 client "
+			"retries an I/O request before returning an error. ");
diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c
index f61af8307dc8..3bc2550cfe4e 100644
--- a/fs/nfs/inode.c
+++ b/fs/nfs/inode.c
@@ -885,10 +885,14 @@ struct nfs_lock_context *nfs_get_lock_context(struct nfs_open_context *ctx)
 		spin_lock(&inode->i_lock);
 		res = __nfs_find_lock_context(ctx);
 		if (res == NULL) {
-			list_add_tail_rcu(&new->list, &ctx->lock_context.list);
-			new->open_context = ctx;
-			res = new;
-			new = NULL;
+			new->open_context = get_nfs_open_context(ctx);
+			if (new->open_context) {
+				list_add_tail_rcu(&new->list,
+						&ctx->lock_context.list);
+				res = new;
+				new = NULL;
+			} else
+				res = ERR_PTR(-EBADF);
 		}
 		spin_unlock(&inode->i_lock);
 		kfree(new);
@@ -906,6 +910,7 @@ void nfs_put_lock_context(struct nfs_lock_context *l_ctx)
 		return;
 	list_del_rcu(&l_ctx->list);
 	spin_unlock(&inode->i_lock);
+	put_nfs_open_context(ctx);
 	kfree_rcu(l_ctx, rcu_head);
 }
 EXPORT_SYMBOL_GPL(nfs_put_lock_context);
diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h
index 331a0504eaf8..498fab72f70b 100644
--- a/fs/nfs/internal.h
+++ b/fs/nfs/internal.h
@@ -84,6 +84,7 @@ struct nfs_client_initdata {
 	u32 minorversion;
 	struct net *net;
 	const struct rpc_timeout *timeparms;
+	const struct cred *cred;
 };
 
 /*
@@ -766,15 +767,10 @@ static inline bool nfs_error_is_fatal(int err)
 	case -ESTALE:
 	case -E2BIG:
 	case -ENOMEM:
+	case -ETIMEDOUT:
 		return true;
 	default:
 		return false;
 	}
 }
 
-static inline void nfs_context_set_write_error(struct nfs_open_context *ctx, int error)
-{
-	ctx->error = error;
-	smp_wmb();
-	set_bit(NFS_CONTEXT_ERROR_WRITE, &ctx->flags);
-}
diff --git a/fs/nfs/mount_clnt.c b/fs/nfs/mount_clnt.c
index d979ff4fee7e..cb7c10e9721e 100644
--- a/fs/nfs/mount_clnt.c
+++ b/fs/nfs/mount_clnt.c
@@ -163,6 +163,7 @@ int nfs_mount(struct nfs_mount_request *info)
 		.program	= &mnt_program,
 		.version	= info->version,
 		.authflavor	= RPC_AUTH_UNIX,
+		.cred		= current_cred(),
 	};
 	struct rpc_clnt		*mnt_clnt;
 	int			status;
@@ -249,6 +250,7 @@ void nfs_umount(const struct nfs_mount_request *info)
 		.version	= info->version,
 		.authflavor	= RPC_AUTH_UNIX,
 		.flags		= RPC_CLNT_CREATE_NOPING,
+		.cred		= current_cred(),
 	};
 	struct rpc_message msg	= {
 		.rpc_argp	= info->dirpath,
diff --git a/fs/nfs/nfs2xdr.c b/fs/nfs/nfs2xdr.c
index a7ed29de0a40..572794dab4b1 100644
--- a/fs/nfs/nfs2xdr.c
+++ b/fs/nfs/nfs2xdr.c
@@ -76,6 +76,20 @@ static int nfs_stat_to_errno(enum nfs_stat);
  * or decoded inline.
  */
 
+static struct user_namespace *rpc_userns(const struct rpc_clnt *clnt)
+{
+	if (clnt && clnt->cl_cred)
+		return clnt->cl_cred->user_ns;
+	return &init_user_ns;
+}
+
+static struct user_namespace *rpc_rqst_userns(const struct rpc_rqst *rqstp)
+{
+	if (rqstp->rq_task)
+		return rpc_userns(rqstp->rq_task->tk_client);
+	return &init_user_ns;
+}
+
 /*
  *	typedef opaque	nfsdata<>;
  */
@@ -248,7 +262,8 @@ static __be32 *xdr_decode_time(__be32 *p, struct timespec *timep)
  *	};
  *
  */
-static int decode_fattr(struct xdr_stream *xdr, struct nfs_fattr *fattr)
+static int decode_fattr(struct xdr_stream *xdr, struct nfs_fattr *fattr,
+		struct user_namespace *userns)
 {
 	u32 rdev, type;
 	__be32 *p;
@@ -263,10 +278,10 @@ static int decode_fattr(struct xdr_stream *xdr, struct nfs_fattr *fattr)
 
 	fattr->mode = be32_to_cpup(p++);
 	fattr->nlink = be32_to_cpup(p++);
-	fattr->uid = make_kuid(&init_user_ns, be32_to_cpup(p++));
+	fattr->uid = make_kuid(userns, be32_to_cpup(p++));
 	if (!uid_valid(fattr->uid))
 		goto out_uid;
-	fattr->gid = make_kgid(&init_user_ns, be32_to_cpup(p++));
+	fattr->gid = make_kgid(userns, be32_to_cpup(p++));
 	if (!gid_valid(fattr->gid))
 		goto out_gid;
 		
@@ -321,7 +336,8 @@ static __be32 *xdr_time_not_set(__be32 *p)
 	return p;
 }
 
-static void encode_sattr(struct xdr_stream *xdr, const struct iattr *attr)
+static void encode_sattr(struct xdr_stream *xdr, const struct iattr *attr,
+		struct user_namespace *userns)
 {
 	struct timespec ts;
 	__be32 *p;
@@ -333,11 +349,11 @@ static void encode_sattr(struct xdr_stream *xdr, const struct iattr *attr)
 	else
 		*p++ = cpu_to_be32(NFS2_SATTR_NOT_SET);
 	if (attr->ia_valid & ATTR_UID)
-		*p++ = cpu_to_be32(from_kuid(&init_user_ns, attr->ia_uid));
+		*p++ = cpu_to_be32(from_kuid_munged(userns, attr->ia_uid));
 	else
 		*p++ = cpu_to_be32(NFS2_SATTR_NOT_SET);
 	if (attr->ia_valid & ATTR_GID)
-		*p++ = cpu_to_be32(from_kgid(&init_user_ns, attr->ia_gid));
+		*p++ = cpu_to_be32(from_kgid_munged(userns, attr->ia_gid));
 	else
 		*p++ = cpu_to_be32(NFS2_SATTR_NOT_SET);
 	if (attr->ia_valid & ATTR_SIZE)
@@ -451,7 +467,8 @@ out_cheating:
  *	};
  */
 static int decode_attrstat(struct xdr_stream *xdr, struct nfs_fattr *result,
-			   __u32 *op_status)
+			   __u32 *op_status,
+			   struct user_namespace *userns)
 {
 	enum nfs_stat status;
 	int error;
@@ -463,7 +480,7 @@ static int decode_attrstat(struct xdr_stream *xdr, struct nfs_fattr *result,
 		*op_status = status;
 	if (status != NFS_OK)
 		goto out_default;
-	error = decode_fattr(xdr, result);
+	error = decode_fattr(xdr, result, userns);
 out:
 	return error;
 out_default:
@@ -498,19 +515,21 @@ static void encode_diropargs(struct xdr_stream *xdr, const struct nfs_fh *fh,
  *		void;
  *	};
  */
-static int decode_diropok(struct xdr_stream *xdr, struct nfs_diropok *result)
+static int decode_diropok(struct xdr_stream *xdr, struct nfs_diropok *result,
+		struct user_namespace *userns)
 {
 	int error;
 
 	error = decode_fhandle(xdr, result->fh);
 	if (unlikely(error))
 		goto out;
-	error = decode_fattr(xdr, result->fattr);
+	error = decode_fattr(xdr, result->fattr, userns);
 out:
 	return error;
 }
 
-static int decode_diropres(struct xdr_stream *xdr, struct nfs_diropok *result)
+static int decode_diropres(struct xdr_stream *xdr, struct nfs_diropok *result,
+		struct user_namespace *userns)
 {
 	enum nfs_stat status;
 	int error;
@@ -520,7 +539,7 @@ static int decode_diropres(struct xdr_stream *xdr, struct nfs_diropok *result)
 		goto out;
 	if (status != NFS_OK)
 		goto out_default;
-	error = decode_diropok(xdr, result);
+	error = decode_diropok(xdr, result, userns);
 out:
 	return error;
 out_default:
@@ -559,7 +578,7 @@ static void nfs2_xdr_enc_sattrargs(struct rpc_rqst *req,
 	const struct nfs_sattrargs *args = data;
 
 	encode_fhandle(xdr, args->fh);
-	encode_sattr(xdr, args->sattr);
+	encode_sattr(xdr, args->sattr, rpc_rqst_userns(req));
 }
 
 static void nfs2_xdr_enc_diropargs(struct rpc_rqst *req,
@@ -674,7 +693,7 @@ static void nfs2_xdr_enc_createargs(struct rpc_rqst *req,
 	const struct nfs_createargs *args = data;
 
 	encode_diropargs(xdr, args->fh, args->name, args->len);
-	encode_sattr(xdr, args->sattr);
+	encode_sattr(xdr, args->sattr, rpc_rqst_userns(req));
 }
 
 static void nfs2_xdr_enc_removeargs(struct rpc_rqst *req,
@@ -741,7 +760,7 @@ static void nfs2_xdr_enc_symlinkargs(struct rpc_rqst *req,
 
 	encode_diropargs(xdr, args->fromfh, args->fromname, args->fromlen);
 	encode_path(xdr, args->pages, args->pathlen);
-	encode_sattr(xdr, args->sattr);
+	encode_sattr(xdr, args->sattr, rpc_rqst_userns(req));
 }
 
 /*
@@ -803,13 +822,13 @@ out_default:
 static int nfs2_xdr_dec_attrstat(struct rpc_rqst *req, struct xdr_stream *xdr,
 				 void *result)
 {
-	return decode_attrstat(xdr, result, NULL);
+	return decode_attrstat(xdr, result, NULL, rpc_rqst_userns(req));
 }
 
 static int nfs2_xdr_dec_diropres(struct rpc_rqst *req, struct xdr_stream *xdr,
 				 void *result)
 {
-	return decode_diropres(xdr, result);
+	return decode_diropres(xdr, result, rpc_rqst_userns(req));
 }
 
 /*
@@ -864,7 +883,7 @@ static int nfs2_xdr_dec_readres(struct rpc_rqst *req, struct xdr_stream *xdr,
 	result->op_status = status;
 	if (status != NFS_OK)
 		goto out_default;
-	error = decode_fattr(xdr, result->fattr);
+	error = decode_fattr(xdr, result->fattr, rpc_rqst_userns(req));
 	if (unlikely(error))
 		goto out;
 	error = decode_nfsdata(xdr, result);
@@ -881,7 +900,8 @@ static int nfs2_xdr_dec_writeres(struct rpc_rqst *req, struct xdr_stream *xdr,
 
 	/* All NFSv2 writes are "file sync" writes */
 	result->verf->committed = NFS_FILE_SYNC;
-	return decode_attrstat(xdr, result->fattr, &result->op_status);
+	return decode_attrstat(xdr, result->fattr, &result->op_status,
+			rpc_rqst_userns(req));
 }
 
 /**
diff --git a/fs/nfs/nfs3client.c b/fs/nfs/nfs3client.c
index 7879f2a0fcfd..1afdb0f7473f 100644
--- a/fs/nfs/nfs3client.c
+++ b/fs/nfs/nfs3client.c
@@ -91,6 +91,7 @@ struct nfs_client *nfs3_set_ds_client(struct nfs_server *mds_srv,
 		.proto = ds_proto,
 		.net = mds_clp->cl_net,
 		.timeparms = &ds_timeout,
+		.cred = mds_srv->cred,
 	};
 	struct nfs_client *clp;
 	char buf[INET6_ADDRSTRLEN + 1];
diff --git a/fs/nfs/nfs3xdr.c b/fs/nfs/nfs3xdr.c
index 110358f4986d..abbbdde97e31 100644
--- a/fs/nfs/nfs3xdr.c
+++ b/fs/nfs/nfs3xdr.c
@@ -104,6 +104,20 @@ static const umode_t nfs_type2fmt[] = {
 	[NF3FIFO] = S_IFIFO,
 };
 
+static struct user_namespace *rpc_userns(const struct rpc_clnt *clnt)
+{
+	if (clnt && clnt->cl_cred)
+		return clnt->cl_cred->user_ns;
+	return &init_user_ns;
+}
+
+static struct user_namespace *rpc_rqst_userns(const struct rpc_rqst *rqstp)
+{
+	if (rqstp->rq_task)
+		return rpc_userns(rqstp->rq_task->tk_client);
+	return &init_user_ns;
+}
+
 /*
  * Encode/decode NFSv3 basic data types
  *
@@ -516,7 +530,8 @@ static __be32 *xdr_decode_nfstime3(__be32 *p, struct timespec *timep)
  *		set_mtime	mtime;
  *	};
  */
-static void encode_sattr3(struct xdr_stream *xdr, const struct iattr *attr)
+static void encode_sattr3(struct xdr_stream *xdr, const struct iattr *attr,
+		struct user_namespace *userns)
 {
 	struct timespec ts;
 	u32 nbytes;
@@ -551,13 +566,13 @@ static void encode_sattr3(struct xdr_stream *xdr, const struct iattr *attr)
 
 	if (attr->ia_valid & ATTR_UID) {
 		*p++ = xdr_one;
-		*p++ = cpu_to_be32(from_kuid(&init_user_ns, attr->ia_uid));
+		*p++ = cpu_to_be32(from_kuid_munged(userns, attr->ia_uid));
 	} else
 		*p++ = xdr_zero;
 
 	if (attr->ia_valid & ATTR_GID) {
 		*p++ = xdr_one;
-		*p++ = cpu_to_be32(from_kgid(&init_user_ns, attr->ia_gid));
+		*p++ = cpu_to_be32(from_kgid_munged(userns, attr->ia_gid));
 	} else
 		*p++ = xdr_zero;
 
@@ -606,7 +621,8 @@ static void encode_sattr3(struct xdr_stream *xdr, const struct iattr *attr)
  *		nfstime3	ctime;
  *	};
  */
-static int decode_fattr3(struct xdr_stream *xdr, struct nfs_fattr *fattr)
+static int decode_fattr3(struct xdr_stream *xdr, struct nfs_fattr *fattr,
+		struct user_namespace *userns)
 {
 	umode_t fmode;
 	__be32 *p;
@@ -619,10 +635,10 @@ static int decode_fattr3(struct xdr_stream *xdr, struct nfs_fattr *fattr)
 
 	fattr->mode = (be32_to_cpup(p++) & ~S_IFMT) | fmode;
 	fattr->nlink = be32_to_cpup(p++);
-	fattr->uid = make_kuid(&init_user_ns, be32_to_cpup(p++));
+	fattr->uid = make_kuid(userns, be32_to_cpup(p++));
 	if (!uid_valid(fattr->uid))
 		goto out_uid;
-	fattr->gid = make_kgid(&init_user_ns, be32_to_cpup(p++));
+	fattr->gid = make_kgid(userns, be32_to_cpup(p++));
 	if (!gid_valid(fattr->gid))
 		goto out_gid;
 
@@ -659,7 +675,8 @@ out_gid:
  *		void;
  *	};
  */
-static int decode_post_op_attr(struct xdr_stream *xdr, struct nfs_fattr *fattr)
+static int decode_post_op_attr(struct xdr_stream *xdr, struct nfs_fattr *fattr,
+		struct user_namespace *userns)
 {
 	__be32 *p;
 
@@ -667,7 +684,7 @@ static int decode_post_op_attr(struct xdr_stream *xdr, struct nfs_fattr *fattr)
 	if (unlikely(!p))
 		return -EIO;
 	if (*p != xdr_zero)
-		return decode_fattr3(xdr, fattr);
+		return decode_fattr3(xdr, fattr, userns);
 	return 0;
 }
 
@@ -728,14 +745,15 @@ static int decode_pre_op_attr(struct xdr_stream *xdr, struct nfs_fattr *fattr)
 	return 0;
 }
 
-static int decode_wcc_data(struct xdr_stream *xdr, struct nfs_fattr *fattr)
+static int decode_wcc_data(struct xdr_stream *xdr, struct nfs_fattr *fattr,
+		struct user_namespace *userns)
 {
 	int error;
 
 	error = decode_pre_op_attr(xdr, fattr);
 	if (unlikely(error))
 		goto out;
-	error = decode_post_op_attr(xdr, fattr);
+	error = decode_post_op_attr(xdr, fattr, userns);
 out:
 	return error;
 }
@@ -837,7 +855,7 @@ static void nfs3_xdr_enc_setattr3args(struct rpc_rqst *req,
 {
 	const struct nfs3_sattrargs *args = data;
 	encode_nfs_fh3(xdr, args->fh);
-	encode_sattr3(xdr, args->sattr);
+	encode_sattr3(xdr, args->sattr, rpc_rqst_userns(req));
 	encode_sattrguard3(xdr, args);
 }
 
@@ -998,13 +1016,14 @@ static void nfs3_xdr_enc_write3args(struct rpc_rqst *req,
  *	};
  */
 static void encode_createhow3(struct xdr_stream *xdr,
-			      const struct nfs3_createargs *args)
+			      const struct nfs3_createargs *args,
+			      struct user_namespace *userns)
 {
 	encode_uint32(xdr, args->createmode);
 	switch (args->createmode) {
 	case NFS3_CREATE_UNCHECKED:
 	case NFS3_CREATE_GUARDED:
-		encode_sattr3(xdr, args->sattr);
+		encode_sattr3(xdr, args->sattr, userns);
 		break;
 	case NFS3_CREATE_EXCLUSIVE:
 		encode_createverf3(xdr, args->verifier);
@@ -1021,7 +1040,7 @@ static void nfs3_xdr_enc_create3args(struct rpc_rqst *req,
 	const struct nfs3_createargs *args = data;
 
 	encode_diropargs3(xdr, args->fh, args->name, args->len);
-	encode_createhow3(xdr, args);
+	encode_createhow3(xdr, args, rpc_rqst_userns(req));
 }
 
 /*
@@ -1039,7 +1058,7 @@ static void nfs3_xdr_enc_mkdir3args(struct rpc_rqst *req,
 	const struct nfs3_mkdirargs *args = data;
 
 	encode_diropargs3(xdr, args->fh, args->name, args->len);
-	encode_sattr3(xdr, args->sattr);
+	encode_sattr3(xdr, args->sattr, rpc_rqst_userns(req));
 }
 
 /*
@@ -1056,11 +1075,12 @@ static void nfs3_xdr_enc_mkdir3args(struct rpc_rqst *req,
  *	};
  */
 static void encode_symlinkdata3(struct xdr_stream *xdr,
-				const void *data)
+				const void *data,
+				struct user_namespace *userns)
 {
 	const struct nfs3_symlinkargs *args = data;
 
-	encode_sattr3(xdr, args->sattr);
+	encode_sattr3(xdr, args->sattr, userns);
 	encode_nfspath3(xdr, args->pages, args->pathlen);
 }
 
@@ -1071,7 +1091,7 @@ static void nfs3_xdr_enc_symlink3args(struct rpc_rqst *req,
 	const struct nfs3_symlinkargs *args = data;
 
 	encode_diropargs3(xdr, args->fromfh, args->fromname, args->fromlen);
-	encode_symlinkdata3(xdr, args);
+	encode_symlinkdata3(xdr, args, rpc_rqst_userns(req));
 	xdr->buf->flags |= XDRBUF_WRITE;
 }
 
@@ -1100,24 +1120,26 @@ static void nfs3_xdr_enc_symlink3args(struct rpc_rqst *req,
  *	};
  */
 static void encode_devicedata3(struct xdr_stream *xdr,
-			       const struct nfs3_mknodargs *args)
+			       const struct nfs3_mknodargs *args,
+			       struct user_namespace *userns)
 {
-	encode_sattr3(xdr, args->sattr);
+	encode_sattr3(xdr, args->sattr, userns);
 	encode_specdata3(xdr, args->rdev);
 }
 
 static void encode_mknoddata3(struct xdr_stream *xdr,
-			      const struct nfs3_mknodargs *args)
+			      const struct nfs3_mknodargs *args,
+			      struct user_namespace *userns)
 {
 	encode_ftype3(xdr, args->type);
 	switch (args->type) {
 	case NF3CHR:
 	case NF3BLK:
-		encode_devicedata3(xdr, args);
+		encode_devicedata3(xdr, args, userns);
 		break;
 	case NF3SOCK:
 	case NF3FIFO:
-		encode_sattr3(xdr, args->sattr);
+		encode_sattr3(xdr, args->sattr, userns);
 		break;
 	case NF3REG:
 	case NF3DIR:
@@ -1134,7 +1156,7 @@ static void nfs3_xdr_enc_mknod3args(struct rpc_rqst *req,
 	const struct nfs3_mknodargs *args = data;
 
 	encode_diropargs3(xdr, args->fh, args->name, args->len);
-	encode_mknoddata3(xdr, args);
+	encode_mknoddata3(xdr, args, rpc_rqst_userns(req));
 }
 
 /*
@@ -1379,7 +1401,7 @@ static int nfs3_xdr_dec_getattr3res(struct rpc_rqst *req,
 		goto out;
 	if (status != NFS3_OK)
 		goto out_default;
-	error = decode_fattr3(xdr, result);
+	error = decode_fattr3(xdr, result, rpc_rqst_userns(req));
 out:
 	return error;
 out_default:
@@ -1414,7 +1436,7 @@ static int nfs3_xdr_dec_setattr3res(struct rpc_rqst *req,
 	error = decode_nfsstat3(xdr, &status);
 	if (unlikely(error))
 		goto out;
-	error = decode_wcc_data(xdr, result);
+	error = decode_wcc_data(xdr, result, rpc_rqst_userns(req));
 	if (unlikely(error))
 		goto out;
 	if (status != NFS3_OK)
@@ -1449,6 +1471,7 @@ static int nfs3_xdr_dec_lookup3res(struct rpc_rqst *req,
 				   struct xdr_stream *xdr,
 				   void *data)
 {
+	struct user_namespace *userns = rpc_rqst_userns(req);
 	struct nfs3_diropres *result = data;
 	enum nfs_stat status;
 	int error;
@@ -1461,14 +1484,14 @@ static int nfs3_xdr_dec_lookup3res(struct rpc_rqst *req,
 	error = decode_nfs_fh3(xdr, result->fh);
 	if (unlikely(error))
 		goto out;
-	error = decode_post_op_attr(xdr, result->fattr);
+	error = decode_post_op_attr(xdr, result->fattr, userns);
 	if (unlikely(error))
 		goto out;
-	error = decode_post_op_attr(xdr, result->dir_attr);
+	error = decode_post_op_attr(xdr, result->dir_attr, userns);
 out:
 	return error;
 out_default:
-	error = decode_post_op_attr(xdr, result->dir_attr);
+	error = decode_post_op_attr(xdr, result->dir_attr, userns);
 	if (unlikely(error))
 		goto out;
 	return nfs3_stat_to_errno(status);
@@ -1504,7 +1527,7 @@ static int nfs3_xdr_dec_access3res(struct rpc_rqst *req,
 	error = decode_nfsstat3(xdr, &status);
 	if (unlikely(error))
 		goto out;
-	error = decode_post_op_attr(xdr, result->fattr);
+	error = decode_post_op_attr(xdr, result->fattr, rpc_rqst_userns(req));
 	if (unlikely(error))
 		goto out;
 	if (status != NFS3_OK)
@@ -1545,7 +1568,7 @@ static int nfs3_xdr_dec_readlink3res(struct rpc_rqst *req,
 	error = decode_nfsstat3(xdr, &status);
 	if (unlikely(error))
 		goto out;
-	error = decode_post_op_attr(xdr, result);
+	error = decode_post_op_attr(xdr, result, rpc_rqst_userns(req));
 	if (unlikely(error))
 		goto out;
 	if (status != NFS3_OK)
@@ -1623,7 +1646,7 @@ static int nfs3_xdr_dec_read3res(struct rpc_rqst *req, struct xdr_stream *xdr,
 	error = decode_nfsstat3(xdr, &status);
 	if (unlikely(error))
 		goto out;
-	error = decode_post_op_attr(xdr, result->fattr);
+	error = decode_post_op_attr(xdr, result->fattr, rpc_rqst_userns(req));
 	if (unlikely(error))
 		goto out;
 	result->op_status = status;
@@ -1694,7 +1717,7 @@ static int nfs3_xdr_dec_write3res(struct rpc_rqst *req, struct xdr_stream *xdr,
 	error = decode_nfsstat3(xdr, &status);
 	if (unlikely(error))
 		goto out;
-	error = decode_wcc_data(xdr, result->fattr);
+	error = decode_wcc_data(xdr, result->fattr, rpc_rqst_userns(req));
 	if (unlikely(error))
 		goto out;
 	result->op_status = status;
@@ -1728,14 +1751,15 @@ out_status:
  *	};
  */
 static int decode_create3resok(struct xdr_stream *xdr,
-			       struct nfs3_diropres *result)
+			       struct nfs3_diropres *result,
+			       struct user_namespace *userns)
 {
 	int error;
 
 	error = decode_post_op_fh3(xdr, result->fh);
 	if (unlikely(error))
 		goto out;
-	error = decode_post_op_attr(xdr, result->fattr);
+	error = decode_post_op_attr(xdr, result->fattr, userns);
 	if (unlikely(error))
 		goto out;
 	/* The server isn't required to return a file handle.
@@ -1744,7 +1768,7 @@ static int decode_create3resok(struct xdr_stream *xdr,
 	 * values for the new object. */
 	if (result->fh->size == 0)
 		result->fattr->valid = 0;
-	error = decode_wcc_data(xdr, result->dir_attr);
+	error = decode_wcc_data(xdr, result->dir_attr, userns);
 out:
 	return error;
 }
@@ -1753,6 +1777,7 @@ static int nfs3_xdr_dec_create3res(struct rpc_rqst *req,
 				   struct xdr_stream *xdr,
 				   void *data)
 {
+	struct user_namespace *userns = rpc_rqst_userns(req);
 	struct nfs3_diropres *result = data;
 	enum nfs_stat status;
 	int error;
@@ -1762,11 +1787,11 @@ static int nfs3_xdr_dec_create3res(struct rpc_rqst *req,
 		goto out;
 	if (status != NFS3_OK)
 		goto out_default;
-	error = decode_create3resok(xdr, result);
+	error = decode_create3resok(xdr, result, userns);
 out:
 	return error;
 out_default:
-	error = decode_wcc_data(xdr, result->dir_attr);
+	error = decode_wcc_data(xdr, result->dir_attr, userns);
 	if (unlikely(error))
 		goto out;
 	return nfs3_stat_to_errno(status);
@@ -1801,7 +1826,7 @@ static int nfs3_xdr_dec_remove3res(struct rpc_rqst *req,
 	error = decode_nfsstat3(xdr, &status);
 	if (unlikely(error))
 		goto out;
-	error = decode_wcc_data(xdr, result->dir_attr);
+	error = decode_wcc_data(xdr, result->dir_attr, rpc_rqst_userns(req));
 	if (unlikely(error))
 		goto out;
 	if (status != NFS3_OK)
@@ -1836,6 +1861,7 @@ static int nfs3_xdr_dec_rename3res(struct rpc_rqst *req,
 				   struct xdr_stream *xdr,
 				   void *data)
 {
+	struct user_namespace *userns = rpc_rqst_userns(req);
 	struct nfs_renameres *result = data;
 	enum nfs_stat status;
 	int error;
@@ -1843,10 +1869,10 @@ static int nfs3_xdr_dec_rename3res(struct rpc_rqst *req,
 	error = decode_nfsstat3(xdr, &status);
 	if (unlikely(error))
 		goto out;
-	error = decode_wcc_data(xdr, result->old_fattr);
+	error = decode_wcc_data(xdr, result->old_fattr, userns);
 	if (unlikely(error))
 		goto out;
-	error = decode_wcc_data(xdr, result->new_fattr);
+	error = decode_wcc_data(xdr, result->new_fattr, userns);
 	if (unlikely(error))
 		goto out;
 	if (status != NFS3_OK)
@@ -1880,6 +1906,7 @@ out_status:
 static int nfs3_xdr_dec_link3res(struct rpc_rqst *req, struct xdr_stream *xdr,
 				 void *data)
 {
+	struct user_namespace *userns = rpc_rqst_userns(req);
 	struct nfs3_linkres *result = data;
 	enum nfs_stat status;
 	int error;
@@ -1887,10 +1914,10 @@ static int nfs3_xdr_dec_link3res(struct rpc_rqst *req, struct xdr_stream *xdr,
 	error = decode_nfsstat3(xdr, &status);
 	if (unlikely(error))
 		goto out;
-	error = decode_post_op_attr(xdr, result->fattr);
+	error = decode_post_op_attr(xdr, result->fattr, userns);
 	if (unlikely(error))
 		goto out;
-	error = decode_wcc_data(xdr, result->dir_attr);
+	error = decode_wcc_data(xdr, result->dir_attr, userns);
 	if (unlikely(error))
 		goto out;
 	if (status != NFS3_OK)
@@ -1939,6 +1966,7 @@ out_status:
 int nfs3_decode_dirent(struct xdr_stream *xdr, struct nfs_entry *entry,
 		       bool plus)
 {
+	struct user_namespace *userns = rpc_userns(entry->server->client);
 	struct nfs_entry old = *entry;
 	__be32 *p;
 	int error;
@@ -1973,7 +2001,7 @@ int nfs3_decode_dirent(struct xdr_stream *xdr, struct nfs_entry *entry,
 
 	if (plus) {
 		entry->fattr->valid = 0;
-		error = decode_post_op_attr(xdr, entry->fattr);
+		error = decode_post_op_attr(xdr, entry->fattr, userns);
 		if (unlikely(error))
 			return error;
 		if (entry->fattr->valid & NFS_ATTR_FATTR_V3)
@@ -2045,11 +2073,12 @@ static int decode_dirlist3(struct xdr_stream *xdr)
 }
 
 static int decode_readdir3resok(struct xdr_stream *xdr,
-				struct nfs3_readdirres *result)
+				struct nfs3_readdirres *result,
+				struct user_namespace *userns)
 {
 	int error;
 
-	error = decode_post_op_attr(xdr, result->dir_attr);
+	error = decode_post_op_attr(xdr, result->dir_attr, userns);
 	if (unlikely(error))
 		goto out;
 	/* XXX: do we need to check if result->verf != NULL ? */
@@ -2074,11 +2103,11 @@ static int nfs3_xdr_dec_readdir3res(struct rpc_rqst *req,
 		goto out;
 	if (status != NFS3_OK)
 		goto out_default;
-	error = decode_readdir3resok(xdr, result);
+	error = decode_readdir3resok(xdr, result, rpc_rqst_userns(req));
 out:
 	return error;
 out_default:
-	error = decode_post_op_attr(xdr, result->dir_attr);
+	error = decode_post_op_attr(xdr, result->dir_attr, rpc_rqst_userns(req));
 	if (unlikely(error))
 		goto out;
 	return nfs3_stat_to_errno(status);
@@ -2138,7 +2167,7 @@ static int nfs3_xdr_dec_fsstat3res(struct rpc_rqst *req,
 	error = decode_nfsstat3(xdr, &status);
 	if (unlikely(error))
 		goto out;
-	error = decode_post_op_attr(xdr, result->fattr);
+	error = decode_post_op_attr(xdr, result->fattr, rpc_rqst_userns(req));
 	if (unlikely(error))
 		goto out;
 	if (status != NFS3_OK)
@@ -2212,7 +2241,7 @@ static int nfs3_xdr_dec_fsinfo3res(struct rpc_rqst *req,
 	error = decode_nfsstat3(xdr, &status);
 	if (unlikely(error))
 		goto out;
-	error = decode_post_op_attr(xdr, result->fattr);
+	error = decode_post_op_attr(xdr, result->fattr, rpc_rqst_userns(req));
 	if (unlikely(error))
 		goto out;
 	if (status != NFS3_OK)
@@ -2273,7 +2302,7 @@ static int nfs3_xdr_dec_pathconf3res(struct rpc_rqst *req,
 	error = decode_nfsstat3(xdr, &status);
 	if (unlikely(error))
 		goto out;
-	error = decode_post_op_attr(xdr, result->fattr);
+	error = decode_post_op_attr(xdr, result->fattr, rpc_rqst_userns(req));
 	if (unlikely(error))
 		goto out;
 	if (status != NFS3_OK)
@@ -2315,7 +2344,7 @@ static int nfs3_xdr_dec_commit3res(struct rpc_rqst *req,
 	error = decode_nfsstat3(xdr, &status);
 	if (unlikely(error))
 		goto out;
-	error = decode_wcc_data(xdr, result->fattr);
+	error = decode_wcc_data(xdr, result->fattr, rpc_rqst_userns(req));
 	if (unlikely(error))
 		goto out;
 	result->op_status = status;
@@ -2331,14 +2360,15 @@ out_status:
 #ifdef CONFIG_NFS_V3_ACL
 
 static inline int decode_getacl3resok(struct xdr_stream *xdr,
-				      struct nfs3_getaclres *result)
+				      struct nfs3_getaclres *result,
+				      struct user_namespace *userns)
 {
 	struct posix_acl **acl;
 	unsigned int *aclcnt;
 	size_t hdrlen;
 	int error;
 
-	error = decode_post_op_attr(xdr, result->fattr);
+	error = decode_post_op_attr(xdr, result->fattr, userns);
 	if (unlikely(error))
 		goto out;
 	error = decode_uint32(xdr, &result->mask);
@@ -2386,7 +2416,7 @@ static int nfs3_xdr_dec_getacl3res(struct rpc_rqst *req,
 		goto out;
 	if (status != NFS3_OK)
 		goto out_default;
-	error = decode_getacl3resok(xdr, result);
+	error = decode_getacl3resok(xdr, result, rpc_rqst_userns(req));
 out:
 	return error;
 out_default:
@@ -2405,7 +2435,7 @@ static int nfs3_xdr_dec_setacl3res(struct rpc_rqst *req,
 		goto out;
 	if (status != NFS3_OK)
 		goto out_default;
-	error = decode_post_op_attr(xdr, result);
+	error = decode_post_op_attr(xdr, result, rpc_rqst_userns(req));
 out:
 	return error;
 out_default:
diff --git a/fs/nfs/nfs4_fs.h b/fs/nfs/nfs4_fs.h
index 06ac3d9ac7c6..8a38a254f516 100644
--- a/fs/nfs/nfs4_fs.h
+++ b/fs/nfs/nfs4_fs.h
@@ -206,6 +206,7 @@ struct nfs4_exception {
 	unsigned char delay : 1,
 		      recovering : 1,
 		      retry : 1;
+	bool interruptible;
 };
 
 struct nfs4_state_recovery_ops {
diff --git a/fs/nfs/nfs4client.c b/fs/nfs/nfs4client.c
index 1339ede979af..3ce246346f02 100644
--- a/fs/nfs/nfs4client.c
+++ b/fs/nfs/nfs4client.c
@@ -870,6 +870,7 @@ static int nfs4_set_client(struct nfs_server *server,
 		.minorversion = minorversion,
 		.net = net,
 		.timeparms = timeparms,
+		.cred = server->cred,
 	};
 	struct nfs_client *clp;
 
@@ -931,6 +932,7 @@ struct nfs_client *nfs4_set_ds_client(struct nfs_server *mds_srv,
 		.minorversion = minor_version,
 		.net = mds_clp->cl_net,
 		.timeparms = &ds_timeout,
+		.cred = mds_srv->cred,
 	};
 	char buf[INET6_ADDRSTRLEN + 1];
 
@@ -1107,6 +1109,8 @@ struct nfs_server *nfs4_create_server(struct nfs_mount_info *mount_info,
 	if (!server)
 		return ERR_PTR(-ENOMEM);
 
+	server->cred = get_cred(current_cred());
+
 	auth_probe = mount_info->parsed->auth_info.flavor_len < 1;
 
 	/* set up the general RPC client */
@@ -1143,6 +1147,8 @@ struct nfs_server *nfs4_create_referral_server(struct nfs_clone_mount *data,
 	parent_server = NFS_SB(data->sb);
 	parent_client = parent_server->nfs_client;
 
+	server->cred = get_cred(parent_server->cred);
+
 	/* Initialise the client representation from the parent server */
 	nfs_server_copy_userdata(server, parent_server);
 
diff --git a/fs/nfs/nfs4file.c b/fs/nfs/nfs4file.c
index 00d17198ee12..cf42a8b939e3 100644
--- a/fs/nfs/nfs4file.c
+++ b/fs/nfs/nfs4file.c
@@ -125,7 +125,7 @@ nfs4_file_flush(struct file *file, fl_owner_t id)
 		return filemap_fdatawrite(file->f_mapping);
 
 	/* Flush writes to the server and return any errors */
-	return vfs_fsync(file, 0);
+	return nfs_wb_all(inode);
 }
 
 #ifdef CONFIG_NFS_V4_2
@@ -187,7 +187,7 @@ static loff_t nfs42_remap_file_range(struct file *src_file, loff_t src_off,
 	bool same_inode = false;
 	int ret;
 
-	if (remap_flags & ~REMAP_FILE_ADVISORY)
+	if (remap_flags & ~(REMAP_FILE_DEDUP | REMAP_FILE_ADVISORY))
 		return -EINVAL;
 
 	/* check alignment w.r.t. clone_blksize */
diff --git a/fs/nfs/nfs4idmap.c b/fs/nfs/nfs4idmap.c
index bf34ddaa2ad7..4884fdae28fb 100644
--- a/fs/nfs/nfs4idmap.c
+++ b/fs/nfs/nfs4idmap.c
@@ -69,8 +69,16 @@ struct idmap {
 	struct rpc_pipe		*idmap_pipe;
 	struct idmap_legacy_upcalldata *idmap_upcall_data;
 	struct mutex		idmap_mutex;
+	const struct cred	*cred;
 };
 
+static struct user_namespace *idmap_userns(const struct idmap *idmap)
+{
+	if (idmap && idmap->cred)
+		return idmap->cred->user_ns;
+	return &init_user_ns;
+}
+
 /**
  * nfs_fattr_init_names - initialise the nfs_fattr owner_name/group_name fields
  * @fattr: fully initialised struct nfs_fattr
@@ -271,14 +279,15 @@ static struct key *nfs_idmap_request_key(const char *name, size_t namelen,
 					 const char *type, struct idmap *idmap)
 {
 	char *desc;
-	struct key *rkey;
+	struct key *rkey = ERR_PTR(-EAGAIN);
 	ssize_t ret;
 
 	ret = nfs_idmap_get_desc(name, namelen, type, strlen(type), &desc);
 	if (ret < 0)
 		return ERR_PTR(ret);
 
-	rkey = request_key(&key_type_id_resolver, desc, "");
+	if (!idmap->cred || idmap->cred->user_ns == &init_user_ns)
+		rkey = request_key(&key_type_id_resolver, desc, "");
 	if (IS_ERR(rkey)) {
 		mutex_lock(&idmap->idmap_mutex);
 		rkey = request_key_with_auxdata(&key_type_id_resolver_legacy,
@@ -452,6 +461,9 @@ nfs_idmap_new(struct nfs_client *clp)
 	if (idmap == NULL)
 		return -ENOMEM;
 
+	mutex_init(&idmap->idmap_mutex);
+	idmap->cred = get_cred(clp->cl_rpcclient->cl_cred);
+
 	rpc_init_pipe_dir_object(&idmap->idmap_pdo,
 			&nfs_idmap_pipe_dir_object_ops,
 			idmap);
@@ -462,7 +474,6 @@ nfs_idmap_new(struct nfs_client *clp)
 		goto err;
 	}
 	idmap->idmap_pipe = pipe;
-	mutex_init(&idmap->idmap_mutex);
 
 	error = rpc_add_pipe_dir_object(clp->cl_net,
 			&clp->cl_rpcclient->cl_pipedir_objects,
@@ -475,6 +486,7 @@ nfs_idmap_new(struct nfs_client *clp)
 err_destroy_pipe:
 	rpc_destroy_pipe_data(idmap->idmap_pipe);
 err:
+	put_cred(idmap->cred);
 	kfree(idmap);
 	return error;
 }
@@ -491,6 +503,7 @@ nfs_idmap_delete(struct nfs_client *clp)
 			&clp->cl_rpcclient->cl_pipedir_objects,
 			&idmap->idmap_pdo);
 	rpc_destroy_pipe_data(idmap->idmap_pipe);
+	put_cred(idmap->cred);
 	kfree(idmap);
 }
 
@@ -735,7 +748,7 @@ int nfs_map_name_to_uid(const struct nfs_server *server, const char *name, size_
 	if (!nfs_map_string_to_numeric(name, namelen, &id))
 		ret = nfs_idmap_lookup_id(name, namelen, "uid", &id, idmap);
 	if (ret == 0) {
-		*uid = make_kuid(&init_user_ns, id);
+		*uid = make_kuid(idmap_userns(idmap), id);
 		if (!uid_valid(*uid))
 			ret = -ERANGE;
 	}
@@ -752,7 +765,7 @@ int nfs_map_group_to_gid(const struct nfs_server *server, const char *name, size
 	if (!nfs_map_string_to_numeric(name, namelen, &id))
 		ret = nfs_idmap_lookup_id(name, namelen, "gid", &id, idmap);
 	if (ret == 0) {
-		*gid = make_kgid(&init_user_ns, id);
+		*gid = make_kgid(idmap_userns(idmap), id);
 		if (!gid_valid(*gid))
 			ret = -ERANGE;
 	}
@@ -766,7 +779,7 @@ int nfs_map_uid_to_name(const struct nfs_server *server, kuid_t uid, char *buf,
 	int ret = -EINVAL;
 	__u32 id;
 
-	id = from_kuid(&init_user_ns, uid);
+	id = from_kuid_munged(idmap_userns(idmap), uid);
 	if (!(server->caps & NFS_CAP_UIDGID_NOMAP))
 		ret = nfs_idmap_lookup_name(id, "user", buf, buflen, idmap);
 	if (ret < 0)
@@ -780,7 +793,7 @@ int nfs_map_gid_to_group(const struct nfs_server *server, kgid_t gid, char *buf,
 	int ret = -EINVAL;
 	__u32 id;
 
-	id = from_kgid(&init_user_ns, gid);
+	id = from_kgid_munged(idmap_userns(idmap), gid);
 	if (!(server->caps & NFS_CAP_UIDGID_NOMAP))
 		ret = nfs_idmap_lookup_name(id, "group", buf, buflen, idmap);
 	if (ret < 0)
diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c
index 741ff8c9c6ed..c29cbef6b53f 100644
--- a/fs/nfs/nfs4proc.c
+++ b/fs/nfs/nfs4proc.c
@@ -400,17 +400,32 @@ static long nfs4_update_delay(long *timeout)
 	return ret;
 }
 
-static int nfs4_delay(struct rpc_clnt *clnt, long *timeout)
+static int nfs4_delay_killable(long *timeout)
 {
-	int res = 0;
-
 	might_sleep();
 
 	freezable_schedule_timeout_killable_unsafe(
 		nfs4_update_delay(timeout));
-	if (fatal_signal_pending(current))
-		res = -ERESTARTSYS;
-	return res;
+	if (!__fatal_signal_pending(current))
+		return 0;
+	return -EINTR;
+}
+
+static int nfs4_delay_interruptible(long *timeout)
+{
+	might_sleep();
+
+	freezable_schedule_timeout_interruptible(nfs4_update_delay(timeout));
+	if (!signal_pending(current))
+		return 0;
+	return __fatal_signal_pending(current) ? -EINTR :-ERESTARTSYS;
+}
+
+static int nfs4_delay(long *timeout, bool interruptible)
+{
+	if (interruptible)
+		return nfs4_delay_interruptible(timeout);
+	return nfs4_delay_killable(timeout);
 }
 
 /* This is the error handling routine for processes that are allowed
@@ -546,7 +561,8 @@ int nfs4_handle_exception(struct nfs_server *server, int errorcode, struct nfs4_
 
 	ret = nfs4_do_handle_exception(server, errorcode, exception);
 	if (exception->delay) {
-		ret = nfs4_delay(server->client, &exception->timeout);
+		ret = nfs4_delay(&exception->timeout,
+				exception->interruptible);
 		goto out_retry;
 	}
 	if (exception->recovering) {
@@ -978,10 +994,8 @@ int nfs4_setup_sequence(struct nfs_client *client,
 	if (res->sr_slot != NULL)
 		goto out_start;
 
-	if (session) {
+	if (session)
 		tbl = &session->fc_slot_table;
-		task->tk_timeout = 0;
-	}
 
 	spin_lock(&tbl->slot_tbl_lock);
 	/* The state manager will wait until the slot table is empty */
@@ -990,9 +1004,8 @@ int nfs4_setup_sequence(struct nfs_client *client,
 
 	slot = nfs4_alloc_slot(tbl);
 	if (IS_ERR(slot)) {
-		/* Try again in 1/4 second */
 		if (slot == ERR_PTR(-ENOMEM))
-			task->tk_timeout = HZ >> 2;
+			goto out_sleep_timeout;
 		goto out_sleep;
 	}
 	spin_unlock(&tbl->slot_tbl_lock);
@@ -1004,11 +1017,20 @@ out_start:
 	nfs41_sequence_res_init(res);
 	rpc_call_start(task);
 	return 0;
-
+out_sleep_timeout:
+	/* Try again in 1/4 second */
+	if (args->sa_privileged)
+		rpc_sleep_on_priority_timeout(&tbl->slot_tbl_waitq, task,
+				jiffies + (HZ >> 2), RPC_PRIORITY_PRIVILEGED);
+	else
+		rpc_sleep_on_timeout(&tbl->slot_tbl_waitq, task,
+				NULL, jiffies + (HZ >> 2));
+	spin_unlock(&tbl->slot_tbl_lock);
+	return -EAGAIN;
 out_sleep:
 	if (args->sa_privileged)
 		rpc_sleep_on_priority(&tbl->slot_tbl_waitq, task,
-				NULL, RPC_PRIORITY_PRIVILEGED);
+				RPC_PRIORITY_PRIVILEGED);
 	else
 		rpc_sleep_on(&tbl->slot_tbl_waitq, task, NULL);
 	spin_unlock(&tbl->slot_tbl_lock);
@@ -3060,7 +3082,9 @@ static struct nfs4_state *nfs4_do_open(struct inode *dir,
 					int *opened)
 {
 	struct nfs_server *server = NFS_SERVER(dir);
-	struct nfs4_exception exception = { };
+	struct nfs4_exception exception = {
+		.interruptible = true,
+	};
 	struct nfs4_state *res;
 	struct nfs4_open_createattrs c = {
 		.label = label,
@@ -3673,7 +3697,9 @@ static int _nfs4_server_capabilities(struct nfs_server *server, struct nfs_fh *f
 
 int nfs4_server_capabilities(struct nfs_server *server, struct nfs_fh *fhandle)
 {
-	struct nfs4_exception exception = { };
+	struct nfs4_exception exception = {
+		.interruptible = true,
+	};
 	int err;
 	do {
 		err = nfs4_handle_exception(server,
@@ -3715,7 +3741,9 @@ static int _nfs4_lookup_root(struct nfs_server *server, struct nfs_fh *fhandle,
 static int nfs4_lookup_root(struct nfs_server *server, struct nfs_fh *fhandle,
 		struct nfs_fsinfo *info)
 {
-	struct nfs4_exception exception = { };
+	struct nfs4_exception exception = {
+		.interruptible = true,
+	};
 	int err;
 	do {
 		err = _nfs4_lookup_root(server, fhandle, info);
@@ -3942,7 +3970,9 @@ static int nfs4_proc_getattr(struct nfs_server *server, struct nfs_fh *fhandle,
 				struct nfs_fattr *fattr, struct nfs4_label *label,
 				struct inode *inode)
 {
-	struct nfs4_exception exception = { };
+	struct nfs4_exception exception = {
+		.interruptible = true,
+	};
 	int err;
 	do {
 		err = _nfs4_proc_getattr(server, fhandle, fattr, label, inode);
@@ -4065,7 +4095,9 @@ static int nfs4_proc_lookup_common(struct rpc_clnt **clnt, struct inode *dir,
 				   const struct qstr *name, struct nfs_fh *fhandle,
 				   struct nfs_fattr *fattr, struct nfs4_label *label)
 {
-	struct nfs4_exception exception = { };
+	struct nfs4_exception exception = {
+		.interruptible = true,
+	};
 	struct rpc_clnt *client = *clnt;
 	int err;
 	do {
@@ -4169,7 +4201,9 @@ static int _nfs4_proc_lookupp(struct inode *inode,
 static int nfs4_proc_lookupp(struct inode *inode, struct nfs_fh *fhandle,
 			     struct nfs_fattr *fattr, struct nfs4_label *label)
 {
-	struct nfs4_exception exception = { };
+	struct nfs4_exception exception = {
+		.interruptible = true,
+	};
 	int err;
 	do {
 		err = _nfs4_proc_lookupp(inode, fhandle, fattr, label);
@@ -4216,7 +4250,9 @@ static int _nfs4_proc_access(struct inode *inode, struct nfs_access_entry *entry
 
 static int nfs4_proc_access(struct inode *inode, struct nfs_access_entry *entry)
 {
-	struct nfs4_exception exception = { };
+	struct nfs4_exception exception = {
+		.interruptible = true,
+	};
 	int err;
 	do {
 		err = _nfs4_proc_access(inode, entry);
@@ -4271,7 +4307,9 @@ static int _nfs4_proc_readlink(struct inode *inode, struct page *page,
 static int nfs4_proc_readlink(struct inode *inode, struct page *page,
 		unsigned int pgbase, unsigned int pglen)
 {
-	struct nfs4_exception exception = { };
+	struct nfs4_exception exception = {
+		.interruptible = true,
+	};
 	int err;
 	do {
 		err = _nfs4_proc_readlink(inode, page, pgbase, pglen);
@@ -4347,7 +4385,9 @@ _nfs4_proc_remove(struct inode *dir, const struct qstr *name, u32 ftype)
 
 static int nfs4_proc_remove(struct inode *dir, struct dentry *dentry)
 {
-	struct nfs4_exception exception = { };
+	struct nfs4_exception exception = {
+		.interruptible = true,
+	};
 	struct inode *inode = d_inode(dentry);
 	int err;
 
@@ -4368,7 +4408,9 @@ static int nfs4_proc_remove(struct inode *dir, struct dentry *dentry)
 
 static int nfs4_proc_rmdir(struct inode *dir, const struct qstr *name)
 {
-	struct nfs4_exception exception = { };
+	struct nfs4_exception exception = {
+		.interruptible = true,
+	};
 	int err;
 
 	do {
@@ -4527,7 +4569,9 @@ out:
 
 static int nfs4_proc_link(struct inode *inode, struct inode *dir, const struct qstr *name)
 {
-	struct nfs4_exception exception = { };
+	struct nfs4_exception exception = {
+		.interruptible = true,
+	};
 	int err;
 	do {
 		err = nfs4_handle_exception(NFS_SERVER(inode),
@@ -4634,7 +4678,9 @@ out:
 static int nfs4_proc_symlink(struct inode *dir, struct dentry *dentry,
 		struct page *page, unsigned int len, struct iattr *sattr)
 {
-	struct nfs4_exception exception = { };
+	struct nfs4_exception exception = {
+		.interruptible = true,
+	};
 	struct nfs4_label l, *label = NULL;
 	int err;
 
@@ -4673,7 +4719,9 @@ static int nfs4_proc_mkdir(struct inode *dir, struct dentry *dentry,
 		struct iattr *sattr)
 {
 	struct nfs_server *server = NFS_SERVER(dir);
-	struct nfs4_exception exception = { };
+	struct nfs4_exception exception = {
+		.interruptible = true,
+	};
 	struct nfs4_label l, *label = NULL;
 	int err;
 
@@ -4733,7 +4781,9 @@ static int _nfs4_proc_readdir(struct dentry *dentry, const struct cred *cred,
 static int nfs4_proc_readdir(struct dentry *dentry, const struct cred *cred,
 		u64 cookie, struct page **pages, unsigned int count, bool plus)
 {
-	struct nfs4_exception exception = { };
+	struct nfs4_exception exception = {
+		.interruptible = true,
+	};
 	int err;
 	do {
 		err = _nfs4_proc_readdir(dentry, cred, cookie,
@@ -4784,7 +4834,9 @@ static int nfs4_proc_mknod(struct inode *dir, struct dentry *dentry,
 		struct iattr *sattr, dev_t rdev)
 {
 	struct nfs_server *server = NFS_SERVER(dir);
-	struct nfs4_exception exception = { };
+	struct nfs4_exception exception = {
+		.interruptible = true,
+	};
 	struct nfs4_label l, *label = NULL;
 	int err;
 
@@ -4826,7 +4878,9 @@ static int _nfs4_proc_statfs(struct nfs_server *server, struct nfs_fh *fhandle,
 
 static int nfs4_proc_statfs(struct nfs_server *server, struct nfs_fh *fhandle, struct nfs_fsstat *fsstat)
 {
-	struct nfs4_exception exception = { };
+	struct nfs4_exception exception = {
+		.interruptible = true,
+	};
 	int err;
 	do {
 		err = nfs4_handle_exception(server,
@@ -4857,7 +4911,9 @@ static int _nfs4_do_fsinfo(struct nfs_server *server, struct nfs_fh *fhandle,
 
 static int nfs4_do_fsinfo(struct nfs_server *server, struct nfs_fh *fhandle, struct nfs_fsinfo *fsinfo)
 {
-	struct nfs4_exception exception = { };
+	struct nfs4_exception exception = {
+		.interruptible = true,
+	};
 	unsigned long now = jiffies;
 	int err;
 
@@ -4919,7 +4975,9 @@ static int _nfs4_proc_pathconf(struct nfs_server *server, struct nfs_fh *fhandle
 static int nfs4_proc_pathconf(struct nfs_server *server, struct nfs_fh *fhandle,
 		struct nfs_pathconf *pathconf)
 {
-	struct nfs4_exception exception = { };
+	struct nfs4_exception exception = {
+		.interruptible = true,
+	};
 	int err;
 
 	do {
@@ -5488,7 +5546,9 @@ out_free:
 
 static ssize_t nfs4_get_acl_uncached(struct inode *inode, void *buf, size_t buflen)
 {
-	struct nfs4_exception exception = { };
+	struct nfs4_exception exception = {
+		.interruptible = true,
+	};
 	ssize_t ret;
 	do {
 		ret = __nfs4_get_acl_uncached(inode, buf, buflen);
@@ -5622,7 +5682,9 @@ static int _nfs4_get_security_label(struct inode *inode, void *buf,
 static int nfs4_get_security_label(struct inode *inode, void *buf,
 					size_t buflen)
 {
-	struct nfs4_exception exception = { };
+	struct nfs4_exception exception = {
+		.interruptible = true,
+	};
 	int err;
 
 	if (!nfs_server_capable(inode, NFS_CAP_SECURITY_LABEL))
@@ -6263,7 +6325,9 @@ out:
 
 static int nfs4_proc_getlk(struct nfs4_state *state, int cmd, struct file_lock *request)
 {
-	struct nfs4_exception exception = { };
+	struct nfs4_exception exception = {
+		.interruptible = true,
+	};
 	int err;
 
 	do {
@@ -6827,6 +6891,7 @@ static int nfs4_proc_setlk(struct nfs4_state *state, int cmd, struct file_lock *
 	struct nfs4_exception exception = {
 		.state = state,
 		.inode = state->inode,
+		.interruptible = true,
 	};
 	int err;
 
@@ -7240,7 +7305,9 @@ int nfs4_proc_fs_locations(struct rpc_clnt *client, struct inode *dir,
 			   struct nfs4_fs_locations *fs_locations,
 			   struct page *page)
 {
-	struct nfs4_exception exception = { };
+	struct nfs4_exception exception = {
+		.interruptible = true,
+	};
 	int err;
 	do {
 		err = _nfs4_proc_fs_locations(client, dir, name,
@@ -7383,7 +7450,9 @@ int nfs4_proc_get_locations(struct inode *inode,
 	struct nfs_client *clp = server->nfs_client;
 	const struct nfs4_mig_recovery_ops *ops =
 					clp->cl_mvops->mig_recovery_ops;
-	struct nfs4_exception exception = { };
+	struct nfs4_exception exception = {
+		.interruptible = true,
+	};
 	int status;
 
 	dprintk("%s: FSID %llx:%llx on \"%s\"\n", __func__,
@@ -7507,7 +7576,9 @@ int nfs4_proc_fsid_present(struct inode *inode, const struct cred *cred)
 	struct nfs_client *clp = server->nfs_client;
 	const struct nfs4_mig_recovery_ops *ops =
 					clp->cl_mvops->mig_recovery_ops;
-	struct nfs4_exception exception = { };
+	struct nfs4_exception exception = {
+		.interruptible = true,
+	};
 	int status;
 
 	dprintk("%s: FSID %llx:%llx on \"%s\"\n", __func__,
@@ -7573,7 +7644,9 @@ static int _nfs4_proc_secinfo(struct inode *dir, const struct qstr *name, struct
 int nfs4_proc_secinfo(struct inode *dir, const struct qstr *name,
 		      struct nfs4_secinfo_flavors *flavors)
 {
-	struct nfs4_exception exception = { };
+	struct nfs4_exception exception = {
+		.interruptible = true,
+	};
 	int err;
 	do {
 		err = -NFS4ERR_WRONGSEC;
@@ -9263,7 +9336,9 @@ static int
 nfs41_proc_secinfo_no_name(struct nfs_server *server, struct nfs_fh *fhandle,
 			   struct nfs_fsinfo *info, struct nfs4_secinfo_flavors *flavors)
 {
-	struct nfs4_exception exception = { };
+	struct nfs4_exception exception = {
+		.interruptible = true,
+	};
 	int err;
 	do {
 		/* first try using integrity protection */
@@ -9430,7 +9505,9 @@ static int nfs41_test_stateid(struct nfs_server *server,
 		nfs4_stateid *stateid,
 		const struct cred *cred)
 {
-	struct nfs4_exception exception = { };
+	struct nfs4_exception exception = {
+		.interruptible = true,
+	};
 	int err;
 	do {
 		err = _nfs41_test_stateid(server, stateid, cred);
diff --git a/fs/nfs/nfs4state.c b/fs/nfs/nfs4state.c
index 3de36479ed7a..e2e3c4f04d3e 100644
--- a/fs/nfs/nfs4state.c
+++ b/fs/nfs/nfs4state.c
@@ -159,6 +159,10 @@ int nfs40_discover_server_trunking(struct nfs_client *clp,
 		/* Sustain the lease, even if it's empty.  If the clientid4
 		 * goes stale it's of no use for trunking discovery. */
 		nfs4_schedule_state_renewal(*result);
+
+		/* If the client state need to recover, do it. */
+		if (clp->cl_state)
+			nfs4_schedule_state_manager(clp);
 	}
 out:
 	return status;
@@ -2346,8 +2350,7 @@ static void nfs41_handle_recallable_state_revoked(struct nfs_client *clp)
 {
 	/* FIXME: For now, we destroy all layouts. */
 	pnfs_destroy_all_layouts(clp);
-	/* FIXME: For now, we test all delegations+open state+locks. */
-	nfs41_handle_some_state_revoked(clp);
+	nfs_test_expired_all_delegations(clp);
 	dprintk("%s: Recallable state revoked on server %s!\n", __func__,
 			clp->cl_hostname);
 }
diff --git a/fs/nfs/pagelist.c b/fs/nfs/pagelist.c
index e9f39fa5964b..6ec30014a439 100644
--- a/fs/nfs/pagelist.c
+++ b/fs/nfs/pagelist.c
@@ -16,8 +16,8 @@
 #include <linux/nfs.h>
 #include <linux/nfs3.h>
 #include <linux/nfs4.h>
-#include <linux/nfs_page.h>
 #include <linux/nfs_fs.h>
+#include <linux/nfs_page.h>
 #include <linux/nfs_mount.h>
 #include <linux/export.h>
 
@@ -47,7 +47,7 @@ void nfs_pgheader_init(struct nfs_pageio_descriptor *desc,
 
 	hdr->req = nfs_list_entry(mirror->pg_list.next);
 	hdr->inode = desc->pg_inode;
-	hdr->cred = hdr->req->wb_context->cred;
+	hdr->cred = nfs_req_openctx(hdr->req)->cred;
 	hdr->io_start = req_offset(hdr->req);
 	hdr->good_bytes = mirror->pg_count;
 	hdr->io_completion = desc->pg_io_completion;
@@ -295,25 +295,13 @@ out:
 		nfs_release_request(head);
 }
 
-/**
- * nfs_create_request - Create an NFS read/write request.
- * @ctx: open context to use
- * @page: page to write
- * @last: last nfs request created for this page group or NULL if head
- * @offset: starting offset within the page for the write
- * @count: number of bytes to read/write
- *
- * The page must be locked by the caller. This makes sure we never
- * create two different requests for the same page.
- * User should ensure it is safe to sleep in this function.
- */
-struct nfs_page *
-nfs_create_request(struct nfs_open_context *ctx, struct page *page,
-		   struct nfs_page *last, unsigned int offset,
+static struct nfs_page *
+__nfs_create_request(struct nfs_lock_context *l_ctx, struct page *page,
+		   unsigned int pgbase, unsigned int offset,
 		   unsigned int count)
 {
 	struct nfs_page		*req;
-	struct nfs_lock_context *l_ctx;
+	struct nfs_open_context *ctx = l_ctx->open_context;
 
 	if (test_bit(NFS_CONTEXT_BAD, &ctx->flags))
 		return ERR_PTR(-EBADF);
@@ -322,13 +310,8 @@ nfs_create_request(struct nfs_open_context *ctx, struct page *page,
 	if (req == NULL)
 		return ERR_PTR(-ENOMEM);
 
-	/* get lock context early so we can deal with alloc failures */
-	l_ctx = nfs_get_lock_context(ctx);
-	if (IS_ERR(l_ctx)) {
-		nfs_page_free(req);
-		return ERR_CAST(l_ctx);
-	}
 	req->wb_lock_context = l_ctx;
+	refcount_inc(&l_ctx->count);
 	atomic_inc(&l_ctx->io_count);
 
 	/* Initialize the request struct. Initially, we assume a
@@ -340,15 +323,59 @@ nfs_create_request(struct nfs_open_context *ctx, struct page *page,
 		get_page(page);
 	}
 	req->wb_offset  = offset;
-	req->wb_pgbase	= offset;
+	req->wb_pgbase	= pgbase;
 	req->wb_bytes   = count;
-	req->wb_context = get_nfs_open_context(ctx);
 	kref_init(&req->wb_kref);
-	nfs_page_group_init(req, last);
+	req->wb_nio = 0;
 	return req;
 }
 
 /**
+ * nfs_create_request - Create an NFS read/write request.
+ * @ctx: open context to use
+ * @page: page to write
+ * @offset: starting offset within the page for the write
+ * @count: number of bytes to read/write
+ *
+ * The page must be locked by the caller. This makes sure we never
+ * create two different requests for the same page.
+ * User should ensure it is safe to sleep in this function.
+ */
+struct nfs_page *
+nfs_create_request(struct nfs_open_context *ctx, struct page *page,
+		   unsigned int offset, unsigned int count)
+{
+	struct nfs_lock_context *l_ctx = nfs_get_lock_context(ctx);
+	struct nfs_page *ret;
+
+	if (IS_ERR(l_ctx))
+		return ERR_CAST(l_ctx);
+	ret = __nfs_create_request(l_ctx, page, offset, offset, count);
+	if (!IS_ERR(ret))
+		nfs_page_group_init(ret, NULL);
+	nfs_put_lock_context(l_ctx);
+	return ret;
+}
+
+static struct nfs_page *
+nfs_create_subreq(struct nfs_page *req, struct nfs_page *last,
+		  unsigned int pgbase, unsigned int offset,
+		  unsigned int count)
+{
+	struct nfs_page *ret;
+
+	ret = __nfs_create_request(req->wb_lock_context, req->wb_page,
+			pgbase, offset, count);
+	if (!IS_ERR(ret)) {
+		nfs_lock_request(ret);
+		ret->wb_index = req->wb_index;
+		nfs_page_group_init(ret, last);
+		ret->wb_nio = req->wb_nio;
+	}
+	return ret;
+}
+
+/**
  * nfs_unlock_request - Unlock request and wake up sleepers.
  * @req: pointer to request
  */
@@ -386,8 +413,8 @@ void nfs_unlock_and_release_request(struct nfs_page *req)
 static void nfs_clear_request(struct nfs_page *req)
 {
 	struct page *page = req->wb_page;
-	struct nfs_open_context *ctx = req->wb_context;
 	struct nfs_lock_context *l_ctx = req->wb_lock_context;
+	struct nfs_open_context *ctx;
 
 	if (page != NULL) {
 		put_page(page);
@@ -396,16 +423,13 @@ static void nfs_clear_request(struct nfs_page *req)
 	if (l_ctx != NULL) {
 		if (atomic_dec_and_test(&l_ctx->io_count)) {
 			wake_up_var(&l_ctx->io_count);
+			ctx = l_ctx->open_context;
 			if (test_bit(NFS_CONTEXT_UNLOCK, &ctx->flags))
 				rpc_wake_up(&NFS_SERVER(d_inode(ctx->dentry))->uoc_rpcwaitq);
 		}
 		nfs_put_lock_context(l_ctx);
 		req->wb_lock_context = NULL;
 	}
-	if (ctx != NULL) {
-		put_nfs_open_context(ctx);
-		req->wb_context = NULL;
-	}
 }
 
 /**
@@ -550,7 +574,7 @@ static void nfs_pgio_rpcsetup(struct nfs_pgio_header *hdr,
 	hdr->args.pgbase = req->wb_pgbase;
 	hdr->args.pages  = hdr->page_array.pagevec;
 	hdr->args.count  = count;
-	hdr->args.context = get_nfs_open_context(req->wb_context);
+	hdr->args.context = get_nfs_open_context(nfs_req_openctx(req));
 	hdr->args.lock_context = req->wb_lock_context;
 	hdr->args.stable  = NFS_UNSTABLE;
 	switch (how & (FLUSH_STABLE | FLUSH_COND_STABLE)) {
@@ -698,6 +722,7 @@ void nfs_pageio_init(struct nfs_pageio_descriptor *desc,
 	desc->pg_mirrors_dynamic = NULL;
 	desc->pg_mirrors = desc->pg_mirrors_static;
 	nfs_pageio_mirror_init(&desc->pg_mirrors[0], bsize);
+	desc->pg_maxretrans = 0;
 }
 
 /**
@@ -906,9 +931,9 @@ static bool nfs_can_coalesce_requests(struct nfs_page *prev,
 	struct file_lock_context *flctx;
 
 	if (prev) {
-		if (!nfs_match_open_context(req->wb_context, prev->wb_context))
+		if (!nfs_match_open_context(nfs_req_openctx(req), nfs_req_openctx(prev)))
 			return false;
-		flctx = d_inode(req->wb_context->dentry)->i_flctx;
+		flctx = d_inode(nfs_req_openctx(req)->dentry)->i_flctx;
 		if (flctx != NULL &&
 		    !(list_empty_careful(&flctx->flc_posix) &&
 		      list_empty_careful(&flctx->flc_flock)) &&
@@ -957,6 +982,15 @@ static int nfs_pageio_do_add_request(struct nfs_pageio_descriptor *desc,
 			return 0;
 		mirror->pg_base = req->wb_pgbase;
 	}
+
+	if (desc->pg_maxretrans && req->wb_nio > desc->pg_maxretrans) {
+		if (NFS_SERVER(desc->pg_inode)->flags & NFS_MOUNT_SOFTERR)
+			desc->pg_error = -ETIMEDOUT;
+		else
+			desc->pg_error = -EIO;
+		return 0;
+	}
+
 	if (!nfs_can_coalesce_requests(prev, req, desc))
 		return 0;
 	nfs_list_move_request(req, &mirror->pg_list);
@@ -1049,14 +1083,10 @@ static int __nfs_pageio_add_request(struct nfs_pageio_descriptor *desc,
 		pgbase += subreq->wb_bytes;
 
 		if (bytes_left) {
-			subreq = nfs_create_request(req->wb_context,
-					req->wb_page,
-					subreq, pgbase, bytes_left);
+			subreq = nfs_create_subreq(req, subreq, pgbase,
+					offset, bytes_left);
 			if (IS_ERR(subreq))
 				goto err_ptr;
-			nfs_lock_request(subreq);
-			subreq->wb_offset  = offset;
-			subreq->wb_index = req->wb_index;
 		}
 	} while (bytes_left > 0);
 
@@ -1158,19 +1188,14 @@ int nfs_pageio_add_request(struct nfs_pageio_descriptor *desc,
 			     lastreq = lastreq->wb_this_page)
 				;
 
-			dupreq = nfs_create_request(req->wb_context,
-					req->wb_page, lastreq, pgbase, bytes);
+			dupreq = nfs_create_subreq(req, lastreq,
+					pgbase, offset, bytes);
 
+			nfs_page_group_unlock(req);
 			if (IS_ERR(dupreq)) {
-				nfs_page_group_unlock(req);
 				desc->pg_error = PTR_ERR(dupreq);
 				goto out_failed;
 			}
-
-			nfs_lock_request(dupreq);
-			nfs_page_group_unlock(req);
-			dupreq->wb_offset = offset;
-			dupreq->wb_index = req->wb_index;
 		} else
 			dupreq = req;
 
diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index 7066cd7c7aff..83722e936b4a 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -2436,7 +2436,7 @@ pnfs_generic_pg_init_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *r
 			rd_size = nfs_dreq_bytes_left(pgio->pg_dreq);
 
 		pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
-						   req->wb_context,
+						   nfs_req_openctx(req),
 						   req_offset(req),
 						   rd_size,
 						   IOMODE_READ,
@@ -2463,7 +2463,7 @@ pnfs_generic_pg_init_write(struct nfs_pageio_descriptor *pgio,
 	pnfs_generic_pg_check_range(pgio, req);
 	if (pgio->pg_lseg == NULL) {
 		pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
-						   req->wb_context,
+						   nfs_req_openctx(req),
 						   req_offset(req),
 						   wb_size,
 						   IOMODE_RW,
diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
index c0420b979d88..f15609c003d8 100644
--- a/fs/nfs/pnfs.h
+++ b/fs/nfs/pnfs.h
@@ -459,7 +459,7 @@ static inline bool
 pnfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg,
 			 struct nfs_commit_info *cinfo, u32 ds_commit_idx)
 {
-	struct inode *inode = d_inode(req->wb_context->dentry);
+	struct inode *inode = d_inode(nfs_req_openctx(req)->dentry);
 	struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld;
 
 	if (lseg == NULL || ld->mark_request_commit == NULL)
@@ -471,7 +471,7 @@ pnfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg,
 static inline bool
 pnfs_clear_request_commit(struct nfs_page *req, struct nfs_commit_info *cinfo)
 {
-	struct inode *inode = d_inode(req->wb_context->dentry);
+	struct inode *inode = d_inode(nfs_req_openctx(req)->dentry);
 	struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld;
 
 	if (ld == NULL || ld->clear_request_commit == NULL)
diff --git a/fs/nfs/read.c b/fs/nfs/read.c
index 1d95a60b2586..c799e540ed1e 100644
--- a/fs/nfs/read.c
+++ b/fs/nfs/read.c
@@ -92,7 +92,7 @@ EXPORT_SYMBOL_GPL(nfs_pageio_reset_read_mds);
 
 static void nfs_readpage_release(struct nfs_page *req)
 {
-	struct inode *inode = d_inode(req->wb_context->dentry);
+	struct inode *inode = d_inode(nfs_req_openctx(req)->dentry);
 
 	dprintk("NFS: read done (%s/%llu %d@%lld)\n", inode->i_sb->s_id,
 		(unsigned long long)NFS_FILEID(inode), req->wb_bytes,
@@ -118,7 +118,7 @@ int nfs_readpage_async(struct nfs_open_context *ctx, struct inode *inode,
 	len = nfs_page_length(page);
 	if (len == 0)
 		return nfs_return_empty_page(page);
-	new = nfs_create_request(ctx, page, NULL, 0, len);
+	new = nfs_create_request(ctx, page, 0, len);
 	if (IS_ERR(new)) {
 		unlock_page(page);
 		return PTR_ERR(new);
@@ -363,7 +363,7 @@ readpage_async_filler(void *data, struct page *page)
 	if (len == 0)
 		return nfs_return_empty_page(page);
 
-	new = nfs_create_request(desc->ctx, page, NULL, 0, len);
+	new = nfs_create_request(desc->ctx, page, 0, len);
 	if (IS_ERR(new))
 		goto out_error;
 
diff --git a/fs/nfs/super.c b/fs/nfs/super.c
index 450ae77d19bf..d6c687419a81 100644
--- a/fs/nfs/super.c
+++ b/fs/nfs/super.c
@@ -78,7 +78,7 @@
 
 enum {
 	/* Mount options that take no arguments */
-	Opt_soft, Opt_hard,
+	Opt_soft, Opt_softerr, Opt_hard,
 	Opt_posix, Opt_noposix,
 	Opt_cto, Opt_nocto,
 	Opt_ac, Opt_noac,
@@ -125,6 +125,7 @@ static const match_table_t nfs_mount_option_tokens = {
 	{ Opt_sloppy, "sloppy" },
 
 	{ Opt_soft, "soft" },
+	{ Opt_softerr, "softerr" },
 	{ Opt_hard, "hard" },
 	{ Opt_deprecated, "intr" },
 	{ Opt_deprecated, "nointr" },
@@ -628,7 +629,8 @@ static void nfs_show_mount_options(struct seq_file *m, struct nfs_server *nfss,
 		const char *str;
 		const char *nostr;
 	} nfs_info[] = {
-		{ NFS_MOUNT_SOFT, ",soft", ",hard" },
+		{ NFS_MOUNT_SOFT, ",soft", "" },
+		{ NFS_MOUNT_SOFTERR, ",softerr", "" },
 		{ NFS_MOUNT_POSIX, ",posix", "" },
 		{ NFS_MOUNT_NOCTO, ",nocto", "" },
 		{ NFS_MOUNT_NOAC, ",noac", "" },
@@ -658,6 +660,8 @@ static void nfs_show_mount_options(struct seq_file *m, struct nfs_server *nfss,
 		seq_printf(m, ",acdirmin=%u", nfss->acdirmin/HZ);
 	if (nfss->acdirmax != NFS_DEF_ACDIRMAX*HZ || showdefaults)
 		seq_printf(m, ",acdirmax=%u", nfss->acdirmax/HZ);
+	if (!(nfss->flags & (NFS_MOUNT_SOFT|NFS_MOUNT_SOFTERR)))
+			seq_puts(m, ",hard");
 	for (nfs_infop = nfs_info; nfs_infop->flag; nfs_infop++) {
 		if (nfss->flags & nfs_infop->flag)
 			seq_puts(m, nfs_infop->str);
@@ -1239,10 +1243,15 @@ static int nfs_parse_mount_options(char *raw,
 		 */
 		case Opt_soft:
 			mnt->flags |= NFS_MOUNT_SOFT;
+			mnt->flags &= ~NFS_MOUNT_SOFTERR;
 			break;
-		case Opt_hard:
+		case Opt_softerr:
+			mnt->flags |= NFS_MOUNT_SOFTERR;
 			mnt->flags &= ~NFS_MOUNT_SOFT;
 			break;
+		case Opt_hard:
+			mnt->flags &= ~(NFS_MOUNT_SOFT|NFS_MOUNT_SOFTERR);
+			break;
 		case Opt_posix:
 			mnt->flags |= NFS_MOUNT_POSIX;
 			break;
@@ -2476,6 +2485,21 @@ static int nfs_compare_super_address(struct nfs_server *server1,
 	return 1;
 }
 
+static int nfs_compare_userns(const struct nfs_server *old,
+		const struct nfs_server *new)
+{
+	const struct user_namespace *oldns = &init_user_ns;
+	const struct user_namespace *newns = &init_user_ns;
+
+	if (old->client && old->client->cl_cred)
+		oldns = old->client->cl_cred->user_ns;
+	if (new->client && new->client->cl_cred)
+		newns = new->client->cl_cred->user_ns;
+	if (oldns != newns)
+		return 0;
+	return 1;
+}
+
 static int nfs_compare_super(struct super_block *sb, void *data)
 {
 	struct nfs_sb_mountdata *sb_mntdata = data;
@@ -2489,6 +2513,8 @@ static int nfs_compare_super(struct super_block *sb, void *data)
 		return 0;
 	if (memcmp(&old->fsid, &server->fsid, sizeof(old->fsid)) != 0)
 		return 0;
+	if (!nfs_compare_userns(old, server))
+		return 0;
 	return nfs_compare_mount_options(sb, server, mntflags);
 }
 
diff --git a/fs/nfs/symlink.c b/fs/nfs/symlink.c
index 06eb44b47885..25ba299fdac2 100644
--- a/fs/nfs/symlink.c
+++ b/fs/nfs/symlink.c
@@ -26,8 +26,9 @@
  * and straight-forward than readdir caching.
  */
 
-static int nfs_symlink_filler(struct inode *inode, struct page *page)
+static int nfs_symlink_filler(void *data, struct page *page)
 {
+	struct inode *inode = data;
 	int error;
 
 	error = NFS_PROTO(inode)->readlink(inode, page, 0, PAGE_SIZE);
@@ -65,8 +66,8 @@ static const char *nfs_get_link(struct dentry *dentry,
 		err = ERR_PTR(nfs_revalidate_mapping(inode, inode->i_mapping));
 		if (err)
 			return err;
-		page = read_cache_page(&inode->i_data, 0,
-					(filler_t *)nfs_symlink_filler, inode);
+		page = read_cache_page(&inode->i_data, 0, nfs_symlink_filler,
+				inode);
 		if (IS_ERR(page))
 			return ERR_CAST(page);
 	}
diff --git a/fs/nfs/write.c b/fs/nfs/write.c
index f3ebabaa291d..bc5bb9323412 100644
--- a/fs/nfs/write.c
+++ b/fs/nfs/write.c
@@ -244,6 +244,12 @@ static void nfs_set_pageerror(struct address_space *mapping)
 	nfs_zap_mapping(mapping->host, mapping);
 }
 
+static void nfs_mapping_set_error(struct page *page, int error)
+{
+	SetPageError(page);
+	mapping_set_error(page_file_mapping(page), error);
+}
+
 /*
  * nfs_page_group_search_locked
  * @head - head request of page group
@@ -582,11 +588,10 @@ release_request:
 	return ERR_PTR(ret);
 }
 
-static void nfs_write_error_remove_page(struct nfs_page *req)
+static void nfs_write_error(struct nfs_page *req, int error)
 {
+	nfs_mapping_set_error(req->wb_page, error);
 	nfs_end_page_writeback(req);
-	generic_error_remove_page(page_file_mapping(req->wb_page),
-				  req->wb_page);
 	nfs_release_request(req);
 }
 
@@ -609,6 +614,7 @@ nfs_error_is_fatal_on_server(int err)
 static int nfs_page_async_flush(struct nfs_pageio_descriptor *pgio,
 				struct page *page)
 {
+	struct address_space *mapping;
 	struct nfs_page *req;
 	int ret = 0;
 
@@ -622,19 +628,19 @@ static int nfs_page_async_flush(struct nfs_pageio_descriptor *pgio,
 	nfs_set_page_writeback(page);
 	WARN_ON_ONCE(test_bit(PG_CLEAN, &req->wb_flags));
 
-	ret = req->wb_context->error;
 	/* If there is a fatal error that covers this write, just exit */
-	if (nfs_error_is_fatal_on_server(ret))
+	ret = 0;
+	mapping = page_file_mapping(page);
+	if (test_bit(AS_ENOSPC, &mapping->flags) ||
+	    test_bit(AS_EIO, &mapping->flags))
 		goto out_launder;
 
-	ret = 0;
 	if (!nfs_pageio_add_request(pgio, req)) {
 		ret = pgio->pg_error;
 		/*
 		 * Remove the problematic req upon fatal errors on the server
 		 */
 		if (nfs_error_is_fatal(ret)) {
-			nfs_context_set_write_error(req->wb_context, ret);
 			if (nfs_error_is_fatal_on_server(ret))
 				goto out_launder;
 		} else
@@ -646,8 +652,8 @@ static int nfs_page_async_flush(struct nfs_pageio_descriptor *pgio,
 out:
 	return ret;
 out_launder:
-	nfs_write_error_remove_page(req);
-	return ret;
+	nfs_write_error(req, ret);
+	return 0;
 }
 
 static int nfs_do_writepage(struct page *page, struct writeback_control *wbc,
@@ -958,7 +964,8 @@ static void
 nfs_clear_request_commit(struct nfs_page *req)
 {
 	if (test_bit(PG_CLEAN, &req->wb_flags)) {
-		struct inode *inode = d_inode(req->wb_context->dentry);
+		struct nfs_open_context *ctx = nfs_req_openctx(req);
+		struct inode *inode = d_inode(ctx->dentry);
 		struct nfs_commit_info cinfo;
 
 		nfs_init_cinfo_from_inode(&cinfo, inode);
@@ -999,10 +1006,12 @@ static void nfs_write_completion(struct nfs_pgio_header *hdr)
 		if (test_bit(NFS_IOHDR_ERROR, &hdr->flags) &&
 		    (hdr->good_bytes < bytes)) {
 			nfs_set_pageerror(page_file_mapping(req->wb_page));
-			nfs_context_set_write_error(req->wb_context, hdr->error);
+			nfs_mapping_set_error(req->wb_page, hdr->error);
 			goto remove_req;
 		}
 		if (nfs_write_need_commit(hdr)) {
+			/* Reset wb_nio, since the write was successful. */
+			req->wb_nio = 0;
 			memcpy(&req->wb_verf, &hdr->verf.verifier, sizeof(req->wb_verf));
 			nfs_mark_request_commit(req, hdr->lseg, &cinfo,
 				hdr->pgio_mirror_idx);
@@ -1136,6 +1145,7 @@ static struct nfs_page *nfs_try_to_update_request(struct inode *inode,
 		req->wb_bytes = end - req->wb_offset;
 	else
 		req->wb_bytes = rqend - req->wb_offset;
+	req->wb_nio = 0;
 	return req;
 out_flushme:
 	/*
@@ -1165,7 +1175,7 @@ static struct nfs_page * nfs_setup_write_request(struct nfs_open_context* ctx,
 	req = nfs_try_to_update_request(inode, page, offset, bytes);
 	if (req != NULL)
 		goto out;
-	req = nfs_create_request(ctx, page, NULL, offset, bytes);
+	req = nfs_create_request(ctx, page, offset, bytes);
 	if (IS_ERR(req))
 		goto out;
 	nfs_inode_add_request(inode, req);
@@ -1210,7 +1220,7 @@ int nfs_flush_incompatible(struct file *file, struct page *page)
 			return 0;
 		l_ctx = req->wb_lock_context;
 		do_flush = req->wb_page != page ||
-			!nfs_match_open_context(req->wb_context, ctx);
+			!nfs_match_open_context(nfs_req_openctx(req), ctx);
 		if (l_ctx && flctx &&
 		    !(list_empty_careful(&flctx->flc_posix) &&
 		      list_empty_careful(&flctx->flc_flock))) {
@@ -1410,8 +1420,10 @@ static void nfs_initiate_write(struct nfs_pgio_header *hdr,
  */
 static void nfs_redirty_request(struct nfs_page *req)
 {
+	/* Bump the transmission count */
+	req->wb_nio++;
 	nfs_mark_request_dirty(req);
-	set_bit(NFS_CONTEXT_RESEND_WRITES, &req->wb_context->flags);
+	set_bit(NFS_CONTEXT_RESEND_WRITES, &nfs_req_openctx(req)->flags);
 	nfs_end_page_writeback(req);
 	nfs_release_request(req);
 }
@@ -1423,14 +1435,10 @@ static void nfs_async_write_error(struct list_head *head, int error)
 	while (!list_empty(head)) {
 		req = nfs_list_entry(head->next);
 		nfs_list_remove_request(req);
-		if (nfs_error_is_fatal(error)) {
-			nfs_context_set_write_error(req->wb_context, error);
-			if (nfs_error_is_fatal_on_server(error)) {
-				nfs_write_error_remove_page(req);
-				continue;
-			}
-		}
-		nfs_redirty_request(req);
+		if (nfs_error_is_fatal(error))
+			nfs_write_error(req, error);
+		else
+			nfs_redirty_request(req);
 	}
 }
 
@@ -1735,7 +1743,8 @@ void nfs_init_commit(struct nfs_commit_data *data,
 		     struct nfs_commit_info *cinfo)
 {
 	struct nfs_page *first = nfs_list_entry(head->next);
-	struct inode *inode = d_inode(first->wb_context->dentry);
+	struct nfs_open_context *ctx = nfs_req_openctx(first);
+	struct inode *inode = d_inode(ctx->dentry);
 
 	/* Set up the RPC argument and reply structs
 	 * NB: take care not to mess about with data->commit et al. */
@@ -1743,7 +1752,7 @@ void nfs_init_commit(struct nfs_commit_data *data,
 	list_splice_init(head, &data->pages);
 
 	data->inode	  = inode;
-	data->cred	  = first->wb_context->cred;
+	data->cred	  = ctx->cred;
 	data->lseg	  = lseg; /* reference transferred */
 	/* only set lwb for pnfs commit */
 	if (lseg)
@@ -1756,7 +1765,7 @@ void nfs_init_commit(struct nfs_commit_data *data,
 	/* Note: we always request a commit of the entire inode */
 	data->args.offset = 0;
 	data->args.count  = 0;
-	data->context     = get_nfs_open_context(first->wb_context);
+	data->context     = get_nfs_open_context(ctx);
 	data->res.fattr   = &data->fattr;
 	data->res.verf    = &data->verf;
 	nfs_fattr_init(&data->fattr);
@@ -1839,14 +1848,15 @@ static void nfs_commit_release_pages(struct nfs_commit_data *data)
 			nfs_clear_page_commit(req->wb_page);
 
 		dprintk("NFS:       commit (%s/%llu %d@%lld)",
-			req->wb_context->dentry->d_sb->s_id,
-			(unsigned long long)NFS_FILEID(d_inode(req->wb_context->dentry)),
+			nfs_req_openctx(req)->dentry->d_sb->s_id,
+			(unsigned long long)NFS_FILEID(d_inode(nfs_req_openctx(req)->dentry)),
 			req->wb_bytes,
 			(long long)req_offset(req));
 		if (status < 0) {
-			nfs_context_set_write_error(req->wb_context, status);
-			if (req->wb_page)
+			if (req->wb_page) {
+				nfs_mapping_set_error(req->wb_page, status);
 				nfs_inode_remove_request(req);
+			}
 			dprintk_cont(", error = %d\n", status);
 			goto next;
 		}
@@ -1863,7 +1873,7 @@ static void nfs_commit_release_pages(struct nfs_commit_data *data)
 		/* We have a mismatch. Write the page again */
 		dprintk_cont(" mismatch\n");
 		nfs_mark_request_dirty(req);
-		set_bit(NFS_CONTEXT_RESEND_WRITES, &req->wb_context->flags);
+		set_bit(NFS_CONTEXT_RESEND_WRITES, &nfs_req_openctx(req)->flags);
 	next:
 		nfs_unlock_and_release_request(req);
 		/* Latency breaker */
diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c
index 7caa3801ce72..9b93e7a9a26d 100644
--- a/fs/nfsd/nfs4callback.c
+++ b/fs/nfsd/nfs4callback.c
@@ -868,6 +868,7 @@ static int setup_callback_client(struct nfs4_client *clp, struct nfs4_cb_conn *c
 		.program	= &cb_program,
 		.version	= 1,
 		.flags		= (RPC_CLNT_CREATE_NOPING | RPC_CLNT_CREATE_QUIET),
+		.cred		= current_cred(),
 	};
 	struct rpc_clnt *client;
 	const struct cred *cred;
@@ -1033,7 +1034,7 @@ static bool nfsd4_cb_sequence_done(struct rpc_task *task, struct nfsd4_callback
 		 * the submission code will error out, so we don't need to
 		 * handle that case here.
 		 */
-		if (task->tk_flags & RPC_TASK_KILLED)
+		if (RPC_SIGNALLED(task))
 			goto need_restart;
 
 		return true;
@@ -1086,7 +1087,7 @@ static bool nfsd4_cb_sequence_done(struct rpc_task *task, struct nfsd4_callback
 	dprintk("%s: freed slot, new seqid=%d\n", __func__,
 		clp->cl_cb_session->se_cb_seq_nr);
 
-	if (task->tk_flags & RPC_TASK_KILLED)
+	if (RPC_SIGNALLED(task))
 		goto need_restart;
 out:
 	return ret;
diff --git a/fs/orangefs/acl.c b/fs/orangefs/acl.c
index 72d2ff17d27b..eced272a3c57 100644
--- a/fs/orangefs/acl.c
+++ b/fs/orangefs/acl.c
@@ -142,7 +142,7 @@ int orangefs_set_acl(struct inode *inode, struct posix_acl *acl, int type)
 			rc = __orangefs_set_acl(inode, acl, type);
 		} else {
 			iattr.ia_valid = ATTR_MODE;
-			rc = orangefs_inode_setattr(inode, &iattr);
+			rc = __orangefs_setattr(inode, &iattr);
 		}
 
 		return rc;
@@ -185,7 +185,7 @@ int orangefs_init_acl(struct inode *inode, struct inode *dir)
 		inode->i_mode = mode;
 		iattr.ia_mode = mode;
 		iattr.ia_valid |= ATTR_MODE;
-		orangefs_inode_setattr(inode, &iattr);
+		__orangefs_setattr(inode, &iattr);
 	}
 
 	return error;
diff --git a/fs/orangefs/file.c b/fs/orangefs/file.c
index b094d3d79354..a35c17017210 100644
--- a/fs/orangefs/file.c
+++ b/fs/orangefs/file.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
  * (C) 2001 Clemson University and The University of Chicago
+ * Copyright 2018 Omnibond Systems, L.L.C.
  *
  * See COPYING in top-level directory.
  */
@@ -44,15 +45,16 @@ static int flush_racache(struct inode *inode)
 /*
  * Post and wait for the I/O upcall to finish
  */
-static ssize_t wait_for_direct_io(enum ORANGEFS_io_type type, struct inode *inode,
-		loff_t *offset, struct iov_iter *iter,
-		size_t total_size, loff_t readahead_size)
+ssize_t wait_for_direct_io(enum ORANGEFS_io_type type, struct inode *inode,
+    loff_t *offset, struct iov_iter *iter, size_t total_size,
+    loff_t readahead_size, struct orangefs_write_range *wr, int *index_return)
 {
 	struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode);
 	struct orangefs_khandle *handle = &orangefs_inode->refn.khandle;
 	struct orangefs_kernel_op_s *new_op = NULL;
 	int buffer_index = -1;
 	ssize_t ret;
+	size_t copy_amount;
 
 	new_op = op_alloc(ORANGEFS_VFS_OP_FILE_IO);
 	if (!new_op)
@@ -84,6 +86,10 @@ populate_shared_memory:
 	new_op->upcall.req.io.buf_index = buffer_index;
 	new_op->upcall.req.io.count = total_size;
 	new_op->upcall.req.io.offset = *offset;
+	if (type == ORANGEFS_IO_WRITE && wr) {
+		new_op->upcall.uid = from_kuid(&init_user_ns, wr->uid);
+		new_op->upcall.gid = from_kgid(&init_user_ns, wr->gid);
+	}
 
 	gossip_debug(GOSSIP_FILE_DEBUG,
 		     "%s(%pU): offset: %llu total_size: %zd\n",
@@ -168,7 +174,10 @@ populate_shared_memory:
 			 * trigger the write.
 			 */
 			case OP_VFS_STATE_INPROGR:
-				ret = total_size;
+				if (type == ORANGEFS_IO_READ)
+					ret = -EINTR;
+				else
+					ret = total_size;
 				break;
 			default:
 				gossip_err("%s: unexpected op state :%d:.\n",
@@ -204,8 +213,25 @@ populate_shared_memory:
 		 *       can futher be kernel-space or user-space addresses.
 		 *       or it can pointers to struct page's
 		 */
+
+		/*
+		 * When reading, readahead_size will only be zero when
+		 * we're doing O_DIRECT, otherwise we got here from
+		 * orangefs_readpage.
+		 *
+		 * If we got here from orangefs_readpage we want to
+		 * copy either a page or the whole file into the io
+		 * vector, whichever is smaller.
+		 */
+		if (readahead_size)
+			copy_amount =
+				min(new_op->downcall.resp.io.amt_complete,
+					(__s64)PAGE_SIZE);
+		else
+			copy_amount = new_op->downcall.resp.io.amt_complete;
+
 		ret = orangefs_bufmap_copy_to_iovec(iter, buffer_index,
-		    new_op->downcall.resp.io.amt_complete);
+			copy_amount);
 		if (ret < 0) {
 			gossip_err("%s: Failed to copy-out buffers. Please make sure that the pvfs2-client is running (%ld)\n",
 			    __func__, (long)ret);
@@ -223,246 +249,112 @@ populate_shared_memory:
 
 out:
 	if (buffer_index >= 0) {
-		orangefs_bufmap_put(buffer_index);
-		gossip_debug(GOSSIP_FILE_DEBUG,
-			     "%s(%pU): PUT buffer_index %d\n",
-			     __func__, handle, buffer_index);
+		if ((readahead_size) && (type == ORANGEFS_IO_READ)) {
+			/* readpage */
+			*index_return = buffer_index;
+			gossip_debug(GOSSIP_FILE_DEBUG,
+				"%s: hold on to buffer_index :%d:\n",
+				__func__, buffer_index);
+		} else {
+			/* O_DIRECT */
+			orangefs_bufmap_put(buffer_index);
+			gossip_debug(GOSSIP_FILE_DEBUG,
+				"%s(%pU): PUT buffer_index %d\n",
+				__func__, handle, buffer_index);
+		}
 		buffer_index = -1;
 	}
 	op_release(new_op);
 	return ret;
 }
 
-/*
- * Common entry point for read/write/readv/writev
- * This function will dispatch it to either the direct I/O
- * or buffered I/O path depending on the mount options and/or
- * augmented/extended metadata attached to the file.
- * Note: File extended attributes override any mount options.
- */
-static ssize_t do_readv_writev(enum ORANGEFS_io_type type, struct file *file,
-		loff_t *offset, struct iov_iter *iter)
+int orangefs_revalidate_mapping(struct inode *inode)
 {
-	struct inode *inode = file->f_mapping->host;
 	struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode);
-	struct orangefs_khandle *handle = &orangefs_inode->refn.khandle;
-	size_t count = iov_iter_count(iter);
-	ssize_t total_count = 0;
-	ssize_t ret = -EINVAL;
-
-	gossip_debug(GOSSIP_FILE_DEBUG,
-		"%s-BEGIN(%pU): count(%d) after estimate_max_iovecs.\n",
-		__func__,
-		handle,
-		(int)count);
-
-	if (type == ORANGEFS_IO_WRITE) {
-		gossip_debug(GOSSIP_FILE_DEBUG,
-			     "%s(%pU): proceeding with offset : %llu, "
-			     "size %d\n",
-			     __func__,
-			     handle,
-			     llu(*offset),
-			     (int)count);
-	}
+	struct address_space *mapping = inode->i_mapping;
+	unsigned long *bitlock = &orangefs_inode->bitlock;
+	int ret;
 
-	if (count == 0) {
-		ret = 0;
-		goto out;
+	while (1) {
+		ret = wait_on_bit(bitlock, 1, TASK_KILLABLE);
+		if (ret)
+			return ret;
+		spin_lock(&inode->i_lock);
+		if (test_bit(1, bitlock)) {
+			spin_unlock(&inode->i_lock);
+			continue;
+		}
+		if (!time_before(jiffies, orangefs_inode->mapping_time))
+			break;
+		spin_unlock(&inode->i_lock);
+		return 0;
 	}
 
-	while (iov_iter_count(iter)) {
-		size_t each_count = iov_iter_count(iter);
-		size_t amt_complete;
-
-		/* how much to transfer in this loop iteration */
-		if (each_count > orangefs_bufmap_size_query())
-			each_count = orangefs_bufmap_size_query();
+	set_bit(1, bitlock);
+	smp_wmb();
+	spin_unlock(&inode->i_lock);
 
-		gossip_debug(GOSSIP_FILE_DEBUG,
-			     "%s(%pU): size of each_count(%d)\n",
-			     __func__,
-			     handle,
-			     (int)each_count);
-		gossip_debug(GOSSIP_FILE_DEBUG,
-			     "%s(%pU): BEFORE wait_for_io: offset is %d\n",
-			     __func__,
-			     handle,
-			     (int)*offset);
-
-		ret = wait_for_direct_io(type, inode, offset, iter,
-				each_count, 0);
-		gossip_debug(GOSSIP_FILE_DEBUG,
-			     "%s(%pU): return from wait_for_io:%d\n",
-			     __func__,
-			     handle,
-			     (int)ret);
+	unmap_mapping_range(mapping, 0, 0, 0);
+	ret = filemap_write_and_wait(mapping);
+	if (!ret)
+		ret = invalidate_inode_pages2(mapping);
 
-		if (ret < 0)
-			goto out;
-
-		*offset += ret;
-		total_count += ret;
-		amt_complete = ret;
+	orangefs_inode->mapping_time = jiffies +
+	    orangefs_cache_timeout_msecs*HZ/1000;
 
-		gossip_debug(GOSSIP_FILE_DEBUG,
-			     "%s(%pU): AFTER wait_for_io: offset is %d\n",
-			     __func__,
-			     handle,
-			     (int)*offset);
-
-		/*
-		 * if we got a short I/O operations,
-		 * fall out and return what we got so far
-		 */
-		if (amt_complete < each_count)
-			break;
-	} /*end while */
-
-out:
-	if (total_count > 0)
-		ret = total_count;
-	if (ret > 0) {
-		if (type == ORANGEFS_IO_READ) {
-			file_accessed(file);
-		} else {
-			file_update_time(file);
-			/*
-			 * Must invalidate to ensure write loop doesn't
-			 * prevent kernel from reading updated
-			 * attribute.  Size probably changed because of
-			 * the write, and other clients could update
-			 * any other attribute.
-			 */
-			orangefs_inode->getattr_time = jiffies - 1;
-		}
-	}
-
-	gossip_debug(GOSSIP_FILE_DEBUG,
-		     "%s(%pU): Value(%d) returned.\n",
-		     __func__,
-		     handle,
-		     (int)ret);
+	clear_bit(1, bitlock);
+	smp_mb__after_atomic();
+	wake_up_bit(bitlock, 1);
 
 	return ret;
 }
 
-/*
- * Read data from a specified offset in a file (referenced by inode).
- * Data may be placed either in a user or kernel buffer.
- */
-ssize_t orangefs_inode_read(struct inode *inode,
-			    struct iov_iter *iter,
-			    loff_t *offset,
-			    loff_t readahead_size)
+static ssize_t orangefs_file_read_iter(struct kiocb *iocb,
+    struct iov_iter *iter)
 {
-	struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode);
-	size_t count = iov_iter_count(iter);
-	size_t bufmap_size;
-	ssize_t ret = -EINVAL;
+	int ret;
+	struct orangefs_read_options *ro;
 
 	orangefs_stats.reads++;
 
-	bufmap_size = orangefs_bufmap_size_query();
-	if (count > bufmap_size) {
-		gossip_debug(GOSSIP_FILE_DEBUG,
-			     "%s: count is too large (%zd/%zd)!\n",
-			     __func__, count, bufmap_size);
-		return -EINVAL;
+	/*
+	 * Remember how they set "count" in read(2) or pread(2) or whatever -
+	 * users can use count as a knob to control orangefs io size and later
+	 * we can try to help them fill as many pages as possible in readpage.
+	 */
+	if (!iocb->ki_filp->private_data) {
+		iocb->ki_filp->private_data = kmalloc(sizeof *ro, GFP_KERNEL);
+		if (!iocb->ki_filp->private_data)
+			return(ENOMEM);
+		ro = iocb->ki_filp->private_data;
+		ro->blksiz = iter->count;
 	}
 
-	gossip_debug(GOSSIP_FILE_DEBUG,
-		     "%s(%pU) %zd@%llu\n",
-		     __func__,
-		     &orangefs_inode->refn.khandle,
-		     count,
-		     llu(*offset));
-
-	ret = wait_for_direct_io(ORANGEFS_IO_READ, inode, offset, iter,
-			count, readahead_size);
-	if (ret > 0)
-		*offset += ret;
-
-	gossip_debug(GOSSIP_FILE_DEBUG,
-		     "%s(%pU): Value(%zd) returned.\n",
-		     __func__,
-		     &orangefs_inode->refn.khandle,
-		     ret);
+	down_read(&file_inode(iocb->ki_filp)->i_rwsem);
+	ret = orangefs_revalidate_mapping(file_inode(iocb->ki_filp));
+	if (ret)
+		goto out;
 
+	ret = generic_file_read_iter(iocb, iter);
+out:
+	up_read(&file_inode(iocb->ki_filp)->i_rwsem);
 	return ret;
 }
 
-static ssize_t orangefs_file_read_iter(struct kiocb *iocb, struct iov_iter *iter)
+static ssize_t orangefs_file_write_iter(struct kiocb *iocb,
+    struct iov_iter *iter)
 {
-	struct file *file = iocb->ki_filp;
-	loff_t pos = iocb->ki_pos;
-	ssize_t rc = 0;
-
-	gossip_debug(GOSSIP_FILE_DEBUG, "orangefs_file_read_iter\n");
-
-	orangefs_stats.reads++;
-
-	rc = do_readv_writev(ORANGEFS_IO_READ, file, &pos, iter);
-	iocb->ki_pos = pos;
-
-	return rc;
-}
-
-static ssize_t orangefs_file_write_iter(struct kiocb *iocb, struct iov_iter *iter)
-{
-	struct file *file = iocb->ki_filp;
-	loff_t pos;
-	ssize_t rc;
-
-	gossip_debug(GOSSIP_FILE_DEBUG, "orangefs_file_write_iter\n");
-
-	inode_lock(file->f_mapping->host);
-
-	/* Make sure generic_write_checks sees an up to date inode size. */
-	if (file->f_flags & O_APPEND) {
-		rc = orangefs_inode_getattr(file->f_mapping->host, 0, 1,
-		    STATX_SIZE);
-		if (rc == -ESTALE)
-			rc = -EIO;
-		if (rc) {
-			gossip_err("%s: orangefs_inode_getattr failed, "
-			    "rc:%zd:.\n", __func__, rc);
-			goto out;
-		}
-	}
-
-	rc = generic_write_checks(iocb, iter);
-
-	if (rc <= 0) {
-		gossip_err("%s: generic_write_checks failed, rc:%zd:.\n",
-			   __func__, rc);
-		goto out;
-	}
-
-	/*
-	 * if we are appending, generic_write_checks would have updated
-	 * pos to the end of the file, so we will wait till now to set
-	 * pos...
-	 */
-	pos = iocb->ki_pos;
-
-	rc = do_readv_writev(ORANGEFS_IO_WRITE,
-			     file,
-			     &pos,
-			     iter);
-	if (rc < 0) {
-		gossip_err("%s: do_readv_writev failed, rc:%zd:.\n",
-			   __func__, rc);
-		goto out;
-	}
-
-	iocb->ki_pos = pos;
+	int ret;
 	orangefs_stats.writes++;
 
-out:
+	if (iocb->ki_pos > i_size_read(file_inode(iocb->ki_filp))) {
+		ret = orangefs_revalidate_mapping(file_inode(iocb->ki_filp));
+		if (ret)
+			return ret;
+	}
 
-	inode_unlock(file->f_mapping->host);
-	return rc;
+	ret = generic_file_write_iter(iocb, iter);
+	return ret;
 }
 
 /*
@@ -528,14 +420,13 @@ static vm_fault_t orangefs_fault(struct vm_fault *vmf)
 {
 	struct file *file = vmf->vma->vm_file;
 	int ret;
-
-	ret = orangefs_inode_getattr(file->f_mapping->host, 0, 1,
-	    STATX_SIZE);
+	ret = orangefs_inode_getattr(file->f_mapping->host,
+	    ORANGEFS_GETATTR_SIZE);
 	if (ret == -ESTALE)
 		ret = -EIO;
 	if (ret) {
-		gossip_err("%s: orangefs_inode_getattr failed, ret:%d:.\n",
-				__func__, ret);
+		gossip_err("%s: orangefs_inode_getattr failed, "
+		    "ret:%d:.\n", __func__, ret);
 		return VM_FAULT_SIGBUS;
 	}
 	return filemap_fault(vmf);
@@ -544,7 +435,7 @@ static vm_fault_t orangefs_fault(struct vm_fault *vmf)
 static const struct vm_operations_struct orangefs_file_vm_ops = {
 	.fault = orangefs_fault,
 	.map_pages = filemap_map_pages,
-	.page_mkwrite = filemap_page_mkwrite,
+	.page_mkwrite = orangefs_page_mkwrite,
 };
 
 /*
@@ -552,15 +443,18 @@ static const struct vm_operations_struct orangefs_file_vm_ops = {
  */
 static int orangefs_file_mmap(struct file *file, struct vm_area_struct *vma)
 {
+	int ret;
+
+	ret = orangefs_revalidate_mapping(file_inode(file));
+	if (ret)
+		return ret;
+
 	gossip_debug(GOSSIP_FILE_DEBUG,
 		     "orangefs_file_mmap: called on %s\n",
 		     (file ?
 			(char *)file->f_path.dentry->d_name.name :
 			(char *)"Unknown"));
 
-	if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE))
-		return -EINVAL;
-
 	/* set the sequential readahead hint */
 	vma->vm_flags |= VM_SEQ_READ;
 	vma->vm_flags &= ~VM_RAND_READ;
@@ -600,8 +494,7 @@ static int orangefs_file_release(struct inode *inode, struct file *file)
 			gossip_debug(GOSSIP_INODE_DEBUG,
 			    "flush_racache finished\n");
 		}
-		truncate_inode_pages(file_inode(file)->i_mapping,
-				     0);
+
 	}
 	return 0;
 }
@@ -619,6 +512,11 @@ static int orangefs_fsync(struct file *file,
 		ORANGEFS_I(file_inode(file));
 	struct orangefs_kernel_op_s *new_op = NULL;
 
+	ret = filemap_write_and_wait_range(file_inode(file)->i_mapping,
+	    start, end);
+	if (ret < 0)
+		return ret;
+
 	new_op = op_alloc(ORANGEFS_VFS_OP_FSYNC);
 	if (!new_op)
 		return -ENOMEM;
@@ -656,8 +554,8 @@ static loff_t orangefs_file_llseek(struct file *file, loff_t offset, int origin)
 		 * NOTE: We are only interested in file size here,
 		 * so we set mask accordingly.
 		 */
-		ret = orangefs_inode_getattr(file->f_mapping->host, 0, 1,
-		    STATX_SIZE);
+		ret = orangefs_inode_getattr(file->f_mapping->host,
+		    ORANGEFS_GETATTR_SIZE);
 		if (ret == -ESTALE)
 			ret = -EIO;
 		if (ret) {
@@ -700,6 +598,42 @@ static int orangefs_lock(struct file *filp, int cmd, struct file_lock *fl)
 	return rc;
 }
 
+static int orangefs_file_open(struct inode * inode, struct file *file)
+{
+	file->private_data = NULL;
+	return generic_file_open(inode, file);
+}
+
+static int orangefs_flush(struct file *file, fl_owner_t id)
+{
+	/*
+	 * This is vfs_fsync_range(file, 0, LLONG_MAX, 0) without the
+	 * service_operation in orangefs_fsync.
+	 *
+	 * Do not send fsync to OrangeFS server on a close.  Do send fsync
+	 * on an explicit fsync call.  This duplicates historical OrangeFS
+	 * behavior.
+	 */
+	struct inode *inode = file->f_mapping->host;
+	int r;
+
+	kfree(file->private_data);
+	file->private_data = NULL;
+
+	if (inode->i_state & I_DIRTY_TIME) {
+		spin_lock(&inode->i_lock);
+		inode->i_state &= ~I_DIRTY_TIME;
+		spin_unlock(&inode->i_lock);
+		mark_inode_dirty_sync(inode);
+	}
+
+	r = filemap_write_and_wait_range(file->f_mapping, 0, LLONG_MAX);
+	if (r > 0)
+		return 0;
+	else
+		return r;
+}
+
 /** ORANGEFS implementation of VFS file operations */
 const struct file_operations orangefs_file_operations = {
 	.llseek		= orangefs_file_llseek,
@@ -708,7 +642,8 @@ const struct file_operations orangefs_file_operations = {
 	.lock		= orangefs_lock,
 	.unlocked_ioctl	= orangefs_ioctl,
 	.mmap		= orangefs_file_mmap,
-	.open		= generic_file_open,
+	.open		= orangefs_file_open,
+	.flush		= orangefs_flush,
 	.release	= orangefs_file_release,
 	.fsync		= orangefs_fsync,
 };
diff --git a/fs/orangefs/inode.c b/fs/orangefs/inode.c
index c3334eca18c7..0c337d8bdaab 100644
--- a/fs/orangefs/inode.c
+++ b/fs/orangefs/inode.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
  * (C) 2001 Clemson University and The University of Chicago
+ * Copyright 2018 Omnibond Systems, L.L.C.
  *
  * See COPYING in top-level directory.
  */
@@ -14,40 +15,312 @@
 #include "orangefs-kernel.h"
 #include "orangefs-bufmap.h"
 
-static int read_one_page(struct page *page)
+static int orangefs_writepage_locked(struct page *page,
+    struct writeback_control *wbc)
 {
-	int ret;
-	int max_block;
-	ssize_t bytes_read = 0;
 	struct inode *inode = page->mapping->host;
-	const __u32 blocksize = PAGE_SIZE;
-	const __u32 blockbits = PAGE_SHIFT;
-	struct iov_iter to;
-	struct bio_vec bv = {.bv_page = page, .bv_len = PAGE_SIZE};
+	struct orangefs_write_range *wr = NULL;
+	struct iov_iter iter;
+	struct bio_vec bv;
+	size_t len, wlen;
+	ssize_t ret;
+	loff_t off;
+
+	set_page_writeback(page);
+
+	len = i_size_read(inode);
+	if (PagePrivate(page)) {
+		wr = (struct orangefs_write_range *)page_private(page);
+		WARN_ON(wr->pos >= len);
+		off = wr->pos;
+		if (off + wr->len > len)
+			wlen = len - off;
+		else
+			wlen = wr->len;
+	} else {
+		WARN_ON(1);
+		off = page_offset(page);
+		if (off + PAGE_SIZE > len)
+			wlen = len - off;
+		else
+			wlen = PAGE_SIZE;
+	}
+	/* Should've been handled in orangefs_invalidatepage. */
+	WARN_ON(off == len || off + wlen > len);
+
+	bv.bv_page = page;
+	bv.bv_len = wlen;
+	bv.bv_offset = off % PAGE_SIZE;
+	WARN_ON(wlen == 0);
+	iov_iter_bvec(&iter, WRITE, &bv, 1, wlen);
+
+	ret = wait_for_direct_io(ORANGEFS_IO_WRITE, inode, &off, &iter, wlen,
+	    len, wr, NULL);
+	if (ret < 0) {
+		SetPageError(page);
+		mapping_set_error(page->mapping, ret);
+	} else {
+		ret = 0;
+	}
+	if (wr) {
+		kfree(wr);
+		set_page_private(page, 0);
+		ClearPagePrivate(page);
+		put_page(page);
+	}
+	return ret;
+}
+
+static int orangefs_writepage(struct page *page, struct writeback_control *wbc)
+{
+	int ret;
+	ret = orangefs_writepage_locked(page, wbc);
+	unlock_page(page);
+	end_page_writeback(page);
+	return ret;
+}
 
-	iov_iter_bvec(&to, READ, &bv, 1, PAGE_SIZE);
+struct orangefs_writepages {
+	loff_t off;
+	size_t len;
+	kuid_t uid;
+	kgid_t gid;
+	int maxpages;
+	int npages;
+	struct page **pages;
+	struct bio_vec *bv;
+};
 
-	gossip_debug(GOSSIP_INODE_DEBUG,
-		    "orangefs_readpage called with page %p\n",
-		     page);
+static int orangefs_writepages_work(struct orangefs_writepages *ow,
+    struct writeback_control *wbc)
+{
+	struct inode *inode = ow->pages[0]->mapping->host;
+	struct orangefs_write_range *wrp, wr;
+	struct iov_iter iter;
+	ssize_t ret;
+	size_t len;
+	loff_t off;
+	int i;
+
+	len = i_size_read(inode);
+
+	for (i = 0; i < ow->npages; i++) {
+		set_page_writeback(ow->pages[i]);
+		ow->bv[i].bv_page = ow->pages[i];
+		ow->bv[i].bv_len = min(page_offset(ow->pages[i]) + PAGE_SIZE,
+		    ow->off + ow->len) -
+		    max(ow->off, page_offset(ow->pages[i]));
+		if (i == 0)
+			ow->bv[i].bv_offset = ow->off -
+			    page_offset(ow->pages[i]);
+		else
+			ow->bv[i].bv_offset = 0;
+	}
+	iov_iter_bvec(&iter, WRITE, ow->bv, ow->npages, ow->len);
+
+	WARN_ON(ow->off >= len);
+	if (ow->off + ow->len > len)
+		ow->len = len - ow->off;
+
+	off = ow->off;
+	wr.uid = ow->uid;
+	wr.gid = ow->gid;
+	ret = wait_for_direct_io(ORANGEFS_IO_WRITE, inode, &off, &iter, ow->len,
+	    0, &wr, NULL);
+	if (ret < 0) {
+		for (i = 0; i < ow->npages; i++) {
+			SetPageError(ow->pages[i]);
+			mapping_set_error(ow->pages[i]->mapping, ret);
+			if (PagePrivate(ow->pages[i])) {
+				wrp = (struct orangefs_write_range *)
+				    page_private(ow->pages[i]);
+				ClearPagePrivate(ow->pages[i]);
+				put_page(ow->pages[i]);
+				kfree(wrp);
+			}
+			end_page_writeback(ow->pages[i]);
+			unlock_page(ow->pages[i]);
+		}
+	} else {
+		ret = 0;
+		for (i = 0; i < ow->npages; i++) {
+			if (PagePrivate(ow->pages[i])) {
+				wrp = (struct orangefs_write_range *)
+				    page_private(ow->pages[i]);
+				ClearPagePrivate(ow->pages[i]);
+				put_page(ow->pages[i]);
+				kfree(wrp);
+			}
+			end_page_writeback(ow->pages[i]);
+			unlock_page(ow->pages[i]);
+		}
+	}
+	return ret;
+}
+
+static int orangefs_writepages_callback(struct page *page,
+    struct writeback_control *wbc, void *data)
+{
+	struct orangefs_writepages *ow = data;
+	struct orangefs_write_range *wr;
+	int ret;
+
+	if (!PagePrivate(page)) {
+		unlock_page(page);
+		/* It's not private so there's nothing to write, right? */
+		printk("writepages_callback not private!\n");
+		BUG();
+		return 0;
+	}
+	wr = (struct orangefs_write_range *)page_private(page);
+
+	ret = -1;
+	if (ow->npages == 0) {
+		ow->off = wr->pos;
+		ow->len = wr->len;
+		ow->uid = wr->uid;
+		ow->gid = wr->gid;
+		ow->pages[ow->npages++] = page;
+		ret = 0;
+		goto done;
+	}
+	if (!uid_eq(ow->uid, wr->uid) || !gid_eq(ow->gid, wr->gid)) {
+		orangefs_writepages_work(ow, wbc);
+		ow->npages = 0;
+		ret = -1;
+		goto done;
+	}
+	if (ow->off + ow->len == wr->pos) {
+		ow->len += wr->len;
+		ow->pages[ow->npages++] = page;
+		ret = 0;
+		goto done;
+	}
+done:
+	if (ret == -1) {
+		if (ow->npages) {
+			orangefs_writepages_work(ow, wbc);
+			ow->npages = 0;
+		}
+		ret = orangefs_writepage_locked(page, wbc);
+		mapping_set_error(page->mapping, ret);
+		unlock_page(page);
+		end_page_writeback(page);
+	} else {
+		if (ow->npages == ow->maxpages) {
+			orangefs_writepages_work(ow, wbc);
+			ow->npages = 0;
+		}
+	}
+	return ret;
+}
+
+static int orangefs_writepages(struct address_space *mapping,
+    struct writeback_control *wbc)
+{
+	struct orangefs_writepages *ow;
+	struct blk_plug plug;
+	int ret;
+	ow = kzalloc(sizeof(struct orangefs_writepages), GFP_KERNEL);
+	if (!ow)
+		return -ENOMEM;
+	ow->maxpages = orangefs_bufmap_size_query()/PAGE_SIZE;
+	ow->pages = kcalloc(ow->maxpages, sizeof(struct page *), GFP_KERNEL);
+	if (!ow->pages) {
+		kfree(ow);
+		return -ENOMEM;
+	}
+	ow->bv = kcalloc(ow->maxpages, sizeof(struct bio_vec), GFP_KERNEL);
+	if (!ow->bv) {
+		kfree(ow->pages);
+		kfree(ow);
+		return -ENOMEM;
+	}
+	blk_start_plug(&plug);
+	ret = write_cache_pages(mapping, wbc, orangefs_writepages_callback, ow);
+	if (ow->npages)
+		ret = orangefs_writepages_work(ow, wbc);
+	blk_finish_plug(&plug);
+	kfree(ow->pages);
+	kfree(ow->bv);
+	kfree(ow);
+	return ret;
+}
 
-	max_block = ((inode->i_size / blocksize) + 1);
+static int orangefs_launder_page(struct page *);
 
-	if (page->index < max_block) {
-		loff_t blockptr_offset = (((loff_t) page->index) << blockbits);
+static int orangefs_readpage(struct file *file, struct page *page)
+{
+	struct inode *inode = page->mapping->host;
+	struct iov_iter iter;
+	struct bio_vec bv;
+	ssize_t ret;
+	loff_t off; /* offset into this page */
+	pgoff_t index; /* which page */
+	struct page *next_page;
+	char *kaddr;
+	struct orangefs_read_options *ro = file->private_data;
+	loff_t read_size;
+	loff_t roundedup;
+	int buffer_index = -1; /* orangefs shared memory slot */
+	int slot_index;   /* index into slot */
+	int remaining;
 
-		bytes_read = orangefs_inode_read(inode,
-						 &to,
-						 &blockptr_offset,
-						 inode->i_size);
+	/*
+	 * If they set some miniscule size for "count" in read(2)
+	 * (for example) then let's try to read a page, or the whole file
+	 * if it is smaller than a page. Once "count" goes over a page
+	 * then lets round up to the highest page size multiple that is
+	 * less than or equal to "count" and do that much orangefs IO and
+	 * try to fill as many pages as we can from it.
+	 *
+	 * "count" should be represented in ro->blksiz.
+	 *
+	 * inode->i_size = file size.
+	 */
+	if (ro) {
+		if (ro->blksiz < PAGE_SIZE) {
+			if (inode->i_size < PAGE_SIZE)
+				read_size = inode->i_size;
+			else
+				read_size = PAGE_SIZE;
+		} else {
+			roundedup = ((PAGE_SIZE - 1) & ro->blksiz) ?
+				((ro->blksiz + PAGE_SIZE) & ~(PAGE_SIZE -1)) :
+				ro->blksiz;
+			if (roundedup > inode->i_size)
+				read_size = inode->i_size;
+			else
+				read_size = roundedup;
+
+		}
+	} else {
+		read_size = PAGE_SIZE;
 	}
+	if (!read_size)
+		read_size = PAGE_SIZE;
+
+	if (PageDirty(page))
+		orangefs_launder_page(page);
+
+	off = page_offset(page);
+	index = off >> PAGE_SHIFT;
+	bv.bv_page = page;
+	bv.bv_len = PAGE_SIZE;
+	bv.bv_offset = 0;
+	iov_iter_bvec(&iter, READ, &bv, 1, PAGE_SIZE);
+
+	ret = wait_for_direct_io(ORANGEFS_IO_READ, inode, &off, &iter,
+	    read_size, inode->i_size, NULL, &buffer_index);
+	remaining = ret;
 	/* this will only zero remaining unread portions of the page data */
-	iov_iter_zero(~0U, &to);
+	iov_iter_zero(~0U, &iter);
 	/* takes care of potential aliasing */
 	flush_dcache_page(page);
-	if (bytes_read < 0) {
-		ret = bytes_read;
+	if (ret < 0) {
 		SetPageError(page);
+		unlock_page(page);
+		goto out;
 	} else {
 		SetPageUptodate(page);
 		if (PageError(page))
@@ -56,96 +329,469 @@ static int read_one_page(struct page *page)
 	}
 	/* unlock the page after the ->readpage() routine completes */
 	unlock_page(page);
+
+	if (remaining > PAGE_SIZE) {
+		slot_index = 0;
+		while ((remaining - PAGE_SIZE) >= PAGE_SIZE) {
+			remaining -= PAGE_SIZE;
+			/*
+			 * It is an optimization to try and fill more than one
+			 * page... by now we've already gotten the single
+			 * page we were after, if stuff doesn't seem to
+			 * be going our way at this point just return
+			 * and hope for the best.
+			 *
+			 * If we look for pages and they're already there is
+			 * one reason to give up, and if they're not there
+			 * and we can't create them is another reason.
+			 */
+
+			index++;
+			slot_index++;
+			next_page = find_get_page(inode->i_mapping, index);
+			if (next_page) {
+				gossip_debug(GOSSIP_FILE_DEBUG,
+					"%s: found next page, quitting\n",
+					__func__);
+				put_page(next_page);
+				goto out;
+			}
+			next_page = find_or_create_page(inode->i_mapping,
+							index,
+							GFP_KERNEL);
+			/*
+			 * I've never hit this, leave it as a printk for
+			 * now so it will be obvious.
+			 */
+			if (!next_page) {
+				printk("%s: can't create next page, quitting\n",
+					__func__);
+				goto out;
+			}
+			kaddr = kmap_atomic(next_page);
+			orangefs_bufmap_page_fill(kaddr,
+						buffer_index,
+						slot_index);
+			kunmap_atomic(kaddr);
+			SetPageUptodate(next_page);
+			unlock_page(next_page);
+			put_page(next_page);
+		}
+	}
+
+out:
+	if (buffer_index != -1)
+		orangefs_bufmap_put(buffer_index);
 	return ret;
 }
 
-static int orangefs_readpage(struct file *file, struct page *page)
+static int orangefs_write_begin(struct file *file,
+    struct address_space *mapping,
+    loff_t pos, unsigned len, unsigned flags, struct page **pagep,
+    void **fsdata)
 {
-	return read_one_page(page);
+	struct orangefs_write_range *wr;
+	struct page *page;
+	pgoff_t index;
+	int ret;
+
+	index = pos >> PAGE_SHIFT;
+
+	page = grab_cache_page_write_begin(mapping, index, flags);
+	if (!page)
+		return -ENOMEM;
+
+	*pagep = page;
+
+	if (PageDirty(page) && !PagePrivate(page)) {
+		/*
+		 * Should be impossible.  If it happens, launder the page
+		 * since we don't know what's dirty.  This will WARN in
+		 * orangefs_writepage_locked.
+		 */
+		ret = orangefs_launder_page(page);
+		if (ret)
+			return ret;
+	}
+	if (PagePrivate(page)) {
+		struct orangefs_write_range *wr;
+		wr = (struct orangefs_write_range *)page_private(page);
+		if (wr->pos + wr->len == pos &&
+		    uid_eq(wr->uid, current_fsuid()) &&
+		    gid_eq(wr->gid, current_fsgid())) {
+			wr->len += len;
+			goto okay;
+		} else {
+			ret = orangefs_launder_page(page);
+			if (ret)
+				return ret;
+		}
+	}
+
+	wr = kmalloc(sizeof *wr, GFP_KERNEL);
+	if (!wr)
+		return -ENOMEM;
+
+	wr->pos = pos;
+	wr->len = len;
+	wr->uid = current_fsuid();
+	wr->gid = current_fsgid();
+	SetPagePrivate(page);
+	set_page_private(page, (unsigned long)wr);
+	get_page(page);
+okay:
+	return 0;
 }
 
-static int orangefs_readpages(struct file *file,
-			   struct address_space *mapping,
-			   struct list_head *pages,
-			   unsigned nr_pages)
+static int orangefs_write_end(struct file *file, struct address_space *mapping,
+    loff_t pos, unsigned len, unsigned copied, struct page *page, void *fsdata)
 {
-	int page_idx;
-	int ret;
+	struct inode *inode = page->mapping->host;
+	loff_t last_pos = pos + copied;
 
-	gossip_debug(GOSSIP_INODE_DEBUG, "orangefs_readpages called\n");
-
-	for (page_idx = 0; page_idx < nr_pages; page_idx++) {
-		struct page *page;
-
-		page = lru_to_page(pages);
-		list_del(&page->lru);
-		if (!add_to_page_cache(page,
-				       mapping,
-				       page->index,
-				       readahead_gfp_mask(mapping))) {
-			ret = read_one_page(page);
-			gossip_debug(GOSSIP_INODE_DEBUG,
-				"failure adding page to cache, read_one_page returned: %d\n",
-				ret);
-	      } else {
-			put_page(page);
-	      }
-	}
-	BUG_ON(!list_empty(pages));
-	return 0;
+	/*
+	 * No need to use i_size_read() here, the i_size
+	 * cannot change under us because we hold the i_mutex.
+	 */
+	if (last_pos > inode->i_size)
+		i_size_write(inode, last_pos);
+
+	/* zero the stale part of the page if we did a short copy */
+	if (!PageUptodate(page)) {
+		unsigned from = pos & (PAGE_SIZE - 1);
+		if (copied < len) {
+			zero_user(page, from + copied, len - copied);
+		}
+		/* Set fully written pages uptodate. */
+		if (pos == page_offset(page) &&
+		    (len == PAGE_SIZE || pos + len == inode->i_size)) {
+			zero_user_segment(page, from + copied, PAGE_SIZE);
+			SetPageUptodate(page);
+		}
+	}
+
+	set_page_dirty(page);
+	unlock_page(page);
+	put_page(page);
+
+	mark_inode_dirty_sync(file_inode(file));
+	return copied;
 }
 
 static void orangefs_invalidatepage(struct page *page,
 				 unsigned int offset,
 				 unsigned int length)
 {
-	gossip_debug(GOSSIP_INODE_DEBUG,
-		     "orangefs_invalidatepage called on page %p "
-		     "(offset is %u)\n",
-		     page,
-		     offset);
+	struct orangefs_write_range *wr;
+	wr = (struct orangefs_write_range *)page_private(page);
+
+	if (offset == 0 && length == PAGE_SIZE) {
+		kfree((struct orangefs_write_range *)page_private(page));
+		set_page_private(page, 0);
+		ClearPagePrivate(page);
+		put_page(page);
+		return;
+	/* write range entirely within invalidate range (or equal) */
+	} else if (page_offset(page) + offset <= wr->pos &&
+	    wr->pos + wr->len <= page_offset(page) + offset + length) {
+		kfree((struct orangefs_write_range *)page_private(page));
+		set_page_private(page, 0);
+		ClearPagePrivate(page);
+		put_page(page);
+		/* XXX is this right? only caller in fs */
+		cancel_dirty_page(page);
+		return;
+	/* invalidate range chops off end of write range */
+	} else if (wr->pos < page_offset(page) + offset &&
+	    wr->pos + wr->len <= page_offset(page) + offset + length &&
+	     page_offset(page) + offset < wr->pos + wr->len) {
+		size_t x;
+		x = wr->pos + wr->len - (page_offset(page) + offset);
+		WARN_ON(x > wr->len);
+		wr->len -= x;
+		wr->uid = current_fsuid();
+		wr->gid = current_fsgid();
+	/* invalidate range chops off beginning of write range */
+	} else if (page_offset(page) + offset <= wr->pos &&
+	    page_offset(page) + offset + length < wr->pos + wr->len &&
+	    wr->pos < page_offset(page) + offset + length) {
+		size_t x;
+		x = page_offset(page) + offset + length - wr->pos;
+		WARN_ON(x > wr->len);
+		wr->pos += x;
+		wr->len -= x;
+		wr->uid = current_fsuid();
+		wr->gid = current_fsgid();
+	/* invalidate range entirely within write range (punch hole) */
+	} else if (wr->pos < page_offset(page) + offset &&
+	    page_offset(page) + offset + length < wr->pos + wr->len) {
+		/* XXX what do we do here... should not WARN_ON */
+		WARN_ON(1);
+		/* punch hole */
+		/*
+		 * should we just ignore this and write it out anyway?
+		 * it hardly makes sense
+		 */
+		return;
+	/* non-overlapping ranges */
+	} else {
+		/* WARN if they do overlap */
+		if (!((page_offset(page) + offset + length <= wr->pos) ^
+		    (wr->pos + wr->len <= page_offset(page) + offset))) {
+			WARN_ON(1);
+			printk("invalidate range offset %llu length %u\n",
+			    page_offset(page) + offset, length);
+			printk("write range offset %llu length %zu\n",
+			    wr->pos, wr->len);
+		}
+		return;
+	}
 
-	ClearPageUptodate(page);
-	ClearPageMappedToDisk(page);
-	return;
+	/*
+	 * Above there are returns where wr is freed or where we WARN.
+	 * Thus the following runs if wr was modified above.
+	 */
 
+	orangefs_launder_page(page);
 }
 
 static int orangefs_releasepage(struct page *page, gfp_t foo)
 {
-	gossip_debug(GOSSIP_INODE_DEBUG,
-		     "orangefs_releasepage called on page %p\n",
-		     page);
-	return 0;
+	return !PagePrivate(page);
 }
 
-/*
- * Having a direct_IO entry point in the address_space_operations
- * struct causes the kernel to allows us to use O_DIRECT on
- * open. Nothing will ever call this thing, but in the future we
- * will need to be able to use O_DIRECT on open in order to support
- * AIO. Modeled after NFS, they do this too.
- */
+static void orangefs_freepage(struct page *page)
+{
+	if (PagePrivate(page)) {
+		kfree((struct orangefs_write_range *)page_private(page));
+		set_page_private(page, 0);
+		ClearPagePrivate(page);
+		put_page(page);
+	}
+}
+
+static int orangefs_launder_page(struct page *page)
+{
+	int r = 0;
+	struct writeback_control wbc = {
+		.sync_mode = WB_SYNC_ALL,
+		.nr_to_write = 0,
+	};
+	wait_on_page_writeback(page);
+	if (clear_page_dirty_for_io(page)) {
+		r = orangefs_writepage_locked(page, &wbc);
+		end_page_writeback(page);
+	}
+	return r;
+}
 
 static ssize_t orangefs_direct_IO(struct kiocb *iocb,
 				  struct iov_iter *iter)
 {
-	gossip_debug(GOSSIP_INODE_DEBUG,
-		     "orangefs_direct_IO: %pD\n",
-		     iocb->ki_filp);
+	/*
+	 * Comment from original do_readv_writev:
+	 * Common entry point for read/write/readv/writev
+	 * This function will dispatch it to either the direct I/O
+	 * or buffered I/O path depending on the mount options and/or
+	 * augmented/extended metadata attached to the file.
+	 * Note: File extended attributes override any mount options.
+	 */
+	struct file *file = iocb->ki_filp;
+	loff_t pos = iocb->ki_pos;
+	enum ORANGEFS_io_type type = iov_iter_rw(iter) == WRITE ?
+            ORANGEFS_IO_WRITE : ORANGEFS_IO_READ;
+	loff_t *offset = &pos;
+	struct inode *inode = file->f_mapping->host;
+	struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode);
+	struct orangefs_khandle *handle = &orangefs_inode->refn.khandle;
+	size_t count = iov_iter_count(iter);
+	ssize_t total_count = 0;
+	ssize_t ret = -EINVAL;
+	int i = 0;
+
+	gossip_debug(GOSSIP_FILE_DEBUG,
+		"%s-BEGIN(%pU): count(%d) after estimate_max_iovecs.\n",
+		__func__,
+		handle,
+		(int)count);
+
+	if (type == ORANGEFS_IO_WRITE) {
+		gossip_debug(GOSSIP_FILE_DEBUG,
+			     "%s(%pU): proceeding with offset : %llu, "
+			     "size %d\n",
+			     __func__,
+			     handle,
+			     llu(*offset),
+			     (int)count);
+	}
+
+	if (count == 0) {
+		ret = 0;
+		goto out;
+	}
 
-	return -EINVAL;
+	while (iov_iter_count(iter)) {
+		size_t each_count = iov_iter_count(iter);
+		size_t amt_complete;
+		i++;
+
+		/* how much to transfer in this loop iteration */
+		if (each_count > orangefs_bufmap_size_query())
+			each_count = orangefs_bufmap_size_query();
+
+		gossip_debug(GOSSIP_FILE_DEBUG,
+			     "%s(%pU): size of each_count(%d)\n",
+			     __func__,
+			     handle,
+			     (int)each_count);
+		gossip_debug(GOSSIP_FILE_DEBUG,
+			     "%s(%pU): BEFORE wait_for_io: offset is %d\n",
+			     __func__,
+			     handle,
+			     (int)*offset);
+
+		ret = wait_for_direct_io(type, inode, offset, iter,
+				each_count, 0, NULL, NULL);
+		gossip_debug(GOSSIP_FILE_DEBUG,
+			     "%s(%pU): return from wait_for_io:%d\n",
+			     __func__,
+			     handle,
+			     (int)ret);
+
+		if (ret < 0)
+			goto out;
+
+		*offset += ret;
+		total_count += ret;
+		amt_complete = ret;
+
+		gossip_debug(GOSSIP_FILE_DEBUG,
+			     "%s(%pU): AFTER wait_for_io: offset is %d\n",
+			     __func__,
+			     handle,
+			     (int)*offset);
+
+		/*
+		 * if we got a short I/O operations,
+		 * fall out and return what we got so far
+		 */
+		if (amt_complete < each_count)
+			break;
+	} /*end while */
+
+out:
+	if (total_count > 0)
+		ret = total_count;
+	if (ret > 0) {
+		if (type == ORANGEFS_IO_READ) {
+			file_accessed(file);
+		} else {
+			file_update_time(file);
+			if (*offset > i_size_read(inode))
+				i_size_write(inode, *offset);
+		}
+	}
+
+	gossip_debug(GOSSIP_FILE_DEBUG,
+		     "%s(%pU): Value(%d) returned.\n",
+		     __func__,
+		     handle,
+		     (int)ret);
+
+	return ret;
 }
 
 /** ORANGEFS2 implementation of address space operations */
 static const struct address_space_operations orangefs_address_operations = {
+	.writepage = orangefs_writepage,
 	.readpage = orangefs_readpage,
-	.readpages = orangefs_readpages,
+	.writepages = orangefs_writepages,
+	.set_page_dirty = __set_page_dirty_nobuffers,
+	.write_begin = orangefs_write_begin,
+	.write_end = orangefs_write_end,
 	.invalidatepage = orangefs_invalidatepage,
 	.releasepage = orangefs_releasepage,
+	.freepage = orangefs_freepage,
+	.launder_page = orangefs_launder_page,
 	.direct_IO = orangefs_direct_IO,
 };
 
+vm_fault_t orangefs_page_mkwrite(struct vm_fault *vmf)
+{
+	struct page *page = vmf->page;
+	struct inode *inode = file_inode(vmf->vma->vm_file);
+	struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode);
+	unsigned long *bitlock = &orangefs_inode->bitlock;
+	vm_fault_t ret;
+	struct orangefs_write_range *wr;
+
+	sb_start_pagefault(inode->i_sb);
+
+	if (wait_on_bit(bitlock, 1, TASK_KILLABLE)) {
+		ret = VM_FAULT_RETRY;
+		goto out;
+	}
+
+	lock_page(page);
+	if (PageDirty(page) && !PagePrivate(page)) {
+		/*
+		 * Should be impossible.  If it happens, launder the page
+		 * since we don't know what's dirty.  This will WARN in
+		 * orangefs_writepage_locked.
+		 */
+		if (orangefs_launder_page(page)) {
+			ret = VM_FAULT_LOCKED|VM_FAULT_RETRY;
+			goto out;
+		}
+	}
+	if (PagePrivate(page)) {
+		wr = (struct orangefs_write_range *)page_private(page);
+		if (uid_eq(wr->uid, current_fsuid()) &&
+		    gid_eq(wr->gid, current_fsgid())) {
+			wr->pos = page_offset(page);
+			wr->len = PAGE_SIZE;
+			goto okay;
+		} else {
+			if (orangefs_launder_page(page)) {
+				ret = VM_FAULT_LOCKED|VM_FAULT_RETRY;
+				goto out;
+			}
+		}
+	}
+	wr = kmalloc(sizeof *wr, GFP_KERNEL);
+	if (!wr) {
+		ret = VM_FAULT_LOCKED|VM_FAULT_RETRY;
+		goto out;
+	}
+	wr->pos = page_offset(page);
+	wr->len = PAGE_SIZE;
+	wr->uid = current_fsuid();
+	wr->gid = current_fsgid();
+	SetPagePrivate(page);
+	set_page_private(page, (unsigned long)wr);
+	get_page(page);
+okay:
+
+	file_update_time(vmf->vma->vm_file);
+	if (page->mapping != inode->i_mapping) {
+		unlock_page(page);
+		ret = VM_FAULT_LOCKED|VM_FAULT_NOPAGE;
+		goto out;
+	}
+
+	/*
+	 * We mark the page dirty already here so that when freeze is in
+	 * progress, we are guaranteed that writeback during freezing will
+	 * see the dirty page and writeprotect it again.
+	 */
+	set_page_dirty(page);
+	wait_for_stable_page(page);
+	ret = VM_FAULT_LOCKED;
+out:
+	sb_end_pagefault(inode->i_sb);
+	return ret;
+}
+
 static int orangefs_setattr_size(struct inode *inode, struct iattr *iattr)
 {
 	struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode);
@@ -162,7 +808,7 @@ static int orangefs_setattr_size(struct inode *inode, struct iattr *iattr)
 		     iattr->ia_size);
 
 	/* Ensure that we have a up to date size, so we know if it changed. */
-	ret = orangefs_inode_getattr(inode, 0, 1, STATX_SIZE);
+	ret = orangefs_inode_getattr(inode, ORANGEFS_GETATTR_SIZE);
 	if (ret == -ESTALE)
 		ret = -EIO;
 	if (ret) {
@@ -172,7 +818,11 @@ static int orangefs_setattr_size(struct inode *inode, struct iattr *iattr)
 	}
 	orig_size = i_size_read(inode);
 
-	truncate_setsize(inode, iattr->ia_size);
+	/* This is truncate_setsize in a different order. */
+	truncate_pagecache(inode, iattr->ia_size);
+	i_size_write(inode, iattr->ia_size);
+	if (iattr->ia_size > orig_size)
+		pagecache_isize_extended(inode, orig_size, iattr->ia_size);
 
 	new_op = op_alloc(ORANGEFS_VFS_OP_TRUNCATE);
 	if (!new_op)
@@ -202,22 +852,33 @@ static int orangefs_setattr_size(struct inode *inode, struct iattr *iattr)
 	return ret;
 }
 
-/*
- * Change attributes of an object referenced by dentry.
- */
-int orangefs_setattr(struct dentry *dentry, struct iattr *iattr)
+int __orangefs_setattr(struct inode *inode, struct iattr *iattr)
 {
-	int ret = -EINVAL;
-	struct inode *inode = dentry->d_inode;
-
-	gossip_debug(GOSSIP_INODE_DEBUG,
-		"%s: called on %pd\n",
-		__func__,
-		dentry);
+	int ret;
 
-	ret = setattr_prepare(dentry, iattr);
-	if (ret)
-		goto out;
+	if (iattr->ia_valid & ATTR_MODE) {
+		if (iattr->ia_mode & (S_ISVTX)) {
+			if (is_root_handle(inode)) {
+				/*
+				 * allow sticky bit to be set on root (since
+				 * it shows up that way by default anyhow),
+				 * but don't show it to the server
+				 */
+				iattr->ia_mode -= S_ISVTX;
+			} else {
+				gossip_debug(GOSSIP_UTILS_DEBUG,
+					     "User attempted to set sticky bit on non-root directory; returning EINVAL.\n");
+				ret = -EINVAL;
+				goto out;
+			}
+		}
+		if (iattr->ia_mode & (S_ISUID)) {
+			gossip_debug(GOSSIP_UTILS_DEBUG,
+				     "Attempting to set setuid bit (not supported); returning EINVAL.\n");
+			ret = -EINVAL;
+			goto out;
+		}
+	}
 
 	if (iattr->ia_valid & ATTR_SIZE) {
 		ret = orangefs_setattr_size(inode, iattr);
@@ -225,21 +886,51 @@ int orangefs_setattr(struct dentry *dentry, struct iattr *iattr)
 			goto out;
 	}
 
+again:
+	spin_lock(&inode->i_lock);
+	if (ORANGEFS_I(inode)->attr_valid) {
+		if (uid_eq(ORANGEFS_I(inode)->attr_uid, current_fsuid()) &&
+		    gid_eq(ORANGEFS_I(inode)->attr_gid, current_fsgid())) {
+			ORANGEFS_I(inode)->attr_valid = iattr->ia_valid;
+		} else {
+			spin_unlock(&inode->i_lock);
+			write_inode_now(inode, 1);
+			goto again;
+		}
+	} else {
+		ORANGEFS_I(inode)->attr_valid = iattr->ia_valid;
+		ORANGEFS_I(inode)->attr_uid = current_fsuid();
+		ORANGEFS_I(inode)->attr_gid = current_fsgid();
+	}
 	setattr_copy(inode, iattr);
+	spin_unlock(&inode->i_lock);
 	mark_inode_dirty(inode);
 
-	ret = orangefs_inode_setattr(inode, iattr);
-	gossip_debug(GOSSIP_INODE_DEBUG,
-		"%s: orangefs_inode_setattr returned %d\n",
-		__func__,
-		ret);
-
-	if (!ret && (iattr->ia_valid & ATTR_MODE))
+	if (iattr->ia_valid & ATTR_MODE)
 		/* change mod on a file that has ACLs */
 		ret = posix_acl_chmod(inode, inode->i_mode);
 
+	ret = 0;
 out:
-	gossip_debug(GOSSIP_INODE_DEBUG, "%s: ret:%d:\n", __func__, ret);
+	return ret;
+}
+
+/*
+ * Change attributes of an object referenced by dentry.
+ */
+int orangefs_setattr(struct dentry *dentry, struct iattr *iattr)
+{
+	int ret;
+	gossip_debug(GOSSIP_INODE_DEBUG, "__orangefs_setattr: called on %pd\n",
+	    dentry);
+	ret = setattr_prepare(dentry, iattr);
+	if (ret)
+	        goto out;
+	ret = __orangefs_setattr(d_inode(dentry), iattr);
+	sync_inode_metadata(d_inode(dentry), 1);
+out:
+	gossip_debug(GOSSIP_INODE_DEBUG, "orangefs_setattr: returning %d\n",
+	    ret);
 	return ret;
 }
 
@@ -253,10 +944,11 @@ int orangefs_getattr(const struct path *path, struct kstat *stat,
 	struct inode *inode = path->dentry->d_inode;
 
 	gossip_debug(GOSSIP_INODE_DEBUG,
-		     "orangefs_getattr: called on %pd\n",
-		     path->dentry);
+		     "orangefs_getattr: called on %pd mask %u\n",
+		     path->dentry, request_mask);
 
-	ret = orangefs_inode_getattr(inode, 0, 0, request_mask);
+	ret = orangefs_inode_getattr(inode,
+	    request_mask & STATX_SIZE ? ORANGEFS_GETATTR_SIZE : 0);
 	if (ret == 0) {
 		generic_fillattr(inode, stat);
 
@@ -284,7 +976,7 @@ int orangefs_permission(struct inode *inode, int mask)
 	gossip_debug(GOSSIP_INODE_DEBUG, "%s: refreshing\n", __func__);
 
 	/* Make sure the permission (and other common attrs) are up to date. */
-	ret = orangefs_inode_getattr(inode, 0, 0, STATX_MODE);
+	ret = orangefs_inode_getattr(inode, 0);
 	if (ret < 0)
 		return ret;
 
@@ -304,7 +996,7 @@ int orangefs_update_time(struct inode *inode, struct timespec64 *time, int flags
 		iattr.ia_valid |= ATTR_CTIME;
 	if (flags & S_MTIME)
 		iattr.ia_valid |= ATTR_MTIME;
-	return orangefs_inode_setattr(inode, &iattr);
+	return __orangefs_setattr(inode, &iattr);
 }
 
 /* ORANGEFS2 implementation of VFS inode operations for files */
@@ -364,6 +1056,10 @@ static int orangefs_set_inode(struct inode *inode, void *data)
 	struct orangefs_object_kref *ref = (struct orangefs_object_kref *) data;
 	ORANGEFS_I(inode)->refn.fs_id = ref->fs_id;
 	ORANGEFS_I(inode)->refn.khandle = ref->khandle;
+	ORANGEFS_I(inode)->attr_valid = 0;
+	hash_init(ORANGEFS_I(inode)->xattr_cache);
+	ORANGEFS_I(inode)->mapping_time = jiffies - 1;
+	ORANGEFS_I(inode)->bitlock = 0;
 	return 0;
 }
 
@@ -409,7 +1105,7 @@ struct inode *orangefs_iget(struct super_block *sb,
 	if (!(inode->i_state & I_NEW))
 		return inode;
 
-	error = orangefs_inode_getattr(inode, 1, 1, STATX_ALL);
+	error = orangefs_inode_getattr(inode, ORANGEFS_GETATTR_NEW);
 	if (error) {
 		iget_failed(inode);
 		return ERR_PTR(error);
@@ -454,17 +1150,11 @@ struct inode *orangefs_new_inode(struct super_block *sb, struct inode *dir,
 	orangefs_set_inode(inode, ref);
 	inode->i_ino = hash;	/* needed for stat etc */
 
-	error = orangefs_inode_getattr(inode, 1, 1, STATX_ALL);
+	error = orangefs_inode_getattr(inode, ORANGEFS_GETATTR_NEW);
 	if (error)
 		goto out_iput;
 
 	orangefs_init_iops(inode);
-
-	inode->i_mode = mode;
-	inode->i_uid = current_fsuid();
-	inode->i_gid = current_fsgid();
-	inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode);
-	inode->i_size = PAGE_SIZE;
 	inode->i_rdev = dev;
 
 	error = insert_inode_locked4(inode, hash, orangefs_test_inode, ref);
diff --git a/fs/orangefs/namei.c b/fs/orangefs/namei.c
index c8676c996249..1dd710e5f376 100644
--- a/fs/orangefs/namei.c
+++ b/fs/orangefs/namei.c
@@ -76,19 +76,16 @@ static int orangefs_create(struct inode *dir,
 
 	d_instantiate_new(dentry, inode);
 	orangefs_set_timeout(dentry);
-	ORANGEFS_I(inode)->getattr_time = jiffies - 1;
-	ORANGEFS_I(inode)->getattr_mask = STATX_BASIC_STATS;
 
 	gossip_debug(GOSSIP_NAME_DEBUG,
 		     "%s: dentry instantiated for %pd\n",
 		     __func__,
 		     dentry);
 
-	dir->i_mtime = dir->i_ctime = current_time(dir);
 	memset(&iattr, 0, sizeof iattr);
-	iattr.ia_valid |= ATTR_MTIME;
-	orangefs_inode_setattr(dir, &iattr);
-	mark_inode_dirty_sync(dir);
+	iattr.ia_valid |= ATTR_MTIME | ATTR_CTIME;
+	iattr.ia_mtime = iattr.ia_ctime = current_time(dir);
+	__orangefs_setattr(dir, &iattr);
 	ret = 0;
 out:
 	op_release(new_op);
@@ -210,11 +207,10 @@ static int orangefs_unlink(struct inode *dir, struct dentry *dentry)
 	if (!ret) {
 		drop_nlink(inode);
 
-		dir->i_mtime = dir->i_ctime = current_time(dir);
 		memset(&iattr, 0, sizeof iattr);
-		iattr.ia_valid |= ATTR_MTIME;
-		orangefs_inode_setattr(dir, &iattr);
-		mark_inode_dirty_sync(dir);
+		iattr.ia_valid |= ATTR_MTIME | ATTR_CTIME;
+		iattr.ia_mtime = iattr.ia_ctime = current_time(dir);
+		__orangefs_setattr(dir, &iattr);
 	}
 	return ret;
 }
@@ -291,19 +287,16 @@ static int orangefs_symlink(struct inode *dir,
 
 	d_instantiate_new(dentry, inode);
 	orangefs_set_timeout(dentry);
-	ORANGEFS_I(inode)->getattr_time = jiffies - 1;
-	ORANGEFS_I(inode)->getattr_mask = STATX_BASIC_STATS;
 
 	gossip_debug(GOSSIP_NAME_DEBUG,
 		     "Inode (Symlink) %pU -> %pd\n",
 		     get_khandle_from_ino(inode),
 		     dentry);
 
-	dir->i_mtime = dir->i_ctime = current_time(dir);
 	memset(&iattr, 0, sizeof iattr);
-	iattr.ia_valid |= ATTR_MTIME;
-	orangefs_inode_setattr(dir, &iattr);
-	mark_inode_dirty_sync(dir);
+	iattr.ia_valid |= ATTR_MTIME | ATTR_CTIME;
+	iattr.ia_mtime = iattr.ia_ctime = current_time(dir);
+	__orangefs_setattr(dir, &iattr);
 	ret = 0;
 out:
 	op_release(new_op);
@@ -360,8 +353,6 @@ static int orangefs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode
 
 	d_instantiate_new(dentry, inode);
 	orangefs_set_timeout(dentry);
-	ORANGEFS_I(inode)->getattr_time = jiffies - 1;
-	ORANGEFS_I(inode)->getattr_mask = STATX_BASIC_STATS;
 
 	gossip_debug(GOSSIP_NAME_DEBUG,
 		     "Inode (Directory) %pU -> %pd\n",
@@ -372,11 +363,10 @@ static int orangefs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode
 	 * NOTE: we have no good way to keep nlink consistent for directories
 	 * across clients; keep constant at 1.
 	 */
-	dir->i_mtime = dir->i_ctime = current_time(dir);
 	memset(&iattr, 0, sizeof iattr);
-	iattr.ia_valid |= ATTR_MTIME;
-	orangefs_inode_setattr(dir, &iattr);
-	mark_inode_dirty_sync(dir);
+	iattr.ia_valid |= ATTR_MTIME | ATTR_CTIME;
+	iattr.ia_mtime = iattr.ia_ctime = current_time(dir);
+	__orangefs_setattr(dir, &iattr);
 out:
 	op_release(new_op);
 	return ret;
@@ -389,6 +379,7 @@ static int orangefs_rename(struct inode *old_dir,
 			unsigned int flags)
 {
 	struct orangefs_kernel_op_s *new_op;
+	struct iattr iattr;
 	int ret;
 
 	if (flags)
@@ -398,7 +389,10 @@ static int orangefs_rename(struct inode *old_dir,
 		     "orangefs_rename: called (%pd2 => %pd2) ct=%d\n",
 		     old_dentry, new_dentry, d_count(new_dentry));
 
-	ORANGEFS_I(new_dentry->d_parent->d_inode)->getattr_time = jiffies - 1;
+	memset(&iattr, 0, sizeof iattr);
+	iattr.ia_valid |= ATTR_MTIME | ATTR_CTIME;
+	iattr.ia_mtime = iattr.ia_ctime = current_time(new_dir);
+	__orangefs_setattr(new_dir, &iattr);
 
 	new_op = op_alloc(ORANGEFS_VFS_OP_RENAME);
 	if (!new_op)
diff --git a/fs/orangefs/orangefs-bufmap.c b/fs/orangefs/orangefs-bufmap.c
index 443bcd8c3c19..d4811f981608 100644
--- a/fs/orangefs/orangefs-bufmap.c
+++ b/fs/orangefs/orangefs-bufmap.c
@@ -538,3 +538,16 @@ int orangefs_bufmap_copy_to_iovec(struct iov_iter *iter,
 	}
 	return 0;
 }
+
+void orangefs_bufmap_page_fill(void *page_to,
+				int buffer_index,
+				int slot_index)
+{
+	struct orangefs_bufmap_desc *from;
+	void *page_from;
+
+	from = &__orangefs_bufmap->desc_array[buffer_index];
+	page_from = kmap_atomic(from->page_array[slot_index]);
+	memcpy(page_to, page_from, PAGE_SIZE);
+	kunmap_atomic(page_from);
+}
diff --git a/fs/orangefs/orangefs-bufmap.h b/fs/orangefs/orangefs-bufmap.h
index c2c3c5a0eeab..75b2d2833af1 100644
--- a/fs/orangefs/orangefs-bufmap.h
+++ b/fs/orangefs/orangefs-bufmap.h
@@ -34,4 +34,6 @@ int orangefs_bufmap_copy_to_iovec(struct iov_iter *iter,
 			      int buffer_index,
 			      size_t size);
 
+void orangefs_bufmap_page_fill(void *kaddr, int buffer_index, int slot_index);
+
 #endif /* __ORANGEFS_BUFMAP_H */
diff --git a/fs/orangefs/orangefs-debugfs.c b/fs/orangefs/orangefs-debugfs.c
index 0732cb08173e..87b1a6fce628 100644
--- a/fs/orangefs/orangefs-debugfs.c
+++ b/fs/orangefs/orangefs-debugfs.c
@@ -963,7 +963,7 @@ int orangefs_debugfs_new_client_mask(void __user *arg)
 	return ret;
 }
 
-int orangefs_debugfs_new_client_string(void __user *arg) 
+int orangefs_debugfs_new_client_string(void __user *arg)
 {
 	int ret;
 
@@ -1016,7 +1016,7 @@ int orangefs_debugfs_new_client_string(void __user *arg)
 	return 0;
 }
 
-int orangefs_debugfs_new_debug(void __user *arg) 
+int orangefs_debugfs_new_debug(void __user *arg)
 {
 	struct dev_mask_info_s mask_info = {0};
 	int ret;
diff --git a/fs/orangefs/orangefs-kernel.h b/fs/orangefs/orangefs-kernel.h
index 17b24ad6b264..572dd29fbd54 100644
--- a/fs/orangefs/orangefs-kernel.h
+++ b/fs/orangefs/orangefs-kernel.h
@@ -51,6 +51,7 @@
 #include <linux/rwsem.h>
 #include <linux/xattr.h>
 #include <linux/exportfs.h>
+#include <linux/hashtable.h>
 
 #include <asm/unaligned.h>
 
@@ -192,7 +193,13 @@ struct orangefs_inode_s {
 	sector_t last_failed_block_index_read;
 
 	unsigned long getattr_time;
-	u32 getattr_mask;
+	unsigned long mapping_time;
+	int attr_valid;
+	kuid_t attr_uid;
+	kgid_t attr_gid;
+	unsigned long bitlock;
+
+	DECLARE_HASHTABLE(xattr_cache, 4);
 };
 
 /* per superblock private orangefs info */
@@ -217,6 +224,25 @@ struct orangefs_stats {
 	unsigned long writes;
 };
 
+struct orangefs_cached_xattr {
+	struct hlist_node node;
+	char key[ORANGEFS_MAX_XATTR_NAMELEN];
+	char val[ORANGEFS_MAX_XATTR_VALUELEN];
+	ssize_t length;
+	unsigned long timeout;
+};
+
+struct orangefs_write_range {
+	loff_t pos;
+	size_t len;
+	kuid_t uid;
+	kgid_t gid;
+};
+
+struct orangefs_read_options {
+	ssize_t blksiz;
+};
+
 extern struct orangefs_stats orangefs_stats;
 
 /*
@@ -329,13 +355,15 @@ void fsid_key_table_finalize(void);
 /*
  * defined in inode.c
  */
+vm_fault_t orangefs_page_mkwrite(struct vm_fault *);
 struct inode *orangefs_new_inode(struct super_block *sb,
 			      struct inode *dir,
 			      int mode,
 			      dev_t dev,
 			      struct orangefs_object_kref *ref);
 
-int orangefs_setattr(struct dentry *dentry, struct iattr *iattr);
+int __orangefs_setattr(struct inode *, struct iattr *);
+int orangefs_setattr(struct dentry *, struct iattr *);
 
 int orangefs_getattr(const struct path *path, struct kstat *stat,
 		     u32 request_mask, unsigned int flags);
@@ -355,11 +383,6 @@ ssize_t orangefs_listxattr(struct dentry *dentry, char *buffer, size_t size);
 struct inode *orangefs_iget(struct super_block *sb,
 			 struct orangefs_object_kref *ref);
 
-ssize_t orangefs_inode_read(struct inode *inode,
-			    struct iov_iter *iter,
-			    loff_t *offset,
-			    loff_t readahead_size);
-
 /*
  * defined in devorangefs-req.c
  */
@@ -371,6 +394,15 @@ int is_daemon_in_service(void);
 bool __is_daemon_in_service(void);
 
 /*
+ * defined in file.c
+ */
+int orangefs_revalidate_mapping(struct inode *);
+ssize_t wait_for_direct_io(enum ORANGEFS_io_type, struct inode *, loff_t *,
+    struct iov_iter *, size_t, loff_t, struct orangefs_write_range *, int *);
+ssize_t do_readv_writev(enum ORANGEFS_io_type, struct file *, loff_t *,
+    struct iov_iter *);
+
+/*
  * defined in orangefs-utils.c
  */
 __s32 fsid_of_op(struct orangefs_kernel_op_s *op);
@@ -386,12 +418,14 @@ int orangefs_inode_setxattr(struct inode *inode,
 			 size_t size,
 			 int flags);
 
-int orangefs_inode_getattr(struct inode *inode, int new, int bypass,
-    u32 request_mask);
+#define ORANGEFS_GETATTR_NEW 1
+#define ORANGEFS_GETATTR_SIZE 2
+
+int orangefs_inode_getattr(struct inode *, int);
 
 int orangefs_inode_check_changed(struct inode *inode);
 
-int orangefs_inode_setattr(struct inode *inode, struct iattr *iattr);
+int orangefs_inode_setattr(struct inode *inode);
 
 bool orangefs_cancel_op_in_progress(struct orangefs_kernel_op_s *op);
 
@@ -400,6 +434,7 @@ int orangefs_normalize_to_errno(__s32 error_code);
 extern struct mutex orangefs_request_mutex;
 extern int op_timeout_secs;
 extern int slot_timeout_secs;
+extern int orangefs_cache_timeout_msecs;
 extern int orangefs_dcache_timeout_msecs;
 extern int orangefs_getattr_timeout_msecs;
 extern struct list_head orangefs_superblocks;
@@ -426,6 +461,7 @@ extern const struct dentry_operations orangefs_dentry_operations;
 #define ORANGEFS_OP_CANCELLATION  4   /* this is a cancellation */
 #define ORANGEFS_OP_NO_MUTEX      8   /* don't acquire request_mutex */
 #define ORANGEFS_OP_ASYNC         16  /* Queue it, but don't wait */
+#define ORANGEFS_OP_WRITEBACK     32
 
 int service_operation(struct orangefs_kernel_op_s *op,
 		      const char *op_name,
diff --git a/fs/orangefs/orangefs-mod.c b/fs/orangefs/orangefs-mod.c
index 85ef87245a87..82cf8b3e568b 100644
--- a/fs/orangefs/orangefs-mod.c
+++ b/fs/orangefs/orangefs-mod.c
@@ -30,6 +30,7 @@ static ulong module_parm_debug_mask;
 __u64 orangefs_gossip_debug_mask;
 int op_timeout_secs = ORANGEFS_DEFAULT_OP_TIMEOUT_SECS;
 int slot_timeout_secs = ORANGEFS_DEFAULT_SLOT_TIMEOUT_SECS;
+int orangefs_cache_timeout_msecs = 50;
 int orangefs_dcache_timeout_msecs = 50;
 int orangefs_getattr_timeout_msecs = 50;
 
diff --git a/fs/orangefs/orangefs-sysfs.c b/fs/orangefs/orangefs-sysfs.c
index 19739aaee675..3627ea946402 100644
--- a/fs/orangefs/orangefs-sysfs.c
+++ b/fs/orangefs/orangefs-sysfs.c
@@ -62,6 +62,14 @@
  *			Slots are requested and waited for,
  *			the wait times out after slot_timeout_secs.
  *
+ * What:		/sys/fs/orangefs/cache_timeout_msecs
+ * Date:		Mar 2018
+ * Contact:		Martin Brandenburg <martin@omnibond.com>
+ * Description:
+ *			Time in milliseconds between which
+ *			orangefs_revalidate_mapping will invalidate the page
+ *			cache.
+ *
  * What:		/sys/fs/orangefs/dcache_timeout_msecs
  * Date:		Jul 2016
  * Contact:		Martin Brandenburg <martin@omnibond.com>
@@ -222,6 +230,13 @@ static ssize_t sysfs_int_show(struct kobject *kobj,
 				       slot_timeout_secs);
 			goto out;
 		} else if (!strcmp(attr->attr.name,
+				   "cache_timeout_msecs")) {
+			rc = scnprintf(buf,
+				       PAGE_SIZE,
+				       "%d\n",
+				       orangefs_cache_timeout_msecs);
+			goto out;
+		} else if (!strcmp(attr->attr.name,
 				   "dcache_timeout_msecs")) {
 			rc = scnprintf(buf,
 				       PAGE_SIZE,
@@ -277,6 +292,9 @@ static ssize_t sysfs_int_store(struct kobject *kobj,
 	} else if (!strcmp(attr->attr.name, "slot_timeout_secs")) {
 		rc = kstrtoint(buf, 0, &slot_timeout_secs);
 		goto out;
+	} else if (!strcmp(attr->attr.name, "cache_timeout_msecs")) {
+		rc = kstrtoint(buf, 0, &orangefs_cache_timeout_msecs);
+		goto out;
 	} else if (!strcmp(attr->attr.name, "dcache_timeout_msecs")) {
 		rc = kstrtoint(buf, 0, &orangefs_dcache_timeout_msecs);
 		goto out;
@@ -818,6 +836,9 @@ static struct orangefs_attribute op_timeout_secs_attribute =
 static struct orangefs_attribute slot_timeout_secs_attribute =
 	__ATTR(slot_timeout_secs, 0664, sysfs_int_show, sysfs_int_store);
 
+static struct orangefs_attribute cache_timeout_msecs_attribute =
+	__ATTR(cache_timeout_msecs, 0664, sysfs_int_show, sysfs_int_store);
+
 static struct orangefs_attribute dcache_timeout_msecs_attribute =
 	__ATTR(dcache_timeout_msecs, 0664, sysfs_int_show, sysfs_int_store);
 
@@ -861,6 +882,7 @@ static struct orangefs_attribute perf_time_interval_secs_attribute =
 static struct attribute *orangefs_default_attrs[] = {
 	&op_timeout_secs_attribute.attr,
 	&slot_timeout_secs_attribute.attr,
+	&cache_timeout_msecs_attribute.attr,
 	&dcache_timeout_msecs_attribute.attr,
 	&getattr_timeout_msecs_attribute.attr,
 	&readahead_count_attribute.attr,
diff --git a/fs/orangefs/orangefs-utils.c b/fs/orangefs/orangefs-utils.c
index 804c8a261e4b..d4b7ae763186 100644
--- a/fs/orangefs/orangefs-utils.c
+++ b/fs/orangefs/orangefs-utils.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
  * (C) 2001 Clemson University and The University of Chicago
+ * Copyright 2018 Omnibond Systems, L.L.C.
  *
  * See COPYING in top-level directory.
  */
@@ -135,51 +136,37 @@ static int orangefs_inode_perms(struct ORANGEFS_sys_attr_s *attrs)
  * NOTE: in kernel land, we never use the sys_attr->link_target for
  * anything, so don't bother copying it into the sys_attr object here.
  */
-static inline int copy_attributes_from_inode(struct inode *inode,
-					     struct ORANGEFS_sys_attr_s *attrs,
-					     struct iattr *iattr)
+static inline void copy_attributes_from_inode(struct inode *inode,
+    struct ORANGEFS_sys_attr_s *attrs)
 {
-	umode_t tmp_mode;
-
-	if (!iattr || !inode || !attrs) {
-		gossip_err("NULL iattr (%p), inode (%p), attrs (%p) "
-			   "in copy_attributes_from_inode!\n",
-			   iattr,
-			   inode,
-			   attrs);
-		return -EINVAL;
-	}
-	/*
-	 * We need to be careful to only copy the attributes out of the
-	 * iattr object that we know are valid.
-	 */
+	struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode);
 	attrs->mask = 0;
-	if (iattr->ia_valid & ATTR_UID) {
-		attrs->owner = from_kuid(&init_user_ns, iattr->ia_uid);
+	if (orangefs_inode->attr_valid & ATTR_UID) {
+		attrs->owner = from_kuid(&init_user_ns, inode->i_uid);
 		attrs->mask |= ORANGEFS_ATTR_SYS_UID;
 		gossip_debug(GOSSIP_UTILS_DEBUG, "(UID) %d\n", attrs->owner);
 	}
-	if (iattr->ia_valid & ATTR_GID) {
-		attrs->group = from_kgid(&init_user_ns, iattr->ia_gid);
+	if (orangefs_inode->attr_valid & ATTR_GID) {
+		attrs->group = from_kgid(&init_user_ns, inode->i_gid);
 		attrs->mask |= ORANGEFS_ATTR_SYS_GID;
 		gossip_debug(GOSSIP_UTILS_DEBUG, "(GID) %d\n", attrs->group);
 	}
 
-	if (iattr->ia_valid & ATTR_ATIME) {
+	if (orangefs_inode->attr_valid & ATTR_ATIME) {
 		attrs->mask |= ORANGEFS_ATTR_SYS_ATIME;
-		if (iattr->ia_valid & ATTR_ATIME_SET) {
-			attrs->atime = (time64_t)iattr->ia_atime.tv_sec;
+		if (orangefs_inode->attr_valid & ATTR_ATIME_SET) {
+			attrs->atime = (time64_t)inode->i_atime.tv_sec;
 			attrs->mask |= ORANGEFS_ATTR_SYS_ATIME_SET;
 		}
 	}
-	if (iattr->ia_valid & ATTR_MTIME) {
+	if (orangefs_inode->attr_valid & ATTR_MTIME) {
 		attrs->mask |= ORANGEFS_ATTR_SYS_MTIME;
-		if (iattr->ia_valid & ATTR_MTIME_SET) {
-			attrs->mtime = (time64_t)iattr->ia_mtime.tv_sec;
+		if (orangefs_inode->attr_valid & ATTR_MTIME_SET) {
+			attrs->mtime = (time64_t)inode->i_mtime.tv_sec;
 			attrs->mask |= ORANGEFS_ATTR_SYS_MTIME_SET;
 		}
 	}
-	if (iattr->ia_valid & ATTR_CTIME)
+	if (orangefs_inode->attr_valid & ATTR_CTIME)
 		attrs->mask |= ORANGEFS_ATTR_SYS_CTIME;
 
 	/*
@@ -188,36 +175,10 @@ static inline int copy_attributes_from_inode(struct inode *inode,
 	 * worry about ATTR_SIZE
 	 */
 
-	if (iattr->ia_valid & ATTR_MODE) {
-		tmp_mode = iattr->ia_mode;
-		if (tmp_mode & (S_ISVTX)) {
-			if (is_root_handle(inode)) {
-				/*
-				 * allow sticky bit to be set on root (since
-				 * it shows up that way by default anyhow),
-				 * but don't show it to the server
-				 */
-				tmp_mode -= S_ISVTX;
-			} else {
-				gossip_debug(GOSSIP_UTILS_DEBUG,
-					"%s: setting sticky bit not supported.\n",
-					__func__);
-				return -EINVAL;
-			}
-		}
-
-		if (tmp_mode & (S_ISUID)) {
-			gossip_debug(GOSSIP_UTILS_DEBUG,
-				"%s: setting setuid bit not supported.\n",
-				__func__);
-			return -EINVAL;
-		}
-
-		attrs->perms = ORANGEFS_util_translate_mode(tmp_mode);
+	if (orangefs_inode->attr_valid & ATTR_MODE) {
+		attrs->perms = ORANGEFS_util_translate_mode(inode->i_mode);
 		attrs->mask |= ORANGEFS_ATTR_SYS_PERM;
 	}
-
-	return 0;
 }
 
 static int orangefs_inode_type(enum orangefs_ds_type objtype)
@@ -272,27 +233,30 @@ static int orangefs_inode_is_stale(struct inode *inode,
 	return 0;
 }
 
-int orangefs_inode_getattr(struct inode *inode, int new, int bypass,
-    u32 request_mask)
+int orangefs_inode_getattr(struct inode *inode, int flags)
 {
 	struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode);
 	struct orangefs_kernel_op_s *new_op;
 	loff_t inode_size;
 	int ret, type;
 
-	gossip_debug(GOSSIP_UTILS_DEBUG, "%s: called on inode %pU\n", __func__,
-	    get_khandle_from_ino(inode));
+	gossip_debug(GOSSIP_UTILS_DEBUG, "%s: called on inode %pU flags %d\n",
+	    __func__, get_khandle_from_ino(inode), flags);
 
-	if (!new && !bypass) {
-		/*
-		 * Must have all the attributes in the mask and be within cache
-		 * time.
-		 */
-		if ((request_mask & orangefs_inode->getattr_mask) ==
-		    request_mask &&
-		    time_before(jiffies, orangefs_inode->getattr_time))
-			return 0;
+again:
+	spin_lock(&inode->i_lock);
+	/* Must have all the attributes in the mask and be within cache time. */
+	if ((!flags && time_before(jiffies, orangefs_inode->getattr_time)) ||
+	    orangefs_inode->attr_valid || inode->i_state & I_DIRTY_PAGES) {
+		if (orangefs_inode->attr_valid) {
+			spin_unlock(&inode->i_lock);
+			write_inode_now(inode, 1);
+			goto again;
+		}
+		spin_unlock(&inode->i_lock);
+		return 0;
 	}
+	spin_unlock(&inode->i_lock);
 
 	new_op = op_alloc(ORANGEFS_VFS_OP_GETATTR);
 	if (!new_op)
@@ -302,7 +266,7 @@ int orangefs_inode_getattr(struct inode *inode, int new, int bypass,
 	 * Size is the hardest attribute to get.  The incremental cost of any
 	 * other attribute is essentially zero.
 	 */
-	if (request_mask & STATX_SIZE || new)
+	if (flags)
 		new_op->upcall.req.getattr.mask = ORANGEFS_ATTR_SYS_ALL_NOHINT;
 	else
 		new_op->upcall.req.getattr.mask =
@@ -313,13 +277,33 @@ int orangefs_inode_getattr(struct inode *inode, int new, int bypass,
 	if (ret != 0)
 		goto out;
 
-	if (!new) {
+again2:
+	spin_lock(&inode->i_lock);
+	/* Must have all the attributes in the mask and be within cache time. */
+	if ((!flags && time_before(jiffies, orangefs_inode->getattr_time)) ||
+	    orangefs_inode->attr_valid || inode->i_state & I_DIRTY_PAGES) {
+		if (orangefs_inode->attr_valid) {
+			spin_unlock(&inode->i_lock);
+			write_inode_now(inode, 1);
+			goto again2;
+		}
+		if (inode->i_state & I_DIRTY_PAGES) {
+			ret = 0;
+			goto out_unlock;
+		}
+		gossip_debug(GOSSIP_UTILS_DEBUG, "%s: in cache or dirty\n",
+		    __func__);
+		ret = 0;
+		goto out_unlock;
+	}
+
+	if (!(flags & ORANGEFS_GETATTR_NEW)) {
 		ret = orangefs_inode_is_stale(inode,
 		    &new_op->downcall.resp.getattr.attributes,
 		    new_op->downcall.resp.getattr.link_target);
 		if (ret) {
 			ret = -ESTALE;
-			goto out;
+			goto out_unlock;
 		}
 	}
 
@@ -329,30 +313,26 @@ int orangefs_inode_getattr(struct inode *inode, int new, int bypass,
 	case S_IFREG:
 		inode->i_flags = orangefs_inode_flags(&new_op->
 		    downcall.resp.getattr.attributes);
-		if (request_mask & STATX_SIZE || new) {
+		if (flags) {
 			inode_size = (loff_t)new_op->
 			    downcall.resp.getattr.attributes.size;
 			inode->i_size = inode_size;
 			inode->i_blkbits = ffs(new_op->downcall.resp.getattr.
 			    attributes.blksize);
-			spin_lock(&inode->i_lock);
 			inode->i_bytes = inode_size;
 			inode->i_blocks =
 			    (inode_size + 512 - inode_size % 512)/512;
-			spin_unlock(&inode->i_lock);
 		}
 		break;
 	case S_IFDIR:
-		if (request_mask & STATX_SIZE || new) {
+		if (flags) {
 			inode->i_size = PAGE_SIZE;
-			spin_lock(&inode->i_lock);
 			inode_set_bytes(inode, inode->i_size);
-			spin_unlock(&inode->i_lock);
 		}
 		set_nlink(inode, 1);
 		break;
 	case S_IFLNK:
-		if (new) {
+		if (flags & ORANGEFS_GETATTR_NEW) {
 			inode->i_size = (loff_t)strlen(new_op->
 			    downcall.resp.getattr.link_target);
 			ret = strscpy(orangefs_inode->link_target,
@@ -360,7 +340,7 @@ int orangefs_inode_getattr(struct inode *inode, int new, int bypass,
 			    ORANGEFS_NAME_MAX);
 			if (ret == -E2BIG) {
 				ret = -EIO;
-				goto out;
+				goto out_unlock;
 			}
 			inode->i_link = orangefs_inode->link_target;
 		}
@@ -370,7 +350,7 @@ int orangefs_inode_getattr(struct inode *inode, int new, int bypass,
 		/* XXX: ESTALE?  This is what is done if it is not new. */
 		orangefs_make_bad_inode(inode);
 		ret = -ESTALE;
-		goto out;
+		goto out_unlock;
 	}
 
 	inode->i_uid = make_kuid(&init_user_ns, new_op->
@@ -393,11 +373,9 @@ int orangefs_inode_getattr(struct inode *inode, int new, int bypass,
 
 	orangefs_inode->getattr_time = jiffies +
 	    orangefs_getattr_timeout_msecs*HZ/1000;
-	if (request_mask & STATX_SIZE || new)
-		orangefs_inode->getattr_mask = STATX_BASIC_STATS;
-	else
-		orangefs_inode->getattr_mask = STATX_BASIC_STATS & ~STATX_SIZE;
 	ret = 0;
+out_unlock:
+	spin_unlock(&inode->i_lock);
 out:
 	op_release(new_op);
 	return ret;
@@ -436,7 +414,7 @@ out:
  * issues a orangefs setattr request to make sure the new attribute values
  * take effect if successful.  returns 0 on success; -errno otherwise
  */
-int orangefs_inode_setattr(struct inode *inode, struct iattr *iattr)
+int orangefs_inode_setattr(struct inode *inode)
 {
 	struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode);
 	struct orangefs_kernel_op_s *new_op;
@@ -446,24 +424,31 @@ int orangefs_inode_setattr(struct inode *inode, struct iattr *iattr)
 	if (!new_op)
 		return -ENOMEM;
 
+	spin_lock(&inode->i_lock);
+	new_op->upcall.uid = from_kuid(&init_user_ns, orangefs_inode->attr_uid);
+	new_op->upcall.gid = from_kgid(&init_user_ns, orangefs_inode->attr_gid);
 	new_op->upcall.req.setattr.refn = orangefs_inode->refn;
-	ret = copy_attributes_from_inode(inode,
-		       &new_op->upcall.req.setattr.attributes,
-		       iattr);
-	if (ret >= 0) {
-		ret = service_operation(new_op, __func__,
-				get_interruptible_flag(inode));
-
-		gossip_debug(GOSSIP_UTILS_DEBUG,
-			     "orangefs_inode_setattr: returning %d\n",
-			     ret);
+	copy_attributes_from_inode(inode,
+	    &new_op->upcall.req.setattr.attributes);
+	orangefs_inode->attr_valid = 0;
+	if (!new_op->upcall.req.setattr.attributes.mask) {
+		spin_unlock(&inode->i_lock);
+		op_release(new_op);
+		return 0;
 	}
+	spin_unlock(&inode->i_lock);
+
+	ret = service_operation(new_op, __func__,
+	    get_interruptible_flag(inode) | ORANGEFS_OP_WRITEBACK);
+	gossip_debug(GOSSIP_UTILS_DEBUG,
+	    "orangefs_inode_setattr: returning %d\n", ret);
+	if (ret)
+		orangefs_make_bad_inode(inode);
 
 	op_release(new_op);
 
 	if (ret == 0)
 		orangefs_inode->getattr_time = jiffies - 1;
-
 	return ret;
 }
 
diff --git a/fs/orangefs/super.c b/fs/orangefs/super.c
index 3784f7e8b603..ee5efdc35cc1 100644
--- a/fs/orangefs/super.c
+++ b/fs/orangefs/super.c
@@ -10,6 +10,7 @@
 #include "orangefs-bufmap.h"
 
 #include <linux/parser.h>
+#include <linux/hashtable.h>
 
 /* a cache for orangefs-inode objects (i.e. orangefs inode private data) */
 static struct kmem_cache *orangefs_inode_cache;
@@ -126,7 +127,17 @@ static struct inode *orangefs_alloc_inode(struct super_block *sb)
 
 static void orangefs_free_inode(struct inode *inode)
 {
-	kmem_cache_free(orangefs_inode_cache, ORANGEFS_I(inode));
+	struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode);
+	struct orangefs_cached_xattr *cx;
+	struct hlist_node *tmp;
+	int i;
+
+	hash_for_each_safe(orangefs_inode->xattr_cache, i, tmp, cx, node) {
+		hlist_del(&cx->node);
+		kfree(cx);
+	}
+
+	kmem_cache_free(orangefs_inode_cache, orangefs_inode);
 }
 
 static void orangefs_destroy_inode(struct inode *inode)
@@ -138,6 +149,13 @@ static void orangefs_destroy_inode(struct inode *inode)
 			__func__, orangefs_inode, get_khandle_from_ino(inode));
 }
 
+static int orangefs_write_inode(struct inode *inode,
+				struct writeback_control *wbc)
+{
+	gossip_debug(GOSSIP_SUPER_DEBUG, "orangefs_write_inode\n");
+	return orangefs_inode_setattr(inode);
+}
+
 /*
  * NOTE: information filled in here is typically reflected in the
  * output of the system command 'df'
@@ -297,6 +315,7 @@ static const struct super_operations orangefs_s_ops = {
 	.alloc_inode = orangefs_alloc_inode,
 	.free_inode = orangefs_free_inode,
 	.destroy_inode = orangefs_destroy_inode,
+	.write_inode = orangefs_write_inode,
 	.drop_inode = generic_delete_inode,
 	.statfs = orangefs_statfs,
 	.remount_fs = orangefs_remount_fs,
@@ -394,15 +413,11 @@ static int orangefs_fill_sb(struct super_block *sb,
 		struct orangefs_fs_mount_response *fs_mount,
 		void *data, int silent)
 {
-	int ret = -EINVAL;
-	struct inode *root = NULL;
-	struct dentry *root_dentry = NULL;
+	int ret;
+	struct inode *root;
+	struct dentry *root_dentry;
 	struct orangefs_object_kref root_object;
 
-	/* alloc and init our private orangefs sb info */
-	sb->s_fs_info = kzalloc(sizeof(struct orangefs_sb_info_s), GFP_KERNEL);
-	if (!ORANGEFS_SB(sb))
-		return -ENOMEM;
 	ORANGEFS_SB(sb)->sb = sb;
 
 	ORANGEFS_SB(sb)->root_khandle = fs_mount->root_khandle;
@@ -425,6 +440,10 @@ static int orangefs_fill_sb(struct super_block *sb,
 	sb->s_blocksize_bits = PAGE_SHIFT;
 	sb->s_maxbytes = MAX_LFS_FILESIZE;
 
+	ret = super_setup_bdi(sb);
+	if (ret)
+		return ret;
+
 	root_object.khandle = ORANGEFS_SB(sb)->root_khandle;
 	root_object.fs_id = ORANGEFS_SB(sb)->fs_id;
 	gossip_debug(GOSSIP_SUPER_DEBUG,
@@ -503,6 +522,13 @@ struct dentry *orangefs_mount(struct file_system_type *fst,
 		goto free_op;
 	}
 
+	/* alloc and init our private orangefs sb info */
+	sb->s_fs_info = kzalloc(sizeof(struct orangefs_sb_info_s), GFP_KERNEL);
+	if (!ORANGEFS_SB(sb)) {
+		d = ERR_PTR(-ENOMEM);
+		goto free_op;
+	}
+
 	ret = orangefs_fill_sb(sb,
 	      &new_op->downcall.resp.fs_mount, data,
 	      flags & SB_SILENT ? 1 : 0);
diff --git a/fs/orangefs/waitqueue.c b/fs/orangefs/waitqueue.c
index 0729d2645d6a..beafc33d57be 100644
--- a/fs/orangefs/waitqueue.c
+++ b/fs/orangefs/waitqueue.c
@@ -19,7 +19,7 @@
 
 static int wait_for_matching_downcall(struct orangefs_kernel_op_s *op,
 		long timeout,
-		bool interruptible)
+		int flags)
 			__acquires(op->lock);
 static void orangefs_clean_up_interrupted_operation(struct orangefs_kernel_op_s *op)
 	__releases(op->lock);
@@ -143,9 +143,7 @@ retry_servicing:
 	if (!(flags & ORANGEFS_OP_NO_MUTEX))
 		mutex_unlock(&orangefs_request_mutex);
 
-	ret = wait_for_matching_downcall(op, timeout,
-					 flags & ORANGEFS_OP_INTERRUPTIBLE);
-
+	ret = wait_for_matching_downcall(op, timeout, flags);
 	gossip_debug(GOSSIP_WAIT_DEBUG,
 		     "%s: wait_for_matching_downcall returned %d for %p\n",
 		     __func__,
@@ -319,10 +317,12 @@ static void
  */
 static int wait_for_matching_downcall(struct orangefs_kernel_op_s *op,
 		long timeout,
-		bool interruptible)
+		int flags)
 			__acquires(op->lock)
 {
 	long n;
+	int writeback = flags & ORANGEFS_OP_WRITEBACK,
+	    interruptible = flags & ORANGEFS_OP_INTERRUPTIBLE;
 
 	/*
 	 * There's a "schedule_timeout" inside of these wait
@@ -330,10 +330,12 @@ static int wait_for_matching_downcall(struct orangefs_kernel_op_s *op,
 	 * user process that needs something done and is being
 	 * manipulated by the client-core process.
 	 */
-	if (interruptible)
+	if (writeback)
+		n = wait_for_completion_io_timeout(&op->waitq, timeout);
+	else if (!writeback && interruptible)
 		n = wait_for_completion_interruptible_timeout(&op->waitq,
-							      timeout);
-	else
+								      timeout);
+	else /* !writeback && !interruptible but compiler complains */
 		n = wait_for_completion_killable_timeout(&op->waitq, timeout);
 
 	spin_lock(&op->lock);
diff --git a/fs/orangefs/xattr.c b/fs/orangefs/xattr.c
index 03bcb871544d..bdc285aea360 100644
--- a/fs/orangefs/xattr.c
+++ b/fs/orangefs/xattr.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
  * (C) 2001 Clemson University and The University of Chicago
+ * Copyright 2018 Omnibond Systems, L.L.C.
  *
  * See COPYING in top-level directory.
  */
@@ -14,7 +15,7 @@
 #include "orangefs-bufmap.h"
 #include <linux/posix_acl_xattr.h>
 #include <linux/xattr.h>
-
+#include <linux/hashtable.h>
 
 #define SYSTEM_ORANGEFS_KEY "system.pvfs2."
 #define SYSTEM_ORANGEFS_KEY_LEN 13
@@ -50,6 +51,35 @@ static inline int convert_to_internal_xattr_flags(int setxattr_flags)
 	return internal_flag;
 }
 
+static unsigned int xattr_key(const char *key)
+{
+	unsigned int i = 0;
+	while (key)
+		i += *key++;
+	return i % 16;
+}
+
+static struct orangefs_cached_xattr *find_cached_xattr(struct inode *inode,
+    const char *key)
+{
+	struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode);
+	struct orangefs_cached_xattr *cx;
+	struct hlist_head *h;
+	struct hlist_node *tmp;
+	h = &orangefs_inode->xattr_cache[xattr_key(key)];
+	if (hlist_empty(h))
+		return NULL;
+	hlist_for_each_entry_safe(cx, tmp, h, node) {
+/*		if (!time_before(jiffies, cx->timeout)) {
+			hlist_del(&cx->node);
+			kfree(cx);
+			continue;
+		}*/
+		if (!strcmp(cx->key, key))
+			return cx;
+	}
+	return NULL;
+}
 
 /*
  * Tries to get a specified key's attributes of a given
@@ -65,6 +95,7 @@ ssize_t orangefs_inode_getxattr(struct inode *inode, const char *name,
 {
 	struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode);
 	struct orangefs_kernel_op_s *new_op = NULL;
+	struct orangefs_cached_xattr *cx;
 	ssize_t ret = -ENOMEM;
 	ssize_t length = 0;
 	int fsuid;
@@ -93,6 +124,27 @@ ssize_t orangefs_inode_getxattr(struct inode *inode, const char *name,
 
 	down_read(&orangefs_inode->xattr_sem);
 
+	cx = find_cached_xattr(inode, name);
+	if (cx && time_before(jiffies, cx->timeout)) {
+		if (cx->length == -1) {
+			ret = -ENODATA;
+			goto out_unlock;
+		} else {
+			if (size == 0) {
+				ret = cx->length;
+				goto out_unlock;
+			}
+			if (cx->length > size) {
+				ret = -ERANGE;
+				goto out_unlock;
+			}
+			memcpy(buffer, cx->val, cx->length);
+			memset(buffer + cx->length, 0, size - cx->length);
+			ret = cx->length;
+			goto out_unlock;
+		}
+	}
+
 	new_op = op_alloc(ORANGEFS_VFS_OP_GETXATTR);
 	if (!new_op)
 		goto out_unlock;
@@ -117,6 +169,15 @@ ssize_t orangefs_inode_getxattr(struct inode *inode, const char *name,
 				     " does not exist!\n",
 				     get_khandle_from_ino(inode),
 				     (char *)new_op->upcall.req.getxattr.key);
+			cx = kmalloc(sizeof *cx, GFP_KERNEL);
+			if (cx) {
+				strcpy(cx->key, name);
+				cx->length = -1;
+				cx->timeout = jiffies +
+				    orangefs_getattr_timeout_msecs*HZ/1000;
+				hash_add(orangefs_inode->xattr_cache, &cx->node,
+				    xattr_key(cx->key));
+			}
 		}
 		goto out_release_op;
 	}
@@ -156,6 +217,23 @@ ssize_t orangefs_inode_getxattr(struct inode *inode, const char *name,
 
 	ret = length;
 
+	if (cx) {
+		strcpy(cx->key, name);
+		memcpy(cx->val, buffer, length);
+		cx->length = length;
+		cx->timeout = jiffies + HZ;
+	} else {
+		cx = kmalloc(sizeof *cx, GFP_KERNEL);
+		if (cx) {
+			strcpy(cx->key, name);
+			memcpy(cx->val, buffer, length);
+			cx->length = length;
+			cx->timeout = jiffies + HZ;
+			hash_add(orangefs_inode->xattr_cache, &cx->node,
+			    xattr_key(cx->key));
+		}
+	}
+
 out_release_op:
 	op_release(new_op);
 out_unlock:
@@ -168,6 +246,9 @@ static int orangefs_inode_removexattr(struct inode *inode, const char *name,
 {
 	struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode);
 	struct orangefs_kernel_op_s *new_op = NULL;
+	struct orangefs_cached_xattr *cx;
+	struct hlist_head *h;
+	struct hlist_node *tmp;
 	int ret = -ENOMEM;
 
 	if (strlen(name) >= ORANGEFS_MAX_XATTR_NAMELEN)
@@ -209,6 +290,16 @@ static int orangefs_inode_removexattr(struct inode *inode, const char *name,
 		     "orangefs_inode_removexattr: returning %d\n", ret);
 
 	op_release(new_op);
+
+	h = &orangefs_inode->xattr_cache[xattr_key(name)];
+	hlist_for_each_entry_safe(cx, tmp, h, node) {
+		if (!strcmp(cx->key, name)) {
+			hlist_del(&cx->node);
+			kfree(cx);
+			break;
+		}
+	}
+
 out_unlock:
 	up_write(&orangefs_inode->xattr_sem);
 	return ret;
@@ -226,6 +317,9 @@ int orangefs_inode_setxattr(struct inode *inode, const char *name,
 	struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode);
 	struct orangefs_kernel_op_s *new_op;
 	int internal_flag = 0;
+	struct orangefs_cached_xattr *cx;
+	struct hlist_head *h;
+	struct hlist_node *tmp;
 	int ret = -ENOMEM;
 
 	gossip_debug(GOSSIP_XATTR_DEBUG,
@@ -287,6 +381,16 @@ int orangefs_inode_setxattr(struct inode *inode, const char *name,
 
 	/* when request is serviced properly, free req op struct */
 	op_release(new_op);
+
+	h = &orangefs_inode->xattr_cache[xattr_key(name)];
+	hlist_for_each_entry_safe(cx, tmp, h, node) {
+		if (!strcmp(cx->key, name)) {
+			hlist_del(&cx->node);
+			kfree(cx);
+			break;
+		}
+	}
+
 out_unlock:
 	up_write(&orangefs_inode->xattr_sem);
 	return ret;
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index f8f6f04c4453..bbb9e332f2fe 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -844,6 +844,7 @@
 	EXIT_CALL							\
 	*(.discard)							\
 	*(.discard.*)							\
+	*(.modinfo)							\
 	}
 
 /**
diff --git a/include/drm/drm_atomic.h b/include/drm/drm_atomic.h
index 824a5ed4e216..e937ff2beb04 100644
--- a/include/drm/drm_atomic.h
+++ b/include/drm/drm_atomic.h
@@ -452,6 +452,12 @@ void drm_atomic_private_obj_fini(struct drm_private_obj *obj);
 struct drm_private_state * __must_check
 drm_atomic_get_private_obj_state(struct drm_atomic_state *state,
 				 struct drm_private_obj *obj);
+struct drm_private_state *
+drm_atomic_get_old_private_obj_state(struct drm_atomic_state *state,
+				     struct drm_private_obj *obj);
+struct drm_private_state *
+drm_atomic_get_new_private_obj_state(struct drm_atomic_state *state,
+				     struct drm_private_obj *obj);
 
 /**
  * drm_atomic_get_existing_crtc_state - get crtc state, if it exists
diff --git a/include/drm/drm_audio_component.h b/include/drm/drm_audio_component.h
index 93a386be38fa..a45f93487039 100644
--- a/include/drm/drm_audio_component.h
+++ b/include/drm/drm_audio_component.h
@@ -19,14 +19,17 @@ struct drm_audio_component_ops {
 	 * @get_power: get the POWER_DOMAIN_AUDIO power well
 	 *
 	 * Request the power well to be turned on.
+	 *
+	 * Returns a wakeref cookie to be passed back to the corresponding
+	 * call to @put_power.
 	 */
-	void (*get_power)(struct device *);
+	unsigned long (*get_power)(struct device *);
 	/**
 	 * @put_power: put the POWER_DOMAIN_AUDIO power well
 	 *
 	 * Allow the power well to be turned off.
 	 */
-	void (*put_power)(struct device *);
+	void (*put_power)(struct device *, unsigned long);
 	/**
 	 * @codec_wake_override: Enable/disable codec wake signal
 	 */
diff --git a/include/drm/drm_auth.h b/include/drm/drm_auth.h
index 86bff9841b54..871008118bab 100644
--- a/include/drm/drm_auth.h
+++ b/include/drm/drm_auth.h
@@ -50,7 +50,6 @@ struct drm_lock_data {
  *
  * @refcount: Refcount for this master object.
  * @dev: Link back to the DRM device
- * @lock: DRI1 lock information.
  * @driver_priv: Pointer to driver-private information.
  * @lessor: Lease holder
  * @lessee_id: id for lessees. Owners always have id 0
@@ -80,7 +79,6 @@ struct drm_master {
 	 * &drm_device.master_mutex.
 	 */
 	struct idr magic_map;
-	struct drm_lock_data lock;
 	void *driver_priv;
 
 	/* Tree of display resource leases, each of which is a drm_master struct
@@ -95,6 +93,10 @@ struct drm_master {
 	struct list_head lessees;
 	struct idr leases;
 	struct idr lessee_idr;
+	/* private: */
+#if IS_ENABLED(CONFIG_DRM_LEGACY)
+	struct drm_lock_data lock;
+#endif
 };
 
 struct drm_master *drm_master_get(struct drm_master *master);
diff --git a/include/drm/drm_bridge.h b/include/drm/drm_bridge.h
index 9da8c93f7976..d4428913a4e1 100644
--- a/include/drm/drm_bridge.h
+++ b/include/drm/drm_bridge.h
@@ -244,14 +244,13 @@ struct drm_bridge_funcs {
  */
 struct drm_bridge_timings {
 	/**
-	 * @sampling_edge:
+	 * @input_bus_flags:
 	 *
-	 * Tells whether the bridge samples the digital input signal
-	 * from the display engine on the positive or negative edge of the
-	 * clock, this should reuse the DRM_BUS_FLAG_PIXDATA_[POS|NEG]EDGE
-	 * bitwise flags from the DRM connector (bit 2 and 3 valid).
+	 * Tells what additional settings for the pixel data on the bus
+	 * this bridge requires (like pixel signal polarity). See also
+	 * &drm_display_info->bus_flags.
 	 */
-	u32 sampling_edge;
+	u32 input_bus_flags;
 	/**
 	 * @setup_time_ps:
 	 *
diff --git a/include/drm/drm_cache.h b/include/drm/drm_cache.h
index 97fc498dc767..987ff16b9420 100644
--- a/include/drm/drm_cache.h
+++ b/include/drm/drm_cache.h
@@ -38,7 +38,7 @@
 void drm_clflush_pages(struct page *pages[], unsigned long num_pages);
 void drm_clflush_sg(struct sg_table *st);
 void drm_clflush_virt_range(void *addr, unsigned long length);
-u64 drm_get_max_iomem(void);
+bool drm_need_swiotlb(int dma_bits);
 
 
 static inline bool drm_arch_can_wc_memory(void)
diff --git a/include/drm/drm_client.h b/include/drm/drm_client.h
index 8b552b1a6ce9..268b2cf0052a 100644
--- a/include/drm/drm_client.h
+++ b/include/drm/drm_client.h
@@ -90,7 +90,7 @@ struct drm_client_dev {
 int drm_client_init(struct drm_device *dev, struct drm_client_dev *client,
 		    const char *name, const struct drm_client_funcs *funcs);
 void drm_client_release(struct drm_client_dev *client);
-void drm_client_add(struct drm_client_dev *client);
+void drm_client_register(struct drm_client_dev *client);
 
 void drm_client_dev_unregister(struct drm_device *dev);
 void drm_client_dev_hotplug(struct drm_device *dev);
diff --git a/include/drm/drm_connector.h b/include/drm/drm_connector.h
index 8fe22abb1e10..02a131202add 100644
--- a/include/drm/drm_connector.h
+++ b/include/drm/drm_connector.h
@@ -253,6 +253,96 @@ enum drm_panel_orientation {
 	DRM_MODE_PANEL_ORIENTATION_RIGHT_UP,
 };
 
+/*
+ * This is a consolidated colorimetry list supported by HDMI and
+ * DP protocol standard. The respective connectors will register
+ * a property with the subset of this list (supported by that
+ * respective protocol). Userspace will set the colorspace through
+ * a colorspace property which will be created and exposed to
+ * userspace.
+ */
+
+/* For Default case, driver will set the colorspace */
+#define DRM_MODE_COLORIMETRY_DEFAULT			0
+/* CEA 861 Normal Colorimetry options */
+#define DRM_MODE_COLORIMETRY_NO_DATA			0
+#define DRM_MODE_COLORIMETRY_SMPTE_170M_YCC		1
+#define DRM_MODE_COLORIMETRY_BT709_YCC			2
+/* CEA 861 Extended Colorimetry Options */
+#define DRM_MODE_COLORIMETRY_XVYCC_601			3
+#define DRM_MODE_COLORIMETRY_XVYCC_709			4
+#define DRM_MODE_COLORIMETRY_SYCC_601			5
+#define DRM_MODE_COLORIMETRY_OPYCC_601			6
+#define DRM_MODE_COLORIMETRY_OPRGB			7
+#define DRM_MODE_COLORIMETRY_BT2020_CYCC		8
+#define DRM_MODE_COLORIMETRY_BT2020_RGB			9
+#define DRM_MODE_COLORIMETRY_BT2020_YCC			10
+/* Additional Colorimetry extension added as part of CTA 861.G */
+#define DRM_MODE_COLORIMETRY_DCI_P3_RGB_D65		11
+#define DRM_MODE_COLORIMETRY_DCI_P3_RGB_THEATER		12
+
+/**
+ * enum drm_bus_flags - bus_flags info for &drm_display_info
+ *
+ * This enum defines signal polarities and clock edge information for signals on
+ * a bus as bitmask flags.
+ *
+ * The clock edge information is conveyed by two sets of symbols,
+ * DRM_BUS_FLAGS_*_DRIVE_\* and DRM_BUS_FLAGS_*_SAMPLE_\*. When this enum is
+ * used to describe a bus from the point of view of the transmitter, the
+ * \*_DRIVE_\* flags should be used. When used from the point of view of the
+ * receiver, the \*_SAMPLE_\* flags should be used. The \*_DRIVE_\* and
+ * \*_SAMPLE_\* flags alias each other, with the \*_SAMPLE_POSEDGE and
+ * \*_SAMPLE_NEGEDGE flags being equal to \*_DRIVE_NEGEDGE and \*_DRIVE_POSEDGE
+ * respectively. This simplifies code as signals are usually sampled on the
+ * opposite edge of the driving edge. Transmitters and receivers may however
+ * need to take other signal timings into account to convert between driving
+ * and sample edges.
+ *
+ * @DRM_BUS_FLAG_DE_LOW:		The Data Enable signal is active low
+ * @DRM_BUS_FLAG_DE_HIGH:		The Data Enable signal is active high
+ * @DRM_BUS_FLAG_PIXDATA_POSEDGE:	Legacy value, do not use
+ * @DRM_BUS_FLAG_PIXDATA_NEGEDGE:	Legacy value, do not use
+ * @DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE:	Data is driven on the rising edge of
+ *					the pixel clock
+ * @DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE:	Data is driven on the falling edge of
+ *					the pixel clock
+ * @DRM_BUS_FLAG_PIXDATA_SAMPLE_POSEDGE: Data is sampled on the rising edge of
+ *					the pixel clock
+ * @DRM_BUS_FLAG_PIXDATA_SAMPLE_NEGEDGE: Data is sampled on the falling edge of
+ *					the pixel clock
+ * @DRM_BUS_FLAG_DATA_MSB_TO_LSB:	Data is transmitted MSB to LSB on the bus
+ * @DRM_BUS_FLAG_DATA_LSB_TO_MSB:	Data is transmitted LSB to MSB on the bus
+ * @DRM_BUS_FLAG_SYNC_POSEDGE:		Legacy value, do not use
+ * @DRM_BUS_FLAG_SYNC_NEGEDGE:		Legacy value, do not use
+ * @DRM_BUS_FLAG_SYNC_DRIVE_POSEDGE:	Sync signals are driven on the rising
+ *					edge of the pixel clock
+ * @DRM_BUS_FLAG_SYNC_DRIVE_NEGEDGE:	Sync signals are driven on the falling
+ *					edge of the pixel clock
+ * @DRM_BUS_FLAG_SYNC_SAMPLE_POSEDGE:	Sync signals are sampled on the rising
+ *					edge of the pixel clock
+ * @DRM_BUS_FLAG_SYNC_SAMPLE_NEGEDGE:	Sync signals are sampled on the falling
+ *					edge of the pixel clock
+ */
+enum drm_bus_flags {
+	DRM_BUS_FLAG_DE_LOW = BIT(0),
+	DRM_BUS_FLAG_DE_HIGH = BIT(1),
+	DRM_BUS_FLAG_PIXDATA_POSEDGE = BIT(2),
+	DRM_BUS_FLAG_PIXDATA_NEGEDGE = BIT(3),
+	DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE = DRM_BUS_FLAG_PIXDATA_POSEDGE,
+	DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE = DRM_BUS_FLAG_PIXDATA_NEGEDGE,
+	DRM_BUS_FLAG_PIXDATA_SAMPLE_POSEDGE = DRM_BUS_FLAG_PIXDATA_NEGEDGE,
+	DRM_BUS_FLAG_PIXDATA_SAMPLE_NEGEDGE = DRM_BUS_FLAG_PIXDATA_POSEDGE,
+	DRM_BUS_FLAG_DATA_MSB_TO_LSB = BIT(4),
+	DRM_BUS_FLAG_DATA_LSB_TO_MSB = BIT(5),
+	DRM_BUS_FLAG_SYNC_POSEDGE = BIT(6),
+	DRM_BUS_FLAG_SYNC_NEGEDGE = BIT(7),
+	DRM_BUS_FLAG_SYNC_DRIVE_POSEDGE = DRM_BUS_FLAG_SYNC_POSEDGE,
+	DRM_BUS_FLAG_SYNC_DRIVE_NEGEDGE = DRM_BUS_FLAG_SYNC_NEGEDGE,
+	DRM_BUS_FLAG_SYNC_SAMPLE_POSEDGE = DRM_BUS_FLAG_SYNC_NEGEDGE,
+	DRM_BUS_FLAG_SYNC_SAMPLE_NEGEDGE = DRM_BUS_FLAG_SYNC_POSEDGE,
+};
+
 /**
  * struct drm_display_info - runtime data about the connected sink
  *
@@ -266,26 +356,16 @@ enum drm_panel_orientation {
  */
 struct drm_display_info {
 	/**
-	 * @name: Name of the display.
-	 */
-	char name[DRM_DISPLAY_INFO_LEN];
-
-	/**
 	 * @width_mm: Physical width in mm.
 	 */
-        unsigned int width_mm;
+	unsigned int width_mm;
+
 	/**
 	 * @height_mm: Physical height in mm.
 	 */
 	unsigned int height_mm;
 
 	/**
-	 * @pixel_clock: Maximum pixel clock supported by the sink, in units of
-	 * 100Hz. This mismatches the clock in &drm_display_mode (which is in
-	 * kHZ), because that's what the EDID uses as base unit.
-	 */
-	unsigned int pixel_clock;
-	/**
 	 * @bpc: Maximum bits per color channel. Used by HDMI and DP outputs.
 	 */
 	unsigned int bpc;
@@ -328,24 +408,10 @@ struct drm_display_info {
 	 */
 	unsigned int num_bus_formats;
 
-#define DRM_BUS_FLAG_DE_LOW		(1<<0)
-#define DRM_BUS_FLAG_DE_HIGH		(1<<1)
-/* drive data on pos. edge */
-#define DRM_BUS_FLAG_PIXDATA_POSEDGE	(1<<2)
-/* drive data on neg. edge */
-#define DRM_BUS_FLAG_PIXDATA_NEGEDGE	(1<<3)
-/* data is transmitted MSB to LSB on the bus */
-#define DRM_BUS_FLAG_DATA_MSB_TO_LSB	(1<<4)
-/* data is transmitted LSB to MSB on the bus */
-#define DRM_BUS_FLAG_DATA_LSB_TO_MSB	(1<<5)
-/* drive sync on pos. edge */
-#define DRM_BUS_FLAG_SYNC_POSEDGE	(1<<6)
-/* drive sync on neg. edge */
-#define DRM_BUS_FLAG_SYNC_NEGEDGE	(1<<7)
-
 	/**
 	 * @bus_flags: Additional information (like pixel signal polarity) for
-	 * the pixel data on the bus, using DRM_BUS_FLAGS\_ defines.
+	 * the pixel data on the bus, using &enum drm_bus_flags values
+	 * DRM_BUS_FLAGS\_.
 	 */
 	u32 bus_flags;
 
@@ -503,6 +569,13 @@ struct drm_connector_state {
 	unsigned int content_protection;
 
 	/**
+	 * @colorspace: State variable for Connector property to request
+	 * colorspace change on Sink. This is most commonly used to switch
+	 * to wider color gamuts like BT2020.
+	 */
+	u32 colorspace;
+
+	/**
 	 * @writeback_job: Writeback job for writeback connectors
 	 *
 	 * Holds the framebuffer and out-fence for a writeback connector. As
@@ -995,6 +1068,12 @@ struct drm_connector {
 	struct drm_property *content_protection_property;
 
 	/**
+	 * @colorspace_property: Connector property to set the suitable
+	 * colorspace supported by the sink.
+	 */
+	struct drm_property *colorspace_property;
+
+	/**
 	 * @path_blob_ptr:
 	 *
 	 * DRM blob property data for the DP MST path property. This should only
@@ -1269,6 +1348,7 @@ int drm_connector_attach_vrr_capable_property(
 int drm_connector_attach_content_protection_property(
 		struct drm_connector *connector);
 int drm_mode_create_aspect_ratio_property(struct drm_device *dev);
+int drm_mode_create_colorspace_property(struct drm_connector *connector);
 int drm_mode_create_content_type_property(struct drm_device *dev);
 void drm_hdmi_avi_infoframe_content_type(struct hdmi_avi_infoframe *frame,
 					 const struct drm_connector_state *conn_state);
diff --git a/include/drm/drm_crtc.h b/include/drm/drm_crtc.h
index 85abd3fe9e83..58ad983d7cd6 100644
--- a/include/drm/drm_crtc.h
+++ b/include/drm/drm_crtc.h
@@ -78,7 +78,7 @@ struct drm_plane_helper_funcs;
 /**
  * struct drm_crtc_state - mutable CRTC state
  *
- * Note that the distinction between @enable and @active is rather subtile:
+ * Note that the distinction between @enable and @active is rather subtle:
  * Flipping @active while @enable is set without changing anything else may
  * never return in a failure from the &drm_mode_config_funcs.atomic_check
  * callback. Userspace assumes that a DPMS On will always succeed. In other
@@ -472,7 +472,7 @@ struct drm_crtc_funcs {
 	/**
 	 * @destroy:
 	 *
-	 * Clean up plane resources. This is only called at driver unload time
+	 * Clean up CRTC resources. This is only called at driver unload time
 	 * through drm_mode_config_cleanup() since a CRTC cannot be hotplugged
 	 * in DRM.
 	 */
diff --git a/include/drm/drm_device.h b/include/drm/drm_device.h
index d5e092dccf3e..7f9ef709b2b6 100644
--- a/include/drm/drm_device.h
+++ b/include/drm/drm_device.h
@@ -306,7 +306,7 @@ struct drm_device {
 
 	/* Everything below here is for legacy driver, never use! */
 	/* private: */
-
+#if IS_ENABLED(CONFIG_DRM_LEGACY)
 	/* Context handle management - linked list of context handles */
 	struct list_head ctxlist;
 
@@ -353,6 +353,7 @@ struct drm_device {
 
 	/* Scatter gather memory */
 	struct drm_sg_mem *sg;
+#endif
 };
 
 #endif
diff --git a/include/drm/drm_drv.h b/include/drm/drm_drv.h
index 570f9d03b2eb..68ca736c548d 100644
--- a/include/drm/drm_drv.h
+++ b/include/drm/drm_drv.h
@@ -91,6 +91,13 @@ enum drm_driver_feature {
 	 * submission.
 	 */
 	DRIVER_SYNCOBJ                  = BIT(5),
+	/**
+	 * @DRIVER_SYNCOBJ_TIMELINE:
+	 *
+	 * Driver supports the timeline flavor of &drm_syncobj for explicit
+	 * synchronization of command submission.
+	 */
+	DRIVER_SYNCOBJ_TIMELINE         = BIT(6),
 
 	/* IMPORTANT: Below are all the legacy flags, add new ones above. */
 
@@ -152,9 +159,9 @@ enum drm_driver_feature {
 /**
  * struct drm_driver - DRM driver structure
  *
- * This structure represent the common code for a family of cards. There will
- * one drm_device for each card present in this family. It contains lots of
- * vfunc entries, and a pile of those probably should be moved to more
+ * This structure represent the common code for a family of cards. There will be
+ * one &struct drm_device for each card present in this family. It contains lots
+ * of vfunc entries, and a pile of those probably should be moved to more
  * appropriate places like &drm_mode_config_funcs or into a new operations
  * structure for GEM drivers.
  */
@@ -718,6 +725,9 @@ extern unsigned int drm_debug;
 int drm_dev_init(struct drm_device *dev,
 		 struct drm_driver *driver,
 		 struct device *parent);
+int devm_drm_dev_init(struct device *parent,
+		      struct drm_device *dev,
+		      struct drm_driver *driver);
 void drm_dev_fini(struct drm_device *dev);
 
 struct drm_device *drm_dev_alloc(struct drm_driver *driver,
diff --git a/include/drm/drm_dsc.h b/include/drm/drm_dsc.h
index 9c26f083c70f..887954cbfc60 100644
--- a/include/drm/drm_dsc.h
+++ b/include/drm/drm_dsc.h
@@ -101,9 +101,9 @@ struct drm_dsc_config {
 	 */
 	u16 slice_height;
 	/**
-	 * @enable422: True for 4_2_2 sampling, false for 4_4_4 sampling
+	 * @simple_422: True if simple 4_2_2 mode is enabled else False
 	 */
-	bool enable422;
+	bool simple_422;
 	/**
 	 * @pic_width: Width of the input display frame in pixels
 	 */
@@ -601,8 +601,9 @@ struct drm_dsc_pps_infoframe {
 	struct drm_dsc_picture_parameter_set pps_payload;
 } __packed;
 
-void drm_dsc_dp_pps_header_init(struct drm_dsc_pps_infoframe *pps_sdp);
-void drm_dsc_pps_infoframe_pack(struct drm_dsc_pps_infoframe *pps_sdp,
+void drm_dsc_dp_pps_header_init(struct dp_sdp_header *pps_header);
+void drm_dsc_pps_payload_pack(struct drm_dsc_picture_parameter_set *pps_sdp,
 				const struct drm_dsc_config *dsc_cfg);
+int drm_dsc_compute_rc_parameters(struct drm_dsc_config *vdsc_cfg);
 
 #endif /* _DRM_DSC_H_ */
diff --git a/include/drm/drm_edid.h b/include/drm/drm_edid.h
index 8dc1a081fb36..9d3b5b93102c 100644
--- a/include/drm/drm_edid.h
+++ b/include/drm/drm_edid.h
@@ -331,6 +331,7 @@ struct cea_sad {
 
 struct drm_encoder;
 struct drm_connector;
+struct drm_connector_state;
 struct drm_display_mode;
 
 int drm_edid_to_sad(struct edid *edid, struct cea_sad **sads);
@@ -358,6 +359,11 @@ int
 drm_hdmi_vendor_infoframe_from_display_mode(struct hdmi_vendor_infoframe *frame,
 					    struct drm_connector *connector,
 					    const struct drm_display_mode *mode);
+
+void
+drm_hdmi_avi_infoframe_colorspace(struct hdmi_avi_infoframe *frame,
+				  const struct drm_connector_state *conn_state);
+
 void
 drm_hdmi_avi_infoframe_quant_range(struct hdmi_avi_infoframe *frame,
 				   struct drm_connector *connector,
diff --git a/include/drm/drm_fb_helper.h b/include/drm/drm_fb_helper.h
index 286d58efed5d..40af2866c26a 100644
--- a/include/drm/drm_fb_helper.h
+++ b/include/drm/drm_fb_helper.h
@@ -68,10 +68,8 @@ struct drm_fb_helper_crtc {
  * according to the largest width/height (so it is large enough for all CRTCs
  * to scanout).  But the fbdev width/height is sized to the minimum width/
  * height of all the displays.  This ensures that fbcon fits on the smallest
- * of the attached displays.
- *
- * So what is passed to drm_fb_helper_fill_var() should be fb_width/fb_height,
- * rather than the surface size.
+ * of the attached displays. fb_width/fb_height is used by
+ * drm_fb_helper_fill_info() to fill out the &fb_info.var structure.
  */
 struct drm_fb_helper_surface_size {
 	u32 fb_width;
@@ -104,29 +102,6 @@ struct drm_fb_helper_funcs {
 	 */
 	int (*fb_probe)(struct drm_fb_helper *helper,
 			struct drm_fb_helper_surface_size *sizes);
-
-	/**
-	 * @initial_config:
-	 *
-	 * Driver callback to setup an initial fbdev display configuration.
-	 * Drivers can use this callback to tell the fbdev emulation what the
-	 * preferred initial configuration is. This is useful to implement
-	 * smooth booting where the fbdev (and subsequently all userspace) never
-	 * changes the mode, but always inherits the existing configuration.
-	 *
-	 * This callback is optional.
-	 *
-	 * RETURNS:
-	 *
-	 * The driver should return true if a suitable initial configuration has
-	 * been filled out and false when the fbdev helper should fall back to
-	 * the default probing logic.
-	 */
-	bool (*initial_config)(struct drm_fb_helper *fb_helper,
-			       struct drm_fb_helper_crtc **crtcs,
-			       struct drm_display_mode **modes,
-			       struct drm_fb_offset *offsets,
-			       bool *enabled, int width, int height);
 };
 
 struct drm_fb_helper_connector {
@@ -289,10 +264,9 @@ int drm_fb_helper_restore_fbdev_mode_unlocked(struct drm_fb_helper *fb_helper);
 
 struct fb_info *drm_fb_helper_alloc_fbi(struct drm_fb_helper *fb_helper);
 void drm_fb_helper_unregister_fbi(struct drm_fb_helper *fb_helper);
-void drm_fb_helper_fill_var(struct fb_info *info, struct drm_fb_helper *fb_helper,
-			    uint32_t fb_width, uint32_t fb_height);
-void drm_fb_helper_fill_fix(struct fb_info *info, uint32_t pitch,
-			    uint32_t depth);
+void drm_fb_helper_fill_info(struct fb_info *info,
+			     struct drm_fb_helper *fb_helper,
+			     struct drm_fb_helper_surface_size *sizes);
 
 void drm_fb_helper_unlink_fbi(struct drm_fb_helper *fb_helper);
 
@@ -418,14 +392,10 @@ static inline void drm_fb_helper_unregister_fbi(struct drm_fb_helper *fb_helper)
 {
 }
 
-static inline void drm_fb_helper_fill_var(struct fb_info *info,
-					  struct drm_fb_helper *fb_helper,
-					  uint32_t fb_width, uint32_t fb_height)
-{
-}
-
-static inline void drm_fb_helper_fill_fix(struct fb_info *info, uint32_t pitch,
-					  uint32_t depth)
+static inline void
+drm_fb_helper_fill_info(struct fb_info *info,
+			struct drm_fb_helper *fb_helper,
+			struct drm_fb_helper_surface_size *sizes)
 {
 }
 
diff --git a/include/drm/drm_file.h b/include/drm/drm_file.h
index 6710b612e2f6..67af60bb527a 100644
--- a/include/drm/drm_file.h
+++ b/include/drm/drm_file.h
@@ -335,7 +335,9 @@ struct drm_file {
 	struct drm_prime_file_private prime;
 
 	/* private: */
+#if IS_ENABLED(CONFIG_DRM_LEGACY)
 	unsigned long lock_count; /* DRI1 legacy lock count */
+#endif
 };
 
 /**
diff --git a/include/drm/drm_format_helper.h b/include/drm/drm_format_helper.h
new file mode 100644
index 000000000000..085d63faee12
--- /dev/null
+++ b/include/drm/drm_format_helper.h
@@ -0,0 +1,35 @@
+/*
+ * Copyright (C) 2016 Noralf Trønnes
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#ifndef __LINUX_DRM_FORMAT_HELPER_H
+#define __LINUX_DRM_FORMAT_HELPER_H
+
+struct drm_framebuffer;
+struct drm_rect;
+
+void drm_fb_memcpy(void *dst, void *vaddr, struct drm_framebuffer *fb,
+		   struct drm_rect *clip);
+void drm_fb_memcpy_dstclip(void __iomem *dst, void *vaddr,
+			   struct drm_framebuffer *fb,
+			   struct drm_rect *clip);
+void drm_fb_swab16(u16 *dst, void *vaddr, struct drm_framebuffer *fb,
+		   struct drm_rect *clip);
+void drm_fb_xrgb8888_to_rgb565(void *dst, void *vaddr,
+			       struct drm_framebuffer *fb,
+			       struct drm_rect *clip, bool swab);
+void drm_fb_xrgb8888_to_rgb565_dstclip(void __iomem *dst, unsigned int dst_pitch,
+				       void *vaddr, struct drm_framebuffer *fb,
+				       struct drm_rect *clip, bool swab);
+void drm_fb_xrgb8888_to_rgb888_dstclip(void __iomem *dst, unsigned int dst_pitch,
+				       void *vaddr, struct drm_framebuffer *fb,
+				       struct drm_rect *clip);
+void drm_fb_xrgb8888_to_gray8(u8 *dst, void *vaddr, struct drm_framebuffer *fb,
+			      struct drm_rect *clip);
+
+#endif /* __LINUX_DRM_FORMAT_HELPER_H */
diff --git a/include/drm/drm_framebuffer.h b/include/drm/drm_framebuffer.h
index f0b34c977ec5..c23016748e3f 100644
--- a/include/drm/drm_framebuffer.h
+++ b/include/drm/drm_framebuffer.h
@@ -32,6 +32,7 @@
 struct drm_clip_rect;
 struct drm_device;
 struct drm_file;
+struct drm_format_info;
 struct drm_framebuffer;
 struct drm_gem_object;
 
diff --git a/include/drm/drm_gem.h b/include/drm/drm_gem.h
index c95727425284..5047c7ee25f5 100644
--- a/include/drm/drm_gem.h
+++ b/include/drm/drm_gem.h
@@ -35,6 +35,7 @@
  */
 
 #include <linux/kref.h>
+#include <linux/reservation.h>
 
 #include <drm/drm_vma_manager.h>
 
@@ -263,6 +264,24 @@ struct drm_gem_object {
 	struct dma_buf_attachment *import_attach;
 
 	/**
+	 * @resv:
+	 *
+	 * Pointer to reservation object associated with the this GEM object.
+	 *
+	 * Normally (@resv == &@_resv) except for imported GEM objects.
+	 */
+	struct reservation_object *resv;
+
+	/**
+	 * @_resv:
+	 *
+	 * A reservation object for this GEM object.
+	 *
+	 * This is unused for imported GEM objects.
+	 */
+	struct reservation_object _resv;
+
+	/**
 	 * @funcs:
 	 *
 	 * Optional GEM object functions. If this is set, it will be used instead of the
@@ -362,7 +381,20 @@ struct page **drm_gem_get_pages(struct drm_gem_object *obj);
 void drm_gem_put_pages(struct drm_gem_object *obj, struct page **pages,
 		bool dirty, bool accessed);
 
+int drm_gem_objects_lookup(struct drm_file *filp, void __user *bo_handles,
+			   int count, struct drm_gem_object ***objs_out);
 struct drm_gem_object *drm_gem_object_lookup(struct drm_file *filp, u32 handle);
+long drm_gem_reservation_object_wait(struct drm_file *filep, u32 handle,
+				    bool wait_all, unsigned long timeout);
+int drm_gem_lock_reservations(struct drm_gem_object **objs, int count,
+			      struct ww_acquire_ctx *acquire_ctx);
+void drm_gem_unlock_reservations(struct drm_gem_object **objs, int count,
+				 struct ww_acquire_ctx *acquire_ctx);
+int drm_gem_fence_array_add(struct xarray *fence_array,
+			    struct dma_fence *fence);
+int drm_gem_fence_array_add_implicit(struct xarray *fence_array,
+				     struct drm_gem_object *obj,
+				     bool write);
 int drm_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev,
 			    u32 handle, u64 *offset);
 int drm_gem_dumb_destroy(struct drm_file *file,
diff --git a/include/drm/drm_gem_shmem_helper.h b/include/drm/drm_gem_shmem_helper.h
new file mode 100644
index 000000000000..038b6d313447
--- /dev/null
+++ b/include/drm/drm_gem_shmem_helper.h
@@ -0,0 +1,159 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef __DRM_GEM_SHMEM_HELPER_H__
+#define __DRM_GEM_SHMEM_HELPER_H__
+
+#include <linux/fs.h>
+#include <linux/mm.h>
+#include <linux/mutex.h>
+
+#include <drm/drm_file.h>
+#include <drm/drm_gem.h>
+#include <drm/drm_ioctl.h>
+#include <drm/drm_prime.h>
+
+struct dma_buf_attachment;
+struct drm_mode_create_dumb;
+struct drm_printer;
+struct sg_table;
+
+/**
+ * struct drm_gem_shmem_object - GEM object backed by shmem
+ */
+struct drm_gem_shmem_object {
+	/**
+	 * @base: Base GEM object
+	 */
+	struct drm_gem_object base;
+
+	/**
+	 * @pages_lock: Protects the page table and use count
+	 */
+	struct mutex pages_lock;
+
+	/**
+	 * @pages: Page table
+	 */
+	struct page **pages;
+
+	/**
+	 * @pages_use_count:
+	 *
+	 * Reference count on the pages table.
+	 * The pages are put when the count reaches zero.
+	 */
+	unsigned int pages_use_count;
+
+	/**
+	 * @pages_mark_dirty_on_put:
+	 *
+	 * Mark pages as dirty when they are put.
+	 */
+	unsigned int pages_mark_dirty_on_put    : 1;
+
+	/**
+	 * @pages_mark_accessed_on_put:
+	 *
+	 * Mark pages as accessed when they are put.
+	 */
+	unsigned int pages_mark_accessed_on_put : 1;
+
+	/**
+	 * @sgt: Scatter/gather table for imported PRIME buffers
+	 */
+	struct sg_table *sgt;
+
+	/**
+	 * @vmap_lock: Protects the vmap address and use count
+	 */
+	struct mutex vmap_lock;
+
+	/**
+	 * @vaddr: Kernel virtual address of the backing memory
+	 */
+	void *vaddr;
+
+	/**
+	 * @vmap_use_count:
+	 *
+	 * Reference count on the virtual address.
+	 * The address are un-mapped when the count reaches zero.
+	 */
+	unsigned int vmap_use_count;
+};
+
+#define to_drm_gem_shmem_obj(obj) \
+	container_of(obj, struct drm_gem_shmem_object, base)
+
+/**
+ * DEFINE_DRM_GEM_SHMEM_FOPS() - Macro to generate file operations for shmem drivers
+ * @name: name for the generated structure
+ *
+ * This macro autogenerates a suitable &struct file_operations for shmem based
+ * drivers, which can be assigned to &drm_driver.fops. Note that this structure
+ * cannot be shared between drivers, because it contains a reference to the
+ * current module using THIS_MODULE.
+ *
+ * Note that the declaration is already marked as static - if you need a
+ * non-static version of this you're probably doing it wrong and will break the
+ * THIS_MODULE reference by accident.
+ */
+#define DEFINE_DRM_GEM_SHMEM_FOPS(name) \
+	static const struct file_operations name = {\
+		.owner		= THIS_MODULE,\
+		.open		= drm_open,\
+		.release	= drm_release,\
+		.unlocked_ioctl	= drm_ioctl,\
+		.compat_ioctl	= drm_compat_ioctl,\
+		.poll		= drm_poll,\
+		.read		= drm_read,\
+		.llseek		= noop_llseek,\
+		.mmap		= drm_gem_shmem_mmap, \
+	}
+
+struct drm_gem_shmem_object *drm_gem_shmem_create(struct drm_device *dev, size_t size);
+void drm_gem_shmem_free_object(struct drm_gem_object *obj);
+
+int drm_gem_shmem_get_pages(struct drm_gem_shmem_object *shmem);
+void drm_gem_shmem_put_pages(struct drm_gem_shmem_object *shmem);
+int drm_gem_shmem_pin(struct drm_gem_object *obj);
+void drm_gem_shmem_unpin(struct drm_gem_object *obj);
+void *drm_gem_shmem_vmap(struct drm_gem_object *obj);
+void drm_gem_shmem_vunmap(struct drm_gem_object *obj, void *vaddr);
+
+struct drm_gem_shmem_object *
+drm_gem_shmem_create_with_handle(struct drm_file *file_priv,
+				 struct drm_device *dev, size_t size,
+				 uint32_t *handle);
+int drm_gem_shmem_dumb_create(struct drm_file *file, struct drm_device *dev,
+			      struct drm_mode_create_dumb *args);
+
+int drm_gem_shmem_mmap(struct file *filp, struct vm_area_struct *vma);
+
+extern const struct vm_operations_struct drm_gem_shmem_vm_ops;
+
+void drm_gem_shmem_print_info(struct drm_printer *p, unsigned int indent,
+			      const struct drm_gem_object *obj);
+
+struct sg_table *drm_gem_shmem_get_sg_table(struct drm_gem_object *obj);
+struct drm_gem_object *
+drm_gem_shmem_prime_import_sg_table(struct drm_device *dev,
+				    struct dma_buf_attachment *attach,
+				    struct sg_table *sgt);
+
+struct sg_table *drm_gem_shmem_get_pages_sgt(struct drm_gem_object *obj);
+
+/**
+ * DRM_GEM_SHMEM_DRIVER_OPS - Default shmem GEM operations
+ *
+ * This macro provides a shortcut for setting the shmem GEM operations in
+ * the &drm_driver structure.
+ */
+#define DRM_GEM_SHMEM_DRIVER_OPS \
+	.prime_handle_to_fd	= drm_gem_prime_handle_to_fd, \
+	.prime_fd_to_handle	= drm_gem_prime_fd_to_handle, \
+	.gem_prime_import_sg_table = drm_gem_shmem_prime_import_sg_table, \
+	.gem_prime_mmap		= drm_gem_prime_mmap, \
+	.dumb_create		= drm_gem_shmem_dumb_create
+
+#endif /* __DRM_GEM_SHMEM_HELPER_H__ */
diff --git a/include/drm/drm_hdcp.h b/include/drm/drm_hdcp.h
index 7260b31af276..f243408ecf26 100644
--- a/include/drm/drm_hdcp.h
+++ b/include/drm/drm_hdcp.h
@@ -13,6 +13,7 @@
 
 /* Period of hdcp checks (to ensure we're still authenticated) */
 #define DRM_HDCP_CHECK_PERIOD_MS		(128 * 16)
+#define DRM_HDCP2_CHECK_PERIOD_MS		500
 
 /* Shared lengths/masks between HDMI/DVI/DisplayPort */
 #define DRM_HDCP_AN_LEN				8
@@ -68,7 +69,6 @@
 #define HDCP_2_2_REP_SEND_ACK			15
 #define HDCP_2_2_REP_STREAM_MANAGE		16
 #define HDCP_2_2_REP_STREAM_READY		17
-#define HDCP_2_2_ERRATA_DP_STREAM_TYPE		50
 
 #define HDCP_2_2_RTX_LEN			8
 #define HDCP_2_2_RRX_LEN			8
@@ -219,11 +219,6 @@ struct hdcp2_rep_stream_ready {
 	u8	m_prime[HDCP_2_2_MPRIME_LEN];
 } __packed;
 
-struct hdcp2_dp_errata_stream_type {
-	u8	msg_id;
-	u8	stream_type;
-} __packed;
-
 /* HDCP2.2 TIMEOUTs in mSec */
 #define HDCP_2_2_CERT_TIMEOUT_MS		100
 #define HDCP_2_2_HPRIME_NO_PAIRED_TIMEOUT_MS	1000
diff --git a/include/drm/drm_legacy.h b/include/drm/drm_legacy.h
index 3e99ab69c122..2182a56ac421 100644
--- a/include/drm/drm_legacy.h
+++ b/include/drm/drm_legacy.h
@@ -162,8 +162,6 @@ int drm_legacy_addmap(struct drm_device *d, resource_size_t offset,
 struct drm_local_map *drm_legacy_findmap(struct drm_device *dev, unsigned int token);
 void drm_legacy_rmmap(struct drm_device *d, struct drm_local_map *map);
 int drm_legacy_rmmap_locked(struct drm_device *d, struct drm_local_map *map);
-void drm_legacy_master_rmmaps(struct drm_device *dev,
-			      struct drm_master *master);
 struct drm_local_map *drm_legacy_getsarea(struct drm_device *dev);
 int drm_legacy_mmap(struct file *filp, struct vm_area_struct *vma);
 
diff --git a/include/drm/drm_modes.h b/include/drm/drm_modes.h
index be4fed97e727..083f16747369 100644
--- a/include/drm/drm_modes.h
+++ b/include/drm/drm_modes.h
@@ -138,6 +138,23 @@ enum drm_mode_status {
 	.vsync_start = (vss), .vsync_end = (vse), .vtotal = (vt), \
 	.vscan = (vs), .flags = (f)
 
+/**
+ * DRM_SIMPLE_MODE - Simple display mode
+ * @hd: Horizontal resolution, width
+ * @vd: Vertical resolution, height
+ * @hd_mm: Display width in millimeters
+ * @vd_mm: Display height in millimeters
+ *
+ * This macro initializes a &drm_display_mode that only contains info about
+ * resolution and physical size.
+ */
+#define DRM_SIMPLE_MODE(hd, vd, hd_mm, vd_mm) \
+	.type = DRM_MODE_TYPE_DRIVER, .clock = 1 /* pass validation */, \
+	.hdisplay = (hd), .hsync_start = (hd), .hsync_end = (hd), \
+	.htotal = (hd), .vdisplay = (vd), .vsync_start = (vd), \
+	.vsync_end = (vd), .vtotal = (vd), .width_mm = (hd_mm), \
+	.height_mm = (vd_mm)
+
 #define CRTC_INTERLACE_HALVE_V	(1 << 0) /* halve V values for interlacing */
 #define CRTC_STEREO_DOUBLE	(1 << 1) /* adjust timings for stereo modes */
 #define CRTC_NO_DBLSCAN		(1 << 2) /* don't adjust doublescan */
diff --git a/include/drm/drm_modeset_helper_vtables.h b/include/drm/drm_modeset_helper_vtables.h
index ce4de6b1e444..f9c94c2a1364 100644
--- a/include/drm/drm_modeset_helper_vtables.h
+++ b/include/drm/drm_modeset_helper_vtables.h
@@ -49,6 +49,8 @@
  */
 
 enum mode_set_atomic;
+struct drm_writeback_connector;
+struct drm_writeback_job;
 
 /**
  * struct drm_crtc_helper_funcs - helper operations for CRTCs
@@ -993,6 +995,11 @@ struct drm_connector_helper_funcs {
 	 */
 	void (*atomic_commit)(struct drm_connector *connector,
 			      struct drm_connector_state *state);
+
+	int (*prepare_writeback_job)(struct drm_writeback_connector *connector,
+				     struct drm_writeback_job *job);
+	void (*cleanup_writeback_job)(struct drm_writeback_connector *connector,
+				      struct drm_writeback_job *job);
 };
 
 /**
diff --git a/include/drm/drm_print.h b/include/drm/drm_print.h
index afbc3beef089..3a4247319e63 100644
--- a/include/drm/drm_print.h
+++ b/include/drm/drm_print.h
@@ -30,6 +30,7 @@
 #include <linux/printk.h>
 #include <linux/seq_file.h>
 #include <linux/device.h>
+#include <linux/debugfs.h>
 
 /**
  * DOC: print
@@ -84,6 +85,7 @@ void __drm_printfn_debug(struct drm_printer *p, struct va_format *vaf);
 __printf(2, 3)
 void drm_printf(struct drm_printer *p, const char *f, ...);
 void drm_puts(struct drm_printer *p, const char *str);
+void drm_print_regset32(struct drm_printer *p, struct debugfs_regset32 *regset);
 
 __printf(2, 0)
 /**
diff --git a/include/drm/drm_syncobj.h b/include/drm/drm_syncobj.h
index 0311c9fdbd2f..6cf7243a1dc5 100644
--- a/include/drm/drm_syncobj.h
+++ b/include/drm/drm_syncobj.h
@@ -27,6 +27,7 @@
 #define __DRM_SYNCOBJ_H__
 
 #include <linux/dma-fence.h>
+#include <linux/dma-fence-chain.h>
 
 struct drm_file;
 
@@ -112,6 +113,10 @@ drm_syncobj_fence_get(struct drm_syncobj *syncobj)
 
 struct drm_syncobj *drm_syncobj_find(struct drm_file *file_private,
 				     u32 handle);
+void drm_syncobj_add_point(struct drm_syncobj *syncobj,
+			   struct dma_fence_chain *chain,
+			   struct dma_fence *fence,
+			   uint64_t point);
 void drm_syncobj_replace_fence(struct drm_syncobj *syncobj,
 			       struct dma_fence *fence);
 int drm_syncobj_find_fence(struct drm_file *file_private,
diff --git a/include/drm/drm_utils.h b/include/drm/drm_utils.h
index a803988d8579..70775748d243 100644
--- a/include/drm/drm_utils.h
+++ b/include/drm/drm_utils.h
@@ -10,6 +10,10 @@
 #ifndef __DRM_UTILS_H__
 #define __DRM_UTILS_H__
 
+#include <linux/types.h>
+
 int drm_get_panel_orientation_quirk(int width, int height);
 
+signed long drm_timeout_abs_to_jiffies(int64_t timeout_nsec);
+
 #endif
diff --git a/include/drm/drm_vma_manager.h b/include/drm/drm_vma_manager.h
index c7987daeaed0..76ac5e97a559 100644
--- a/include/drm/drm_vma_manager.h
+++ b/include/drm/drm_vma_manager.h
@@ -29,6 +29,18 @@
 #include <linux/spinlock.h>
 #include <linux/types.h>
 
+/* We make up offsets for buffer objects so we can recognize them at
+ * mmap time. pgoff in mmap is an unsigned long, so we need to make sure
+ * that the faked up offset will fit
+ */
+#if BITS_PER_LONG == 64
+#define DRM_FILE_PAGE_OFFSET_START ((0xFFFFFFFFUL >> PAGE_SHIFT) + 1)
+#define DRM_FILE_PAGE_OFFSET_SIZE ((0xFFFFFFFFUL >> PAGE_SHIFT) * 256)
+#else
+#define DRM_FILE_PAGE_OFFSET_START ((0xFFFFFFFUL >> PAGE_SHIFT) + 1)
+#define DRM_FILE_PAGE_OFFSET_SIZE ((0xFFFFFFFUL >> PAGE_SHIFT) * 16)
+#endif
+
 struct drm_file;
 
 struct drm_vma_offset_file {
diff --git a/include/drm/drm_writeback.h b/include/drm/drm_writeback.h
index 23df9d463003..777c14c847f0 100644
--- a/include/drm/drm_writeback.h
+++ b/include/drm/drm_writeback.h
@@ -80,6 +80,20 @@ struct drm_writeback_connector {
 
 struct drm_writeback_job {
 	/**
+	 * @connector:
+	 *
+	 * Back-pointer to the writeback connector associated with the job
+	 */
+	struct drm_writeback_connector *connector;
+
+	/**
+	 * @prepared:
+	 *
+	 * Set when the job has been prepared with drm_writeback_prepare_job()
+	 */
+	bool prepared;
+
+	/**
 	 * @cleanup_work:
 	 *
 	 * Used to allow drm_writeback_signal_completion to defer dropping the
@@ -98,7 +112,7 @@ struct drm_writeback_job {
 	 * @fb:
 	 *
 	 * Framebuffer to be written to by the writeback connector. Do not set
-	 * directly, use drm_atomic_set_writeback_fb_for_connector()
+	 * directly, use drm_writeback_set_fb()
 	 */
 	struct drm_framebuffer *fb;
 
@@ -108,6 +122,13 @@ struct drm_writeback_job {
 	 * Fence which will signal once the writeback has completed
 	 */
 	struct dma_fence *out_fence;
+
+	/**
+	 * @priv:
+	 *
+	 * Driver-private data
+	 */
+	void *priv;
 };
 
 static inline struct drm_writeback_connector *
@@ -122,8 +143,13 @@ int drm_writeback_connector_init(struct drm_device *dev,
 				 const struct drm_encoder_helper_funcs *enc_helper_funcs,
 				 const u32 *formats, int n_formats);
 
+int drm_writeback_set_fb(struct drm_connector_state *conn_state,
+			 struct drm_framebuffer *fb);
+
+int drm_writeback_prepare_job(struct drm_writeback_job *job);
+
 void drm_writeback_queue_job(struct drm_writeback_connector *wb_connector,
-			     struct drm_writeback_job *job);
+			     struct drm_connector_state *conn_state);
 
 void drm_writeback_cleanup_job(struct drm_writeback_job *job);
 
diff --git a/include/drm/i915_pciids.h b/include/drm/i915_pciids.h
index d2fad7b0fcf6..6477da22af28 100644
--- a/include/drm/i915_pciids.h
+++ b/include/drm/i915_pciids.h
@@ -108,8 +108,10 @@
 	INTEL_VGA_DEVICE(0x2e42, info), /* B43_G */ \
 	INTEL_VGA_DEVICE(0x2e92, info)	/* B43_G.1 */
 
-#define INTEL_PINEVIEW_IDS(info)			\
-	INTEL_VGA_DEVICE(0xa001, info),			\
+#define INTEL_PINEVIEW_G_IDS(info) \
+	INTEL_VGA_DEVICE(0xa001, info)
+
+#define INTEL_PINEVIEW_M_IDS(info) \
 	INTEL_VGA_DEVICE(0xa011, info)
 
 #define INTEL_IRONLAKE_D_IDS(info) \
@@ -166,7 +168,18 @@
 #define INTEL_IVB_Q_IDS(info) \
 	INTEL_QUANTA_VGA_DEVICE(info) /* Quanta transcode */
 
+#define INTEL_HSW_ULT_GT1_IDS(info) \
+	INTEL_VGA_DEVICE(0x0A02, info), /* ULT GT1 desktop */ \
+	INTEL_VGA_DEVICE(0x0A0A, info), /* ULT GT1 server */ \
+	INTEL_VGA_DEVICE(0x0A0B, info), /* ULT GT1 reserved */ \
+	INTEL_VGA_DEVICE(0x0A06, info)  /* ULT GT1 mobile */
+
+#define INTEL_HSW_ULX_GT1_IDS(info) \
+	INTEL_VGA_DEVICE(0x0A0E, info) /* ULX GT1 mobile */
+
 #define INTEL_HSW_GT1_IDS(info) \
+	INTEL_HSW_ULT_GT1_IDS(info), \
+	INTEL_HSW_ULX_GT1_IDS(info), \
 	INTEL_VGA_DEVICE(0x0402, info), /* GT1 desktop */ \
 	INTEL_VGA_DEVICE(0x040a, info), /* GT1 server */ \
 	INTEL_VGA_DEVICE(0x040B, info), /* GT1 reserved */ \
@@ -175,20 +188,26 @@
 	INTEL_VGA_DEVICE(0x0C0A, info), /* SDV GT1 server */ \
 	INTEL_VGA_DEVICE(0x0C0B, info), /* SDV GT1 reserved */ \
 	INTEL_VGA_DEVICE(0x0C0E, info), /* SDV GT1 reserved */ \
-	INTEL_VGA_DEVICE(0x0A02, info), /* ULT GT1 desktop */ \
-	INTEL_VGA_DEVICE(0x0A0A, info), /* ULT GT1 server */ \
-	INTEL_VGA_DEVICE(0x0A0B, info), /* ULT GT1 reserved */ \
 	INTEL_VGA_DEVICE(0x0D02, info), /* CRW GT1 desktop */ \
 	INTEL_VGA_DEVICE(0x0D0A, info), /* CRW GT1 server */ \
 	INTEL_VGA_DEVICE(0x0D0B, info), /* CRW GT1 reserved */ \
 	INTEL_VGA_DEVICE(0x0D0E, info), /* CRW GT1 reserved */ \
 	INTEL_VGA_DEVICE(0x0406, info), /* GT1 mobile */ \
 	INTEL_VGA_DEVICE(0x0C06, info), /* SDV GT1 mobile */ \
-	INTEL_VGA_DEVICE(0x0A06, info), /* ULT GT1 mobile */ \
-	INTEL_VGA_DEVICE(0x0A0E, info), /* ULX GT1 mobile */ \
 	INTEL_VGA_DEVICE(0x0D06, info)  /* CRW GT1 mobile */
 
+#define INTEL_HSW_ULT_GT2_IDS(info) \
+	INTEL_VGA_DEVICE(0x0A12, info), /* ULT GT2 desktop */ \
+	INTEL_VGA_DEVICE(0x0A1A, info), /* ULT GT2 server */ \
+	INTEL_VGA_DEVICE(0x0A1B, info), /* ULT GT2 reserved */ \
+	INTEL_VGA_DEVICE(0x0A16, info)  /* ULT GT2 mobile */
+
+#define INTEL_HSW_ULX_GT2_IDS(info) \
+	INTEL_VGA_DEVICE(0x0A1E, info) /* ULX GT2 mobile */ \
+
 #define INTEL_HSW_GT2_IDS(info) \
+	INTEL_HSW_ULT_GT2_IDS(info), \
+	INTEL_HSW_ULX_GT2_IDS(info), \
 	INTEL_VGA_DEVICE(0x0412, info), /* GT2 desktop */ \
 	INTEL_VGA_DEVICE(0x041a, info), /* GT2 server */ \
 	INTEL_VGA_DEVICE(0x041B, info), /* GT2 reserved */ \
@@ -197,9 +216,6 @@
 	INTEL_VGA_DEVICE(0x0C1A, info), /* SDV GT2 server */ \
 	INTEL_VGA_DEVICE(0x0C1B, info), /* SDV GT2 reserved */ \
 	INTEL_VGA_DEVICE(0x0C1E, info), /* SDV GT2 reserved */ \
-	INTEL_VGA_DEVICE(0x0A12, info), /* ULT GT2 desktop */ \
-	INTEL_VGA_DEVICE(0x0A1A, info), /* ULT GT2 server */ \
-	INTEL_VGA_DEVICE(0x0A1B, info), /* ULT GT2 reserved */ \
 	INTEL_VGA_DEVICE(0x0D12, info), /* CRW GT2 desktop */ \
 	INTEL_VGA_DEVICE(0x0D1A, info), /* CRW GT2 server */ \
 	INTEL_VGA_DEVICE(0x0D1B, info), /* CRW GT2 reserved */ \
@@ -207,11 +223,17 @@
 	INTEL_VGA_DEVICE(0x0416, info), /* GT2 mobile */ \
 	INTEL_VGA_DEVICE(0x0426, info), /* GT2 mobile */ \
 	INTEL_VGA_DEVICE(0x0C16, info), /* SDV GT2 mobile */ \
-	INTEL_VGA_DEVICE(0x0A16, info), /* ULT GT2 mobile */ \
-	INTEL_VGA_DEVICE(0x0A1E, info), /* ULX GT2 mobile */ \
 	INTEL_VGA_DEVICE(0x0D16, info)  /* CRW GT2 mobile */
 
+#define INTEL_HSW_ULT_GT3_IDS(info) \
+	INTEL_VGA_DEVICE(0x0A22, info), /* ULT GT3 desktop */ \
+	INTEL_VGA_DEVICE(0x0A2A, info), /* ULT GT3 server */ \
+	INTEL_VGA_DEVICE(0x0A2B, info), /* ULT GT3 reserved */ \
+	INTEL_VGA_DEVICE(0x0A26, info), /* ULT GT3 mobile */ \
+	INTEL_VGA_DEVICE(0x0A2E, info)  /* ULT GT3 reserved */
+
 #define INTEL_HSW_GT3_IDS(info) \
+	INTEL_HSW_ULT_GT3_IDS(info), \
 	INTEL_VGA_DEVICE(0x0422, info), /* GT3 desktop */ \
 	INTEL_VGA_DEVICE(0x042a, info), /* GT3 server */ \
 	INTEL_VGA_DEVICE(0x042B, info), /* GT3 reserved */ \
@@ -220,16 +242,11 @@
 	INTEL_VGA_DEVICE(0x0C2A, info), /* SDV GT3 server */ \
 	INTEL_VGA_DEVICE(0x0C2B, info), /* SDV GT3 reserved */ \
 	INTEL_VGA_DEVICE(0x0C2E, info), /* SDV GT3 reserved */ \
-	INTEL_VGA_DEVICE(0x0A22, info), /* ULT GT3 desktop */ \
-	INTEL_VGA_DEVICE(0x0A2A, info), /* ULT GT3 server */ \
-	INTEL_VGA_DEVICE(0x0A2B, info), /* ULT GT3 reserved */ \
 	INTEL_VGA_DEVICE(0x0D22, info), /* CRW GT3 desktop */ \
 	INTEL_VGA_DEVICE(0x0D2A, info), /* CRW GT3 server */ \
 	INTEL_VGA_DEVICE(0x0D2B, info), /* CRW GT3 reserved */ \
 	INTEL_VGA_DEVICE(0x0D2E, info), /* CRW GT3 reserved */ \
 	INTEL_VGA_DEVICE(0x0C26, info), /* SDV GT3 mobile */ \
-	INTEL_VGA_DEVICE(0x0A26, info), /* ULT GT3 mobile */ \
-	INTEL_VGA_DEVICE(0x0A2E, info), /* ULT GT3 reserved */ \
 	INTEL_VGA_DEVICE(0x0D26, info)  /* CRW GT3 mobile */
 
 #define INTEL_HSW_IDS(info) \
@@ -245,35 +262,59 @@
 	INTEL_VGA_DEVICE(0x0157, info), \
 	INTEL_VGA_DEVICE(0x0155, info)
 
-#define INTEL_BDW_GT1_IDS(info)  \
-	INTEL_VGA_DEVICE(0x1602, info), /* GT1 ULT */ \
+#define INTEL_BDW_ULT_GT1_IDS(info) \
 	INTEL_VGA_DEVICE(0x1606, info), /* GT1 ULT */ \
-	INTEL_VGA_DEVICE(0x160B, info), /* GT1 Iris */ \
-	INTEL_VGA_DEVICE(0x160E, info), /* GT1 ULX */ \
+	INTEL_VGA_DEVICE(0x160B, info)  /* GT1 Iris */
+
+#define INTEL_BDW_ULX_GT1_IDS(info) \
+	INTEL_VGA_DEVICE(0x160E, info) /* GT1 ULX */
+
+#define INTEL_BDW_GT1_IDS(info) \
+	INTEL_BDW_ULT_GT1_IDS(info), \
+	INTEL_BDW_ULX_GT1_IDS(info), \
+	INTEL_VGA_DEVICE(0x1602, info), /* GT1 ULT */ \
 	INTEL_VGA_DEVICE(0x160A, info), /* GT1 Server */ \
 	INTEL_VGA_DEVICE(0x160D, info)  /* GT1 Workstation */
 
-#define INTEL_BDW_GT2_IDS(info)  \
-	INTEL_VGA_DEVICE(0x1612, info), /* GT2 Halo */	\
+#define INTEL_BDW_ULT_GT2_IDS(info) \
 	INTEL_VGA_DEVICE(0x1616, info), /* GT2 ULT */ \
-	INTEL_VGA_DEVICE(0x161B, info), /* GT2 ULT */ \
-	INTEL_VGA_DEVICE(0x161E, info), /* GT2 ULX */ \
+	INTEL_VGA_DEVICE(0x161B, info)  /* GT2 ULT */
+
+#define INTEL_BDW_ULX_GT2_IDS(info) \
+	INTEL_VGA_DEVICE(0x161E, info) /* GT2 ULX */
+
+#define INTEL_BDW_GT2_IDS(info) \
+	INTEL_BDW_ULT_GT2_IDS(info), \
+	INTEL_BDW_ULX_GT2_IDS(info), \
+	INTEL_VGA_DEVICE(0x1612, info), /* GT2 Halo */	\
 	INTEL_VGA_DEVICE(0x161A, info), /* GT2 Server */ \
 	INTEL_VGA_DEVICE(0x161D, info)  /* GT2 Workstation */
 
+#define INTEL_BDW_ULT_GT3_IDS(info) \
+	INTEL_VGA_DEVICE(0x1626, info), /* ULT */ \
+	INTEL_VGA_DEVICE(0x162B, info)  /* Iris */ \
+
+#define INTEL_BDW_ULX_GT3_IDS(info) \
+	INTEL_VGA_DEVICE(0x162E, info)  /* ULX */
+
 #define INTEL_BDW_GT3_IDS(info) \
+	INTEL_BDW_ULT_GT3_IDS(info), \
+	INTEL_BDW_ULX_GT3_IDS(info), \
 	INTEL_VGA_DEVICE(0x1622, info), /* ULT */ \
-	INTEL_VGA_DEVICE(0x1626, info), /* ULT */ \
-	INTEL_VGA_DEVICE(0x162B, info), /* Iris */ \
-	INTEL_VGA_DEVICE(0x162E, info),  /* ULX */\
 	INTEL_VGA_DEVICE(0x162A, info), /* Server */ \
 	INTEL_VGA_DEVICE(0x162D, info)  /* Workstation */
 
+#define INTEL_BDW_ULT_RSVD_IDS(info) \
+	INTEL_VGA_DEVICE(0x1636, info), /* ULT */ \
+	INTEL_VGA_DEVICE(0x163B, info)  /* Iris */
+
+#define INTEL_BDW_ULX_RSVD_IDS(info) \
+	INTEL_VGA_DEVICE(0x163E, info) /* ULX */
+
 #define INTEL_BDW_RSVD_IDS(info) \
+	INTEL_BDW_ULT_RSVD_IDS(info), \
+	INTEL_BDW_ULX_RSVD_IDS(info), \
 	INTEL_VGA_DEVICE(0x1632, info), /* ULT */ \
-	INTEL_VGA_DEVICE(0x1636, info), /* ULT */ \
-	INTEL_VGA_DEVICE(0x163B, info), /* Iris */ \
-	INTEL_VGA_DEVICE(0x163E, info), /* ULX */ \
 	INTEL_VGA_DEVICE(0x163A, info), /* Server */ \
 	INTEL_VGA_DEVICE(0x163D, info)  /* Workstation */
 
@@ -289,25 +330,40 @@
 	INTEL_VGA_DEVICE(0x22b2, info), \
 	INTEL_VGA_DEVICE(0x22b3, info)
 
+#define INTEL_SKL_ULT_GT1_IDS(info) \
+	INTEL_VGA_DEVICE(0x1906, info) /* ULT GT1 */
+
+#define INTEL_SKL_ULX_GT1_IDS(info) \
+	INTEL_VGA_DEVICE(0x190E, info) /* ULX GT1 */
+
 #define INTEL_SKL_GT1_IDS(info)	\
-	INTEL_VGA_DEVICE(0x1906, info), /* ULT GT1 */ \
-	INTEL_VGA_DEVICE(0x190E, info), /* ULX GT1 */ \
+	INTEL_SKL_ULT_GT1_IDS(info), \
+	INTEL_SKL_ULX_GT1_IDS(info), \
 	INTEL_VGA_DEVICE(0x1902, info), /* DT  GT1 */ \
 	INTEL_VGA_DEVICE(0x190B, info), /* Halo GT1 */ \
 	INTEL_VGA_DEVICE(0x190A, info) /* SRV GT1 */
 
-#define INTEL_SKL_GT2_IDS(info)	\
+#define INTEL_SKL_ULT_GT2_IDS(info) \
 	INTEL_VGA_DEVICE(0x1916, info), /* ULT GT2 */ \
-	INTEL_VGA_DEVICE(0x1921, info), /* ULT GT2F */ \
-	INTEL_VGA_DEVICE(0x191E, info), /* ULX GT2 */ \
+	INTEL_VGA_DEVICE(0x1921, info)  /* ULT GT2F */
+
+#define INTEL_SKL_ULX_GT2_IDS(info) \
+	INTEL_VGA_DEVICE(0x191E, info) /* ULX GT2 */
+
+#define INTEL_SKL_GT2_IDS(info)	\
+	INTEL_SKL_ULT_GT2_IDS(info), \
+	INTEL_SKL_ULX_GT2_IDS(info), \
 	INTEL_VGA_DEVICE(0x1912, info), /* DT  GT2 */ \
 	INTEL_VGA_DEVICE(0x191B, info), /* Halo GT2 */ \
 	INTEL_VGA_DEVICE(0x191A, info), /* SRV GT2 */ \
 	INTEL_VGA_DEVICE(0x191D, info)  /* WKS GT2 */
 
+#define INTEL_SKL_ULT_GT3_IDS(info) \
+	INTEL_VGA_DEVICE(0x1926, info) /* ULT GT3 */
+
 #define INTEL_SKL_GT3_IDS(info) \
+	INTEL_SKL_ULT_GT3_IDS(info), \
 	INTEL_VGA_DEVICE(0x1923, info), /* ULT GT3 */ \
-	INTEL_VGA_DEVICE(0x1926, info), /* ULT GT3 */ \
 	INTEL_VGA_DEVICE(0x1927, info), /* ULT GT3 */ \
 	INTEL_VGA_DEVICE(0x192B, info), /* Halo GT3 */ \
 	INTEL_VGA_DEVICE(0x192D, info)  /* SRV GT3 */
@@ -336,29 +392,44 @@
 	INTEL_VGA_DEVICE(0x3184, info), \
 	INTEL_VGA_DEVICE(0x3185, info)
 
-#define INTEL_KBL_GT1_IDS(info)	\
-	INTEL_VGA_DEVICE(0x5913, info), /* ULT GT1.5 */ \
-	INTEL_VGA_DEVICE(0x5915, info), /* ULX GT1.5 */ \
+#define INTEL_KBL_ULT_GT1_IDS(info) \
 	INTEL_VGA_DEVICE(0x5906, info), /* ULT GT1 */ \
+	INTEL_VGA_DEVICE(0x5913, info)  /* ULT GT1.5 */
+
+#define INTEL_KBL_ULX_GT1_IDS(info) \
 	INTEL_VGA_DEVICE(0x590E, info), /* ULX GT1 */ \
+	INTEL_VGA_DEVICE(0x5915, info)  /* ULX GT1.5 */
+
+#define INTEL_KBL_GT1_IDS(info)	\
+	INTEL_KBL_ULT_GT1_IDS(info), \
+	INTEL_KBL_ULX_GT1_IDS(info), \
 	INTEL_VGA_DEVICE(0x5902, info), /* DT  GT1 */ \
 	INTEL_VGA_DEVICE(0x5908, info), /* Halo GT1 */ \
 	INTEL_VGA_DEVICE(0x590B, info), /* Halo GT1 */ \
 	INTEL_VGA_DEVICE(0x590A, info) /* SRV GT1 */
 
-#define INTEL_KBL_GT2_IDS(info)	\
+#define INTEL_KBL_ULT_GT2_IDS(info) \
 	INTEL_VGA_DEVICE(0x5916, info), /* ULT GT2 */ \
+	INTEL_VGA_DEVICE(0x5921, info)  /* ULT GT2F */
+
+#define INTEL_KBL_ULX_GT2_IDS(info) \
+	INTEL_VGA_DEVICE(0x591E, info)  /* ULX GT2 */
+
+#define INTEL_KBL_GT2_IDS(info)	\
+	INTEL_KBL_ULT_GT2_IDS(info), \
+	INTEL_KBL_ULX_GT2_IDS(info), \
 	INTEL_VGA_DEVICE(0x5917, info), /* Mobile GT2 */ \
-	INTEL_VGA_DEVICE(0x5921, info), /* ULT GT2F */ \
-	INTEL_VGA_DEVICE(0x591E, info), /* ULX GT2 */ \
 	INTEL_VGA_DEVICE(0x5912, info), /* DT  GT2 */ \
 	INTEL_VGA_DEVICE(0x591B, info), /* Halo GT2 */ \
 	INTEL_VGA_DEVICE(0x591A, info), /* SRV GT2 */ \
 	INTEL_VGA_DEVICE(0x591D, info) /* WKS GT2 */
 
+#define INTEL_KBL_ULT_GT3_IDS(info) \
+	INTEL_VGA_DEVICE(0x5926, info) /* ULT GT3 */
+
 #define INTEL_KBL_GT3_IDS(info) \
+	INTEL_KBL_ULT_GT3_IDS(info), \
 	INTEL_VGA_DEVICE(0x5923, info), /* ULT GT3 */ \
-	INTEL_VGA_DEVICE(0x5926, info), /* ULT GT3 */ \
 	INTEL_VGA_DEVICE(0x5927, info) /* ULT GT3 */
 
 #define INTEL_KBL_GT4_IDS(info) \
@@ -373,6 +444,30 @@
 #define INTEL_AML_CFL_GT2_IDS(info) \
 	INTEL_VGA_DEVICE(0x87CA, info)
 
+/* CML GT1 */
+#define INTEL_CML_GT1_IDS(info)	\
+	INTEL_VGA_DEVICE(0x9B21, info), \
+	INTEL_VGA_DEVICE(0x9BAA, info), \
+	INTEL_VGA_DEVICE(0x9BAB, info), \
+	INTEL_VGA_DEVICE(0x9BAC, info), \
+	INTEL_VGA_DEVICE(0x9BA0, info), \
+	INTEL_VGA_DEVICE(0x9BA5, info), \
+	INTEL_VGA_DEVICE(0x9BA8, info), \
+	INTEL_VGA_DEVICE(0x9BA4, info), \
+	INTEL_VGA_DEVICE(0x9BA2, info)
+
+/* CML GT2 */
+#define INTEL_CML_GT2_IDS(info)	\
+	INTEL_VGA_DEVICE(0x9B41, info), \
+	INTEL_VGA_DEVICE(0x9BCA, info), \
+	INTEL_VGA_DEVICE(0x9BCB, info), \
+	INTEL_VGA_DEVICE(0x9BCC, info), \
+	INTEL_VGA_DEVICE(0x9BC0, info), \
+	INTEL_VGA_DEVICE(0x9BC5, info), \
+	INTEL_VGA_DEVICE(0x9BC8, info), \
+	INTEL_VGA_DEVICE(0x9BC4, info), \
+	INTEL_VGA_DEVICE(0x9BC2, info)
+
 #define INTEL_KBL_IDS(info) \
 	INTEL_KBL_GT1_IDS(info), \
 	INTEL_KBL_GT2_IDS(info), \
@@ -436,10 +531,19 @@
 	INTEL_WHL_U_GT1_IDS(info), \
 	INTEL_WHL_U_GT2_IDS(info), \
 	INTEL_WHL_U_GT3_IDS(info), \
-	INTEL_AML_CFL_GT2_IDS(info)
+	INTEL_AML_CFL_GT2_IDS(info), \
+	INTEL_CML_GT1_IDS(info), \
+	INTEL_CML_GT2_IDS(info)
 
 /* CNL */
+#define INTEL_CNL_PORT_F_IDS(info) \
+	INTEL_VGA_DEVICE(0x5A54, info), \
+	INTEL_VGA_DEVICE(0x5A5C, info), \
+	INTEL_VGA_DEVICE(0x5A44, info), \
+	INTEL_VGA_DEVICE(0x5A4C, info)
+
 #define INTEL_CNL_IDS(info) \
+	INTEL_CNL_PORT_F_IDS(info), \
 	INTEL_VGA_DEVICE(0x5A51, info), \
 	INTEL_VGA_DEVICE(0x5A59, info), \
 	INTEL_VGA_DEVICE(0x5A41, info), \
@@ -449,16 +553,11 @@
 	INTEL_VGA_DEVICE(0x5A42, info), \
 	INTEL_VGA_DEVICE(0x5A4A, info), \
 	INTEL_VGA_DEVICE(0x5A50, info), \
-	INTEL_VGA_DEVICE(0x5A40, info), \
-	INTEL_VGA_DEVICE(0x5A54, info), \
-	INTEL_VGA_DEVICE(0x5A5C, info), \
-	INTEL_VGA_DEVICE(0x5A44, info), \
-	INTEL_VGA_DEVICE(0x5A4C, info)
+	INTEL_VGA_DEVICE(0x5A40, info)
 
 /* ICL */
-#define INTEL_ICL_11_IDS(info) \
+#define INTEL_ICL_PORT_F_IDS(info) \
 	INTEL_VGA_DEVICE(0x8A50, info), \
-	INTEL_VGA_DEVICE(0x8A51, info), \
 	INTEL_VGA_DEVICE(0x8A5C, info), \
 	INTEL_VGA_DEVICE(0x8A5D, info), \
 	INTEL_VGA_DEVICE(0x8A59, info),	\
@@ -469,6 +568,18 @@
 	INTEL_VGA_DEVICE(0x8A57, info), \
 	INTEL_VGA_DEVICE(0x8A56, info), \
 	INTEL_VGA_DEVICE(0x8A71, info), \
-	INTEL_VGA_DEVICE(0x8A70, info)
+	INTEL_VGA_DEVICE(0x8A70, info), \
+	INTEL_VGA_DEVICE(0x8A53, info)
+
+#define INTEL_ICL_11_IDS(info) \
+	INTEL_ICL_PORT_F_IDS(info), \
+	INTEL_VGA_DEVICE(0x8A51, info)
+
+/* EHL */
+#define INTEL_EHL_IDS(info) \
+	INTEL_VGA_DEVICE(0x4500, info),	\
+	INTEL_VGA_DEVICE(0x4571, info), \
+	INTEL_VGA_DEVICE(0x4551, info), \
+	INTEL_VGA_DEVICE(0x4541, info)
 
 #endif /* _I915_PCIIDS_H */
diff --git a/include/drm/tinydrm/mipi-dbi.h b/include/drm/tinydrm/mipi-dbi.h
index f4ec2834bc22..af203b37d87a 100644
--- a/include/drm/tinydrm/mipi-dbi.h
+++ b/include/drm/tinydrm/mipi-dbi.h
@@ -12,7 +12,9 @@
 #ifndef __LINUX_MIPI_DBI_H
 #define __LINUX_MIPI_DBI_H
 
-#include <drm/tinydrm/tinydrm.h>
+#include <linux/mutex.h>
+#include <drm/drm_device.h>
+#include <drm/drm_simple_kms_helper.h>
 
 struct drm_rect;
 struct spi_device;
@@ -21,7 +23,6 @@ struct regulator;
 
 /**
  * struct mipi_dbi - MIPI DBI controller
- * @tinydrm: tinydrm base
  * @spi: SPI device
  * @enabled: Pipeline is enabled
  * @cmdlock: Command lock
@@ -39,11 +40,20 @@ struct regulator;
  * @regulator: power regulator (optional)
  */
 struct mipi_dbi {
-	struct tinydrm_device tinydrm;
+	/**
+	 * @drm: DRM device
+	 */
+	struct drm_device drm;
+
+	/**
+	 * @pipe: Display pipe structure
+	 */
+	struct drm_simple_display_pipe pipe;
+
 	struct spi_device *spi;
 	bool enabled;
 	struct mutex cmdlock;
-	int (*command)(struct mipi_dbi *mipi, u8 cmd, u8 *param, size_t num);
+	int (*command)(struct mipi_dbi *mipi, u8 *cmd, u8 *param, size_t num);
 	const u8 *read_commands;
 	struct gpio_desc *dc;
 	u16 *tx_buf;
@@ -56,18 +66,17 @@ struct mipi_dbi {
 	struct regulator *regulator;
 };
 
-static inline struct mipi_dbi *
-mipi_dbi_from_tinydrm(struct tinydrm_device *tdev)
+static inline struct mipi_dbi *drm_to_mipi_dbi(struct drm_device *drm)
 {
-	return container_of(tdev, struct mipi_dbi, tinydrm);
+	return container_of(drm, struct mipi_dbi, drm);
 }
 
 int mipi_dbi_spi_init(struct spi_device *spi, struct mipi_dbi *mipi,
 		      struct gpio_desc *dc);
-int mipi_dbi_init(struct device *dev, struct mipi_dbi *mipi,
-		  const struct drm_simple_display_pipe_funcs *pipe_funcs,
-		  struct drm_driver *driver,
+int mipi_dbi_init(struct mipi_dbi *mipi,
+		  const struct drm_simple_display_pipe_funcs *funcs,
 		  const struct drm_display_mode *mode, unsigned int rotation);
+void mipi_dbi_release(struct drm_device *drm);
 void mipi_dbi_pipe_update(struct drm_simple_display_pipe *pipe,
 			  struct drm_plane_state *old_state);
 void mipi_dbi_enable_flush(struct mipi_dbi *mipi,
@@ -82,6 +91,7 @@ u32 mipi_dbi_spi_cmd_max_speed(struct spi_device *spi, size_t len);
 
 int mipi_dbi_command_read(struct mipi_dbi *mipi, u8 cmd, u8 *val);
 int mipi_dbi_command_buf(struct mipi_dbi *mipi, u8 cmd, u8 *data, size_t len);
+int mipi_dbi_command_stackbuf(struct mipi_dbi *mipi, u8 cmd, u8 *data, size_t len);
 int mipi_dbi_buf_copy(void *dst, struct drm_framebuffer *fb,
 		      struct drm_rect *clip, bool swap);
 /**
@@ -99,7 +109,7 @@ int mipi_dbi_buf_copy(void *dst, struct drm_framebuffer *fb,
 #define mipi_dbi_command(mipi, cmd, seq...) \
 ({ \
 	u8 d[] = { seq }; \
-	mipi_dbi_command_buf(mipi, cmd, d, ARRAY_SIZE(d)); \
+	mipi_dbi_command_stackbuf(mipi, cmd, d, ARRAY_SIZE(d)); \
 })
 
 #ifdef CONFIG_DEBUG_FS
diff --git a/include/drm/tinydrm/tinydrm-helpers.h b/include/drm/tinydrm/tinydrm-helpers.h
index f0d598789e4d..7d259acb8826 100644
--- a/include/drm/tinydrm/tinydrm-helpers.h
+++ b/include/drm/tinydrm/tinydrm-helpers.h
@@ -11,8 +11,12 @@
 #define __LINUX_TINYDRM_HELPERS_H
 
 struct backlight_device;
+struct drm_device;
+struct drm_display_mode;
 struct drm_framebuffer;
 struct drm_rect;
+struct drm_simple_display_pipe;
+struct drm_simple_display_pipe_funcs;
 struct spi_transfer;
 struct spi_message;
 struct spi_device;
@@ -33,15 +37,14 @@ static inline bool tinydrm_machine_little_endian(void)
 #endif
 }
 
-void tinydrm_memcpy(void *dst, void *vaddr, struct drm_framebuffer *fb,
-		    struct drm_rect *clip);
-void tinydrm_swab16(u16 *dst, void *vaddr, struct drm_framebuffer *fb,
-		    struct drm_rect *clip);
-void tinydrm_xrgb8888_to_rgb565(u16 *dst, void *vaddr,
-				struct drm_framebuffer *fb,
-				struct drm_rect *clip, bool swap);
-void tinydrm_xrgb8888_to_gray8(u8 *dst, void *vaddr, struct drm_framebuffer *fb,
-			       struct drm_rect *clip);
+int tinydrm_display_pipe_init(struct drm_device *drm,
+			      struct drm_simple_display_pipe *pipe,
+			      const struct drm_simple_display_pipe_funcs *funcs,
+			      int connector_type,
+			      const uint32_t *formats,
+			      unsigned int format_count,
+			      const struct drm_display_mode *mode,
+			      unsigned int rotation);
 
 size_t tinydrm_spi_max_transfer_size(struct spi_device *spi, size_t max_len);
 bool tinydrm_spi_bpw_supported(struct spi_device *spi, u8 bpw);
diff --git a/include/drm/tinydrm/tinydrm.h b/include/drm/tinydrm/tinydrm.h
deleted file mode 100644
index 5621688edcc0..000000000000
--- a/include/drm/tinydrm/tinydrm.h
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Copyright (C) 2016 Noralf Trønnes
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
-
-#ifndef __LINUX_TINYDRM_H
-#define __LINUX_TINYDRM_H
-
-#include <drm/drm_simple_kms_helper.h>
-
-struct drm_driver;
-
-/**
- * struct tinydrm_device - tinydrm device
- */
-struct tinydrm_device {
-	/**
-	 * @drm: DRM device
-	 */
-	struct drm_device *drm;
-
-	/**
-	 * @pipe: Display pipe structure
-	 */
-	struct drm_simple_display_pipe pipe;
-};
-
-static inline struct tinydrm_device *
-pipe_to_tinydrm(struct drm_simple_display_pipe *pipe)
-{
-	return container_of(pipe, struct tinydrm_device, pipe);
-}
-
-/**
- * TINYDRM_MODE - tinydrm display mode
- * @hd: Horizontal resolution, width
- * @vd: Vertical resolution, height
- * @hd_mm: Display width in millimeters
- * @vd_mm: Display height in millimeters
- *
- * This macro creates a &drm_display_mode for use with tinydrm.
- */
-#define TINYDRM_MODE(hd, vd, hd_mm, vd_mm) \
-	.hdisplay = (hd), \
-	.hsync_start = (hd), \
-	.hsync_end = (hd), \
-	.htotal = (hd), \
-	.vdisplay = (vd), \
-	.vsync_start = (vd), \
-	.vsync_end = (vd), \
-	.vtotal = (vd), \
-	.width_mm = (hd_mm), \
-	.height_mm = (vd_mm), \
-	.type = DRM_MODE_TYPE_DRIVER, \
-	.clock = 1 /* pass validation */
-
-int devm_tinydrm_init(struct device *parent, struct tinydrm_device *tdev,
-		      struct drm_driver *driver);
-int devm_tinydrm_register(struct tinydrm_device *tdev);
-void tinydrm_shutdown(struct tinydrm_device *tdev);
-
-int
-tinydrm_display_pipe_init(struct tinydrm_device *tdev,
-			  const struct drm_simple_display_pipe_funcs *funcs,
-			  int connector_type,
-			  const uint32_t *formats,
-			  unsigned int format_count,
-			  const struct drm_display_mode *mode,
-			  unsigned int rotation);
-
-#endif /* __LINUX_TINYDRM_H */
diff --git a/include/drm/ttm/ttm_bo_driver.h b/include/drm/ttm/ttm_bo_driver.h
index 668ad971cd7b..129dabbc002d 100644
--- a/include/drm/ttm/ttm_bo_driver.h
+++ b/include/drm/ttm/ttm_bo_driver.h
@@ -596,7 +596,7 @@ int ttm_bo_device_release(struct ttm_bo_device *bdev);
 int ttm_bo_device_init(struct ttm_bo_device *bdev,
 		       struct ttm_bo_driver *driver,
 		       struct address_space *mapping,
-		       uint64_t file_page_offset, bool need_dma32);
+		       bool need_dma32);
 
 /**
  * ttm_bo_unmap_virtual
diff --git a/include/dt-bindings/clock/axg-audio-clkc.h b/include/dt-bindings/clock/axg-audio-clkc.h
index fd9c362099d9..75901c636893 100644
--- a/include/dt-bindings/clock/axg-audio-clkc.h
+++ b/include/dt-bindings/clock/axg-audio-clkc.h
@@ -7,26 +7,6 @@
 #ifndef __AXG_AUDIO_CLKC_BINDINGS_H
 #define __AXG_AUDIO_CLKC_BINDINGS_H
 
-#define AUD_CLKID_SLV_SCLK0		9
-#define AUD_CLKID_SLV_SCLK1		10
-#define AUD_CLKID_SLV_SCLK2		11
-#define AUD_CLKID_SLV_SCLK3		12
-#define AUD_CLKID_SLV_SCLK4		13
-#define AUD_CLKID_SLV_SCLK5		14
-#define AUD_CLKID_SLV_SCLK6		15
-#define AUD_CLKID_SLV_SCLK7		16
-#define AUD_CLKID_SLV_SCLK8		17
-#define AUD_CLKID_SLV_SCLK9		18
-#define AUD_CLKID_SLV_LRCLK0		19
-#define AUD_CLKID_SLV_LRCLK1		20
-#define AUD_CLKID_SLV_LRCLK2		21
-#define AUD_CLKID_SLV_LRCLK3		22
-#define AUD_CLKID_SLV_LRCLK4		23
-#define AUD_CLKID_SLV_LRCLK5		24
-#define AUD_CLKID_SLV_LRCLK6		25
-#define AUD_CLKID_SLV_LRCLK7		26
-#define AUD_CLKID_SLV_LRCLK8		27
-#define AUD_CLKID_SLV_LRCLK9		28
 #define AUD_CLKID_DDR_ARB		29
 #define AUD_CLKID_PDM			30
 #define AUD_CLKID_TDMIN_A		31
@@ -90,5 +70,15 @@
 #define AUD_CLKID_TDMOUT_A_LRCLK	134
 #define AUD_CLKID_TDMOUT_B_LRCLK	135
 #define AUD_CLKID_TDMOUT_C_LRCLK	136
+#define AUD_CLKID_SPDIFOUT_B		151
+#define AUD_CLKID_SPDIFOUT_B_CLK	152
+#define AUD_CLKID_TDM_MCLK_PAD0		155
+#define AUD_CLKID_TDM_MCLK_PAD1		156
+#define AUD_CLKID_TDM_LRCLK_PAD0	157
+#define AUD_CLKID_TDM_LRCLK_PAD1	158
+#define AUD_CLKID_TDM_LRCLK_PAD2	159
+#define AUD_CLKID_TDM_SCLK_PAD0		160
+#define AUD_CLKID_TDM_SCLK_PAD1		161
+#define AUD_CLKID_TDM_SCLK_PAD2		162
 
 #endif /* __AXG_AUDIO_CLKC_BINDINGS_H */
diff --git a/include/dt-bindings/clock/exynos5410.h b/include/dt-bindings/clock/exynos5410.h
index f179eabbcdb7..86c2ad56c5ef 100644
--- a/include/dt-bindings/clock/exynos5410.h
+++ b/include/dt-bindings/clock/exynos5410.h
@@ -36,6 +36,7 @@
 #define CLK_UART0		257
 #define CLK_UART1		258
 #define CLK_UART2		259
+#define CLK_UART3		260
 #define CLK_I2C0		261
 #define CLK_I2C1		262
 #define CLK_I2C2		263
@@ -44,7 +45,7 @@
 #define CLK_USI1		266
 #define CLK_USI2		267
 #define CLK_USI3		268
-#define CLK_UART3		260
+#define CLK_TSADC		270
 #define CLK_PWM			279
 #define CLK_MCT			315
 #define CLK_WDT			316
diff --git a/include/dt-bindings/clock/g12a-aoclkc.h b/include/dt-bindings/clock/g12a-aoclkc.h
index 8db01ffbeb06..e916e49ff288 100644
--- a/include/dt-bindings/clock/g12a-aoclkc.h
+++ b/include/dt-bindings/clock/g12a-aoclkc.h
@@ -26,7 +26,9 @@
 #define CLKID_AO_M4_FCLK	13
 #define CLKID_AO_M4_HCLK	14
 #define CLKID_AO_CLK81		15
+#define CLKID_AO_SAR_ADC_SEL	16
 #define CLKID_AO_SAR_ADC_CLK	18
+#define CLKID_AO_CTS_OSCIN	19
 #define CLKID_AO_32K		23
 #define CLKID_AO_CEC		27
 #define CLKID_AO_CTS_RTC_OSCIN	28
diff --git a/include/dt-bindings/clock/g12a-clkc.h b/include/dt-bindings/clock/g12a-clkc.h
index 83b657038d1e..82c9e0c020b2 100644
--- a/include/dt-bindings/clock/g12a-clkc.h
+++ b/include/dt-bindings/clock/g12a-clkc.h
@@ -131,5 +131,10 @@
 #define CLKID_MALI_1				174
 #define CLKID_MALI				175
 #define CLKID_MPLL_5OM				177
+#define CLKID_CPU_CLK				187
+#define CLKID_PCIE_PLL				201
+#define CLKID_VDEC_1				204
+#define CLKID_VDEC_HEVC				207
+#define CLKID_VDEC_HEVCF			210
 
 #endif /* __G12A_CLKC_H */
diff --git a/include/dt-bindings/clock/imx7ulp-clock.h b/include/dt-bindings/clock/imx7ulp-clock.h
index 21d872e69cb1..6f66f9005c81 100644
--- a/include/dt-bindings/clock/imx7ulp-clock.h
+++ b/include/dt-bindings/clock/imx7ulp-clock.h
@@ -65,7 +65,6 @@
 #define IMX7ULP_CLK_FLEXBUS		2
 #define IMX7ULP_CLK_SEMA42_1		3
 #define IMX7ULP_CLK_DMA_MUX1		4
-#define IMX7ULP_CLK_SNVS		5
 #define IMX7ULP_CLK_CAAM		6
 #define IMX7ULP_CLK_LPTPM4		7
 #define IMX7ULP_CLK_LPTPM5		8
diff --git a/include/dt-bindings/clock/jz4725b-cgu.h b/include/dt-bindings/clock/jz4725b-cgu.h
index 460bbeff6ab8..31f1ab0fe42c 100644
--- a/include/dt-bindings/clock/jz4725b-cgu.h
+++ b/include/dt-bindings/clock/jz4725b-cgu.h
@@ -31,5 +31,6 @@
 #define JZ4725B_CLK_TCU		22
 #define JZ4725B_CLK_EXT512	23
 #define JZ4725B_CLK_RTC		24
+#define JZ4725B_CLK_UDC_PHY	25
 
 #endif /* __DT_BINDINGS_CLOCK_JZ4725B_CGU_H__ */
diff --git a/include/dt-bindings/clock/meson8b-clkc.h b/include/dt-bindings/clock/meson8b-clkc.h
index 8067077a62ca..47556539f0ee 100644
--- a/include/dt-bindings/clock/meson8b-clkc.h
+++ b/include/dt-bindings/clock/meson8b-clkc.h
@@ -103,10 +103,14 @@
 #define CLKID_MPLL1		94
 #define CLKID_MPLL2		95
 #define CLKID_NAND_CLK		112
-#define CLKID_ABP		124
 #define CLKID_APB		124
 #define CLKID_PERIPH		126
 #define CLKID_AXI		128
 #define CLKID_L2_DRAM		130
+#define CLKID_VPU		190
+#define CLKID_VDEC_1		196
+#define CLKID_VDEC_HCODEC	199
+#define CLKID_VDEC_2		202
+#define CLKID_VDEC_HEVC		206
 
 #endif /* __MESON8B_CLKC_H */
diff --git a/include/dt-bindings/clock/mt8183-clk.h b/include/dt-bindings/clock/mt8183-clk.h
new file mode 100644
index 000000000000..0046506eb24c
--- /dev/null
+++ b/include/dt-bindings/clock/mt8183-clk.h
@@ -0,0 +1,422 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2018 MediaTek Inc.
+ * Author: Weiyi Lu <weiyi.lu@mediatek.com>
+ */
+
+#ifndef _DT_BINDINGS_CLK_MT8183_H
+#define _DT_BINDINGS_CLK_MT8183_H
+
+/* APMIXED */
+#define CLK_APMIXED_ARMPLL_LL		0
+#define CLK_APMIXED_ARMPLL_L		1
+#define CLK_APMIXED_CCIPLL		2
+#define CLK_APMIXED_MAINPLL		3
+#define CLK_APMIXED_UNIV2PLL		4
+#define CLK_APMIXED_MSDCPLL		5
+#define CLK_APMIXED_MMPLL		6
+#define CLK_APMIXED_MFGPLL		7
+#define CLK_APMIXED_TVDPLL		8
+#define CLK_APMIXED_APLL1		9
+#define CLK_APMIXED_APLL2		10
+#define CLK_APMIXED_SSUSB_26M		11
+#define CLK_APMIXED_APPLL_26M		12
+#define CLK_APMIXED_MIPIC0_26M		13
+#define CLK_APMIXED_MDPLLGP_26M		14
+#define CLK_APMIXED_MMSYS_26M		15
+#define CLK_APMIXED_UFS_26M		16
+#define CLK_APMIXED_MIPIC1_26M		17
+#define CLK_APMIXED_MEMPLL_26M		18
+#define CLK_APMIXED_CLKSQ_LVPLL_26M	19
+#define CLK_APMIXED_MIPID0_26M		20
+#define CLK_APMIXED_MIPID1_26M		21
+#define CLK_APMIXED_NR_CLK		22
+
+/* TOPCKGEN */
+#define CLK_TOP_MUX_AXI			0
+#define CLK_TOP_MUX_MM			1
+#define CLK_TOP_MUX_CAM			2
+#define CLK_TOP_MUX_MFG			3
+#define CLK_TOP_MUX_CAMTG		4
+#define CLK_TOP_MUX_UART		5
+#define CLK_TOP_MUX_SPI			6
+#define CLK_TOP_MUX_MSDC50_0_HCLK	7
+#define CLK_TOP_MUX_MSDC50_0		8
+#define CLK_TOP_MUX_MSDC30_1		9
+#define CLK_TOP_MUX_MSDC30_2		10
+#define CLK_TOP_MUX_AUDIO		11
+#define CLK_TOP_MUX_AUD_INTBUS		12
+#define CLK_TOP_MUX_FPWRAP_ULPOSC	13
+#define CLK_TOP_MUX_SCP			14
+#define CLK_TOP_MUX_ATB			15
+#define CLK_TOP_MUX_SSPM		16
+#define CLK_TOP_MUX_DPI0		17
+#define CLK_TOP_MUX_SCAM		18
+#define CLK_TOP_MUX_AUD_1		19
+#define CLK_TOP_MUX_AUD_2		20
+#define CLK_TOP_MUX_DISP_PWM		21
+#define CLK_TOP_MUX_SSUSB_TOP_XHCI	22
+#define CLK_TOP_MUX_USB_TOP		23
+#define CLK_TOP_MUX_SPM			24
+#define CLK_TOP_MUX_I2C			25
+#define CLK_TOP_MUX_F52M_MFG		26
+#define CLK_TOP_MUX_SENINF		27
+#define CLK_TOP_MUX_DXCC		28
+#define CLK_TOP_MUX_CAMTG2		29
+#define CLK_TOP_MUX_AUD_ENG1		30
+#define CLK_TOP_MUX_AUD_ENG2		31
+#define CLK_TOP_MUX_FAES_UFSFDE		32
+#define CLK_TOP_MUX_FUFS		33
+#define CLK_TOP_MUX_IMG			34
+#define CLK_TOP_MUX_DSP			35
+#define CLK_TOP_MUX_DSP1		36
+#define CLK_TOP_MUX_DSP2		37
+#define CLK_TOP_MUX_IPU_IF		38
+#define CLK_TOP_MUX_CAMTG3		39
+#define CLK_TOP_MUX_CAMTG4		40
+#define CLK_TOP_MUX_PMICSPI		41
+#define CLK_TOP_SYSPLL_CK		42
+#define CLK_TOP_SYSPLL_D2		43
+#define CLK_TOP_SYSPLL_D3		44
+#define CLK_TOP_SYSPLL_D5		45
+#define CLK_TOP_SYSPLL_D7		46
+#define CLK_TOP_SYSPLL_D2_D2		47
+#define CLK_TOP_SYSPLL_D2_D4		48
+#define CLK_TOP_SYSPLL_D2_D8		49
+#define CLK_TOP_SYSPLL_D2_D16		50
+#define CLK_TOP_SYSPLL_D3_D2		51
+#define CLK_TOP_SYSPLL_D3_D4		52
+#define CLK_TOP_SYSPLL_D3_D8		53
+#define CLK_TOP_SYSPLL_D5_D2		54
+#define CLK_TOP_SYSPLL_D5_D4		55
+#define CLK_TOP_SYSPLL_D7_D2		56
+#define CLK_TOP_SYSPLL_D7_D4		57
+#define CLK_TOP_UNIVPLL_CK		58
+#define CLK_TOP_UNIVPLL_D2		59
+#define CLK_TOP_UNIVPLL_D3		60
+#define CLK_TOP_UNIVPLL_D5		61
+#define CLK_TOP_UNIVPLL_D7		62
+#define CLK_TOP_UNIVPLL_D2_D2		63
+#define CLK_TOP_UNIVPLL_D2_D4		64
+#define CLK_TOP_UNIVPLL_D2_D8		65
+#define CLK_TOP_UNIVPLL_D3_D2		66
+#define CLK_TOP_UNIVPLL_D3_D4		67
+#define CLK_TOP_UNIVPLL_D3_D8		68
+#define CLK_TOP_UNIVPLL_D5_D2		69
+#define CLK_TOP_UNIVPLL_D5_D4		70
+#define CLK_TOP_UNIVPLL_D5_D8		71
+#define CLK_TOP_APLL1_CK		72
+#define CLK_TOP_APLL1_D2		73
+#define CLK_TOP_APLL1_D4		74
+#define CLK_TOP_APLL1_D8		75
+#define CLK_TOP_APLL2_CK		76
+#define CLK_TOP_APLL2_D2		77
+#define CLK_TOP_APLL2_D4		78
+#define CLK_TOP_APLL2_D8		79
+#define CLK_TOP_TVDPLL_CK		80
+#define CLK_TOP_TVDPLL_D2		81
+#define CLK_TOP_TVDPLL_D4		82
+#define CLK_TOP_TVDPLL_D8		83
+#define CLK_TOP_TVDPLL_D16		84
+#define CLK_TOP_MSDCPLL_CK		85
+#define CLK_TOP_MSDCPLL_D2		86
+#define CLK_TOP_MSDCPLL_D4		87
+#define CLK_TOP_MSDCPLL_D8		88
+#define CLK_TOP_MSDCPLL_D16		89
+#define CLK_TOP_AD_OSC_CK		90
+#define CLK_TOP_OSC_D2			91
+#define CLK_TOP_OSC_D4			92
+#define CLK_TOP_OSC_D8			93
+#define CLK_TOP_OSC_D16			94
+#define CLK_TOP_F26M_CK_D2		95
+#define CLK_TOP_MFGPLL_CK		96
+#define CLK_TOP_UNIVP_192M_CK		97
+#define CLK_TOP_UNIVP_192M_D2		98
+#define CLK_TOP_UNIVP_192M_D4		99
+#define CLK_TOP_UNIVP_192M_D8		100
+#define CLK_TOP_UNIVP_192M_D16		101
+#define CLK_TOP_UNIVP_192M_D32		102
+#define CLK_TOP_MMPLL_CK		103
+#define CLK_TOP_MMPLL_D4		104
+#define CLK_TOP_MMPLL_D4_D2		105
+#define CLK_TOP_MMPLL_D4_D4		106
+#define CLK_TOP_MMPLL_D5		107
+#define CLK_TOP_MMPLL_D5_D2		108
+#define CLK_TOP_MMPLL_D5_D4		109
+#define CLK_TOP_MMPLL_D6		110
+#define CLK_TOP_MMPLL_D7		111
+#define CLK_TOP_CLK26M			112
+#define CLK_TOP_CLK13M			113
+#define CLK_TOP_ULPOSC			114
+#define CLK_TOP_UNIVP_192M		115
+#define CLK_TOP_MUX_APLL_I2S0		116
+#define CLK_TOP_MUX_APLL_I2S1		117
+#define CLK_TOP_MUX_APLL_I2S2		118
+#define CLK_TOP_MUX_APLL_I2S3		119
+#define CLK_TOP_MUX_APLL_I2S4		120
+#define CLK_TOP_MUX_APLL_I2S5		121
+#define CLK_TOP_APLL12_DIV0		122
+#define CLK_TOP_APLL12_DIV1		123
+#define CLK_TOP_APLL12_DIV2		124
+#define CLK_TOP_APLL12_DIV3		125
+#define CLK_TOP_APLL12_DIV4		126
+#define CLK_TOP_APLL12_DIVB		127
+#define CLK_TOP_UNIVPLL			128
+#define CLK_TOP_ARMPLL_DIV_PLL1		129
+#define CLK_TOP_ARMPLL_DIV_PLL2		130
+#define CLK_TOP_UNIVPLL_D3_D16		131
+#define CLK_TOP_NR_CLK			132
+
+/* CAMSYS */
+#define CLK_CAM_LARB6			0
+#define CLK_CAM_DFP_VAD			1
+#define CLK_CAM_CAM			2
+#define CLK_CAM_CAMTG			3
+#define CLK_CAM_SENINF			4
+#define CLK_CAM_CAMSV0			5
+#define CLK_CAM_CAMSV1			6
+#define CLK_CAM_CAMSV2			7
+#define CLK_CAM_CCU			8
+#define CLK_CAM_LARB3			9
+#define CLK_CAM_NR_CLK			10
+
+/* INFRACFG_AO */
+#define CLK_INFRA_PMIC_TMR		0
+#define CLK_INFRA_PMIC_AP		1
+#define CLK_INFRA_PMIC_MD		2
+#define CLK_INFRA_PMIC_CONN		3
+#define CLK_INFRA_SCPSYS		4
+#define CLK_INFRA_SEJ			5
+#define CLK_INFRA_APXGPT		6
+#define CLK_INFRA_ICUSB			7
+#define CLK_INFRA_GCE			8
+#define CLK_INFRA_THERM			9
+#define CLK_INFRA_I2C0			10
+#define CLK_INFRA_I2C1			11
+#define CLK_INFRA_I2C2			12
+#define CLK_INFRA_I2C3			13
+#define CLK_INFRA_PWM_HCLK		14
+#define CLK_INFRA_PWM1			15
+#define CLK_INFRA_PWM2			16
+#define CLK_INFRA_PWM3			17
+#define CLK_INFRA_PWM4			18
+#define CLK_INFRA_PWM			19
+#define CLK_INFRA_UART0			20
+#define CLK_INFRA_UART1			21
+#define CLK_INFRA_UART2			22
+#define CLK_INFRA_UART3			23
+#define CLK_INFRA_GCE_26M		24
+#define CLK_INFRA_CQ_DMA_FPC		25
+#define CLK_INFRA_BTIF			26
+#define CLK_INFRA_SPI0			27
+#define CLK_INFRA_MSDC0			28
+#define CLK_INFRA_MSDC1			29
+#define CLK_INFRA_MSDC2			30
+#define CLK_INFRA_MSDC0_SCK		31
+#define CLK_INFRA_DVFSRC		32
+#define CLK_INFRA_GCPU			33
+#define CLK_INFRA_TRNG			34
+#define CLK_INFRA_AUXADC		35
+#define CLK_INFRA_CPUM			36
+#define CLK_INFRA_CCIF1_AP		37
+#define CLK_INFRA_CCIF1_MD		38
+#define CLK_INFRA_AUXADC_MD		39
+#define CLK_INFRA_MSDC1_SCK		40
+#define CLK_INFRA_MSDC2_SCK		41
+#define CLK_INFRA_AP_DMA		42
+#define CLK_INFRA_XIU			43
+#define CLK_INFRA_DEVICE_APC		44
+#define CLK_INFRA_CCIF_AP		45
+#define CLK_INFRA_DEBUGSYS		46
+#define CLK_INFRA_AUDIO			47
+#define CLK_INFRA_CCIF_MD		48
+#define CLK_INFRA_DXCC_SEC_CORE		49
+#define CLK_INFRA_DXCC_AO		50
+#define CLK_INFRA_DRAMC_F26M		51
+#define CLK_INFRA_IRTX			52
+#define CLK_INFRA_DISP_PWM		53
+#define CLK_INFRA_CLDMA_BCLK		54
+#define CLK_INFRA_AUDIO_26M_BCLK	55
+#define CLK_INFRA_SPI1			56
+#define CLK_INFRA_I2C4			57
+#define CLK_INFRA_MODEM_TEMP_SHARE	58
+#define CLK_INFRA_SPI2			59
+#define CLK_INFRA_SPI3			60
+#define CLK_INFRA_UNIPRO_SCK		61
+#define CLK_INFRA_UNIPRO_TICK		62
+#define CLK_INFRA_UFS_MP_SAP_BCLK	63
+#define CLK_INFRA_MD32_BCLK		64
+#define CLK_INFRA_SSPM			65
+#define CLK_INFRA_UNIPRO_MBIST		66
+#define CLK_INFRA_SSPM_BUS_HCLK		67
+#define CLK_INFRA_I2C5			68
+#define CLK_INFRA_I2C5_ARBITER		69
+#define CLK_INFRA_I2C5_IMM		70
+#define CLK_INFRA_I2C1_ARBITER		71
+#define CLK_INFRA_I2C1_IMM		72
+#define CLK_INFRA_I2C2_ARBITER		73
+#define CLK_INFRA_I2C2_IMM		74
+#define CLK_INFRA_SPI4			75
+#define CLK_INFRA_SPI5			76
+#define CLK_INFRA_CQ_DMA		77
+#define CLK_INFRA_UFS			78
+#define CLK_INFRA_AES_UFSFDE		79
+#define CLK_INFRA_UFS_TICK		80
+#define CLK_INFRA_MSDC0_SELF		81
+#define CLK_INFRA_MSDC1_SELF		82
+#define CLK_INFRA_MSDC2_SELF		83
+#define CLK_INFRA_SSPM_26M_SELF		84
+#define CLK_INFRA_SSPM_32K_SELF		85
+#define CLK_INFRA_UFS_AXI		86
+#define CLK_INFRA_I2C6			87
+#define CLK_INFRA_AP_MSDC0		88
+#define CLK_INFRA_MD_MSDC0		89
+#define CLK_INFRA_USB			90
+#define CLK_INFRA_DEVMPU_BCLK		91
+#define CLK_INFRA_CCIF2_AP		92
+#define CLK_INFRA_CCIF2_MD		93
+#define CLK_INFRA_CCIF3_AP		94
+#define CLK_INFRA_CCIF3_MD		95
+#define CLK_INFRA_SEJ_F13M		96
+#define CLK_INFRA_AES_BCLK		97
+#define CLK_INFRA_I2C7			98
+#define CLK_INFRA_I2C8			99
+#define CLK_INFRA_FBIST2FPC		100
+#define CLK_INFRA_NR_CLK		101
+
+/* MFGCFG */
+#define CLK_MFG_BG3D			0
+#define CLK_MFG_NR_CLK			1
+
+/* IMG */
+#define CLK_IMG_OWE			0
+#define CLK_IMG_WPE_B			1
+#define CLK_IMG_WPE_A			2
+#define CLK_IMG_MFB			3
+#define CLK_IMG_RSC			4
+#define CLK_IMG_DPE			5
+#define CLK_IMG_FDVT			6
+#define CLK_IMG_DIP			7
+#define CLK_IMG_LARB2			8
+#define CLK_IMG_LARB5			9
+#define CLK_IMG_NR_CLK			10
+
+/* MMSYS_CONFIG */
+#define CLK_MM_SMI_COMMON		0
+#define CLK_MM_SMI_LARB0		1
+#define CLK_MM_SMI_LARB1		2
+#define CLK_MM_GALS_COMM0		3
+#define CLK_MM_GALS_COMM1		4
+#define CLK_MM_GALS_CCU2MM		5
+#define CLK_MM_GALS_IPU12MM		6
+#define CLK_MM_GALS_IMG2MM		7
+#define CLK_MM_GALS_CAM2MM		8
+#define CLK_MM_GALS_IPU2MM		9
+#define CLK_MM_MDP_DL_TXCK		10
+#define CLK_MM_IPU_DL_TXCK		11
+#define CLK_MM_MDP_RDMA0		12
+#define CLK_MM_MDP_RDMA1		13
+#define CLK_MM_MDP_RSZ0			14
+#define CLK_MM_MDP_RSZ1			15
+#define CLK_MM_MDP_TDSHP		16
+#define CLK_MM_MDP_WROT0		17
+#define CLK_MM_FAKE_ENG			18
+#define CLK_MM_DISP_OVL0		19
+#define CLK_MM_DISP_OVL0_2L		20
+#define CLK_MM_DISP_OVL1_2L		21
+#define CLK_MM_DISP_RDMA0		22
+#define CLK_MM_DISP_RDMA1		23
+#define CLK_MM_DISP_WDMA0		24
+#define CLK_MM_DISP_COLOR0		25
+#define CLK_MM_DISP_CCORR0		26
+#define CLK_MM_DISP_AAL0		27
+#define CLK_MM_DISP_GAMMA0		28
+#define CLK_MM_DISP_DITHER0		29
+#define CLK_MM_DISP_SPLIT		30
+#define CLK_MM_DSI0_MM			31
+#define CLK_MM_DSI0_IF			32
+#define CLK_MM_DPI_MM			33
+#define CLK_MM_DPI_IF			34
+#define CLK_MM_FAKE_ENG2		35
+#define CLK_MM_MDP_DL_RX		36
+#define CLK_MM_IPU_DL_RX		37
+#define CLK_MM_26M			38
+#define CLK_MM_MMSYS_R2Y		39
+#define CLK_MM_DISP_RSZ			40
+#define CLK_MM_MDP_WDMA0		41
+#define CLK_MM_MDP_AAL			42
+#define CLK_MM_MDP_CCORR		43
+#define CLK_MM_DBI_MM			44
+#define CLK_MM_DBI_IF			45
+#define CLK_MM_NR_CLK			46
+
+/* VDEC_GCON */
+#define CLK_VDEC_VDEC			0
+#define CLK_VDEC_LARB1			1
+#define CLK_VDEC_NR_CLK			2
+
+/* VENC_GCON */
+#define CLK_VENC_LARB			0
+#define CLK_VENC_VENC			1
+#define CLK_VENC_JPGENC			2
+#define CLK_VENC_NR_CLK			3
+
+/* AUDIO */
+#define CLK_AUDIO_TML			0
+#define CLK_AUDIO_DAC_PREDIS		1
+#define CLK_AUDIO_DAC			2
+#define CLK_AUDIO_ADC			3
+#define CLK_AUDIO_APLL_TUNER		4
+#define CLK_AUDIO_APLL2_TUNER		5
+#define CLK_AUDIO_24M			6
+#define CLK_AUDIO_22M			7
+#define CLK_AUDIO_AFE			8
+#define CLK_AUDIO_I2S4			9
+#define CLK_AUDIO_I2S3			10
+#define CLK_AUDIO_I2S2			11
+#define CLK_AUDIO_I2S1			12
+#define CLK_AUDIO_PDN_ADDA6_ADC		13
+#define CLK_AUDIO_TDM			14
+#define CLK_AUDIO_NR_CLK		15
+
+/* IPU_CONN */
+#define CLK_IPU_CONN_IPU		0
+#define CLK_IPU_CONN_AHB		1
+#define CLK_IPU_CONN_AXI		2
+#define CLK_IPU_CONN_ISP		3
+#define CLK_IPU_CONN_CAM_ADL		4
+#define CLK_IPU_CONN_IMG_ADL		5
+#define CLK_IPU_CONN_DAP_RX		6
+#define CLK_IPU_CONN_APB2AXI		7
+#define CLK_IPU_CONN_APB2AHB		8
+#define CLK_IPU_CONN_IPU_CAB1TO2	9
+#define CLK_IPU_CONN_IPU1_CAB1TO2	10
+#define CLK_IPU_CONN_IPU2_CAB1TO2	11
+#define CLK_IPU_CONN_CAB3TO3		12
+#define CLK_IPU_CONN_CAB2TO1		13
+#define CLK_IPU_CONN_CAB3TO1_SLICE	14
+#define CLK_IPU_CONN_NR_CLK		15
+
+/* IPU_ADL */
+#define CLK_IPU_ADL_CABGEN		0
+#define CLK_IPU_ADL_NR_CLK		1
+
+/* IPU_CORE0 */
+#define CLK_IPU_CORE0_JTAG		0
+#define CLK_IPU_CORE0_AXI		1
+#define CLK_IPU_CORE0_IPU		2
+#define CLK_IPU_CORE0_NR_CLK		3
+
+/* IPU_CORE1 */
+#define CLK_IPU_CORE1_JTAG		0
+#define CLK_IPU_CORE1_AXI		1
+#define CLK_IPU_CORE1_IPU		2
+#define CLK_IPU_CORE1_NR_CLK		3
+
+/* MCUCFG */
+#define CLK_MCU_MP0_SEL			0
+#define CLK_MCU_MP2_SEL			1
+#define CLK_MCU_BUS_SEL			2
+#define CLK_MCU_NR_CLK			3
+
+#endif /* _DT_BINDINGS_CLK_MT8183_H */
diff --git a/include/dt-bindings/clock/mt8516-clk.h b/include/dt-bindings/clock/mt8516-clk.h
new file mode 100644
index 000000000000..9cfca53cd78d
--- /dev/null
+++ b/include/dt-bindings/clock/mt8516-clk.h
@@ -0,0 +1,211 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2019 MediaTek Inc.
+ * Copyright (c) 2019 BayLibre, SAS.
+ * Author: James Liao <jamesjj.liao@mediatek.com>
+ */
+
+#ifndef _DT_BINDINGS_CLK_MT8516_H
+#define _DT_BINDINGS_CLK_MT8516_H
+
+/* APMIXEDSYS */
+
+#define CLK_APMIXED_ARMPLL		0
+#define CLK_APMIXED_MAINPLL		1
+#define CLK_APMIXED_UNIVPLL		2
+#define CLK_APMIXED_MMPLL		3
+#define CLK_APMIXED_APLL1		4
+#define CLK_APMIXED_APLL2		5
+#define CLK_APMIXED_NR_CLK		6
+
+/* INFRACFG */
+
+#define CLK_IFR_MUX1_SEL		0
+#define CLK_IFR_ETH_25M_SEL		1
+#define CLK_IFR_I2C0_SEL		2
+#define CLK_IFR_I2C1_SEL		3
+#define CLK_IFR_I2C2_SEL		4
+#define CLK_IFR_NR_CLK			5
+
+/* TOPCKGEN */
+
+#define CLK_TOP_CLK_NULL		0
+#define CLK_TOP_I2S_INFRA_BCK		1
+#define CLK_TOP_MEMPLL			2
+#define CLK_TOP_DMPLL			3
+#define CLK_TOP_MAINPLL_D2		4
+#define CLK_TOP_MAINPLL_D4		5
+#define CLK_TOP_MAINPLL_D8		6
+#define CLK_TOP_MAINPLL_D16		7
+#define CLK_TOP_MAINPLL_D11		8
+#define CLK_TOP_MAINPLL_D22		9
+#define CLK_TOP_MAINPLL_D3		10
+#define CLK_TOP_MAINPLL_D6		11
+#define CLK_TOP_MAINPLL_D12		12
+#define CLK_TOP_MAINPLL_D5		13
+#define CLK_TOP_MAINPLL_D10		14
+#define CLK_TOP_MAINPLL_D20		15
+#define CLK_TOP_MAINPLL_D40		16
+#define CLK_TOP_MAINPLL_D7		17
+#define CLK_TOP_MAINPLL_D14		18
+#define CLK_TOP_UNIVPLL_D2		19
+#define CLK_TOP_UNIVPLL_D4		20
+#define CLK_TOP_UNIVPLL_D8		21
+#define CLK_TOP_UNIVPLL_D16		22
+#define CLK_TOP_UNIVPLL_D3		23
+#define CLK_TOP_UNIVPLL_D6		24
+#define CLK_TOP_UNIVPLL_D12		25
+#define CLK_TOP_UNIVPLL_D24		26
+#define CLK_TOP_UNIVPLL_D5		27
+#define CLK_TOP_UNIVPLL_D20		28
+#define CLK_TOP_MMPLL380M		29
+#define CLK_TOP_MMPLL_D2		30
+#define CLK_TOP_MMPLL_200M		31
+#define CLK_TOP_USB_PHY48M		32
+#define CLK_TOP_APLL1			33
+#define CLK_TOP_APLL1_D2		34
+#define CLK_TOP_APLL1_D4		35
+#define CLK_TOP_APLL1_D8		36
+#define CLK_TOP_APLL2			37
+#define CLK_TOP_APLL2_D2		38
+#define CLK_TOP_APLL2_D4		39
+#define CLK_TOP_APLL2_D8		40
+#define CLK_TOP_CLK26M			41
+#define CLK_TOP_CLK26M_D2		42
+#define CLK_TOP_AHB_INFRA_D2		43
+#define CLK_TOP_NFI1X			44
+#define CLK_TOP_ETH_D2			45
+#define CLK_TOP_THEM			46
+#define CLK_TOP_APDMA			47
+#define CLK_TOP_I2C0			48
+#define CLK_TOP_I2C1			49
+#define CLK_TOP_AUXADC1			50
+#define CLK_TOP_NFI			51
+#define CLK_TOP_NFIECC			52
+#define CLK_TOP_DEBUGSYS		53
+#define CLK_TOP_PWM			54
+#define CLK_TOP_UART0			55
+#define CLK_TOP_UART1			56
+#define CLK_TOP_BTIF			57
+#define CLK_TOP_USB			58
+#define CLK_TOP_FLASHIF_26M		59
+#define CLK_TOP_AUXADC2			60
+#define CLK_TOP_I2C2			61
+#define CLK_TOP_MSDC0			62
+#define CLK_TOP_MSDC1			63
+#define CLK_TOP_NFI2X			64
+#define CLK_TOP_PMICWRAP_AP		65
+#define CLK_TOP_SEJ			66
+#define CLK_TOP_MEMSLP_DLYER		67
+#define CLK_TOP_SPI			68
+#define CLK_TOP_APXGPT			69
+#define CLK_TOP_AUDIO			70
+#define CLK_TOP_PMICWRAP_MD		71
+#define CLK_TOP_PMICWRAP_CONN		72
+#define CLK_TOP_PMICWRAP_26M		73
+#define CLK_TOP_AUX_ADC			74
+#define CLK_TOP_AUX_TP			75
+#define CLK_TOP_MSDC2			76
+#define CLK_TOP_RBIST			77
+#define CLK_TOP_NFI_BUS			78
+#define CLK_TOP_GCE			79
+#define CLK_TOP_TRNG			80
+#define CLK_TOP_SEJ_13M			81
+#define CLK_TOP_AES			82
+#define CLK_TOP_PWM_B			83
+#define CLK_TOP_PWM1_FB			84
+#define CLK_TOP_PWM2_FB			85
+#define CLK_TOP_PWM3_FB			86
+#define CLK_TOP_PWM4_FB			87
+#define CLK_TOP_PWM5_FB			88
+#define CLK_TOP_USB_1P			89
+#define CLK_TOP_FLASHIF_FREERUN		90
+#define CLK_TOP_66M_ETH			91
+#define CLK_TOP_133M_ETH		92
+#define CLK_TOP_FETH_25M		93
+#define CLK_TOP_FETH_50M		94
+#define CLK_TOP_FLASHIF_AXI		95
+#define CLK_TOP_USBIF			96
+#define CLK_TOP_UART2			97
+#define CLK_TOP_BSI			98
+#define CLK_TOP_RG_SPINOR		99
+#define CLK_TOP_RG_MSDC2		100
+#define CLK_TOP_RG_ETH			101
+#define CLK_TOP_RG_AUD1			102
+#define CLK_TOP_RG_AUD2			103
+#define CLK_TOP_RG_AUD_ENGEN1		104
+#define CLK_TOP_RG_AUD_ENGEN2		105
+#define CLK_TOP_RG_I2C			106
+#define CLK_TOP_RG_PWM_INFRA		107
+#define CLK_TOP_RG_AUD_SPDIF_IN		108
+#define CLK_TOP_RG_UART2		109
+#define CLK_TOP_RG_BSI			110
+#define CLK_TOP_RG_DBG_ATCLK		111
+#define CLK_TOP_RG_NFIECC		112
+#define CLK_TOP_RG_APLL1_D2_EN		113
+#define CLK_TOP_RG_APLL1_D4_EN		114
+#define CLK_TOP_RG_APLL1_D8_EN		115
+#define CLK_TOP_RG_APLL2_D2_EN		116
+#define CLK_TOP_RG_APLL2_D4_EN		117
+#define CLK_TOP_RG_APLL2_D8_EN		118
+#define CLK_TOP_APLL12_DIV0		119
+#define CLK_TOP_APLL12_DIV1		120
+#define CLK_TOP_APLL12_DIV2		121
+#define CLK_TOP_APLL12_DIV3		122
+#define CLK_TOP_APLL12_DIV4		123
+#define CLK_TOP_APLL12_DIV4B		124
+#define CLK_TOP_APLL12_DIV5		125
+#define CLK_TOP_APLL12_DIV5B		126
+#define CLK_TOP_APLL12_DIV6		127
+#define CLK_TOP_UART0_SEL		128
+#define CLK_TOP_EMI_DDRPHY_SEL		129
+#define CLK_TOP_AHB_INFRA_SEL		130
+#define CLK_TOP_MSDC0_SEL		131
+#define CLK_TOP_UART1_SEL		132
+#define CLK_TOP_MSDC1_SEL		133
+#define CLK_TOP_PMICSPI_SEL		134
+#define CLK_TOP_QAXI_AUD26M_SEL		135
+#define CLK_TOP_AUD_INTBUS_SEL		136
+#define CLK_TOP_NFI2X_PAD_SEL		137
+#define CLK_TOP_NFI1X_PAD_SEL		138
+#define CLK_TOP_DDRPHYCFG_SEL		139
+#define CLK_TOP_USB_78M_SEL		140
+#define CLK_TOP_SPINOR_SEL		141
+#define CLK_TOP_MSDC2_SEL		142
+#define CLK_TOP_ETH_SEL			143
+#define CLK_TOP_AUD1_SEL		144
+#define CLK_TOP_AUD2_SEL		145
+#define CLK_TOP_AUD_ENGEN1_SEL		146
+#define CLK_TOP_AUD_ENGEN2_SEL		147
+#define CLK_TOP_I2C_SEL			148
+#define CLK_TOP_AUD_I2S0_M_SEL		149
+#define CLK_TOP_AUD_I2S1_M_SEL		150
+#define CLK_TOP_AUD_I2S2_M_SEL		151
+#define CLK_TOP_AUD_I2S3_M_SEL		152
+#define CLK_TOP_AUD_I2S4_M_SEL		153
+#define CLK_TOP_AUD_I2S5_M_SEL		154
+#define CLK_TOP_AUD_SPDIF_B_SEL		155
+#define CLK_TOP_PWM_SEL			156
+#define CLK_TOP_SPI_SEL			157
+#define CLK_TOP_AUD_SPDIFIN_SEL		158
+#define CLK_TOP_UART2_SEL		159
+#define CLK_TOP_BSI_SEL			160
+#define CLK_TOP_DBG_ATCLK_SEL		161
+#define CLK_TOP_CSW_NFIECC_SEL		162
+#define CLK_TOP_NFIECC_SEL		163
+#define CLK_TOP_APLL12_CK_DIV0		164
+#define CLK_TOP_APLL12_CK_DIV1		165
+#define CLK_TOP_APLL12_CK_DIV2		166
+#define CLK_TOP_APLL12_CK_DIV3		167
+#define CLK_TOP_APLL12_CK_DIV4		168
+#define CLK_TOP_APLL12_CK_DIV4B		169
+#define CLK_TOP_APLL12_CK_DIV5		170
+#define CLK_TOP_APLL12_CK_DIV5B		171
+#define CLK_TOP_APLL12_CK_DIV6		172
+#define CLK_TOP_USB_78M			173
+#define CLK_TOP_MSDC0_INFRA		174
+#define CLK_TOP_MSDC1_INFRA		175
+#define CLK_TOP_MSDC2_INFRA		176
+#define CLK_TOP_NR_CLK			177
+
+#endif /* _DT_BINDINGS_CLK_MT8516_H */
diff --git a/include/dt-bindings/clock/qcom,gcc-qcs404.h b/include/dt-bindings/clock/qcom,gcc-qcs404.h
index 6ceb55ed72c6..454b3f43f538 100644
--- a/include/dt-bindings/clock/qcom,gcc-qcs404.h
+++ b/include/dt-bindings/clock/qcom,gcc-qcs404.h
@@ -146,6 +146,10 @@
 #define GCC_MDP_TBU_CLK					138
 #define GCC_QDSS_DAP_CLK				139
 #define GCC_DCC_XO_CLK					140
+#define GCC_CDSP_CFG_AHB_CLK				143
+#define GCC_BIMC_CDSP_CLK				144
+#define GCC_CDSP_TBU_CLK				145
+#define GCC_CDSP_BIMC_CLK_SRC				146
 
 #define GCC_GENI_IR_BCR					0
 #define GCC_USB_HS_BCR					1
@@ -161,5 +165,6 @@
 #define GCC_PCIE_0_LINK_DOWN_BCR			11
 #define GCC_PCIEPHY_0_PHY_BCR				12
 #define GCC_EMAC_BCR					13
+#define GCC_CDSP_RESTART				14
 
 #endif
diff --git a/include/dt-bindings/clock/qcom,turingcc-qcs404.h b/include/dt-bindings/clock/qcom,turingcc-qcs404.h
new file mode 100644
index 000000000000..838faef57c67
--- /dev/null
+++ b/include/dt-bindings/clock/qcom,turingcc-qcs404.h
@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2019, Linaro Ltd
+ */
+
+#ifndef _DT_BINDINGS_CLK_TURING_QCS404_H
+#define _DT_BINDINGS_CLK_TURING_QCS404_H
+
+#define TURING_Q6SS_Q6_AXIM_CLK		0
+#define TURING_Q6SS_AHBM_AON_CLK	1
+#define TURING_WRAPPER_AON_CLK		2
+#define TURING_Q6SS_AHBS_AON_CLK	3
+#define TURING_WRAPPER_QOS_AHBS_AON_CLK	4
+
+#endif
diff --git a/include/dt-bindings/clock/stm32fx-clock.h b/include/dt-bindings/clock/stm32fx-clock.h
index 58d8b515be55..7d34e297049c 100644
--- a/include/dt-bindings/clock/stm32fx-clock.h
+++ b/include/dt-bindings/clock/stm32fx-clock.h
@@ -54,7 +54,10 @@
 #define CLK_I2C3		28
 #define CLK_I2C4		29
 #define CLK_LPTIMER		30
-
-#define END_PRIMARY_CLK_F7	31
+#define CLK_PLL_SRC		31
+#define CLK_DFSDM1		32
+#define CLK_ADFSDM1		33
+#define CLK_F769_DSI		34
+#define END_PRIMARY_CLK_F7	35
 
 #endif
diff --git a/include/dt-bindings/clock/sun5i-ccu.h b/include/dt-bindings/clock/sun5i-ccu.h
index 81f34d477aeb..2e6b9ddcc24e 100644
--- a/include/dt-bindings/clock/sun5i-ccu.h
+++ b/include/dt-bindings/clock/sun5i-ccu.h
@@ -100,7 +100,7 @@
 #define CLK_AVS			96
 #define CLK_HDMI		97
 #define CLK_GPU			98
-
+#define CLK_MBUS		99
 #define CLK_IEP			100
 
 #endif /* _DT_BINDINGS_CLK_SUN5I_H_ */
diff --git a/include/dt-bindings/phy/phy-am654-serdes.h b/include/dt-bindings/phy/phy-am654-serdes.h
new file mode 100644
index 000000000000..e8d901729ed9
--- /dev/null
+++ b/include/dt-bindings/phy/phy-am654-serdes.h
@@ -0,0 +1,13 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * This header provides constants for AM654 SERDES.
+ */
+
+#ifndef _DT_BINDINGS_AM654_SERDES
+#define _DT_BINDINGS_AM654_SERDES
+
+#define AM654_SERDES_CMU_REFCLK	0
+#define AM654_SERDES_LO_REFCLK	1
+#define AM654_SERDES_RO_REFCLK	2
+
+#endif /* _DT_BINDINGS_AM654_SERDES */
diff --git a/include/dt-bindings/pinctrl/stm32-pinfunc.h b/include/dt-bindings/pinctrl/stm32-pinfunc.h
index b5a2174a6386..e6fb8ada3f4d 100644
--- a/include/dt-bindings/pinctrl/stm32-pinfunc.h
+++ b/include/dt-bindings/pinctrl/stm32-pinfunc.h
@@ -32,5 +32,11 @@
 
 #define STM32_PINMUX(port, line, mode) (((PIN_NO(port, line)) << 8) | (mode))
 
+/*  package information */
+#define STM32MP_PKG_AA	0x1
+#define STM32MP_PKG_AB	0x2
+#define STM32MP_PKG_AC	0x4
+#define STM32MP_PKG_AD	0x8
+
 #endif /* _DT_BINDINGS_STM32_PINFUNC_H */
 
diff --git a/include/linux/cgroup-defs.h b/include/linux/cgroup-defs.h
index 1c70803e9f77..77258d276f93 100644
--- a/include/linux/cgroup-defs.h
+++ b/include/linux/cgroup-defs.h
@@ -65,6 +65,12 @@ enum {
 	 * specified at mount time and thus is implemented here.
 	 */
 	CGRP_CPUSET_CLONE_CHILDREN,
+
+	/* Control group has to be frozen. */
+	CGRP_FREEZE,
+
+	/* Cgroup is frozen. */
+	CGRP_FROZEN,
 };
 
 /* cgroup_root->flags */
@@ -317,6 +323,25 @@ struct cgroup_rstat_cpu {
 	struct cgroup *updated_next;		/* NULL iff not on the list */
 };
 
+struct cgroup_freezer_state {
+	/* Should the cgroup and its descendants be frozen. */
+	bool freeze;
+
+	/* Should the cgroup actually be frozen? */
+	int e_freeze;
+
+	/* Fields below are protected by css_set_lock */
+
+	/* Number of frozen descendant cgroups */
+	int nr_frozen_descendants;
+
+	/*
+	 * Number of tasks, which are counted as frozen:
+	 * frozen, SIGSTOPped, and PTRACEd.
+	 */
+	int nr_frozen_tasks;
+};
+
 struct cgroup {
 	/* self css with NULL ->ss, points back to this cgroup */
 	struct cgroup_subsys_state self;
@@ -349,6 +374,11 @@ struct cgroup {
 	 * Dying cgroups are cgroups which were deleted by a user,
 	 * but are still existing because someone else is holding a reference.
 	 * max_descendants is a maximum allowed number of descent cgroups.
+	 *
+	 * nr_descendants and nr_dying_descendants are protected
+	 * by cgroup_mutex and css_set_lock. It's fine to read them holding
+	 * any of cgroup_mutex and css_set_lock; for writing both locks
+	 * should be held.
 	 */
 	int nr_descendants;
 	int nr_dying_descendants;
@@ -448,6 +478,9 @@ struct cgroup {
 	/* If there is block congestion on this cgroup. */
 	atomic_t congestion_count;
 
+	/* Used to store internal freezer state */
+	struct cgroup_freezer_state freezer;
+
 	/* ids of the ancestors at each level including self */
 	int ancestor_ids[];
 };
diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h
index 81f58b4a5418..c0077adeea83 100644
--- a/include/linux/cgroup.h
+++ b/include/linux/cgroup.h
@@ -881,4 +881,47 @@ static inline void put_cgroup_ns(struct cgroup_namespace *ns)
 		free_cgroup_ns(ns);
 }
 
+#ifdef CONFIG_CGROUPS
+
+void cgroup_enter_frozen(void);
+void cgroup_leave_frozen(bool always_leave);
+void cgroup_update_frozen(struct cgroup *cgrp);
+void cgroup_freeze(struct cgroup *cgrp, bool freeze);
+void cgroup_freezer_migrate_task(struct task_struct *task, struct cgroup *src,
+				 struct cgroup *dst);
+
+static inline bool cgroup_task_freeze(struct task_struct *task)
+{
+	bool ret;
+
+	if (task->flags & PF_KTHREAD)
+		return false;
+
+	rcu_read_lock();
+	ret = test_bit(CGRP_FREEZE, &task_dfl_cgroup(task)->flags);
+	rcu_read_unlock();
+
+	return ret;
+}
+
+static inline bool cgroup_task_frozen(struct task_struct *task)
+{
+	return task->frozen;
+}
+
+#else /* !CONFIG_CGROUPS */
+
+static inline void cgroup_enter_frozen(void) { }
+static inline void cgroup_leave_frozen(bool always_leave) { }
+static inline bool cgroup_task_freeze(struct task_struct *task)
+{
+	return false;
+}
+static inline bool cgroup_task_frozen(struct task_struct *task)
+{
+	return false;
+}
+
+#endif /* !CONFIG_CGROUPS */
+
 #endif /* _LINUX_CGROUP_H */
diff --git a/include/linux/clk-provider.h b/include/linux/clk-provider.h
index b7cf80a71293..491d992d045d 100644
--- a/include/linux/clk-provider.h
+++ b/include/linux/clk-provider.h
@@ -24,7 +24,7 @@
 #define CLK_SET_RATE_PARENT	BIT(2) /* propagate rate change up one level */
 #define CLK_IGNORE_UNUSED	BIT(3) /* do not gate even if unused */
 				/* unused */
-#define CLK_IS_BASIC		BIT(5) /* Basic clk, can't do a to_clk_foo() */
+				/* unused */
 #define CLK_GET_RATE_NOCACHE	BIT(6) /* do not use the cached clk rate */
 #define CLK_SET_RATE_NO_REPARENT BIT(7) /* don't re-parent on rate change */
 #define CLK_GET_ACCURACY_NOCACHE BIT(8) /* do not use the cached clk accuracy */
@@ -251,19 +251,40 @@ struct clk_ops {
 };
 
 /**
+ * struct clk_parent_data - clk parent information
+ * @hw: parent clk_hw pointer (used for clk providers with internal clks)
+ * @fw_name: parent name local to provider registering clk
+ * @name: globally unique parent name (used as a fallback)
+ * @index: parent index local to provider registering clk (if @fw_name absent)
+ */
+struct clk_parent_data {
+	const struct clk_hw	*hw;
+	const char		*fw_name;
+	const char		*name;
+	int			index;
+};
+
+/**
  * struct clk_init_data - holds init data that's common to all clocks and is
  * shared between the clock provider and the common clock framework.
  *
  * @name: clock name
  * @ops: operations this clock supports
  * @parent_names: array of string names for all possible parents
+ * @parent_data: array of parent data for all possible parents (when some
+ *               parents are external to the clk controller)
+ * @parent_hws: array of pointers to all possible parents (when all parents
+ *              are internal to the clk controller)
  * @num_parents: number of possible parents
  * @flags: framework-level hints and quirks
  */
 struct clk_init_data {
 	const char		*name;
 	const struct clk_ops	*ops;
+	/* Only one of the following three should be assigned */
 	const char		* const *parent_names;
+	const struct clk_parent_data	*parent_data;
+	const struct clk_hw		**parent_hws;
 	u8			num_parents;
 	unsigned long		flags;
 };
@@ -307,7 +328,6 @@ struct clk_fixed_rate {
 	struct		clk_hw hw;
 	unsigned long	fixed_rate;
 	unsigned long	fixed_accuracy;
-	u8		flags;
 };
 
 #define to_clk_fixed_rate(_hw) container_of(_hw, struct clk_fixed_rate, hw)
@@ -349,6 +369,9 @@ void of_fixed_clk_setup(struct device_node *np);
  *	of this register, and mask of gate bits are in higher 16-bit of this
  *	register.  While setting the gate bits, higher 16-bit should also be
  *	updated to indicate changing gate bits.
+ * CLK_GATE_BIG_ENDIAN - by default little endian register accesses are used for
+ *	the gate register.  Setting this flag makes the register accesses big
+ *	endian.
  */
 struct clk_gate {
 	struct clk_hw hw;
@@ -362,6 +385,7 @@ struct clk_gate {
 
 #define CLK_GATE_SET_TO_DISABLE		BIT(0)
 #define CLK_GATE_HIWORD_MASK		BIT(1)
+#define CLK_GATE_BIG_ENDIAN		BIT(2)
 
 extern const struct clk_ops clk_gate_ops;
 struct clk *clk_register_gate(struct device *dev, const char *name,
@@ -417,6 +441,9 @@ struct clk_div_table {
  * CLK_DIVIDER_MAX_AT_ZERO - For dividers which are like CLK_DIVIDER_ONE_BASED
  *	except when the value read from the register is zero, the divisor is
  *	2^width of the field.
+ * CLK_DIVIDER_BIG_ENDIAN - By default little endian register accesses are used
+ *	for the divider register.  Setting this flag makes the register accesses
+ *	big endian.
  */
 struct clk_divider {
 	struct clk_hw	hw;
@@ -438,6 +465,7 @@ struct clk_divider {
 #define CLK_DIVIDER_ROUND_CLOSEST	BIT(4)
 #define CLK_DIVIDER_READ_ONLY		BIT(5)
 #define CLK_DIVIDER_MAX_AT_ZERO		BIT(6)
+#define CLK_DIVIDER_BIG_ENDIAN		BIT(7)
 
 extern const struct clk_ops clk_divider_ops;
 extern const struct clk_ops clk_divider_ro_ops;
@@ -499,8 +527,13 @@ void clk_hw_unregister_divider(struct clk_hw *hw);
  *	register, and mask of mux bits are in higher 16-bit of this register.
  *	While setting the mux bits, higher 16-bit should also be updated to
  *	indicate changing mux bits.
+ * CLK_MUX_READ_ONLY - The mux registers can't be written, only read in the
+ * 	.get_parent clk_op.
  * CLK_MUX_ROUND_CLOSEST - Use the parent rate that is closest to the desired
  *	frequency.
+ * CLK_MUX_BIG_ENDIAN - By default little endian register accesses are used for
+ *	the mux register.  Setting this flag makes the register accesses big
+ *	endian.
  */
 struct clk_mux {
 	struct clk_hw	hw;
@@ -519,6 +552,7 @@ struct clk_mux {
 #define CLK_MUX_HIWORD_MASK		BIT(2)
 #define CLK_MUX_READ_ONLY		BIT(3) /* mux can't be changed */
 #define CLK_MUX_ROUND_CLOSEST		BIT(4)
+#define CLK_MUX_BIG_ENDIAN		BIT(5)
 
 extern const struct clk_ops clk_mux_ops;
 extern const struct clk_ops clk_mux_ro_ops;
@@ -602,6 +636,9 @@ void clk_hw_unregister_fixed_factor(struct clk_hw *hw);
  *	is the value read from the register. If CLK_FRAC_DIVIDER_ZERO_BASED
  *	is set then the numerator and denominator are both the value read
  *	plus one.
+ * CLK_FRAC_DIVIDER_BIG_ENDIAN - By default little endian register accesses are
+ *	used for the divider register.  Setting this flag makes the register
+ *	accesses big endian.
  */
 struct clk_fractional_divider {
 	struct clk_hw	hw;
@@ -622,6 +659,7 @@ struct clk_fractional_divider {
 #define to_clk_fd(_hw) container_of(_hw, struct clk_fractional_divider, hw)
 
 #define CLK_FRAC_DIVIDER_ZERO_BASED		BIT(0)
+#define CLK_FRAC_DIVIDER_BIG_ENDIAN		BIT(1)
 
 extern const struct clk_ops clk_fractional_divider_ops;
 struct clk *clk_register_fractional_divider(struct device *dev,
@@ -654,6 +692,9 @@ void clk_hw_unregister_fractional_divider(struct clk_hw *hw);
  *	leaving the parent rate unmodified.
  * CLK_MULTIPLIER_ROUND_CLOSEST - Makes the best calculated divider to be
  *	rounded to the closest integer instead of the down one.
+ * CLK_MULTIPLIER_BIG_ENDIAN - By default little endian register accesses are
+ *	used for the multiplier register.  Setting this flag makes the register
+ *	accesses big endian.
  */
 struct clk_multiplier {
 	struct clk_hw	hw;
@@ -668,6 +709,7 @@ struct clk_multiplier {
 
 #define CLK_MULTIPLIER_ZERO_BYPASS		BIT(0)
 #define CLK_MULTIPLIER_ROUND_CLOSEST	BIT(1)
+#define CLK_MULTIPLIER_BIG_ENDIAN		BIT(2)
 
 extern const struct clk_ops clk_multiplier_ops;
 
@@ -712,16 +754,19 @@ struct clk_hw *clk_hw_register_composite(struct device *dev, const char *name,
 		unsigned long flags);
 void clk_hw_unregister_composite(struct clk_hw *hw);
 
-/***
- * struct clk_gpio_gate - gpio gated clock
+/**
+ * struct clk_gpio - gpio gated clock
  *
  * @hw:		handle between common and hardware-specific interfaces
  * @gpiod:	gpio descriptor
  *
- * Clock with a gpio control for enabling and disabling the parent clock.
- * Implements .enable, .disable and .is_enabled
+ * Clock with a gpio control for enabling and disabling the parent clock
+ * or switching between two parents by asserting or deasserting the gpio.
+ *
+ * Implements .enable, .disable and .is_enabled or
+ * .get_parent, .set_parent and .determine_rate depending on which clk_ops
+ * is used.
  */
-
 struct clk_gpio {
 	struct clk_hw	hw;
 	struct gpio_desc *gpiod;
@@ -738,16 +783,6 @@ struct clk_hw *clk_hw_register_gpio_gate(struct device *dev, const char *name,
 		unsigned long flags);
 void clk_hw_unregister_gpio_gate(struct clk_hw *hw);
 
-/**
- * struct clk_gpio_mux - gpio controlled clock multiplexer
- *
- * @hw:		see struct clk_gpio
- * @gpiod:	gpio descriptor to select the parent of this clock multiplexer
- *
- * Clock with a gpio control for selecting the parent clock.
- * Implements .get_parent, .set_parent and .determine_rate
- */
-
 extern const struct clk_ops clk_gpio_mux_ops;
 struct clk *clk_register_gpio_mux(struct device *dev, const char *name,
 		const char * const *parent_names, u8 num_parents, struct gpio_desc *gpiod,
@@ -757,22 +792,12 @@ struct clk_hw *clk_hw_register_gpio_mux(struct device *dev, const char *name,
 		unsigned long flags);
 void clk_hw_unregister_gpio_mux(struct clk_hw *hw);
 
-/**
- * clk_register - allocate a new clock, register it and return an opaque cookie
- * @dev: device that is registering this clock
- * @hw: link to hardware-specific clock data
- *
- * clk_register is the primary interface for populating the clock tree with new
- * clock nodes.  It returns a pointer to the newly allocated struct clk which
- * cannot be dereferenced by driver code but may be used in conjuction with the
- * rest of the clock API.  In the event of an error clk_register will return an
- * error code; drivers must test for an error code after calling clk_register.
- */
 struct clk *clk_register(struct device *dev, struct clk_hw *hw);
 struct clk *devm_clk_register(struct device *dev, struct clk_hw *hw);
 
 int __must_check clk_hw_register(struct device *dev, struct clk_hw *hw);
 int __must_check devm_clk_hw_register(struct device *dev, struct clk_hw *hw);
+int __must_check of_clk_hw_register(struct device_node *node, struct clk_hw *hw);
 
 void clk_unregister(struct clk *clk);
 void devm_clk_unregister(struct device *dev, struct clk *clk);
@@ -993,37 +1018,6 @@ static inline int of_clk_detect_critical(struct device_node *np, int index,
 }
 #endif /* CONFIG_OF */
 
-/*
- * wrap access to peripherals in accessor routines
- * for improved portability across platforms
- */
-
-#if IS_ENABLED(CONFIG_PPC)
-
-static inline u32 clk_readl(u32 __iomem *reg)
-{
-	return ioread32be(reg);
-}
-
-static inline void clk_writel(u32 val, u32 __iomem *reg)
-{
-	iowrite32be(val, reg);
-}
-
-#else	/* platform dependent I/O accessors */
-
-static inline u32 clk_readl(u32 __iomem *reg)
-{
-	return readl(reg);
-}
-
-static inline void clk_writel(u32 val, u32 __iomem *reg)
-{
-	writel(val, reg);
-}
-
-#endif	/* platform dependent I/O accessors */
-
 void clk_gate_restore_context(struct clk_hw *hw);
 
 #endif /* CONFIG_COMMON_CLK */
diff --git a/include/linux/clk/analogbits-wrpll-cln28hpc.h b/include/linux/clk/analogbits-wrpll-cln28hpc.h
new file mode 100644
index 000000000000..03279097e138
--- /dev/null
+++ b/include/linux/clk/analogbits-wrpll-cln28hpc.h
@@ -0,0 +1,79 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2018-2019 SiFive, Inc.
+ * Wesley Terpstra
+ * Paul Walmsley
+ */
+
+#ifndef __LINUX_CLK_ANALOGBITS_WRPLL_CLN28HPC_H
+#define __LINUX_CLK_ANALOGBITS_WRPLL_CLN28HPC_H
+
+#include <linux/types.h>
+
+/* DIVQ_VALUES: number of valid DIVQ values */
+#define DIVQ_VALUES				6
+
+/*
+ * Bit definitions for struct wrpll_cfg.flags
+ *
+ * WRPLL_FLAGS_BYPASS_FLAG: if set, the PLL is either in bypass, or should be
+ *	programmed to enter bypass
+ * WRPLL_FLAGS_RESET_FLAG: if set, the PLL is in reset
+ * WRPLL_FLAGS_INT_FEEDBACK_FLAG: if set, the PLL is configured for internal
+ *	feedback mode
+ * WRPLL_FLAGS_EXT_FEEDBACK_FLAG: if set, the PLL is configured for external
+ *	feedback mode (not yet supported by this driver)
+ */
+#define WRPLL_FLAGS_BYPASS_SHIFT		0
+#define WRPLL_FLAGS_BYPASS_MASK		BIT(WRPLL_FLAGS_BYPASS_SHIFT)
+#define WRPLL_FLAGS_RESET_SHIFT		1
+#define WRPLL_FLAGS_RESET_MASK		BIT(WRPLL_FLAGS_RESET_SHIFT)
+#define WRPLL_FLAGS_INT_FEEDBACK_SHIFT	2
+#define WRPLL_FLAGS_INT_FEEDBACK_MASK	BIT(WRPLL_FLAGS_INT_FEEDBACK_SHIFT)
+#define WRPLL_FLAGS_EXT_FEEDBACK_SHIFT	3
+#define WRPLL_FLAGS_EXT_FEEDBACK_MASK	BIT(WRPLL_FLAGS_EXT_FEEDBACK_SHIFT)
+
+/**
+ * struct wrpll_cfg - WRPLL configuration values
+ * @divr: reference divider value (6 bits), as presented to the PLL signals
+ * @divf: feedback divider value (9 bits), as presented to the PLL signals
+ * @divq: output divider value (3 bits), as presented to the PLL signals
+ * @flags: PLL configuration flags.  See above for more information
+ * @range: PLL loop filter range.  See below for more information
+ * @output_rate_cache: cached output rates, swept across DIVQ
+ * @parent_rate: PLL refclk rate for which values are valid
+ * @max_r: maximum possible R divider value, given @parent_rate
+ * @init_r: initial R divider value to start the search from
+ *
+ * @divr, @divq, @divq, @range represent what the PLL expects to see
+ * on its input signals.  Thus @divr and @divf are the actual divisors
+ * minus one.  @divq is a power-of-two divider; for example, 1 =
+ * divide-by-2 and 6 = divide-by-64.  0 is an invalid @divq value.
+ *
+ * When initially passing a struct wrpll_cfg record, the
+ * record should be zero-initialized with the exception of the @flags
+ * field.  The only flag bits that need to be set are either
+ * WRPLL_FLAGS_INT_FEEDBACK or WRPLL_FLAGS_EXT_FEEDBACK.
+ */
+struct wrpll_cfg {
+	u8 divr;
+	u8 divq;
+	u8 range;
+	u8 flags;
+	u16 divf;
+/* private: */
+	u32 output_rate_cache[DIVQ_VALUES];
+	unsigned long parent_rate;
+	u8 max_r;
+	u8 init_r;
+};
+
+int wrpll_configure_for_rate(struct wrpll_cfg *c, u32 target_rate,
+			     unsigned long parent_rate);
+
+unsigned int wrpll_calc_max_lock_us(const struct wrpll_cfg *c);
+
+unsigned long wrpll_calc_output_rate(const struct wrpll_cfg *c,
+				     unsigned long parent_rate);
+
+#endif /* __LINUX_CLK_ANALOGBITS_WRPLL_CLN28HPC_H */
diff --git a/include/linux/clk/at91_pmc.h b/include/linux/clk/at91_pmc.h
index 931ab05f771d..0c53f26ae3d3 100644
--- a/include/linux/clk/at91_pmc.h
+++ b/include/linux/clk/at91_pmc.h
@@ -74,6 +74,8 @@
 #define			AT91_PMC_USBDIV_4		(2 << 28)
 #define		AT91_PMC_USB96M		(1     << 28)		/* Divider by 2 Enable (PLLB only) */
 
+#define AT91_PMC_CPU_CKR	0x28			/* CPU Clock Register */
+
 #define	AT91_PMC_MCKR		0x30			/* Master Clock Register */
 #define		AT91_PMC_CSS		(3 <<  0)		/* Master Clock Selection */
 #define			AT91_PMC_CSS_SLOW		(0 << 0)
@@ -187,16 +189,8 @@
 
 #define AT91_PMC_PCR		0x10c			/* Peripheral Control Register [some SAM9 and SAMA5] */
 #define		AT91_PMC_PCR_PID_MASK		0x3f
-#define		AT91_PMC_PCR_GCKCSS_OFFSET	8
-#define		AT91_PMC_PCR_GCKCSS_MASK	(0x7  << AT91_PMC_PCR_GCKCSS_OFFSET)
-#define		AT91_PMC_PCR_GCKCSS(n)		((n)  << AT91_PMC_PCR_GCKCSS_OFFSET)	/* GCK Clock Source Selection */
 #define		AT91_PMC_PCR_CMD		(0x1  <<  12)				/* Command (read=0, write=1) */
-#define		AT91_PMC_PCR_DIV_OFFSET		16
-#define		AT91_PMC_PCR_DIV_MASK		(0x3  << AT91_PMC_PCR_DIV_OFFSET)
-#define		AT91_PMC_PCR_DIV(n)		((n)  << AT91_PMC_PCR_DIV_OFFSET)	/* Divisor Value */
-#define		AT91_PMC_PCR_GCKDIV_OFFSET	20
-#define		AT91_PMC_PCR_GCKDIV_MASK	(0xff  << AT91_PMC_PCR_GCKDIV_OFFSET)
-#define		AT91_PMC_PCR_GCKDIV(n)		((n)  << AT91_PMC_PCR_GCKDIV_OFFSET)	/* Generated Clock Divisor Value */
+#define		AT91_PMC_PCR_GCKDIV_MASK	GENMASK(27, 20)
 #define		AT91_PMC_PCR_EN			(0x1  <<  28)				/* Enable */
 #define		AT91_PMC_PCR_GCKEN		(0x1  <<  29)				/* GCK Enable */
 
diff --git a/include/linux/clk/ti.h b/include/linux/clk/ti.h
index 78872efc7be0..1e8ef96555ce 100644
--- a/include/linux/clk/ti.h
+++ b/include/linux/clk/ti.h
@@ -243,6 +243,7 @@ struct ti_clk_ll_ops {
 
 #define to_clk_hw_omap(_hw) container_of(_hw, struct clk_hw_omap, hw)
 
+bool omap2_clk_is_hw_omap(struct clk_hw *hw);
 int omap2_clk_disable_autoidle_all(void);
 int omap2_clk_enable_autoidle_all(void);
 int omap2_clk_allow_idle(struct clk *clk);
@@ -293,6 +294,7 @@ struct ti_clk_features {
 #define TI_CLK_DISABLE_CLKDM_CONTROL		BIT(2)
 #define TI_CLK_ERRATA_I810			BIT(3)
 #define TI_CLK_CLKCTRL_COMPAT			BIT(4)
+#define TI_CLK_DEVICE_TYPE_GP			BIT(5)
 
 void ti_clk_setup_features(struct ti_clk_features *features);
 const struct ti_clk_features *ti_clk_get_features(void);
diff --git a/include/linux/device.h b/include/linux/device.h
index 4457e560bc2b..e85264fb6616 100644
--- a/include/linux/device.h
+++ b/include/linux/device.h
@@ -1229,7 +1229,7 @@ static inline void device_lock_assert(struct device *dev)
 
 static inline struct device_node *dev_of_node(struct device *dev)
 {
-	if (!IS_ENABLED(CONFIG_OF))
+	if (!IS_ENABLED(CONFIG_OF) || !dev)
 		return NULL;
 	return dev->of_node;
 }
diff --git a/include/linux/dma-fence-chain.h b/include/linux/dma-fence-chain.h
new file mode 100644
index 000000000000..934a442db8ac
--- /dev/null
+++ b/include/linux/dma-fence-chain.h
@@ -0,0 +1,81 @@
+/*
+ * fence-chain: chain fences together in a timeline
+ *
+ * Copyright (C) 2018 Advanced Micro Devices, Inc.
+ * Authors:
+ *	Christian König <christian.koenig@amd.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#ifndef __LINUX_DMA_FENCE_CHAIN_H
+#define __LINUX_DMA_FENCE_CHAIN_H
+
+#include <linux/dma-fence.h>
+#include <linux/irq_work.h>
+
+/**
+ * struct dma_fence_chain - fence to represent an node of a fence chain
+ * @base: fence base class
+ * @lock: spinlock for fence handling
+ * @prev: previous fence of the chain
+ * @prev_seqno: original previous seqno before garbage collection
+ * @fence: encapsulated fence
+ * @cb: callback structure for signaling
+ * @work: irq work item for signaling
+ */
+struct dma_fence_chain {
+	struct dma_fence base;
+	spinlock_t lock;
+	struct dma_fence __rcu *prev;
+	u64 prev_seqno;
+	struct dma_fence *fence;
+	struct dma_fence_cb cb;
+	struct irq_work work;
+};
+
+extern const struct dma_fence_ops dma_fence_chain_ops;
+
+/**
+ * to_dma_fence_chain - cast a fence to a dma_fence_chain
+ * @fence: fence to cast to a dma_fence_array
+ *
+ * Returns NULL if the fence is not a dma_fence_chain,
+ * or the dma_fence_chain otherwise.
+ */
+static inline struct dma_fence_chain *
+to_dma_fence_chain(struct dma_fence *fence)
+{
+	if (!fence || fence->ops != &dma_fence_chain_ops)
+		return NULL;
+
+	return container_of(fence, struct dma_fence_chain, base);
+}
+
+/**
+ * dma_fence_chain_for_each - iterate over all fences in chain
+ * @iter: current fence
+ * @head: starting point
+ *
+ * Iterate over all fences in the chain. We keep a reference to the current
+ * fence while inside the loop which must be dropped when breaking out.
+ */
+#define dma_fence_chain_for_each(iter, head)	\
+	for (iter = dma_fence_get(head); iter; \
+	     iter = dma_fence_chain_walk(iter))
+
+struct dma_fence *dma_fence_chain_walk(struct dma_fence *fence);
+int dma_fence_chain_find_seqno(struct dma_fence **pfence, uint64_t seqno);
+void dma_fence_chain_init(struct dma_fence_chain *chain,
+			  struct dma_fence *prev,
+			  struct dma_fence *fence,
+			  uint64_t seqno);
+
+#endif /* __LINUX_DMA_FENCE_CHAIN_H */
diff --git a/include/linux/dma-fence.h b/include/linux/dma-fence.h
index 6b788467b2e3..974717d6ac0c 100644
--- a/include/linux/dma-fence.h
+++ b/include/linux/dma-fence.h
@@ -112,6 +112,14 @@ struct dma_fence_cb {
  */
 struct dma_fence_ops {
 	/**
+	 * @use_64bit_seqno:
+	 *
+	 * True if this dma_fence implementation uses 64bit seqno, false
+	 * otherwise.
+	 */
+	bool use_64bit_seqno;
+
+	/**
 	 * @get_driver_name:
 	 *
 	 * Returns the driver name. This is a callback to allow drivers to
@@ -410,18 +418,19 @@ dma_fence_is_signaled(struct dma_fence *fence)
  * __dma_fence_is_later - return if f1 is chronologically later than f2
  * @f1: the first fence's seqno
  * @f2: the second fence's seqno from the same context
+ * @ops: dma_fence_ops associated with the seqno
  *
  * Returns true if f1 is chronologically later than f2. Both fences must be
  * from the same context, since a seqno is not common across contexts.
  */
-static inline bool __dma_fence_is_later(u64 f1, u64 f2)
+static inline bool __dma_fence_is_later(u64 f1, u64 f2,
+					const struct dma_fence_ops *ops)
 {
 	/* This is for backward compatibility with drivers which can only handle
-	 * 32bit sequence numbers. Use a 64bit compare when any of the higher
-	 * bits are none zero, otherwise use a 32bit compare with wrap around
-	 * handling.
+	 * 32bit sequence numbers. Use a 64bit compare when the driver says to
+	 * do so.
 	 */
-	if (upper_32_bits(f1) || upper_32_bits(f2))
+	if (ops->use_64bit_seqno)
 		return f1 > f2;
 
 	return (int)(lower_32_bits(f1) - lower_32_bits(f2)) > 0;
@@ -441,7 +450,7 @@ static inline bool dma_fence_is_later(struct dma_fence *f1,
 	if (WARN_ON(f1->context != f2->context))
 		return false;
 
-	return __dma_fence_is_later(f1->seqno, f2->seqno);
+	return __dma_fence_is_later(f1->seqno, f2->seqno, f1->ops);
 }
 
 /**
diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h
index 75e60be91e5f..6309a721394b 100644
--- a/include/linux/dma-mapping.h
+++ b/include/linux/dma-mapping.h
@@ -267,9 +267,9 @@ size_t dma_direct_max_mapping_size(struct device *dev);
 
 static inline const struct dma_map_ops *get_dma_ops(struct device *dev)
 {
-	if (dev && dev->dma_ops)
+	if (dev->dma_ops)
 		return dev->dma_ops;
-	return get_arch_dma_ops(dev ? dev->bus : NULL);
+	return get_arch_dma_ops(dev->bus);
 }
 
 static inline void set_dma_ops(struct device *dev,
@@ -650,7 +650,7 @@ static inline void dma_free_coherent(struct device *dev, size_t size,
 
 static inline u64 dma_get_mask(struct device *dev)
 {
-	if (dev && dev->dma_mask && *dev->dma_mask)
+	if (dev->dma_mask && *dev->dma_mask)
 		return *dev->dma_mask;
 	return DMA_BIT_MASK(32);
 }
diff --git a/include/linux/dma-noncoherent.h b/include/linux/dma-noncoherent.h
index 69b36ed31a99..9741767e400f 100644
--- a/include/linux/dma-noncoherent.h
+++ b/include/linux/dma-noncoherent.h
@@ -72,6 +72,12 @@ static inline void arch_sync_dma_for_cpu_all(struct device *dev)
 }
 #endif /* CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL */
 
+#ifdef CONFIG_ARCH_HAS_DMA_PREP_COHERENT
 void arch_dma_prep_coherent(struct page *page, size_t size);
+#else
+static inline void arch_dma_prep_coherent(struct page *page, size_t size)
+{
+}
+#endif /* CONFIG_ARCH_HAS_DMA_PREP_COHERENT */
 
 #endif /* _LINUX_DMA_NONCOHERENT_H */
diff --git a/include/linux/dma/idma64.h b/include/linux/dma/idma64.h
new file mode 100644
index 000000000000..621cfae60554
--- /dev/null
+++ b/include/linux/dma/idma64.h
@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Definitions for the Intel integrated DMA 64-bit
+ *
+ * Copyright (C) 2019 Intel Corporation
+ */
+
+#ifndef __LINUX_DMA_IDMA64_H__
+#define __LINUX_DMA_IDMA64_H__
+
+/* Platform driver name */
+#define LPSS_IDMA64_DRIVER_NAME		"idma64"
+
+#endif /* __LINUX_DMA_IDMA64_H__ */
diff --git a/include/linux/dynamic_debug.h b/include/linux/dynamic_debug.h
index c2be029b9b53..6c809440f319 100644
--- a/include/linux/dynamic_debug.h
+++ b/include/linux/dynamic_debug.h
@@ -71,6 +71,13 @@ void __dynamic_netdev_dbg(struct _ddebug *descriptor,
 			  const struct net_device *dev,
 			  const char *fmt, ...);
 
+struct ib_device;
+
+extern __printf(3, 4)
+void __dynamic_ibdev_dbg(struct _ddebug *descriptor,
+			 const struct ib_device *ibdev,
+			 const char *fmt, ...);
+
 #define DEFINE_DYNAMIC_DEBUG_METADATA(name, fmt)		\
 	static struct _ddebug  __aligned(8)			\
 	__attribute__((section("__verbose"))) name = {		\
@@ -154,6 +161,10 @@ void __dynamic_netdev_dbg(struct _ddebug *descriptor,
 	_dynamic_func_call(fmt, __dynamic_netdev_dbg,		\
 			   dev, fmt, ##__VA_ARGS__)
 
+#define dynamic_ibdev_dbg(dev, fmt, ...)			\
+	_dynamic_func_call(fmt, __dynamic_ibdev_dbg,		\
+			   dev, fmt, ##__VA_ARGS__)
+
 #define dynamic_hex_dump(prefix_str, prefix_type, rowsize,		\
 			 groupsize, buf, len, ascii)			\
 	_dynamic_func_call_no_desc(__builtin_constant_p(prefix_str) ? prefix_str : "hexdump", \
diff --git a/include/linux/gpio/driver.h b/include/linux/gpio/driver.h
index 01497910f023..951be1715c12 100644
--- a/include/linux/gpio/driver.h
+++ b/include/linux/gpio/driver.h
@@ -614,6 +614,9 @@ struct gpio_desc *gpiochip_request_own_desc(struct gpio_chip *chip, u16 hwnum,
 					    enum gpiod_flags flags);
 void gpiochip_free_own_desc(struct gpio_desc *desc);
 
+void devprop_gpiochip_set_names(struct gpio_chip *chip,
+				const struct fwnode_handle *fwnode);
+
 #else /* CONFIG_GPIOLIB */
 
 static inline struct gpio_chip *gpiod_to_chip(const struct gpio_desc *desc)
diff --git a/include/linux/i2c-algo-bit.h b/include/linux/i2c-algo-bit.h
index 69045df78e2d..7fd5575a368f 100644
--- a/include/linux/i2c-algo-bit.h
+++ b/include/linux/i2c-algo-bit.h
@@ -33,6 +33,7 @@ struct i2c_algo_bit_data {
 				   minimum 5 us for standard-mode I2C and SMBus,
 				   maximum 50 us for SMBus */
 	int timeout;		/* in jiffies */
+	bool can_do_atomic;	/* callbacks don't sleep, we can be atomic */
 };
 
 int i2c_bit_add_bus(struct i2c_adapter *);
diff --git a/include/linux/i2c.h b/include/linux/i2c.h
index 383510b4f083..be27062f8ed1 100644
--- a/include/linux/i2c.h
+++ b/include/linux/i2c.h
@@ -499,9 +499,13 @@ i2c_register_board_info(int busnum, struct i2c_board_info const *info,
  * @master_xfer: Issue a set of i2c transactions to the given I2C adapter
  *   defined by the msgs array, with num messages available to transfer via
  *   the adapter specified by adap.
+ * @master_xfer_atomic: same as @master_xfer. Yet, only using atomic context
+ *   so e.g. PMICs can be accessed very late before shutdown. Optional.
  * @smbus_xfer: Issue smbus transactions to the given I2C adapter. If this
  *   is not present, then the bus layer will try and convert the SMBus calls
  *   into I2C transfers instead.
+ * @smbus_xfer_atomic: same as @smbus_xfer. Yet, only using atomic context
+ *   so e.g. PMICs can be accessed very late before shutdown. Optional.
  * @functionality: Return the flags that this algorithm/adapter pair supports
  *   from the I2C_FUNC_* flags.
  * @reg_slave: Register given client to I2C slave mode of this adapter
@@ -512,25 +516,33 @@ i2c_register_board_info(int busnum, struct i2c_board_info const *info,
  * be addressed using the same bus algorithms - i.e. bit-banging or the PCF8584
  * to name two of the most common.
  *
- * The return codes from the @master_xfer field should indicate the type of
- * error code that occurred during the transfer, as documented in the kernel
- * Documentation file Documentation/i2c/fault-codes.
+ * The return codes from the @master_xfer{_atomic} fields should indicate the
+ * type of error code that occurred during the transfer, as documented in the
+ * Kernel Documentation file Documentation/i2c/fault-codes.
  */
 struct i2c_algorithm {
-	/* If an adapter algorithm can't do I2C-level access, set master_xfer
-	   to NULL. If an adapter algorithm can do SMBus access, set
-	   smbus_xfer. If set to NULL, the SMBus protocol is simulated
-	   using common I2C messages */
-	/* master_xfer should return the number of messages successfully
-	   processed, or a negative value on error */
+	/*
+	 * If an adapter algorithm can't do I2C-level access, set master_xfer
+	 * to NULL. If an adapter algorithm can do SMBus access, set
+	 * smbus_xfer. If set to NULL, the SMBus protocol is simulated
+	 * using common I2C messages.
+	 *
+	 * master_xfer should return the number of messages successfully
+	 * processed, or a negative value on error
+	 */
 	int (*master_xfer)(struct i2c_adapter *adap, struct i2c_msg *msgs,
 			   int num);
-	int (*smbus_xfer) (struct i2c_adapter *adap, u16 addr,
-			   unsigned short flags, char read_write,
-			   u8 command, int size, union i2c_smbus_data *data);
+	int (*master_xfer_atomic)(struct i2c_adapter *adap,
+				   struct i2c_msg *msgs, int num);
+	int (*smbus_xfer)(struct i2c_adapter *adap, u16 addr,
+			  unsigned short flags, char read_write,
+			  u8 command, int size, union i2c_smbus_data *data);
+	int (*smbus_xfer_atomic)(struct i2c_adapter *adap, u16 addr,
+				 unsigned short flags, char read_write,
+				 u8 command, int size, union i2c_smbus_data *data);
 
 	/* To determine what the adapter supports */
-	u32 (*functionality) (struct i2c_adapter *);
+	u32 (*functionality)(struct i2c_adapter *adap);
 
 #if IS_ENABLED(CONFIG_I2C_SLAVE)
 	int (*reg_slave)(struct i2c_client *client);
@@ -682,7 +694,8 @@ struct i2c_adapter {
 	int retries;
 	struct device dev;		/* the adapter device */
 	unsigned long locked_flags;	/* owned by the I2C core */
-#define I2C_ALF_IS_SUSPENDED	0
+#define I2C_ALF_IS_SUSPENDED		0
+#define I2C_ALF_SUSPEND_REPORTED	1
 
 	int nr;
 	char name[48];
diff --git a/include/linux/io-pgtable.h b/include/linux/io-pgtable.h
index 47d5ae559329..76969a564831 100644
--- a/include/linux/io-pgtable.h
+++ b/include/linux/io-pgtable.h
@@ -12,6 +12,7 @@ enum io_pgtable_fmt {
 	ARM_64_LPAE_S1,
 	ARM_64_LPAE_S2,
 	ARM_V7S,
+	ARM_MALI_LPAE,
 	IO_PGTABLE_NUM_FMTS,
 };
 
@@ -108,6 +109,11 @@ struct io_pgtable_cfg {
 			u32	nmrr;
 			u32	prrr;
 		} arm_v7s_cfg;
+
+		struct {
+			u64	transtab;
+			u64	memattr;
+		} arm_mali_lpae_cfg;
 	};
 };
 
@@ -209,5 +215,6 @@ extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns;
 extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns;
 extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns;
 extern struct io_pgtable_init_fns io_pgtable_arm_v7s_init_fns;
+extern struct io_pgtable_init_fns io_pgtable_arm_mali_lpae_init_fns;
 
 #endif /* __IO_PGTABLE_H */
diff --git a/include/linux/lockd/bind.h b/include/linux/lockd/bind.h
index 053a4ef3d431..8c0cf1059443 100644
--- a/include/linux/lockd/bind.h
+++ b/include/linux/lockd/bind.h
@@ -46,6 +46,7 @@ struct nlmclnt_initdata {
 	int			noresvport;
 	struct net		*net;
 	const struct nlmclnt_operations	*nlmclnt_ops;
+	const struct cred	*cred;
 };
 
 /*
diff --git a/include/linux/lockd/lockd.h b/include/linux/lockd/lockd.h
index b065ef406770..c9b422dde542 100644
--- a/include/linux/lockd/lockd.h
+++ b/include/linux/lockd/lockd.h
@@ -70,6 +70,7 @@ struct nlm_host {
 	struct nsm_handle	*h_nsmhandle;	/* NSM status handle */
 	char			*h_addrbuf;	/* address eyecatcher */
 	struct net		*net;		/* host net */
+	const struct cred	*h_cred;
 	char			nodename[UNX_MAXNODENAME + 1];
 	const struct nlmclnt_operations	*h_nlmclnt_ops;	/* Callback ops for NLM users */
 };
@@ -229,7 +230,8 @@ struct nlm_host  *nlmclnt_lookup_host(const struct sockaddr *sap,
 					const u32 version,
 					const char *hostname,
 					int noresvport,
-					struct net *net);
+					struct net *net,
+					const struct cred *cred);
 void		  nlmclnt_release_host(struct nlm_host *);
 struct nlm_host  *nlmsvc_lookup_host(const struct svc_rqst *rqstp,
 					const char *hostname,
diff --git a/include/linux/math64.h b/include/linux/math64.h
index bb2c84afb80c..65bef21cdddb 100644
--- a/include/linux/math64.h
+++ b/include/linux/math64.h
@@ -284,4 +284,17 @@ static inline u64 mul_u64_u32_div(u64 a, u32 mul, u32 divisor)
 #define DIV64_U64_ROUND_UP(ll, d)	\
 	({ u64 _tmp = (d); div64_u64((ll) + _tmp - 1, _tmp); })
 
+/**
+ * DIV64_U64_ROUND_CLOSEST - unsigned 64bit divide with 64bit divisor rounded to nearest integer
+ * @dividend: unsigned 64bit dividend
+ * @divisor: unsigned 64bit divisor
+ *
+ * Divide unsigned 64bit dividend by unsigned 64bit divisor
+ * and round to closest integer.
+ *
+ * Return: dividend / divisor rounded to nearest integer
+ */
+#define DIV64_U64_ROUND_CLOSEST(dividend, divisor)	\
+	({ u64 _tmp = (divisor); div64_u64((dividend) + _tmp / 2, _tmp); })
+
 #endif /* _LINUX_MATH64_H */
diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h
index 5a39b323c52e..5a27246db883 100644
--- a/include/linux/mlx5/driver.h
+++ b/include/linux/mlx5/driver.h
@@ -689,7 +689,6 @@ struct mlx5_core_dev {
 #endif
 	struct mlx5_clock        clock;
 	struct mlx5_ib_clock_info  *clock_info;
-	struct page             *clock_info_page;
 	struct mlx5_fw_tracer   *tracer;
 };
 
diff --git a/include/linux/module.h b/include/linux/module.h
index 7dc4dc79b634..8f75277d4cef 100644
--- a/include/linux/module.h
+++ b/include/linux/module.h
@@ -253,6 +253,7 @@ extern typeof(name) __mod_##type##__##name##_device_table		\
 #define MODULE_VERSION(_version) MODULE_INFO(version, _version)
 #else
 #define MODULE_VERSION(_version)					\
+	MODULE_INFO(version, _version);					\
 	static struct module_version_attribute ___modver_attr = {	\
 		.mattr	= {						\
 			.attr	= {					\
@@ -676,6 +677,7 @@ static inline bool is_livepatch_module(struct module *mod)
 #endif /* CONFIG_LIVEPATCH */
 
 bool is_module_sig_enforced(void);
+void set_module_sig_enforced(void);
 
 #else /* !CONFIG_MODULES... */
 
@@ -802,6 +804,10 @@ static inline bool is_module_sig_enforced(void)
 	return false;
 }
 
+static inline void set_module_sig_enforced(void)
+{
+}
+
 /* Dereference module function descriptor */
 static inline
 void *dereference_module_function_descriptor(struct module *mod, void *ptr)
diff --git a/include/linux/moduleparam.h b/include/linux/moduleparam.h
index ba36506db4fb..5ba250d9172a 100644
--- a/include/linux/moduleparam.h
+++ b/include/linux/moduleparam.h
@@ -10,23 +10,21 @@
    module name. */
 #ifdef MODULE
 #define MODULE_PARAM_PREFIX /* empty */
+#define __MODULE_INFO_PREFIX /* empty */
 #else
 #define MODULE_PARAM_PREFIX KBUILD_MODNAME "."
+/* We cannot use MODULE_PARAM_PREFIX because some modules override it. */
+#define __MODULE_INFO_PREFIX KBUILD_MODNAME "."
 #endif
 
 /* Chosen so that structs with an unsigned long line up. */
 #define MAX_PARAM_PREFIX_LEN (64 - sizeof(unsigned long))
 
-#ifdef MODULE
 #define __MODULE_INFO(tag, name, info)					  \
 static const char __UNIQUE_ID(name)[]					  \
   __used __attribute__((section(".modinfo"), unused, aligned(1)))	  \
-  = __stringify(tag) "=" info
-#else  /* !MODULE */
-/* This struct is here for syntactic coherency, it is not used */
-#define __MODULE_INFO(tag, name, info)					  \
-  struct __UNIQUE_ID(name) {}
-#endif
+  = __MODULE_INFO_PREFIX __stringify(tag) "=" info
+
 #define __MODULE_PARM_TYPE(name, _type)					  \
   __MODULE_INFO(parmtype, name##type, #name ":" _type)
 
diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h
index 40e30376130b..d363d5765cdf 100644
--- a/include/linux/nfs_fs.h
+++ b/include/linux/nfs_fs.h
@@ -76,7 +76,6 @@ struct nfs_open_context {
 	fmode_t mode;
 
 	unsigned long flags;
-#define NFS_CONTEXT_ERROR_WRITE		(0)
 #define NFS_CONTEXT_RESEND_WRITES	(1)
 #define NFS_CONTEXT_BAD			(2)
 #define NFS_CONTEXT_UNLOCK	(3)
diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h
index c827d31298cc..1e78032a174b 100644
--- a/include/linux/nfs_fs_sb.h
+++ b/include/linux/nfs_fs_sb.h
@@ -139,6 +139,16 @@ struct nfs_server {
 	struct nfs_iostats __percpu *io_stats;	/* I/O statistics */
 	atomic_long_t		writeback;	/* number of writeback pages */
 	int			flags;		/* various flags */
+
+/* The following are for internal use only. Also see uapi/linux/nfs_mount.h */
+#define NFS_MOUNT_LOOKUP_CACHE_NONEG	0x10000
+#define NFS_MOUNT_LOOKUP_CACHE_NONE	0x20000
+#define NFS_MOUNT_NORESVPORT		0x40000
+#define NFS_MOUNT_LEGACY_INTERFACE	0x80000
+#define NFS_MOUNT_LOCAL_FLOCK		0x100000
+#define NFS_MOUNT_LOCAL_FCNTL		0x200000
+#define NFS_MOUNT_SOFTERR		0x400000
+
 	unsigned int		caps;		/* server capabilities */
 	unsigned int		rsize;		/* read size */
 	unsigned int		rpages;		/* read size (in pages) */
@@ -231,6 +241,9 @@ struct nfs_server {
 
 	/* XDR related information */
 	unsigned int		read_hdrsize;
+
+	/* User namespace info */
+	const struct cred	*cred;
 };
 
 /* Server capabilities */
diff --git a/include/linux/nfs_page.h b/include/linux/nfs_page.h
index ad69430fd0eb..0bbd587fac6a 100644
--- a/include/linux/nfs_page.h
+++ b/include/linux/nfs_page.h
@@ -42,7 +42,6 @@ struct nfs_inode;
 struct nfs_page {
 	struct list_head	wb_list;	/* Defines state of page: */
 	struct page		*wb_page;	/* page to read in/write out */
-	struct nfs_open_context	*wb_context;	/* File state context info */
 	struct nfs_lock_context	*wb_lock_context;	/* lock context info */
 	pgoff_t			wb_index;	/* Offset >> PAGE_SHIFT */
 	unsigned int		wb_offset,	/* Offset & ~PAGE_MASK */
@@ -53,6 +52,7 @@ struct nfs_page {
 	struct nfs_write_verifier	wb_verf;	/* Commit cookie */
 	struct nfs_page		*wb_this_page;  /* list of reqs for this page */
 	struct nfs_page		*wb_head;       /* head pointer for req list */
+	unsigned short		wb_nio;		/* Number of I/O attempts */
 };
 
 struct nfs_pageio_descriptor;
@@ -87,7 +87,6 @@ struct nfs_pgio_mirror {
 };
 
 struct nfs_pageio_descriptor {
-	unsigned char		pg_moreio : 1;
 	struct inode		*pg_inode;
 	const struct nfs_pageio_ops *pg_ops;
 	const struct nfs_rw_ops *pg_rw_ops;
@@ -105,6 +104,8 @@ struct nfs_pageio_descriptor {
 	struct nfs_pgio_mirror	pg_mirrors_static[1];
 	struct nfs_pgio_mirror	*pg_mirrors_dynamic;
 	u32			pg_mirror_idx;	/* current mirror */
+	unsigned short		pg_maxretrans;
+	unsigned char		pg_moreio : 1;
 };
 
 /* arbitrarily selected limit to number of mirrors */
@@ -114,7 +115,6 @@ struct nfs_pageio_descriptor {
 
 extern	struct nfs_page *nfs_create_request(struct nfs_open_context *ctx,
 					    struct page *page,
-					    struct nfs_page *last,
 					    unsigned int offset,
 					    unsigned int count);
 extern	void nfs_release_request(struct nfs_page *);
@@ -199,4 +199,10 @@ loff_t req_offset(struct nfs_page *req)
 	return (((loff_t)req->wb_index) << PAGE_SHIFT) + req->wb_offset;
 }
 
+static inline struct nfs_open_context *
+nfs_req_openctx(struct nfs_page *req)
+{
+	return req->wb_lock_context->open_context;
+}
+
 #endif /* _LINUX_NFS_PAGE_H */
diff --git a/include/linux/nvme-fc-driver.h b/include/linux/nvme-fc-driver.h
index 2bb349035431..c48e96436f56 100644
--- a/include/linux/nvme-fc-driver.h
+++ b/include/linux/nvme-fc-driver.h
@@ -17,12 +17,6 @@
 
 
 
-/* FC Port role bitmask - can merge with FC Port Roles in fc transport */
-#define FC_PORT_ROLE_NVME_INITIATOR	0x10
-#define FC_PORT_ROLE_NVME_TARGET	0x20
-#define FC_PORT_ROLE_NVME_DISCOVERY	0x40
-
-
 /**
  * struct nvme_fc_port_info - port-specific ids and FC connection-specific
  *                            data element used during NVME Host role
diff --git a/include/linux/overflow.h b/include/linux/overflow.h
index 40b48e2133cb..15eb85de9226 100644
--- a/include/linux/overflow.h
+++ b/include/linux/overflow.h
@@ -36,6 +36,12 @@
 #define type_max(T) ((T)((__type_half_max(T) - 1) + __type_half_max(T)))
 #define type_min(T) ((T)((T)-type_max(T)-(T)1))
 
+/*
+ * Avoids triggering -Wtype-limits compilation warning,
+ * while using unsigned data types to check a < 0.
+ */
+#define is_non_negative(a) ((a) > 0 || (a) == 0)
+#define is_negative(a) (!(is_non_negative(a)))
 
 #ifdef COMPILER_HAS_GENERIC_BUILTIN_OVERFLOW
 /*
@@ -227,10 +233,10 @@
 	typeof(d) _d = d;						\
 	u64 _a_full = _a;						\
 	unsigned int _to_shift =					\
-		_s >= 0 && _s < 8 * sizeof(*d) ? _s : 0;		\
+		is_non_negative(_s) && _s < 8 * sizeof(*d) ? _s : 0;	\
 	*_d = (_a_full << _to_shift);					\
-	(_to_shift != _s || *_d < 0 || _a < 0 ||			\
-		(*_d >> _to_shift) != _a);				\
+	(_to_shift != _s || is_negative(*_d) || is_negative(_a) ||	\
+	(*_d >> _to_shift) != _a);					\
 })
 
 /**
diff --git a/include/linux/phy/phy.h b/include/linux/phy/phy.h
index 3f350e2749fe..ef13aea1d370 100644
--- a/include/linux/phy/phy.h
+++ b/include/linux/phy/phy.h
@@ -64,6 +64,7 @@ union phy_configure_opts {
  * @set_mode: set the mode of the phy
  * @reset: resetting the phy
  * @calibrate: calibrate the phy
+ * @release: ops to be performed while the consumer relinquishes the PHY
  * @owner: the module owner containing the ops
  */
 struct phy_ops {
@@ -105,6 +106,7 @@ struct phy_ops {
 			    union phy_configure_opts *opts);
 	int	(*reset)(struct phy *phy);
 	int	(*calibrate)(struct phy *phy);
+	void	(*release)(struct phy *phy);
 	struct module *owner;
 };
 
diff --git a/include/linux/platform_data/pca954x.h b/include/linux/platform_data/pca954x.h
deleted file mode 100644
index 1712677d5904..000000000000
--- a/include/linux/platform_data/pca954x.h
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- *
- * pca954x.h - I2C multiplexer/switch support
- *
- * Copyright (c) 2008-2009 Rodolfo Giometti <giometti@linux.it>
- * Copyright (c) 2008-2009 Eurotech S.p.A. <info@eurotech.it>
- * Michael Lawnick <michael.lawnick.ext@nsn.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
- */
-
-
-#ifndef _LINUX_I2C_PCA954X_H
-#define _LINUX_I2C_PCA954X_H
-
-/* Platform data for the PCA954x I2C multiplexers */
-
-/* Per channel initialisation data:
- * @adap_id: bus number for the adapter. 0 = don't care
- * @deselect_on_exit: set this entry to 1, if your H/W needs deselection
- *                    of this channel after transaction.
- *
- */
-struct pca954x_platform_mode {
-	int		adap_id;
-	unsigned int	deselect_on_exit:1;
-	unsigned int	class;
-};
-
-/* Per mux/switch data, used with i2c_register_board_info */
-struct pca954x_platform_data {
-	struct pca954x_platform_mode *modes;
-	int num_modes;
-};
-
-#endif /* _LINUX_I2C_PCA954X_H */
diff --git a/include/linux/reservation.h b/include/linux/reservation.h
index 2f0ffca35780..ee750765cc94 100644
--- a/include/linux/reservation.h
+++ b/include/linux/reservation.h
@@ -228,7 +228,8 @@ reservation_object_unlock(struct reservation_object *obj)
  * @obj: the reservation object
  *
  * Returns the exclusive fence (if any).  Does NOT take a
- * reference.  The obj->lock must be held.
+ * reference. Writers must hold obj->lock, readers may only
+ * hold a RCU read side lock.
  *
  * RETURNS
  * The exclusive fence or NULL
diff --git a/include/linux/reset.h b/include/linux/reset.h
index c1901b61ca30..95d555c2130a 100644
--- a/include/linux/reset.h
+++ b/include/linux/reset.h
@@ -14,23 +14,26 @@ int reset_control_reset(struct reset_control *rstc);
 int reset_control_assert(struct reset_control *rstc);
 int reset_control_deassert(struct reset_control *rstc);
 int reset_control_status(struct reset_control *rstc);
+int reset_control_acquire(struct reset_control *rstc);
+void reset_control_release(struct reset_control *rstc);
 
 struct reset_control *__of_reset_control_get(struct device_node *node,
 				     const char *id, int index, bool shared,
-				     bool optional);
+				     bool optional, bool acquired);
 struct reset_control *__reset_control_get(struct device *dev, const char *id,
 					  int index, bool shared,
-					  bool optional);
+					  bool optional, bool acquired);
 void reset_control_put(struct reset_control *rstc);
 int __device_reset(struct device *dev, bool optional);
 struct reset_control *__devm_reset_control_get(struct device *dev,
 				     const char *id, int index, bool shared,
-				     bool optional);
+				     bool optional, bool acquired);
 
 struct reset_control *devm_reset_control_array_get(struct device *dev,
 						   bool shared, bool optional);
 struct reset_control *of_reset_control_array_get(struct device_node *np,
-						 bool shared, bool optional);
+						 bool shared, bool optional,
+						 bool acquired);
 
 int reset_control_get_count(struct device *dev);
 
@@ -56,6 +59,15 @@ static inline int reset_control_status(struct reset_control *rstc)
 	return 0;
 }
 
+static inline int reset_control_acquire(struct reset_control *rstc)
+{
+	return 0;
+}
+
+static inline void reset_control_release(struct reset_control *rstc)
+{
+}
+
 static inline void reset_control_put(struct reset_control *rstc)
 {
 }
@@ -68,21 +80,23 @@ static inline int __device_reset(struct device *dev, bool optional)
 static inline struct reset_control *__of_reset_control_get(
 					struct device_node *node,
 					const char *id, int index, bool shared,
-					bool optional)
+					bool optional, bool acquired)
 {
 	return optional ? NULL : ERR_PTR(-ENOTSUPP);
 }
 
 static inline struct reset_control *__reset_control_get(
 					struct device *dev, const char *id,
-					int index, bool shared, bool optional)
+					int index, bool shared, bool optional,
+					bool acquired)
 {
 	return optional ? NULL : ERR_PTR(-ENOTSUPP);
 }
 
 static inline struct reset_control *__devm_reset_control_get(
 					struct device *dev, const char *id,
-					int index, bool shared, bool optional)
+					int index, bool shared, bool optional,
+					bool acquired)
 {
 	return optional ? NULL : ERR_PTR(-ENOTSUPP);
 }
@@ -94,7 +108,8 @@ devm_reset_control_array_get(struct device *dev, bool shared, bool optional)
 }
 
 static inline struct reset_control *
-of_reset_control_array_get(struct device_node *np, bool shared, bool optional)
+of_reset_control_array_get(struct device_node *np, bool shared, bool optional,
+			   bool acquired)
 {
 	return optional ? NULL : ERR_PTR(-ENOTSUPP);
 }
@@ -134,7 +149,28 @@ static inline int device_reset_optional(struct device *dev)
 static inline struct reset_control *
 __must_check reset_control_get_exclusive(struct device *dev, const char *id)
 {
-	return __reset_control_get(dev, id, 0, false, false);
+	return __reset_control_get(dev, id, 0, false, false, true);
+}
+
+/**
+ * reset_control_get_exclusive_released - Lookup and obtain a temoprarily
+ *                                        exclusive reference to a reset
+ *                                        controller.
+ * @dev: device to be reset by the controller
+ * @id: reset line name
+ *
+ * Returns a struct reset_control or IS_ERR() condition containing errno.
+ * reset-controls returned by this function must be acquired via
+ * reset_control_acquire() before they can be used and should be released
+ * via reset_control_release() afterwards.
+ *
+ * Use of id names is optional.
+ */
+static inline struct reset_control *
+__must_check reset_control_get_exclusive_released(struct device *dev,
+						  const char *id)
+{
+	return __reset_control_get(dev, id, 0, false, false, false);
 }
 
 /**
@@ -162,19 +198,19 @@ __must_check reset_control_get_exclusive(struct device *dev, const char *id)
 static inline struct reset_control *reset_control_get_shared(
 					struct device *dev, const char *id)
 {
-	return __reset_control_get(dev, id, 0, true, false);
+	return __reset_control_get(dev, id, 0, true, false, false);
 }
 
 static inline struct reset_control *reset_control_get_optional_exclusive(
 					struct device *dev, const char *id)
 {
-	return __reset_control_get(dev, id, 0, false, true);
+	return __reset_control_get(dev, id, 0, false, true, true);
 }
 
 static inline struct reset_control *reset_control_get_optional_shared(
 					struct device *dev, const char *id)
 {
-	return __reset_control_get(dev, id, 0, true, true);
+	return __reset_control_get(dev, id, 0, true, true, false);
 }
 
 /**
@@ -190,7 +226,7 @@ static inline struct reset_control *reset_control_get_optional_shared(
 static inline struct reset_control *of_reset_control_get_exclusive(
 				struct device_node *node, const char *id)
 {
-	return __of_reset_control_get(node, id, 0, false, false);
+	return __of_reset_control_get(node, id, 0, false, false, true);
 }
 
 /**
@@ -215,7 +251,7 @@ static inline struct reset_control *of_reset_control_get_exclusive(
 static inline struct reset_control *of_reset_control_get_shared(
 				struct device_node *node, const char *id)
 {
-	return __of_reset_control_get(node, id, 0, true, false);
+	return __of_reset_control_get(node, id, 0, true, false, false);
 }
 
 /**
@@ -232,7 +268,7 @@ static inline struct reset_control *of_reset_control_get_shared(
 static inline struct reset_control *of_reset_control_get_exclusive_by_index(
 					struct device_node *node, int index)
 {
-	return __of_reset_control_get(node, NULL, index, false, false);
+	return __of_reset_control_get(node, NULL, index, false, false, true);
 }
 
 /**
@@ -260,7 +296,7 @@ static inline struct reset_control *of_reset_control_get_exclusive_by_index(
 static inline struct reset_control *of_reset_control_get_shared_by_index(
 					struct device_node *node, int index)
 {
-	return __of_reset_control_get(node, NULL, index, true, false);
+	return __of_reset_control_get(node, NULL, index, true, false, false);
 }
 
 /**
@@ -279,7 +315,26 @@ static inline struct reset_control *
 __must_check devm_reset_control_get_exclusive(struct device *dev,
 					      const char *id)
 {
-	return __devm_reset_control_get(dev, id, 0, false, false);
+	return __devm_reset_control_get(dev, id, 0, false, false, true);
+}
+
+/**
+ * devm_reset_control_get_exclusive_released - resource managed
+ *                                             reset_control_get_exclusive_released()
+ * @dev: device to be reset by the controller
+ * @id: reset line name
+ *
+ * Managed reset_control_get_exclusive_released(). For reset controllers
+ * returned from this function, reset_control_put() is called automatically on
+ * driver detach.
+ *
+ * See reset_control_get_exclusive_released() for more information.
+ */
+static inline struct reset_control *
+__must_check devm_reset_control_get_exclusive_released(struct device *dev,
+						       const char *id)
+{
+	return __devm_reset_control_get(dev, id, 0, false, false, false);
 }
 
 /**
@@ -294,19 +349,19 @@ __must_check devm_reset_control_get_exclusive(struct device *dev,
 static inline struct reset_control *devm_reset_control_get_shared(
 					struct device *dev, const char *id)
 {
-	return __devm_reset_control_get(dev, id, 0, true, false);
+	return __devm_reset_control_get(dev, id, 0, true, false, false);
 }
 
 static inline struct reset_control *devm_reset_control_get_optional_exclusive(
 					struct device *dev, const char *id)
 {
-	return __devm_reset_control_get(dev, id, 0, false, true);
+	return __devm_reset_control_get(dev, id, 0, false, true, true);
 }
 
 static inline struct reset_control *devm_reset_control_get_optional_shared(
 					struct device *dev, const char *id)
 {
-	return __devm_reset_control_get(dev, id, 0, true, true);
+	return __devm_reset_control_get(dev, id, 0, true, true, false);
 }
 
 /**
@@ -324,7 +379,7 @@ static inline struct reset_control *devm_reset_control_get_optional_shared(
 static inline struct reset_control *
 devm_reset_control_get_exclusive_by_index(struct device *dev, int index)
 {
-	return __devm_reset_control_get(dev, NULL, index, false, false);
+	return __devm_reset_control_get(dev, NULL, index, false, false, true);
 }
 
 /**
@@ -340,7 +395,7 @@ devm_reset_control_get_exclusive_by_index(struct device *dev, int index)
 static inline struct reset_control *
 devm_reset_control_get_shared_by_index(struct device *dev, int index)
 {
-	return __devm_reset_control_get(dev, NULL, index, true, false);
+	return __devm_reset_control_get(dev, NULL, index, true, false, false);
 }
 
 /*
@@ -412,24 +467,30 @@ devm_reset_control_array_get_optional_shared(struct device *dev)
 static inline struct reset_control *
 of_reset_control_array_get_exclusive(struct device_node *node)
 {
-	return of_reset_control_array_get(node, false, false);
+	return of_reset_control_array_get(node, false, false, true);
+}
+
+static inline struct reset_control *
+of_reset_control_array_get_exclusive_released(struct device_node *node)
+{
+	return of_reset_control_array_get(node, false, false, false);
 }
 
 static inline struct reset_control *
 of_reset_control_array_get_shared(struct device_node *node)
 {
-	return of_reset_control_array_get(node, true, false);
+	return of_reset_control_array_get(node, true, false, true);
 }
 
 static inline struct reset_control *
 of_reset_control_array_get_optional_exclusive(struct device_node *node)
 {
-	return of_reset_control_array_get(node, false, true);
+	return of_reset_control_array_get(node, false, true, true);
 }
 
 static inline struct reset_control *
 of_reset_control_array_get_optional_shared(struct device_node *node)
 {
-	return of_reset_control_array_get(node, true, true);
+	return of_reset_control_array_get(node, true, true, true);
 }
 #endif
diff --git a/include/linux/rtc.h b/include/linux/rtc.h
index f89bfbb54902..df666cf29ef1 100644
--- a/include/linux/rtc.h
+++ b/include/linux/rtc.h
@@ -79,8 +79,6 @@ struct rtc_class_ops {
 	int (*read_alarm)(struct device *, struct rtc_wkalrm *);
 	int (*set_alarm)(struct device *, struct rtc_wkalrm *);
 	int (*proc)(struct device *, struct seq_file *);
-	int (*set_mmss64)(struct device *, time64_t secs);
-	int (*set_mmss)(struct device *, unsigned long secs);
 	int (*alarm_irq_enable)(struct device *, unsigned int enabled);
 	int (*read_offset)(struct device *, long *offset);
 	int (*set_offset)(struct device *, long offset);
@@ -162,9 +160,11 @@ struct rtc_device {
 #define to_rtc_device(d) container_of(d, struct rtc_device, dev)
 
 /* useful timestamps */
-#define RTC_TIMESTAMP_BEGIN_1900	-2208989361LL /* 1900-01-01 00:00:00 */
+#define RTC_TIMESTAMP_BEGIN_1900	-2208988800LL /* 1900-01-01 00:00:00 */
 #define RTC_TIMESTAMP_BEGIN_2000	946684800LL /* 2000-01-01 00:00:00 */
+#define RTC_TIMESTAMP_END_2063		2966371199LL /* 2063-12-31 23:59:59 */
 #define RTC_TIMESTAMP_END_2099		4102444799LL /* 2099-12-31 23:59:59 */
+#define RTC_TIMESTAMP_END_9999		253402300799LL /* 9999-12-31 23:59:59 */
 
 extern struct rtc_device *devm_rtc_device_register(struct device *dev,
 					const char *name,
diff --git a/include/linux/rtc/ds1685.h b/include/linux/rtc/ds1685.h
index e6337a56d741..a00b332c505f 100644
--- a/include/linux/rtc/ds1685.h
+++ b/include/linux/rtc/ds1685.h
@@ -48,8 +48,6 @@ struct ds1685_priv {
 	u32 regstep;
 	resource_size_t baseaddr;
 	size_t size;
-	spinlock_t lock;
-	struct work_struct work;
 	int irq_num;
 	bool bcd_mode;
 	bool no_irq;
diff --git a/include/linux/scatterlist.h b/include/linux/scatterlist.h
index b4be960c7e5d..30a9a55c28ba 100644
--- a/include/linux/scatterlist.h
+++ b/include/linux/scatterlist.h
@@ -340,11 +340,11 @@ int sg_alloc_table_chained(struct sg_table *table, int nents,
  * sg page iterator
  *
  * Iterates over sg entries page-by-page.  On each successful iteration, you
- * can call sg_page_iter_page(@piter) to get the current page and its dma
- * address. @piter->sg will point to the sg holding this page and
- * @piter->sg_pgoffset to the page's page offset within the sg. The iteration
- * will stop either when a maximum number of sg entries was reached or a
- * terminating sg (sg_last(sg) == true) was reached.
+ * can call sg_page_iter_page(@piter) to get the current page.
+ * @piter->sg will point to the sg holding this page and @piter->sg_pgoffset to
+ * the page's page offset within the sg. The iteration will stop either when a
+ * maximum number of sg entries was reached or a terminating sg
+ * (sg_last(sg) == true) was reached.
  */
 struct sg_page_iter {
 	struct scatterlist	*sg;		/* sg holding the page */
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 50606a6e73d6..a2cd15855bad 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -726,6 +726,8 @@ struct task_struct {
 #ifdef CONFIG_CGROUPS
 	/* disallow userland-initiated cgroup migration */
 	unsigned			no_cgroup_migration:1;
+	/* task is frozen/stopped (used by the cgroup freezer) */
+	unsigned			frozen:1;
 #endif
 #ifdef CONFIG_BLK_CGROUP
 	/* to be used once the psi infrastructure lands upstream. */
diff --git a/include/linux/sched/jobctl.h b/include/linux/sched/jobctl.h
index 98228bd48aee..fa067de9f1a9 100644
--- a/include/linux/sched/jobctl.h
+++ b/include/linux/sched/jobctl.h
@@ -18,6 +18,7 @@ struct task_struct;
 #define JOBCTL_TRAP_NOTIFY_BIT	20	/* trap for NOTIFY */
 #define JOBCTL_TRAPPING_BIT	21	/* switching to TRACED */
 #define JOBCTL_LISTENING_BIT	22	/* ptracer is listening for events */
+#define JOBCTL_TRAP_FREEZE_BIT	23	/* trap for cgroup freezer */
 
 #define JOBCTL_STOP_DEQUEUED	(1UL << JOBCTL_STOP_DEQUEUED_BIT)
 #define JOBCTL_STOP_PENDING	(1UL << JOBCTL_STOP_PENDING_BIT)
@@ -26,6 +27,7 @@ struct task_struct;
 #define JOBCTL_TRAP_NOTIFY	(1UL << JOBCTL_TRAP_NOTIFY_BIT)
 #define JOBCTL_TRAPPING		(1UL << JOBCTL_TRAPPING_BIT)
 #define JOBCTL_LISTENING	(1UL << JOBCTL_LISTENING_BIT)
+#define JOBCTL_TRAP_FREEZE	(1UL << JOBCTL_TRAP_FREEZE_BIT)
 
 #define JOBCTL_TRAP_MASK	(JOBCTL_TRAP_STOP | JOBCTL_TRAP_NOTIFY)
 #define JOBCTL_PENDING_MASK	(JOBCTL_STOP_PENDING | JOBCTL_TRAP_MASK)
diff --git a/include/linux/serial_core.h b/include/linux/serial_core.h
index 5fe2b037e833..fea2216a893f 100644
--- a/include/linux/serial_core.h
+++ b/include/linux/serial_core.h
@@ -45,7 +45,7 @@ struct device;
 
 /*
  * This structure describes all the operations that can be done on the
- * physical hardware.  See Documentation/serial/driver for details.
+ * physical hardware.  See Documentation/serial/driver.rst for details.
  */
 struct uart_ops {
 	unsigned int	(*tx_empty)(struct uart_port *);
diff --git a/include/linux/sunrpc/clnt.h b/include/linux/sunrpc/clnt.h
index 98bc9883b230..6e8073140a5d 100644
--- a/include/linux/sunrpc/clnt.h
+++ b/include/linux/sunrpc/clnt.h
@@ -50,6 +50,7 @@ struct rpc_clnt {
 	struct rpc_iostats *	cl_metrics;	/* per-client statistics */
 
 	unsigned int		cl_softrtry : 1,/* soft timeouts */
+				cl_softerr  : 1,/* Timeouts return errors */
 				cl_discrtry : 1,/* disconnect before retry */
 				cl_noretranstimeo: 1,/* No retransmit timeouts */
 				cl_autobind : 1,/* use getport() */
@@ -71,6 +72,7 @@ struct rpc_clnt {
 	struct dentry		*cl_debugfs;	/* debugfs directory */
 #endif
 	struct rpc_xprt_iter	cl_xpi;
+	const struct cred	*cl_cred;
 };
 
 /*
@@ -125,6 +127,7 @@ struct rpc_create_args {
 	unsigned long		flags;
 	char			*client_name;
 	struct svc_xprt		*bc_xprt;	/* NFSv4.1 backchannel */
+	const struct cred	*cred;
 };
 
 struct rpc_add_xprt_test {
@@ -144,6 +147,7 @@ struct rpc_add_xprt_test {
 #define RPC_CLNT_CREATE_INFINITE_SLOTS	(1UL << 7)
 #define RPC_CLNT_CREATE_NO_IDLE_TIMEOUT	(1UL << 8)
 #define RPC_CLNT_CREATE_NO_RETRANS_TIMEOUT	(1UL << 9)
+#define RPC_CLNT_CREATE_SOFTERR		(1UL << 10)
 
 struct rpc_clnt *rpc_create(struct rpc_create_args *args);
 struct rpc_clnt	*rpc_bind_new_program(struct rpc_clnt *,
diff --git a/include/linux/sunrpc/sched.h b/include/linux/sunrpc/sched.h
index 52d41d0c1ae1..d0e451868f02 100644
--- a/include/linux/sunrpc/sched.h
+++ b/include/linux/sunrpc/sched.h
@@ -35,7 +35,6 @@ struct rpc_wait {
 	struct list_head	list;		/* wait queue links */
 	struct list_head	links;		/* Links to related tasks */
 	struct list_head	timer_list;	/* Timer list */
-	unsigned long		expires;
 };
 
 /*
@@ -62,6 +61,8 @@ struct rpc_task {
 		struct rpc_wait		tk_wait;	/* RPC wait */
 	} u;
 
+	int			tk_rpc_status;	/* Result of last RPC operation */
+
 	/*
 	 * RPC call state
 	 */
@@ -125,7 +126,6 @@ struct rpc_task_setup {
 #define RPC_CALL_MAJORSEEN	0x0020		/* major timeout seen */
 #define RPC_TASK_ROOTCREDS	0x0040		/* force root creds */
 #define RPC_TASK_DYNAMIC	0x0080		/* task was kmalloc'ed */
-#define RPC_TASK_KILLED		0x0100		/* task was killed */
 #define RPC_TASK_SOFT		0x0200		/* Use soft timeouts */
 #define RPC_TASK_SOFTCONN	0x0400		/* Fail if can't connect */
 #define RPC_TASK_SENT		0x0800		/* message was sent */
@@ -135,7 +135,6 @@ struct rpc_task_setup {
 
 #define RPC_IS_ASYNC(t)		((t)->tk_flags & RPC_TASK_ASYNC)
 #define RPC_IS_SWAPPER(t)	((t)->tk_flags & RPC_TASK_SWAPPER)
-#define RPC_ASSASSINATED(t)	((t)->tk_flags & RPC_TASK_KILLED)
 #define RPC_IS_SOFT(t)		((t)->tk_flags & (RPC_TASK_SOFT|RPC_TASK_TIMEOUT))
 #define RPC_IS_SOFTCONN(t)	((t)->tk_flags & RPC_TASK_SOFTCONN)
 #define RPC_WAS_SENT(t)		((t)->tk_flags & RPC_TASK_SENT)
@@ -146,6 +145,7 @@ struct rpc_task_setup {
 #define RPC_TASK_NEED_XMIT	3
 #define RPC_TASK_NEED_RECV	4
 #define RPC_TASK_MSG_PIN_WAIT	5
+#define RPC_TASK_SIGNALLED	6
 
 #define RPC_IS_RUNNING(t)	test_bit(RPC_TASK_RUNNING, &(t)->tk_runstate)
 #define rpc_set_running(t)	set_bit(RPC_TASK_RUNNING, &(t)->tk_runstate)
@@ -169,6 +169,8 @@ struct rpc_task_setup {
 
 #define RPC_IS_ACTIVATED(t)	test_bit(RPC_TASK_ACTIVE, &(t)->tk_runstate)
 
+#define RPC_SIGNALLED(t)	test_bit(RPC_TASK_SIGNALLED, &(t)->tk_runstate)
+
 /*
  * Task priorities.
  * Note: if you change these, you must also change
@@ -183,7 +185,6 @@ struct rpc_task_setup {
 struct rpc_timer {
 	struct timer_list timer;
 	struct list_head list;
-	unsigned long expires;
 };
 
 /*
@@ -217,6 +218,7 @@ struct rpc_task *rpc_run_task(const struct rpc_task_setup *);
 struct rpc_task *rpc_run_bc_task(struct rpc_rqst *req);
 void		rpc_put_task(struct rpc_task *);
 void		rpc_put_task_async(struct rpc_task *);
+void		rpc_signal_task(struct rpc_task *);
 void		rpc_exit_task(struct rpc_task *);
 void		rpc_exit(struct rpc_task *, int);
 void		rpc_release_calldata(const struct rpc_call_ops *, void *);
@@ -225,11 +227,19 @@ void		rpc_execute(struct rpc_task *);
 void		rpc_init_priority_wait_queue(struct rpc_wait_queue *, const char *);
 void		rpc_init_wait_queue(struct rpc_wait_queue *, const char *);
 void		rpc_destroy_wait_queue(struct rpc_wait_queue *);
+unsigned long	rpc_task_timeout(const struct rpc_task *task);
+void		rpc_sleep_on_timeout(struct rpc_wait_queue *queue,
+					struct rpc_task *task,
+					rpc_action action,
+					unsigned long timeout);
 void		rpc_sleep_on(struct rpc_wait_queue *, struct rpc_task *,
 					rpc_action action);
+void		rpc_sleep_on_priority_timeout(struct rpc_wait_queue *queue,
+					struct rpc_task *task,
+					unsigned long timeout,
+					int priority);
 void		rpc_sleep_on_priority(struct rpc_wait_queue *,
 					struct rpc_task *,
-					rpc_action action,
 					int priority);
 void rpc_wake_up_queued_task_on_wq(struct workqueue_struct *wq,
 		struct rpc_wait_queue *queue,
diff --git a/include/linux/sunrpc/xprt.h b/include/linux/sunrpc/xprt.h
index 3a391544299e..a6d9fce7f20e 100644
--- a/include/linux/sunrpc/xprt.h
+++ b/include/linux/sunrpc/xprt.h
@@ -143,7 +143,7 @@ struct rpc_xprt_ops {
 	void		(*buf_free)(struct rpc_task *task);
 	void		(*prepare_request)(struct rpc_rqst *req);
 	int		(*send_request)(struct rpc_rqst *req);
-	void		(*set_retrans_timeout)(struct rpc_task *task);
+	void		(*wait_for_reply_request)(struct rpc_task *task);
 	void		(*timer)(struct rpc_xprt *xprt, struct rpc_task *task);
 	void		(*release_request)(struct rpc_task *task);
 	void		(*close)(struct rpc_xprt *xprt);
@@ -378,8 +378,8 @@ xprt_disable_swap(struct rpc_xprt *xprt)
 int			xprt_register_transport(struct xprt_class *type);
 int			xprt_unregister_transport(struct xprt_class *type);
 int			xprt_load_transport(const char *);
-void			xprt_set_retrans_timeout_def(struct rpc_task *task);
-void			xprt_set_retrans_timeout_rtt(struct rpc_task *task);
+void			xprt_wait_for_reply_request_def(struct rpc_task *task);
+void			xprt_wait_for_reply_request_rtt(struct rpc_task *task);
 void			xprt_wake_pending_tasks(struct rpc_xprt *xprt, int status);
 void			xprt_wait_for_buffer_space(struct rpc_xprt *xprt);
 bool			xprt_write_space(struct rpc_xprt *xprt);
diff --git a/include/linux/usb.h b/include/linux/usb.h
index ff010d1fd1c7..ae82d9d1112b 100644
--- a/include/linux/usb.h
+++ b/include/linux/usb.h
@@ -1543,10 +1543,10 @@ typedef void (*usb_complete_t)(struct urb *);
 struct urb {
 	/* private: usb core and host controller only fields in the urb */
 	struct kref kref;		/* reference count of the URB */
+	int unlinked;			/* unlink error code */
 	void *hcpriv;			/* private data for host controller */
 	atomic_t use_count;		/* concurrent submissions counter */
 	atomic_t reject;		/* submissions will fail */
-	int unlinked;			/* unlink error code */
 
 	/* public: documented fields in the urb that can be used by drivers */
 	struct list_head urb_list;	/* list head for use by the urb's
diff --git a/include/linux/usb/ch9.h b/include/linux/usb/ch9.h
index 523aa088f6ab..da82606be605 100644
--- a/include/linux/usb/ch9.h
+++ b/include/linux/usb/ch9.h
@@ -37,6 +37,14 @@
 #include <uapi/linux/usb/ch9.h>
 
 /**
+ * usb_ep_type_string() - Returns human readable-name of the endpoint type.
+ * @ep_type: The endpoint type to return human-readable name for.  If it's not
+ *   any of the types: USB_ENDPOINT_XFER_{CONTROL, ISOC, BULK, INT},
+ *   usually got by usb_endpoint_type(), the string 'unknown' will be returned.
+ */
+extern const char *usb_ep_type_string(int ep_type);
+
+/**
  * usb_speed_string() - Returns human readable-name of the speed.
  * @speed: The speed to return human-readable name for.  If it's not
  *   any of the speeds defined in usb_device_speed enum, string for
diff --git a/include/linux/usb/hcd.h b/include/linux/usb/hcd.h
index 695931b03684..bb57b5af4700 100644
--- a/include/linux/usb/hcd.h
+++ b/include/linux/usb/hcd.h
@@ -98,6 +98,7 @@ struct usb_hcd {
 #ifdef CONFIG_PM
 	struct work_struct	wakeup_work;	/* for remote wakeup */
 #endif
+	struct work_struct	died_work;	/* for when the device dies */
 
 	/*
 	 * hardware info/state
@@ -652,11 +653,16 @@ extern wait_queue_head_t usb_kill_urb_queue;
 #define usb_endpoint_out(ep_dir)	(!((ep_dir) & USB_DIR_IN))
 
 #ifdef CONFIG_PM
+extern unsigned usb_wakeup_enabled_descendants(struct usb_device *udev);
 extern void usb_root_hub_lost_power(struct usb_device *rhdev);
 extern int hcd_bus_suspend(struct usb_device *rhdev, pm_message_t msg);
 extern int hcd_bus_resume(struct usb_device *rhdev, pm_message_t msg);
 extern void usb_hcd_resume_root_hub(struct usb_hcd *hcd);
 #else
+static inline unsigned usb_wakeup_enabled_descendants(struct usb_device *udev)
+{
+	return 0;
+}
 static inline void usb_hcd_resume_root_hub(struct usb_hcd *hcd)
 {
 	return;
diff --git a/include/linux/usb/serial.h b/include/linux/usb/serial.h
index 1c19f77ed541..14cac4a1ae8f 100644
--- a/include/linux/usb/serial.h
+++ b/include/linux/usb/serial.h
@@ -23,11 +23,9 @@
 /* The maximum number of ports one device can grab at once */
 #define MAX_NUM_PORTS		16
 
-/* parity check flag */
-#define RELEVANT_IFLAG(iflag)	(iflag & (IGNBRK|BRKINT|IGNPAR|PARMRK|INPCK))
-
 /* USB serial flags */
 #define USB_SERIAL_WRITE_BUSY	0
+#define USB_SERIAL_THROTTLED	1
 
 /**
  * usb_serial_port: structure for the specific ports of a device.
@@ -67,8 +65,6 @@
  * @flags: usb serial port flags
  * @write_wait: a wait_queue_head_t used by the port.
  * @work: work queue entry for the line discipline waking up.
- * @throttled: nonzero if the read urb is inactive to throttle the device
- * @throttle_req: nonzero if the tty wants to throttle us
  * @dev: pointer to the serial device
  *
  * This structure is used by the usb-serial core and drivers for the specific
@@ -115,8 +111,6 @@ struct usb_serial_port {
 	unsigned long		flags;
 	wait_queue_head_t	write_wait;
 	struct work_struct	work;
-	char			throttled;
-	char			throttle_req;
 	unsigned long		sysrq; /* sysrq timeout */
 	struct device		dev;
 };
diff --git a/include/linux/usb/tcpm.h b/include/linux/usb/tcpm.h
index 0c532ca3f079..36a15dcadc53 100644
--- a/include/linux/usb/tcpm.h
+++ b/include/linux/usb/tcpm.h
@@ -121,10 +121,10 @@ struct tcpc_config {
  *		with partner.
  * @set_pd_rx:	Called to enable or disable reception of PD messages
  * @set_roles:	Called to set power and data roles
- * @start_drp_toggling:
- *		Optional; if supported by hardware, called to start DRP
- *		toggling. DRP toggling is stopped automatically if
- *		a connection is established.
+ * @start_toggling:
+ *		Optional; if supported by hardware, called to start dual-role
+ *		toggling or single-role connection detection. Toggling stops
+ *		automatically if a connection is established.
  * @try_role:	Optional; called to set a preferred role
  * @pd_transmit:Called to transmit PD message
  * @mux:	Pointer to multiplexer data
@@ -147,8 +147,9 @@ struct tcpc_dev {
 	int (*set_pd_rx)(struct tcpc_dev *dev, bool on);
 	int (*set_roles)(struct tcpc_dev *dev, bool attached,
 			 enum typec_role role, enum typec_data_role data);
-	int (*start_drp_toggling)(struct tcpc_dev *dev,
-				  enum typec_cc_status cc);
+	int (*start_toggling)(struct tcpc_dev *dev,
+			      enum typec_port_type port_type,
+			      enum typec_cc_status cc);
 	int (*try_role)(struct tcpc_dev *dev, int role);
 	int (*pd_transmit)(struct tcpc_dev *dev, enum tcpm_transmit_type type,
 			   const struct pd_message *msg);
diff --git a/include/linux/usb/typec_dp.h b/include/linux/usb/typec_dp.h
index 7fa12ef8d09a..fc4c7edb2e8a 100644
--- a/include/linux/usb/typec_dp.h
+++ b/include/linux/usb/typec_dp.h
@@ -5,6 +5,11 @@
 #include <linux/usb/typec_altmode.h>
 
 #define USB_TYPEC_DP_SID	0xff01
+/* USB IF has not assigned a Standard ID (SID) for VirtualLink,
+ * so the manufacturers of VirtualLink adapters use their Vendor
+ * IDs as the SVID.
+ */
+#define USB_TYPEC_NVIDIA_VLINK_SID	0x955	/* NVIDIA VirtualLink */
 #define USB_TYPEC_DP_MODE	1
 
 /*
diff --git a/include/linux/wait.h b/include/linux/wait.h
index 5f3efabc36f4..b6f77cf60dd7 100644
--- a/include/linux/wait.h
+++ b/include/linux/wait.h
@@ -101,7 +101,7 @@ init_waitqueue_func_entry(struct wait_queue_entry *wq_entry, wait_queue_func_t f
  * lead to sporadic and non-obvious failure.
  *
  * Use either while holding wait_queue_head::lock or when used for wakeups
- * with an extra smp_mb() like:
+ * with an extra smp_mb() like::
  *
  *      CPU0 - waker                    CPU1 - waiter
  *
diff --git a/include/media/cec-notifier.h b/include/media/cec-notifier.h
index 814eeef35a5c..57b3a9f6ea1d 100644
--- a/include/media/cec-notifier.h
+++ b/include/media/cec-notifier.h
@@ -9,7 +9,7 @@
 #ifndef LINUX_CEC_NOTIFIER_H
 #define LINUX_CEC_NOTIFIER_H
 
-#include <linux/types.h>
+#include <linux/err.h>
 #include <media/cec.h>
 
 struct device;
@@ -87,6 +87,17 @@ void cec_notifier_unregister(struct cec_notifier *n);
 void cec_register_cec_notifier(struct cec_adapter *adap,
 			       struct cec_notifier *notifier);
 
+/**
+ * cec_notifier_parse_hdmi_phandle - find the hdmi device from "hdmi-phandle"
+ * @dev: the device with the "hdmi-phandle" device tree property
+ *
+ * Returns the device pointer referenced by the "hdmi-phandle" property.
+ * Note that the refcount of the returned device is not incremented.
+ * This device pointer is only used as a key value in the notifier
+ * list, but it is never accessed by the CEC driver.
+ */
+struct device *cec_notifier_parse_hdmi_phandle(struct device *dev);
+
 #else
 static inline struct cec_notifier *cec_notifier_get_conn(struct device *dev,
 							 const char *conn)
@@ -122,6 +133,12 @@ static inline void cec_register_cec_notifier(struct cec_adapter *adap,
 					     struct cec_notifier *notifier)
 {
 }
+
+static inline struct device *cec_notifier_parse_hdmi_phandle(struct device *dev)
+{
+	return ERR_PTR(-ENODEV);
+}
+
 #endif
 
 /**
diff --git a/include/media/dvb-usb-ids.h b/include/media/dvb-usb-ids.h
index f9e73b4a6e89..52875e3eee71 100644
--- a/include/media/dvb-usb-ids.h
+++ b/include/media/dvb-usb-ids.h
@@ -258,6 +258,7 @@
 #define USB_PID_AVERMEDIA_A867				0xa867
 #define USB_PID_AVERMEDIA_H335				0x0335
 #define USB_PID_AVERMEDIA_TD110				0xa110
+#define USB_PID_AVERMEDIA_TD310				0x1871
 #define USB_PID_AVERMEDIA_TWINSTAR			0x0825
 #define USB_PID_TECHNOTREND_CONNECT_S2400               0x3006
 #define USB_PID_TECHNOTREND_CONNECT_S2400_8KEEPROM	0x3009
diff --git a/include/media/fwht-ctrls.h b/include/media/fwht-ctrls.h
new file mode 100644
index 000000000000..615027410e47
--- /dev/null
+++ b/include/media/fwht-ctrls.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * These are the FWHT state controls for use with stateless FWHT
+ * codec drivers.
+ *
+ * It turns out that these structs are not stable yet and will undergo
+ * more changes. So keep them private until they are stable and ready to
+ * become part of the official public API.
+ */
+
+#ifndef _FWHT_CTRLS_H_
+#define _FWHT_CTRLS_H_
+
+#define V4L2_CTRL_TYPE_FWHT_PARAMS 0x0105
+
+#define V4L2_CID_MPEG_VIDEO_FWHT_PARAMS	(V4L2_CID_MPEG_BASE + 292)
+
+struct v4l2_ctrl_fwht_params {
+	__u64 backward_ref_ts;
+	__u32 version;
+	__u32 width;
+	__u32 height;
+	__u32 flags;
+	__u32 colorspace;
+	__u32 xfer_func;
+	__u32 ycbcr_enc;
+	__u32 quantization;
+};
+
+
+#endif
diff --git a/include/media/media-dev-allocator.h b/include/media/media-dev-allocator.h
new file mode 100644
index 000000000000..b35ea6062596
--- /dev/null
+++ b/include/media/media-dev-allocator.h
@@ -0,0 +1,63 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * media-dev-allocator.h - Media Controller Device Allocator API
+ *
+ * Copyright (c) 2019 Shuah Khan <shuah@kernel.org>
+ *
+ * Credits: Suggested by Laurent Pinchart <laurent.pinchart@ideasonboard.com>
+ */
+
+/*
+ * This file adds a global ref-counted Media Controller Device Instance API.
+ * A system wide global media device list is managed and each media device
+ * includes a kref count. The last put on the media device releases the media
+ * device instance.
+ */
+
+#ifndef _MEDIA_DEV_ALLOCATOR_H
+#define _MEDIA_DEV_ALLOCATOR_H
+
+struct usb_device;
+
+#if defined(CONFIG_MEDIA_CONTROLLER) && defined(CONFIG_USB)
+/**
+ * media_device_usb_allocate() - Allocate and return struct &media device
+ *
+ * @udev:		struct &usb_device pointer
+ * @module_name:	should be filled with %KBUILD_MODNAME
+ * @owner:		struct module pointer %THIS_MODULE for the driver.
+ *			%THIS_MODULE is null for a built-in driver.
+ *			It is safe even when %THIS_MODULE is null.
+ *
+ * This interface should be called to allocate a Media Device when multiple
+ * drivers share usb_device and the media device. This interface allocates
+ * &media_device structure and calls media_device_usb_init() to initialize
+ * it.
+ *
+ */
+struct media_device *media_device_usb_allocate(struct usb_device *udev,
+					       const char *module_name,
+					       struct module *owner);
+/**
+ * media_device_delete() - Release media device. Calls kref_put().
+ *
+ * @mdev:		struct &media_device pointer
+ * @module_name:	should be filled with %KBUILD_MODNAME
+ * @owner:		struct module pointer %THIS_MODULE for the driver.
+ *			%THIS_MODULE is null for a built-in driver.
+ *			It is safe even when %THIS_MODULE is null.
+ *
+ * This interface should be called to put Media Device Instance kref.
+ */
+void media_device_delete(struct media_device *mdev, const char *module_name,
+			 struct module *owner);
+#else
+static inline struct media_device *media_device_usb_allocate(
+			struct usb_device *udev, const char *module_name,
+			struct module *owner)
+			{ return NULL; }
+static inline void media_device_delete(
+			struct media_device *mdev, const char *module_name,
+			struct module *owner) { }
+#endif /* CONFIG_MEDIA_CONTROLLER && CONFIG_USB */
+#endif /* _MEDIA_DEV_ALLOCATOR_H */
diff --git a/include/media/media-entity.h b/include/media/media-entity.h
index e5f6960d92f6..3cbad42e3693 100644
--- a/include/media/media-entity.h
+++ b/include/media/media-entity.h
@@ -865,19 +865,6 @@ struct media_link *media_entity_find_link(struct media_pad *source,
 struct media_pad *media_entity_remote_pad(const struct media_pad *pad);
 
 /**
- * media_entity_get - Get a reference to the parent module
- *
- * @entity: The entity
- *
- * Get a reference to the parent media device module.
- *
- * The function will return immediately if @entity is %NULL.
- *
- * Return: returns a pointer to the entity on success or %NULL on failure.
- */
-struct media_entity *media_entity_get(struct media_entity *entity);
-
-/**
  * media_entity_get_fwnode_pad - Get pad number from fwnode
  *
  * @entity: The entity
@@ -917,17 +904,6 @@ __must_check int media_graph_walk_init(
 void media_graph_walk_cleanup(struct media_graph *graph);
 
 /**
- * media_entity_put - Release the reference to the parent module
- *
- * @entity: The entity
- *
- * Release the reference count acquired by media_entity_get().
- *
- * The function will return immediately if @entity is %NULL.
- */
-void media_entity_put(struct media_entity *entity);
-
-/**
  * media_graph_walk_start - Start walking the media graph at a
  *	given entity
  *
diff --git a/include/media/media-request.h b/include/media/media-request.h
index bd36d7431698..3cd25a2717ce 100644
--- a/include/media/media-request.h
+++ b/include/media/media-request.h
@@ -198,7 +198,7 @@ void media_request_put(struct media_request *req);
  * Get the request represented by @request_fd that is owned
  * by the media device.
  *
- * Return a -EACCES error pointer if requests are not supported
+ * Return a -EBADR error pointer if requests are not supported
  * by this driver. Return -EINVAL if the request was not found.
  * Return the pointer to the request if found: the caller will
  * have to call @media_request_put when it finished using the
@@ -231,7 +231,7 @@ static inline void media_request_put(struct media_request *req)
 static inline struct media_request *
 media_request_get_by_fd(struct media_device *mdev, int request_fd)
 {
-	return ERR_PTR(-EACCES);
+	return ERR_PTR(-EBADR);
 }
 
 #endif
diff --git a/include/media/rc-map.h b/include/media/rc-map.h
index 5e684bb0d64c..367d983188f7 100644
--- a/include/media/rc-map.h
+++ b/include/media/rc-map.h
@@ -40,6 +40,7 @@
 #define RC_PROTO_BIT_RCMM12		BIT_ULL(RC_PROTO_RCMM12)
 #define RC_PROTO_BIT_RCMM24		BIT_ULL(RC_PROTO_RCMM24)
 #define RC_PROTO_BIT_RCMM32		BIT_ULL(RC_PROTO_RCMM32)
+#define RC_PROTO_BIT_XBOX_DVD		BIT_ULL(RC_PROTO_XBOX_DVD)
 
 #define RC_PROTO_BIT_ALL \
 			(RC_PROTO_BIT_UNKNOWN | RC_PROTO_BIT_OTHER | \
@@ -55,7 +56,8 @@
 			 RC_PROTO_BIT_RC6_MCE | RC_PROTO_BIT_SHARP | \
 			 RC_PROTO_BIT_XMP | RC_PROTO_BIT_CEC | \
 			 RC_PROTO_BIT_IMON | RC_PROTO_BIT_RCMM12 | \
-			 RC_PROTO_BIT_RCMM24 | RC_PROTO_BIT_RCMM32)
+			 RC_PROTO_BIT_RCMM24 | RC_PROTO_BIT_RCMM32 | \
+			 RC_PROTO_BIT_XBOX_DVD)
 /* All rc protocols for which we have decoders */
 #define RC_PROTO_BIT_ALL_IR_DECODER \
 			(RC_PROTO_BIT_RC5 | RC_PROTO_BIT_RC5X_20 | \
diff --git a/include/media/v4l2-common.h b/include/media/v4l2-common.h
index 2b93cb281fa5..0a41bbecf3d3 100644
--- a/include/media/v4l2-common.h
+++ b/include/media/v4l2-common.h
@@ -392,4 +392,37 @@ int v4l2_s_parm_cap(struct video_device *vdev,
 	((u64)(a).numerator * (b).denominator OP	\
 	(u64)(b).numerator * (a).denominator)
 
+/* ------------------------------------------------------------------------- */
+
+/* Pixel format and FourCC helpers */
+
+/**
+ * struct v4l2_format_info - information about a V4L2 format
+ * @format: 4CC format identifier (V4L2_PIX_FMT_*)
+ * @mem_planes: Number of memory planes, which includes the alpha plane (1 to 4).
+ * @comp_planes: Number of component planes, which includes the alpha plane (1 to 4).
+ * @bpp: Array of per-plane bytes per pixel
+ * @hdiv: Horizontal chroma subsampling factor
+ * @vdiv: Vertical chroma subsampling factor
+ * @block_w: Per-plane macroblock pixel width (optional)
+ * @block_h: Per-plane macroblock pixel height (optional)
+ */
+struct v4l2_format_info {
+	u32 format;
+	u8 mem_planes;
+	u8 comp_planes;
+	u8 bpp[4];
+	u8 hdiv;
+	u8 vdiv;
+	u8 block_w[4];
+	u8 block_h[4];
+};
+
+const struct v4l2_format_info *v4l2_format_info(u32 format);
+
+int v4l2_fill_pixfmt(struct v4l2_pix_format *pixfmt, int pixelformat,
+		     int width, int height);
+int v4l2_fill_pixfmt_mp(struct v4l2_pix_format_mplane *pixfmt, int pixelformat,
+			int width, int height);
+
 #endif /* V4L2_COMMON_H_ */
diff --git a/include/media/v4l2-ctrls.h b/include/media/v4l2-ctrls.h
index e5cae37ced2d..ee026387f513 100644
--- a/include/media/v4l2-ctrls.h
+++ b/include/media/v4l2-ctrls.h
@@ -23,10 +23,11 @@
 #include <media/media-request.h>
 
 /*
- * Include the mpeg2 stateless codec compound control definitions.
+ * Include the mpeg2 and fwht stateless codec compound control definitions.
  * This will move to the public headers once this API is fully stable.
  */
 #include <media/mpeg2-ctrls.h>
+#include <media/fwht-ctrls.h>
 
 /* forward references */
 struct file;
@@ -49,6 +50,7 @@ struct poll_table_struct;
  * @p_char:			Pointer to a string.
  * @p_mpeg2_slice_params:	Pointer to a MPEG2 slice parameters structure.
  * @p_mpeg2_quantization:	Pointer to a MPEG2 quantization data structure.
+ * @p_fwht_params:		Pointer to a FWHT stateless parameters structure.
  * @p:				Pointer to a compound value.
  */
 union v4l2_ctrl_ptr {
@@ -60,6 +62,7 @@ union v4l2_ctrl_ptr {
 	char *p_char;
 	struct v4l2_ctrl_mpeg2_slice_params *p_mpeg2_slice_params;
 	struct v4l2_ctrl_mpeg2_quantization *p_mpeg2_quantization;
+	struct v4l2_ctrl_fwht_params *p_fwht_params;
 	void *p;
 };
 
@@ -934,7 +937,7 @@ s32 v4l2_ctrl_g_ctrl(struct v4l2_ctrl *ctrl);
  * __v4l2_ctrl_s_ctrl() - Unlocked variant of v4l2_ctrl_s_ctrl().
  *
  * @ctrl:	The control.
- * @val:	TheControls name new value.
+ * @val:	The new value.
  *
  * This sets the control's new value safely by going through the control
  * framework. This function assumes the control's handler is already locked,
@@ -1039,7 +1042,7 @@ int __v4l2_ctrl_s_ctrl_string(struct v4l2_ctrl *ctrl, const char *s);
  *
  * @ctrl:	The control.
  * @s:		The new string.
- *Controls name
+ *
  * This sets the control's new string safely by going through the control
  * framework. This function will lock the control's handler, so it cannot be
  * used from within the &v4l2_ctrl_ops functions.
@@ -1127,7 +1130,7 @@ __poll_t v4l2_ctrl_poll(struct file *file, struct poll_table_struct *wait);
  * applying control values in a request is only applicable to memory-to-memory
  * devices.
  */
-void v4l2_ctrl_request_setup(struct media_request *req,
+int v4l2_ctrl_request_setup(struct media_request *req,
 			     struct v4l2_ctrl_handler *parent);
 
 /**
diff --git a/include/media/v4l2-subdev.h b/include/media/v4l2-subdev.h
index 349e1c18cf48..a7fa5b80915a 100644
--- a/include/media/v4l2-subdev.h
+++ b/include/media/v4l2-subdev.h
@@ -755,7 +755,17 @@ struct v4l2_subdev_ops {
  *
  * @open: called when the subdev device node is opened by an application.
  *
- * @close: called when the subdev device node is closed.
+ * @close: called when the subdev device node is closed. Please note that
+ *	it is possible for @close to be called after @unregistered!
+ *
+ * @release: called when the last user of the subdev device is gone. This
+ *	happens after the @unregistered callback and when the last open
+ *	filehandle to the v4l-subdevX device node was closed. If no device
+ *	node was created for this sub-device, then the @release callback
+ *	is called right after the @unregistered callback.
+ *	The @release callback is typically used to free the memory containing
+ *	the v4l2_subdev structure. It is almost certainly required for any
+ *	sub-device that sets the V4L2_SUBDEV_FL_HAS_DEVNODE flag.
  *
  * .. note::
  *	Never call this from drivers, only the v4l2 framework can call
@@ -766,6 +776,7 @@ struct v4l2_subdev_internal_ops {
 	void (*unregistered)(struct v4l2_subdev *sd);
 	int (*open)(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh);
 	int (*close)(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh);
+	void (*release)(struct v4l2_subdev *sd);
 };
 
 #define V4L2_SUBDEV_NAME_SIZE 32
@@ -899,9 +910,11 @@ struct v4l2_subdev {
  *
  * @vfh: pointer to &struct v4l2_fh
  * @pad: pointer to &struct v4l2_subdev_pad_config
+ * @owner: module pointer to the owner of this file handle
  */
 struct v4l2_subdev_fh {
 	struct v4l2_fh vfh;
+	struct module *owner;
 #if defined(CONFIG_VIDEO_V4L2_SUBDEV_API)
 	struct v4l2_subdev_pad_config *pad;
 #endif
diff --git a/include/media/videobuf2-core.h b/include/media/videobuf2-core.h
index 910f3d469005..22f3ff76a8b5 100644
--- a/include/media/videobuf2-core.h
+++ b/include/media/videobuf2-core.h
@@ -207,7 +207,6 @@ enum vb2_io_modes {
  * @VB2_BUF_STATE_IN_REQUEST:	buffer is queued in media request.
  * @VB2_BUF_STATE_PREPARING:	buffer is being prepared in videobuf.
  * @VB2_BUF_STATE_QUEUED:	buffer queued in videobuf, but not in driver.
- * @VB2_BUF_STATE_REQUEUEING:	re-queue a buffer to the driver.
  * @VB2_BUF_STATE_ACTIVE:	buffer queued in driver and possibly used
  *				in a hardware operation.
  * @VB2_BUF_STATE_DONE:		buffer returned from driver to videobuf, but
@@ -221,7 +220,6 @@ enum vb2_buffer_state {
 	VB2_BUF_STATE_IN_REQUEST,
 	VB2_BUF_STATE_PREPARING,
 	VB2_BUF_STATE_QUEUED,
-	VB2_BUF_STATE_REQUEUEING,
 	VB2_BUF_STATE_ACTIVE,
 	VB2_BUF_STATE_DONE,
 	VB2_BUF_STATE_ERROR,
@@ -384,10 +382,10 @@ struct vb2_buffer {
  *			driver can return an error if hardware fails, in that
  *			case all buffers that have been already given by
  *			the @buf_queue callback are to be returned by the driver
- *			by calling vb2_buffer_done() with %VB2_BUF_STATE_QUEUED
- *			or %VB2_BUF_STATE_REQUEUEING. If you need a minimum
- *			number of buffers before you can start streaming, then
- *			set &vb2_queue->min_buffers_needed. If that is non-zero
+ *			by calling vb2_buffer_done() with %VB2_BUF_STATE_QUEUED.
+ *			If you need a minimum number of buffers before you can
+ *			start streaming, then set
+ *			&vb2_queue->min_buffers_needed. If that is non-zero
  *			then @start_streaming won't be called until at least
  *			that many buffers have been queued up by userspace.
  * @stop_streaming:	called when 'streaming' state must be disabled; driver
@@ -484,6 +482,8 @@ struct vb2_buf_ops {
  *              has not been called. This is a vb1 idiom that has been adopted
  *              also by vb2.
  * @supports_requests: this queue supports the Request API.
+ * @requires_requests: this queue requires the Request API. If this is set to 1,
+ *		then supports_requests must be set to 1 as well.
  * @uses_qbuf:	qbuf was used directly for this queue. Set to 1 the first
  *		time this is called. Set to 0 when the queue is canceled.
  *		If this is 1, then you cannot queue buffers from a request.
@@ -558,6 +558,7 @@ struct vb2_queue {
 	unsigned			allow_zero_bytesused:1;
 	unsigned		   quirk_poll_must_check_waiting_for_buffers:1;
 	unsigned			supports_requests:1;
+	unsigned			requires_requests:1;
 	unsigned			uses_qbuf:1;
 	unsigned			uses_requests:1;
 
@@ -595,6 +596,7 @@ struct vb2_queue {
 	unsigned int			start_streaming_called:1;
 	unsigned int			error:1;
 	unsigned int			waiting_for_buffers:1;
+	unsigned int			waiting_in_dqbuf:1;
 	unsigned int			is_multiplanar:1;
 	unsigned int			is_output:1;
 	unsigned int			copy_timestamp:1;
@@ -648,9 +650,7 @@ void *vb2_plane_cookie(struct vb2_buffer *vb, unsigned int plane_no);
  * @state:	state of the buffer, as defined by &enum vb2_buffer_state.
  *		Either %VB2_BUF_STATE_DONE if the operation finished
  *		successfully, %VB2_BUF_STATE_ERROR if the operation finished
- *		with an error or any of %VB2_BUF_STATE_QUEUED or
- *		%VB2_BUF_STATE_REQUEUEING if the driver wants to
- *		requeue buffers (see below).
+ *		with an error or %VB2_BUF_STATE_QUEUED.
  *
  * This function should be called by the driver after a hardware operation on
  * a buffer is finished and the buffer may be returned to userspace. The driver
@@ -661,12 +661,7 @@ void *vb2_plane_cookie(struct vb2_buffer *vb, unsigned int plane_no);
  * While streaming a buffer can only be returned in state DONE or ERROR.
  * The &vb2_ops->start_streaming op can also return them in case the DMA engine
  * cannot be started for some reason. In that case the buffers should be
- * returned with state QUEUED or REQUEUEING to put them back into the queue.
- *
- * %VB2_BUF_STATE_REQUEUEING is like %VB2_BUF_STATE_QUEUED, but it also calls
- * &vb2_ops->buf_queue to queue buffers back to the driver. Note that calling
- * vb2_buffer_done(..., VB2_BUF_STATE_REQUEUEING) from interrupt context will
- * result in &vb2_ops->buf_queue being called in interrupt context as well.
+ * returned with state QUEUED to put them back into the queue.
  */
 void vb2_buffer_done(struct vb2_buffer *vb, enum vb2_buffer_state state);
 
diff --git a/include/media/vsp1.h b/include/media/vsp1.h
index 1cf868360701..cc1b0d42ce95 100644
--- a/include/media/vsp1.h
+++ b/include/media/vsp1.h
@@ -17,6 +17,9 @@ struct device;
 
 int vsp1_du_init(struct device *dev);
 
+#define VSP1_DU_STATUS_COMPLETE		BIT(0)
+#define VSP1_DU_STATUS_WRITEBACK	BIT(1)
+
 /**
  * struct vsp1_du_lif_config - VSP LIF configuration
  * @width: output frame width
@@ -32,7 +35,7 @@ struct vsp1_du_lif_config {
 	unsigned int height;
 	bool interlaced;
 
-	void (*callback)(void *data, bool completed, u32 crc);
+	void (*callback)(void *data, unsigned int status, u32 crc);
 	void *callback_data;
 };
 
@@ -82,11 +85,25 @@ struct vsp1_du_crc_config {
 };
 
 /**
+ * struct vsp1_du_writeback_config - VSP writeback configuration parameters
+ * @pixelformat: plane pixel format (V4L2 4CC)
+ * @pitch: line pitch in bytes for the first plane
+ * @mem: DMA memory address for each plane of the frame buffer
+ */
+struct vsp1_du_writeback_config {
+	u32 pixelformat;
+	unsigned int pitch;
+	dma_addr_t mem[3];
+};
+
+/**
  * struct vsp1_du_atomic_pipe_config - VSP atomic pipe configuration parameters
  * @crc: CRC computation configuration
+ * @writeback: writeback configuration
  */
 struct vsp1_du_atomic_pipe_config {
 	struct vsp1_du_crc_config crc;
+	struct vsp1_du_writeback_config writeback;
 };
 
 void vsp1_du_atomic_begin(struct device *dev, unsigned int pipe_index);
diff --git a/include/net/tcp.h b/include/net/tcp.h
index 7cf1181630a3..985aa5db570c 100644
--- a/include/net/tcp.h
+++ b/include/net/tcp.h
@@ -2198,7 +2198,7 @@ extern struct static_key_false tcp_have_smc;
 void clean_acked_data_enable(struct inet_connection_sock *icsk,
 			     void (*cad)(struct sock *sk, u32 ack_seq));
 void clean_acked_data_disable(struct inet_connection_sock *icsk);
-
+void clean_acked_data_flush(void);
 #endif
 
 #endif	/* _TCP_H */
diff --git a/include/rdma/ib_cache.h b/include/rdma/ib_cache.h
index 62e990b620aa..870b5e6c06db 100644
--- a/include/rdma/ib_cache.h
+++ b/include/rdma/ib_cache.h
@@ -54,6 +54,10 @@ const struct ib_gid_attr *rdma_find_gid_by_filter(
 		       void *),
 	void *context);
 
+int rdma_read_gid_l2_fields(const struct ib_gid_attr *attr,
+			    u16 *vlan_id, u8 *smac);
+struct net_device *rdma_read_gid_attr_ndev_rcu(const struct ib_gid_attr *attr);
+
 /**
  * ib_get_cached_pkey - Returns a cached PKey table entry
  * @device: The device to query.
diff --git a/include/rdma/ib_mad.h b/include/rdma/ib_mad.h
index 79ba8219e7dc..eea946fcc819 100644
--- a/include/rdma/ib_mad.h
+++ b/include/rdma/ib_mad.h
@@ -198,7 +198,7 @@ struct ib_sa_hdr {
 	__be16			attr_offset;
 	__be16			reserved;
 	ib_sa_comp_mask		comp_mask;
-} __attribute__ ((packed));
+} __packed;
 
 struct ib_mad {
 	struct ib_mad_hdr	mad_hdr;
@@ -227,7 +227,7 @@ struct ib_sa_mad {
 	struct ib_rmpp_hdr	rmpp_hdr;
 	struct ib_sa_hdr	sa_hdr;
 	u8			data[IB_MGMT_SA_DATA];
-} __attribute__ ((packed));
+} __packed;
 
 struct ib_vendor_mad {
 	struct ib_mad_hdr	mad_hdr;
diff --git a/include/rdma/ib_smi.h b/include/rdma/ib_smi.h
index b439e988408e..7be0028f155c 100644
--- a/include/rdma/ib_smi.h
+++ b/include/rdma/ib_smi.h
@@ -61,7 +61,7 @@ struct ib_smp {
 	u8	data[IB_SMP_DATA_SIZE];
 	u8	initial_path[IB_SMP_MAX_PATH_HOPS];
 	u8	return_path[IB_SMP_MAX_PATH_HOPS];
-} __attribute__ ((packed));
+} __packed;
 
 #define IB_SMP_DIRECTION			cpu_to_be16(0x8000)
 
diff --git a/include/rdma/ib_umem.h b/include/rdma/ib_umem.h
index 73af05db04c7..040d853077c6 100644
--- a/include/rdma/ib_umem.h
+++ b/include/rdma/ib_umem.h
@@ -48,12 +48,11 @@ struct ib_umem {
 	unsigned long		address;
 	int			page_shift;
 	u32 writable : 1;
-	u32 hugetlb : 1;
 	u32 is_odp : 1;
 	struct work_struct	work;
 	struct sg_table sg_head;
 	int             nmap;
-	int             npages;
+	unsigned int    sg_nents;
 };
 
 /* Returns the offset of the umem start relative to the first page. */
@@ -87,6 +86,9 @@ void ib_umem_release(struct ib_umem *umem);
 int ib_umem_page_count(struct ib_umem *umem);
 int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset,
 		      size_t length);
+unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem,
+				     unsigned long pgsz_bitmap,
+				     unsigned long virt);
 
 #else /* CONFIG_INFINIBAND_USER_MEM */
 
@@ -104,6 +106,12 @@ static inline int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offs
 		      		    size_t length) {
 	return -EINVAL;
 }
+static inline int ib_umem_find_best_pgsz(struct ib_umem *umem,
+					 unsigned long pgsz_bitmap,
+					 unsigned long virt) {
+	return -EINVAL;
+}
+
 #endif /* CONFIG_INFINIBAND_USER_MEM */
 
 #endif /* IB_UMEM_H */
diff --git a/include/rdma/ib_umem_odp.h b/include/rdma/ib_umem_odp.h
index dadc96dea39c..eeec4e53c448 100644
--- a/include/rdma/ib_umem_odp.h
+++ b/include/rdma/ib_umem_odp.h
@@ -69,6 +69,7 @@ struct ib_umem_odp {
 
 	int notifiers_seq;
 	int notifiers_count;
+	int npages;
 
 	/* Tree tracking */
 	struct umem_odp_node	interval_tree;
diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
index 9b9e17bcc201..0742095355f2 100644
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -59,6 +59,8 @@
 #include <linux/mmu_notifier.h>
 #include <linux/uaccess.h>
 #include <linux/cgroup_rdma.h>
+#include <linux/irqflags.h>
+#include <linux/preempt.h>
 #include <uapi/rdma/ib_user_verbs.h>
 #include <rdma/restrack.h>
 #include <uapi/rdma/rdma_user_ioctl.h>
@@ -72,6 +74,36 @@ extern struct workqueue_struct *ib_wq;
 extern struct workqueue_struct *ib_comp_wq;
 extern struct workqueue_struct *ib_comp_unbound_wq;
 
+__printf(3, 4) __cold
+void ibdev_printk(const char *level, const struct ib_device *ibdev,
+		  const char *format, ...);
+__printf(2, 3) __cold
+void ibdev_emerg(const struct ib_device *ibdev, const char *format, ...);
+__printf(2, 3) __cold
+void ibdev_alert(const struct ib_device *ibdev, const char *format, ...);
+__printf(2, 3) __cold
+void ibdev_crit(const struct ib_device *ibdev, const char *format, ...);
+__printf(2, 3) __cold
+void ibdev_err(const struct ib_device *ibdev, const char *format, ...);
+__printf(2, 3) __cold
+void ibdev_warn(const struct ib_device *ibdev, const char *format, ...);
+__printf(2, 3) __cold
+void ibdev_notice(const struct ib_device *ibdev, const char *format, ...);
+__printf(2, 3) __cold
+void ibdev_info(const struct ib_device *ibdev, const char *format, ...);
+
+#if defined(CONFIG_DYNAMIC_DEBUG)
+#define ibdev_dbg(__dev, format, args...)                       \
+	dynamic_ibdev_dbg(__dev, format, ##args)
+#elif defined(DEBUG)
+#define ibdev_dbg(__dev, format, args...)                       \
+	ibdev_printk(KERN_DEBUG, __dev, format, ##args)
+#else
+__printf(2, 3) __cold
+static inline
+void ibdev_dbg(const struct ib_device *ibdev, const char *format, ...) {}
+#endif
+
 union ib_gid {
 	u8	raw[16];
 	struct {
@@ -92,7 +124,7 @@ enum ib_gid_type {
 
 #define ROCE_V2_UDP_DPORT      4791
 struct ib_gid_attr {
-	struct net_device	*ndev;
+	struct net_device __rcu	*ndev;
 	struct ib_device	*device;
 	union ib_gid		gid;
 	enum ib_gid_type	gid_type;
@@ -108,6 +140,7 @@ enum rdma_node_type {
 	RDMA_NODE_RNIC,
 	RDMA_NODE_USNIC,
 	RDMA_NODE_USNIC_UDP,
+	RDMA_NODE_UNSPECIFIED,
 };
 
 enum {
@@ -119,7 +152,8 @@ enum rdma_transport_type {
 	RDMA_TRANSPORT_IB,
 	RDMA_TRANSPORT_IWARP,
 	RDMA_TRANSPORT_USNIC,
-	RDMA_TRANSPORT_USNIC_UDP
+	RDMA_TRANSPORT_USNIC_UDP,
+	RDMA_TRANSPORT_UNSPECIFIED,
 };
 
 enum rdma_protocol_type {
@@ -2189,8 +2223,6 @@ struct ib_cache {
 	struct ib_event_handler event_handler;
 };
 
-struct iw_cm_verbs;
-
 struct ib_port_immutable {
 	int                           pkey_tbl_len;
 	int                           gid_tbl_len;
@@ -2272,6 +2304,8 @@ struct ib_counters_read_attr {
 };
 
 struct uverbs_attr_bundle;
+struct iw_cm_id;
+struct iw_cm_conn_param;
 
 #define INIT_RDMA_OBJ_SIZE(ib_struct, drv_struct, member)                      \
 	.size_##ib_struct =                                                    \
@@ -2281,8 +2315,11 @@ struct uverbs_attr_bundle;
 			 !__same_type(((struct drv_struct *)NULL)->member,     \
 				      struct ib_struct)))
 
+#define rdma_zalloc_drv_obj_gfp(ib_dev, ib_type, gfp)                         \
+	((struct ib_type *)kzalloc(ib_dev->ops.size_##ib_type, gfp))
+
 #define rdma_zalloc_drv_obj(ib_dev, ib_type)                                   \
-	((struct ib_type *)kzalloc(ib_dev->ops.size_##ib_type, GFP_KERNEL))
+	rdma_zalloc_drv_obj_gfp(ib_dev, ib_type, GFP_KERNEL)
 
 #define DECLARE_RDMA_OBJ_SIZE(ib_struct) size_t size_##ib_struct
 
@@ -2394,23 +2431,21 @@ struct ib_device_ops {
 	void (*dealloc_ucontext)(struct ib_ucontext *context);
 	int (*mmap)(struct ib_ucontext *context, struct vm_area_struct *vma);
 	void (*disassociate_ucontext)(struct ib_ucontext *ibcontext);
-	int (*alloc_pd)(struct ib_pd *pd, struct ib_ucontext *context,
-			struct ib_udata *udata);
-	void (*dealloc_pd)(struct ib_pd *pd);
-	struct ib_ah *(*create_ah)(struct ib_pd *pd,
-				   struct rdma_ah_attr *ah_attr, u32 flags,
-				   struct ib_udata *udata);
+	int (*alloc_pd)(struct ib_pd *pd, struct ib_udata *udata);
+	void (*dealloc_pd)(struct ib_pd *pd, struct ib_udata *udata);
+	int (*create_ah)(struct ib_ah *ah, struct rdma_ah_attr *ah_attr,
+			 u32 flags, struct ib_udata *udata);
 	int (*modify_ah)(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);
 	int (*query_ah)(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);
-	int (*destroy_ah)(struct ib_ah *ah, u32 flags);
-	struct ib_srq *(*create_srq)(struct ib_pd *pd,
-				     struct ib_srq_init_attr *srq_init_attr,
-				     struct ib_udata *udata);
+	void (*destroy_ah)(struct ib_ah *ah, u32 flags);
+	int (*create_srq)(struct ib_srq *srq,
+			  struct ib_srq_init_attr *srq_init_attr,
+			  struct ib_udata *udata);
 	int (*modify_srq)(struct ib_srq *srq, struct ib_srq_attr *srq_attr,
 			  enum ib_srq_attr_mask srq_attr_mask,
 			  struct ib_udata *udata);
 	int (*query_srq)(struct ib_srq *srq, struct ib_srq_attr *srq_attr);
-	int (*destroy_srq)(struct ib_srq *srq);
+	void (*destroy_srq)(struct ib_srq *srq, struct ib_udata *udata);
 	struct ib_qp *(*create_qp)(struct ib_pd *pd,
 				   struct ib_qp_init_attr *qp_init_attr,
 				   struct ib_udata *udata);
@@ -2418,13 +2453,12 @@ struct ib_device_ops {
 			 int qp_attr_mask, struct ib_udata *udata);
 	int (*query_qp)(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
 			int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr);
-	int (*destroy_qp)(struct ib_qp *qp);
+	int (*destroy_qp)(struct ib_qp *qp, struct ib_udata *udata);
 	struct ib_cq *(*create_cq)(struct ib_device *device,
 				   const struct ib_cq_init_attr *attr,
-				   struct ib_ucontext *context,
 				   struct ib_udata *udata);
 	int (*modify_cq)(struct ib_cq *cq, u16 cq_count, u16 cq_period);
-	int (*destroy_cq)(struct ib_cq *cq);
+	int (*destroy_cq)(struct ib_cq *cq, struct ib_udata *udata);
 	int (*resize_cq)(struct ib_cq *cq, int cqe, struct ib_udata *udata);
 	struct ib_mr *(*get_dma_mr)(struct ib_pd *pd, int mr_access_flags);
 	struct ib_mr *(*reg_user_mr)(struct ib_pd *pd, u64 start, u64 length,
@@ -2433,9 +2467,9 @@ struct ib_device_ops {
 	int (*rereg_user_mr)(struct ib_mr *mr, int flags, u64 start, u64 length,
 			     u64 virt_addr, int mr_access_flags,
 			     struct ib_pd *pd, struct ib_udata *udata);
-	int (*dereg_mr)(struct ib_mr *mr);
+	int (*dereg_mr)(struct ib_mr *mr, struct ib_udata *udata);
 	struct ib_mr *(*alloc_mr)(struct ib_pd *pd, enum ib_mr_type mr_type,
-				  u32 max_num_sg);
+				  u32 max_num_sg, struct ib_udata *udata);
 	int (*advise_mr)(struct ib_pd *pd,
 			 enum ib_uverbs_advise_mr_advice advice, u32 flags,
 			 struct ib_sge *sg_list, u32 num_sge,
@@ -2456,9 +2490,8 @@ struct ib_device_ops {
 	int (*attach_mcast)(struct ib_qp *qp, union ib_gid *gid, u16 lid);
 	int (*detach_mcast)(struct ib_qp *qp, union ib_gid *gid, u16 lid);
 	struct ib_xrcd *(*alloc_xrcd)(struct ib_device *device,
-				      struct ib_ucontext *ucontext,
 				      struct ib_udata *udata);
-	int (*dealloc_xrcd)(struct ib_xrcd *xrcd);
+	int (*dealloc_xrcd)(struct ib_xrcd *xrcd, struct ib_udata *udata);
 	struct ib_flow *(*create_flow)(struct ib_qp *qp,
 				       struct ib_flow_attr *flow_attr,
 				       int domain, struct ib_udata *udata);
@@ -2483,7 +2516,7 @@ struct ib_device_ops {
 	struct ib_wq *(*create_wq)(struct ib_pd *pd,
 				   struct ib_wq_init_attr *init_attr,
 				   struct ib_udata *udata);
-	int (*destroy_wq)(struct ib_wq *wq);
+	int (*destroy_wq)(struct ib_wq *wq, struct ib_udata *udata);
 	int (*modify_wq)(struct ib_wq *wq, struct ib_wq_attr *attr,
 			 u32 wq_attr_mask, struct ib_udata *udata);
 	struct ib_rwq_ind_table *(*create_rwq_ind_table)(
@@ -2495,7 +2528,7 @@ struct ib_device_ops {
 				  struct ib_ucontext *context,
 				  struct ib_dm_alloc_attr *attr,
 				  struct uverbs_attr_bundle *attrs);
-	int (*dealloc_dm)(struct ib_dm *dm);
+	int (*dealloc_dm)(struct ib_dm *dm, struct uverbs_attr_bundle *attrs);
 	struct ib_mr *(*reg_dm_mr)(struct ib_pd *pd, struct ib_dm *dm,
 				   struct ib_dm_mr_attr *attr,
 				   struct uverbs_attr_bundle *attrs);
@@ -2550,12 +2583,37 @@ struct ib_device_ops {
 	 */
 	void (*dealloc_driver)(struct ib_device *dev);
 
+	/* iWarp CM callbacks */
+	void (*iw_add_ref)(struct ib_qp *qp);
+	void (*iw_rem_ref)(struct ib_qp *qp);
+	struct ib_qp *(*iw_get_qp)(struct ib_device *device, int qpn);
+	int (*iw_connect)(struct iw_cm_id *cm_id,
+			  struct iw_cm_conn_param *conn_param);
+	int (*iw_accept)(struct iw_cm_id *cm_id,
+			 struct iw_cm_conn_param *conn_param);
+	int (*iw_reject)(struct iw_cm_id *cm_id, const void *pdata,
+			 u8 pdata_len);
+	int (*iw_create_listen)(struct iw_cm_id *cm_id, int backlog);
+	int (*iw_destroy_listen)(struct iw_cm_id *cm_id);
+
+	DECLARE_RDMA_OBJ_SIZE(ib_ah);
 	DECLARE_RDMA_OBJ_SIZE(ib_pd);
+	DECLARE_RDMA_OBJ_SIZE(ib_srq);
 	DECLARE_RDMA_OBJ_SIZE(ib_ucontext);
 };
 
-struct rdma_restrack_root;
+struct ib_core_device {
+	/* device must be the first element in structure until,
+	 * union of ib_core_device and device exists in ib_device.
+	 */
+	struct device dev;
+	possible_net_t rdma_net;
+	struct kobject *ports_kobj;
+	struct list_head port_list;
+	struct ib_device *owner; /* reach back to owner ib_device */
+};
 
+struct rdma_restrack_root;
 struct ib_device {
 	/* Do not access @dma_device directly from ULP nor from HW drivers. */
 	struct device                *dma_device;
@@ -2578,19 +2636,18 @@ struct ib_device {
 
 	int			      num_comp_vectors;
 
-	struct iw_cm_verbs	     *iwcm;
-
 	struct module               *owner;
-	struct device                dev;
+	union {
+		struct device		dev;
+		struct ib_core_device	coredev;
+	};
+
 	/* First group for device attributes,
 	 * Second group for driver provided attributes (optional).
 	 * It is NULL terminated array.
 	 */
 	const struct attribute_group	*groups[3];
 
-	struct kobject			*ports_kobj;
-	struct list_head             port_list;
-
 	int			     uverbs_abi_ver;
 	u64			     uverbs_cmd_mask;
 	u64			     uverbs_ex_cmd_mask;
@@ -2626,6 +2683,15 @@ struct ib_device {
 	struct work_struct unregistration_work;
 
 	const struct rdma_link_ops *link_ops;
+
+	/* Protects compat_devs xarray modifications */
+	struct mutex compat_devs_mutex;
+	/* Maintains compat devices for each net namespace */
+	struct xarray compat_devs;
+
+	/* Used by iWarp CM */
+	char iw_ifname[IFNAMSIZ];
+	u32 iw_driver_flags;
 };
 
 struct ib_client {
@@ -2662,6 +2728,21 @@ struct ib_client {
 	u8 no_kverbs_req:1;
 };
 
+/*
+ * IB block DMA iterator
+ *
+ * Iterates the DMA-mapped SGL in contiguous memory blocks aligned
+ * to a HW supported page size.
+ */
+struct ib_block_iter {
+	/* internal states */
+	struct scatterlist *__sg;	/* sg holding the current aligned block */
+	dma_addr_t __dma_addr;		/* unaligned DMA address of this block */
+	unsigned int __sg_nents;	/* number of SG entries */
+	unsigned int __sg_advance;	/* number of bytes to advance in sg in next step */
+	unsigned int __pg_bit;		/* alignment of current block */
+};
+
 struct ib_device *_ib_alloc_device(size_t size);
 #define ib_alloc_device(drv_struct, member)                                    \
 	container_of(_ib_alloc_device(sizeof(struct drv_struct) +              \
@@ -2682,6 +2763,38 @@ void ib_unregister_device_queued(struct ib_device *ib_dev);
 int ib_register_client   (struct ib_client *client);
 void ib_unregister_client(struct ib_client *client);
 
+void __rdma_block_iter_start(struct ib_block_iter *biter,
+			     struct scatterlist *sglist,
+			     unsigned int nents,
+			     unsigned long pgsz);
+bool __rdma_block_iter_next(struct ib_block_iter *biter);
+
+/**
+ * rdma_block_iter_dma_address - get the aligned dma address of the current
+ * block held by the block iterator.
+ * @biter: block iterator holding the memory block
+ */
+static inline dma_addr_t
+rdma_block_iter_dma_address(struct ib_block_iter *biter)
+{
+	return biter->__dma_addr & ~(BIT_ULL(biter->__pg_bit) - 1);
+}
+
+/**
+ * rdma_for_each_block - iterate over contiguous memory blocks of the sg list
+ * @sglist: sglist to iterate over
+ * @biter: block iterator holding the memory block
+ * @nents: maximum number of sg entries to iterate over
+ * @pgsz: best HW supported page size to use
+ *
+ * Callers may use rdma_block_iter_dma_address() to get each
+ * blocks aligned DMA address.
+ */
+#define rdma_for_each_block(sglist, biter, nents, pgsz)		\
+	for (__rdma_block_iter_start(biter, sglist, nents,	\
+				     pgsz);			\
+	     __rdma_block_iter_next(biter);)
+
 /**
  * ib_get_client_data - Get IB client context
  * @device:Device to get context for
@@ -2705,9 +2818,6 @@ void ib_set_device_ops(struct ib_device *device,
 #if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS)
 int rdma_user_mmap_io(struct ib_ucontext *ucontext, struct vm_area_struct *vma,
 		      unsigned long pfn, unsigned long size, pgprot_t prot);
-int rdma_user_mmap_page(struct ib_ucontext *ucontext,
-			struct vm_area_struct *vma, struct page *page,
-			unsigned long size);
 #else
 static inline int rdma_user_mmap_io(struct ib_ucontext *ucontext,
 				    struct vm_area_struct *vma,
@@ -2716,12 +2826,6 @@ static inline int rdma_user_mmap_io(struct ib_ucontext *ucontext,
 {
 	return -EINVAL;
 }
-static inline int rdma_user_mmap_page(struct ib_ucontext *ucontext,
-				struct vm_area_struct *vma, struct page *page,
-				unsigned long size)
-{
-	return -EINVAL;
-}
 #endif
 
 static inline int ib_copy_from_udata(void *dest, struct ib_udata *udata, size_t len)
@@ -2978,8 +3082,8 @@ static inline bool rdma_cap_ib_mad(const struct ib_device *device, u8 port_num)
  */
 static inline bool rdma_cap_opa_mad(struct ib_device *device, u8 port_num)
 {
-	return (device->port_data[port_num].immutable.core_cap_flags &
-		RDMA_CORE_CAP_OPA_MAD) == RDMA_CORE_CAP_OPA_MAD;
+	return device->port_data[port_num].immutable.core_cap_flags &
+		RDMA_CORE_CAP_OPA_MAD;
 }
 
 /**
@@ -3195,6 +3299,30 @@ static inline bool rdma_cap_read_inv(struct ib_device *dev, u32 port_num)
 	return rdma_protocol_iwarp(dev, port_num);
 }
 
+/**
+ * rdma_find_pg_bit - Find page bit given address and HW supported page sizes
+ *
+ * @addr: address
+ * @pgsz_bitmap: bitmap of HW supported page sizes
+ */
+static inline unsigned int rdma_find_pg_bit(unsigned long addr,
+					    unsigned long pgsz_bitmap)
+{
+	unsigned long align;
+	unsigned long pgsz;
+
+	align = addr & -addr;
+
+	/* Find page bit such that addr is aligned to the highest supported
+	 * HW page size
+	 */
+	pgsz = pgsz_bitmap & ~(-align << 1);
+	if (!pgsz)
+		return __ffs(pgsz_bitmap);
+
+	return __fls(pgsz);
+}
+
 int ib_set_vf_link_state(struct ib_device *device, int vf, u8 port,
 			 int state);
 int ib_get_vf_config(struct ib_device *device, int vf, u8 port,
@@ -3236,9 +3364,27 @@ enum ib_pd_flags {
 
 struct ib_pd *__ib_alloc_pd(struct ib_device *device, unsigned int flags,
 		const char *caller);
+
 #define ib_alloc_pd(device, flags) \
 	__ib_alloc_pd((device), (flags), KBUILD_MODNAME)
-void ib_dealloc_pd(struct ib_pd *pd);
+
+/**
+ * ib_dealloc_pd_user - Deallocate kernel/user PD
+ * @pd: The protection domain
+ * @udata: Valid user data or NULL for kernel objects
+ */
+void ib_dealloc_pd_user(struct ib_pd *pd, struct ib_udata *udata);
+
+/**
+ * ib_dealloc_pd - Deallocate kernel PD
+ * @pd: The protection domain
+ *
+ * NOTE: for user PD use ib_dealloc_pd_user with valid udata!
+ */
+static inline void ib_dealloc_pd(struct ib_pd *pd)
+{
+	ib_dealloc_pd_user(pd, NULL);
+}
 
 enum rdma_create_ah_flags {
 	/* In a sleepable context */
@@ -3351,11 +3497,24 @@ enum rdma_destroy_ah_flags {
 };
 
 /**
- * rdma_destroy_ah - Destroys an address handle.
+ * rdma_destroy_ah_user - Destroys an address handle.
  * @ah: The address handle to destroy.
  * @flags: Destroy address handle flags (see enum rdma_destroy_ah_flags).
+ * @udata: Valid user data or NULL for kernel objects
  */
-int rdma_destroy_ah(struct ib_ah *ah, u32 flags);
+int rdma_destroy_ah_user(struct ib_ah *ah, u32 flags, struct ib_udata *udata);
+
+/**
+ * rdma_destroy_ah - Destroys an kernel address handle.
+ * @ah: The address handle to destroy.
+ * @flags: Destroy address handle flags (see enum rdma_destroy_ah_flags).
+ *
+ * NOTE: for user ah use rdma_destroy_ah_user with valid udata!
+ */
+static inline int rdma_destroy_ah(struct ib_ah *ah, u32 flags)
+{
+	return rdma_destroy_ah_user(ah, flags, NULL);
+}
 
 /**
  * ib_create_srq - Creates a SRQ associated with the specified protection
@@ -3399,10 +3558,22 @@ int ib_query_srq(struct ib_srq *srq,
 		 struct ib_srq_attr *srq_attr);
 
 /**
- * ib_destroy_srq - Destroys the specified SRQ.
+ * ib_destroy_srq_user - Destroys the specified SRQ.
  * @srq: The SRQ to destroy.
+ * @udata: Valid user data or NULL for kernel objects
  */
-int ib_destroy_srq(struct ib_srq *srq);
+int ib_destroy_srq_user(struct ib_srq *srq, struct ib_udata *udata);
+
+/**
+ * ib_destroy_srq - Destroys the specified kernel SRQ.
+ * @srq: The SRQ to destroy.
+ *
+ * NOTE: for user srq use ib_destroy_srq_user with valid udata!
+ */
+static inline int ib_destroy_srq(struct ib_srq *srq)
+{
+	return ib_destroy_srq_user(srq, NULL);
+}
 
 /**
  * ib_post_srq_recv - Posts a list of work requests to the specified SRQ.
@@ -3422,15 +3593,34 @@ static inline int ib_post_srq_recv(struct ib_srq *srq,
 }
 
 /**
- * ib_create_qp - Creates a QP associated with the specified protection
+ * ib_create_qp_user - Creates a QP associated with the specified protection
  *   domain.
  * @pd: The protection domain associated with the QP.
  * @qp_init_attr: A list of initial attributes required to create the
  *   QP.  If QP creation succeeds, then the attributes are updated to
  *   the actual capabilities of the created QP.
+ * @udata: Valid user data or NULL for kernel objects
  */
-struct ib_qp *ib_create_qp(struct ib_pd *pd,
-			   struct ib_qp_init_attr *qp_init_attr);
+struct ib_qp *ib_create_qp_user(struct ib_pd *pd,
+				struct ib_qp_init_attr *qp_init_attr,
+				struct ib_udata *udata);
+
+/**
+ * ib_create_qp - Creates a kernel QP associated with the specified protection
+ *   domain.
+ * @pd: The protection domain associated with the QP.
+ * @qp_init_attr: A list of initial attributes required to create the
+ *   QP.  If QP creation succeeds, then the attributes are updated to
+ *   the actual capabilities of the created QP.
+ * @udata: Valid user data or NULL for kernel objects
+ *
+ * NOTE: for user qp use ib_create_qp_user with valid udata!
+ */
+static inline struct ib_qp *ib_create_qp(struct ib_pd *pd,
+					 struct ib_qp_init_attr *qp_init_attr)
+{
+	return ib_create_qp_user(pd, qp_init_attr, NULL);
+}
 
 /**
  * ib_modify_qp_with_udata - Modifies the attributes for the specified QP.
@@ -3480,8 +3670,20 @@ int ib_query_qp(struct ib_qp *qp,
 /**
  * ib_destroy_qp - Destroys the specified QP.
  * @qp: The QP to destroy.
+ * @udata: Valid udata or NULL for kernel objects
  */
-int ib_destroy_qp(struct ib_qp *qp);
+int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata);
+
+/**
+ * ib_destroy_qp - Destroys the specified kernel QP.
+ * @qp: The QP to destroy.
+ *
+ * NOTE: for user qp use ib_destroy_qp_user with valid udata!
+ */
+static inline int ib_destroy_qp(struct ib_qp *qp)
+{
+	return ib_destroy_qp_user(qp, NULL);
+}
 
 /**
  * ib_open_qp - Obtain a reference to an existing sharable QP.
@@ -3541,13 +3743,66 @@ static inline int ib_post_recv(struct ib_qp *qp,
 	return qp->device->ops.post_recv(qp, recv_wr, bad_recv_wr ? : &dummy);
 }
 
-struct ib_cq *__ib_alloc_cq(struct ib_device *dev, void *private,
-			    int nr_cqe, int comp_vector,
-			    enum ib_poll_context poll_ctx, const char *caller);
-#define ib_alloc_cq(device, priv, nr_cqe, comp_vect, poll_ctx) \
-	__ib_alloc_cq((device), (priv), (nr_cqe), (comp_vect), (poll_ctx), KBUILD_MODNAME)
+struct ib_cq *__ib_alloc_cq_user(struct ib_device *dev, void *private,
+				 int nr_cqe, int comp_vector,
+				 enum ib_poll_context poll_ctx,
+				 const char *caller, struct ib_udata *udata);
+
+/**
+ * ib_alloc_cq_user: Allocate kernel/user CQ
+ * @dev: The IB device
+ * @private: Private data attached to the CQE
+ * @nr_cqe: Number of CQEs in the CQ
+ * @comp_vector: Completion vector used for the IRQs
+ * @poll_ctx: Context used for polling the CQ
+ * @udata: Valid user data or NULL for kernel objects
+ */
+static inline struct ib_cq *ib_alloc_cq_user(struct ib_device *dev,
+					     void *private, int nr_cqe,
+					     int comp_vector,
+					     enum ib_poll_context poll_ctx,
+					     struct ib_udata *udata)
+{
+	return __ib_alloc_cq_user(dev, private, nr_cqe, comp_vector, poll_ctx,
+				  KBUILD_MODNAME, udata);
+}
+
+/**
+ * ib_alloc_cq: Allocate kernel CQ
+ * @dev: The IB device
+ * @private: Private data attached to the CQE
+ * @nr_cqe: Number of CQEs in the CQ
+ * @comp_vector: Completion vector used for the IRQs
+ * @poll_ctx: Context used for polling the CQ
+ *
+ * NOTE: for user cq use ib_alloc_cq_user with valid udata!
+ */
+static inline struct ib_cq *ib_alloc_cq(struct ib_device *dev, void *private,
+					int nr_cqe, int comp_vector,
+					enum ib_poll_context poll_ctx)
+{
+	return ib_alloc_cq_user(dev, private, nr_cqe, comp_vector, poll_ctx,
+				NULL);
+}
+
+/**
+ * ib_free_cq_user - Free kernel/user CQ
+ * @cq: The CQ to free
+ * @udata: Valid user data or NULL for kernel objects
+ */
+void ib_free_cq_user(struct ib_cq *cq, struct ib_udata *udata);
+
+/**
+ * ib_free_cq - Free kernel CQ
+ * @cq: The CQ to free
+ *
+ * NOTE: for user cq use ib_free_cq_user with valid udata!
+ */
+static inline void ib_free_cq(struct ib_cq *cq)
+{
+	ib_free_cq_user(cq, NULL);
+}
 
-void ib_free_cq(struct ib_cq *cq);
 int ib_process_cq_direct(struct ib_cq *cq, int budget);
 
 /**
@@ -3591,10 +3846,22 @@ int ib_resize_cq(struct ib_cq *cq, int cqe);
 int rdma_set_cq_moderation(struct ib_cq *cq, u16 cq_count, u16 cq_period);
 
 /**
- * ib_destroy_cq - Destroys the specified CQ.
+ * ib_destroy_cq_user - Destroys the specified CQ.
  * @cq: The CQ to destroy.
+ * @udata: Valid user data or NULL for kernel objects
  */
-int ib_destroy_cq(struct ib_cq *cq);
+int ib_destroy_cq_user(struct ib_cq *cq, struct ib_udata *udata);
+
+/**
+ * ib_destroy_cq - Destroys the specified kernel CQ.
+ * @cq: The CQ to destroy.
+ *
+ * NOTE: for user cq use ib_destroy_cq_user with valid udata!
+ */
+static inline int ib_destroy_cq(struct ib_cq *cq)
+{
+	return ib_destroy_cq_user(cq, NULL);
+}
 
 /**
  * ib_poll_cq - poll a CQ for completion(s)
@@ -3848,17 +4115,37 @@ static inline void ib_dma_free_coherent(struct ib_device *dev,
 }
 
 /**
- * ib_dereg_mr - Deregisters a memory region and removes it from the
+ * ib_dereg_mr_user - Deregisters a memory region and removes it from the
+ *   HCA translation table.
+ * @mr: The memory region to deregister.
+ * @udata: Valid user data or NULL for kernel object
+ *
+ * This function can fail, if the memory region has memory windows bound to it.
+ */
+int ib_dereg_mr_user(struct ib_mr *mr, struct ib_udata *udata);
+
+/**
+ * ib_dereg_mr - Deregisters a kernel memory region and removes it from the
  *   HCA translation table.
  * @mr: The memory region to deregister.
  *
  * This function can fail, if the memory region has memory windows bound to it.
+ *
+ * NOTE: for user mr use ib_dereg_mr_user with valid udata!
  */
-int ib_dereg_mr(struct ib_mr *mr);
+static inline int ib_dereg_mr(struct ib_mr *mr)
+{
+	return ib_dereg_mr_user(mr, NULL);
+}
+
+struct ib_mr *ib_alloc_mr_user(struct ib_pd *pd, enum ib_mr_type mr_type,
+			       u32 max_num_sg, struct ib_udata *udata);
 
-struct ib_mr *ib_alloc_mr(struct ib_pd *pd,
-			  enum ib_mr_type mr_type,
-			  u32 max_num_sg);
+static inline struct ib_mr *ib_alloc_mr(struct ib_pd *pd,
+					enum ib_mr_type mr_type, u32 max_num_sg)
+{
+	return ib_alloc_mr_user(pd, mr_type, max_num_sg, NULL);
+}
 
 /**
  * ib_update_fast_reg_key - updates the key portion of the fast_reg MR
@@ -3956,8 +4243,9 @@ struct ib_xrcd *__ib_alloc_xrcd(struct ib_device *device, const char *caller);
 /**
  * ib_dealloc_xrcd - Deallocates an XRC domain.
  * @xrcd: The XRC domain to deallocate.
+ * @udata: Valid user data or NULL for kernel object
  */
-int ib_dealloc_xrcd(struct ib_xrcd *xrcd);
+int ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata);
 
 static inline int ib_check_mr_access(int flags)
 {
@@ -4033,7 +4321,7 @@ struct net_device *ib_device_netdev(struct ib_device *dev, u8 port);
 
 struct ib_wq *ib_create_wq(struct ib_pd *pd,
 			   struct ib_wq_init_attr *init_attr);
-int ib_destroy_wq(struct ib_wq *wq);
+int ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata);
 int ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *attr,
 		 u32 wq_attr_mask);
 struct ib_rwq_ind_table *ib_create_rwq_ind_table(struct ib_device *device,
@@ -4349,7 +4637,10 @@ rdma_set_device_sysfs_group(struct ib_device *dev,
  */
 static inline struct ib_device *rdma_device_to_ibdev(struct device *device)
 {
-	return container_of(device, struct ib_device, dev);
+	struct ib_core_device *coredev =
+		container_of(device, struct ib_core_device, dev);
+
+	return coredev->owner;
 }
 
 /**
@@ -4362,4 +4653,7 @@ static inline struct ib_device *rdma_device_to_ibdev(struct device *device)
  */
 #define rdma_device_to_drv_device(dev, drv_dev_struct, ibdev_member)           \
 	container_of(rdma_device_to_ibdev(dev), drv_dev_struct, ibdev_member)
+
+bool rdma_dev_access_netns(const struct ib_device *device,
+			   const struct net *net);
 #endif /* IB_VERBS_H */
diff --git a/include/rdma/iw_cm.h b/include/rdma/iw_cm.h
index 0e1f02815643..5aa8a9c76aa0 100644
--- a/include/rdma/iw_cm.h
+++ b/include/rdma/iw_cm.h
@@ -118,31 +118,6 @@ enum iw_flags {
 	IW_F_NO_PORT_MAP = (1 << 0),
 };
 
-struct iw_cm_verbs {
-	void		(*add_ref)(struct ib_qp *qp);
-
-	void		(*rem_ref)(struct ib_qp *qp);
-
-	struct ib_qp *	(*get_qp)(struct ib_device *device,
-				  int qpn);
-
-	int		(*connect)(struct iw_cm_id *cm_id,
-				   struct iw_cm_conn_param *conn_param);
-
-	int		(*accept)(struct iw_cm_id *cm_id,
-				  struct iw_cm_conn_param *conn_param);
-
-	int		(*reject)(struct iw_cm_id *cm_id,
-				  const void *pdata, u8 pdata_len);
-
-	int		(*create_listen)(struct iw_cm_id *cm_id,
-					 int backlog);
-
-	int		(*destroy_listen)(struct iw_cm_id *cm_id);
-	char		ifname[IFNAMSIZ];
-	enum iw_flags	driver_flags;
-};
-
 /**
  * iw_create_cm_id - Create an IW CM identifier.
  *
diff --git a/include/rdma/opa_port_info.h b/include/rdma/opa_port_info.h
index b4f0ac02f283..7147a9263011 100644
--- a/include/rdma/opa_port_info.h
+++ b/include/rdma/opa_port_info.h
@@ -413,6 +413,6 @@ struct opa_port_info {
 	u8     local_port_num;
 	u8     reserved12;
 	u8     reserved13;                       /* was guid_cap */
-} __attribute__ ((packed));
+} __packed;
 
 #endif /* OPA_PORT_INFO_H */
diff --git a/include/rdma/opa_smi.h b/include/rdma/opa_smi.h
index f7896117936e..c7b2ef12792d 100644
--- a/include/rdma/opa_smi.h
+++ b/include/rdma/opa_smi.h
@@ -98,7 +98,7 @@ struct opa_smp {
 
 struct opa_node_description {
 	u8 data[64];
-} __attribute__ ((packed));
+} __packed;
 
 struct opa_node_info {
 	u8      base_version;
@@ -114,7 +114,7 @@ struct opa_node_info {
 	__be32  revision;
 	u8      local_port_num;
 	u8      vendor_id[3];   /* network byte order */
-} __attribute__ ((packed));
+} __packed;
 
 #define OPA_PARTITION_TABLE_BLK_SIZE 32
 
diff --git a/include/rdma/rdma_vt.h b/include/rdma/rdma_vt.h
index 4c257aff7d32..b9cd06db1a71 100644
--- a/include/rdma/rdma_vt.h
+++ b/include/rdma/rdma_vt.h
@@ -59,7 +59,6 @@
 #include <rdma/ib_verbs.h>
 #include <rdma/ib_mad.h>
 #include <rdma/rdmavt_mr.h>
-#include <rdma/rdmavt_qp.h>
 
 #define RVT_MAX_PKEY_VALUES 16
 
@@ -72,6 +71,8 @@ struct trap_list {
 	struct list_head list;
 };
 
+struct rvt_qp;
+struct rvt_qpn_table;
 struct rvt_ibport {
 	struct rvt_qp __rcu *qp[2];
 	struct ib_mad_agent *send_agent;	/* agent for SMI (traps) */
@@ -206,6 +207,20 @@ struct rvt_ah {
 	u8 log_pmtu;
 };
 
+/*
+ * This structure is used by rvt_mmap() to validate an offset
+ * when an mmap() request is made.  The vm_area_struct then uses
+ * this as its vm_private_data.
+ */
+struct rvt_mmap_info {
+	struct list_head pending_mmaps;
+	struct ib_ucontext *context;
+	void *obj;
+	__u64 offset;
+	struct kref ref;
+	u32 size;
+};
+
 /* memory working set size */
 struct rvt_wss {
 	unsigned long *entries;
@@ -501,16 +516,6 @@ static inline struct rvt_dev_info *ib_to_rvt(struct ib_device *ibdev)
 	return  container_of(ibdev, struct rvt_dev_info, ibdev);
 }
 
-static inline struct rvt_srq *ibsrq_to_rvtsrq(struct ib_srq *ibsrq)
-{
-	return container_of(ibsrq, struct rvt_srq, ibsrq);
-}
-
-static inline struct rvt_qp *ibqp_to_rvtqp(struct ib_qp *ibqp)
-{
-	return container_of(ibqp, struct rvt_qp, ibqp);
-}
-
 static inline unsigned rvt_get_npkeys(struct rvt_dev_info *rdi)
 {
 	/*
@@ -548,57 +553,6 @@ static inline u16 rvt_get_pkey(struct rvt_dev_info *rdi,
 		return rdi->ports[port_index]->pkey_table[index];
 }
 
-/**
- * rvt_lookup_qpn - return the QP with the given QPN
- * @ibp: the ibport
- * @qpn: the QP number to look up
- *
- * The caller must hold the rcu_read_lock(), and keep the lock until
- * the returned qp is no longer in use.
- */
-/* TODO: Remove this and put in rdmavt/qp.h when no longer needed by drivers */
-static inline struct rvt_qp *rvt_lookup_qpn(struct rvt_dev_info *rdi,
-					    struct rvt_ibport *rvp,
-					    u32 qpn) __must_hold(RCU)
-{
-	struct rvt_qp *qp = NULL;
-
-	if (unlikely(qpn <= 1)) {
-		qp = rcu_dereference(rvp->qp[qpn]);
-	} else {
-		u32 n = hash_32(qpn, rdi->qp_dev->qp_table_bits);
-
-		for (qp = rcu_dereference(rdi->qp_dev->qp_table[n]); qp;
-			qp = rcu_dereference(qp->next))
-			if (qp->ibqp.qp_num == qpn)
-				break;
-	}
-	return qp;
-}
-
-/**
- * rvt_mod_retry_timer - mod a retry timer
- * @qp - the QP
- * @shift - timeout shift to wait for multiple packets
- * Modify a potentially already running retry timer
- */
-static inline void rvt_mod_retry_timer_ext(struct rvt_qp *qp, u8 shift)
-{
-	struct ib_qp *ibqp = &qp->ibqp;
-	struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device);
-
-	lockdep_assert_held(&qp->s_lock);
-	qp->s_flags |= RVT_S_TIMER;
-	/* 4.096 usec. * (1 << qp->timeout) */
-	mod_timer(&qp->s_timer, jiffies + rdi->busy_jiffies +
-		  (qp->timeout_jiffies << shift));
-}
-
-static inline void rvt_mod_retry_timer(struct rvt_qp *qp)
-{
-	return rvt_mod_retry_timer_ext(qp, 0);
-}
-
 struct rvt_dev_info *rvt_alloc_device(size_t size, int nports);
 void rvt_dealloc_device(struct rvt_dev_info *rdi);
 int rvt_register_device(struct rvt_dev_info *rvd, u32 driver_id);
diff --git a/include/rdma/rdmavt_qp.h b/include/rdma/rdmavt_qp.h
index f0fbd4063fef..68e38c20afc0 100644
--- a/include/rdma/rdmavt_qp.h
+++ b/include/rdma/rdmavt_qp.h
@@ -83,7 +83,6 @@
  * RVT_S_WAIT_DMA - waiting for send DMA queue to drain before generating
  *                  next send completion entry not via send DMA
  * RVT_S_WAIT_PIO - waiting for a send buffer to be available
- * RVT_S_WAIT_PIO_DRAIN - waiting for a qp to drain pio packets
  * RVT_S_WAIT_TX - waiting for a struct verbs_txreq to be available
  * RVT_S_WAIT_DMA_DESC - waiting for DMA descriptors to be available
  * RVT_S_WAIT_KMEM - waiting for kernel memory to be available
@@ -212,20 +211,6 @@ struct rvt_rq {
 };
 
 /*
- * This structure is used by rvt_mmap() to validate an offset
- * when an mmap() request is made.  The vm_area_struct then uses
- * this as its vm_private_data.
- */
-struct rvt_mmap_info {
-	struct list_head pending_mmaps;
-	struct ib_ucontext *context;
-	void *obj;
-	__u64 offset;
-	struct kref ref;
-	unsigned size;
-};
-
-/*
  * This structure holds the information that the send tasklet needs
  * to send a RDMA read response or atomic operation.
  */
@@ -399,6 +384,16 @@ struct rvt_srq {
 	u32 limit;
 };
 
+static inline struct rvt_srq *ibsrq_to_rvtsrq(struct ib_srq *ibsrq)
+{
+	return container_of(ibsrq, struct rvt_srq, ibsrq);
+}
+
+static inline struct rvt_qp *ibqp_to_rvtqp(struct ib_qp *ibqp)
+{
+	return container_of(ibqp, struct rvt_qp, ibqp);
+}
+
 #define RVT_QPN_MAX                 BIT(24)
 #define RVT_QPNMAP_ENTRIES          (RVT_QPN_MAX / PAGE_SIZE / BITS_PER_BYTE)
 #define RVT_BITS_PER_PAGE           (PAGE_SIZE * BITS_PER_BYTE)
@@ -678,6 +673,70 @@ static inline unsigned long rvt_timeout_to_jiffies(u8 timeout)
 	return usecs_to_jiffies(1U << timeout) * 4096UL / 1000UL;
 }
 
+/**
+ * rvt_lookup_qpn - return the QP with the given QPN
+ * @ibp: the ibport
+ * @qpn: the QP number to look up
+ *
+ * The caller must hold the rcu_read_lock(), and keep the lock until
+ * the returned qp is no longer in use.
+ */
+static inline struct rvt_qp *rvt_lookup_qpn(struct rvt_dev_info *rdi,
+					    struct rvt_ibport *rvp,
+					    u32 qpn) __must_hold(RCU)
+{
+	struct rvt_qp *qp = NULL;
+
+	if (unlikely(qpn <= 1)) {
+		qp = rcu_dereference(rvp->qp[qpn]);
+	} else {
+		u32 n = hash_32(qpn, rdi->qp_dev->qp_table_bits);
+
+		for (qp = rcu_dereference(rdi->qp_dev->qp_table[n]); qp;
+			qp = rcu_dereference(qp->next))
+			if (qp->ibqp.qp_num == qpn)
+				break;
+	}
+	return qp;
+}
+
+/**
+ * rvt_mod_retry_timer - mod a retry timer
+ * @qp - the QP
+ * @shift - timeout shift to wait for multiple packets
+ * Modify a potentially already running retry timer
+ */
+static inline void rvt_mod_retry_timer_ext(struct rvt_qp *qp, u8 shift)
+{
+	struct ib_qp *ibqp = &qp->ibqp;
+	struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device);
+
+	lockdep_assert_held(&qp->s_lock);
+	qp->s_flags |= RVT_S_TIMER;
+	/* 4.096 usec. * (1 << qp->timeout) */
+	mod_timer(&qp->s_timer, jiffies + rdi->busy_jiffies +
+		  (qp->timeout_jiffies << shift));
+}
+
+static inline void rvt_mod_retry_timer(struct rvt_qp *qp)
+{
+	return rvt_mod_retry_timer_ext(qp, 0);
+}
+
+/**
+ * rvt_put_qp_swqe - drop refs held by swqe
+ * @qp: the send qp
+ * @wqe: the send wqe
+ *
+ * This drops any references held by the swqe
+ */
+static inline void rvt_put_qp_swqe(struct rvt_qp *qp, struct rvt_swqe *wqe)
+{
+	rvt_put_swqe(wqe);
+	if (qp->allowed_ops == IB_OPCODE_UD)
+		atomic_dec(&ibah_to_rvtah(wqe->ud_wr.ah)->refcount);
+}
+
 extern const int  ib_rvt_state_ops[];
 
 struct rvt_dev_info;
diff --git a/include/rdma/uverbs_std_types.h b/include/rdma/uverbs_std_types.h
index 794c47565971..05eabfd5d0d3 100644
--- a/include/rdma/uverbs_std_types.h
+++ b/include/rdma/uverbs_std_types.h
@@ -48,17 +48,15 @@
 #define uobj_get_type(_attrs, _object)                                         \
 	uapi_get_object((_attrs)->ufile->device->uapi, _object)
 
-struct ib_uobject *_uobj_get_read(enum uverbs_default_objects type,
-				  u32 object_id,
-				  struct uverbs_attr_bundle *attrs);
-
 #define uobj_get_read(_type, _id, _attrs)                                      \
-	_uobj_get_read(_type, _uobj_check_id(_id), _attrs)
+	rdma_lookup_get_uobject(uobj_get_type(_attrs, _type), (_attrs)->ufile, \
+				_uobj_check_id(_id), UVERBS_LOOKUP_READ,       \
+				_attrs)
 
 #define ufd_get_read(_type, _fdnum, _attrs)                                    \
 	rdma_lookup_get_uobject(uobj_get_type(_attrs, _type), (_attrs)->ufile, \
 				(_fdnum)*typecheck(s32, _fdnum),               \
-				UVERBS_LOOKUP_READ)
+				UVERBS_LOOKUP_READ, _attrs)
 
 static inline void *_uobj_get_obj_read(struct ib_uobject *uobj)
 {
@@ -70,22 +68,19 @@ static inline void *_uobj_get_obj_read(struct ib_uobject *uobj)
 	((struct ib_##_object *)_uobj_get_obj_read(                            \
 		uobj_get_read(_type, _id, _attrs)))
 
-struct ib_uobject *_uobj_get_write(enum uverbs_default_objects type,
-				   u32 object_id,
-				   struct uverbs_attr_bundle *attrs);
-
 #define uobj_get_write(_type, _id, _attrs)                                     \
-	_uobj_get_write(_type, _uobj_check_id(_id), _attrs)
+	rdma_lookup_get_uobject(uobj_get_type(_attrs, _type), (_attrs)->ufile, \
+				_uobj_check_id(_id), UVERBS_LOOKUP_WRITE,      \
+				_attrs)
 
 int __uobj_perform_destroy(const struct uverbs_api_object *obj, u32 id,
-			   const struct uverbs_attr_bundle *attrs);
+			   struct uverbs_attr_bundle *attrs);
 #define uobj_perform_destroy(_type, _id, _attrs)                               \
 	__uobj_perform_destroy(uobj_get_type(_attrs, _type),                   \
 			       _uobj_check_id(_id), _attrs)
 
 struct ib_uobject *__uobj_get_destroy(const struct uverbs_api_object *obj,
-				      u32 id,
-				      const struct uverbs_attr_bundle *attrs);
+				      u32 id, struct uverbs_attr_bundle *attrs);
 
 #define uobj_get_destroy(_type, _id, _attrs)                                   \
 	__uobj_get_destroy(uobj_get_type(_attrs, _type), _uobj_check_id(_id),  \
@@ -109,30 +104,31 @@ static inline void uobj_put_write(struct ib_uobject *uobj)
 	rdma_lookup_put_uobject(uobj, UVERBS_LOOKUP_WRITE);
 }
 
-static inline int __must_check uobj_alloc_commit(struct ib_uobject *uobj)
+static inline int __must_check
+uobj_alloc_commit(struct ib_uobject *uobj, struct uverbs_attr_bundle *attrs)
 {
-	int ret = rdma_alloc_commit_uobject(uobj);
+	int ret = rdma_alloc_commit_uobject(uobj, attrs);
 
 	if (ret)
 		return ret;
 	return 0;
 }
 
-static inline void uobj_alloc_abort(struct ib_uobject *uobj)
+static inline void uobj_alloc_abort(struct ib_uobject *uobj,
+				    struct uverbs_attr_bundle *attrs)
 {
-	rdma_alloc_abort_uobject(uobj);
+	rdma_alloc_abort_uobject(uobj, attrs);
 }
 
 static inline struct ib_uobject *
 __uobj_alloc(const struct uverbs_api_object *obj,
 	     struct uverbs_attr_bundle *attrs, struct ib_device **ib_dev)
 {
-	struct ib_uobject *uobj = rdma_alloc_begin_uobject(obj, attrs->ufile);
+	struct ib_uobject *uobj =
+		rdma_alloc_begin_uobject(obj, attrs->ufile, attrs);
 
-	if (!IS_ERR(uobj)) {
-		*ib_dev = uobj->context->device;
-		attrs->context = uobj->context;
-	}
+	if (!IS_ERR(uobj))
+		*ib_dev = attrs->context->device;
 	return uobj;
 }
 
diff --git a/include/rdma/uverbs_types.h b/include/rdma/uverbs_types.h
index 175d761695e1..d57a5ba00c74 100644
--- a/include/rdma/uverbs_types.h
+++ b/include/rdma/uverbs_types.h
@@ -95,7 +95,8 @@ struct uverbs_obj_type_class {
 	void (*lookup_put)(struct ib_uobject *uobj, enum rdma_lookup_mode mode);
 	/* This does not consume the kref on uobj */
 	int __must_check (*destroy_hw)(struct ib_uobject *uobj,
-				       enum rdma_remove_reason why);
+				       enum rdma_remove_reason why,
+				       struct uverbs_attr_bundle *attrs);
 	void (*remove_handle)(struct ib_uobject *uobj);
 	u8    needs_kfree_rcu;
 };
@@ -126,18 +127,23 @@ struct uverbs_obj_idr_type {
 	 * completely unchanged.
 	 */
 	int __must_check (*destroy_object)(struct ib_uobject *uobj,
-					   enum rdma_remove_reason why);
+					   enum rdma_remove_reason why,
+					   struct uverbs_attr_bundle *attrs);
 };
 
 struct ib_uobject *rdma_lookup_get_uobject(const struct uverbs_api_object *obj,
 					   struct ib_uverbs_file *ufile, s64 id,
-					   enum rdma_lookup_mode mode);
+					   enum rdma_lookup_mode mode,
+					   struct uverbs_attr_bundle *attrs);
 void rdma_lookup_put_uobject(struct ib_uobject *uobj,
 			     enum rdma_lookup_mode mode);
 struct ib_uobject *rdma_alloc_begin_uobject(const struct uverbs_api_object *obj,
-					    struct ib_uverbs_file *ufile);
-void rdma_alloc_abort_uobject(struct ib_uobject *uobj);
-int __must_check rdma_alloc_commit_uobject(struct ib_uobject *uobj);
+					    struct ib_uverbs_file *ufile,
+					    struct uverbs_attr_bundle *attrs);
+void rdma_alloc_abort_uobject(struct ib_uobject *uobj,
+			      struct uverbs_attr_bundle *attrs);
+int __must_check rdma_alloc_commit_uobject(struct ib_uobject *uobj,
+					   struct uverbs_attr_bundle *attrs);
 
 struct uverbs_obj_fd_type {
 	/*
diff --git a/include/scsi/libsas.h b/include/scsi/libsas.h
index 56b2dba7d911..b08febec7895 100644
--- a/include/scsi/libsas.h
+++ b/include/scsi/libsas.h
@@ -224,6 +224,13 @@ struct sas_work {
 	struct work_struct work;
 };
 
+/* Lots of code duplicates this in the SCSI tree, which can be factored out */
+static inline bool sas_dev_type_is_expander(enum sas_device_type type)
+{
+	return type == SAS_EDGE_EXPANDER_DEVICE ||
+	       type == SAS_FANOUT_EXPANDER_DEVICE;
+}
+
 static inline void INIT_SAS_WORK(struct sas_work *sw, void (*fn)(struct work_struct *))
 {
 	INIT_WORK(&sw->work, fn);
@@ -245,9 +252,9 @@ static inline struct sas_discovery_event *to_sas_discovery_event(struct work_str
 struct sas_discovery {
 	struct sas_discovery_event disc_work[DISC_NUM_EVENTS];
 	unsigned long    pending;
-	u8     fanout_sas_addr[8];
-	u8     eeds_a[8];
-	u8     eeds_b[8];
+	u8     fanout_sas_addr[SAS_ADDR_SIZE];
+	u8     eeds_a[SAS_ADDR_SIZE];
+	u8     eeds_b[SAS_ADDR_SIZE];
 	int    max_level;
 };
 
diff --git a/include/scsi/osd_attributes.h b/include/scsi/osd_attributes.h
deleted file mode 100644
index 8a6acd054e4e..000000000000
--- a/include/scsi/osd_attributes.h
+++ /dev/null
@@ -1,398 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __OSD_ATTRIBUTES_H__
-#define __OSD_ATTRIBUTES_H__
-
-#include <scsi/osd_protocol.h>
-
-/*
- * Contains types and constants that define attribute pages and attribute
- * numbers and their data types.
- */
-
-#define ATTR_SET(pg, id, l, ptr) \
-	{ .attr_page = pg, .attr_id = id, .len = l, .val_ptr = ptr }
-
-#define ATTR_DEF(pg, id, l) ATTR_SET(pg, id, l, NULL)
-
-/* osd-r10 4.7.3 Attributes pages */
-enum {
-	OSD_APAGE_OBJECT_FIRST		= 0x0,
-	OSD_APAGE_OBJECT_DIRECTORY	= 0,
-	OSD_APAGE_OBJECT_INFORMATION	= 1,
-	OSD_APAGE_OBJECT_QUOTAS		= 2,
-	OSD_APAGE_OBJECT_TIMESTAMP	= 3,
-	OSD_APAGE_OBJECT_COLLECTIONS	= 4,
-	OSD_APAGE_OBJECT_SECURITY	= 5,
-	OSD_APAGE_OBJECT_LAST		= 0x2fffffff,
-
-	OSD_APAGE_PARTITION_FIRST	= 0x30000000,
-	OSD_APAGE_PARTITION_DIRECTORY	= OSD_APAGE_PARTITION_FIRST + 0,
-	OSD_APAGE_PARTITION_INFORMATION = OSD_APAGE_PARTITION_FIRST + 1,
-	OSD_APAGE_PARTITION_QUOTAS	= OSD_APAGE_PARTITION_FIRST + 2,
-	OSD_APAGE_PARTITION_TIMESTAMP	= OSD_APAGE_PARTITION_FIRST + 3,
-	OSD_APAGE_PARTITION_ATTR_ACCESS = OSD_APAGE_PARTITION_FIRST + 4,
-	OSD_APAGE_PARTITION_SECURITY	= OSD_APAGE_PARTITION_FIRST + 5,
-	OSD_APAGE_PARTITION_LAST	= 0x5FFFFFFF,
-
-	OSD_APAGE_COLLECTION_FIRST	= 0x60000000,
-	OSD_APAGE_COLLECTION_DIRECTORY	= OSD_APAGE_COLLECTION_FIRST + 0,
-	OSD_APAGE_COLLECTION_INFORMATION = OSD_APAGE_COLLECTION_FIRST + 1,
-	OSD_APAGE_COLLECTION_TIMESTAMP	= OSD_APAGE_COLLECTION_FIRST + 3,
-	OSD_APAGE_COLLECTION_SECURITY	= OSD_APAGE_COLLECTION_FIRST + 5,
-	OSD_APAGE_COLLECTION_LAST	= 0x8FFFFFFF,
-
-	OSD_APAGE_ROOT_FIRST		= 0x90000000,
-	OSD_APAGE_ROOT_DIRECTORY	= OSD_APAGE_ROOT_FIRST + 0,
-	OSD_APAGE_ROOT_INFORMATION	= OSD_APAGE_ROOT_FIRST + 1,
-	OSD_APAGE_ROOT_QUOTAS		= OSD_APAGE_ROOT_FIRST + 2,
-	OSD_APAGE_ROOT_TIMESTAMP	= OSD_APAGE_ROOT_FIRST + 3,
-	OSD_APAGE_ROOT_SECURITY		= OSD_APAGE_ROOT_FIRST + 5,
-	OSD_APAGE_ROOT_LAST		= 0xBFFFFFFF,
-
-	OSD_APAGE_RESERVED_TYPE_FIRST	= 0xC0000000,
-	OSD_APAGE_RESERVED_TYPE_LAST	= 0xEFFFFFFF,
-
-	OSD_APAGE_COMMON_FIRST		= 0xF0000000,
-	OSD_APAGE_COMMON_LAST		= 0xFFFFFFFD,
-
-	OSD_APAGE_CURRENT_COMMAND	= 0xFFFFFFFE,
-
-	OSD_APAGE_REQUEST_ALL		= 0xFFFFFFFF,
-};
-
-/* subcategories of attr pages within each range above */
-enum {
-	OSD_APAGE_STD_FIRST		= 0x0,
-	OSD_APAGE_STD_DIRECTORY		= 0,
-	OSD_APAGE_STD_INFORMATION	= 1,
-	OSD_APAGE_STD_QUOTAS		= 2,
-	OSD_APAGE_STD_TIMESTAMP		= 3,
-	OSD_APAGE_STD_COLLECTIONS	= 4,
-	OSD_APAGE_STD_POLICY_SECURITY	= 5,
-	OSD_APAGE_STD_LAST		= 0x0000007F,
-
-	OSD_APAGE_RESERVED_FIRST	= 0x00000080,
-	OSD_APAGE_RESERVED_LAST		= 0x00007FFF,
-
-	OSD_APAGE_OTHER_STD_FIRST	= 0x00008000,
-	OSD_APAGE_OTHER_STD_LAST	= 0x0000EFFF,
-
-	OSD_APAGE_PUBLIC_FIRST		= 0x0000F000,
-	OSD_APAGE_PUBLIC_LAST		= 0x0000FFFF,
-
-	OSD_APAGE_APP_DEFINED_FIRST	= 0x00010000,
-	OSD_APAGE_APP_DEFINED_LAST	= 0x1FFFFFFF,
-
-	OSD_APAGE_VENDOR_SPECIFIC_FIRST	= 0x20000000,
-	OSD_APAGE_VENDOR_SPECIFIC_LAST	= 0x2FFFFFFF,
-};
-
-enum {
-	OSD_ATTR_PAGE_IDENTIFICATION = 0, /* in all pages 40 bytes */
-};
-
-struct page_identification {
-	u8 vendor_identification[8];
-	u8 page_identification[32];
-}  __packed;
-
-struct osd_attr_page_header {
-	__be32 page_number;
-	__be32 page_length;
-} __packed;
-
-/* 7.1.2.8 Root Information attributes page (OSD_APAGE_ROOT_INFORMATION) */
-enum {
-	OSD_ATTR_RI_OSD_SYSTEM_ID            = 0x3,   /* 20       */
-	OSD_ATTR_RI_VENDOR_IDENTIFICATION    = 0x4,   /* 8        */
-	OSD_ATTR_RI_PRODUCT_IDENTIFICATION   = 0x5,   /* 16       */
-	OSD_ATTR_RI_PRODUCT_MODEL            = 0x6,   /* 32       */
-	OSD_ATTR_RI_PRODUCT_REVISION_LEVEL   = 0x7,   /* 4        */
-	OSD_ATTR_RI_PRODUCT_SERIAL_NUMBER    = 0x8,   /* variable */
-	OSD_ATTR_RI_OSD_NAME                 = 0x9,   /* variable */
-	OSD_ATTR_RI_MAX_CDB_CONTINUATION_LEN = 0xA,   /* 4        */
-	OSD_ATTR_RI_TOTAL_CAPACITY           = 0x80,  /* 8        */
-	OSD_ATTR_RI_USED_CAPACITY            = 0x81,  /* 8        */
-	OSD_ATTR_RI_NUMBER_OF_PARTITIONS     = 0xC0,  /* 8        */
-	OSD_ATTR_RI_CLOCK                    = 0x100, /* 6        */
-	OARI_DEFAULT_ISOLATION_METHOD        = 0X110, /* 1        */
-	OARI_SUPPORTED_ISOLATION_METHODS     = 0X111, /* 32       */
-
-	OARI_DATA_ATOMICITY_GUARANTEE                   = 0X120,   /* 8       */
-	OARI_DATA_ATOMICITY_ALIGNMENT                   = 0X121,   /* 8       */
-	OARI_ATTRIBUTES_ATOMICITY_GUARANTEE             = 0X122,   /* 8       */
-	OARI_DATA_ATTRIBUTES_ATOMICITY_MULTIPLIER       = 0X123,   /* 1       */
-
-	OARI_MAXIMUM_SNAPSHOTS_COUNT                    = 0X1C1,    /* 0 or 4 */
-	OARI_MAXIMUM_CLONES_COUNT                       = 0X1C2,    /* 0 or 4 */
-	OARI_MAXIMUM_BRANCH_DEPTH                       = 0X1CC,    /* 0 or 4 */
-	OARI_SUPPORTED_OBJECT_DUPLICATION_METHOD_FIRST  = 0X200,    /* 0 or 4 */
-	OARI_SUPPORTED_OBJECT_DUPLICATION_METHOD_LAST   = 0X2ff,    /* 0 or 4 */
-	OARI_SUPPORTED_TIME_OF_DUPLICATION_METHOD_FIRST = 0X300,    /* 0 or 4 */
-	OARI_SUPPORTED_TIME_OF_DUPLICATION_METHOD_LAST  = 0X30F,    /* 0 or 4 */
-	OARI_SUPPORT_FOR_DUPLICATED_OBJECT_FREEZING     = 0X310,    /* 0 or 4 */
-	OARI_SUPPORT_FOR_SNAPSHOT_REFRESHING            = 0X311,    /* 0 or 1 */
-	OARI_SUPPORTED_CDB_CONTINUATION_DESC_TYPE_FIRST = 0X7000001,/* 0 or 4 */
-	OARI_SUPPORTED_CDB_CONTINUATION_DESC_TYPE_LAST  = 0X700FFFF,/* 0 or 4 */
-};
-/* Root_Information_attributes_page does not have a get_page structure */
-
-/* 7.1.2.9 Partition Information attributes page
- * (OSD_APAGE_PARTITION_INFORMATION)
- */
-enum {
-	OSD_ATTR_PI_PARTITION_ID            = 0x1,     /* 8        */
-	OSD_ATTR_PI_USERNAME                = 0x9,     /* variable */
-	OSD_ATTR_PI_USED_CAPACITY           = 0x81,    /* 8        */
-	OSD_ATTR_PI_USED_CAPACITY_INCREMENT = 0x84,    /* 0 or 8   */
-	OSD_ATTR_PI_NUMBER_OF_OBJECTS       = 0xC1,    /* 8        */
-
-	OSD_ATTR_PI_ACTUAL_DATA_SPACE                      = 0xD1, /* 0 or 8 */
-	OSD_ATTR_PI_RESERVED_DATA_SPACE                    = 0xD2, /* 0 or 8 */
-	OSD_ATTR_PI_DEFAULT_SNAPSHOT_DUPLICATION_METHOD    = 0x200,/* 0 or 4 */
-	OSD_ATTR_PI_DEFAULT_CLONE_DUPLICATION_METHOD       = 0x201,/* 0 or 4 */
-	OSD_ATTR_PI_DEFAULT_SP_TIME_OF_DUPLICATION         = 0x300,/* 0 or 4 */
-	OSD_ATTR_PI_DEFAULT_CLONE_TIME_OF_DUPLICATION      = 0x301,/* 0 or 4 */
-};
-/* Partition Information attributes page does not have a get_page structure */
-
-/* 7.1.2.10 Collection Information attributes page
- * (OSD_APAGE_COLLECTION_INFORMATION)
- */
-enum {
-	OSD_ATTR_CI_PARTITION_ID           = 0x1,       /* 8        */
-	OSD_ATTR_CI_COLLECTION_OBJECT_ID   = 0x2,       /* 8        */
-	OSD_ATTR_CI_USERNAME               = 0x9,       /* variable */
-	OSD_ATTR_CI_COLLECTION_TYPE        = 0xA,       /* 1        */
-	OSD_ATTR_CI_USED_CAPACITY          = 0x81,      /* 8        */
-};
-/* Collection Information attributes page does not have a get_page structure */
-
-/* 7.1.2.11 User Object Information attributes page
- * (OSD_APAGE_OBJECT_INFORMATION)
- */
-enum {
-	OSD_ATTR_OI_PARTITION_ID         = 0x1,       /* 8        */
-	OSD_ATTR_OI_OBJECT_ID            = 0x2,       /* 8        */
-	OSD_ATTR_OI_USERNAME             = 0x9,       /* variable */
-	OSD_ATTR_OI_USED_CAPACITY        = 0x81,      /* 8        */
-	OSD_ATTR_OI_LOGICAL_LENGTH       = 0x82,      /* 8        */
-	SD_ATTR_OI_ACTUAL_DATA_SPACE     = 0XD1,      /* 0 OR 8   */
-	SD_ATTR_OI_RESERVED_DATA_SPACE   = 0XD2,      /* 0 OR 8   */
-};
-/* Object Information attributes page does not have a get_page structure */
-
-/* 7.1.2.12 Root Quotas attributes page (OSD_APAGE_ROOT_QUOTAS) */
-enum {
-	OSD_ATTR_RQ_DEFAULT_MAXIMUM_USER_OBJECT_LENGTH     = 0x1,      /* 8  */
-	OSD_ATTR_RQ_PARTITION_CAPACITY_QUOTA               = 0x10001,  /* 8  */
-	OSD_ATTR_RQ_PARTITION_OBJECT_COUNT                 = 0x10002,  /* 8  */
-	OSD_ATTR_RQ_PARTITION_COLLECTIONS_PER_USER_OBJECT  = 0x10081,  /* 4  */
-	OSD_ATTR_RQ_PARTITION_COUNT                        = 0x20002,  /* 8  */
-};
-
-struct Root_Quotas_attributes_page {
-	struct osd_attr_page_header hdr; /* id=R+2, size=0x24 */
-	__be64 default_maximum_user_object_length;
-	__be64 partition_capacity_quota;
-	__be64 partition_object_count;
-	__be64 partition_collections_per_user_object;
-	__be64 partition_count;
-}  __packed;
-
-/* 7.1.2.13 Partition Quotas attributes page (OSD_APAGE_PARTITION_QUOTAS)*/
-enum {
-	OSD_ATTR_PQ_DEFAULT_MAXIMUM_USER_OBJECT_LENGTH  = 0x1,        /* 8 */
-	OSD_ATTR_PQ_CAPACITY_QUOTA                      = 0x10001,    /* 8 */
-	OSD_ATTR_PQ_OBJECT_COUNT                        = 0x10002,    /* 8 */
-	OSD_ATTR_PQ_COLLECTIONS_PER_USER_OBJECT         = 0x10081,    /* 4 */
-};
-
-struct Partition_Quotas_attributes_page {
-	struct osd_attr_page_header hdr; /* id=P+2, size=0x1C */
-	__be64 default_maximum_user_object_length;
-	__be64 capacity_quota;
-	__be64 object_count;
-	__be64 collections_per_user_object;
-}  __packed;
-
-/* 7.1.2.14 User Object Quotas attributes page (OSD_APAGE_OBJECT_QUOTAS) */
-enum {
-	OSD_ATTR_OQ_MAXIMUM_LENGTH  = 0x1,        /* 8 */
-};
-
-struct Object_Quotas_attributes_page {
-	struct osd_attr_page_header hdr; /* id=U+2, size=0x8 */
-	__be64 maximum_length;
-}  __packed;
-
-/* 7.1.2.15 Root Timestamps attributes page (OSD_APAGE_ROOT_TIMESTAMP) */
-enum {
-	OSD_ATTR_RT_ATTRIBUTES_ACCESSED_TIME  = 0x2,        /* 6 */
-	OSD_ATTR_RT_ATTRIBUTES_MODIFIED_TIME  = 0x3,        /* 6 */
-	OSD_ATTR_RT_TIMESTAMP_BYPASS          = 0xFFFFFFFE, /* 1 */
-};
-
-struct root_timestamps_attributes_page {
-	struct osd_attr_page_header hdr; /* id=R+3, size=0xD */
-	struct osd_timestamp attributes_accessed_time;
-	struct osd_timestamp attributes_modified_time;
-	u8 timestamp_bypass;
-}  __packed;
-
-/* 7.1.2.16 Partition Timestamps attributes page
- * (OSD_APAGE_PARTITION_TIMESTAMP)
- */
-enum {
-	OSD_ATTR_PT_CREATED_TIME              = 0x1,        /* 6 */
-	OSD_ATTR_PT_ATTRIBUTES_ACCESSED_TIME  = 0x2,        /* 6 */
-	OSD_ATTR_PT_ATTRIBUTES_MODIFIED_TIME  = 0x3,        /* 6 */
-	OSD_ATTR_PT_DATA_ACCESSED_TIME        = 0x4,        /* 6 */
-	OSD_ATTR_PT_DATA_MODIFIED_TIME        = 0x5,        /* 6 */
-	OSD_ATTR_PT_TIMESTAMP_BYPASS          = 0xFFFFFFFE, /* 1 */
-};
-
-struct partition_timestamps_attributes_page {
-	struct osd_attr_page_header hdr; /* id=P+3, size=0x1F */
-	struct osd_timestamp created_time;
-	struct osd_timestamp attributes_accessed_time;
-	struct osd_timestamp attributes_modified_time;
-	struct osd_timestamp data_accessed_time;
-	struct osd_timestamp data_modified_time;
-	u8 timestamp_bypass;
-}  __packed;
-
-/* 7.1.2.17/18 Collection/Object Timestamps attributes page
- * (OSD_APAGE_COLLECTION_TIMESTAMP/OSD_APAGE_OBJECT_TIMESTAMP)
- */
-enum {
-	OSD_ATTR_OT_CREATED_TIME              = 0x1,        /* 6 */
-	OSD_ATTR_OT_ATTRIBUTES_ACCESSED_TIME  = 0x2,        /* 6 */
-	OSD_ATTR_OT_ATTRIBUTES_MODIFIED_TIME  = 0x3,        /* 6 */
-	OSD_ATTR_OT_DATA_ACCESSED_TIME        = 0x4,        /* 6 */
-	OSD_ATTR_OT_DATA_MODIFIED_TIME        = 0x5,        /* 6 */
-};
-
-/* same for collection */
-struct object_timestamps_attributes_page {
-	struct osd_attr_page_header hdr; /* id=C+3/3, size=0x1E */
-	struct osd_timestamp created_time;
-	struct osd_timestamp attributes_accessed_time;
-	struct osd_timestamp attributes_modified_time;
-	struct osd_timestamp data_accessed_time;
-	struct osd_timestamp data_modified_time;
-}  __packed;
-
-/* OSD2r05: 7.1.3.19 Attributes Access attributes page
- * (OSD_APAGE_PARTITION_ATTR_ACCESS)
- *
- * each attribute is of the form below. Total array length is deduced
- * from the attribute's length
- * (See allowed_attributes_access of the struct osd_cap_object_descriptor)
- */
-struct attributes_access_attr {
-	struct osd_attributes_list_attrid attr_list[0];
-} __packed;
-
-/* OSD2r05: 7.1.2.21 Collections attributes page */
-/* TBD */
-
-/* 7.1.2.20 Root Policy/Security attributes page (OSD_APAGE_ROOT_SECURITY) */
-enum {
-	OSD_ATTR_RS_DEFAULT_SECURITY_METHOD           = 0x1,       /* 1      */
-	OSD_ATTR_RS_OLDEST_VALID_NONCE_LIMIT          = 0x2,       /* 6      */
-	OSD_ATTR_RS_NEWEST_VALID_NONCE_LIMIT          = 0x3,       /* 6      */
-	OSD_ATTR_RS_PARTITION_DEFAULT_SECURITY_METHOD = 0x6,       /* 1      */
-	OSD_ATTR_RS_SUPPORTED_SECURITY_METHODS        = 0x7,       /* 2      */
-	OSD_ATTR_RS_ADJUSTABLE_CLOCK                  = 0x9,       /* 6      */
-	OSD_ATTR_RS_MASTER_KEY_IDENTIFIER             = 0x7FFD,    /* 0 or 7 */
-	OSD_ATTR_RS_ROOT_KEY_IDENTIFIER               = 0x7FFE,    /* 0 or 7 */
-	OSD_ATTR_RS_SUPPORTED_INTEGRITY_ALGORITHM_0   = 0x80000000,/* 1,(x16)*/
-	OSD_ATTR_RS_SUPPORTED_DH_GROUP_0              = 0x80000010,/* 1,(x16)*/
-};
-
-struct root_security_attributes_page {
-	struct osd_attr_page_header hdr; /* id=R+5, size=0x3F */
-	u8 default_security_method;
-	u8 partition_default_security_method;
-	__be16 supported_security_methods;
-	u8 mki_valid_rki_valid;
-	struct osd_timestamp oldest_valid_nonce_limit;
-	struct osd_timestamp newest_valid_nonce_limit;
-	struct osd_timestamp adjustable_clock;
-	u8 master_key_identifier[32-25];
-	u8 root_key_identifier[39-32];
-	u8 supported_integrity_algorithm[16];
-	u8 supported_dh_group[16];
-}  __packed;
-
-/* 7.1.2.21 Partition Policy/Security attributes page
- * (OSD_APAGE_PARTITION_SECURITY)
- */
-enum {
-	OSD_ATTR_PS_DEFAULT_SECURITY_METHOD        = 0x1,        /* 1      */
-	OSD_ATTR_PS_OLDEST_VALID_NONCE             = 0x2,        /* 6      */
-	OSD_ATTR_PS_NEWEST_VALID_NONCE             = 0x3,        /* 6      */
-	OSD_ATTR_PS_REQUEST_NONCE_LIST_DEPTH       = 0x4,        /* 2      */
-	OSD_ATTR_PS_FROZEN_WORKING_KEY_BIT_MASK    = 0x5,        /* 2      */
-	OSD_ATTR_PS_PARTITION_KEY_IDENTIFIER       = 0x7FFF,     /* 0 or 7 */
-	OSD_ATTR_PS_WORKING_KEY_IDENTIFIER_FIRST   = 0x8000,     /* 0 or 7 */
-	OSD_ATTR_PS_WORKING_KEY_IDENTIFIER_LAST    = 0x800F,     /* 0 or 7 */
-	OSD_ATTR_PS_POLICY_ACCESS_TAG              = 0x40000001, /* 4      */
-	OSD_ATTR_PS_USER_OBJECT_POLICY_ACCESS_TAG  = 0x40000002, /* 4      */
-};
-
-struct partition_security_attributes_page {
-	struct osd_attr_page_header hdr; /* id=p+5, size=0x8f */
-	u8 reserved[3];
-	u8 default_security_method;
-	struct osd_timestamp oldest_valid_nonce;
-	struct osd_timestamp newest_valid_nonce;
-	__be16 request_nonce_list_depth;
-	__be16 frozen_working_key_bit_mask;
-	__be32 policy_access_tag;
-	__be32 user_object_policy_access_tag;
-	u8 pki_valid;
-	__be16 wki_00_0f_vld;
-	struct osd_key_identifier partition_key_identifier;
-	struct osd_key_identifier working_key_identifiers[16];
-}  __packed;
-
-/* 7.1.2.22/23 Collection/Object Policy-Security attributes page
- * (OSD_APAGE_COLLECTION_SECURITY/OSD_APAGE_OBJECT_SECURITY)
- */
-enum {
-	OSD_ATTR_OS_POLICY_ACCESS_TAG              = 0x40000001, /* 4      */
-};
-
-struct object_security_attributes_page {
-	struct osd_attr_page_header hdr; /* id=C+5/5, size=4 */
-	__be32 policy_access_tag;
-}  __packed;
-
-/* OSD2r05: 7.1.3.31 Current Command attributes page
- * (OSD_APAGE_CURRENT_COMMAND)
- */
-enum {
-	OSD_ATTR_CC_RESPONSE_INTEGRITY_CHECK_VALUE     = 0x1, /* 32  */
-	OSD_ATTR_CC_OBJECT_TYPE                        = 0x2, /* 1   */
-	OSD_ATTR_CC_PARTITION_ID                       = 0x3, /* 8   */
-	OSD_ATTR_CC_OBJECT_ID                          = 0x4, /* 8   */
-	OSD_ATTR_CC_STARTING_BYTE_ADDRESS_OF_APPEND    = 0x5, /* 8   */
-	OSD_ATTR_CC_CHANGE_IN_USED_CAPACITY            = 0x6, /* 8   */
-};
-
-/*TBD: osdv1_current_command_attributes_page */
-
-struct osdv2_current_command_attributes_page {
-	struct osd_attr_page_header hdr;  /* id=0xFFFFFFFE, size=0x44 */
-	u8 response_integrity_check_value[OSD_CRYPTO_KEYID_SIZE];
-	u8 object_type;
-	u8 reserved[3];
-	__be64 partition_id;
-	__be64 object_id;
-	__be64 starting_byte_address_of_append;
-	__be64 change_in_used_capacity;
-};
-
-#endif /*ndef __OSD_ATTRIBUTES_H__*/
diff --git a/include/scsi/osd_protocol.h b/include/scsi/osd_protocol.h
deleted file mode 100644
index e0ca835e7bf7..000000000000
--- a/include/scsi/osd_protocol.h
+++ /dev/null
@@ -1,676 +0,0 @@
-/*
- * osd_protocol.h - OSD T10 standard C definitions.
- *
- * Copyright (C) 2008 Panasas Inc.  All rights reserved.
- *
- * Authors:
- *   Boaz Harrosh <ooo@electrozaur.com>
- *   Benny Halevy <bhalevy@panasas.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2
- *
- * This file contains types and constants that are defined by the protocol
- * Note: All names and symbols are taken from the OSD standard's text.
- */
-#ifndef __OSD_PROTOCOL_H__
-#define __OSD_PROTOCOL_H__
-
-#include <linux/types.h>
-#include <linux/kernel.h>
-#include <asm/unaligned.h>
-#include <scsi/scsi.h>
-
-enum {
-	OSDv1_ADDITIONAL_CDB_LENGTH = 192,
-	OSDv1_TOTAL_CDB_LEN = OSDv1_ADDITIONAL_CDB_LENGTH + 8,
-	OSDv1_CAP_LEN = 80,
-
-	/* Latest supported version */
-	OSDv2_ADDITIONAL_CDB_LENGTH = 228,
-	OSD_ADDITIONAL_CDB_LENGTH =
-		OSDv2_ADDITIONAL_CDB_LENGTH,
-	OSD_TOTAL_CDB_LEN = OSD_ADDITIONAL_CDB_LENGTH + 8,
-	OSD_CAP_LEN = 104,
-
-	OSD_SYSTEMID_LEN = 20,
-	OSDv1_CRYPTO_KEYID_SIZE = 20,
-	OSDv2_CRYPTO_KEYID_SIZE = 32,
-	OSD_CRYPTO_KEYID_SIZE = OSDv2_CRYPTO_KEYID_SIZE,
-	OSD_CRYPTO_SEED_SIZE = 4,
-	OSD_CRYPTO_NONCE_SIZE = 12,
-	OSD_MAX_SENSE_LEN = 252, /* from SPC-3 */
-
-	OSD_PARTITION_FIRST_ID = 0x10000,
-	OSD_OBJECT_FIRST_ID = 0x10000,
-};
-
-/* (osd-r10 5.2.4)
- * osd2r03: 5.2.3 Caching control bits
- */
-enum osd_options_byte {
-	OSD_CDB_FUA = 0x08,	/* Force Unit Access */
-	OSD_CDB_DPO = 0x10,	/* Disable Page Out */
-};
-
-/*
- * osd2r03: 5.2.5 Isolation.
- * First 3 bits, V2-only.
- * Also for attr 110h "default isolation method" at Root Information page
- */
-enum osd_options_byte_isolation {
-	OSD_ISOLATION_DEFAULT = 0,
-	OSD_ISOLATION_NONE = 1,
-	OSD_ISOLATION_STRICT = 2,
-	OSD_ISOLATION_RANGE = 4,
-	OSD_ISOLATION_FUNCTIONAL = 5,
-	OSD_ISOLATION_VENDOR = 7,
-};
-
-/* (osd-r10: 6.7)
- * osd2r03: 6.8 FLUSH, FLUSH COLLECTION, FLUSH OSD, FLUSH PARTITION
- */
-enum osd_options_flush_scope_values {
-	OSD_CDB_FLUSH_ALL = 0,
-	OSD_CDB_FLUSH_ATTR_ONLY = 1,
-
-	OSD_CDB_FLUSH_ALL_RECURSIVE = 2,
-	/* V2-only */
-	OSD_CDB_FLUSH_ALL_RANGE = 2,
-};
-
-/* osd2r03: 5.2.10 Timestamps control */
-enum {
-	OSD_CDB_NORMAL_TIMESTAMPS = 0,
-	OSD_CDB_BYPASS_TIMESTAMPS = 0x7f,
-};
-
-/* (osd-r10: 5.2.2.1)
- * osd2r03: 5.2.4.1 Get and set attributes CDB format selection
- *	2 bits at second nibble of command_specific_options byte
- */
-enum osd_attributes_mode {
-	/* V2-only */
-	OSD_CDB_SET_ONE_ATTR = 0x10,
-
-	OSD_CDB_GET_ATTR_PAGE_SET_ONE = 0x20,
-	OSD_CDB_GET_SET_ATTR_LISTS = 0x30,
-
-	OSD_CDB_GET_SET_ATTR_MASK = 0x30,
-};
-
-/* (osd-r10: 4.12.5)
- * osd2r03: 4.14.5 Data-In and Data-Out buffer offsets
- *	byte offset = mantissa * (2^(exponent+8))
- *	struct {
- *		unsigned mantissa: 28;
- *		int exponent: 04;
- *	}
- */
-typedef __be32 osd_cdb_offset;
-
-enum {
-	OSD_OFFSET_UNUSED = 0xFFFFFFFF,
-	OSD_OFFSET_MAX_BITS = 28,
-
-	OSDv1_OFFSET_MIN_SHIFT = 8,
-	OSD_OFFSET_MIN_SHIFT = 3,
-	OSD_OFFSET_MAX_SHIFT = 16,
-};
-
-/* Return the smallest allowed encoded offset that contains @offset.
- *
- * The actual encoded offset returned is @offset + *padding.
- * (up to max_shift, non-inclusive)
- */
-osd_cdb_offset __osd_encode_offset(u64 offset, unsigned *padding,
-	int min_shift, int max_shift);
-
-/* Minimum alignment is 256 bytes
- * Note: Seems from std v1 that exponent can be from 0+8 to 0xE+8 (inclusive)
- * which is 8 to 23 but IBM code restricts it to 16, so be it.
- */
-static inline osd_cdb_offset osd_encode_offset_v1(u64 offset, unsigned *padding)
-{
-	return __osd_encode_offset(offset, padding,
-				OSDv1_OFFSET_MIN_SHIFT, OSD_OFFSET_MAX_SHIFT);
-}
-
-/* Minimum 8 bytes alignment
- * Same as v1 but since exponent can be signed than a less than
- * 256 alignment can be reached with small offsets (<2GB)
- */
-static inline osd_cdb_offset osd_encode_offset_v2(u64 offset, unsigned *padding)
-{
-	return __osd_encode_offset(offset, padding,
-				   OSD_OFFSET_MIN_SHIFT, OSD_OFFSET_MAX_SHIFT);
-}
-
-/* osd2r03: 5.2.1 Overview */
-struct osd_cdb_head {
-	struct scsi_varlen_cdb_hdr varlen_cdb;
-/*10*/	u8		options;
-	u8		command_specific_options;
-	u8		timestamp_control;
-/*13*/	u8		reserved1[3];
-/*16*/	__be64		partition;
-/*24*/	__be64		object;
-/*32*/	union { /* V1 vs V2 alignment differences */
-		struct __osdv1_cdb_addr_len {
-/*32*/			__be32 		list_identifier;/* Rarely used */
-/*36*/			__be64		length;
-/*44*/			__be64		start_address;
-		} __packed v1;
-
-		struct __osdv2_cdb_addr_len {
-			/* called allocation_length in some commands */
-/*32*/			__be64	length;
-/*40*/			__be64	start_address;
-			union {
-/*48*/				__be32 list_identifier;/* Rarely used */
-				/* OSD2r05 5.2.5 CDB continuation length */
-/*48*/				__be32 cdb_continuation_length;
-			};
-		} __packed v2;
-	};
-/*52*/	union { /* selected attributes mode Page/List/Single */
-		struct osd_attributes_page_mode {
-/*52*/			__be32		get_attr_page;
-/*56*/			__be32		get_attr_alloc_length;
-/*60*/			osd_cdb_offset	get_attr_offset;
-
-/*64*/			__be32		set_attr_page;
-/*68*/			__be32		set_attr_id;
-/*72*/			__be32		set_attr_length;
-/*76*/			osd_cdb_offset	set_attr_offset;
-/*80*/		} __packed attrs_page;
-
-		struct osd_attributes_list_mode {
-/*52*/			__be32		get_attr_desc_bytes;
-/*56*/			osd_cdb_offset	get_attr_desc_offset;
-
-/*60*/			__be32		get_attr_alloc_length;
-/*64*/			osd_cdb_offset	get_attr_offset;
-
-/*68*/			__be32		set_attr_bytes;
-/*72*/			osd_cdb_offset	set_attr_offset;
-			__be32 not_used;
-/*80*/		} __packed attrs_list;
-
-		/* osd2r03:5.2.4.2 Set one attribute value using CDB fields */
-		struct osd_attributes_cdb_mode {
-/*52*/			__be32		set_attr_page;
-/*56*/			__be32		set_attr_id;
-/*60*/			__be16		set_attr_len;
-/*62*/			u8		set_attr_val[18];
-/*80*/		} __packed attrs_cdb;
-/*52*/		u8 get_set_attributes_parameters[28];
-	};
-} __packed;
-/*80*/
-
-/*160 v1*/
-struct osdv1_security_parameters {
-/*160*/u8	integrity_check_value[OSDv1_CRYPTO_KEYID_SIZE];
-/*180*/u8	request_nonce[OSD_CRYPTO_NONCE_SIZE];
-/*192*/osd_cdb_offset	data_in_integrity_check_offset;
-/*196*/osd_cdb_offset	data_out_integrity_check_offset;
-} __packed;
-/*200 v1*/
-
-/*184 v2*/
-struct osdv2_security_parameters {
-/*184*/u8	integrity_check_value[OSDv2_CRYPTO_KEYID_SIZE];
-/*216*/u8	request_nonce[OSD_CRYPTO_NONCE_SIZE];
-/*228*/osd_cdb_offset	data_in_integrity_check_offset;
-/*232*/osd_cdb_offset	data_out_integrity_check_offset;
-} __packed;
-/*236 v2*/
-
-struct osd_security_parameters {
-	union {
-		struct osdv1_security_parameters v1;
-		struct osdv2_security_parameters v2;
-	};
-};
-
-struct osdv1_cdb {
-	struct osd_cdb_head h;
-	u8 caps[OSDv1_CAP_LEN];
-	struct osdv1_security_parameters sec_params;
-} __packed;
-
-struct osdv2_cdb {
-	struct osd_cdb_head h;
-	u8 caps[OSD_CAP_LEN];
-	struct osdv2_security_parameters sec_params;
-} __packed;
-
-struct osd_cdb {
-	union {
-		struct osdv1_cdb v1;
-		struct osdv2_cdb v2;
-		u8 buff[OSD_TOTAL_CDB_LEN];
-	};
-} __packed;
-
-static inline struct osd_cdb_head *osd_cdb_head(struct osd_cdb *ocdb)
-{
-	return (struct osd_cdb_head *)ocdb->buff;
-}
-
-/* define both version actions
- * Ex name = FORMAT_OSD we have OSD_ACT_FORMAT_OSD && OSDv1_ACT_FORMAT_OSD
- */
-#define OSD_ACT___(Name, Num) \
-	OSD_ACT_##Name = cpu_to_be16(0x8880 + Num), \
-	OSDv1_ACT_##Name = cpu_to_be16(0x8800 + Num),
-
-/* V2 only actions */
-#define OSD_ACT_V2(Name, Num) \
-	OSD_ACT_##Name = cpu_to_be16(0x8880 + Num),
-
-#define OSD_ACT_V1_V2(Name, Num1, Num2) \
-	OSD_ACT_##Name = cpu_to_be16(Num2), \
-	OSDv1_ACT_##Name = cpu_to_be16(Num1),
-
-enum osd_service_actions {
-	OSD_ACT_V2(OBJECT_STRUCTURE_CHECK,	0x00)
-	OSD_ACT___(FORMAT_OSD,			0x01)
-	OSD_ACT___(CREATE,			0x02)
-	OSD_ACT___(LIST,			0x03)
-	OSD_ACT_V2(PUNCH,			0x04)
-	OSD_ACT___(READ,			0x05)
-	OSD_ACT___(WRITE,			0x06)
-	OSD_ACT___(APPEND,			0x07)
-	OSD_ACT___(FLUSH,			0x08)
-	OSD_ACT_V2(CLEAR,			0x09)
-	OSD_ACT___(REMOVE,			0x0A)
-	OSD_ACT___(CREATE_PARTITION,		0x0B)
-	OSD_ACT___(REMOVE_PARTITION,		0x0C)
-	OSD_ACT___(GET_ATTRIBUTES,		0x0E)
-	OSD_ACT___(SET_ATTRIBUTES,		0x0F)
-	OSD_ACT___(CREATE_AND_WRITE,		0x12)
-	OSD_ACT___(CREATE_COLLECTION,		0x15)
-	OSD_ACT___(REMOVE_COLLECTION,		0x16)
-	OSD_ACT___(LIST_COLLECTION,		0x17)
-	OSD_ACT___(SET_KEY,			0x18)
-	OSD_ACT___(SET_MASTER_KEY,		0x19)
-	OSD_ACT___(FLUSH_COLLECTION,		0x1A)
-	OSD_ACT___(FLUSH_PARTITION,		0x1B)
-	OSD_ACT___(FLUSH_OSD,			0x1C)
-
-	OSD_ACT_V2(QUERY,			0x20)
-	OSD_ACT_V2(REMOVE_MEMBER_OBJECTS,	0x21)
-	OSD_ACT_V2(GET_MEMBER_ATTRIBUTES,	0x22)
-	OSD_ACT_V2(SET_MEMBER_ATTRIBUTES,	0x23)
-
-	OSD_ACT_V2(CREATE_CLONE,		0x28)
-	OSD_ACT_V2(CREATE_SNAPSHOT,		0x29)
-	OSD_ACT_V2(DETACH_CLONE,		0x2A)
-	OSD_ACT_V2(REFRESH_SNAPSHOT_CLONE,	0x2B)
-	OSD_ACT_V2(RESTORE_PARTITION_FROM_SNAPSHOT, 0x2C)
-
-	OSD_ACT_V2(READ_MAP,			0x31)
-	OSD_ACT_V2(READ_MAPS_COMPARE,		0x32)
-
-	OSD_ACT_V1_V2(PERFORM_SCSI_COMMAND,	0x8F7E, 0x8F7C)
-	OSD_ACT_V1_V2(SCSI_TASK_MANAGEMENT,	0x8F7F, 0x8F7D)
-	/* 0x8F80 to 0x8FFF are Vendor specific */
-};
-
-/* osd2r03: 7.1.3.2 List entry format for retrieving attributes */
-struct osd_attributes_list_attrid {
-	__be32 attr_page;
-	__be32 attr_id;
-} __packed;
-
-/*
- * NOTE: v1: is not aligned.
- */
-struct osdv1_attributes_list_element {
-	__be32 attr_page;
-	__be32 attr_id;
-	__be16 attr_bytes; /* valid bytes at attr_val without padding */
-	u8 attr_val[0];
-} __packed;
-
-/*
- * osd2r03: 7.1.3.3 List entry format for retrieved attributes and
- *                  for setting attributes
- * NOTE: v2 is 8-bytes aligned
- */
-struct osdv2_attributes_list_element {
-	__be32 attr_page;
-	__be32 attr_id;
-	u8 reserved[6];
-	__be16 attr_bytes; /* valid bytes at attr_val without padding */
-	u8 attr_val[0];
-} __packed;
-
-enum {
-	OSDv1_ATTRIBUTES_ELEM_ALIGN = 1,
-	OSD_ATTRIBUTES_ELEM_ALIGN = 8,
-};
-
-enum {
-	OSD_ATTR_LIST_ALL_PAGES = 0xFFFFFFFF,
-	OSD_ATTR_LIST_ALL_IN_PAGE = 0xFFFFFFFF,
-};
-
-static inline unsigned osdv1_attr_list_elem_size(unsigned len)
-{
-	return ALIGN(len + sizeof(struct osdv1_attributes_list_element),
-		     OSDv1_ATTRIBUTES_ELEM_ALIGN);
-}
-
-static inline unsigned osdv2_attr_list_elem_size(unsigned len)
-{
-	return ALIGN(len + sizeof(struct osdv2_attributes_list_element),
-		     OSD_ATTRIBUTES_ELEM_ALIGN);
-}
-
-/*
- * osd2r03: 7.1.3 OSD attributes lists (Table 184) — List type values
- */
-enum osd_attr_list_types {
-	OSD_ATTR_LIST_GET = 0x1, 	/* descriptors only */
-	OSD_ATTR_LIST_SET_RETRIEVE = 0x9, /*descriptors/values variable-length*/
-	OSD_V2_ATTR_LIST_MULTIPLE = 0xE,  /* ver2, Multiple Objects lists*/
-	OSD_V1_ATTR_LIST_CREATE_MULTIPLE = 0xF,/*ver1, used by create_multple*/
-};
-
-/* osd2r03: 7.1.3.4 Multi-object retrieved attributes format */
-struct osd_attributes_list_multi_header {
-	__be64 object_id;
-	u8 object_type; /* object_type enum below */
-	u8 reserved[5];
-	__be16 list_bytes;
-	/* followed by struct osd_attributes_list_element's */
-};
-
-struct osdv1_attributes_list_header {
-	u8 type;	/* low 4-bit only */
-	u8 pad;
-	__be16 list_bytes; /* Initiator shall set to Zero. Only set by target */
-	/*
-	 * type=9 followed by struct osd_attributes_list_element's
-	 * type=E followed by struct osd_attributes_list_multi_header's
-	 */
-} __packed;
-
-static inline unsigned osdv1_list_size(struct osdv1_attributes_list_header *h)
-{
-	return be16_to_cpu(h->list_bytes);
-}
-
-struct osdv2_attributes_list_header {
-	u8 type;	/* lower 4-bits only */
-	u8 pad[3];
-/*4*/	__be32 list_bytes; /* Initiator shall set to zero. Only set by target */
-	/*
-	 * type=9 followed by struct osd_attributes_list_element's
-	 * type=E followed by struct osd_attributes_list_multi_header's
-	 */
-} __packed;
-
-static inline unsigned osdv2_list_size(struct osdv2_attributes_list_header *h)
-{
-	return be32_to_cpu(h->list_bytes);
-}
-
-/* (osd-r10 6.13)
- * osd2r03: 6.15 LIST (Table 79) LIST command parameter data.
- *	for root_lstchg below
- */
-enum {
-	OSD_OBJ_ID_LIST_PAR = 0x1, /* V1-only. Not used in V2 */
-	OSD_OBJ_ID_LIST_LSTCHG = 0x2,
-};
-
-/*
- * osd2r03: 6.15.2 LIST command parameter data
- * (Also for LIST COLLECTION)
- */
-struct osd_obj_id_list {
-	__be64 list_bytes; /* bytes in list excluding list_bytes (-8) */
-	__be64 continuation_id;
-	__be32 list_identifier;
-	u8 pad[3];
-	u8 root_lstchg;
-	__be64 object_ids[0];
-} __packed;
-
-static inline bool osd_is_obj_list_done(struct osd_obj_id_list *list,
-	bool *is_changed)
-{
-	*is_changed = (0 != (list->root_lstchg & OSD_OBJ_ID_LIST_LSTCHG));
-	return 0 != list->continuation_id;
-}
-
-/*
- * osd2r03: 4.12.4.5 The ALLDATA security method
- */
-struct osd_data_out_integrity_info {
-	__be64 data_bytes;
-	__be64 set_attributes_bytes;
-	__be64 get_attributes_bytes;
-	__u8 integrity_check_value[OSD_CRYPTO_KEYID_SIZE];
-} __packed;
-
-/* Same osd_data_out_integrity_info is used for OSD2/OSD1. The only difference
- * Is the sizeof the structure since in OSD1 the last array is smaller. Use
- * below for version independent handling of this structure
- */
-static inline int osd_data_out_integrity_info_sizeof(bool is_ver1)
-{
-	return sizeof(struct osd_data_out_integrity_info) -
-		(is_ver1 * (OSDv2_CRYPTO_KEYID_SIZE - OSDv1_CRYPTO_KEYID_SIZE));
-}
-
-struct osd_data_in_integrity_info {
-	__be64 data_bytes;
-	__be64 retrieved_attributes_bytes;
-	__u8 integrity_check_value[OSD_CRYPTO_KEYID_SIZE];
-} __packed;
-
-/* Same osd_data_in_integrity_info is used for OSD2/OSD1. The only difference
- * Is the sizeof the structure since in OSD1 the last array is smaller. Use
- * below for version independent handling of this structure
- */
-static inline int osd_data_in_integrity_info_sizeof(bool is_ver1)
-{
-	return sizeof(struct osd_data_in_integrity_info) -
-		(is_ver1 * (OSDv2_CRYPTO_KEYID_SIZE - OSDv1_CRYPTO_KEYID_SIZE));
-}
-
-struct osd_timestamp {
-	u8 time[6]; /* number of milliseconds since 1/1/1970 UT (big endian) */
-} __packed;
-/* FIXME: define helper functions to convert to/from osd time format */
-
-/*
- * Capability & Security definitions
- * osd2r03: 4.11.2.2 Capability format
- * osd2r03: 5.2.8 Security parameters
- */
-
-struct osd_key_identifier {
-	u8 id[7]; /* if you know why 7 please email ooo@electrozaur.com */
-} __packed;
-
-/* for osd_capability.format */
-enum {
-	OSD_SEC_CAP_FORMAT_NO_CAPS = 0,
-	OSD_SEC_CAP_FORMAT_VER1 = 1,
-	OSD_SEC_CAP_FORMAT_VER2 = 2,
-};
-
-/* security_method */
-enum {
-	OSD_SEC_NOSEC = 0,
-	OSD_SEC_CAPKEY = 1,
-	OSD_SEC_CMDRSP = 2,
-	OSD_SEC_ALLDATA = 3,
-};
-
-enum object_type {
-	OSD_SEC_OBJ_ROOT = 0x1,
-	OSD_SEC_OBJ_PARTITION = 0x2,
-	OSD_SEC_OBJ_COLLECTION = 0x40,
-	OSD_SEC_OBJ_USER = 0x80,
-};
-
-enum osd_capability_bit_masks {
-	OSD_SEC_CAP_APPEND	= BIT(0),
-	OSD_SEC_CAP_OBJ_MGMT	= BIT(1),
-	OSD_SEC_CAP_REMOVE	= BIT(2),
-	OSD_SEC_CAP_CREATE	= BIT(3),
-	OSD_SEC_CAP_SET_ATTR	= BIT(4),
-	OSD_SEC_CAP_GET_ATTR	= BIT(5),
-	OSD_SEC_CAP_WRITE	= BIT(6),
-	OSD_SEC_CAP_READ	= BIT(7),
-
-	OSD_SEC_CAP_NONE1	= BIT(8),
-	OSD_SEC_CAP_NONE2	= BIT(9),
-	OSD_SEC_GBL_REM 	= BIT(10), /*v2 only*/
-	OSD_SEC_CAP_QUERY	= BIT(11), /*v2 only*/
-	OSD_SEC_CAP_M_OBJECT	= BIT(12), /*v2 only*/
-	OSD_SEC_CAP_POL_SEC	= BIT(13),
-	OSD_SEC_CAP_GLOBAL	= BIT(14),
-	OSD_SEC_CAP_DEV_MGMT	= BIT(15),
-};
-
-/* for object_descriptor_type (hi nibble used) */
-enum {
-	OSD_SEC_OBJ_DESC_NONE = 0,     /* Not allowed */
-	OSD_SEC_OBJ_DESC_OBJ = 1 << 4, /* v1: also collection */
-	OSD_SEC_OBJ_DESC_PAR = 2 << 4, /* also root */
-	OSD_SEC_OBJ_DESC_COL = 3 << 4, /* v2 only */
-};
-
-/* (osd-r10:4.9.2.2)
- * osd2r03:4.11.2.2 Capability format
- */
-struct osd_capability_head {
-	u8 format; /* low nibble */
-	u8 integrity_algorithm__key_version; /* MAKE_BYTE(integ_alg, key_ver) */
-	u8 security_method;
-	u8 reserved1;
-/*04*/	struct osd_timestamp expiration_time;
-/*10*/	u8 audit[20];
-/*30*/	u8 discriminator[12];
-/*42*/	struct osd_timestamp object_created_time;
-/*48*/	u8 object_type;
-/*49*/	u8 permissions_bit_mask[5];
-/*54*/	u8 reserved2;
-/*55*/	u8 object_descriptor_type; /* high nibble */
-} __packed;
-
-/*56 v1*/
-struct osdv1_cap_object_descriptor {
-	union {
-		struct {
-/*56*/			__be32 policy_access_tag;
-/*60*/			__be64 allowed_partition_id;
-/*68*/			__be64 allowed_object_id;
-/*76*/			__be32 reserved;
-		} __packed obj_desc;
-
-/*56*/		u8 object_descriptor[24];
-	};
-} __packed;
-/*80 v1*/
-
-/*56 v2*/
-struct osd_cap_object_descriptor {
-	union {
-		struct {
-/*56*/			__be32 allowed_attributes_access;
-/*60*/			__be32 policy_access_tag;
-/*64*/			__be16 boot_epoch;
-/*66*/			u8 reserved[6];
-/*72*/			__be64 allowed_partition_id;
-/*80*/			__be64 allowed_object_id;
-/*88*/			__be64 allowed_range_length;
-/*96*/			__be64 allowed_range_start;
-		} __packed obj_desc;
-
-/*56*/		u8 object_descriptor[48];
-	};
-} __packed;
-/*104 v2*/
-
-struct osdv1_capability {
-	struct osd_capability_head h;
-	struct osdv1_cap_object_descriptor od;
-} __packed;
-
-struct osd_capability {
-	struct osd_capability_head h;
-	struct osd_cap_object_descriptor od;
-} __packed;
-
-/**
- * osd_sec_set_caps - set cap-bits into the capabilities header
- *
- * @cap:	The osd_capability_head to set cap bits to.
- * @bit_mask: 	Use an ORed list of enum osd_capability_bit_masks values
- *
- * permissions_bit_mask is unaligned use below to set into caps
- * in a version independent way
- */
-static inline void osd_sec_set_caps(struct osd_capability_head *cap,
-	u16 bit_mask)
-{
-	/*
-	 *Note: The bits above are defined LE order this is because this way
-	 *      they can grow in the future to more then 16, and still retain
-	 *      there constant values.
-	 */
-	put_unaligned_le16(bit_mask, &cap->permissions_bit_mask);
-}
-
-/* osd2r05a sec 5.3: CDB continuation segment formats */
-enum osd_continuation_segment_format {
-	CDB_CONTINUATION_FORMAT_V2 = 0x01,
-};
-
-struct osd_continuation_segment_header {
-	u8	format;
-	u8	reserved1;
-	__be16	service_action;
-	__be32	reserved2;
-	u8	integrity_check[OSDv2_CRYPTO_KEYID_SIZE];
-} __packed;
-
-/* osd2r05a sec 5.4.1: CDB continuation descriptors */
-enum osd_continuation_descriptor_type {
-	NO_MORE_DESCRIPTORS = 0x0000,
-	SCATTER_GATHER_LIST = 0x0001,
-	QUERY_LIST = 0x0002,
-	USER_OBJECT = 0x0003,
-	COPY_USER_OBJECT_SOURCE = 0x0101,
-	EXTENSION_CAPABILITIES = 0xFFEE
-};
-
-struct osd_continuation_descriptor_header {
-	__be16	type;
-	u8	reserved;
-	u8	pad_length;
-	__be32	length;
-} __packed;
-
-
-/* osd2r05a sec 5.4.2: Scatter/gather list */
-struct osd_sg_list_entry {
-	__be64 offset;
-	__be64 len;
-};
-
-struct osd_sg_continuation_descriptor {
-	struct osd_continuation_descriptor_header hdr;
-	struct osd_sg_list_entry entries[];
-};
-
-#endif /* ndef __OSD_PROTOCOL_H__ */
diff --git a/include/scsi/osd_sec.h b/include/scsi/osd_sec.h
deleted file mode 100644
index 7abeb0f0db30..000000000000
--- a/include/scsi/osd_sec.h
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * osd_sec.h - OSD security manager API
- *
- * Copyright (C) 2008 Panasas Inc.  All rights reserved.
- *
- * Authors:
- *   Boaz Harrosh <ooo@electrozaur.com>
- *   Benny Halevy <bhalevy@panasas.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2
- *
- */
-#ifndef __OSD_SEC_H__
-#define __OSD_SEC_H__
-
-#include <scsi/osd_protocol.h>
-#include <scsi/osd_types.h>
-
-/*
- * Contains types and constants of osd capabilities and security
- * encoding/decoding.
- * API is trying to keep security abstract so initiator of an object
- * based pNFS client knows as little as possible about security and
- * capabilities. It is the Server's osd-initiator place to know more.
- * Also can be used by osd-target.
- */
-void osd_sec_encode_caps(void *caps, ...);/* NI */
-void osd_sec_init_nosec_doall_caps(void *caps,
-	const struct osd_obj_id *obj, bool is_collection, const bool is_v1);
-
-bool osd_is_sec_alldata(struct osd_security_parameters *sec_params);
-
-/* Conditionally sign the CDB according to security setting in ocdb
- * with cap_key */
-void osd_sec_sign_cdb(struct osd_cdb *ocdb, const u8 *cap_key);
-
-/* Unconditionally sign the BIO data with cap_key.
- * Check for osd_is_sec_alldata() was done prior to calling this. */
-void osd_sec_sign_data(void *data_integ, struct bio *bio, const u8 *cap_key);
-
-/* Version independent copy of caps into the cdb */
-void osd_set_caps(struct osd_cdb *cdb, const void *caps);
-
-#endif /* ndef __OSD_SEC_H__ */
diff --git a/include/scsi/osd_sense.h b/include/scsi/osd_sense.h
deleted file mode 100644
index d52aa93a0b2d..000000000000
--- a/include/scsi/osd_sense.h
+++ /dev/null
@@ -1,263 +0,0 @@
-/*
- * osd_sense.h - OSD Related sense handling definitions.
- *
- * Copyright (C) 2008 Panasas Inc.  All rights reserved.
- *
- * Authors:
- *   Boaz Harrosh <ooo@electrozaur.com>
- *   Benny Halevy <bhalevy@panasas.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2
- *
- * This file contains types and constants that are defined by the protocol
- * Note: All names and symbols are taken from the OSD standard's text.
- */
-#ifndef __OSD_SENSE_H__
-#define __OSD_SENSE_H__
-
-#include <scsi/osd_protocol.h>
-
-/* SPC3r23 4.5.6 Sense key and sense code definitions table 27 */
-enum scsi_sense_keys {
-	scsi_sk_no_sense        = 0x0,
-	scsi_sk_recovered_error = 0x1,
-	scsi_sk_not_ready       = 0x2,
-	scsi_sk_medium_error    = 0x3,
-	scsi_sk_hardware_error  = 0x4,
-	scsi_sk_illegal_request = 0x5,
-	scsi_sk_unit_attention  = 0x6,
-	scsi_sk_data_protect    = 0x7,
-	scsi_sk_blank_check     = 0x8,
-	scsi_sk_vendor_specific = 0x9,
-	scsi_sk_copy_aborted    = 0xa,
-	scsi_sk_aborted_command = 0xb,
-	scsi_sk_volume_overflow = 0xd,
-	scsi_sk_miscompare      = 0xe,
-	scsi_sk_reserved        = 0xf,
-};
-
-/* SPC3r23 4.5.6 Sense key and sense code definitions table 28 */
-/* Note: only those which can be returned by an OSD target. Most of
- *       these errors are taken care of by the generic scsi layer.
- */
-enum osd_additional_sense_codes {
-	scsi_no_additional_sense_information			= 0x0000,
-	scsi_operation_in_progress				= 0x0016,
-	scsi_cleaning_requested					= 0x0017,
-	scsi_lunr_cause_not_reportable				= 0x0400,
-	scsi_logical_unit_is_in_process_of_becoming_ready	= 0x0401,
-	scsi_lunr_initializing_command_required			= 0x0402,
-	scsi_lunr_manual_intervention_required			= 0x0403,
-	scsi_lunr_operation_in_progress				= 0x0407,
-	scsi_lunr_selftest_in_progress				= 0x0409,
-	scsi_luna_asymmetric_access_state_transition		= 0x040a,
-	scsi_luna_target_port_in_standby_state			= 0x040b,
-	scsi_luna_target_port_in_unavailable_state		= 0x040c,
-	scsi_lunr_notify_enable_spinup_required			= 0x0411,
-	scsi_logical_unit_does_not_respond_to_selection		= 0x0500,
-	scsi_logical_unit_communication_failure			= 0x0800,
-	scsi_logical_unit_communication_timeout			= 0x0801,
-	scsi_logical_unit_communication_parity_error		= 0x0802,
-	scsi_error_log_overflow					= 0x0a00,
-	scsi_warning						= 0x0b00,
-	scsi_warning_specified_temperature_exceeded		= 0x0b01,
-	scsi_warning_enclosure_degraded				= 0x0b02,
-	scsi_write_error_unexpected_unsolicited_data		= 0x0c0c,
-	scsi_write_error_not_enough_unsolicited_data		= 0x0c0d,
-	scsi_invalid_information_unit				= 0x0e00,
-	scsi_invalid_field_in_command_information_unit		= 0x0e03,
-	scsi_read_error_failed_retransmission_request		= 0x1113,
-	scsi_parameter_list_length_error			= 0x1a00,
-	scsi_invalid_command_operation_code			= 0x2000,
-	scsi_invalid_field_in_cdb				= 0x2400,
-	osd_security_audit_value_frozen				= 0x2404,
-	osd_security_working_key_frozen				= 0x2405,
-	osd_nonce_not_unique					= 0x2406,
-	osd_nonce_timestamp_out_of_range			= 0x2407,
-	scsi_logical_unit_not_supported				= 0x2500,
-	scsi_invalid_field_in_parameter_list			= 0x2600,
-	scsi_parameter_not_supported				= 0x2601,
-	scsi_parameter_value_invalid				= 0x2602,
-	scsi_invalid_release_of_persistent_reservation		= 0x2604,
-	osd_invalid_dataout_buffer_integrity_check_value	= 0x260f,
-	scsi_not_ready_to_ready_change_medium_may_have_changed	= 0x2800,
-	scsi_power_on_reset_or_bus_device_reset_occurred	= 0x2900,
-	scsi_power_on_occurred					= 0x2901,
-	scsi_scsi_bus_reset_occurred				= 0x2902,
-	scsi_bus_device_reset_function_occurred			= 0x2903,
-	scsi_device_internal_reset				= 0x2904,
-	scsi_transceiver_mode_changed_to_single_ended		= 0x2905,
-	scsi_transceiver_mode_changed_to_lvd			= 0x2906,
-	scsi_i_t_nexus_loss_occurred				= 0x2907,
-	scsi_parameters_changed					= 0x2a00,
-	scsi_mode_parameters_changed				= 0x2a01,
-	scsi_asymmetric_access_state_changed			= 0x2a06,
-	scsi_priority_changed					= 0x2a08,
-	scsi_command_sequence_error				= 0x2c00,
-	scsi_previous_busy_status				= 0x2c07,
-	scsi_previous_task_set_full_status			= 0x2c08,
-	scsi_previous_reservation_conflict_status		= 0x2c09,
-	osd_partition_or_collection_contains_user_objects	= 0x2c0a,
-	scsi_commands_cleared_by_another_initiator		= 0x2f00,
-	scsi_cleaning_failure					= 0x3007,
-	scsi_enclosure_failure					= 0x3400,
-	scsi_enclosure_services_failure				= 0x3500,
-	scsi_unsupported_enclosure_function			= 0x3501,
-	scsi_enclosure_services_unavailable			= 0x3502,
-	scsi_enclosure_services_transfer_failure		= 0x3503,
-	scsi_enclosure_services_transfer_refused		= 0x3504,
-	scsi_enclosure_services_checksum_error			= 0x3505,
-	scsi_rounded_parameter					= 0x3700,
-	osd_read_past_end_of_user_object			= 0x3b17,
-	scsi_logical_unit_has_not_self_configured_yet		= 0x3e00,
-	scsi_logical_unit_failure				= 0x3e01,
-	scsi_timeout_on_logical_unit				= 0x3e02,
-	scsi_logical_unit_failed_selftest			= 0x3e03,
-	scsi_logical_unit_unable_to_update_selftest_log		= 0x3e04,
-	scsi_target_operating_conditions_have_changed		= 0x3f00,
-	scsi_microcode_has_been_changed				= 0x3f01,
-	scsi_inquiry_data_has_changed				= 0x3f03,
-	scsi_echo_buffer_overwritten				= 0x3f0f,
-	scsi_diagnostic_failure_on_component_nn_first		= 0x4080,
-	scsi_diagnostic_failure_on_component_nn_last		= 0x40ff,
-	scsi_message_error					= 0x4300,
-	scsi_internal_target_failure				= 0x4400,
-	scsi_select_or_reselect_failure				= 0x4500,
-	scsi_scsi_parity_error					= 0x4700,
-	scsi_data_phase_crc_error_detected			= 0x4701,
-	scsi_scsi_parity_error_detected_during_st_data_phase	= 0x4702,
-	scsi_asynchronous_information_protection_error_detected	= 0x4704,
-	scsi_protocol_service_crc_error				= 0x4705,
-	scsi_phy_test_function_in_progress			= 0x4706,
-	scsi_invalid_message_error				= 0x4900,
-	scsi_command_phase_error				= 0x4a00,
-	scsi_data_phase_error					= 0x4b00,
-	scsi_logical_unit_failed_self_configuration		= 0x4c00,
-	scsi_overlapped_commands_attempted			= 0x4e00,
-	osd_quota_error						= 0x5507,
-	scsi_failure_prediction_threshold_exceeded		= 0x5d00,
-	scsi_failure_prediction_threshold_exceeded_false	= 0x5dff,
-	scsi_voltage_fault					= 0x6500,
-};
-
-enum scsi_descriptor_types {
-	scsi_sense_information			= 0x0,
-	scsi_sense_command_specific_information	= 0x1,
-	scsi_sense_key_specific			= 0x2,
-	scsi_sense_field_replaceable_unit	= 0x3,
-	scsi_sense_stream_commands		= 0x4,
-	scsi_sense_block_commands		= 0x5,
-	osd_sense_object_identification		= 0x6,
-	osd_sense_response_integrity_check	= 0x7,
-	osd_sense_attribute_identification	= 0x8,
-	scsi_sense_ata_return			= 0x9,
-
-	scsi_sense_Reserved_first		= 0x0A,
-	scsi_sense_Reserved_last		= 0x7F,
-	scsi_sense_Vendor_specific_first	= 0x80,
-	scsi_sense_Vendor_specific_last		= 0xFF,
-};
-
-struct scsi_sense_descriptor { /* for picking into desc type */
-	u8	descriptor_type; /* one of enum scsi_descriptor_types */
-	u8	additional_length; /* n - 1 */
-	u8	data[];
-} __packed;
-
-/* OSD deploys only scsi descriptor_based sense buffers */
-struct scsi_sense_descriptor_based {
-/*0*/	u8 	response_code; /* 0x72 or 0x73 */
-/*1*/	u8 	sense_key; /* one of enum scsi_sense_keys (4 lower bits) */
-/*2*/	__be16	additional_sense_code; /* enum osd_additional_sense_codes */
-/*4*/	u8	Reserved[3];
-/*7*/	u8	additional_sense_length; /* n - 7 */
-/*8*/	struct	scsi_sense_descriptor ssd[0]; /* variable length, 1 or more */
-} __packed;
-
-/* some descriptors deployed by OSD */
-
-/* SPC3r23 4.5.2.3 Command-specific information sense data descriptor */
-/* Note: this is the same for descriptor_type=00 but with type=00 the
- *        Reserved[0] == 0x80 (ie. bit-7 set)
- */
-struct scsi_sense_command_specific_data_descriptor {
-/*0*/	u8	descriptor_type; /* (00h/01h) */
-/*1*/	u8	additional_length; /* (0Ah) */
-/*2*/	u8	Reserved[2];
-/*4*/	__be64  information;
-} __packed;
-/*12*/
-
-struct scsi_sense_key_specific_data_descriptor {
-/*0*/	u8	descriptor_type; /* (02h) */
-/*1*/	u8	additional_length; /* (06h) */
-/*2*/	u8	Reserved[2];
-/* SKSV, C/D, Reserved (2), BPV, BIT POINTER (3) */
-/*4*/	u8	sksv_cd_bpv_bp;
-/*5*/	__be16	value; /* field-pointer/progress-value/retry-count/... */
-/*7*/	u8	Reserved2;
-} __packed;
-/*8*/
-
-/* 4.16.2.1 OSD error identification sense data descriptor - table 52 */
-/* Note: these bits are defined LE order for easy definition, this way the BIT()
- * number is the same as in the documentation. Below members at
- * osd_sense_identification_data_descriptor are therefore defined __le32.
- */
-enum osd_command_functions_bits {
-	OSD_CFB_COMMAND		 = BIT(4),
-	OSD_CFB_CMD_CAP_VERIFIED = BIT(5),
-	OSD_CFB_VALIDATION	 = BIT(7),
-	OSD_CFB_IMP_ST_ATT	 = BIT(12),
-	OSD_CFB_SET_ATT		 = BIT(20),
-	OSD_CFB_SA_CAP_VERIFIED	 = BIT(21),
-	OSD_CFB_GET_ATT		 = BIT(28),
-	OSD_CFB_GA_CAP_VERIFIED	 = BIT(29),
-};
-
-struct osd_sense_identification_data_descriptor {
-/*0*/	u8	descriptor_type; /* (06h) */
-/*1*/	u8	additional_length; /* (1Eh) */
-/*2*/	u8	Reserved[6];
-/*8*/	__le32	not_initiated_functions; /*osd_command_functions_bits*/
-/*12*/	__le32	completed_functions; /*osd_command_functions_bits*/
-/*16*/ 	__be64	partition_id;
-/*24*/	__be64	object_id;
-} __packed;
-/*32*/
-
-struct osd_sense_response_integrity_check_descriptor {
-/*0*/	u8	descriptor_type; /* (07h) */
-/*1*/	u8	additional_length; /* (20h) */
-/*2*/	u8	integrity_check_value[32]; /*FIXME: OSDv2_CRYPTO_KEYID_SIZE*/
-} __packed;
-/*34*/
-
-struct osd_sense_attributes_data_descriptor {
-/*0*/	u8	descriptor_type; /* (08h) */
-/*1*/	u8	additional_length; /* (n-2) */
-/*2*/	u8	Reserved[6];
-	struct osd_sense_attr {
-/*8*/		__be32	attr_page;
-/*12*/		__be32	attr_id;
-/*16*/	} sense_attrs[0]; /* 1 or more */
-} __packed;
-/*variable*/
-
-/* Dig into scsi_sk_illegal_request/scsi_invalid_field_in_cdb errors */
-
-/*FIXME: Support also field in CAPS*/
-#define OSD_CDB_OFFSET(F) offsetof(struct osd_cdb_head, F)
-
-enum osdv2_cdb_field_offset {
-	OSDv1_CFO_STARTING_BYTE	= OSD_CDB_OFFSET(v1.start_address),
-	OSD_CFO_STARTING_BYTE	= OSD_CDB_OFFSET(v2.start_address),
-	OSD_CFO_PARTITION_ID	= OSD_CDB_OFFSET(partition),
-	OSD_CFO_OBJECT_ID	= OSD_CDB_OFFSET(object),
-	OSD_CFO_PERMISSIONS	= sizeof(struct osd_cdb_head) +
-					offsetof(struct osd_capability_head,
-						 permissions_bit_mask),
-};
-
-#endif /* ndef __OSD_SENSE_H__ */
diff --git a/include/scsi/osd_types.h b/include/scsi/osd_types.h
deleted file mode 100644
index 48e8a165e136..000000000000
--- a/include/scsi/osd_types.h
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * osd_types.h - Types and constants which are not part of the protocol.
- *
- * Copyright (C) 2008 Panasas Inc.  All rights reserved.
- *
- * Authors:
- *   Boaz Harrosh <ooo@electrozaur.com>
- *   Benny Halevy <bhalevy@panasas.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2
- *
- * Contains types and constants that are implementation specific and are
- * used by more than one part of the osd library.
- *     (Eg initiator/target/security_manager/...)
- */
-#ifndef __OSD_TYPES_H__
-#define __OSD_TYPES_H__
-
-struct osd_systemid {
-	u8 data[OSD_SYSTEMID_LEN];
-};
-
-typedef u64 __bitwise osd_id;
-
-struct osd_obj_id {
-	osd_id partition;
-	osd_id id;
-};
-
-static const struct __weak osd_obj_id osd_root_object = {0, 0};
-
-struct osd_attr {
-	u32 attr_page;
-	u32 attr_id;
-	u16 len;		/* byte count of operand */
-	void *val_ptr;		/* in network order */
-};
-
-struct osd_sg_entry {
-	u64 offset;
-	u64 len;
-};
-
-#endif /* ndef __OSD_TYPES_H__ */
diff --git a/include/scsi/scsi_host.h b/include/scsi/scsi_host.h
index 2b539a1b3f62..a5fcdad4a03e 100644
--- a/include/scsi/scsi_host.h
+++ b/include/scsi/scsi_host.h
@@ -628,6 +628,9 @@ struct Scsi_Host {
 	/* Host responded with short (<36 bytes) INQUIRY result */
 	unsigned short_inquiry:1;
 
+	/* The transport requires the LUN bits NOT to be stored in CDB[1] */
+	unsigned no_scsi2_lun_in_cdb:1;
+
 	/*
 	 * Optional work queue to be utilized by the transport
 	 */
@@ -639,9 +642,6 @@ struct Scsi_Host {
 	 */
 	struct workqueue_struct *tmf_work_q;
 
-	/* The transport requires the LUN bits NOT to be stored in CDB[1] */
-	unsigned no_scsi2_lun_in_cdb:1;
-
 	/*
 	 * Value host_blocked counts down from
 	 */
diff --git a/include/scsi/scsi_transport_fc.h b/include/scsi/scsi_transport_fc.h
index 15da45dc2a5d..b375c3303fe2 100644
--- a/include/scsi/scsi_transport_fc.h
+++ b/include/scsi/scsi_transport_fc.h
@@ -165,6 +165,9 @@ enum fc_tgtid_binding_type  {
 #define FC_PORT_ROLE_FCP_INITIATOR		0x02
 #define FC_PORT_ROLE_IP_PORT			0x04
 #define FC_PORT_ROLE_FCP_DUMMY_INITIATOR	0x08
+#define FC_PORT_ROLE_NVME_INITIATOR		0x10
+#define FC_PORT_ROLE_NVME_TARGET		0x20
+#define FC_PORT_ROLE_NVME_DISCOVERY		0x40
 
 /* The following are for compatibility */
 #define FC_RPORT_ROLE_UNKNOWN			FC_PORT_ROLE_UNKNOWN
@@ -473,6 +476,7 @@ enum fc_host_event_code  {
 	FCH_EVT_PORT_ONLINE		= 0x202,
 	FCH_EVT_PORT_FABRIC		= 0x204,
 	FCH_EVT_LINK_UNKNOWN		= 0x500,
+	FCH_EVT_LINK_FPIN		= 0x501,
 	FCH_EVT_VENDOR_UNIQUE		= 0xffff,
 };
 
@@ -755,7 +759,7 @@ fc_remote_port_chkready(struct fc_rport *rport)
 	return result;
 }
 
-static inline u64 wwn_to_u64(u8 *wwn)
+static inline u64 wwn_to_u64(const u8 *wwn)
 {
 	return get_unaligned_be64(wwn);
 }
@@ -798,11 +802,17 @@ u32 fc_get_event_number(void);
 void fc_host_post_event(struct Scsi_Host *shost, u32 event_number,
 		enum fc_host_event_code event_code, u32 event_data);
 void fc_host_post_vendor_event(struct Scsi_Host *shost, u32 event_number,
-		u32 data_len, char * data_buf, u64 vendor_id);
+		u32 data_len, char *data_buf, u64 vendor_id);
+void fc_host_post_fc_event(struct Scsi_Host *shost, u32 event_number,
+		enum fc_host_event_code event_code,
+		u32 data_len, char *data_buf, u64 vendor_id);
 	/* Note: when specifying vendor_id to fc_host_post_vendor_event()
-	 *   be sure to read the Vendor Type and ID formatting requirements
-	 *   specified in scsi_netlink.h
+	 *   or fc_host_post_fc_event(), be sure to read the Vendor Type
+	 *   and ID formatting requirements specified in scsi_netlink.h
+	 * Note: when calling fc_host_post_fc_event(), vendor_id may be
+	 *   specified as 0.
 	 */
+void fc_host_fpin_rcv(struct Scsi_Host *shost, u32 fpin_len, char *fpin_buf);
 struct fc_vport *fc_vport_create(struct Scsi_Host *shost, int channel,
 		struct fc_vport_identifiers *);
 int fc_vport_terminate(struct fc_vport *vport);
diff --git a/include/sound/core.h b/include/sound/core.h
index e923c23e05dd..c90ebbc8d9c4 100644
--- a/include/sound/core.h
+++ b/include/sound/core.h
@@ -226,7 +226,6 @@ int copy_from_user_toio(volatile void __iomem *dst, const void __user *src, size
 
 /* init.c */
 
-extern struct snd_card *snd_cards[SNDRV_CARDS];
 int snd_card_locked(int card);
 #if IS_ENABLED(CONFIG_SND_MIXER_OSS)
 #define SND_MIXER_OSS_NOTIFY_REGISTER	0
@@ -251,7 +250,20 @@ int snd_card_add_dev_attr(struct snd_card *card,
 int snd_component_add(struct snd_card *card, const char *component);
 int snd_card_file_add(struct snd_card *card, struct file *file);
 int snd_card_file_remove(struct snd_card *card, struct file *file);
-#define snd_card_unref(card)	put_device(&(card)->card_dev)
+
+struct snd_card *snd_card_ref(int card);
+
+/**
+ * snd_card_unref - Unreference the card object
+ * @card: the card object to unreference
+ *
+ * Call this function for the card object that was obtained via snd_card_ref()
+ * or snd_lookup_minor_data().
+ */
+static inline void snd_card_unref(struct snd_card *card)
+{
+	put_device(&card->card_dev);
+}
 
 #define snd_card_set_dev(card, devptr) ((card)->dev = (devptr))
 
diff --git a/include/sound/da7219.h b/include/sound/da7219.h
index 1bfcb16f2d10..4a36954c86c5 100644
--- a/include/sound/da7219.h
+++ b/include/sound/da7219.h
@@ -33,10 +33,16 @@ enum da7219_mic_amp_in_sel {
 
 struct da7219_aad_pdata;
 
+enum da7219_dai_clks {
+	DA7219_DAI_WCLK_IDX = 0,
+	DA7219_DAI_BCLK_IDX,
+	DA7219_DAI_NUM_CLKS,
+};
+
 struct da7219_pdata {
 	bool wakeup_source;
 
-	const char *dai_clks_name;
+	const char *dai_clk_names[DA7219_DAI_NUM_CLKS];
 
 	/* Mic */
 	enum da7219_micbias_voltage micbias_lvl;
diff --git a/include/sound/hdaudio.h b/include/sound/hdaudio.h
index 45f944d57982..896c3f45503b 100644
--- a/include/sound/hdaudio.h
+++ b/include/sound/hdaudio.h
@@ -297,7 +297,7 @@ struct hdac_rb {
  * @num_streams: streams supported
  * @idx: HDA link index
  * @hlink_list: link list of HDA links
- * @lock: lock for link mgmt
+ * @lock: lock for link and display power mgmt
  * @cmd_dma_state: state of cmd DMAs: CORB and RIRB
  */
 struct hdac_bus {
@@ -363,21 +363,20 @@ struct hdac_bus {
 	/* locks */
 	spinlock_t reg_lock;
 	struct mutex cmd_mutex;
+	struct mutex lock;
 
 	/* DRM component interface */
 	struct drm_audio_component *audio_component;
 	long display_power_status;
-	bool display_power_active;
+	unsigned long display_power_active;
 
 	/* parameters required for enhanced capabilities */
 	int num_streams;
 	int idx;
 
+	/* link management */
 	struct list_head hlink_list;
-
-	struct mutex lock;
 	bool cmd_dma_state;
-
 };
 
 int snd_hdac_bus_init(struct hdac_bus *bus, struct device *dev,
diff --git a/include/sound/memalloc.h b/include/sound/memalloc.h
index 1ac0dd82a916..4c6f3b5a7cff 100644
--- a/include/sound/memalloc.h
+++ b/include/sound/memalloc.h
@@ -151,9 +151,5 @@ int snd_dma_alloc_pages_fallback(int type, struct device *dev, size_t size,
                                  struct snd_dma_buffer *dmab);
 void snd_dma_free_pages(struct snd_dma_buffer *dmab);
 
-/* basic memory allocation functions */
-void *snd_malloc_pages(size_t size, gfp_t gfp_flags);
-void snd_free_pages(void *ptr, size_t size);
-
 #endif /* __SOUND_MEMALLOC_H */
 
diff --git a/include/sound/seq_kernel.h b/include/sound/seq_kernel.h
index 4b9ee3009aa0..c7a5433e109a 100644
--- a/include/sound/seq_kernel.h
+++ b/include/sound/seq_kernel.h
@@ -73,7 +73,8 @@ __printf(3, 4)
 int snd_seq_create_kernel_client(struct snd_card *card, int client_index,
 				 const char *name_fmt, ...);
 int snd_seq_delete_kernel_client(int client);
-int snd_seq_kernel_client_enqueue(int client, struct snd_seq_event *ev, int atomic, int hop);
+int snd_seq_kernel_client_enqueue(int client, struct snd_seq_event *ev,
+				  struct file *file, bool blocking);
 int snd_seq_kernel_client_dispatch(int client, struct snd_seq_event *ev, int atomic, int hop);
 int snd_seq_kernel_client_ctl(int client, unsigned int cmd, void *arg);
 
diff --git a/include/sound/simple_card_utils.h b/include/sound/simple_card_utils.h
index 7afe45389972..3429888347e7 100644
--- a/include/sound/simple_card_utils.h
+++ b/include/sound/simple_card_utils.h
@@ -10,10 +10,10 @@
 
 #include <sound/soc.h>
 
-#define asoc_simple_card_init_hp(card, sjack, prefix) \
-	asoc_simple_card_init_jack(card, sjack, 1, prefix)
-#define asoc_simple_card_init_mic(card, sjack, prefix) \
-	asoc_simple_card_init_jack(card, sjack, 0, prefix)
+#define asoc_simple_init_hp(card, sjack, prefix) \
+	asoc_simple_init_jack(card, sjack, 1, prefix)
+#define asoc_simple_init_mic(card, sjack, prefix) \
+	asoc_simple_init_jack(card, sjack, 0, prefix)
 
 struct asoc_simple_dai {
 	const char *name;
@@ -26,7 +26,7 @@ struct asoc_simple_dai {
 	struct clk *clk;
 };
 
-struct asoc_simple_card_data {
+struct asoc_simple_data {
 	u32 convert_rate;
 	u32 convert_channels;
 };
@@ -37,96 +37,180 @@ struct asoc_simple_jack {
 	struct snd_soc_jack_gpio gpio;
 };
 
-int asoc_simple_card_parse_daifmt(struct device *dev,
-				  struct device_node *node,
-				  struct device_node *codec,
-				  char *prefix,
-				  unsigned int *retfmt);
+struct asoc_simple_priv {
+	struct snd_soc_card snd_card;
+	struct simple_dai_props {
+		struct asoc_simple_dai *cpu_dai;
+		struct asoc_simple_dai *codec_dai;
+		struct snd_soc_dai_link_component codecs; /* single codec */
+		struct snd_soc_dai_link_component platforms;
+		struct asoc_simple_data adata;
+		struct snd_soc_codec_conf *codec_conf;
+		unsigned int mclk_fs;
+	} *dai_props;
+	struct asoc_simple_jack hp_jack;
+	struct asoc_simple_jack mic_jack;
+	struct snd_soc_dai_link *dai_link;
+	struct asoc_simple_dai *dais;
+	struct snd_soc_codec_conf *codec_conf;
+	struct gpio_desc *pa_gpio;
+};
+#define simple_priv_to_card(priv)	(&(priv)->snd_card)
+#define simple_priv_to_props(priv, i)	((priv)->dai_props + (i))
+#define simple_priv_to_dev(priv)	(simple_priv_to_card(priv)->dev)
+#define simple_priv_to_link(priv, i)	(simple_priv_to_card(priv)->dai_link + (i))
+
+struct link_info {
+	int dais; /* number of dai  */
+	int link; /* number of link */
+	int conf; /* number of codec_conf */
+	int cpu;  /* turn for CPU / Codec */
+};
+
+int asoc_simple_parse_daifmt(struct device *dev,
+			     struct device_node *node,
+			     struct device_node *codec,
+			     char *prefix,
+			     unsigned int *retfmt);
 __printf(3, 4)
-int asoc_simple_card_set_dailink_name(struct device *dev,
-				      struct snd_soc_dai_link *dai_link,
-				      const char *fmt, ...);
-int asoc_simple_card_parse_card_name(struct snd_soc_card *card,
-				     char *prefix);
-
-#define asoc_simple_card_parse_clk_cpu(dev, node, dai_link, simple_dai)		\
-	asoc_simple_card_parse_clk(dev, node, dai_link->cpu_of_node, simple_dai, \
+int asoc_simple_set_dailink_name(struct device *dev,
+				 struct snd_soc_dai_link *dai_link,
+				 const char *fmt, ...);
+int asoc_simple_parse_card_name(struct snd_soc_card *card,
+				char *prefix);
+
+#define asoc_simple_parse_clk_cpu(dev, node, dai_link, simple_dai)		\
+	asoc_simple_parse_clk(dev, node, dai_link->cpu_of_node, simple_dai, \
 				   dai_link->cpu_dai_name, NULL)
-#define asoc_simple_card_parse_clk_codec(dev, node, dai_link, simple_dai)	\
-	asoc_simple_card_parse_clk(dev, node, dai_link->codec_of_node, simple_dai,\
+#define asoc_simple_parse_clk_codec(dev, node, dai_link, simple_dai)	\
+	asoc_simple_parse_clk(dev, node, dai_link->codec_of_node, simple_dai,\
 				   dai_link->codec_dai_name, dai_link->codecs)
-int asoc_simple_card_parse_clk(struct device *dev,
-			       struct device_node *node,
-			       struct device_node *dai_of_node,
-			       struct asoc_simple_dai *simple_dai,
-			       const char *dai_name,
-			       struct snd_soc_dai_link_component *dlc);
-int asoc_simple_card_clk_enable(struct asoc_simple_dai *dai);
-void asoc_simple_card_clk_disable(struct asoc_simple_dai *dai);
-
-#define asoc_simple_card_parse_cpu(node, dai_link,				\
-				   list_name, cells_name, is_single_link)	\
-	asoc_simple_card_parse_dai(node, NULL,					\
-		&dai_link->cpu_of_node,						\
-		&dai_link->cpu_dai_name, list_name, cells_name, is_single_link)
-#define asoc_simple_card_parse_codec(node, dai_link, list_name, cells_name)	\
-	asoc_simple_card_parse_dai(node, dai_link->codecs,			\
+int asoc_simple_parse_clk(struct device *dev,
+			  struct device_node *node,
+			  struct device_node *dai_of_node,
+			  struct asoc_simple_dai *simple_dai,
+			  const char *dai_name,
+			  struct snd_soc_dai_link_component *dlc);
+int asoc_simple_startup(struct snd_pcm_substream *substream);
+void asoc_simple_shutdown(struct snd_pcm_substream *substream);
+int asoc_simple_hw_params(struct snd_pcm_substream *substream,
+			  struct snd_pcm_hw_params *params);
+int asoc_simple_dai_init(struct snd_soc_pcm_runtime *rtd);
+int asoc_simple_be_hw_params_fixup(struct snd_soc_pcm_runtime *rtd,
+				   struct snd_pcm_hw_params *params);
+
+#define asoc_simple_parse_cpu(node, dai_link, is_single_link)	\
+	asoc_simple_parse_dai(node, NULL,				\
+		&dai_link->cpu_of_node,					\
+		&dai_link->cpu_dai_name, is_single_link)
+#define asoc_simple_parse_codec(node, dai_link)	\
+	asoc_simple_parse_dai(node, dai_link->codecs,			\
 				   &dai_link->codec_of_node,			\
-				   &dai_link->codec_dai_name,			\
-				   list_name, cells_name, NULL)
-#define asoc_simple_card_parse_platform(node, dai_link, list_name, cells_name)	\
-	asoc_simple_card_parse_dai(node, dai_link->platforms,			\
-		&dai_link->platform_of_node,					\
-		NULL, list_name, cells_name, NULL)
-int asoc_simple_card_parse_dai(struct device_node *node,
-				  struct snd_soc_dai_link_component *dlc,
-				  struct device_node **endpoint_np,
-				  const char **dai_name,
-				  const char *list_name,
-				  const char *cells_name,
-				  int *is_single_links);
-
-#define asoc_simple_card_parse_graph_cpu(ep, dai_link)			\
-	asoc_simple_card_parse_graph_dai(ep, NULL,			\
-					 &dai_link->cpu_of_node,	\
-					 &dai_link->cpu_dai_name)
-#define asoc_simple_card_parse_graph_codec(ep, dai_link)		\
-	asoc_simple_card_parse_graph_dai(ep, dai_link->codecs,		\
-					 &dai_link->codec_of_node,	\
-					 &dai_link->codec_dai_name)
-int asoc_simple_card_parse_graph_dai(struct device_node *ep,
-				     struct snd_soc_dai_link_component *dlc,
-				     struct device_node **endpoint_np,
-				     const char **dai_name);
-
-#define asoc_simple_card_of_parse_tdm(np, dai)			\
+				   &dai_link->codec_dai_name, NULL)
+#define asoc_simple_parse_platform(node, dai_link)	\
+	asoc_simple_parse_dai(node, dai_link->platforms,			\
+		&dai_link->platform_of_node, NULL, NULL)
+
+#define asoc_simple_parse_tdm(np, dai)			\
 	snd_soc_of_parse_tdm_slot(np,	&(dai)->tx_slot_mask,	\
 					&(dai)->rx_slot_mask,	\
 					&(dai)->slots,		\
 					&(dai)->slot_width);
 
-int asoc_simple_card_init_dai(struct snd_soc_dai *dai,
-			      struct asoc_simple_dai *simple_dai);
-
-void asoc_simple_card_canonicalize_platform(struct snd_soc_dai_link *dai_link);
-void asoc_simple_card_canonicalize_cpu(struct snd_soc_dai_link *dai_link,
+void asoc_simple_canonicalize_platform(struct snd_soc_dai_link *dai_link);
+void asoc_simple_canonicalize_cpu(struct snd_soc_dai_link *dai_link,
 				      int is_single_links);
 
-int asoc_simple_card_clean_reference(struct snd_soc_card *card);
+int asoc_simple_clean_reference(struct snd_soc_card *card);
 
-void asoc_simple_card_convert_fixup(struct asoc_simple_card_data *data,
+void asoc_simple_convert_fixup(struct asoc_simple_data *data,
 				      struct snd_pcm_hw_params *params);
-void asoc_simple_card_parse_convert(struct device *dev,
-				    struct device_node *np, char *prefix,
-				    struct asoc_simple_card_data *data);
+void asoc_simple_parse_convert(struct device *dev,
+			       struct device_node *np, char *prefix,
+			       struct asoc_simple_data *data);
 
-int asoc_simple_card_of_parse_routing(struct snd_soc_card *card,
+int asoc_simple_parse_routing(struct snd_soc_card *card,
 				      char *prefix);
-int asoc_simple_card_of_parse_widgets(struct snd_soc_card *card,
+int asoc_simple_parse_widgets(struct snd_soc_card *card,
 				      char *prefix);
+int asoc_simple_parse_pin_switches(struct snd_soc_card *card,
+				   char *prefix);
 
-int asoc_simple_card_init_jack(struct snd_soc_card *card,
+int asoc_simple_init_jack(struct snd_soc_card *card,
 			       struct asoc_simple_jack *sjack,
 			       int is_hp, char *prefix);
+int asoc_simple_init_priv(struct asoc_simple_priv *priv,
+			       struct link_info *li);
+
+#ifdef DEBUG
+inline void asoc_simple_debug_dai(struct asoc_simple_priv *priv,
+				  char *name,
+				  struct asoc_simple_dai *dai)
+{
+	struct device *dev = simple_priv_to_dev(priv);
+
+	if (dai->name)
+		dev_dbg(dev, "%s dai name = %s\n",
+			name, dai->name);
+	if (dai->sysclk)
+		dev_dbg(dev, "%s sysclk = %d\n",
+			name, dai->sysclk);
+
+	dev_dbg(dev, "%s direction = %s\n",
+		name, dai->clk_direction ? "OUT" : "IN");
+
+	if (dai->slots)
+		dev_dbg(dev, "%s slots = %d\n", name, dai->slots);
+	if (dai->slot_width)
+		dev_dbg(dev, "%s slot width = %d\n", name, dai->slot_width);
+	if (dai->tx_slot_mask)
+		dev_dbg(dev, "%s tx slot mask = %d\n", name, dai->tx_slot_mask);
+	if (dai->rx_slot_mask)
+		dev_dbg(dev, "%s rx slot mask = %d\n", name, dai->rx_slot_mask);
+	if (dai->clk)
+		dev_dbg(dev, "%s clk %luHz\n", name, clk_get_rate(dai->clk));
+}
+
+inline void asoc_simple_debug_info(struct asoc_simple_priv *priv)
+{
+	struct snd_soc_card *card = simple_priv_to_card(priv);
+	struct device *dev = simple_priv_to_dev(priv);
+
+	int i;
+
+	if (card->name)
+		dev_dbg(dev, "Card Name: %s\n", card->name);
+
+	for (i = 0; i < card->num_links; i++) {
+		struct simple_dai_props *props = simple_priv_to_props(priv, i);
+		struct snd_soc_dai_link *link = simple_priv_to_link(priv, i);
+
+		dev_dbg(dev, "DAI%d\n", i);
+
+		asoc_simple_debug_dai(priv, "cpu", props->cpu_dai);
+		asoc_simple_debug_dai(priv, "codec", props->codec_dai);
+
+		if (link->name)
+			dev_dbg(dev, "dai name = %s\n", link->name);
+
+		dev_dbg(dev, "dai format = %04x\n", link->dai_fmt);
+
+		if (props->adata.convert_rate)
+			dev_dbg(dev, "convert_rate = %d\n",
+				props->adata.convert_rate);
+		if (props->adata.convert_channels)
+			dev_dbg(dev, "convert_channels = %d\n",
+				props->adata.convert_channels);
+		if (props->codec_conf && props->codec_conf->name_prefix)
+			dev_dbg(dev, "name prefix = %s\n",
+				props->codec_conf->name_prefix);
+		if (props->mclk_fs)
+			dev_dbg(dev, "mclk-fs = %d\n",
+				props->mclk_fs);
+	}
+}
+#else
+#define  asoc_simple_debug_info(priv)
+#endif /* DEBUG */
 
 #endif /* __SIMPLE_CARD_UTILS_H */
diff --git a/include/sound/sof.h b/include/sound/sof.h
new file mode 100644
index 000000000000..4640566b54fe
--- /dev/null
+++ b/include/sound/sof.h
@@ -0,0 +1,100 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */
+/*
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * Copyright(c) 2018 Intel Corporation. All rights reserved.
+ *
+ * Author: Liam Girdwood <liam.r.girdwood@linux.intel.com>
+ */
+
+#ifndef __INCLUDE_SOUND_SOF_H
+#define __INCLUDE_SOUND_SOF_H
+
+#include <linux/pci.h>
+#include <sound/soc.h>
+#include <sound/soc-acpi.h>
+
+struct snd_sof_dsp_ops;
+
+/*
+ * SOF Platform data.
+ */
+struct snd_sof_pdata {
+	const struct firmware *fw;
+	const char *drv_name;
+	const char *name;
+	const char *platform;
+
+	struct device *dev;
+
+	/*
+	 * notification callback used if the hardware initialization
+	 * can take time or is handled in a workqueue. This callback
+	 * can be used by the caller to e.g. enable runtime_pm
+	 * or limit functionality until all low-level inits are
+	 * complete.
+	 */
+	void (*sof_probe_complete)(struct device *dev);
+
+	/* descriptor */
+	const struct sof_dev_desc *desc;
+
+	/* firmware and topology filenames */
+	const char *fw_filename_prefix;
+	const char *fw_filename;
+	const char *tplg_filename_prefix;
+	const char *tplg_filename;
+
+	/* machine */
+	struct platform_device *pdev_mach;
+	const struct snd_soc_acpi_mach *machine;
+
+	void *hw_pdata;
+};
+
+/*
+ * Descriptor used for setting up SOF platform data. This is used when
+ * ACPI/PCI data is missing or mapped differently.
+ */
+struct sof_dev_desc {
+	/* list of machines using this configuration */
+	struct snd_soc_acpi_mach *machines;
+
+	/* Platform resource indexes in BAR / ACPI resources. */
+	/* Must set to -1 if not used - add new items to end */
+	int resindex_lpe_base;
+	int resindex_pcicfg_base;
+	int resindex_imr_base;
+	int irqindex_host_ipc;
+	int resindex_dma_base;
+
+	/* DMA only valid when resindex_dma_base != -1*/
+	int dma_engine;
+	int dma_size;
+
+	/* IPC timeouts in ms */
+	int ipc_timeout;
+	int boot_timeout;
+
+	/* chip information for dsp */
+	const void *chip_info;
+
+	/* defaults for no codec mode */
+	const char *nocodec_fw_filename;
+	const char *nocodec_tplg_filename;
+
+	/* defaults paths for firmware and topology files */
+	const char *default_fw_path;
+	const char *default_tplg_path;
+
+	const struct snd_sof_dsp_ops *ops;
+	const struct sof_arch_ops *arch_ops;
+};
+
+int sof_nocodec_setup(struct device *dev,
+		      struct snd_sof_pdata *sof_pdata,
+		      struct snd_soc_acpi_mach *mach,
+		      const struct sof_dev_desc *desc,
+		      const struct snd_sof_dsp_ops *ops);
+#endif
diff --git a/include/sound/sof/control.h b/include/sound/sof/control.h
new file mode 100644
index 000000000000..bded69e696d4
--- /dev/null
+++ b/include/sound/sof/control.h
@@ -0,0 +1,158 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */
+/*
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * Copyright(c) 2018 Intel Corporation. All rights reserved.
+ */
+
+#ifndef __INCLUDE_SOUND_SOF_CONTROL_H__
+#define __INCLUDE_SOUND_SOF_CONTROL_H__
+
+#include <uapi/sound/sof/header.h>
+#include <sound/sof/header.h>
+
+/*
+ * Component Mixers and Controls
+ */
+
+/* channel positions - uses same values as ALSA */
+enum sof_ipc_chmap {
+	SOF_CHMAP_UNKNOWN = 0,
+	SOF_CHMAP_NA,		/**< N/A, silent */
+	SOF_CHMAP_MONO,		/**< mono stream */
+	SOF_CHMAP_FL,		/**< front left */
+	SOF_CHMAP_FR,		/**< front right */
+	SOF_CHMAP_RL,		/**< rear left */
+	SOF_CHMAP_RR,		/**< rear right */
+	SOF_CHMAP_FC,		/**< front centre */
+	SOF_CHMAP_LFE,		/**< LFE */
+	SOF_CHMAP_SL,		/**< side left */
+	SOF_CHMAP_SR,		/**< side right */
+	SOF_CHMAP_RC,		/**< rear centre */
+	SOF_CHMAP_FLC,		/**< front left centre */
+	SOF_CHMAP_FRC,		/**< front right centre */
+	SOF_CHMAP_RLC,		/**< rear left centre */
+	SOF_CHMAP_RRC,		/**< rear right centre */
+	SOF_CHMAP_FLW,		/**< front left wide */
+	SOF_CHMAP_FRW,		/**< front right wide */
+	SOF_CHMAP_FLH,		/**< front left high */
+	SOF_CHMAP_FCH,		/**< front centre high */
+	SOF_CHMAP_FRH,		/**< front right high */
+	SOF_CHMAP_TC,		/**< top centre */
+	SOF_CHMAP_TFL,		/**< top front left */
+	SOF_CHMAP_TFR,		/**< top front right */
+	SOF_CHMAP_TFC,		/**< top front centre */
+	SOF_CHMAP_TRL,		/**< top rear left */
+	SOF_CHMAP_TRR,		/**< top rear right */
+	SOF_CHMAP_TRC,		/**< top rear centre */
+	SOF_CHMAP_TFLC,		/**< top front left centre */
+	SOF_CHMAP_TFRC,		/**< top front right centre */
+	SOF_CHMAP_TSL,		/**< top side left */
+	SOF_CHMAP_TSR,		/**< top side right */
+	SOF_CHMAP_LLFE,		/**< left LFE */
+	SOF_CHMAP_RLFE,		/**< right LFE */
+	SOF_CHMAP_BC,		/**< bottom centre */
+	SOF_CHMAP_BLC,		/**< bottom left centre */
+	SOF_CHMAP_BRC,		/**< bottom right centre */
+	SOF_CHMAP_LAST = SOF_CHMAP_BRC,
+};
+
+/* control data type and direction */
+enum sof_ipc_ctrl_type {
+	/*  per channel data - uses struct sof_ipc_ctrl_value_chan */
+	SOF_CTRL_TYPE_VALUE_CHAN_GET = 0,
+	SOF_CTRL_TYPE_VALUE_CHAN_SET,
+	/* component data - uses struct sof_ipc_ctrl_value_comp */
+	SOF_CTRL_TYPE_VALUE_COMP_GET,
+	SOF_CTRL_TYPE_VALUE_COMP_SET,
+	/* bespoke data - uses struct sof_abi_hdr */
+	SOF_CTRL_TYPE_DATA_GET,
+	SOF_CTRL_TYPE_DATA_SET,
+};
+
+/* control command type */
+enum sof_ipc_ctrl_cmd {
+	SOF_CTRL_CMD_VOLUME = 0, /**< maps to ALSA volume style controls */
+	SOF_CTRL_CMD_ENUM,	/**< maps to ALSA enum style controls */
+	SOF_CTRL_CMD_SWITCH,	/**< maps to ALSA switch style controls */
+	SOF_CTRL_CMD_BINARY,	/**< maps to ALSA binary style controls */
+};
+
+/* generic channel mapped value data */
+struct sof_ipc_ctrl_value_chan {
+	uint32_t channel;	/**< channel map - enum sof_ipc_chmap */
+	uint32_t value;
+} __packed;
+
+/* generic component mapped value data */
+struct sof_ipc_ctrl_value_comp {
+	uint32_t index;	/**< component source/sink/control index in control */
+	union {
+		uint32_t uvalue;
+		int32_t svalue;
+	};
+} __packed;
+
+/* generic control data */
+struct sof_ipc_ctrl_data {
+	struct sof_ipc_reply rhdr;
+	uint32_t comp_id;
+
+	/* control access and data type */
+	uint32_t type;		/**< enum sof_ipc_ctrl_type */
+	uint32_t cmd;		/**< enum sof_ipc_ctrl_cmd */
+	uint32_t index;		/**< control index for comps > 1 control */
+
+	/* control data - can either be appended or DMAed from host */
+	struct sof_ipc_host_buffer buffer;
+	uint32_t num_elems;	/**< in array elems or bytes for data type */
+	uint32_t elems_remaining;	/**< elems remaining if sent in parts */
+
+	uint32_t msg_index;	/**< for large messages sent in parts */
+
+	/* reserved for future use */
+	uint32_t reserved[6];
+
+	/* control data - add new types if needed */
+	union {
+		/* channel values can be used by volume type controls */
+		struct sof_ipc_ctrl_value_chan chanv[0];
+		/* component values used by routing controls like mux, mixer */
+		struct sof_ipc_ctrl_value_comp compv[0];
+		/* data can be used by binary controls */
+		struct sof_abi_hdr data[0];
+	};
+} __packed;
+
+/** Event type */
+enum sof_ipc_ctrl_event_type {
+	SOF_CTRL_EVENT_GENERIC = 0,	/**< generic event */
+	SOF_CTRL_EVENT_GENERIC_METADATA,	/**< generic event with metadata */
+	SOF_CTRL_EVENT_KD,	/**< keyword detection event */
+	SOF_CTRL_EVENT_VAD,	/**< voice activity detection event */
+};
+
+/**
+ * Generic notification data.
+ */
+struct sof_ipc_comp_event {
+	struct sof_ipc_reply rhdr;
+	uint16_t src_comp_type;	/**< COMP_TYPE_ */
+	uint32_t src_comp_id;	/**< source component id */
+	uint32_t event_type;	/**< event type - SOF_CTRL_EVENT_* */
+	uint32_t num_elems;	/**< in array elems or bytes for data type */
+
+	/* reserved for future use */
+	uint32_t reserved[8];
+
+	/* control data - add new types if needed */
+	union {
+		/* data can be used by binary controls */
+		struct sof_abi_hdr data[0];
+		/* event specific values */
+		uint32_t event_value;
+	};
+} __packed;
+
+#endif
diff --git a/include/sound/sof/dai-intel.h b/include/sound/sof/dai-intel.h
new file mode 100644
index 000000000000..4bd83f7adddf
--- /dev/null
+++ b/include/sound/sof/dai-intel.h
@@ -0,0 +1,178 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */
+/*
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * Copyright(c) 2018 Intel Corporation. All rights reserved.
+ */
+
+#ifndef __INCLUDE_SOUND_SOF_DAI_INTEL_H__
+#define __INCLUDE_SOUND_SOF_DAI_INTEL_H__
+
+#include <sound/sof/header.h>
+
+ /* ssc1: TINTE */
+#define SOF_DAI_INTEL_SSP_QUIRK_TINTE		(1 << 0)
+ /* ssc1: PINTE */
+#define SOF_DAI_INTEL_SSP_QUIRK_PINTE		(1 << 1)
+ /* ssc2: SMTATF */
+#define SOF_DAI_INTEL_SSP_QUIRK_SMTATF		(1 << 2)
+ /* ssc2: MMRATF */
+#define SOF_DAI_INTEL_SSP_QUIRK_MMRATF		(1 << 3)
+ /* ssc2: PSPSTWFDFD */
+#define SOF_DAI_INTEL_SSP_QUIRK_PSPSTWFDFD	(1 << 4)
+ /* ssc2: PSPSRWFDFD */
+#define SOF_DAI_INTEL_SSP_QUIRK_PSPSRWFDFD	(1 << 5)
+/* ssc1: LBM */
+#define SOF_DAI_INTEL_SSP_QUIRK_LBM		(1 << 6)
+
+ /* here is the possibility to define others aux macros */
+
+#define SOF_DAI_INTEL_SSP_FRAME_PULSE_WIDTH_MAX		38
+#define SOF_DAI_INTEL_SSP_SLOT_PADDING_MAX		31
+
+/* SSP clocks control settings
+ *
+ * Macros for clks_control field in sof_ipc_dai_ssp_params struct.
+ */
+
+/* mclk 0 disable */
+#define SOF_DAI_INTEL_SSP_MCLK_0_DISABLE		BIT(0)
+/* mclk 1 disable */
+#define SOF_DAI_INTEL_SSP_MCLK_1_DISABLE		BIT(1)
+/* mclk keep active */
+#define SOF_DAI_INTEL_SSP_CLKCTRL_MCLK_KA		BIT(2)
+/* bclk keep active */
+#define SOF_DAI_INTEL_SSP_CLKCTRL_BCLK_KA		BIT(3)
+/* fs keep active */
+#define SOF_DAI_INTEL_SSP_CLKCTRL_FS_KA			BIT(4)
+/* bclk idle */
+#define SOF_DAI_INTEL_SSP_CLKCTRL_BCLK_IDLE_HIGH	BIT(5)
+
+/* SSP Configuration Request - SOF_IPC_DAI_SSP_CONFIG */
+struct sof_ipc_dai_ssp_params {
+	struct sof_ipc_hdr hdr;
+	uint16_t reserved1;
+	uint16_t mclk_id;
+
+	uint32_t mclk_rate;	/* mclk frequency in Hz */
+	uint32_t fsync_rate;	/* fsync frequency in Hz */
+	uint32_t bclk_rate;	/* bclk frequency in Hz */
+
+	/* TDM */
+	uint32_t tdm_slots;
+	uint32_t rx_slots;
+	uint32_t tx_slots;
+
+	/* data */
+	uint32_t sample_valid_bits;
+	uint16_t tdm_slot_width;
+	uint16_t reserved2;	/* alignment */
+
+	/* MCLK */
+	uint32_t mclk_direction;
+
+	uint16_t frame_pulse_width;
+	uint16_t tdm_per_slot_padding_flag;
+	uint32_t clks_control;
+	uint32_t quirks;
+} __packed;
+
+/* HDA Configuration Request - SOF_IPC_DAI_HDA_CONFIG */
+struct sof_ipc_dai_hda_params {
+	struct sof_ipc_hdr hdr;
+	uint32_t link_dma_ch;
+} __packed;
+
+/* DMIC Configuration Request - SOF_IPC_DAI_DMIC_CONFIG */
+
+/* This struct is defined per 2ch PDM controller available in the platform.
+ * Normally it is sufficient to set the used microphone specific enables to 1
+ * and keep other parameters as zero. The customizations are:
+ *
+ * 1. If a device mixes different microphones types with different polarity
+ * and/or the absolute polarity matters the PCM signal from a microphone
+ * can be inverted with the controls.
+ *
+ * 2. If the microphones in a stereo pair do not appear in captured stream
+ * in desired order due to board schematics choises they can be swapped with
+ * the clk_edge parameter.
+ *
+ * 3. If PDM bit errors are seen in capture (poor quality) the skew parameter
+ * that delays the sampling time of data by half cycles of DMIC source clock
+ * can be tried for improvement. However there is no guarantee for this to fix
+ * data integrity problems.
+ */
+struct sof_ipc_dai_dmic_pdm_ctrl {
+	struct sof_ipc_hdr hdr;
+	uint16_t id;		/**< PDM controller ID */
+
+	uint16_t enable_mic_a;	/**< Use A (left) channel mic (0 or 1)*/
+	uint16_t enable_mic_b;	/**< Use B (right) channel mic (0 or 1)*/
+
+	uint16_t polarity_mic_a; /**< Optionally invert mic A signal (0 or 1) */
+	uint16_t polarity_mic_b; /**< Optionally invert mic B signal (0 or 1) */
+
+	uint16_t clk_edge;	/**< Optionally swap data clock edge (0 or 1) */
+	uint16_t skew;		/**< Adjust PDM data sampling vs. clock (0..15) */
+
+	uint16_t reserved[3];	/**< Make sure the total size is 4 bytes aligned */
+} __packed;
+
+/* This struct contains the global settings for all 2ch PDM controllers. The
+ * version number used in configuration data is checked vs. version used by
+ * device driver src/drivers/dmic.c need to match. It is incremented from
+ * initial value 1 if updates done for the to driver would alter the operation
+ * of the microhone.
+ *
+ * Note: The microphone clock (pdmclk_min, pdmclk_max, duty_min, duty_max)
+ * parameters need to be set as defined in microphone data sheet. E.g. clock
+ * range 1.0 - 3.2 MHz is usually supported microphones. Some microphones are
+ * multi-mode capable and there may be denied mic clock frequencies between
+ * the modes. In such case set the clock range limits of the desired mode to
+ * avoid the driver to set clock to an illegal rate.
+ *
+ * The duty cycle could be set to 48-52% if not known. Generally these
+ * parameters can be altered within data sheet specified limits to match
+ * required audio application performance power.
+ *
+ * The microphone clock needs to be usually about 50-80 times the used audio
+ * sample rate. With highest sample rates above 48 kHz this can relaxed
+ * somewhat.
+ *
+ * The parameter wake_up_time describes how long time the microphone needs
+ * for the data line to produce valid output from mic clock start. The driver
+ * will mute the captured audio for the given time. The min_clock_on_time
+ * parameter is used to prevent too short clock bursts to happen. The driver
+ * will keep the clock active after capture stop if this time is not yet
+ * met. The unit for both is microseconds (us). Exceed of 100 ms will be
+ * treated as an error.
+ */
+struct sof_ipc_dai_dmic_params {
+	struct sof_ipc_hdr hdr;
+	uint32_t driver_ipc_version;	/**< Version (1..N) */
+
+	uint32_t pdmclk_min;	/**< Minimum microphone clock in Hz (100000..N) */
+	uint32_t pdmclk_max;	/**< Maximum microphone clock in Hz (min...N) */
+
+	uint32_t fifo_fs;	/**< FIFO sample rate in Hz (8000..96000) */
+	uint32_t reserved_1;	/**< Reserved */
+	uint16_t fifo_bits;	/**< FIFO word length (16 or 32) */
+	uint16_t reserved_2;	/**< Reserved */
+
+	uint16_t duty_min;	/**< Min. mic clock duty cycle in % (20..80) */
+	uint16_t duty_max;	/**< Max. mic clock duty cycle in % (min..80) */
+
+	uint32_t num_pdm_active; /**< Number of active pdm controllers */
+
+	uint32_t wake_up_time;      /**< Time from clock start to data (us) */
+	uint32_t min_clock_on_time; /**< Min. time that clk is kept on (us) */
+
+	/* reserved for future use */
+	uint32_t reserved[6];
+
+	/**< variable number of pdm controller config */
+	struct sof_ipc_dai_dmic_pdm_ctrl pdm[0];
+} __packed;
+
+#endif
diff --git a/include/sound/sof/dai.h b/include/sound/sof/dai.h
new file mode 100644
index 000000000000..3b67c93ff101
--- /dev/null
+++ b/include/sound/sof/dai.h
@@ -0,0 +1,75 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */
+/*
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * Copyright(c) 2018 Intel Corporation. All rights reserved.
+ */
+
+#ifndef __INCLUDE_SOUND_SOF_DAI_H__
+#define __INCLUDE_SOUND_SOF_DAI_H__
+
+#include <sound/sof/header.h>
+#include <sound/sof/dai-intel.h>
+
+/*
+ * DAI Configuration.
+ *
+ * Each different DAI type will have it's own structure and IPC cmd.
+ */
+
+#define SOF_DAI_FMT_I2S		1 /**< I2S mode */
+#define SOF_DAI_FMT_RIGHT_J	2 /**< Right Justified mode */
+#define SOF_DAI_FMT_LEFT_J	3 /**< Left Justified mode */
+#define SOF_DAI_FMT_DSP_A	4 /**< L data MSB after FRM LRC */
+#define SOF_DAI_FMT_DSP_B	5 /**< L data MSB during FRM LRC */
+#define SOF_DAI_FMT_PDM		6 /**< Pulse density modulation */
+
+#define SOF_DAI_FMT_CONT	(1 << 4) /**< continuous clock */
+#define SOF_DAI_FMT_GATED	(0 << 4) /**< clock is gated */
+
+#define SOF_DAI_FMT_NB_NF	(0 << 8) /**< normal bit clock + frame */
+#define SOF_DAI_FMT_NB_IF	(2 << 8) /**< normal BCLK + inv FRM */
+#define SOF_DAI_FMT_IB_NF	(3 << 8) /**< invert BCLK + nor FRM */
+#define SOF_DAI_FMT_IB_IF	(4 << 8) /**< invert BCLK + FRM */
+
+#define SOF_DAI_FMT_CBM_CFM	(0 << 12) /**< codec clk & FRM master */
+#define SOF_DAI_FMT_CBS_CFM	(2 << 12) /**< codec clk slave & FRM master */
+#define SOF_DAI_FMT_CBM_CFS	(3 << 12) /**< codec clk master & frame slave */
+#define SOF_DAI_FMT_CBS_CFS	(4 << 12) /**< codec clk & FRM slave */
+
+#define SOF_DAI_FMT_FORMAT_MASK		0x000f
+#define SOF_DAI_FMT_CLOCK_MASK		0x00f0
+#define SOF_DAI_FMT_INV_MASK		0x0f00
+#define SOF_DAI_FMT_MASTER_MASK		0xf000
+
+/** \brief Types of DAI */
+enum sof_ipc_dai_type {
+	SOF_DAI_INTEL_NONE = 0,		/**< None */
+	SOF_DAI_INTEL_SSP,		/**< Intel SSP */
+	SOF_DAI_INTEL_DMIC,		/**< Intel DMIC */
+	SOF_DAI_INTEL_HDA,		/**< Intel HD/A */
+};
+
+/* general purpose DAI configuration */
+struct sof_ipc_dai_config {
+	struct sof_ipc_cmd_hdr hdr;
+	uint32_t type;		/**< DAI type - enum sof_ipc_dai_type */
+	uint32_t dai_index;	/**< index of this type dai */
+
+	/* physical protocol and clocking */
+	uint16_t format;	/**< SOF_DAI_FMT_ */
+	uint16_t reserved16;	/**< alignment */
+
+	/* reserved for future use */
+	uint32_t reserved[8];
+
+	/* HW specific data */
+	union {
+		struct sof_ipc_dai_ssp_params ssp;
+		struct sof_ipc_dai_dmic_params dmic;
+		struct sof_ipc_dai_hda_params hda;
+	};
+} __packed;
+
+#endif
diff --git a/include/sound/sof/header.h b/include/sound/sof/header.h
new file mode 100644
index 000000000000..ccb6a004b37b
--- /dev/null
+++ b/include/sound/sof/header.h
@@ -0,0 +1,158 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */
+/*
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * Copyright(c) 2018 Intel Corporation. All rights reserved.
+ */
+
+#ifndef __INCLUDE_SOUND_SOF_HEADER_H__
+#define __INCLUDE_SOUND_SOF_HEADER_H__
+
+#include <uapi/sound/sof/abi.h>
+
+/** \addtogroup sof_uapi uAPI
+ *  SOF uAPI specification.
+ *  @{
+ */
+
+/*
+ * IPC messages have a prefixed 32 bit identifier made up as follows :-
+ *
+ * 0xGCCCNNNN where
+ * G is global cmd type (4 bits)
+ * C is command type (12 bits)
+ * I is the ID number (16 bits) - monotonic and overflows
+ *
+ * This is sent at the start of the IPM message in the mailbox. Messages should
+ * not be sent in the doorbell (special exceptions for firmware .
+ */
+
+/* Global Message - Generic */
+#define SOF_GLB_TYPE_SHIFT			28
+#define SOF_GLB_TYPE_MASK			(0xf << SOF_GLB_TYPE_SHIFT)
+#define SOF_GLB_TYPE(x)				((x) << SOF_GLB_TYPE_SHIFT)
+
+/* Command Message - Generic */
+#define SOF_CMD_TYPE_SHIFT			16
+#define SOF_CMD_TYPE_MASK			(0xfff << SOF_CMD_TYPE_SHIFT)
+#define SOF_CMD_TYPE(x)				((x) << SOF_CMD_TYPE_SHIFT)
+
+/* Global Message Types */
+#define SOF_IPC_GLB_REPLY			SOF_GLB_TYPE(0x1U)
+#define SOF_IPC_GLB_COMPOUND			SOF_GLB_TYPE(0x2U)
+#define SOF_IPC_GLB_TPLG_MSG			SOF_GLB_TYPE(0x3U)
+#define SOF_IPC_GLB_PM_MSG			SOF_GLB_TYPE(0x4U)
+#define SOF_IPC_GLB_COMP_MSG			SOF_GLB_TYPE(0x5U)
+#define SOF_IPC_GLB_STREAM_MSG			SOF_GLB_TYPE(0x6U)
+#define SOF_IPC_FW_READY			SOF_GLB_TYPE(0x7U)
+#define SOF_IPC_GLB_DAI_MSG			SOF_GLB_TYPE(0x8U)
+#define SOF_IPC_GLB_TRACE_MSG			SOF_GLB_TYPE(0x9U)
+
+/*
+ * DSP Command Message Types
+ */
+
+/* topology */
+#define SOF_IPC_TPLG_COMP_NEW			SOF_CMD_TYPE(0x001)
+#define SOF_IPC_TPLG_COMP_FREE			SOF_CMD_TYPE(0x002)
+#define SOF_IPC_TPLG_COMP_CONNECT		SOF_CMD_TYPE(0x003)
+#define SOF_IPC_TPLG_PIPE_NEW			SOF_CMD_TYPE(0x010)
+#define SOF_IPC_TPLG_PIPE_FREE			SOF_CMD_TYPE(0x011)
+#define SOF_IPC_TPLG_PIPE_CONNECT		SOF_CMD_TYPE(0x012)
+#define SOF_IPC_TPLG_PIPE_COMPLETE		SOF_CMD_TYPE(0x013)
+#define SOF_IPC_TPLG_BUFFER_NEW			SOF_CMD_TYPE(0x020)
+#define SOF_IPC_TPLG_BUFFER_FREE		SOF_CMD_TYPE(0x021)
+
+/* PM */
+#define SOF_IPC_PM_CTX_SAVE			SOF_CMD_TYPE(0x001)
+#define SOF_IPC_PM_CTX_RESTORE			SOF_CMD_TYPE(0x002)
+#define SOF_IPC_PM_CTX_SIZE			SOF_CMD_TYPE(0x003)
+#define SOF_IPC_PM_CLK_SET			SOF_CMD_TYPE(0x004)
+#define SOF_IPC_PM_CLK_GET			SOF_CMD_TYPE(0x005)
+#define SOF_IPC_PM_CLK_REQ			SOF_CMD_TYPE(0x006)
+#define SOF_IPC_PM_CORE_ENABLE			SOF_CMD_TYPE(0x007)
+
+/* component runtime config - multiple different types */
+#define SOF_IPC_COMP_SET_VALUE			SOF_CMD_TYPE(0x001)
+#define SOF_IPC_COMP_GET_VALUE			SOF_CMD_TYPE(0x002)
+#define SOF_IPC_COMP_SET_DATA			SOF_CMD_TYPE(0x003)
+#define SOF_IPC_COMP_GET_DATA			SOF_CMD_TYPE(0x004)
+
+/* DAI messages */
+#define SOF_IPC_DAI_CONFIG			SOF_CMD_TYPE(0x001)
+#define SOF_IPC_DAI_LOOPBACK			SOF_CMD_TYPE(0x002)
+
+/* stream */
+#define SOF_IPC_STREAM_PCM_PARAMS		SOF_CMD_TYPE(0x001)
+#define SOF_IPC_STREAM_PCM_PARAMS_REPLY		SOF_CMD_TYPE(0x002)
+#define SOF_IPC_STREAM_PCM_FREE			SOF_CMD_TYPE(0x003)
+#define SOF_IPC_STREAM_TRIG_START		SOF_CMD_TYPE(0x004)
+#define SOF_IPC_STREAM_TRIG_STOP		SOF_CMD_TYPE(0x005)
+#define SOF_IPC_STREAM_TRIG_PAUSE		SOF_CMD_TYPE(0x006)
+#define SOF_IPC_STREAM_TRIG_RELEASE		SOF_CMD_TYPE(0x007)
+#define SOF_IPC_STREAM_TRIG_DRAIN		SOF_CMD_TYPE(0x008)
+#define SOF_IPC_STREAM_TRIG_XRUN		SOF_CMD_TYPE(0x009)
+#define SOF_IPC_STREAM_POSITION			SOF_CMD_TYPE(0x00a)
+#define SOF_IPC_STREAM_VORBIS_PARAMS		SOF_CMD_TYPE(0x010)
+#define SOF_IPC_STREAM_VORBIS_FREE		SOF_CMD_TYPE(0x011)
+
+/* trace and debug */
+#define SOF_IPC_TRACE_DMA_PARAMS		SOF_CMD_TYPE(0x001)
+#define SOF_IPC_TRACE_DMA_POSITION		SOF_CMD_TYPE(0x002)
+
+/* Get message component id */
+#define SOF_IPC_MESSAGE_ID(x)			((x) & 0xffff)
+
+/* maximum message size for mailbox Tx/Rx */
+#define SOF_IPC_MSG_MAX_SIZE			384
+
+/*
+ * Structure Header - Header for all IPC structures except command structs.
+ * The size can be greater than the structure size and that means there is
+ * extended bespoke data beyond the end of the structure including variable
+ * arrays.
+ */
+
+struct sof_ipc_hdr {
+	uint32_t size;			/**< size of structure */
+} __packed;
+
+/*
+ * Command Header - Header for all IPC commands. Identifies IPC message.
+ * The size can be greater than the structure size and that means there is
+ * extended bespoke data beyond the end of the structure including variable
+ * arrays.
+ */
+
+struct sof_ipc_cmd_hdr {
+	uint32_t size;			/**< size of structure */
+	uint32_t cmd;			/**< SOF_IPC_GLB_ + cmd */
+} __packed;
+
+/*
+ * Generic reply message. Some commands override this with their own reply
+ * types that must include this at start.
+ */
+struct sof_ipc_reply {
+	struct sof_ipc_cmd_hdr hdr;
+	int32_t error;			/**< negative error numbers */
+}  __packed;
+
+/*
+ * Compound commands - SOF_IPC_GLB_COMPOUND.
+ *
+ * Compound commands are sent to the DSP as a single IPC operation. The
+ * commands are split into blocks and each block has a header. This header
+ * identifies the command type and the number of commands before the next
+ * header.
+ */
+
+struct sof_ipc_compound_hdr {
+	struct sof_ipc_cmd_hdr hdr;
+	uint32_t count;		/**< count of 0 means end of compound sequence */
+}  __packed;
+
+/** @}*/
+
+#endif
diff --git a/include/sound/sof/info.h b/include/sound/sof/info.h
new file mode 100644
index 000000000000..21dae04d8183
--- /dev/null
+++ b/include/sound/sof/info.h
@@ -0,0 +1,118 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */
+/*
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * Copyright(c) 2018 Intel Corporation. All rights reserved.
+ */
+
+#ifndef __INCLUDE_SOUND_SOF_INFO_H__
+#define __INCLUDE_SOUND_SOF_INFO_H__
+
+#include <sound/sof/header.h>
+#include <sound/sof/stream.h>
+
+/*
+ * Firmware boot and version
+ */
+
+#define SOF_IPC_MAX_ELEMS	16
+
+/* extended data types that can be appended onto end of sof_ipc_fw_ready */
+enum sof_ipc_ext_data {
+	SOF_IPC_EXT_DMA_BUFFER = 0,
+	SOF_IPC_EXT_WINDOW,
+};
+
+/* FW version - SOF_IPC_GLB_VERSION */
+struct sof_ipc_fw_version {
+	struct sof_ipc_hdr hdr;
+	uint16_t major;
+	uint16_t minor;
+	uint16_t micro;
+	uint16_t build;
+	uint8_t date[12];
+	uint8_t time[10];
+	uint8_t tag[6];
+	uint32_t abi_version;
+
+	/* reserved for future use */
+	uint32_t reserved[4];
+} __packed;
+
+/* FW ready Message - sent by firmware when boot has completed */
+struct sof_ipc_fw_ready {
+	struct sof_ipc_cmd_hdr hdr;
+	uint32_t dspbox_offset;	 /* dsp initiated IPC mailbox */
+	uint32_t hostbox_offset; /* host initiated IPC mailbox */
+	uint32_t dspbox_size;
+	uint32_t hostbox_size;
+	struct sof_ipc_fw_version version;
+
+	/* Miscellaneous debug flags showing build/debug features enabled */
+	union {
+		uint64_t reserved;
+		struct {
+			uint64_t build:1;
+			uint64_t locks:1;
+			uint64_t locks_verbose:1;
+			uint64_t gdb:1;
+		} bits;
+	} debug;
+
+	/* reserved for future use */
+	uint32_t reserved[4];
+} __packed;
+
+/*
+ * Extended Firmware data. All optional, depends on platform/arch.
+ */
+enum sof_ipc_region {
+	SOF_IPC_REGION_DOWNBOX	= 0,
+	SOF_IPC_REGION_UPBOX,
+	SOF_IPC_REGION_TRACE,
+	SOF_IPC_REGION_DEBUG,
+	SOF_IPC_REGION_STREAM,
+	SOF_IPC_REGION_REGS,
+	SOF_IPC_REGION_EXCEPTION,
+};
+
+struct sof_ipc_ext_data_hdr {
+	struct sof_ipc_cmd_hdr hdr;
+	uint32_t type;		/**< SOF_IPC_EXT_ */
+} __packed;
+
+struct sof_ipc_dma_buffer_elem {
+	struct sof_ipc_hdr hdr;
+	uint32_t type;		/**< SOF_IPC_REGION_ */
+	uint32_t id;		/**< platform specific - used to map to host memory */
+	struct sof_ipc_host_buffer buffer;
+} __packed;
+
+/* extended data DMA buffers for IPC, trace and debug */
+struct sof_ipc_dma_buffer_data {
+	struct sof_ipc_ext_data_hdr ext_hdr;
+	uint32_t num_buffers;
+
+	/* host files in buffer[n].buffer */
+	struct sof_ipc_dma_buffer_elem buffer[];
+}  __packed;
+
+struct sof_ipc_window_elem {
+	struct sof_ipc_hdr hdr;
+	uint32_t type;		/**< SOF_IPC_REGION_ */
+	uint32_t id;		/**< platform specific - used to map to host memory */
+	uint32_t flags;		/**< R, W, RW, etc - to define */
+	uint32_t size;		/**< size of region in bytes */
+	/* offset in window region as windows can be partitioned */
+	uint32_t offset;
+} __packed;
+
+/* extended data memory windows for IPC, trace and debug */
+struct sof_ipc_window {
+	struct sof_ipc_ext_data_hdr ext_hdr;
+	uint32_t num_windows;
+	struct sof_ipc_window_elem window[];
+}  __packed;
+
+#endif
diff --git a/include/sound/sof/pm.h b/include/sound/sof/pm.h
new file mode 100644
index 000000000000..8ae3ad45bdf7
--- /dev/null
+++ b/include/sound/sof/pm.h
@@ -0,0 +1,48 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */
+/*
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * Copyright(c) 2018 Intel Corporation. All rights reserved.
+ */
+
+#ifndef __INCLUDE_SOUND_SOF_PM_H__
+#define __INCLUDE_SOUND_SOF_PM_H__
+
+#include <sound/sof/header.h>
+
+/*
+ * PM
+ */
+
+/* PM context element */
+struct sof_ipc_pm_ctx_elem {
+	struct sof_ipc_hdr hdr;
+	uint32_t type;
+	uint32_t size;
+	uint64_t addr;
+}  __packed;
+
+/*
+ * PM context - SOF_IPC_PM_CTX_SAVE, SOF_IPC_PM_CTX_RESTORE,
+ * SOF_IPC_PM_CTX_SIZE
+ */
+struct sof_ipc_pm_ctx {
+	struct sof_ipc_cmd_hdr hdr;
+	struct sof_ipc_host_buffer buffer;
+	uint32_t num_elems;
+	uint32_t size;
+
+	/* reserved for future use */
+	uint32_t reserved[8];
+
+	struct sof_ipc_pm_ctx_elem elems[];
+} __packed;
+
+/* enable or disable cores - SOF_IPC_PM_CORE_ENABLE */
+struct sof_ipc_pm_core_config {
+	struct sof_ipc_cmd_hdr hdr;
+	uint32_t enable_mask;
+} __packed;
+
+#endif
diff --git a/include/sound/sof/stream.h b/include/sound/sof/stream.h
new file mode 100644
index 000000000000..643f175cb479
--- /dev/null
+++ b/include/sound/sof/stream.h
@@ -0,0 +1,148 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */
+/*
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * Copyright(c) 2018 Intel Corporation. All rights reserved.
+ */
+
+#ifndef __INCLUDE_SOUND_SOF_STREAM_H__
+#define __INCLUDE_SOUND_SOF_STREAM_H__
+
+#include <sound/sof/header.h>
+
+/*
+ * Stream configuration.
+ */
+
+#define SOF_IPC_MAX_CHANNELS			8
+
+/* common sample rates for use in masks */
+#define SOF_RATE_8000		(1 <<  0) /**< 8000Hz  */
+#define SOF_RATE_11025		(1 <<  1) /**< 11025Hz */
+#define SOF_RATE_12000		(1 <<  2) /**< 12000Hz */
+#define SOF_RATE_16000		(1 <<  3) /**< 16000Hz */
+#define SOF_RATE_22050		(1 <<  4) /**< 22050Hz */
+#define SOF_RATE_24000		(1 <<  5) /**< 24000Hz */
+#define SOF_RATE_32000		(1 <<  6) /**< 32000Hz */
+#define SOF_RATE_44100		(1 <<  7) /**< 44100Hz */
+#define SOF_RATE_48000		(1 <<  8) /**< 48000Hz */
+#define SOF_RATE_64000		(1 <<  9) /**< 64000Hz */
+#define SOF_RATE_88200		(1 << 10) /**< 88200Hz */
+#define SOF_RATE_96000		(1 << 11) /**< 96000Hz */
+#define SOF_RATE_176400		(1 << 12) /**< 176400Hz */
+#define SOF_RATE_192000		(1 << 13) /**< 192000Hz */
+
+/* continuous and non-standard rates for flexibility */
+#define SOF_RATE_CONTINUOUS	(1 << 30)  /**< range */
+#define SOF_RATE_KNOT		(1 << 31)  /**< non-continuous */
+
+/* generic PCM flags for runtime settings */
+#define SOF_PCM_FLAG_XRUN_STOP	(1 << 0) /**< Stop on any XRUN */
+
+/* stream PCM frame format */
+enum sof_ipc_frame {
+	SOF_IPC_FRAME_S16_LE = 0,
+	SOF_IPC_FRAME_S24_4LE,
+	SOF_IPC_FRAME_S32_LE,
+	SOF_IPC_FRAME_FLOAT,
+	/* other formats here */
+};
+
+/* stream buffer format */
+enum sof_ipc_buffer_format {
+	SOF_IPC_BUFFER_INTERLEAVED,
+	SOF_IPC_BUFFER_NONINTERLEAVED,
+	/* other formats here */
+};
+
+/* stream direction */
+enum sof_ipc_stream_direction {
+	SOF_IPC_STREAM_PLAYBACK = 0,
+	SOF_IPC_STREAM_CAPTURE,
+};
+
+/* stream ring info */
+struct sof_ipc_host_buffer {
+	struct sof_ipc_hdr hdr;
+	uint32_t phy_addr;
+	uint32_t pages;
+	uint32_t size;
+	uint32_t reserved[3];
+} __packed;
+
+struct sof_ipc_stream_params {
+	struct sof_ipc_hdr hdr;
+	struct sof_ipc_host_buffer buffer;
+	uint32_t direction;	/**< enum sof_ipc_stream_direction */
+	uint32_t frame_fmt;	/**< enum sof_ipc_frame */
+	uint32_t buffer_fmt;	/**< enum sof_ipc_buffer_format */
+	uint32_t rate;
+	uint16_t stream_tag;
+	uint16_t channels;
+	uint16_t sample_valid_bytes;
+	uint16_t sample_container_bytes;
+
+	/* for notifying host period has completed - 0 means no period IRQ */
+	uint32_t host_period_bytes;
+
+	uint32_t reserved[2];
+	uint16_t chmap[SOF_IPC_MAX_CHANNELS];	/**< channel map - SOF_CHMAP_ */
+} __packed;
+
+/* PCM params info - SOF_IPC_STREAM_PCM_PARAMS */
+struct sof_ipc_pcm_params {
+	struct sof_ipc_cmd_hdr hdr;
+	uint32_t comp_id;
+	uint32_t flags;		/**< generic PCM flags - SOF_PCM_FLAG_ */
+	uint32_t reserved[2];
+	struct sof_ipc_stream_params params;
+}  __packed;
+
+/* PCM params info reply - SOF_IPC_STREAM_PCM_PARAMS_REPLY */
+struct sof_ipc_pcm_params_reply {
+	struct sof_ipc_reply rhdr;
+	uint32_t comp_id;
+	uint32_t posn_offset;
+} __packed;
+
+/* free stream - SOF_IPC_STREAM_PCM_PARAMS */
+struct sof_ipc_stream {
+	struct sof_ipc_cmd_hdr hdr;
+	uint32_t comp_id;
+} __packed;
+
+/* flags indicating which time stamps are in sync with each other */
+#define	SOF_TIME_HOST_SYNC	(1 << 0)
+#define	SOF_TIME_DAI_SYNC	(1 << 1)
+#define	SOF_TIME_WALL_SYNC	(1 << 2)
+#define	SOF_TIME_STAMP_SYNC	(1 << 3)
+
+/* flags indicating which time stamps are valid */
+#define	SOF_TIME_HOST_VALID	(1 << 8)
+#define	SOF_TIME_DAI_VALID	(1 << 9)
+#define	SOF_TIME_WALL_VALID	(1 << 10)
+#define	SOF_TIME_STAMP_VALID	(1 << 11)
+
+/* flags indicating time stamps are 64bit else 3use low 32bit */
+#define	SOF_TIME_HOST_64	(1 << 16)
+#define	SOF_TIME_DAI_64		(1 << 17)
+#define	SOF_TIME_WALL_64	(1 << 18)
+#define	SOF_TIME_STAMP_64	(1 << 19)
+
+struct sof_ipc_stream_posn {
+	struct sof_ipc_reply rhdr;
+	uint32_t comp_id;	/**< host component ID */
+	uint32_t flags;		/**< SOF_TIME_ */
+	uint32_t wallclock_hz;	/**< frequency of wallclock in Hz */
+	uint32_t timestamp_ns;	/**< resolution of timestamp in ns */
+	uint64_t host_posn;	/**< host DMA position in bytes */
+	uint64_t dai_posn;	/**< DAI DMA position in bytes */
+	uint64_t comp_posn;	/**< comp position in bytes */
+	uint64_t wallclock;	/**< audio wall clock */
+	uint64_t timestamp;	/**< system time stamp */
+	uint32_t xrun_comp_id;	/**< comp ID of XRUN component */
+	int32_t xrun_size;	/**< XRUN size in bytes */
+}  __packed;
+
+#endif
diff --git a/include/sound/sof/topology.h b/include/sound/sof/topology.h
new file mode 100644
index 000000000000..46b2a7e63167
--- /dev/null
+++ b/include/sound/sof/topology.h
@@ -0,0 +1,256 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */
+/*
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * Copyright(c) 2018 Intel Corporation. All rights reserved.
+ */
+
+#ifndef __INCLUDE_SOUND_SOF_TOPOLOGY_H__
+#define __INCLUDE_SOUND_SOF_TOPOLOGY_H__
+
+#include <sound/sof/header.h>
+
+/*
+ * Component
+ */
+
+/* types of component */
+enum sof_comp_type {
+	SOF_COMP_NONE = 0,
+	SOF_COMP_HOST,
+	SOF_COMP_DAI,
+	SOF_COMP_SG_HOST,	/**< scatter gather variant */
+	SOF_COMP_SG_DAI,	/**< scatter gather variant */
+	SOF_COMP_VOLUME,
+	SOF_COMP_MIXER,
+	SOF_COMP_MUX,
+	SOF_COMP_SRC,
+	SOF_COMP_SPLITTER,
+	SOF_COMP_TONE,
+	SOF_COMP_SWITCH,
+	SOF_COMP_BUFFER,
+	SOF_COMP_EQ_IIR,
+	SOF_COMP_EQ_FIR,
+	SOF_COMP_KEYWORD_DETECT,
+	SOF_COMP_KPB,			/* A key phrase buffer component */
+	SOF_COMP_SELECTOR,		/**< channel selector component */
+	/* keep FILEREAD/FILEWRITE as the last ones */
+	SOF_COMP_FILEREAD = 10000,	/**< host test based file IO */
+	SOF_COMP_FILEWRITE = 10001,	/**< host test based file IO */
+};
+
+/* XRUN action for component */
+#define SOF_XRUN_STOP		1	/**< stop stream */
+#define SOF_XRUN_UNDER_ZERO	2	/**< send 0s to sink */
+#define SOF_XRUN_OVER_NULL	4	/**< send data to NULL */
+
+/* create new generic component - SOF_IPC_TPLG_COMP_NEW */
+struct sof_ipc_comp {
+	struct sof_ipc_cmd_hdr hdr;
+	uint32_t id;
+	enum sof_comp_type type;
+	uint32_t pipeline_id;
+
+	/* reserved for future use */
+	uint32_t reserved[2];
+} __packed;
+
+/*
+ * Component Buffers
+ */
+
+/*
+ * SOF memory capabilities, add new ones at the end
+ */
+#define SOF_MEM_CAPS_RAM			(1 << 0)
+#define SOF_MEM_CAPS_ROM			(1 << 1)
+#define SOF_MEM_CAPS_EXT			(1 << 2) /**< external */
+#define SOF_MEM_CAPS_LP			(1 << 3) /**< low power */
+#define SOF_MEM_CAPS_HP			(1 << 4) /**< high performance */
+#define SOF_MEM_CAPS_DMA			(1 << 5) /**< DMA'able */
+#define SOF_MEM_CAPS_CACHE			(1 << 6) /**< cacheable */
+#define SOF_MEM_CAPS_EXEC			(1 << 7) /**< executable */
+
+/* create new component buffer - SOF_IPC_TPLG_BUFFER_NEW */
+struct sof_ipc_buffer {
+	struct sof_ipc_comp comp;
+	uint32_t size;		/**< buffer size in bytes */
+	uint32_t caps;		/**< SOF_MEM_CAPS_ */
+} __packed;
+
+/* generic component config data - must always be after struct sof_ipc_comp */
+struct sof_ipc_comp_config {
+	struct sof_ipc_cmd_hdr hdr;
+	uint32_t periods_sink;	/**< 0 means variable */
+	uint32_t periods_source;	/**< 0 means variable */
+	uint32_t reserved1;	/**< reserved */
+	uint32_t frame_fmt;		/**< SOF_IPC_FRAME_ */
+	uint32_t xrun_action;
+
+	/* reserved for future use */
+	uint32_t reserved[2];
+} __packed;
+
+/* generic host component */
+struct sof_ipc_comp_host {
+	struct sof_ipc_comp comp;
+	struct sof_ipc_comp_config config;
+	uint32_t direction;	/**< SOF_IPC_STREAM_ */
+	uint32_t no_irq;	/**< don't send periodic IRQ to host/DSP */
+	uint32_t dmac_config; /**< DMA engine specific */
+}  __packed;
+
+/* generic DAI component */
+struct sof_ipc_comp_dai {
+	struct sof_ipc_comp comp;
+	struct sof_ipc_comp_config config;
+	uint32_t direction;	/**< SOF_IPC_STREAM_ */
+	uint32_t dai_index;	/**< index of this type dai */
+	uint32_t type;		/**< DAI type - SOF_DAI_ */
+	uint32_t reserved;	/**< reserved */
+}  __packed;
+
+/* generic mixer component */
+struct sof_ipc_comp_mixer {
+	struct sof_ipc_comp comp;
+	struct sof_ipc_comp_config config;
+}  __packed;
+
+/* volume ramping types */
+enum sof_volume_ramp {
+	SOF_VOLUME_LINEAR	= 0,
+	SOF_VOLUME_LOG,
+	SOF_VOLUME_LINEAR_ZC,
+	SOF_VOLUME_LOG_ZC,
+};
+
+/* generic volume component */
+struct sof_ipc_comp_volume {
+	struct sof_ipc_comp comp;
+	struct sof_ipc_comp_config config;
+	uint32_t channels;
+	uint32_t min_value;
+	uint32_t max_value;
+	uint32_t ramp;		/**< SOF_VOLUME_ */
+	uint32_t initial_ramp;	/**< ramp space in ms */
+}  __packed;
+
+/* generic SRC component */
+struct sof_ipc_comp_src {
+	struct sof_ipc_comp comp;
+	struct sof_ipc_comp_config config;
+	/* either source or sink rate must be non zero */
+	uint32_t source_rate;	/**< source rate or 0 for variable */
+	uint32_t sink_rate;	/**< sink rate or 0 for variable */
+	uint32_t rate_mask;	/**< SOF_RATE_ supported rates */
+} __packed;
+
+/* generic MUX component */
+struct sof_ipc_comp_mux {
+	struct sof_ipc_comp comp;
+	struct sof_ipc_comp_config config;
+} __packed;
+
+/* generic tone generator component */
+struct sof_ipc_comp_tone {
+	struct sof_ipc_comp comp;
+	struct sof_ipc_comp_config config;
+	int32_t sample_rate;
+	int32_t frequency;
+	int32_t amplitude;
+	int32_t freq_mult;
+	int32_t ampl_mult;
+	int32_t length;
+	int32_t period;
+	int32_t repeats;
+	int32_t ramp_step;
+} __packed;
+
+/** \brief Types of processing components */
+enum sof_ipc_process_type {
+	SOF_PROCESS_NONE = 0,		/**< None */
+	SOF_PROCESS_EQFIR,		/**< Intel FIR */
+	SOF_PROCESS_EQIIR,		/**< Intel IIR */
+	SOF_PROCESS_KEYWORD_DETECT,	/**< Keyword Detection */
+	SOF_PROCESS_KPB,		/**< KeyPhrase Buffer Manager */
+	SOF_PROCESS_CHAN_SELECTOR,	/**< Channel Selector */
+};
+
+/* generic "effect", "codec" or proprietary processing component */
+struct sof_ipc_comp_process {
+	struct sof_ipc_comp comp;
+	struct sof_ipc_comp_config config;
+	uint32_t size;	/**< size of bespoke data section in bytes */
+	uint32_t type;	/**< sof_ipc_process_type */
+
+	/* reserved for future use */
+	uint32_t reserved[7];
+
+	unsigned char data[0];
+} __packed;
+
+/* frees components, buffers and pipelines
+ * SOF_IPC_TPLG_COMP_FREE, SOF_IPC_TPLG_PIPE_FREE, SOF_IPC_TPLG_BUFFER_FREE
+ */
+struct sof_ipc_free {
+	struct sof_ipc_cmd_hdr hdr;
+	uint32_t id;
+} __packed;
+
+struct sof_ipc_comp_reply {
+	struct sof_ipc_reply rhdr;
+	uint32_t id;
+	uint32_t offset;
+} __packed;
+
+/*
+ * Pipeline
+ */
+
+/** \brief Types of pipeline scheduling time domains */
+enum sof_ipc_pipe_sched_time_domain {
+	SOF_TIME_DOMAIN_DMA = 0,	/**< DMA interrupt */
+	SOF_TIME_DOMAIN_TIMER,		/**< Timer interrupt */
+};
+
+/* new pipeline - SOF_IPC_TPLG_PIPE_NEW */
+struct sof_ipc_pipe_new {
+	struct sof_ipc_cmd_hdr hdr;
+	uint32_t comp_id;	/**< component id for pipeline */
+	uint32_t pipeline_id;	/**< pipeline id */
+	uint32_t sched_id;	/**< Scheduling component id */
+	uint32_t core;		/**< core we run on */
+	uint32_t period;	/**< execution period in us*/
+	uint32_t priority;	/**< priority level 0 (low) to 10 (max) */
+	uint32_t period_mips;	/**< worst case instruction count per period */
+	uint32_t frames_per_sched;/**< output frames of pipeline, 0 is variable */
+	uint32_t xrun_limit_usecs; /**< report xruns greater than limit */
+	uint32_t time_domain;	/**< scheduling time domain */
+}  __packed;
+
+/* pipeline construction complete - SOF_IPC_TPLG_PIPE_COMPLETE */
+struct sof_ipc_pipe_ready {
+	struct sof_ipc_cmd_hdr hdr;
+	uint32_t comp_id;
+}  __packed;
+
+struct sof_ipc_pipe_free {
+	struct sof_ipc_cmd_hdr hdr;
+	uint32_t comp_id;
+}  __packed;
+
+/* connect two components in pipeline - SOF_IPC_TPLG_COMP_CONNECT */
+struct sof_ipc_pipe_comp_connect {
+	struct sof_ipc_cmd_hdr hdr;
+	uint32_t source_id;
+	uint32_t sink_id;
+}  __packed;
+
+/* external events */
+enum sof_event_types {
+	SOF_EVENT_NONE = 0,
+	SOF_KEYWORD_DETECT_DAPM_EVENT,
+};
+
+#endif
diff --git a/include/sound/sof/trace.h b/include/sound/sof/trace.h
new file mode 100644
index 000000000000..7d211f319a92
--- /dev/null
+++ b/include/sound/sof/trace.h
@@ -0,0 +1,67 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */
+/*
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * Copyright(c) 2018 Intel Corporation. All rights reserved.
+ */
+
+#ifndef __INCLUDE_SOUND_SOF_TRACE_H__
+#define __INCLUDE_SOUND_SOF_TRACE_H__
+
+#include <sound/sof/header.h>
+#include <sound/sof/stream.h>
+
+/*
+ * DMA for Trace
+ */
+
+#define SOF_TRACE_FILENAME_SIZE		32
+
+/* DMA for Trace params info - SOF_IPC_DEBUG_DMA_PARAMS */
+struct sof_ipc_dma_trace_params {
+	struct sof_ipc_cmd_hdr hdr;
+	struct sof_ipc_host_buffer buffer;
+	uint32_t stream_tag;
+}  __packed;
+
+/* DMA for Trace params info - SOF_IPC_DEBUG_DMA_PARAMS */
+struct sof_ipc_dma_trace_posn {
+	struct sof_ipc_reply rhdr;
+	uint32_t host_offset;	/* Offset of DMA host buffer */
+	uint32_t overflow;	/* overflow bytes if any */
+	uint32_t messages;	/* total trace messages */
+}  __packed;
+
+/*
+ * Commom debug
+ */
+
+/*
+ * SOF panic codes
+ */
+#define SOF_IPC_PANIC_MAGIC			0x0dead000
+#define SOF_IPC_PANIC_MAGIC_MASK		0x0ffff000
+#define SOF_IPC_PANIC_CODE_MASK			0x00000fff
+#define SOF_IPC_PANIC_MEM			(SOF_IPC_PANIC_MAGIC | 0x0)
+#define SOF_IPC_PANIC_WORK			(SOF_IPC_PANIC_MAGIC | 0x1)
+#define SOF_IPC_PANIC_IPC			(SOF_IPC_PANIC_MAGIC | 0x2)
+#define SOF_IPC_PANIC_ARCH			(SOF_IPC_PANIC_MAGIC | 0x3)
+#define SOF_IPC_PANIC_PLATFORM			(SOF_IPC_PANIC_MAGIC | 0x4)
+#define SOF_IPC_PANIC_TASK			(SOF_IPC_PANIC_MAGIC | 0x5)
+#define SOF_IPC_PANIC_EXCEPTION			(SOF_IPC_PANIC_MAGIC | 0x6)
+#define SOF_IPC_PANIC_DEADLOCK			(SOF_IPC_PANIC_MAGIC | 0x7)
+#define SOF_IPC_PANIC_STACK			(SOF_IPC_PANIC_MAGIC | 0x8)
+#define SOF_IPC_PANIC_IDLE			(SOF_IPC_PANIC_MAGIC | 0x9)
+#define SOF_IPC_PANIC_WFI			(SOF_IPC_PANIC_MAGIC | 0xa)
+#define SOF_IPC_PANIC_ASSERT			(SOF_IPC_PANIC_MAGIC | 0xb)
+
+/* panic info include filename and line number */
+struct sof_ipc_panic_info {
+	struct sof_ipc_hdr hdr;
+	uint32_t code;			/* SOF_IPC_PANIC_ */
+	char filename[SOF_TRACE_FILENAME_SIZE];
+	uint32_t linenum;
+}  __packed;
+
+#endif
diff --git a/include/sound/sof/xtensa.h b/include/sound/sof/xtensa.h
new file mode 100644
index 000000000000..a7189984000d
--- /dev/null
+++ b/include/sound/sof/xtensa.h
@@ -0,0 +1,44 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */
+/*
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * Copyright(c) 2018 Intel Corporation. All rights reserved.
+ */
+
+#ifndef __INCLUDE_SOUND_SOF_XTENSA_H__
+#define __INCLUDE_SOUND_SOF_XTENSA_H__
+
+#include <sound/sof/header.h>
+
+/*
+ * Architecture specific debug
+ */
+
+/* Xtensa Firmware Oops data */
+struct sof_ipc_dsp_oops_xtensa {
+	struct sof_ipc_hdr hdr;
+	uint32_t exccause;
+	uint32_t excvaddr;
+	uint32_t ps;
+	uint32_t epc1;
+	uint32_t epc2;
+	uint32_t epc3;
+	uint32_t epc4;
+	uint32_t epc5;
+	uint32_t epc6;
+	uint32_t epc7;
+	uint32_t eps2;
+	uint32_t eps3;
+	uint32_t eps4;
+	uint32_t eps5;
+	uint32_t eps6;
+	uint32_t eps7;
+	uint32_t depc;
+	uint32_t intenable;
+	uint32_t interrupt;
+	uint32_t sar;
+	uint32_t stack;
+}  __packed;
+
+#endif
diff --git a/include/target/iscsi/iscsi_target_core.h b/include/target/iscsi/iscsi_target_core.h
index 24c398f4a68f..a49d37140a64 100644
--- a/include/target/iscsi/iscsi_target_core.h
+++ b/include/target/iscsi/iscsi_target_core.h
@@ -473,6 +473,7 @@ struct iscsi_cmd {
 	struct timer_list	dataout_timer;
 	/* Iovecs for SCSI data payload RX/TX w/ kernel level sockets */
 	struct kvec		*iov_data;
+	void			*overflow_buf;
 	/* Iovecs for miscellaneous purposes */
 #define ISCSI_MISC_IOVECS			5
 	struct kvec		iov_misc[ISCSI_MISC_IOVECS];
diff --git a/include/target/target_core_base.h b/include/target/target_core_base.h
index 19a5bf4214fc..7c9716fe731e 100644
--- a/include/target/target_core_base.h
+++ b/include/target/target_core_base.h
@@ -795,8 +795,8 @@ struct se_device {
 	spinlock_t		se_tmr_lock;
 	spinlock_t		qf_cmd_lock;
 	struct semaphore	caw_sem;
-	/* Used for legacy SPC-2 reservationsa */
-	struct se_node_acl	*dev_reserved_node_acl;
+	/* Used for legacy SPC-2 reservations */
+	struct se_session	*reservation_holder;
 	/* Used for ALUA Logical Unit Group membership */
 	struct t10_alua_lu_gp_member *dev_alua_lu_gp_mem;
 	/* Used for SPC-3 Persistent Reservations */
diff --git a/include/target/target_core_fabric.h b/include/target/target_core_fabric.h
index 8ed90407f062..063f133e47c2 100644
--- a/include/target/target_core_fabric.h
+++ b/include/target/target_core_fabric.h
@@ -142,6 +142,7 @@ void	transport_register_session(struct se_portal_group *,
 		struct se_node_acl *, struct se_session *, void *);
 ssize_t	target_show_dynamic_sessions(struct se_portal_group *, char *);
 void	transport_free_session(struct se_session *);
+void	target_spc2_release(struct se_node_acl *nacl);
 void	target_put_nacl(struct se_node_acl *);
 void	transport_deregister_session_configfs(struct se_session *);
 void	transport_deregister_session(struct se_session *);
diff --git a/include/trace/events/cgroup.h b/include/trace/events/cgroup.h
index a401ff5e7847..a566cc521476 100644
--- a/include/trace/events/cgroup.h
+++ b/include/trace/events/cgroup.h
@@ -103,6 +103,20 @@ DEFINE_EVENT(cgroup, cgroup_rename,
 	TP_ARGS(cgrp, path)
 );
 
+DEFINE_EVENT(cgroup, cgroup_freeze,
+
+	TP_PROTO(struct cgroup *cgrp, const char *path),
+
+	TP_ARGS(cgrp, path)
+);
+
+DEFINE_EVENT(cgroup, cgroup_unfreeze,
+
+	TP_PROTO(struct cgroup *cgrp, const char *path),
+
+	TP_ARGS(cgrp, path)
+);
+
 DECLARE_EVENT_CLASS(cgroup_migrate,
 
 	TP_PROTO(struct cgroup *dst_cgrp, const char *path,
@@ -149,6 +163,47 @@ DEFINE_EVENT(cgroup_migrate, cgroup_transfer_tasks,
 	TP_ARGS(dst_cgrp, path, task, threadgroup)
 );
 
+DECLARE_EVENT_CLASS(cgroup_event,
+
+	TP_PROTO(struct cgroup *cgrp, const char *path, int val),
+
+	TP_ARGS(cgrp, path, val),
+
+	TP_STRUCT__entry(
+		__field(	int,		root			)
+		__field(	int,		id			)
+		__field(	int,		level			)
+		__string(	path,		path			)
+		__field(	int,		val			)
+	),
+
+	TP_fast_assign(
+		__entry->root = cgrp->root->hierarchy_id;
+		__entry->id = cgrp->id;
+		__entry->level = cgrp->level;
+		__assign_str(path, path);
+		__entry->val = val;
+	),
+
+	TP_printk("root=%d id=%d level=%d path=%s val=%d",
+		  __entry->root, __entry->id, __entry->level, __get_str(path),
+		  __entry->val)
+);
+
+DEFINE_EVENT(cgroup_event, cgroup_notify_populated,
+
+	TP_PROTO(struct cgroup *cgrp, const char *path, int val),
+
+	TP_ARGS(cgrp, path, val)
+);
+
+DEFINE_EVENT(cgroup_event, cgroup_notify_frozen,
+
+	TP_PROTO(struct cgroup *cgrp, const char *path, int val),
+
+	TP_ARGS(cgrp, path, val)
+);
+
 #endif /* _TRACE_CGROUP_H */
 
 /* This part must be outside protection */
diff --git a/include/trace/events/ib_mad.h b/include/trace/events/ib_mad.h
new file mode 100644
index 000000000000..59363a083ecb
--- /dev/null
+++ b/include/trace/events/ib_mad.h
@@ -0,0 +1,390 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
+
+/*
+ * Copyright (c) 2018 Intel Corporation.  All rights reserved.
+ */
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM ib_mad
+
+#if !defined(_TRACE_IB_MAD_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_IB_MAD_H
+
+#include <linux/tracepoint.h>
+#include <rdma/ib_mad.h>
+
+#ifdef CONFIG_TRACEPOINTS
+struct trace_event_raw_ib_mad_send_template;
+static void create_mad_addr_info(struct ib_mad_send_wr_private *mad_send_wr,
+			  struct ib_mad_qp_info *qp_info,
+			  struct trace_event_raw_ib_mad_send_template *entry);
+#endif
+
+DECLARE_EVENT_CLASS(ib_mad_send_template,
+	TP_PROTO(struct ib_mad_send_wr_private *wr,
+		 struct ib_mad_qp_info *qp_info),
+	TP_ARGS(wr, qp_info),
+
+	TP_STRUCT__entry(
+		__field(u8,             base_version)
+		__field(u8,             mgmt_class)
+		__field(u8,             class_version)
+		__field(u8,             port_num)
+		__field(u32,            qp_num)
+		__field(u8,             method)
+		__field(u8,             sl)
+		__field(u16,            attr_id)
+		__field(u32,            attr_mod)
+		__field(u64,            wrtid)
+		__field(u64,            tid)
+		__field(u16,            status)
+		__field(u16,            class_specific)
+		__field(u32,            length)
+		__field(u32,            dlid)
+		__field(u32,            rqpn)
+		__field(u32,            rqkey)
+		__field(u32,            dev_index)
+		__field(void *,         agent_priv)
+		__field(unsigned long,  timeout)
+		__field(int,            retries_left)
+		__field(int,            max_retries)
+		__field(int,            retry)
+		__field(u16,            pkey)
+	),
+
+	TP_fast_assign(
+		__entry->dev_index = wr->mad_agent_priv->agent.device->index;
+		__entry->port_num = wr->mad_agent_priv->agent.port_num;
+		__entry->qp_num = wr->mad_agent_priv->qp_info->qp->qp_num;
+		__entry->agent_priv = wr->mad_agent_priv;
+		__entry->wrtid = wr->tid;
+		__entry->max_retries = wr->max_retries;
+		__entry->retries_left = wr->retries_left;
+		__entry->retry = wr->retry;
+		__entry->timeout = wr->timeout;
+		__entry->length = wr->send_buf.hdr_len +
+				  wr->send_buf.data_len;
+		__entry->base_version =
+			((struct ib_mad_hdr *)wr->send_buf.mad)->base_version;
+		__entry->mgmt_class =
+			((struct ib_mad_hdr *)wr->send_buf.mad)->mgmt_class;
+		__entry->class_version =
+			((struct ib_mad_hdr *)wr->send_buf.mad)->class_version;
+		__entry->method =
+			((struct ib_mad_hdr *)wr->send_buf.mad)->method;
+		__entry->status =
+			((struct ib_mad_hdr *)wr->send_buf.mad)->status;
+		__entry->class_specific =
+			((struct ib_mad_hdr *)wr->send_buf.mad)->class_specific;
+		__entry->tid = ((struct ib_mad_hdr *)wr->send_buf.mad)->tid;
+		__entry->attr_id =
+			((struct ib_mad_hdr *)wr->send_buf.mad)->attr_id;
+		__entry->attr_mod =
+			((struct ib_mad_hdr *)wr->send_buf.mad)->attr_mod;
+		create_mad_addr_info(wr, qp_info, __entry);
+	),
+
+	TP_printk("%d:%d QP%d agent %p: " \
+		  "wrtid 0x%llx; %d/%d retries(%d); timeout %lu length %d : " \
+		  "hdr : base_ver 0x%x class 0x%x class_ver 0x%x " \
+		  "method 0x%x status 0x%x class_specific 0x%x tid 0x%llx " \
+		  "attr_id 0x%x attr_mod 0x%x  => dlid 0x%08x sl %d "\
+		  "pkey 0x%x rpqn 0x%x rqpkey 0x%x",
+		__entry->dev_index, __entry->port_num, __entry->qp_num,
+		__entry->agent_priv, be64_to_cpu(__entry->wrtid),
+		__entry->retries_left, __entry->max_retries,
+		__entry->retry, __entry->timeout, __entry->length,
+		__entry->base_version, __entry->mgmt_class,
+		__entry->class_version,
+		__entry->method, be16_to_cpu(__entry->status),
+		be16_to_cpu(__entry->class_specific),
+		be64_to_cpu(__entry->tid), be16_to_cpu(__entry->attr_id),
+		be32_to_cpu(__entry->attr_mod),
+		be32_to_cpu(__entry->dlid), __entry->sl, __entry->pkey,
+		__entry->rqpn, __entry->rqkey
+	)
+);
+
+DEFINE_EVENT(ib_mad_send_template, ib_mad_error_handler,
+	TP_PROTO(struct ib_mad_send_wr_private *wr,
+		 struct ib_mad_qp_info *qp_info),
+	TP_ARGS(wr, qp_info));
+DEFINE_EVENT(ib_mad_send_template, ib_mad_ib_send_mad,
+	TP_PROTO(struct ib_mad_send_wr_private *wr,
+		 struct ib_mad_qp_info *qp_info),
+	TP_ARGS(wr, qp_info));
+DEFINE_EVENT(ib_mad_send_template, ib_mad_send_done_resend,
+	TP_PROTO(struct ib_mad_send_wr_private *wr,
+		 struct ib_mad_qp_info *qp_info),
+	TP_ARGS(wr, qp_info));
+
+TRACE_EVENT(ib_mad_send_done_handler,
+	TP_PROTO(struct ib_mad_send_wr_private *wr, struct ib_wc *wc),
+	TP_ARGS(wr, wc),
+
+	TP_STRUCT__entry(
+		__field(u8,             port_num)
+		__field(u8,             base_version)
+		__field(u8,             mgmt_class)
+		__field(u8,             class_version)
+		__field(u32,            qp_num)
+		__field(u64,            wrtid)
+		__field(u16,            status)
+		__field(u16,            wc_status)
+		__field(u32,            length)
+		__field(void *,         agent_priv)
+		__field(unsigned long,  timeout)
+		__field(u32,            dev_index)
+		__field(int,            retries_left)
+		__field(int,            max_retries)
+		__field(int,            retry)
+		__field(u8,             method)
+	),
+
+	TP_fast_assign(
+		__entry->dev_index = wr->mad_agent_priv->agent.device->index;
+		__entry->port_num = wr->mad_agent_priv->agent.port_num;
+		__entry->qp_num = wr->mad_agent_priv->qp_info->qp->qp_num;
+		__entry->agent_priv = wr->mad_agent_priv;
+		__entry->wrtid = wr->tid;
+		__entry->max_retries = wr->max_retries;
+		__entry->retries_left = wr->retries_left;
+		__entry->retry = wr->retry;
+		__entry->timeout = wr->timeout;
+		__entry->base_version =
+			((struct ib_mad_hdr *)wr->send_buf.mad)->base_version;
+		__entry->mgmt_class =
+			((struct ib_mad_hdr *)wr->send_buf.mad)->mgmt_class;
+		__entry->class_version =
+			((struct ib_mad_hdr *)wr->send_buf.mad)->class_version;
+		__entry->method =
+			((struct ib_mad_hdr *)wr->send_buf.mad)->method;
+		__entry->status =
+			((struct ib_mad_hdr *)wr->send_buf.mad)->status;
+		__entry->wc_status = wc->status;
+		__entry->length = wc->byte_len;
+	),
+
+	TP_printk("%d:%d QP%d : SEND WC Status %d : agent %p: " \
+		  "wrtid 0x%llx %d/%d retries(%d) timeout %lu length %d: " \
+		  "hdr : base_ver 0x%x class 0x%x class_ver 0x%x " \
+		  "method 0x%x status 0x%x",
+		__entry->dev_index, __entry->port_num, __entry->qp_num,
+		__entry->wc_status,
+		__entry->agent_priv, be64_to_cpu(__entry->wrtid),
+		__entry->retries_left, __entry->max_retries,
+		__entry->retry, __entry->timeout,
+		__entry->length,
+		__entry->base_version, __entry->mgmt_class,
+		__entry->class_version, __entry->method,
+		be16_to_cpu(__entry->status)
+	)
+);
+
+TRACE_EVENT(ib_mad_recv_done_handler,
+	TP_PROTO(struct ib_mad_qp_info *qp_info, struct ib_wc *wc,
+		 struct ib_mad_hdr *mad_hdr),
+	TP_ARGS(qp_info, wc, mad_hdr),
+
+	TP_STRUCT__entry(
+		__field(u8,             base_version)
+		__field(u8,             mgmt_class)
+		__field(u8,             class_version)
+		__field(u8,             port_num)
+		__field(u32,            qp_num)
+		__field(u16,            status)
+		__field(u16,            class_specific)
+		__field(u32,            length)
+		__field(u64,            tid)
+		__field(u8,             method)
+		__field(u8,             sl)
+		__field(u16,            attr_id)
+		__field(u32,            attr_mod)
+		__field(u16,            src_qp)
+		__field(u16,            wc_status)
+		__field(u32,            slid)
+		__field(u32,            dev_index)
+		__field(u16,            pkey)
+	),
+
+	TP_fast_assign(
+		__entry->dev_index = qp_info->port_priv->device->index;
+		__entry->port_num = qp_info->port_priv->port_num;
+		__entry->qp_num = qp_info->qp->qp_num;
+		__entry->length = wc->byte_len;
+		__entry->base_version = mad_hdr->base_version;
+		__entry->mgmt_class = mad_hdr->mgmt_class;
+		__entry->class_version = mad_hdr->class_version;
+		__entry->method = mad_hdr->method;
+		__entry->status = mad_hdr->status;
+		__entry->class_specific = mad_hdr->class_specific;
+		__entry->tid = mad_hdr->tid;
+		__entry->attr_id = mad_hdr->attr_id;
+		__entry->attr_mod = mad_hdr->attr_mod;
+		__entry->slid = wc->slid;
+		__entry->src_qp = wc->src_qp;
+		__entry->sl = wc->sl;
+		ib_query_pkey(qp_info->port_priv->device,
+			      qp_info->port_priv->port_num,
+			      wc->pkey_index, &__entry->pkey);
+		__entry->wc_status = wc->status;
+	),
+
+	TP_printk("%d:%d QP%d : RECV WC Status %d : length %d : hdr : " \
+		  "base_ver 0x%02x class 0x%02x class_ver 0x%02x " \
+		  "method 0x%02x status 0x%04x class_specific 0x%04x " \
+		  "tid 0x%016llx attr_id 0x%04x attr_mod 0x%08x " \
+		  "slid 0x%08x src QP%d, sl %d pkey 0x%04x",
+		__entry->dev_index, __entry->port_num, __entry->qp_num,
+		__entry->wc_status,
+		__entry->length,
+		__entry->base_version, __entry->mgmt_class,
+		__entry->class_version, __entry->method,
+		be16_to_cpu(__entry->status),
+		be16_to_cpu(__entry->class_specific),
+		be64_to_cpu(__entry->tid), be16_to_cpu(__entry->attr_id),
+		be32_to_cpu(__entry->attr_mod),
+		__entry->slid, __entry->src_qp, __entry->sl, __entry->pkey
+	)
+);
+
+DECLARE_EVENT_CLASS(ib_mad_agent_template,
+	TP_PROTO(struct ib_mad_agent_private *agent),
+	TP_ARGS(agent),
+
+	TP_STRUCT__entry(
+		__field(u32,            dev_index)
+		__field(u32,            hi_tid)
+		__field(u8,             port_num)
+		__field(u8,             mgmt_class)
+		__field(u8,             mgmt_class_version)
+	),
+
+	TP_fast_assign(
+		__entry->dev_index = agent->agent.device->index;
+		__entry->port_num = agent->agent.port_num;
+		__entry->hi_tid = agent->agent.hi_tid;
+
+		if (agent->reg_req) {
+			__entry->mgmt_class = agent->reg_req->mgmt_class;
+			__entry->mgmt_class_version =
+				agent->reg_req->mgmt_class_version;
+		} else {
+			__entry->mgmt_class = 0;
+			__entry->mgmt_class_version = 0;
+		}
+	),
+
+	TP_printk("%d:%d mad agent : hi_tid 0x%08x class 0x%02x class_ver 0x%02x",
+		__entry->dev_index, __entry->port_num,
+		__entry->hi_tid, __entry->mgmt_class,
+		__entry->mgmt_class_version
+	)
+);
+DEFINE_EVENT(ib_mad_agent_template, ib_mad_recv_done_agent,
+	TP_PROTO(struct ib_mad_agent_private *agent),
+	TP_ARGS(agent));
+DEFINE_EVENT(ib_mad_agent_template, ib_mad_send_done_agent,
+	TP_PROTO(struct ib_mad_agent_private *agent),
+	TP_ARGS(agent));
+DEFINE_EVENT(ib_mad_agent_template, ib_mad_create_agent,
+	TP_PROTO(struct ib_mad_agent_private *agent),
+	TP_ARGS(agent));
+DEFINE_EVENT(ib_mad_agent_template, ib_mad_unregister_agent,
+	TP_PROTO(struct ib_mad_agent_private *agent),
+	TP_ARGS(agent));
+
+
+
+DECLARE_EVENT_CLASS(ib_mad_opa_smi_template,
+	TP_PROTO(struct opa_smp *smp),
+	TP_ARGS(smp),
+
+	TP_STRUCT__entry(
+		__field(u64,            mkey)
+		__field(u32,            dr_slid)
+		__field(u32,            dr_dlid)
+		__field(u8,             hop_ptr)
+		__field(u8,             hop_cnt)
+		__array(u8,             initial_path, OPA_SMP_MAX_PATH_HOPS)
+		__array(u8,             return_path, OPA_SMP_MAX_PATH_HOPS)
+	),
+
+	TP_fast_assign(
+		__entry->hop_ptr = smp->hop_ptr;
+		__entry->hop_cnt = smp->hop_cnt;
+		__entry->mkey = smp->mkey;
+		__entry->dr_slid = smp->route.dr.dr_slid;
+		__entry->dr_dlid = smp->route.dr.dr_dlid;
+		memcpy(__entry->initial_path, smp->route.dr.initial_path,
+			OPA_SMP_MAX_PATH_HOPS);
+		memcpy(__entry->return_path, smp->route.dr.return_path,
+			OPA_SMP_MAX_PATH_HOPS);
+	),
+
+	TP_printk("OPA SMP: hop_ptr %d hop_cnt %d " \
+		  "mkey 0x%016llx dr_slid 0x%08x dr_dlid 0x%08x " \
+		  "initial_path %*ph return_path %*ph ",
+		__entry->hop_ptr, __entry->hop_cnt,
+		be64_to_cpu(__entry->mkey), be32_to_cpu(__entry->dr_slid),
+		be32_to_cpu(__entry->dr_dlid),
+		OPA_SMP_MAX_PATH_HOPS, __entry->initial_path,
+		OPA_SMP_MAX_PATH_HOPS, __entry->return_path
+	)
+);
+
+DEFINE_EVENT(ib_mad_opa_smi_template, ib_mad_handle_opa_smi,
+	TP_PROTO(struct opa_smp *smp),
+	TP_ARGS(smp));
+DEFINE_EVENT(ib_mad_opa_smi_template, ib_mad_handle_out_opa_smi,
+	TP_PROTO(struct opa_smp *smp),
+	TP_ARGS(smp));
+
+
+DECLARE_EVENT_CLASS(ib_mad_opa_ib_template,
+	TP_PROTO(struct ib_smp *smp),
+	TP_ARGS(smp),
+
+	TP_STRUCT__entry(
+		__field(u64,            mkey)
+		__field(u32,            dr_slid)
+		__field(u32,            dr_dlid)
+		__field(u8,             hop_ptr)
+		__field(u8,             hop_cnt)
+		__array(u8,             initial_path, IB_SMP_MAX_PATH_HOPS)
+		__array(u8,             return_path, IB_SMP_MAX_PATH_HOPS)
+	),
+
+	TP_fast_assign(
+		__entry->hop_ptr = smp->hop_ptr;
+		__entry->hop_cnt = smp->hop_cnt;
+		__entry->mkey = smp->mkey;
+		__entry->dr_slid = smp->dr_slid;
+		__entry->dr_dlid = smp->dr_dlid;
+		memcpy(__entry->initial_path, smp->initial_path,
+			IB_SMP_MAX_PATH_HOPS);
+		memcpy(__entry->return_path, smp->return_path,
+			IB_SMP_MAX_PATH_HOPS);
+	),
+
+	TP_printk("OPA SMP: hop_ptr %d hop_cnt %d " \
+		  "mkey 0x%016llx dr_slid 0x%04x dr_dlid 0x%04x " \
+		  "initial_path %*ph return_path %*ph ",
+		__entry->hop_ptr, __entry->hop_cnt,
+		be64_to_cpu(__entry->mkey), be16_to_cpu(__entry->dr_slid),
+		be16_to_cpu(__entry->dr_dlid),
+		IB_SMP_MAX_PATH_HOPS, __entry->initial_path,
+		IB_SMP_MAX_PATH_HOPS, __entry->return_path
+	)
+);
+
+DEFINE_EVENT(ib_mad_opa_ib_template, ib_mad_handle_ib_smi,
+	TP_PROTO(struct ib_smp *smp),
+	TP_ARGS(smp));
+DEFINE_EVENT(ib_mad_opa_ib_template, ib_mad_handle_out_ib_smi,
+	TP_PROTO(struct ib_smp *smp),
+	TP_ARGS(smp));
+
+#endif /* _TRACE_IB_MAD_H */
+
+#include <trace/define_trace.h>
diff --git a/include/trace/events/ib_umad.h b/include/trace/events/ib_umad.h
new file mode 100644
index 000000000000..c393a19a0f60
--- /dev/null
+++ b/include/trace/events/ib_umad.h
@@ -0,0 +1,126 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
+
+/*
+ * Copyright (c) 2018 Intel Corporation.  All rights reserved.
+ *
+ */
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM ib_umad
+
+#if !defined(_TRACE_IB_UMAD_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_IB_UMAD_H
+
+#include <linux/tracepoint.h>
+
+DECLARE_EVENT_CLASS(ib_umad_template,
+	TP_PROTO(struct ib_umad_file *file, struct ib_user_mad_hdr *umad_hdr,
+		 struct ib_mad_hdr *mad_hdr),
+	TP_ARGS(file, umad_hdr, mad_hdr),
+
+	TP_STRUCT__entry(
+		__field(u8, port_num)
+		__field(u8, sl)
+		__field(u8, path_bits)
+		__field(u8, grh_present)
+		__field(u32, id)
+		__field(u32, status)
+		__field(u32, timeout_ms)
+		__field(u32, retires)
+		__field(u32, length)
+		__field(u32, qpn)
+		__field(u32, qkey)
+		__field(u8, gid_index)
+		__field(u8, hop_limit)
+		__field(u16, lid)
+		__field(u16, attr_id)
+		__field(u16, pkey_index)
+		__field(u8, base_version)
+		__field(u8, mgmt_class)
+		__field(u8, class_version)
+		__field(u8, method)
+		__field(u32, flow_label)
+		__field(u16, mad_status)
+		__field(u16, class_specific)
+		__field(u32, attr_mod)
+		__field(u64, tid)
+		__array(u8, gid, 16)
+		__field(u32, dev_index)
+		__field(u8,  traffic_class)
+	),
+
+	TP_fast_assign(
+		__entry->dev_index = file->port->ib_dev->index;
+		__entry->port_num = file->port->port_num;
+
+		__entry->id = umad_hdr->id;
+		__entry->status = umad_hdr->status;
+		__entry->timeout_ms = umad_hdr->timeout_ms;
+		__entry->retires = umad_hdr->retries;
+		__entry->length = umad_hdr->length;
+		__entry->qpn = umad_hdr->qpn;
+		__entry->qkey = umad_hdr->qkey;
+		__entry->lid = umad_hdr->lid;
+		__entry->sl = umad_hdr->sl;
+		__entry->path_bits = umad_hdr->path_bits;
+		__entry->grh_present = umad_hdr->grh_present;
+		__entry->gid_index = umad_hdr->gid_index;
+		__entry->hop_limit = umad_hdr->hop_limit;
+		__entry->traffic_class = umad_hdr->traffic_class;
+		memcpy(__entry->gid, umad_hdr->gid, sizeof(umad_hdr->gid));
+		__entry->flow_label = umad_hdr->flow_label;
+		__entry->pkey_index = umad_hdr->pkey_index;
+
+		__entry->base_version = mad_hdr->base_version;
+		__entry->mgmt_class = mad_hdr->mgmt_class;
+		__entry->class_version = mad_hdr->class_version;
+		__entry->method = mad_hdr->method;
+		__entry->mad_status = mad_hdr->status;
+		__entry->class_specific = mad_hdr->class_specific;
+		__entry->tid = mad_hdr->tid;
+		__entry->attr_id = mad_hdr->attr_id;
+		__entry->attr_mod = mad_hdr->attr_mod;
+	),
+
+	TP_printk("%d:%d umad_hdr: id 0x%08x status 0x%08x ms %u ret %u " \
+		  "len %u QP%u qkey 0x%08x lid 0x%04x sl %u path_bits 0x%x " \
+		  "grh 0x%x gidi %u hop_lim %u traf_cl %u gid %pI6c " \
+		  "flow 0x%08x pkeyi %u  MAD: base_ver 0x%x class 0x%x " \
+		  "class_ver 0x%x method 0x%x status 0x%04x " \
+		  "class_specific 0x%04x tid 0x%016llx attr_id 0x%04x " \
+		  "attr_mod 0x%08x ",
+		__entry->dev_index, __entry->port_num,
+		__entry->id, __entry->status, __entry->timeout_ms,
+		__entry->retires, __entry->length, be32_to_cpu(__entry->qpn),
+		be32_to_cpu(__entry->qkey), be16_to_cpu(__entry->lid),
+		__entry->sl, __entry->path_bits, __entry->grh_present,
+		__entry->gid_index, __entry->hop_limit,
+		__entry->traffic_class, &__entry->gid,
+		be32_to_cpu(__entry->flow_label), __entry->pkey_index,
+		__entry->base_version, __entry->mgmt_class,
+		__entry->class_version, __entry->method,
+		be16_to_cpu(__entry->mad_status),
+		be16_to_cpu(__entry->class_specific),
+		be64_to_cpu(__entry->tid), be16_to_cpu(__entry->attr_id),
+		be32_to_cpu(__entry->attr_mod)
+	)
+);
+
+DEFINE_EVENT(ib_umad_template, ib_umad_write,
+	TP_PROTO(struct ib_umad_file *file, struct ib_user_mad_hdr *umad_hdr,
+		 struct ib_mad_hdr *mad_hdr),
+	TP_ARGS(file, umad_hdr, mad_hdr));
+
+DEFINE_EVENT(ib_umad_template, ib_umad_read_recv,
+	TP_PROTO(struct ib_umad_file *file, struct ib_user_mad_hdr *umad_hdr,
+		 struct ib_mad_hdr *mad_hdr),
+	TP_ARGS(file, umad_hdr, mad_hdr));
+
+DEFINE_EVENT(ib_umad_template, ib_umad_read_send,
+	TP_PROTO(struct ib_umad_file *file, struct ib_user_mad_hdr *umad_hdr,
+		 struct ib_mad_hdr *mad_hdr),
+	TP_ARGS(file, umad_hdr, mad_hdr));
+
+#endif /* _TRACE_IB_UMAD_H */
+
+#include <trace/define_trace.h>
diff --git a/include/trace/events/rpcrdma.h b/include/trace/events/rpcrdma.h
index 962975b4313f..df9851cb82b2 100644
--- a/include/trace/events/rpcrdma.h
+++ b/include/trace/events/rpcrdma.h
@@ -511,6 +511,33 @@ TRACE_EVENT(xprtrdma_marshal,
 	)
 );
 
+TRACE_EVENT(xprtrdma_marshal_failed,
+	TP_PROTO(const struct rpc_rqst *rqst,
+		 int ret
+	),
+
+	TP_ARGS(rqst, ret),
+
+	TP_STRUCT__entry(
+		__field(unsigned int, task_id)
+		__field(unsigned int, client_id)
+		__field(u32, xid)
+		__field(int, ret)
+	),
+
+	TP_fast_assign(
+		__entry->task_id = rqst->rq_task->tk_pid;
+		__entry->client_id = rqst->rq_task->tk_client->cl_clid;
+		__entry->xid = be32_to_cpu(rqst->rq_xid);
+		__entry->ret = ret;
+	),
+
+	TP_printk("task:%u@%u xid=0x%08x: ret=%d",
+		__entry->task_id, __entry->client_id, __entry->xid,
+		__entry->ret
+	)
+);
+
 TRACE_EVENT(xprtrdma_post_send,
 	TP_PROTO(
 		const struct rpcrdma_req *req,
diff --git a/include/trace/events/sunrpc.h b/include/trace/events/sunrpc.h
index f0a6f0c5549c..ffa3c51dbb1a 100644
--- a/include/trace/events/sunrpc.h
+++ b/include/trace/events/sunrpc.h
@@ -82,7 +82,6 @@ TRACE_DEFINE_ENUM(RPC_TASK_SWAPPER);
 TRACE_DEFINE_ENUM(RPC_CALL_MAJORSEEN);
 TRACE_DEFINE_ENUM(RPC_TASK_ROOTCREDS);
 TRACE_DEFINE_ENUM(RPC_TASK_DYNAMIC);
-TRACE_DEFINE_ENUM(RPC_TASK_KILLED);
 TRACE_DEFINE_ENUM(RPC_TASK_SOFT);
 TRACE_DEFINE_ENUM(RPC_TASK_SOFTCONN);
 TRACE_DEFINE_ENUM(RPC_TASK_SENT);
@@ -97,7 +96,6 @@ TRACE_DEFINE_ENUM(RPC_TASK_NO_RETRANS_TIMEOUT);
 		{ RPC_CALL_MAJORSEEN, "MAJORSEEN" },			\
 		{ RPC_TASK_ROOTCREDS, "ROOTCREDS" },			\
 		{ RPC_TASK_DYNAMIC, "DYNAMIC" },			\
-		{ RPC_TASK_KILLED, "KILLED" },				\
 		{ RPC_TASK_SOFT, "SOFT" },				\
 		{ RPC_TASK_SOFTCONN, "SOFTCONN" },			\
 		{ RPC_TASK_SENT, "SENT" },				\
@@ -111,6 +109,7 @@ TRACE_DEFINE_ENUM(RPC_TASK_ACTIVE);
 TRACE_DEFINE_ENUM(RPC_TASK_NEED_XMIT);
 TRACE_DEFINE_ENUM(RPC_TASK_NEED_RECV);
 TRACE_DEFINE_ENUM(RPC_TASK_MSG_PIN_WAIT);
+TRACE_DEFINE_ENUM(RPC_TASK_SIGNALLED);
 
 #define rpc_show_runstate(flags)					\
 	__print_flags(flags, "|",					\
@@ -119,7 +118,8 @@ TRACE_DEFINE_ENUM(RPC_TASK_MSG_PIN_WAIT);
 		{ (1UL << RPC_TASK_ACTIVE), "ACTIVE" },			\
 		{ (1UL << RPC_TASK_NEED_XMIT), "NEED_XMIT" },		\
 		{ (1UL << RPC_TASK_NEED_RECV), "NEED_RECV" },		\
-		{ (1UL << RPC_TASK_MSG_PIN_WAIT), "MSG_PIN_WAIT" })
+		{ (1UL << RPC_TASK_MSG_PIN_WAIT), "MSG_PIN_WAIT" },	\
+		{ (1UL << RPC_TASK_SIGNALLED), "SIGNALLED" })
 
 DECLARE_EVENT_CLASS(rpc_task_running,
 
@@ -186,7 +186,7 @@ DECLARE_EVENT_CLASS(rpc_task_queued,
 		__entry->client_id = task->tk_client ?
 				     task->tk_client->cl_clid : -1;
 		__entry->task_id = task->tk_pid;
-		__entry->timeout = task->tk_timeout;
+		__entry->timeout = rpc_task_timeout(task);
 		__entry->runstate = task->tk_runstate;
 		__entry->status = task->tk_status;
 		__entry->flags = task->tk_flags;
diff --git a/include/uapi/drm/amdgpu_drm.h b/include/uapi/drm/amdgpu_drm.h
index 4a53f6cfa034..4788730dbe78 100644
--- a/include/uapi/drm/amdgpu_drm.h
+++ b/include/uapi/drm/amdgpu_drm.h
@@ -210,6 +210,9 @@ union drm_amdgpu_bo_list {
 #define AMDGPU_CTX_QUERY2_FLAGS_VRAMLOST (1<<1)
 /* indicate some job from this context once cause gpu hang */
 #define AMDGPU_CTX_QUERY2_FLAGS_GUILTY   (1<<2)
+/* indicate some errors are detected by RAS */
+#define AMDGPU_CTX_QUERY2_FLAGS_RAS_CE   (1<<3)
+#define AMDGPU_CTX_QUERY2_FLAGS_RAS_UE   (1<<4)
 
 /* Context priority level */
 #define AMDGPU_CTX_PRIORITY_UNSET       -2048
@@ -525,6 +528,8 @@ struct drm_amdgpu_gem_va {
 #define AMDGPU_CHUNK_ID_SYNCOBJ_OUT     0x05
 #define AMDGPU_CHUNK_ID_BO_HANDLES      0x06
 #define AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES	0x07
+#define AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT    0x08
+#define AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL  0x09
 
 struct drm_amdgpu_cs_chunk {
 	__u32		chunk_id;
@@ -605,6 +610,12 @@ struct drm_amdgpu_cs_chunk_sem {
 	__u32 handle;
 };
 
+struct drm_amdgpu_cs_chunk_syncobj {
+       __u32 handle;
+       __u32 flags;
+       __u64 point;
+};
+
 #define AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ	0
 #define AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ_FD	1
 #define AMDGPU_FENCE_TO_HANDLE_GET_SYNC_FILE_FD	2
@@ -680,6 +691,7 @@ struct drm_amdgpu_cs_chunk_data {
 	#define AMDGPU_INFO_FW_GFX_RLC_RESTORE_LIST_SRM_MEM 0x11
 	/* Subquery id: Query DMCU firmware version */
 	#define AMDGPU_INFO_FW_DMCU		0x12
+	#define AMDGPU_INFO_FW_TA		0x13
 /* number of bytes moved for TTM migration */
 #define AMDGPU_INFO_NUM_BYTES_MOVED		0x0f
 /* the used VRAM size */
@@ -733,6 +745,37 @@ struct drm_amdgpu_cs_chunk_data {
 /* Number of VRAM page faults on CPU access. */
 #define AMDGPU_INFO_NUM_VRAM_CPU_PAGE_FAULTS	0x1E
 #define AMDGPU_INFO_VRAM_LOST_COUNTER		0x1F
+/* query ras mask of enabled features*/
+#define AMDGPU_INFO_RAS_ENABLED_FEATURES	0x20
+
+/* RAS MASK: UMC (VRAM) */
+#define AMDGPU_INFO_RAS_ENABLED_UMC			(1 << 0)
+/* RAS MASK: SDMA */
+#define AMDGPU_INFO_RAS_ENABLED_SDMA			(1 << 1)
+/* RAS MASK: GFX */
+#define AMDGPU_INFO_RAS_ENABLED_GFX			(1 << 2)
+/* RAS MASK: MMHUB */
+#define AMDGPU_INFO_RAS_ENABLED_MMHUB			(1 << 3)
+/* RAS MASK: ATHUB */
+#define AMDGPU_INFO_RAS_ENABLED_ATHUB			(1 << 4)
+/* RAS MASK: PCIE */
+#define AMDGPU_INFO_RAS_ENABLED_PCIE			(1 << 5)
+/* RAS MASK: HDP */
+#define AMDGPU_INFO_RAS_ENABLED_HDP			(1 << 6)
+/* RAS MASK: XGMI */
+#define AMDGPU_INFO_RAS_ENABLED_XGMI			(1 << 7)
+/* RAS MASK: DF */
+#define AMDGPU_INFO_RAS_ENABLED_DF			(1 << 8)
+/* RAS MASK: SMN */
+#define AMDGPU_INFO_RAS_ENABLED_SMN			(1 << 9)
+/* RAS MASK: SEM */
+#define AMDGPU_INFO_RAS_ENABLED_SEM			(1 << 10)
+/* RAS MASK: MP0 */
+#define AMDGPU_INFO_RAS_ENABLED_MP0			(1 << 11)
+/* RAS MASK: MP1 */
+#define AMDGPU_INFO_RAS_ENABLED_MP1			(1 << 12)
+/* RAS MASK: FUSE */
+#define AMDGPU_INFO_RAS_ENABLED_FUSE			(1 << 13)
 
 #define AMDGPU_INFO_MMR_SE_INDEX_SHIFT	0
 #define AMDGPU_INFO_MMR_SE_INDEX_MASK	0xff
diff --git a/include/uapi/drm/drm.h b/include/uapi/drm/drm.h
index 300f336633f2..661d73f9a919 100644
--- a/include/uapi/drm/drm.h
+++ b/include/uapi/drm/drm.h
@@ -649,6 +649,7 @@ struct drm_gem_open {
 #define DRM_CAP_PAGE_FLIP_TARGET	0x11
 #define DRM_CAP_CRTC_IN_VBLANK_EVENT	0x12
 #define DRM_CAP_SYNCOBJ		0x13
+#define DRM_CAP_SYNCOBJ_TIMELINE	0x14
 
 /** DRM_IOCTL_GET_CAP ioctl argument type */
 struct drm_get_cap {
@@ -735,8 +736,18 @@ struct drm_syncobj_handle {
 	__u32 pad;
 };
 
+struct drm_syncobj_transfer {
+	__u32 src_handle;
+	__u32 dst_handle;
+	__u64 src_point;
+	__u64 dst_point;
+	__u32 flags;
+	__u32 pad;
+};
+
 #define DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL (1 << 0)
 #define DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT (1 << 1)
+#define DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE (1 << 2) /* wait for time point to become available */
 struct drm_syncobj_wait {
 	__u64 handles;
 	/* absolute timeout */
@@ -747,12 +758,33 @@ struct drm_syncobj_wait {
 	__u32 pad;
 };
 
+struct drm_syncobj_timeline_wait {
+	__u64 handles;
+	/* wait on specific timeline point for every handles*/
+	__u64 points;
+	/* absolute timeout */
+	__s64 timeout_nsec;
+	__u32 count_handles;
+	__u32 flags;
+	__u32 first_signaled; /* only valid when not waiting all */
+	__u32 pad;
+};
+
+
 struct drm_syncobj_array {
 	__u64 handles;
 	__u32 count_handles;
 	__u32 pad;
 };
 
+struct drm_syncobj_timeline_array {
+	__u64 handles;
+	__u64 points;
+	__u32 count_handles;
+	__u32 pad;
+};
+
+
 /* Query current scanout sequence number */
 struct drm_crtc_get_sequence {
 	__u32 crtc_id;		/* requested crtc_id */
@@ -909,6 +941,11 @@ extern "C" {
 #define DRM_IOCTL_MODE_GET_LEASE	DRM_IOWR(0xC8, struct drm_mode_get_lease)
 #define DRM_IOCTL_MODE_REVOKE_LEASE	DRM_IOWR(0xC9, struct drm_mode_revoke_lease)
 
+#define DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT	DRM_IOWR(0xCA, struct drm_syncobj_timeline_wait)
+#define DRM_IOCTL_SYNCOBJ_QUERY		DRM_IOWR(0xCB, struct drm_syncobj_timeline_array)
+#define DRM_IOCTL_SYNCOBJ_TRANSFER	DRM_IOWR(0xCC, struct drm_syncobj_transfer)
+#define DRM_IOCTL_SYNCOBJ_TIMELINE_SIGNAL	DRM_IOWR(0xCD, struct drm_syncobj_timeline_array)
+
 /**
  * Device specific ioctls should only be in their respective headers
  * The device specific ioctl range is from 0x40 to 0x9f.
diff --git a/include/uapi/drm/drm_fourcc.h b/include/uapi/drm/drm_fourcc.h
index bab20298f422..3feeaa3f987a 100644
--- a/include/uapi/drm/drm_fourcc.h
+++ b/include/uapi/drm/drm_fourcc.h
@@ -144,6 +144,17 @@ extern "C" {
 #define DRM_FORMAT_RGBA1010102	fourcc_code('R', 'A', '3', '0') /* [31:0] R:G:B:A 10:10:10:2 little endian */
 #define DRM_FORMAT_BGRA1010102	fourcc_code('B', 'A', '3', '0') /* [31:0] B:G:R:A 10:10:10:2 little endian */
 
+/*
+ * Floating point 64bpp RGB
+ * IEEE 754-2008 binary16 half-precision float
+ * [15:0] sign:exponent:mantissa 1:5:10
+ */
+#define DRM_FORMAT_XRGB16161616F fourcc_code('X', 'R', '4', 'H') /* [63:0] x:R:G:B 16:16:16:16 little endian */
+#define DRM_FORMAT_XBGR16161616F fourcc_code('X', 'B', '4', 'H') /* [63:0] x:B:G:R 16:16:16:16 little endian */
+
+#define DRM_FORMAT_ARGB16161616F fourcc_code('A', 'R', '4', 'H') /* [63:0] A:R:G:B 16:16:16:16 little endian */
+#define DRM_FORMAT_ABGR16161616F fourcc_code('A', 'B', '4', 'H') /* [63:0] A:B:G:R 16:16:16:16 little endian */
+
 /* packed YCbCr */
 #define DRM_FORMAT_YUYV		fourcc_code('Y', 'U', 'Y', 'V') /* [31:0] Cr0:Y1:Cb0:Y0 8:8:8:8 little endian */
 #define DRM_FORMAT_YVYU		fourcc_code('Y', 'V', 'Y', 'U') /* [31:0] Cb0:Y1:Cr0:Y0 8:8:8:8 little endian */
@@ -151,7 +162,29 @@ extern "C" {
 #define DRM_FORMAT_VYUY		fourcc_code('V', 'Y', 'U', 'Y') /* [31:0] Y1:Cb0:Y0:Cr0 8:8:8:8 little endian */
 
 #define DRM_FORMAT_AYUV		fourcc_code('A', 'Y', 'U', 'V') /* [31:0] A:Y:Cb:Cr 8:8:8:8 little endian */
-#define DRM_FORMAT_XYUV8888		fourcc_code('X', 'Y', 'U', 'V') /* [31:0] X:Y:Cb:Cr 8:8:8:8 little endian */
+#define DRM_FORMAT_XYUV8888	fourcc_code('X', 'Y', 'U', 'V') /* [31:0] X:Y:Cb:Cr 8:8:8:8 little endian */
+#define DRM_FORMAT_VUY888	fourcc_code('V', 'U', '2', '4') /* [23:0] Cr:Cb:Y 8:8:8 little endian */
+#define DRM_FORMAT_VUY101010	fourcc_code('V', 'U', '3', '0') /* Y followed by U then V, 10:10:10. Non-linear modifier only */
+
+/*
+ * packed Y2xx indicate for each component, xx valid data occupy msb
+ * 16-xx padding occupy lsb
+ */
+#define DRM_FORMAT_Y210         fourcc_code('Y', '2', '1', '0') /* [63:0] Cr0:0:Y1:0:Cb0:0:Y0:0 10:6:10:6:10:6:10:6 little endian per 2 Y pixels */
+#define DRM_FORMAT_Y212         fourcc_code('Y', '2', '1', '2') /* [63:0] Cr0:0:Y1:0:Cb0:0:Y0:0 12:4:12:4:12:4:12:4 little endian per 2 Y pixels */
+#define DRM_FORMAT_Y216         fourcc_code('Y', '2', '1', '6') /* [63:0] Cr0:Y1:Cb0:Y0 16:16:16:16 little endian per 2 Y pixels */
+
+/*
+ * packed Y4xx indicate for each component, xx valid data occupy msb
+ * 16-xx padding occupy lsb except Y410
+ */
+#define DRM_FORMAT_Y410         fourcc_code('Y', '4', '1', '0') /* [31:0] A:Cr:Y:Cb 2:10:10:10 little endian */
+#define DRM_FORMAT_Y412         fourcc_code('Y', '4', '1', '2') /* [63:0] A:0:Cr:0:Y:0:Cb:0 12:4:12:4:12:4:12:4 little endian */
+#define DRM_FORMAT_Y416         fourcc_code('Y', '4', '1', '6') /* [63:0] A:Cr:Y:Cb 16:16:16:16 little endian */
+
+#define DRM_FORMAT_XVYU2101010	fourcc_code('X', 'V', '3', '0') /* [31:0] X:Cr:Y:Cb 2:10:10:10 little endian */
+#define DRM_FORMAT_XVYU12_16161616	fourcc_code('X', 'V', '3', '6') /* [63:0] X:0:Cr:0:Y:0:Cb:0 12:4:12:4:12:4:12:4 little endian */
+#define DRM_FORMAT_XVYU16161616	fourcc_code('X', 'V', '4', '8') /* [63:0] X:Cr:Y:Cb 16:16:16:16 little endian */
 
 /*
  * packed YCbCr420 2x2 tiled formats
@@ -168,6 +201,15 @@ extern "C" {
 #define DRM_FORMAT_X0L2		fourcc_code('X', '0', 'L', '2')
 
 /*
+ * 1-plane YUV 4:2:0
+ * In these formats, the component ordering is specified (Y, followed by U
+ * then V), but the exact Linear layout is undefined.
+ * These formats can only be used with a non-Linear modifier.
+ */
+#define DRM_FORMAT_YUV420_8BIT	fourcc_code('Y', 'U', '0', '8')
+#define DRM_FORMAT_YUV420_10BIT	fourcc_code('Y', 'U', '1', '0')
+
+/*
  * 2 plane RGB + A
  * index 0 = RGB plane, same format as the corresponding non _A8 format has
  * index 1 = A plane, [7:0] A
@@ -200,6 +242,13 @@ extern "C" {
  * index 0 = Y plane, [15:0] Y:x [10:6] little endian
  * index 1 = Cr:Cb plane, [31:0] Cr:x:Cb:x [10:6:10:6] little endian
  */
+#define DRM_FORMAT_P210		fourcc_code('P', '2', '1', '0') /* 2x1 subsampled Cr:Cb plane, 10 bit per channel */
+
+/*
+ * 2 plane YCbCr MSB aligned
+ * index 0 = Y plane, [15:0] Y:x [10:6] little endian
+ * index 1 = Cr:Cb plane, [31:0] Cr:x:Cb:x [10:6:10:6] little endian
+ */
 #define DRM_FORMAT_P010		fourcc_code('P', '0', '1', '0') /* 2x2 subsampled Cr:Cb plane 10 bits per channel */
 
 /*
diff --git a/include/uapi/drm/drm_mode.h b/include/uapi/drm/drm_mode.h
index a439c2e67896..83cd1636b9be 100644
--- a/include/uapi/drm/drm_mode.h
+++ b/include/uapi/drm/drm_mode.h
@@ -33,7 +33,6 @@
 extern "C" {
 #endif
 
-#define DRM_DISPLAY_INFO_LEN	32
 #define DRM_CONNECTOR_NAME_LEN	32
 #define DRM_DISPLAY_MODE_LEN	32
 #define DRM_PROP_NAME_LEN	32
@@ -622,7 +621,8 @@ struct drm_color_ctm {
 
 struct drm_color_lut {
 	/*
-	 * Data is U0.16 fixed point format.
+	 * Values are mapped linearly to 0.0 - 1.0 range, with 0x0 == 0.0 and
+	 * 0xffff == 1.0.
 	 */
 	__u16 red;
 	__u16 green;
diff --git a/include/uapi/drm/i915_drm.h b/include/uapi/drm/i915_drm.h
index 397810fa2d33..3a73f5316766 100644
--- a/include/uapi/drm/i915_drm.h
+++ b/include/uapi/drm/i915_drm.h
@@ -63,6 +63,28 @@ extern "C" {
 #define I915_RESET_UEVENT		"RESET"
 
 /*
+ * i915_user_extension: Base class for defining a chain of extensions
+ *
+ * Many interfaces need to grow over time. In most cases we can simply
+ * extend the struct and have userspace pass in more data. Another option,
+ * as demonstrated by Vulkan's approach to providing extensions for forward
+ * and backward compatibility, is to use a list of optional structs to
+ * provide those extra details.
+ *
+ * The key advantage to using an extension chain is that it allows us to
+ * redefine the interface more easily than an ever growing struct of
+ * increasing complexity, and for large parts of that interface to be
+ * entirely optional. The downside is more pointer chasing; chasing across
+ * the __user boundary with pointers encapsulated inside u64.
+ */
+struct i915_user_extension {
+	__u64 next_extension;
+	__u32 name;
+	__u32 flags; /* All undefined bits must be zero. */
+	__u32 rsvd[4]; /* Reserved for future use; must be zero. */
+};
+
+/*
  * MOCS indexes used for GPU surfaces, defining the cacheability of the
  * surface data and the coherency for this data wrt. CPU vs. GPU accesses.
  */
@@ -99,9 +121,23 @@ enum drm_i915_gem_engine_class {
 	I915_ENGINE_CLASS_VIDEO		= 2,
 	I915_ENGINE_CLASS_VIDEO_ENHANCE	= 3,
 
+	/* should be kept compact */
+
 	I915_ENGINE_CLASS_INVALID	= -1
 };
 
+/*
+ * There may be more than one engine fulfilling any role within the system.
+ * Each engine of a class is given a unique instance number and therefore
+ * any engine can be specified by its class:instance tuplet. APIs that allow
+ * access to any engine in the system will use struct i915_engine_class_instance
+ * for this identification.
+ */
+struct i915_engine_class_instance {
+	__u16 engine_class; /* see enum drm_i915_gem_engine_class */
+	__u16 engine_instance;
+};
+
 /**
  * DOC: perf_events exposed by i915 through /sys/bus/event_sources/drivers/i915
  *
@@ -319,6 +355,7 @@ typedef struct _drm_i915_sarea {
 #define DRM_I915_PERF_ADD_CONFIG	0x37
 #define DRM_I915_PERF_REMOVE_CONFIG	0x38
 #define DRM_I915_QUERY			0x39
+/* Must be kept compact -- no holes */
 
 #define DRM_IOCTL_I915_INIT		DRM_IOW( DRM_COMMAND_BASE + DRM_I915_INIT, drm_i915_init_t)
 #define DRM_IOCTL_I915_FLUSH		DRM_IO ( DRM_COMMAND_BASE + DRM_I915_FLUSH)
@@ -367,6 +404,7 @@ typedef struct _drm_i915_sarea {
 #define DRM_IOCTL_I915_GET_SPRITE_COLORKEY DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GET_SPRITE_COLORKEY, struct drm_intel_sprite_colorkey)
 #define DRM_IOCTL_I915_GEM_WAIT		DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GEM_WAIT, struct drm_i915_gem_wait)
 #define DRM_IOCTL_I915_GEM_CONTEXT_CREATE	DRM_IOWR (DRM_COMMAND_BASE + DRM_I915_GEM_CONTEXT_CREATE, struct drm_i915_gem_context_create)
+#define DRM_IOCTL_I915_GEM_CONTEXT_CREATE_EXT	DRM_IOWR (DRM_COMMAND_BASE + DRM_I915_GEM_CONTEXT_CREATE, struct drm_i915_gem_context_create_ext)
 #define DRM_IOCTL_I915_GEM_CONTEXT_DESTROY	DRM_IOW (DRM_COMMAND_BASE + DRM_I915_GEM_CONTEXT_DESTROY, struct drm_i915_gem_context_destroy)
 #define DRM_IOCTL_I915_REG_READ			DRM_IOWR (DRM_COMMAND_BASE + DRM_I915_REG_READ, struct drm_i915_reg_read)
 #define DRM_IOCTL_I915_GET_RESET_STATS		DRM_IOWR (DRM_COMMAND_BASE + DRM_I915_GET_RESET_STATS, struct drm_i915_reset_stats)
@@ -476,6 +514,7 @@ typedef struct drm_i915_irq_wait {
 #define   I915_SCHEDULER_CAP_ENABLED	(1ul << 0)
 #define   I915_SCHEDULER_CAP_PRIORITY	(1ul << 1)
 #define   I915_SCHEDULER_CAP_PREEMPTION	(1ul << 2)
+#define   I915_SCHEDULER_CAP_SEMAPHORES	(1ul << 3)
 
 #define I915_PARAM_HUC_STATUS		 42
 
@@ -559,6 +598,8 @@ typedef struct drm_i915_irq_wait {
  */
 #define I915_PARAM_MMAP_GTT_COHERENT	52
 
+/* Must be kept compact -- no holes and well documented */
+
 typedef struct drm_i915_getparam {
 	__s32 param;
 	/*
@@ -574,6 +615,7 @@ typedef struct drm_i915_getparam {
 #define I915_SETPARAM_TEX_LRU_LOG_GRANULARITY             2
 #define I915_SETPARAM_ALLOW_BATCHBUFFER                   3
 #define I915_SETPARAM_NUM_USED_FENCES                     4
+/* Must be kept compact -- no holes */
 
 typedef struct drm_i915_setparam {
 	int param;
@@ -972,7 +1014,7 @@ struct drm_i915_gem_execbuffer2 {
 	 * struct drm_i915_gem_exec_fence *fences.
 	 */
 	__u64 cliprects_ptr;
-#define I915_EXEC_RING_MASK              (7<<0)
+#define I915_EXEC_RING_MASK              (0x3f)
 #define I915_EXEC_DEFAULT                (0<<0)
 #define I915_EXEC_RENDER                 (1<<0)
 #define I915_EXEC_BSD                    (2<<0)
@@ -1120,32 +1162,34 @@ struct drm_i915_gem_busy {
 	 * as busy may become idle before the ioctl is completed.
 	 *
 	 * Furthermore, if the object is busy, which engine is busy is only
-	 * provided as a guide. There are race conditions which prevent the
-	 * report of which engines are busy from being always accurate.
-	 * However, the converse is not true. If the object is idle, the
-	 * result of the ioctl, that all engines are idle, is accurate.
+	 * provided as a guide and only indirectly by reporting its class
+	 * (there may be more than one engine in each class). There are race
+	 * conditions which prevent the report of which engines are busy from
+	 * being always accurate.  However, the converse is not true. If the
+	 * object is idle, the result of the ioctl, that all engines are idle,
+	 * is accurate.
 	 *
 	 * The returned dword is split into two fields to indicate both
-	 * the engines on which the object is being read, and the
-	 * engine on which it is currently being written (if any).
+	 * the engine classess on which the object is being read, and the
+	 * engine class on which it is currently being written (if any).
 	 *
 	 * The low word (bits 0:15) indicate if the object is being written
 	 * to by any engine (there can only be one, as the GEM implicit
 	 * synchronisation rules force writes to be serialised). Only the
-	 * engine for the last write is reported.
+	 * engine class (offset by 1, I915_ENGINE_CLASS_RENDER is reported as
+	 * 1 not 0 etc) for the last write is reported.
 	 *
-	 * The high word (bits 16:31) are a bitmask of which engines are
-	 * currently reading from the object. Multiple engines may be
+	 * The high word (bits 16:31) are a bitmask of which engines classes
+	 * are currently reading from the object. Multiple engines may be
 	 * reading from the object simultaneously.
 	 *
-	 * The value of each engine is the same as specified in the
-	 * EXECBUFFER2 ioctl, i.e. I915_EXEC_RENDER, I915_EXEC_BSD etc.
-	 * Note I915_EXEC_DEFAULT is a symbolic value and is mapped to
-	 * the I915_EXEC_RENDER engine for execution, and so it is never
+	 * The value of each engine class is the same as specified in the
+	 * I915_CONTEXT_SET_ENGINES parameter and via perf, i.e.
+	 * I915_ENGINE_CLASS_RENDER, I915_ENGINE_CLASS_COPY, etc.
 	 * reported as active itself. Some hardware may have parallel
 	 * execution engines, e.g. multiple media engines, which are
-	 * mapped to the same identifier in the EXECBUFFER2 ioctl and
-	 * so are not separately reported for busyness.
+	 * mapped to the same class identifier and so are not separately
+	 * reported for busyness.
 	 *
 	 * Caveat emptor:
 	 * Only the boolean result of this query is reliable; that is whether
@@ -1412,65 +1456,17 @@ struct drm_i915_gem_wait {
 };
 
 struct drm_i915_gem_context_create {
-	/*  output: id of new context*/
-	__u32 ctx_id;
-	__u32 pad;
-};
-
-struct drm_i915_gem_context_destroy {
-	__u32 ctx_id;
+	__u32 ctx_id; /* output: id of new context*/
 	__u32 pad;
 };
 
-struct drm_i915_reg_read {
-	/*
-	 * Register offset.
-	 * For 64bit wide registers where the upper 32bits don't immediately
-	 * follow the lower 32bits, the offset of the lower 32bits must
-	 * be specified
-	 */
-	__u64 offset;
-#define I915_REG_READ_8B_WA (1ul << 0)
-
-	__u64 val; /* Return value */
-};
-/* Known registers:
- *
- * Render engine timestamp - 0x2358 + 64bit - gen7+
- * - Note this register returns an invalid value if using the default
- *   single instruction 8byte read, in order to workaround that pass
- *   flag I915_REG_READ_8B_WA in offset field.
- *
- */
-
-struct drm_i915_reset_stats {
-	__u32 ctx_id;
-	__u32 flags;
-
-	/* All resets since boot/module reload, for all contexts */
-	__u32 reset_count;
-
-	/* Number of batches lost when active in GPU, for this context */
-	__u32 batch_active;
-
-	/* Number of batches lost pending for execution, for this context */
-	__u32 batch_pending;
-
-	__u32 pad;
-};
-
-struct drm_i915_gem_userptr {
-	__u64 user_ptr;
-	__u64 user_size;
+struct drm_i915_gem_context_create_ext {
+	__u32 ctx_id; /* output: id of new context*/
 	__u32 flags;
-#define I915_USERPTR_READ_ONLY 0x1
-#define I915_USERPTR_UNSYNCHRONIZED 0x80000000
-	/**
-	 * Returned handle for the object.
-	 *
-	 * Object handles are nonzero.
-	 */
-	__u32 handle;
+#define I915_CONTEXT_CREATE_FLAGS_USE_EXTENSIONS	(1u << 0)
+#define I915_CONTEXT_CREATE_FLAGS_UNKNOWN \
+	(-(I915_CONTEXT_CREATE_FLAGS_USE_EXTENSIONS << 1))
+	__u64 extensions;
 };
 
 struct drm_i915_gem_context_param {
@@ -1491,6 +1487,28 @@ struct drm_i915_gem_context_param {
 	 * drm_i915_gem_context_param_sseu.
 	 */
 #define I915_CONTEXT_PARAM_SSEU		0x7
+
+/*
+ * Not all clients may want to attempt automatic recover of a context after
+ * a hang (for example, some clients may only submit very small incremental
+ * batches relying on known logical state of previous batches which will never
+ * recover correctly and each attempt will hang), and so would prefer that
+ * the context is forever banned instead.
+ *
+ * If set to false (0), after a reset, subsequent (and in flight) rendering
+ * from this context is discarded, and the client will need to create a new
+ * context to use instead.
+ *
+ * If set to true (1), the kernel will automatically attempt to recover the
+ * context by skipping the hanging batch and executing the next batch starting
+ * from the default context state (discarding the incomplete logical context
+ * state lost due to the reset).
+ *
+ * On creation, all new contexts are marked as recoverable.
+ */
+#define I915_CONTEXT_PARAM_RECOVERABLE	0x8
+/* Must be kept compact -- no holes and well documented */
+
 	__u64 value;
 };
 
@@ -1519,8 +1537,7 @@ struct drm_i915_gem_context_param_sseu {
 	/*
 	 * Engine class & instance to be configured or queried.
 	 */
-	__u16 engine_class;
-	__u16 engine_instance;
+	struct i915_engine_class_instance engine;
 
 	/*
 	 * Unused for now. Must be cleared to zero.
@@ -1553,6 +1570,96 @@ struct drm_i915_gem_context_param_sseu {
 	__u32 rsvd;
 };
 
+struct drm_i915_gem_context_create_ext_setparam {
+#define I915_CONTEXT_CREATE_EXT_SETPARAM 0
+	struct i915_user_extension base;
+	struct drm_i915_gem_context_param param;
+};
+
+struct drm_i915_gem_context_destroy {
+	__u32 ctx_id;
+	__u32 pad;
+};
+
+/*
+ * DRM_I915_GEM_VM_CREATE -
+ *
+ * Create a new virtual memory address space (ppGTT) for use within a context
+ * on the same file. Extensions can be provided to configure exactly how the
+ * address space is setup upon creation.
+ *
+ * The id of new VM (bound to the fd) for use with I915_CONTEXT_PARAM_VM is
+ * returned in the outparam @id.
+ *
+ * No flags are defined, with all bits reserved and must be zero.
+ *
+ * An extension chain maybe provided, starting with @extensions, and terminated
+ * by the @next_extension being 0. Currently, no extensions are defined.
+ *
+ * DRM_I915_GEM_VM_DESTROY -
+ *
+ * Destroys a previously created VM id, specified in @id.
+ *
+ * No extensions or flags are allowed currently, and so must be zero.
+ */
+struct drm_i915_gem_vm_control {
+	__u64 extensions;
+	__u32 flags;
+	__u32 vm_id;
+};
+
+struct drm_i915_reg_read {
+	/*
+	 * Register offset.
+	 * For 64bit wide registers where the upper 32bits don't immediately
+	 * follow the lower 32bits, the offset of the lower 32bits must
+	 * be specified
+	 */
+	__u64 offset;
+#define I915_REG_READ_8B_WA (1ul << 0)
+
+	__u64 val; /* Return value */
+};
+
+/* Known registers:
+ *
+ * Render engine timestamp - 0x2358 + 64bit - gen7+
+ * - Note this register returns an invalid value if using the default
+ *   single instruction 8byte read, in order to workaround that pass
+ *   flag I915_REG_READ_8B_WA in offset field.
+ *
+ */
+
+struct drm_i915_reset_stats {
+	__u32 ctx_id;
+	__u32 flags;
+
+	/* All resets since boot/module reload, for all contexts */
+	__u32 reset_count;
+
+	/* Number of batches lost when active in GPU, for this context */
+	__u32 batch_active;
+
+	/* Number of batches lost pending for execution, for this context */
+	__u32 batch_pending;
+
+	__u32 pad;
+};
+
+struct drm_i915_gem_userptr {
+	__u64 user_ptr;
+	__u64 user_size;
+	__u32 flags;
+#define I915_USERPTR_READ_ONLY 0x1
+#define I915_USERPTR_UNSYNCHRONIZED 0x80000000
+	/**
+	 * Returned handle for the object.
+	 *
+	 * Object handles are nonzero.
+	 */
+	__u32 handle;
+};
+
 enum drm_i915_oa_format {
 	I915_OA_FORMAT_A13 = 1,	    /* HSW only */
 	I915_OA_FORMAT_A29,	    /* HSW only */
@@ -1714,6 +1821,7 @@ struct drm_i915_perf_oa_config {
 struct drm_i915_query_item {
 	__u64 query_id;
 #define DRM_I915_QUERY_TOPOLOGY_INFO    1
+/* Must be kept compact -- no holes and well documented */
 
 	/*
 	 * When set to zero by userspace, this is filled with the size of the
diff --git a/include/uapi/drm/lima_drm.h b/include/uapi/drm/lima_drm.h
new file mode 100644
index 000000000000..95a00fb867e6
--- /dev/null
+++ b/include/uapi/drm/lima_drm.h
@@ -0,0 +1,169 @@
+/* SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) OR MIT */
+/* Copyright 2017-2018 Qiang Yu <yuq825@gmail.com> */
+
+#ifndef __LIMA_DRM_H__
+#define __LIMA_DRM_H__
+
+#include "drm.h"
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+enum drm_lima_param_gpu_id {
+	DRM_LIMA_PARAM_GPU_ID_UNKNOWN,
+	DRM_LIMA_PARAM_GPU_ID_MALI400,
+	DRM_LIMA_PARAM_GPU_ID_MALI450,
+};
+
+enum drm_lima_param {
+	DRM_LIMA_PARAM_GPU_ID,
+	DRM_LIMA_PARAM_NUM_PP,
+	DRM_LIMA_PARAM_GP_VERSION,
+	DRM_LIMA_PARAM_PP_VERSION,
+};
+
+/**
+ * get various information of the GPU
+ */
+struct drm_lima_get_param {
+	__u32 param; /* in, value in enum drm_lima_param */
+	__u32 pad;   /* pad, must be zero */
+	__u64 value; /* out, parameter value */
+};
+
+/**
+ * create a buffer for used by GPU
+ */
+struct drm_lima_gem_create {
+	__u32 size;    /* in, buffer size */
+	__u32 flags;   /* in, currently no flags, must be zero */
+	__u32 handle;  /* out, GEM buffer handle */
+	__u32 pad;     /* pad, must be zero */
+};
+
+/**
+ * get information of a buffer
+ */
+struct drm_lima_gem_info {
+	__u32 handle;  /* in, GEM buffer handle */
+	__u32 va;      /* out, virtual address mapped into GPU MMU */
+	__u64 offset;  /* out, used to mmap this buffer to CPU */
+};
+
+#define LIMA_SUBMIT_BO_READ   0x01
+#define LIMA_SUBMIT_BO_WRITE  0x02
+
+/* buffer information used by one task */
+struct drm_lima_gem_submit_bo {
+	__u32 handle;  /* in, GEM buffer handle */
+	__u32 flags;   /* in, buffer read/write by GPU */
+};
+
+#define LIMA_GP_FRAME_REG_NUM 6
+
+/* frame used to setup GP for each task */
+struct drm_lima_gp_frame {
+	__u32 frame[LIMA_GP_FRAME_REG_NUM];
+};
+
+#define LIMA_PP_FRAME_REG_NUM 23
+#define LIMA_PP_WB_REG_NUM 12
+
+/* frame used to setup mali400 GPU PP for each task */
+struct drm_lima_m400_pp_frame {
+	__u32 frame[LIMA_PP_FRAME_REG_NUM];
+	__u32 num_pp;
+	__u32 wb[3 * LIMA_PP_WB_REG_NUM];
+	__u32 plbu_array_address[4];
+	__u32 fragment_stack_address[4];
+};
+
+/* frame used to setup mali450 GPU PP for each task */
+struct drm_lima_m450_pp_frame {
+	__u32 frame[LIMA_PP_FRAME_REG_NUM];
+	__u32 num_pp;
+	__u32 wb[3 * LIMA_PP_WB_REG_NUM];
+	__u32 use_dlbu;
+	__u32 _pad;
+	union {
+		__u32 plbu_array_address[8];
+		__u32 dlbu_regs[4];
+	};
+	__u32 fragment_stack_address[8];
+};
+
+#define LIMA_PIPE_GP  0x00
+#define LIMA_PIPE_PP  0x01
+
+#define LIMA_SUBMIT_FLAG_EXPLICIT_FENCE (1 << 0)
+
+/**
+ * submit a task to GPU
+ *
+ * User can always merge multi sync_file and drm_syncobj
+ * into one drm_syncobj as in_sync[0], but we reserve
+ * in_sync[1] for another task's out_sync to avoid the
+ * export/import/merge pass when explicit sync.
+ */
+struct drm_lima_gem_submit {
+	__u32 ctx;         /* in, context handle task is submitted to */
+	__u32 pipe;        /* in, which pipe to use, GP/PP */
+	__u32 nr_bos;      /* in, array length of bos field */
+	__u32 frame_size;  /* in, size of frame field */
+	__u64 bos;         /* in, array of drm_lima_gem_submit_bo */
+	__u64 frame;       /* in, GP/PP frame */
+	__u32 flags;       /* in, submit flags */
+	__u32 out_sync;    /* in, drm_syncobj handle used to wait task finish after submission */
+	__u32 in_sync[2];  /* in, drm_syncobj handle used to wait before start this task */
+};
+
+#define LIMA_GEM_WAIT_READ   0x01
+#define LIMA_GEM_WAIT_WRITE  0x02
+
+/**
+ * wait pending GPU task finish of a buffer
+ */
+struct drm_lima_gem_wait {
+	__u32 handle;      /* in, GEM buffer handle */
+	__u32 op;          /* in, CPU want to read/write this buffer */
+	__s64 timeout_ns;  /* in, wait timeout in absulute time */
+};
+
+/**
+ * create a context
+ */
+struct drm_lima_ctx_create {
+	__u32 id;          /* out, context handle */
+	__u32 _pad;        /* pad, must be zero */
+};
+
+/**
+ * free a context
+ */
+struct drm_lima_ctx_free {
+	__u32 id;          /* in, context handle */
+	__u32 _pad;        /* pad, must be zero */
+};
+
+#define DRM_LIMA_GET_PARAM   0x00
+#define DRM_LIMA_GEM_CREATE  0x01
+#define DRM_LIMA_GEM_INFO    0x02
+#define DRM_LIMA_GEM_SUBMIT  0x03
+#define DRM_LIMA_GEM_WAIT    0x04
+#define DRM_LIMA_CTX_CREATE  0x05
+#define DRM_LIMA_CTX_FREE    0x06
+
+#define DRM_IOCTL_LIMA_GET_PARAM DRM_IOWR(DRM_COMMAND_BASE + DRM_LIMA_GET_PARAM, struct drm_lima_get_param)
+#define DRM_IOCTL_LIMA_GEM_CREATE DRM_IOWR(DRM_COMMAND_BASE + DRM_LIMA_GEM_CREATE, struct drm_lima_gem_create)
+#define DRM_IOCTL_LIMA_GEM_INFO DRM_IOWR(DRM_COMMAND_BASE + DRM_LIMA_GEM_INFO, struct drm_lima_gem_info)
+#define DRM_IOCTL_LIMA_GEM_SUBMIT DRM_IOW(DRM_COMMAND_BASE + DRM_LIMA_GEM_SUBMIT, struct drm_lima_gem_submit)
+#define DRM_IOCTL_LIMA_GEM_WAIT DRM_IOW(DRM_COMMAND_BASE + DRM_LIMA_GEM_WAIT, struct drm_lima_gem_wait)
+#define DRM_IOCTL_LIMA_CTX_CREATE DRM_IOR(DRM_COMMAND_BASE + DRM_LIMA_CTX_CREATE, struct drm_lima_ctx_create)
+#define DRM_IOCTL_LIMA_CTX_FREE DRM_IOW(DRM_COMMAND_BASE + DRM_LIMA_CTX_FREE, struct drm_lima_ctx_free)
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* __LIMA_DRM_H__ */
diff --git a/include/uapi/drm/msm_drm.h b/include/uapi/drm/msm_drm.h
index 91a16b333c69..0b85ed6a3710 100644
--- a/include/uapi/drm/msm_drm.h
+++ b/include/uapi/drm/msm_drm.h
@@ -74,6 +74,8 @@ struct drm_msm_timespec {
 #define MSM_PARAM_TIMESTAMP  0x05
 #define MSM_PARAM_GMEM_BASE  0x06
 #define MSM_PARAM_NR_RINGS   0x07
+#define MSM_PARAM_PP_PGTABLE 0x08  /* => 1 for per-process pagetables, else 0 */
+#define MSM_PARAM_FAULTS     0x09
 
 struct drm_msm_param {
 	__u32 pipe;           /* in, MSM_PIPE_x */
@@ -286,6 +288,16 @@ struct drm_msm_submitqueue {
 	__u32 id;      /* out, identifier */
 };
 
+#define MSM_SUBMITQUEUE_PARAM_FAULTS   0
+
+struct drm_msm_submitqueue_query {
+	__u64 data;
+	__u32 id;
+	__u32 param;
+	__u32 len;
+	__u32 pad;
+};
+
 #define DRM_MSM_GET_PARAM              0x00
 /* placeholder:
 #define DRM_MSM_SET_PARAM              0x01
@@ -302,6 +314,7 @@ struct drm_msm_submitqueue {
  */
 #define DRM_MSM_SUBMITQUEUE_NEW        0x0A
 #define DRM_MSM_SUBMITQUEUE_CLOSE      0x0B
+#define DRM_MSM_SUBMITQUEUE_QUERY      0x0C
 
 #define DRM_IOCTL_MSM_GET_PARAM        DRM_IOWR(DRM_COMMAND_BASE + DRM_MSM_GET_PARAM, struct drm_msm_param)
 #define DRM_IOCTL_MSM_GEM_NEW          DRM_IOWR(DRM_COMMAND_BASE + DRM_MSM_GEM_NEW, struct drm_msm_gem_new)
@@ -313,6 +326,7 @@ struct drm_msm_submitqueue {
 #define DRM_IOCTL_MSM_GEM_MADVISE      DRM_IOWR(DRM_COMMAND_BASE + DRM_MSM_GEM_MADVISE, struct drm_msm_gem_madvise)
 #define DRM_IOCTL_MSM_SUBMITQUEUE_NEW    DRM_IOWR(DRM_COMMAND_BASE + DRM_MSM_SUBMITQUEUE_NEW, struct drm_msm_submitqueue)
 #define DRM_IOCTL_MSM_SUBMITQUEUE_CLOSE  DRM_IOW (DRM_COMMAND_BASE + DRM_MSM_SUBMITQUEUE_CLOSE, __u32)
+#define DRM_IOCTL_MSM_SUBMITQUEUE_QUERY  DRM_IOW (DRM_COMMAND_BASE + DRM_MSM_SUBMITQUEUE_QUERY, struct drm_msm_submitqueue_query)
 
 #if defined(__cplusplus)
 }
diff --git a/include/uapi/drm/panfrost_drm.h b/include/uapi/drm/panfrost_drm.h
new file mode 100644
index 000000000000..a52e0283b90d
--- /dev/null
+++ b/include/uapi/drm/panfrost_drm.h
@@ -0,0 +1,142 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2014-2018 Broadcom
+ * Copyright © 2019 Collabora ltd.
+ */
+#ifndef _PANFROST_DRM_H_
+#define _PANFROST_DRM_H_
+
+#include "drm.h"
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#define DRM_PANFROST_SUBMIT			0x00
+#define DRM_PANFROST_WAIT_BO			0x01
+#define DRM_PANFROST_CREATE_BO			0x02
+#define DRM_PANFROST_MMAP_BO			0x03
+#define DRM_PANFROST_GET_PARAM			0x04
+#define DRM_PANFROST_GET_BO_OFFSET		0x05
+
+#define DRM_IOCTL_PANFROST_SUBMIT		DRM_IOW(DRM_COMMAND_BASE + DRM_PANFROST_SUBMIT, struct drm_panfrost_submit)
+#define DRM_IOCTL_PANFROST_WAIT_BO		DRM_IOW(DRM_COMMAND_BASE + DRM_PANFROST_WAIT_BO, struct drm_panfrost_wait_bo)
+#define DRM_IOCTL_PANFROST_CREATE_BO		DRM_IOWR(DRM_COMMAND_BASE + DRM_PANFROST_CREATE_BO, struct drm_panfrost_create_bo)
+#define DRM_IOCTL_PANFROST_MMAP_BO		DRM_IOWR(DRM_COMMAND_BASE + DRM_PANFROST_MMAP_BO, struct drm_panfrost_mmap_bo)
+#define DRM_IOCTL_PANFROST_GET_PARAM		DRM_IOWR(DRM_COMMAND_BASE + DRM_PANFROST_GET_PARAM, struct drm_panfrost_get_param)
+#define DRM_IOCTL_PANFROST_GET_BO_OFFSET	DRM_IOWR(DRM_COMMAND_BASE + DRM_PANFROST_GET_BO_OFFSET, struct drm_panfrost_get_bo_offset)
+
+#define PANFROST_JD_REQ_FS (1 << 0)
+/**
+ * struct drm_panfrost_submit - ioctl argument for submitting commands to the 3D
+ * engine.
+ *
+ * This asks the kernel to have the GPU execute a render command list.
+ */
+struct drm_panfrost_submit {
+
+	/** Address to GPU mapping of job descriptor */
+	__u64 jc;
+
+	/** An optional array of sync objects to wait on before starting this job. */
+	__u64 in_syncs;
+
+	/** Number of sync objects to wait on before starting this job. */
+	__u32 in_sync_count;
+
+	/** An optional sync object to place the completion fence in. */
+	__u32 out_sync;
+
+	/** Pointer to a u32 array of the BOs that are referenced by the job. */
+	__u64 bo_handles;
+
+	/** Number of BO handles passed in (size is that times 4). */
+	__u32 bo_handle_count;
+
+	/** A combination of PANFROST_JD_REQ_* */
+	__u32 requirements;
+};
+
+/**
+ * struct drm_panfrost_wait_bo - ioctl argument for waiting for
+ * completion of the last DRM_PANFROST_SUBMIT on a BO.
+ *
+ * This is useful for cases where multiple processes might be
+ * rendering to a BO and you want to wait for all rendering to be
+ * completed.
+ */
+struct drm_panfrost_wait_bo {
+	__u32 handle;
+	__u32 pad;
+	__s64 timeout_ns;	/* absolute */
+};
+
+/**
+ * struct drm_panfrost_create_bo - ioctl argument for creating Panfrost BOs.
+ *
+ * There are currently no values for the flags argument, but it may be
+ * used in a future extension.
+ */
+struct drm_panfrost_create_bo {
+	__u32 size;
+	__u32 flags;
+	/** Returned GEM handle for the BO. */
+	__u32 handle;
+	/* Pad, must be zero-filled. */
+	__u32 pad;
+	/**
+	 * Returned offset for the BO in the GPU address space.  This offset
+	 * is private to the DRM fd and is valid for the lifetime of the GEM
+	 * handle.
+	 *
+	 * This offset value will always be nonzero, since various HW
+	 * units treat 0 specially.
+	 */
+	__u64 offset;
+};
+
+/**
+ * struct drm_panfrost_mmap_bo - ioctl argument for mapping Panfrost BOs.
+ *
+ * This doesn't actually perform an mmap.  Instead, it returns the
+ * offset you need to use in an mmap on the DRM device node.  This
+ * means that tools like valgrind end up knowing about the mapped
+ * memory.
+ *
+ * There are currently no values for the flags argument, but it may be
+ * used in a future extension.
+ */
+struct drm_panfrost_mmap_bo {
+	/** Handle for the object being mapped. */
+	__u32 handle;
+	__u32 flags;
+	/** offset into the drm node to use for subsequent mmap call. */
+	__u64 offset;
+};
+
+enum drm_panfrost_param {
+	DRM_PANFROST_PARAM_GPU_PROD_ID,
+};
+
+struct drm_panfrost_get_param {
+	__u32 param;
+	__u32 pad;
+	__u64 value;
+};
+
+/**
+ * Returns the offset for the BO in the GPU address space for this DRM fd.
+ * This is the same value returned by drm_panfrost_create_bo, if that was called
+ * from this DRM fd.
+ */
+struct drm_panfrost_get_bo_offset {
+	__u32 handle;
+	__u32 pad;
+	__u64 offset;
+};
+
+#if defined(__cplusplus)
+}
+#endif
+
+#endif /* _PANFROST_DRM_H_ */
diff --git a/include/uapi/linux/kfd_ioctl.h b/include/uapi/linux/kfd_ioctl.h
index e622fd1fbd46..dc067ed0b72d 100644
--- a/include/uapi/linux/kfd_ioctl.h
+++ b/include/uapi/linux/kfd_ioctl.h
@@ -211,6 +211,11 @@ struct kfd_ioctl_dbg_wave_control_args {
 #define KFD_HW_EXCEPTION_GPU_HANG	0
 #define KFD_HW_EXCEPTION_ECC		1
 
+/* For kfd_hsa_memory_exception_data.ErrorType */
+#define KFD_MEM_ERR_NO_RAS		0
+#define KFD_MEM_ERR_SRAM_ECC		1
+#define KFD_MEM_ERR_POISON_CONSUMED	2
+#define KFD_MEM_ERR_GPU_HANG		3
 
 struct kfd_ioctl_create_event_args {
 	__u64 event_page_offset;	/* from KFD */
@@ -250,7 +255,12 @@ struct kfd_hsa_memory_exception_data {
 	struct kfd_memory_exception_failure failure;
 	__u64 va;
 	__u32 gpu_id;
-	__u32 pad;
+	__u32 ErrorType; /* 0 = no RAS error,
+			  * 1 = ECC_SRAM,
+			  * 2 = Link_SYNFLOOD (poison),
+			  * 3 = GPU hang (not attributable to a specific cause),
+			  * other values reserved
+			  */
 };
 
 /* hw exception data */
diff --git a/include/uapi/linux/lirc.h b/include/uapi/linux/lirc.h
index 45fcbf99d72e..f99d9dcae667 100644
--- a/include/uapi/linux/lirc.h
+++ b/include/uapi/linux/lirc.h
@@ -195,6 +195,7 @@ struct lirc_scancode {
  * @RC_PROTO_RCMM12: RC-MM protocol 12 bits
  * @RC_PROTO_RCMM24: RC-MM protocol 24 bits
  * @RC_PROTO_RCMM32: RC-MM protocol 32 bits
+ * @RC_PROTO_XBOX_DVD: Xbox DVD Movie Playback Kit protocol
  */
 enum rc_proto {
 	RC_PROTO_UNKNOWN	= 0,
@@ -224,6 +225,7 @@ enum rc_proto {
 	RC_PROTO_RCMM12		= 24,
 	RC_PROTO_RCMM24		= 25,
 	RC_PROTO_RCMM32		= 26,
+	RC_PROTO_XBOX_DVD	= 27,
 };
 
 #endif
diff --git a/include/uapi/linux/media-bus-format.h b/include/uapi/linux/media-bus-format.h
index d6a5a3bfe6c4..2a6b253cfb05 100644
--- a/include/uapi/linux/media-bus-format.h
+++ b/include/uapi/linux/media-bus-format.h
@@ -34,7 +34,7 @@
 
 #define MEDIA_BUS_FMT_FIXED			0x0001
 
-/* RGB - next is	0x101b */
+/* RGB - next is	0x101c */
 #define MEDIA_BUS_FMT_RGB444_1X12		0x1016
 #define MEDIA_BUS_FMT_RGB444_2X8_PADHI_BE	0x1001
 #define MEDIA_BUS_FMT_RGB444_2X8_PADHI_LE	0x1002
@@ -50,6 +50,7 @@
 #define MEDIA_BUS_FMT_RGB666_1X24_CPADHI	0x1015
 #define MEDIA_BUS_FMT_RGB666_1X7X3_SPWG		0x1010
 #define MEDIA_BUS_FMT_BGR888_1X24		0x1013
+#define MEDIA_BUS_FMT_BGR888_3X8		0x101b
 #define MEDIA_BUS_FMT_GBR888_1X24		0x1014
 #define MEDIA_BUS_FMT_RGB888_1X24		0x100a
 #define MEDIA_BUS_FMT_RGB888_2X12_BE		0x100b
diff --git a/include/uapi/linux/media.h b/include/uapi/linux/media.h
index e5d0c5c611b5..9aedb187bc48 100644
--- a/include/uapi/linux/media.h
+++ b/include/uapi/linux/media.h
@@ -262,6 +262,11 @@ struct media_links_enum {
 #define MEDIA_INTF_T_V4L_SWRADIO		(MEDIA_INTF_T_V4L_BASE + 4)
 #define MEDIA_INTF_T_V4L_TOUCH			(MEDIA_INTF_T_V4L_BASE + 5)
 
+#define MEDIA_INTF_T_ALSA_BASE			0x00000300
+#define MEDIA_INTF_T_ALSA_PCM_CAPTURE		(MEDIA_INTF_T_ALSA_BASE)
+#define MEDIA_INTF_T_ALSA_PCM_PLAYBACK		(MEDIA_INTF_T_ALSA_BASE + 1)
+#define MEDIA_INTF_T_ALSA_CONTROL		(MEDIA_INTF_T_ALSA_BASE + 2)
+
 #if defined(__KERNEL__)
 
 /*
@@ -413,19 +418,19 @@ struct media_v2_topology {
 #define MEDIA_ENT_F_DTV_DECODER			MEDIA_ENT_F_DV_DECODER
 
 /*
- * There is still no ALSA support in the media controller. These
+ * There is still no full ALSA support in the media controller. These
  * defines should not have been added and we leave them here only
  * in case some application tries to use these defines.
+ *
+ * The ALSA defines that are in use have been moved into __KERNEL__
+ * scope. As support gets added to these interface types, they should
+ * be moved into __KERNEL__ scope with the code that uses them.
  */
-#define MEDIA_INTF_T_ALSA_BASE			0x00000300
-#define MEDIA_INTF_T_ALSA_PCM_CAPTURE		(MEDIA_INTF_T_ALSA_BASE)
-#define MEDIA_INTF_T_ALSA_PCM_PLAYBACK		(MEDIA_INTF_T_ALSA_BASE + 1)
-#define MEDIA_INTF_T_ALSA_CONTROL		(MEDIA_INTF_T_ALSA_BASE + 2)
-#define MEDIA_INTF_T_ALSA_COMPRESS		(MEDIA_INTF_T_ALSA_BASE + 3)
-#define MEDIA_INTF_T_ALSA_RAWMIDI		(MEDIA_INTF_T_ALSA_BASE + 4)
-#define MEDIA_INTF_T_ALSA_HWDEP			(MEDIA_INTF_T_ALSA_BASE + 5)
-#define MEDIA_INTF_T_ALSA_SEQUENCER		(MEDIA_INTF_T_ALSA_BASE + 6)
-#define MEDIA_INTF_T_ALSA_TIMER			(MEDIA_INTF_T_ALSA_BASE + 7)
+#define MEDIA_INTF_T_ALSA_COMPRESS             (MEDIA_INTF_T_ALSA_BASE + 3)
+#define MEDIA_INTF_T_ALSA_RAWMIDI              (MEDIA_INTF_T_ALSA_BASE + 4)
+#define MEDIA_INTF_T_ALSA_HWDEP                (MEDIA_INTF_T_ALSA_BASE + 5)
+#define MEDIA_INTF_T_ALSA_SEQUENCER            (MEDIA_INTF_T_ALSA_BASE + 6)
+#define MEDIA_INTF_T_ALSA_TIMER                (MEDIA_INTF_T_ALSA_BASE + 7)
 
 /* Obsolete symbol for media_version, no longer used in the kernel */
 #define MEDIA_API_VERSION			((0 << 16) | (1 << 8) | 0)
diff --git a/include/uapi/linux/nfs_mount.h b/include/uapi/linux/nfs_mount.h
index e44e00616ab5..e3bcfc6aa3b0 100644
--- a/include/uapi/linux/nfs_mount.h
+++ b/include/uapi/linux/nfs_mount.h
@@ -66,13 +66,4 @@ struct nfs_mount_data {
 #define NFS_MOUNT_UNSHARED	0x8000	/* 5 */
 #define NFS_MOUNT_FLAGMASK	0xFFFF
 
-/* The following are for internal use only */
-#define NFS_MOUNT_LOOKUP_CACHE_NONEG	0x10000
-#define NFS_MOUNT_LOOKUP_CACHE_NONE	0x20000
-#define NFS_MOUNT_NORESVPORT		0x40000
-#define NFS_MOUNT_LEGACY_INTERFACE	0x80000
-
-#define NFS_MOUNT_LOCAL_FLOCK	0x100000
-#define NFS_MOUNT_LOCAL_FCNTL	0x200000
-
 #endif
diff --git a/include/uapi/linux/serial_core.h b/include/uapi/linux/serial_core.h
index 6009ee2c2e99..67c4aaaa2308 100644
--- a/include/uapi/linux/serial_core.h
+++ b/include/uapi/linux/serial_core.h
@@ -287,4 +287,10 @@
 /* RDA UART */
 #define PORT_RDA	118
 
+/* Socionext Milbeaut UART */
+#define PORT_MLB_USIO	119
+
+/* SiFive UART */
+#define PORT_SIFIVE_V0	120
+
 #endif /* _UAPILINUX_SERIAL_CORE_H */
diff --git a/include/uapi/linux/v4l2-controls.h b/include/uapi/linux/v4l2-controls.h
index 06479f2fb3ae..37807f23231e 100644
--- a/include/uapi/linux/v4l2-controls.h
+++ b/include/uapi/linux/v4l2-controls.h
@@ -404,6 +404,10 @@ enum v4l2_mpeg_video_multi_slice_mode {
 #define V4L2_CID_MPEG_VIDEO_MV_V_SEARCH_RANGE		(V4L2_CID_MPEG_BASE+228)
 #define V4L2_CID_MPEG_VIDEO_FORCE_KEY_FRAME		(V4L2_CID_MPEG_BASE+229)
 
+/* CIDs for the FWHT codec as used by the vicodec driver. */
+#define V4L2_CID_FWHT_I_FRAME_QP             (V4L2_CID_MPEG_BASE + 290)
+#define V4L2_CID_FWHT_P_FRAME_QP             (V4L2_CID_MPEG_BASE + 291)
+
 #define V4L2_CID_MPEG_VIDEO_H263_I_FRAME_QP		(V4L2_CID_MPEG_BASE+300)
 #define V4L2_CID_MPEG_VIDEO_H263_P_FRAME_QP		(V4L2_CID_MPEG_BASE+301)
 #define V4L2_CID_MPEG_VIDEO_H263_B_FRAME_QP		(V4L2_CID_MPEG_BASE+302)
@@ -535,6 +539,10 @@ enum v4l2_mpeg_video_h264_hierarchical_coding_type {
 #define V4L2_CID_MPEG_VIDEO_H264_HIERARCHICAL_CODING_LAYER_QP	(V4L2_CID_MPEG_BASE+382)
 #define V4L2_CID_MPEG_VIDEO_H264_CONSTRAINED_INTRA_PREDICTION	(V4L2_CID_MPEG_BASE+383)
 #define V4L2_CID_MPEG_VIDEO_H264_CHROMA_QP_INDEX_OFFSET		(V4L2_CID_MPEG_BASE+384)
+#define V4L2_CID_MPEG_VIDEO_H264_I_FRAME_MIN_QP	(V4L2_CID_MPEG_BASE+385)
+#define V4L2_CID_MPEG_VIDEO_H264_I_FRAME_MAX_QP	(V4L2_CID_MPEG_BASE+386)
+#define V4L2_CID_MPEG_VIDEO_H264_P_FRAME_MIN_QP	(V4L2_CID_MPEG_BASE+387)
+#define V4L2_CID_MPEG_VIDEO_H264_P_FRAME_MAX_QP	(V4L2_CID_MPEG_BASE+388)
 #define V4L2_CID_MPEG_VIDEO_MPEG4_I_FRAME_QP	(V4L2_CID_MPEG_BASE+400)
 #define V4L2_CID_MPEG_VIDEO_MPEG4_P_FRAME_QP	(V4L2_CID_MPEG_BASE+401)
 #define V4L2_CID_MPEG_VIDEO_MPEG4_B_FRAME_QP	(V4L2_CID_MPEG_BASE+402)
diff --git a/include/uapi/linux/videodev2.h b/include/uapi/linux/videodev2.h
index 1db220da3bcc..1050a75fb7ef 100644
--- a/include/uapi/linux/videodev2.h
+++ b/include/uapi/linux/videodev2.h
@@ -514,9 +514,21 @@ struct v4l2_pix_format {
 #define V4L2_PIX_FMT_RGB444  v4l2_fourcc('R', '4', '4', '4') /* 16  xxxxrrrr ggggbbbb */
 #define V4L2_PIX_FMT_ARGB444 v4l2_fourcc('A', 'R', '1', '2') /* 16  aaaarrrr ggggbbbb */
 #define V4L2_PIX_FMT_XRGB444 v4l2_fourcc('X', 'R', '1', '2') /* 16  xxxxrrrr ggggbbbb */
+#define V4L2_PIX_FMT_RGBA444 v4l2_fourcc('R', 'A', '1', '2') /* 16  rrrrgggg bbbbaaaa */
+#define V4L2_PIX_FMT_RGBX444 v4l2_fourcc('R', 'X', '1', '2') /* 16  rrrrgggg bbbbxxxx */
+#define V4L2_PIX_FMT_ABGR444 v4l2_fourcc('A', 'B', '1', '2') /* 16  aaaabbbb ggggrrrr */
+#define V4L2_PIX_FMT_XBGR444 v4l2_fourcc('X', 'B', '1', '2') /* 16  xxxxbbbb ggggrrrr */
+#define V4L2_PIX_FMT_BGRA444 v4l2_fourcc('B', 'A', '1', '2') /* 16  bbbbgggg rrrraaaa */
+#define V4L2_PIX_FMT_BGRX444 v4l2_fourcc('B', 'X', '1', '2') /* 16  bbbbgggg rrrrxxxx */
 #define V4L2_PIX_FMT_RGB555  v4l2_fourcc('R', 'G', 'B', 'O') /* 16  RGB-5-5-5     */
 #define V4L2_PIX_FMT_ARGB555 v4l2_fourcc('A', 'R', '1', '5') /* 16  ARGB-1-5-5-5  */
 #define V4L2_PIX_FMT_XRGB555 v4l2_fourcc('X', 'R', '1', '5') /* 16  XRGB-1-5-5-5  */
+#define V4L2_PIX_FMT_RGBA555 v4l2_fourcc('R', 'A', '1', '5') /* 16  RGBA-5-5-5-1  */
+#define V4L2_PIX_FMT_RGBX555 v4l2_fourcc('R', 'X', '1', '5') /* 16  RGBX-5-5-5-1  */
+#define V4L2_PIX_FMT_ABGR555 v4l2_fourcc('A', 'B', '1', '5') /* 16  ABGR-1-5-5-5  */
+#define V4L2_PIX_FMT_XBGR555 v4l2_fourcc('X', 'B', '1', '5') /* 16  XBGR-1-5-5-5  */
+#define V4L2_PIX_FMT_BGRA555 v4l2_fourcc('B', 'A', '1', '5') /* 16  BGRA-5-5-5-1  */
+#define V4L2_PIX_FMT_BGRX555 v4l2_fourcc('B', 'X', '1', '5') /* 16  BGRX-5-5-5-1  */
 #define V4L2_PIX_FMT_RGB565  v4l2_fourcc('R', 'G', 'B', 'P') /* 16  RGB-5-6-5     */
 #define V4L2_PIX_FMT_RGB555X v4l2_fourcc('R', 'G', 'B', 'Q') /* 16  RGB-5-5-5 BE  */
 #define V4L2_PIX_FMT_ARGB555X v4l2_fourcc_be('A', 'R', '1', '5') /* 16  ARGB-5-5-5 BE */
@@ -528,7 +540,11 @@ struct v4l2_pix_format {
 #define V4L2_PIX_FMT_BGR32   v4l2_fourcc('B', 'G', 'R', '4') /* 32  BGR-8-8-8-8   */
 #define V4L2_PIX_FMT_ABGR32  v4l2_fourcc('A', 'R', '2', '4') /* 32  BGRA-8-8-8-8  */
 #define V4L2_PIX_FMT_XBGR32  v4l2_fourcc('X', 'R', '2', '4') /* 32  BGRX-8-8-8-8  */
+#define V4L2_PIX_FMT_BGRA32  v4l2_fourcc('R', 'A', '2', '4') /* 32  ABGR-8-8-8-8  */
+#define V4L2_PIX_FMT_BGRX32  v4l2_fourcc('R', 'X', '2', '4') /* 32  XBGR-8-8-8-8  */
 #define V4L2_PIX_FMT_RGB32   v4l2_fourcc('R', 'G', 'B', '4') /* 32  RGB-8-8-8-8   */
+#define V4L2_PIX_FMT_RGBA32  v4l2_fourcc('A', 'B', '2', '4') /* 32  RGBA-8-8-8-8  */
+#define V4L2_PIX_FMT_RGBX32  v4l2_fourcc('X', 'B', '2', '4') /* 32  RGBX-8-8-8-8  */
 #define V4L2_PIX_FMT_ARGB32  v4l2_fourcc('B', 'A', '2', '4') /* 32  ARGB-8-8-8-8  */
 #define V4L2_PIX_FMT_XRGB32  v4l2_fourcc('B', 'X', '2', '4') /* 32  XRGB-8-8-8-8  */
 
@@ -669,6 +685,7 @@ struct v4l2_pix_format {
 #define V4L2_PIX_FMT_VP9      v4l2_fourcc('V', 'P', '9', '0') /* VP9 */
 #define V4L2_PIX_FMT_HEVC     v4l2_fourcc('H', 'E', 'V', 'C') /* HEVC aka H.265 */
 #define V4L2_PIX_FMT_FWHT     v4l2_fourcc('F', 'W', 'H', 'T') /* Fast Walsh Hadamard Transform (vicodec) */
+#define V4L2_PIX_FMT_FWHT_STATELESS     v4l2_fourcc('S', 'F', 'W', 'H') /* Stateless FWHT (vicodec) */
 
 /*  Vendor-specific formats   */
 #define V4L2_PIX_FMT_CPIA1    v4l2_fourcc('C', 'P', 'I', 'A') /* cpia1 YUV */
diff --git a/include/uapi/linux/virtio_gpu.h b/include/uapi/linux/virtio_gpu.h
index 8e88eba1fa7a..0c85914d9369 100644
--- a/include/uapi/linux/virtio_gpu.h
+++ b/include/uapi/linux/virtio_gpu.h
@@ -40,8 +40,16 @@
 
 #include <linux/types.h>
 
-#define VIRTIO_GPU_F_VIRGL 0
-#define VIRTIO_GPU_F_EDID  1
+/*
+ * VIRTIO_GPU_CMD_CTX_*
+ * VIRTIO_GPU_CMD_*_3D
+ */
+#define VIRTIO_GPU_F_VIRGL               0
+
+/*
+ * VIRTIO_GPU_CMD_GET_EDID
+ */
+#define VIRTIO_GPU_F_EDID                1
 
 enum virtio_gpu_ctrl_type {
 	VIRTIO_GPU_UNDEFINED = 0,
diff --git a/include/uapi/rdma/efa-abi.h b/include/uapi/rdma/efa-abi.h
new file mode 100644
index 000000000000..9599a2a62be8
--- /dev/null
+++ b/include/uapi/rdma/efa-abi.h
@@ -0,0 +1,101 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-2-Clause) */
+/*
+ * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
+ */
+
+#ifndef EFA_ABI_USER_H
+#define EFA_ABI_USER_H
+
+#include <linux/types.h>
+
+/*
+ * Increment this value if any changes that break userspace ABI
+ * compatibility are made.
+ */
+#define EFA_UVERBS_ABI_VERSION 1
+
+/*
+ * Keep structs aligned to 8 bytes.
+ * Keep reserved fields as arrays of __u8 named reserved_XXX where XXX is the
+ * hex bit offset of the field.
+ */
+
+enum efa_ibv_user_cmds_supp_udata {
+	EFA_USER_CMDS_SUPP_UDATA_QUERY_DEVICE = 1 << 0,
+	EFA_USER_CMDS_SUPP_UDATA_CREATE_AH    = 1 << 1,
+};
+
+struct efa_ibv_alloc_ucontext_resp {
+	__u32 comp_mask;
+	__u32 cmds_supp_udata_mask;
+	__u16 sub_cqs_per_cq;
+	__u16 inline_buf_size;
+	__u32 max_llq_size; /* bytes */
+};
+
+struct efa_ibv_alloc_pd_resp {
+	__u32 comp_mask;
+	__u16 pdn;
+	__u8 reserved_30[2];
+};
+
+struct efa_ibv_create_cq {
+	__u32 comp_mask;
+	__u32 cq_entry_size;
+	__u16 num_sub_cqs;
+	__u8 reserved_50[6];
+};
+
+struct efa_ibv_create_cq_resp {
+	__u32 comp_mask;
+	__u8 reserved_20[4];
+	__aligned_u64 q_mmap_key;
+	__aligned_u64 q_mmap_size;
+	__u16 cq_idx;
+	__u8 reserved_d0[6];
+};
+
+enum {
+	EFA_QP_DRIVER_TYPE_SRD = 0,
+};
+
+struct efa_ibv_create_qp {
+	__u32 comp_mask;
+	__u32 rq_ring_size; /* bytes */
+	__u32 sq_ring_size; /* bytes */
+	__u32 driver_qp_type;
+};
+
+struct efa_ibv_create_qp_resp {
+	__u32 comp_mask;
+	/* the offset inside the page of the rq db */
+	__u32 rq_db_offset;
+	/* the offset inside the page of the sq db */
+	__u32 sq_db_offset;
+	/* the offset inside the page of descriptors buffer */
+	__u32 llq_desc_offset;
+	__aligned_u64 rq_mmap_key;
+	__aligned_u64 rq_mmap_size;
+	__aligned_u64 rq_db_mmap_key;
+	__aligned_u64 sq_db_mmap_key;
+	__aligned_u64 llq_desc_mmap_key;
+	__u16 send_sub_cq_idx;
+	__u16 recv_sub_cq_idx;
+	__u8 reserved_1e0[4];
+};
+
+struct efa_ibv_create_ah_resp {
+	__u32 comp_mask;
+	__u16 efa_address_handle;
+	__u8 reserved_30[2];
+};
+
+struct efa_ibv_ex_query_device_resp {
+	__u32 comp_mask;
+	__u32 max_sq_wr;
+	__u32 max_rq_wr;
+	__u16 max_sq_sge;
+	__u16 max_rq_sge;
+};
+
+#endif /* EFA_ABI_USER_H */
diff --git a/include/uapi/rdma/mlx5-abi.h b/include/uapi/rdma/mlx5-abi.h
index f4d4010b7e3e..624f5b53eb1f 100644
--- a/include/uapi/rdma/mlx5-abi.h
+++ b/include/uapi/rdma/mlx5-abi.h
@@ -360,6 +360,7 @@ enum mlx5_ib_create_qp_resp_mask {
 	MLX5_IB_CREATE_QP_RESP_MASK_TISN = 1UL << 1,
 	MLX5_IB_CREATE_QP_RESP_MASK_RQN  = 1UL << 2,
 	MLX5_IB_CREATE_QP_RESP_MASK_SQN  = 1UL << 3,
+	MLX5_IB_CREATE_QP_RESP_MASK_TIR_ICM_ADDR  = 1UL << 4,
 };
 
 struct mlx5_ib_create_qp_resp {
@@ -371,6 +372,7 @@ struct mlx5_ib_create_qp_resp {
 	__u32	rqn;
 	__u32	sqn;
 	__u32   reserved1;
+	__u64	tir_icm_addr;
 };
 
 struct mlx5_ib_alloc_mw {
diff --git a/include/uapi/rdma/mlx5_user_ioctl_cmds.h b/include/uapi/rdma/mlx5_user_ioctl_cmds.h
index 8149d224030b..d404c951954c 100644
--- a/include/uapi/rdma/mlx5_user_ioctl_cmds.h
+++ b/include/uapi/rdma/mlx5_user_ioctl_cmds.h
@@ -44,6 +44,7 @@ enum mlx5_ib_create_flow_action_attrs {
 enum mlx5_ib_alloc_dm_attrs {
 	MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET = (1U << UVERBS_ID_NS_SHIFT),
 	MLX5_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX,
+	MLX5_IB_ATTR_ALLOC_DM_REQ_TYPE,
 };
 
 enum mlx5_ib_devx_methods {
@@ -144,6 +145,7 @@ enum mlx5_ib_flow_matcher_create_attrs {
 	MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE,
 	MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA,
 	MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS,
+	MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE,
 };
 
 enum mlx5_ib_flow_matcher_destroy_attrs {
diff --git a/include/uapi/rdma/mlx5_user_ioctl_verbs.h b/include/uapi/rdma/mlx5_user_ioctl_verbs.h
index 4a701033b93f..a8f34c237458 100644
--- a/include/uapi/rdma/mlx5_user_ioctl_verbs.h
+++ b/include/uapi/rdma/mlx5_user_ioctl_verbs.h
@@ -42,6 +42,7 @@ enum mlx5_ib_uapi_flow_action_flags {
 enum mlx5_ib_uapi_flow_table_type {
 	MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX     = 0x0,
 	MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX	= 0x1,
+	MLX5_IB_UAPI_FLOW_TABLE_TYPE_FDB	= 0x2,
 };
 
 enum mlx5_ib_uapi_flow_action_packet_reformat_type {
@@ -56,5 +57,11 @@ struct mlx5_ib_uapi_devx_async_cmd_hdr {
 	__u8		out_data[];
 };
 
+enum mlx5_ib_uapi_dm_type {
+	MLX5_IB_UAPI_DM_TYPE_MEMIC,
+	MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM,
+	MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM,
+};
+
 #endif
 
diff --git a/include/uapi/rdma/rdma_netlink.h b/include/uapi/rdma/rdma_netlink.h
index 5cc592728071..42a8bdc40a14 100644
--- a/include/uapi/rdma/rdma_netlink.h
+++ b/include/uapi/rdma/rdma_netlink.h
@@ -49,17 +49,6 @@ enum {
 	RDMA_NL_IWPM_NUM_OPS
 };
 
-struct rdma_cm_id_stats {
-	__u32	qp_num;
-	__u32	bound_dev_if;
-	__u32	port_space;
-	__s32	pid;
-	__u8	cm_state;
-	__u8	node_type;
-	__u8	port_num;
-	__u8	qp_type;
-};
-
 enum {
 	IWPM_NLA_REG_PID_UNSPEC = 0,
 	IWPM_NLA_REG_PID_SEQ,
@@ -261,7 +250,10 @@ enum rdma_nldev_command {
 
 	RDMA_NLDEV_CMD_PORT_GET, /* can dump */
 
-	/* 6 - 8 are free to use */
+	RDMA_NLDEV_CMD_SYS_GET, /* can dump */
+	RDMA_NLDEV_CMD_SYS_SET,
+
+	/* 8 is free to use */
 
 	RDMA_NLDEV_CMD_RES_GET = 9, /* can dump */
 
@@ -473,6 +465,21 @@ enum rdma_nldev_attr {
 	RDMA_NLDEV_ATTR_LINK_TYPE,		/* string */
 
 	/*
+	 * net namespace mode for rdma subsystem:
+	 * either shared or exclusive among multiple net namespaces.
+	 */
+	RDMA_NLDEV_SYS_ATTR_NETNS_MODE,		/* u8 */
+	/*
+	 * Device protocol, e.g. ib, iw, usnic, roce and opa
+	 */
+	RDMA_NLDEV_ATTR_DEV_PROTOCOL,		/* string */
+
+	/*
+	 * File descriptor handle of the net namespace object
+	 */
+	RDMA_NLDEV_NET_NS_FD,			/* u32 */
+
+	/*
 	 * Always the end
 	 */
 	RDMA_NLDEV_ATTR_MAX
diff --git a/include/uapi/rdma/rdma_user_ioctl_cmds.h b/include/uapi/rdma/rdma_user_ioctl_cmds.h
index 06c34d99be85..26213f49f5c8 100644
--- a/include/uapi/rdma/rdma_user_ioctl_cmds.h
+++ b/include/uapi/rdma/rdma_user_ioctl_cmds.h
@@ -102,6 +102,7 @@ enum rdma_driver_id {
 	RDMA_DRIVER_RXE,
 	RDMA_DRIVER_HFI1,
 	RDMA_DRIVER_QIB,
+	RDMA_DRIVER_EFA,
 };
 
 #endif
diff --git a/include/uapi/scsi/fc/fc_els.h b/include/uapi/scsi/fc/fc_els.h
index b7e0a5ed40de..a81c53508cc6 100644
--- a/include/uapi/scsi/fc/fc_els.h
+++ b/include/uapi/scsi/fc/fc_els.h
@@ -52,6 +52,7 @@ enum fc_els_cmd {
 	ELS_RRQ =	0x12,	/* reinstate recovery qualifier */
 	ELS_REC =	0x13,	/* read exchange concise */
 	ELS_SRR =	0x14,	/* sequence retransmission request */
+	ELS_FPIN =	0x16,	/* Fabric Performance Impact Notification */
 	ELS_PRLI =	0x20,	/* process login */
 	ELS_PRLO =	0x21,	/* process logout */
 	ELS_SCN =	0x22,	/* state change notification */
@@ -119,6 +120,7 @@ enum fc_els_cmd {
 	[ELS_RRQ] =	"RRQ",			\
 	[ELS_REC] =	"REC",			\
 	[ELS_SRR] =	"SRR",			\
+	[ELS_FPIN] =	"FPIN",			\
 	[ELS_PRLI] =	"PRLI",			\
 	[ELS_PRLO] =	"PRLO",			\
 	[ELS_SCN] =	"SCN",			\
@@ -829,4 +831,35 @@ enum fc_els_clid_ic {
 	ELS_CLID_IC_LIP =	8,	/* receiving LIP */
 };
 
+
+/*
+ * Fabric Notification Descriptor Tag values
+ */
+enum fc_fn_dtag {
+	ELS_FN_DTAG_LNK_INTEGRITY =	0x00020001,	/* Link Integrity */
+	ELS_FN_DTAG_PEER_CONGEST =	0x00020003,	/* Peer Congestion */
+	ELS_FN_DTAG_CONGESTION =	0x00020004,	/* Congestion */
+};
+
+/*
+ * Fabric Notification Descriptor
+ */
+struct fc_fn_desc {
+	__be32		fn_desc_tag;	/* Notification Descriptor Tag */
+	__be32		fn_desc_value_len; /* Length of Descriptor Value field
+					    * (in bytes)
+					    */
+	__u8		fn_desc_value[0];  /* Descriptor Value */
+};
+
+/*
+ * ELS_FPIN - Fabric Performance Impact Notification
+ */
+struct fc_els_fpin {
+	__u8		fpin_cmd;	/* command (0x16) */
+	__u8		fpin_zero[3];	/* specified as zero - part of cmd */
+	__be32		fpin_desc_cnt;	/* count of descriptors */
+	struct fc_fn_desc	fpin_desc[0];	/* Descriptor list */
+};
+
 #endif /* _FC_ELS_H_ */
diff --git a/include/uapi/sound/sof/abi.h b/include/uapi/sound/sof/abi.h
new file mode 100644
index 000000000000..37e0a90dc9e6
--- /dev/null
+++ b/include/uapi/sound/sof/abi.h
@@ -0,0 +1,62 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */
+/*
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * Copyright(c) 2018 Intel Corporation. All rights reserved.
+ */
+
+/**
+ * SOF ABI versioning is based on Semantic Versioning where we have a given
+ * MAJOR.MINOR.PATCH version number. See https://semver.org/
+ *
+ * Rules for incrementing or changing version :-
+ *
+ * 1) Increment MAJOR version if you make incompatible API changes. MINOR and
+ *    PATCH should be reset to 0.
+ *
+ * 2) Increment MINOR version if you add backwards compatible features or
+ *    changes. PATCH should be reset to 0.
+ *
+ * 3) Increment PATCH version if you add backwards compatible bug fixes.
+ */
+
+#ifndef __INCLUDE_UAPI_SOUND_SOF_ABI_H__
+#define __INCLUDE_UAPI_SOUND_SOF_ABI_H__
+
+/* SOF ABI version major, minor and patch numbers */
+#define SOF_ABI_MAJOR 3
+#define SOF_ABI_MINOR 4
+#define SOF_ABI_PATCH 0
+
+/* SOF ABI version number. Format within 32bit word is MMmmmppp */
+#define SOF_ABI_MAJOR_SHIFT	24
+#define SOF_ABI_MAJOR_MASK	0xff
+#define SOF_ABI_MINOR_SHIFT	12
+#define SOF_ABI_MINOR_MASK	0xfff
+#define SOF_ABI_PATCH_SHIFT	0
+#define SOF_ABI_PATCH_MASK	0xfff
+
+#define SOF_ABI_VER(major, minor, patch) \
+	(((major) << SOF_ABI_MAJOR_SHIFT) | \
+	((minor) << SOF_ABI_MINOR_SHIFT) | \
+	((patch) << SOF_ABI_PATCH_SHIFT))
+
+#define SOF_ABI_VERSION_MAJOR(version) \
+	(((version) >> SOF_ABI_MAJOR_SHIFT) & SOF_ABI_MAJOR_MASK)
+#define SOF_ABI_VERSION_MINOR(version)	\
+	(((version) >> SOF_ABI_MINOR_SHIFT) & SOF_ABI_MINOR_MASK)
+#define SOF_ABI_VERSION_PATCH(version)	\
+	(((version) >> SOF_ABI_PATCH_SHIFT) & SOF_ABI_PATCH_MASK)
+
+#define SOF_ABI_VERSION_INCOMPATIBLE(sof_ver, client_ver)		\
+	(SOF_ABI_VERSION_MAJOR((sof_ver)) !=				\
+		SOF_ABI_VERSION_MAJOR((client_ver))			\
+	)
+
+#define SOF_ABI_VERSION SOF_ABI_VER(SOF_ABI_MAJOR, SOF_ABI_MINOR, SOF_ABI_PATCH)
+
+/* SOF ABI magic number "SOF\0". */
+#define SOF_ABI_MAGIC		0x00464F53
+
+#endif
diff --git a/include/uapi/sound/sof/eq.h b/include/uapi/sound/sof/eq.h
new file mode 100644
index 000000000000..666c2b6a3229
--- /dev/null
+++ b/include/uapi/sound/sof/eq.h
@@ -0,0 +1,172 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */
+/*
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * Copyright(c) 2018 Intel Corporation. All rights reserved.
+ */
+
+#ifndef __INCLUDE_UAPI_SOUND_SOF_USER_EQ_H__
+#define __INCLUDE_UAPI_SOUND_SOF_USER_EQ_H__
+
+/* FIR EQ type */
+
+#define SOF_EQ_FIR_IDX_SWITCH	0
+
+#define SOF_EQ_FIR_MAX_SIZE 4096 /* Max size allowed for coef data in bytes */
+
+#define SOF_EQ_FIR_MAX_LENGTH 192 /* Max length for individual filter */
+
+#define SOF_EQ_FIR_MAX_RESPONSES 8 /* A blob can define max 8 FIR EQs */
+
+/*
+ * eq_fir_configuration data structure contains this information
+ *     uint32_t size
+ *	   This is the number of bytes need to store the received EQ
+ *	   configuration.
+ *     uint16_t channels_in_config
+ *         This describes the number of channels in this EQ config data. It
+ *         can be different from PLATFORM_MAX_CHANNELS.
+ *     uint16_t number_of_responses
+ *         0=no responses, 1=one response defined, 2=two responses defined, etc.
+ *     int16_t data[]
+ *         assign_response[channels_in_config]
+ *             0 = use first response, 1 = use 2nd response, etc.
+ *             E.g. {0, 0, 0, 0, 1, 1, 1, 1} would apply to channels 0-3 the
+ *	       same first defined response and for to channels 4-7 the second.
+ *         coef_data[]
+ *             Repeated data
+ *             { filter_length, output_shift, h[] }
+ *	       for every EQ response defined where vector h has filter_length
+ *             number of coefficients. Coefficients in h[] are in Q1.15 format.
+ *             E.g. 16384 (Q1.15) = 0.5. The shifts are number of right shifts.
+ *
+ * NOTE: The channels_in_config must be even to have coef_data aligned to
+ * 32 bit word in RAM. Therefore a mono EQ assign must be duplicated to 2ch
+ * even if it would never used. Similarly a 5ch EQ assign must be increased
+ * to 6ch. EQ init will return an error if this is not met.
+ *
+ * NOTE: The filter_length must be multiple of four. Therefore the filter must
+ * be padded from the end with zeros have this condition met.
+ */
+
+struct sof_eq_fir_config {
+	uint32_t size;
+	uint16_t channels_in_config;
+	uint16_t number_of_responses;
+
+	/* reserved */
+	uint32_t reserved[4];
+
+	int16_t data[];
+} __packed;
+
+struct sof_eq_fir_coef_data {
+	int16_t length; /* Number of FIR taps */
+	int16_t out_shift; /* Amount of right shifts at output */
+
+	/* reserved */
+	uint32_t reserved[4];
+
+	int16_t coef[]; /* FIR coefficients */
+} __packed;
+
+/* In the struct above there's two 16 bit words (length, shift) and four
+ * reserved 32 bit words before the actual FIR coefficients. This information
+ * is used in parsing of the configuration blob.
+ */
+#define SOF_EQ_FIR_COEF_NHEADER \
+	(sizeof(struct sof_eq_fir_coef_data) / sizeof(int16_t))
+
+/* IIR EQ type */
+
+#define SOF_EQ_IIR_IDX_SWITCH   0
+
+#define SOF_EQ_IIR_MAX_SIZE 1024 /* Max size allowed for coef data in bytes */
+
+#define SOF_EQ_IIR_MAX_RESPONSES 8 /* A blob can define max 8 IIR EQs */
+
+/* eq_iir_configuration
+ *     uint32_t channels_in_config
+ *         This describes the number of channels in this EQ config data. It
+ *         can be different from PLATFORM_MAX_CHANNELS.
+ *     uint32_t number_of_responses_defined
+ *         0=no responses, 1=one response defined, 2=two responses defined, etc.
+ *     int32_t data[]
+ *         Data consist of two parts. First is the response assign vector that
+ *	   has length of channels_in_config. The latter part is coefficient
+ *         data.
+ *         uint32_t assign_response[channels_in_config]
+ *             -1 = not defined, 0 = use first response, 1 = use 2nd, etc.
+ *             E.g. {0, 0, 0, 0, -1, -1, -1, -1} would apply to channels 0-3 the
+ *             same first defined response and leave channels 4-7 unequalized.
+ *         coefficient_data[]
+ *             <1st EQ>
+ *             uint32_t num_biquads
+ *             uint32_t num_biquads_in_series
+ *             <1st biquad>
+ *             int32_t coef_a2       Q2.30 format
+ *             int32_t coef_a1       Q2.30 format
+ *             int32_t coef_b2       Q2.30 format
+ *             int32_t coef_b1       Q2.30 format
+ *             int32_t coef_b0       Q2.30 format
+ *             int32_t output_shift  number of shifts right, shift left is negative
+ *             int32_t output_gain   Q2.14 format
+ *             <2nd biquad>
+ *             ...
+ *             <2nd EQ>
+ *
+ *         Note: A flat response biquad can be made with a section set to
+ *         b0 = 1.0, gain = 1.0, and other parameters set to 0
+ *         {0, 0, 0, 0, 1073741824, 0, 16484}
+ */
+
+struct sof_eq_iir_config {
+	uint32_t size;
+	uint32_t channels_in_config;
+	uint32_t number_of_responses;
+
+	/* reserved */
+	uint32_t reserved[4];
+
+	int32_t data[]; /* eq_assign[channels], eq 0, eq 1, ... */
+} __packed;
+
+struct sof_eq_iir_header_df2t {
+	uint32_t num_sections;
+	uint32_t num_sections_in_series;
+
+	/* reserved */
+	uint32_t reserved[4];
+
+	int32_t biquads[]; /* Repeated biquad coefficients */
+} __packed;
+
+struct sof_eq_iir_biquad_df2t {
+	int32_t a2; /* Q2.30 */
+	int32_t a1; /* Q2.30 */
+	int32_t b2; /* Q2.30 */
+	int32_t b1; /* Q2.30 */
+	int32_t b0; /* Q2.30 */
+	int32_t output_shift; /* Number of right shifts */
+	int32_t output_gain;  /* Q2.14 */
+} __packed;
+
+/* A full 22th order equalizer with 11 biquads cover octave bands 1-11 in
+ * in the 0 - 20 kHz bandwidth.
+ */
+#define SOF_EQ_IIR_DF2T_BIQUADS_MAX 11
+
+/* The number of int32_t words in sof_eq_iir_header_df2t:
+ *	num_sections, num_sections_in_series, reserved[4]
+ */
+#define SOF_EQ_IIR_NHEADER_DF2T \
+	(sizeof(struct sof_eq_iir_header_df2t) / sizeof(int32_t))
+
+/* The number of int32_t words in sof_eq_iir_biquad_df2t:
+ *	a2, a1, b2, b1, b0, output_shift, output_gain
+ */
+#define SOF_EQ_IIR_NBIQUAD_DF2T \
+	(sizeof(struct sof_eq_iir_biquad_df2t) / sizeof(int32_t))
+
+#endif
diff --git a/include/uapi/sound/sof/fw.h b/include/uapi/sound/sof/fw.h
new file mode 100644
index 000000000000..1afca973eb09
--- /dev/null
+++ b/include/uapi/sound/sof/fw.h
@@ -0,0 +1,78 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */
+/*
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * Copyright(c) 2018 Intel Corporation. All rights reserved.
+ */
+
+/*
+ * Firmware file format .
+ */
+
+#ifndef __INCLUDE_UAPI_SOF_FW_H__
+#define __INCLUDE_UAPI_SOF_FW_H__
+
+#define SND_SOF_FW_SIG_SIZE	4
+#define SND_SOF_FW_ABI		1
+#define SND_SOF_FW_SIG		"Reef"
+
+/*
+ * Firmware module is made up of 1 . N blocks of different types. The
+ * Block header is used to determine where and how block is to be copied in the
+ * DSP/host memory space.
+ */
+enum snd_sof_fw_blk_type {
+	SOF_FW_BLK_TYPE_INVALID	= -1,
+	SOF_FW_BLK_TYPE_START	= 0,
+	SOF_FW_BLK_TYPE_RSRVD0	= SOF_FW_BLK_TYPE_START,
+	SOF_FW_BLK_TYPE_IRAM	= 1,	/* local instruction RAM */
+	SOF_FW_BLK_TYPE_DRAM	= 2,	/* local data RAM */
+	SOF_FW_BLK_TYPE_SRAM	= 3,	/* system RAM */
+	SOF_FW_BLK_TYPE_ROM	= 4,
+	SOF_FW_BLK_TYPE_IMR	= 5,
+	SOF_FW_BLK_TYPE_RSRVD6	= 6,
+	SOF_FW_BLK_TYPE_RSRVD7	= 7,
+	SOF_FW_BLK_TYPE_RSRVD8	= 8,
+	SOF_FW_BLK_TYPE_RSRVD9	= 9,
+	SOF_FW_BLK_TYPE_RSRVD10	= 10,
+	SOF_FW_BLK_TYPE_RSRVD11	= 11,
+	SOF_FW_BLK_TYPE_RSRVD12	= 12,
+	SOF_FW_BLK_TYPE_RSRVD13	= 13,
+	SOF_FW_BLK_TYPE_RSRVD14	= 14,
+	/* use SOF_FW_BLK_TYPE_RSVRDX for new block types */
+	SOF_FW_BLK_TYPE_NUM
+};
+
+struct snd_sof_blk_hdr {
+	enum snd_sof_fw_blk_type type;
+	uint32_t size;		/* bytes minus this header */
+	uint32_t offset;	/* offset from base */
+} __packed;
+
+/*
+ * Firmware file is made up of 1 .. N different modules types. The module
+ * type is used to determine how to load and parse the module.
+ */
+enum snd_sof_fw_mod_type {
+	SOF_FW_BASE	= 0,	/* base firmware image */
+	SOF_FW_MODULE	= 1,	/* firmware module */
+};
+
+struct snd_sof_mod_hdr {
+	enum snd_sof_fw_mod_type type;
+	uint32_t size;		/* bytes minus this header */
+	uint32_t num_blocks;	/* number of blocks */
+} __packed;
+
+/*
+ * Firmware file header.
+ */
+struct snd_sof_fw_header {
+	unsigned char sig[SND_SOF_FW_SIG_SIZE]; /* "Reef" */
+	uint32_t file_size;	/* size of file minus this header */
+	uint32_t num_modules;	/* number of modules */
+	uint32_t abi;		/* version of header format */
+} __packed;
+
+#endif
diff --git a/include/uapi/sound/sof/header.h b/include/uapi/sound/sof/header.h
new file mode 100644
index 000000000000..7868990b0d6f
--- /dev/null
+++ b/include/uapi/sound/sof/header.h
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */
+/*
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * Copyright(c) 2018 Intel Corporation. All rights reserved.
+ */
+
+#ifndef __INCLUDE_UAPI_SOUND_SOF_USER_HEADER_H__
+#define __INCLUDE_UAPI_SOUND_SOF_USER_HEADER_H__
+
+/*
+ * Header for all non IPC ABI data.
+ *
+ * Identifies data type, size and ABI.
+ * Used by any bespoke component data structures or binary blobs.
+ */
+struct sof_abi_hdr {
+	uint32_t magic;		/**< 'S', 'O', 'F', '\0' */
+	uint32_t type;		/**< component specific type */
+	uint32_t size;		/**< size in bytes of data excl. this struct */
+	uint32_t abi;		/**< SOF ABI version */
+	uint32_t reserved[4];	/**< reserved for future use */
+	uint32_t data[0];	/**< Component data - opaque to core */
+}  __packed;
+
+#endif
diff --git a/include/uapi/sound/sof/manifest.h b/include/uapi/sound/sof/manifest.h
new file mode 100644
index 000000000000..2009ee30fad0
--- /dev/null
+++ b/include/uapi/sound/sof/manifest.h
@@ -0,0 +1,188 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */
+/*
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * Copyright(c) 2018 Intel Corporation. All rights reserved.
+ */
+
+#ifndef __INCLUDE_UAPI_SOUND_SOF_USER_MANIFEST_H__
+#define __INCLUDE_UAPI_SOUND_SOF_USER_MANIFEST_H__
+
+/* start offset for base FW module */
+#define SOF_MAN_ELF_TEXT_OFFSET		0x2000
+
+/* FW Extended Manifest Header id = $AE1 */
+#define SOF_MAN_EXT_HEADER_MAGIC	0x31454124
+
+/* module type load type */
+#define SOF_MAN_MOD_TYPE_BUILTIN	0
+#define SOF_MAN_MOD_TYPE_MODULE		1
+
+struct sof_man_module_type {
+	uint32_t load_type:4;	/* SOF_MAN_MOD_TYPE_ */
+	uint32_t auto_start:1;
+	uint32_t domain_ll:1;
+	uint32_t domain_dp:1;
+	uint32_t rsvd_:25;
+};
+
+/* segment flags.type */
+#define SOF_MAN_SEGMENT_TEXT		0
+#define SOF_MAN_SEGMENT_RODATA		1
+#define SOF_MAN_SEGMENT_DATA		1
+#define SOF_MAN_SEGMENT_BSS		2
+#define SOF_MAN_SEGMENT_EMPTY		15
+
+union sof_man_segment_flags {
+	uint32_t ul;
+	struct {
+		uint32_t contents:1;
+		uint32_t alloc:1;
+		uint32_t load:1;
+		uint32_t readonly:1;
+		uint32_t code:1;
+		uint32_t data:1;
+		uint32_t _rsvd0:2;
+		uint32_t type:4;	/* MAN_SEGMENT_ */
+		uint32_t _rsvd1:4;
+		uint32_t length:16;	/* of segment in pages */
+	} r;
+} __packed;
+
+/*
+ * Module segment descriptor. Used by ROM - Immutable.
+ */
+struct sof_man_segment_desc {
+	union sof_man_segment_flags flags;
+	uint32_t v_base_addr;
+	uint32_t file_offset;
+} __packed;
+
+/*
+ * The firmware binary can be split into several modules.
+ */
+
+#define SOF_MAN_MOD_ID_LEN		4
+#define SOF_MAN_MOD_NAME_LEN		8
+#define SOF_MAN_MOD_SHA256_LEN		32
+#define SOF_MAN_MOD_ID			{'$', 'A', 'M', 'E'}
+
+/*
+ * Each module has an entry in the FW header. Used by ROM - Immutable.
+ */
+struct sof_man_module {
+	uint8_t struct_id[SOF_MAN_MOD_ID_LEN];	/* SOF_MAN_MOD_ID */
+	uint8_t name[SOF_MAN_MOD_NAME_LEN];
+	uint8_t uuid[16];
+	struct sof_man_module_type type;
+	uint8_t hash[SOF_MAN_MOD_SHA256_LEN];
+	uint32_t entry_point;
+	uint16_t cfg_offset;
+	uint16_t cfg_count;
+	uint32_t affinity_mask;
+	uint16_t instance_max_count;	/* max number of instances */
+	uint16_t instance_bss_size;	/* instance (pages) */
+	struct sof_man_segment_desc segment[3];
+} __packed;
+
+/*
+ * Each module has a configuration in the FW header. Used by ROM - Immutable.
+ */
+struct sof_man_mod_config {
+	uint32_t par[4];	/* module parameters */
+	uint32_t is_pages;	/* actual size of instance .bss (pages) */
+	uint32_t cps;		/* cycles per second */
+	uint32_t ibs;		/* input buffer size (bytes) */
+	uint32_t obs;		/* output buffer size (bytes) */
+	uint32_t module_flags;	/* flags, reserved for future use */
+	uint32_t cpc;		/* cycles per single run */
+	uint32_t obls;		/* output block size, reserved for future use */
+} __packed;
+
+/*
+ * FW Manifest Header
+ */
+
+#define SOF_MAN_FW_HDR_FW_NAME_LEN	8
+#define SOF_MAN_FW_HDR_ID		{'$', 'A', 'M', '1'}
+#define SOF_MAN_FW_HDR_NAME		"ADSPFW"
+#define SOF_MAN_FW_HDR_FLAGS		0x0
+#define SOF_MAN_FW_HDR_FEATURES		0xff
+
+/*
+ * The firmware has a standard header that is checked by the ROM on firmware
+ * loading. preload_page_count is used by DMA code loader and is entire
+ * image size on CNL. i.e. CNL: total size of the binary’s .text and .rodata
+ * Used by ROM - Immutable.
+ */
+struct sof_man_fw_header {
+	uint8_t header_id[4];
+	uint32_t header_len;
+	uint8_t name[SOF_MAN_FW_HDR_FW_NAME_LEN];
+	/* number of pages of preloaded image loaded by driver */
+	uint32_t preload_page_count;
+	uint32_t fw_image_flags;
+	uint32_t feature_mask;
+	uint16_t major_version;
+	uint16_t minor_version;
+	uint16_t hotfix_version;
+	uint16_t build_version;
+	uint32_t num_module_entries;
+	uint32_t hw_buf_base_addr;
+	uint32_t hw_buf_length;
+	/* target address for binary loading as offset in IMR - must be == base offset */
+	uint32_t load_offset;
+} __packed;
+
+/*
+ * Firmware manifest descriptor. This can contain N modules and N module
+ * configs. Used by ROM - Immutable.
+ */
+struct sof_man_fw_desc {
+	struct sof_man_fw_header header;
+
+	/* Warning - hack for module arrays. For some unknown reason the we
+	 * have a variable size array of struct man_module followed by a
+	 * variable size array of struct mod_config. These should have been
+	 * merged into a variable array of a parent structure. We have to hack
+	 * around this in many places....
+	 *
+	 * struct sof_man_module man_module[];
+	 * struct sof_man_mod_config mod_config[];
+	 */
+
+} __packed;
+
+/*
+ * Component Descriptor. Used by ROM - Immutable.
+ */
+struct sof_man_component_desc {
+	uint32_t reserved[2];	/* all 0 */
+	uint32_t version;
+	uint8_t hash[SOF_MAN_MOD_SHA256_LEN];
+	uint32_t base_offset;
+	uint32_t limit_offset;
+	uint32_t attributes[4];
+} __packed;
+
+/*
+ * Audio DSP extended metadata. Used by ROM - Immutable.
+ */
+struct sof_man_adsp_meta_file_ext {
+	uint32_t ext_type;	/* always 17 for ADSP extension */
+	uint32_t ext_len;
+	uint32_t imr_type;
+	uint8_t reserved[16];	/* all 0 */
+	struct sof_man_component_desc comp_desc[1];
+} __packed;
+
+/*
+ * Module Manifest for rimage module metadata. Not used by ROM.
+ */
+struct sof_man_module_manifest {
+	struct sof_man_module module;
+	uint32_t text_size;
+} __packed;
+
+#endif
diff --git a/include/uapi/sound/sof/tokens.h b/include/uapi/sound/sof/tokens.h
new file mode 100644
index 000000000000..53ea94bf1c08
--- /dev/null
+++ b/include/uapi/sound/sof/tokens.h
@@ -0,0 +1,107 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */
+/*
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * Copyright(c) 2018 Intel Corporation. All rights reserved.
+ * Author: Liam Girdwood <liam.r.girdwood@linux.intel.com>
+ *         Keyon Jie <yang.jie@linux.intel.com>
+ */
+
+/*
+ * Topology IDs and tokens.
+ *
+ * ** MUST BE ALIGNED WITH TOPOLOGY CONFIGURATION TOKEN VALUES **
+ */
+
+#ifndef __INCLUDE_UAPI_SOF_TOPOLOGY_H__
+#define __INCLUDE_UAPI_SOF_TOPOLOGY_H__
+
+/*
+ * Kcontrol IDs
+ */
+#define SOF_TPLG_KCTL_VOL_ID	256
+#define SOF_TPLG_KCTL_ENUM_ID	257
+#define SOF_TPLG_KCTL_BYTES_ID	258
+#define SOF_TPLG_KCTL_SWITCH_ID	259
+
+/*
+ * Tokens - must match values in topology configurations
+ */
+
+/* buffers */
+#define SOF_TKN_BUF_SIZE			100
+#define SOF_TKN_BUF_CAPS			101
+
+/* DAI */
+/* Token retired with ABI 3.2, do not use for new capabilities
+ * #define	SOF_TKN_DAI_DMAC_CONFIG			153
+ */
+#define SOF_TKN_DAI_TYPE			154
+#define SOF_TKN_DAI_INDEX			155
+#define SOF_TKN_DAI_DIRECTION			156
+
+/* scheduling */
+#define SOF_TKN_SCHED_PERIOD			200
+#define SOF_TKN_SCHED_PRIORITY			201
+#define SOF_TKN_SCHED_MIPS			202
+#define SOF_TKN_SCHED_CORE			203
+#define SOF_TKN_SCHED_FRAMES			204
+#define SOF_TKN_SCHED_TIME_DOMAIN		205
+
+/* volume */
+#define SOF_TKN_VOLUME_RAMP_STEP_TYPE		250
+#define SOF_TKN_VOLUME_RAMP_STEP_MS		251
+
+/* SRC */
+#define SOF_TKN_SRC_RATE_IN			300
+#define SOF_TKN_SRC_RATE_OUT			301
+
+/* PCM */
+#define SOF_TKN_PCM_DMAC_CONFIG			353
+
+/* Generic components */
+#define SOF_TKN_COMP_PERIOD_SINK_COUNT		400
+#define SOF_TKN_COMP_PERIOD_SOURCE_COUNT	401
+#define SOF_TKN_COMP_FORMAT			402
+/* Token retired with ABI 3.2, do not use for new capabilities
+ * #define SOF_TKN_COMP_PRELOAD_COUNT		403
+ */
+
+/* SSP */
+#define SOF_TKN_INTEL_SSP_CLKS_CONTROL		500
+#define SOF_TKN_INTEL_SSP_MCLK_ID		501
+#define SOF_TKN_INTEL_SSP_SAMPLE_BITS		502
+#define SOF_TKN_INTEL_SSP_FRAME_PULSE_WIDTH	503
+#define SOF_TKN_INTEL_SSP_QUIRKS		504
+#define SOF_TKN_INTEL_SSP_TDM_PADDING_PER_SLOT	505
+
+/* DMIC */
+#define SOF_TKN_INTEL_DMIC_DRIVER_VERSION	600
+#define SOF_TKN_INTEL_DMIC_CLK_MIN		601
+#define SOF_TKN_INTEL_DMIC_CLK_MAX		602
+#define SOF_TKN_INTEL_DMIC_DUTY_MIN		603
+#define SOF_TKN_INTEL_DMIC_DUTY_MAX		604
+#define SOF_TKN_INTEL_DMIC_NUM_PDM_ACTIVE	605
+#define SOF_TKN_INTEL_DMIC_SAMPLE_RATE		608
+#define SOF_TKN_INTEL_DMIC_FIFO_WORD_LENGTH	609
+
+/* DMIC PDM */
+#define SOF_TKN_INTEL_DMIC_PDM_CTRL_ID		700
+#define SOF_TKN_INTEL_DMIC_PDM_MIC_A_Enable	701
+#define SOF_TKN_INTEL_DMIC_PDM_MIC_B_Enable	702
+#define SOF_TKN_INTEL_DMIC_PDM_POLARITY_A	703
+#define SOF_TKN_INTEL_DMIC_PDM_POLARITY_B	704
+#define SOF_TKN_INTEL_DMIC_PDM_CLK_EDGE		705
+#define SOF_TKN_INTEL_DMIC_PDM_SKEW		706
+
+/* Tone */
+#define SOF_TKN_TONE_SAMPLE_RATE		800
+
+/* Processing Components */
+#define SOF_TKN_PROCESS_TYPE                    900
+
+/* for backward compatibility */
+#define SOF_TKN_EFFECT_TYPE	SOF_TKN_PROCESS_TYPE
+
+#endif
diff --git a/include/uapi/sound/sof/tone.h b/include/uapi/sound/sof/tone.h
new file mode 100644
index 000000000000..d7c6e5d8317e
--- /dev/null
+++ b/include/uapi/sound/sof/tone.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */
+/*
+* This file is provided under a dual BSD/GPLv2 license.  When using or
+* redistributing this file, you may do so under either license.
+*
+* Copyright(c) 2018 Intel Corporation. All rights reserved.
+*/
+
+#ifndef __INCLUDE_UAPI_SOUND_SOF_USER_TONE_H__
+#define __INCLUDE_UAPI_SOUND_SOF_USER_TONE_H__
+
+#define SOF_TONE_IDX_FREQUENCY		0
+#define SOF_TONE_IDX_AMPLITUDE		1
+#define SOF_TONE_IDX_FREQ_MULT		2
+#define SOF_TONE_IDX_AMPL_MULT		3
+#define SOF_TONE_IDX_LENGTH		4
+#define SOF_TONE_IDX_PERIOD		5
+#define SOF_TONE_IDX_REPEATS		6
+#define SOF_TONE_IDX_LIN_RAMP_STEP	7
+
+#endif
diff --git a/include/uapi/sound/sof/trace.h b/include/uapi/sound/sof/trace.h
new file mode 100644
index 000000000000..ffa7288a0f16
--- /dev/null
+++ b/include/uapi/sound/sof/trace.h
@@ -0,0 +1,66 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */
+/*
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * Copyright(c) 2018 Intel Corporation. All rights reserved.
+ */
+
+#ifndef __INCLUDE_UAPI_SOUND_SOF_USER_TRACE_H__
+#define __INCLUDE_UAPI_SOUND_SOF_USER_TRACE_H__
+
+/*
+ * Host system time.
+ *
+ * This property is used by the driver to pass down information about
+ * current system time. It is expressed in us.
+ * FW translates timestamps (in log entries, probe pockets) to this time
+ * domain.
+ *
+ * (cavs: SystemTime).
+ */
+struct system_time {
+	uint32_t val_l;  /* Lower dword of current host time value */
+	uint32_t val_u;  /* Upper dword of current host time value */
+} __packed;
+
+#define LOG_ENABLE		1  /* Enable logging */
+#define LOG_DISABLE		0  /* Disable logging */
+
+#define LOG_LEVEL_CRITICAL	1  /* (FDK fatal) */
+#define LOG_LEVEL_VERBOSE	2
+
+/*
+ * Layout of a log fifo.
+ */
+struct log_buffer_layout {
+	uint32_t read_ptr;  /*read pointer */
+	uint32_t write_ptr; /* write pointer */
+	uint32_t buffer[0]; /* buffer */
+} __packed;
+
+/*
+ * Log buffer status reported by FW.
+ */
+struct log_buffer_status {
+	uint32_t core_id;  /* ID of core that logged to other half */
+} __packed;
+
+#define TRACE_ID_LENGTH 12
+
+/*
+ *  Log entry header.
+ *
+ * The header is followed by an array of arguments (uint32_t[]).
+ * Number of arguments is specified by the params_num field of log_entry
+ */
+struct log_entry_header {
+	uint32_t id_0 : TRACE_ID_LENGTH;	/* e.g. Pipeline ID */
+	uint32_t id_1 : TRACE_ID_LENGTH;	/* e.g. Component ID */
+	uint32_t core_id : 8;		/* Reporting core's id */
+
+	uint64_t timestamp;		/* Timestamp (in dsp ticks) */
+	uint32_t log_entry_address;	/* Address of log entry in ELF */
+} __packed;
+
+#endif
diff --git a/kernel/cgroup/Makefile b/kernel/cgroup/Makefile
index bfcdae896122..5d7a76bfbbb7 100644
--- a/kernel/cgroup/Makefile
+++ b/kernel/cgroup/Makefile
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0
-obj-y := cgroup.o rstat.o namespace.o cgroup-v1.o
+obj-y := cgroup.o rstat.o namespace.o cgroup-v1.o freezer.o
 
-obj-$(CONFIG_CGROUP_FREEZER) += freezer.o
+obj-$(CONFIG_CGROUP_FREEZER) += legacy_freezer.o
 obj-$(CONFIG_CGROUP_PIDS) += pids.o
 obj-$(CONFIG_CGROUP_RDMA) += rdma.o
 obj-$(CONFIG_CPUSETS) += cpuset.o
diff --git a/kernel/cgroup/cgroup-internal.h b/kernel/cgroup/cgroup-internal.h
index 30e39f3932ad..809e34a3c017 100644
--- a/kernel/cgroup/cgroup-internal.h
+++ b/kernel/cgroup/cgroup-internal.h
@@ -28,12 +28,15 @@ extern void __init enable_debug_cgroup(void);
 #define TRACE_CGROUP_PATH(type, cgrp, ...)				\
 	do {								\
 		if (trace_cgroup_##type##_enabled()) {			\
-			spin_lock(&trace_cgroup_path_lock);		\
+			unsigned long flags;				\
+			spin_lock_irqsave(&trace_cgroup_path_lock,	\
+					  flags);			\
 			cgroup_path(cgrp, trace_cgroup_path,		\
 				    TRACE_CGROUP_PATH_LEN);		\
 			trace_cgroup_##type(cgrp, trace_cgroup_path,	\
 					    ##__VA_ARGS__);		\
-			spin_unlock(&trace_cgroup_path_lock);		\
+			spin_unlock_irqrestore(&trace_cgroup_path_lock, \
+					       flags);			\
 		}							\
 	} while (0)
 
@@ -240,6 +243,7 @@ int cgroup_rmdir(struct kernfs_node *kn);
 int cgroup_show_path(struct seq_file *sf, struct kernfs_node *kf_node,
 		     struct kernfs_root *kf_root);
 
+int __cgroup_task_count(const struct cgroup *cgrp);
 int cgroup_task_count(const struct cgroup *cgrp);
 
 /*
diff --git a/kernel/cgroup/cgroup-v1.c b/kernel/cgroup/cgroup-v1.c
index c126b34fd4ff..68ca5de7ec27 100644
--- a/kernel/cgroup/cgroup-v1.c
+++ b/kernel/cgroup/cgroup-v1.c
@@ -342,22 +342,6 @@ static struct cgroup_pidlist *cgroup_pidlist_find_create(struct cgroup *cgrp,
 	return l;
 }
 
-/**
- * cgroup_task_count - count the number of tasks in a cgroup.
- * @cgrp: the cgroup in question
- */
-int cgroup_task_count(const struct cgroup *cgrp)
-{
-	int count = 0;
-	struct cgrp_cset_link *link;
-
-	spin_lock_irq(&css_set_lock);
-	list_for_each_entry(link, &cgrp->cset_links, cset_link)
-		count += link->cset->nr_tasks;
-	spin_unlock_irq(&css_set_lock);
-	return count;
-}
-
 /*
  * Load a cgroup's pidarray with either procs' tgids or tasks' pids
  */
diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c
index 3f2b4bde0f9c..327f37c9fdfa 100644
--- a/kernel/cgroup/cgroup.c
+++ b/kernel/cgroup/cgroup.c
@@ -593,6 +593,39 @@ static void cgroup_get_live(struct cgroup *cgrp)
 	css_get(&cgrp->self);
 }
 
+/**
+ * __cgroup_task_count - count the number of tasks in a cgroup. The caller
+ * is responsible for taking the css_set_lock.
+ * @cgrp: the cgroup in question
+ */
+int __cgroup_task_count(const struct cgroup *cgrp)
+{
+	int count = 0;
+	struct cgrp_cset_link *link;
+
+	lockdep_assert_held(&css_set_lock);
+
+	list_for_each_entry(link, &cgrp->cset_links, cset_link)
+		count += link->cset->nr_tasks;
+
+	return count;
+}
+
+/**
+ * cgroup_task_count - count the number of tasks in a cgroup.
+ * @cgrp: the cgroup in question
+ */
+int cgroup_task_count(const struct cgroup *cgrp)
+{
+	int count;
+
+	spin_lock_irq(&css_set_lock);
+	count = __cgroup_task_count(cgrp);
+	spin_unlock_irq(&css_set_lock);
+
+	return count;
+}
+
 struct cgroup_subsys_state *of_css(struct kernfs_open_file *of)
 {
 	struct cgroup *cgrp = of->kn->parent->priv;
@@ -783,6 +816,8 @@ static void cgroup_update_populated(struct cgroup *cgrp, bool populated)
 			break;
 
 		cgroup1_check_for_release(cgrp);
+		TRACE_CGROUP_PATH(notify_populated, cgrp,
+				  cgroup_is_populated(cgrp));
 		cgroup_file_notify(&cgrp->events_file);
 
 		child = cgrp;
@@ -2402,8 +2437,15 @@ static int cgroup_migrate_execute(struct cgroup_mgctx *mgctx)
 			get_css_set(to_cset);
 			to_cset->nr_tasks++;
 			css_set_move_task(task, from_cset, to_cset, true);
-			put_css_set_locked(from_cset);
 			from_cset->nr_tasks--;
+			/*
+			 * If the source or destination cgroup is frozen,
+			 * the task might require to change its state.
+			 */
+			cgroup_freezer_migrate_task(task, from_cset->dfl_cgrp,
+						    to_cset->dfl_cgrp);
+			put_css_set_locked(from_cset);
+
 		}
 	}
 	spin_unlock_irq(&css_set_lock);
@@ -2602,7 +2644,7 @@ int cgroup_migrate_prepare_dst(struct cgroup_mgctx *mgctx)
 
 		dst_cset = find_css_set(src_cset, src_cset->mg_dst_cgrp);
 		if (!dst_cset)
-			goto err;
+			return -ENOMEM;
 
 		WARN_ON_ONCE(src_cset->mg_dst_cset || dst_cset->mg_dst_cset);
 
@@ -2634,9 +2676,6 @@ int cgroup_migrate_prepare_dst(struct cgroup_mgctx *mgctx)
 	}
 
 	return 0;
-err:
-	cgroup_migrate_finish(mgctx);
-	return -ENOMEM;
 }
 
 /**
@@ -3447,8 +3486,11 @@ static ssize_t cgroup_max_depth_write(struct kernfs_open_file *of,
 
 static int cgroup_events_show(struct seq_file *seq, void *v)
 {
-	seq_printf(seq, "populated %d\n",
-		   cgroup_is_populated(seq_css(seq)->cgroup));
+	struct cgroup *cgrp = seq_css(seq)->cgroup;
+
+	seq_printf(seq, "populated %d\n", cgroup_is_populated(cgrp));
+	seq_printf(seq, "frozen %d\n", test_bit(CGRP_FROZEN, &cgrp->flags));
+
 	return 0;
 }
 
@@ -3510,6 +3552,40 @@ static int cgroup_cpu_pressure_show(struct seq_file *seq, void *v)
 }
 #endif
 
+static int cgroup_freeze_show(struct seq_file *seq, void *v)
+{
+	struct cgroup *cgrp = seq_css(seq)->cgroup;
+
+	seq_printf(seq, "%d\n", cgrp->freezer.freeze);
+
+	return 0;
+}
+
+static ssize_t cgroup_freeze_write(struct kernfs_open_file *of,
+				   char *buf, size_t nbytes, loff_t off)
+{
+	struct cgroup *cgrp;
+	ssize_t ret;
+	int freeze;
+
+	ret = kstrtoint(strstrip(buf), 0, &freeze);
+	if (ret)
+		return ret;
+
+	if (freeze < 0 || freeze > 1)
+		return -ERANGE;
+
+	cgrp = cgroup_kn_lock_live(of->kn, false);
+	if (!cgrp)
+		return -ENOENT;
+
+	cgroup_freeze(cgrp, freeze);
+
+	cgroup_kn_unlock(of->kn);
+
+	return nbytes;
+}
+
 static int cgroup_file_open(struct kernfs_open_file *of)
 {
 	struct cftype *cft = of->kn->priv;
@@ -4654,6 +4730,12 @@ static struct cftype cgroup_base_files[] = {
 		.seq_show = cgroup_stat_show,
 	},
 	{
+		.name = "cgroup.freeze",
+		.flags = CFTYPE_NOT_ON_ROOT,
+		.seq_show = cgroup_freeze_show,
+		.write = cgroup_freeze_write,
+	},
+	{
 		.name = "cpu.stat",
 		.flags = CFTYPE_NOT_ON_ROOT,
 		.seq_show = cpu_stat_show,
@@ -4781,9 +4863,11 @@ static void css_release_work_fn(struct work_struct *work)
 		if (cgroup_on_dfl(cgrp))
 			cgroup_rstat_flush(cgrp);
 
+		spin_lock_irq(&css_set_lock);
 		for (tcgrp = cgroup_parent(cgrp); tcgrp;
 		     tcgrp = cgroup_parent(tcgrp))
 			tcgrp->nr_dying_descendants--;
+		spin_unlock_irq(&css_set_lock);
 
 		cgroup_idr_remove(&cgrp->root->cgroup_idr, cgrp->id);
 		cgrp->id = -1;
@@ -5001,12 +5085,31 @@ static struct cgroup *cgroup_create(struct cgroup *parent)
 	if (ret)
 		goto out_psi_free;
 
+	/*
+	 * New cgroup inherits effective freeze counter, and
+	 * if the parent has to be frozen, the child has too.
+	 */
+	cgrp->freezer.e_freeze = parent->freezer.e_freeze;
+	if (cgrp->freezer.e_freeze)
+		set_bit(CGRP_FROZEN, &cgrp->flags);
+
+	spin_lock_irq(&css_set_lock);
 	for (tcgrp = cgrp; tcgrp; tcgrp = cgroup_parent(tcgrp)) {
 		cgrp->ancestor_ids[tcgrp->level] = tcgrp->id;
 
-		if (tcgrp != cgrp)
+		if (tcgrp != cgrp) {
 			tcgrp->nr_descendants++;
+
+			/*
+			 * If the new cgroup is frozen, all ancestor cgroups
+			 * get a new frozen descendant, but their state can't
+			 * change because of this.
+			 */
+			if (cgrp->freezer.e_freeze)
+				tcgrp->freezer.nr_frozen_descendants++;
+		}
 	}
+	spin_unlock_irq(&css_set_lock);
 
 	if (notify_on_release(parent))
 		set_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);
@@ -5291,10 +5394,18 @@ static int cgroup_destroy_locked(struct cgroup *cgrp)
 	if (parent && cgroup_is_threaded(cgrp))
 		parent->nr_threaded_children--;
 
+	spin_lock_irq(&css_set_lock);
 	for (tcgrp = cgroup_parent(cgrp); tcgrp; tcgrp = cgroup_parent(tcgrp)) {
 		tcgrp->nr_descendants--;
 		tcgrp->nr_dying_descendants++;
+		/*
+		 * If the dying cgroup is frozen, decrease frozen descendants
+		 * counters of ancestor cgroups.
+		 */
+		if (test_bit(CGRP_FROZEN, &cgrp->flags))
+			tcgrp->freezer.nr_frozen_descendants--;
 	}
+	spin_unlock_irq(&css_set_lock);
 
 	cgroup1_check_for_release(parent);
 
@@ -5746,6 +5857,26 @@ void cgroup_post_fork(struct task_struct *child)
 			cset->nr_tasks++;
 			css_set_move_task(child, NULL, cset, false);
 		}
+
+		/*
+		 * If the cgroup has to be frozen, the new task has too.
+		 * Let's set the JOBCTL_TRAP_FREEZE jobctl bit to get
+		 * the task into the frozen state.
+		 */
+		if (unlikely(cgroup_task_freeze(child))) {
+			spin_lock(&child->sighand->siglock);
+			WARN_ON_ONCE(child->frozen);
+			child->jobctl |= JOBCTL_TRAP_FREEZE;
+			spin_unlock(&child->sighand->siglock);
+
+			/*
+			 * Calling cgroup_update_frozen() isn't required here,
+			 * because it will be called anyway a bit later
+			 * from do_freezer_trap(). So we avoid cgroup's
+			 * transient switch from the frozen state and back.
+			 */
+		}
+
 		spin_unlock_irq(&css_set_lock);
 	}
 
@@ -5794,6 +5925,11 @@ void cgroup_exit(struct task_struct *tsk)
 		spin_lock_irq(&css_set_lock);
 		css_set_move_task(tsk, cset, NULL, false);
 		cset->nr_tasks--;
+
+		WARN_ON_ONCE(cgroup_task_frozen(tsk));
+		if (unlikely(cgroup_task_freeze(tsk)))
+			cgroup_update_frozen(task_dfl_cgroup(tsk));
+
 		spin_unlock_irq(&css_set_lock);
 	} else {
 		get_css_set(cset);
diff --git a/kernel/cgroup/debug.c b/kernel/cgroup/debug.c
index 5f1b87330bee..80aa3f027ac3 100644
--- a/kernel/cgroup/debug.c
+++ b/kernel/cgroup/debug.c
@@ -64,8 +64,8 @@ static int current_css_set_read(struct seq_file *seq, void *v)
 		css = cset->subsys[ss->id];
 		if (!css)
 			continue;
-		seq_printf(seq, "%2d: %-4s\t- %lx[%d]\n", ss->id, ss->name,
-			  (unsigned long)css, css->id);
+		seq_printf(seq, "%2d: %-4s\t- %p[%d]\n", ss->id, ss->name,
+			  css, css->id);
 	}
 	rcu_read_unlock();
 	spin_unlock_irq(&css_set_lock);
@@ -224,8 +224,8 @@ static int cgroup_subsys_states_read(struct seq_file *seq, void *v)
 		if (css->parent)
 			snprintf(pbuf, sizeof(pbuf) - 1, " P=%d",
 				 css->parent->id);
-		seq_printf(seq, "%2d: %-4s\t- %lx[%d] %d%s\n", ss->id, ss->name,
-			  (unsigned long)css, css->id,
+		seq_printf(seq, "%2d: %-4s\t- %p[%d] %d%s\n", ss->id, ss->name,
+			  css, css->id,
 			  atomic_read(&css->online_cnt), pbuf);
 	}
 
diff --git a/kernel/cgroup/freezer.c b/kernel/cgroup/freezer.c
index 08236798d173..8cf010680678 100644
--- a/kernel/cgroup/freezer.c
+++ b/kernel/cgroup/freezer.c
@@ -1,481 +1,314 @@
-/*
- * cgroup_freezer.c -  control group freezer subsystem
- *
- * Copyright IBM Corporation, 2007
- *
- * Author : Cedric Le Goater <clg@fr.ibm.com>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of version 2.1 of the GNU Lesser General Public License
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it would be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
- */
-
-#include <linux/export.h>
-#include <linux/slab.h>
+//SPDX-License-Identifier: GPL-2.0
 #include <linux/cgroup.h>
-#include <linux/fs.h>
-#include <linux/uaccess.h>
-#include <linux/freezer.h>
-#include <linux/seq_file.h>
-#include <linux/mutex.h>
-
-/*
- * A cgroup is freezing if any FREEZING flags are set.  FREEZING_SELF is
- * set if "FROZEN" is written to freezer.state cgroupfs file, and cleared
- * for "THAWED".  FREEZING_PARENT is set if the parent freezer is FREEZING
- * for whatever reason.  IOW, a cgroup has FREEZING_PARENT set if one of
- * its ancestors has FREEZING_SELF set.
- */
-enum freezer_state_flags {
-	CGROUP_FREEZER_ONLINE	= (1 << 0), /* freezer is fully online */
-	CGROUP_FREEZING_SELF	= (1 << 1), /* this freezer is freezing */
-	CGROUP_FREEZING_PARENT	= (1 << 2), /* the parent freezer is freezing */
-	CGROUP_FROZEN		= (1 << 3), /* this and its descendants frozen */
+#include <linux/sched.h>
+#include <linux/sched/task.h>
+#include <linux/sched/signal.h>
 
-	/* mask for all FREEZING flags */
-	CGROUP_FREEZING		= CGROUP_FREEZING_SELF | CGROUP_FREEZING_PARENT,
-};
+#include "cgroup-internal.h"
 
-struct freezer {
-	struct cgroup_subsys_state	css;
-	unsigned int			state;
-};
+#include <trace/events/cgroup.h>
 
-static DEFINE_MUTEX(freezer_mutex);
-
-static inline struct freezer *css_freezer(struct cgroup_subsys_state *css)
+/*
+ * Propagate the cgroup frozen state upwards by the cgroup tree.
+ */
+static void cgroup_propagate_frozen(struct cgroup *cgrp, bool frozen)
 {
-	return css ? container_of(css, struct freezer, css) : NULL;
-}
+	int desc = 1;
 
-static inline struct freezer *task_freezer(struct task_struct *task)
-{
-	return css_freezer(task_css(task, freezer_cgrp_id));
+	/*
+	 * If the new state is frozen, some freezing ancestor cgroups may change
+	 * their state too, depending on if all their descendants are frozen.
+	 *
+	 * Otherwise, all ancestor cgroups are forced into the non-frozen state.
+	 */
+	while ((cgrp = cgroup_parent(cgrp))) {
+		if (frozen) {
+			cgrp->freezer.nr_frozen_descendants += desc;
+			if (!test_bit(CGRP_FROZEN, &cgrp->flags) &&
+			    test_bit(CGRP_FREEZE, &cgrp->flags) &&
+			    cgrp->freezer.nr_frozen_descendants ==
+			    cgrp->nr_descendants) {
+				set_bit(CGRP_FROZEN, &cgrp->flags);
+				cgroup_file_notify(&cgrp->events_file);
+				TRACE_CGROUP_PATH(notify_frozen, cgrp, 1);
+				desc++;
+			}
+		} else {
+			cgrp->freezer.nr_frozen_descendants -= desc;
+			if (test_bit(CGRP_FROZEN, &cgrp->flags)) {
+				clear_bit(CGRP_FROZEN, &cgrp->flags);
+				cgroup_file_notify(&cgrp->events_file);
+				TRACE_CGROUP_PATH(notify_frozen, cgrp, 0);
+				desc++;
+			}
+		}
+	}
 }
 
-static struct freezer *parent_freezer(struct freezer *freezer)
+/*
+ * Revisit the cgroup frozen state.
+ * Checks if the cgroup is really frozen and perform all state transitions.
+ */
+void cgroup_update_frozen(struct cgroup *cgrp)
 {
-	return css_freezer(freezer->css.parent);
-}
+	bool frozen;
 
-bool cgroup_freezing(struct task_struct *task)
-{
-	bool ret;
+	lockdep_assert_held(&css_set_lock);
 
-	rcu_read_lock();
-	ret = task_freezer(task)->state & CGROUP_FREEZING;
-	rcu_read_unlock();
+	/*
+	 * If the cgroup has to be frozen (CGRP_FREEZE bit set),
+	 * and all tasks are frozen and/or stopped, let's consider
+	 * the cgroup frozen. Otherwise it's not frozen.
+	 */
+	frozen = test_bit(CGRP_FREEZE, &cgrp->flags) &&
+		cgrp->freezer.nr_frozen_tasks == __cgroup_task_count(cgrp);
 
-	return ret;
-}
+	if (frozen) {
+		/* Already there? */
+		if (test_bit(CGRP_FROZEN, &cgrp->flags))
+			return;
 
-static const char *freezer_state_strs(unsigned int state)
-{
-	if (state & CGROUP_FROZEN)
-		return "FROZEN";
-	if (state & CGROUP_FREEZING)
-		return "FREEZING";
-	return "THAWED";
-};
-
-static struct cgroup_subsys_state *
-freezer_css_alloc(struct cgroup_subsys_state *parent_css)
-{
-	struct freezer *freezer;
+		set_bit(CGRP_FROZEN, &cgrp->flags);
+	} else {
+		/* Already there? */
+		if (!test_bit(CGRP_FROZEN, &cgrp->flags))
+			return;
 
-	freezer = kzalloc(sizeof(struct freezer), GFP_KERNEL);
-	if (!freezer)
-		return ERR_PTR(-ENOMEM);
+		clear_bit(CGRP_FROZEN, &cgrp->flags);
+	}
+	cgroup_file_notify(&cgrp->events_file);
+	TRACE_CGROUP_PATH(notify_frozen, cgrp, frozen);
 
-	return &freezer->css;
+	/* Update the state of ancestor cgroups. */
+	cgroup_propagate_frozen(cgrp, frozen);
 }
 
-/**
- * freezer_css_online - commit creation of a freezer css
- * @css: css being created
- *
- * We're committing to creation of @css.  Mark it online and inherit
- * parent's freezing state while holding both parent's and our
- * freezer->lock.
+/*
+ * Increment cgroup's nr_frozen_tasks.
  */
-static int freezer_css_online(struct cgroup_subsys_state *css)
+static void cgroup_inc_frozen_cnt(struct cgroup *cgrp)
 {
-	struct freezer *freezer = css_freezer(css);
-	struct freezer *parent = parent_freezer(freezer);
-
-	mutex_lock(&freezer_mutex);
-
-	freezer->state |= CGROUP_FREEZER_ONLINE;
-
-	if (parent && (parent->state & CGROUP_FREEZING)) {
-		freezer->state |= CGROUP_FREEZING_PARENT | CGROUP_FROZEN;
-		atomic_inc(&system_freezing_cnt);
-	}
-
-	mutex_unlock(&freezer_mutex);
-	return 0;
+	cgrp->freezer.nr_frozen_tasks++;
 }
 
-/**
- * freezer_css_offline - initiate destruction of a freezer css
- * @css: css being destroyed
- *
- * @css is going away.  Mark it dead and decrement system_freezing_count if
- * it was holding one.
+/*
+ * Decrement cgroup's nr_frozen_tasks.
  */
-static void freezer_css_offline(struct cgroup_subsys_state *css)
+static void cgroup_dec_frozen_cnt(struct cgroup *cgrp)
 {
-	struct freezer *freezer = css_freezer(css);
-
-	mutex_lock(&freezer_mutex);
-
-	if (freezer->state & CGROUP_FREEZING)
-		atomic_dec(&system_freezing_cnt);
-
-	freezer->state = 0;
-
-	mutex_unlock(&freezer_mutex);
+	cgrp->freezer.nr_frozen_tasks--;
+	WARN_ON_ONCE(cgrp->freezer.nr_frozen_tasks < 0);
 }
 
-static void freezer_css_free(struct cgroup_subsys_state *css)
+/*
+ * Enter frozen/stopped state, if not yet there. Update cgroup's counters,
+ * and revisit the state of the cgroup, if necessary.
+ */
+void cgroup_enter_frozen(void)
 {
-	kfree(css_freezer(css));
+	struct cgroup *cgrp;
+
+	if (current->frozen)
+		return;
+
+	spin_lock_irq(&css_set_lock);
+	current->frozen = true;
+	cgrp = task_dfl_cgroup(current);
+	cgroup_inc_frozen_cnt(cgrp);
+	cgroup_update_frozen(cgrp);
+	spin_unlock_irq(&css_set_lock);
 }
 
 /*
- * Tasks can be migrated into a different freezer anytime regardless of its
- * current state.  freezer_attach() is responsible for making new tasks
- * conform to the current state.
+ * Conditionally leave frozen/stopped state. Update cgroup's counters,
+ * and revisit the state of the cgroup, if necessary.
  *
- * Freezer state changes and task migration are synchronized via
- * @freezer->lock.  freezer_attach() makes the new tasks conform to the
- * current state and all following state changes can see the new tasks.
+ * If always_leave is not set, and the cgroup is freezing,
+ * we're racing with the cgroup freezing. In this case, we don't
+ * drop the frozen counter to avoid a transient switch to
+ * the unfrozen state.
  */
-static void freezer_attach(struct cgroup_taskset *tset)
+void cgroup_leave_frozen(bool always_leave)
 {
-	struct task_struct *task;
-	struct cgroup_subsys_state *new_css;
-
-	mutex_lock(&freezer_mutex);
-
-	/*
-	 * Make the new tasks conform to the current state of @new_css.
-	 * For simplicity, when migrating any task to a FROZEN cgroup, we
-	 * revert it to FREEZING and let update_if_frozen() determine the
-	 * correct state later.
-	 *
-	 * Tasks in @tset are on @new_css but may not conform to its
-	 * current state before executing the following - !frozen tasks may
-	 * be visible in a FROZEN cgroup and frozen tasks in a THAWED one.
-	 */
-	cgroup_taskset_for_each(task, new_css, tset) {
-		struct freezer *freezer = css_freezer(new_css);
-
-		if (!(freezer->state & CGROUP_FREEZING)) {
-			__thaw_task(task);
-		} else {
-			freeze_task(task);
-			/* clear FROZEN and propagate upwards */
-			while (freezer && (freezer->state & CGROUP_FROZEN)) {
-				freezer->state &= ~CGROUP_FROZEN;
-				freezer = parent_freezer(freezer);
-			}
-		}
+	struct cgroup *cgrp;
+
+	spin_lock_irq(&css_set_lock);
+	cgrp = task_dfl_cgroup(current);
+	if (always_leave || !test_bit(CGRP_FREEZE, &cgrp->flags)) {
+		cgroup_dec_frozen_cnt(cgrp);
+		cgroup_update_frozen(cgrp);
+		WARN_ON_ONCE(!current->frozen);
+		current->frozen = false;
+	} else if (!(current->jobctl & JOBCTL_TRAP_FREEZE)) {
+		spin_lock(&current->sighand->siglock);
+		current->jobctl |= JOBCTL_TRAP_FREEZE;
+		set_thread_flag(TIF_SIGPENDING);
+		spin_unlock(&current->sighand->siglock);
 	}
-
-	mutex_unlock(&freezer_mutex);
+	spin_unlock_irq(&css_set_lock);
 }
 
-/**
- * freezer_fork - cgroup post fork callback
- * @task: a task which has just been forked
- *
- * @task has just been created and should conform to the current state of
- * the cgroup_freezer it belongs to.  This function may race against
- * freezer_attach().  Losing to freezer_attach() means that we don't have
- * to do anything as freezer_attach() will put @task into the appropriate
- * state.
+/*
+ * Freeze or unfreeze the task by setting or clearing the JOBCTL_TRAP_FREEZE
+ * jobctl bit.
  */
-static void freezer_fork(struct task_struct *task)
+static void cgroup_freeze_task(struct task_struct *task, bool freeze)
 {
-	struct freezer *freezer;
+	unsigned long flags;
 
-	/*
-	 * The root cgroup is non-freezable, so we can skip locking the
-	 * freezer.  This is safe regardless of race with task migration.
-	 * If we didn't race or won, skipping is obviously the right thing
-	 * to do.  If we lost and root is the new cgroup, noop is still the
-	 * right thing to do.
-	 */
-	if (task_css_is_root(task, freezer_cgrp_id))
+	/* If the task is about to die, don't bother with freezing it. */
+	if (!lock_task_sighand(task, &flags))
 		return;
 
-	mutex_lock(&freezer_mutex);
-	rcu_read_lock();
-
-	freezer = task_freezer(task);
-	if (freezer->state & CGROUP_FREEZING)
-		freeze_task(task);
+	if (freeze) {
+		task->jobctl |= JOBCTL_TRAP_FREEZE;
+		signal_wake_up(task, false);
+	} else {
+		task->jobctl &= ~JOBCTL_TRAP_FREEZE;
+		wake_up_process(task);
+	}
 
-	rcu_read_unlock();
-	mutex_unlock(&freezer_mutex);
+	unlock_task_sighand(task, &flags);
 }
 
-/**
- * update_if_frozen - update whether a cgroup finished freezing
- * @css: css of interest
- *
- * Once FREEZING is initiated, transition to FROZEN is lazily updated by
- * calling this function.  If the current state is FREEZING but not FROZEN,
- * this function checks whether all tasks of this cgroup and the descendant
- * cgroups finished freezing and, if so, sets FROZEN.
- *
- * The caller is responsible for grabbing RCU read lock and calling
- * update_if_frozen() on all descendants prior to invoking this function.
- *
- * Task states and freezer state might disagree while tasks are being
- * migrated into or out of @css, so we can't verify task states against
- * @freezer state here.  See freezer_attach() for details.
+/*
+ * Freeze or unfreeze all tasks in the given cgroup.
  */
-static void update_if_frozen(struct cgroup_subsys_state *css)
+static void cgroup_do_freeze(struct cgroup *cgrp, bool freeze)
 {
-	struct freezer *freezer = css_freezer(css);
-	struct cgroup_subsys_state *pos;
 	struct css_task_iter it;
 	struct task_struct *task;
 
-	lockdep_assert_held(&freezer_mutex);
-
-	if (!(freezer->state & CGROUP_FREEZING) ||
-	    (freezer->state & CGROUP_FROZEN))
-		return;
+	lockdep_assert_held(&cgroup_mutex);
 
-	/* are all (live) children frozen? */
-	rcu_read_lock();
-	css_for_each_child(pos, css) {
-		struct freezer *child = css_freezer(pos);
-
-		if ((child->state & CGROUP_FREEZER_ONLINE) &&
-		    !(child->state & CGROUP_FROZEN)) {
-			rcu_read_unlock();
-			return;
-		}
-	}
-	rcu_read_unlock();
+	spin_lock_irq(&css_set_lock);
+	if (freeze)
+		set_bit(CGRP_FREEZE, &cgrp->flags);
+	else
+		clear_bit(CGRP_FREEZE, &cgrp->flags);
+	spin_unlock_irq(&css_set_lock);
 
-	/* are all tasks frozen? */
-	css_task_iter_start(css, 0, &it);
+	if (freeze)
+		TRACE_CGROUP_PATH(freeze, cgrp);
+	else
+		TRACE_CGROUP_PATH(unfreeze, cgrp);
 
+	css_task_iter_start(&cgrp->self, 0, &it);
 	while ((task = css_task_iter_next(&it))) {
-		if (freezing(task)) {
-			/*
-			 * freezer_should_skip() indicates that the task
-			 * should be skipped when determining freezing
-			 * completion.  Consider it frozen in addition to
-			 * the usual frozen condition.
-			 */
-			if (!frozen(task) && !freezer_should_skip(task))
-				goto out_iter_end;
-		}
-	}
-
-	freezer->state |= CGROUP_FROZEN;
-out_iter_end:
-	css_task_iter_end(&it);
-}
-
-static int freezer_read(struct seq_file *m, void *v)
-{
-	struct cgroup_subsys_state *css = seq_css(m), *pos;
-
-	mutex_lock(&freezer_mutex);
-	rcu_read_lock();
-
-	/* update states bottom-up */
-	css_for_each_descendant_post(pos, css) {
-		if (!css_tryget_online(pos))
+		/*
+		 * Ignore kernel threads here. Freezing cgroups containing
+		 * kthreads isn't supported.
+		 */
+		if (task->flags & PF_KTHREAD)
 			continue;
-		rcu_read_unlock();
-
-		update_if_frozen(pos);
-
-		rcu_read_lock();
-		css_put(pos);
+		cgroup_freeze_task(task, freeze);
 	}
-
-	rcu_read_unlock();
-	mutex_unlock(&freezer_mutex);
-
-	seq_puts(m, freezer_state_strs(css_freezer(css)->state));
-	seq_putc(m, '\n');
-	return 0;
-}
-
-static void freeze_cgroup(struct freezer *freezer)
-{
-	struct css_task_iter it;
-	struct task_struct *task;
-
-	css_task_iter_start(&freezer->css, 0, &it);
-	while ((task = css_task_iter_next(&it)))
-		freeze_task(task);
 	css_task_iter_end(&it);
-}
 
-static void unfreeze_cgroup(struct freezer *freezer)
-{
-	struct css_task_iter it;
-	struct task_struct *task;
-
-	css_task_iter_start(&freezer->css, 0, &it);
-	while ((task = css_task_iter_next(&it)))
-		__thaw_task(task);
-	css_task_iter_end(&it);
+	/*
+	 * Cgroup state should be revisited here to cover empty leaf cgroups
+	 * and cgroups which descendants are already in the desired state.
+	 */
+	spin_lock_irq(&css_set_lock);
+	if (cgrp->nr_descendants == cgrp->freezer.nr_frozen_descendants)
+		cgroup_update_frozen(cgrp);
+	spin_unlock_irq(&css_set_lock);
 }
 
-/**
- * freezer_apply_state - apply state change to a single cgroup_freezer
- * @freezer: freezer to apply state change to
- * @freeze: whether to freeze or unfreeze
- * @state: CGROUP_FREEZING_* flag to set or clear
- *
- * Set or clear @state on @cgroup according to @freeze, and perform
- * freezing or thawing as necessary.
+/*
+ * Adjust the task state (freeze or unfreeze) and revisit the state of
+ * source and destination cgroups.
  */
-static void freezer_apply_state(struct freezer *freezer, bool freeze,
-				unsigned int state)
+void cgroup_freezer_migrate_task(struct task_struct *task,
+				 struct cgroup *src, struct cgroup *dst)
 {
-	/* also synchronizes against task migration, see freezer_attach() */
-	lockdep_assert_held(&freezer_mutex);
+	lockdep_assert_held(&css_set_lock);
 
-	if (!(freezer->state & CGROUP_FREEZER_ONLINE))
+	/*
+	 * Kernel threads are not supposed to be frozen at all.
+	 */
+	if (task->flags & PF_KTHREAD)
 		return;
 
-	if (freeze) {
-		if (!(freezer->state & CGROUP_FREEZING))
-			atomic_inc(&system_freezing_cnt);
-		freezer->state |= state;
-		freeze_cgroup(freezer);
-	} else {
-		bool was_freezing = freezer->state & CGROUP_FREEZING;
-
-		freezer->state &= ~state;
-
-		if (!(freezer->state & CGROUP_FREEZING)) {
-			if (was_freezing)
-				atomic_dec(&system_freezing_cnt);
-			freezer->state &= ~CGROUP_FROZEN;
-			unfreeze_cgroup(freezer);
-		}
+	/*
+	 * Adjust counters of freezing and frozen tasks.
+	 * Note, that if the task is frozen, but the destination cgroup is not
+	 * frozen, we bump both counters to keep them balanced.
+	 */
+	if (task->frozen) {
+		cgroup_inc_frozen_cnt(dst);
+		cgroup_dec_frozen_cnt(src);
 	}
-}
-
-/**
- * freezer_change_state - change the freezing state of a cgroup_freezer
- * @freezer: freezer of interest
- * @freeze: whether to freeze or thaw
- *
- * Freeze or thaw @freezer according to @freeze.  The operations are
- * recursive - all descendants of @freezer will be affected.
- */
-static void freezer_change_state(struct freezer *freezer, bool freeze)
-{
-	struct cgroup_subsys_state *pos;
+	cgroup_update_frozen(dst);
+	cgroup_update_frozen(src);
 
 	/*
-	 * Update all its descendants in pre-order traversal.  Each
-	 * descendant will try to inherit its parent's FREEZING state as
-	 * CGROUP_FREEZING_PARENT.
+	 * Force the task to the desired state.
 	 */
-	mutex_lock(&freezer_mutex);
-	rcu_read_lock();
-	css_for_each_descendant_pre(pos, &freezer->css) {
-		struct freezer *pos_f = css_freezer(pos);
-		struct freezer *parent = parent_freezer(pos_f);
-
-		if (!css_tryget_online(pos))
-			continue;
-		rcu_read_unlock();
-
-		if (pos_f == freezer)
-			freezer_apply_state(pos_f, freeze,
-					    CGROUP_FREEZING_SELF);
-		else
-			freezer_apply_state(pos_f,
-					    parent->state & CGROUP_FREEZING,
-					    CGROUP_FREEZING_PARENT);
-
-		rcu_read_lock();
-		css_put(pos);
-	}
-	rcu_read_unlock();
-	mutex_unlock(&freezer_mutex);
+	cgroup_freeze_task(task, test_bit(CGRP_FREEZE, &dst->flags));
 }
 
-static ssize_t freezer_write(struct kernfs_open_file *of,
-			     char *buf, size_t nbytes, loff_t off)
+void cgroup_freeze(struct cgroup *cgrp, bool freeze)
 {
-	bool freeze;
+	struct cgroup_subsys_state *css;
+	struct cgroup *dsct;
+	bool applied = false;
 
-	buf = strstrip(buf);
+	lockdep_assert_held(&cgroup_mutex);
 
-	if (strcmp(buf, freezer_state_strs(0)) == 0)
-		freeze = false;
-	else if (strcmp(buf, freezer_state_strs(CGROUP_FROZEN)) == 0)
-		freeze = true;
-	else
-		return -EINVAL;
+	/*
+	 * Nothing changed? Just exit.
+	 */
+	if (cgrp->freezer.freeze == freeze)
+		return;
 
-	freezer_change_state(css_freezer(of_css(of)), freeze);
-	return nbytes;
-}
+	cgrp->freezer.freeze = freeze;
 
-static u64 freezer_self_freezing_read(struct cgroup_subsys_state *css,
-				      struct cftype *cft)
-{
-	struct freezer *freezer = css_freezer(css);
+	/*
+	 * Propagate changes downwards the cgroup tree.
+	 */
+	css_for_each_descendant_pre(css, &cgrp->self) {
+		dsct = css->cgroup;
 
-	return (bool)(freezer->state & CGROUP_FREEZING_SELF);
-}
+		if (cgroup_is_dead(dsct))
+			continue;
 
-static u64 freezer_parent_freezing_read(struct cgroup_subsys_state *css,
-					struct cftype *cft)
-{
-	struct freezer *freezer = css_freezer(css);
+		if (freeze) {
+			dsct->freezer.e_freeze++;
+			/*
+			 * Already frozen because of ancestor's settings?
+			 */
+			if (dsct->freezer.e_freeze > 1)
+				continue;
+		} else {
+			dsct->freezer.e_freeze--;
+			/*
+			 * Still frozen because of ancestor's settings?
+			 */
+			if (dsct->freezer.e_freeze > 0)
+				continue;
 
-	return (bool)(freezer->state & CGROUP_FREEZING_PARENT);
-}
+			WARN_ON_ONCE(dsct->freezer.e_freeze < 0);
+		}
+
+		/*
+		 * Do change actual state: freeze or unfreeze.
+		 */
+		cgroup_do_freeze(dsct, freeze);
+		applied = true;
+	}
 
-static struct cftype files[] = {
-	{
-		.name = "state",
-		.flags = CFTYPE_NOT_ON_ROOT,
-		.seq_show = freezer_read,
-		.write = freezer_write,
-	},
-	{
-		.name = "self_freezing",
-		.flags = CFTYPE_NOT_ON_ROOT,
-		.read_u64 = freezer_self_freezing_read,
-	},
-	{
-		.name = "parent_freezing",
-		.flags = CFTYPE_NOT_ON_ROOT,
-		.read_u64 = freezer_parent_freezing_read,
-	},
-	{ }	/* terminate */
-};
-
-struct cgroup_subsys freezer_cgrp_subsys = {
-	.css_alloc	= freezer_css_alloc,
-	.css_online	= freezer_css_online,
-	.css_offline	= freezer_css_offline,
-	.css_free	= freezer_css_free,
-	.attach		= freezer_attach,
-	.fork		= freezer_fork,
-	.legacy_cftypes	= files,
-};
+	/*
+	 * Even if the actual state hasn't changed, let's notify a user.
+	 * The state can be enforced by an ancestor cgroup: the cgroup
+	 * can already be in the desired state or it can be locked in the
+	 * opposite state, so that the transition will never happen.
+	 * In both cases it's better to notify a user, that there is
+	 * nothing to wait for.
+	 */
+	if (!applied) {
+		TRACE_CGROUP_PATH(notify_frozen, cgrp,
+				  test_bit(CGRP_FROZEN, &cgrp->flags));
+		cgroup_file_notify(&cgrp->events_file);
+	}
+}
diff --git a/kernel/cgroup/legacy_freezer.c b/kernel/cgroup/legacy_freezer.c
new file mode 100644
index 000000000000..08236798d173
--- /dev/null
+++ b/kernel/cgroup/legacy_freezer.c
@@ -0,0 +1,481 @@
+/*
+ * cgroup_freezer.c -  control group freezer subsystem
+ *
+ * Copyright IBM Corporation, 2007
+ *
+ * Author : Cedric Le Goater <clg@fr.ibm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2.1 of the GNU Lesser General Public License
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it would be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ */
+
+#include <linux/export.h>
+#include <linux/slab.h>
+#include <linux/cgroup.h>
+#include <linux/fs.h>
+#include <linux/uaccess.h>
+#include <linux/freezer.h>
+#include <linux/seq_file.h>
+#include <linux/mutex.h>
+
+/*
+ * A cgroup is freezing if any FREEZING flags are set.  FREEZING_SELF is
+ * set if "FROZEN" is written to freezer.state cgroupfs file, and cleared
+ * for "THAWED".  FREEZING_PARENT is set if the parent freezer is FREEZING
+ * for whatever reason.  IOW, a cgroup has FREEZING_PARENT set if one of
+ * its ancestors has FREEZING_SELF set.
+ */
+enum freezer_state_flags {
+	CGROUP_FREEZER_ONLINE	= (1 << 0), /* freezer is fully online */
+	CGROUP_FREEZING_SELF	= (1 << 1), /* this freezer is freezing */
+	CGROUP_FREEZING_PARENT	= (1 << 2), /* the parent freezer is freezing */
+	CGROUP_FROZEN		= (1 << 3), /* this and its descendants frozen */
+
+	/* mask for all FREEZING flags */
+	CGROUP_FREEZING		= CGROUP_FREEZING_SELF | CGROUP_FREEZING_PARENT,
+};
+
+struct freezer {
+	struct cgroup_subsys_state	css;
+	unsigned int			state;
+};
+
+static DEFINE_MUTEX(freezer_mutex);
+
+static inline struct freezer *css_freezer(struct cgroup_subsys_state *css)
+{
+	return css ? container_of(css, struct freezer, css) : NULL;
+}
+
+static inline struct freezer *task_freezer(struct task_struct *task)
+{
+	return css_freezer(task_css(task, freezer_cgrp_id));
+}
+
+static struct freezer *parent_freezer(struct freezer *freezer)
+{
+	return css_freezer(freezer->css.parent);
+}
+
+bool cgroup_freezing(struct task_struct *task)
+{
+	bool ret;
+
+	rcu_read_lock();
+	ret = task_freezer(task)->state & CGROUP_FREEZING;
+	rcu_read_unlock();
+
+	return ret;
+}
+
+static const char *freezer_state_strs(unsigned int state)
+{
+	if (state & CGROUP_FROZEN)
+		return "FROZEN";
+	if (state & CGROUP_FREEZING)
+		return "FREEZING";
+	return "THAWED";
+};
+
+static struct cgroup_subsys_state *
+freezer_css_alloc(struct cgroup_subsys_state *parent_css)
+{
+	struct freezer *freezer;
+
+	freezer = kzalloc(sizeof(struct freezer), GFP_KERNEL);
+	if (!freezer)
+		return ERR_PTR(-ENOMEM);
+
+	return &freezer->css;
+}
+
+/**
+ * freezer_css_online - commit creation of a freezer css
+ * @css: css being created
+ *
+ * We're committing to creation of @css.  Mark it online and inherit
+ * parent's freezing state while holding both parent's and our
+ * freezer->lock.
+ */
+static int freezer_css_online(struct cgroup_subsys_state *css)
+{
+	struct freezer *freezer = css_freezer(css);
+	struct freezer *parent = parent_freezer(freezer);
+
+	mutex_lock(&freezer_mutex);
+
+	freezer->state |= CGROUP_FREEZER_ONLINE;
+
+	if (parent && (parent->state & CGROUP_FREEZING)) {
+		freezer->state |= CGROUP_FREEZING_PARENT | CGROUP_FROZEN;
+		atomic_inc(&system_freezing_cnt);
+	}
+
+	mutex_unlock(&freezer_mutex);
+	return 0;
+}
+
+/**
+ * freezer_css_offline - initiate destruction of a freezer css
+ * @css: css being destroyed
+ *
+ * @css is going away.  Mark it dead and decrement system_freezing_count if
+ * it was holding one.
+ */
+static void freezer_css_offline(struct cgroup_subsys_state *css)
+{
+	struct freezer *freezer = css_freezer(css);
+
+	mutex_lock(&freezer_mutex);
+
+	if (freezer->state & CGROUP_FREEZING)
+		atomic_dec(&system_freezing_cnt);
+
+	freezer->state = 0;
+
+	mutex_unlock(&freezer_mutex);
+}
+
+static void freezer_css_free(struct cgroup_subsys_state *css)
+{
+	kfree(css_freezer(css));
+}
+
+/*
+ * Tasks can be migrated into a different freezer anytime regardless of its
+ * current state.  freezer_attach() is responsible for making new tasks
+ * conform to the current state.
+ *
+ * Freezer state changes and task migration are synchronized via
+ * @freezer->lock.  freezer_attach() makes the new tasks conform to the
+ * current state and all following state changes can see the new tasks.
+ */
+static void freezer_attach(struct cgroup_taskset *tset)
+{
+	struct task_struct *task;
+	struct cgroup_subsys_state *new_css;
+
+	mutex_lock(&freezer_mutex);
+
+	/*
+	 * Make the new tasks conform to the current state of @new_css.
+	 * For simplicity, when migrating any task to a FROZEN cgroup, we
+	 * revert it to FREEZING and let update_if_frozen() determine the
+	 * correct state later.
+	 *
+	 * Tasks in @tset are on @new_css but may not conform to its
+	 * current state before executing the following - !frozen tasks may
+	 * be visible in a FROZEN cgroup and frozen tasks in a THAWED one.
+	 */
+	cgroup_taskset_for_each(task, new_css, tset) {
+		struct freezer *freezer = css_freezer(new_css);
+
+		if (!(freezer->state & CGROUP_FREEZING)) {
+			__thaw_task(task);
+		} else {
+			freeze_task(task);
+			/* clear FROZEN and propagate upwards */
+			while (freezer && (freezer->state & CGROUP_FROZEN)) {
+				freezer->state &= ~CGROUP_FROZEN;
+				freezer = parent_freezer(freezer);
+			}
+		}
+	}
+
+	mutex_unlock(&freezer_mutex);
+}
+
+/**
+ * freezer_fork - cgroup post fork callback
+ * @task: a task which has just been forked
+ *
+ * @task has just been created and should conform to the current state of
+ * the cgroup_freezer it belongs to.  This function may race against
+ * freezer_attach().  Losing to freezer_attach() means that we don't have
+ * to do anything as freezer_attach() will put @task into the appropriate
+ * state.
+ */
+static void freezer_fork(struct task_struct *task)
+{
+	struct freezer *freezer;
+
+	/*
+	 * The root cgroup is non-freezable, so we can skip locking the
+	 * freezer.  This is safe regardless of race with task migration.
+	 * If we didn't race or won, skipping is obviously the right thing
+	 * to do.  If we lost and root is the new cgroup, noop is still the
+	 * right thing to do.
+	 */
+	if (task_css_is_root(task, freezer_cgrp_id))
+		return;
+
+	mutex_lock(&freezer_mutex);
+	rcu_read_lock();
+
+	freezer = task_freezer(task);
+	if (freezer->state & CGROUP_FREEZING)
+		freeze_task(task);
+
+	rcu_read_unlock();
+	mutex_unlock(&freezer_mutex);
+}
+
+/**
+ * update_if_frozen - update whether a cgroup finished freezing
+ * @css: css of interest
+ *
+ * Once FREEZING is initiated, transition to FROZEN is lazily updated by
+ * calling this function.  If the current state is FREEZING but not FROZEN,
+ * this function checks whether all tasks of this cgroup and the descendant
+ * cgroups finished freezing and, if so, sets FROZEN.
+ *
+ * The caller is responsible for grabbing RCU read lock and calling
+ * update_if_frozen() on all descendants prior to invoking this function.
+ *
+ * Task states and freezer state might disagree while tasks are being
+ * migrated into or out of @css, so we can't verify task states against
+ * @freezer state here.  See freezer_attach() for details.
+ */
+static void update_if_frozen(struct cgroup_subsys_state *css)
+{
+	struct freezer *freezer = css_freezer(css);
+	struct cgroup_subsys_state *pos;
+	struct css_task_iter it;
+	struct task_struct *task;
+
+	lockdep_assert_held(&freezer_mutex);
+
+	if (!(freezer->state & CGROUP_FREEZING) ||
+	    (freezer->state & CGROUP_FROZEN))
+		return;
+
+	/* are all (live) children frozen? */
+	rcu_read_lock();
+	css_for_each_child(pos, css) {
+		struct freezer *child = css_freezer(pos);
+
+		if ((child->state & CGROUP_FREEZER_ONLINE) &&
+		    !(child->state & CGROUP_FROZEN)) {
+			rcu_read_unlock();
+			return;
+		}
+	}
+	rcu_read_unlock();
+
+	/* are all tasks frozen? */
+	css_task_iter_start(css, 0, &it);
+
+	while ((task = css_task_iter_next(&it))) {
+		if (freezing(task)) {
+			/*
+			 * freezer_should_skip() indicates that the task
+			 * should be skipped when determining freezing
+			 * completion.  Consider it frozen in addition to
+			 * the usual frozen condition.
+			 */
+			if (!frozen(task) && !freezer_should_skip(task))
+				goto out_iter_end;
+		}
+	}
+
+	freezer->state |= CGROUP_FROZEN;
+out_iter_end:
+	css_task_iter_end(&it);
+}
+
+static int freezer_read(struct seq_file *m, void *v)
+{
+	struct cgroup_subsys_state *css = seq_css(m), *pos;
+
+	mutex_lock(&freezer_mutex);
+	rcu_read_lock();
+
+	/* update states bottom-up */
+	css_for_each_descendant_post(pos, css) {
+		if (!css_tryget_online(pos))
+			continue;
+		rcu_read_unlock();
+
+		update_if_frozen(pos);
+
+		rcu_read_lock();
+		css_put(pos);
+	}
+
+	rcu_read_unlock();
+	mutex_unlock(&freezer_mutex);
+
+	seq_puts(m, freezer_state_strs(css_freezer(css)->state));
+	seq_putc(m, '\n');
+	return 0;
+}
+
+static void freeze_cgroup(struct freezer *freezer)
+{
+	struct css_task_iter it;
+	struct task_struct *task;
+
+	css_task_iter_start(&freezer->css, 0, &it);
+	while ((task = css_task_iter_next(&it)))
+		freeze_task(task);
+	css_task_iter_end(&it);
+}
+
+static void unfreeze_cgroup(struct freezer *freezer)
+{
+	struct css_task_iter it;
+	struct task_struct *task;
+
+	css_task_iter_start(&freezer->css, 0, &it);
+	while ((task = css_task_iter_next(&it)))
+		__thaw_task(task);
+	css_task_iter_end(&it);
+}
+
+/**
+ * freezer_apply_state - apply state change to a single cgroup_freezer
+ * @freezer: freezer to apply state change to
+ * @freeze: whether to freeze or unfreeze
+ * @state: CGROUP_FREEZING_* flag to set or clear
+ *
+ * Set or clear @state on @cgroup according to @freeze, and perform
+ * freezing or thawing as necessary.
+ */
+static void freezer_apply_state(struct freezer *freezer, bool freeze,
+				unsigned int state)
+{
+	/* also synchronizes against task migration, see freezer_attach() */
+	lockdep_assert_held(&freezer_mutex);
+
+	if (!(freezer->state & CGROUP_FREEZER_ONLINE))
+		return;
+
+	if (freeze) {
+		if (!(freezer->state & CGROUP_FREEZING))
+			atomic_inc(&system_freezing_cnt);
+		freezer->state |= state;
+		freeze_cgroup(freezer);
+	} else {
+		bool was_freezing = freezer->state & CGROUP_FREEZING;
+
+		freezer->state &= ~state;
+
+		if (!(freezer->state & CGROUP_FREEZING)) {
+			if (was_freezing)
+				atomic_dec(&system_freezing_cnt);
+			freezer->state &= ~CGROUP_FROZEN;
+			unfreeze_cgroup(freezer);
+		}
+	}
+}
+
+/**
+ * freezer_change_state - change the freezing state of a cgroup_freezer
+ * @freezer: freezer of interest
+ * @freeze: whether to freeze or thaw
+ *
+ * Freeze or thaw @freezer according to @freeze.  The operations are
+ * recursive - all descendants of @freezer will be affected.
+ */
+static void freezer_change_state(struct freezer *freezer, bool freeze)
+{
+	struct cgroup_subsys_state *pos;
+
+	/*
+	 * Update all its descendants in pre-order traversal.  Each
+	 * descendant will try to inherit its parent's FREEZING state as
+	 * CGROUP_FREEZING_PARENT.
+	 */
+	mutex_lock(&freezer_mutex);
+	rcu_read_lock();
+	css_for_each_descendant_pre(pos, &freezer->css) {
+		struct freezer *pos_f = css_freezer(pos);
+		struct freezer *parent = parent_freezer(pos_f);
+
+		if (!css_tryget_online(pos))
+			continue;
+		rcu_read_unlock();
+
+		if (pos_f == freezer)
+			freezer_apply_state(pos_f, freeze,
+					    CGROUP_FREEZING_SELF);
+		else
+			freezer_apply_state(pos_f,
+					    parent->state & CGROUP_FREEZING,
+					    CGROUP_FREEZING_PARENT);
+
+		rcu_read_lock();
+		css_put(pos);
+	}
+	rcu_read_unlock();
+	mutex_unlock(&freezer_mutex);
+}
+
+static ssize_t freezer_write(struct kernfs_open_file *of,
+			     char *buf, size_t nbytes, loff_t off)
+{
+	bool freeze;
+
+	buf = strstrip(buf);
+
+	if (strcmp(buf, freezer_state_strs(0)) == 0)
+		freeze = false;
+	else if (strcmp(buf, freezer_state_strs(CGROUP_FROZEN)) == 0)
+		freeze = true;
+	else
+		return -EINVAL;
+
+	freezer_change_state(css_freezer(of_css(of)), freeze);
+	return nbytes;
+}
+
+static u64 freezer_self_freezing_read(struct cgroup_subsys_state *css,
+				      struct cftype *cft)
+{
+	struct freezer *freezer = css_freezer(css);
+
+	return (bool)(freezer->state & CGROUP_FREEZING_SELF);
+}
+
+static u64 freezer_parent_freezing_read(struct cgroup_subsys_state *css,
+					struct cftype *cft)
+{
+	struct freezer *freezer = css_freezer(css);
+
+	return (bool)(freezer->state & CGROUP_FREEZING_PARENT);
+}
+
+static struct cftype files[] = {
+	{
+		.name = "state",
+		.flags = CFTYPE_NOT_ON_ROOT,
+		.seq_show = freezer_read,
+		.write = freezer_write,
+	},
+	{
+		.name = "self_freezing",
+		.flags = CFTYPE_NOT_ON_ROOT,
+		.read_u64 = freezer_self_freezing_read,
+	},
+	{
+		.name = "parent_freezing",
+		.flags = CFTYPE_NOT_ON_ROOT,
+		.read_u64 = freezer_parent_freezing_read,
+	},
+	{ }	/* terminate */
+};
+
+struct cgroup_subsys freezer_cgrp_subsys = {
+	.css_alloc	= freezer_css_alloc,
+	.css_online	= freezer_css_online,
+	.css_offline	= freezer_css_offline,
+	.css_free	= freezer_css_free,
+	.attach		= freezer_attach,
+	.fork		= freezer_fork,
+	.legacy_cftypes	= files,
+};
diff --git a/kernel/dma/Kconfig b/kernel/dma/Kconfig
index a06ba3013b3b..83d711f8d665 100644
--- a/kernel/dma/Kconfig
+++ b/kernel/dma/Kconfig
@@ -38,6 +38,9 @@ config ARCH_HAS_SYNC_DMA_FOR_CPU
 config ARCH_HAS_SYNC_DMA_FOR_CPU_ALL
 	bool
 
+config ARCH_HAS_DMA_PREP_COHERENT
+	bool
+
 config ARCH_HAS_DMA_COHERENT_TO_PFN
 	bool
 
@@ -57,6 +60,7 @@ config SWIOTLB
 
 config DMA_REMAP
 	depends on MMU
+	select GENERIC_ALLOCATOR
 	bool
 
 config DMA_DIRECT_REMAP
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index fcdb23e8d2fc..2c2772e9702a 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -311,7 +311,7 @@ static inline bool dma_direct_possible(struct device *dev, dma_addr_t dma_addr,
 		size_t size)
 {
 	return swiotlb_force != SWIOTLB_FORCE &&
-		(!dev || dma_capable(dev, dma_addr, size));
+		dma_capable(dev, dma_addr, size);
 }
 
 dma_addr_t dma_direct_map_page(struct device *dev, struct page *page,
diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c
index c000906348c9..f7afdadb6770 100644
--- a/kernel/dma/mapping.c
+++ b/kernel/dma/mapping.c
@@ -238,17 +238,13 @@ u64 dma_get_required_mask(struct device *dev)
 }
 EXPORT_SYMBOL_GPL(dma_get_required_mask);
 
-#ifndef arch_dma_alloc_attrs
-#define arch_dma_alloc_attrs(dev)	(true)
-#endif
-
 void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
 		gfp_t flag, unsigned long attrs)
 {
 	const struct dma_map_ops *ops = get_dma_ops(dev);
 	void *cpu_addr;
 
-	WARN_ON_ONCE(dev && !dev->coherent_dma_mask);
+	WARN_ON_ONCE(!dev->coherent_dma_mask);
 
 	if (dma_alloc_from_dev_coherent(dev, size, dma_handle, &cpu_addr))
 		return cpu_addr;
@@ -256,9 +252,6 @@ void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
 	/* let the implementation decide on the zone to allocate from: */
 	flag &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM);
 
-	if (!arch_dma_alloc_attrs(&dev))
-		return NULL;
-
 	if (dma_is_direct(ops))
 		cpu_addr = dma_direct_alloc(dev, size, dma_handle, flag, attrs);
 	else if (ops->alloc)
diff --git a/kernel/fork.c b/kernel/fork.c
index 8b03d93ba068..5359facf9867 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -1225,7 +1225,9 @@ static int wait_for_vfork_done(struct task_struct *child,
 	int killed;
 
 	freezer_do_not_count();
+	cgroup_enter_frozen();
 	killed = wait_for_completion_killable(vfork);
+	cgroup_leave_frozen(false);
 	freezer_count();
 
 	if (killed) {
diff --git a/kernel/module.c b/kernel/module.c
index a9020bdd4cf6..a9e1e7f2c224 100644
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -290,6 +290,11 @@ bool is_module_sig_enforced(void)
 }
 EXPORT_SYMBOL(is_module_sig_enforced);
 
+void set_module_sig_enforced(void)
+{
+	sig_enforce = true;
+}
+
 /* Block module loading/unloading? */
 int modules_disabled = 0;
 core_param(nomodule, modules_disabled, bint, 0);
diff --git a/kernel/signal.c b/kernel/signal.c
index cd83cc376767..62f9aea4a15a 100644
--- a/kernel/signal.c
+++ b/kernel/signal.c
@@ -43,6 +43,7 @@
 #include <linux/compiler.h>
 #include <linux/posix-timers.h>
 #include <linux/livepatch.h>
+#include <linux/cgroup.h>
 
 #define CREATE_TRACE_POINTS
 #include <trace/events/signal.h>
@@ -146,9 +147,10 @@ static inline bool has_pending_signals(sigset_t *signal, sigset_t *blocked)
 
 static bool recalc_sigpending_tsk(struct task_struct *t)
 {
-	if ((t->jobctl & JOBCTL_PENDING_MASK) ||
+	if ((t->jobctl & (JOBCTL_PENDING_MASK | JOBCTL_TRAP_FREEZE)) ||
 	    PENDING(&t->pending, &t->blocked) ||
-	    PENDING(&t->signal->shared_pending, &t->blocked)) {
+	    PENDING(&t->signal->shared_pending, &t->blocked) ||
+	    cgroup_task_frozen(t)) {
 		set_tsk_thread_flag(t, TIF_SIGPENDING);
 		return true;
 	}
@@ -2108,6 +2110,7 @@ static void ptrace_stop(int exit_code, int why, int clear_code, kernel_siginfo_t
 		preempt_disable();
 		read_unlock(&tasklist_lock);
 		preempt_enable_no_resched();
+		cgroup_enter_frozen();
 		freezable_schedule();
 	} else {
 		/*
@@ -2286,6 +2289,7 @@ static bool do_signal_stop(int signr)
 		}
 
 		/* Now we don't run again until woken by SIGCONT or SIGKILL */
+		cgroup_enter_frozen();
 		freezable_schedule();
 		return true;
 	} else {
@@ -2332,6 +2336,43 @@ static void do_jobctl_trap(void)
 	}
 }
 
+/**
+ * do_freezer_trap - handle the freezer jobctl trap
+ *
+ * Puts the task into frozen state, if only the task is not about to quit.
+ * In this case it drops JOBCTL_TRAP_FREEZE.
+ *
+ * CONTEXT:
+ * Must be called with @current->sighand->siglock held,
+ * which is always released before returning.
+ */
+static void do_freezer_trap(void)
+	__releases(&current->sighand->siglock)
+{
+	/*
+	 * If there are other trap bits pending except JOBCTL_TRAP_FREEZE,
+	 * let's make another loop to give it a chance to be handled.
+	 * In any case, we'll return back.
+	 */
+	if ((current->jobctl & (JOBCTL_PENDING_MASK | JOBCTL_TRAP_FREEZE)) !=
+	     JOBCTL_TRAP_FREEZE) {
+		spin_unlock_irq(&current->sighand->siglock);
+		return;
+	}
+
+	/*
+	 * Now we're sure that there is no pending fatal signal and no
+	 * pending traps. Clear TIF_SIGPENDING to not get out of schedule()
+	 * immediately (if there is a non-fatal signal pending), and
+	 * put the task into sleep.
+	 */
+	__set_current_state(TASK_INTERRUPTIBLE);
+	clear_thread_flag(TIF_SIGPENDING);
+	spin_unlock_irq(&current->sighand->siglock);
+	cgroup_enter_frozen();
+	freezable_schedule();
+}
+
 static int ptrace_signal(int signr, kernel_siginfo_t *info)
 {
 	/*
@@ -2452,9 +2493,24 @@ relock:
 		    do_signal_stop(0))
 			goto relock;
 
-		if (unlikely(current->jobctl & JOBCTL_TRAP_MASK)) {
-			do_jobctl_trap();
+		if (unlikely(current->jobctl &
+			     (JOBCTL_TRAP_MASK | JOBCTL_TRAP_FREEZE))) {
+			if (current->jobctl & JOBCTL_TRAP_MASK) {
+				do_jobctl_trap();
+				spin_unlock_irq(&sighand->siglock);
+			} else if (current->jobctl & JOBCTL_TRAP_FREEZE)
+				do_freezer_trap();
+
+			goto relock;
+		}
+
+		/*
+		 * If the task is leaving the frozen state, let's update
+		 * cgroup counters and reset the frozen bit.
+		 */
+		if (unlikely(cgroup_task_frozen(current))) {
 			spin_unlock_irq(&sighand->siglock);
+			cgroup_leave_frozen(false);
 			goto relock;
 		}
 
@@ -2550,6 +2606,8 @@ relock:
 
 	fatal:
 		spin_unlock_irq(&sighand->siglock);
+		if (unlikely(cgroup_task_frozen(current)))
+			cgroup_leave_frozen(true);
 
 		/*
 		 * Anything else is fatal, maybe with a core dump.
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index faf7622246da..9657315405de 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -127,16 +127,16 @@ enum {
  *
  * PL: wq_pool_mutex protected.
  *
- * PR: wq_pool_mutex protected for writes.  Sched-RCU protected for reads.
+ * PR: wq_pool_mutex protected for writes.  RCU protected for reads.
  *
  * PW: wq_pool_mutex and wq->mutex protected for writes.  Either for reads.
  *
  * PWR: wq_pool_mutex and wq->mutex protected for writes.  Either or
- *      sched-RCU for reads.
+ *      RCU for reads.
  *
  * WQ: wq->mutex protected.
  *
- * WR: wq->mutex protected for writes.  Sched-RCU protected for reads.
+ * WR: wq->mutex protected for writes.  RCU protected for reads.
  *
  * MD: wq_mayday_lock protected.
  */
@@ -183,7 +183,7 @@ struct worker_pool {
 	atomic_t		nr_running ____cacheline_aligned_in_smp;
 
 	/*
-	 * Destruction of pool is sched-RCU protected to allow dereferences
+	 * Destruction of pool is RCU protected to allow dereferences
 	 * from get_work_pool().
 	 */
 	struct rcu_head		rcu;
@@ -212,7 +212,7 @@ struct pool_workqueue {
 	/*
 	 * Release of unbound pwq is punted to system_wq.  See put_pwq()
 	 * and pwq_unbound_release_workfn() for details.  pool_workqueue
-	 * itself is also sched-RCU protected so that the first pwq can be
+	 * itself is also RCU protected so that the first pwq can be
 	 * determined without grabbing wq->mutex.
 	 */
 	struct work_struct	unbound_release_work;
@@ -266,8 +266,8 @@ struct workqueue_struct {
 	char			name[WQ_NAME_LEN]; /* I: workqueue name */
 
 	/*
-	 * Destruction of workqueue_struct is sched-RCU protected to allow
-	 * walking the workqueues list without grabbing wq_pool_mutex.
+	 * Destruction of workqueue_struct is RCU protected to allow walking
+	 * the workqueues list without grabbing wq_pool_mutex.
 	 * This is used to dump all workqueues from sysrq.
 	 */
 	struct rcu_head		rcu;
@@ -359,20 +359,20 @@ static void workqueue_sysfs_unregister(struct workqueue_struct *wq);
 #include <trace/events/workqueue.h>
 
 #define assert_rcu_or_pool_mutex()					\
-	RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held() &&			\
+	RCU_LOCKDEP_WARN(!rcu_read_lock_held() &&			\
 			 !lockdep_is_held(&wq_pool_mutex),		\
-			 "sched RCU or wq_pool_mutex should be held")
+			 "RCU or wq_pool_mutex should be held")
 
 #define assert_rcu_or_wq_mutex(wq)					\
-	RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held() &&			\
+	RCU_LOCKDEP_WARN(!rcu_read_lock_held() &&			\
 			 !lockdep_is_held(&wq->mutex),			\
-			 "sched RCU or wq->mutex should be held")
+			 "RCU or wq->mutex should be held")
 
 #define assert_rcu_or_wq_mutex_or_pool_mutex(wq)			\
-	RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held() &&			\
+	RCU_LOCKDEP_WARN(!rcu_read_lock_held() &&			\
 			 !lockdep_is_held(&wq->mutex) &&		\
 			 !lockdep_is_held(&wq_pool_mutex),		\
-			 "sched RCU, wq->mutex or wq_pool_mutex should be held")
+			 "RCU, wq->mutex or wq_pool_mutex should be held")
 
 #define for_each_cpu_worker_pool(pool, cpu)				\
 	for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0];		\
@@ -384,7 +384,7 @@ static void workqueue_sysfs_unregister(struct workqueue_struct *wq);
  * @pool: iteration cursor
  * @pi: integer used for iteration
  *
- * This must be called either with wq_pool_mutex held or sched RCU read
+ * This must be called either with wq_pool_mutex held or RCU read
  * locked.  If the pool needs to be used beyond the locking in effect, the
  * caller is responsible for guaranteeing that the pool stays online.
  *
@@ -416,7 +416,7 @@ static void workqueue_sysfs_unregister(struct workqueue_struct *wq);
  * @pwq: iteration cursor
  * @wq: the target workqueue
  *
- * This must be called either with wq->mutex held or sched RCU read locked.
+ * This must be called either with wq->mutex held or RCU read locked.
  * If the pwq needs to be used beyond the locking in effect, the caller is
  * responsible for guaranteeing that the pwq stays online.
  *
@@ -552,7 +552,7 @@ static int worker_pool_assign_id(struct worker_pool *pool)
  * @wq: the target workqueue
  * @node: the node ID
  *
- * This must be called with any of wq_pool_mutex, wq->mutex or sched RCU
+ * This must be called with any of wq_pool_mutex, wq->mutex or RCU
  * read locked.
  * If the pwq needs to be used beyond the locking in effect, the caller is
  * responsible for guaranteeing that the pwq stays online.
@@ -696,8 +696,8 @@ static struct pool_workqueue *get_work_pwq(struct work_struct *work)
  * @work: the work item of interest
  *
  * Pools are created and destroyed under wq_pool_mutex, and allows read
- * access under sched-RCU read lock.  As such, this function should be
- * called under wq_pool_mutex or with preemption disabled.
+ * access under RCU read lock.  As such, this function should be
+ * called under wq_pool_mutex or inside of a rcu_read_lock() region.
  *
  * All fields of the returned pool are accessible as long as the above
  * mentioned locking is in effect.  If the returned pool needs to be used
@@ -907,6 +907,7 @@ void wq_worker_sleeping(struct task_struct *task)
 
 /**
  * wq_worker_last_func - retrieve worker's last work function
+ * @task: Task to retrieve last work function of.
  *
  * Determine the last function a worker executed. This is called from
  * the scheduler to get a worker's last known identity.
@@ -1126,7 +1127,7 @@ static void put_pwq_unlocked(struct pool_workqueue *pwq)
 {
 	if (pwq) {
 		/*
-		 * As both pwqs and pools are sched-RCU protected, the
+		 * As both pwqs and pools are RCU protected, the
 		 * following lock operations are safe.
 		 */
 		spin_lock_irq(&pwq->pool->lock);
@@ -1254,6 +1255,7 @@ static int try_to_grab_pending(struct work_struct *work, bool is_dwork,
 	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)))
 		return 0;
 
+	rcu_read_lock();
 	/*
 	 * The queueing is in progress, or it is already queued. Try to
 	 * steal it from ->worklist without clearing WORK_STRUCT_PENDING.
@@ -1292,10 +1294,12 @@ static int try_to_grab_pending(struct work_struct *work, bool is_dwork,
 		set_work_pool_and_keep_pending(work, pool->id);
 
 		spin_unlock(&pool->lock);
+		rcu_read_unlock();
 		return 1;
 	}
 	spin_unlock(&pool->lock);
 fail:
+	rcu_read_unlock();
 	local_irq_restore(*flags);
 	if (work_is_canceling(work))
 		return -ENOENT;
@@ -1409,6 +1413,7 @@ static void __queue_work(int cpu, struct workqueue_struct *wq,
 	if (unlikely(wq->flags & __WQ_DRAINING) &&
 	    WARN_ON_ONCE(!is_chained_work(wq)))
 		return;
+	rcu_read_lock();
 retry:
 	if (req_cpu == WORK_CPU_UNBOUND)
 		cpu = wq_select_unbound_cpu(raw_smp_processor_id());
@@ -1465,10 +1470,8 @@ retry:
 	/* pwq determined, queue */
 	trace_workqueue_queue_work(req_cpu, pwq, work);
 
-	if (WARN_ON(!list_empty(&work->entry))) {
-		spin_unlock(&pwq->pool->lock);
-		return;
-	}
+	if (WARN_ON(!list_empty(&work->entry)))
+		goto out;
 
 	pwq->nr_in_flight[pwq->work_color]++;
 	work_flags = work_color_to_flags(pwq->work_color);
@@ -1486,7 +1489,9 @@ retry:
 
 	insert_work(pwq, work, worklist, work_flags);
 
+out:
 	spin_unlock(&pwq->pool->lock);
+	rcu_read_unlock();
 }
 
 /**
@@ -2968,14 +2973,14 @@ static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr,
 
 	might_sleep();
 
-	local_irq_disable();
+	rcu_read_lock();
 	pool = get_work_pool(work);
 	if (!pool) {
-		local_irq_enable();
+		rcu_read_unlock();
 		return false;
 	}
 
-	spin_lock(&pool->lock);
+	spin_lock_irq(&pool->lock);
 	/* see the comment in try_to_grab_pending() with the same code */
 	pwq = get_work_pwq(work);
 	if (pwq) {
@@ -3007,10 +3012,11 @@ static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr,
 		lock_map_acquire(&pwq->wq->lockdep_map);
 		lock_map_release(&pwq->wq->lockdep_map);
 	}
-
+	rcu_read_unlock();
 	return true;
 already_gone:
 	spin_unlock_irq(&pool->lock);
+	rcu_read_unlock();
 	return false;
 }
 
@@ -3497,7 +3503,7 @@ static void rcu_free_pool(struct rcu_head *rcu)
  * put_unbound_pool - put a worker_pool
  * @pool: worker_pool to put
  *
- * Put @pool.  If its refcnt reaches zero, it gets destroyed in sched-RCU
+ * Put @pool.  If its refcnt reaches zero, it gets destroyed in RCU
  * safe manner.  get_unbound_pool() calls this function on its failure path
  * and this function should be able to release pools which went through,
  * successfully or not, init_worker_pool().
@@ -3551,7 +3557,7 @@ static void put_unbound_pool(struct worker_pool *pool)
 	del_timer_sync(&pool->idle_timer);
 	del_timer_sync(&pool->mayday_timer);
 
-	/* sched-RCU protected to allow dereferences from get_work_pool() */
+	/* RCU protected to allow dereferences from get_work_pool() */
 	call_rcu(&pool->rcu, rcu_free_pool);
 }
 
@@ -4202,6 +4208,7 @@ static int init_rescuer(struct workqueue_struct *wq)
 	return 0;
 }
 
+__printf(1, 4)
 struct workqueue_struct *alloc_workqueue(const char *fmt,
 					 unsigned int flags,
 					 int max_active, ...)
@@ -4465,7 +4472,8 @@ bool workqueue_congested(int cpu, struct workqueue_struct *wq)
 	struct pool_workqueue *pwq;
 	bool ret;
 
-	rcu_read_lock_sched();
+	rcu_read_lock();
+	preempt_disable();
 
 	if (cpu == WORK_CPU_UNBOUND)
 		cpu = smp_processor_id();
@@ -4476,7 +4484,8 @@ bool workqueue_congested(int cpu, struct workqueue_struct *wq)
 		pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
 
 	ret = !list_empty(&pwq->delayed_works);
-	rcu_read_unlock_sched();
+	preempt_enable();
+	rcu_read_unlock();
 
 	return ret;
 }
@@ -4502,15 +4511,15 @@ unsigned int work_busy(struct work_struct *work)
 	if (work_pending(work))
 		ret |= WORK_BUSY_PENDING;
 
-	local_irq_save(flags);
+	rcu_read_lock();
 	pool = get_work_pool(work);
 	if (pool) {
-		spin_lock(&pool->lock);
+		spin_lock_irqsave(&pool->lock, flags);
 		if (find_worker_executing_work(pool, work))
 			ret |= WORK_BUSY_RUNNING;
-		spin_unlock(&pool->lock);
+		spin_unlock_irqrestore(&pool->lock, flags);
 	}
-	local_irq_restore(flags);
+	rcu_read_unlock();
 
 	return ret;
 }
@@ -4694,7 +4703,7 @@ void show_workqueue_state(void)
 	unsigned long flags;
 	int pi;
 
-	rcu_read_lock_sched();
+	rcu_read_lock();
 
 	pr_info("Showing busy workqueues and worker pools:\n");
 
@@ -4759,7 +4768,7 @@ void show_workqueue_state(void)
 		touch_nmi_watchdog();
 	}
 
-	rcu_read_unlock_sched();
+	rcu_read_unlock();
 }
 
 /* used to show worker information through /proc/PID/{comm,stat,status} */
@@ -5146,16 +5155,16 @@ bool freeze_workqueues_busy(void)
 		 * nr_active is monotonically decreasing.  It's safe
 		 * to peek without lock.
 		 */
-		rcu_read_lock_sched();
+		rcu_read_lock();
 		for_each_pwq(pwq, wq) {
 			WARN_ON_ONCE(pwq->nr_active < 0);
 			if (pwq->nr_active) {
 				busy = true;
-				rcu_read_unlock_sched();
+				rcu_read_unlock();
 				goto out_unlock;
 			}
 		}
-		rcu_read_unlock_sched();
+		rcu_read_unlock();
 	}
 out_unlock:
 	mutex_unlock(&wq_pool_mutex);
@@ -5350,7 +5359,8 @@ static ssize_t wq_pool_ids_show(struct device *dev,
 	const char *delim = "";
 	int node, written = 0;
 
-	rcu_read_lock_sched();
+	get_online_cpus();
+	rcu_read_lock();
 	for_each_node(node) {
 		written += scnprintf(buf + written, PAGE_SIZE - written,
 				     "%s%d:%d", delim, node,
@@ -5358,7 +5368,8 @@ static ssize_t wq_pool_ids_show(struct device *dev,
 		delim = " ";
 	}
 	written += scnprintf(buf + written, PAGE_SIZE - written, "\n");
-	rcu_read_unlock_sched();
+	rcu_read_unlock();
+	put_online_cpus();
 
 	return written;
 }
diff --git a/lib/dynamic_debug.c b/lib/dynamic_debug.c
index 7bdf98c37e91..8a16c2d498e9 100644
--- a/lib/dynamic_debug.c
+++ b/lib/dynamic_debug.c
@@ -37,6 +37,8 @@
 #include <linux/device.h>
 #include <linux/netdevice.h>
 
+#include <rdma/ib_verbs.h>
+
 extern struct _ddebug __start___verbose[];
 extern struct _ddebug __stop___verbose[];
 
@@ -636,6 +638,41 @@ EXPORT_SYMBOL(__dynamic_netdev_dbg);
 
 #endif
 
+#if IS_ENABLED(CONFIG_INFINIBAND)
+
+void __dynamic_ibdev_dbg(struct _ddebug *descriptor,
+			 const struct ib_device *ibdev, const char *fmt, ...)
+{
+	struct va_format vaf;
+	va_list args;
+
+	va_start(args, fmt);
+
+	vaf.fmt = fmt;
+	vaf.va = &args;
+
+	if (ibdev && ibdev->dev.parent) {
+		char buf[PREFIX_SIZE];
+
+		dev_printk_emit(LOGLEVEL_DEBUG, ibdev->dev.parent,
+				"%s%s %s %s: %pV",
+				dynamic_emit_prefix(descriptor, buf),
+				dev_driver_string(ibdev->dev.parent),
+				dev_name(ibdev->dev.parent),
+				dev_name(&ibdev->dev),
+				&vaf);
+	} else if (ibdev) {
+		printk(KERN_DEBUG "%s: %pV", dev_name(&ibdev->dev), &vaf);
+	} else {
+		printk(KERN_DEBUG "(NULL ib_device): %pV", &vaf);
+	}
+
+	va_end(args);
+}
+EXPORT_SYMBOL(__dynamic_ibdev_dbg);
+
+#endif
+
 #define DDEBUG_STRING_SIZE 1024
 static __initdata char ddebug_setup_string[DDEBUG_STRING_SIZE];
 
diff --git a/net/8021q/vlan_dev.c b/net/8021q/vlan_dev.c
index f044ae56a313..2a9a60733594 100644
--- a/net/8021q/vlan_dev.c
+++ b/net/8021q/vlan_dev.c
@@ -370,10 +370,12 @@ static int vlan_dev_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
 	ifrr.ifr_ifru = ifr->ifr_ifru;
 
 	switch (cmd) {
+	case SIOCSHWTSTAMP:
+		if (!net_eq(dev_net(dev), &init_net))
+			break;
 	case SIOCGMIIPHY:
 	case SIOCGMIIREG:
 	case SIOCSMIIREG:
-	case SIOCSHWTSTAMP:
 	case SIOCGHWTSTAMP:
 		if (netif_device_present(real_dev) && ops->ndo_do_ioctl)
 			err = ops->ndo_do_ioctl(real_dev, &ifrr, cmd);
diff --git a/net/batman-adv/main.c b/net/batman-adv/main.c
index dabcaff87e34..4a89177def64 100644
--- a/net/batman-adv/main.c
+++ b/net/batman-adv/main.c
@@ -168,6 +168,7 @@ int batadv_mesh_init(struct net_device *soft_iface)
 	spin_lock_init(&bat_priv->tt.commit_lock);
 	spin_lock_init(&bat_priv->gw.list_lock);
 #ifdef CONFIG_BATMAN_ADV_MCAST
+	spin_lock_init(&bat_priv->mcast.mla_lock);
 	spin_lock_init(&bat_priv->mcast.want_lists_lock);
 #endif
 	spin_lock_init(&bat_priv->tvlv.container_list_lock);
diff --git a/net/batman-adv/main.h b/net/batman-adv/main.h
index f827e441025f..c59afcba31e0 100644
--- a/net/batman-adv/main.h
+++ b/net/batman-adv/main.h
@@ -13,7 +13,7 @@
 #define BATADV_DRIVER_DEVICE "batman-adv"
 
 #ifndef BATADV_SOURCE_VERSION
-#define BATADV_SOURCE_VERSION "2019.1"
+#define BATADV_SOURCE_VERSION "2019.2"
 #endif
 
 /* B.A.T.M.A.N. parameters */
diff --git a/net/batman-adv/multicast.c b/net/batman-adv/multicast.c
index 3feb9435b715..ec54e236e345 100644
--- a/net/batman-adv/multicast.c
+++ b/net/batman-adv/multicast.c
@@ -314,8 +314,6 @@ static void batadv_mcast_mla_list_free(struct hlist_head *mcast_list)
  * translation table except the ones listed in the given mcast_list.
  *
  * If mcast_list is NULL then all are retracted.
- *
- * Do not call outside of the mcast worker! (or cancel mcast worker first)
  */
 static void batadv_mcast_mla_tt_retract(struct batadv_priv *bat_priv,
 					struct hlist_head *mcast_list)
@@ -323,8 +321,6 @@ static void batadv_mcast_mla_tt_retract(struct batadv_priv *bat_priv,
 	struct batadv_hw_addr *mcast_entry;
 	struct hlist_node *tmp;
 
-	WARN_ON(delayed_work_pending(&bat_priv->mcast.work));
-
 	hlist_for_each_entry_safe(mcast_entry, tmp, &bat_priv->mcast.mla_list,
 				  list) {
 		if (mcast_list &&
@@ -348,8 +344,6 @@ static void batadv_mcast_mla_tt_retract(struct batadv_priv *bat_priv,
  *
  * Adds multicast listener announcements from the given mcast_list to the
  * translation table if they have not been added yet.
- *
- * Do not call outside of the mcast worker! (or cancel mcast worker first)
  */
 static void batadv_mcast_mla_tt_add(struct batadv_priv *bat_priv,
 				    struct hlist_head *mcast_list)
@@ -357,8 +351,6 @@ static void batadv_mcast_mla_tt_add(struct batadv_priv *bat_priv,
 	struct batadv_hw_addr *mcast_entry;
 	struct hlist_node *tmp;
 
-	WARN_ON(delayed_work_pending(&bat_priv->mcast.work));
-
 	if (!mcast_list)
 		return;
 
@@ -647,7 +639,10 @@ static void batadv_mcast_mla_update(struct work_struct *work)
 	priv_mcast = container_of(delayed_work, struct batadv_priv_mcast, work);
 	bat_priv = container_of(priv_mcast, struct batadv_priv, mcast);
 
+	spin_lock(&bat_priv->mcast.mla_lock);
 	__batadv_mcast_mla_update(bat_priv);
+	spin_unlock(&bat_priv->mcast.mla_lock);
+
 	batadv_mcast_start_timer(bat_priv);
 }
 
diff --git a/net/batman-adv/types.h b/net/batman-adv/types.h
index 357ca119329a..74b644738a36 100644
--- a/net/batman-adv/types.h
+++ b/net/batman-adv/types.h
@@ -1212,6 +1212,11 @@ struct batadv_priv_mcast {
 	unsigned char bridged:1;
 
 	/**
+	 * @mla_lock: a lock protecting mla_list and mla_flags
+	 */
+	spinlock_t mla_lock;
+
+	/**
 	 * @num_want_all_unsnoopables: number of nodes wanting unsnoopable IP
 	 *  traffic
 	 */
diff --git a/net/core/fib_rules.c b/net/core/fib_rules.c
index 18f8dd8329ed..43f0115cce9c 100644
--- a/net/core/fib_rules.c
+++ b/net/core/fib_rules.c
@@ -757,9 +757,9 @@ int fib_nl_newrule(struct sk_buff *skb, struct nlmsghdr *nlh,
 	if (err)
 		goto errout;
 
-	if ((nlh->nlmsg_flags & NLM_F_EXCL) &&
-	    rule_exists(ops, frh, tb, rule)) {
-		err = -EEXIST;
+	if (rule_exists(ops, frh, tb, rule)) {
+		if (nlh->nlmsg_flags & NLM_F_EXCL)
+			err = -EEXIST;
 		goto errout_free;
 	}
 
diff --git a/net/ipv4/raw.c b/net/ipv4/raw.c
index c55a5432cf37..dc91c27bb788 100644
--- a/net/ipv4/raw.c
+++ b/net/ipv4/raw.c
@@ -173,6 +173,7 @@ static int icmp_filter(const struct sock *sk, const struct sk_buff *skb)
 static int raw_v4_input(struct sk_buff *skb, const struct iphdr *iph, int hash)
 {
 	int sdif = inet_sdif(skb);
+	int dif = inet_iif(skb);
 	struct sock *sk;
 	struct hlist_head *head;
 	int delivered = 0;
@@ -185,8 +186,7 @@ static int raw_v4_input(struct sk_buff *skb, const struct iphdr *iph, int hash)
 
 	net = dev_net(skb->dev);
 	sk = __raw_v4_lookup(net, __sk_head(head), iph->protocol,
-			     iph->saddr, iph->daddr,
-			     skb->dev->ifindex, sdif);
+			     iph->saddr, iph->daddr, dif, sdif);
 
 	while (sk) {
 		delivered = 1;
diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c
index 077d9abdfcf5..20f6fac5882e 100644
--- a/net/ipv4/tcp_input.c
+++ b/net/ipv4/tcp_input.c
@@ -77,7 +77,7 @@
 #include <asm/unaligned.h>
 #include <linux/errqueue.h>
 #include <trace/events/tcp.h>
-#include <linux/static_key.h>
+#include <linux/jump_label_ratelimit.h>
 #include <net/busy_poll.h>
 
 int sysctl_tcp_max_orphans __read_mostly = NR_FILE;
@@ -113,22 +113,28 @@ int sysctl_tcp_max_orphans __read_mostly = NR_FILE;
 #define REXMIT_NEW	2 /* FRTO-style transmit of unsent/new packets */
 
 #if IS_ENABLED(CONFIG_TLS_DEVICE)
-static DEFINE_STATIC_KEY_FALSE(clean_acked_data_enabled);
+static DEFINE_STATIC_KEY_DEFERRED_FALSE(clean_acked_data_enabled, HZ);
 
 void clean_acked_data_enable(struct inet_connection_sock *icsk,
 			     void (*cad)(struct sock *sk, u32 ack_seq))
 {
 	icsk->icsk_clean_acked = cad;
-	static_branch_inc(&clean_acked_data_enabled);
+	static_branch_inc(&clean_acked_data_enabled.key);
 }
 EXPORT_SYMBOL_GPL(clean_acked_data_enable);
 
 void clean_acked_data_disable(struct inet_connection_sock *icsk)
 {
-	static_branch_dec(&clean_acked_data_enabled);
+	static_branch_slow_dec_deferred(&clean_acked_data_enabled);
 	icsk->icsk_clean_acked = NULL;
 }
 EXPORT_SYMBOL_GPL(clean_acked_data_disable);
+
+void clean_acked_data_flush(void)
+{
+	static_key_deferred_flush(&clean_acked_data_enabled);
+}
+EXPORT_SYMBOL_GPL(clean_acked_data_flush);
 #endif
 
 static void tcp_gro_dev_warn(struct sock *sk, const struct sk_buff *skb,
@@ -3598,7 +3604,7 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag)
 		icsk->icsk_retransmits = 0;
 
 #if IS_ENABLED(CONFIG_TLS_DEVICE)
-		if (static_branch_unlikely(&clean_acked_data_enabled))
+		if (static_branch_unlikely(&clean_acked_data_enabled.key))
 			if (icsk->icsk_clean_acked)
 				icsk->icsk_clean_acked(sk, ack);
 #endif
diff --git a/net/openvswitch/conntrack.c b/net/openvswitch/conntrack.c
index 333ec5f298fe..4c597a0bb168 100644
--- a/net/openvswitch/conntrack.c
+++ b/net/openvswitch/conntrack.c
@@ -1322,7 +1322,7 @@ static int ovs_ct_add_helper(struct ovs_conntrack_info *info, const char *name,
 		return -ENOMEM;
 	}
 
-#ifdef CONFIG_NF_NAT_NEEDED
+#if IS_ENABLED(CONFIG_NF_NAT)
 	if (info->nat) {
 		ret = nf_nat_helper_try_module_get(name, info->family,
 						   key->ip.proto);
@@ -1811,7 +1811,7 @@ void ovs_ct_free_action(const struct nlattr *a)
 static void __ovs_ct_free_action(struct ovs_conntrack_info *ct_info)
 {
 	if (ct_info->helper) {
-#ifdef CONFIG_NF_NAT_NEEDED
+#if IS_ENABLED(CONFIG_NF_NAT)
 		if (ct_info->nat)
 			nf_nat_helper_put(ct_info->helper);
 #endif
diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c
index 90d4e3ce00e5..fbc775fbf712 100644
--- a/net/packet/af_packet.c
+++ b/net/packet/af_packet.c
@@ -4598,14 +4598,29 @@ static void __exit packet_exit(void)
 
 static int __init packet_init(void)
 {
-	int rc = proto_register(&packet_proto, 0);
+	int rc;
 
-	if (rc != 0)
+	rc = proto_register(&packet_proto, 0);
+	if (rc)
 		goto out;
+	rc = sock_register(&packet_family_ops);
+	if (rc)
+		goto out_proto;
+	rc = register_pernet_subsys(&packet_net_ops);
+	if (rc)
+		goto out_sock;
+	rc = register_netdevice_notifier(&packet_netdev_notifier);
+	if (rc)
+		goto out_pernet;
 
-	sock_register(&packet_family_ops);
-	register_pernet_subsys(&packet_net_ops);
-	register_netdevice_notifier(&packet_netdev_notifier);
+	return 0;
+
+out_pernet:
+	unregister_pernet_subsys(&packet_net_ops);
+out_sock:
+	sock_unregister(PF_PACKET);
+out_proto:
+	proto_unregister(&packet_proto);
 out:
 	return rc;
 }
diff --git a/net/sched/cls_matchall.c b/net/sched/cls_matchall.c
index 1e98a517fb0b..db42d97a2006 100644
--- a/net/sched/cls_matchall.c
+++ b/net/sched/cls_matchall.c
@@ -308,6 +308,7 @@ static int mall_reoffload(struct tcf_proto *tp, bool add, tc_setup_cb_t *cb,
 			NL_SET_ERR_MSG_MOD(extack, "Failed to setup flow action");
 			return err;
 		}
+		return 0;
 	}
 
 	err = cb(TC_SETUP_CLSMATCHALL, &cls_mall, cb_priv);
diff --git a/net/smc/smc_ib.c b/net/smc/smc_ib.c
index 53f429c04843..d14ca4af6f94 100644
--- a/net/smc/smc_ib.c
+++ b/net/smc/smc_ib.c
@@ -146,18 +146,13 @@ out:
 static int smc_ib_fill_mac(struct smc_ib_device *smcibdev, u8 ibport)
 {
 	const struct ib_gid_attr *attr;
-	int rc = 0;
+	int rc;
 
 	attr = rdma_get_gid_attr(smcibdev->ibdev, ibport, 0);
 	if (IS_ERR(attr))
 		return -ENODEV;
 
-	if (attr->ndev)
-		memcpy(smcibdev->mac[ibport - 1], attr->ndev->dev_addr,
-		       ETH_ALEN);
-	else
-		rc = -ENODEV;
-
+	rc = rdma_read_gid_l2_fields(attr, NULL, smcibdev->mac[ibport - 1]);
 	rdma_put_gid_attr(attr);
 	return rc;
 }
@@ -185,6 +180,7 @@ int smc_ib_determine_gid(struct smc_ib_device *smcibdev, u8 ibport,
 			 unsigned short vlan_id, u8 gid[], u8 *sgid_index)
 {
 	const struct ib_gid_attr *attr;
+	const struct net_device *ndev;
 	int i;
 
 	for (i = 0; i < smcibdev->pattr[ibport - 1].gid_tbl_len; i++) {
@@ -192,11 +188,14 @@ int smc_ib_determine_gid(struct smc_ib_device *smcibdev, u8 ibport,
 		if (IS_ERR(attr))
 			continue;
 
-		if (attr->ndev &&
+		rcu_read_lock();
+		ndev = rdma_read_gid_attr_ndev_rcu(attr);
+		if (!IS_ERR(ndev) &&
 		    ((!vlan_id && !is_vlan_dev(attr->ndev)) ||
 		     (vlan_id && is_vlan_dev(attr->ndev) &&
 		      vlan_dev_vlan_id(attr->ndev) == vlan_id)) &&
 		    attr->gid_type == IB_GID_TYPE_ROCE) {
+			rcu_read_unlock();
 			if (gid)
 				memcpy(gid, &attr->gid, SMC_GID_SIZE);
 			if (sgid_index)
@@ -204,6 +203,7 @@ int smc_ib_determine_gid(struct smc_ib_device *smcibdev, u8 ibport,
 			rdma_put_gid_attr(attr);
 			return 0;
 		}
+		rcu_read_unlock();
 		rdma_put_gid_attr(attr);
 	}
 	return -ENODEV;
diff --git a/net/sunrpc/auth_gss/auth_gss.c b/net/sunrpc/auth_gss/auth_gss.c
index 3fd56c0c90ae..4ce42c62458e 100644
--- a/net/sunrpc/auth_gss/auth_gss.c
+++ b/net/sunrpc/auth_gss/auth_gss.c
@@ -269,6 +269,7 @@ err:
 struct gss_upcall_msg {
 	refcount_t count;
 	kuid_t	uid;
+	const char *service_name;
 	struct rpc_pipe_msg msg;
 	struct list_head list;
 	struct gss_auth *auth;
@@ -316,6 +317,7 @@ gss_release_msg(struct gss_upcall_msg *gss_msg)
 		gss_put_ctx(gss_msg->ctx);
 	rpc_destroy_wait_queue(&gss_msg->rpc_waitqueue);
 	gss_put_auth(gss_msg->auth);
+	kfree_const(gss_msg->service_name);
 	kfree(gss_msg);
 }
 
@@ -410,9 +412,12 @@ gss_upcall_callback(struct rpc_task *task)
 	gss_release_msg(gss_msg);
 }
 
-static void gss_encode_v0_msg(struct gss_upcall_msg *gss_msg)
+static void gss_encode_v0_msg(struct gss_upcall_msg *gss_msg,
+			      const struct cred *cred)
 {
-	uid_t uid = from_kuid(&init_user_ns, gss_msg->uid);
+	struct user_namespace *userns = cred->user_ns;
+
+	uid_t uid = from_kuid_munged(userns, gss_msg->uid);
 	memcpy(gss_msg->databuf, &uid, sizeof(uid));
 	gss_msg->msg.data = gss_msg->databuf;
 	gss_msg->msg.len = sizeof(uid);
@@ -420,17 +425,31 @@ static void gss_encode_v0_msg(struct gss_upcall_msg *gss_msg)
 	BUILD_BUG_ON(sizeof(uid) > sizeof(gss_msg->databuf));
 }
 
+static ssize_t
+gss_v0_upcall(struct file *file, struct rpc_pipe_msg *msg,
+		char __user *buf, size_t buflen)
+{
+	struct gss_upcall_msg *gss_msg = container_of(msg,
+						      struct gss_upcall_msg,
+						      msg);
+	if (msg->copied == 0)
+		gss_encode_v0_msg(gss_msg, file->f_cred);
+	return rpc_pipe_generic_upcall(file, msg, buf, buflen);
+}
+
 static int gss_encode_v1_msg(struct gss_upcall_msg *gss_msg,
 				const char *service_name,
-				const char *target_name)
+				const char *target_name,
+				const struct cred *cred)
 {
+	struct user_namespace *userns = cred->user_ns;
 	struct gss_api_mech *mech = gss_msg->auth->mech;
 	char *p = gss_msg->databuf;
 	size_t buflen = sizeof(gss_msg->databuf);
 	int len;
 
 	len = scnprintf(p, buflen, "mech=%s uid=%d", mech->gm_name,
-			from_kuid(&init_user_ns, gss_msg->uid));
+			from_kuid_munged(userns, gss_msg->uid));
 	buflen -= len;
 	p += len;
 	gss_msg->msg.len = len;
@@ -491,6 +510,25 @@ out_overflow:
 	return -ENOMEM;
 }
 
+static ssize_t
+gss_v1_upcall(struct file *file, struct rpc_pipe_msg *msg,
+		char __user *buf, size_t buflen)
+{
+	struct gss_upcall_msg *gss_msg = container_of(msg,
+						      struct gss_upcall_msg,
+						      msg);
+	int err;
+	if (msg->copied == 0) {
+		err = gss_encode_v1_msg(gss_msg,
+					gss_msg->service_name,
+					gss_msg->auth->target_name,
+					file->f_cred);
+		if (err)
+			return err;
+	}
+	return rpc_pipe_generic_upcall(file, msg, buf, buflen);
+}
+
 static struct gss_upcall_msg *
 gss_alloc_msg(struct gss_auth *gss_auth,
 		kuid_t uid, const char *service_name)
@@ -513,16 +551,14 @@ gss_alloc_msg(struct gss_auth *gss_auth,
 	refcount_set(&gss_msg->count, 1);
 	gss_msg->uid = uid;
 	gss_msg->auth = gss_auth;
-	switch (vers) {
-	case 0:
-		gss_encode_v0_msg(gss_msg);
-		break;
-	default:
-		err = gss_encode_v1_msg(gss_msg, service_name, gss_auth->target_name);
-		if (err)
+	kref_get(&gss_auth->kref);
+	if (service_name) {
+		gss_msg->service_name = kstrdup_const(service_name, GFP_NOFS);
+		if (!gss_msg->service_name) {
+			err = -ENOMEM;
 			goto err_put_pipe_version;
+		}
 	}
-	kref_get(&gss_auth->kref);
 	return gss_msg;
 err_put_pipe_version:
 	put_pipe_version(gss_auth->net);
@@ -581,8 +617,8 @@ gss_refresh_upcall(struct rpc_task *task)
 		/* XXX: warning on the first, under the assumption we
 		 * shouldn't normally hit this case on a refresh. */
 		warn_gssd();
-		task->tk_timeout = 15*HZ;
-		rpc_sleep_on(&pipe_version_rpc_waitqueue, task, NULL);
+		rpc_sleep_on_timeout(&pipe_version_rpc_waitqueue,
+				task, NULL, jiffies + (15 * HZ));
 		err = -EAGAIN;
 		goto out;
 	}
@@ -595,7 +631,6 @@ gss_refresh_upcall(struct rpc_task *task)
 	if (gss_cred->gc_upcall != NULL)
 		rpc_sleep_on(&gss_cred->gc_upcall->rpc_waitqueue, task, NULL);
 	else if (gss_msg->ctx == NULL && gss_msg->msg.errno >= 0) {
-		task->tk_timeout = 0;
 		gss_cred->gc_upcall = gss_msg;
 		/* gss_upcall_callback will release the reference to gss_upcall_msg */
 		refcount_inc(&gss_msg->count);
@@ -707,7 +742,7 @@ gss_pipe_downcall(struct file *filp, const char __user *src, size_t mlen)
 		goto err;
 	}
 
-	uid = make_kuid(&init_user_ns, id);
+	uid = make_kuid(current_user_ns(), id);
 	if (!uid_valid(uid)) {
 		err = -EINVAL;
 		goto err;
@@ -2116,7 +2151,7 @@ static const struct rpc_credops gss_nullops = {
 };
 
 static const struct rpc_pipe_ops gss_upcall_ops_v0 = {
-	.upcall		= rpc_pipe_generic_upcall,
+	.upcall		= gss_v0_upcall,
 	.downcall	= gss_pipe_downcall,
 	.destroy_msg	= gss_pipe_destroy_msg,
 	.open_pipe	= gss_pipe_open_v0,
@@ -2124,7 +2159,7 @@ static const struct rpc_pipe_ops gss_upcall_ops_v0 = {
 };
 
 static const struct rpc_pipe_ops gss_upcall_ops_v1 = {
-	.upcall		= rpc_pipe_generic_upcall,
+	.upcall		= gss_v1_upcall,
 	.downcall	= gss_pipe_downcall,
 	.destroy_msg	= gss_pipe_destroy_msg,
 	.open_pipe	= gss_pipe_open_v1,
diff --git a/net/sunrpc/auth_unix.c b/net/sunrpc/auth_unix.c
index d4018e5a24c5..e7df1f782b2e 100644
--- a/net/sunrpc/auth_unix.c
+++ b/net/sunrpc/auth_unix.c
@@ -107,6 +107,8 @@ unx_marshal(struct rpc_task *task, struct xdr_stream *xdr)
 	__be32		*p, *cred_len, *gidarr_len;
 	int		i;
 	struct group_info *gi = cred->cr_cred->group_info;
+	struct user_namespace *userns = clnt->cl_cred ?
+		clnt->cl_cred->user_ns : &init_user_ns;
 
 	/* Credential */
 
@@ -122,14 +124,13 @@ unx_marshal(struct rpc_task *task, struct xdr_stream *xdr)
 	p = xdr_reserve_space(xdr, 3 * sizeof(*p));
 	if (!p)
 		goto marshal_failed;
-	*p++ = cpu_to_be32(from_kuid(&init_user_ns, cred->cr_cred->fsuid));
-	*p++ = cpu_to_be32(from_kgid(&init_user_ns, cred->cr_cred->fsgid));
+	*p++ = cpu_to_be32(from_kuid_munged(userns, cred->cr_cred->fsuid));
+	*p++ = cpu_to_be32(from_kgid_munged(userns, cred->cr_cred->fsgid));
 
 	gidarr_len = p++;
 	if (gi)
 		for (i = 0; i < UNX_NGROUPS && i < gi->ngroups; i++)
-			*p++ = cpu_to_be32(from_kgid(&init_user_ns,
-						     gi->gid[i]));
+			*p++ = cpu_to_be32(from_kgid_munged(userns, gi->gid[i]));
 	*gidarr_len = cpu_to_be32(p - gidarr_len - 1);
 	*cred_len = cpu_to_be32((p - cred_len - 1) << 2);
 	p = xdr_reserve_space(xdr, (p - gidarr_len - 1) << 2);
diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c
index 8ff11dc98d7f..c1f1afabd024 100644
--- a/net/sunrpc/clnt.c
+++ b/net/sunrpc/clnt.c
@@ -394,6 +394,7 @@ static struct rpc_clnt * rpc_new_client(const struct rpc_create_args *args,
 	if (err)
 		goto out_no_clid;
 
+	clnt->cl_cred	  = get_cred(args->cred);
 	clnt->cl_procinfo = version->procs;
 	clnt->cl_maxproc  = version->nrprocs;
 	clnt->cl_prog     = args->prognumber ? : program->number;
@@ -439,6 +440,7 @@ static struct rpc_clnt * rpc_new_client(const struct rpc_create_args *args,
 out_no_path:
 	rpc_free_iostats(clnt->cl_metrics);
 out_no_stats:
+	put_cred(clnt->cl_cred);
 	rpc_free_clid(clnt);
 out_no_clid:
 	kfree(clnt);
@@ -484,8 +486,11 @@ static struct rpc_clnt *rpc_create_xprt(struct rpc_create_args *args,
 	}
 
 	clnt->cl_softrtry = 1;
-	if (args->flags & RPC_CLNT_CREATE_HARDRTRY)
+	if (args->flags & (RPC_CLNT_CREATE_HARDRTRY|RPC_CLNT_CREATE_SOFTERR)) {
 		clnt->cl_softrtry = 0;
+		if (args->flags & RPC_CLNT_CREATE_SOFTERR)
+			clnt->cl_softerr = 1;
+	}
 
 	if (args->flags & RPC_CLNT_CREATE_AUTOBIND)
 		clnt->cl_autobind = 1;
@@ -623,10 +628,12 @@ static struct rpc_clnt *__rpc_clone_client(struct rpc_create_args *args,
 	/* Turn off autobind on clones */
 	new->cl_autobind = 0;
 	new->cl_softrtry = clnt->cl_softrtry;
+	new->cl_softerr = clnt->cl_softerr;
 	new->cl_noretranstimeo = clnt->cl_noretranstimeo;
 	new->cl_discrtry = clnt->cl_discrtry;
 	new->cl_chatty = clnt->cl_chatty;
 	new->cl_principal = clnt->cl_principal;
+	new->cl_cred = get_cred(clnt->cl_cred);
 	return new;
 
 out_err:
@@ -648,6 +655,7 @@ struct rpc_clnt *rpc_clone_client(struct rpc_clnt *clnt)
 		.prognumber	= clnt->cl_prog,
 		.version	= clnt->cl_vers,
 		.authflavor	= clnt->cl_auth->au_flavor,
+		.cred		= clnt->cl_cred,
 	};
 	return __rpc_clone_client(&args, clnt);
 }
@@ -669,6 +677,7 @@ rpc_clone_client_set_auth(struct rpc_clnt *clnt, rpc_authflavor_t flavor)
 		.prognumber	= clnt->cl_prog,
 		.version	= clnt->cl_vers,
 		.authflavor	= flavor,
+		.cred		= clnt->cl_cred,
 	};
 	return __rpc_clone_client(&args, clnt);
 }
@@ -827,14 +836,8 @@ void rpc_killall_tasks(struct rpc_clnt *clnt)
 	 * Spin lock all_tasks to prevent changes...
 	 */
 	spin_lock(&clnt->cl_lock);
-	list_for_each_entry(rovr, &clnt->cl_tasks, tk_task) {
-		if (!RPC_IS_ACTIVATED(rovr))
-			continue;
-		if (!(rovr->tk_flags & RPC_TASK_KILLED)) {
-			rovr->tk_flags |= RPC_TASK_KILLED;
-			rpc_exit(rovr, -EIO);
-		}
-	}
+	list_for_each_entry(rovr, &clnt->cl_tasks, tk_task)
+		rpc_signal_task(rovr);
 	spin_unlock(&clnt->cl_lock);
 }
 EXPORT_SYMBOL_GPL(rpc_killall_tasks);
@@ -882,6 +885,7 @@ rpc_free_client(struct rpc_clnt *clnt)
 	xprt_put(rcu_dereference_raw(clnt->cl_xprt));
 	xprt_iter_destroy(&clnt->cl_xpi);
 	rpciod_down();
+	put_cred(clnt->cl_cred);
 	rpc_free_clid(clnt);
 	kfree(clnt);
 	return parent;
@@ -946,6 +950,7 @@ struct rpc_clnt *rpc_bind_new_program(struct rpc_clnt *old,
 		.prognumber	= program->number,
 		.version	= vers,
 		.authflavor	= old->cl_auth->au_flavor,
+		.cred		= old->cl_cred,
 	};
 	struct rpc_clnt *clnt;
 	int err;
@@ -1007,6 +1012,8 @@ void rpc_task_set_client(struct rpc_task *task, struct rpc_clnt *clnt)
 		atomic_inc(&clnt->cl_count);
 		if (clnt->cl_softrtry)
 			task->tk_flags |= RPC_TASK_SOFT;
+		if (clnt->cl_softerr)
+			task->tk_flags |= RPC_TASK_TIMEOUT;
 		if (clnt->cl_noretranstimeo)
 			task->tk_flags |= RPC_TASK_NO_RETRANS_TIMEOUT;
 		if (atomic_read(&clnt->cl_swapper))
@@ -1470,22 +1477,14 @@ void rpc_force_rebind(struct rpc_clnt *clnt)
 }
 EXPORT_SYMBOL_GPL(rpc_force_rebind);
 
-/*
- * Restart an (async) RPC call from the call_prepare state.
- * Usually called from within the exit handler.
- */
-int
-rpc_restart_call_prepare(struct rpc_task *task)
+static int
+__rpc_restart_call(struct rpc_task *task, void (*action)(struct rpc_task *))
 {
-	if (RPC_ASSASSINATED(task))
-		return 0;
-	task->tk_action = call_start;
 	task->tk_status = 0;
-	if (task->tk_ops->rpc_call_prepare != NULL)
-		task->tk_action = rpc_prepare_task;
+	task->tk_rpc_status = 0;
+	task->tk_action = action;
 	return 1;
 }
-EXPORT_SYMBOL_GPL(rpc_restart_call_prepare);
 
 /*
  * Restart an (async) RPC call. Usually called from within the
@@ -1494,14 +1493,23 @@ EXPORT_SYMBOL_GPL(rpc_restart_call_prepare);
 int
 rpc_restart_call(struct rpc_task *task)
 {
-	if (RPC_ASSASSINATED(task))
-		return 0;
-	task->tk_action = call_start;
-	task->tk_status = 0;
-	return 1;
+	return __rpc_restart_call(task, call_start);
 }
 EXPORT_SYMBOL_GPL(rpc_restart_call);
 
+/*
+ * Restart an (async) RPC call from the call_prepare state.
+ * Usually called from within the exit handler.
+ */
+int
+rpc_restart_call_prepare(struct rpc_task *task)
+{
+	if (task->tk_ops->rpc_call_prepare != NULL)
+		return __rpc_restart_call(task, rpc_prepare_task);
+	return rpc_restart_call(task);
+}
+EXPORT_SYMBOL_GPL(rpc_restart_call_prepare);
+
 const char
 *rpc_proc_name(const struct rpc_task *task)
 {
@@ -1516,6 +1524,19 @@ const char
 		return "no proc";
 }
 
+static void
+__rpc_call_rpcerror(struct rpc_task *task, int tk_status, int rpc_status)
+{
+	task->tk_rpc_status = rpc_status;
+	rpc_exit(task, tk_status);
+}
+
+static void
+rpc_call_rpcerror(struct rpc_task *task, int status)
+{
+	__rpc_call_rpcerror(task, status, status);
+}
+
 /*
  * 0.  Initial state
  *
@@ -1580,7 +1601,7 @@ call_reserveresult(struct rpc_task *task)
 
 		printk(KERN_ERR "%s: status=%d, but no request slot, exiting\n",
 				__func__, status);
-		rpc_exit(task, -EIO);
+		rpc_call_rpcerror(task, -EIO);
 		return;
 	}
 
@@ -1608,7 +1629,7 @@ call_reserveresult(struct rpc_task *task)
 				__func__, status);
 		break;
 	}
-	rpc_exit(task, status);
+	rpc_call_rpcerror(task, status);
 }
 
 /*
@@ -1676,7 +1697,7 @@ call_refreshresult(struct rpc_task *task)
 	}
 	dprintk("RPC: %5u %s: refresh creds failed with error %d\n",
 				task->tk_pid, __func__, status);
-	rpc_exit(task, status);
+	rpc_call_rpcerror(task, status);
 }
 
 /*
@@ -1727,7 +1748,7 @@ call_allocate(struct rpc_task *task)
 	if (status == 0)
 		return;
 	if (status != -ENOMEM) {
-		rpc_exit(task, status);
+		rpc_call_rpcerror(task, status);
 		return;
 	}
 
@@ -1793,10 +1814,17 @@ call_encode(struct rpc_task *task)
 			rpc_delay(task, HZ >> 4);
 			break;
 		case -EKEYEXPIRED:
-			task->tk_action = call_refresh;
+			if (!task->tk_cred_retry) {
+				rpc_exit(task, task->tk_status);
+			} else {
+				task->tk_action = call_refresh;
+				task->tk_cred_retry--;
+				dprintk("RPC: %5u %s: retry refresh creds\n",
+					task->tk_pid, __func__);
+			}
 			break;
 		default:
-			rpc_exit(task, task->tk_status);
+			rpc_call_rpcerror(task, task->tk_status);
 		}
 		return;
 	} else {
@@ -1857,7 +1885,6 @@ call_bind(struct rpc_task *task)
 	if (!xprt_prepare_transmit(task))
 		return;
 
-	task->tk_timeout = xprt->bind_timeout;
 	xprt->ops->rpcbind(task);
 }
 
@@ -1938,7 +1965,7 @@ call_bind_status(struct rpc_task *task)
 				task->tk_pid, -task->tk_status);
 	}
 
-	rpc_exit(task, status);
+	rpc_call_rpcerror(task, status);
 	return;
 
 retry_timeout:
@@ -1973,7 +2000,7 @@ call_connect(struct rpc_task *task)
 	if (task->tk_status < 0)
 		return;
 	if (task->tk_flags & RPC_TASK_NOCONNECT) {
-		rpc_exit(task, -ENOTCONN);
+		rpc_call_rpcerror(task, -ENOTCONN);
 		return;
 	}
 	if (!xprt_prepare_transmit(task))
@@ -2033,7 +2060,7 @@ call_connect_status(struct rpc_task *task)
 		task->tk_action = call_transmit;
 		return;
 	}
-	rpc_exit(task, status);
+	rpc_call_rpcerror(task, status);
 	return;
 out_retry:
 	/* Check for timeouts before looping back to call_bind */
@@ -2118,7 +2145,7 @@ call_transmit_status(struct rpc_task *task)
 			if (!task->tk_msg.rpc_proc->p_proc)
 				trace_xprt_ping(task->tk_xprt,
 						task->tk_status);
-			rpc_exit(task, task->tk_status);
+			rpc_call_rpcerror(task, task->tk_status);
 			return;
 		}
 		/* fall through */
@@ -2282,7 +2309,7 @@ call_status(struct rpc_task *task)
 	rpc_check_timeout(task);
 	return;
 out_exit:
-	rpc_exit(task, status);
+	rpc_call_rpcerror(task, status);
 }
 
 static bool
@@ -2306,29 +2333,40 @@ rpc_check_timeout(struct rpc_task *task)
 	task->tk_timeouts++;
 
 	if (RPC_IS_SOFTCONN(task) && !rpc_check_connected(task->tk_rqstp)) {
-		rpc_exit(task, -ETIMEDOUT);
+		rpc_call_rpcerror(task, -ETIMEDOUT);
 		return;
 	}
 
 	if (RPC_IS_SOFT(task)) {
+		/*
+		 * Once a "no retrans timeout" soft tasks (a.k.a NFSv4) has
+		 * been sent, it should time out only if the transport
+		 * connection gets terminally broken.
+		 */
+		if ((task->tk_flags & RPC_TASK_NO_RETRANS_TIMEOUT) &&
+		    rpc_check_connected(task->tk_rqstp))
+			return;
+
 		if (clnt->cl_chatty) {
-			printk(KERN_NOTICE "%s: server %s not responding, timed out\n",
+			pr_notice_ratelimited(
+				"%s: server %s not responding, timed out\n",
 				clnt->cl_program->name,
 				task->tk_xprt->servername);
 		}
 		if (task->tk_flags & RPC_TASK_TIMEOUT)
-			rpc_exit(task, -ETIMEDOUT);
+			rpc_call_rpcerror(task, -ETIMEDOUT);
 		else
-			rpc_exit(task, -EIO);
+			__rpc_call_rpcerror(task, -EIO, -ETIMEDOUT);
 		return;
 	}
 
 	if (!(task->tk_flags & RPC_CALL_MAJORSEEN)) {
 		task->tk_flags |= RPC_CALL_MAJORSEEN;
 		if (clnt->cl_chatty) {
-			printk(KERN_NOTICE "%s: server %s not responding, still trying\n",
-			clnt->cl_program->name,
-			task->tk_xprt->servername);
+			pr_notice_ratelimited(
+				"%s: server %s not responding, still trying\n",
+				clnt->cl_program->name,
+				task->tk_xprt->servername);
 		}
 	}
 	rpc_force_rebind(clnt);
@@ -2358,7 +2396,7 @@ call_decode(struct rpc_task *task)
 
 	if (task->tk_flags & RPC_CALL_MAJORSEEN) {
 		if (clnt->cl_chatty) {
-			printk(KERN_NOTICE "%s: server %s OK\n",
+			pr_notice_ratelimited("%s: server %s OK\n",
 				clnt->cl_program->name,
 				task->tk_xprt->servername);
 		}
@@ -2881,7 +2919,7 @@ static void rpc_show_task(const struct rpc_clnt *clnt,
 
 	printk(KERN_INFO "%5u %04x %6d %8p %8p %8ld %8p %sv%u %s a:%ps q:%s\n",
 		task->tk_pid, task->tk_flags, task->tk_status,
-		clnt, task->tk_rqstp, task->tk_timeout, task->tk_ops,
+		clnt, task->tk_rqstp, rpc_task_timeout(task), task->tk_ops,
 		clnt->cl_program->name, clnt->cl_vers, rpc_proc_name(task),
 		task->tk_action, rpc_waitq);
 }
diff --git a/net/sunrpc/debugfs.c b/net/sunrpc/debugfs.c
index 19bb356230ed..95ebd76b132d 100644
--- a/net/sunrpc/debugfs.c
+++ b/net/sunrpc/debugfs.c
@@ -33,7 +33,7 @@ tasks_show(struct seq_file *f, void *v)
 
 	seq_printf(f, "%5u %04x %6d 0x%x 0x%x %8ld %ps %sv%u %s a:%ps q:%s\n",
 		task->tk_pid, task->tk_flags, task->tk_status,
-		clnt->cl_clid, xid, task->tk_timeout, task->tk_ops,
+		clnt->cl_clid, xid, rpc_task_timeout(task), task->tk_ops,
 		clnt->cl_program->name, clnt->cl_vers, rpc_proc_name(task),
 		task->tk_action, rpc_waitq);
 	return 0;
diff --git a/net/sunrpc/rpcb_clnt.c b/net/sunrpc/rpcb_clnt.c
index 41a971ac1c63..2277b7cdad27 100644
--- a/net/sunrpc/rpcb_clnt.c
+++ b/net/sunrpc/rpcb_clnt.c
@@ -240,6 +240,7 @@ static int rpcb_create_local_unix(struct net *net)
 		.program	= &rpcb_program,
 		.version	= RPCBVERS_2,
 		.authflavor	= RPC_AUTH_NULL,
+		.cred		= current_cred(),
 		/*
 		 * We turn off the idle timeout to prevent the kernel
 		 * from automatically disconnecting the socket.
@@ -299,6 +300,7 @@ static int rpcb_create_local_net(struct net *net)
 		.program	= &rpcb_program,
 		.version	= RPCBVERS_2,
 		.authflavor	= RPC_AUTH_UNIX,
+		.cred		= current_cred(),
 		.flags		= RPC_CLNT_CREATE_NOPING,
 	};
 	struct rpc_clnt *clnt, *clnt4;
@@ -358,7 +360,8 @@ out:
 static struct rpc_clnt *rpcb_create(struct net *net, const char *nodename,
 				    const char *hostname,
 				    struct sockaddr *srvaddr, size_t salen,
-				    int proto, u32 version)
+				    int proto, u32 version,
+				    const struct cred *cred)
 {
 	struct rpc_create_args args = {
 		.net		= net,
@@ -370,6 +373,7 @@ static struct rpc_clnt *rpcb_create(struct net *net, const char *nodename,
 		.program	= &rpcb_program,
 		.version	= version,
 		.authflavor	= RPC_AUTH_UNIX,
+		.cred		= cred,
 		.flags		= (RPC_CLNT_CREATE_NOPING |
 					RPC_CLNT_CREATE_NONPRIVPORT),
 	};
@@ -694,7 +698,8 @@ void rpcb_getport_async(struct rpc_task *task)
 
 	/* Put self on the wait queue to ensure we get notified if
 	 * some other task is already attempting to bind the port */
-	rpc_sleep_on(&xprt->binding, task, NULL);
+	rpc_sleep_on_timeout(&xprt->binding, task,
+			NULL, jiffies + xprt->bind_timeout);
 
 	if (xprt_test_and_set_binding(xprt)) {
 		dprintk("RPC: %5u %s: waiting for another binder\n",
@@ -744,7 +749,8 @@ void rpcb_getport_async(struct rpc_task *task)
 	rpcb_clnt = rpcb_create(xprt->xprt_net,
 				clnt->cl_nodename,
 				xprt->servername, sap, salen,
-				xprt->prot, bind_version);
+				xprt->prot, bind_version,
+				clnt->cl_cred);
 	if (IS_ERR(rpcb_clnt)) {
 		status = PTR_ERR(rpcb_clnt);
 		dprintk("RPC: %5u %s: rpcb_create failed, error %ld\n",
diff --git a/net/sunrpc/sched.c b/net/sunrpc/sched.c
index 28956c70100a..1a12fb03e611 100644
--- a/net/sunrpc/sched.c
+++ b/net/sunrpc/sched.c
@@ -58,6 +58,20 @@ static struct rpc_wait_queue delay_queue;
 struct workqueue_struct *rpciod_workqueue __read_mostly;
 struct workqueue_struct *xprtiod_workqueue __read_mostly;
 
+unsigned long
+rpc_task_timeout(const struct rpc_task *task)
+{
+	unsigned long timeout = READ_ONCE(task->tk_timeout);
+
+	if (timeout != 0) {
+		unsigned long now = jiffies;
+		if (time_before(now, timeout))
+			return timeout - now;
+	}
+	return 0;
+}
+EXPORT_SYMBOL_GPL(rpc_task_timeout);
+
 /*
  * Disable the timer for a given RPC task. Should be called with
  * queue->lock and bh_disabled in order to avoid races within
@@ -66,7 +80,7 @@ struct workqueue_struct *xprtiod_workqueue __read_mostly;
 static void
 __rpc_disable_timer(struct rpc_wait_queue *queue, struct rpc_task *task)
 {
-	if (task->tk_timeout == 0)
+	if (list_empty(&task->u.tk_wait.timer_list))
 		return;
 	dprintk("RPC: %5u disabling timer\n", task->tk_pid);
 	task->tk_timeout = 0;
@@ -78,25 +92,21 @@ __rpc_disable_timer(struct rpc_wait_queue *queue, struct rpc_task *task)
 static void
 rpc_set_queue_timer(struct rpc_wait_queue *queue, unsigned long expires)
 {
-	queue->timer_list.expires = expires;
-	mod_timer(&queue->timer_list.timer, expires);
+	timer_reduce(&queue->timer_list.timer, expires);
 }
 
 /*
  * Set up a timer for the current task.
  */
 static void
-__rpc_add_timer(struct rpc_wait_queue *queue, struct rpc_task *task)
+__rpc_add_timer(struct rpc_wait_queue *queue, struct rpc_task *task,
+		unsigned long timeout)
 {
-	if (!task->tk_timeout)
-		return;
-
 	dprintk("RPC: %5u setting alarm for %u ms\n",
-		task->tk_pid, jiffies_to_msecs(task->tk_timeout));
+		task->tk_pid, jiffies_to_msecs(timeout - jiffies));
 
-	task->u.tk_wait.expires = jiffies + task->tk_timeout;
-	if (list_empty(&queue->timer_list.list) || time_before(task->u.tk_wait.expires, queue->timer_list.expires))
-		rpc_set_queue_timer(queue, task->u.tk_wait.expires);
+	task->tk_timeout = timeout;
+	rpc_set_queue_timer(queue, timeout);
 	list_add(&task->u.tk_wait.timer_list, &queue->timer_list.list);
 }
 
@@ -188,6 +198,7 @@ static void __rpc_add_wait_queue(struct rpc_wait_queue *queue,
 	if (RPC_IS_QUEUED(task))
 		return;
 
+	INIT_LIST_HEAD(&task->u.tk_wait.timer_list);
 	if (RPC_IS_PRIORITY(queue))
 		__rpc_add_wait_queue_priority(queue, task, queue_priority);
 	else if (RPC_IS_SWAPPER(task))
@@ -238,7 +249,9 @@ static void __rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const c
 	queue->maxpriority = nr_queues - 1;
 	rpc_reset_waitqueue_priority(queue);
 	queue->qlen = 0;
-	timer_setup(&queue->timer_list.timer, __rpc_queue_timer_fn, 0);
+	timer_setup(&queue->timer_list.timer,
+			__rpc_queue_timer_fn,
+			TIMER_DEFERRABLE);
 	INIT_LIST_HEAD(&queue->timer_list.list);
 	rpc_assign_waitqueue_name(queue, qname);
 }
@@ -362,7 +375,6 @@ static void rpc_make_runnable(struct workqueue_struct *wq,
  */
 static void __rpc_sleep_on_priority(struct rpc_wait_queue *q,
 		struct rpc_task *task,
-		rpc_action action,
 		unsigned char queue_priority)
 {
 	dprintk("RPC: %5u sleep_on(queue \"%s\" time %lu)\n",
@@ -372,47 +384,100 @@ static void __rpc_sleep_on_priority(struct rpc_wait_queue *q,
 
 	__rpc_add_wait_queue(q, task, queue_priority);
 
-	WARN_ON_ONCE(task->tk_callback != NULL);
-	task->tk_callback = action;
-	__rpc_add_timer(q, task);
 }
 
-void rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task,
-				rpc_action action)
+static void __rpc_sleep_on_priority_timeout(struct rpc_wait_queue *q,
+		struct rpc_task *task, unsigned long timeout,
+		unsigned char queue_priority)
+{
+	if (time_is_after_jiffies(timeout)) {
+		__rpc_sleep_on_priority(q, task, queue_priority);
+		__rpc_add_timer(q, task, timeout);
+	} else
+		task->tk_status = -ETIMEDOUT;
+}
+
+static void rpc_set_tk_callback(struct rpc_task *task, rpc_action action)
+{
+	if (action && !WARN_ON_ONCE(task->tk_callback != NULL))
+		task->tk_callback = action;
+}
+
+static bool rpc_sleep_check_activated(struct rpc_task *task)
 {
 	/* We shouldn't ever put an inactive task to sleep */
-	WARN_ON_ONCE(!RPC_IS_ACTIVATED(task));
-	if (!RPC_IS_ACTIVATED(task)) {
+	if (WARN_ON_ONCE(!RPC_IS_ACTIVATED(task))) {
 		task->tk_status = -EIO;
 		rpc_put_task_async(task);
-		return;
+		return false;
 	}
+	return true;
+}
+
+void rpc_sleep_on_timeout(struct rpc_wait_queue *q, struct rpc_task *task,
+				rpc_action action, unsigned long timeout)
+{
+	if (!rpc_sleep_check_activated(task))
+		return;
+
+	rpc_set_tk_callback(task, action);
+
+	/*
+	 * Protect the queue operations.
+	 */
+	spin_lock_bh(&q->lock);
+	__rpc_sleep_on_priority_timeout(q, task, timeout, task->tk_priority);
+	spin_unlock_bh(&q->lock);
+}
+EXPORT_SYMBOL_GPL(rpc_sleep_on_timeout);
 
+void rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task,
+				rpc_action action)
+{
+	if (!rpc_sleep_check_activated(task))
+		return;
+
+	rpc_set_tk_callback(task, action);
+
+	WARN_ON_ONCE(task->tk_timeout != 0);
 	/*
 	 * Protect the queue operations.
 	 */
 	spin_lock_bh(&q->lock);
-	__rpc_sleep_on_priority(q, task, action, task->tk_priority);
+	__rpc_sleep_on_priority(q, task, task->tk_priority);
 	spin_unlock_bh(&q->lock);
 }
 EXPORT_SYMBOL_GPL(rpc_sleep_on);
 
+void rpc_sleep_on_priority_timeout(struct rpc_wait_queue *q,
+		struct rpc_task *task, unsigned long timeout, int priority)
+{
+	if (!rpc_sleep_check_activated(task))
+		return;
+
+	priority -= RPC_PRIORITY_LOW;
+	/*
+	 * Protect the queue operations.
+	 */
+	spin_lock_bh(&q->lock);
+	__rpc_sleep_on_priority_timeout(q, task, timeout, priority);
+	spin_unlock_bh(&q->lock);
+}
+EXPORT_SYMBOL_GPL(rpc_sleep_on_priority_timeout);
+
 void rpc_sleep_on_priority(struct rpc_wait_queue *q, struct rpc_task *task,
-		rpc_action action, int priority)
+		int priority)
 {
-	/* We shouldn't ever put an inactive task to sleep */
-	WARN_ON_ONCE(!RPC_IS_ACTIVATED(task));
-	if (!RPC_IS_ACTIVATED(task)) {
-		task->tk_status = -EIO;
-		rpc_put_task_async(task);
+	if (!rpc_sleep_check_activated(task))
 		return;
-	}
 
+	WARN_ON_ONCE(task->tk_timeout != 0);
+	priority -= RPC_PRIORITY_LOW;
 	/*
 	 * Protect the queue operations.
 	 */
 	spin_lock_bh(&q->lock);
-	__rpc_sleep_on_priority(q, task, action, priority - RPC_PRIORITY_LOW);
+	__rpc_sleep_on_priority(q, task, priority);
 	spin_unlock_bh(&q->lock);
 }
 EXPORT_SYMBOL_GPL(rpc_sleep_on_priority);
@@ -704,7 +769,7 @@ static void __rpc_queue_timer_fn(struct timer_list *t)
 	spin_lock(&queue->lock);
 	expires = now = jiffies;
 	list_for_each_entry_safe(task, n, &queue->timer_list.list, u.tk_wait.timer_list) {
-		timeo = task->u.tk_wait.expires;
+		timeo = task->tk_timeout;
 		if (time_after_eq(now, timeo)) {
 			dprintk("RPC: %5u timeout\n", task->tk_pid);
 			task->tk_status = -ETIMEDOUT;
@@ -730,8 +795,7 @@ static void __rpc_atrun(struct rpc_task *task)
  */
 void rpc_delay(struct rpc_task *task, unsigned long delay)
 {
-	task->tk_timeout = delay;
-	rpc_sleep_on(&delay_queue, task, __rpc_atrun);
+	rpc_sleep_on_timeout(&delay_queue, task, __rpc_atrun, jiffies + delay);
 }
 EXPORT_SYMBOL_GPL(rpc_delay);
 
@@ -759,8 +823,7 @@ static void
 rpc_reset_task_statistics(struct rpc_task *task)
 {
 	task->tk_timeouts = 0;
-	task->tk_flags &= ~(RPC_CALL_MAJORSEEN|RPC_TASK_KILLED|RPC_TASK_SENT);
-
+	task->tk_flags &= ~(RPC_CALL_MAJORSEEN|RPC_TASK_SENT);
 	rpc_init_task_statistics(task);
 }
 
@@ -773,7 +836,6 @@ void rpc_exit_task(struct rpc_task *task)
 	if (task->tk_ops->rpc_call_done != NULL) {
 		task->tk_ops->rpc_call_done(task, task->tk_calldata);
 		if (task->tk_action != NULL) {
-			WARN_ON(RPC_ASSASSINATED(task));
 			/* Always release the RPC slot and buffer memory */
 			xprt_release(task);
 			rpc_reset_task_statistics(task);
@@ -781,6 +843,19 @@ void rpc_exit_task(struct rpc_task *task)
 	}
 }
 
+void rpc_signal_task(struct rpc_task *task)
+{
+	struct rpc_wait_queue *queue;
+
+	if (!RPC_IS_ACTIVATED(task))
+		return;
+	set_bit(RPC_TASK_SIGNALLED, &task->tk_runstate);
+	smp_mb__after_atomic();
+	queue = READ_ONCE(task->tk_waitqueue);
+	if (queue)
+		rpc_wake_up_queued_task_set_status(queue, task, -ERESTARTSYS);
+}
+
 void rpc_exit(struct rpc_task *task, int status)
 {
 	task->tk_status = status;
@@ -836,6 +911,13 @@ static void __rpc_execute(struct rpc_task *task)
 		 */
 		if (!RPC_IS_QUEUED(task))
 			continue;
+
+		/*
+		 * Signalled tasks should exit rather than sleep.
+		 */
+		if (RPC_SIGNALLED(task))
+			rpc_exit(task, -ERESTARTSYS);
+
 		/*
 		 * The queue->lock protects against races with
 		 * rpc_make_runnable().
@@ -861,7 +943,7 @@ static void __rpc_execute(struct rpc_task *task)
 		status = out_of_line_wait_on_bit(&task->tk_runstate,
 				RPC_TASK_QUEUED, rpc_wait_bit_killable,
 				TASK_KILLABLE);
-		if (status == -ERESTARTSYS) {
+		if (status < 0) {
 			/*
 			 * When a sync task receives a signal, it exits with
 			 * -ERESTARTSYS. In order to catch any callbacks that
@@ -869,7 +951,7 @@ static void __rpc_execute(struct rpc_task *task)
 			 * break the loop here, but go around once more.
 			 */
 			dprintk("RPC: %5u got signal\n", task->tk_pid);
-			task->tk_flags |= RPC_TASK_KILLED;
+			set_bit(RPC_TASK_SIGNALLED, &task->tk_runstate);
 			rpc_exit(task, -ERESTARTSYS);
 		}
 		dprintk("RPC: %5u sync task resuming\n", task->tk_pid);
diff --git a/net/sunrpc/socklib.c b/net/sunrpc/socklib.c
index 7e55cfc69697..9faea12624a6 100644
--- a/net/sunrpc/socklib.c
+++ b/net/sunrpc/socklib.c
@@ -106,7 +106,7 @@ xdr_partial_copy_from_skb(struct xdr_buf *xdr, unsigned int base, struct xdr_skb
 		/* ACL likes to be lazy in allocating pages - ACLs
 		 * are small by default but can get huge. */
 		if ((xdr->flags & XDRBUF_SPARSE_PAGES) && *ppage == NULL) {
-			*ppage = alloc_page(GFP_ATOMIC);
+			*ppage = alloc_page(GFP_NOWAIT | __GFP_NOWARN);
 			if (unlikely(*ppage == NULL)) {
 				if (copied == 0)
 					copied = -ENOMEM;
diff --git a/net/sunrpc/xprt.c b/net/sunrpc/xprt.c
index d7117d241460..a9d40bc7ebed 100644
--- a/net/sunrpc/xprt.c
+++ b/net/sunrpc/xprt.c
@@ -73,6 +73,15 @@ static void	 xprt_destroy(struct rpc_xprt *xprt);
 static DEFINE_SPINLOCK(xprt_list_lock);
 static LIST_HEAD(xprt_list);
 
+static unsigned long xprt_request_timeout(const struct rpc_rqst *req)
+{
+	unsigned long timeout = jiffies + req->rq_timeout;
+
+	if (time_before(timeout, req->rq_majortimeo))
+		return timeout;
+	return req->rq_majortimeo;
+}
+
 /**
  * xprt_register_transport - register a transport implementation
  * @transport: transport to register
@@ -209,9 +218,12 @@ out_unlock:
 out_sleep:
 	dprintk("RPC: %5u failed to lock transport %p\n",
 			task->tk_pid, xprt);
-	task->tk_timeout = RPC_IS_SOFT(task) ? req->rq_timeout : 0;
 	task->tk_status = -EAGAIN;
-	rpc_sleep_on(&xprt->sending, task, NULL);
+	if  (RPC_IS_SOFT(task))
+		rpc_sleep_on_timeout(&xprt->sending, task, NULL,
+				xprt_request_timeout(req));
+	else
+		rpc_sleep_on(&xprt->sending, task, NULL);
 	return 0;
 }
 EXPORT_SYMBOL_GPL(xprt_reserve_xprt);
@@ -273,9 +285,12 @@ out_unlock:
 	xprt_clear_locked(xprt);
 out_sleep:
 	dprintk("RPC: %5u failed to lock transport %p\n", task->tk_pid, xprt);
-	task->tk_timeout = RPC_IS_SOFT(task) ? req->rq_timeout : 0;
 	task->tk_status = -EAGAIN;
-	rpc_sleep_on(&xprt->sending, task, NULL);
+	if (RPC_IS_SOFT(task))
+		rpc_sleep_on_timeout(&xprt->sending, task, NULL,
+				xprt_request_timeout(req));
+	else
+		rpc_sleep_on(&xprt->sending, task, NULL);
 	return 0;
 }
 EXPORT_SYMBOL_GPL(xprt_reserve_xprt_cong);
@@ -554,53 +569,44 @@ bool xprt_write_space(struct rpc_xprt *xprt)
 }
 EXPORT_SYMBOL_GPL(xprt_write_space);
 
-/**
- * xprt_set_retrans_timeout_def - set a request's retransmit timeout
- * @task: task whose timeout is to be set
- *
- * Set a request's retransmit timeout based on the transport's
- * default timeout parameters.  Used by transports that don't adjust
- * the retransmit timeout based on round-trip time estimation.
- */
-void xprt_set_retrans_timeout_def(struct rpc_task *task)
+static unsigned long xprt_abs_ktime_to_jiffies(ktime_t abstime)
 {
-	task->tk_timeout = task->tk_rqstp->rq_timeout;
+	s64 delta = ktime_to_ns(ktime_get() - abstime);
+	return likely(delta >= 0) ?
+		jiffies - nsecs_to_jiffies(delta) :
+		jiffies + nsecs_to_jiffies(-delta);
 }
-EXPORT_SYMBOL_GPL(xprt_set_retrans_timeout_def);
 
-/**
- * xprt_set_retrans_timeout_rtt - set a request's retransmit timeout
- * @task: task whose timeout is to be set
- *
- * Set a request's retransmit timeout using the RTT estimator.
- */
-void xprt_set_retrans_timeout_rtt(struct rpc_task *task)
+static unsigned long xprt_calc_majortimeo(struct rpc_rqst *req)
 {
-	int timer = task->tk_msg.rpc_proc->p_timer;
-	struct rpc_clnt *clnt = task->tk_client;
-	struct rpc_rtt *rtt = clnt->cl_rtt;
-	struct rpc_rqst *req = task->tk_rqstp;
-	unsigned long max_timeout = clnt->cl_timeout->to_maxval;
+	const struct rpc_timeout *to = req->rq_task->tk_client->cl_timeout;
+	unsigned long majortimeo = req->rq_timeout;
 
-	task->tk_timeout = rpc_calc_rto(rtt, timer);
-	task->tk_timeout <<= rpc_ntimeo(rtt, timer) + req->rq_retries;
-	if (task->tk_timeout > max_timeout || task->tk_timeout == 0)
-		task->tk_timeout = max_timeout;
+	if (to->to_exponential)
+		majortimeo <<= to->to_retries;
+	else
+		majortimeo += to->to_increment * to->to_retries;
+	if (majortimeo > to->to_maxval || majortimeo == 0)
+		majortimeo = to->to_maxval;
+	return majortimeo;
 }
-EXPORT_SYMBOL_GPL(xprt_set_retrans_timeout_rtt);
 
 static void xprt_reset_majortimeo(struct rpc_rqst *req)
 {
-	const struct rpc_timeout *to = req->rq_task->tk_client->cl_timeout;
+	req->rq_majortimeo += xprt_calc_majortimeo(req);
+}
 
-	req->rq_majortimeo = req->rq_timeout;
-	if (to->to_exponential)
-		req->rq_majortimeo <<= to->to_retries;
+static void xprt_init_majortimeo(struct rpc_task *task, struct rpc_rqst *req)
+{
+	unsigned long time_init;
+	struct rpc_xprt *xprt = req->rq_xprt;
+
+	if (likely(xprt && xprt_connected(xprt)))
+		time_init = jiffies;
 	else
-		req->rq_majortimeo += to->to_increment * to->to_retries;
-	if (req->rq_majortimeo > to->to_maxval || req->rq_majortimeo == 0)
-		req->rq_majortimeo = to->to_maxval;
-	req->rq_majortimeo += jiffies;
+		time_init = xprt_abs_ktime_to_jiffies(task->tk_start);
+	req->rq_timeout = task->tk_client->cl_timeout->to_initval;
+	req->rq_majortimeo = time_init + xprt_calc_majortimeo(req);
 }
 
 /**
@@ -822,9 +828,9 @@ void xprt_connect(struct rpc_task *task)
 		xprt->ops->close(xprt);
 
 	if (!xprt_connected(xprt)) {
-		task->tk_timeout = task->tk_rqstp->rq_timeout;
 		task->tk_rqstp->rq_connect_cookie = xprt->connect_cookie;
-		rpc_sleep_on(&xprt->pending, task, NULL);
+		rpc_sleep_on_timeout(&xprt->pending, task, NULL,
+				xprt_request_timeout(task->tk_rqstp));
 
 		if (test_bit(XPRT_CLOSING, &xprt->state))
 			return;
@@ -949,7 +955,7 @@ xprt_is_pinned_rqst(struct rpc_rqst *req)
  * @req: Request to pin
  *
  * Caller must ensure this is atomic with the call to xprt_lookup_rqst()
- * so should be holding the xprt receive lock.
+ * so should be holding xprt->queue_lock.
  */
 void xprt_pin_rqst(struct rpc_rqst *req)
 {
@@ -961,7 +967,7 @@ EXPORT_SYMBOL_GPL(xprt_pin_rqst);
  * xprt_unpin_rqst - Unpin a request on the transport receive list
  * @req: Request to pin
  *
- * Caller should be holding the xprt receive lock.
+ * Caller should be holding xprt->queue_lock.
  */
 void xprt_unpin_rqst(struct rpc_rqst *req)
 {
@@ -1017,7 +1023,6 @@ xprt_request_enqueue_receive(struct rpc_task *task)
 	set_bit(RPC_TASK_NEED_RECV, &task->tk_runstate);
 	spin_unlock(&xprt->queue_lock);
 
-	xprt_reset_majortimeo(req);
 	/* Turn off autodisconnect */
 	del_singleshot_timer_sync(&xprt->timer);
 }
@@ -1103,6 +1108,49 @@ static void xprt_timer(struct rpc_task *task)
 }
 
 /**
+ * xprt_wait_for_reply_request_def - wait for reply
+ * @task: pointer to rpc_task
+ *
+ * Set a request's retransmit timeout based on the transport's
+ * default timeout parameters.  Used by transports that don't adjust
+ * the retransmit timeout based on round-trip time estimation,
+ * and put the task to sleep on the pending queue.
+ */
+void xprt_wait_for_reply_request_def(struct rpc_task *task)
+{
+	struct rpc_rqst *req = task->tk_rqstp;
+
+	rpc_sleep_on_timeout(&req->rq_xprt->pending, task, xprt_timer,
+			xprt_request_timeout(req));
+}
+EXPORT_SYMBOL_GPL(xprt_wait_for_reply_request_def);
+
+/**
+ * xprt_wait_for_reply_request_rtt - wait for reply using RTT estimator
+ * @task: pointer to rpc_task
+ *
+ * Set a request's retransmit timeout using the RTT estimator,
+ * and put the task to sleep on the pending queue.
+ */
+void xprt_wait_for_reply_request_rtt(struct rpc_task *task)
+{
+	int timer = task->tk_msg.rpc_proc->p_timer;
+	struct rpc_clnt *clnt = task->tk_client;
+	struct rpc_rtt *rtt = clnt->cl_rtt;
+	struct rpc_rqst *req = task->tk_rqstp;
+	unsigned long max_timeout = clnt->cl_timeout->to_maxval;
+	unsigned long timeout;
+
+	timeout = rpc_calc_rto(rtt, timer);
+	timeout <<= rpc_ntimeo(rtt, timer) + req->rq_retries;
+	if (timeout > max_timeout || timeout == 0)
+		timeout = max_timeout;
+	rpc_sleep_on_timeout(&req->rq_xprt->pending, task, xprt_timer,
+			jiffies + timeout);
+}
+EXPORT_SYMBOL_GPL(xprt_wait_for_reply_request_rtt);
+
+/**
  * xprt_request_wait_receive - wait for the reply to an RPC request
  * @task: RPC task about to send a request
  *
@@ -1121,8 +1169,7 @@ void xprt_request_wait_receive(struct rpc_task *task)
 	 */
 	spin_lock(&xprt->queue_lock);
 	if (test_bit(RPC_TASK_NEED_RECV, &task->tk_runstate)) {
-		xprt->ops->set_retrans_timeout(task);
-		rpc_sleep_on(&xprt->pending, task, xprt_timer);
+		xprt->ops->wait_for_reply_request(task);
 		/*
 		 * Send an extra queue wakeup call if the
 		 * connection was dropped in case the call to
@@ -1337,6 +1384,10 @@ xprt_request_transmit(struct rpc_rqst *req, struct rpc_task *snd_task)
 			if (status < 0)
 				goto out_dequeue;
 		}
+		if (RPC_SIGNALLED(task)) {
+			status = -ERESTARTSYS;
+			goto out_dequeue;
+		}
 	}
 
 	/*
@@ -1605,7 +1656,6 @@ xprt_request_init(struct rpc_task *task)
 	struct rpc_xprt *xprt = task->tk_xprt;
 	struct rpc_rqst	*req = task->tk_rqstp;
 
-	req->rq_timeout = task->tk_client->cl_timeout->to_initval;
 	req->rq_task	= task;
 	req->rq_xprt    = xprt;
 	req->rq_buffer  = NULL;
@@ -1618,7 +1668,7 @@ xprt_request_init(struct rpc_task *task)
 	req->rq_snd_buf.bvec = NULL;
 	req->rq_rcv_buf.bvec = NULL;
 	req->rq_release_snd_buf = NULL;
-	xprt_reset_majortimeo(req);
+	xprt_init_majortimeo(task, req);
 	dprintk("RPC: %5u reserved req %p xid %08x\n", task->tk_pid,
 			req, ntohl(req->rq_xid));
 }
@@ -1647,7 +1697,6 @@ void xprt_reserve(struct rpc_task *task)
 	if (task->tk_rqstp != NULL)
 		return;
 
-	task->tk_timeout = 0;
 	task->tk_status = -EAGAIN;
 	if (!xprt_throttle_congested(xprt, task))
 		xprt_do_reserve(xprt, task);
@@ -1670,7 +1719,6 @@ void xprt_retry_reserve(struct rpc_task *task)
 	if (task->tk_rqstp != NULL)
 		return;
 
-	task->tk_timeout = 0;
 	task->tk_status = -EAGAIN;
 	xprt_do_reserve(xprt, task);
 }
@@ -1827,7 +1875,9 @@ found:
 		xprt->idle_timeout = 0;
 	INIT_WORK(&xprt->task_cleanup, xprt_autoclose);
 	if (xprt_has_timer(xprt))
-		timer_setup(&xprt->timer, xprt_init_autodisconnect, 0);
+		timer_setup(&xprt->timer,
+				xprt_init_autodisconnect,
+				TIMER_DEFERRABLE);
 	else
 		timer_setup(&xprt->timer, NULL, 0);
 
diff --git a/net/sunrpc/xprtrdma/backchannel.c b/net/sunrpc/xprtrdma/backchannel.c
index d79b18c1f4cd..ce986591f213 100644
--- a/net/sunrpc/xprtrdma/backchannel.c
+++ b/net/sunrpc/xprtrdma/backchannel.c
@@ -19,45 +19,6 @@
 
 #undef RPCRDMA_BACKCHANNEL_DEBUG
 
-static int rpcrdma_bc_setup_reqs(struct rpcrdma_xprt *r_xprt,
-				 unsigned int count)
-{
-	struct rpc_xprt *xprt = &r_xprt->rx_xprt;
-	struct rpcrdma_req *req;
-	struct rpc_rqst *rqst;
-	unsigned int i;
-
-	for (i = 0; i < (count << 1); i++) {
-		struct rpcrdma_regbuf *rb;
-		size_t size;
-
-		req = rpcrdma_create_req(r_xprt);
-		if (IS_ERR(req))
-			return PTR_ERR(req);
-		rqst = &req->rl_slot;
-
-		rqst->rq_xprt = xprt;
-		INIT_LIST_HEAD(&rqst->rq_bc_list);
-		__set_bit(RPC_BC_PA_IN_USE, &rqst->rq_bc_pa_state);
-		spin_lock(&xprt->bc_pa_lock);
-		list_add(&rqst->rq_bc_pa_list, &xprt->bc_pa_list);
-		spin_unlock(&xprt->bc_pa_lock);
-
-		size = r_xprt->rx_data.inline_rsize;
-		rb = rpcrdma_alloc_regbuf(size, DMA_TO_DEVICE, GFP_KERNEL);
-		if (IS_ERR(rb))
-			goto out_fail;
-		req->rl_sendbuf = rb;
-		xdr_buf_init(&rqst->rq_snd_buf, rb->rg_base,
-			     min_t(size_t, size, PAGE_SIZE));
-	}
-	return 0;
-
-out_fail:
-	rpcrdma_req_destroy(req);
-	return -ENOMEM;
-}
-
 /**
  * xprt_rdma_bc_setup - Pre-allocate resources for handling backchannel requests
  * @xprt: transport associated with these backchannel resources
@@ -68,34 +29,10 @@ out_fail:
 int xprt_rdma_bc_setup(struct rpc_xprt *xprt, unsigned int reqs)
 {
 	struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt);
-	int rc;
 
-	/* The backchannel reply path returns each rpc_rqst to the
-	 * bc_pa_list _after_ the reply is sent. If the server is
-	 * faster than the client, it can send another backward
-	 * direction request before the rpc_rqst is returned to the
-	 * list. The client rejects the request in this case.
-	 *
-	 * Twice as many rpc_rqsts are prepared to ensure there is
-	 * always an rpc_rqst available as soon as a reply is sent.
-	 */
-	if (reqs > RPCRDMA_BACKWARD_WRS >> 1)
-		goto out_err;
-
-	rc = rpcrdma_bc_setup_reqs(r_xprt, reqs);
-	if (rc)
-		goto out_free;
-
-	r_xprt->rx_buf.rb_bc_srv_max_requests = reqs;
+	r_xprt->rx_buf.rb_bc_srv_max_requests = RPCRDMA_BACKWARD_WRS >> 1;
 	trace_xprtrdma_cb_setup(r_xprt, reqs);
 	return 0;
-
-out_free:
-	xprt_rdma_bc_destroy(xprt, reqs);
-
-out_err:
-	pr_err("RPC:       %s: setup backchannel transport failed\n", __func__);
-	return -ENOMEM;
 }
 
 /**
@@ -107,10 +44,10 @@ out_err:
 size_t xprt_rdma_bc_maxpayload(struct rpc_xprt *xprt)
 {
 	struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt);
-	struct rpcrdma_create_data_internal *cdata = &r_xprt->rx_data;
+	struct rpcrdma_ep *ep = &r_xprt->rx_ep;
 	size_t maxmsg;
 
-	maxmsg = min_t(unsigned int, cdata->inline_rsize, cdata->inline_wsize);
+	maxmsg = min_t(unsigned int, ep->rep_inline_send, ep->rep_inline_recv);
 	maxmsg = min_t(unsigned int, maxmsg, PAGE_SIZE);
 	return maxmsg - RPCRDMA_HDRLEN_MIN;
 }
@@ -123,7 +60,7 @@ static int rpcrdma_bc_marshal_reply(struct rpc_rqst *rqst)
 
 	rpcrdma_set_xdrlen(&req->rl_hdrbuf, 0);
 	xdr_init_encode(&req->rl_stream, &req->rl_hdrbuf,
-			req->rl_rdmabuf->rg_base, rqst);
+			rdmab_data(req->rl_rdmabuf), rqst);
 
 	p = xdr_reserve_space(&req->rl_stream, 28);
 	if (unlikely(!p))
@@ -223,6 +160,43 @@ void xprt_rdma_bc_free_rqst(struct rpc_rqst *rqst)
 	spin_unlock(&xprt->bc_pa_lock);
 }
 
+static struct rpc_rqst *rpcrdma_bc_rqst_get(struct rpcrdma_xprt *r_xprt)
+{
+	struct rpc_xprt *xprt = &r_xprt->rx_xprt;
+	struct rpcrdma_req *req;
+	struct rpc_rqst *rqst;
+	size_t size;
+
+	spin_lock(&xprt->bc_pa_lock);
+	rqst = list_first_entry_or_null(&xprt->bc_pa_list, struct rpc_rqst,
+					rq_bc_pa_list);
+	if (!rqst)
+		goto create_req;
+	list_del(&rqst->rq_bc_pa_list);
+	spin_unlock(&xprt->bc_pa_lock);
+	return rqst;
+
+create_req:
+	spin_unlock(&xprt->bc_pa_lock);
+
+	/* Set a limit to prevent a remote from overrunning our resources.
+	 */
+	if (xprt->bc_alloc_count >= RPCRDMA_BACKWARD_WRS)
+		return NULL;
+
+	size = min_t(size_t, r_xprt->rx_ep.rep_inline_recv, PAGE_SIZE);
+	req = rpcrdma_req_create(r_xprt, size, GFP_KERNEL);
+	if (!req)
+		return NULL;
+
+	xprt->bc_alloc_count++;
+	rqst = &req->rl_slot;
+	rqst->rq_xprt = xprt;
+	__set_bit(RPC_BC_PA_IN_USE, &rqst->rq_bc_pa_state);
+	xdr_buf_init(&rqst->rq_snd_buf, rdmab_data(req->rl_sendbuf), size);
+	return rqst;
+}
+
 /**
  * rpcrdma_bc_receive_call - Handle a backward direction call
  * @r_xprt: transport receiving the call
@@ -254,18 +228,10 @@ void rpcrdma_bc_receive_call(struct rpcrdma_xprt *r_xprt,
 	pr_info("RPC:       %s: %*ph\n", __func__, size, p);
 #endif
 
-	/* Grab a free bc rqst */
-	spin_lock(&xprt->bc_pa_lock);
-	if (list_empty(&xprt->bc_pa_list)) {
-		spin_unlock(&xprt->bc_pa_lock);
+	rqst = rpcrdma_bc_rqst_get(r_xprt);
+	if (!rqst)
 		goto out_overflow;
-	}
-	rqst = list_first_entry(&xprt->bc_pa_list,
-				struct rpc_rqst, rq_bc_pa_list);
-	list_del(&rqst->rq_bc_pa_list);
-	spin_unlock(&xprt->bc_pa_lock);
 
-	/* Prepare rqst */
 	rqst->rq_reply_bytes_recvd = 0;
 	rqst->rq_xid = *p;
 
diff --git a/net/sunrpc/xprtrdma/frwr_ops.c b/net/sunrpc/xprtrdma/frwr_ops.c
index 52cb6c1b0c2b..794ba4ca0994 100644
--- a/net/sunrpc/xprtrdma/frwr_ops.c
+++ b/net/sunrpc/xprtrdma/frwr_ops.c
@@ -82,13 +82,13 @@
 
 /**
  * frwr_is_supported - Check if device supports FRWR
- * @ia: interface adapter to check
+ * @device: interface adapter to check
  *
  * Returns true if device supports FRWR, otherwise false
  */
-bool frwr_is_supported(struct rpcrdma_ia *ia)
+bool frwr_is_supported(struct ib_device *device)
 {
-	struct ib_device_attr *attrs = &ia->ri_device->attrs;
+	struct ib_device_attr *attrs = &device->attrs;
 
 	if (!(attrs->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS))
 		goto out_not_supported;
@@ -98,7 +98,7 @@ bool frwr_is_supported(struct rpcrdma_ia *ia)
 
 out_not_supported:
 	pr_info("rpcrdma: 'frwr' mode is not supported by device %s\n",
-		ia->ri_device->name);
+		device->name);
 	return false;
 }
 
@@ -131,7 +131,7 @@ frwr_mr_recycle_worker(struct work_struct *work)
 
 	if (mr->mr_dir != DMA_NONE) {
 		trace_xprtrdma_mr_unmap(mr);
-		ib_dma_unmap_sg(r_xprt->rx_ia.ri_device,
+		ib_dma_unmap_sg(r_xprt->rx_ia.ri_id->device,
 				mr->mr_sg, mr->mr_nents, mr->mr_dir);
 		mr->mr_dir = DMA_NONE;
 	}
@@ -194,12 +194,11 @@ out_list_err:
  * frwr_open - Prepare an endpoint for use with FRWR
  * @ia: interface adapter this endpoint will use
  * @ep: endpoint to prepare
- * @cdata: transport parameters
  *
  * On success, sets:
  *	ep->rep_attr.cap.max_send_wr
  *	ep->rep_attr.cap.max_recv_wr
- *	cdata->max_requests
+ *	ep->rep_max_requests
  *	ia->ri_max_segs
  *
  * And these FRWR-related fields:
@@ -208,10 +207,9 @@ out_list_err:
  *
  * On failure, a negative errno is returned.
  */
-int frwr_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep,
-	      struct rpcrdma_create_data_internal *cdata)
+int frwr_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep)
 {
-	struct ib_device_attr *attrs = &ia->ri_device->attrs;
+	struct ib_device_attr *attrs = &ia->ri_id->device->attrs;
 	int max_qp_wr, depth, delta;
 
 	ia->ri_mrtype = IB_MR_TYPE_MEM_REG;
@@ -253,24 +251,23 @@ int frwr_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep,
 		} while (delta > 0);
 	}
 
-	max_qp_wr = ia->ri_device->attrs.max_qp_wr;
+	max_qp_wr = ia->ri_id->device->attrs.max_qp_wr;
 	max_qp_wr -= RPCRDMA_BACKWARD_WRS;
 	max_qp_wr -= 1;
 	if (max_qp_wr < RPCRDMA_MIN_SLOT_TABLE)
 		return -ENOMEM;
-	if (cdata->max_requests > max_qp_wr)
-		cdata->max_requests = max_qp_wr;
-	ep->rep_attr.cap.max_send_wr = cdata->max_requests * depth;
+	if (ep->rep_max_requests > max_qp_wr)
+		ep->rep_max_requests = max_qp_wr;
+	ep->rep_attr.cap.max_send_wr = ep->rep_max_requests * depth;
 	if (ep->rep_attr.cap.max_send_wr > max_qp_wr) {
-		cdata->max_requests = max_qp_wr / depth;
-		if (!cdata->max_requests)
+		ep->rep_max_requests = max_qp_wr / depth;
+		if (!ep->rep_max_requests)
 			return -EINVAL;
-		ep->rep_attr.cap.max_send_wr = cdata->max_requests *
-					       depth;
+		ep->rep_attr.cap.max_send_wr = ep->rep_max_requests * depth;
 	}
 	ep->rep_attr.cap.max_send_wr += RPCRDMA_BACKWARD_WRS;
 	ep->rep_attr.cap.max_send_wr += 1; /* for ib_drain_sq */
-	ep->rep_attr.cap.max_recv_wr = cdata->max_requests;
+	ep->rep_attr.cap.max_recv_wr = ep->rep_max_requests;
 	ep->rep_attr.cap.max_recv_wr += RPCRDMA_BACKWARD_WRS;
 	ep->rep_attr.cap.max_recv_wr += 1; /* for ib_drain_rq */
 
@@ -300,15 +297,6 @@ size_t frwr_maxpages(struct rpcrdma_xprt *r_xprt)
 		     (ia->ri_max_segs - 2) * ia->ri_max_frwr_depth);
 }
 
-static void
-__frwr_sendcompletion_flush(struct ib_wc *wc, const char *wr)
-{
-	if (wc->status != IB_WC_WR_FLUSH_ERR)
-		pr_err("rpcrdma: %s: %s (%u/0x%x)\n",
-		       wr, ib_wc_status_msg(wc->status),
-		       wc->status, wc->vendor_err);
-}
-
 /**
  * frwr_wc_fastreg - Invoked by RDMA provider for a flushed FastReg WC
  * @cq:	completion queue (ignored)
@@ -323,10 +311,8 @@ frwr_wc_fastreg(struct ib_cq *cq, struct ib_wc *wc)
 			container_of(cqe, struct rpcrdma_frwr, fr_cqe);
 
 	/* WARNING: Only wr_cqe and status are reliable at this point */
-	if (wc->status != IB_WC_SUCCESS) {
+	if (wc->status != IB_WC_SUCCESS)
 		frwr->fr_state = FRWR_FLUSHED_FR;
-		__frwr_sendcompletion_flush(wc, "fastreg");
-	}
 	trace_xprtrdma_wc_fastreg(wc, frwr);
 }
 
@@ -344,10 +330,8 @@ frwr_wc_localinv(struct ib_cq *cq, struct ib_wc *wc)
 						 fr_cqe);
 
 	/* WARNING: Only wr_cqe and status are reliable at this point */
-	if (wc->status != IB_WC_SUCCESS) {
+	if (wc->status != IB_WC_SUCCESS)
 		frwr->fr_state = FRWR_FLUSHED_LI;
-		__frwr_sendcompletion_flush(wc, "localinv");
-	}
 	trace_xprtrdma_wc_li(wc, frwr);
 }
 
@@ -366,12 +350,10 @@ frwr_wc_localinv_wake(struct ib_cq *cq, struct ib_wc *wc)
 						 fr_cqe);
 
 	/* WARNING: Only wr_cqe and status are reliable at this point */
-	if (wc->status != IB_WC_SUCCESS) {
+	if (wc->status != IB_WC_SUCCESS)
 		frwr->fr_state = FRWR_FLUSHED_LI;
-		__frwr_sendcompletion_flush(wc, "localinv");
-	}
-	complete(&frwr->fr_linv_done);
 	trace_xprtrdma_wc_li_wake(wc, frwr);
+	complete(&frwr->fr_linv_done);
 }
 
 /**
@@ -436,7 +418,8 @@ struct rpcrdma_mr_seg *frwr_map(struct rpcrdma_xprt *r_xprt,
 	}
 	mr->mr_dir = rpcrdma_data_dir(writing);
 
-	mr->mr_nents = ib_dma_map_sg(ia->ri_device, mr->mr_sg, i, mr->mr_dir);
+	mr->mr_nents =
+		ib_dma_map_sg(ia->ri_id->device, mr->mr_sg, i, mr->mr_dir);
 	if (!mr->mr_nents)
 		goto out_dmamap_err;
 
@@ -466,7 +449,7 @@ struct rpcrdma_mr_seg *frwr_map(struct rpcrdma_xprt *r_xprt,
 	return seg;
 
 out_dmamap_err:
-	frwr->fr_state = FRWR_IS_INVALID;
+	mr->mr_dir = DMA_NONE;
 	trace_xprtrdma_frwr_sgerr(mr, i);
 	rpcrdma_mr_put(mr);
 	return ERR_PTR(-EIO);
diff --git a/net/sunrpc/xprtrdma/rpc_rdma.c b/net/sunrpc/xprtrdma/rpc_rdma.c
index 6c1fb270f127..85115a2e2639 100644
--- a/net/sunrpc/xprtrdma/rpc_rdma.c
+++ b/net/sunrpc/xprtrdma/rpc_rdma.c
@@ -105,16 +105,23 @@ static unsigned int rpcrdma_max_reply_header_size(unsigned int maxsegs)
 	return size;
 }
 
+/**
+ * rpcrdma_set_max_header_sizes - Initialize inline payload sizes
+ * @r_xprt: transport instance to initialize
+ *
+ * The max_inline fields contain the maximum size of an RPC message
+ * so the marshaling code doesn't have to repeat this calculation
+ * for every RPC.
+ */
 void rpcrdma_set_max_header_sizes(struct rpcrdma_xprt *r_xprt)
 {
-	struct rpcrdma_create_data_internal *cdata = &r_xprt->rx_data;
-	struct rpcrdma_ia *ia = &r_xprt->rx_ia;
-	unsigned int maxsegs = ia->ri_max_segs;
-
-	ia->ri_max_inline_write = cdata->inline_wsize -
-				  rpcrdma_max_call_header_size(maxsegs);
-	ia->ri_max_inline_read = cdata->inline_rsize -
-				 rpcrdma_max_reply_header_size(maxsegs);
+	unsigned int maxsegs = r_xprt->rx_ia.ri_max_segs;
+	struct rpcrdma_ep *ep = &r_xprt->rx_ep;
+
+	ep->rep_max_inline_send =
+		ep->rep_inline_send - rpcrdma_max_call_header_size(maxsegs);
+	ep->rep_max_inline_recv =
+		ep->rep_inline_recv - rpcrdma_max_reply_header_size(maxsegs);
 }
 
 /* The client can send a request inline as long as the RPCRDMA header
@@ -131,7 +138,7 @@ static bool rpcrdma_args_inline(struct rpcrdma_xprt *r_xprt,
 	struct xdr_buf *xdr = &rqst->rq_snd_buf;
 	unsigned int count, remaining, offset;
 
-	if (xdr->len > r_xprt->rx_ia.ri_max_inline_write)
+	if (xdr->len > r_xprt->rx_ep.rep_max_inline_send)
 		return false;
 
 	if (xdr->page_len) {
@@ -159,9 +166,7 @@ static bool rpcrdma_args_inline(struct rpcrdma_xprt *r_xprt,
 static bool rpcrdma_results_inline(struct rpcrdma_xprt *r_xprt,
 				   struct rpc_rqst *rqst)
 {
-	struct rpcrdma_ia *ia = &r_xprt->rx_ia;
-
-	return rqst->rq_rcv_buf.buflen <= ia->ri_max_inline_read;
+	return rqst->rq_rcv_buf.buflen <= r_xprt->rx_ep.rep_max_inline_recv;
 }
 
 /* The client is required to provide a Reply chunk if the maximum
@@ -173,10 +178,9 @@ rpcrdma_nonpayload_inline(const struct rpcrdma_xprt *r_xprt,
 			  const struct rpc_rqst *rqst)
 {
 	const struct xdr_buf *buf = &rqst->rq_rcv_buf;
-	const struct rpcrdma_ia *ia = &r_xprt->rx_ia;
 
-	return buf->head[0].iov_len + buf->tail[0].iov_len <
-		ia->ri_max_inline_read;
+	return (buf->head[0].iov_len + buf->tail[0].iov_len) <
+		r_xprt->rx_ep.rep_max_inline_recv;
 }
 
 /* Split @vec on page boundaries into SGEs. FMR registers pages, not
@@ -238,7 +242,7 @@ rpcrdma_convert_iovs(struct rpcrdma_xprt *r_xprt, struct xdr_buf *xdrbuf,
 		 */
 		if (unlikely(xdrbuf->flags & XDRBUF_SPARSE_PAGES)) {
 			if (!*ppages)
-				*ppages = alloc_page(GFP_ATOMIC);
+				*ppages = alloc_page(GFP_NOWAIT | __GFP_NOWARN);
 			if (!*ppages)
 				return -ENOBUFS;
 		}
@@ -508,50 +512,45 @@ rpcrdma_encode_reply_chunk(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req,
 }
 
 /**
- * rpcrdma_unmap_sendctx - DMA-unmap Send buffers
+ * rpcrdma_sendctx_unmap - DMA-unmap Send buffer
  * @sc: sendctx containing SGEs to unmap
  *
  */
-void
-rpcrdma_unmap_sendctx(struct rpcrdma_sendctx *sc)
+void rpcrdma_sendctx_unmap(struct rpcrdma_sendctx *sc)
 {
-	struct rpcrdma_ia *ia = &sc->sc_xprt->rx_ia;
 	struct ib_sge *sge;
-	unsigned int count;
 
 	/* The first two SGEs contain the transport header and
 	 * the inline buffer. These are always left mapped so
 	 * they can be cheaply re-used.
 	 */
-	sge = &sc->sc_sges[2];
-	for (count = sc->sc_unmap_count; count; ++sge, --count)
-		ib_dma_unmap_page(ia->ri_device,
-				  sge->addr, sge->length, DMA_TO_DEVICE);
+	for (sge = &sc->sc_sges[2]; sc->sc_unmap_count;
+	     ++sge, --sc->sc_unmap_count)
+		ib_dma_unmap_page(sc->sc_device, sge->addr, sge->length,
+				  DMA_TO_DEVICE);
 
-	if (test_and_clear_bit(RPCRDMA_REQ_F_TX_RESOURCES, &sc->sc_req->rl_flags)) {
-		smp_mb__after_atomic();
+	if (test_and_clear_bit(RPCRDMA_REQ_F_TX_RESOURCES,
+			       &sc->sc_req->rl_flags))
 		wake_up_bit(&sc->sc_req->rl_flags, RPCRDMA_REQ_F_TX_RESOURCES);
-	}
 }
 
 /* Prepare an SGE for the RPC-over-RDMA transport header.
  */
-static bool
-rpcrdma_prepare_hdr_sge(struct rpcrdma_ia *ia, struct rpcrdma_req *req,
-			u32 len)
+static bool rpcrdma_prepare_hdr_sge(struct rpcrdma_xprt *r_xprt,
+				    struct rpcrdma_req *req, u32 len)
 {
 	struct rpcrdma_sendctx *sc = req->rl_sendctx;
 	struct rpcrdma_regbuf *rb = req->rl_rdmabuf;
 	struct ib_sge *sge = sc->sc_sges;
 
-	if (!rpcrdma_dma_map_regbuf(ia, rb))
+	if (!rpcrdma_regbuf_dma_map(r_xprt, rb))
 		goto out_regbuf;
 	sge->addr = rdmab_addr(rb);
 	sge->length = len;
 	sge->lkey = rdmab_lkey(rb);
 
-	ib_dma_sync_single_for_device(rdmab_device(rb), sge->addr,
-				      sge->length, DMA_TO_DEVICE);
+	ib_dma_sync_single_for_device(rdmab_device(rb), sge->addr, sge->length,
+				      DMA_TO_DEVICE);
 	sc->sc_wr.num_sge++;
 	return true;
 
@@ -563,23 +562,23 @@ out_regbuf:
 /* Prepare the Send SGEs. The head and tail iovec, and each entry
  * in the page list, gets its own SGE.
  */
-static bool
-rpcrdma_prepare_msg_sges(struct rpcrdma_ia *ia, struct rpcrdma_req *req,
-			 struct xdr_buf *xdr, enum rpcrdma_chunktype rtype)
+static bool rpcrdma_prepare_msg_sges(struct rpcrdma_xprt *r_xprt,
+				     struct rpcrdma_req *req,
+				     struct xdr_buf *xdr,
+				     enum rpcrdma_chunktype rtype)
 {
 	struct rpcrdma_sendctx *sc = req->rl_sendctx;
 	unsigned int sge_no, page_base, len, remaining;
 	struct rpcrdma_regbuf *rb = req->rl_sendbuf;
-	struct ib_device *device = ia->ri_device;
 	struct ib_sge *sge = sc->sc_sges;
-	u32 lkey = ia->ri_pd->local_dma_lkey;
 	struct page *page, **ppages;
 
 	/* The head iovec is straightforward, as it is already
 	 * DMA-mapped. Sync the content that has changed.
 	 */
-	if (!rpcrdma_dma_map_regbuf(ia, rb))
+	if (!rpcrdma_regbuf_dma_map(r_xprt, rb))
 		goto out_regbuf;
+	sc->sc_device = rdmab_device(rb);
 	sge_no = 1;
 	sge[sge_no].addr = rdmab_addr(rb);
 	sge[sge_no].length = xdr->head[0].iov_len;
@@ -626,13 +625,14 @@ rpcrdma_prepare_msg_sges(struct rpcrdma_ia *ia, struct rpcrdma_req *req,
 				goto out_mapping_overflow;
 
 			len = min_t(u32, PAGE_SIZE - page_base, remaining);
-			sge[sge_no].addr = ib_dma_map_page(device, *ppages,
-							   page_base, len,
-							   DMA_TO_DEVICE);
-			if (ib_dma_mapping_error(device, sge[sge_no].addr))
+			sge[sge_no].addr =
+				ib_dma_map_page(rdmab_device(rb), *ppages,
+						page_base, len, DMA_TO_DEVICE);
+			if (ib_dma_mapping_error(rdmab_device(rb),
+						 sge[sge_no].addr))
 				goto out_mapping_err;
 			sge[sge_no].length = len;
-			sge[sge_no].lkey = lkey;
+			sge[sge_no].lkey = rdmab_lkey(rb);
 
 			sc->sc_unmap_count++;
 			ppages++;
@@ -653,13 +653,13 @@ rpcrdma_prepare_msg_sges(struct rpcrdma_ia *ia, struct rpcrdma_req *req,
 
 map_tail:
 		sge_no++;
-		sge[sge_no].addr = ib_dma_map_page(device, page,
-						   page_base, len,
-						   DMA_TO_DEVICE);
-		if (ib_dma_mapping_error(device, sge[sge_no].addr))
+		sge[sge_no].addr =
+			ib_dma_map_page(rdmab_device(rb), page, page_base, len,
+					DMA_TO_DEVICE);
+		if (ib_dma_mapping_error(rdmab_device(rb), sge[sge_no].addr))
 			goto out_mapping_err;
 		sge[sge_no].length = len;
-		sge[sge_no].lkey = lkey;
+		sge[sge_no].lkey = rdmab_lkey(rb);
 		sc->sc_unmap_count++;
 	}
 
@@ -674,12 +674,12 @@ out_regbuf:
 	return false;
 
 out_mapping_overflow:
-	rpcrdma_unmap_sendctx(sc);
+	rpcrdma_sendctx_unmap(sc);
 	pr_err("rpcrdma: too many Send SGEs (%u)\n", sge_no);
 	return false;
 
 out_mapping_err:
-	rpcrdma_unmap_sendctx(sc);
+	rpcrdma_sendctx_unmap(sc);
 	trace_xprtrdma_dma_maperr(sge[sge_no].addr);
 	return false;
 }
@@ -699,7 +699,7 @@ rpcrdma_prepare_send_sges(struct rpcrdma_xprt *r_xprt,
 			  struct rpcrdma_req *req, u32 hdrlen,
 			  struct xdr_buf *xdr, enum rpcrdma_chunktype rtype)
 {
-	req->rl_sendctx = rpcrdma_sendctx_get_locked(&r_xprt->rx_buf);
+	req->rl_sendctx = rpcrdma_sendctx_get_locked(r_xprt);
 	if (!req->rl_sendctx)
 		return -EAGAIN;
 	req->rl_sendctx->sc_wr.num_sge = 0;
@@ -707,11 +707,11 @@ rpcrdma_prepare_send_sges(struct rpcrdma_xprt *r_xprt,
 	req->rl_sendctx->sc_req = req;
 	__clear_bit(RPCRDMA_REQ_F_TX_RESOURCES, &req->rl_flags);
 
-	if (!rpcrdma_prepare_hdr_sge(&r_xprt->rx_ia, req, hdrlen))
+	if (!rpcrdma_prepare_hdr_sge(r_xprt, req, hdrlen))
 		return -EIO;
 
 	if (rtype != rpcrdma_areadch)
-		if (!rpcrdma_prepare_msg_sges(&r_xprt->rx_ia, req, xdr, rtype))
+		if (!rpcrdma_prepare_msg_sges(r_xprt, req, xdr, rtype))
 			return -EIO;
 
 	return 0;
@@ -747,8 +747,8 @@ rpcrdma_marshal_req(struct rpcrdma_xprt *r_xprt, struct rpc_rqst *rqst)
 	int ret;
 
 	rpcrdma_set_xdrlen(&req->rl_hdrbuf, 0);
-	xdr_init_encode(xdr, &req->rl_hdrbuf,
-			req->rl_rdmabuf->rg_base, rqst);
+	xdr_init_encode(xdr, &req->rl_hdrbuf, rdmab_data(req->rl_rdmabuf),
+			rqst);
 
 	/* Fixed header fields */
 	ret = -EMSGSIZE;
@@ -876,6 +876,7 @@ rpcrdma_marshal_req(struct rpcrdma_xprt *r_xprt, struct rpc_rqst *rqst)
 	return 0;
 
 out_err:
+	trace_xprtrdma_marshal_failed(rqst, ret);
 	switch (ret) {
 	case -EAGAIN:
 		xprt_wait_for_buffer_space(rqst->rq_xprt);
diff --git a/net/sunrpc/xprtrdma/svc_rdma_backchannel.c b/net/sunrpc/xprtrdma/svc_rdma_backchannel.c
index 907464c2a9f0..bed57d8b5c19 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_backchannel.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_backchannel.c
@@ -261,7 +261,7 @@ static const struct rpc_xprt_ops xprt_rdma_bc_procs = {
 	.buf_alloc		= xprt_rdma_bc_allocate,
 	.buf_free		= xprt_rdma_bc_free,
 	.send_request		= xprt_rdma_bc_send_request,
-	.set_retrans_timeout	= xprt_set_retrans_timeout_def,
+	.wait_for_reply_request	= xprt_wait_for_reply_request_def,
 	.close			= xprt_rdma_bc_close,
 	.destroy		= xprt_rdma_bc_put,
 	.print_stats		= xprt_rdma_print_stats
diff --git a/net/sunrpc/xprtrdma/transport.c b/net/sunrpc/xprtrdma/transport.c
index 5d261353bd90..1f73a6a7e43c 100644
--- a/net/sunrpc/xprtrdma/transport.c
+++ b/net/sunrpc/xprtrdma/transport.c
@@ -68,9 +68,9 @@
  * tunables
  */
 
-static unsigned int xprt_rdma_slot_table_entries = RPCRDMA_DEF_SLOT_TABLE;
+unsigned int xprt_rdma_slot_table_entries = RPCRDMA_DEF_SLOT_TABLE;
 unsigned int xprt_rdma_max_inline_read = RPCRDMA_DEF_INLINE;
-static unsigned int xprt_rdma_max_inline_write = RPCRDMA_DEF_INLINE;
+unsigned int xprt_rdma_max_inline_write = RPCRDMA_DEF_INLINE;
 unsigned int xprt_rdma_memreg_strategy		= RPCRDMA_FRWR;
 int xprt_rdma_pad_optimize;
 
@@ -288,7 +288,7 @@ xprt_rdma_destroy(struct rpc_xprt *xprt)
 
 	cancel_delayed_work_sync(&r_xprt->rx_connect_worker);
 
-	rpcrdma_ep_destroy(&r_xprt->rx_ep, &r_xprt->rx_ia);
+	rpcrdma_ep_destroy(r_xprt);
 	rpcrdma_buffer_destroy(&r_xprt->rx_buf);
 	rpcrdma_ia_close(&r_xprt->rx_ia);
 
@@ -311,10 +311,8 @@ static const struct rpc_timeout xprt_rdma_default_timeout = {
 static struct rpc_xprt *
 xprt_setup_rdma(struct xprt_create *args)
 {
-	struct rpcrdma_create_data_internal cdata;
 	struct rpc_xprt *xprt;
 	struct rpcrdma_xprt *new_xprt;
-	struct rpcrdma_ep *new_ep;
 	struct sockaddr *sap;
 	int rc;
 
@@ -349,40 +347,12 @@ xprt_setup_rdma(struct xprt_create *args)
 		xprt_set_bound(xprt);
 	xprt_rdma_format_addresses(xprt, sap);
 
-	cdata.max_requests = xprt_rdma_slot_table_entries;
-
-	cdata.rsize = RPCRDMA_MAX_SEGS * PAGE_SIZE; /* RDMA write max */
-	cdata.wsize = RPCRDMA_MAX_SEGS * PAGE_SIZE; /* RDMA read max */
-
-	cdata.inline_wsize = xprt_rdma_max_inline_write;
-	if (cdata.inline_wsize > cdata.wsize)
-		cdata.inline_wsize = cdata.wsize;
-
-	cdata.inline_rsize = xprt_rdma_max_inline_read;
-	if (cdata.inline_rsize > cdata.rsize)
-		cdata.inline_rsize = cdata.rsize;
-
-	/*
-	 * Create new transport instance, which includes initialized
-	 *  o ia
-	 *  o endpoint
-	 *  o buffers
-	 */
-
 	new_xprt = rpcx_to_rdmax(xprt);
-
 	rc = rpcrdma_ia_open(new_xprt);
 	if (rc)
 		goto out1;
 
-	/*
-	 * initialize and create ep
-	 */
-	new_xprt->rx_data = cdata;
-	new_ep = &new_xprt->rx_ep;
-
-	rc = rpcrdma_ep_create(&new_xprt->rx_ep,
-				&new_xprt->rx_ia, &new_xprt->rx_data);
+	rc = rpcrdma_ep_create(new_xprt);
 	if (rc)
 		goto out2;
 
@@ -413,7 +383,7 @@ out4:
 	rpcrdma_buffer_destroy(&new_xprt->rx_buf);
 	rc = -ENODEV;
 out3:
-	rpcrdma_ep_destroy(new_ep, &new_xprt->rx_ia);
+	rpcrdma_ep_destroy(new_xprt);
 out2:
 	rpcrdma_ia_close(&new_xprt->rx_ia);
 out1:
@@ -585,52 +555,15 @@ xprt_rdma_free_slot(struct rpc_xprt *xprt, struct rpc_rqst *rqst)
 	rpc_wake_up_next(&xprt->backlog);
 }
 
-static bool
-rpcrdma_get_sendbuf(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req,
-		    size_t size, gfp_t flags)
+static bool rpcrdma_check_regbuf(struct rpcrdma_xprt *r_xprt,
+				 struct rpcrdma_regbuf *rb, size_t size,
+				 gfp_t flags)
 {
-	struct rpcrdma_regbuf *rb;
-
-	if (req->rl_sendbuf && rdmab_length(req->rl_sendbuf) >= size)
-		return true;
-
-	rb = rpcrdma_alloc_regbuf(size, DMA_TO_DEVICE, flags);
-	if (IS_ERR(rb))
-		return false;
-
-	rpcrdma_free_regbuf(req->rl_sendbuf);
-	r_xprt->rx_stats.hardway_register_count += size;
-	req->rl_sendbuf = rb;
-	return true;
-}
-
-/* The rq_rcv_buf is used only if a Reply chunk is necessary.
- * The decision to use a Reply chunk is made later in
- * rpcrdma_marshal_req. This buffer is registered at that time.
- *
- * Otherwise, the associated RPC Reply arrives in a separate
- * Receive buffer, arbitrarily chosen by the HCA. The buffer
- * allocated here for the RPC Reply is not utilized in that
- * case. See rpcrdma_inline_fixup.
- *
- * A regbuf is used here to remember the buffer size.
- */
-static bool
-rpcrdma_get_recvbuf(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req,
-		    size_t size, gfp_t flags)
-{
-	struct rpcrdma_regbuf *rb;
-
-	if (req->rl_recvbuf && rdmab_length(req->rl_recvbuf) >= size)
-		return true;
-
-	rb = rpcrdma_alloc_regbuf(size, DMA_NONE, flags);
-	if (IS_ERR(rb))
-		return false;
-
-	rpcrdma_free_regbuf(req->rl_recvbuf);
-	r_xprt->rx_stats.hardway_register_count += size;
-	req->rl_recvbuf = rb;
+	if (unlikely(rdmab_length(rb) < size)) {
+		if (!rpcrdma_regbuf_realloc(rb, size, flags))
+			return false;
+		r_xprt->rx_stats.hardway_register_count += size;
+	}
 	return true;
 }
 
@@ -655,13 +588,15 @@ xprt_rdma_allocate(struct rpc_task *task)
 	if (RPC_IS_SWAPPER(task))
 		flags = __GFP_MEMALLOC | GFP_NOWAIT | __GFP_NOWARN;
 
-	if (!rpcrdma_get_sendbuf(r_xprt, req, rqst->rq_callsize, flags))
+	if (!rpcrdma_check_regbuf(r_xprt, req->rl_sendbuf, rqst->rq_callsize,
+				  flags))
 		goto out_fail;
-	if (!rpcrdma_get_recvbuf(r_xprt, req, rqst->rq_rcvsize, flags))
+	if (!rpcrdma_check_regbuf(r_xprt, req->rl_recvbuf, rqst->rq_rcvsize,
+				  flags))
 		goto out_fail;
 
-	rqst->rq_buffer = req->rl_sendbuf->rg_base;
-	rqst->rq_rbuffer = req->rl_recvbuf->rg_base;
+	rqst->rq_buffer = rdmab_data(req->rl_sendbuf);
+	rqst->rq_rbuffer = rdmab_data(req->rl_recvbuf);
 	trace_xprtrdma_op_allocate(task, req);
 	return 0;
 
@@ -815,7 +750,7 @@ static const struct rpc_xprt_ops xprt_rdma_procs = {
 	.alloc_slot		= xprt_rdma_alloc_slot,
 	.free_slot		= xprt_rdma_free_slot,
 	.release_request	= xprt_release_rqst_cong,       /* ditto */
-	.set_retrans_timeout	= xprt_set_retrans_timeout_def, /* ditto */
+	.wait_for_reply_request	= xprt_wait_for_reply_request_def, /* ditto */
 	.timer			= xprt_rdma_timer,
 	.rpcbind		= rpcb_getport_async,	/* sunrpc/rpcb_clnt.c */
 	.set_port		= xprt_rdma_set_port,
diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c
index 30cfc0efe699..bef5eac8ab38 100644
--- a/net/sunrpc/xprtrdma/verbs.c
+++ b/net/sunrpc/xprtrdma/verbs.c
@@ -76,11 +76,16 @@
 static void rpcrdma_sendctx_put_locked(struct rpcrdma_sendctx *sc);
 static void rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt);
 static void rpcrdma_mrs_destroy(struct rpcrdma_buffer *buf);
-static int rpcrdma_create_rep(struct rpcrdma_xprt *r_xprt, bool temp);
-static void rpcrdma_dma_unmap_regbuf(struct rpcrdma_regbuf *rb);
+static struct rpcrdma_regbuf *
+rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction,
+		     gfp_t flags);
+static void rpcrdma_regbuf_dma_unmap(struct rpcrdma_regbuf *rb);
+static void rpcrdma_regbuf_free(struct rpcrdma_regbuf *rb);
 static void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, bool temp);
 
-/* Wait for outstanding transport work to finish.
+/* Wait for outstanding transport work to finish. ib_drain_qp
+ * handles the drains in the wrong order for us, so open code
+ * them here.
  */
 static void rpcrdma_xprt_drain(struct rpcrdma_xprt *r_xprt)
 {
@@ -132,11 +137,6 @@ rpcrdma_wc_send(struct ib_cq *cq, struct ib_wc *wc)
 
 	/* WARNING: Only wr_cqe and status are reliable at this point */
 	trace_xprtrdma_wc_send(sc, wc);
-	if (wc->status != IB_WC_SUCCESS && wc->status != IB_WC_WR_FLUSH_ERR)
-		pr_err("rpcrdma: Send: %s (%u/0x%x)\n",
-		       ib_wc_status_msg(wc->status),
-		       wc->status, wc->vendor_err);
-
 	rpcrdma_sendctx_put_locked(sc);
 }
 
@@ -174,10 +174,6 @@ rpcrdma_wc_receive(struct ib_cq *cq, struct ib_wc *wc)
 	return;
 
 out_flushed:
-	if (wc->status != IB_WC_WR_FLUSH_ERR)
-		pr_err("rpcrdma: Recv: %s (%u/0x%x)\n",
-		       ib_wc_status_msg(wc->status),
-		       wc->status, wc->vendor_err);
 	rpcrdma_recv_buffer_put(rep);
 }
 
@@ -185,7 +181,6 @@ static void
 rpcrdma_update_connect_private(struct rpcrdma_xprt *r_xprt,
 			       struct rdma_conn_param *param)
 {
-	struct rpcrdma_create_data_internal *cdata = &r_xprt->rx_data;
 	const struct rpcrdma_connect_private *pmsg = param->private_data;
 	unsigned int rsize, wsize;
 
@@ -202,12 +197,13 @@ rpcrdma_update_connect_private(struct rpcrdma_xprt *r_xprt,
 		wsize = rpcrdma_decode_buffer_size(pmsg->cp_recv_size);
 	}
 
-	if (rsize < cdata->inline_rsize)
-		cdata->inline_rsize = rsize;
-	if (wsize < cdata->inline_wsize)
-		cdata->inline_wsize = wsize;
-	dprintk("RPC:       %s: max send %u, max recv %u\n",
-		__func__, cdata->inline_wsize, cdata->inline_rsize);
+	if (rsize < r_xprt->rx_ep.rep_inline_recv)
+		r_xprt->rx_ep.rep_inline_recv = rsize;
+	if (wsize < r_xprt->rx_ep.rep_inline_send)
+		r_xprt->rx_ep.rep_inline_send = wsize;
+	dprintk("RPC:       %s: max send %u, max recv %u\n", __func__,
+		r_xprt->rx_ep.rep_inline_send,
+		r_xprt->rx_ep.rep_inline_recv);
 	rpcrdma_set_max_header_sizes(r_xprt);
 }
 
@@ -247,7 +243,7 @@ rpcrdma_cm_event_handler(struct rdma_cm_id *id, struct rdma_cm_event *event)
 	case RDMA_CM_EVENT_DEVICE_REMOVAL:
 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
 		pr_info("rpcrdma: removing device %s for %s:%s\n",
-			ia->ri_device->name,
+			ia->ri_id->device->name,
 			rpcrdma_addrstr(r_xprt), rpcrdma_portstr(r_xprt));
 #endif
 		set_bit(RPCRDMA_IAF_REMOVING, &ia->ri_flags);
@@ -256,7 +252,6 @@ rpcrdma_cm_event_handler(struct rdma_cm_id *id, struct rdma_cm_event *event)
 		wait_for_completion(&ia->ri_remove_done);
 
 		ia->ri_id = NULL;
-		ia->ri_device = NULL;
 		/* Return 1 to ensure the core destroys the id. */
 		return 1;
 	case RDMA_CM_EVENT_ESTABLISHED:
@@ -291,7 +286,7 @@ disconnected:
 
 	dprintk("RPC:       %s: %s:%s on %s/frwr: %s\n", __func__,
 		rpcrdma_addrstr(r_xprt), rpcrdma_portstr(r_xprt),
-		ia->ri_device->name, rdma_event_msg(event->event));
+		ia->ri_id->device->name, rdma_event_msg(event->event));
 	return 0;
 }
 
@@ -370,9 +365,8 @@ rpcrdma_ia_open(struct rpcrdma_xprt *xprt)
 		rc = PTR_ERR(ia->ri_id);
 		goto out_err;
 	}
-	ia->ri_device = ia->ri_id->device;
 
-	ia->ri_pd = ib_alloc_pd(ia->ri_device, 0);
+	ia->ri_pd = ib_alloc_pd(ia->ri_id->device, 0);
 	if (IS_ERR(ia->ri_pd)) {
 		rc = PTR_ERR(ia->ri_pd);
 		pr_err("rpcrdma: ib_alloc_pd() returned %d\n", rc);
@@ -381,12 +375,12 @@ rpcrdma_ia_open(struct rpcrdma_xprt *xprt)
 
 	switch (xprt_rdma_memreg_strategy) {
 	case RPCRDMA_FRWR:
-		if (frwr_is_supported(ia))
+		if (frwr_is_supported(ia->ri_id->device))
 			break;
 		/*FALLTHROUGH*/
 	default:
 		pr_err("rpcrdma: Device %s does not support memreg mode %d\n",
-		       ia->ri_device->name, xprt_rdma_memreg_strategy);
+		       ia->ri_id->device->name, xprt_rdma_memreg_strategy);
 		rc = -EINVAL;
 		goto out_err;
 	}
@@ -438,11 +432,11 @@ rpcrdma_ia_remove(struct rpcrdma_ia *ia)
 	 * mappings and MRs are gone.
 	 */
 	list_for_each_entry(rep, &buf->rb_recv_bufs, rr_list)
-		rpcrdma_dma_unmap_regbuf(rep->rr_rdmabuf);
+		rpcrdma_regbuf_dma_unmap(rep->rr_rdmabuf);
 	list_for_each_entry(req, &buf->rb_allreqs, rl_all) {
-		rpcrdma_dma_unmap_regbuf(req->rl_rdmabuf);
-		rpcrdma_dma_unmap_regbuf(req->rl_sendbuf);
-		rpcrdma_dma_unmap_regbuf(req->rl_recvbuf);
+		rpcrdma_regbuf_dma_unmap(req->rl_rdmabuf);
+		rpcrdma_regbuf_dma_unmap(req->rl_sendbuf);
+		rpcrdma_regbuf_dma_unmap(req->rl_recvbuf);
 	}
 	rpcrdma_mrs_destroy(buf);
 	ib_dealloc_pd(ia->ri_pd);
@@ -468,7 +462,6 @@ rpcrdma_ia_close(struct rpcrdma_ia *ia)
 		rdma_destroy_id(ia->ri_id);
 	}
 	ia->ri_id = NULL;
-	ia->ri_device = NULL;
 
 	/* If the pd is still busy, xprtrdma missed freeing a resource */
 	if (ia->ri_pd && !IS_ERR(ia->ri_pd))
@@ -476,19 +469,26 @@ rpcrdma_ia_close(struct rpcrdma_ia *ia)
 	ia->ri_pd = NULL;
 }
 
-/*
- * Create unconnected endpoint.
+/**
+ * rpcrdma_ep_create - Create unconnected endpoint
+ * @r_xprt: transport to instantiate
+ *
+ * Returns zero on success, or a negative errno.
  */
-int
-rpcrdma_ep_create(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia,
-		  struct rpcrdma_create_data_internal *cdata)
+int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt)
 {
+	struct rpcrdma_ep *ep = &r_xprt->rx_ep;
+	struct rpcrdma_ia *ia = &r_xprt->rx_ia;
 	struct rpcrdma_connect_private *pmsg = &ep->rep_cm_private;
 	struct ib_cq *sendcq, *recvcq;
 	unsigned int max_sge;
 	int rc;
 
-	max_sge = min_t(unsigned int, ia->ri_device->attrs.max_send_sge,
+	ep->rep_max_requests = xprt_rdma_slot_table_entries;
+	ep->rep_inline_send = xprt_rdma_max_inline_write;
+	ep->rep_inline_recv = xprt_rdma_max_inline_read;
+
+	max_sge = min_t(unsigned int, ia->ri_id->device->attrs.max_send_sge,
 			RPCRDMA_MAX_SEND_SGES);
 	if (max_sge < RPCRDMA_MIN_SEND_SGES) {
 		pr_warn("rpcrdma: HCA provides only %d send SGEs\n", max_sge);
@@ -496,7 +496,7 @@ rpcrdma_ep_create(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia,
 	}
 	ia->ri_max_send_sges = max_sge;
 
-	rc = frwr_open(ia, ep, cdata);
+	rc = frwr_open(ia, ep);
 	if (rc)
 		return rc;
 
@@ -518,23 +518,21 @@ rpcrdma_ep_create(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia,
 		ep->rep_attr.cap.max_send_sge,
 		ep->rep_attr.cap.max_recv_sge);
 
-	/* set trigger for requesting send completion */
-	ep->rep_send_batch = min_t(unsigned int, RPCRDMA_MAX_SEND_BATCH,
-				   cdata->max_requests >> 2);
+	ep->rep_send_batch = ep->rep_max_requests >> 3;
 	ep->rep_send_count = ep->rep_send_batch;
 	init_waitqueue_head(&ep->rep_connect_wait);
 	ep->rep_receive_count = 0;
 
-	sendcq = ib_alloc_cq(ia->ri_device, NULL,
+	sendcq = ib_alloc_cq(ia->ri_id->device, NULL,
 			     ep->rep_attr.cap.max_send_wr + 1,
-			     ia->ri_device->num_comp_vectors > 1 ? 1 : 0,
+			     ia->ri_id->device->num_comp_vectors > 1 ? 1 : 0,
 			     IB_POLL_WORKQUEUE);
 	if (IS_ERR(sendcq)) {
 		rc = PTR_ERR(sendcq);
 		goto out1;
 	}
 
-	recvcq = ib_alloc_cq(ia->ri_device, NULL,
+	recvcq = ib_alloc_cq(ia->ri_id->device, NULL,
 			     ep->rep_attr.cap.max_recv_wr + 1,
 			     0, IB_POLL_WORKQUEUE);
 	if (IS_ERR(recvcq)) {
@@ -552,15 +550,15 @@ rpcrdma_ep_create(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia,
 	pmsg->cp_magic = rpcrdma_cmp_magic;
 	pmsg->cp_version = RPCRDMA_CMP_VERSION;
 	pmsg->cp_flags |= RPCRDMA_CMP_F_SND_W_INV_OK;
-	pmsg->cp_send_size = rpcrdma_encode_buffer_size(cdata->inline_wsize);
-	pmsg->cp_recv_size = rpcrdma_encode_buffer_size(cdata->inline_rsize);
+	pmsg->cp_send_size = rpcrdma_encode_buffer_size(ep->rep_inline_send);
+	pmsg->cp_recv_size = rpcrdma_encode_buffer_size(ep->rep_inline_recv);
 	ep->rep_remote_cma.private_data = pmsg;
 	ep->rep_remote_cma.private_data_len = sizeof(*pmsg);
 
 	/* Client offers RDMA Read but does not initiate */
 	ep->rep_remote_cma.initiator_depth = 0;
 	ep->rep_remote_cma.responder_resources =
-		min_t(int, U8_MAX, ia->ri_device->attrs.max_qp_rd_atom);
+		min_t(int, U8_MAX, ia->ri_id->device->attrs.max_qp_rd_atom);
 
 	/* Limit transport retries so client can detect server
 	 * GID changes quickly. RPC layer handles re-establishing
@@ -583,16 +581,16 @@ out1:
 	return rc;
 }
 
-/*
- * rpcrdma_ep_destroy
+/**
+ * rpcrdma_ep_destroy - Disconnect and destroy endpoint.
+ * @r_xprt: transport instance to shut down
  *
- * Disconnect and destroy endpoint. After this, the only
- * valid operations on the ep are to free it (if dynamically
- * allocated) or re-create it.
  */
-void
-rpcrdma_ep_destroy(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia)
+void rpcrdma_ep_destroy(struct rpcrdma_xprt *r_xprt)
 {
+	struct rpcrdma_ep *ep = &r_xprt->rx_ep;
+	struct rpcrdma_ia *ia = &r_xprt->rx_ia;
+
 	if (ia->ri_id && ia->ri_id->qp) {
 		rpcrdma_ep_disconnect(ep, ia);
 		rdma_destroy_qp(ia->ri_id);
@@ -622,7 +620,7 @@ rpcrdma_ep_recreate_xprt(struct rpcrdma_xprt *r_xprt,
 		goto out1;
 
 	rc = -ENOMEM;
-	err = rpcrdma_ep_create(ep, ia, &r_xprt->rx_data);
+	err = rpcrdma_ep_create(r_xprt);
 	if (err) {
 		pr_err("rpcrdma: rpcrdma_ep_create returned %d\n", err);
 		goto out2;
@@ -639,7 +637,7 @@ rpcrdma_ep_recreate_xprt(struct rpcrdma_xprt *r_xprt,
 	return 0;
 
 out3:
-	rpcrdma_ep_destroy(ep, ia);
+	rpcrdma_ep_destroy(r_xprt);
 out2:
 	rpcrdma_ia_close(ia);
 out1:
@@ -672,7 +670,7 @@ rpcrdma_ep_reconnect(struct rpcrdma_xprt *r_xprt, struct rpcrdma_ep *ep,
 	 */
 	old = id;
 	rc = -ENETUNREACH;
-	if (ia->ri_device != id->device) {
+	if (ia->ri_id->device != id->device) {
 		pr_err("rpcrdma: can't reconnect on different device!\n");
 		goto out_destroy;
 	}
@@ -796,8 +794,8 @@ rpcrdma_ep_disconnect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia)
  */
 
 /* rpcrdma_sendctxs_destroy() assumes caller has already quiesced
- * queue activity, and ib_drain_qp has flushed all remaining Send
- * requests.
+ * queue activity, and rpcrdma_xprt_drain has flushed all remaining
+ * Send requests.
  */
 static void rpcrdma_sendctxs_destroy(struct rpcrdma_buffer *buf)
 {
@@ -867,20 +865,20 @@ static unsigned long rpcrdma_sendctx_next(struct rpcrdma_buffer *buf,
 
 /**
  * rpcrdma_sendctx_get_locked - Acquire a send context
- * @buf: transport buffers from which to acquire an unused context
+ * @r_xprt: controlling transport instance
  *
  * Returns pointer to a free send completion context; or NULL if
  * the queue is empty.
  *
  * Usage: Called to acquire an SGE array before preparing a Send WR.
  *
- * The caller serializes calls to this function (per rpcrdma_buffer),
- * and provides an effective memory barrier that flushes the new value
+ * The caller serializes calls to this function (per transport), and
+ * provides an effective memory barrier that flushes the new value
  * of rb_sc_head.
  */
-struct rpcrdma_sendctx *rpcrdma_sendctx_get_locked(struct rpcrdma_buffer *buf)
+struct rpcrdma_sendctx *rpcrdma_sendctx_get_locked(struct rpcrdma_xprt *r_xprt)
 {
-	struct rpcrdma_xprt *r_xprt;
+	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;
 	struct rpcrdma_sendctx *sc;
 	unsigned long next_head;
 
@@ -905,7 +903,6 @@ out_emptyq:
 	 * backing up. Cause the caller to pause and try again.
 	 */
 	set_bit(RPCRDMA_BUF_F_EMPTY_SCQ, &buf->rb_flags);
-	r_xprt = container_of(buf, struct rpcrdma_xprt, rx_buf);
 	r_xprt->rx_stats.empty_sendctx_q++;
 	return NULL;
 }
@@ -917,7 +914,7 @@ out_emptyq:
  * Usage: Called from Send completion to return a sendctxt
  * to the queue.
  *
- * The caller serializes calls to this function (per rpcrdma_buffer).
+ * The caller serializes calls to this function (per transport).
  */
 static void
 rpcrdma_sendctx_put_locked(struct rpcrdma_sendctx *sc)
@@ -925,7 +922,7 @@ rpcrdma_sendctx_put_locked(struct rpcrdma_sendctx *sc)
 	struct rpcrdma_buffer *buf = &sc->sc_xprt->rx_buf;
 	unsigned long next_tail;
 
-	/* Unmap SGEs of previously completed by unsignaled
+	/* Unmap SGEs of previously completed but unsignaled
 	 * Sends by walking up the queue until @sc is found.
 	 */
 	next_tail = buf->rb_sc_tail;
@@ -933,7 +930,7 @@ rpcrdma_sendctx_put_locked(struct rpcrdma_sendctx *sc)
 		next_tail = rpcrdma_sendctx_next(buf, next_tail);
 
 		/* ORDER: item must be accessed _before_ tail is updated */
-		rpcrdma_unmap_sendctx(buf->rb_sc_ctxs[next_tail]);
+		rpcrdma_sendctx_unmap(buf->rb_sc_ctxs[next_tail]);
 
 	} while (buf->rb_sc_ctxs[next_tail] != sc);
 
@@ -996,54 +993,70 @@ rpcrdma_mr_refresh_worker(struct work_struct *work)
 	rpcrdma_mrs_create(r_xprt);
 }
 
-struct rpcrdma_req *
-rpcrdma_create_req(struct rpcrdma_xprt *r_xprt)
+/**
+ * rpcrdma_req_create - Allocate an rpcrdma_req object
+ * @r_xprt: controlling r_xprt
+ * @size: initial size, in bytes, of send and receive buffers
+ * @flags: GFP flags passed to memory allocators
+ *
+ * Returns an allocated and fully initialized rpcrdma_req or NULL.
+ */
+struct rpcrdma_req *rpcrdma_req_create(struct rpcrdma_xprt *r_xprt, size_t size,
+				       gfp_t flags)
 {
 	struct rpcrdma_buffer *buffer = &r_xprt->rx_buf;
 	struct rpcrdma_regbuf *rb;
 	struct rpcrdma_req *req;
 
-	req = kzalloc(sizeof(*req), GFP_KERNEL);
+	req = kzalloc(sizeof(*req), flags);
 	if (req == NULL)
-		return ERR_PTR(-ENOMEM);
+		goto out1;
 
-	rb = rpcrdma_alloc_regbuf(RPCRDMA_HDRBUF_SIZE,
-				  DMA_TO_DEVICE, GFP_KERNEL);
-	if (IS_ERR(rb)) {
-		kfree(req);
-		return ERR_PTR(-ENOMEM);
-	}
+	rb = rpcrdma_regbuf_alloc(RPCRDMA_HDRBUF_SIZE, DMA_TO_DEVICE, flags);
+	if (!rb)
+		goto out2;
 	req->rl_rdmabuf = rb;
-	xdr_buf_init(&req->rl_hdrbuf, rb->rg_base, rdmab_length(rb));
+	xdr_buf_init(&req->rl_hdrbuf, rdmab_data(rb), rdmab_length(rb));
+
+	req->rl_sendbuf = rpcrdma_regbuf_alloc(size, DMA_TO_DEVICE, flags);
+	if (!req->rl_sendbuf)
+		goto out3;
+
+	req->rl_recvbuf = rpcrdma_regbuf_alloc(size, DMA_NONE, flags);
+	if (!req->rl_recvbuf)
+		goto out4;
+
 	req->rl_buffer = buffer;
 	INIT_LIST_HEAD(&req->rl_registered);
-
 	spin_lock(&buffer->rb_lock);
 	list_add(&req->rl_all, &buffer->rb_allreqs);
 	spin_unlock(&buffer->rb_lock);
 	return req;
+
+out4:
+	kfree(req->rl_sendbuf);
+out3:
+	kfree(req->rl_rdmabuf);
+out2:
+	kfree(req);
+out1:
+	return NULL;
 }
 
-static int
-rpcrdma_create_rep(struct rpcrdma_xprt *r_xprt, bool temp)
+static bool rpcrdma_rep_create(struct rpcrdma_xprt *r_xprt, bool temp)
 {
-	struct rpcrdma_create_data_internal *cdata = &r_xprt->rx_data;
 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;
 	struct rpcrdma_rep *rep;
-	int rc;
 
-	rc = -ENOMEM;
 	rep = kzalloc(sizeof(*rep), GFP_KERNEL);
 	if (rep == NULL)
 		goto out;
 
-	rep->rr_rdmabuf = rpcrdma_alloc_regbuf(cdata->inline_rsize,
+	rep->rr_rdmabuf = rpcrdma_regbuf_alloc(r_xprt->rx_ep.rep_inline_recv,
 					       DMA_FROM_DEVICE, GFP_KERNEL);
-	if (IS_ERR(rep->rr_rdmabuf)) {
-		rc = PTR_ERR(rep->rr_rdmabuf);
+	if (!rep->rr_rdmabuf)
 		goto out_free;
-	}
-	xdr_buf_init(&rep->rr_hdrbuf, rep->rr_rdmabuf->rg_base,
+	xdr_buf_init(&rep->rr_hdrbuf, rdmab_data(rep->rr_rdmabuf),
 		     rdmab_length(rep->rr_rdmabuf));
 
 	rep->rr_cqe.done = rpcrdma_wc_receive;
@@ -1058,22 +1071,27 @@ rpcrdma_create_rep(struct rpcrdma_xprt *r_xprt, bool temp)
 	spin_lock(&buf->rb_lock);
 	list_add(&rep->rr_list, &buf->rb_recv_bufs);
 	spin_unlock(&buf->rb_lock);
-	return 0;
+	return true;
 
 out_free:
 	kfree(rep);
 out:
-	return rc;
+	return false;
 }
 
-int
-rpcrdma_buffer_create(struct rpcrdma_xprt *r_xprt)
+/**
+ * rpcrdma_buffer_create - Create initial set of req/rep objects
+ * @r_xprt: transport instance to (re)initialize
+ *
+ * Returns zero on success, otherwise a negative errno.
+ */
+int rpcrdma_buffer_create(struct rpcrdma_xprt *r_xprt)
 {
 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;
 	int i, rc;
 
 	buf->rb_flags = 0;
-	buf->rb_max_requests = r_xprt->rx_data.max_requests;
+	buf->rb_max_requests = r_xprt->rx_ep.rep_max_requests;
 	buf->rb_bc_srv_max_requests = 0;
 	spin_lock_init(&buf->rb_mrlock);
 	spin_lock_init(&buf->rb_lock);
@@ -1086,16 +1104,15 @@ rpcrdma_buffer_create(struct rpcrdma_xprt *r_xprt)
 
 	INIT_LIST_HEAD(&buf->rb_send_bufs);
 	INIT_LIST_HEAD(&buf->rb_allreqs);
+
+	rc = -ENOMEM;
 	for (i = 0; i < buf->rb_max_requests; i++) {
 		struct rpcrdma_req *req;
 
-		req = rpcrdma_create_req(r_xprt);
-		if (IS_ERR(req)) {
-			dprintk("RPC:       %s: request buffer %d alloc"
-				" failed\n", __func__, i);
-			rc = PTR_ERR(req);
+		req = rpcrdma_req_create(r_xprt, RPCRDMA_V1_DEF_INLINE_SIZE,
+					 GFP_KERNEL);
+		if (!req)
 			goto out;
-		}
 		list_add(&req->rl_list, &buf->rb_send_bufs);
 	}
 
@@ -1121,10 +1138,9 @@ out:
 	return rc;
 }
 
-static void
-rpcrdma_destroy_rep(struct rpcrdma_rep *rep)
+static void rpcrdma_rep_destroy(struct rpcrdma_rep *rep)
 {
-	rpcrdma_free_regbuf(rep->rr_rdmabuf);
+	rpcrdma_regbuf_free(rep->rr_rdmabuf);
 	kfree(rep);
 }
 
@@ -1140,9 +1156,9 @@ rpcrdma_req_destroy(struct rpcrdma_req *req)
 {
 	list_del(&req->rl_all);
 
-	rpcrdma_free_regbuf(req->rl_recvbuf);
-	rpcrdma_free_regbuf(req->rl_sendbuf);
-	rpcrdma_free_regbuf(req->rl_rdmabuf);
+	rpcrdma_regbuf_free(req->rl_recvbuf);
+	rpcrdma_regbuf_free(req->rl_sendbuf);
+	rpcrdma_regbuf_free(req->rl_rdmabuf);
 	kfree(req);
 }
 
@@ -1180,7 +1196,7 @@ rpcrdma_mrs_destroy(struct rpcrdma_buffer *buf)
  * rpcrdma_buffer_destroy - Release all hw resources
  * @buf: root control block for resources
  *
- * ORDERING: relies on a prior ib_drain_qp :
+ * ORDERING: relies on a prior rpcrdma_xprt_drain :
  * - No more Send or Receive completions can occur
  * - All MRs, reps, and reqs are returned to their free lists
  */
@@ -1202,7 +1218,7 @@ rpcrdma_buffer_destroy(struct rpcrdma_buffer *buf)
 		rep = list_first_entry(&buf->rb_recv_bufs,
 				       struct rpcrdma_rep, rr_list);
 		list_del(&rep->rr_list);
-		rpcrdma_destroy_rep(rep);
+		rpcrdma_rep_destroy(rep);
 	}
 
 	while (!list_empty(&buf->rb_send_bufs)) {
@@ -1281,7 +1297,7 @@ rpcrdma_mr_unmap_and_put(struct rpcrdma_mr *mr)
 
 	if (mr->mr_dir != DMA_NONE) {
 		trace_xprtrdma_mr_unmap(mr);
-		ib_dma_unmap_sg(r_xprt->rx_ia.ri_device,
+		ib_dma_unmap_sg(r_xprt->rx_ia.ri_id->device,
 				mr->mr_sg, mr->mr_nents, mr->mr_dir);
 		mr->mr_dir = DMA_NONE;
 	}
@@ -1331,7 +1347,7 @@ rpcrdma_buffer_put(struct rpcrdma_req *req)
 	}
 	spin_unlock(&buffers->rb_lock);
 	if (rep)
-		rpcrdma_destroy_rep(rep);
+		rpcrdma_rep_destroy(rep);
 }
 
 /*
@@ -1348,69 +1364,90 @@ rpcrdma_recv_buffer_put(struct rpcrdma_rep *rep)
 		list_add(&rep->rr_list, &buffers->rb_recv_bufs);
 		spin_unlock(&buffers->rb_lock);
 	} else {
-		rpcrdma_destroy_rep(rep);
+		rpcrdma_rep_destroy(rep);
 	}
 }
 
-/**
- * rpcrdma_alloc_regbuf - allocate and DMA-map memory for SEND/RECV buffers
- * @size: size of buffer to be allocated, in bytes
- * @direction: direction of data movement
- * @flags: GFP flags
- *
- * Returns an ERR_PTR, or a pointer to a regbuf, a buffer that
- * can be persistently DMA-mapped for I/O.
+/* Returns a pointer to a rpcrdma_regbuf object, or NULL.
  *
  * xprtrdma uses a regbuf for posting an outgoing RDMA SEND, or for
  * receiving the payload of RDMA RECV operations. During Long Calls
  * or Replies they may be registered externally via frwr_map.
  */
-struct rpcrdma_regbuf *
-rpcrdma_alloc_regbuf(size_t size, enum dma_data_direction direction,
+static struct rpcrdma_regbuf *
+rpcrdma_regbuf_alloc(size_t size, enum dma_data_direction direction,
 		     gfp_t flags)
 {
 	struct rpcrdma_regbuf *rb;
 
-	rb = kmalloc(sizeof(*rb) + size, flags);
-	if (rb == NULL)
-		return ERR_PTR(-ENOMEM);
+	rb = kmalloc(sizeof(*rb), flags);
+	if (!rb)
+		return NULL;
+	rb->rg_data = kmalloc(size, flags);
+	if (!rb->rg_data) {
+		kfree(rb);
+		return NULL;
+	}
 
 	rb->rg_device = NULL;
 	rb->rg_direction = direction;
 	rb->rg_iov.length = size;
-
 	return rb;
 }
 
 /**
- * __rpcrdma_map_regbuf - DMA-map a regbuf
- * @ia: controlling rpcrdma_ia
+ * rpcrdma_regbuf_realloc - re-allocate a SEND/RECV buffer
+ * @rb: regbuf to reallocate
+ * @size: size of buffer to be allocated, in bytes
+ * @flags: GFP flags
+ *
+ * Returns true if reallocation was successful. If false is
+ * returned, @rb is left untouched.
+ */
+bool rpcrdma_regbuf_realloc(struct rpcrdma_regbuf *rb, size_t size, gfp_t flags)
+{
+	void *buf;
+
+	buf = kmalloc(size, flags);
+	if (!buf)
+		return false;
+
+	rpcrdma_regbuf_dma_unmap(rb);
+	kfree(rb->rg_data);
+
+	rb->rg_data = buf;
+	rb->rg_iov.length = size;
+	return true;
+}
+
+/**
+ * __rpcrdma_regbuf_dma_map - DMA-map a regbuf
+ * @r_xprt: controlling transport instance
  * @rb: regbuf to be mapped
+ *
+ * Returns true if the buffer is now DMA mapped to @r_xprt's device
  */
-bool
-__rpcrdma_dma_map_regbuf(struct rpcrdma_ia *ia, struct rpcrdma_regbuf *rb)
+bool __rpcrdma_regbuf_dma_map(struct rpcrdma_xprt *r_xprt,
+			      struct rpcrdma_regbuf *rb)
 {
-	struct ib_device *device = ia->ri_device;
+	struct ib_device *device = r_xprt->rx_ia.ri_id->device;
 
 	if (rb->rg_direction == DMA_NONE)
 		return false;
 
-	rb->rg_iov.addr = ib_dma_map_single(device,
-					    (void *)rb->rg_base,
-					    rdmab_length(rb),
-					    rb->rg_direction);
+	rb->rg_iov.addr = ib_dma_map_single(device, rdmab_data(rb),
+					    rdmab_length(rb), rb->rg_direction);
 	if (ib_dma_mapping_error(device, rdmab_addr(rb))) {
 		trace_xprtrdma_dma_maperr(rdmab_addr(rb));
 		return false;
 	}
 
 	rb->rg_device = device;
-	rb->rg_iov.lkey = ia->ri_pd->local_dma_lkey;
+	rb->rg_iov.lkey = r_xprt->rx_ia.ri_pd->local_dma_lkey;
 	return true;
 }
 
-static void
-rpcrdma_dma_unmap_regbuf(struct rpcrdma_regbuf *rb)
+static void rpcrdma_regbuf_dma_unmap(struct rpcrdma_regbuf *rb)
 {
 	if (!rb)
 		return;
@@ -1418,19 +1455,16 @@ rpcrdma_dma_unmap_regbuf(struct rpcrdma_regbuf *rb)
 	if (!rpcrdma_regbuf_is_mapped(rb))
 		return;
 
-	ib_dma_unmap_single(rb->rg_device, rdmab_addr(rb),
-			    rdmab_length(rb), rb->rg_direction);
+	ib_dma_unmap_single(rb->rg_device, rdmab_addr(rb), rdmab_length(rb),
+			    rb->rg_direction);
 	rb->rg_device = NULL;
 }
 
-/**
- * rpcrdma_free_regbuf - deregister and free registered buffer
- * @rb: regbuf to be deregistered and freed
- */
-void
-rpcrdma_free_regbuf(struct rpcrdma_regbuf *rb)
+static void rpcrdma_regbuf_free(struct rpcrdma_regbuf *rb)
 {
-	rpcrdma_dma_unmap_regbuf(rb);
+	rpcrdma_regbuf_dma_unmap(rb);
+	if (rb)
+		kfree(rb->rg_data);
 	kfree(rb);
 }
 
@@ -1497,17 +1531,15 @@ rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, bool temp)
 			list_del(&rep->rr_list);
 		spin_unlock(&buf->rb_lock);
 		if (!rep) {
-			if (rpcrdma_create_rep(r_xprt, temp))
+			if (!rpcrdma_rep_create(r_xprt, temp))
 				break;
 			continue;
 		}
 
 		rb = rep->rr_rdmabuf;
-		if (!rpcrdma_regbuf_is_mapped(rb)) {
-			if (!__rpcrdma_dma_map_regbuf(&r_xprt->rx_ia, rb)) {
-				rpcrdma_recv_buffer_put(rep);
-				break;
-			}
+		if (!rpcrdma_regbuf_dma_map(r_xprt, rb)) {
+			rpcrdma_recv_buffer_put(rep);
+			break;
 		}
 
 		trace_xprtrdma_post_recv(rep->rr_recv_wr.wr_cqe);
diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h
index 10f6593e1a6a..d1e0749bcbc4 100644
--- a/net/sunrpc/xprtrdma/xprt_rdma.h
+++ b/net/sunrpc/xprtrdma/xprt_rdma.h
@@ -66,20 +66,17 @@
  * Interface Adapter -- one per transport instance
  */
 struct rpcrdma_ia {
-	struct ib_device	*ri_device;
 	struct rdma_cm_id 	*ri_id;
 	struct ib_pd		*ri_pd;
-	struct completion	ri_done;
-	struct completion	ri_remove_done;
 	int			ri_async_rc;
 	unsigned int		ri_max_segs;
 	unsigned int		ri_max_frwr_depth;
-	unsigned int		ri_max_inline_write;
-	unsigned int		ri_max_inline_read;
 	unsigned int		ri_max_send_sges;
 	bool			ri_implicit_roundup;
 	enum ib_mr_type		ri_mrtype;
 	unsigned long		ri_flags;
+	struct completion	ri_done;
+	struct completion	ri_remove_done;
 };
 
 enum {
@@ -93,22 +90,29 @@ enum {
 struct rpcrdma_ep {
 	unsigned int		rep_send_count;
 	unsigned int		rep_send_batch;
+	unsigned int		rep_max_inline_send;
+	unsigned int		rep_max_inline_recv;
 	int			rep_connected;
 	struct ib_qp_init_attr	rep_attr;
 	wait_queue_head_t 	rep_connect_wait;
 	struct rpcrdma_connect_private	rep_cm_private;
 	struct rdma_conn_param	rep_remote_cma;
+	unsigned int		rep_max_requests;	/* set by /proc */
+	unsigned int		rep_inline_send;	/* negotiated */
+	unsigned int		rep_inline_recv;	/* negotiated */
 	int			rep_receive_count;
 };
 
 /* Pre-allocate extra Work Requests for handling backward receives
  * and sends. This is a fixed value because the Work Queues are
- * allocated when the forward channel is set up.
+ * allocated when the forward channel is set up, long before the
+ * backchannel is provisioned. This value is two times
+ * NFS4_DEF_CB_SLOT_TABLE_SIZE.
  */
 #if defined(CONFIG_SUNRPC_BACKCHANNEL)
-#define RPCRDMA_BACKWARD_WRS		(8)
+#define RPCRDMA_BACKWARD_WRS (32)
 #else
-#define RPCRDMA_BACKWARD_WRS		(0)
+#define RPCRDMA_BACKWARD_WRS (0)
 #endif
 
 /* Registered buffer -- registered kmalloc'd memory for RDMA SEND/RECV
@@ -121,33 +125,34 @@ struct rpcrdma_regbuf {
 	struct ib_sge		rg_iov;
 	struct ib_device	*rg_device;
 	enum dma_data_direction	rg_direction;
-	__be32			rg_base[0] __attribute__ ((aligned(256)));
+	void			*rg_data;
 };
 
-static inline u64
-rdmab_addr(struct rpcrdma_regbuf *rb)
+static inline u64 rdmab_addr(struct rpcrdma_regbuf *rb)
 {
 	return rb->rg_iov.addr;
 }
 
-static inline u32
-rdmab_length(struct rpcrdma_regbuf *rb)
+static inline u32 rdmab_length(struct rpcrdma_regbuf *rb)
 {
 	return rb->rg_iov.length;
 }
 
-static inline u32
-rdmab_lkey(struct rpcrdma_regbuf *rb)
+static inline u32 rdmab_lkey(struct rpcrdma_regbuf *rb)
 {
 	return rb->rg_iov.lkey;
 }
 
-static inline struct ib_device *
-rdmab_device(struct rpcrdma_regbuf *rb)
+static inline struct ib_device *rdmab_device(struct rpcrdma_regbuf *rb)
 {
 	return rb->rg_device;
 }
 
+static inline void *rdmab_data(const struct rpcrdma_regbuf *rb)
+{
+	return rb->rg_data;
+}
+
 #define RPCRDMA_DEF_GFP		(GFP_NOIO | __GFP_NOWARN)
 
 /* To ensure a transport can always make forward progress,
@@ -222,34 +227,18 @@ struct rpcrdma_xprt;
 struct rpcrdma_sendctx {
 	struct ib_send_wr	sc_wr;
 	struct ib_cqe		sc_cqe;
+	struct ib_device	*sc_device;
 	struct rpcrdma_xprt	*sc_xprt;
 	struct rpcrdma_req	*sc_req;
 	unsigned int		sc_unmap_count;
 	struct ib_sge		sc_sges[];
 };
 
-/* Limit the number of SGEs that can be unmapped during one
- * Send completion. This caps the amount of work a single
- * completion can do before returning to the provider.
- *
- * Setting this to zero disables Send completion batching.
- */
-enum {
-	RPCRDMA_MAX_SEND_BATCH = 7,
-};
-
 /*
  * struct rpcrdma_mr - external memory region metadata
  *
  * An external memory region is any buffer or page that is registered
  * on the fly (ie, not pre-registered).
- *
- * Each rpcrdma_buffer has a list of free MWs anchored in rb_mrs. During
- * call_allocate, rpcrdma_buffer_get() assigns one to each segment in
- * an rpcrdma_req. Then rpcrdma_register_external() grabs these to keep
- * track of registration metadata while each RPC is pending.
- * rpcrdma_deregister_external() uses this metadata to unmap and
- * release these resources when an RPC is complete.
  */
 enum rpcrdma_frwr_state {
 	FRWR_IS_INVALID,	/* ready to be used */
@@ -419,20 +408,6 @@ enum {
 };
 
 /*
- * Internal structure for transport instance creation. This
- * exists primarily for modularity.
- *
- * This data should be set with mount options
- */
-struct rpcrdma_create_data_internal {
-	unsigned int	max_requests;	/* max requests (slots) in flight */
-	unsigned int	rsize;		/* mount rsize - max read hdr+data */
-	unsigned int	wsize;		/* mount wsize - max write hdr+data */
-	unsigned int	inline_rsize;	/* max non-rdma read data payload */
-	unsigned int	inline_wsize;	/* max non-rdma write data payload */
-};
-
-/*
  * Statistics for RPCRDMA
  */
 struct rpcrdma_stats {
@@ -476,13 +451,11 @@ struct rpcrdma_xprt {
 	struct rpcrdma_ia	rx_ia;
 	struct rpcrdma_ep	rx_ep;
 	struct rpcrdma_buffer	rx_buf;
-	struct rpcrdma_create_data_internal rx_data;
 	struct delayed_work	rx_connect_worker;
 	struct rpcrdma_stats	rx_stats;
 };
 
 #define rpcx_to_rdmax(x) container_of(x, struct rpcrdma_xprt, rx_xprt)
-#define rpcx_to_rdmad(x) (rpcx_to_rdmax(x)->rx_data)
 
 static inline const char *
 rpcrdma_addrstr(const struct rpcrdma_xprt *r_xprt)
@@ -516,9 +489,8 @@ void rpcrdma_ia_close(struct rpcrdma_ia *);
 /*
  * Endpoint calls - xprtrdma/verbs.c
  */
-int rpcrdma_ep_create(struct rpcrdma_ep *, struct rpcrdma_ia *,
-				struct rpcrdma_create_data_internal *);
-void rpcrdma_ep_destroy(struct rpcrdma_ep *, struct rpcrdma_ia *);
+int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt);
+void rpcrdma_ep_destroy(struct rpcrdma_xprt *r_xprt);
 int rpcrdma_ep_connect(struct rpcrdma_ep *, struct rpcrdma_ia *);
 void rpcrdma_ep_disconnect(struct rpcrdma_ep *, struct rpcrdma_ia *);
 
@@ -528,11 +500,12 @@ int rpcrdma_ep_post(struct rpcrdma_ia *, struct rpcrdma_ep *,
 /*
  * Buffer calls - xprtrdma/verbs.c
  */
-struct rpcrdma_req *rpcrdma_create_req(struct rpcrdma_xprt *);
+struct rpcrdma_req *rpcrdma_req_create(struct rpcrdma_xprt *r_xprt, size_t size,
+				       gfp_t flags);
 void rpcrdma_req_destroy(struct rpcrdma_req *req);
 int rpcrdma_buffer_create(struct rpcrdma_xprt *);
 void rpcrdma_buffer_destroy(struct rpcrdma_buffer *);
-struct rpcrdma_sendctx *rpcrdma_sendctx_get_locked(struct rpcrdma_buffer *buf);
+struct rpcrdma_sendctx *rpcrdma_sendctx_get_locked(struct rpcrdma_xprt *r_xprt);
 
 struct rpcrdma_mr *rpcrdma_mr_get(struct rpcrdma_xprt *r_xprt);
 void rpcrdma_mr_put(struct rpcrdma_mr *mr);
@@ -548,23 +521,34 @@ struct rpcrdma_req *rpcrdma_buffer_get(struct rpcrdma_buffer *);
 void rpcrdma_buffer_put(struct rpcrdma_req *);
 void rpcrdma_recv_buffer_put(struct rpcrdma_rep *);
 
-struct rpcrdma_regbuf *rpcrdma_alloc_regbuf(size_t, enum dma_data_direction,
-					    gfp_t);
-bool __rpcrdma_dma_map_regbuf(struct rpcrdma_ia *, struct rpcrdma_regbuf *);
-void rpcrdma_free_regbuf(struct rpcrdma_regbuf *);
+bool rpcrdma_regbuf_realloc(struct rpcrdma_regbuf *rb, size_t size,
+			    gfp_t flags);
+bool __rpcrdma_regbuf_dma_map(struct rpcrdma_xprt *r_xprt,
+			      struct rpcrdma_regbuf *rb);
 
-static inline bool
-rpcrdma_regbuf_is_mapped(struct rpcrdma_regbuf *rb)
+/**
+ * rpcrdma_regbuf_is_mapped - check if buffer is DMA mapped
+ *
+ * Returns true if the buffer is now mapped to rb->rg_device.
+ */
+static inline bool rpcrdma_regbuf_is_mapped(struct rpcrdma_regbuf *rb)
 {
 	return rb->rg_device != NULL;
 }
 
-static inline bool
-rpcrdma_dma_map_regbuf(struct rpcrdma_ia *ia, struct rpcrdma_regbuf *rb)
+/**
+ * rpcrdma_regbuf_dma_map - DMA-map a regbuf
+ * @r_xprt: controlling transport instance
+ * @rb: regbuf to be mapped
+ *
+ * Returns true if the buffer is currently DMA mapped.
+ */
+static inline bool rpcrdma_regbuf_dma_map(struct rpcrdma_xprt *r_xprt,
+					  struct rpcrdma_regbuf *rb)
 {
 	if (likely(rpcrdma_regbuf_is_mapped(rb)))
 		return true;
-	return __rpcrdma_dma_map_regbuf(ia, rb);
+	return __rpcrdma_regbuf_dma_map(r_xprt, rb);
 }
 
 /*
@@ -579,9 +563,8 @@ rpcrdma_data_dir(bool writing)
 
 /* Memory registration calls xprtrdma/frwr_ops.c
  */
-bool frwr_is_supported(struct rpcrdma_ia *);
-int frwr_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep,
-	      struct rpcrdma_create_data_internal *cdata);
+bool frwr_is_supported(struct ib_device *device);
+int frwr_open(struct rpcrdma_ia *ia, struct rpcrdma_ep *ep);
 int frwr_init_mr(struct rpcrdma_ia *ia, struct rpcrdma_mr *mr);
 void frwr_release_mr(struct rpcrdma_mr *mr);
 size_t frwr_maxpages(struct rpcrdma_xprt *r_xprt);
@@ -610,7 +593,7 @@ int rpcrdma_prepare_send_sges(struct rpcrdma_xprt *r_xprt,
 			      struct rpcrdma_req *req, u32 hdrlen,
 			      struct xdr_buf *xdr,
 			      enum rpcrdma_chunktype rtype);
-void rpcrdma_unmap_sendctx(struct rpcrdma_sendctx *sc);
+void rpcrdma_sendctx_unmap(struct rpcrdma_sendctx *sc);
 int rpcrdma_marshal_req(struct rpcrdma_xprt *r_xprt, struct rpc_rqst *rqst);
 void rpcrdma_set_max_header_sizes(struct rpcrdma_xprt *);
 void rpcrdma_complete_rqst(struct rpcrdma_rep *rep);
@@ -627,7 +610,9 @@ static inline void rpcrdma_set_xdrlen(struct xdr_buf *xdr, size_t len)
 
 /* RPC/RDMA module init - xprtrdma/transport.c
  */
+extern unsigned int xprt_rdma_slot_table_entries;
 extern unsigned int xprt_rdma_max_inline_read;
+extern unsigned int xprt_rdma_max_inline_write;
 void xprt_rdma_format_addresses(struct rpc_xprt *xprt, struct sockaddr *sap);
 void xprt_rdma_free_addresses(struct rpc_xprt *xprt);
 void xprt_rdma_close(struct rpc_xprt *xprt);
diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c
index 732d4b57411a..c69951ed2ebc 100644
--- a/net/sunrpc/xprtsock.c
+++ b/net/sunrpc/xprtsock.c
@@ -2017,6 +2017,7 @@ static void xs_local_connect(struct rpc_xprt *xprt, struct rpc_task *task)
 		 * we'll need to figure out how to pass a namespace to
 		 * connect.
 		 */
+		task->tk_rpc_status = -ENOTCONN;
 		rpc_exit(task, -ENOTCONN);
 		return;
 	}
@@ -2690,7 +2691,7 @@ static const struct rpc_xprt_ops xs_local_ops = {
 	.buf_free		= rpc_free,
 	.prepare_request	= xs_stream_prepare_request,
 	.send_request		= xs_local_send_request,
-	.set_retrans_timeout	= xprt_set_retrans_timeout_def,
+	.wait_for_reply_request	= xprt_wait_for_reply_request_def,
 	.close			= xs_close,
 	.destroy		= xs_destroy,
 	.print_stats		= xs_local_print_stats,
@@ -2710,7 +2711,7 @@ static const struct rpc_xprt_ops xs_udp_ops = {
 	.buf_alloc		= rpc_malloc,
 	.buf_free		= rpc_free,
 	.send_request		= xs_udp_send_request,
-	.set_retrans_timeout	= xprt_set_retrans_timeout_rtt,
+	.wait_for_reply_request	= xprt_wait_for_reply_request_rtt,
 	.timer			= xs_udp_timer,
 	.release_request	= xprt_release_rqst_cong,
 	.close			= xs_close,
@@ -2733,7 +2734,7 @@ static const struct rpc_xprt_ops xs_tcp_ops = {
 	.buf_free		= rpc_free,
 	.prepare_request	= xs_stream_prepare_request,
 	.send_request		= xs_tcp_send_request,
-	.set_retrans_timeout	= xprt_set_retrans_timeout_def,
+	.wait_for_reply_request	= xprt_wait_for_reply_request_def,
 	.close			= xs_tcp_shutdown,
 	.destroy		= xs_destroy,
 	.set_connect_timeout	= xs_tcp_set_connect_timeout,
@@ -2761,7 +2762,7 @@ static const struct rpc_xprt_ops bc_tcp_ops = {
 	.buf_alloc		= bc_malloc,
 	.buf_free		= bc_free,
 	.send_request		= bc_send_request,
-	.set_retrans_timeout	= xprt_set_retrans_timeout_def,
+	.wait_for_reply_request	= xprt_wait_for_reply_request_def,
 	.close			= bc_close,
 	.destroy		= bc_destroy,
 	.print_stats		= xs_tcp_print_stats,
diff --git a/net/tipc/socket.c b/net/tipc/socket.c
index 145e4decb0c9..dd8537f988c4 100644
--- a/net/tipc/socket.c
+++ b/net/tipc/socket.c
@@ -736,11 +736,11 @@ static __poll_t tipc_poll(struct file *file, struct socket *sock,
 
 	switch (sk->sk_state) {
 	case TIPC_ESTABLISHED:
-	case TIPC_CONNECTING:
 		if (!tsk->cong_link_cnt && !tsk_conn_cong(tsk))
 			revents |= EPOLLOUT;
 		/* fall through */
 	case TIPC_LISTEN:
+	case TIPC_CONNECTING:
 		if (!skb_queue_empty(&sk->sk_receive_queue))
 			revents |= EPOLLIN | EPOLLRDNORM;
 		break;
@@ -2043,7 +2043,7 @@ static bool tipc_sk_filter_connect(struct tipc_sock *tsk, struct sk_buff *skb)
 			if (msg_data_sz(hdr))
 				return true;
 			/* Empty ACK-, - wake up sleeping connect() and drop */
-			sk->sk_data_ready(sk);
+			sk->sk_state_change(sk);
 			msg_set_dest_droppable(hdr, 1);
 			return false;
 		}
diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c
index e225c81e6b35..ca54a7c7ec81 100644
--- a/net/tls/tls_device.c
+++ b/net/tls/tls_device.c
@@ -541,14 +541,11 @@ static int tls_device_push_pending_record(struct sock *sk, int flags)
 
 void tls_device_write_space(struct sock *sk, struct tls_context *ctx)
 {
-	int rc = 0;
-
 	if (!sk->sk_write_pending && tls_is_partially_sent_record(ctx)) {
 		gfp_t sk_allocation = sk->sk_allocation;
 
 		sk->sk_allocation = GFP_ATOMIC;
-		rc = tls_push_partial_record(sk, ctx,
-					     MSG_DONTWAIT | MSG_NOSIGNAL);
+		tls_push_partial_record(sk, ctx, MSG_DONTWAIT | MSG_NOSIGNAL);
 		sk->sk_allocation = sk_allocation;
 	}
 }
@@ -1036,4 +1033,5 @@ void __exit tls_device_cleanup(void)
 {
 	unregister_netdevice_notifier(&tls_dev_notifier);
 	flush_work(&tls_device_gc_work);
+	clean_acked_data_flush();
 }
diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c
index c02293fb10e6..d93f83f77864 100644
--- a/net/tls/tls_sw.c
+++ b/net/tls/tls_sw.c
@@ -119,23 +119,25 @@ static int skb_nsg(struct sk_buff *skb, int offset, int len)
 }
 
 static int padding_length(struct tls_sw_context_rx *ctx,
-			  struct tls_context *tls_ctx, struct sk_buff *skb)
+			  struct tls_prot_info *prot, struct sk_buff *skb)
 {
 	struct strp_msg *rxm = strp_msg(skb);
 	int sub = 0;
 
 	/* Determine zero-padding length */
-	if (tls_ctx->prot_info.version == TLS_1_3_VERSION) {
+	if (prot->version == TLS_1_3_VERSION) {
 		char content_type = 0;
 		int err;
 		int back = 17;
 
 		while (content_type == 0) {
-			if (back > rxm->full_len)
+			if (back > rxm->full_len - prot->prepend_size)
 				return -EBADMSG;
 			err = skb_copy_bits(skb,
 					    rxm->offset + rxm->full_len - back,
 					    &content_type, 1);
+			if (err)
+				return err;
 			if (content_type)
 				break;
 			sub++;
@@ -170,9 +172,17 @@ static void tls_decrypt_done(struct crypto_async_request *req, int err)
 		tls_err_abort(skb->sk, err);
 	} else {
 		struct strp_msg *rxm = strp_msg(skb);
-		rxm->full_len -= padding_length(ctx, tls_ctx, skb);
-		rxm->offset += prot->prepend_size;
-		rxm->full_len -= prot->overhead_size;
+		int pad;
+
+		pad = padding_length(ctx, prot, skb);
+		if (pad < 0) {
+			ctx->async_wait.err = pad;
+			tls_err_abort(skb->sk, pad);
+		} else {
+			rxm->full_len -= pad;
+			rxm->offset += prot->prepend_size;
+			rxm->full_len -= prot->overhead_size;
+		}
 	}
 
 	/* After using skb->sk to propagate sk through crypto async callback
@@ -1478,7 +1488,7 @@ static int decrypt_skb_update(struct sock *sk, struct sk_buff *skb,
 	struct tls_prot_info *prot = &tls_ctx->prot_info;
 	int version = prot->version;
 	struct strp_msg *rxm = strp_msg(skb);
-	int err = 0;
+	int pad, err = 0;
 
 	if (!ctx->decrypted) {
 #ifdef CONFIG_TLS_DEVICE
@@ -1501,7 +1511,11 @@ static int decrypt_skb_update(struct sock *sk, struct sk_buff *skb,
 			*zc = false;
 		}
 
-		rxm->full_len -= padding_length(ctx, tls_ctx, skb);
+		pad = padding_length(ctx, prot, skb);
+		if (pad < 0)
+			return pad;
+
+		rxm->full_len -= pad;
 		rxm->offset += prot->prepend_size;
 		rxm->full_len -= prot->overhead_size;
 		tls_advance_record_sn(sk, &tls_ctx->rx, version);
diff --git a/samples/Kconfig b/samples/Kconfig
index 30a89425009c..0561a94f6fdb 100644
--- a/samples/Kconfig
+++ b/samples/Kconfig
@@ -20,8 +20,7 @@ config SAMPLE_TRACE_PRINTK
 	 test various trace_printk() calls from a module.
 
 config SAMPLE_KOBJECT
-	tristate "Build kobject examples -- loadable modules only"
-	depends on m
+	tristate "Build kobject examples"
 	help
 	  This config option will allow you to build a number of
 	  different kobject sample modules showing how to use kobjects,
@@ -103,8 +102,8 @@ config SAMPLE_CONNECTOR
 	  See also Documentation/connector/connector.txt
 
 config SAMPLE_SECCOMP
-	tristate "Build seccomp sample code -- loadable modules only"
-	depends on SECCOMP_FILTER && m
+	bool "Build seccomp sample code"
+	depends on SECCOMP_FILTER
 	help
 	  Build samples of seccomp filters using various methods of
 	  BPF filter construction.
diff --git a/samples/Makefile b/samples/Makefile
index 2484cc262d3e..8e096e0d45d1 100644
--- a/samples/Makefile
+++ b/samples/Makefile
@@ -1,6 +1,6 @@
 # Makefile for Linux samples code
 
-obj-$(CONFIG_SAMPLES)	+= kobject/ kprobes/ trace_events/ livepatch/ \
+obj-y			+= kobject/ kprobes/ trace_events/ livepatch/ \
 			   hw_breakpoint/ kfifo/ kdb/ hidraw/ rpmsg/ seccomp/ \
 			   configfs/ connector/ v4l/ trace_printk/ \
 			   vfio-mdev/ vfs/ qmi/ binderfs/ pidfd/
diff --git a/samples/bpf/Makefile b/samples/bpf/Makefile
index 65e667bdf979..4f0a1cdbfe7c 100644
--- a/samples/bpf/Makefile
+++ b/samples/bpf/Makefile
@@ -52,6 +52,7 @@ hostprogs-y += xdpsock
 hostprogs-y += xdp_fwd
 hostprogs-y += task_fd_query
 hostprogs-y += xdp_sample_pkts
+hostprogs-y += ibumad
 hostprogs-y += hbm
 
 # Libbpf dependencies
@@ -108,6 +109,7 @@ xdpsock-objs := xdpsock_user.o
 xdp_fwd-objs := xdp_fwd_user.o
 task_fd_query-objs := bpf_load.o task_fd_query_user.o $(TRACE_HELPERS)
 xdp_sample_pkts-objs := xdp_sample_pkts_user.o $(TRACE_HELPERS)
+ibumad-objs := bpf_load.o ibumad_user.o $(TRACE_HELPERS)
 hbm-objs := bpf_load.o hbm.o $(CGROUP_HELPERS)
 
 # Tell kbuild to always build the programs
@@ -166,6 +168,7 @@ always += xdp_adjust_tail_kern.o
 always += xdp_fwd_kern.o
 always += task_fd_query_kern.o
 always += xdp_sample_pkts_kern.o
+always += ibumad_kern.o
 always += hbm_out_kern.o
 
 KBUILD_HOSTCFLAGS += -I$(objtree)/usr/include
diff --git a/samples/bpf/ibumad_kern.c b/samples/bpf/ibumad_kern.c
new file mode 100644
index 000000000000..38b2b3f22049
--- /dev/null
+++ b/samples/bpf/ibumad_kern.c
@@ -0,0 +1,144 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+
+/**
+ * ibumad BPF sample kernel side
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of version 2 of the GNU General Public
+ * License as published by the Free Software Foundation.
+ *
+ * Copyright(c) 2018 Ira Weiny, Intel Corporation
+ */
+
+#define KBUILD_MODNAME "ibumad_count_pkts_by_class"
+#include <uapi/linux/bpf.h>
+
+#include "bpf_helpers.h"
+
+
+struct bpf_map_def SEC("maps") read_count = {
+	.type        = BPF_MAP_TYPE_ARRAY,
+	.key_size    = sizeof(u32), /* class; u32 required */
+	.value_size  = sizeof(u64), /* count of mads read */
+	.max_entries = 256, /* Room for all Classes */
+};
+
+struct bpf_map_def SEC("maps") write_count = {
+	.type        = BPF_MAP_TYPE_ARRAY,
+	.key_size    = sizeof(u32), /* class; u32 required */
+	.value_size  = sizeof(u64), /* count of mads written */
+	.max_entries = 256, /* Room for all Classes */
+};
+
+#undef DEBUG
+#ifdef DEBUG
+#define bpf_debug(fmt, ...)                         \
+({                                                  \
+	char ____fmt[] = fmt;                       \
+	bpf_trace_printk(____fmt, sizeof(____fmt),  \
+			 ##__VA_ARGS__);            \
+})
+#else
+#define bpf_debug(fmt, ...)
+#endif
+
+/* Taken from the current format defined in
+ * include/trace/events/ib_umad.h
+ * and
+ * /sys/kernel/debug/tracing/events/ib_umad/ib_umad_read/format
+ * /sys/kernel/debug/tracing/events/ib_umad/ib_umad_write/format
+ */
+struct ib_umad_rw_args {
+	u64 pad;
+	u8 port_num;
+	u8 sl;
+	u8 path_bits;
+	u8 grh_present;
+	u32 id;
+	u32 status;
+	u32 timeout_ms;
+	u32 retires;
+	u32 length;
+	u32 qpn;
+	u32 qkey;
+	u8 gid_index;
+	u8 hop_limit;
+	u16 lid;
+	u16 attr_id;
+	u16 pkey_index;
+	u8 base_version;
+	u8 mgmt_class;
+	u8 class_version;
+	u8 method;
+	u32 flow_label;
+	u16 mad_status;
+	u16 class_specific;
+	u32 attr_mod;
+	u64 tid;
+	u8 gid[16];
+	u32 dev_index;
+	u8 traffic_class;
+};
+
+SEC("tracepoint/ib_umad/ib_umad_read_recv")
+int on_ib_umad_read_recv(struct ib_umad_rw_args *ctx)
+{
+	u64 zero = 0, *val;
+	u8 class = ctx->mgmt_class;
+
+	bpf_debug("ib_umad read recv : class 0x%x\n", class);
+
+	val = bpf_map_lookup_elem(&read_count, &class);
+	if (!val) {
+		bpf_map_update_elem(&read_count, &class, &zero, BPF_NOEXIST);
+		val = bpf_map_lookup_elem(&read_count, &class);
+		if (!val)
+			return 0;
+	}
+
+	(*val) += 1;
+
+	return 0;
+}
+SEC("tracepoint/ib_umad/ib_umad_read_send")
+int on_ib_umad_read_send(struct ib_umad_rw_args *ctx)
+{
+	u64 zero = 0, *val;
+	u8 class = ctx->mgmt_class;
+
+	bpf_debug("ib_umad read send : class 0x%x\n", class);
+
+	val = bpf_map_lookup_elem(&read_count, &class);
+	if (!val) {
+		bpf_map_update_elem(&read_count, &class, &zero, BPF_NOEXIST);
+		val = bpf_map_lookup_elem(&read_count, &class);
+		if (!val)
+			return 0;
+	}
+
+	(*val) += 1;
+
+	return 0;
+}
+SEC("tracepoint/ib_umad/ib_umad_write")
+int on_ib_umad_write(struct ib_umad_rw_args *ctx)
+{
+	u64 zero = 0, *val;
+	u8 class = ctx->mgmt_class;
+
+	bpf_debug("ib_umad write : class 0x%x\n", class);
+
+	val = bpf_map_lookup_elem(&write_count, &class);
+	if (!val) {
+		bpf_map_update_elem(&write_count, &class, &zero, BPF_NOEXIST);
+		val = bpf_map_lookup_elem(&write_count, &class);
+		if (!val)
+			return 0;
+	}
+
+	(*val) += 1;
+
+	return 0;
+}
+
+char _license[] SEC("license") = "GPL";
diff --git a/samples/bpf/ibumad_user.c b/samples/bpf/ibumad_user.c
new file mode 100644
index 000000000000..097d76143363
--- /dev/null
+++ b/samples/bpf/ibumad_user.c
@@ -0,0 +1,122 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+
+/**
+ * ibumad BPF sample user side
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of version 2 of the GNU General Public
+ * License as published by the Free Software Foundation.
+ *
+ * Copyright(c) 2018 Ira Weiny, Intel Corporation
+ */
+
+#include <linux/bpf.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <limits.h>
+
+#include <sys/resource.h>
+#include <getopt.h>
+#include <net/if.h>
+
+#include "bpf_load.h"
+#include "bpf_util.h"
+#include "bpf/libbpf.h"
+
+static void dump_counts(int fd)
+{
+	__u32 key;
+	__u64 value;
+
+	for (key = 0; key < 256; key++) {
+		if (bpf_map_lookup_elem(fd, &key, &value)) {
+			printf("failed to read key %u\n", key);
+			continue;
+		}
+		if (value)
+			printf("0x%02x : %llu\n", key, value);
+	}
+}
+
+static void dump_all_counts(void)
+{
+	printf("Read 'Class : count'\n");
+	dump_counts(map_fd[0]);
+	printf("Write 'Class : count'\n");
+	dump_counts(map_fd[1]);
+}
+
+static void dump_exit(int sig)
+{
+	dump_all_counts();
+	exit(0);
+}
+
+static const struct option long_options[] = {
+	{"help",      no_argument,       NULL, 'h'},
+	{"delay",     required_argument, NULL, 'd'},
+};
+
+static void usage(char *cmd)
+{
+	printf("eBPF test program to count packets from various IP addresses\n"
+		"Usage: %s <options>\n"
+		"       --help,   -h  this menu\n"
+		"       --delay,  -d  <delay>  wait <delay> sec between prints [1 - 1000000]\n"
+		, cmd
+		);
+}
+
+int main(int argc, char **argv)
+{
+	unsigned long delay = 5;
+	int longindex = 0;
+	int opt;
+	char bpf_file[256];
+
+	/* Create the eBPF kernel code path name.
+	 * This follows the pattern of all of the other bpf samples
+	 */
+	snprintf(bpf_file, sizeof(bpf_file), "%s_kern.o", argv[0]);
+
+	/* Do one final dump when exiting */
+	signal(SIGINT, dump_exit);
+	signal(SIGTERM, dump_exit);
+
+	while ((opt = getopt_long(argc, argv, "hd:rSw",
+				  long_options, &longindex)) != -1) {
+		switch (opt) {
+		case 'd':
+			delay = strtoul(optarg, NULL, 0);
+			if (delay == ULONG_MAX || delay < 0 ||
+			    delay > 1000000) {
+				fprintf(stderr, "ERROR: invalid delay : %s\n",
+					optarg);
+				usage(argv[0]);
+				return 1;
+			}
+			break;
+		default:
+		case 'h':
+			usage(argv[0]);
+			return 1;
+		}
+	}
+
+	if (load_bpf_file(bpf_file)) {
+		fprintf(stderr, "ERROR: failed to load eBPF from file : %s\n",
+			bpf_file);
+		return 1;
+	}
+
+	while (1) {
+		sleep(delay);
+		dump_all_counts();
+	}
+
+	return 0;
+}
diff --git a/samples/seccomp/Makefile b/samples/seccomp/Makefile
index fb43a814d4c0..00e0b5e90bd0 100644
--- a/samples/seccomp/Makefile
+++ b/samples/seccomp/Makefile
@@ -40,5 +40,5 @@ HOSTLDLIBS_bpf-fancy += $(MFLAG)
 HOSTLDLIBS_dropper += $(MFLAG)
 HOSTLDLIBS_user-trap += $(MFLAG)
 endif
-always := $(hostprogs-m)
+always := $(hostprogs-y)
 endif
diff --git a/scripts/Makefile.build b/scripts/Makefile.build
index 0c5969fa795f..ae9cf740633e 100644
--- a/scripts/Makefile.build
+++ b/scripts/Makefile.build
@@ -504,7 +504,7 @@ existing-targets := $(wildcard $(sort $(targets)))
 
 -include $(foreach f,$(existing-targets),$(dir $(f)).$(notdir $(f)).cmd)
 
-ifneq ($(KBUILD_SRC),)
+ifneq ($(srctree),.)
 # Create directories for object files if they do not exist
 obj-dirs := $(sort $(obj) $(patsubst %/,%, $(dir $(targets))))
 # If targets exist, their directories apparently exist. Skip mkdir.
diff --git a/scripts/Makefile.host b/scripts/Makefile.host
index a115259b57e7..73b804197fca 100644
--- a/scripts/Makefile.host
+++ b/scripts/Makefile.host
@@ -71,7 +71,7 @@ __hostc_flags	= $(_hostc_flags)
 __hostcxx_flags	= $(_hostcxx_flags)
 
 ifeq ($(KBUILD_EXTMOD),)
-ifneq ($(KBUILD_SRC),)
+ifneq ($(srctree),.)
 __hostc_flags	= -I$(obj) $(call flags,_hostc_flags)
 __hostcxx_flags	= -I$(obj) $(call flags,_hostcxx_flags)
 endif
diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib
index 8a1f64f17740..41e98fa66b91 100644
--- a/scripts/Makefile.lib
+++ b/scripts/Makefile.lib
@@ -144,7 +144,7 @@ __cpp_flags     = $(_cpp_flags)
 # If building the kernel in a separate objtree expand all occurrences
 # of -Idir to -I$(srctree)/dir except for absolute paths (starting with '/').
 ifeq ($(KBUILD_EXTMOD),)
-ifneq ($(KBUILD_SRC),)
+ifneq ($(srctree),.)
 
 # -I$(obj) locates generated .h files
 # $(call addtree,-I$(obj)) locates .h files in srctree, from generated .c files
diff --git a/scripts/Makefile.modbuiltin b/scripts/Makefile.modbuiltin
index a072a4267746..ea90a90b41a0 100644
--- a/scripts/Makefile.modbuiltin
+++ b/scripts/Makefile.modbuiltin
@@ -15,7 +15,7 @@ include include/config/tristate.conf
 
 include scripts/Kbuild.include
 
-ifneq ($(KBUILD_SRC),)
+ifneq ($(srctree),.)
 # Create output directory if not already present
 _dummy := $(shell [ -d $(obj) ] || mkdir -p $(obj))
 endif
diff --git a/scripts/Makefile.modpost b/scripts/Makefile.modpost
index 6b7f354f189a..fec6ec2ffa47 100644
--- a/scripts/Makefile.modpost
+++ b/scripts/Makefile.modpost
@@ -78,7 +78,7 @@ modpost = scripts/mod/modpost                    \
  $(if $(KBUILD_EXTRA_SYMBOLS), $(patsubst %, -e %,$(KBUILD_EXTRA_SYMBOLS))) \
  $(if $(KBUILD_EXTMOD),-o $(modulesymfile))      \
  $(if $(CONFIG_SECTION_MISMATCH_WARN_ONLY),,-E)  \
- $(if $(KBUILD_EXTMOD)$(KBUILD_MODPOST_WARN),-w)
+ $(if $(KBUILD_MODPOST_WARN),-w)
 
 MODPOST_OPT=$(subst -i,-n,$(filter -i,$(MAKEFLAGS)))
 
diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl
index a09333fd7cef..bb28b178d929 100755
--- a/scripts/checkpatch.pl
+++ b/scripts/checkpatch.pl
@@ -2687,6 +2687,24 @@ sub process {
 			} else {
 				$signatures{$sig_nospace} = 1;
 			}
+
+# Check Co-developed-by: immediately followed by Signed-off-by: with same name and email
+			if ($sign_off =~ /^co-developed-by:$/i) {
+				if ($email eq $author) {
+					WARN("BAD_SIGN_OFF",
+					      "Co-developed-by: should not be used to attribute nominal patch author '$author'\n" . "$here\n" . $rawline);
+				}
+				if (!defined $lines[$linenr]) {
+					WARN("BAD_SIGN_OFF",
+                                             "Co-developed-by: must be immediately followed by Signed-off-by:\n" . "$here\n" . $rawline);
+				} elsif ($rawlines[$linenr] !~ /^\s*signed-off-by:\s*(.*)/i) {
+					WARN("BAD_SIGN_OFF",
+					     "Co-developed-by: must be immediately followed by Signed-off-by:\n" . "$here\n" . $rawline . "\n" .$rawlines[$linenr]);
+				} elsif ($1 ne $email) {
+					WARN("BAD_SIGN_OFF",
+					     "Co-developed-by and Signed-off-by: name/email do not match \n" . "$here\n" . $rawline . "\n" .$rawlines[$linenr]);
+				}
+			}
 		}
 
 # Check email subject for common tools that don't need to be mentioned
diff --git a/scripts/documentation-file-ref-check b/scripts/documentation-file-ref-check
index ad9db6821824..63e9542656f1 100755
--- a/scripts/documentation-file-ref-check
+++ b/scripts/documentation-file-ref-check
@@ -30,6 +30,34 @@ print "Finding broken references. This may take a while...  " if ($fix);
 
 my %broken_ref;
 
+my $doc_fix = 0;
+
+open IN, "git grep ':doc:\`' Documentation/|"
+     or die "Failed to run git grep";
+while (<IN>) {
+	next if (!m,^([^:]+):.*\:doc\:\`([^\`]+)\`,);
+
+	my $d = $1;
+	my $doc_ref = $2;
+
+	my $f = $doc_ref;
+
+	$d =~ s,(.*/).*,$1,;
+	$f =~ s,.*\<([^\>]+)\>,$1,;
+
+	$f ="$d$f.rst";
+
+	next if (grep -e, glob("$f"));
+
+	if ($fix && !$doc_fix) {
+		print STDERR "\nWARNING: Currently, can't fix broken :doc:`` fields\n";
+	}
+	$doc_fix++;
+
+	print STDERR "$f: :doc:`$doc_ref`\n";
+}
+close IN;
+
 open IN, "git grep 'Documentation/'|"
      or die "Failed to run git grep";
 while (<IN>) {
@@ -38,6 +66,9 @@ while (<IN>) {
 	my $f = $1;
 	my $ln = $2;
 
+	# On linux-next, discard the Next/ directory
+	next if ($f =~ m,^Next/,);
+
 	# Makefiles and scripts contain nasty expressions to parse docs
 	next if ($f =~ m/Makefile/ || $f =~ m/\.sh$/);
 
@@ -100,6 +131,7 @@ while (<IN>) {
 		}
 	}
 }
+close IN;
 
 exit 0 if (!$fix);
 
diff --git a/scripts/gdb/linux/Makefile b/scripts/gdb/linux/Makefile
index 3df395a9c2ce..9fd3d8ed731a 100644
--- a/scripts/gdb/linux/Makefile
+++ b/scripts/gdb/linux/Makefile
@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0
 
-ifneq ($(KBUILD_SRC),)
+ifneq ($(srctree),.)
 
 symlinks := $(patsubst $(srctree)/$(src)/%,%,$(wildcard $(srctree)/$(src)/*.py))
 
diff --git a/scripts/link-vmlinux.sh b/scripts/link-vmlinux.sh
index e3c06b9482a2..a7124f895b24 100755
--- a/scripts/link-vmlinux.sh
+++ b/scripts/link-vmlinux.sh
@@ -212,6 +212,9 @@ modpost_link vmlinux.o
 # modpost vmlinux.o to check for section mismatches
 ${MAKE} -f "${srctree}/scripts/Makefile.modpost" vmlinux.o
 
+info MODINFO modules.builtin.modinfo
+${OBJCOPY} -j .modinfo -O binary vmlinux.o modules.builtin.modinfo
+
 kallsymso=""
 kallsyms_vmlinux=""
 if [ -n "${CONFIG_KALLSYMS}" ]; then
diff --git a/scripts/mkmakefile b/scripts/mkmakefile
index 412f13fdff52..4d0faebb1719 100755
--- a/scripts/mkmakefile
+++ b/scripts/mkmakefile
@@ -7,33 +7,11 @@
 # Usage
 # $1 - Kernel src directory
 
-# Only overwrite automatically generated Makefiles
-# (so we do not overwrite kernel Makefile)
-if test -e Makefile && ! grep -q Automatically Makefile
-then
-	exit 0
-fi
 if [ "${quiet}" != "silent_" ]; then
 	echo "  GEN     Makefile"
 fi
 
 cat << EOF > Makefile
-# Automatically generated by $0: don't edit
-
-ifeq ("\$(origin V)", "command line")
-VERBOSE := \$(V)
-endif
-ifneq (\$(VERBOSE),1)
-Q := @
-endif
-
-MAKEFLAGS += --no-print-directory
-
-.PHONY: __sub-make \$(MAKECMDGOALS)
-
-__sub-make:
-	\$(Q)\$(MAKE) -C $1 O=\$(CURDIR) \$(MAKECMDGOALS)
-
-\$(filter-out __sub-make, \$(MAKECMDGOALS)): __sub-make
-	@:
+# Automatically generated by $(realpath $0): don't edit
+include $(realpath $1/Makefile)
 EOF
diff --git a/scripts/recordmcount.pl b/scripts/recordmcount.pl
index 68841d01162c..d24759214efd 100755
--- a/scripts/recordmcount.pl
+++ b/scripts/recordmcount.pl
@@ -397,6 +397,9 @@ if ($arch eq "x86_64") {
 } elsif ($arch eq "nds32") {
     $mcount_regex = "^\\s*([0-9a-fA-F]+):\\s*R_NDS32_HI20_RELA\\s+_mcount\$";
     $alignment = 2;
+} elsif ($arch eq "csky") {
+    $mcount_regex = "^\\s*([0-9a-fA-F]+):\\s*R_CKCORE_PCREL_JSR_IMM26BY2\\s+_mcount\$";
+    $alignment = 2;
 } else {
     die "Arch $arch is not supported with CONFIG_FTRACE_MCOUNT_RECORD";
 }
@@ -493,7 +496,7 @@ sub update_funcs
 #
 # Step 2: find the sections and mcount call sites
 #
-open(IN, "$objdump -hdr $inputfile|") || die "error running $objdump";
+open(IN, "LANG=C $objdump -hdr $inputfile|") || die "error running $objdump";
 
 my $text;
 
diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install
index 067459760a7b..f6a5c0bae31e 100755
--- a/scripts/sphinx-pre-install
+++ b/scripts/sphinx-pre-install
@@ -532,6 +532,7 @@ sub check_needs()
 	check_program("dot", 1);
 	check_program("convert", 1);
 	check_program("rsvg-convert", 1) if ($pdf);
+	check_program("latexmk", 1) if ($pdf);
 
 	check_distros();
 
diff --git a/scripts/tags.sh b/scripts/tags.sh
index f470d9919ed7..70e14c67bde7 100755
--- a/scripts/tags.sh
+++ b/scripts/tags.sh
@@ -19,7 +19,7 @@ ignore="$ignore ( -name *.mod.c ) -prune -o"
 # Do not use full path if we do not use O=.. builds
 # Use make O=. {tags|cscope}
 # to force full paths for a non-O= build
-if [ "${KBUILD_SRC}" = "" ]; then
+if [ "${srctree}" = "." -o -z "${srctree}" ]; then
 	tree=
 else
 	tree=${srctree}/
diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c
index c61787b15f27..d82b87c16b0a 100644
--- a/security/selinux/hooks.c
+++ b/security/selinux/hooks.c
@@ -4649,7 +4649,7 @@ static int selinux_socket_connect_helper(struct socket *sock,
 		struct lsm_network_audit net = {0,};
 		struct sockaddr_in *addr4 = NULL;
 		struct sockaddr_in6 *addr6 = NULL;
-		unsigned short snum;
+		unsigned short snum = 0;
 		u32 sid, perm;
 
 		/* sctp_connectx(3) calls via selinux_sctp_bind_connect()
@@ -4674,12 +4674,12 @@ static int selinux_socket_connect_helper(struct socket *sock,
 			break;
 		default:
 			/* Note that SCTP services expect -EINVAL, whereas
-			 * others expect -EAFNOSUPPORT.
+			 * others must handle this at the protocol level:
+			 * connect(AF_UNSPEC) on a connected socket is
+			 * a documented way disconnect the socket.
 			 */
 			if (sksec->sclass == SECCLASS_SCTP_SOCKET)
 				return -EINVAL;
-			else
-				return -EAFNOSUPPORT;
 		}
 
 		err = sel_netport_sid(sk->sk_protocol, snum, &sid);
diff --git a/security/smack/smack.h b/security/smack/smack.h
index cf52af77d15e..e41ca1d58484 100644
--- a/security/smack/smack.h
+++ b/security/smack/smack.h
@@ -348,6 +348,7 @@ extern struct list_head smack_onlycap_list;
 
 #define SMACK_HASH_SLOTS 16
 extern struct hlist_head smack_known_hash[SMACK_HASH_SLOTS];
+extern struct kmem_cache *smack_rule_cache;
 
 static inline struct task_smack *smack_cred(const struct cred *cred)
 {
diff --git a/security/smack/smack_lsm.c b/security/smack/smack_lsm.c
index 5c1613519d5a..0de725f88bed 100644
--- a/security/smack/smack_lsm.c
+++ b/security/smack/smack_lsm.c
@@ -59,6 +59,7 @@ DEFINE_MUTEX(smack_ipv6_lock);
 static LIST_HEAD(smk_ipv6_port_list);
 #endif
 static struct kmem_cache *smack_inode_cache;
+struct kmem_cache *smack_rule_cache;
 int smack_enabled;
 
 #define A(s) {"smack"#s, sizeof("smack"#s) - 1, Opt_##s}
@@ -354,7 +355,7 @@ static int smk_copy_rules(struct list_head *nhead, struct list_head *ohead,
 	int rc = 0;
 
 	list_for_each_entry_rcu(orp, ohead, list) {
-		nrp = kzalloc(sizeof(struct smack_rule), gfp);
+		nrp = kmem_cache_zalloc(smack_rule_cache, gfp);
 		if (nrp == NULL) {
 			rc = -ENOMEM;
 			break;
@@ -1931,7 +1932,7 @@ static void smack_cred_free(struct cred *cred)
 	list_for_each_safe(l, n, &tsp->smk_rules) {
 		rp = list_entry(l, struct smack_rule, list);
 		list_del(&rp->list);
-		kfree(rp);
+		kmem_cache_free(smack_rule_cache, rp);
 	}
 }
 
@@ -2805,13 +2806,17 @@ static int smack_socket_socketpair(struct socket *socka,
  *
  * Records the label bound to a port.
  *
- * Returns 0
+ * Returns 0 on success, and error code otherwise
  */
 static int smack_socket_bind(struct socket *sock, struct sockaddr *address,
 				int addrlen)
 {
-	if (sock->sk != NULL && sock->sk->sk_family == PF_INET6)
+	if (sock->sk != NULL && sock->sk->sk_family == PF_INET6) {
+		if (addrlen < SIN6_LEN_RFC2133 ||
+		    address->sa_family != AF_INET6)
+			return -EINVAL;
 		smk_ipv6_port_label(sock, address);
+	}
 	return 0;
 }
 #endif /* SMACK_IPV6_PORT_LABELING */
@@ -2847,12 +2852,13 @@ static int smack_socket_connect(struct socket *sock, struct sockaddr *sap,
 
 	switch (sock->sk->sk_family) {
 	case PF_INET:
-		if (addrlen < sizeof(struct sockaddr_in))
+		if (addrlen < sizeof(struct sockaddr_in) ||
+		    sap->sa_family != AF_INET)
 			return -EINVAL;
 		rc = smack_netlabel_send(sock->sk, (struct sockaddr_in *)sap);
 		break;
 	case PF_INET6:
-		if (addrlen < sizeof(struct sockaddr_in6))
+		if (addrlen < SIN6_LEN_RFC2133 || sap->sa_family != AF_INET6)
 			return -EINVAL;
 #ifdef SMACK_IPV6_SECMARK_LABELING
 		rsp = smack_ipv6host_label(sip);
@@ -3682,9 +3688,16 @@ static int smack_socket_sendmsg(struct socket *sock, struct msghdr *msg,
 
 	switch (sock->sk->sk_family) {
 	case AF_INET:
+		if (msg->msg_namelen < sizeof(struct sockaddr_in) ||
+		    sip->sin_family != AF_INET)
+			return -EINVAL;
 		rc = smack_netlabel_send(sock->sk, sip);
 		break;
+#if IS_ENABLED(CONFIG_IPV6)
 	case AF_INET6:
+		if (msg->msg_namelen < SIN6_LEN_RFC2133 ||
+		    sap->sin6_family != AF_INET6)
+			return -EINVAL;
 #ifdef SMACK_IPV6_SECMARK_LABELING
 		rsp = smack_ipv6host_label(sap);
 		if (rsp != NULL)
@@ -3694,6 +3707,7 @@ static int smack_socket_sendmsg(struct socket *sock, struct msghdr *msg,
 #ifdef SMACK_IPV6_PORT_LABELING
 		rc = smk_ipv6_port_check(sock->sk, sap, SMK_SENDING);
 #endif
+#endif /* IS_ENABLED(CONFIG_IPV6) */
 		break;
 	}
 	return rc;
@@ -3906,6 +3920,8 @@ access_check:
 #ifdef SMACK_IPV6_SECMARK_LABELING
 		if (skb && skb->secmark != 0)
 			skp = smack_from_secid(skb->secmark);
+		else if (smk_ipv6_localhost(&sadd))
+			break;
 		else
 			skp = smack_ipv6host_label(&sadd);
 		if (skp == NULL)
@@ -4758,6 +4774,12 @@ static __init int smack_init(void)
 	if (!smack_inode_cache)
 		return -ENOMEM;
 
+	smack_rule_cache = KMEM_CACHE(smack_rule, 0);
+	if (!smack_rule_cache) {
+		kmem_cache_destroy(smack_inode_cache);
+		return -ENOMEM;
+	}
+
 	/*
 	 * Set the security state for the initial task.
 	 */
diff --git a/security/smack/smackfs.c b/security/smack/smackfs.c
index faf2ea3968b3..47f73a0dabb1 100644
--- a/security/smack/smackfs.c
+++ b/security/smack/smackfs.c
@@ -67,7 +67,6 @@ enum smk_inos {
 /*
  * List locks
  */
-static DEFINE_MUTEX(smack_master_list_lock);
 static DEFINE_MUTEX(smack_cipso_lock);
 static DEFINE_MUTEX(smack_ambient_lock);
 static DEFINE_MUTEX(smk_net4addr_lock);
@@ -134,15 +133,7 @@ LIST_HEAD(smk_net6addr_list);
 
 /*
  * Rule lists are maintained for each label.
- * This master list is just for reading /smack/load and /smack/load2.
  */
-struct smack_master_list {
-	struct list_head	list;
-	struct smack_rule	*smk_rule;
-};
-
-static LIST_HEAD(smack_rule_list);
-
 struct smack_parsed_rule {
 	struct smack_known	*smk_subject;
 	struct smack_known	*smk_object;
@@ -211,7 +202,6 @@ static void smk_netlabel_audit_set(struct netlbl_audit *nap)
  * @srp: the rule to add or replace
  * @rule_list: the list of rules
  * @rule_lock: the rule list lock
- * @global: if non-zero, indicates a global rule
  *
  * Looks through the current subject/object/access list for
  * the subject/object pair and replaces the access that was
@@ -223,10 +213,9 @@ static void smk_netlabel_audit_set(struct netlbl_audit *nap)
  */
 static int smk_set_access(struct smack_parsed_rule *srp,
 				struct list_head *rule_list,
-				struct mutex *rule_lock, int global)
+				struct mutex *rule_lock)
 {
 	struct smack_rule *sp;
-	struct smack_master_list *smlp;
 	int found = 0;
 	int rc = 0;
 
@@ -247,7 +236,7 @@ static int smk_set_access(struct smack_parsed_rule *srp,
 	}
 
 	if (found == 0) {
-		sp = kzalloc(sizeof(*sp), GFP_KERNEL);
+		sp = kmem_cache_zalloc(smack_rule_cache, GFP_KERNEL);
 		if (sp == NULL) {
 			rc = -ENOMEM;
 			goto out;
@@ -258,22 +247,6 @@ static int smk_set_access(struct smack_parsed_rule *srp,
 		sp->smk_access = srp->smk_access1 & ~srp->smk_access2;
 
 		list_add_rcu(&sp->list, rule_list);
-		/*
-		 * If this is a global as opposed to self and a new rule
-		 * it needs to get added for reporting.
-		 */
-		if (global) {
-			mutex_unlock(rule_lock);
-			smlp = kzalloc(sizeof(*smlp), GFP_KERNEL);
-			if (smlp != NULL) {
-				smlp->smk_rule = sp;
-				mutex_lock(&smack_master_list_lock);
-				list_add_rcu(&smlp->list, &smack_rule_list);
-				mutex_unlock(&smack_master_list_lock);
-			} else
-				rc = -ENOMEM;
-			return rc;
-		}
 	}
 
 out:
@@ -540,9 +513,9 @@ static ssize_t smk_write_rules_list(struct file *file, const char __user *buf,
 
 		if (rule_list == NULL)
 			rc = smk_set_access(&rule, &rule.smk_subject->smk_rules,
-				&rule.smk_subject->smk_rules_lock, 1);
+				&rule.smk_subject->smk_rules_lock);
 		else
-			rc = smk_set_access(&rule, rule_list, rule_lock, 0);
+			rc = smk_set_access(&rule, rule_list, rule_lock);
 
 		if (rc)
 			goto out;
@@ -636,21 +609,23 @@ static void smk_rule_show(struct seq_file *s, struct smack_rule *srp, int max)
 
 static void *load2_seq_start(struct seq_file *s, loff_t *pos)
 {
-	return smk_seq_start(s, pos, &smack_rule_list);
+	return smk_seq_start(s, pos, &smack_known_list);
 }
 
 static void *load2_seq_next(struct seq_file *s, void *v, loff_t *pos)
 {
-	return smk_seq_next(s, v, pos, &smack_rule_list);
+	return smk_seq_next(s, v, pos, &smack_known_list);
 }
 
 static int load_seq_show(struct seq_file *s, void *v)
 {
 	struct list_head *list = v;
-	struct smack_master_list *smlp =
-		list_entry_rcu(list, struct smack_master_list, list);
+	struct smack_rule *srp;
+	struct smack_known *skp =
+		list_entry_rcu(list, struct smack_known, list);
 
-	smk_rule_show(s, smlp->smk_rule, SMK_LABELLEN);
+	list_for_each_entry_rcu(srp, &skp->smk_rules, list)
+		smk_rule_show(s, srp, SMK_LABELLEN);
 
 	return 0;
 }
@@ -2352,10 +2327,12 @@ static const struct file_operations smk_access_ops = {
 static int load2_seq_show(struct seq_file *s, void *v)
 {
 	struct list_head *list = v;
-	struct smack_master_list *smlp =
-		list_entry_rcu(list, struct smack_master_list, list);
+	struct smack_rule *srp;
+	struct smack_known *skp =
+		list_entry_rcu(list, struct smack_known, list);
 
-	smk_rule_show(s, smlp->smk_rule, SMK_LONGLABEL);
+	list_for_each_entry_rcu(srp, &skp->smk_rules, list)
+		smk_rule_show(s, srp, SMK_LONGLABEL);
 
 	return 0;
 }
diff --git a/sound/core/init.c b/sound/core/init.c
index 079c12d64b0e..d64416f0a281 100644
--- a/sound/core/init.c
+++ b/sound/core/init.c
@@ -49,8 +49,7 @@ static const struct file_operations snd_shutdown_f_ops;
 
 /* locked for registering/using */
 static DECLARE_BITMAP(snd_cards_lock, SNDRV_CARDS);
-struct snd_card *snd_cards[SNDRV_CARDS];
-EXPORT_SYMBOL(snd_cards);
+static struct snd_card *snd_cards[SNDRV_CARDS];
 
 static DEFINE_MUTEX(snd_card_mutex);
 
@@ -268,6 +267,26 @@ int snd_card_new(struct device *parent, int idx, const char *xid,
 }
 EXPORT_SYMBOL(snd_card_new);
 
+/**
+ * snd_card_ref - Get the card object from the index
+ * @idx: the card index
+ *
+ * Returns a card object corresponding to the given index or NULL if not found.
+ * Release the object via snd_card_unref().
+ */
+struct snd_card *snd_card_ref(int idx)
+{
+	struct snd_card *card;
+
+	mutex_lock(&snd_card_mutex);
+	card = snd_cards[idx];
+	if (card)
+		get_device(&card->card_dev);
+	mutex_unlock(&snd_card_mutex);
+	return card;
+}
+EXPORT_SYMBOL_GPL(snd_card_ref);
+
 /* return non-zero if a card is already locked */
 int snd_card_locked(int card)
 {
diff --git a/sound/core/memalloc.c b/sound/core/memalloc.c
index eb974235c92b..9f48e1d3a257 100644
--- a/sound/core/memalloc.c
+++ b/sound/core/memalloc.c
@@ -32,53 +32,6 @@
 
 /*
  *
- *  Generic memory allocators
- *
- */
-
-/**
- * snd_malloc_pages - allocate pages with the given size
- * @size: the size to allocate in bytes
- * @gfp_flags: the allocation conditions, GFP_XXX
- *
- * Allocates the physically contiguous pages with the given size.
- *
- * Return: The pointer of the buffer, or %NULL if no enough memory.
- */
-void *snd_malloc_pages(size_t size, gfp_t gfp_flags)
-{
-	int pg;
-
-	if (WARN_ON(!size))
-		return NULL;
-	if (WARN_ON(!gfp_flags))
-		return NULL;
-	gfp_flags |= __GFP_COMP;	/* compound page lets parts be mapped */
-	pg = get_order(size);
-	return (void *) __get_free_pages(gfp_flags, pg);
-}
-EXPORT_SYMBOL(snd_malloc_pages);
-
-/**
- * snd_free_pages - release the pages
- * @ptr: the buffer pointer to release
- * @size: the allocated buffer size
- *
- * Releases the buffer allocated via snd_malloc_pages().
- */
-void snd_free_pages(void *ptr, size_t size)
-{
-	int pg;
-
-	if (ptr == NULL)
-		return;
-	pg = get_order(size);
-	free_pages((unsigned long) ptr, pg);
-}
-EXPORT_SYMBOL(snd_free_pages);
-
-/*
- *
  *  Bus-specific memory allocators
  *
  */
@@ -190,8 +143,8 @@ int snd_dma_alloc_pages(int type, struct device *device, size_t size,
 	dmab->bytes = 0;
 	switch (type) {
 	case SNDRV_DMA_TYPE_CONTINUOUS:
-		dmab->area = snd_malloc_pages(size,
-					(__force gfp_t)(unsigned long)device);
+		dmab->area = alloc_pages_exact(size,
+					       (__force gfp_t)(unsigned long)device);
 		dmab->addr = 0;
 		break;
 #ifdef CONFIG_HAS_DMA
@@ -275,7 +228,7 @@ void snd_dma_free_pages(struct snd_dma_buffer *dmab)
 {
 	switch (dmab->dev.type) {
 	case SNDRV_DMA_TYPE_CONTINUOUS:
-		snd_free_pages(dmab->area, dmab->bytes);
+		free_pages_exact(dmab->area, dmab->bytes);
 		break;
 #ifdef CONFIG_HAS_DMA
 #ifdef CONFIG_GENERIC_ALLOCATOR
diff --git a/sound/core/oss/mixer_oss.c b/sound/core/oss/mixer_oss.c
index 64d904bee8bb..c8618678649c 100644
--- a/sound/core/oss/mixer_oss.c
+++ b/sound/core/oss/mixer_oss.c
@@ -1403,24 +1403,32 @@ static int snd_mixer_oss_notify_handler(struct snd_card *card, int cmd)
 
 static int __init alsa_mixer_oss_init(void)
 {
+	struct snd_card *card;
 	int idx;
 	
 	snd_mixer_oss_notify_callback = snd_mixer_oss_notify_handler;
 	for (idx = 0; idx < SNDRV_CARDS; idx++) {
-		if (snd_cards[idx])
-			snd_mixer_oss_notify_handler(snd_cards[idx], SND_MIXER_OSS_NOTIFY_REGISTER);
+		card = snd_card_ref(idx);
+		if (card) {
+			snd_mixer_oss_notify_handler(card, SND_MIXER_OSS_NOTIFY_REGISTER);
+			snd_card_unref(card);
+		}
 	}
 	return 0;
 }
 
 static void __exit alsa_mixer_oss_exit(void)
 {
+	struct snd_card *card;
 	int idx;
 
 	snd_mixer_oss_notify_callback = NULL;
 	for (idx = 0; idx < SNDRV_CARDS; idx++) {
-		if (snd_cards[idx])
-			snd_mixer_oss_notify_handler(snd_cards[idx], SND_MIXER_OSS_NOTIFY_FREE);
+		card = snd_card_ref(idx);
+		if (card) {
+			snd_mixer_oss_notify_handler(card, SND_MIXER_OSS_NOTIFY_FREE);
+			snd_card_unref(card);
+		}
 	}
 }
 
diff --git a/sound/core/pcm.c b/sound/core/pcm.c
index 7b63aee124af..998e477522fd 100644
--- a/sound/core/pcm.c
+++ b/sound/core/pcm.c
@@ -959,22 +959,22 @@ int snd_pcm_attach_substream(struct snd_pcm *pcm, int stream,
 		return -ENOMEM;
 
 	size = PAGE_ALIGN(sizeof(struct snd_pcm_mmap_status));
-	runtime->status = snd_malloc_pages(size, GFP_KERNEL);
+	runtime->status = alloc_pages_exact(size, GFP_KERNEL);
 	if (runtime->status == NULL) {
 		kfree(runtime);
 		return -ENOMEM;
 	}
-	memset((void*)runtime->status, 0, size);
+	memset(runtime->status, 0, size);
 
 	size = PAGE_ALIGN(sizeof(struct snd_pcm_mmap_control));
-	runtime->control = snd_malloc_pages(size, GFP_KERNEL);
+	runtime->control = alloc_pages_exact(size, GFP_KERNEL);
 	if (runtime->control == NULL) {
-		snd_free_pages((void*)runtime->status,
+		free_pages_exact(runtime->status,
 			       PAGE_ALIGN(sizeof(struct snd_pcm_mmap_status)));
 		kfree(runtime);
 		return -ENOMEM;
 	}
-	memset((void*)runtime->control, 0, size);
+	memset(runtime->control, 0, size);
 
 	init_waitqueue_head(&runtime->sleep);
 	init_waitqueue_head(&runtime->tsleep);
@@ -1000,9 +1000,9 @@ void snd_pcm_detach_substream(struct snd_pcm_substream *substream)
 	runtime = substream->runtime;
 	if (runtime->private_free != NULL)
 		runtime->private_free(runtime);
-	snd_free_pages((void*)runtime->status,
+	free_pages_exact(runtime->status,
 		       PAGE_ALIGN(sizeof(struct snd_pcm_mmap_status)));
-	snd_free_pages((void*)runtime->control,
+	free_pages_exact(runtime->control,
 		       PAGE_ALIGN(sizeof(struct snd_pcm_mmap_control)));
 	kfree(runtime->hw_constraints.rules);
 	/* Avoid concurrent access to runtime via PCM timer interface */
diff --git a/sound/core/seq/oss/seq_oss_device.h b/sound/core/seq/oss/seq_oss_device.h
index 2d0e9eaf13aa..77eb1fe1155c 100644
--- a/sound/core/seq/oss/seq_oss_device.h
+++ b/sound/core/seq/oss/seq_oss_device.h
@@ -30,6 +30,7 @@
 #include <sound/rawmidi.h>
 #include <sound/seq_kernel.h>
 #include <sound/info.h>
+#include "../seq_clientmgr.h"
 
 /* max. applications */
 #define SNDRV_SEQ_OSS_MAX_CLIENTS	16
@@ -150,11 +151,16 @@ snd_seq_oss_dispatch(struct seq_oss_devinfo *dp, struct snd_seq_event *ev, int a
 	return snd_seq_kernel_client_dispatch(dp->cseq, ev, atomic, hop);
 }
 
-/* ioctl */
+/* ioctl for writeq */
 static inline int
 snd_seq_oss_control(struct seq_oss_devinfo *dp, unsigned int type, void *arg)
 {
-	return snd_seq_kernel_client_ctl(dp->cseq, type, arg);
+	int err;
+
+	snd_seq_client_ioctl_lock(dp->cseq);
+	err = snd_seq_kernel_client_ctl(dp->cseq, type, arg);
+	snd_seq_client_ioctl_unlock(dp->cseq);
+	return err;
 }
 
 /* fill the addresses in header */
diff --git a/sound/core/seq/oss/seq_oss_rw.c b/sound/core/seq/oss/seq_oss_rw.c
index 30886f5fb100..eb1ef12181f3 100644
--- a/sound/core/seq/oss/seq_oss_rw.c
+++ b/sound/core/seq/oss/seq_oss_rw.c
@@ -180,14 +180,11 @@ insert_queue(struct seq_oss_devinfo *dp, union evrec *rec, struct file *opt)
 		return 0; /* invalid event - no need to insert queue */
 
 	event.time.tick = snd_seq_oss_timer_cur_tick(dp->timer);
-	if (dp->timer->realtime || !dp->timer->running) {
+	if (dp->timer->realtime || !dp->timer->running)
 		snd_seq_oss_dispatch(dp, &event, 0, 0);
-	} else {
-		if (is_nonblock_mode(dp->file_mode))
-			rc = snd_seq_kernel_client_enqueue(dp->cseq, &event, 0, 0);
-		else
-			rc = snd_seq_kernel_client_enqueue_blocking(dp->cseq, &event, opt, 0, 0);
-	}
+	else
+		rc = snd_seq_kernel_client_enqueue(dp->cseq, &event, opt,
+						   !is_nonblock_mode(dp->file_mode));
 	return rc;
 }
 		
diff --git a/sound/core/seq/oss/seq_oss_writeq.c b/sound/core/seq/oss/seq_oss_writeq.c
index 5e04f4df10e4..b2f69617591f 100644
--- a/sound/core/seq/oss/seq_oss_writeq.c
+++ b/sound/core/seq/oss/seq_oss_writeq.c
@@ -116,7 +116,7 @@ snd_seq_oss_writeq_sync(struct seq_oss_writeq *q)
 		rec->t.code = SEQ_SYNCTIMER;
 		rec->t.time = time;
 		q->sync_event_put = 1;
-		snd_seq_kernel_client_enqueue_blocking(dp->cseq, &ev, NULL, 0, 0);
+		snd_seq_kernel_client_enqueue(dp->cseq, &ev, NULL, true);
 	}
 
 	wait_event_interruptible_timeout(q->sync_sleep, ! q->sync_event_put, HZ);
diff --git a/sound/core/seq/seq_clientmgr.c b/sound/core/seq/seq_clientmgr.c
index a11bdc0350fc..b3f593ee752e 100644
--- a/sound/core/seq/seq_clientmgr.c
+++ b/sound/core/seq/seq_clientmgr.c
@@ -179,6 +179,41 @@ struct snd_seq_client *snd_seq_client_use_ptr(int clientid)
 	return client;
 }
 
+/* Take refcount and perform ioctl_mutex lock on the given client;
+ * used only for OSS sequencer
+ * Unlock via snd_seq_client_ioctl_unlock() below
+ */
+bool snd_seq_client_ioctl_lock(int clientid)
+{
+	struct snd_seq_client *client;
+
+	client = snd_seq_client_use_ptr(clientid);
+	if (!client)
+		return false;
+	mutex_lock(&client->ioctl_mutex);
+	/* The client isn't unrefed here; see snd_seq_client_ioctl_unlock() */
+	return true;
+}
+EXPORT_SYMBOL_GPL(snd_seq_client_ioctl_lock);
+
+/* Unlock and unref the given client; for OSS sequencer use only */
+void snd_seq_client_ioctl_unlock(int clientid)
+{
+	struct snd_seq_client *client;
+
+	client = snd_seq_client_use_ptr(clientid);
+	if (WARN_ON(!client))
+		return;
+	mutex_unlock(&client->ioctl_mutex);
+	/* The doubly unrefs below are intentional; the first one releases the
+	 * leftover from snd_seq_client_ioctl_lock() above, and the second one
+	 * is for releasing snd_seq_client_use_ptr() in this function
+	 */
+	snd_seq_client_unlock(client);
+	snd_seq_client_unlock(client);
+}
+EXPORT_SYMBOL_GPL(snd_seq_client_ioctl_unlock);
+
 static void usage_alloc(struct snd_seq_usage *res, int num)
 {
 	res->cur += num;
@@ -203,7 +238,6 @@ int __init client_init_data(void)
 
 static struct snd_seq_client *seq_create_client1(int client_index, int poolsize)
 {
-	unsigned long flags;
 	int c;
 	struct snd_seq_client *client;
 
@@ -224,7 +258,7 @@ static struct snd_seq_client *seq_create_client1(int client_index, int poolsize)
 	mutex_init(&client->ioctl_mutex);
 
 	/* find free slot in the client table */
-	spin_lock_irqsave(&clients_lock, flags);
+	spin_lock_irq(&clients_lock);
 	if (client_index < 0) {
 		for (c = SNDRV_SEQ_DYNAMIC_CLIENTS_BEGIN;
 		     c < SNDRV_SEQ_MAX_CLIENTS;
@@ -232,17 +266,17 @@ static struct snd_seq_client *seq_create_client1(int client_index, int poolsize)
 			if (clienttab[c] || clienttablock[c])
 				continue;
 			clienttab[client->number = c] = client;
-			spin_unlock_irqrestore(&clients_lock, flags);
+			spin_unlock_irq(&clients_lock);
 			return client;
 		}
 	} else {
 		if (clienttab[client_index] == NULL && !clienttablock[client_index]) {
 			clienttab[client->number = client_index] = client;
-			spin_unlock_irqrestore(&clients_lock, flags);
+			spin_unlock_irq(&clients_lock);
 			return client;
 		}
 	}
-	spin_unlock_irqrestore(&clients_lock, flags);
+	spin_unlock_irq(&clients_lock);
 	snd_seq_pool_delete(&client->pool);
 	kfree(client);
 	return NULL;	/* no free slot found or busy, return failure code */
@@ -251,23 +285,21 @@ static struct snd_seq_client *seq_create_client1(int client_index, int poolsize)
 
 static int seq_free_client1(struct snd_seq_client *client)
 {
-	unsigned long flags;
-
 	if (!client)
 		return 0;
-	spin_lock_irqsave(&clients_lock, flags);
+	spin_lock_irq(&clients_lock);
 	clienttablock[client->number] = 1;
 	clienttab[client->number] = NULL;
-	spin_unlock_irqrestore(&clients_lock, flags);
+	spin_unlock_irq(&clients_lock);
 	snd_seq_delete_all_ports(client);
 	snd_seq_queue_client_leave(client->number);
 	snd_use_lock_sync(&client->use_lock);
 	snd_seq_queue_client_termination(client->number);
 	if (client->pool)
 		snd_seq_pool_delete(&client->pool);
-	spin_lock_irqsave(&clients_lock, flags);
+	spin_lock_irq(&clients_lock);
 	clienttablock[client->number] = 0;
-	spin_unlock_irqrestore(&clients_lock, flags);
+	spin_unlock_irq(&clients_lock);
 	return 0;
 }
 
@@ -1900,20 +1932,14 @@ static int snd_seq_ioctl_get_subscription(struct snd_seq_client *client,
 	int result;
 	struct snd_seq_client *sender = NULL;
 	struct snd_seq_client_port *sport = NULL;
-	struct snd_seq_subscribers *p;
 
 	result = -EINVAL;
 	if ((sender = snd_seq_client_use_ptr(subs->sender.client)) == NULL)
 		goto __end;
 	if ((sport = snd_seq_port_use_ptr(sender, subs->sender.port)) == NULL)
 		goto __end;
-	p = snd_seq_port_get_subscription(&sport->c_src, &subs->dest);
-	if (p) {
-		result = 0;
-		*subs = p->info;
-	} else
-		result = -ENOENT;
-
+	result = snd_seq_port_get_subscription(&sport->c_src, &subs->dest,
+					       subs);
       __end:
       	if (sport)
 		snd_seq_port_unlock(sport);
@@ -2227,12 +2253,13 @@ int snd_seq_delete_kernel_client(int client)
 }
 EXPORT_SYMBOL(snd_seq_delete_kernel_client);
 
-/* skeleton to enqueue event, called from snd_seq_kernel_client_enqueue
- * and snd_seq_kernel_client_enqueue_blocking
+/*
+ * exported, called by kernel clients to enqueue events (w/o blocking)
+ *
+ * RETURN VALUE: zero if succeed, negative if error
  */
-static int kernel_client_enqueue(int client, struct snd_seq_event *ev,
-				 struct file *file, int blocking,
-				 int atomic, int hop)
+int snd_seq_kernel_client_enqueue(int client, struct snd_seq_event *ev,
+				  struct file *file, bool blocking)
 {
 	struct snd_seq_client *cptr;
 	int result;
@@ -2255,41 +2282,21 @@ static int kernel_client_enqueue(int client, struct snd_seq_event *ev,
 	if (cptr == NULL)
 		return -EINVAL;
 	
-	if (! cptr->accept_output)
+	if (!cptr->accept_output) {
 		result = -EPERM;
-	else /* send it */
+	} else { /* send it */
+		mutex_lock(&cptr->ioctl_mutex);
 		result = snd_seq_client_enqueue_event(cptr, ev, file, blocking,
-						      atomic, hop, NULL);
+						      false, 0,
+						      &cptr->ioctl_mutex);
+		mutex_unlock(&cptr->ioctl_mutex);
+	}
 
 	snd_seq_client_unlock(cptr);
 	return result;
 }
-
-/*
- * exported, called by kernel clients to enqueue events (w/o blocking)
- *
- * RETURN VALUE: zero if succeed, negative if error
- */
-int snd_seq_kernel_client_enqueue(int client, struct snd_seq_event * ev,
-				  int atomic, int hop)
-{
-	return kernel_client_enqueue(client, ev, NULL, 0, atomic, hop);
-}
 EXPORT_SYMBOL(snd_seq_kernel_client_enqueue);
 
-/*
- * exported, called by kernel clients to enqueue events (with blocking)
- *
- * RETURN VALUE: zero if succeed, negative if error
- */
-int snd_seq_kernel_client_enqueue_blocking(int client, struct snd_seq_event * ev,
-					   struct file *file,
-					   int atomic, int hop)
-{
-	return kernel_client_enqueue(client, ev, file, 1, atomic, hop);
-}
-EXPORT_SYMBOL(snd_seq_kernel_client_enqueue_blocking);
-
 /* 
  * exported, called by kernel clients to dispatch events directly to other
  * clients, bypassing the queues.  Event time-stamp will be updated.
diff --git a/sound/core/seq/seq_clientmgr.h b/sound/core/seq/seq_clientmgr.h
index 0611e1e0ed5b..28a51dcc0190 100644
--- a/sound/core/seq/seq_clientmgr.h
+++ b/sound/core/seq/seq_clientmgr.h
@@ -93,14 +93,14 @@ struct snd_seq_client *snd_seq_client_use_ptr(int clientid);
 /* dispatch event to client(s) */
 int snd_seq_dispatch_event(struct snd_seq_event_cell *cell, int atomic, int hop);
 
-/* exported to other modules */
-int snd_seq_kernel_client_enqueue(int client, struct snd_seq_event *ev, int atomic, int hop);
-int snd_seq_kernel_client_enqueue_blocking(int client, struct snd_seq_event * ev,
-					   struct file *file, int atomic, int hop);
 int snd_seq_kernel_client_write_poll(int clientid, struct file *file, poll_table *wait);
 int snd_seq_client_notify_subscription(int client, int port,
 				       struct snd_seq_port_subscribe *info, int evtype);
 
+/* only for OSS sequencer */
+bool snd_seq_client_ioctl_lock(int clientid);
+void snd_seq_client_ioctl_unlock(int clientid);
+
 extern int seq_client_load[15];
 
 #endif
diff --git a/sound/core/seq/seq_fifo.c b/sound/core/seq/seq_fifo.c
index 72c0302a55d2..97ee89cb6426 100644
--- a/sound/core/seq/seq_fifo.c
+++ b/sound/core/seq/seq_fifo.c
@@ -98,18 +98,17 @@ static struct snd_seq_event_cell *fifo_cell_out(struct snd_seq_fifo *f);
 void snd_seq_fifo_clear(struct snd_seq_fifo *f)
 {
 	struct snd_seq_event_cell *cell;
-	unsigned long flags;
 
 	/* clear overflow flag */
 	atomic_set(&f->overflow, 0);
 
 	snd_use_lock_sync(&f->use_lock);
-	spin_lock_irqsave(&f->lock, flags);
+	spin_lock_irq(&f->lock);
 	/* drain the fifo */
 	while ((cell = fifo_cell_out(f)) != NULL) {
 		snd_seq_cell_free(cell);
 	}
-	spin_unlock_irqrestore(&f->lock, flags);
+	spin_unlock_irq(&f->lock);
 }
 
 
@@ -195,9 +194,9 @@ int snd_seq_fifo_cell_out(struct snd_seq_fifo *f,
 		}
 		set_current_state(TASK_INTERRUPTIBLE);
 		add_wait_queue(&f->input_sleep, &wait);
-		spin_unlock_irq(&f->lock);
+		spin_unlock_irqrestore(&f->lock, flags);
 		schedule();
-		spin_lock_irq(&f->lock);
+		spin_lock_irqsave(&f->lock, flags);
 		remove_wait_queue(&f->input_sleep, &wait);
 		if (signal_pending(current)) {
 			spin_unlock_irqrestore(&f->lock, flags);
@@ -239,7 +238,6 @@ int snd_seq_fifo_poll_wait(struct snd_seq_fifo *f, struct file *file,
 /* change the size of pool; all old events are removed */
 int snd_seq_fifo_resize(struct snd_seq_fifo *f, int poolsize)
 {
-	unsigned long flags;
 	struct snd_seq_pool *newpool, *oldpool;
 	struct snd_seq_event_cell *cell, *next, *oldhead;
 
@@ -255,7 +253,7 @@ int snd_seq_fifo_resize(struct snd_seq_fifo *f, int poolsize)
 		return -ENOMEM;
 	}
 
-	spin_lock_irqsave(&f->lock, flags);
+	spin_lock_irq(&f->lock);
 	/* remember old pool */
 	oldpool = f->pool;
 	oldhead = f->head;
@@ -265,7 +263,7 @@ int snd_seq_fifo_resize(struct snd_seq_fifo *f, int poolsize)
 	f->tail = NULL;
 	f->cells = 0;
 	/* NOTE: overflow flag is not cleared */
-	spin_unlock_irqrestore(&f->lock, flags);
+	spin_unlock_irq(&f->lock);
 
 	/* close the old pool and wait until all users are gone */
 	snd_seq_pool_mark_closing(oldpool);
diff --git a/sound/core/seq/seq_memory.c b/sound/core/seq/seq_memory.c
index 5b0388202bac..19b718e871c5 100644
--- a/sound/core/seq/seq_memory.c
+++ b/sound/core/seq/seq_memory.c
@@ -24,7 +24,7 @@
 #include <linux/export.h>
 #include <linux/slab.h>
 #include <linux/sched/signal.h>
-#include <linux/vmalloc.h>
+#include <linux/mm.h>
 #include <sound/core.h>
 
 #include <sound/seq_kernel.h>
@@ -244,13 +244,13 @@ static int snd_seq_cell_alloc(struct snd_seq_pool *pool,
 
 		set_current_state(TASK_INTERRUPTIBLE);
 		add_wait_queue(&pool->output_sleep, &wait);
-		spin_unlock_irq(&pool->lock);
+		spin_unlock_irqrestore(&pool->lock, flags);
 		if (mutexp)
 			mutex_unlock(mutexp);
 		schedule();
 		if (mutexp)
 			mutex_lock(mutexp);
-		spin_lock_irq(&pool->lock);
+		spin_lock_irqsave(&pool->lock, flags);
 		remove_wait_queue(&pool->output_sleep, &wait);
 		/* interrupted? */
 		if (signal_pending(current)) {
@@ -384,21 +384,20 @@ int snd_seq_pool_init(struct snd_seq_pool *pool)
 {
 	int cell;
 	struct snd_seq_event_cell *cellptr;
-	unsigned long flags;
 
 	if (snd_BUG_ON(!pool))
 		return -EINVAL;
 
-	cellptr = vmalloc(array_size(sizeof(struct snd_seq_event_cell),
-				     pool->size));
+	cellptr = kvmalloc_array(sizeof(struct snd_seq_event_cell), pool->size,
+				 GFP_KERNEL);
 	if (!cellptr)
 		return -ENOMEM;
 
 	/* add new cells to the free cell list */
-	spin_lock_irqsave(&pool->lock, flags);
+	spin_lock_irq(&pool->lock);
 	if (pool->ptr) {
-		spin_unlock_irqrestore(&pool->lock, flags);
-		vfree(cellptr);
+		spin_unlock_irq(&pool->lock);
+		kvfree(cellptr);
 		return 0;
 	}
 
@@ -416,7 +415,7 @@ int snd_seq_pool_init(struct snd_seq_pool *pool)
 	/* init statistics */
 	pool->max_used = 0;
 	pool->total_elements = pool->size;
-	spin_unlock_irqrestore(&pool->lock, flags);
+	spin_unlock_irq(&pool->lock);
 	return 0;
 }
 
@@ -435,7 +434,6 @@ void snd_seq_pool_mark_closing(struct snd_seq_pool *pool)
 /* remove events */
 int snd_seq_pool_done(struct snd_seq_pool *pool)
 {
-	unsigned long flags;
 	struct snd_seq_event_cell *ptr;
 
 	if (snd_BUG_ON(!pool))
@@ -449,18 +447,18 @@ int snd_seq_pool_done(struct snd_seq_pool *pool)
 		schedule_timeout_uninterruptible(1);
 	
 	/* release all resources */
-	spin_lock_irqsave(&pool->lock, flags);
+	spin_lock_irq(&pool->lock);
 	ptr = pool->ptr;
 	pool->ptr = NULL;
 	pool->free = NULL;
 	pool->total_elements = 0;
-	spin_unlock_irqrestore(&pool->lock, flags);
+	spin_unlock_irq(&pool->lock);
 
-	vfree(ptr);
+	kvfree(ptr);
 
-	spin_lock_irqsave(&pool->lock, flags);
+	spin_lock_irq(&pool->lock);
 	pool->closing = 0;
-	spin_unlock_irqrestore(&pool->lock, flags);
+	spin_unlock_irq(&pool->lock);
 
 	return 0;
 }
diff --git a/sound/core/seq/seq_ports.c b/sound/core/seq/seq_ports.c
index 24d90abfc64d..ac7556ab531c 100644
--- a/sound/core/seq/seq_ports.c
+++ b/sound/core/seq/seq_ports.c
@@ -128,7 +128,6 @@ static void port_subs_info_init(struct snd_seq_port_subs_info *grp)
 struct snd_seq_client_port *snd_seq_create_port(struct snd_seq_client *client,
 						int port)
 {
-	unsigned long flags;
 	struct snd_seq_client_port *new_port, *p;
 	int num = -1;
 	
@@ -157,7 +156,7 @@ struct snd_seq_client_port *snd_seq_create_port(struct snd_seq_client *client,
 
 	num = port >= 0 ? port : 0;
 	mutex_lock(&client->ports_mutex);
-	write_lock_irqsave(&client->ports_lock, flags);
+	write_lock_irq(&client->ports_lock);
 	list_for_each_entry(p, &client->ports_list_head, list) {
 		if (p->addr.port > num)
 			break;
@@ -169,7 +168,7 @@ struct snd_seq_client_port *snd_seq_create_port(struct snd_seq_client *client,
 	client->num_ports++;
 	new_port->addr.port = num;	/* store the port number in the port */
 	sprintf(new_port->name, "port-%d", num);
-	write_unlock_irqrestore(&client->ports_lock, flags);
+	write_unlock_irq(&client->ports_lock);
 	mutex_unlock(&client->ports_mutex);
 
 	return new_port;
@@ -283,11 +282,10 @@ static int port_delete(struct snd_seq_client *client,
 /* delete a port with the given port id */
 int snd_seq_delete_port(struct snd_seq_client *client, int port)
 {
-	unsigned long flags;
 	struct snd_seq_client_port *found = NULL, *p;
 
 	mutex_lock(&client->ports_mutex);
-	write_lock_irqsave(&client->ports_lock, flags);
+	write_lock_irq(&client->ports_lock);
 	list_for_each_entry(p, &client->ports_list_head, list) {
 		if (p->addr.port == port) {
 			/* ok found.  delete from the list at first */
@@ -297,7 +295,7 @@ int snd_seq_delete_port(struct snd_seq_client *client, int port)
 			break;
 		}
 	}
-	write_unlock_irqrestore(&client->ports_lock, flags);
+	write_unlock_irq(&client->ports_lock);
 	mutex_unlock(&client->ports_mutex);
 	if (found)
 		return port_delete(client, found);
@@ -308,7 +306,6 @@ int snd_seq_delete_port(struct snd_seq_client *client, int port)
 /* delete the all ports belonging to the given client */
 int snd_seq_delete_all_ports(struct snd_seq_client *client)
 {
-	unsigned long flags;
 	struct list_head deleted_list;
 	struct snd_seq_client_port *port, *tmp;
 	
@@ -316,7 +313,7 @@ int snd_seq_delete_all_ports(struct snd_seq_client *client)
 	 * clear the port list in the client data.
 	 */
 	mutex_lock(&client->ports_mutex);
-	write_lock_irqsave(&client->ports_lock, flags);
+	write_lock_irq(&client->ports_lock);
 	if (! list_empty(&client->ports_list_head)) {
 		list_add(&deleted_list, &client->ports_list_head);
 		list_del_init(&client->ports_list_head);
@@ -324,7 +321,7 @@ int snd_seq_delete_all_ports(struct snd_seq_client *client)
 		INIT_LIST_HEAD(&deleted_list);
 	}
 	client->num_ports = 0;
-	write_unlock_irqrestore(&client->ports_lock, flags);
+	write_unlock_irq(&client->ports_lock);
 
 	/* remove each port in deleted_list */
 	list_for_each_entry_safe(port, tmp, &deleted_list, list) {
@@ -550,10 +547,10 @@ static void delete_and_unsubscribe_port(struct snd_seq_client *client,
 		list_del_init(list);
 	grp->exclusive = 0;
 	write_unlock_irq(&grp->list_lock);
-	up_write(&grp->list_mutex);
 
 	if (!empty)
 		unsubscribe_port(client, port, grp, &subs->info, ack);
+	up_write(&grp->list_mutex);
 }
 
 /* connect two ports */
@@ -635,20 +632,23 @@ int snd_seq_port_disconnect(struct snd_seq_client *connector,
 
 
 /* get matched subscriber */
-struct snd_seq_subscribers *snd_seq_port_get_subscription(struct snd_seq_port_subs_info *src_grp,
-							  struct snd_seq_addr *dest_addr)
+int snd_seq_port_get_subscription(struct snd_seq_port_subs_info *src_grp,
+				  struct snd_seq_addr *dest_addr,
+				  struct snd_seq_port_subscribe *subs)
 {
-	struct snd_seq_subscribers *s, *found = NULL;
+	struct snd_seq_subscribers *s;
+	int err = -ENOENT;
 
 	down_read(&src_grp->list_mutex);
 	list_for_each_entry(s, &src_grp->list_head, src_list) {
 		if (addr_match(dest_addr, &s->info.dest)) {
-			found = s;
+			*subs = s->info;
+			err = 0;
 			break;
 		}
 	}
 	up_read(&src_grp->list_mutex);
-	return found;
+	return err;
 }
 
 /*
diff --git a/sound/core/seq/seq_ports.h b/sound/core/seq/seq_ports.h
index 26bd71f36c41..06003b36652e 100644
--- a/sound/core/seq/seq_ports.h
+++ b/sound/core/seq/seq_ports.h
@@ -135,7 +135,8 @@ int snd_seq_port_subscribe(struct snd_seq_client_port *port,
 			   struct snd_seq_port_subscribe *info);
 
 /* get matched subscriber */
-struct snd_seq_subscribers *snd_seq_port_get_subscription(struct snd_seq_port_subs_info *src_grp,
-							  struct snd_seq_addr *dest_addr);
+int snd_seq_port_get_subscription(struct snd_seq_port_subs_info *src_grp,
+				  struct snd_seq_addr *dest_addr,
+				  struct snd_seq_port_subscribe *subs);
 
 #endif
diff --git a/sound/core/sound.c b/sound/core/sound.c
index b30f027eb0fe..a9ad4379523b 100644
--- a/sound/core/sound.c
+++ b/sound/core/sound.c
@@ -134,8 +134,11 @@ static struct snd_minor *autoload_device(unsigned int minor)
 	if (dev == SNDRV_MINOR_CONTROL) {
 		/* /dev/aloadC? */
 		int card = SNDRV_MINOR_CARD(minor);
-		if (snd_cards[card] == NULL)
+		struct snd_card *ref = snd_card_ref(card);
+		if (!ref)
 			snd_request_card(card);
+		else
+			snd_card_unref(ref);
 	} else if (dev == SNDRV_MINOR_GLOBAL) {
 		/* /dev/aloadSEQ */
 		snd_request_other(minor);
diff --git a/sound/core/timer.c b/sound/core/timer.c
index b842b61f66c2..e3973957b392 100644
--- a/sound/core/timer.c
+++ b/sound/core/timer.c
@@ -38,6 +38,7 @@
 
 /* internal flags */
 #define SNDRV_TIMER_IFLG_PAUSED		0x00010000
+#define SNDRV_TIMER_IFLG_DEAD		0x00020000
 
 #if IS_ENABLED(CONFIG_SND_HRTIMER)
 #define DEFAULT_TIMER_LIMIT 4
@@ -254,19 +255,20 @@ int snd_timer_open(struct snd_timer_instance **ti,
 	struct snd_timer_instance *timeri = NULL;
 	int err;
 
+	mutex_lock(&register_mutex);
 	if (tid->dev_class == SNDRV_TIMER_CLASS_SLAVE) {
 		/* open a slave instance */
 		if (tid->dev_sclass <= SNDRV_TIMER_SCLASS_NONE ||
 		    tid->dev_sclass > SNDRV_TIMER_SCLASS_OSS_SEQUENCER) {
 			pr_debug("ALSA: timer: invalid slave class %i\n",
 				 tid->dev_sclass);
-			return -EINVAL;
+			err = -EINVAL;
+			goto unlock;
 		}
-		mutex_lock(&register_mutex);
 		timeri = snd_timer_instance_new(owner, NULL);
 		if (!timeri) {
-			mutex_unlock(&register_mutex);
-			return -ENOMEM;
+			err = -ENOMEM;
+			goto unlock;
 		}
 		timeri->slave_class = tid->dev_sclass;
 		timeri->slave_id = tid->device;
@@ -277,13 +279,10 @@ int snd_timer_open(struct snd_timer_instance **ti,
 			snd_timer_close_locked(timeri);
 			timeri = NULL;
 		}
-		mutex_unlock(&register_mutex);
-		*ti = timeri;
-		return err;
+		goto unlock;
 	}
 
 	/* open a master instance */
-	mutex_lock(&register_mutex);
 	timer = snd_timer_find(tid);
 #ifdef CONFIG_MODULES
 	if (!timer) {
@@ -294,25 +293,26 @@ int snd_timer_open(struct snd_timer_instance **ti,
 	}
 #endif
 	if (!timer) {
-		mutex_unlock(&register_mutex);
-		return -ENODEV;
+		err = -ENODEV;
+		goto unlock;
 	}
 	if (!list_empty(&timer->open_list_head)) {
 		timeri = list_entry(timer->open_list_head.next,
 				    struct snd_timer_instance, open_list);
 		if (timeri->flags & SNDRV_TIMER_IFLG_EXCLUSIVE) {
-			mutex_unlock(&register_mutex);
-			return -EBUSY;
+			err = -EBUSY;
+			timeri = NULL;
+			goto unlock;
 		}
 	}
 	if (timer->num_instances >= timer->max_instances) {
-		mutex_unlock(&register_mutex);
-		return -EBUSY;
+		err = -EBUSY;
+		goto unlock;
 	}
 	timeri = snd_timer_instance_new(owner, timer);
 	if (!timeri) {
-		mutex_unlock(&register_mutex);
-		return -ENOMEM;
+		err = -ENOMEM;
+		goto unlock;
 	}
 	/* take a card refcount for safe disconnection */
 	if (timer->card)
@@ -321,16 +321,16 @@ int snd_timer_open(struct snd_timer_instance **ti,
 	timeri->slave_id = slave_id;
 
 	if (list_empty(&timer->open_list_head) && timer->hw.open) {
-		int err = timer->hw.open(timer);
+		err = timer->hw.open(timer);
 		if (err) {
 			kfree(timeri->owner);
 			kfree(timeri);
+			timeri = NULL;
 
 			if (timer->card)
 				put_device(&timer->card->card_dev);
 			module_put(timer->module);
-			mutex_unlock(&register_mutex);
-			return err;
+			goto unlock;
 		}
 	}
 
@@ -341,6 +341,8 @@ int snd_timer_open(struct snd_timer_instance **ti,
 		snd_timer_close_locked(timeri);
 		timeri = NULL;
 	}
+
+ unlock:
 	mutex_unlock(&register_mutex);
 	*ti = timeri;
 	return err;
@@ -353,15 +355,20 @@ EXPORT_SYMBOL(snd_timer_open);
  */
 static int snd_timer_close_locked(struct snd_timer_instance *timeri)
 {
-	struct snd_timer *timer = NULL;
+	struct snd_timer *timer = timeri->timer;
 	struct snd_timer_instance *slave, *tmp;
 
+	if (timer) {
+		spin_lock_irq(&timer->lock);
+		timeri->flags |= SNDRV_TIMER_IFLG_DEAD;
+		spin_unlock_irq(&timer->lock);
+	}
+
 	list_del(&timeri->open_list);
 
 	/* force to stop the timer */
 	snd_timer_stop(timeri);
 
-	timer = timeri->timer;
 	if (timer) {
 		timer->num_instances--;
 		/* wait, until the active callback is finished */
@@ -497,6 +504,10 @@ static int snd_timer_start1(struct snd_timer_instance *timeri,
 		return -EINVAL;
 
 	spin_lock_irqsave(&timer->lock, flags);
+	if (timeri->flags & SNDRV_TIMER_IFLG_DEAD) {
+		result = -EINVAL;
+		goto unlock;
+	}
 	if (timer->card && timer->card->shutdown) {
 		result = -ENODEV;
 		goto unlock;
@@ -541,11 +552,16 @@ static int snd_timer_start_slave(struct snd_timer_instance *timeri,
 				 bool start)
 {
 	unsigned long flags;
+	int err;
 
 	spin_lock_irqsave(&slave_active_lock, flags);
+	if (timeri->flags & SNDRV_TIMER_IFLG_DEAD) {
+		err = -EINVAL;
+		goto unlock;
+	}
 	if (timeri->flags & SNDRV_TIMER_IFLG_RUNNING) {
-		spin_unlock_irqrestore(&slave_active_lock, flags);
-		return -EBUSY;
+		err = -EBUSY;
+		goto unlock;
 	}
 	timeri->flags |= SNDRV_TIMER_IFLG_RUNNING;
 	if (timeri->master && timeri->timer) {
@@ -556,8 +572,10 @@ static int snd_timer_start_slave(struct snd_timer_instance *timeri,
 				  SNDRV_TIMER_EVENT_CONTINUE);
 		spin_unlock(&timeri->timer->lock);
 	}
+	err = 1; /* delayed start */
+ unlock:
 	spin_unlock_irqrestore(&slave_active_lock, flags);
-	return 1; /* delayed start */
+	return err;
 }
 
 /* stop/pause a master timer */
@@ -720,6 +738,46 @@ static void snd_timer_reschedule(struct snd_timer * timer, unsigned long ticks_l
 	timer->sticks = ticks;
 }
 
+/* call callbacks in timer ack list */
+static void snd_timer_process_callbacks(struct snd_timer *timer,
+					struct list_head *head)
+{
+	struct snd_timer_instance *ti;
+	unsigned long resolution, ticks;
+
+	while (!list_empty(head)) {
+		ti = list_first_entry(head, struct snd_timer_instance,
+				      ack_list);
+
+		/* remove from ack_list and make empty */
+		list_del_init(&ti->ack_list);
+
+		if (!(ti->flags & SNDRV_TIMER_IFLG_DEAD)) {
+			ticks = ti->pticks;
+			ti->pticks = 0;
+			resolution = ti->resolution;
+			ti->flags |= SNDRV_TIMER_IFLG_CALLBACK;
+			spin_unlock(&timer->lock);
+			if (ti->callback)
+				ti->callback(ti, resolution, ticks);
+			spin_lock(&timer->lock);
+			ti->flags &= ~SNDRV_TIMER_IFLG_CALLBACK;
+		}
+	}
+}
+
+/* clear pending instances from ack list */
+static void snd_timer_clear_callbacks(struct snd_timer *timer,
+				      struct list_head *head)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&timer->lock, flags);
+	while (!list_empty(head))
+		list_del_init(head->next);
+	spin_unlock_irqrestore(&timer->lock, flags);
+}
+
 /*
  * timer tasklet
  *
@@ -727,34 +785,15 @@ static void snd_timer_reschedule(struct snd_timer * timer, unsigned long ticks_l
 static void snd_timer_tasklet(unsigned long arg)
 {
 	struct snd_timer *timer = (struct snd_timer *) arg;
-	struct snd_timer_instance *ti;
-	struct list_head *p;
-	unsigned long resolution, ticks;
 	unsigned long flags;
 
-	if (timer->card && timer->card->shutdown)
+	if (timer->card && timer->card->shutdown) {
+		snd_timer_clear_callbacks(timer, &timer->sack_list_head);
 		return;
+	}
 
 	spin_lock_irqsave(&timer->lock, flags);
-	/* now process all callbacks */
-	while (!list_empty(&timer->sack_list_head)) {
-		p = timer->sack_list_head.next;		/* get first item */
-		ti = list_entry(p, struct snd_timer_instance, ack_list);
-
-		/* remove from ack_list and make empty */
-		list_del_init(p);
-
-		ticks = ti->pticks;
-		ti->pticks = 0;
-		resolution = ti->resolution;
-
-		ti->flags |= SNDRV_TIMER_IFLG_CALLBACK;
-		spin_unlock(&timer->lock);
-		if (ti->callback)
-			ti->callback(ti, resolution, ticks);
-		spin_lock(&timer->lock);
-		ti->flags &= ~SNDRV_TIMER_IFLG_CALLBACK;
-	}
+	snd_timer_process_callbacks(timer, &timer->sack_list_head);
 	spin_unlock_irqrestore(&timer->lock, flags);
 }
 
@@ -767,16 +806,18 @@ static void snd_timer_tasklet(unsigned long arg)
 void snd_timer_interrupt(struct snd_timer * timer, unsigned long ticks_left)
 {
 	struct snd_timer_instance *ti, *ts, *tmp;
-	unsigned long resolution, ticks;
-	struct list_head *p, *ack_list_head;
+	unsigned long resolution;
+	struct list_head *ack_list_head;
 	unsigned long flags;
 	int use_tasklet = 0;
 
 	if (timer == NULL)
 		return;
 
-	if (timer->card && timer->card->shutdown)
+	if (timer->card && timer->card->shutdown) {
+		snd_timer_clear_callbacks(timer, &timer->ack_list_head);
 		return;
+	}
 
 	spin_lock_irqsave(&timer->lock, flags);
 
@@ -790,6 +831,8 @@ void snd_timer_interrupt(struct snd_timer * timer, unsigned long ticks_left)
 	 */
 	list_for_each_entry_safe(ti, tmp, &timer->active_list_head,
 				 active_list) {
+		if (ti->flags & SNDRV_TIMER_IFLG_DEAD)
+			continue;
 		if (!(ti->flags & SNDRV_TIMER_IFLG_RUNNING))
 			continue;
 		ti->pticks += ticks_left;
@@ -839,23 +882,7 @@ void snd_timer_interrupt(struct snd_timer * timer, unsigned long ticks_left)
 	}
 
 	/* now process all fast callbacks */
-	while (!list_empty(&timer->ack_list_head)) {
-		p = timer->ack_list_head.next;		/* get first item */
-		ti = list_entry(p, struct snd_timer_instance, ack_list);
-
-		/* remove from ack_list and make empty */
-		list_del_init(p);
-
-		ticks = ti->pticks;
-		ti->pticks = 0;
-
-		ti->flags |= SNDRV_TIMER_IFLG_CALLBACK;
-		spin_unlock(&timer->lock);
-		if (ti->callback)
-			ti->callback(ti, resolution, ticks);
-		spin_lock(&timer->lock);
-		ti->flags &= ~SNDRV_TIMER_IFLG_CALLBACK;
-	}
+	snd_timer_process_callbacks(timer, &timer->ack_list_head);
 
 	/* do we have any slow callbacks? */
 	use_tasklet = !list_empty(&timer->sack_list_head);
@@ -1882,7 +1909,10 @@ static int snd_timer_user_start(struct file *file)
 	snd_timer_stop(tu->timeri);
 	tu->timeri->lost = 0;
 	tu->last_resolution = 0;
-	return (err = snd_timer_start(tu->timeri, tu->ticks)) < 0 ? err : 0;
+	err = snd_timer_start(tu->timeri, tu->ticks);
+	if (err < 0)
+		return err;
+	return 0;
 }
 
 static int snd_timer_user_stop(struct file *file)
@@ -1893,7 +1923,10 @@ static int snd_timer_user_stop(struct file *file)
 	tu = file->private_data;
 	if (!tu->timeri)
 		return -EBADFD;
-	return (err = snd_timer_stop(tu->timeri)) < 0 ? err : 0;
+	err = snd_timer_stop(tu->timeri);
+	if (err < 0)
+		return err;
+	return 0;
 }
 
 static int snd_timer_user_continue(struct file *file)
@@ -1908,7 +1941,10 @@ static int snd_timer_user_continue(struct file *file)
 	if (!(tu->timeri->flags & SNDRV_TIMER_IFLG_PAUSED))
 		return snd_timer_user_start(file);
 	tu->timeri->lost = 0;
-	return (err = snd_timer_continue(tu->timeri)) < 0 ? err : 0;
+	err = snd_timer_continue(tu->timeri);
+	if (err < 0)
+		return err;
+	return 0;
 }
 
 static int snd_timer_user_pause(struct file *file)
@@ -1919,7 +1955,10 @@ static int snd_timer_user_pause(struct file *file)
 	tu = file->private_data;
 	if (!tu->timeri)
 		return -EBADFD;
-	return (err = snd_timer_pause(tu->timeri)) < 0 ? err : 0;
+	err = snd_timer_pause(tu->timeri);
+	if (err < 0)
+		return err;
+	return 0;
 }
 
 enum {
diff --git a/sound/drivers/aloop.c b/sound/drivers/aloop.c
index 8c3fbe1276be..c14e57b2a135 100644
--- a/sound/drivers/aloop.c
+++ b/sound/drivers/aloop.c
@@ -337,7 +337,7 @@ static int loopback_prepare(struct snd_pcm_substream *substream)
 
 	loopback_timer_stop_sync(dpcm);
 
-	salign = (snd_pcm_format_width(runtime->format) *
+	salign = (snd_pcm_format_physical_width(runtime->format) *
 						runtime->channels) / 8;
 	bps = salign * runtime->rate;
 	if (bps <= 0 || salign <= 0)
@@ -562,6 +562,8 @@ static const struct snd_pcm_hardware loopback_pcm_hardware =
 			 SNDRV_PCM_INFO_MMAP_VALID | SNDRV_PCM_INFO_PAUSE |
 			 SNDRV_PCM_INFO_RESUME),
 	.formats =	(SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S16_BE |
+			 SNDRV_PCM_FMTBIT_S24_LE | SNDRV_PCM_FMTBIT_S24_BE |
+			 SNDRV_PCM_FMTBIT_S24_3LE | SNDRV_PCM_FMTBIT_S24_3BE |
 			 SNDRV_PCM_FMTBIT_S32_LE | SNDRV_PCM_FMTBIT_S32_BE |
 			 SNDRV_PCM_FMTBIT_FLOAT_LE | SNDRV_PCM_FMTBIT_FLOAT_BE),
 	.rates =	SNDRV_PCM_RATE_CONTINUOUS | SNDRV_PCM_RATE_8000_192000,
diff --git a/sound/firewire/amdtp-stream.c b/sound/firewire/amdtp-stream.c
index 3ada55ed5381..43f28b813386 100644
--- a/sound/firewire/amdtp-stream.c
+++ b/sound/firewire/amdtp-stream.c
@@ -56,8 +56,9 @@
 #define INTERRUPT_INTERVAL	16
 #define QUEUE_LENGTH		48
 
-#define IN_PACKET_HEADER_SIZE	4
+#define IR_HEADER_SIZE		8	// For header and timestamp.
 #define OUT_PACKET_HEADER_SIZE	0
+#define HEADER_TSTAMP_MASK	0x0000ffff
 
 static void pcm_period_tasklet(unsigned long data);
 
@@ -456,7 +457,7 @@ static inline int queue_out_packet(struct amdtp_stream *s,
 
 static inline int queue_in_packet(struct amdtp_stream *s)
 {
-	return queue_packet(s, IN_PACKET_HEADER_SIZE, s->max_payload_length);
+	return queue_packet(s, IR_HEADER_SIZE, s->max_payload_length);
 }
 
 static int handle_out_packet(struct amdtp_stream *s,
@@ -701,13 +702,6 @@ static inline u32 increment_cycle_count(u32 cycle, unsigned int addend)
 	return cycle;
 }
 
-static inline u32 decrement_cycle_count(u32 cycle, unsigned int subtrahend)
-{
-	if (cycle < subtrahend)
-		cycle += 8 * CYCLES_PER_SECOND;
-	return cycle - subtrahend;
-}
-
 static void out_stream_callback(struct fw_iso_context *context, u32 tstamp,
 				size_t header_length, void *header,
 				void *private_data)
@@ -745,29 +739,26 @@ static void in_stream_callback(struct fw_iso_context *context, u32 tstamp,
 	struct amdtp_stream *s = private_data;
 	unsigned int i, packets;
 	unsigned int payload_length, max_payload_length;
-	__be32 *headers = header;
-	u32 cycle;
+	__be32 *ctx_header = header;
 
 	if (s->packet_index < 0)
 		return;
 
 	/* The number of packets in buffer */
-	packets = header_length / IN_PACKET_HEADER_SIZE;
-
-	cycle = compute_cycle_count(tstamp);
-
-	/* Align to actual cycle count for the last packet. */
-	cycle = decrement_cycle_count(cycle, packets);
+	packets = header_length / IR_HEADER_SIZE;
 
 	/* For buffer-over-run prevention. */
 	max_payload_length = s->max_payload_length;
 
 	for (i = 0; i < packets; i++) {
-		cycle = increment_cycle_count(cycle, 1);
+		u32 iso_header = be32_to_cpu(ctx_header[0]);
+		unsigned int cycle;
+
+		tstamp = be32_to_cpu(ctx_header[1]) & HEADER_TSTAMP_MASK;
+		cycle = compute_cycle_count(tstamp);
 
 		/* The number of bytes in this packet */
-		payload_length =
-			(be32_to_cpu(headers[i]) >> ISO_DATA_LENGTH_SHIFT);
+		payload_length = iso_header >> ISO_DATA_LENGTH_SHIFT;
 		if (payload_length > max_payload_length) {
 			dev_err(&s->unit->device,
 				"Detect jumbo payload: %04x %04x\n",
@@ -777,6 +768,8 @@ static void in_stream_callback(struct fw_iso_context *context, u32 tstamp,
 
 		if (s->handle_packet(s, payload_length, cycle, i) < 0)
 			break;
+
+		ctx_header += IR_HEADER_SIZE / sizeof(__be32);
 	}
 
 	/* Queueing error or detecting invalid payload. */
@@ -797,6 +790,7 @@ static void amdtp_stream_first_callback(struct fw_iso_context *context,
 					void *header, void *private_data)
 {
 	struct amdtp_stream *s = private_data;
+	__be32 *ctx_header = header;
 	u32 cycle;
 	unsigned int packets;
 
@@ -807,11 +801,10 @@ static void amdtp_stream_first_callback(struct fw_iso_context *context,
 	s->callbacked = true;
 	wake_up(&s->callback_wait);
 
-	cycle = compute_cycle_count(tstamp);
-
 	if (s->direction == AMDTP_IN_STREAM) {
-		packets = header_length / IN_PACKET_HEADER_SIZE;
-		cycle = decrement_cycle_count(cycle, packets);
+		tstamp = be32_to_cpu(ctx_header[1]) & HEADER_TSTAMP_MASK;
+		cycle = compute_cycle_count(tstamp);
+
 		context->callback.sc = in_stream_callback;
 		if (s->flags & CIP_NO_HEADER)
 			s->handle_packet = handle_in_packet_without_header;
@@ -819,6 +812,7 @@ static void amdtp_stream_first_callback(struct fw_iso_context *context,
 			s->handle_packet = handle_in_packet;
 	} else {
 		packets = header_length / 4;
+		cycle = compute_cycle_count(tstamp);
 		cycle = increment_cycle_count(cycle, QUEUE_LENGTH - packets);
 		context->callback.sc = out_stream_callback;
 		if (s->flags & CIP_NO_HEADER)
@@ -880,7 +874,7 @@ int amdtp_stream_start(struct amdtp_stream *s, int channel, int speed)
 	if (s->direction == AMDTP_IN_STREAM) {
 		dir = DMA_FROM_DEVICE;
 		type = FW_ISO_CONTEXT_RECEIVE;
-		header_size = IN_PACKET_HEADER_SIZE;
+		header_size = IR_HEADER_SIZE;
 	} else {
 		dir = DMA_TO_DEVICE;
 		type = FW_ISO_CONTEXT_TRANSMIT;
diff --git a/sound/firewire/motu/amdtp-motu.c b/sound/firewire/motu/amdtp-motu.c
index 6c9b743ea74b..cb0c967dea63 100644
--- a/sound/firewire/motu/amdtp-motu.c
+++ b/sound/firewire/motu/amdtp-motu.c
@@ -412,6 +412,12 @@ int amdtp_motu_init(struct amdtp_stream *s, struct fw_unit *unit,
 				 CIP_HEADER_WITHOUT_EOH;
 			fmt = CIP_FMT_MOTU_TX_V3;
 		}
+
+		if (protocol == &snd_motu_protocol_v2) {
+			// 8pre has some quirks.
+			flags |= CIP_WRONG_DBS |
+				 CIP_SKIP_DBC_ZERO_CHECK;
+		}
 	} else {
 		process_data_blocks = process_rx_data_blocks;
 		flags |= CIP_DBC_IS_END_EVENT;
diff --git a/sound/firewire/motu/motu-protocol-v2.c b/sound/firewire/motu/motu-protocol-v2.c
index 453fc29fade7..848fffe7387e 100644
--- a/sound/firewire/motu/motu-protocol-v2.c
+++ b/sound/firewire/motu/motu-protocol-v2.c
@@ -15,6 +15,8 @@
 #define  V2_CLOCK_SRC_SHIFT			0
 #define  V2_CLOCK_TRAVELER_FETCH_DISABLE	0x04000000
 #define  V2_CLOCK_TRAVELER_FETCH_ENABLE		0x03000000
+#define  V2_CLOCK_8PRE_FETCH_DISABLE		0x02000000
+#define  V2_CLOCK_8PRE_FETCH_ENABLE		0x00000000
 
 #define V2_IN_OUT_CONF_OFFSET			0x0c04
 #define  V2_OPT_OUT_IFACE_MASK			0x00000c00
@@ -132,20 +134,31 @@ static int v2_switch_fetching_mode(struct snd_motu *motu, bool enable)
 	u32 data;
 	int err = 0;
 
-	if (motu->spec == &snd_motu_spec_traveler) {
+	if (motu->spec == &snd_motu_spec_traveler ||
+	    motu->spec == &snd_motu_spec_8pre) {
 		err = snd_motu_transaction_read(motu, V2_CLOCK_STATUS_OFFSET,
 						&reg, sizeof(reg));
 		if (err < 0)
 			return err;
 		data = be32_to_cpu(reg);
 
-		data &= ~(V2_CLOCK_TRAVELER_FETCH_DISABLE |
-			  V2_CLOCK_TRAVELER_FETCH_ENABLE);
-
-		if (enable)
-			data |= V2_CLOCK_TRAVELER_FETCH_ENABLE;
-		else
-			data |= V2_CLOCK_TRAVELER_FETCH_DISABLE;
+		if (motu->spec == &snd_motu_spec_traveler) {
+			data &= ~(V2_CLOCK_TRAVELER_FETCH_DISABLE |
+				  V2_CLOCK_TRAVELER_FETCH_ENABLE);
+
+			if (enable)
+				data |= V2_CLOCK_TRAVELER_FETCH_ENABLE;
+			else
+				data |= V2_CLOCK_TRAVELER_FETCH_DISABLE;
+		} else if (motu->spec == &snd_motu_spec_8pre) {
+			data &= ~(V2_CLOCK_8PRE_FETCH_DISABLE |
+				  V2_CLOCK_8PRE_FETCH_ENABLE);
+
+			if (enable)
+				data |= V2_CLOCK_8PRE_FETCH_DISABLE;
+			else
+				data |= V2_CLOCK_8PRE_FETCH_ENABLE;
+		}
 
 		reg = cpu_to_be32(data);
 		err = snd_motu_transaction_write(motu, V2_CLOCK_STATUS_OFFSET,
@@ -220,10 +233,16 @@ static void calculate_differed_part(struct snd_motu_packet_format *formats,
 	 * interfaces.
 	 */
 	data = (data & mask) >> shift;
-	if ((flags & SND_MOTU_SPEC_HAS_OPT_IFACE_A) &&
-	    data == V2_OPT_IFACE_MODE_ADAT) {
-		pcm_chunks[0] += 8;
-		pcm_chunks[1] += 4;
+	if (data == V2_OPT_IFACE_MODE_ADAT) {
+		if (flags & SND_MOTU_SPEC_HAS_OPT_IFACE_A) {
+			pcm_chunks[0] += 8;
+			pcm_chunks[1] += 4;
+		}
+		// 8pre has two sets of optical interface and doesn't reduce
+		// chunks for ADAT signals.
+		if (flags & SND_MOTU_SPEC_HAS_OPT_IFACE_B) {
+			pcm_chunks[1] += 4;
+		}
 	}
 
 	/* At mode x4, no data chunks are supported in this part. */
diff --git a/sound/firewire/motu/motu.c b/sound/firewire/motu/motu.c
index 513291ba0ab0..201539d4488c 100644
--- a/sound/firewire/motu/motu.c
+++ b/sound/firewire/motu/motu.c
@@ -203,6 +203,20 @@ const struct snd_motu_spec snd_motu_spec_traveler = {
 	.analog_out_ports = 8,
 };
 
+const struct snd_motu_spec snd_motu_spec_8pre = {
+	.name = "8pre",
+	.protocol = &snd_motu_protocol_v2,
+	// In tx, use coax chunks for mix-return 1/2. In rx, use coax chunks for
+	// dummy 1/2.
+	.flags = SND_MOTU_SPEC_SUPPORT_CLOCK_X2 |
+		 SND_MOTU_SPEC_HAS_OPT_IFACE_A |
+		 SND_MOTU_SPEC_HAS_OPT_IFACE_B |
+		 SND_MOTU_SPEC_RX_MIDI_2ND_Q |
+		 SND_MOTU_SPEC_TX_MIDI_2ND_Q,
+	.analog_in_ports = 8,
+	.analog_out_ports = 2,
+};
+
 static const struct snd_motu_spec motu_828mk3 = {
 	.name = "828mk3",
 	.protocol = &snd_motu_protocol_v3,
@@ -248,6 +262,7 @@ static const struct snd_motu_spec motu_audio_express = {
 static const struct ieee1394_device_id motu_id_table[] = {
 	SND_MOTU_DEV_ENTRY(0x000003, &motu_828mk2),
 	SND_MOTU_DEV_ENTRY(0x000009, &snd_motu_spec_traveler),
+	SND_MOTU_DEV_ENTRY(0x00000f, &snd_motu_spec_8pre),
 	SND_MOTU_DEV_ENTRY(0x000015, &motu_828mk3),	/* FireWire only. */
 	SND_MOTU_DEV_ENTRY(0x000035, &motu_828mk3),	/* Hybrid. */
 	SND_MOTU_DEV_ENTRY(0x000033, &motu_audio_express),
diff --git a/sound/firewire/motu/motu.h b/sound/firewire/motu/motu.h
index fd5327d30ab1..1cd112be7dad 100644
--- a/sound/firewire/motu/motu.h
+++ b/sound/firewire/motu/motu.h
@@ -130,6 +130,7 @@ extern const struct snd_motu_protocol snd_motu_protocol_v2;
 extern const struct snd_motu_protocol snd_motu_protocol_v3;
 
 extern const struct snd_motu_spec snd_motu_spec_traveler;
+extern const struct snd_motu_spec snd_motu_spec_8pre;
 
 int amdtp_motu_init(struct amdtp_stream *s, struct fw_unit *unit,
 		    enum amdtp_stream_direction dir,
diff --git a/sound/hda/ext/hdac_ext_bus.c b/sound/hda/ext/hdac_ext_bus.c
index ec7715c6b0c0..c203af71a099 100644
--- a/sound/hda/ext/hdac_ext_bus.c
+++ b/sound/hda/ext/hdac_ext_bus.c
@@ -104,9 +104,7 @@ int snd_hdac_ext_bus_init(struct hdac_bus *bus, struct device *dev,
 		return ret;
 
 	bus->ext_ops = ext_ops;
-	INIT_LIST_HEAD(&bus->hlink_list);
 	bus->idx = idx++;
-
 	bus->cmd_dma_state = true;
 
 	return 0;
diff --git a/sound/hda/hdac_bus.c b/sound/hda/hdac_bus.c
index ad8eee08013f..10e5d261fde1 100644
--- a/sound/hda/hdac_bus.c
+++ b/sound/hda/hdac_bus.c
@@ -39,6 +39,7 @@ int snd_hdac_bus_init(struct hdac_bus *bus, struct device *dev,
 	spin_lock_init(&bus->reg_lock);
 	mutex_init(&bus->cmd_mutex);
 	mutex_init(&bus->lock);
+	INIT_LIST_HEAD(&bus->hlink_list);
 	bus->irq = -1;
 	return 0;
 }
diff --git a/sound/hda/hdac_component.c b/sound/hda/hdac_component.c
index 1ea51e3b942a..dfe7e755f594 100644
--- a/sound/hda/hdac_component.c
+++ b/sound/hda/hdac_component.c
@@ -81,17 +81,23 @@ void snd_hdac_display_power(struct hdac_bus *bus, unsigned int idx, bool enable)
 
 	if (bus->display_power_status) {
 		if (!bus->display_power_active) {
+			unsigned long cookie = -1;
+
 			if (acomp->ops->get_power)
-				acomp->ops->get_power(acomp->dev);
+				cookie = acomp->ops->get_power(acomp->dev);
+
 			snd_hdac_set_codec_wakeup(bus, true);
 			snd_hdac_set_codec_wakeup(bus, false);
-			bus->display_power_active = true;
+			bus->display_power_active = cookie;
 		}
 	} else {
 		if (bus->display_power_active) {
+			unsigned long cookie = bus->display_power_active;
+
 			if (acomp->ops->put_power)
-				acomp->ops->put_power(acomp->dev);
-			bus->display_power_active = false;
+				acomp->ops->put_power(acomp->dev, cookie);
+
+			bus->display_power_active = 0;
 		}
 	}
  unlock:
@@ -329,9 +335,9 @@ int snd_hdac_acomp_exit(struct hdac_bus *bus)
 		return 0;
 
 	if (WARN_ON(bus->display_power_active) && acomp->ops)
-		acomp->ops->put_power(acomp->dev);
+		acomp->ops->put_power(acomp->dev, bus->display_power_active);
 
-	bus->display_power_active = false;
+	bus->display_power_active = 0;
 	bus->display_power_status = 0;
 
 	component_master_del(dev, &hdac_component_master_ops);
diff --git a/sound/isa/gus/gus_mem.c b/sound/isa/gus/gus_mem.c
index 4ac76f46dd76..d708ae1525e4 100644
--- a/sound/isa/gus/gus_mem.c
+++ b/sound/isa/gus/gus_mem.c
@@ -306,7 +306,7 @@ static void snd_gf1_mem_info_read(struct snd_info_entry *entry,
 	used = 0;
 	for (block = alloc->first, i = 0; block; block = block->next, i++) {
 		used += block->size;
-		snd_iprintf(buffer, "Block %i at 0x%lx onboard 0x%x size %i (0x%x):\n", i, (long) block, block->ptr, block->size, block->size);
+		snd_iprintf(buffer, "Block %i onboard 0x%x size %i (0x%x):\n", i, block->ptr, block->size, block->size);
 		if (block->share ||
 		    block->share_id[0] || block->share_id[1] ||
 		    block->share_id[2] || block->share_id[3])
diff --git a/sound/last.c b/sound/last.c
index 43f222825038..4f5a624ab438 100644
--- a/sound/last.c
+++ b/sound/last.c
@@ -24,14 +24,18 @@
 
 static int __init alsa_sound_last_init(void)
 {
+	struct snd_card *card;
 	int idx, ok = 0;
 	
 	printk(KERN_INFO "ALSA device list:\n");
-	for (idx = 0; idx < SNDRV_CARDS; idx++)
-		if (snd_cards[idx] != NULL) {
-			printk(KERN_INFO "  #%i: %s\n", idx, snd_cards[idx]->longname);
+	for (idx = 0; idx < SNDRV_CARDS; idx++) {
+		card = snd_card_ref(idx);
+		if (card) {
+			printk(KERN_INFO "  #%i: %s\n", idx, card->longname);
+			snd_card_unref(card);
 			ok++;
 		}
+	}
 	if (ok == 0)
 		printk(KERN_INFO "  No soundcards found.\n");
 	return 0;
diff --git a/sound/pci/emu10k1/emu10k1_main.c b/sound/pci/emu10k1/emu10k1_main.c
index 61f85ff91cd9..0419c75bdf5a 100644
--- a/sound/pci/emu10k1/emu10k1_main.c
+++ b/sound/pci/emu10k1/emu10k1_main.c
@@ -1882,22 +1882,8 @@ int snd_emu10k1_create(struct snd_card *card,
 			c->name, pci->vendor, pci->device,
 			emu->serial);
 
-	if (!*card->id && c->id) {
-		int i, n = 0;
+	if (!*card->id && c->id)
 		strlcpy(card->id, c->id, sizeof(card->id));
-		for (;;) {
-			for (i = 0; i < snd_ecards_limit; i++) {
-				if (snd_cards[i] && !strcmp(snd_cards[i]->id, card->id))
-					break;
-			}
-			if (i >= snd_ecards_limit)
-				break;
-			n++;
-			if (n >= SNDRV_CARDS)
-				break;
-			snprintf(card->id, sizeof(card->id), "%s_%d", c->id, n);
-		}
-	}
 
 	is_audigy = emu->audigy = c->emu10k2_chip;
 
diff --git a/sound/pci/hda/hda_codec.c b/sound/pci/hda/hda_codec.c
index 701a69d856f5..b20eb7fc83eb 100644
--- a/sound/pci/hda/hda_codec.c
+++ b/sound/pci/hda/hda_codec.c
@@ -832,7 +832,13 @@ static int snd_hda_codec_dev_free(struct snd_device *device)
 	struct hda_codec *codec = device->device_data;
 
 	codec->in_freeing = 1;
-	snd_hdac_device_unregister(&codec->core);
+	/*
+	 * snd_hda_codec_device_new() is used by legacy HDA and ASoC driver.
+	 * We can't unregister ASoC device since it will be unregistered in
+	 * snd_hdac_ext_bus_device_remove().
+	 */
+	if (codec->core.type == HDA_DEV_LEGACY)
+		snd_hdac_device_unregister(&codec->core);
 	codec_display_power(codec, false);
 	put_device(hda_codec_dev(codec));
 	return 0;
diff --git a/sound/pci/hda/hda_intel.c b/sound/pci/hda/hda_intel.c
index 2ec91085fa3e..0741eae23f10 100644
--- a/sound/pci/hda/hda_intel.c
+++ b/sound/pci/hda/hda_intel.c
@@ -1788,9 +1788,6 @@ static int azx_first_init(struct azx *chip)
 			chip->msi = 0;
 	}
 
-	if (azx_acquire_irq(chip, 0) < 0)
-		return -EBUSY;
-
 	pci_set_master(pci);
 	synchronize_irq(bus->irq);
 
@@ -1904,6 +1901,9 @@ static int azx_first_init(struct azx *chip)
 		return -ENODEV;
 	}
 
+	if (azx_acquire_irq(chip, 0) < 0)
+		return -EBUSY;
+
 	strcpy(card->driver, "HDA-Intel");
 	strlcpy(card->shortname, driver_short_names[chip->driver_type],
 		sizeof(card->shortname));
@@ -2378,6 +2378,12 @@ static const struct pci_device_id azx_ids[] = {
 	/* Cannonlake */
 	{ PCI_DEVICE(0x8086, 0x9dc8),
 	  .driver_data = AZX_DRIVER_SKL | AZX_DCAPS_INTEL_SKYLAKE},
+	/* CometLake-LP */
+	{ PCI_DEVICE(0x8086, 0x02C8),
+	  .driver_data = AZX_DRIVER_SKL | AZX_DCAPS_INTEL_SKYLAKE},
+	/* CometLake-H */
+	{ PCI_DEVICE(0x8086, 0x06C8),
+	  .driver_data = AZX_DRIVER_SKL | AZX_DCAPS_INTEL_SKYLAKE},
 	/* Icelake */
 	{ PCI_DEVICE(0x8086, 0x34c8),
 	  .driver_data = AZX_DRIVER_SKL | AZX_DCAPS_INTEL_SKYLAKE},
diff --git a/sound/pci/hda/patch_hdmi.c b/sound/pci/hda/patch_hdmi.c
index 8b3ac690efa3..0c61c05503f5 100644
--- a/sound/pci/hda/patch_hdmi.c
+++ b/sound/pci/hda/patch_hdmi.c
@@ -1551,9 +1551,11 @@ static bool hdmi_present_sense_via_verbs(struct hdmi_spec_per_pin *per_pin,
 	ret = !repoll || !eld->monitor_present || eld->eld_valid;
 
 	jack = snd_hda_jack_tbl_get(codec, pin_nid);
-	if (jack)
+	if (jack) {
 		jack->block_report = !ret;
-
+		jack->pin_sense = (eld->monitor_present && eld->eld_valid) ?
+			AC_PINSENSE_PRESENCE : 0;
+	}
 	mutex_unlock(&per_pin->lock);
 	return ret;
 }
@@ -1663,6 +1665,11 @@ static void hdmi_repoll_eld(struct work_struct *work)
 	container_of(to_delayed_work(work), struct hdmi_spec_per_pin, work);
 	struct hda_codec *codec = per_pin->codec;
 	struct hdmi_spec *spec = codec->spec;
+	struct hda_jack_tbl *jack;
+
+	jack = snd_hda_jack_tbl_get(codec, per_pin->pin_nid);
+	if (jack)
+		jack->jack_dirty = 1;
 
 	if (per_pin->repoll_count++ > 6)
 		per_pin->repoll_count = 0;
diff --git a/sound/pci/hda/patch_realtek.c b/sound/pci/hda/patch_realtek.c
index 42cd3945e0de..c53ca589c930 100644
--- a/sound/pci/hda/patch_realtek.c
+++ b/sound/pci/hda/patch_realtek.c
@@ -119,6 +119,7 @@ struct alc_spec {
 	unsigned int no_depop_delay:1;
 	unsigned int done_hp_init:1;
 	unsigned int no_shutup_pins:1;
+	unsigned int ultra_low_power:1;
 
 	/* for PLL fix */
 	hda_nid_t pll_nid;
@@ -803,11 +804,10 @@ static int alc_init(struct hda_codec *codec)
 	if (spec->init_hook)
 		spec->init_hook(codec);
 
+	snd_hda_gen_init(codec);
 	alc_fix_pll(codec);
 	alc_auto_init_amp(codec, spec->init_amp);
 
-	snd_hda_gen_init(codec);
-
 	snd_hda_apply_fixup(codec, HDA_FIXUP_ACT_INIT);
 
 	return 0;
@@ -3197,7 +3197,7 @@ static void alc256_init(struct hda_codec *codec)
 	bool hp_pin_sense;
 
 	if (!hp_pin)
-		return;
+		hp_pin = 0x21;
 
 	msleep(30);
 
@@ -3207,17 +3207,25 @@ static void alc256_init(struct hda_codec *codec)
 		msleep(2);
 
 	alc_update_coefex_idx(codec, 0x57, 0x04, 0x0007, 0x1); /* Low power */
+	if (spec->ultra_low_power) {
+		alc_update_coef_idx(codec, 0x03, 1<<1, 1<<1);
+		alc_update_coef_idx(codec, 0x08, 3<<2, 3<<2);
+		alc_update_coef_idx(codec, 0x08, 7<<4, 0);
+		alc_update_coef_idx(codec, 0x3b, 1<<15, 0);
+		alc_update_coef_idx(codec, 0x0e, 7<<6, 7<<6);
+		msleep(30);
+	}
 
 	snd_hda_codec_write(codec, hp_pin, 0,
 			    AC_VERB_SET_AMP_GAIN_MUTE, AMP_OUT_MUTE);
 
-	if (hp_pin_sense)
+	if (hp_pin_sense || spec->ultra_low_power)
 		msleep(85);
 
 	snd_hda_codec_write(codec, hp_pin, 0,
 			    AC_VERB_SET_PIN_WIDGET_CONTROL, PIN_OUT);
 
-	if (hp_pin_sense)
+	if (hp_pin_sense || spec->ultra_low_power)
 		msleep(100);
 
 	alc_update_coef_idx(codec, 0x46, 3 << 12, 0);
@@ -3232,10 +3240,8 @@ static void alc256_shutup(struct hda_codec *codec)
 	hda_nid_t hp_pin = alc_get_hp_pin(spec);
 	bool hp_pin_sense;
 
-	if (!hp_pin) {
-		alc269_shutup(codec);
-		return;
-	}
+	if (!hp_pin)
+		hp_pin = 0x21;
 
 	hp_pin_sense = snd_hda_jack_detect(codec, hp_pin);
 
@@ -3245,7 +3251,7 @@ static void alc256_shutup(struct hda_codec *codec)
 	snd_hda_codec_write(codec, hp_pin, 0,
 			    AC_VERB_SET_AMP_GAIN_MUTE, AMP_OUT_MUTE);
 
-	if (hp_pin_sense)
+	if (hp_pin_sense || spec->ultra_low_power)
 		msleep(85);
 
 	/* 3k pull low control for Headset jack. */
@@ -3256,11 +3262,20 @@ static void alc256_shutup(struct hda_codec *codec)
 		snd_hda_codec_write(codec, hp_pin, 0,
 				    AC_VERB_SET_PIN_WIDGET_CONTROL, 0x0);
 
-	if (hp_pin_sense)
+	if (hp_pin_sense || spec->ultra_low_power)
 		msleep(100);
 
 	alc_auto_setup_eapd(codec, false);
 	alc_shutup_pins(codec);
+	if (spec->ultra_low_power) {
+		msleep(50);
+		alc_update_coef_idx(codec, 0x03, 1<<1, 0);
+		alc_update_coef_idx(codec, 0x08, 7<<4, 7<<4);
+		alc_update_coef_idx(codec, 0x08, 3<<2, 0);
+		alc_update_coef_idx(codec, 0x3b, 1<<15, 1<<15);
+		alc_update_coef_idx(codec, 0x0e, 7<<6, 0);
+		msleep(30);
+	}
 }
 
 static void alc225_init(struct hda_codec *codec)
@@ -3270,8 +3285,7 @@ static void alc225_init(struct hda_codec *codec)
 	bool hp1_pin_sense, hp2_pin_sense;
 
 	if (!hp_pin)
-		return;
-
+		hp_pin = 0x21;
 	msleep(30);
 
 	hp1_pin_sense = snd_hda_jack_detect(codec, hp_pin);
@@ -3281,25 +3295,31 @@ static void alc225_init(struct hda_codec *codec)
 		msleep(2);
 
 	alc_update_coefex_idx(codec, 0x57, 0x04, 0x0007, 0x1); /* Low power */
+	if (spec->ultra_low_power) {
+		alc_update_coef_idx(codec, 0x08, 0x0f << 2, 3<<2);
+		alc_update_coef_idx(codec, 0x0e, 7<<6, 7<<6);
+		alc_update_coef_idx(codec, 0x33, 1<<11, 0);
+		msleep(30);
+	}
 
-	if (hp1_pin_sense)
+	if (hp1_pin_sense || spec->ultra_low_power)
 		snd_hda_codec_write(codec, hp_pin, 0,
 			    AC_VERB_SET_AMP_GAIN_MUTE, AMP_OUT_MUTE);
 	if (hp2_pin_sense)
 		snd_hda_codec_write(codec, 0x16, 0,
 			    AC_VERB_SET_AMP_GAIN_MUTE, AMP_OUT_MUTE);
 
-	if (hp1_pin_sense || hp2_pin_sense)
+	if (hp1_pin_sense || hp2_pin_sense || spec->ultra_low_power)
 		msleep(85);
 
-	if (hp1_pin_sense)
+	if (hp1_pin_sense || spec->ultra_low_power)
 		snd_hda_codec_write(codec, hp_pin, 0,
 			    AC_VERB_SET_PIN_WIDGET_CONTROL, PIN_OUT);
 	if (hp2_pin_sense)
 		snd_hda_codec_write(codec, 0x16, 0,
 			    AC_VERB_SET_PIN_WIDGET_CONTROL, PIN_OUT);
 
-	if (hp1_pin_sense || hp2_pin_sense)
+	if (hp1_pin_sense || hp2_pin_sense || spec->ultra_low_power)
 		msleep(100);
 
 	alc_update_coef_idx(codec, 0x4a, 3 << 10, 0);
@@ -3312,11 +3332,8 @@ static void alc225_shutup(struct hda_codec *codec)
 	hda_nid_t hp_pin = alc_get_hp_pin(spec);
 	bool hp1_pin_sense, hp2_pin_sense;
 
-	if (!hp_pin) {
-		alc269_shutup(codec);
-		return;
-	}
-
+	if (!hp_pin)
+		hp_pin = 0x21;
 	/* 3k pull low control for Headset jack. */
 	alc_update_coef_idx(codec, 0x4a, 0, 3 << 10);
 
@@ -3326,28 +3343,36 @@ static void alc225_shutup(struct hda_codec *codec)
 	if (hp1_pin_sense || hp2_pin_sense)
 		msleep(2);
 
-	if (hp1_pin_sense)
+	if (hp1_pin_sense || spec->ultra_low_power)
 		snd_hda_codec_write(codec, hp_pin, 0,
 			    AC_VERB_SET_AMP_GAIN_MUTE, AMP_OUT_MUTE);
 	if (hp2_pin_sense)
 		snd_hda_codec_write(codec, 0x16, 0,
 			    AC_VERB_SET_AMP_GAIN_MUTE, AMP_OUT_MUTE);
 
-	if (hp1_pin_sense || hp2_pin_sense)
+	if (hp1_pin_sense || hp2_pin_sense || spec->ultra_low_power)
 		msleep(85);
 
-	if (hp1_pin_sense)
+	if (hp1_pin_sense || spec->ultra_low_power)
 		snd_hda_codec_write(codec, hp_pin, 0,
 			    AC_VERB_SET_PIN_WIDGET_CONTROL, 0x0);
 	if (hp2_pin_sense)
 		snd_hda_codec_write(codec, 0x16, 0,
 			    AC_VERB_SET_PIN_WIDGET_CONTROL, 0x0);
 
-	if (hp1_pin_sense || hp2_pin_sense)
+	if (hp1_pin_sense || hp2_pin_sense || spec->ultra_low_power)
 		msleep(100);
 
 	alc_auto_setup_eapd(codec, false);
 	alc_shutup_pins(codec);
+	if (spec->ultra_low_power) {
+		msleep(50);
+		alc_update_coef_idx(codec, 0x08, 0x0f << 2, 0x0c << 2);
+		alc_update_coef_idx(codec, 0x0e, 7<<6, 0);
+		alc_update_coef_idx(codec, 0x33, 1<<11, 1<<11);
+		alc_update_coef_idx(codec, 0x4a, 3<<4, 2<<4);
+		msleep(30);
+	}
 }
 
 static void alc_default_init(struct hda_codec *codec)
@@ -5527,7 +5552,12 @@ static void alc_fixup_headset_jack(struct hda_codec *codec,
 static void alc295_fixup_chromebook(struct hda_codec *codec,
 				    const struct hda_fixup *fix, int action)
 {
+	struct alc_spec *spec = codec->spec;
+
 	switch (action) {
+	case HDA_FIXUP_ACT_PRE_PROBE:
+		spec->ultra_low_power = true;
+		break;
 	case HDA_FIXUP_ACT_INIT:
 		switch (codec->core.vendor_id) {
 		case 0x10ec0295:
@@ -6933,6 +6963,8 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x1462, 0xb120, "MSI Cubi MS-B120", ALC283_FIXUP_HEADSET_MIC),
 	SND_PCI_QUIRK(0x1462, 0xb171, "Cubi N 8GL (MS-B171)", ALC283_FIXUP_HEADSET_MIC),
 	SND_PCI_QUIRK(0x1558, 0x1325, "System76 Darter Pro (darp5)", ALC293_FIXUP_SYSTEM76_MIC_NO_PRESENCE),
+	SND_PCI_QUIRK(0x1558, 0x8550, "System76 Gazelle (gaze14)", ALC293_FIXUP_SYSTEM76_MIC_NO_PRESENCE),
+	SND_PCI_QUIRK(0x1558, 0x8560, "System76 Gazelle (gaze14)", ALC293_FIXUP_SYSTEM76_MIC_NO_PRESENCE),
 	SND_PCI_QUIRK(0x17aa, 0x1036, "Lenovo P520", ALC233_FIXUP_LENOVO_MULTI_CODECS),
 	SND_PCI_QUIRK(0x17aa, 0x20f2, "Thinkpad SL410/510", ALC269_FIXUP_SKU_IGNORE),
 	SND_PCI_QUIRK(0x17aa, 0x215e, "Thinkpad L512", ALC269_FIXUP_SKU_IGNORE),
diff --git a/sound/ppc/snd_ps3.c b/sound/ppc/snd_ps3.c
index 521236efcc4d..f77a0d5c0385 100644
--- a/sound/ppc/snd_ps3.c
+++ b/sound/ppc/snd_ps3.c
@@ -233,7 +233,6 @@ static int snd_ps3_program_dma(struct snd_ps3_card_info *card,
 	int fill_stages, dma_ch, stage;
 	enum snd_ps3_ch ch;
 	uint32_t ch0_kick_event = 0; /* initialize to mute gcc */
-	void *start_vaddr;
 	unsigned long irqsave;
 	int silent = 0;
 
@@ -257,7 +256,6 @@ static int snd_ps3_program_dma(struct snd_ps3_card_info *card,
 	fill_stages = 4;
 	spin_lock_irqsave(&card->dma_lock, irqsave);
 	for (ch = 0; ch < 2; ch++) {
-		start_vaddr = card->dma_next_transfer_vaddr[0];
 		for (stage = 0; stage < fill_stages; stage++) {
 			dma_ch = stage * 2 + ch;
 			if (silent)
@@ -526,9 +524,7 @@ static int snd_ps3_pcm_open(struct snd_pcm_substream *substream)
 {
 	struct snd_pcm_runtime *runtime = substream->runtime;
 	struct snd_ps3_card_info *card = snd_pcm_substream_chip(substream);
-	int pcm_index;
 
-	pcm_index = substream->pcm->device;
 	/* to retrieve substream/runtime in interrupt handler */
 	card->substream = substream;
 
diff --git a/sound/sh/aica.c b/sound/sh/aica.c
index e7fef3fce44a..a24e486d9d83 100644
--- a/sound/sh/aica.c
+++ b/sound/sh/aica.c
@@ -303,7 +303,7 @@ static void aica_period_elapsed(struct timer_list *t)
 {
 	struct snd_card_aica *dreamcastcard = from_timer(dreamcastcard,
 							      t, timer);
-	struct snd_pcm_substream *substream = dreamcastcard->timer_substream;
+	struct snd_pcm_substream *substream = dreamcastcard->substream;
 	/*timer function - so cannot sleep */
 	int play_period;
 	struct snd_pcm_runtime *runtime;
@@ -335,13 +335,6 @@ static void spu_begin_dma(struct snd_pcm_substream *substream)
 	dreamcastcard = substream->pcm->private_data;
 	/*get the queue to do the work */
 	schedule_work(&(dreamcastcard->spu_dma_work));
-	/* Timer may already be running */
-	if (unlikely(dreamcastcard->timer_substream)) {
-		mod_timer(&dreamcastcard->timer, jiffies + 4);
-		return;
-	}
-	timer_setup(&dreamcastcard->timer, aica_period_elapsed, 0);
-	dreamcastcard->timer_substream = substream;
 	mod_timer(&dreamcastcard->timer, jiffies + 4);
 }
 
@@ -379,8 +372,8 @@ static int snd_aicapcm_pcm_close(struct snd_pcm_substream
 {
 	struct snd_card_aica *dreamcastcard = substream->pcm->private_data;
 	flush_work(&(dreamcastcard->spu_dma_work));
-	if (dreamcastcard->timer_substream)
-		del_timer(&dreamcastcard->timer);
+	del_timer(&dreamcastcard->timer);
+	dreamcastcard->substream = NULL;
 	kfree(dreamcastcard->channel);
 	spu_disable();
 	return 0;
@@ -613,6 +606,7 @@ static int snd_aica_probe(struct platform_device *devptr)
 	       "Yamaha AICA Super Intelligent Sound Processor for SEGA Dreamcast");
 	/* Prepare to use the queue */
 	INIT_WORK(&(dreamcastcard->spu_dma_work), run_spu_dma);
+	timer_setup(&dreamcastcard->timer, aica_period_elapsed, 0);
 	/* Load the PCM 'chip' */
 	err = snd_aicapcmchip(dreamcastcard, 0);
 	if (unlikely(err < 0))
diff --git a/sound/soc/Kconfig b/sound/soc/Kconfig
index aa35940f5c50..297be0ca3dbc 100644
--- a/sound/soc/Kconfig
+++ b/sound/soc/Kconfig
@@ -63,6 +63,7 @@ source "sound/soc/rockchip/Kconfig"
 source "sound/soc/samsung/Kconfig"
 source "sound/soc/sh/Kconfig"
 source "sound/soc/sirf/Kconfig"
+source "sound/soc/sof/Kconfig"
 source "sound/soc/spear/Kconfig"
 source "sound/soc/sprd/Kconfig"
 source "sound/soc/sti/Kconfig"
diff --git a/sound/soc/Makefile b/sound/soc/Makefile
index 974fb9821e17..d90ce8a32887 100644
--- a/sound/soc/Makefile
+++ b/sound/soc/Makefile
@@ -47,6 +47,7 @@ obj-$(CONFIG_SND_SOC)	+= rockchip/
 obj-$(CONFIG_SND_SOC)	+= samsung/
 obj-$(CONFIG_SND_SOC)	+= sh/
 obj-$(CONFIG_SND_SOC)	+= sirf/
+obj-$(CONFIG_SND_SOC)	+= sof/
 obj-$(CONFIG_SND_SOC)	+= spear/
 obj-$(CONFIG_SND_SOC)	+= sprd/
 obj-$(CONFIG_SND_SOC)	+= sti/
diff --git a/sound/soc/adi/axi-i2s.c b/sound/soc/adi/axi-i2s.c
index 4c23381727a1..273c543e8ff3 100644
--- a/sound/soc/adi/axi-i2s.c
+++ b/sound/soc/adi/axi-i2s.c
@@ -43,6 +43,9 @@ struct axi_i2s {
 	struct clk *clk;
 	struct clk *clk_ref;
 
+	bool   has_capture;
+	bool   has_playback;
+
 	struct snd_soc_dai_driver dai_driver;
 
 	struct snd_dmaengine_dai_dma_data capture_dma_data;
@@ -136,8 +139,10 @@ static int axi_i2s_dai_probe(struct snd_soc_dai *dai)
 {
 	struct axi_i2s *i2s = snd_soc_dai_get_drvdata(dai);
 
-	snd_soc_dai_init_dma_data(dai, &i2s->playback_dma_data,
-		&i2s->capture_dma_data);
+	snd_soc_dai_init_dma_data(
+		dai,
+		i2s->has_playback ? &i2s->playback_dma_data : NULL,
+		i2s->has_capture  ? &i2s->capture_dma_data  : NULL);
 
 	return 0;
 }
@@ -151,18 +156,6 @@ static const struct snd_soc_dai_ops axi_i2s_dai_ops = {
 
 static struct snd_soc_dai_driver axi_i2s_dai = {
 	.probe = axi_i2s_dai_probe,
-	.playback = {
-		.channels_min = 2,
-		.channels_max = 2,
-		.rates = SNDRV_PCM_RATE_KNOT,
-		.formats = SNDRV_PCM_FMTBIT_S32_LE | SNDRV_PCM_FMTBIT_U32_LE,
-	},
-	.capture = {
-		.channels_min = 2,
-		.channels_max = 2,
-		.rates = SNDRV_PCM_RATE_KNOT,
-		.formats = SNDRV_PCM_FMTBIT_S32_LE | SNDRV_PCM_FMTBIT_U32_LE,
-	},
 	.ops = &axi_i2s_dai_ops,
 	.symmetric_rates = 1,
 };
@@ -178,6 +171,19 @@ static const struct regmap_config axi_i2s_regmap_config = {
 	.max_register = AXI_I2S_REG_STATUS,
 };
 
+static void axi_i2s_parse_of(struct axi_i2s *i2s, const struct device_node *np)
+{
+	struct property *dma_names;
+	const char *dma_name;
+
+	of_property_for_each_string(np, "dma-names", dma_names, dma_name) {
+		if (strcmp(dma_name, "rx") == 0)
+			i2s->has_capture = true;
+		if (strcmp(dma_name, "tx") == 0)
+			i2s->has_playback = true;
+	}
+}
+
 static int axi_i2s_probe(struct platform_device *pdev)
 {
 	struct resource *res;
@@ -191,6 +197,8 @@ static int axi_i2s_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, i2s);
 
+	axi_i2s_parse_of(i2s, pdev->dev.of_node);
+
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	base = devm_ioremap_resource(&pdev->dev, res);
 	if (IS_ERR(base))
@@ -213,13 +221,29 @@ static int axi_i2s_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	i2s->playback_dma_data.addr = res->start + AXI_I2S_REG_TX_FIFO;
-	i2s->playback_dma_data.addr_width = 4;
-	i2s->playback_dma_data.maxburst = 1;
+	if (i2s->has_playback) {
+		axi_i2s_dai.playback.channels_min = 2;
+		axi_i2s_dai.playback.channels_max = 2;
+		axi_i2s_dai.playback.rates = SNDRV_PCM_RATE_KNOT;
+		axi_i2s_dai.playback.formats =
+			SNDRV_PCM_FMTBIT_S32_LE | SNDRV_PCM_FMTBIT_U32_LE;
+
+		i2s->playback_dma_data.addr = res->start + AXI_I2S_REG_TX_FIFO;
+		i2s->playback_dma_data.addr_width = 4;
+		i2s->playback_dma_data.maxburst = 1;
+	}
+
+	if (i2s->has_capture) {
+		axi_i2s_dai.capture.channels_min = 2;
+		axi_i2s_dai.capture.channels_max = 2;
+		axi_i2s_dai.capture.rates = SNDRV_PCM_RATE_KNOT;
+		axi_i2s_dai.capture.formats =
+			SNDRV_PCM_FMTBIT_S32_LE | SNDRV_PCM_FMTBIT_U32_LE;
 
-	i2s->capture_dma_data.addr = res->start + AXI_I2S_REG_RX_FIFO;
-	i2s->capture_dma_data.addr_width = 4;
-	i2s->capture_dma_data.maxburst = 1;
+		i2s->capture_dma_data.addr = res->start + AXI_I2S_REG_RX_FIFO;
+		i2s->capture_dma_data.addr_width = 4;
+		i2s->capture_dma_data.maxburst = 1;
+	}
 
 	i2s->ratnum.num = clk_get_rate(i2s->clk_ref) / 2 / AXI_I2S_BITS_PER_FRAME;
 	i2s->ratnum.den_step = 1;
@@ -240,6 +264,10 @@ static int axi_i2s_probe(struct platform_device *pdev)
 	if (ret)
 		goto err_clk_disable;
 
+	dev_info(&pdev->dev, "probed, capture %s, playback %s\n",
+		 i2s->has_capture ? "enabled" : "disabled",
+		 i2s->has_playback ? "enabled" : "disabled");
+
 	return 0;
 
 err_clk_disable:
diff --git a/sound/soc/amd/acp-da7219-max98357a.c b/sound/soc/amd/acp-da7219-max98357a.c
index a5daad973ce5..16b0ea3a3d72 100644
--- a/sound/soc/amd/acp-da7219-max98357a.c
+++ b/sound/soc/amd/acp-da7219-max98357a.c
@@ -46,8 +46,9 @@
 #define DUAL_CHANNEL		2
 
 static struct snd_soc_jack cz_jack;
-static struct clk *da7219_dai_clk;
-extern int bt_uart_enable;
+static struct clk *da7219_dai_wclk;
+static struct clk *da7219_dai_bclk;
+extern bool bt_uart_enable;
 
 static int cz_da7219_init(struct snd_soc_pcm_runtime *rtd)
 {
@@ -72,7 +73,8 @@ static int cz_da7219_init(struct snd_soc_pcm_runtime *rtd)
 		return ret;
 	}
 
-	da7219_dai_clk = clk_get(component->dev, "da7219-dai-clks");
+	da7219_dai_wclk = clk_get(component->dev, "da7219-dai-wclk");
+	da7219_dai_bclk = clk_get(component->dev, "da7219-dai-bclk");
 
 	ret = snd_soc_card_jack_new(card, "Headset Jack",
 				SND_JACK_HEADSET | SND_JACK_LINEOUT |
@@ -94,12 +96,15 @@ static int cz_da7219_init(struct snd_soc_pcm_runtime *rtd)
 	return 0;
 }
 
-static int da7219_clk_enable(struct snd_pcm_substream *substream)
+static int da7219_clk_enable(struct snd_pcm_substream *substream,
+			     int wclk_rate, int bclk_rate)
 {
 	int ret = 0;
 	struct snd_soc_pcm_runtime *rtd = substream->private_data;
 
-	ret = clk_prepare_enable(da7219_dai_clk);
+	clk_set_rate(da7219_dai_wclk, wclk_rate);
+	clk_set_rate(da7219_dai_bclk, bclk_rate);
+	ret = clk_prepare_enable(da7219_dai_bclk);
 	if (ret < 0) {
 		dev_err(rtd->dev, "can't enable master clock %d\n", ret);
 		return ret;
@@ -110,7 +115,7 @@ static int da7219_clk_enable(struct snd_pcm_substream *substream)
 
 static void da7219_clk_disable(void)
 {
-	clk_disable_unprepare(da7219_dai_clk);
+	clk_disable_unprepare(da7219_dai_bclk);
 }
 
 static const unsigned int channels[] = {
@@ -151,7 +156,7 @@ static int cz_da7219_play_startup(struct snd_pcm_substream *substream)
 				   &constraints_rates);
 
 	machine->play_i2s_instance = I2S_SP_INSTANCE;
-	return da7219_clk_enable(substream);
+	return 0;
 }
 
 static int cz_da7219_cap_startup(struct snd_pcm_substream *substream)
@@ -173,12 +178,7 @@ static int cz_da7219_cap_startup(struct snd_pcm_substream *substream)
 
 	machine->cap_i2s_instance = I2S_SP_INSTANCE;
 	machine->capture_channel = CAP_CHANNEL1;
-	return da7219_clk_enable(substream);
-}
-
-static void cz_da7219_shutdown(struct snd_pcm_substream *substream)
-{
-	da7219_clk_disable();
+	return 0;
 }
 
 static int cz_max_startup(struct snd_pcm_substream *substream)
@@ -199,12 +199,7 @@ static int cz_max_startup(struct snd_pcm_substream *substream)
 				   &constraints_rates);
 
 	machine->play_i2s_instance = I2S_BT_INSTANCE;
-	return da7219_clk_enable(substream);
-}
-
-static void cz_max_shutdown(struct snd_pcm_substream *substream)
-{
-	da7219_clk_disable();
+	return 0;
 }
 
 static int cz_dmic0_startup(struct snd_pcm_substream *substream)
@@ -225,7 +220,7 @@ static int cz_dmic0_startup(struct snd_pcm_substream *substream)
 				   &constraints_rates);
 
 	machine->cap_i2s_instance = I2S_BT_INSTANCE;
-	return da7219_clk_enable(substream);
+	return 0;
 }
 
 static int cz_dmic1_startup(struct snd_pcm_substream *substream)
@@ -247,10 +242,28 @@ static int cz_dmic1_startup(struct snd_pcm_substream *substream)
 
 	machine->cap_i2s_instance = I2S_SP_INSTANCE;
 	machine->capture_channel = CAP_CHANNEL0;
-	return da7219_clk_enable(substream);
+	return 0;
 }
 
-static void cz_dmic_shutdown(struct snd_pcm_substream *substream)
+static int cz_da7219_params(struct snd_pcm_substream *substream,
+				      struct snd_pcm_hw_params *params)
+{
+	int wclk, bclk;
+
+	wclk = params_rate(params);
+	bclk = wclk * params_channels(params) *
+		snd_pcm_format_width(params_format(params));
+	/* ADAU7002 spec: "The ADAU7002 requires a BCLK rate
+	 * that is minimum of 64x the LRCLK sample rate."
+	 * DA7219 is the only clk source so for all codecs
+	 * we have to limit bclk to 64X lrclk.
+	 */
+	if (bclk < (wclk * 64))
+		bclk = wclk * 64;
+	return da7219_clk_enable(substream, wclk, bclk);
+}
+
+static void cz_da7219_shutdown(struct snd_pcm_substream *substream)
 {
 	da7219_clk_disable();
 }
@@ -258,26 +271,31 @@ static void cz_dmic_shutdown(struct snd_pcm_substream *substream)
 static const struct snd_soc_ops cz_da7219_play_ops = {
 	.startup = cz_da7219_play_startup,
 	.shutdown = cz_da7219_shutdown,
+	.hw_params = cz_da7219_params,
 };
 
 static const struct snd_soc_ops cz_da7219_cap_ops = {
 	.startup = cz_da7219_cap_startup,
 	.shutdown = cz_da7219_shutdown,
+	.hw_params = cz_da7219_params,
 };
 
 static const struct snd_soc_ops cz_max_play_ops = {
 	.startup = cz_max_startup,
-	.shutdown = cz_max_shutdown,
+	.shutdown = cz_da7219_shutdown,
+	.hw_params = cz_da7219_params,
 };
 
 static const struct snd_soc_ops cz_dmic0_cap_ops = {
 	.startup = cz_dmic0_startup,
-	.shutdown = cz_dmic_shutdown,
+	.shutdown = cz_da7219_shutdown,
+	.hw_params = cz_da7219_params,
 };
 
 static const struct snd_soc_ops cz_dmic1_cap_ops = {
 	.startup = cz_dmic1_startup,
-	.shutdown = cz_dmic_shutdown,
+	.shutdown = cz_da7219_shutdown,
+	.hw_params = cz_da7219_params,
 };
 
 static struct snd_soc_dai_link cz_dai_7219_98357[] = {
diff --git a/sound/soc/amd/raven/acp3x-pcm-dma.c b/sound/soc/amd/raven/acp3x-pcm-dma.c
index 1a2e15ff1456..9775bda2a4ca 100644
--- a/sound/soc/amd/raven/acp3x-pcm-dma.c
+++ b/sound/soc/amd/raven/acp3x-pcm-dma.c
@@ -558,7 +558,7 @@ static int acp3x_dai_i2s_trigger(struct snd_pcm_substream *substream,
 	return ret;
 }
 
-struct snd_soc_dai_ops acp3x_dai_i2s_ops = {
+static struct snd_soc_dai_ops acp3x_dai_i2s_ops = {
 	.hw_params = acp3x_dai_i2s_hwparams,
 	.trigger   = acp3x_dai_i2s_trigger,
 	.set_fmt = acp3x_dai_i2s_set_fmt,
diff --git a/sound/soc/atmel/Kconfig b/sound/soc/atmel/Kconfig
index 64f86f0b87e5..c473b9e463ab 100644
--- a/sound/soc/atmel/Kconfig
+++ b/sound/soc/atmel/Kconfig
@@ -109,4 +109,18 @@ config SND_SOC_MIKROE_PROTO
 	  using I2C over SDA (MPU Data Input) and SCL (MPU Clock Input) pins.
 	  Both playback and capture are supported.
 
+config SND_MCHP_SOC_I2S_MCC
+	tristate "Microchip ASoC driver for boards using I2S MCC"
+	depends on OF && (ARCH_AT91 || COMPILE_TEST)
+	select SND_SOC_GENERIC_DMAENGINE_PCM
+	select REGMAP_MMIO
+	help
+	  Say Y or M if you want to add support for I2S Multi-Channel ASoC
+	  driver on the following Microchip platforms:
+	  - sam9x60
+
+	  The I2SMCC complies with the Inter-IC Sound (I2S) bus specification
+	  and supports a Time Division Multiplexed (TDM) interface with
+	  external multi-channel audio codecs.
+
 endif
diff --git a/sound/soc/atmel/Makefile b/sound/soc/atmel/Makefile
index 9f41bfa0fea3..1f6890ed3738 100644
--- a/sound/soc/atmel/Makefile
+++ b/sound/soc/atmel/Makefile
@@ -4,11 +4,13 @@ snd-soc-atmel-pcm-pdc-objs := atmel-pcm-pdc.o
 snd-soc-atmel-pcm-dma-objs := atmel-pcm-dma.o
 snd-soc-atmel_ssc_dai-objs := atmel_ssc_dai.o
 snd-soc-atmel-i2s-objs := atmel-i2s.o
+snd-soc-mchp-i2s-mcc-objs := mchp-i2s-mcc.o
 
 obj-$(CONFIG_SND_ATMEL_SOC_PDC) += snd-soc-atmel-pcm-pdc.o
 obj-$(CONFIG_SND_ATMEL_SOC_DMA) += snd-soc-atmel-pcm-dma.o
 obj-$(CONFIG_SND_ATMEL_SOC_SSC) += snd-soc-atmel_ssc_dai.o
 obj-$(CONFIG_SND_ATMEL_SOC_I2S) += snd-soc-atmel-i2s.o
+obj-$(CONFIG_SND_MCHP_SOC_I2S_MCC) += snd-soc-mchp-i2s-mcc.o
 
 # AT91 Machine Support
 snd-soc-sam9g20-wm8731-objs := sam9g20_wm8731.o
diff --git a/sound/soc/atmel/mchp-i2s-mcc.c b/sound/soc/atmel/mchp-i2s-mcc.c
new file mode 100644
index 000000000000..86495883ca3f
--- /dev/null
+++ b/sound/soc/atmel/mchp-i2s-mcc.c
@@ -0,0 +1,974 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Driver for Microchip I2S Multi-channel controller
+//
+// Copyright (C) 2018 Microchip Technology Inc. and its subsidiaries
+//
+// Author: Codrin Ciubotariu <codrin.ciubotariu@microchip.com>
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/slab.h>
+
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/clk.h>
+#include <linux/mfd/syscon.h>
+#include <linux/lcm.h>
+
+#include <sound/core.h>
+#include <sound/pcm.h>
+#include <sound/pcm_params.h>
+#include <sound/initval.h>
+#include <sound/soc.h>
+#include <sound/dmaengine_pcm.h>
+
+/*
+ * ---- I2S Controller Register map ----
+ */
+#define MCHP_I2SMCC_CR		0x0000	/* Control Register */
+#define MCHP_I2SMCC_MRA		0x0004	/* Mode Register A */
+#define MCHP_I2SMCC_MRB		0x0008	/* Mode Register B */
+#define MCHP_I2SMCC_SR		0x000C	/* Status Register */
+#define MCHP_I2SMCC_IERA	0x0010	/* Interrupt Enable Register A */
+#define MCHP_I2SMCC_IDRA	0x0014	/* Interrupt Disable Register A */
+#define MCHP_I2SMCC_IMRA	0x0018	/* Interrupt Mask Register A */
+#define MCHP_I2SMCC_ISRA	0X001C	/* Interrupt Status Register A */
+
+#define MCHP_I2SMCC_IERB	0x0020	/* Interrupt Enable Register B */
+#define MCHP_I2SMCC_IDRB	0x0024	/* Interrupt Disable Register B */
+#define MCHP_I2SMCC_IMRB	0x0028	/* Interrupt Mask Register B */
+#define MCHP_I2SMCC_ISRB	0X002C	/* Interrupt Status Register B */
+
+#define MCHP_I2SMCC_RHR		0x0030	/* Receiver Holding Register */
+#define MCHP_I2SMCC_THR		0x0034	/* Transmitter Holding Register */
+
+#define MCHP_I2SMCC_RHL0R	0x0040	/* Receiver Holding Left 0 Register */
+#define MCHP_I2SMCC_RHR0R	0x0044	/* Receiver Holding Right 0 Register */
+
+#define MCHP_I2SMCC_RHL1R	0x0048	/* Receiver Holding Left 1 Register */
+#define MCHP_I2SMCC_RHR1R	0x004C	/* Receiver Holding Right 1 Register */
+
+#define MCHP_I2SMCC_RHL2R	0x0050	/* Receiver Holding Left 2 Register */
+#define MCHP_I2SMCC_RHR2R	0x0054	/* Receiver Holding Right 2 Register */
+
+#define MCHP_I2SMCC_RHL3R	0x0058	/* Receiver Holding Left 3 Register */
+#define MCHP_I2SMCC_RHR3R	0x005C	/* Receiver Holding Right 3 Register */
+
+#define MCHP_I2SMCC_THL0R	0x0060	/* Transmitter Holding Left 0 Register */
+#define MCHP_I2SMCC_THR0R	0x0064	/* Transmitter Holding Right 0 Register */
+
+#define MCHP_I2SMCC_THL1R	0x0068	/* Transmitter Holding Left 1 Register */
+#define MCHP_I2SMCC_THR1R	0x006C	/* Transmitter Holding Right 1 Register */
+
+#define MCHP_I2SMCC_THL2R	0x0070	/* Transmitter Holding Left 2 Register */
+#define MCHP_I2SMCC_THR2R	0x0074	/* Transmitter Holding Right 2 Register */
+
+#define MCHP_I2SMCC_THL3R	0x0078	/* Transmitter Holding Left 3 Register */
+#define MCHP_I2SMCC_THR3R	0x007C	/* Transmitter Holding Right 3 Register */
+
+#define MCHP_I2SMCC_VERSION	0x00FC	/* Version Register */
+
+/*
+ * ---- Control Register (Write-only) ----
+ */
+#define MCHP_I2SMCC_CR_RXEN		BIT(0)	/* Receiver Enable */
+#define MCHP_I2SMCC_CR_RXDIS		BIT(1)	/* Receiver Disable */
+#define MCHP_I2SMCC_CR_CKEN		BIT(2)	/* Clock Enable */
+#define MCHP_I2SMCC_CR_CKDIS		BIT(3)	/* Clock Disable */
+#define MCHP_I2SMCC_CR_TXEN		BIT(4)	/* Transmitter Enable */
+#define MCHP_I2SMCC_CR_TXDIS		BIT(5)	/* Transmitter Disable */
+#define MCHP_I2SMCC_CR_SWRST		BIT(7)	/* Software Reset */
+
+/*
+ * ---- Mode Register A (Read/Write) ----
+ */
+#define MCHP_I2SMCC_MRA_MODE_MASK		GENMASK(0, 0)
+#define MCHP_I2SMCC_MRA_MODE_SLAVE		(0 << 0)
+#define MCHP_I2SMCC_MRA_MODE_MASTER		(1 << 0)
+
+#define MCHP_I2SMCC_MRA_DATALENGTH_MASK			GENMASK(3, 1)
+#define MCHP_I2SMCC_MRA_DATALENGTH_32_BITS		(0 << 1)
+#define MCHP_I2SMCC_MRA_DATALENGTH_24_BITS		(1 << 1)
+#define MCHP_I2SMCC_MRA_DATALENGTH_20_BITS		(2 << 1)
+#define MCHP_I2SMCC_MRA_DATALENGTH_18_BITS		(3 << 1)
+#define MCHP_I2SMCC_MRA_DATALENGTH_16_BITS		(4 << 1)
+#define MCHP_I2SMCC_MRA_DATALENGTH_16_BITS_COMPACT	(5 << 1)
+#define MCHP_I2SMCC_MRA_DATALENGTH_8_BITS		(6 << 1)
+#define MCHP_I2SMCC_MRA_DATALENGTH_8_BITS_COMPACT	(7 << 1)
+
+#define MCHP_I2SMCC_MRA_WIRECFG_MASK		GENMASK(5, 4)
+#define MCHP_I2SMCC_MRA_WIRECFG_I2S_1_TDM_0	(0 << 4)
+#define MCHP_I2SMCC_MRA_WIRECFG_I2S_2_TDM_1	(1 << 4)
+#define MCHP_I2SMCC_MRA_WIRECFG_I2S_4_TDM_2	(2 << 4)
+#define MCHP_I2SMCC_MRA_WIRECFG_TDM_3		(3 << 4)
+
+#define MCHP_I2SMCC_MRA_FORMAT_MASK		GENMASK(7, 6)
+#define MCHP_I2SMCC_MRA_FORMAT_I2S		(0 << 6)
+#define MCHP_I2SMCC_MRA_FORMAT_LJ		(1 << 6) /* Left Justified */
+#define MCHP_I2SMCC_MRA_FORMAT_TDM		(2 << 6)
+#define MCHP_I2SMCC_MRA_FORMAT_TDMLJ		(3 << 6)
+
+/* Transmitter uses one DMA channel ... */
+/* Left audio samples duplicated to right audio channel */
+#define MCHP_I2SMCC_MRA_RXMONO			BIT(8)
+
+/* I2SDO output of I2SC is internally connected to I2SDI input */
+#define MCHP_I2SMCC_MRA_RXLOOP			BIT(9)
+
+/* Receiver uses one DMA channel ... */
+/* Left audio samples duplicated to right audio channel */
+#define MCHP_I2SMCC_MRA_TXMONO			BIT(10)
+
+/* x sample transmitted when underrun */
+#define MCHP_I2SMCC_MRA_TXSAME_ZERO		(0 << 11) /* Zero sample */
+#define MCHP_I2SMCC_MRA_TXSAME_PREVIOUS		(1 << 11) /* Previous sample */
+
+/* select between peripheral clock and generated clock */
+#define MCHP_I2SMCC_MRA_SRCCLK_PCLK		(0 << 12)
+#define MCHP_I2SMCC_MRA_SRCCLK_GCLK		(1 << 12)
+
+/* Number of TDM Channels - 1 */
+#define MCHP_I2SMCC_MRA_NBCHAN_MASK		GENMASK(15, 13)
+#define MCHP_I2SMCC_MRA_NBCHAN(ch) \
+	((((ch) - 1) << 13) & MCHP_I2SMCC_MRA_NBCHAN_MASK)
+
+/* Selected Clock to I2SMCC Master Clock ratio */
+#define MCHP_I2SMCC_MRA_IMCKDIV_MASK		GENMASK(21, 16)
+#define MCHP_I2SMCC_MRA_IMCKDIV(div) \
+	(((div) << 16) & MCHP_I2SMCC_MRA_IMCKDIV_MASK)
+
+/* TDM Frame Synchronization */
+#define MCHP_I2SMCC_MRA_TDMFS_MASK		GENMASK(23, 22)
+#define MCHP_I2SMCC_MRA_TDMFS_SLOT		(0 << 22)
+#define MCHP_I2SMCC_MRA_TDMFS_HALF		(1 << 22)
+#define MCHP_I2SMCC_MRA_TDMFS_BIT		(2 << 22)
+
+/* Selected Clock to I2SMC Serial Clock ratio */
+#define MCHP_I2SMCC_MRA_ISCKDIV_MASK		GENMASK(29, 24)
+#define MCHP_I2SMCC_MRA_ISCKDIV(div) \
+	(((div) << 24) & MCHP_I2SMCC_MRA_ISCKDIV_MASK)
+
+/* Master Clock mode */
+#define MCHP_I2SMCC_MRA_IMCKMODE_MASK		GENMASK(30, 30)
+/* 0: No master clock generated*/
+#define MCHP_I2SMCC_MRA_IMCKMODE_NONE		(0 << 30)
+/* 1: master clock generated (internally generated clock drives I2SMCK pin) */
+#define MCHP_I2SMCC_MRA_IMCKMODE_GEN		(1 << 30)
+
+/* Slot Width */
+/* 0: slot is 32 bits wide for DATALENGTH = 18/20/24 bits. */
+/* 1: slot is 24 bits wide for DATALENGTH = 18/20/24 bits. */
+#define MCHP_I2SMCC_MRA_IWS			BIT(31)
+
+/*
+ * ---- Mode Register B (Read/Write) ----
+ */
+/* all enabled I2S left channels are filled first, then I2S right channels */
+#define MCHP_I2SMCC_MRB_CRAMODE_LEFT_FIRST	(0 << 0)
+/*
+ * an enabled I2S left channel is filled, then the corresponding right
+ * channel, until all channels are filled
+ */
+#define MCHP_I2SMCC_MRB_CRAMODE_REGULAR		(1 << 0)
+
+#define MCHP_I2SMCC_MRB_FIFOEN			BIT(1)
+
+#define MCHP_I2SMCC_MRB_DMACHUNK_MASK		GENMASK(9, 8)
+#define MCHP_I2SMCC_MRB_DMACHUNK(no_words) \
+	(((fls(no_words) - 1) << 8) & MCHP_I2SMCC_MRB_DMACHUNK_MASK)
+
+#define MCHP_I2SMCC_MRB_CLKSEL_MASK		GENMASK(16, 16)
+#define MCHP_I2SMCC_MRB_CLKSEL_EXT		(0 << 16)
+#define MCHP_I2SMCC_MRB_CLKSEL_INT		(1 << 16)
+
+/*
+ * ---- Status Registers (Read-only) ----
+ */
+#define MCHP_I2SMCC_SR_RXEN		BIT(0)	/* Receiver Enabled */
+#define MCHP_I2SMCC_SR_TXEN		BIT(4)	/* Transmitter Enabled */
+
+/*
+ * ---- Interrupt Enable/Disable/Mask/Status Registers A ----
+ */
+#define MCHP_I2SMCC_INT_TXRDY_MASK(ch)		GENMASK((ch) - 1, 0)
+#define MCHP_I2SMCC_INT_TXRDYCH(ch)		BIT(ch)
+#define MCHP_I2SMCC_INT_TXUNF_MASK(ch)		GENMASK((ch) + 7, 8)
+#define MCHP_I2SMCC_INT_TXUNFCH(ch)		BIT((ch) + 8)
+#define MCHP_I2SMCC_INT_RXRDY_MASK(ch)		GENMASK((ch) + 15, 16)
+#define MCHP_I2SMCC_INT_RXRDYCH(ch)		BIT((ch) + 16)
+#define MCHP_I2SMCC_INT_RXOVF_MASK(ch)		GENMASK((ch) + 23, 24)
+#define MCHP_I2SMCC_INT_RXOVFCH(ch)		BIT((ch) + 24)
+
+/*
+ * ---- Interrupt Enable/Disable/Mask/Status Registers B ----
+ */
+#define MCHP_I2SMCC_INT_WERR			BIT(0)
+#define MCHP_I2SMCC_INT_TXFFRDY			BIT(8)
+#define MCHP_I2SMCC_INT_TXFFEMP			BIT(9)
+#define MCHP_I2SMCC_INT_RXFFRDY			BIT(12)
+#define MCHP_I2SMCC_INT_RXFFFUL			BIT(13)
+
+/*
+ * ---- Version Register (Read-only) ----
+ */
+#define MCHP_I2SMCC_VERSION_MASK		GENMASK(11, 0)
+
+#define MCHP_I2SMCC_MAX_CHANNELS		8
+#define MCHP_I2MCC_TDM_SLOT_WIDTH		32
+
+static const struct regmap_config mchp_i2s_mcc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = MCHP_I2SMCC_VERSION,
+};
+
+struct mchp_i2s_mcc_dev {
+	struct wait_queue_head			wq_txrdy;
+	struct wait_queue_head			wq_rxrdy;
+	struct device				*dev;
+	struct regmap				*regmap;
+	struct clk				*pclk;
+	struct clk				*gclk;
+	struct snd_dmaengine_dai_dma_data	playback;
+	struct snd_dmaengine_dai_dma_data	capture;
+	unsigned int				fmt;
+	unsigned int				sysclk;
+	unsigned int				frame_length;
+	int					tdm_slots;
+	int					channels;
+	int					gclk_use:1;
+	int					gclk_running:1;
+	int					tx_rdy:1;
+	int					rx_rdy:1;
+};
+
+static irqreturn_t mchp_i2s_mcc_interrupt(int irq, void *dev_id)
+{
+	struct mchp_i2s_mcc_dev *dev = dev_id;
+	u32 sra, imra, srb, imrb, pendinga, pendingb, idra = 0;
+	irqreturn_t ret = IRQ_NONE;
+
+	regmap_read(dev->regmap, MCHP_I2SMCC_IMRA, &imra);
+	regmap_read(dev->regmap, MCHP_I2SMCC_ISRA, &sra);
+	pendinga = imra & sra;
+
+	regmap_read(dev->regmap, MCHP_I2SMCC_IMRB, &imrb);
+	regmap_read(dev->regmap, MCHP_I2SMCC_ISRB, &srb);
+	pendingb = imrb & srb;
+
+	if (!pendinga && !pendingb)
+		return IRQ_NONE;
+
+	/*
+	 * Tx/Rx ready interrupts are enabled when stopping only, to assure
+	 * availability and to disable clocks if necessary
+	 */
+	idra |= pendinga & (MCHP_I2SMCC_INT_TXRDY_MASK(dev->channels) |
+			    MCHP_I2SMCC_INT_RXRDY_MASK(dev->channels));
+	if (idra)
+		ret = IRQ_HANDLED;
+
+	if ((imra & MCHP_I2SMCC_INT_TXRDY_MASK(dev->channels)) &&
+	    (imra & MCHP_I2SMCC_INT_TXRDY_MASK(dev->channels)) ==
+	    (idra & MCHP_I2SMCC_INT_TXRDY_MASK(dev->channels))) {
+		dev->tx_rdy = 1;
+		wake_up_interruptible(&dev->wq_txrdy);
+	}
+	if ((imra & MCHP_I2SMCC_INT_RXRDY_MASK(dev->channels)) &&
+	    (imra & MCHP_I2SMCC_INT_RXRDY_MASK(dev->channels)) ==
+	    (idra & MCHP_I2SMCC_INT_RXRDY_MASK(dev->channels))) {
+		dev->rx_rdy = 1;
+		wake_up_interruptible(&dev->wq_rxrdy);
+	}
+	regmap_write(dev->regmap, MCHP_I2SMCC_IDRA, idra);
+
+	return ret;
+}
+
+static int mchp_i2s_mcc_set_sysclk(struct snd_soc_dai *dai,
+				   int clk_id, unsigned int freq, int dir)
+{
+	struct mchp_i2s_mcc_dev *dev = snd_soc_dai_get_drvdata(dai);
+
+	dev_dbg(dev->dev, "%s() clk_id=%d freq=%u dir=%d\n",
+		__func__, clk_id, freq, dir);
+
+	/* We do not need SYSCLK */
+	if (dir == SND_SOC_CLOCK_IN)
+		return 0;
+
+	dev->sysclk = freq;
+
+	return 0;
+}
+
+static int mchp_i2s_mcc_set_bclk_ratio(struct snd_soc_dai *dai,
+				       unsigned int ratio)
+{
+	struct mchp_i2s_mcc_dev *dev = snd_soc_dai_get_drvdata(dai);
+
+	dev_dbg(dev->dev, "%s() ratio=%u\n", __func__, ratio);
+
+	dev->frame_length = ratio;
+
+	return 0;
+}
+
+static int mchp_i2s_mcc_set_dai_fmt(struct snd_soc_dai *dai, unsigned int fmt)
+{
+	struct mchp_i2s_mcc_dev *dev = snd_soc_dai_get_drvdata(dai);
+
+	dev_dbg(dev->dev, "%s() fmt=%#x\n", __func__, fmt);
+
+	/* We don't support any kind of clock inversion */
+	if ((fmt & SND_SOC_DAIFMT_INV_MASK) != SND_SOC_DAIFMT_NB_NF)
+		return -EINVAL;
+
+	/* We can't generate only FSYNC */
+	if ((fmt & SND_SOC_DAIFMT_MASTER_MASK) == SND_SOC_DAIFMT_CBM_CFS)
+		return -EINVAL;
+
+	/* We can only reconfigure the IP when it's stopped */
+	if (fmt & SND_SOC_DAIFMT_CONT)
+		return -EINVAL;
+
+	dev->fmt = fmt;
+
+	return 0;
+}
+
+static int mchp_i2s_mcc_set_dai_tdm_slot(struct snd_soc_dai *dai,
+					 unsigned int tx_mask,
+					 unsigned int rx_mask,
+					 int slots, int slot_width)
+{
+	struct mchp_i2s_mcc_dev *dev = snd_soc_dai_get_drvdata(dai);
+
+	dev_dbg(dev->dev,
+		"%s() tx_mask=0x%08x rx_mask=0x%08x slots=%d width=%d\n",
+		__func__, tx_mask, rx_mask, slots, slot_width);
+
+	if (slots < 0 || slots > MCHP_I2SMCC_MAX_CHANNELS ||
+	    slot_width != MCHP_I2MCC_TDM_SLOT_WIDTH)
+		return -EINVAL;
+
+	if (slots) {
+		/* We do not support daisy chain */
+		if (rx_mask != GENMASK(slots - 1, 0) ||
+		    rx_mask != tx_mask)
+			return -EINVAL;
+	}
+
+	dev->tdm_slots = slots;
+	dev->frame_length = slots * MCHP_I2MCC_TDM_SLOT_WIDTH;
+
+	return 0;
+}
+
+static int mchp_i2s_mcc_clk_get_rate_diff(struct clk *clk,
+					  unsigned long rate,
+					  struct clk **best_clk,
+					  unsigned long *best_rate,
+					  unsigned long *best_diff_rate)
+{
+	long round_rate;
+	unsigned int diff_rate;
+
+	round_rate = clk_round_rate(clk, rate);
+	if (round_rate < 0)
+		return (int)round_rate;
+
+	diff_rate = abs(rate - round_rate);
+	if (diff_rate < *best_diff_rate) {
+		*best_clk = clk;
+		*best_diff_rate = diff_rate;
+		*best_rate = rate;
+	}
+
+	return 0;
+}
+
+static int mchp_i2s_mcc_config_divs(struct mchp_i2s_mcc_dev *dev,
+				    unsigned int bclk, unsigned int *mra)
+{
+	unsigned long clk_rate;
+	unsigned long lcm_rate;
+	unsigned long best_rate = 0;
+	unsigned long best_diff_rate = ~0;
+	unsigned int sysclk;
+	struct clk *best_clk = NULL;
+	int ret;
+
+	/* For code simplification */
+	if (!dev->sysclk)
+		sysclk = bclk;
+	else
+		sysclk = dev->sysclk;
+
+	/*
+	 * MCLK is Selected CLK / (2 * IMCKDIV),
+	 * BCLK is Selected CLK / (2 * ISCKDIV);
+	 * if IMCKDIV or ISCKDIV are 0, MCLK or BCLK = Selected CLK
+	 */
+	lcm_rate = lcm(sysclk, bclk);
+	if ((lcm_rate / sysclk % 2 == 1 && lcm_rate / sysclk > 2) ||
+	    (lcm_rate / bclk % 2 == 1 && lcm_rate / bclk > 2))
+		lcm_rate *= 2;
+
+	for (clk_rate = lcm_rate;
+	     (clk_rate == sysclk || clk_rate / (sysclk * 2) <= GENMASK(5, 0)) &&
+	     (clk_rate == bclk || clk_rate / (bclk * 2) <= GENMASK(5, 0));
+	     clk_rate += lcm_rate) {
+		ret = mchp_i2s_mcc_clk_get_rate_diff(dev->gclk, clk_rate,
+						     &best_clk, &best_rate,
+						     &best_diff_rate);
+		if (ret) {
+			dev_err(dev->dev, "gclk error for rate %lu: %d",
+				clk_rate, ret);
+		} else {
+			if (!best_diff_rate) {
+				dev_dbg(dev->dev, "found perfect rate on gclk: %lu\n",
+					clk_rate);
+				break;
+			}
+		}
+
+		ret = mchp_i2s_mcc_clk_get_rate_diff(dev->pclk, clk_rate,
+						     &best_clk, &best_rate,
+						     &best_diff_rate);
+		if (ret) {
+			dev_err(dev->dev, "pclk error for rate %lu: %d",
+				clk_rate, ret);
+		} else {
+			if (!best_diff_rate) {
+				dev_dbg(dev->dev, "found perfect rate on pclk: %lu\n",
+					clk_rate);
+				break;
+			}
+		}
+	}
+
+	/* check if clocks returned only errors */
+	if (!best_clk) {
+		dev_err(dev->dev, "unable to change rate to clocks\n");
+		return -EINVAL;
+	}
+
+	dev_dbg(dev->dev, "source CLK is %s with rate %lu, diff %lu\n",
+		best_clk == dev->pclk ? "pclk" : "gclk",
+		best_rate, best_diff_rate);
+
+	/* set the rate */
+	ret = clk_set_rate(best_clk, best_rate);
+	if (ret) {
+		dev_err(dev->dev, "unable to set rate %lu to %s: %d\n",
+			best_rate, best_clk == dev->pclk ? "PCLK" : "GCLK",
+			ret);
+		return ret;
+	}
+
+	/* Configure divisors */
+	if (dev->sysclk)
+		*mra |= MCHP_I2SMCC_MRA_IMCKDIV(best_rate / (2 * sysclk));
+	*mra |= MCHP_I2SMCC_MRA_ISCKDIV(best_rate / (2 * bclk));
+
+	if (best_clk == dev->gclk) {
+		*mra |= MCHP_I2SMCC_MRA_SRCCLK_GCLK;
+		ret = clk_prepare(dev->gclk);
+		if (ret < 0)
+			dev_err(dev->dev, "unable to prepare GCLK: %d\n", ret);
+		else
+			dev->gclk_use = 1;
+	} else {
+		*mra |= MCHP_I2SMCC_MRA_SRCCLK_PCLK;
+		dev->gclk_use = 0;
+	}
+
+	return 0;
+}
+
+static int mchp_i2s_mcc_is_running(struct mchp_i2s_mcc_dev *dev)
+{
+	u32 sr;
+
+	regmap_read(dev->regmap, MCHP_I2SMCC_SR, &sr);
+	return !!(sr & (MCHP_I2SMCC_SR_TXEN | MCHP_I2SMCC_SR_RXEN));
+}
+
+static int mchp_i2s_mcc_hw_params(struct snd_pcm_substream *substream,
+				  struct snd_pcm_hw_params *params,
+				  struct snd_soc_dai *dai)
+{
+	struct mchp_i2s_mcc_dev *dev = snd_soc_dai_get_drvdata(dai);
+	u32 mra = 0;
+	u32 mrb = 0;
+	unsigned int channels = params_channels(params);
+	unsigned int frame_length = dev->frame_length;
+	unsigned int bclk_rate;
+	int set_divs = 0;
+	int ret;
+	bool is_playback = (substream->stream == SNDRV_PCM_STREAM_PLAYBACK);
+
+	dev_dbg(dev->dev, "%s() rate=%u format=%#x width=%u channels=%u\n",
+		__func__, params_rate(params), params_format(params),
+		params_width(params), params_channels(params));
+
+	switch (dev->fmt & SND_SOC_DAIFMT_FORMAT_MASK) {
+	case SND_SOC_DAIFMT_I2S:
+		if (dev->tdm_slots) {
+			dev_err(dev->dev, "I2S with TDM is not supported\n");
+			return -EINVAL;
+		}
+		mra |= MCHP_I2SMCC_MRA_FORMAT_I2S;
+		break;
+	case SND_SOC_DAIFMT_LEFT_J:
+		if (dev->tdm_slots) {
+			dev_err(dev->dev, "Left-Justified with TDM is not supported\n");
+			return -EINVAL;
+		}
+		mra |= MCHP_I2SMCC_MRA_FORMAT_LJ;
+		break;
+	case SND_SOC_DAIFMT_DSP_A:
+		mra |= MCHP_I2SMCC_MRA_FORMAT_TDM;
+		break;
+	default:
+		dev_err(dev->dev, "unsupported bus format\n");
+		return -EINVAL;
+	}
+
+	switch (dev->fmt & SND_SOC_DAIFMT_MASTER_MASK) {
+	case SND_SOC_DAIFMT_CBS_CFS:
+		/* cpu is BCLK and LRC master */
+		mra |= MCHP_I2SMCC_MRA_MODE_MASTER;
+		if (dev->sysclk)
+			mra |= MCHP_I2SMCC_MRA_IMCKMODE_GEN;
+		set_divs = 1;
+		break;
+	case SND_SOC_DAIFMT_CBS_CFM:
+		/* cpu is BCLK master */
+		mrb |= MCHP_I2SMCC_MRB_CLKSEL_INT;
+		set_divs = 1;
+		/* fall through */
+	case SND_SOC_DAIFMT_CBM_CFM:
+		/* cpu is slave */
+		mra |= MCHP_I2SMCC_MRA_MODE_SLAVE;
+		if (dev->sysclk)
+			dev_warn(dev->dev, "Unable to generate MCLK in Slave mode\n");
+		break;
+	default:
+		dev_err(dev->dev, "unsupported master/slave mode\n");
+		return -EINVAL;
+	}
+
+	if (dev->fmt & (SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_LEFT_J)) {
+		switch (channels) {
+		case 1:
+			if (is_playback)
+				mra |= MCHP_I2SMCC_MRA_TXMONO;
+			else
+				mra |= MCHP_I2SMCC_MRA_RXMONO;
+			break;
+		case 2:
+			break;
+		default:
+			dev_err(dev->dev, "unsupported number of audio channels\n");
+			return -EINVAL;
+		}
+
+		if (!frame_length)
+			frame_length = 2 * params_physical_width(params);
+	} else if (dev->fmt & SND_SOC_DAIFMT_DSP_A) {
+		if (dev->tdm_slots) {
+			if (channels % 2 && channels * 2 <= dev->tdm_slots) {
+				/*
+				 * Duplicate data for even-numbered channels
+				 * to odd-numbered channels
+				 */
+				if (is_playback)
+					mra |= MCHP_I2SMCC_MRA_TXMONO;
+				else
+					mra |= MCHP_I2SMCC_MRA_RXMONO;
+			}
+			channels = dev->tdm_slots;
+		}
+
+		mra |= MCHP_I2SMCC_MRA_NBCHAN(channels);
+		if (!frame_length)
+			frame_length = channels * MCHP_I2MCC_TDM_SLOT_WIDTH;
+	}
+
+	/*
+	 * We must have the same burst size configured
+	 * in the DMA transfer and in out IP
+	 */
+	mrb |= MCHP_I2SMCC_MRB_DMACHUNK(channels);
+	if (is_playback)
+		dev->playback.maxburst = 1 << (fls(channels) - 1);
+	else
+		dev->capture.maxburst = 1 << (fls(channels) - 1);
+
+	switch (params_format(params)) {
+	case SNDRV_PCM_FORMAT_S8:
+		mra |= MCHP_I2SMCC_MRA_DATALENGTH_8_BITS;
+		break;
+	case SNDRV_PCM_FORMAT_S16_LE:
+		mra |= MCHP_I2SMCC_MRA_DATALENGTH_16_BITS;
+		break;
+	case SNDRV_PCM_FORMAT_S18_3LE:
+		mra |= MCHP_I2SMCC_MRA_DATALENGTH_18_BITS |
+		       MCHP_I2SMCC_MRA_IWS;
+		break;
+	case SNDRV_PCM_FORMAT_S20_3LE:
+		mra |= MCHP_I2SMCC_MRA_DATALENGTH_20_BITS |
+		       MCHP_I2SMCC_MRA_IWS;
+		break;
+	case SNDRV_PCM_FORMAT_S24_3LE:
+		mra |= MCHP_I2SMCC_MRA_DATALENGTH_24_BITS |
+		       MCHP_I2SMCC_MRA_IWS;
+		break;
+	case SNDRV_PCM_FORMAT_S24_LE:
+		mra |= MCHP_I2SMCC_MRA_DATALENGTH_24_BITS;
+		break;
+	case SNDRV_PCM_FORMAT_S32_LE:
+		mra |= MCHP_I2SMCC_MRA_DATALENGTH_32_BITS;
+		break;
+	default:
+		dev_err(dev->dev, "unsupported size/endianness for audio samples\n");
+		return -EINVAL;
+	}
+
+	/*
+	 * If we are already running, the wanted setup must be
+	 * the same with the one that's currently ongoing
+	 */
+	if (mchp_i2s_mcc_is_running(dev)) {
+		u32 mra_cur;
+		u32 mrb_cur;
+
+		regmap_read(dev->regmap, MCHP_I2SMCC_MRA, &mra_cur);
+		regmap_read(dev->regmap, MCHP_I2SMCC_MRB, &mrb_cur);
+		if (mra != mra_cur || mrb != mrb_cur)
+			return -EINVAL;
+
+		return 0;
+	}
+
+	/* Save the number of channels to know what interrupts to enable */
+	dev->channels = channels;
+
+	if (set_divs) {
+		bclk_rate = frame_length * params_rate(params);
+		ret = mchp_i2s_mcc_config_divs(dev, bclk_rate, &mra);
+		if (ret) {
+			dev_err(dev->dev, "unable to configure the divisors: %d\n",
+				ret);
+			return ret;
+		}
+	}
+
+	ret = regmap_write(dev->regmap, MCHP_I2SMCC_MRA, mra);
+	if (ret < 0)
+		return ret;
+	return regmap_write(dev->regmap, MCHP_I2SMCC_MRB, mrb);
+}
+
+static int mchp_i2s_mcc_hw_free(struct snd_pcm_substream *substream,
+				struct snd_soc_dai *dai)
+{
+	struct mchp_i2s_mcc_dev *dev = snd_soc_dai_get_drvdata(dai);
+	bool is_playback = (substream->stream == SNDRV_PCM_STREAM_PLAYBACK);
+	long err;
+
+	if (is_playback) {
+		err = wait_event_interruptible_timeout(dev->wq_txrdy,
+						       dev->tx_rdy,
+						       msecs_to_jiffies(500));
+	} else {
+		err = wait_event_interruptible_timeout(dev->wq_rxrdy,
+						       dev->rx_rdy,
+						       msecs_to_jiffies(500));
+	}
+
+	if (err == 0) {
+		u32 idra;
+
+		dev_warn_once(dev->dev, "Timeout waiting for %s\n",
+			      is_playback ? "Tx ready" : "Rx ready");
+		if (is_playback)
+			idra = MCHP_I2SMCC_INT_TXRDY_MASK(dev->channels);
+		else
+			idra = MCHP_I2SMCC_INT_RXRDY_MASK(dev->channels);
+		regmap_write(dev->regmap, MCHP_I2SMCC_IDRA, idra);
+	}
+
+	if (!mchp_i2s_mcc_is_running(dev)) {
+		regmap_write(dev->regmap, MCHP_I2SMCC_CR, MCHP_I2SMCC_CR_CKDIS);
+
+		if (dev->gclk_running) {
+			clk_disable_unprepare(dev->gclk);
+			dev->gclk_running = 0;
+		}
+	}
+
+	return 0;
+}
+
+static int mchp_i2s_mcc_trigger(struct snd_pcm_substream *substream, int cmd,
+				struct snd_soc_dai *dai)
+{
+	struct mchp_i2s_mcc_dev *dev = snd_soc_dai_get_drvdata(dai);
+	bool is_playback = (substream->stream == SNDRV_PCM_STREAM_PLAYBACK);
+	u32 cr = 0;
+	u32 iera = 0;
+	u32 sr;
+	int err;
+
+	switch (cmd) {
+	case SNDRV_PCM_TRIGGER_START:
+	case SNDRV_PCM_TRIGGER_RESUME:
+	case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
+		if (is_playback)
+			cr = MCHP_I2SMCC_CR_TXEN | MCHP_I2SMCC_CR_CKEN;
+		else
+			cr = MCHP_I2SMCC_CR_RXEN | MCHP_I2SMCC_CR_CKEN;
+		break;
+	case SNDRV_PCM_TRIGGER_STOP:
+	case SNDRV_PCM_TRIGGER_SUSPEND:
+	case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
+		regmap_read(dev->regmap, MCHP_I2SMCC_SR, &sr);
+		if (is_playback && (sr & MCHP_I2SMCC_SR_TXEN)) {
+			cr = MCHP_I2SMCC_CR_TXDIS;
+			dev->tx_rdy = 0;
+			/*
+			 * Enable Tx Ready interrupts on all channels
+			 * to assure all data is sent
+			 */
+			iera = MCHP_I2SMCC_INT_TXRDY_MASK(dev->channels);
+		} else if (!is_playback && (sr & MCHP_I2SMCC_SR_RXEN)) {
+			cr = MCHP_I2SMCC_CR_RXDIS;
+			dev->rx_rdy = 0;
+			/*
+			 * Enable Rx Ready interrupts on all channels
+			 * to assure all data is received
+			 */
+			iera = MCHP_I2SMCC_INT_RXRDY_MASK(dev->channels);
+		}
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if ((cr & MCHP_I2SMCC_CR_CKEN) && dev->gclk_use &&
+	    !dev->gclk_running) {
+		err = clk_enable(dev->gclk);
+		if (err) {
+			dev_err_once(dev->dev, "failed to enable GCLK: %d\n",
+				     err);
+		} else {
+			dev->gclk_running = 1;
+		}
+	}
+
+	regmap_write(dev->regmap, MCHP_I2SMCC_IERA, iera);
+	regmap_write(dev->regmap, MCHP_I2SMCC_CR, cr);
+
+	return 0;
+}
+
+static int mchp_i2s_mcc_startup(struct snd_pcm_substream *substream,
+				struct snd_soc_dai *dai)
+{
+	struct mchp_i2s_mcc_dev *dev = snd_soc_dai_get_drvdata(dai);
+
+	/* Software reset the IP if it's not running */
+	if (!mchp_i2s_mcc_is_running(dev)) {
+		return regmap_write(dev->regmap, MCHP_I2SMCC_CR,
+				    MCHP_I2SMCC_CR_SWRST);
+	}
+
+	return 0;
+}
+
+static const struct snd_soc_dai_ops mchp_i2s_mcc_dai_ops = {
+	.set_sysclk	= mchp_i2s_mcc_set_sysclk,
+	.set_bclk_ratio = mchp_i2s_mcc_set_bclk_ratio,
+	.startup	= mchp_i2s_mcc_startup,
+	.trigger	= mchp_i2s_mcc_trigger,
+	.hw_params	= mchp_i2s_mcc_hw_params,
+	.hw_free	= mchp_i2s_mcc_hw_free,
+	.set_fmt	= mchp_i2s_mcc_set_dai_fmt,
+	.set_tdm_slot	= mchp_i2s_mcc_set_dai_tdm_slot,
+};
+
+static int mchp_i2s_mcc_dai_probe(struct snd_soc_dai *dai)
+{
+	struct mchp_i2s_mcc_dev *dev = snd_soc_dai_get_drvdata(dai);
+
+	init_waitqueue_head(&dev->wq_txrdy);
+	init_waitqueue_head(&dev->wq_rxrdy);
+
+	snd_soc_dai_init_dma_data(dai, &dev->playback, &dev->capture);
+
+	return 0;
+}
+
+#define MCHP_I2SMCC_RATES              SNDRV_PCM_RATE_8000_192000
+
+#define MCHP_I2SMCC_FORMATS	(SNDRV_PCM_FMTBIT_S8 |          \
+				 SNDRV_PCM_FMTBIT_S16_LE |      \
+				 SNDRV_PCM_FMTBIT_S18_3LE |     \
+				 SNDRV_PCM_FMTBIT_S20_3LE |     \
+				 SNDRV_PCM_FMTBIT_S24_3LE |     \
+				 SNDRV_PCM_FMTBIT_S24_LE |      \
+				 SNDRV_PCM_FMTBIT_S32_LE)
+
+static struct snd_soc_dai_driver mchp_i2s_mcc_dai = {
+	.probe	= mchp_i2s_mcc_dai_probe,
+	.playback = {
+		.stream_name = "I2SMCC-Playback",
+		.channels_min = 1,
+		.channels_max = 8,
+		.rates = MCHP_I2SMCC_RATES,
+		.formats = MCHP_I2SMCC_FORMATS,
+	},
+	.capture = {
+		.stream_name = "I2SMCC-Capture",
+		.channels_min = 1,
+		.channels_max = 8,
+		.rates = MCHP_I2SMCC_RATES,
+		.formats = MCHP_I2SMCC_FORMATS,
+	},
+	.ops = &mchp_i2s_mcc_dai_ops,
+	.symmetric_rates = 1,
+	.symmetric_samplebits = 1,
+	.symmetric_channels = 1,
+};
+
+static const struct snd_soc_component_driver mchp_i2s_mcc_component = {
+	.name	= "mchp-i2s-mcc",
+};
+
+#ifdef CONFIG_OF
+static const struct of_device_id mchp_i2s_mcc_dt_ids[] = {
+	{
+		.compatible = "microchip,sam9x60-i2smcc",
+	},
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, mchp_i2s_mcc_dt_ids);
+#endif
+
+static int mchp_i2s_mcc_probe(struct platform_device *pdev)
+{
+	struct mchp_i2s_mcc_dev *dev;
+	struct resource *mem;
+	struct regmap *regmap;
+	void __iomem *base;
+	u32 version;
+	int irq;
+	int err;
+
+	dev = devm_kzalloc(&pdev->dev, sizeof(*dev), GFP_KERNEL);
+	if (!dev)
+		return -ENOMEM;
+
+	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	base = devm_ioremap_resource(&pdev->dev, mem);
+	if (IS_ERR(base))
+		return PTR_ERR(base);
+
+	regmap = devm_regmap_init_mmio(&pdev->dev, base,
+				       &mchp_i2s_mcc_regmap_config);
+	if (IS_ERR(regmap))
+		return PTR_ERR(regmap);
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0)
+		return irq;
+
+	err = devm_request_irq(&pdev->dev, irq, mchp_i2s_mcc_interrupt, 0,
+			       dev_name(&pdev->dev), dev);
+	if (err)
+		return err;
+
+	dev->pclk = devm_clk_get(&pdev->dev, "pclk");
+	if (IS_ERR(dev->pclk)) {
+		err = PTR_ERR(dev->pclk);
+		dev_err(&pdev->dev,
+			"failed to get the peripheral clock: %d\n", err);
+		return err;
+	}
+
+	/* Get the optional generated clock */
+	dev->gclk = devm_clk_get(&pdev->dev, "gclk");
+	if (IS_ERR(dev->gclk)) {
+		if (PTR_ERR(dev->gclk) == -EPROBE_DEFER)
+			return -EPROBE_DEFER;
+		dev_warn(&pdev->dev,
+			 "generated clock not found: %d\n", err);
+		dev->gclk = NULL;
+	}
+
+	dev->dev = &pdev->dev;
+	dev->regmap = regmap;
+	platform_set_drvdata(pdev, dev);
+
+	err = clk_prepare_enable(dev->pclk);
+	if (err) {
+		dev_err(&pdev->dev,
+			"failed to enable the peripheral clock: %d\n", err);
+		return err;
+	}
+
+	err = devm_snd_soc_register_component(&pdev->dev,
+					      &mchp_i2s_mcc_component,
+					      &mchp_i2s_mcc_dai, 1);
+	if (err) {
+		dev_err(&pdev->dev, "failed to register DAI: %d\n", err);
+		clk_disable_unprepare(dev->pclk);
+		return err;
+	}
+
+	dev->playback.addr	= (dma_addr_t)mem->start + MCHP_I2SMCC_THR;
+	dev->capture.addr	= (dma_addr_t)mem->start + MCHP_I2SMCC_RHR;
+
+	err = devm_snd_dmaengine_pcm_register(&pdev->dev, NULL, 0);
+	if (err) {
+		dev_err(&pdev->dev, "failed to register PCM: %d\n", err);
+		clk_disable_unprepare(dev->pclk);
+		return err;
+	}
+
+	/* Get IP version. */
+	regmap_read(dev->regmap, MCHP_I2SMCC_VERSION, &version);
+	dev_info(&pdev->dev, "hw version: %#lx\n",
+		 version & MCHP_I2SMCC_VERSION_MASK);
+
+	return 0;
+}
+
+static int mchp_i2s_mcc_remove(struct platform_device *pdev)
+{
+	struct mchp_i2s_mcc_dev *dev = platform_get_drvdata(pdev);
+
+	clk_disable_unprepare(dev->pclk);
+
+	return 0;
+}
+
+static struct platform_driver mchp_i2s_mcc_driver = {
+	.driver		= {
+		.name	= "mchp_i2s_mcc",
+		.of_match_table	= of_match_ptr(mchp_i2s_mcc_dt_ids),
+	},
+	.probe		= mchp_i2s_mcc_probe,
+	.remove		= mchp_i2s_mcc_remove,
+};
+module_platform_driver(mchp_i2s_mcc_driver);
+
+MODULE_DESCRIPTION("Microchip I2S Multi-Channel Controller driver");
+MODULE_AUTHOR("Codrin Ciubotariu <codrin.ciubotariu@microchip.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/sound/soc/atmel/tse850-pcm5142.c b/sound/soc/atmel/tse850-pcm5142.c
index 214adcad5419..ae445184614a 100644
--- a/sound/soc/atmel/tse850-pcm5142.c
+++ b/sound/soc/atmel/tse850-pcm5142.c
@@ -117,8 +117,8 @@ static int tse850_put_mux2(struct snd_kcontrol *kctrl,
 	return snd_soc_dapm_put_enum_double(kctrl, ucontrol);
 }
 
-int tse850_get_mix(struct snd_kcontrol *kctrl,
-		   struct snd_ctl_elem_value *ucontrol)
+static int tse850_get_mix(struct snd_kcontrol *kctrl,
+			  struct snd_ctl_elem_value *ucontrol)
 {
 	struct snd_soc_dapm_context *dapm = snd_soc_dapm_kcontrol_dapm(kctrl);
 	struct snd_soc_card *card = dapm->card;
@@ -129,8 +129,8 @@ int tse850_get_mix(struct snd_kcontrol *kctrl,
 	return 0;
 }
 
-int tse850_put_mix(struct snd_kcontrol *kctrl,
-		   struct snd_ctl_elem_value *ucontrol)
+static int tse850_put_mix(struct snd_kcontrol *kctrl,
+			  struct snd_ctl_elem_value *ucontrol)
 {
 	struct snd_soc_dapm_context *dapm = snd_soc_dapm_kcontrol_dapm(kctrl);
 	struct snd_soc_card *card = dapm->card;
@@ -151,8 +151,8 @@ int tse850_put_mix(struct snd_kcontrol *kctrl,
 	return 1;
 }
 
-int tse850_get_ana(struct snd_kcontrol *kctrl,
-		   struct snd_ctl_elem_value *ucontrol)
+static int tse850_get_ana(struct snd_kcontrol *kctrl,
+			  struct snd_ctl_elem_value *ucontrol)
 {
 	struct snd_soc_dapm_context *dapm = snd_soc_dapm_kcontrol_dapm(kctrl);
 	struct snd_soc_card *card = dapm->card;
@@ -184,8 +184,8 @@ int tse850_get_ana(struct snd_kcontrol *kctrl,
 	return 0;
 }
 
-int tse850_put_ana(struct snd_kcontrol *kctrl,
-		   struct snd_ctl_elem_value *ucontrol)
+static int tse850_put_ana(struct snd_kcontrol *kctrl,
+			  struct snd_ctl_elem_value *ucontrol)
 {
 	struct snd_soc_dapm_context *dapm = snd_soc_dapm_kcontrol_dapm(kctrl);
 	struct snd_soc_card *card = dapm->card;
diff --git a/sound/soc/codecs/Kconfig b/sound/soc/codecs/Kconfig
index 667fc1d59e18..8f577258080b 100644
--- a/sound/soc/codecs/Kconfig
+++ b/sound/soc/codecs/Kconfig
@@ -94,6 +94,7 @@ config SND_SOC_ALL_CODECS
 	select SND_SOC_JZ4725B_CODEC
 	select SND_SOC_LM4857 if I2C
 	select SND_SOC_LM49453 if I2C
+	select SND_SOC_LOCHNAGAR_SC if MFD_LOCHNAGAR
 	select SND_SOC_MAX98088 if I2C
 	select SND_SOC_MAX98090 if I2C
 	select SND_SOC_MAX98095 if I2C
@@ -179,8 +180,8 @@ config SND_SOC_ALL_CODECS
 	select SND_SOC_TLV320AIC23_SPI if SPI_MASTER
 	select SND_SOC_TLV320AIC26 if SPI_MASTER
 	select SND_SOC_TLV320AIC31XX if I2C
-	select SND_SOC_TLV320AIC32X4_I2C if I2C
-	select SND_SOC_TLV320AIC32X4_SPI if SPI_MASTER
+	select SND_SOC_TLV320AIC32X4_I2C if I2C && COMMON_CLK
+	select SND_SOC_TLV320AIC32X4_SPI if SPI_MASTER && COMMON_CLK
 	select SND_SOC_TLV320AIC3X if I2C
 	select SND_SOC_TPA6130A2 if I2C
 	select SND_SOC_TLV320DAC33 if I2C
@@ -688,6 +689,13 @@ config SND_SOC_ISABELLE
 config SND_SOC_LM49453
 	tristate
 
+config SND_SOC_LOCHNAGAR_SC
+	tristate "Lochnagar Sound Card"
+	depends on MFD_LOCHNAGAR
+	help
+	  This driver support the sound card functionality of the Cirrus
+	  Logic Lochnagar audio development board.
+
 config SND_SOC_MAX98088
 	tristate "Maxim MAX98088/9 Low-Power, Stereo Audio Codec"
 	depends on I2C
@@ -1097,15 +1105,18 @@ config SND_SOC_TLV320AIC31XX
 
 config SND_SOC_TLV320AIC32X4
 	tristate
+	depends on COMMON_CLK
 
 config SND_SOC_TLV320AIC32X4_I2C
 	tristate "Texas Instruments TLV320AIC32x4 audio CODECs - I2C"
 	depends on I2C
+	depends on COMMON_CLK
 	select SND_SOC_TLV320AIC32X4
 
 config SND_SOC_TLV320AIC32X4_SPI
 	tristate "Texas Instruments TLV320AIC32x4 audio CODECs - SPI"
 	depends on SPI_MASTER
+	depends on COMMON_CLK
 	select SND_SOC_TLV320AIC32X4
 
 config SND_SOC_TLV320AIC3X
diff --git a/sound/soc/codecs/Makefile b/sound/soc/codecs/Makefile
index aab2ad95a137..aa7720a7a0aa 100644
--- a/sound/soc/codecs/Makefile
+++ b/sound/soc/codecs/Makefile
@@ -91,6 +91,7 @@ snd-soc-jz4725b-codec-objs := jz4725b.o
 snd-soc-l3-objs := l3.o
 snd-soc-lm4857-objs := lm4857.o
 snd-soc-lm49453-objs := lm49453.o
+snd-soc-lochnagar-sc-objs := lochnagar-sc.o
 snd-soc-max9759-objs := max9759.o
 snd-soc-max9768-objs := max9768.o
 snd-soc-max98088-objs := max98088.o
@@ -192,7 +193,7 @@ snd-soc-tlv320aic23-i2c-objs := tlv320aic23-i2c.o
 snd-soc-tlv320aic23-spi-objs := tlv320aic23-spi.o
 snd-soc-tlv320aic26-objs := tlv320aic26.o
 snd-soc-tlv320aic31xx-objs := tlv320aic31xx.o
-snd-soc-tlv320aic32x4-objs := tlv320aic32x4.o
+snd-soc-tlv320aic32x4-objs := tlv320aic32x4.o tlv320aic32x4-clk.o
 snd-soc-tlv320aic32x4-i2c-objs := tlv320aic32x4-i2c.o
 snd-soc-tlv320aic32x4-spi-objs := tlv320aic32x4-spi.o
 snd-soc-tlv320aic3x-objs := tlv320aic3x.o
@@ -364,6 +365,7 @@ obj-$(CONFIG_SND_SOC_JZ4725B_CODEC)	+= snd-soc-jz4725b-codec.o
 obj-$(CONFIG_SND_SOC_L3)	+= snd-soc-l3.o
 obj-$(CONFIG_SND_SOC_LM4857)	+= snd-soc-lm4857.o
 obj-$(CONFIG_SND_SOC_LM49453)   += snd-soc-lm49453.o
+obj-$(CONFIG_SND_SOC_LOCHNAGAR_SC)	+= snd-soc-lochnagar-sc.o
 obj-$(CONFIG_SND_SOC_MAX9759)	+= snd-soc-max9759.o
 obj-$(CONFIG_SND_SOC_MAX9768)	+= snd-soc-max9768.o
 obj-$(CONFIG_SND_SOC_MAX98088)	+= snd-soc-max98088.o
diff --git a/sound/soc/codecs/cs42l51-i2c.c b/sound/soc/codecs/cs42l51-i2c.c
index 4b5731a41876..116221e581ce 100644
--- a/sound/soc/codecs/cs42l51-i2c.c
+++ b/sound/soc/codecs/cs42l51-i2c.c
@@ -29,18 +29,27 @@ static int cs42l51_i2c_probe(struct i2c_client *i2c,
 	struct regmap_config config;
 
 	config = cs42l51_regmap;
-	config.val_bits = 8;
-	config.reg_bits = 8;
 
 	return cs42l51_probe(&i2c->dev, devm_regmap_init_i2c(i2c, &config));
 }
 
+static int cs42l51_i2c_remove(struct i2c_client *i2c)
+{
+	return cs42l51_remove(&i2c->dev);
+}
+
+static const struct dev_pm_ops cs42l51_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(cs42l51_suspend, cs42l51_resume)
+};
+
 static struct i2c_driver cs42l51_i2c_driver = {
 	.driver = {
 		.name = "cs42l51",
 		.of_match_table = cs42l51_of_match,
+		.pm = &cs42l51_pm_ops,
 	},
 	.probe = cs42l51_i2c_probe,
+	.remove = cs42l51_i2c_remove,
 	.id_table = cs42l51_i2c_id,
 };
 
diff --git a/sound/soc/codecs/cs42l51.c b/sound/soc/codecs/cs42l51.c
index fd2bd74024c1..991e4ebd7a04 100644
--- a/sound/soc/codecs/cs42l51.c
+++ b/sound/soc/codecs/cs42l51.c
@@ -30,7 +30,9 @@
 #include <sound/initval.h>
 #include <sound/pcm_params.h>
 #include <sound/pcm.h>
+#include <linux/gpio/consumer.h>
 #include <linux/regmap.h>
+#include <linux/regulator/consumer.h>
 
 #include "cs42l51.h"
 
@@ -40,11 +42,21 @@ enum master_slave_mode {
 	MODE_MASTER,
 };
 
+static const char * const cs42l51_supply_names[] = {
+	"VL",
+	"VD",
+	"VA",
+	"VAHP",
+};
+
 struct cs42l51_private {
 	unsigned int mclk;
 	struct clk *mclk_handle;
 	unsigned int audio_mode;	/* The mode (I2S or left-justified) */
 	enum master_slave_mode func;
+	struct regulator_bulk_data supplies[ARRAY_SIZE(cs42l51_supply_names)];
+	struct gpio_desc *reset_gpio;
+	struct regmap *regmap;
 };
 
 #define CS42L51_FORMATS ( \
@@ -111,6 +123,7 @@ static const DECLARE_TLV_DB_SCALE(tone_tlv, -1050, 150, 0);
 static const DECLARE_TLV_DB_SCALE(aout_tlv, -10200, 50, 0);
 
 static const DECLARE_TLV_DB_SCALE(boost_tlv, 1600, 1600, 0);
+static const DECLARE_TLV_DB_SCALE(adc_boost_tlv, 2000, 2000, 0);
 static const char *chan_mix[] = {
 	"L R",
 	"L+R",
@@ -139,6 +152,8 @@ static const struct snd_kcontrol_new cs42l51_snd_controls[] = {
 	SOC_SINGLE("Zero Cross Switch", CS42L51_DAC_CTL, 0, 0, 0),
 	SOC_DOUBLE_TLV("Mic Boost Volume",
 			CS42L51_MIC_CTL, 0, 1, 1, 0, boost_tlv),
+	SOC_DOUBLE_TLV("ADC Boost Volume",
+		       CS42L51_MIC_CTL, 5, 6, 1, 0, adc_boost_tlv),
 	SOC_SINGLE_TLV("Bass Volume", CS42L51_TONE_CTL, 0, 0xf, 1, tone_tlv),
 	SOC_SINGLE_TLV("Treble Volume", CS42L51_TONE_CTL, 4, 0xf, 1, tone_tlv),
 	SOC_ENUM_EXT("PCM channel mixer",
@@ -195,7 +210,8 @@ static const struct snd_kcontrol_new cs42l51_adcr_mux_controls =
 	SOC_DAPM_ENUM("Route", cs42l51_adcr_mux_enum);
 
 static const struct snd_soc_dapm_widget cs42l51_dapm_widgets[] = {
-	SND_SOC_DAPM_MICBIAS("Mic Bias", CS42L51_MIC_POWER_CTL, 1, 1),
+	SND_SOC_DAPM_SUPPLY("Mic Bias", CS42L51_MIC_POWER_CTL, 1, 1, NULL,
+			    SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD),
 	SND_SOC_DAPM_PGA_E("Left PGA", CS42L51_POWER_CTL1, 3, 1, NULL, 0,
 		cs42l51_pdn_event, SND_SOC_DAPM_PRE_POST_PMD),
 	SND_SOC_DAPM_PGA_E("Right PGA", CS42L51_POWER_CTL1, 4, 1, NULL, 0,
@@ -329,6 +345,19 @@ static struct cs42l51_ratios slave_auto_ratios[] = {
 	{  256, CS42L51_DSM_MODE, 1 }, {  384, CS42L51_DSM_MODE, 1 },
 };
 
+/*
+ * Master mode mclk/fs ratios.
+ * Recommended configurations are SSM for 4-50khz and DSM for 50-100kHz ranges
+ * The table below provides support of following ratios:
+ * 128: SSM (%128) with div2 disabled
+ * 256: SSM (%128) with div2 enabled
+ * In both cases, if sampling rate is above 50kHz, SSM is overridden
+ * with DSM (%128) configuration
+ */
+static struct cs42l51_ratios master_ratios[] = {
+	{ 128, CS42L51_SSM_MODE, 0 }, { 256, CS42L51_SSM_MODE, 1 },
+};
+
 static int cs42l51_set_dai_sysclk(struct snd_soc_dai *codec_dai,
 		int clk_id, unsigned int freq, int dir)
 {
@@ -351,11 +380,13 @@ static int cs42l51_hw_params(struct snd_pcm_substream *substream,
 	unsigned int ratio;
 	struct cs42l51_ratios *ratios = NULL;
 	int nr_ratios = 0;
-	int intf_ctl, power_ctl, fmt;
+	int intf_ctl, power_ctl, fmt, mode;
 
 	switch (cs42l51->func) {
 	case MODE_MASTER:
-		return -EINVAL;
+		ratios = master_ratios;
+		nr_ratios = ARRAY_SIZE(master_ratios);
+		break;
 	case MODE_SLAVE:
 		ratios = slave_ratios;
 		nr_ratios = ARRAY_SIZE(slave_ratios);
@@ -391,7 +422,16 @@ static int cs42l51_hw_params(struct snd_pcm_substream *substream,
 	switch (cs42l51->func) {
 	case MODE_MASTER:
 		intf_ctl |= CS42L51_INTF_CTL_MASTER;
-		power_ctl |= CS42L51_MIC_POWER_CTL_SPEED(ratios[i].speed_mode);
+		mode = ratios[i].speed_mode;
+		/* Force DSM mode if sampling rate is above 50kHz */
+		if (rate > 50000)
+			mode = CS42L51_DSM_MODE;
+		power_ctl |= CS42L51_MIC_POWER_CTL_SPEED(mode);
+		/*
+		 * Auto detect mode is not applicable for master mode and has to
+		 * be disabled. Otherwise SPEED[1:0] bits will be ignored.
+		 */
+		power_ctl &= ~CS42L51_MIC_POWER_CTL_AUTO;
 		break;
 	case MODE_SLAVE:
 		power_ctl |= CS42L51_MIC_POWER_CTL_SPEED(ratios[i].speed_mode);
@@ -464,6 +504,13 @@ static int cs42l51_dai_mute(struct snd_soc_dai *dai, int mute)
 	return snd_soc_component_write(component, CS42L51_DAC_OUT_CTL, reg);
 }
 
+static int cs42l51_of_xlate_dai_id(struct snd_soc_component *component,
+				   struct device_node *endpoint)
+{
+	/* return dai id 0, whatever the endpoint index */
+	return 0;
+}
+
 static const struct snd_soc_dai_ops cs42l51_dai_ops = {
 	.hw_params      = cs42l51_hw_params,
 	.set_sysclk     = cs42l51_set_dai_sysclk,
@@ -526,13 +573,113 @@ static const struct snd_soc_component_driver soc_component_device_cs42l51 = {
 	.num_dapm_widgets	= ARRAY_SIZE(cs42l51_dapm_widgets),
 	.dapm_routes		= cs42l51_routes,
 	.num_dapm_routes	= ARRAY_SIZE(cs42l51_routes),
+	.of_xlate_dai_id	= cs42l51_of_xlate_dai_id,
 	.idle_bias_on		= 1,
 	.use_pmdown_time	= 1,
 	.endianness		= 1,
 	.non_legacy_dai_naming	= 1,
 };
 
+static bool cs42l51_writeable_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case CS42L51_POWER_CTL1:
+	case CS42L51_MIC_POWER_CTL:
+	case CS42L51_INTF_CTL:
+	case CS42L51_MIC_CTL:
+	case CS42L51_ADC_CTL:
+	case CS42L51_ADC_INPUT:
+	case CS42L51_DAC_OUT_CTL:
+	case CS42L51_DAC_CTL:
+	case CS42L51_ALC_PGA_CTL:
+	case CS42L51_ALC_PGB_CTL:
+	case CS42L51_ADCA_ATT:
+	case CS42L51_ADCB_ATT:
+	case CS42L51_ADCA_VOL:
+	case CS42L51_ADCB_VOL:
+	case CS42L51_PCMA_VOL:
+	case CS42L51_PCMB_VOL:
+	case CS42L51_BEEP_FREQ:
+	case CS42L51_BEEP_VOL:
+	case CS42L51_BEEP_CONF:
+	case CS42L51_TONE_CTL:
+	case CS42L51_AOUTA_VOL:
+	case CS42L51_AOUTB_VOL:
+	case CS42L51_PCM_MIXER:
+	case CS42L51_LIMIT_THRES_DIS:
+	case CS42L51_LIMIT_REL:
+	case CS42L51_LIMIT_ATT:
+	case CS42L51_ALC_EN:
+	case CS42L51_ALC_REL:
+	case CS42L51_ALC_THRES:
+	case CS42L51_NOISE_CONF:
+	case CS42L51_CHARGE_FREQ:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static bool cs42l51_volatile_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case CS42L51_STATUS:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static bool cs42l51_readable_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case CS42L51_CHIP_REV_ID:
+	case CS42L51_POWER_CTL1:
+	case CS42L51_MIC_POWER_CTL:
+	case CS42L51_INTF_CTL:
+	case CS42L51_MIC_CTL:
+	case CS42L51_ADC_CTL:
+	case CS42L51_ADC_INPUT:
+	case CS42L51_DAC_OUT_CTL:
+	case CS42L51_DAC_CTL:
+	case CS42L51_ALC_PGA_CTL:
+	case CS42L51_ALC_PGB_CTL:
+	case CS42L51_ADCA_ATT:
+	case CS42L51_ADCB_ATT:
+	case CS42L51_ADCA_VOL:
+	case CS42L51_ADCB_VOL:
+	case CS42L51_PCMA_VOL:
+	case CS42L51_PCMB_VOL:
+	case CS42L51_BEEP_FREQ:
+	case CS42L51_BEEP_VOL:
+	case CS42L51_BEEP_CONF:
+	case CS42L51_TONE_CTL:
+	case CS42L51_AOUTA_VOL:
+	case CS42L51_AOUTB_VOL:
+	case CS42L51_PCM_MIXER:
+	case CS42L51_LIMIT_THRES_DIS:
+	case CS42L51_LIMIT_REL:
+	case CS42L51_LIMIT_ATT:
+	case CS42L51_ALC_EN:
+	case CS42L51_ALC_REL:
+	case CS42L51_ALC_THRES:
+	case CS42L51_NOISE_CONF:
+	case CS42L51_STATUS:
+	case CS42L51_CHARGE_FREQ:
+		return true;
+	default:
+		return false;
+	}
+}
+
 const struct regmap_config cs42l51_regmap = {
+	.reg_bits = 8,
+	.reg_stride = 1,
+	.val_bits = 8,
+	.use_single_write = true,
+	.readable_reg = cs42l51_readable_reg,
+	.volatile_reg = cs42l51_volatile_reg,
+	.writeable_reg = cs42l51_writeable_reg,
 	.max_register = CS42L51_CHARGE_FREQ,
 	.cache_type = REGCACHE_RBTREE,
 };
@@ -542,7 +689,7 @@ int cs42l51_probe(struct device *dev, struct regmap *regmap)
 {
 	struct cs42l51_private *cs42l51;
 	unsigned int val;
-	int ret;
+	int ret, i;
 
 	if (IS_ERR(regmap))
 		return PTR_ERR(regmap);
@@ -553,6 +700,7 @@ int cs42l51_probe(struct device *dev, struct regmap *regmap)
 		return -ENOMEM;
 
 	dev_set_drvdata(dev, cs42l51);
+	cs42l51->regmap = regmap;
 
 	cs42l51->mclk_handle = devm_clk_get(dev, "MCLK");
 	if (IS_ERR(cs42l51->mclk_handle)) {
@@ -561,6 +709,34 @@ int cs42l51_probe(struct device *dev, struct regmap *regmap)
 		cs42l51->mclk_handle = NULL;
 	}
 
+	for (i = 0; i < ARRAY_SIZE(cs42l51->supplies); i++)
+		cs42l51->supplies[i].supply = cs42l51_supply_names[i];
+
+	ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(cs42l51->supplies),
+				      cs42l51->supplies);
+	if (ret != 0) {
+		dev_err(dev, "Failed to request supplies: %d\n", ret);
+		return ret;
+	}
+
+	ret = regulator_bulk_enable(ARRAY_SIZE(cs42l51->supplies),
+				    cs42l51->supplies);
+	if (ret != 0) {
+		dev_err(dev, "Failed to enable supplies: %d\n", ret);
+		return ret;
+	}
+
+	cs42l51->reset_gpio = devm_gpiod_get_optional(dev, "reset",
+						      GPIOD_OUT_LOW);
+	if (IS_ERR(cs42l51->reset_gpio))
+		return PTR_ERR(cs42l51->reset_gpio);
+
+	if (cs42l51->reset_gpio) {
+		dev_dbg(dev, "Release reset gpio\n");
+		gpiod_set_value_cansleep(cs42l51->reset_gpio, 0);
+		mdelay(2);
+	}
+
 	/* Verify that we have a CS42L51 */
 	ret = regmap_read(regmap, CS42L51_CHIP_REV_ID, &val);
 	if (ret < 0) {
@@ -579,11 +755,50 @@ int cs42l51_probe(struct device *dev, struct regmap *regmap)
 
 	ret = devm_snd_soc_register_component(dev,
 			&soc_component_device_cs42l51, &cs42l51_dai, 1);
+	if (ret < 0)
+		goto error;
+
+	return 0;
+
 error:
+	regulator_bulk_disable(ARRAY_SIZE(cs42l51->supplies),
+			       cs42l51->supplies);
 	return ret;
 }
 EXPORT_SYMBOL_GPL(cs42l51_probe);
 
+int cs42l51_remove(struct device *dev)
+{
+	struct cs42l51_private *cs42l51 = dev_get_drvdata(dev);
+
+	gpiod_set_value_cansleep(cs42l51->reset_gpio, 1);
+
+	return regulator_bulk_disable(ARRAY_SIZE(cs42l51->supplies),
+				      cs42l51->supplies);
+}
+EXPORT_SYMBOL_GPL(cs42l51_remove);
+
+int __maybe_unused cs42l51_suspend(struct device *dev)
+{
+	struct cs42l51_private *cs42l51 = dev_get_drvdata(dev);
+
+	regcache_cache_only(cs42l51->regmap, true);
+	regcache_mark_dirty(cs42l51->regmap);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(cs42l51_suspend);
+
+int __maybe_unused cs42l51_resume(struct device *dev)
+{
+	struct cs42l51_private *cs42l51 = dev_get_drvdata(dev);
+
+	regcache_cache_only(cs42l51->regmap, false);
+
+	return regcache_sync(cs42l51->regmap);
+}
+EXPORT_SYMBOL_GPL(cs42l51_resume);
+
 const struct of_device_id cs42l51_of_match[] = {
 	{ .compatible = "cirrus,cs42l51", },
 	{ }
diff --git a/sound/soc/codecs/cs42l51.h b/sound/soc/codecs/cs42l51.h
index 0ca805492ac4..79dee01137c8 100644
--- a/sound/soc/codecs/cs42l51.h
+++ b/sound/soc/codecs/cs42l51.h
@@ -22,6 +22,9 @@ struct device;
 
 extern const struct regmap_config cs42l51_regmap;
 int cs42l51_probe(struct device *dev, struct regmap *regmap);
+int cs42l51_remove(struct device *dev);
+int __maybe_unused cs42l51_suspend(struct device *dev);
+int __maybe_unused cs42l51_resume(struct device *dev);
 extern const struct of_device_id cs42l51_of_match[];
 
 #define CS42L51_CHIP_ID			0x1B
diff --git a/sound/soc/codecs/cs43130.c b/sound/soc/codecs/cs43130.c
index 3f7b255587e6..80d672710eae 100644
--- a/sound/soc/codecs/cs43130.c
+++ b/sound/soc/codecs/cs43130.c
@@ -2322,6 +2322,8 @@ static int cs43130_probe(struct snd_soc_component *component)
 			return ret;
 
 		cs43130->wq = create_singlethread_workqueue("cs43130_hp");
+		if (!cs43130->wq)
+			return -ENOMEM;
 		INIT_WORK(&cs43130->work, cs43130_imp_meas);
 	}
 
diff --git a/sound/soc/codecs/cs47l24.c b/sound/soc/codecs/cs47l24.c
index b16832a6a9af..eebbf02e1c39 100644
--- a/sound/soc/codecs/cs47l24.c
+++ b/sound/soc/codecs/cs47l24.c
@@ -75,7 +75,9 @@ static int cs47l24_adsp_power_ev(struct snd_soc_dapm_widget *w,
 
 	v = (v & ARIZONA_SYSCLK_FREQ_MASK) >> ARIZONA_SYSCLK_FREQ_SHIFT;
 
-	return wm_adsp2_early_event(w, kcontrol, event, v);
+	wm_adsp2_set_dspclk(w, v);
+
+	return wm_adsp_early_event(w, kcontrol, event);
 }
 
 static DECLARE_TLV_DB_SCALE(eq_tlv, -1200, 100, 0);
diff --git a/sound/soc/codecs/da7213.c b/sound/soc/codecs/da7213.c
index 92d006a5283e..425c11d63e49 100644
--- a/sound/soc/codecs/da7213.c
+++ b/sound/soc/codecs/da7213.c
@@ -1305,7 +1305,10 @@ static int da7213_set_dai_fmt(struct snd_soc_dai *codec_dai, unsigned int fmt)
 	/* By default only 64 BCLK per WCLK is supported */
 	dai_clk_mode |= DA7213_DAI_BCLKS_PER_WCLK_64;
 
-	snd_soc_component_write(component, DA7213_DAI_CLK_MODE, dai_clk_mode);
+	snd_soc_component_update_bits(component, DA7213_DAI_CLK_MODE,
+			    DA7213_DAI_BCLKS_PER_WCLK_MASK |
+			    DA7213_DAI_CLK_POL_MASK | DA7213_DAI_WCLK_POL_MASK,
+			    dai_clk_mode);
 	snd_soc_component_update_bits(component, DA7213_DAI_CTRL, DA7213_DAI_FORMAT_MASK,
 			    dai_ctrl);
 	snd_soc_component_write(component, DA7213_DAI_OFFSET, dai_offset);
diff --git a/sound/soc/codecs/da7213.h b/sound/soc/codecs/da7213.h
index 5a78dba1dcb5..9d31efc3cfe5 100644
--- a/sound/soc/codecs/da7213.h
+++ b/sound/soc/codecs/da7213.h
@@ -181,7 +181,9 @@
 #define DA7213_DAI_BCLKS_PER_WCLK_256				(0x3 << 0)
 #define DA7213_DAI_BCLKS_PER_WCLK_MASK				(0x3 << 0)
 #define DA7213_DAI_CLK_POL_INV					(0x1 << 2)
+#define DA7213_DAI_CLK_POL_MASK					(0x1 << 2)
 #define DA7213_DAI_WCLK_POL_INV					(0x1 << 3)
+#define DA7213_DAI_WCLK_POL_MASK				(0x1 << 3)
 #define DA7213_DAI_CLK_EN_MASK					(0x1 << 7)
 
 /* DA7213_DAI_CTRL = 0x29 */
diff --git a/sound/soc/codecs/da7219.c b/sound/soc/codecs/da7219.c
index 121a8190f93e..7d9d1f84eed8 100644
--- a/sound/soc/codecs/da7219.c
+++ b/sound/soc/codecs/da7219.c
@@ -797,6 +797,7 @@ static int da7219_dai_event(struct snd_soc_dapm_widget *w,
 {
 	struct snd_soc_component *component = snd_soc_dapm_to_component(w->dapm);
 	struct da7219_priv *da7219 = snd_soc_component_get_drvdata(component);
+	struct clk *bclk = da7219->dai_clks[DA7219_DAI_BCLK_IDX];
 	u8 pll_ctrl, pll_status;
 	int i = 0, ret;
 	bool srm_lock = false;
@@ -805,11 +806,11 @@ static int da7219_dai_event(struct snd_soc_dapm_widget *w,
 	case SND_SOC_DAPM_PRE_PMU:
 		if (da7219->master) {
 			/* Enable DAI clks for master mode */
-			if (da7219->dai_clks) {
-				ret = clk_prepare_enable(da7219->dai_clks);
+			if (bclk) {
+				ret = clk_prepare_enable(bclk);
 				if (ret) {
 					dev_err(component->dev,
-						"Failed to enable dai_clks\n");
+						"Failed to enable DAI clks\n");
 					return ret;
 				}
 			} else {
@@ -852,8 +853,8 @@ static int da7219_dai_event(struct snd_soc_dapm_widget *w,
 
 		/* Disable DAI clks if in master mode */
 		if (da7219->master) {
-			if (da7219->dai_clks)
-				clk_disable_unprepare(da7219->dai_clks);
+			if (bclk)
+				clk_disable_unprepare(bclk);
 			else
 				snd_soc_component_update_bits(component,
 							      DA7219_DAI_CLK_MODE,
@@ -1385,17 +1386,50 @@ static int da7219_set_dai_fmt(struct snd_soc_dai *codec_dai, unsigned int fmt)
 	return 0;
 }
 
+static int da7219_set_bclks_per_wclk(struct snd_soc_component *component,
+				     unsigned long factor)
+{
+	u8 bclks_per_wclk;
+
+	switch (factor) {
+	case 32:
+		bclks_per_wclk = DA7219_DAI_BCLKS_PER_WCLK_32;
+		break;
+	case 64:
+		bclks_per_wclk = DA7219_DAI_BCLKS_PER_WCLK_64;
+		break;
+	case 128:
+		bclks_per_wclk = DA7219_DAI_BCLKS_PER_WCLK_128;
+		break;
+	case 256:
+		bclks_per_wclk = DA7219_DAI_BCLKS_PER_WCLK_256;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	snd_soc_component_update_bits(component, DA7219_DAI_CLK_MODE,
+				      DA7219_DAI_BCLKS_PER_WCLK_MASK,
+				      bclks_per_wclk);
+
+	return 0;
+}
+
 static int da7219_set_dai_tdm_slot(struct snd_soc_dai *dai,
 				   unsigned int tx_mask, unsigned int rx_mask,
 				   int slots, int slot_width)
 {
 	struct snd_soc_component *component = dai->component;
 	struct da7219_priv *da7219 = snd_soc_component_get_drvdata(component);
+	struct clk *wclk = da7219->dai_clks[DA7219_DAI_WCLK_IDX];
+	struct clk *bclk = da7219->dai_clks[DA7219_DAI_BCLK_IDX];
 	unsigned int ch_mask;
-	u8 dai_bclks_per_wclk, slot_offset;
+	unsigned long sr, bclk_rate;
+	u8 slot_offset;
 	u16 offset;
 	__le16 dai_offset;
 	u32 frame_size;
+	int ret;
 
 	/* No channels enabled so disable TDM */
 	if (!tx_mask) {
@@ -1432,28 +1466,26 @@ static int da7219_set_dai_tdm_slot(struct snd_soc_dai *dai,
 	 */
 	if (da7219->master) {
 		frame_size = slots * slot_width;
-		switch (frame_size) {
-		case 32:
-			dai_bclks_per_wclk = DA7219_DAI_BCLKS_PER_WCLK_32;
-			break;
-		case 64:
-			dai_bclks_per_wclk = DA7219_DAI_BCLKS_PER_WCLK_64;
-			break;
-		case 128:
-			dai_bclks_per_wclk = DA7219_DAI_BCLKS_PER_WCLK_128;
-			break;
-		case 256:
-			dai_bclks_per_wclk = DA7219_DAI_BCLKS_PER_WCLK_256;
-			break;
-		default:
-			dev_err(component->dev, "Invalid frame size %d\n",
-				frame_size);
-			return -EINVAL;
-		}
 
-		snd_soc_component_update_bits(component, DA7219_DAI_CLK_MODE,
-				DA7219_DAI_BCLKS_PER_WCLK_MASK,
-				dai_bclks_per_wclk);
+		if (bclk) {
+			sr = clk_get_rate(wclk);
+			bclk_rate = sr * frame_size;
+			ret = clk_set_rate(bclk, bclk_rate);
+			if (ret) {
+				dev_err(component->dev,
+					"Failed to set TDM BCLK rate %lu: %d\n",
+					bclk_rate, ret);
+				return ret;
+			}
+		} else {
+			ret = da7219_set_bclks_per_wclk(component, frame_size);
+			if (ret) {
+				dev_err(component->dev,
+					"Failed to set TDM BCLKs per WCLK %d: %d\n",
+					frame_size, ret);
+				return ret;
+			}
+		}
 	}
 
 	dai_offset = cpu_to_le16(offset);
@@ -1471,44 +1503,12 @@ static int da7219_set_dai_tdm_slot(struct snd_soc_dai *dai,
 	return 0;
 }
 
-static int da7219_hw_params(struct snd_pcm_substream *substream,
-			    struct snd_pcm_hw_params *params,
-			    struct snd_soc_dai *dai)
+static int da7219_set_sr(struct snd_soc_component *component,
+			 unsigned long rate)
 {
-	struct snd_soc_component *component = dai->component;
-	struct da7219_priv *da7219 = snd_soc_component_get_drvdata(component);
-	u8 dai_ctrl = 0, dai_bclks_per_wclk = 0, fs;
-	unsigned int channels;
-	int word_len = params_width(params);
-	int frame_size;
+	u8 fs;
 
-	switch (word_len) {
-	case 16:
-		dai_ctrl |= DA7219_DAI_WORD_LENGTH_S16_LE;
-		break;
-	case 20:
-		dai_ctrl |= DA7219_DAI_WORD_LENGTH_S20_LE;
-		break;
-	case 24:
-		dai_ctrl |= DA7219_DAI_WORD_LENGTH_S24_LE;
-		break;
-	case 32:
-		dai_ctrl |= DA7219_DAI_WORD_LENGTH_S32_LE;
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	channels = params_channels(params);
-	if ((channels < 1) || (channels > DA7219_DAI_CH_NUM_MAX)) {
-		dev_err(component->dev,
-			"Invalid number of channels, only 1 to %d supported\n",
-			DA7219_DAI_CH_NUM_MAX);
-		return -EINVAL;
-	}
-	dai_ctrl |= channels << DA7219_DAI_CH_NUM_SHIFT;
-
-	switch (params_rate(params)) {
+	switch (rate) {
 	case 8000:
 		fs = DA7219_SR_8000;
 		break;
@@ -1546,28 +1546,118 @@ static int da7219_hw_params(struct snd_pcm_substream *substream,
 		return -EINVAL;
 	}
 
+	snd_soc_component_write(component, DA7219_SR, fs);
+
+	return 0;
+}
+
+static int da7219_hw_params(struct snd_pcm_substream *substream,
+			    struct snd_pcm_hw_params *params,
+			    struct snd_soc_dai *dai)
+{
+	struct snd_soc_component *component = dai->component;
+	struct da7219_priv *da7219 = snd_soc_component_get_drvdata(component);
+	struct clk *wclk = da7219->dai_clks[DA7219_DAI_WCLK_IDX];
+	struct clk *bclk = da7219->dai_clks[DA7219_DAI_BCLK_IDX];
+	u8 dai_ctrl = 0;
+	unsigned int channels;
+	unsigned long sr, bclk_rate;
+	int word_len = params_width(params);
+	int frame_size, ret;
+
+	switch (word_len) {
+	case 16:
+		dai_ctrl |= DA7219_DAI_WORD_LENGTH_S16_LE;
+		break;
+	case 20:
+		dai_ctrl |= DA7219_DAI_WORD_LENGTH_S20_LE;
+		break;
+	case 24:
+		dai_ctrl |= DA7219_DAI_WORD_LENGTH_S24_LE;
+		break;
+	case 32:
+		dai_ctrl |= DA7219_DAI_WORD_LENGTH_S32_LE;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	channels = params_channels(params);
+	if ((channels < 1) || (channels > DA7219_DAI_CH_NUM_MAX)) {
+		dev_err(component->dev,
+			"Invalid number of channels, only 1 to %d supported\n",
+			DA7219_DAI_CH_NUM_MAX);
+		return -EINVAL;
+	}
+	dai_ctrl |= channels << DA7219_DAI_CH_NUM_SHIFT;
+
+	sr = params_rate(params);
+	if (da7219->master && wclk) {
+		ret = clk_set_rate(wclk, sr);
+		if (ret) {
+			dev_err(component->dev,
+				"Failed to set WCLK SR %lu: %d\n", sr, ret);
+			return ret;
+		}
+	} else {
+		ret = da7219_set_sr(component, sr);
+		if (ret) {
+			dev_err(component->dev,
+				"Failed to set SR %lu: %d\n", sr, ret);
+			return ret;
+		}
+	}
+
 	/*
 	 * If we're master, then we have a limited set of BCLK rates we
 	 * support. For slave mode this isn't the case and the codec can detect
 	 * the BCLK rate automatically.
 	 */
 	if (da7219->master && !da7219->tdm_en) {
-		frame_size = word_len * 2;
-		if (frame_size <= 32)
-			dai_bclks_per_wclk = DA7219_DAI_BCLKS_PER_WCLK_32;
+		if ((word_len * DA7219_DAI_CH_NUM_MAX) <= 32)
+			frame_size = 32;
 		else
-			dai_bclks_per_wclk = DA7219_DAI_BCLKS_PER_WCLK_64;
+			frame_size = 64;
+
+		if (bclk) {
+			bclk_rate = frame_size * sr;
+			/*
+			 * Rounding the rate here avoids failure trying to set a
+			 * new rate on an already enabled bclk. In that
+			 * instance this will just set the same rate as is
+			 * currently in use, and so should continue without
+			 * problem, as long as the BCLK rate is suitable for the
+			 * desired frame size.
+			 */
+			bclk_rate = clk_round_rate(bclk, bclk_rate);
+			if ((bclk_rate / sr) < frame_size) {
+				dev_err(component->dev,
+					"BCLK rate mismatch against frame size");
+				return -EINVAL;
+			}
 
-		snd_soc_component_update_bits(component, DA7219_DAI_CLK_MODE,
-					      DA7219_DAI_BCLKS_PER_WCLK_MASK,
-					      dai_bclks_per_wclk);
+			ret = clk_set_rate(bclk, bclk_rate);
+			if (ret) {
+				dev_err(component->dev,
+					"Failed to set BCLK rate %lu: %d\n",
+					bclk_rate, ret);
+				return ret;
+			}
+		} else {
+			ret = da7219_set_bclks_per_wclk(component, frame_size);
+			if (ret) {
+				dev_err(component->dev,
+					"Failed to set BCLKs per WCLK %d: %d\n",
+					frame_size, ret);
+				return ret;
+			}
+		}
 	}
 
 	snd_soc_component_update_bits(component, DA7219_DAI_CTRL,
 			    DA7219_DAI_WORD_LENGTH_MASK |
 			    DA7219_DAI_CH_NUM_MASK,
 			    dai_ctrl);
-	snd_soc_component_write(component, DA7219_SR, fs);
 
 	return 0;
 }
@@ -1583,20 +1673,26 @@ static const struct snd_soc_dai_ops da7219_dai_ops = {
 #define DA7219_FORMATS (SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S20_3LE |\
 			SNDRV_PCM_FMTBIT_S24_LE | SNDRV_PCM_FMTBIT_S32_LE)
 
+#define DA7219_RATES (SNDRV_PCM_RATE_8000 | SNDRV_PCM_RATE_11025 |\
+		      SNDRV_PCM_RATE_16000 | SNDRV_PCM_RATE_22050 |\
+		      SNDRV_PCM_RATE_32000 | SNDRV_PCM_RATE_44100 |\
+		      SNDRV_PCM_RATE_48000 | SNDRV_PCM_RATE_88200 |\
+		      SNDRV_PCM_RATE_96000)
+
 static struct snd_soc_dai_driver da7219_dai = {
 	.name = "da7219-hifi",
 	.playback = {
 		.stream_name = "Playback",
 		.channels_min = 1,
 		.channels_max = DA7219_DAI_CH_NUM_MAX,
-		.rates = SNDRV_PCM_RATE_8000_96000,
+		.rates = DA7219_RATES,
 		.formats = DA7219_FORMATS,
 	},
 	.capture = {
 		.stream_name = "Capture",
 		.channels_min = 1,
 		.channels_max = DA7219_DAI_CH_NUM_MAX,
-		.rates = SNDRV_PCM_RATE_8000_96000,
+		.rates = DA7219_RATES,
 		.formats = DA7219_FORMATS,
 	},
 	.ops = &da7219_dai_ops,
@@ -1672,11 +1768,14 @@ static struct da7219_pdata *da7219_fw_to_pdata(struct snd_soc_component *compone
 
 	pdata->wakeup_source = device_property_read_bool(dev, "wakeup-source");
 
-	pdata->dai_clks_name = "da7219-dai-clks";
-	if (device_property_read_string(dev, "clock-output-names",
-					&pdata->dai_clks_name))
-		dev_warn(dev, "Using default clk name: %s\n",
-			 pdata->dai_clks_name);
+	pdata->dai_clk_names[DA7219_DAI_WCLK_IDX] = "da7219-dai-wclk";
+	pdata->dai_clk_names[DA7219_DAI_BCLK_IDX] = "da7219-dai-bclk";
+	if (device_property_read_string_array(dev, "clock-output-names",
+					      pdata->dai_clk_names,
+					      DA7219_DAI_NUM_CLKS) < 0)
+		dev_warn(dev, "Using default DAI clk names: %s, %s\n",
+			 pdata->dai_clk_names[DA7219_DAI_WCLK_IDX],
+			 pdata->dai_clk_names[DA7219_DAI_BCLK_IDX]);
 
 	if (device_property_read_u32(dev, "dlg,micbias-lvl", &of_val32) >= 0)
 		pdata->micbias_lvl = da7219_fw_micbias_lvl(dev, of_val32);
@@ -1793,12 +1892,16 @@ static int da7219_handle_supplies(struct snd_soc_component *component)
 }
 
 #ifdef CONFIG_COMMON_CLK
-static int da7219_dai_clks_prepare(struct clk_hw *hw)
+static int da7219_wclk_prepare(struct clk_hw *hw)
 {
 	struct da7219_priv *da7219 =
-		container_of(hw, struct da7219_priv, dai_clks_hw);
+		container_of(hw, struct da7219_priv,
+			     dai_clks_hw[DA7219_DAI_WCLK_IDX]);
 	struct snd_soc_component *component = da7219->component;
 
+	if (!da7219->master)
+		return -EINVAL;
+
 	snd_soc_component_update_bits(component, DA7219_DAI_CLK_MODE,
 				      DA7219_DAI_CLK_EN_MASK,
 				      DA7219_DAI_CLK_EN_MASK);
@@ -1806,33 +1909,42 @@ static int da7219_dai_clks_prepare(struct clk_hw *hw)
 	return 0;
 }
 
-static void da7219_dai_clks_unprepare(struct clk_hw *hw)
+static void da7219_wclk_unprepare(struct clk_hw *hw)
 {
 	struct da7219_priv *da7219 =
-		container_of(hw, struct da7219_priv, dai_clks_hw);
+		container_of(hw, struct da7219_priv,
+			     dai_clks_hw[DA7219_DAI_WCLK_IDX]);
 	struct snd_soc_component *component = da7219->component;
 
+	if (!da7219->master)
+		return;
+
 	snd_soc_component_update_bits(component, DA7219_DAI_CLK_MODE,
 				      DA7219_DAI_CLK_EN_MASK, 0);
 }
 
-static int da7219_dai_clks_is_prepared(struct clk_hw *hw)
+static int da7219_wclk_is_prepared(struct clk_hw *hw)
 {
 	struct da7219_priv *da7219 =
-		container_of(hw, struct da7219_priv, dai_clks_hw);
+		container_of(hw, struct da7219_priv,
+			     dai_clks_hw[DA7219_DAI_WCLK_IDX]);
 	struct snd_soc_component *component = da7219->component;
 	u8 clk_reg;
 
+	if (!da7219->master)
+		return -EINVAL;
+
 	clk_reg = snd_soc_component_read32(component, DA7219_DAI_CLK_MODE);
 
 	return !!(clk_reg & DA7219_DAI_CLK_EN_MASK);
 }
 
-static unsigned long da7219_dai_clks_recalc_rate(struct clk_hw *hw,
-						 unsigned long parent_rate)
+static unsigned long da7219_wclk_recalc_rate(struct clk_hw *hw,
+					     unsigned long parent_rate)
 {
 	struct da7219_priv *da7219 =
-		container_of(hw, struct da7219_priv, dai_clks_hw);
+		container_of(hw, struct da7219_priv,
+			     dai_clks_hw[DA7219_DAI_WCLK_IDX]);
 	struct snd_soc_component *component = da7219->component;
 	u8 fs = snd_soc_component_read32(component, DA7219_SR);
 
@@ -1864,11 +1976,148 @@ static unsigned long da7219_dai_clks_recalc_rate(struct clk_hw *hw,
 	}
 }
 
-static const struct clk_ops da7219_dai_clks_ops = {
-	.prepare = da7219_dai_clks_prepare,
-	.unprepare = da7219_dai_clks_unprepare,
-	.is_prepared = da7219_dai_clks_is_prepared,
-	.recalc_rate = da7219_dai_clks_recalc_rate,
+static long da7219_wclk_round_rate(struct clk_hw *hw, unsigned long rate,
+				   unsigned long *parent_rate)
+{
+	struct da7219_priv *da7219 =
+		container_of(hw, struct da7219_priv,
+			     dai_clks_hw[DA7219_DAI_WCLK_IDX]);
+
+	if (!da7219->master)
+		return -EINVAL;
+
+	if (rate < 11025)
+		return 8000;
+	else if (rate < 12000)
+		return 11025;
+	else if (rate < 16000)
+		return 12000;
+	else if (rate < 22050)
+		return 16000;
+	else if (rate < 24000)
+		return 22050;
+	else if (rate < 32000)
+		return 24000;
+	else if (rate < 44100)
+		return 32000;
+	else if (rate < 48000)
+		return 44100;
+	else if (rate < 88200)
+		return 48000;
+	else if (rate < 96000)
+		return 88200;
+	else
+		return 96000;
+}
+
+static int da7219_wclk_set_rate(struct clk_hw *hw, unsigned long rate,
+				unsigned long parent_rate)
+{
+	struct da7219_priv *da7219 =
+		container_of(hw, struct da7219_priv,
+			     dai_clks_hw[DA7219_DAI_WCLK_IDX]);
+	struct snd_soc_component *component = da7219->component;
+
+	if (!da7219->master)
+		return -EINVAL;
+
+	return da7219_set_sr(component, rate);
+}
+
+static unsigned long da7219_bclk_recalc_rate(struct clk_hw *hw,
+					     unsigned long parent_rate)
+{
+	struct da7219_priv *da7219 =
+		container_of(hw, struct da7219_priv,
+			     dai_clks_hw[DA7219_DAI_BCLK_IDX]);
+	struct snd_soc_component *component = da7219->component;
+	u8 bclks_per_wclk = snd_soc_component_read32(component,
+						     DA7219_DAI_CLK_MODE);
+
+	switch (bclks_per_wclk & DA7219_DAI_BCLKS_PER_WCLK_MASK) {
+	case DA7219_DAI_BCLKS_PER_WCLK_32:
+		return parent_rate * 32;
+	case DA7219_DAI_BCLKS_PER_WCLK_64:
+		return parent_rate * 64;
+	case DA7219_DAI_BCLKS_PER_WCLK_128:
+		return parent_rate * 128;
+	case DA7219_DAI_BCLKS_PER_WCLK_256:
+		return parent_rate * 256;
+	default:
+		return 0;
+	}
+}
+
+static unsigned long da7219_bclk_get_factor(unsigned long rate,
+					    unsigned long parent_rate)
+{
+	unsigned long factor;
+
+	factor = rate / parent_rate;
+	if (factor < 64)
+		return 32;
+	else if (factor < 128)
+		return 64;
+	else if (factor < 256)
+		return 128;
+	else
+		return 256;
+}
+
+static long da7219_bclk_round_rate(struct clk_hw *hw, unsigned long rate,
+				   unsigned long *parent_rate)
+{
+	struct da7219_priv *da7219 =
+		container_of(hw, struct da7219_priv,
+			     dai_clks_hw[DA7219_DAI_BCLK_IDX]);
+	unsigned long factor;
+
+	if (!*parent_rate || !da7219->master)
+		return -EINVAL;
+
+	/*
+	 * We don't allow changing the parent rate as some BCLK rates can be
+	 * derived from multiple parent WCLK rates (BCLK rates are set as a
+	 * multiplier of WCLK in HW). We just do some rounding down based on the
+	 * parent WCLK rate set and find the appropriate multiplier of BCLK to
+	 * get the rounded down BCLK value.
+	 */
+	factor = da7219_bclk_get_factor(rate, *parent_rate);
+
+	return *parent_rate * factor;
+}
+
+static int da7219_bclk_set_rate(struct clk_hw *hw, unsigned long rate,
+				unsigned long parent_rate)
+{
+	struct da7219_priv *da7219 =
+		container_of(hw, struct da7219_priv,
+			     dai_clks_hw[DA7219_DAI_BCLK_IDX]);
+	struct snd_soc_component *component = da7219->component;
+	unsigned long factor;
+
+	if (!da7219->master)
+		return -EINVAL;
+
+	factor = da7219_bclk_get_factor(rate, parent_rate);
+
+	return da7219_set_bclks_per_wclk(component, factor);
+}
+
+static const struct clk_ops da7219_dai_clk_ops[DA7219_DAI_NUM_CLKS] = {
+	[DA7219_DAI_WCLK_IDX] = {
+		.prepare = da7219_wclk_prepare,
+		.unprepare = da7219_wclk_unprepare,
+		.is_prepared = da7219_wclk_is_prepared,
+		.recalc_rate = da7219_wclk_recalc_rate,
+		.round_rate = da7219_wclk_round_rate,
+		.set_rate = da7219_wclk_set_rate,
+	},
+	[DA7219_DAI_BCLK_IDX] = {
+		.recalc_rate = da7219_bclk_recalc_rate,
+		.round_rate = da7219_bclk_round_rate,
+		.set_rate = da7219_bclk_set_rate,
+	},
 };
 
 static int da7219_register_dai_clks(struct snd_soc_component *component)
@@ -1876,47 +2125,81 @@ static int da7219_register_dai_clks(struct snd_soc_component *component)
 	struct device *dev = component->dev;
 	struct da7219_priv *da7219 = snd_soc_component_get_drvdata(component);
 	struct da7219_pdata *pdata = da7219->pdata;
-	struct clk_init_data init = {};
-	struct clk *dai_clks;
-	struct clk_lookup *dai_clks_lookup;
 	const char *parent_name;
+	int i, ret;
 
-	if (da7219->mclk) {
-		parent_name = __clk_get_name(da7219->mclk);
-		init.parent_names = &parent_name;
-		init.num_parents = 1;
-	} else {
-		init.parent_names = NULL;
-		init.num_parents = 0;
-	}
+	for (i = 0; i < DA7219_DAI_NUM_CLKS; ++i) {
+		struct clk_init_data init = {};
+		struct clk *dai_clk;
+		struct clk_lookup *dai_clk_lookup;
+		struct clk_hw *dai_clk_hw = &da7219->dai_clks_hw[i];
 
-	init.name = pdata->dai_clks_name;
-	init.ops = &da7219_dai_clks_ops;
-	init.flags = CLK_GET_RATE_NOCACHE;
-	da7219->dai_clks_hw.init = &init;
+		switch (i) {
+		case DA7219_DAI_WCLK_IDX:
+			/*
+			 * If we can, make MCLK the parent of WCLK to ensure
+			 * it's enabled as required.
+			 */
+			if (da7219->mclk) {
+				parent_name = __clk_get_name(da7219->mclk);
+				init.parent_names = &parent_name;
+				init.num_parents = 1;
+			} else {
+				init.parent_names = NULL;
+				init.num_parents = 0;
+			}
+			break;
+		case DA7219_DAI_BCLK_IDX:
+			/* Make WCLK the parent of BCLK */
+			parent_name = __clk_get_name(da7219->dai_clks[DA7219_DAI_WCLK_IDX]);
+			init.parent_names = &parent_name;
+			init.num_parents = 1;
+			break;
+		default:
+			dev_err(dev, "Invalid clock index\n");
+			ret = -EINVAL;
+			goto err;
+		}
 
-	dai_clks = devm_clk_register(dev, &da7219->dai_clks_hw);
-	if (IS_ERR(dai_clks)) {
-		dev_warn(dev, "Failed to register DAI clocks: %ld\n",
-			 PTR_ERR(dai_clks));
-		return PTR_ERR(dai_clks);
-	}
-	da7219->dai_clks = dai_clks;
+		init.name = pdata->dai_clk_names[i];
+		init.ops = &da7219_dai_clk_ops[i];
+		init.flags = CLK_GET_RATE_NOCACHE | CLK_SET_RATE_GATE;
+		dai_clk_hw->init = &init;
+
+		dai_clk = devm_clk_register(dev, dai_clk_hw);
+		if (IS_ERR(dai_clk)) {
+			dev_warn(dev, "Failed to register %s: %ld\n",
+				 init.name, PTR_ERR(dai_clk));
+			ret = PTR_ERR(dai_clk);
+			goto err;
+		}
+		da7219->dai_clks[i] = dai_clk;
 
-	/* If we're using DT, then register as provider accordingly */
-	if (dev->of_node) {
-		devm_of_clk_add_hw_provider(dev, of_clk_hw_simple_get,
-					    &da7219->dai_clks_hw);
-	} else {
-		dai_clks_lookup = clkdev_create(dai_clks, pdata->dai_clks_name,
-						"%s", dev_name(dev));
-		if (!dai_clks_lookup)
-			return -ENOMEM;
-		else
-			da7219->dai_clks_lookup = dai_clks_lookup;
+		/* If we're using DT, then register as provider accordingly */
+		if (dev->of_node) {
+			devm_of_clk_add_hw_provider(dev, of_clk_hw_simple_get,
+						    dai_clk_hw);
+		} else {
+			dai_clk_lookup = clkdev_create(dai_clk, init.name,
+						       "%s", dev_name(dev));
+			if (!dai_clk_lookup) {
+				ret = -ENOMEM;
+				goto err;
+			} else {
+				da7219->dai_clks_lookup[i] = dai_clk_lookup;
+			}
+		}
 	}
 
 	return 0;
+
+err:
+	do {
+		if (da7219->dai_clks_lookup[i])
+			clkdev_drop(da7219->dai_clks_lookup[i]);
+	} while (i-- > 0);
+
+	return ret;
 }
 #else
 static inline int da7219_register_dai_clks(struct snd_soc_component *component)
@@ -2080,12 +2363,17 @@ err_disable_reg:
 static void da7219_remove(struct snd_soc_component *component)
 {
 	struct da7219_priv *da7219 = snd_soc_component_get_drvdata(component);
+#ifdef CONFIG_COMMON_CLK
+	int i;
+#endif
 
 	da7219_aad_exit(component);
 
 #ifdef CONFIG_COMMON_CLK
-	if (da7219->dai_clks_lookup)
-		clkdev_drop(da7219->dai_clks_lookup);
+	for (i = DA7219_DAI_NUM_CLKS - 1; i >= 0; --i) {
+		if (da7219->dai_clks_lookup[i])
+			clkdev_drop(da7219->dai_clks_lookup[i]);
+	}
 #endif
 
 	/* Supplies */
diff --git a/sound/soc/codecs/da7219.h b/sound/soc/codecs/da7219.h
index 018819c631fb..f3b180bc986f 100644
--- a/sound/soc/codecs/da7219.h
+++ b/sound/soc/codecs/da7219.h
@@ -820,10 +820,10 @@ struct da7219_priv {
 	struct mutex pll_lock;
 
 #ifdef CONFIG_COMMON_CLK
-	struct clk_hw dai_clks_hw;
+	struct clk_hw dai_clks_hw[DA7219_DAI_NUM_CLKS];
 #endif
-	struct clk_lookup *dai_clks_lookup;
-	struct clk *dai_clks;
+	struct clk_lookup *dai_clks_lookup[DA7219_DAI_NUM_CLKS];
+	struct clk *dai_clks[DA7219_DAI_NUM_CLKS];
 
 	struct clk *mclk;
 	unsigned int mclk_rate;
diff --git a/sound/soc/codecs/es8316.c b/sound/soc/codecs/es8316.c
index 6d4a323f786b..ec2770b3f77d 100644
--- a/sound/soc/codecs/es8316.c
+++ b/sound/soc/codecs/es8316.c
@@ -43,6 +43,7 @@ struct es8316_priv {
 	unsigned int sysclk;
 	unsigned int allowed_rates[NR_SUPPORTED_MCLK_LRCK_RATIOS];
 	struct snd_pcm_hw_constraint_list sysclk_constraints;
+	bool jd_inverted;
 };
 
 /*
@@ -577,6 +578,9 @@ static irqreturn_t es8316_irq(int irq, void *data)
 	if (!es8316->jack)
 		goto out;
 
+	if (es8316->jd_inverted)
+		flags ^= ES8316_GPIO_FLAG_HP_NOT_INSERTED;
+
 	dev_dbg(comp->dev, "gpio flags %#04x\n", flags);
 	if (flags & ES8316_GPIO_FLAG_HP_NOT_INSERTED) {
 		/* Jack removed, or spurious IRQ? */
@@ -592,6 +596,8 @@ static irqreturn_t es8316_irq(int irq, void *data)
 		/* Jack inserted, determine type */
 		es8316_enable_micbias_for_mic_gnd_short_detect(comp);
 		regmap_read(es8316->regmap, ES8316_GPIO_FLAG, &flags);
+		if (es8316->jd_inverted)
+			flags ^= ES8316_GPIO_FLAG_HP_NOT_INSERTED;
 		dev_dbg(comp->dev, "gpio flags %#04x\n", flags);
 		if (flags & ES8316_GPIO_FLAG_HP_NOT_INSERTED) {
 			/* Jack unplugged underneath us */
@@ -633,6 +639,14 @@ static void es8316_enable_jack_detect(struct snd_soc_component *component,
 {
 	struct es8316_priv *es8316 = snd_soc_component_get_drvdata(component);
 
+	/*
+	 * Init es8316->jd_inverted here and not in the probe, as we cannot
+	 * guarantee that the bytchr-es8316 driver, which might set this
+	 * property, will probe before us.
+	 */
+	es8316->jd_inverted = device_property_read_bool(component->dev,
+							"everest,jack-detect-inverted");
+
 	mutex_lock(&es8316->lock);
 
 	es8316->jack = jack;
diff --git a/sound/soc/codecs/hdac_hda.c b/sound/soc/codecs/hdac_hda.c
index f889d94c8e3c..7d4940256914 100644
--- a/sound/soc/codecs/hdac_hda.c
+++ b/sound/soc/codecs/hdac_hda.c
@@ -328,6 +328,12 @@ static int hdac_hda_codec_probe(struct snd_soc_component *component)
 		dev_err(&hdev->dev, "failed to create hda codec %d\n", ret);
 		goto error_no_pm;
 	}
+	/*
+	 * Overwrite type to HDA_DEV_ASOC since it is a ASoC driver
+	 * hda_codec.c will check this flag to determine if unregister
+	 * device is needed.
+	 */
+	hdev->type = HDA_DEV_ASOC;
 
 	/*
 	 * snd_hda_codec_device_new decrements the usage count so call get pm
diff --git a/sound/soc/codecs/hdac_hdmi.c b/sound/soc/codecs/hdac_hdmi.c
index 5eeb0fe836a9..660e0587f399 100644
--- a/sound/soc/codecs/hdac_hdmi.c
+++ b/sound/soc/codecs/hdac_hdmi.c
@@ -455,24 +455,11 @@ static int hdac_hdmi_set_hw_params(struct snd_pcm_substream *substream,
 	struct snd_pcm_hw_params *hparams, struct snd_soc_dai *dai)
 {
 	struct hdac_hdmi_priv *hdmi = snd_soc_dai_get_drvdata(dai);
-	struct hdac_device *hdev = hdmi->hdev;
 	struct hdac_hdmi_dai_port_map *dai_map;
-	struct hdac_hdmi_port *port;
 	struct hdac_hdmi_pcm *pcm;
 	int format;
 
 	dai_map = &hdmi->dai_map[dai->id];
-	port = dai_map->port;
-
-	if (!port)
-		return -ENODEV;
-
-	if ((!port->eld.monitor_present) || (!port->eld.eld_valid)) {
-		dev_err(&hdev->dev,
-			"device is not configured for this pin:port%d:%d\n",
-					port->pin->nid, port->id);
-		return -ENODEV;
-	}
 
 	format = snd_hdac_calc_stream_format(params_rate(hparams),
 			params_channels(hparams), params_format(hparams),
@@ -1855,6 +1842,17 @@ static int hdmi_codec_probe(struct snd_soc_component *component)
 	hdmi->card = dapm->card->snd_card;
 
 	/*
+	 * Setup a device_link between card device and HDMI codec device.
+	 * The card device is the consumer and the HDMI codec device is
+	 * the supplier. With this setting, we can make sure that the audio
+	 * domain in display power will be always turned on before operating
+	 * on the HDMI audio codec registers.
+	 * Let's use the flag DL_FLAG_AUTOREMOVE_CONSUMER. This can make
+	 * sure the device link is freed when the machine driver is removed.
+	 */
+	device_link_add(component->card->dev, &hdev->dev, DL_FLAG_RPM_ACTIVE |
+			DL_FLAG_AUTOREMOVE_CONSUMER);
+	/*
 	 * hdac_device core already sets the state to active and calls
 	 * get_noresume. So enable runtime and set the device to suspend.
 	 */
diff --git a/sound/soc/codecs/hdmi-codec.c b/sound/soc/codecs/hdmi-codec.c
index 35df73e42cbc..39caf19abb0b 100644
--- a/sound/soc/codecs/hdmi-codec.c
+++ b/sound/soc/codecs/hdmi-codec.c
@@ -439,8 +439,12 @@ static int hdmi_codec_startup(struct snd_pcm_substream *substream,
 		if (!ret) {
 			ret = snd_pcm_hw_constraint_eld(substream->runtime,
 							hcp->eld);
-			if (ret)
+			if (ret) {
+				mutex_lock(&hcp->current_stream_lock);
+				hcp->current_stream = NULL;
+				mutex_unlock(&hcp->current_stream_lock);
 				return ret;
+			}
 		}
 		/* Select chmap supported */
 		hdmi_codec_eld_chmap(hcp);
@@ -492,10 +496,6 @@ static int hdmi_codec_hw_params(struct snd_pcm_substream *substream,
 		return ret;
 	}
 
-	ret = hdmi_codec_new_stream(substream, dai);
-	if (ret)
-		return ret;
-
 	hdmi_audio_infoframe_init(&hp.cea);
 	hp.cea.channels = params_channels(params);
 	hp.cea.coding_type = HDMI_AUDIO_CODING_TYPE_STREAM;
@@ -757,7 +757,7 @@ static int hdmi_codec_probe(struct platform_device *pdev)
 	dev_dbg(dev, "%s()\n", __func__);
 
 	if (!hcd) {
-		dev_err(dev, "%s: No plalform data\n", __func__);
+		dev_err(dev, "%s: No platform data\n", __func__);
 		return -EINVAL;
 	}
 
diff --git a/sound/soc/codecs/lochnagar-sc.c b/sound/soc/codecs/lochnagar-sc.c
new file mode 100644
index 000000000000..3209b39e46af
--- /dev/null
+++ b/sound/soc/codecs/lochnagar-sc.c
@@ -0,0 +1,266 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Lochnagar sound card driver
+//
+// Copyright (c) 2017-2019 Cirrus Logic, Inc. and
+//                         Cirrus Logic International Semiconductor Ltd.
+//
+// Author: Charles Keepax <ckeepax@opensource.cirrus.com>
+//         Piotr Stankiewicz <piotrs@opensource.cirrus.com>
+
+#include <linux/clk.h>
+#include <linux/module.h>
+#include <sound/soc.h>
+
+#include <linux/mfd/lochnagar.h>
+#include <linux/mfd/lochnagar1_regs.h>
+#include <linux/mfd/lochnagar2_regs.h>
+
+struct lochnagar_sc_priv {
+	struct clk *mclk;
+};
+
+static const struct snd_soc_dapm_widget lochnagar_sc_widgets[] = {
+	SND_SOC_DAPM_LINE("Line Jack", NULL),
+	SND_SOC_DAPM_LINE("USB Audio", NULL),
+};
+
+static const struct snd_soc_dapm_route lochnagar_sc_routes[] = {
+	{ "Line Jack", NULL, "AIF1 Playback" },
+	{ "AIF1 Capture", NULL, "Line Jack" },
+
+	{ "USB Audio", NULL, "USB1 Playback" },
+	{ "USB Audio", NULL, "USB2 Playback" },
+	{ "USB1 Capture", NULL, "USB Audio" },
+	{ "USB2 Capture", NULL, "USB Audio" },
+};
+
+static const unsigned int lochnagar_sc_chan_vals[] = {
+	4, 8,
+};
+
+static const struct snd_pcm_hw_constraint_list lochnagar_sc_chan_constraint = {
+	.count = ARRAY_SIZE(lochnagar_sc_chan_vals),
+	.list = lochnagar_sc_chan_vals,
+};
+
+static const unsigned int lochnagar_sc_rate_vals[] = {
+	8000, 16000, 24000, 32000, 48000, 96000, 192000,
+	22050, 44100, 88200, 176400,
+};
+
+static const struct snd_pcm_hw_constraint_list lochnagar_sc_rate_constraint = {
+	.count = ARRAY_SIZE(lochnagar_sc_rate_vals),
+	.list = lochnagar_sc_rate_vals,
+};
+
+static int lochnagar_sc_hw_rule_rate(struct snd_pcm_hw_params *params,
+				     struct snd_pcm_hw_rule *rule)
+{
+	struct snd_interval range = {
+		.min = 8000,
+		.max = 24576000 / hw_param_interval(params, rule->deps[0])->max,
+	};
+
+	return snd_interval_refine(hw_param_interval(params, rule->var),
+				   &range);
+}
+
+static int lochnagar_sc_startup(struct snd_pcm_substream *substream,
+				struct snd_soc_dai *dai)
+{
+	struct snd_soc_component *comp = dai->component;
+	struct lochnagar_sc_priv *priv = snd_soc_component_get_drvdata(comp);
+	int ret;
+
+	ret = snd_pcm_hw_constraint_list(substream->runtime, 0,
+					 SNDRV_PCM_HW_PARAM_RATE,
+					 &lochnagar_sc_rate_constraint);
+	if (ret)
+		return ret;
+
+	return snd_pcm_hw_rule_add(substream->runtime, 0,
+				   SNDRV_PCM_HW_PARAM_RATE,
+				   lochnagar_sc_hw_rule_rate, priv,
+				   SNDRV_PCM_HW_PARAM_FRAME_BITS, -1);
+}
+
+static int lochnagar_sc_line_startup(struct snd_pcm_substream *substream,
+				     struct snd_soc_dai *dai)
+{
+	struct snd_soc_component *comp = dai->component;
+	struct lochnagar_sc_priv *priv = snd_soc_component_get_drvdata(comp);
+	int ret;
+
+	ret = clk_prepare_enable(priv->mclk);
+	if (ret < 0) {
+		dev_err(dai->dev, "Failed to enable MCLK: %d\n", ret);
+		return ret;
+	}
+
+	ret = lochnagar_sc_startup(substream, dai);
+	if (ret)
+		return ret;
+
+	return snd_pcm_hw_constraint_list(substream->runtime, 0,
+					  SNDRV_PCM_HW_PARAM_CHANNELS,
+					  &lochnagar_sc_chan_constraint);
+}
+
+static void lochnagar_sc_line_shutdown(struct snd_pcm_substream *substream,
+				       struct snd_soc_dai *dai)
+{
+	struct snd_soc_component *comp = dai->component;
+	struct lochnagar_sc_priv *priv = snd_soc_component_get_drvdata(comp);
+
+	clk_disable_unprepare(priv->mclk);
+}
+
+static int lochnagar_sc_check_fmt(struct snd_soc_dai *dai, unsigned int fmt,
+				  unsigned int tar)
+{
+	tar |= SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_NB_NF;
+
+	if ((fmt & ~SND_SOC_DAIFMT_CLOCK_MASK) != tar)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int lochnagar_sc_set_line_fmt(struct snd_soc_dai *dai, unsigned int fmt)
+{
+	return lochnagar_sc_check_fmt(dai, fmt, SND_SOC_DAIFMT_CBS_CFS);
+}
+
+static int lochnagar_sc_set_usb_fmt(struct snd_soc_dai *dai, unsigned int fmt)
+{
+	return lochnagar_sc_check_fmt(dai, fmt, SND_SOC_DAIFMT_CBM_CFM);
+}
+
+static const struct snd_soc_dai_ops lochnagar_sc_line_ops = {
+	.startup = lochnagar_sc_line_startup,
+	.shutdown = lochnagar_sc_line_shutdown,
+	.set_fmt = lochnagar_sc_set_line_fmt,
+};
+
+static const struct snd_soc_dai_ops lochnagar_sc_usb_ops = {
+	.startup = lochnagar_sc_startup,
+	.set_fmt = lochnagar_sc_set_usb_fmt,
+};
+
+static struct snd_soc_dai_driver lochnagar_sc_dai[] = {
+	{
+		.name = "lochnagar-line",
+		.playback = {
+			.stream_name = "AIF1 Playback",
+			.channels_min = 4,
+			.channels_max = 8,
+			.rates = SNDRV_PCM_RATE_KNOT,
+			.formats = SNDRV_PCM_FMTBIT_S32_LE,
+		},
+		.capture = {
+			.stream_name = "AIF1 Capture",
+			.channels_min = 4,
+			.channels_max = 8,
+			.rates = SNDRV_PCM_RATE_KNOT,
+			.formats = SNDRV_PCM_FMTBIT_S32_LE,
+		},
+		.ops = &lochnagar_sc_line_ops,
+		.symmetric_rates = true,
+		.symmetric_samplebits = true,
+	},
+	{
+		.name = "lochnagar-usb1",
+		.playback = {
+			.stream_name = "USB1 Playback",
+			.channels_min = 1,
+			.channels_max = 8,
+			.rates = SNDRV_PCM_RATE_KNOT,
+			.formats = SNDRV_PCM_FMTBIT_S32_LE,
+		},
+		.capture = {
+			.stream_name = "USB1 Capture",
+			.channels_min = 1,
+			.channels_max = 8,
+			.rates = SNDRV_PCM_RATE_KNOT,
+			.formats = SNDRV_PCM_FMTBIT_S32_LE,
+		},
+		.ops = &lochnagar_sc_usb_ops,
+		.symmetric_rates = true,
+		.symmetric_samplebits = true,
+	},
+	{
+		.name = "lochnagar-usb2",
+		.playback = {
+			.stream_name = "USB2 Playback",
+			.channels_min = 1,
+			.channels_max = 8,
+			.rates = SNDRV_PCM_RATE_KNOT,
+			.formats = SNDRV_PCM_FMTBIT_S32_LE,
+		},
+		.capture = {
+			.stream_name = "USB2 Capture",
+			.channels_min = 1,
+			.channels_max = 8,
+			.rates = SNDRV_PCM_RATE_KNOT,
+			.formats = SNDRV_PCM_FMTBIT_S32_LE,
+		},
+		.ops = &lochnagar_sc_usb_ops,
+		.symmetric_rates = true,
+		.symmetric_samplebits = true,
+	},
+};
+
+static const struct snd_soc_component_driver lochnagar_sc_driver = {
+	.non_legacy_dai_naming = 1,
+
+	.dapm_widgets = lochnagar_sc_widgets,
+	.num_dapm_widgets = ARRAY_SIZE(lochnagar_sc_widgets),
+	.dapm_routes = lochnagar_sc_routes,
+	.num_dapm_routes = ARRAY_SIZE(lochnagar_sc_routes),
+};
+
+static int lochnagar_sc_probe(struct platform_device *pdev)
+{
+	struct lochnagar_sc_priv *priv;
+	int ret;
+
+	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->mclk = devm_clk_get(&pdev->dev, "mclk");
+	if (IS_ERR(priv->mclk)) {
+		ret = PTR_ERR(priv->mclk);
+		dev_err(&pdev->dev, "Failed to get MCLK: %d\n", ret);
+		return ret;
+	}
+
+	platform_set_drvdata(pdev, priv);
+
+	return devm_snd_soc_register_component(&pdev->dev,
+					       &lochnagar_sc_driver,
+					       lochnagar_sc_dai,
+					       ARRAY_SIZE(lochnagar_sc_dai));
+}
+
+static const struct of_device_id lochnagar_of_match[] = {
+	{ .compatible = "cirrus,lochnagar2-soundcard" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, lochnagar_of_match);
+
+static struct platform_driver lochnagar_sc_codec_driver = {
+	.driver = {
+		.name = "lochnagar-soundcard",
+		.of_match_table = of_match_ptr(lochnagar_of_match),
+	},
+
+	.probe = lochnagar_sc_probe,
+};
+module_platform_driver(lochnagar_sc_codec_driver);
+
+MODULE_DESCRIPTION("ASoC Lochnagar Sound Card Driver");
+MODULE_AUTHOR("Piotr Stankiewicz <piotrs@opensource.cirrus.com>");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:lochnagar-soundcard");
diff --git a/sound/soc/codecs/max98090.c b/sound/soc/codecs/max98090.c
index 30c242c38d99..7619ea31ab50 100644
--- a/sound/soc/codecs/max98090.c
+++ b/sound/soc/codecs/max98090.c
@@ -1194,14 +1194,14 @@ static const struct snd_soc_dapm_widget max98090_dapm_widgets[] = {
 		&max98090_right_rcv_mixer_controls[0],
 		ARRAY_SIZE(max98090_right_rcv_mixer_controls)),
 
-	SND_SOC_DAPM_MUX("LINMOD Mux", M98090_REG_LOUTR_MIXER,
-		M98090_LINMOD_SHIFT, 0, &max98090_linmod_mux),
+	SND_SOC_DAPM_MUX("LINMOD Mux", SND_SOC_NOPM, 0, 0,
+		&max98090_linmod_mux),
 
-	SND_SOC_DAPM_MUX("MIXHPLSEL Mux", M98090_REG_HP_CONTROL,
-		M98090_MIXHPLSEL_SHIFT, 0, &max98090_mixhplsel_mux),
+	SND_SOC_DAPM_MUX("MIXHPLSEL Mux", SND_SOC_NOPM, 0, 0,
+		&max98090_mixhplsel_mux),
 
-	SND_SOC_DAPM_MUX("MIXHPRSEL Mux", M98090_REG_HP_CONTROL,
-		M98090_MIXHPRSEL_SHIFT, 0, &max98090_mixhprsel_mux),
+	SND_SOC_DAPM_MUX("MIXHPRSEL Mux", SND_SOC_NOPM, 0, 0,
+		&max98090_mixhprsel_mux),
 
 	SND_SOC_DAPM_PGA("HP Left Out", M98090_REG_OUTPUT_ENABLE,
 		M98090_HPLEN_SHIFT, 0, NULL, 0),
diff --git a/sound/soc/codecs/max98357a.c b/sound/soc/codecs/max98357a.c
index d469576b5a7b..d037a3e4d323 100644
--- a/sound/soc/codecs/max98357a.c
+++ b/sound/soc/codecs/max98357a.c
@@ -97,7 +97,10 @@ static struct snd_soc_dai_driver max98357a_dai_driver = {
 					SNDRV_PCM_FMTBIT_S32,
 		.rates		= SNDRV_PCM_RATE_8000 |
 					SNDRV_PCM_RATE_16000 |
+					SNDRV_PCM_RATE_32000 |
+					SNDRV_PCM_RATE_44100 |
 					SNDRV_PCM_RATE_48000 |
+					SNDRV_PCM_RATE_88200 |
 					SNDRV_PCM_RATE_96000,
 		.rate_min	= 8000,
 		.rate_max	= 96000,
diff --git a/sound/soc/codecs/mt6358.c b/sound/soc/codecs/mt6358.c
index d4c4fee6d3d9..50b3fc5457ea 100644
--- a/sound/soc/codecs/mt6358.c
+++ b/sound/soc/codecs/mt6358.c
@@ -320,32 +320,6 @@ enum {
 #define DL_GAIN_N_40DB_REG (DL_GAIN_N_40DB << 7 | DL_GAIN_N_40DB)
 #define DL_GAIN_REG_MASK 0x0f9f
 
-static void lo_store_gain(struct mt6358_priv *priv)
-{
-	unsigned int reg;
-	unsigned int gain_l, gain_r;
-
-	regmap_read(priv->regmap, MT6358_ZCD_CON1, &reg);
-	gain_l = (reg >> RG_AUDLOLGAIN_SFT) & RG_AUDLOLGAIN_MASK;
-	gain_r = (reg >> RG_AUDLORGAIN_SFT) & RG_AUDLORGAIN_MASK;
-
-	priv->ana_gain[AUDIO_ANALOG_VOLUME_LINEOUTL] = gain_l;
-	priv->ana_gain[AUDIO_ANALOG_VOLUME_LINEOUTR] = gain_r;
-}
-
-static void hp_store_gain(struct mt6358_priv *priv)
-{
-	unsigned int reg;
-	unsigned int gain_l, gain_r;
-
-	regmap_read(priv->regmap, MT6358_ZCD_CON2, &reg);
-	gain_l = (reg >> RG_AUDHPLGAIN_SFT) & RG_AUDHPLGAIN_MASK;
-	gain_r = (reg >> RG_AUDHPRGAIN_SFT) & RG_AUDHPRGAIN_MASK;
-
-	priv->ana_gain[AUDIO_ANALOG_VOLUME_HPOUTL] = gain_l;
-	priv->ana_gain[AUDIO_ANALOG_VOLUME_HPOUTR] = gain_r;
-}
-
 static void hp_zcd_disable(struct mt6358_priv *priv)
 {
 	regmap_write(priv->regmap, MT6358_ZCD_CON0, 0x0000);
@@ -405,10 +379,9 @@ static bool is_valid_hp_pga_idx(int reg_idx)
 	       reg_idx == DL_GAIN_N_40DB;
 }
 
-static void headset_volume_ramp(struct mt6358_priv *priv,
-				int from, int to)
+static void headset_volume_ramp(struct mt6358_priv *priv, int from, int to)
 {
-	int offset = 0, count = 1, reg_idx;
+	int offset = 0, count = 0, reg_idx;
 
 	if (!is_valid_hp_pga_idx(from) || !is_valid_hp_pga_idx(to))
 		dev_warn(priv->dev, "%s(), volume index is not valid, from %d, to %d\n",
@@ -422,7 +395,7 @@ static void headset_volume_ramp(struct mt6358_priv *priv,
 	else
 		offset = from - to;
 
-	while (offset > 0) {
+	while (offset >= 0) {
 		if (to > from)
 			reg_idx = from + count;
 		else
@@ -440,25 +413,76 @@ static void headset_volume_ramp(struct mt6358_priv *priv,
 	}
 }
 
+static int mt6358_put_volsw(struct snd_kcontrol *kcontrol,
+			    struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_component *component =
+			snd_soc_kcontrol_component(kcontrol);
+	struct mt6358_priv *priv = snd_soc_component_get_drvdata(component);
+	struct soc_mixer_control *mc =
+			(struct soc_mixer_control *)kcontrol->private_value;
+	unsigned int reg;
+	int ret;
+
+	ret = snd_soc_put_volsw(kcontrol, ucontrol);
+	if (ret < 0)
+		return ret;
+
+	switch (mc->reg) {
+	case MT6358_ZCD_CON2:
+		regmap_read(priv->regmap, MT6358_ZCD_CON2, &reg);
+		priv->ana_gain[AUDIO_ANALOG_VOLUME_HPOUTL] =
+			(reg >> RG_AUDHPLGAIN_SFT) & RG_AUDHPLGAIN_MASK;
+		priv->ana_gain[AUDIO_ANALOG_VOLUME_HPOUTR] =
+			(reg >> RG_AUDHPRGAIN_SFT) & RG_AUDHPRGAIN_MASK;
+		break;
+	case MT6358_ZCD_CON1:
+		regmap_read(priv->regmap, MT6358_ZCD_CON1, &reg);
+		priv->ana_gain[AUDIO_ANALOG_VOLUME_LINEOUTL] =
+			(reg >> RG_AUDLOLGAIN_SFT) & RG_AUDLOLGAIN_MASK;
+		priv->ana_gain[AUDIO_ANALOG_VOLUME_LINEOUTR] =
+			(reg >> RG_AUDLORGAIN_SFT) & RG_AUDLORGAIN_MASK;
+		break;
+	case MT6358_ZCD_CON3:
+		regmap_read(priv->regmap, MT6358_ZCD_CON3, &reg);
+		priv->ana_gain[AUDIO_ANALOG_VOLUME_HSOUTL] =
+			(reg >> RG_AUDHSGAIN_SFT) & RG_AUDHSGAIN_MASK;
+		priv->ana_gain[AUDIO_ANALOG_VOLUME_HSOUTR] =
+			(reg >> RG_AUDHSGAIN_SFT) & RG_AUDHSGAIN_MASK;
+		break;
+	case MT6358_AUDENC_ANA_CON0:
+	case MT6358_AUDENC_ANA_CON1:
+		regmap_read(priv->regmap, MT6358_AUDENC_ANA_CON0, &reg);
+		priv->ana_gain[AUDIO_ANALOG_VOLUME_MICAMP1] =
+			(reg >> RG_AUDPREAMPLGAIN_SFT) & RG_AUDPREAMPLGAIN_MASK;
+		regmap_read(priv->regmap, MT6358_AUDENC_ANA_CON1, &reg);
+		priv->ana_gain[AUDIO_ANALOG_VOLUME_MICAMP2] =
+			(reg >> RG_AUDPREAMPRGAIN_SFT) & RG_AUDPREAMPRGAIN_MASK;
+		break;
+	}
+
+	return ret;
+}
+
 static const DECLARE_TLV_DB_SCALE(playback_tlv, -1000, 100, 0);
 static const DECLARE_TLV_DB_SCALE(pga_tlv, 0, 600, 0);
 
 static const struct snd_kcontrol_new mt6358_snd_controls[] = {
 	/* dl pga gain */
-	SOC_DOUBLE_TLV("Headphone Volume",
-		       MT6358_ZCD_CON2, 0, 7, 0x12, 1,
-		       playback_tlv),
-	SOC_DOUBLE_TLV("Lineout Volume",
-		       MT6358_ZCD_CON1, 0, 7, 0x12, 1,
-		       playback_tlv),
-	SOC_SINGLE_TLV("Handset Volume",
-		       MT6358_ZCD_CON3, 0, 0x12, 1,
-		       playback_tlv),
+	SOC_DOUBLE_EXT_TLV("Headphone Volume",
+			   MT6358_ZCD_CON2, 0, 7, 0x12, 1,
+			   snd_soc_get_volsw, mt6358_put_volsw, playback_tlv),
+	SOC_DOUBLE_EXT_TLV("Lineout Volume",
+			   MT6358_ZCD_CON1, 0, 7, 0x12, 1,
+			   snd_soc_get_volsw, mt6358_put_volsw, playback_tlv),
+	SOC_SINGLE_EXT_TLV("Handset Volume",
+			   MT6358_ZCD_CON3, 0, 0x12, 1,
+			   snd_soc_get_volsw, mt6358_put_volsw, playback_tlv),
 	/* ul pga gain */
-	SOC_DOUBLE_R_TLV("PGA Volume",
-			 MT6358_AUDENC_ANA_CON0, MT6358_AUDENC_ANA_CON1,
-			 8, 4, 0,
-			 pga_tlv),
+	SOC_DOUBLE_R_EXT_TLV("PGA Volume",
+			     MT6358_AUDENC_ANA_CON0, MT6358_AUDENC_ANA_CON1,
+			     8, 4, 0,
+			     snd_soc_get_volsw, mt6358_put_volsw, pga_tlv),
 };
 
 /* MUX */
@@ -832,8 +856,6 @@ static int mtk_hp_enable(struct mt6358_priv *priv)
 	/* Reduce ESD resistance of AU_REFN */
 	regmap_write(priv->regmap, MT6358_AUDDEC_ANA_CON2, 0x4000);
 
-	/* save target gain to restore after hardware open complete */
-	hp_store_gain(priv);
 	/* Set HPR/HPL gain as minimum (~ -40dB) */
 	regmap_write(priv->regmap, MT6358_ZCD_CON2, DL_GAIN_N_40DB_REG);
 
@@ -1043,8 +1065,6 @@ static int mtk_hp_spk_enable(struct mt6358_priv *priv)
 	/* Reduce ESD resistance of AU_REFN */
 	regmap_write(priv->regmap, MT6358_AUDDEC_ANA_CON2, 0x4000);
 
-	/* save target gain to restore after hardware open complete */
-	hp_store_gain(priv);
 	/* Set HPR/HPL gain to -10dB */
 	regmap_write(priv->regmap, MT6358_ZCD_CON2, DL_GAIN_N_10DB_REG);
 
@@ -1104,7 +1124,6 @@ static int mtk_hp_spk_enable(struct mt6358_priv *priv)
 	hp_main_output_ramp(priv, true);
 
 	/* Set LO gain as minimum (~ -40dB) */
-	lo_store_gain(priv);
 	regmap_write(priv->regmap, MT6358_ZCD_CON1, DL_GAIN_N_40DB_REG);
 	/* apply volume setting */
 	headset_volume_ramp(priv,
@@ -1740,6 +1759,21 @@ static void mt6358_dmic_disable(struct mt6358_priv *priv)
 	regmap_write(priv->regmap, MT6358_AUDENC_ANA_CON9, 0x0000);
 }
 
+static void mt6358_restore_pga(struct mt6358_priv *priv)
+{
+	unsigned int gain_l, gain_r;
+
+	gain_l = priv->ana_gain[AUDIO_ANALOG_VOLUME_MICAMP1];
+	gain_r = priv->ana_gain[AUDIO_ANALOG_VOLUME_MICAMP2];
+
+	regmap_update_bits(priv->regmap, MT6358_AUDENC_ANA_CON0,
+			   RG_AUDPREAMPLGAIN_MASK_SFT,
+			   gain_l << RG_AUDPREAMPLGAIN_SFT);
+	regmap_update_bits(priv->regmap, MT6358_AUDENC_ANA_CON1,
+			   RG_AUDPREAMPRGAIN_MASK_SFT,
+			   gain_r << RG_AUDPREAMPRGAIN_SFT);
+}
+
 static int mt_mic_type_event(struct snd_soc_dapm_widget *w,
 			     struct snd_kcontrol *kcontrol,
 			     int event)
@@ -1764,6 +1798,7 @@ static int mt_mic_type_event(struct snd_soc_dapm_widget *w,
 			mt6358_amic_enable(priv);
 			break;
 		}
+		mt6358_restore_pga(priv);
 
 		break;
 	case SND_SOC_DAPM_POST_PMD:
diff --git a/sound/soc/codecs/nau8810.c b/sound/soc/codecs/nau8810.c
index 645aa0794123..dd82c65cfa7f 100644
--- a/sound/soc/codecs/nau8810.c
+++ b/sound/soc/codecs/nau8810.c
@@ -493,7 +493,7 @@ static int nau8810_set_sysclk(struct snd_soc_dai *dai,
 	return 0;
 }
 
-static int nau88l0_calc_pll(unsigned int pll_in,
+static int nau8810_calc_pll(unsigned int pll_in,
 	unsigned int fs, struct nau8810_pll *pll_param)
 {
 	u64 f2, f2_max, pll_ratio;
@@ -505,7 +505,8 @@ static int nau88l0_calc_pll(unsigned int pll_in,
 	f2_max = 0;
 	scal_sel = ARRAY_SIZE(nau8810_mclk_scaler);
 	for (i = 0; i < ARRAY_SIZE(nau8810_mclk_scaler); i++) {
-		f2 = 256 * fs * 4 * nau8810_mclk_scaler[i] / 10;
+		f2 = 256ULL * fs * 4 * nau8810_mclk_scaler[i];
+		f2 = div_u64(f2, 10);
 		if (f2 > NAU_PLL_FREQ_MIN && f2 < NAU_PLL_FREQ_MAX &&
 			f2_max < f2) {
 			f2_max = f2;
@@ -542,7 +543,7 @@ static int nau8810_set_pll(struct snd_soc_dai *codec_dai, int pll_id,
 	int ret, fs;
 
 	fs = freq_out / 256;
-	ret = nau88l0_calc_pll(freq_in, fs, pll_param);
+	ret = nau8810_calc_pll(freq_in, fs, pll_param);
 	if (ret < 0) {
 		dev_err(nau8810->dev, "Unsupported input clock %d\n", freq_in);
 		return ret;
@@ -667,6 +668,24 @@ static int nau8810_pcm_hw_params(struct snd_pcm_substream *substream,
 	struct snd_soc_component *component = dai->component;
 	struct nau8810 *nau8810 = snd_soc_component_get_drvdata(component);
 	int val_len = 0, val_rate = 0, ret = 0;
+	unsigned int ctrl_val, bclk_fs, bclk_div;
+
+	/* Select BCLK configuration if the codec as master. */
+	regmap_read(nau8810->regmap, NAU8810_REG_CLOCK, &ctrl_val);
+	if (ctrl_val & NAU8810_CLKIO_MASTER) {
+		/* get the bclk and fs ratio */
+		bclk_fs = snd_soc_params_to_bclk(params) / params_rate(params);
+		if (bclk_fs <= 32)
+			bclk_div = NAU8810_BCLKDIV_8;
+		else if (bclk_fs <= 64)
+			bclk_div = NAU8810_BCLKDIV_4;
+		else if (bclk_fs <= 128)
+			bclk_div = NAU8810_BCLKDIV_2;
+		else
+			return -EINVAL;
+		regmap_update_bits(nau8810->regmap, NAU8810_REG_CLOCK,
+			NAU8810_BCLKSEL_MASK, bclk_div);
+	}
 
 	switch (params_width(params)) {
 	case 16:
diff --git a/sound/soc/codecs/pcm3168a.c b/sound/soc/codecs/pcm3168a.c
index 08d3fe192e65..e0d5839fe1a7 100644
--- a/sound/soc/codecs/pcm3168a.c
+++ b/sound/soc/codecs/pcm3168a.c
@@ -457,13 +457,16 @@ static int pcm3168a_hw_params(struct snd_pcm_substream *substream,
 	if (chan > 2) {
 		switch (fmt) {
 		case PCM3168A_FMT_I2S:
+		case PCM3168A_FMT_DSP_A:
 			fmt = PCM3168A_FMT_I2S_TDM;
 			break;
 		case PCM3168A_FMT_LEFT_J:
+		case PCM3168A_FMT_DSP_B:
 			fmt = PCM3168A_FMT_LEFT_J_TDM;
 			break;
 		default:
-			dev_err(component->dev, "TDM is supported under I2S/Left_J only\n");
+			dev_err(component->dev,
+				"TDM is supported under DSP/I2S/Left_J only\n");
 			return -EINVAL;
 		}
 	}
@@ -526,6 +529,8 @@ static int pcm3168a_startup(struct snd_pcm_substream *substream,
 		break;
 	case PCM3168A_FMT_LEFT_J:
 	case PCM3168A_FMT_I2S:
+	case PCM3168A_FMT_DSP_A:
+	case PCM3168A_FMT_DSP_B:
 		sample_min  = 24;
 		channel_max = channel_maxs[tx];
 		break;
diff --git a/sound/soc/codecs/rt5645.c b/sound/soc/codecs/rt5645.c
index 9a0751978090..cd45d41df4ec 100644
--- a/sound/soc/codecs/rt5645.c
+++ b/sound/soc/codecs/rt5645.c
@@ -3419,6 +3419,9 @@ static int rt5645_probe(struct snd_soc_component *component)
 		RT5645_HWEQ_NUM, sizeof(struct rt5645_eq_param_s),
 		GFP_KERNEL);
 
+	if (!rt5645->eq_param)
+		return -ENOMEM;
+
 	return 0;
 }
 
@@ -3631,6 +3634,11 @@ static const struct rt5645_platform_data jd_mode3_platform_data = {
 	.jd_mode = 3,
 };
 
+static const struct rt5645_platform_data lattepanda_board_platform_data = {
+	.jd_mode = 2,
+	.inv_jd1_1 = true
+};
+
 static const struct dmi_system_id dmi_platform_data[] = {
 	{
 		.ident = "Chrome Buddy",
@@ -3728,6 +3736,15 @@ static const struct dmi_system_id dmi_platform_data[] = {
 		},
 		.driver_data = (void *)&intel_braswell_platform_data,
 	},
+	{
+		.ident = "LattePanda board",
+		.matches = {
+		  DMI_EXACT_MATCH(DMI_BOARD_VENDOR, "AMI Corporation"),
+		  DMI_EXACT_MATCH(DMI_BOARD_NAME, "Cherry Trail CR"),
+		  DMI_EXACT_MATCH(DMI_BOARD_VERSION, "Default string"),
+		},
+		.driver_data = (void *)&lattepanda_board_platform_data,
+	},
 	{ }
 };
 
diff --git a/sound/soc/codecs/rt5651.c b/sound/soc/codecs/rt5651.c
index 29b2d60076b0..cb8252ff31cb 100644
--- a/sound/soc/codecs/rt5651.c
+++ b/sound/soc/codecs/rt5651.c
@@ -1645,7 +1645,10 @@ static bool rt5651_jack_inserted(struct snd_soc_component *component)
 		break;
 	}
 
-	return val == 0;
+	if (rt5651->jd_active_high)
+		return val != 0;
+	else
+		return val == 0;
 }
 
 /* Jack detect and button-press timings */
@@ -1868,20 +1871,47 @@ static void rt5651_enable_jack_detect(struct snd_soc_component *component,
 	case RT5651_JD1_1:
 		snd_soc_component_update_bits(component, RT5651_JD_CTRL2,
 			RT5651_JD_TRG_SEL_MASK, RT5651_JD_TRG_SEL_JD1_1);
-		snd_soc_component_update_bits(component, RT5651_IRQ_CTRL1,
-			RT5651_JD1_1_IRQ_EN, RT5651_JD1_1_IRQ_EN);
+		/* active-low is normal, set inv flag for active-high */
+		if (rt5651->jd_active_high)
+			snd_soc_component_update_bits(component,
+				RT5651_IRQ_CTRL1,
+				RT5651_JD1_1_IRQ_EN | RT5651_JD1_1_INV,
+				RT5651_JD1_1_IRQ_EN | RT5651_JD1_1_INV);
+		else
+			snd_soc_component_update_bits(component,
+				RT5651_IRQ_CTRL1,
+				RT5651_JD1_1_IRQ_EN | RT5651_JD1_1_INV,
+				RT5651_JD1_1_IRQ_EN);
 		break;
 	case RT5651_JD1_2:
 		snd_soc_component_update_bits(component, RT5651_JD_CTRL2,
 			RT5651_JD_TRG_SEL_MASK, RT5651_JD_TRG_SEL_JD1_2);
-		snd_soc_component_update_bits(component, RT5651_IRQ_CTRL1,
-			RT5651_JD1_2_IRQ_EN, RT5651_JD1_2_IRQ_EN);
+		/* active-low is normal, set inv flag for active-high */
+		if (rt5651->jd_active_high)
+			snd_soc_component_update_bits(component,
+				RT5651_IRQ_CTRL1,
+				RT5651_JD1_2_IRQ_EN | RT5651_JD1_2_INV,
+				RT5651_JD1_2_IRQ_EN | RT5651_JD1_2_INV);
+		else
+			snd_soc_component_update_bits(component,
+				RT5651_IRQ_CTRL1,
+				RT5651_JD1_2_IRQ_EN | RT5651_JD1_2_INV,
+				RT5651_JD1_2_IRQ_EN);
 		break;
 	case RT5651_JD2:
 		snd_soc_component_update_bits(component, RT5651_JD_CTRL2,
 			RT5651_JD_TRG_SEL_MASK, RT5651_JD_TRG_SEL_JD2);
-		snd_soc_component_update_bits(component, RT5651_IRQ_CTRL1,
-			RT5651_JD2_IRQ_EN, RT5651_JD2_IRQ_EN);
+		/* active-low is normal, set inv flag for active-high */
+		if (rt5651->jd_active_high)
+			snd_soc_component_update_bits(component,
+				RT5651_IRQ_CTRL1,
+				RT5651_JD2_IRQ_EN | RT5651_JD2_INV,
+				RT5651_JD2_IRQ_EN | RT5651_JD2_INV);
+		else
+			snd_soc_component_update_bits(component,
+				RT5651_IRQ_CTRL1,
+				RT5651_JD2_IRQ_EN | RT5651_JD2_INV,
+				RT5651_JD2_IRQ_EN);
 		break;
 	default:
 		dev_err(component->dev, "Currently only JD1_1 / JD1_2 / JD2 are supported\n");
@@ -1986,6 +2016,9 @@ static void rt5651_apply_properties(struct snd_soc_component *component)
 				     "realtek,jack-detect-source", &val) == 0)
 		rt5651->jd_src = val;
 
+	if (device_property_read_bool(component->dev, "realtek,jack-detect-not-inverted"))
+		rt5651->jd_active_high = true;
+
 	/*
 	 * Testing on various boards has shown that good defaults for the OVCD
 	 * threshold and scale-factor are 2000µA and 0.75. For an effective
diff --git a/sound/soc/codecs/rt5651.h b/sound/soc/codecs/rt5651.h
index 41fcb8b5eb40..05b0f6f8b95d 100644
--- a/sound/soc/codecs/rt5651.h
+++ b/sound/soc/codecs/rt5651.h
@@ -2083,6 +2083,7 @@ struct rt5651_priv {
 	int release_count;
 	int poll_count;
 	unsigned int jd_src;
+	bool jd_active_high;
 	unsigned int ovcd_th;
 	unsigned int ovcd_sf;
 
diff --git a/sound/soc/codecs/rt5677-spi.c b/sound/soc/codecs/rt5677-spi.c
index 84501c2020c7..84b6bd8b50e1 100644
--- a/sound/soc/codecs/rt5677-spi.c
+++ b/sound/soc/codecs/rt5677-spi.c
@@ -25,6 +25,7 @@
 #include <linux/sysfs.h>
 #include <linux/clk.h>
 #include <linux/firmware.h>
+#include <linux/acpi.h>
 
 #include "rt5677-spi.h"
 
@@ -57,13 +58,15 @@ static DEFINE_MUTEX(spi_mutex);
  * RT5677_SPI_READ/WRITE_32:	Transfer 4 bytes
  * RT5677_SPI_READ/WRITE_BURST:	Transfer any multiples of 8 bytes
  *
- * For example, reading 260 bytes at 0x60030002 uses the following commands:
- * 0x60030002 RT5677_SPI_READ_16	2 bytes
+ * Note:
+ * 16 Bit writes and reads are restricted to the address range
+ * 0x18020000 ~ 0x18021000
+ *
+ * For example, reading 256 bytes at 0x60030004 uses the following commands:
  * 0x60030004 RT5677_SPI_READ_32	4 bytes
  * 0x60030008 RT5677_SPI_READ_BURST	240 bytes
  * 0x600300F8 RT5677_SPI_READ_BURST	8 bytes
  * 0x60030100 RT5677_SPI_READ_32	4 bytes
- * 0x60030104 RT5677_SPI_READ_16	2 bytes
  *
  * Input:
  * @read: true for read commands; false for write commands
@@ -78,15 +81,13 @@ static u8 rt5677_spi_select_cmd(bool read, u32 align, u32 remain, u32 *len)
 {
 	u8 cmd;
 
-	if (align == 2 || align == 6 || remain == 2) {
-		cmd = RT5677_SPI_READ_16;
-		*len = 2;
-	} else if (align == 4 || remain <= 6) {
+	if (align == 4 || remain <= 4) {
 		cmd = RT5677_SPI_READ_32;
 		*len = 4;
 	} else {
 		cmd = RT5677_SPI_READ_BURST;
-		*len = min_t(u32, remain & ~7, RT5677_SPI_BURST_LEN);
+		*len = (((remain - 1) >> 3) + 1) << 3;
+		*len = min_t(u32, *len, RT5677_SPI_BURST_LEN);
 	}
 	return read ? cmd : cmd + 1;
 }
@@ -107,7 +108,7 @@ static void rt5677_spi_reverse(u8 *dst, u32 dstlen, const u8 *src, u32 srclen)
 	}
 }
 
-/* Read DSP address space using SPI. addr and len have to be 2-byte aligned. */
+/* Read DSP address space using SPI. addr and len have to be 4-byte aligned. */
 int rt5677_spi_read(u32 addr, void *rxbuf, size_t len)
 {
 	u32 offset;
@@ -123,7 +124,7 @@ int rt5677_spi_read(u32 addr, void *rxbuf, size_t len)
 	if (!g_spi)
 		return -ENODEV;
 
-	if ((addr & 1) || (len & 1)) {
+	if ((addr & 3) || (len & 3)) {
 		dev_err(&g_spi->dev, "Bad read align 0x%x(%zu)\n", addr, len);
 		return -EACCES;
 	}
@@ -158,13 +159,13 @@ int rt5677_spi_read(u32 addr, void *rxbuf, size_t len)
 }
 EXPORT_SYMBOL_GPL(rt5677_spi_read);
 
-/* Write DSP address space using SPI. addr has to be 2-byte aligned.
- * If len is not 2-byte aligned, an extra byte of zero is written at the end
+/* Write DSP address space using SPI. addr has to be 4-byte aligned.
+ * If len is not 4-byte aligned, then extra zeros are written at the end
  * as padding.
  */
 int rt5677_spi_write(u32 addr, const void *txbuf, size_t len)
 {
-	u32 offset, len_with_pad = len;
+	u32 offset;
 	int status = 0;
 	struct spi_transfer t;
 	struct spi_message m;
@@ -177,22 +178,19 @@ int rt5677_spi_write(u32 addr, const void *txbuf, size_t len)
 	if (!g_spi)
 		return -ENODEV;
 
-	if (addr & 1) {
+	if (addr & 3) {
 		dev_err(&g_spi->dev, "Bad write align 0x%x(%zu)\n", addr, len);
 		return -EACCES;
 	}
 
-	if (len & 1)
-		len_with_pad = len + 1;
-
 	memset(&t, 0, sizeof(t));
 	t.tx_buf = buf;
 	t.speed_hz = RT5677_SPI_FREQ;
 	spi_message_init_with_transfers(&m, &t, 1);
 
-	for (offset = 0; offset < len_with_pad;) {
+	for (offset = 0; offset < len;) {
 		spi_cmd = rt5677_spi_select_cmd(false, (addr + offset) & 7,
-				len_with_pad - offset, &t.len);
+				len - offset, &t.len);
 
 		/* Construct SPI message header */
 		buf[0] = spi_cmd;
@@ -226,9 +224,16 @@ static int rt5677_spi_probe(struct spi_device *spi)
 	return 0;
 }
 
+static const struct acpi_device_id rt5677_spi_acpi_id[] = {
+	{ "RT5677AA", 0 },
+	{ }
+};
+MODULE_DEVICE_TABLE(acpi, rt5677_spi_acpi_id);
+
 static struct spi_driver rt5677_spi_driver = {
 	.driver = {
 		.name = "rt5677",
+		.acpi_match_table = ACPI_PTR(rt5677_spi_acpi_id),
 	},
 	.probe = rt5677_spi_probe,
 };
diff --git a/sound/soc/codecs/rt5682.c b/sound/soc/codecs/rt5682.c
index 86a7fa31c294..505fb3d7b1c5 100644
--- a/sound/soc/codecs/rt5682.c
+++ b/sound/soc/codecs/rt5682.c
@@ -2588,6 +2588,7 @@ static int rt5682_i2c_probe(struct i2c_client *i2c,
 
 	rt5682_reset(rt5682->regmap);
 
+	mutex_init(&rt5682->calibrate_mutex);
 	rt5682_calibrate(rt5682);
 
 	ret = regmap_multi_reg_write(rt5682->regmap, patch_list,
@@ -2654,7 +2655,6 @@ static int rt5682_i2c_probe(struct i2c_client *i2c,
 	INIT_DELAYED_WORK(&rt5682->jd_check_work,
 				rt5682_jd_check_handler);
 
-	mutex_init(&rt5682->calibrate_mutex);
 
 	if (i2c->irq) {
 		ret = devm_request_threaded_irq(&i2c->dev, i2c->irq, NULL,
diff --git a/sound/soc/codecs/simple-amplifier.c b/sound/soc/codecs/simple-amplifier.c
index c07e8a80b4b7..351aa55c384e 100644
--- a/sound/soc/codecs/simple-amplifier.c
+++ b/sound/soc/codecs/simple-amplifier.c
@@ -89,7 +89,8 @@ static int simple_amp_probe(struct platform_device *pdev)
 		return -ENOMEM;
 	platform_set_drvdata(pdev, priv);
 
-	priv->gpiod_enable = devm_gpiod_get(dev, "enable", GPIOD_OUT_LOW);
+	priv->gpiod_enable = devm_gpiod_get_optional(dev, "enable",
+						     GPIOD_OUT_LOW);
 	if (IS_ERR(priv->gpiod_enable)) {
 		err = PTR_ERR(priv->gpiod_enable);
 		if (err != -EPROBE_DEFER)
diff --git a/sound/soc/codecs/sirf-audio-codec.c b/sound/soc/codecs/sirf-audio-codec.c
index e424499a8450..e0af21050078 100644
--- a/sound/soc/codecs/sirf-audio-codec.c
+++ b/sound/soc/codecs/sirf-audio-codec.c
@@ -461,9 +461,6 @@ static int sirf_audio_codec_driver_probe(struct platform_device *pdev)
 	struct sirf_audio_codec *sirf_audio_codec;
 	void __iomem *base;
 	struct resource *mem_res;
-	const struct of_device_id *match;
-
-	match = of_match_node(sirf_audio_codec_of_match, pdev->dev.of_node);
 
 	sirf_audio_codec = devm_kzalloc(&pdev->dev,
 		sizeof(struct sirf_audio_codec), GFP_KERNEL);
diff --git a/sound/soc/codecs/tlv320aic31xx.c b/sound/soc/codecs/tlv320aic31xx.c
index c544a1e35f5e..9b37e98da0db 100644
--- a/sound/soc/codecs/tlv320aic31xx.c
+++ b/sound/soc/codecs/tlv320aic31xx.c
@@ -25,6 +25,7 @@
 #include <linux/of_gpio.h>
 #include <linux/slab.h>
 #include <sound/core.h>
+#include <sound/jack.h>
 #include <sound/pcm.h>
 #include <sound/pcm_params.h>
 #include <sound/soc.h>
@@ -89,6 +90,7 @@ static bool aic31xx_volatile(struct device *dev, unsigned int reg)
 	case AIC31XX_INTRADCFLAG: /* Sticky interrupt flags */
 	case AIC31XX_INTRDACFLAG2:
 	case AIC31XX_INTRADCFLAG2:
+	case AIC31XX_HSDETECT:
 		return true;
 	}
 	return false;
@@ -163,6 +165,7 @@ struct aic31xx_priv {
 	struct aic31xx_pdata pdata;
 	struct regulator_bulk_data supplies[AIC31XX_NUM_SUPPLIES];
 	struct aic31xx_disable_nb disable_nb[AIC31XX_NUM_SUPPLIES];
+	struct snd_soc_jack *jack;
 	unsigned int sysclk;
 	u8 p_div;
 	int rate_div_line;
@@ -1261,6 +1264,20 @@ static int aic31xx_set_bias_level(struct snd_soc_component *component,
 	return 0;
 }
 
+static int aic31xx_set_jack(struct snd_soc_component *component,
+			    struct snd_soc_jack *jack, void *data)
+{
+	struct aic31xx_priv *aic31xx = snd_soc_component_get_drvdata(component);
+
+	aic31xx->jack = jack;
+
+	/* Enable/Disable jack detection */
+	regmap_write(aic31xx->regmap, AIC31XX_HSDETECT,
+		     jack ? AIC31XX_HSD_ENABLE : 0);
+
+	return 0;
+}
+
 static int aic31xx_codec_probe(struct snd_soc_component *component)
 {
 	struct aic31xx_priv *aic31xx = snd_soc_component_get_drvdata(component);
@@ -1301,6 +1318,7 @@ static int aic31xx_codec_probe(struct snd_soc_component *component)
 
 static const struct snd_soc_component_driver soc_codec_driver_aic31xx = {
 	.probe			= aic31xx_codec_probe,
+	.set_jack		= aic31xx_set_jack,
 	.set_bias_level		= aic31xx_set_bias_level,
 	.controls		= common31xx_snd_controls,
 	.num_controls		= ARRAY_SIZE(common31xx_snd_controls),
@@ -1405,8 +1423,47 @@ static irqreturn_t aic31xx_irq(int irq, void *data)
 		dev_err(dev, "Short circuit on Left output is detected\n");
 	if (value & AIC31XX_HPRSCDETECT)
 		dev_err(dev, "Short circuit on Right output is detected\n");
+	if (value & (AIC31XX_HSPLUG | AIC31XX_BUTTONPRESS)) {
+		unsigned int val;
+		int status = 0;
+
+		ret = regmap_read(aic31xx->regmap, AIC31XX_INTRDACFLAG2,
+				  &val);
+		if (ret) {
+			dev_err(dev, "Failed to read interrupt mask: %d\n",
+				ret);
+			goto exit;
+		}
+
+		if (val & AIC31XX_BUTTONPRESS)
+			status |= SND_JACK_BTN_0;
+
+		ret = regmap_read(aic31xx->regmap, AIC31XX_HSDETECT, &val);
+		if (ret) {
+			dev_err(dev, "Failed to read headset type: %d\n", ret);
+			goto exit;
+		}
+
+		switch ((val & AIC31XX_HSD_TYPE_MASK) >>
+			AIC31XX_HSD_TYPE_SHIFT) {
+		case AIC31XX_HSD_HP:
+			status |= SND_JACK_HEADPHONE;
+			break;
+		case AIC31XX_HSD_HS:
+			status |= SND_JACK_HEADSET;
+			break;
+		default:
+			break;
+		}
+
+		if (aic31xx->jack)
+			snd_soc_jack_report(aic31xx->jack, status,
+					    AIC31XX_JACK_MASK);
+	}
 	if (value & ~(AIC31XX_HPLSCDETECT |
-		      AIC31XX_HPRSCDETECT))
+		      AIC31XX_HPRSCDETECT |
+		      AIC31XX_HSPLUG |
+		      AIC31XX_BUTTONPRESS))
 		dev_err(dev, "Unknown DAC interrupt flags: 0x%08x\n", value);
 
 read_overflow:
@@ -1518,6 +1575,8 @@ static int aic31xx_i2c_probe(struct i2c_client *i2c,
 				   AIC31XX_GPIO1_FUNC_SHIFT);
 
 		regmap_write(aic31xx->regmap, AIC31XX_INT1CTRL,
+			     AIC31XX_HSPLUGDET |
+			     AIC31XX_BUTTONPRESSDET |
 			     AIC31XX_SC |
 			     AIC31XX_ENGINE);
 
diff --git a/sound/soc/codecs/tlv320aic31xx.h b/sound/soc/codecs/tlv320aic31xx.h
index 2636f2c6bc79..cb024955c978 100644
--- a/sound/soc/codecs/tlv320aic31xx.h
+++ b/sound/soc/codecs/tlv320aic31xx.h
@@ -20,6 +20,10 @@
 #define AIC31XX_MINIDSP_BIT		BIT(2)
 #define DAC31XX_BIT			BIT(3)
 
+#define AIC31XX_JACK_MASK (SND_JACK_HEADPHONE | \
+			   SND_JACK_HEADSET | \
+			   SND_JACK_BTN_0)
+
 enum aic31xx_type {
 	AIC3100	= 0,
 	AIC3110 = AIC31XX_STEREO_CLASS_D_BIT,
@@ -220,6 +224,14 @@ struct aic31xx_pdata {
 /* AIC31XX_DACMUTE */
 #define AIC31XX_DACMUTE_MASK		GENMASK(3, 2)
 
+/* AIC31XX_HSDETECT */
+#define AIC31XX_HSD_ENABLE		BIT(7)
+#define AIC31XX_HSD_TYPE_MASK		GENMASK(6, 5)
+#define AIC31XX_HSD_TYPE_SHIFT		5
+#define AIC31XX_HSD_NONE		0x00
+#define AIC31XX_HSD_HP			0x01
+#define AIC31XX_HSD_HS			0x03
+
 /* AIC31XX_MICBIAS */
 #define AIC31XX_MICBIAS_MASK		GENMASK(1, 0)
 #define AIC31XX_MICBIAS_SHIFT		0
diff --git a/sound/soc/codecs/tlv320aic32x4-clk.c b/sound/soc/codecs/tlv320aic32x4-clk.c
new file mode 100644
index 000000000000..156c153c12ab
--- /dev/null
+++ b/sound/soc/codecs/tlv320aic32x4-clk.c
@@ -0,0 +1,483 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * Clock Tree for the Texas Instruments TLV320AIC32x4
+ *
+ * Copyright 2019 Annaliese McDermond
+ *
+ * Author: Annaliese McDermond <nh6z@nh6z.net>
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/clkdev.h>
+#include <linux/regmap.h>
+#include <linux/device.h>
+
+#include "tlv320aic32x4.h"
+
+#define to_clk_aic32x4(_hw) container_of(_hw, struct clk_aic32x4, hw)
+struct clk_aic32x4 {
+	struct clk_hw hw;
+	struct device *dev;
+	struct regmap *regmap;
+	unsigned int reg;
+};
+
+/*
+ * struct clk_aic32x4_pll_muldiv - Multiplier/divider settings
+ * @p:		Divider
+ * @r:		first multiplier
+ * @j:		integer part of second multiplier
+ * @d:		decimal part of second multiplier
+ */
+struct clk_aic32x4_pll_muldiv {
+	u8 p;
+	u16 r;
+	u8 j;
+	u16 d;
+};
+
+struct aic32x4_clkdesc {
+	const char *name;
+	const char * const *parent_names;
+	unsigned int num_parents;
+	const struct clk_ops *ops;
+	unsigned int reg;
+};
+
+static int clk_aic32x4_pll_prepare(struct clk_hw *hw)
+{
+	struct clk_aic32x4 *pll = to_clk_aic32x4(hw);
+
+	return regmap_update_bits(pll->regmap, AIC32X4_PLLPR,
+				AIC32X4_PLLEN, AIC32X4_PLLEN);
+}
+
+static void clk_aic32x4_pll_unprepare(struct clk_hw *hw)
+{
+	struct clk_aic32x4 *pll = to_clk_aic32x4(hw);
+
+	regmap_update_bits(pll->regmap, AIC32X4_PLLPR,
+				AIC32X4_PLLEN, 0);
+}
+
+static int clk_aic32x4_pll_is_prepared(struct clk_hw *hw)
+{
+	struct clk_aic32x4 *pll = to_clk_aic32x4(hw);
+
+	unsigned int val;
+	int ret;
+
+	ret = regmap_read(pll->regmap, AIC32X4_PLLPR, &val);
+	if (ret < 0)
+		return ret;
+
+	return !!(val & AIC32X4_PLLEN);
+}
+
+static int clk_aic32x4_pll_get_muldiv(struct clk_aic32x4 *pll,
+			struct clk_aic32x4_pll_muldiv *settings)
+{
+	/*	Change to use regmap_bulk_read? */
+	unsigned int val;
+	int ret;
+
+	ret = regmap_read(pll->regmap, AIC32X4_PLLPR, &val);
+	if (ret < 0)
+		return ret;
+	settings->r = val & AIC32X4_PLL_R_MASK;
+	settings->p = (val & AIC32X4_PLL_P_MASK) >> AIC32X4_PLL_P_SHIFT;
+
+	ret = regmap_read(pll->regmap, AIC32X4_PLLJ, &val);
+	if (ret < 0)
+		return ret;
+	settings->j = val;
+
+	ret = regmap_read(pll->regmap, AIC32X4_PLLDMSB, &val);
+	if (ret < 0)
+		return ret;
+	settings->d = val << 8;
+
+	ret = regmap_read(pll->regmap, AIC32X4_PLLDLSB,	 &val);
+	if (ret < 0)
+		return ret;
+	settings->d |= val;
+
+	return 0;
+}
+
+static int clk_aic32x4_pll_set_muldiv(struct clk_aic32x4 *pll,
+			struct clk_aic32x4_pll_muldiv *settings)
+{
+	int ret;
+	/*	Change to use regmap_bulk_write for some if not all? */
+
+	ret = regmap_update_bits(pll->regmap, AIC32X4_PLLPR,
+				AIC32X4_PLL_R_MASK, settings->r);
+	if (ret < 0)
+		return ret;
+
+	ret = regmap_update_bits(pll->regmap, AIC32X4_PLLPR,
+				AIC32X4_PLL_P_MASK,
+				settings->p << AIC32X4_PLL_P_SHIFT);
+	if (ret < 0)
+		return ret;
+
+	ret = regmap_write(pll->regmap, AIC32X4_PLLJ, settings->j);
+	if (ret < 0)
+		return ret;
+
+	ret = regmap_write(pll->regmap, AIC32X4_PLLDMSB, (settings->d >> 8));
+	if (ret < 0)
+		return ret;
+	ret = regmap_write(pll->regmap, AIC32X4_PLLDLSB, (settings->d & 0xff));
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static unsigned long clk_aic32x4_pll_calc_rate(
+			struct clk_aic32x4_pll_muldiv *settings,
+			unsigned long parent_rate)
+{
+	u64 rate;
+	/*
+	 * We scale j by 10000 to account for the decimal part of P and divide
+	 * it back out later.
+	 */
+	rate = (u64) parent_rate * settings->r *
+				((settings->j * 10000) + settings->d);
+
+	return (unsigned long) DIV_ROUND_UP_ULL(rate, settings->p * 10000);
+}
+
+static int clk_aic32x4_pll_calc_muldiv(struct clk_aic32x4_pll_muldiv *settings,
+			unsigned long rate, unsigned long parent_rate)
+{
+	u64 multiplier;
+
+	settings->p = parent_rate / AIC32X4_MAX_PLL_CLKIN + 1;
+	if (settings->p > 8)
+		return -1;
+
+	/*
+	 * We scale this figure by 10000 so that we can get the decimal part
+	 * of the multiplier.	This is because we can't do floating point
+	 * math in the kernel.
+	 */
+	multiplier = (u64) rate * settings->p * 10000;
+	do_div(multiplier, parent_rate);
+
+	/*
+	 * J can't be over 64, so R can scale this.
+	 * R can't be greater than 4.
+	 */
+	settings->r = ((u32) multiplier / 640000) + 1;
+	if (settings->r > 4)
+		return -1;
+	do_div(multiplier, settings->r);
+
+	/*
+	 * J can't be < 1.
+	 */
+	if (multiplier < 10000)
+		return -1;
+
+	/* Figure out the integer part, J, and the fractional part, D. */
+	settings->j = (u32) multiplier / 10000;
+	settings->d = (u32) multiplier % 10000;
+
+	return 0;
+}
+
+static unsigned long clk_aic32x4_pll_recalc_rate(struct clk_hw *hw,
+			unsigned long parent_rate)
+{
+	struct clk_aic32x4 *pll = to_clk_aic32x4(hw);
+	struct clk_aic32x4_pll_muldiv settings;
+	int ret;
+
+	ret =  clk_aic32x4_pll_get_muldiv(pll, &settings);
+	if (ret < 0)
+		return 0;
+
+	return clk_aic32x4_pll_calc_rate(&settings, parent_rate);
+}
+
+static long clk_aic32x4_pll_round_rate(struct clk_hw *hw,
+			unsigned long rate,
+			unsigned long *parent_rate)
+{
+	struct clk_aic32x4_pll_muldiv settings;
+	int ret;
+
+	ret = clk_aic32x4_pll_calc_muldiv(&settings, rate, *parent_rate);
+	if (ret < 0)
+		return 0;
+
+	return clk_aic32x4_pll_calc_rate(&settings, *parent_rate);
+}
+
+static int clk_aic32x4_pll_set_rate(struct clk_hw *hw,
+			unsigned long rate,
+			unsigned long parent_rate)
+{
+	struct clk_aic32x4 *pll = to_clk_aic32x4(hw);
+	struct clk_aic32x4_pll_muldiv settings;
+	int ret;
+
+	ret = clk_aic32x4_pll_calc_muldiv(&settings, rate, parent_rate);
+	if (ret < 0)
+		return -EINVAL;
+
+	return clk_aic32x4_pll_set_muldiv(pll, &settings);
+}
+
+static int clk_aic32x4_pll_set_parent(struct clk_hw *hw, u8 index)
+{
+	struct clk_aic32x4 *pll = to_clk_aic32x4(hw);
+
+	return regmap_update_bits(pll->regmap,
+				AIC32X4_CLKMUX,
+				AIC32X4_PLL_CLKIN_MASK,
+				index << AIC32X4_PLL_CLKIN_SHIFT);
+}
+
+static u8 clk_aic32x4_pll_get_parent(struct clk_hw *hw)
+{
+	struct clk_aic32x4 *pll = to_clk_aic32x4(hw);
+	unsigned int val;
+
+	regmap_read(pll->regmap, AIC32X4_PLLPR, &val);
+
+	return (val & AIC32X4_PLL_CLKIN_MASK) >> AIC32X4_PLL_CLKIN_SHIFT;
+}
+
+
+static const struct clk_ops aic32x4_pll_ops = {
+	.prepare = clk_aic32x4_pll_prepare,
+	.unprepare = clk_aic32x4_pll_unprepare,
+	.is_prepared = clk_aic32x4_pll_is_prepared,
+	.recalc_rate = clk_aic32x4_pll_recalc_rate,
+	.round_rate = clk_aic32x4_pll_round_rate,
+	.set_rate = clk_aic32x4_pll_set_rate,
+	.set_parent = clk_aic32x4_pll_set_parent,
+	.get_parent = clk_aic32x4_pll_get_parent,
+};
+
+static int clk_aic32x4_codec_clkin_set_parent(struct clk_hw *hw, u8 index)
+{
+	struct clk_aic32x4 *mux = to_clk_aic32x4(hw);
+
+	return regmap_update_bits(mux->regmap,
+		AIC32X4_CLKMUX,
+		AIC32X4_CODEC_CLKIN_MASK, index << AIC32X4_CODEC_CLKIN_SHIFT);
+}
+
+static u8 clk_aic32x4_codec_clkin_get_parent(struct clk_hw *hw)
+{
+	struct clk_aic32x4 *mux = to_clk_aic32x4(hw);
+	unsigned int val;
+
+	regmap_read(mux->regmap, AIC32X4_CLKMUX, &val);
+
+	return (val & AIC32X4_CODEC_CLKIN_MASK) >> AIC32X4_CODEC_CLKIN_SHIFT;
+}
+
+static const struct clk_ops aic32x4_codec_clkin_ops = {
+	.set_parent = clk_aic32x4_codec_clkin_set_parent,
+	.get_parent = clk_aic32x4_codec_clkin_get_parent,
+};
+
+static int clk_aic32x4_div_prepare(struct clk_hw *hw)
+{
+	struct clk_aic32x4 *div = to_clk_aic32x4(hw);
+
+	return regmap_update_bits(div->regmap, div->reg,
+				AIC32X4_DIVEN, AIC32X4_DIVEN);
+}
+
+static void clk_aic32x4_div_unprepare(struct clk_hw *hw)
+{
+	struct clk_aic32x4 *div = to_clk_aic32x4(hw);
+
+	regmap_update_bits(div->regmap, div->reg,
+			AIC32X4_DIVEN, 0);
+}
+
+static int clk_aic32x4_div_set_rate(struct clk_hw *hw, unsigned long rate,
+				unsigned long parent_rate)
+{
+	struct clk_aic32x4 *div = to_clk_aic32x4(hw);
+	u8 divisor;
+
+	divisor = DIV_ROUND_UP(parent_rate, rate);
+	if (divisor > 128)
+		return -EINVAL;
+
+	return regmap_update_bits(div->regmap, div->reg,
+				AIC32X4_DIV_MASK, divisor);
+}
+
+static long clk_aic32x4_div_round_rate(struct clk_hw *hw, unsigned long rate,
+				unsigned long *parent_rate)
+{
+	unsigned long divisor;
+
+	divisor = DIV_ROUND_UP(*parent_rate, rate);
+	if (divisor > 128)
+		return -EINVAL;
+
+	return DIV_ROUND_UP(*parent_rate, divisor);
+}
+
+static unsigned long clk_aic32x4_div_recalc_rate(struct clk_hw *hw,
+						unsigned long parent_rate)
+{
+	struct clk_aic32x4 *div = to_clk_aic32x4(hw);
+
+	unsigned int val;
+
+	regmap_read(div->regmap, div->reg, &val);
+
+	return DIV_ROUND_UP(parent_rate, val & AIC32X4_DIV_MASK);
+}
+
+static const struct clk_ops aic32x4_div_ops = {
+	.prepare = clk_aic32x4_div_prepare,
+	.unprepare = clk_aic32x4_div_unprepare,
+	.set_rate = clk_aic32x4_div_set_rate,
+	.round_rate = clk_aic32x4_div_round_rate,
+	.recalc_rate = clk_aic32x4_div_recalc_rate,
+};
+
+static int clk_aic32x4_bdiv_set_parent(struct clk_hw *hw, u8 index)
+{
+	struct clk_aic32x4 *mux = to_clk_aic32x4(hw);
+
+	return regmap_update_bits(mux->regmap, AIC32X4_IFACE3,
+				AIC32X4_BDIVCLK_MASK, index);
+}
+
+static u8 clk_aic32x4_bdiv_get_parent(struct clk_hw *hw)
+{
+	struct clk_aic32x4 *mux = to_clk_aic32x4(hw);
+	unsigned int val;
+
+	regmap_read(mux->regmap, AIC32X4_IFACE3, &val);
+
+	return val & AIC32X4_BDIVCLK_MASK;
+}
+
+static const struct clk_ops aic32x4_bdiv_ops = {
+	.prepare = clk_aic32x4_div_prepare,
+	.unprepare = clk_aic32x4_div_unprepare,
+	.set_parent = clk_aic32x4_bdiv_set_parent,
+	.get_parent = clk_aic32x4_bdiv_get_parent,
+	.set_rate = clk_aic32x4_div_set_rate,
+	.round_rate = clk_aic32x4_div_round_rate,
+	.recalc_rate = clk_aic32x4_div_recalc_rate,
+};
+
+static struct aic32x4_clkdesc aic32x4_clkdesc_array[] = {
+	{
+		.name = "pll",
+		.parent_names =
+			(const char* []) { "mclk", "bclk", "gpio", "din" },
+		.num_parents = 4,
+		.ops = &aic32x4_pll_ops,
+		.reg = 0,
+	},
+	{
+		.name = "codec_clkin",
+		.parent_names =
+			(const char *[]) { "mclk", "bclk", "gpio", "pll" },
+		.num_parents = 4,
+		.ops = &aic32x4_codec_clkin_ops,
+		.reg = 0,
+	},
+	{
+		.name = "ndac",
+		.parent_names = (const char * []) { "codec_clkin" },
+		.num_parents = 1,
+		.ops = &aic32x4_div_ops,
+		.reg = AIC32X4_NDAC,
+	},
+	{
+		.name = "mdac",
+		.parent_names = (const char * []) { "ndac" },
+		.num_parents = 1,
+		.ops = &aic32x4_div_ops,
+		.reg = AIC32X4_MDAC,
+	},
+	{
+		.name = "nadc",
+		.parent_names = (const char * []) { "codec_clkin" },
+		.num_parents = 1,
+		.ops = &aic32x4_div_ops,
+		.reg = AIC32X4_NADC,
+	},
+	{
+		.name = "madc",
+		.parent_names = (const char * []) { "nadc" },
+		.num_parents = 1,
+		.ops = &aic32x4_div_ops,
+		.reg = AIC32X4_MADC,
+	},
+	{
+		.name = "bdiv",
+		.parent_names =
+			(const char *[]) { "ndac", "mdac", "nadc", "madc" },
+		.num_parents = 4,
+		.ops = &aic32x4_bdiv_ops,
+		.reg = AIC32X4_BCLKN,
+	},
+};
+
+static struct clk *aic32x4_register_clk(struct device *dev,
+			struct aic32x4_clkdesc *desc)
+{
+	struct clk_init_data init;
+	struct clk_aic32x4 *priv;
+	const char *devname = dev_name(dev);
+
+	init.ops = desc->ops;
+	init.name = desc->name;
+	init.parent_names = desc->parent_names;
+	init.num_parents = desc->num_parents;
+	init.flags = 0;
+
+	priv = devm_kzalloc(dev, sizeof(struct clk_aic32x4), GFP_KERNEL);
+	if (priv == NULL)
+		return (struct clk *) -ENOMEM;
+
+	priv->dev = dev;
+	priv->hw.init = &init;
+	priv->regmap = dev_get_regmap(dev, NULL);
+	priv->reg = desc->reg;
+
+	clk_hw_register_clkdev(&priv->hw, desc->name, devname);
+	return devm_clk_register(dev, &priv->hw);
+}
+
+int aic32x4_register_clocks(struct device *dev, const char *mclk_name)
+{
+	int i;
+
+	/*
+	 * These lines are here to preserve the current functionality of
+	 * the driver with regard to the DT.  These should eventually be set
+	 * by DT nodes so that the connections can be set up in configuration
+	 * rather than code.
+	 */
+	aic32x4_clkdesc_array[0].parent_names =
+			(const char* []) { mclk_name, "bclk", "gpio", "din" };
+	aic32x4_clkdesc_array[1].parent_names =
+			(const char *[]) { mclk_name, "bclk", "gpio", "pll" };
+
+	for (i = 0; i < ARRAY_SIZE(aic32x4_clkdesc_array); ++i)
+		aic32x4_register_clk(dev, &aic32x4_clkdesc_array[i]);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(aic32x4_register_clocks);
diff --git a/sound/soc/codecs/tlv320aic32x4-i2c.c b/sound/soc/codecs/tlv320aic32x4-i2c.c
index 22c3a6bc0b6c..6d54cbf70a0b 100644
--- a/sound/soc/codecs/tlv320aic32x4-i2c.c
+++ b/sound/soc/codecs/tlv320aic32x4-i2c.c
@@ -1,21 +1,11 @@
-/*
- * linux/sound/soc/codecs/tlv320aic32x4-i2c.c
+/* SPDX-License-Identifier: GPL-2.0
  *
- * Copyright 2011 NW Digital Radio
+ * Copyright 2011-2019 NW Digital Radio
  *
  * Author: Annaliese McDermond <nh6z@nh6z.net>
  *
  * Based on sound/soc/codecs/wm8974 and TI driver for kernel 2.6.27.
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #include <linux/i2c.h>
diff --git a/sound/soc/codecs/tlv320aic32x4-spi.c b/sound/soc/codecs/tlv320aic32x4-spi.c
index aa5b7ba0254b..a22e7700bfc8 100644
--- a/sound/soc/codecs/tlv320aic32x4-spi.c
+++ b/sound/soc/codecs/tlv320aic32x4-spi.c
@@ -1,21 +1,11 @@
-/*
- * linux/sound/soc/codecs/tlv320aic32x4-spi.c
+/* SPDX-License-Identifier: GPL-2.0
  *
- * Copyright 2011 NW Digital Radio
+ * Copyright 2011-2019 NW Digital Radio
  *
  * Author: Annaliese McDermond <nh6z@nh6z.net>
  *
  * Based on sound/soc/codecs/wm8974 and TI driver for kernel 2.6.27.
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
  */
 
 #include <linux/spi/spi.h>
diff --git a/sound/soc/codecs/tlv320aic32x4.c b/sound/soc/codecs/tlv320aic32x4.c
index 5520044929f4..83608f386aef 100644
--- a/sound/soc/codecs/tlv320aic32x4.c
+++ b/sound/soc/codecs/tlv320aic32x4.c
@@ -14,7 +14,7 @@
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
@@ -33,6 +33,7 @@
 #include <linux/cdev.h>
 #include <linux/slab.h>
 #include <linux/clk.h>
+#include <linux/of_clk.h>
 #include <linux/regulator/consumer.h>
 
 #include <sound/tlv320aic32x4.h>
@@ -46,29 +47,13 @@
 
 #include "tlv320aic32x4.h"
 
-struct aic32x4_rate_divs {
-	u32 mclk;
-	u32 rate;
-	u8 p_val;
-	u8 pll_j;
-	u16 pll_d;
-	u16 dosr;
-	u8 ndac;
-	u8 mdac;
-	u8 aosr;
-	u8 nadc;
-	u8 madc;
-	u8 blck_N;
-};
-
 struct aic32x4_priv {
 	struct regmap *regmap;
-	u32 sysclk;
 	u32 power_cfg;
 	u32 micpga_routing;
 	bool swapdacs;
 	int rstn_gpio;
-	struct clk *mclk;
+	const char *mclk_name;
 
 	struct regulator *supply_ldo;
 	struct regulator *supply_iov;
@@ -257,9 +242,24 @@ static DECLARE_TLV_DB_SCALE(tlv_driver_gain, -600, 100, 0);
 /* -12dB min, 0.5dB steps */
 static DECLARE_TLV_DB_SCALE(tlv_adc_vol, -1200, 50, 0);
 
+static const char * const lo_cm_text[] = {
+	"Full Chip", "1.65V",
+};
+
+static SOC_ENUM_SINGLE_DECL(lo_cm_enum, AIC32X4_CMMODE, 3, lo_cm_text);
+
+static const char * const ptm_text[] = {
+	"P3", "P2", "P1",
+};
+
+static SOC_ENUM_SINGLE_DECL(l_ptm_enum, AIC32X4_LPLAYBACK, 2, ptm_text);
+static SOC_ENUM_SINGLE_DECL(r_ptm_enum, AIC32X4_RPLAYBACK, 2, ptm_text);
+
 static const struct snd_kcontrol_new aic32x4_snd_controls[] = {
 	SOC_DOUBLE_R_S_TLV("PCM Playback Volume", AIC32X4_LDACVOL,
 			AIC32X4_RDACVOL, 0, -0x7f, 0x30, 7, 0, tlv_pcm),
+	SOC_ENUM("DAC Left Playback PowerTune Switch", l_ptm_enum),
+	SOC_ENUM("DAC Right Playback PowerTune Switch", r_ptm_enum),
 	SOC_DOUBLE_R_S_TLV("HP Driver Gain Volume", AIC32X4_HPLGAIN,
 			AIC32X4_HPRGAIN, 0, -0x6, 0x1d, 5, 0,
 			tlv_driver_gain),
@@ -270,6 +270,7 @@ static const struct snd_kcontrol_new aic32x4_snd_controls[] = {
 			AIC32X4_HPRGAIN, 6, 0x01, 1),
 	SOC_DOUBLE_R("LO DAC Playback Switch", AIC32X4_LOLGAIN,
 			AIC32X4_LORGAIN, 6, 0x01, 1),
+	SOC_ENUM("LO Playback Common Mode Switch", lo_cm_enum),
 	SOC_DOUBLE_R("Mic PGA Switch", AIC32X4_LMICPGAVOL,
 			AIC32X4_RMICPGAVOL, 7, 0x01, 1),
 
@@ -305,38 +306,6 @@ static const struct snd_kcontrol_new aic32x4_snd_controls[] = {
 			0, 0x0F, 0),
 };
 
-static const struct aic32x4_rate_divs aic32x4_divs[] = {
-	/* 8k rate */
-	{12000000, 8000, 1, 7, 6800, 768, 5, 3, 128, 5, 18, 24},
-	{24000000, 8000, 2, 7, 6800, 768, 15, 1, 64, 45, 4, 24},
-	{25000000, 8000, 2, 7, 3728, 768, 15, 1, 64, 45, 4, 24},
-	/* 11.025k rate */
-	{12000000, 11025, 1, 7, 5264, 512, 8, 2, 128, 8, 8, 16},
-	{24000000, 11025, 2, 7, 5264, 512, 16, 1, 64, 32, 4, 16},
-	/* 16k rate */
-	{12000000, 16000, 1, 7, 6800, 384, 5, 3, 128, 5, 9, 12},
-	{24000000, 16000, 2, 7, 6800, 384, 15, 1, 64, 18, 5, 12},
-	{25000000, 16000, 2, 7, 3728, 384, 15, 1, 64, 18, 5, 12},
-	/* 22.05k rate */
-	{12000000, 22050, 1, 7, 5264, 256, 4, 4, 128, 4, 8, 8},
-	{24000000, 22050, 2, 7, 5264, 256, 16, 1, 64, 16, 4, 8},
-	{25000000, 22050, 2, 7, 2253, 256, 16, 1, 64, 16, 4, 8},
-	/* 32k rate */
-	{12000000, 32000, 1, 7, 1680, 192, 2, 7, 64, 2, 21, 6},
-	{24000000, 32000, 2, 7, 1680, 192, 7, 2, 64, 7, 6, 6},
-	/* 44.1k rate */
-	{12000000, 44100, 1, 7, 5264, 128, 2, 8, 128, 2, 8, 4},
-	{24000000, 44100, 2, 7, 5264, 128, 8, 2, 64, 8, 4, 4},
-	{25000000, 44100, 2, 7, 2253, 128, 8, 2, 64, 8, 4, 4},
-	/* 48k rate */
-	{12000000, 48000, 1, 8, 1920, 128, 2, 8, 128, 2, 8, 4},
-	{24000000, 48000, 2, 8, 1920, 128, 8, 2, 64, 8, 4, 4},
-	{25000000, 48000, 2, 7, 8643, 128, 8, 2, 64, 8, 4, 4},
-
-	/* 96k rate */
-	{25000000, 96000, 2, 7, 8643, 64, 4, 4, 64, 4, 4, 1},
-};
-
 static const struct snd_kcontrol_new hpl_output_mixer_controls[] = {
 	SOC_DAPM_SINGLE("L_DAC Switch", AIC32X4_HPLROUTE, 3, 1, 0),
 	SOC_DAPM_SINGLE("IN1_L Switch", AIC32X4_HPLROUTE, 2, 1, 0),
@@ -391,7 +360,7 @@ static const struct snd_kcontrol_new in3r_to_lmixer_controls[] = {
 	SOC_DAPM_ENUM("IN3_R L- Switch", in3r_lpga_n_enum),
 };
 
-/*  Right mixer pins */
+/*	Right mixer pins */
 static SOC_ENUM_SINGLE_DECL(in1r_rpga_p_enum, AIC32X4_RMICPGAPIN, 6, resistor_text);
 static SOC_ENUM_SINGLE_DECL(in2r_rpga_p_enum, AIC32X4_RMICPGAPIN, 4, resistor_text);
 static SOC_ENUM_SINGLE_DECL(in3r_rpga_p_enum, AIC32X4_RMICPGAPIN, 2, resistor_text);
@@ -595,7 +564,7 @@ static const struct snd_soc_dapm_route aic32x4_dapm_routes[] = {
 static const struct regmap_range_cfg aic32x4_regmap_pages[] = {
 	{
 		.selector_reg = 0,
-		.selector_mask  = 0xff,
+		.selector_mask	= 0xff,
 		.window_start = 0,
 		.window_len = 128,
 		.range_min = 0,
@@ -610,35 +579,17 @@ const struct regmap_config aic32x4_regmap_config = {
 };
 EXPORT_SYMBOL(aic32x4_regmap_config);
 
-static inline int aic32x4_get_divs(int mclk, int rate)
-{
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(aic32x4_divs); i++) {
-		if ((aic32x4_divs[i].rate == rate)
-		    && (aic32x4_divs[i].mclk == mclk)) {
-			return i;
-		}
-	}
-	printk(KERN_ERR "aic32x4: master clock and sample rate is not supported\n");
-	return -EINVAL;
-}
-
 static int aic32x4_set_dai_sysclk(struct snd_soc_dai *codec_dai,
 				  int clk_id, unsigned int freq, int dir)
 {
 	struct snd_soc_component *component = codec_dai->component;
-	struct aic32x4_priv *aic32x4 = snd_soc_component_get_drvdata(component);
+	struct clk *mclk;
+	struct clk *pll;
 
-	switch (freq) {
-	case 12000000:
-	case 24000000:
-	case 25000000:
-		aic32x4->sysclk = freq;
-		return 0;
-	}
-	printk(KERN_ERR "aic32x4: invalid frequency to set DAI system clock\n");
-	return -EINVAL;
+	pll = devm_clk_get(component->dev, "pll");
+	mclk = clk_get_parent(pll);
+
+	return clk_set_rate(mclk, freq);
 }
 
 static int aic32x4_set_dai_fmt(struct snd_soc_dai *codec_dai, unsigned int fmt)
@@ -688,103 +639,175 @@ static int aic32x4_set_dai_fmt(struct snd_soc_dai *codec_dai, unsigned int fmt)
 	}
 
 	snd_soc_component_update_bits(component, AIC32X4_IFACE1,
-			    AIC32X4_IFACE1_DATATYPE_MASK |
-			    AIC32X4_IFACE1_MASTER_MASK, iface_reg_1);
+				AIC32X4_IFACE1_DATATYPE_MASK |
+				AIC32X4_IFACE1_MASTER_MASK, iface_reg_1);
 	snd_soc_component_update_bits(component, AIC32X4_IFACE2,
-			    AIC32X4_DATA_OFFSET_MASK, iface_reg_2);
+				AIC32X4_DATA_OFFSET_MASK, iface_reg_2);
 	snd_soc_component_update_bits(component, AIC32X4_IFACE3,
-			    AIC32X4_BCLKINV_MASK, iface_reg_3);
+				AIC32X4_BCLKINV_MASK, iface_reg_3);
 
 	return 0;
 }
 
-static int aic32x4_hw_params(struct snd_pcm_substream *substream,
-			     struct snd_pcm_hw_params *params,
-			     struct snd_soc_dai *dai)
+static int aic32x4_set_aosr(struct snd_soc_component *component, u8 aosr)
 {
-	struct snd_soc_component *component = dai->component;
-	struct aic32x4_priv *aic32x4 = snd_soc_component_get_drvdata(component);
-	u8 iface1_reg = 0;
-	u8 dacsetup_reg = 0;
-	int i;
-
-	i = aic32x4_get_divs(aic32x4->sysclk, params_rate(params));
-	if (i < 0) {
-		printk(KERN_ERR "aic32x4: sampling rate not supported\n");
-		return i;
-	}
+	return snd_soc_component_write(component, AIC32X4_AOSR, aosr);
+}
 
-	/* MCLK as PLL_CLKIN */
-	snd_soc_component_update_bits(component, AIC32X4_CLKMUX, AIC32X4_PLL_CLKIN_MASK,
-			    AIC32X4_PLL_CLKIN_MCLK << AIC32X4_PLL_CLKIN_SHIFT);
-	/* PLL as CODEC_CLKIN */
-	snd_soc_component_update_bits(component, AIC32X4_CLKMUX, AIC32X4_CODEC_CLKIN_MASK,
-			    AIC32X4_CODEC_CLKIN_PLL << AIC32X4_CODEC_CLKIN_SHIFT);
-	/* DAC_MOD_CLK as BDIV_CLKIN */
-	snd_soc_component_update_bits(component, AIC32X4_IFACE3, AIC32X4_BDIVCLK_MASK,
-			    AIC32X4_DACMOD2BCLK << AIC32X4_BDIVCLK_SHIFT);
+static int aic32x4_set_dosr(struct snd_soc_component *component, u16 dosr)
+{
+	snd_soc_component_write(component, AIC32X4_DOSRMSB, dosr >> 8);
+	snd_soc_component_write(component, AIC32X4_DOSRLSB,
+		      (dosr & 0xff));
 
-	/* We will fix R value to 1 and will make P & J=K.D as variable */
-	snd_soc_component_update_bits(component, AIC32X4_PLLPR, AIC32X4_PLL_R_MASK, 0x01);
+	return 0;
+}
 
-	/* PLL P value */
-	snd_soc_component_update_bits(component, AIC32X4_PLLPR, AIC32X4_PLL_P_MASK,
-			    aic32x4_divs[i].p_val << AIC32X4_PLL_P_SHIFT);
+static int aic32x4_set_processing_blocks(struct snd_soc_component *component,
+						u8 r_block, u8 p_block)
+{
+	if (r_block > 18 || p_block > 25)
+		return -EINVAL;
 
-	/* PLL J value */
-	snd_soc_component_write(component, AIC32X4_PLLJ, aic32x4_divs[i].pll_j);
+	snd_soc_component_write(component, AIC32X4_ADCSPB, r_block);
+	snd_soc_component_write(component, AIC32X4_DACSPB, p_block);
 
-	/* PLL D value */
-	snd_soc_component_write(component, AIC32X4_PLLDMSB, (aic32x4_divs[i].pll_d >> 8));
-	snd_soc_component_write(component, AIC32X4_PLLDLSB, (aic32x4_divs[i].pll_d & 0xff));
+	return 0;
+}
 
-	/* NDAC divider value */
-	snd_soc_component_update_bits(component, AIC32X4_NDAC,
-			    AIC32X4_NDAC_MASK, aic32x4_divs[i].ndac);
+static int aic32x4_setup_clocks(struct snd_soc_component *component,
+				unsigned int sample_rate)
+{
+	u8 aosr;
+	u16 dosr;
+	u8 adc_resource_class, dac_resource_class;
+	u8 madc, nadc, mdac, ndac, max_nadc, min_mdac, max_ndac;
+	u8 dosr_increment;
+	u16 max_dosr, min_dosr;
+	unsigned long adc_clock_rate, dac_clock_rate;
+	int ret;
 
-	/* MDAC divider value */
-	snd_soc_component_update_bits(component, AIC32X4_MDAC,
-			    AIC32X4_MDAC_MASK, aic32x4_divs[i].mdac);
+	struct clk_bulk_data clocks[] = {
+		{ .id = "pll" },
+		{ .id = "nadc" },
+		{ .id = "madc" },
+		{ .id = "ndac" },
+		{ .id = "mdac" },
+		{ .id = "bdiv" },
+	};
+	ret = devm_clk_bulk_get(component->dev, ARRAY_SIZE(clocks), clocks);
+	if (ret)
+		return ret;
 
-	/* DOSR MSB & LSB values */
-	snd_soc_component_write(component, AIC32X4_DOSRMSB, aic32x4_divs[i].dosr >> 8);
-	snd_soc_component_write(component, AIC32X4_DOSRLSB, (aic32x4_divs[i].dosr & 0xff));
+	if (sample_rate <= 48000) {
+		aosr = 128;
+		adc_resource_class = 6;
+		dac_resource_class = 8;
+		dosr_increment = 8;
+		aic32x4_set_processing_blocks(component, 1, 1);
+	} else if (sample_rate <= 96000) {
+		aosr = 64;
+		adc_resource_class = 6;
+		dac_resource_class = 8;
+		dosr_increment = 4;
+		aic32x4_set_processing_blocks(component, 1, 9);
+	} else if (sample_rate == 192000) {
+		aosr = 32;
+		adc_resource_class = 3;
+		dac_resource_class = 4;
+		dosr_increment = 2;
+		aic32x4_set_processing_blocks(component, 13, 19);
+	} else {
+		dev_err(component->dev, "Sampling rate not supported\n");
+		return -EINVAL;
+	}
 
-	/* NADC divider value */
-	snd_soc_component_update_bits(component, AIC32X4_NADC,
-			    AIC32X4_NADC_MASK, aic32x4_divs[i].nadc);
+	madc = DIV_ROUND_UP((32 * adc_resource_class), aosr);
+	max_dosr = (AIC32X4_MAX_DOSR_FREQ / sample_rate / dosr_increment) *
+			dosr_increment;
+	min_dosr = (AIC32X4_MIN_DOSR_FREQ / sample_rate / dosr_increment) *
+			dosr_increment;
+	max_nadc = AIC32X4_MAX_CODEC_CLKIN_FREQ / (madc * aosr * sample_rate);
+
+	for (nadc = max_nadc; nadc > 0; --nadc) {
+		adc_clock_rate = nadc * madc * aosr * sample_rate;
+		for (dosr = max_dosr; dosr >= min_dosr;
+				dosr -= dosr_increment) {
+			min_mdac = DIV_ROUND_UP((32 * dac_resource_class), dosr);
+			max_ndac = AIC32X4_MAX_CODEC_CLKIN_FREQ /
+					(min_mdac * dosr * sample_rate);
+			for (mdac = min_mdac; mdac <= 128; ++mdac) {
+				for (ndac = max_ndac; ndac > 0; --ndac) {
+					dac_clock_rate = ndac * mdac * dosr *
+							sample_rate;
+					if (dac_clock_rate == adc_clock_rate) {
+						if (clk_round_rate(clocks[0].clk, dac_clock_rate) == 0)
+							continue;
+
+						clk_set_rate(clocks[0].clk,
+							dac_clock_rate);
+
+						clk_set_rate(clocks[1].clk,
+							sample_rate * aosr *
+							madc);
+						clk_set_rate(clocks[2].clk,
+							sample_rate * aosr);
+						aic32x4_set_aosr(component,
+							aosr);
+
+						clk_set_rate(clocks[3].clk,
+							sample_rate * dosr *
+							mdac);
+						clk_set_rate(clocks[4].clk,
+							sample_rate * dosr);
+						aic32x4_set_dosr(component,
+							dosr);
+
+						clk_set_rate(clocks[5].clk,
+							sample_rate * 32);
+						return 0;
+					}
+				}
+			}
+		}
+	}
 
-	/* MADC divider value */
-	snd_soc_component_update_bits(component, AIC32X4_MADC,
-			    AIC32X4_MADC_MASK, aic32x4_divs[i].madc);
+	dev_err(component->dev,
+		"Could not set clocks to support sample rate.\n");
+	return -EINVAL;
+}
 
-	/* AOSR value */
-	snd_soc_component_write(component, AIC32X4_AOSR, aic32x4_divs[i].aosr);
+static int aic32x4_hw_params(struct snd_pcm_substream *substream,
+				 struct snd_pcm_hw_params *params,
+				 struct snd_soc_dai *dai)
+{
+	struct snd_soc_component *component = dai->component;
+	struct aic32x4_priv *aic32x4 = snd_soc_component_get_drvdata(component);
+	u8 iface1_reg = 0;
+	u8 dacsetup_reg = 0;
 
-	/* BCLK N divider */
-	snd_soc_component_update_bits(component, AIC32X4_BCLKN,
-			    AIC32X4_BCLK_MASK, aic32x4_divs[i].blck_N);
+	aic32x4_setup_clocks(component, params_rate(params));
 
 	switch (params_width(params)) {
 	case 16:
 		iface1_reg |= (AIC32X4_WORD_LEN_16BITS <<
-			       AIC32X4_IFACE1_DATALEN_SHIFT);
+				   AIC32X4_IFACE1_DATALEN_SHIFT);
 		break;
 	case 20:
 		iface1_reg |= (AIC32X4_WORD_LEN_20BITS <<
-			       AIC32X4_IFACE1_DATALEN_SHIFT);
+				   AIC32X4_IFACE1_DATALEN_SHIFT);
 		break;
 	case 24:
 		iface1_reg |= (AIC32X4_WORD_LEN_24BITS <<
-			       AIC32X4_IFACE1_DATALEN_SHIFT);
+				   AIC32X4_IFACE1_DATALEN_SHIFT);
 		break;
 	case 32:
 		iface1_reg |= (AIC32X4_WORD_LEN_32BITS <<
-			       AIC32X4_IFACE1_DATALEN_SHIFT);
+				   AIC32X4_IFACE1_DATALEN_SHIFT);
 		break;
 	}
 	snd_soc_component_update_bits(component, AIC32X4_IFACE1,
-			    AIC32X4_IFACE1_DATALEN_MASK, iface1_reg);
+				AIC32X4_IFACE1_DATALEN_MASK, iface1_reg);
 
 	if (params_channels(params) == 1) {
 		dacsetup_reg = AIC32X4_RDAC2LCHN | AIC32X4_LDAC2LCHN;
@@ -795,7 +818,7 @@ static int aic32x4_hw_params(struct snd_pcm_substream *substream,
 			dacsetup_reg = AIC32X4_LDAC2LCHN | AIC32X4_RDAC2RCHN;
 	}
 	snd_soc_component_update_bits(component, AIC32X4_DACSETUP,
-			    AIC32X4_DAC_CHAN_MASK, dacsetup_reg);
+				AIC32X4_DAC_CHAN_MASK, dacsetup_reg);
 
 	return 0;
 }
@@ -805,7 +828,7 @@ static int aic32x4_mute(struct snd_soc_dai *dai, int mute)
 	struct snd_soc_component *component = dai->component;
 
 	snd_soc_component_update_bits(component, AIC32X4_DACMUTE,
-			    AIC32X4_MUTEON, mute ? AIC32X4_MUTEON : 0);
+				AIC32X4_MUTEON, mute ? AIC32X4_MUTEON : 0);
 
 	return 0;
 }
@@ -813,41 +836,25 @@ static int aic32x4_mute(struct snd_soc_dai *dai, int mute)
 static int aic32x4_set_bias_level(struct snd_soc_component *component,
 				  enum snd_soc_bias_level level)
 {
-	struct aic32x4_priv *aic32x4 = snd_soc_component_get_drvdata(component);
 	int ret;
 
+	struct clk_bulk_data clocks[] = {
+		{ .id = "madc" },
+		{ .id = "mdac" },
+		{ .id = "bdiv" },
+	};
+
+	ret = devm_clk_bulk_get(component->dev, ARRAY_SIZE(clocks), clocks);
+	if (ret)
+		return ret;
+
 	switch (level) {
 	case SND_SOC_BIAS_ON:
-		/* Switch on master clock */
-		ret = clk_prepare_enable(aic32x4->mclk);
+		ret = clk_bulk_prepare_enable(ARRAY_SIZE(clocks), clocks);
 		if (ret) {
-			dev_err(component->dev, "Failed to enable master clock\n");
+			dev_err(component->dev, "Failed to enable clocks\n");
 			return ret;
 		}
-
-		/* Switch on PLL */
-		snd_soc_component_update_bits(component, AIC32X4_PLLPR,
-				    AIC32X4_PLLEN, AIC32X4_PLLEN);
-
-		/* Switch on NDAC Divider */
-		snd_soc_component_update_bits(component, AIC32X4_NDAC,
-				    AIC32X4_NDACEN, AIC32X4_NDACEN);
-
-		/* Switch on MDAC Divider */
-		snd_soc_component_update_bits(component, AIC32X4_MDAC,
-				    AIC32X4_MDACEN, AIC32X4_MDACEN);
-
-		/* Switch on NADC Divider */
-		snd_soc_component_update_bits(component, AIC32X4_NADC,
-				    AIC32X4_NADCEN, AIC32X4_NADCEN);
-
-		/* Switch on MADC Divider */
-		snd_soc_component_update_bits(component, AIC32X4_MADC,
-				    AIC32X4_MADCEN, AIC32X4_MADCEN);
-
-		/* Switch on BCLK_N Divider */
-		snd_soc_component_update_bits(component, AIC32X4_BCLKN,
-				    AIC32X4_BCLKEN, AIC32X4_BCLKEN);
 		break;
 	case SND_SOC_BIAS_PREPARE:
 		break;
@@ -856,32 +863,7 @@ static int aic32x4_set_bias_level(struct snd_soc_component *component,
 		if (snd_soc_component_get_bias_level(component) == SND_SOC_BIAS_OFF)
 			break;
 
-		/* Switch off BCLK_N Divider */
-		snd_soc_component_update_bits(component, AIC32X4_BCLKN,
-				    AIC32X4_BCLKEN, 0);
-
-		/* Switch off MADC Divider */
-		snd_soc_component_update_bits(component, AIC32X4_MADC,
-				    AIC32X4_MADCEN, 0);
-
-		/* Switch off NADC Divider */
-		snd_soc_component_update_bits(component, AIC32X4_NADC,
-				    AIC32X4_NADCEN, 0);
-
-		/* Switch off MDAC Divider */
-		snd_soc_component_update_bits(component, AIC32X4_MDAC,
-				    AIC32X4_MDACEN, 0);
-
-		/* Switch off NDAC Divider */
-		snd_soc_component_update_bits(component, AIC32X4_NDAC,
-				    AIC32X4_NDACEN, 0);
-
-		/* Switch off PLL */
-		snd_soc_component_update_bits(component, AIC32X4_PLLPR,
-				    AIC32X4_PLLEN, 0);
-
-		/* Switch off master clock */
-		clk_disable_unprepare(aic32x4->mclk);
+		clk_bulk_disable_unprepare(ARRAY_SIZE(clocks), clocks);
 		break;
 	case SND_SOC_BIAS_OFF:
 		break;
@@ -889,8 +871,8 @@ static int aic32x4_set_bias_level(struct snd_soc_component *component,
 	return 0;
 }
 
-#define AIC32X4_RATES	SNDRV_PCM_RATE_8000_96000
-#define AIC32X4_FORMATS	(SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S20_3LE \
+#define AIC32X4_RATES	SNDRV_PCM_RATE_8000_192000
+#define AIC32X4_FORMATS (SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S20_3LE \
 			 | SNDRV_PCM_FMTBIT_S24_3LE | SNDRV_PCM_FMTBIT_S32_LE)
 
 static const struct snd_soc_dai_ops aic32x4_ops = {
@@ -903,17 +885,17 @@ static const struct snd_soc_dai_ops aic32x4_ops = {
 static struct snd_soc_dai_driver aic32x4_dai = {
 	.name = "tlv320aic32x4-hifi",
 	.playback = {
-		     .stream_name = "Playback",
-		     .channels_min = 1,
-		     .channels_max = 2,
-		     .rates = AIC32X4_RATES,
-		     .formats = AIC32X4_FORMATS,},
+			 .stream_name = "Playback",
+			 .channels_min = 1,
+			 .channels_max = 2,
+			 .rates = AIC32X4_RATES,
+			 .formats = AIC32X4_FORMATS,},
 	.capture = {
-		    .stream_name = "Capture",
-		    .channels_min = 1,
-		    .channels_max = 2,
-		    .rates = AIC32X4_RATES,
-		    .formats = AIC32X4_FORMATS,},
+			.stream_name = "Capture",
+			.channels_min = 1,
+			.channels_max = 2,
+			.rates = AIC32X4_RATES,
+			.formats = AIC32X4_FORMATS,},
 	.ops = &aic32x4_ops,
 	.symmetric_rates = 1,
 };
@@ -926,7 +908,7 @@ static void aic32x4_setup_gpios(struct snd_soc_component *component)
 	/* MFP1 */
 	if (aic32x4->setup->gpio_func[0] != AIC32X4_MFPX_DEFAULT_VALUE) {
 		snd_soc_component_write(component, AIC32X4_DINCTL,
-		      aic32x4->setup->gpio_func[0]);
+			  aic32x4->setup->gpio_func[0]);
 		snd_soc_add_component_controls(component, aic32x4_mfp1,
 			ARRAY_SIZE(aic32x4_mfp1));
 	}
@@ -934,7 +916,7 @@ static void aic32x4_setup_gpios(struct snd_soc_component *component)
 	/* MFP2 */
 	if (aic32x4->setup->gpio_func[1] != AIC32X4_MFPX_DEFAULT_VALUE) {
 		snd_soc_component_write(component, AIC32X4_DOUTCTL,
-		      aic32x4->setup->gpio_func[1]);
+			  aic32x4->setup->gpio_func[1]);
 		snd_soc_add_component_controls(component, aic32x4_mfp2,
 			ARRAY_SIZE(aic32x4_mfp2));
 	}
@@ -942,7 +924,7 @@ static void aic32x4_setup_gpios(struct snd_soc_component *component)
 	/* MFP3 */
 	if (aic32x4->setup->gpio_func[2] != AIC32X4_MFPX_DEFAULT_VALUE) {
 		snd_soc_component_write(component, AIC32X4_SCLKCTL,
-		      aic32x4->setup->gpio_func[2]);
+			  aic32x4->setup->gpio_func[2]);
 		snd_soc_add_component_controls(component, aic32x4_mfp3,
 			ARRAY_SIZE(aic32x4_mfp3));
 	}
@@ -950,7 +932,7 @@ static void aic32x4_setup_gpios(struct snd_soc_component *component)
 	/* MFP4 */
 	if (aic32x4->setup->gpio_func[3] != AIC32X4_MFPX_DEFAULT_VALUE) {
 		snd_soc_component_write(component, AIC32X4_MISOCTL,
-		      aic32x4->setup->gpio_func[3]);
+			  aic32x4->setup->gpio_func[3]);
 		snd_soc_add_component_controls(component, aic32x4_mfp4,
 			ARRAY_SIZE(aic32x4_mfp4));
 	}
@@ -958,7 +940,7 @@ static void aic32x4_setup_gpios(struct snd_soc_component *component)
 	/* MFP5 */
 	if (aic32x4->setup->gpio_func[4] != AIC32X4_MFPX_DEFAULT_VALUE) {
 		snd_soc_component_write(component, AIC32X4_GPIOCTL,
-		      aic32x4->setup->gpio_func[4]);
+			  aic32x4->setup->gpio_func[4]);
 		snd_soc_add_component_controls(component, aic32x4_mfp5,
 			ARRAY_SIZE(aic32x4_mfp5));
 	}
@@ -968,6 +950,18 @@ static int aic32x4_component_probe(struct snd_soc_component *component)
 {
 	struct aic32x4_priv *aic32x4 = snd_soc_component_get_drvdata(component);
 	u32 tmp_reg;
+	int ret;
+
+	struct clk_bulk_data clocks[] = {
+		{ .id = "codec_clkin" },
+		{ .id = "pll" },
+		{ .id = "bdiv" },
+		{ .id = "mdac" },
+	};
+
+	ret = devm_clk_bulk_get(component->dev, ARRAY_SIZE(clocks), clocks);
+	if (ret)
+		return ret;
 
 	if (gpio_is_valid(aic32x4->rstn_gpio)) {
 		ndelay(10);
@@ -980,10 +974,13 @@ static int aic32x4_component_probe(struct snd_soc_component *component)
 	if (aic32x4->setup)
 		aic32x4_setup_gpios(component);
 
+	clk_set_parent(clocks[0].clk, clocks[1].clk);
+	clk_set_parent(clocks[2].clk, clocks[3].clk);
+
 	/* Power platform configuration */
 	if (aic32x4->power_cfg & AIC32X4_PWR_MICBIAS_2075_LDOIN) {
-		snd_soc_component_write(component, AIC32X4_MICBIAS, AIC32X4_MICBIAS_LDOIN |
-						      AIC32X4_MICBIAS_2075V);
+		snd_soc_component_write(component, AIC32X4_MICBIAS,
+				AIC32X4_MICBIAS_LDOIN | AIC32X4_MICBIAS_2075V);
 	}
 	if (aic32x4->power_cfg & AIC32X4_PWR_AVDD_DVDD_WEAK_DISABLE)
 		snd_soc_component_write(component, AIC32X4_PWRCFG, AIC32X4_AVDDWEAKDISABLE);
@@ -1046,12 +1043,18 @@ static int aic32x4_parse_dt(struct aic32x4_priv *aic32x4,
 		struct device_node *np)
 {
 	struct aic32x4_setup_data *aic32x4_setup;
+	int ret;
 
 	aic32x4_setup = devm_kzalloc(aic32x4->dev, sizeof(*aic32x4_setup),
 							GFP_KERNEL);
 	if (!aic32x4_setup)
 		return -ENOMEM;
 
+	ret = of_property_match_string(np, "clock-names", "mclk");
+	if (ret < 0)
+		return -EINVAL;
+	aic32x4->mclk_name = of_clk_get_parent_name(np, ret);
+
 	aic32x4->swapdacs = false;
 	aic32x4->micpga_routing = 0;
 	aic32x4->rstn_gpio = of_get_named_gpio(np, "reset-gpios", 0);
@@ -1173,7 +1176,7 @@ int aic32x4_probe(struct device *dev, struct regmap *regmap)
 		return PTR_ERR(regmap);
 
 	aic32x4 = devm_kzalloc(dev, sizeof(struct aic32x4_priv),
-			       GFP_KERNEL);
+				   GFP_KERNEL);
 	if (aic32x4 == NULL)
 		return -ENOMEM;
 
@@ -1185,6 +1188,7 @@ int aic32x4_probe(struct device *dev, struct regmap *regmap)
 		aic32x4->swapdacs = pdata->swapdacs;
 		aic32x4->micpga_routing = pdata->micpga_routing;
 		aic32x4->rstn_gpio = pdata->rstn_gpio;
+		aic32x4->mclk_name = "mclk";
 	} else if (np) {
 		ret = aic32x4_parse_dt(aic32x4, np);
 		if (ret) {
@@ -1196,13 +1200,12 @@ int aic32x4_probe(struct device *dev, struct regmap *regmap)
 		aic32x4->swapdacs = false;
 		aic32x4->micpga_routing = 0;
 		aic32x4->rstn_gpio = -1;
+		aic32x4->mclk_name = "mclk";
 	}
 
-	aic32x4->mclk = devm_clk_get(dev, "mclk");
-	if (IS_ERR(aic32x4->mclk)) {
-		dev_err(dev, "Failed getting the mclk. The current implementation does not support the usage of this codec without mclk\n");
-		return PTR_ERR(aic32x4->mclk);
-	}
+	ret = aic32x4_register_clocks(dev, aic32x4->mclk_name);
+	if (ret)
+		return ret;
 
 	if (gpio_is_valid(aic32x4->rstn_gpio)) {
 		ret = devm_gpio_request_one(dev, aic32x4->rstn_gpio,
diff --git a/sound/soc/codecs/tlv320aic32x4.h b/sound/soc/codecs/tlv320aic32x4.h
index c2d74025bf4b..40734211bc0e 100644
--- a/sound/soc/codecs/tlv320aic32x4.h
+++ b/sound/soc/codecs/tlv320aic32x4.h
@@ -16,6 +16,7 @@ struct regmap_config;
 extern const struct regmap_config aic32x4_regmap_config;
 int aic32x4_probe(struct device *dev, struct regmap *regmap);
 int aic32x4_remove(struct device *dev);
+int aic32x4_register_clocks(struct device *dev, const char *mclk_name);
 
 /* tlv320aic32x4 register space (in decimal to match datasheet) */
 
@@ -77,6 +78,8 @@ int aic32x4_remove(struct device *dev);
 
 #define AIC32X4_PWRCFG		AIC32X4_REG(1, 1)
 #define AIC32X4_LDOCTL		AIC32X4_REG(1, 2)
+#define AIC32X4_LPLAYBACK	AIC32X4_REG(1, 3)
+#define AIC32X4_RPLAYBACK	AIC32X4_REG(1, 4)
 #define AIC32X4_OUTPWRCTL	AIC32X4_REG(1, 9)
 #define AIC32X4_CMMODE		AIC32X4_REG(1, 10)
 #define AIC32X4_HPLROUTE	AIC32X4_REG(1, 12)
@@ -205,4 +208,14 @@ int aic32x4_remove(struct device *dev);
 #define AIC32X4_RMICPGANIN_IN1L_10K	0x10
 #define AIC32X4_RMICPGANIN_CM1R_10K	0x40
 
+/* Common mask and enable for all of the dividers */
+#define AIC32X4_DIVEN           BIT(7)
+#define AIC32X4_DIV_MASK        GENMASK(6, 0)
+
+/* Clock Limits */
+#define AIC32X4_MAX_DOSR_FREQ		6200000
+#define AIC32X4_MIN_DOSR_FREQ		2800000
+#define AIC32X4_MAX_CODEC_CLKIN_FREQ    110000000
+#define AIC32X4_MAX_PLL_CLKIN		20000000
+
 #endif				/* _TLV320AIC32X4_H */
diff --git a/sound/soc/codecs/wcd9335.c b/sound/soc/codecs/wcd9335.c
index 981f88a5f615..a04a7cedd99d 100644
--- a/sound/soc/codecs/wcd9335.c
+++ b/sound/soc/codecs/wcd9335.c
@@ -5188,6 +5188,7 @@ static int wcd9335_slim_status(struct slim_device *sdev,
 
 	wcd->slim = sdev;
 	wcd->slim_ifc_dev = of_slim_get_device(sdev->ctrl, ifc_dev_np);
+	of_node_put(ifc_dev_np);
 	if (!wcd->slim_ifc_dev) {
 		dev_err(dev, "Unable to get SLIM Interface device\n");
 		return -EINVAL;
diff --git a/sound/soc/codecs/wm5102.c b/sound/soc/codecs/wm5102.c
index 4466e195b66d..b32e8313954d 100644
--- a/sound/soc/codecs/wm5102.c
+++ b/sound/soc/codecs/wm5102.c
@@ -646,6 +646,8 @@ static int wm5102_adsp_power_ev(struct snd_soc_dapm_widget *w,
 				return ret;
 			}
 		}
+
+		wm_adsp2_set_dspclk(w, v);
 		break;
 
 	case SND_SOC_DAPM_POST_PMD:
@@ -659,7 +661,7 @@ static int wm5102_adsp_power_ev(struct snd_soc_dapm_widget *w,
 		break;
 	}
 
-	return wm_adsp2_early_event(w, kcontrol, event, v);
+	return wm_adsp_early_event(w, kcontrol, event);
 }
 
 static int wm5102_out_comp_coeff_get(struct snd_kcontrol *kcontrol,
diff --git a/sound/soc/codecs/wm5110.c b/sound/soc/codecs/wm5110.c
index b25877fa529d..1f500cc8d96a 100644
--- a/sound/soc/codecs/wm5110.c
+++ b/sound/soc/codecs/wm5110.c
@@ -211,7 +211,9 @@ static int wm5110_adsp_power_ev(struct snd_soc_dapm_widget *w,
 
 	v = (v & ARIZONA_SYSCLK_FREQ_MASK) >> ARIZONA_SYSCLK_FREQ_SHIFT;
 
-	return wm_adsp2_early_event(w, kcontrol, event, v);
+	wm_adsp2_set_dspclk(w, v);
+
+	return wm_adsp_early_event(w, kcontrol, event);
 }
 
 static const struct reg_sequence wm5110_no_dre_left_enable[] = {
diff --git a/sound/soc/codecs/wm_adsp.c b/sound/soc/codecs/wm_adsp.c
index b0b48eb9c7c9..b26e6b825a90 100644
--- a/sound/soc/codecs/wm_adsp.c
+++ b/sound/soc/codecs/wm_adsp.c
@@ -227,6 +227,89 @@
  */
 #define WM_ADSP_FW_EVENT_SHUTDOWN            0x000001
 
+/*
+ * HALO system info
+ */
+#define HALO_AHBM_WINDOW_DEBUG_0             0x02040
+#define HALO_AHBM_WINDOW_DEBUG_1             0x02044
+
+/*
+ * HALO core
+ */
+#define HALO_SCRATCH1                        0x005c0
+#define HALO_SCRATCH2                        0x005c8
+#define HALO_SCRATCH3                        0x005d0
+#define HALO_SCRATCH4                        0x005d8
+#define HALO_CCM_CORE_CONTROL                0x41000
+#define HALO_CORE_SOFT_RESET                 0x00010
+#define HALO_WDT_CONTROL                     0x47000
+
+/*
+ * HALO MPU banks
+ */
+#define HALO_MPU_XMEM_ACCESS_0               0x43000
+#define HALO_MPU_YMEM_ACCESS_0               0x43004
+#define HALO_MPU_WINDOW_ACCESS_0             0x43008
+#define HALO_MPU_XREG_ACCESS_0               0x4300C
+#define HALO_MPU_YREG_ACCESS_0               0x43014
+#define HALO_MPU_XMEM_ACCESS_1               0x43018
+#define HALO_MPU_YMEM_ACCESS_1               0x4301C
+#define HALO_MPU_WINDOW_ACCESS_1             0x43020
+#define HALO_MPU_XREG_ACCESS_1               0x43024
+#define HALO_MPU_YREG_ACCESS_1               0x4302C
+#define HALO_MPU_XMEM_ACCESS_2               0x43030
+#define HALO_MPU_YMEM_ACCESS_2               0x43034
+#define HALO_MPU_WINDOW_ACCESS_2             0x43038
+#define HALO_MPU_XREG_ACCESS_2               0x4303C
+#define HALO_MPU_YREG_ACCESS_2               0x43044
+#define HALO_MPU_XMEM_ACCESS_3               0x43048
+#define HALO_MPU_YMEM_ACCESS_3               0x4304C
+#define HALO_MPU_WINDOW_ACCESS_3             0x43050
+#define HALO_MPU_XREG_ACCESS_3               0x43054
+#define HALO_MPU_YREG_ACCESS_3               0x4305C
+#define HALO_MPU_XM_VIO_ADDR                 0x43100
+#define HALO_MPU_XM_VIO_STATUS               0x43104
+#define HALO_MPU_YM_VIO_ADDR                 0x43108
+#define HALO_MPU_YM_VIO_STATUS               0x4310C
+#define HALO_MPU_PM_VIO_ADDR                 0x43110
+#define HALO_MPU_PM_VIO_STATUS               0x43114
+#define HALO_MPU_LOCK_CONFIG                 0x43140
+
+/*
+ * HALO_AHBM_WINDOW_DEBUG_1
+ */
+#define HALO_AHBM_CORE_ERR_ADDR_MASK         0x0fffff00
+#define HALO_AHBM_CORE_ERR_ADDR_SHIFT                 8
+#define HALO_AHBM_FLAGS_ERR_MASK             0x000000ff
+
+/*
+ * HALO_CCM_CORE_CONTROL
+ */
+#define HALO_CORE_EN                        0x00000001
+
+/*
+ * HALO_CORE_SOFT_RESET
+ */
+#define HALO_CORE_SOFT_RESET_MASK           0x00000001
+
+/*
+ * HALO_WDT_CONTROL
+ */
+#define HALO_WDT_EN_MASK                    0x00000001
+
+/*
+ * HALO_MPU_?M_VIO_STATUS
+ */
+#define HALO_MPU_VIO_STS_MASK               0x007e0000
+#define HALO_MPU_VIO_STS_SHIFT                      17
+#define HALO_MPU_VIO_ERR_WR_MASK            0x00008000
+#define HALO_MPU_VIO_ERR_SRC_MASK           0x00007fff
+#define HALO_MPU_VIO_ERR_SRC_SHIFT                   0
+
+static struct wm_adsp_ops wm_adsp1_ops;
+static struct wm_adsp_ops wm_adsp2_ops[];
+static struct wm_adsp_ops wm_halo_ops;
+
 struct wm_adsp_buf {
 	struct list_head list;
 	void *buf;
@@ -306,6 +389,12 @@ struct wm_adsp_system_config_xm_hdr {
 	__be32 build_job_number;
 };
 
+struct wm_halo_system_config_xm_hdr {
+	__be32 halo_heartbeat;
+	__be32 build_job_name[3];
+	__be32 build_job_number;
+};
+
 struct wm_adsp_alg_xm_struct {
 	__be32 magic;
 	__be32 smoothing;
@@ -532,12 +621,18 @@ static const char *wm_adsp_mem_region_name(unsigned int type)
 	switch (type) {
 	case WMFW_ADSP1_PM:
 		return "PM";
+	case WMFW_HALO_PM_PACKED:
+		return "PM_PACKED";
 	case WMFW_ADSP1_DM:
 		return "DM";
 	case WMFW_ADSP2_XM:
 		return "XM";
+	case WMFW_HALO_XM_PACKED:
+		return "XM_PACKED";
 	case WMFW_ADSP2_YM:
 		return "YM";
+	case WMFW_HALO_YM_PACKED:
+		return "YM_PACKED";
 	case WMFW_ADSP1_ZM:
 		return "ZM";
 	default:
@@ -769,17 +864,12 @@ static struct wm_adsp_region const *wm_adsp_find_region(struct wm_adsp *dsp,
 static unsigned int wm_adsp_region_to_reg(struct wm_adsp_region const *mem,
 					  unsigned int offset)
 {
-	if (WARN_ON(!mem))
-		return offset;
 	switch (mem->type) {
 	case WMFW_ADSP1_PM:
 		return mem->base + (offset * 3);
 	case WMFW_ADSP1_DM:
-		return mem->base + (offset * 2);
 	case WMFW_ADSP2_XM:
-		return mem->base + (offset * 2);
 	case WMFW_ADSP2_YM:
-		return mem->base + (offset * 2);
 	case WMFW_ADSP1_ZM:
 		return mem->base + (offset * 2);
 	default:
@@ -788,49 +878,72 @@ static unsigned int wm_adsp_region_to_reg(struct wm_adsp_region const *mem,
 	}
 }
 
-static void wm_adsp2_show_fw_status(struct wm_adsp *dsp)
+static unsigned int wm_halo_region_to_reg(struct wm_adsp_region const *mem,
+					  unsigned int offset)
+{
+	switch (mem->type) {
+	case WMFW_ADSP2_XM:
+	case WMFW_ADSP2_YM:
+		return mem->base + (offset * 4);
+	case WMFW_HALO_XM_PACKED:
+	case WMFW_HALO_YM_PACKED:
+		return (mem->base + (offset * 3)) & ~0x3;
+	case WMFW_HALO_PM_PACKED:
+		return mem->base + (offset * 5);
+	default:
+		WARN(1, "Unknown memory region type");
+		return offset;
+	}
+}
+
+static void wm_adsp_read_fw_status(struct wm_adsp *dsp,
+				   int noffs, unsigned int *offs)
 {
-	unsigned int scratch[4];
-	unsigned int addr = dsp->base + ADSP2_SCRATCH0;
 	unsigned int i;
 	int ret;
 
-	for (i = 0; i < ARRAY_SIZE(scratch); ++i) {
-		ret = regmap_read(dsp->regmap, addr + i, &scratch[i]);
+	for (i = 0; i < noffs; ++i) {
+		ret = regmap_read(dsp->regmap, dsp->base + offs[i], &offs[i]);
 		if (ret) {
 			adsp_err(dsp, "Failed to read SCRATCH%u: %d\n", i, ret);
 			return;
 		}
 	}
+}
+
+static void wm_adsp2_show_fw_status(struct wm_adsp *dsp)
+{
+	unsigned int offs[] = {
+		ADSP2_SCRATCH0, ADSP2_SCRATCH1, ADSP2_SCRATCH2, ADSP2_SCRATCH3,
+	};
+
+	wm_adsp_read_fw_status(dsp, ARRAY_SIZE(offs), offs);
 
 	adsp_dbg(dsp, "FW SCRATCH 0:0x%x 1:0x%x 2:0x%x 3:0x%x\n",
-		 scratch[0], scratch[1], scratch[2], scratch[3]);
+		 offs[0], offs[1], offs[2], offs[3]);
 }
 
 static void wm_adsp2v2_show_fw_status(struct wm_adsp *dsp)
 {
-	unsigned int scratch[2];
-	int ret;
+	unsigned int offs[] = { ADSP2V2_SCRATCH0_1, ADSP2V2_SCRATCH2_3 };
 
-	ret = regmap_read(dsp->regmap, dsp->base + ADSP2V2_SCRATCH0_1,
-			  &scratch[0]);
-	if (ret) {
-		adsp_err(dsp, "Failed to read SCRATCH0_1: %d\n", ret);
-		return;
-	}
+	wm_adsp_read_fw_status(dsp, ARRAY_SIZE(offs), offs);
 
-	ret = regmap_read(dsp->regmap, dsp->base + ADSP2V2_SCRATCH2_3,
-			  &scratch[1]);
-	if (ret) {
-		adsp_err(dsp, "Failed to read SCRATCH2_3: %d\n", ret);
-		return;
-	}
+	adsp_dbg(dsp, "FW SCRATCH 0:0x%x 1:0x%x 2:0x%x 3:0x%x\n",
+		 offs[0] & 0xFFFF, offs[0] >> 16,
+		 offs[1] & 0xFFFF, offs[1] >> 16);
+}
+
+static void wm_halo_show_fw_status(struct wm_adsp *dsp)
+{
+	unsigned int offs[] = {
+		HALO_SCRATCH1, HALO_SCRATCH2, HALO_SCRATCH3, HALO_SCRATCH4,
+	};
+
+	wm_adsp_read_fw_status(dsp, ARRAY_SIZE(offs), offs);
 
 	adsp_dbg(dsp, "FW SCRATCH 0:0x%x 1:0x%x 2:0x%x 3:0x%x\n",
-		 scratch[0] & 0xFFFF,
-		 scratch[0] >> 16,
-		 scratch[1] & 0xFFFF,
-		 scratch[1] >> 16);
+		 offs[0], offs[1], offs[2], offs[3]);
 }
 
 static inline struct wm_coeff_ctl *bytes_ext_to_ctl(struct soc_bytes_ext *ext)
@@ -851,7 +964,7 @@ static int wm_coeff_base_reg(struct wm_coeff_ctl *ctl, unsigned int *reg)
 		return -EINVAL;
 	}
 
-	*reg = wm_adsp_region_to_reg(mem, ctl->alg_region.base + ctl->offset);
+	*reg = dsp->ops->region_to_reg(mem, ctl->alg_region.base + ctl->offset);
 
 	return 0;
 }
@@ -1339,28 +1452,33 @@ static int wm_adsp_create_control(struct wm_adsp *dsp,
 	case 1:
 		snprintf(name, SNDRV_CTL_ELEM_ID_NAME_MAXLEN, "%s %s %x",
 			 dsp->name, region_name, alg_region->alg);
+		subname = NULL; /* don't append subname */
 		break;
-	default:
+	case 2:
 		ret = snprintf(name, SNDRV_CTL_ELEM_ID_NAME_MAXLEN,
 				"%s%c %.12s %x", dsp->name, *region_name,
 				wm_adsp_fw_text[dsp->fw], alg_region->alg);
+		break;
+	default:
+		ret = snprintf(name, SNDRV_CTL_ELEM_ID_NAME_MAXLEN,
+				"%s %.12s %x", dsp->name,
+				wm_adsp_fw_text[dsp->fw], alg_region->alg);
+		break;
+	}
 
-		/* Truncate the subname from the start if it is too long */
-		if (subname) {
-			int avail = SNDRV_CTL_ELEM_ID_NAME_MAXLEN - ret - 2;
-			int skip = 0;
+	if (subname) {
+		int avail = SNDRV_CTL_ELEM_ID_NAME_MAXLEN - ret - 2;
+		int skip = 0;
 
-			if (dsp->component->name_prefix)
-				avail -= strlen(dsp->component->name_prefix) + 1;
+		if (dsp->component->name_prefix)
+			avail -= strlen(dsp->component->name_prefix) + 1;
 
-			if (subname_len > avail)
-				skip = subname_len - avail;
+		/* Truncate the subname from the start if it is too long */
+		if (subname_len > avail)
+			skip = subname_len - avail;
 
-			snprintf(name + ret,
-				 SNDRV_CTL_ELEM_ID_NAME_MAXLEN - ret, " %.*s",
-				 subname_len - skip, subname + skip);
-		}
-		break;
+		snprintf(name + ret, SNDRV_CTL_ELEM_ID_NAME_MAXLEN - ret,
+			 " %.*s", subname_len - skip, subname + skip);
 	}
 
 	list_for_each_entry(ctl, &dsp->ctl_list, list) {
@@ -1647,6 +1765,62 @@ static int wm_adsp_parse_coeff(struct wm_adsp *dsp,
 	return 0;
 }
 
+static unsigned int wm_adsp1_parse_sizes(struct wm_adsp *dsp,
+					 const char * const file,
+					 unsigned int pos,
+					 const struct firmware *firmware)
+{
+	const struct wmfw_adsp1_sizes *adsp1_sizes;
+
+	adsp1_sizes = (void *)&firmware->data[pos];
+
+	adsp_dbg(dsp, "%s: %d DM, %d PM, %d ZM\n", file,
+		 le32_to_cpu(adsp1_sizes->dm), le32_to_cpu(adsp1_sizes->pm),
+		 le32_to_cpu(adsp1_sizes->zm));
+
+	return pos + sizeof(*adsp1_sizes);
+}
+
+static unsigned int wm_adsp2_parse_sizes(struct wm_adsp *dsp,
+					 const char * const file,
+					 unsigned int pos,
+					 const struct firmware *firmware)
+{
+	const struct wmfw_adsp2_sizes *adsp2_sizes;
+
+	adsp2_sizes = (void *)&firmware->data[pos];
+
+	adsp_dbg(dsp, "%s: %d XM, %d YM %d PM, %d ZM\n", file,
+		 le32_to_cpu(adsp2_sizes->xm), le32_to_cpu(adsp2_sizes->ym),
+		 le32_to_cpu(adsp2_sizes->pm), le32_to_cpu(adsp2_sizes->zm));
+
+	return pos + sizeof(*adsp2_sizes);
+}
+
+static bool wm_adsp_validate_version(struct wm_adsp *dsp, unsigned int version)
+{
+	switch (version) {
+	case 0:
+		adsp_warn(dsp, "Deprecated file format %d\n", version);
+		return true;
+	case 1:
+	case 2:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static bool wm_halo_validate_version(struct wm_adsp *dsp, unsigned int version)
+{
+	switch (version) {
+	case 3:
+		return true;
+	default:
+		return false;
+	}
+}
+
 static int wm_adsp_load(struct wm_adsp *dsp)
 {
 	LIST_HEAD(buf_list);
@@ -1655,7 +1829,6 @@ static int wm_adsp_load(struct wm_adsp *dsp)
 	unsigned int pos = 0;
 	const struct wmfw_header *header;
 	const struct wmfw_adsp1_sizes *adsp1_sizes;
-	const struct wmfw_adsp2_sizes *adsp2_sizes;
 	const struct wmfw_footer *footer;
 	const struct wmfw_region *region;
 	const struct wm_adsp_region *mem;
@@ -1664,7 +1837,7 @@ static int wm_adsp_load(struct wm_adsp *dsp)
 	struct wm_adsp_buf *buf;
 	unsigned int reg;
 	int regions = 0;
-	int ret, offset, type, sizes;
+	int ret, offset, type;
 
 	file = kzalloc(PAGE_SIZE, GFP_KERNEL);
 	if (file == NULL)
@@ -1695,15 +1868,7 @@ static int wm_adsp_load(struct wm_adsp *dsp)
 		goto out_fw;
 	}
 
-	switch (header->ver) {
-	case 0:
-		adsp_warn(dsp, "%s: Depreciated file format %d\n",
-			  file, header->ver);
-		break;
-	case 1:
-	case 2:
-		break;
-	default:
+	if (!dsp->ops->validate_version(dsp, header->ver)) {
 		adsp_err(dsp, "%s: unknown file format %d\n",
 			 file, header->ver);
 		goto out_fw;
@@ -1718,39 +1883,13 @@ static int wm_adsp_load(struct wm_adsp *dsp)
 		goto out_fw;
 	}
 
-	switch (dsp->type) {
-	case WMFW_ADSP1:
-		pos = sizeof(*header) + sizeof(*adsp1_sizes) + sizeof(*footer);
-		adsp1_sizes = (void *)&(header[1]);
-		footer = (void *)&(adsp1_sizes[1]);
-		sizes = sizeof(*adsp1_sizes);
+	pos = sizeof(*header);
+	pos = dsp->ops->parse_sizes(dsp, file, pos, firmware);
 
-		adsp_dbg(dsp, "%s: %d DM, %d PM, %d ZM\n",
-			 file, le32_to_cpu(adsp1_sizes->dm),
-			 le32_to_cpu(adsp1_sizes->pm),
-			 le32_to_cpu(adsp1_sizes->zm));
-		break;
-
-	case WMFW_ADSP2:
-		pos = sizeof(*header) + sizeof(*adsp2_sizes) + sizeof(*footer);
-		adsp2_sizes = (void *)&(header[1]);
-		footer = (void *)&(adsp2_sizes[1]);
-		sizes = sizeof(*adsp2_sizes);
-
-		adsp_dbg(dsp, "%s: %d XM, %d YM %d PM, %d ZM\n",
-			 file, le32_to_cpu(adsp2_sizes->xm),
-			 le32_to_cpu(adsp2_sizes->ym),
-			 le32_to_cpu(adsp2_sizes->pm),
-			 le32_to_cpu(adsp2_sizes->zm));
-		break;
-
-	default:
-		WARN(1, "Unknown DSP type");
-		goto out_fw;
-	}
+	footer = (void *)&firmware->data[pos];
+	pos += sizeof(*footer);
 
-	if (le32_to_cpu(header->len) != sizeof(*header) +
-	    sizes + sizeof(*footer)) {
+	if (le32_to_cpu(header->len) != pos) {
 		adsp_err(dsp, "%s: unexpected header length %d\n",
 			 file, le32_to_cpu(header->len));
 		goto out_fw;
@@ -1767,7 +1906,6 @@ static int wm_adsp_load(struct wm_adsp *dsp)
 		text = NULL;
 		offset = le32_to_cpu(region->offset) & 0xffffff;
 		type = be32_to_cpu(region->type) & 0xff;
-		mem = wm_adsp_find_region(dsp, type);
 
 		switch (type) {
 		case WMFW_NAME_TEXT:
@@ -1795,8 +1933,17 @@ static int wm_adsp_load(struct wm_adsp *dsp)
 		case WMFW_ADSP2_XM:
 		case WMFW_ADSP2_YM:
 		case WMFW_ADSP1_ZM:
+		case WMFW_HALO_PM_PACKED:
+		case WMFW_HALO_XM_PACKED:
+		case WMFW_HALO_YM_PACKED:
+			mem = wm_adsp_find_region(dsp, type);
+			if (!mem) {
+				adsp_err(dsp, "No region of type: %x\n", type);
+				goto out_fw;
+			}
+
 			region_name = wm_adsp_mem_region_name(type);
-			reg = wm_adsp_region_to_reg(mem, offset);
+			reg = dsp->ops->region_to_reg(mem, offset);
 			break;
 		default:
 			adsp_warn(dsp,
@@ -1909,7 +2056,7 @@ static void *wm_adsp_read_algs(struct wm_adsp *dsp, size_t n_algs,
 	}
 
 	/* Read the terminator first to validate the length */
-	reg = wm_adsp_region_to_reg(mem, pos + len);
+	reg = dsp->ops->region_to_reg(mem, pos + len);
 
 	ret = regmap_raw_read(dsp->regmap, reg, &val, sizeof(val));
 	if (ret != 0) {
@@ -1929,7 +2076,7 @@ static void *wm_adsp_read_algs(struct wm_adsp *dsp, size_t n_algs,
 	if (!alg)
 		return ERR_PTR(-ENOMEM);
 
-	reg = wm_adsp_region_to_reg(mem, pos);
+	reg = dsp->ops->region_to_reg(mem, pos);
 
 	ret = regmap_raw_read(dsp->regmap, reg, alg, len);
 	if (ret != 0) {
@@ -1989,6 +2136,47 @@ static void wm_adsp_free_alg_regions(struct wm_adsp *dsp)
 	}
 }
 
+static void wmfw_parse_id_header(struct wm_adsp *dsp,
+				 struct wmfw_id_hdr *fw, int nalgs)
+{
+	dsp->fw_id = be32_to_cpu(fw->id);
+	dsp->fw_id_version = be32_to_cpu(fw->ver);
+
+	adsp_info(dsp, "Firmware: %x v%d.%d.%d, %d algorithms\n",
+		  dsp->fw_id, (dsp->fw_id_version & 0xff0000) >> 16,
+		  (dsp->fw_id_version & 0xff00) >> 8, dsp->fw_id_version & 0xff,
+		  nalgs);
+}
+
+static void wmfw_v3_parse_id_header(struct wm_adsp *dsp,
+				    struct wmfw_v3_id_hdr *fw, int nalgs)
+{
+	dsp->fw_id = be32_to_cpu(fw->id);
+	dsp->fw_id_version = be32_to_cpu(fw->ver);
+	dsp->fw_vendor_id = be32_to_cpu(fw->vendor_id);
+
+	adsp_info(dsp, "Firmware: %x vendor: 0x%x v%d.%d.%d, %d algorithms\n",
+		  dsp->fw_id, dsp->fw_vendor_id,
+		  (dsp->fw_id_version & 0xff0000) >> 16,
+		  (dsp->fw_id_version & 0xff00) >> 8, dsp->fw_id_version & 0xff,
+		  nalgs);
+}
+
+static int wm_adsp_create_regions(struct wm_adsp *dsp, __be32 id, int nregions,
+				int *type, __be32 *base)
+{
+	struct wm_adsp_alg_region *alg_region;
+	int i;
+
+	for (i = 0; i < nregions; i++) {
+		alg_region = wm_adsp_create_region(dsp, type[i], id, base[i]);
+		if (IS_ERR(alg_region))
+			return PTR_ERR(alg_region);
+	}
+
+	return 0;
+}
+
 static int wm_adsp1_setup_algs(struct wm_adsp *dsp)
 {
 	struct wmfw_adsp1_id_hdr adsp1_id;
@@ -2012,13 +2200,8 @@ static int wm_adsp1_setup_algs(struct wm_adsp *dsp)
 	}
 
 	n_algs = be32_to_cpu(adsp1_id.n_algs);
-	dsp->fw_id = be32_to_cpu(adsp1_id.fw.id);
-	adsp_info(dsp, "Firmware: %x v%d.%d.%d, %zu algorithms\n",
-		  dsp->fw_id,
-		  (be32_to_cpu(adsp1_id.fw.ver) & 0xff0000) >> 16,
-		  (be32_to_cpu(adsp1_id.fw.ver) & 0xff00) >> 8,
-		  be32_to_cpu(adsp1_id.fw.ver) & 0xff,
-		  n_algs);
+
+	wmfw_parse_id_header(dsp, &adsp1_id.fw, n_algs);
 
 	alg_region = wm_adsp_create_region(dsp, WMFW_ADSP1_ZM,
 					   adsp1_id.fw.id, adsp1_id.zm);
@@ -2118,14 +2301,8 @@ static int wm_adsp2_setup_algs(struct wm_adsp *dsp)
 	}
 
 	n_algs = be32_to_cpu(adsp2_id.n_algs);
-	dsp->fw_id = be32_to_cpu(adsp2_id.fw.id);
-	dsp->fw_id_version = be32_to_cpu(adsp2_id.fw.ver);
-	adsp_info(dsp, "Firmware: %x v%d.%d.%d, %zu algorithms\n",
-		  dsp->fw_id,
-		  (dsp->fw_id_version & 0xff0000) >> 16,
-		  (dsp->fw_id_version & 0xff00) >> 8,
-		  dsp->fw_id_version & 0xff,
-		  n_algs);
+
+	wmfw_parse_id_header(dsp, &adsp2_id.fw, n_algs);
 
 	alg_region = wm_adsp_create_region(dsp, WMFW_ADSP2_XM,
 					   adsp2_id.fw.id, adsp2_id.xm);
@@ -2230,6 +2407,78 @@ out:
 	return ret;
 }
 
+static int wm_halo_create_regions(struct wm_adsp *dsp, __be32 id,
+				  __be32 xm_base, __be32 ym_base)
+{
+	int types[] = {
+		WMFW_ADSP2_XM, WMFW_HALO_XM_PACKED,
+		WMFW_ADSP2_YM, WMFW_HALO_YM_PACKED
+	};
+	__be32 bases[] = { xm_base, xm_base, ym_base, ym_base };
+
+	return wm_adsp_create_regions(dsp, id, ARRAY_SIZE(types), types, bases);
+}
+
+static int wm_halo_setup_algs(struct wm_adsp *dsp)
+{
+	struct wmfw_halo_id_hdr halo_id;
+	struct wmfw_halo_alg_hdr *halo_alg;
+	const struct wm_adsp_region *mem;
+	unsigned int pos, len;
+	size_t n_algs;
+	int i, ret;
+
+	mem = wm_adsp_find_region(dsp, WMFW_ADSP2_XM);
+	if (WARN_ON(!mem))
+		return -EINVAL;
+
+	ret = regmap_raw_read(dsp->regmap, mem->base, &halo_id,
+			      sizeof(halo_id));
+	if (ret != 0) {
+		adsp_err(dsp, "Failed to read algorithm info: %d\n",
+			 ret);
+		return ret;
+	}
+
+	n_algs = be32_to_cpu(halo_id.n_algs);
+
+	wmfw_v3_parse_id_header(dsp, &halo_id.fw, n_algs);
+
+	ret = wm_halo_create_regions(dsp, halo_id.fw.id,
+				     halo_id.xm_base, halo_id.ym_base);
+	if (ret)
+		return ret;
+
+	/* Calculate offset and length in DSP words */
+	pos = sizeof(halo_id) / sizeof(u32);
+	len = (sizeof(*halo_alg) * n_algs) / sizeof(u32);
+
+	halo_alg = wm_adsp_read_algs(dsp, n_algs, mem, pos, len);
+	if (IS_ERR(halo_alg))
+		return PTR_ERR(halo_alg);
+
+	for (i = 0; i < n_algs; i++) {
+		adsp_info(dsp,
+			  "%d: ID %x v%d.%d.%d XM@%x YM@%x\n",
+			  i, be32_to_cpu(halo_alg[i].alg.id),
+			  (be32_to_cpu(halo_alg[i].alg.ver) & 0xff0000) >> 16,
+			  (be32_to_cpu(halo_alg[i].alg.ver) & 0xff00) >> 8,
+			  be32_to_cpu(halo_alg[i].alg.ver) & 0xff,
+			  be32_to_cpu(halo_alg[i].xm_base),
+			  be32_to_cpu(halo_alg[i].ym_base));
+
+		ret = wm_halo_create_regions(dsp, halo_alg[i].alg.id,
+					     halo_alg[i].xm_base,
+					     halo_alg[i].ym_base);
+		if (ret)
+			goto out;
+	}
+
+out:
+	kfree(halo_alg);
+	return ret;
+}
+
 static int wm_adsp_load_coeff(struct wm_adsp *dsp)
 {
 	LIST_HEAD(buf_list);
@@ -2324,7 +2573,7 @@ static int wm_adsp_load_coeff(struct wm_adsp *dsp)
 					adsp_err(dsp, "No ZM\n");
 					break;
 				}
-				reg = wm_adsp_region_to_reg(mem, 0);
+				reg = dsp->ops->region_to_reg(mem, 0);
 
 			} else {
 				region_name = "register";
@@ -2336,6 +2585,9 @@ static int wm_adsp_load_coeff(struct wm_adsp *dsp)
 		case WMFW_ADSP1_ZM:
 		case WMFW_ADSP2_XM:
 		case WMFW_ADSP2_YM:
+		case WMFW_HALO_XM_PACKED:
+		case WMFW_HALO_YM_PACKED:
+		case WMFW_HALO_PM_PACKED:
 			adsp_dbg(dsp, "%s.%d: %d bytes in %x for %x\n",
 				 file, blocks, le32_to_cpu(blk->len),
 				 type, le32_to_cpu(blk->id));
@@ -2350,7 +2602,7 @@ static int wm_adsp_load_coeff(struct wm_adsp *dsp)
 						le32_to_cpu(blk->id));
 			if (alg_region) {
 				reg = alg_region->base;
-				reg = wm_adsp_region_to_reg(mem, reg);
+				reg = dsp->ops->region_to_reg(mem, reg);
 				reg += offset;
 			} else {
 				adsp_err(dsp, "No %x for algorithm %x\n",
@@ -2464,6 +2716,8 @@ static int wm_adsp_common_init(struct wm_adsp *dsp)
 
 int wm_adsp1_init(struct wm_adsp *dsp)
 {
+	dsp->ops = &wm_adsp1_ops;
+
 	return wm_adsp_common_init(dsp);
 }
 EXPORT_SYMBOL_GPL(wm_adsp1_init);
@@ -2583,23 +2837,11 @@ err_mutex:
 }
 EXPORT_SYMBOL_GPL(wm_adsp1_event);
 
-static int wm_adsp2_ena(struct wm_adsp *dsp)
+static int wm_adsp2v2_enable_core(struct wm_adsp *dsp)
 {
 	unsigned int val;
 	int ret, count;
 
-	switch (dsp->rev) {
-	case 0:
-		ret = regmap_update_bits_async(dsp->regmap,
-					       dsp->base + ADSP2_CONTROL,
-					       ADSP2_SYS_ENA, ADSP2_SYS_ENA);
-		if (ret != 0)
-			return ret;
-		break;
-	default:
-		break;
-	}
-
 	/* Wait for the RAM to start, should be near instantaneous */
 	for (count = 0; count < 10; ++count) {
 		ret = regmap_read(dsp->regmap, dsp->base + ADSP2_STATUS1, &val);
@@ -2622,7 +2864,78 @@ static int wm_adsp2_ena(struct wm_adsp *dsp)
 	return 0;
 }
 
-static void wm_adsp2_boot_work(struct work_struct *work)
+static int wm_adsp2_enable_core(struct wm_adsp *dsp)
+{
+	int ret;
+
+	ret = regmap_update_bits_async(dsp->regmap, dsp->base + ADSP2_CONTROL,
+				       ADSP2_SYS_ENA, ADSP2_SYS_ENA);
+	if (ret != 0)
+		return ret;
+
+	return wm_adsp2v2_enable_core(dsp);
+}
+
+static int wm_adsp2_lock(struct wm_adsp *dsp, unsigned int lock_regions)
+{
+	struct regmap *regmap = dsp->regmap;
+	unsigned int code0, code1, lock_reg;
+
+	if (!(lock_regions & WM_ADSP2_REGION_ALL))
+		return 0;
+
+	lock_regions &= WM_ADSP2_REGION_ALL;
+	lock_reg = dsp->base + ADSP2_LOCK_REGION_1_LOCK_REGION_0;
+
+	while (lock_regions) {
+		code0 = code1 = 0;
+		if (lock_regions & BIT(0)) {
+			code0 = ADSP2_LOCK_CODE_0;
+			code1 = ADSP2_LOCK_CODE_1;
+		}
+		if (lock_regions & BIT(1)) {
+			code0 |= ADSP2_LOCK_CODE_0 << ADSP2_LOCK_REGION_SHIFT;
+			code1 |= ADSP2_LOCK_CODE_1 << ADSP2_LOCK_REGION_SHIFT;
+		}
+		regmap_write(regmap, lock_reg, code0);
+		regmap_write(regmap, lock_reg, code1);
+		lock_regions >>= 2;
+		lock_reg += 2;
+	}
+
+	return 0;
+}
+
+static int wm_adsp2_enable_memory(struct wm_adsp *dsp)
+{
+	return regmap_update_bits(dsp->regmap, dsp->base + ADSP2_CONTROL,
+				  ADSP2_MEM_ENA, ADSP2_MEM_ENA);
+}
+
+static void wm_adsp2_disable_memory(struct wm_adsp *dsp)
+{
+	regmap_update_bits(dsp->regmap, dsp->base + ADSP2_CONTROL,
+			   ADSP2_MEM_ENA, 0);
+}
+
+static void wm_adsp2_disable_core(struct wm_adsp *dsp)
+{
+	regmap_write(dsp->regmap, dsp->base + ADSP2_RDMA_CONFIG_1, 0);
+	regmap_write(dsp->regmap, dsp->base + ADSP2_WDMA_CONFIG_1, 0);
+	regmap_write(dsp->regmap, dsp->base + ADSP2_WDMA_CONFIG_2, 0);
+
+	regmap_update_bits(dsp->regmap, dsp->base + ADSP2_CONTROL,
+			   ADSP2_SYS_ENA, 0);
+}
+
+static void wm_adsp2v2_disable_core(struct wm_adsp *dsp)
+{
+	regmap_write(dsp->regmap, dsp->base + ADSP2_RDMA_CONFIG_1, 0);
+	regmap_write(dsp->regmap, dsp->base + ADSP2_WDMA_CONFIG_1, 0);
+	regmap_write(dsp->regmap, dsp->base + ADSP2V2_WDMA_CONFIG_2, 0);
+}
+
+static void wm_adsp_boot_work(struct work_struct *work)
 {
 	struct wm_adsp *dsp = container_of(work,
 					   struct wm_adsp,
@@ -2631,20 +2944,23 @@ static void wm_adsp2_boot_work(struct work_struct *work)
 
 	mutex_lock(&dsp->pwr_lock);
 
-	ret = regmap_update_bits(dsp->regmap, dsp->base + ADSP2_CONTROL,
-				 ADSP2_MEM_ENA, ADSP2_MEM_ENA);
-	if (ret != 0)
-		goto err_mutex;
+	if (dsp->ops->enable_memory) {
+		ret = dsp->ops->enable_memory(dsp);
+		if (ret != 0)
+			goto err_mutex;
+	}
 
-	ret = wm_adsp2_ena(dsp);
-	if (ret != 0)
-		goto err_mem;
+	if (dsp->ops->enable_core) {
+		ret = dsp->ops->enable_core(dsp);
+		if (ret != 0)
+			goto err_mem;
+	}
 
 	ret = wm_adsp_load(dsp);
 	if (ret != 0)
 		goto err_ena;
 
-	ret = wm_adsp2_setup_algs(dsp);
+	ret = dsp->ops->setup_algs(dsp);
 	if (ret != 0)
 		goto err_ena;
 
@@ -2657,17 +2973,8 @@ static void wm_adsp2_boot_work(struct work_struct *work)
 	if (ret != 0)
 		goto err_ena;
 
-	switch (dsp->rev) {
-	case 0:
-		/* Turn DSP back off until we are ready to run */
-		ret = regmap_update_bits(dsp->regmap, dsp->base + ADSP2_CONTROL,
-					 ADSP2_SYS_ENA, 0);
-		if (ret != 0)
-			goto err_ena;
-		break;
-	default:
-		break;
-	}
+	if (dsp->ops->disable_core)
+		dsp->ops->disable_core(dsp);
 
 	dsp->booted = true;
 
@@ -2676,35 +2983,62 @@ static void wm_adsp2_boot_work(struct work_struct *work)
 	return;
 
 err_ena:
-	regmap_update_bits(dsp->regmap, dsp->base + ADSP2_CONTROL,
-			   ADSP2_SYS_ENA | ADSP2_CORE_ENA | ADSP2_START, 0);
+	if (dsp->ops->disable_core)
+		dsp->ops->disable_core(dsp);
 err_mem:
-	regmap_update_bits(dsp->regmap, dsp->base + ADSP2_CONTROL,
-			   ADSP2_MEM_ENA, 0);
+	if (dsp->ops->disable_memory)
+		dsp->ops->disable_memory(dsp);
 err_mutex:
 	mutex_unlock(&dsp->pwr_lock);
 }
 
-static void wm_adsp2_set_dspclk(struct wm_adsp *dsp, unsigned int freq)
+static int wm_halo_configure_mpu(struct wm_adsp *dsp, unsigned int lock_regions)
+{
+	struct reg_sequence config[] = {
+		{ dsp->base + HALO_MPU_LOCK_CONFIG,     0x5555 },
+		{ dsp->base + HALO_MPU_LOCK_CONFIG,     0xAAAA },
+		{ dsp->base + HALO_MPU_XMEM_ACCESS_0,   0xFFFFFFFF },
+		{ dsp->base + HALO_MPU_YMEM_ACCESS_0,   0xFFFFFFFF },
+		{ dsp->base + HALO_MPU_WINDOW_ACCESS_0, lock_regions },
+		{ dsp->base + HALO_MPU_XREG_ACCESS_0,   lock_regions },
+		{ dsp->base + HALO_MPU_YREG_ACCESS_0,   lock_regions },
+		{ dsp->base + HALO_MPU_XMEM_ACCESS_1,   0xFFFFFFFF },
+		{ dsp->base + HALO_MPU_YMEM_ACCESS_1,   0xFFFFFFFF },
+		{ dsp->base + HALO_MPU_WINDOW_ACCESS_1, lock_regions },
+		{ dsp->base + HALO_MPU_XREG_ACCESS_1,   lock_regions },
+		{ dsp->base + HALO_MPU_YREG_ACCESS_1,   lock_regions },
+		{ dsp->base + HALO_MPU_XMEM_ACCESS_2,   0xFFFFFFFF },
+		{ dsp->base + HALO_MPU_YMEM_ACCESS_2,   0xFFFFFFFF },
+		{ dsp->base + HALO_MPU_WINDOW_ACCESS_2, lock_regions },
+		{ dsp->base + HALO_MPU_XREG_ACCESS_2,   lock_regions },
+		{ dsp->base + HALO_MPU_YREG_ACCESS_2,   lock_regions },
+		{ dsp->base + HALO_MPU_XMEM_ACCESS_3,   0xFFFFFFFF },
+		{ dsp->base + HALO_MPU_YMEM_ACCESS_3,   0xFFFFFFFF },
+		{ dsp->base + HALO_MPU_WINDOW_ACCESS_3, lock_regions },
+		{ dsp->base + HALO_MPU_XREG_ACCESS_3,   lock_regions },
+		{ dsp->base + HALO_MPU_YREG_ACCESS_3,   lock_regions },
+		{ dsp->base + HALO_MPU_LOCK_CONFIG,     0 },
+	};
+
+	return regmap_multi_reg_write(dsp->regmap, config, ARRAY_SIZE(config));
+}
+
+int wm_adsp2_set_dspclk(struct snd_soc_dapm_widget *w, unsigned int freq)
 {
+	struct snd_soc_component *component = snd_soc_dapm_to_component(w->dapm);
+	struct wm_adsp *dsps = snd_soc_component_get_drvdata(component);
+	struct wm_adsp *dsp = &dsps[w->shift];
 	int ret;
 
-	switch (dsp->rev) {
-	case 0:
-		ret = regmap_update_bits_async(dsp->regmap,
-					       dsp->base + ADSP2_CLOCKING,
-					       ADSP2_CLK_SEL_MASK,
-					       freq << ADSP2_CLK_SEL_SHIFT);
-		if (ret) {
-			adsp_err(dsp, "Failed to set clock rate: %d\n", ret);
-			return;
-		}
-		break;
-	default:
-		/* clock is handled by parent codec driver */
-		break;
-	}
+	ret = regmap_update_bits(dsp->regmap, dsp->base + ADSP2_CLOCKING,
+				 ADSP2_CLK_SEL_MASK,
+				 freq << ADSP2_CLK_SEL_SHIFT);
+	if (ret)
+		adsp_err(dsp, "Failed to set clock rate: %d\n", ret);
+
+	return ret;
 }
+EXPORT_SYMBOL_GPL(wm_adsp2_set_dspclk);
 
 int wm_adsp2_preloader_get(struct snd_kcontrol *kcontrol,
 			   struct snd_ctl_elem_value *ucontrol)
@@ -2751,19 +3085,18 @@ EXPORT_SYMBOL_GPL(wm_adsp2_preloader_put);
 
 static void wm_adsp_stop_watchdog(struct wm_adsp *dsp)
 {
-	switch (dsp->rev) {
-	case 0:
-	case 1:
-		return;
-	default:
-		regmap_update_bits(dsp->regmap, dsp->base + ADSP2_WATCHDOG,
-				   ADSP2_WDT_ENA_MASK, 0);
-	}
+	regmap_update_bits(dsp->regmap, dsp->base + ADSP2_WATCHDOG,
+			   ADSP2_WDT_ENA_MASK, 0);
 }
 
-int wm_adsp2_early_event(struct snd_soc_dapm_widget *w,
-			 struct snd_kcontrol *kcontrol, int event,
-			 unsigned int freq)
+static void wm_halo_stop_watchdog(struct wm_adsp *dsp)
+{
+	regmap_update_bits(dsp->regmap, dsp->base + HALO_WDT_CONTROL,
+			   HALO_WDT_EN_MASK, 0);
+}
+
+int wm_adsp_early_event(struct snd_soc_dapm_widget *w,
+			struct snd_kcontrol *kcontrol, int event)
 {
 	struct snd_soc_component *component = snd_soc_dapm_to_component(w->dapm);
 	struct wm_adsp *dsps = snd_soc_component_get_drvdata(component);
@@ -2772,7 +3105,6 @@ int wm_adsp2_early_event(struct snd_soc_dapm_widget *w,
 
 	switch (event) {
 	case SND_SOC_DAPM_PRE_PMU:
-		wm_adsp2_set_dspclk(dsp, freq);
 		queue_work(system_unbound_wq, &dsp->boot_work);
 		break;
 	case SND_SOC_DAPM_PRE_PMD:
@@ -2785,8 +3117,8 @@ int wm_adsp2_early_event(struct snd_soc_dapm_widget *w,
 
 		dsp->booted = false;
 
-		regmap_update_bits(dsp->regmap, dsp->base + ADSP2_CONTROL,
-				   ADSP2_MEM_ENA, 0);
+		if (dsp->ops->disable_memory)
+			dsp->ops->disable_memory(dsp);
 
 		list_for_each_entry(ctl, &dsp->ctl_list, list)
 			ctl->enabled = 0;
@@ -2803,10 +3135,23 @@ int wm_adsp2_early_event(struct snd_soc_dapm_widget *w,
 
 	return 0;
 }
-EXPORT_SYMBOL_GPL(wm_adsp2_early_event);
+EXPORT_SYMBOL_GPL(wm_adsp_early_event);
+
+static int wm_adsp2_start_core(struct wm_adsp *dsp)
+{
+	return regmap_update_bits(dsp->regmap, dsp->base + ADSP2_CONTROL,
+				 ADSP2_CORE_ENA | ADSP2_START,
+				 ADSP2_CORE_ENA | ADSP2_START);
+}
 
-int wm_adsp2_event(struct snd_soc_dapm_widget *w,
-		   struct snd_kcontrol *kcontrol, int event)
+static void wm_adsp2_stop_core(struct wm_adsp *dsp)
+{
+	regmap_update_bits(dsp->regmap, dsp->base + ADSP2_CONTROL,
+			   ADSP2_CORE_ENA | ADSP2_START, 0);
+}
+
+int wm_adsp_event(struct snd_soc_dapm_widget *w,
+		  struct snd_kcontrol *kcontrol, int event)
 {
 	struct snd_soc_component *component = snd_soc_dapm_to_component(w->dapm);
 	struct wm_adsp *dsps = snd_soc_component_get_drvdata(component);
@@ -2824,23 +3169,31 @@ int wm_adsp2_event(struct snd_soc_dapm_widget *w,
 			goto err;
 		}
 
-		ret = wm_adsp2_ena(dsp);
-		if (ret != 0)
-			goto err;
+		if (dsp->ops->enable_core) {
+			ret = dsp->ops->enable_core(dsp);
+			if (ret != 0)
+				goto err;
+		}
 
 		/* Sync set controls */
 		ret = wm_coeff_sync_controls(dsp);
 		if (ret != 0)
 			goto err;
 
-		wm_adsp2_lock(dsp, dsp->lock_regions);
+		if (dsp->ops->lock_memory) {
+			ret = dsp->ops->lock_memory(dsp, dsp->lock_regions);
+			if (ret != 0) {
+				adsp_err(dsp, "Error configuring MPU: %d\n",
+					 ret);
+				goto err;
+			}
+		}
 
-		ret = regmap_update_bits(dsp->regmap,
-					 dsp->base + ADSP2_CONTROL,
-					 ADSP2_CORE_ENA | ADSP2_START,
-					 ADSP2_CORE_ENA | ADSP2_START);
-		if (ret != 0)
-			goto err;
+		if (dsp->ops->start_core) {
+			ret = dsp->ops->start_core(dsp);
+			if (ret != 0)
+				goto err;
+		}
 
 		if (wm_adsp_fw[dsp->fw].num_caps != 0) {
 			ret = wm_adsp_buffer_init(dsp);
@@ -2851,56 +3204,27 @@ int wm_adsp2_event(struct snd_soc_dapm_widget *w,
 		dsp->running = true;
 
 		mutex_unlock(&dsp->pwr_lock);
-
 		break;
 
 	case SND_SOC_DAPM_PRE_PMD:
 		/* Tell the firmware to cleanup */
 		wm_adsp_signal_event_controls(dsp, WM_ADSP_FW_EVENT_SHUTDOWN);
 
-		wm_adsp_stop_watchdog(dsp);
+		if (dsp->ops->stop_watchdog)
+			dsp->ops->stop_watchdog(dsp);
 
 		/* Log firmware state, it can be useful for analysis */
-		switch (dsp->rev) {
-		case 0:
-			wm_adsp2_show_fw_status(dsp);
-			break;
-		default:
-			wm_adsp2v2_show_fw_status(dsp);
-			break;
-		}
+		if (dsp->ops->show_fw_status)
+			dsp->ops->show_fw_status(dsp);
 
 		mutex_lock(&dsp->pwr_lock);
 
 		dsp->running = false;
 
-		regmap_update_bits(dsp->regmap,
-				   dsp->base + ADSP2_CONTROL,
-				   ADSP2_CORE_ENA | ADSP2_START, 0);
-
-		/* Make sure DMAs are quiesced */
-		switch (dsp->rev) {
-		case 0:
-			regmap_write(dsp->regmap,
-				     dsp->base + ADSP2_RDMA_CONFIG_1, 0);
-			regmap_write(dsp->regmap,
-				     dsp->base + ADSP2_WDMA_CONFIG_1, 0);
-			regmap_write(dsp->regmap,
-				     dsp->base + ADSP2_WDMA_CONFIG_2, 0);
-
-			regmap_update_bits(dsp->regmap,
-					   dsp->base + ADSP2_CONTROL,
-					   ADSP2_SYS_ENA, 0);
-			break;
-		default:
-			regmap_write(dsp->regmap,
-				     dsp->base + ADSP2_RDMA_CONFIG_1, 0);
-			regmap_write(dsp->regmap,
-				     dsp->base + ADSP2_WDMA_CONFIG_1, 0);
-			regmap_write(dsp->regmap,
-				     dsp->base + ADSP2V2_WDMA_CONFIG_2, 0);
-			break;
-		}
+		if (dsp->ops->stop_core)
+			dsp->ops->stop_core(dsp);
+		if (dsp->ops->disable_core)
+			dsp->ops->disable_core(dsp);
 
 		if (wm_adsp_fw[dsp->fw].num_caps != 0)
 			wm_adsp_buffer_free(dsp);
@@ -2918,12 +3242,31 @@ int wm_adsp2_event(struct snd_soc_dapm_widget *w,
 
 	return 0;
 err:
-	regmap_update_bits(dsp->regmap, dsp->base + ADSP2_CONTROL,
-			   ADSP2_SYS_ENA | ADSP2_CORE_ENA | ADSP2_START, 0);
+	if (dsp->ops->stop_core)
+		dsp->ops->stop_core(dsp);
+	if (dsp->ops->disable_core)
+		dsp->ops->disable_core(dsp);
 	mutex_unlock(&dsp->pwr_lock);
 	return ret;
 }
-EXPORT_SYMBOL_GPL(wm_adsp2_event);
+EXPORT_SYMBOL_GPL(wm_adsp_event);
+
+static int wm_halo_start_core(struct wm_adsp *dsp)
+{
+	return regmap_update_bits(dsp->regmap,
+				  dsp->base + HALO_CCM_CORE_CONTROL,
+				  HALO_CORE_EN, HALO_CORE_EN);
+}
+
+static void wm_halo_stop_core(struct wm_adsp *dsp)
+{
+	regmap_update_bits(dsp->regmap, dsp->base + HALO_CCM_CORE_CONTROL,
+			   HALO_CORE_EN, 0);
+
+	/* reset halo core with CORE_SOFT_RESET */
+	regmap_update_bits(dsp->regmap, dsp->base + HALO_CORE_SOFT_RESET,
+			   HALO_CORE_SOFT_RESET_MASK, 1);
+}
 
 int wm_adsp2_component_probe(struct wm_adsp *dsp, struct snd_soc_component *component)
 {
@@ -2969,17 +3312,39 @@ int wm_adsp2_init(struct wm_adsp *dsp)
 				 "Failed to clear memory retention: %d\n", ret);
 			return ret;
 		}
+
+		dsp->ops = &wm_adsp2_ops[0];
+		break;
+	case 1:
+		dsp->ops = &wm_adsp2_ops[1];
 		break;
 	default:
+		dsp->ops = &wm_adsp2_ops[2];
 		break;
 	}
 
-	INIT_WORK(&dsp->boot_work, wm_adsp2_boot_work);
+	INIT_WORK(&dsp->boot_work, wm_adsp_boot_work);
 
 	return 0;
 }
 EXPORT_SYMBOL_GPL(wm_adsp2_init);
 
+int wm_halo_init(struct wm_adsp *dsp)
+{
+	int ret;
+
+	ret = wm_adsp_common_init(dsp);
+	if (ret)
+		return ret;
+
+	dsp->ops = &wm_halo_ops;
+
+	INIT_WORK(&dsp->boot_work, wm_adsp_boot_work);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(wm_halo_init);
+
 void wm_adsp2_remove(struct wm_adsp *dsp)
 {
 	struct wm_coeff_ctl *ctl;
@@ -3016,7 +3381,7 @@ static int wm_adsp_compr_attach(struct wm_adsp_compr *compr)
 		return -EINVAL;
 
 	compr->buf = buf;
-	compr->buf->compr = compr;
+	buf->compr = compr;
 
 	return 0;
 }
@@ -3224,7 +3589,7 @@ static int wm_adsp_read_data_block(struct wm_adsp *dsp, int mem_type,
 	if (!mem)
 		return -EINVAL;
 
-	reg = wm_adsp_region_to_reg(mem, mem_addr);
+	reg = dsp->ops->region_to_reg(mem, mem_addr);
 
 	ret = regmap_raw_read(dsp->regmap, reg, data,
 			      sizeof(*data) * num_words);
@@ -3252,7 +3617,7 @@ static int wm_adsp_write_data_word(struct wm_adsp *dsp, int mem_type,
 	if (!mem)
 		return -EINVAL;
 
-	reg = wm_adsp_region_to_reg(mem, mem_addr);
+	reg = dsp->ops->region_to_reg(mem, mem_addr);
 
 	data = cpu_to_be32(data & 0x00ffffffu);
 
@@ -3363,7 +3728,7 @@ static int wm_adsp_buffer_parse_legacy(struct wm_adsp *dsp)
 		return -ENOMEM;
 
 	alg_region = wm_adsp_find_alg_region(dsp, WMFW_ADSP2_XM, dsp->fw_id);
-	xmalg = sizeof(struct wm_adsp_system_config_xm_hdr) / sizeof(__be32);
+	xmalg = dsp->ops->sys_config_size / sizeof(__be32);
 
 	addr = alg_region->base + xmalg + ALG_XM_FIELD(magic);
 	ret = wm_adsp_read_data_word(dsp, WMFW_ADSP2_XM, addr, &magic);
@@ -3522,8 +3887,7 @@ static int wm_adsp_buffer_free(struct wm_adsp *dsp)
 	struct wm_adsp_compr_buf *buf, *tmp;
 
 	list_for_each_entry_safe(buf, tmp, &dsp->buffer_list, list) {
-		if (buf->compr)
-			wm_adsp_compr_detach(buf->compr);
+		wm_adsp_compr_detach(buf->compr);
 
 		kfree(buf->name);
 		kfree(buf->regions);
@@ -3728,7 +4092,7 @@ int wm_adsp_compr_pointer(struct snd_compr_stream *stream,
 
 	buf = compr->buf;
 
-	if (!compr->buf || compr->buf->error) {
+	if (dsp->fatal_error || !buf || buf->error) {
 		snd_compr_stop_error(stream, SNDRV_PCM_STATE_XRUN);
 		ret = -EIO;
 		goto out;
@@ -3748,7 +4112,7 @@ int wm_adsp_compr_pointer(struct snd_compr_stream *stream,
 		if (buf->avail < wm_adsp_compr_frag_words(compr)) {
 			ret = wm_adsp_buffer_get_error(buf);
 			if (ret < 0) {
-				if (compr->buf->error)
+				if (buf->error)
 					snd_compr_stop_error(stream,
 							SNDRV_PCM_STATE_XRUN);
 				goto out;
@@ -3832,12 +4196,13 @@ static int wm_adsp_buffer_capture_block(struct wm_adsp_compr *compr, int target)
 static int wm_adsp_compr_read(struct wm_adsp_compr *compr,
 			      char __user *buf, size_t count)
 {
+	struct wm_adsp *dsp = compr->dsp;
 	int ntotal = 0;
 	int nwords, nbytes;
 
 	compr_dbg(compr, "Requested read of %zu bytes\n", count);
 
-	if (!compr->buf || compr->buf->error) {
+	if (dsp->fatal_error || !compr->buf || compr->buf->error) {
 		snd_compr_stop_error(compr->stream, SNDRV_PCM_STATE_XRUN);
 		return -EIO;
 	}
@@ -3891,37 +4256,6 @@ int wm_adsp_compr_copy(struct snd_compr_stream *stream, char __user *buf,
 }
 EXPORT_SYMBOL_GPL(wm_adsp_compr_copy);
 
-int wm_adsp2_lock(struct wm_adsp *dsp, unsigned int lock_regions)
-{
-	struct regmap *regmap = dsp->regmap;
-	unsigned int code0, code1, lock_reg;
-
-	if (!(lock_regions & WM_ADSP2_REGION_ALL))
-		return 0;
-
-	lock_regions &= WM_ADSP2_REGION_ALL;
-	lock_reg = dsp->base + ADSP2_LOCK_REGION_1_LOCK_REGION_0;
-
-	while (lock_regions) {
-		code0 = code1 = 0;
-		if (lock_regions & BIT(0)) {
-			code0 = ADSP2_LOCK_CODE_0;
-			code1 = ADSP2_LOCK_CODE_1;
-		}
-		if (lock_regions & BIT(1)) {
-			code0 |= ADSP2_LOCK_CODE_0 << ADSP2_LOCK_REGION_SHIFT;
-			code1 |= ADSP2_LOCK_CODE_1 << ADSP2_LOCK_REGION_SHIFT;
-		}
-		regmap_write(regmap, lock_reg, code0);
-		regmap_write(regmap, lock_reg, code1);
-		lock_regions >>= 2;
-		lock_reg += 2;
-	}
-
-	return 0;
-}
-EXPORT_SYMBOL_GPL(wm_adsp2_lock);
-
 static void wm_adsp_fatal_error(struct wm_adsp *dsp)
 {
 	struct wm_adsp_compr *compr;
@@ -3929,11 +4263,8 @@ static void wm_adsp_fatal_error(struct wm_adsp *dsp)
 	dsp->fatal_error = true;
 
 	list_for_each_entry(compr, &dsp->compr_list, list) {
-		if (compr->stream) {
-			snd_compr_stop_error(compr->stream,
-					     SNDRV_PCM_STATE_XRUN);
+		if (compr->stream)
 			snd_compr_fragment_elapsed(compr->stream);
-		}
 	}
 }
 
@@ -3954,7 +4285,7 @@ irqreturn_t wm_adsp2_bus_error(struct wm_adsp *dsp)
 
 	if (val & ADSP2_WDT_TIMEOUT_STS_MASK) {
 		adsp_err(dsp, "watchdog timeout error\n");
-		wm_adsp_stop_watchdog(dsp);
+		dsp->ops->stop_watchdog(dsp);
 		wm_adsp_fatal_error(dsp);
 	}
 
@@ -4002,4 +4333,159 @@ error:
 }
 EXPORT_SYMBOL_GPL(wm_adsp2_bus_error);
 
+irqreturn_t wm_halo_bus_error(struct wm_adsp *dsp)
+{
+	struct regmap *regmap = dsp->regmap;
+	unsigned int fault[6];
+	struct reg_sequence clear[] = {
+		{ dsp->base + HALO_MPU_XM_VIO_STATUS,     0x0 },
+		{ dsp->base + HALO_MPU_YM_VIO_STATUS,     0x0 },
+		{ dsp->base + HALO_MPU_PM_VIO_STATUS,     0x0 },
+	};
+	int ret;
+
+	mutex_lock(&dsp->pwr_lock);
+
+	ret = regmap_read(regmap, dsp->base_sysinfo + HALO_AHBM_WINDOW_DEBUG_1,
+			  fault);
+	if (ret) {
+		adsp_warn(dsp, "Failed to read AHB DEBUG_1: %d\n", ret);
+		goto exit_unlock;
+	}
+
+	adsp_warn(dsp, "AHB: STATUS: 0x%x ADDR: 0x%x\n",
+		  *fault & HALO_AHBM_FLAGS_ERR_MASK,
+		  (*fault & HALO_AHBM_CORE_ERR_ADDR_MASK) >>
+		  HALO_AHBM_CORE_ERR_ADDR_SHIFT);
+
+	ret = regmap_read(regmap, dsp->base_sysinfo + HALO_AHBM_WINDOW_DEBUG_0,
+			  fault);
+	if (ret) {
+		adsp_warn(dsp, "Failed to read AHB DEBUG_0: %d\n", ret);
+		goto exit_unlock;
+	}
+
+	adsp_warn(dsp, "AHB: SYS_ADDR: 0x%x\n", *fault);
+
+	ret = regmap_bulk_read(regmap, dsp->base + HALO_MPU_XM_VIO_ADDR,
+			       fault, ARRAY_SIZE(fault));
+	if (ret) {
+		adsp_warn(dsp, "Failed to read MPU fault info: %d\n", ret);
+		goto exit_unlock;
+	}
+
+	adsp_warn(dsp, "XM: STATUS:0x%x ADDR:0x%x\n", fault[1], fault[0]);
+	adsp_warn(dsp, "YM: STATUS:0x%x ADDR:0x%x\n", fault[3], fault[2]);
+	adsp_warn(dsp, "PM: STATUS:0x%x ADDR:0x%x\n", fault[5], fault[4]);
+
+	ret = regmap_multi_reg_write(dsp->regmap, clear, ARRAY_SIZE(clear));
+	if (ret)
+		adsp_warn(dsp, "Failed to clear MPU status: %d\n", ret);
+
+exit_unlock:
+	mutex_unlock(&dsp->pwr_lock);
+
+	return IRQ_HANDLED;
+}
+EXPORT_SYMBOL_GPL(wm_halo_bus_error);
+
+irqreturn_t wm_halo_wdt_expire(int irq, void *data)
+{
+	struct wm_adsp *dsp = data;
+
+	mutex_lock(&dsp->pwr_lock);
+
+	adsp_warn(dsp, "WDT Expiry Fault\n");
+	dsp->ops->stop_watchdog(dsp);
+	wm_adsp_fatal_error(dsp);
+
+	mutex_unlock(&dsp->pwr_lock);
+
+	return IRQ_HANDLED;
+}
+EXPORT_SYMBOL_GPL(wm_halo_wdt_expire);
+
+static struct wm_adsp_ops wm_adsp1_ops = {
+	.validate_version = wm_adsp_validate_version,
+	.parse_sizes = wm_adsp1_parse_sizes,
+	.region_to_reg = wm_adsp_region_to_reg,
+};
+
+static struct wm_adsp_ops wm_adsp2_ops[] = {
+	{
+		.sys_config_size = sizeof(struct wm_adsp_system_config_xm_hdr),
+		.parse_sizes = wm_adsp2_parse_sizes,
+		.validate_version = wm_adsp_validate_version,
+		.setup_algs = wm_adsp2_setup_algs,
+		.region_to_reg = wm_adsp_region_to_reg,
+
+		.show_fw_status = wm_adsp2_show_fw_status,
+
+		.enable_memory = wm_adsp2_enable_memory,
+		.disable_memory = wm_adsp2_disable_memory,
+
+		.enable_core = wm_adsp2_enable_core,
+		.disable_core = wm_adsp2_disable_core,
+
+		.start_core = wm_adsp2_start_core,
+		.stop_core = wm_adsp2_stop_core,
+
+	},
+	{
+		.sys_config_size = sizeof(struct wm_adsp_system_config_xm_hdr),
+		.parse_sizes = wm_adsp2_parse_sizes,
+		.validate_version = wm_adsp_validate_version,
+		.setup_algs = wm_adsp2_setup_algs,
+		.region_to_reg = wm_adsp_region_to_reg,
+
+		.show_fw_status = wm_adsp2v2_show_fw_status,
+
+		.enable_memory = wm_adsp2_enable_memory,
+		.disable_memory = wm_adsp2_disable_memory,
+		.lock_memory = wm_adsp2_lock,
+
+		.enable_core = wm_adsp2v2_enable_core,
+		.disable_core = wm_adsp2v2_disable_core,
+
+		.start_core = wm_adsp2_start_core,
+		.stop_core = wm_adsp2_stop_core,
+	},
+	{
+		.sys_config_size = sizeof(struct wm_adsp_system_config_xm_hdr),
+		.parse_sizes = wm_adsp2_parse_sizes,
+		.validate_version = wm_adsp_validate_version,
+		.setup_algs = wm_adsp2_setup_algs,
+		.region_to_reg = wm_adsp_region_to_reg,
+
+		.show_fw_status = wm_adsp2v2_show_fw_status,
+		.stop_watchdog = wm_adsp_stop_watchdog,
+
+		.enable_memory = wm_adsp2_enable_memory,
+		.disable_memory = wm_adsp2_disable_memory,
+		.lock_memory = wm_adsp2_lock,
+
+		.enable_core = wm_adsp2v2_enable_core,
+		.disable_core = wm_adsp2v2_disable_core,
+
+		.start_core = wm_adsp2_start_core,
+		.stop_core = wm_adsp2_stop_core,
+	},
+};
+
+static struct wm_adsp_ops wm_halo_ops = {
+	.sys_config_size = sizeof(struct wm_halo_system_config_xm_hdr),
+	.parse_sizes = wm_adsp2_parse_sizes,
+	.validate_version = wm_halo_validate_version,
+	.setup_algs = wm_halo_setup_algs,
+	.region_to_reg = wm_halo_region_to_reg,
+
+	.show_fw_status = wm_halo_show_fw_status,
+	.stop_watchdog = wm_halo_stop_watchdog,
+
+	.lock_memory = wm_halo_configure_mpu,
+
+	.start_core = wm_halo_start_core,
+	.stop_core = wm_halo_stop_core,
+};
+
 MODULE_LICENSE("GPL v2");
diff --git a/sound/soc/codecs/wm_adsp.h b/sound/soc/codecs/wm_adsp.h
index 8f09b4419a91..3631c9200c5d 100644
--- a/sound/soc/codecs/wm_adsp.h
+++ b/sound/soc/codecs/wm_adsp.h
@@ -54,6 +54,7 @@ struct wm_adsp_alg_region {
 
 struct wm_adsp_compr;
 struct wm_adsp_compr_buf;
+struct wm_adsp_ops;
 
 struct wm_adsp {
 	const char *part;
@@ -66,7 +67,10 @@ struct wm_adsp {
 	struct regmap *regmap;
 	struct snd_soc_component *component;
 
+	struct wm_adsp_ops *ops;
+
 	unsigned int base;
+	unsigned int base_sysinfo;
 	unsigned int sysclk_reg;
 	unsigned int sysclk_mask;
 	unsigned int sysclk_shift;
@@ -75,6 +79,7 @@ struct wm_adsp {
 
 	unsigned int fw_id;
 	unsigned int fw_id_version;
+	unsigned int fw_vendor_id;
 
 	const struct wm_adsp_region *mem;
 	int num_mems;
@@ -106,6 +111,32 @@ struct wm_adsp {
 
 };
 
+struct wm_adsp_ops {
+	unsigned int sys_config_size;
+
+	bool (*validate_version)(struct wm_adsp *dsp, unsigned int version);
+	unsigned int (*parse_sizes)(struct wm_adsp *dsp,
+				    const char * const file,
+				    unsigned int pos,
+				    const struct firmware *firmware);
+	int (*setup_algs)(struct wm_adsp *dsp);
+	unsigned int (*region_to_reg)(struct wm_adsp_region const *mem,
+				      unsigned int offset);
+
+	void (*show_fw_status)(struct wm_adsp *dsp);
+	void (*stop_watchdog)(struct wm_adsp *dsp);
+
+	int (*enable_memory)(struct wm_adsp *dsp);
+	void (*disable_memory)(struct wm_adsp *dsp);
+	int (*lock_memory)(struct wm_adsp *dsp, unsigned int lock_regions);
+
+	int (*enable_core)(struct wm_adsp *dsp);
+	void (*disable_core)(struct wm_adsp *dsp);
+
+	int (*start_core)(struct wm_adsp *dsp);
+	void (*stop_core)(struct wm_adsp *dsp);
+};
+
 #define WM_ADSP1(wname, num) \
 	SND_SOC_DAPM_PGA_E(wname, SND_SOC_NOPM, num, 0, NULL, 0, \
 		wm_adsp1_event, SND_SOC_DAPM_POST_PMU | SND_SOC_DAPM_PRE_PMD)
@@ -121,7 +152,7 @@ struct wm_adsp {
 	.event_flags = SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_PRE_PMD, \
 	.subseq = 100, /* Ensure we run after SYSCLK supply widget */ }, \
 {	.id = snd_soc_dapm_out_drv, .name = wname, \
-	.reg = SND_SOC_NOPM, .shift = num, .event = wm_adsp2_event, \
+	.reg = SND_SOC_NOPM, .shift = num, .event = wm_adsp_event, \
 	.event_flags = SND_SOC_DAPM_POST_PMU | SND_SOC_DAPM_PRE_PMD }
 
 #define WM_ADSP_FW_CONTROL(dspname, num) \
@@ -135,17 +166,22 @@ int wm_adsp2_init(struct wm_adsp *dsp);
 void wm_adsp2_remove(struct wm_adsp *dsp);
 int wm_adsp2_component_probe(struct wm_adsp *dsp, struct snd_soc_component *component);
 int wm_adsp2_component_remove(struct wm_adsp *dsp, struct snd_soc_component *component);
+int wm_halo_init(struct wm_adsp *dsp);
+
 int wm_adsp1_event(struct snd_soc_dapm_widget *w,
 		   struct snd_kcontrol *kcontrol, int event);
-int wm_adsp2_early_event(struct snd_soc_dapm_widget *w,
-			 struct snd_kcontrol *kcontrol, int event,
-			 unsigned int freq);
 
-int wm_adsp2_lock(struct wm_adsp *adsp, unsigned int regions);
+int wm_adsp_early_event(struct snd_soc_dapm_widget *w,
+			struct snd_kcontrol *kcontrol, int event);
+
 irqreturn_t wm_adsp2_bus_error(struct wm_adsp *adsp);
+irqreturn_t wm_halo_bus_error(struct wm_adsp *dsp);
+irqreturn_t wm_halo_wdt_expire(int irq, void *data);
 
-int wm_adsp2_event(struct snd_soc_dapm_widget *w,
-		   struct snd_kcontrol *kcontrol, int event);
+int wm_adsp_event(struct snd_soc_dapm_widget *w,
+		  struct snd_kcontrol *kcontrol, int event);
+
+int wm_adsp2_set_dspclk(struct snd_soc_dapm_widget *w, unsigned int freq);
 
 int wm_adsp2_preloader_get(struct snd_kcontrol *kcontrol,
 			   struct snd_ctl_elem_value *ucontrol);
diff --git a/sound/soc/codecs/wmfw.h b/sound/soc/codecs/wmfw.h
index 0c3f50acb8b1..14b2d1a2fc59 100644
--- a/sound/soc/codecs/wmfw.h
+++ b/sound/soc/codecs/wmfw.h
@@ -73,6 +73,14 @@ struct wmfw_id_hdr {
 	__be32 ver;
 } __packed;
 
+struct wmfw_v3_id_hdr {
+	__be32 core_id;
+	__be32 block_rev;
+	__be32 vendor_id;
+	__be32 id;
+	__be32 ver;
+} __packed;
+
 struct wmfw_adsp1_id_hdr {
 	struct wmfw_id_hdr fw;
 	__be32 zm;
@@ -88,6 +96,15 @@ struct wmfw_adsp2_id_hdr {
 	__be32 n_algs;
 } __packed;
 
+struct wmfw_halo_id_hdr {
+	struct wmfw_v3_id_hdr fw;
+	__be32 xm_base;
+	__be32 xm_size;
+	__be32 ym_base;
+	__be32 ym_size;
+	__be32 n_algs;
+} __packed;
+
 struct wmfw_alg_hdr {
 	__be32 id;
 	__be32 ver;
@@ -106,6 +123,14 @@ struct wmfw_adsp2_alg_hdr {
 	__be32 ym;
 } __packed;
 
+struct wmfw_halo_alg_hdr {
+	struct wmfw_alg_hdr alg;
+	__be32 xm_base;
+	__be32 xm_size;
+	__be32 ym_base;
+	__be32 ym_size;
+} __packed;
+
 struct wmfw_adsp_alg_data {
 	__le32 id;
 	u8 name[WMFW_MAX_ALG_NAME];
@@ -154,6 +179,7 @@ struct wmfw_coeff_item {
 
 #define WMFW_ADSP1 1
 #define WMFW_ADSP2 2
+#define WMFW_HALO 4
 
 #define WMFW_ABSOLUTE         0xf0
 #define WMFW_ALGORITHM_DATA   0xf2
@@ -169,4 +195,8 @@ struct wmfw_coeff_item {
 #define WMFW_ADSP2_XM 5
 #define WMFW_ADSP2_YM 6
 
+#define WMFW_HALO_PM_PACKED 0x10
+#define WMFW_HALO_XM_PACKED 0x11
+#define WMFW_HALO_YM_PACKED 0x12
+
 #endif
diff --git a/sound/soc/fsl/Kconfig b/sound/soc/fsl/Kconfig
index 7b1d9970be8b..55ed47c599e2 100644
--- a/sound/soc/fsl/Kconfig
+++ b/sound/soc/fsl/Kconfig
@@ -24,6 +24,13 @@ config SND_SOC_FSL_SAI
 	  This option is only useful for out-of-tree drivers since
 	  in-tree drivers select it automatically.
 
+config SND_SOC_FSL_AUDMIX
+	tristate "Audio Mixer (AUDMIX) module support"
+	select REGMAP_MMIO
+	help
+	  Say Y if you want to add Audio Mixer (AUDMIX)
+	  support for the NXP iMX CPUs.
+
 config SND_SOC_FSL_SSI
 	tristate "Synchronous Serial Interface module (SSI) support"
 	select SND_SOC_IMX_PCM_DMA if SND_IMX_SOC != n
@@ -182,16 +189,17 @@ config SND_MPC52xx_SOC_EFIKA
 
 endif # SND_POWERPC_SOC
 
+config SND_SOC_IMX_PCM_FIQ
+	tristate
+	default y if SND_SOC_IMX_SSI=y && (SND_SOC_FSL_SSI=m || SND_SOC_FSL_SPDIF=m) && (MXC_TZIC || MXC_AVIC)
+	select FIQ
+
 if SND_IMX_SOC
 
 config SND_SOC_IMX_SSI
 	tristate
 	select SND_SOC_FSL_UTILS
 
-config SND_SOC_IMX_PCM_FIQ
-	tristate
-	select FIQ
-
 comment "SoC Audio support for Freescale i.MX boards:"
 
 config SND_MXC_SOC_WM1133_EV1
@@ -296,6 +304,15 @@ config SND_SOC_FSL_ASOC_CARD
 	 CS4271, CS4272 and SGTL5000.
 	 Say Y if you want to add support for Freescale Generic ASoC Sound Card.
 
+config SND_SOC_IMX_AUDMIX
+	tristate "SoC Audio support for i.MX boards with AUDMIX"
+	select SND_SOC_FSL_AUDMIX
+	select SND_SOC_FSL_SAI
+	help
+	  SoC Audio support for i.MX boards with Audio Mixer
+	  Say Y if you want to add support for SoC audio on an i.MX board with
+	  an Audio Mixer.
+
 endif # SND_IMX_SOC
 
 endmenu
diff --git a/sound/soc/fsl/Makefile b/sound/soc/fsl/Makefile
index 3c0ff315b971..c0dd04422fe9 100644
--- a/sound/soc/fsl/Makefile
+++ b/sound/soc/fsl/Makefile
@@ -12,6 +12,7 @@ snd-soc-p1022-rdk-objs := p1022_rdk.o
 obj-$(CONFIG_SND_SOC_P1022_RDK) += snd-soc-p1022-rdk.o
 
 # Freescale SSI/DMA/SAI/SPDIF Support
+snd-soc-fsl-audmix-objs := fsl_audmix.o
 snd-soc-fsl-asoc-card-objs := fsl-asoc-card.o
 snd-soc-fsl-asrc-objs := fsl_asrc.o fsl_asrc_dma.o
 snd-soc-fsl-sai-objs := fsl_sai.o
@@ -22,6 +23,8 @@ snd-soc-fsl-esai-objs := fsl_esai.o
 snd-soc-fsl-micfil-objs := fsl_micfil.o
 snd-soc-fsl-utils-objs := fsl_utils.o
 snd-soc-fsl-dma-objs := fsl_dma.o
+
+obj-$(CONFIG_SND_SOC_FSL_AUDMIX) += snd-soc-fsl-audmix.o
 obj-$(CONFIG_SND_SOC_FSL_ASOC_CARD) += snd-soc-fsl-asoc-card.o
 obj-$(CONFIG_SND_SOC_FSL_ASRC) += snd-soc-fsl-asrc.o
 obj-$(CONFIG_SND_SOC_FSL_SAI) += snd-soc-fsl-sai.o
@@ -59,6 +62,7 @@ snd-soc-imx-es8328-objs := imx-es8328.o
 snd-soc-imx-sgtl5000-objs := imx-sgtl5000.o
 snd-soc-imx-spdif-objs := imx-spdif.o
 snd-soc-imx-mc13783-objs := imx-mc13783.o
+snd-soc-imx-audmix-objs := imx-audmix.o
 
 obj-$(CONFIG_SND_SOC_EUKREA_TLV320) += snd-soc-eukrea-tlv320.o
 obj-$(CONFIG_SND_SOC_PHYCORE_AC97) += snd-soc-phycore-ac97.o
@@ -68,3 +72,4 @@ obj-$(CONFIG_SND_SOC_IMX_ES8328) += snd-soc-imx-es8328.o
 obj-$(CONFIG_SND_SOC_IMX_SGTL5000) += snd-soc-imx-sgtl5000.o
 obj-$(CONFIG_SND_SOC_IMX_SPDIF) += snd-soc-imx-spdif.o
 obj-$(CONFIG_SND_SOC_IMX_MC13783) += snd-soc-imx-mc13783.o
+obj-$(CONFIG_SND_SOC_IMX_AUDMIX) += snd-soc-imx-audmix.o
diff --git a/sound/soc/fsl/eukrea-tlv320.c b/sound/soc/fsl/eukrea-tlv320.c
index 191426a6d9ad..d648268cb454 100644
--- a/sound/soc/fsl/eukrea-tlv320.c
+++ b/sound/soc/fsl/eukrea-tlv320.c
@@ -1,19 +1,13 @@
-/*
- * eukrea-tlv320.c  --  SoC audio for eukrea_cpuimxXX in I2S mode
- *
- * Copyright 2010 Eric Bénard, Eukréa Electromatique <eric@eukrea.com>
- *
- * based on sound/soc/s3c24xx/s3c24xx_simtec_tlv320aic23.c
- * which is Copyright 2009 Simtec Electronics
- * and on sound/soc/imx/phycore-ac97.c which is
- * Copyright 2009 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>
- *
- *  This program is free software; you can redistribute  it and/or modify it
- *  under  the terms of  the GNU General  Public License as published by the
- *  Free Software Foundation;  either version 2 of the  License, or (at your
- *  option) any later version.
- *
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// eukrea-tlv320.c  --  SoC audio for eukrea_cpuimxXX in I2S mode
+//
+// Copyright 2010 Eric Bénard, Eukréa Electromatique <eric@eukrea.com>
+//
+// based on sound/soc/s3c24xx/s3c24xx_simtec_tlv320aic23.c
+// which is Copyright 2009 Simtec Electronics
+// and on sound/soc/imx/phycore-ac97.c which is
+// Copyright 2009 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>
 
 #include <linux/errno.h>
 #include <linux/module.h>
@@ -118,13 +112,13 @@ static int eukrea_tlv320_probe(struct platform_device *pdev)
 		if (ret) {
 			dev_err(&pdev->dev,
 				"fsl,mux-int-port node missing or invalid.\n");
-			return ret;
+			goto err;
 		}
 		ret = of_property_read_u32(np, "fsl,mux-ext-port", &ext_port);
 		if (ret) {
 			dev_err(&pdev->dev,
 				"fsl,mux-ext-port node missing or invalid.\n");
-			return ret;
+			goto err;
 		}
 
 		/*
diff --git a/sound/soc/fsl/fsl_audmix.c b/sound/soc/fsl/fsl_audmix.c
new file mode 100644
index 000000000000..3897a54a11fe
--- /dev/null
+++ b/sound/soc/fsl/fsl_audmix.c
@@ -0,0 +1,578 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * NXP AUDMIX ALSA SoC Digital Audio Interface (DAI) driver
+ *
+ * Copyright 2017 NXP
+ */
+
+#include <linux/clk.h>
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <linux/pm_runtime.h>
+#include <sound/soc.h>
+#include <sound/pcm_params.h>
+
+#include "fsl_audmix.h"
+
+#define SOC_ENUM_SINGLE_S(xreg, xshift, xtexts) \
+	SOC_ENUM_SINGLE(xreg, xshift, ARRAY_SIZE(xtexts), xtexts)
+
+static const char
+	*tdm_sel[] = { "TDM1", "TDM2", },
+	*mode_sel[] = { "Disabled", "TDM1", "TDM2", "Mixed", },
+	*width_sel[] = { "16b", "18b", "20b", "24b", "32b", },
+	*endis_sel[] = { "Disabled", "Enabled", },
+	*updn_sel[] = { "Downward", "Upward", },
+	*mask_sel[] = { "Unmask", "Mask", };
+
+static const struct soc_enum fsl_audmix_enum[] = {
+/* FSL_AUDMIX_CTR enums */
+SOC_ENUM_SINGLE_S(FSL_AUDMIX_CTR, FSL_AUDMIX_CTR_MIXCLK_SHIFT, tdm_sel),
+SOC_ENUM_SINGLE_S(FSL_AUDMIX_CTR, FSL_AUDMIX_CTR_OUTSRC_SHIFT, mode_sel),
+SOC_ENUM_SINGLE_S(FSL_AUDMIX_CTR, FSL_AUDMIX_CTR_OUTWIDTH_SHIFT, width_sel),
+SOC_ENUM_SINGLE_S(FSL_AUDMIX_CTR, FSL_AUDMIX_CTR_MASKRTDF_SHIFT, mask_sel),
+SOC_ENUM_SINGLE_S(FSL_AUDMIX_CTR, FSL_AUDMIX_CTR_MASKCKDF_SHIFT, mask_sel),
+SOC_ENUM_SINGLE_S(FSL_AUDMIX_CTR, FSL_AUDMIX_CTR_SYNCMODE_SHIFT, endis_sel),
+SOC_ENUM_SINGLE_S(FSL_AUDMIX_CTR, FSL_AUDMIX_CTR_SYNCSRC_SHIFT, tdm_sel),
+/* FSL_AUDMIX_ATCR0 enums */
+SOC_ENUM_SINGLE_S(FSL_AUDMIX_ATCR0, 0, endis_sel),
+SOC_ENUM_SINGLE_S(FSL_AUDMIX_ATCR0, 1, updn_sel),
+/* FSL_AUDMIX_ATCR1 enums */
+SOC_ENUM_SINGLE_S(FSL_AUDMIX_ATCR1, 0, endis_sel),
+SOC_ENUM_SINGLE_S(FSL_AUDMIX_ATCR1, 1, updn_sel),
+};
+
+struct fsl_audmix_state {
+	u8 tdms;
+	u8 clk;
+	char msg[64];
+};
+
+static const struct fsl_audmix_state prms[4][4] = {{
+	/* DIS->DIS, do nothing */
+	{ .tdms = 0, .clk = 0, .msg = "" },
+	/* DIS->TDM1*/
+	{ .tdms = 1, .clk = 1, .msg = "DIS->TDM1: TDM1 not started!\n" },
+	/* DIS->TDM2*/
+	{ .tdms = 2, .clk = 2, .msg = "DIS->TDM2: TDM2 not started!\n" },
+	/* DIS->MIX */
+	{ .tdms = 3, .clk = 0, .msg = "DIS->MIX: Please start both TDMs!\n" }
+}, {	/* TDM1->DIS */
+	{ .tdms = 1, .clk = 0, .msg = "TDM1->DIS: TDM1 not started!\n" },
+	/* TDM1->TDM1, do nothing */
+	{ .tdms = 0, .clk = 0, .msg = "" },
+	/* TDM1->TDM2 */
+	{ .tdms = 3, .clk = 2, .msg = "TDM1->TDM2: Please start both TDMs!\n" },
+	/* TDM1->MIX */
+	{ .tdms = 3, .clk = 0, .msg = "TDM1->MIX: Please start both TDMs!\n" }
+}, {	/* TDM2->DIS */
+	{ .tdms = 2, .clk = 0, .msg = "TDM2->DIS: TDM2 not started!\n" },
+	/* TDM2->TDM1 */
+	{ .tdms = 3, .clk = 1, .msg = "TDM2->TDM1: Please start both TDMs!\n" },
+	/* TDM2->TDM2, do nothing */
+	{ .tdms = 0, .clk = 0, .msg = "" },
+	/* TDM2->MIX */
+	{ .tdms = 3, .clk = 0, .msg = "TDM2->MIX: Please start both TDMs!\n" }
+}, {	/* MIX->DIS */
+	{ .tdms = 3, .clk = 0, .msg = "MIX->DIS: Please start both TDMs!\n" },
+	/* MIX->TDM1 */
+	{ .tdms = 3, .clk = 1, .msg = "MIX->TDM1: Please start both TDMs!\n" },
+	/* MIX->TDM2 */
+	{ .tdms = 3, .clk = 2, .msg = "MIX->TDM2: Please start both TDMs!\n" },
+	/* MIX->MIX, do nothing */
+	{ .tdms = 0, .clk = 0, .msg = "" }
+}, };
+
+static int fsl_audmix_state_trans(struct snd_soc_component *comp,
+				  unsigned int *mask, unsigned int *ctr,
+				  const struct fsl_audmix_state prm)
+{
+	struct fsl_audmix *priv = snd_soc_component_get_drvdata(comp);
+	/* Enforce all required TDMs are started */
+	if ((priv->tdms & prm.tdms) != prm.tdms) {
+		dev_dbg(comp->dev, "%s", prm.msg);
+		return -EINVAL;
+	}
+
+	switch (prm.clk) {
+	case 1:
+	case 2:
+		/* Set mix clock */
+		(*mask) |= FSL_AUDMIX_CTR_MIXCLK_MASK;
+		(*ctr)  |= FSL_AUDMIX_CTR_MIXCLK(prm.clk - 1);
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static int fsl_audmix_put_mix_clk_src(struct snd_kcontrol *kcontrol,
+				      struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_component *comp = snd_kcontrol_chip(kcontrol);
+	struct fsl_audmix *priv = snd_soc_component_get_drvdata(comp);
+	struct soc_enum *e = (struct soc_enum *)kcontrol->private_value;
+	unsigned int *item = ucontrol->value.enumerated.item;
+	unsigned int reg_val, val, mix_clk;
+	int ret = 0;
+
+	/* Get current state */
+	ret = snd_soc_component_read(comp, FSL_AUDMIX_CTR, &reg_val);
+	if (ret)
+		return ret;
+
+	mix_clk = ((reg_val & FSL_AUDMIX_CTR_MIXCLK_MASK)
+			>> FSL_AUDMIX_CTR_MIXCLK_SHIFT);
+	val = snd_soc_enum_item_to_val(e, item[0]);
+
+	dev_dbg(comp->dev, "TDMs=x%08x, val=x%08x\n", priv->tdms, val);
+
+	/**
+	 * Ensure the current selected mixer clock is available
+	 * for configuration propagation
+	 */
+	if (!(priv->tdms & BIT(mix_clk))) {
+		dev_err(comp->dev,
+			"Started TDM%d needed for config propagation!\n",
+			mix_clk + 1);
+		return -EINVAL;
+	}
+
+	if (!(priv->tdms & BIT(val))) {
+		dev_err(comp->dev,
+			"The selected clock source has no TDM%d enabled!\n",
+			val + 1);
+		return -EINVAL;
+	}
+
+	return snd_soc_put_enum_double(kcontrol, ucontrol);
+}
+
+static int fsl_audmix_put_out_src(struct snd_kcontrol *kcontrol,
+				  struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_component *comp = snd_kcontrol_chip(kcontrol);
+	struct fsl_audmix *priv = snd_soc_component_get_drvdata(comp);
+	struct soc_enum *e = (struct soc_enum *)kcontrol->private_value;
+	unsigned int *item = ucontrol->value.enumerated.item;
+	u32 out_src, mix_clk;
+	unsigned int reg_val, val, mask = 0, ctr = 0;
+	int ret = 0;
+
+	/* Get current state */
+	ret = snd_soc_component_read(comp, FSL_AUDMIX_CTR, &reg_val);
+	if (ret)
+		return ret;
+
+	/* "From" state */
+	out_src = ((reg_val & FSL_AUDMIX_CTR_OUTSRC_MASK)
+			>> FSL_AUDMIX_CTR_OUTSRC_SHIFT);
+	mix_clk = ((reg_val & FSL_AUDMIX_CTR_MIXCLK_MASK)
+			>> FSL_AUDMIX_CTR_MIXCLK_SHIFT);
+
+	/* "To" state */
+	val = snd_soc_enum_item_to_val(e, item[0]);
+
+	dev_dbg(comp->dev, "TDMs=x%08x, val=x%08x\n", priv->tdms, val);
+
+	/* Check if state is changing ... */
+	if (out_src == val)
+		return 0;
+	/**
+	 * Ensure the current selected mixer clock is available
+	 * for configuration propagation
+	 */
+	if (!(priv->tdms & BIT(mix_clk))) {
+		dev_err(comp->dev,
+			"Started TDM%d needed for config propagation!\n",
+			mix_clk + 1);
+		return -EINVAL;
+	}
+
+	/* Check state transition constraints */
+	ret = fsl_audmix_state_trans(comp, &mask, &ctr, prms[out_src][val]);
+	if (ret)
+		return ret;
+
+	/* Complete transition to new state */
+	mask |= FSL_AUDMIX_CTR_OUTSRC_MASK;
+	ctr  |= FSL_AUDMIX_CTR_OUTSRC(val);
+
+	return snd_soc_component_update_bits(comp, FSL_AUDMIX_CTR, mask, ctr);
+}
+
+static const struct snd_kcontrol_new fsl_audmix_snd_controls[] = {
+	/* FSL_AUDMIX_CTR controls */
+	SOC_ENUM_EXT("Mixing Clock Source", fsl_audmix_enum[0],
+		     snd_soc_get_enum_double, fsl_audmix_put_mix_clk_src),
+	SOC_ENUM_EXT("Output Source", fsl_audmix_enum[1],
+		     snd_soc_get_enum_double, fsl_audmix_put_out_src),
+	SOC_ENUM("Output Width", fsl_audmix_enum[2]),
+	SOC_ENUM("Frame Rate Diff Error", fsl_audmix_enum[3]),
+	SOC_ENUM("Clock Freq Diff Error", fsl_audmix_enum[4]),
+	SOC_ENUM("Sync Mode Config", fsl_audmix_enum[5]),
+	SOC_ENUM("Sync Mode Clk Source", fsl_audmix_enum[6]),
+	/* TDM1 Attenuation controls */
+	SOC_ENUM("TDM1 Attenuation", fsl_audmix_enum[7]),
+	SOC_ENUM("TDM1 Attenuation Direction", fsl_audmix_enum[8]),
+	SOC_SINGLE("TDM1 Attenuation Step Divider", FSL_AUDMIX_ATCR0,
+		   2, 0x00fff, 0),
+	SOC_SINGLE("TDM1 Attenuation Initial Value", FSL_AUDMIX_ATIVAL0,
+		   0, 0x3ffff, 0),
+	SOC_SINGLE("TDM1 Attenuation Step Up Factor", FSL_AUDMIX_ATSTPUP0,
+		   0, 0x3ffff, 0),
+	SOC_SINGLE("TDM1 Attenuation Step Down Factor", FSL_AUDMIX_ATSTPDN0,
+		   0, 0x3ffff, 0),
+	SOC_SINGLE("TDM1 Attenuation Step Target", FSL_AUDMIX_ATSTPTGT0,
+		   0, 0x3ffff, 0),
+	/* TDM2 Attenuation controls */
+	SOC_ENUM("TDM2 Attenuation", fsl_audmix_enum[9]),
+	SOC_ENUM("TDM2 Attenuation Direction", fsl_audmix_enum[10]),
+	SOC_SINGLE("TDM2 Attenuation Step Divider", FSL_AUDMIX_ATCR1,
+		   2, 0x00fff, 0),
+	SOC_SINGLE("TDM2 Attenuation Initial Value", FSL_AUDMIX_ATIVAL1,
+		   0, 0x3ffff, 0),
+	SOC_SINGLE("TDM2 Attenuation Step Up Factor", FSL_AUDMIX_ATSTPUP1,
+		   0, 0x3ffff, 0),
+	SOC_SINGLE("TDM2 Attenuation Step Down Factor", FSL_AUDMIX_ATSTPDN1,
+		   0, 0x3ffff, 0),
+	SOC_SINGLE("TDM2 Attenuation Step Target", FSL_AUDMIX_ATSTPTGT1,
+		   0, 0x3ffff, 0),
+};
+
+static int fsl_audmix_dai_set_fmt(struct snd_soc_dai *dai, unsigned int fmt)
+{
+	struct snd_soc_component *comp = dai->component;
+	u32 mask = 0, ctr = 0;
+
+	/* AUDMIX is working in DSP_A format only */
+	switch (fmt & SND_SOC_DAIFMT_FORMAT_MASK) {
+	case SND_SOC_DAIFMT_DSP_A:
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	/* For playback the AUDMIX is slave, and for record is master */
+	switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) {
+	case SND_SOC_DAIFMT_CBM_CFM:
+	case SND_SOC_DAIFMT_CBS_CFS:
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	switch (fmt & SND_SOC_DAIFMT_INV_MASK) {
+	case SND_SOC_DAIFMT_IB_NF:
+		/* Output data will be written on positive edge of the clock */
+		ctr |= FSL_AUDMIX_CTR_OUTCKPOL(0);
+		break;
+	case SND_SOC_DAIFMT_NB_NF:
+		/* Output data will be written on negative edge of the clock */
+		ctr |= FSL_AUDMIX_CTR_OUTCKPOL(1);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	mask |= FSL_AUDMIX_CTR_OUTCKPOL_MASK;
+
+	return snd_soc_component_update_bits(comp, FSL_AUDMIX_CTR, mask, ctr);
+}
+
+static int fsl_audmix_dai_trigger(struct snd_pcm_substream *substream, int cmd,
+				  struct snd_soc_dai *dai)
+{
+	struct fsl_audmix *priv = snd_soc_dai_get_drvdata(dai);
+
+	/* Capture stream shall not be handled */
+	if (substream->stream == SNDRV_PCM_STREAM_CAPTURE)
+		return 0;
+
+	switch (cmd) {
+	case SNDRV_PCM_TRIGGER_START:
+	case SNDRV_PCM_TRIGGER_RESUME:
+	case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
+		priv->tdms |= BIT(dai->driver->id);
+		break;
+	case SNDRV_PCM_TRIGGER_STOP:
+	case SNDRV_PCM_TRIGGER_SUSPEND:
+	case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
+		priv->tdms &= ~BIT(dai->driver->id);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static const struct snd_soc_dai_ops fsl_audmix_dai_ops = {
+	.set_fmt      = fsl_audmix_dai_set_fmt,
+	.trigger      = fsl_audmix_dai_trigger,
+};
+
+static struct snd_soc_dai_driver fsl_audmix_dai[] = {
+	{
+		.id   = 0,
+		.name = "audmix-0",
+		.playback = {
+			.stream_name = "AUDMIX-Playback-0",
+			.channels_min = 8,
+			.channels_max = 8,
+			.rate_min = 8000,
+			.rate_max = 96000,
+			.rates = SNDRV_PCM_RATE_8000_96000,
+			.formats = FSL_AUDMIX_FORMATS,
+		},
+		.capture = {
+			.stream_name = "AUDMIX-Capture-0",
+			.channels_min = 8,
+			.channels_max = 8,
+			.rate_min = 8000,
+			.rate_max = 96000,
+			.rates = SNDRV_PCM_RATE_8000_96000,
+			.formats = FSL_AUDMIX_FORMATS,
+		},
+		.ops = &fsl_audmix_dai_ops,
+	},
+	{
+		.id   = 1,
+		.name = "audmix-1",
+		.playback = {
+			.stream_name = "AUDMIX-Playback-1",
+			.channels_min = 8,
+			.channels_max = 8,
+			.rate_min = 8000,
+			.rate_max = 96000,
+			.rates = SNDRV_PCM_RATE_8000_96000,
+			.formats = FSL_AUDMIX_FORMATS,
+		},
+		.capture = {
+			.stream_name = "AUDMIX-Capture-1",
+			.channels_min = 8,
+			.channels_max = 8,
+			.rate_min = 8000,
+			.rate_max = 96000,
+			.rates = SNDRV_PCM_RATE_8000_96000,
+			.formats = FSL_AUDMIX_FORMATS,
+		},
+		.ops = &fsl_audmix_dai_ops,
+	},
+};
+
+static const struct snd_soc_component_driver fsl_audmix_component = {
+	.name		  = "fsl-audmix-dai",
+	.controls	  = fsl_audmix_snd_controls,
+	.num_controls	  = ARRAY_SIZE(fsl_audmix_snd_controls),
+};
+
+static bool fsl_audmix_readable_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case FSL_AUDMIX_CTR:
+	case FSL_AUDMIX_STR:
+	case FSL_AUDMIX_ATCR0:
+	case FSL_AUDMIX_ATIVAL0:
+	case FSL_AUDMIX_ATSTPUP0:
+	case FSL_AUDMIX_ATSTPDN0:
+	case FSL_AUDMIX_ATSTPTGT0:
+	case FSL_AUDMIX_ATTNVAL0:
+	case FSL_AUDMIX_ATSTP0:
+	case FSL_AUDMIX_ATCR1:
+	case FSL_AUDMIX_ATIVAL1:
+	case FSL_AUDMIX_ATSTPUP1:
+	case FSL_AUDMIX_ATSTPDN1:
+	case FSL_AUDMIX_ATSTPTGT1:
+	case FSL_AUDMIX_ATTNVAL1:
+	case FSL_AUDMIX_ATSTP1:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static bool fsl_audmix_writeable_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case FSL_AUDMIX_CTR:
+	case FSL_AUDMIX_ATCR0:
+	case FSL_AUDMIX_ATIVAL0:
+	case FSL_AUDMIX_ATSTPUP0:
+	case FSL_AUDMIX_ATSTPDN0:
+	case FSL_AUDMIX_ATSTPTGT0:
+	case FSL_AUDMIX_ATCR1:
+	case FSL_AUDMIX_ATIVAL1:
+	case FSL_AUDMIX_ATSTPUP1:
+	case FSL_AUDMIX_ATSTPDN1:
+	case FSL_AUDMIX_ATSTPTGT1:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static const struct reg_default fsl_audmix_reg[] = {
+	{ FSL_AUDMIX_CTR,       0x00060 },
+	{ FSL_AUDMIX_STR,       0x00003 },
+	{ FSL_AUDMIX_ATCR0,     0x00000 },
+	{ FSL_AUDMIX_ATIVAL0,   0x3FFFF },
+	{ FSL_AUDMIX_ATSTPUP0,  0x2AAAA },
+	{ FSL_AUDMIX_ATSTPDN0,  0x30000 },
+	{ FSL_AUDMIX_ATSTPTGT0, 0x00010 },
+	{ FSL_AUDMIX_ATTNVAL0,  0x00000 },
+	{ FSL_AUDMIX_ATSTP0,    0x00000 },
+	{ FSL_AUDMIX_ATCR1,     0x00000 },
+	{ FSL_AUDMIX_ATIVAL1,   0x3FFFF },
+	{ FSL_AUDMIX_ATSTPUP1,  0x2AAAA },
+	{ FSL_AUDMIX_ATSTPDN1,  0x30000 },
+	{ FSL_AUDMIX_ATSTPTGT1, 0x00010 },
+	{ FSL_AUDMIX_ATTNVAL1,  0x00000 },
+	{ FSL_AUDMIX_ATSTP1,    0x00000 },
+};
+
+static const struct regmap_config fsl_audmix_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = FSL_AUDMIX_ATSTP1,
+	.reg_defaults = fsl_audmix_reg,
+	.num_reg_defaults = ARRAY_SIZE(fsl_audmix_reg),
+	.readable_reg = fsl_audmix_readable_reg,
+	.writeable_reg = fsl_audmix_writeable_reg,
+	.cache_type = REGCACHE_FLAT,
+};
+
+static const struct of_device_id fsl_audmix_ids[] = {
+	{
+		.compatible = "fsl,imx8qm-audmix",
+		.data = "imx-audmix",
+	},
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, fsl_audmix_ids);
+
+static int fsl_audmix_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct fsl_audmix *priv;
+	struct resource *res;
+	const char *mdrv;
+	const struct of_device_id *of_id;
+	void __iomem *regs;
+	int ret;
+
+	of_id = of_match_device(fsl_audmix_ids, dev);
+	if (!of_id || !of_id->data)
+		return -EINVAL;
+
+	mdrv = of_id->data;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	/* Get the addresses */
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	regs = devm_ioremap_resource(dev, res);
+	if (IS_ERR(regs))
+		return PTR_ERR(regs);
+
+	priv->regmap = devm_regmap_init_mmio_clk(dev, "ipg", regs,
+						 &fsl_audmix_regmap_config);
+	if (IS_ERR(priv->regmap)) {
+		dev_err(dev, "failed to init regmap\n");
+		return PTR_ERR(priv->regmap);
+	}
+
+	priv->ipg_clk = devm_clk_get(dev, "ipg");
+	if (IS_ERR(priv->ipg_clk)) {
+		dev_err(dev, "failed to get ipg clock\n");
+		return PTR_ERR(priv->ipg_clk);
+	}
+
+	platform_set_drvdata(pdev, priv);
+	pm_runtime_enable(dev);
+
+	ret = devm_snd_soc_register_component(dev, &fsl_audmix_component,
+					      fsl_audmix_dai,
+					      ARRAY_SIZE(fsl_audmix_dai));
+	if (ret) {
+		dev_err(dev, "failed to register ASoC DAI\n");
+		return ret;
+	}
+
+	priv->pdev = platform_device_register_data(dev, mdrv, 0, NULL, 0);
+	if (IS_ERR(priv->pdev)) {
+		ret = PTR_ERR(priv->pdev);
+		dev_err(dev, "failed to register platform %s: %d\n", mdrv, ret);
+	}
+
+	return ret;
+}
+
+static int fsl_audmix_remove(struct platform_device *pdev)
+{
+	struct fsl_audmix *priv = dev_get_drvdata(&pdev->dev);
+
+	if (priv->pdev)
+		platform_device_unregister(priv->pdev);
+
+	return 0;
+}
+
+#ifdef CONFIG_PM
+static int fsl_audmix_runtime_resume(struct device *dev)
+{
+	struct fsl_audmix *priv = dev_get_drvdata(dev);
+	int ret;
+
+	ret = clk_prepare_enable(priv->ipg_clk);
+	if (ret) {
+		dev_err(dev, "Failed to enable IPG clock: %d\n", ret);
+		return ret;
+	}
+
+	regcache_cache_only(priv->regmap, false);
+	regcache_mark_dirty(priv->regmap);
+
+	return regcache_sync(priv->regmap);
+}
+
+static int fsl_audmix_runtime_suspend(struct device *dev)
+{
+	struct fsl_audmix *priv = dev_get_drvdata(dev);
+
+	regcache_cache_only(priv->regmap, true);
+
+	clk_disable_unprepare(priv->ipg_clk);
+
+	return 0;
+}
+#endif /* CONFIG_PM */
+
+static const struct dev_pm_ops fsl_audmix_pm = {
+	SET_RUNTIME_PM_OPS(fsl_audmix_runtime_suspend,
+			   fsl_audmix_runtime_resume,
+			   NULL)
+	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				pm_runtime_force_resume)
+};
+
+static struct platform_driver fsl_audmix_driver = {
+	.probe = fsl_audmix_probe,
+	.remove = fsl_audmix_remove,
+	.driver = {
+		.name = "fsl-audmix",
+		.of_match_table = fsl_audmix_ids,
+		.pm = &fsl_audmix_pm,
+	},
+};
+module_platform_driver(fsl_audmix_driver);
+
+MODULE_DESCRIPTION("NXP AUDMIX ASoC DAI driver");
+MODULE_AUTHOR("Viorel Suman <viorel.suman@nxp.com>");
+MODULE_ALIAS("platform:fsl-audmix");
+MODULE_LICENSE("GPL v2");
diff --git a/sound/soc/fsl/fsl_audmix.h b/sound/soc/fsl/fsl_audmix.h
new file mode 100644
index 000000000000..7812ffec45c5
--- /dev/null
+++ b/sound/soc/fsl/fsl_audmix.h
@@ -0,0 +1,102 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * NXP AUDMIX ALSA SoC Digital Audio Interface (DAI) driver
+ *
+ * Copyright 2017 NXP
+ */
+
+#ifndef __FSL_AUDMIX_H
+#define __FSL_AUDMIX_H
+
+#define FSL_AUDMIX_FORMATS (SNDRV_PCM_FMTBIT_S16_LE |\
+			SNDRV_PCM_FMTBIT_S24_LE |\
+			SNDRV_PCM_FMTBIT_S32_LE)
+/* AUDMIX Registers */
+#define FSL_AUDMIX_CTR		0x200 /* Control */
+#define FSL_AUDMIX_STR		0x204 /* Status */
+
+#define FSL_AUDMIX_ATCR0	0x208 /* Attenuation Control */
+#define FSL_AUDMIX_ATIVAL0	0x20c /* Attenuation Initial Value */
+#define FSL_AUDMIX_ATSTPUP0	0x210 /* Attenuation step up factor */
+#define FSL_AUDMIX_ATSTPDN0	0x214 /* Attenuation step down factor */
+#define FSL_AUDMIX_ATSTPTGT0	0x218 /* Attenuation step target */
+#define FSL_AUDMIX_ATTNVAL0	0x21c /* Attenuation Value */
+#define FSL_AUDMIX_ATSTP0	0x220 /* Attenuation step number */
+
+#define FSL_AUDMIX_ATCR1	0x228 /* Attenuation Control */
+#define FSL_AUDMIX_ATIVAL1	0x22c /* Attenuation Initial Value */
+#define FSL_AUDMIX_ATSTPUP1	0x230 /* Attenuation step up factor */
+#define FSL_AUDMIX_ATSTPDN1	0x234 /* Attenuation step down factor */
+#define FSL_AUDMIX_ATSTPTGT1	0x238 /* Attenuation step target */
+#define FSL_AUDMIX_ATTNVAL1	0x23c /* Attenuation Value */
+#define FSL_AUDMIX_ATSTP1	0x240 /* Attenuation step number */
+
+/* AUDMIX Control Register */
+#define FSL_AUDMIX_CTR_MIXCLK_SHIFT	0
+#define FSL_AUDMIX_CTR_MIXCLK_MASK	BIT(FSL_AUDMIX_CTR_MIXCLK_SHIFT)
+#define FSL_AUDMIX_CTR_MIXCLK(i)	((i) << FSL_AUDMIX_CTR_MIXCLK_SHIFT)
+#define FSL_AUDMIX_CTR_OUTSRC_SHIFT	1
+#define FSL_AUDMIX_CTR_OUTSRC_MASK	(0x3 << FSL_AUDMIX_CTR_OUTSRC_SHIFT)
+#define FSL_AUDMIX_CTR_OUTSRC(i)	(((i) << FSL_AUDMIX_CTR_OUTSRC_SHIFT)\
+					      & FSL_AUDMIX_CTR_OUTSRC_MASK)
+#define FSL_AUDMIX_CTR_OUTWIDTH_SHIFT	3
+#define FSL_AUDMIX_CTR_OUTWIDTH_MASK	(0x7 << FSL_AUDMIX_CTR_OUTWIDTH_SHIFT)
+#define FSL_AUDMIX_CTR_OUTWIDTH(i)	(((i) << FSL_AUDMIX_CTR_OUTWIDTH_SHIFT)\
+					      & FSL_AUDMIX_CTR_OUTWIDTH_MASK)
+#define FSL_AUDMIX_CTR_OUTCKPOL_SHIFT	6
+#define FSL_AUDMIX_CTR_OUTCKPOL_MASK	BIT(FSL_AUDMIX_CTR_OUTCKPOL_SHIFT)
+#define FSL_AUDMIX_CTR_OUTCKPOL(i)	((i) << FSL_AUDMIX_CTR_OUTCKPOL_SHIFT)
+#define FSL_AUDMIX_CTR_MASKRTDF_SHIFT	7
+#define FSL_AUDMIX_CTR_MASKRTDF_MASK	BIT(FSL_AUDMIX_CTR_MASKRTDF_SHIFT)
+#define FSL_AUDMIX_CTR_MASKRTDF(i)	((i) << FSL_AUDMIX_CTR_MASKRTDF_SHIFT)
+#define FSL_AUDMIX_CTR_MASKCKDF_SHIFT	8
+#define FSL_AUDMIX_CTR_MASKCKDF_MASK	BIT(FSL_AUDMIX_CTR_MASKCKDF_SHIFT)
+#define FSL_AUDMIX_CTR_MASKCKDF(i)	((i) << FSL_AUDMIX_CTR_MASKCKDF_SHIFT)
+#define FSL_AUDMIX_CTR_SYNCMODE_SHIFT	9
+#define FSL_AUDMIX_CTR_SYNCMODE_MASK	BIT(FSL_AUDMIX_CTR_SYNCMODE_SHIFT)
+#define FSL_AUDMIX_CTR_SYNCMODE(i)	((i) << FSL_AUDMIX_CTR_SYNCMODE_SHIFT)
+#define FSL_AUDMIX_CTR_SYNCSRC_SHIFT	10
+#define FSL_AUDMIX_CTR_SYNCSRC_MASK	BIT(FSL_AUDMIX_CTR_SYNCSRC_SHIFT)
+#define FSL_AUDMIX_CTR_SYNCSRC(i)	((i) << FSL_AUDMIX_CTR_SYNCSRC_SHIFT)
+
+/* AUDMIX Status Register */
+#define FSL_AUDMIX_STR_RATEDIFF		BIT(0)
+#define FSL_AUDMIX_STR_CLKDIFF		BIT(1)
+#define FSL_AUDMIX_STR_MIXSTAT_SHIFT	2
+#define FSL_AUDMIX_STR_MIXSTAT_MASK	(0x3 << FSL_AUDMIX_STR_MIXSTAT_SHIFT)
+#define FSL_AUDMIX_STR_MIXSTAT(i)	(((i) & FSL_AUDMIX_STR_MIXSTAT_MASK) \
+					   >> FSL_AUDMIX_STR_MIXSTAT_SHIFT)
+/* AUDMIX Attenuation Control Register */
+#define FSL_AUDMIX_ATCR_AT_EN		BIT(0)
+#define FSL_AUDMIX_ATCR_AT_UPDN		BIT(1)
+#define FSL_AUDMIX_ATCR_ATSTPDIF_SHIFT	2
+#define FSL_AUDMIX_ATCR_ATSTPDFI_MASK	\
+				(0xfff << FSL_AUDMIX_ATCR_ATSTPDIF_SHIFT)
+
+/* AUDMIX Attenuation Initial Value Register */
+#define FSL_AUDMIX_ATIVAL_ATINVAL_MASK	0x3FFFF
+
+/* AUDMIX Attenuation Step Up Factor Register */
+#define FSL_AUDMIX_ATSTPUP_ATSTEPUP_MASK	0x3FFFF
+
+/* AUDMIX Attenuation Step Down Factor Register */
+#define FSL_AUDMIX_ATSTPDN_ATSTEPDN_MASK	0x3FFFF
+
+/* AUDMIX Attenuation Step Target Register */
+#define FSL_AUDMIX_ATSTPTGT_ATSTPTG_MASK	0x3FFFF
+
+/* AUDMIX Attenuation Value Register */
+#define FSL_AUDMIX_ATTNVAL_ATCURVAL_MASK	0x3FFFF
+
+/* AUDMIX Attenuation Step Number Register */
+#define FSL_AUDMIX_ATSTP_STPCTR_MASK	0x3FFFF
+
+#define FSL_AUDMIX_MAX_DAIS		2
+struct fsl_audmix {
+	struct platform_device *pdev;
+	struct regmap *regmap;
+	struct clk *ipg_clk;
+	u8 tdms;
+};
+
+#endif /* __FSL_AUDMIX_H */
diff --git a/sound/soc/fsl/fsl_dma.c b/sound/soc/fsl/fsl_dma.c
index 78871de35086..e22508301412 100644
--- a/sound/soc/fsl/fsl_dma.c
+++ b/sound/soc/fsl/fsl_dma.c
@@ -1,18 +1,14 @@
-/*
- * Freescale DMA ALSA SoC PCM driver
- *
- * Author: Timur Tabi <timur@freescale.com>
- *
- * Copyright 2007-2010 Freescale Semiconductor, Inc.
- *
- * This file is licensed under the terms of the GNU General Public License
- * version 2.  This program is licensed "as is" without any warranty of any
- * kind, whether express or implied.
- *
- * This driver implements ASoC support for the Elo DMA controller, which is
- * the DMA controller on Freescale 83xx, 85xx, and 86xx SOCs. In ALSA terms,
- * the PCM driver is what handles the DMA buffer.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Freescale DMA ALSA SoC PCM driver
+//
+// Author: Timur Tabi <timur@freescale.com>
+//
+// Copyright 2007-2010 Freescale Semiconductor, Inc.
+//
+// This driver implements ASoC support for the Elo DMA controller, which is
+// the DMA controller on Freescale 83xx, 85xx, and 86xx SOCs. In ALSA terms,
+// the PCM driver is what handles the DMA buffer.
 
 #include <linux/module.h>
 #include <linux/init.h>
diff --git a/sound/soc/fsl/fsl_dma.h b/sound/soc/fsl/fsl_dma.h
index 78fee97e8036..f19ae765b656 100644
--- a/sound/soc/fsl/fsl_dma.h
+++ b/sound/soc/fsl/fsl_dma.h
@@ -1,9 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * mpc8610-pcm.h - ALSA PCM interface for the Freescale MPC8610 SoC
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #ifndef _MPC8610_PCM_H
diff --git a/sound/soc/fsl/fsl_esai.c b/sound/soc/fsl/fsl_esai.c
index 3623aa9a6f2e..bad0dfed6b68 100644
--- a/sound/soc/fsl/fsl_esai.c
+++ b/sound/soc/fsl/fsl_esai.c
@@ -218,7 +218,7 @@ static int fsl_esai_set_dai_sysclk(struct snd_soc_dai *dai, int clk_id,
 {
 	struct fsl_esai *esai_priv = snd_soc_dai_get_drvdata(dai);
 	struct clk *clksrc = esai_priv->extalclk;
-	bool tx = clk_id <= ESAI_HCKT_EXTAL;
+	bool tx = (clk_id <= ESAI_HCKT_EXTAL || esai_priv->synchronous);
 	bool in = dir == SND_SOC_CLOCK_IN;
 	u32 ratio, ecr = 0;
 	unsigned long clk_rate;
@@ -251,9 +251,9 @@ static int fsl_esai_set_dai_sysclk(struct snd_soc_dai *dai, int clk_id,
 		break;
 	case ESAI_HCKT_EXTAL:
 		ecr |= ESAI_ECR_ETI;
-		/* fall through */
+		break;
 	case ESAI_HCKR_EXTAL:
-		ecr |= ESAI_ECR_ERI;
+		ecr |= esai_priv->synchronous ? ESAI_ECR_ETI : ESAI_ECR_ERI;
 		break;
 	default:
 		return -EINVAL;
@@ -537,10 +537,18 @@ static int fsl_esai_hw_params(struct snd_pcm_substream *substream,
 
 	bclk = params_rate(params) * slot_width * esai_priv->slots;
 
-	ret = fsl_esai_set_bclk(dai, tx, bclk);
+	ret = fsl_esai_set_bclk(dai, esai_priv->synchronous || tx, bclk);
 	if (ret)
 		return ret;
 
+	mask = ESAI_xCR_xSWS_MASK;
+	val = ESAI_xCR_xSWS(slot_width, width);
+
+	regmap_update_bits(esai_priv->regmap, REG_ESAI_xCR(tx), mask, val);
+	/* Recording in synchronous mode needs to set TCR also */
+	if (!tx && esai_priv->synchronous)
+		regmap_update_bits(esai_priv->regmap, REG_ESAI_TCR, mask, val);
+
 	/* Use Normal mode to support monaural audio */
 	regmap_update_bits(esai_priv->regmap, REG_ESAI_xCR(tx),
 			   ESAI_xCR_xMOD_MASK, params_channels(params) > 1 ?
@@ -556,10 +564,9 @@ static int fsl_esai_hw_params(struct snd_pcm_substream *substream,
 
 	regmap_update_bits(esai_priv->regmap, REG_ESAI_xFCR(tx), mask, val);
 
-	mask = ESAI_xCR_xSWS_MASK | (tx ? ESAI_xCR_PADC : 0);
-	val = ESAI_xCR_xSWS(slot_width, width) | (tx ? ESAI_xCR_PADC : 0);
-
-	regmap_update_bits(esai_priv->regmap, REG_ESAI_xCR(tx), mask, val);
+	if (tx)
+		regmap_update_bits(esai_priv->regmap, REG_ESAI_TCR,
+				ESAI_xCR_PADC, ESAI_xCR_PADC);
 
 	/* Remove ESAI personal reset by configuring ESAI_PCRC and ESAI_PRRC */
 	regmap_update_bits(esai_priv->regmap, REG_ESAI_PRRC,
diff --git a/sound/soc/fsl/fsl_micfil.c b/sound/soc/fsl/fsl_micfil.c
index 40c07e756481..f7f2d29f1bfe 100644
--- a/sound/soc/fsl/fsl_micfil.c
+++ b/sound/soc/fsl/fsl_micfil.c
@@ -151,12 +151,9 @@ static inline int get_clk_div(struct fsl_micfil *micfil,
 {
 	u32 ctrl2_reg;
 	long mclk_rate;
-	int osr;
 	int clk_div;
 
 	regmap_read(micfil->regmap, REG_MICFIL_CTRL2, &ctrl2_reg);
-	osr = 16 - ((ctrl2_reg & MICFIL_CTRL2_CICOSR_MASK)
-		    >> MICFIL_CTRL2_CICOSR_SHIFT);
 
 	mclk_rate = clk_get_rate(micfil->mclk);
 
diff --git a/sound/soc/fsl/fsl_sai.c b/sound/soc/fsl/fsl_sai.c
index db9e0872f73d..8593269156bd 100644
--- a/sound/soc/fsl/fsl_sai.c
+++ b/sound/soc/fsl/fsl_sai.c
@@ -9,6 +9,7 @@
 #include <linux/dmaengine.h>
 #include <linux/module.h>
 #include <linux/of_address.h>
+#include <linux/pm_runtime.h>
 #include <linux/regmap.h>
 #include <linux/slab.h>
 #include <linux/time.h>
@@ -268,12 +269,14 @@ static int fsl_sai_set_dai_fmt_tr(struct snd_soc_dai *cpu_dai,
 	case SND_SOC_DAIFMT_CBS_CFS:
 		val_cr2 |= FSL_SAI_CR2_BCD_MSTR;
 		val_cr4 |= FSL_SAI_CR4_FSD_MSTR;
+		sai->is_slave_mode = false;
 		break;
 	case SND_SOC_DAIFMT_CBM_CFM:
 		sai->is_slave_mode = true;
 		break;
 	case SND_SOC_DAIFMT_CBS_CFM:
 		val_cr2 |= FSL_SAI_CR2_BCD_MSTR;
+		sai->is_slave_mode = false;
 		break;
 	case SND_SOC_DAIFMT_CBM_CFS:
 		val_cr4 |= FSL_SAI_CR4_FSD_MSTR;
@@ -899,6 +902,8 @@ static int fsl_sai_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, sai);
 
+	pm_runtime_enable(&pdev->dev);
+
 	ret = devm_snd_soc_register_component(&pdev->dev, &fsl_component,
 			&fsl_sai_dai, 1);
 	if (ret)
@@ -910,6 +915,13 @@ static int fsl_sai_probe(struct platform_device *pdev)
 		return devm_snd_dmaengine_pcm_register(&pdev->dev, NULL, 0);
 }
 
+static int fsl_sai_remove(struct platform_device *pdev)
+{
+	pm_runtime_disable(&pdev->dev);
+
+	return 0;
+}
+
 static const struct of_device_id fsl_sai_ids[] = {
 	{ .compatible = "fsl,vf610-sai", },
 	{ .compatible = "fsl,imx6sx-sai", },
@@ -918,8 +930,8 @@ static const struct of_device_id fsl_sai_ids[] = {
 };
 MODULE_DEVICE_TABLE(of, fsl_sai_ids);
 
-#ifdef CONFIG_PM_SLEEP
-static int fsl_sai_suspend(struct device *dev)
+#ifdef CONFIG_PM
+static int fsl_sai_runtime_suspend(struct device *dev)
 {
 	struct fsl_sai *sai = dev_get_drvdata(dev);
 
@@ -929,7 +941,7 @@ static int fsl_sai_suspend(struct device *dev)
 	return 0;
 }
 
-static int fsl_sai_resume(struct device *dev)
+static int fsl_sai_runtime_resume(struct device *dev)
 {
 	struct fsl_sai *sai = dev_get_drvdata(dev);
 
@@ -941,14 +953,18 @@ static int fsl_sai_resume(struct device *dev)
 	regmap_write(sai->regmap, FSL_SAI_RCSR, 0);
 	return regcache_sync(sai->regmap);
 }
-#endif /* CONFIG_PM_SLEEP */
+#endif /* CONFIG_PM */
 
 static const struct dev_pm_ops fsl_sai_pm_ops = {
-	SET_SYSTEM_SLEEP_PM_OPS(fsl_sai_suspend, fsl_sai_resume)
+	SET_RUNTIME_PM_OPS(fsl_sai_runtime_suspend,
+			   fsl_sai_runtime_resume, NULL)
+	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
+				pm_runtime_force_resume)
 };
 
 static struct platform_driver fsl_sai_driver = {
 	.probe = fsl_sai_probe,
+	.remove = fsl_sai_remove,
 	.driver = {
 		.name = "fsl-sai",
 		.pm = &fsl_sai_pm_ops,
diff --git a/sound/soc/fsl/fsl_utils.c b/sound/soc/fsl/fsl_utils.c
index 9981668ab590..040d06b89f00 100644
--- a/sound/soc/fsl/fsl_utils.c
+++ b/sound/soc/fsl/fsl_utils.c
@@ -71,6 +71,7 @@ int fsl_asoc_get_dma_channel(struct device_node *ssi_np,
 	iprop = of_get_property(dma_np, "cell-index", NULL);
 	if (!iprop) {
 		of_node_put(dma_np);
+		of_node_put(dma_channel_np);
 		return -EINVAL;
 	}
 	*dma_id = be32_to_cpup(iprop);
diff --git a/sound/soc/fsl/imx-audmix.c b/sound/soc/fsl/imx-audmix.c
new file mode 100644
index 000000000000..9aaf3e5b45b9
--- /dev/null
+++ b/sound/soc/fsl/imx-audmix.c
@@ -0,0 +1,331 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2017 NXP
+ *
+ * The code contained herein is licensed under the GNU General Public
+ * License. You may obtain a copy of the GNU General Public License
+ * Version 2 or later at the following locations:
+ *
+ * http://www.opensource.org/licenses/gpl-license.html
+ * http://www.gnu.org/copyleft/gpl.html
+ */
+
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <linux/clk.h>
+#include <sound/soc.h>
+#include <sound/soc-dapm.h>
+#include <linux/pm_runtime.h>
+#include "fsl_sai.h"
+#include "fsl_audmix.h"
+
+struct imx_audmix {
+	struct platform_device *pdev;
+	struct snd_soc_card card;
+	struct platform_device *audmix_pdev;
+	struct platform_device *out_pdev;
+	struct clk *cpu_mclk;
+	int num_dai;
+	struct snd_soc_dai_link *dai;
+	int num_dai_conf;
+	struct snd_soc_codec_conf *dai_conf;
+	int num_dapm_routes;
+	struct snd_soc_dapm_route *dapm_routes;
+};
+
+static const u32 imx_audmix_rates[] = {
+	8000, 12000, 16000, 24000, 32000, 48000, 64000, 96000,
+};
+
+static const struct snd_pcm_hw_constraint_list imx_audmix_rate_constraints = {
+	.count = ARRAY_SIZE(imx_audmix_rates),
+	.list = imx_audmix_rates,
+};
+
+static int imx_audmix_fe_startup(struct snd_pcm_substream *substream)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct imx_audmix *priv = snd_soc_card_get_drvdata(rtd->card);
+	struct snd_pcm_runtime *runtime = substream->runtime;
+	struct device *dev = rtd->card->dev;
+	unsigned long clk_rate = clk_get_rate(priv->cpu_mclk);
+	int ret;
+
+	if (clk_rate % 24576000 == 0) {
+		ret = snd_pcm_hw_constraint_list(runtime, 0,
+						 SNDRV_PCM_HW_PARAM_RATE,
+						 &imx_audmix_rate_constraints);
+		if (ret < 0)
+			return ret;
+	} else {
+		dev_warn(dev, "mclk may be not supported %lu\n", clk_rate);
+	}
+
+	ret = snd_pcm_hw_constraint_minmax(runtime, SNDRV_PCM_HW_PARAM_CHANNELS,
+					   1, 8);
+	if (ret < 0)
+		return ret;
+
+	return snd_pcm_hw_constraint_mask64(runtime, SNDRV_PCM_HW_PARAM_FORMAT,
+					    FSL_AUDMIX_FORMATS);
+}
+
+static int imx_audmix_fe_hw_params(struct snd_pcm_substream *substream,
+				   struct snd_pcm_hw_params *params)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct device *dev = rtd->card->dev;
+	bool tx = substream->stream == SNDRV_PCM_STREAM_PLAYBACK;
+	unsigned int fmt = SND_SOC_DAIFMT_DSP_A | SND_SOC_DAIFMT_NB_NF;
+	u32 channels = params_channels(params);
+	int ret, dir;
+
+	/* For playback the AUDMIX is slave, and for record is master */
+	fmt |= tx ? SND_SOC_DAIFMT_CBS_CFS : SND_SOC_DAIFMT_CBM_CFM;
+	dir  = tx ? SND_SOC_CLOCK_OUT : SND_SOC_CLOCK_IN;
+
+	/* set DAI configuration */
+	ret = snd_soc_dai_set_fmt(rtd->cpu_dai, fmt);
+	if (ret) {
+		dev_err(dev, "failed to set cpu dai fmt: %d\n", ret);
+		return ret;
+	}
+
+	ret = snd_soc_dai_set_sysclk(rtd->cpu_dai, FSL_SAI_CLK_MAST1, 0, dir);
+	if (ret) {
+		dev_err(dev, "failed to set cpu sysclk: %d\n", ret);
+		return ret;
+	}
+
+	/*
+	 * Per datasheet, AUDMIX expects 8 slots and 32 bits
+	 * for every slot in TDM mode.
+	 */
+	ret = snd_soc_dai_set_tdm_slot(rtd->cpu_dai, BIT(channels) - 1,
+				       BIT(channels) - 1, 8, 32);
+	if (ret)
+		dev_err(dev, "failed to set cpu dai tdm slot: %d\n", ret);
+
+	return ret;
+}
+
+static int imx_audmix_be_hw_params(struct snd_pcm_substream *substream,
+				   struct snd_pcm_hw_params *params)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct device *dev = rtd->card->dev;
+	bool tx = substream->stream == SNDRV_PCM_STREAM_PLAYBACK;
+	unsigned int fmt = SND_SOC_DAIFMT_DSP_A | SND_SOC_DAIFMT_NB_NF;
+	int ret;
+
+	if (!tx)
+		return 0;
+
+	/* For playback the AUDMIX is slave */
+	fmt |= SND_SOC_DAIFMT_CBM_CFM;
+
+	/* set AUDMIX DAI configuration */
+	ret = snd_soc_dai_set_fmt(rtd->cpu_dai, fmt);
+	if (ret)
+		dev_err(dev, "failed to set AUDMIX DAI fmt: %d\n", ret);
+
+	return ret;
+}
+
+static struct snd_soc_ops imx_audmix_fe_ops = {
+	.startup = imx_audmix_fe_startup,
+	.hw_params = imx_audmix_fe_hw_params,
+};
+
+static struct snd_soc_ops imx_audmix_be_ops = {
+	.hw_params = imx_audmix_be_hw_params,
+};
+
+static int imx_audmix_probe(struct platform_device *pdev)
+{
+	struct device_node *np = pdev->dev.of_node;
+	struct device_node *audmix_np = NULL, *out_cpu_np = NULL;
+	struct platform_device *audmix_pdev = NULL;
+	struct platform_device *cpu_pdev;
+	struct of_phandle_args args;
+	struct imx_audmix *priv;
+	int i, num_dai, ret;
+	const char *fe_name_pref = "HiFi-AUDMIX-FE-";
+	char *be_name, *be_pb, *be_cp, *dai_name, *capture_dai_name;
+
+	if (pdev->dev.parent) {
+		audmix_np = pdev->dev.parent->of_node;
+	} else {
+		dev_err(&pdev->dev, "Missing parent device.\n");
+		return -EINVAL;
+	}
+
+	if (!audmix_np) {
+		dev_err(&pdev->dev, "Missing DT node for parent device.\n");
+		return -EINVAL;
+	}
+
+	audmix_pdev = of_find_device_by_node(audmix_np);
+	if (!audmix_pdev) {
+		dev_err(&pdev->dev, "Missing AUDMIX platform device for %s\n",
+			np->full_name);
+		return -EINVAL;
+	}
+	put_device(&audmix_pdev->dev);
+
+	num_dai = of_count_phandle_with_args(audmix_np, "dais", NULL);
+	if (num_dai != FSL_AUDMIX_MAX_DAIS) {
+		dev_err(&pdev->dev, "Need 2 dais to be provided for %s\n",
+			audmix_np->full_name);
+		return -EINVAL;
+	}
+
+	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->num_dai = 2 * num_dai;
+	priv->dai = devm_kzalloc(&pdev->dev, priv->num_dai *
+				 sizeof(struct snd_soc_dai_link), GFP_KERNEL);
+	if (!priv->dai)
+		return -ENOMEM;
+
+	priv->num_dai_conf = num_dai;
+	priv->dai_conf = devm_kzalloc(&pdev->dev, priv->num_dai_conf *
+				      sizeof(struct snd_soc_codec_conf),
+				      GFP_KERNEL);
+	if (!priv->dai_conf)
+		return -ENOMEM;
+
+	priv->num_dapm_routes = 3 * num_dai;
+	priv->dapm_routes = devm_kzalloc(&pdev->dev, priv->num_dapm_routes *
+					 sizeof(struct snd_soc_dapm_route),
+					 GFP_KERNEL);
+	if (!priv->dapm_routes)
+		return -ENOMEM;
+
+	for (i = 0; i < num_dai; i++) {
+		ret = of_parse_phandle_with_args(audmix_np, "dais", NULL, i,
+						 &args);
+		if (ret < 0) {
+			dev_err(&pdev->dev, "of_parse_phandle_with_args failed\n");
+			return ret;
+		}
+
+		cpu_pdev = of_find_device_by_node(args.np);
+		if (!cpu_pdev) {
+			dev_err(&pdev->dev, "failed to find SAI platform device\n");
+			return -EINVAL;
+		}
+		put_device(&cpu_pdev->dev);
+
+		dai_name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s%s",
+					  fe_name_pref, args.np->full_name + 1);
+
+		dev_info(pdev->dev.parent, "DAI FE name:%s\n", dai_name);
+
+		if (i == 0) {
+			out_cpu_np = args.np;
+			capture_dai_name =
+				devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s %s",
+					       dai_name, "CPU-Capture");
+		}
+
+		priv->dai[i].name = dai_name;
+		priv->dai[i].stream_name = "HiFi-AUDMIX-FE";
+		priv->dai[i].codec_dai_name = "snd-soc-dummy-dai";
+		priv->dai[i].codec_name = "snd-soc-dummy";
+		priv->dai[i].cpu_of_node = args.np;
+		priv->dai[i].cpu_dai_name = dev_name(&cpu_pdev->dev);
+		priv->dai[i].platform_of_node = args.np;
+		priv->dai[i].dynamic = 1;
+		priv->dai[i].dpcm_playback = 1;
+		priv->dai[i].dpcm_capture = (i == 0 ? 1 : 0);
+		priv->dai[i].ignore_pmdown_time = 1;
+		priv->dai[i].ops = &imx_audmix_fe_ops;
+
+		/* Add AUDMIX Backend */
+		be_name = devm_kasprintf(&pdev->dev, GFP_KERNEL,
+					 "audmix-%d", i);
+		be_pb = devm_kasprintf(&pdev->dev, GFP_KERNEL,
+				       "AUDMIX-Playback-%d", i);
+		be_cp = devm_kasprintf(&pdev->dev, GFP_KERNEL,
+				       "AUDMIX-Capture-%d", i);
+
+		priv->dai[num_dai + i].name = be_name;
+		priv->dai[num_dai + i].codec_dai_name = "snd-soc-dummy-dai";
+		priv->dai[num_dai + i].codec_name = "snd-soc-dummy";
+		priv->dai[num_dai + i].cpu_of_node = audmix_np;
+		priv->dai[num_dai + i].cpu_dai_name = be_name;
+		priv->dai[num_dai + i].platform_name = "snd-soc-dummy";
+		priv->dai[num_dai + i].no_pcm = 1;
+		priv->dai[num_dai + i].dpcm_playback = 1;
+		priv->dai[num_dai + i].dpcm_capture  = 1;
+		priv->dai[num_dai + i].ignore_pmdown_time = 1;
+		priv->dai[num_dai + i].ops = &imx_audmix_be_ops;
+
+		priv->dai_conf[i].of_node = args.np;
+		priv->dai_conf[i].name_prefix = dai_name;
+
+		priv->dapm_routes[i].source =
+			devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s %s",
+				       dai_name, "CPU-Playback");
+		priv->dapm_routes[i].sink = be_pb;
+		priv->dapm_routes[num_dai + i].source   = be_pb;
+		priv->dapm_routes[num_dai + i].sink     = be_cp;
+		priv->dapm_routes[2 * num_dai + i].source = be_cp;
+		priv->dapm_routes[2 * num_dai + i].sink   = capture_dai_name;
+	}
+
+	cpu_pdev = of_find_device_by_node(out_cpu_np);
+	if (!cpu_pdev) {
+		dev_err(&pdev->dev, "failed to find SAI platform device\n");
+		return -EINVAL;
+	}
+	put_device(&cpu_pdev->dev);
+
+	priv->cpu_mclk = devm_clk_get(&cpu_pdev->dev, "mclk1");
+	if (IS_ERR(priv->cpu_mclk)) {
+		ret = PTR_ERR(priv->cpu_mclk);
+		dev_err(&cpu_pdev->dev, "failed to get DAI mclk1: %d\n", ret);
+		return -EINVAL;
+	}
+
+	priv->audmix_pdev = audmix_pdev;
+	priv->out_pdev  = cpu_pdev;
+
+	priv->card.dai_link = priv->dai;
+	priv->card.num_links = priv->num_dai;
+	priv->card.codec_conf = priv->dai_conf;
+	priv->card.num_configs = priv->num_dai_conf;
+	priv->card.dapm_routes = priv->dapm_routes;
+	priv->card.num_dapm_routes = priv->num_dapm_routes;
+	priv->card.dev = pdev->dev.parent;
+	priv->card.owner = THIS_MODULE;
+	priv->card.name = "imx-audmix";
+
+	platform_set_drvdata(pdev, &priv->card);
+	snd_soc_card_set_drvdata(&priv->card, priv);
+
+	ret = devm_snd_soc_register_card(pdev->dev.parent, &priv->card);
+	if (ret) {
+		dev_err(&pdev->dev, "snd_soc_register_card failed\n");
+		return ret;
+	}
+
+	return ret;
+}
+
+static struct platform_driver imx_audmix_driver = {
+	.probe = imx_audmix_probe,
+	.driver = {
+		.name = "imx-audmix",
+		.pm = &snd_soc_pm_ops,
+	},
+};
+module_platform_driver(imx_audmix_driver);
+
+MODULE_DESCRIPTION("NXP AUDMIX ASoC machine driver");
+MODULE_AUTHOR("Viorel Suman <viorel.suman@nxp.com>");
+MODULE_ALIAS("platform:imx-audmix");
+MODULE_LICENSE("GPL v2");
diff --git a/sound/soc/fsl/imx-audmux.c b/sound/soc/fsl/imx-audmux.c
index 99e07b01a2ce..04e59e66711d 100644
--- a/sound/soc/fsl/imx-audmux.c
+++ b/sound/soc/fsl/imx-audmux.c
@@ -1,21 +1,11 @@
-/*
- * Copyright 2012 Freescale Semiconductor, Inc.
- * Copyright 2012 Linaro Ltd.
- * Copyright 2009 Pengutronix, Sascha Hauer <s.hauer@pengutronix.de>
- *
- * Initial development of this code was funded by
- * Phytec Messtechnik GmbH, http://www.phytec.de
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright 2012 Freescale Semiconductor, Inc.
+// Copyright 2012 Linaro Ltd.
+// Copyright 2009 Pengutronix, Sascha Hauer <s.hauer@pengutronix.de>
+//
+// Initial development of this code was funded by
+// Phytec Messtechnik GmbH, http://www.phytec.de
 
 #include <linux/clk.h>
 #include <linux/debugfs.h>
diff --git a/sound/soc/fsl/imx-es8328.c b/sound/soc/fsl/imx-es8328.c
index 9953438086e4..c9d8739b04a9 100644
--- a/sound/soc/fsl/imx-es8328.c
+++ b/sound/soc/fsl/imx-es8328.c
@@ -1,14 +1,7 @@
-/*
- * Copyright 2012 Freescale Semiconductor, Inc.
- * Copyright 2012 Linaro Ltd.
- *
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright 2012 Freescale Semiconductor, Inc.
+// Copyright 2012 Linaro Ltd.
 
 #include <linux/gpio.h>
 #include <linux/module.h>
diff --git a/sound/soc/fsl/imx-mc13783.c b/sound/soc/fsl/imx-mc13783.c
index 9d19b808f634..545815a27074 100644
--- a/sound/soc/fsl/imx-mc13783.c
+++ b/sound/soc/fsl/imx-mc13783.c
@@ -1,17 +1,11 @@
-/*
- * imx-mc13783.c  --  SoC audio for imx based boards with mc13783 codec
- *
- * Copyright 2012 Philippe Retornaz, <philippe.retornaz@epfl.ch>
- *
- * Heavly based on phycore-mc13783:
- * Copyright 2009 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>
- *
- *  This program is free software; you can redistribute  it and/or modify it
- *  under  the terms of  the GNU General  Public License as published by the
- *  Free Software Foundation;  either version 2 of the  License, or (at your
- *  option) any later version.
- *
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// imx-mc13783.c  --  SoC audio for imx based boards with mc13783 codec
+//
+// Copyright 2012 Philippe Retornaz, <philippe.retornaz@epfl.ch>
+//
+// Heavly based on phycore-mc13783:
+// Copyright 2009 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>
 
 #include <linux/module.h>
 #include <linux/moduleparam.h>
diff --git a/sound/soc/fsl/imx-pcm-fiq.c b/sound/soc/fsl/imx-pcm-fiq.c
index 0578f3486847..c49aea4fba56 100644
--- a/sound/soc/fsl/imx-pcm-fiq.c
+++ b/sound/soc/fsl/imx-pcm-fiq.c
@@ -1,16 +1,11 @@
-/*
- * imx-pcm-fiq.c  --  ALSA Soc Audio Layer
- *
- * Copyright 2009 Sascha Hauer <s.hauer@pengutronix.de>
- *
- * This code is based on code copyrighted by Freescale,
- * Liam Girdwood, Javier Martin and probably others.
- *
- *  This program is free software; you can redistribute  it and/or modify it
- *  under  the terms of  the GNU General  Public License as published by the
- *  Free Software Foundation;  either version 2 of the  License, or (at your
- *  option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+// imx-pcm-fiq.c  --  ALSA Soc Audio Layer
+//
+// Copyright 2009 Sascha Hauer <s.hauer@pengutronix.de>
+//
+// This code is based on code copyrighted by Freescale,
+// Liam Girdwood, Javier Martin and probably others.
+
 #include <linux/clk.h>
 #include <linux/delay.h>
 #include <linux/device.h>
diff --git a/sound/soc/fsl/imx-pcm.h b/sound/soc/fsl/imx-pcm.h
index 133c4470acad..5dd406774d3e 100644
--- a/sound/soc/fsl/imx-pcm.h
+++ b/sound/soc/fsl/imx-pcm.h
@@ -1,13 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * Copyright 2009 Sascha Hauer <s.hauer@pengutronix.de>
  *
  * This code is based on code copyrighted by Freescale,
  * Liam Girdwood, Javier Martin and probably others.
- *
- * This program is free software; you can redistribute  it and/or modify it
- * under  the terms of  the GNU General  Public License as published by the
- * Free Software Foundation;  either version 2 of the  License, or (at your
- * option) any later version.
  */
 
 #ifndef _IMX_PCM_H
diff --git a/sound/soc/fsl/imx-spdif.c b/sound/soc/fsl/imx-spdif.c
index 797d66e43d49..4f7f210beb18 100644
--- a/sound/soc/fsl/imx-spdif.c
+++ b/sound/soc/fsl/imx-spdif.c
@@ -1,13 +1,6 @@
-/*
- * Copyright (C) 2013 Freescale Semiconductor, Inc.
- *
- * The code contained herein is licensed under the GNU General Public
- * License. You may obtain a copy of the GNU General Public License
- * Version 2 or later at the following locations:
- *
- * http://www.opensource.org/licenses/gpl-license.html
- * http://www.gnu.org/copyleft/gpl.html
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright (C) 2013 Freescale Semiconductor, Inc.
 
 #include <linux/module.h>
 #include <linux/of_platform.h>
diff --git a/sound/soc/fsl/imx-ssi.c b/sound/soc/fsl/imx-ssi.c
index 06790615e04e..9038b61317be 100644
--- a/sound/soc/fsl/imx-ssi.c
+++ b/sound/soc/fsl/imx-ssi.c
@@ -1,35 +1,28 @@
-/*
- * imx-ssi.c  --  ALSA Soc Audio Layer
- *
- * Copyright 2009 Sascha Hauer <s.hauer@pengutronix.de>
- *
- * This code is based on code copyrighted by Freescale,
- * Liam Girdwood, Javier Martin and probably others.
- *
- *  This program is free software; you can redistribute  it and/or modify it
- *  under  the terms of  the GNU General  Public License as published by the
- *  Free Software Foundation;  either version 2 of the  License, or (at your
- *  option) any later version.
- *
- *
- * The i.MX SSI core has some nasty limitations in AC97 mode. While most
- * sane processor vendors have a FIFO per AC97 slot, the i.MX has only
- * one FIFO which combines all valid receive slots. We cannot even select
- * which slots we want to receive. The WM9712 with which this driver
- * was developed with always sends GPIO status data in slot 12 which
- * we receive in our (PCM-) data stream. The only chance we have is to
- * manually skip this data in the FIQ handler. With sampling rates different
- * from 48000Hz not every frame has valid receive data, so the ratio
- * between pcm data and GPIO status data changes. Our FIQ handler is not
- * able to handle this, hence this driver only works with 48000Hz sampling
- * rate.
- * Reading and writing AC97 registers is another challenge. The core
- * provides us status bits when the read register is updated with *another*
- * value. When we read the same register two times (and the register still
- * contains the same value) these status bits are not set. We work
- * around this by not polling these bits but only wait a fixed delay.
- *
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// imx-ssi.c  --  ALSA Soc Audio Layer
+//
+// Copyright 2009 Sascha Hauer <s.hauer@pengutronix.de>
+//
+// This code is based on code copyrighted by Freescale,
+// Liam Girdwood, Javier Martin and probably others.
+//
+// The i.MX SSI core has some nasty limitations in AC97 mode. While most
+// sane processor vendors have a FIFO per AC97 slot, the i.MX has only
+// one FIFO which combines all valid receive slots. We cannot even select
+// which slots we want to receive. The WM9712 with which this driver
+// was developed with always sends GPIO status data in slot 12 which
+// we receive in our (PCM-) data stream. The only chance we have is to
+// manually skip this data in the FIQ handler. With sampling rates different
+// from 48000Hz not every frame has valid receive data, so the ratio
+// between pcm data and GPIO status data changes. Our FIQ handler is not
+// able to handle this, hence this driver only works with 48000Hz sampling
+// rate.
+// Reading and writing AC97 registers is another challenge. The core
+// provides us status bits when the read register is updated with *another*
+// value. When we read the same register two times (and the register still
+// contains the same value) these status bits are not set. We work
+// around this by not polling these bits but only wait a fixed delay.
 
 #include <linux/clk.h>
 #include <linux/delay.h>
diff --git a/sound/soc/fsl/imx-ssi.h b/sound/soc/fsl/imx-ssi.h
index be6562365b6a..19cd0937e740 100644
--- a/sound/soc/fsl/imx-ssi.h
+++ b/sound/soc/fsl/imx-ssi.h
@@ -1,8 +1,4 @@
-/*
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+/* SPDX-License-Identifier: GPL-2.0 */
 
 #ifndef _IMX_SSI_H
 #define _IMX_SSI_H
diff --git a/sound/soc/fsl/mpc5200_dma.c b/sound/soc/fsl/mpc5200_dma.c
index c1a4544eb16b..ccf9301889fe 100644
--- a/sound/soc/fsl/mpc5200_dma.c
+++ b/sound/soc/fsl/mpc5200_dma.c
@@ -1,10 +1,10 @@
-/*
- * Freescale MPC5200 PSC DMA
- * ALSA SoC Platform driver
- *
- * Copyright (C) 2008 Secret Lab Technologies Ltd.
- * Copyright (C) 2009 Jon Smirl, Digispeaker
- */
+// SPDX-License-Identifier: GPL-2.0-only
+//
+// Freescale MPC5200 PSC DMA
+// ALSA SoC Platform driver
+//
+// Copyright (C) 2008 Secret Lab Technologies Ltd.
+// Copyright (C) 2009 Jon Smirl, Digispeaker
 
 #include <linux/module.h>
 #include <linux/of_device.h>
diff --git a/sound/soc/fsl/mpc5200_psc_ac97.c b/sound/soc/fsl/mpc5200_psc_ac97.c
index 07ee355ee385..e5b9c04d1565 100644
--- a/sound/soc/fsl/mpc5200_psc_ac97.c
+++ b/sound/soc/fsl/mpc5200_psc_ac97.c
@@ -1,13 +1,9 @@
-/*
- * linux/sound/mpc5200-ac97.c -- AC97 support for the Freescale MPC52xx chip.
- *
- * Copyright (C) 2009 Jon Smirl, Digispeaker
- * Author: Jon Smirl <jonsmirl@gmail.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// linux/sound/mpc5200-ac97.c -- AC97 support for the Freescale MPC52xx chip.
+//
+// Copyright (C) 2009 Jon Smirl, Digispeaker
+// Author: Jon Smirl <jonsmirl@gmail.com>
 
 #include <linux/module.h>
 #include <linux/of_device.h>
diff --git a/sound/soc/fsl/mpc5200_psc_i2s.c b/sound/soc/fsl/mpc5200_psc_i2s.c
index d8232943ccb6..9bc01f374b39 100644
--- a/sound/soc/fsl/mpc5200_psc_i2s.c
+++ b/sound/soc/fsl/mpc5200_psc_i2s.c
@@ -1,10 +1,10 @@
-/*
- * Freescale MPC5200 PSC in I2S mode
- * ALSA SoC Digital Audio Interface (DAI) driver
- *
- * Copyright (C) 2008 Secret Lab Technologies Ltd.
- * Copyright (C) 2009 Jon Smirl, Digispeaker
- */
+// SPDX-License-Identifier: GPL-2.0-only
+//
+// Freescale MPC5200 PSC in I2S mode
+// ALSA SoC Digital Audio Interface (DAI) driver
+//
+// Copyright (C) 2008 Secret Lab Technologies Ltd.
+// Copyright (C) 2009 Jon Smirl, Digispeaker
 
 #include <linux/module.h>
 #include <linux/of_device.h>
diff --git a/sound/soc/fsl/mpc8610_hpcd.c b/sound/soc/fsl/mpc8610_hpcd.c
index a639b52c16f6..f6261a3eeb0f 100644
--- a/sound/soc/fsl/mpc8610_hpcd.c
+++ b/sound/soc/fsl/mpc8610_hpcd.c
@@ -1,14 +1,10 @@
-/**
- * Freescale MPC8610HPCD ALSA SoC Machine driver
- *
- * Author: Timur Tabi <timur@freescale.com>
- *
- * Copyright 2007-2010 Freescale Semiconductor, Inc.
- *
- * This file is licensed under the terms of the GNU General Public License
- * version 2.  This program is licensed "as is" without any warranty of any
- * kind, whether express or implied.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Freescale MPC8610HPCD ALSA SoC Machine driver
+//
+// Author: Timur Tabi <timur@freescale.com>
+//
+// Copyright 2007-2010 Freescale Semiconductor, Inc.
 
 #include <linux/module.h>
 #include <linux/interrupt.h>
diff --git a/sound/soc/fsl/mx27vis-aic32x4.c b/sound/soc/fsl/mx27vis-aic32x4.c
index d7ec3d20065c..37a4520aef62 100644
--- a/sound/soc/fsl/mx27vis-aic32x4.c
+++ b/sound/soc/fsl/mx27vis-aic32x4.c
@@ -1,25 +1,10 @@
-/*
- * mx27vis-aic32x4.c
- *
- * Copyright 2011 Vista Silicon S.L.
- *
- * Author: Javier Martin <javier.martin@vista-silicon.com>
- *
- * This program is free software; you can redistribute  it and/or modify it
- * under  the terms of  the GNU General  Public License as published by the
- * Free Software Foundation;  either version 2 of the  License, or (at your
- * option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- * MA 02110-1301, USA.
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// mx27vis-aic32x4.c
+//
+// Copyright 2011 Vista Silicon S.L.
+//
+// Author: Javier Martin <javier.martin@vista-silicon.com>
 
 #include <linux/module.h>
 #include <linux/moduleparam.h>
diff --git a/sound/soc/fsl/p1022_ds.c b/sound/soc/fsl/p1022_ds.c
index 41c623c55c16..80384f70878d 100644
--- a/sound/soc/fsl/p1022_ds.c
+++ b/sound/soc/fsl/p1022_ds.c
@@ -1,14 +1,10 @@
-/**
- * Freescale P1022DS ALSA SoC Machine driver
- *
- * Author: Timur Tabi <timur@freescale.com>
- *
- * Copyright 2010 Freescale Semiconductor, Inc.
- *
- * This file is licensed under the terms of the GNU General Public License
- * version 2.  This program is licensed "as is" without any warranty of any
- * kind, whether express or implied.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Freescale P1022DS ALSA SoC Machine driver
+//
+// Author: Timur Tabi <timur@freescale.com>
+//
+// Copyright 2010 Freescale Semiconductor, Inc.
 
 #include <linux/module.h>
 #include <linux/fsl/guts.h>
diff --git a/sound/soc/fsl/p1022_rdk.c b/sound/soc/fsl/p1022_rdk.c
index 4afbdd610bfa..1c32c2d8c6b0 100644
--- a/sound/soc/fsl/p1022_rdk.c
+++ b/sound/soc/fsl/p1022_rdk.c
@@ -1,21 +1,17 @@
-/**
- * Freescale P1022RDK ALSA SoC Machine driver
- *
- * Author: Timur Tabi <timur@freescale.com>
- *
- * Copyright 2012 Freescale Semiconductor, Inc.
- *
- * This file is licensed under the terms of the GNU General Public License
- * version 2.  This program is licensed "as is" without any warranty of any
- * kind, whether express or implied.
- *
- * Note: in order for audio to work correctly, the output controls need
- * to be enabled, because they control the clock.  So for playback, for
- * example:
- *
- *      amixer sset 'Left Output Mixer PCM' on
- *      amixer sset 'Right Output Mixer PCM' on
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Freescale P1022RDK ALSA SoC Machine driver
+//
+// Author: Timur Tabi <timur@freescale.com>
+//
+// Copyright 2012 Freescale Semiconductor, Inc.
+//
+// Note: in order for audio to work correctly, the output controls need
+// to be enabled, because they control the clock.  So for playback, for
+// example:
+//
+//      amixer sset 'Left Output Mixer PCM' on
+//      amixer sset 'Right Output Mixer PCM' on
 
 #include <linux/module.h>
 #include <linux/fsl/guts.h>
diff --git a/sound/soc/fsl/pcm030-audio-fabric.c b/sound/soc/fsl/pcm030-audio-fabric.c
index e339f36cea95..a7fe4ad25c52 100644
--- a/sound/soc/fsl/pcm030-audio-fabric.c
+++ b/sound/soc/fsl/pcm030-audio-fabric.c
@@ -1,14 +1,10 @@
-/*
- * Phytec pcm030 driver for the PSC of the Freescale MPC52xx
- * configured as AC97 interface
- *
- * Copyright 2008 Jon Smirl, Digispeaker
- * Author: Jon Smirl <jonsmirl@gmail.com>
- *
- * This file is licensed under the terms of the GNU General Public License
- * version 2. This program is licensed "as is" without any warranty of any
- * kind, whether express or implied.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Phytec pcm030 driver for the PSC of the Freescale MPC52xx
+// configured as AC97 interface
+//
+// Copyright 2008 Jon Smirl, Digispeaker
+// Author: Jon Smirl <jonsmirl@gmail.com>
 
 #include <linux/init.h>
 #include <linux/module.h>
diff --git a/sound/soc/fsl/phycore-ac97.c b/sound/soc/fsl/phycore-ac97.c
index 66fb6c4614d2..fe7ba6db7c96 100644
--- a/sound/soc/fsl/phycore-ac97.c
+++ b/sound/soc/fsl/phycore-ac97.c
@@ -1,14 +1,8 @@
-/*
- * phycore-ac97.c  --  SoC audio for imx_phycore in AC97 mode
- *
- * Copyright 2009 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>
- *
- *  This program is free software; you can redistribute  it and/or modify it
- *  under  the terms of  the GNU General  Public License as published by the
- *  Free Software Foundation;  either version 2 of the  License, or (at your
- *  option) any later version.
- *
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// phycore-ac97.c  --  SoC audio for imx_phycore in AC97 mode
+//
+// Copyright 2009 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>
 
 #include <linux/module.h>
 #include <linux/moduleparam.h>
diff --git a/sound/soc/fsl/wm1133-ev1.c b/sound/soc/fsl/wm1133-ev1.c
index 2f80b21b2921..aad24ccbef90 100644
--- a/sound/soc/fsl/wm1133-ev1.c
+++ b/sound/soc/fsl/wm1133-ev1.c
@@ -1,16 +1,11 @@
-/*
- *  wm1133-ev1.c - Audio for WM1133-EV1 on i.MX31ADS
- *
- *  Copyright (c) 2010 Wolfson Microelectronics plc
- *  Author: Mark Brown <broonie@opensource.wolfsonmicro.com>
- *
- *  Based on an earlier driver for the same hardware by Liam Girdwood.
- *
- *  This program is free software; you can redistribute  it and/or modify it
- *  under  the terms of  the GNU General  Public License as published by the
- *  Free Software Foundation;  either version 2 of the  License, or (at your
- *  option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+//  wm1133-ev1.c - Audio for WM1133-EV1 on i.MX31ADS
+//
+//  Copyright (c) 2010 Wolfson Microelectronics plc
+//  Author: Mark Brown <broonie@opensource.wolfsonmicro.com>
+//
+//  Based on an earlier driver for the same hardware by Liam Girdwood.
 
 #include <linux/platform_device.h>
 #include <linux/clk.h>
diff --git a/sound/soc/generic/audio-graph-card.c b/sound/soc/generic/audio-graph-card.c
index 69bc4848d787..ec7e673ba475 100644
--- a/sound/soc/generic/audio-graph-card.c
+++ b/sound/soc/generic/audio-graph-card.c
@@ -22,37 +22,6 @@
 
 #define DPCM_SELECTABLE 1
 
-struct graph_priv {
-	struct snd_soc_card snd_card;
-	struct graph_dai_props {
-		struct asoc_simple_dai *cpu_dai;
-		struct asoc_simple_dai *codec_dai;
-		struct snd_soc_dai_link_component codecs; /* single codec */
-		struct snd_soc_dai_link_component platforms;
-		struct asoc_simple_card_data adata;
-		struct snd_soc_codec_conf *codec_conf;
-		unsigned int mclk_fs;
-	} *dai_props;
-	struct asoc_simple_jack hp_jack;
-	struct asoc_simple_jack mic_jack;
-	struct snd_soc_dai_link *dai_link;
-	struct asoc_simple_dai *dais;
-	struct snd_soc_codec_conf *codec_conf;
-	struct gpio_desc *pa_gpio;
-};
-
-struct link_info {
-	int dais; /* number of dai  */
-	int link; /* number of link */
-	int conf; /* number of codec_conf */
-	int cpu;  /* turn for CPU / Codec */
-};
-
-#define graph_priv_to_card(priv) (&(priv)->snd_card)
-#define graph_priv_to_props(priv, i) ((priv)->dai_props + (i))
-#define graph_priv_to_dev(priv) (graph_priv_to_card(priv)->dev)
-#define graph_priv_to_link(priv, i) (graph_priv_to_card(priv)->dai_link + (i))
-
 #define PREFIX	"audio-graph-card,"
 
 static int graph_outdrv_event(struct snd_soc_dapm_widget *w,
@@ -60,7 +29,7 @@ static int graph_outdrv_event(struct snd_soc_dapm_widget *w,
 			      int event)
 {
 	struct snd_soc_dapm_context *dapm = w->dapm;
-	struct graph_priv *priv = snd_soc_card_get_drvdata(dapm->card);
+	struct asoc_simple_priv *priv = snd_soc_card_get_drvdata(dapm->card);
 
 	switch (event) {
 	case SND_SOC_DAPM_POST_PMU:
@@ -82,127 +51,156 @@ static const struct snd_soc_dapm_widget graph_dapm_widgets[] = {
 			       SND_SOC_DAPM_POST_PMU | SND_SOC_DAPM_PRE_PMD),
 };
 
-static int graph_startup(struct snd_pcm_substream *substream)
+static const struct snd_soc_ops graph_ops = {
+	.startup	= asoc_simple_startup,
+	.shutdown	= asoc_simple_shutdown,
+	.hw_params	= asoc_simple_hw_params,
+};
+
+static int graph_get_dai_id(struct device_node *ep)
 {
-	struct snd_soc_pcm_runtime *rtd = substream->private_data;
-	struct graph_priv *priv = snd_soc_card_get_drvdata(rtd->card);
-	struct graph_dai_props *dai_props = graph_priv_to_props(priv, rtd->num);
+	struct device_node *node;
+	struct device_node *endpoint;
+	struct of_endpoint info;
+	int i, id;
 	int ret;
 
-	ret = asoc_simple_card_clk_enable(dai_props->cpu_dai);
-	if (ret)
+	/* use driver specified DAI ID if exist */
+	ret = snd_soc_get_dai_id(ep);
+	if (ret != -ENOTSUPP)
 		return ret;
 
-	ret = asoc_simple_card_clk_enable(dai_props->codec_dai);
-	if (ret)
-		asoc_simple_card_clk_disable(dai_props->cpu_dai);
+	/* use endpoint/port reg if exist */
+	ret = of_graph_parse_endpoint(ep, &info);
+	if (ret == 0) {
+		/*
+		 * Because it will count port/endpoint if it doesn't have "reg".
+		 * But, we can't judge whether it has "no reg", or "reg = <0>"
+		 * only of_graph_parse_endpoint().
+		 * We need to check "reg" property
+		 */
+		if (of_get_property(ep,   "reg", NULL))
+			return info.id;
 
-	return ret;
-}
+		node = of_get_parent(ep);
+		of_node_put(node);
+		if (of_get_property(node, "reg", NULL))
+			return info.port;
+	}
+	node = of_graph_get_port_parent(ep);
 
-static void graph_shutdown(struct snd_pcm_substream *substream)
-{
-	struct snd_soc_pcm_runtime *rtd = substream->private_data;
-	struct graph_priv *priv = snd_soc_card_get_drvdata(rtd->card);
-	struct graph_dai_props *dai_props = graph_priv_to_props(priv, rtd->num);
+	/*
+	 * Non HDMI sound case, counting port/endpoint on its DT
+	 * is enough. Let's count it.
+	 */
+	i = 0;
+	id = -1;
+	for_each_endpoint_of_node(node, endpoint) {
+		if (endpoint == ep)
+			id = i;
+		i++;
+	}
 
-	asoc_simple_card_clk_disable(dai_props->cpu_dai);
+	of_node_put(node);
+
+	if (id < 0)
+		return -ENODEV;
 
-	asoc_simple_card_clk_disable(dai_props->codec_dai);
+	return id;
 }
 
-static int graph_hw_params(struct snd_pcm_substream *substream,
-			   struct snd_pcm_hw_params *params)
+static int asoc_simple_parse_dai(struct device_node *ep,
+				 struct snd_soc_dai_link_component *dlc,
+				 struct device_node **dai_of_node,
+				 const char **dai_name,
+				 int *is_single_link)
 {
-	struct snd_soc_pcm_runtime *rtd = substream->private_data;
-	struct snd_soc_dai *codec_dai = rtd->codec_dai;
-	struct snd_soc_dai *cpu_dai = rtd->cpu_dai;
-	struct graph_priv *priv = snd_soc_card_get_drvdata(rtd->card);
-	struct graph_dai_props *dai_props = graph_priv_to_props(priv, rtd->num);
-	unsigned int mclk, mclk_fs = 0;
-	int ret = 0;
-
-	if (dai_props->mclk_fs)
-		mclk_fs = dai_props->mclk_fs;
-
-	if (mclk_fs) {
-		mclk = params_rate(params) * mclk_fs;
-		ret = snd_soc_dai_set_sysclk(codec_dai, 0, mclk,
-					     SND_SOC_CLOCK_IN);
-		if (ret && ret != -ENOTSUPP)
-			goto err;
-
-		ret = snd_soc_dai_set_sysclk(cpu_dai, 0, mclk,
-					     SND_SOC_CLOCK_OUT);
-		if (ret && ret != -ENOTSUPP)
-			goto err;
+	struct device_node *node;
+	struct of_phandle_args args;
+	int ret;
+
+	/*
+	 * Use snd_soc_dai_link_component instead of legacy style.
+	 * It is only for codec, but cpu will be supported in the future.
+	 * see
+	 *	soc-core.c :: snd_soc_init_multicodec()
+	 */
+	if (dlc) {
+		dai_name	= &dlc->dai_name;
+		dai_of_node	= &dlc->of_node;
 	}
-	return 0;
-err:
-	return ret;
-}
 
-static const struct snd_soc_ops graph_ops = {
-	.startup	= graph_startup,
-	.shutdown	= graph_shutdown,
-	.hw_params	= graph_hw_params,
-};
+	if (!ep)
+		return 0;
+	if (!dai_name)
+		return 0;
 
-static int graph_dai_init(struct snd_soc_pcm_runtime *rtd)
-{
-	struct graph_priv *priv = snd_soc_card_get_drvdata(rtd->card);
-	struct graph_dai_props *dai_props = graph_priv_to_props(priv, rtd->num);
-	int ret = 0;
+	node = of_graph_get_port_parent(ep);
 
-	ret = asoc_simple_card_init_dai(rtd->codec_dai,
-					dai_props->codec_dai);
-	if (ret < 0)
-		return ret;
+	/* Get dai->name */
+	args.np		= node;
+	args.args[0]	= graph_get_dai_id(ep);
+	args.args_count	= (of_graph_get_endpoint_count(node) > 1);
 
-	ret = asoc_simple_card_init_dai(rtd->cpu_dai,
-					dai_props->cpu_dai);
+	ret = snd_soc_get_dai_name(&args, dai_name);
 	if (ret < 0)
 		return ret;
 
-	return 0;
-}
-
-static int graph_be_hw_params_fixup(struct snd_soc_pcm_runtime *rtd,
-				    struct snd_pcm_hw_params *params)
-{
-	struct graph_priv *priv = snd_soc_card_get_drvdata(rtd->card);
-	struct graph_dai_props *dai_props = graph_priv_to_props(priv, rtd->num);
+	*dai_of_node = node;
 
-	asoc_simple_card_convert_fixup(&dai_props->adata, params);
+	if (is_single_link)
+		*is_single_link = of_graph_get_endpoint_count(node) == 1;
 
 	return 0;
 }
 
-static void graph_get_conversion(struct device *dev,
-				 struct device_node *ep,
-				 struct asoc_simple_card_data *adata)
+static void graph_parse_convert(struct device *dev,
+				struct device_node *ep,
+				struct asoc_simple_data *adata)
 {
 	struct device_node *top = dev->of_node;
 	struct device_node *port = of_get_parent(ep);
 	struct device_node *ports = of_get_parent(port);
 	struct device_node *node = of_graph_get_port_parent(ep);
 
-	asoc_simple_card_parse_convert(dev, top,   NULL,   adata);
-	asoc_simple_card_parse_convert(dev, node,  PREFIX, adata);
-	asoc_simple_card_parse_convert(dev, ports, NULL,   adata);
-	asoc_simple_card_parse_convert(dev, port,  NULL,   adata);
-	asoc_simple_card_parse_convert(dev, ep,    NULL,   adata);
+	asoc_simple_parse_convert(dev, top,   NULL,   adata);
+	asoc_simple_parse_convert(dev, node,  PREFIX, adata);
+	asoc_simple_parse_convert(dev, ports, NULL,   adata);
+	asoc_simple_parse_convert(dev, port,  NULL,   adata);
+	asoc_simple_parse_convert(dev, ep,    NULL,   adata);
+
+	of_node_put(port);
+	of_node_put(ports);
+	of_node_put(node);
 }
 
-static int graph_dai_link_of_dpcm(struct graph_priv *priv,
+static void graph_parse_mclk_fs(struct device_node *top,
+				struct device_node *ep,
+				struct simple_dai_props *props)
+{
+	struct device_node *port	= of_get_parent(ep);
+	struct device_node *ports	= of_get_parent(port);
+	struct device_node *node	= of_graph_get_port_parent(ep);
+
+	of_property_read_u32(top,	"mclk-fs", &props->mclk_fs);
+	of_property_read_u32(ports,	"mclk-fs", &props->mclk_fs);
+	of_property_read_u32(port,	"mclk-fs", &props->mclk_fs);
+	of_property_read_u32(ep,	"mclk-fs", &props->mclk_fs);
+
+	of_node_put(port);
+	of_node_put(ports);
+	of_node_put(node);
+}
+
+static int graph_dai_link_of_dpcm(struct asoc_simple_priv *priv,
 				  struct device_node *cpu_ep,
 				  struct device_node *codec_ep,
 				  struct link_info *li,
 				  int dup_codec)
 {
-	struct device *dev = graph_priv_to_dev(priv);
-	struct snd_soc_dai_link *dai_link = graph_priv_to_link(priv, li->link);
-	struct graph_dai_props *dai_props = graph_priv_to_props(priv, li->link);
+	struct device *dev = simple_priv_to_dev(priv);
+	struct snd_soc_dai_link *dai_link = simple_priv_to_link(priv, li->link);
+	struct simple_dai_props *dai_props = simple_priv_to_props(priv, li->link);
 	struct device_node *top = dev->of_node;
 	struct device_node *ep = li->cpu ? cpu_ep : codec_ep;
 	struct device_node *port;
@@ -224,18 +222,12 @@ static int graph_dai_link_of_dpcm(struct graph_priv *priv,
 
 	dev_dbg(dev, "link_of DPCM (%pOF)\n", ep);
 
-	of_property_read_u32(top,   "mclk-fs", &dai_props->mclk_fs);
-	of_property_read_u32(ports, "mclk-fs", &dai_props->mclk_fs);
-	of_property_read_u32(port,  "mclk-fs", &dai_props->mclk_fs);
-	of_property_read_u32(ep,    "mclk-fs", &dai_props->mclk_fs);
-
-	graph_get_conversion(dev, ep, &dai_props->adata);
-
 	of_node_put(ports);
 	of_node_put(port);
 	of_node_put(node);
 
 	if (li->cpu) {
+		int is_single_links = 0;
 
 		/* BE is dummy */
 		codecs->of_node		= NULL;
@@ -249,23 +241,22 @@ static int graph_dai_link_of_dpcm(struct graph_priv *priv,
 		dai =
 		dai_props->cpu_dai	= &priv->dais[li->dais++];
 
-		ret = asoc_simple_card_parse_graph_cpu(ep, dai_link);
+		ret = asoc_simple_parse_cpu(ep, dai_link, &is_single_links);
 		if (ret)
 			return ret;
 
-		ret = asoc_simple_card_parse_clk_cpu(dev, ep, dai_link, dai);
+		ret = asoc_simple_parse_clk_cpu(dev, ep, dai_link, dai);
 		if (ret < 0)
 			return ret;
 
-		ret = asoc_simple_card_set_dailink_name(dev, dai_link,
-							"fe.%s",
-							dai_link->cpu_dai_name);
+		ret = asoc_simple_set_dailink_name(dev, dai_link,
+						   "fe.%s",
+						   dai_link->cpu_dai_name);
 		if (ret < 0)
 			return ret;
 
 		/* card->num_links includes Codec */
-		asoc_simple_card_canonicalize_cpu(dai_link,
-			of_graph_get_endpoint_count(dai_link->cpu_of_node) == 1);
+		asoc_simple_canonicalize_cpu(dai_link, is_single_links);
 	} else {
 		struct snd_soc_codec_conf *cconf;
 
@@ -276,7 +267,7 @@ static int graph_dai_link_of_dpcm(struct graph_priv *priv,
 
 		/* BE settings */
 		dai_link->no_pcm		= 1;
-		dai_link->be_hw_params_fixup	= graph_be_hw_params_fixup;
+		dai_link->be_hw_params_fixup	= asoc_simple_be_hw_params_fixup;
 
 		dai =
 		dai_props->codec_dai	= &priv->dais[li->dais++];
@@ -284,17 +275,17 @@ static int graph_dai_link_of_dpcm(struct graph_priv *priv,
 		cconf =
 		dai_props->codec_conf	= &priv->codec_conf[li->conf++];
 
-		ret = asoc_simple_card_parse_graph_codec(ep, dai_link);
+		ret = asoc_simple_parse_codec(ep, dai_link);
 		if (ret < 0)
 			return ret;
 
-		ret = asoc_simple_card_parse_clk_codec(dev, ep, dai_link, dai);
+		ret = asoc_simple_parse_clk_codec(dev, ep, dai_link, dai);
 		if (ret < 0)
 			return ret;
 
-		ret = asoc_simple_card_set_dailink_name(dev, dai_link,
-							"be.%s",
-							codecs->dai_name);
+		ret = asoc_simple_set_dailink_name(dev, dai_link,
+						   "be.%s",
+						   codecs->dai_name);
 		if (ret < 0)
 			return ret;
 
@@ -309,51 +300,45 @@ static int graph_dai_link_of_dpcm(struct graph_priv *priv,
 					     "prefix");
 	}
 
-	asoc_simple_card_canonicalize_platform(dai_link);
+	graph_parse_convert(dev, ep, &dai_props->adata);
+	graph_parse_mclk_fs(top, ep, dai_props);
 
-	ret = asoc_simple_card_of_parse_tdm(ep, dai);
+	asoc_simple_canonicalize_platform(dai_link);
+
+	ret = asoc_simple_parse_tdm(ep, dai);
 	if (ret)
 		return ret;
 
-	ret = asoc_simple_card_parse_daifmt(dev, cpu_ep, codec_ep,
-					    NULL, &dai_link->dai_fmt);
+	ret = asoc_simple_parse_daifmt(dev, cpu_ep, codec_ep,
+				       NULL, &dai_link->dai_fmt);
 	if (ret < 0)
 		return ret;
 
 	dai_link->dpcm_playback		= 1;
 	dai_link->dpcm_capture		= 1;
 	dai_link->ops			= &graph_ops;
-	dai_link->init			= graph_dai_init;
+	dai_link->init			= asoc_simple_dai_init;
 
 	return 0;
 }
 
-static int graph_dai_link_of(struct graph_priv *priv,
+static int graph_dai_link_of(struct asoc_simple_priv *priv,
 			     struct device_node *cpu_ep,
 			     struct device_node *codec_ep,
 			     struct link_info *li)
 {
-	struct device *dev = graph_priv_to_dev(priv);
-	struct snd_soc_dai_link *dai_link = graph_priv_to_link(priv, li->link);
-	struct graph_dai_props *dai_props = graph_priv_to_props(priv, li->link);
+	struct device *dev = simple_priv_to_dev(priv);
+	struct snd_soc_dai_link *dai_link = simple_priv_to_link(priv, li->link);
+	struct simple_dai_props *dai_props = simple_priv_to_props(priv, li->link);
 	struct device_node *top = dev->of_node;
-	struct device_node *cpu_port;
-	struct device_node *cpu_ports;
-	struct device_node *codec_port;
-	struct device_node *codec_ports;
 	struct asoc_simple_dai *cpu_dai;
 	struct asoc_simple_dai *codec_dai;
-	int ret;
+	int ret, single_cpu;
 
 	/* Do it only CPU turn */
 	if (!li->cpu)
 		return 0;
 
-	cpu_port	= of_get_parent(cpu_ep);
-	cpu_ports	= of_get_parent(cpu_port);
-	codec_port	= of_get_parent(codec_ep);
-	codec_ports	= of_get_parent(codec_port);
-
 	dev_dbg(dev, "link_of (%pOF)\n", cpu_ep);
 
 	li->link++;
@@ -364,84 +349,74 @@ static int graph_dai_link_of(struct graph_priv *priv,
 	dai_props->codec_dai	= &priv->dais[li->dais++];
 
 	/* Factor to mclk, used in hw_params() */
-	of_property_read_u32(top,         "mclk-fs", &dai_props->mclk_fs);
-	of_property_read_u32(cpu_ports,   "mclk-fs", &dai_props->mclk_fs);
-	of_property_read_u32(codec_ports, "mclk-fs", &dai_props->mclk_fs);
-	of_property_read_u32(cpu_port,    "mclk-fs", &dai_props->mclk_fs);
-	of_property_read_u32(codec_port,  "mclk-fs", &dai_props->mclk_fs);
-	of_property_read_u32(cpu_ep,      "mclk-fs", &dai_props->mclk_fs);
-	of_property_read_u32(codec_ep,    "mclk-fs", &dai_props->mclk_fs);
-	of_node_put(cpu_port);
-	of_node_put(cpu_ports);
-	of_node_put(codec_port);
-	of_node_put(codec_ports);
-
-	ret = asoc_simple_card_parse_daifmt(dev, cpu_ep, codec_ep,
-					    NULL, &dai_link->dai_fmt);
+	graph_parse_mclk_fs(top, cpu_ep,   dai_props);
+	graph_parse_mclk_fs(top, codec_ep, dai_props);
+
+	ret = asoc_simple_parse_daifmt(dev, cpu_ep, codec_ep,
+				       NULL, &dai_link->dai_fmt);
 	if (ret < 0)
 		return ret;
 
-	ret = asoc_simple_card_parse_graph_cpu(cpu_ep, dai_link);
+	ret = asoc_simple_parse_cpu(cpu_ep, dai_link, &single_cpu);
 	if (ret < 0)
 		return ret;
 
-	ret = asoc_simple_card_parse_graph_codec(codec_ep, dai_link);
+	ret = asoc_simple_parse_codec(codec_ep, dai_link);
 	if (ret < 0)
 		return ret;
 
-	ret = asoc_simple_card_of_parse_tdm(cpu_ep, cpu_dai);
+	ret = asoc_simple_parse_tdm(cpu_ep, cpu_dai);
 	if (ret < 0)
 		return ret;
 
-	ret = asoc_simple_card_of_parse_tdm(codec_ep, codec_dai);
+	ret = asoc_simple_parse_tdm(codec_ep, codec_dai);
 	if (ret < 0)
 		return ret;
 
-	ret = asoc_simple_card_parse_clk_cpu(dev, cpu_ep, dai_link, cpu_dai);
+	ret = asoc_simple_parse_clk_cpu(dev, cpu_ep, dai_link, cpu_dai);
 	if (ret < 0)
 		return ret;
 
-	ret = asoc_simple_card_parse_clk_codec(dev, codec_ep, dai_link, codec_dai);
+	ret = asoc_simple_parse_clk_codec(dev, codec_ep, dai_link, codec_dai);
 	if (ret < 0)
 		return ret;
 
-	ret = asoc_simple_card_set_dailink_name(dev, dai_link,
-						"%s-%s",
-						dai_link->cpu_dai_name,
-						dai_link->codecs->dai_name);
+	ret = asoc_simple_set_dailink_name(dev, dai_link,
+					   "%s-%s",
+					   dai_link->cpu_dai_name,
+					   dai_link->codecs->dai_name);
 	if (ret < 0)
 		return ret;
 
 	dai_link->ops = &graph_ops;
-	dai_link->init = graph_dai_init;
+	dai_link->init = asoc_simple_dai_init;
 
-	asoc_simple_card_canonicalize_platform(dai_link);
-	asoc_simple_card_canonicalize_cpu(dai_link,
-		of_graph_get_endpoint_count(dai_link->cpu_of_node) == 1);
+	asoc_simple_canonicalize_cpu(dai_link, single_cpu);
+	asoc_simple_canonicalize_platform(dai_link);
 
 	return 0;
 }
 
-static int graph_for_each_link(struct graph_priv *priv,
+static int graph_for_each_link(struct asoc_simple_priv *priv,
 			struct link_info *li,
-			int (*func_noml)(struct graph_priv *priv,
+			int (*func_noml)(struct asoc_simple_priv *priv,
 					 struct device_node *cpu_ep,
 					 struct device_node *codec_ep,
 					 struct link_info *li),
-			int (*func_dpcm)(struct graph_priv *priv,
+			int (*func_dpcm)(struct asoc_simple_priv *priv,
 					 struct device_node *cpu_ep,
 					 struct device_node *codec_ep,
 					 struct link_info *li, int dup_codec))
 {
 	struct of_phandle_iterator it;
-	struct device *dev = graph_priv_to_dev(priv);
+	struct device *dev = simple_priv_to_dev(priv);
 	struct device_node *node = dev->of_node;
 	struct device_node *cpu_port;
 	struct device_node *cpu_ep;
 	struct device_node *codec_ep;
 	struct device_node *codec_port;
 	struct device_node *codec_port_old = NULL;
-	struct asoc_simple_card_data adata;
+	struct asoc_simple_data adata;
 	uintptr_t dpcm_selectable = (uintptr_t)of_device_get_match_data(dev);
 	int rc, ret;
 
@@ -465,8 +440,8 @@ static int graph_for_each_link(struct graph_priv *priv,
 
 			/* get convert-xxx property */
 			memset(&adata, 0, sizeof(adata));
-			graph_get_conversion(dev, codec_ep, &adata);
-			graph_get_conversion(dev, cpu_ep,   &adata);
+			graph_parse_convert(dev, codec_ep, &adata);
+			graph_parse_convert(dev, cpu_ep,   &adata);
 
 			/*
 			 * It is DPCM
@@ -492,17 +467,17 @@ static int graph_for_each_link(struct graph_priv *priv,
 	return 0;
 }
 
-static int graph_parse_of(struct graph_priv *priv)
+static int graph_parse_of(struct asoc_simple_priv *priv)
 {
-	struct snd_soc_card *card = graph_priv_to_card(priv);
+	struct snd_soc_card *card = simple_priv_to_card(priv);
 	struct link_info li;
 	int ret;
 
-	ret = asoc_simple_card_of_parse_widgets(card, NULL);
+	ret = asoc_simple_parse_widgets(card, NULL);
 	if (ret < 0)
 		return ret;
 
-	ret = asoc_simple_card_of_parse_routing(card, NULL);
+	ret = asoc_simple_parse_routing(card, NULL);
 	if (ret < 0)
 		return ret;
 
@@ -527,15 +502,15 @@ static int graph_parse_of(struct graph_priv *priv)
 			return ret;
 	}
 
-	return asoc_simple_card_parse_card_name(card, NULL);
+	return asoc_simple_parse_card_name(card, NULL);
 }
 
-static int graph_count_noml(struct graph_priv *priv,
+static int graph_count_noml(struct asoc_simple_priv *priv,
 			    struct device_node *cpu_ep,
 			    struct device_node *codec_ep,
 			    struct link_info *li)
 {
-	struct device *dev = graph_priv_to_dev(priv);
+	struct device *dev = simple_priv_to_dev(priv);
 
 	li->link += 1; /* 1xCPU-Codec */
 	li->dais += 2; /* 1xCPU + 1xCodec */
@@ -545,13 +520,13 @@ static int graph_count_noml(struct graph_priv *priv,
 	return 0;
 }
 
-static int graph_count_dpcm(struct graph_priv *priv,
+static int graph_count_dpcm(struct asoc_simple_priv *priv,
 			    struct device_node *cpu_ep,
 			    struct device_node *codec_ep,
 			    struct link_info *li,
 			    int dup_codec)
 {
-	struct device *dev = graph_priv_to_dev(priv);
+	struct device *dev = simple_priv_to_dev(priv);
 
 	li->link++; /* 1xCPU-dummy */
 	li->dais++; /* 1xCPU */
@@ -567,10 +542,10 @@ static int graph_count_dpcm(struct graph_priv *priv,
 	return 0;
 }
 
-static void graph_get_dais_count(struct graph_priv *priv,
+static void graph_get_dais_count(struct asoc_simple_priv *priv,
 				 struct link_info *li)
 {
-	struct device *dev = graph_priv_to_dev(priv);
+	struct device *dev = simple_priv_to_dev(priv);
 
 	/*
 	 * link_num :	number of links.
@@ -627,14 +602,14 @@ static void graph_get_dais_count(struct graph_priv *priv,
 
 static int graph_card_probe(struct snd_soc_card *card)
 {
-	struct graph_priv *priv = snd_soc_card_get_drvdata(card);
+	struct asoc_simple_priv *priv = snd_soc_card_get_drvdata(card);
 	int ret;
 
-	ret = asoc_simple_card_init_hp(card, &priv->hp_jack, NULL);
+	ret = asoc_simple_init_hp(card, &priv->hp_jack, NULL);
 	if (ret < 0)
 		return ret;
 
-	ret = asoc_simple_card_init_mic(card, &priv->mic_jack, NULL);
+	ret = asoc_simple_init_mic(card, &priv->mic_jack, NULL);
 	if (ret < 0)
 		return ret;
 
@@ -643,22 +618,18 @@ static int graph_card_probe(struct snd_soc_card *card)
 
 static int graph_probe(struct platform_device *pdev)
 {
-	struct graph_priv *priv;
-	struct snd_soc_dai_link *dai_link;
-	struct graph_dai_props *dai_props;
-	struct asoc_simple_dai *dais;
+	struct asoc_simple_priv *priv;
 	struct device *dev = &pdev->dev;
 	struct snd_soc_card *card;
-	struct snd_soc_codec_conf *cconf;
 	struct link_info li;
-	int ret, i;
+	int ret;
 
 	/* Allocate the private data and the DAI link array */
 	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
 	if (!priv)
 		return -ENOMEM;
 
-	card = graph_priv_to_card(priv);
+	card = simple_priv_to_card(priv);
 	card->owner		= THIS_MODULE;
 	card->dev		= dev;
 	card->dapm_widgets	= graph_dapm_widgets;
@@ -670,25 +641,9 @@ static int graph_probe(struct platform_device *pdev)
 	if (!li.link || !li.dais)
 		return -EINVAL;
 
-	dai_props = devm_kcalloc(dev, li.link, sizeof(*dai_props), GFP_KERNEL);
-	dai_link  = devm_kcalloc(dev, li.link, sizeof(*dai_link),  GFP_KERNEL);
-	dais      = devm_kcalloc(dev, li.dais, sizeof(*dais),      GFP_KERNEL);
-	cconf     = devm_kcalloc(dev, li.conf, sizeof(*cconf),     GFP_KERNEL);
-	if (!dai_props || !dai_link || !dais)
-		return -ENOMEM;
-
-	/*
-	 * Use snd_soc_dai_link_component instead of legacy style
-	 * It is codec only. but cpu/platform will be supported in the future.
-	 * see
-	 *	soc-core.c :: snd_soc_init_multicodec()
-	 */
-	for (i = 0; i < li.link; i++) {
-		dai_link[i].codecs	= &dai_props[i].codecs;
-		dai_link[i].num_codecs	= 1;
-		dai_link[i].platforms	= &dai_props[i].platforms;
-		dai_link[i].num_platforms = 1;
-	}
+	ret = asoc_simple_init_priv(priv, &li);
+	if (ret < 0)
+		return ret;
 
 	priv->pa_gpio = devm_gpiod_get_optional(dev, "pa", GPIOD_OUT_LOW);
 	if (IS_ERR(priv->pa_gpio)) {
@@ -697,16 +652,6 @@ static int graph_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	priv->dai_props		= dai_props;
-	priv->dai_link		= dai_link;
-	priv->dais		= dais;
-	priv->codec_conf	= cconf;
-
-	card->dai_link		= dai_link;
-	card->num_links		= li.link;
-	card->codec_conf	= cconf;
-	card->num_configs	= li.conf;
-
 	ret = graph_parse_of(priv);
 	if (ret < 0) {
 		if (ret != -EPROBE_DEFER)
@@ -716,13 +661,15 @@ static int graph_probe(struct platform_device *pdev)
 
 	snd_soc_card_set_drvdata(card, priv);
 
+	asoc_simple_debug_info(priv);
+
 	ret = devm_snd_soc_register_card(dev, card);
 	if (ret < 0)
 		goto err;
 
 	return 0;
 err:
-	asoc_simple_card_clean_reference(card);
+	asoc_simple_clean_reference(card);
 
 	return ret;
 }
@@ -731,7 +678,7 @@ static int graph_remove(struct platform_device *pdev)
 {
 	struct snd_soc_card *card = platform_get_drvdata(pdev);
 
-	return asoc_simple_card_clean_reference(card);
+	return asoc_simple_clean_reference(card);
 }
 
 static const struct of_device_id graph_of_match[] = {
diff --git a/sound/soc/generic/simple-card-utils.c b/sound/soc/generic/simple-card-utils.c
index 5c1424f03620..f4c6375d11c7 100644
--- a/sound/soc/generic/simple-card-utils.c
+++ b/sound/soc/generic/simple-card-utils.c
@@ -14,8 +14,8 @@
 #include <sound/jack.h>
 #include <sound/simple_card_utils.h>
 
-void asoc_simple_card_convert_fixup(struct asoc_simple_card_data *data,
-				    struct snd_pcm_hw_params *params)
+void asoc_simple_convert_fixup(struct asoc_simple_data *data,
+			       struct snd_pcm_hw_params *params)
 {
 	struct snd_interval *rate = hw_param_interval(params,
 						SNDRV_PCM_HW_PARAM_RATE);
@@ -30,12 +30,12 @@ void asoc_simple_card_convert_fixup(struct asoc_simple_card_data *data,
 		channels->min =
 		channels->max = data->convert_channels;
 }
-EXPORT_SYMBOL_GPL(asoc_simple_card_convert_fixup);
+EXPORT_SYMBOL_GPL(asoc_simple_convert_fixup);
 
-void asoc_simple_card_parse_convert(struct device *dev,
-				    struct device_node *np,
-				    char *prefix,
-				    struct asoc_simple_card_data *data)
+void asoc_simple_parse_convert(struct device *dev,
+			       struct device_node *np,
+			       char *prefix,
+			       struct asoc_simple_data *data)
 {
 	char prop[128];
 
@@ -49,17 +49,14 @@ void asoc_simple_card_parse_convert(struct device *dev,
 	/* channels transfer */
 	snprintf(prop, sizeof(prop), "%s%s", prefix, "convert-channels");
 	of_property_read_u32(np, prop, &data->convert_channels);
-
-	dev_dbg(dev, "convert_rate     %d\n", data->convert_rate);
-	dev_dbg(dev, "convert_channels %d\n", data->convert_channels);
 }
-EXPORT_SYMBOL_GPL(asoc_simple_card_parse_convert);
+EXPORT_SYMBOL_GPL(asoc_simple_parse_convert);
 
-int asoc_simple_card_parse_daifmt(struct device *dev,
-				  struct device_node *node,
-				  struct device_node *codec,
-				  char *prefix,
-				  unsigned int *retfmt)
+int asoc_simple_parse_daifmt(struct device *dev,
+			     struct device_node *node,
+			     struct device_node *codec,
+			     char *prefix,
+			     unsigned int *retfmt)
 {
 	struct device_node *bitclkmaster = NULL;
 	struct device_node *framemaster = NULL;
@@ -93,15 +90,13 @@ int asoc_simple_card_parse_daifmt(struct device *dev,
 
 	*retfmt = daifmt;
 
-	dev_dbg(dev, "format : %04x\n", daifmt);
-
 	return 0;
 }
-EXPORT_SYMBOL_GPL(asoc_simple_card_parse_daifmt);
+EXPORT_SYMBOL_GPL(asoc_simple_parse_daifmt);
 
-int asoc_simple_card_set_dailink_name(struct device *dev,
-				      struct snd_soc_dai_link *dai_link,
-				      const char *fmt, ...)
+int asoc_simple_set_dailink_name(struct device *dev,
+				 struct snd_soc_dai_link *dai_link,
+				 const char *fmt, ...)
 {
 	va_list ap;
 	char *name = NULL;
@@ -116,16 +111,14 @@ int asoc_simple_card_set_dailink_name(struct device *dev,
 
 		dai_link->name		= name;
 		dai_link->stream_name	= name;
-
-		dev_dbg(dev, "name : %s\n", name);
 	}
 
 	return ret;
 }
-EXPORT_SYMBOL_GPL(asoc_simple_card_set_dailink_name);
+EXPORT_SYMBOL_GPL(asoc_simple_set_dailink_name);
 
-int asoc_simple_card_parse_card_name(struct snd_soc_card *card,
-				     char *prefix)
+int asoc_simple_parse_card_name(struct snd_soc_card *card,
+				char *prefix)
 {
 	int ret;
 
@@ -146,34 +139,30 @@ int asoc_simple_card_parse_card_name(struct snd_soc_card *card,
 	if (!card->name && card->dai_link)
 		card->name = card->dai_link->name;
 
-	dev_dbg(card->dev, "Card Name: %s\n", card->name ? card->name : "");
-
 	return 0;
 }
-EXPORT_SYMBOL_GPL(asoc_simple_card_parse_card_name);
+EXPORT_SYMBOL_GPL(asoc_simple_parse_card_name);
 
-int asoc_simple_card_clk_enable(struct asoc_simple_dai *dai)
+static int asoc_simple_clk_enable(struct asoc_simple_dai *dai)
 {
 	if (dai)
 		return clk_prepare_enable(dai->clk);
 
 	return 0;
 }
-EXPORT_SYMBOL_GPL(asoc_simple_card_clk_enable);
 
-void asoc_simple_card_clk_disable(struct asoc_simple_dai *dai)
+static void asoc_simple_clk_disable(struct asoc_simple_dai *dai)
 {
 	if (dai)
 		clk_disable_unprepare(dai->clk);
 }
-EXPORT_SYMBOL_GPL(asoc_simple_card_clk_disable);
-
-int asoc_simple_card_parse_clk(struct device *dev,
-			       struct device_node *node,
-			       struct device_node *dai_of_node,
-			       struct asoc_simple_dai *simple_dai,
-			       const char *dai_name,
-			       struct snd_soc_dai_link_component *dlc)
+
+int asoc_simple_parse_clk(struct device *dev,
+			  struct device_node *node,
+			  struct device_node *dai_of_node,
+			  struct asoc_simple_dai *simple_dai,
+			  const char *dai_name,
+			  struct snd_soc_dai_link_component *dlc)
 {
 	struct clk *clk;
 	u32 val;
@@ -184,10 +173,8 @@ int asoc_simple_card_parse_clk(struct device *dev,
 	 * see
 	 *	soc-core.c :: snd_soc_init_multicodec()
 	 */
-	if (dlc) {
+	if (dlc)
 		dai_of_node	= dlc->of_node;
-		dai_name	= dlc->dai_name;
-	}
 
 	/*
 	 * Parse dai->sysclk come from "clocks = <&xxx>"
@@ -211,158 +198,113 @@ int asoc_simple_card_parse_clk(struct device *dev,
 	if (of_property_read_bool(node, "system-clock-direction-out"))
 		simple_dai->clk_direction = SND_SOC_CLOCK_OUT;
 
-	dev_dbg(dev, "%s : sysclk = %d, direction %d\n", dai_name,
-		simple_dai->sysclk, simple_dai->clk_direction);
-
 	return 0;
 }
-EXPORT_SYMBOL_GPL(asoc_simple_card_parse_clk);
-
-int asoc_simple_card_parse_dai(struct device_node *node,
-				    struct snd_soc_dai_link_component *dlc,
-				    struct device_node **dai_of_node,
-				    const char **dai_name,
-				    const char *list_name,
-				    const char *cells_name,
-				    int *is_single_link)
+EXPORT_SYMBOL_GPL(asoc_simple_parse_clk);
+
+int asoc_simple_startup(struct snd_pcm_substream *substream)
 {
-	struct of_phandle_args args;
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct asoc_simple_priv *priv = snd_soc_card_get_drvdata(rtd->card);
+	struct simple_dai_props *dai_props = simple_priv_to_props(priv, rtd->num);
 	int ret;
 
-	if (!node)
-		return 0;
-
-	/*
-	 * Use snd_soc_dai_link_component instead of legacy style.
-	 * It is only for codec, but cpu will be supported in the future.
-	 * see
-	 *	soc-core.c :: snd_soc_init_multicodec()
-	 */
-	if (dlc) {
-		dai_name	= &dlc->dai_name;
-		dai_of_node	= &dlc->of_node;
-	}
-
-	/*
-	 * Get node via "sound-dai = <&phandle port>"
-	 * it will be used as xxx_of_node on soc_bind_dai_link()
-	 */
-	ret = of_parse_phandle_with_args(node, list_name, cells_name, 0, &args);
+	ret = asoc_simple_clk_enable(dai_props->cpu_dai);
 	if (ret)
 		return ret;
 
-	/* Get dai->name */
-	if (dai_name) {
-		ret = snd_soc_of_get_dai_name(node, dai_name);
-		if (ret < 0)
-			return ret;
-	}
-
-	*dai_of_node = args.np;
-
-	if (is_single_link)
-		*is_single_link = !args.args_count;
+	ret = asoc_simple_clk_enable(dai_props->codec_dai);
+	if (ret)
+		asoc_simple_clk_disable(dai_props->cpu_dai);
 
-	return 0;
+	return ret;
 }
-EXPORT_SYMBOL_GPL(asoc_simple_card_parse_dai);
+EXPORT_SYMBOL_GPL(asoc_simple_startup);
 
-static int asoc_simple_card_get_dai_id(struct device_node *ep)
+void asoc_simple_shutdown(struct snd_pcm_substream *substream)
 {
-	struct device_node *node;
-	struct device_node *endpoint;
-	struct of_endpoint info;
-	int i, id;
-	int ret;
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct asoc_simple_priv *priv = snd_soc_card_get_drvdata(rtd->card);
+	struct simple_dai_props *dai_props =
+		simple_priv_to_props(priv, rtd->num);
 
-	/* use driver specified DAI ID if exist */
-	ret = snd_soc_get_dai_id(ep);
-	if (ret != -ENOTSUPP)
-		return ret;
+	asoc_simple_clk_disable(dai_props->cpu_dai);
 
-	/* use endpoint/port reg if exist */
-	ret = of_graph_parse_endpoint(ep, &info);
-	if (ret == 0) {
-		/*
-		 * Because it will count port/endpoint if it doesn't have "reg".
-		 * But, we can't judge whether it has "no reg", or "reg = <0>"
-		 * only of_graph_parse_endpoint().
-		 * We need to check "reg" property
-		 */
-		if (of_get_property(ep,   "reg", NULL))
-			return info.id;
-
-		node = of_get_parent(ep);
-		of_node_put(node);
-		if (of_get_property(node, "reg", NULL))
-			return info.port;
-	}
-	node = of_graph_get_port_parent(ep);
+	asoc_simple_clk_disable(dai_props->codec_dai);
+}
+EXPORT_SYMBOL_GPL(asoc_simple_shutdown);
 
-	/*
-	 * Non HDMI sound case, counting port/endpoint on its DT
-	 * is enough. Let's count it.
-	 */
-	i = 0;
-	id = -1;
-	for_each_endpoint_of_node(node, endpoint) {
-		if (endpoint == ep)
-			id = i;
-		i++;
-	}
+static int asoc_simple_set_clk_rate(struct asoc_simple_dai *simple_dai,
+				    unsigned long rate)
+{
+	if (!simple_dai)
+		return 0;
 
-	of_node_put(node);
+	if (!simple_dai->clk)
+		return 0;
 
-	if (id < 0)
-		return -ENODEV;
+	if (clk_get_rate(simple_dai->clk) == rate)
+		return 0;
 
-	return id;
+	return clk_set_rate(simple_dai->clk, rate);
 }
 
-int asoc_simple_card_parse_graph_dai(struct device_node *ep,
-				     struct snd_soc_dai_link_component *dlc,
-				     struct device_node **dai_of_node,
-				     const char **dai_name)
+int asoc_simple_hw_params(struct snd_pcm_substream *substream,
+			  struct snd_pcm_hw_params *params)
 {
-	struct device_node *node;
-	struct of_phandle_args args;
-	int ret;
-
-	/*
-	 * Use snd_soc_dai_link_component instead of legacy style.
-	 * It is only for codec, but cpu will be supported in the future.
-	 * see
-	 *	soc-core.c :: snd_soc_init_multicodec()
-	 */
-	if (dlc) {
-		dai_name	= &dlc->dai_name;
-		dai_of_node	= &dlc->of_node;
-	}
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct snd_soc_dai *codec_dai = rtd->codec_dai;
+	struct snd_soc_dai *cpu_dai = rtd->cpu_dai;
+	struct asoc_simple_priv *priv = snd_soc_card_get_drvdata(rtd->card);
+	struct simple_dai_props *dai_props =
+		simple_priv_to_props(priv, rtd->num);
+	unsigned int mclk, mclk_fs = 0;
+	int ret = 0;
+
+	if (dai_props->mclk_fs)
+		mclk_fs = dai_props->mclk_fs;
+
+	if (mclk_fs) {
+		mclk = params_rate(params) * mclk_fs;
+
+		ret = asoc_simple_set_clk_rate(dai_props->codec_dai, mclk);
+		if (ret < 0)
+			return ret;
 
-	if (!ep)
-		return 0;
-	if (!dai_name)
-		return 0;
+		ret = asoc_simple_set_clk_rate(dai_props->cpu_dai, mclk);
+		if (ret < 0)
+			return ret;
 
-	node = of_graph_get_port_parent(ep);
+		ret = snd_soc_dai_set_sysclk(codec_dai, 0, mclk,
+					     SND_SOC_CLOCK_IN);
+		if (ret && ret != -ENOTSUPP)
+			goto err;
 
-	/* Get dai->name */
-	args.np		= node;
-	args.args[0]	= asoc_simple_card_get_dai_id(ep);
-	args.args_count	= (of_graph_get_endpoint_count(node) > 1);
+		ret = snd_soc_dai_set_sysclk(cpu_dai, 0, mclk,
+					     SND_SOC_CLOCK_OUT);
+		if (ret && ret != -ENOTSUPP)
+			goto err;
+	}
+	return 0;
+err:
+	return ret;
+}
+EXPORT_SYMBOL_GPL(asoc_simple_hw_params);
 
-	ret = snd_soc_get_dai_name(&args, dai_name);
-	if (ret < 0)
-		return ret;
+int asoc_simple_be_hw_params_fixup(struct snd_soc_pcm_runtime *rtd,
+				   struct snd_pcm_hw_params *params)
+{
+	struct asoc_simple_priv *priv = snd_soc_card_get_drvdata(rtd->card);
+	struct simple_dai_props *dai_props = simple_priv_to_props(priv, rtd->num);
 
-	*dai_of_node = node;
+	asoc_simple_convert_fixup(&dai_props->adata, params);
 
 	return 0;
 }
-EXPORT_SYMBOL_GPL(asoc_simple_card_parse_graph_dai);
+EXPORT_SYMBOL_GPL(asoc_simple_be_hw_params_fixup);
 
-int asoc_simple_card_init_dai(struct snd_soc_dai *dai,
-			      struct asoc_simple_dai *simple_dai)
+static int asoc_simple_init_dai(struct snd_soc_dai *dai,
+				     struct asoc_simple_dai *simple_dai)
 {
 	int ret;
 
@@ -392,18 +334,37 @@ int asoc_simple_card_init_dai(struct snd_soc_dai *dai,
 
 	return 0;
 }
-EXPORT_SYMBOL_GPL(asoc_simple_card_init_dai);
 
-void asoc_simple_card_canonicalize_platform(struct snd_soc_dai_link *dai_link)
+int asoc_simple_dai_init(struct snd_soc_pcm_runtime *rtd)
+{
+	struct asoc_simple_priv *priv = snd_soc_card_get_drvdata(rtd->card);
+	struct simple_dai_props *dai_props = simple_priv_to_props(priv, rtd->num);
+	int ret;
+
+	ret = asoc_simple_init_dai(rtd->codec_dai,
+				   dai_props->codec_dai);
+	if (ret < 0)
+		return ret;
+
+	ret = asoc_simple_init_dai(rtd->cpu_dai,
+				   dai_props->cpu_dai);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(asoc_simple_dai_init);
+
+void asoc_simple_canonicalize_platform(struct snd_soc_dai_link *dai_link)
 {
 	/* Assumes platform == cpu */
 	if (!dai_link->platforms->of_node)
 		dai_link->platforms->of_node = dai_link->cpu_of_node;
 }
-EXPORT_SYMBOL_GPL(asoc_simple_card_canonicalize_platform);
+EXPORT_SYMBOL_GPL(asoc_simple_canonicalize_platform);
 
-void asoc_simple_card_canonicalize_cpu(struct snd_soc_dai_link *dai_link,
-				       int is_single_links)
+void asoc_simple_canonicalize_cpu(struct snd_soc_dai_link *dai_link,
+				  int is_single_links)
 {
 	/*
 	 * In soc_bind_dai_link() will check cpu name after
@@ -417,9 +378,9 @@ void asoc_simple_card_canonicalize_cpu(struct snd_soc_dai_link *dai_link,
 	if (is_single_links)
 		dai_link->cpu_dai_name = NULL;
 }
-EXPORT_SYMBOL_GPL(asoc_simple_card_canonicalize_cpu);
+EXPORT_SYMBOL_GPL(asoc_simple_canonicalize_cpu);
 
-int asoc_simple_card_clean_reference(struct snd_soc_card *card)
+int asoc_simple_clean_reference(struct snd_soc_card *card)
 {
 	struct snd_soc_dai_link *dai_link;
 	int i;
@@ -430,10 +391,10 @@ int asoc_simple_card_clean_reference(struct snd_soc_card *card)
 	}
 	return 0;
 }
-EXPORT_SYMBOL_GPL(asoc_simple_card_clean_reference);
+EXPORT_SYMBOL_GPL(asoc_simple_clean_reference);
 
-int asoc_simple_card_of_parse_routing(struct snd_soc_card *card,
-				      char *prefix)
+int asoc_simple_parse_routing(struct snd_soc_card *card,
+			      char *prefix)
 {
 	struct device_node *node = card->dev->of_node;
 	char prop[128];
@@ -448,10 +409,10 @@ int asoc_simple_card_of_parse_routing(struct snd_soc_card *card,
 
 	return snd_soc_of_parse_audio_routing(card, prop);
 }
-EXPORT_SYMBOL_GPL(asoc_simple_card_of_parse_routing);
+EXPORT_SYMBOL_GPL(asoc_simple_parse_routing);
 
-int asoc_simple_card_of_parse_widgets(struct snd_soc_card *card,
-				      char *prefix)
+int asoc_simple_parse_widgets(struct snd_soc_card *card,
+			      char *prefix)
 {
 	struct device_node *node = card->dev->of_node;
 	char prop[128];
@@ -467,11 +428,68 @@ int asoc_simple_card_of_parse_widgets(struct snd_soc_card *card,
 	/* no widgets is not error */
 	return 0;
 }
-EXPORT_SYMBOL_GPL(asoc_simple_card_of_parse_widgets);
+EXPORT_SYMBOL_GPL(asoc_simple_parse_widgets);
+
+int asoc_simple_parse_pin_switches(struct snd_soc_card *card,
+				   char *prefix)
+{
+	const unsigned int nb_controls_max = 16;
+	const char **strings, *control_name;
+	struct snd_kcontrol_new *controls;
+	struct device *dev = card->dev;
+	unsigned int i, nb_controls;
+	char prop[128];
+	int ret;
+
+	if (!prefix)
+		prefix = "";
+
+	snprintf(prop, sizeof(prop), "%s%s", prefix, "pin-switches");
+
+	if (!of_property_read_bool(dev->of_node, prop))
+		return 0;
+
+	strings = devm_kcalloc(dev, nb_controls_max,
+			       sizeof(*strings), GFP_KERNEL);
+	if (!strings)
+		return -ENOMEM;
+
+	ret = of_property_read_string_array(dev->of_node, prop,
+					    strings, nb_controls_max);
+	if (ret < 0)
+		return ret;
+
+	nb_controls = (unsigned int)ret;
+
+	controls = devm_kcalloc(dev, nb_controls,
+				sizeof(*controls), GFP_KERNEL);
+	if (!controls)
+		return -ENOMEM;
+
+	for (i = 0; i < nb_controls; i++) {
+		control_name = devm_kasprintf(dev, GFP_KERNEL,
+					      "%s Switch", strings[i]);
+		if (!control_name)
+			return -ENOMEM;
+
+		controls[i].iface = SNDRV_CTL_ELEM_IFACE_MIXER;
+		controls[i].name = control_name;
+		controls[i].info = snd_soc_dapm_info_pin_switch;
+		controls[i].get = snd_soc_dapm_get_pin_switch;
+		controls[i].put = snd_soc_dapm_put_pin_switch;
+		controls[i].private_value = (unsigned long)strings[i];
+	}
+
+	card->controls = controls;
+	card->num_controls = nb_controls;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(asoc_simple_parse_pin_switches);
 
-int asoc_simple_card_init_jack(struct snd_soc_card *card,
-			       struct asoc_simple_jack *sjack,
-			       int is_hp, char *prefix)
+int asoc_simple_init_jack(struct snd_soc_card *card,
+			  struct asoc_simple_jack *sjack,
+			  int is_hp, char *prefix)
 {
 	struct device *dev = card->dev;
 	enum of_gpio_flags flags;
@@ -522,7 +540,61 @@ int asoc_simple_card_init_jack(struct snd_soc_card *card,
 
 	return 0;
 }
-EXPORT_SYMBOL_GPL(asoc_simple_card_init_jack);
+EXPORT_SYMBOL_GPL(asoc_simple_init_jack);
+
+int asoc_simple_init_priv(struct asoc_simple_priv *priv,
+			  struct link_info *li)
+{
+	struct snd_soc_card *card = simple_priv_to_card(priv);
+	struct device *dev = simple_priv_to_dev(priv);
+	struct snd_soc_dai_link *dai_link;
+	struct simple_dai_props *dai_props;
+	struct asoc_simple_dai *dais;
+	struct snd_soc_codec_conf *cconf = NULL;
+	int i;
+
+	dai_props = devm_kcalloc(dev, li->link, sizeof(*dai_props), GFP_KERNEL);
+	dai_link  = devm_kcalloc(dev, li->link, sizeof(*dai_link),  GFP_KERNEL);
+	dais      = devm_kcalloc(dev, li->dais, sizeof(*dais),      GFP_KERNEL);
+	if (!dai_props || !dai_link || !dais)
+		return -ENOMEM;
+
+	if (li->conf) {
+		cconf = devm_kcalloc(dev, li->conf, sizeof(*cconf), GFP_KERNEL);
+		if (!cconf)
+			return -ENOMEM;
+	}
+
+	/*
+	 * Use snd_soc_dai_link_component instead of legacy style
+	 * It is codec only. but cpu/platform will be supported in the future.
+	 * see
+	 *	soc-core.c :: snd_soc_init_multicodec()
+	 *
+	 * "platform" might be removed
+	 * see
+	 *	simple-card-utils.c :: asoc_simple_canonicalize_platform()
+	 */
+	for (i = 0; i < li->link; i++) {
+		dai_link[i].codecs		= &dai_props[i].codecs;
+		dai_link[i].num_codecs		= 1;
+		dai_link[i].platforms		= &dai_props[i].platforms;
+		dai_link[i].num_platforms	= 1;
+	}
+
+	priv->dai_props		= dai_props;
+	priv->dai_link		= dai_link;
+	priv->dais		= dais;
+	priv->codec_conf	= cconf;
+
+	card->dai_link		= priv->dai_link;
+	card->num_links		= li->link;
+	card->codec_conf	= cconf;
+	card->num_configs	= li->conf;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(asoc_simple_init_priv);
 
 /* Module information */
 MODULE_AUTHOR("Kuninori Morimoto <kuninori.morimoto.gx@renesas.com>");
diff --git a/sound/soc/generic/simple-card.c b/sound/soc/generic/simple-card.c
index 34de32efc4c4..9b568f578bcd 100644
--- a/sound/soc/generic/simple-card.c
+++ b/sound/soc/generic/simple-card.c
@@ -18,179 +18,98 @@
 
 #define DPCM_SELECTABLE 1
 
-struct simple_priv {
-	struct snd_soc_card snd_card;
-	struct simple_dai_props {
-		struct asoc_simple_dai *cpu_dai;
-		struct asoc_simple_dai *codec_dai;
-		struct snd_soc_dai_link_component codecs; /* single codec */
-		struct snd_soc_dai_link_component platforms;
-		struct asoc_simple_card_data adata;
-		struct snd_soc_codec_conf *codec_conf;
-		unsigned int mclk_fs;
-	} *dai_props;
-	struct asoc_simple_jack hp_jack;
-	struct asoc_simple_jack mic_jack;
-	struct snd_soc_dai_link *dai_link;
-	struct asoc_simple_dai *dais;
-	struct snd_soc_codec_conf *codec_conf;
-};
-
-struct link_info {
-	int dais; /* number of dai  */
-	int link; /* number of link */
-	int conf; /* number of codec_conf */
-	int cpu;  /* turn for CPU / Codec */
-};
-
-#define simple_priv_to_card(priv) (&(priv)->snd_card)
-#define simple_priv_to_props(priv, i) ((priv)->dai_props + (i))
-#define simple_priv_to_dev(priv) (simple_priv_to_card(priv)->dev)
-#define simple_priv_to_link(priv, i) (simple_priv_to_card(priv)->dai_link + (i))
-
 #define DAI	"sound-dai"
 #define CELL	"#sound-dai-cells"
 #define PREFIX	"simple-audio-card,"
 
-static int simple_startup(struct snd_pcm_substream *substream)
-{
-	struct snd_soc_pcm_runtime *rtd = substream->private_data;
-	struct simple_priv *priv = snd_soc_card_get_drvdata(rtd->card);
-	struct simple_dai_props *dai_props =
-		simple_priv_to_props(priv, rtd->num);
-	int ret;
-
-	ret = asoc_simple_card_clk_enable(dai_props->cpu_dai);
-	if (ret)
-		return ret;
-
-	ret = asoc_simple_card_clk_enable(dai_props->codec_dai);
-	if (ret)
-		asoc_simple_card_clk_disable(dai_props->cpu_dai);
-
-	return ret;
-}
-
-static void simple_shutdown(struct snd_pcm_substream *substream)
-{
-	struct snd_soc_pcm_runtime *rtd = substream->private_data;
-	struct simple_priv *priv = snd_soc_card_get_drvdata(rtd->card);
-	struct simple_dai_props *dai_props =
-		simple_priv_to_props(priv, rtd->num);
-
-	asoc_simple_card_clk_disable(dai_props->cpu_dai);
-
-	asoc_simple_card_clk_disable(dai_props->codec_dai);
-}
+static const struct snd_soc_ops simple_ops = {
+	.startup	= asoc_simple_startup,
+	.shutdown	= asoc_simple_shutdown,
+	.hw_params	= asoc_simple_hw_params,
+};
 
-static int simple_set_clk_rate(struct asoc_simple_dai *simple_dai,
-			       unsigned long rate)
+static int asoc_simple_parse_dai(struct device_node *node,
+				 struct snd_soc_dai_link_component *dlc,
+				 struct device_node **dai_of_node,
+				 const char **dai_name,
+				 int *is_single_link)
 {
-	if (!simple_dai)
-		return 0;
-
-	if (!simple_dai->clk)
-		return 0;
+	struct of_phandle_args args;
+	int ret;
 
-	if (clk_get_rate(simple_dai->clk) == rate)
+	if (!node)
 		return 0;
 
-	return clk_set_rate(simple_dai->clk, rate);
-}
-
-static int simple_hw_params(struct snd_pcm_substream *substream,
-			    struct snd_pcm_hw_params *params)
-{
-	struct snd_soc_pcm_runtime *rtd = substream->private_data;
-	struct snd_soc_dai *codec_dai = rtd->codec_dai;
-	struct snd_soc_dai *cpu_dai = rtd->cpu_dai;
-	struct simple_priv *priv = snd_soc_card_get_drvdata(rtd->card);
-	struct simple_dai_props *dai_props =
-		simple_priv_to_props(priv, rtd->num);
-	unsigned int mclk, mclk_fs = 0;
-	int ret = 0;
-
-	if (dai_props->mclk_fs)
-		mclk_fs = dai_props->mclk_fs;
-
-	if (mclk_fs) {
-		mclk = params_rate(params) * mclk_fs;
+	/*
+	 * Use snd_soc_dai_link_component instead of legacy style.
+	 * It is only for codec, but cpu will be supported in the future.
+	 * see
+	 *	soc-core.c :: snd_soc_init_multicodec()
+	 */
+	if (dlc) {
+		dai_name	= &dlc->dai_name;
+		dai_of_node	= &dlc->of_node;
+	}
 
-		ret = simple_set_clk_rate(dai_props->codec_dai, mclk);
-		if (ret < 0)
-			return ret;
+	/*
+	 * Get node via "sound-dai = <&phandle port>"
+	 * it will be used as xxx_of_node on soc_bind_dai_link()
+	 */
+	ret = of_parse_phandle_with_args(node, DAI, CELL, 0, &args);
+	if (ret)
+		return ret;
 
-		ret = simple_set_clk_rate(dai_props->cpu_dai, mclk);
+	/* Get dai->name */
+	if (dai_name) {
+		ret = snd_soc_of_get_dai_name(node, dai_name);
 		if (ret < 0)
 			return ret;
-
-		ret = snd_soc_dai_set_sysclk(codec_dai, 0, mclk,
-					     SND_SOC_CLOCK_IN);
-		if (ret && ret != -ENOTSUPP)
-			goto err;
-
-		ret = snd_soc_dai_set_sysclk(cpu_dai, 0, mclk,
-					     SND_SOC_CLOCK_OUT);
-		if (ret && ret != -ENOTSUPP)
-			goto err;
 	}
-	return 0;
-err:
-	return ret;
-}
-
-static const struct snd_soc_ops simple_ops = {
-	.startup	= simple_startup,
-	.shutdown	= simple_shutdown,
-	.hw_params	= simple_hw_params,
-};
 
-static int simple_dai_init(struct snd_soc_pcm_runtime *rtd)
-{
-	struct simple_priv *priv = snd_soc_card_get_drvdata(rtd->card);
-	struct simple_dai_props *dai_props = simple_priv_to_props(priv, rtd->num);
-	int ret;
+	*dai_of_node = args.np;
 
-	ret = asoc_simple_card_init_dai(rtd->codec_dai,
-					dai_props->codec_dai);
-	if (ret < 0)
-		return ret;
-
-	ret = asoc_simple_card_init_dai(rtd->cpu_dai,
-					dai_props->cpu_dai);
-	if (ret < 0)
-		return ret;
+	if (is_single_link)
+		*is_single_link = !args.args_count;
 
 	return 0;
 }
 
-static int simple_be_hw_params_fixup(struct snd_soc_pcm_runtime *rtd,
-				     struct snd_pcm_hw_params *params)
+static void simple_parse_convert(struct device *dev,
+				 struct device_node *np,
+				 struct asoc_simple_data *adata)
 {
-	struct simple_priv *priv = snd_soc_card_get_drvdata(rtd->card);
-	struct simple_dai_props *dai_props = simple_priv_to_props(priv, rtd->num);
+	struct device_node *top = dev->of_node;
+	struct device_node *node = of_get_parent(np);
 
-	asoc_simple_card_convert_fixup(&dai_props->adata, params);
+	asoc_simple_parse_convert(dev, top,  PREFIX, adata);
+	asoc_simple_parse_convert(dev, node, PREFIX, adata);
+	asoc_simple_parse_convert(dev, node, NULL,   adata);
+	asoc_simple_parse_convert(dev, np,   NULL,   adata);
 
-	return 0;
+	of_node_put(node);
 }
 
-static void simple_get_conversion(struct device *dev,
-				  struct device_node *np,
-				  struct asoc_simple_card_data *adata)
+static void simple_parse_mclk_fs(struct device_node *top,
+				 struct device_node *cpu,
+				 struct device_node *codec,
+				 struct simple_dai_props *props,
+				 char *prefix)
 {
-	struct device_node *top = dev->of_node;
-	struct device_node *node = of_get_parent(np);
+	struct device_node *node = of_get_parent(cpu);
+	char prop[128];
 
-	asoc_simple_card_parse_convert(dev, top,  PREFIX, adata);
-	asoc_simple_card_parse_convert(dev, node, PREFIX, adata);
-	asoc_simple_card_parse_convert(dev, node, NULL,   adata);
-	asoc_simple_card_parse_convert(dev, np,   NULL,   adata);
+	snprintf(prop, sizeof(prop), "%smclk-fs", PREFIX);
+	of_property_read_u32(top,	prop, &props->mclk_fs);
+
+	snprintf(prop, sizeof(prop), "%smclk-fs", prefix);
+	of_property_read_u32(node,	prop, &props->mclk_fs);
+	of_property_read_u32(cpu,	prop, &props->mclk_fs);
+	of_property_read_u32(codec,	prop, &props->mclk_fs);
 
 	of_node_put(node);
 }
 
-static int simple_dai_link_of_dpcm(struct simple_priv *priv,
+static int simple_dai_link_of_dpcm(struct asoc_simple_priv *priv,
 				   struct device_node *np,
 				   struct device_node *codec,
 				   struct link_info *li,
@@ -203,7 +122,6 @@ static int simple_dai_link_of_dpcm(struct simple_priv *priv,
 	struct snd_soc_dai_link_component *codecs = dai_link->codecs;
 	struct device_node *top = dev->of_node;
 	struct device_node *node = of_get_parent(np);
-	char prop[128];
 	char *prefix = "";
 	int ret;
 
@@ -241,22 +159,21 @@ static int simple_dai_link_of_dpcm(struct simple_priv *priv,
 		dai =
 		dai_props->cpu_dai	= &priv->dais[li->dais++];
 
-		ret = asoc_simple_card_parse_cpu(np, dai_link, DAI, CELL,
-						 &is_single_links);
+		ret = asoc_simple_parse_cpu(np, dai_link, &is_single_links);
 		if (ret)
 			return ret;
 
-		ret = asoc_simple_card_parse_clk_cpu(dev, np, dai_link, dai);
+		ret = asoc_simple_parse_clk_cpu(dev, np, dai_link, dai);
 		if (ret < 0)
 			return ret;
 
-		ret = asoc_simple_card_set_dailink_name(dev, dai_link,
-							"fe.%s",
-							dai_link->cpu_dai_name);
+		ret = asoc_simple_set_dailink_name(dev, dai_link,
+						   "fe.%s",
+						   dai_link->cpu_dai_name);
 		if (ret < 0)
 			return ret;
 
-		asoc_simple_card_canonicalize_cpu(dai_link, is_single_links);
+		asoc_simple_canonicalize_cpu(dai_link, is_single_links);
 	} else {
 		struct snd_soc_codec_conf *cconf;
 
@@ -267,7 +184,7 @@ static int simple_dai_link_of_dpcm(struct simple_priv *priv,
 
 		/* BE settings */
 		dai_link->no_pcm		= 1;
-		dai_link->be_hw_params_fixup	= simple_be_hw_params_fixup;
+		dai_link->be_hw_params_fixup	= asoc_simple_be_hw_params_fixup;
 
 		dai =
 		dai_props->codec_dai	= &priv->dais[li->dais++];
@@ -275,17 +192,17 @@ static int simple_dai_link_of_dpcm(struct simple_priv *priv,
 		cconf =
 		dai_props->codec_conf	= &priv->codec_conf[li->conf++];
 
-		ret = asoc_simple_card_parse_codec(np, dai_link, DAI, CELL);
+		ret = asoc_simple_parse_codec(np, dai_link);
 		if (ret < 0)
 			return ret;
 
-		ret = asoc_simple_card_parse_clk_codec(dev, np, dai_link, dai);
+		ret = asoc_simple_parse_clk_codec(dev, np, dai_link, dai);
 		if (ret < 0)
 			return ret;
 
-		ret = asoc_simple_card_set_dailink_name(dev, dai_link,
-							"be.%s",
-							codecs->dai_name);
+		ret = asoc_simple_set_dailink_name(dev, dai_link,
+						   "be.%s",
+						   codecs->dai_name);
 		if (ret < 0)
 			return ret;
 
@@ -298,33 +215,29 @@ static int simple_dai_link_of_dpcm(struct simple_priv *priv,
 					     "prefix");
 	}
 
-	simple_get_conversion(dev, np, &dai_props->adata);
+	simple_parse_convert(dev, np, &dai_props->adata);
+	simple_parse_mclk_fs(top, np, codec, dai_props, prefix);
 
-	asoc_simple_card_canonicalize_platform(dai_link);
+	asoc_simple_canonicalize_platform(dai_link);
 
-	ret = asoc_simple_card_of_parse_tdm(np, dai);
+	ret = asoc_simple_parse_tdm(np, dai);
 	if (ret)
 		return ret;
 
-	snprintf(prop, sizeof(prop), "%smclk-fs", prefix);
-	of_property_read_u32(top,  PREFIX "mclk-fs", &dai_props->mclk_fs);
-	of_property_read_u32(node, prop, &dai_props->mclk_fs);
-	of_property_read_u32(np,   prop, &dai_props->mclk_fs);
-
-	ret = asoc_simple_card_parse_daifmt(dev, node, codec,
-					    prefix, &dai_link->dai_fmt);
+	ret = asoc_simple_parse_daifmt(dev, node, codec,
+				       prefix, &dai_link->dai_fmt);
 	if (ret < 0)
 		return ret;
 
 	dai_link->dpcm_playback		= 1;
 	dai_link->dpcm_capture		= 1;
 	dai_link->ops			= &simple_ops;
-	dai_link->init			= simple_dai_init;
+	dai_link->init			= asoc_simple_dai_init;
 
 	return 0;
 }
 
-static int simple_dai_link_of(struct simple_priv *priv,
+static int simple_dai_link_of(struct asoc_simple_priv *priv,
 			      struct device_node *np,
 			      struct device_node *codec,
 			      struct link_info *li,
@@ -370,58 +283,53 @@ static int simple_dai_link_of(struct simple_priv *priv,
 	codec_dai		=
 	dai_props->codec_dai	= &priv->dais[li->dais++];
 
-	ret = asoc_simple_card_parse_daifmt(dev, node, codec,
-					    prefix, &dai_link->dai_fmt);
+	ret = asoc_simple_parse_daifmt(dev, node, codec,
+				       prefix, &dai_link->dai_fmt);
 	if (ret < 0)
 		goto dai_link_of_err;
 
-	snprintf(prop, sizeof(prop), "%smclk-fs", prefix);
-	of_property_read_u32(top,  PREFIX "mclk-fs", &dai_props->mclk_fs);
-	of_property_read_u32(node,  prop, &dai_props->mclk_fs);
-	of_property_read_u32(cpu,   prop, &dai_props->mclk_fs);
-	of_property_read_u32(codec, prop, &dai_props->mclk_fs);
+	simple_parse_mclk_fs(top, cpu, codec, dai_props, prefix);
 
-	ret = asoc_simple_card_parse_cpu(cpu, dai_link,
-					 DAI, CELL, &single_cpu);
+	ret = asoc_simple_parse_cpu(cpu, dai_link, &single_cpu);
 	if (ret < 0)
 		goto dai_link_of_err;
 
-	ret = asoc_simple_card_parse_codec(codec, dai_link, DAI, CELL);
+	ret = asoc_simple_parse_codec(codec, dai_link);
 	if (ret < 0)
 		goto dai_link_of_err;
 
-	ret = asoc_simple_card_parse_platform(plat, dai_link, DAI, CELL);
+	ret = asoc_simple_parse_platform(plat, dai_link);
 	if (ret < 0)
 		goto dai_link_of_err;
 
-	ret = asoc_simple_card_of_parse_tdm(cpu, cpu_dai);
+	ret = asoc_simple_parse_tdm(cpu, cpu_dai);
 	if (ret < 0)
 		goto dai_link_of_err;
 
-	ret = asoc_simple_card_of_parse_tdm(codec, codec_dai);
+	ret = asoc_simple_parse_tdm(codec, codec_dai);
 	if (ret < 0)
 		goto dai_link_of_err;
 
-	ret = asoc_simple_card_parse_clk_cpu(dev, cpu, dai_link, cpu_dai);
+	ret = asoc_simple_parse_clk_cpu(dev, cpu, dai_link, cpu_dai);
 	if (ret < 0)
 		goto dai_link_of_err;
 
-	ret = asoc_simple_card_parse_clk_codec(dev, codec, dai_link, codec_dai);
+	ret = asoc_simple_parse_clk_codec(dev, codec, dai_link, codec_dai);
 	if (ret < 0)
 		goto dai_link_of_err;
 
-	ret = asoc_simple_card_set_dailink_name(dev, dai_link,
-						"%s-%s",
-						dai_link->cpu_dai_name,
-						dai_link->codecs->dai_name);
+	ret = asoc_simple_set_dailink_name(dev, dai_link,
+					   "%s-%s",
+					   dai_link->cpu_dai_name,
+					   dai_link->codecs->dai_name);
 	if (ret < 0)
 		goto dai_link_of_err;
 
 	dai_link->ops = &simple_ops;
-	dai_link->init = simple_dai_init;
+	dai_link->init = asoc_simple_dai_init;
 
-	asoc_simple_card_canonicalize_cpu(dai_link, single_cpu);
-	asoc_simple_card_canonicalize_platform(dai_link);
+	asoc_simple_canonicalize_cpu(dai_link, single_cpu);
+	asoc_simple_canonicalize_platform(dai_link);
 
 dai_link_of_err:
 	of_node_put(plat);
@@ -430,13 +338,13 @@ dai_link_of_err:
 	return ret;
 }
 
-static int simple_for_each_link(struct simple_priv *priv,
+static int simple_for_each_link(struct asoc_simple_priv *priv,
 			struct link_info *li,
-			int (*func_noml)(struct simple_priv *priv,
+			int (*func_noml)(struct asoc_simple_priv *priv,
 					 struct device_node *np,
 					 struct device_node *codec,
 					 struct link_info *li, bool is_top),
-			int (*func_dpcm)(struct simple_priv *priv,
+			int (*func_dpcm)(struct asoc_simple_priv *priv,
 					 struct device_node *np,
 					 struct device_node *codec,
 					 struct link_info *li, bool is_top))
@@ -457,7 +365,7 @@ static int simple_for_each_link(struct simple_priv *priv,
 
 	/* loop for all dai-link */
 	do {
-		struct asoc_simple_card_data adata;
+		struct asoc_simple_data adata;
 		struct device_node *codec;
 		struct device_node *np;
 		int num = of_get_child_count(node);
@@ -475,7 +383,7 @@ static int simple_for_each_link(struct simple_priv *priv,
 		/* get convert-xxx property */
 		memset(&adata, 0, sizeof(adata));
 		for_each_child_of_node(node, np)
-			simple_get_conversion(dev, np, &adata);
+			simple_parse_convert(dev, np, &adata);
 
 		/* loop for all CPU/Codec node */
 		for_each_child_of_node(node, np) {
@@ -507,7 +415,7 @@ static int simple_for_each_link(struct simple_priv *priv,
 }
 
 static int simple_parse_aux_devs(struct device_node *node,
-				 struct simple_priv *priv)
+				 struct asoc_simple_priv *priv)
 {
 	struct device *dev = simple_priv_to_dev(priv);
 	struct device_node *aux_node;
@@ -537,7 +445,7 @@ static int simple_parse_aux_devs(struct device_node *node,
 	return 0;
 }
 
-static int simple_parse_of(struct simple_priv *priv)
+static int simple_parse_of(struct asoc_simple_priv *priv)
 {
 	struct device *dev = simple_priv_to_dev(priv);
 	struct device_node *top = dev->of_node;
@@ -548,11 +456,15 @@ static int simple_parse_of(struct simple_priv *priv)
 	if (!top)
 		return -EINVAL;
 
-	ret = asoc_simple_card_of_parse_widgets(card, PREFIX);
+	ret = asoc_simple_parse_widgets(card, PREFIX);
 	if (ret < 0)
 		return ret;
 
-	ret = asoc_simple_card_of_parse_routing(card, PREFIX);
+	ret = asoc_simple_parse_routing(card, PREFIX);
+	if (ret < 0)
+		return ret;
+
+	ret = asoc_simple_parse_pin_switches(card, PREFIX);
 	if (ret < 0)
 		return ret;
 
@@ -578,7 +490,7 @@ static int simple_parse_of(struct simple_priv *priv)
 			return ret;
 	}
 
-	ret = asoc_simple_card_parse_card_name(card, PREFIX);
+	ret = asoc_simple_parse_card_name(card, PREFIX);
 	if (ret < 0)
 		return ret;
 
@@ -587,7 +499,7 @@ static int simple_parse_of(struct simple_priv *priv)
 	return ret;
 }
 
-static int simple_count_noml(struct simple_priv *priv,
+static int simple_count_noml(struct asoc_simple_priv *priv,
 			     struct device_node *np,
 			     struct device_node *codec,
 			     struct link_info *li, bool is_top)
@@ -599,7 +511,7 @@ static int simple_count_noml(struct simple_priv *priv,
 	return 0;
 }
 
-static int simple_count_dpcm(struct simple_priv *priv,
+static int simple_count_dpcm(struct asoc_simple_priv *priv,
 			     struct device_node *np,
 			     struct device_node *codec,
 			     struct link_info *li, bool is_top)
@@ -612,7 +524,7 @@ static int simple_count_dpcm(struct simple_priv *priv,
 	return 0;
 }
 
-static void simple_get_dais_count(struct simple_priv *priv,
+static void simple_get_dais_count(struct asoc_simple_priv *priv,
 				  struct link_info *li)
 {
 	struct device *dev = simple_priv_to_dev(priv);
@@ -681,14 +593,14 @@ static void simple_get_dais_count(struct simple_priv *priv,
 
 static int simple_soc_probe(struct snd_soc_card *card)
 {
-	struct simple_priv *priv = snd_soc_card_get_drvdata(card);
+	struct asoc_simple_priv *priv = snd_soc_card_get_drvdata(card);
 	int ret;
 
-	ret = asoc_simple_card_init_hp(card, &priv->hp_jack, PREFIX);
+	ret = asoc_simple_init_hp(card, &priv->hp_jack, PREFIX);
 	if (ret < 0)
 		return ret;
 
-	ret = asoc_simple_card_init_mic(card, &priv->mic_jack, PREFIX);
+	ret = asoc_simple_init_mic(card, &priv->mic_jack, PREFIX);
 	if (ret < 0)
 		return ret;
 
@@ -697,16 +609,12 @@ static int simple_soc_probe(struct snd_soc_card *card)
 
 static int simple_probe(struct platform_device *pdev)
 {
-	struct simple_priv *priv;
-	struct snd_soc_dai_link *dai_link;
-	struct simple_dai_props *dai_props;
-	struct asoc_simple_dai *dais;
+	struct asoc_simple_priv *priv;
 	struct device *dev = &pdev->dev;
 	struct device_node *np = dev->of_node;
 	struct snd_soc_card *card;
-	struct snd_soc_codec_conf *cconf;
 	struct link_info li;
-	int ret, i;
+	int ret;
 
 	/* Allocate the private data and the DAI link array */
 	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
@@ -723,35 +631,9 @@ static int simple_probe(struct platform_device *pdev)
 	if (!li.link || !li.dais)
 		return -EINVAL;
 
-	dai_props = devm_kcalloc(dev, li.link, sizeof(*dai_props), GFP_KERNEL);
-	dai_link  = devm_kcalloc(dev, li.link, sizeof(*dai_link),  GFP_KERNEL);
-	dais      = devm_kcalloc(dev, li.dais, sizeof(*dais),      GFP_KERNEL);
-	cconf     = devm_kcalloc(dev, li.conf, sizeof(*cconf),     GFP_KERNEL);
-	if (!dai_props || !dai_link || !dais)
-		return -ENOMEM;
-
-	/*
-	 * Use snd_soc_dai_link_component instead of legacy style
-	 * It is codec only. but cpu/platform will be supported in the future.
-	 * see
-	 *	soc-core.c :: snd_soc_init_multicodec()
-	 */
-	for (i = 0; i < li.link; i++) {
-		dai_link[i].codecs	= &dai_props[i].codecs;
-		dai_link[i].num_codecs	= 1;
-		dai_link[i].platforms	= &dai_props[i].platforms;
-		dai_link[i].num_platforms = 1;
-	}
-
-	priv->dai_props		= dai_props;
-	priv->dai_link		= dai_link;
-	priv->dais		= dais;
-	priv->codec_conf	= cconf;
-
-	card->dai_link		= priv->dai_link;
-	card->num_links		= li.link;
-	card->codec_conf	= cconf;
-	card->num_configs	= li.conf;
+	ret = asoc_simple_init_priv(priv, &li);
+	if (ret < 0)
+		return ret;
 
 	if (np && of_device_is_available(np)) {
 
@@ -766,6 +648,9 @@ static int simple_probe(struct platform_device *pdev)
 		struct asoc_simple_card_info *cinfo;
 		struct snd_soc_dai_link_component *codecs;
 		struct snd_soc_dai_link_component *platform;
+		struct snd_soc_dai_link *dai_link = priv->dai_link;
+		struct simple_dai_props *dai_props = priv->dai_props;
+
 		int dai_idx = 0;
 
 		cinfo = dev->platform_data;
@@ -798,22 +683,24 @@ static int simple_probe(struct platform_device *pdev)
 		dai_link->stream_name	= cinfo->name;
 		dai_link->cpu_dai_name	= cinfo->cpu_dai.name;
 		dai_link->dai_fmt	= cinfo->daifmt;
-		dai_link->init		= simple_dai_init;
-		memcpy(priv->dai_props->cpu_dai, &cinfo->cpu_dai,
-					sizeof(*priv->dai_props->cpu_dai));
-		memcpy(priv->dai_props->codec_dai, &cinfo->codec_dai,
-					sizeof(*priv->dai_props->codec_dai));
+		dai_link->init		= asoc_simple_dai_init;
+		memcpy(dai_props->cpu_dai, &cinfo->cpu_dai,
+					sizeof(*dai_props->cpu_dai));
+		memcpy(dai_props->codec_dai, &cinfo->codec_dai,
+					sizeof(*dai_props->codec_dai));
 	}
 
 	snd_soc_card_set_drvdata(card, priv);
 
+	asoc_simple_debug_info(priv);
+
 	ret = devm_snd_soc_register_card(dev, card);
 	if (ret < 0)
 		goto err;
 
 	return 0;
 err:
-	asoc_simple_card_clean_reference(card);
+	asoc_simple_clean_reference(card);
 
 	return ret;
 }
@@ -822,7 +709,7 @@ static int simple_remove(struct platform_device *pdev)
 {
 	struct snd_soc_card *card = platform_get_drvdata(pdev);
 
-	return asoc_simple_card_clean_reference(card);
+	return asoc_simple_clean_reference(card);
 }
 
 static const struct of_device_id simple_of_match[] = {
diff --git a/sound/soc/intel/Kconfig b/sound/soc/intel/Kconfig
index bd9fd2035c55..fc1396adde71 100644
--- a/sound/soc/intel/Kconfig
+++ b/sound/soc/intel/Kconfig
@@ -196,13 +196,18 @@ config SND_SOC_INTEL_SKYLAKE_COMMON
 
 endif ## SND_SOC_INTEL_SKYLAKE_FAMILY
 
+endif ## SND_SOC_INTEL_SST_TOPLEVEL
+
+if SND_SOC_INTEL_SST_TOPLEVEL || SND_SOC_SOF_INTEL_TOPLEVEL
+
 config SND_SOC_ACPI_INTEL_MATCH
 	tristate
 	select SND_SOC_ACPI if ACPI
 	# this option controls the compilation of ACPI matching tables and
 	# helpers and is not meant to be selected by the user.
 
-endif ## SND_SOC_INTEL_SST_TOPLEVEL
+endif ## SND_SOC_INTEL_SST_TOPLEVEL || SND_SOC_SOF_INTEL_TOPLEVEL
+
 
 # ASoC codec drivers
 source "sound/soc/intel/boards/Kconfig"
diff --git a/sound/soc/intel/boards/Kconfig b/sound/soc/intel/boards/Kconfig
index 12d6b73e9531..e39473a6a5d9 100644
--- a/sound/soc/intel/boards/Kconfig
+++ b/sound/soc/intel/boards/Kconfig
@@ -1,6 +1,6 @@
 menuconfig SND_SOC_INTEL_MACH
 	bool "Intel Machine drivers"
-	depends on SND_SOC_INTEL_SST_TOPLEVEL
+	depends on SND_SOC_INTEL_SST_TOPLEVEL || SND_SOC_SOF_INTEL_TOPLEVEL
 	help
          Intel ASoC Machine Drivers. If you have a Intel machine that
          has an audio controller with a DSP and I2S or DMIC port, then
@@ -16,7 +16,9 @@ if SND_SOC_INTEL_HASWELL
 
 config SND_SOC_INTEL_HASWELL_MACH
 	tristate "Haswell Lynxpoint"
-	depends on X86_INTEL_LPSS && I2C && I2C_DESIGNWARE_PLATFORM
+	depends on I2C
+	depends on I2C_DESIGNWARE_PLATFORM || COMPILE_TEST
+	depends on X86_INTEL_LPSS || COMPILE_TEST
 	select SND_SOC_RT5640
 	help
 	  This adds support for the Lynxpoint Audio DSP on Intel(R) Haswell
@@ -24,9 +26,16 @@ config SND_SOC_INTEL_HASWELL_MACH
 	  Say Y or m if you have such a device.
 	  If unsure select "N".
 
+endif ## SND_SOC_INTEL_HASWELL
+
+if SND_SOC_INTEL_HASWELL || SND_SOC_SOF_BROADWELL
+
 config SND_SOC_INTEL_BDW_RT5677_MACH
 	tristate "Broadwell with RT5677 codec"
-	depends on X86_INTEL_LPSS && I2C && I2C_DESIGNWARE_PLATFORM && GPIOLIB
+	depends on I2C
+	depends on I2C_DESIGNWARE_PLATFORM || COMPILE_TEST
+	depends on GPIOLIB || COMPILE_TEST
+	depends on X86_INTEL_LPSS || COMPILE_TEST
 	select SND_SOC_RT5677
 	help
 	  This adds support for Intel Broadwell platform based boards with
@@ -36,20 +45,23 @@ config SND_SOC_INTEL_BDW_RT5677_MACH
 
 config SND_SOC_INTEL_BROADWELL_MACH
 	tristate "Broadwell Wildcatpoint"
-	depends on X86_INTEL_LPSS && I2C && I2C_DESIGNWARE_PLATFORM
+	depends on I2C
+	depends on I2C_DESIGNWARE_PLATFORM || COMPILE_TEST
+	depends on X86_INTEL_LPSS || COMPILE_TEST
 	select SND_SOC_RT286
 	help
 	  This adds support for the Wilcatpoint Audio DSP on Intel(R) Broadwell
 	  Ultrabook platforms.
 	  Say Y or m if you have such a device. This is a recommended option.
 	  If unsure select "N".
-endif ## SND_SOC_INTEL_HASWELL
+endif ## SND_SOC_INTEL_HASWELL || SND_SOC_SOF_BROADWELL
 
 if SND_SOC_INTEL_BAYTRAIL
 
 config SND_SOC_INTEL_BYT_MAX98090_MACH
 	tristate "Baytrail with MAX98090 codec"
-	depends on X86_INTEL_LPSS && I2C
+	depends on I2C
+	depends on X86_INTEL_LPSS || COMPILE_TEST
 	select SND_SOC_MAX98090
 	help
 	  This adds audio driver for Intel Baytrail platform based boards
@@ -59,7 +71,8 @@ config SND_SOC_INTEL_BYT_MAX98090_MACH
 
 config SND_SOC_INTEL_BYT_RT5640_MACH
 	tristate "Baytrail with RT5640 codec"
-	depends on X86_INTEL_LPSS && I2C
+	depends on I2C
+	depends on X86_INTEL_LPSS || COMPILE_TEST
 	select SND_SOC_RT5640
 	help
 	  This adds audio driver for Intel Baytrail platform based boards
@@ -68,11 +81,12 @@ config SND_SOC_INTEL_BYT_RT5640_MACH
 
 endif ## SND_SOC_INTEL_BAYTRAIL
 
-if SND_SST_ATOM_HIFI2_PLATFORM
+if SND_SST_ATOM_HIFI2_PLATFORM || SND_SOC_SOF_BAYTRAIL
 
 config SND_SOC_INTEL_BYTCR_RT5640_MACH
 	tristate "Baytrail and Baytrail-CR with RT5640 codec"
-	depends on X86_INTEL_LPSS && I2C && ACPI
+	depends on I2C && ACPI
+	depends on X86_INTEL_LPSS || COMPILE_TEST
 	select SND_SOC_ACPI
 	select SND_SOC_RT5640
 	help
@@ -83,7 +97,8 @@ config SND_SOC_INTEL_BYTCR_RT5640_MACH
 
 config SND_SOC_INTEL_BYTCR_RT5651_MACH
 	tristate "Baytrail and Baytrail-CR with RT5651 codec"
-	depends on X86_INTEL_LPSS && I2C && ACPI
+	depends on I2C && ACPI
+	depends on X86_INTEL_LPSS || COMPILE_TEST
 	select SND_SOC_ACPI
 	select SND_SOC_RT5651
 	help
@@ -94,7 +109,8 @@ config SND_SOC_INTEL_BYTCR_RT5651_MACH
 
 config SND_SOC_INTEL_CHT_BSW_RT5672_MACH
 	tristate "Cherrytrail & Braswell with RT5672 codec"
-	depends on X86_INTEL_LPSS && I2C && ACPI
+	depends on I2C && ACPI
+	depends on X86_INTEL_LPSS || COMPILE_TEST
 	select SND_SOC_ACPI
 	select SND_SOC_RT5670
         help
@@ -105,7 +121,8 @@ config SND_SOC_INTEL_CHT_BSW_RT5672_MACH
 
 config SND_SOC_INTEL_CHT_BSW_RT5645_MACH
 	tristate "Cherrytrail & Braswell with RT5645/5650 codec"
-	depends on X86_INTEL_LPSS && I2C && ACPI
+	depends on I2C && ACPI
+	depends on X86_INTEL_LPSS || COMPILE_TEST
 	select SND_SOC_ACPI
 	select SND_SOC_RT5645
 	help
@@ -116,7 +133,8 @@ config SND_SOC_INTEL_CHT_BSW_RT5645_MACH
 
 config SND_SOC_INTEL_CHT_BSW_MAX98090_TI_MACH
 	tristate "Cherrytrail & Braswell with MAX98090 & TI codec"
-	depends on X86_INTEL_LPSS && I2C && ACPI
+	depends on I2C && ACPI
+	depends on X86_INTEL_LPSS || COMPILE_TEST
 	select SND_SOC_MAX98090
 	select SND_SOC_TS3A227E
 	help
@@ -127,7 +145,8 @@ config SND_SOC_INTEL_CHT_BSW_MAX98090_TI_MACH
 
 config SND_SOC_INTEL_CHT_BSW_NAU8824_MACH
 	tristate "Cherrytrail & Braswell with NAU88L24 codec"
-	depends on X86_INTEL_LPSS && I2C && ACPI
+	depends on I2C && ACPI
+	depends on X86_INTEL_LPSS || COMPILE_TEST
 	select SND_SOC_ACPI
 	select SND_SOC_NAU8824
 	help
@@ -138,7 +157,8 @@ config SND_SOC_INTEL_CHT_BSW_NAU8824_MACH
 
 config SND_SOC_INTEL_BYT_CHT_DA7213_MACH
 	tristate "Baytrail & Cherrytrail with DA7212/7213 codec"
-	depends on X86_INTEL_LPSS && I2C && ACPI
+	depends on I2C && ACPI
+	depends on X86_INTEL_LPSS || COMPILE_TEST
 	select SND_SOC_ACPI
 	select SND_SOC_DA7213
 	help
@@ -149,7 +169,8 @@ config SND_SOC_INTEL_BYT_CHT_DA7213_MACH
 
 config SND_SOC_INTEL_BYT_CHT_ES8316_MACH
 	tristate "Baytrail & Cherrytrail with ES8316 codec"
-	depends on X86_INTEL_LPSS && I2C && ACPI
+	depends on I2C && ACPI
+	depends on X86_INTEL_LPSS || COMPILE_TEST
 	select SND_SOC_ACPI
 	select SND_SOC_ES8316
 	help
@@ -158,9 +179,14 @@ config SND_SOC_INTEL_BYT_CHT_ES8316_MACH
 	  Say Y or m if you have such a device. This is a recommended option.
 	  If unsure select "N".
 
+endif ## SND_SST_ATOM_HIFI2_PLATFORM || SND_SOC_SOF_BAYTRAIL
+
+if SND_SST_ATOM_HIFI2_PLATFORM
+
 config SND_SOC_INTEL_BYT_CHT_NOCODEC_MACH
 	tristate "Baytrail & Cherrytrail platform with no codec (MinnowBoard MAX, Up)"
-	depends on X86_INTEL_LPSS && I2C && ACPI
+	depends on I2C && ACPI
+	depends on X86_INTEL_LPSS || COMPILE_TEST
 	help
 	  This adds support for ASoC machine driver for the MinnowBoard Max or
 	  Up boards and provides access to I2S signals on the Low-Speed
@@ -176,7 +202,8 @@ if SND_SOC_INTEL_SKL
 
 config SND_SOC_INTEL_SKL_RT286_MACH
 	tristate "SKL with RT286 I2S mode"
-	depends on MFD_INTEL_LPSS && I2C && ACPI
+	depends on I2C && ACPI
+	depends on MFD_INTEL_LPSS || COMPILE_TEST
 	select SND_SOC_RT286
 	select SND_SOC_DMIC
 	select SND_SOC_HDAC_HDMI
@@ -188,7 +215,8 @@ config SND_SOC_INTEL_SKL_RT286_MACH
 
 config SND_SOC_INTEL_SKL_NAU88L25_SSM4567_MACH
 	tristate "SKL with NAU88L25 and SSM4567 in I2S Mode"
-	depends on MFD_INTEL_LPSS && I2C && ACPI
+	depends on I2C && ACPI
+	depends on MFD_INTEL_LPSS || COMPILE_TEST
 	select SND_SOC_NAU8825
 	select SND_SOC_SSM4567
 	select SND_SOC_DMIC
@@ -201,7 +229,8 @@ config SND_SOC_INTEL_SKL_NAU88L25_SSM4567_MACH
 
 config SND_SOC_INTEL_SKL_NAU88L25_MAX98357A_MACH
 	tristate "SKL with NAU88L25 and MAX98357A in I2S Mode"
-	depends on MFD_INTEL_LPSS && I2C && ACPI
+	depends on I2C && ACPI
+	depends on MFD_INTEL_LPSS || COMPILE_TEST
 	select SND_SOC_NAU8825
 	select SND_SOC_MAX98357A
 	select SND_SOC_DMIC
@@ -218,7 +247,8 @@ if SND_SOC_INTEL_APL
 
 config SND_SOC_INTEL_BXT_DA7219_MAX98357A_MACH
 	tristate "Broxton with DA7219 and MAX98357A in I2S Mode"
-	depends on MFD_INTEL_LPSS && I2C && ACPI
+	depends on I2C && ACPI
+	depends on MFD_INTEL_LPSS || COMPILE_TEST
 	select SND_SOC_DA7219
 	select SND_SOC_MAX98357A
 	select SND_SOC_DMIC
@@ -232,7 +262,8 @@ config SND_SOC_INTEL_BXT_DA7219_MAX98357A_MACH
 
 config SND_SOC_INTEL_BXT_RT298_MACH
 	tristate "Broxton with RT298 I2S mode"
-	depends on MFD_INTEL_LPSS && I2C && ACPI
+	depends on I2C && ACPI
+	depends on MFD_INTEL_LPSS || COMPILE_TEST
 	select SND_SOC_RT298
 	select SND_SOC_DMIC
 	select SND_SOC_HDAC_HDMI
@@ -249,7 +280,8 @@ if SND_SOC_INTEL_KBL
 
 config SND_SOC_INTEL_KBL_RT5663_MAX98927_MACH
 	tristate "KBL with RT5663 and MAX98927 in I2S Mode"
-	depends on MFD_INTEL_LPSS && I2C && ACPI
+	depends on I2C && ACPI
+	depends on MFD_INTEL_LPSS || COMPILE_TEST
 	select SND_SOC_RT5663
 	select SND_SOC_MAX98927
 	select SND_SOC_DMIC
@@ -263,7 +295,8 @@ config SND_SOC_INTEL_KBL_RT5663_MAX98927_MACH
 
 config SND_SOC_INTEL_KBL_RT5663_RT5514_MAX98927_MACH
         tristate "KBL with RT5663, RT5514 and MAX98927 in I2S Mode"
-        depends on MFD_INTEL_LPSS && I2C && ACPI
+	depends on I2C && ACPI
+	depends on MFD_INTEL_LPSS || COMPILE_TEST
         depends on SPI
         select SND_SOC_RT5663
         select SND_SOC_RT5514
@@ -278,7 +311,8 @@ config SND_SOC_INTEL_KBL_RT5663_RT5514_MAX98927_MACH
 
 config SND_SOC_INTEL_KBL_DA7219_MAX98357A_MACH
 	tristate "KBL with DA7219 and MAX98357A in I2S Mode"
-	depends on MFD_INTEL_LPSS && I2C && ACPI
+	depends on I2C && ACPI
+	depends on MFD_INTEL_LPSS || COMPILE_TEST
 	select SND_SOC_DA7219
 	select SND_SOC_MAX98357A
 	select SND_SOC_DMIC
@@ -290,7 +324,8 @@ config SND_SOC_INTEL_KBL_DA7219_MAX98357A_MACH
 
 config SND_SOC_INTEL_KBL_DA7219_MAX98927_MACH
 	tristate "KBL with DA7219 and MAX98927 in I2S Mode"
-	depends on MFD_INTEL_LPSS && I2C && ACPI
+	depends on I2C && ACPI
+	depends on MFD_INTEL_LPSS || COMPILE_TEST
 	select SND_SOC_DA7219
 	select SND_SOC_MAX98927
 	select SND_SOC_MAX98373
@@ -304,7 +339,8 @@ config SND_SOC_INTEL_KBL_DA7219_MAX98927_MACH
 
 config SND_SOC_INTEL_KBL_RT5660_MACH
 	tristate "KBL with RT5660 in I2S Mode"
-	depends on MFD_INTEL_LPSS && I2C && ACPI
+	depends on I2C && ACPI
+	depends on MFD_INTEL_LPSS || COMPILE_TEST
 	select SND_SOC_RT5660
 	select SND_SOC_HDAC_HDMI
 	help
@@ -314,11 +350,12 @@ config SND_SOC_INTEL_KBL_RT5660_MACH
 
 endif ## SND_SOC_INTEL_KBL
 
-if SND_SOC_INTEL_GLK
+if SND_SOC_INTEL_GLK || (SND_SOC_SOF_GEMINILAKE  && SND_SOC_SOF_HDA_LINK)
 
 config SND_SOC_INTEL_GLK_RT5682_MAX98357A_MACH
 	tristate "GLK with RT5682 and MAX98357A in I2S Mode"
-	depends on MFD_INTEL_LPSS && I2C && ACPI
+	depends on I2C && ACPI
+	depends on MFD_INTEL_LPSS || COMPILE_TEST
 	select SND_SOC_RT5682
 	select SND_SOC_MAX98357A
 	select SND_SOC_DMIC
@@ -330,9 +367,9 @@ config SND_SOC_INTEL_GLK_RT5682_MAX98357A_MACH
 	   Say Y if you have such a device.
 	   If unsure select "N".
 
-endif ## SND_SOC_INTEL_GLK
+endif ## SND_SOC_INTEL_GLK || (SND_SOC_SOF_GEMINILAKE  && SND_SOC_SOF_HDA_LINK)
 
-if SND_SOC_INTEL_SKYLAKE_HDAUDIO_CODEC
+if SND_SOC_INTEL_SKYLAKE_HDAUDIO_CODEC || SND_SOC_SOF_HDA_AUDIO_CODEC
 
 config SND_SOC_INTEL_SKL_HDA_DSP_GENERIC_MACH
 	tristate "SKL/KBL/BXT/APL with HDA Codecs"
@@ -344,6 +381,22 @@ config SND_SOC_INTEL_SKL_HDA_DSP_GENERIC_MACH
           Say Y or m if you have such a device. This is a recommended option.
 	  If unsure select "N".
 
-endif ## SND_SOC_INTEL_SKYLAKE_HDAUDIO_CODEC
+endif ## SND_SOC_INTEL_SKYLAKE_HDAUDIO_CODEC || SND_SOC_SOF_HDA_AUDIO_CODEC
+
+if SND_SOC_SOF_HDA_COMMON || SND_SOC_SOF_BAYTRAIL
+config SND_SOC_INTEL_SOF_RT5682_MACH
+	tristate "SOF with rt5682 codec in I2S Mode"
+	depends on I2C && ACPI
+	depends on (SND_SOC_SOF_HDA_COMMON && MFD_INTEL_LPSS) ||\
+		   (SND_SOC_SOF_BAYTRAIL && X86_INTEL_LPSS)
+	select SND_SOC_RT5682
+	select SND_SOC_DMIC
+	select SND_SOC_HDAC_HDMI if SND_SOC_SOF_HDA_COMMON
+	help
+	   This adds support for ASoC machine driver for SOF platforms
+	   with rt5682 codec.
+	   Say Y if you have such a device.
+	   If unsure select "N".
+endif ## SND_SOC_SOF_HDA_COMMON || SND_SOC_SOF_BAYTRAIL
 
 endif ## SND_SOC_INTEL_MACH
diff --git a/sound/soc/intel/boards/Makefile b/sound/soc/intel/boards/Makefile
index bf072ea299b7..451b3bd7d9c5 100644
--- a/sound/soc/intel/boards/Makefile
+++ b/sound/soc/intel/boards/Makefile
@@ -16,6 +16,7 @@ snd-soc-sst-cht-bsw-nau8824-objs := cht_bsw_nau8824.o
 snd-soc-sst-byt-cht-da7213-objs := bytcht_da7213.o
 snd-soc-sst-byt-cht-es8316-objs := bytcht_es8316.o
 snd-soc-sst-byt-cht-nocodec-objs := bytcht_nocodec.o
+snd-soc-sof_rt5682-objs := sof_rt5682.o
 snd-soc-kbl_da7219_max98357a-objs := kbl_da7219_max98357a.o
 snd-soc-kbl_da7219_max98927-objs := kbl_da7219_max98927.o
 snd-soc-kbl_rt5663_max98927-objs := kbl_rt5663_max98927.o
@@ -26,6 +27,7 @@ snd-soc-skl_hda_dsp-objs := skl_hda_dsp_generic.o skl_hda_dsp_common.o
 snd-skl_nau88l25_max98357a-objs := skl_nau88l25_max98357a.o
 snd-soc-skl_nau88l25_ssm4567-objs := skl_nau88l25_ssm4567.o
 
+obj-$(CONFIG_SND_SOC_INTEL_SOF_RT5682_MACH) += snd-soc-sof_rt5682.o
 obj-$(CONFIG_SND_SOC_INTEL_HASWELL_MACH) += snd-soc-sst-haswell.o
 obj-$(CONFIG_SND_SOC_INTEL_BYT_RT5640_MACH) += snd-soc-sst-byt-rt5640-mach.o
 obj-$(CONFIG_SND_SOC_INTEL_BYT_MAX98090_MACH) += snd-soc-sst-byt-max98090-mach.o
diff --git a/sound/soc/intel/boards/bdw-rt5677.c b/sound/soc/intel/boards/bdw-rt5677.c
index 1844c88ea4e2..6520a8ea5537 100644
--- a/sound/soc/intel/boards/bdw-rt5677.c
+++ b/sound/soc/intel/boards/bdw-rt5677.c
@@ -180,6 +180,7 @@ static const struct snd_soc_ops bdw_rt5677_ops = {
 	.hw_params = bdw_rt5677_hw_params,
 };
 
+#if !IS_ENABLED(CONFIG_SND_SOC_SOF_BROADWELL)
 static int bdw_rt5677_rtd_init(struct snd_soc_pcm_runtime *rtd)
 {
 	struct snd_soc_component *component = snd_soc_rtdcom_lookup(rtd, DRV_NAME);
@@ -198,6 +199,7 @@ static int bdw_rt5677_rtd_init(struct snd_soc_pcm_runtime *rtd)
 
 	return 0;
 }
+#endif
 
 static int bdw_rt5677_init(struct snd_soc_pcm_runtime *rtd)
 {
@@ -265,7 +267,9 @@ static struct snd_soc_dai_link bdw_rt5677_dais[] = {
 		.dynamic = 1,
 		.codec_name = "snd-soc-dummy",
 		.codec_dai_name = "snd-soc-dummy-dai",
+#if !IS_ENABLED(CONFIG_SND_SOC_SOF_BROADWELL)
 		.init = bdw_rt5677_rtd_init,
+#endif
 		.trigger = {
 			SND_SOC_DPCM_TRIGGER_POST,
 			SND_SOC_DPCM_TRIGGER_POST
diff --git a/sound/soc/intel/boards/broadwell.c b/sound/soc/intel/boards/broadwell.c
index b86c746d9b7a..0f18f8964f51 100644
--- a/sound/soc/intel/boards/broadwell.c
+++ b/sound/soc/intel/boards/broadwell.c
@@ -131,6 +131,7 @@ static const struct snd_soc_ops broadwell_rt286_ops = {
 	.hw_params = broadwell_rt286_hw_params,
 };
 
+#if !IS_ENABLED(CONFIG_SND_SOC_SOF_BROADWELL)
 static int broadwell_rtd_init(struct snd_soc_pcm_runtime *rtd)
 {
 	struct snd_soc_component *component = snd_soc_rtdcom_lookup(rtd, DRV_NAME);
@@ -149,6 +150,7 @@ static int broadwell_rtd_init(struct snd_soc_pcm_runtime *rtd)
 
 	return 0;
 }
+#endif
 
 /* broadwell digital audio interface glue - connects codec <--> CPU */
 static struct snd_soc_dai_link broadwell_rt286_dais[] = {
@@ -161,7 +163,9 @@ static struct snd_soc_dai_link broadwell_rt286_dais[] = {
 		.dynamic = 1,
 		.codec_name = "snd-soc-dummy",
 		.codec_dai_name = "snd-soc-dummy-dai",
+#if !IS_ENABLED(CONFIG_SND_SOC_SOF_BROADWELL)
 		.init = broadwell_rtd_init,
+#endif
 		.trigger = {SND_SOC_DPCM_TRIGGER_POST, SND_SOC_DPCM_TRIGGER_POST},
 		.dpcm_playback = 1,
 		.dpcm_capture = 1,
diff --git a/sound/soc/intel/boards/bytcht_es8316.c b/sound/soc/intel/boards/bytcht_es8316.c
index 6937c00cf63d..e8c585ffd04d 100644
--- a/sound/soc/intel/boards/bytcht_es8316.c
+++ b/sound/soc/intel/boards/bytcht_es8316.c
@@ -22,6 +22,7 @@
 #include <linux/acpi.h>
 #include <linux/clk.h>
 #include <linux/device.h>
+#include <linux/dmi.h>
 #include <linux/gpio/consumer.h>
 #include <linux/i2c.h>
 #include <linux/init.h>
@@ -40,6 +41,9 @@
 #include "../atom/sst-atom-controls.h"
 #include "../common/sst-dsp.h"
 
+/* jd-inv + terminating entry */
+#define MAX_NO_PROPS 2
+
 struct byt_cht_es8316_private {
 	struct clk *mclk;
 	struct snd_soc_jack jack;
@@ -55,8 +59,9 @@ enum {
 #define BYT_CHT_ES8316_MAP(quirk)		((quirk) & GENMASK(3, 0))
 #define BYT_CHT_ES8316_SSP0			BIT(16)
 #define BYT_CHT_ES8316_MONO_SPEAKER		BIT(17)
+#define BYT_CHT_ES8316_JD_INVERTED		BIT(18)
 
-static int quirk;
+static unsigned long quirk;
 
 static int quirk_override = -1;
 module_param_named(quirk, quirk_override, int, 0444);
@@ -72,6 +77,8 @@ static void log_quirks(struct device *dev)
 		dev_info(dev, "quirk SSP0 enabled");
 	if (quirk & BYT_CHT_ES8316_MONO_SPEAKER)
 		dev_info(dev, "quirk MONO_SPEAKER enabled\n");
+	if (quirk & BYT_CHT_ES8316_JD_INVERTED)
+		dev_info(dev, "quirk JD_INVERTED enabled\n");
 }
 
 static int byt_cht_es8316_speaker_power_event(struct snd_soc_dapm_widget *w,
@@ -435,15 +442,31 @@ static const struct acpi_gpio_mapping byt_cht_es8316_gpios[] = {
 	{ },
 };
 
+/* Please keep this list alphabetically sorted */
+static const struct dmi_system_id byt_cht_es8316_quirk_table[] = {
+	{	/* Teclast X98 Plus II */
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "TECLAST"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "X98 Plus II"),
+		},
+		.driver_data = (void *)(BYT_CHT_ES8316_INTMIC_IN1_MAP
+					| BYT_CHT_ES8316_JD_INVERTED),
+	},
+	{}
+};
+
 static int snd_byt_cht_es8316_mc_probe(struct platform_device *pdev)
 {
 	static const char * const mic_name[] = { "in1", "in2" };
+	struct property_entry props[MAX_NO_PROPS] = {};
 	struct byt_cht_es8316_private *priv;
+	const struct dmi_system_id *dmi_id;
 	struct device *dev = &pdev->dev;
 	struct snd_soc_acpi_mach *mach;
 	const char *platform_name;
 	struct acpi_device *adev;
 	struct device *codec_dev;
+	unsigned int cnt = 0;
 	int dai_index = 0;
 	int i;
 	int ret = 0;
@@ -480,7 +503,10 @@ static int snd_byt_cht_es8316_mc_probe(struct platform_device *pdev)
 		return ret;
 
 	/* Check for BYTCR or other platform and setup quirks */
-	if (x86_match_cpu(baytrail_cpu_ids) &&
+	dmi_id = dmi_first_match(byt_cht_es8316_quirk_table);
+	if (dmi_id) {
+		quirk = (unsigned long)dmi_id->driver_data;
+	} else if (x86_match_cpu(baytrail_cpu_ids) &&
 	    mach->mach_params.acpi_ipc_irq_index == 0) {
 		/* On BYTCR default to SSP0, internal-mic-in2-map, mono-spk */
 		quirk = BYT_CHT_ES8316_SSP0 | BYT_CHT_ES8316_INTMIC_IN2_MAP |
@@ -491,7 +517,8 @@ static int snd_byt_cht_es8316_mc_probe(struct platform_device *pdev)
 			BYT_CHT_ES8316_MONO_SPEAKER;
 	}
 	if (quirk_override != -1) {
-		dev_info(dev, "Overriding quirk 0x%x => 0x%x\n", quirk,
+		dev_info(dev, "Overriding quirk 0x%x => 0x%x\n",
+			 (unsigned int)quirk,
 			 quirk_override);
 		quirk = quirk_override;
 	}
@@ -513,6 +540,15 @@ static int snd_byt_cht_es8316_mc_probe(struct platform_device *pdev)
 	if (!codec_dev)
 		return -EPROBE_DEFER;
 
+	if (quirk & BYT_CHT_ES8316_JD_INVERTED)
+		props[cnt++] = PROPERTY_ENTRY_BOOL("everest,jack-detect-inverted");
+
+	if (cnt) {
+		ret = device_add_properties(codec_dev, props);
+		if (ret)
+			return ret;
+	}
+
 	devm_acpi_dev_add_driver_gpios(codec_dev, byt_cht_es8316_gpios);
 	priv->speaker_en_gpio =
 		gpiod_get_index(codec_dev, "speaker-enable", 0,
diff --git a/sound/soc/intel/boards/bytcr_rt5640.c b/sound/soc/intel/boards/bytcr_rt5640.c
index f9175cf6747e..dc22df9a99fb 100644
--- a/sound/soc/intel/boards/bytcr_rt5640.c
+++ b/sound/soc/intel/boards/bytcr_rt5640.c
@@ -98,8 +98,8 @@ struct byt_rt5640_private {
 static bool is_bytcr;
 
 static unsigned long byt_rt5640_quirk = BYT_RT5640_MCLK_EN;
-static unsigned int quirk_override;
-module_param_named(quirk, quirk_override, uint, 0444);
+static int quirk_override = -1;
+module_param_named(quirk, quirk_override, int, 0444);
 MODULE_PARM_DESC(quirk, "Board-specific quirk override");
 
 static void log_quirks(struct device *dev)
@@ -1254,7 +1254,7 @@ static int snd_byt_rt5640_mc_probe(struct platform_device *pdev)
 	dmi_id = dmi_first_match(byt_rt5640_quirk_table);
 	if (dmi_id)
 		byt_rt5640_quirk = (unsigned long)dmi_id->driver_data;
-	if (quirk_override) {
+	if (quirk_override != -1) {
 		dev_info(&pdev->dev, "Overriding quirk 0x%x => 0x%x\n",
 			 (unsigned int)byt_rt5640_quirk, quirk_override);
 		byt_rt5640_quirk = quirk_override;
diff --git a/sound/soc/intel/boards/bytcr_rt5651.c b/sound/soc/intel/boards/bytcr_rt5651.c
index b744add01d12..ca657c3e5726 100644
--- a/sound/soc/intel/boards/bytcr_rt5651.c
+++ b/sound/soc/intel/boards/bytcr_rt5651.c
@@ -79,14 +79,15 @@ enum {
 #define BYT_RT5651_SSP0_AIF2		BIT(21)
 #define BYT_RT5651_HP_LR_SWAPPED	BIT(22)
 #define BYT_RT5651_MONO_SPEAKER		BIT(23)
+#define BYT_RT5651_JD_NOT_INV		BIT(24)
 
 #define BYT_RT5651_DEFAULT_QUIRKS	(BYT_RT5651_MCLK_EN | \
 					 BYT_RT5651_JD1_1   | \
 					 BYT_RT5651_OVCD_TH_2000UA | \
 					 BYT_RT5651_OVCD_SF_0P75)
 
-/* jack-detect-source + dmic-en + ovcd-th + -sf + terminating empty entry */
-#define MAX_NO_PROPS 5
+/* jack-detect-source + inv + dmic-en + ovcd-th + -sf + terminating entry */
+#define MAX_NO_PROPS 6
 
 struct byt_rt5651_private {
 	struct clk *mclk;
@@ -101,8 +102,8 @@ static const struct acpi_gpio_mapping *byt_rt5651_gpios;
 static unsigned long byt_rt5651_quirk = BYT_RT5651_DEFAULT_QUIRKS |
 					BYT_RT5651_IN2_MAP;
 
-static unsigned int quirk_override;
-module_param_named(quirk, quirk_override, uint, 0444);
+static int quirk_override = -1;
+module_param_named(quirk, quirk_override, int, 0444);
 MODULE_PARM_DESC(quirk, "Board-specific quirk override");
 
 static void log_quirks(struct device *dev)
@@ -137,6 +138,8 @@ static void log_quirks(struct device *dev)
 		dev_info(dev, "quirk SSP0_AIF2 enabled\n");
 	if (byt_rt5651_quirk & BYT_RT5651_MONO_SPEAKER)
 		dev_info(dev, "quirk MONO_SPEAKER enabled\n");
+	if (byt_rt5651_quirk & BYT_RT5651_JD_NOT_INV)
+		dev_info(dev, "quirk JD_NOT_INV enabled\n");
 }
 
 #define BYT_CODEC_DAI1	"rt5651-aif1"
@@ -415,6 +418,18 @@ static const struct dmi_system_id byt_rt5651_quirk_table[] = {
 					BYT_RT5651_MONO_SPEAKER),
 	},
 	{
+		/* Complet Electro Serv MY8307 */
+		.callback = byt_rt5651_quirk_cb,
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Complet Electro Serv"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "MY8307"),
+		},
+		.driver_data = (void *)(BYT_RT5651_DEFAULT_QUIRKS |
+					BYT_RT5651_IN2_MAP |
+					BYT_RT5651_MONO_SPEAKER |
+					BYT_RT5651_JD_NOT_INV),
+	},
+	{
 		/* I.T.Works TW701, Ployer Momo7w and Trekstor ST70416-6
 		 * (these all use the same mainboard) */
 		.callback = byt_rt5651_quirk_cb,
@@ -525,6 +540,9 @@ static int byt_rt5651_add_codec_device_props(struct device *i2c_dev)
 	if (byt_rt5651_quirk & BYT_RT5651_DMIC_EN)
 		props[cnt++] = PROPERTY_ENTRY_BOOL("realtek,dmic-en");
 
+	if (byt_rt5651_quirk & BYT_RT5651_JD_NOT_INV)
+		props[cnt++] = PROPERTY_ENTRY_BOOL("realtek,jack-detect-not-inverted");
+
 	return device_add_properties(i2c_dev, props);
 }
 
@@ -969,7 +987,7 @@ static int snd_byt_rt5651_mc_probe(struct platform_device *pdev)
 	/* check quirks before creating card */
 	dmi_check_system(byt_rt5651_quirk_table);
 
-	if (quirk_override) {
+	if (quirk_override != -1) {
 		dev_info(&pdev->dev, "Overriding quirk 0x%x => 0x%x\n",
 			 (unsigned int)byt_rt5651_quirk, quirk_override);
 		byt_rt5651_quirk = quirk_override;
diff --git a/sound/soc/intel/boards/kbl_da7219_max98357a.c b/sound/soc/intel/boards/kbl_da7219_max98357a.c
index 38f6ab74709d..07491a0f8fb8 100644
--- a/sound/soc/intel/boards/kbl_da7219_max98357a.c
+++ b/sound/soc/intel/boards/kbl_da7219_max98357a.c
@@ -188,7 +188,7 @@ static int kabylake_da7219_codec_init(struct snd_soc_pcm_runtime *rtd)
 
 	jack = &ctx->kabylake_headset;
 
-	snd_jack_set_key(jack->jack, SND_JACK_BTN_0, KEY_MEDIA);
+	snd_jack_set_key(jack->jack, SND_JACK_BTN_0, KEY_PLAYPAUSE);
 	snd_jack_set_key(jack->jack, SND_JACK_BTN_1, KEY_VOLUMEUP);
 	snd_jack_set_key(jack->jack, SND_JACK_BTN_2, KEY_VOLUMEDOWN);
 	snd_jack_set_key(jack->jack, SND_JACK_BTN_3, KEY_VOICECOMMAND);
diff --git a/sound/soc/intel/boards/kbl_da7219_max98927.c b/sound/soc/intel/boards/kbl_da7219_max98927.c
index 2768a572d065..f72a7bf028d7 100644
--- a/sound/soc/intel/boards/kbl_da7219_max98927.c
+++ b/sound/soc/intel/boards/kbl_da7219_max98927.c
@@ -52,7 +52,6 @@ struct kbl_codec_private {
 
 enum {
 	KBL_DPCM_AUDIO_PB = 0,
-	KBL_DPCM_AUDIO_CP,
 	KBL_DPCM_AUDIO_ECHO_REF_CP,
 	KBL_DPCM_AUDIO_REF_CP,
 	KBL_DPCM_AUDIO_DMIC_CP,
@@ -60,6 +59,7 @@ enum {
 	KBL_DPCM_AUDIO_HDMI2_PB,
 	KBL_DPCM_AUDIO_HDMI3_PB,
 	KBL_DPCM_AUDIO_HS_PB,
+	KBL_DPCM_AUDIO_CP,
 };
 
 static int platform_clock_control(struct snd_soc_dapm_widget *w,
@@ -311,6 +311,12 @@ static int kabylake_da7219_codec_init(struct snd_soc_pcm_runtime *rtd)
 
 	da7219_aad_jack_det(component, &ctx->kabylake_headset);
 
+	return 0;
+}
+
+static int kabylake_dmic_init(struct snd_soc_pcm_runtime *rtd)
+{
+	int ret;
 	ret = snd_soc_dapm_ignore_suspend(&rtd->card->dapm, "SoC DMIC");
 	if (ret)
 		dev_err(rtd->dev, "SoC DMIC - Ignore suspend failed %d\n", ret);
@@ -581,20 +587,6 @@ static struct snd_soc_dai_link kabylake_dais[] = {
 		.dpcm_playback = 1,
 		.ops = &kabylake_da7219_fe_ops,
 	},
-	[KBL_DPCM_AUDIO_CP] = {
-		.name = "Kbl Audio Capture Port",
-		.stream_name = "Audio Record",
-		.cpu_dai_name = "System Pin",
-		.platform_name = "0000:00:1f.3",
-		.dynamic = 1,
-		.codec_name = "snd-soc-dummy",
-		.codec_dai_name = "snd-soc-dummy-dai",
-		.nonatomic = 1,
-		.trigger = {
-			SND_SOC_DPCM_TRIGGER_POST, SND_SOC_DPCM_TRIGGER_POST},
-		.dpcm_capture = 1,
-		.ops = &kabylake_da7219_fe_ops,
-	},
 	[KBL_DPCM_AUDIO_ECHO_REF_CP] = {
 		.name = "Kbl Audio Echo Reference cap",
 		.stream_name = "Echoreference Capture",
@@ -690,6 +682,20 @@ static struct snd_soc_dai_link kabylake_dais[] = {
 		.ops = &kabylake_da7219_fe_ops,
 
 	},
+	[KBL_DPCM_AUDIO_CP] = {
+		.name = "Kbl Audio Capture Port",
+		.stream_name = "Audio Record",
+		.cpu_dai_name = "System Pin",
+		.platform_name = "0000:00:1f.3",
+		.dynamic = 1,
+		.codec_name = "snd-soc-dummy",
+		.codec_dai_name = "snd-soc-dummy-dai",
+		.nonatomic = 1,
+		.trigger = {
+			SND_SOC_DPCM_TRIGGER_POST, SND_SOC_DPCM_TRIGGER_POST},
+		.dpcm_capture = 1,
+		.ops = &kabylake_da7219_fe_ops,
+	},
 
 	/* Back End DAI links */
 	{
@@ -733,6 +739,7 @@ static struct snd_soc_dai_link kabylake_dais[] = {
 		.cpu_dai_name = "DMIC01 Pin",
 		.codec_name = "dmic-codec",
 		.codec_dai_name = "dmic-hifi",
+		.init = kabylake_dmic_init,
 		.platform_name = "0000:00:1f.3",
 		.be_hw_params_fixup = kabylake_dmic_fixup,
 		.ignore_suspend = 1,
@@ -792,20 +799,6 @@ static struct snd_soc_dai_link kabylake_max98_927_373_dais[] = {
 		.dpcm_playback = 1,
 		.ops = &kabylake_da7219_fe_ops,
 	},
-	[KBL_DPCM_AUDIO_CP] = {
-		.name = "Kbl Audio Capture Port",
-		.stream_name = "Audio Record",
-		.cpu_dai_name = "System Pin",
-		.platform_name = "0000:00:1f.3",
-		.dynamic = 1,
-		.codec_name = "snd-soc-dummy",
-		.codec_dai_name = "snd-soc-dummy-dai",
-		.nonatomic = 1,
-		.trigger = {
-			SND_SOC_DPCM_TRIGGER_POST, SND_SOC_DPCM_TRIGGER_POST},
-		.dpcm_capture = 1,
-		.ops = &kabylake_da7219_fe_ops,
-	},
 	[KBL_DPCM_AUDIO_ECHO_REF_CP] = {
 		.name = "Kbl Audio Echo Reference cap",
 		.stream_name = "Echoreference Capture",
@@ -911,6 +904,7 @@ static struct snd_soc_dai_link kabylake_max98_927_373_dais[] = {
 		.cpu_dai_name = "DMIC01 Pin",
 		.codec_name = "dmic-codec",
 		.codec_dai_name = "dmic-hifi",
+		.init = kabylake_dmic_init,
 		.platform_name = "0000:00:1f.3",
 		.be_hw_params_fixup = kabylake_dmic_fixup,
 		.ignore_suspend = 1,
diff --git a/sound/soc/intel/boards/skl_hda_dsp_common.c b/sound/soc/intel/boards/skl_hda_dsp_common.c
index 3fdbf239da74..8b68f41a5b88 100644
--- a/sound/soc/intel/boards/skl_hda_dsp_common.c
+++ b/sound/soc/intel/boards/skl_hda_dsp_common.c
@@ -78,7 +78,6 @@ struct snd_soc_dai_link skl_hda_be_dai_links[HDA_DSP_MAX_BE_DAI_LINKS] = {
 		.platform_name = "0000:00:1f.3",
 		.dpcm_playback = 1,
 		.dpcm_capture = 1,
-		.init = NULL,
 		.no_pcm = 1,
 	},
 	{
@@ -90,7 +89,26 @@ struct snd_soc_dai_link skl_hda_be_dai_links[HDA_DSP_MAX_BE_DAI_LINKS] = {
 		.platform_name = "0000:00:1f.3",
 		.dpcm_playback = 1,
 		.dpcm_capture = 1,
-		.init = NULL,
+		.no_pcm = 1,
+	},
+	{
+		.name = "dmic01",
+		.id = 6,
+		.cpu_dai_name = "DMIC01 Pin",
+		.codec_name = "dmic-codec",
+		.codec_dai_name = "dmic-hifi",
+		.platform_name = "0000:00:1f.3",
+		.dpcm_capture = 1,
+		.no_pcm = 1,
+	},
+	{
+		.name = "dmic16k",
+		.id = 7,
+		.cpu_dai_name = "DMIC16k Pin",
+		.codec_name = "dmic-codec",
+		.codec_dai_name = "dmic-hifi",
+		.platform_name = "0000:00:1f.3",
+		.dpcm_capture = 1,
 		.no_pcm = 1,
 	},
 };
diff --git a/sound/soc/intel/boards/skl_hda_dsp_common.h b/sound/soc/intel/boards/skl_hda_dsp_common.h
index 87c50aff56cd..daa582e513b2 100644
--- a/sound/soc/intel/boards/skl_hda_dsp_common.h
+++ b/sound/soc/intel/boards/skl_hda_dsp_common.h
@@ -15,7 +15,7 @@
 #include <sound/core.h>
 #include <sound/jack.h>
 
-#define HDA_DSP_MAX_BE_DAI_LINKS 5
+#define HDA_DSP_MAX_BE_DAI_LINKS 7
 
 struct skl_hda_hdmi_pcm {
 	struct list_head head;
diff --git a/sound/soc/intel/boards/skl_hda_dsp_generic.c b/sound/soc/intel/boards/skl_hda_dsp_generic.c
index b9a21e64ead2..fc52d3a32354 100644
--- a/sound/soc/intel/boards/skl_hda_dsp_generic.c
+++ b/sound/soc/intel/boards/skl_hda_dsp_generic.c
@@ -97,6 +97,9 @@ static struct snd_soc_card hda_soc_card = {
 };
 
 #define IDISP_DAI_COUNT		3
+#define HDAC_DAI_COUNT		2
+#define DMIC_DAI_COUNT		2
+
 /* there are two routes per iDisp output */
 #define IDISP_ROUTE_COUNT	(IDISP_DAI_COUNT * 2)
 #define IDISP_CODEC_MASK	0x4
@@ -112,11 +115,23 @@ static int skl_hda_fill_card_info(struct snd_soc_acpi_mach_params *mach_params)
 	codec_count = hweight_long(codec_mask);
 
 	if (codec_count == 1 && codec_mask & IDISP_CODEC_MASK) {
-		num_links = IDISP_DAI_COUNT;
+		num_links = IDISP_DAI_COUNT + DMIC_DAI_COUNT;
 		num_route = IDISP_ROUTE_COUNT;
+
+		/*
+		 * rearrange the dai link array and make the
+		 * dmic dai links follow idsp dai links for only
+		 * num_links of dai links need to be registered
+		 * to ASoC.
+		 */
+		for (i = 0; i < DMIC_DAI_COUNT; i++) {
+			skl_hda_be_dai_links[IDISP_DAI_COUNT + i] =
+				skl_hda_be_dai_links[IDISP_DAI_COUNT +
+					HDAC_DAI_COUNT + i];
+		}
 	} else if (codec_count == 2 && codec_mask & IDISP_CODEC_MASK) {
 		num_links = ARRAY_SIZE(skl_hda_be_dai_links);
-		num_route = ARRAY_SIZE(skl_hda_map),
+		num_route = ARRAY_SIZE(skl_hda_map);
 		card->dapm_widgets = skl_hda_widgets;
 		card->num_dapm_widgets = ARRAY_SIZE(skl_hda_widgets);
 	} else {
diff --git a/sound/soc/intel/boards/sof_rt5682.c b/sound/soc/intel/boards/sof_rt5682.c
new file mode 100644
index 000000000000..f28fb98cc306
--- /dev/null
+++ b/sound/soc/intel/boards/sof_rt5682.c
@@ -0,0 +1,563 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright(c) 2019 Intel Corporation.
+
+/*
+ * Intel SOF Machine Driver with Realtek rt5682 Codec
+ * and speaker codec MAX98357A
+ */
+#include <linux/i2c.h>
+#include <linux/input.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/dmi.h>
+#include <asm/cpu_device_id.h>
+#include <asm/intel-family.h>
+#include <sound/core.h>
+#include <sound/jack.h>
+#include <sound/pcm.h>
+#include <sound/pcm_params.h>
+#include <sound/soc.h>
+#include <sound/rt5682.h>
+#include <sound/soc-acpi.h>
+#include "../../codecs/rt5682.h"
+#include "../../codecs/hdac_hdmi.h"
+
+#define NAME_SIZE 32
+
+#define SOF_RT5682_SSP_CODEC(quirk)		((quirk) & GENMASK(2, 0))
+#define SOF_RT5682_SSP_CODEC_MASK			(GENMASK(2, 0))
+#define SOF_RT5682_MCLK_EN			BIT(3)
+#define SOF_RT5682_MCLK_24MHZ			BIT(4)
+#define SOF_SPEAKER_AMP_PRESENT		BIT(5)
+#define SOF_RT5682_SSP_AMP(quirk)		((quirk) & GENMASK(8, 6))
+#define SOF_RT5682_SSP_AMP_MASK			(GENMASK(8, 6))
+#define SOF_RT5682_SSP_AMP_SHIFT		6
+
+/* Default: MCLK on, MCLK 19.2M, SSP0  */
+static unsigned long sof_rt5682_quirk = SOF_RT5682_MCLK_EN |
+					SOF_RT5682_SSP_CODEC(0);
+
+static int is_legacy_cpu;
+
+static struct snd_soc_jack sof_hdmi[3];
+
+struct sof_hdmi_pcm {
+	struct list_head head;
+	struct snd_soc_dai *codec_dai;
+	int device;
+};
+
+struct sof_card_private {
+	struct snd_soc_jack sof_headset;
+	struct list_head hdmi_pcm_list;
+};
+
+static int sof_rt5682_quirk_cb(const struct dmi_system_id *id)
+{
+	sof_rt5682_quirk = (unsigned long)id->driver_data;
+	return 1;
+}
+
+static const struct dmi_system_id sof_rt5682_quirk_table[] = {
+	{
+		.callback = sof_rt5682_quirk_cb,
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Intel Corporation"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "WhiskeyLake Client"),
+		},
+		.driver_data = (void *)(SOF_RT5682_MCLK_EN |
+					SOF_RT5682_MCLK_24MHZ |
+					SOF_RT5682_SSP_CODEC(1)),
+	},
+	{
+		.callback = sof_rt5682_quirk_cb,
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Google"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Hatch"),
+		},
+		.driver_data = (void *)(SOF_RT5682_MCLK_EN |
+					SOF_RT5682_MCLK_24MHZ |
+					SOF_RT5682_SSP_CODEC(0) |
+					SOF_SPEAKER_AMP_PRESENT |
+					SOF_RT5682_SSP_AMP(1)),
+	},
+	{
+		.callback = sof_rt5682_quirk_cb,
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Intel Corporation"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Ice Lake Client"),
+		},
+		.driver_data = (void *)(SOF_RT5682_MCLK_EN |
+					SOF_RT5682_SSP_CODEC(0)),
+	},
+	{}
+};
+
+static int sof_hdmi_init(struct snd_soc_pcm_runtime *rtd)
+{
+	struct sof_card_private *ctx = snd_soc_card_get_drvdata(rtd->card);
+	struct snd_soc_dai *dai = rtd->codec_dai;
+	struct sof_hdmi_pcm *pcm;
+
+	pcm = devm_kzalloc(rtd->card->dev, sizeof(*pcm), GFP_KERNEL);
+	if (!pcm)
+		return -ENOMEM;
+
+	/* dai_link id is 1:1 mapped to the PCM device */
+	pcm->device = rtd->dai_link->id;
+	pcm->codec_dai = dai;
+
+	list_add_tail(&pcm->head, &ctx->hdmi_pcm_list);
+
+	return 0;
+}
+
+static int sof_rt5682_codec_init(struct snd_soc_pcm_runtime *rtd)
+{
+	struct sof_card_private *ctx = snd_soc_card_get_drvdata(rtd->card);
+	struct snd_soc_component *component = rtd->codec_dai->component;
+	struct snd_soc_jack *jack;
+	int ret;
+
+	/* need to enable ASRC function for 24MHz mclk rate */
+	if ((sof_rt5682_quirk & SOF_RT5682_MCLK_EN) &&
+	    (sof_rt5682_quirk & SOF_RT5682_MCLK_24MHZ)) {
+		rt5682_sel_asrc_clk_src(component, RT5682_DA_STEREO1_FILTER |
+					RT5682_AD_STEREO1_FILTER,
+					RT5682_CLK_SEL_I2S1_ASRC);
+	}
+
+	/*
+	 * Headset buttons map to the google Reference headset.
+	 * These can be configured by userspace.
+	 */
+	ret = snd_soc_card_jack_new(rtd->card, "Headset Jack",
+				    SND_JACK_HEADSET | SND_JACK_BTN_0 |
+				    SND_JACK_BTN_1 | SND_JACK_BTN_2 |
+				    SND_JACK_BTN_3,
+				    &ctx->sof_headset, NULL, 0);
+	if (ret) {
+		dev_err(rtd->dev, "Headset Jack creation failed: %d\n", ret);
+		return ret;
+	}
+
+	jack = &ctx->sof_headset;
+
+	snd_jack_set_key(jack->jack, SND_JACK_BTN_0, KEY_PLAYPAUSE);
+	snd_jack_set_key(jack->jack, SND_JACK_BTN_1, KEY_VOLUMEUP);
+	snd_jack_set_key(jack->jack, SND_JACK_BTN_2, KEY_VOLUMEDOWN);
+	snd_jack_set_key(jack->jack, SND_JACK_BTN_3, KEY_VOICECOMMAND);
+	ret = snd_soc_component_set_jack(component, jack, NULL);
+
+	if (ret) {
+		dev_err(rtd->dev, "Headset Jack call-back failed: %d\n", ret);
+		return ret;
+	}
+
+	return ret;
+};
+
+static int sof_rt5682_hw_params(struct snd_pcm_substream *substream,
+				struct snd_pcm_hw_params *params)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct snd_soc_dai *codec_dai = rtd->codec_dai;
+	int clk_id, clk_freq, pll_out, ret;
+
+	if (sof_rt5682_quirk & SOF_RT5682_MCLK_EN) {
+		clk_id = RT5682_PLL1_S_MCLK;
+		if (sof_rt5682_quirk & SOF_RT5682_MCLK_24MHZ)
+			clk_freq = 24000000;
+		else
+			clk_freq = 19200000;
+	} else {
+		clk_id = RT5682_PLL1_S_BCLK1;
+		clk_freq = params_rate(params) * 50;
+	}
+
+	pll_out = params_rate(params) * 512;
+
+	ret = snd_soc_dai_set_pll(codec_dai, 0, clk_id, clk_freq, pll_out);
+	if (ret < 0)
+		dev_err(rtd->dev, "snd_soc_dai_set_pll err = %d\n", ret);
+
+	/* Configure sysclk for codec */
+	ret = snd_soc_dai_set_sysclk(codec_dai, RT5682_SCLK_S_PLL1,
+				     pll_out, SND_SOC_CLOCK_IN);
+	if (ret < 0)
+		dev_err(rtd->dev, "snd_soc_dai_set_sysclk err = %d\n", ret);
+
+	/*
+	 * slot_width should equal or large than data length, set them
+	 * be the same
+	 */
+	ret = snd_soc_dai_set_tdm_slot(codec_dai, 0x0, 0x0, 2,
+				       params_width(params));
+	if (ret < 0) {
+		dev_err(rtd->dev, "set TDM slot err:%d\n", ret);
+		return ret;
+	}
+
+	return ret;
+}
+
+static struct snd_soc_ops sof_rt5682_ops = {
+	.hw_params = sof_rt5682_hw_params,
+};
+
+static struct snd_soc_dai_link_component platform_component[] = {
+	{
+		/* name might be overridden during probe */
+		.name = "0000:00:1f.3"
+	}
+};
+
+static int sof_card_late_probe(struct snd_soc_card *card)
+{
+	struct sof_card_private *ctx = snd_soc_card_get_drvdata(card);
+	struct snd_soc_component *component = NULL;
+	char jack_name[NAME_SIZE];
+	struct sof_hdmi_pcm *pcm;
+	int err = 0;
+	int i = 0;
+
+	/* HDMI is not supported by SOF on Baytrail/CherryTrail */
+	if (is_legacy_cpu)
+		return 0;
+
+	list_for_each_entry(pcm, &ctx->hdmi_pcm_list, head) {
+		component = pcm->codec_dai->component;
+		snprintf(jack_name, sizeof(jack_name),
+			 "HDMI/DP, pcm=%d Jack", pcm->device);
+		err = snd_soc_card_jack_new(card, jack_name,
+					    SND_JACK_AVOUT, &sof_hdmi[i],
+					    NULL, 0);
+
+		if (err)
+			return err;
+
+		err = hdac_hdmi_jack_init(pcm->codec_dai, pcm->device,
+					  &sof_hdmi[i]);
+		if (err < 0)
+			return err;
+
+		i++;
+	}
+	if (!component)
+		return -EINVAL;
+
+	return hdac_hdmi_jack_port_init(component, &card->dapm);
+}
+
+static const struct snd_kcontrol_new sof_controls[] = {
+	SOC_DAPM_PIN_SWITCH("Headphone Jack"),
+	SOC_DAPM_PIN_SWITCH("Headset Mic"),
+	SOC_DAPM_PIN_SWITCH("Spk"),
+};
+
+static const struct snd_soc_dapm_widget sof_widgets[] = {
+	SND_SOC_DAPM_HP("Headphone Jack", NULL),
+	SND_SOC_DAPM_MIC("Headset Mic", NULL),
+	SND_SOC_DAPM_SPK("Spk", NULL),
+};
+
+static const struct snd_soc_dapm_route sof_map[] = {
+	/* HP jack connectors - unknown if we have jack detection */
+	{ "Headphone Jack", NULL, "HPOL" },
+	{ "Headphone Jack", NULL, "HPOR" },
+
+	/* other jacks */
+	{ "IN1P", NULL, "Headset Mic" },
+
+};
+
+static const struct snd_soc_dapm_route speaker_map[] = {
+	/* speaker */
+	{ "Spk", NULL, "Speaker" },
+};
+
+static int speaker_codec_init(struct snd_soc_pcm_runtime *rtd)
+{
+	struct snd_soc_card *card = rtd->card;
+	int ret;
+
+	ret = snd_soc_dapm_add_routes(&card->dapm, speaker_map,
+				      ARRAY_SIZE(speaker_map));
+
+	if (ret)
+		dev_err(rtd->dev, "Speaker map addition failed: %d\n", ret);
+	return ret;
+}
+
+/* sof audio machine driver for rt5682 codec */
+static struct snd_soc_card sof_audio_card_rt5682 = {
+	.name = "sof_rt5682",
+	.owner = THIS_MODULE,
+	.controls = sof_controls,
+	.num_controls = ARRAY_SIZE(sof_controls),
+	.dapm_widgets = sof_widgets,
+	.num_dapm_widgets = ARRAY_SIZE(sof_widgets),
+	.dapm_routes = sof_map,
+	.num_dapm_routes = ARRAY_SIZE(sof_map),
+	.fully_routed = true,
+	.late_probe = sof_card_late_probe,
+};
+
+static const struct x86_cpu_id legacy_cpi_ids[] = {
+	{ X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_SILVERMONT }, /* Baytrail */
+	{ X86_VENDOR_INTEL, 6, INTEL_FAM6_ATOM_AIRMONT }, /* Cherrytrail */
+	{}
+};
+
+static struct snd_soc_dai_link_component rt5682_component[] = {
+	{
+		.name = "i2c-10EC5682:00",
+		.dai_name = "rt5682-aif1",
+	}
+};
+
+static struct snd_soc_dai_link_component dmic_component[] = {
+	{
+		.name = "dmic-codec",
+		.dai_name = "dmic-hifi",
+	}
+};
+
+static struct snd_soc_dai_link_component max98357a_component[] = {
+	{
+		.name = "MX98357A:00",
+		.dai_name = "HiFi",
+	}
+};
+
+static struct snd_soc_dai_link *sof_card_dai_links_create(struct device *dev,
+							  int ssp_codec,
+							  int ssp_amp,
+							  int dmic_num,
+							  int hdmi_num)
+{
+	struct snd_soc_dai_link_component *idisp_components;
+	struct snd_soc_dai_link *links;
+	int i, id = 0;
+
+	links = devm_kzalloc(dev, sizeof(struct snd_soc_dai_link) *
+			     sof_audio_card_rt5682.num_links, GFP_KERNEL);
+	if (!links)
+		goto devm_err;
+
+	/* codec SSP */
+	links[id].name = devm_kasprintf(dev, GFP_KERNEL,
+					"SSP%d-Codec", ssp_codec);
+	if (!links[id].name)
+		goto devm_err;
+
+	links[id].id = id;
+	links[id].codecs = rt5682_component;
+	links[id].num_codecs = ARRAY_SIZE(rt5682_component);
+	links[id].platforms = platform_component;
+	links[id].num_platforms = ARRAY_SIZE(platform_component);
+	links[id].init = sof_rt5682_codec_init;
+	links[id].ops = &sof_rt5682_ops;
+	links[id].nonatomic = true;
+	links[id].dpcm_playback = 1;
+	links[id].dpcm_capture = 1;
+	links[id].no_pcm = 1;
+	if (is_legacy_cpu) {
+		links[id].cpu_dai_name = devm_kasprintf(dev, GFP_KERNEL,
+							"ssp%d-port",
+							ssp_codec);
+		if (!links[id].cpu_dai_name)
+			goto devm_err;
+	} else {
+		/*
+		 * Currently, On SKL+ platforms MCLK will be turned off in sof
+		 * runtime suspended, and it will go into runtime suspended
+		 * right after playback is stop. However, rt5682 will output
+		 * static noise if sysclk turns off during playback. Set
+		 * ignore_pmdown_time to power down rt5682 immediately and
+		 * avoid the noise.
+		 * It can be removed once we can control MCLK by driver.
+		 */
+		links[id].ignore_pmdown_time = 1;
+		links[id].cpu_dai_name = devm_kasprintf(dev, GFP_KERNEL,
+							"SSP%d Pin",
+							ssp_codec);
+		if (!links[id].cpu_dai_name)
+			goto devm_err;
+	}
+	id++;
+
+	/* dmic */
+	for (i = 1; i <= dmic_num; i++) {
+		links[id].name = devm_kasprintf(dev, GFP_KERNEL,
+						"dmic%02d", i);
+		if (!links[id].name)
+			goto devm_err;
+
+		links[id].id = id;
+		links[id].cpu_dai_name = devm_kasprintf(dev, GFP_KERNEL,
+							"DMIC%02d Pin", i);
+		if (!links[id].cpu_dai_name)
+			goto devm_err;
+
+		links[id].codecs = dmic_component;
+		links[id].num_codecs = ARRAY_SIZE(dmic_component);
+		links[id].platforms = platform_component;
+		links[id].num_platforms = ARRAY_SIZE(platform_component);
+		links[id].ignore_suspend = 1;
+		links[id].dpcm_capture = 1;
+		links[id].no_pcm = 1;
+		id++;
+	}
+
+	/* HDMI */
+	if (hdmi_num > 0) {
+		idisp_components = devm_kzalloc(dev,
+				   sizeof(struct snd_soc_dai_link_component) *
+				   hdmi_num, GFP_KERNEL);
+		if (!idisp_components)
+			goto devm_err;
+	}
+	for (i = 1; i <= hdmi_num; i++) {
+		links[id].name = devm_kasprintf(dev, GFP_KERNEL,
+						"iDisp%d", i);
+		if (!links[id].name)
+			goto devm_err;
+
+		links[id].id = id;
+		links[id].cpu_dai_name = devm_kasprintf(dev, GFP_KERNEL,
+							"iDisp%d Pin", i);
+		if (!links[id].cpu_dai_name)
+			goto devm_err;
+
+		idisp_components[i - 1].name = "ehdaudio0D2";
+		idisp_components[i - 1].dai_name = devm_kasprintf(dev,
+								  GFP_KERNEL,
+								  "intel-hdmi-hifi%d",
+								  i);
+		if (!idisp_components[i - 1].dai_name)
+			goto devm_err;
+
+		links[id].codecs = &idisp_components[i - 1];
+		links[id].num_codecs = 1;
+		links[id].platforms = platform_component;
+		links[id].num_platforms = ARRAY_SIZE(platform_component);
+		links[id].init = sof_hdmi_init;
+		links[id].dpcm_playback = 1;
+		links[id].no_pcm = 1;
+		id++;
+	}
+
+	/* speaker amp */
+	if (sof_rt5682_quirk & SOF_SPEAKER_AMP_PRESENT) {
+		links[id].name = devm_kasprintf(dev, GFP_KERNEL,
+						"SSP%d-Codec", ssp_amp);
+		if (!links[id].name)
+			goto devm_err;
+
+		links[id].id = id;
+		links[id].codecs = max98357a_component;
+		links[id].num_codecs = ARRAY_SIZE(max98357a_component);
+		links[id].platforms = platform_component;
+		links[id].num_platforms = ARRAY_SIZE(platform_component);
+		links[id].init = speaker_codec_init,
+		links[id].nonatomic = true;
+		links[id].dpcm_playback = 1;
+		links[id].no_pcm = 1;
+		if (is_legacy_cpu) {
+			links[id].cpu_dai_name = devm_kasprintf(dev, GFP_KERNEL,
+								"ssp%d-port",
+								ssp_amp);
+			if (!links[id].cpu_dai_name)
+				goto devm_err;
+
+		} else {
+			links[id].cpu_dai_name = devm_kasprintf(dev, GFP_KERNEL,
+								"SSP%d Pin",
+								ssp_amp);
+			if (!links[id].cpu_dai_name)
+				goto devm_err;
+		}
+	}
+
+	return links;
+devm_err:
+	return NULL;
+}
+
+static int sof_audio_probe(struct platform_device *pdev)
+{
+	struct snd_soc_dai_link *dai_links;
+	struct snd_soc_acpi_mach *mach;
+	struct sof_card_private *ctx;
+	int dmic_num, hdmi_num;
+	int ret, ssp_amp, ssp_codec;
+
+	ctx = devm_kzalloc(&pdev->dev, sizeof(*ctx), GFP_ATOMIC);
+	if (!ctx)
+		return -ENOMEM;
+
+	if (x86_match_cpu(legacy_cpi_ids)) {
+		is_legacy_cpu = 1;
+		dmic_num = 0;
+		hdmi_num = 0;
+		/* default quirk for legacy cpu */
+		sof_rt5682_quirk = SOF_RT5682_SSP_CODEC(2);
+	} else {
+		dmic_num = 1;
+		hdmi_num = 3;
+	}
+
+	dmi_check_system(sof_rt5682_quirk_table);
+
+	dev_dbg(&pdev->dev, "sof_rt5682_quirk = %lx\n", sof_rt5682_quirk);
+
+	ssp_amp = (sof_rt5682_quirk & SOF_RT5682_SSP_AMP_MASK) >>
+			SOF_RT5682_SSP_AMP_SHIFT;
+
+	ssp_codec = sof_rt5682_quirk & SOF_RT5682_SSP_CODEC_MASK;
+
+	/* compute number of dai links */
+	sof_audio_card_rt5682.num_links = 1 + dmic_num + hdmi_num;
+	if (sof_rt5682_quirk & SOF_SPEAKER_AMP_PRESENT)
+		sof_audio_card_rt5682.num_links++;
+
+	dai_links = sof_card_dai_links_create(&pdev->dev, ssp_codec, ssp_amp,
+					      dmic_num, hdmi_num);
+	if (!dai_links)
+		return -ENOMEM;
+
+	sof_audio_card_rt5682.dai_link = dai_links;
+
+	INIT_LIST_HEAD(&ctx->hdmi_pcm_list);
+
+	sof_audio_card_rt5682.dev = &pdev->dev;
+	mach = (&pdev->dev)->platform_data;
+
+	/* set platform name for each dailink */
+	ret = snd_soc_fixup_dai_links_platform_name(&sof_audio_card_rt5682,
+						    mach->mach_params.platform);
+	if (ret)
+		return ret;
+
+	snd_soc_card_set_drvdata(&sof_audio_card_rt5682, ctx);
+
+	return devm_snd_soc_register_card(&pdev->dev,
+					  &sof_audio_card_rt5682);
+}
+
+static struct platform_driver sof_audio = {
+	.probe = sof_audio_probe,
+	.driver = {
+		.name = "sof_rt5682",
+		.pm = &snd_soc_pm_ops,
+	},
+};
+module_platform_driver(sof_audio)
+
+/* Module information */
+MODULE_DESCRIPTION("SOF Audio Machine driver");
+MODULE_AUTHOR("Bard Liao <bard.liao@intel.com>");
+MODULE_AUTHOR("Sathya Prakash M R <sathya.prakash.m.r@intel.com>");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:sof_rt5682");
diff --git a/sound/soc/intel/common/soc-acpi-intel-byt-match.c b/sound/soc/intel/common/soc-acpi-intel-byt-match.c
index fe812a909db4..0cfab247876a 100644
--- a/sound/soc/intel/common/soc-acpi-intel-byt-match.c
+++ b/sound/soc/intel/common/soc-acpi-intel-byt-match.c
@@ -185,6 +185,12 @@ struct snd_soc_acpi_mach  snd_soc_acpi_intel_baytrail_machines[] = {
 		.sof_fw_filename = "sof-byt.ri",
 		.sof_tplg_filename = "sof-byt-es8316.tplg",
 	},
+	{
+		.id = "10EC5682",
+		.drv_name = "sof_rt5682",
+		.sof_fw_filename = "sof-byt.ri",
+		.sof_tplg_filename = "sof-byt-rt5682.tplg",
+	},
 	/* some Baytrail platforms rely on RT5645, use CHT machine driver */
 	{
 		.id = "10EC5645",
diff --git a/sound/soc/intel/common/soc-acpi-intel-cht-match.c b/sound/soc/intel/common/soc-acpi-intel-cht-match.c
index deafd87cc764..ff9c31a39ad4 100644
--- a/sound/soc/intel/common/soc-acpi-intel-cht-match.c
+++ b/sound/soc/intel/common/soc-acpi-intel-cht-match.c
@@ -160,6 +160,12 @@ struct snd_soc_acpi_mach  snd_soc_acpi_intel_cherrytrail_machines[] = {
 		.sof_fw_filename = "sof-cht.ri",
 		.sof_tplg_filename = "sof-cht-rt5640.tplg",
 	},
+	{
+		.id = "10EC5682",
+		.drv_name = "sof_rt5682",
+		.sof_fw_filename = "sof-cht.ri",
+		.sof_tplg_filename = "sof-cht-rt5682.tplg",
+	},
 	/* some CHT-T platforms rely on RT5651, use Baytrail machine driver */
 	{
 		.id = "10EC5651",
diff --git a/sound/soc/intel/common/soc-acpi-intel-cnl-match.c b/sound/soc/intel/common/soc-acpi-intel-cnl-match.c
index a914dd238d0a..df7c52cad5c3 100644
--- a/sound/soc/intel/common/soc-acpi-intel-cnl-match.c
+++ b/sound/soc/intel/common/soc-acpi-intel-cnl-match.c
@@ -14,6 +14,11 @@ static struct skl_machine_pdata cnl_pdata = {
 	.use_tplg_pcm = true,
 };
 
+static struct snd_soc_acpi_codecs cml_codecs = {
+	.num_codecs = 1,
+	.codecs = {"10EC5682"}
+};
+
 struct snd_soc_acpi_mach snd_soc_acpi_intel_cnl_machines[] = {
 	{
 		.id = "INT34C2",
@@ -23,6 +28,20 @@ struct snd_soc_acpi_mach snd_soc_acpi_intel_cnl_machines[] = {
 		.sof_fw_filename = "sof-cnl.ri",
 		.sof_tplg_filename = "sof-cnl-rt274.tplg",
 	},
+	{
+		.id = "10EC5682",
+		.drv_name = "sof_rt5682",
+		.sof_fw_filename = "sof-cnl.ri",
+		.sof_tplg_filename = "sof-cml-rt5682.tplg",
+	},
+	{
+		.id = "MX98357A",
+		.drv_name = "sof_rt5682",
+		.quirk_data = &cml_codecs,
+		.sof_fw_filename = "sof-cnl.ri",
+		.sof_tplg_filename = "sof-cml-rt5682-max98357a.tplg",
+	},
+
 	{},
 };
 EXPORT_SYMBOL_GPL(snd_soc_acpi_intel_cnl_machines);
diff --git a/sound/soc/intel/common/soc-acpi-intel-glk-match.c b/sound/soc/intel/common/soc-acpi-intel-glk-match.c
index 3f2061475ae4..616eb09e78a0 100644
--- a/sound/soc/intel/common/soc-acpi-intel-glk-match.c
+++ b/sound/soc/intel/common/soc-acpi-intel-glk-match.c
@@ -31,6 +31,15 @@ struct snd_soc_acpi_mach snd_soc_acpi_intel_glk_machines[] = {
 		.sof_fw_filename = "sof-glk.ri",
 		.sof_tplg_filename = "sof-glk-da7219.tplg",
 	},
+	{
+		.id = "10EC5682",
+		.drv_name = "glk_rt5682_max98357a",
+		.fw_filename = "intel/dsp_fw_glk.bin",
+		.machine_quirk = snd_soc_acpi_codec_list,
+		.quirk_data = &glk_codecs,
+		.sof_fw_filename = "sof-glk.ri",
+		.sof_tplg_filename = "sof-glk-rt5682.tplg",
+	},
 	{},
 };
 EXPORT_SYMBOL_GPL(snd_soc_acpi_intel_glk_machines);
diff --git a/sound/soc/intel/common/soc-acpi-intel-icl-match.c b/sound/soc/intel/common/soc-acpi-intel-icl-match.c
index e5a6be5bc0ee..0b430b9b3673 100644
--- a/sound/soc/intel/common/soc-acpi-intel-icl-match.c
+++ b/sound/soc/intel/common/soc-acpi-intel-icl-match.c
@@ -23,6 +23,12 @@ struct snd_soc_acpi_mach snd_soc_acpi_intel_icl_machines[] = {
 		.sof_fw_filename = "sof-icl.ri",
 		.sof_tplg_filename = "sof-icl-rt274.tplg",
 	},
+	{
+		.id = "10EC5682",
+		.drv_name = "sof_rt5682",
+		.sof_fw_filename = "sof-icl.ri",
+		.sof_tplg_filename = "sof-icl-rt5682.tplg",
+	},
 	{},
 };
 EXPORT_SYMBOL_GPL(snd_soc_acpi_intel_icl_machines);
diff --git a/sound/soc/intel/common/sst-firmware.c b/sound/soc/intel/common/sst-firmware.c
index 1e067504b604..f830e59f93ea 100644
--- a/sound/soc/intel/common/sst-firmware.c
+++ b/sound/soc/intel/common/sst-firmware.c
@@ -1251,11 +1251,15 @@ struct sst_dsp *sst_dsp_new(struct device *dev,
 		goto irq_err;
 
 	err = sst_dma_new(sst);
-	if (err)
-		dev_warn(dev, "sst_dma_new failed %d\n", err);
+	if (err)  {
+		dev_err(dev, "sst_dma_new failed %d\n", err);
+		goto dma_err;
+	}
 
 	return sst;
 
+dma_err:
+	free_irq(sst->irq, sst);
 irq_err:
 	if (sst->ops->free)
 		sst->ops->free(sst);
diff --git a/sound/soc/intel/haswell/sst-haswell-ipc.c b/sound/soc/intel/haswell/sst-haswell-ipc.c
index 31fcdf12c67d..74acf9c65161 100644
--- a/sound/soc/intel/haswell/sst-haswell-ipc.c
+++ b/sound/soc/intel/haswell/sst-haswell-ipc.c
@@ -345,11 +345,6 @@ static inline u32 msg_get_stream_type(u32 msg)
 	return (msg & IPC_STR_TYPE_MASK) >>  IPC_STR_TYPE_SHIFT;
 }
 
-static inline u32 msg_get_stage_type(u32 msg)
-{
-	return (msg & IPC_STG_TYPE_MASK) >>  IPC_STG_TYPE_SHIFT;
-}
-
 static inline u32 msg_get_stream_id(u32 msg)
 {
 	return (msg & IPC_STR_ID_MASK) >>  IPC_STR_ID_SHIFT;
@@ -666,13 +661,12 @@ static int hsw_module_message(struct sst_hsw *hsw, u32 header)
 
 static int hsw_stream_message(struct sst_hsw *hsw, u32 header)
 {
-	u32 stream_msg, stream_id, stage_type;
+	u32 stream_msg, stream_id;
 	struct sst_hsw_stream *stream;
 	int handled = 0;
 
 	stream_msg = msg_get_stream_type(header);
 	stream_id = msg_get_stream_id(header);
-	stage_type = msg_get_stage_type(header);
 
 	stream = get_stream_by_id(hsw, stream_id);
 	if (stream == NULL)
diff --git a/sound/soc/jz4740/Kconfig b/sound/soc/jz4740/Kconfig
index 1a354a6b6e87..b3f9c41b4319 100644
--- a/sound/soc/jz4740/Kconfig
+++ b/sound/soc/jz4740/Kconfig
@@ -1,6 +1,6 @@
 config SND_JZ4740_SOC
 	tristate "SoC Audio for Ingenic JZ4740 SoC"
-	depends on MACH_JZ4740 || COMPILE_TEST
+	depends on MIPS || COMPILE_TEST
 	select SND_SOC_GENERIC_DMAENGINE_PCM
 	help
 	  Say Y or M if you want to add support for codecs attached to
diff --git a/sound/soc/mediatek/Kconfig b/sound/soc/mediatek/Kconfig
index b35410e4020e..f70b7109f2b6 100644
--- a/sound/soc/mediatek/Kconfig
+++ b/sound/soc/mediatek/Kconfig
@@ -116,6 +116,33 @@ config SND_SOC_MT8183
 	  Select Y if you have such device.
 	  If unsure select "N".
 
+config SND_SOC_MT8183_MT6358_TS3A227E_MAX98357A
+	tristate "ASoC Audio driver for MT8183 with MT6358 TS3A227E MAX98357A codec"
+	depends on I2C
+	depends on SND_SOC_MT8183
+	select SND_SOC_MT6358
+	select SND_SOC_MAX98357A
+	select SND_SOC_BT_SCO
+	select SND_SOC_TS3A227E
+	help
+	  This adds ASoC driver for Mediatek MT8183 boards
+	  with the MT6358 TS3A227E MAX98357A audio codec.
+	  Select Y if you have such device.
+	  If unsure select "N".
+
+config SND_SOC_MT8183_DA7219_MAX98357A
+	tristate "ASoC Audio driver for MT8183 with DA7219 MAX98357A codec"
+	depends on SND_SOC_MT8183
+	select SND_SOC_MT6358
+	select SND_SOC_MAX98357A
+	select SND_SOC_DA7219
+	select SND_SOC_BT_SCO
+	help
+	  This adds ASoC driver for Mediatek MT8183 boards
+	  with the DA7219 MAX98357A audio codec.
+	  Select Y if you have such device.
+	  If unsure select "N".
+
 config SND_SOC_MTK_BTCVSD
 	tristate "ALSA BT SCO CVSD/MSBC Driver"
 	help
diff --git a/sound/soc/mediatek/common/mtk-afe-fe-dai.c b/sound/soc/mediatek/common/mtk-afe-fe-dai.c
index cf4978be062f..fded11d14cde 100644
--- a/sound/soc/mediatek/common/mtk-afe-fe-dai.c
+++ b/sound/soc/mediatek/common/mtk-afe-fe-dai.c
@@ -18,11 +18,11 @@
 
 static int mtk_regmap_update_bits(struct regmap *map, int reg,
 			   unsigned int mask,
-			   unsigned int val)
+			   unsigned int val, int shift)
 {
-	if (reg < 0)
+	if (reg < 0 || WARN_ON_ONCE(shift < 0))
 		return 0;
-	return regmap_update_bits(map, reg, mask, val);
+	return regmap_update_bits(map, reg, mask << shift, val << shift);
 }
 
 static int mtk_regmap_write(struct regmap *map, int reg, unsigned int val)
@@ -49,8 +49,7 @@ int mtk_afe_fe_startup(struct snd_pcm_substream *substream,
 				   SNDRV_PCM_HW_PARAM_BUFFER_BYTES, 16);
 	/* enable agent */
 	mtk_regmap_update_bits(afe->regmap, memif->data->agent_disable_reg,
-			       1 << memif->data->agent_disable_shift,
-			       0 << memif->data->agent_disable_shift);
+			       1, 0, memif->data->agent_disable_shift);
 
 	snd_soc_set_runtime_hwparams(substream, mtk_afe_hardware);
 
@@ -105,8 +104,7 @@ void mtk_afe_fe_shutdown(struct snd_pcm_substream *substream,
 	irq_id = memif->irq_usage;
 
 	mtk_regmap_update_bits(afe->regmap, memif->data->agent_disable_reg,
-			       1 << memif->data->agent_disable_shift,
-			       1 << memif->data->agent_disable_shift);
+			       1, 1, memif->data->agent_disable_shift);
 
 	if (!memif->const_irq) {
 		mtk_dynamic_irq_release(afe, irq_id);
@@ -144,16 +142,14 @@ int mtk_afe_fe_hw_params(struct snd_pcm_substream *substream,
 
 	/* set MSB to 33-bit */
 	mtk_regmap_update_bits(afe->regmap, memif->data->msb_reg,
-			       1 << memif->data->msb_shift,
-			       msb_at_bit33 << memif->data->msb_shift);
+			       1, msb_at_bit33, memif->data->msb_shift);
 
 	/* set channel */
 	if (memif->data->mono_shift >= 0) {
 		unsigned int mono = (params_channels(params) == 1) ? 1 : 0;
 
 		mtk_regmap_update_bits(afe->regmap, memif->data->mono_reg,
-				       1 << memif->data->mono_shift,
-				       mono << memif->data->mono_shift);
+				       1, mono, memif->data->mono_shift);
 	}
 
 	/* set rate */
@@ -166,8 +162,8 @@ int mtk_afe_fe_hw_params(struct snd_pcm_substream *substream,
 		return -EINVAL;
 
 	mtk_regmap_update_bits(afe->regmap, memif->data->fs_reg,
-			       memif->data->fs_maskbit << memif->data->fs_shift,
-			       fs << memif->data->fs_shift);
+			       memif->data->fs_maskbit, fs,
+			       memif->data->fs_shift);
 
 	return 0;
 }
@@ -197,17 +193,14 @@ int mtk_afe_fe_trigger(struct snd_pcm_substream *substream, int cmd,
 	switch (cmd) {
 	case SNDRV_PCM_TRIGGER_START:
 	case SNDRV_PCM_TRIGGER_RESUME:
-		if (memif->data->enable_shift >= 0)
-			mtk_regmap_update_bits(afe->regmap,
-					       memif->data->enable_reg,
-					       1 << memif->data->enable_shift,
-					       1 << memif->data->enable_shift);
+		mtk_regmap_update_bits(afe->regmap,
+				       memif->data->enable_reg,
+				       1, 1, memif->data->enable_shift);
 
 		/* set irq counter */
 		mtk_regmap_update_bits(afe->regmap, irq_data->irq_cnt_reg,
-				       irq_data->irq_cnt_maskbit
-				       << irq_data->irq_cnt_shift,
-				       counter << irq_data->irq_cnt_shift);
+				       irq_data->irq_cnt_maskbit, counter,
+				       irq_data->irq_cnt_shift);
 
 		/* set irq fs */
 		fs = afe->irq_fs(substream, runtime->rate);
@@ -216,24 +209,21 @@ int mtk_afe_fe_trigger(struct snd_pcm_substream *substream, int cmd,
 			return -EINVAL;
 
 		mtk_regmap_update_bits(afe->regmap, irq_data->irq_fs_reg,
-				       irq_data->irq_fs_maskbit
-				       << irq_data->irq_fs_shift,
-				       fs << irq_data->irq_fs_shift);
+				       irq_data->irq_fs_maskbit, fs,
+				       irq_data->irq_fs_shift);
 
 		/* enable interrupt */
 		mtk_regmap_update_bits(afe->regmap, irq_data->irq_en_reg,
-				       1 << irq_data->irq_en_shift,
-				       1 << irq_data->irq_en_shift);
+				       1, 1, irq_data->irq_en_shift);
 
 		return 0;
 	case SNDRV_PCM_TRIGGER_STOP:
 	case SNDRV_PCM_TRIGGER_SUSPEND:
 		mtk_regmap_update_bits(afe->regmap, memif->data->enable_reg,
-				       1 << memif->data->enable_shift, 0);
+				       1, 0, memif->data->enable_shift);
 		/* disable interrupt */
 		mtk_regmap_update_bits(afe->regmap, irq_data->irq_en_reg,
-				       1 << irq_data->irq_en_shift,
-				       0 << irq_data->irq_en_shift);
+				       1, 0, irq_data->irq_en_shift);
 		/* and clear pending IRQ */
 		mtk_regmap_write(afe->regmap, irq_data->irq_clr_reg,
 				 1 << irq_data->irq_clr_shift);
@@ -270,8 +260,7 @@ int mtk_afe_fe_prepare(struct snd_pcm_substream *substream,
 	}
 
 	mtk_regmap_update_bits(afe->regmap, memif->data->hd_reg,
-			       1 << memif->data->hd_shift,
-			       hd_audio << memif->data->hd_shift);
+			       1, hd_audio, memif->data->hd_shift);
 
 	return 0;
 }
diff --git a/sound/soc/mediatek/common/mtk-btcvsd.c b/sound/soc/mediatek/common/mtk-btcvsd.c
index 9a163d7064d1..bd55c546e790 100644
--- a/sound/soc/mediatek/common/mtk-btcvsd.c
+++ b/sound/soc/mediatek/common/mtk-btcvsd.c
@@ -193,13 +193,13 @@ static const u8 table_msbc_silence[SCO_PACKET_180] = {
 static void mtk_btcvsd_snd_irq_enable(struct mtk_btcvsd_snd *bt)
 {
 	regmap_update_bits(bt->infra, bt->infra_misc_offset,
-			   bt->conn_bt_cvsd_mask, bt->conn_bt_cvsd_mask);
+			   bt->conn_bt_cvsd_mask, 0);
 }
 
 static void mtk_btcvsd_snd_irq_disable(struct mtk_btcvsd_snd *bt)
 {
 	regmap_update_bits(bt->infra, bt->infra_misc_offset,
-			   bt->conn_bt_cvsd_mask, 0);
+			   bt->conn_bt_cvsd_mask, bt->conn_bt_cvsd_mask);
 }
 
 static void mtk_btcvsd_snd_set_state(struct mtk_btcvsd_snd *bt,
diff --git a/sound/soc/mediatek/mt2701/mt2701-afe-pcm.c b/sound/soc/mediatek/mt2701/mt2701-afe-pcm.c
index 968fba4d7533..7064a9fd6f74 100644
--- a/sound/soc/mediatek/mt2701/mt2701-afe-pcm.c
+++ b/sound/soc/mediatek/mt2701/mt2701-afe-pcm.c
@@ -994,7 +994,6 @@ static const struct mtk_base_memif_data memif_data[MT2701_MEMIF_NUM] = {
 		.agent_disable_reg = AUDIO_TOP_CON5,
 		.agent_disable_shift = 6,
 		.msb_reg = -1,
-		.msb_shift = -1,
 	},
 	{
 		.name = "DL2",
@@ -1013,7 +1012,6 @@ static const struct mtk_base_memif_data memif_data[MT2701_MEMIF_NUM] = {
 		.agent_disable_reg = AUDIO_TOP_CON5,
 		.agent_disable_shift = 7,
 		.msb_reg = -1,
-		.msb_shift = -1,
 	},
 	{
 		.name = "DL3",
@@ -1032,7 +1030,6 @@ static const struct mtk_base_memif_data memif_data[MT2701_MEMIF_NUM] = {
 		.agent_disable_reg = AUDIO_TOP_CON5,
 		.agent_disable_shift = 8,
 		.msb_reg = -1,
-		.msb_shift = -1,
 	},
 	{
 		.name = "DL4",
@@ -1051,7 +1048,6 @@ static const struct mtk_base_memif_data memif_data[MT2701_MEMIF_NUM] = {
 		.agent_disable_reg = AUDIO_TOP_CON5,
 		.agent_disable_shift = 9,
 		.msb_reg = -1,
-		.msb_shift = -1,
 	},
 	{
 		.name = "DL5",
@@ -1070,7 +1066,6 @@ static const struct mtk_base_memif_data memif_data[MT2701_MEMIF_NUM] = {
 		.agent_disable_reg = AUDIO_TOP_CON5,
 		.agent_disable_shift = 10,
 		.msb_reg = -1,
-		.msb_shift = -1,
 	},
 	{
 		.name = "DLM",
@@ -1089,7 +1084,6 @@ static const struct mtk_base_memif_data memif_data[MT2701_MEMIF_NUM] = {
 		.agent_disable_reg = AUDIO_TOP_CON5,
 		.agent_disable_shift = 12,
 		.msb_reg = -1,
-		.msb_shift = -1,
 	},
 	{
 		.name = "UL1",
@@ -1108,7 +1102,6 @@ static const struct mtk_base_memif_data memif_data[MT2701_MEMIF_NUM] = {
 		.agent_disable_reg = AUDIO_TOP_CON5,
 		.agent_disable_shift = 0,
 		.msb_reg = -1,
-		.msb_shift = -1,
 	},
 	{
 		.name = "UL2",
@@ -1127,7 +1120,6 @@ static const struct mtk_base_memif_data memif_data[MT2701_MEMIF_NUM] = {
 		.agent_disable_reg = AUDIO_TOP_CON5,
 		.agent_disable_shift = 1,
 		.msb_reg = -1,
-		.msb_shift = -1,
 	},
 	{
 		.name = "UL3",
@@ -1146,7 +1138,6 @@ static const struct mtk_base_memif_data memif_data[MT2701_MEMIF_NUM] = {
 		.agent_disable_reg = AUDIO_TOP_CON5,
 		.agent_disable_shift = 2,
 		.msb_reg = -1,
-		.msb_shift = -1,
 	},
 	{
 		.name = "UL4",
@@ -1165,7 +1156,6 @@ static const struct mtk_base_memif_data memif_data[MT2701_MEMIF_NUM] = {
 		.agent_disable_reg = AUDIO_TOP_CON5,
 		.agent_disable_shift = 3,
 		.msb_reg = -1,
-		.msb_shift = -1,
 	},
 	{
 		.name = "UL5",
@@ -1184,7 +1174,6 @@ static const struct mtk_base_memif_data memif_data[MT2701_MEMIF_NUM] = {
 		.agent_disable_reg = AUDIO_TOP_CON5,
 		.agent_disable_shift = 4,
 		.msb_reg = -1,
-		.msb_shift = -1,
 	},
 	{
 		.name = "DLBT",
@@ -1203,7 +1192,6 @@ static const struct mtk_base_memif_data memif_data[MT2701_MEMIF_NUM] = {
 		.agent_disable_reg = AUDIO_TOP_CON5,
 		.agent_disable_shift = 13,
 		.msb_reg = -1,
-		.msb_shift = -1,
 	},
 	{
 		.name = "ULBT",
@@ -1222,7 +1210,6 @@ static const struct mtk_base_memif_data memif_data[MT2701_MEMIF_NUM] = {
 		.agent_disable_reg = AUDIO_TOP_CON5,
 		.agent_disable_shift = 16,
 		.msb_reg = -1,
-		.msb_shift = -1,
 	},
 };
 
diff --git a/sound/soc/mediatek/mt6797/mt6797-afe-pcm.c b/sound/soc/mediatek/mt6797/mt6797-afe-pcm.c
index bff7d71d0742..08a6532da322 100644
--- a/sound/soc/mediatek/mt6797/mt6797-afe-pcm.c
+++ b/sound/soc/mediatek/mt6797/mt6797-afe-pcm.c
@@ -401,9 +401,7 @@ static const struct mtk_base_memif_data memif_data[MT6797_MEMIF_NUM] = {
 		.hd_reg = AFE_MEMIF_HD_MODE,
 		.hd_shift = DL1_HD_SFT,
 		.agent_disable_reg = -1,
-		.agent_disable_shift = -1,
 		.msb_reg = -1,
-		.msb_shift = -1,
 	},
 	[MT6797_MEMIF_DL2] = {
 		.name = "DL2",
@@ -420,9 +418,7 @@ static const struct mtk_base_memif_data memif_data[MT6797_MEMIF_NUM] = {
 		.hd_reg = AFE_MEMIF_HD_MODE,
 		.hd_shift = DL2_HD_SFT,
 		.agent_disable_reg = -1,
-		.agent_disable_shift = -1,
 		.msb_reg = -1,
-		.msb_shift = -1,
 	},
 	[MT6797_MEMIF_DL3] = {
 		.name = "DL3",
@@ -439,9 +435,7 @@ static const struct mtk_base_memif_data memif_data[MT6797_MEMIF_NUM] = {
 		.hd_reg = AFE_MEMIF_HD_MODE,
 		.hd_shift = DL3_HD_SFT,
 		.agent_disable_reg = -1,
-		.agent_disable_shift = -1,
 		.msb_reg = -1,
-		.msb_shift = -1,
 	},
 	[MT6797_MEMIF_VUL] = {
 		.name = "VUL",
@@ -458,9 +452,7 @@ static const struct mtk_base_memif_data memif_data[MT6797_MEMIF_NUM] = {
 		.hd_reg = AFE_MEMIF_HD_MODE,
 		.hd_shift = VUL_HD_SFT,
 		.agent_disable_reg = -1,
-		.agent_disable_shift = -1,
 		.msb_reg = -1,
-		.msb_shift = -1,
 	},
 	[MT6797_MEMIF_AWB] = {
 		.name = "AWB",
@@ -477,9 +469,7 @@ static const struct mtk_base_memif_data memif_data[MT6797_MEMIF_NUM] = {
 		.hd_reg = AFE_MEMIF_HD_MODE,
 		.hd_shift = AWB_HD_SFT,
 		.agent_disable_reg = -1,
-		.agent_disable_shift = -1,
 		.msb_reg = -1,
-		.msb_shift = -1,
 	},
 	[MT6797_MEMIF_VUL12] = {
 		.name = "VUL12",
@@ -496,9 +486,7 @@ static const struct mtk_base_memif_data memif_data[MT6797_MEMIF_NUM] = {
 		.hd_reg = AFE_MEMIF_HD_MODE,
 		.hd_shift = VUL_DATA2_HD_SFT,
 		.agent_disable_reg = -1,
-		.agent_disable_shift = -1,
 		.msb_reg = -1,
-		.msb_shift = -1,
 	},
 	[MT6797_MEMIF_DAI] = {
 		.name = "DAI",
@@ -515,9 +503,7 @@ static const struct mtk_base_memif_data memif_data[MT6797_MEMIF_NUM] = {
 		.hd_reg = AFE_MEMIF_HD_MODE,
 		.hd_shift = DAI_HD_SFT,
 		.agent_disable_reg = -1,
-		.agent_disable_shift = -1,
 		.msb_reg = -1,
-		.msb_shift = -1,
 	},
 	[MT6797_MEMIF_MOD_DAI] = {
 		.name = "MOD_DAI",
@@ -534,9 +520,7 @@ static const struct mtk_base_memif_data memif_data[MT6797_MEMIF_NUM] = {
 		.hd_reg = AFE_MEMIF_HD_MODE,
 		.hd_shift = MOD_DAI_HD_SFT,
 		.agent_disable_reg = -1,
-		.agent_disable_shift = -1,
 		.msb_reg = -1,
-		.msb_shift = -1,
 	},
 };
 
diff --git a/sound/soc/mediatek/mt8173/mt8173-afe-pcm.c b/sound/soc/mediatek/mt8173/mt8173-afe-pcm.c
index 166aed28330d..0382896c162e 100644
--- a/sound/soc/mediatek/mt8173/mt8173-afe-pcm.c
+++ b/sound/soc/mediatek/mt8173/mt8173-afe-pcm.c
@@ -714,13 +714,11 @@ static const struct mtk_base_memif_data memif_data[MT8173_AFE_MEMIF_NUM] = {
 		.mono_reg = AFE_DAC_CON1,
 		.mono_shift = 21,
 		.hd_reg = -1,
-		.hd_shift = -1,
 		.enable_reg = AFE_DAC_CON0,
 		.enable_shift = 1,
 		.msb_reg = AFE_MEMIF_MSB,
 		.msb_shift = 0,
 		.agent_disable_reg = -1,
-		.agent_disable_shift = -1,
 	}, {
 		.name = "DL2",
 		.id = MT8173_AFE_MEMIF_DL2,
@@ -732,13 +730,11 @@ static const struct mtk_base_memif_data memif_data[MT8173_AFE_MEMIF_NUM] = {
 		.mono_reg = AFE_DAC_CON1,
 		.mono_shift = 22,
 		.hd_reg = -1,
-		.hd_shift = -1,
 		.enable_reg = AFE_DAC_CON0,
 		.enable_shift = 2,
 		.msb_reg = AFE_MEMIF_MSB,
 		.msb_shift = 1,
 		.agent_disable_reg = -1,
-		.agent_disable_shift = -1,
 	}, {
 		.name = "VUL",
 		.id = MT8173_AFE_MEMIF_VUL,
@@ -750,13 +746,11 @@ static const struct mtk_base_memif_data memif_data[MT8173_AFE_MEMIF_NUM] = {
 		.mono_reg = AFE_DAC_CON1,
 		.mono_shift = 27,
 		.hd_reg = -1,
-		.hd_shift = -1,
 		.enable_reg = AFE_DAC_CON0,
 		.enable_shift = 3,
 		.msb_reg = AFE_MEMIF_MSB,
 		.msb_shift = 6,
 		.agent_disable_reg = -1,
-		.agent_disable_shift = -1,
 	}, {
 		.name = "DAI",
 		.id = MT8173_AFE_MEMIF_DAI,
@@ -768,13 +762,11 @@ static const struct mtk_base_memif_data memif_data[MT8173_AFE_MEMIF_NUM] = {
 		.mono_reg = -1,
 		.mono_shift = -1,
 		.hd_reg = -1,
-		.hd_shift = -1,
 		.enable_reg = AFE_DAC_CON0,
 		.enable_shift = 4,
 		.msb_reg = AFE_MEMIF_MSB,
 		.msb_shift = 5,
 		.agent_disable_reg = -1,
-		.agent_disable_shift = -1,
 	}, {
 		.name = "AWB",
 		.id = MT8173_AFE_MEMIF_AWB,
@@ -786,13 +778,11 @@ static const struct mtk_base_memif_data memif_data[MT8173_AFE_MEMIF_NUM] = {
 		.mono_reg = AFE_DAC_CON1,
 		.mono_shift = 24,
 		.hd_reg = -1,
-		.hd_shift = -1,
 		.enable_reg = AFE_DAC_CON0,
 		.enable_shift = 6,
 		.msb_reg = AFE_MEMIF_MSB,
 		.msb_shift = 3,
 		.agent_disable_reg = -1,
-		.agent_disable_shift = -1,
 	}, {
 		.name = "MOD_DAI",
 		.id = MT8173_AFE_MEMIF_MOD_DAI,
@@ -804,13 +794,11 @@ static const struct mtk_base_memif_data memif_data[MT8173_AFE_MEMIF_NUM] = {
 		.mono_reg = AFE_DAC_CON1,
 		.mono_shift = 30,
 		.hd_reg = -1,
-		.hd_shift = -1,
 		.enable_reg = AFE_DAC_CON0,
 		.enable_shift = 7,
 		.msb_reg = AFE_MEMIF_MSB,
 		.msb_shift = 4,
 		.agent_disable_reg = -1,
-		.agent_disable_shift = -1,
 	}, {
 		.name = "HDMI",
 		.id = MT8173_AFE_MEMIF_HDMI,
@@ -822,13 +810,10 @@ static const struct mtk_base_memif_data memif_data[MT8173_AFE_MEMIF_NUM] = {
 		.mono_reg = -1,
 		.mono_shift = -1,
 		.hd_reg = -1,
-		.hd_shift = -1,
 		.enable_reg = -1,
-		.enable_shift = -1,
 		.msb_reg = AFE_MEMIF_MSB,
 		.msb_shift = 8,
 		.agent_disable_reg = -1,
-		.agent_disable_shift = -1,
 	},
 };
 
@@ -914,7 +899,6 @@ static const struct mtk_base_irq_data irq_data[MT8173_AFE_IRQ_NUM] = {
 		.irq_en_reg = AFE_IRQ_MCU_CON,
 		.irq_en_shift = 12,
 		.irq_fs_reg = -1,
-		.irq_fs_shift = -1,
 		.irq_fs_maskbit = -1,
 		.irq_clr_reg = AFE_IRQ_CLR,
 		.irq_clr_shift = 4,
diff --git a/sound/soc/mediatek/mt8183/Makefile b/sound/soc/mediatek/mt8183/Makefile
index f3ee6ac98fe8..c0a3bbc2c1f6 100644
--- a/sound/soc/mediatek/mt8183/Makefile
+++ b/sound/soc/mediatek/mt8183/Makefile
@@ -11,3 +11,5 @@ snd-soc-mt8183-afe-objs := \
 	mt8183-dai-adda.o
 
 obj-$(CONFIG_SND_SOC_MT8183) += snd-soc-mt8183-afe.o
+obj-$(CONFIG_SND_SOC_MT8183_MT6358_TS3A227E_MAX98357A) += mt8183-mt6358-ts3a227-max98357.o
+obj-$(CONFIG_SND_SOC_MT8183_DA7219_MAX98357A) += mt8183-da7219-max98357.o
diff --git a/sound/soc/mediatek/mt8183/mt8183-afe-pcm.c b/sound/soc/mediatek/mt8183/mt8183-afe-pcm.c
index 4e045dd305a7..1bc0fafe5e29 100644
--- a/sound/soc/mediatek/mt8183/mt8183-afe-pcm.c
+++ b/sound/soc/mediatek/mt8183/mt8183-afe-pcm.c
@@ -291,11 +291,15 @@ static struct snd_soc_dai_driver mt8183_memif_dai_driver[] = {
 static const struct snd_kcontrol_new memif_ul1_ch1_mix[] = {
 	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN21,
 				    I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("I2S0_CH1", AFE_CONN21,
+				    I_I2S0_CH1, 1, 0),
 };
 
 static const struct snd_kcontrol_new memif_ul1_ch2_mix[] = {
 	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN22,
 				    I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("I2S0_CH2", AFE_CONN21,
+				    I_I2S0_CH2, 1, 0),
 };
 
 static const struct snd_kcontrol_new memif_ul2_ch1_mix[] = {
@@ -307,6 +311,8 @@ static const struct snd_kcontrol_new memif_ul2_ch1_mix[] = {
 				    I_DL2_CH1, 1, 0),
 	SOC_DAPM_SINGLE_AUTODISABLE("DL3_CH1", AFE_CONN5,
 				    I_DL3_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("I2S2_CH1", AFE_CONN5,
+				    I_I2S2_CH1, 1, 0),
 };
 
 static const struct snd_kcontrol_new memif_ul2_ch2_mix[] = {
@@ -318,16 +324,22 @@ static const struct snd_kcontrol_new memif_ul2_ch2_mix[] = {
 				    I_DL2_CH2, 1, 0),
 	SOC_DAPM_SINGLE_AUTODISABLE("DL3_CH2", AFE_CONN6,
 				    I_DL3_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("I2S2_CH2", AFE_CONN6,
+				    I_I2S2_CH2, 1, 0),
 };
 
 static const struct snd_kcontrol_new memif_ul3_ch1_mix[] = {
 	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN32,
 				    I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("I2S2_CH1", AFE_CONN32,
+				    I_I2S2_CH1, 1, 0),
 };
 
 static const struct snd_kcontrol_new memif_ul3_ch2_mix[] = {
 	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN33,
 				    I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("I2S2_CH2", AFE_CONN33,
+				    I_I2S2_CH2, 1, 0),
 };
 
 static const struct snd_kcontrol_new memif_ul4_ch1_mix[] = {
@@ -380,16 +392,22 @@ static const struct snd_soc_dapm_route mt8183_memif_routes[] = {
 	{"UL1", NULL, "UL1_CH2"},
 	{"UL1_CH1", "ADDA_UL_CH1", "ADDA Capture"},
 	{"UL1_CH2", "ADDA_UL_CH2", "ADDA Capture"},
+	{"UL1_CH1", "I2S0_CH1", "I2S0"},
+	{"UL1_CH2", "I2S0_CH2", "I2S0"},
 
 	{"UL2", NULL, "UL2_CH1"},
 	{"UL2", NULL, "UL2_CH2"},
 	{"UL2_CH1", "ADDA_UL_CH1", "ADDA Capture"},
 	{"UL2_CH2", "ADDA_UL_CH2", "ADDA Capture"},
+	{"UL2_CH1", "I2S2_CH1", "I2S2"},
+	{"UL2_CH2", "I2S2_CH2", "I2S2"},
 
 	{"UL3", NULL, "UL3_CH1"},
 	{"UL3", NULL, "UL3_CH2"},
 	{"UL3_CH1", "ADDA_UL_CH1", "ADDA Capture"},
 	{"UL3_CH2", "ADDA_UL_CH2", "ADDA Capture"},
+	{"UL3_CH1", "I2S2_CH1", "I2S2"},
+	{"UL3_CH2", "I2S2_CH2", "I2S2"},
 
 	{"UL4", NULL, "UL4_CH1"},
 	{"UL4", NULL, "UL4_CH2"},
diff --git a/sound/soc/mediatek/mt8183/mt8183-da7219-max98357.c b/sound/soc/mediatek/mt8183/mt8183-da7219-max98357.c
new file mode 100644
index 000000000000..31ea8632c397
--- /dev/null
+++ b/sound/soc/mediatek/mt8183/mt8183-da7219-max98357.c
@@ -0,0 +1,471 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// mt8183-da7219-max98357.c
+//	--  MT8183-DA7219-MAX98357 ALSA SoC machine driver
+//
+// Copyright (c) 2018 MediaTek Inc.
+// Author: Shunli Wang <shunli.wang@mediatek.com>
+
+#include <linux/module.h>
+#include <sound/pcm_params.h>
+#include <sound/soc.h>
+#include <sound/jack.h>
+#include <linux/pinctrl/consumer.h>
+
+#include "mt8183-afe-common.h"
+#include "../../codecs/da7219-aad.h"
+#include "../../codecs/da7219.h"
+
+static struct snd_soc_jack headset_jack;
+
+/* Headset jack detection DAPM pins */
+static struct snd_soc_jack_pin headset_jack_pins[] = {
+	{
+		.pin = "Headphone",
+		.mask = SND_JACK_HEADPHONE,
+	},
+	{
+		.pin = "Headset Mic",
+		.mask = SND_JACK_MICROPHONE,
+	},
+};
+
+static struct snd_soc_dai_link_component
+mt8183_da7219_max98357_external_codecs[] = {
+	{
+		.name = "max98357a",
+		.dai_name = "HiFi",
+	},
+	{
+		.name = "da7219.5-001a",
+		.dai_name = "da7219-hifi",
+	},
+};
+
+static int mt8183_mt6358_i2s_hw_params(struct snd_pcm_substream *substream,
+				       struct snd_pcm_hw_params *params)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	unsigned int rate = params_rate(params);
+	unsigned int mclk_fs_ratio = 128;
+	unsigned int mclk_fs = rate * mclk_fs_ratio;
+
+	return snd_soc_dai_set_sysclk(rtd->cpu_dai,
+				      0, mclk_fs, SND_SOC_CLOCK_OUT);
+}
+
+static const struct snd_soc_ops mt8183_mt6358_i2s_ops = {
+	.hw_params = mt8183_mt6358_i2s_hw_params,
+};
+
+static int mt8183_da7219_i2s_hw_params(struct snd_pcm_substream *substream,
+				       struct snd_pcm_hw_params *params)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	unsigned int rate = params_rate(params);
+	unsigned int mclk_fs_ratio = 256;
+	unsigned int mclk_fs = rate * mclk_fs_ratio;
+	unsigned int freq;
+	int ret = 0, j;
+
+	ret = snd_soc_dai_set_sysclk(rtd->cpu_dai, 0,
+				     mclk_fs, SND_SOC_CLOCK_OUT);
+	if (ret < 0)
+		dev_err(rtd->dev, "failed to set cpu dai sysclk\n");
+
+	for (j = 0; j < rtd->num_codecs; j++) {
+		struct snd_soc_dai *codec_dai = rtd->codec_dais[j];
+
+		if (!strcmp(codec_dai->component->name, "da7219.5-001a")) {
+			ret = snd_soc_dai_set_sysclk(codec_dai,
+						     DA7219_CLKSRC_MCLK,
+						     mclk_fs,
+						     SND_SOC_CLOCK_IN);
+			if (ret < 0)
+				dev_err(rtd->dev, "failed to set sysclk\n");
+
+			if ((rate % 8000) == 0)
+				freq = DA7219_PLL_FREQ_OUT_98304;
+			else
+				freq = DA7219_PLL_FREQ_OUT_90316;
+
+			ret = snd_soc_dai_set_pll(codec_dai, 0,
+						  DA7219_SYSCLK_PLL_SRM,
+						  0, freq);
+			if (ret)
+				dev_err(rtd->dev, "failed to start PLL: %d\n",
+					ret);
+		}
+	}
+
+	return ret;
+}
+
+static int mt8183_da7219_hw_free(struct snd_pcm_substream *substream)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	int ret = 0, j;
+
+	for (j = 0; j < rtd->num_codecs; j++) {
+		struct snd_soc_dai *codec_dai = rtd->codec_dais[j];
+
+		if (!strcmp(codec_dai->component->name, "da7219.5-001a")) {
+			ret = snd_soc_dai_set_pll(codec_dai,
+						  0, DA7219_SYSCLK_MCLK, 0, 0);
+			if (ret < 0) {
+				dev_err(rtd->dev, "failed to stop PLL: %d\n",
+					ret);
+				break;
+			}
+		}
+	}
+
+	return ret;
+}
+
+static const struct snd_soc_ops mt8183_da7219_i2s_ops = {
+	.hw_params = mt8183_da7219_i2s_hw_params,
+	.hw_free = mt8183_da7219_hw_free,
+};
+
+static int mt8183_i2s_hw_params_fixup(struct snd_soc_pcm_runtime *rtd,
+				      struct snd_pcm_hw_params *params)
+{
+	/* fix BE i2s format to 32bit, clean param mask first */
+	snd_mask_reset_range(hw_param_mask(params, SNDRV_PCM_HW_PARAM_FORMAT),
+			     0, SNDRV_PCM_FORMAT_LAST);
+
+	params_set_format(params, SNDRV_PCM_FORMAT_S32_LE);
+
+	return 0;
+}
+
+static const struct snd_soc_dapm_widget
+mt8183_da7219_max98357_dapm_widgets[] = {
+	SND_SOC_DAPM_OUTPUT("IT6505_8CH"),
+};
+
+static const struct snd_soc_dapm_route mt8183_da7219_max98357_dapm_routes[] = {
+	{"IT6505_8CH", NULL, "TDM"},
+};
+
+static struct snd_soc_dai_link mt8183_da7219_max98357_dai_links[] = {
+	/* FE */
+	{
+		.name = "Playback_1",
+		.stream_name = "Playback_1",
+		.cpu_dai_name = "DL1",
+		.codec_name = "snd-soc-dummy",
+		.codec_dai_name = "snd-soc-dummy-dai",
+		.trigger = {SND_SOC_DPCM_TRIGGER_PRE,
+			    SND_SOC_DPCM_TRIGGER_PRE},
+		.dynamic = 1,
+		.dpcm_playback = 1,
+	},
+	{
+		.name = "Playback_2",
+		.stream_name = "Playback_2",
+		.cpu_dai_name = "DL2",
+		.codec_name = "snd-soc-dummy",
+		.codec_dai_name = "snd-soc-dummy-dai",
+		.trigger = {SND_SOC_DPCM_TRIGGER_PRE,
+			    SND_SOC_DPCM_TRIGGER_PRE},
+		.dynamic = 1,
+		.dpcm_playback = 1,
+	},
+	{
+		.name = "Playback_3",
+		.stream_name = "Playback_3",
+		.cpu_dai_name = "DL3",
+		.codec_name = "snd-soc-dummy",
+		.codec_dai_name = "snd-soc-dummy-dai",
+		.trigger = {SND_SOC_DPCM_TRIGGER_PRE,
+			    SND_SOC_DPCM_TRIGGER_PRE},
+		.dynamic = 1,
+		.dpcm_playback = 1,
+	},
+	{
+		.name = "Capture_1",
+		.stream_name = "Capture_1",
+		.cpu_dai_name = "UL1",
+		.codec_name = "snd-soc-dummy",
+		.codec_dai_name = "snd-soc-dummy-dai",
+		.trigger = {SND_SOC_DPCM_TRIGGER_PRE,
+			    SND_SOC_DPCM_TRIGGER_PRE},
+		.dynamic = 1,
+		.dpcm_capture = 1,
+	},
+	{
+		.name = "Capture_2",
+		.stream_name = "Capture_2",
+		.cpu_dai_name = "UL2",
+		.codec_name = "snd-soc-dummy",
+		.codec_dai_name = "snd-soc-dummy-dai",
+		.trigger = {SND_SOC_DPCM_TRIGGER_PRE,
+			    SND_SOC_DPCM_TRIGGER_PRE},
+		.dynamic = 1,
+		.dpcm_capture = 1,
+	},
+	{
+		.name = "Capture_3",
+		.stream_name = "Capture_3",
+		.cpu_dai_name = "UL3",
+		.codec_name = "snd-soc-dummy",
+		.codec_dai_name = "snd-soc-dummy-dai",
+		.trigger = {SND_SOC_DPCM_TRIGGER_PRE,
+			    SND_SOC_DPCM_TRIGGER_PRE},
+		.dynamic = 1,
+		.dpcm_capture = 1,
+	},
+	{
+		.name = "Capture_Mono_1",
+		.stream_name = "Capture_Mono_1",
+		.cpu_dai_name = "UL_MONO_1",
+		.codec_name = "snd-soc-dummy",
+		.codec_dai_name = "snd-soc-dummy-dai",
+		.trigger = {SND_SOC_DPCM_TRIGGER_PRE,
+			    SND_SOC_DPCM_TRIGGER_PRE},
+		.dynamic = 1,
+		.dpcm_capture = 1,
+	},
+	{
+		.name = "Playback_HDMI",
+		.stream_name = "Playback_HDMI",
+		.cpu_dai_name = "HDMI",
+		.codec_name = "snd-soc-dummy",
+		.codec_dai_name = "snd-soc-dummy-dai",
+		.trigger = {SND_SOC_DPCM_TRIGGER_PRE,
+			    SND_SOC_DPCM_TRIGGER_PRE},
+		.dynamic = 1,
+		.dpcm_playback = 1,
+	},
+	/* BE */
+	{
+		.name = "Primary Codec",
+		.cpu_dai_name = "ADDA",
+		.codec_dai_name = "mt6358-snd-codec-aif1",
+		.codec_name = "mt6358-sound",
+		.no_pcm = 1,
+		.dpcm_playback = 1,
+		.dpcm_capture = 1,
+		.ignore_suspend = 1,
+	},
+	{
+		.name = "PCM 1",
+		.cpu_dai_name = "PCM 1",
+		.codec_name = "snd-soc-dummy",
+		.codec_dai_name = "snd-soc-dummy-dai",
+		.no_pcm = 1,
+		.dpcm_playback = 1,
+		.dpcm_capture = 1,
+		.ignore_suspend = 1,
+	},
+	{
+		.name = "PCM 2",
+		.cpu_dai_name = "PCM 2",
+		.codec_name = "snd-soc-dummy",
+		.codec_dai_name = "snd-soc-dummy-dai",
+		.no_pcm = 1,
+		.dpcm_playback = 1,
+		.dpcm_capture = 1,
+		.ignore_suspend = 1,
+	},
+	{
+		.name = "I2S0",
+		.cpu_dai_name = "I2S0",
+		.codec_dai_name = "bt-sco-pcm",
+		.codec_name = "bt-sco",
+		.no_pcm = 1,
+		.dpcm_capture = 1,
+		.ignore_suspend = 1,
+		.be_hw_params_fixup = mt8183_i2s_hw_params_fixup,
+		.ops = &mt8183_mt6358_i2s_ops,
+	},
+	{
+		.name = "I2S1",
+		.cpu_dai_name = "I2S1",
+		.codec_dai_name = "snd-soc-dummy-dai",
+		.codec_name = "snd-soc-dummy",
+		.no_pcm = 1,
+		.dpcm_playback = 1,
+		.ignore_suspend = 1,
+		.be_hw_params_fixup = mt8183_i2s_hw_params_fixup,
+		.ops = &mt8183_mt6358_i2s_ops,
+	},
+	{
+		.name = "I2S2",
+		.cpu_dai_name = "I2S2",
+		.codec_dai_name = "da7219-hifi",
+		.codec_name = "da7219.5-001a",
+		.no_pcm = 1,
+		.dpcm_capture = 1,
+		.ignore_suspend = 1,
+		.be_hw_params_fixup = mt8183_i2s_hw_params_fixup,
+		.ops = &mt8183_da7219_i2s_ops,
+	},
+	{
+		.name = "I2S3",
+		.cpu_dai_name = "I2S3",
+		.codecs = mt8183_da7219_max98357_external_codecs,
+		.num_codecs =
+			ARRAY_SIZE(mt8183_da7219_max98357_external_codecs),
+		.no_pcm = 1,
+		.dpcm_playback = 1,
+		.ignore_suspend = 1,
+		.be_hw_params_fixup = mt8183_i2s_hw_params_fixup,
+		.ops = &mt8183_da7219_i2s_ops,
+	},
+	{
+		.name = "I2S5",
+		.cpu_dai_name = "I2S5",
+		.codec_dai_name = "bt-sco-pcm",
+		.codec_name = "bt-sco",
+		.no_pcm = 1,
+		.dpcm_playback = 1,
+		.ignore_suspend = 1,
+		.be_hw_params_fixup = mt8183_i2s_hw_params_fixup,
+		.ops = &mt8183_mt6358_i2s_ops,
+	},
+	{
+		.name = "TDM",
+		.cpu_dai_name = "TDM",
+		.codec_name = "snd-soc-dummy",
+		.codec_dai_name = "snd-soc-dummy-dai",
+		.no_pcm = 1,
+		.dpcm_playback = 1,
+		.ignore_suspend = 1,
+	},
+};
+
+static int
+mt8183_da7219_max98357_headset_init(struct snd_soc_component *component);
+
+static struct snd_soc_aux_dev mt8183_da7219_max98357_headset_dev = {
+	.name = "Headset Chip",
+	.init = mt8183_da7219_max98357_headset_init,
+};
+
+static struct snd_soc_codec_conf mt6358_codec_conf[] = {
+	{
+		.dev_name = "mt6358-sound",
+		.name_prefix = "Mt6358",
+	},
+};
+
+static struct snd_soc_card mt8183_da7219_max98357_card = {
+	.name = "mt8183_da7219_max98357",
+	.owner = THIS_MODULE,
+	.dai_link = mt8183_da7219_max98357_dai_links,
+	.num_links = ARRAY_SIZE(mt8183_da7219_max98357_dai_links),
+	.aux_dev = &mt8183_da7219_max98357_headset_dev,
+	.num_aux_devs = 1,
+	.codec_conf = mt6358_codec_conf,
+	.num_configs = ARRAY_SIZE(mt6358_codec_conf),
+};
+
+static int
+mt8183_da7219_max98357_headset_init(struct snd_soc_component *component)
+{
+	int ret;
+
+	/* Enable Headset and 4 Buttons Jack detection */
+	ret = snd_soc_card_jack_new(&mt8183_da7219_max98357_card,
+				    "Headset Jack",
+				    SND_JACK_HEADSET |
+				    SND_JACK_BTN_0 | SND_JACK_BTN_1 |
+				    SND_JACK_BTN_2 | SND_JACK_BTN_3,
+				    &headset_jack,
+				    headset_jack_pins,
+				    ARRAY_SIZE(headset_jack_pins));
+	if (ret)
+		return ret;
+
+	da7219_aad_jack_det(component, &headset_jack);
+
+	return ret;
+}
+
+static int mt8183_da7219_max98357_dev_probe(struct platform_device *pdev)
+{
+	struct snd_soc_card *card = &mt8183_da7219_max98357_card;
+	struct device_node *platform_node;
+	struct snd_soc_dai_link *dai_link;
+	struct pinctrl *default_pins;
+	int ret, i;
+
+	card->dev = &pdev->dev;
+
+	platform_node = of_parse_phandle(pdev->dev.of_node,
+					 "mediatek,platform", 0);
+	if (!platform_node) {
+		dev_err(&pdev->dev, "Property 'platform' missing or invalid\n");
+		return -EINVAL;
+	}
+
+	for_each_card_prelinks(card, i, dai_link) {
+		/* In the alsa soc-core, the "platform" will be
+		 * allocated by devm_kzalloc if null.
+		 * There is a special case that registerring
+		 * sound card is failed at the first time, but
+		 * the "platform" will not null when probe is trying
+		 * again. It's not expected normally.
+		 */
+		dai_link->platforms = NULL;
+
+		if (dai_link->platform_name)
+			continue;
+		dai_link->platform_of_node = platform_node;
+	}
+
+	mt8183_da7219_max98357_headset_dev.codec_of_node =
+		of_parse_phandle(pdev->dev.of_node,
+				 "mediatek,headset-codec", 0);
+	if (!mt8183_da7219_max98357_headset_dev.codec_of_node) {
+		dev_err(&pdev->dev,
+			"Property 'mediatek,headset-codec' missing/invalid\n");
+		return -EINVAL;
+	}
+
+	ret = devm_snd_soc_register_card(&pdev->dev, card);
+	if (ret) {
+		dev_err(&pdev->dev, "%s snd_soc_register_card fail %d\n",
+			__func__, ret);
+		return ret;
+	}
+
+	default_pins =
+		devm_pinctrl_get_select(&pdev->dev, PINCTRL_STATE_DEFAULT);
+	if (IS_ERR(default_pins)) {
+		dev_err(&pdev->dev, "%s set pins failed\n",
+			__func__);
+		return PTR_ERR(default_pins);
+	}
+
+	return ret;
+}
+
+#ifdef CONFIG_OF
+static const struct of_device_id mt8183_da7219_max98357_dt_match[] = {
+	{.compatible = "mediatek,mt8183_da7219_max98357",},
+	{}
+};
+#endif
+
+static struct platform_driver mt8183_da7219_max98357_driver = {
+	.driver = {
+		.name = "mt8183_da7219_max98357",
+#ifdef CONFIG_OF
+		.of_match_table = mt8183_da7219_max98357_dt_match,
+#endif
+	},
+	.probe = mt8183_da7219_max98357_dev_probe,
+};
+
+module_platform_driver(mt8183_da7219_max98357_driver);
+
+/* Module information */
+MODULE_DESCRIPTION("MT8183-DA7219-MAX98357 ALSA SoC machine driver");
+MODULE_AUTHOR("Shunli Wang <shunli.wang@mediatek.com>");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("mt8183_da7219_max98357 soc card");
+
diff --git a/sound/soc/mediatek/mt8183/mt8183-mt6358-ts3a227-max98357.c b/sound/soc/mediatek/mt8183/mt8183-mt6358-ts3a227-max98357.c
new file mode 100644
index 000000000000..4e44e5689d6f
--- /dev/null
+++ b/sound/soc/mediatek/mt8183/mt8183-mt6358-ts3a227-max98357.c
@@ -0,0 +1,423 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// mt8183-mt6358.c  --
+//	MT8183-MT6358-TS3A227-MAX98357 ALSA SoC machine driver
+//
+// Copyright (c) 2018 MediaTek Inc.
+// Author: Shunli Wang <shunli.wang@mediatek.com>
+
+#include <linux/module.h>
+#include <sound/pcm_params.h>
+#include <sound/soc.h>
+#include <sound/jack.h>
+#include <linux/pinctrl/consumer.h>
+
+#include "mt8183-afe-common.h"
+#include "../../codecs/ts3a227e.h"
+
+static struct snd_soc_jack headset_jack;
+
+/* Headset jack detection DAPM pins */
+static struct snd_soc_jack_pin headset_jack_pins[] = {
+	{
+		.pin = "Headphone",
+		.mask = SND_JACK_HEADPHONE,
+	},
+	{
+		.pin = "Headset Mic",
+		.mask = SND_JACK_MICROPHONE,
+	},
+
+};
+
+static int mt8183_mt6358_i2s_hw_params(struct snd_pcm_substream *substream,
+				       struct snd_pcm_hw_params *params)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	unsigned int rate = params_rate(params);
+	unsigned int mclk_fs_ratio = 128;
+	unsigned int mclk_fs = rate * mclk_fs_ratio;
+
+	return snd_soc_dai_set_sysclk(rtd->cpu_dai,
+				      0, mclk_fs, SND_SOC_CLOCK_OUT);
+}
+
+static const struct snd_soc_ops mt8183_mt6358_i2s_ops = {
+	.hw_params = mt8183_mt6358_i2s_hw_params,
+};
+
+static int mt8183_i2s_hw_params_fixup(struct snd_soc_pcm_runtime *rtd,
+				      struct snd_pcm_hw_params *params)
+{
+	dev_dbg(rtd->dev, "%s(), fix format to 32bit\n", __func__);
+
+	/* fix BE i2s format to 32bit, clean param mask first */
+	snd_mask_reset_range(hw_param_mask(params, SNDRV_PCM_HW_PARAM_FORMAT),
+			     0, SNDRV_PCM_FORMAT_LAST);
+
+	params_set_format(params, SNDRV_PCM_FORMAT_S32_LE);
+	return 0;
+}
+
+static const struct snd_soc_dapm_widget
+mt8183_mt6358_ts3a227_max98357_dapm_widgets[] = {
+	SND_SOC_DAPM_OUTPUT("IT6505_8CH"),
+};
+
+static const struct snd_soc_dapm_route
+mt8183_mt6358_ts3a227_max98357_dapm_routes[] = {
+	{"IT6505_8CH", NULL, "TDM"},
+};
+
+static int
+mt8183_mt6358_ts3a227_max98357_bt_sco_startup(
+	struct snd_pcm_substream *substream)
+{
+	static const unsigned int rates[] = {
+		8000, 16000
+	};
+	static const struct snd_pcm_hw_constraint_list constraints_rates = {
+		.count = ARRAY_SIZE(rates),
+		.list  = rates,
+		.mask = 0,
+	};
+	static const unsigned int channels[] = {
+		1,
+	};
+	static const struct snd_pcm_hw_constraint_list constraints_channels = {
+		.count = ARRAY_SIZE(channels),
+		.list = channels,
+		.mask = 0,
+	};
+
+	struct snd_pcm_runtime *runtime = substream->runtime;
+
+	snd_pcm_hw_constraint_list(runtime, 0,
+			SNDRV_PCM_HW_PARAM_RATE, &constraints_rates);
+	runtime->hw.channels_max = 1;
+	snd_pcm_hw_constraint_list(runtime, 0,
+			SNDRV_PCM_HW_PARAM_CHANNELS,
+			&constraints_channels);
+
+	runtime->hw.formats = SNDRV_PCM_FMTBIT_S16_LE;
+	snd_pcm_hw_constraint_msbits(runtime, 0, 16, 16);
+
+	return 0;
+}
+
+static const struct snd_soc_ops mt8183_mt6358_ts3a227_max98357_bt_sco_ops = {
+	.startup = mt8183_mt6358_ts3a227_max98357_bt_sco_startup,
+};
+
+static struct snd_soc_dai_link
+mt8183_mt6358_ts3a227_max98357_dai_links[] = {
+	/* FE */
+	{
+		.name = "Playback_1",
+		.stream_name = "Playback_1",
+		.cpu_dai_name = "DL1",
+		.codec_name = "snd-soc-dummy",
+		.codec_dai_name = "snd-soc-dummy-dai",
+		.trigger = {SND_SOC_DPCM_TRIGGER_PRE,
+			    SND_SOC_DPCM_TRIGGER_PRE},
+		.dynamic = 1,
+		.dpcm_playback = 1,
+	},
+	{
+		.name = "Playback_2",
+		.stream_name = "Playback_2",
+		.cpu_dai_name = "DL2",
+		.codec_name = "snd-soc-dummy",
+		.codec_dai_name = "snd-soc-dummy-dai",
+		.trigger = {SND_SOC_DPCM_TRIGGER_PRE,
+			    SND_SOC_DPCM_TRIGGER_PRE},
+		.dynamic = 1,
+		.dpcm_playback = 1,
+		.ops = &mt8183_mt6358_ts3a227_max98357_bt_sco_ops,
+	},
+	{
+		.name = "Playback_3",
+		.stream_name = "Playback_3",
+		.cpu_dai_name = "DL3",
+		.codec_name = "snd-soc-dummy",
+		.codec_dai_name = "snd-soc-dummy-dai",
+		.trigger = {SND_SOC_DPCM_TRIGGER_PRE,
+			    SND_SOC_DPCM_TRIGGER_PRE},
+		.dynamic = 1,
+		.dpcm_playback = 1,
+	},
+	{
+		.name = "Capture_1",
+		.stream_name = "Capture_1",
+		.cpu_dai_name = "UL1",
+		.codec_name = "snd-soc-dummy",
+		.codec_dai_name = "snd-soc-dummy-dai",
+		.trigger = {SND_SOC_DPCM_TRIGGER_PRE,
+			    SND_SOC_DPCM_TRIGGER_PRE},
+		.dynamic = 1,
+		.dpcm_capture = 1,
+		.ops = &mt8183_mt6358_ts3a227_max98357_bt_sco_ops,
+	},
+	{
+		.name = "Capture_2",
+		.stream_name = "Capture_2",
+		.cpu_dai_name = "UL2",
+		.codec_name = "snd-soc-dummy",
+		.codec_dai_name = "snd-soc-dummy-dai",
+		.trigger = {SND_SOC_DPCM_TRIGGER_PRE,
+			    SND_SOC_DPCM_TRIGGER_PRE},
+		.dynamic = 1,
+		.dpcm_capture = 1,
+	},
+	{
+		.name = "Capture_3",
+		.stream_name = "Capture_3",
+		.cpu_dai_name = "UL3",
+		.codec_name = "snd-soc-dummy",
+		.codec_dai_name = "snd-soc-dummy-dai",
+		.trigger = {SND_SOC_DPCM_TRIGGER_PRE,
+			    SND_SOC_DPCM_TRIGGER_PRE},
+		.dynamic = 1,
+		.dpcm_capture = 1,
+	},
+	{
+		.name = "Capture_Mono_1",
+		.stream_name = "Capture_Mono_1",
+		.cpu_dai_name = "UL_MONO_1",
+		.codec_name = "snd-soc-dummy",
+		.codec_dai_name = "snd-soc-dummy-dai",
+		.trigger = {SND_SOC_DPCM_TRIGGER_PRE,
+			    SND_SOC_DPCM_TRIGGER_PRE},
+		.dynamic = 1,
+		.dpcm_capture = 1,
+	},
+	{
+		.name = "Playback_HDMI",
+		.stream_name = "Playback_HDMI",
+		.cpu_dai_name = "HDMI",
+		.codec_name = "snd-soc-dummy",
+		.codec_dai_name = "snd-soc-dummy-dai",
+		.trigger = {SND_SOC_DPCM_TRIGGER_PRE,
+			    SND_SOC_DPCM_TRIGGER_PRE},
+		.dynamic = 1,
+		.dpcm_playback = 1,
+	},
+	/* BE */
+	{
+		.name = "Primary Codec",
+		.cpu_dai_name = "ADDA",
+		.codec_dai_name = "mt6358-snd-codec-aif1",
+		.codec_name = "mt6358-sound",
+		.no_pcm = 1,
+		.dpcm_playback = 1,
+		.dpcm_capture = 1,
+		.ignore_suspend = 1,
+	},
+	{
+		.name = "PCM 1",
+		.cpu_dai_name = "PCM 1",
+		.codec_name = "snd-soc-dummy",
+		.codec_dai_name = "snd-soc-dummy-dai",
+		.no_pcm = 1,
+		.dpcm_playback = 1,
+		.dpcm_capture = 1,
+		.ignore_suspend = 1,
+	},
+	{
+		.name = "PCM 2",
+		.cpu_dai_name = "PCM 2",
+		.codec_name = "snd-soc-dummy",
+		.codec_dai_name = "snd-soc-dummy-dai",
+		.no_pcm = 1,
+		.dpcm_playback = 1,
+		.dpcm_capture = 1,
+		.ignore_suspend = 1,
+	},
+	{
+		.name = "I2S0",
+		.cpu_dai_name = "I2S0",
+		.codec_dai_name = "bt-sco-pcm",
+		.codec_name = "bt-sco",
+		.no_pcm = 1,
+		.dpcm_capture = 1,
+		.ignore_suspend = 1,
+		.be_hw_params_fixup = mt8183_i2s_hw_params_fixup,
+		.ops = &mt8183_mt6358_i2s_ops,
+	},
+	{
+		.name = "I2S1",
+		.cpu_dai_name = "I2S1",
+		.codec_dai_name = "snd-soc-dummy-dai",
+		.codec_name = "snd-soc-dummy",
+		.no_pcm = 1,
+		.dpcm_playback = 1,
+		.ignore_suspend = 1,
+		.be_hw_params_fixup = mt8183_i2s_hw_params_fixup,
+		.ops = &mt8183_mt6358_i2s_ops,
+	},
+	{
+		.name = "I2S2",
+		.cpu_dai_name = "I2S2",
+		.codec_dai_name = "snd-soc-dummy-dai",
+		.codec_name = "snd-soc-dummy",
+		.no_pcm = 1,
+		.dpcm_capture = 1,
+		.ignore_suspend = 1,
+		.be_hw_params_fixup = mt8183_i2s_hw_params_fixup,
+		.ops = &mt8183_mt6358_i2s_ops,
+	},
+	{
+		.name = "I2S3",
+		.cpu_dai_name = "I2S3",
+		.codec_dai_name = "HiFi",
+		.codec_name = "max98357a",
+		.no_pcm = 1,
+		.dpcm_playback = 1,
+		.ignore_suspend = 1,
+		.be_hw_params_fixup = mt8183_i2s_hw_params_fixup,
+		.ops = &mt8183_mt6358_i2s_ops,
+	},
+	{
+		.name = "I2S5",
+		.cpu_dai_name = "I2S5",
+		.codec_dai_name = "bt-sco-pcm",
+		.codec_name = "bt-sco",
+		.no_pcm = 1,
+		.dpcm_playback = 1,
+		.ignore_suspend = 1,
+		.be_hw_params_fixup = mt8183_i2s_hw_params_fixup,
+		.ops = &mt8183_mt6358_i2s_ops,
+	},
+	{
+		.name = "TDM",
+		.cpu_dai_name = "TDM",
+		.codec_name = "snd-soc-dummy",
+		.codec_dai_name = "snd-soc-dummy-dai",
+		.no_pcm = 1,
+		.dpcm_playback = 1,
+		.ignore_suspend = 1,
+	},
+};
+
+static int
+mt8183_mt6358_ts3a227_max98357_headset_init(struct snd_soc_component *cpnt);
+
+static struct snd_soc_aux_dev mt8183_mt6358_ts3a227_max98357_headset_dev = {
+	.name = "Headset Chip",
+	.init = mt8183_mt6358_ts3a227_max98357_headset_init,
+};
+
+static struct snd_soc_card mt8183_mt6358_ts3a227_max98357_card = {
+	.name = "mt8183_mt6358_ts3a227_max98357",
+	.owner = THIS_MODULE,
+	.dai_link = mt8183_mt6358_ts3a227_max98357_dai_links,
+	.num_links = ARRAY_SIZE(mt8183_mt6358_ts3a227_max98357_dai_links),
+	.aux_dev = &mt8183_mt6358_ts3a227_max98357_headset_dev,
+	.num_aux_devs = 1,
+};
+
+static int
+mt8183_mt6358_ts3a227_max98357_headset_init(struct snd_soc_component *component)
+{
+	int ret;
+
+	/* Enable Headset and 4 Buttons Jack detection */
+	ret = snd_soc_card_jack_new(&mt8183_mt6358_ts3a227_max98357_card,
+				    "Headset Jack",
+				    SND_JACK_HEADSET |
+				    SND_JACK_BTN_0 | SND_JACK_BTN_1 |
+				    SND_JACK_BTN_2 | SND_JACK_BTN_3,
+				    &headset_jack,
+				    headset_jack_pins,
+				    ARRAY_SIZE(headset_jack_pins));
+	if (ret)
+		return ret;
+
+	ret = ts3a227e_enable_jack_detect(component, &headset_jack);
+
+	return ret;
+}
+
+static int
+mt8183_mt6358_ts3a227_max98357_dev_probe(struct platform_device *pdev)
+{
+	struct snd_soc_card *card = &mt8183_mt6358_ts3a227_max98357_card;
+	struct device_node *platform_node;
+	struct snd_soc_dai_link *dai_link;
+	struct pinctrl *default_pins;
+	int ret, i;
+
+	card->dev = &pdev->dev;
+
+	platform_node = of_parse_phandle(pdev->dev.of_node,
+					 "mediatek,platform", 0);
+	if (!platform_node) {
+		dev_err(&pdev->dev, "Property 'platform' missing or invalid\n");
+		return -EINVAL;
+	}
+
+	for_each_card_prelinks(card, i, dai_link) {
+		/* In the alsa soc-core, the "platform" will be
+		 * allocated by devm_kzalloc if null.
+		 * There is a special case that registerring
+		 * sound card is failed at the first time, but
+		 * the "platform" will not null when probe is trying
+		 * again. It's not expected normally.
+		 */
+		dai_link->platforms = NULL;
+
+		if (dai_link->platform_name)
+			continue;
+		dai_link->platform_of_node = platform_node;
+	}
+
+	mt8183_mt6358_ts3a227_max98357_headset_dev.codec_of_node =
+		of_parse_phandle(pdev->dev.of_node,
+				 "mediatek,headset-codec", 0);
+	if (!mt8183_mt6358_ts3a227_max98357_headset_dev.codec_of_node) {
+		dev_err(&pdev->dev,
+			"Property 'mediatek,headset-codec' missing/invalid\n");
+		return -EINVAL;
+	}
+
+	ret = devm_snd_soc_register_card(&pdev->dev, card);
+	if (ret)
+		dev_err(&pdev->dev, "%s snd_soc_register_card fail %d\n",
+			__func__, ret);
+
+	default_pins =
+		devm_pinctrl_get_select(&pdev->dev, PINCTRL_STATE_DEFAULT);
+	if (IS_ERR(default_pins)) {
+		dev_err(&pdev->dev, "%s set pins failed\n",
+			__func__);
+		return PTR_ERR(default_pins);
+	}
+
+	return ret;
+}
+
+#ifdef CONFIG_OF
+static const struct of_device_id mt8183_mt6358_ts3a227_max98357_dt_match[] = {
+	{.compatible = "mediatek,mt8183_mt6358_ts3a227_max98357",},
+	{}
+};
+#endif
+
+static struct platform_driver mt8183_mt6358_ts3a227_max98357_driver = {
+	.driver = {
+		.name = "mt8183_mt6358_ts3a227_max98357",
+#ifdef CONFIG_OF
+		.of_match_table = mt8183_mt6358_ts3a227_max98357_dt_match,
+#endif
+	},
+	.probe = mt8183_mt6358_ts3a227_max98357_dev_probe,
+};
+
+module_platform_driver(mt8183_mt6358_ts3a227_max98357_driver);
+
+/* Module information */
+MODULE_DESCRIPTION("MT8183-MT6358-TS3A227-MAX98357 ALSA SoC machine driver");
+MODULE_AUTHOR("Shunli Wang <shunli.wang@mediatek.com>");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("mt8183_mt6358_ts3a227_max98357 soc card");
+
diff --git a/sound/soc/meson/axg-fifo.c b/sound/soc/meson/axg-fifo.c
index 75e5e480fda2..01c1c7db2510 100644
--- a/sound/soc/meson/axg-fifo.c
+++ b/sound/soc/meson/axg-fifo.c
@@ -19,7 +19,7 @@
  * This file implements the platform operations common to the playback and
  * capture frontend DAI. The logic behind this two types of fifo is very
  * similar but some difference exist.
- * These differences the respective DAI drivers
+ * These differences are handled in the respective DAI drivers
  */
 
 static struct snd_pcm_hardware axg_fifo_hw = {
@@ -133,6 +133,23 @@ static int axg_fifo_pcm_hw_params(struct snd_pcm_substream *ss,
 	return 0;
 }
 
+static int g12a_fifo_pcm_hw_params(struct snd_pcm_substream *ss,
+				   struct snd_pcm_hw_params *params)
+{
+	struct axg_fifo *fifo = axg_fifo_data(ss);
+	struct snd_pcm_runtime *runtime = ss->runtime;
+	int ret;
+
+	ret = axg_fifo_pcm_hw_params(ss, params);
+	if (ret)
+		return ret;
+
+	/* Set the initial memory address of the DMA */
+	regmap_write(fifo->map, FIFO_INIT_ADDR, runtime->dma_addr);
+
+	return 0;
+}
+
 static int axg_fifo_pcm_hw_free(struct snd_pcm_substream *ss)
 {
 	struct axg_fifo *fifo = axg_fifo_data(ss);
@@ -262,6 +279,17 @@ const struct snd_pcm_ops axg_fifo_pcm_ops = {
 };
 EXPORT_SYMBOL_GPL(axg_fifo_pcm_ops);
 
+const struct snd_pcm_ops g12a_fifo_pcm_ops = {
+	.open =		axg_fifo_pcm_open,
+	.close =        axg_fifo_pcm_close,
+	.ioctl =	snd_pcm_lib_ioctl,
+	.hw_params =	g12a_fifo_pcm_hw_params,
+	.hw_free =      axg_fifo_pcm_hw_free,
+	.pointer =	axg_fifo_pcm_pointer,
+	.trigger =	axg_fifo_pcm_trigger,
+};
+EXPORT_SYMBOL_GPL(g12a_fifo_pcm_ops);
+
 int axg_fifo_pcm_new(struct snd_soc_pcm_runtime *rtd, unsigned int type)
 {
 	struct snd_card *card = rtd->card->snd_card;
@@ -278,7 +306,7 @@ static const struct regmap_config axg_fifo_regmap_cfg = {
 	.reg_bits	= 32,
 	.val_bits	= 32,
 	.reg_stride	= 4,
-	.max_register	= FIFO_STATUS2,
+	.max_register	= FIFO_INIT_ADDR,
 };
 
 int axg_fifo_probe(struct platform_device *pdev)
@@ -339,6 +367,6 @@ int axg_fifo_probe(struct platform_device *pdev)
 }
 EXPORT_SYMBOL_GPL(axg_fifo_probe);
 
-MODULE_DESCRIPTION("Amlogic AXG fifo driver");
+MODULE_DESCRIPTION("Amlogic AXG/G12A fifo driver");
 MODULE_AUTHOR("Jerome Brunet <jbrunet@baylibre.com>");
 MODULE_LICENSE("GPL v2");
diff --git a/sound/soc/meson/axg-fifo.h b/sound/soc/meson/axg-fifo.h
index d9f516cfbeda..5caf81241dfe 100644
--- a/sound/soc/meson/axg-fifo.h
+++ b/sound/soc/meson/axg-fifo.h
@@ -60,6 +60,7 @@ struct snd_soc_pcm_runtime;
 #define FIFO_STATUS1			0x14
 #define  STATUS1_INT_STS(x)		((x) << 0)
 #define FIFO_STATUS2			0x18
+#define FIFO_INIT_ADDR			0x24
 
 struct axg_fifo {
 	struct regmap *map;
@@ -74,6 +75,7 @@ struct axg_fifo_match_data {
 };
 
 extern const struct snd_pcm_ops axg_fifo_pcm_ops;
+extern const struct snd_pcm_ops g12a_fifo_pcm_ops;
 
 int axg_fifo_pcm_new(struct snd_soc_pcm_runtime *rtd, unsigned int type);
 int axg_fifo_probe(struct platform_device *pdev);
diff --git a/sound/soc/meson/axg-frddr.c b/sound/soc/meson/axg-frddr.c
index a6f6f6a2eca8..2b8807737b2b 100644
--- a/sound/soc/meson/axg-frddr.c
+++ b/sound/soc/meson/axg-frddr.c
@@ -3,7 +3,9 @@
 // Copyright (c) 2018 BayLibre, SAS.
 // Author: Jerome Brunet <jbrunet@baylibre.com>
 
-/* This driver implements the frontend playback DAI of AXG based SoCs */
+/*
+ * This driver implements the frontend playback DAI of AXG and G12A based SoCs
+ */
 
 #include <linux/clk.h>
 #include <linux/regmap.h>
@@ -14,7 +16,29 @@
 
 #include "axg-fifo.h"
 
-#define CTRL0_FRDDR_PP_MODE	BIT(30)
+#define CTRL0_FRDDR_PP_MODE		BIT(30)
+#define CTRL0_SEL1_EN_SHIFT		3
+#define CTRL0_SEL2_SHIFT		4
+#define CTRL0_SEL2_EN_SHIFT		7
+#define CTRL0_SEL3_SHIFT		8
+#define CTRL0_SEL3_EN_SHIFT		11
+#define CTRL1_FRDDR_FORCE_FINISH	BIT(12)
+
+static int g12a_frddr_dai_prepare(struct snd_pcm_substream *substream,
+				  struct snd_soc_dai *dai)
+{
+	struct axg_fifo *fifo = snd_soc_dai_get_drvdata(dai);
+
+	/* Reset the read pointer to the FIFO_INIT_ADDR */
+	regmap_update_bits(fifo->map, FIFO_CTRL1,
+			   CTRL1_FRDDR_FORCE_FINISH, 0);
+	regmap_update_bits(fifo->map, FIFO_CTRL1,
+			   CTRL1_FRDDR_FORCE_FINISH, CTRL1_FRDDR_FORCE_FINISH);
+	regmap_update_bits(fifo->map, FIFO_CTRL1,
+			   CTRL1_FRDDR_FORCE_FINISH, 0);
+
+	return 0;
+}
 
 static int axg_frddr_dai_startup(struct snd_pcm_substream *substream,
 				 struct snd_soc_dai *dai)
@@ -119,10 +143,123 @@ static const struct axg_fifo_match_data axg_frddr_match_data = {
 	.dai_drv	= &axg_frddr_dai_drv
 };
 
+static const struct snd_soc_dai_ops g12a_frddr_ops = {
+	.prepare	= g12a_frddr_dai_prepare,
+	.startup	= axg_frddr_dai_startup,
+	.shutdown	= axg_frddr_dai_shutdown,
+};
+
+static struct snd_soc_dai_driver g12a_frddr_dai_drv = {
+	.name = "FRDDR",
+	.playback = {
+		.stream_name	= "Playback",
+		.channels_min	= 1,
+		.channels_max	= AXG_FIFO_CH_MAX,
+		.rates		= AXG_FIFO_RATES,
+		.formats	= AXG_FIFO_FORMATS,
+	},
+	.ops		= &g12a_frddr_ops,
+	.pcm_new	= axg_frddr_pcm_new,
+};
+
+static const char * const g12a_frddr_sel_texts[] = {
+	"OUT 0", "OUT 1", "OUT 2", "OUT 3", "OUT 4",
+};
+
+static SOC_ENUM_SINGLE_DECL(g12a_frddr_sel1_enum, FIFO_CTRL0, CTRL0_SEL_SHIFT,
+			    g12a_frddr_sel_texts);
+static SOC_ENUM_SINGLE_DECL(g12a_frddr_sel2_enum, FIFO_CTRL0, CTRL0_SEL2_SHIFT,
+			    g12a_frddr_sel_texts);
+static SOC_ENUM_SINGLE_DECL(g12a_frddr_sel3_enum, FIFO_CTRL0, CTRL0_SEL3_SHIFT,
+			    g12a_frddr_sel_texts);
+
+static const struct snd_kcontrol_new g12a_frddr_out1_demux =
+	SOC_DAPM_ENUM("Output Src 1", g12a_frddr_sel1_enum);
+static const struct snd_kcontrol_new g12a_frddr_out2_demux =
+	SOC_DAPM_ENUM("Output Src 2", g12a_frddr_sel2_enum);
+static const struct snd_kcontrol_new g12a_frddr_out3_demux =
+	SOC_DAPM_ENUM("Output Src 3", g12a_frddr_sel3_enum);
+
+static const struct snd_kcontrol_new g12a_frddr_out1_enable =
+	SOC_DAPM_SINGLE_AUTODISABLE("Switch", FIFO_CTRL0,
+				    CTRL0_SEL1_EN_SHIFT, 1, 0);
+static const struct snd_kcontrol_new g12a_frddr_out2_enable =
+	SOC_DAPM_SINGLE_AUTODISABLE("Switch", FIFO_CTRL0,
+				    CTRL0_SEL2_EN_SHIFT, 1, 0);
+static const struct snd_kcontrol_new g12a_frddr_out3_enable =
+	SOC_DAPM_SINGLE_AUTODISABLE("Switch", FIFO_CTRL0,
+				    CTRL0_SEL3_EN_SHIFT, 1, 0);
+
+static const struct snd_soc_dapm_widget g12a_frddr_dapm_widgets[] = {
+	SND_SOC_DAPM_AIF_OUT("SRC 1", NULL, 0, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_OUT("SRC 2", NULL, 0, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_OUT("SRC 3", NULL, 0, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_SWITCH("SRC 1 EN", SND_SOC_NOPM, 0, 0,
+			    &g12a_frddr_out1_enable),
+	SND_SOC_DAPM_SWITCH("SRC 2 EN", SND_SOC_NOPM, 0, 0,
+			    &g12a_frddr_out2_enable),
+	SND_SOC_DAPM_SWITCH("SRC 3 EN", SND_SOC_NOPM, 0, 0,
+			    &g12a_frddr_out3_enable),
+	SND_SOC_DAPM_DEMUX("SINK 1 SEL", SND_SOC_NOPM, 0, 0,
+			   &g12a_frddr_out1_demux),
+	SND_SOC_DAPM_DEMUX("SINK 2 SEL", SND_SOC_NOPM, 0, 0,
+			   &g12a_frddr_out2_demux),
+	SND_SOC_DAPM_DEMUX("SINK 3 SEL", SND_SOC_NOPM, 0, 0,
+			   &g12a_frddr_out3_demux),
+	SND_SOC_DAPM_AIF_OUT("OUT 0", NULL, 0, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_OUT("OUT 1", NULL, 0, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_OUT("OUT 2", NULL, 0, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_OUT("OUT 3", NULL, 0, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_OUT("OUT 4", NULL, 0, SND_SOC_NOPM, 0, 0),
+};
+
+static const struct snd_soc_dapm_route g12a_frddr_dapm_routes[] = {
+	{ "SRC 1", NULL, "Playback" },
+	{ "SRC 2", NULL, "Playback" },
+	{ "SRC 3", NULL, "Playback" },
+	{ "SRC 1 EN", "Switch", "SRC 1" },
+	{ "SRC 2 EN", "Switch", "SRC 2" },
+	{ "SRC 3 EN", "Switch", "SRC 3" },
+	{ "SINK 1 SEL", NULL, "SRC 1 EN" },
+	{ "SINK 2 SEL", NULL, "SRC 2 EN" },
+	{ "SINK 3 SEL", NULL, "SRC 3 EN" },
+	{ "OUT 0", "OUT 0", "SINK 1 SEL" },
+	{ "OUT 1", "OUT 1", "SINK 1 SEL" },
+	{ "OUT 2", "OUT 2", "SINK 1 SEL" },
+	{ "OUT 3", "OUT 3", "SINK 1 SEL" },
+	{ "OUT 4", "OUT 4", "SINK 1 SEL" },
+	{ "OUT 0", "OUT 0", "SINK 2 SEL" },
+	{ "OUT 1", "OUT 1", "SINK 2 SEL" },
+	{ "OUT 2", "OUT 2", "SINK 2 SEL" },
+	{ "OUT 3", "OUT 3", "SINK 2 SEL" },
+	{ "OUT 4", "OUT 4", "SINK 2 SEL" },
+	{ "OUT 0", "OUT 0", "SINK 3 SEL" },
+	{ "OUT 1", "OUT 1", "SINK 3 SEL" },
+	{ "OUT 2", "OUT 2", "SINK 3 SEL" },
+	{ "OUT 3", "OUT 3", "SINK 3 SEL" },
+	{ "OUT 4", "OUT 4", "SINK 3 SEL" },
+};
+
+static const struct snd_soc_component_driver g12a_frddr_component_drv = {
+	.dapm_widgets		= g12a_frddr_dapm_widgets,
+	.num_dapm_widgets	= ARRAY_SIZE(g12a_frddr_dapm_widgets),
+	.dapm_routes		= g12a_frddr_dapm_routes,
+	.num_dapm_routes	= ARRAY_SIZE(g12a_frddr_dapm_routes),
+	.ops			= &g12a_fifo_pcm_ops
+};
+
+static const struct axg_fifo_match_data g12a_frddr_match_data = {
+	.component_drv	= &g12a_frddr_component_drv,
+	.dai_drv	= &g12a_frddr_dai_drv
+};
+
 static const struct of_device_id axg_frddr_of_match[] = {
 	{
 		.compatible = "amlogic,axg-frddr",
 		.data = &axg_frddr_match_data,
+	}, {
+		.compatible = "amlogic,g12a-frddr",
+		.data = &g12a_frddr_match_data,
 	}, {}
 };
 MODULE_DEVICE_TABLE(of, axg_frddr_of_match);
@@ -136,6 +273,6 @@ static struct platform_driver axg_frddr_pdrv = {
 };
 module_platform_driver(axg_frddr_pdrv);
 
-MODULE_DESCRIPTION("Amlogic AXG playback fifo driver");
+MODULE_DESCRIPTION("Amlogic AXG/G12A playback fifo driver");
 MODULE_AUTHOR("Jerome Brunet <jbrunet@baylibre.com>");
 MODULE_LICENSE("GPL v2");
diff --git a/sound/soc/meson/axg-tdm-formatter.c b/sound/soc/meson/axg-tdm-formatter.c
index 43e390f9358a..0c6cce5c5773 100644
--- a/sound/soc/meson/axg-tdm-formatter.c
+++ b/sound/soc/meson/axg-tdm-formatter.c
@@ -68,7 +68,7 @@ EXPORT_SYMBOL_GPL(axg_tdm_formatter_set_channel_masks);
 static int axg_tdm_formatter_enable(struct axg_tdm_formatter *formatter)
 {
 	struct axg_tdm_stream *ts = formatter->stream;
-	bool invert = formatter->drv->invert_sclk;
+	bool invert = formatter->drv->quirks->invert_sclk;
 	int ret;
 
 	/* Do nothing if the formatter is already enabled */
@@ -85,7 +85,9 @@ static int axg_tdm_formatter_enable(struct axg_tdm_formatter *formatter)
 		return ret;
 
 	/* Setup the stream parameter in the formatter */
-	ret = formatter->drv->ops->prepare(formatter->map, formatter->stream);
+	ret = formatter->drv->ops->prepare(formatter->map,
+					   formatter->drv->quirks,
+					   formatter->stream);
 	if (ret)
 		return ret;
 
diff --git a/sound/soc/meson/axg-tdm-formatter.h b/sound/soc/meson/axg-tdm-formatter.h
index cf947caf3cb1..9ef98e955cb2 100644
--- a/sound/soc/meson/axg-tdm-formatter.h
+++ b/sound/soc/meson/axg-tdm-formatter.h
@@ -14,18 +14,25 @@ struct regmap;
 struct snd_soc_dapm_widget;
 struct snd_kcontrol;
 
+struct axg_tdm_formatter_hw {
+	unsigned int skew_offset;
+	bool invert_sclk;
+};
+
 struct axg_tdm_formatter_ops {
 	struct axg_tdm_stream *(*get_stream)(struct snd_soc_dapm_widget *w);
 	void (*enable)(struct regmap *map);
 	void (*disable)(struct regmap *map);
-	int (*prepare)(struct regmap *map, struct axg_tdm_stream *ts);
+	int (*prepare)(struct regmap *map,
+		       const struct axg_tdm_formatter_hw *quirks,
+		       struct axg_tdm_stream *ts);
 };
 
 struct axg_tdm_formatter_driver {
 	const struct snd_soc_component_driver *component_drv;
 	const struct regmap_config *regmap_cfg;
 	const struct axg_tdm_formatter_ops *ops;
-	bool invert_sclk;
+	const struct axg_tdm_formatter_hw *quirks;
 };
 
 int axg_tdm_formatter_set_channel_masks(struct regmap *map,
diff --git a/sound/soc/meson/axg-tdmin.c b/sound/soc/meson/axg-tdmin.c
index bbac44c81688..a790f925a4ef 100644
--- a/sound/soc/meson/axg-tdmin.c
+++ b/sound/soc/meson/axg-tdmin.c
@@ -107,21 +107,22 @@ static void axg_tdmin_disable(struct regmap *map)
 	regmap_update_bits(map, TDMIN_CTRL, TDMIN_CTRL_ENABLE, 0);
 }
 
-static int axg_tdmin_prepare(struct regmap *map, struct axg_tdm_stream *ts)
+static int axg_tdmin_prepare(struct regmap *map,
+			     const struct axg_tdm_formatter_hw *quirks,
+			     struct axg_tdm_stream *ts)
 {
-	unsigned int val = 0;
+	unsigned int val, skew = quirks->skew_offset;
 
 	/* Set stream skew */
 	switch (ts->iface->fmt & SND_SOC_DAIFMT_FORMAT_MASK) {
 	case SND_SOC_DAIFMT_I2S:
 	case SND_SOC_DAIFMT_DSP_A:
-		val |= TDMIN_CTRL_IN_BIT_SKEW(3);
+		skew += 1;
 		break;
 
 	case SND_SOC_DAIFMT_LEFT_J:
 	case SND_SOC_DAIFMT_RIGHT_J:
 	case SND_SOC_DAIFMT_DSP_B:
-		val = TDMIN_CTRL_IN_BIT_SKEW(2);
 		break;
 
 	default:
@@ -130,6 +131,8 @@ static int axg_tdmin_prepare(struct regmap *map, struct axg_tdm_stream *ts)
 		return -EINVAL;
 	}
 
+	val = TDMIN_CTRL_IN_BIT_SKEW(skew);
+
 	/* Set stream format mode */
 	switch (ts->iface->fmt & SND_SOC_DAIFMT_FORMAT_MASK) {
 	case SND_SOC_DAIFMT_I2S:
@@ -204,7 +207,10 @@ static const struct axg_tdm_formatter_driver axg_tdmin_drv = {
 	.component_drv	= &axg_tdmin_component_drv,
 	.regmap_cfg	= &axg_tdmin_regmap_cfg,
 	.ops		= &axg_tdmin_ops,
-	.invert_sclk	= false,
+	.quirks		= &(const struct axg_tdm_formatter_hw) {
+		.invert_sclk	= false,
+		.skew_offset	= 2,
+	},
 };
 
 static const struct of_device_id axg_tdmin_of_match[] = {
diff --git a/sound/soc/meson/axg-tdmout.c b/sound/soc/meson/axg-tdmout.c
index f73368ee1088..527bfc4487e0 100644
--- a/sound/soc/meson/axg-tdmout.c
+++ b/sound/soc/meson/axg-tdmout.c
@@ -124,21 +124,22 @@ static void axg_tdmout_disable(struct regmap *map)
 	regmap_update_bits(map, TDMOUT_CTRL0, TDMOUT_CTRL0_ENABLE, 0);
 }
 
-static int axg_tdmout_prepare(struct regmap *map, struct axg_tdm_stream *ts)
+static int axg_tdmout_prepare(struct regmap *map,
+			      const struct axg_tdm_formatter_hw *quirks,
+			      struct axg_tdm_stream *ts)
 {
-	unsigned int val = 0;
+	unsigned int val, skew = quirks->skew_offset;
 
 	/* Set the stream skew */
 	switch (ts->iface->fmt & SND_SOC_DAIFMT_FORMAT_MASK) {
 	case SND_SOC_DAIFMT_I2S:
 	case SND_SOC_DAIFMT_DSP_A:
-		val |= TDMOUT_CTRL0_INIT_BITNUM(1);
 		break;
 
 	case SND_SOC_DAIFMT_LEFT_J:
 	case SND_SOC_DAIFMT_RIGHT_J:
 	case SND_SOC_DAIFMT_DSP_B:
-		val |= TDMOUT_CTRL0_INIT_BITNUM(2);
+		skew += 1;
 		break;
 
 	default:
@@ -147,6 +148,8 @@ static int axg_tdmout_prepare(struct regmap *map, struct axg_tdm_stream *ts)
 		return -EINVAL;
 	}
 
+	val = TDMOUT_CTRL0_INIT_BITNUM(skew);
+
 	/* Set the slot width */
 	val |= TDMOUT_CTRL0_BITNUM(ts->iface->slot_width - 1);
 
@@ -234,13 +237,29 @@ static const struct axg_tdm_formatter_driver axg_tdmout_drv = {
 	.component_drv	= &axg_tdmout_component_drv,
 	.regmap_cfg	= &axg_tdmout_regmap_cfg,
 	.ops		= &axg_tdmout_ops,
-	.invert_sclk	= true,
+	.quirks		= &(const struct axg_tdm_formatter_hw) {
+		.invert_sclk = true,
+		.skew_offset = 1,
+	},
+};
+
+static const struct axg_tdm_formatter_driver g12a_tdmout_drv = {
+	.component_drv	= &axg_tdmout_component_drv,
+	.regmap_cfg	= &axg_tdmout_regmap_cfg,
+	.ops		= &axg_tdmout_ops,
+	.quirks		= &(const struct axg_tdm_formatter_hw) {
+		.invert_sclk = true,
+		.skew_offset = 2,
+	},
 };
 
 static const struct of_device_id axg_tdmout_of_match[] = {
 	{
 		.compatible = "amlogic,axg-tdmout",
 		.data = &axg_tdmout_drv,
+	}, {
+		.compatible = "amlogic,g12a-tdmout",
+		.data = &g12a_tdmout_drv,
 	}, {}
 };
 MODULE_DEVICE_TABLE(of, axg_tdmout_of_match);
diff --git a/sound/soc/meson/axg-toddr.c b/sound/soc/meson/axg-toddr.c
index 0e9ca3882ae5..4f63e434fad4 100644
--- a/sound/soc/meson/axg-toddr.c
+++ b/sound/soc/meson/axg-toddr.c
@@ -24,6 +24,7 @@
 #define CTRL0_TODDR_MSB_POS(x)		((x) << 8)
 #define CTRL0_TODDR_LSB_POS_MASK	GENMASK(7, 3)
 #define CTRL0_TODDR_LSB_POS(x)		((x) << 3)
+#define CTRL1_TODDR_FORCE_FINISH	BIT(25)
 
 #define TODDR_MSB_POS	31
 
@@ -33,6 +34,22 @@ static int axg_toddr_pcm_new(struct snd_soc_pcm_runtime *rtd,
 	return axg_fifo_pcm_new(rtd, SNDRV_PCM_STREAM_CAPTURE);
 }
 
+static int g12a_toddr_dai_prepare(struct snd_pcm_substream *substream,
+				  struct snd_soc_dai *dai)
+{
+	struct axg_fifo *fifo = snd_soc_dai_get_drvdata(dai);
+
+	/* Reset the write pointer to the FIFO_INIT_ADDR */
+	regmap_update_bits(fifo->map, FIFO_CTRL1,
+			   CTRL1_TODDR_FORCE_FINISH, 0);
+	regmap_update_bits(fifo->map, FIFO_CTRL1,
+			   CTRL1_TODDR_FORCE_FINISH, CTRL1_TODDR_FORCE_FINISH);
+	regmap_update_bits(fifo->map, FIFO_CTRL1,
+			   CTRL1_TODDR_FORCE_FINISH, 0);
+
+	return 0;
+}
+
 static int axg_toddr_dai_hw_params(struct snd_pcm_substream *substream,
 				   struct snd_pcm_hw_params *params,
 				   struct snd_soc_dai *dai)
@@ -172,10 +189,46 @@ static const struct axg_fifo_match_data axg_toddr_match_data = {
 	.dai_drv	= &axg_toddr_dai_drv
 };
 
+static const struct snd_soc_dai_ops g12a_toddr_ops = {
+	.prepare	= g12a_toddr_dai_prepare,
+	.hw_params	= axg_toddr_dai_hw_params,
+	.startup	= axg_toddr_dai_startup,
+	.shutdown	= axg_toddr_dai_shutdown,
+};
+
+static struct snd_soc_dai_driver g12a_toddr_dai_drv = {
+	.name = "TODDR",
+	.capture = {
+		.stream_name	= "Capture",
+		.channels_min	= 1,
+		.channels_max	= AXG_FIFO_CH_MAX,
+		.rates		= AXG_FIFO_RATES,
+		.formats	= AXG_FIFO_FORMATS,
+	},
+	.ops		= &g12a_toddr_ops,
+	.pcm_new	= axg_toddr_pcm_new,
+};
+
+static const struct snd_soc_component_driver g12a_toddr_component_drv = {
+	.dapm_widgets		= axg_toddr_dapm_widgets,
+	.num_dapm_widgets	= ARRAY_SIZE(axg_toddr_dapm_widgets),
+	.dapm_routes		= axg_toddr_dapm_routes,
+	.num_dapm_routes	= ARRAY_SIZE(axg_toddr_dapm_routes),
+	.ops			= &g12a_fifo_pcm_ops
+};
+
+static const struct axg_fifo_match_data g12a_toddr_match_data = {
+	.component_drv	= &g12a_toddr_component_drv,
+	.dai_drv	= &g12a_toddr_dai_drv
+};
+
 static const struct of_device_id axg_toddr_of_match[] = {
 	{
 		.compatible = "amlogic,axg-toddr",
 		.data = &axg_toddr_match_data,
+	}, {
+		.compatible = "amlogic,g12a-toddr",
+		.data = &g12a_toddr_match_data,
 	}, {}
 };
 MODULE_DEVICE_TABLE(of, axg_toddr_of_match);
diff --git a/sound/soc/qcom/Kconfig b/sound/soc/qcom/Kconfig
index 75ceb04d8bf0..b1764af858ba 100644
--- a/sound/soc/qcom/Kconfig
+++ b/sound/soc/qcom/Kconfig
@@ -98,7 +98,7 @@ config SND_SOC_MSM8996
 
 config SND_SOC_SDM845
 	tristate "SoC Machine driver for SDM845 boards"
-	depends on QCOM_APR && MFD_CROS_EC
+	depends on QCOM_APR && MFD_CROS_EC && I2C
 	select SND_SOC_QDSP6
 	select SND_SOC_QCOM_COMMON
 	select SND_SOC_RT5663
diff --git a/sound/soc/rockchip/rockchip_pdm.c b/sound/soc/rockchip/rockchip_pdm.c
index d0b403a0e27b..b9c1d8ad77c1 100644
--- a/sound/soc/rockchip/rockchip_pdm.c
+++ b/sound/soc/rockchip/rockchip_pdm.c
@@ -17,14 +17,23 @@
 #include <linux/module.h>
 #include <linux/clk.h>
 #include <linux/of.h>
+#include <linux/of_device.h>
 #include <linux/pm_runtime.h>
+#include <linux/rational.h>
 #include <linux/regmap.h>
+#include <linux/reset.h>
 #include <sound/dmaengine_pcm.h>
 #include <sound/pcm_params.h>
 
 #include "rockchip_pdm.h"
 
 #define PDM_DMA_BURST_SIZE	(8) /* size * width: 8*4 = 32 bytes */
+#define PDM_SIGNOFF_CLK_RATE	(100000000)
+
+enum rk_pdm_version {
+	RK_PDM_RK3229,
+	RK_PDM_RK3308,
+};
 
 struct rk_pdm_dev {
 	struct device *dev;
@@ -32,22 +41,51 @@ struct rk_pdm_dev {
 	struct clk *hclk;
 	struct regmap *regmap;
 	struct snd_dmaengine_dai_dma_data capture_dma_data;
+	struct reset_control *reset;
+	enum rk_pdm_version version;
 };
 
 struct rk_pdm_clkref {
 	unsigned int sr;
 	unsigned int clk;
+	unsigned int clk_out;
+};
+
+struct rk_pdm_ds_ratio {
+	unsigned int ratio;
+	unsigned int sr;
 };
 
 static struct rk_pdm_clkref clkref[] = {
-	{ 8000, 40960000 },
-	{ 11025, 56448000 },
-	{ 12000, 61440000 },
+	{ 8000, 40960000, 2048000 },
+	{ 11025, 56448000, 2822400 },
+	{ 12000, 61440000, 3072000 },
+	{ 8000, 98304000, 2048000 },
+	{ 12000, 98304000, 3072000 },
+};
+
+static struct rk_pdm_ds_ratio ds_ratio[] = {
+	{ 0, 192000 },
+	{ 0, 176400 },
+	{ 0, 128000 },
+	{ 1, 96000 },
+	{ 1, 88200 },
+	{ 1, 64000 },
+	{ 2, 48000 },
+	{ 2, 44100 },
+	{ 2, 32000 },
+	{ 3, 24000 },
+	{ 3, 22050 },
+	{ 3, 16000 },
+	{ 4, 12000 },
+	{ 4, 11025 },
+	{ 4, 8000 },
 };
 
-static unsigned int get_pdm_clk(unsigned int sr)
+static unsigned int get_pdm_clk(struct rk_pdm_dev *pdm, unsigned int sr,
+				unsigned int *clk_src, unsigned int *clk_out)
 {
-	unsigned int i, count, clk, div;
+	unsigned int i, count, clk, div, rate;
 
 	clk = 0;
 	if (!sr)
@@ -59,14 +97,39 @@ static unsigned int get_pdm_clk(unsigned int sr)
 			continue;
 		div = sr / clkref[i].sr;
 		if ((div & (div - 1)) == 0) {
+			*clk_out = clkref[i].clk_out;
+			rate = clk_round_rate(pdm->clk, clkref[i].clk);
+			if (rate != clkref[i].clk)
+				continue;
 			clk = clkref[i].clk;
+			*clk_src = clkref[i].clk;
 			break;
 		}
 	}
 
+	if (!clk) {
+		clk = clk_round_rate(pdm->clk, PDM_SIGNOFF_CLK_RATE);
+		*clk_src = clk;
+	}
 	return clk;
 }
 
+static unsigned int get_pdm_ds_ratio(unsigned int sr)
+{
+	unsigned int i, count, ratio;
+
+	ratio = 0;
+	if (!sr)
+		return ratio;
+
+	count = ARRAY_SIZE(ds_ratio);
+	for (i = 0; i < count; i++) {
+		if (sr == ds_ratio[i].sr)
+			ratio = ds_ratio[i].ratio;
+	}
+	return ratio;
+}
+
 static inline struct rk_pdm_dev *to_info(struct snd_soc_dai *dai)
 {
 	return snd_soc_dai_get_drvdata(dai);
@@ -95,46 +158,61 @@ static int rockchip_pdm_hw_params(struct snd_pcm_substream *substream,
 	struct rk_pdm_dev *pdm = to_info(dai);
 	unsigned int val = 0;
 	unsigned int clk_rate, clk_div, samplerate;
+	unsigned int clk_src, clk_out = 0;
+	unsigned long m, n;
+	bool change;
 	int ret;
 
+	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
+		return 0;
+
 	samplerate = params_rate(params);
-	clk_rate = get_pdm_clk(samplerate);
+	clk_rate = get_pdm_clk(pdm, samplerate, &clk_src, &clk_out);
 	if (!clk_rate)
 		return -EINVAL;
 
-	ret = clk_set_rate(pdm->clk, clk_rate);
+	ret = clk_set_rate(pdm->clk, clk_src);
 	if (ret)
 		return -EINVAL;
 
-	clk_div = DIV_ROUND_CLOSEST(clk_rate, samplerate);
-
-	switch (clk_div) {
-	case 320:
-		val = PDM_CLK_320FS;
-		break;
-	case 640:
-		val = PDM_CLK_640FS;
-		break;
-	case 1280:
-		val = PDM_CLK_1280FS;
-		break;
-	case 2560:
-		val = PDM_CLK_2560FS;
-		break;
-	case 5120:
-		val = PDM_CLK_5120FS;
-		break;
-	default:
-		dev_err(pdm->dev, "unsupported div: %d\n", clk_div);
-		return -EINVAL;
+	if (pdm->version == RK_PDM_RK3308) {
+		rational_best_approximation(clk_out, clk_src,
+					    GENMASK(16 - 1, 0),
+					    GENMASK(16 - 1, 0),
+					    &m, &n);
+
+		val = (m << PDM_FD_NUMERATOR_SFT) |
+			(n << PDM_FD_DENOMINATOR_SFT);
+		regmap_update_bits_check(pdm->regmap, PDM_CTRL1,
+					 PDM_FD_NUMERATOR_MSK |
+					 PDM_FD_DENOMINATOR_MSK,
+					 val, &change);
+		if (change) {
+			reset_control_assert(pdm->reset);
+			reset_control_deassert(pdm->reset);
+			rockchip_pdm_rxctrl(pdm, 0);
+		}
+		clk_div = n / m;
+		if (clk_div >= 40)
+			val = PDM_CLK_FD_RATIO_40;
+		else if (clk_div <= 35)
+			val = PDM_CLK_FD_RATIO_35;
+		else
+			return -EINVAL;
+		regmap_update_bits(pdm->regmap, PDM_CLK_CTRL,
+				   PDM_CLK_FD_RATIO_MSK,
+				   val);
 	}
-
+	val = get_pdm_ds_ratio(samplerate);
 	regmap_update_bits(pdm->regmap, PDM_CLK_CTRL, PDM_DS_RATIO_MSK, val);
 	regmap_update_bits(pdm->regmap, PDM_HPF_CTRL,
 			   PDM_HPF_CF_MSK, PDM_HPF_60HZ);
 	regmap_update_bits(pdm->regmap, PDM_HPF_CTRL,
 			   PDM_HPF_LE | PDM_HPF_RE, PDM_HPF_LE | PDM_HPF_RE);
 	regmap_update_bits(pdm->regmap, PDM_CLK_CTRL, PDM_CLK_EN, PDM_CLK_EN);
+	if (pdm->version != RK_PDM_RK3229)
+		regmap_update_bits(pdm->regmap, PDM_CTRL0,
+				   PDM_MODE_MSK, PDM_MODE_LJ);
 
 	val = 0;
 	switch (params_format(params)) {
@@ -176,16 +254,12 @@ static int rockchip_pdm_hw_params(struct snd_pcm_substream *substream,
 		return -EINVAL;
 	}
 
-	if (substream->stream == SNDRV_PCM_STREAM_CAPTURE) {
-		regmap_update_bits(pdm->regmap, PDM_CTRL0,
-				   PDM_PATH_MSK | PDM_VDW_MSK,
-				   val);
-		regmap_update_bits(pdm->regmap, PDM_DMA_CTRL, PDM_DMA_RDL_MSK,
-				   PDM_DMA_RDL(16));
-		regmap_update_bits(pdm->regmap, PDM_SYSCONFIG,
-				   PDM_RX_MASK | PDM_RX_CLR_MASK,
-				   PDM_RX_STOP | PDM_RX_CLR_WR);
-	}
+	regmap_update_bits(pdm->regmap, PDM_CTRL0,
+			   PDM_PATH_MSK | PDM_VDW_MSK,
+			   val);
+	/* all channels share the single FIFO */
+	regmap_update_bits(pdm->regmap, PDM_DMA_CTRL, PDM_DMA_RDL_MSK,
+			   PDM_DMA_RDL(8 * params_channels(params)));
 
 	return 0;
 }
@@ -343,6 +417,7 @@ static bool rockchip_pdm_rd_reg(struct device *dev, unsigned int reg)
 	case PDM_INT_CLR:
 	case PDM_INT_ST:
 	case PDM_DATA_VALID:
+	case PDM_RXFIFO_DATA:
 	case PDM_VERSION:
 		return true;
 	default:
@@ -354,27 +429,62 @@ static bool rockchip_pdm_volatile_reg(struct device *dev, unsigned int reg)
 {
 	switch (reg) {
 	case PDM_SYSCONFIG:
+	case PDM_FIFO_CTRL:
 	case PDM_INT_CLR:
 	case PDM_INT_ST:
+	case PDM_RXFIFO_DATA:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static bool rockchip_pdm_precious_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case PDM_RXFIFO_DATA:
 		return true;
 	default:
 		return false;
 	}
 }
 
+static const struct reg_default rockchip_pdm_reg_defaults[] = {
+	{0x04, 0x78000017},
+	{0x08, 0x0bb8ea60},
+	{0x18, 0x0000001f},
+};
+
 static const struct regmap_config rockchip_pdm_regmap_config = {
 	.reg_bits = 32,
 	.reg_stride = 4,
 	.val_bits = 32,
 	.max_register = PDM_VERSION,
+	.reg_defaults = rockchip_pdm_reg_defaults,
+	.num_reg_defaults = ARRAY_SIZE(rockchip_pdm_reg_defaults),
 	.writeable_reg = rockchip_pdm_wr_reg,
 	.readable_reg = rockchip_pdm_rd_reg,
 	.volatile_reg = rockchip_pdm_volatile_reg,
+	.precious_reg = rockchip_pdm_precious_reg,
 	.cache_type = REGCACHE_FLAT,
 };
 
+static const struct of_device_id rockchip_pdm_match[] = {
+	{ .compatible = "rockchip,pdm",
+	  .data = (void *)RK_PDM_RK3229 },
+	{ .compatible = "rockchip,px30-pdm",
+	  .data = (void *)RK_PDM_RK3308 },
+	{ .compatible = "rockchip,rk1808-pdm",
+	  .data = (void *)RK_PDM_RK3308 },
+	{ .compatible = "rockchip,rk3308-pdm",
+	  .data = (void *)RK_PDM_RK3308 },
+	{},
+};
+MODULE_DEVICE_TABLE(of, rockchip_pdm_match);
+
 static int rockchip_pdm_probe(struct platform_device *pdev)
 {
+	const struct of_device_id *match;
 	struct rk_pdm_dev *pdm;
 	struct resource *res;
 	void __iomem *regs;
@@ -384,6 +494,16 @@ static int rockchip_pdm_probe(struct platform_device *pdev)
 	if (!pdm)
 		return -ENOMEM;
 
+	match = of_match_device(rockchip_pdm_match, &pdev->dev);
+	if (match)
+		pdm->version = (enum rk_pdm_version)match->data;
+
+	if (pdm->version == RK_PDM_RK3308) {
+		pdm->reset = devm_reset_control_get(&pdev->dev, "pdm-m");
+		if (IS_ERR(pdm->reset))
+			return PTR_ERR(pdm->reset);
+	}
+
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	regs = devm_ioremap_resource(&pdev->dev, res);
 	if (IS_ERR(regs))
@@ -429,6 +549,7 @@ static int rockchip_pdm_probe(struct platform_device *pdev)
 		goto err_suspend;
 	}
 
+	rockchip_pdm_rxctrl(pdm, 0);
 	ret = devm_snd_dmaengine_pcm_register(&pdev->dev, NULL, 0);
 	if (ret) {
 		dev_err(&pdev->dev, "could not register pcm: %d\n", ret);
@@ -495,12 +616,6 @@ static const struct dev_pm_ops rockchip_pdm_pm_ops = {
 	SET_SYSTEM_SLEEP_PM_OPS(rockchip_pdm_suspend, rockchip_pdm_resume)
 };
 
-static const struct of_device_id rockchip_pdm_match[] = {
-	{ .compatible = "rockchip,pdm", },
-	{},
-};
-MODULE_DEVICE_TABLE(of, rockchip_pdm_match);
-
 static struct platform_driver rockchip_pdm_driver = {
 	.probe  = rockchip_pdm_probe,
 	.remove = rockchip_pdm_remove,
diff --git a/sound/soc/rockchip/rockchip_pdm.h b/sound/soc/rockchip/rockchip_pdm.h
index 886b48d128fd..ae88644aa334 100644
--- a/sound/soc/rockchip/rockchip_pdm.h
+++ b/sound/soc/rockchip/rockchip_pdm.h
@@ -42,6 +42,9 @@
 
 /* PDM CTRL0 */
 #define PDM_PATH_MSK		(0xf << 27)
+#define PDM_MODE_MSK		BIT(31)
+#define PDM_MODE_RJ		0
+#define PDM_MODE_LJ		BIT(31)
 #define PDM_PATH3_EN		BIT(30)
 #define PDM_PATH2_EN		BIT(29)
 #define PDM_PATH1_EN		BIT(28)
@@ -50,7 +53,16 @@
 #define PDM_VDW_MSK		(0x1f << 0)
 #define PDM_VDW(X)		((X - 1) << 0)
 
+/* PDM CTRL1 */
+#define PDM_FD_NUMERATOR_SFT	16
+#define PDM_FD_NUMERATOR_MSK	GENMASK(31, 16)
+#define PDM_FD_DENOMINATOR_SFT	0
+#define PDM_FD_DENOMINATOR_MSK	GENMASK(15, 0)
+
 /* PDM CLK CTRL */
+#define PDM_CLK_FD_RATIO_MSK	BIT(6)
+#define PDM_CLK_FD_RATIO_40	(0X0 << 6)
+#define PDM_CLK_FD_RATIO_35	BIT(6)
 #define PDM_CLK_MSK		BIT(5)
 #define PDM_CLK_EN		BIT(5)
 #define PDM_CLK_DIS		(0x0 << 5)
diff --git a/sound/soc/samsung/arndale_rt5631.c b/sound/soc/samsung/arndale_rt5631.c
index ee1fda92f2f4..cc334e1866f6 100644
--- a/sound/soc/samsung/arndale_rt5631.c
+++ b/sound/soc/samsung/arndale_rt5631.c
@@ -1,15 +1,8 @@
-/*
- *  arndale_rt5631.c
- *
- *  Copyright (c) 2014, Insignal Co., Ltd.
- *
- *  Author: Claude <claude@insginal.co.kr>
- *
- *  This program is free software; you can redistribute  it and/or modify it
- *  under  the terms of  the GNU General  Public License as published by the
- *  Free Software Foundation;  either version 2 of the  License, or (at your
- *  option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright (c) 2014, Insignal Co., Ltd.
+//
+//  Author: Claude <claude@insginal.co.kr>
 
 #include <linux/module.h>
 #include <linux/platform_device.h>
diff --git a/sound/soc/samsung/bells.c b/sound/soc/samsung/bells.c
index 0e66cd8ef2f9..770845e2507a 100644
--- a/sound/soc/samsung/bells.c
+++ b/sound/soc/samsung/bells.c
@@ -1,13 +1,8 @@
-/*
- * Bells audio support
- *
- * Copyright 2012 Wolfson Microelectronics
- *
- * This program is free software; you can redistribute  it and/or modify it
- * under  the terms of  the GNU General  Public License as published by the
- * Free Software Foundation;  either version 2 of the  License, or (at your
- * option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Bells audio support
+//
+// Copyright 2012 Wolfson Microelectronics
 
 #include <sound/soc.h>
 #include <sound/soc-dapm.h>
diff --git a/sound/soc/samsung/dma.h b/sound/soc/samsung/dma.h
index 0ae15d01a3f6..7b5d4556e0fd 100644
--- a/sound/soc/samsung/dma.h
+++ b/sound/soc/samsung/dma.h
@@ -1,10 +1,6 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
- *  This program is free software; you can redistribute  it and/or modify it
- *  under  the terms of  the GNU General  Public License as published by the
- *  Free Software Foundation;  either version 2 of the  License, or (at your
- *  option) any later version.
- *
- *  ALSA PCM interface for the Samsung SoC
+ * ALSA PCM interface for the Samsung SoC
  */
 
 #ifndef _SAMSUNG_DMA_H
diff --git a/sound/soc/samsung/dmaengine.c b/sound/soc/samsung/dmaengine.c
index 302871974cb3..2802789a323e 100644
--- a/sound/soc/samsung/dmaengine.c
+++ b/sound/soc/samsung/dmaengine.c
@@ -1,19 +1,9 @@
-/*
- * dmaengine.c - Samsung dmaengine wrapper
- *
- * Author: Mark Brown <broonie@linaro.org>
- * Copyright 2013 Linaro
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// dmaengine.c - Samsung dmaengine wrapper
+//
+// Author: Mark Brown <broonie@linaro.org>
+// Copyright 2013 Linaro
 
 #include <linux/module.h>
 #include <sound/core.h>
diff --git a/sound/soc/samsung/h1940_uda1380.c b/sound/soc/samsung/h1940_uda1380.c
index 051935162d7b..95925c4a5964 100644
--- a/sound/soc/samsung/h1940_uda1380.c
+++ b/sound/soc/samsung/h1940_uda1380.c
@@ -1,17 +1,11 @@
-/*
- * h1940-uda1380.c  --  ALSA Soc Audio Layer
- *
- * Copyright (c) 2010 Arnaud Patard <arnaud.patard@rtp-net.org>
- * Copyright (c) 2010 Vasily Khoruzhick <anarsoul@gmail.com>
- *
- * Based on version from Arnaud Patard <arnaud.patard@rtp-net.org>
- *
- * This program is free software; you can redistribute  it and/or modify it
- * under  the terms of  the GNU General  Public License as published by the
- * Free Software Foundation;  either version 2 of the  License, or (at your
- * option) any later version.
- *
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// h1940_uda1380.c - ALSA SoC Audio Layer
+//
+// Copyright (c) 2010 Arnaud Patard <arnaud.patard@rtp-net.org>
+// Copyright (c) 2010 Vasily Khoruzhick <anarsoul@gmail.com>
+//
+// Based on version from Arnaud Patard <arnaud.patard@rtp-net.org>
 
 #include <linux/types.h>
 #include <linux/gpio.h>
diff --git a/sound/soc/samsung/i2s-regs.h b/sound/soc/samsung/i2s-regs.h
index 964985ea2e80..b4b5d6053503 100644
--- a/sound/soc/samsung/i2s-regs.h
+++ b/sound/soc/samsung/i2s-regs.h
@@ -1,15 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
- * linux/sound/soc/samsung/i2s-regs.h
- *
  * Copyright (c) 2011 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com
  *
  * Samsung I2S driver's register header
- *
- * This program is free software; you can redistribute  it and/or modify it
- * under  the terms of  the GNU General  Public License as published by the
- * Free Software Foundation;  either version 2 of the  License, or (at your
- * option) any later version.
  */
 
 #ifndef __SND_SOC_SAMSUNG_I2S_REGS_H
diff --git a/sound/soc/samsung/i2s.c b/sound/soc/samsung/i2s.c
index ab471d550d17..9722940da6a4 100644
--- a/sound/soc/samsung/i2s.c
+++ b/sound/soc/samsung/i2s.c
@@ -88,12 +88,6 @@ struct samsung_i2s_priv {
 	struct platform_device *pdev;
 	struct platform_device *pdev_sec;
 
-	/* Memory mapped SFR region */
-	void __iomem *addr;
-
-	/* Spinlock protecting access to the device's registers */
-	spinlock_t lock;
-
 	/* Lock for cross interface checks */
 	spinlock_t pcm_lock;
 
@@ -122,6 +116,15 @@ struct samsung_i2s_priv {
 	/* The clock provider's data */
 	struct clk *clk_table[3];
 	struct clk_onecell_data clk_data;
+
+	/* Spinlock protecting member fields below */
+	spinlock_t lock;
+
+	/* Memory mapped SFR region */
+	void __iomem *addr;
+
+	/* A flag indicating the I2S slave mode operation */
+	bool slave_mode;
 };
 
 /* Returns true if this is the 'overlay' stereo DAI */
@@ -130,15 +133,6 @@ static inline bool is_secondary(struct i2s_dai *i2s)
 	return i2s->drv->id == SAMSUNG_I2S_ID_SECONDARY;
 }
 
-/* If operating in SoC-Slave mode */
-static inline bool is_slave(struct i2s_dai *i2s)
-{
-	struct samsung_i2s_priv *priv = i2s->priv;
-
-	u32 mod = readl(priv->addr + I2SMOD);
-	return (mod & (1 << priv->variant_regs->mss_off)) ? true : false;
-}
-
 /* If this interface of the controller is transmitting data */
 static inline bool tx_active(struct i2s_dai *i2s)
 {
@@ -715,6 +709,7 @@ static int i2s_set_fmt(struct snd_soc_dai *dai, unsigned int fmt)
 	mod &= ~(sdf_mask | lrp_rlow | mod_slave);
 	mod |= tmp;
 	writel(mod, priv->addr + I2SMOD);
+	priv->slave_mode = (mod & mod_slave);
 	spin_unlock_irqrestore(&priv->lock, flags);
 	pm_runtime_put(dai->dev);
 
@@ -917,7 +912,7 @@ static int config_setup(struct i2s_dai *i2s)
 	set_rfs(i2s, rfs);
 
 	/* Don't bother with PSR in Slave mode */
-	if (is_slave(i2s))
+	if (priv->slave_mode)
 		return 0;
 
 	if (!(priv->quirks & QUIRK_NO_MUXPSR)) {
diff --git a/sound/soc/samsung/i2s.h b/sound/soc/samsung/i2s.h
index a9832a9555cb..78b475ef98d9 100644
--- a/sound/soc/samsung/i2s.h
+++ b/sound/soc/samsung/i2s.h
@@ -1,13 +1,9 @@
-/* sound/soc/samsung/i2s.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * ALSA SoC Audio Layer - Samsung I2S Controller driver
  *
  * Copyright (c) 2010 Samsung Electronics Co. Ltd.
  *	Jaswinder Singh <jassisinghbrar@gmail.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #ifndef __SND_SOC_SAMSUNG_I2S_H
diff --git a/sound/soc/samsung/idma.c b/sound/soc/samsung/idma.c
index b1f09b942410..65497cd477a5 100644
--- a/sound/soc/samsung/idma.c
+++ b/sound/soc/samsung/idma.c
@@ -1,16 +1,10 @@
-/*
- * sound/soc/samsung/idma.c
- *
- * Copyright (c) 2011 Samsung Electronics Co., Ltd.
- *		http://www.samsung.com
- *
- * I2S0's Internal DMA driver
- *
- * This program is free software; you can redistribute  it and/or modify it
- * under  the terms of  the GNU General  Public License as published by the
- * Free Software Foundation;  either version 2 of the  License, or (at your
- * option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// idma.c - I2S0 internal DMA driver
+//
+// Copyright (c) 2011 Samsung Electronics Co., Ltd.
+//		http://www.samsung.com
+
 #include <linux/interrupt.h>
 #include <linux/platform_device.h>
 #include <linux/dma-mapping.h>
diff --git a/sound/soc/samsung/idma.h b/sound/soc/samsung/idma.h
index 8644946973e5..8a46a918ed2a 100644
--- a/sound/soc/samsung/idma.h
+++ b/sound/soc/samsung/idma.h
@@ -1,14 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
- * sound/soc/samsung/idma.h
- *
  * Copyright (c) 2011 Samsung Electronics Co., Ltd
  *		http://www.samsung.com
- *
- *  This program is free software; you can redistribute  it and/or modify it
- *  under  the terms of  the GNU General  Public License as published by the
- *  Free Software Foundation;  either version 2 of the  License, or (at your
- *  option) any later version.
- *
  */
 
 #ifndef __SND_SOC_SAMSUNG_IDMA_H_
diff --git a/sound/soc/samsung/jive_wm8750.c b/sound/soc/samsung/jive_wm8750.c
index 529b10dc532b..f05f9e03f07d 100644
--- a/sound/soc/samsung/jive_wm8750.c
+++ b/sound/soc/samsung/jive_wm8750.c
@@ -1,15 +1,10 @@
-/* sound/soc/samsung/jive_wm8750.c
- *
- * Copyright 2007,2008 Simtec Electronics
- *
- * Based on sound/soc/pxa/spitz.c
- *	Copyright 2005 Wolfson Microelectronics PLC.
- *	Copyright 2005 Openedhand Ltd.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2007,2008 Simtec Electronics
+//
+// Based on sound/soc/pxa/spitz.c
+//	Copyright 2005 Wolfson Microelectronics PLC.
+//	Copyright 2005 Openedhand Ltd.
 
 #include <linux/module.h>
 #include <sound/soc.h>
diff --git a/sound/soc/samsung/littlemill.c b/sound/soc/samsung/littlemill.c
index 087f8d738dfb..cd70b06cc99d 100644
--- a/sound/soc/samsung/littlemill.c
+++ b/sound/soc/samsung/littlemill.c
@@ -1,13 +1,8 @@
-/*
- * Littlemill audio support
- *
- * Copyright 2011 Wolfson Microelectronics
- *
- * This program is free software; you can redistribute  it and/or modify it
- * under  the terms of  the GNU General  Public License as published by the
- * Free Software Foundation;  either version 2 of the  License, or (at your
- * option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Littlemill audio support
+//
+// Copyright 2011 Wolfson Microelectronics
 
 #include <sound/soc.h>
 #include <sound/soc-dapm.h>
diff --git a/sound/soc/samsung/lowland.c b/sound/soc/samsung/lowland.c
index c9081f42f373..2fdab2ac8e8c 100644
--- a/sound/soc/samsung/lowland.c
+++ b/sound/soc/samsung/lowland.c
@@ -1,13 +1,8 @@
-/*
- * Lowland audio support
- *
- * Copyright 2011 Wolfson Microelectronics
- *
- * This program is free software; you can redistribute  it and/or modify it
- * under  the terms of  the GNU General  Public License as published by the
- * Free Software Foundation;  either version 2 of the  License, or (at your
- * option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Lowland audio support
+//
+// Copyright 2011 Wolfson Microelectronics
 
 #include <sound/soc.h>
 #include <sound/soc-dapm.h>
diff --git a/sound/soc/samsung/neo1973_wm8753.c b/sound/soc/samsung/neo1973_wm8753.c
index 65602b935377..7e625066ddcd 100644
--- a/sound/soc/samsung/neo1973_wm8753.c
+++ b/sound/soc/samsung/neo1973_wm8753.c
@@ -1,18 +1,13 @@
-/*
- * neo1973_wm8753.c  --  SoC audio for Openmoko Neo1973 and Freerunner devices
- *
- * Copyright 2007 Openmoko Inc
- * Author: Graeme Gregory <graeme@openmoko.org>
- * Copyright 2007 Wolfson Microelectronics PLC.
- * Author: Graeme Gregory
- *         graeme.gregory@wolfsonmicro.com or linux@wolfsonmicro.com
- * Copyright 2009 Wolfson Microelectronics
- *
- *  This program is free software; you can redistribute  it and/or modify it
- *  under  the terms of  the GNU General  Public License as published by the
- *  Free Software Foundation;  either version 2 of the  License, or (at your
- *  option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// neo1973_wm8753.c - SoC audio for Openmoko Neo1973 and Freerunner devices
+//
+// Copyright 2007 Openmoko Inc
+// Author: Graeme Gregory <graeme@openmoko.org>
+// Copyright 2007 Wolfson Microelectronics PLC.
+// Author: Graeme Gregory
+//         graeme.gregory@wolfsonmicro.com or linux@wolfsonmicro.com
+// Copyright 2009 Wolfson Microelectronics
 
 #include <linux/module.h>
 #include <linux/platform_device.h>
diff --git a/sound/soc/samsung/odroid.c b/sound/soc/samsung/odroid.c
index 1dc54c4206f0..e688169ff12a 100644
--- a/sound/soc/samsung/odroid.c
+++ b/sound/soc/samsung/odroid.c
@@ -1,10 +1,6 @@
-/*
- * Copyright (C) 2017 Samsung Electronics Co., Ltd.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (C) 2017 Samsung Electronics Co., Ltd.
 
 #include <linux/clk.h>
 #include <linux/clk-provider.h>
diff --git a/sound/soc/samsung/pcm.c b/sound/soc/samsung/pcm.c
index 3c7baa561084..f6e67d0e7882 100644
--- a/sound/soc/samsung/pcm.c
+++ b/sound/soc/samsung/pcm.c
@@ -1,15 +1,10 @@
-/* sound/soc/samsung/pcm.c
- *
- * ALSA SoC Audio Layer - S3C PCM-Controller driver
- *
- * Copyright (c) 2009 Samsung Electronics Co. Ltd
- * Author: Jaswinder Singh <jassisinghbrar@gmail.com>
- * based upon I2S drivers by Ben Dooks.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// ALSA SoC Audio Layer - S3C PCM-Controller driver
+//
+// Copyright (c) 2009 Samsung Electronics Co. Ltd
+// Author: Jaswinder Singh <jassisinghbrar@gmail.com>
+// based upon I2S drivers by Ben Dooks.
 
 #include <linux/clk.h>
 #include <linux/io.h>
diff --git a/sound/soc/samsung/pcm.h b/sound/soc/samsung/pcm.h
index 726baf814613..208d8da27de1 100644
--- a/sound/soc/samsung/pcm.h
+++ b/sound/soc/samsung/pcm.h
@@ -1,10 +1,4 @@
-/*  sound/soc/samsung/pcm.h
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
- */
+/* SPDX-License-Identifier: GPL-2.0 */
 
 #ifndef __S3C_PCM_H
 #define __S3C_PCM_H __FILE__
diff --git a/sound/soc/samsung/regs-i2s-v2.h b/sound/soc/samsung/regs-i2s-v2.h
index 5e5e5680580b..867984e75709 100644
--- a/sound/soc/samsung/regs-i2s-v2.h
+++ b/sound/soc/samsung/regs-i2s-v2.h
@@ -1,14 +1,10 @@
-/* linux/include/asm-arm/plat-s3c24xx/regs-s3c2412-iis.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright 2007 Simtec Electronics <linux@simtec.co.uk>
  *	http://armlinux.simtec.co.uk/
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
  * S3C2412 IIS register definition
-*/
+ */
 
 #ifndef __ASM_ARCH_REGS_S3C2412_IIS_H
 #define __ASM_ARCH_REGS_S3C2412_IIS_H
diff --git a/sound/soc/samsung/regs-iis.h b/sound/soc/samsung/regs-iis.h
index dc6cbbe9c4f0..253e172ad3b6 100644
--- a/sound/soc/samsung/regs-iis.h
+++ b/sound/soc/samsung/regs-iis.h
@@ -1,13 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (c) 2003 Simtec Electronics <linux@simtec.co.uk>
  *		      http://www.simtec.co.uk/products/SWLINUX/
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
  * S3C2410 IIS register definition
-*/
+ */
 
 #ifndef __SAMSUNG_REGS_IIS_H__
 #define __SAMSUNG_REGS_IIS_H__
diff --git a/sound/soc/samsung/rx1950_uda1380.c b/sound/soc/samsung/rx1950_uda1380.c
index a064ca7d78c3..1dcc1b252ad1 100644
--- a/sound/soc/samsung/rx1950_uda1380.c
+++ b/sound/soc/samsung/rx1950_uda1380.c
@@ -1,21 +1,15 @@
-/*
- * rx1950.c  --  ALSA Soc Audio Layer
- *
- * Copyright (c) 2010 Vasily Khoruzhick <anarsoul@gmail.com>
- *
- * Based on smdk2440.c and magician.c
- *
- * Authors: Graeme Gregory graeme.gregory@wolfsonmicro.com
- *          Philipp Zabel <philipp.zabel@gmail.com>
- *          Denis Grigoriev <dgreenday@gmail.com>
- *          Vasily Khoruzhick <anarsoul@gmail.com>
- *
- *  This program is free software; you can redistribute  it and/or modify it
- *  under  the terms of  the GNU General  Public License as published by the
- *  Free Software Foundation;  either version 2 of the  License, or (at your
- *  option) any later version.
- *
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// rx1950.c - ALSA SoC Audio Layer
+//
+// Copyright (c) 2010 Vasily Khoruzhick <anarsoul@gmail.com>
+//
+// Based on smdk2440.c and magician.c
+//
+// Authors: Graeme Gregory graeme.gregory@wolfsonmicro.com
+//          Philipp Zabel <philipp.zabel@gmail.com>
+//          Denis Grigoriev <dgreenday@gmail.com>
+//          Vasily Khoruzhick <anarsoul@gmail.com>
 
 #include <linux/types.h>
 #include <linux/gpio.h>
diff --git a/sound/soc/samsung/s3c-i2s-v2.c b/sound/soc/samsung/s3c-i2s-v2.c
index 58c3e9bfc6b7..7e196b599be1 100644
--- a/sound/soc/samsung/s3c-i2s-v2.c
+++ b/sound/soc/samsung/s3c-i2s-v2.c
@@ -1,18 +1,14 @@
-/* ALSA Soc Audio Layer - I2S core for newer Samsung SoCs.
- *
- * Copyright (c) 2006 Wolfson Microelectronics PLC.
- *	Graeme Gregory graeme.gregory@wolfsonmicro.com
- *	linux@wolfsonmicro.com
- *
- * Copyright (c) 2008, 2007, 2004-2005 Simtec Electronics
- *	http://armlinux.simtec.co.uk/
- *	Ben Dooks <ben@simtec.co.uk>
- *
- * This program is free software; you can redistribute  it and/or modify it
- * under  the terms of  the GNU General  Public License as published by the
- * Free Software Foundation;  either version 2 of the  License, or (at your
- * option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// ALSA Soc Audio Layer - I2S core for newer Samsung SoCs.
+//
+// Copyright (c) 2006 Wolfson Microelectronics PLC.
+//	Graeme Gregory graeme.gregory@wolfsonmicro.com
+//	linux@wolfsonmicro.com
+//
+// Copyright (c) 2008, 2007, 2004-2005 Simtec Electronics
+//	http://armlinux.simtec.co.uk/
+//	Ben Dooks <ben@simtec.co.uk>
 
 #include <linux/module.h>
 #include <linux/delay.h>
diff --git a/sound/soc/samsung/s3c-i2s-v2.h b/sound/soc/samsung/s3c-i2s-v2.h
index 3fca20f7a853..fe42b77999fd 100644
--- a/sound/soc/samsung/s3c-i2s-v2.h
+++ b/sound/soc/samsung/s3c-i2s-v2.h
@@ -1,16 +1,11 @@
-/* sound/soc/samsung/s3c-i2s-v2.h
- *
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
  * ALSA Soc Audio Layer - S3C_I2SV2 I2S driver
  *
  * Copyright (c) 2007 Simtec Electronics
  *	http://armlinux.simtec.co.uk/
  *	Ben Dooks <ben@simtec.co.uk>
- *
- *  This program is free software; you can redistribute  it and/or modify it
- *  under  the terms of  the GNU General  Public License as published by the
- *  Free Software Foundation;  either version 2 of the  License, or (at your
- *  option) any later version.
-*/
+ */
 
 /* This code is the core support for the I2S block found in a number of
  * Samsung SoC devices which is unofficially named I2S-V2. Currently the
diff --git a/sound/soc/samsung/s3c2412-i2s.c b/sound/soc/samsung/s3c2412-i2s.c
index c08638b0e458..787a3f6e9f24 100644
--- a/sound/soc/samsung/s3c2412-i2s.c
+++ b/sound/soc/samsung/s3c2412-i2s.c
@@ -1,20 +1,14 @@
-/* sound/soc/samsung/s3c2412-i2s.c
- *
- * ALSA Soc Audio Layer - S3C2412 I2S driver
- *
- * Copyright (c) 2006 Wolfson Microelectronics PLC.
- *	Graeme Gregory graeme.gregory@wolfsonmicro.com
- *	linux@wolfsonmicro.com
- *
- * Copyright (c) 2007, 2004-2005 Simtec Electronics
- *	http://armlinux.simtec.co.uk/
- *	Ben Dooks <ben@simtec.co.uk>
- *
- * This program is free software; you can redistribute  it and/or modify it
- * under  the terms of  the GNU General  Public License as published by the
- * Free Software Foundation;  either version 2 of the  License, or (at your
- * option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// ALSA Soc Audio Layer - S3C2412 I2S driver
+//
+// Copyright (c) 2006 Wolfson Microelectronics PLC.
+//	Graeme Gregory graeme.gregory@wolfsonmicro.com
+//	linux@wolfsonmicro.com
+//
+// Copyright (c) 2007, 2004-2005 Simtec Electronics
+//	http://armlinux.simtec.co.uk/
+//	Ben Dooks <ben@simtec.co.uk>
 
 #include <linux/delay.h>
 #include <linux/gpio.h>
diff --git a/sound/soc/samsung/s3c2412-i2s.h b/sound/soc/samsung/s3c2412-i2s.h
index 02ad5794c0a9..bff2a797cb08 100644
--- a/sound/soc/samsung/s3c2412-i2s.h
+++ b/sound/soc/samsung/s3c2412-i2s.h
@@ -1,16 +1,11 @@
-/* sound/soc/samsung/s3c2412-i2s.c
- *
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
  * ALSA Soc Audio Layer - S3C2412 I2S driver
  *
  * Copyright (c) 2007 Simtec Electronics
  *	http://armlinux.simtec.co.uk/
  *	Ben Dooks <ben@simtec.co.uk>
- *
- *  This program is free software; you can redistribute  it and/or modify it
- *  under  the terms of  the GNU General  Public License as published by the
- *  Free Software Foundation;  either version 2 of the  License, or (at your
- *  option) any later version.
-*/
+ */
 
 #ifndef __SND_SOC_S3C24XX_S3C2412_I2S_H
 #define __SND_SOC_S3C24XX_S3C2412_I2S_H __FILE__
diff --git a/sound/soc/samsung/s3c24xx-i2s.c b/sound/soc/samsung/s3c24xx-i2s.c
index a8026b640c95..92bdaf0878f8 100644
--- a/sound/soc/samsung/s3c24xx-i2s.c
+++ b/sound/soc/samsung/s3c24xx-i2s.c
@@ -1,18 +1,13 @@
-/*
- * s3c24xx-i2s.c  --  ALSA Soc Audio Layer
- *
- * (c) 2006 Wolfson Microelectronics PLC.
- * Graeme Gregory graeme.gregory@wolfsonmicro.com or linux@wolfsonmicro.com
- *
- * Copyright 2004-2005 Simtec Electronics
- *	http://armlinux.simtec.co.uk/
- *	Ben Dooks <ben@simtec.co.uk>
- *
- *  This program is free software; you can redistribute  it and/or modify it
- *  under  the terms of  the GNU General  Public License as published by the
- *  Free Software Foundation;  either version 2 of the  License, or (at your
- *  option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// s3c24xx-i2s.c  --  ALSA Soc Audio Layer
+//
+// (c) 2006 Wolfson Microelectronics PLC.
+// Graeme Gregory graeme.gregory@wolfsonmicro.com or linux@wolfsonmicro.com
+//
+// Copyright 2004-2005 Simtec Electronics
+//	http://armlinux.simtec.co.uk/
+//	Ben Dooks <ben@simtec.co.uk>
 
 #include <linux/delay.h>
 #include <linux/clk.h>
diff --git a/sound/soc/samsung/s3c24xx-i2s.h b/sound/soc/samsung/s3c24xx-i2s.h
index f9ca04edacb7..e073e31855d0 100644
--- a/sound/soc/samsung/s3c24xx-i2s.h
+++ b/sound/soc/samsung/s3c24xx-i2s.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
 /*
  * s3c24xx-i2s.c  --  ALSA Soc Audio Layer
  *
@@ -5,11 +6,6 @@
  * Author: Graeme Gregory
  *         graeme.gregory@wolfsonmicro.com or linux@wolfsonmicro.com
  *
- *  This program is free software; you can redistribute  it and/or modify it
- *  under  the terms of  the GNU General  Public License as published by the
- *  Free Software Foundation;  either version 2 of the  License, or (at your
- *  option) any later version.
- *
  *  Revision history
  *    10th Nov 2006   Initial version.
  */
diff --git a/sound/soc/samsung/s3c24xx_simtec.c b/sound/soc/samsung/s3c24xx_simtec.c
index 6de63f3e37d5..4543705b8d87 100644
--- a/sound/soc/samsung/s3c24xx_simtec.c
+++ b/sound/soc/samsung/s3c24xx_simtec.c
@@ -1,11 +1,6 @@
-/* sound/soc/samsung/s3c24xx_simtec.c
- *
- * Copyright 2009 Simtec Electronics
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2009 Simtec Electronics
 
 #include <linux/gpio.h>
 #include <linux/clk.h>
diff --git a/sound/soc/samsung/s3c24xx_simtec.h b/sound/soc/samsung/s3c24xx_simtec.h
index 8270748a2c41..38d8384755cd 100644
--- a/sound/soc/samsung/s3c24xx_simtec.h
+++ b/sound/soc/samsung/s3c24xx_simtec.h
@@ -1,11 +1,7 @@
-/* sound/soc/samsung/s3c24xx_simtec.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * Copyright 2009 Simtec Electronics
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+ */
 
 extern void simtec_audio_init(struct snd_soc_pcm_runtime *rtd);
 
diff --git a/sound/soc/samsung/s3c24xx_simtec_hermes.c b/sound/soc/samsung/s3c24xx_simtec_hermes.c
index 7ac924c595bf..e3528e74a338 100644
--- a/sound/soc/samsung/s3c24xx_simtec_hermes.c
+++ b/sound/soc/samsung/s3c24xx_simtec_hermes.c
@@ -1,11 +1,6 @@
-/* sound/soc/samsung/s3c24xx_simtec_hermes.c
- *
- * Copyright 2009 Simtec Electronics
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2009 Simtec Electronics
 
 #include <linux/module.h>
 #include <sound/soc.h>
diff --git a/sound/soc/samsung/s3c24xx_simtec_tlv320aic23.c b/sound/soc/samsung/s3c24xx_simtec_tlv320aic23.c
index b4ed2fc1a65c..1360b881400d 100644
--- a/sound/soc/samsung/s3c24xx_simtec_tlv320aic23.c
+++ b/sound/soc/samsung/s3c24xx_simtec_tlv320aic23.c
@@ -1,11 +1,6 @@
-/* sound/soc/samsung/s3c24xx_simtec_tlv320aic23.c
- *
- * Copyright 2009 Simtec Electronics
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
-*/
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright 2009 Simtec Electronics
 
 #include <linux/module.h>
 #include <sound/soc.h>
diff --git a/sound/soc/samsung/s3c24xx_uda134x.c b/sound/soc/samsung/s3c24xx_uda134x.c
index 5fb3bab6bbfe..9d68f8ca1fcc 100644
--- a/sound/soc/samsung/s3c24xx_uda134x.c
+++ b/sound/soc/samsung/s3c24xx_uda134x.c
@@ -1,15 +1,11 @@
-/*
- * Modifications by Christian Pellegrin <chripell@evolware.org>
- *
- * s3c24xx_uda134x.c  --  S3C24XX_UDA134X ALSA SoC Audio board driver
- *
- * Copyright 2007 Dension Audio Systems Ltd.
- * Author: Zoltan Devai
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// Modifications by Christian Pellegrin <chripell@evolware.org>
+//
+// s3c24xx_uda134x.c - S3C24XX_UDA134X ALSA SoC Audio board driver
+//
+// Copyright 2007 Dension Audio Systems Ltd.
+// Author: Zoltan Devai
 
 #include <linux/clk.h>
 #include <linux/gpio.h>
diff --git a/sound/soc/samsung/smartq_wm8987.c b/sound/soc/samsung/smartq_wm8987.c
index cf0f54e652c1..b9e887ea60b2 100644
--- a/sound/soc/samsung/smartq_wm8987.c
+++ b/sound/soc/samsung/smartq_wm8987.c
@@ -1,17 +1,10 @@
-/* sound/soc/samsung/smartq_wm8987.c
- *
- * Copyright 2010 Maurus Cuelenaere <mcuelenaere@gmail.com>
- *
- * Based on smdk6410_wm8987.c
- *     Copyright 2007 Wolfson Microelectronics PLC. - linux@wolfsonmicro.com
- *     Graeme Gregory - graeme.gregory@wolfsonmicro.com
- *
- *  This program is free software; you can redistribute  it and/or modify it
- *  under  the terms of  the GNU General  Public License as published by the
- *  Free Software Foundation;  either version 2 of the  License, or (at your
- *  option) any later version.
- *
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright 2010 Maurus Cuelenaere <mcuelenaere@gmail.com>
+//
+// Based on smdk6410_wm8987.c
+//     Copyright 2007 Wolfson Microelectronics PLC. - linux@wolfsonmicro.com
+//     Graeme Gregory - graeme.gregory@wolfsonmicro.com
 
 #include <linux/gpio/consumer.h>
 #include <linux/module.h>
diff --git a/sound/soc/samsung/smdk_spdif.c b/sound/soc/samsung/smdk_spdif.c
index 7fc7cc6d1530..87a70d872c00 100644
--- a/sound/soc/samsung/smdk_spdif.c
+++ b/sound/soc/samsung/smdk_spdif.c
@@ -1,14 +1,8 @@
-/*
- * smdk_spdif.c  --  S/PDIF audio for SMDK
- *
- * Copyright 2010 Samsung Electronics Co. Ltd.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of the
- * License, or (at your option) any later version.
- *
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// smdk_spdif.c - S/PDIF audio for SMDK
+//
+// Copyright (C) 2010 Samsung Electronics Co., Ltd.
 
 #include <linux/clk.h>
 #include <linux/module.h>
diff --git a/sound/soc/samsung/smdk_wm8580.c b/sound/soc/samsung/smdk_wm8580.c
index 6e4dfa7e2c89..987807e6f8c3 100644
--- a/sound/soc/samsung/smdk_wm8580.c
+++ b/sound/soc/samsung/smdk_wm8580.c
@@ -1,14 +1,7 @@
-/*
- *  smdk_wm8580.c
- *
- *  Copyright (c) 2009 Samsung Electronics Co. Ltd
- *  Author: Jaswinder Singh <jassisinghbrar@gmail.com>
- *
- *  This program is free software; you can redistribute  it and/or modify it
- *  under  the terms of  the GNU General  Public License as published by the
- *  Free Software Foundation;  either version 2 of the  License, or (at your
- *  option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright (c) 2009 Samsung Electronics Co. Ltd
+// Author: Jaswinder Singh <jassisinghbrar@gmail.com>
 
 #include <linux/module.h>
 #include <sound/soc.h>
diff --git a/sound/soc/samsung/smdk_wm8994.c b/sound/soc/samsung/smdk_wm8994.c
index ff57b192d37d..135d8c2745be 100644
--- a/sound/soc/samsung/smdk_wm8994.c
+++ b/sound/soc/samsung/smdk_wm8994.c
@@ -1,11 +1,4 @@
-/*
- *  smdk_wm8994.c
- *
- *  This program is free software; you can redistribute  it and/or modify it
- *  under  the terms of  the GNU General  Public License as published by the
- *  Free Software Foundation;  either version 2 of the  License, or (at your
- *  option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
 
 #include "../codecs/wm8994.h"
 #include <sound/pcm_params.h>
diff --git a/sound/soc/samsung/smdk_wm8994pcm.c b/sound/soc/samsung/smdk_wm8994pcm.c
index 2e621496be8b..43171d6457fa 100644
--- a/sound/soc/samsung/smdk_wm8994pcm.c
+++ b/sound/soc/samsung/smdk_wm8994pcm.c
@@ -1,14 +1,8 @@
-/*
- *  sound/soc/samsung/smdk_wm8994pcm.c
- *
- *  Copyright (c) 2011 Samsung Electronics Co., Ltd
- *		http://www.samsung.com
- *
- *  This program is free software; you can redistribute  it and/or  modify it
- *  under  the terms of  the GNU General  Public License as published by the
- *  Free Software Foundation;  either version 2 of the  License, or (at your
- *  option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright (c) 2011 Samsung Electronics Co., Ltd
+//		http://www.samsung.com
+
 #include <linux/module.h>
 #include <sound/soc.h>
 #include <sound/pcm.h>
diff --git a/sound/soc/samsung/snow.c b/sound/soc/samsung/snow.c
index 5d8efc2d5c38..57ce90fe5004 100644
--- a/sound/soc/samsung/snow.c
+++ b/sound/soc/samsung/snow.c
@@ -1,15 +1,6 @@
-/*
- * ASoC machine driver for Snow boards
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// ASoC machine driver for Snow boards
 
 #include <linux/clk.h>
 #include <linux/module.h>
diff --git a/sound/soc/samsung/spdif.c b/sound/soc/samsung/spdif.c
index 5e4afb330416..805c57986e0b 100644
--- a/sound/soc/samsung/spdif.c
+++ b/sound/soc/samsung/spdif.c
@@ -1,14 +1,9 @@
-/* sound/soc/samsung/spdif.c
- *
- * ALSA SoC Audio Layer - Samsung S/PDIF Controller driver
- *
- * Copyright (c) 2010 Samsung Electronics Co. Ltd
- *		http://www.samsung.com/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
+// SPDX-License-Identifier: GPL-2.0
+//
+// ALSA SoC Audio Layer - Samsung S/PDIF Controller driver
+//
+// Copyright (c) 2010 Samsung Electronics Co. Ltd
+//		http://www.samsung.com/
 
 #include <linux/clk.h>
 #include <linux/io.h>
diff --git a/sound/soc/samsung/spdif.h b/sound/soc/samsung/spdif.h
index 4f72cb446dbf..461da60ab040 100644
--- a/sound/soc/samsung/spdif.h
+++ b/sound/soc/samsung/spdif.h
@@ -1,13 +1,9 @@
-/* sound/soc/samsung/spdif.h
- *
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
  * ALSA SoC Audio Layer - Samsung S/PDIF Controller driver
  *
  * Copyright (c) 2010 Samsung Electronics Co. Ltd
  *		http://www.samsung.com/
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #ifndef __SND_SOC_SAMSUNG_SPDIF_H
diff --git a/sound/soc/samsung/speyside.c b/sound/soc/samsung/speyside.c
index 4b4147d07804..15465c84daa3 100644
--- a/sound/soc/samsung/speyside.c
+++ b/sound/soc/samsung/speyside.c
@@ -1,13 +1,8 @@
-/*
- * Speyside audio support
- *
- * Copyright 2011 Wolfson Microelectronics
- *
- * This program is free software; you can redistribute  it and/or modify it
- * under  the terms of  the GNU General  Public License as published by the
- * Free Software Foundation;  either version 2 of the  License, or (at your
- * option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Speyside audio support
+//
+// Copyright 2011 Wolfson Microelectronics
 
 #include <sound/soc.h>
 #include <sound/soc-dapm.h>
diff --git a/sound/soc/samsung/tm2_wm5110.c b/sound/soc/samsung/tm2_wm5110.c
index dc93941e01c3..31f4256c6c65 100644
--- a/sound/soc/samsung/tm2_wm5110.c
+++ b/sound/soc/samsung/tm2_wm5110.c
@@ -1,14 +1,9 @@
-/*
- * Copyright (C) 2015 - 2016 Samsung Electronics Co., Ltd.
- *
- * Authors: Inha Song <ideal.song@samsung.com>
- *          Sylwester Nawrocki <s.nawrocki@samsung.com>
- *
- * This program is free software; you can redistribute  it and/or modify it
- * under  the terms of  the GNU General  Public License as published by the
- * Free Software Foundation;  either version 2 of the  License, or (at your
- * option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Copyright (C) 2015 - 2016 Samsung Electronics Co., Ltd.
+//
+// Authors: Inha Song <ideal.song@samsung.com>
+//          Sylwester Nawrocki <s.nawrocki@samsung.com>
 
 #include <linux/clk.h>
 #include <linux/gpio.h>
diff --git a/sound/soc/samsung/tobermory.c b/sound/soc/samsung/tobermory.c
index 998727cb4c31..14b11acb12a4 100644
--- a/sound/soc/samsung/tobermory.c
+++ b/sound/soc/samsung/tobermory.c
@@ -1,13 +1,8 @@
-/*
- * Tobermory audio support
- *
- * Copyright 2011 Wolfson Microelectronics
- *
- * This program is free software; you can redistribute  it and/or modify it
- * under  the terms of  the GNU General  Public License as published by the
- * Free Software Foundation;  either version 2 of the  License, or (at your
- * option) any later version.
- */
+// SPDX-License-Identifier: GPL-2.0+
+//
+// Tobermory audio support
+//
+// Copyright 2011 Wolfson Microelectronics
 
 #include <sound/soc.h>
 #include <sound/soc-dapm.h>
diff --git a/sound/soc/sh/rcar/core.c b/sound/soc/sh/rcar/core.c
index 4fe83e611c01..37cb61553d5f 100644
--- a/sound/soc/sh/rcar/core.c
+++ b/sound/soc/sh/rcar/core.c
@@ -300,6 +300,18 @@ int rsnd_runtime_channel_after_ctu_with_params(struct rsnd_dai_stream *io,
 	return chan;
 }
 
+int rsnd_channel_normalization(int chan)
+{
+	if ((chan > 8) || (chan < 0))
+		return 0;
+
+	/* TDM Extend Mode needs 8ch */
+	if (chan == 6)
+		chan = 8;
+
+	return chan;
+}
+
 int rsnd_runtime_channel_for_ssi_with_params(struct rsnd_dai_stream *io,
 					     struct snd_pcm_hw_params *params)
 {
@@ -312,11 +324,7 @@ int rsnd_runtime_channel_for_ssi_with_params(struct rsnd_dai_stream *io,
 	if (rsnd_runtime_is_multi_ssi(io))
 		chan /= rsnd_rdai_ssi_lane_get(rdai);
 
-	/* TDM Extend Mode needs 8ch */
-	if (chan == 6)
-		chan = 8;
-
-	return chan;
+	return rsnd_channel_normalization(chan);
 }
 
 int rsnd_runtime_is_multi_ssi(struct rsnd_dai_stream *io)
diff --git a/sound/soc/sh/rcar/rsnd.h b/sound/soc/sh/rcar/rsnd.h
index 0e6ef4e18400..7727add3eb1a 100644
--- a/sound/soc/sh/rcar/rsnd.h
+++ b/sound/soc/sh/rcar/rsnd.h
@@ -446,6 +446,7 @@ void rsnd_parse_connect_common(struct rsnd_dai *rdai,
 		struct device_node *playback,
 		struct device_node *capture);
 
+int rsnd_channel_normalization(int chan);
 #define rsnd_runtime_channel_original(io) \
 	rsnd_runtime_channel_original_with_params(io, NULL)
 int rsnd_runtime_channel_original_with_params(struct rsnd_dai_stream *io,
diff --git a/sound/soc/sh/rcar/ssi.c b/sound/soc/sh/rcar/ssi.c
index f5afab631abb..44bda210256e 100644
--- a/sound/soc/sh/rcar/ssi.c
+++ b/sound/soc/sh/rcar/ssi.c
@@ -303,6 +303,8 @@ static int rsnd_ssi_master_clk_start(struct rsnd_mod *mod,
 	if (rsnd_runtime_is_tdm_split(io))
 		chan = rsnd_io_converted_chan(io);
 
+	chan = rsnd_channel_normalization(chan);
+
 	main_rate = rsnd_ssi_clk_query(rdai, rate, chan, &idx);
 	if (!main_rate) {
 		dev_err(dev, "unsupported clock rate\n");
diff --git a/sound/soc/soc-core.c b/sound/soc/soc-core.c
index 46e3ab0fced4..2403bec2fccf 100644
--- a/sound/soc/soc-core.c
+++ b/sound/soc/soc-core.c
@@ -1974,10 +1974,13 @@ static void soc_check_tplg_fes(struct snd_soc_card *card)
 			continue;
 
 		/* for this machine ? */
+		if (!strcmp(component->driver->ignore_machine,
+			    card->dev->driver->name))
+			goto match;
 		if (strcmp(component->driver->ignore_machine,
-			   card->dev->driver->name))
+			   dev_name(card->dev)))
 			continue;
-
+match:
 		/* machine matches, so override the rtd data */
 		for_each_card_prelinks(card, i, dai_link) {
 
@@ -2828,10 +2831,21 @@ EXPORT_SYMBOL_GPL(snd_soc_register_card);
 
 static void snd_soc_unbind_card(struct snd_soc_card *card, bool unregister)
 {
+	struct snd_soc_pcm_runtime *rtd;
+	int order;
+
 	if (card->instantiated) {
 		card->instantiated = false;
 		snd_soc_dapm_shutdown(card);
 		snd_soc_flush_all_delayed_work(card);
+
+		/* remove all components used by DAI links on this card */
+		for_each_comp_order(order) {
+			for_each_card_rtds(card, rtd) {
+				soc_remove_link_components(card, rtd, order);
+			}
+		}
+
 		soc_cleanup_card_resources(card);
 		if (!unregister)
 			list_add(&card->list, &unbind_card_list);
diff --git a/sound/soc/soc-dapm.c b/sound/soc/soc-dapm.c
index 0382a47b30bd..81a7a12196ff 100644
--- a/sound/soc/soc-dapm.c
+++ b/sound/soc/soc-dapm.c
@@ -883,6 +883,7 @@ static int dapm_create_or_share_kcontrol(struct snd_soc_dapm_widget *w,
 			case snd_soc_dapm_switch:
 			case snd_soc_dapm_mixer:
 			case snd_soc_dapm_pga:
+			case snd_soc_dapm_effect:
 			case snd_soc_dapm_out_drv:
 				wname_in_long_name = true;
 				kcname_in_long_name = true;
@@ -2370,6 +2371,7 @@ static ssize_t dapm_widget_show_component(struct snd_soc_component *cmpnt,
 		case snd_soc_dapm_dac:
 		case snd_soc_dapm_adc:
 		case snd_soc_dapm_pga:
+		case snd_soc_dapm_effect:
 		case snd_soc_dapm_out_drv:
 		case snd_soc_dapm_mixer:
 		case snd_soc_dapm_mixer_named_ctl:
@@ -3197,6 +3199,7 @@ int snd_soc_dapm_new_widgets(struct snd_soc_card *card)
 			dapm_new_mux(w);
 			break;
 		case snd_soc_dapm_pga:
+		case snd_soc_dapm_effect:
 		case snd_soc_dapm_out_drv:
 			dapm_new_pga(w);
 			break;
@@ -4049,7 +4052,7 @@ snd_soc_dapm_new_dai(struct snd_soc_card *card, struct snd_soc_pcm_runtime *rtd,
 	struct snd_soc_dapm_widget template;
 	struct snd_soc_dapm_widget *w;
 	const char **w_param_text;
-	unsigned long private_value;
+	unsigned long private_value = 0;
 	char *link_name;
 	int ret;
 
diff --git a/sound/soc/soc-pcm.c b/sound/soc/soc-pcm.c
index be80a12fba27..0a4f60c7a188 100644
--- a/sound/soc/soc-pcm.c
+++ b/sound/soc/soc-pcm.c
@@ -43,8 +43,8 @@ static bool snd_soc_dai_stream_valid(struct snd_soc_dai *dai, int stream)
 	else
 		codec_stream = &dai->driver->capture;
 
-	/* If the codec specifies any rate at all, it supports the stream. */
-	return codec_stream->rates;
+	/* If the codec specifies any channels at all, it supports the stream */
+	return codec_stream->channels_min;
 }
 
 /**
@@ -1033,6 +1033,9 @@ interface_err:
 
 codec_err:
 	for_each_rtd_codec_dai_rollback(rtd, i, codec_dai) {
+		if (!snd_soc_dai_stream_valid(codec_dai, substream->stream))
+			continue;
+
 		if (codec_dai->driver->ops->hw_free)
 			codec_dai->driver->ops->hw_free(substream, codec_dai);
 		codec_dai->rate = 0;
@@ -1090,6 +1093,9 @@ static int soc_pcm_hw_free(struct snd_pcm_substream *substream)
 
 	/* now free hw params for the DAIs  */
 	for_each_rtd_codec_dai(rtd, i, codec_dai) {
+		if (!snd_soc_dai_stream_valid(codec_dai, substream->stream))
+			continue;
+
 		if (codec_dai->driver->ops->hw_free)
 			codec_dai->driver->ops->hw_free(substream, codec_dai);
 	}
@@ -2166,6 +2172,10 @@ int dpcm_be_dai_hw_params(struct snd_soc_pcm_runtime *fe, int stream)
 			}
 		}
 
+		/* copy the fixed-up hw params for BE dai */
+		memcpy(&be->dpcm[stream].hw_params, &dpcm->hw_params,
+		       sizeof(struct snd_pcm_hw_params));
+
 		/* only allow hw_params() if no connected FEs are running */
 		if (!snd_soc_dpcm_can_be_params(fe, be, stream))
 			continue;
diff --git a/sound/soc/soc-topology.c b/sound/soc/soc-topology.c
index 96852d250619..3299ebb48c1a 100644
--- a/sound/soc/soc-topology.c
+++ b/sound/soc/soc-topology.c
@@ -30,6 +30,8 @@
 #include <sound/soc-topology.h>
 #include <sound/tlv.h>
 
+#define SOC_TPLG_MAGIC_BIG_ENDIAN            0x436F5341 /* ASoC in reverse */
+
 /*
  * We make several passes over the data (since it wont necessarily be ordered)
  * and process objects in the following order. This guarantees the component
@@ -197,8 +199,8 @@ static int tplc_chan_get_reg(struct soc_tplg *tplg,
 	int i;
 
 	for (i = 0; i < SND_SOC_TPLG_MAX_CHAN; i++) {
-		if (chan[i].id == map)
-			return chan[i].reg;
+		if (le32_to_cpu(chan[i].id) == map)
+			return le32_to_cpu(chan[i].reg);
 	}
 
 	return -EINVAL;
@@ -210,8 +212,8 @@ static int tplc_chan_get_shift(struct soc_tplg *tplg,
 	int i;
 
 	for (i = 0; i < SND_SOC_TPLG_MAX_CHAN; i++) {
-		if (chan[i].id == map)
-			return chan[i].shift;
+		if (le32_to_cpu(chan[i].id) == map)
+			return le32_to_cpu(chan[i].shift);
 	}
 
 	return -EINVAL;
@@ -536,6 +538,8 @@ static void remove_dai(struct snd_soc_component *comp,
 		if (dai->driver == dai_drv)
 			dai->driver = NULL;
 
+	kfree(dai_drv->playback.stream_name);
+	kfree(dai_drv->capture.stream_name);
 	kfree(dai_drv->name);
 	list_del(&dobj->list);
 	kfree(dai_drv);
@@ -591,7 +595,7 @@ static int soc_tplg_kcontrol_bind_io(struct snd_soc_tplg_ctl_hdr *hdr,
 	const struct snd_soc_tplg_bytes_ext_ops *ext_ops;
 	int num_ops, i;
 
-	if (hdr->ops.info == SND_SOC_TPLG_CTL_BYTES
+	if (le32_to_cpu(hdr->ops.info) == SND_SOC_TPLG_CTL_BYTES
 		&& k->iface & SNDRV_CTL_ELEM_IFACE_MIXER
 		&& k->access & SNDRV_CTL_ELEM_ACCESS_TLV_READWRITE
 		&& k->access & SNDRV_CTL_ELEM_ACCESS_TLV_CALLBACK) {
@@ -707,9 +711,9 @@ static int soc_tplg_create_tlv_db_scale(struct soc_tplg *tplg,
 
 	p[0] = SNDRV_CTL_TLVT_DB_SCALE;
 	p[1] = item_len;
-	p[2] = scale->min;
-	p[3] = (scale->step & TLV_DB_SCALE_MASK)
-			| (scale->mute ? TLV_DB_SCALE_MUTE : 0);
+	p[2] = le32_to_cpu(scale->min);
+	p[3] = (le32_to_cpu(scale->step) & TLV_DB_SCALE_MASK)
+		| (le32_to_cpu(scale->mute) ? TLV_DB_SCALE_MUTE : 0);
 
 	kc->tlv.p = (void *)p;
 	return 0;
@@ -719,13 +723,14 @@ static int soc_tplg_create_tlv(struct soc_tplg *tplg,
 	struct snd_kcontrol_new *kc, struct snd_soc_tplg_ctl_hdr *tc)
 {
 	struct snd_soc_tplg_ctl_tlv *tplg_tlv;
+	u32 access = le32_to_cpu(tc->access);
 
-	if (!(tc->access & SNDRV_CTL_ELEM_ACCESS_TLV_READWRITE))
+	if (!(access & SNDRV_CTL_ELEM_ACCESS_TLV_READWRITE))
 		return 0;
 
-	if (!(tc->access & SNDRV_CTL_ELEM_ACCESS_TLV_CALLBACK)) {
+	if (!(access & SNDRV_CTL_ELEM_ACCESS_TLV_CALLBACK)) {
 		tplg_tlv = &tc->tlv;
-		switch (tplg_tlv->type) {
+		switch (le32_to_cpu(tplg_tlv->type)) {
 		case SNDRV_CTL_TLVT_DB_SCALE:
 			return soc_tplg_create_tlv_db_scale(tplg, kc,
 					&tplg_tlv->scale);
@@ -776,7 +781,7 @@ static int soc_tplg_dbytes_create(struct soc_tplg *tplg, unsigned int count,
 			return -ENOMEM;
 
 		tplg->pos += (sizeof(struct snd_soc_tplg_bytes_control) +
-			be->priv.size);
+			      le32_to_cpu(be->priv.size));
 
 		dev_dbg(tplg->dev,
 			"ASoC: adding bytes kcontrol %s with access 0x%x\n",
@@ -786,9 +791,9 @@ static int soc_tplg_dbytes_create(struct soc_tplg *tplg, unsigned int count,
 		kc.name = be->hdr.name;
 		kc.private_value = (long)sbe;
 		kc.iface = SNDRV_CTL_ELEM_IFACE_MIXER;
-		kc.access = be->hdr.access;
+		kc.access = le32_to_cpu(be->hdr.access);
 
-		sbe->max = be->max;
+		sbe->max = le32_to_cpu(be->max);
 		sbe->dobj.type = SND_SOC_DOBJ_BYTES;
 		sbe->dobj.ops = tplg->ops;
 		INIT_LIST_HEAD(&sbe->dobj.list);
@@ -856,7 +861,7 @@ static int soc_tplg_dmixer_create(struct soc_tplg *tplg, unsigned int count,
 		if (sm == NULL)
 			return -ENOMEM;
 		tplg->pos += (sizeof(struct snd_soc_tplg_mixer_control) +
-			mc->priv.size);
+			      le32_to_cpu(mc->priv.size));
 
 		dev_dbg(tplg->dev,
 			"ASoC: adding mixer kcontrol %s with access 0x%x\n",
@@ -866,7 +871,7 @@ static int soc_tplg_dmixer_create(struct soc_tplg *tplg, unsigned int count,
 		kc.name = mc->hdr.name;
 		kc.private_value = (long)sm;
 		kc.iface = SNDRV_CTL_ELEM_IFACE_MIXER;
-		kc.access = mc->hdr.access;
+		kc.access = le32_to_cpu(mc->hdr.access);
 
 		/* we only support FL/FR channel mapping atm */
 		sm->reg = tplc_chan_get_reg(tplg, mc->channel,
@@ -878,10 +883,10 @@ static int soc_tplg_dmixer_create(struct soc_tplg *tplg, unsigned int count,
 		sm->rshift = tplc_chan_get_shift(tplg, mc->channel,
 			SNDRV_CHMAP_FR);
 
-		sm->max = mc->max;
-		sm->min = mc->min;
-		sm->invert = mc->invert;
-		sm->platform_max = mc->platform_max;
+		sm->max = le32_to_cpu(mc->max);
+		sm->min = le32_to_cpu(mc->min);
+		sm->invert = le32_to_cpu(mc->invert);
+		sm->platform_max = le32_to_cpu(mc->platform_max);
 		sm->dobj.index = tplg->index;
 		sm->dobj.ops = tplg->ops;
 		sm->dobj.type = SND_SOC_DOBJ_MIXER;
@@ -895,19 +900,20 @@ static int soc_tplg_dmixer_create(struct soc_tplg *tplg, unsigned int count,
 			continue;
 		}
 
+		/* create any TLV data */
+		soc_tplg_create_tlv(tplg, &kc, &mc->hdr);
+
 		/* pass control to driver for optional further init */
 		err = soc_tplg_init_kcontrol(tplg, &kc,
 			(struct snd_soc_tplg_ctl_hdr *) mc);
 		if (err < 0) {
 			dev_err(tplg->dev, "ASoC: failed to init %s\n",
 				mc->hdr.name);
+			soc_tplg_free_tlv(tplg, &kc);
 			kfree(sm);
 			continue;
 		}
 
-		/* create any TLV data */
-		soc_tplg_create_tlv(tplg, &kc, &mc->hdr);
-
 		/* register control here */
 		err = soc_tplg_add_kcontrol(tplg, &kc,
 			&sm->dobj.control.kcontrol);
@@ -931,7 +937,7 @@ static int soc_tplg_denum_create_texts(struct soc_enum *se,
 	int i, ret;
 
 	se->dobj.control.dtexts =
-		kcalloc(ec->items, sizeof(char *), GFP_KERNEL);
+		kcalloc(le32_to_cpu(ec->items), sizeof(char *), GFP_KERNEL);
 	if (se->dobj.control.dtexts == NULL)
 		return -ENOMEM;
 
@@ -963,15 +969,22 @@ err:
 static int soc_tplg_denum_create_values(struct soc_enum *se,
 	struct snd_soc_tplg_enum_control *ec)
 {
-	if (ec->items > sizeof(*ec->values))
+	int i;
+
+	if (le32_to_cpu(ec->items) > sizeof(*ec->values))
 		return -EINVAL;
 
-	se->dobj.control.dvalues = kmemdup(ec->values,
-					   ec->items * sizeof(u32),
+	se->dobj.control.dvalues = kzalloc(le32_to_cpu(ec->items) *
+					   sizeof(u32),
 					   GFP_KERNEL);
 	if (!se->dobj.control.dvalues)
 		return -ENOMEM;
 
+	/* convert from little-endian */
+	for (i = 0; i < le32_to_cpu(ec->items); i++) {
+		se->dobj.control.dvalues[i] = le32_to_cpu(ec->values[i]);
+	}
+
 	return 0;
 }
 
@@ -994,8 +1007,6 @@ static int soc_tplg_denum_create(struct soc_tplg *tplg, unsigned int count,
 
 	for (i = 0; i < count; i++) {
 		ec = (struct snd_soc_tplg_enum_control *)tplg->pos;
-		tplg->pos += (sizeof(struct snd_soc_tplg_enum_control) +
-			ec->priv.size);
 
 		/* validate kcontrol */
 		if (strnlen(ec->hdr.name, SNDRV_CTL_ELEM_ID_NAME_MAXLEN) ==
@@ -1006,6 +1017,9 @@ static int soc_tplg_denum_create(struct soc_tplg *tplg, unsigned int count,
 		if (se == NULL)
 			return -ENOMEM;
 
+		tplg->pos += (sizeof(struct snd_soc_tplg_enum_control) +
+			      le32_to_cpu(ec->priv.size));
+
 		dev_dbg(tplg->dev, "ASoC: adding enum kcontrol %s size %d\n",
 			ec->hdr.name, ec->items);
 
@@ -1013,7 +1027,7 @@ static int soc_tplg_denum_create(struct soc_tplg *tplg, unsigned int count,
 		kc.name = ec->hdr.name;
 		kc.private_value = (long)se;
 		kc.iface = SNDRV_CTL_ELEM_IFACE_MIXER;
-		kc.access = ec->hdr.access;
+		kc.access = le32_to_cpu(ec->hdr.access);
 
 		se->reg = tplc_chan_get_reg(tplg, ec->channel, SNDRV_CHMAP_FL);
 		se->shift_l = tplc_chan_get_shift(tplg, ec->channel,
@@ -1021,14 +1035,14 @@ static int soc_tplg_denum_create(struct soc_tplg *tplg, unsigned int count,
 		se->shift_r = tplc_chan_get_shift(tplg, ec->channel,
 			SNDRV_CHMAP_FL);
 
-		se->items = ec->items;
-		se->mask = ec->mask;
+		se->items = le32_to_cpu(ec->items);
+		se->mask = le32_to_cpu(ec->mask);
 		se->dobj.index = tplg->index;
 		se->dobj.type = SND_SOC_DOBJ_ENUM;
 		se->dobj.ops = tplg->ops;
 		INIT_LIST_HEAD(&se->dobj.list);
 
-		switch (ec->hdr.ops.info) {
+		switch (le32_to_cpu(ec->hdr.ops.info)) {
 		case SND_SOC_TPLG_DAPM_CTL_ENUM_VALUE:
 		case SND_SOC_TPLG_CTL_ENUM_VALUE:
 			err = soc_tplg_denum_create_values(se, ec);
@@ -1101,23 +1115,24 @@ static int soc_tplg_kcontrol_elems_load(struct soc_tplg *tplg,
 	int i;
 
 	if (tplg->pass != SOC_TPLG_PASS_MIXER) {
-		tplg->pos += hdr->size + hdr->payload_size;
+		tplg->pos += le32_to_cpu(hdr->size) +
+			le32_to_cpu(hdr->payload_size);
 		return 0;
 	}
 
 	dev_dbg(tplg->dev, "ASoC: adding %d kcontrols at 0x%lx\n", hdr->count,
 		soc_tplg_get_offset(tplg));
 
-	for (i = 0; i < hdr->count; i++) {
+	for (i = 0; i < le32_to_cpu(hdr->count); i++) {
 
 		control_hdr = (struct snd_soc_tplg_ctl_hdr *)tplg->pos;
 
-		if (control_hdr->size != sizeof(*control_hdr)) {
+		if (le32_to_cpu(control_hdr->size) != sizeof(*control_hdr)) {
 			dev_err(tplg->dev, "ASoC: invalid control size\n");
 			return -EINVAL;
 		}
 
-		switch (control_hdr->ops.info) {
+		switch (le32_to_cpu(control_hdr->ops.info)) {
 		case SND_SOC_TPLG_CTL_VOLSW:
 		case SND_SOC_TPLG_CTL_STROBE:
 		case SND_SOC_TPLG_CTL_VOLSW_SX:
@@ -1125,17 +1140,20 @@ static int soc_tplg_kcontrol_elems_load(struct soc_tplg *tplg,
 		case SND_SOC_TPLG_CTL_RANGE:
 		case SND_SOC_TPLG_DAPM_CTL_VOLSW:
 		case SND_SOC_TPLG_DAPM_CTL_PIN:
-			soc_tplg_dmixer_create(tplg, 1, hdr->payload_size);
+			soc_tplg_dmixer_create(tplg, 1,
+					       le32_to_cpu(hdr->payload_size));
 			break;
 		case SND_SOC_TPLG_CTL_ENUM:
 		case SND_SOC_TPLG_CTL_ENUM_VALUE:
 		case SND_SOC_TPLG_DAPM_CTL_ENUM_DOUBLE:
 		case SND_SOC_TPLG_DAPM_CTL_ENUM_VIRT:
 		case SND_SOC_TPLG_DAPM_CTL_ENUM_VALUE:
-			soc_tplg_denum_create(tplg, 1, hdr->payload_size);
+			soc_tplg_denum_create(tplg, 1,
+					      le32_to_cpu(hdr->payload_size));
 			break;
 		case SND_SOC_TPLG_CTL_BYTES:
-			soc_tplg_dbytes_create(tplg, 1, hdr->payload_size);
+			soc_tplg_dbytes_create(tplg, 1,
+					       le32_to_cpu(hdr->payload_size));
 			break;
 		default:
 			soc_bind_err(tplg, control_hdr, i);
@@ -1163,17 +1181,22 @@ static int soc_tplg_dapm_graph_elems_load(struct soc_tplg *tplg,
 	struct snd_soc_dapm_context *dapm = &tplg->comp->dapm;
 	struct snd_soc_tplg_dapm_graph_elem *elem;
 	struct snd_soc_dapm_route **routes;
-	int count = hdr->count, i, j;
+	int count, i, j;
 	int ret = 0;
 
+	count = le32_to_cpu(hdr->count);
+
 	if (tplg->pass != SOC_TPLG_PASS_GRAPH) {
-		tplg->pos += hdr->size + hdr->payload_size;
+		tplg->pos +=
+			le32_to_cpu(hdr->size) +
+			le32_to_cpu(hdr->payload_size);
+
 		return 0;
 	}
 
 	if (soc_tplg_check_elem_count(tplg,
 		sizeof(struct snd_soc_tplg_dapm_graph_elem),
-		count, hdr->payload_size, "graph")) {
+		count, le32_to_cpu(hdr->payload_size), "graph")) {
 
 		dev_err(tplg->dev, "ASoC: invalid count %d for DAPM routes\n",
 			count);
@@ -1282,14 +1305,14 @@ static struct snd_kcontrol_new *soc_tplg_dapm_widget_dmixer_create(
 		if (sm == NULL)
 			goto err;
 
-		tplg->pos += (sizeof(struct snd_soc_tplg_mixer_control) +
-			mc->priv.size);
-
 		/* validate kcontrol */
 		if (strnlen(mc->hdr.name, SNDRV_CTL_ELEM_ID_NAME_MAXLEN) ==
 			SNDRV_CTL_ELEM_ID_NAME_MAXLEN)
 			goto err_str;
 
+		tplg->pos += (sizeof(struct snd_soc_tplg_mixer_control) +
+			      le32_to_cpu(mc->priv.size));
+
 		dev_dbg(tplg->dev, " adding DAPM widget mixer control %s at %d\n",
 			mc->hdr.name, i);
 
@@ -1325,18 +1348,19 @@ static struct snd_kcontrol_new *soc_tplg_dapm_widget_dmixer_create(
 			continue;
 		}
 
+		/* create any TLV data */
+		soc_tplg_create_tlv(tplg, &kc[i], &mc->hdr);
+
 		/* pass control to driver for optional further init */
 		err = soc_tplg_init_kcontrol(tplg, &kc[i],
 			(struct snd_soc_tplg_ctl_hdr *)mc);
 		if (err < 0) {
 			dev_err(tplg->dev, "ASoC: failed to init %s\n",
 				mc->hdr.name);
+			soc_tplg_free_tlv(tplg, &kc[i]);
 			kfree(sm);
 			continue;
 		}
-
-		/* create any TLV data */
-		soc_tplg_create_tlv(tplg, &kc[i], &mc->hdr);
 	}
 	return kc;
 
@@ -1374,6 +1398,9 @@ static struct snd_kcontrol_new *soc_tplg_dapm_widget_denum_create(
 		if (se == NULL)
 			goto err;
 
+		tplg->pos += (sizeof(struct snd_soc_tplg_enum_control) +
+				ec->priv.size);
+
 		dev_dbg(tplg->dev, " adding DAPM widget enum control %s\n",
 			ec->hdr.name);
 
@@ -1397,7 +1424,7 @@ static struct snd_kcontrol_new *soc_tplg_dapm_widget_denum_create(
 		se->mask = ec->mask;
 		se->dobj.index = tplg->index;
 
-		switch (ec->hdr.ops.info) {
+		switch (le32_to_cpu(ec->hdr.ops.info)) {
 		case SND_SOC_TPLG_CTL_ENUM_VALUE:
 		case SND_SOC_TPLG_DAPM_CTL_ENUM_VALUE:
 			err = soc_tplg_denum_create_values(se, ec);
@@ -1438,9 +1465,6 @@ static struct snd_kcontrol_new *soc_tplg_dapm_widget_denum_create(
 				ec->hdr.name);
 			goto err_se;
 		}
-
-		tplg->pos += (sizeof(struct snd_soc_tplg_enum_control) +
-				ec->priv.size);
 	}
 
 	return kc;
@@ -1491,7 +1515,7 @@ static struct snd_kcontrol_new *soc_tplg_dapm_widget_dbytes_create(
 			goto err;
 
 		tplg->pos += (sizeof(struct snd_soc_tplg_bytes_control) +
-			be->priv.size);
+			      le32_to_cpu(be->priv.size));
 
 		dev_dbg(tplg->dev,
 			"ASoC: adding bytes kcontrol %s with access 0x%x\n",
@@ -1563,7 +1587,7 @@ static int soc_tplg_dapm_widget_create(struct soc_tplg *tplg,
 	memset(&template, 0, sizeof(template));
 
 	/* map user to kernel widget ID */
-	template.id = get_widget_id(w->id);
+	template.id = get_widget_id(le32_to_cpu(w->id));
 	if (template.id < 0)
 		return template.id;
 
@@ -1576,18 +1600,20 @@ static int soc_tplg_dapm_widget_create(struct soc_tplg *tplg,
 		ret = -ENOMEM;
 		goto err;
 	}
-	template.reg = w->reg;
-	template.shift = w->shift;
-	template.mask = w->mask;
-	template.subseq = w->subseq;
+	template.reg = le32_to_cpu(w->reg);
+	template.shift = le32_to_cpu(w->shift);
+	template.mask = le32_to_cpu(w->mask);
+	template.subseq = le32_to_cpu(w->subseq);
 	template.on_val = w->invert ? 0 : 1;
 	template.off_val = w->invert ? 1 : 0;
-	template.ignore_suspend = w->ignore_suspend;
-	template.event_flags = w->event_flags;
+	template.ignore_suspend = le32_to_cpu(w->ignore_suspend);
+	template.event_flags = le16_to_cpu(w->event_flags);
 	template.dobj.index = tplg->index;
 
 	tplg->pos +=
-		(sizeof(struct snd_soc_tplg_dapm_widget) + w->priv.size);
+		(sizeof(struct snd_soc_tplg_dapm_widget) +
+		 le32_to_cpu(w->priv.size));
+
 	if (w->num_kcontrols == 0) {
 		kcontrol_type = 0;
 		template.num_kcontrols = 0;
@@ -1598,7 +1624,7 @@ static int soc_tplg_dapm_widget_create(struct soc_tplg *tplg,
 	dev_dbg(tplg->dev, "ASoC: template %s has %d controls of type %x\n",
 		w->name, w->num_kcontrols, control_hdr->type);
 
-	switch (control_hdr->ops.info) {
+	switch (le32_to_cpu(control_hdr->ops.info)) {
 	case SND_SOC_TPLG_CTL_VOLSW:
 	case SND_SOC_TPLG_CTL_STROBE:
 	case SND_SOC_TPLG_CTL_VOLSW_SX:
@@ -1606,7 +1632,7 @@ static int soc_tplg_dapm_widget_create(struct soc_tplg *tplg,
 	case SND_SOC_TPLG_CTL_RANGE:
 	case SND_SOC_TPLG_DAPM_CTL_VOLSW:
 		kcontrol_type = SND_SOC_TPLG_TYPE_MIXER;  /* volume mixer */
-		template.num_kcontrols = w->num_kcontrols;
+		template.num_kcontrols = le32_to_cpu(w->num_kcontrols);
 		template.kcontrol_news =
 			soc_tplg_dapm_widget_dmixer_create(tplg,
 			template.num_kcontrols);
@@ -1621,7 +1647,7 @@ static int soc_tplg_dapm_widget_create(struct soc_tplg *tplg,
 	case SND_SOC_TPLG_DAPM_CTL_ENUM_VIRT:
 	case SND_SOC_TPLG_DAPM_CTL_ENUM_VALUE:
 		kcontrol_type = SND_SOC_TPLG_TYPE_ENUM;	/* enumerated mixer */
-		template.num_kcontrols = w->num_kcontrols;
+		template.num_kcontrols = le32_to_cpu(w->num_kcontrols);
 		template.kcontrol_news =
 			soc_tplg_dapm_widget_denum_create(tplg,
 			template.num_kcontrols);
@@ -1632,7 +1658,7 @@ static int soc_tplg_dapm_widget_create(struct soc_tplg *tplg,
 		break;
 	case SND_SOC_TPLG_CTL_BYTES:
 		kcontrol_type = SND_SOC_TPLG_TYPE_BYTES; /* bytes control */
-		template.num_kcontrols = w->num_kcontrols;
+		template.num_kcontrols = le32_to_cpu(w->num_kcontrols);
 		template.kcontrol_news =
 			soc_tplg_dapm_widget_dbytes_create(tplg,
 				template.num_kcontrols);
@@ -1644,7 +1670,7 @@ static int soc_tplg_dapm_widget_create(struct soc_tplg *tplg,
 	default:
 		dev_err(tplg->dev, "ASoC: invalid widget control type %d:%d:%d\n",
 			control_hdr->ops.get, control_hdr->ops.put,
-			control_hdr->ops.info);
+			le32_to_cpu(control_hdr->ops.info));
 		ret = -EINVAL;
 		goto hdr_err;
 	}
@@ -1694,7 +1720,9 @@ static int soc_tplg_dapm_widget_elems_load(struct soc_tplg *tplg,
 	struct snd_soc_tplg_hdr *hdr)
 {
 	struct snd_soc_tplg_dapm_widget *widget;
-	int ret, count = hdr->count, i;
+	int ret, count, i;
+
+	count = le32_to_cpu(hdr->count);
 
 	if (tplg->pass != SOC_TPLG_PASS_WIDGET)
 		return 0;
@@ -1703,7 +1731,7 @@ static int soc_tplg_dapm_widget_elems_load(struct soc_tplg *tplg,
 
 	for (i = 0; i < count; i++) {
 		widget = (struct snd_soc_tplg_dapm_widget *) tplg->pos;
-		if (widget->size != sizeof(*widget)) {
+		if (le32_to_cpu(widget->size) != sizeof(*widget)) {
 			dev_err(tplg->dev, "ASoC: invalid widget size\n");
 			return -EINVAL;
 		}
@@ -1745,13 +1773,13 @@ static void set_stream_info(struct snd_soc_pcm_stream *stream,
 	struct snd_soc_tplg_stream_caps *caps)
 {
 	stream->stream_name = kstrdup(caps->name, GFP_KERNEL);
-	stream->channels_min = caps->channels_min;
-	stream->channels_max = caps->channels_max;
-	stream->rates = caps->rates;
-	stream->rate_min = caps->rate_min;
-	stream->rate_max = caps->rate_max;
-	stream->formats = caps->formats;
-	stream->sig_bits = caps->sig_bits;
+	stream->channels_min = le32_to_cpu(caps->channels_min);
+	stream->channels_max = le32_to_cpu(caps->channels_max);
+	stream->rates = le32_to_cpu(caps->rates);
+	stream->rate_min = le32_to_cpu(caps->rate_min);
+	stream->rate_max = le32_to_cpu(caps->rate_max);
+	stream->formats = le64_to_cpu(caps->formats);
+	stream->sig_bits = le32_to_cpu(caps->sig_bits);
 }
 
 static void set_dai_flags(struct snd_soc_dai_driver *dai_drv,
@@ -1786,7 +1814,7 @@ static int soc_tplg_dai_create(struct soc_tplg *tplg,
 
 	if (strlen(pcm->dai_name))
 		dai_drv->name = kstrdup(pcm->dai_name, GFP_KERNEL);
-	dai_drv->id = pcm->dai_id;
+	dai_drv->id = le32_to_cpu(pcm->dai_id);
 
 	if (pcm->playback) {
 		stream = &dai_drv->playback;
@@ -1807,6 +1835,9 @@ static int soc_tplg_dai_create(struct soc_tplg *tplg,
 	ret = soc_tplg_dai_load(tplg, dai_drv, pcm, NULL);
 	if (ret < 0) {
 		dev_err(tplg->comp->dev, "ASoC: DAI loading failed\n");
+		kfree(dai_drv->playback.stream_name);
+		kfree(dai_drv->capture.stream_name);
+		kfree(dai_drv->name);
 		kfree(dai_drv);
 		return ret;
 	}
@@ -1858,7 +1889,7 @@ static int soc_tplg_fe_link_create(struct soc_tplg *tplg,
 		link->name = kstrdup(pcm->pcm_name, GFP_KERNEL);
 		link->stream_name = kstrdup(pcm->pcm_name, GFP_KERNEL);
 	}
-	link->id = pcm->pcm_id;
+	link->id = le32_to_cpu(pcm->pcm_id);
 
 	if (strlen(pcm->dai_name))
 		link->cpu_dai_name = kstrdup(pcm->dai_name, GFP_KERNEL);
@@ -1868,15 +1899,20 @@ static int soc_tplg_fe_link_create(struct soc_tplg *tplg,
 
 	/* enable DPCM */
 	link->dynamic = 1;
-	link->dpcm_playback = pcm->playback;
-	link->dpcm_capture = pcm->capture;
+	link->dpcm_playback = le32_to_cpu(pcm->playback);
+	link->dpcm_capture = le32_to_cpu(pcm->capture);
 	if (pcm->flag_mask)
-		set_link_flags(link, pcm->flag_mask, pcm->flags);
+		set_link_flags(link,
+			       le32_to_cpu(pcm->flag_mask),
+			       le32_to_cpu(pcm->flags));
 
 	/* pass control to component driver for optional further init */
 	ret = soc_tplg_dai_link_load(tplg, link, NULL);
 	if (ret < 0) {
 		dev_err(tplg->comp->dev, "ASoC: FE link loading failed\n");
+		kfree(link->name);
+		kfree(link->stream_name);
+		kfree(link->cpu_dai_name);
 		kfree(link);
 		return ret;
 	}
@@ -1907,7 +1943,7 @@ static int soc_tplg_pcm_create(struct soc_tplg *tplg,
 static void stream_caps_new_ver(struct snd_soc_tplg_stream_caps *dest,
 				struct snd_soc_tplg_stream_caps_v4 *src)
 {
-	dest->size = sizeof(*dest);
+	dest->size = cpu_to_le32(sizeof(*dest));
 	memcpy(dest->name, src->name, SNDRV_CTL_ELEM_ID_NAME_MAXLEN);
 	dest->formats = src->formats;
 	dest->rates = src->rates;
@@ -1941,7 +1977,7 @@ static int pcm_new_ver(struct soc_tplg *tplg,
 
 	*pcm = NULL;
 
-	if (src->size != sizeof(*src_v4)) {
+	if (le32_to_cpu(src->size) != sizeof(*src_v4)) {
 		dev_err(tplg->dev, "ASoC: invalid PCM size\n");
 		return -EINVAL;
 	}
@@ -1952,7 +1988,7 @@ static int pcm_new_ver(struct soc_tplg *tplg,
 	if (!dest)
 		return -ENOMEM;
 
-	dest->size = sizeof(*dest);	/* size of latest abi version */
+	dest->size = cpu_to_le32(sizeof(*dest)); /* size of latest abi version */
 	memcpy(dest->pcm_name, src_v4->pcm_name, SNDRV_CTL_ELEM_ID_NAME_MAXLEN);
 	memcpy(dest->dai_name, src_v4->dai_name, SNDRV_CTL_ELEM_ID_NAME_MAXLEN);
 	dest->pcm_id = src_v4->pcm_id;
@@ -1961,7 +1997,7 @@ static int pcm_new_ver(struct soc_tplg *tplg,
 	dest->capture = src_v4->capture;
 	dest->compress = src_v4->compress;
 	dest->num_streams = src_v4->num_streams;
-	for (i = 0; i < dest->num_streams; i++)
+	for (i = 0; i < le32_to_cpu(dest->num_streams); i++)
 		memcpy(&dest->stream[i], &src_v4->stream[i],
 		       sizeof(struct snd_soc_tplg_stream));
 
@@ -1976,25 +2012,30 @@ static int soc_tplg_pcm_elems_load(struct soc_tplg *tplg,
 	struct snd_soc_tplg_hdr *hdr)
 {
 	struct snd_soc_tplg_pcm *pcm, *_pcm;
-	int count = hdr->count;
+	int count;
+	int size;
 	int i;
 	bool abi_match;
 
+	count = le32_to_cpu(hdr->count);
+
 	if (tplg->pass != SOC_TPLG_PASS_PCM_DAI)
 		return 0;
 
 	/* check the element size and count */
 	pcm = (struct snd_soc_tplg_pcm *)tplg->pos;
-	if (pcm->size > sizeof(struct snd_soc_tplg_pcm)
-		|| pcm->size < sizeof(struct snd_soc_tplg_pcm_v4)) {
+	size = le32_to_cpu(pcm->size);
+	if (size > sizeof(struct snd_soc_tplg_pcm)
+		|| size < sizeof(struct snd_soc_tplg_pcm_v4)) {
 		dev_err(tplg->dev, "ASoC: invalid size %d for PCM elems\n",
-			pcm->size);
+			size);
 		return -EINVAL;
 	}
 
 	if (soc_tplg_check_elem_count(tplg,
-		pcm->size, count,
-		hdr->payload_size, "PCM DAI")) {
+				      size, count,
+				      le32_to_cpu(hdr->payload_size),
+				      "PCM DAI")) {
 		dev_err(tplg->dev, "ASoC: invalid count %d for PCM DAI elems\n",
 			count);
 		return -EINVAL;
@@ -2002,11 +2043,12 @@ static int soc_tplg_pcm_elems_load(struct soc_tplg *tplg,
 
 	for (i = 0; i < count; i++) {
 		pcm = (struct snd_soc_tplg_pcm *)tplg->pos;
+		size = le32_to_cpu(pcm->size);
 
 		/* check ABI version by size, create a new version of pcm
 		 * if abi not match.
 		 */
-		if (pcm->size == sizeof(*pcm)) {
+		if (size == sizeof(*pcm)) {
 			abi_match = true;
 			_pcm = pcm;
 		} else {
@@ -2020,7 +2062,7 @@ static int soc_tplg_pcm_elems_load(struct soc_tplg *tplg,
 		/* offset by version-specific struct size and
 		 * real priv data size
 		 */
-		tplg->pos += pcm->size + _pcm->priv.size;
+		tplg->pos += size + le32_to_cpu(_pcm->priv.size);
 
 		if (!abi_match)
 			kfree(_pcm); /* free the duplicated one */
@@ -2048,12 +2090,13 @@ static void set_link_hw_format(struct snd_soc_dai_link *link,
 	unsigned char invert_bclk, invert_fsync;
 	int i;
 
-	for (i = 0; i < cfg->num_hw_configs; i++) {
+	for (i = 0; i < le32_to_cpu(cfg->num_hw_configs); i++) {
 		hw_config = &cfg->hw_config[i];
 		if (hw_config->id != cfg->default_hw_config_id)
 			continue;
 
-		link->dai_fmt = hw_config->fmt & SND_SOC_DAIFMT_FORMAT_MASK;
+		link->dai_fmt = le32_to_cpu(hw_config->fmt) &
+			SND_SOC_DAIFMT_FORMAT_MASK;
 
 		/* clock gating */
 		switch (hw_config->clock_gated) {
@@ -2117,7 +2160,8 @@ static int link_new_ver(struct soc_tplg *tplg,
 
 	*link = NULL;
 
-	if (src->size != sizeof(struct snd_soc_tplg_link_config_v4)) {
+	if (le32_to_cpu(src->size) !=
+	    sizeof(struct snd_soc_tplg_link_config_v4)) {
 		dev_err(tplg->dev, "ASoC: invalid physical link config size\n");
 		return -EINVAL;
 	}
@@ -2129,10 +2173,10 @@ static int link_new_ver(struct soc_tplg *tplg,
 	if (!dest)
 		return -ENOMEM;
 
-	dest->size = sizeof(*dest);
+	dest->size = cpu_to_le32(sizeof(*dest));
 	dest->id = src_v4->id;
 	dest->num_streams = src_v4->num_streams;
-	for (i = 0; i < dest->num_streams; i++)
+	for (i = 0; i < le32_to_cpu(dest->num_streams); i++)
 		memcpy(&dest->stream[i], &src_v4->stream[i],
 		       sizeof(struct snd_soc_tplg_stream));
 
@@ -2165,7 +2209,7 @@ static int soc_tplg_link_config(struct soc_tplg *tplg,
 	else
 		stream_name = NULL;
 
-	link = snd_soc_find_dai_link(tplg->comp->card, cfg->id,
+	link = snd_soc_find_dai_link(tplg->comp->card, le32_to_cpu(cfg->id),
 				     name, stream_name);
 	if (!link) {
 		dev_err(tplg->dev, "ASoC: physical link %s (id %d) not exist\n",
@@ -2179,7 +2223,9 @@ static int soc_tplg_link_config(struct soc_tplg *tplg,
 
 	/* flags */
 	if (cfg->flag_mask)
-		set_link_flags(link, cfg->flag_mask, cfg->flags);
+		set_link_flags(link,
+			       le32_to_cpu(cfg->flag_mask),
+			       le32_to_cpu(cfg->flags));
 
 	/* pass control to component driver for optional further init */
 	ret = soc_tplg_dai_link_load(tplg, link, cfg);
@@ -2203,27 +2249,33 @@ static int soc_tplg_link_elems_load(struct soc_tplg *tplg,
 	struct snd_soc_tplg_hdr *hdr)
 {
 	struct snd_soc_tplg_link_config *link, *_link;
-	int count = hdr->count;
+	int count;
+	int size;
 	int i, ret;
 	bool abi_match;
 
+	count = le32_to_cpu(hdr->count);
+
 	if (tplg->pass != SOC_TPLG_PASS_LINK) {
-		tplg->pos += hdr->size + hdr->payload_size;
+		tplg->pos += le32_to_cpu(hdr->size) +
+			le32_to_cpu(hdr->payload_size);
 		return 0;
 	};
 
 	/* check the element size and count */
 	link = (struct snd_soc_tplg_link_config *)tplg->pos;
-	if (link->size > sizeof(struct snd_soc_tplg_link_config)
-		|| link->size < sizeof(struct snd_soc_tplg_link_config_v4)) {
+	size = le32_to_cpu(link->size);
+	if (size > sizeof(struct snd_soc_tplg_link_config)
+		|| size < sizeof(struct snd_soc_tplg_link_config_v4)) {
 		dev_err(tplg->dev, "ASoC: invalid size %d for physical link elems\n",
-			link->size);
+			size);
 		return -EINVAL;
 	}
 
 	if (soc_tplg_check_elem_count(tplg,
-		link->size, count,
-		hdr->payload_size, "physical link config")) {
+				      size, count,
+				      le32_to_cpu(hdr->payload_size),
+				      "physical link config")) {
 		dev_err(tplg->dev, "ASoC: invalid count %d for physical link elems\n",
 			count);
 		return -EINVAL;
@@ -2232,7 +2284,8 @@ static int soc_tplg_link_elems_load(struct soc_tplg *tplg,
 	/* config physical DAI links */
 	for (i = 0; i < count; i++) {
 		link = (struct snd_soc_tplg_link_config *)tplg->pos;
-		if (link->size == sizeof(*link)) {
+		size = le32_to_cpu(link->size);
+		if (size == sizeof(*link)) {
 			abi_match = true;
 			_link = link;
 		} else {
@@ -2249,7 +2302,7 @@ static int soc_tplg_link_elems_load(struct soc_tplg *tplg,
 		/* offset by version-specific struct size and
 		 * real priv data size
 		 */
-		tplg->pos += link->size + _link->priv.size;
+		tplg->pos += size + le32_to_cpu(_link->priv.size);
 
 		if (!abi_match)
 			kfree(_link); /* free the duplicated one */
@@ -2269,13 +2322,15 @@ static int soc_tplg_link_elems_load(struct soc_tplg *tplg,
 static int soc_tplg_dai_config(struct soc_tplg *tplg,
 			       struct snd_soc_tplg_dai *d)
 {
-	struct snd_soc_dai_link_component dai_component = {0};
+	struct snd_soc_dai_link_component dai_component;
 	struct snd_soc_dai *dai;
 	struct snd_soc_dai_driver *dai_drv;
 	struct snd_soc_pcm_stream *stream;
 	struct snd_soc_tplg_stream_caps *caps;
 	int ret;
 
+	memset(&dai_component, 0, sizeof(dai_component));
+
 	dai_component.dai_name = d->dai_name;
 	dai = snd_soc_find_dai(&dai_component);
 	if (!dai) {
@@ -2284,7 +2339,7 @@ static int soc_tplg_dai_config(struct soc_tplg *tplg,
 		return -EINVAL;
 	}
 
-	if (d->dai_id != dai->id) {
+	if (le32_to_cpu(d->dai_id) != dai->id) {
 		dev_err(tplg->dev, "ASoC: physical DAI %s id mismatch\n",
 			d->dai_name);
 		return -EINVAL;
@@ -2307,7 +2362,9 @@ static int soc_tplg_dai_config(struct soc_tplg *tplg,
 	}
 
 	if (d->flag_mask)
-		set_dai_flags(dai_drv, d->flag_mask, d->flags);
+		set_dai_flags(dai_drv,
+			      le32_to_cpu(d->flag_mask),
+			      le32_to_cpu(d->flags));
 
 	/* pass control to component driver for optional further init */
 	ret = soc_tplg_dai_load(tplg, dai_drv, NULL, dai);
@@ -2324,22 +2381,24 @@ static int soc_tplg_dai_elems_load(struct soc_tplg *tplg,
 				   struct snd_soc_tplg_hdr *hdr)
 {
 	struct snd_soc_tplg_dai *dai;
-	int count = hdr->count;
+	int count;
 	int i;
 
+	count = le32_to_cpu(hdr->count);
+
 	if (tplg->pass != SOC_TPLG_PASS_BE_DAI)
 		return 0;
 
 	/* config the existing BE DAIs */
 	for (i = 0; i < count; i++) {
 		dai = (struct snd_soc_tplg_dai *)tplg->pos;
-		if (dai->size != sizeof(*dai)) {
+		if (le32_to_cpu(dai->size) != sizeof(*dai)) {
 			dev_err(tplg->dev, "ASoC: invalid physical DAI size\n");
 			return -EINVAL;
 		}
 
 		soc_tplg_dai_config(tplg, dai);
-		tplg->pos += (sizeof(*dai) + dai->priv.size);
+		tplg->pos += (sizeof(*dai) + le32_to_cpu(dai->priv.size));
 	}
 
 	dev_dbg(tplg->dev, "ASoC: Configure %d BE DAIs\n", count);
@@ -2361,25 +2420,28 @@ static int manifest_new_ver(struct soc_tplg *tplg,
 {
 	struct snd_soc_tplg_manifest *dest;
 	struct snd_soc_tplg_manifest_v4 *src_v4;
+	int size;
 
 	*manifest = NULL;
 
-	if (src->size != sizeof(*src_v4)) {
+	size = le32_to_cpu(src->size);
+	if (size != sizeof(*src_v4)) {
 		dev_warn(tplg->dev, "ASoC: invalid manifest size %d\n",
-			 src->size);
-		if (src->size)
+			 size);
+		if (size)
 			return -EINVAL;
-		src->size = sizeof(*src_v4);
+		src->size = cpu_to_le32(sizeof(*src_v4));
 	}
 
 	dev_warn(tplg->dev, "ASoC: old version of manifest\n");
 
 	src_v4 = (struct snd_soc_tplg_manifest_v4 *)src;
-	dest = kzalloc(sizeof(*dest) + src_v4->priv.size, GFP_KERNEL);
+	dest = kzalloc(sizeof(*dest) + le32_to_cpu(src_v4->priv.size),
+		       GFP_KERNEL);
 	if (!dest)
 		return -ENOMEM;
 
-	dest->size = sizeof(*dest);	/* size of latest abi version */
+	dest->size = cpu_to_le32(sizeof(*dest)); /* size of latest abi version */
 	dest->control_elems = src_v4->control_elems;
 	dest->widget_elems = src_v4->widget_elems;
 	dest->graph_elems = src_v4->graph_elems;
@@ -2388,7 +2450,7 @@ static int manifest_new_ver(struct soc_tplg *tplg,
 	dest->priv.size = src_v4->priv.size;
 	if (dest->priv.size)
 		memcpy(dest->priv.data, src_v4->priv.data,
-		       src_v4->priv.size);
+		       le32_to_cpu(src_v4->priv.size));
 
 	*manifest = dest;
 	return 0;
@@ -2407,7 +2469,7 @@ static int soc_tplg_manifest_load(struct soc_tplg *tplg,
 	manifest = (struct snd_soc_tplg_manifest *)tplg->pos;
 
 	/* check ABI version by size, create a new manifest if abi not match */
-	if (manifest->size == sizeof(*manifest)) {
+	if (le32_to_cpu(manifest->size) == sizeof(*manifest)) {
 		abi_match = true;
 		_manifest = manifest;
 	} else {
@@ -2434,16 +2496,16 @@ static int soc_valid_header(struct soc_tplg *tplg,
 	if (soc_tplg_get_hdr_offset(tplg) >= tplg->fw->size)
 		return 0;
 
-	if (hdr->size != sizeof(*hdr)) {
+	if (le32_to_cpu(hdr->size) != sizeof(*hdr)) {
 		dev_err(tplg->dev,
 			"ASoC: invalid header size for type %d at offset 0x%lx size 0x%zx.\n",
-			hdr->type, soc_tplg_get_hdr_offset(tplg),
+			le32_to_cpu(hdr->type), soc_tplg_get_hdr_offset(tplg),
 			tplg->fw->size);
 		return -EINVAL;
 	}
 
 	/* big endian firmware objects not supported atm */
-	if (hdr->magic == cpu_to_be32(SND_SOC_TPLG_MAGIC)) {
+	if (hdr->magic == SOC_TPLG_MAGIC_BIG_ENDIAN) {
 		dev_err(tplg->dev,
 			"ASoC: pass %d big endian not supported header got %x at offset 0x%lx size 0x%zx.\n",
 			tplg->pass, hdr->magic,
@@ -2451,7 +2513,7 @@ static int soc_valid_header(struct soc_tplg *tplg,
 		return -EINVAL;
 	}
 
-	if (hdr->magic != SND_SOC_TPLG_MAGIC) {
+	if (le32_to_cpu(hdr->magic) != SND_SOC_TPLG_MAGIC) {
 		dev_err(tplg->dev,
 			"ASoC: pass %d does not have a valid header got %x at offset 0x%lx size 0x%zx.\n",
 			tplg->pass, hdr->magic,
@@ -2460,8 +2522,8 @@ static int soc_valid_header(struct soc_tplg *tplg,
 	}
 
 	/* Support ABI from version 4 */
-	if (hdr->abi > SND_SOC_TPLG_ABI_VERSION
-		|| hdr->abi < SND_SOC_TPLG_ABI_VERSION_MIN) {
+	if (le32_to_cpu(hdr->abi) > SND_SOC_TPLG_ABI_VERSION ||
+	    le32_to_cpu(hdr->abi) < SND_SOC_TPLG_ABI_VERSION_MIN) {
 		dev_err(tplg->dev,
 			"ASoC: pass %d invalid ABI version got 0x%x need 0x%x at offset 0x%lx size 0x%zx.\n",
 			tplg->pass, hdr->abi,
@@ -2476,7 +2538,7 @@ static int soc_valid_header(struct soc_tplg *tplg,
 		return -EINVAL;
 	}
 
-	if (tplg->pass == hdr->type)
+	if (tplg->pass == le32_to_cpu(hdr->type))
 		dev_dbg(tplg->dev,
 			"ASoC: Got 0x%x bytes of type %d version %d vendor %d at pass %d\n",
 			hdr->payload_size, hdr->type, hdr->version,
@@ -2492,13 +2554,13 @@ static int soc_tplg_load_header(struct soc_tplg *tplg,
 	tplg->pos = tplg->hdr_pos + sizeof(struct snd_soc_tplg_hdr);
 
 	/* check for matching ID */
-	if (hdr->index != tplg->req_index &&
+	if (le32_to_cpu(hdr->index) != tplg->req_index &&
 		tplg->req_index != SND_SOC_TPLG_INDEX_ALL)
 		return 0;
 
-	tplg->index = hdr->index;
+	tplg->index = le32_to_cpu(hdr->index);
 
-	switch (hdr->type) {
+	switch (le32_to_cpu(hdr->type)) {
 	case SND_SOC_TPLG_TYPE_MIXER:
 	case SND_SOC_TPLG_TYPE_ENUM:
 	case SND_SOC_TPLG_TYPE_BYTES:
@@ -2554,7 +2616,7 @@ static int soc_tplg_process_headers(struct soc_tplg *tplg)
 				return ret;
 
 			/* goto next header */
-			tplg->hdr_pos += hdr->payload_size +
+			tplg->hdr_pos += le32_to_cpu(hdr->payload_size) +
 				sizeof(struct snd_soc_tplg_hdr);
 			hdr = (struct snd_soc_tplg_hdr *)tplg->hdr_pos;
 		}
diff --git a/sound/soc/sof/Kconfig b/sound/soc/sof/Kconfig
new file mode 100644
index 000000000000..b204c65698f9
--- /dev/null
+++ b/sound/soc/sof/Kconfig
@@ -0,0 +1,156 @@
+config SND_SOC_SOF_TOPLEVEL
+	bool "Sound Open Firmware Support"
+	help
+	  This adds support for Sound Open Firmware (SOF). SOF is a free and
+	  generic open source audio DSP firmware for multiple devices.
+	  Say Y if you have such a device that is supported by SOF.
+	  If unsure select "N".
+
+if SND_SOC_SOF_TOPLEVEL
+
+config SND_SOC_SOF_PCI
+	tristate "SOF PCI enumeration support"
+	depends on PCI
+	select SND_SOC_SOF
+	select SND_SOC_ACPI if ACPI
+	select SND_SOC_SOF_OPTIONS
+	select SND_SOC_SOF_INTEL_PCI if SND_SOC_SOF_INTEL_TOPLEVEL
+	help
+	  This adds support for PCI enumeration. This option is
+	  required to enable Intel Skylake+ devices
+	  Say Y if you need this option
+	  If unsure select "N".
+
+config SND_SOC_SOF_ACPI
+	tristate "SOF ACPI enumeration support"
+	depends on ACPI || COMPILE_TEST
+	select SND_SOC_SOF
+	select SND_SOC_ACPI if ACPI
+	select SND_SOC_SOF_OPTIONS
+	select SND_SOC_SOF_INTEL_ACPI if SND_SOC_SOF_INTEL_TOPLEVEL
+	select IOSF_MBI if X86 && PCI
+	help
+	  This adds support for ACPI enumeration. This option is required
+	  to enable Intel Haswell/Broadwell/Baytrail/Cherrytrail devices
+	  Say Y if you need this option
+	  If unsure select "N".
+
+config SND_SOC_SOF_OPTIONS
+	tristate
+	help
+	  This option is not user-selectable but automagically handled by
+	  'select' statements at a higher level
+
+if SND_SOC_SOF_OPTIONS
+
+config SND_SOC_SOF_NOCODEC
+	tristate "SOF nocodec mode Support"
+	help
+	  This adds support for a dummy/nocodec machine driver fallback
+	  option if no known codec is detected. This is typically only
+	  enabled for developers or devices where the sound card is
+	  controlled externally
+	  Say Y if you need this nocodec fallback option
+	  If unsure select "N".
+
+config SND_SOC_SOF_STRICT_ABI_CHECKS
+	bool "SOF strict ABI checks"
+	help
+	  This option enables strict ABI checks for firmware and topology
+	  files.
+	  When these files are more recent than the kernel, the kernel
+	  will handle the functionality it supports and may report errors
+	  during topology creation or run-time usage if new functionality
+	  is invoked.
+	  This option will stop topology creation and firmware load upfront.
+	  It is intended for SOF CI/releases and not for users or distros.
+	  Say Y if you want strict ABI checks for an SOF release
+	  If you are not involved in SOF releases and CI development
+	  select "N".
+
+config SND_SOC_SOF_DEBUG
+	bool "SOF debugging features"
+	help
+	  This option can be used to enable or disable individual SOF firmware
+	  and driver debugging options.
+	  Say Y if you are debugging SOF FW or drivers.
+	  If unsure select "N".
+
+if SND_SOC_SOF_DEBUG
+
+config SND_SOC_SOF_FORCE_NOCODEC_MODE
+	bool "SOF force nocodec Mode"
+	depends on SND_SOC_SOF_NOCODEC
+	help
+	  This forces SOF to use dummy/nocodec as machine driver, even
+	  though there is a codec detected on the real platform. This is
+	  typically only enabled for developers for debug purposes, before
+	  codec/machine driver is ready, or to exclude the impact of those
+	  drivers
+	  Say Y if you need this force nocodec mode option
+	  If unsure select "N".
+
+config SND_SOC_SOF_DEBUG_XRUN_STOP
+	bool "SOF stop on XRUN"
+	help
+	  This option forces PCMs to stop on any XRUN event. This is useful to
+	  preserve any trace data ond pipeline status prior to the XRUN.
+	  Say Y if you are debugging SOF FW pipeline XRUNs.
+	  If unsure select "N".
+
+config SND_SOC_SOF_DEBUG_VERBOSE_IPC
+	bool "SOF verbose IPC logs"
+	help
+	  This option enables more verbose IPC logs, with command types in
+	  human-readable form instead of just 32-bit hex dumps. This is useful
+	  if you are trying to debug IPC with the DSP firmware.
+	  If unsure select "N".
+
+config SND_SOC_SOF_DEBUG_FORCE_IPC_POSITION
+	bool "SOF force to use IPC for position update on SKL+"
+	help
+	  This option force to handle stream position update IPCs and run pcm
+	  elapse to inform ALSA about that, on platforms (e.g. Intel SKL+) that
+	  with other approach (e.g. HDAC DPIB/posbuf) to elapse PCM.
+	  On platforms (e.g. Intel SKL-) where position update IPC is the only
+	  one choice, this setting won't impact anything.
+	  if you are trying to debug pointer update with position IPCs or where
+	  DPIB/posbuf is not ready, select "Y".
+	  If unsure select "N".
+
+config SND_SOC_SOF_DEBUG_ENABLE_DEBUGFS_CACHE
+	bool "SOF enable debugfs caching"
+	help
+	  This option enables caching of debugfs
+	  memory -> DSP resource (memory, register, etc)
+	  before the audio DSP is suspended. This will increase the suspend
+	  latency and therefore should be used for debug purposes only.
+	  Say Y if you want to enable caching the memory windows.
+	  If unsure, select "N".
+
+endif ## SND_SOC_SOF_DEBUG
+
+endif ## SND_SOC_SOF_OPTIONS
+
+config SND_SOC_SOF
+	tristate
+	select SND_SOC_TOPOLOGY
+	help
+	  This option is not user-selectable but automagically handled by
+	  'select' statements at a higher level
+	  The selection is made at the top level and does not exactly follow
+	  module dependencies but since the module or built-in type is decided
+	  at the top level it doesn't matter.
+
+config SND_SOC_SOF_PROBE_WORK_QUEUE
+	bool
+	help
+	  This option is not user-selectable but automagically handled by
+	  'select' statements at a higher level
+	  When selected, the probe is handled in two steps, for example to
+	  avoid lockdeps if request_module is used in the probe.
+
+source "sound/soc/sof/intel/Kconfig"
+source "sound/soc/sof/xtensa/Kconfig"
+
+endif
diff --git a/sound/soc/sof/Makefile b/sound/soc/sof/Makefile
new file mode 100644
index 000000000000..8f14c9d2950b
--- /dev/null
+++ b/sound/soc/sof/Makefile
@@ -0,0 +1,18 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+
+snd-sof-objs := core.o ops.o loader.o ipc.o pcm.o pm.o debug.o topology.o\
+		control.o trace.o utils.o
+
+snd-sof-pci-objs := sof-pci-dev.o
+snd-sof-acpi-objs := sof-acpi-dev.o
+snd-sof-nocodec-objs := nocodec.o
+
+obj-$(CONFIG_SND_SOC_SOF) += snd-sof.o
+obj-$(CONFIG_SND_SOC_SOF_NOCODEC) += snd-sof-nocodec.o
+
+
+obj-$(CONFIG_SND_SOC_SOF_ACPI) += sof-acpi-dev.o
+obj-$(CONFIG_SND_SOC_SOF_PCI) += sof-pci-dev.o
+
+obj-$(CONFIG_SND_SOC_SOF_INTEL_TOPLEVEL) += intel/
+obj-$(CONFIG_SND_SOC_SOF_XTENSA) += xtensa/
diff --git a/sound/soc/sof/control.c b/sound/soc/sof/control.c
new file mode 100644
index 000000000000..11762c4580f1
--- /dev/null
+++ b/sound/soc/sof/control.c
@@ -0,0 +1,552 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license.  When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2018 Intel Corporation. All rights reserved.
+//
+// Author: Liam Girdwood <liam.r.girdwood@linux.intel.com>
+//
+
+/* Mixer Controls */
+
+#include <linux/pm_runtime.h>
+#include "sof-priv.h"
+
+static inline u32 mixer_to_ipc(unsigned int value, u32 *volume_map, int size)
+{
+	if (value >= size)
+		return volume_map[size - 1];
+
+	return volume_map[value];
+}
+
+static inline u32 ipc_to_mixer(u32 value, u32 *volume_map, int size)
+{
+	int i;
+
+	for (i = 0; i < size; i++) {
+		if (volume_map[i] >= value)
+			return i;
+	}
+
+	return i - 1;
+}
+
+int snd_sof_volume_get(struct snd_kcontrol *kcontrol,
+		       struct snd_ctl_elem_value *ucontrol)
+{
+	struct soc_mixer_control *sm =
+		(struct soc_mixer_control *)kcontrol->private_value;
+	struct snd_sof_control *scontrol = sm->dobj.private;
+	struct snd_sof_dev *sdev = scontrol->sdev;
+	struct sof_ipc_ctrl_data *cdata = scontrol->control_data;
+	unsigned int i, channels = scontrol->num_channels;
+	int err, ret;
+
+	ret = pm_runtime_get_sync(sdev->dev);
+	if (ret < 0) {
+		dev_err_ratelimited(sdev->dev,
+				    "error: volume get failed to resume %d\n",
+				    ret);
+		pm_runtime_put_noidle(sdev->dev);
+		return ret;
+	}
+
+	/* get all the mixer data from DSP */
+	snd_sof_ipc_set_get_comp_data(sdev->ipc, scontrol,
+				      SOF_IPC_COMP_GET_VALUE,
+				      SOF_CTRL_TYPE_VALUE_CHAN_GET,
+				      SOF_CTRL_CMD_VOLUME,
+				      false);
+
+	/* read back each channel */
+	for (i = 0; i < channels; i++)
+		ucontrol->value.integer.value[i] =
+			ipc_to_mixer(cdata->chanv[i].value,
+				     scontrol->volume_table, sm->max + 1);
+
+	pm_runtime_mark_last_busy(sdev->dev);
+	err = pm_runtime_put_autosuspend(sdev->dev);
+	if (err < 0)
+		dev_err_ratelimited(sdev->dev,
+				    "error: volume get failed to idle %d\n",
+				    err);
+	return 0;
+}
+
+int snd_sof_volume_put(struct snd_kcontrol *kcontrol,
+		       struct snd_ctl_elem_value *ucontrol)
+{
+	struct soc_mixer_control *sm =
+		(struct soc_mixer_control *)kcontrol->private_value;
+	struct snd_sof_control *scontrol = sm->dobj.private;
+	struct snd_sof_dev *sdev = scontrol->sdev;
+	struct sof_ipc_ctrl_data *cdata = scontrol->control_data;
+	unsigned int i, channels = scontrol->num_channels;
+	int ret, err;
+
+	ret = pm_runtime_get_sync(sdev->dev);
+	if (ret < 0) {
+		dev_err_ratelimited(sdev->dev,
+				    "error: volume put failed to resume %d\n",
+				    ret);
+		pm_runtime_put_noidle(sdev->dev);
+		return ret;
+	}
+
+	/* update each channel */
+	for (i = 0; i < channels; i++) {
+		cdata->chanv[i].value =
+			mixer_to_ipc(ucontrol->value.integer.value[i],
+				     scontrol->volume_table, sm->max + 1);
+		cdata->chanv[i].channel = i;
+	}
+
+	/* notify DSP of mixer updates */
+	snd_sof_ipc_set_get_comp_data(sdev->ipc, scontrol,
+				      SOF_IPC_COMP_SET_VALUE,
+				      SOF_CTRL_TYPE_VALUE_CHAN_GET,
+				      SOF_CTRL_CMD_VOLUME,
+				      true);
+
+	pm_runtime_mark_last_busy(sdev->dev);
+	err = pm_runtime_put_autosuspend(sdev->dev);
+	if (err < 0)
+		dev_err_ratelimited(sdev->dev,
+				    "error: volume put failed to idle %d\n",
+				    err);
+	return 0;
+}
+
+int snd_sof_switch_get(struct snd_kcontrol *kcontrol,
+		       struct snd_ctl_elem_value *ucontrol)
+{
+	struct soc_mixer_control *sm =
+		(struct soc_mixer_control *)kcontrol->private_value;
+	struct snd_sof_control *scontrol = sm->dobj.private;
+	struct snd_sof_dev *sdev = scontrol->sdev;
+	struct sof_ipc_ctrl_data *cdata = scontrol->control_data;
+	unsigned int i, channels = scontrol->num_channels;
+	int err, ret;
+
+	ret = pm_runtime_get_sync(sdev->dev);
+	if (ret < 0) {
+		dev_err_ratelimited(sdev->dev,
+				    "error: switch get failed to resume %d\n",
+				    ret);
+		pm_runtime_put_noidle(sdev->dev);
+		return ret;
+	}
+
+	/* get all the mixer data from DSP */
+	snd_sof_ipc_set_get_comp_data(sdev->ipc, scontrol,
+				      SOF_IPC_COMP_GET_VALUE,
+				      SOF_CTRL_TYPE_VALUE_CHAN_GET,
+				      SOF_CTRL_CMD_SWITCH,
+				      false);
+
+	/* read back each channel */
+	for (i = 0; i < channels; i++)
+		ucontrol->value.integer.value[i] = cdata->chanv[i].value;
+
+	pm_runtime_mark_last_busy(sdev->dev);
+	err = pm_runtime_put_autosuspend(sdev->dev);
+	if (err < 0)
+		dev_err_ratelimited(sdev->dev,
+				    "error: switch get failed to idle %d\n",
+				    err);
+	return 0;
+}
+
+int snd_sof_switch_put(struct snd_kcontrol *kcontrol,
+		       struct snd_ctl_elem_value *ucontrol)
+{
+	struct soc_mixer_control *sm =
+		(struct soc_mixer_control *)kcontrol->private_value;
+	struct snd_sof_control *scontrol = sm->dobj.private;
+	struct snd_sof_dev *sdev = scontrol->sdev;
+	struct sof_ipc_ctrl_data *cdata = scontrol->control_data;
+	unsigned int i, channels = scontrol->num_channels;
+	int ret, err;
+
+	ret = pm_runtime_get_sync(sdev->dev);
+	if (ret < 0) {
+		dev_err_ratelimited(sdev->dev,
+				    "error: switch put failed to resume %d\n",
+				    ret);
+		pm_runtime_put_noidle(sdev->dev);
+		return ret;
+	}
+
+	/* update each channel */
+	for (i = 0; i < channels; i++) {
+		cdata->chanv[i].value = ucontrol->value.integer.value[i];
+		cdata->chanv[i].channel = i;
+	}
+
+	/* notify DSP of mixer updates */
+	snd_sof_ipc_set_get_comp_data(sdev->ipc, scontrol,
+				      SOF_IPC_COMP_SET_VALUE,
+				      SOF_CTRL_TYPE_VALUE_CHAN_GET,
+				      SOF_CTRL_CMD_SWITCH,
+				      true);
+
+	pm_runtime_mark_last_busy(sdev->dev);
+	err = pm_runtime_put_autosuspend(sdev->dev);
+	if (err < 0)
+		dev_err_ratelimited(sdev->dev,
+				    "error: switch put failed to idle %d\n",
+				    err);
+	return 0;
+}
+
+int snd_sof_enum_get(struct snd_kcontrol *kcontrol,
+		     struct snd_ctl_elem_value *ucontrol)
+{
+	struct soc_enum *se =
+		(struct soc_enum *)kcontrol->private_value;
+	struct snd_sof_control *scontrol = se->dobj.private;
+	struct snd_sof_dev *sdev = scontrol->sdev;
+	struct sof_ipc_ctrl_data *cdata = scontrol->control_data;
+	unsigned int i, channels = scontrol->num_channels;
+	int err, ret;
+
+	ret = pm_runtime_get_sync(sdev->dev);
+	if (ret < 0) {
+		dev_err_ratelimited(sdev->dev,
+				    "error: enum get failed to resume %d\n",
+				    ret);
+		pm_runtime_put_noidle(sdev->dev);
+		return ret;
+	}
+
+	/* get all the enum data from DSP */
+	snd_sof_ipc_set_get_comp_data(sdev->ipc, scontrol,
+				      SOF_IPC_COMP_GET_VALUE,
+				      SOF_CTRL_TYPE_VALUE_CHAN_GET,
+				      SOF_CTRL_CMD_ENUM,
+				      false);
+
+	/* read back each channel */
+	for (i = 0; i < channels; i++)
+		ucontrol->value.enumerated.item[i] = cdata->chanv[i].value;
+
+	pm_runtime_mark_last_busy(sdev->dev);
+	err = pm_runtime_put_autosuspend(sdev->dev);
+	if (err < 0)
+		dev_err_ratelimited(sdev->dev,
+				    "error: enum get failed to idle %d\n",
+				    err);
+	return 0;
+}
+
+int snd_sof_enum_put(struct snd_kcontrol *kcontrol,
+		     struct snd_ctl_elem_value *ucontrol)
+{
+	struct soc_enum *se =
+		(struct soc_enum *)kcontrol->private_value;
+	struct snd_sof_control *scontrol = se->dobj.private;
+	struct snd_sof_dev *sdev = scontrol->sdev;
+	struct sof_ipc_ctrl_data *cdata = scontrol->control_data;
+	unsigned int i, channels = scontrol->num_channels;
+	int ret, err;
+
+	ret = pm_runtime_get_sync(sdev->dev);
+	if (ret < 0) {
+		dev_err_ratelimited(sdev->dev,
+				    "error: enum put failed to resume %d\n",
+				    ret);
+		pm_runtime_put_noidle(sdev->dev);
+		return ret;
+	}
+
+	/* update each channel */
+	for (i = 0; i < channels; i++) {
+		cdata->chanv[i].value = ucontrol->value.enumerated.item[i];
+		cdata->chanv[i].channel = i;
+	}
+
+	/* notify DSP of enum updates */
+	snd_sof_ipc_set_get_comp_data(sdev->ipc, scontrol,
+				      SOF_IPC_COMP_SET_VALUE,
+				      SOF_CTRL_TYPE_VALUE_CHAN_GET,
+				      SOF_CTRL_CMD_ENUM,
+				      true);
+
+	pm_runtime_mark_last_busy(sdev->dev);
+	err = pm_runtime_put_autosuspend(sdev->dev);
+	if (err < 0)
+		dev_err_ratelimited(sdev->dev,
+				    "error: enum put failed to idle %d\n",
+				    err);
+	return 0;
+}
+
+int snd_sof_bytes_get(struct snd_kcontrol *kcontrol,
+		      struct snd_ctl_elem_value *ucontrol)
+{
+	struct soc_bytes_ext *be =
+		(struct soc_bytes_ext *)kcontrol->private_value;
+	struct snd_sof_control *scontrol = be->dobj.private;
+	struct snd_sof_dev *sdev = scontrol->sdev;
+	struct sof_ipc_ctrl_data *cdata = scontrol->control_data;
+	struct sof_abi_hdr *data = cdata->data;
+	size_t size;
+	int ret, err;
+
+	if (be->max > sizeof(ucontrol->value.bytes.data)) {
+		dev_err_ratelimited(sdev->dev,
+				    "error: data max %d exceeds ucontrol data array size\n",
+				    be->max);
+		return -EINVAL;
+	}
+
+	ret = pm_runtime_get_sync(sdev->dev);
+	if (ret < 0) {
+		dev_err_ratelimited(sdev->dev,
+				    "error: bytes get failed to resume %d\n",
+				    ret);
+		pm_runtime_put_noidle(sdev->dev);
+		return ret;
+	}
+
+	/* get all the binary data from DSP */
+	snd_sof_ipc_set_get_comp_data(sdev->ipc, scontrol,
+				      SOF_IPC_COMP_GET_DATA,
+				      SOF_CTRL_TYPE_DATA_GET,
+				      scontrol->cmd,
+				      false);
+
+	size = data->size + sizeof(*data);
+	if (size > be->max) {
+		dev_err_ratelimited(sdev->dev,
+				    "error: DSP sent %zu bytes max is %d\n",
+				    size, be->max);
+		ret = -EINVAL;
+		goto out;
+	}
+
+	/* copy back to kcontrol */
+	memcpy(ucontrol->value.bytes.data, data, size);
+
+out:
+	pm_runtime_mark_last_busy(sdev->dev);
+	err = pm_runtime_put_autosuspend(sdev->dev);
+	if (err < 0)
+		dev_err_ratelimited(sdev->dev,
+				    "error: bytes get failed to idle %d\n",
+				    err);
+	return ret;
+}
+
+int snd_sof_bytes_put(struct snd_kcontrol *kcontrol,
+		      struct snd_ctl_elem_value *ucontrol)
+{
+	struct soc_bytes_ext *be =
+		(struct soc_bytes_ext *)kcontrol->private_value;
+	struct snd_sof_control *scontrol = be->dobj.private;
+	struct snd_sof_dev *sdev = scontrol->sdev;
+	struct sof_ipc_ctrl_data *cdata = scontrol->control_data;
+	struct sof_abi_hdr *data = cdata->data;
+	int ret, err;
+
+	if (be->max > sizeof(ucontrol->value.bytes.data)) {
+		dev_err_ratelimited(sdev->dev,
+				    "error: data max %d exceeds ucontrol data array size\n",
+				    be->max);
+		return -EINVAL;
+	}
+
+	if (data->size > be->max) {
+		dev_err_ratelimited(sdev->dev,
+				    "error: size too big %d bytes max is %d\n",
+				    data->size, be->max);
+		return -EINVAL;
+	}
+
+	ret = pm_runtime_get_sync(sdev->dev);
+	if (ret < 0) {
+		dev_err_ratelimited(sdev->dev,
+				    "error: bytes put failed to resume %d\n",
+				    ret);
+		pm_runtime_put_noidle(sdev->dev);
+		return ret;
+	}
+
+	/* copy from kcontrol */
+	memcpy(data, ucontrol->value.bytes.data, data->size);
+
+	/* notify DSP of byte control updates */
+	snd_sof_ipc_set_get_comp_data(sdev->ipc, scontrol,
+				      SOF_IPC_COMP_SET_DATA,
+				      SOF_CTRL_TYPE_DATA_SET,
+				      scontrol->cmd,
+				      true);
+
+	pm_runtime_mark_last_busy(sdev->dev);
+	err = pm_runtime_put_autosuspend(sdev->dev);
+	if (err < 0)
+		dev_err_ratelimited(sdev->dev,
+				    "error: bytes put failed to idle %d\n",
+				    err);
+	return ret;
+}
+
+int snd_sof_bytes_ext_put(struct snd_kcontrol *kcontrol,
+			  const unsigned int __user *binary_data,
+			  unsigned int size)
+{
+	struct soc_bytes_ext *be =
+		(struct soc_bytes_ext *)kcontrol->private_value;
+	struct snd_sof_control *scontrol = be->dobj.private;
+	struct snd_sof_dev *sdev = scontrol->sdev;
+	struct sof_ipc_ctrl_data *cdata = scontrol->control_data;
+	struct snd_ctl_tlv header;
+	const struct snd_ctl_tlv __user *tlvd =
+		(const struct snd_ctl_tlv __user *)binary_data;
+	int ret;
+	int err;
+
+	/*
+	 * The beginning of bytes data contains a header from where
+	 * the length (as bytes) is needed to know the correct copy
+	 * length of data from tlvd->tlv.
+	 */
+	if (copy_from_user(&header, tlvd, sizeof(const struct snd_ctl_tlv)))
+		return -EFAULT;
+
+	/* be->max is coming from topology */
+	if (header.length > be->max) {
+		dev_err_ratelimited(sdev->dev, "error: Bytes data size %d exceeds max %d.\n",
+				    header.length, be->max);
+		return -EINVAL;
+	}
+
+	/* Check that header id matches the command */
+	if (header.numid != scontrol->cmd) {
+		dev_err_ratelimited(sdev->dev,
+				    "error: incorrect numid %d\n",
+				    header.numid);
+		return -EINVAL;
+	}
+
+	if (copy_from_user(cdata->data, tlvd->tlv, header.length))
+		return -EFAULT;
+
+	if (cdata->data->magic != SOF_ABI_MAGIC) {
+		dev_err_ratelimited(sdev->dev,
+				    "error: Wrong ABI magic 0x%08x.\n",
+				    cdata->data->magic);
+		return -EINVAL;
+	}
+
+	if (SOF_ABI_VERSION_INCOMPATIBLE(SOF_ABI_VERSION, cdata->data->abi)) {
+		dev_err_ratelimited(sdev->dev, "error: Incompatible ABI version 0x%08x.\n",
+				    cdata->data->abi);
+		return -EINVAL;
+	}
+
+	if (cdata->data->size + sizeof(const struct sof_abi_hdr) > be->max) {
+		dev_err_ratelimited(sdev->dev, "error: Mismatch in ABI data size (truncated?).\n");
+		return -EINVAL;
+	}
+
+	ret = pm_runtime_get_sync(sdev->dev);
+	if (ret < 0) {
+		dev_err_ratelimited(sdev->dev,
+				    "error: bytes_ext put failed to resume %d\n",
+				    ret);
+		pm_runtime_put_noidle(sdev->dev);
+		return ret;
+	}
+
+	/* notify DSP of byte control updates */
+	snd_sof_ipc_set_get_comp_data(sdev->ipc, scontrol,
+				      SOF_IPC_COMP_SET_DATA,
+				      SOF_CTRL_TYPE_DATA_SET,
+				      scontrol->cmd,
+				      true);
+
+	pm_runtime_mark_last_busy(sdev->dev);
+	err = pm_runtime_put_autosuspend(sdev->dev);
+	if (err < 0)
+		dev_err_ratelimited(sdev->dev,
+				    "error: bytes_ext put failed to idle %d\n",
+				    err);
+
+	return ret;
+}
+
+int snd_sof_bytes_ext_get(struct snd_kcontrol *kcontrol,
+			  unsigned int __user *binary_data,
+			  unsigned int size)
+{
+	struct soc_bytes_ext *be =
+		(struct soc_bytes_ext *)kcontrol->private_value;
+	struct snd_sof_control *scontrol = be->dobj.private;
+	struct snd_sof_dev *sdev = scontrol->sdev;
+	struct sof_ipc_ctrl_data *cdata = scontrol->control_data;
+	struct snd_ctl_tlv header;
+	struct snd_ctl_tlv __user *tlvd =
+		(struct snd_ctl_tlv __user *)binary_data;
+	int data_size;
+	int err;
+	int ret;
+
+	ret = pm_runtime_get_sync(sdev->dev);
+	if (ret < 0) {
+		dev_err_ratelimited(sdev->dev,
+				    "error: bytes_ext get failed to resume %d\n",
+				    ret);
+		pm_runtime_put_noidle(sdev->dev);
+		return ret;
+	}
+
+	/*
+	 * Decrement the limit by ext bytes header size to
+	 * ensure the user space buffer is not exceeded.
+	 */
+	size -= sizeof(const struct snd_ctl_tlv);
+
+	/* set the ABI header values */
+	cdata->data->magic = SOF_ABI_MAGIC;
+	cdata->data->abi = SOF_ABI_VERSION;
+
+	/* get all the component data from DSP */
+	ret = snd_sof_ipc_set_get_comp_data(sdev->ipc, scontrol,
+					    SOF_IPC_COMP_GET_DATA,
+					    SOF_CTRL_TYPE_DATA_GET,
+					    scontrol->cmd,
+					    false);
+
+	/* Prevent read of other kernel data or possibly corrupt response */
+	data_size = cdata->data->size + sizeof(const struct sof_abi_hdr);
+
+	/* check data size doesn't exceed max coming from topology */
+	if (data_size > be->max) {
+		dev_err_ratelimited(sdev->dev, "error: user data size %d exceeds max size %d.\n",
+				    data_size, be->max);
+		ret = -EINVAL;
+		goto out;
+	}
+
+	header.numid = scontrol->cmd;
+	header.length = data_size;
+	if (copy_to_user(tlvd, &header, sizeof(const struct snd_ctl_tlv))) {
+		ret = -EFAULT;
+		goto out;
+	}
+
+	if (copy_to_user(tlvd->tlv, cdata->data, data_size))
+		ret = -EFAULT;
+
+out:
+	pm_runtime_mark_last_busy(sdev->dev);
+	err = pm_runtime_put_autosuspend(sdev->dev);
+	if (err < 0)
+		dev_err_ratelimited(sdev->dev,
+				    "error: bytes_ext get failed to idle %d\n",
+				    err);
+	return ret;
+}
diff --git a/sound/soc/sof/core.c b/sound/soc/sof/core.c
new file mode 100644
index 000000000000..32105e0fabe8
--- /dev/null
+++ b/sound/soc/sof/core.c
@@ -0,0 +1,508 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license.  When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2018 Intel Corporation. All rights reserved.
+//
+// Author: Liam Girdwood <liam.r.girdwood@linux.intel.com>
+//
+
+#include <linux/firmware.h>
+#include <linux/module.h>
+#include <asm/unaligned.h>
+#include <sound/soc.h>
+#include <sound/sof.h>
+#include "sof-priv.h"
+#include "ops.h"
+
+/* SOF defaults if not provided by the platform in ms */
+#define TIMEOUT_DEFAULT_IPC_MS  5
+#define TIMEOUT_DEFAULT_BOOT_MS 100
+
+/*
+ * Generic object lookup APIs.
+ */
+
+struct snd_sof_pcm *snd_sof_find_spcm_name(struct snd_sof_dev *sdev,
+					   const char *name)
+{
+	struct snd_sof_pcm *spcm;
+
+	list_for_each_entry(spcm, &sdev->pcm_list, list) {
+		/* match with PCM dai name */
+		if (strcmp(spcm->pcm.dai_name, name) == 0)
+			return spcm;
+
+		/* match with playback caps name if set */
+		if (*spcm->pcm.caps[0].name &&
+		    !strcmp(spcm->pcm.caps[0].name, name))
+			return spcm;
+
+		/* match with capture caps name if set */
+		if (*spcm->pcm.caps[1].name &&
+		    !strcmp(spcm->pcm.caps[1].name, name))
+			return spcm;
+	}
+
+	return NULL;
+}
+
+struct snd_sof_pcm *snd_sof_find_spcm_comp(struct snd_sof_dev *sdev,
+					   unsigned int comp_id,
+					   int *direction)
+{
+	struct snd_sof_pcm *spcm;
+
+	list_for_each_entry(spcm, &sdev->pcm_list, list) {
+		if (spcm->stream[SNDRV_PCM_STREAM_PLAYBACK].comp_id == comp_id) {
+			*direction = SNDRV_PCM_STREAM_PLAYBACK;
+			return spcm;
+		}
+		if (spcm->stream[SNDRV_PCM_STREAM_CAPTURE].comp_id == comp_id) {
+			*direction = SNDRV_PCM_STREAM_CAPTURE;
+			return spcm;
+		}
+	}
+
+	return NULL;
+}
+
+struct snd_sof_pcm *snd_sof_find_spcm_pcm_id(struct snd_sof_dev *sdev,
+					     unsigned int pcm_id)
+{
+	struct snd_sof_pcm *spcm;
+
+	list_for_each_entry(spcm, &sdev->pcm_list, list) {
+		if (le32_to_cpu(spcm->pcm.pcm_id) == pcm_id)
+			return spcm;
+	}
+
+	return NULL;
+}
+
+struct snd_sof_widget *snd_sof_find_swidget(struct snd_sof_dev *sdev,
+					    const char *name)
+{
+	struct snd_sof_widget *swidget;
+
+	list_for_each_entry(swidget, &sdev->widget_list, list) {
+		if (strcmp(name, swidget->widget->name) == 0)
+			return swidget;
+	}
+
+	return NULL;
+}
+
+/* find widget by stream name and direction */
+struct snd_sof_widget *snd_sof_find_swidget_sname(struct snd_sof_dev *sdev,
+						  const char *pcm_name, int dir)
+{
+	struct snd_sof_widget *swidget;
+	enum snd_soc_dapm_type type;
+
+	if (dir == SNDRV_PCM_STREAM_PLAYBACK)
+		type = snd_soc_dapm_aif_in;
+	else
+		type = snd_soc_dapm_aif_out;
+
+	list_for_each_entry(swidget, &sdev->widget_list, list) {
+		if (!strcmp(pcm_name, swidget->widget->sname) && swidget->id == type)
+			return swidget;
+	}
+
+	return NULL;
+}
+
+struct snd_sof_dai *snd_sof_find_dai(struct snd_sof_dev *sdev,
+				     const char *name)
+{
+	struct snd_sof_dai *dai;
+
+	list_for_each_entry(dai, &sdev->dai_list, list) {
+		if (dai->name && (strcmp(name, dai->name) == 0))
+			return dai;
+	}
+
+	return NULL;
+}
+
+/*
+ * FW Panic/fault handling.
+ */
+
+struct sof_panic_msg {
+	u32 id;
+	const char *msg;
+};
+
+/* standard FW panic types */
+static const struct sof_panic_msg panic_msg[] = {
+	{SOF_IPC_PANIC_MEM, "out of memory"},
+	{SOF_IPC_PANIC_WORK, "work subsystem init failed"},
+	{SOF_IPC_PANIC_IPC, "IPC subsystem init failed"},
+	{SOF_IPC_PANIC_ARCH, "arch init failed"},
+	{SOF_IPC_PANIC_PLATFORM, "platform init failed"},
+	{SOF_IPC_PANIC_TASK, "scheduler init failed"},
+	{SOF_IPC_PANIC_EXCEPTION, "runtime exception"},
+	{SOF_IPC_PANIC_DEADLOCK, "deadlock"},
+	{SOF_IPC_PANIC_STACK, "stack overflow"},
+	{SOF_IPC_PANIC_IDLE, "can't enter idle"},
+	{SOF_IPC_PANIC_WFI, "invalid wait state"},
+	{SOF_IPC_PANIC_ASSERT, "assertion failed"},
+};
+
+/*
+ * helper to be called from .dbg_dump callbacks. No error code is
+ * provided, it's left as an exercise for the caller of .dbg_dump
+ * (typically IPC or loader)
+ */
+void snd_sof_get_status(struct snd_sof_dev *sdev, u32 panic_code,
+			u32 tracep_code, void *oops,
+			struct sof_ipc_panic_info *panic_info,
+			void *stack, size_t stack_words)
+{
+	u32 code;
+	int i;
+
+	/* is firmware dead ? */
+	if ((panic_code & SOF_IPC_PANIC_MAGIC_MASK) != SOF_IPC_PANIC_MAGIC) {
+		dev_err(sdev->dev, "error: unexpected fault 0x%8.8x trace 0x%8.8x\n",
+			panic_code, tracep_code);
+		return; /* no fault ? */
+	}
+
+	code = panic_code & (SOF_IPC_PANIC_MAGIC_MASK | SOF_IPC_PANIC_CODE_MASK);
+
+	for (i = 0; i < ARRAY_SIZE(panic_msg); i++) {
+		if (panic_msg[i].id == code) {
+			dev_err(sdev->dev, "error: %s\n", panic_msg[i].msg);
+			dev_err(sdev->dev, "error: trace point %8.8x\n",
+				tracep_code);
+			goto out;
+		}
+	}
+
+	/* unknown error */
+	dev_err(sdev->dev, "error: unknown reason %8.8x\n", panic_code);
+	dev_err(sdev->dev, "error: trace point %8.8x\n", tracep_code);
+
+out:
+	dev_err(sdev->dev, "error: panic at %s:%d\n",
+		panic_info->filename, panic_info->linenum);
+	sof_oops(sdev, oops);
+	sof_stack(sdev, oops, stack, stack_words);
+}
+EXPORT_SYMBOL(snd_sof_get_status);
+
+/*
+ * Generic buffer page table creation.
+ * Take the each physical page address and drop the least significant unused
+ * bits from each (based on PAGE_SIZE). Then pack valid page address bits
+ * into compressed page table.
+ */
+
+int snd_sof_create_page_table(struct snd_sof_dev *sdev,
+			      struct snd_dma_buffer *dmab,
+			      unsigned char *page_table, size_t size)
+{
+	int i, pages;
+
+	pages = snd_sgbuf_aligned_pages(size);
+
+	dev_dbg(sdev->dev, "generating page table for %p size 0x%zx pages %d\n",
+		dmab->area, size, pages);
+
+	for (i = 0; i < pages; i++) {
+		/*
+		 * The number of valid address bits for each page is 20.
+		 * idx determines the byte position within page_table
+		 * where the current page's address is stored
+		 * in the compressed page_table.
+		 * This can be calculated by multiplying the page number by 2.5.
+		 */
+		u32 idx = (5 * i) >> 1;
+		u32 pfn = snd_sgbuf_get_addr(dmab, i * PAGE_SIZE) >> PAGE_SHIFT;
+		u8 *pg_table;
+
+		dev_vdbg(sdev->dev, "pfn i %i idx %d pfn %x\n", i, idx, pfn);
+
+		pg_table = (u8 *)(page_table + idx);
+
+		/*
+		 * pagetable compression:
+		 * byte 0     byte 1     byte 2     byte 3     byte 4     byte 5
+		 * ___________pfn 0__________ __________pfn 1___________  _pfn 2...
+		 * .... ....  .... ....  .... ....  .... ....  .... ....  ....
+		 * It is created by:
+		 * 1. set current location to 0, PFN index i to 0
+		 * 2. put pfn[i] at current location in Little Endian byte order
+		 * 3. calculate an intermediate value as
+		 *    x = (pfn[i+1] << 4) | (pfn[i] & 0xf)
+		 * 4. put x at offset (current location + 2) in LE byte order
+		 * 5. increment current location by 5 bytes, increment i by 2
+		 * 6. continue to (2)
+		 */
+		if (i & 1)
+			put_unaligned_le32((pg_table[0] & 0xf) | pfn << 4,
+					   pg_table);
+		else
+			put_unaligned_le32(pfn, pg_table);
+	}
+
+	return pages;
+}
+
+/*
+ * SOF Driver enumeration.
+ */
+static int sof_machine_check(struct snd_sof_dev *sdev)
+{
+	struct snd_sof_pdata *plat_data = sdev->pdata;
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_NOCODEC)
+	struct snd_soc_acpi_mach *machine;
+	int ret;
+#endif
+
+	if (plat_data->machine)
+		return 0;
+
+#if !IS_ENABLED(CONFIG_SND_SOC_SOF_NOCODEC)
+	dev_err(sdev->dev, "error: no matching ASoC machine driver found - aborting probe\n");
+	return -ENODEV;
+#else
+	/* fallback to nocodec mode */
+	dev_warn(sdev->dev, "No ASoC machine driver found - using nocodec\n");
+	machine = devm_kzalloc(sdev->dev, sizeof(*machine), GFP_KERNEL);
+	if (!machine)
+		return -ENOMEM;
+
+	ret = sof_nocodec_setup(sdev->dev, plat_data, machine,
+				plat_data->desc, plat_data->desc->ops);
+	if (ret < 0)
+		return ret;
+
+	plat_data->machine = machine;
+
+	return 0;
+#endif
+}
+
+static int sof_probe_continue(struct snd_sof_dev *sdev)
+{
+	struct snd_sof_pdata *plat_data = sdev->pdata;
+	const char *drv_name;
+	const void *mach;
+	int size;
+	int ret;
+
+	/* probe the DSP hardware */
+	ret = snd_sof_probe(sdev);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: failed to probe DSP %d\n", ret);
+		return ret;
+	}
+
+	/* check machine info */
+	ret = sof_machine_check(sdev);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: failed to get machine info %d\n",
+			ret);
+		goto dbg_err;
+	}
+
+	/* set up platform component driver */
+	snd_sof_new_platform_drv(sdev);
+
+	/* register any debug/trace capabilities */
+	ret = snd_sof_dbg_init(sdev);
+	if (ret < 0) {
+		/*
+		 * debugfs issues are suppressed in snd_sof_dbg_init() since
+		 * we cannot rely on debugfs
+		 * here we trap errors due to memory allocation only.
+		 */
+		dev_err(sdev->dev, "error: failed to init DSP trace/debug %d\n",
+			ret);
+		goto dbg_err;
+	}
+
+	/* init the IPC */
+	sdev->ipc = snd_sof_ipc_init(sdev);
+	if (!sdev->ipc) {
+		dev_err(sdev->dev, "error: failed to init DSP IPC %d\n", ret);
+		goto ipc_err;
+	}
+
+	/* load the firmware */
+	ret = snd_sof_load_firmware(sdev);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: failed to load DSP firmware %d\n",
+			ret);
+		goto fw_load_err;
+	}
+
+	/* boot the firmware */
+	ret = snd_sof_run_firmware(sdev);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: failed to boot DSP firmware %d\n",
+			ret);
+		goto fw_run_err;
+	}
+
+	/* init DMA trace */
+	ret = snd_sof_init_trace(sdev);
+	if (ret < 0) {
+		/* non fatal */
+		dev_warn(sdev->dev,
+			 "warning: failed to initialize trace %d\n", ret);
+	}
+
+	/* hereafter all FW boot flows are for PM reasons */
+	sdev->first_boot = false;
+
+	/* now register audio DSP platform driver and dai */
+	ret = devm_snd_soc_register_component(sdev->dev, &sdev->plat_drv,
+					      sof_ops(sdev)->drv,
+					      sof_ops(sdev)->num_drv);
+	if (ret < 0) {
+		dev_err(sdev->dev,
+			"error: failed to register DSP DAI driver %d\n", ret);
+		goto fw_run_err;
+	}
+
+	drv_name = plat_data->machine->drv_name;
+	mach = (const void *)plat_data->machine;
+	size = sizeof(*plat_data->machine);
+
+	/* register machine driver, pass machine info as pdata */
+	plat_data->pdev_mach =
+		platform_device_register_data(sdev->dev, drv_name,
+					      PLATFORM_DEVID_NONE, mach, size);
+
+	if (IS_ERR(plat_data->pdev_mach)) {
+		ret = PTR_ERR(plat_data->pdev_mach);
+		goto comp_err;
+	}
+
+	dev_dbg(sdev->dev, "created machine %s\n",
+		dev_name(&plat_data->pdev_mach->dev));
+
+	if (plat_data->sof_probe_complete)
+		plat_data->sof_probe_complete(sdev->dev);
+
+	return 0;
+
+comp_err:
+	snd_soc_unregister_component(sdev->dev);
+fw_run_err:
+	snd_sof_fw_unload(sdev);
+fw_load_err:
+	snd_sof_ipc_free(sdev);
+ipc_err:
+	snd_sof_free_debug(sdev);
+dbg_err:
+	snd_sof_remove(sdev);
+
+	return ret;
+}
+
+static void sof_probe_work(struct work_struct *work)
+{
+	struct snd_sof_dev *sdev =
+		container_of(work, struct snd_sof_dev, probe_work);
+	int ret;
+
+	ret = sof_probe_continue(sdev);
+	if (ret < 0) {
+		/* errors cannot be propagated, log */
+		dev_err(sdev->dev, "error: %s failed err: %d\n", __func__, ret);
+	}
+}
+
+int snd_sof_device_probe(struct device *dev, struct snd_sof_pdata *plat_data)
+{
+	struct snd_sof_dev *sdev;
+
+	sdev = devm_kzalloc(dev, sizeof(*sdev), GFP_KERNEL);
+	if (!sdev)
+		return -ENOMEM;
+
+	/* initialize sof device */
+	sdev->dev = dev;
+
+	sdev->pdata = plat_data;
+	sdev->first_boot = true;
+	dev_set_drvdata(dev, sdev);
+
+	/* check all mandatory ops */
+	if (!sof_ops(sdev) || !sof_ops(sdev)->probe || !sof_ops(sdev)->run ||
+	    !sof_ops(sdev)->block_read || !sof_ops(sdev)->block_write ||
+	    !sof_ops(sdev)->send_msg || !sof_ops(sdev)->load_firmware ||
+	    !sof_ops(sdev)->ipc_msg_data || !sof_ops(sdev)->ipc_pcm_params)
+		return -EINVAL;
+
+	INIT_LIST_HEAD(&sdev->pcm_list);
+	INIT_LIST_HEAD(&sdev->kcontrol_list);
+	INIT_LIST_HEAD(&sdev->widget_list);
+	INIT_LIST_HEAD(&sdev->dai_list);
+	INIT_LIST_HEAD(&sdev->route_list);
+	spin_lock_init(&sdev->ipc_lock);
+	spin_lock_init(&sdev->hw_lock);
+
+	if (IS_ENABLED(CONFIG_SND_SOC_SOF_PROBE_WORK_QUEUE))
+		INIT_WORK(&sdev->probe_work, sof_probe_work);
+
+	/* set default timeouts if none provided */
+	if (plat_data->desc->ipc_timeout == 0)
+		sdev->ipc_timeout = TIMEOUT_DEFAULT_IPC_MS;
+	else
+		sdev->ipc_timeout = plat_data->desc->ipc_timeout;
+	if (plat_data->desc->boot_timeout == 0)
+		sdev->boot_timeout = TIMEOUT_DEFAULT_BOOT_MS;
+	else
+		sdev->boot_timeout = plat_data->desc->boot_timeout;
+
+	if (IS_ENABLED(CONFIG_SND_SOC_SOF_PROBE_WORK_QUEUE)) {
+		schedule_work(&sdev->probe_work);
+		return 0;
+	}
+
+	return sof_probe_continue(sdev);
+}
+EXPORT_SYMBOL(snd_sof_device_probe);
+
+int snd_sof_device_remove(struct device *dev)
+{
+	struct snd_sof_dev *sdev = dev_get_drvdata(dev);
+	struct snd_sof_pdata *pdata = sdev->pdata;
+
+	if (IS_ENABLED(CONFIG_SND_SOC_SOF_PROBE_WORK_QUEUE))
+		cancel_work_sync(&sdev->probe_work);
+
+	snd_sof_fw_unload(sdev);
+	snd_sof_ipc_free(sdev);
+	snd_sof_free_debug(sdev);
+	snd_sof_free_trace(sdev);
+	snd_sof_remove(sdev);
+
+	/*
+	 * Unregister machine driver. This will unbind the snd_card which
+	 * will remove the component driver and unload the topology
+	 * before freeing the snd_card.
+	 */
+	if (!IS_ERR_OR_NULL(pdata->pdev_mach))
+		platform_device_unregister(pdata->pdev_mach);
+
+	/* release firmware */
+	release_firmware(pdata->fw);
+	pdata->fw = NULL;
+
+	return 0;
+}
+EXPORT_SYMBOL(snd_sof_device_remove);
+
+MODULE_AUTHOR("Liam Girdwood");
+MODULE_DESCRIPTION("Sound Open Firmware (SOF) Core");
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_ALIAS("platform:sof-audio");
diff --git a/sound/soc/sof/debug.c b/sound/soc/sof/debug.c
new file mode 100644
index 000000000000..55f1d808dba0
--- /dev/null
+++ b/sound/soc/sof/debug.c
@@ -0,0 +1,232 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license.  When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2018 Intel Corporation. All rights reserved.
+//
+// Author: Liam Girdwood <liam.r.girdwood@linux.intel.com>
+//
+// Generic debug routines used to export DSP MMIO and memories to userspace
+// for firmware debugging.
+//
+
+#include <linux/debugfs.h>
+#include <linux/io.h>
+#include <linux/pm_runtime.h>
+#include "sof-priv.h"
+#include "ops.h"
+
+static ssize_t sof_dfsentry_read(struct file *file, char __user *buffer,
+				 size_t count, loff_t *ppos)
+{
+	struct snd_sof_dfsentry *dfse = file->private_data;
+	struct snd_sof_dev *sdev = dfse->sdev;
+	loff_t pos = *ppos;
+	size_t size_ret;
+	int skip = 0;
+	int size;
+	u8 *buf;
+
+	size = dfse->size;
+
+	/* validate position & count */
+	if (pos < 0)
+		return -EINVAL;
+	if (pos >= size || !count)
+		return 0;
+	/* find the minimum. min() is not used since it adds sparse warnings */
+	if (count > size - pos)
+		count = size - pos;
+
+	/* align io read start to u32 multiple */
+	pos = ALIGN_DOWN(pos, 4);
+
+	/* intermediate buffer size must be u32 multiple */
+	size = ALIGN(count, 4);
+
+	/* if start position is unaligned, read extra u32 */
+	if (unlikely(pos != *ppos)) {
+		skip = *ppos - pos;
+		if (pos + size + 4 < dfse->size)
+			size += 4;
+	}
+
+	buf = kzalloc(size, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	if (dfse->type == SOF_DFSENTRY_TYPE_IOMEM) {
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG_ENABLE_DEBUGFS_CACHE)
+		/*
+		 * If the DSP is active: copy from IO.
+		 * If the DSP is suspended:
+		 *	- Copy from IO if the memory is always accessible.
+		 *	- Otherwise, copy from cached buffer.
+		 */
+		if (pm_runtime_active(sdev->dev) ||
+		    dfse->access_type == SOF_DEBUGFS_ACCESS_ALWAYS) {
+			memcpy_fromio(buf, dfse->io_mem + pos, size);
+		} else {
+			dev_info(sdev->dev,
+				 "Copying cached debugfs data\n");
+			memcpy(buf, dfse->cache_buf + pos, size);
+		}
+#else
+		/* if the DSP is in D3 */
+		if (!pm_runtime_active(sdev->dev) &&
+		    dfse->access_type == SOF_DEBUGFS_ACCESS_D0_ONLY) {
+			dev_err(sdev->dev,
+				"error: debugfs entry %s cannot be read in DSP D3\n",
+				dfse->dfsentry->d_name.name);
+			kfree(buf);
+			return -EINVAL;
+		}
+
+		memcpy_fromio(buf, dfse->io_mem + pos, size);
+#endif
+	} else {
+		memcpy(buf, ((u8 *)(dfse->buf) + pos), size);
+	}
+
+	/* copy to userspace */
+	size_ret = copy_to_user(buffer, buf + skip, count);
+
+	kfree(buf);
+
+	/* update count & position if copy succeeded */
+	if (size_ret)
+		return -EFAULT;
+
+	*ppos = pos + count;
+
+	return count;
+}
+
+static const struct file_operations sof_dfs_fops = {
+	.open = simple_open,
+	.read = sof_dfsentry_read,
+	.llseek = default_llseek,
+};
+
+/* create FS entry for debug files that can expose DSP memories, registers */
+int snd_sof_debugfs_io_item(struct snd_sof_dev *sdev,
+			    void __iomem *base, size_t size,
+			    const char *name,
+			    enum sof_debugfs_access_type access_type)
+{
+	struct snd_sof_dfsentry *dfse;
+
+	if (!sdev)
+		return -EINVAL;
+
+	dfse = devm_kzalloc(sdev->dev, sizeof(*dfse), GFP_KERNEL);
+	if (!dfse)
+		return -ENOMEM;
+
+	dfse->type = SOF_DFSENTRY_TYPE_IOMEM;
+	dfse->io_mem = base;
+	dfse->size = size;
+	dfse->sdev = sdev;
+	dfse->access_type = access_type;
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG_ENABLE_DEBUGFS_CACHE)
+	/*
+	 * allocate cache buffer that will be used to save the mem window
+	 * contents prior to suspend
+	 */
+	if (access_type == SOF_DEBUGFS_ACCESS_D0_ONLY) {
+		dfse->cache_buf = devm_kzalloc(sdev->dev, size, GFP_KERNEL);
+		if (!dfse->cache_buf)
+			return -ENOMEM;
+	}
+#endif
+
+	dfse->dfsentry = debugfs_create_file(name, 0444, sdev->debugfs_root,
+					     dfse, &sof_dfs_fops);
+	if (!dfse->dfsentry) {
+		/* can't rely on debugfs, only log error and keep going */
+		dev_err(sdev->dev, "error: cannot create debugfs entry %s\n",
+			name);
+	} else {
+		/* add to dfsentry list */
+		list_add(&dfse->list, &sdev->dfsentry_list);
+
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(snd_sof_debugfs_io_item);
+
+/* create FS entry for debug files to expose kernel memory */
+int snd_sof_debugfs_buf_item(struct snd_sof_dev *sdev,
+			     void *base, size_t size,
+			     const char *name)
+{
+	struct snd_sof_dfsentry *dfse;
+
+	if (!sdev)
+		return -EINVAL;
+
+	dfse = devm_kzalloc(sdev->dev, sizeof(*dfse), GFP_KERNEL);
+	if (!dfse)
+		return -ENOMEM;
+
+	dfse->type = SOF_DFSENTRY_TYPE_BUF;
+	dfse->buf = base;
+	dfse->size = size;
+	dfse->sdev = sdev;
+
+	dfse->dfsentry = debugfs_create_file(name, 0444, sdev->debugfs_root,
+					     dfse, &sof_dfs_fops);
+	if (!dfse->dfsentry) {
+		/* can't rely on debugfs, only log error and keep going */
+		dev_err(sdev->dev, "error: cannot create debugfs entry %s\n",
+			name);
+	} else {
+		/* add to dfsentry list */
+		list_add(&dfse->list, &sdev->dfsentry_list);
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(snd_sof_debugfs_buf_item);
+
+int snd_sof_dbg_init(struct snd_sof_dev *sdev)
+{
+	const struct snd_sof_dsp_ops *ops = sof_ops(sdev);
+	const struct snd_sof_debugfs_map *map;
+	int i;
+	int err;
+
+	/* use "sof" as top level debugFS dir */
+	sdev->debugfs_root = debugfs_create_dir("sof", NULL);
+	if (IS_ERR_OR_NULL(sdev->debugfs_root)) {
+		dev_err(sdev->dev, "error: failed to create debugfs directory\n");
+		return 0;
+	}
+
+	/* init dfsentry list */
+	INIT_LIST_HEAD(&sdev->dfsentry_list);
+
+	/* create debugFS files for platform specific MMIO/DSP memories */
+	for (i = 0; i < ops->debug_map_count; i++) {
+		map = &ops->debug_map[i];
+
+		err = snd_sof_debugfs_io_item(sdev, sdev->bar[map->bar] +
+					      map->offset, map->size,
+					      map->name, map->access_type);
+		/* errors are only due to memory allocation, not debugfs */
+		if (err < 0)
+			return err;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(snd_sof_dbg_init);
+
+void snd_sof_free_debug(struct snd_sof_dev *sdev)
+{
+	debugfs_remove_recursive(sdev->debugfs_root);
+}
+EXPORT_SYMBOL_GPL(snd_sof_free_debug);
diff --git a/sound/soc/sof/intel/Kconfig b/sound/soc/sof/intel/Kconfig
new file mode 100644
index 000000000000..603e0db4f012
--- /dev/null
+++ b/sound/soc/sof/intel/Kconfig
@@ -0,0 +1,230 @@
+config SND_SOC_SOF_INTEL_TOPLEVEL
+	bool "SOF support for Intel audio DSPs"
+	depends on X86 || COMPILE_TEST
+	help
+	  This adds support for Sound Open Firmware for Intel(R) platforms.
+	  Say Y if you have such a device.
+	  If unsure select "N".
+
+if SND_SOC_SOF_INTEL_TOPLEVEL
+
+config SND_SOC_SOF_INTEL_ACPI
+	tristate
+	select SND_SOC_SOF_BAYTRAIL  if SND_SOC_SOF_BAYTRAIL_SUPPORT
+	select SND_SOC_SOF_BROADWELL if SND_SOC_SOF_BROADWELL_SUPPORT
+	help
+	  This option is not user-selectable but automagically handled by
+	  'select' statements at a higher level
+
+config SND_SOC_SOF_INTEL_PCI
+	tristate
+	select SND_SOC_SOF_MERRIFIELD  if SND_SOC_SOF_MERRIFIELD_SUPPORT
+	select SND_SOC_SOF_APOLLOLAKE  if SND_SOC_SOF_APOLLOLAKE_SUPPORT
+	select SND_SOC_SOF_GEMINILAKE  if SND_SOC_SOF_GEMINILAKE_SUPPORT
+	select SND_SOC_SOF_CANNONLAKE  if SND_SOC_SOF_CANNONLAKE_SUPPORT
+	select SND_SOC_SOF_COFFEELAKE  if SND_SOC_SOF_COFFEELAKE_SUPPORT
+	select SND_SOC_SOF_ICELAKE     if SND_SOC_SOF_ICELAKE_SUPPORT
+	help
+	  This option is not user-selectable but automagically handled by
+	  'select' statements at a higher level
+
+config SND_SOC_SOF_INTEL_HIFI_EP_IPC
+	tristate
+	help
+          This option is not user-selectable but automagically handled by
+	  'select' statements at a higher level
+
+config SND_SOC_SOF_INTEL_ATOM_HIFI_EP
+	tristate
+	select SND_SOC_SOF_INTEL_COMMON
+	select SND_SOC_SOF_INTEL_HIFI_EP_IPC
+	help
+	  This option is not user-selectable but automagically handled by
+	  'select' statements at a higher level
+
+config SND_SOC_SOF_INTEL_COMMON
+	tristate
+	select SND_SOC_ACPI_INTEL_MATCH
+	select SND_SOC_SOF_XTENSA
+	select SND_SOC_INTEL_MACH
+	select SND_SOC_ACPI if ACPI
+	help
+	  This option is not user-selectable but automagically handled by
+	  'select' statements at a higher level
+
+if SND_SOC_SOF_INTEL_ACPI
+
+config SND_SOC_SOF_BAYTRAIL_SUPPORT
+	bool "SOF support for Baytrail, Braswell and Cherrytrail"
+	help
+	  This adds support for Sound Open Firmware for Intel(R) platforms
+	  using the Baytrail, Braswell or Cherrytrail processors.
+	  Say Y if you have such a device.
+	  If unsure select "N".
+
+config SND_SOC_SOF_BAYTRAIL
+	tristate
+	select SND_SOC_SOF_INTEL_ATOM_HIFI_EP
+	help
+	  This option is not user-selectable but automagically handled by
+	  'select' statements at a higher level
+
+config SND_SOC_SOF_BROADWELL_SUPPORT
+	bool "SOF support for Broadwell"
+	help
+	  This adds support for Sound Open Firmware for Intel(R) platforms
+	  using the Broadwell processors.
+	  Say Y if you have such a device.
+	  If unsure select "N".
+
+config SND_SOC_SOF_BROADWELL
+	tristate
+	select SND_SOC_SOF_INTEL_COMMON
+	select SND_SOC_SOF_INTEL_HIFI_EP_IPC
+	help
+	  This option is not user-selectable but automagically handled by
+	  'select' statements at a higher level
+
+endif ## SND_SOC_SOF_INTEL_ACPI
+
+if SND_SOC_SOF_INTEL_PCI
+
+config SND_SOC_SOF_MERRIFIELD_SUPPORT
+	bool "SOF support for Tangier/Merrifield"
+	help
+	  This adds support for Sound Open Firmware for Intel(R) platforms
+	  using the Tangier/Merrifield processors.
+	  Say Y if you have such a device.
+	  If unsure select "N".
+
+config SND_SOC_SOF_MERRIFIELD
+	tristate
+	select SND_SOC_SOF_INTEL_ATOM_HIFI_EP
+	help
+	  This option is not user-selectable but automagically handled by
+	  'select' statements at a higher level
+
+config SND_SOC_SOF_APOLLOLAKE_SUPPORT
+	bool "SOF support for Apollolake"
+	help
+	  This adds support for Sound Open Firmware for Intel(R) platforms
+	  using the Apollolake processors.
+	  Say Y if you have such a device.
+	  If unsure select "N".
+
+config SND_SOC_SOF_APOLLOLAKE
+	tristate
+	select SND_SOC_SOF_HDA_COMMON
+	help
+	  This option is not user-selectable but automagically handled by
+	  'select' statements at a higher level
+
+config SND_SOC_SOF_GEMINILAKE_SUPPORT
+	bool "SOF support for GeminiLake"
+	help
+	  This adds support for Sound Open Firmware for Intel(R) platforms
+	  using the Geminilake processors.
+	  Say Y if you have such a device.
+	  If unsure select "N".
+
+config SND_SOC_SOF_GEMINILAKE
+	tristate
+	select SND_SOC_SOF_HDA_COMMON
+	help
+	  This option is not user-selectable but automagically handled by
+	  'select' statements at a higher level
+
+config SND_SOC_SOF_CANNONLAKE_SUPPORT
+	bool "SOF support for Cannonlake"
+	help
+	  This adds support for Sound Open Firmware for Intel(R) platforms
+	  using the Cannonlake processors.
+	  Say Y if you have such a device.
+	  If unsure select "N".
+
+config SND_SOC_SOF_CANNONLAKE
+	tristate
+	select SND_SOC_SOF_HDA_COMMON
+	help
+	  This option is not user-selectable but automagically handled by
+	  'select' statements at a higher level
+
+config SND_SOC_SOF_COFFEELAKE_SUPPORT
+	bool "SOF support for CoffeeLake"
+	help
+	  This adds support for Sound Open Firmware for Intel(R) platforms
+	  using the Coffeelake processors.
+	  Say Y if you have such a device.
+	  If unsure select "N".
+
+config SND_SOC_SOF_COFFEELAKE
+	tristate
+	select SND_SOC_SOF_HDA_COMMON
+	help
+	  This option is not user-selectable but automagically handled by
+	  'select' statements at a higher level
+
+config SND_SOC_SOF_ICELAKE_SUPPORT
+	bool "SOF support for Icelake"
+	help
+	  This adds support for Sound Open Firmware for Intel(R) platforms
+	  using the Icelake processors.
+	  Say Y if you have such a device.
+	  If unsure select "N".
+
+config SND_SOC_SOF_ICELAKE
+	tristate
+	select SND_SOC_SOF_HDA_COMMON
+	help
+	  This option is not user-selectable but automagically handled by
+	  'select' statements at a higher level
+
+config SND_SOC_SOF_HDA_COMMON
+	tristate
+	select SND_SOC_SOF_INTEL_COMMON
+	select SND_SOC_SOF_HDA_LINK_BASELINE
+	help
+	  This option is not user-selectable but automagically handled by
+	  'select' statements at a higher level
+
+if SND_SOC_SOF_HDA_COMMON
+
+config SND_SOC_SOF_HDA_LINK
+	bool "SOF support for HDA Links(HDA/HDMI)"
+	depends on SND_SOC_SOF_NOCODEC=n
+	select SND_SOC_SOF_PROBE_WORK_QUEUE
+	help
+	  This adds support for HDA links(HDA/HDMI) with Sound Open Firmware
+		  for Intel(R) platforms.
+	  Say Y if you want to enable HDA links with SOF.
+	  If unsure select "N".
+
+config SND_SOC_SOF_HDA_AUDIO_CODEC
+	bool "SOF support for HDAudio codecs"
+	depends on SND_SOC_SOF_HDA_LINK
+	help
+	  This adds support for HDAudio codecs with Sound Open Firmware
+		  for Intel(R) platforms.
+	  Say Y if you want to enable HDAudio codecs with SOF.
+	  If unsure select "N".
+
+endif ## SND_SOC_SOF_HDA_COMMON
+
+config SND_SOC_SOF_HDA_LINK_BASELINE
+	tristate
+	select SND_SOC_SOF_HDA if SND_SOC_SOF_HDA_LINK
+	help
+	  This option is not user-selectable but automagically handled by
+	  'select' statements at a higher level
+
+config SND_SOC_SOF_HDA
+	tristate
+	select SND_HDA_EXT_CORE if SND_SOC_SOF_HDA_LINK
+	select SND_SOC_HDAC_HDA if SND_SOC_SOF_HDA_AUDIO_CODEC
+	help
+	  This option is not user-selectable but automagically handled by
+	  'select' statements at a higher level
+
+endif ## SND_SOC_SOF_INTEL_PCI
+
+endif ## SND_SOC_SOF_INTEL_TOPLEVEL
diff --git a/sound/soc/sof/intel/Makefile b/sound/soc/sof/intel/Makefile
new file mode 100644
index 000000000000..b8f58e006e29
--- /dev/null
+++ b/sound/soc/sof/intel/Makefile
@@ -0,0 +1,19 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+
+snd-sof-intel-byt-objs := byt.o
+snd-sof-intel-bdw-objs := bdw.o
+
+snd-sof-intel-ipc-objs := intel-ipc.o
+
+snd-sof-intel-hda-common-objs := hda.o hda-loader.o hda-stream.o hda-trace.o \
+				 hda-dsp.o hda-ipc.o hda-ctrl.o hda-pcm.o \
+				 hda-dai.o hda-bus.o \
+				 apl.o cnl.o
+
+snd-sof-intel-hda-objs := hda-codec.o
+
+obj-$(CONFIG_SND_SOC_SOF_INTEL_ATOM_HIFI_EP) += snd-sof-intel-byt.o
+obj-$(CONFIG_SND_SOC_SOF_BROADWELL) += snd-sof-intel-bdw.o
+obj-$(CONFIG_SND_SOC_SOF_INTEL_HIFI_EP_IPC) += snd-sof-intel-ipc.o
+obj-$(CONFIG_SND_SOC_SOF_HDA_COMMON) += snd-sof-intel-hda-common.o
+obj-$(CONFIG_SND_SOC_SOF_HDA) += snd-sof-intel-hda.o
diff --git a/sound/soc/sof/intel/apl.c b/sound/soc/sof/intel/apl.c
new file mode 100644
index 000000000000..f215d80dce2c
--- /dev/null
+++ b/sound/soc/sof/intel/apl.c
@@ -0,0 +1,113 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license.  When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2018 Intel Corporation. All rights reserved.
+//
+// Authors: Liam Girdwood <liam.r.girdwood@linux.intel.com>
+//	    Ranjani Sridharan <ranjani.sridharan@linux.intel.com>
+//	    Rander Wang <rander.wang@intel.com>
+//          Keyon Jie <yang.jie@linux.intel.com>
+//
+
+/*
+ * Hardware interface for audio DSP on Apollolake and GeminiLake
+ */
+
+#include "../sof-priv.h"
+#include "hda.h"
+
+static const struct snd_sof_debugfs_map apl_dsp_debugfs[] = {
+	{"hda", HDA_DSP_HDA_BAR, 0, 0x4000, SOF_DEBUGFS_ACCESS_ALWAYS},
+	{"pp", HDA_DSP_PP_BAR,  0, 0x1000, SOF_DEBUGFS_ACCESS_ALWAYS},
+	{"dsp", HDA_DSP_BAR,  0, 0x10000, SOF_DEBUGFS_ACCESS_ALWAYS},
+};
+
+/* apollolake ops */
+const struct snd_sof_dsp_ops sof_apl_ops = {
+	/* probe and remove */
+	.probe		= hda_dsp_probe,
+	.remove		= hda_dsp_remove,
+
+	/* Register IO */
+	.write		= sof_io_write,
+	.read		= sof_io_read,
+	.write64	= sof_io_write64,
+	.read64		= sof_io_read64,
+
+	/* Block IO */
+	.block_read	= sof_block_read,
+	.block_write	= sof_block_write,
+
+	/* doorbell */
+	.irq_handler	= hda_dsp_ipc_irq_handler,
+	.irq_thread	= hda_dsp_ipc_irq_thread,
+
+	/* ipc */
+	.send_msg	= hda_dsp_ipc_send_msg,
+	.fw_ready	= hda_dsp_ipc_fw_ready,
+
+	.ipc_msg_data	= hda_ipc_msg_data,
+	.ipc_pcm_params	= hda_ipc_pcm_params,
+
+	/* debug */
+	.debug_map	= apl_dsp_debugfs,
+	.debug_map_count	= ARRAY_SIZE(apl_dsp_debugfs),
+	.dbg_dump	= hda_dsp_dump,
+	.ipc_dump	= hda_ipc_dump,
+
+	/* stream callbacks */
+	.pcm_open	= hda_dsp_pcm_open,
+	.pcm_close	= hda_dsp_pcm_close,
+	.pcm_hw_params	= hda_dsp_pcm_hw_params,
+	.pcm_trigger	= hda_dsp_pcm_trigger,
+	.pcm_pointer	= hda_dsp_pcm_pointer,
+
+	/* firmware loading */
+	.load_firmware = snd_sof_load_firmware_raw,
+
+	/* firmware run */
+	.run = hda_dsp_cl_boot_firmware,
+
+	/* pre/post fw run */
+	.pre_fw_run = hda_dsp_pre_fw_run,
+	.post_fw_run = hda_dsp_post_fw_run,
+
+	/* dsp core power up/down */
+	.core_power_up = hda_dsp_enable_core,
+	.core_power_down = hda_dsp_core_reset_power_down,
+
+	/* trace callback */
+	.trace_init = hda_dsp_trace_init,
+	.trace_release = hda_dsp_trace_release,
+	.trace_trigger = hda_dsp_trace_trigger,
+
+	/* DAI drivers */
+	.drv		= skl_dai,
+	.num_drv	= SOF_SKL_NUM_DAIS,
+
+	/* PM */
+	.suspend		= hda_dsp_suspend,
+	.resume			= hda_dsp_resume,
+	.runtime_suspend	= hda_dsp_runtime_suspend,
+	.runtime_resume		= hda_dsp_runtime_resume,
+	.set_hw_params_upon_resume = hda_dsp_set_hw_params_upon_resume,
+};
+EXPORT_SYMBOL(sof_apl_ops);
+
+const struct sof_intel_dsp_desc apl_chip_info = {
+	/* Apollolake */
+	.cores_num = 2,
+	.init_core_mask = 1,
+	.cores_mask = HDA_DSP_CORE_MASK(0) | HDA_DSP_CORE_MASK(1),
+	.ipc_req = HDA_DSP_REG_HIPCI,
+	.ipc_req_mask = HDA_DSP_REG_HIPCI_BUSY,
+	.ipc_ack = HDA_DSP_REG_HIPCIE,
+	.ipc_ack_mask = HDA_DSP_REG_HIPCIE_DONE,
+	.ipc_ctl = HDA_DSP_REG_HIPCCTL,
+	.rom_init_timeout	= 150,
+	.ssp_count = APL_SSP_COUNT,
+	.ssp_base_offset = APL_SSP_BASE_OFFSET,
+};
+EXPORT_SYMBOL(apl_chip_info);
diff --git a/sound/soc/sof/intel/bdw.c b/sound/soc/sof/intel/bdw.c
new file mode 100644
index 000000000000..065cb868bdfa
--- /dev/null
+++ b/sound/soc/sof/intel/bdw.c
@@ -0,0 +1,713 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license.  When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2018 Intel Corporation. All rights reserved.
+//
+// Author: Liam Girdwood <liam.r.girdwood@linux.intel.com>
+//
+
+/*
+ * Hardware interface for audio DSP on Broadwell
+ */
+
+#include <linux/module.h>
+#include <sound/sof.h>
+#include <sound/sof/xtensa.h>
+#include "../ops.h"
+#include "shim.h"
+
+/* BARs */
+#define BDW_DSP_BAR 0
+#define BDW_PCI_BAR 1
+
+/*
+ * Debug
+ */
+
+/* DSP memories for BDW */
+#define IRAM_OFFSET     0xA0000
+#define BDW_IRAM_SIZE       (10 * 32 * 1024)
+#define DRAM_OFFSET     0x00000
+#define BDW_DRAM_SIZE       (20 * 32 * 1024)
+#define SHIM_OFFSET     0xFB000
+#define SHIM_SIZE       0x100
+#define MBOX_OFFSET     0x9E000
+#define MBOX_SIZE       0x1000
+#define MBOX_DUMP_SIZE 0x30
+#define EXCEPT_OFFSET	0x800
+
+/* DSP peripherals */
+#define DMAC0_OFFSET    0xFE000
+#define DMAC1_OFFSET    0xFF000
+#define DMAC_SIZE       0x420
+#define SSP0_OFFSET     0xFC000
+#define SSP1_OFFSET     0xFD000
+#define SSP_SIZE	0x100
+
+#define BDW_STACK_DUMP_SIZE	32
+
+#define BDW_PANIC_OFFSET(x)	((x) & 0xFFFF)
+
+static const struct snd_sof_debugfs_map bdw_debugfs[] = {
+	{"dmac0", BDW_DSP_BAR, DMAC0_OFFSET, DMAC_SIZE,
+	 SOF_DEBUGFS_ACCESS_ALWAYS},
+	{"dmac1", BDW_DSP_BAR, DMAC1_OFFSET, DMAC_SIZE,
+	 SOF_DEBUGFS_ACCESS_ALWAYS},
+	{"ssp0", BDW_DSP_BAR, SSP0_OFFSET, SSP_SIZE,
+	 SOF_DEBUGFS_ACCESS_ALWAYS},
+	{"ssp1", BDW_DSP_BAR, SSP1_OFFSET, SSP_SIZE,
+	 SOF_DEBUGFS_ACCESS_ALWAYS},
+	{"iram", BDW_DSP_BAR, IRAM_OFFSET, BDW_IRAM_SIZE,
+	 SOF_DEBUGFS_ACCESS_D0_ONLY},
+	{"dram", BDW_DSP_BAR, DRAM_OFFSET, BDW_DRAM_SIZE,
+	 SOF_DEBUGFS_ACCESS_D0_ONLY},
+	{"shim", BDW_DSP_BAR, SHIM_OFFSET, SHIM_SIZE,
+	 SOF_DEBUGFS_ACCESS_ALWAYS},
+};
+
+static void bdw_host_done(struct snd_sof_dev *sdev);
+static void bdw_dsp_done(struct snd_sof_dev *sdev);
+static void bdw_get_reply(struct snd_sof_dev *sdev);
+
+/*
+ * DSP Control.
+ */
+
+static int bdw_run(struct snd_sof_dev *sdev)
+{
+	/* set opportunistic mode on engine 0,1 for all channels */
+	snd_sof_dsp_update_bits(sdev, BDW_DSP_BAR, SHIM_HMDC,
+				SHIM_HMDC_HDDA_E0_ALLCH |
+				SHIM_HMDC_HDDA_E1_ALLCH, 0);
+
+	/* set DSP to RUN */
+	snd_sof_dsp_update_bits_unlocked(sdev, BDW_DSP_BAR, SHIM_CSR,
+					 SHIM_CSR_STALL, 0x0);
+
+	/* return init core mask */
+	return 1;
+}
+
+static int bdw_reset(struct snd_sof_dev *sdev)
+{
+	/* put DSP into reset and stall */
+	snd_sof_dsp_update_bits_unlocked(sdev, BDW_DSP_BAR, SHIM_CSR,
+					 SHIM_CSR_RST | SHIM_CSR_STALL,
+					 SHIM_CSR_RST | SHIM_CSR_STALL);
+
+	/* keep in reset for 10ms */
+	mdelay(10);
+
+	/* take DSP out of reset and keep stalled for FW loading */
+	snd_sof_dsp_update_bits_unlocked(sdev, BDW_DSP_BAR, SHIM_CSR,
+					 SHIM_CSR_RST | SHIM_CSR_STALL,
+					 SHIM_CSR_STALL);
+
+	return 0;
+}
+
+static int bdw_set_dsp_D0(struct snd_sof_dev *sdev)
+{
+	int tries = 10;
+	u32 reg;
+
+	/* Disable core clock gating (VDRTCTL2.DCLCGE = 0) */
+	snd_sof_dsp_update_bits_unlocked(sdev, BDW_PCI_BAR, PCI_VDRTCTL2,
+					 PCI_VDRTCL2_DCLCGE |
+					 PCI_VDRTCL2_DTCGE, 0);
+
+	/* Disable D3PG (VDRTCTL0.D3PGD = 1) */
+	snd_sof_dsp_update_bits_unlocked(sdev, BDW_PCI_BAR, PCI_VDRTCTL0,
+					 PCI_VDRTCL0_D3PGD, PCI_VDRTCL0_D3PGD);
+
+	/* Set D0 state */
+	snd_sof_dsp_update_bits_unlocked(sdev, BDW_PCI_BAR, PCI_PMCS,
+					 PCI_PMCS_PS_MASK, 0);
+
+	/* check that ADSP shim is enabled */
+	while (tries--) {
+		reg = readl(sdev->bar[BDW_PCI_BAR] + PCI_PMCS)
+			& PCI_PMCS_PS_MASK;
+		if (reg == 0)
+			goto finish;
+
+		msleep(20);
+	}
+
+	return -ENODEV;
+
+finish:
+	/*
+	 * select SSP1 19.2MHz base clock, SSP clock 0,
+	 * turn off Low Power Clock
+	 */
+	snd_sof_dsp_update_bits_unlocked(sdev, BDW_DSP_BAR, SHIM_CSR,
+					 SHIM_CSR_S1IOCS | SHIM_CSR_SBCS1 |
+					 SHIM_CSR_LPCS, 0x0);
+
+	/* stall DSP core, set clk to 192/96Mhz */
+	snd_sof_dsp_update_bits_unlocked(sdev, BDW_DSP_BAR,
+					 SHIM_CSR, SHIM_CSR_STALL |
+					 SHIM_CSR_DCS_MASK,
+					 SHIM_CSR_STALL |
+					 SHIM_CSR_DCS(4));
+
+	/* Set 24MHz MCLK, prevent local clock gating, enable SSP0 clock */
+	snd_sof_dsp_update_bits_unlocked(sdev, BDW_DSP_BAR, SHIM_CLKCTL,
+					 SHIM_CLKCTL_MASK |
+					 SHIM_CLKCTL_DCPLCG |
+					 SHIM_CLKCTL_SCOE0,
+					 SHIM_CLKCTL_MASK |
+					 SHIM_CLKCTL_DCPLCG |
+					 SHIM_CLKCTL_SCOE0);
+
+	/* Stall and reset core, set CSR */
+	bdw_reset(sdev);
+
+	/* Enable core clock gating (VDRTCTL2.DCLCGE = 1), delay 50 us */
+	snd_sof_dsp_update_bits_unlocked(sdev, BDW_PCI_BAR, PCI_VDRTCTL2,
+					 PCI_VDRTCL2_DCLCGE |
+					 PCI_VDRTCL2_DTCGE,
+					 PCI_VDRTCL2_DCLCGE |
+					 PCI_VDRTCL2_DTCGE);
+
+	usleep_range(50, 55);
+
+	/* switch on audio PLL */
+	snd_sof_dsp_update_bits_unlocked(sdev, BDW_PCI_BAR, PCI_VDRTCTL2,
+					 PCI_VDRTCL2_APLLSE_MASK, 0);
+
+	/*
+	 * set default power gating control, enable power gating control for
+	 * all blocks. that is, can't be accessed, please enable each block
+	 * before accessing.
+	 */
+	snd_sof_dsp_update_bits_unlocked(sdev, BDW_PCI_BAR, PCI_VDRTCTL0,
+					 0xfffffffC, 0x0);
+
+	/* disable DMA finish function for SSP0 & SSP1 */
+	snd_sof_dsp_update_bits_unlocked(sdev, BDW_DSP_BAR,  SHIM_CSR2,
+					 SHIM_CSR2_SDFD_SSP1,
+					 SHIM_CSR2_SDFD_SSP1);
+
+	/* set on-demond mode on engine 0,1 for all channels */
+	snd_sof_dsp_update_bits(sdev, BDW_DSP_BAR, SHIM_HMDC,
+				SHIM_HMDC_HDDA_E0_ALLCH |
+				SHIM_HMDC_HDDA_E1_ALLCH,
+				SHIM_HMDC_HDDA_E0_ALLCH |
+				SHIM_HMDC_HDDA_E1_ALLCH);
+
+	/* Enable Interrupt from both sides */
+	snd_sof_dsp_update_bits(sdev, BDW_DSP_BAR, SHIM_IMRX,
+				(SHIM_IMRX_BUSY | SHIM_IMRX_DONE), 0x0);
+	snd_sof_dsp_update_bits(sdev, BDW_DSP_BAR, SHIM_IMRD,
+				(SHIM_IMRD_DONE | SHIM_IMRD_BUSY |
+				SHIM_IMRD_SSP0 | SHIM_IMRD_DMAC), 0x0);
+
+	/* clear IPC registers */
+	snd_sof_dsp_write(sdev, BDW_DSP_BAR, SHIM_IPCX, 0x0);
+	snd_sof_dsp_write(sdev, BDW_DSP_BAR, SHIM_IPCD, 0x0);
+	snd_sof_dsp_write(sdev, BDW_DSP_BAR, 0x80, 0x6);
+	snd_sof_dsp_write(sdev, BDW_DSP_BAR, 0xe0, 0x300a);
+
+	return 0;
+}
+
+static void bdw_get_registers(struct snd_sof_dev *sdev,
+			      struct sof_ipc_dsp_oops_xtensa *xoops,
+			      struct sof_ipc_panic_info *panic_info,
+			      u32 *stack, size_t stack_words)
+{
+	/* first read regsisters */
+	sof_mailbox_read(sdev, sdev->dsp_oops_offset, xoops, sizeof(*xoops));
+
+	/* then get panic info */
+	sof_mailbox_read(sdev, sdev->dsp_oops_offset + sizeof(*xoops),
+			 panic_info, sizeof(*panic_info));
+
+	/* then get the stack */
+	sof_mailbox_read(sdev, sdev->dsp_oops_offset + sizeof(*xoops) +
+			   sizeof(*panic_info), stack,
+			   stack_words * sizeof(u32));
+}
+
+static void bdw_dump(struct snd_sof_dev *sdev, u32 flags)
+{
+	struct sof_ipc_dsp_oops_xtensa xoops;
+	struct sof_ipc_panic_info panic_info;
+	u32 stack[BDW_STACK_DUMP_SIZE];
+	u32 status, panic;
+
+	/* now try generic SOF status messages */
+	status = snd_sof_dsp_read(sdev, BDW_DSP_BAR, SHIM_IPCD);
+	panic = snd_sof_dsp_read(sdev, BDW_DSP_BAR, SHIM_IPCX);
+	bdw_get_registers(sdev, &xoops, &panic_info, stack,
+			  BDW_STACK_DUMP_SIZE);
+	snd_sof_get_status(sdev, status, panic, &xoops, &panic_info, stack,
+			   BDW_STACK_DUMP_SIZE);
+}
+
+/*
+ * IPC Doorbell IRQ handler and thread.
+ */
+
+static irqreturn_t bdw_irq_handler(int irq, void *context)
+{
+	struct snd_sof_dev *sdev = context;
+	u32 isr;
+	int ret = IRQ_NONE;
+
+	/* Interrupt arrived, check src */
+	isr = snd_sof_dsp_read(sdev, BDW_DSP_BAR, SHIM_ISRX);
+	if (isr & (SHIM_ISRX_DONE | SHIM_ISRX_BUSY))
+		ret = IRQ_WAKE_THREAD;
+
+	return ret;
+}
+
+static irqreturn_t bdw_irq_thread(int irq, void *context)
+{
+	struct snd_sof_dev *sdev = context;
+	u32 ipcx, ipcd, imrx;
+
+	imrx = snd_sof_dsp_read64(sdev, BDW_DSP_BAR, SHIM_IMRX);
+	ipcx = snd_sof_dsp_read(sdev, BDW_DSP_BAR, SHIM_IPCX);
+
+	/* reply message from DSP */
+	if (ipcx & SHIM_IPCX_DONE &&
+	    !(imrx & SHIM_IMRX_DONE)) {
+		/* Mask Done interrupt before return */
+		snd_sof_dsp_update_bits_unlocked(sdev, BDW_DSP_BAR,
+						 SHIM_IMRX, SHIM_IMRX_DONE,
+						 SHIM_IMRX_DONE);
+
+		/*
+		 * handle immediate reply from DSP core. If the msg is
+		 * found, set done bit in cmd_done which is called at the
+		 * end of message processing function, else set it here
+		 * because the done bit can't be set in cmd_done function
+		 * which is triggered by msg
+		 */
+		bdw_get_reply(sdev);
+		snd_sof_ipc_reply(sdev, ipcx);
+
+		bdw_dsp_done(sdev);
+	}
+
+	ipcd = snd_sof_dsp_read(sdev, BDW_DSP_BAR, SHIM_IPCD);
+
+	/* new message from DSP */
+	if (ipcd & SHIM_IPCD_BUSY &&
+	    !(imrx & SHIM_IMRX_BUSY)) {
+		/* Mask Busy interrupt before return */
+		snd_sof_dsp_update_bits_unlocked(sdev, BDW_DSP_BAR,
+						 SHIM_IMRX, SHIM_IMRX_BUSY,
+						 SHIM_IMRX_BUSY);
+
+		/* Handle messages from DSP Core */
+		if ((ipcd & SOF_IPC_PANIC_MAGIC_MASK) == SOF_IPC_PANIC_MAGIC) {
+			snd_sof_dsp_panic(sdev, BDW_PANIC_OFFSET(ipcx) +
+					  MBOX_OFFSET);
+		} else {
+			snd_sof_ipc_msgs_rx(sdev);
+		}
+
+		bdw_host_done(sdev);
+	}
+
+	return IRQ_HANDLED;
+}
+
+/*
+ * IPC Firmware ready.
+ */
+static void bdw_get_windows(struct snd_sof_dev *sdev)
+{
+	struct sof_ipc_window_elem *elem;
+	u32 outbox_offset = 0;
+	u32 stream_offset = 0;
+	u32 inbox_offset = 0;
+	u32 outbox_size = 0;
+	u32 stream_size = 0;
+	u32 inbox_size = 0;
+	int i;
+
+	if (!sdev->info_window) {
+		dev_err(sdev->dev, "error: have no window info\n");
+		return;
+	}
+
+	for (i = 0; i < sdev->info_window->num_windows; i++) {
+		elem = &sdev->info_window->window[i];
+
+		switch (elem->type) {
+		case SOF_IPC_REGION_UPBOX:
+			inbox_offset = elem->offset + MBOX_OFFSET;
+			inbox_size = elem->size;
+			snd_sof_debugfs_io_item(sdev,
+						sdev->bar[BDW_DSP_BAR] +
+						inbox_offset,
+						elem->size, "inbox",
+						SOF_DEBUGFS_ACCESS_D0_ONLY);
+			break;
+		case SOF_IPC_REGION_DOWNBOX:
+			outbox_offset = elem->offset + MBOX_OFFSET;
+			outbox_size = elem->size;
+			snd_sof_debugfs_io_item(sdev,
+						sdev->bar[BDW_DSP_BAR] +
+						outbox_offset,
+						elem->size, "outbox",
+						SOF_DEBUGFS_ACCESS_D0_ONLY);
+			break;
+		case SOF_IPC_REGION_TRACE:
+			snd_sof_debugfs_io_item(sdev,
+						sdev->bar[BDW_DSP_BAR] +
+						elem->offset +
+						MBOX_OFFSET,
+						elem->size, "etrace",
+						SOF_DEBUGFS_ACCESS_D0_ONLY);
+			break;
+		case SOF_IPC_REGION_DEBUG:
+			snd_sof_debugfs_io_item(sdev,
+						sdev->bar[BDW_DSP_BAR] +
+						elem->offset +
+						MBOX_OFFSET,
+						elem->size, "debug",
+						SOF_DEBUGFS_ACCESS_D0_ONLY);
+			break;
+		case SOF_IPC_REGION_STREAM:
+			stream_offset = elem->offset + MBOX_OFFSET;
+			stream_size = elem->size;
+			snd_sof_debugfs_io_item(sdev,
+						sdev->bar[BDW_DSP_BAR] +
+						stream_offset,
+						elem->size, "stream",
+						SOF_DEBUGFS_ACCESS_D0_ONLY);
+			break;
+		case SOF_IPC_REGION_REGS:
+			snd_sof_debugfs_io_item(sdev,
+						sdev->bar[BDW_DSP_BAR] +
+						elem->offset +
+						MBOX_OFFSET,
+						elem->size, "regs",
+						SOF_DEBUGFS_ACCESS_D0_ONLY);
+			break;
+		case SOF_IPC_REGION_EXCEPTION:
+			sdev->dsp_oops_offset = elem->offset + MBOX_OFFSET;
+			snd_sof_debugfs_io_item(sdev,
+						sdev->bar[BDW_DSP_BAR] +
+						elem->offset +
+						MBOX_OFFSET,
+						elem->size, "exception",
+						SOF_DEBUGFS_ACCESS_D0_ONLY);
+			break;
+		default:
+			dev_err(sdev->dev, "error: get illegal window info\n");
+			return;
+		}
+	}
+
+	if (outbox_size == 0 || inbox_size == 0) {
+		dev_err(sdev->dev, "error: get illegal mailbox window\n");
+		return;
+	}
+
+	snd_sof_dsp_mailbox_init(sdev, inbox_offset, inbox_size,
+				 outbox_offset, outbox_size);
+	sdev->stream_box.offset = stream_offset;
+	sdev->stream_box.size = stream_size;
+
+	dev_dbg(sdev->dev, " mailbox upstream 0x%x - size 0x%x\n",
+		inbox_offset, inbox_size);
+	dev_dbg(sdev->dev, " mailbox downstream 0x%x - size 0x%x\n",
+		outbox_offset, outbox_size);
+	dev_dbg(sdev->dev, " stream region 0x%x - size 0x%x\n",
+		stream_offset, stream_size);
+}
+
+/* check for ABI compatibility and create memory windows on first boot */
+static int bdw_fw_ready(struct snd_sof_dev *sdev, u32 msg_id)
+{
+	struct sof_ipc_fw_ready *fw_ready = &sdev->fw_ready;
+	u32 offset;
+	int ret;
+
+	/* mailbox must be on 4k boundary */
+	offset = MBOX_OFFSET;
+
+	dev_dbg(sdev->dev, "ipc: DSP is ready 0x%8.8x offset %d\n",
+		msg_id, offset);
+
+	/* no need to re-check version/ABI for subsequent boots */
+	if (!sdev->first_boot)
+		return 0;
+
+	/* copy data from the DSP FW ready offset */
+	sof_block_read(sdev, sdev->mmio_bar, offset, fw_ready,
+		       sizeof(*fw_ready));
+
+	snd_sof_dsp_mailbox_init(sdev, fw_ready->dspbox_offset,
+				 fw_ready->dspbox_size,
+				 fw_ready->hostbox_offset,
+				 fw_ready->hostbox_size);
+
+	/* make sure ABI version is compatible */
+	ret = snd_sof_ipc_valid(sdev);
+	if (ret < 0)
+		return ret;
+
+	/* now check for extended data */
+	snd_sof_fw_parse_ext_data(sdev, sdev->mmio_bar, MBOX_OFFSET +
+				  sizeof(struct sof_ipc_fw_ready));
+
+	bdw_get_windows(sdev);
+
+	return 0;
+}
+
+/*
+ * IPC Mailbox IO
+ */
+
+static int bdw_send_msg(struct snd_sof_dev *sdev, struct snd_sof_ipc_msg *msg)
+{
+	/* send the message */
+	sof_mailbox_write(sdev, sdev->host_box.offset, msg->msg_data,
+			  msg->msg_size);
+	snd_sof_dsp_write(sdev, BDW_DSP_BAR, SHIM_IPCX, SHIM_IPCX_BUSY);
+
+	return 0;
+}
+
+static void bdw_get_reply(struct snd_sof_dev *sdev)
+{
+	struct snd_sof_ipc_msg *msg = sdev->msg;
+	struct sof_ipc_reply reply;
+	unsigned long flags;
+	int ret = 0;
+
+	/*
+	 * Sometimes, there is unexpected reply ipc arriving. The reply
+	 * ipc belongs to none of the ipcs sent from driver.
+	 * In this case, the driver must ignore the ipc.
+	 */
+	if (!msg) {
+		dev_warn(sdev->dev, "unexpected ipc interrupt raised!\n");
+		return;
+	}
+
+	/* get reply */
+	sof_mailbox_read(sdev, sdev->host_box.offset, &reply, sizeof(reply));
+
+	spin_lock_irqsave(&sdev->ipc_lock, flags);
+
+	if (reply.error < 0) {
+		memcpy(msg->reply_data, &reply, sizeof(reply));
+		ret = reply.error;
+	} else {
+		/* reply correct size ? */
+		if (reply.hdr.size != msg->reply_size) {
+			dev_err(sdev->dev, "error: reply expected %zu got %u bytes\n",
+				msg->reply_size, reply.hdr.size);
+			ret = -EINVAL;
+		}
+
+		/* read the message */
+		if (msg->reply_size > 0)
+			sof_mailbox_read(sdev, sdev->host_box.offset,
+					 msg->reply_data, msg->reply_size);
+	}
+
+	msg->reply_error = ret;
+
+	spin_unlock_irqrestore(&sdev->ipc_lock, flags);
+}
+
+static void bdw_host_done(struct snd_sof_dev *sdev)
+{
+	/* clear BUSY bit and set DONE bit - accept new messages */
+	snd_sof_dsp_update_bits_unlocked(sdev, BDW_DSP_BAR, SHIM_IPCD,
+					 SHIM_IPCD_BUSY | SHIM_IPCD_DONE,
+					 SHIM_IPCD_DONE);
+
+	/* unmask busy interrupt */
+	snd_sof_dsp_update_bits_unlocked(sdev, BDW_DSP_BAR, SHIM_IMRX,
+					 SHIM_IMRX_BUSY, 0);
+}
+
+static void bdw_dsp_done(struct snd_sof_dev *sdev)
+{
+	/* clear DONE bit - tell DSP we have completed */
+	snd_sof_dsp_update_bits_unlocked(sdev, BDW_DSP_BAR, SHIM_IPCX,
+					 SHIM_IPCX_DONE, 0);
+
+	/* unmask Done interrupt */
+	snd_sof_dsp_update_bits_unlocked(sdev, BDW_DSP_BAR, SHIM_IMRX,
+					 SHIM_IMRX_DONE, 0);
+}
+
+/*
+ * Probe and remove.
+ */
+static int bdw_probe(struct snd_sof_dev *sdev)
+{
+	struct snd_sof_pdata *pdata = sdev->pdata;
+	const struct sof_dev_desc *desc = pdata->desc;
+	struct platform_device *pdev =
+		container_of(sdev->dev, struct platform_device, dev);
+	struct resource *mmio;
+	u32 base, size;
+	int ret;
+
+	/* LPE base */
+	mmio = platform_get_resource(pdev, IORESOURCE_MEM,
+				     desc->resindex_lpe_base);
+	if (mmio) {
+		base = mmio->start;
+		size = resource_size(mmio);
+	} else {
+		dev_err(sdev->dev, "error: failed to get LPE base at idx %d\n",
+			desc->resindex_lpe_base);
+		return -EINVAL;
+	}
+
+	dev_dbg(sdev->dev, "LPE PHY base at 0x%x size 0x%x", base, size);
+	sdev->bar[BDW_DSP_BAR] = devm_ioremap(sdev->dev, base, size);
+	if (!sdev->bar[BDW_DSP_BAR]) {
+		dev_err(sdev->dev,
+			"error: failed to ioremap LPE base 0x%x size 0x%x\n",
+			base, size);
+		return -ENODEV;
+	}
+	dev_dbg(sdev->dev, "LPE VADDR %p\n", sdev->bar[BDW_DSP_BAR]);
+
+	/* TODO: add offsets */
+	sdev->mmio_bar = BDW_DSP_BAR;
+	sdev->mailbox_bar = BDW_DSP_BAR;
+
+	/* PCI base */
+	mmio = platform_get_resource(pdev, IORESOURCE_MEM,
+				     desc->resindex_pcicfg_base);
+	if (mmio) {
+		base = mmio->start;
+		size = resource_size(mmio);
+	} else {
+		dev_err(sdev->dev, "error: failed to get PCI base at idx %d\n",
+			desc->resindex_pcicfg_base);
+		return -ENODEV;
+	}
+
+	dev_dbg(sdev->dev, "PCI base at 0x%x size 0x%x", base, size);
+	sdev->bar[BDW_PCI_BAR] = devm_ioremap(sdev->dev, base, size);
+	if (!sdev->bar[BDW_PCI_BAR]) {
+		dev_err(sdev->dev,
+			"error: failed to ioremap PCI base 0x%x size 0x%x\n",
+			base, size);
+		return -ENODEV;
+	}
+	dev_dbg(sdev->dev, "PCI VADDR %p\n", sdev->bar[BDW_PCI_BAR]);
+
+	/* register our IRQ */
+	sdev->ipc_irq = platform_get_irq(pdev, desc->irqindex_host_ipc);
+	if (sdev->ipc_irq < 0) {
+		dev_err(sdev->dev, "error: failed to get IRQ at index %d\n",
+			desc->irqindex_host_ipc);
+		return sdev->ipc_irq;
+	}
+
+	dev_dbg(sdev->dev, "using IRQ %d\n", sdev->ipc_irq);
+	ret = devm_request_threaded_irq(sdev->dev, sdev->ipc_irq,
+					bdw_irq_handler, bdw_irq_thread,
+					IRQF_SHARED, "AudioDSP", sdev);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: failed to register IRQ %d\n",
+			sdev->ipc_irq);
+		return ret;
+	}
+
+	/* enable the DSP SHIM */
+	ret = bdw_set_dsp_D0(sdev);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: failed to set DSP D0\n");
+		return ret;
+	}
+
+	/* DSP DMA can only access low 31 bits of host memory */
+	ret = dma_coerce_mask_and_coherent(sdev->dev, DMA_BIT_MASK(31));
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: failed to set DMA mask %d\n", ret);
+		return ret;
+	}
+
+	/* set default mailbox */
+	snd_sof_dsp_mailbox_init(sdev, MBOX_OFFSET, MBOX_SIZE, 0, 0);
+
+	return ret;
+}
+
+/* Broadwell DAIs */
+static struct snd_soc_dai_driver bdw_dai[] = {
+{
+	.name = "ssp0-port",
+},
+{
+	.name = "ssp1-port",
+},
+};
+
+/* broadwell ops */
+const struct snd_sof_dsp_ops sof_bdw_ops = {
+	/*Device init */
+	.probe          = bdw_probe,
+
+	/* DSP Core Control */
+	.run            = bdw_run,
+	.reset          = bdw_reset,
+
+	/* Register IO */
+	.write		= sof_io_write,
+	.read		= sof_io_read,
+	.write64	= sof_io_write64,
+	.read64		= sof_io_read64,
+
+	/* Block IO */
+	.block_read	= sof_block_read,
+	.block_write	= sof_block_write,
+
+	/* ipc */
+	.send_msg	= bdw_send_msg,
+	.fw_ready	= bdw_fw_ready,
+
+	.ipc_msg_data	= intel_ipc_msg_data,
+	.ipc_pcm_params	= intel_ipc_pcm_params,
+
+	/* debug */
+	.debug_map  = bdw_debugfs,
+	.debug_map_count    = ARRAY_SIZE(bdw_debugfs),
+	.dbg_dump   = bdw_dump,
+
+	/* stream callbacks */
+	.pcm_open	= intel_pcm_open,
+	.pcm_close	= intel_pcm_close,
+
+	/* Module loading */
+	.load_module    = snd_sof_parse_module_memcpy,
+
+	/*Firmware loading */
+	.load_firmware	= snd_sof_load_firmware_memcpy,
+
+	/* DAI drivers */
+	.drv = bdw_dai,
+	.num_drv = ARRAY_SIZE(bdw_dai)
+};
+EXPORT_SYMBOL(sof_bdw_ops);
+
+const struct sof_intel_dsp_desc bdw_chip_info = {
+	.cores_num = 1,
+	.cores_mask = 1,
+};
+EXPORT_SYMBOL(bdw_chip_info);
+
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/sound/soc/sof/intel/byt.c b/sound/soc/sof/intel/byt.c
new file mode 100644
index 000000000000..7bf9143d3106
--- /dev/null
+++ b/sound/soc/sof/intel/byt.c
@@ -0,0 +1,874 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license.  When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2018 Intel Corporation. All rights reserved.
+//
+// Author: Liam Girdwood <liam.r.girdwood@linux.intel.com>
+//
+
+/*
+ * Hardware interface for audio DSP on Baytrail, Braswell and Cherrytrail.
+ */
+
+#include <linux/module.h>
+#include <sound/sof.h>
+#include <sound/sof/xtensa.h>
+#include "../ops.h"
+#include "shim.h"
+
+/* DSP memories */
+#define IRAM_OFFSET		0x0C0000
+#define IRAM_SIZE		(80 * 1024)
+#define DRAM_OFFSET		0x100000
+#define DRAM_SIZE		(160 * 1024)
+#define SHIM_OFFSET		0x140000
+#define SHIM_SIZE		0x100
+#define MBOX_OFFSET		0x144000
+#define MBOX_SIZE		0x1000
+#define EXCEPT_OFFSET		0x800
+
+/* DSP peripherals */
+#define DMAC0_OFFSET		0x098000
+#define DMAC1_OFFSET		0x09c000
+#define DMAC2_OFFSET		0x094000
+#define DMAC_SIZE		0x420
+#define SSP0_OFFSET		0x0a0000
+#define SSP1_OFFSET		0x0a1000
+#define SSP2_OFFSET		0x0a2000
+#define SSP3_OFFSET		0x0a4000
+#define SSP4_OFFSET		0x0a5000
+#define SSP5_OFFSET		0x0a6000
+#define SSP_SIZE		0x100
+
+#define BYT_STACK_DUMP_SIZE	32
+
+#define BYT_PCI_BAR_SIZE	0x200000
+
+#define BYT_PANIC_OFFSET(x)	(((x) & GENMASK_ULL(47, 32)) >> 32)
+
+/*
+ * Debug
+ */
+
+#define MBOX_DUMP_SIZE	0x30
+
+/* BARs */
+#define BYT_DSP_BAR		0
+#define BYT_PCI_BAR		1
+#define BYT_IMR_BAR		2
+
+static const struct snd_sof_debugfs_map byt_debugfs[] = {
+	{"dmac0", BYT_DSP_BAR, DMAC0_OFFSET, DMAC_SIZE,
+	 SOF_DEBUGFS_ACCESS_ALWAYS},
+	{"dmac1", BYT_DSP_BAR,  DMAC1_OFFSET, DMAC_SIZE,
+	 SOF_DEBUGFS_ACCESS_ALWAYS},
+	{"ssp0",  BYT_DSP_BAR, SSP0_OFFSET, SSP_SIZE,
+	 SOF_DEBUGFS_ACCESS_ALWAYS},
+	{"ssp1", BYT_DSP_BAR, SSP1_OFFSET, SSP_SIZE,
+	 SOF_DEBUGFS_ACCESS_ALWAYS},
+	{"ssp2", BYT_DSP_BAR, SSP2_OFFSET, SSP_SIZE,
+	 SOF_DEBUGFS_ACCESS_ALWAYS},
+	{"iram", BYT_DSP_BAR, IRAM_OFFSET, IRAM_SIZE,
+	 SOF_DEBUGFS_ACCESS_D0_ONLY},
+	{"dram", BYT_DSP_BAR, DRAM_OFFSET, DRAM_SIZE,
+	 SOF_DEBUGFS_ACCESS_D0_ONLY},
+	{"shim", BYT_DSP_BAR, SHIM_OFFSET, SHIM_SIZE,
+	 SOF_DEBUGFS_ACCESS_ALWAYS},
+};
+
+static const struct snd_sof_debugfs_map cht_debugfs[] = {
+	{"dmac0", BYT_DSP_BAR, DMAC0_OFFSET, DMAC_SIZE,
+	 SOF_DEBUGFS_ACCESS_ALWAYS},
+	{"dmac1", BYT_DSP_BAR,  DMAC1_OFFSET, DMAC_SIZE,
+	 SOF_DEBUGFS_ACCESS_ALWAYS},
+	{"dmac2", BYT_DSP_BAR,  DMAC2_OFFSET, DMAC_SIZE,
+	 SOF_DEBUGFS_ACCESS_ALWAYS},
+	{"ssp0",  BYT_DSP_BAR, SSP0_OFFSET, SSP_SIZE,
+	 SOF_DEBUGFS_ACCESS_ALWAYS},
+	{"ssp1", BYT_DSP_BAR, SSP1_OFFSET, SSP_SIZE,
+	 SOF_DEBUGFS_ACCESS_ALWAYS},
+	{"ssp2", BYT_DSP_BAR, SSP2_OFFSET, SSP_SIZE,
+	 SOF_DEBUGFS_ACCESS_ALWAYS},
+	{"ssp3", BYT_DSP_BAR, SSP3_OFFSET, SSP_SIZE,
+	 SOF_DEBUGFS_ACCESS_ALWAYS},
+	{"ssp4", BYT_DSP_BAR, SSP4_OFFSET, SSP_SIZE,
+	 SOF_DEBUGFS_ACCESS_ALWAYS},
+	{"ssp5", BYT_DSP_BAR, SSP5_OFFSET, SSP_SIZE,
+	 SOF_DEBUGFS_ACCESS_ALWAYS},
+	{"iram", BYT_DSP_BAR, IRAM_OFFSET, IRAM_SIZE,
+	 SOF_DEBUGFS_ACCESS_D0_ONLY},
+	{"dram", BYT_DSP_BAR, DRAM_OFFSET, DRAM_SIZE,
+	 SOF_DEBUGFS_ACCESS_D0_ONLY},
+	{"shim", BYT_DSP_BAR, SHIM_OFFSET, SHIM_SIZE,
+	 SOF_DEBUGFS_ACCESS_ALWAYS},
+};
+
+static void byt_host_done(struct snd_sof_dev *sdev);
+static void byt_dsp_done(struct snd_sof_dev *sdev);
+static void byt_get_reply(struct snd_sof_dev *sdev);
+
+/*
+ * IPC Firmware ready.
+ */
+static void byt_get_windows(struct snd_sof_dev *sdev)
+{
+	struct sof_ipc_window_elem *elem;
+	u32 outbox_offset = 0;
+	u32 stream_offset = 0;
+	u32 inbox_offset = 0;
+	u32 outbox_size = 0;
+	u32 stream_size = 0;
+	u32 inbox_size = 0;
+	int i;
+
+	if (!sdev->info_window) {
+		dev_err(sdev->dev, "error: have no window info\n");
+		return;
+	}
+
+	for (i = 0; i < sdev->info_window->num_windows; i++) {
+		elem = &sdev->info_window->window[i];
+
+		switch (elem->type) {
+		case SOF_IPC_REGION_UPBOX:
+			inbox_offset = elem->offset + MBOX_OFFSET;
+			inbox_size = elem->size;
+			snd_sof_debugfs_io_item(sdev,
+						sdev->bar[BYT_DSP_BAR] +
+						inbox_offset,
+						elem->size, "inbox",
+						SOF_DEBUGFS_ACCESS_D0_ONLY);
+			break;
+		case SOF_IPC_REGION_DOWNBOX:
+			outbox_offset = elem->offset + MBOX_OFFSET;
+			outbox_size = elem->size;
+			snd_sof_debugfs_io_item(sdev,
+						sdev->bar[BYT_DSP_BAR] +
+						outbox_offset,
+						elem->size, "outbox",
+						SOF_DEBUGFS_ACCESS_D0_ONLY);
+			break;
+		case SOF_IPC_REGION_TRACE:
+			snd_sof_debugfs_io_item(sdev,
+						sdev->bar[BYT_DSP_BAR] +
+						elem->offset +
+						MBOX_OFFSET,
+						elem->size, "etrace",
+						SOF_DEBUGFS_ACCESS_D0_ONLY);
+			break;
+		case SOF_IPC_REGION_DEBUG:
+			snd_sof_debugfs_io_item(sdev,
+						sdev->bar[BYT_DSP_BAR] +
+						elem->offset +
+						MBOX_OFFSET,
+						elem->size, "debug",
+						SOF_DEBUGFS_ACCESS_D0_ONLY);
+			break;
+		case SOF_IPC_REGION_STREAM:
+			stream_offset = elem->offset + MBOX_OFFSET;
+			stream_size = elem->size;
+			snd_sof_debugfs_io_item(sdev,
+						sdev->bar[BYT_DSP_BAR] +
+						stream_offset,
+						elem->size, "stream",
+						SOF_DEBUGFS_ACCESS_D0_ONLY);
+			break;
+		case SOF_IPC_REGION_REGS:
+			snd_sof_debugfs_io_item(sdev,
+						sdev->bar[BYT_DSP_BAR] +
+						elem->offset +
+						MBOX_OFFSET,
+						elem->size, "regs",
+						SOF_DEBUGFS_ACCESS_D0_ONLY);
+			break;
+		case SOF_IPC_REGION_EXCEPTION:
+			sdev->dsp_oops_offset = elem->offset + MBOX_OFFSET;
+			snd_sof_debugfs_io_item(sdev,
+						sdev->bar[BYT_DSP_BAR] +
+						elem->offset +
+						MBOX_OFFSET,
+						elem->size, "exception",
+						SOF_DEBUGFS_ACCESS_D0_ONLY);
+			break;
+		default:
+			dev_err(sdev->dev, "error: get illegal window info\n");
+			return;
+		}
+	}
+
+	if (outbox_size == 0 || inbox_size == 0) {
+		dev_err(sdev->dev, "error: get illegal mailbox window\n");
+		return;
+	}
+
+	snd_sof_dsp_mailbox_init(sdev, inbox_offset, inbox_size,
+				 outbox_offset, outbox_size);
+	sdev->stream_box.offset = stream_offset;
+	sdev->stream_box.size = stream_size;
+
+	dev_dbg(sdev->dev, " mailbox upstream 0x%x - size 0x%x\n",
+		inbox_offset, inbox_size);
+	dev_dbg(sdev->dev, " mailbox downstream 0x%x - size 0x%x\n",
+		outbox_offset, outbox_size);
+	dev_dbg(sdev->dev, " stream region 0x%x - size 0x%x\n",
+		stream_offset, stream_size);
+}
+
+/* check for ABI compatibility and create memory windows on first boot */
+static int byt_fw_ready(struct snd_sof_dev *sdev, u32 msg_id)
+{
+	struct sof_ipc_fw_ready *fw_ready = &sdev->fw_ready;
+	u32 offset;
+	int ret;
+
+	/* mailbox must be on 4k boundary */
+	offset = MBOX_OFFSET;
+
+	dev_dbg(sdev->dev, "ipc: DSP is ready 0x%8.8x offset 0x%x\n",
+		msg_id, offset);
+
+	/* no need to re-check version/ABI for subsequent boots */
+	if (!sdev->first_boot)
+		return 0;
+
+	/* copy data from the DSP FW ready offset */
+	sof_block_read(sdev, sdev->mmio_bar, offset, fw_ready,
+		       sizeof(*fw_ready));
+
+	snd_sof_dsp_mailbox_init(sdev, fw_ready->dspbox_offset,
+				 fw_ready->dspbox_size,
+				 fw_ready->hostbox_offset,
+				 fw_ready->hostbox_size);
+
+	/* make sure ABI version is compatible */
+	ret = snd_sof_ipc_valid(sdev);
+	if (ret < 0)
+		return ret;
+
+	/* now check for extended data */
+	snd_sof_fw_parse_ext_data(sdev, sdev->mmio_bar, MBOX_OFFSET +
+				  sizeof(struct sof_ipc_fw_ready));
+
+	byt_get_windows(sdev);
+
+	return 0;
+}
+
+/*
+ * Debug
+ */
+
+static void byt_get_registers(struct snd_sof_dev *sdev,
+			      struct sof_ipc_dsp_oops_xtensa *xoops,
+			      struct sof_ipc_panic_info *panic_info,
+			      u32 *stack, size_t stack_words)
+{
+	/* first read regsisters */
+	sof_mailbox_read(sdev, sdev->dsp_oops_offset, xoops, sizeof(*xoops));
+
+	/* then get panic info */
+	sof_mailbox_read(sdev, sdev->dsp_oops_offset + sizeof(*xoops),
+			 panic_info, sizeof(*panic_info));
+
+	/* then get the stack */
+	sof_mailbox_read(sdev, sdev->dsp_oops_offset + sizeof(*xoops) +
+			   sizeof(*panic_info), stack,
+			   stack_words * sizeof(u32));
+}
+
+static void byt_dump(struct snd_sof_dev *sdev, u32 flags)
+{
+	struct sof_ipc_dsp_oops_xtensa xoops;
+	struct sof_ipc_panic_info panic_info;
+	u32 stack[BYT_STACK_DUMP_SIZE];
+	u32 status, panic;
+
+	/* now try generic SOF status messages */
+	status = snd_sof_dsp_read(sdev, BYT_DSP_BAR, SHIM_IPCD);
+	panic = snd_sof_dsp_read(sdev, BYT_DSP_BAR, SHIM_IPCX);
+	byt_get_registers(sdev, &xoops, &panic_info, stack,
+			  BYT_STACK_DUMP_SIZE);
+	snd_sof_get_status(sdev, status, panic, &xoops, &panic_info, stack,
+			   BYT_STACK_DUMP_SIZE);
+}
+
+/*
+ * IPC Doorbell IRQ handler and thread.
+ */
+
+static irqreturn_t byt_irq_handler(int irq, void *context)
+{
+	struct snd_sof_dev *sdev = context;
+	u64 isr;
+	int ret = IRQ_NONE;
+
+	/* Interrupt arrived, check src */
+	isr = snd_sof_dsp_read64(sdev, BYT_DSP_BAR, SHIM_ISRX);
+	if (isr & (SHIM_ISRX_DONE | SHIM_ISRX_BUSY))
+		ret = IRQ_WAKE_THREAD;
+
+	return ret;
+}
+
+static irqreturn_t byt_irq_thread(int irq, void *context)
+{
+	struct snd_sof_dev *sdev = context;
+	u64 ipcx, ipcd;
+	u64 imrx;
+
+	imrx = snd_sof_dsp_read64(sdev, BYT_DSP_BAR, SHIM_IMRX);
+	ipcx = snd_sof_dsp_read64(sdev, BYT_DSP_BAR, SHIM_IPCX);
+
+	/* reply message from DSP */
+	if (ipcx & SHIM_BYT_IPCX_DONE &&
+	    !(imrx & SHIM_IMRX_DONE)) {
+		/* Mask Done interrupt before first */
+		snd_sof_dsp_update_bits64_unlocked(sdev, BYT_DSP_BAR,
+						   SHIM_IMRX,
+						   SHIM_IMRX_DONE,
+						   SHIM_IMRX_DONE);
+		/*
+		 * handle immediate reply from DSP core. If the msg is
+		 * found, set done bit in cmd_done which is called at the
+		 * end of message processing function, else set it here
+		 * because the done bit can't be set in cmd_done function
+		 * which is triggered by msg
+		 */
+		byt_get_reply(sdev);
+		snd_sof_ipc_reply(sdev, ipcx);
+
+		byt_dsp_done(sdev);
+	}
+
+	/* new message from DSP */
+	ipcd = snd_sof_dsp_read64(sdev, BYT_DSP_BAR, SHIM_IPCD);
+	if (ipcd & SHIM_BYT_IPCD_BUSY &&
+	    !(imrx & SHIM_IMRX_BUSY)) {
+		/* Mask Busy interrupt before return */
+		snd_sof_dsp_update_bits64_unlocked(sdev, BYT_DSP_BAR,
+						   SHIM_IMRX,
+						   SHIM_IMRX_BUSY,
+						   SHIM_IMRX_BUSY);
+
+		/* Handle messages from DSP Core */
+		if ((ipcd & SOF_IPC_PANIC_MAGIC_MASK) == SOF_IPC_PANIC_MAGIC) {
+			snd_sof_dsp_panic(sdev, BYT_PANIC_OFFSET(ipcd) +
+					  MBOX_OFFSET);
+		} else {
+			snd_sof_ipc_msgs_rx(sdev);
+		}
+
+		byt_host_done(sdev);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static int byt_send_msg(struct snd_sof_dev *sdev, struct snd_sof_ipc_msg *msg)
+{
+	u64 cmd = msg->header;
+
+	/* send the message */
+	sof_mailbox_write(sdev, sdev->host_box.offset, msg->msg_data,
+			  msg->msg_size);
+	snd_sof_dsp_write64(sdev, BYT_DSP_BAR, SHIM_IPCX,
+			    cmd | SHIM_BYT_IPCX_BUSY);
+
+	return 0;
+}
+
+static void byt_get_reply(struct snd_sof_dev *sdev)
+{
+	struct snd_sof_ipc_msg *msg = sdev->msg;
+	struct sof_ipc_reply reply;
+	unsigned long flags;
+	int ret = 0;
+
+	/*
+	 * Sometimes, there is unexpected reply ipc arriving. The reply
+	 * ipc belongs to none of the ipcs sent from driver.
+	 * In this case, the driver must ignore the ipc.
+	 */
+	if (!msg) {
+		dev_warn(sdev->dev, "unexpected ipc interrupt raised!\n");
+		return;
+	}
+
+	/* get reply */
+	sof_mailbox_read(sdev, sdev->host_box.offset, &reply, sizeof(reply));
+
+	spin_lock_irqsave(&sdev->ipc_lock, flags);
+
+	if (reply.error < 0) {
+		memcpy(msg->reply_data, &reply, sizeof(reply));
+		ret = reply.error;
+	} else {
+		/* reply correct size ? */
+		if (reply.hdr.size != msg->reply_size) {
+			dev_err(sdev->dev, "error: reply expected %zu got %u bytes\n",
+				msg->reply_size, reply.hdr.size);
+			ret = -EINVAL;
+		}
+
+		/* read the message */
+		if (msg->reply_size > 0)
+			sof_mailbox_read(sdev, sdev->host_box.offset,
+					 msg->reply_data, msg->reply_size);
+	}
+
+	msg->reply_error = ret;
+
+	spin_unlock_irqrestore(&sdev->ipc_lock, flags);
+}
+
+static void byt_host_done(struct snd_sof_dev *sdev)
+{
+	/* clear BUSY bit and set DONE bit - accept new messages */
+	snd_sof_dsp_update_bits64_unlocked(sdev, BYT_DSP_BAR, SHIM_IPCD,
+					   SHIM_BYT_IPCD_BUSY |
+					   SHIM_BYT_IPCD_DONE,
+					   SHIM_BYT_IPCD_DONE);
+
+	/* unmask busy interrupt */
+	snd_sof_dsp_update_bits64_unlocked(sdev, BYT_DSP_BAR, SHIM_IMRX,
+					   SHIM_IMRX_BUSY, 0);
+}
+
+static void byt_dsp_done(struct snd_sof_dev *sdev)
+{
+	/* clear DONE bit - tell DSP we have completed */
+	snd_sof_dsp_update_bits64_unlocked(sdev, BYT_DSP_BAR, SHIM_IPCX,
+					   SHIM_BYT_IPCX_DONE, 0);
+
+	/* unmask Done interrupt */
+	snd_sof_dsp_update_bits64_unlocked(sdev, BYT_DSP_BAR, SHIM_IMRX,
+					   SHIM_IMRX_DONE, 0);
+}
+
+/*
+ * DSP control.
+ */
+
+static int byt_run(struct snd_sof_dev *sdev)
+{
+	int tries = 10;
+
+	/* release stall and wait to unstall */
+	snd_sof_dsp_update_bits64(sdev, BYT_DSP_BAR, SHIM_CSR,
+				  SHIM_BYT_CSR_STALL, 0x0);
+	while (tries--) {
+		if (!(snd_sof_dsp_read64(sdev, BYT_DSP_BAR, SHIM_CSR) &
+		      SHIM_BYT_CSR_PWAITMODE))
+			break;
+		msleep(100);
+	}
+	if (tries < 0) {
+		dev_err(sdev->dev, "error:  unable to run DSP firmware\n");
+		byt_dump(sdev, SOF_DBG_REGS | SOF_DBG_MBOX);
+		return -ENODEV;
+	}
+
+	/* return init core mask */
+	return 1;
+}
+
+static int byt_reset(struct snd_sof_dev *sdev)
+{
+	/* put DSP into reset, set reset vector and stall */
+	snd_sof_dsp_update_bits64(sdev, BYT_DSP_BAR, SHIM_CSR,
+				  SHIM_BYT_CSR_RST | SHIM_BYT_CSR_VECTOR_SEL |
+				  SHIM_BYT_CSR_STALL,
+				  SHIM_BYT_CSR_RST | SHIM_BYT_CSR_VECTOR_SEL |
+				  SHIM_BYT_CSR_STALL);
+
+	usleep_range(10, 15);
+
+	/* take DSP out of reset and keep stalled for FW loading */
+	snd_sof_dsp_update_bits64(sdev, BYT_DSP_BAR, SHIM_CSR,
+				  SHIM_BYT_CSR_RST, 0);
+
+	return 0;
+}
+
+/* Baytrail DAIs */
+static struct snd_soc_dai_driver byt_dai[] = {
+{
+	.name = "ssp0-port",
+},
+{
+	.name = "ssp1-port",
+},
+{
+	.name = "ssp2-port",
+},
+{
+	.name = "ssp3-port",
+},
+{
+	.name = "ssp4-port",
+},
+{
+	.name = "ssp5-port",
+},
+};
+
+/*
+ * Probe and remove.
+ */
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_MERRIFIELD)
+
+static int tangier_pci_probe(struct snd_sof_dev *sdev)
+{
+	struct snd_sof_pdata *pdata = sdev->pdata;
+	const struct sof_dev_desc *desc = pdata->desc;
+	struct pci_dev *pci = to_pci_dev(sdev->dev);
+	u32 base, size;
+	int ret;
+
+	/* DSP DMA can only access low 31 bits of host memory */
+	ret = dma_coerce_mask_and_coherent(&pci->dev, DMA_BIT_MASK(31));
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: failed to set DMA mask %d\n", ret);
+		return ret;
+	}
+
+	/* LPE base */
+	base = pci_resource_start(pci, desc->resindex_lpe_base) - IRAM_OFFSET;
+	size = BYT_PCI_BAR_SIZE;
+
+	dev_dbg(sdev->dev, "LPE PHY base at 0x%x size 0x%x", base, size);
+	sdev->bar[BYT_DSP_BAR] = devm_ioremap(sdev->dev, base, size);
+	if (!sdev->bar[BYT_DSP_BAR]) {
+		dev_err(sdev->dev, "error: failed to ioremap LPE base 0x%x size 0x%x\n",
+			base, size);
+		return -ENODEV;
+	}
+	dev_dbg(sdev->dev, "LPE VADDR %p\n", sdev->bar[BYT_DSP_BAR]);
+
+	/* IMR base - optional */
+	if (desc->resindex_imr_base == -1)
+		goto irq;
+
+	base = pci_resource_start(pci, desc->resindex_imr_base);
+	size = pci_resource_len(pci, desc->resindex_imr_base);
+
+	/* some BIOSes don't map IMR */
+	if (base == 0x55aa55aa || base == 0x0) {
+		dev_info(sdev->dev, "IMR not set by BIOS. Ignoring\n");
+		goto irq;
+	}
+
+	dev_dbg(sdev->dev, "IMR base at 0x%x size 0x%x", base, size);
+	sdev->bar[BYT_IMR_BAR] = devm_ioremap(sdev->dev, base, size);
+	if (!sdev->bar[BYT_IMR_BAR]) {
+		dev_err(sdev->dev, "error: failed to ioremap IMR base 0x%x size 0x%x\n",
+			base, size);
+		return -ENODEV;
+	}
+	dev_dbg(sdev->dev, "IMR VADDR %p\n", sdev->bar[BYT_IMR_BAR]);
+
+irq:
+	/* register our IRQ */
+	sdev->ipc_irq = pci->irq;
+	dev_dbg(sdev->dev, "using IRQ %d\n", sdev->ipc_irq);
+	ret = devm_request_threaded_irq(sdev->dev, sdev->ipc_irq,
+					byt_irq_handler, byt_irq_thread,
+					0, "AudioDSP", sdev);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: failed to register IRQ %d\n",
+			sdev->ipc_irq);
+		return ret;
+	}
+
+	/* enable Interrupt from both sides */
+	snd_sof_dsp_update_bits64(sdev, BYT_DSP_BAR, SHIM_IMRX, 0x3, 0x0);
+	snd_sof_dsp_update_bits64(sdev, BYT_DSP_BAR, SHIM_IMRD, 0x3, 0x0);
+
+	/* set default mailbox offset for FW ready message */
+	sdev->dsp_box.offset = MBOX_OFFSET;
+
+	return ret;
+}
+
+const struct snd_sof_dsp_ops sof_tng_ops = {
+	/* device init */
+	.probe		= tangier_pci_probe,
+
+	/* DSP core boot / reset */
+	.run		= byt_run,
+	.reset		= byt_reset,
+
+	/* Register IO */
+	.write		= sof_io_write,
+	.read		= sof_io_read,
+	.write64	= sof_io_write64,
+	.read64		= sof_io_read64,
+
+	/* Block IO */
+	.block_read	= sof_block_read,
+	.block_write	= sof_block_write,
+
+	/* doorbell */
+	.irq_handler	= byt_irq_handler,
+	.irq_thread	= byt_irq_thread,
+
+	/* ipc */
+	.send_msg	= byt_send_msg,
+	.fw_ready	= byt_fw_ready,
+
+	.ipc_msg_data	= intel_ipc_msg_data,
+	.ipc_pcm_params	= intel_ipc_pcm_params,
+
+	/* debug */
+	.debug_map	= byt_debugfs,
+	.debug_map_count	= ARRAY_SIZE(byt_debugfs),
+	.dbg_dump	= byt_dump,
+
+	/* stream callbacks */
+	.pcm_open	= intel_pcm_open,
+	.pcm_close	= intel_pcm_close,
+
+	/* module loading */
+	.load_module	= snd_sof_parse_module_memcpy,
+
+	/*Firmware loading */
+	.load_firmware	= snd_sof_load_firmware_memcpy,
+
+	/* DAI drivers */
+	.drv = byt_dai,
+	.num_drv = 3, /* we have only 3 SSPs on byt*/
+};
+EXPORT_SYMBOL(sof_tng_ops);
+
+const struct sof_intel_dsp_desc tng_chip_info = {
+	.cores_num = 1,
+	.cores_mask = 1,
+};
+EXPORT_SYMBOL(tng_chip_info);
+
+#endif /* CONFIG_SND_SOC_SOF_MERRIFIELD */
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_BAYTRAIL)
+
+static int byt_acpi_probe(struct snd_sof_dev *sdev)
+{
+	struct snd_sof_pdata *pdata = sdev->pdata;
+	const struct sof_dev_desc *desc = pdata->desc;
+	struct platform_device *pdev =
+		container_of(sdev->dev, struct platform_device, dev);
+	struct resource *mmio;
+	u32 base, size;
+	int ret;
+
+	/* DSP DMA can only access low 31 bits of host memory */
+	ret = dma_coerce_mask_and_coherent(sdev->dev, DMA_BIT_MASK(31));
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: failed to set DMA mask %d\n", ret);
+		return ret;
+	}
+
+	/* LPE base */
+	mmio = platform_get_resource(pdev, IORESOURCE_MEM,
+				     desc->resindex_lpe_base);
+	if (mmio) {
+		base = mmio->start;
+		size = resource_size(mmio);
+	} else {
+		dev_err(sdev->dev, "error: failed to get LPE base at idx %d\n",
+			desc->resindex_lpe_base);
+		return -EINVAL;
+	}
+
+	dev_dbg(sdev->dev, "LPE PHY base at 0x%x size 0x%x", base, size);
+	sdev->bar[BYT_DSP_BAR] = devm_ioremap(sdev->dev, base, size);
+	if (!sdev->bar[BYT_DSP_BAR]) {
+		dev_err(sdev->dev, "error: failed to ioremap LPE base 0x%x size 0x%x\n",
+			base, size);
+		return -ENODEV;
+	}
+	dev_dbg(sdev->dev, "LPE VADDR %p\n", sdev->bar[BYT_DSP_BAR]);
+
+	/* TODO: add offsets */
+	sdev->mmio_bar = BYT_DSP_BAR;
+	sdev->mailbox_bar = BYT_DSP_BAR;
+
+	/* IMR base - optional */
+	if (desc->resindex_imr_base == -1)
+		goto irq;
+
+	mmio = platform_get_resource(pdev, IORESOURCE_MEM,
+				     desc->resindex_imr_base);
+	if (mmio) {
+		base = mmio->start;
+		size = resource_size(mmio);
+	} else {
+		dev_err(sdev->dev, "error: failed to get IMR base at idx %d\n",
+			desc->resindex_imr_base);
+		return -ENODEV;
+	}
+
+	/* some BIOSes don't map IMR */
+	if (base == 0x55aa55aa || base == 0x0) {
+		dev_info(sdev->dev, "IMR not set by BIOS. Ignoring\n");
+		goto irq;
+	}
+
+	dev_dbg(sdev->dev, "IMR base at 0x%x size 0x%x", base, size);
+	sdev->bar[BYT_IMR_BAR] = devm_ioremap(sdev->dev, base, size);
+	if (!sdev->bar[BYT_IMR_BAR]) {
+		dev_err(sdev->dev, "error: failed to ioremap IMR base 0x%x size 0x%x\n",
+			base, size);
+		return -ENODEV;
+	}
+	dev_dbg(sdev->dev, "IMR VADDR %p\n", sdev->bar[BYT_IMR_BAR]);
+
+irq:
+	/* register our IRQ */
+	sdev->ipc_irq = platform_get_irq(pdev, desc->irqindex_host_ipc);
+	if (sdev->ipc_irq < 0) {
+		dev_err(sdev->dev, "error: failed to get IRQ at index %d\n",
+			desc->irqindex_host_ipc);
+		return sdev->ipc_irq;
+	}
+
+	dev_dbg(sdev->dev, "using IRQ %d\n", sdev->ipc_irq);
+	ret = devm_request_threaded_irq(sdev->dev, sdev->ipc_irq,
+					byt_irq_handler, byt_irq_thread,
+					IRQF_SHARED, "AudioDSP", sdev);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: failed to register IRQ %d\n",
+			sdev->ipc_irq);
+		return ret;
+	}
+
+	/* enable Interrupt from both sides */
+	snd_sof_dsp_update_bits64(sdev, BYT_DSP_BAR, SHIM_IMRX, 0x3, 0x0);
+	snd_sof_dsp_update_bits64(sdev, BYT_DSP_BAR, SHIM_IMRD, 0x3, 0x0);
+
+	/* set default mailbox offset for FW ready message */
+	sdev->dsp_box.offset = MBOX_OFFSET;
+
+	return ret;
+}
+
+/* baytrail ops */
+const struct snd_sof_dsp_ops sof_byt_ops = {
+	/* device init */
+	.probe		= byt_acpi_probe,
+
+	/* DSP core boot / reset */
+	.run		= byt_run,
+	.reset		= byt_reset,
+
+	/* Register IO */
+	.write		= sof_io_write,
+	.read		= sof_io_read,
+	.write64	= sof_io_write64,
+	.read64		= sof_io_read64,
+
+	/* Block IO */
+	.block_read	= sof_block_read,
+	.block_write	= sof_block_write,
+
+	/* doorbell */
+	.irq_handler	= byt_irq_handler,
+	.irq_thread	= byt_irq_thread,
+
+	/* ipc */
+	.send_msg	= byt_send_msg,
+	.fw_ready	= byt_fw_ready,
+
+	.ipc_msg_data	= intel_ipc_msg_data,
+	.ipc_pcm_params	= intel_ipc_pcm_params,
+
+	/* debug */
+	.debug_map	= byt_debugfs,
+	.debug_map_count	= ARRAY_SIZE(byt_debugfs),
+	.dbg_dump	= byt_dump,
+
+	/* stream callbacks */
+	.pcm_open	= intel_pcm_open,
+	.pcm_close	= intel_pcm_close,
+
+	/* module loading */
+	.load_module	= snd_sof_parse_module_memcpy,
+
+	/*Firmware loading */
+	.load_firmware	= snd_sof_load_firmware_memcpy,
+
+	/* DAI drivers */
+	.drv = byt_dai,
+	.num_drv = 3, /* we have only 3 SSPs on byt*/
+};
+EXPORT_SYMBOL(sof_byt_ops);
+
+const struct sof_intel_dsp_desc byt_chip_info = {
+	.cores_num = 1,
+	.cores_mask = 1,
+};
+EXPORT_SYMBOL(byt_chip_info);
+
+/* cherrytrail and braswell ops */
+const struct snd_sof_dsp_ops sof_cht_ops = {
+	/* device init */
+	.probe		= byt_acpi_probe,
+
+	/* DSP core boot / reset */
+	.run		= byt_run,
+	.reset		= byt_reset,
+
+	/* Register IO */
+	.write		= sof_io_write,
+	.read		= sof_io_read,
+	.write64	= sof_io_write64,
+	.read64		= sof_io_read64,
+
+	/* Block IO */
+	.block_read	= sof_block_read,
+	.block_write	= sof_block_write,
+
+	/* doorbell */
+	.irq_handler	= byt_irq_handler,
+	.irq_thread	= byt_irq_thread,
+
+	/* ipc */
+	.send_msg	= byt_send_msg,
+	.fw_ready	= byt_fw_ready,
+
+	.ipc_msg_data	= intel_ipc_msg_data,
+	.ipc_pcm_params	= intel_ipc_pcm_params,
+
+	/* debug */
+	.debug_map	= cht_debugfs,
+	.debug_map_count	= ARRAY_SIZE(cht_debugfs),
+	.dbg_dump	= byt_dump,
+
+	/* stream callbacks */
+	.pcm_open	= intel_pcm_open,
+	.pcm_close	= intel_pcm_close,
+
+	/* module loading */
+	.load_module	= snd_sof_parse_module_memcpy,
+
+	/*Firmware loading */
+	.load_firmware	= snd_sof_load_firmware_memcpy,
+
+	/* DAI drivers */
+	.drv = byt_dai,
+	/* all 6 SSPs may be available for cherrytrail */
+	.num_drv = ARRAY_SIZE(byt_dai),
+};
+EXPORT_SYMBOL(sof_cht_ops);
+
+const struct sof_intel_dsp_desc cht_chip_info = {
+	.cores_num = 1,
+	.cores_mask = 1,
+};
+EXPORT_SYMBOL(cht_chip_info);
+
+#endif /* CONFIG_SND_SOC_SOF_BAYTRAIL */
+
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/sound/soc/sof/intel/cnl.c b/sound/soc/sof/intel/cnl.c
new file mode 100644
index 000000000000..08a1a3d3c08d
--- /dev/null
+++ b/sound/soc/sof/intel/cnl.c
@@ -0,0 +1,268 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license.  When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2018 Intel Corporation. All rights reserved.
+//
+// Authors: Liam Girdwood <liam.r.girdwood@linux.intel.com>
+//	    Ranjani Sridharan <ranjani.sridharan@linux.intel.com>
+//	    Rander Wang <rander.wang@intel.com>
+//          Keyon Jie <yang.jie@linux.intel.com>
+//
+
+/*
+ * Hardware interface for audio DSP on Cannonlake.
+ */
+
+#include "../ops.h"
+#include "hda.h"
+
+static const struct snd_sof_debugfs_map cnl_dsp_debugfs[] = {
+	{"hda", HDA_DSP_HDA_BAR, 0, 0x4000, SOF_DEBUGFS_ACCESS_ALWAYS},
+	{"pp", HDA_DSP_PP_BAR,  0, 0x1000, SOF_DEBUGFS_ACCESS_ALWAYS},
+	{"dsp", HDA_DSP_BAR,  0, 0x10000, SOF_DEBUGFS_ACCESS_ALWAYS},
+};
+
+static void cnl_ipc_host_done(struct snd_sof_dev *sdev);
+static void cnl_ipc_dsp_done(struct snd_sof_dev *sdev);
+
+static irqreturn_t cnl_ipc_irq_thread(int irq, void *context)
+{
+	struct snd_sof_dev *sdev = context;
+	u32 hipci;
+	u32 hipcctl;
+	u32 hipcida;
+	u32 hipctdr;
+	u32 hipctdd;
+	u32 msg;
+	u32 msg_ext;
+	irqreturn_t ret = IRQ_NONE;
+
+	hipcida = snd_sof_dsp_read(sdev, HDA_DSP_BAR, CNL_DSP_REG_HIPCIDA);
+	hipcctl = snd_sof_dsp_read(sdev, HDA_DSP_BAR, CNL_DSP_REG_HIPCCTL);
+	hipctdr = snd_sof_dsp_read(sdev, HDA_DSP_BAR, CNL_DSP_REG_HIPCTDR);
+
+	/* reenable IPC interrupt */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, HDA_DSP_REG_ADSPIC,
+				HDA_DSP_ADSPIC_IPC, HDA_DSP_ADSPIC_IPC);
+
+	/* reply message from DSP */
+	if (hipcida & CNL_DSP_REG_HIPCIDA_DONE &&
+	    hipcctl & CNL_DSP_REG_HIPCCTL_DONE) {
+		hipci = snd_sof_dsp_read(sdev, HDA_DSP_BAR,
+					 CNL_DSP_REG_HIPCIDR);
+		msg_ext = hipci & CNL_DSP_REG_HIPCIDR_MSG_MASK;
+		msg = hipcida & CNL_DSP_REG_HIPCIDA_MSG_MASK;
+
+		dev_vdbg(sdev->dev,
+			 "ipc: firmware response, msg:0x%x, msg_ext:0x%x\n",
+			 msg, msg_ext);
+
+		/* mask Done interrupt */
+		snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR,
+					CNL_DSP_REG_HIPCCTL,
+					CNL_DSP_REG_HIPCCTL_DONE, 0);
+
+		/* handle immediate reply from DSP core */
+		hda_dsp_ipc_get_reply(sdev);
+		snd_sof_ipc_reply(sdev, msg);
+
+		if (sdev->code_loading)	{
+			sdev->code_loading = 0;
+			wake_up(&sdev->waitq);
+		}
+
+		cnl_ipc_dsp_done(sdev);
+
+		ret = IRQ_HANDLED;
+	}
+
+	/* new message from DSP */
+	if (hipctdr & CNL_DSP_REG_HIPCTDR_BUSY) {
+		hipctdd = snd_sof_dsp_read(sdev, HDA_DSP_BAR,
+					   CNL_DSP_REG_HIPCTDD);
+		msg = hipctdr & CNL_DSP_REG_HIPCTDR_MSG_MASK;
+		msg_ext = hipctdd & CNL_DSP_REG_HIPCTDD_MSG_MASK;
+
+		dev_vdbg(sdev->dev,
+			 "ipc: firmware initiated, msg:0x%x, msg_ext:0x%x\n",
+			 msg, msg_ext);
+
+		/* handle messages from DSP */
+		if ((hipctdr & SOF_IPC_PANIC_MAGIC_MASK) ==
+		   SOF_IPC_PANIC_MAGIC) {
+			snd_sof_dsp_panic(sdev, HDA_DSP_PANIC_OFFSET(msg_ext));
+		} else {
+			snd_sof_ipc_msgs_rx(sdev);
+		}
+
+		/*
+		 * clear busy interrupt to tell dsp controller this
+		 * interrupt has been accepted, not trigger it again
+		 */
+		snd_sof_dsp_update_bits_forced(sdev, HDA_DSP_BAR,
+					       CNL_DSP_REG_HIPCTDR,
+					       CNL_DSP_REG_HIPCTDR_BUSY,
+					       CNL_DSP_REG_HIPCTDR_BUSY);
+
+		cnl_ipc_host_done(sdev);
+
+		ret = IRQ_HANDLED;
+	}
+
+	return ret;
+}
+
+static void cnl_ipc_host_done(struct snd_sof_dev *sdev)
+{
+	/*
+	 * set done bit to ack dsp the msg has been
+	 * processed and send reply msg to dsp
+	 */
+	snd_sof_dsp_update_bits_forced(sdev, HDA_DSP_BAR,
+				       CNL_DSP_REG_HIPCTDA,
+				       CNL_DSP_REG_HIPCTDA_DONE,
+				       CNL_DSP_REG_HIPCTDA_DONE);
+}
+
+static void cnl_ipc_dsp_done(struct snd_sof_dev *sdev)
+{
+	/*
+	 * set DONE bit - tell DSP we have received the reply msg
+	 * from DSP, and processed it, don't send more reply to host
+	 */
+	snd_sof_dsp_update_bits_forced(sdev, HDA_DSP_BAR,
+				       CNL_DSP_REG_HIPCIDA,
+				       CNL_DSP_REG_HIPCIDA_DONE,
+				       CNL_DSP_REG_HIPCIDA_DONE);
+
+	/* unmask Done interrupt */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR,
+				CNL_DSP_REG_HIPCCTL,
+				CNL_DSP_REG_HIPCCTL_DONE,
+				CNL_DSP_REG_HIPCCTL_DONE);
+}
+
+static int cnl_ipc_send_msg(struct snd_sof_dev *sdev,
+			    struct snd_sof_ipc_msg *msg)
+{
+	u32 cmd = msg->header;
+
+	/* send the message */
+	sof_mailbox_write(sdev, sdev->host_box.offset, msg->msg_data,
+			  msg->msg_size);
+	snd_sof_dsp_write(sdev, HDA_DSP_BAR, CNL_DSP_REG_HIPCIDR,
+			  cmd | CNL_DSP_REG_HIPCIDR_BUSY);
+
+	return 0;
+}
+
+static void cnl_ipc_dump(struct snd_sof_dev *sdev)
+{
+	u32 hipcctl;
+	u32 hipcida;
+	u32 hipctdr;
+
+	/* read IPC status */
+	hipcida = snd_sof_dsp_read(sdev, HDA_DSP_BAR, CNL_DSP_REG_HIPCIDA);
+	hipcctl = snd_sof_dsp_read(sdev, HDA_DSP_BAR, CNL_DSP_REG_HIPCCTL);
+	hipctdr = snd_sof_dsp_read(sdev, HDA_DSP_BAR, CNL_DSP_REG_HIPCTDR);
+
+	/* dump the IPC regs */
+	/* TODO: parse the raw msg */
+	dev_err(sdev->dev,
+		"error: host status 0x%8.8x dsp status 0x%8.8x mask 0x%8.8x\n",
+		hipcida, hipctdr, hipcctl);
+}
+
+/* cannonlake ops */
+const struct snd_sof_dsp_ops sof_cnl_ops = {
+	/* probe and remove */
+	.probe		= hda_dsp_probe,
+	.remove		= hda_dsp_remove,
+
+	/* Register IO */
+	.write		= sof_io_write,
+	.read		= sof_io_read,
+	.write64	= sof_io_write64,
+	.read64		= sof_io_read64,
+
+	/* Block IO */
+	.block_read	= sof_block_read,
+	.block_write	= sof_block_write,
+
+	/* doorbell */
+	.irq_handler	= hda_dsp_ipc_irq_handler,
+	.irq_thread	= cnl_ipc_irq_thread,
+
+	/* ipc */
+	.send_msg	= cnl_ipc_send_msg,
+	.fw_ready	= hda_dsp_ipc_fw_ready,
+
+	.ipc_msg_data	= hda_ipc_msg_data,
+	.ipc_pcm_params	= hda_ipc_pcm_params,
+
+	/* debug */
+	.debug_map	= cnl_dsp_debugfs,
+	.debug_map_count	= ARRAY_SIZE(cnl_dsp_debugfs),
+	.dbg_dump	= hda_dsp_dump,
+	.ipc_dump	= cnl_ipc_dump,
+
+	/* stream callbacks */
+	.pcm_open	= hda_dsp_pcm_open,
+	.pcm_close	= hda_dsp_pcm_close,
+	.pcm_hw_params	= hda_dsp_pcm_hw_params,
+	.pcm_trigger	= hda_dsp_pcm_trigger,
+	.pcm_pointer	= hda_dsp_pcm_pointer,
+
+	/* firmware loading */
+	.load_firmware = snd_sof_load_firmware_raw,
+
+	/* pre/post fw run */
+	.pre_fw_run = hda_dsp_pre_fw_run,
+	.post_fw_run = hda_dsp_post_fw_run,
+
+	/* dsp core power up/down */
+	.core_power_up = hda_dsp_enable_core,
+	.core_power_down = hda_dsp_core_reset_power_down,
+
+	/* firmware run */
+	.run = hda_dsp_cl_boot_firmware,
+
+	/* trace callback */
+	.trace_init = hda_dsp_trace_init,
+	.trace_release = hda_dsp_trace_release,
+	.trace_trigger = hda_dsp_trace_trigger,
+
+	/* DAI drivers */
+	.drv		= skl_dai,
+	.num_drv	= SOF_SKL_NUM_DAIS,
+
+	/* PM */
+	.suspend		= hda_dsp_suspend,
+	.resume			= hda_dsp_resume,
+	.runtime_suspend	= hda_dsp_runtime_suspend,
+	.runtime_resume		= hda_dsp_runtime_resume,
+	.set_hw_params_upon_resume = hda_dsp_set_hw_params_upon_resume,
+};
+EXPORT_SYMBOL(sof_cnl_ops);
+
+const struct sof_intel_dsp_desc cnl_chip_info = {
+	/* Cannonlake */
+	.cores_num = 4,
+	.init_core_mask = 1,
+	.cores_mask = HDA_DSP_CORE_MASK(0) |
+				HDA_DSP_CORE_MASK(1) |
+				HDA_DSP_CORE_MASK(2) |
+				HDA_DSP_CORE_MASK(3),
+	.ipc_req = CNL_DSP_REG_HIPCIDR,
+	.ipc_req_mask = CNL_DSP_REG_HIPCIDR_BUSY,
+	.ipc_ack = CNL_DSP_REG_HIPCIDA,
+	.ipc_ack_mask = CNL_DSP_REG_HIPCIDA_DONE,
+	.ipc_ctl = CNL_DSP_REG_HIPCCTL,
+	.rom_init_timeout	= 300,
+	.ssp_count = CNL_SSP_COUNT,
+	.ssp_base_offset = CNL_SSP_BASE_OFFSET,
+};
+EXPORT_SYMBOL(cnl_chip_info);
diff --git a/sound/soc/sof/intel/hda-bus.c b/sound/soc/sof/intel/hda-bus.c
new file mode 100644
index 000000000000..a7e6d8227df6
--- /dev/null
+++ b/sound/soc/sof/intel/hda-bus.c
@@ -0,0 +1,111 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license.  When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2018 Intel Corporation. All rights reserved.
+//
+// Authors: Keyon Jie <yang.jie@linux.intel.com>
+
+#include <linux/io.h>
+#include <sound/hdaudio.h>
+#include "../sof-priv.h"
+#include "hda.h"
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
+
+static const struct hdac_bus_ops bus_ops = {
+	.command = snd_hdac_bus_send_cmd,
+	.get_response = snd_hdac_bus_get_response,
+};
+
+#endif
+
+static void sof_hda_writel(u32 value, u32 __iomem *addr)
+{
+	writel(value, addr);
+}
+
+static u32 sof_hda_readl(u32 __iomem *addr)
+{
+	return readl(addr);
+}
+
+static void sof_hda_writew(u16 value, u16 __iomem *addr)
+{
+	writew(value, addr);
+}
+
+static u16 sof_hda_readw(u16 __iomem *addr)
+{
+	return readw(addr);
+}
+
+static void sof_hda_writeb(u8 value, u8 __iomem *addr)
+{
+	writeb(value, addr);
+}
+
+static u8 sof_hda_readb(u8 __iomem *addr)
+{
+	return readb(addr);
+}
+
+static int sof_hda_dma_alloc_pages(struct hdac_bus *bus, int type,
+				   size_t size, struct snd_dma_buffer *buf)
+{
+	return snd_dma_alloc_pages(type, bus->dev, size, buf);
+}
+
+static void sof_hda_dma_free_pages(struct hdac_bus *bus,
+				   struct snd_dma_buffer *buf)
+{
+	snd_dma_free_pages(buf);
+}
+
+static const struct hdac_io_ops io_ops = {
+	.reg_writel = sof_hda_writel,
+	.reg_readl = sof_hda_readl,
+	.reg_writew = sof_hda_writew,
+	.reg_readw = sof_hda_readw,
+	.reg_writeb = sof_hda_writeb,
+	.reg_readb = sof_hda_readb,
+	.dma_alloc_pages = sof_hda_dma_alloc_pages,
+	.dma_free_pages = sof_hda_dma_free_pages,
+};
+
+/*
+ * This can be used for both with/without hda link support.
+ */
+void sof_hda_bus_init(struct hdac_bus *bus, struct device *dev,
+		      const struct hdac_ext_bus_ops *ext_ops)
+{
+	memset(bus, 0, sizeof(*bus));
+	bus->dev = dev;
+
+	bus->io_ops = &io_ops;
+	INIT_LIST_HEAD(&bus->stream_list);
+
+	bus->irq = -1;
+	bus->ext_ops = ext_ops;
+
+	/*
+	 * There is only one HDA bus atm. keep the index as 0.
+	 * Need to fix when there are more than one HDA bus.
+	 */
+	bus->idx = 0;
+
+	spin_lock_init(&bus->reg_lock);
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
+	INIT_LIST_HEAD(&bus->codec_list);
+	INIT_LIST_HEAD(&bus->hlink_list);
+
+	mutex_init(&bus->cmd_mutex);
+	mutex_init(&bus->lock);
+	bus->ops = &bus_ops;
+	INIT_WORK(&bus->unsol_work, snd_hdac_bus_process_unsol_events);
+	bus->cmd_dma_state = true;
+#endif
+
+}
diff --git a/sound/soc/sof/intel/hda-codec.c b/sound/soc/sof/intel/hda-codec.c
new file mode 100644
index 000000000000..b8b37f082309
--- /dev/null
+++ b/sound/soc/sof/intel/hda-codec.c
@@ -0,0 +1,171 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license.  When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2018 Intel Corporation. All rights reserved.
+//
+// Authors: Keyon Jie <yang.jie@linux.intel.com>
+//
+
+#include <linux/module.h>
+#include <sound/hdaudio_ext.h>
+#include <sound/hda_codec.h>
+#include <sound/hda_i915.h>
+#include <sound/sof.h>
+#include "../ops.h"
+#include "hda.h"
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA_AUDIO_CODEC)
+#include "../../codecs/hdac_hda.h"
+#endif /* CONFIG_SND_SOC_SOF_HDA_AUDIO_CODEC */
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA_AUDIO_CODEC)
+#define IDISP_VID_INTEL	0x80860000
+
+/* load the legacy HDA codec driver */
+#ifdef MODULE
+static void hda_codec_load_module(struct hda_codec *codec)
+{
+	char alias[MODULE_NAME_LEN];
+	const char *module = alias;
+
+	snd_hdac_codec_modalias(&codec->core, alias, sizeof(alias));
+	dev_dbg(&codec->core.dev, "loading codec module: %s\n", module);
+	request_module(module);
+}
+#else
+static void hda_codec_load_module(struct hda_codec *codec) {}
+#endif
+
+#endif /* CONFIG_SND_SOC_SOF_HDA_AUDIO_CODEC */
+
+/* probe individual codec */
+static int hda_codec_probe(struct snd_sof_dev *sdev, int address)
+{
+	struct hda_bus *hbus = sof_to_hbus(sdev);
+	struct hdac_device *hdev;
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA_AUDIO_CODEC)
+	struct hdac_hda_priv *hda_priv;
+#endif
+	u32 hda_cmd = (address << 28) | (AC_NODE_ROOT << 20) |
+		(AC_VERB_PARAMETERS << 8) | AC_PAR_VENDOR_ID;
+	u32 resp = -1;
+	int ret;
+
+	mutex_lock(&hbus->core.cmd_mutex);
+	snd_hdac_bus_send_cmd(&hbus->core, hda_cmd);
+	snd_hdac_bus_get_response(&hbus->core, address, &resp);
+	mutex_unlock(&hbus->core.cmd_mutex);
+	if (resp == -1)
+		return -EIO;
+	dev_dbg(sdev->dev, "HDA codec #%d probed OK: response: %x\n",
+		address, resp);
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA_AUDIO_CODEC)
+	/* snd_hdac_ext_bus_device_exit will use kfree to free hdev */
+	hda_priv = kzalloc(sizeof(*hda_priv), GFP_KERNEL);
+	if (!hda_priv)
+		return -ENOMEM;
+
+	hda_priv->codec.bus = hbus;
+	hdev = &hda_priv->codec.core;
+
+	ret = snd_hdac_ext_bus_device_init(&hbus->core, address, hdev);
+	if (ret < 0)
+		return ret;
+
+	/* use legacy bus only for HDA codecs, idisp uses ext bus */
+	if ((resp & 0xFFFF0000) != IDISP_VID_INTEL) {
+		hdev->type = HDA_DEV_LEGACY;
+		hda_codec_load_module(&hda_priv->codec);
+	}
+
+	return 0;
+#else
+	/* snd_hdac_ext_bus_device_exit will use kfree to free hdev */
+	hdev = kzalloc(sizeof(*hdev), GFP_KERNEL);
+	if (!hdev)
+		return -ENOMEM;
+
+	ret = snd_hdac_ext_bus_device_init(&hbus->core, address, hdev);
+
+	return ret;
+#endif
+}
+
+/* Codec initialization */
+int hda_codec_probe_bus(struct snd_sof_dev *sdev)
+{
+	struct hdac_bus *bus = sof_to_bus(sdev);
+	int i, ret;
+
+	/* probe codecs in avail slots */
+	for (i = 0; i < HDA_MAX_CODECS; i++) {
+
+		if (!(bus->codec_mask & (1 << i)))
+			continue;
+
+		ret = hda_codec_probe(sdev, i);
+		if (ret < 0) {
+			dev_err(bus->dev, "error: codec #%d probe error, ret: %d\n",
+				i, ret);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL(hda_codec_probe_bus);
+
+#if IS_ENABLED(CONFIG_SND_SOC_HDAC_HDMI)
+
+void hda_codec_i915_get(struct snd_sof_dev *sdev)
+{
+	struct hdac_bus *bus = sof_to_bus(sdev);
+
+	dev_dbg(bus->dev, "Turning i915 HDAC power on\n");
+	snd_hdac_display_power(bus, HDA_CODEC_IDX_CONTROLLER, true);
+}
+EXPORT_SYMBOL(hda_codec_i915_get);
+
+void hda_codec_i915_put(struct snd_sof_dev *sdev)
+{
+	struct hdac_bus *bus = sof_to_bus(sdev);
+
+	dev_dbg(bus->dev, "Turning i915 HDAC power off\n");
+	snd_hdac_display_power(bus, HDA_CODEC_IDX_CONTROLLER, false);
+}
+EXPORT_SYMBOL(hda_codec_i915_put);
+
+int hda_codec_i915_init(struct snd_sof_dev *sdev)
+{
+	struct hdac_bus *bus = sof_to_bus(sdev);
+	int ret;
+
+	/* i915 exposes a HDA codec for HDMI audio */
+	ret = snd_hdac_i915_init(bus);
+	if (ret < 0)
+		return ret;
+
+	hda_codec_i915_get(sdev);
+
+	return 0;
+}
+EXPORT_SYMBOL(hda_codec_i915_init);
+
+int hda_codec_i915_exit(struct snd_sof_dev *sdev)
+{
+	struct hdac_bus *bus = sof_to_bus(sdev);
+	int ret;
+
+	hda_codec_i915_put(sdev);
+
+	ret = snd_hdac_i915_exit(bus);
+
+	return ret;
+}
+EXPORT_SYMBOL(hda_codec_i915_exit);
+
+#endif /* CONFIG_SND_SOC_HDAC_HDMI */
+
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/sound/soc/sof/intel/hda-ctrl.c b/sound/soc/sof/intel/hda-ctrl.c
new file mode 100644
index 000000000000..2c3645736e1f
--- /dev/null
+++ b/sound/soc/sof/intel/hda-ctrl.c
@@ -0,0 +1,181 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license.  When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2018 Intel Corporation. All rights reserved.
+//
+// Authors: Liam Girdwood <liam.r.girdwood@linux.intel.com>
+//	    Ranjani Sridharan <ranjani.sridharan@linux.intel.com>
+//	    Rander Wang <rander.wang@intel.com>
+//          Keyon Jie <yang.jie@linux.intel.com>
+//
+
+/*
+ * Hardware interface for generic Intel audio DSP HDA IP
+ */
+
+#include <sound/hdaudio_ext.h>
+#include <sound/hda_register.h>
+#include "../ops.h"
+#include "hda.h"
+
+/*
+ * HDA Operations.
+ */
+
+int hda_dsp_ctrl_link_reset(struct snd_sof_dev *sdev, bool reset)
+{
+	unsigned long timeout;
+	u32 gctl = 0;
+	u32 val;
+
+	/* 0 to enter reset and 1 to exit reset */
+	val = reset ? 0 : SOF_HDA_GCTL_RESET;
+
+	/* enter/exit HDA controller reset */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR, SOF_HDA_GCTL,
+				SOF_HDA_GCTL_RESET, val);
+
+	/* wait to enter/exit reset */
+	timeout = jiffies + msecs_to_jiffies(HDA_DSP_CTRL_RESET_TIMEOUT);
+	while (time_before(jiffies, timeout)) {
+		gctl = snd_sof_dsp_read(sdev, HDA_DSP_HDA_BAR, SOF_HDA_GCTL);
+		if ((gctl & SOF_HDA_GCTL_RESET) == val)
+			return 0;
+		usleep_range(500, 1000);
+	}
+
+	/* enter/exit reset failed */
+	dev_err(sdev->dev, "error: failed to %s HDA controller gctl 0x%x\n",
+		reset ? "reset" : "ready", gctl);
+	return -EIO;
+}
+
+int hda_dsp_ctrl_get_caps(struct snd_sof_dev *sdev)
+{
+	struct hdac_bus *bus = sof_to_bus(sdev);
+	u32 cap, offset, feature;
+	int count = 0;
+
+	offset = snd_sof_dsp_read(sdev, HDA_DSP_HDA_BAR, SOF_HDA_LLCH);
+
+	do {
+		cap = snd_sof_dsp_read(sdev, HDA_DSP_HDA_BAR, offset);
+
+		dev_dbg(sdev->dev, "checking for capabilities at offset 0x%x\n",
+			offset & SOF_HDA_CAP_NEXT_MASK);
+
+		feature = (cap & SOF_HDA_CAP_ID_MASK) >> SOF_HDA_CAP_ID_OFF;
+
+		switch (feature) {
+		case SOF_HDA_PP_CAP_ID:
+			dev_dbg(sdev->dev, "found DSP capability at 0x%x\n",
+				offset);
+			bus->ppcap = bus->remap_addr + offset;
+			sdev->bar[HDA_DSP_PP_BAR] = bus->ppcap;
+			break;
+		case SOF_HDA_SPIB_CAP_ID:
+			dev_dbg(sdev->dev, "found SPIB capability at 0x%x\n",
+				offset);
+			bus->spbcap = bus->remap_addr + offset;
+			sdev->bar[HDA_DSP_SPIB_BAR] = bus->spbcap;
+			break;
+		case SOF_HDA_DRSM_CAP_ID:
+			dev_dbg(sdev->dev, "found DRSM capability at 0x%x\n",
+				offset);
+			bus->drsmcap = bus->remap_addr + offset;
+			sdev->bar[HDA_DSP_DRSM_BAR] = bus->drsmcap;
+			break;
+		case SOF_HDA_GTS_CAP_ID:
+			dev_dbg(sdev->dev, "found GTS capability at 0x%x\n",
+				offset);
+			bus->gtscap = bus->remap_addr + offset;
+			break;
+		case SOF_HDA_ML_CAP_ID:
+			dev_dbg(sdev->dev, "found ML capability at 0x%x\n",
+				offset);
+			bus->mlcap = bus->remap_addr + offset;
+			break;
+		default:
+			dev_vdbg(sdev->dev, "found capability %d at 0x%x\n",
+				 feature, offset);
+			break;
+		}
+
+		offset = cap & SOF_HDA_CAP_NEXT_MASK;
+	} while (count++ <= SOF_HDA_MAX_CAPS && offset);
+
+	return 0;
+}
+
+void hda_dsp_ctrl_ppcap_enable(struct snd_sof_dev *sdev, bool enable)
+{
+	u32 val = enable ? SOF_HDA_PPCTL_GPROCEN : 0;
+
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_PP_BAR, SOF_HDA_REG_PP_PPCTL,
+				SOF_HDA_PPCTL_GPROCEN, val);
+}
+
+void hda_dsp_ctrl_ppcap_int_enable(struct snd_sof_dev *sdev, bool enable)
+{
+	u32 val	= enable ? SOF_HDA_PPCTL_PIE : 0;
+
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_PP_BAR, SOF_HDA_REG_PP_PPCTL,
+				SOF_HDA_PPCTL_PIE, val);
+}
+
+void hda_dsp_ctrl_misc_clock_gating(struct snd_sof_dev *sdev, bool enable)
+{
+	u32 val = enable ? PCI_CGCTL_MISCBDCGE_MASK : 0;
+
+	snd_sof_pci_update_bits(sdev, PCI_CGCTL, PCI_CGCTL_MISCBDCGE_MASK, val);
+}
+
+/*
+ * enable/disable audio dsp clock gating and power gating bits.
+ * This allows the HW to opportunistically power and clock gate
+ * the audio dsp when it is idle
+ */
+int hda_dsp_ctrl_clock_power_gating(struct snd_sof_dev *sdev, bool enable)
+{
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
+	struct hdac_bus *bus = sof_to_bus(sdev);
+#endif
+	u32 val;
+
+	/* enable/disable audio dsp clock gating */
+	val = enable ? PCI_CGCTL_ADSPDCGE : 0;
+	snd_sof_pci_update_bits(sdev, PCI_CGCTL, PCI_CGCTL_ADSPDCGE, val);
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
+	/* enable/disable L1 support */
+	val = enable ? SOF_HDA_VS_EM2_L1SEN : 0;
+	snd_hdac_chip_updatel(bus, VS_EM2, SOF_HDA_VS_EM2_L1SEN, val);
+#endif
+
+	/* enable/disable audio dsp power gating */
+	val = enable ? 0 : PCI_PGCTL_ADSPPGD;
+	snd_sof_pci_update_bits(sdev, PCI_PGCTL, PCI_PGCTL_ADSPPGD, val);
+
+	return 0;
+}
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
+/*
+ * While performing reset, controller may not come back properly and causing
+ * issues, so recommendation is to set CGCTL.MISCBDCGE to 0 then do reset
+ * (init chip) and then again set CGCTL.MISCBDCGE to 1
+ */
+int hda_dsp_ctrl_init_chip(struct snd_sof_dev *sdev, bool full_reset)
+{
+	struct hdac_bus *bus = sof_to_bus(sdev);
+	int ret;
+
+	hda_dsp_ctrl_misc_clock_gating(sdev, false);
+	ret = snd_hdac_bus_init_chip(bus, full_reset);
+	hda_dsp_ctrl_misc_clock_gating(sdev, true);
+
+	return ret;
+}
+#endif
diff --git a/sound/soc/sof/intel/hda-dai.c b/sound/soc/sof/intel/hda-dai.c
new file mode 100644
index 000000000000..e1decf25aeac
--- /dev/null
+++ b/sound/soc/sof/intel/hda-dai.c
@@ -0,0 +1,356 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license.  When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2018 Intel Corporation. All rights reserved.
+//
+// Authors: Keyon Jie <yang.jie@linux.intel.com>
+//
+
+#include <sound/pcm_params.h>
+#include <sound/hdaudio_ext.h>
+#include "../sof-priv.h"
+#include "hda.h"
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
+
+struct hda_pipe_params {
+	u8 host_dma_id;
+	u8 link_dma_id;
+	u32 ch;
+	u32 s_freq;
+	u32 s_fmt;
+	u8 linktype;
+	snd_pcm_format_t format;
+	int link_index;
+	int stream;
+	unsigned int host_bps;
+	unsigned int link_bps;
+};
+
+/*
+ * Unlike GP dma, there is a set of stream registers in hda controller
+ * to control the link dma channels. Each register controls one link
+ * dma channel and the relation is fixed. To make sure FW uses correct
+ * link dma channels, host allocates stream registers and sends the
+ * corresponding link dma channels to FW to allocate link dma channel
+ *
+ * FIXME: this API is abused in the sense that tx_num and rx_num are
+ * passed as arguments, not returned. We need to find a better way to
+ * retrieve the stream tag allocated for the link DMA
+ */
+static int hda_link_dma_get_channels(struct snd_soc_dai *dai,
+				     unsigned int *tx_num,
+				     unsigned int *tx_slot,
+				     unsigned int *rx_num,
+				     unsigned int *rx_slot)
+{
+	struct hdac_bus *bus;
+	struct hdac_ext_stream *stream;
+	struct snd_pcm_substream substream;
+	struct snd_sof_dev *sdev =
+		snd_soc_component_get_drvdata(dai->component);
+
+	bus = sof_to_bus(sdev);
+
+	memset(&substream, 0, sizeof(substream));
+	if (*tx_num == 1) {
+		substream.stream = SNDRV_PCM_STREAM_PLAYBACK;
+		stream = snd_hdac_ext_stream_assign(bus, &substream,
+						    HDAC_EXT_STREAM_TYPE_LINK);
+		if (!stream) {
+			dev_err(bus->dev, "error: failed to find a free hda ext stream for playback");
+			return -EBUSY;
+		}
+
+		snd_soc_dai_set_dma_data(dai, &substream, stream);
+		*tx_slot = hdac_stream(stream)->stream_tag - 1;
+
+		dev_dbg(bus->dev, "link dma channel %d for playback", *tx_slot);
+	}
+
+	if (*rx_num == 1) {
+		substream.stream = SNDRV_PCM_STREAM_CAPTURE;
+		stream = snd_hdac_ext_stream_assign(bus, &substream,
+						    HDAC_EXT_STREAM_TYPE_LINK);
+		if (!stream) {
+			dev_err(bus->dev, "error: failed to find a free hda ext stream for capture");
+			return -EBUSY;
+		}
+
+		snd_soc_dai_set_dma_data(dai, &substream, stream);
+		*rx_slot = hdac_stream(stream)->stream_tag - 1;
+
+		dev_dbg(bus->dev, "link dma channel %d for capture", *rx_slot);
+	}
+
+	return 0;
+}
+
+static int hda_link_dma_params(struct hdac_ext_stream *stream,
+			       struct hda_pipe_params *params)
+{
+	struct hdac_stream *hstream = &stream->hstream;
+	unsigned char stream_tag = hstream->stream_tag;
+	struct hdac_bus *bus = hstream->bus;
+	struct hdac_ext_link *link;
+	unsigned int format_val;
+
+	snd_hdac_ext_stream_decouple(bus, stream, true);
+	snd_hdac_ext_link_stream_reset(stream);
+
+	format_val = snd_hdac_calc_stream_format(params->s_freq, params->ch,
+						 params->format,
+						 params->link_bps, 0);
+
+	dev_dbg(bus->dev, "format_val=%d, rate=%d, ch=%d, format=%d\n",
+		format_val, params->s_freq, params->ch, params->format);
+
+	snd_hdac_ext_link_stream_setup(stream, format_val);
+
+	if (stream->hstream.direction == SNDRV_PCM_STREAM_PLAYBACK) {
+		list_for_each_entry(link, &bus->hlink_list, list) {
+			if (link->index == params->link_index)
+				snd_hdac_ext_link_set_stream_id(link,
+								stream_tag);
+		}
+	}
+
+	stream->link_prepared = 1;
+
+	return 0;
+}
+
+static int hda_link_hw_params(struct snd_pcm_substream *substream,
+			      struct snd_pcm_hw_params *params,
+			      struct snd_soc_dai *dai)
+{
+	struct hdac_stream *hstream = substream->runtime->private_data;
+	struct hdac_bus *bus = hstream->bus;
+	struct hdac_ext_stream *link_dev;
+	struct snd_soc_pcm_runtime *rtd = snd_pcm_substream_chip(substream);
+	struct snd_soc_dai *codec_dai = rtd->codec_dai;
+	struct sof_intel_hda_stream *hda_stream;
+	struct hda_pipe_params p_params = {0};
+	struct hdac_ext_link *link;
+	int stream_tag;
+
+	link_dev = snd_soc_dai_get_dma_data(dai, substream);
+
+	hda_stream = container_of(link_dev, struct sof_intel_hda_stream,
+				  hda_stream);
+	hda_stream->hw_params_upon_resume = 0;
+
+	link = snd_hdac_ext_bus_get_link(bus, codec_dai->component->name);
+	if (!link)
+		return -EINVAL;
+
+	stream_tag = hdac_stream(link_dev)->stream_tag;
+
+	/* set the stream tag in the codec dai dma params  */
+	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
+		snd_soc_dai_set_tdm_slot(codec_dai, stream_tag, 0, 0, 0);
+	else
+		snd_soc_dai_set_tdm_slot(codec_dai, 0, stream_tag, 0, 0);
+
+	p_params.s_fmt = snd_pcm_format_width(params_format(params));
+	p_params.ch = params_channels(params);
+	p_params.s_freq = params_rate(params);
+	p_params.stream = substream->stream;
+	p_params.link_dma_id = stream_tag - 1;
+	p_params.link_index = link->index;
+	p_params.format = params_format(params);
+
+	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
+		p_params.link_bps = codec_dai->driver->playback.sig_bits;
+	else
+		p_params.link_bps = codec_dai->driver->capture.sig_bits;
+
+	return hda_link_dma_params(link_dev, &p_params);
+}
+
+static int hda_link_pcm_prepare(struct snd_pcm_substream *substream,
+				struct snd_soc_dai *dai)
+{
+	struct hdac_ext_stream *link_dev =
+				snd_soc_dai_get_dma_data(dai, substream);
+	struct sof_intel_hda_stream *hda_stream;
+	struct snd_sof_dev *sdev =
+				snd_soc_component_get_drvdata(dai->component);
+	struct snd_soc_pcm_runtime *rtd = snd_pcm_substream_chip(substream);
+	int stream = substream->stream;
+
+	hda_stream = container_of(link_dev, struct sof_intel_hda_stream,
+				  hda_stream);
+
+	/* setup hw_params again only if resuming from system suspend */
+	if (!hda_stream->hw_params_upon_resume)
+		return 0;
+
+	dev_dbg(sdev->dev, "hda: prepare stream dir %d\n", substream->stream);
+
+	return hda_link_hw_params(substream, &rtd->dpcm[stream].hw_params,
+				  dai);
+}
+
+static int hda_link_pcm_trigger(struct snd_pcm_substream *substream,
+				int cmd, struct snd_soc_dai *dai)
+{
+	struct hdac_ext_stream *link_dev =
+				snd_soc_dai_get_dma_data(dai, substream);
+	int ret;
+
+	dev_dbg(dai->dev, "In %s cmd=%d\n", __func__, cmd);
+	switch (cmd) {
+	case SNDRV_PCM_TRIGGER_RESUME:
+		/* set up hw_params */
+		ret = hda_link_pcm_prepare(substream, dai);
+		if (ret < 0) {
+			dev_err(dai->dev,
+				"error: setting up hw_params during resume\n");
+			return ret;
+		}
+
+		/* fallthrough */
+	case SNDRV_PCM_TRIGGER_START:
+	case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
+		snd_hdac_ext_link_stream_start(link_dev);
+		break;
+	case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
+	case SNDRV_PCM_TRIGGER_SUSPEND:
+	case SNDRV_PCM_TRIGGER_STOP:
+		snd_hdac_ext_link_stream_clear(link_dev);
+		break;
+	default:
+		return -EINVAL;
+	}
+	return 0;
+}
+
+/*
+ * FIXME: This API is also abused since it's used for two purposes.
+ * when the substream argument is NULL this function is used for cleanups
+ * that aren't necessarily required, and called explicitly by handling
+ * ASoC core structures, which is not recommended.
+ * This part will be reworked in follow-up patches.
+ */
+static int hda_link_hw_free(struct snd_pcm_substream *substream,
+			    struct snd_soc_dai *dai)
+{
+	const char *name;
+	unsigned int stream_tag;
+	struct hdac_bus *bus;
+	struct hdac_ext_link *link;
+	struct hdac_stream *hstream;
+	struct hdac_ext_stream *stream;
+	struct snd_soc_pcm_runtime *rtd;
+	struct hdac_ext_stream *link_dev;
+	struct snd_pcm_substream pcm_substream;
+
+	memset(&pcm_substream, 0, sizeof(pcm_substream));
+	if (substream) {
+		hstream = substream->runtime->private_data;
+		bus = hstream->bus;
+		rtd = snd_pcm_substream_chip(substream);
+		link_dev = snd_soc_dai_get_dma_data(dai, substream);
+		snd_hdac_ext_stream_decouple(bus, link_dev, false);
+		name = rtd->codec_dai->component->name;
+		link = snd_hdac_ext_bus_get_link(bus, name);
+		if (!link)
+			return -EINVAL;
+
+		if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) {
+			stream_tag = hdac_stream(link_dev)->stream_tag;
+			snd_hdac_ext_link_clear_stream_id(link, stream_tag);
+		}
+
+		link_dev->link_prepared = 0;
+	} else {
+		/* release all hda streams when dai link is unloaded */
+		pcm_substream.stream = SNDRV_PCM_STREAM_PLAYBACK;
+		stream = snd_soc_dai_get_dma_data(dai, &pcm_substream);
+		if (stream) {
+			snd_soc_dai_set_dma_data(dai, &pcm_substream, NULL);
+			snd_hdac_ext_stream_release(stream,
+						    HDAC_EXT_STREAM_TYPE_LINK);
+		}
+
+		pcm_substream.stream = SNDRV_PCM_STREAM_CAPTURE;
+		stream = snd_soc_dai_get_dma_data(dai, &pcm_substream);
+		if (stream) {
+			snd_soc_dai_set_dma_data(dai, &pcm_substream, NULL);
+			snd_hdac_ext_stream_release(stream,
+						    HDAC_EXT_STREAM_TYPE_LINK);
+		}
+	}
+
+	return 0;
+}
+
+static const struct snd_soc_dai_ops hda_link_dai_ops = {
+	.hw_params = hda_link_hw_params,
+	.hw_free = hda_link_hw_free,
+	.trigger = hda_link_pcm_trigger,
+	.prepare = hda_link_pcm_prepare,
+	.get_channel_map = hda_link_dma_get_channels,
+};
+#endif
+
+/*
+ * common dai driver for skl+ platforms.
+ * some products who use this DAI array only physically have a subset of
+ * the DAIs, but no harm is done here by adding the whole set.
+ */
+struct snd_soc_dai_driver skl_dai[] = {
+{
+	.name = "SSP0 Pin",
+},
+{
+	.name = "SSP1 Pin",
+},
+{
+	.name = "SSP2 Pin",
+},
+{
+	.name = "SSP3 Pin",
+},
+{
+	.name = "SSP4 Pin",
+},
+{
+	.name = "SSP5 Pin",
+},
+{
+	.name = "DMIC01 Pin",
+},
+{
+	.name = "DMIC16k Pin",
+},
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
+{
+	.name = "iDisp1 Pin",
+	.ops = &hda_link_dai_ops,
+},
+{
+	.name = "iDisp2 Pin",
+	.ops = &hda_link_dai_ops,
+},
+{
+	.name = "iDisp3 Pin",
+	.ops = &hda_link_dai_ops,
+},
+{
+	.name = "Analog CPU DAI",
+	.ops = &hda_link_dai_ops,
+},
+{
+	.name = "Digital CPU DAI",
+	.ops = &hda_link_dai_ops,
+},
+{
+	.name = "Alt Analog CPU DAI",
+	.ops = &hda_link_dai_ops,
+},
+#endif
+};
diff --git a/sound/soc/sof/intel/hda-dsp.c b/sound/soc/sof/intel/hda-dsp.c
new file mode 100644
index 000000000000..5b73115a0b78
--- /dev/null
+++ b/sound/soc/sof/intel/hda-dsp.c
@@ -0,0 +1,471 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license.  When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2018 Intel Corporation. All rights reserved.
+//
+// Authors: Liam Girdwood <liam.r.girdwood@linux.intel.com>
+//	    Ranjani Sridharan <ranjani.sridharan@linux.intel.com>
+//	    Rander Wang <rander.wang@intel.com>
+//          Keyon Jie <yang.jie@linux.intel.com>
+//
+
+/*
+ * Hardware interface for generic Intel audio DSP HDA IP
+ */
+
+#include <sound/hdaudio_ext.h>
+#include <sound/hda_register.h>
+#include "../ops.h"
+#include "hda.h"
+
+/*
+ * DSP Core control.
+ */
+
+int hda_dsp_core_reset_enter(struct snd_sof_dev *sdev, unsigned int core_mask)
+{
+	u32 adspcs;
+	u32 reset;
+	int ret;
+
+	/* set reset bits for cores */
+	reset = HDA_DSP_ADSPCS_CRST_MASK(core_mask);
+	snd_sof_dsp_update_bits_unlocked(sdev, HDA_DSP_BAR,
+					 HDA_DSP_REG_ADSPCS,
+					 reset, reset),
+
+	/* poll with timeout to check if operation successful */
+	ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR,
+					HDA_DSP_REG_ADSPCS, adspcs,
+					((adspcs & reset) == reset),
+					HDA_DSP_REG_POLL_INTERVAL_US,
+					HDA_DSP_RESET_TIMEOUT_US);
+
+	/* has core entered reset ? */
+	adspcs = snd_sof_dsp_read(sdev, HDA_DSP_BAR,
+				  HDA_DSP_REG_ADSPCS);
+	if ((adspcs & HDA_DSP_ADSPCS_CRST_MASK(core_mask)) !=
+		HDA_DSP_ADSPCS_CRST_MASK(core_mask)) {
+		dev_err(sdev->dev,
+			"error: reset enter failed: core_mask %x adspcs 0x%x\n",
+			core_mask, adspcs);
+		ret = -EIO;
+	}
+
+	return ret;
+}
+
+int hda_dsp_core_reset_leave(struct snd_sof_dev *sdev, unsigned int core_mask)
+{
+	unsigned int crst;
+	u32 adspcs;
+	int ret;
+
+	/* clear reset bits for cores */
+	snd_sof_dsp_update_bits_unlocked(sdev, HDA_DSP_BAR,
+					 HDA_DSP_REG_ADSPCS,
+					 HDA_DSP_ADSPCS_CRST_MASK(core_mask),
+					 0);
+
+	/* poll with timeout to check if operation successful */
+	crst = HDA_DSP_ADSPCS_CRST_MASK(core_mask);
+	ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR,
+					    HDA_DSP_REG_ADSPCS, adspcs,
+					    !(adspcs & crst),
+					    HDA_DSP_REG_POLL_INTERVAL_US,
+					    HDA_DSP_RESET_TIMEOUT_US);
+
+	/* has core left reset ? */
+	adspcs = snd_sof_dsp_read(sdev, HDA_DSP_BAR,
+				  HDA_DSP_REG_ADSPCS);
+	if ((adspcs & HDA_DSP_ADSPCS_CRST_MASK(core_mask)) != 0) {
+		dev_err(sdev->dev,
+			"error: reset leave failed: core_mask %x adspcs 0x%x\n",
+			core_mask, adspcs);
+		ret = -EIO;
+	}
+
+	return ret;
+}
+
+int hda_dsp_core_stall_reset(struct snd_sof_dev *sdev, unsigned int core_mask)
+{
+	/* stall core */
+	snd_sof_dsp_update_bits_unlocked(sdev, HDA_DSP_BAR,
+					 HDA_DSP_REG_ADSPCS,
+					 HDA_DSP_ADSPCS_CSTALL_MASK(core_mask),
+					 HDA_DSP_ADSPCS_CSTALL_MASK(core_mask));
+
+	/* set reset state */
+	return hda_dsp_core_reset_enter(sdev, core_mask);
+}
+
+int hda_dsp_core_run(struct snd_sof_dev *sdev, unsigned int core_mask)
+{
+	int ret;
+
+	/* leave reset state */
+	ret = hda_dsp_core_reset_leave(sdev, core_mask);
+	if (ret < 0)
+		return ret;
+
+	/* run core */
+	dev_dbg(sdev->dev, "unstall/run core: core_mask = %x\n", core_mask);
+	snd_sof_dsp_update_bits_unlocked(sdev, HDA_DSP_BAR,
+					 HDA_DSP_REG_ADSPCS,
+					 HDA_DSP_ADSPCS_CSTALL_MASK(core_mask),
+					 0);
+
+	/* is core now running ? */
+	if (!hda_dsp_core_is_enabled(sdev, core_mask)) {
+		hda_dsp_core_stall_reset(sdev, core_mask);
+		dev_err(sdev->dev, "error: DSP start core failed: core_mask %x\n",
+			core_mask);
+		ret = -EIO;
+	}
+
+	return ret;
+}
+
+/*
+ * Power Management.
+ */
+
+int hda_dsp_core_power_up(struct snd_sof_dev *sdev, unsigned int core_mask)
+{
+	unsigned int cpa;
+	u32 adspcs;
+	int ret;
+
+	/* update bits */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, HDA_DSP_REG_ADSPCS,
+				HDA_DSP_ADSPCS_SPA_MASK(core_mask),
+				HDA_DSP_ADSPCS_SPA_MASK(core_mask));
+
+	/* poll with timeout to check if operation successful */
+	cpa = HDA_DSP_ADSPCS_CPA_MASK(core_mask);
+	ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR,
+					    HDA_DSP_REG_ADSPCS, adspcs,
+					    (adspcs & cpa) == cpa,
+					    HDA_DSP_REG_POLL_INTERVAL_US,
+					    HDA_DSP_RESET_TIMEOUT_US);
+	if (ret < 0)
+		dev_err(sdev->dev, "error: timeout on core powerup\n");
+
+	/* did core power up ? */
+	adspcs = snd_sof_dsp_read(sdev, HDA_DSP_BAR,
+				  HDA_DSP_REG_ADSPCS);
+	if ((adspcs & HDA_DSP_ADSPCS_CPA_MASK(core_mask)) !=
+		HDA_DSP_ADSPCS_CPA_MASK(core_mask)) {
+		dev_err(sdev->dev,
+			"error: power up core failed core_mask %xadspcs 0x%x\n",
+			core_mask, adspcs);
+		ret = -EIO;
+	}
+
+	return ret;
+}
+
+int hda_dsp_core_power_down(struct snd_sof_dev *sdev, unsigned int core_mask)
+{
+	u32 adspcs;
+
+	/* update bits */
+	snd_sof_dsp_update_bits_unlocked(sdev, HDA_DSP_BAR,
+					 HDA_DSP_REG_ADSPCS,
+					 HDA_DSP_ADSPCS_SPA_MASK(core_mask), 0);
+
+	return snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR,
+				HDA_DSP_REG_ADSPCS, adspcs,
+				!(adspcs & HDA_DSP_ADSPCS_SPA_MASK(core_mask)),
+				HDA_DSP_REG_POLL_INTERVAL_US,
+				HDA_DSP_PD_TIMEOUT * USEC_PER_MSEC);
+}
+
+bool hda_dsp_core_is_enabled(struct snd_sof_dev *sdev,
+			     unsigned int core_mask)
+{
+	int val;
+	bool is_enable;
+
+	val = snd_sof_dsp_read(sdev, HDA_DSP_BAR, HDA_DSP_REG_ADSPCS);
+
+	is_enable = ((val & HDA_DSP_ADSPCS_CPA_MASK(core_mask)) &&
+			(val & HDA_DSP_ADSPCS_SPA_MASK(core_mask)) &&
+			!(val & HDA_DSP_ADSPCS_CRST_MASK(core_mask)) &&
+			!(val & HDA_DSP_ADSPCS_CSTALL_MASK(core_mask)));
+
+	dev_dbg(sdev->dev, "DSP core(s) enabled? %d : core_mask %x\n",
+		is_enable, core_mask);
+
+	return is_enable;
+}
+
+int hda_dsp_enable_core(struct snd_sof_dev *sdev, unsigned int core_mask)
+{
+	int ret;
+
+	/* return if core is already enabled */
+	if (hda_dsp_core_is_enabled(sdev, core_mask))
+		return 0;
+
+	/* power up */
+	ret = hda_dsp_core_power_up(sdev, core_mask);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: dsp core power up failed: core_mask %x\n",
+			core_mask);
+		return ret;
+	}
+
+	return hda_dsp_core_run(sdev, core_mask);
+}
+
+int hda_dsp_core_reset_power_down(struct snd_sof_dev *sdev,
+				  unsigned int core_mask)
+{
+	int ret;
+
+	/* place core in reset prior to power down */
+	ret = hda_dsp_core_stall_reset(sdev, core_mask);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: dsp core reset failed: core_mask %x\n",
+			core_mask);
+		return ret;
+	}
+
+	/* power down core */
+	ret = hda_dsp_core_power_down(sdev, core_mask);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: dsp core power down fail mask %x: %d\n",
+			core_mask, ret);
+		return ret;
+	}
+
+	/* make sure we are in OFF state */
+	if (hda_dsp_core_is_enabled(sdev, core_mask)) {
+		dev_err(sdev->dev, "error: dsp core disable fail mask %x: %d\n",
+			core_mask, ret);
+		ret = -EIO;
+	}
+
+	return ret;
+}
+
+void hda_dsp_ipc_int_enable(struct snd_sof_dev *sdev)
+{
+	struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata;
+	const struct sof_intel_dsp_desc *chip = hda->desc;
+
+	/* enable IPC DONE and BUSY interrupts */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, chip->ipc_ctl,
+			HDA_DSP_REG_HIPCCTL_DONE | HDA_DSP_REG_HIPCCTL_BUSY,
+			HDA_DSP_REG_HIPCCTL_DONE | HDA_DSP_REG_HIPCCTL_BUSY);
+
+	/* enable IPC interrupt */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, HDA_DSP_REG_ADSPIC,
+				HDA_DSP_ADSPIC_IPC, HDA_DSP_ADSPIC_IPC);
+}
+
+void hda_dsp_ipc_int_disable(struct snd_sof_dev *sdev)
+{
+	struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata;
+	const struct sof_intel_dsp_desc *chip = hda->desc;
+
+	/* disable IPC interrupt */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, HDA_DSP_REG_ADSPIC,
+				HDA_DSP_ADSPIC_IPC, 0);
+
+	/* disable IPC BUSY and DONE interrupt */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, chip->ipc_ctl,
+			HDA_DSP_REG_HIPCCTL_BUSY | HDA_DSP_REG_HIPCCTL_DONE, 0);
+}
+
+static int hda_suspend(struct snd_sof_dev *sdev, int state)
+{
+	struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata;
+	const struct sof_intel_dsp_desc *chip = hda->desc;
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
+	struct hdac_bus *bus = sof_to_bus(sdev);
+#endif
+	int ret;
+
+	/* disable IPC interrupts */
+	hda_dsp_ipc_int_disable(sdev);
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
+	/* power down all hda link */
+	snd_hdac_ext_bus_link_power_down_all(bus);
+#endif
+
+	/* power down DSP */
+	ret = hda_dsp_core_reset_power_down(sdev, chip->cores_mask);
+	if (ret < 0) {
+		dev_err(sdev->dev,
+			"error: failed to power down core during suspend\n");
+		return ret;
+	}
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
+	/* disable ppcap interrupt */
+	snd_hdac_ext_bus_ppcap_int_enable(bus, false);
+	snd_hdac_ext_bus_ppcap_enable(bus, false);
+
+	/* disable hda bus irq and i/o */
+	snd_hdac_bus_stop_chip(bus);
+#else
+	/* disable ppcap interrupt */
+	hda_dsp_ctrl_ppcap_enable(sdev, false);
+	hda_dsp_ctrl_ppcap_int_enable(sdev, false);
+
+	/* disable hda bus irq */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR, SOF_HDA_INTCTL,
+				SOF_HDA_INT_CTRL_EN | SOF_HDA_INT_GLOBAL_EN,
+				0);
+#endif
+
+	/* disable LP retention mode */
+	snd_sof_pci_update_bits(sdev, PCI_PGCTL,
+				PCI_PGCTL_LSRMD_MASK, PCI_PGCTL_LSRMD_MASK);
+
+	/* reset controller */
+	ret = hda_dsp_ctrl_link_reset(sdev, true);
+	if (ret < 0) {
+		dev_err(sdev->dev,
+			"error: failed to reset controller during suspend\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static int hda_resume(struct snd_sof_dev *sdev)
+{
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
+	struct hdac_bus *bus = sof_to_bus(sdev);
+	struct hdac_ext_link *hlink = NULL;
+#endif
+	int ret;
+
+	/*
+	 * clear TCSEL to clear playback on some HD Audio
+	 * codecs. PCI TCSEL is defined in the Intel manuals.
+	 */
+	snd_sof_pci_update_bits(sdev, PCI_TCSEL, 0x07, 0);
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
+	/* reset and start hda controller */
+	ret = hda_dsp_ctrl_init_chip(sdev, true);
+	if (ret < 0) {
+		dev_err(sdev->dev,
+			"error: failed to start controller after resume\n");
+		return ret;
+	}
+
+	hda_dsp_ctrl_misc_clock_gating(sdev, false);
+
+	/* Reset stream-to-link mapping */
+	list_for_each_entry(hlink, &bus->hlink_list, list)
+		bus->io_ops->reg_writel(0, hlink->ml_addr + AZX_REG_ML_LOSIDV);
+
+	hda_dsp_ctrl_misc_clock_gating(sdev, true);
+
+	/* enable ppcap interrupt */
+	snd_hdac_ext_bus_ppcap_enable(bus, true);
+	snd_hdac_ext_bus_ppcap_int_enable(bus, true);
+#else
+
+	hda_dsp_ctrl_misc_clock_gating(sdev, false);
+
+	/* reset controller */
+	ret = hda_dsp_ctrl_link_reset(sdev, true);
+	if (ret < 0) {
+		dev_err(sdev->dev,
+			"error: failed to reset controller during resume\n");
+		return ret;
+	}
+
+	/* take controller out of reset */
+	ret = hda_dsp_ctrl_link_reset(sdev, false);
+	if (ret < 0) {
+		dev_err(sdev->dev,
+			"error: failed to ready controller during resume\n");
+		return ret;
+	}
+
+	/* enable hda bus irq */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR, SOF_HDA_INTCTL,
+				SOF_HDA_INT_CTRL_EN | SOF_HDA_INT_GLOBAL_EN,
+				SOF_HDA_INT_CTRL_EN | SOF_HDA_INT_GLOBAL_EN);
+
+	hda_dsp_ctrl_misc_clock_gating(sdev, true);
+
+	/* enable ppcap interrupt */
+	hda_dsp_ctrl_ppcap_enable(sdev, true);
+	hda_dsp_ctrl_ppcap_int_enable(sdev, true);
+#endif
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
+	/* turn off the links that were off before suspend */
+	list_for_each_entry(hlink, &bus->hlink_list, list) {
+		if (!hlink->ref_count)
+			snd_hdac_ext_bus_link_power_down(hlink);
+	}
+
+	/* check dma status and clean up CORB/RIRB buffers */
+	if (!bus->cmd_dma_state)
+		snd_hdac_bus_stop_cmd_io(bus);
+#endif
+
+	return 0;
+}
+
+int hda_dsp_resume(struct snd_sof_dev *sdev)
+{
+	/* init hda controller. DSP cores will be powered up during fw boot */
+	return hda_resume(sdev);
+}
+
+int hda_dsp_runtime_resume(struct snd_sof_dev *sdev)
+{
+	/* init hda controller. DSP cores will be powered up during fw boot */
+	return hda_resume(sdev);
+}
+
+int hda_dsp_runtime_suspend(struct snd_sof_dev *sdev, int state)
+{
+	/* stop hda controller and power dsp off */
+	return hda_suspend(sdev, state);
+}
+
+int hda_dsp_suspend(struct snd_sof_dev *sdev, int state)
+{
+	struct hdac_bus *bus = sof_to_bus(sdev);
+	int ret;
+
+	/* stop hda controller and power dsp off */
+	ret = hda_suspend(sdev, state);
+	if (ret < 0) {
+		dev_err(bus->dev, "error: suspending dsp\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+void hda_dsp_set_hw_params_upon_resume(struct snd_sof_dev *sdev)
+{
+	struct hdac_bus *bus = sof_to_bus(sdev);
+	struct sof_intel_hda_stream *hda_stream;
+	struct hdac_ext_stream *stream;
+	struct hdac_stream *s;
+
+	/* set internal flag for BE */
+	list_for_each_entry(s, &bus->stream_list, list) {
+		stream = stream_to_hdac_ext_stream(s);
+		hda_stream = container_of(stream, struct sof_intel_hda_stream,
+					  hda_stream);
+		hda_stream->hw_params_upon_resume = 1;
+	}
+}
diff --git a/sound/soc/sof/intel/hda-ipc.c b/sound/soc/sof/intel/hda-ipc.c
new file mode 100644
index 000000000000..73ead7070cde
--- /dev/null
+++ b/sound/soc/sof/intel/hda-ipc.c
@@ -0,0 +1,455 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license.  When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2018 Intel Corporation. All rights reserved.
+//
+// Authors: Liam Girdwood <liam.r.girdwood@linux.intel.com>
+//	    Ranjani Sridharan <ranjani.sridharan@linux.intel.com>
+//	    Rander Wang <rander.wang@intel.com>
+//          Keyon Jie <yang.jie@linux.intel.com>
+//
+
+/*
+ * Hardware interface for generic Intel audio DSP HDA IP
+ */
+
+#include "../ops.h"
+#include "hda.h"
+
+static void hda_dsp_ipc_host_done(struct snd_sof_dev *sdev)
+{
+	/*
+	 * tell DSP cmd is done - clear busy
+	 * interrupt and send reply msg to dsp
+	 */
+	snd_sof_dsp_update_bits_forced(sdev, HDA_DSP_BAR,
+				       HDA_DSP_REG_HIPCT,
+				       HDA_DSP_REG_HIPCT_BUSY,
+				       HDA_DSP_REG_HIPCT_BUSY);
+
+	/* unmask BUSY interrupt */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR,
+				HDA_DSP_REG_HIPCCTL,
+				HDA_DSP_REG_HIPCCTL_BUSY,
+				HDA_DSP_REG_HIPCCTL_BUSY);
+}
+
+static void hda_dsp_ipc_dsp_done(struct snd_sof_dev *sdev)
+{
+	/*
+	 * set DONE bit - tell DSP we have received the reply msg
+	 * from DSP, and processed it, don't send more reply to host
+	 */
+	snd_sof_dsp_update_bits_forced(sdev, HDA_DSP_BAR,
+				       HDA_DSP_REG_HIPCIE,
+				       HDA_DSP_REG_HIPCIE_DONE,
+				       HDA_DSP_REG_HIPCIE_DONE);
+
+	/* unmask Done interrupt */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR,
+				HDA_DSP_REG_HIPCCTL,
+				HDA_DSP_REG_HIPCCTL_DONE,
+				HDA_DSP_REG_HIPCCTL_DONE);
+}
+
+int hda_dsp_ipc_send_msg(struct snd_sof_dev *sdev, struct snd_sof_ipc_msg *msg)
+{
+	u32 cmd = msg->header;
+
+	/* send IPC message to DSP */
+	sof_mailbox_write(sdev, sdev->host_box.offset, msg->msg_data,
+			  msg->msg_size);
+	snd_sof_dsp_write(sdev, HDA_DSP_BAR, HDA_DSP_REG_HIPCI,
+			  cmd | HDA_DSP_REG_HIPCI_BUSY);
+
+	return 0;
+}
+
+void hda_dsp_ipc_get_reply(struct snd_sof_dev *sdev)
+{
+	struct snd_sof_ipc_msg *msg = sdev->msg;
+	struct sof_ipc_reply reply;
+	struct sof_ipc_cmd_hdr *hdr;
+	unsigned long flags;
+	int ret = 0;
+
+	/*
+	 * Sometimes, there is unexpected reply ipc arriving. The reply
+	 * ipc belongs to none of the ipcs sent from driver.
+	 * In this case, the driver must ignore the ipc.
+	 */
+	if (!msg) {
+		dev_warn(sdev->dev, "unexpected ipc interrupt raised!\n");
+		return;
+	}
+	spin_lock_irqsave(&sdev->ipc_lock, flags);
+
+	hdr = msg->msg_data;
+	if (hdr->cmd == (SOF_IPC_GLB_PM_MSG | SOF_IPC_PM_CTX_SAVE)) {
+		/*
+		 * memory windows are powered off before sending IPC reply,
+		 * so we can't read the mailbox for CTX_SAVE reply.
+		 */
+		reply.error = 0;
+		reply.hdr.cmd = SOF_IPC_GLB_REPLY;
+		reply.hdr.size = sizeof(reply);
+		memcpy(msg->reply_data, &reply, sizeof(reply));
+		goto out;
+	}
+
+	/* get IPC reply from DSP in the mailbox */
+	sof_mailbox_read(sdev, sdev->host_box.offset, &reply,
+			 sizeof(reply));
+
+	if (reply.error < 0) {
+		memcpy(msg->reply_data, &reply, sizeof(reply));
+		ret = reply.error;
+	} else {
+		/* reply correct size ? */
+		if (reply.hdr.size != msg->reply_size) {
+			dev_err(sdev->dev, "error: reply expected %zu got %u bytes\n",
+				msg->reply_size, reply.hdr.size);
+			ret = -EINVAL;
+		}
+
+		/* read the message */
+		if (msg->reply_size > 0)
+			sof_mailbox_read(sdev, sdev->host_box.offset,
+					 msg->reply_data, msg->reply_size);
+	}
+
+out:
+	msg->reply_error = ret;
+
+	spin_unlock_irqrestore(&sdev->ipc_lock, flags);
+}
+
+static bool hda_dsp_ipc_is_sof(uint32_t msg)
+{
+	return (msg & (HDA_DSP_IPC_PURGE_FW | 0xf << 9)) != msg ||
+		(msg & HDA_DSP_IPC_PURGE_FW) != HDA_DSP_IPC_PURGE_FW;
+}
+
+/* IPC handler thread */
+irqreturn_t hda_dsp_ipc_irq_thread(int irq, void *context)
+{
+	struct snd_sof_dev *sdev = context;
+	irqreturn_t ret = IRQ_NONE;
+	u32 hipci;
+	u32 hipcie;
+	u32 hipct;
+	u32 hipcte;
+	u32 hipcctl;
+	u32 msg;
+	u32 msg_ext;
+
+	/* read IPC status */
+	hipcie = snd_sof_dsp_read(sdev, HDA_DSP_BAR,
+				  HDA_DSP_REG_HIPCIE);
+	hipct = snd_sof_dsp_read(sdev, HDA_DSP_BAR, HDA_DSP_REG_HIPCT);
+	hipcctl = snd_sof_dsp_read(sdev, HDA_DSP_BAR, HDA_DSP_REG_HIPCCTL);
+
+	/* reenable IPC interrupt */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, HDA_DSP_REG_ADSPIC,
+				HDA_DSP_ADSPIC_IPC, HDA_DSP_ADSPIC_IPC);
+
+	/* is this a reply message from the DSP */
+	if (hipcie & HDA_DSP_REG_HIPCIE_DONE &&
+	    hipcctl & HDA_DSP_REG_HIPCCTL_DONE) {
+		hipci = snd_sof_dsp_read(sdev, HDA_DSP_BAR,
+					 HDA_DSP_REG_HIPCI);
+		msg = hipci & HDA_DSP_REG_HIPCI_MSG_MASK;
+		msg_ext = hipcie & HDA_DSP_REG_HIPCIE_MSG_MASK;
+
+		dev_vdbg(sdev->dev,
+			 "ipc: firmware response, msg:0x%x, msg_ext:0x%x\n",
+			 msg, msg_ext);
+
+		/* mask Done interrupt */
+		snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR,
+					HDA_DSP_REG_HIPCCTL,
+					HDA_DSP_REG_HIPCCTL_DONE, 0);
+
+		/* handle immediate reply from DSP core - ignore ROM messages */
+		if (hda_dsp_ipc_is_sof(msg)) {
+			hda_dsp_ipc_get_reply(sdev);
+			snd_sof_ipc_reply(sdev, msg);
+		}
+
+		/* wake up sleeper if we are loading code */
+		if (sdev->code_loading)	{
+			sdev->code_loading = 0;
+			wake_up(&sdev->waitq);
+		}
+
+		/* set the done bit */
+		hda_dsp_ipc_dsp_done(sdev);
+
+		ret = IRQ_HANDLED;
+	}
+
+	/* is this a new message from DSP */
+	if (hipct & HDA_DSP_REG_HIPCT_BUSY &&
+	    hipcctl & HDA_DSP_REG_HIPCCTL_BUSY) {
+
+		hipcte = snd_sof_dsp_read(sdev, HDA_DSP_BAR,
+					  HDA_DSP_REG_HIPCTE);
+		msg = hipct & HDA_DSP_REG_HIPCT_MSG_MASK;
+		msg_ext = hipcte & HDA_DSP_REG_HIPCTE_MSG_MASK;
+
+		dev_vdbg(sdev->dev,
+			 "ipc: firmware initiated, msg:0x%x, msg_ext:0x%x\n",
+			 msg, msg_ext);
+
+		/* mask BUSY interrupt */
+		snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR,
+					HDA_DSP_REG_HIPCCTL,
+					HDA_DSP_REG_HIPCCTL_BUSY, 0);
+
+		/* handle messages from DSP */
+		if ((hipct & SOF_IPC_PANIC_MAGIC_MASK) == SOF_IPC_PANIC_MAGIC) {
+			/* this is a PANIC message !! */
+			snd_sof_dsp_panic(sdev, HDA_DSP_PANIC_OFFSET(msg_ext));
+		} else {
+			/* normal message - process normally */
+			snd_sof_ipc_msgs_rx(sdev);
+		}
+
+		hda_dsp_ipc_host_done(sdev);
+
+		ret = IRQ_HANDLED;
+	}
+
+	return ret;
+}
+
+/* is this IRQ for ADSP ? - we only care about IPC here */
+irqreturn_t hda_dsp_ipc_irq_handler(int irq, void *context)
+{
+	struct snd_sof_dev *sdev = context;
+	int ret = IRQ_NONE;
+	u32 irq_status;
+
+	spin_lock(&sdev->hw_lock);
+
+	/* store status */
+	irq_status = snd_sof_dsp_read(sdev, HDA_DSP_BAR, HDA_DSP_REG_ADSPIS);
+	dev_vdbg(sdev->dev, "irq handler: irq_status:0x%x\n", irq_status);
+
+	/* invalid message ? */
+	if (irq_status == 0xffffffff)
+		goto out;
+
+	/* IPC message ? */
+	if (irq_status & HDA_DSP_ADSPIS_IPC) {
+		/* disable IPC interrupt */
+		snd_sof_dsp_update_bits_unlocked(sdev, HDA_DSP_BAR,
+						 HDA_DSP_REG_ADSPIC,
+						 HDA_DSP_ADSPIC_IPC, 0);
+		ret = IRQ_WAKE_THREAD;
+	}
+
+out:
+	spin_unlock(&sdev->hw_lock);
+	return ret;
+}
+
+/* IPC Firmware ready */
+
+static void ipc_get_windows(struct snd_sof_dev *sdev)
+{
+	struct sof_ipc_window_elem *elem;
+	u32 outbox_offset = 0;
+	u32 stream_offset = 0;
+	u32 inbox_offset = 0;
+	u32 outbox_size = 0;
+	u32 stream_size = 0;
+	u32 inbox_size = 0;
+	int i;
+
+	if (!sdev->info_window) {
+		dev_err(sdev->dev, "error: have no window info\n");
+		return;
+	}
+
+	for (i = 0; i < sdev->info_window->num_windows; i++) {
+		elem = &sdev->info_window->window[i];
+
+		switch (elem->type) {
+		case SOF_IPC_REGION_UPBOX:
+			inbox_offset =
+				elem->offset + SRAM_WINDOW_OFFSET(elem->id);
+			inbox_size = elem->size;
+			snd_sof_debugfs_io_item(sdev,
+						sdev->bar[HDA_DSP_BAR] +
+						inbox_offset,
+						elem->size, "inbox",
+						SOF_DEBUGFS_ACCESS_D0_ONLY);
+			break;
+		case SOF_IPC_REGION_DOWNBOX:
+			outbox_offset =
+				elem->offset + SRAM_WINDOW_OFFSET(elem->id);
+			outbox_size = elem->size;
+			snd_sof_debugfs_io_item(sdev,
+						sdev->bar[HDA_DSP_BAR] +
+						outbox_offset,
+						elem->size, "outbox",
+						SOF_DEBUGFS_ACCESS_D0_ONLY);
+			break;
+		case SOF_IPC_REGION_TRACE:
+			snd_sof_debugfs_io_item(sdev,
+						sdev->bar[HDA_DSP_BAR] +
+						elem->offset +
+						SRAM_WINDOW_OFFSET
+						(elem->id),
+						elem->size, "etrace",
+						SOF_DEBUGFS_ACCESS_D0_ONLY);
+			break;
+		case SOF_IPC_REGION_DEBUG:
+			snd_sof_debugfs_io_item(sdev,
+						sdev->bar[HDA_DSP_BAR] +
+						elem->offset +
+						SRAM_WINDOW_OFFSET
+						(elem->id),
+						elem->size, "debug",
+						SOF_DEBUGFS_ACCESS_D0_ONLY);
+			break;
+		case SOF_IPC_REGION_STREAM:
+			stream_offset =
+				elem->offset + SRAM_WINDOW_OFFSET(elem->id);
+			stream_size = elem->size;
+			snd_sof_debugfs_io_item(sdev,
+						sdev->bar[HDA_DSP_BAR] +
+						elem->offset +
+						SRAM_WINDOW_OFFSET
+						(elem->id),
+						elem->size, "stream",
+						SOF_DEBUGFS_ACCESS_D0_ONLY);
+			break;
+		case SOF_IPC_REGION_REGS:
+			snd_sof_debugfs_io_item(sdev,
+						sdev->bar[HDA_DSP_BAR] +
+						elem->offset +
+						SRAM_WINDOW_OFFSET
+						(elem->id),
+						elem->size, "regs",
+						SOF_DEBUGFS_ACCESS_D0_ONLY);
+			break;
+		case SOF_IPC_REGION_EXCEPTION:
+			sdev->dsp_oops_offset = elem->offset +
+						SRAM_WINDOW_OFFSET(elem->id);
+			snd_sof_debugfs_io_item(sdev,
+						sdev->bar[HDA_DSP_BAR] +
+						elem->offset +
+						SRAM_WINDOW_OFFSET
+						(elem->id),
+						elem->size, "exception",
+						SOF_DEBUGFS_ACCESS_D0_ONLY);
+			break;
+		default:
+			dev_err(sdev->dev, "error: get illegal window info\n");
+			return;
+		}
+	}
+
+	if (outbox_size == 0 || inbox_size == 0) {
+		dev_err(sdev->dev, "error: get illegal mailbox window\n");
+		return;
+	}
+
+	snd_sof_dsp_mailbox_init(sdev, inbox_offset, inbox_size,
+				 outbox_offset, outbox_size);
+	sdev->stream_box.offset = stream_offset;
+	sdev->stream_box.size = stream_size;
+
+	dev_dbg(sdev->dev, " mailbox upstream 0x%x - size 0x%x\n",
+		inbox_offset, inbox_size);
+	dev_dbg(sdev->dev, " mailbox downstream 0x%x - size 0x%x\n",
+		outbox_offset, outbox_size);
+	dev_dbg(sdev->dev, " stream region 0x%x - size 0x%x\n",
+		stream_offset, stream_size);
+}
+
+/* check for ABI compatibility and create memory windows on first boot */
+int hda_dsp_ipc_fw_ready(struct snd_sof_dev *sdev, u32 msg_id)
+{
+	struct sof_ipc_fw_ready *fw_ready = &sdev->fw_ready;
+	u32 offset;
+	int ret;
+
+	/* mailbox must be on 4k boundary */
+	offset = HDA_DSP_MBOX_UPLINK_OFFSET;
+
+	dev_dbg(sdev->dev, "ipc: DSP is ready 0x%8.8x offset 0x%x\n",
+		msg_id, offset);
+
+	/* no need to re-check version/ABI for subsequent boots */
+	if (!sdev->first_boot)
+		return 0;
+
+	/* copy data from the DSP FW ready offset */
+	sof_block_read(sdev, sdev->mmio_bar, offset, fw_ready,
+		       sizeof(*fw_ready));
+
+	/* make sure ABI version is compatible */
+	ret = snd_sof_ipc_valid(sdev);
+	if (ret < 0)
+		return ret;
+
+	/* now check for extended data */
+	snd_sof_fw_parse_ext_data(sdev, sdev->mmio_bar,
+				  HDA_DSP_MBOX_UPLINK_OFFSET +
+				  sizeof(struct sof_ipc_fw_ready));
+
+	ipc_get_windows(sdev);
+
+	return 0;
+}
+
+void hda_ipc_msg_data(struct snd_sof_dev *sdev,
+		      struct snd_pcm_substream *substream,
+		      void *p, size_t sz)
+{
+	if (!substream || !sdev->stream_box.size) {
+		sof_mailbox_read(sdev, sdev->dsp_box.offset, p, sz);
+	} else {
+		struct hdac_stream *hstream = substream->runtime->private_data;
+		struct sof_intel_hda_stream *hda_stream;
+
+		hda_stream = container_of(hstream,
+					  struct sof_intel_hda_stream,
+					  hda_stream.hstream);
+
+		/* The stream might already be closed */
+		if (hstream)
+			sof_mailbox_read(sdev, hda_stream->stream.posn_offset,
+					 p, sz);
+	}
+}
+
+int hda_ipc_pcm_params(struct snd_sof_dev *sdev,
+		       struct snd_pcm_substream *substream,
+		       const struct sof_ipc_pcm_params_reply *reply)
+{
+	struct hdac_stream *hstream = substream->runtime->private_data;
+	struct sof_intel_hda_stream *hda_stream;
+	/* validate offset */
+	size_t posn_offset = reply->posn_offset;
+
+	hda_stream = container_of(hstream, struct sof_intel_hda_stream,
+				  hda_stream.hstream);
+
+	/* check for unaligned offset or overflow */
+	if (posn_offset > sdev->stream_box.size ||
+	    posn_offset % sizeof(struct sof_ipc_stream_posn) != 0)
+		return -EINVAL;
+
+	hda_stream->stream.posn_offset = sdev->stream_box.offset + posn_offset;
+
+	dev_dbg(sdev->dev, "pcm: stream dir %d, posn mailbox offset is %zu",
+		substream->stream, hda_stream->stream.posn_offset);
+
+	return 0;
+}
diff --git a/sound/soc/sof/intel/hda-loader.c b/sound/soc/sof/intel/hda-loader.c
new file mode 100644
index 000000000000..6427f0b3a2f1
--- /dev/null
+++ b/sound/soc/sof/intel/hda-loader.c
@@ -0,0 +1,382 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license.  When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2018 Intel Corporation. All rights reserved.
+//
+// Authors: Liam Girdwood <liam.r.girdwood@linux.intel.com>
+//	    Ranjani Sridharan <ranjani.sridharan@linux.intel.com>
+//	    Rander Wang <rander.wang@intel.com>
+//          Keyon Jie <yang.jie@linux.intel.com>
+//
+
+/*
+ * Hardware interface for HDA DSP code loader
+ */
+
+#include <linux/firmware.h>
+#include <sound/hdaudio_ext.h>
+#include <sound/sof.h>
+#include "../ops.h"
+#include "hda.h"
+
+#define HDA_FW_BOOT_ATTEMPTS	3
+
+static int cl_stream_prepare(struct snd_sof_dev *sdev, unsigned int format,
+			     unsigned int size, struct snd_dma_buffer *dmab,
+			     int direction)
+{
+	struct hdac_ext_stream *dsp_stream;
+	struct hdac_stream *hstream;
+	struct pci_dev *pci = to_pci_dev(sdev->dev);
+	int ret;
+
+	if (direction != SNDRV_PCM_STREAM_PLAYBACK) {
+		dev_err(sdev->dev, "error: code loading DMA is playback only\n");
+		return -EINVAL;
+	}
+
+	dsp_stream = hda_dsp_stream_get(sdev, direction);
+
+	if (!dsp_stream) {
+		dev_err(sdev->dev, "error: no stream available\n");
+		return -ENODEV;
+	}
+	hstream = &dsp_stream->hstream;
+
+	/* allocate DMA buffer */
+	ret = snd_dma_alloc_pages(SNDRV_DMA_TYPE_DEV_SG, &pci->dev, size, dmab);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: memory alloc failed: %x\n", ret);
+		goto error;
+	}
+
+	hstream->period_bytes = 0;/* initialize period_bytes */
+	hstream->format_val = format;
+	hstream->bufsize = size;
+
+	ret = hda_dsp_stream_hw_params(sdev, dsp_stream, dmab, NULL);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: hdac prepare failed: %x\n", ret);
+		goto error;
+	}
+
+	hda_dsp_stream_spib_config(sdev, dsp_stream, HDA_DSP_SPIB_ENABLE, size);
+
+	return hstream->stream_tag;
+
+error:
+	hda_dsp_stream_put(sdev, direction, hstream->stream_tag);
+	snd_dma_free_pages(dmab);
+	return ret;
+}
+
+/*
+ * first boot sequence has some extra steps. core 0 waits for power
+ * status on core 1, so power up core 1 also momentarily, keep it in
+ * reset/stall and then turn it off
+ */
+static int cl_dsp_init(struct snd_sof_dev *sdev, const void *fwdata,
+		       u32 fwsize, int stream_tag)
+{
+	struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata;
+	const struct sof_intel_dsp_desc *chip = hda->desc;
+	unsigned int status;
+	int ret;
+	int i;
+
+	/* step 1: power up corex */
+	ret = hda_dsp_core_power_up(sdev, chip->cores_mask);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: dsp core 0/1 power up failed\n");
+		goto err;
+	}
+
+	/* DSP is powered up, set all SSPs to slave mode */
+	for (i = 0; i < chip->ssp_count; i++) {
+		snd_sof_dsp_update_bits_unlocked(sdev, HDA_DSP_BAR,
+						 chip->ssp_base_offset
+						 + i * SSP_DEV_MEM_SIZE
+						 + SSP_SSC1_OFFSET,
+						 SSP_SET_SLAVE,
+						 SSP_SET_SLAVE);
+	}
+
+	/* step 2: purge FW request */
+	snd_sof_dsp_write(sdev, HDA_DSP_BAR, chip->ipc_req,
+			  chip->ipc_req_mask | (HDA_DSP_IPC_PURGE_FW |
+			  ((stream_tag - 1) << 9)));
+
+	/* step 3: unset core 0 reset state & unstall/run core 0 */
+	ret = hda_dsp_core_run(sdev, HDA_DSP_CORE_MASK(0));
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: dsp core start failed %d\n", ret);
+		ret = -EIO;
+		goto err;
+	}
+
+	/* step 4: wait for IPC DONE bit from ROM */
+	ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR,
+					    chip->ipc_ack, status,
+					    ((status & chip->ipc_ack_mask)
+						    == chip->ipc_ack_mask),
+					    HDA_DSP_REG_POLL_INTERVAL_US,
+					    HDA_DSP_INIT_TIMEOUT_US);
+
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: waiting for HIPCIE done\n");
+		goto err;
+	}
+
+	/* step 5: power down corex */
+	ret = hda_dsp_core_power_down(sdev,
+				  chip->cores_mask & ~(HDA_DSP_CORE_MASK(0)));
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: dsp core x power down failed\n");
+		goto err;
+	}
+
+	/* step 6: enable IPC interrupts */
+	hda_dsp_ipc_int_enable(sdev);
+
+	/* step 7: wait for ROM init */
+	ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR,
+					HDA_DSP_SRAM_REG_ROM_STATUS, status,
+					((status & HDA_DSP_ROM_STS_MASK)
+						== HDA_DSP_ROM_INIT),
+					HDA_DSP_REG_POLL_INTERVAL_US,
+					chip->rom_init_timeout *
+					USEC_PER_MSEC);
+	if (!ret)
+		return 0;
+
+err:
+	hda_dsp_dump(sdev, SOF_DBG_REGS | SOF_DBG_PCI | SOF_DBG_MBOX);
+	hda_dsp_core_reset_power_down(sdev, chip->cores_mask);
+
+	return ret;
+}
+
+static int cl_trigger(struct snd_sof_dev *sdev,
+		      struct hdac_ext_stream *stream, int cmd)
+{
+	struct hdac_stream *hstream = &stream->hstream;
+	int sd_offset = SOF_STREAM_SD_OFFSET(hstream);
+
+	/* code loader is special case that reuses stream ops */
+	switch (cmd) {
+	case SNDRV_PCM_TRIGGER_START:
+		wait_event_timeout(sdev->waitq, !sdev->code_loading,
+				   HDA_DSP_CL_TRIGGER_TIMEOUT);
+
+		snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR, SOF_HDA_INTCTL,
+					1 << hstream->index,
+					1 << hstream->index);
+
+		snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR,
+					sd_offset,
+					SOF_HDA_SD_CTL_DMA_START |
+					SOF_HDA_CL_DMA_SD_INT_MASK,
+					SOF_HDA_SD_CTL_DMA_START |
+					SOF_HDA_CL_DMA_SD_INT_MASK);
+
+		hstream->running = true;
+		return 0;
+	default:
+		return hda_dsp_stream_trigger(sdev, stream, cmd);
+	}
+}
+
+static struct hdac_ext_stream *get_stream_with_tag(struct snd_sof_dev *sdev,
+						   int tag)
+{
+	struct hdac_bus *bus = sof_to_bus(sdev);
+	struct hdac_stream *s;
+
+	/* get stream with tag */
+	list_for_each_entry(s, &bus->stream_list, list) {
+		if (s->direction == SNDRV_PCM_STREAM_PLAYBACK &&
+		    s->stream_tag == tag) {
+			return stream_to_hdac_ext_stream(s);
+		}
+	}
+
+	return NULL;
+}
+
+static int cl_cleanup(struct snd_sof_dev *sdev, struct snd_dma_buffer *dmab,
+		      struct hdac_ext_stream *stream)
+{
+	struct hdac_stream *hstream = &stream->hstream;
+	int sd_offset = SOF_STREAM_SD_OFFSET(hstream);
+	int ret;
+
+	ret = hda_dsp_stream_spib_config(sdev, stream, HDA_DSP_SPIB_DISABLE, 0);
+
+	hda_dsp_stream_put(sdev, SNDRV_PCM_STREAM_PLAYBACK,
+			   hstream->stream_tag);
+	hstream->running = 0;
+	hstream->substream = NULL;
+
+	/* reset BDL address */
+	snd_sof_dsp_write(sdev, HDA_DSP_HDA_BAR,
+			  sd_offset + SOF_HDA_ADSP_REG_CL_SD_BDLPL, 0);
+	snd_sof_dsp_write(sdev, HDA_DSP_HDA_BAR,
+			  sd_offset + SOF_HDA_ADSP_REG_CL_SD_BDLPU, 0);
+
+	snd_sof_dsp_write(sdev, HDA_DSP_HDA_BAR, sd_offset, 0);
+	snd_dma_free_pages(dmab);
+	dmab->area = NULL;
+	hstream->bufsize = 0;
+	hstream->format_val = 0;
+
+	return ret;
+}
+
+static int cl_copy_fw(struct snd_sof_dev *sdev, struct hdac_ext_stream *stream)
+{
+	unsigned int reg;
+	int ret, status;
+
+	ret = cl_trigger(sdev, stream, SNDRV_PCM_TRIGGER_START);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: DMA trigger start failed\n");
+		return ret;
+	}
+
+	status = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR,
+					HDA_DSP_SRAM_REG_ROM_STATUS, reg,
+					((reg & HDA_DSP_ROM_STS_MASK)
+						== HDA_DSP_ROM_FW_ENTERED),
+					HDA_DSP_REG_POLL_INTERVAL_US,
+					HDA_DSP_BASEFW_TIMEOUT_US);
+
+	ret = cl_trigger(sdev, stream, SNDRV_PCM_TRIGGER_STOP);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: DMA trigger stop failed\n");
+		return ret;
+	}
+
+	return status;
+}
+
+int hda_dsp_cl_boot_firmware(struct snd_sof_dev *sdev)
+{
+	struct snd_sof_pdata *plat_data = sdev->pdata;
+	const struct sof_dev_desc *desc = plat_data->desc;
+	const struct sof_intel_dsp_desc *chip_info;
+	struct hdac_ext_stream *stream;
+	struct firmware stripped_firmware;
+	int ret, ret1, tag, i;
+
+	chip_info = desc->chip_info;
+
+	stripped_firmware.data = plat_data->fw->data;
+	stripped_firmware.size = plat_data->fw->size;
+
+	/* init for booting wait */
+	init_waitqueue_head(&sdev->boot_wait);
+	sdev->boot_complete = false;
+
+	/* prepare DMA for code loader stream */
+	tag = cl_stream_prepare(sdev, 0x40, stripped_firmware.size,
+				&sdev->dmab, SNDRV_PCM_STREAM_PLAYBACK);
+
+	if (tag < 0) {
+		dev_err(sdev->dev, "error: dma prepare for fw loading err: %x\n",
+			tag);
+		return tag;
+	}
+
+	/* get stream with tag */
+	stream = get_stream_with_tag(sdev, tag);
+	if (!stream) {
+		dev_err(sdev->dev,
+			"error: could not get stream with stream tag %d\n",
+			tag);
+		ret = -ENODEV;
+		goto err;
+	}
+
+	memcpy(sdev->dmab.area, stripped_firmware.data,
+	       stripped_firmware.size);
+
+	/* try ROM init a few times before giving up */
+	for (i = 0; i < HDA_FW_BOOT_ATTEMPTS; i++) {
+		ret = cl_dsp_init(sdev, stripped_firmware.data,
+				  stripped_firmware.size, tag);
+
+		/* don't retry anymore if successful */
+		if (!ret)
+			break;
+
+		dev_err(sdev->dev, "error: Error code=0x%x: FW status=0x%x\n",
+			snd_sof_dsp_read(sdev, HDA_DSP_BAR,
+					 HDA_DSP_SRAM_REG_ROM_ERROR),
+			snd_sof_dsp_read(sdev, HDA_DSP_BAR,
+					 HDA_DSP_SRAM_REG_ROM_STATUS));
+		dev_err(sdev->dev, "error: iteration %d of Core En/ROM load failed: %d\n",
+			i, ret);
+	}
+
+	if (i == HDA_FW_BOOT_ATTEMPTS) {
+		dev_err(sdev->dev, "error: dsp init failed after %d attempts with err: %d\n",
+			i, ret);
+		goto cleanup;
+	}
+
+	/*
+	 * at this point DSP ROM has been initialized and
+	 * should be ready for code loading and firmware boot
+	 */
+	ret = cl_copy_fw(sdev, stream);
+	if (!ret)
+		dev_dbg(sdev->dev, "Firmware download successful, booting...\n");
+	else
+		dev_err(sdev->dev, "error: load fw failed ret: %d\n", ret);
+
+cleanup:
+	/*
+	 * Perform codeloader stream cleanup.
+	 * This should be done even if firmware loading fails.
+	 */
+	ret1 = cl_cleanup(sdev, &sdev->dmab, stream);
+	if (ret1 < 0) {
+		dev_err(sdev->dev, "error: Code loader DSP cleanup failed\n");
+
+		/* set return value to indicate cleanup failure */
+		ret = ret1;
+	}
+
+	/*
+	 * return master core id if both fw copy
+	 * and stream clean up are successful
+	 */
+	if (!ret)
+		return chip_info->init_core_mask;
+
+	/* dump dsp registers and disable DSP upon error */
+err:
+	hda_dsp_dump(sdev, SOF_DBG_REGS | SOF_DBG_PCI | SOF_DBG_MBOX);
+
+	/* disable DSP */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_PP_BAR,
+				SOF_HDA_REG_PP_PPCTL,
+				SOF_HDA_PPCTL_GPROCEN, 0);
+	return ret;
+}
+
+/* pre fw run operations */
+int hda_dsp_pre_fw_run(struct snd_sof_dev *sdev)
+{
+	/* disable clock gating and power gating */
+	return hda_dsp_ctrl_clock_power_gating(sdev, false);
+}
+
+/* post fw run operations */
+int hda_dsp_post_fw_run(struct snd_sof_dev *sdev)
+{
+	/* re-enable clock gating and power gating */
+	return hda_dsp_ctrl_clock_power_gating(sdev, true);
+}
diff --git a/sound/soc/sof/intel/hda-pcm.c b/sound/soc/sof/intel/hda-pcm.c
new file mode 100644
index 000000000000..9b730f183529
--- /dev/null
+++ b/sound/soc/sof/intel/hda-pcm.c
@@ -0,0 +1,239 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license.  When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2018 Intel Corporation. All rights reserved.
+//
+// Authors: Liam Girdwood <liam.r.girdwood@linux.intel.com>
+//	    Ranjani Sridharan <ranjani.sridharan@linux.intel.com>
+//	    Rander Wang <rander.wang@intel.com>
+//          Keyon Jie <yang.jie@linux.intel.com>
+//
+
+/*
+ * Hardware interface for generic Intel audio DSP HDA IP
+ */
+
+#include <sound/hda_register.h>
+#include <sound/pcm_params.h>
+#include "../ops.h"
+#include "hda.h"
+
+#define SDnFMT_BASE(x)	((x) << 14)
+#define SDnFMT_MULT(x)	(((x) - 1) << 11)
+#define SDnFMT_DIV(x)	(((x) - 1) << 8)
+#define SDnFMT_BITS(x)	((x) << 4)
+#define SDnFMT_CHAN(x)	((x) << 0)
+
+static inline u32 get_mult_div(struct snd_sof_dev *sdev, int rate)
+{
+	switch (rate) {
+	case 8000:
+		return SDnFMT_DIV(6);
+	case 9600:
+		return SDnFMT_DIV(5);
+	case 11025:
+		return SDnFMT_BASE(1) | SDnFMT_DIV(4);
+	case 16000:
+		return SDnFMT_DIV(3);
+	case 22050:
+		return SDnFMT_BASE(1) | SDnFMT_DIV(2);
+	case 32000:
+		return SDnFMT_DIV(3) | SDnFMT_MULT(2);
+	case 44100:
+		return SDnFMT_BASE(1);
+	case 48000:
+		return 0;
+	case 88200:
+		return SDnFMT_BASE(1) | SDnFMT_MULT(2);
+	case 96000:
+		return SDnFMT_MULT(2);
+	case 176400:
+		return SDnFMT_BASE(1) | SDnFMT_MULT(4);
+	case 192000:
+		return SDnFMT_MULT(4);
+	default:
+		dev_warn(sdev->dev, "can't find div rate %d using 48kHz\n",
+			 rate);
+		return 0; /* use 48KHz if not found */
+	}
+};
+
+static inline u32 get_bits(struct snd_sof_dev *sdev, int sample_bits)
+{
+	switch (sample_bits) {
+	case 8:
+		return SDnFMT_BITS(0);
+	case 16:
+		return SDnFMT_BITS(1);
+	case 20:
+		return SDnFMT_BITS(2);
+	case 24:
+		return SDnFMT_BITS(3);
+	case 32:
+		return SDnFMT_BITS(4);
+	default:
+		dev_warn(sdev->dev, "can't find %d bits using 16bit\n",
+			 sample_bits);
+		return SDnFMT_BITS(1); /* use 16bits format if not found */
+	}
+};
+
+int hda_dsp_pcm_hw_params(struct snd_sof_dev *sdev,
+			  struct snd_pcm_substream *substream,
+			  struct snd_pcm_hw_params *params,
+			  struct sof_ipc_stream_params *ipc_params)
+{
+	struct hdac_stream *hstream = substream->runtime->private_data;
+	struct hdac_ext_stream *stream = stream_to_hdac_ext_stream(hstream);
+	struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata;
+	struct snd_dma_buffer *dmab;
+	int ret;
+	u32 size, rate, bits;
+
+	size = params_buffer_bytes(params);
+	rate = get_mult_div(sdev, params_rate(params));
+	bits = get_bits(sdev, params_width(params));
+
+	hstream->substream = substream;
+
+	dmab = substream->runtime->dma_buffer_p;
+
+	hstream->format_val = rate | bits | (params_channels(params) - 1);
+	hstream->bufsize = size;
+	hstream->period_bytes = params_period_bytes(params);
+	hstream->no_period_wakeup  =
+			(params->info & SNDRV_PCM_INFO_NO_PERIOD_WAKEUP) &&
+			(params->flags & SNDRV_PCM_HW_PARAMS_NO_PERIOD_WAKEUP);
+
+	ret = hda_dsp_stream_hw_params(sdev, stream, dmab, params);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: hdac prepare failed: %x\n", ret);
+		return ret;
+	}
+
+	/* disable SPIB, to enable buffer wrap for stream */
+	hda_dsp_stream_spib_config(sdev, stream, HDA_DSP_SPIB_DISABLE, 0);
+
+	/* set host_period_bytes to 0 if no IPC position */
+	if (hda && hda->no_ipc_position)
+		ipc_params->host_period_bytes = 0;
+
+	ipc_params->stream_tag = hstream->stream_tag;
+
+	return 0;
+}
+
+int hda_dsp_pcm_trigger(struct snd_sof_dev *sdev,
+			struct snd_pcm_substream *substream, int cmd)
+{
+	struct hdac_stream *hstream = substream->runtime->private_data;
+	struct hdac_ext_stream *stream = stream_to_hdac_ext_stream(hstream);
+
+	return hda_dsp_stream_trigger(sdev, stream, cmd);
+}
+
+snd_pcm_uframes_t hda_dsp_pcm_pointer(struct snd_sof_dev *sdev,
+				      struct snd_pcm_substream *substream)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct hdac_stream *hstream = substream->runtime->private_data;
+	struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata;
+	struct snd_sof_pcm *spcm;
+	snd_pcm_uframes_t pos;
+
+	spcm = snd_sof_find_spcm_dai(sdev, rtd);
+	if (!spcm) {
+		dev_warn_ratelimited(sdev->dev, "warn: can't find PCM with DAI ID %d\n",
+				     rtd->dai_link->id);
+		return 0;
+	}
+
+	if (hda && !hda->no_ipc_position) {
+		/* read position from IPC position */
+		pos = spcm->stream[substream->stream].posn.host_posn;
+		goto found;
+	}
+
+	/*
+	 * DPIB/posbuf position mode:
+	 * For Playback, Use DPIB register from HDA space which
+	 * reflects the actual data transferred.
+	 * For Capture, Use the position buffer for pointer, as DPIB
+	 * is not accurate enough, its update may be completed
+	 * earlier than the data written to DDR.
+	 */
+	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) {
+		pos = snd_sof_dsp_read(sdev, HDA_DSP_HDA_BAR,
+				       AZX_REG_VS_SDXDPIB_XBASE +
+				       (AZX_REG_VS_SDXDPIB_XINTERVAL *
+					hstream->index));
+	} else {
+		/*
+		 * For capture stream, we need more workaround to fix the
+		 * position incorrect issue:
+		 *
+		 * 1. Wait at least 20us before reading position buffer after
+		 * the interrupt generated(IOC), to make sure position update
+		 * happens on frame boundary i.e. 20.833uSec for 48KHz.
+		 * 2. Perform a dummy Read to DPIB register to flush DMA
+		 * position value.
+		 * 3. Read the DMA Position from posbuf. Now the readback
+		 * value should be >= period boundary.
+		 */
+		usleep_range(20, 21);
+		snd_sof_dsp_read(sdev, HDA_DSP_HDA_BAR,
+				 AZX_REG_VS_SDXDPIB_XBASE +
+				 (AZX_REG_VS_SDXDPIB_XINTERVAL *
+				  hstream->index));
+		pos = snd_hdac_stream_get_pos_posbuf(hstream);
+	}
+
+	if (pos >= hstream->bufsize)
+		pos = 0;
+
+found:
+	pos = bytes_to_frames(substream->runtime, pos);
+
+	dev_vdbg(sdev->dev, "PCM: stream %d dir %d position %lu\n",
+		 hstream->index, substream->stream, pos);
+	return pos;
+}
+
+int hda_dsp_pcm_open(struct snd_sof_dev *sdev,
+		     struct snd_pcm_substream *substream)
+{
+	struct hdac_ext_stream *dsp_stream;
+	int direction = substream->stream;
+
+	dsp_stream = hda_dsp_stream_get(sdev, direction);
+
+	if (!dsp_stream) {
+		dev_err(sdev->dev, "error: no stream available\n");
+		return -ENODEV;
+	}
+
+	/* binding pcm substream to hda stream */
+	substream->runtime->private_data = &dsp_stream->hstream;
+	return 0;
+}
+
+int hda_dsp_pcm_close(struct snd_sof_dev *sdev,
+		      struct snd_pcm_substream *substream)
+{
+	struct hdac_stream *hstream = substream->runtime->private_data;
+	int direction = substream->stream;
+	int ret;
+
+	ret = hda_dsp_stream_put(sdev, direction, hstream->stream_tag);
+
+	if (ret) {
+		dev_dbg(sdev->dev, "stream %s not opened!\n", substream->name);
+		return -ENODEV;
+	}
+
+	/* unbinding pcm substream to hda stream */
+	substream->runtime->private_data = NULL;
+	return 0;
+}
diff --git a/sound/soc/sof/intel/hda-stream.c b/sound/soc/sof/intel/hda-stream.c
new file mode 100644
index 000000000000..c92006f89499
--- /dev/null
+++ b/sound/soc/sof/intel/hda-stream.c
@@ -0,0 +1,701 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license.  When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2018 Intel Corporation. All rights reserved.
+//
+// Authors: Liam Girdwood <liam.r.girdwood@linux.intel.com>
+//	    Ranjani Sridharan <ranjani.sridharan@linux.intel.com>
+//	    Rander Wang <rander.wang@intel.com>
+//          Keyon Jie <yang.jie@linux.intel.com>
+//
+
+/*
+ * Hardware interface for generic Intel audio DSP HDA IP
+ */
+
+#include <linux/pm_runtime.h>
+#include <sound/hdaudio_ext.h>
+#include <sound/hda_register.h>
+#include <sound/sof.h>
+#include "../ops.h"
+#include "hda.h"
+
+/*
+ * set up one of BDL entries for a stream
+ */
+static int hda_setup_bdle(struct snd_sof_dev *sdev,
+			  struct snd_dma_buffer *dmab,
+			  struct hdac_stream *stream,
+			  struct sof_intel_dsp_bdl **bdlp,
+			  int offset, int size, int ioc)
+{
+	struct hdac_bus *bus = sof_to_bus(sdev);
+	struct sof_intel_dsp_bdl *bdl = *bdlp;
+
+	while (size > 0) {
+		dma_addr_t addr;
+		int chunk;
+
+		if (stream->frags >= HDA_DSP_MAX_BDL_ENTRIES) {
+			dev_err(sdev->dev, "error: stream frags exceeded\n");
+			return -EINVAL;
+		}
+
+		addr = snd_sgbuf_get_addr(dmab, offset);
+		/* program BDL addr */
+		bdl->addr_l = cpu_to_le32(lower_32_bits(addr));
+		bdl->addr_h = cpu_to_le32(upper_32_bits(addr));
+		/* program BDL size */
+		chunk = snd_sgbuf_get_chunk_size(dmab, offset, size);
+		/* one BDLE should not cross 4K boundary */
+		if (bus->align_bdle_4k) {
+			u32 remain = 0x1000 - (offset & 0xfff);
+
+			if (chunk > remain)
+				chunk = remain;
+		}
+		bdl->size = cpu_to_le32(chunk);
+		/* only program IOC when the whole segment is processed */
+		size -= chunk;
+		bdl->ioc = (size || !ioc) ? 0 : cpu_to_le32(0x01);
+		bdl++;
+		stream->frags++;
+		offset += chunk;
+
+		dev_vdbg(sdev->dev, "bdl, frags:%d, chunk size:0x%x;\n",
+			 stream->frags, chunk);
+	}
+
+	*bdlp = bdl;
+	return offset;
+}
+
+/*
+ * set up Buffer Descriptor List (BDL) for host memory transfer
+ * BDL describes the location of the individual buffers and is little endian.
+ */
+int hda_dsp_stream_setup_bdl(struct snd_sof_dev *sdev,
+			     struct snd_dma_buffer *dmab,
+			     struct hdac_stream *stream)
+{
+	struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata;
+	struct sof_intel_dsp_bdl *bdl;
+	int i, offset, period_bytes, periods;
+	int remain, ioc;
+
+	period_bytes = stream->period_bytes;
+	dev_dbg(sdev->dev, "period_bytes:0x%x\n", period_bytes);
+	if (!period_bytes)
+		period_bytes = stream->bufsize;
+
+	periods = stream->bufsize / period_bytes;
+
+	dev_dbg(sdev->dev, "periods:%d\n", periods);
+
+	remain = stream->bufsize % period_bytes;
+	if (remain)
+		periods++;
+
+	/* program the initial BDL entries */
+	bdl = (struct sof_intel_dsp_bdl *)stream->bdl.area;
+	offset = 0;
+	stream->frags = 0;
+
+	/*
+	 * set IOC if don't use position IPC
+	 * and period_wakeup needed.
+	 */
+	ioc = hda->no_ipc_position ?
+	      !stream->no_period_wakeup : 0;
+
+	for (i = 0; i < periods; i++) {
+		if (i == (periods - 1) && remain)
+			/* set the last small entry */
+			offset = hda_setup_bdle(sdev, dmab,
+						stream, &bdl, offset,
+						remain, 0);
+		else
+			offset = hda_setup_bdle(sdev, dmab,
+						stream, &bdl, offset,
+						period_bytes, ioc);
+	}
+
+	return offset;
+}
+
+int hda_dsp_stream_spib_config(struct snd_sof_dev *sdev,
+			       struct hdac_ext_stream *stream,
+			       int enable, u32 size)
+{
+	struct hdac_stream *hstream = &stream->hstream;
+	u32 mask;
+
+	if (!sdev->bar[HDA_DSP_SPIB_BAR]) {
+		dev_err(sdev->dev, "error: address of spib capability is NULL\n");
+		return -EINVAL;
+	}
+
+	mask = (1 << hstream->index);
+
+	/* enable/disable SPIB for the stream */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_SPIB_BAR,
+				SOF_HDA_ADSP_REG_CL_SPBFIFO_SPBFCCTL, mask,
+				enable << hstream->index);
+
+	/* set the SPIB value */
+	sof_io_write(sdev, stream->spib_addr, size);
+
+	return 0;
+}
+
+/* get next unused stream */
+struct hdac_ext_stream *
+hda_dsp_stream_get(struct snd_sof_dev *sdev, int direction)
+{
+	struct hdac_bus *bus = sof_to_bus(sdev);
+	struct hdac_ext_stream *stream = NULL;
+	struct hdac_stream *s;
+
+	spin_lock_irq(&bus->reg_lock);
+
+	/* get an unused stream */
+	list_for_each_entry(s, &bus->stream_list, list) {
+		if (s->direction == direction && !s->opened) {
+			s->opened = true;
+			stream = stream_to_hdac_ext_stream(s);
+			break;
+		}
+	}
+
+	spin_unlock_irq(&bus->reg_lock);
+
+	/* stream found ? */
+	if (!stream)
+		dev_err(sdev->dev, "error: no free %s streams\n",
+			direction == SNDRV_PCM_STREAM_PLAYBACK ?
+			"playback" : "capture");
+
+	return stream;
+}
+
+/* free a stream */
+int hda_dsp_stream_put(struct snd_sof_dev *sdev, int direction, int stream_tag)
+{
+	struct hdac_bus *bus = sof_to_bus(sdev);
+	struct hdac_stream *s;
+
+	spin_lock_irq(&bus->reg_lock);
+
+	/* find used stream */
+	list_for_each_entry(s, &bus->stream_list, list) {
+		if (s->direction == direction &&
+		    s->opened && s->stream_tag == stream_tag) {
+			s->opened = false;
+			spin_unlock_irq(&bus->reg_lock);
+			return 0;
+		}
+	}
+
+	spin_unlock_irq(&bus->reg_lock);
+
+	dev_dbg(sdev->dev, "stream_tag %d not opened!\n", stream_tag);
+	return -ENODEV;
+}
+
+int hda_dsp_stream_trigger(struct snd_sof_dev *sdev,
+			   struct hdac_ext_stream *stream, int cmd)
+{
+	struct hdac_stream *hstream = &stream->hstream;
+	int sd_offset = SOF_STREAM_SD_OFFSET(hstream);
+
+	/* cmd must be for audio stream */
+	switch (cmd) {
+	case SNDRV_PCM_TRIGGER_RESUME:
+	case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
+	case SNDRV_PCM_TRIGGER_START:
+		snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR, SOF_HDA_INTCTL,
+					1 << hstream->index,
+					1 << hstream->index);
+
+		snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR,
+					sd_offset,
+					SOF_HDA_SD_CTL_DMA_START |
+					SOF_HDA_CL_DMA_SD_INT_MASK,
+					SOF_HDA_SD_CTL_DMA_START |
+					SOF_HDA_CL_DMA_SD_INT_MASK);
+
+		hstream->running = true;
+		break;
+	case SNDRV_PCM_TRIGGER_SUSPEND:
+	case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
+	case SNDRV_PCM_TRIGGER_STOP:
+		snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR,
+					sd_offset,
+					SOF_HDA_SD_CTL_DMA_START |
+					SOF_HDA_CL_DMA_SD_INT_MASK, 0x0);
+
+		snd_sof_dsp_write(sdev, HDA_DSP_HDA_BAR, sd_offset +
+				  SOF_HDA_ADSP_REG_CL_SD_STS,
+				  SOF_HDA_CL_DMA_SD_INT_MASK);
+
+		hstream->running = false;
+		snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR, SOF_HDA_INTCTL,
+					1 << hstream->index, 0x0);
+		break;
+	default:
+		dev_err(sdev->dev, "error: unknown command: %d\n", cmd);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+/*
+ * prepare for common hdac registers settings, for both code loader
+ * and normal stream.
+ */
+int hda_dsp_stream_hw_params(struct snd_sof_dev *sdev,
+			     struct hdac_ext_stream *stream,
+			     struct snd_dma_buffer *dmab,
+			     struct snd_pcm_hw_params *params)
+{
+	struct hdac_bus *bus = sof_to_bus(sdev);
+	struct hdac_stream *hstream = &stream->hstream;
+	int sd_offset = SOF_STREAM_SD_OFFSET(hstream);
+	int ret, timeout = HDA_DSP_STREAM_RESET_TIMEOUT;
+	u32 val, mask;
+
+	if (!stream) {
+		dev_err(sdev->dev, "error: no stream available\n");
+		return -ENODEV;
+	}
+
+	/* decouple host and link DMA */
+	mask = 0x1 << hstream->index;
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_PP_BAR, SOF_HDA_REG_PP_PPCTL,
+				mask, mask);
+
+	if (!dmab) {
+		dev_err(sdev->dev, "error: no dma buffer allocated!\n");
+		return -ENODEV;
+	}
+
+	/* clear stream status */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR, sd_offset,
+				SOF_HDA_CL_DMA_SD_INT_MASK |
+				SOF_HDA_SD_CTL_DMA_START, 0);
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR,
+				sd_offset + SOF_HDA_ADSP_REG_CL_SD_STS,
+				SOF_HDA_CL_DMA_SD_INT_MASK,
+				SOF_HDA_CL_DMA_SD_INT_MASK);
+
+	/* stream reset */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR, sd_offset, 0x1,
+				0x1);
+	udelay(3);
+	do {
+		val = snd_sof_dsp_read(sdev, HDA_DSP_HDA_BAR,
+				       sd_offset);
+		if (val & 0x1)
+			break;
+	} while (--timeout);
+	if (timeout == 0) {
+		dev_err(sdev->dev, "error: stream reset failed\n");
+		return -ETIMEDOUT;
+	}
+
+	timeout = HDA_DSP_STREAM_RESET_TIMEOUT;
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR, sd_offset, 0x1,
+				0x0);
+
+	/* wait for hardware to report that stream is out of reset */
+	udelay(3);
+	do {
+		val = snd_sof_dsp_read(sdev, HDA_DSP_HDA_BAR,
+				       sd_offset);
+		if ((val & 0x1) == 0)
+			break;
+	} while (--timeout);
+	if (timeout == 0) {
+		dev_err(sdev->dev, "error: timeout waiting for stream reset\n");
+		return -ETIMEDOUT;
+	}
+
+	if (hstream->posbuf)
+		*hstream->posbuf = 0;
+
+	/* reset BDL address */
+	snd_sof_dsp_write(sdev, HDA_DSP_HDA_BAR,
+			  sd_offset + SOF_HDA_ADSP_REG_CL_SD_BDLPL,
+			  0x0);
+	snd_sof_dsp_write(sdev, HDA_DSP_HDA_BAR,
+			  sd_offset + SOF_HDA_ADSP_REG_CL_SD_BDLPU,
+			  0x0);
+
+	/* clear stream status */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR, sd_offset,
+				SOF_HDA_CL_DMA_SD_INT_MASK |
+				SOF_HDA_SD_CTL_DMA_START, 0);
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR,
+				sd_offset + SOF_HDA_ADSP_REG_CL_SD_STS,
+				SOF_HDA_CL_DMA_SD_INT_MASK,
+				SOF_HDA_CL_DMA_SD_INT_MASK);
+
+	hstream->frags = 0;
+
+	ret = hda_dsp_stream_setup_bdl(sdev, dmab, hstream);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: set up of BDL failed\n");
+		return ret;
+	}
+
+	/* program stream tag to set up stream descriptor for DMA */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR, sd_offset,
+				SOF_HDA_CL_SD_CTL_STREAM_TAG_MASK,
+				hstream->stream_tag <<
+				SOF_HDA_CL_SD_CTL_STREAM_TAG_SHIFT);
+
+	/* program cyclic buffer length */
+	snd_sof_dsp_write(sdev, HDA_DSP_HDA_BAR,
+			  sd_offset + SOF_HDA_ADSP_REG_CL_SD_CBL,
+			  hstream->bufsize);
+
+	/*
+	 * Recommended hardware programming sequence for HDAudio DMA format
+	 *
+	 * 1. Put DMA into coupled mode by clearing PPCTL.PROCEN bit
+	 *    for corresponding stream index before the time of writing
+	 *    format to SDxFMT register.
+	 * 2. Write SDxFMT
+	 * 3. Set PPCTL.PROCEN bit for corresponding stream index to
+	 *    enable decoupled mode
+	 */
+
+	/* couple host and link DMA, disable DSP features */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_PP_BAR, SOF_HDA_REG_PP_PPCTL,
+				mask, 0);
+
+	/* program stream format */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR,
+				sd_offset +
+				SOF_HDA_ADSP_REG_CL_SD_FORMAT,
+				0xffff, hstream->format_val);
+
+	/* decouple host and link DMA, enable DSP features */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_PP_BAR, SOF_HDA_REG_PP_PPCTL,
+				mask, mask);
+
+	/* program last valid index */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR,
+				sd_offset + SOF_HDA_ADSP_REG_CL_SD_LVI,
+				0xffff, (hstream->frags - 1));
+
+	/* program BDL address */
+	snd_sof_dsp_write(sdev, HDA_DSP_HDA_BAR,
+			  sd_offset + SOF_HDA_ADSP_REG_CL_SD_BDLPL,
+			  (u32)hstream->bdl.addr);
+	snd_sof_dsp_write(sdev, HDA_DSP_HDA_BAR,
+			  sd_offset + SOF_HDA_ADSP_REG_CL_SD_BDLPU,
+			  upper_32_bits(hstream->bdl.addr));
+
+	/* enable position buffer */
+	if (!(snd_sof_dsp_read(sdev, HDA_DSP_HDA_BAR, SOF_HDA_ADSP_DPLBASE)
+				& SOF_HDA_ADSP_DPLBASE_ENABLE)) {
+		snd_sof_dsp_write(sdev, HDA_DSP_HDA_BAR, SOF_HDA_ADSP_DPUBASE,
+				  upper_32_bits(bus->posbuf.addr));
+		snd_sof_dsp_write(sdev, HDA_DSP_HDA_BAR, SOF_HDA_ADSP_DPLBASE,
+				  (u32)bus->posbuf.addr |
+				  SOF_HDA_ADSP_DPLBASE_ENABLE);
+	}
+
+	/* set interrupt enable bits */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR, sd_offset,
+				SOF_HDA_CL_DMA_SD_INT_MASK,
+				SOF_HDA_CL_DMA_SD_INT_MASK);
+
+	/* read FIFO size */
+	if (hstream->direction == SNDRV_PCM_STREAM_PLAYBACK) {
+		hstream->fifo_size =
+			snd_sof_dsp_read(sdev, HDA_DSP_HDA_BAR,
+					 sd_offset +
+					 SOF_HDA_ADSP_REG_CL_SD_FIFOSIZE);
+		hstream->fifo_size &= 0xffff;
+		hstream->fifo_size += 1;
+	} else {
+		hstream->fifo_size = 0;
+	}
+
+	return ret;
+}
+
+irqreturn_t hda_dsp_stream_interrupt(int irq, void *context)
+{
+	struct hdac_bus *bus = context;
+	struct sof_intel_hda_dev *sof_hda = bus_to_sof_hda(bus);
+	u32 stream_mask;
+	u32 status;
+
+	if (!pm_runtime_active(bus->dev))
+		return IRQ_NONE;
+
+	spin_lock(&bus->reg_lock);
+
+	status = snd_hdac_chip_readl(bus, INTSTS);
+	stream_mask = GENMASK(sof_hda->stream_max - 1, 0) | AZX_INT_CTRL_EN;
+
+	/* Not stream interrupt or register inaccessible, ignore it.*/
+	if (!(status & stream_mask) || status == 0xffffffff) {
+		spin_unlock(&bus->reg_lock);
+		return IRQ_NONE;
+	}
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
+	/* clear rirb int */
+	status = snd_hdac_chip_readb(bus, RIRBSTS);
+	if (status & RIRB_INT_MASK) {
+		if (status & RIRB_INT_RESPONSE)
+			snd_hdac_bus_update_rirb(bus);
+		snd_hdac_chip_writeb(bus, RIRBSTS, RIRB_INT_MASK);
+	}
+#endif
+
+	spin_unlock(&bus->reg_lock);
+
+	return snd_hdac_chip_readl(bus, INTSTS) ? IRQ_WAKE_THREAD : IRQ_HANDLED;
+}
+
+irqreturn_t hda_dsp_stream_threaded_handler(int irq, void *context)
+{
+	struct hdac_bus *bus = context;
+	struct sof_intel_hda_dev *sof_hda = bus_to_sof_hda(bus);
+	u32 status = snd_hdac_chip_readl(bus, INTSTS);
+	struct hdac_stream *s;
+	u32 sd_status;
+
+	/* check streams */
+	list_for_each_entry(s, &bus->stream_list, list) {
+		if (status & (1 << s->index) && s->opened) {
+			sd_status = snd_hdac_stream_readb(s, SD_STS);
+
+			dev_vdbg(bus->dev, "stream %d status 0x%x\n",
+				 s->index, sd_status);
+
+			snd_hdac_stream_writeb(s, SD_STS, SD_INT_MASK);
+
+			if (!s->substream ||
+			    !s->running ||
+			    (sd_status & SOF_HDA_CL_DMA_SD_INT_COMPLETE) == 0)
+				continue;
+
+			/* Inform ALSA only in case not do that with IPC */
+			if (sof_hda->no_ipc_position)
+				snd_sof_pcm_period_elapsed(s->substream);
+
+		}
+	}
+
+	return IRQ_HANDLED;
+}
+
+int hda_dsp_stream_init(struct snd_sof_dev *sdev)
+{
+	struct hdac_bus *bus = sof_to_bus(sdev);
+	struct hdac_ext_stream *stream;
+	struct hdac_stream *hstream;
+	struct pci_dev *pci = to_pci_dev(sdev->dev);
+	struct sof_intel_hda_dev *sof_hda = bus_to_sof_hda(bus);
+	int sd_offset;
+	int i, num_playback, num_capture, num_total, ret;
+	u32 gcap;
+
+	gcap = snd_sof_dsp_read(sdev, HDA_DSP_HDA_BAR, SOF_HDA_GCAP);
+	dev_dbg(sdev->dev, "hda global caps = 0x%x\n", gcap);
+
+	/* get stream count from GCAP */
+	num_capture = (gcap >> 8) & 0x0f;
+	num_playback = (gcap >> 12) & 0x0f;
+	num_total = num_playback + num_capture;
+
+	dev_dbg(sdev->dev, "detected %d playback and %d capture streams\n",
+		num_playback, num_capture);
+
+	if (num_playback >= SOF_HDA_PLAYBACK_STREAMS) {
+		dev_err(sdev->dev, "error: too many playback streams %d\n",
+			num_playback);
+		return -EINVAL;
+	}
+
+	if (num_capture >= SOF_HDA_CAPTURE_STREAMS) {
+		dev_err(sdev->dev, "error: too many capture streams %d\n",
+			num_playback);
+		return -EINVAL;
+	}
+
+	/*
+	 * mem alloc for the position buffer
+	 * TODO: check position buffer update
+	 */
+	ret = snd_dma_alloc_pages(SNDRV_DMA_TYPE_DEV, &pci->dev,
+				  SOF_HDA_DPIB_ENTRY_SIZE * num_total,
+				  &bus->posbuf);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: posbuffer dma alloc failed\n");
+		return -ENOMEM;
+	}
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
+	/* mem alloc for the CORB/RIRB ringbuffers */
+	ret = snd_dma_alloc_pages(SNDRV_DMA_TYPE_DEV, &pci->dev,
+				  PAGE_SIZE, &bus->rb);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: RB alloc failed\n");
+		return -ENOMEM;
+	}
+#endif
+
+	/* create capture streams */
+	for (i = 0; i < num_capture; i++) {
+		struct sof_intel_hda_stream *hda_stream;
+
+		hda_stream = devm_kzalloc(sdev->dev, sizeof(*hda_stream),
+					  GFP_KERNEL);
+		if (!hda_stream)
+			return -ENOMEM;
+
+		stream = &hda_stream->hda_stream;
+
+		stream->pphc_addr = sdev->bar[HDA_DSP_PP_BAR] +
+			SOF_HDA_PPHC_BASE + SOF_HDA_PPHC_INTERVAL * i;
+
+		stream->pplc_addr = sdev->bar[HDA_DSP_PP_BAR] +
+			SOF_HDA_PPLC_BASE + SOF_HDA_PPLC_MULTI * num_total +
+			SOF_HDA_PPLC_INTERVAL * i;
+
+		/* do we support SPIB */
+		if (sdev->bar[HDA_DSP_SPIB_BAR]) {
+			stream->spib_addr = sdev->bar[HDA_DSP_SPIB_BAR] +
+				SOF_HDA_SPIB_BASE + SOF_HDA_SPIB_INTERVAL * i +
+				SOF_HDA_SPIB_SPIB;
+
+			stream->fifo_addr = sdev->bar[HDA_DSP_SPIB_BAR] +
+				SOF_HDA_SPIB_BASE + SOF_HDA_SPIB_INTERVAL * i +
+				SOF_HDA_SPIB_MAXFIFO;
+		}
+
+		hstream = &stream->hstream;
+		hstream->bus = bus;
+		hstream->sd_int_sta_mask = 1 << i;
+		hstream->index = i;
+		sd_offset = SOF_STREAM_SD_OFFSET(hstream);
+		hstream->sd_addr = sdev->bar[HDA_DSP_HDA_BAR] + sd_offset;
+		hstream->stream_tag = i + 1;
+		hstream->opened = false;
+		hstream->running = false;
+		hstream->direction = SNDRV_PCM_STREAM_CAPTURE;
+
+		/* memory alloc for stream BDL */
+		ret = snd_dma_alloc_pages(SNDRV_DMA_TYPE_DEV, &pci->dev,
+					  HDA_DSP_BDL_SIZE, &hstream->bdl);
+		if (ret < 0) {
+			dev_err(sdev->dev, "error: stream bdl dma alloc failed\n");
+			return -ENOMEM;
+		}
+		hstream->posbuf = (__le32 *)(bus->posbuf.area +
+			(hstream->index) * 8);
+
+		list_add_tail(&hstream->list, &bus->stream_list);
+	}
+
+	/* create playback streams */
+	for (i = num_capture; i < num_total; i++) {
+		struct sof_intel_hda_stream *hda_stream;
+
+		hda_stream = devm_kzalloc(sdev->dev, sizeof(*hda_stream),
+					  GFP_KERNEL);
+		if (!hda_stream)
+			return -ENOMEM;
+
+		stream = &hda_stream->hda_stream;
+
+		/* we always have DSP support */
+		stream->pphc_addr = sdev->bar[HDA_DSP_PP_BAR] +
+			SOF_HDA_PPHC_BASE + SOF_HDA_PPHC_INTERVAL * i;
+
+		stream->pplc_addr = sdev->bar[HDA_DSP_PP_BAR] +
+			SOF_HDA_PPLC_BASE + SOF_HDA_PPLC_MULTI * num_total +
+			SOF_HDA_PPLC_INTERVAL * i;
+
+		/* do we support SPIB */
+		if (sdev->bar[HDA_DSP_SPIB_BAR]) {
+			stream->spib_addr = sdev->bar[HDA_DSP_SPIB_BAR] +
+				SOF_HDA_SPIB_BASE + SOF_HDA_SPIB_INTERVAL * i +
+				SOF_HDA_SPIB_SPIB;
+
+			stream->fifo_addr = sdev->bar[HDA_DSP_SPIB_BAR] +
+				SOF_HDA_SPIB_BASE + SOF_HDA_SPIB_INTERVAL * i +
+				SOF_HDA_SPIB_MAXFIFO;
+		}
+
+		hstream = &stream->hstream;
+		hstream->bus = bus;
+		hstream->sd_int_sta_mask = 1 << i;
+		hstream->index = i;
+		sd_offset = SOF_STREAM_SD_OFFSET(hstream);
+		hstream->sd_addr = sdev->bar[HDA_DSP_HDA_BAR] + sd_offset;
+		hstream->stream_tag = i - num_capture + 1;
+		hstream->opened = false;
+		hstream->running = false;
+		hstream->direction = SNDRV_PCM_STREAM_PLAYBACK;
+
+		/* mem alloc for stream BDL */
+		ret = snd_dma_alloc_pages(SNDRV_DMA_TYPE_DEV, &pci->dev,
+					  HDA_DSP_BDL_SIZE, &hstream->bdl);
+		if (ret < 0) {
+			dev_err(sdev->dev, "error: stream bdl dma alloc failed\n");
+			return -ENOMEM;
+		}
+
+		hstream->posbuf = (__le32 *)(bus->posbuf.area +
+			(hstream->index) * 8);
+
+		list_add_tail(&hstream->list, &bus->stream_list);
+	}
+
+	/* store total stream count (playback + capture) from GCAP */
+	sof_hda->stream_max = num_total;
+
+	return 0;
+}
+
+void hda_dsp_stream_free(struct snd_sof_dev *sdev)
+{
+	struct hdac_bus *bus = sof_to_bus(sdev);
+	struct hdac_stream *s, *_s;
+	struct hdac_ext_stream *stream;
+	struct sof_intel_hda_stream *hda_stream;
+
+	/* free position buffer */
+	if (bus->posbuf.area)
+		snd_dma_free_pages(&bus->posbuf);
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
+	/* free position buffer */
+	if (bus->rb.area)
+		snd_dma_free_pages(&bus->rb);
+#endif
+
+	list_for_each_entry_safe(s, _s, &bus->stream_list, list) {
+		/* TODO: decouple */
+
+		/* free bdl buffer */
+		if (s->bdl.area)
+			snd_dma_free_pages(&s->bdl);
+		list_del(&s->list);
+		stream = stream_to_hdac_ext_stream(s);
+		hda_stream = container_of(stream, struct sof_intel_hda_stream,
+					  hda_stream);
+		devm_kfree(sdev->dev, hda_stream);
+	}
+}
diff --git a/sound/soc/sof/intel/hda-trace.c b/sound/soc/sof/intel/hda-trace.c
new file mode 100644
index 000000000000..33b23bd6a01e
--- /dev/null
+++ b/sound/soc/sof/intel/hda-trace.c
@@ -0,0 +1,94 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license.  When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2018 Intel Corporation. All rights reserved.
+//
+// Authors: Liam Girdwood <liam.r.girdwood@linux.intel.com>
+//	    Ranjani Sridharan <ranjani.sridharan@linux.intel.com>
+//	    Rander Wang <rander.wang@intel.com>
+//          Keyon Jie <yang.jie@linux.intel.com>
+//
+
+/*
+ * Hardware interface for generic Intel audio DSP HDA IP
+ */
+
+#include <sound/hdaudio_ext.h>
+#include "../ops.h"
+#include "hda.h"
+
+static int hda_dsp_trace_prepare(struct snd_sof_dev *sdev)
+{
+	struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata;
+	struct hdac_ext_stream *stream = hda->dtrace_stream;
+	struct hdac_stream *hstream = &stream->hstream;
+	struct snd_dma_buffer *dmab = &sdev->dmatb;
+	int ret;
+
+	hstream->period_bytes = 0;/* initialize period_bytes */
+	hstream->bufsize = sdev->dmatb.bytes;
+
+	ret = hda_dsp_stream_hw_params(sdev, stream, dmab, NULL);
+	if (ret < 0)
+		dev_err(sdev->dev, "error: hdac prepare failed: %x\n", ret);
+
+	return ret;
+}
+
+int hda_dsp_trace_init(struct snd_sof_dev *sdev, u32 *stream_tag)
+{
+	struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata;
+	int ret;
+
+	hda->dtrace_stream = hda_dsp_stream_get(sdev,
+						SNDRV_PCM_STREAM_CAPTURE);
+
+	if (!hda->dtrace_stream) {
+		dev_err(sdev->dev,
+			"error: no available capture stream for DMA trace\n");
+		return -ENODEV;
+	}
+
+	*stream_tag = hda->dtrace_stream->hstream.stream_tag;
+
+	/*
+	 * initialize capture stream, set BDL address and return corresponding
+	 * stream tag which will be sent to the firmware by IPC message.
+	 */
+	ret = hda_dsp_trace_prepare(sdev);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: hdac trace init failed: %x\n", ret);
+		hda_dsp_stream_put(sdev, SNDRV_PCM_STREAM_CAPTURE, *stream_tag);
+		hda->dtrace_stream = NULL;
+		*stream_tag = 0;
+	}
+
+	return ret;
+}
+
+int hda_dsp_trace_release(struct snd_sof_dev *sdev)
+{
+	struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata;
+	struct hdac_stream *hstream;
+
+	if (hda->dtrace_stream) {
+		hstream = &hda->dtrace_stream->hstream;
+		hda_dsp_stream_put(sdev,
+				   SNDRV_PCM_STREAM_CAPTURE,
+				   hstream->stream_tag);
+		hda->dtrace_stream = NULL;
+		return 0;
+	}
+
+	dev_dbg(sdev->dev, "DMA trace stream is not opened!\n");
+	return -ENODEV;
+}
+
+int hda_dsp_trace_trigger(struct snd_sof_dev *sdev, int cmd)
+{
+	struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata;
+
+	return hda_dsp_stream_trigger(sdev, hda->dtrace_stream, cmd);
+}
diff --git a/sound/soc/sof/intel/hda.c b/sound/soc/sof/intel/hda.c
new file mode 100644
index 000000000000..7e3980a2f7ba
--- /dev/null
+++ b/sound/soc/sof/intel/hda.c
@@ -0,0 +1,689 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license.  When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2018 Intel Corporation. All rights reserved.
+//
+// Authors: Liam Girdwood <liam.r.girdwood@linux.intel.com>
+//	    Ranjani Sridharan <ranjani.sridharan@linux.intel.com>
+//	    Rander Wang <rander.wang@intel.com>
+//          Keyon Jie <yang.jie@linux.intel.com>
+//
+
+/*
+ * Hardware interface for generic Intel audio DSP HDA IP
+ */
+
+#include <linux/module.h>
+#include <sound/hdaudio_ext.h>
+#include <sound/sof.h>
+#include <sound/sof/xtensa.h>
+#include "../ops.h"
+#include "hda.h"
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA_AUDIO_CODEC)
+#include "../../codecs/hdac_hda.h"
+#endif
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
+#include <sound/soc-acpi-intel-match.h>
+#endif
+
+/* platform specific devices */
+#include "shim.h"
+
+/*
+ * Debug
+ */
+
+struct hda_dsp_msg_code {
+	u32 code;
+	const char *msg;
+};
+
+static const struct hda_dsp_msg_code hda_dsp_rom_msg[] = {
+	{HDA_DSP_ROM_FW_MANIFEST_LOADED, "status: manifest loaded"},
+	{HDA_DSP_ROM_FW_FW_LOADED, "status: fw loaded"},
+	{HDA_DSP_ROM_FW_ENTERED, "status: fw entered"},
+	{HDA_DSP_ROM_CSE_ERROR, "error: cse error"},
+	{HDA_DSP_ROM_CSE_WRONG_RESPONSE, "error: cse wrong response"},
+	{HDA_DSP_ROM_IMR_TO_SMALL, "error: IMR too small"},
+	{HDA_DSP_ROM_BASE_FW_NOT_FOUND, "error: base fw not found"},
+	{HDA_DSP_ROM_CSE_VALIDATION_FAILED, "error: signature verification failed"},
+	{HDA_DSP_ROM_IPC_FATAL_ERROR, "error: ipc fatal error"},
+	{HDA_DSP_ROM_L2_CACHE_ERROR, "error: L2 cache error"},
+	{HDA_DSP_ROM_LOAD_OFFSET_TO_SMALL, "error: load offset too small"},
+	{HDA_DSP_ROM_API_PTR_INVALID, "error: API ptr invalid"},
+	{HDA_DSP_ROM_BASEFW_INCOMPAT, "error: base fw incompatible"},
+	{HDA_DSP_ROM_UNHANDLED_INTERRUPT, "error: unhandled interrupt"},
+	{HDA_DSP_ROM_MEMORY_HOLE_ECC, "error: ECC memory hole"},
+	{HDA_DSP_ROM_KERNEL_EXCEPTION, "error: kernel exception"},
+	{HDA_DSP_ROM_USER_EXCEPTION, "error: user exception"},
+	{HDA_DSP_ROM_UNEXPECTED_RESET, "error: unexpected reset"},
+	{HDA_DSP_ROM_NULL_FW_ENTRY,	"error: null FW entry point"},
+};
+
+static void hda_dsp_get_status_skl(struct snd_sof_dev *sdev)
+{
+	u32 status;
+	int i;
+
+	status = snd_sof_dsp_read(sdev, HDA_DSP_BAR,
+				  HDA_ADSP_FW_STATUS_SKL);
+
+	for (i = 0; i < ARRAY_SIZE(hda_dsp_rom_msg); i++) {
+		if (status == hda_dsp_rom_msg[i].code) {
+			dev_err(sdev->dev, "%s - code %8.8x\n",
+				hda_dsp_rom_msg[i].msg, status);
+			return;
+		}
+	}
+
+	/* not for us, must be generic sof message */
+	dev_dbg(sdev->dev, "unknown ROM status value %8.8x\n", status);
+}
+
+static void hda_dsp_get_status(struct snd_sof_dev *sdev)
+{
+	u32 status;
+	int i;
+
+	status = snd_sof_dsp_read(sdev, HDA_DSP_BAR,
+				  HDA_DSP_SRAM_REG_ROM_STATUS);
+
+	for (i = 0; i < ARRAY_SIZE(hda_dsp_rom_msg); i++) {
+		if (status == hda_dsp_rom_msg[i].code) {
+			dev_err(sdev->dev, "%s - code %8.8x\n",
+				hda_dsp_rom_msg[i].msg, status);
+			return;
+		}
+	}
+
+	/* not for us, must be generic sof message */
+	dev_dbg(sdev->dev, "unknown ROM status value %8.8x\n", status);
+}
+
+static void hda_dsp_get_registers(struct snd_sof_dev *sdev,
+				  struct sof_ipc_dsp_oops_xtensa *xoops,
+				  struct sof_ipc_panic_info *panic_info,
+				  u32 *stack, size_t stack_words)
+{
+	/* first read registers */
+	sof_block_read(sdev, sdev->mmio_bar, sdev->dsp_oops_offset, xoops,
+		       sizeof(*xoops));
+
+	/* then get panic info */
+	sof_block_read(sdev, sdev->mmio_bar, sdev->dsp_oops_offset +
+		       sizeof(*xoops), panic_info, sizeof(*panic_info));
+
+	/* then get the stack */
+	sof_block_read(sdev, sdev->mmio_bar, sdev->dsp_oops_offset +
+		       sizeof(*xoops) + sizeof(*panic_info), stack,
+		       stack_words * sizeof(u32));
+}
+
+void hda_dsp_dump_skl(struct snd_sof_dev *sdev, u32 flags)
+{
+	struct sof_ipc_dsp_oops_xtensa xoops;
+	struct sof_ipc_panic_info panic_info;
+	u32 stack[HDA_DSP_STACK_DUMP_SIZE];
+	u32 status, panic;
+
+	/* try APL specific status message types first */
+	hda_dsp_get_status_skl(sdev);
+
+	/* now try generic SOF status messages */
+	status = snd_sof_dsp_read(sdev, HDA_DSP_BAR,
+				  HDA_ADSP_ERROR_CODE_SKL);
+
+	/*TODO: Check: there is no define in spec, but it is used in the code*/
+	panic = snd_sof_dsp_read(sdev, HDA_DSP_BAR,
+				 HDA_ADSP_ERROR_CODE_SKL + 0x4);
+
+	if (sdev->boot_complete) {
+		hda_dsp_get_registers(sdev, &xoops, &panic_info, stack,
+				      HDA_DSP_STACK_DUMP_SIZE);
+		snd_sof_get_status(sdev, status, panic, &xoops, &panic_info,
+				   stack, HDA_DSP_STACK_DUMP_SIZE);
+	} else {
+		dev_err(sdev->dev, "error: status = 0x%8.8x panic = 0x%8.8x\n",
+			status, panic);
+		hda_dsp_get_status_skl(sdev);
+	}
+}
+
+void hda_dsp_dump(struct snd_sof_dev *sdev, u32 flags)
+{
+	struct sof_ipc_dsp_oops_xtensa xoops;
+	struct sof_ipc_panic_info panic_info;
+	u32 stack[HDA_DSP_STACK_DUMP_SIZE];
+	u32 status, panic;
+
+	/* try APL specific status message types first */
+	hda_dsp_get_status(sdev);
+
+	/* now try generic SOF status messages */
+	status = snd_sof_dsp_read(sdev, HDA_DSP_BAR,
+				  HDA_DSP_SRAM_REG_FW_STATUS);
+	panic = snd_sof_dsp_read(sdev, HDA_DSP_BAR, HDA_DSP_SRAM_REG_FW_TRACEP);
+
+	if (sdev->boot_complete) {
+		hda_dsp_get_registers(sdev, &xoops, &panic_info, stack,
+				      HDA_DSP_STACK_DUMP_SIZE);
+		snd_sof_get_status(sdev, status, panic, &xoops, &panic_info,
+				   stack, HDA_DSP_STACK_DUMP_SIZE);
+	} else {
+		dev_err(sdev->dev, "error: status = 0x%8.8x panic = 0x%8.8x\n",
+			status, panic);
+		hda_dsp_get_status(sdev);
+	}
+}
+
+void hda_ipc_dump(struct snd_sof_dev *sdev)
+{
+	u32 hipcie;
+	u32 hipct;
+	u32 hipcctl;
+
+	/* read IPC status */
+	hipcie = snd_sof_dsp_read(sdev, HDA_DSP_BAR, HDA_DSP_REG_HIPCIE);
+	hipct = snd_sof_dsp_read(sdev, HDA_DSP_BAR, HDA_DSP_REG_HIPCT);
+	hipcctl = snd_sof_dsp_read(sdev, HDA_DSP_BAR, HDA_DSP_REG_HIPCCTL);
+
+	/* dump the IPC regs */
+	/* TODO: parse the raw msg */
+	dev_err(sdev->dev,
+		"error: host status 0x%8.8x dsp status 0x%8.8x mask 0x%8.8x\n",
+		hipcie, hipct, hipcctl);
+}
+
+static int hda_init(struct snd_sof_dev *sdev)
+{
+	struct hda_bus *hbus;
+	struct hdac_bus *bus;
+	struct hdac_ext_bus_ops *ext_ops = NULL;
+	struct pci_dev *pci = to_pci_dev(sdev->dev);
+	int ret;
+
+	hbus = sof_to_hbus(sdev);
+	bus = sof_to_bus(sdev);
+
+	/* HDA bus init */
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA_AUDIO_CODEC)
+	ext_ops = snd_soc_hdac_hda_get_ops();
+#endif
+	sof_hda_bus_init(bus, &pci->dev, ext_ops);
+	bus->use_posbuf = 1;
+	bus->bdl_pos_adj = 0;
+
+	mutex_init(&hbus->prepare_mutex);
+	hbus->pci = pci;
+	hbus->mixer_assigned = -1;
+	hbus->modelname = "sofbus";
+
+	/* initialise hdac bus */
+	bus->addr = pci_resource_start(pci, 0);
+	bus->remap_addr = pci_ioremap_bar(pci, 0);
+	if (!bus->remap_addr) {
+		dev_err(bus->dev, "error: ioremap error\n");
+		return -ENXIO;
+	}
+
+	/* HDA base */
+	sdev->bar[HDA_DSP_HDA_BAR] = bus->remap_addr;
+
+	/* get controller capabilities */
+	ret = hda_dsp_ctrl_get_caps(sdev);
+	if (ret < 0)
+		dev_err(sdev->dev, "error: get caps error\n");
+
+	return ret;
+}
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
+
+static const char *fixup_tplg_name(struct snd_sof_dev *sdev,
+				   const char *sof_tplg_filename)
+{
+	const char *tplg_filename = NULL;
+	char *filename;
+	char *split_ext;
+
+	filename = devm_kstrdup(sdev->dev, sof_tplg_filename, GFP_KERNEL);
+	if (!filename)
+		return NULL;
+
+	/* this assumes a .tplg extension */
+	split_ext = strsep(&filename, ".");
+	if (split_ext) {
+		tplg_filename = devm_kasprintf(sdev->dev, GFP_KERNEL,
+					       "%s-idisp.tplg", split_ext);
+		if (!tplg_filename)
+			return NULL;
+	}
+	return tplg_filename;
+}
+
+static int hda_init_caps(struct snd_sof_dev *sdev)
+{
+	struct hdac_bus *bus = sof_to_bus(sdev);
+	struct hdac_ext_link *hlink;
+	struct snd_soc_acpi_mach_params *mach_params;
+	struct snd_soc_acpi_mach *hda_mach;
+	struct snd_sof_pdata *pdata = sdev->pdata;
+	struct snd_soc_acpi_mach *mach;
+	const char *tplg_filename;
+	int codec_num = 0;
+	int ret = 0;
+	int i;
+
+	device_disable_async_suspend(bus->dev);
+
+	/* check if dsp is there */
+	if (bus->ppcap)
+		dev_dbg(sdev->dev, "PP capability, will probe DSP later.\n");
+
+	if (bus->mlcap)
+		snd_hdac_ext_bus_get_ml_capabilities(bus);
+
+	/* init i915 and HDMI codecs */
+	ret = hda_codec_i915_init(sdev);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: no HDMI audio devices found\n");
+		return ret;
+	}
+
+	ret = hda_dsp_ctrl_init_chip(sdev, true);
+	if (ret < 0) {
+		dev_err(bus->dev, "error: init chip failed with ret: %d\n", ret);
+		goto out;
+	}
+
+	/* codec detection */
+	if (!bus->codec_mask) {
+		dev_info(bus->dev, "no hda codecs found!\n");
+	} else {
+		dev_info(bus->dev, "hda codecs found, mask %lx\n",
+			 bus->codec_mask);
+
+		for (i = 0; i < HDA_MAX_CODECS; i++) {
+			if (bus->codec_mask & (1 << i))
+				codec_num++;
+		}
+
+		/*
+		 * If no machine driver is found, then:
+		 *
+		 * hda machine driver is used if :
+		 * 1. there is one HDMI codec and one external HDAudio codec
+		 * 2. only HDMI codec
+		 */
+		if (!pdata->machine && codec_num <= 2 &&
+		    HDA_IDISP_CODEC(bus->codec_mask)) {
+			hda_mach = snd_soc_acpi_intel_hda_machines;
+			pdata->machine = hda_mach;
+
+			/* topology: use the info from hda_machines */
+			pdata->tplg_filename =
+				hda_mach->sof_tplg_filename;
+
+			/* firmware: pick the first in machine list */
+			mach = pdata->desc->machines;
+			pdata->fw_filename = mach->sof_fw_filename;
+
+			dev_info(bus->dev, "using HDA machine driver %s now\n",
+				 hda_mach->drv_name);
+
+			/* fixup topology file for HDMI only platforms */
+			if (codec_num == 1) {
+				/* use local variable for readability */
+				tplg_filename = pdata->tplg_filename;
+				tplg_filename = fixup_tplg_name(sdev, tplg_filename);
+				if (!tplg_filename)
+					goto out;
+				pdata->tplg_filename = tplg_filename;
+			}
+		}
+	}
+
+	/* used by hda machine driver to create dai links */
+	if (pdata->machine) {
+		mach_params = (struct snd_soc_acpi_mach_params *)
+			&pdata->machine->mach_params;
+		mach_params->codec_mask = bus->codec_mask;
+		mach_params->platform = dev_name(sdev->dev);
+	}
+
+	/* create codec instances */
+	hda_codec_probe_bus(sdev);
+
+	hda_codec_i915_put(sdev);
+
+	/*
+	 * we are done probing so decrement link counts
+	 */
+	list_for_each_entry(hlink, &bus->hlink_list, list)
+		snd_hdac_ext_bus_link_put(bus, hlink);
+
+	return 0;
+
+out:
+	hda_codec_i915_exit(sdev);
+	return ret;
+}
+
+#else
+
+static int hda_init_caps(struct snd_sof_dev *sdev)
+{
+	/*
+	 * set CGCTL.MISCBDCGE to 0 during reset and set back to 1
+	 * when reset finished.
+	 * TODO: maybe no need for init_caps?
+	 */
+	hda_dsp_ctrl_misc_clock_gating(sdev, 0);
+
+	/* clear WAKESTS */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR, SOF_HDA_WAKESTS,
+				SOF_HDA_WAKESTS_INT_MASK,
+				SOF_HDA_WAKESTS_INT_MASK);
+
+	return 0;
+}
+
+#endif
+
+static const struct sof_intel_dsp_desc
+	*get_chip_info(struct snd_sof_pdata *pdata)
+{
+	const struct sof_dev_desc *desc = pdata->desc;
+	const struct sof_intel_dsp_desc *chip_info;
+
+	chip_info = desc->chip_info;
+
+	return chip_info;
+}
+
+int hda_dsp_probe(struct snd_sof_dev *sdev)
+{
+	struct pci_dev *pci = to_pci_dev(sdev->dev);
+	struct sof_intel_hda_dev *hdev;
+	struct hdac_bus *bus;
+	struct hdac_stream *stream;
+	const struct sof_intel_dsp_desc *chip;
+	int sd_offset, ret = 0;
+
+	/*
+	 * detect DSP by checking class/subclass/prog-id information
+	 * class=04 subclass 03 prog-if 00: no DSP, legacy driver is required
+	 * class=04 subclass 01 prog-if 00: DSP is present
+	 *   (and may be required e.g. for DMIC or SSP support)
+	 * class=04 subclass 03 prog-if 80: either of DSP or legacy mode works
+	 */
+	if (pci->class == 0x040300) {
+		dev_err(sdev->dev, "error: the DSP is not enabled on this platform, aborting probe\n");
+		return -ENODEV;
+	} else if (pci->class != 0x040100 && pci->class != 0x040380) {
+		dev_err(sdev->dev, "error: unknown PCI class/subclass/prog-if 0x%06x found, aborting probe\n", pci->class);
+		return -ENODEV;
+	}
+	dev_info(sdev->dev, "DSP detected with PCI class/subclass/prog-if 0x%06x\n", pci->class);
+
+	chip = get_chip_info(sdev->pdata);
+	if (!chip) {
+		dev_err(sdev->dev, "error: no such device supported, chip id:%x\n",
+			pci->device);
+		ret = -EIO;
+		goto err;
+	}
+
+	hdev = devm_kzalloc(sdev->dev, sizeof(*hdev), GFP_KERNEL);
+	if (!hdev)
+		return -ENOMEM;
+	sdev->pdata->hw_pdata = hdev;
+	hdev->desc = chip;
+
+	hdev->dmic_dev = platform_device_register_data(sdev->dev, "dmic-codec",
+						       PLATFORM_DEVID_NONE,
+						       NULL, 0);
+	if (IS_ERR(hdev->dmic_dev)) {
+		dev_err(sdev->dev, "error: failed to create DMIC device\n");
+		return PTR_ERR(hdev->dmic_dev);
+	}
+
+	/*
+	 * use position update IPC if either it is forced
+	 * or we don't have other choice
+	 */
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG_FORCE_IPC_POSITION)
+	hdev->no_ipc_position = 0;
+#else
+	hdev->no_ipc_position = sof_ops(sdev)->pcm_pointer ? 1 : 0;
+#endif
+
+	/* set up HDA base */
+	bus = sof_to_bus(sdev);
+	ret = hda_init(sdev);
+	if (ret < 0)
+		goto hdac_bus_unmap;
+
+	/* DSP base */
+	sdev->bar[HDA_DSP_BAR] = pci_ioremap_bar(pci, HDA_DSP_BAR);
+	if (!sdev->bar[HDA_DSP_BAR]) {
+		dev_err(sdev->dev, "error: ioremap error\n");
+		ret = -ENXIO;
+		goto hdac_bus_unmap;
+	}
+
+	sdev->mmio_bar = HDA_DSP_BAR;
+	sdev->mailbox_bar = HDA_DSP_BAR;
+
+	/* allow 64bit DMA address if supported by H/W */
+	if (!dma_set_mask(&pci->dev, DMA_BIT_MASK(64))) {
+		dev_dbg(sdev->dev, "DMA mask is 64 bit\n");
+		dma_set_coherent_mask(&pci->dev, DMA_BIT_MASK(64));
+	} else {
+		dev_dbg(sdev->dev, "DMA mask is 32 bit\n");
+		dma_set_mask(&pci->dev, DMA_BIT_MASK(32));
+		dma_set_coherent_mask(&pci->dev, DMA_BIT_MASK(32));
+	}
+
+	/* init streams */
+	ret = hda_dsp_stream_init(sdev);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: failed to init streams\n");
+		/*
+		 * not all errors are due to memory issues, but trying
+		 * to free everything does not harm
+		 */
+		goto free_streams;
+	}
+
+	/*
+	 * register our IRQ
+	 * let's try to enable msi firstly
+	 * if it fails, use legacy interrupt mode
+	 * TODO: support interrupt mode selection with kernel parameter
+	 *       support msi multiple vectors
+	 */
+	ret = pci_alloc_irq_vectors(pci, 1, 1, PCI_IRQ_MSI);
+	if (ret < 0) {
+		dev_info(sdev->dev, "use legacy interrupt mode\n");
+		/*
+		 * in IO-APIC mode, hda->irq and ipc_irq are using the same
+		 * irq number of pci->irq
+		 */
+		hdev->irq = pci->irq;
+		sdev->ipc_irq = pci->irq;
+		sdev->msi_enabled = 0;
+	} else {
+		dev_info(sdev->dev, "use msi interrupt mode\n");
+		hdev->irq = pci_irq_vector(pci, 0);
+		/* ipc irq number is the same of hda irq */
+		sdev->ipc_irq = hdev->irq;
+		sdev->msi_enabled = 1;
+	}
+
+	dev_dbg(sdev->dev, "using HDA IRQ %d\n", hdev->irq);
+	ret = request_threaded_irq(hdev->irq, hda_dsp_stream_interrupt,
+				   hda_dsp_stream_threaded_handler,
+				   IRQF_SHARED, "AudioHDA", bus);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: failed to register HDA IRQ %d\n",
+			hdev->irq);
+		goto free_irq_vector;
+	}
+
+	dev_dbg(sdev->dev, "using IPC IRQ %d\n", sdev->ipc_irq);
+	ret = request_threaded_irq(sdev->ipc_irq, hda_dsp_ipc_irq_handler,
+				   sof_ops(sdev)->irq_thread, IRQF_SHARED,
+				   "AudioDSP", sdev);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: failed to register IPC IRQ %d\n",
+			sdev->ipc_irq);
+		goto free_hda_irq;
+	}
+
+	pci_set_master(pci);
+	synchronize_irq(pci->irq);
+
+	/*
+	 * clear TCSEL to clear playback on some HD Audio
+	 * codecs. PCI TCSEL is defined in the Intel manuals.
+	 */
+	snd_sof_pci_update_bits(sdev, PCI_TCSEL, 0x07, 0);
+
+	/* init HDA capabilities */
+	ret = hda_init_caps(sdev);
+	if (ret < 0)
+		goto free_ipc_irq;
+
+	/* reset HDA controller */
+	ret = hda_dsp_ctrl_link_reset(sdev, true);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: failed to reset HDA controller\n");
+		goto free_ipc_irq;
+	}
+
+	/* exit HDA controller reset */
+	ret = hda_dsp_ctrl_link_reset(sdev, false);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: failed to exit HDA controller reset\n");
+		goto free_ipc_irq;
+	}
+
+	/* clear stream status */
+	list_for_each_entry(stream, &bus->stream_list, list) {
+		sd_offset = SOF_STREAM_SD_OFFSET(stream);
+		snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR,
+					sd_offset +
+					SOF_HDA_ADSP_REG_CL_SD_STS,
+					SOF_HDA_CL_DMA_SD_INT_MASK,
+					SOF_HDA_CL_DMA_SD_INT_MASK);
+	}
+
+	/* clear WAKESTS */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR, SOF_HDA_WAKESTS,
+				SOF_HDA_WAKESTS_INT_MASK,
+				SOF_HDA_WAKESTS_INT_MASK);
+
+	/* clear interrupt status register */
+	snd_sof_dsp_write(sdev, HDA_DSP_HDA_BAR, SOF_HDA_INTSTS,
+			  SOF_HDA_INT_CTRL_EN | SOF_HDA_INT_ALL_STREAM);
+
+	/* enable CIE and GIE interrupts */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR, SOF_HDA_INTCTL,
+				SOF_HDA_INT_CTRL_EN | SOF_HDA_INT_GLOBAL_EN,
+				SOF_HDA_INT_CTRL_EN | SOF_HDA_INT_GLOBAL_EN);
+
+	/* re-enable CGCTL.MISCBDCGE after reset */
+	hda_dsp_ctrl_misc_clock_gating(sdev, true);
+
+	device_disable_async_suspend(&pci->dev);
+
+	/* enable DSP features */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_PP_BAR, SOF_HDA_REG_PP_PPCTL,
+				SOF_HDA_PPCTL_GPROCEN, SOF_HDA_PPCTL_GPROCEN);
+
+	/* enable DSP IRQ */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_PP_BAR, SOF_HDA_REG_PP_PPCTL,
+				SOF_HDA_PPCTL_PIE, SOF_HDA_PPCTL_PIE);
+
+	/* initialize waitq for code loading */
+	init_waitqueue_head(&sdev->waitq);
+
+	/* set default mailbox offset for FW ready message */
+	sdev->dsp_box.offset = HDA_DSP_MBOX_UPLINK_OFFSET;
+
+	return 0;
+
+free_ipc_irq:
+	free_irq(sdev->ipc_irq, sdev);
+free_hda_irq:
+	free_irq(hdev->irq, bus);
+free_irq_vector:
+	if (sdev->msi_enabled)
+		pci_free_irq_vectors(pci);
+free_streams:
+	hda_dsp_stream_free(sdev);
+/* dsp_unmap: not currently used */
+	iounmap(sdev->bar[HDA_DSP_BAR]);
+hdac_bus_unmap:
+	iounmap(bus->remap_addr);
+err:
+	return ret;
+}
+
+int hda_dsp_remove(struct snd_sof_dev *sdev)
+{
+	struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata;
+	struct hdac_bus *bus = sof_to_bus(sdev);
+	struct pci_dev *pci = to_pci_dev(sdev->dev);
+	const struct sof_intel_dsp_desc *chip = hda->desc;
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
+	/* codec removal, invoke bus_device_remove */
+	snd_hdac_ext_bus_device_remove(bus);
+#endif
+
+	if (!IS_ERR_OR_NULL(hda->dmic_dev))
+		platform_device_unregister(hda->dmic_dev);
+
+	/* disable DSP IRQ */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_PP_BAR, SOF_HDA_REG_PP_PPCTL,
+				SOF_HDA_PPCTL_PIE, 0);
+
+	/* disable CIE and GIE interrupts */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR, SOF_HDA_INTCTL,
+				SOF_HDA_INT_CTRL_EN | SOF_HDA_INT_GLOBAL_EN, 0);
+
+	/* disable cores */
+	if (chip)
+		hda_dsp_core_reset_power_down(sdev, chip->cores_mask);
+
+	/* disable DSP */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_PP_BAR, SOF_HDA_REG_PP_PPCTL,
+				SOF_HDA_PPCTL_GPROCEN, 0);
+
+	free_irq(sdev->ipc_irq, sdev);
+	free_irq(hda->irq, bus);
+	if (sdev->msi_enabled)
+		pci_free_irq_vectors(pci);
+
+	hda_dsp_stream_free(sdev);
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
+	snd_hdac_link_free_all(bus);
+#endif
+
+	iounmap(sdev->bar[HDA_DSP_BAR]);
+	iounmap(bus->remap_addr);
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
+	snd_hdac_ext_bus_exit(bus);
+#endif
+	hda_codec_i915_exit(sdev);
+
+	return 0;
+}
+
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/sound/soc/sof/intel/hda.h b/sound/soc/sof/intel/hda.h
new file mode 100644
index 000000000000..92d45c43b4b1
--- /dev/null
+++ b/sound/soc/sof/intel/hda.h
@@ -0,0 +1,583 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */
+/*
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * Copyright(c) 2017 Intel Corporation. All rights reserved.
+ *
+ * Author: Liam Girdwood <liam.r.girdwood@linux.intel.com>
+ */
+
+#ifndef __SOF_INTEL_HDA_H
+#define __SOF_INTEL_HDA_H
+
+#include <sound/hda_codec.h>
+#include <sound/hdaudio_ext.h>
+#include "shim.h"
+
+/* PCI registers */
+#define PCI_TCSEL			0x44
+#define PCI_PGCTL			PCI_TCSEL
+#define PCI_CGCTL			0x48
+
+/* PCI_PGCTL bits */
+#define PCI_PGCTL_ADSPPGD               BIT(2)
+#define PCI_PGCTL_LSRMD_MASK		BIT(4)
+
+/* PCI_CGCTL bits */
+#define PCI_CGCTL_MISCBDCGE_MASK	BIT(6)
+#define PCI_CGCTL_ADSPDCGE              BIT(1)
+
+/* Legacy HDA registers and bits used - widths are variable */
+#define SOF_HDA_GCAP			0x0
+#define SOF_HDA_GCTL			0x8
+/* accept unsol. response enable */
+#define SOF_HDA_GCTL_UNSOL		BIT(8)
+#define SOF_HDA_LLCH			0x14
+#define SOF_HDA_INTCTL			0x20
+#define SOF_HDA_INTSTS			0x24
+#define SOF_HDA_WAKESTS			0x0E
+#define SOF_HDA_WAKESTS_INT_MASK	((1 << 8) - 1)
+#define SOF_HDA_RIRBSTS			0x5d
+#define SOF_HDA_VS_EM2_L1SEN            BIT(13)
+
+/* SOF_HDA_GCTL register bist */
+#define SOF_HDA_GCTL_RESET		BIT(0)
+
+/* SOF_HDA_INCTL and SOF_HDA_INTSTS regs */
+#define SOF_HDA_INT_GLOBAL_EN		BIT(31)
+#define SOF_HDA_INT_CTRL_EN		BIT(30)
+#define SOF_HDA_INT_ALL_STREAM		0xff
+
+#define SOF_HDA_MAX_CAPS		10
+#define SOF_HDA_CAP_ID_OFF		16
+#define SOF_HDA_CAP_ID_MASK		GENMASK(SOF_HDA_CAP_ID_OFF + 11,\
+						SOF_HDA_CAP_ID_OFF)
+#define SOF_HDA_CAP_NEXT_MASK		0xFFFF
+
+#define SOF_HDA_GTS_CAP_ID			0x1
+#define SOF_HDA_ML_CAP_ID			0x2
+
+#define SOF_HDA_PP_CAP_ID		0x3
+#define SOF_HDA_REG_PP_PPCH		0x10
+#define SOF_HDA_REG_PP_PPCTL		0x04
+#define SOF_HDA_PPCTL_PIE		BIT(31)
+#define SOF_HDA_PPCTL_GPROCEN		BIT(30)
+
+/* DPIB entry size: 8 Bytes = 2 DWords */
+#define SOF_HDA_DPIB_ENTRY_SIZE	0x8
+
+#define SOF_HDA_SPIB_CAP_ID		0x4
+#define SOF_HDA_DRSM_CAP_ID		0x5
+
+#define SOF_HDA_SPIB_BASE		0x08
+#define SOF_HDA_SPIB_INTERVAL		0x08
+#define SOF_HDA_SPIB_SPIB		0x00
+#define SOF_HDA_SPIB_MAXFIFO		0x04
+
+#define SOF_HDA_PPHC_BASE		0x10
+#define SOF_HDA_PPHC_INTERVAL		0x10
+
+#define SOF_HDA_PPLC_BASE		0x10
+#define SOF_HDA_PPLC_MULTI		0x10
+#define SOF_HDA_PPLC_INTERVAL		0x10
+
+#define SOF_HDA_DRSM_BASE		0x08
+#define SOF_HDA_DRSM_INTERVAL		0x08
+
+/* Descriptor error interrupt */
+#define SOF_HDA_CL_DMA_SD_INT_DESC_ERR		0x10
+
+/* FIFO error interrupt */
+#define SOF_HDA_CL_DMA_SD_INT_FIFO_ERR		0x08
+
+/* Buffer completion interrupt */
+#define SOF_HDA_CL_DMA_SD_INT_COMPLETE		0x04
+
+#define SOF_HDA_CL_DMA_SD_INT_MASK \
+	(SOF_HDA_CL_DMA_SD_INT_DESC_ERR | \
+	SOF_HDA_CL_DMA_SD_INT_FIFO_ERR | \
+	SOF_HDA_CL_DMA_SD_INT_COMPLETE)
+#define SOF_HDA_SD_CTL_DMA_START		0x02 /* Stream DMA start bit */
+
+/* Intel HD Audio Code Loader DMA Registers */
+#define SOF_HDA_ADSP_LOADER_BASE		0x80
+#define SOF_HDA_ADSP_DPLBASE			0x70
+#define SOF_HDA_ADSP_DPUBASE			0x74
+#define SOF_HDA_ADSP_DPLBASE_ENABLE		0x01
+
+/* Stream Registers */
+#define SOF_HDA_ADSP_REG_CL_SD_CTL		0x00
+#define SOF_HDA_ADSP_REG_CL_SD_STS		0x03
+#define SOF_HDA_ADSP_REG_CL_SD_LPIB		0x04
+#define SOF_HDA_ADSP_REG_CL_SD_CBL		0x08
+#define SOF_HDA_ADSP_REG_CL_SD_LVI		0x0C
+#define SOF_HDA_ADSP_REG_CL_SD_FIFOW		0x0E
+#define SOF_HDA_ADSP_REG_CL_SD_FIFOSIZE		0x10
+#define SOF_HDA_ADSP_REG_CL_SD_FORMAT		0x12
+#define SOF_HDA_ADSP_REG_CL_SD_FIFOL		0x14
+#define SOF_HDA_ADSP_REG_CL_SD_BDLPL		0x18
+#define SOF_HDA_ADSP_REG_CL_SD_BDLPU		0x1C
+#define SOF_HDA_ADSP_SD_ENTRY_SIZE		0x20
+
+/* CL: Software Position Based FIFO Capability Registers */
+#define SOF_DSP_REG_CL_SPBFIFO \
+	(SOF_HDA_ADSP_LOADER_BASE + 0x20)
+#define SOF_HDA_ADSP_REG_CL_SPBFIFO_SPBFCH	0x0
+#define SOF_HDA_ADSP_REG_CL_SPBFIFO_SPBFCCTL	0x4
+#define SOF_HDA_ADSP_REG_CL_SPBFIFO_SPIB	0x8
+#define SOF_HDA_ADSP_REG_CL_SPBFIFO_MAXFIFOS	0xc
+
+/* Stream Number */
+#define SOF_HDA_CL_SD_CTL_STREAM_TAG_SHIFT	20
+#define SOF_HDA_CL_SD_CTL_STREAM_TAG_MASK \
+	GENMASK(SOF_HDA_CL_SD_CTL_STREAM_TAG_SHIFT + 3,\
+		SOF_HDA_CL_SD_CTL_STREAM_TAG_SHIFT)
+
+#define HDA_DSP_HDA_BAR				0
+#define HDA_DSP_PP_BAR				1
+#define HDA_DSP_SPIB_BAR			2
+#define HDA_DSP_DRSM_BAR			3
+#define HDA_DSP_BAR				4
+
+#define SRAM_WINDOW_OFFSET(x)			(0x80000 + (x) * 0x20000)
+
+#define HDA_DSP_MBOX_OFFSET			SRAM_WINDOW_OFFSET(0)
+
+#define HDA_DSP_PANIC_OFFSET(x) \
+	(((x) & 0xFFFFFF) + HDA_DSP_MBOX_OFFSET)
+
+/* SRAM window 0 FW "registers" */
+#define HDA_DSP_SRAM_REG_ROM_STATUS		(HDA_DSP_MBOX_OFFSET + 0x0)
+#define HDA_DSP_SRAM_REG_ROM_ERROR		(HDA_DSP_MBOX_OFFSET + 0x4)
+/* FW and ROM share offset 4 */
+#define HDA_DSP_SRAM_REG_FW_STATUS		(HDA_DSP_MBOX_OFFSET + 0x4)
+#define HDA_DSP_SRAM_REG_FW_TRACEP		(HDA_DSP_MBOX_OFFSET + 0x8)
+#define HDA_DSP_SRAM_REG_FW_END			(HDA_DSP_MBOX_OFFSET + 0xc)
+
+#define HDA_DSP_MBOX_UPLINK_OFFSET		0x81000
+
+#define HDA_DSP_STREAM_RESET_TIMEOUT		300
+#define HDA_DSP_CL_TRIGGER_TIMEOUT		300
+
+#define HDA_DSP_SPIB_ENABLE			1
+#define HDA_DSP_SPIB_DISABLE			0
+
+#define SOF_HDA_MAX_BUFFER_SIZE			(32 * PAGE_SIZE)
+
+#define HDA_DSP_STACK_DUMP_SIZE			32
+
+/* ROM  status/error values */
+#define HDA_DSP_ROM_STS_MASK			0xf
+#define HDA_DSP_ROM_INIT			0x1
+#define HDA_DSP_ROM_FW_MANIFEST_LOADED		0x3
+#define HDA_DSP_ROM_FW_FW_LOADED		0x4
+#define HDA_DSP_ROM_FW_ENTERED			0x5
+#define HDA_DSP_ROM_RFW_START			0xf
+#define HDA_DSP_ROM_CSE_ERROR			40
+#define HDA_DSP_ROM_CSE_WRONG_RESPONSE		41
+#define HDA_DSP_ROM_IMR_TO_SMALL		42
+#define HDA_DSP_ROM_BASE_FW_NOT_FOUND		43
+#define HDA_DSP_ROM_CSE_VALIDATION_FAILED	44
+#define HDA_DSP_ROM_IPC_FATAL_ERROR		45
+#define HDA_DSP_ROM_L2_CACHE_ERROR		46
+#define HDA_DSP_ROM_LOAD_OFFSET_TO_SMALL	47
+#define HDA_DSP_ROM_API_PTR_INVALID		50
+#define HDA_DSP_ROM_BASEFW_INCOMPAT		51
+#define HDA_DSP_ROM_UNHANDLED_INTERRUPT		0xBEE00000
+#define HDA_DSP_ROM_MEMORY_HOLE_ECC		0xECC00000
+#define HDA_DSP_ROM_KERNEL_EXCEPTION		0xCAFE0000
+#define HDA_DSP_ROM_USER_EXCEPTION		0xBEEF0000
+#define HDA_DSP_ROM_UNEXPECTED_RESET		0xDECAF000
+#define HDA_DSP_ROM_NULL_FW_ENTRY		0x4c4c4e55
+#define HDA_DSP_IPC_PURGE_FW			0x01004000
+
+/* various timeout values */
+#define HDA_DSP_PU_TIMEOUT		50
+#define HDA_DSP_PD_TIMEOUT		50
+#define HDA_DSP_RESET_TIMEOUT_US	50000
+#define HDA_DSP_BASEFW_TIMEOUT_US       3000000
+#define HDA_DSP_INIT_TIMEOUT_US	500000
+#define HDA_DSP_CTRL_RESET_TIMEOUT		100
+#define HDA_DSP_WAIT_TIMEOUT		500	/* 500 msec */
+#define HDA_DSP_REG_POLL_INTERVAL_US		500	/* 0.5 msec */
+
+#define HDA_DSP_ADSPIC_IPC			1
+#define HDA_DSP_ADSPIS_IPC			1
+
+/* Intel HD Audio General DSP Registers */
+#define HDA_DSP_GEN_BASE		0x0
+#define HDA_DSP_REG_ADSPCS		(HDA_DSP_GEN_BASE + 0x04)
+#define HDA_DSP_REG_ADSPIC		(HDA_DSP_GEN_BASE + 0x08)
+#define HDA_DSP_REG_ADSPIS		(HDA_DSP_GEN_BASE + 0x0C)
+#define HDA_DSP_REG_ADSPIC2		(HDA_DSP_GEN_BASE + 0x10)
+#define HDA_DSP_REG_ADSPIS2		(HDA_DSP_GEN_BASE + 0x14)
+
+/* Intel HD Audio Inter-Processor Communication Registers */
+#define HDA_DSP_IPC_BASE		0x40
+#define HDA_DSP_REG_HIPCT		(HDA_DSP_IPC_BASE + 0x00)
+#define HDA_DSP_REG_HIPCTE		(HDA_DSP_IPC_BASE + 0x04)
+#define HDA_DSP_REG_HIPCI		(HDA_DSP_IPC_BASE + 0x08)
+#define HDA_DSP_REG_HIPCIE		(HDA_DSP_IPC_BASE + 0x0C)
+#define HDA_DSP_REG_HIPCCTL		(HDA_DSP_IPC_BASE + 0x10)
+
+/*  HIPCI */
+#define HDA_DSP_REG_HIPCI_BUSY		BIT(31)
+#define HDA_DSP_REG_HIPCI_MSG_MASK	0x7FFFFFFF
+
+/* HIPCIE */
+#define HDA_DSP_REG_HIPCIE_DONE	BIT(30)
+#define HDA_DSP_REG_HIPCIE_MSG_MASK	0x3FFFFFFF
+
+/* HIPCCTL */
+#define HDA_DSP_REG_HIPCCTL_DONE	BIT(1)
+#define HDA_DSP_REG_HIPCCTL_BUSY	BIT(0)
+
+/* HIPCT */
+#define HDA_DSP_REG_HIPCT_BUSY		BIT(31)
+#define HDA_DSP_REG_HIPCT_MSG_MASK	0x7FFFFFFF
+
+/* HIPCTE */
+#define HDA_DSP_REG_HIPCTE_MSG_MASK	0x3FFFFFFF
+
+#define HDA_DSP_ADSPIC_CL_DMA		0x2
+#define HDA_DSP_ADSPIS_CL_DMA		0x2
+
+/* Delay before scheduling D0i3 entry */
+#define BXT_D0I3_DELAY 5000
+
+#define FW_CL_STREAM_NUMBER		0x1
+
+/* ADSPCS - Audio DSP Control & Status */
+
+/*
+ * Core Reset - asserted high
+ * CRST Mask for a given core mask pattern, cm
+ */
+#define HDA_DSP_ADSPCS_CRST_SHIFT	0
+#define HDA_DSP_ADSPCS_CRST_MASK(cm)	((cm) << HDA_DSP_ADSPCS_CRST_SHIFT)
+
+/*
+ * Core run/stall - when set to '1' core is stalled
+ * CSTALL Mask for a given core mask pattern, cm
+ */
+#define HDA_DSP_ADSPCS_CSTALL_SHIFT	8
+#define HDA_DSP_ADSPCS_CSTALL_MASK(cm)	((cm) << HDA_DSP_ADSPCS_CSTALL_SHIFT)
+
+/*
+ * Set Power Active - when set to '1' turn cores on
+ * SPA Mask for a given core mask pattern, cm
+ */
+#define HDA_DSP_ADSPCS_SPA_SHIFT	16
+#define HDA_DSP_ADSPCS_SPA_MASK(cm)	((cm) << HDA_DSP_ADSPCS_SPA_SHIFT)
+
+/*
+ * Current Power Active - power status of cores, set by hardware
+ * CPA Mask for a given core mask pattern, cm
+ */
+#define HDA_DSP_ADSPCS_CPA_SHIFT	24
+#define HDA_DSP_ADSPCS_CPA_MASK(cm)	((cm) << HDA_DSP_ADSPCS_CPA_SHIFT)
+
+/* Mask for a given core index, c = 0.. number of supported cores - 1 */
+#define HDA_DSP_CORE_MASK(c)		BIT(c)
+
+/*
+ * Mask for a given number of cores
+ * nc = number of supported cores
+ */
+#define SOF_DSP_CORES_MASK(nc)	GENMASK(((nc) - 1), 0)
+
+/* Intel HD Audio Inter-Processor Communication Registers for Cannonlake*/
+#define CNL_DSP_IPC_BASE		0xc0
+#define CNL_DSP_REG_HIPCTDR		(CNL_DSP_IPC_BASE + 0x00)
+#define CNL_DSP_REG_HIPCTDA		(CNL_DSP_IPC_BASE + 0x04)
+#define CNL_DSP_REG_HIPCTDD		(CNL_DSP_IPC_BASE + 0x08)
+#define CNL_DSP_REG_HIPCIDR		(CNL_DSP_IPC_BASE + 0x10)
+#define CNL_DSP_REG_HIPCIDA		(CNL_DSP_IPC_BASE + 0x14)
+#define CNL_DSP_REG_HIPCCTL		(CNL_DSP_IPC_BASE + 0x28)
+
+/*  HIPCI */
+#define CNL_DSP_REG_HIPCIDR_BUSY		BIT(31)
+#define CNL_DSP_REG_HIPCIDR_MSG_MASK	0x7FFFFFFF
+
+/* HIPCIE */
+#define CNL_DSP_REG_HIPCIDA_DONE	BIT(31)
+#define CNL_DSP_REG_HIPCIDA_MSG_MASK	0x7FFFFFFF
+
+/* HIPCCTL */
+#define CNL_DSP_REG_HIPCCTL_DONE	BIT(1)
+#define CNL_DSP_REG_HIPCCTL_BUSY	BIT(0)
+
+/* HIPCT */
+#define CNL_DSP_REG_HIPCTDR_BUSY		BIT(31)
+#define CNL_DSP_REG_HIPCTDR_MSG_MASK	0x7FFFFFFF
+
+/* HIPCTDA */
+#define CNL_DSP_REG_HIPCTDA_DONE	BIT(31)
+#define CNL_DSP_REG_HIPCTDA_MSG_MASK	0x7FFFFFFF
+
+/* HIPCTDD */
+#define CNL_DSP_REG_HIPCTDD_MSG_MASK	0x7FFFFFFF
+
+/* BDL */
+#define HDA_DSP_BDL_SIZE			4096
+#define HDA_DSP_MAX_BDL_ENTRIES			\
+	(HDA_DSP_BDL_SIZE / sizeof(struct sof_intel_dsp_bdl))
+
+/* Number of DAIs */
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
+#define SOF_SKL_NUM_DAIS		14
+#else
+#define SOF_SKL_NUM_DAIS		8
+#endif
+
+/* Intel HD Audio SRAM Window 0*/
+#define HDA_ADSP_SRAM0_BASE_SKL		0x8000
+
+/* Firmware status window */
+#define HDA_ADSP_FW_STATUS_SKL		HDA_ADSP_SRAM0_BASE_SKL
+#define HDA_ADSP_ERROR_CODE_SKL		(HDA_ADSP_FW_STATUS_SKL + 0x4)
+
+/* Host Device Memory Space */
+#define APL_SSP_BASE_OFFSET	0x2000
+#define CNL_SSP_BASE_OFFSET	0x10000
+
+/* Host Device Memory Size of a Single SSP */
+#define SSP_DEV_MEM_SIZE	0x1000
+
+/* SSP Count of the Platform */
+#define APL_SSP_COUNT		6
+#define CNL_SSP_COUNT		3
+
+/* SSP Registers */
+#define SSP_SSC1_OFFSET		0x4
+#define SSP_SET_SCLK_SLAVE	BIT(25)
+#define SSP_SET_SFRM_SLAVE	BIT(24)
+#define SSP_SET_SLAVE		(SSP_SET_SCLK_SLAVE | SSP_SET_SFRM_SLAVE)
+
+#define HDA_IDISP_CODEC(x) ((x) & BIT(2))
+
+struct sof_intel_dsp_bdl {
+	__le32 addr_l;
+	__le32 addr_h;
+	__le32 size;
+	__le32 ioc;
+} __attribute((packed));
+
+#define SOF_HDA_PLAYBACK_STREAMS	16
+#define SOF_HDA_CAPTURE_STREAMS		16
+#define SOF_HDA_PLAYBACK		0
+#define SOF_HDA_CAPTURE			1
+
+/* represents DSP HDA controller frontend - i.e. host facing control */
+struct sof_intel_hda_dev {
+
+	struct hda_bus hbus;
+
+	/* hw config */
+	const struct sof_intel_dsp_desc *desc;
+
+	/* trace */
+	struct hdac_ext_stream *dtrace_stream;
+
+	/* if position update IPC needed */
+	u32 no_ipc_position;
+
+	/* the maximum number of streams (playback + capture) supported */
+	u32 stream_max;
+
+	int irq;
+
+	/* DMIC device */
+	struct platform_device *dmic_dev;
+};
+
+static inline struct hdac_bus *sof_to_bus(struct snd_sof_dev *s)
+{
+	struct sof_intel_hda_dev *hda = s->pdata->hw_pdata;
+
+	return &hda->hbus.core;
+}
+
+static inline struct hda_bus *sof_to_hbus(struct snd_sof_dev *s)
+{
+	struct sof_intel_hda_dev *hda = s->pdata->hw_pdata;
+
+	return &hda->hbus;
+}
+
+struct sof_intel_hda_stream {
+	struct hdac_ext_stream hda_stream;
+	struct sof_intel_stream stream;
+	int hw_params_upon_resume; /* set up hw_params upon resume */
+};
+
+#define bus_to_sof_hda(bus) \
+	container_of(bus, struct sof_intel_hda_dev, hbus.core)
+
+#define SOF_STREAM_SD_OFFSET(s) \
+	(SOF_HDA_ADSP_SD_ENTRY_SIZE * ((s)->index) \
+	 + SOF_HDA_ADSP_LOADER_BASE)
+
+/*
+ * DSP Core services.
+ */
+int hda_dsp_probe(struct snd_sof_dev *sdev);
+int hda_dsp_remove(struct snd_sof_dev *sdev);
+int hda_dsp_core_reset_enter(struct snd_sof_dev *sdev,
+			     unsigned int core_mask);
+int hda_dsp_core_reset_leave(struct snd_sof_dev *sdev,
+			     unsigned int core_mask);
+int hda_dsp_core_stall_reset(struct snd_sof_dev *sdev, unsigned int core_mask);
+int hda_dsp_core_run(struct snd_sof_dev *sdev, unsigned int core_mask);
+int hda_dsp_core_power_up(struct snd_sof_dev *sdev, unsigned int core_mask);
+int hda_dsp_enable_core(struct snd_sof_dev *sdev, unsigned int core_mask);
+int hda_dsp_core_power_down(struct snd_sof_dev *sdev, unsigned int core_mask);
+bool hda_dsp_core_is_enabled(struct snd_sof_dev *sdev,
+			     unsigned int core_mask);
+int hda_dsp_core_reset_power_down(struct snd_sof_dev *sdev,
+				  unsigned int core_mask);
+void hda_dsp_ipc_int_enable(struct snd_sof_dev *sdev);
+void hda_dsp_ipc_int_disable(struct snd_sof_dev *sdev);
+
+int hda_dsp_suspend(struct snd_sof_dev *sdev, int state);
+int hda_dsp_resume(struct snd_sof_dev *sdev);
+int hda_dsp_runtime_suspend(struct snd_sof_dev *sdev, int state);
+int hda_dsp_runtime_resume(struct snd_sof_dev *sdev);
+void hda_dsp_set_hw_params_upon_resume(struct snd_sof_dev *sdev);
+void hda_dsp_dump_skl(struct snd_sof_dev *sdev, u32 flags);
+void hda_dsp_dump(struct snd_sof_dev *sdev, u32 flags);
+void hda_ipc_dump(struct snd_sof_dev *sdev);
+
+/*
+ * DSP PCM Operations.
+ */
+int hda_dsp_pcm_open(struct snd_sof_dev *sdev,
+		     struct snd_pcm_substream *substream);
+int hda_dsp_pcm_close(struct snd_sof_dev *sdev,
+		      struct snd_pcm_substream *substream);
+int hda_dsp_pcm_hw_params(struct snd_sof_dev *sdev,
+			  struct snd_pcm_substream *substream,
+			  struct snd_pcm_hw_params *params,
+			  struct sof_ipc_stream_params *ipc_params);
+int hda_dsp_pcm_trigger(struct snd_sof_dev *sdev,
+			struct snd_pcm_substream *substream, int cmd);
+snd_pcm_uframes_t hda_dsp_pcm_pointer(struct snd_sof_dev *sdev,
+				      struct snd_pcm_substream *substream);
+
+/*
+ * DSP Stream Operations.
+ */
+
+int hda_dsp_stream_init(struct snd_sof_dev *sdev);
+void hda_dsp_stream_free(struct snd_sof_dev *sdev);
+int hda_dsp_stream_hw_params(struct snd_sof_dev *sdev,
+			     struct hdac_ext_stream *stream,
+			     struct snd_dma_buffer *dmab,
+			     struct snd_pcm_hw_params *params);
+int hda_dsp_stream_trigger(struct snd_sof_dev *sdev,
+			   struct hdac_ext_stream *stream, int cmd);
+irqreturn_t hda_dsp_stream_interrupt(int irq, void *context);
+irqreturn_t hda_dsp_stream_threaded_handler(int irq, void *context);
+int hda_dsp_stream_setup_bdl(struct snd_sof_dev *sdev,
+			     struct snd_dma_buffer *dmab,
+			     struct hdac_stream *stream);
+
+struct hdac_ext_stream *
+	hda_dsp_stream_get(struct snd_sof_dev *sdev, int direction);
+int hda_dsp_stream_put(struct snd_sof_dev *sdev, int direction, int stream_tag);
+int hda_dsp_stream_spib_config(struct snd_sof_dev *sdev,
+			       struct hdac_ext_stream *stream,
+			       int enable, u32 size);
+
+void hda_ipc_msg_data(struct snd_sof_dev *sdev,
+		      struct snd_pcm_substream *substream,
+		      void *p, size_t sz);
+int hda_ipc_pcm_params(struct snd_sof_dev *sdev,
+		       struct snd_pcm_substream *substream,
+		       const struct sof_ipc_pcm_params_reply *reply);
+
+/*
+ * DSP IPC Operations.
+ */
+int hda_dsp_ipc_send_msg(struct snd_sof_dev *sdev,
+			 struct snd_sof_ipc_msg *msg);
+void hda_dsp_ipc_get_reply(struct snd_sof_dev *sdev);
+int hda_dsp_ipc_fw_ready(struct snd_sof_dev *sdev, u32 msg_id);
+irqreturn_t hda_dsp_ipc_irq_handler(int irq, void *context);
+irqreturn_t hda_dsp_ipc_irq_thread(int irq, void *context);
+int hda_dsp_ipc_cmd_done(struct snd_sof_dev *sdev, int dir);
+
+/*
+ * DSP Code loader.
+ */
+int hda_dsp_cl_boot_firmware(struct snd_sof_dev *sdev);
+int hda_dsp_cl_boot_firmware_skl(struct snd_sof_dev *sdev);
+
+/* pre and post fw run ops */
+int hda_dsp_pre_fw_run(struct snd_sof_dev *sdev);
+int hda_dsp_post_fw_run(struct snd_sof_dev *sdev);
+
+/*
+ * HDA Controller Operations.
+ */
+int hda_dsp_ctrl_get_caps(struct snd_sof_dev *sdev);
+void hda_dsp_ctrl_ppcap_enable(struct snd_sof_dev *sdev, bool enable);
+void hda_dsp_ctrl_ppcap_int_enable(struct snd_sof_dev *sdev, bool enable);
+int hda_dsp_ctrl_link_reset(struct snd_sof_dev *sdev, bool reset);
+void hda_dsp_ctrl_misc_clock_gating(struct snd_sof_dev *sdev, bool enable);
+int hda_dsp_ctrl_clock_power_gating(struct snd_sof_dev *sdev, bool enable);
+int hda_dsp_ctrl_init_chip(struct snd_sof_dev *sdev, bool full_reset);
+
+/*
+ * HDA bus operations.
+ */
+void sof_hda_bus_init(struct hdac_bus *bus, struct device *dev,
+		      const struct hdac_ext_bus_ops *ext_ops);
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
+/*
+ * HDA Codec operations.
+ */
+int hda_codec_probe_bus(struct snd_sof_dev *sdev);
+
+#endif /* CONFIG_SND_SOC_SOF_HDA */
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA) && IS_ENABLED(CONFIG_SND_SOC_HDAC_HDMI)
+
+void hda_codec_i915_get(struct snd_sof_dev *sdev);
+void hda_codec_i915_put(struct snd_sof_dev *sdev);
+int hda_codec_i915_init(struct snd_sof_dev *sdev);
+int hda_codec_i915_exit(struct snd_sof_dev *sdev);
+
+#else
+
+static inline void hda_codec_i915_get(struct snd_sof_dev *sdev)  { }
+static inline void hda_codec_i915_put(struct snd_sof_dev *sdev)  { }
+static inline int hda_codec_i915_init(struct snd_sof_dev *sdev) { return 0; }
+static inline int hda_codec_i915_exit(struct snd_sof_dev *sdev) { return 0; }
+
+#endif /* CONFIG_SND_SOC_SOF_HDA && CONFIG_SND_SOC_HDAC_HDMI */
+
+/*
+ * Trace Control.
+ */
+int hda_dsp_trace_init(struct snd_sof_dev *sdev, u32 *stream_tag);
+int hda_dsp_trace_release(struct snd_sof_dev *sdev);
+int hda_dsp_trace_trigger(struct snd_sof_dev *sdev, int cmd);
+
+/* common dai driver */
+extern struct snd_soc_dai_driver skl_dai[];
+
+/*
+ * Platform Specific HW abstraction Ops.
+ */
+extern const struct snd_sof_dsp_ops sof_apl_ops;
+extern const struct snd_sof_dsp_ops sof_cnl_ops;
+extern const struct snd_sof_dsp_ops sof_skl_ops;
+
+extern const struct sof_intel_dsp_desc apl_chip_info;
+extern const struct sof_intel_dsp_desc cnl_chip_info;
+extern const struct sof_intel_dsp_desc skl_chip_info;
+
+#endif
diff --git a/sound/soc/sof/intel/intel-ipc.c b/sound/soc/sof/intel/intel-ipc.c
new file mode 100644
index 000000000000..4edd92151fd5
--- /dev/null
+++ b/sound/soc/sof/intel/intel-ipc.c
@@ -0,0 +1,92 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license.  When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2019 Intel Corporation. All rights reserved.
+//
+// Authors: Guennadi Liakhovetski <guennadi.liakhovetski@linux.intel.com>
+
+/* Intel-specific SOF IPC code */
+
+#include <linux/device.h>
+#include <linux/export.h>
+#include <linux/module.h>
+#include <linux/types.h>
+
+#include <sound/pcm.h>
+#include <sound/sof/stream.h>
+
+#include "../ops.h"
+#include "../sof-priv.h"
+
+struct intel_stream {
+	size_t posn_offset;
+};
+
+/* Mailbox-based Intel IPC implementation */
+void intel_ipc_msg_data(struct snd_sof_dev *sdev,
+			struct snd_pcm_substream *substream,
+			void *p, size_t sz)
+{
+	if (!substream || !sdev->stream_box.size) {
+		sof_mailbox_read(sdev, sdev->dsp_box.offset, p, sz);
+	} else {
+		struct intel_stream *stream = substream->runtime->private_data;
+
+		/* The stream might already be closed */
+		if (stream)
+			sof_mailbox_read(sdev, stream->posn_offset, p, sz);
+	}
+}
+EXPORT_SYMBOL(intel_ipc_msg_data);
+
+int intel_ipc_pcm_params(struct snd_sof_dev *sdev,
+			 struct snd_pcm_substream *substream,
+			 const struct sof_ipc_pcm_params_reply *reply)
+{
+	struct intel_stream *stream = substream->runtime->private_data;
+	size_t posn_offset = reply->posn_offset;
+
+	/* check if offset is overflow or it is not aligned */
+	if (posn_offset > sdev->stream_box.size ||
+	    posn_offset % sizeof(struct sof_ipc_stream_posn) != 0)
+		return -EINVAL;
+
+	stream->posn_offset = sdev->stream_box.offset + posn_offset;
+
+	dev_dbg(sdev->dev, "pcm: stream dir %d, posn mailbox offset is %zu",
+		substream->stream, stream->posn_offset);
+
+	return 0;
+}
+EXPORT_SYMBOL(intel_ipc_pcm_params);
+
+int intel_pcm_open(struct snd_sof_dev *sdev,
+		   struct snd_pcm_substream *substream)
+{
+	struct intel_stream *stream = kmalloc(sizeof(*stream), GFP_KERNEL);
+
+	if (!stream)
+		return -ENOMEM;
+
+	/* binding pcm substream to hda stream */
+	substream->runtime->private_data = stream;
+
+	return 0;
+}
+EXPORT_SYMBOL(intel_pcm_open);
+
+int intel_pcm_close(struct snd_sof_dev *sdev,
+		    struct snd_pcm_substream *substream)
+{
+	struct intel_stream *stream = substream->runtime->private_data;
+
+	substream->runtime->private_data = NULL;
+	kfree(stream);
+
+	return 0;
+}
+EXPORT_SYMBOL(intel_pcm_close);
+
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/sound/soc/sof/intel/shim.h b/sound/soc/sof/intel/shim.h
new file mode 100644
index 000000000000..f7a3f62e45d4
--- /dev/null
+++ b/sound/soc/sof/intel/shim.h
@@ -0,0 +1,185 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */
+/*
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * Copyright(c) 2017 Intel Corporation. All rights reserved.
+ *
+ * Author: Liam Girdwood <liam.r.girdwood@linux.intel.com>
+ */
+
+#ifndef __SOF_INTEL_SHIM_H
+#define __SOF_INTEL_SHIM_H
+
+/*
+ * SHIM registers for BYT, BSW, CHT, HSW, BDW
+ */
+
+#define SHIM_CSR		(SHIM_OFFSET + 0x00)
+#define SHIM_PISR		(SHIM_OFFSET + 0x08)
+#define SHIM_PIMR		(SHIM_OFFSET + 0x10)
+#define SHIM_ISRX		(SHIM_OFFSET + 0x18)
+#define SHIM_ISRD		(SHIM_OFFSET + 0x20)
+#define SHIM_IMRX		(SHIM_OFFSET + 0x28)
+#define SHIM_IMRD		(SHIM_OFFSET + 0x30)
+#define SHIM_IPCX		(SHIM_OFFSET + 0x38)
+#define SHIM_IPCD		(SHIM_OFFSET + 0x40)
+#define SHIM_ISRSC		(SHIM_OFFSET + 0x48)
+#define SHIM_ISRLPESC		(SHIM_OFFSET + 0x50)
+#define SHIM_IMRSC		(SHIM_OFFSET + 0x58)
+#define SHIM_IMRLPESC		(SHIM_OFFSET + 0x60)
+#define SHIM_IPCSC		(SHIM_OFFSET + 0x68)
+#define SHIM_IPCLPESC		(SHIM_OFFSET + 0x70)
+#define SHIM_CLKCTL		(SHIM_OFFSET + 0x78)
+#define SHIM_CSR2		(SHIM_OFFSET + 0x80)
+#define SHIM_LTRC		(SHIM_OFFSET + 0xE0)
+#define SHIM_HMDC		(SHIM_OFFSET + 0xE8)
+
+#define SHIM_PWMCTRL		0x1000
+
+/*
+ * SST SHIM register bits for BYT, BSW, CHT HSW, BDW
+ * Register bit naming and functionaility can differ between devices.
+ */
+
+/* CSR / CS */
+#define SHIM_CSR_RST		BIT(1)
+#define SHIM_CSR_SBCS0		BIT(2)
+#define SHIM_CSR_SBCS1		BIT(3)
+#define SHIM_CSR_DCS(x)		((x) << 4)
+#define SHIM_CSR_DCS_MASK	(0x7 << 4)
+#define SHIM_CSR_STALL		BIT(10)
+#define SHIM_CSR_S0IOCS		BIT(21)
+#define SHIM_CSR_S1IOCS		BIT(23)
+#define SHIM_CSR_LPCS		BIT(31)
+#define SHIM_CSR_24MHZ_LPCS \
+	(SHIM_CSR_SBCS0 | SHIM_CSR_SBCS1 | SHIM_CSR_LPCS)
+#define SHIM_CSR_24MHZ_NO_LPCS	(SHIM_CSR_SBCS0 | SHIM_CSR_SBCS1)
+#define SHIM_BYT_CSR_RST	BIT(0)
+#define SHIM_BYT_CSR_VECTOR_SEL	BIT(1)
+#define SHIM_BYT_CSR_STALL	BIT(2)
+#define SHIM_BYT_CSR_PWAITMODE	BIT(3)
+
+/*  ISRX / ISC */
+#define SHIM_ISRX_BUSY		BIT(1)
+#define SHIM_ISRX_DONE		BIT(0)
+#define SHIM_BYT_ISRX_REQUEST	BIT(1)
+
+/*  ISRD / ISD */
+#define SHIM_ISRD_BUSY		BIT(1)
+#define SHIM_ISRD_DONE		BIT(0)
+
+/* IMRX / IMC */
+#define SHIM_IMRX_BUSY		BIT(1)
+#define SHIM_IMRX_DONE		BIT(0)
+#define SHIM_BYT_IMRX_REQUEST	BIT(1)
+
+/* IMRD / IMD */
+#define SHIM_IMRD_DONE		BIT(0)
+#define SHIM_IMRD_BUSY		BIT(1)
+#define SHIM_IMRD_SSP0		BIT(16)
+#define SHIM_IMRD_DMAC0		BIT(21)
+#define SHIM_IMRD_DMAC1		BIT(22)
+#define SHIM_IMRD_DMAC		(SHIM_IMRD_DMAC0 | SHIM_IMRD_DMAC1)
+
+/*  IPCX / IPCC */
+#define	SHIM_IPCX_DONE		BIT(30)
+#define	SHIM_IPCX_BUSY		BIT(31)
+#define SHIM_BYT_IPCX_DONE	BIT_ULL(62)
+#define SHIM_BYT_IPCX_BUSY	BIT_ULL(63)
+
+/*  IPCD */
+#define	SHIM_IPCD_DONE		BIT(30)
+#define	SHIM_IPCD_BUSY		BIT(31)
+#define SHIM_BYT_IPCD_DONE	BIT_ULL(62)
+#define SHIM_BYT_IPCD_BUSY	BIT_ULL(63)
+
+/* CLKCTL */
+#define SHIM_CLKCTL_SMOS(x)	((x) << 24)
+#define SHIM_CLKCTL_MASK	(3 << 24)
+#define SHIM_CLKCTL_DCPLCG	BIT(18)
+#define SHIM_CLKCTL_SCOE1	BIT(17)
+#define SHIM_CLKCTL_SCOE0	BIT(16)
+
+/* CSR2 / CS2 */
+#define SHIM_CSR2_SDFD_SSP0	BIT(1)
+#define SHIM_CSR2_SDFD_SSP1	BIT(2)
+
+/* LTRC */
+#define SHIM_LTRC_VAL(x)	((x) << 0)
+
+/* HMDC */
+#define SHIM_HMDC_HDDA0(x)	((x) << 0)
+#define SHIM_HMDC_HDDA1(x)	((x) << 7)
+#define SHIM_HMDC_HDDA_E0_CH0	1
+#define SHIM_HMDC_HDDA_E0_CH1	2
+#define SHIM_HMDC_HDDA_E0_CH2	4
+#define SHIM_HMDC_HDDA_E0_CH3	8
+#define SHIM_HMDC_HDDA_E1_CH0	SHIM_HMDC_HDDA1(SHIM_HMDC_HDDA_E0_CH0)
+#define SHIM_HMDC_HDDA_E1_CH1	SHIM_HMDC_HDDA1(SHIM_HMDC_HDDA_E0_CH1)
+#define SHIM_HMDC_HDDA_E1_CH2	SHIM_HMDC_HDDA1(SHIM_HMDC_HDDA_E0_CH2)
+#define SHIM_HMDC_HDDA_E1_CH3	SHIM_HMDC_HDDA1(SHIM_HMDC_HDDA_E0_CH3)
+#define SHIM_HMDC_HDDA_E0_ALLCH	\
+	(SHIM_HMDC_HDDA_E0_CH0 | SHIM_HMDC_HDDA_E0_CH1 | \
+	 SHIM_HMDC_HDDA_E0_CH2 | SHIM_HMDC_HDDA_E0_CH3)
+#define SHIM_HMDC_HDDA_E1_ALLCH	\
+	(SHIM_HMDC_HDDA_E1_CH0 | SHIM_HMDC_HDDA_E1_CH1 | \
+	 SHIM_HMDC_HDDA_E1_CH2 | SHIM_HMDC_HDDA_E1_CH3)
+
+/* Audio DSP PCI registers */
+#define PCI_VDRTCTL0		0xa0
+#define PCI_VDRTCTL1		0xa4
+#define PCI_VDRTCTL2		0xa8
+#define PCI_VDRTCTL3		0xaC
+
+/* VDRTCTL0 */
+#define PCI_VDRTCL0_D3PGD		BIT(0)
+#define PCI_VDRTCL0_D3SRAMPGD		BIT(1)
+#define PCI_VDRTCL0_DSRAMPGE_SHIFT	12
+#define PCI_VDRTCL0_DSRAMPGE_MASK	GENMASK(PCI_VDRTCL0_DSRAMPGE_SHIFT + 19,\
+						PCI_VDRTCL0_DSRAMPGE_SHIFT)
+#define PCI_VDRTCL0_ISRAMPGE_SHIFT	2
+#define PCI_VDRTCL0_ISRAMPGE_MASK	GENMASK(PCI_VDRTCL0_ISRAMPGE_SHIFT + 9,\
+						PCI_VDRTCL0_ISRAMPGE_SHIFT)
+
+/* VDRTCTL2 */
+#define PCI_VDRTCL2_DCLCGE		BIT(1)
+#define PCI_VDRTCL2_DTCGE		BIT(10)
+#define PCI_VDRTCL2_APLLSE_MASK		BIT(31)
+
+/* PMCS */
+#define PCI_PMCS		0x84
+#define PCI_PMCS_PS_MASK	0x3
+
+/* DSP hardware descriptor */
+struct sof_intel_dsp_desc {
+	int cores_num;
+	int cores_mask;
+	int init_core_mask; /* cores available after fw boot */
+	int ipc_req;
+	int ipc_req_mask;
+	int ipc_ack;
+	int ipc_ack_mask;
+	int ipc_ctl;
+	int rom_init_timeout;
+	int ssp_count;			/* ssp count of the platform */
+	int ssp_base_offset;		/* base address of the SSPs */
+};
+
+extern const struct snd_sof_dsp_ops sof_tng_ops;
+extern const struct snd_sof_dsp_ops sof_byt_ops;
+extern const struct snd_sof_dsp_ops sof_cht_ops;
+extern const struct snd_sof_dsp_ops sof_hsw_ops;
+extern const struct snd_sof_dsp_ops sof_bdw_ops;
+
+extern const struct sof_intel_dsp_desc byt_chip_info;
+extern const struct sof_intel_dsp_desc cht_chip_info;
+extern const struct sof_intel_dsp_desc bdw_chip_info;
+extern const struct sof_intel_dsp_desc hsw_chip_info;
+extern const struct sof_intel_dsp_desc tng_chip_info;
+
+struct sof_intel_stream {
+	size_t posn_offset;
+};
+
+#endif
diff --git a/sound/soc/sof/ipc.c b/sound/soc/sof/ipc.c
new file mode 100644
index 000000000000..f0b9d3c53f6f
--- /dev/null
+++ b/sound/soc/sof/ipc.c
@@ -0,0 +1,846 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license.  When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2018 Intel Corporation. All rights reserved.
+//
+// Author: Liam Girdwood <liam.r.girdwood@linux.intel.com>
+//
+// Generic IPC layer that can work over MMIO and SPI/I2C. PHY layer provided
+// by platform driver code.
+//
+
+#include <linux/mutex.h>
+#include <linux/types.h>
+
+#include "sof-priv.h"
+#include "ops.h"
+
+/*
+ * IPC message default size and timeout (ms).
+ * TODO: allow platforms to set size and timeout.
+ */
+#define IPC_TIMEOUT_MS		300
+
+static void ipc_trace_message(struct snd_sof_dev *sdev, u32 msg_id);
+static void ipc_stream_message(struct snd_sof_dev *sdev, u32 msg_cmd);
+
+/*
+ * IPC message Tx/Rx message handling.
+ */
+
+/* SOF generic IPC data */
+struct snd_sof_ipc {
+	struct snd_sof_dev *sdev;
+
+	/* protects messages and the disable flag */
+	struct mutex tx_mutex;
+	/* disables further sending of ipc's */
+	bool disable_ipc_tx;
+
+	struct snd_sof_ipc_msg msg;
+};
+
+struct sof_ipc_ctrl_data_params {
+	size_t msg_bytes;
+	size_t hdr_bytes;
+	size_t pl_size;
+	size_t elems;
+	u32 num_msg;
+	u8 *src;
+	u8 *dst;
+};
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG_VERBOSE_IPC)
+static void ipc_log_header(struct device *dev, u8 *text, u32 cmd)
+{
+	u8 *str;
+	u8 *str2 = NULL;
+	u32 glb;
+	u32 type;
+
+	glb = cmd & SOF_GLB_TYPE_MASK;
+	type = cmd & SOF_CMD_TYPE_MASK;
+
+	switch (glb) {
+	case SOF_IPC_GLB_REPLY:
+		str = "GLB_REPLY"; break;
+	case SOF_IPC_GLB_COMPOUND:
+		str = "GLB_COMPOUND"; break;
+	case SOF_IPC_GLB_TPLG_MSG:
+		str = "GLB_TPLG_MSG";
+		switch (type) {
+		case SOF_IPC_TPLG_COMP_NEW:
+			str2 = "COMP_NEW"; break;
+		case SOF_IPC_TPLG_COMP_FREE:
+			str2 = "COMP_FREE"; break;
+		case SOF_IPC_TPLG_COMP_CONNECT:
+			str2 = "COMP_CONNECT"; break;
+		case SOF_IPC_TPLG_PIPE_NEW:
+			str2 = "PIPE_NEW"; break;
+		case SOF_IPC_TPLG_PIPE_FREE:
+			str2 = "PIPE_FREE"; break;
+		case SOF_IPC_TPLG_PIPE_CONNECT:
+			str2 = "PIPE_CONNECT"; break;
+		case SOF_IPC_TPLG_PIPE_COMPLETE:
+			str2 = "PIPE_COMPLETE"; break;
+		case SOF_IPC_TPLG_BUFFER_NEW:
+			str2 = "BUFFER_NEW"; break;
+		case SOF_IPC_TPLG_BUFFER_FREE:
+			str2 = "BUFFER_FREE"; break;
+		default:
+			str2 = "unknown type"; break;
+		}
+		break;
+	case SOF_IPC_GLB_PM_MSG:
+		str = "GLB_PM_MSG";
+		switch (type) {
+		case SOF_IPC_PM_CTX_SAVE:
+			str2 = "CTX_SAVE"; break;
+		case SOF_IPC_PM_CTX_RESTORE:
+			str2 = "CTX_RESTORE"; break;
+		case SOF_IPC_PM_CTX_SIZE:
+			str2 = "CTX_SIZE"; break;
+		case SOF_IPC_PM_CLK_SET:
+			str2 = "CLK_SET"; break;
+		case SOF_IPC_PM_CLK_GET:
+			str2 = "CLK_GET"; break;
+		case SOF_IPC_PM_CLK_REQ:
+			str2 = "CLK_REQ"; break;
+		case SOF_IPC_PM_CORE_ENABLE:
+			str2 = "CORE_ENABLE"; break;
+		default:
+			str2 = "unknown type"; break;
+		}
+		break;
+	case SOF_IPC_GLB_COMP_MSG:
+		str = "GLB_COMP_MSG: SET_VALUE";
+		switch (type) {
+		case SOF_IPC_COMP_SET_VALUE:
+			str2 = "SET_VALUE"; break;
+		case SOF_IPC_COMP_GET_VALUE:
+			str2 = "GET_VALUE"; break;
+		case SOF_IPC_COMP_SET_DATA:
+			str2 = "SET_DATA"; break;
+		case SOF_IPC_COMP_GET_DATA:
+			str2 = "GET_DATA"; break;
+		default:
+			str2 = "unknown type"; break;
+		}
+		break;
+	case SOF_IPC_GLB_STREAM_MSG:
+		str = "GLB_STREAM_MSG";
+		switch (type) {
+		case SOF_IPC_STREAM_PCM_PARAMS:
+			str2 = "PCM_PARAMS"; break;
+		case SOF_IPC_STREAM_PCM_PARAMS_REPLY:
+			str2 = "PCM_REPLY"; break;
+		case SOF_IPC_STREAM_PCM_FREE:
+			str2 = "PCM_FREE"; break;
+		case SOF_IPC_STREAM_TRIG_START:
+			str2 = "TRIG_START"; break;
+		case SOF_IPC_STREAM_TRIG_STOP:
+			str2 = "TRIG_STOP"; break;
+		case SOF_IPC_STREAM_TRIG_PAUSE:
+			str2 = "TRIG_PAUSE"; break;
+		case SOF_IPC_STREAM_TRIG_RELEASE:
+			str2 = "TRIG_RELEASE"; break;
+		case SOF_IPC_STREAM_TRIG_DRAIN:
+			str2 = "TRIG_DRAIN"; break;
+		case SOF_IPC_STREAM_TRIG_XRUN:
+			str2 = "TRIG_XRUN"; break;
+		case SOF_IPC_STREAM_POSITION:
+			str2 = "POSITION"; break;
+		case SOF_IPC_STREAM_VORBIS_PARAMS:
+			str2 = "VORBIS_PARAMS"; break;
+		case SOF_IPC_STREAM_VORBIS_FREE:
+			str2 = "VORBIS_FREE"; break;
+		default:
+			str2 = "unknown type"; break;
+		}
+		break;
+	case SOF_IPC_FW_READY:
+		str = "FW_READY"; break;
+	case SOF_IPC_GLB_DAI_MSG:
+		str = "GLB_DAI_MSG";
+		switch (type) {
+		case SOF_IPC_DAI_CONFIG:
+			str2 = "CONFIG"; break;
+		case SOF_IPC_DAI_LOOPBACK:
+			str2 = "LOOPBACK"; break;
+		default:
+			str2 = "unknown type"; break;
+		}
+		break;
+	case SOF_IPC_GLB_TRACE_MSG:
+		str = "GLB_TRACE_MSG"; break;
+	default:
+		str = "unknown GLB command"; break;
+	}
+
+	if (str2)
+		dev_dbg(dev, "%s: 0x%x: %s: %s\n", text, cmd, str, str2);
+	else
+		dev_dbg(dev, "%s: 0x%x: %s\n", text, cmd, str);
+}
+#else
+static inline void ipc_log_header(struct device *dev, u8 *text, u32 cmd)
+{
+	dev_dbg(dev, "%s: 0x%x\n", text, cmd);
+}
+#endif
+
+/* wait for IPC message reply */
+static int tx_wait_done(struct snd_sof_ipc *ipc, struct snd_sof_ipc_msg *msg,
+			void *reply_data)
+{
+	struct snd_sof_dev *sdev = ipc->sdev;
+	struct sof_ipc_cmd_hdr *hdr = msg->msg_data;
+	int ret;
+
+	/* wait for DSP IPC completion */
+	ret = wait_event_timeout(msg->waitq, msg->ipc_complete,
+				 msecs_to_jiffies(IPC_TIMEOUT_MS));
+
+	if (ret == 0) {
+		dev_err(sdev->dev, "error: ipc timed out for 0x%x size %d\n",
+			hdr->cmd, hdr->size);
+		snd_sof_dsp_dbg_dump(ipc->sdev, SOF_DBG_REGS | SOF_DBG_MBOX);
+		snd_sof_ipc_dump(ipc->sdev);
+		snd_sof_trace_notify_for_error(ipc->sdev);
+		ret = -ETIMEDOUT;
+	} else {
+		/* copy the data returned from DSP */
+		ret = msg->reply_error;
+		if (msg->reply_size)
+			memcpy(reply_data, msg->reply_data, msg->reply_size);
+		if (ret < 0)
+			dev_err(sdev->dev, "error: ipc error for 0x%x size %zu\n",
+				hdr->cmd, msg->reply_size);
+		else
+			ipc_log_header(sdev->dev, "ipc tx succeeded", hdr->cmd);
+	}
+
+	return ret;
+}
+
+/* send IPC message from host to DSP */
+static int sof_ipc_tx_message_unlocked(struct snd_sof_ipc *ipc, u32 header,
+				       void *msg_data, size_t msg_bytes,
+				       void *reply_data, size_t reply_bytes)
+{
+	struct snd_sof_dev *sdev = ipc->sdev;
+	struct snd_sof_ipc_msg *msg;
+	int ret;
+
+	if (ipc->disable_ipc_tx)
+		return -ENODEV;
+
+	/*
+	 * The spin-lock is also still needed to protect message objects against
+	 * other atomic contexts.
+	 */
+	spin_lock_irq(&sdev->ipc_lock);
+
+	/* initialise the message */
+	msg = &ipc->msg;
+
+	msg->header = header;
+	msg->msg_size = msg_bytes;
+	msg->reply_size = reply_bytes;
+	msg->reply_error = 0;
+
+	/* attach any data */
+	if (msg_bytes)
+		memcpy(msg->msg_data, msg_data, msg_bytes);
+
+	sdev->msg = msg;
+
+	ret = snd_sof_dsp_send_msg(sdev, msg);
+	/* Next reply that we receive will be related to this message */
+	if (!ret)
+		msg->ipc_complete = false;
+
+	spin_unlock_irq(&sdev->ipc_lock);
+
+	if (ret < 0) {
+		/* So far IPC TX never fails, consider making the above void */
+		dev_err_ratelimited(sdev->dev,
+				    "error: ipc tx failed with error %d\n",
+				    ret);
+		return ret;
+	}
+
+	ipc_log_header(sdev->dev, "ipc tx", msg->header);
+
+	/* now wait for completion */
+	if (!ret)
+		ret = tx_wait_done(ipc, msg, reply_data);
+
+	return ret;
+}
+
+/* send IPC message from host to DSP */
+int sof_ipc_tx_message(struct snd_sof_ipc *ipc, u32 header,
+		       void *msg_data, size_t msg_bytes, void *reply_data,
+		       size_t reply_bytes)
+{
+	int ret;
+
+	if (msg_bytes > SOF_IPC_MSG_MAX_SIZE ||
+	    reply_bytes > SOF_IPC_MSG_MAX_SIZE)
+		return -ENOBUFS;
+
+	/* Serialise IPC TX */
+	mutex_lock(&ipc->tx_mutex);
+
+	ret = sof_ipc_tx_message_unlocked(ipc, header, msg_data, msg_bytes,
+					  reply_data, reply_bytes);
+
+	mutex_unlock(&ipc->tx_mutex);
+
+	return ret;
+}
+EXPORT_SYMBOL(sof_ipc_tx_message);
+
+/* handle reply message from DSP */
+int snd_sof_ipc_reply(struct snd_sof_dev *sdev, u32 msg_id)
+{
+	struct snd_sof_ipc_msg *msg = &sdev->ipc->msg;
+	unsigned long flags;
+
+	/*
+	 * Protect against a theoretical race with sof_ipc_tx_message(): if the
+	 * DSP is fast enough to receive an IPC message, reply to it, and the
+	 * host interrupt processing calls this function on a different core
+	 * from the one, where the sending is taking place, the message might
+	 * not yet be marked as expecting a reply.
+	 */
+	spin_lock_irqsave(&sdev->ipc_lock, flags);
+
+	if (msg->ipc_complete) {
+		spin_unlock_irqrestore(&sdev->ipc_lock, flags);
+		dev_err(sdev->dev, "error: no reply expected, received 0x%x",
+			msg_id);
+		return -EINVAL;
+	}
+
+	/* wake up and return the error if we have waiters on this message ? */
+	msg->ipc_complete = true;
+	wake_up(&msg->waitq);
+
+	spin_unlock_irqrestore(&sdev->ipc_lock, flags);
+
+	return 0;
+}
+EXPORT_SYMBOL(snd_sof_ipc_reply);
+
+/* DSP firmware has sent host a message  */
+void snd_sof_ipc_msgs_rx(struct snd_sof_dev *sdev)
+{
+	struct sof_ipc_cmd_hdr hdr;
+	u32 cmd, type;
+	int err = 0;
+
+	/* read back header */
+	snd_sof_ipc_msg_data(sdev, NULL, &hdr, sizeof(hdr));
+	ipc_log_header(sdev->dev, "ipc rx", hdr.cmd);
+
+	cmd = hdr.cmd & SOF_GLB_TYPE_MASK;
+	type = hdr.cmd & SOF_CMD_TYPE_MASK;
+
+	/* check message type */
+	switch (cmd) {
+	case SOF_IPC_GLB_REPLY:
+		dev_err(sdev->dev, "error: ipc reply unknown\n");
+		break;
+	case SOF_IPC_FW_READY:
+		/* check for FW boot completion */
+		if (!sdev->boot_complete) {
+			err = sof_ops(sdev)->fw_ready(sdev, cmd);
+			if (err < 0) {
+				/*
+				 * this indicates a mismatch in ABI
+				 * between the driver and fw
+				 */
+				dev_err(sdev->dev, "error: ABI mismatch %d\n",
+					err);
+			} else {
+				/* firmware boot completed OK */
+				sdev->boot_complete = true;
+			}
+
+			/* wake up firmware loader */
+			wake_up(&sdev->boot_wait);
+		}
+		break;
+	case SOF_IPC_GLB_COMPOUND:
+	case SOF_IPC_GLB_TPLG_MSG:
+	case SOF_IPC_GLB_PM_MSG:
+	case SOF_IPC_GLB_COMP_MSG:
+		break;
+	case SOF_IPC_GLB_STREAM_MSG:
+		/* need to pass msg id into the function */
+		ipc_stream_message(sdev, hdr.cmd);
+		break;
+	case SOF_IPC_GLB_TRACE_MSG:
+		ipc_trace_message(sdev, type);
+		break;
+	default:
+		dev_err(sdev->dev, "error: unknown DSP message 0x%x\n", cmd);
+		break;
+	}
+
+	ipc_log_header(sdev->dev, "ipc rx done", hdr.cmd);
+}
+EXPORT_SYMBOL(snd_sof_ipc_msgs_rx);
+
+/*
+ * IPC trace mechanism.
+ */
+
+static void ipc_trace_message(struct snd_sof_dev *sdev, u32 msg_id)
+{
+	struct sof_ipc_dma_trace_posn posn;
+
+	switch (msg_id) {
+	case SOF_IPC_TRACE_DMA_POSITION:
+		/* read back full message */
+		snd_sof_ipc_msg_data(sdev, NULL, &posn, sizeof(posn));
+		snd_sof_trace_update_pos(sdev, &posn);
+		break;
+	default:
+		dev_err(sdev->dev, "error: unhandled trace message %x\n",
+			msg_id);
+		break;
+	}
+}
+
+/*
+ * IPC stream position.
+ */
+
+static void ipc_period_elapsed(struct snd_sof_dev *sdev, u32 msg_id)
+{
+	struct snd_sof_pcm_stream *stream;
+	struct sof_ipc_stream_posn posn;
+	struct snd_sof_pcm *spcm;
+	int direction;
+
+	spcm = snd_sof_find_spcm_comp(sdev, msg_id, &direction);
+	if (!spcm) {
+		dev_err(sdev->dev,
+			"error: period elapsed for unknown stream, msg_id %d\n",
+			msg_id);
+		return;
+	}
+
+	stream = &spcm->stream[direction];
+	snd_sof_ipc_msg_data(sdev, stream->substream, &posn, sizeof(posn));
+
+	dev_dbg(sdev->dev, "posn : host 0x%llx dai 0x%llx wall 0x%llx\n",
+		posn.host_posn, posn.dai_posn, posn.wallclock);
+
+	memcpy(&stream->posn, &posn, sizeof(posn));
+
+	/* only inform ALSA for period_wakeup mode */
+	if (!stream->substream->runtime->no_period_wakeup)
+		snd_sof_pcm_period_elapsed(stream->substream);
+}
+
+/* DSP notifies host of an XRUN within FW */
+static void ipc_xrun(struct snd_sof_dev *sdev, u32 msg_id)
+{
+	struct snd_sof_pcm_stream *stream;
+	struct sof_ipc_stream_posn posn;
+	struct snd_sof_pcm *spcm;
+	int direction;
+
+	spcm = snd_sof_find_spcm_comp(sdev, msg_id, &direction);
+	if (!spcm) {
+		dev_err(sdev->dev, "error: XRUN for unknown stream, msg_id %d\n",
+			msg_id);
+		return;
+	}
+
+	stream = &spcm->stream[direction];
+	snd_sof_ipc_msg_data(sdev, stream->substream, &posn, sizeof(posn));
+
+	dev_dbg(sdev->dev,  "posn XRUN: host %llx comp %d size %d\n",
+		posn.host_posn, posn.xrun_comp_id, posn.xrun_size);
+
+#if defined(CONFIG_SND_SOC_SOF_DEBUG_XRUN_STOP)
+	/* stop PCM on XRUN - used for pipeline debug */
+	memcpy(&stream->posn, &posn, sizeof(posn));
+	snd_pcm_stop_xrun(stream->substream);
+#endif
+}
+
+/* stream notifications from DSP FW */
+static void ipc_stream_message(struct snd_sof_dev *sdev, u32 msg_cmd)
+{
+	/* get msg cmd type and msd id */
+	u32 msg_type = msg_cmd & SOF_CMD_TYPE_MASK;
+	u32 msg_id = SOF_IPC_MESSAGE_ID(msg_cmd);
+
+	switch (msg_type) {
+	case SOF_IPC_STREAM_POSITION:
+		ipc_period_elapsed(sdev, msg_id);
+		break;
+	case SOF_IPC_STREAM_TRIG_XRUN:
+		ipc_xrun(sdev, msg_id);
+		break;
+	default:
+		dev_err(sdev->dev, "error: unhandled stream message %x\n",
+			msg_id);
+		break;
+	}
+}
+
+/* get stream position IPC - use faster MMIO method if available on platform */
+int snd_sof_ipc_stream_posn(struct snd_sof_dev *sdev,
+			    struct snd_sof_pcm *spcm, int direction,
+			    struct sof_ipc_stream_posn *posn)
+{
+	struct sof_ipc_stream stream;
+	int err;
+
+	/* read position via slower IPC */
+	stream.hdr.size = sizeof(stream);
+	stream.hdr.cmd = SOF_IPC_GLB_STREAM_MSG | SOF_IPC_STREAM_POSITION;
+	stream.comp_id = spcm->stream[direction].comp_id;
+
+	/* send IPC to the DSP */
+	err = sof_ipc_tx_message(sdev->ipc,
+				 stream.hdr.cmd, &stream, sizeof(stream), &posn,
+				 sizeof(*posn));
+	if (err < 0) {
+		dev_err(sdev->dev, "error: failed to get stream %d position\n",
+			stream.comp_id);
+		return err;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL(snd_sof_ipc_stream_posn);
+
+static int sof_get_ctrl_copy_params(enum sof_ipc_ctrl_type ctrl_type,
+				    struct sof_ipc_ctrl_data *src,
+				    struct sof_ipc_ctrl_data *dst,
+				    struct sof_ipc_ctrl_data_params *sparams)
+{
+	switch (ctrl_type) {
+	case SOF_CTRL_TYPE_VALUE_CHAN_GET:
+	case SOF_CTRL_TYPE_VALUE_CHAN_SET:
+		sparams->src = (u8 *)src->chanv;
+		sparams->dst = (u8 *)dst->chanv;
+		break;
+	case SOF_CTRL_TYPE_VALUE_COMP_GET:
+	case SOF_CTRL_TYPE_VALUE_COMP_SET:
+		sparams->src = (u8 *)src->compv;
+		sparams->dst = (u8 *)dst->compv;
+		break;
+	case SOF_CTRL_TYPE_DATA_GET:
+	case SOF_CTRL_TYPE_DATA_SET:
+		sparams->src = (u8 *)src->data->data;
+		sparams->dst = (u8 *)dst->data->data;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	/* calculate payload size and number of messages */
+	sparams->pl_size = SOF_IPC_MSG_MAX_SIZE - sparams->hdr_bytes;
+	sparams->num_msg = DIV_ROUND_UP(sparams->msg_bytes, sparams->pl_size);
+
+	return 0;
+}
+
+static int sof_set_get_large_ctrl_data(struct snd_sof_dev *sdev,
+				       struct sof_ipc_ctrl_data *cdata,
+				       struct sof_ipc_ctrl_data_params *sparams,
+				       bool send)
+{
+	struct sof_ipc_ctrl_data *partdata;
+	size_t send_bytes;
+	size_t offset = 0;
+	size_t msg_bytes;
+	size_t pl_size;
+	int err;
+	int i;
+
+	/* allocate max ipc size because we have at least one */
+	partdata = kzalloc(SOF_IPC_MSG_MAX_SIZE, GFP_KERNEL);
+	if (!partdata)
+		return -ENOMEM;
+
+	if (send)
+		err = sof_get_ctrl_copy_params(cdata->type, cdata, partdata,
+					       sparams);
+	else
+		err = sof_get_ctrl_copy_params(cdata->type, partdata, cdata,
+					       sparams);
+	if (err < 0)
+		return err;
+
+	msg_bytes = sparams->msg_bytes;
+	pl_size = sparams->pl_size;
+
+	/* copy the header data */
+	memcpy(partdata, cdata, sparams->hdr_bytes);
+
+	/* Serialise IPC TX */
+	mutex_lock(&sdev->ipc->tx_mutex);
+
+	/* copy the payload data in a loop */
+	for (i = 0; i < sparams->num_msg; i++) {
+		send_bytes = min(msg_bytes, pl_size);
+		partdata->num_elems = send_bytes;
+		partdata->rhdr.hdr.size = sparams->hdr_bytes + send_bytes;
+		partdata->msg_index = i;
+		msg_bytes -= send_bytes;
+		partdata->elems_remaining = msg_bytes;
+
+		if (send)
+			memcpy(sparams->dst, sparams->src + offset, send_bytes);
+
+		err = sof_ipc_tx_message_unlocked(sdev->ipc,
+						  partdata->rhdr.hdr.cmd,
+						  partdata,
+						  partdata->rhdr.hdr.size,
+						  partdata,
+						  partdata->rhdr.hdr.size);
+		if (err < 0)
+			break;
+
+		if (!send)
+			memcpy(sparams->dst + offset, sparams->src, send_bytes);
+
+		offset += pl_size;
+	}
+
+	mutex_unlock(&sdev->ipc->tx_mutex);
+
+	kfree(partdata);
+	return err;
+}
+
+/*
+ * IPC get()/set() for kcontrols.
+ */
+int snd_sof_ipc_set_get_comp_data(struct snd_sof_ipc *ipc,
+				  struct snd_sof_control *scontrol,
+				  u32 ipc_cmd,
+				  enum sof_ipc_ctrl_type ctrl_type,
+				  enum sof_ipc_ctrl_cmd ctrl_cmd,
+				  bool send)
+{
+	struct sof_ipc_ctrl_data *cdata = scontrol->control_data;
+	struct snd_sof_dev *sdev = ipc->sdev;
+	struct sof_ipc_fw_ready *ready = &sdev->fw_ready;
+	struct sof_ipc_fw_version *v = &ready->version;
+	struct sof_ipc_ctrl_data_params sparams;
+	size_t send_bytes;
+	int err;
+
+	/* read or write firmware volume */
+	if (scontrol->readback_offset != 0) {
+		/* write/read value header via mmaped region */
+		send_bytes = sizeof(struct sof_ipc_ctrl_value_chan) *
+		cdata->num_elems;
+		if (send)
+			snd_sof_dsp_block_write(sdev, sdev->mmio_bar,
+						scontrol->readback_offset,
+						cdata->chanv, send_bytes);
+
+		else
+			snd_sof_dsp_block_read(sdev, sdev->mmio_bar,
+					       scontrol->readback_offset,
+					       cdata->chanv, send_bytes);
+		return 0;
+	}
+
+	cdata->rhdr.hdr.cmd = SOF_IPC_GLB_COMP_MSG | ipc_cmd;
+	cdata->cmd = ctrl_cmd;
+	cdata->type = ctrl_type;
+	cdata->comp_id = scontrol->comp_id;
+	cdata->msg_index = 0;
+
+	/* calculate header and data size */
+	switch (cdata->type) {
+	case SOF_CTRL_TYPE_VALUE_CHAN_GET:
+	case SOF_CTRL_TYPE_VALUE_CHAN_SET:
+		sparams.msg_bytes = scontrol->num_channels *
+			sizeof(struct sof_ipc_ctrl_value_chan);
+		sparams.hdr_bytes = sizeof(struct sof_ipc_ctrl_data);
+		sparams.elems = scontrol->num_channels;
+		break;
+	case SOF_CTRL_TYPE_VALUE_COMP_GET:
+	case SOF_CTRL_TYPE_VALUE_COMP_SET:
+		sparams.msg_bytes = scontrol->num_channels *
+			sizeof(struct sof_ipc_ctrl_value_comp);
+		sparams.hdr_bytes = sizeof(struct sof_ipc_ctrl_data);
+		sparams.elems = scontrol->num_channels;
+		break;
+	case SOF_CTRL_TYPE_DATA_GET:
+	case SOF_CTRL_TYPE_DATA_SET:
+		sparams.msg_bytes = cdata->data->size;
+		sparams.hdr_bytes = sizeof(struct sof_ipc_ctrl_data) +
+			sizeof(struct sof_abi_hdr);
+		sparams.elems = cdata->data->size;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	cdata->rhdr.hdr.size = sparams.msg_bytes + sparams.hdr_bytes;
+	cdata->num_elems = sparams.elems;
+	cdata->elems_remaining = 0;
+
+	/* send normal size ipc in one part */
+	if (cdata->rhdr.hdr.size <= SOF_IPC_MSG_MAX_SIZE) {
+		err = sof_ipc_tx_message(sdev->ipc, cdata->rhdr.hdr.cmd, cdata,
+					 cdata->rhdr.hdr.size, cdata,
+					 cdata->rhdr.hdr.size);
+
+		if (err < 0)
+			dev_err(sdev->dev, "error: set/get ctrl ipc comp %d\n",
+				cdata->comp_id);
+
+		return err;
+	}
+
+	/* data is bigger than max ipc size, chop into smaller pieces */
+	dev_dbg(sdev->dev, "large ipc size %u, control size %u\n",
+		cdata->rhdr.hdr.size, scontrol->size);
+
+	/* large messages is only supported from ABI 3.3.0 onwards */
+	if (v->abi_version < SOF_ABI_VER(3, 3, 0)) {
+		dev_err(sdev->dev, "error: incompatible FW ABI version\n");
+		return -EINVAL;
+	}
+
+	err = sof_set_get_large_ctrl_data(sdev, cdata, &sparams, send);
+
+	if (err < 0)
+		dev_err(sdev->dev, "error: set/get large ctrl ipc comp %d\n",
+			cdata->comp_id);
+
+	return err;
+}
+EXPORT_SYMBOL(snd_sof_ipc_set_get_comp_data);
+
+/*
+ * IPC layer enumeration.
+ */
+
+int snd_sof_dsp_mailbox_init(struct snd_sof_dev *sdev, u32 dspbox,
+			     size_t dspbox_size, u32 hostbox,
+			     size_t hostbox_size)
+{
+	sdev->dsp_box.offset = dspbox;
+	sdev->dsp_box.size = dspbox_size;
+	sdev->host_box.offset = hostbox;
+	sdev->host_box.size = hostbox_size;
+	return 0;
+}
+EXPORT_SYMBOL(snd_sof_dsp_mailbox_init);
+
+int snd_sof_ipc_valid(struct snd_sof_dev *sdev)
+{
+	struct sof_ipc_fw_ready *ready = &sdev->fw_ready;
+	struct sof_ipc_fw_version *v = &ready->version;
+
+	dev_info(sdev->dev,
+		 "Firmware info: version %d:%d:%d-%s\n",  v->major, v->minor,
+		 v->micro, v->tag);
+	dev_info(sdev->dev,
+		 "Firmware: ABI %d:%d:%d Kernel ABI %d:%d:%d\n",
+		 SOF_ABI_VERSION_MAJOR(v->abi_version),
+		 SOF_ABI_VERSION_MINOR(v->abi_version),
+		 SOF_ABI_VERSION_PATCH(v->abi_version),
+		 SOF_ABI_MAJOR, SOF_ABI_MINOR, SOF_ABI_PATCH);
+
+	if (SOF_ABI_VERSION_INCOMPATIBLE(SOF_ABI_VERSION, v->abi_version)) {
+		dev_err(sdev->dev, "error: incompatible FW ABI version\n");
+		return -EINVAL;
+	}
+
+	if (v->abi_version > SOF_ABI_VERSION) {
+		if (!IS_ENABLED(CONFIG_SND_SOC_SOF_STRICT_ABI_CHECKS)) {
+			dev_warn(sdev->dev, "warn: FW ABI is more recent than kernel\n");
+		} else {
+			dev_err(sdev->dev, "error: FW ABI is more recent than kernel\n");
+			return -EINVAL;
+		}
+	}
+
+	if (ready->debug.bits.build) {
+		dev_info(sdev->dev,
+			 "Firmware debug build %d on %s-%s - options:\n"
+			 " GDB: %s\n"
+			 " lock debug: %s\n"
+			 " lock vdebug: %s\n",
+			 v->build, v->date, v->time,
+			 ready->debug.bits.gdb ? "enabled" : "disabled",
+			 ready->debug.bits.locks ? "enabled" : "disabled",
+			 ready->debug.bits.locks_verbose ? "enabled" : "disabled");
+	}
+
+	/* copy the fw_version into debugfs at first boot */
+	memcpy(&sdev->fw_version, v, sizeof(*v));
+
+	return 0;
+}
+EXPORT_SYMBOL(snd_sof_ipc_valid);
+
+struct snd_sof_ipc *snd_sof_ipc_init(struct snd_sof_dev *sdev)
+{
+	struct snd_sof_ipc *ipc;
+	struct snd_sof_ipc_msg *msg;
+
+	/* check if mandatory ops required for ipc are defined */
+	if (!sof_ops(sdev)->fw_ready) {
+		dev_err(sdev->dev, "error: ipc mandatory ops not defined\n");
+		return NULL;
+	}
+
+	ipc = devm_kzalloc(sdev->dev, sizeof(*ipc), GFP_KERNEL);
+	if (!ipc)
+		return NULL;
+
+	mutex_init(&ipc->tx_mutex);
+	ipc->sdev = sdev;
+	msg = &ipc->msg;
+
+	/* indicate that we aren't sending a message ATM */
+	msg->ipc_complete = true;
+
+	/* pre-allocate message data */
+	msg->msg_data = devm_kzalloc(sdev->dev, SOF_IPC_MSG_MAX_SIZE,
+				     GFP_KERNEL);
+	if (!msg->msg_data)
+		return NULL;
+
+	msg->reply_data = devm_kzalloc(sdev->dev, SOF_IPC_MSG_MAX_SIZE,
+				       GFP_KERNEL);
+	if (!msg->reply_data)
+		return NULL;
+
+	init_waitqueue_head(&msg->waitq);
+
+	return ipc;
+}
+EXPORT_SYMBOL(snd_sof_ipc_init);
+
+void snd_sof_ipc_free(struct snd_sof_dev *sdev)
+{
+	struct snd_sof_ipc *ipc = sdev->ipc;
+
+	/* disable sending of ipc's */
+	mutex_lock(&ipc->tx_mutex);
+	ipc->disable_ipc_tx = true;
+	mutex_unlock(&ipc->tx_mutex);
+}
+EXPORT_SYMBOL(snd_sof_ipc_free);
diff --git a/sound/soc/sof/loader.c b/sound/soc/sof/loader.c
new file mode 100644
index 000000000000..81c7452aae17
--- /dev/null
+++ b/sound/soc/sof/loader.c
@@ -0,0 +1,400 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license.  When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2018 Intel Corporation. All rights reserved.
+//
+// Author: Liam Girdwood <liam.r.girdwood@linux.intel.com>
+//
+// Generic firmware loader.
+//
+
+#include <linux/firmware.h>
+#include <sound/sof.h>
+#include "ops.h"
+
+static int get_ext_windows(struct snd_sof_dev *sdev,
+			   struct sof_ipc_ext_data_hdr *ext_hdr)
+{
+	struct sof_ipc_window *w =
+		container_of(ext_hdr, struct sof_ipc_window, ext_hdr);
+	size_t size;
+
+	if (w->num_windows == 0 || w->num_windows > SOF_IPC_MAX_ELEMS)
+		return -EINVAL;
+
+	size = sizeof(*w) + sizeof(struct sof_ipc_window_elem) * w->num_windows;
+
+	/* keep a local copy of the data */
+	sdev->info_window = kmemdup(w, size, GFP_KERNEL);
+	if (!sdev->info_window)
+		return -ENOMEM;
+
+	return 0;
+}
+
+/* parse the extended FW boot data structures from FW boot message */
+int snd_sof_fw_parse_ext_data(struct snd_sof_dev *sdev, u32 bar, u32 offset)
+{
+	struct sof_ipc_ext_data_hdr *ext_hdr;
+	void *ext_data;
+	int ret = 0;
+
+	ext_data = kzalloc(PAGE_SIZE, GFP_KERNEL);
+	if (!ext_data)
+		return -ENOMEM;
+
+	/* get first header */
+	snd_sof_dsp_block_read(sdev, bar, offset, ext_data,
+			       sizeof(*ext_hdr));
+	ext_hdr = ext_data;
+
+	while (ext_hdr->hdr.cmd == SOF_IPC_FW_READY) {
+		/* read in ext structure */
+		offset += sizeof(*ext_hdr);
+		snd_sof_dsp_block_read(sdev, bar, offset,
+				   (void *)((u8 *)ext_data + sizeof(*ext_hdr)),
+				   ext_hdr->hdr.size - sizeof(*ext_hdr));
+
+		dev_dbg(sdev->dev, "found ext header type %d size 0x%x\n",
+			ext_hdr->type, ext_hdr->hdr.size);
+
+		/* process structure data */
+		switch (ext_hdr->type) {
+		case SOF_IPC_EXT_DMA_BUFFER:
+			break;
+		case SOF_IPC_EXT_WINDOW:
+			ret = get_ext_windows(sdev, ext_hdr);
+			break;
+		default:
+			break;
+		}
+
+		if (ret < 0) {
+			dev_err(sdev->dev, "error: failed to parse ext data type %d\n",
+				ext_hdr->type);
+			break;
+		}
+
+		/* move to next header */
+		offset += ext_hdr->hdr.size;
+		snd_sof_dsp_block_read(sdev, bar, offset, ext_data,
+				       sizeof(*ext_hdr));
+		ext_hdr = ext_data;
+	}
+
+	kfree(ext_data);
+	return ret;
+}
+EXPORT_SYMBOL(snd_sof_fw_parse_ext_data);
+
+/* generic module parser for mmaped DSPs */
+int snd_sof_parse_module_memcpy(struct snd_sof_dev *sdev,
+				struct snd_sof_mod_hdr *module)
+{
+	struct snd_sof_blk_hdr *block;
+	int count;
+	u32 offset;
+	size_t remaining;
+
+	dev_dbg(sdev->dev, "new module size 0x%x blocks 0x%x type 0x%x\n",
+		module->size, module->num_blocks, module->type);
+
+	block = (struct snd_sof_blk_hdr *)((u8 *)module + sizeof(*module));
+
+	/* module->size doesn't include header size */
+	remaining = module->size;
+	for (count = 0; count < module->num_blocks; count++) {
+		/* check for wrap */
+		if (remaining < sizeof(*block)) {
+			dev_err(sdev->dev, "error: not enough data remaining\n");
+			return -EINVAL;
+		}
+
+		/* minus header size of block */
+		remaining -= sizeof(*block);
+
+		if (block->size == 0) {
+			dev_warn(sdev->dev,
+				 "warning: block %d size zero\n", count);
+			dev_warn(sdev->dev, " type 0x%x offset 0x%x\n",
+				 block->type, block->offset);
+			continue;
+		}
+
+		switch (block->type) {
+		case SOF_FW_BLK_TYPE_RSRVD0:
+		case SOF_FW_BLK_TYPE_SRAM...SOF_FW_BLK_TYPE_RSRVD14:
+			continue;	/* not handled atm */
+		case SOF_FW_BLK_TYPE_IRAM:
+		case SOF_FW_BLK_TYPE_DRAM:
+			offset = block->offset;
+			break;
+		default:
+			dev_err(sdev->dev, "error: bad type 0x%x for block 0x%x\n",
+				block->type, count);
+			return -EINVAL;
+		}
+
+		dev_dbg(sdev->dev,
+			"block %d type 0x%x size 0x%x ==>  offset 0x%x\n",
+			count, block->type, block->size, offset);
+
+		/* checking block->size to avoid unaligned access */
+		if (block->size % sizeof(u32)) {
+			dev_err(sdev->dev, "error: invalid block size 0x%x\n",
+				block->size);
+			return -EINVAL;
+		}
+		snd_sof_dsp_block_write(sdev, sdev->mmio_bar, offset,
+					block + 1, block->size);
+
+		if (remaining < block->size) {
+			dev_err(sdev->dev, "error: not enough data remaining\n");
+			return -EINVAL;
+		}
+
+		/* minus body size of block */
+		remaining -= block->size;
+		/* next block */
+		block = (struct snd_sof_blk_hdr *)((u8 *)block + sizeof(*block)
+			+ block->size);
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL(snd_sof_parse_module_memcpy);
+
+static int check_header(struct snd_sof_dev *sdev, const struct firmware *fw)
+{
+	struct snd_sof_fw_header *header;
+
+	/* Read the header information from the data pointer */
+	header = (struct snd_sof_fw_header *)fw->data;
+
+	/* verify FW sig */
+	if (strncmp(header->sig, SND_SOF_FW_SIG, SND_SOF_FW_SIG_SIZE) != 0) {
+		dev_err(sdev->dev, "error: invalid firmware signature\n");
+		return -EINVAL;
+	}
+
+	/* check size is valid */
+	if (fw->size != header->file_size + sizeof(*header)) {
+		dev_err(sdev->dev, "error: invalid filesize mismatch got 0x%zx expected 0x%zx\n",
+			fw->size, header->file_size + sizeof(*header));
+		return -EINVAL;
+	}
+
+	dev_dbg(sdev->dev, "header size=0x%x modules=0x%x abi=0x%x size=%zu\n",
+		header->file_size, header->num_modules,
+		header->abi, sizeof(*header));
+
+	return 0;
+}
+
+static int load_modules(struct snd_sof_dev *sdev, const struct firmware *fw)
+{
+	struct snd_sof_fw_header *header;
+	struct snd_sof_mod_hdr *module;
+	int (*load_module)(struct snd_sof_dev *sof_dev,
+			   struct snd_sof_mod_hdr *hdr);
+	int ret, count;
+	size_t remaining;
+
+	header = (struct snd_sof_fw_header *)fw->data;
+	load_module = sof_ops(sdev)->load_module;
+	if (!load_module)
+		return -EINVAL;
+
+	/* parse each module */
+	module = (struct snd_sof_mod_hdr *)((u8 *)(fw->data) + sizeof(*header));
+	remaining = fw->size - sizeof(*header);
+	/* check for wrap */
+	if (remaining > fw->size) {
+		dev_err(sdev->dev, "error: fw size smaller than header size\n");
+		return -EINVAL;
+	}
+
+	for (count = 0; count < header->num_modules; count++) {
+		/* check for wrap */
+		if (remaining < sizeof(*module)) {
+			dev_err(sdev->dev, "error: not enough data remaining\n");
+			return -EINVAL;
+		}
+
+		/* minus header size of module */
+		remaining -= sizeof(*module);
+
+		/* module */
+		ret = load_module(sdev, module);
+		if (ret < 0) {
+			dev_err(sdev->dev, "error: invalid module %d\n", count);
+			return ret;
+		}
+
+		if (remaining < module->size) {
+			dev_err(sdev->dev, "error: not enough data remaining\n");
+			return -EINVAL;
+		}
+
+		/* minus body size of module */
+		remaining -=  module->size;
+		module = (struct snd_sof_mod_hdr *)((u8 *)module
+			+ sizeof(*module) + module->size);
+	}
+
+	return 0;
+}
+
+int snd_sof_load_firmware_raw(struct snd_sof_dev *sdev)
+{
+	struct snd_sof_pdata *plat_data = sdev->pdata;
+	const char *fw_filename;
+	int ret;
+
+	/* set code loading condition to true */
+	sdev->code_loading = 1;
+
+	/* Don't request firmware again if firmware is already requested */
+	if (plat_data->fw)
+		return 0;
+
+	fw_filename = kasprintf(GFP_KERNEL, "%s/%s",
+				plat_data->fw_filename_prefix,
+				plat_data->fw_filename);
+	if (!fw_filename)
+		return -ENOMEM;
+
+	ret = request_firmware(&plat_data->fw, fw_filename, sdev->dev);
+
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: request firmware %s failed err: %d\n",
+			fw_filename, ret);
+	}
+
+	kfree(fw_filename);
+
+	return ret;
+}
+EXPORT_SYMBOL(snd_sof_load_firmware_raw);
+
+int snd_sof_load_firmware_memcpy(struct snd_sof_dev *sdev)
+{
+	struct snd_sof_pdata *plat_data = sdev->pdata;
+	int ret;
+
+	ret = snd_sof_load_firmware_raw(sdev);
+	if (ret < 0)
+		return ret;
+
+	/* make sure the FW header and file is valid */
+	ret = check_header(sdev, plat_data->fw);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: invalid FW header\n");
+		goto error;
+	}
+
+	/* prepare the DSP for FW loading */
+	ret = snd_sof_dsp_reset(sdev);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: failed to reset DSP\n");
+		goto error;
+	}
+
+	/* parse and load firmware modules to DSP */
+	ret = load_modules(sdev, plat_data->fw);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: invalid FW modules\n");
+		goto error;
+	}
+
+	return 0;
+
+error:
+	release_firmware(plat_data->fw);
+	plat_data->fw = NULL;
+	return ret;
+
+}
+EXPORT_SYMBOL(snd_sof_load_firmware_memcpy);
+
+int snd_sof_load_firmware(struct snd_sof_dev *sdev)
+{
+	dev_dbg(sdev->dev, "loading firmware\n");
+
+	if (sof_ops(sdev)->load_firmware)
+		return sof_ops(sdev)->load_firmware(sdev);
+	return 0;
+}
+EXPORT_SYMBOL(snd_sof_load_firmware);
+
+int snd_sof_run_firmware(struct snd_sof_dev *sdev)
+{
+	int ret;
+	int init_core_mask;
+
+	init_waitqueue_head(&sdev->boot_wait);
+	sdev->boot_complete = false;
+
+	/* create fw_version debugfs to store boot version info */
+	if (sdev->first_boot) {
+		ret = snd_sof_debugfs_buf_item(sdev, &sdev->fw_version,
+					       sizeof(sdev->fw_version),
+					       "fw_version");
+		/* errors are only due to memory allocation, not debugfs */
+		if (ret < 0) {
+			dev_err(sdev->dev, "error: snd_sof_debugfs_buf_item failed\n");
+			return ret;
+		}
+	}
+
+	/* perform pre fw run operations */
+	ret = snd_sof_dsp_pre_fw_run(sdev);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: failed pre fw run op\n");
+		return ret;
+	}
+
+	dev_dbg(sdev->dev, "booting DSP firmware\n");
+
+	/* boot the firmware on the DSP */
+	ret = snd_sof_dsp_run(sdev);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: failed to reset DSP\n");
+		return ret;
+	}
+
+	init_core_mask = ret;
+
+	/* now wait for the DSP to boot */
+	ret = wait_event_timeout(sdev->boot_wait, sdev->boot_complete,
+				 msecs_to_jiffies(sdev->boot_timeout));
+	if (ret == 0) {
+		dev_err(sdev->dev, "error: firmware boot failure\n");
+		snd_sof_dsp_dbg_dump(sdev, SOF_DBG_REGS | SOF_DBG_MBOX |
+			SOF_DBG_TEXT | SOF_DBG_PCI);
+		return -EIO;
+	}
+
+	dev_info(sdev->dev, "firmware boot complete\n");
+
+	/* perform post fw run operations */
+	ret = snd_sof_dsp_post_fw_run(sdev);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: failed post fw run op\n");
+		return ret;
+	}
+
+	/* fw boot is complete. Update the active cores mask */
+	sdev->enabled_cores_mask = init_core_mask;
+
+	return 0;
+}
+EXPORT_SYMBOL(snd_sof_run_firmware);
+
+void snd_sof_fw_unload(struct snd_sof_dev *sdev)
+{
+	/* TODO: support module unloading at runtime */
+}
+EXPORT_SYMBOL(snd_sof_fw_unload);
diff --git a/sound/soc/sof/nocodec.c b/sound/soc/sof/nocodec.c
new file mode 100644
index 000000000000..f84b4344dcc3
--- /dev/null
+++ b/sound/soc/sof/nocodec.c
@@ -0,0 +1,109 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license.  When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2018 Intel Corporation. All rights reserved.
+//
+// Author: Liam Girdwood <liam.r.girdwood@linux.intel.com>
+//
+
+#include <linux/module.h>
+#include <sound/sof.h>
+#include "sof-priv.h"
+
+static struct snd_soc_card sof_nocodec_card = {
+	.name = "nocodec", /* the sof- prefix is added by the core */
+};
+
+static int sof_nocodec_bes_setup(struct device *dev,
+				 const struct snd_sof_dsp_ops *ops,
+				 struct snd_soc_dai_link *links,
+				 int link_num, struct snd_soc_card *card)
+{
+	int i;
+
+	if (!ops || !links || !card)
+		return -EINVAL;
+
+	/* set up BE dai_links */
+	for (i = 0; i < link_num; i++) {
+		links[i].name = devm_kasprintf(dev, GFP_KERNEL,
+					       "NoCodec-%d", i);
+		if (!links[i].name)
+			return -ENOMEM;
+
+		links[i].id = i;
+		links[i].no_pcm = 1;
+		links[i].cpu_dai_name = ops->drv[i].name;
+		links[i].platform_name = dev_name(dev);
+		links[i].codec_dai_name = "snd-soc-dummy-dai";
+		links[i].codec_name = "snd-soc-dummy";
+		links[i].dpcm_playback = 1;
+		links[i].dpcm_capture = 1;
+	}
+
+	card->dai_link = links;
+	card->num_links = link_num;
+
+	return 0;
+}
+
+int sof_nocodec_setup(struct device *dev,
+		      struct snd_sof_pdata *sof_pdata,
+		      struct snd_soc_acpi_mach *mach,
+		      const struct sof_dev_desc *desc,
+		      const struct snd_sof_dsp_ops *ops)
+{
+	struct snd_soc_dai_link *links;
+	int ret;
+
+	if (!mach)
+		return -EINVAL;
+
+	sof_pdata->drv_name = "sof-nocodec";
+
+	mach->drv_name = "sof-nocodec";
+	sof_pdata->fw_filename = desc->nocodec_fw_filename;
+	sof_pdata->tplg_filename = desc->nocodec_tplg_filename;
+
+	/* create dummy BE dai_links */
+	links = devm_kzalloc(dev, sizeof(struct snd_soc_dai_link) *
+			     ops->num_drv, GFP_KERNEL);
+	if (!links)
+		return -ENOMEM;
+
+	ret = sof_nocodec_bes_setup(dev, ops, links, ops->num_drv,
+				    &sof_nocodec_card);
+	return ret;
+}
+EXPORT_SYMBOL(sof_nocodec_setup);
+
+static int sof_nocodec_probe(struct platform_device *pdev)
+{
+	struct snd_soc_card *card = &sof_nocodec_card;
+
+	card->dev = &pdev->dev;
+
+	return devm_snd_soc_register_card(&pdev->dev, card);
+}
+
+static int sof_nocodec_remove(struct platform_device *pdev)
+{
+	return 0;
+}
+
+static struct platform_driver sof_nocodec_audio = {
+	.probe = sof_nocodec_probe,
+	.remove = sof_nocodec_remove,
+	.driver = {
+		.name = "sof-nocodec",
+		.pm = &snd_soc_pm_ops,
+	},
+};
+module_platform_driver(sof_nocodec_audio)
+
+MODULE_DESCRIPTION("ASoC sof nocodec");
+MODULE_AUTHOR("Liam Girdwood");
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_ALIAS("platform:sof-nocodec");
diff --git a/sound/soc/sof/ops.c b/sound/soc/sof/ops.c
new file mode 100644
index 000000000000..7a27c3b719e7
--- /dev/null
+++ b/sound/soc/sof/ops.c
@@ -0,0 +1,163 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license.  When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2018 Intel Corporation. All rights reserved.
+//
+// Author: Liam Girdwood <liam.r.girdwood@linux.intel.com>
+//
+
+#include <linux/pci.h>
+#include "ops.h"
+
+static
+bool snd_sof_pci_update_bits_unlocked(struct snd_sof_dev *sdev, u32 offset,
+				      u32 mask, u32 value)
+{
+	struct pci_dev *pci = to_pci_dev(sdev->dev);
+	unsigned int old, new;
+	u32 ret = 0;
+
+	pci_read_config_dword(pci, offset, &ret);
+	old = ret;
+	dev_dbg(sdev->dev, "Debug PCIR: %8.8x at  %8.8x\n", old & mask, offset);
+
+	new = (old & ~mask) | (value & mask);
+
+	if (old == new)
+		return false;
+
+	pci_write_config_dword(pci, offset, new);
+	dev_dbg(sdev->dev, "Debug PCIW: %8.8x at  %8.8x\n", value,
+		offset);
+
+	return true;
+}
+
+bool snd_sof_pci_update_bits(struct snd_sof_dev *sdev, u32 offset,
+			     u32 mask, u32 value)
+{
+	unsigned long flags;
+	bool change;
+
+	spin_lock_irqsave(&sdev->hw_lock, flags);
+	change = snd_sof_pci_update_bits_unlocked(sdev, offset, mask, value);
+	spin_unlock_irqrestore(&sdev->hw_lock, flags);
+	return change;
+}
+EXPORT_SYMBOL(snd_sof_pci_update_bits);
+
+bool snd_sof_dsp_update_bits_unlocked(struct snd_sof_dev *sdev, u32 bar,
+				      u32 offset, u32 mask, u32 value)
+{
+	unsigned int old, new;
+	u32 ret;
+
+	ret = snd_sof_dsp_read(sdev, bar, offset);
+
+	old = ret;
+	new = (old & ~mask) | (value & mask);
+
+	if (old == new)
+		return false;
+
+	snd_sof_dsp_write(sdev, bar, offset, new);
+
+	return true;
+}
+EXPORT_SYMBOL(snd_sof_dsp_update_bits_unlocked);
+
+bool snd_sof_dsp_update_bits64_unlocked(struct snd_sof_dev *sdev, u32 bar,
+					u32 offset, u64 mask, u64 value)
+{
+	u64 old, new;
+
+	old = snd_sof_dsp_read64(sdev, bar, offset);
+
+	new = (old & ~mask) | (value & mask);
+
+	if (old == new)
+		return false;
+
+	snd_sof_dsp_write64(sdev, bar, offset, new);
+
+	return true;
+}
+EXPORT_SYMBOL(snd_sof_dsp_update_bits64_unlocked);
+
+/* This is for registers bits with attribute RWC */
+bool snd_sof_dsp_update_bits(struct snd_sof_dev *sdev, u32 bar, u32 offset,
+			     u32 mask, u32 value)
+{
+	unsigned long flags;
+	bool change;
+
+	spin_lock_irqsave(&sdev->hw_lock, flags);
+	change = snd_sof_dsp_update_bits_unlocked(sdev, bar, offset, mask,
+						  value);
+	spin_unlock_irqrestore(&sdev->hw_lock, flags);
+	return change;
+}
+EXPORT_SYMBOL(snd_sof_dsp_update_bits);
+
+bool snd_sof_dsp_update_bits64(struct snd_sof_dev *sdev, u32 bar, u32 offset,
+			       u64 mask, u64 value)
+{
+	unsigned long flags;
+	bool change;
+
+	spin_lock_irqsave(&sdev->hw_lock, flags);
+	change = snd_sof_dsp_update_bits64_unlocked(sdev, bar, offset, mask,
+						    value);
+	spin_unlock_irqrestore(&sdev->hw_lock, flags);
+	return change;
+}
+EXPORT_SYMBOL(snd_sof_dsp_update_bits64);
+
+static
+void snd_sof_dsp_update_bits_forced_unlocked(struct snd_sof_dev *sdev, u32 bar,
+					     u32 offset, u32 mask, u32 value)
+{
+	unsigned int old, new;
+	u32 ret;
+
+	ret = snd_sof_dsp_read(sdev, bar, offset);
+
+	old = ret;
+	new = (old & ~mask) | (value & mask);
+
+	snd_sof_dsp_write(sdev, bar, offset, new);
+}
+
+/* This is for registers bits with attribute RWC */
+void snd_sof_dsp_update_bits_forced(struct snd_sof_dev *sdev, u32 bar,
+				    u32 offset, u32 mask, u32 value)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&sdev->hw_lock, flags);
+	snd_sof_dsp_update_bits_forced_unlocked(sdev, bar, offset, mask, value);
+	spin_unlock_irqrestore(&sdev->hw_lock, flags);
+}
+EXPORT_SYMBOL(snd_sof_dsp_update_bits_forced);
+
+void snd_sof_dsp_panic(struct snd_sof_dev *sdev, u32 offset)
+{
+	dev_err(sdev->dev, "error : DSP panic!\n");
+
+	/*
+	 * check if DSP is not ready and did not set the dsp_oops_offset.
+	 * if the dsp_oops_offset is not set, set it from the panic message.
+	 * Also add a check to memory window setting with panic message.
+	 */
+	if (!sdev->dsp_oops_offset)
+		sdev->dsp_oops_offset = offset;
+	else
+		dev_dbg(sdev->dev, "panic: dsp_oops_offset %zu offset %d\n",
+			sdev->dsp_oops_offset, offset);
+
+	snd_sof_dsp_dbg_dump(sdev, SOF_DBG_REGS | SOF_DBG_MBOX);
+	snd_sof_trace_notify_for_error(sdev);
+}
+EXPORT_SYMBOL(snd_sof_dsp_panic);
diff --git a/sound/soc/sof/ops.h b/sound/soc/sof/ops.h
new file mode 100644
index 000000000000..80fc3b374c2b
--- /dev/null
+++ b/sound/soc/sof/ops.h
@@ -0,0 +1,411 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */
+/*
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * Copyright(c) 2018 Intel Corporation. All rights reserved.
+ *
+ * Author: Liam Girdwood <liam.r.girdwood@linux.intel.com>
+ */
+
+#ifndef __SOUND_SOC_SOF_IO_H
+#define __SOUND_SOC_SOF_IO_H
+
+#include <linux/device.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <sound/pcm.h>
+#include "sof-priv.h"
+
+#define sof_ops(sdev) \
+	((sdev)->pdata->desc->ops)
+
+/* Mandatory operations are verified during probing */
+
+/* init */
+static inline int snd_sof_probe(struct snd_sof_dev *sdev)
+{
+	return sof_ops(sdev)->probe(sdev);
+}
+
+static inline int snd_sof_remove(struct snd_sof_dev *sdev)
+{
+	if (sof_ops(sdev)->remove)
+		return sof_ops(sdev)->remove(sdev);
+
+	return 0;
+}
+
+/* control */
+
+/*
+ * snd_sof_dsp_run returns the core mask of the cores that are available
+ * after successful fw boot
+ */
+static inline int snd_sof_dsp_run(struct snd_sof_dev *sdev)
+{
+	return sof_ops(sdev)->run(sdev);
+}
+
+static inline int snd_sof_dsp_stall(struct snd_sof_dev *sdev)
+{
+	if (sof_ops(sdev)->stall)
+		return sof_ops(sdev)->stall(sdev);
+
+	return 0;
+}
+
+static inline int snd_sof_dsp_reset(struct snd_sof_dev *sdev)
+{
+	if (sof_ops(sdev)->reset)
+		return sof_ops(sdev)->reset(sdev);
+
+	return 0;
+}
+
+/* dsp core power up/power down */
+static inline int snd_sof_dsp_core_power_up(struct snd_sof_dev *sdev,
+					    unsigned int core_mask)
+{
+	if (sof_ops(sdev)->core_power_up)
+		return sof_ops(sdev)->core_power_up(sdev, core_mask);
+
+	return 0;
+}
+
+static inline int snd_sof_dsp_core_power_down(struct snd_sof_dev *sdev,
+					      unsigned int core_mask)
+{
+	if (sof_ops(sdev)->core_power_down)
+		return sof_ops(sdev)->core_power_down(sdev, core_mask);
+
+	return 0;
+}
+
+/* pre/post fw load */
+static inline int snd_sof_dsp_pre_fw_run(struct snd_sof_dev *sdev)
+{
+	if (sof_ops(sdev)->pre_fw_run)
+		return sof_ops(sdev)->pre_fw_run(sdev);
+
+	return 0;
+}
+
+static inline int snd_sof_dsp_post_fw_run(struct snd_sof_dev *sdev)
+{
+	if (sof_ops(sdev)->post_fw_run)
+		return sof_ops(sdev)->post_fw_run(sdev);
+
+	return 0;
+}
+
+/* power management */
+static inline int snd_sof_dsp_resume(struct snd_sof_dev *sdev)
+{
+	if (sof_ops(sdev)->resume)
+		return sof_ops(sdev)->resume(sdev);
+
+	return 0;
+}
+
+static inline int snd_sof_dsp_suspend(struct snd_sof_dev *sdev, int state)
+{
+	if (sof_ops(sdev)->suspend)
+		return sof_ops(sdev)->suspend(sdev, state);
+
+	return 0;
+}
+
+static inline int snd_sof_dsp_runtime_resume(struct snd_sof_dev *sdev)
+{
+	if (sof_ops(sdev)->runtime_resume)
+		return sof_ops(sdev)->runtime_resume(sdev);
+
+	return 0;
+}
+
+static inline int snd_sof_dsp_runtime_suspend(struct snd_sof_dev *sdev,
+					      int state)
+{
+	if (sof_ops(sdev)->runtime_suspend)
+		return sof_ops(sdev)->runtime_suspend(sdev, state);
+
+	return 0;
+}
+
+static inline void snd_sof_dsp_hw_params_upon_resume(struct snd_sof_dev *sdev)
+{
+	if (sof_ops(sdev)->set_hw_params_upon_resume)
+		sof_ops(sdev)->set_hw_params_upon_resume(sdev);
+}
+
+static inline int snd_sof_dsp_set_clk(struct snd_sof_dev *sdev, u32 freq)
+{
+	if (sof_ops(sdev)->set_clk)
+		return sof_ops(sdev)->set_clk(sdev, freq);
+
+	return 0;
+}
+
+/* debug */
+static inline void snd_sof_dsp_dbg_dump(struct snd_sof_dev *sdev, u32 flags)
+{
+	if (sof_ops(sdev)->dbg_dump)
+		return sof_ops(sdev)->dbg_dump(sdev, flags);
+}
+
+static inline void snd_sof_ipc_dump(struct snd_sof_dev *sdev)
+{
+	if (sof_ops(sdev)->ipc_dump)
+		return sof_ops(sdev)->ipc_dump(sdev);
+}
+
+/* register IO */
+static inline void snd_sof_dsp_write(struct snd_sof_dev *sdev, u32 bar,
+				     u32 offset, u32 value)
+{
+	if (sof_ops(sdev)->write) {
+		sof_ops(sdev)->write(sdev, sdev->bar[bar] + offset, value);
+		return;
+	}
+
+	dev_err_ratelimited(sdev->dev, "error: %s not defined\n", __func__);
+}
+
+static inline void snd_sof_dsp_write64(struct snd_sof_dev *sdev, u32 bar,
+				       u32 offset, u64 value)
+{
+	if (sof_ops(sdev)->write64) {
+		sof_ops(sdev)->write64(sdev, sdev->bar[bar] + offset, value);
+		return;
+	}
+
+	dev_err_ratelimited(sdev->dev, "error: %s not defined\n", __func__);
+}
+
+static inline u32 snd_sof_dsp_read(struct snd_sof_dev *sdev, u32 bar,
+				   u32 offset)
+{
+	if (sof_ops(sdev)->read)
+		return sof_ops(sdev)->read(sdev, sdev->bar[bar] + offset);
+
+	dev_err(sdev->dev, "error: %s not defined\n", __func__);
+	return -ENOTSUPP;
+}
+
+static inline u64 snd_sof_dsp_read64(struct snd_sof_dev *sdev, u32 bar,
+				     u32 offset)
+{
+	if (sof_ops(sdev)->read64)
+		return sof_ops(sdev)->read64(sdev, sdev->bar[bar] + offset);
+
+	dev_err(sdev->dev, "error: %s not defined\n", __func__);
+	return -ENOTSUPP;
+}
+
+/* block IO */
+static inline void snd_sof_dsp_block_read(struct snd_sof_dev *sdev, u32 bar,
+					  u32 offset, void *dest, size_t bytes)
+{
+	sof_ops(sdev)->block_read(sdev, bar, offset, dest, bytes);
+}
+
+static inline void snd_sof_dsp_block_write(struct snd_sof_dev *sdev, u32 bar,
+					   u32 offset, void *src, size_t bytes)
+{
+	sof_ops(sdev)->block_write(sdev, bar, offset, src, bytes);
+}
+
+/* ipc */
+static inline int snd_sof_dsp_send_msg(struct snd_sof_dev *sdev,
+				       struct snd_sof_ipc_msg *msg)
+{
+	return sof_ops(sdev)->send_msg(sdev, msg);
+}
+
+/* host DMA trace */
+static inline int snd_sof_dma_trace_init(struct snd_sof_dev *sdev,
+					 u32 *stream_tag)
+{
+	if (sof_ops(sdev)->trace_init)
+		return sof_ops(sdev)->trace_init(sdev, stream_tag);
+
+	return 0;
+}
+
+static inline int snd_sof_dma_trace_release(struct snd_sof_dev *sdev)
+{
+	if (sof_ops(sdev)->trace_release)
+		return sof_ops(sdev)->trace_release(sdev);
+
+	return 0;
+}
+
+static inline int snd_sof_dma_trace_trigger(struct snd_sof_dev *sdev, int cmd)
+{
+	if (sof_ops(sdev)->trace_trigger)
+		return sof_ops(sdev)->trace_trigger(sdev, cmd);
+
+	return 0;
+}
+
+/* host PCM ops */
+static inline int
+snd_sof_pcm_platform_open(struct snd_sof_dev *sdev,
+			  struct snd_pcm_substream *substream)
+{
+	if (sof_ops(sdev) && sof_ops(sdev)->pcm_open)
+		return sof_ops(sdev)->pcm_open(sdev, substream);
+
+	return 0;
+}
+
+/* disconnect pcm substream to a host stream */
+static inline int
+snd_sof_pcm_platform_close(struct snd_sof_dev *sdev,
+			   struct snd_pcm_substream *substream)
+{
+	if (sof_ops(sdev) && sof_ops(sdev)->pcm_close)
+		return sof_ops(sdev)->pcm_close(sdev, substream);
+
+	return 0;
+}
+
+/* host stream hw params */
+static inline int
+snd_sof_pcm_platform_hw_params(struct snd_sof_dev *sdev,
+			       struct snd_pcm_substream *substream,
+			       struct snd_pcm_hw_params *params,
+			       struct sof_ipc_stream_params *ipc_params)
+{
+	if (sof_ops(sdev) && sof_ops(sdev)->pcm_hw_params)
+		return sof_ops(sdev)->pcm_hw_params(sdev, substream,
+						    params, ipc_params);
+
+	return 0;
+}
+
+/* host stream trigger */
+static inline int
+snd_sof_pcm_platform_trigger(struct snd_sof_dev *sdev,
+			     struct snd_pcm_substream *substream, int cmd)
+{
+	if (sof_ops(sdev) && sof_ops(sdev)->pcm_trigger)
+		return sof_ops(sdev)->pcm_trigger(sdev, substream, cmd);
+
+	return 0;
+}
+
+/* host DSP message data */
+static inline void snd_sof_ipc_msg_data(struct snd_sof_dev *sdev,
+					struct snd_pcm_substream *substream,
+					void *p, size_t sz)
+{
+	sof_ops(sdev)->ipc_msg_data(sdev, substream, p, sz);
+}
+
+/* host configure DSP HW parameters */
+static inline int
+snd_sof_ipc_pcm_params(struct snd_sof_dev *sdev,
+		       struct snd_pcm_substream *substream,
+		       const struct sof_ipc_pcm_params_reply *reply)
+{
+	return sof_ops(sdev)->ipc_pcm_params(sdev, substream, reply);
+}
+
+/* host stream pointer */
+static inline snd_pcm_uframes_t
+snd_sof_pcm_platform_pointer(struct snd_sof_dev *sdev,
+			     struct snd_pcm_substream *substream)
+{
+	if (sof_ops(sdev) && sof_ops(sdev)->pcm_pointer)
+		return sof_ops(sdev)->pcm_pointer(sdev, substream);
+
+	return 0;
+}
+
+static inline const struct snd_sof_dsp_ops
+*sof_get_ops(const struct sof_dev_desc *d,
+	     const struct sof_ops_table mach_ops[], int asize)
+{
+	int i;
+
+	for (i = 0; i < asize; i++) {
+		if (d == mach_ops[i].desc)
+			return mach_ops[i].ops;
+	}
+
+	/* not found */
+	return NULL;
+}
+
+/**
+ * snd_sof_dsp_register_poll_timeout - Periodically poll an address
+ * until a condition is met or a timeout occurs
+ * @op: accessor function (takes @addr as its only argument)
+ * @addr: Address to poll
+ * @val: Variable to read the value into
+ * @cond: Break condition (usually involving @val)
+ * @sleep_us: Maximum time to sleep between reads in us (0
+ *            tight-loops).  Should be less than ~20ms since usleep_range
+ *            is used (see Documentation/timers/timers-howto.txt).
+ * @timeout_us: Timeout in us, 0 means never timeout
+ *
+ * Returns 0 on success and -ETIMEDOUT upon a timeout. In either
+ * case, the last read value at @addr is stored in @val. Must not
+ * be called from atomic context if sleep_us or timeout_us are used.
+ *
+ * This is modelled after the readx_poll_timeout macros in linux/iopoll.h.
+ */
+#define snd_sof_dsp_read_poll_timeout(sdev, bar, offset, val, cond, sleep_us, timeout_us) \
+({ \
+	u64 __timeout_us = (timeout_us); \
+	unsigned long __sleep_us = (sleep_us); \
+	ktime_t __timeout = ktime_add_us(ktime_get(), __timeout_us); \
+	might_sleep_if((__sleep_us) != 0); \
+	for (;;) {							\
+		(val) = snd_sof_dsp_read(sdev, bar, offset);		\
+		if (cond) { \
+			dev_dbg(sdev->dev, \
+				"FW Poll Status: reg=%#x successful\n", (val)); \
+			break; \
+		} \
+		if (__timeout_us && \
+		    ktime_compare(ktime_get(), __timeout) > 0) { \
+			(val) = snd_sof_dsp_read(sdev, bar, offset); \
+			dev_dbg(sdev->dev, \
+				"FW Poll Status: reg=%#x timedout\n", (val)); \
+			break; \
+		} \
+		if (__sleep_us) \
+			usleep_range((__sleep_us >> 2) + 1, __sleep_us); \
+	} \
+	(cond) ? 0 : -ETIMEDOUT; \
+})
+
+/* This is for registers bits with attribute RWC */
+bool snd_sof_pci_update_bits(struct snd_sof_dev *sdev, u32 offset,
+			     u32 mask, u32 value);
+
+bool snd_sof_dsp_update_bits_unlocked(struct snd_sof_dev *sdev, u32 bar,
+				      u32 offset, u32 mask, u32 value);
+
+bool snd_sof_dsp_update_bits64_unlocked(struct snd_sof_dev *sdev, u32 bar,
+					u32 offset, u64 mask, u64 value);
+
+bool snd_sof_dsp_update_bits(struct snd_sof_dev *sdev, u32 bar, u32 offset,
+			     u32 mask, u32 value);
+
+bool snd_sof_dsp_update_bits64(struct snd_sof_dev *sdev, u32 bar,
+			       u32 offset, u64 mask, u64 value);
+
+void snd_sof_dsp_update_bits_forced(struct snd_sof_dev *sdev, u32 bar,
+				    u32 offset, u32 mask, u32 value);
+
+int snd_sof_dsp_register_poll(struct snd_sof_dev *sdev, u32 bar, u32 offset,
+			      u32 mask, u32 target, u32 timeout_ms,
+			      u32 interval_us);
+
+void snd_sof_dsp_panic(struct snd_sof_dev *sdev, u32 offset);
+#endif
diff --git a/sound/soc/sof/pcm.c b/sound/soc/sof/pcm.c
new file mode 100644
index 000000000000..649968841dad
--- /dev/null
+++ b/sound/soc/sof/pcm.c
@@ -0,0 +1,767 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license.  When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2018 Intel Corporation. All rights reserved.
+//
+// Author: Liam Girdwood <liam.r.girdwood@linux.intel.com>
+//
+// PCM Layer, interface between ALSA and IPC.
+//
+
+#include <linux/pm_runtime.h>
+#include <sound/pcm_params.h>
+#include <sound/sof.h>
+#include "sof-priv.h"
+#include "ops.h"
+
+#define DRV_NAME	"sof-audio-component"
+
+/* Create DMA buffer page table for DSP */
+static int create_page_table(struct snd_pcm_substream *substream,
+			     unsigned char *dma_area, size_t size)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct snd_soc_component *component =
+		snd_soc_rtdcom_lookup(rtd, DRV_NAME);
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(component);
+	struct snd_sof_pcm *spcm;
+	struct snd_dma_buffer *dmab = snd_pcm_get_dma_buf(substream);
+	int stream = substream->stream;
+
+	spcm = snd_sof_find_spcm_dai(sdev, rtd);
+	if (!spcm)
+		return -EINVAL;
+
+	return snd_sof_create_page_table(sdev, dmab,
+		spcm->stream[stream].page_table.area, size);
+}
+
+static int sof_pcm_dsp_params(struct snd_sof_pcm *spcm, struct snd_pcm_substream *substream,
+			      const struct sof_ipc_pcm_params_reply *reply)
+{
+	struct snd_sof_dev *sdev = spcm->sdev;
+	/* validate offset */
+	int ret = snd_sof_ipc_pcm_params(sdev, substream, reply);
+
+	if (ret < 0)
+		dev_err(sdev->dev, "error: got wrong reply for PCM %d\n",
+			spcm->pcm.pcm_id);
+
+	return ret;
+}
+
+/*
+ * sof pcm period elapse work
+ */
+static void sof_pcm_period_elapsed_work(struct work_struct *work)
+{
+	struct snd_sof_pcm_stream *sps =
+		container_of(work, struct snd_sof_pcm_stream,
+			     period_elapsed_work);
+
+	snd_pcm_period_elapsed(sps->substream);
+}
+
+/*
+ * sof pcm period elapse, this could be called at irq thread context.
+ */
+void snd_sof_pcm_period_elapsed(struct snd_pcm_substream *substream)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct snd_soc_component *component =
+		snd_soc_rtdcom_lookup(rtd, DRV_NAME);
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(component);
+	struct snd_sof_pcm *spcm;
+
+	spcm = snd_sof_find_spcm_dai(sdev, rtd);
+	if (!spcm) {
+		dev_err(sdev->dev,
+			"error: period elapsed for unknown stream!\n");
+		return;
+	}
+
+	/*
+	 * snd_pcm_period_elapsed() can be called in interrupt context
+	 * before IRQ_HANDLED is returned. Inside snd_pcm_period_elapsed(),
+	 * when the PCM is done draining or xrun happened, a STOP IPC will
+	 * then be sent and this IPC will hit IPC timeout.
+	 * To avoid sending IPC before the previous IPC is handled, we
+	 * schedule delayed work here to call the snd_pcm_period_elapsed().
+	 */
+	schedule_work(&spcm->stream[substream->stream].period_elapsed_work);
+}
+EXPORT_SYMBOL(snd_sof_pcm_period_elapsed);
+
+/* this may get called several times by oss emulation */
+static int sof_pcm_hw_params(struct snd_pcm_substream *substream,
+			     struct snd_pcm_hw_params *params)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct snd_pcm_runtime *runtime = substream->runtime;
+	struct snd_soc_component *component =
+		snd_soc_rtdcom_lookup(rtd, DRV_NAME);
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(component);
+	struct snd_sof_pcm *spcm;
+	struct sof_ipc_pcm_params pcm;
+	struct sof_ipc_pcm_params_reply ipc_params_reply;
+	int ret;
+
+	/* nothing to do for BE */
+	if (rtd->dai_link->no_pcm)
+		return 0;
+
+	spcm = snd_sof_find_spcm_dai(sdev, rtd);
+	if (!spcm)
+		return -EINVAL;
+
+	dev_dbg(sdev->dev, "pcm: hw params stream %d dir %d\n",
+		spcm->pcm.pcm_id, substream->stream);
+
+	memset(&pcm, 0, sizeof(pcm));
+
+	/* allocate audio buffer pages */
+	ret = snd_pcm_lib_malloc_pages(substream, params_buffer_bytes(params));
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: could not allocate %d bytes for PCM %d\n",
+			params_buffer_bytes(params), spcm->pcm.pcm_id);
+		return ret;
+	}
+	if (ret) {
+		/*
+		 * ret == 1 means the buffer is changed
+		 * create compressed page table for audio firmware
+		 * ret == 0 means the buffer is not changed
+		 * so no need to regenerate the page table
+		 */
+		ret = create_page_table(substream, runtime->dma_area,
+					runtime->dma_bytes);
+		if (ret < 0)
+			return ret;
+	}
+
+	/* number of pages should be rounded up */
+	pcm.params.buffer.pages = PFN_UP(runtime->dma_bytes);
+
+	/* set IPC PCM parameters */
+	pcm.hdr.size = sizeof(pcm);
+	pcm.hdr.cmd = SOF_IPC_GLB_STREAM_MSG | SOF_IPC_STREAM_PCM_PARAMS;
+	pcm.comp_id = spcm->stream[substream->stream].comp_id;
+	pcm.params.hdr.size = sizeof(pcm.params);
+	pcm.params.buffer.phy_addr =
+		spcm->stream[substream->stream].page_table.addr;
+	pcm.params.buffer.size = runtime->dma_bytes;
+	pcm.params.direction = substream->stream;
+	pcm.params.sample_valid_bytes = params_width(params) >> 3;
+	pcm.params.buffer_fmt = SOF_IPC_BUFFER_INTERLEAVED;
+	pcm.params.rate = params_rate(params);
+	pcm.params.channels = params_channels(params);
+	pcm.params.host_period_bytes = params_period_bytes(params);
+
+	/* container size */
+	ret = snd_pcm_format_physical_width(params_format(params));
+	if (ret < 0)
+		return ret;
+	pcm.params.sample_container_bytes = ret >> 3;
+
+	/* format */
+	switch (params_format(params)) {
+	case SNDRV_PCM_FORMAT_S16:
+		pcm.params.frame_fmt = SOF_IPC_FRAME_S16_LE;
+		break;
+	case SNDRV_PCM_FORMAT_S24:
+		pcm.params.frame_fmt = SOF_IPC_FRAME_S24_4LE;
+		break;
+	case SNDRV_PCM_FORMAT_S32:
+		pcm.params.frame_fmt = SOF_IPC_FRAME_S32_LE;
+		break;
+	case SNDRV_PCM_FORMAT_FLOAT:
+		pcm.params.frame_fmt = SOF_IPC_FRAME_FLOAT;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	/* firmware already configured host stream */
+	ret = snd_sof_pcm_platform_hw_params(sdev,
+					     substream,
+					     params,
+					     &pcm.params);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: platform hw params failed\n");
+		return ret;
+	}
+
+	dev_dbg(sdev->dev, "stream_tag %d", pcm.params.stream_tag);
+
+	/* send IPC to the DSP */
+	ret = sof_ipc_tx_message(sdev->ipc, pcm.hdr.cmd, &pcm, sizeof(pcm),
+				 &ipc_params_reply, sizeof(ipc_params_reply));
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: hw params ipc failed for stream %d\n",
+			pcm.params.stream_tag);
+		return ret;
+	}
+
+	ret = sof_pcm_dsp_params(spcm, substream, &ipc_params_reply);
+	if (ret < 0)
+		return ret;
+
+	/* save pcm hw_params */
+	memcpy(&spcm->params[substream->stream], params, sizeof(*params));
+
+	INIT_WORK(&spcm->stream[substream->stream].period_elapsed_work,
+		  sof_pcm_period_elapsed_work);
+
+	return ret;
+}
+
+static int sof_pcm_hw_free(struct snd_pcm_substream *substream)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct snd_soc_component *component =
+		snd_soc_rtdcom_lookup(rtd, DRV_NAME);
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(component);
+	struct snd_sof_pcm *spcm;
+	struct sof_ipc_stream stream;
+	struct sof_ipc_reply reply;
+	int ret;
+
+	/* nothing to do for BE */
+	if (rtd->dai_link->no_pcm)
+		return 0;
+
+	spcm = snd_sof_find_spcm_dai(sdev, rtd);
+	if (!spcm)
+		return -EINVAL;
+
+	dev_dbg(sdev->dev, "pcm: free stream %d dir %d\n", spcm->pcm.pcm_id,
+		substream->stream);
+
+	stream.hdr.size = sizeof(stream);
+	stream.hdr.cmd = SOF_IPC_GLB_STREAM_MSG | SOF_IPC_STREAM_PCM_FREE;
+	stream.comp_id = spcm->stream[substream->stream].comp_id;
+
+	/* send IPC to the DSP */
+	ret = sof_ipc_tx_message(sdev->ipc, stream.hdr.cmd, &stream,
+				 sizeof(stream), &reply, sizeof(reply));
+
+	snd_pcm_lib_free_pages(substream);
+
+	cancel_work_sync(&spcm->stream[substream->stream].period_elapsed_work);
+
+	return ret;
+}
+
+static int sof_pcm_prepare(struct snd_pcm_substream *substream)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct snd_soc_component *component =
+		snd_soc_rtdcom_lookup(rtd, DRV_NAME);
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(component);
+	struct snd_sof_pcm *spcm;
+	int ret;
+
+	/* nothing to do for BE */
+	if (rtd->dai_link->no_pcm)
+		return 0;
+
+	spcm = snd_sof_find_spcm_dai(sdev, rtd);
+	if (!spcm)
+		return -EINVAL;
+
+	/*
+	 * check if hw_params needs to be set-up again.
+	 * This is only needed when resuming from system sleep.
+	 */
+	if (!spcm->hw_params_upon_resume[substream->stream])
+		return 0;
+
+	dev_dbg(sdev->dev, "pcm: prepare stream %d dir %d\n", spcm->pcm.pcm_id,
+		substream->stream);
+
+	/* set hw_params */
+	ret = sof_pcm_hw_params(substream, &spcm->params[substream->stream]);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: set pcm hw_params after resume\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+/*
+ * FE dai link trigger actions are always executed in non-atomic context because
+ * they involve IPC's.
+ */
+static int sof_pcm_trigger(struct snd_pcm_substream *substream, int cmd)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct snd_soc_component *component =
+		snd_soc_rtdcom_lookup(rtd, DRV_NAME);
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(component);
+	struct snd_sof_pcm *spcm;
+	struct sof_ipc_stream stream;
+	struct sof_ipc_reply reply;
+	int ret;
+
+	/* nothing to do for BE */
+	if (rtd->dai_link->no_pcm)
+		return 0;
+
+	spcm = snd_sof_find_spcm_dai(sdev, rtd);
+	if (!spcm)
+		return -EINVAL;
+
+	dev_dbg(sdev->dev, "pcm: trigger stream %d dir %d cmd %d\n",
+		spcm->pcm.pcm_id, substream->stream, cmd);
+
+	stream.hdr.size = sizeof(stream);
+	stream.hdr.cmd = SOF_IPC_GLB_STREAM_MSG;
+	stream.comp_id = spcm->stream[substream->stream].comp_id;
+
+	switch (cmd) {
+	case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
+		stream.hdr.cmd |= SOF_IPC_STREAM_TRIG_PAUSE;
+		break;
+	case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
+		stream.hdr.cmd |= SOF_IPC_STREAM_TRIG_RELEASE;
+		break;
+	case SNDRV_PCM_TRIGGER_RESUME:
+		/* set up hw_params */
+		ret = sof_pcm_prepare(substream);
+		if (ret < 0) {
+			dev_err(sdev->dev,
+				"error: failed to set up hw_params upon resume\n");
+			return ret;
+		}
+
+		/* fallthrough */
+	case SNDRV_PCM_TRIGGER_START:
+		stream.hdr.cmd |= SOF_IPC_STREAM_TRIG_START;
+		break;
+	case SNDRV_PCM_TRIGGER_SUSPEND:
+	case SNDRV_PCM_TRIGGER_STOP:
+		stream.hdr.cmd |= SOF_IPC_STREAM_TRIG_STOP;
+		break;
+	default:
+		dev_err(sdev->dev, "error: unhandled trigger cmd %d\n", cmd);
+		return -EINVAL;
+	}
+
+	snd_sof_pcm_platform_trigger(sdev, substream, cmd);
+
+	/* send IPC to the DSP */
+	ret = sof_ipc_tx_message(sdev->ipc, stream.hdr.cmd, &stream,
+				 sizeof(stream), &reply, sizeof(reply));
+
+	if (ret < 0 || cmd != SNDRV_PCM_TRIGGER_SUSPEND)
+		return ret;
+
+	/*
+	 * The hw_free op is usually called when the pcm stream is closed.
+	 * Since the stream is not closed during suspend, the DSP needs to be
+	 * notified explicitly to free pcm to prevent errors upon resume.
+	 */
+	stream.hdr.size = sizeof(stream);
+	stream.hdr.cmd = SOF_IPC_GLB_STREAM_MSG | SOF_IPC_STREAM_PCM_FREE;
+	stream.comp_id = spcm->stream[substream->stream].comp_id;
+
+	/* send IPC to the DSP */
+	return sof_ipc_tx_message(sdev->ipc, stream.hdr.cmd, &stream,
+				  sizeof(stream), &reply, sizeof(reply));
+}
+
+static snd_pcm_uframes_t sof_pcm_pointer(struct snd_pcm_substream *substream)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct snd_soc_component *component =
+		snd_soc_rtdcom_lookup(rtd, DRV_NAME);
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(component);
+	struct snd_sof_pcm *spcm;
+	snd_pcm_uframes_t host, dai;
+
+	/* nothing to do for BE */
+	if (rtd->dai_link->no_pcm)
+		return 0;
+
+	/* use dsp ops pointer callback directly if set */
+	if (sof_ops(sdev)->pcm_pointer)
+		return sof_ops(sdev)->pcm_pointer(sdev, substream);
+
+	spcm = snd_sof_find_spcm_dai(sdev, rtd);
+	if (!spcm)
+		return -EINVAL;
+
+	/* read position from DSP */
+	host = bytes_to_frames(substream->runtime,
+			       spcm->stream[substream->stream].posn.host_posn);
+	dai = bytes_to_frames(substream->runtime,
+			      spcm->stream[substream->stream].posn.dai_posn);
+
+	dev_dbg(sdev->dev, "PCM: stream %d dir %d DMA position %lu DAI position %lu\n",
+		spcm->pcm.pcm_id, substream->stream, host, dai);
+
+	return host;
+}
+
+static int sof_pcm_open(struct snd_pcm_substream *substream)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct snd_pcm_runtime *runtime = substream->runtime;
+	struct snd_soc_component *component =
+		snd_soc_rtdcom_lookup(rtd, DRV_NAME);
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(component);
+	struct snd_sof_pcm *spcm;
+	struct snd_soc_tplg_stream_caps *caps;
+	int ret;
+	int err;
+
+	/* nothing to do for BE */
+	if (rtd->dai_link->no_pcm)
+		return 0;
+
+	spcm = snd_sof_find_spcm_dai(sdev, rtd);
+	if (!spcm)
+		return -EINVAL;
+
+	dev_dbg(sdev->dev, "pcm: open stream %d dir %d\n", spcm->pcm.pcm_id,
+		substream->stream);
+
+	/* clear hw_params_upon_resume flag */
+	spcm->hw_params_upon_resume[substream->stream] = 0;
+
+	caps = &spcm->pcm.caps[substream->stream];
+
+	ret = pm_runtime_get_sync(sdev->dev);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: pcm open failed to resume %d\n",
+			ret);
+		pm_runtime_put_noidle(sdev->dev);
+		return ret;
+	}
+
+	/* set any runtime constraints based on topology */
+	snd_pcm_hw_constraint_step(substream->runtime, 0,
+				   SNDRV_PCM_HW_PARAM_BUFFER_BYTES,
+				   le32_to_cpu(caps->period_size_min));
+	snd_pcm_hw_constraint_step(substream->runtime, 0,
+				   SNDRV_PCM_HW_PARAM_PERIOD_BYTES,
+				   le32_to_cpu(caps->period_size_min));
+
+	/* set runtime config */
+	runtime->hw.info = SNDRV_PCM_INFO_MMAP |
+			  SNDRV_PCM_INFO_MMAP_VALID |
+			  SNDRV_PCM_INFO_INTERLEAVED |
+			  SNDRV_PCM_INFO_PAUSE |
+			  SNDRV_PCM_INFO_NO_PERIOD_WAKEUP;
+	runtime->hw.formats = le64_to_cpu(caps->formats);
+	runtime->hw.period_bytes_min = le32_to_cpu(caps->period_size_min);
+	runtime->hw.period_bytes_max = le32_to_cpu(caps->period_size_max);
+	runtime->hw.periods_min = le32_to_cpu(caps->periods_min);
+	runtime->hw.periods_max = le32_to_cpu(caps->periods_max);
+
+	/*
+	 * caps->buffer_size_min is not used since the
+	 * snd_pcm_hardware structure only defines buffer_bytes_max
+	 */
+	runtime->hw.buffer_bytes_max = le32_to_cpu(caps->buffer_size_max);
+
+	dev_dbg(sdev->dev, "period min %zd max %zd bytes\n",
+		runtime->hw.period_bytes_min,
+		runtime->hw.period_bytes_max);
+	dev_dbg(sdev->dev, "period count %d max %d\n",
+		runtime->hw.periods_min,
+		runtime->hw.periods_max);
+	dev_dbg(sdev->dev, "buffer max %zd bytes\n",
+		runtime->hw.buffer_bytes_max);
+
+	/* set wait time - TODO: come from topology */
+	substream->wait_time = 500;
+
+	spcm->stream[substream->stream].posn.host_posn = 0;
+	spcm->stream[substream->stream].posn.dai_posn = 0;
+	spcm->stream[substream->stream].substream = substream;
+
+	ret = snd_sof_pcm_platform_open(sdev, substream);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: pcm open failed %d\n",
+			ret);
+
+		pm_runtime_mark_last_busy(sdev->dev);
+
+		err = pm_runtime_put_autosuspend(sdev->dev);
+		if (err < 0)
+			dev_err(sdev->dev, "error: pcm close failed to idle %d\n",
+				err);
+	}
+
+	return ret;
+}
+
+static int sof_pcm_close(struct snd_pcm_substream *substream)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct snd_soc_component *component =
+		snd_soc_rtdcom_lookup(rtd, DRV_NAME);
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(component);
+	struct snd_sof_pcm *spcm;
+	int err;
+
+	/* nothing to do for BE */
+	if (rtd->dai_link->no_pcm)
+		return 0;
+
+	spcm = snd_sof_find_spcm_dai(sdev, rtd);
+	if (!spcm)
+		return -EINVAL;
+
+	dev_dbg(sdev->dev, "pcm: close stream %d dir %d\n", spcm->pcm.pcm_id,
+		substream->stream);
+
+	err = snd_sof_pcm_platform_close(sdev, substream);
+	if (err < 0) {
+		dev_err(sdev->dev, "error: pcm close failed %d\n",
+			err);
+		/*
+		 * keep going, no point in preventing the close
+		 * from happening
+		 */
+	}
+
+	pm_runtime_mark_last_busy(sdev->dev);
+
+	err = pm_runtime_put_autosuspend(sdev->dev);
+	if (err < 0)
+		dev_err(sdev->dev, "error: pcm close failed to idle %d\n",
+			err);
+
+	return 0;
+}
+
+static struct snd_pcm_ops sof_pcm_ops = {
+	.open		= sof_pcm_open,
+	.close		= sof_pcm_close,
+	.ioctl		= snd_pcm_lib_ioctl,
+	.hw_params	= sof_pcm_hw_params,
+	.prepare	= sof_pcm_prepare,
+	.hw_free	= sof_pcm_hw_free,
+	.trigger	= sof_pcm_trigger,
+	.pointer	= sof_pcm_pointer,
+	.page		= snd_pcm_sgbuf_ops_page,
+};
+
+/*
+ * Pre-allocate playback/capture audio buffer pages.
+ * no need to explicitly release memory preallocated by sof_pcm_new in pcm_free
+ * snd_pcm_lib_preallocate_free_for_all() is called by the core.
+ */
+static int sof_pcm_new(struct snd_soc_pcm_runtime *rtd)
+{
+	struct snd_soc_component *component =
+		snd_soc_rtdcom_lookup(rtd, DRV_NAME);
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(component);
+	struct snd_sof_pcm *spcm;
+	struct snd_pcm *pcm = rtd->pcm;
+	struct snd_soc_tplg_stream_caps *caps;
+	int stream = SNDRV_PCM_STREAM_PLAYBACK;
+
+	/* find SOF PCM for this RTD */
+	spcm = snd_sof_find_spcm_dai(sdev, rtd);
+	if (!spcm) {
+		dev_warn(sdev->dev, "warn: can't find PCM with DAI ID %d\n",
+			 rtd->dai_link->id);
+		return 0;
+	}
+
+	dev_dbg(sdev->dev, "creating new PCM %s\n", spcm->pcm.pcm_name);
+
+	/* do we need to pre-allocate playback audio buffer pages */
+	if (!spcm->pcm.playback)
+		goto capture;
+
+	caps = &spcm->pcm.caps[stream];
+
+	/* pre-allocate playback audio buffer pages */
+	dev_dbg(sdev->dev, "spcm: allocate %s playback DMA buffer size 0x%x max 0x%x\n",
+		caps->name, caps->buffer_size_min, caps->buffer_size_max);
+
+	snd_pcm_lib_preallocate_pages(pcm->streams[stream].substream,
+				      SNDRV_DMA_TYPE_DEV_SG, sdev->dev,
+				      le32_to_cpu(caps->buffer_size_min),
+				      le32_to_cpu(caps->buffer_size_max));
+capture:
+	stream = SNDRV_PCM_STREAM_CAPTURE;
+
+	/* do we need to pre-allocate capture audio buffer pages */
+	if (!spcm->pcm.capture)
+		return 0;
+
+	caps = &spcm->pcm.caps[stream];
+
+	/* pre-allocate capture audio buffer pages */
+	dev_dbg(sdev->dev, "spcm: allocate %s capture DMA buffer size 0x%x max 0x%x\n",
+		caps->name, caps->buffer_size_min, caps->buffer_size_max);
+
+	snd_pcm_lib_preallocate_pages(pcm->streams[stream].substream,
+				      SNDRV_DMA_TYPE_DEV_SG, sdev->dev,
+				      le32_to_cpu(caps->buffer_size_min),
+				      le32_to_cpu(caps->buffer_size_max));
+
+	return 0;
+}
+
+/* fixup the BE DAI link to match any values from topology */
+static int sof_pcm_dai_link_fixup(struct snd_soc_pcm_runtime *rtd,
+				  struct snd_pcm_hw_params *params)
+{
+	struct snd_interval *rate = hw_param_interval(params,
+			SNDRV_PCM_HW_PARAM_RATE);
+	struct snd_interval *channels = hw_param_interval(params,
+						SNDRV_PCM_HW_PARAM_CHANNELS);
+	struct snd_mask *fmt = hw_param_mask(params, SNDRV_PCM_HW_PARAM_FORMAT);
+	struct snd_soc_component *component =
+		snd_soc_rtdcom_lookup(rtd, DRV_NAME);
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(component);
+	struct snd_sof_dai *dai =
+		snd_sof_find_dai(sdev, (char *)rtd->dai_link->name);
+
+	/* no topology exists for this BE, try a common configuration */
+	if (!dai) {
+		dev_warn(sdev->dev, "warning: no topology found for BE DAI %s config\n",
+			 rtd->dai_link->name);
+
+		/*  set 48k, stereo, 16bits by default */
+		rate->min = 48000;
+		rate->max = 48000;
+
+		channels->min = 2;
+		channels->max = 2;
+
+		snd_mask_none(fmt);
+		snd_mask_set_format(fmt, SNDRV_PCM_FORMAT_S16_LE);
+
+		return 0;
+	}
+
+	/* read format from topology */
+	snd_mask_none(fmt);
+
+	switch (dai->comp_dai.config.frame_fmt) {
+	case SOF_IPC_FRAME_S16_LE:
+		snd_mask_set_format(fmt, SNDRV_PCM_FORMAT_S16_LE);
+		break;
+	case SOF_IPC_FRAME_S24_4LE:
+		snd_mask_set_format(fmt, SNDRV_PCM_FORMAT_S24_LE);
+		break;
+	case SOF_IPC_FRAME_S32_LE:
+		snd_mask_set_format(fmt, SNDRV_PCM_FORMAT_S32_LE);
+		break;
+	default:
+		dev_err(sdev->dev, "error: No available DAI format!\n");
+		return -EINVAL;
+	}
+
+	/* read rate and channels from topology */
+	switch (dai->dai_config->type) {
+	case SOF_DAI_INTEL_SSP:
+		rate->min = dai->dai_config->ssp.fsync_rate;
+		rate->max = dai->dai_config->ssp.fsync_rate;
+		channels->min = dai->dai_config->ssp.tdm_slots;
+		channels->max = dai->dai_config->ssp.tdm_slots;
+
+		dev_dbg(sdev->dev,
+			"rate_min: %d rate_max: %d\n", rate->min, rate->max);
+		dev_dbg(sdev->dev,
+			"channels_min: %d channels_max: %d\n",
+			channels->min, channels->max);
+
+		break;
+	case SOF_DAI_INTEL_DMIC:
+		/* DMIC only supports 16 or 32 bit formats */
+		if (dai->comp_dai.config.frame_fmt == SOF_IPC_FRAME_S24_4LE) {
+			dev_err(sdev->dev,
+				"error: invalid fmt %d for DAI type %d\n",
+				dai->comp_dai.config.frame_fmt,
+				dai->dai_config->type);
+		}
+		break;
+	case SOF_DAI_INTEL_HDA:
+		/* do nothing for HDA dai_link */
+		break;
+	default:
+		dev_err(sdev->dev, "error: invalid DAI type %d\n",
+			dai->dai_config->type);
+		break;
+	}
+
+	return 0;
+}
+
+static int sof_pcm_probe(struct snd_soc_component *component)
+{
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(component);
+	struct snd_sof_pdata *plat_data = sdev->pdata;
+	const char *tplg_filename;
+	int ret;
+
+	/* load the default topology */
+	sdev->component = component;
+
+	tplg_filename = devm_kasprintf(sdev->dev, GFP_KERNEL,
+				       "%s/%s",
+				       plat_data->tplg_filename_prefix,
+				       plat_data->tplg_filename);
+	if (!tplg_filename)
+		return -ENOMEM;
+
+	ret = snd_sof_load_topology(sdev, tplg_filename);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: failed to load DSP topology %d\n",
+			ret);
+		return ret;
+	}
+
+	/*
+	 * Some platforms in SOF, ex: BYT, may not have their platform PM
+	 * callbacks set. Increment the usage count so as to
+	 * prevent the device from entering runtime suspend.
+	 */
+	if (!sof_ops(sdev)->runtime_suspend || !sof_ops(sdev)->runtime_resume)
+		pm_runtime_get_noresume(sdev->dev);
+
+	return ret;
+}
+
+static void sof_pcm_remove(struct snd_soc_component *component)
+{
+	/* remove topology */
+	snd_soc_tplg_component_remove(component, SND_SOC_TPLG_INDEX_ALL);
+}
+
+void snd_sof_new_platform_drv(struct snd_sof_dev *sdev)
+{
+	struct snd_soc_component_driver *pd = &sdev->plat_drv;
+	struct snd_sof_pdata *plat_data = sdev->pdata;
+	const char *drv_name;
+
+	drv_name = plat_data->machine->drv_name;
+
+	pd->name = "sof-audio-component";
+	pd->probe = sof_pcm_probe;
+	pd->remove = sof_pcm_remove;
+	pd->ops	= &sof_pcm_ops;
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_COMPRESS)
+	pd->compr_ops = &sof_compressed_ops;
+#endif
+	pd->pcm_new = sof_pcm_new;
+	pd->ignore_machine = drv_name;
+	pd->be_hw_params_fixup = sof_pcm_dai_link_fixup;
+	pd->be_pcm_base = SOF_BE_PCM_BASE;
+	pd->use_dai_pcm_id = true;
+	pd->topology_name_prefix = "sof";
+
+	 /* increment module refcount when a pcm is opened */
+	pd->module_get_upon_open = 1;
+}
diff --git a/sound/soc/sof/pm.c b/sound/soc/sof/pm.c
new file mode 100644
index 000000000000..8ef1d51025d8
--- /dev/null
+++ b/sound/soc/sof/pm.c
@@ -0,0 +1,388 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license.  When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2018 Intel Corporation. All rights reserved.
+//
+// Author: Liam Girdwood <liam.r.girdwood@linux.intel.com>
+//
+
+#include "ops.h"
+#include "sof-priv.h"
+
+static int sof_restore_kcontrols(struct snd_sof_dev *sdev)
+{
+	struct snd_sof_control *scontrol;
+	int ipc_cmd, ctrl_type;
+	int ret = 0;
+
+	/* restore kcontrol values */
+	list_for_each_entry(scontrol, &sdev->kcontrol_list, list) {
+		/* reset readback offset for scontrol after resuming */
+		scontrol->readback_offset = 0;
+
+		/* notify DSP of kcontrol values */
+		switch (scontrol->cmd) {
+		case SOF_CTRL_CMD_VOLUME:
+		case SOF_CTRL_CMD_ENUM:
+		case SOF_CTRL_CMD_SWITCH:
+			ipc_cmd = SOF_IPC_COMP_SET_VALUE;
+			ctrl_type = SOF_CTRL_TYPE_VALUE_CHAN_SET;
+			ret = snd_sof_ipc_set_get_comp_data(sdev->ipc, scontrol,
+							    ipc_cmd, ctrl_type,
+							    scontrol->cmd,
+							    true);
+			break;
+		case SOF_CTRL_CMD_BINARY:
+			ipc_cmd = SOF_IPC_COMP_SET_DATA;
+			ctrl_type = SOF_CTRL_TYPE_DATA_SET;
+			ret = snd_sof_ipc_set_get_comp_data(sdev->ipc, scontrol,
+							    ipc_cmd, ctrl_type,
+							    scontrol->cmd,
+							    true);
+			break;
+
+		default:
+			break;
+		}
+
+		if (ret < 0) {
+			dev_err(sdev->dev,
+				"error: failed kcontrol value set for widget: %d\n",
+				scontrol->comp_id);
+
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int sof_restore_pipelines(struct snd_sof_dev *sdev)
+{
+	struct snd_sof_widget *swidget;
+	struct snd_sof_route *sroute;
+	struct sof_ipc_pipe_new *pipeline;
+	struct snd_sof_dai *dai;
+	struct sof_ipc_comp_dai *comp_dai;
+	struct sof_ipc_cmd_hdr *hdr;
+	int ret;
+
+	/* restore pipeline components */
+	list_for_each_entry_reverse(swidget, &sdev->widget_list, list) {
+		struct sof_ipc_comp_reply r;
+
+		/* skip if there is no private data */
+		if (!swidget->private)
+			continue;
+
+		switch (swidget->id) {
+		case snd_soc_dapm_dai_in:
+		case snd_soc_dapm_dai_out:
+			dai = swidget->private;
+			comp_dai = &dai->comp_dai;
+			ret = sof_ipc_tx_message(sdev->ipc,
+						 comp_dai->comp.hdr.cmd,
+						 comp_dai, sizeof(*comp_dai),
+						 &r, sizeof(r));
+			break;
+		case snd_soc_dapm_scheduler:
+
+			/*
+			 * During suspend, all DSP cores are powered off.
+			 * Therefore upon resume, create the pipeline comp
+			 * and power up the core that the pipeline is
+			 * scheduled on.
+			 */
+			pipeline = swidget->private;
+			ret = sof_load_pipeline_ipc(sdev, pipeline, &r);
+			break;
+		default:
+			hdr = swidget->private;
+			ret = sof_ipc_tx_message(sdev->ipc, hdr->cmd,
+						 swidget->private, hdr->size,
+						 &r, sizeof(r));
+			break;
+		}
+		if (ret < 0) {
+			dev_err(sdev->dev,
+				"error: failed to load widget type %d with ID: %d\n",
+				swidget->widget->id, swidget->comp_id);
+
+			return ret;
+		}
+	}
+
+	/* restore pipeline connections */
+	list_for_each_entry_reverse(sroute, &sdev->route_list, list) {
+		struct sof_ipc_pipe_comp_connect *connect;
+		struct sof_ipc_reply reply;
+
+		/* skip if there's no private data */
+		if (!sroute->private)
+			continue;
+
+		connect = sroute->private;
+
+		/* send ipc */
+		ret = sof_ipc_tx_message(sdev->ipc,
+					 connect->hdr.cmd,
+					 connect, sizeof(*connect),
+					 &reply, sizeof(reply));
+		if (ret < 0) {
+			dev_err(sdev->dev,
+				"error: failed to load route sink %s control %s source %s\n",
+				sroute->route->sink,
+				sroute->route->control ? sroute->route->control
+					: "none",
+				sroute->route->source);
+
+			return ret;
+		}
+	}
+
+	/* restore dai links */
+	list_for_each_entry_reverse(dai, &sdev->dai_list, list) {
+		struct sof_ipc_reply reply;
+		struct sof_ipc_dai_config *config = dai->dai_config;
+
+		if (!config) {
+			dev_err(sdev->dev, "error: no config for DAI %s\n",
+				dai->name);
+			continue;
+		}
+
+		ret = sof_ipc_tx_message(sdev->ipc,
+					 config->hdr.cmd, config,
+					 config->hdr.size,
+					 &reply, sizeof(reply));
+
+		if (ret < 0) {
+			dev_err(sdev->dev,
+				"error: failed to set dai config for %s\n",
+				dai->name);
+
+			return ret;
+		}
+	}
+
+	/* complete pipeline */
+	list_for_each_entry(swidget, &sdev->widget_list, list) {
+		switch (swidget->id) {
+		case snd_soc_dapm_scheduler:
+			swidget->complete =
+				snd_sof_complete_pipeline(sdev, swidget);
+			break;
+		default:
+			break;
+		}
+	}
+
+	/* restore pipeline kcontrols */
+	ret = sof_restore_kcontrols(sdev);
+	if (ret < 0)
+		dev_err(sdev->dev,
+			"error: restoring kcontrols after resume\n");
+
+	return ret;
+}
+
+static int sof_send_pm_ipc(struct snd_sof_dev *sdev, int cmd)
+{
+	struct sof_ipc_pm_ctx pm_ctx;
+	struct sof_ipc_reply reply;
+
+	memset(&pm_ctx, 0, sizeof(pm_ctx));
+
+	/* configure ctx save ipc message */
+	pm_ctx.hdr.size = sizeof(pm_ctx);
+	pm_ctx.hdr.cmd = SOF_IPC_GLB_PM_MSG | cmd;
+
+	/* send ctx save ipc to dsp */
+	return sof_ipc_tx_message(sdev->ipc, pm_ctx.hdr.cmd, &pm_ctx,
+				 sizeof(pm_ctx), &reply, sizeof(reply));
+}
+
+static void sof_set_hw_params_upon_resume(struct snd_sof_dev *sdev)
+{
+	struct snd_pcm_substream *substream;
+	struct snd_sof_pcm *spcm;
+	snd_pcm_state_t state;
+	int dir;
+
+	/*
+	 * SOF requires hw_params to be set-up internally upon resume.
+	 * So, set the flag to indicate this for those streams that
+	 * have been suspended.
+	 */
+	list_for_each_entry(spcm, &sdev->pcm_list, list) {
+		for (dir = 0; dir <= SNDRV_PCM_STREAM_CAPTURE; dir++) {
+			substream = spcm->stream[dir].substream;
+			if (!substream || !substream->runtime)
+				continue;
+
+			state = substream->runtime->status->state;
+			if (state == SNDRV_PCM_STATE_SUSPENDED)
+				spcm->hw_params_upon_resume[dir] = 1;
+		}
+	}
+
+	/* set internal flag for BE */
+	snd_sof_dsp_hw_params_upon_resume(sdev);
+}
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG_ENABLE_DEBUGFS_CACHE)
+static void sof_cache_debugfs(struct snd_sof_dev *sdev)
+{
+	struct snd_sof_dfsentry *dfse;
+
+	list_for_each_entry(dfse, &sdev->dfsentry_list, list) {
+
+		/* nothing to do if debugfs buffer is not IO mem */
+		if (dfse->type == SOF_DFSENTRY_TYPE_BUF)
+			continue;
+
+		/* cache memory that is only accessible in D0 */
+		if (dfse->access_type == SOF_DEBUGFS_ACCESS_D0_ONLY)
+			memcpy_fromio(dfse->cache_buf, dfse->io_mem,
+				      dfse->size);
+	}
+}
+#endif
+
+static int sof_resume(struct device *dev, bool runtime_resume)
+{
+	struct snd_sof_dev *sdev = dev_get_drvdata(dev);
+	int ret;
+
+	/* do nothing if dsp resume callbacks are not set */
+	if (!sof_ops(sdev)->resume || !sof_ops(sdev)->runtime_resume)
+		return 0;
+
+	/*
+	 * if the runtime_resume flag is set, call the runtime_resume routine
+	 * or else call the system resume routine
+	 */
+	if (runtime_resume)
+		ret = snd_sof_dsp_runtime_resume(sdev);
+	else
+		ret = snd_sof_dsp_resume(sdev);
+	if (ret < 0) {
+		dev_err(sdev->dev,
+			"error: failed to power up DSP after resume\n");
+		return ret;
+	}
+
+	/* load the firmware */
+	ret = snd_sof_load_firmware(sdev);
+	if (ret < 0) {
+		dev_err(sdev->dev,
+			"error: failed to load DSP firmware after resume %d\n",
+			ret);
+		return ret;
+	}
+
+	/* boot the firmware */
+	ret = snd_sof_run_firmware(sdev);
+	if (ret < 0) {
+		dev_err(sdev->dev,
+			"error: failed to boot DSP firmware after resume %d\n",
+			ret);
+		return ret;
+	}
+
+	/* resume DMA trace, only need send ipc */
+	ret = snd_sof_init_trace_ipc(sdev);
+	if (ret < 0) {
+		/* non fatal */
+		dev_warn(sdev->dev,
+			 "warning: failed to init trace after resume %d\n",
+			 ret);
+	}
+
+	/* restore pipelines */
+	ret = sof_restore_pipelines(sdev);
+	if (ret < 0) {
+		dev_err(sdev->dev,
+			"error: failed to restore pipeline after resume %d\n",
+			ret);
+		return ret;
+	}
+
+	/* notify DSP of system resume */
+	ret = sof_send_pm_ipc(sdev, SOF_IPC_PM_CTX_RESTORE);
+	if (ret < 0)
+		dev_err(sdev->dev,
+			"error: ctx_restore ipc error during resume %d\n",
+			ret);
+
+	return ret;
+}
+
+static int sof_suspend(struct device *dev, bool runtime_suspend)
+{
+	struct snd_sof_dev *sdev = dev_get_drvdata(dev);
+	int ret;
+
+	/* do nothing if dsp suspend callback is not set */
+	if (!sof_ops(sdev)->suspend)
+		return 0;
+
+	/* release trace */
+	snd_sof_release_trace(sdev);
+
+	/* set restore_stream for all streams during system suspend */
+	if (!runtime_suspend)
+		sof_set_hw_params_upon_resume(sdev);
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG_ENABLE_DEBUGFS_CACHE)
+	/* cache debugfs contents during runtime suspend */
+	if (runtime_suspend)
+		sof_cache_debugfs(sdev);
+#endif
+	/* notify DSP of upcoming power down */
+	ret = sof_send_pm_ipc(sdev, SOF_IPC_PM_CTX_SAVE);
+	if (ret < 0) {
+		dev_err(sdev->dev,
+			"error: ctx_save ipc error during suspend %d\n",
+			ret);
+		return ret;
+	}
+
+	/* power down all DSP cores */
+	if (runtime_suspend)
+		ret = snd_sof_dsp_runtime_suspend(sdev, 0);
+	else
+		ret = snd_sof_dsp_suspend(sdev, 0);
+	if (ret < 0)
+		dev_err(sdev->dev,
+			"error: failed to power down DSP during suspend %d\n",
+			ret);
+
+	return ret;
+}
+
+int snd_sof_runtime_suspend(struct device *dev)
+{
+	return sof_suspend(dev, true);
+}
+EXPORT_SYMBOL(snd_sof_runtime_suspend);
+
+int snd_sof_runtime_resume(struct device *dev)
+{
+	return sof_resume(dev, true);
+}
+EXPORT_SYMBOL(snd_sof_runtime_resume);
+
+int snd_sof_resume(struct device *dev)
+{
+	return sof_resume(dev, false);
+}
+EXPORT_SYMBOL(snd_sof_resume);
+
+int snd_sof_suspend(struct device *dev)
+{
+	return sof_suspend(dev, false);
+}
+EXPORT_SYMBOL(snd_sof_suspend);
diff --git a/sound/soc/sof/sof-acpi-dev.c b/sound/soc/sof/sof-acpi-dev.c
new file mode 100644
index 000000000000..e9cf69874b5b
--- /dev/null
+++ b/sound/soc/sof/sof-acpi-dev.c
@@ -0,0 +1,312 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license.  When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2018 Intel Corporation. All rights reserved.
+//
+// Author: Liam Girdwood <liam.r.girdwood@linux.intel.com>
+//
+
+#include <linux/acpi.h>
+#include <linux/firmware.h>
+#include <linux/module.h>
+#include <linux/pm_runtime.h>
+#include <sound/soc-acpi.h>
+#include <sound/soc-acpi-intel-match.h>
+#include <sound/sof.h>
+#ifdef CONFIG_X86
+#include <asm/iosf_mbi.h>
+#endif
+
+#include "ops.h"
+
+/* platform specific devices */
+#include "intel/shim.h"
+
+static char *fw_path;
+module_param(fw_path, charp, 0444);
+MODULE_PARM_DESC(fw_path, "alternate path for SOF firmware.");
+
+static char *tplg_path;
+module_param(tplg_path, charp, 0444);
+MODULE_PARM_DESC(tplg_path, "alternate path for SOF topology.");
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_HASWELL)
+static const struct sof_dev_desc sof_acpi_haswell_desc = {
+	.machines = snd_soc_acpi_intel_haswell_machines,
+	.resindex_lpe_base = 0,
+	.resindex_pcicfg_base = 1,
+	.resindex_imr_base = -1,
+	.irqindex_host_ipc = 0,
+	.chip_info = &hsw_chip_info,
+	.default_fw_path = "intel/sof",
+	.default_tplg_path = "intel/sof-tplg",
+	.nocodec_fw_filename = "sof-hsw.ri",
+	.nocodec_tplg_filename = "sof-hsw-nocodec.tplg",
+	.ops = &sof_hsw_ops,
+	.arch_ops = &sof_xtensa_arch_ops
+};
+#endif
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_BROADWELL)
+static const struct sof_dev_desc sof_acpi_broadwell_desc = {
+	.machines = snd_soc_acpi_intel_broadwell_machines,
+	.resindex_lpe_base = 0,
+	.resindex_pcicfg_base = 1,
+	.resindex_imr_base = -1,
+	.irqindex_host_ipc = 0,
+	.chip_info = &bdw_chip_info,
+	.default_fw_path = "intel/sof",
+	.default_tplg_path = "intel/sof-tplg",
+	.nocodec_fw_filename = "sof-bdw.ri",
+	.nocodec_tplg_filename = "sof-bdw-nocodec.tplg",
+	.ops = &sof_bdw_ops,
+	.arch_ops = &sof_xtensa_arch_ops
+};
+#endif
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_BAYTRAIL)
+
+/* BYTCR uses different IRQ index */
+static const struct sof_dev_desc sof_acpi_baytrailcr_desc = {
+	.machines = snd_soc_acpi_intel_baytrail_machines,
+	.resindex_lpe_base = 0,
+	.resindex_pcicfg_base = 1,
+	.resindex_imr_base = 2,
+	.irqindex_host_ipc = 0,
+	.chip_info = &byt_chip_info,
+	.default_fw_path = "intel/sof",
+	.default_tplg_path = "intel/sof-tplg",
+	.nocodec_fw_filename = "sof-byt.ri",
+	.nocodec_tplg_filename = "sof-byt-nocodec.tplg",
+	.ops = &sof_byt_ops,
+	.arch_ops = &sof_xtensa_arch_ops
+};
+
+static const struct sof_dev_desc sof_acpi_baytrail_desc = {
+	.machines = snd_soc_acpi_intel_baytrail_machines,
+	.resindex_lpe_base = 0,
+	.resindex_pcicfg_base = 1,
+	.resindex_imr_base = 2,
+	.irqindex_host_ipc = 5,
+	.chip_info = &byt_chip_info,
+	.default_fw_path = "intel/sof",
+	.default_tplg_path = "intel/sof-tplg",
+	.nocodec_fw_filename = "sof-byt.ri",
+	.nocodec_tplg_filename = "sof-byt-nocodec.tplg",
+	.ops = &sof_byt_ops,
+	.arch_ops = &sof_xtensa_arch_ops
+};
+
+#ifdef CONFIG_X86 /* TODO: move this to common helper */
+
+static bool is_byt_cr(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	int status;
+
+	if (iosf_mbi_available()) {
+		u32 bios_status;
+		status = iosf_mbi_read(BT_MBI_UNIT_PMC, /* 0x04 PUNIT */
+				       MBI_REG_READ, /* 0x10 */
+				       0x006, /* BIOS_CONFIG */
+				       &bios_status);
+
+		if (status) {
+			dev_err(dev, "could not read PUNIT BIOS_CONFIG\n");
+		} else {
+			/* bits 26:27 mirror PMIC options */
+			bios_status = (bios_status >> 26) & 3;
+
+			if (bios_status == 1 || bios_status == 3) {
+				dev_info(dev, "Detected Baytrail-CR platform\n");
+				return true;
+			}
+
+			dev_info(dev, "BYT-CR not detected\n");
+		}
+	} else {
+		dev_info(dev, "IOSF_MBI not available, no BYT-CR detection\n");
+	}
+
+	if (platform_get_resource(pdev, IORESOURCE_IRQ, 5) == NULL) {
+		/*
+		 * Some devices detected as BYT-T have only a single IRQ listed,
+		 * causing platform_get_irq with index 5 to return -ENXIO.
+		 * The correct IRQ in this case is at index 0, as on BYT-CR.
+		 */
+		dev_info(dev, "Falling back to Baytrail-CR platform\n");
+		return true;
+	}
+
+	return false;
+}
+#else
+static int is_byt_cr(struct platform_device *pdev)
+{
+	return 0;
+}
+#endif
+
+static const struct sof_dev_desc sof_acpi_cherrytrail_desc = {
+	.machines = snd_soc_acpi_intel_cherrytrail_machines,
+	.resindex_lpe_base = 0,
+	.resindex_pcicfg_base = 1,
+	.resindex_imr_base = 2,
+	.irqindex_host_ipc = 5,
+	.chip_info = &cht_chip_info,
+	.default_fw_path = "intel/sof",
+	.default_tplg_path = "intel/sof-tplg",
+	.nocodec_fw_filename = "sof-cht.ri",
+	.nocodec_tplg_filename = "sof-cht-nocodec.tplg",
+	.ops = &sof_cht_ops,
+	.arch_ops = &sof_xtensa_arch_ops
+};
+
+#endif
+
+static const struct dev_pm_ops sof_acpi_pm = {
+	SET_SYSTEM_SLEEP_PM_OPS(snd_sof_suspend, snd_sof_resume)
+	SET_RUNTIME_PM_OPS(snd_sof_runtime_suspend, snd_sof_runtime_resume,
+			   NULL)
+};
+
+static void sof_acpi_probe_complete(struct device *dev)
+{
+	/* allow runtime_pm */
+	pm_runtime_set_autosuspend_delay(dev, SND_SOF_SUSPEND_DELAY_MS);
+	pm_runtime_use_autosuspend(dev);
+	pm_runtime_enable(dev);
+}
+
+static int sof_acpi_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	const struct sof_dev_desc *desc;
+	struct snd_soc_acpi_mach *mach;
+	struct snd_sof_pdata *sof_pdata;
+	const struct snd_sof_dsp_ops *ops;
+	int ret;
+
+	dev_dbg(&pdev->dev, "ACPI DSP detected");
+
+	sof_pdata = devm_kzalloc(dev, sizeof(*sof_pdata), GFP_KERNEL);
+	if (!sof_pdata)
+		return -ENOMEM;
+
+	desc = device_get_match_data(dev);
+	if (!desc)
+		return -ENODEV;
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_BAYTRAIL)
+	if (desc == &sof_acpi_baytrail_desc && is_byt_cr(pdev))
+		desc = &sof_acpi_baytrailcr_desc;
+#endif
+
+	/* get ops for platform */
+	ops = desc->ops;
+	if (!ops) {
+		dev_err(dev, "error: no matching ACPI descriptor ops\n");
+		return -ENODEV;
+	}
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_FORCE_NOCODEC_MODE)
+	/* force nocodec mode */
+	dev_warn(dev, "Force to use nocodec mode\n");
+	mach = devm_kzalloc(dev, sizeof(*mach), GFP_KERNEL);
+	if (!mach)
+		return -ENOMEM;
+	ret = sof_nocodec_setup(dev, sof_pdata, mach, desc, ops);
+	if (ret < 0)
+		return ret;
+#else
+	/* find machine */
+	mach = snd_soc_acpi_find_machine(desc->machines);
+	if (!mach) {
+		dev_warn(dev, "warning: No matching ASoC machine driver found\n");
+	} else {
+		sof_pdata->fw_filename = mach->sof_fw_filename;
+		sof_pdata->tplg_filename = mach->sof_tplg_filename;
+	}
+#endif
+
+	if (mach) {
+		mach->mach_params.platform = dev_name(dev);
+		mach->mach_params.acpi_ipc_irq_index = desc->irqindex_host_ipc;
+	}
+
+	sof_pdata->machine = mach;
+	sof_pdata->desc = desc;
+	sof_pdata->dev = &pdev->dev;
+	sof_pdata->platform = dev_name(dev);
+
+	/* alternate fw and tplg filenames ? */
+	if (fw_path)
+		sof_pdata->fw_filename_prefix = fw_path;
+	else
+		sof_pdata->fw_filename_prefix =
+			sof_pdata->desc->default_fw_path;
+
+	if (tplg_path)
+		sof_pdata->tplg_filename_prefix = tplg_path;
+	else
+		sof_pdata->tplg_filename_prefix =
+			sof_pdata->desc->default_tplg_path;
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_PROBE_WORK_QUEUE)
+	/* set callback to enable runtime_pm */
+	sof_pdata->sof_probe_complete = sof_acpi_probe_complete;
+#endif
+	/* call sof helper for DSP hardware probe */
+	ret = snd_sof_device_probe(dev, sof_pdata);
+	if (ret) {
+		dev_err(dev, "error: failed to probe DSP hardware!\n");
+		return ret;
+	}
+
+#if !IS_ENABLED(CONFIG_SND_SOC_SOF_PROBE_WORK_QUEUE)
+	sof_acpi_probe_complete(dev);
+#endif
+
+	return ret;
+}
+
+static int sof_acpi_remove(struct platform_device *pdev)
+{
+	pm_runtime_disable(&pdev->dev);
+
+	/* call sof helper for DSP hardware remove */
+	snd_sof_device_remove(&pdev->dev);
+
+	return 0;
+}
+
+static const struct acpi_device_id sof_acpi_match[] = {
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_HASWELL)
+	{ "INT33C8", (unsigned long)&sof_acpi_haswell_desc },
+#endif
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_BROADWELL)
+	{ "INT3438", (unsigned long)&sof_acpi_broadwell_desc },
+#endif
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_BAYTRAIL)
+	{ "80860F28", (unsigned long)&sof_acpi_baytrail_desc },
+	{ "808622A8", (unsigned long)&sof_acpi_cherrytrail_desc },
+#endif
+	{ }
+};
+MODULE_DEVICE_TABLE(acpi, sof_acpi_match);
+
+/* acpi_driver definition */
+static struct platform_driver snd_sof_acpi_driver = {
+	.probe = sof_acpi_probe,
+	.remove = sof_acpi_remove,
+	.driver = {
+		.name = "sof-audio-acpi",
+		.pm = &sof_acpi_pm,
+		.acpi_match_table = ACPI_PTR(sof_acpi_match),
+	},
+};
+module_platform_driver(snd_sof_acpi_driver);
+
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/sound/soc/sof/sof-pci-dev.c b/sound/soc/sof/sof-pci-dev.c
new file mode 100644
index 000000000000..b778dffb2d25
--- /dev/null
+++ b/sound/soc/sof/sof-pci-dev.c
@@ -0,0 +1,373 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license.  When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2018 Intel Corporation. All rights reserved.
+//
+// Author: Liam Girdwood <liam.r.girdwood@linux.intel.com>
+//
+
+#include <linux/firmware.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/pm_runtime.h>
+#include <sound/soc-acpi.h>
+#include <sound/soc-acpi-intel-match.h>
+#include <sound/sof.h>
+#include "ops.h"
+
+/* platform specific devices */
+#include "intel/shim.h"
+#include "intel/hda.h"
+
+static char *fw_path;
+module_param(fw_path, charp, 0444);
+MODULE_PARM_DESC(fw_path, "alternate path for SOF firmware.");
+
+static char *tplg_path;
+module_param(tplg_path, charp, 0444);
+MODULE_PARM_DESC(tplg_path, "alternate path for SOF topology.");
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_APOLLOLAKE)
+static const struct sof_dev_desc bxt_desc = {
+	.machines		= snd_soc_acpi_intel_bxt_machines,
+	.resindex_lpe_base	= 0,
+	.resindex_pcicfg_base	= -1,
+	.resindex_imr_base	= -1,
+	.irqindex_host_ipc	= -1,
+	.resindex_dma_base	= -1,
+	.chip_info = &apl_chip_info,
+	.default_fw_path = "intel/sof",
+	.default_tplg_path = "intel/sof-tplg",
+	.nocodec_fw_filename = "sof-apl.ri",
+	.nocodec_tplg_filename = "sof-apl-nocodec.tplg",
+	.ops = &sof_apl_ops,
+	.arch_ops = &sof_xtensa_arch_ops
+};
+#endif
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_GEMINILAKE)
+static const struct sof_dev_desc glk_desc = {
+	.machines		= snd_soc_acpi_intel_glk_machines,
+	.resindex_lpe_base	= 0,
+	.resindex_pcicfg_base	= -1,
+	.resindex_imr_base	= -1,
+	.irqindex_host_ipc	= -1,
+	.resindex_dma_base	= -1,
+	.chip_info = &apl_chip_info,
+	.default_fw_path = "intel/sof",
+	.default_tplg_path = "intel/sof-tplg",
+	.nocodec_fw_filename = "sof-glk.ri",
+	.nocodec_tplg_filename = "sof-glk-nocodec.tplg",
+	.ops = &sof_apl_ops,
+	.arch_ops = &sof_xtensa_arch_ops
+};
+#endif
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_MERRIFIELD)
+static struct snd_soc_acpi_mach sof_tng_machines[] = {
+	{
+		.id = "INT343A",
+		.drv_name = "edison",
+		.sof_fw_filename = "sof-byt.ri",
+		.sof_tplg_filename = "sof-byt.tplg",
+	},
+	{}
+};
+
+static const struct sof_dev_desc tng_desc = {
+	.machines		= sof_tng_machines,
+	.resindex_lpe_base	= 3,	/* IRAM, but subtract IRAM offset */
+	.resindex_pcicfg_base	= -1,
+	.resindex_imr_base	= 0,
+	.irqindex_host_ipc	= -1,
+	.resindex_dma_base	= -1,
+	.chip_info = &tng_chip_info,
+	.default_fw_path = "intel/sof",
+	.default_tplg_path = "intel/sof-tplg",
+	.nocodec_fw_filename = "sof-byt.ri",
+	.nocodec_tplg_filename = "sof-byt.tplg",
+	.ops = &sof_tng_ops,
+	.arch_ops = &sof_xtensa_arch_ops
+};
+#endif
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_CANNONLAKE)
+static const struct sof_dev_desc cnl_desc = {
+	.machines		= snd_soc_acpi_intel_cnl_machines,
+	.resindex_lpe_base	= 0,
+	.resindex_pcicfg_base	= -1,
+	.resindex_imr_base	= -1,
+	.irqindex_host_ipc	= -1,
+	.resindex_dma_base	= -1,
+	.chip_info = &cnl_chip_info,
+	.default_fw_path = "intel/sof",
+	.default_tplg_path = "intel/sof-tplg",
+	.nocodec_fw_filename = "sof-cnl.ri",
+	.nocodec_tplg_filename = "sof-cnl-nocodec.tplg",
+	.ops = &sof_cnl_ops,
+	.arch_ops = &sof_xtensa_arch_ops
+};
+#endif
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_COFFEELAKE)
+static const struct sof_dev_desc cfl_desc = {
+	.machines		= snd_soc_acpi_intel_cnl_machines,
+	.resindex_lpe_base	= 0,
+	.resindex_pcicfg_base	= -1,
+	.resindex_imr_base	= -1,
+	.irqindex_host_ipc	= -1,
+	.resindex_dma_base	= -1,
+	.chip_info = &cnl_chip_info,
+	.default_fw_path = "intel/sof",
+	.default_tplg_path = "intel/sof-tplg",
+	.nocodec_fw_filename = "sof-cnl.ri",
+	.nocodec_tplg_filename = "sof-cnl-nocodec.tplg",
+	.ops = &sof_cnl_ops,
+	.arch_ops = &sof_xtensa_arch_ops
+};
+#endif
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_ICELAKE)
+static const struct sof_dev_desc icl_desc = {
+	.machines               = snd_soc_acpi_intel_icl_machines,
+	.resindex_lpe_base      = 0,
+	.resindex_pcicfg_base   = -1,
+	.resindex_imr_base      = -1,
+	.irqindex_host_ipc      = -1,
+	.resindex_dma_base      = -1,
+	.chip_info = &cnl_chip_info,
+	.default_fw_path = "intel/sof",
+	.default_tplg_path = "intel/sof-tplg",
+	.nocodec_fw_filename = "sof-icl.ri",
+	.nocodec_tplg_filename = "sof-icl-nocodec.tplg",
+	.ops = &sof_cnl_ops,
+	.arch_ops = &sof_xtensa_arch_ops
+};
+#endif
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_SKYLAKE)
+static const struct sof_dev_desc skl_desc = {
+	.machines		= snd_soc_acpi_intel_skl_machines,
+	.resindex_lpe_base	= 0,
+	.resindex_pcicfg_base	= -1,
+	.resindex_imr_base	= -1,
+	.irqindex_host_ipc	= -1,
+	.resindex_dma_base	= -1,
+	.chip_info = &skl_chip_info,
+	.default_fw_path = "intel/sof",
+	.default_tplg_path = "intel/sof-tplg",
+	.nocodec_fw_filename = "sof-skl.ri",
+	.nocodec_tplg_filename = "sof-skl-nocodec.tplg",
+	.ops = &sof_skl_ops,
+	.arch_ops = &sof_xtensa_arch_ops
+};
+#endif
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_KABYLAKE)
+static const struct sof_dev_desc kbl_desc = {
+	.machines		= snd_soc_acpi_intel_kbl_machines,
+	.resindex_lpe_base	= 0,
+	.resindex_pcicfg_base	= -1,
+	.resindex_imr_base	= -1,
+	.irqindex_host_ipc	= -1,
+	.resindex_dma_base	= -1,
+	.chip_info = &skl_chip_info,
+	.default_fw_path = "intel/sof",
+	.default_tplg_path = "intel/sof-tplg",
+	.nocodec_fw_filename = "sof-kbl.ri",
+	.nocodec_tplg_filename = "sof-kbl-nocodec.tplg",
+	.ops = &sof_skl_ops,
+	.arch_ops = &sof_xtensa_arch_ops
+};
+#endif
+
+static const struct dev_pm_ops sof_pci_pm = {
+	SET_SYSTEM_SLEEP_PM_OPS(snd_sof_suspend, snd_sof_resume)
+	SET_RUNTIME_PM_OPS(snd_sof_runtime_suspend, snd_sof_runtime_resume,
+			   NULL)
+};
+
+static void sof_pci_probe_complete(struct device *dev)
+{
+	dev_dbg(dev, "Completing SOF PCI probe");
+
+	/* allow runtime_pm */
+	pm_runtime_set_autosuspend_delay(dev, SND_SOF_SUSPEND_DELAY_MS);
+	pm_runtime_use_autosuspend(dev);
+
+	/*
+	 * runtime pm for pci device is "forbidden" by default.
+	 * so call pm_runtime_allow() to enable it.
+	 */
+	pm_runtime_allow(dev);
+
+	/* follow recommendation in pci-driver.c to decrement usage counter */
+	pm_runtime_put_noidle(dev);
+}
+
+static int sof_pci_probe(struct pci_dev *pci,
+			 const struct pci_device_id *pci_id)
+{
+	struct device *dev = &pci->dev;
+	const struct sof_dev_desc *desc =
+		(const struct sof_dev_desc *)pci_id->driver_data;
+	struct snd_soc_acpi_mach *mach;
+	struct snd_sof_pdata *sof_pdata;
+	const struct snd_sof_dsp_ops *ops;
+	int ret;
+
+	dev_dbg(&pci->dev, "PCI DSP detected");
+
+	/* get ops for platform */
+	ops = desc->ops;
+	if (!ops) {
+		dev_err(dev, "error: no matching PCI descriptor ops\n");
+		return -ENODEV;
+	}
+
+	sof_pdata = devm_kzalloc(dev, sizeof(*sof_pdata), GFP_KERNEL);
+	if (!sof_pdata)
+		return -ENOMEM;
+
+	ret = pcim_enable_device(pci);
+	if (ret < 0)
+		return ret;
+
+	ret = pci_request_regions(pci, "Audio DSP");
+	if (ret < 0)
+		return ret;
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_FORCE_NOCODEC_MODE)
+	/* force nocodec mode */
+	dev_warn(dev, "Force to use nocodec mode\n");
+	mach = devm_kzalloc(dev, sizeof(*mach), GFP_KERNEL);
+	if (!mach) {
+		ret = -ENOMEM;
+		goto release_regions;
+	}
+	ret = sof_nocodec_setup(dev, sof_pdata, mach, desc, ops);
+	if (ret < 0)
+		goto release_regions;
+
+#else
+	/* find machine */
+	mach = snd_soc_acpi_find_machine(desc->machines);
+	if (!mach) {
+		dev_warn(dev, "warning: No matching ASoC machine driver found\n");
+	} else {
+		mach->mach_params.platform = dev_name(dev);
+		sof_pdata->fw_filename = mach->sof_fw_filename;
+		sof_pdata->tplg_filename = mach->sof_tplg_filename;
+	}
+#endif /* CONFIG_SND_SOC_SOF_FORCE_NOCODEC_MODE */
+
+	sof_pdata->name = pci_name(pci);
+	sof_pdata->machine = mach;
+	sof_pdata->desc = (struct sof_dev_desc *)pci_id->driver_data;
+	sof_pdata->dev = dev;
+	sof_pdata->platform = dev_name(dev);
+
+	/* alternate fw and tplg filenames ? */
+	if (fw_path)
+		sof_pdata->fw_filename_prefix = fw_path;
+	else
+		sof_pdata->fw_filename_prefix =
+			sof_pdata->desc->default_fw_path;
+
+	if (tplg_path)
+		sof_pdata->tplg_filename_prefix = tplg_path;
+	else
+		sof_pdata->tplg_filename_prefix =
+			sof_pdata->desc->default_tplg_path;
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_PROBE_WORK_QUEUE)
+	/* set callback to enable runtime_pm */
+	sof_pdata->sof_probe_complete = sof_pci_probe_complete;
+#endif
+	/* call sof helper for DSP hardware probe */
+	ret = snd_sof_device_probe(dev, sof_pdata);
+	if (ret) {
+		dev_err(dev, "error: failed to probe DSP hardware!\n");
+		goto release_regions;
+	}
+
+#if !IS_ENABLED(CONFIG_SND_SOC_SOF_PROBE_WORK_QUEUE)
+	sof_pci_probe_complete(dev);
+#endif
+
+	return ret;
+
+release_regions:
+	pci_release_regions(pci);
+
+	return ret;
+}
+
+static void sof_pci_remove(struct pci_dev *pci)
+{
+	/* call sof helper for DSP hardware remove */
+	snd_sof_device_remove(&pci->dev);
+
+	/* follow recommendation in pci-driver.c to increment usage counter */
+	pm_runtime_get_noresume(&pci->dev);
+
+	/* release pci regions and disable device */
+	pci_release_regions(pci);
+}
+
+/* PCI IDs */
+static const struct pci_device_id sof_pci_ids[] = {
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_MERRIFIELD)
+	{ PCI_DEVICE(0x8086, 0x119a),
+		.driver_data = (unsigned long)&tng_desc},
+#endif
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_APOLLOLAKE)
+	/* BXT-P & Apollolake */
+	{ PCI_DEVICE(0x8086, 0x5a98),
+		.driver_data = (unsigned long)&bxt_desc},
+	{ PCI_DEVICE(0x8086, 0x1a98),
+		.driver_data = (unsigned long)&bxt_desc},
+#endif
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_GEMINILAKE)
+	{ PCI_DEVICE(0x8086, 0x3198),
+		.driver_data = (unsigned long)&glk_desc},
+#endif
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_CANNONLAKE)
+	{ PCI_DEVICE(0x8086, 0x9dc8),
+		.driver_data = (unsigned long)&cnl_desc},
+#endif
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_COFFEELAKE)
+	{ PCI_DEVICE(0x8086, 0xa348),
+		.driver_data = (unsigned long)&cfl_desc},
+#endif
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_KABYLAKE)
+	{ PCI_DEVICE(0x8086, 0x9d71),
+		.driver_data = (unsigned long)&kbl_desc},
+#endif
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_SKYLAKE)
+	{ PCI_DEVICE(0x8086, 0x9d70),
+		.driver_data = (unsigned long)&skl_desc},
+#endif
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_ICELAKE)
+	{ PCI_DEVICE(0x8086, 0x34C8),
+		.driver_data = (unsigned long)&icl_desc},
+#endif
+	{ 0, }
+};
+MODULE_DEVICE_TABLE(pci, sof_pci_ids);
+
+/* pci_driver definition */
+static struct pci_driver snd_sof_pci_driver = {
+	.name = "sof-audio-pci",
+	.id_table = sof_pci_ids,
+	.probe = sof_pci_probe,
+	.remove = sof_pci_remove,
+	.driver = {
+		.pm = &sof_pci_pm,
+	},
+};
+module_pci_driver(snd_sof_pci_driver);
+
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/sound/soc/sof/sof-priv.h b/sound/soc/sof/sof-priv.h
new file mode 100644
index 000000000000..1e85d6f9c5c3
--- /dev/null
+++ b/sound/soc/sof/sof-priv.h
@@ -0,0 +1,635 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */
+/*
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * Copyright(c) 2018 Intel Corporation. All rights reserved.
+ *
+ * Author: Liam Girdwood <liam.r.girdwood@linux.intel.com>
+ */
+
+#ifndef __SOUND_SOC_SOF_PRIV_H
+#define __SOUND_SOC_SOF_PRIV_H
+
+#include <linux/device.h>
+
+#include <sound/hdaudio.h>
+#include <sound/soc.h>
+
+#include <sound/sof.h>
+#include <sound/sof/stream.h> /* needs to be included before control.h */
+#include <sound/sof/control.h>
+#include <sound/sof/dai.h>
+#include <sound/sof/info.h>
+#include <sound/sof/pm.h>
+#include <sound/sof/topology.h>
+#include <sound/sof/trace.h>
+
+#include <uapi/sound/sof/fw.h>
+
+/* debug flags */
+#define SOF_DBG_REGS	BIT(1)
+#define SOF_DBG_MBOX	BIT(2)
+#define SOF_DBG_TEXT	BIT(3)
+#define SOF_DBG_PCI	BIT(4)
+
+/* max BARs mmaped devices can use */
+#define SND_SOF_BARS	8
+
+/* time in ms for runtime suspend delay */
+#define SND_SOF_SUSPEND_DELAY_MS	2000
+
+/* DMA buffer size for trace */
+#define DMA_BUF_SIZE_FOR_TRACE (PAGE_SIZE * 16)
+
+/* max number of FE PCMs before BEs */
+#define SOF_BE_PCM_BASE		16
+
+#define SOF_IPC_DSP_REPLY		0
+#define SOF_IPC_HOST_REPLY		1
+
+/* convenience constructor for DAI driver streams */
+#define SOF_DAI_STREAM(sname, scmin, scmax, srates, sfmt) \
+	{.stream_name = sname, .channels_min = scmin, .channels_max = scmax, \
+	 .rates = srates, .formats = sfmt}
+
+#define SOF_FORMATS (SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S24_LE | \
+	SNDRV_PCM_FMTBIT_S32_LE | SNDRV_PCM_FMTBIT_FLOAT)
+
+struct snd_sof_dev;
+struct snd_sof_ipc_msg;
+struct snd_sof_ipc;
+struct snd_sof_debugfs_map;
+struct snd_soc_tplg_ops;
+struct snd_soc_component;
+struct snd_sof_pdata;
+
+/*
+ * SOF DSP HW abstraction operations.
+ * Used to abstract DSP HW architecture and any IO busses between host CPU
+ * and DSP device(s).
+ */
+struct snd_sof_dsp_ops {
+
+	/* probe and remove */
+	int (*probe)(struct snd_sof_dev *sof_dev); /* mandatory */
+	int (*remove)(struct snd_sof_dev *sof_dev); /* optional */
+
+	/* DSP core boot / reset */
+	int (*run)(struct snd_sof_dev *sof_dev); /* mandatory */
+	int (*stall)(struct snd_sof_dev *sof_dev); /* optional */
+	int (*reset)(struct snd_sof_dev *sof_dev); /* optional */
+	int (*core_power_up)(struct snd_sof_dev *sof_dev,
+			     unsigned int core_mask); /* optional */
+	int (*core_power_down)(struct snd_sof_dev *sof_dev,
+			       unsigned int core_mask); /* optional */
+
+	/*
+	 * Register IO: only used by respective drivers themselves,
+	 * TODO: consider removing these operations and calling respective
+	 * implementations directly
+	 */
+	void (*write)(struct snd_sof_dev *sof_dev, void __iomem *addr,
+		      u32 value); /* optional */
+	u32 (*read)(struct snd_sof_dev *sof_dev,
+		    void __iomem *addr); /* optional */
+	void (*write64)(struct snd_sof_dev *sof_dev, void __iomem *addr,
+			u64 value); /* optional */
+	u64 (*read64)(struct snd_sof_dev *sof_dev,
+		      void __iomem *addr); /* optional */
+
+	/* memcpy IO */
+	void (*block_read)(struct snd_sof_dev *sof_dev, u32 bar,
+			   u32 offset, void *dest,
+			   size_t size); /* mandatory */
+	void (*block_write)(struct snd_sof_dev *sof_dev, u32 bar,
+			    u32 offset, void *src,
+			    size_t size); /* mandatory */
+
+	/* doorbell */
+	irqreturn_t (*irq_handler)(int irq, void *context); /* optional */
+	irqreturn_t (*irq_thread)(int irq, void *context); /* optional */
+
+	/* ipc */
+	int (*send_msg)(struct snd_sof_dev *sof_dev,
+			struct snd_sof_ipc_msg *msg); /* mandatory */
+
+	/* FW loading */
+	int (*load_firmware)(struct snd_sof_dev *sof_dev); /* mandatory */
+	int (*load_module)(struct snd_sof_dev *sof_dev,
+			   struct snd_sof_mod_hdr *hdr); /* optional */
+	/*
+	 * FW ready checks for ABI compatibility and creates
+	 * memory windows at first boot
+	 */
+	int (*fw_ready)(struct snd_sof_dev *sdev, u32 msg_id); /* optional */
+
+	/* connect pcm substream to a host stream */
+	int (*pcm_open)(struct snd_sof_dev *sdev,
+			struct snd_pcm_substream *substream); /* optional */
+	/* disconnect pcm substream to a host stream */
+	int (*pcm_close)(struct snd_sof_dev *sdev,
+			 struct snd_pcm_substream *substream); /* optional */
+
+	/* host stream hw params */
+	int (*pcm_hw_params)(struct snd_sof_dev *sdev,
+			     struct snd_pcm_substream *substream,
+			     struct snd_pcm_hw_params *params,
+			     struct sof_ipc_stream_params *ipc_params); /* optional */
+
+	/* host stream trigger */
+	int (*pcm_trigger)(struct snd_sof_dev *sdev,
+			   struct snd_pcm_substream *substream,
+			   int cmd); /* optional */
+
+	/* host stream pointer */
+	snd_pcm_uframes_t (*pcm_pointer)(struct snd_sof_dev *sdev,
+					 struct snd_pcm_substream *substream); /* optional */
+
+	/* host read DSP stream data */
+	void (*ipc_msg_data)(struct snd_sof_dev *sdev,
+			     struct snd_pcm_substream *substream,
+			     void *p, size_t sz); /* mandatory */
+
+	/* host configure DSP HW parameters */
+	int (*ipc_pcm_params)(struct snd_sof_dev *sdev,
+			      struct snd_pcm_substream *substream,
+			      const struct sof_ipc_pcm_params_reply *reply); /* mandatory */
+
+	/* pre/post firmware run */
+	int (*pre_fw_run)(struct snd_sof_dev *sof_dev); /* optional */
+	int (*post_fw_run)(struct snd_sof_dev *sof_dev); /* optional */
+
+	/* DSP PM */
+	int (*suspend)(struct snd_sof_dev *sof_dev, int state); /* optional */
+	int (*resume)(struct snd_sof_dev *sof_dev); /* optional */
+	int (*runtime_suspend)(struct snd_sof_dev *sof_dev,
+			       int state); /* optional */
+	int (*runtime_resume)(struct snd_sof_dev *sof_dev); /* optional */
+	void (*set_hw_params_upon_resume)(struct snd_sof_dev *sdev); /* optional */
+
+	/* DSP clocking */
+	int (*set_clk)(struct snd_sof_dev *sof_dev, u32 freq); /* optional */
+
+	/* debug */
+	const struct snd_sof_debugfs_map *debug_map; /* optional */
+	int debug_map_count; /* optional */
+	void (*dbg_dump)(struct snd_sof_dev *sof_dev,
+			 u32 flags); /* optional */
+	void (*ipc_dump)(struct snd_sof_dev *sof_dev); /* optional */
+
+	/* host DMA trace initialization */
+	int (*trace_init)(struct snd_sof_dev *sdev,
+			  u32 *stream_tag); /* optional */
+	int (*trace_release)(struct snd_sof_dev *sdev); /* optional */
+	int (*trace_trigger)(struct snd_sof_dev *sdev,
+			     int cmd); /* optional */
+
+	/* DAI ops */
+	struct snd_soc_dai_driver *drv;
+	int num_drv;
+};
+
+/* DSP architecture specific callbacks for oops and stack dumps */
+struct sof_arch_ops {
+	void (*dsp_oops)(struct snd_sof_dev *sdev, void *oops);
+	void (*dsp_stack)(struct snd_sof_dev *sdev, void *oops,
+			  u32 *stack, u32 stack_words);
+};
+
+#define sof_arch_ops(sdev) ((sdev)->pdata->desc->arch_ops)
+
+/* DSP device HW descriptor mapping between bus ID and ops */
+struct sof_ops_table {
+	const struct sof_dev_desc *desc;
+	const struct snd_sof_dsp_ops *ops;
+};
+
+enum sof_dfsentry_type {
+	SOF_DFSENTRY_TYPE_IOMEM = 0,
+	SOF_DFSENTRY_TYPE_BUF,
+};
+
+enum sof_debugfs_access_type {
+	SOF_DEBUGFS_ACCESS_ALWAYS = 0,
+	SOF_DEBUGFS_ACCESS_D0_ONLY,
+};
+
+/* FS entry for debug files that can expose DSP memories, registers */
+struct snd_sof_dfsentry {
+	struct dentry *dfsentry;
+	size_t size;
+	enum sof_dfsentry_type type;
+	/*
+	 * access_type specifies if the
+	 * memory -> DSP resource (memory, register etc) is always accessible
+	 * or if it is accessible only when the DSP is in D0.
+	 */
+	enum sof_debugfs_access_type access_type;
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG_ENABLE_DEBUGFS_CACHE)
+	char *cache_buf; /* buffer to cache the contents of debugfs memory */
+#endif
+	struct snd_sof_dev *sdev;
+	struct list_head list;  /* list in sdev dfsentry list */
+	union {
+		void __iomem *io_mem;
+		void *buf;
+	};
+};
+
+/* Debug mapping for any DSP memory or registers that can used for debug */
+struct snd_sof_debugfs_map {
+	const char *name;
+	u32 bar;
+	u32 offset;
+	u32 size;
+	/*
+	 * access_type specifies if the memory is always accessible
+	 * or if it is accessible only when the DSP is in D0.
+	 */
+	enum sof_debugfs_access_type access_type;
+};
+
+/* mailbox descriptor, used for host <-> DSP IPC */
+struct snd_sof_mailbox {
+	u32 offset;
+	size_t size;
+};
+
+/* IPC message descriptor for host <-> DSP IO */
+struct snd_sof_ipc_msg {
+	/* message data */
+	u32 header;
+	void *msg_data;
+	void *reply_data;
+	size_t msg_size;
+	size_t reply_size;
+	int reply_error;
+
+	wait_queue_head_t waitq;
+	bool ipc_complete;
+};
+
+/* PCM stream, mapped to FW component  */
+struct snd_sof_pcm_stream {
+	u32 comp_id;
+	struct snd_dma_buffer page_table;
+	struct sof_ipc_stream_posn posn;
+	struct snd_pcm_substream *substream;
+	struct work_struct period_elapsed_work;
+};
+
+/* ALSA SOF PCM device */
+struct snd_sof_pcm {
+	struct snd_sof_dev *sdev;
+	struct snd_soc_tplg_pcm pcm;
+	struct snd_sof_pcm_stream stream[2];
+	struct list_head list;	/* list in sdev pcm list */
+	struct snd_pcm_hw_params params[2];
+	int hw_params_upon_resume[2]; /* set up hw_params upon resume */
+};
+
+/* ALSA SOF Kcontrol device */
+struct snd_sof_control {
+	struct snd_sof_dev *sdev;
+	int comp_id;
+	int num_channels;
+	u32 readback_offset; /* offset to mmaped data if used */
+	struct sof_ipc_ctrl_data *control_data;
+	u32 size;	/* cdata size */
+	enum sof_ipc_ctrl_cmd cmd;
+	u32 *volume_table; /* volume table computed from tlv data*/
+
+	struct list_head list;	/* list in sdev control list */
+};
+
+/* ASoC SOF DAPM widget */
+struct snd_sof_widget {
+	struct snd_sof_dev *sdev;
+	int comp_id;
+	int pipeline_id;
+	int complete;
+	int id;
+
+	struct snd_soc_dapm_widget *widget;
+	struct list_head list;	/* list in sdev widget list */
+
+	void *private;		/* core does not touch this */
+};
+
+/* ASoC SOF DAPM route */
+struct snd_sof_route {
+	struct snd_sof_dev *sdev;
+
+	struct snd_soc_dapm_route *route;
+	struct list_head list;	/* list in sdev route list */
+
+	void *private;
+};
+
+/* ASoC DAI device */
+struct snd_sof_dai {
+	struct snd_sof_dev *sdev;
+	const char *name;
+
+	struct sof_ipc_comp_dai comp_dai;
+	struct sof_ipc_dai_config *dai_config;
+	struct list_head list;	/* list in sdev dai list */
+};
+
+/*
+ * SOF Device Level.
+ */
+struct snd_sof_dev {
+	struct device *dev;
+	spinlock_t ipc_lock;	/* lock for IPC users */
+	spinlock_t hw_lock;	/* lock for HW IO access */
+
+	/*
+	 * ASoC components. plat_drv fields are set dynamically so
+	 * can't use const
+	 */
+	struct snd_soc_component_driver plat_drv;
+
+	/* DSP firmware boot */
+	wait_queue_head_t boot_wait;
+	u32 boot_complete;
+	u32 first_boot;
+
+	/* work queue in case the probe is implemented in two steps */
+	struct work_struct probe_work;
+
+	/* DSP HW differentiation */
+	struct snd_sof_pdata *pdata;
+
+	/* IPC */
+	struct snd_sof_ipc *ipc;
+	struct snd_sof_mailbox dsp_box;		/* DSP initiated IPC */
+	struct snd_sof_mailbox host_box;	/* Host initiated IPC */
+	struct snd_sof_mailbox stream_box;	/* Stream position update */
+	struct snd_sof_ipc_msg *msg;
+	int ipc_irq;
+	u32 next_comp_id; /* monotonic - reset during S3 */
+
+	/* memory bases for mmaped DSPs - set by dsp_init() */
+	void __iomem *bar[SND_SOF_BARS];	/* DSP base address */
+	int mmio_bar;
+	int mailbox_bar;
+	size_t dsp_oops_offset;
+
+	/* debug */
+	struct dentry *debugfs_root;
+	struct list_head dfsentry_list;
+
+	/* firmware loader */
+	struct snd_dma_buffer dmab;
+	struct snd_dma_buffer dmab_bdl;
+	struct sof_ipc_fw_ready fw_ready;
+	struct sof_ipc_fw_version fw_version;
+
+	/* topology */
+	struct snd_soc_tplg_ops *tplg_ops;
+	struct list_head pcm_list;
+	struct list_head kcontrol_list;
+	struct list_head widget_list;
+	struct list_head dai_list;
+	struct list_head route_list;
+	struct snd_soc_component *component;
+	u32 enabled_cores_mask; /* keep track of enabled cores */
+
+	/* FW configuration */
+	struct sof_ipc_dma_buffer_data *info_buffer;
+	struct sof_ipc_window *info_window;
+
+	/* IPC timeouts in ms */
+	int ipc_timeout;
+	int boot_timeout;
+
+	/* Wait queue for code loading */
+	wait_queue_head_t waitq;
+	int code_loading;
+
+	/* DMA for Trace */
+	struct snd_dma_buffer dmatb;
+	struct snd_dma_buffer dmatp;
+	int dma_trace_pages;
+	wait_queue_head_t trace_sleep;
+	u32 host_offset;
+	u32 dtrace_is_enabled;
+	u32 dtrace_error;
+	u32 msi_enabled;
+
+	void *private;			/* core does not touch this */
+};
+
+/*
+ * Device Level.
+ */
+
+int snd_sof_device_probe(struct device *dev, struct snd_sof_pdata *plat_data);
+int snd_sof_device_remove(struct device *dev);
+
+int snd_sof_runtime_suspend(struct device *dev);
+int snd_sof_runtime_resume(struct device *dev);
+int snd_sof_resume(struct device *dev);
+int snd_sof_suspend(struct device *dev);
+
+void snd_sof_new_platform_drv(struct snd_sof_dev *sdev);
+
+int snd_sof_create_page_table(struct snd_sof_dev *sdev,
+			      struct snd_dma_buffer *dmab,
+			      unsigned char *page_table, size_t size);
+
+/*
+ * Firmware loading.
+ */
+int snd_sof_load_firmware(struct snd_sof_dev *sdev);
+int snd_sof_load_firmware_raw(struct snd_sof_dev *sdev);
+int snd_sof_load_firmware_memcpy(struct snd_sof_dev *sdev);
+int snd_sof_run_firmware(struct snd_sof_dev *sdev);
+int snd_sof_parse_module_memcpy(struct snd_sof_dev *sdev,
+				struct snd_sof_mod_hdr *module);
+void snd_sof_fw_unload(struct snd_sof_dev *sdev);
+int snd_sof_fw_parse_ext_data(struct snd_sof_dev *sdev, u32 bar, u32 offset);
+
+/*
+ * IPC low level APIs.
+ */
+struct snd_sof_ipc *snd_sof_ipc_init(struct snd_sof_dev *sdev);
+void snd_sof_ipc_free(struct snd_sof_dev *sdev);
+int snd_sof_ipc_reply(struct snd_sof_dev *sdev, u32 msg_id);
+void snd_sof_ipc_msgs_rx(struct snd_sof_dev *sdev);
+int snd_sof_ipc_stream_pcm_params(struct snd_sof_dev *sdev,
+				  struct sof_ipc_pcm_params *params);
+int snd_sof_dsp_mailbox_init(struct snd_sof_dev *sdev, u32 dspbox,
+			     size_t dspbox_size, u32 hostbox,
+			     size_t hostbox_size);
+int snd_sof_ipc_valid(struct snd_sof_dev *sdev);
+int sof_ipc_tx_message(struct snd_sof_ipc *ipc, u32 header,
+		       void *msg_data, size_t msg_bytes, void *reply_data,
+		       size_t reply_bytes);
+struct snd_sof_widget *snd_sof_find_swidget(struct snd_sof_dev *sdev,
+					    const char *name);
+struct snd_sof_widget *snd_sof_find_swidget_sname(struct snd_sof_dev *sdev,
+						  const char *pcm_name,
+						  int dir);
+struct snd_sof_dai *snd_sof_find_dai(struct snd_sof_dev *sdev,
+				     const char *name);
+
+static inline
+struct snd_sof_pcm *snd_sof_find_spcm_dai(struct snd_sof_dev *sdev,
+					  struct snd_soc_pcm_runtime *rtd)
+{
+	struct snd_sof_pcm *spcm = NULL;
+
+	list_for_each_entry(spcm, &sdev->pcm_list, list) {
+		if (le32_to_cpu(spcm->pcm.dai_id) == rtd->dai_link->id)
+			return spcm;
+	}
+
+	return NULL;
+}
+
+struct snd_sof_pcm *snd_sof_find_spcm_name(struct snd_sof_dev *sdev,
+					   const char *name);
+struct snd_sof_pcm *snd_sof_find_spcm_comp(struct snd_sof_dev *sdev,
+					   unsigned int comp_id,
+					   int *direction);
+struct snd_sof_pcm *snd_sof_find_spcm_pcm_id(struct snd_sof_dev *sdev,
+					     unsigned int pcm_id);
+void snd_sof_pcm_period_elapsed(struct snd_pcm_substream *substream);
+
+/*
+ * Stream IPC
+ */
+int snd_sof_ipc_stream_posn(struct snd_sof_dev *sdev,
+			    struct snd_sof_pcm *spcm, int direction,
+			    struct sof_ipc_stream_posn *posn);
+
+/*
+ * Mixer IPC
+ */
+int snd_sof_ipc_set_get_comp_data(struct snd_sof_ipc *ipc,
+				  struct snd_sof_control *scontrol, u32 ipc_cmd,
+				  enum sof_ipc_ctrl_type ctrl_type,
+				  enum sof_ipc_ctrl_cmd ctrl_cmd,
+				  bool send);
+
+/*
+ * Topology.
+ * There is no snd_sof_free_topology since topology components will
+ * be freed by snd_soc_unregister_component,
+ */
+int snd_sof_init_topology(struct snd_sof_dev *sdev,
+			  struct snd_soc_tplg_ops *ops);
+int snd_sof_load_topology(struct snd_sof_dev *sdev, const char *file);
+int snd_sof_complete_pipeline(struct snd_sof_dev *sdev,
+			      struct snd_sof_widget *swidget);
+
+int sof_load_pipeline_ipc(struct snd_sof_dev *sdev,
+			  struct sof_ipc_pipe_new *pipeline,
+			  struct sof_ipc_comp_reply *r);
+
+/*
+ * Trace/debug
+ */
+int snd_sof_init_trace(struct snd_sof_dev *sdev);
+void snd_sof_release_trace(struct snd_sof_dev *sdev);
+void snd_sof_free_trace(struct snd_sof_dev *sdev);
+int snd_sof_dbg_init(struct snd_sof_dev *sdev);
+void snd_sof_free_debug(struct snd_sof_dev *sdev);
+int snd_sof_debugfs_io_item(struct snd_sof_dev *sdev,
+			    void __iomem *base, size_t size,
+			    const char *name,
+			    enum sof_debugfs_access_type access_type);
+int snd_sof_debugfs_buf_item(struct snd_sof_dev *sdev,
+			     void *base, size_t size,
+			     const char *name);
+int snd_sof_trace_update_pos(struct snd_sof_dev *sdev,
+			     struct sof_ipc_dma_trace_posn *posn);
+void snd_sof_trace_notify_for_error(struct snd_sof_dev *sdev);
+void snd_sof_get_status(struct snd_sof_dev *sdev, u32 panic_code,
+			u32 tracep_code, void *oops,
+			struct sof_ipc_panic_info *panic_info,
+			void *stack, size_t stack_words);
+int snd_sof_init_trace_ipc(struct snd_sof_dev *sdev);
+
+/*
+ * Platform specific ops.
+ */
+extern struct snd_compr_ops sof_compressed_ops;
+
+/*
+ * Kcontrols.
+ */
+
+int snd_sof_volume_get(struct snd_kcontrol *kcontrol,
+		       struct snd_ctl_elem_value *ucontrol);
+int snd_sof_volume_put(struct snd_kcontrol *kcontrol,
+		       struct snd_ctl_elem_value *ucontrol);
+int snd_sof_switch_get(struct snd_kcontrol *kcontrol,
+		       struct snd_ctl_elem_value *ucontrol);
+int snd_sof_switch_put(struct snd_kcontrol *kcontrol,
+		       struct snd_ctl_elem_value *ucontrol);
+int snd_sof_enum_get(struct snd_kcontrol *kcontrol,
+		     struct snd_ctl_elem_value *ucontrol);
+int snd_sof_enum_put(struct snd_kcontrol *kcontrol,
+		     struct snd_ctl_elem_value *ucontrol);
+int snd_sof_bytes_get(struct snd_kcontrol *kcontrol,
+		      struct snd_ctl_elem_value *ucontrol);
+int snd_sof_bytes_put(struct snd_kcontrol *kcontrol,
+		      struct snd_ctl_elem_value *ucontrol);
+int snd_sof_bytes_ext_put(struct snd_kcontrol *kcontrol,
+			  const unsigned int __user *binary_data,
+			  unsigned int size);
+int snd_sof_bytes_ext_get(struct snd_kcontrol *kcontrol,
+			  unsigned int __user *binary_data,
+			  unsigned int size);
+
+/*
+ * DSP Architectures.
+ */
+static inline void sof_stack(struct snd_sof_dev *sdev, void *oops, u32 *stack,
+			     u32 stack_words)
+{
+	if (sof_arch_ops(sdev)->dsp_stack)
+		sof_arch_ops(sdev)->dsp_stack(sdev, oops, stack, stack_words);
+}
+
+static inline void sof_oops(struct snd_sof_dev *sdev, void *oops)
+{
+	if (sof_arch_ops(sdev)->dsp_oops)
+		sof_arch_ops(sdev)->dsp_oops(sdev, oops);
+}
+
+extern const struct sof_arch_ops sof_xtensa_arch_ops;
+
+/*
+ * Utilities
+ */
+void sof_io_write(struct snd_sof_dev *sdev, void __iomem *addr, u32 value);
+void sof_io_write64(struct snd_sof_dev *sdev, void __iomem *addr, u64 value);
+u32 sof_io_read(struct snd_sof_dev *sdev, void __iomem *addr);
+u64 sof_io_read64(struct snd_sof_dev *sdev, void __iomem *addr);
+void sof_mailbox_write(struct snd_sof_dev *sdev, u32 offset,
+		       void *message, size_t bytes);
+void sof_mailbox_read(struct snd_sof_dev *sdev, u32 offset,
+		      void *message, size_t bytes);
+void sof_block_write(struct snd_sof_dev *sdev, u32 bar, u32 offset, void *src,
+		     size_t size);
+void sof_block_read(struct snd_sof_dev *sdev, u32 bar, u32 offset, void *dest,
+		    size_t size);
+
+void intel_ipc_msg_data(struct snd_sof_dev *sdev,
+			struct snd_pcm_substream *substream,
+			void *p, size_t sz);
+int intel_ipc_pcm_params(struct snd_sof_dev *sdev,
+			 struct snd_pcm_substream *substream,
+			 const struct sof_ipc_pcm_params_reply *reply);
+
+int intel_pcm_open(struct snd_sof_dev *sdev,
+		   struct snd_pcm_substream *substream);
+int intel_pcm_close(struct snd_sof_dev *sdev,
+		    struct snd_pcm_substream *substream);
+
+#endif
diff --git a/sound/soc/sof/topology.c b/sound/soc/sof/topology.c
new file mode 100644
index 000000000000..c88afa872a58
--- /dev/null
+++ b/sound/soc/sof/topology.c
@@ -0,0 +1,3179 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license.  When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2018 Intel Corporation. All rights reserved.
+//
+// Author: Liam Girdwood <liam.r.girdwood@linux.intel.com>
+//
+
+#include <linux/firmware.h>
+#include <sound/tlv.h>
+#include <sound/pcm_params.h>
+#include <uapi/sound/sof/tokens.h>
+#include "sof-priv.h"
+#include "ops.h"
+
+#define COMP_ID_UNASSIGNED		0xffffffff
+/*
+ * Constants used in the computation of linear volume gain
+ * from dB gain 20th root of 10 in Q1.16 fixed-point notation
+ */
+#define VOL_TWENTIETH_ROOT_OF_TEN	73533
+/* 40th root of 10 in Q1.16 fixed-point notation*/
+#define VOL_FORTIETH_ROOT_OF_TEN	69419
+/*
+ * Volume fractional word length define to 16 sets
+ * the volume linear gain value to use Qx.16 format
+ */
+#define VOLUME_FWL	16
+/* 0.5 dB step value in topology TLV */
+#define VOL_HALF_DB_STEP	50
+/* Full volume for default values */
+#define VOL_ZERO_DB	BIT(VOLUME_FWL)
+
+/* TLV data items */
+#define TLV_ITEMS	3
+#define TLV_MIN		0
+#define TLV_STEP	1
+#define TLV_MUTE	2
+
+/* size of tplg abi in byte */
+#define SOF_TPLG_ABI_SIZE 3
+
+/* send pcm params ipc */
+static int ipc_pcm_params(struct snd_sof_widget *swidget, int dir)
+{
+	struct sof_ipc_pcm_params_reply ipc_params_reply;
+	struct snd_sof_dev *sdev = swidget->sdev;
+	struct sof_ipc_pcm_params pcm;
+	struct snd_pcm_hw_params *params;
+	struct snd_sof_pcm *spcm;
+	int ret = 0;
+
+	memset(&pcm, 0, sizeof(pcm));
+
+	/* get runtime PCM params using widget's stream name */
+	spcm = snd_sof_find_spcm_name(sdev, swidget->widget->sname);
+	if (!spcm) {
+		dev_err(sdev->dev, "error: cannot find PCM for %s\n",
+			swidget->widget->name);
+		return -EINVAL;
+	}
+
+	params = &spcm->params[dir];
+
+	/* set IPC PCM params */
+	pcm.hdr.size = sizeof(pcm);
+	pcm.hdr.cmd = SOF_IPC_GLB_STREAM_MSG | SOF_IPC_STREAM_PCM_PARAMS;
+	pcm.comp_id = swidget->comp_id;
+	pcm.params.hdr.size = sizeof(pcm.params);
+	pcm.params.direction = dir;
+	pcm.params.sample_valid_bytes = params_width(params) >> 3;
+	pcm.params.buffer_fmt = SOF_IPC_BUFFER_INTERLEAVED;
+	pcm.params.rate = params_rate(params);
+	pcm.params.channels = params_channels(params);
+	pcm.params.host_period_bytes = params_period_bytes(params);
+
+	/* set format */
+	switch (params_format(params)) {
+	case SNDRV_PCM_FORMAT_S16:
+		pcm.params.frame_fmt = SOF_IPC_FRAME_S16_LE;
+		break;
+	case SNDRV_PCM_FORMAT_S24:
+		pcm.params.frame_fmt = SOF_IPC_FRAME_S24_4LE;
+		break;
+	case SNDRV_PCM_FORMAT_S32:
+		pcm.params.frame_fmt = SOF_IPC_FRAME_S32_LE;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	/* send IPC to the DSP */
+	ret = sof_ipc_tx_message(sdev->ipc, pcm.hdr.cmd, &pcm, sizeof(pcm),
+				 &ipc_params_reply, sizeof(ipc_params_reply));
+	if (ret < 0)
+		dev_err(sdev->dev, "error: pcm params failed for %s\n",
+			swidget->widget->name);
+
+	return ret;
+}
+
+ /* send stream trigger ipc */
+static int ipc_trigger(struct snd_sof_widget *swidget, int cmd)
+{
+	struct snd_sof_dev *sdev = swidget->sdev;
+	struct sof_ipc_stream stream;
+	struct sof_ipc_reply reply;
+	int ret = 0;
+
+	/* set IPC stream params */
+	stream.hdr.size = sizeof(stream);
+	stream.hdr.cmd = SOF_IPC_GLB_STREAM_MSG | cmd;
+	stream.comp_id = swidget->comp_id;
+
+	/* send IPC to the DSP */
+	ret = sof_ipc_tx_message(sdev->ipc, stream.hdr.cmd, &stream,
+				 sizeof(stream), &reply, sizeof(reply));
+	if (ret < 0)
+		dev_err(sdev->dev, "error: failed to trigger %s\n",
+			swidget->widget->name);
+
+	return ret;
+}
+
+static int sof_keyword_dapm_event(struct snd_soc_dapm_widget *w,
+				  struct snd_kcontrol *k, int event)
+{
+	struct snd_sof_widget *swidget = w->dobj.private;
+	struct snd_sof_dev *sdev;
+	int ret = 0;
+
+	if (!swidget)
+		return 0;
+
+	sdev = swidget->sdev;
+
+	dev_dbg(sdev->dev, "received event %d for widget %s\n",
+		event, w->name);
+
+	/* process events */
+	switch (event) {
+	case SND_SOC_DAPM_PRE_PMU:
+		/* set pcm params */
+		ret = ipc_pcm_params(swidget, SOF_IPC_STREAM_CAPTURE);
+		if (ret < 0) {
+			dev_err(sdev->dev,
+				"error: failed to set pcm params for widget %s\n",
+				swidget->widget->name);
+			break;
+		}
+
+		/* start trigger */
+		ret = ipc_trigger(swidget, SOF_IPC_STREAM_TRIG_START);
+		if (ret < 0)
+			dev_err(sdev->dev,
+				"error: failed to trigger widget %s\n",
+				swidget->widget->name);
+		break;
+	case SND_SOC_DAPM_POST_PMD:
+		/* stop trigger */
+		ret = ipc_trigger(swidget, SOF_IPC_STREAM_TRIG_STOP);
+		if (ret < 0)
+			dev_err(sdev->dev,
+				"error: failed to trigger widget %s\n",
+				swidget->widget->name);
+
+		/* pcm free */
+		ret = ipc_trigger(swidget, SOF_IPC_STREAM_PCM_FREE);
+		if (ret < 0)
+			dev_err(sdev->dev,
+				"error: failed to trigger widget %s\n",
+				swidget->widget->name);
+		break;
+	default:
+		break;
+	}
+
+	return ret;
+}
+
+/* event handlers for keyword detect component */
+static const struct snd_soc_tplg_widget_events sof_kwd_events[] = {
+	{SOF_KEYWORD_DETECT_DAPM_EVENT, sof_keyword_dapm_event},
+};
+
+static inline int get_tlv_data(const int *p, int tlv[TLV_ITEMS])
+{
+	/* we only support dB scale TLV type at the moment */
+	if ((int)p[SNDRV_CTL_TLVO_TYPE] != SNDRV_CTL_TLVT_DB_SCALE)
+		return -EINVAL;
+
+	/* min value in topology tlv data is multiplied by 100 */
+	tlv[TLV_MIN] = (int)p[SNDRV_CTL_TLVO_DB_SCALE_MIN] / 100;
+
+	/* volume steps */
+	tlv[TLV_STEP] = (int)(p[SNDRV_CTL_TLVO_DB_SCALE_MUTE_AND_STEP] &
+				TLV_DB_SCALE_MASK);
+
+	/* mute ON/OFF */
+	if ((p[SNDRV_CTL_TLVO_DB_SCALE_MUTE_AND_STEP] &
+		TLV_DB_SCALE_MUTE) == 0)
+		tlv[TLV_MUTE] = 0;
+	else
+		tlv[TLV_MUTE] = 1;
+
+	return 0;
+}
+
+/*
+ * Function to truncate an unsigned 64-bit number
+ * by x bits and return 32-bit unsigned number. This
+ * function also takes care of rounding while truncating
+ */
+static inline u32 vol_shift_64(u64 i, u32 x)
+{
+	/* do not truncate more than 32 bits */
+	if (x > 32)
+		x = 32;
+
+	if (x == 0)
+		return (u32)i;
+
+	return (u32)(((i >> (x - 1)) + 1) >> 1);
+}
+
+/*
+ * Function to compute a ^ exp where,
+ * a is a fractional number represented by a fixed-point
+ * integer with a fractional world length of "fwl"
+ * exp is an integer
+ * fwl is the fractional word length
+ * Return value is a fractional number represented by a
+ * fixed-point integer with a fractional word length of "fwl"
+ */
+static u32 vol_pow32(u32 a, int exp, u32 fwl)
+{
+	int i, iter;
+	u32 power = 1 << fwl;
+	u64 numerator;
+
+	/* if exponent is 0, return 1 */
+	if (exp == 0)
+		return power;
+
+	/* determine the number of iterations based on the exponent */
+	if (exp < 0)
+		iter = exp * -1;
+	else
+		iter = exp;
+
+	/* mutiply a "iter" times to compute power */
+	for (i = 0; i < iter; i++) {
+		/*
+		 * Product of 2 Qx.fwl fixed-point numbers yields a Q2*x.2*fwl
+		 * Truncate product back to fwl fractional bits with rounding
+		 */
+		power = vol_shift_64((u64)power * a, fwl);
+	}
+
+	if (exp > 0) {
+		/* if exp is positive, return the result */
+		return power;
+	}
+
+	/* if exp is negative, return the multiplicative inverse */
+	numerator = (u64)1 << (fwl << 1);
+	do_div(numerator, power);
+
+	return (u32)numerator;
+}
+
+/*
+ * Function to calculate volume gain from TLV data.
+ * This function can only handle gain steps that are multiples of 0.5 dB
+ */
+static u32 vol_compute_gain(u32 value, int *tlv)
+{
+	int dB_gain;
+	u32 linear_gain;
+	int f_step;
+
+	/* mute volume */
+	if (value == 0 && tlv[TLV_MUTE])
+		return 0;
+
+	/*
+	 * compute dB gain from tlv. tlv_step
+	 * in topology is multiplied by 100
+	 */
+	dB_gain = tlv[TLV_MIN] + (value * tlv[TLV_STEP]) / 100;
+
+	/*
+	 * compute linear gain represented by fixed-point
+	 * int with VOLUME_FWL fractional bits
+	 */
+	linear_gain = vol_pow32(VOL_TWENTIETH_ROOT_OF_TEN, dB_gain, VOLUME_FWL);
+
+	/* extract the fractional part of volume step */
+	f_step = tlv[TLV_STEP] - (tlv[TLV_STEP] / 100);
+
+	/* if volume step is an odd multiple of 0.5 dB */
+	if (f_step == VOL_HALF_DB_STEP && (value & 1))
+		linear_gain = vol_shift_64((u64)linear_gain *
+						  VOL_FORTIETH_ROOT_OF_TEN,
+						  VOLUME_FWL);
+
+	return linear_gain;
+}
+
+/*
+ * Set up volume table for kcontrols from tlv data
+ * "size" specifies the number of entries in the table
+ */
+static int set_up_volume_table(struct snd_sof_control *scontrol,
+			       int tlv[TLV_ITEMS], int size)
+{
+	int j;
+
+	/* init the volume table */
+	scontrol->volume_table = kcalloc(size, sizeof(u32), GFP_KERNEL);
+	if (!scontrol->volume_table)
+		return -ENOMEM;
+
+	/* populate the volume table */
+	for (j = 0; j < size ; j++)
+		scontrol->volume_table[j] = vol_compute_gain(j, tlv);
+
+	return 0;
+}
+
+struct sof_dai_types {
+	const char *name;
+	enum sof_ipc_dai_type type;
+};
+
+static const struct sof_dai_types sof_dais[] = {
+	{"SSP", SOF_DAI_INTEL_SSP},
+	{"HDA", SOF_DAI_INTEL_HDA},
+	{"DMIC", SOF_DAI_INTEL_DMIC},
+};
+
+static enum sof_ipc_dai_type find_dai(const char *name)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(sof_dais); i++) {
+		if (strcmp(name, sof_dais[i].name) == 0)
+			return sof_dais[i].type;
+	}
+
+	return SOF_DAI_INTEL_NONE;
+}
+
+/*
+ * Supported Frame format types and lookup, add new ones to end of list.
+ */
+
+struct sof_frame_types {
+	const char *name;
+	enum sof_ipc_frame frame;
+};
+
+static const struct sof_frame_types sof_frames[] = {
+	{"s16le", SOF_IPC_FRAME_S16_LE},
+	{"s24le", SOF_IPC_FRAME_S24_4LE},
+	{"s32le", SOF_IPC_FRAME_S32_LE},
+	{"float", SOF_IPC_FRAME_FLOAT},
+};
+
+static enum sof_ipc_frame find_format(const char *name)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(sof_frames); i++) {
+		if (strcmp(name, sof_frames[i].name) == 0)
+			return sof_frames[i].frame;
+	}
+
+	/* use s32le if nothing is specified */
+	return SOF_IPC_FRAME_S32_LE;
+}
+
+struct sof_process_types {
+	const char *name;
+	enum sof_ipc_process_type type;
+	enum sof_comp_type comp_type;
+};
+
+static const struct sof_process_types sof_process[] = {
+	{"EQFIR", SOF_PROCESS_EQFIR, SOF_COMP_EQ_FIR},
+	{"EQIIR", SOF_PROCESS_EQIIR, SOF_COMP_EQ_IIR},
+	{"KEYWORD_DETECT", SOF_PROCESS_KEYWORD_DETECT, SOF_COMP_KEYWORD_DETECT},
+	{"KPB", SOF_PROCESS_KPB, SOF_COMP_KPB},
+	{"CHAN_SELECTOR", SOF_PROCESS_CHAN_SELECTOR, SOF_COMP_SELECTOR},
+};
+
+static enum sof_ipc_process_type find_process(const char *name)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(sof_process); i++) {
+		if (strcmp(name, sof_process[i].name) == 0)
+			return sof_process[i].type;
+	}
+
+	return SOF_PROCESS_NONE;
+}
+
+static enum sof_comp_type find_process_comp_type(enum sof_ipc_process_type type)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(sof_process); i++) {
+		if (sof_process[i].type == type)
+			return sof_process[i].comp_type;
+	}
+
+	return SOF_COMP_NONE;
+}
+
+/*
+ * Standard Kcontrols.
+ */
+
+static int sof_control_load_volume(struct snd_soc_component *scomp,
+				   struct snd_sof_control *scontrol,
+				   struct snd_kcontrol_new *kc,
+				   struct snd_soc_tplg_ctl_hdr *hdr)
+{
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
+	struct snd_soc_tplg_mixer_control *mc =
+		container_of(hdr, struct snd_soc_tplg_mixer_control, hdr);
+	struct sof_ipc_ctrl_data *cdata;
+	int tlv[TLV_ITEMS];
+	unsigned int i;
+	int ret;
+
+	/* validate topology data */
+	if (le32_to_cpu(mc->num_channels) > SND_SOC_TPLG_MAX_CHAN)
+		return -EINVAL;
+
+	/* init the volume get/put data */
+	scontrol->size = sizeof(struct sof_ipc_ctrl_data) +
+			 sizeof(struct sof_ipc_ctrl_value_chan) *
+			 le32_to_cpu(mc->num_channels);
+	scontrol->control_data = kzalloc(scontrol->size, GFP_KERNEL);
+	if (!scontrol->control_data)
+		return -ENOMEM;
+
+	scontrol->comp_id = sdev->next_comp_id;
+	scontrol->num_channels = le32_to_cpu(mc->num_channels);
+
+	/* set cmd for mixer control */
+	if (le32_to_cpu(mc->max) == 1) {
+		scontrol->cmd = SOF_CTRL_CMD_SWITCH;
+		goto out;
+	}
+
+	scontrol->cmd = SOF_CTRL_CMD_VOLUME;
+
+	/* extract tlv data */
+	if (get_tlv_data(kc->tlv.p, tlv) < 0) {
+		dev_err(sdev->dev, "error: invalid TLV data\n");
+		return -EINVAL;
+	}
+
+	/* set up volume table */
+	ret = set_up_volume_table(scontrol, tlv, le32_to_cpu(mc->max) + 1);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: setting up volume table\n");
+		return ret;
+	}
+
+	/* set default volume values to 0dB in control */
+	cdata = scontrol->control_data;
+	for (i = 0; i < scontrol->num_channels; i++) {
+		cdata->chanv[i].channel = i;
+		cdata->chanv[i].value = VOL_ZERO_DB;
+	}
+
+out:
+	dev_dbg(sdev->dev, "tplg: load kcontrol index %d chans %d\n",
+		scontrol->comp_id, scontrol->num_channels);
+
+	return 0;
+}
+
+static int sof_control_load_enum(struct snd_soc_component *scomp,
+				 struct snd_sof_control *scontrol,
+				 struct snd_kcontrol_new *kc,
+				 struct snd_soc_tplg_ctl_hdr *hdr)
+{
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
+	struct snd_soc_tplg_enum_control *ec =
+		container_of(hdr, struct snd_soc_tplg_enum_control, hdr);
+
+	/* validate topology data */
+	if (le32_to_cpu(ec->num_channels) > SND_SOC_TPLG_MAX_CHAN)
+		return -EINVAL;
+
+	/* init the enum get/put data */
+	scontrol->size = sizeof(struct sof_ipc_ctrl_data) +
+			 sizeof(struct sof_ipc_ctrl_value_chan) *
+			 le32_to_cpu(ec->num_channels);
+	scontrol->control_data = kzalloc(scontrol->size, GFP_KERNEL);
+	if (!scontrol->control_data)
+		return -ENOMEM;
+
+	scontrol->comp_id = sdev->next_comp_id;
+	scontrol->num_channels = le32_to_cpu(ec->num_channels);
+
+	scontrol->cmd = SOF_CTRL_CMD_ENUM;
+
+	dev_dbg(sdev->dev, "tplg: load kcontrol index %d chans %d comp_id %d\n",
+		scontrol->comp_id, scontrol->num_channels, scontrol->comp_id);
+
+	return 0;
+}
+
+static int sof_control_load_bytes(struct snd_soc_component *scomp,
+				  struct snd_sof_control *scontrol,
+				  struct snd_kcontrol_new *kc,
+				  struct snd_soc_tplg_ctl_hdr *hdr)
+{
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
+	struct sof_ipc_ctrl_data *cdata;
+	struct snd_soc_tplg_bytes_control *control =
+		container_of(hdr, struct snd_soc_tplg_bytes_control, hdr);
+	struct soc_bytes_ext *sbe = (struct soc_bytes_ext *)kc->private_value;
+	int max_size = sbe->max;
+
+	if (le32_to_cpu(control->priv.size) > max_size) {
+		dev_err(sdev->dev, "err: bytes data size %d exceeds max %d.\n",
+			control->priv.size, max_size);
+		return -EINVAL;
+	}
+
+	/* init the get/put bytes data */
+	scontrol->size = sizeof(struct sof_ipc_ctrl_data) +
+		le32_to_cpu(control->priv.size);
+	scontrol->control_data = kzalloc(max_size, GFP_KERNEL);
+	cdata = scontrol->control_data;
+	if (!scontrol->control_data)
+		return -ENOMEM;
+
+	scontrol->comp_id = sdev->next_comp_id;
+	scontrol->cmd = SOF_CTRL_CMD_BINARY;
+
+	dev_dbg(sdev->dev, "tplg: load kcontrol index %d chans %d\n",
+		scontrol->comp_id, scontrol->num_channels);
+
+	if (le32_to_cpu(control->priv.size) > 0) {
+		memcpy(cdata->data, control->priv.data,
+		       le32_to_cpu(control->priv.size));
+
+		if (cdata->data->magic != SOF_ABI_MAGIC) {
+			dev_err(sdev->dev, "error: Wrong ABI magic 0x%08x.\n",
+				cdata->data->magic);
+			return -EINVAL;
+		}
+		if (SOF_ABI_VERSION_INCOMPATIBLE(SOF_ABI_VERSION,
+						 cdata->data->abi)) {
+			dev_err(sdev->dev,
+				"error: Incompatible ABI version 0x%08x.\n",
+				cdata->data->abi);
+			return -EINVAL;
+		}
+		if (cdata->data->size + sizeof(const struct sof_abi_hdr) !=
+		    le32_to_cpu(control->priv.size)) {
+			dev_err(sdev->dev,
+				"error: Conflict in bytes vs. priv size.\n");
+			return -EINVAL;
+		}
+	}
+	return 0;
+}
+
+/*
+ * Topology Token Parsing.
+ * New tokens should be added to headers and parsing tables below.
+ */
+
+struct sof_topology_token {
+	u32 token;
+	u32 type;
+	int (*get_token)(void *elem, void *object, u32 offset, u32 size);
+	u32 offset;
+	u32 size;
+};
+
+static int get_token_u32(void *elem, void *object, u32 offset, u32 size)
+{
+	struct snd_soc_tplg_vendor_value_elem *velem = elem;
+	u32 *val = (u32 *)((u8 *)object + offset);
+
+	*val = le32_to_cpu(velem->value);
+	return 0;
+}
+
+static int get_token_u16(void *elem, void *object, u32 offset, u32 size)
+{
+	struct snd_soc_tplg_vendor_value_elem *velem = elem;
+	u16 *val = (u16 *)((u8 *)object + offset);
+
+	*val = (u16)le32_to_cpu(velem->value);
+	return 0;
+}
+
+static int get_token_comp_format(void *elem, void *object, u32 offset, u32 size)
+{
+	struct snd_soc_tplg_vendor_string_elem *velem = elem;
+	u32 *val = (u32 *)((u8 *)object + offset);
+
+	*val = find_format(velem->string);
+	return 0;
+}
+
+static int get_token_dai_type(void *elem, void *object, u32 offset, u32 size)
+{
+	struct snd_soc_tplg_vendor_string_elem *velem = elem;
+	u32 *val = (u32 *)((u8 *)object + offset);
+
+	*val = find_dai(velem->string);
+	return 0;
+}
+
+static int get_token_process_type(void *elem, void *object, u32 offset,
+				  u32 size)
+{
+	struct snd_soc_tplg_vendor_string_elem *velem = elem;
+	u32 *val = (u32 *)((u8 *)object + offset);
+
+	*val = find_process(velem->string);
+	return 0;
+}
+
+/* Buffers */
+static const struct sof_topology_token buffer_tokens[] = {
+	{SOF_TKN_BUF_SIZE, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
+		offsetof(struct sof_ipc_buffer, size), 0},
+	{SOF_TKN_BUF_CAPS, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
+		offsetof(struct sof_ipc_buffer, caps), 0},
+};
+
+/* DAI */
+static const struct sof_topology_token dai_tokens[] = {
+	{SOF_TKN_DAI_TYPE, SND_SOC_TPLG_TUPLE_TYPE_STRING, get_token_dai_type,
+		offsetof(struct sof_ipc_comp_dai, type), 0},
+	{SOF_TKN_DAI_INDEX, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
+		offsetof(struct sof_ipc_comp_dai, dai_index), 0},
+	{SOF_TKN_DAI_DIRECTION, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
+		offsetof(struct sof_ipc_comp_dai, direction), 0},
+};
+
+/* BE DAI link */
+static const struct sof_topology_token dai_link_tokens[] = {
+	{SOF_TKN_DAI_TYPE, SND_SOC_TPLG_TUPLE_TYPE_STRING, get_token_dai_type,
+		offsetof(struct sof_ipc_dai_config, type), 0},
+	{SOF_TKN_DAI_INDEX, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
+		offsetof(struct sof_ipc_dai_config, dai_index), 0},
+};
+
+/* scheduling */
+static const struct sof_topology_token sched_tokens[] = {
+	{SOF_TKN_SCHED_PERIOD, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
+		offsetof(struct sof_ipc_pipe_new, period), 0},
+	{SOF_TKN_SCHED_PRIORITY, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
+		offsetof(struct sof_ipc_pipe_new, priority), 0},
+	{SOF_TKN_SCHED_MIPS, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
+		offsetof(struct sof_ipc_pipe_new, period_mips), 0},
+	{SOF_TKN_SCHED_CORE, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
+		offsetof(struct sof_ipc_pipe_new, core), 0},
+	{SOF_TKN_SCHED_FRAMES, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
+		offsetof(struct sof_ipc_pipe_new, frames_per_sched), 0},
+	{SOF_TKN_SCHED_TIME_DOMAIN, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
+		offsetof(struct sof_ipc_pipe_new, time_domain), 0},
+};
+
+/* volume */
+static const struct sof_topology_token volume_tokens[] = {
+	{SOF_TKN_VOLUME_RAMP_STEP_TYPE, SND_SOC_TPLG_TUPLE_TYPE_WORD,
+		get_token_u32, offsetof(struct sof_ipc_comp_volume, ramp), 0},
+	{SOF_TKN_VOLUME_RAMP_STEP_MS,
+		SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
+		offsetof(struct sof_ipc_comp_volume, initial_ramp), 0},
+};
+
+/* SRC */
+static const struct sof_topology_token src_tokens[] = {
+	{SOF_TKN_SRC_RATE_IN, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
+		offsetof(struct sof_ipc_comp_src, source_rate), 0},
+	{SOF_TKN_SRC_RATE_OUT, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
+		offsetof(struct sof_ipc_comp_src, sink_rate), 0},
+};
+
+/* Tone */
+static const struct sof_topology_token tone_tokens[] = {
+};
+
+/* EFFECT */
+static const struct sof_topology_token process_tokens[] = {
+	{SOF_TKN_PROCESS_TYPE, SND_SOC_TPLG_TUPLE_TYPE_STRING,
+		get_token_process_type,
+		offsetof(struct sof_ipc_comp_process, type), 0},
+};
+
+/* PCM */
+static const struct sof_topology_token pcm_tokens[] = {
+	{SOF_TKN_PCM_DMAC_CONFIG, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
+		offsetof(struct sof_ipc_comp_host, dmac_config), 0},
+};
+
+/* Generic components */
+static const struct sof_topology_token comp_tokens[] = {
+	{SOF_TKN_COMP_PERIOD_SINK_COUNT,
+		SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
+		offsetof(struct sof_ipc_comp_config, periods_sink), 0},
+	{SOF_TKN_COMP_PERIOD_SOURCE_COUNT,
+		SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
+		offsetof(struct sof_ipc_comp_config, periods_source), 0},
+	{SOF_TKN_COMP_FORMAT,
+		SND_SOC_TPLG_TUPLE_TYPE_STRING, get_token_comp_format,
+		offsetof(struct sof_ipc_comp_config, frame_fmt), 0},
+};
+
+/* SSP */
+static const struct sof_topology_token ssp_tokens[] = {
+	{SOF_TKN_INTEL_SSP_CLKS_CONTROL,
+		SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
+		offsetof(struct sof_ipc_dai_ssp_params, clks_control), 0},
+	{SOF_TKN_INTEL_SSP_MCLK_ID,
+		SND_SOC_TPLG_TUPLE_TYPE_SHORT, get_token_u16,
+		offsetof(struct sof_ipc_dai_ssp_params, mclk_id), 0},
+	{SOF_TKN_INTEL_SSP_SAMPLE_BITS, SND_SOC_TPLG_TUPLE_TYPE_WORD,
+		get_token_u32,
+		offsetof(struct sof_ipc_dai_ssp_params, sample_valid_bits), 0},
+	{SOF_TKN_INTEL_SSP_FRAME_PULSE_WIDTH, SND_SOC_TPLG_TUPLE_TYPE_SHORT,
+		get_token_u16,
+		offsetof(struct sof_ipc_dai_ssp_params, frame_pulse_width), 0},
+	{SOF_TKN_INTEL_SSP_QUIRKS, SND_SOC_TPLG_TUPLE_TYPE_WORD,
+		get_token_u32,
+		offsetof(struct sof_ipc_dai_ssp_params, quirks), 0},
+	{SOF_TKN_INTEL_SSP_TDM_PADDING_PER_SLOT, SND_SOC_TPLG_TUPLE_TYPE_BOOL,
+		get_token_u16,
+		offsetof(struct sof_ipc_dai_ssp_params,
+			 tdm_per_slot_padding_flag), 0},
+
+};
+
+/* DMIC */
+static const struct sof_topology_token dmic_tokens[] = {
+	{SOF_TKN_INTEL_DMIC_DRIVER_VERSION,
+		SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
+		offsetof(struct sof_ipc_dai_dmic_params, driver_ipc_version),
+		0},
+	{SOF_TKN_INTEL_DMIC_CLK_MIN,
+		SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
+		offsetof(struct sof_ipc_dai_dmic_params, pdmclk_min), 0},
+	{SOF_TKN_INTEL_DMIC_CLK_MAX,
+		SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
+		offsetof(struct sof_ipc_dai_dmic_params, pdmclk_max), 0},
+	{SOF_TKN_INTEL_DMIC_SAMPLE_RATE,
+		SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
+		offsetof(struct sof_ipc_dai_dmic_params, fifo_fs), 0},
+	{SOF_TKN_INTEL_DMIC_DUTY_MIN,
+		SND_SOC_TPLG_TUPLE_TYPE_SHORT, get_token_u16,
+		offsetof(struct sof_ipc_dai_dmic_params, duty_min), 0},
+	{SOF_TKN_INTEL_DMIC_DUTY_MAX,
+		SND_SOC_TPLG_TUPLE_TYPE_SHORT, get_token_u16,
+		offsetof(struct sof_ipc_dai_dmic_params, duty_max), 0},
+	{SOF_TKN_INTEL_DMIC_NUM_PDM_ACTIVE,
+		SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
+		offsetof(struct sof_ipc_dai_dmic_params,
+			 num_pdm_active), 0},
+	{SOF_TKN_INTEL_DMIC_FIFO_WORD_LENGTH,
+		SND_SOC_TPLG_TUPLE_TYPE_SHORT, get_token_u16,
+		offsetof(struct sof_ipc_dai_dmic_params, fifo_bits), 0},
+};
+
+/*
+ * DMIC PDM Tokens
+ * SOF_TKN_INTEL_DMIC_PDM_CTRL_ID should be the first token
+ * as it increments the index while parsing the array of pdm tokens
+ * and determines the correct offset
+ */
+static const struct sof_topology_token dmic_pdm_tokens[] = {
+	{SOF_TKN_INTEL_DMIC_PDM_CTRL_ID,
+		SND_SOC_TPLG_TUPLE_TYPE_SHORT, get_token_u16,
+		offsetof(struct sof_ipc_dai_dmic_pdm_ctrl, id),
+		0},
+	{SOF_TKN_INTEL_DMIC_PDM_MIC_A_Enable,
+		SND_SOC_TPLG_TUPLE_TYPE_SHORT, get_token_u16,
+		offsetof(struct sof_ipc_dai_dmic_pdm_ctrl, enable_mic_a),
+		0},
+	{SOF_TKN_INTEL_DMIC_PDM_MIC_B_Enable,
+		SND_SOC_TPLG_TUPLE_TYPE_SHORT, get_token_u16,
+		offsetof(struct sof_ipc_dai_dmic_pdm_ctrl, enable_mic_b),
+		0},
+	{SOF_TKN_INTEL_DMIC_PDM_POLARITY_A,
+		SND_SOC_TPLG_TUPLE_TYPE_SHORT, get_token_u16,
+		offsetof(struct sof_ipc_dai_dmic_pdm_ctrl, polarity_mic_a),
+		0},
+	{SOF_TKN_INTEL_DMIC_PDM_POLARITY_B,
+		SND_SOC_TPLG_TUPLE_TYPE_SHORT, get_token_u16,
+		offsetof(struct sof_ipc_dai_dmic_pdm_ctrl, polarity_mic_b),
+		0},
+	{SOF_TKN_INTEL_DMIC_PDM_CLK_EDGE,
+		SND_SOC_TPLG_TUPLE_TYPE_SHORT, get_token_u16,
+		offsetof(struct sof_ipc_dai_dmic_pdm_ctrl, clk_edge),
+		0},
+	{SOF_TKN_INTEL_DMIC_PDM_SKEW,
+		SND_SOC_TPLG_TUPLE_TYPE_SHORT, get_token_u16,
+		offsetof(struct sof_ipc_dai_dmic_pdm_ctrl, skew),
+		0},
+};
+
+/* HDA */
+static const struct sof_topology_token hda_tokens[] = {
+};
+
+static void sof_parse_uuid_tokens(struct snd_soc_component *scomp,
+				  void *object,
+				  const struct sof_topology_token *tokens,
+				  int count,
+				  struct snd_soc_tplg_vendor_array *array)
+{
+	struct snd_soc_tplg_vendor_uuid_elem *elem;
+	int i, j;
+
+	/* parse element by element */
+	for (i = 0; i < le32_to_cpu(array->num_elems); i++) {
+		elem = &array->uuid[i];
+
+		/* search for token */
+		for (j = 0; j < count; j++) {
+			/* match token type */
+			if (tokens[j].type != SND_SOC_TPLG_TUPLE_TYPE_UUID)
+				continue;
+
+			/* match token id */
+			if (tokens[j].token != le32_to_cpu(elem->token))
+				continue;
+
+			/* matched - now load token */
+			tokens[j].get_token(elem, object, tokens[j].offset,
+					    tokens[j].size);
+		}
+	}
+}
+
+static void sof_parse_string_tokens(struct snd_soc_component *scomp,
+				    void *object,
+				    const struct sof_topology_token *tokens,
+				    int count,
+				    struct snd_soc_tplg_vendor_array *array)
+{
+	struct snd_soc_tplg_vendor_string_elem *elem;
+	int i, j;
+
+	/* parse element by element */
+	for (i = 0; i < le32_to_cpu(array->num_elems); i++) {
+		elem = &array->string[i];
+
+		/* search for token */
+		for (j = 0; j < count; j++) {
+			/* match token type */
+			if (tokens[j].type != SND_SOC_TPLG_TUPLE_TYPE_STRING)
+				continue;
+
+			/* match token id */
+			if (tokens[j].token != le32_to_cpu(elem->token))
+				continue;
+
+			/* matched - now load token */
+			tokens[j].get_token(elem, object, tokens[j].offset,
+					    tokens[j].size);
+		}
+	}
+}
+
+static void sof_parse_word_tokens(struct snd_soc_component *scomp,
+				  void *object,
+				  const struct sof_topology_token *tokens,
+				  int count,
+				  struct snd_soc_tplg_vendor_array *array)
+{
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
+	struct snd_soc_tplg_vendor_value_elem *elem;
+	size_t size = sizeof(struct sof_ipc_dai_dmic_pdm_ctrl);
+	int i, j;
+	u32 offset;
+	u32 *index = NULL;
+
+	/* parse element by element */
+	for (i = 0; i < le32_to_cpu(array->num_elems); i++) {
+		elem = &array->value[i];
+
+		/* search for token */
+		for (j = 0; j < count; j++) {
+			/* match token type */
+			if (!(tokens[j].type == SND_SOC_TPLG_TUPLE_TYPE_WORD ||
+			      tokens[j].type == SND_SOC_TPLG_TUPLE_TYPE_SHORT))
+				continue;
+
+			/* match token id */
+			if (tokens[j].token != le32_to_cpu(elem->token))
+				continue;
+
+			/* pdm config array index */
+			if (sdev->private)
+				index = sdev->private;
+
+			/* matched - determine offset */
+			switch (tokens[j].token) {
+			case SOF_TKN_INTEL_DMIC_PDM_CTRL_ID:
+
+				/* inc number of pdm array index */
+				if (index)
+					(*index)++;
+				/* fallthrough */
+			case SOF_TKN_INTEL_DMIC_PDM_MIC_A_Enable:
+			case SOF_TKN_INTEL_DMIC_PDM_MIC_B_Enable:
+			case SOF_TKN_INTEL_DMIC_PDM_POLARITY_A:
+			case SOF_TKN_INTEL_DMIC_PDM_POLARITY_B:
+			case SOF_TKN_INTEL_DMIC_PDM_CLK_EDGE:
+			case SOF_TKN_INTEL_DMIC_PDM_SKEW:
+
+				/* check if array index is valid */
+				if (!index || *index == 0) {
+					dev_err(sdev->dev,
+						"error: invalid array offset\n");
+					continue;
+				} else {
+					/* offset within the pdm config array */
+					offset = size * (*index - 1);
+				}
+				break;
+			default:
+				offset = 0;
+				break;
+			}
+
+			/* load token */
+			tokens[j].get_token(elem, object,
+					    offset + tokens[j].offset,
+					    tokens[j].size);
+		}
+	}
+}
+
+static int sof_parse_tokens(struct snd_soc_component *scomp,
+			    void *object,
+			    const struct sof_topology_token *tokens,
+			    int count,
+			    struct snd_soc_tplg_vendor_array *array,
+			    int priv_size)
+{
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
+	int asize;
+
+	while (priv_size > 0) {
+		asize = le32_to_cpu(array->size);
+
+		/* validate asize */
+		if (asize < 0) { /* FIXME: A zero-size array makes no sense */
+			dev_err(sdev->dev, "error: invalid array size 0x%x\n",
+				asize);
+			return -EINVAL;
+		}
+
+		/* make sure there is enough data before parsing */
+		priv_size -= asize;
+		if (priv_size < 0) {
+			dev_err(sdev->dev, "error: invalid array size 0x%x\n",
+				asize);
+			return -EINVAL;
+		}
+
+		/* call correct parser depending on type */
+		switch (le32_to_cpu(array->type)) {
+		case SND_SOC_TPLG_TUPLE_TYPE_UUID:
+			sof_parse_uuid_tokens(scomp, object, tokens, count,
+					      array);
+			break;
+		case SND_SOC_TPLG_TUPLE_TYPE_STRING:
+			sof_parse_string_tokens(scomp, object, tokens, count,
+						array);
+			break;
+		case SND_SOC_TPLG_TUPLE_TYPE_BOOL:
+		case SND_SOC_TPLG_TUPLE_TYPE_BYTE:
+		case SND_SOC_TPLG_TUPLE_TYPE_WORD:
+		case SND_SOC_TPLG_TUPLE_TYPE_SHORT:
+			sof_parse_word_tokens(scomp, object, tokens, count,
+					      array);
+			break;
+		default:
+			dev_err(sdev->dev, "error: unknown token type %d\n",
+				array->type);
+			return -EINVAL;
+		}
+
+		/* next array */
+		array = (struct snd_soc_tplg_vendor_array *)((u8 *)array
+			+ asize);
+	}
+	return 0;
+}
+
+static void sof_dbg_comp_config(struct snd_soc_component *scomp,
+				struct sof_ipc_comp_config *config)
+{
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
+
+	dev_dbg(sdev->dev, " config: periods snk %d src %d fmt %d\n",
+		config->periods_sink, config->periods_source,
+		config->frame_fmt);
+}
+
+/* external kcontrol init - used for any driver specific init */
+static int sof_control_load(struct snd_soc_component *scomp, int index,
+			    struct snd_kcontrol_new *kc,
+			    struct snd_soc_tplg_ctl_hdr *hdr)
+{
+	struct soc_mixer_control *sm;
+	struct soc_bytes_ext *sbe;
+	struct soc_enum *se;
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
+	struct snd_soc_dobj *dobj;
+	struct snd_sof_control *scontrol;
+	int ret = -EINVAL;
+
+	dev_dbg(sdev->dev, "tplg: load control type %d name : %s\n",
+		hdr->type, hdr->name);
+
+	scontrol = kzalloc(sizeof(*scontrol), GFP_KERNEL);
+	if (!scontrol)
+		return -ENOMEM;
+
+	scontrol->sdev = sdev;
+
+	switch (le32_to_cpu(hdr->ops.info)) {
+	case SND_SOC_TPLG_CTL_VOLSW:
+	case SND_SOC_TPLG_CTL_VOLSW_SX:
+	case SND_SOC_TPLG_CTL_VOLSW_XR_SX:
+		sm = (struct soc_mixer_control *)kc->private_value;
+		dobj = &sm->dobj;
+		ret = sof_control_load_volume(scomp, scontrol, kc, hdr);
+		break;
+	case SND_SOC_TPLG_CTL_BYTES:
+		sbe = (struct soc_bytes_ext *)kc->private_value;
+		dobj = &sbe->dobj;
+		ret = sof_control_load_bytes(scomp, scontrol, kc, hdr);
+		break;
+	case SND_SOC_TPLG_CTL_ENUM:
+	case SND_SOC_TPLG_CTL_ENUM_VALUE:
+		se = (struct soc_enum *)kc->private_value;
+		dobj = &se->dobj;
+		ret = sof_control_load_enum(scomp, scontrol, kc, hdr);
+		break;
+	case SND_SOC_TPLG_CTL_RANGE:
+	case SND_SOC_TPLG_CTL_STROBE:
+	case SND_SOC_TPLG_DAPM_CTL_VOLSW:
+	case SND_SOC_TPLG_DAPM_CTL_ENUM_DOUBLE:
+	case SND_SOC_TPLG_DAPM_CTL_ENUM_VIRT:
+	case SND_SOC_TPLG_DAPM_CTL_ENUM_VALUE:
+	case SND_SOC_TPLG_DAPM_CTL_PIN:
+	default:
+		dev_warn(sdev->dev, "control type not supported %d:%d:%d\n",
+			 hdr->ops.get, hdr->ops.put, hdr->ops.info);
+		kfree(scontrol);
+		return 0;
+	}
+
+	dobj->private = scontrol;
+	list_add(&scontrol->list, &sdev->kcontrol_list);
+	return ret;
+}
+
+static int sof_control_unload(struct snd_soc_component *scomp,
+			      struct snd_soc_dobj *dobj)
+{
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
+	struct sof_ipc_free fcomp;
+	struct snd_sof_control *scontrol = dobj->private;
+
+	dev_dbg(sdev->dev, "tplg: unload control name : %s\n", scomp->name);
+
+	fcomp.hdr.cmd = SOF_IPC_GLB_TPLG_MSG | SOF_IPC_TPLG_COMP_FREE;
+	fcomp.hdr.size = sizeof(fcomp);
+	fcomp.id = scontrol->comp_id;
+
+	kfree(scontrol->control_data);
+	list_del(&scontrol->list);
+	kfree(scontrol);
+	/* send IPC to the DSP */
+	return sof_ipc_tx_message(sdev->ipc,
+				  fcomp.hdr.cmd, &fcomp, sizeof(fcomp),
+				  NULL, 0);
+}
+
+/*
+ * DAI Topology
+ */
+
+static int sof_connect_dai_widget(struct snd_soc_component *scomp,
+				  struct snd_soc_dapm_widget *w,
+				  struct snd_soc_tplg_dapm_widget *tw,
+				  struct snd_sof_dai *dai)
+{
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
+	struct snd_soc_card *card = scomp->card;
+	struct snd_soc_pcm_runtime *rtd;
+
+	list_for_each_entry(rtd, &card->rtd_list, list) {
+		dev_vdbg(sdev->dev, "tplg: check widget: %s stream: %s dai stream: %s\n",
+			 w->name,  w->sname, rtd->dai_link->stream_name);
+
+		if (!w->sname || !rtd->dai_link->stream_name)
+			continue;
+
+		/* does stream match DAI link ? */
+		if (strcmp(w->sname, rtd->dai_link->stream_name))
+			continue;
+
+		switch (w->id) {
+		case snd_soc_dapm_dai_out:
+			rtd->cpu_dai->capture_widget = w;
+			dai->name = rtd->dai_link->name;
+			dev_dbg(sdev->dev, "tplg: connected widget %s -> DAI link %s\n",
+				w->name, rtd->dai_link->name);
+			break;
+		case snd_soc_dapm_dai_in:
+			rtd->cpu_dai->playback_widget = w;
+			dai->name = rtd->dai_link->name;
+			dev_dbg(sdev->dev, "tplg: connected widget %s -> DAI link %s\n",
+				w->name, rtd->dai_link->name);
+			break;
+		default:
+			break;
+		}
+	}
+
+	/* check we have a connection */
+	if (!dai->name) {
+		dev_err(sdev->dev, "error: can't connect DAI %s stream %s\n",
+			w->name, w->sname);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int sof_widget_load_dai(struct snd_soc_component *scomp, int index,
+			       struct snd_sof_widget *swidget,
+			       struct snd_soc_tplg_dapm_widget *tw,
+			       struct sof_ipc_comp_reply *r,
+			       struct snd_sof_dai *dai)
+{
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
+	struct snd_soc_tplg_private *private = &tw->priv;
+	struct sof_ipc_comp_dai comp_dai;
+	int ret;
+
+	/* configure dai IPC message */
+	memset(&comp_dai, 0, sizeof(comp_dai));
+	comp_dai.comp.hdr.size = sizeof(comp_dai);
+	comp_dai.comp.hdr.cmd = SOF_IPC_GLB_TPLG_MSG | SOF_IPC_TPLG_COMP_NEW;
+	comp_dai.comp.id = swidget->comp_id;
+	comp_dai.comp.type = SOF_COMP_DAI;
+	comp_dai.comp.pipeline_id = index;
+	comp_dai.config.hdr.size = sizeof(comp_dai.config);
+
+	ret = sof_parse_tokens(scomp, &comp_dai, dai_tokens,
+			       ARRAY_SIZE(dai_tokens), private->array,
+			       le32_to_cpu(private->size));
+	if (ret != 0) {
+		dev_err(sdev->dev, "error: parse dai tokens failed %d\n",
+			le32_to_cpu(private->size));
+		return ret;
+	}
+
+	ret = sof_parse_tokens(scomp, &comp_dai.config, comp_tokens,
+			       ARRAY_SIZE(comp_tokens), private->array,
+			       le32_to_cpu(private->size));
+	if (ret != 0) {
+		dev_err(sdev->dev, "error: parse dai.cfg tokens failed %d\n",
+			private->size);
+		return ret;
+	}
+
+	dev_dbg(sdev->dev, "dai %s: type %d index %d\n",
+		swidget->widget->name, comp_dai.type, comp_dai.dai_index);
+	sof_dbg_comp_config(scomp, &comp_dai.config);
+
+	ret = sof_ipc_tx_message(sdev->ipc, comp_dai.comp.hdr.cmd,
+				 &comp_dai, sizeof(comp_dai), r, sizeof(*r));
+
+	if (ret == 0 && dai) {
+		dai->sdev = sdev;
+		memcpy(&dai->comp_dai, &comp_dai, sizeof(comp_dai));
+	}
+
+	return ret;
+}
+
+/*
+ * Buffer topology
+ */
+
+static int sof_widget_load_buffer(struct snd_soc_component *scomp, int index,
+				  struct snd_sof_widget *swidget,
+				  struct snd_soc_tplg_dapm_widget *tw,
+				  struct sof_ipc_comp_reply *r)
+{
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
+	struct snd_soc_tplg_private *private = &tw->priv;
+	struct sof_ipc_buffer *buffer;
+	int ret;
+
+	buffer = kzalloc(sizeof(*buffer), GFP_KERNEL);
+	if (!buffer)
+		return -ENOMEM;
+
+	/* configure dai IPC message */
+	buffer->comp.hdr.size = sizeof(*buffer);
+	buffer->comp.hdr.cmd = SOF_IPC_GLB_TPLG_MSG | SOF_IPC_TPLG_BUFFER_NEW;
+	buffer->comp.id = swidget->comp_id;
+	buffer->comp.type = SOF_COMP_BUFFER;
+	buffer->comp.pipeline_id = index;
+
+	ret = sof_parse_tokens(scomp, buffer, buffer_tokens,
+			       ARRAY_SIZE(buffer_tokens), private->array,
+			       le32_to_cpu(private->size));
+	if (ret != 0) {
+		dev_err(sdev->dev, "error: parse buffer tokens failed %d\n",
+			private->size);
+		kfree(buffer);
+		return ret;
+	}
+
+	dev_dbg(sdev->dev, "buffer %s: size %d caps 0x%x\n",
+		swidget->widget->name, buffer->size, buffer->caps);
+
+	swidget->private = buffer;
+
+	ret = sof_ipc_tx_message(sdev->ipc, buffer->comp.hdr.cmd, buffer,
+				 sizeof(*buffer), r, sizeof(*r));
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: buffer %s load failed\n",
+			swidget->widget->name);
+		kfree(buffer);
+	}
+
+	return ret;
+}
+
+/* bind PCM ID to host component ID */
+static int spcm_bind(struct snd_sof_dev *sdev, struct snd_sof_pcm *spcm,
+		     int dir)
+{
+	struct snd_sof_widget *host_widget;
+
+	host_widget = snd_sof_find_swidget_sname(sdev,
+						 spcm->pcm.caps[dir].name,
+						 dir);
+	if (!host_widget) {
+		dev_err(sdev->dev, "can't find host comp to bind pcm\n");
+		return -EINVAL;
+	}
+
+	spcm->stream[dir].comp_id = host_widget->comp_id;
+
+	return 0;
+}
+
+/*
+ * PCM Topology
+ */
+
+static int sof_widget_load_pcm(struct snd_soc_component *scomp, int index,
+			       struct snd_sof_widget *swidget,
+			       enum sof_ipc_stream_direction dir,
+			       struct snd_soc_tplg_dapm_widget *tw,
+			       struct sof_ipc_comp_reply *r)
+{
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
+	struct snd_soc_tplg_private *private = &tw->priv;
+	struct sof_ipc_comp_host *host;
+	int ret;
+
+	host = kzalloc(sizeof(*host), GFP_KERNEL);
+	if (!host)
+		return -ENOMEM;
+
+	/* configure host comp IPC message */
+	host->comp.hdr.size = sizeof(*host);
+	host->comp.hdr.cmd = SOF_IPC_GLB_TPLG_MSG | SOF_IPC_TPLG_COMP_NEW;
+	host->comp.id = swidget->comp_id;
+	host->comp.type = SOF_COMP_HOST;
+	host->comp.pipeline_id = index;
+	host->direction = dir;
+	host->config.hdr.size = sizeof(host->config);
+
+	ret = sof_parse_tokens(scomp, host, pcm_tokens,
+			       ARRAY_SIZE(pcm_tokens), private->array,
+			       le32_to_cpu(private->size));
+	if (ret != 0) {
+		dev_err(sdev->dev, "error: parse host tokens failed %d\n",
+			private->size);
+		goto err;
+	}
+
+	ret = sof_parse_tokens(scomp, &host->config, comp_tokens,
+			       ARRAY_SIZE(comp_tokens), private->array,
+			       le32_to_cpu(private->size));
+	if (ret != 0) {
+		dev_err(sdev->dev, "error: parse host.cfg tokens failed %d\n",
+			le32_to_cpu(private->size));
+		goto err;
+	}
+
+	dev_dbg(sdev->dev, "loaded host %s\n", swidget->widget->name);
+	sof_dbg_comp_config(scomp, &host->config);
+
+	swidget->private = host;
+
+	ret = sof_ipc_tx_message(sdev->ipc, host->comp.hdr.cmd, host,
+				 sizeof(*host), r, sizeof(*r));
+	if (ret >= 0)
+		return ret;
+err:
+	kfree(host);
+	return ret;
+}
+
+/*
+ * Pipeline Topology
+ */
+int sof_load_pipeline_ipc(struct snd_sof_dev *sdev,
+			  struct sof_ipc_pipe_new *pipeline,
+			  struct sof_ipc_comp_reply *r)
+{
+	struct sof_ipc_pm_core_config pm_core_config;
+	int ret;
+
+	ret = sof_ipc_tx_message(sdev->ipc, pipeline->hdr.cmd, pipeline,
+				 sizeof(*pipeline), r, sizeof(*r));
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: load pipeline ipc failure\n");
+		return ret;
+	}
+
+	/* power up the core that this pipeline is scheduled on */
+	ret = snd_sof_dsp_core_power_up(sdev, 1 << pipeline->core);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: powering up pipeline schedule core %d\n",
+			pipeline->core);
+		return ret;
+	}
+
+	/* update enabled cores mask */
+	sdev->enabled_cores_mask |= 1 << pipeline->core;
+
+	/*
+	 * Now notify DSP that the core that this pipeline is scheduled on
+	 * has been powered up
+	 */
+	memset(&pm_core_config, 0, sizeof(pm_core_config));
+	pm_core_config.enable_mask = sdev->enabled_cores_mask;
+
+	/* configure CORE_ENABLE ipc message */
+	pm_core_config.hdr.size = sizeof(pm_core_config);
+	pm_core_config.hdr.cmd = SOF_IPC_GLB_PM_MSG | SOF_IPC_PM_CORE_ENABLE;
+
+	/* send ipc */
+	ret = sof_ipc_tx_message(sdev->ipc, pm_core_config.hdr.cmd,
+				 &pm_core_config, sizeof(pm_core_config),
+				 &pm_core_config, sizeof(pm_core_config));
+	if (ret < 0)
+		dev_err(sdev->dev, "error: core enable ipc failure\n");
+
+	return ret;
+}
+
+static int sof_widget_load_pipeline(struct snd_soc_component *scomp,
+				    int index, struct snd_sof_widget *swidget,
+				    struct snd_soc_tplg_dapm_widget *tw,
+				    struct sof_ipc_comp_reply *r)
+{
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
+	struct snd_soc_tplg_private *private = &tw->priv;
+	struct sof_ipc_pipe_new *pipeline;
+	struct snd_sof_widget *comp_swidget;
+	int ret;
+
+	pipeline = kzalloc(sizeof(*pipeline), GFP_KERNEL);
+	if (!pipeline)
+		return -ENOMEM;
+
+	/* configure dai IPC message */
+	pipeline->hdr.size = sizeof(*pipeline);
+	pipeline->hdr.cmd = SOF_IPC_GLB_TPLG_MSG | SOF_IPC_TPLG_PIPE_NEW;
+	pipeline->pipeline_id = index;
+	pipeline->comp_id = swidget->comp_id;
+
+	/* component at start of pipeline is our stream id */
+	comp_swidget = snd_sof_find_swidget(sdev, tw->sname);
+	if (!comp_swidget) {
+		dev_err(sdev->dev, "error: widget %s refers to non existent widget %s\n",
+			tw->name, tw->sname);
+		ret = -EINVAL;
+		goto err;
+	}
+
+	pipeline->sched_id = comp_swidget->comp_id;
+
+	dev_dbg(sdev->dev, "tplg: pipeline id %d comp %d scheduling comp id %d\n",
+		pipeline->pipeline_id, pipeline->comp_id, pipeline->sched_id);
+
+	ret = sof_parse_tokens(scomp, pipeline, sched_tokens,
+			       ARRAY_SIZE(sched_tokens), private->array,
+			       le32_to_cpu(private->size));
+	if (ret != 0) {
+		dev_err(sdev->dev, "error: parse pipeline tokens failed %d\n",
+			private->size);
+		goto err;
+	}
+
+	dev_dbg(sdev->dev, "pipeline %s: period %d pri %d mips %d core %d frames %d\n",
+		swidget->widget->name, pipeline->period, pipeline->priority,
+		pipeline->period_mips, pipeline->core, pipeline->frames_per_sched);
+
+	swidget->private = pipeline;
+
+	/* send ipc's to create pipeline comp and power up schedule core */
+	ret = sof_load_pipeline_ipc(sdev, pipeline, r);
+	if (ret >= 0)
+		return ret;
+err:
+	kfree(pipeline);
+	return ret;
+}
+
+/*
+ * Mixer topology
+ */
+
+static int sof_widget_load_mixer(struct snd_soc_component *scomp, int index,
+				 struct snd_sof_widget *swidget,
+				 struct snd_soc_tplg_dapm_widget *tw,
+				 struct sof_ipc_comp_reply *r)
+{
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
+	struct snd_soc_tplg_private *private = &tw->priv;
+	struct sof_ipc_comp_mixer *mixer;
+	int ret;
+
+	mixer = kzalloc(sizeof(*mixer), GFP_KERNEL);
+	if (!mixer)
+		return -ENOMEM;
+
+	/* configure mixer IPC message */
+	mixer->comp.hdr.size = sizeof(*mixer);
+	mixer->comp.hdr.cmd = SOF_IPC_GLB_TPLG_MSG | SOF_IPC_TPLG_COMP_NEW;
+	mixer->comp.id = swidget->comp_id;
+	mixer->comp.type = SOF_COMP_MIXER;
+	mixer->comp.pipeline_id = index;
+	mixer->config.hdr.size = sizeof(mixer->config);
+
+	ret = sof_parse_tokens(scomp, &mixer->config, comp_tokens,
+			       ARRAY_SIZE(comp_tokens), private->array,
+			       le32_to_cpu(private->size));
+	if (ret != 0) {
+		dev_err(sdev->dev, "error: parse mixer.cfg tokens failed %d\n",
+			private->size);
+		kfree(mixer);
+		return ret;
+	}
+
+	sof_dbg_comp_config(scomp, &mixer->config);
+
+	swidget->private = mixer;
+
+	ret = sof_ipc_tx_message(sdev->ipc, mixer->comp.hdr.cmd, mixer,
+				 sizeof(*mixer), r, sizeof(*r));
+	if (ret < 0)
+		kfree(mixer);
+
+	return ret;
+}
+
+/*
+ * Mux topology
+ */
+static int sof_widget_load_mux(struct snd_soc_component *scomp, int index,
+			       struct snd_sof_widget *swidget,
+			       struct snd_soc_tplg_dapm_widget *tw,
+			       struct sof_ipc_comp_reply *r)
+{
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
+	struct snd_soc_tplg_private *private = &tw->priv;
+	struct sof_ipc_comp_mux *mux;
+	int ret;
+
+	mux = kzalloc(sizeof(*mux), GFP_KERNEL);
+	if (!mux)
+		return -ENOMEM;
+
+	/* configure mux IPC message */
+	mux->comp.hdr.size = sizeof(*mux);
+	mux->comp.hdr.cmd = SOF_IPC_GLB_TPLG_MSG | SOF_IPC_TPLG_COMP_NEW;
+	mux->comp.id = swidget->comp_id;
+	mux->comp.type = SOF_COMP_MUX;
+	mux->comp.pipeline_id = index;
+	mux->config.hdr.size = sizeof(mux->config);
+
+	ret = sof_parse_tokens(scomp, &mux->config, comp_tokens,
+			       ARRAY_SIZE(comp_tokens), private->array,
+			       le32_to_cpu(private->size));
+	if (ret != 0) {
+		dev_err(sdev->dev, "error: parse mux.cfg tokens failed %d\n",
+			private->size);
+		kfree(mux);
+		return ret;
+	}
+
+	sof_dbg_comp_config(scomp, &mux->config);
+
+	swidget->private = mux;
+
+	ret = sof_ipc_tx_message(sdev->ipc, mux->comp.hdr.cmd, mux,
+				 sizeof(*mux), r, sizeof(*r));
+	if (ret < 0)
+		kfree(mux);
+
+	return ret;
+}
+
+/*
+ * PGA Topology
+ */
+
+static int sof_widget_load_pga(struct snd_soc_component *scomp, int index,
+			       struct snd_sof_widget *swidget,
+			       struct snd_soc_tplg_dapm_widget *tw,
+			       struct sof_ipc_comp_reply *r)
+{
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
+	struct snd_soc_tplg_private *private = &tw->priv;
+	struct sof_ipc_comp_volume *volume;
+	int ret;
+
+	volume = kzalloc(sizeof(*volume), GFP_KERNEL);
+	if (!volume)
+		return -ENOMEM;
+
+	if (le32_to_cpu(tw->num_kcontrols) != 1) {
+		dev_err(sdev->dev, "error: invalid kcontrol count %d for volume\n",
+			tw->num_kcontrols);
+		ret = -EINVAL;
+		goto err;
+	}
+
+	/* configure volume IPC message */
+	volume->comp.hdr.size = sizeof(*volume);
+	volume->comp.hdr.cmd = SOF_IPC_GLB_TPLG_MSG | SOF_IPC_TPLG_COMP_NEW;
+	volume->comp.id = swidget->comp_id;
+	volume->comp.type = SOF_COMP_VOLUME;
+	volume->comp.pipeline_id = index;
+	volume->config.hdr.size = sizeof(volume->config);
+
+	ret = sof_parse_tokens(scomp, volume, volume_tokens,
+			       ARRAY_SIZE(volume_tokens), private->array,
+			       le32_to_cpu(private->size));
+	if (ret != 0) {
+		dev_err(sdev->dev, "error: parse volume tokens failed %d\n",
+			private->size);
+		goto err;
+	}
+	ret = sof_parse_tokens(scomp, &volume->config, comp_tokens,
+			       ARRAY_SIZE(comp_tokens), private->array,
+			       le32_to_cpu(private->size));
+	if (ret != 0) {
+		dev_err(sdev->dev, "error: parse volume.cfg tokens failed %d\n",
+			le32_to_cpu(private->size));
+		goto err;
+	}
+
+	sof_dbg_comp_config(scomp, &volume->config);
+
+	swidget->private = volume;
+
+	ret = sof_ipc_tx_message(sdev->ipc, volume->comp.hdr.cmd, volume,
+				 sizeof(*volume), r, sizeof(*r));
+	if (ret >= 0)
+		return ret;
+err:
+	kfree(volume);
+	return ret;
+}
+
+/*
+ * SRC Topology
+ */
+
+static int sof_widget_load_src(struct snd_soc_component *scomp, int index,
+			       struct snd_sof_widget *swidget,
+			       struct snd_soc_tplg_dapm_widget *tw,
+			       struct sof_ipc_comp_reply *r)
+{
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
+	struct snd_soc_tplg_private *private = &tw->priv;
+	struct sof_ipc_comp_src *src;
+	int ret;
+
+	src = kzalloc(sizeof(*src), GFP_KERNEL);
+	if (!src)
+		return -ENOMEM;
+
+	/* configure src IPC message */
+	src->comp.hdr.size = sizeof(*src);
+	src->comp.hdr.cmd = SOF_IPC_GLB_TPLG_MSG | SOF_IPC_TPLG_COMP_NEW;
+	src->comp.id = swidget->comp_id;
+	src->comp.type = SOF_COMP_SRC;
+	src->comp.pipeline_id = index;
+	src->config.hdr.size = sizeof(src->config);
+
+	ret = sof_parse_tokens(scomp, src, src_tokens,
+			       ARRAY_SIZE(src_tokens), private->array,
+			       le32_to_cpu(private->size));
+	if (ret != 0) {
+		dev_err(sdev->dev, "error: parse src tokens failed %d\n",
+			private->size);
+		goto err;
+	}
+
+	ret = sof_parse_tokens(scomp, &src->config, comp_tokens,
+			       ARRAY_SIZE(comp_tokens), private->array,
+			       le32_to_cpu(private->size));
+	if (ret != 0) {
+		dev_err(sdev->dev, "error: parse src.cfg tokens failed %d\n",
+			le32_to_cpu(private->size));
+		goto err;
+	}
+
+	dev_dbg(sdev->dev, "src %s: source rate %d sink rate %d\n",
+		swidget->widget->name, src->source_rate, src->sink_rate);
+	sof_dbg_comp_config(scomp, &src->config);
+
+	swidget->private = src;
+
+	ret = sof_ipc_tx_message(sdev->ipc, src->comp.hdr.cmd, src,
+				 sizeof(*src), r, sizeof(*r));
+	if (ret >= 0)
+		return ret;
+err:
+	kfree(src);
+	return ret;
+}
+
+/*
+ * Signal Generator Topology
+ */
+
+static int sof_widget_load_siggen(struct snd_soc_component *scomp, int index,
+				  struct snd_sof_widget *swidget,
+				  struct snd_soc_tplg_dapm_widget *tw,
+				  struct sof_ipc_comp_reply *r)
+{
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
+	struct snd_soc_tplg_private *private = &tw->priv;
+	struct sof_ipc_comp_tone *tone;
+	int ret;
+
+	tone = kzalloc(sizeof(*tone), GFP_KERNEL);
+	if (!tone)
+		return -ENOMEM;
+
+	/* configure siggen IPC message */
+	tone->comp.hdr.size = sizeof(*tone);
+	tone->comp.hdr.cmd = SOF_IPC_GLB_TPLG_MSG | SOF_IPC_TPLG_COMP_NEW;
+	tone->comp.id = swidget->comp_id;
+	tone->comp.type = SOF_COMP_TONE;
+	tone->comp.pipeline_id = index;
+	tone->config.hdr.size = sizeof(tone->config);
+
+	ret = sof_parse_tokens(scomp, tone, tone_tokens,
+			       ARRAY_SIZE(tone_tokens), private->array,
+			       le32_to_cpu(private->size));
+	if (ret != 0) {
+		dev_err(sdev->dev, "error: parse tone tokens failed %d\n",
+			le32_to_cpu(private->size));
+		goto err;
+	}
+
+	ret = sof_parse_tokens(scomp, &tone->config, comp_tokens,
+			       ARRAY_SIZE(comp_tokens), private->array,
+			       le32_to_cpu(private->size));
+	if (ret != 0) {
+		dev_err(sdev->dev, "error: parse tone.cfg tokens failed %d\n",
+			le32_to_cpu(private->size));
+		goto err;
+	}
+
+	dev_dbg(sdev->dev, "tone %s: frequency %d amplitude %d\n",
+		swidget->widget->name, tone->frequency, tone->amplitude);
+	sof_dbg_comp_config(scomp, &tone->config);
+
+	swidget->private = tone;
+
+	ret = sof_ipc_tx_message(sdev->ipc, tone->comp.hdr.cmd, tone,
+				 sizeof(*tone), r, sizeof(*r));
+	if (ret >= 0)
+		return ret;
+err:
+	kfree(tone);
+	return ret;
+}
+
+static int sof_process_load(struct snd_soc_component *scomp, int index,
+			    struct snd_sof_widget *swidget,
+			    struct snd_soc_tplg_dapm_widget *tw,
+			    struct sof_ipc_comp_reply *r,
+			    int type)
+{
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
+	struct snd_soc_tplg_private *private = &tw->priv;
+	struct snd_soc_dapm_widget *widget = swidget->widget;
+	const struct snd_kcontrol_new *kc;
+	struct soc_bytes_ext *sbe;
+	struct soc_mixer_control *sm;
+	struct soc_enum *se;
+	struct snd_sof_control *scontrol = NULL;
+	struct sof_abi_hdr *pdata = NULL;
+	struct sof_ipc_comp_process *process;
+	size_t ipc_size, ipc_data_size = 0;
+	int ret, i, offset = 0;
+
+	if (type == SOF_COMP_NONE) {
+		dev_err(sdev->dev, "error: invalid process comp type %d\n",
+			type);
+		return -EINVAL;
+	}
+
+	/*
+	 * get possible component controls - get size of all pdata,
+	 * then memcpy with headers
+	 */
+	for (i = 0; i < widget->num_kcontrols; i++) {
+
+		kc = &widget->kcontrol_news[i];
+
+		switch (widget->dobj.widget.kcontrol_type) {
+		case SND_SOC_TPLG_TYPE_MIXER:
+			sm = (struct soc_mixer_control *)kc->private_value;
+			scontrol = sm->dobj.private;
+			break;
+		case SND_SOC_TPLG_TYPE_BYTES:
+			sbe = (struct soc_bytes_ext *)kc->private_value;
+			scontrol = sbe->dobj.private;
+			break;
+		case SND_SOC_TPLG_TYPE_ENUM:
+			se = (struct soc_enum *)kc->private_value;
+			scontrol = se->dobj.private;
+			break;
+		default:
+			dev_err(sdev->dev, "error: unknown kcontrol type %d in widget %s\n",
+				widget->dobj.widget.kcontrol_type,
+				widget->name);
+			return -EINVAL;
+		}
+
+		if (!scontrol) {
+			dev_err(sdev->dev, "error: no scontrol for widget %s\n",
+				widget->name);
+			return -EINVAL;
+		}
+
+		/* don't include if no private data */
+		pdata = scontrol->control_data->data;
+		if (!pdata)
+			continue;
+
+		/* make sure data is valid - data can be updated at runtime */
+		if (pdata->magic != SOF_ABI_MAGIC)
+			continue;
+
+		ipc_data_size += pdata->size;
+	}
+
+	ipc_size = sizeof(struct sof_ipc_comp_process) +
+		le32_to_cpu(private->size) +
+		ipc_data_size;
+
+	process = kzalloc(ipc_size, GFP_KERNEL);
+	if (!process)
+		return -ENOMEM;
+
+	/* configure iir IPC message */
+	process->comp.hdr.size = ipc_size;
+	process->comp.hdr.cmd = SOF_IPC_GLB_TPLG_MSG | SOF_IPC_TPLG_COMP_NEW;
+	process->comp.id = swidget->comp_id;
+	process->comp.type = type;
+	process->comp.pipeline_id = index;
+	process->config.hdr.size = sizeof(process->config);
+
+	ret = sof_parse_tokens(scomp, &process->config, comp_tokens,
+			       ARRAY_SIZE(comp_tokens), private->array,
+			       le32_to_cpu(private->size));
+	if (ret != 0) {
+		dev_err(sdev->dev, "error: parse process.cfg tokens failed %d\n",
+			le32_to_cpu(private->size));
+		goto err;
+	}
+
+	sof_dbg_comp_config(scomp, &process->config);
+
+	/*
+	 * found private data in control, so copy it.
+	 * get possible component controls - get size of all pdata,
+	 * then memcpy with headers
+	 */
+	for (i = 0; i < widget->num_kcontrols; i++) {
+		kc = &widget->kcontrol_news[i];
+
+		switch (widget->dobj.widget.kcontrol_type) {
+		case SND_SOC_TPLG_TYPE_MIXER:
+			sm = (struct soc_mixer_control *)kc->private_value;
+			scontrol = sm->dobj.private;
+			break;
+		case SND_SOC_TPLG_TYPE_BYTES:
+			sbe = (struct soc_bytes_ext *)kc->private_value;
+			scontrol = sbe->dobj.private;
+			break;
+		case SND_SOC_TPLG_TYPE_ENUM:
+			se = (struct soc_enum *)kc->private_value;
+			scontrol = se->dobj.private;
+			break;
+		default:
+			dev_err(sdev->dev, "error: unknown kcontrol type %d in widget %s\n",
+				widget->dobj.widget.kcontrol_type,
+				widget->name);
+			return -EINVAL;
+		}
+
+		/* don't include if no private data */
+		pdata = scontrol->control_data->data;
+		if (!pdata)
+			continue;
+
+		/* make sure data is valid - data can be updated at runtime */
+		if (pdata->magic != SOF_ABI_MAGIC)
+			continue;
+
+		memcpy(&process->data + offset, pdata->data, pdata->size);
+		offset += pdata->size;
+	}
+
+	process->size = ipc_data_size;
+	swidget->private = process;
+
+	ret = sof_ipc_tx_message(sdev->ipc, process->comp.hdr.cmd, process,
+				 ipc_size, r, sizeof(*r));
+	if (ret >= 0)
+		return ret;
+err:
+	kfree(process);
+	return ret;
+}
+
+/*
+ * Processing Component Topology - can be "effect", "codec", or general
+ * "processing".
+ */
+
+static int sof_widget_load_process(struct snd_soc_component *scomp, int index,
+				   struct snd_sof_widget *swidget,
+				   struct snd_soc_tplg_dapm_widget *tw,
+				   struct sof_ipc_comp_reply *r)
+{
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
+	struct snd_soc_tplg_private *private = &tw->priv;
+	struct sof_ipc_comp_process config;
+	int ret;
+
+	/* check we have some tokens - we need at least process type */
+	if (le32_to_cpu(private->size) == 0) {
+		dev_err(sdev->dev, "error: process tokens not found\n");
+		return -EINVAL;
+	}
+
+	memset(&config, 0, sizeof(config));
+
+	/* get the process token */
+	ret = sof_parse_tokens(scomp, &config, process_tokens,
+			       ARRAY_SIZE(process_tokens), private->array,
+			       le32_to_cpu(private->size));
+	if (ret != 0) {
+		dev_err(sdev->dev, "error: parse process tokens failed %d\n",
+			le32_to_cpu(private->size));
+		return ret;
+	}
+
+	/* now load process specific data and send IPC */
+	ret = sof_process_load(scomp, index, swidget, tw, r,
+			       find_process_comp_type(config.type));
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: process loading failed\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static int sof_widget_bind_event(struct snd_sof_dev *sdev,
+				 struct snd_sof_widget *swidget,
+				 u16 event_type)
+{
+	struct sof_ipc_comp *ipc_comp;
+
+	/* validate widget event type */
+	switch (event_type) {
+	case SOF_KEYWORD_DETECT_DAPM_EVENT:
+		/* only KEYWORD_DETECT comps should handle this */
+		if (swidget->id != snd_soc_dapm_effect)
+			break;
+
+		ipc_comp = swidget->private;
+		if (ipc_comp && ipc_comp->type != SOF_COMP_KEYWORD_DETECT)
+			break;
+
+		/* bind event to keyword detect comp */
+		return snd_soc_tplg_widget_bind_event(swidget->widget,
+						      sof_kwd_events,
+						      ARRAY_SIZE(sof_kwd_events),
+						      event_type);
+	default:
+		break;
+	}
+
+	dev_err(sdev->dev,
+		"error: invalid event type %d for widget %s\n",
+		event_type, swidget->widget->name);
+	return -EINVAL;
+}
+
+/* external widget init - used for any driver specific init */
+static int sof_widget_ready(struct snd_soc_component *scomp, int index,
+			    struct snd_soc_dapm_widget *w,
+			    struct snd_soc_tplg_dapm_widget *tw)
+{
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
+	struct snd_sof_widget *swidget;
+	struct snd_sof_dai *dai;
+	struct sof_ipc_comp_reply reply;
+	struct snd_sof_control *scontrol;
+	int ret = 0;
+
+	swidget = kzalloc(sizeof(*swidget), GFP_KERNEL);
+	if (!swidget)
+		return -ENOMEM;
+
+	swidget->sdev = sdev;
+	swidget->widget = w;
+	swidget->comp_id = sdev->next_comp_id++;
+	swidget->complete = 0;
+	swidget->id = w->id;
+	swidget->pipeline_id = index;
+	swidget->private = NULL;
+	memset(&reply, 0, sizeof(reply));
+
+	dev_dbg(sdev->dev, "tplg: ready widget id %d pipe %d type %d name : %s stream %s\n",
+		swidget->comp_id, index, swidget->id, tw->name,
+		strnlen(tw->sname, SNDRV_CTL_ELEM_ID_NAME_MAXLEN) > 0
+			? tw->sname : "none");
+
+	/* handle any special case widgets */
+	switch (w->id) {
+	case snd_soc_dapm_dai_in:
+	case snd_soc_dapm_dai_out:
+		dai = kzalloc(sizeof(*dai), GFP_KERNEL);
+		if (!dai) {
+			kfree(swidget);
+			return -ENOMEM;
+		}
+
+		ret = sof_widget_load_dai(scomp, index, swidget, tw, &reply,
+					  dai);
+		if (ret == 0) {
+			sof_connect_dai_widget(scomp, w, tw, dai);
+			list_add(&dai->list, &sdev->dai_list);
+			swidget->private = dai;
+		} else {
+			kfree(dai);
+		}
+		break;
+	case snd_soc_dapm_mixer:
+		ret = sof_widget_load_mixer(scomp, index, swidget, tw, &reply);
+		break;
+	case snd_soc_dapm_pga:
+		ret = sof_widget_load_pga(scomp, index, swidget, tw, &reply);
+		/* Find scontrol for this pga and set readback offset*/
+		list_for_each_entry(scontrol, &sdev->kcontrol_list, list) {
+			if (scontrol->comp_id == swidget->comp_id) {
+				scontrol->readback_offset = reply.offset;
+				break;
+			}
+		}
+		break;
+	case snd_soc_dapm_buffer:
+		ret = sof_widget_load_buffer(scomp, index, swidget, tw, &reply);
+		break;
+	case snd_soc_dapm_scheduler:
+		ret = sof_widget_load_pipeline(scomp, index, swidget, tw,
+					       &reply);
+		break;
+	case snd_soc_dapm_aif_out:
+		ret = sof_widget_load_pcm(scomp, index, swidget,
+					  SOF_IPC_STREAM_CAPTURE, tw, &reply);
+		break;
+	case snd_soc_dapm_aif_in:
+		ret = sof_widget_load_pcm(scomp, index, swidget,
+					  SOF_IPC_STREAM_PLAYBACK, tw, &reply);
+		break;
+	case snd_soc_dapm_src:
+		ret = sof_widget_load_src(scomp, index, swidget, tw, &reply);
+		break;
+	case snd_soc_dapm_siggen:
+		ret = sof_widget_load_siggen(scomp, index, swidget, tw, &reply);
+		break;
+	case snd_soc_dapm_effect:
+		ret = sof_widget_load_process(scomp, index, swidget, tw,
+					      &reply);
+		break;
+	case snd_soc_dapm_mux:
+	case snd_soc_dapm_demux:
+		ret = sof_widget_load_mux(scomp, index, swidget, tw, &reply);
+		break;
+	case snd_soc_dapm_switch:
+	case snd_soc_dapm_dai_link:
+	case snd_soc_dapm_kcontrol:
+	default:
+		dev_warn(sdev->dev, "warning: widget type %d name %s not handled\n",
+			 swidget->id, tw->name);
+		break;
+	}
+
+	/* check IPC reply */
+	if (ret < 0 || reply.rhdr.error < 0) {
+		dev_err(sdev->dev,
+			"error: DSP failed to add widget id %d type %d name : %s stream %s reply %d\n",
+			tw->shift, swidget->id, tw->name,
+			strnlen(tw->sname, SNDRV_CTL_ELEM_ID_NAME_MAXLEN) > 0
+				? tw->sname : "none", reply.rhdr.error);
+		kfree(swidget);
+		return ret;
+	}
+
+	/* bind widget to external event */
+	if (tw->event_type) {
+		ret = sof_widget_bind_event(sdev, swidget,
+					    le16_to_cpu(tw->event_type));
+		if (ret) {
+			dev_err(sdev->dev, "error: widget event binding failed\n");
+			kfree(swidget->private);
+			kfree(swidget);
+			return ret;
+		}
+	}
+
+	w->dobj.private = swidget;
+	list_add(&swidget->list, &sdev->widget_list);
+	return ret;
+}
+
+static int sof_route_unload(struct snd_soc_component *scomp,
+			    struct snd_soc_dobj *dobj)
+{
+	struct snd_sof_route *sroute;
+
+	sroute = dobj->private;
+	if (!sroute)
+		return 0;
+
+	/* free sroute and its private data */
+	kfree(sroute->private);
+	list_del(&sroute->list);
+	kfree(sroute);
+
+	return 0;
+}
+
+static int sof_widget_unload(struct snd_soc_component *scomp,
+			     struct snd_soc_dobj *dobj)
+{
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
+	const struct snd_kcontrol_new *kc;
+	struct snd_soc_dapm_widget *widget;
+	struct sof_ipc_pipe_new *pipeline;
+	struct snd_sof_control *scontrol;
+	struct snd_sof_widget *swidget;
+	struct soc_mixer_control *sm;
+	struct soc_bytes_ext *sbe;
+	struct snd_sof_dai *dai;
+	struct soc_enum *se;
+	int ret = 0;
+	int i;
+
+	swidget = dobj->private;
+	if (!swidget)
+		return 0;
+
+	widget = swidget->widget;
+
+	switch (swidget->id) {
+	case snd_soc_dapm_dai_in:
+	case snd_soc_dapm_dai_out:
+		dai = swidget->private;
+
+		if (dai) {
+			/* free dai config */
+			kfree(dai->dai_config);
+			list_del(&dai->list);
+		}
+		break;
+	case snd_soc_dapm_scheduler:
+
+		/* power down the pipeline schedule core */
+		pipeline = swidget->private;
+		ret = snd_sof_dsp_core_power_down(sdev, 1 << pipeline->core);
+		if (ret < 0)
+			dev_err(sdev->dev, "error: powering down pipeline schedule core %d\n",
+				pipeline->core);
+
+		/* update enabled cores mask */
+		sdev->enabled_cores_mask &= ~(1 << pipeline->core);
+
+		break;
+	default:
+		break;
+	}
+	for (i = 0; i < widget->num_kcontrols; i++) {
+		kc = &widget->kcontrol_news[i];
+		switch (dobj->widget.kcontrol_type) {
+		case SND_SOC_TPLG_TYPE_MIXER:
+			sm = (struct soc_mixer_control *)kc->private_value;
+			scontrol = sm->dobj.private;
+			if (sm->max > 1)
+				kfree(scontrol->volume_table);
+			break;
+		case SND_SOC_TPLG_TYPE_ENUM:
+			se = (struct soc_enum *)kc->private_value;
+			scontrol = se->dobj.private;
+			break;
+		case SND_SOC_TPLG_TYPE_BYTES:
+			sbe = (struct soc_bytes_ext *)kc->private_value;
+			scontrol = sbe->dobj.private;
+			break;
+		default:
+			dev_warn(sdev->dev, "unsupported kcontrol_type\n");
+			goto out;
+		}
+		kfree(scontrol->control_data);
+		list_del(&scontrol->list);
+		kfree(scontrol);
+	}
+
+out:
+	/* free private value */
+	kfree(swidget->private);
+
+	/* remove and free swidget object */
+	list_del(&swidget->list);
+	kfree(swidget);
+
+	return ret;
+}
+
+/*
+ * DAI HW configuration.
+ */
+
+/* FE DAI - used for any driver specific init */
+static int sof_dai_load(struct snd_soc_component *scomp, int index,
+			struct snd_soc_dai_driver *dai_drv,
+			struct snd_soc_tplg_pcm *pcm, struct snd_soc_dai *dai)
+{
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
+	struct snd_soc_tplg_stream_caps *caps;
+	struct snd_sof_pcm *spcm;
+	int stream = SNDRV_PCM_STREAM_PLAYBACK;
+	int ret = 0;
+
+	/* nothing to do for BEs atm */
+	if (!pcm)
+		return 0;
+
+	spcm = kzalloc(sizeof(*spcm), GFP_KERNEL);
+	if (!spcm)
+		return -ENOMEM;
+
+	spcm->sdev = sdev;
+	spcm->stream[SNDRV_PCM_STREAM_PLAYBACK].comp_id = COMP_ID_UNASSIGNED;
+	spcm->stream[SNDRV_PCM_STREAM_CAPTURE].comp_id = COMP_ID_UNASSIGNED;
+
+	if (pcm) {
+		spcm->pcm = *pcm;
+		dev_dbg(sdev->dev, "tplg: load pcm %s\n", pcm->dai_name);
+	}
+	dai_drv->dobj.private = spcm;
+	list_add(&spcm->list, &sdev->pcm_list);
+
+	/* do we need to allocate playback PCM DMA pages */
+	if (!spcm->pcm.playback)
+		goto capture;
+
+	caps = &spcm->pcm.caps[stream];
+
+	/* allocate playback page table buffer */
+	ret = snd_dma_alloc_pages(SNDRV_DMA_TYPE_DEV, sdev->dev,
+				  PAGE_SIZE, &spcm->stream[stream].page_table);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: can't alloc page table for %s %d\n",
+			caps->name, ret);
+
+		return ret;
+	}
+
+	/* bind pcm to host comp */
+	ret = spcm_bind(sdev, spcm, stream);
+	if (ret) {
+		dev_err(sdev->dev,
+			"error: can't bind pcm to host\n");
+		goto free_playback_tables;
+	}
+
+capture:
+	stream = SNDRV_PCM_STREAM_CAPTURE;
+
+	/* do we need to allocate capture PCM DMA pages */
+	if (!spcm->pcm.capture)
+		return ret;
+
+	caps = &spcm->pcm.caps[stream];
+
+	/* allocate capture page table buffer */
+	ret = snd_dma_alloc_pages(SNDRV_DMA_TYPE_DEV, sdev->dev,
+				  PAGE_SIZE, &spcm->stream[stream].page_table);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: can't alloc page table for %s %d\n",
+			caps->name, ret);
+		goto free_playback_tables;
+	}
+
+	/* bind pcm to host comp */
+	ret = spcm_bind(sdev, spcm, stream);
+	if (ret) {
+		dev_err(sdev->dev,
+			"error: can't bind pcm to host\n");
+		snd_dma_free_pages(&spcm->stream[stream].page_table);
+		goto free_playback_tables;
+	}
+
+	return ret;
+
+free_playback_tables:
+	if (spcm->pcm.playback)
+		snd_dma_free_pages(&spcm->stream[SNDRV_PCM_STREAM_PLAYBACK].page_table);
+
+	return ret;
+}
+
+static int sof_dai_unload(struct snd_soc_component *scomp,
+			  struct snd_soc_dobj *dobj)
+{
+	struct snd_sof_pcm *spcm = dobj->private;
+
+	/* free PCM DMA pages */
+	if (spcm->pcm.playback)
+		snd_dma_free_pages(&spcm->stream[SNDRV_PCM_STREAM_PLAYBACK].page_table);
+
+	if (spcm->pcm.capture)
+		snd_dma_free_pages(&spcm->stream[SNDRV_PCM_STREAM_CAPTURE].page_table);
+
+	/* remove from list and free spcm */
+	list_del(&spcm->list);
+	kfree(spcm);
+
+	return 0;
+}
+
+static void sof_dai_set_format(struct snd_soc_tplg_hw_config *hw_config,
+			       struct sof_ipc_dai_config *config)
+{
+	/* clock directions wrt codec */
+	if (hw_config->bclk_master == SND_SOC_TPLG_BCLK_CM) {
+		/* codec is bclk master */
+		if (hw_config->fsync_master == SND_SOC_TPLG_FSYNC_CM)
+			config->format |= SOF_DAI_FMT_CBM_CFM;
+		else
+			config->format |= SOF_DAI_FMT_CBM_CFS;
+	} else {
+		/* codec is bclk slave */
+		if (hw_config->fsync_master == SND_SOC_TPLG_FSYNC_CM)
+			config->format |= SOF_DAI_FMT_CBS_CFM;
+		else
+			config->format |= SOF_DAI_FMT_CBS_CFS;
+	}
+
+	/* inverted clocks ? */
+	if (hw_config->invert_bclk) {
+		if (hw_config->invert_fsync)
+			config->format |= SOF_DAI_FMT_IB_IF;
+		else
+			config->format |= SOF_DAI_FMT_IB_NF;
+	} else {
+		if (hw_config->invert_fsync)
+			config->format |= SOF_DAI_FMT_NB_IF;
+		else
+			config->format |= SOF_DAI_FMT_NB_NF;
+	}
+}
+
+/* set config for all DAI's with name matching the link name */
+static int sof_set_dai_config(struct snd_sof_dev *sdev, u32 size,
+			      struct snd_soc_dai_link *link,
+			      struct sof_ipc_dai_config *config)
+{
+	struct snd_sof_dai *dai;
+	int found = 0;
+
+	list_for_each_entry(dai, &sdev->dai_list, list) {
+		if (!dai->name)
+			continue;
+
+		if (strcmp(link->name, dai->name) == 0) {
+			dai->dai_config = kmemdup(config, size, GFP_KERNEL);
+			if (!dai->dai_config)
+				return -ENOMEM;
+
+			found = 1;
+		}
+	}
+
+	/*
+	 * machine driver may define a dai link with playback and capture
+	 * dai enabled, but the dai link in topology would support both, one
+	 * or none of them. Here print a warning message to notify user
+	 */
+	if (!found) {
+		dev_warn(sdev->dev, "warning: failed to find dai for dai link %s",
+			 link->name);
+	}
+
+	return 0;
+}
+
+static int sof_link_ssp_load(struct snd_soc_component *scomp, int index,
+			     struct snd_soc_dai_link *link,
+			     struct snd_soc_tplg_link_config *cfg,
+			     struct snd_soc_tplg_hw_config *hw_config,
+			     struct sof_ipc_dai_config *config)
+{
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
+	struct snd_soc_tplg_private *private = &cfg->priv;
+	struct sof_ipc_reply reply;
+	u32 size = sizeof(*config);
+	int ret;
+
+	/* handle master/slave and inverted clocks */
+	sof_dai_set_format(hw_config, config);
+
+	/* init IPC */
+	memset(&config->ssp, 0, sizeof(struct sof_ipc_dai_ssp_params));
+	config->hdr.size = size;
+
+	ret = sof_parse_tokens(scomp, &config->ssp, ssp_tokens,
+			       ARRAY_SIZE(ssp_tokens), private->array,
+			       le32_to_cpu(private->size));
+	if (ret != 0) {
+		dev_err(sdev->dev, "error: parse ssp tokens failed %d\n",
+			le32_to_cpu(private->size));
+		return ret;
+	}
+
+	config->ssp.mclk_rate = le32_to_cpu(hw_config->mclk_rate);
+	config->ssp.bclk_rate = le32_to_cpu(hw_config->bclk_rate);
+	config->ssp.fsync_rate = le32_to_cpu(hw_config->fsync_rate);
+	config->ssp.tdm_slots = le32_to_cpu(hw_config->tdm_slots);
+	config->ssp.tdm_slot_width = le32_to_cpu(hw_config->tdm_slot_width);
+	config->ssp.mclk_direction = hw_config->mclk_direction;
+	config->ssp.rx_slots = le32_to_cpu(hw_config->rx_slots);
+	config->ssp.tx_slots = le32_to_cpu(hw_config->tx_slots);
+
+	dev_dbg(sdev->dev, "tplg: config SSP%d fmt 0x%x mclk %d bclk %d fclk %d width (%d)%d slots %d mclk id %d quirks %d\n",
+		config->dai_index, config->format,
+		config->ssp.mclk_rate, config->ssp.bclk_rate,
+		config->ssp.fsync_rate, config->ssp.sample_valid_bits,
+		config->ssp.tdm_slot_width, config->ssp.tdm_slots,
+		config->ssp.mclk_id, config->ssp.quirks);
+
+	/* validate SSP fsync rate and channel count */
+	if (config->ssp.fsync_rate < 8000 || config->ssp.fsync_rate > 192000) {
+		dev_err(sdev->dev, "error: invalid fsync rate for SSP%d\n",
+			config->dai_index);
+		return -EINVAL;
+	}
+
+	if (config->ssp.tdm_slots < 1 || config->ssp.tdm_slots > 8) {
+		dev_err(sdev->dev, "error: invalid channel count for SSP%d\n",
+			config->dai_index);
+		return -EINVAL;
+	}
+
+	/* send message to DSP */
+	ret = sof_ipc_tx_message(sdev->ipc,
+				 config->hdr.cmd, config, size, &reply,
+				 sizeof(reply));
+
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: failed to set DAI config for SSP%d\n",
+			config->dai_index);
+		return ret;
+	}
+
+	/* set config for all DAI's with name matching the link name */
+	ret = sof_set_dai_config(sdev, size, link, config);
+	if (ret < 0)
+		dev_err(sdev->dev, "error: failed to save DAI config for SSP%d\n",
+			config->dai_index);
+
+	return ret;
+}
+
+static int sof_link_dmic_load(struct snd_soc_component *scomp, int index,
+			      struct snd_soc_dai_link *link,
+			      struct snd_soc_tplg_link_config *cfg,
+			      struct snd_soc_tplg_hw_config *hw_config,
+			      struct sof_ipc_dai_config *config)
+{
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
+	struct snd_soc_tplg_private *private = &cfg->priv;
+	struct sof_ipc_dai_config *ipc_config;
+	struct sof_ipc_reply reply;
+	struct sof_ipc_fw_ready *ready = &sdev->fw_ready;
+	struct sof_ipc_fw_version *v = &ready->version;
+	u32 size;
+	int ret, j;
+
+	/*
+	 * config is only used for the common params in dmic_params structure
+	 * that does not include the PDM controller config array
+	 * Set the common params to 0.
+	 */
+	memset(&config->dmic, 0, sizeof(struct sof_ipc_dai_dmic_params));
+
+	/* get DMIC tokens */
+	ret = sof_parse_tokens(scomp, &config->dmic, dmic_tokens,
+			       ARRAY_SIZE(dmic_tokens), private->array,
+			       le32_to_cpu(private->size));
+	if (ret != 0) {
+		dev_err(sdev->dev, "error: parse dmic tokens failed %d\n",
+			le32_to_cpu(private->size));
+		return ret;
+	}
+
+	/*
+	 * allocate memory for dmic dai config accounting for the
+	 * variable number of active pdm controllers
+	 * This will be the ipc payload for setting dai config
+	 */
+	size = sizeof(*config) + sizeof(struct sof_ipc_dai_dmic_pdm_ctrl) *
+					config->dmic.num_pdm_active;
+
+	ipc_config = kzalloc(size, GFP_KERNEL);
+	if (!ipc_config)
+		return -ENOMEM;
+
+	/* copy the common dai config and dmic params */
+	memcpy(ipc_config, config, sizeof(*config));
+
+	/*
+	 * alloc memory for private member
+	 * Used to track the pdm config array index currently being parsed
+	 */
+	sdev->private = kzalloc(sizeof(u32), GFP_KERNEL);
+	if (!sdev->private) {
+		kfree(ipc_config);
+		return -ENOMEM;
+	}
+
+	/* get DMIC PDM tokens */
+	ret = sof_parse_tokens(scomp, &ipc_config->dmic.pdm[0], dmic_pdm_tokens,
+			       ARRAY_SIZE(dmic_pdm_tokens), private->array,
+			       le32_to_cpu(private->size));
+	if (ret != 0) {
+		dev_err(sdev->dev, "error: parse dmic pdm tokens failed %d\n",
+			le32_to_cpu(private->size));
+		goto err;
+	}
+
+	/* set IPC header size */
+	ipc_config->hdr.size = size;
+
+	/* debug messages */
+	dev_dbg(sdev->dev, "tplg: config DMIC%d driver version %d\n",
+		ipc_config->dai_index, ipc_config->dmic.driver_ipc_version);
+	dev_dbg(sdev->dev, "pdmclk_min %d pdm_clkmax %d duty_min %hd\n",
+		ipc_config->dmic.pdmclk_min, ipc_config->dmic.pdmclk_max,
+		ipc_config->dmic.duty_min);
+	dev_dbg(sdev->dev, "duty_max %hd fifo_fs %d num_pdms active %d\n",
+		ipc_config->dmic.duty_max, ipc_config->dmic.fifo_fs,
+		ipc_config->dmic.num_pdm_active);
+	dev_dbg(sdev->dev, "fifo word length %hd\n",
+		ipc_config->dmic.fifo_bits);
+
+	for (j = 0; j < ipc_config->dmic.num_pdm_active; j++) {
+		dev_dbg(sdev->dev, "pdm %hd mic a %hd mic b %hd\n",
+			ipc_config->dmic.pdm[j].id,
+			ipc_config->dmic.pdm[j].enable_mic_a,
+			ipc_config->dmic.pdm[j].enable_mic_b);
+		dev_dbg(sdev->dev, "pdm %hd polarity a %hd polarity b %hd\n",
+			ipc_config->dmic.pdm[j].id,
+			ipc_config->dmic.pdm[j].polarity_mic_a,
+			ipc_config->dmic.pdm[j].polarity_mic_b);
+		dev_dbg(sdev->dev, "pdm %hd clk_edge %hd skew %hd\n",
+			ipc_config->dmic.pdm[j].id,
+			ipc_config->dmic.pdm[j].clk_edge,
+			ipc_config->dmic.pdm[j].skew);
+	}
+
+	if (SOF_ABI_VER(v->major, v->minor, v->micro) < SOF_ABI_VER(3, 0, 1)) {
+		/* this takes care of backwards compatible handling of fifo_bits_b */
+		ipc_config->dmic.reserved_2 = ipc_config->dmic.fifo_bits;
+	}
+
+	/* send message to DSP */
+	ret = sof_ipc_tx_message(sdev->ipc,
+				 ipc_config->hdr.cmd, ipc_config, size, &reply,
+				 sizeof(reply));
+
+	if (ret < 0) {
+		dev_err(sdev->dev,
+			"error: failed to set DAI config for DMIC%d\n",
+			config->dai_index);
+		goto err;
+	}
+
+	/* set config for all DAI's with name matching the link name */
+	ret = sof_set_dai_config(sdev, size, link, ipc_config);
+	if (ret < 0)
+		dev_err(sdev->dev, "error: failed to save DAI config for DMIC%d\n",
+			config->dai_index);
+
+err:
+	kfree(sdev->private);
+	kfree(ipc_config);
+
+	return ret;
+}
+
+/*
+ * for hda link, playback and capture are supported by different dai
+ * in FW. Here get the dai_index, set dma channel of each dai
+ * and send config to FW. In FW, each dai sets config by dai_index
+ */
+static int sof_link_hda_process(struct snd_sof_dev *sdev,
+				struct snd_soc_dai_link *link,
+				struct sof_ipc_dai_config *config,
+				int tx_slot,
+				int rx_slot)
+{
+	struct sof_ipc_reply reply;
+	u32 size = sizeof(*config);
+	struct snd_sof_dai *sof_dai;
+	int found = 0;
+	int ret;
+
+	list_for_each_entry(sof_dai, &sdev->dai_list, list) {
+		if (!sof_dai->name)
+			continue;
+
+		if (strcmp(link->name, sof_dai->name) == 0) {
+			if (sof_dai->comp_dai.direction ==
+			    SNDRV_PCM_STREAM_PLAYBACK) {
+				if (!link->dpcm_playback)
+					return -EINVAL;
+
+				config->hda.link_dma_ch = tx_slot;
+			} else {
+				if (!link->dpcm_capture)
+					return -EINVAL;
+
+				config->hda.link_dma_ch = rx_slot;
+			}
+
+			config->dai_index = sof_dai->comp_dai.dai_index;
+			found = 1;
+
+			/* save config in dai component */
+			sof_dai->dai_config = kmemdup(config, size, GFP_KERNEL);
+			if (!sof_dai->dai_config)
+				return -ENOMEM;
+
+			/* send message to DSP */
+			ret = sof_ipc_tx_message(sdev->ipc,
+						 config->hdr.cmd, config, size,
+						 &reply, sizeof(reply));
+
+			if (ret < 0) {
+				dev_err(sdev->dev, "error: failed to set DAI config for direction:%d of HDA dai %d\n",
+					sof_dai->comp_dai.direction,
+					config->dai_index);
+
+				return ret;
+			}
+		}
+	}
+
+	/*
+	 * machine driver may define a dai link with playback and capture
+	 * dai enabled, but the dai link in topology would support both, one
+	 * or none of them. Here print a warning message to notify user
+	 */
+	if (!found) {
+		dev_warn(sdev->dev, "warning: failed to find dai for dai link %s",
+			 link->name);
+	}
+
+	return 0;
+}
+
+static int sof_link_hda_load(struct snd_soc_component *scomp, int index,
+			     struct snd_soc_dai_link *link,
+			     struct snd_soc_tplg_link_config *cfg,
+			     struct snd_soc_tplg_hw_config *hw_config,
+			     struct sof_ipc_dai_config *config)
+{
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
+	struct snd_soc_dai_link_component dai_component;
+	struct snd_soc_tplg_private *private = &cfg->priv;
+	struct snd_soc_dai *dai;
+	u32 size = sizeof(*config);
+	u32 tx_num = 0;
+	u32 tx_slot = 0;
+	u32 rx_num = 0;
+	u32 rx_slot = 0;
+	int ret;
+
+	/* init IPC */
+	memset(&dai_component, 0, sizeof(dai_component));
+	memset(&config->hda, 0, sizeof(struct sof_ipc_dai_hda_params));
+	config->hdr.size = size;
+
+	/* get any bespoke DAI tokens */
+	ret = sof_parse_tokens(scomp, config, hda_tokens,
+			       ARRAY_SIZE(hda_tokens), private->array,
+			       le32_to_cpu(private->size));
+	if (ret != 0) {
+		dev_err(sdev->dev, "error: parse hda tokens failed %d\n",
+			le32_to_cpu(private->size));
+		return ret;
+	}
+
+	dai_component.dai_name = link->cpu_dai_name;
+	dai = snd_soc_find_dai(&dai_component);
+	if (!dai) {
+		dev_err(sdev->dev, "error: failed to find dai %s in %s",
+			dai_component.dai_name, __func__);
+		return -EINVAL;
+	}
+
+	if (link->dpcm_playback)
+		tx_num = 1;
+
+	if (link->dpcm_capture)
+		rx_num = 1;
+
+	ret = snd_soc_dai_get_channel_map(dai, &tx_num, &tx_slot,
+					  &rx_num, &rx_slot);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: failed to get dma channel for HDA%d\n",
+			config->dai_index);
+
+		return ret;
+	}
+
+	ret = sof_link_hda_process(sdev, link, config, tx_slot, rx_slot);
+	if (ret < 0)
+		dev_err(sdev->dev, "error: failed to process hda dai link %s",
+			link->name);
+
+	return ret;
+}
+
+/* DAI link - used for any driver specific init */
+static int sof_link_load(struct snd_soc_component *scomp, int index,
+			 struct snd_soc_dai_link *link,
+			 struct snd_soc_tplg_link_config *cfg)
+{
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
+	struct snd_soc_tplg_private *private = &cfg->priv;
+	struct sof_ipc_dai_config config;
+	struct snd_soc_tplg_hw_config *hw_config;
+	int num_hw_configs;
+	int ret;
+	int i = 0;
+
+	link->platform_name = dev_name(sdev->dev);
+
+	/*
+	 * Set nonatomic property for FE dai links as their trigger action
+	 * involves IPC's.
+	 */
+	if (!link->no_pcm) {
+		link->nonatomic = true;
+
+		/* nothing more to do for FE dai links */
+		return 0;
+	}
+
+	/* check we have some tokens - we need at least DAI type */
+	if (le32_to_cpu(private->size) == 0) {
+		dev_err(sdev->dev, "error: expected tokens for DAI, none found\n");
+		return -EINVAL;
+	}
+
+	/* Send BE DAI link configurations to DSP */
+	memset(&config, 0, sizeof(config));
+
+	/* get any common DAI tokens */
+	ret = sof_parse_tokens(scomp, &config, dai_link_tokens,
+			       ARRAY_SIZE(dai_link_tokens), private->array,
+			       le32_to_cpu(private->size));
+	if (ret != 0) {
+		dev_err(sdev->dev, "error: parse link tokens failed %d\n",
+			le32_to_cpu(private->size));
+		return ret;
+	}
+
+	/*
+	 * DAI links are expected to have at least 1 hw_config.
+	 * But some older topologies might have no hw_config for HDA dai links.
+	 */
+	num_hw_configs = le32_to_cpu(cfg->num_hw_configs);
+	if (!num_hw_configs) {
+		if (config.type != SOF_DAI_INTEL_HDA) {
+			dev_err(sdev->dev, "error: unexpected DAI config count %d!\n",
+				le32_to_cpu(cfg->num_hw_configs));
+			return -EINVAL;
+		}
+	} else {
+		dev_dbg(sdev->dev, "tplg: %d hw_configs found, default id: %d!\n",
+			cfg->num_hw_configs, le32_to_cpu(cfg->default_hw_config_id));
+
+		for (i = 0; i < num_hw_configs; i++) {
+			if (cfg->hw_config[i].id == cfg->default_hw_config_id)
+				break;
+		}
+
+		if (i == num_hw_configs) {
+			dev_err(sdev->dev, "error: default hw_config id: %d not found!\n",
+				le32_to_cpu(cfg->default_hw_config_id));
+			return -EINVAL;
+		}
+	}
+
+	/* configure dai IPC message */
+	hw_config = &cfg->hw_config[i];
+
+	config.hdr.cmd = SOF_IPC_GLB_DAI_MSG | SOF_IPC_DAI_CONFIG;
+	config.format = le32_to_cpu(hw_config->fmt);
+
+	/* now load DAI specific data and send IPC - type comes from token */
+	switch (config.type) {
+	case SOF_DAI_INTEL_SSP:
+		ret = sof_link_ssp_load(scomp, index, link, cfg, hw_config,
+					&config);
+		break;
+	case SOF_DAI_INTEL_DMIC:
+		ret = sof_link_dmic_load(scomp, index, link, cfg, hw_config,
+					 &config);
+		break;
+	case SOF_DAI_INTEL_HDA:
+		ret = sof_link_hda_load(scomp, index, link, cfg, hw_config,
+					&config);
+		break;
+	default:
+		dev_err(sdev->dev, "error: invalid DAI type %d\n", config.type);
+		ret = -EINVAL;
+		break;
+	}
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int sof_link_hda_unload(struct snd_sof_dev *sdev,
+			       struct snd_soc_dai_link *link)
+{
+	struct snd_soc_dai_link_component dai_component;
+	struct snd_soc_dai *dai;
+	int ret = 0;
+
+	memset(&dai_component, 0, sizeof(dai_component));
+	dai_component.dai_name = link->cpu_dai_name;
+	dai = snd_soc_find_dai(&dai_component);
+	if (!dai) {
+		dev_err(sdev->dev, "error: failed to find dai %s in %s",
+			dai_component.dai_name, __func__);
+		return -EINVAL;
+	}
+
+	/*
+	 * FIXME: this call to hw_free is mainly to release the link DMA ID.
+	 * This is abusing the API and handling SOC internals is not
+	 * recommended. This part will be reworked.
+	 */
+	if (dai->driver->ops->hw_free)
+		ret = dai->driver->ops->hw_free(NULL, dai);
+	if (ret < 0)
+		dev_err(sdev->dev, "error: failed to free hda resource for %s\n",
+			link->name);
+
+	return ret;
+}
+
+static int sof_link_unload(struct snd_soc_component *scomp,
+			   struct snd_soc_dobj *dobj)
+{
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
+	struct snd_soc_dai_link *link =
+		container_of(dobj, struct snd_soc_dai_link, dobj);
+
+	struct snd_sof_dai *sof_dai;
+	int ret = 0;
+
+	/* only BE link is loaded by sof */
+	if (!link->no_pcm)
+		return 0;
+
+	list_for_each_entry(sof_dai, &sdev->dai_list, list) {
+		if (!sof_dai->name)
+			continue;
+
+		if (strcmp(link->name, sof_dai->name) == 0)
+			goto found;
+	}
+
+	dev_err(sdev->dev, "error: failed to find dai %s in %s",
+		link->name, __func__);
+	return -EINVAL;
+found:
+
+	switch (sof_dai->dai_config->type) {
+	case SOF_DAI_INTEL_SSP:
+	case SOF_DAI_INTEL_DMIC:
+		/* no resource needs to be released for SSP and DMIC */
+		break;
+	case SOF_DAI_INTEL_HDA:
+		ret = sof_link_hda_unload(sdev, link);
+		break;
+	default:
+		dev_err(sdev->dev, "error: invalid DAI type %d\n",
+			sof_dai->dai_config->type);
+		ret = -EINVAL;
+		break;
+	}
+
+	return ret;
+}
+
+/* DAI link - used for any driver specific init */
+static int sof_route_load(struct snd_soc_component *scomp, int index,
+			  struct snd_soc_dapm_route *route)
+{
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
+	struct sof_ipc_pipe_comp_connect *connect;
+	struct snd_sof_widget *source_swidget, *sink_swidget;
+	struct snd_soc_dobj *dobj = &route->dobj;
+	struct snd_sof_route *sroute;
+	struct sof_ipc_reply reply;
+	int ret = 0;
+
+	/* allocate memory for sroute and connect */
+	sroute = kzalloc(sizeof(*sroute), GFP_KERNEL);
+	if (!sroute)
+		return -ENOMEM;
+
+	sroute->sdev = sdev;
+
+	connect = kzalloc(sizeof(*connect), GFP_KERNEL);
+	if (!connect) {
+		kfree(sroute);
+		return -ENOMEM;
+	}
+
+	connect->hdr.size = sizeof(*connect);
+	connect->hdr.cmd = SOF_IPC_GLB_TPLG_MSG | SOF_IPC_TPLG_COMP_CONNECT;
+
+	dev_dbg(sdev->dev, "sink %s control %s source %s\n",
+		route->sink, route->control ? route->control : "none",
+		route->source);
+
+	/* source component */
+	source_swidget = snd_sof_find_swidget(sdev, (char *)route->source);
+	if (!source_swidget) {
+		dev_err(sdev->dev, "error: source %s not found\n",
+			route->source);
+		ret = -EINVAL;
+		goto err;
+	}
+
+	/*
+	 * Virtual widgets of type output/out_drv may be added in topology
+	 * for compatibility. These are not handled by the FW.
+	 * So, don't send routes whose source/sink widget is of such types
+	 * to the DSP.
+	 */
+	if (source_swidget->id == snd_soc_dapm_out_drv ||
+	    source_swidget->id == snd_soc_dapm_output)
+		goto err;
+
+	connect->source_id = source_swidget->comp_id;
+
+	/* sink component */
+	sink_swidget = snd_sof_find_swidget(sdev, (char *)route->sink);
+	if (!sink_swidget) {
+		dev_err(sdev->dev, "error: sink %s not found\n",
+			route->sink);
+		ret = -EINVAL;
+		goto err;
+	}
+
+	/*
+	 * Don't send routes whose sink widget is of type
+	 * output or out_drv to the DSP
+	 */
+	if (sink_swidget->id == snd_soc_dapm_out_drv ||
+	    sink_swidget->id == snd_soc_dapm_output)
+		goto err;
+
+	connect->sink_id = sink_swidget->comp_id;
+
+	/*
+	 * For virtual routes, both sink and source are not
+	 * buffer. Since only buffer linked to component is supported by
+	 * FW, others are reported as error, add check in route function,
+	 * do not send it to FW when both source and sink are not buffer
+	 */
+	if (source_swidget->id != snd_soc_dapm_buffer &&
+	    sink_swidget->id != snd_soc_dapm_buffer) {
+		dev_dbg(sdev->dev, "warning: neither Linked source component %s nor sink component %s is of buffer type, ignoring link\n",
+			route->source, route->sink);
+		ret = 0;
+		goto err;
+	} else {
+		ret = sof_ipc_tx_message(sdev->ipc,
+					 connect->hdr.cmd,
+					 connect, sizeof(*connect),
+					 &reply, sizeof(reply));
+
+		/* check IPC return value */
+		if (ret < 0) {
+			dev_err(sdev->dev, "error: failed to add route sink %s control %s source %s\n",
+				route->sink,
+				route->control ? route->control : "none",
+				route->source);
+			goto err;
+		}
+
+		/* check IPC reply */
+		if (reply.error < 0) {
+			dev_err(sdev->dev, "error: DSP failed to add route sink %s control %s source %s result %d\n",
+				route->sink,
+				route->control ? route->control : "none",
+				route->source, reply.error);
+			ret = reply.error;
+			goto err;
+		}
+
+		sroute->route = route;
+		dobj->private = sroute;
+		sroute->private = connect;
+
+		/* add route to route list */
+		list_add(&sroute->list, &sdev->route_list);
+
+		return ret;
+	}
+
+err:
+	kfree(connect);
+	kfree(sroute);
+	return ret;
+}
+
+int snd_sof_complete_pipeline(struct snd_sof_dev *sdev,
+			      struct snd_sof_widget *swidget)
+{
+	struct sof_ipc_pipe_ready ready;
+	struct sof_ipc_reply reply;
+	int ret;
+
+	dev_dbg(sdev->dev, "tplg: complete pipeline %s id %d\n",
+		swidget->widget->name, swidget->comp_id);
+
+	memset(&ready, 0, sizeof(ready));
+	ready.hdr.size = sizeof(ready);
+	ready.hdr.cmd = SOF_IPC_GLB_TPLG_MSG | SOF_IPC_TPLG_PIPE_COMPLETE;
+	ready.comp_id = swidget->comp_id;
+
+	ret = sof_ipc_tx_message(sdev->ipc,
+				 ready.hdr.cmd, &ready, sizeof(ready), &reply,
+				 sizeof(reply));
+	if (ret < 0)
+		return ret;
+	return 1;
+}
+
+/* completion - called at completion of firmware loading */
+static void sof_complete(struct snd_soc_component *scomp)
+{
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
+	struct snd_sof_widget *swidget;
+
+	/* some widget types require completion notificattion */
+	list_for_each_entry(swidget, &sdev->widget_list, list) {
+		if (swidget->complete)
+			continue;
+
+		switch (swidget->id) {
+		case snd_soc_dapm_scheduler:
+			swidget->complete =
+				snd_sof_complete_pipeline(sdev, swidget);
+			break;
+		default:
+			break;
+		}
+	}
+}
+
+/* manifest - optional to inform component of manifest */
+static int sof_manifest(struct snd_soc_component *scomp, int index,
+			struct snd_soc_tplg_manifest *man)
+{
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
+	u32 size;
+	u32 abi_version;
+
+	size = le32_to_cpu(man->priv.size);
+
+	/* backward compatible with tplg without ABI info */
+	if (!size) {
+		dev_dbg(sdev->dev, "No topology ABI info\n");
+		return 0;
+	}
+
+	if (size != SOF_TPLG_ABI_SIZE) {
+		dev_err(sdev->dev, "error: invalid topology ABI size\n");
+		return -EINVAL;
+	}
+
+	dev_info(sdev->dev,
+		 "Topology: ABI %d:%d:%d Kernel ABI %d:%d:%d\n",
+		 man->priv.data[0], man->priv.data[1],
+		 man->priv.data[2], SOF_ABI_MAJOR, SOF_ABI_MINOR,
+		 SOF_ABI_PATCH);
+
+	abi_version = SOF_ABI_VER(man->priv.data[0],
+				  man->priv.data[1],
+				  man->priv.data[2]);
+
+	if (SOF_ABI_VERSION_INCOMPATIBLE(SOF_ABI_VERSION, abi_version)) {
+		dev_err(sdev->dev, "error: incompatible topology ABI version\n");
+		return -EINVAL;
+	}
+
+	if (abi_version > SOF_ABI_VERSION) {
+		if (!IS_ENABLED(CONFIG_SND_SOC_SOF_STRICT_ABI_CHECKS)) {
+			dev_warn(sdev->dev, "warn: topology ABI is more recent than kernel\n");
+		} else {
+			dev_err(sdev->dev, "error: topology ABI is more recent than kernel\n");
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+/* vendor specific kcontrol handlers available for binding */
+static const struct snd_soc_tplg_kcontrol_ops sof_io_ops[] = {
+	{SOF_TPLG_KCTL_VOL_ID, snd_sof_volume_get, snd_sof_volume_put},
+	{SOF_TPLG_KCTL_BYTES_ID, snd_sof_bytes_get, snd_sof_bytes_put},
+	{SOF_TPLG_KCTL_ENUM_ID, snd_sof_enum_get, snd_sof_enum_put},
+	{SOF_TPLG_KCTL_SWITCH_ID, snd_sof_switch_get, snd_sof_switch_put},
+};
+
+/* vendor specific bytes ext handlers available for binding */
+static const struct snd_soc_tplg_bytes_ext_ops sof_bytes_ext_ops[] = {
+	{SOF_TPLG_KCTL_BYTES_ID, snd_sof_bytes_ext_get, snd_sof_bytes_ext_put},
+};
+
+static struct snd_soc_tplg_ops sof_tplg_ops = {
+	/* external kcontrol init - used for any driver specific init */
+	.control_load	= sof_control_load,
+	.control_unload	= sof_control_unload,
+
+	/* external kcontrol init - used for any driver specific init */
+	.dapm_route_load	= sof_route_load,
+	.dapm_route_unload	= sof_route_unload,
+
+	/* external widget init - used for any driver specific init */
+	/* .widget_load is not currently used */
+	.widget_ready	= sof_widget_ready,
+	.widget_unload	= sof_widget_unload,
+
+	/* FE DAI - used for any driver specific init */
+	.dai_load	= sof_dai_load,
+	.dai_unload	= sof_dai_unload,
+
+	/* DAI link - used for any driver specific init */
+	.link_load	= sof_link_load,
+	.link_unload	= sof_link_unload,
+
+	/* completion - called at completion of firmware loading */
+	.complete	= sof_complete,
+
+	/* manifest - optional to inform component of manifest */
+	.manifest	= sof_manifest,
+
+	/* vendor specific kcontrol handlers available for binding */
+	.io_ops		= sof_io_ops,
+	.io_ops_count	= ARRAY_SIZE(sof_io_ops),
+
+	/* vendor specific bytes ext handlers available for binding */
+	.bytes_ext_ops	= sof_bytes_ext_ops,
+	.bytes_ext_ops_count	= ARRAY_SIZE(sof_bytes_ext_ops),
+};
+
+int snd_sof_init_topology(struct snd_sof_dev *sdev,
+			  struct snd_soc_tplg_ops *ops)
+{
+	/* TODO: support linked list of topologies */
+	sdev->tplg_ops = ops;
+	return 0;
+}
+EXPORT_SYMBOL(snd_sof_init_topology);
+
+int snd_sof_load_topology(struct snd_sof_dev *sdev, const char *file)
+{
+	const struct firmware *fw;
+	int ret;
+
+	dev_dbg(sdev->dev, "loading topology:%s\n", file);
+
+	ret = request_firmware(&fw, file, sdev->dev);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: tplg request firmware %s failed err: %d\n",
+			file, ret);
+		return ret;
+	}
+
+	ret = snd_soc_tplg_component_load(sdev->component,
+					  &sof_tplg_ops, fw,
+					  SND_SOC_TPLG_INDEX_ALL);
+	if (ret < 0) {
+		dev_err(sdev->dev, "error: tplg component load failed %d\n",
+			ret);
+		ret = -EINVAL;
+	}
+
+	release_firmware(fw);
+	return ret;
+}
+EXPORT_SYMBOL(snd_sof_load_topology);
diff --git a/sound/soc/sof/trace.c b/sound/soc/sof/trace.c
new file mode 100644
index 000000000000..d588e4b70fad
--- /dev/null
+++ b/sound/soc/sof/trace.c
@@ -0,0 +1,297 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license.  When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2018 Intel Corporation. All rights reserved.
+//
+// Author: Liam Girdwood <liam.r.girdwood@linux.intel.com>
+//
+
+#include <linux/debugfs.h>
+#include <linux/sched/signal.h>
+#include "sof-priv.h"
+#include "ops.h"
+
+static size_t sof_wait_trace_avail(struct snd_sof_dev *sdev,
+				   loff_t pos, size_t buffer_size)
+{
+	wait_queue_entry_t wait;
+	loff_t host_offset = READ_ONCE(sdev->host_offset);
+
+	/*
+	 * If host offset is less than local pos, it means write pointer of
+	 * host DMA buffer has been wrapped. We should output the trace data
+	 * at the end of host DMA buffer at first.
+	 */
+	if (host_offset < pos)
+		return buffer_size - pos;
+
+	/* If there is available trace data now, it is unnecessary to wait. */
+	if (host_offset > pos)
+		return host_offset - pos;
+
+	/* wait for available trace data from FW */
+	init_waitqueue_entry(&wait, current);
+	set_current_state(TASK_INTERRUPTIBLE);
+	add_wait_queue(&sdev->trace_sleep, &wait);
+
+	if (!signal_pending(current)) {
+		/* set timeout to max value, no error code */
+		schedule_timeout(MAX_SCHEDULE_TIMEOUT);
+	}
+	remove_wait_queue(&sdev->trace_sleep, &wait);
+
+	/* return bytes available for copy */
+	host_offset = READ_ONCE(sdev->host_offset);
+	if (host_offset < pos)
+		return buffer_size - pos;
+
+	return host_offset - pos;
+}
+
+static ssize_t sof_dfsentry_trace_read(struct file *file, char __user *buffer,
+				       size_t count, loff_t *ppos)
+{
+	struct snd_sof_dfsentry *dfse = file->private_data;
+	struct snd_sof_dev *sdev = dfse->sdev;
+	unsigned long rem;
+	loff_t lpos = *ppos;
+	size_t avail, buffer_size = dfse->size;
+	u64 lpos_64;
+
+	/* make sure we know about any failures on the DSP side */
+	sdev->dtrace_error = false;
+
+	/* check pos and count */
+	if (lpos < 0)
+		return -EINVAL;
+	if (!count)
+		return 0;
+
+	/* check for buffer wrap and count overflow */
+	lpos_64 = lpos;
+	lpos = do_div(lpos_64, buffer_size);
+
+	if (count > buffer_size - lpos) /* min() not used to avoid sparse warnings */
+		count = buffer_size - lpos;
+
+	/* get available count based on current host offset */
+	avail = sof_wait_trace_avail(sdev, lpos, buffer_size);
+	if (sdev->dtrace_error) {
+		dev_err(sdev->dev, "error: trace IO error\n");
+		return -EIO;
+	}
+
+	/* make sure count is <= avail */
+	count = avail > count ? count : avail;
+
+	/* copy available trace data to debugfs */
+	rem = copy_to_user(buffer, ((u8 *)(dfse->buf) + lpos), count);
+	if (rem)
+		return -EFAULT;
+
+	*ppos += count;
+
+	/* move debugfs reading position */
+	return count;
+}
+
+static const struct file_operations sof_dfs_trace_fops = {
+	.open = simple_open,
+	.read = sof_dfsentry_trace_read,
+	.llseek = default_llseek,
+};
+
+static int trace_debugfs_create(struct snd_sof_dev *sdev)
+{
+	struct snd_sof_dfsentry *dfse;
+
+	if (!sdev)
+		return -EINVAL;
+
+	dfse = devm_kzalloc(sdev->dev, sizeof(*dfse), GFP_KERNEL);
+	if (!dfse)
+		return -ENOMEM;
+
+	dfse->type = SOF_DFSENTRY_TYPE_BUF;
+	dfse->buf = sdev->dmatb.area;
+	dfse->size = sdev->dmatb.bytes;
+	dfse->sdev = sdev;
+
+	dfse->dfsentry = debugfs_create_file("trace", 0444, sdev->debugfs_root,
+					     dfse, &sof_dfs_trace_fops);
+	if (!dfse->dfsentry) {
+		/* can't rely on debugfs, only log error and keep going */
+		dev_err(sdev->dev,
+			"error: cannot create debugfs entry for trace\n");
+	}
+
+	return 0;
+}
+
+int snd_sof_init_trace_ipc(struct snd_sof_dev *sdev)
+{
+	struct sof_ipc_dma_trace_params params;
+	struct sof_ipc_reply ipc_reply;
+	int ret;
+
+	if (sdev->dtrace_is_enabled || !sdev->dma_trace_pages)
+		return -EINVAL;
+
+	/* set IPC parameters */
+	params.hdr.size = sizeof(params);
+	params.hdr.cmd = SOF_IPC_GLB_TRACE_MSG | SOF_IPC_TRACE_DMA_PARAMS;
+	params.buffer.phy_addr = sdev->dmatp.addr;
+	params.buffer.size = sdev->dmatb.bytes;
+	params.buffer.pages = sdev->dma_trace_pages;
+	params.stream_tag = 0;
+
+	sdev->host_offset = 0;
+
+	ret = snd_sof_dma_trace_init(sdev, &params.stream_tag);
+	if (ret < 0) {
+		dev_err(sdev->dev,
+			"error: fail in snd_sof_dma_trace_init %d\n", ret);
+		return ret;
+	}
+	dev_dbg(sdev->dev, "stream_tag: %d\n", params.stream_tag);
+
+	/* send IPC to the DSP */
+	ret = sof_ipc_tx_message(sdev->ipc,
+				 params.hdr.cmd, &params, sizeof(params),
+				 &ipc_reply, sizeof(ipc_reply));
+	if (ret < 0) {
+		dev_err(sdev->dev,
+			"error: can't set params for DMA for trace %d\n", ret);
+		goto trace_release;
+	}
+
+	ret = snd_sof_dma_trace_trigger(sdev, SNDRV_PCM_TRIGGER_START);
+	if (ret < 0) {
+		dev_err(sdev->dev,
+			"error: snd_sof_dma_trace_trigger: start: %d\n", ret);
+		goto trace_release;
+	}
+
+	sdev->dtrace_is_enabled = true;
+
+	return 0;
+
+trace_release:
+	snd_sof_dma_trace_release(sdev);
+	return ret;
+}
+
+int snd_sof_init_trace(struct snd_sof_dev *sdev)
+{
+	int ret;
+
+	/* set false before start initialization */
+	sdev->dtrace_is_enabled = false;
+
+	/* allocate trace page table buffer */
+	ret = snd_dma_alloc_pages(SNDRV_DMA_TYPE_DEV, sdev->dev,
+				  PAGE_SIZE, &sdev->dmatp);
+	if (ret < 0) {
+		dev_err(sdev->dev,
+			"error: can't alloc page table for trace %d\n", ret);
+		return ret;
+	}
+
+	/* allocate trace data buffer */
+	ret = snd_dma_alloc_pages(SNDRV_DMA_TYPE_DEV_SG, sdev->dev,
+				  DMA_BUF_SIZE_FOR_TRACE, &sdev->dmatb);
+	if (ret < 0) {
+		dev_err(sdev->dev,
+			"error: can't alloc buffer for trace %d\n", ret);
+		goto page_err;
+	}
+
+	/* create compressed page table for audio firmware */
+	ret = snd_sof_create_page_table(sdev, &sdev->dmatb, sdev->dmatp.area,
+					sdev->dmatb.bytes);
+	if (ret < 0)
+		goto table_err;
+
+	sdev->dma_trace_pages = ret;
+	dev_dbg(sdev->dev, "dma_trace_pages: %d\n", sdev->dma_trace_pages);
+
+	if (sdev->first_boot) {
+		ret = trace_debugfs_create(sdev);
+		if (ret < 0)
+			goto table_err;
+	}
+
+	init_waitqueue_head(&sdev->trace_sleep);
+
+	ret = snd_sof_init_trace_ipc(sdev);
+	if (ret < 0)
+		goto table_err;
+
+	return 0;
+table_err:
+	sdev->dma_trace_pages = 0;
+	snd_dma_free_pages(&sdev->dmatb);
+page_err:
+	snd_dma_free_pages(&sdev->dmatp);
+	return ret;
+}
+EXPORT_SYMBOL(snd_sof_init_trace);
+
+int snd_sof_trace_update_pos(struct snd_sof_dev *sdev,
+			     struct sof_ipc_dma_trace_posn *posn)
+{
+	if (sdev->dtrace_is_enabled && sdev->host_offset != posn->host_offset) {
+		sdev->host_offset = posn->host_offset;
+		wake_up(&sdev->trace_sleep);
+	}
+
+	if (posn->overflow != 0)
+		dev_err(sdev->dev,
+			"error: DSP trace buffer overflow %u bytes. Total messages %d\n",
+			posn->overflow, posn->messages);
+
+	return 0;
+}
+
+/* an error has occurred within the DSP that prevents further trace */
+void snd_sof_trace_notify_for_error(struct snd_sof_dev *sdev)
+{
+	if (sdev->dtrace_is_enabled) {
+		dev_err(sdev->dev, "error: waking up any trace sleepers\n");
+		sdev->dtrace_error = true;
+		wake_up(&sdev->trace_sleep);
+	}
+}
+EXPORT_SYMBOL(snd_sof_trace_notify_for_error);
+
+void snd_sof_release_trace(struct snd_sof_dev *sdev)
+{
+	int ret;
+
+	if (!sdev->dtrace_is_enabled)
+		return;
+
+	ret = snd_sof_dma_trace_trigger(sdev, SNDRV_PCM_TRIGGER_STOP);
+	if (ret < 0)
+		dev_err(sdev->dev,
+			"error: snd_sof_dma_trace_trigger: stop: %d\n", ret);
+
+	ret = snd_sof_dma_trace_release(sdev);
+	if (ret < 0)
+		dev_err(sdev->dev,
+			"error: fail in snd_sof_dma_trace_release %d\n", ret);
+
+	sdev->dtrace_is_enabled = false;
+}
+EXPORT_SYMBOL(snd_sof_release_trace);
+
+void snd_sof_free_trace(struct snd_sof_dev *sdev)
+{
+	snd_sof_release_trace(sdev);
+
+	snd_dma_free_pages(&sdev->dmatb);
+	snd_dma_free_pages(&sdev->dmatp);
+}
+EXPORT_SYMBOL(snd_sof_free_trace);
diff --git a/sound/soc/sof/utils.c b/sound/soc/sof/utils.c
new file mode 100644
index 000000000000..2ac4c3da0320
--- /dev/null
+++ b/sound/soc/sof/utils.c
@@ -0,0 +1,112 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license.  When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2018 Intel Corporation. All rights reserved.
+//
+// Author: Keyon Jie <yang.jie@linux.intel.com>
+//
+
+#include <linux/io-64-nonatomic-lo-hi.h>
+#include <linux/platform_device.h>
+#include <sound/soc.h>
+#include <sound/sof.h>
+#include "sof-priv.h"
+
+/*
+ * Register IO
+ *
+ * The sof_io_xyz() wrappers are typically referenced in snd_sof_dsp_ops
+ * structures and cannot be inlined.
+ */
+
+void sof_io_write(struct snd_sof_dev *sdev, void __iomem *addr, u32 value)
+{
+	writel(value, addr);
+}
+EXPORT_SYMBOL(sof_io_write);
+
+u32 sof_io_read(struct snd_sof_dev *sdev, void __iomem *addr)
+{
+	return readl(addr);
+}
+EXPORT_SYMBOL(sof_io_read);
+
+void sof_io_write64(struct snd_sof_dev *sdev, void __iomem *addr, u64 value)
+{
+	writeq(value, addr);
+}
+EXPORT_SYMBOL(sof_io_write64);
+
+u64 sof_io_read64(struct snd_sof_dev *sdev, void __iomem *addr)
+{
+	return readq(addr);
+}
+EXPORT_SYMBOL(sof_io_read64);
+
+/*
+ * IPC Mailbox IO
+ */
+
+void sof_mailbox_write(struct snd_sof_dev *sdev, u32 offset,
+		       void *message, size_t bytes)
+{
+	void __iomem *dest = sdev->bar[sdev->mailbox_bar] + offset;
+
+	memcpy_toio(dest, message, bytes);
+}
+EXPORT_SYMBOL(sof_mailbox_write);
+
+void sof_mailbox_read(struct snd_sof_dev *sdev, u32 offset,
+		      void *message, size_t bytes)
+{
+	void __iomem *src = sdev->bar[sdev->mailbox_bar] + offset;
+
+	memcpy_fromio(message, src, bytes);
+}
+EXPORT_SYMBOL(sof_mailbox_read);
+
+/*
+ * Memory copy.
+ */
+
+void sof_block_write(struct snd_sof_dev *sdev, u32 bar, u32 offset, void *src,
+		     size_t size)
+{
+	void __iomem *dest = sdev->bar[bar] + offset;
+	const u8 *src_byte = src;
+	u32 affected_mask;
+	u32 tmp;
+	int m, n;
+
+	m = size / 4;
+	n = size % 4;
+
+	/* __iowrite32_copy use 32bit size values so divide by 4 */
+	__iowrite32_copy(dest, src, m);
+
+	if (n) {
+		affected_mask = (1 << (8 * n)) - 1;
+
+		/* first read the 32bit data of dest, then change affected
+		 * bytes, and write back to dest. For unaffected bytes, it
+		 * should not be changed
+		 */
+		tmp = ioread32(dest + m * 4);
+		tmp &= ~affected_mask;
+
+		tmp |= *(u32 *)(src_byte + m * 4) & affected_mask;
+		iowrite32(tmp, dest + m * 4);
+	}
+}
+EXPORT_SYMBOL(sof_block_write);
+
+void sof_block_read(struct snd_sof_dev *sdev, u32 bar, u32 offset, void *dest,
+		    size_t size)
+{
+	void __iomem *src = sdev->bar[bar] + offset;
+
+	memcpy_fromio(dest, src, size);
+}
+EXPORT_SYMBOL(sof_block_read);
diff --git a/sound/soc/sof/xtensa/Kconfig b/sound/soc/sof/xtensa/Kconfig
new file mode 100644
index 000000000000..8a9343b85146
--- /dev/null
+++ b/sound/soc/sof/xtensa/Kconfig
@@ -0,0 +1,2 @@
+config SND_SOC_SOF_XTENSA
+	tristate
diff --git a/sound/soc/sof/xtensa/Makefile b/sound/soc/sof/xtensa/Makefile
new file mode 100644
index 000000000000..cc89c7472a38
--- /dev/null
+++ b/sound/soc/sof/xtensa/Makefile
@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+
+snd-sof-xtensa-dsp-objs := core.o
+
+obj-$(CONFIG_SND_SOC_SOF_XTENSA) += snd-sof-xtensa-dsp.o
diff --git a/sound/soc/sof/xtensa/core.c b/sound/soc/sof/xtensa/core.c
new file mode 100644
index 000000000000..c3ad23a85b99
--- /dev/null
+++ b/sound/soc/sof/xtensa/core.c
@@ -0,0 +1,138 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license.  When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2018 Intel Corporation. All rights reserved.
+//
+// Author: Pan Xiuli <xiuli.pan@linux.intel.com>
+//
+
+#include <linux/module.h>
+#include <sound/sof.h>
+#include <sound/sof/xtensa.h>
+#include "../sof-priv.h"
+
+struct xtensa_exception_cause {
+	u32 id;
+	const char *msg;
+	const char *description;
+};
+
+/*
+ * From 4.4.1.5 table 4-64 Exception Causes of Xtensa
+ * Instruction Set Architecture (ISA) Reference Manual
+ */
+static const struct xtensa_exception_cause xtensa_exception_causes[] = {
+	{0, "IllegalInstructionCause", "Illegal instruction"},
+	{1, "SyscallCause", "SYSCALL instruction"},
+	{2, "InstructionFetchErrorCause",
+	"Processor internal physical address or data error during instruction fetch"},
+	{3, "LoadStoreErrorCause",
+	"Processor internal physical address or data error during load or store"},
+	{4, "Level1InterruptCause",
+	"Level-1 interrupt as indicated by set level-1 bits in the INTERRUPT register"},
+	{5, "AllocaCause",
+	"MOVSP instruction, if caller’s registers are not in the register file"},
+	{6, "IntegerDivideByZeroCause",
+	"QUOS, QUOU, REMS, or REMU divisor operand is zero"},
+	{8, "PrivilegedCause",
+	"Attempt to execute a privileged operation when CRING ? 0"},
+	{9, "LoadStoreAlignmentCause", "Load or store to an unaligned address"},
+	{12, "InstrPIFDataErrorCause",
+	"PIF data error during instruction fetch"},
+	{13, "LoadStorePIFDataErrorCause",
+	"Synchronous PIF data error during LoadStore access"},
+	{14, "InstrPIFAddrErrorCause",
+	"PIF address error during instruction fetch"},
+	{15, "LoadStorePIFAddrErrorCause",
+	"Synchronous PIF address error during LoadStore access"},
+	{16, "InstTLBMissCause", "Error during Instruction TLB refill"},
+	{17, "InstTLBMultiHitCause",
+	"Multiple instruction TLB entries matched"},
+	{18, "InstFetchPrivilegeCause",
+	"An instruction fetch referenced a virtual address at a ring level less than CRING"},
+	{20, "InstFetchProhibitedCause",
+	"An instruction fetch referenced a page mapped with an attribute that does not permit instruction fetch"},
+	{24, "LoadStoreTLBMissCause",
+	"Error during TLB refill for a load or store"},
+	{25, "LoadStoreTLBMultiHitCause",
+	"Multiple TLB entries matched for a load or store"},
+	{26, "LoadStorePrivilegeCause",
+	"A load or store referenced a virtual address at a ring level less than CRING"},
+	{28, "LoadProhibitedCause",
+	"A load referenced a page mapped with an attribute that does not permit loads"},
+	{32, "Coprocessor0Disabled",
+	"Coprocessor 0 instruction when cp0 disabled"},
+	{33, "Coprocessor1Disabled",
+	"Coprocessor 1 instruction when cp1 disabled"},
+	{34, "Coprocessor2Disabled",
+	"Coprocessor 2 instruction when cp2 disabled"},
+	{35, "Coprocessor3Disabled",
+	"Coprocessor 3 instruction when cp3 disabled"},
+	{36, "Coprocessor4Disabled",
+	"Coprocessor 4 instruction when cp4 disabled"},
+	{37, "Coprocessor5Disabled",
+	"Coprocessor 5 instruction when cp5 disabled"},
+	{38, "Coprocessor6Disabled",
+	"Coprocessor 6 instruction when cp6 disabled"},
+	{39, "Coprocessor7Disabled",
+	"Coprocessor 7 instruction when cp7 disabled"},
+};
+
+/* only need xtensa atm */
+static void xtensa_dsp_oops(struct snd_sof_dev *sdev, void *oops)
+{
+	struct sof_ipc_dsp_oops_xtensa *xoops = oops;
+	int i;
+
+	dev_err(sdev->dev, "error: DSP Firmware Oops\n");
+	for (i = 0; i < ARRAY_SIZE(xtensa_exception_causes); i++) {
+		if (xtensa_exception_causes[i].id == xoops->exccause) {
+			dev_err(sdev->dev, "error: Exception Cause: %s, %s\n",
+				xtensa_exception_causes[i].msg,
+				xtensa_exception_causes[i].description);
+		}
+	}
+	dev_err(sdev->dev, "EXCCAUSE 0x%8.8x EXCVADDR 0x%8.8x PS       0x%8.8x SAR     0x%8.8x\n",
+		xoops->exccause, xoops->excvaddr, xoops->ps, xoops->sar);
+	dev_err(sdev->dev, "EPC1     0x%8.8x EPC2     0x%8.8x EPC3     0x%8.8x EPC4    0x%8.8x",
+		xoops->epc1, xoops->epc2, xoops->epc3, xoops->epc4);
+	dev_err(sdev->dev, "EPC5     0x%8.8x EPC6     0x%8.8x EPC7     0x%8.8x DEPC    0x%8.8x",
+		xoops->epc5, xoops->epc6, xoops->epc7, xoops->depc);
+	dev_err(sdev->dev, "EPS2     0x%8.8x EPS3     0x%8.8x EPS4     0x%8.8x EPS5    0x%8.8x",
+		xoops->eps2, xoops->eps3, xoops->eps4, xoops->eps5);
+	dev_err(sdev->dev, "EPS6     0x%8.8x EPS7     0x%8.8x INTENABL 0x%8.8x INTERRU 0x%8.8x",
+		xoops->eps6, xoops->eps7, xoops->intenable, xoops->interrupt);
+}
+
+static void xtensa_stack(struct snd_sof_dev *sdev, void *oops, u32 *stack,
+			 u32 stack_words)
+{
+	struct sof_ipc_dsp_oops_xtensa *xoops = oops;
+	u32 stack_ptr = xoops->stack;
+	/* 4 * 8chars + 3 ws + 1 terminating NUL */
+	unsigned char buf[4 * 8 + 3 + 1];
+	int i;
+
+	dev_err(sdev->dev, "stack dump from 0x%8.8x\n", stack_ptr);
+
+	/*
+	 * example output:
+	 * 0x0049fbb0: 8000f2d0 0049fc00 6f6c6c61 00632e63
+	 */
+	for (i = 0; i < stack_words; i += 4) {
+		hex_dump_to_buffer(stack + i * 4, 16, 16, 4,
+				   buf, sizeof(buf), false);
+		dev_err(sdev->dev, "0x%08x: %s\n", stack_ptr + i, buf);
+	}
+}
+
+const struct sof_arch_ops sof_xtensa_arch_ops = {
+	.dsp_oops = xtensa_dsp_oops,
+	.dsp_stack = xtensa_stack,
+};
+EXPORT_SYMBOL(sof_xtensa_arch_ops);
+
+MODULE_DESCRIPTION("SOF Xtensa DSP support");
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/sound/soc/sprd/Kconfig b/sound/soc/sprd/Kconfig
index 43ece7daf0e9..21f9cc34f8bd 100644
--- a/sound/soc/sprd/Kconfig
+++ b/sound/soc/sprd/Kconfig
@@ -1,6 +1,15 @@
 config SND_SOC_SPRD
 	tristate "SoC Audio for the Spreadtrum SoC chips"
 	depends on ARCH_SPRD || COMPILE_TEST
+	select SND_SOC_COMPRESS
 	help
 	  Say Y or M if you want to add support for codecs attached to
 	  the Spreadtrum SoCs' Audio interfaces.
+
+config SND_SOC_SPRD_MCDT
+	bool "Spreadtrum multi-channel data transfer support"
+	depends on SND_SOC_SPRD
+	help
+	  Say y here to enable multi-channel data transfer support. It
+	  is used for sound stream transmission between audio subsystem
+	  and other AP/CP subsystem.
diff --git a/sound/soc/sprd/Makefile b/sound/soc/sprd/Makefile
index 47620e57a9f2..a95fa56cd000 100644
--- a/sound/soc/sprd/Makefile
+++ b/sound/soc/sprd/Makefile
@@ -1,4 +1,8 @@
 # SPDX-License-Identifier: GPL-2.0
 # Spreadtrum Audio Support
 
-obj-$(CONFIG_SND_SOC_SPRD)	+= sprd-pcm-dma.o
+snd-soc-sprd-platform-objs := sprd-pcm-dma.o sprd-pcm-compress.o
+
+obj-$(CONFIG_SND_SOC_SPRD) += snd-soc-sprd-platform.o
+
+obj-$(CONFIG_SND_SOC_SPRD_MCDT) += sprd-mcdt.o
diff --git a/sound/soc/sprd/sprd-mcdt.c b/sound/soc/sprd/sprd-mcdt.c
new file mode 100644
index 000000000000..7448015a4935
--- /dev/null
+++ b/sound/soc/sprd/sprd-mcdt.c
@@ -0,0 +1,1011 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2019 Spreadtrum Communications Inc.
+
+#include <linux/errno.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/spinlock.h>
+
+#include "sprd-mcdt.h"
+
+/* MCDT registers definition */
+#define MCDT_CH0_TXD		0x0
+#define MCDT_CH0_RXD		0x28
+#define MCDT_DAC0_WTMK		0x60
+#define MCDT_ADC0_WTMK		0x88
+#define MCDT_DMA_EN		0xb0
+
+#define MCDT_INT_EN0		0xb4
+#define MCDT_INT_EN1		0xb8
+#define MCDT_INT_EN2		0xbc
+
+#define MCDT_INT_CLR0		0xc0
+#define MCDT_INT_CLR1		0xc4
+#define MCDT_INT_CLR2		0xc8
+
+#define MCDT_INT_RAW1		0xcc
+#define MCDT_INT_RAW2		0xd0
+#define MCDT_INT_RAW3		0xd4
+
+#define MCDT_INT_MSK1		0xd8
+#define MCDT_INT_MSK2		0xdc
+#define MCDT_INT_MSK3		0xe0
+
+#define MCDT_DAC0_FIFO_ADDR_ST	0xe4
+#define MCDT_ADC0_FIFO_ADDR_ST	0xe8
+
+#define MCDT_CH_FIFO_ST0	0x134
+#define MCDT_CH_FIFO_ST1	0x138
+#define MCDT_CH_FIFO_ST2	0x13c
+
+#define MCDT_INT_MSK_CFG0	0x140
+#define MCDT_INT_MSK_CFG1	0x144
+
+#define MCDT_DMA_CFG0		0x148
+#define MCDT_FIFO_CLR		0x14c
+#define MCDT_DMA_CFG1		0x150
+#define MCDT_DMA_CFG2		0x154
+#define MCDT_DMA_CFG3		0x158
+#define MCDT_DMA_CFG4		0x15c
+#define MCDT_DMA_CFG5		0x160
+
+/* Channel water mark definition */
+#define MCDT_CH_FIFO_AE_SHIFT	16
+#define MCDT_CH_FIFO_AE_MASK	GENMASK(24, 16)
+#define MCDT_CH_FIFO_AF_MASK	GENMASK(8, 0)
+
+/* DMA channel select definition */
+#define MCDT_DMA_CH0_SEL_MASK	GENMASK(3, 0)
+#define MCDT_DMA_CH0_SEL_SHIFT	0
+#define MCDT_DMA_CH1_SEL_MASK	GENMASK(7, 4)
+#define MCDT_DMA_CH1_SEL_SHIFT	4
+#define MCDT_DMA_CH2_SEL_MASK	GENMASK(11, 8)
+#define MCDT_DMA_CH2_SEL_SHIFT	8
+#define MCDT_DMA_CH3_SEL_MASK	GENMASK(15, 12)
+#define MCDT_DMA_CH3_SEL_SHIFT	12
+#define MCDT_DMA_CH4_SEL_MASK	GENMASK(19, 16)
+#define MCDT_DMA_CH4_SEL_SHIFT	16
+#define MCDT_DAC_DMA_SHIFT	16
+
+/* DMA channel ACK select definition */
+#define MCDT_DMA_ACK_SEL_MASK	GENMASK(3, 0)
+
+/* Channel FIFO definition */
+#define MCDT_CH_FIFO_ADDR_SHIFT	16
+#define MCDT_CH_FIFO_ADDR_MASK	GENMASK(9, 0)
+#define MCDT_ADC_FIFO_SHIFT	16
+#define MCDT_FIFO_LENGTH	512
+
+#define MCDT_ADC_CHANNEL_NUM	10
+#define MCDT_DAC_CHANNEL_NUM	10
+#define MCDT_CHANNEL_NUM	(MCDT_ADC_CHANNEL_NUM + MCDT_DAC_CHANNEL_NUM)
+
+enum sprd_mcdt_fifo_int {
+	MCDT_ADC_FIFO_AE_INT,
+	MCDT_ADC_FIFO_AF_INT,
+	MCDT_DAC_FIFO_AE_INT,
+	MCDT_DAC_FIFO_AF_INT,
+	MCDT_ADC_FIFO_OV_INT,
+	MCDT_DAC_FIFO_OV_INT
+};
+
+enum sprd_mcdt_fifo_sts {
+	MCDT_ADC_FIFO_REAL_FULL,
+	MCDT_ADC_FIFO_REAL_EMPTY,
+	MCDT_ADC_FIFO_AF,
+	MCDT_ADC_FIFO_AE,
+	MCDT_DAC_FIFO_REAL_FULL,
+	MCDT_DAC_FIFO_REAL_EMPTY,
+	MCDT_DAC_FIFO_AF,
+	MCDT_DAC_FIFO_AE
+};
+
+struct sprd_mcdt_dev {
+	struct device *dev;
+	void __iomem *base;
+	spinlock_t lock;
+	struct sprd_mcdt_chan chan[MCDT_CHANNEL_NUM];
+};
+
+static LIST_HEAD(sprd_mcdt_chan_list);
+static DEFINE_MUTEX(sprd_mcdt_list_mutex);
+
+static void sprd_mcdt_update(struct sprd_mcdt_dev *mcdt, u32 reg, u32 val,
+			     u32 mask)
+{
+	u32 orig = readl_relaxed(mcdt->base + reg);
+	u32 tmp;
+
+	tmp = (orig & ~mask) | val;
+	writel_relaxed(tmp, mcdt->base + reg);
+}
+
+static void sprd_mcdt_dac_set_watermark(struct sprd_mcdt_dev *mcdt, u8 channel,
+					u32 full, u32 empty)
+{
+	u32 reg = MCDT_DAC0_WTMK + channel * 4;
+	u32 water_mark =
+		(empty << MCDT_CH_FIFO_AE_SHIFT) & MCDT_CH_FIFO_AE_MASK;
+
+	water_mark |= full & MCDT_CH_FIFO_AF_MASK;
+	sprd_mcdt_update(mcdt, reg, water_mark,
+			 MCDT_CH_FIFO_AE_MASK | MCDT_CH_FIFO_AF_MASK);
+}
+
+static void sprd_mcdt_adc_set_watermark(struct sprd_mcdt_dev *mcdt, u8 channel,
+					u32 full, u32 empty)
+{
+	u32 reg = MCDT_ADC0_WTMK + channel * 4;
+	u32 water_mark =
+		(empty << MCDT_CH_FIFO_AE_SHIFT) & MCDT_CH_FIFO_AE_MASK;
+
+	water_mark |= full & MCDT_CH_FIFO_AF_MASK;
+	sprd_mcdt_update(mcdt, reg, water_mark,
+			 MCDT_CH_FIFO_AE_MASK | MCDT_CH_FIFO_AF_MASK);
+}
+
+static void sprd_mcdt_dac_dma_enable(struct sprd_mcdt_dev *mcdt, u8 channel,
+				     bool enable)
+{
+	u32 shift = MCDT_DAC_DMA_SHIFT + channel;
+
+	if (enable)
+		sprd_mcdt_update(mcdt, MCDT_DMA_EN, BIT(shift), BIT(shift));
+	else
+		sprd_mcdt_update(mcdt, MCDT_DMA_EN, 0, BIT(shift));
+}
+
+static void sprd_mcdt_adc_dma_enable(struct sprd_mcdt_dev *mcdt, u8 channel,
+				     bool enable)
+{
+	if (enable)
+		sprd_mcdt_update(mcdt, MCDT_DMA_EN, BIT(channel), BIT(channel));
+	else
+		sprd_mcdt_update(mcdt, MCDT_DMA_EN, 0, BIT(channel));
+}
+
+static void sprd_mcdt_ap_int_enable(struct sprd_mcdt_dev *mcdt, u8 channel,
+				    bool enable)
+{
+	if (enable)
+		sprd_mcdt_update(mcdt, MCDT_INT_MSK_CFG0, BIT(channel),
+				 BIT(channel));
+	else
+		sprd_mcdt_update(mcdt, MCDT_INT_MSK_CFG0, 0, BIT(channel));
+}
+
+static void sprd_mcdt_dac_write_fifo(struct sprd_mcdt_dev *mcdt, u8 channel,
+				     u32 val)
+{
+	u32 reg = MCDT_CH0_TXD + channel * 4;
+
+	writel_relaxed(val, mcdt->base + reg);
+}
+
+static void sprd_mcdt_adc_read_fifo(struct sprd_mcdt_dev *mcdt, u8 channel,
+				    u32 *val)
+{
+	u32 reg = MCDT_CH0_RXD + channel * 4;
+
+	*val = readl_relaxed(mcdt->base + reg);
+}
+
+static void sprd_mcdt_dac_dma_chn_select(struct sprd_mcdt_dev *mcdt, u8 channel,
+					 enum sprd_mcdt_dma_chan dma_chan)
+{
+	switch (dma_chan) {
+	case SPRD_MCDT_DMA_CH0:
+		sprd_mcdt_update(mcdt, MCDT_DMA_CFG0,
+				 channel << MCDT_DMA_CH0_SEL_SHIFT,
+				 MCDT_DMA_CH0_SEL_MASK);
+		break;
+
+	case SPRD_MCDT_DMA_CH1:
+		sprd_mcdt_update(mcdt, MCDT_DMA_CFG0,
+				 channel << MCDT_DMA_CH1_SEL_SHIFT,
+				 MCDT_DMA_CH1_SEL_MASK);
+		break;
+
+	case SPRD_MCDT_DMA_CH2:
+		sprd_mcdt_update(mcdt, MCDT_DMA_CFG0,
+				 channel << MCDT_DMA_CH2_SEL_SHIFT,
+				 MCDT_DMA_CH2_SEL_MASK);
+		break;
+
+	case SPRD_MCDT_DMA_CH3:
+		sprd_mcdt_update(mcdt, MCDT_DMA_CFG0,
+				 channel << MCDT_DMA_CH3_SEL_SHIFT,
+				 MCDT_DMA_CH3_SEL_MASK);
+		break;
+
+	case SPRD_MCDT_DMA_CH4:
+		sprd_mcdt_update(mcdt, MCDT_DMA_CFG0,
+				 channel << MCDT_DMA_CH4_SEL_SHIFT,
+				 MCDT_DMA_CH4_SEL_MASK);
+		break;
+	}
+}
+
+static void sprd_mcdt_adc_dma_chn_select(struct sprd_mcdt_dev *mcdt, u8 channel,
+					 enum sprd_mcdt_dma_chan dma_chan)
+{
+	switch (dma_chan) {
+	case SPRD_MCDT_DMA_CH0:
+		sprd_mcdt_update(mcdt, MCDT_DMA_CFG1,
+				 channel << MCDT_DMA_CH0_SEL_SHIFT,
+				 MCDT_DMA_CH0_SEL_MASK);
+		break;
+
+	case SPRD_MCDT_DMA_CH1:
+		sprd_mcdt_update(mcdt, MCDT_DMA_CFG1,
+				 channel << MCDT_DMA_CH1_SEL_SHIFT,
+				 MCDT_DMA_CH1_SEL_MASK);
+		break;
+
+	case SPRD_MCDT_DMA_CH2:
+		sprd_mcdt_update(mcdt, MCDT_DMA_CFG1,
+				 channel << MCDT_DMA_CH2_SEL_SHIFT,
+				 MCDT_DMA_CH2_SEL_MASK);
+		break;
+
+	case SPRD_MCDT_DMA_CH3:
+		sprd_mcdt_update(mcdt, MCDT_DMA_CFG1,
+				 channel << MCDT_DMA_CH3_SEL_SHIFT,
+				 MCDT_DMA_CH3_SEL_MASK);
+		break;
+
+	case SPRD_MCDT_DMA_CH4:
+		sprd_mcdt_update(mcdt, MCDT_DMA_CFG1,
+				 channel << MCDT_DMA_CH4_SEL_SHIFT,
+				 MCDT_DMA_CH4_SEL_MASK);
+		break;
+	}
+}
+
+static u32 sprd_mcdt_dma_ack_shift(u8 channel)
+{
+	switch (channel) {
+	default:
+	case 0:
+	case 8:
+		return 0;
+	case 1:
+	case 9:
+		return 4;
+	case 2:
+		return 8;
+	case 3:
+		return 12;
+	case 4:
+		return 16;
+	case 5:
+		return 20;
+	case 6:
+		return 24;
+	case 7:
+		return 28;
+	}
+}
+
+static void sprd_mcdt_dac_dma_ack_select(struct sprd_mcdt_dev *mcdt, u8 channel,
+					 enum sprd_mcdt_dma_chan dma_chan)
+{
+	u32 reg, shift = sprd_mcdt_dma_ack_shift(channel), ack = dma_chan;
+
+	switch (channel) {
+	case 0 ... 7:
+		reg = MCDT_DMA_CFG2;
+		break;
+
+	case 8 ... 9:
+		reg = MCDT_DMA_CFG3;
+		break;
+
+	default:
+		return;
+	}
+
+	sprd_mcdt_update(mcdt, reg, ack << shift,
+			 MCDT_DMA_ACK_SEL_MASK << shift);
+}
+
+static void sprd_mcdt_adc_dma_ack_select(struct sprd_mcdt_dev *mcdt, u8 channel,
+					 enum sprd_mcdt_dma_chan dma_chan)
+{
+	u32 reg, shift = sprd_mcdt_dma_ack_shift(channel), ack = dma_chan;
+
+	switch (channel) {
+	case 0 ... 7:
+		reg = MCDT_DMA_CFG4;
+		break;
+
+	case 8 ... 9:
+		reg = MCDT_DMA_CFG5;
+		break;
+
+	default:
+		return;
+	}
+
+	sprd_mcdt_update(mcdt, reg, ack << shift,
+			 MCDT_DMA_ACK_SEL_MASK << shift);
+}
+
+static bool sprd_mcdt_chan_fifo_sts(struct sprd_mcdt_dev *mcdt, u8 channel,
+				    enum sprd_mcdt_fifo_sts fifo_sts)
+{
+	u32 reg, shift;
+
+	switch (channel) {
+	case 0 ... 3:
+		reg = MCDT_CH_FIFO_ST0;
+		break;
+	case 4 ... 7:
+		reg = MCDT_CH_FIFO_ST1;
+		break;
+	case 8 ... 9:
+		reg = MCDT_CH_FIFO_ST2;
+		break;
+	default:
+		return false;
+	}
+
+	switch (channel) {
+	case 0:
+	case 4:
+	case 8:
+		shift = fifo_sts;
+		break;
+
+	case 1:
+	case 5:
+	case 9:
+		shift = 8 + fifo_sts;
+		break;
+
+	case 2:
+	case 6:
+		shift = 16 + fifo_sts;
+		break;
+
+	case 3:
+	case 7:
+		shift = 24 + fifo_sts;
+		break;
+
+	default:
+		return false;
+	}
+
+	return !!(readl_relaxed(mcdt->base + reg) & BIT(shift));
+}
+
+static void sprd_mcdt_dac_fifo_clear(struct sprd_mcdt_dev *mcdt, u8 channel)
+{
+	sprd_mcdt_update(mcdt, MCDT_FIFO_CLR, BIT(channel), BIT(channel));
+}
+
+static void sprd_mcdt_adc_fifo_clear(struct sprd_mcdt_dev *mcdt, u8 channel)
+{
+	u32 shift = MCDT_ADC_FIFO_SHIFT + channel;
+
+	sprd_mcdt_update(mcdt, MCDT_FIFO_CLR, BIT(shift), BIT(shift));
+}
+
+static u32 sprd_mcdt_dac_fifo_avail(struct sprd_mcdt_dev *mcdt, u8 channel)
+{
+	u32 reg = MCDT_DAC0_FIFO_ADDR_ST + channel * 8;
+	u32 r_addr = (readl_relaxed(mcdt->base + reg) >>
+		      MCDT_CH_FIFO_ADDR_SHIFT) & MCDT_CH_FIFO_ADDR_MASK;
+	u32 w_addr = readl_relaxed(mcdt->base + reg) & MCDT_CH_FIFO_ADDR_MASK;
+
+	if (w_addr >= r_addr)
+		return 4 * (MCDT_FIFO_LENGTH - w_addr + r_addr);
+	else
+		return 4 * (r_addr - w_addr);
+}
+
+static u32 sprd_mcdt_adc_fifo_avail(struct sprd_mcdt_dev *mcdt, u8 channel)
+{
+	u32 reg = MCDT_ADC0_FIFO_ADDR_ST + channel * 8;
+	u32 r_addr = (readl_relaxed(mcdt->base + reg) >>
+		      MCDT_CH_FIFO_ADDR_SHIFT) & MCDT_CH_FIFO_ADDR_MASK;
+	u32 w_addr = readl_relaxed(mcdt->base + reg) & MCDT_CH_FIFO_ADDR_MASK;
+
+	if (w_addr >= r_addr)
+		return 4 * (w_addr - r_addr);
+	else
+		return 4 * (MCDT_FIFO_LENGTH - r_addr + w_addr);
+}
+
+static u32 sprd_mcdt_int_type_shift(u8 channel,
+				    enum sprd_mcdt_fifo_int int_type)
+{
+	switch (channel) {
+	case 0:
+	case 4:
+	case 8:
+		return int_type;
+
+	case 1:
+	case 5:
+	case 9:
+		return  8 + int_type;
+
+	case 2:
+	case 6:
+		return 16 + int_type;
+
+	case 3:
+	case 7:
+		return 24 + int_type;
+
+	default:
+		return 0;
+	}
+}
+
+static void sprd_mcdt_chan_int_en(struct sprd_mcdt_dev *mcdt, u8 channel,
+				  enum sprd_mcdt_fifo_int int_type, bool enable)
+{
+	u32 reg, shift = sprd_mcdt_int_type_shift(channel, int_type);
+
+	switch (channel) {
+	case 0 ... 3:
+		reg = MCDT_INT_EN0;
+		break;
+	case 4 ... 7:
+		reg = MCDT_INT_EN1;
+		break;
+	case 8 ... 9:
+		reg = MCDT_INT_EN2;
+		break;
+	default:
+		return;
+	}
+
+	if (enable)
+		sprd_mcdt_update(mcdt, reg, BIT(shift), BIT(shift));
+	else
+		sprd_mcdt_update(mcdt, reg, 0, BIT(shift));
+}
+
+static void sprd_mcdt_chan_int_clear(struct sprd_mcdt_dev *mcdt, u8 channel,
+				     enum sprd_mcdt_fifo_int int_type)
+{
+	u32 reg, shift = sprd_mcdt_int_type_shift(channel, int_type);
+
+	switch (channel) {
+	case 0 ... 3:
+		reg = MCDT_INT_CLR0;
+		break;
+	case 4 ... 7:
+		reg = MCDT_INT_CLR1;
+		break;
+	case 8 ... 9:
+		reg = MCDT_INT_CLR2;
+		break;
+	default:
+		return;
+	}
+
+	sprd_mcdt_update(mcdt, reg, BIT(shift), BIT(shift));
+}
+
+static bool sprd_mcdt_chan_int_sts(struct sprd_mcdt_dev *mcdt, u8 channel,
+				   enum sprd_mcdt_fifo_int int_type)
+{
+	u32 reg, shift = sprd_mcdt_int_type_shift(channel, int_type);
+
+	switch (channel) {
+	case 0 ... 3:
+		reg = MCDT_INT_MSK1;
+		break;
+	case 4 ... 7:
+		reg = MCDT_INT_MSK2;
+		break;
+	case 8 ... 9:
+		reg = MCDT_INT_MSK3;
+		break;
+	default:
+		return false;
+	}
+
+	return !!(readl_relaxed(mcdt->base + reg) & BIT(shift));
+}
+
+static irqreturn_t sprd_mcdt_irq_handler(int irq, void *dev_id)
+{
+	struct sprd_mcdt_dev *mcdt = (struct sprd_mcdt_dev *)dev_id;
+	int i;
+
+	spin_lock(&mcdt->lock);
+
+	for (i = 0; i < MCDT_ADC_CHANNEL_NUM; i++) {
+		if (sprd_mcdt_chan_int_sts(mcdt, i, MCDT_ADC_FIFO_AF_INT)) {
+			struct sprd_mcdt_chan *chan = &mcdt->chan[i];
+
+			sprd_mcdt_chan_int_clear(mcdt, i, MCDT_ADC_FIFO_AF_INT);
+			if (chan->cb)
+				chan->cb->notify(chan->cb->data);
+		}
+	}
+
+	for (i = 0; i < MCDT_DAC_CHANNEL_NUM; i++) {
+		if (sprd_mcdt_chan_int_sts(mcdt, i, MCDT_DAC_FIFO_AE_INT)) {
+			struct sprd_mcdt_chan *chan =
+				&mcdt->chan[i + MCDT_ADC_CHANNEL_NUM];
+
+			sprd_mcdt_chan_int_clear(mcdt, i, MCDT_DAC_FIFO_AE_INT);
+			if (chan->cb)
+				chan->cb->notify(chan->cb->data);
+		}
+	}
+
+	spin_unlock(&mcdt->lock);
+
+	return IRQ_HANDLED;
+}
+
+/**
+ * sprd_mcdt_chan_write - write data to the MCDT channel's fifo
+ * @chan: the MCDT channel
+ * @tx_buf: send buffer
+ * @size: data size
+ *
+ * Note: We can not write data to the channel fifo when enabling the DMA mode,
+ * otherwise the channel fifo data will be invalid.
+ *
+ * If there are not enough space of the channel fifo, it will return errors
+ * to users.
+ *
+ * Returns 0 on success, or an appropriate error code on failure.
+ */
+int sprd_mcdt_chan_write(struct sprd_mcdt_chan *chan, char *tx_buf, u32 size)
+{
+	struct sprd_mcdt_dev *mcdt = chan->mcdt;
+	unsigned long flags;
+	int avail, i = 0, words = size / 4;
+	u32 *buf = (u32 *)tx_buf;
+
+	spin_lock_irqsave(&mcdt->lock, flags);
+
+	if (chan->dma_enable) {
+		dev_err(mcdt->dev,
+			"Can not write data when DMA mode enabled\n");
+		spin_unlock_irqrestore(&mcdt->lock, flags);
+		return -EINVAL;
+	}
+
+	if (sprd_mcdt_chan_fifo_sts(mcdt, chan->id, MCDT_DAC_FIFO_REAL_FULL)) {
+		dev_err(mcdt->dev, "Channel fifo is full now\n");
+		spin_unlock_irqrestore(&mcdt->lock, flags);
+		return -EBUSY;
+	}
+
+	avail = sprd_mcdt_dac_fifo_avail(mcdt, chan->id);
+	if (size > avail) {
+		dev_err(mcdt->dev,
+			"Data size is larger than the available fifo size\n");
+		spin_unlock_irqrestore(&mcdt->lock, flags);
+		return -EBUSY;
+	}
+
+	while (i++ < words)
+		sprd_mcdt_dac_write_fifo(mcdt, chan->id, *buf++);
+
+	spin_unlock_irqrestore(&mcdt->lock, flags);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(sprd_mcdt_chan_write);
+
+/**
+ * sprd_mcdt_chan_read - read data from the MCDT channel's fifo
+ * @chan: the MCDT channel
+ * @rx_buf: receive buffer
+ * @size: data size
+ *
+ * Note: We can not read data from the channel fifo when enabling the DMA mode,
+ * otherwise the reading data will be invalid.
+ *
+ * Usually user need start to read data once receiving the fifo full interrupt.
+ *
+ * Returns data size of reading successfully, or an error code on failure.
+ */
+int sprd_mcdt_chan_read(struct sprd_mcdt_chan *chan, char *rx_buf, u32 size)
+{
+	struct sprd_mcdt_dev *mcdt = chan->mcdt;
+	unsigned long flags;
+	int i = 0, avail, words = size / 4;
+	u32 *buf = (u32 *)rx_buf;
+
+	spin_lock_irqsave(&mcdt->lock, flags);
+
+	if (chan->dma_enable) {
+		dev_err(mcdt->dev, "Can not read data when DMA mode enabled\n");
+		spin_unlock_irqrestore(&mcdt->lock, flags);
+		return -EINVAL;
+	}
+
+	if (sprd_mcdt_chan_fifo_sts(mcdt, chan->id, MCDT_ADC_FIFO_REAL_EMPTY)) {
+		dev_err(mcdt->dev, "Channel fifo is empty\n");
+		spin_unlock_irqrestore(&mcdt->lock, flags);
+		return -EBUSY;
+	}
+
+	avail = sprd_mcdt_adc_fifo_avail(mcdt, chan->id);
+	if (size > avail)
+		words = avail / 4;
+
+	while (i++ < words)
+		sprd_mcdt_adc_read_fifo(mcdt, chan->id, buf++);
+
+	spin_unlock_irqrestore(&mcdt->lock, flags);
+	return words * 4;
+}
+EXPORT_SYMBOL_GPL(sprd_mcdt_chan_read);
+
+/**
+ * sprd_mcdt_chan_int_enable - enable the interrupt mode for the MCDT channel
+ * @chan: the MCDT channel
+ * @water_mark: water mark to trigger a interrupt
+ * @cb: callback when a interrupt happened
+ *
+ * Now it only can enable fifo almost full interrupt for ADC channel and fifo
+ * almost empty interrupt for DAC channel. Morevoer for interrupt mode, user
+ * should use sprd_mcdt_chan_read() or sprd_mcdt_chan_write() to read or write
+ * data manually.
+ *
+ * For ADC channel, user can start to read data once receiving one fifo full
+ * interrupt. For DAC channel, user can start to write data once receiving one
+ * fifo empty interrupt or just call sprd_mcdt_chan_write() to write data
+ * directly.
+ *
+ * Returns 0 on success, or an error code on failure.
+ */
+int sprd_mcdt_chan_int_enable(struct sprd_mcdt_chan *chan, u32 water_mark,
+			      struct sprd_mcdt_chan_callback *cb)
+{
+	struct sprd_mcdt_dev *mcdt = chan->mcdt;
+	unsigned long flags;
+	int ret = 0;
+
+	spin_lock_irqsave(&mcdt->lock, flags);
+
+	if (chan->dma_enable || chan->int_enable) {
+		dev_err(mcdt->dev, "Failed to set interrupt mode.\n");
+		spin_unlock_irqrestore(&mcdt->lock, flags);
+		return -EINVAL;
+	}
+
+	switch (chan->type) {
+	case SPRD_MCDT_ADC_CHAN:
+		sprd_mcdt_adc_fifo_clear(mcdt, chan->id);
+		sprd_mcdt_adc_set_watermark(mcdt, chan->id, water_mark,
+					    MCDT_FIFO_LENGTH - 1);
+		sprd_mcdt_chan_int_en(mcdt, chan->id,
+				      MCDT_ADC_FIFO_AF_INT, true);
+		sprd_mcdt_ap_int_enable(mcdt, chan->id, true);
+		break;
+
+	case SPRD_MCDT_DAC_CHAN:
+		sprd_mcdt_dac_fifo_clear(mcdt, chan->id);
+		sprd_mcdt_dac_set_watermark(mcdt, chan->id,
+					    MCDT_FIFO_LENGTH - 1, water_mark);
+		sprd_mcdt_chan_int_en(mcdt, chan->id,
+				      MCDT_DAC_FIFO_AE_INT, true);
+		sprd_mcdt_ap_int_enable(mcdt, chan->id, true);
+		break;
+
+	default:
+		dev_err(mcdt->dev, "Unsupported channel type\n");
+		ret = -EINVAL;
+	}
+
+	if (!ret) {
+		chan->cb = cb;
+		chan->int_enable = true;
+	}
+
+	spin_unlock_irqrestore(&mcdt->lock, flags);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(sprd_mcdt_chan_int_enable);
+
+/**
+ * sprd_mcdt_chan_int_disable - disable the interrupt mode for the MCDT channel
+ * @chan: the MCDT channel
+ */
+void sprd_mcdt_chan_int_disable(struct sprd_mcdt_chan *chan)
+{
+	struct sprd_mcdt_dev *mcdt = chan->mcdt;
+	unsigned long flags;
+
+	spin_lock_irqsave(&mcdt->lock, flags);
+
+	if (!chan->int_enable) {
+		spin_unlock_irqrestore(&mcdt->lock, flags);
+		return;
+	}
+
+	switch (chan->type) {
+	case SPRD_MCDT_ADC_CHAN:
+		sprd_mcdt_chan_int_en(mcdt, chan->id,
+				      MCDT_ADC_FIFO_AF_INT, false);
+		sprd_mcdt_chan_int_clear(mcdt, chan->id, MCDT_ADC_FIFO_AF_INT);
+		sprd_mcdt_ap_int_enable(mcdt, chan->id, false);
+		break;
+
+	case SPRD_MCDT_DAC_CHAN:
+		sprd_mcdt_chan_int_en(mcdt, chan->id,
+				      MCDT_DAC_FIFO_AE_INT, false);
+		sprd_mcdt_chan_int_clear(mcdt, chan->id, MCDT_DAC_FIFO_AE_INT);
+		sprd_mcdt_ap_int_enable(mcdt, chan->id, false);
+		break;
+
+	default:
+		break;
+	}
+
+	chan->int_enable = false;
+	spin_unlock_irqrestore(&mcdt->lock, flags);
+}
+EXPORT_SYMBOL_GPL(sprd_mcdt_chan_int_disable);
+
+/**
+ * sprd_mcdt_chan_dma_enable - enable the DMA mode for the MCDT channel
+ * @chan: the MCDT channel
+ * @dma_chan: specify which DMA channel will be used for this MCDT channel
+ * @water_mark: water mark to trigger a DMA request
+ *
+ * Enable the DMA mode for the MCDT channel, that means we can use DMA to
+ * transfer data to the channel fifo and do not need reading/writing data
+ * manually.
+ *
+ * Returns 0 on success, or an error code on failure.
+ */
+int sprd_mcdt_chan_dma_enable(struct sprd_mcdt_chan *chan,
+			      enum sprd_mcdt_dma_chan dma_chan,
+			      u32 water_mark)
+{
+	struct sprd_mcdt_dev *mcdt = chan->mcdt;
+	unsigned long flags;
+	int ret = 0;
+
+	spin_lock_irqsave(&mcdt->lock, flags);
+
+	if (chan->dma_enable || chan->int_enable ||
+	    dma_chan > SPRD_MCDT_DMA_CH4) {
+		dev_err(mcdt->dev, "Failed to set DMA mode\n");
+		spin_unlock_irqrestore(&mcdt->lock, flags);
+		return -EINVAL;
+	}
+
+	switch (chan->type) {
+	case SPRD_MCDT_ADC_CHAN:
+		sprd_mcdt_adc_fifo_clear(mcdt, chan->id);
+		sprd_mcdt_adc_set_watermark(mcdt, chan->id,
+					    water_mark, MCDT_FIFO_LENGTH - 1);
+		sprd_mcdt_adc_dma_enable(mcdt, chan->id, true);
+		sprd_mcdt_adc_dma_chn_select(mcdt, chan->id, dma_chan);
+		sprd_mcdt_adc_dma_ack_select(mcdt, chan->id, dma_chan);
+		break;
+
+	case SPRD_MCDT_DAC_CHAN:
+		sprd_mcdt_dac_fifo_clear(mcdt, chan->id);
+		sprd_mcdt_dac_set_watermark(mcdt, chan->id,
+					    MCDT_FIFO_LENGTH - 1, water_mark);
+		sprd_mcdt_dac_dma_enable(mcdt, chan->id, true);
+		sprd_mcdt_dac_dma_chn_select(mcdt, chan->id, dma_chan);
+		sprd_mcdt_dac_dma_ack_select(mcdt, chan->id, dma_chan);
+		break;
+
+	default:
+		dev_err(mcdt->dev, "Unsupported channel type\n");
+		ret = -EINVAL;
+	}
+
+	if (!ret)
+		chan->dma_enable = true;
+
+	spin_unlock_irqrestore(&mcdt->lock, flags);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(sprd_mcdt_chan_dma_enable);
+
+/**
+ * sprd_mcdt_chan_dma_disable - disable the DMA mode for the MCDT channel
+ * @chan: the MCDT channel
+ */
+void sprd_mcdt_chan_dma_disable(struct sprd_mcdt_chan *chan)
+{
+	struct sprd_mcdt_dev *mcdt = chan->mcdt;
+	unsigned long flags;
+
+	spin_lock_irqsave(&mcdt->lock, flags);
+
+	if (!chan->dma_enable) {
+		spin_unlock_irqrestore(&mcdt->lock, flags);
+		return;
+	}
+
+	switch (chan->type) {
+	case SPRD_MCDT_ADC_CHAN:
+		sprd_mcdt_adc_dma_enable(mcdt, chan->id, false);
+		sprd_mcdt_adc_fifo_clear(mcdt, chan->id);
+		break;
+
+	case SPRD_MCDT_DAC_CHAN:
+		sprd_mcdt_dac_dma_enable(mcdt, chan->id, false);
+		sprd_mcdt_dac_fifo_clear(mcdt, chan->id);
+		break;
+
+	default:
+		break;
+	}
+
+	chan->dma_enable = false;
+	spin_unlock_irqrestore(&mcdt->lock, flags);
+}
+EXPORT_SYMBOL_GPL(sprd_mcdt_chan_dma_disable);
+
+/**
+ * sprd_mcdt_request_chan - request one MCDT channel
+ * @channel: channel id
+ * @type: channel type, it can be one ADC channel or DAC channel
+ *
+ * Rreturn NULL if no available channel.
+ */
+struct sprd_mcdt_chan *sprd_mcdt_request_chan(u8 channel,
+					      enum sprd_mcdt_channel_type type)
+{
+	struct sprd_mcdt_chan *temp, *chan = NULL;
+
+	mutex_lock(&sprd_mcdt_list_mutex);
+
+	list_for_each_entry(temp, &sprd_mcdt_chan_list, list) {
+		if (temp->type == type && temp->id == channel) {
+			chan = temp;
+			break;
+		}
+	}
+
+	if (chan)
+		list_del(&chan->list);
+
+	mutex_unlock(&sprd_mcdt_list_mutex);
+
+	return chan;
+}
+EXPORT_SYMBOL_GPL(sprd_mcdt_request_chan);
+
+/**
+ * sprd_mcdt_free_chan - free one MCDT channel
+ * @chan: the channel to be freed
+ */
+void sprd_mcdt_free_chan(struct sprd_mcdt_chan *chan)
+{
+	struct sprd_mcdt_chan *temp;
+
+	sprd_mcdt_chan_dma_disable(chan);
+	sprd_mcdt_chan_int_disable(chan);
+
+	mutex_lock(&sprd_mcdt_list_mutex);
+
+	list_for_each_entry(temp, &sprd_mcdt_chan_list, list) {
+		if (temp == chan) {
+			mutex_unlock(&sprd_mcdt_list_mutex);
+			return;
+		}
+	}
+
+	list_add_tail(&chan->list, &sprd_mcdt_chan_list);
+	mutex_unlock(&sprd_mcdt_list_mutex);
+}
+EXPORT_SYMBOL_GPL(sprd_mcdt_free_chan);
+
+static void sprd_mcdt_init_chans(struct sprd_mcdt_dev *mcdt,
+				 struct resource *res)
+{
+	int i;
+
+	for (i = 0; i < MCDT_CHANNEL_NUM; i++) {
+		struct sprd_mcdt_chan *chan = &mcdt->chan[i];
+
+		if (i < MCDT_ADC_CHANNEL_NUM) {
+			chan->id = i;
+			chan->type = SPRD_MCDT_ADC_CHAN;
+			chan->fifo_phys = res->start + MCDT_CH0_RXD + i * 4;
+		} else {
+			chan->id = i - MCDT_ADC_CHANNEL_NUM;
+			chan->type = SPRD_MCDT_DAC_CHAN;
+			chan->fifo_phys = res->start + MCDT_CH0_TXD +
+				(i - MCDT_ADC_CHANNEL_NUM) * 4;
+		}
+
+		chan->mcdt = mcdt;
+		INIT_LIST_HEAD(&chan->list);
+
+		mutex_lock(&sprd_mcdt_list_mutex);
+		list_add_tail(&chan->list, &sprd_mcdt_chan_list);
+		mutex_unlock(&sprd_mcdt_list_mutex);
+	}
+}
+
+static int sprd_mcdt_probe(struct platform_device *pdev)
+{
+	struct sprd_mcdt_dev *mcdt;
+	struct resource *res;
+	int ret, irq;
+
+	mcdt = devm_kzalloc(&pdev->dev, sizeof(*mcdt), GFP_KERNEL);
+	if (!mcdt)
+		return -ENOMEM;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	mcdt->base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(mcdt->base))
+		return PTR_ERR(mcdt->base);
+
+	mcdt->dev = &pdev->dev;
+	spin_lock_init(&mcdt->lock);
+	platform_set_drvdata(pdev, mcdt);
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0) {
+		dev_err(&pdev->dev, "Failed to get MCDT interrupt\n");
+		return irq;
+	}
+
+	ret = devm_request_irq(&pdev->dev, irq, sprd_mcdt_irq_handler,
+			       0, "sprd-mcdt", mcdt);
+	if (ret) {
+		dev_err(&pdev->dev, "Failed to request MCDT IRQ\n");
+		return ret;
+	}
+
+	sprd_mcdt_init_chans(mcdt, res);
+
+	return 0;
+}
+
+static int sprd_mcdt_remove(struct platform_device *pdev)
+{
+	struct sprd_mcdt_chan *chan, *temp;
+
+	mutex_lock(&sprd_mcdt_list_mutex);
+
+	list_for_each_entry_safe(chan, temp, &sprd_mcdt_chan_list, list)
+		list_del(&chan->list);
+
+	mutex_unlock(&sprd_mcdt_list_mutex);
+
+	return 0;
+}
+
+static const struct of_device_id sprd_mcdt_of_match[] = {
+	{ .compatible = "sprd,sc9860-mcdt", },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, sprd_mcdt_of_match);
+
+static struct platform_driver sprd_mcdt_driver = {
+	.probe = sprd_mcdt_probe,
+	.remove = sprd_mcdt_remove,
+	.driver = {
+		.name = "sprd-mcdt",
+		.of_match_table = sprd_mcdt_of_match,
+	},
+};
+
+module_platform_driver(sprd_mcdt_driver);
+
+MODULE_DESCRIPTION("Spreadtrum Multi-Channel Data Transfer Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/sound/soc/sprd/sprd-mcdt.h b/sound/soc/sprd/sprd-mcdt.h
new file mode 100644
index 000000000000..9cc7e207ac76
--- /dev/null
+++ b/sound/soc/sprd/sprd-mcdt.h
@@ -0,0 +1,107 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#ifndef __SPRD_MCDT_H
+#define __SPRD_MCDT_H
+
+enum sprd_mcdt_channel_type {
+	SPRD_MCDT_DAC_CHAN,
+	SPRD_MCDT_ADC_CHAN,
+	SPRD_MCDT_UNKNOWN_CHAN,
+};
+
+enum sprd_mcdt_dma_chan {
+	SPRD_MCDT_DMA_CH0,
+	SPRD_MCDT_DMA_CH1,
+	SPRD_MCDT_DMA_CH2,
+	SPRD_MCDT_DMA_CH3,
+	SPRD_MCDT_DMA_CH4,
+};
+
+struct sprd_mcdt_chan_callback {
+	void (*notify)(void *data);
+	void *data;
+};
+
+/**
+ * struct sprd_mcdt_chan - this struct represents a single channel instance
+ * @mcdt: the mcdt controller
+ * @id: channel id
+ * @fifo_phys: channel fifo physical address which is used for DMA transfer
+ * @type: channel type
+ * @cb: channel fifo interrupt's callback interface to notify the fifo events
+ * @dma_enable: indicate if use DMA mode to transfer data
+ * @int_enable: indicate if use interrupt mode to notify users to read or
+ * write data manually
+ * @list: used to link into the global list
+ *
+ * Note: users should not modify any members of this structure.
+ */
+struct sprd_mcdt_chan {
+	struct sprd_mcdt_dev *mcdt;
+	u8 id;
+	unsigned long fifo_phys;
+	enum sprd_mcdt_channel_type type;
+	enum sprd_mcdt_dma_chan dma_chan;
+	struct sprd_mcdt_chan_callback *cb;
+	bool dma_enable;
+	bool int_enable;
+	struct list_head list;
+};
+
+#ifdef CONFIG_SND_SOC_SPRD_MCDT
+struct sprd_mcdt_chan *sprd_mcdt_request_chan(u8 channel,
+					      enum sprd_mcdt_channel_type type);
+void sprd_mcdt_free_chan(struct sprd_mcdt_chan *chan);
+
+int sprd_mcdt_chan_write(struct sprd_mcdt_chan *chan, char *tx_buf, u32 size);
+int sprd_mcdt_chan_read(struct sprd_mcdt_chan *chan, char *rx_buf, u32 size);
+int sprd_mcdt_chan_int_enable(struct sprd_mcdt_chan *chan, u32 water_mark,
+			      struct sprd_mcdt_chan_callback *cb);
+void sprd_mcdt_chan_int_disable(struct sprd_mcdt_chan *chan);
+
+int sprd_mcdt_chan_dma_enable(struct sprd_mcdt_chan *chan,
+			      enum sprd_mcdt_dma_chan dma_chan, u32 water_mark);
+void sprd_mcdt_chan_dma_disable(struct sprd_mcdt_chan *chan);
+
+#else
+
+struct sprd_mcdt_chan *sprd_mcdt_request_chan(u8 channel,
+					      enum sprd_mcdt_channel_type type)
+{
+	return NULL;
+}
+
+void sprd_mcdt_free_chan(struct sprd_mcdt_chan *chan)
+{ }
+
+int sprd_mcdt_chan_write(struct sprd_mcdt_chan *chan, char *tx_buf, u32 size)
+{
+	return -EINVAL;
+}
+
+int sprd_mcdt_chan_read(struct sprd_mcdt_chan *chan, char *rx_buf, u32 size)
+{
+	return 0;
+}
+
+int sprd_mcdt_chan_int_enable(struct sprd_mcdt_chan *chan, u32 water_mark,
+			      struct sprd_mcdt_chan_callback *cb)
+{
+	return -EINVAL;
+}
+
+void sprd_mcdt_chan_int_disable(struct sprd_mcdt_chan *chan)
+{ }
+
+int sprd_mcdt_chan_dma_enable(struct sprd_mcdt_chan *chan,
+			      enum sprd_mcdt_dma_chan dma_chan, u32 water_mark)
+{
+	return -EINVAL;
+}
+
+void sprd_mcdt_chan_dma_disable(struct sprd_mcdt_chan *chan)
+{ }
+
+#endif
+
+#endif /* __SPRD_MCDT_H */
diff --git a/sound/soc/sprd/sprd-pcm-compress.c b/sound/soc/sprd/sprd-pcm-compress.c
new file mode 100644
index 000000000000..6cddf551bc11
--- /dev/null
+++ b/sound/soc/sprd/sprd-pcm-compress.c
@@ -0,0 +1,674 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2019 Spreadtrum Communications Inc.
+
+#include <linux/dma-mapping.h>
+#include <linux/dmaengine.h>
+#include <linux/dma/sprd-dma.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <sound/pcm.h>
+#include <sound/pcm_params.h>
+#include <sound/soc.h>
+#include <sound/compress_driver.h>
+
+#include "sprd-pcm-dma.h"
+
+#define SPRD_COMPR_DMA_CHANS		2
+
+/* Default values if userspace does not set */
+#define SPRD_COMPR_MIN_FRAGMENT_SIZE	SZ_8K
+#define SPRD_COMPR_MAX_FRAGMENT_SIZE	SZ_128K
+#define SPRD_COMPR_MIN_NUM_FRAGMENTS	4
+#define SPRD_COMPR_MAX_NUM_FRAGMENTS	64
+
+/* DSP FIFO size */
+#define SPRD_COMPR_MCDT_EMPTY_WMK	0
+#define SPRD_COMPR_MCDT_FIFO_SIZE	512
+
+/* Stage 0 IRAM buffer size definition */
+#define SPRD_COMPR_IRAM_BUF_SIZE	SZ_32K
+#define SPRD_COMPR_IRAM_INFO_SIZE	(sizeof(struct sprd_compr_playinfo))
+#define SPRD_COMPR_IRAM_LINKLIST_SIZE	(1024 - SPRD_COMPR_IRAM_INFO_SIZE)
+#define SPRD_COMPR_IRAM_SIZE		(SPRD_COMPR_IRAM_BUF_SIZE + \
+					 SPRD_COMPR_IRAM_INFO_SIZE + \
+					 SPRD_COMPR_IRAM_LINKLIST_SIZE)
+
+/* Stage 1 DDR buffer size definition */
+#define SPRD_COMPR_AREA_BUF_SIZE	SZ_2M
+#define SPRD_COMPR_AREA_LINKLIST_SIZE	1024
+#define SPRD_COMPR_AREA_SIZE		(SPRD_COMPR_AREA_BUF_SIZE + \
+					 SPRD_COMPR_AREA_LINKLIST_SIZE)
+
+struct sprd_compr_dma {
+	struct dma_chan *chan;
+	struct dma_async_tx_descriptor *desc;
+	dma_cookie_t cookie;
+	dma_addr_t phys;
+	void *virt;
+	int trans_len;
+};
+
+/*
+ * The Spreadtrum Audio compress offload mode will use 2-stage DMA transfer to
+ * save power. That means we can request 2 dma channels, one for source channel,
+ * and another one for destination channel. Once the source channel's transaction
+ * is done, it will trigger the destination channel's transaction automatically
+ * by hardware signal.
+ *
+ * For 2-stage DMA transfer, we can allocate 2 buffers: IRAM buffer (always
+ * power-on) and DDR buffer. The source channel will transfer data from IRAM
+ * buffer to the DSP fifo to decoding/encoding, once IRAM buffer is empty by
+ * transferring done, the destination channel will start to transfer data from
+ * DDR buffer to IRAM buffer.
+ *
+ * Since the DSP fifo is only 512B, IRAM buffer is allocated by 32K, and DDR
+ * buffer is larger to 2M. That means only the IRAM 32k data is transferred
+ * done, we can wake up the AP system to transfer data from DDR to IRAM, and
+ * other time the AP system can be suspended to save power.
+ */
+struct sprd_compr_stream {
+	struct snd_compr_stream *cstream;
+	struct sprd_compr_ops *compr_ops;
+	struct sprd_compr_dma dma[SPRD_COMPR_DMA_CHANS];
+
+	/* DMA engine channel number */
+	int num_channels;
+
+	/* Stage 0 IRAM buffer */
+	struct snd_dma_buffer iram_buffer;
+	/* Stage 1 DDR buffer */
+	struct snd_dma_buffer compr_buffer;
+
+	/* DSP play information IRAM buffer */
+	dma_addr_t info_phys;
+	void *info_area;
+	int info_size;
+
+	/* Data size copied to IRAM buffer */
+	int copied_total;
+	/* Total received data size from userspace */
+	int received_total;
+	/* Stage 0 IRAM buffer received data size */
+	int received_stage0;
+	/* Stage 1 DDR buffer received data size */
+	int received_stage1;
+	/* Stage 1 DDR buffer pointer */
+	int stage1_pointer;
+};
+
+static int sprd_platform_compr_trigger(struct snd_compr_stream *cstream,
+				       int cmd);
+
+static void sprd_platform_compr_drain_notify(void *arg)
+{
+	struct snd_compr_stream *cstream = arg;
+	struct snd_compr_runtime *runtime = cstream->runtime;
+	struct sprd_compr_stream *stream = runtime->private_data;
+
+	memset(stream->info_area, 0, sizeof(struct sprd_compr_playinfo));
+
+	snd_compr_drain_notify(cstream);
+}
+
+static void sprd_platform_compr_dma_complete(void *data)
+{
+	struct snd_compr_stream *cstream = data;
+	struct snd_compr_runtime *runtime = cstream->runtime;
+	struct sprd_compr_stream *stream = runtime->private_data;
+	struct sprd_compr_dma *dma = &stream->dma[1];
+
+	/* Update data size copied to IRAM buffer */
+	stream->copied_total += dma->trans_len;
+	if (stream->copied_total > stream->received_total)
+		stream->copied_total = stream->received_total;
+
+	snd_compr_fragment_elapsed(cstream);
+}
+
+static int sprd_platform_compr_dma_config(struct snd_compr_stream *cstream,
+					  struct snd_compr_params *params,
+					  int channel)
+{
+	struct snd_compr_runtime *runtime = cstream->runtime;
+	struct sprd_compr_stream *stream = runtime->private_data;
+	struct snd_soc_pcm_runtime *rtd = cstream->private_data;
+	struct snd_soc_component *component =
+		snd_soc_rtdcom_lookup(rtd, DRV_NAME);
+	struct device *dev = component->dev;
+	struct sprd_compr_data *data = snd_soc_dai_get_drvdata(rtd->cpu_dai);
+	struct sprd_pcm_dma_params *dma_params = data->dma_params;
+	struct sprd_compr_dma *dma = &stream->dma[channel];
+	struct dma_slave_config config = { };
+	struct sprd_dma_linklist link = { };
+	enum dma_transfer_direction dir;
+	struct scatterlist *sg, *sgt;
+	enum dma_slave_buswidth bus_width;
+	int period, period_cnt, sg_num = 2;
+	dma_addr_t src_addr, dst_addr;
+	unsigned long flags;
+	int ret, j;
+
+	if (!dma_params) {
+		dev_err(dev, "no dma parameters setting\n");
+		return -EINVAL;
+	}
+
+	dma->chan = dma_request_slave_channel(dev,
+					      dma_params->chan_name[channel]);
+	if (!dma->chan) {
+		dev_err(dev, "failed to request dma channel\n");
+		return -ENODEV;
+	}
+
+	sgt = sg = devm_kcalloc(dev, sg_num, sizeof(*sg), GFP_KERNEL);
+	if (!sg) {
+		ret = -ENOMEM;
+		goto sg_err;
+	}
+
+	switch (channel) {
+	case 0:
+		bus_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
+		period = (SPRD_COMPR_MCDT_FIFO_SIZE - SPRD_COMPR_MCDT_EMPTY_WMK) * 4;
+		period_cnt = params->buffer.fragment_size / period;
+		src_addr = stream->iram_buffer.addr;
+		dst_addr = dma_params->dev_phys[channel];
+		flags = SPRD_DMA_FLAGS(SPRD_DMA_SRC_CHN1,
+				       SPRD_DMA_TRANS_DONE_TRG,
+				       SPRD_DMA_FRAG_REQ,
+				       SPRD_DMA_TRANS_INT);
+		break;
+
+	case 1:
+		bus_width = DMA_SLAVE_BUSWIDTH_2_BYTES;
+		period = params->buffer.fragment_size;
+		period_cnt = params->buffer.fragments;
+		src_addr = stream->compr_buffer.addr;
+		dst_addr = stream->iram_buffer.addr;
+		flags = SPRD_DMA_FLAGS(SPRD_DMA_DST_CHN1,
+				       SPRD_DMA_TRANS_DONE_TRG,
+				       SPRD_DMA_FRAG_REQ,
+				       SPRD_DMA_TRANS_INT);
+		break;
+
+	default:
+		ret = -EINVAL;
+		goto config_err;
+	}
+
+	dma->trans_len = period * period_cnt;
+
+	config.src_maxburst = period;
+	config.src_addr_width = bus_width;
+	config.dst_addr_width = bus_width;
+	if (cstream->direction == SND_COMPRESS_PLAYBACK) {
+		config.src_addr = src_addr;
+		config.dst_addr = dst_addr;
+		dir = DMA_MEM_TO_DEV;
+	} else {
+		config.src_addr = dst_addr;
+		config.dst_addr = src_addr;
+		dir = DMA_DEV_TO_MEM;
+	}
+
+	sg_init_table(sgt, sg_num);
+	for (j = 0; j < sg_num; j++, sgt++) {
+		sg_dma_len(sgt) = dma->trans_len;
+		sg_dma_address(sgt) = dst_addr;
+	}
+
+	/*
+	 * Configure the link-list address for the DMA engine link-list
+	 * mode.
+	 */
+	link.virt_addr = (unsigned long)dma->virt;
+	link.phy_addr = dma->phys;
+
+	ret = dmaengine_slave_config(dma->chan, &config);
+	if (ret) {
+		dev_err(dev,
+			"failed to set slave configuration: %d\n", ret);
+		goto config_err;
+	}
+
+	/*
+	 * We configure the DMA request mode, interrupt mode, channel
+	 * mode and channel trigger mode by the flags.
+	 */
+	dma->desc = dma->chan->device->device_prep_slave_sg(dma->chan, sg,
+							    sg_num, dir,
+							    flags, &link);
+	if (!dma->desc) {
+		dev_err(dev, "failed to prepare slave sg\n");
+		ret = -ENOMEM;
+		goto config_err;
+	}
+
+	/* Only channel 1 transfer can wake up the AP system. */
+	if (!params->no_wake_mode && channel == 1) {
+		dma->desc->callback = sprd_platform_compr_dma_complete;
+		dma->desc->callback_param = cstream;
+	}
+
+	devm_kfree(dev, sg);
+
+	return 0;
+
+config_err:
+	devm_kfree(dev, sg);
+sg_err:
+	dma_release_channel(dma->chan);
+	return ret;
+}
+
+static int sprd_platform_compr_set_params(struct snd_compr_stream *cstream,
+					  struct snd_compr_params *params)
+{
+	struct snd_compr_runtime *runtime = cstream->runtime;
+	struct sprd_compr_stream *stream = runtime->private_data;
+	struct snd_soc_pcm_runtime *rtd = cstream->private_data;
+	struct snd_soc_component *component =
+		snd_soc_rtdcom_lookup(rtd, DRV_NAME);
+	struct device *dev = component->dev;
+	struct sprd_compr_params compr_params = { };
+	int ret;
+
+	/*
+	 * Configure the DMA engine 2-stage transfer mode. Channel 1 set as the
+	 * destination channel, and channel 0 set as the source channel, that
+	 * means once the source channel's transaction is done, it will trigger
+	 * the destination channel's transaction automatically.
+	 */
+	ret = sprd_platform_compr_dma_config(cstream, params, 1);
+	if (ret) {
+		dev_err(dev, "failed to config stage 1 DMA: %d\n", ret);
+		return ret;
+	}
+
+	ret = sprd_platform_compr_dma_config(cstream, params, 0);
+	if (ret) {
+		dev_err(dev, "failed to config stage 0 DMA: %d\n", ret);
+		goto config_err;
+	}
+
+	compr_params.direction = cstream->direction;
+	compr_params.sample_rate = params->codec.sample_rate;
+	compr_params.channels = stream->num_channels;
+	compr_params.info_phys = stream->info_phys;
+	compr_params.info_size = stream->info_size;
+	compr_params.rate = params->codec.bit_rate;
+	compr_params.format = params->codec.id;
+
+	ret = stream->compr_ops->set_params(cstream->direction, &compr_params);
+	if (ret) {
+		dev_err(dev, "failed to set parameters: %d\n", ret);
+		goto params_err;
+	}
+
+	return 0;
+
+params_err:
+	dma_release_channel(stream->dma[0].chan);
+config_err:
+	dma_release_channel(stream->dma[1].chan);
+	return ret;
+}
+
+static int sprd_platform_compr_open(struct snd_compr_stream *cstream)
+{
+	struct snd_compr_runtime *runtime = cstream->runtime;
+	struct snd_soc_pcm_runtime *rtd = cstream->private_data;
+	struct snd_soc_component *component =
+		snd_soc_rtdcom_lookup(rtd, DRV_NAME);
+	struct device *dev = component->dev;
+	struct sprd_compr_data *data = snd_soc_dai_get_drvdata(rtd->cpu_dai);
+	struct sprd_compr_stream *stream;
+	struct sprd_compr_callback cb;
+	int stream_id = cstream->direction, ret;
+
+	ret = dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(32));
+	if (ret)
+		return ret;
+
+	stream = devm_kzalloc(dev, sizeof(*stream), GFP_KERNEL);
+	if (!stream)
+		return -ENOMEM;
+
+	stream->cstream = cstream;
+	stream->num_channels = 2;
+	stream->compr_ops = data->ops;
+
+	/*
+	 * Allocate the stage 0 IRAM buffer size, including the DMA 0
+	 * link-list size and play information of DSP address size.
+	 */
+	ret = snd_dma_alloc_pages(SNDRV_DMA_TYPE_DEV_IRAM, dev,
+				  SPRD_COMPR_IRAM_SIZE, &stream->iram_buffer);
+	if (ret < 0)
+		goto err_iram;
+
+	/* Use to save link-list configuration for DMA 0. */
+	stream->dma[0].virt = stream->iram_buffer.area + SPRD_COMPR_IRAM_SIZE;
+	stream->dma[0].phys = stream->iram_buffer.addr + SPRD_COMPR_IRAM_SIZE;
+
+	/* Use to update the current data offset of DSP. */
+	stream->info_phys = stream->iram_buffer.addr + SPRD_COMPR_IRAM_SIZE +
+		SPRD_COMPR_IRAM_LINKLIST_SIZE;
+	stream->info_area = stream->iram_buffer.area + SPRD_COMPR_IRAM_SIZE +
+		SPRD_COMPR_IRAM_LINKLIST_SIZE;
+	stream->info_size = SPRD_COMPR_IRAM_INFO_SIZE;
+
+	/*
+	 * Allocate the stage 1 DDR buffer size, including the DMA 1 link-list
+	 * size.
+	 */
+	ret = snd_dma_alloc_pages(SNDRV_DMA_TYPE_DEV, dev,
+				  SPRD_COMPR_AREA_SIZE, &stream->compr_buffer);
+	if (ret < 0)
+		goto err_compr;
+
+	/* Use to save link-list configuration for DMA 1. */
+	stream->dma[1].virt = stream->compr_buffer.area + SPRD_COMPR_AREA_SIZE;
+	stream->dma[1].phys = stream->compr_buffer.addr + SPRD_COMPR_AREA_SIZE;
+
+	cb.drain_notify = sprd_platform_compr_drain_notify;
+	cb.drain_data = cstream;
+	ret = stream->compr_ops->open(stream_id, &cb);
+	if (ret) {
+		dev_err(dev, "failed to open compress platform: %d\n", ret);
+		goto err_open;
+	}
+
+	runtime->private_data = stream;
+	return 0;
+
+err_open:
+	snd_dma_free_pages(&stream->compr_buffer);
+err_compr:
+	snd_dma_free_pages(&stream->iram_buffer);
+err_iram:
+	devm_kfree(dev, stream);
+
+	return ret;
+}
+
+static int sprd_platform_compr_free(struct snd_compr_stream *cstream)
+{
+	struct snd_compr_runtime *runtime = cstream->runtime;
+	struct sprd_compr_stream *stream = runtime->private_data;
+	struct snd_soc_pcm_runtime *rtd = cstream->private_data;
+	struct snd_soc_component *component =
+		snd_soc_rtdcom_lookup(rtd, DRV_NAME);
+	struct device *dev = component->dev;
+	int stream_id = cstream->direction, i;
+
+	for (i = 0; i < stream->num_channels; i++) {
+		struct sprd_compr_dma *dma = &stream->dma[i];
+
+		if (dma->chan) {
+			dma_release_channel(dma->chan);
+			dma->chan = NULL;
+		}
+	}
+
+	snd_dma_free_pages(&stream->compr_buffer);
+	snd_dma_free_pages(&stream->iram_buffer);
+
+	stream->compr_ops->close(stream_id);
+
+	devm_kfree(dev, stream);
+	return 0;
+}
+
+static int sprd_platform_compr_trigger(struct snd_compr_stream *cstream,
+				       int cmd)
+{
+	struct snd_compr_runtime *runtime = cstream->runtime;
+	struct sprd_compr_stream *stream = runtime->private_data;
+	struct snd_soc_pcm_runtime *rtd = cstream->private_data;
+	struct snd_soc_component *component =
+		snd_soc_rtdcom_lookup(rtd, DRV_NAME);
+	struct device *dev = component->dev;
+	int channels = stream->num_channels, ret = 0, i;
+	int stream_id = cstream->direction;
+
+	if (cstream->direction != SND_COMPRESS_PLAYBACK) {
+		dev_err(dev, "unsupported compress direction\n");
+		return -EINVAL;
+	}
+
+	switch (cmd) {
+	case SNDRV_PCM_TRIGGER_START:
+		for (i = channels - 1; i >= 0; i--) {
+			struct sprd_compr_dma *dma = &stream->dma[i];
+
+			if (!dma->desc)
+				continue;
+
+			dma->cookie = dmaengine_submit(dma->desc);
+			ret = dma_submit_error(dma->cookie);
+			if (ret) {
+				dev_err(dev, "failed to submit request: %d\n",
+					ret);
+				return ret;
+			}
+		}
+
+		for (i = channels - 1; i >= 0; i--) {
+			struct sprd_compr_dma *dma = &stream->dma[i];
+
+			if (dma->chan)
+				dma_async_issue_pending(dma->chan);
+		}
+
+		ret = stream->compr_ops->start(stream_id);
+		break;
+
+	case SNDRV_PCM_TRIGGER_STOP:
+		for (i = channels - 1; i >= 0; i--) {
+			struct sprd_compr_dma *dma = &stream->dma[i];
+
+			if (dma->chan)
+				dmaengine_terminate_async(dma->chan);
+		}
+
+		stream->copied_total = 0;
+		stream->stage1_pointer  = 0;
+		stream->received_total = 0;
+		stream->received_stage0 = 0;
+		stream->received_stage1 = 0;
+
+		ret = stream->compr_ops->stop(stream_id);
+		break;
+
+	case SNDRV_PCM_TRIGGER_SUSPEND:
+	case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
+		for (i = channels - 1; i >= 0; i--) {
+			struct sprd_compr_dma *dma = &stream->dma[i];
+
+			if (dma->chan)
+				dmaengine_pause(dma->chan);
+		}
+
+		ret = stream->compr_ops->pause(stream_id);
+		break;
+
+	case SNDRV_PCM_TRIGGER_RESUME:
+	case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
+		for (i = channels - 1; i >= 0; i--) {
+			struct sprd_compr_dma *dma = &stream->dma[i];
+
+			if (dma->chan)
+				dmaengine_resume(dma->chan);
+		}
+
+		ret = stream->compr_ops->pause_release(stream_id);
+		break;
+
+	case SND_COMPR_TRIGGER_PARTIAL_DRAIN:
+	case SND_COMPR_TRIGGER_DRAIN:
+		ret = stream->compr_ops->drain(stream->received_total);
+		break;
+
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	return ret;
+}
+
+static int sprd_platform_compr_pointer(struct snd_compr_stream *cstream,
+				       struct snd_compr_tstamp *tstamp)
+{
+	struct snd_compr_runtime *runtime = cstream->runtime;
+	struct sprd_compr_stream *stream = runtime->private_data;
+	struct sprd_compr_playinfo *info =
+		(struct sprd_compr_playinfo *)stream->info_area;
+
+	tstamp->copied_total = stream->copied_total;
+	tstamp->pcm_io_frames = info->current_data_offset;
+
+	return 0;
+}
+
+static int sprd_platform_compr_copy(struct snd_compr_stream *cstream,
+				    char __user *buf, size_t count)
+{
+	struct snd_compr_runtime *runtime = cstream->runtime;
+	struct sprd_compr_stream *stream = runtime->private_data;
+	int avail_bytes, data_count = count;
+	void *dst;
+
+	/*
+	 * We usually set fragment size as 32K, and the stage 0 IRAM buffer
+	 * size is 32K too. So if now the received data size of the stage 0
+	 * IRAM buffer is less than 32K, that means we have some available
+	 * spaces for the stage 0 IRAM buffer.
+	 */
+	if (stream->received_stage0 < runtime->fragment_size) {
+		avail_bytes = runtime->fragment_size - stream->received_stage0;
+		dst = stream->iram_buffer.area + stream->received_stage0;
+
+		if (avail_bytes >= data_count) {
+			/*
+			 * Copy data to the stage 0 IRAM buffer directly if
+			 * spaces are enough.
+			 */
+			if (copy_from_user(dst, buf, data_count))
+				return -EFAULT;
+
+			stream->received_stage0 += data_count;
+			stream->copied_total += data_count;
+			goto copy_done;
+		} else {
+			/*
+			 * If the data count is larger than the available spaces
+			 * of the the stage 0 IRAM buffer, we should copy one
+			 * partial data to the stage 0 IRAM buffer, and copy
+			 * the left to the stage 1 DDR buffer.
+			 */
+			if (copy_from_user(dst, buf, avail_bytes))
+				return -EFAULT;
+
+			data_count -= avail_bytes;
+			stream->received_stage0 += avail_bytes;
+			stream->copied_total += avail_bytes;
+			buf += avail_bytes;
+		}
+	}
+
+	/*
+	 * Copy data to the stage 1 DDR buffer if no spaces for the stage 0 IRAM
+	 * buffer.
+	 */
+	dst = stream->compr_buffer.area + stream->stage1_pointer;
+	if (data_count < stream->compr_buffer.bytes - stream->stage1_pointer) {
+		if (copy_from_user(dst, buf, data_count))
+			return -EFAULT;
+
+		stream->stage1_pointer += data_count;
+	} else {
+		avail_bytes = stream->compr_buffer.bytes - stream->stage1_pointer;
+
+		if (copy_from_user(dst, buf, avail_bytes))
+			return -EFAULT;
+
+		if (copy_from_user(stream->compr_buffer.area, buf + avail_bytes,
+				   data_count - avail_bytes))
+			return -EFAULT;
+
+		stream->stage1_pointer = data_count - avail_bytes;
+	}
+
+	stream->received_stage1 += data_count;
+
+copy_done:
+	/* Update the copied data size. */
+	stream->received_total += count;
+	return count;
+}
+
+static int sprd_platform_compr_get_caps(struct snd_compr_stream *cstream,
+					struct snd_compr_caps *caps)
+{
+	caps->direction = cstream->direction;
+	caps->min_fragment_size = SPRD_COMPR_MIN_FRAGMENT_SIZE;
+	caps->max_fragment_size = SPRD_COMPR_MAX_FRAGMENT_SIZE;
+	caps->min_fragments = SPRD_COMPR_MIN_NUM_FRAGMENTS;
+	caps->max_fragments = SPRD_COMPR_MAX_NUM_FRAGMENTS;
+	caps->num_codecs = 2;
+	caps->codecs[0] = SND_AUDIOCODEC_MP3;
+	caps->codecs[1] = SND_AUDIOCODEC_AAC;
+
+	return 0;
+}
+
+static int
+sprd_platform_compr_get_codec_caps(struct snd_compr_stream *cstream,
+				   struct snd_compr_codec_caps *codec)
+{
+	switch (codec->codec) {
+	case SND_AUDIOCODEC_MP3:
+		codec->num_descriptors = 2;
+		codec->descriptor[0].max_ch = 2;
+		codec->descriptor[0].bit_rate[0] = 320;
+		codec->descriptor[0].bit_rate[1] = 128;
+		codec->descriptor[0].num_bitrates = 2;
+		codec->descriptor[0].profiles = 0;
+		codec->descriptor[0].modes = SND_AUDIOCHANMODE_MP3_STEREO;
+		codec->descriptor[0].formats = 0;
+		break;
+
+	case SND_AUDIOCODEC_AAC:
+		codec->num_descriptors = 2;
+		codec->descriptor[1].max_ch = 2;
+		codec->descriptor[1].bit_rate[0] = 320;
+		codec->descriptor[1].bit_rate[1] = 128;
+		codec->descriptor[1].num_bitrates = 2;
+		codec->descriptor[1].profiles = 0;
+		codec->descriptor[1].modes = 0;
+		codec->descriptor[1].formats = 0;
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+const struct snd_compr_ops sprd_platform_compr_ops = {
+	.open = sprd_platform_compr_open,
+	.free = sprd_platform_compr_free,
+	.set_params = sprd_platform_compr_set_params,
+	.trigger = sprd_platform_compr_trigger,
+	.pointer = sprd_platform_compr_pointer,
+	.copy = sprd_platform_compr_copy,
+	.get_caps = sprd_platform_compr_get_caps,
+	.get_codec_caps = sprd_platform_compr_get_codec_caps,
+};
+
+MODULE_DESCRIPTION("Spreadtrum ASoC Compress Platform Driver");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:compress-platform");
diff --git a/sound/soc/sprd/sprd-pcm-dma.c b/sound/soc/sprd/sprd-pcm-dma.c
index cbb27c4abeba..d38ebbbbf169 100644
--- a/sound/soc/sprd/sprd-pcm-dma.c
+++ b/sound/soc/sprd/sprd-pcm-dma.c
@@ -6,6 +6,7 @@
 #include <linux/dma/sprd-dma.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/of_reserved_mem.h>
 #include <linux/platform_device.h>
 #include <sound/pcm.h>
 #include <sound/pcm_params.h>
@@ -13,7 +14,6 @@
 
 #include "sprd-pcm-dma.h"
 
-#define DRV_NAME			"sprd_pcm_dma"
 #define SPRD_PCM_DMA_LINKLIST_SIZE	64
 #define SPRD_PCM_DMA_BRUST_LEN		640
 
@@ -524,14 +524,21 @@ static void sprd_pcm_free(struct snd_pcm *pcm)
 static const struct snd_soc_component_driver sprd_soc_component = {
 	.name		= DRV_NAME,
 	.ops		= &sprd_pcm_ops,
+	.compr_ops	= &sprd_platform_compr_ops,
 	.pcm_new	= sprd_pcm_new,
 	.pcm_free	= sprd_pcm_free,
 };
 
 static int sprd_soc_platform_probe(struct platform_device *pdev)
 {
+	struct device_node *np = pdev->dev.of_node;
 	int ret;
 
+	ret = of_reserved_mem_device_init_by_idx(&pdev->dev, np, 0);
+	if (ret)
+		dev_warn(&pdev->dev,
+			 "no reserved DMA memory for audio platform device\n");
+
 	ret = devm_snd_soc_register_component(&pdev->dev, &sprd_soc_component,
 					      NULL, 0);
 	if (ret)
diff --git a/sound/soc/sprd/sprd-pcm-dma.h b/sound/soc/sprd/sprd-pcm-dma.h
index d85a34f1461d..08e9fdba82f1 100644
--- a/sound/soc/sprd/sprd-pcm-dma.h
+++ b/sound/soc/sprd/sprd-pcm-dma.h
@@ -3,8 +3,11 @@
 #ifndef __SPRD_PCM_DMA_H
 #define __SPRD_PCM_DMA_H
 
+#define DRV_NAME		"sprd_pcm_dma"
 #define SPRD_PCM_CHANNEL_MAX	2
 
+extern const struct snd_compr_ops sprd_platform_compr_ops;
+
 struct sprd_pcm_dma_params {
 	dma_addr_t dev_phys[SPRD_PCM_CHANNEL_MAX];
 	u32 datawidth[SPRD_PCM_CHANNEL_MAX];
@@ -12,4 +15,44 @@ struct sprd_pcm_dma_params {
 	const char *chan_name[SPRD_PCM_CHANNEL_MAX];
 };
 
+struct sprd_compr_playinfo {
+	int total_time;
+	int current_time;
+	int total_data_length;
+	int current_data_offset;
+};
+
+struct sprd_compr_params {
+	u32 direction;
+	u32 rate;
+	u32 sample_rate;
+	u32 channels;
+	u32 format;
+	u32 period;
+	u32 periods;
+	u32 info_phys;
+	u32 info_size;
+};
+
+struct sprd_compr_callback {
+	void (*drain_notify)(void *data);
+	void *drain_data;
+};
+
+struct sprd_compr_ops {
+	int (*open)(int str_id, struct sprd_compr_callback *cb);
+	int (*close)(int str_id);
+	int (*start)(int str_id);
+	int (*stop)(int str_id);
+	int (*pause)(int str_id);
+	int (*pause_release)(int str_id);
+	int (*drain)(int received_total);
+	int (*set_params)(int str_id, struct sprd_compr_params *params);
+};
+
+struct sprd_compr_data {
+	struct sprd_compr_ops *ops;
+	struct sprd_pcm_dma_params *dma_params;
+};
+
 #endif /* __SPRD_PCM_DMA_H */
diff --git a/sound/soc/stm/stm32_adfsdm.c b/sound/soc/stm/stm32_adfsdm.c
index 78bed9734713..cc517e007039 100644
--- a/sound/soc/stm/stm32_adfsdm.c
+++ b/sound/soc/stm/stm32_adfsdm.c
@@ -44,7 +44,7 @@ struct stm32_adfsdm_priv {
 
 static const struct snd_pcm_hardware stm32_adfsdm_pcm_hw = {
 	.info = SNDRV_PCM_INFO_INTERLEAVED | SNDRV_PCM_INFO_BLOCK_TRANSFER |
-	    SNDRV_PCM_INFO_PAUSE,
+		SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_PAUSE,
 	.formats = SNDRV_PCM_FMTBIT_S32_LE,
 
 	.rate_min = 8000,
diff --git a/sound/soc/stm/stm32_i2s.c b/sound/soc/stm/stm32_i2s.c
index 8968458eec62..8ee697ff1f86 100644
--- a/sound/soc/stm/stm32_i2s.c
+++ b/sound/soc/stm/stm32_i2s.c
@@ -179,7 +179,6 @@ enum i2s_datlen {
 	I2S_I2SMOD_DATLEN_32,
 };
 
-#define STM32_I2S_DAI_NAME_SIZE		20
 #define STM32_I2S_FIFO_SIZE		16
 
 #define STM32_I2S_IS_MASTER(x)		((x)->ms_flg == I2S_MS_MASTER)
@@ -202,7 +201,6 @@ enum i2s_datlen {
  * @phys_addr: I2S registers physical base address
  * @lock_fd: lock to manage race conditions in full duplex mode
  * @irq_lock: prevent race condition with IRQ
- * @dais_name: DAI name
  * @mclk_rate: master clock frequency (Hz)
  * @fmt: DAI protocol
  * @refcount: keep count of opened streams on I2S
@@ -224,7 +222,6 @@ struct stm32_i2s_data {
 	dma_addr_t phys_addr;
 	spinlock_t lock_fd; /* Manage race conditions for full duplex */
 	spinlock_t irq_lock; /* used to prevent race condition with IRQ */
-	char dais_name[STM32_I2S_DAI_NAME_SIZE];
 	unsigned int mclk_rate;
 	unsigned int fmt;
 	int refcount;
@@ -495,12 +492,6 @@ static int stm32_i2s_configure(struct snd_soc_dai *cpu_dai,
 	unsigned int fthlv;
 	int ret;
 
-	if ((params_channels(params) == 1) &&
-	    ((i2s->fmt & SND_SOC_DAIFMT_FORMAT_MASK) != SND_SOC_DAIFMT_DSP_A)) {
-		dev_err(cpu_dai->dev, "Mono mode supported only by DSP_A\n");
-		return -EINVAL;
-	}
-
 	switch (format) {
 	case 16:
 		cfgr = I2S_CGFR_DATLEN_SET(I2S_I2SMOD_DATLEN_16);
@@ -550,6 +541,10 @@ static int stm32_i2s_startup(struct snd_pcm_substream *substream,
 	i2s->substream = substream;
 	spin_unlock_irqrestore(&i2s->irq_lock, flags);
 
+	if ((i2s->fmt & SND_SOC_DAIFMT_FORMAT_MASK) != SND_SOC_DAIFMT_DSP_A)
+		snd_pcm_hw_constraint_single(substream->runtime,
+					     SNDRV_PCM_HW_PARAM_CHANNELS, 2);
+
 	ret = clk_prepare_enable(i2s->i2sclk);
 	if (ret < 0) {
 		dev_err(cpu_dai->dev, "Failed to enable clock: %d\n", ret);
@@ -592,7 +587,8 @@ static int stm32_i2s_trigger(struct snd_pcm_substream *substream, int cmd,
 	case SNDRV_PCM_TRIGGER_RESUME:
 	case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
 		/* Enable i2s */
-		dev_dbg(cpu_dai->dev, "start I2S\n");
+		dev_dbg(cpu_dai->dev, "start I2S %s\n",
+			playback_flg ? "playback" : "capture");
 
 		cfg1_mask = I2S_CFG1_RXDMAEN | I2S_CFG1_TXDMAEN;
 		regmap_update_bits(i2s->regmap, STM32_I2S_CFG1_REG,
@@ -637,6 +633,9 @@ static int stm32_i2s_trigger(struct snd_pcm_substream *substream, int cmd,
 	case SNDRV_PCM_TRIGGER_STOP:
 	case SNDRV_PCM_TRIGGER_SUSPEND:
 	case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
+		dev_dbg(cpu_dai->dev, "stop I2S %s\n",
+			playback_flg ? "playback" : "capture");
+
 		if (playback_flg)
 			regmap_update_bits(i2s->regmap, STM32_I2S_IER_REG,
 					   I2S_IER_UDRIE,
@@ -653,8 +652,6 @@ static int stm32_i2s_trigger(struct snd_pcm_substream *substream, int cmd,
 			break;
 		}
 
-		dev_dbg(cpu_dai->dev, "stop I2S\n");
-
 		ret = regmap_update_bits(i2s->regmap, STM32_I2S_CR1_REG,
 					 I2S_CR1_SPE, 0);
 		if (ret < 0) {
@@ -734,7 +731,8 @@ static const struct snd_soc_dai_ops stm32_i2s_pcm_dai_ops = {
 static const struct snd_pcm_hardware stm32_i2s_pcm_hw = {
 	.info = SNDRV_PCM_INFO_INTERLEAVED | SNDRV_PCM_INFO_MMAP,
 	.buffer_bytes_max = 8 * PAGE_SIZE,
-	.period_bytes_max = 2048,
+	.period_bytes_min = 1024,
+	.period_bytes_max = 4 * PAGE_SIZE,
 	.periods_min = 2,
 	.periods_max = 8,
 };
@@ -770,12 +768,8 @@ static int stm32_i2s_dais_init(struct platform_device *pdev,
 	if (!dai_ptr)
 		return -ENOMEM;
 
-	snprintf(i2s->dais_name, STM32_I2S_DAI_NAME_SIZE,
-		 "%s", dev_name(&pdev->dev));
-
 	dai_ptr->probe = stm32_i2s_dai_probe;
 	dai_ptr->ops = &stm32_i2s_pcm_dai_ops;
-	dai_ptr->name = i2s->dais_name;
 	dai_ptr->id = 1;
 	stm32_i2s_dai_init(&dai_ptr->playback, "playback");
 	stm32_i2s_dai_init(&dai_ptr->capture, "capture");
@@ -845,8 +839,9 @@ static int stm32_i2s_parse_dt(struct platform_device *pdev,
 	/* Get irqs */
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0) {
-		dev_err(&pdev->dev, "no irq for node %s\n", pdev->name);
-		return -ENOENT;
+		if (irq != -EPROBE_DEFER)
+			dev_err(&pdev->dev, "no irq for node %s\n", pdev->name);
+		return irq;
 	}
 
 	ret = devm_request_irq(&pdev->dev, irq, stm32_i2s_isr, IRQF_ONESHOT,
diff --git a/sound/soc/stm/stm32_sai.c b/sound/soc/stm/stm32_sai.c
index d68d62f12df5..7550d5f08be3 100644
--- a/sound/soc/stm/stm32_sai.c
+++ b/sound/soc/stm/stm32_sai.c
@@ -21,6 +21,7 @@
 #include <linux/delay.h>
 #include <linux/module.h>
 #include <linux/of_platform.h>
+#include <linux/pinctrl/consumer.h>
 #include <linux/reset.h>
 
 #include <sound/dmaengine_pcm.h>
@@ -44,20 +45,41 @@ static const struct of_device_id stm32_sai_ids[] = {
 	{}
 };
 
-static int stm32_sai_sync_conf_client(struct stm32_sai_data *sai, int synci)
+static int stm32_sai_pclk_disable(struct device *dev)
+{
+	struct stm32_sai_data *sai = dev_get_drvdata(dev);
+
+	clk_disable_unprepare(sai->pclk);
+
+	return 0;
+}
+
+static int stm32_sai_pclk_enable(struct device *dev)
 {
+	struct stm32_sai_data *sai = dev_get_drvdata(dev);
 	int ret;
 
-	/* Enable peripheral clock to allow GCR register access */
 	ret = clk_prepare_enable(sai->pclk);
 	if (ret) {
 		dev_err(&sai->pdev->dev, "failed to enable clock: %d\n", ret);
 		return ret;
 	}
 
+	return 0;
+}
+
+static int stm32_sai_sync_conf_client(struct stm32_sai_data *sai, int synci)
+{
+	int ret;
+
+	/* Enable peripheral clock to allow GCR register access */
+	ret = stm32_sai_pclk_enable(&sai->pdev->dev);
+	if (ret)
+		return ret;
+
 	writel_relaxed(FIELD_PREP(SAI_GCR_SYNCIN_MASK, (synci - 1)), sai->base);
 
-	clk_disable_unprepare(sai->pclk);
+	stm32_sai_pclk_disable(&sai->pdev->dev);
 
 	return 0;
 }
@@ -68,11 +90,9 @@ static int stm32_sai_sync_conf_provider(struct stm32_sai_data *sai, int synco)
 	int ret;
 
 	/* Enable peripheral clock to allow GCR register access */
-	ret = clk_prepare_enable(sai->pclk);
-	if (ret) {
-		dev_err(&sai->pdev->dev, "failed to enable clock: %d\n", ret);
+	ret = stm32_sai_pclk_enable(&sai->pdev->dev);
+	if (ret)
 		return ret;
-	}
 
 	dev_dbg(&sai->pdev->dev, "Set %pOFn%s as synchro provider\n",
 		sai->pdev->dev.of_node,
@@ -83,13 +103,13 @@ static int stm32_sai_sync_conf_provider(struct stm32_sai_data *sai, int synco)
 		dev_err(&sai->pdev->dev, "%pOFn%s already set as sync provider\n",
 			sai->pdev->dev.of_node,
 			prev_synco == STM_SAI_SYNC_OUT_A ? "A" : "B");
-		clk_disable_unprepare(sai->pclk);
+			stm32_sai_pclk_disable(&sai->pdev->dev);
 		return -EINVAL;
 	}
 
 	writel_relaxed(FIELD_PREP(SAI_GCR_SYNCOUT_MASK, synco), sai->base);
 
-	clk_disable_unprepare(sai->pclk);
+	stm32_sai_pclk_disable(&sai->pdev->dev);
 
 	return 0;
 }
@@ -195,12 +215,54 @@ static int stm32_sai_probe(struct platform_device *pdev)
 	return devm_of_platform_populate(&pdev->dev);
 }
 
+#ifdef CONFIG_PM_SLEEP
+/*
+ * When pins are shared by two sai sub instances, pins have to be defined
+ * in sai parent node. In this case, pins state is not managed by alsa fw.
+ * These pins are managed in suspend/resume callbacks.
+ */
+static int stm32_sai_suspend(struct device *dev)
+{
+	struct stm32_sai_data *sai = dev_get_drvdata(dev);
+	int ret;
+
+	ret = stm32_sai_pclk_enable(dev);
+	if (ret)
+		return ret;
+
+	sai->gcr = readl_relaxed(sai->base);
+	stm32_sai_pclk_disable(dev);
+
+	return pinctrl_pm_select_sleep_state(dev);
+}
+
+static int stm32_sai_resume(struct device *dev)
+{
+	struct stm32_sai_data *sai = dev_get_drvdata(dev);
+	int ret;
+
+	ret = stm32_sai_pclk_enable(dev);
+	if (ret)
+		return ret;
+
+	writel_relaxed(sai->gcr, sai->base);
+	stm32_sai_pclk_disable(dev);
+
+	return pinctrl_pm_select_default_state(dev);
+}
+#endif /* CONFIG_PM_SLEEP */
+
+static const struct dev_pm_ops stm32_sai_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(stm32_sai_suspend, stm32_sai_resume)
+};
+
 MODULE_DEVICE_TABLE(of, stm32_sai_ids);
 
 static struct platform_driver stm32_sai_driver = {
 	.driver = {
 		.name = "st,stm32-sai",
 		.of_match_table = stm32_sai_ids,
+		.pm = &stm32_sai_pm_ops,
 	},
 	.probe = stm32_sai_probe,
 };
diff --git a/sound/soc/stm/stm32_sai.h b/sound/soc/stm/stm32_sai.h
index 08de899c766b..9c36a393ab7b 100644
--- a/sound/soc/stm/stm32_sai.h
+++ b/sound/soc/stm/stm32_sai.h
@@ -268,6 +268,7 @@ struct stm32_sai_conf {
  * @version: SOC version
  * @irq: SAI interrupt line
  * @set_sync: pointer to synchro mode configuration callback
+ * @gcr: SAI Global Configuration Register
  */
 struct stm32_sai_data {
 	struct platform_device *pdev;
@@ -279,4 +280,5 @@ struct stm32_sai_data {
 	int irq;
 	int (*set_sync)(struct stm32_sai_data *sai,
 			struct device_node *np_provider, int synco, int synci);
+	u32 gcr;
 };
diff --git a/sound/soc/stm/stm32_sai_sub.c b/sound/soc/stm/stm32_sai_sub.c
index d7045aa520de..2a74ce7c9440 100644
--- a/sound/soc/stm/stm32_sai_sub.c
+++ b/sound/soc/stm/stm32_sai_sub.c
@@ -110,7 +110,7 @@ struct stm32_sai_sub_data {
 	struct regmap *regmap;
 	const struct regmap_config *regmap_config;
 	struct snd_dmaengine_dai_dma_data dma_params;
-	struct snd_soc_dai_driver *cpu_dai_drv;
+	struct snd_soc_dai_driver cpu_dai_drv;
 	struct snd_soc_dai *cpu_dai;
 	struct snd_pcm_substream *substream;
 	struct stm32_sai_data *pdata;
@@ -169,6 +169,7 @@ static bool stm32_sai_sub_volatile_reg(struct device *dev, unsigned int reg)
 {
 	switch (reg) {
 	case STM_SAI_DR_REGX:
+	case STM_SAI_SR_REGX:
 		return true;
 	default:
 		return false;
@@ -183,7 +184,6 @@ static bool stm32_sai_sub_writeable_reg(struct device *dev, unsigned int reg)
 	case STM_SAI_FRCR_REGX:
 	case STM_SAI_SLOTR_REGX:
 	case STM_SAI_IMR_REGX:
-	case STM_SAI_SR_REGX:
 	case STM_SAI_CLRFR_REGX:
 	case STM_SAI_DR_REGX:
 	case STM_SAI_PDMCR_REGX:
@@ -203,6 +203,7 @@ static const struct regmap_config stm32_sai_sub_regmap_config_f4 = {
 	.volatile_reg = stm32_sai_sub_volatile_reg,
 	.writeable_reg = stm32_sai_sub_writeable_reg,
 	.fast_io = true,
+	.cache_type = REGCACHE_FLAT,
 };
 
 static const struct regmap_config stm32_sai_sub_regmap_config_h7 = {
@@ -214,6 +215,7 @@ static const struct regmap_config stm32_sai_sub_regmap_config_h7 = {
 	.volatile_reg = stm32_sai_sub_volatile_reg,
 	.writeable_reg = stm32_sai_sub_writeable_reg,
 	.fast_io = true,
+	.cache_type = REGCACHE_FLAT,
 };
 
 static int snd_pcm_iec958_info(struct snd_kcontrol *kcontrol,
@@ -461,8 +463,8 @@ static irqreturn_t stm32_sai_isr(int irq, void *devid)
 	if (!flags)
 		return IRQ_NONE;
 
-	regmap_update_bits(sai->regmap, STM_SAI_CLRFR_REGX, SAI_XCLRFR_MASK,
-			   SAI_XCLRFR_MASK);
+	regmap_write_bits(sai->regmap, STM_SAI_CLRFR_REGX, SAI_XCLRFR_MASK,
+			  SAI_XCLRFR_MASK);
 
 	if (!sai->substream) {
 		dev_err(&pdev->dev, "Device stopped. Spurious IRQ 0x%x\n", sr);
@@ -728,9 +730,8 @@ static int stm32_sai_startup(struct snd_pcm_substream *substream,
 	}
 
 	/* Enable ITs */
-
-	regmap_update_bits(sai->regmap, STM_SAI_CLRFR_REGX,
-			   SAI_XCLRFR_MASK, SAI_XCLRFR_MASK);
+	regmap_write_bits(sai->regmap, STM_SAI_CLRFR_REGX,
+			  SAI_XCLRFR_MASK, SAI_XCLRFR_MASK);
 
 	imr = SAI_XIMR_OVRUDRIE;
 	if (STM_SAI_IS_CAPTURE(sai)) {
@@ -762,10 +763,10 @@ static int stm32_sai_set_config(struct snd_soc_dai *cpu_dai,
 	 * SAI fifo threshold is set to half fifo, to keep enough space
 	 * for DMA incoming bursts.
 	 */
-	regmap_update_bits(sai->regmap, STM_SAI_CR2_REGX,
-			   SAI_XCR2_FFLUSH | SAI_XCR2_FTH_MASK,
-			   SAI_XCR2_FFLUSH |
-			   SAI_XCR2_FTH_SET(STM_SAI_FIFO_TH_HALF));
+	regmap_write_bits(sai->regmap, STM_SAI_CR2_REGX,
+			  SAI_XCR2_FFLUSH | SAI_XCR2_FTH_MASK,
+			  SAI_XCR2_FFLUSH |
+			  SAI_XCR2_FTH_SET(STM_SAI_FIFO_TH_HALF));
 
 	/* DS bits in CR1 not set for SPDIF (size forced to 24 bits).*/
 	if (STM_SAI_PROTOCOL_IS_SPDIF(sai)) {
@@ -1233,8 +1234,7 @@ static const struct snd_pcm_hardware stm32_sai_pcm_hw = {
 	.periods_max = 8,
 };
 
-static struct snd_soc_dai_driver stm32_sai_playback_dai[] = {
-{
+static struct snd_soc_dai_driver stm32_sai_playback_dai = {
 		.probe = stm32_sai_dai_probe,
 		.pcm_new = stm32_sai_pcm_new,
 		.id = 1, /* avoid call to fmt_single_name() */
@@ -1251,11 +1251,9 @@ static struct snd_soc_dai_driver stm32_sai_playback_dai[] = {
 				SNDRV_PCM_FMTBIT_S32_LE,
 		},
 		.ops = &stm32_sai_pcm_dai_ops,
-	}
 };
 
-static struct snd_soc_dai_driver stm32_sai_capture_dai[] = {
-{
+static struct snd_soc_dai_driver stm32_sai_capture_dai = {
 		.probe = stm32_sai_dai_probe,
 		.id = 1, /* avoid call to fmt_single_name() */
 		.capture = {
@@ -1271,7 +1269,6 @@ static struct snd_soc_dai_driver stm32_sai_capture_dai[] = {
 				SNDRV_PCM_FMTBIT_S32_LE,
 		},
 		.ops = &stm32_sai_pcm_dai_ops,
-	}
 };
 
 static const struct snd_dmaengine_pcm_config stm32_sai_pcm_config = {
@@ -1440,29 +1437,6 @@ static int stm32_sai_sub_parse_of(struct platform_device *pdev,
 	return 0;
 }
 
-static int stm32_sai_sub_dais_init(struct platform_device *pdev,
-				   struct stm32_sai_sub_data *sai)
-{
-	sai->cpu_dai_drv = devm_kzalloc(&pdev->dev,
-					sizeof(struct snd_soc_dai_driver),
-					GFP_KERNEL);
-	if (!sai->cpu_dai_drv)
-		return -ENOMEM;
-
-	if (STM_SAI_IS_PLAYBACK(sai)) {
-		memcpy(sai->cpu_dai_drv, &stm32_sai_playback_dai,
-		       sizeof(stm32_sai_playback_dai));
-		sai->cpu_dai_drv->playback.stream_name = sai->cpu_dai_drv->name;
-	} else {
-		memcpy(sai->cpu_dai_drv, &stm32_sai_capture_dai,
-		       sizeof(stm32_sai_capture_dai));
-		sai->cpu_dai_drv->capture.stream_name = sai->cpu_dai_drv->name;
-	}
-	sai->cpu_dai_drv->name = dev_name(&pdev->dev);
-
-	return 0;
-}
-
 static int stm32_sai_sub_probe(struct platform_device *pdev)
 {
 	struct stm32_sai_sub_data *sai;
@@ -1494,9 +1468,11 @@ static int stm32_sai_sub_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	ret = stm32_sai_sub_dais_init(pdev, sai);
-	if (ret)
-		return ret;
+	if (STM_SAI_IS_PLAYBACK(sai))
+		sai->cpu_dai_drv = stm32_sai_playback_dai;
+	else
+		sai->cpu_dai_drv = stm32_sai_capture_dai;
+	sai->cpu_dai_drv.name = dev_name(&pdev->dev);
 
 	ret = devm_request_irq(&pdev->dev, sai->pdata->irq, stm32_sai_isr,
 			       IRQF_SHARED, dev_name(&pdev->dev), sai);
@@ -1506,7 +1482,7 @@ static int stm32_sai_sub_probe(struct platform_device *pdev)
 	}
 
 	ret = devm_snd_soc_register_component(&pdev->dev, &stm32_component,
-					      sai->cpu_dai_drv, 1);
+					      &sai->cpu_dai_drv, 1);
 	if (ret)
 		return ret;
 
@@ -1522,10 +1498,34 @@ static int stm32_sai_sub_probe(struct platform_device *pdev)
 	return 0;
 }
 
+#ifdef CONFIG_PM_SLEEP
+static int stm32_sai_sub_suspend(struct device *dev)
+{
+	struct stm32_sai_sub_data *sai = dev_get_drvdata(dev);
+
+	regcache_cache_only(sai->regmap, true);
+	regcache_mark_dirty(sai->regmap);
+	return 0;
+}
+
+static int stm32_sai_sub_resume(struct device *dev)
+{
+	struct stm32_sai_sub_data *sai = dev_get_drvdata(dev);
+
+	regcache_cache_only(sai->regmap, false);
+	return regcache_sync(sai->regmap);
+}
+#endif /* CONFIG_PM_SLEEP */
+
+static const struct dev_pm_ops stm32_sai_sub_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(stm32_sai_sub_suspend, stm32_sai_sub_resume)
+};
+
 static struct platform_driver stm32_sai_sub_driver = {
 	.driver = {
 		.name = "st,stm32-sai-sub",
 		.of_match_table = stm32_sai_sub_ids,
+		.pm = &stm32_sai_sub_pm_ops,
 	},
 	.probe = stm32_sai_sub_probe,
 };
diff --git a/sound/soc/stm/stm32_spdifrx.c b/sound/soc/stm/stm32_spdifrx.c
index 373df4f24be1..3d64200edbb5 100644
--- a/sound/soc/stm/stm32_spdifrx.c
+++ b/sound/soc/stm/stm32_spdifrx.c
@@ -21,6 +21,7 @@
 #include <linux/delay.h>
 #include <linux/module.h>
 #include <linux/of_platform.h>
+#include <linux/pinctrl/consumer.h>
 #include <linux/regmap.h>
 #include <linux/reset.h>
 
@@ -471,6 +472,8 @@ static int stm32_spdifrx_get_ctrl_data(struct stm32_spdifrx_data *spdifrx)
 	memset(spdifrx->cs, 0, SPDIFRX_CS_BYTES_NB);
 	memset(spdifrx->ub, 0, SPDIFRX_UB_BYTES_NB);
 
+	pinctrl_pm_select_default_state(&spdifrx->pdev->dev);
+
 	ret = stm32_spdifrx_dma_ctrl_start(spdifrx);
 	if (ret < 0)
 		return ret;
@@ -493,7 +496,7 @@ static int stm32_spdifrx_get_ctrl_data(struct stm32_spdifrx_data *spdifrx)
 	if (wait_for_completion_interruptible_timeout(&spdifrx->cs_completion,
 						      msecs_to_jiffies(100))
 						      <= 0) {
-		dev_err(&spdifrx->pdev->dev, "Failed to get control data\n");
+		dev_dbg(&spdifrx->pdev->dev, "Failed to get control data\n");
 		ret = -EAGAIN;
 	}
 
@@ -502,6 +505,7 @@ static int stm32_spdifrx_get_ctrl_data(struct stm32_spdifrx_data *spdifrx)
 
 end:
 	clk_disable_unprepare(spdifrx->kclk);
+	pinctrl_pm_select_sleep_state(&spdifrx->pdev->dev);
 
 	return ret;
 }
@@ -611,10 +615,15 @@ static bool stm32_spdifrx_readable_reg(struct device *dev, unsigned int reg)
 
 static bool stm32_spdifrx_volatile_reg(struct device *dev, unsigned int reg)
 {
-	if (reg == STM32_SPDIFRX_DR)
+	switch (reg) {
+	case STM32_SPDIFRX_DR:
+	case STM32_SPDIFRX_CSR:
+	case STM32_SPDIFRX_SR:
+	case STM32_SPDIFRX_DIR:
 		return true;
-
-	return false;
+	default:
+		return false;
+	}
 }
 
 static bool stm32_spdifrx_writeable_reg(struct device *dev, unsigned int reg)
@@ -638,6 +647,7 @@ static const struct regmap_config stm32_h7_spdifrx_regmap_conf = {
 	.volatile_reg = stm32_spdifrx_volatile_reg,
 	.writeable_reg = stm32_spdifrx_writeable_reg,
 	.fast_io = true,
+	.cache_type = REGCACHE_FLAT,
 };
 
 static irqreturn_t stm32_spdifrx_isr(int irq, void *devid)
@@ -835,7 +845,8 @@ static struct snd_soc_dai_driver stm32_spdifrx_dai[] = {
 static const struct snd_pcm_hardware stm32_spdifrx_pcm_hw = {
 	.info = SNDRV_PCM_INFO_INTERLEAVED | SNDRV_PCM_INFO_MMAP,
 	.buffer_bytes_max = 8 * PAGE_SIZE,
-	.period_bytes_max = 2048, /* MDMA constraint */
+	.period_bytes_min = 1024,
+	.period_bytes_max = 4 * PAGE_SIZE,
 	.periods_min = 2,
 	.periods_max = 8,
 };
@@ -983,10 +994,36 @@ static int stm32_spdifrx_remove(struct platform_device *pdev)
 
 MODULE_DEVICE_TABLE(of, stm32_spdifrx_ids);
 
+#ifdef CONFIG_PM_SLEEP
+static int stm32_spdifrx_suspend(struct device *dev)
+{
+	struct stm32_spdifrx_data *spdifrx = dev_get_drvdata(dev);
+
+	regcache_cache_only(spdifrx->regmap, true);
+	regcache_mark_dirty(spdifrx->regmap);
+
+	return 0;
+}
+
+static int stm32_spdifrx_resume(struct device *dev)
+{
+	struct stm32_spdifrx_data *spdifrx = dev_get_drvdata(dev);
+
+	regcache_cache_only(spdifrx->regmap, false);
+
+	return regcache_sync(spdifrx->regmap);
+}
+#endif /* CONFIG_PM_SLEEP */
+
+static const struct dev_pm_ops stm32_spdifrx_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(stm32_spdifrx_suspend, stm32_spdifrx_resume)
+};
+
 static struct platform_driver stm32_spdifrx_driver = {
 	.driver = {
 		.name = "st,stm32-spdifrx",
 		.of_match_table = stm32_spdifrx_ids,
+		.pm = &stm32_spdifrx_pm_ops,
 	},
 	.probe = stm32_spdifrx_probe,
 	.remove = stm32_spdifrx_remove,
diff --git a/sound/soc/ti/Kconfig b/sound/soc/ti/Kconfig
index 4bf3c15d4e51..ee7c202c69b7 100644
--- a/sound/soc/ti/Kconfig
+++ b/sound/soc/ti/Kconfig
@@ -21,8 +21,8 @@ config SND_SOC_DAVINCI_ASP
 
 config SND_SOC_DAVINCI_MCASP
 	tristate "Multichannel Audio Serial Port (McASP) support"
-	select SND_SOC_TI_EDMA_PCM if TI_EDMA
-	select SND_SOC_TI_SDMA_PCM if DMA_OMAP
+	select SND_SOC_TI_EDMA_PCM
+	select SND_SOC_TI_SDMA_PCM
 	help
 	  Say Y or M here if you want to have support for McASP IP found in
 	  various Texas Instruments SoCs like:
diff --git a/sound/soc/ti/ams-delta.c b/sound/soc/ti/ams-delta.c
index 4dce494dfbd3..b9611db14c86 100644
--- a/sound/soc/ti/ams-delta.c
+++ b/sound/soc/ti/ams-delta.c
@@ -200,7 +200,7 @@ static int ams_delta_get_audio_mode(struct snd_kcontrol *kcontrol,
 	return 0;
 }
 
-static const SOC_ENUM_SINGLE_EXT_DECL(ams_delta_audio_enum,
+static SOC_ENUM_SINGLE_EXT_DECL(ams_delta_audio_enum,
 				      ams_delta_audio_mode);
 
 static const struct snd_kcontrol_new ams_delta_audio_controls[] = {
diff --git a/sound/soc/ti/davinci-mcasp.c b/sound/soc/ti/davinci-mcasp.c
index a3a67a8f0f54..9fbc759fdefe 100644
--- a/sound/soc/ti/davinci-mcasp.c
+++ b/sound/soc/ti/davinci-mcasp.c
@@ -45,6 +45,7 @@
 
 #define MCASP_MAX_AFIFO_DEPTH	64
 
+#ifdef CONFIG_PM
 static u32 context_regs[] = {
 	DAVINCI_MCASP_TXFMCTL_REG,
 	DAVINCI_MCASP_RXFMCTL_REG,
@@ -68,6 +69,7 @@ struct davinci_mcasp_context {
 	u32	*xrsr_regs; /* for serializer configuration */
 	bool	pm_state;
 };
+#endif
 
 struct davinci_mcasp_ruledata {
 	struct davinci_mcasp *mcasp;
diff --git a/sound/soc/ti/edma-pcm.c b/sound/soc/ti/edma-pcm.c
index 59e588abe54b..fdffb801b185 100644
--- a/sound/soc/ti/edma-pcm.c
+++ b/sound/soc/ti/edma-pcm.c
@@ -23,7 +23,6 @@
 #include <sound/pcm_params.h>
 #include <sound/soc.h>
 #include <sound/dmaengine_pcm.h>
-#include <linux/edma.h>
 
 #include "edma-pcm.h"
 
@@ -43,14 +42,12 @@ static const struct snd_pcm_hardware edma_pcm_hardware = {
 static const struct snd_dmaengine_pcm_config edma_dmaengine_pcm_config = {
 	.pcm_hardware = &edma_pcm_hardware,
 	.prepare_slave_config = snd_dmaengine_pcm_prepare_slave_config,
-	.compat_filter_fn = edma_filter_fn,
 	.prealloc_buffer_size = 128 * 1024,
 };
 
 int edma_pcm_platform_register(struct device *dev)
 {
-	return devm_snd_dmaengine_pcm_register(dev, &edma_dmaengine_pcm_config,
-					SND_DMAENGINE_PCM_FLAG_COMPAT);
+	return devm_snd_dmaengine_pcm_register(dev, &edma_dmaengine_pcm_config, 0);
 }
 EXPORT_SYMBOL_GPL(edma_pcm_platform_register);
 
diff --git a/sound/soc/ti/sdma-pcm.c b/sound/soc/ti/sdma-pcm.c
index 21a9c2499d48..a236350beb10 100644
--- a/sound/soc/ti/sdma-pcm.c
+++ b/sound/soc/ti/sdma-pcm.c
@@ -11,7 +11,6 @@
 #include <sound/pcm_params.h>
 #include <sound/soc.h>
 #include <sound/dmaengine_pcm.h>
-#include <linux/omap-dmaengine.h>
 
 #include "sdma-pcm.h"
 
@@ -31,7 +30,6 @@ static const struct snd_pcm_hardware sdma_pcm_hardware = {
 static const struct snd_dmaengine_pcm_config sdma_dmaengine_pcm_config = {
 	.pcm_hardware = &sdma_pcm_hardware,
 	.prepare_slave_config = snd_dmaengine_pcm_prepare_slave_config,
-	.compat_filter_fn = omap_dma_filter_fn,
 	.prealloc_buffer_size = 128 * 1024,
 };
 
@@ -39,13 +37,12 @@ int sdma_pcm_platform_register(struct device *dev,
 			       char *txdmachan, char *rxdmachan)
 {
 	struct snd_dmaengine_pcm_config *config;
-	unsigned int flags = SND_DMAENGINE_PCM_FLAG_COMPAT;
+	unsigned int flags = 0;
 
 	/* Standard names for the directions: 'tx' and 'rx' */
 	if (!txdmachan && !rxdmachan)
 		return devm_snd_dmaengine_pcm_register(dev,
-						&sdma_dmaengine_pcm_config,
-						flags);
+						&sdma_dmaengine_pcm_config, 0);
 
 	config = devm_kzalloc(dev, sizeof(*config), GFP_KERNEL);
 	if (!config)
@@ -65,7 +62,7 @@ int sdma_pcm_platform_register(struct device *dev,
 	config->chan_names[0] = txdmachan;
 	config->chan_names[1] = rxdmachan;
 
-	return devm_snd_dmaengine_pcm_register(dev, config, flags);
+	return devm_snd_dmaengine_pcm_register(dev, config, 0);
 }
 EXPORT_SYMBOL_GPL(sdma_pcm_platform_register);
 
diff --git a/sound/synth/emux/emux_hwdep.c b/sound/synth/emux/emux_hwdep.c
index d9fcae071b47..fae48d108b97 100644
--- a/sound/synth/emux/emux_hwdep.c
+++ b/sound/synth/emux/emux_hwdep.c
@@ -39,6 +39,11 @@ snd_emux_hwdep_load_patch(struct snd_emux *emu, void __user *arg)
 	if (copy_from_user(&patch, arg, sizeof(patch)))
 		return -EFAULT;
 
+	if (patch.key == GUS_PATCH)
+		return snd_soundfont_load_guspatch(emu->sflist, arg,
+						   patch.len + sizeof(patch),
+						   TMP_CLIENT_ID);
+
 	if (patch.type >= SNDRV_SFNT_LOAD_INFO &&
 	    patch.type <= SNDRV_SFNT_PROBE_DATA) {
 		err = snd_soundfont_load(emu->sflist, arg, patch.len + sizeof(patch), TMP_CLIENT_ID);
diff --git a/sound/synth/emux/soundfont.c b/sound/synth/emux/soundfont.c
index 31a4ea94830e..9b5d70104489 100644
--- a/sound/synth/emux/soundfont.c
+++ b/sound/synth/emux/soundfont.c
@@ -856,6 +856,8 @@ calc_gus_envelope_time(int rate, int start, int end)
 	int r, p, t;
 	r = (3 - ((rate >> 6) & 3)) * 3;
 	p = rate & 0x3f;
+	if (!p)
+		p = 1;
 	t = end - start;
 	if (t < 0) t = -t;
 	if (13 > r)
diff --git a/sound/usb/Kconfig b/sound/usb/Kconfig
index f61b5662bb89..6319b544ba3a 100644
--- a/sound/usb/Kconfig
+++ b/sound/usb/Kconfig
@@ -15,6 +15,7 @@ config SND_USB_AUDIO
 	select SND_RAWMIDI
 	select SND_PCM
 	select BITREVERSE
+	select SND_USB_AUDIO_USE_MEDIA_CONTROLLER if MEDIA_CONTROLLER && (MEDIA_SUPPORT=y || MEDIA_SUPPORT=SND_USB_AUDIO)
 	help
 	  Say Y here to include support for USB audio and USB MIDI
 	  devices.
@@ -22,6 +23,9 @@ config SND_USB_AUDIO
 	  To compile this driver as a module, choose M here: the module
 	  will be called snd-usb-audio.
 
+config SND_USB_AUDIO_USE_MEDIA_CONTROLLER
+	bool
+
 config SND_USB_UA101
 	tristate "Edirol UA-101/UA-1000 driver"
 	select SND_PCM
diff --git a/sound/usb/Makefile b/sound/usb/Makefile
index d330f74c90e6..e1ce257ab705 100644
--- a/sound/usb/Makefile
+++ b/sound/usb/Makefile
@@ -18,6 +18,8 @@ snd-usb-audio-objs := 	card.o \
 			quirks.o \
 			stream.o
 
+snd-usb-audio-$(CONFIG_SND_USB_AUDIO_USE_MEDIA_CONTROLLER) += media.o
+
 snd-usbmidi-lib-objs := midi.o
 
 # Toplevel Module Dependency
diff --git a/sound/usb/card.c b/sound/usb/card.c
index 719e10034553..04465d581204 100644
--- a/sound/usb/card.c
+++ b/sound/usb/card.c
@@ -68,6 +68,7 @@
 #include "format.h"
 #include "power.h"
 #include "stream.h"
+#include "media.h"
 
 MODULE_AUTHOR("Takashi Iwai <tiwai@suse.de>");
 MODULE_DESCRIPTION("USB Audio");
@@ -673,6 +674,11 @@ static int usb_audio_probe(struct usb_interface *intf,
 	if (err < 0)
 		goto __error;
 
+	if (quirk && quirk->shares_media_device) {
+		/* don't want to fail when snd_media_device_create() fails */
+		snd_media_device_create(chip, intf);
+	}
+
 	usb_chip[chip->index] = chip;
 	chip->num_interfaces++;
 	usb_set_intfdata(intf, chip);
@@ -732,6 +738,14 @@ static void usb_audio_disconnect(struct usb_interface *intf)
 		list_for_each(p, &chip->midi_list) {
 			snd_usbmidi_disconnect(p);
 		}
+		/*
+		 * Nice to check quirk && quirk->shares_media_device and
+		 * then call the snd_media_device_delete(). Don't have
+		 * access to the quirk here. snd_media_device_delete()
+		 * accesses mixer_list
+		 */
+		snd_media_device_delete(chip);
+
 		/* release mixer resources */
 		list_for_each_entry(mixer, &chip->mixer_list, list) {
 			snd_usb_mixer_disconnect(mixer);
diff --git a/sound/usb/card.h b/sound/usb/card.h
index 79fa2a19fb7b..2991b9986f66 100644
--- a/sound/usb/card.h
+++ b/sound/usb/card.h
@@ -109,6 +109,8 @@ struct snd_usb_endpoint {
 	struct list_head list;
 };
 
+struct media_ctl;
+
 struct snd_usb_substream {
 	struct snd_usb_stream *stream;
 	struct usb_device *dev;
@@ -161,6 +163,7 @@ struct snd_usb_substream {
 	} dsd_dop;
 
 	bool trigger_tstamp_pending_update; /* trigger timestamp being updated from initial estimate */
+	struct media_ctl *media_ctl;
 };
 
 struct snd_usb_stream {
diff --git a/sound/usb/line6/toneport.c b/sound/usb/line6/toneport.c
index ecbe5f3beda5..e28368d8eba2 100644
--- a/sound/usb/line6/toneport.c
+++ b/sound/usb/line6/toneport.c
@@ -54,8 +54,8 @@ struct usb_line6_toneport {
 	/* Firmware version (x 100) */
 	u8 firmware_version;
 
-	/* Timer for delayed PCM startup */
-	struct timer_list timer;
+	/* Work for delayed PCM startup */
+	struct delayed_work pcm_work;
 
 	/* Device type */
 	enum line6_device_type type;
@@ -241,9 +241,10 @@ static int snd_toneport_source_put(struct snd_kcontrol *kcontrol,
 	return 1;
 }
 
-static void toneport_start_pcm(struct timer_list *t)
+static void toneport_start_pcm(struct work_struct *work)
 {
-	struct usb_line6_toneport *toneport = from_timer(toneport, t, timer);
+	struct usb_line6_toneport *toneport =
+		container_of(work, struct usb_line6_toneport, pcm_work.work);
 	struct usb_line6 *line6 = &toneport->line6;
 
 	line6_pcm_acquire(line6->line6pcm, LINE6_STREAM_MONITOR, true);
@@ -393,7 +394,8 @@ static int toneport_setup(struct usb_line6_toneport *toneport)
 	if (toneport_has_led(toneport))
 		toneport_update_led(toneport);
 
-	mod_timer(&toneport->timer, jiffies + TONEPORT_PCM_DELAY * HZ);
+	schedule_delayed_work(&toneport->pcm_work,
+			      msecs_to_jiffies(TONEPORT_PCM_DELAY * 1000));
 	return 0;
 }
 
@@ -405,7 +407,7 @@ static void line6_toneport_disconnect(struct usb_line6 *line6)
 	struct usb_line6_toneport *toneport =
 		(struct usb_line6_toneport *)line6;
 
-	del_timer_sync(&toneport->timer);
+	cancel_delayed_work_sync(&toneport->pcm_work);
 
 	if (toneport_has_led(toneport))
 		toneport_remove_leds(toneport);
@@ -422,7 +424,7 @@ static int toneport_init(struct usb_line6 *line6,
 	struct usb_line6_toneport *toneport =  (struct usb_line6_toneport *) line6;
 
 	toneport->type = id->driver_info;
-	timer_setup(&toneport->timer, toneport_start_pcm, 0);
+	INIT_DELAYED_WORK(&toneport->pcm_work, toneport_start_pcm);
 
 	line6->disconnect = line6_toneport_disconnect;
 
diff --git a/sound/usb/media.c b/sound/usb/media.c
new file mode 100644
index 000000000000..812017eacbcf
--- /dev/null
+++ b/sound/usb/media.c
@@ -0,0 +1,327 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * media.c - Media Controller specific ALSA driver code
+ *
+ * Copyright (c) 2019 Shuah Khan <shuah@kernel.org>
+ *
+ */
+
+/*
+ * This file adds Media Controller support to the ALSA driver
+ * to use the Media Controller API to share the tuner with DVB
+ * and V4L2 drivers that control the media device.
+ *
+ * The media device is created based on the existing quirks framework.
+ * Using this approach, the media controller API usage can be added for
+ * a specific device.
+ */
+
+#include <linux/init.h>
+#include <linux/list.h>
+#include <linux/mutex.h>
+#include <linux/slab.h>
+#include <linux/usb.h>
+
+#include <sound/pcm.h>
+#include <sound/core.h>
+
+#include "usbaudio.h"
+#include "card.h"
+#include "mixer.h"
+#include "media.h"
+
+int snd_media_stream_init(struct snd_usb_substream *subs, struct snd_pcm *pcm,
+			  int stream)
+{
+	struct media_device *mdev;
+	struct media_ctl *mctl;
+	struct device *pcm_dev = &pcm->streams[stream].dev;
+	u32 intf_type;
+	int ret = 0;
+	u16 mixer_pad;
+	struct media_entity *entity;
+
+	mdev = subs->stream->chip->media_dev;
+	if (!mdev)
+		return 0;
+
+	if (subs->media_ctl)
+		return 0;
+
+	/* allocate media_ctl */
+	mctl = kzalloc(sizeof(*mctl), GFP_KERNEL);
+	if (!mctl)
+		return -ENOMEM;
+
+	mctl->media_dev = mdev;
+	if (stream == SNDRV_PCM_STREAM_PLAYBACK) {
+		intf_type = MEDIA_INTF_T_ALSA_PCM_PLAYBACK;
+		mctl->media_entity.function = MEDIA_ENT_F_AUDIO_PLAYBACK;
+		mctl->media_pad.flags = MEDIA_PAD_FL_SOURCE;
+		mixer_pad = 1;
+	} else {
+		intf_type = MEDIA_INTF_T_ALSA_PCM_CAPTURE;
+		mctl->media_entity.function = MEDIA_ENT_F_AUDIO_CAPTURE;
+		mctl->media_pad.flags = MEDIA_PAD_FL_SINK;
+		mixer_pad = 2;
+	}
+	mctl->media_entity.name = pcm->name;
+	media_entity_pads_init(&mctl->media_entity, 1, &mctl->media_pad);
+	ret =  media_device_register_entity(mctl->media_dev,
+					    &mctl->media_entity);
+	if (ret)
+		goto free_mctl;
+
+	mctl->intf_devnode = media_devnode_create(mdev, intf_type, 0,
+						  MAJOR(pcm_dev->devt),
+						  MINOR(pcm_dev->devt));
+	if (!mctl->intf_devnode) {
+		ret = -ENOMEM;
+		goto unregister_entity;
+	}
+	mctl->intf_link = media_create_intf_link(&mctl->media_entity,
+						 &mctl->intf_devnode->intf,
+						 MEDIA_LNK_FL_ENABLED);
+	if (!mctl->intf_link) {
+		ret = -ENOMEM;
+		goto devnode_remove;
+	}
+
+	/* create link between mixer and audio */
+	media_device_for_each_entity(entity, mdev) {
+		switch (entity->function) {
+		case MEDIA_ENT_F_AUDIO_MIXER:
+			ret = media_create_pad_link(entity, mixer_pad,
+						    &mctl->media_entity, 0,
+						    MEDIA_LNK_FL_ENABLED);
+			if (ret)
+				goto remove_intf_link;
+			break;
+		}
+	}
+
+	subs->media_ctl = mctl;
+	return 0;
+
+remove_intf_link:
+	media_remove_intf_link(mctl->intf_link);
+devnode_remove:
+	media_devnode_remove(mctl->intf_devnode);
+unregister_entity:
+	media_device_unregister_entity(&mctl->media_entity);
+free_mctl:
+	kfree(mctl);
+	return ret;
+}
+
+void snd_media_stream_delete(struct snd_usb_substream *subs)
+{
+	struct media_ctl *mctl = subs->media_ctl;
+
+	if (mctl) {
+		struct media_device *mdev;
+
+		mdev = mctl->media_dev;
+		if (mdev && media_devnode_is_registered(mdev->devnode)) {
+			media_devnode_remove(mctl->intf_devnode);
+			media_device_unregister_entity(&mctl->media_entity);
+			media_entity_cleanup(&mctl->media_entity);
+		}
+		kfree(mctl);
+		subs->media_ctl = NULL;
+	}
+}
+
+int snd_media_start_pipeline(struct snd_usb_substream *subs)
+{
+	struct media_ctl *mctl = subs->media_ctl;
+	int ret = 0;
+
+	if (!mctl)
+		return 0;
+
+	mutex_lock(&mctl->media_dev->graph_mutex);
+	if (mctl->media_dev->enable_source)
+		ret = mctl->media_dev->enable_source(&mctl->media_entity,
+						     &mctl->media_pipe);
+	mutex_unlock(&mctl->media_dev->graph_mutex);
+	return ret;
+}
+
+void snd_media_stop_pipeline(struct snd_usb_substream *subs)
+{
+	struct media_ctl *mctl = subs->media_ctl;
+
+	if (!mctl)
+		return;
+
+	mutex_lock(&mctl->media_dev->graph_mutex);
+	if (mctl->media_dev->disable_source)
+		mctl->media_dev->disable_source(&mctl->media_entity);
+	mutex_unlock(&mctl->media_dev->graph_mutex);
+}
+
+static int snd_media_mixer_init(struct snd_usb_audio *chip)
+{
+	struct device *ctl_dev = &chip->card->ctl_dev;
+	struct media_intf_devnode *ctl_intf;
+	struct usb_mixer_interface *mixer;
+	struct media_device *mdev = chip->media_dev;
+	struct media_mixer_ctl *mctl;
+	u32 intf_type = MEDIA_INTF_T_ALSA_CONTROL;
+	int ret;
+
+	if (!mdev)
+		return -ENODEV;
+
+	ctl_intf = chip->ctl_intf_media_devnode;
+	if (!ctl_intf) {
+		ctl_intf = media_devnode_create(mdev, intf_type, 0,
+						MAJOR(ctl_dev->devt),
+						MINOR(ctl_dev->devt));
+		if (!ctl_intf)
+			return -ENOMEM;
+		chip->ctl_intf_media_devnode = ctl_intf;
+	}
+
+	list_for_each_entry(mixer, &chip->mixer_list, list) {
+
+		if (mixer->media_mixer_ctl)
+			continue;
+
+		/* allocate media_mixer_ctl */
+		mctl = kzalloc(sizeof(*mctl), GFP_KERNEL);
+		if (!mctl)
+			return -ENOMEM;
+
+		mctl->media_dev = mdev;
+		mctl->media_entity.function = MEDIA_ENT_F_AUDIO_MIXER;
+		mctl->media_entity.name = chip->card->mixername;
+		mctl->media_pad[0].flags = MEDIA_PAD_FL_SINK;
+		mctl->media_pad[1].flags = MEDIA_PAD_FL_SOURCE;
+		mctl->media_pad[2].flags = MEDIA_PAD_FL_SOURCE;
+		media_entity_pads_init(&mctl->media_entity, MEDIA_MIXER_PAD_MAX,
+				  mctl->media_pad);
+		ret =  media_device_register_entity(mctl->media_dev,
+						    &mctl->media_entity);
+		if (ret) {
+			kfree(mctl);
+			return ret;
+		}
+
+		mctl->intf_link = media_create_intf_link(&mctl->media_entity,
+							 &ctl_intf->intf,
+							 MEDIA_LNK_FL_ENABLED);
+		if (!mctl->intf_link) {
+			media_device_unregister_entity(&mctl->media_entity);
+			media_entity_cleanup(&mctl->media_entity);
+			kfree(mctl);
+			return -ENOMEM;
+		}
+		mctl->intf_devnode = ctl_intf;
+		mixer->media_mixer_ctl = mctl;
+	}
+	return 0;
+}
+
+static void snd_media_mixer_delete(struct snd_usb_audio *chip)
+{
+	struct usb_mixer_interface *mixer;
+	struct media_device *mdev = chip->media_dev;
+
+	if (!mdev)
+		return;
+
+	list_for_each_entry(mixer, &chip->mixer_list, list) {
+		struct media_mixer_ctl *mctl;
+
+		mctl = mixer->media_mixer_ctl;
+		if (!mixer->media_mixer_ctl)
+			continue;
+
+		if (media_devnode_is_registered(mdev->devnode)) {
+			media_device_unregister_entity(&mctl->media_entity);
+			media_entity_cleanup(&mctl->media_entity);
+		}
+		kfree(mctl);
+		mixer->media_mixer_ctl = NULL;
+	}
+	if (media_devnode_is_registered(mdev->devnode))
+		media_devnode_remove(chip->ctl_intf_media_devnode);
+	chip->ctl_intf_media_devnode = NULL;
+}
+
+int snd_media_device_create(struct snd_usb_audio *chip,
+			struct usb_interface *iface)
+{
+	struct media_device *mdev;
+	struct usb_device *usbdev = interface_to_usbdev(iface);
+	int ret = 0;
+
+	/* usb-audio driver is probed for each usb interface, and
+	 * there are multiple interfaces per device. Avoid calling
+	 * media_device_usb_allocate() each time usb_audio_probe()
+	 * is called. Do it only once.
+	 */
+	if (chip->media_dev) {
+		mdev = chip->media_dev;
+		goto snd_mixer_init;
+	}
+
+	mdev = media_device_usb_allocate(usbdev, KBUILD_MODNAME, THIS_MODULE);
+	if (IS_ERR(mdev))
+		return -ENOMEM;
+
+	/* save media device - avoid lookups */
+	chip->media_dev = mdev;
+
+snd_mixer_init:
+	/* Create media entities for mixer and control dev */
+	ret = snd_media_mixer_init(chip);
+	/* media_device might be registered, print error and continue */
+	if (ret)
+		dev_err(&usbdev->dev,
+			"Couldn't create media mixer entities. Error: %d\n",
+			ret);
+
+	if (!media_devnode_is_registered(mdev->devnode)) {
+		/* dont'register if snd_media_mixer_init() failed */
+		if (ret)
+			goto create_fail;
+
+		/* register media_device */
+		ret = media_device_register(mdev);
+create_fail:
+		if (ret) {
+			snd_media_mixer_delete(chip);
+			media_device_delete(mdev, KBUILD_MODNAME, THIS_MODULE);
+			/* clear saved media_dev */
+			chip->media_dev = NULL;
+			dev_err(&usbdev->dev,
+				"Couldn't register media device. Error: %d\n",
+				ret);
+			return ret;
+		}
+	}
+
+	return ret;
+}
+
+void snd_media_device_delete(struct snd_usb_audio *chip)
+{
+	struct media_device *mdev = chip->media_dev;
+	struct snd_usb_stream *stream;
+
+	/* release resources */
+	list_for_each_entry(stream, &chip->pcm_list, list) {
+		snd_media_stream_delete(&stream->substream[0]);
+		snd_media_stream_delete(&stream->substream[1]);
+	}
+
+	snd_media_mixer_delete(chip);
+
+	if (mdev) {
+		media_device_delete(mdev, KBUILD_MODNAME, THIS_MODULE);
+		chip->media_dev = NULL;
+	}
+}
diff --git a/sound/usb/media.h b/sound/usb/media.h
new file mode 100644
index 000000000000..f5bdec1d602f
--- /dev/null
+++ b/sound/usb/media.h
@@ -0,0 +1,74 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * media.h - Media Controller specific ALSA driver code
+ *
+ * Copyright (c) 2019 Shuah Khan <shuah@kernel.org>
+ *
+ */
+
+/*
+ * This file adds Media Controller support to the ALSA driver
+ * to use the Media Controller API to share the tuner with DVB
+ * and V4L2 drivers that control the media device.
+ *
+ * The media device is created based on the existing quirks framework.
+ * Using this approach, the media controller API usage can be added for
+ * a specific device.
+ */
+#ifndef __MEDIA_H
+
+#ifdef CONFIG_SND_USB_AUDIO_USE_MEDIA_CONTROLLER
+
+#include <linux/media.h>
+#include <media/media-device.h>
+#include <media/media-entity.h>
+#include <media/media-dev-allocator.h>
+#include <sound/asound.h>
+
+struct media_ctl {
+	struct media_device *media_dev;
+	struct media_entity media_entity;
+	struct media_intf_devnode *intf_devnode;
+	struct media_link *intf_link;
+	struct media_pad media_pad;
+	struct media_pipeline media_pipe;
+};
+
+/*
+ * One source pad each for SNDRV_PCM_STREAM_CAPTURE and
+ * SNDRV_PCM_STREAM_PLAYBACK. One for sink pad to link
+ * to AUDIO Source
+ */
+#define MEDIA_MIXER_PAD_MAX    (SNDRV_PCM_STREAM_LAST + 2)
+
+struct media_mixer_ctl {
+	struct media_device *media_dev;
+	struct media_entity media_entity;
+	struct media_intf_devnode *intf_devnode;
+	struct media_link *intf_link;
+	struct media_pad media_pad[MEDIA_MIXER_PAD_MAX];
+	struct media_pipeline media_pipe;
+};
+
+int snd_media_device_create(struct snd_usb_audio *chip,
+			    struct usb_interface *iface);
+void snd_media_device_delete(struct snd_usb_audio *chip);
+int snd_media_stream_init(struct snd_usb_substream *subs, struct snd_pcm *pcm,
+			  int stream);
+void snd_media_stream_delete(struct snd_usb_substream *subs);
+int snd_media_start_pipeline(struct snd_usb_substream *subs);
+void snd_media_stop_pipeline(struct snd_usb_substream *subs);
+#else
+static inline int snd_media_device_create(struct snd_usb_audio *chip,
+					  struct usb_interface *iface)
+						{ return 0; }
+static inline void snd_media_device_delete(struct snd_usb_audio *chip) { }
+static inline int snd_media_stream_init(struct snd_usb_substream *subs,
+					struct snd_pcm *pcm, int stream)
+						{ return 0; }
+static inline void snd_media_stream_delete(struct snd_usb_substream *subs) { }
+static inline int snd_media_start_pipeline(struct snd_usb_substream *subs)
+					{ return 0; }
+static inline void snd_media_stop_pipeline(struct snd_usb_substream *subs) { }
+#endif
+#endif /* __MEDIA_H */
diff --git a/sound/usb/mixer.c b/sound/usb/mixer.c
index 73d7dff425c1..e003b5e7b01a 100644
--- a/sound/usb/mixer.c
+++ b/sound/usb/mixer.c
@@ -2675,6 +2675,8 @@ static int parse_audio_selector_unit(struct mixer_build *state, int unitid,
 	kctl = snd_ctl_new1(&mixer_selectunit_ctl, cval);
 	if (! kctl) {
 		usb_audio_err(state->chip, "cannot malloc kcontrol\n");
+		for (i = 0; i < desc->bNrInPins; i++)
+			kfree(namelist[i]);
 		kfree(namelist);
 		kfree(cval);
 		return -ENOMEM;
@@ -3490,7 +3492,9 @@ int snd_usb_create_mixer(struct snd_usb_audio *chip, int ctrlif,
 	if (err < 0)
 		goto _error;
 
-	snd_usb_mixer_apply_create_quirk(mixer);
+	err = snd_usb_mixer_apply_create_quirk(mixer);
+	if (err < 0)
+		goto _error;
 
 	err = snd_device_new(chip->card, SNDRV_DEV_CODEC, mixer, &dev_ops);
 	if (err < 0)
diff --git a/sound/usb/mixer.h b/sound/usb/mixer.h
index 3d12af8bf191..394cd9107507 100644
--- a/sound/usb/mixer.h
+++ b/sound/usb/mixer.h
@@ -4,6 +4,8 @@
 
 #include <sound/info.h>
 
+struct media_mixer_ctl;
+
 struct usb_mixer_interface {
 	struct snd_usb_audio *chip;
 	struct usb_host_interface *hostif;
@@ -23,6 +25,7 @@ struct usb_mixer_interface {
 	struct urb *rc_urb;
 	struct usb_ctrlrequest *rc_setup_packet;
 	u8 rc_buffer[6];
+	struct media_mixer_ctl *media_mixer_ctl;
 
 	bool disconnected;
 };
diff --git a/sound/usb/pcm.c b/sound/usb/pcm.c
index 056af0a57b22..5d8494b2a026 100644
--- a/sound/usb/pcm.c
+++ b/sound/usb/pcm.c
@@ -35,6 +35,7 @@
 #include "pcm.h"
 #include "clock.h"
 #include "power.h"
+#include "media.h"
 
 #define SUBSTREAM_FLAG_DATA_EP_STARTED	0
 #define SUBSTREAM_FLAG_SYNC_EP_STARTED	1
@@ -787,6 +788,10 @@ static int snd_usb_hw_params(struct snd_pcm_substream *substream,
 	struct audioformat *fmt;
 	int ret;
 
+	ret = snd_media_start_pipeline(subs);
+	if (ret)
+		return ret;
+
 	if (snd_usb_use_vmalloc)
 		ret = snd_pcm_lib_alloc_vmalloc_buffer(substream,
 						       params_buffer_bytes(hw_params));
@@ -794,7 +799,7 @@ static int snd_usb_hw_params(struct snd_pcm_substream *substream,
 		ret = snd_pcm_lib_malloc_pages(substream,
 					       params_buffer_bytes(hw_params));
 	if (ret < 0)
-		return ret;
+		goto stop_pipeline;
 
 	subs->pcm_format = params_format(hw_params);
 	subs->period_bytes = params_period_bytes(hw_params);
@@ -808,12 +813,13 @@ static int snd_usb_hw_params(struct snd_pcm_substream *substream,
 		dev_dbg(&subs->dev->dev,
 			"cannot set format: format = %#x, rate = %d, channels = %d\n",
 			   subs->pcm_format, subs->cur_rate, subs->channels);
-		return -EINVAL;
+		ret = -EINVAL;
+		goto stop_pipeline;
 	}
 
 	ret = snd_usb_lock_shutdown(subs->stream->chip);
 	if (ret < 0)
-		return ret;
+		goto stop_pipeline;
 
 	ret = snd_usb_pcm_change_state(subs, UAC3_PD_STATE_D0);
 	if (ret < 0)
@@ -829,6 +835,12 @@ static int snd_usb_hw_params(struct snd_pcm_substream *substream,
 
  unlock:
 	snd_usb_unlock_shutdown(subs->stream->chip);
+	if (ret < 0)
+		goto stop_pipeline;
+	return ret;
+
+ stop_pipeline:
+	snd_media_stop_pipeline(subs);
 	return ret;
 }
 
@@ -841,6 +853,7 @@ static int snd_usb_hw_free(struct snd_pcm_substream *substream)
 {
 	struct snd_usb_substream *subs = substream->runtime->private_data;
 
+	snd_media_stop_pipeline(subs);
 	subs->cur_audiofmt = NULL;
 	subs->cur_rate = 0;
 	subs->period_bytes = 0;
@@ -1313,6 +1326,7 @@ static int snd_usb_pcm_open(struct snd_pcm_substream *substream)
 	struct snd_usb_stream *as = snd_pcm_substream_chip(substream);
 	struct snd_pcm_runtime *runtime = substream->runtime;
 	struct snd_usb_substream *subs = &as->substream[direction];
+	int ret;
 
 	subs->interface = -1;
 	subs->altset_idx = 0;
@@ -1326,7 +1340,13 @@ static int snd_usb_pcm_open(struct snd_pcm_substream *substream)
 	subs->dsd_dop.channel = 0;
 	subs->dsd_dop.marker = 1;
 
-	return setup_hw_info(runtime, subs);
+	ret = setup_hw_info(runtime, subs);
+	if (ret == 0) {
+		ret = snd_media_stream_init(subs, as->pcm, direction);
+		if (ret)
+			snd_usb_autosuspend(subs->stream->chip);
+	}
+	return ret;
 }
 
 static int snd_usb_pcm_close(struct snd_pcm_substream *substream)
@@ -1337,6 +1357,7 @@ static int snd_usb_pcm_close(struct snd_pcm_substream *substream)
 	int ret;
 
 	stop_endpoints(subs, true);
+	snd_media_stop_pipeline(subs);
 
 	if (!as->chip->keep_iface &&
 	    subs->interface >= 0 &&
diff --git a/sound/usb/quirks-table.h b/sound/usb/quirks-table.h
index 86e80916a029..5600143ff660 100644
--- a/sound/usb/quirks-table.h
+++ b/sound/usb/quirks-table.h
@@ -2770,6 +2770,90 @@ YAMAHA_DEVICE(0x7010, "UB99"),
 		.type = QUIRK_MIDI_NOVATION
 	}
 },
+{
+	/*
+	 * Focusrite Scarlett Solo 2nd generation
+	 * Reports that playback should use Synch: Synchronous
+	 * while still providing a feedback endpoint. Synchronous causes
+	 * snapping on some sample rates.
+	 * Force it to use Synch: Asynchronous.
+	 */
+	USB_DEVICE(0x1235, 0x8205),
+	.driver_info = (unsigned long) & (const struct snd_usb_audio_quirk) {
+		.ifnum = QUIRK_ANY_INTERFACE,
+		.type = QUIRK_COMPOSITE,
+		.data = (const struct snd_usb_audio_quirk[]) {
+			{
+				.ifnum = 1,
+				.type = QUIRK_AUDIO_FIXED_ENDPOINT,
+				.data = & (const struct audioformat) {
+					.formats = SNDRV_PCM_FMTBIT_S32_LE,
+					.channels = 2,
+					.iface = 1,
+					.altsetting = 1,
+					.altset_idx = 1,
+					.attributes = 0,
+					.endpoint = 0x01,
+					.ep_attr = USB_ENDPOINT_XFER_ISOC |
+						   USB_ENDPOINT_SYNC_ASYNC,
+					.protocol = UAC_VERSION_2,
+					.rates = SNDRV_PCM_RATE_44100 |
+						 SNDRV_PCM_RATE_48000 |
+						 SNDRV_PCM_RATE_88200 |
+						 SNDRV_PCM_RATE_96000 |
+						 SNDRV_PCM_RATE_176400 |
+						 SNDRV_PCM_RATE_192000,
+					.rate_min = 44100,
+					.rate_max = 192000,
+					.nr_rates = 6,
+					.rate_table = (unsigned int[]) {
+						44100, 48000, 88200,
+						96000, 176400, 192000
+					},
+					.clock = 41
+				}
+			},
+			{
+				.ifnum = 2,
+				.type = QUIRK_AUDIO_FIXED_ENDPOINT,
+				.data = & (const struct audioformat) {
+					.formats = SNDRV_PCM_FMTBIT_S32_LE,
+					.channels = 2,
+					.iface = 2,
+					.altsetting = 1,
+					.altset_idx = 1,
+					.attributes = 0,
+					.endpoint = 0x82,
+					.ep_attr = USB_ENDPOINT_XFER_ISOC |
+						   USB_ENDPOINT_SYNC_ASYNC |
+						   USB_ENDPOINT_USAGE_IMPLICIT_FB,
+					.protocol = UAC_VERSION_2,
+					.rates = SNDRV_PCM_RATE_44100 |
+						 SNDRV_PCM_RATE_48000 |
+						 SNDRV_PCM_RATE_88200 |
+						 SNDRV_PCM_RATE_96000 |
+						 SNDRV_PCM_RATE_176400 |
+						 SNDRV_PCM_RATE_192000,
+					.rate_min = 44100,
+					.rate_max = 192000,
+					.nr_rates = 6,
+					.rate_table = (unsigned int[]) {
+						44100, 48000, 88200,
+						96000, 176400, 192000
+					},
+					.clock = 41
+				}
+			},
+			{
+				.ifnum = 3,
+				.type = QUIRK_IGNORE_INTERFACE
+			},
+			{
+				.ifnum = -1
+			}
+		}
+	}
+},
 
 /* Access Music devices */
 {
@@ -2887,6 +2971,7 @@ YAMAHA_DEVICE(0x7010, "UB99"),
 		.product_name = pname, \
 		.ifnum = QUIRK_ANY_INTERFACE, \
 		.type = QUIRK_AUDIO_ALIGN_TRANSFER, \
+		.shares_media_device = 1, \
 	} \
 }
 
diff --git a/sound/usb/stream.c b/sound/usb/stream.c
index d9e3de495c16..9f1623e37fb3 100644
--- a/sound/usb/stream.c
+++ b/sound/usb/stream.c
@@ -38,6 +38,7 @@
 #include "clock.h"
 #include "stream.h"
 #include "power.h"
+#include "media.h"
 
 /*
  * free a substream
@@ -55,6 +56,7 @@ static void free_substream(struct snd_usb_substream *subs)
 	}
 	kfree(subs->rate_list.list);
 	kfree(subs->str_pd);
+	snd_media_stream_delete(subs);
 }
 
 
diff --git a/sound/usb/usbaudio.h b/sound/usb/usbaudio.h
index b9faeca645fd..0968a45c8925 100644
--- a/sound/usb/usbaudio.h
+++ b/sound/usb/usbaudio.h
@@ -30,6 +30,9 @@
  *
  */
 
+struct media_device;
+struct media_intf_devnode;
+
 struct snd_usb_audio {
 	int index;
 	struct usb_device *dev;
@@ -66,6 +69,8 @@ struct snd_usb_audio {
 					 */
 
 	struct usb_host_interface *ctrl_intf;	/* the audio control interface */
+	struct media_device *media_dev;
+	struct media_intf_devnode *ctl_intf_media_devnode;
 };
 
 #define usb_audio_err(chip, fmt, args...) \
@@ -117,6 +122,7 @@ struct snd_usb_audio_quirk {
 	const char *profile_name;	/* override the card->longname */
 	int16_t ifnum;
 	uint16_t type;
+	bool shares_media_device;
 	const void *data;
 };
 
diff --git a/sound/usb/usx2y/usX2Yhwdep.c b/sound/usb/usx2y/usX2Yhwdep.c
index c1dd9a7b48df..bfe1108416cf 100644
--- a/sound/usb/usx2y/usX2Yhwdep.c
+++ b/sound/usb/usx2y/usX2Yhwdep.c
@@ -75,7 +75,8 @@ static int snd_us428ctls_mmap(struct snd_hwdep * hw, struct file *filp, struct v
 
 	if (!us428->us428ctls_sharedmem) {
 		init_waitqueue_head(&us428->us428ctls_wait_queue_head);
-		if(!(us428->us428ctls_sharedmem = snd_malloc_pages(sizeof(struct us428ctls_sharedmem), GFP_KERNEL)))
+		us428->us428ctls_sharedmem = alloc_pages_exact(sizeof(struct us428ctls_sharedmem), GFP_KERNEL);
+		if (!us428->us428ctls_sharedmem)
 			return -ENOMEM;
 		memset(us428->us428ctls_sharedmem, -1, sizeof(struct us428ctls_sharedmem));
 		us428->us428ctls_sharedmem->CtlSnapShotLast = -2;
diff --git a/sound/usb/usx2y/usb_stream.c b/sound/usb/usx2y/usb_stream.c
index 221adf68bd0c..51d73111263a 100644
--- a/sound/usb/usx2y/usb_stream.c
+++ b/sound/usb/usx2y/usb_stream.c
@@ -155,9 +155,9 @@ void usb_stream_free(struct usb_stream_kernel *sk)
 	if (!s)
 		return;
 
-	free_pages((unsigned long)sk->write_page, get_order(s->write_size));
+	free_pages_exact(sk->write_page, s->write_size);
 	sk->write_page = NULL;
-	free_pages((unsigned long)s, get_order(s->read_size));
+	free_pages_exact(s, s->read_size);
 	sk->s = NULL;
 }
 
@@ -172,7 +172,6 @@ struct usb_stream *usb_stream_new(struct usb_stream_kernel *sk,
 	int read_size = sizeof(struct usb_stream);
 	int write_size;
 	int usb_frames = dev->speed == USB_SPEED_HIGH ? 8000 : 1000;
-	int pg;
 
 	in_pipe = usb_rcvisocpipe(dev, in_endpoint);
 	out_pipe = usb_sndisocpipe(dev, out_endpoint);
@@ -202,11 +201,10 @@ struct usb_stream *usb_stream_new(struct usb_stream_kernel *sk,
 		goto out;
 	}
 
-	pg = get_order(read_size);
-	sk->s = (void *) __get_free_pages(GFP_KERNEL|__GFP_COMP|__GFP_ZERO|
-					  __GFP_NOWARN, pg);
+	sk->s = alloc_pages_exact(read_size,
+				  GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN);
 	if (!sk->s) {
-		snd_printk(KERN_WARNING "couldn't __get_free_pages()\n");
+		pr_warn("us122l: couldn't allocate read buffer\n");
 		goto out;
 	}
 	sk->s->cfg.version = USB_STREAM_INTERFACE_VERSION;
@@ -221,13 +219,11 @@ struct usb_stream *usb_stream_new(struct usb_stream_kernel *sk,
 	sk->s->period_size = frame_size * period_frames;
 
 	sk->s->write_size = write_size;
-	pg = get_order(write_size);
 
-	sk->write_page =
-		(void *)__get_free_pages(GFP_KERNEL|__GFP_COMP|__GFP_ZERO|
-					 __GFP_NOWARN, pg);
+	sk->write_page = alloc_pages_exact(write_size,
+					   GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN);
 	if (!sk->write_page) {
-		snd_printk(KERN_WARNING "couldn't __get_free_pages()\n");
+		pr_warn("us122l: couldn't allocate write buffer\n");
 		usb_stream_free(sk);
 		return NULL;
 	}
diff --git a/sound/usb/usx2y/usbusx2y.c b/sound/usb/usx2y/usbusx2y.c
index da4a5a541512..e8687b3bd3c8 100644
--- a/sound/usb/usx2y/usbusx2y.c
+++ b/sound/usb/usx2y/usbusx2y.c
@@ -293,10 +293,8 @@ int usX2Y_In04_init(struct usX2Ydev *usX2Y)
 	if (! (usX2Y->In04urb = usb_alloc_urb(0, GFP_KERNEL)))
 		return -ENOMEM;
 
-	if (! (usX2Y->In04Buf = kmalloc(21, GFP_KERNEL))) {
-		usb_free_urb(usX2Y->In04urb);
+	if (! (usX2Y->In04Buf = kmalloc(21, GFP_KERNEL)))
 		return -ENOMEM;
-	}
 	 
 	init_waitqueue_head(&usX2Y->In04WaitQueue);
 	usb_fill_int_urb(usX2Y->In04urb, usX2Y->dev, usb_rcvintpipe(usX2Y->dev, 0x4),
@@ -437,7 +435,8 @@ static void snd_usX2Y_card_private_free(struct snd_card *card)
 	kfree(usX2Y(card)->In04Buf);
 	usb_free_urb(usX2Y(card)->In04urb);
 	if (usX2Y(card)->us428ctls_sharedmem)
-		snd_free_pages(usX2Y(card)->us428ctls_sharedmem, sizeof(*usX2Y(card)->us428ctls_sharedmem));
+		free_pages_exact(usX2Y(card)->us428ctls_sharedmem,
+				 sizeof(*usX2Y(card)->us428ctls_sharedmem));
 	if (usX2Y(card)->card_index >= 0  &&  usX2Y(card)->card_index < SNDRV_CARDS)
 		snd_usX2Y_card_used[usX2Y(card)->card_index] = 0;
 }
diff --git a/sound/usb/usx2y/usx2yhwdeppcm.c b/sound/usb/usx2y/usx2yhwdeppcm.c
index 714cf50d4a4c..ace8185c3f6d 100644
--- a/sound/usb/usx2y/usx2yhwdeppcm.c
+++ b/sound/usb/usx2y/usx2yhwdeppcm.c
@@ -488,7 +488,9 @@ static int snd_usX2Y_usbpcm_prepare(struct snd_pcm_substream *substream)
 	snd_printdd("snd_usX2Y_pcm_prepare(%p)\n", substream);
 
 	if (NULL == usX2Y->hwdep_pcm_shm) {
-		if (NULL == (usX2Y->hwdep_pcm_shm = snd_malloc_pages(sizeof(struct snd_usX2Y_hwdep_pcm_shm), GFP_KERNEL)))
+		usX2Y->hwdep_pcm_shm = alloc_pages_exact(sizeof(struct snd_usX2Y_hwdep_pcm_shm),
+							 GFP_KERNEL);
+		if (!usX2Y->hwdep_pcm_shm)
 			return -ENOMEM;
 		memset(usX2Y->hwdep_pcm_shm, 0, sizeof(struct snd_usX2Y_hwdep_pcm_shm));
 	}
@@ -700,7 +702,7 @@ static void snd_usX2Y_hwdep_pcm_private_free(struct snd_hwdep *hwdep)
 {
 	struct usX2Ydev *usX2Y = hwdep->private_data;
 	if (NULL != usX2Y->hwdep_pcm_shm)
-		snd_free_pages(usX2Y->hwdep_pcm_shm, sizeof(struct snd_usX2Y_hwdep_pcm_shm));
+		free_pages_exact(usX2Y->hwdep_pcm_shm, sizeof(struct snd_usX2Y_hwdep_pcm_shm));
 }
 
 
diff --git a/tools/arch/csky/include/uapi/asm/perf_regs.h b/tools/arch/csky/include/uapi/asm/perf_regs.h
new file mode 100644
index 000000000000..ee323d818592
--- /dev/null
+++ b/tools/arch/csky/include/uapi/asm/perf_regs.h
@@ -0,0 +1,51 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+// Copyright (C) 2019 Hangzhou C-SKY Microsystems co.,ltd.
+
+#ifndef _ASM_CSKY_PERF_REGS_H
+#define _ASM_CSKY_PERF_REGS_H
+
+/* Index of struct pt_regs */
+enum perf_event_csky_regs {
+	PERF_REG_CSKY_TLS,
+	PERF_REG_CSKY_LR,
+	PERF_REG_CSKY_PC,
+	PERF_REG_CSKY_SR,
+	PERF_REG_CSKY_SP,
+	PERF_REG_CSKY_ORIG_A0,
+	PERF_REG_CSKY_A0,
+	PERF_REG_CSKY_A1,
+	PERF_REG_CSKY_A2,
+	PERF_REG_CSKY_A3,
+	PERF_REG_CSKY_REGS0,
+	PERF_REG_CSKY_REGS1,
+	PERF_REG_CSKY_REGS2,
+	PERF_REG_CSKY_REGS3,
+	PERF_REG_CSKY_REGS4,
+	PERF_REG_CSKY_REGS5,
+	PERF_REG_CSKY_REGS6,
+	PERF_REG_CSKY_REGS7,
+	PERF_REG_CSKY_REGS8,
+	PERF_REG_CSKY_REGS9,
+#if defined(__CSKYABIV2__)
+	PERF_REG_CSKY_EXREGS0,
+	PERF_REG_CSKY_EXREGS1,
+	PERF_REG_CSKY_EXREGS2,
+	PERF_REG_CSKY_EXREGS3,
+	PERF_REG_CSKY_EXREGS4,
+	PERF_REG_CSKY_EXREGS5,
+	PERF_REG_CSKY_EXREGS6,
+	PERF_REG_CSKY_EXREGS7,
+	PERF_REG_CSKY_EXREGS8,
+	PERF_REG_CSKY_EXREGS9,
+	PERF_REG_CSKY_EXREGS10,
+	PERF_REG_CSKY_EXREGS11,
+	PERF_REG_CSKY_EXREGS12,
+	PERF_REG_CSKY_EXREGS13,
+	PERF_REG_CSKY_EXREGS14,
+	PERF_REG_CSKY_HI,
+	PERF_REG_CSKY_LO,
+	PERF_REG_CSKY_DCSR,
+#endif
+	PERF_REG_CSKY_MAX,
+};
+#endif /* _ASM_CSKY_PERF_REGS_H */
diff --git a/tools/include/nolibc/nolibc.h b/tools/include/nolibc/nolibc.h
index 1708e9f9f8aa..2551e9b71167 100644
--- a/tools/include/nolibc/nolibc.h
+++ b/tools/include/nolibc/nolibc.h
@@ -1243,6 +1243,200 @@ struct sys_stat_struct {
 	long          st_pad4[14];
 };
 
+#elif defined(__riscv)
+
+#if   __riscv_xlen == 64
+#define PTRLOG "3"
+#define SZREG  "8"
+#elif __riscv_xlen == 32
+#define PTRLOG "2"
+#define SZREG  "4"
+#endif
+
+/* Syscalls for RISCV :
+ *   - stack is 16-byte aligned
+ *   - syscall number is passed in a7
+ *   - arguments are in a0, a1, a2, a3, a4, a5
+ *   - the system call is performed by calling ecall
+ *   - syscall return comes in a0
+ *   - the arguments are cast to long and assigned into the target
+ *     registers which are then simply passed as registers to the asm code,
+ *     so that we don't have to experience issues with register constraints.
+ */
+
+#define my_syscall0(num)                                                      \
+({                                                                            \
+	register long _num  asm("a7") = (num);                                \
+	register long _arg1 asm("a0");                                        \
+									      \
+	asm volatile (                                                        \
+		"ecall\n\t"                                                   \
+		: "=r"(_arg1)                                                 \
+		: "r"(_num)                                                   \
+		: "memory", "cc"                                              \
+	);                                                                    \
+	_arg1;                                                                \
+})
+
+#define my_syscall1(num, arg1)                                                \
+({                                                                            \
+	register long _num  asm("a7") = (num);                                \
+	register long _arg1 asm("a0") = (long)(arg1);		              \
+									      \
+	asm volatile (                                                        \
+		"ecall\n"                                                     \
+		: "+r"(_arg1)                                                 \
+		: "r"(_num)                                                   \
+		: "memory", "cc"                                              \
+	);                                                                    \
+	_arg1;                                                                \
+})
+
+#define my_syscall2(num, arg1, arg2)                                          \
+({                                                                            \
+	register long _num  asm("a7") = (num);                                \
+	register long _arg1 asm("a0") = (long)(arg1);                         \
+	register long _arg2 asm("a1") = (long)(arg2);                         \
+									      \
+	asm volatile (                                                        \
+		"ecall\n"                                                     \
+		: "+r"(_arg1)                                                 \
+		: "r"(_arg2),                                                 \
+		  "r"(_num)                                                   \
+		: "memory", "cc"                                              \
+	);                                                                    \
+	_arg1;                                                                \
+})
+
+#define my_syscall3(num, arg1, arg2, arg3)                                    \
+({                                                                            \
+	register long _num  asm("a7") = (num);                                \
+	register long _arg1 asm("a0") = (long)(arg1);                         \
+	register long _arg2 asm("a1") = (long)(arg2);                         \
+	register long _arg3 asm("a2") = (long)(arg3);                         \
+									      \
+	asm volatile (                                                        \
+		"ecall\n\t"                                                   \
+		: "+r"(_arg1)                                                 \
+		: "r"(_arg2), "r"(_arg3),                                     \
+		  "r"(_num)                                                   \
+		: "memory", "cc"                                              \
+	);                                                                    \
+	_arg1;                                                                \
+})
+
+#define my_syscall4(num, arg1, arg2, arg3, arg4)                              \
+({                                                                            \
+	register long _num  asm("a7") = (num);                                \
+	register long _arg1 asm("a0") = (long)(arg1);                         \
+	register long _arg2 asm("a1") = (long)(arg2);                         \
+	register long _arg3 asm("a2") = (long)(arg3);                         \
+	register long _arg4 asm("a3") = (long)(arg4);                         \
+									      \
+	asm volatile (                                                        \
+		"ecall\n"                                                     \
+		: "+r"(_arg1)                                                 \
+		: "r"(_arg2), "r"(_arg3), "r"(_arg4),                         \
+		  "r"(_num)                                                   \
+		: "memory", "cc"                                              \
+	);                                                                    \
+	_arg1;                                                                \
+})
+
+#define my_syscall5(num, arg1, arg2, arg3, arg4, arg5)                        \
+({                                                                            \
+	register long _num  asm("a7") = (num);                                \
+	register long _arg1 asm("a0") = (long)(arg1);                         \
+	register long _arg2 asm("a1") = (long)(arg2);                         \
+	register long _arg3 asm("a2") = (long)(arg3);                         \
+	register long _arg4 asm("a3") = (long)(arg4);                         \
+	register long _arg5 asm("a4") = (long)(arg5);                         \
+									      \
+	asm volatile (                                                        \
+		"ecall\n"                                                     \
+		: "+r"(_arg1)                                                 \
+		: "r"(_arg2), "r"(_arg3), "r"(_arg4), "r"(_arg5),             \
+		  "r"(_num)                                                   \
+		: "memory", "cc"                                              \
+	);                                                                    \
+	_arg1;                                                                \
+})
+
+#define my_syscall6(num, arg1, arg2, arg3, arg4, arg5, arg6)                  \
+({                                                                            \
+	register long _num  asm("a7") = (num);                                \
+	register long _arg1 asm("a0") = (long)(arg1);                         \
+	register long _arg2 asm("a1") = (long)(arg2);                         \
+	register long _arg3 asm("a2") = (long)(arg3);                         \
+	register long _arg4 asm("a3") = (long)(arg4);                         \
+	register long _arg5 asm("a4") = (long)(arg5);                         \
+	register long _arg6 asm("a5") = (long)(arg6);                         \
+									      \
+	asm volatile (                                                        \
+		"ecall\n"                                                     \
+		: "+r"(_arg1)                                                 \
+		: "r"(_arg2), "r"(_arg3), "r"(_arg4), "r"(_arg5), "r"(_arg6), \
+		  "r"(_num)                                                   \
+		: "memory", "cc"                                              \
+	);                                                                    \
+	_arg1;                                                                \
+})
+
+/* startup code */
+asm(".section .text\n"
+    ".global _start\n"
+    "_start:\n"
+    ".option push\n"
+    ".option norelax\n"
+    "lla   gp, __global_pointer$\n"
+    ".option pop\n"
+    "ld    a0, 0(sp)\n"          // argc (a0) was in the stack
+    "add   a1, sp, "SZREG"\n"    // argv (a1) = sp
+    "slli  a2, a0, "PTRLOG"\n"   // envp (a2) = SZREG*argc ...
+    "add   a2, a2, "SZREG"\n"    //             + SZREG (skip null)
+    "add   a2,a2,a1\n"           //             + argv
+    "andi  sp,a1,-16\n"          // sp must be 16-byte aligned
+    "call  main\n"               // main() returns the status code, we'll exit with it.
+    "andi  a0, a0, 0xff\n"       // limit exit code to 8 bits
+    "li a7, 93\n"                // NR_exit == 93
+    "ecall\n"
+    "");
+
+/* fcntl / open */
+#define O_RDONLY            0
+#define O_WRONLY            1
+#define O_RDWR              2
+#define O_CREAT         0x100
+#define O_EXCL          0x200
+#define O_NOCTTY        0x400
+#define O_TRUNC        0x1000
+#define O_APPEND       0x2000
+#define O_NONBLOCK     0x4000
+#define O_DIRECTORY  0x200000
+
+struct sys_stat_struct {
+	unsigned long	st_dev;		/* Device.  */
+	unsigned long	st_ino;		/* File serial number.  */
+	unsigned int	st_mode;	/* File mode.  */
+	unsigned int	st_nlink;	/* Link count.  */
+	unsigned int	st_uid;		/* User ID of the file's owner.  */
+	unsigned int	st_gid;		/* Group ID of the file's group. */
+	unsigned long	st_rdev;	/* Device number, if device.  */
+	unsigned long	__pad1;
+	long		st_size;	/* Size of file, in bytes.  */
+	int		st_blksize;	/* Optimal block size for I/O.  */
+	int		__pad2;
+	long		st_blocks;	/* Number 512-byte blocks allocated. */
+	long		st_atime;	/* Time of last access.  */
+	unsigned long	st_atime_nsec;
+	long		st_mtime;	/* Time of last modification.  */
+	unsigned long	st_mtime_nsec;
+	long		st_ctime;	/* Time of last status change.  */
+	unsigned long	st_ctime_nsec;
+	unsigned int	__unused4;
+	unsigned int	__unused5;
+};
+
 #endif
 
 
diff --git a/tools/objtool/Documentation/stack-validation.txt b/tools/objtool/Documentation/stack-validation.txt
index 3995735a878f..8df526c80b65 100644
--- a/tools/objtool/Documentation/stack-validation.txt
+++ b/tools/objtool/Documentation/stack-validation.txt
@@ -111,7 +111,7 @@ c) Higher live patching compatibility rate
    be detectable).  Objtool makes that possible.
 
    For more details, see the livepatch documentation in the Linux kernel
-   source tree at Documentation/livepatch/livepatch.txt.
+   source tree at Documentation/livepatch/livepatch.rst.
 
 Rules
 -----
diff --git a/tools/perf/Makefile.config b/tools/perf/Makefile.config
index 0c52a01dc759..e1bb5288ab1f 100644
--- a/tools/perf/Makefile.config
+++ b/tools/perf/Makefile.config
@@ -59,6 +59,10 @@ ifeq ($(SRCARCH),arm64)
   LIBUNWIND_LIBS = -lunwind -lunwind-aarch64
 endif
 
+ifeq ($(SRCARCH),csky)
+  NO_PERF_REGS := 0
+endif
+
 ifeq ($(ARCH),s390)
   NO_PERF_REGS := 0
   NO_SYSCALL_TABLE := 0
@@ -77,7 +81,7 @@ endif
 # Disable it on all other architectures in case libdw unwind
 # support is detected in system. Add supported architectures
 # to the check.
-ifneq ($(SRCARCH),$(filter $(SRCARCH),x86 arm arm64 powerpc s390))
+ifneq ($(SRCARCH),$(filter $(SRCARCH),x86 arm arm64 powerpc s390 csky))
   NO_LIBDW_DWARF_UNWIND := 1
 endif
 
diff --git a/tools/perf/arch/csky/Build b/tools/perf/arch/csky/Build
new file mode 100644
index 000000000000..e4e5f33c84d8
--- /dev/null
+++ b/tools/perf/arch/csky/Build
@@ -0,0 +1 @@
+perf-y += util/
diff --git a/tools/perf/arch/csky/Makefile b/tools/perf/arch/csky/Makefile
new file mode 100644
index 000000000000..7fbca175099e
--- /dev/null
+++ b/tools/perf/arch/csky/Makefile
@@ -0,0 +1,3 @@
+ifndef NO_DWARF
+PERF_HAVE_DWARF_REGS := 1
+endif
diff --git a/tools/perf/arch/csky/include/perf_regs.h b/tools/perf/arch/csky/include/perf_regs.h
new file mode 100644
index 000000000000..8f336ea1161a
--- /dev/null
+++ b/tools/perf/arch/csky/include/perf_regs.h
@@ -0,0 +1,100 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+// Copyright (C) 2019 Hangzhou C-SKY Microsystems co.,ltd.
+
+#ifndef ARCH_PERF_REGS_H
+#define ARCH_PERF_REGS_H
+
+#include <stdlib.h>
+#include <linux/types.h>
+#include <asm/perf_regs.h>
+
+#define PERF_REGS_MASK	((1ULL << PERF_REG_CSKY_MAX) - 1)
+#define PERF_REGS_MAX	PERF_REG_CSKY_MAX
+#define PERF_SAMPLE_REGS_ABI	PERF_SAMPLE_REGS_ABI_32
+
+#define PERF_REG_IP	PERF_REG_CSKY_PC
+#define PERF_REG_SP	PERF_REG_CSKY_SP
+
+static inline const char *perf_reg_name(int id)
+{
+	switch (id) {
+	case PERF_REG_CSKY_A0:
+		return "a0";
+	case PERF_REG_CSKY_A1:
+		return "a1";
+	case PERF_REG_CSKY_A2:
+		return "a2";
+	case PERF_REG_CSKY_A3:
+		return "a3";
+	case PERF_REG_CSKY_REGS0:
+		return "regs0";
+	case PERF_REG_CSKY_REGS1:
+		return "regs1";
+	case PERF_REG_CSKY_REGS2:
+		return "regs2";
+	case PERF_REG_CSKY_REGS3:
+		return "regs3";
+	case PERF_REG_CSKY_REGS4:
+		return "regs4";
+	case PERF_REG_CSKY_REGS5:
+		return "regs5";
+	case PERF_REG_CSKY_REGS6:
+		return "regs6";
+	case PERF_REG_CSKY_REGS7:
+		return "regs7";
+	case PERF_REG_CSKY_REGS8:
+		return "regs8";
+	case PERF_REG_CSKY_REGS9:
+		return "regs9";
+	case PERF_REG_CSKY_SP:
+		return "sp";
+	case PERF_REG_CSKY_LR:
+		return "lr";
+	case PERF_REG_CSKY_PC:
+		return "pc";
+#if defined(__CSKYABIV2__)
+	case PERF_REG_CSKY_EXREGS0:
+		return "exregs0";
+	case PERF_REG_CSKY_EXREGS1:
+		return "exregs1";
+	case PERF_REG_CSKY_EXREGS2:
+		return "exregs2";
+	case PERF_REG_CSKY_EXREGS3:
+		return "exregs3";
+	case PERF_REG_CSKY_EXREGS4:
+		return "exregs4";
+	case PERF_REG_CSKY_EXREGS5:
+		return "exregs5";
+	case PERF_REG_CSKY_EXREGS6:
+		return "exregs6";
+	case PERF_REG_CSKY_EXREGS7:
+		return "exregs7";
+	case PERF_REG_CSKY_EXREGS8:
+		return "exregs8";
+	case PERF_REG_CSKY_EXREGS9:
+		return "exregs9";
+	case PERF_REG_CSKY_EXREGS10:
+		return "exregs10";
+	case PERF_REG_CSKY_EXREGS11:
+		return "exregs11";
+	case PERF_REG_CSKY_EXREGS12:
+		return "exregs12";
+	case PERF_REG_CSKY_EXREGS13:
+		return "exregs13";
+	case PERF_REG_CSKY_EXREGS14:
+		return "exregs14";
+	case PERF_REG_CSKY_TLS:
+		return "tls";
+	case PERF_REG_CSKY_HI:
+		return "hi";
+	case PERF_REG_CSKY_LO:
+		return "lo";
+#endif
+	default:
+		return NULL;
+	}
+
+	return NULL;
+}
+
+#endif /* ARCH_PERF_REGS_H */
diff --git a/tools/perf/arch/csky/util/Build b/tools/perf/arch/csky/util/Build
new file mode 100644
index 000000000000..1160bb2332ba
--- /dev/null
+++ b/tools/perf/arch/csky/util/Build
@@ -0,0 +1,2 @@
+perf-$(CONFIG_DWARF) += dwarf-regs.o
+perf-$(CONFIG_LIBDW_DWARF_UNWIND) += unwind-libdw.o
diff --git a/tools/perf/arch/csky/util/dwarf-regs.c b/tools/perf/arch/csky/util/dwarf-regs.c
new file mode 100644
index 000000000000..ca86ecaeacbb
--- /dev/null
+++ b/tools/perf/arch/csky/util/dwarf-regs.c
@@ -0,0 +1,49 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2019 Hangzhou C-SKY Microsystems co.,ltd.
+// Mapping of DWARF debug register numbers into register names.
+
+#include <stddef.h>
+#include <dwarf-regs.h>
+
+#if defined(__CSKYABIV2__)
+#define CSKY_MAX_REGS 73
+const char *csky_dwarf_regs_table[CSKY_MAX_REGS] = {
+	/* r0 ~ r8 */
+	"%a0", "%a1", "%a2", "%a3", "%regs0", "%regs1", "%regs2", "%regs3",
+	/* r9 ~ r15 */
+	"%regs4", "%regs5", "%regs6", "%regs7", "%regs8", "%regs9", "%sp",
+	"%lr",
+	/* r16 ~ r23 */
+	"%exregs0", "%exregs1", "%exregs2", "%exregs3", "%exregs4",
+	"%exregs5", "%exregs6", "%exregs7",
+	/* r24 ~ r31 */
+	"%exregs8", "%exregs9", "%exregs10", "%exregs11", "%exregs12",
+	"%exregs13", "%exregs14", "%tls",
+	"%pc", NULL, NULL, NULL, "%hi", "%lo", NULL, NULL,
+	NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
+	NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
+	NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
+	NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
+	"%epc",
+};
+#else
+#define CSKY_MAX_REGS 57
+const char *csky_dwarf_regs_table[CSKY_MAX_REGS] = {
+	/* r0 ~ r8 */
+	"%sp", "%regs9", "%a0", "%a1", "%a2", "%a3", "%regs0", "%regs1",
+	/* r9 ~ r15 */
+	"%regs2", "%regs3", "%regs4", "%regs5", "%regs6", "%regs7", "%regs8",
+	"%lr",
+	NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
+	NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
+	NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
+	NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
+	NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
+	"%epc",
+};
+#endif
+
+const char *get_arch_regstr(unsigned int n)
+{
+	return (n < CSKY_MAX_REGS) ? csky_dwarf_regs_table[n] : NULL;
+}
diff --git a/tools/perf/arch/csky/util/unwind-libdw.c b/tools/perf/arch/csky/util/unwind-libdw.c
new file mode 100644
index 000000000000..4bb4a06776e4
--- /dev/null
+++ b/tools/perf/arch/csky/util/unwind-libdw.c
@@ -0,0 +1,77 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2019 Hangzhou C-SKY Microsystems co.,ltd.
+
+#include <elfutils/libdwfl.h>
+#include "../../util/unwind-libdw.h"
+#include "../../util/perf_regs.h"
+#include "../../util/event.h"
+
+bool libdw__arch_set_initial_registers(Dwfl_Thread *thread, void *arg)
+{
+	struct unwind_info *ui = arg;
+	struct regs_dump *user_regs = &ui->sample->user_regs;
+	Dwarf_Word dwarf_regs[PERF_REG_CSKY_MAX];
+
+#define REG(r) ({						\
+	Dwarf_Word val = 0;					\
+	perf_reg_value(&val, user_regs, PERF_REG_CSKY_##r);	\
+	val;							\
+})
+
+#if defined(__CSKYABIV2__)
+	dwarf_regs[0]  = REG(A0);
+	dwarf_regs[1]  = REG(A1);
+	dwarf_regs[2]  = REG(A2);
+	dwarf_regs[3]  = REG(A3);
+	dwarf_regs[4]  = REG(REGS0);
+	dwarf_regs[5]  = REG(REGS1);
+	dwarf_regs[6]  = REG(REGS2);
+	dwarf_regs[7]  = REG(REGS3);
+	dwarf_regs[8]  = REG(REGS4);
+	dwarf_regs[9]  = REG(REGS5);
+	dwarf_regs[10] = REG(REGS6);
+	dwarf_regs[11] = REG(REGS7);
+	dwarf_regs[12] = REG(REGS8);
+	dwarf_regs[13] = REG(REGS9);
+	dwarf_regs[14] = REG(SP);
+	dwarf_regs[15] = REG(LR);
+	dwarf_regs[16] = REG(EXREGS0);
+	dwarf_regs[17] = REG(EXREGS1);
+	dwarf_regs[18] = REG(EXREGS2);
+	dwarf_regs[19] = REG(EXREGS3);
+	dwarf_regs[20] = REG(EXREGS4);
+	dwarf_regs[21] = REG(EXREGS5);
+	dwarf_regs[22] = REG(EXREGS6);
+	dwarf_regs[23] = REG(EXREGS7);
+	dwarf_regs[24] = REG(EXREGS8);
+	dwarf_regs[25] = REG(EXREGS9);
+	dwarf_regs[26] = REG(EXREGS10);
+	dwarf_regs[27] = REG(EXREGS11);
+	dwarf_regs[28] = REG(EXREGS12);
+	dwarf_regs[29] = REG(EXREGS13);
+	dwarf_regs[30] = REG(EXREGS14);
+	dwarf_regs[31] = REG(TLS);
+	dwarf_regs[32] = REG(PC);
+#else
+	dwarf_regs[0]  = REG(SP);
+	dwarf_regs[1]  = REG(REGS9);
+	dwarf_regs[2]  = REG(A0);
+	dwarf_regs[3]  = REG(A1);
+	dwarf_regs[4]  = REG(A2);
+	dwarf_regs[5]  = REG(A3);
+	dwarf_regs[6]  = REG(REGS0);
+	dwarf_regs[7]  = REG(REGS1);
+	dwarf_regs[8]  = REG(REGS2);
+	dwarf_regs[9]  = REG(REGS3);
+	dwarf_regs[10] = REG(REGS4);
+	dwarf_regs[11] = REG(REGS5);
+	dwarf_regs[12] = REG(REGS6);
+	dwarf_regs[13] = REG(REGS7);
+	dwarf_regs[14] = REG(REGS8);
+	dwarf_regs[15] = REG(LR);
+#endif
+	dwfl_thread_state_register_pc(thread, REG(PC));
+
+	return dwfl_thread_state_registers(thread, 0, PERF_REG_CSKY_MAX,
+					   dwarf_regs);
+}
diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c b/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c
index 23b159d95c3f..b74e2f6e96d0 100644
--- a/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c
+++ b/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c
@@ -15,7 +15,7 @@ static int libbpf_debug_print(enum libbpf_print_level level,
 static int check_load(const char *file)
 {
 	struct bpf_prog_load_attr attr;
-	struct bpf_object *obj;
+	struct bpf_object *obj = NULL;
 	int err, prog_fd;
 
 	memset(&attr, 0, sizeof(struct bpf_prog_load_attr));
diff --git a/tools/testing/selftests/bpf/prog_tests/task_fd_query_tp.c b/tools/testing/selftests/bpf/prog_tests/task_fd_query_tp.c
index d636a4f39476..f9b70e81682b 100644
--- a/tools/testing/selftests/bpf/prog_tests/task_fd_query_tp.c
+++ b/tools/testing/selftests/bpf/prog_tests/task_fd_query_tp.c
@@ -9,7 +9,7 @@ static void test_task_fd_query_tp_core(const char *probe_name,
 	struct perf_event_attr attr = {};
 	__u64 probe_offset, probe_addr;
 	__u32 len, prog_id, fd_type;
-	struct bpf_object *obj;
+	struct bpf_object *obj = NULL;
 	__u32 duration = 0;
 	char buf[256];
 
diff --git a/tools/testing/selftests/bpf/prog_tests/tp_attach_query.c b/tools/testing/selftests/bpf/prog_tests/tp_attach_query.c
index a2f476f91637..fb095e5cd9af 100644
--- a/tools/testing/selftests/bpf/prog_tests/tp_attach_query.c
+++ b/tools/testing/selftests/bpf/prog_tests/tp_attach_query.c
@@ -13,6 +13,9 @@ void test_tp_attach_query(void)
 	struct bpf_prog_info prog_info;
 	char buf[256];
 
+	for (i = 0; i < num_progs; i++)
+		obj[i] = NULL;
+
 	snprintf(buf, sizeof(buf),
 		 "/sys/kernel/debug/tracing/events/sched/sched_switch/id");
 	efd = open(buf, O_RDONLY, 0);
diff --git a/tools/testing/selftests/cgroup/.gitignore b/tools/testing/selftests/cgroup/.gitignore
index adacda50a4b2..7f9835624793 100644
--- a/tools/testing/selftests/cgroup/.gitignore
+++ b/tools/testing/selftests/cgroup/.gitignore
@@ -1,2 +1,3 @@
 test_memcontrol
 test_core
+test_freezer
diff --git a/tools/testing/selftests/cgroup/Makefile b/tools/testing/selftests/cgroup/Makefile
index 23fbaa4a9630..8d369b6a2069 100644
--- a/tools/testing/selftests/cgroup/Makefile
+++ b/tools/testing/selftests/cgroup/Makefile
@@ -5,8 +5,10 @@ all:
 
 TEST_GEN_PROGS = test_memcontrol
 TEST_GEN_PROGS += test_core
+TEST_GEN_PROGS += test_freezer
 
 include ../lib.mk
 
 $(OUTPUT)/test_memcontrol: cgroup_util.c
 $(OUTPUT)/test_core: cgroup_util.c
+$(OUTPUT)/test_freezer: cgroup_util.c
diff --git a/tools/testing/selftests/cgroup/cgroup_util.c b/tools/testing/selftests/cgroup/cgroup_util.c
index 14c9fe284806..4c223266299a 100644
--- a/tools/testing/selftests/cgroup/cgroup_util.c
+++ b/tools/testing/selftests/cgroup/cgroup_util.c
@@ -74,6 +74,16 @@ char *cg_name_indexed(const char *root, const char *name, int index)
 	return ret;
 }
 
+char *cg_control(const char *cgroup, const char *control)
+{
+	size_t len = strlen(cgroup) + strlen(control) + 2;
+	char *ret = malloc(len);
+
+	snprintf(ret, len, "%s/%s", cgroup, control);
+
+	return ret;
+}
+
 int cg_read(const char *cgroup, const char *control, char *buf, size_t len)
 {
 	char path[PATH_MAX];
@@ -196,7 +206,32 @@ int cg_create(const char *cgroup)
 	return mkdir(cgroup, 0644);
 }
 
-static int cg_killall(const char *cgroup)
+int cg_wait_for_proc_count(const char *cgroup, int count)
+{
+	char buf[10 * PAGE_SIZE] = {0};
+	int attempts;
+	char *ptr;
+
+	for (attempts = 10; attempts >= 0; attempts--) {
+		int nr = 0;
+
+		if (cg_read(cgroup, "cgroup.procs", buf, sizeof(buf)))
+			break;
+
+		for (ptr = buf; *ptr; ptr++)
+			if (*ptr == '\n')
+				nr++;
+
+		if (nr >= count)
+			return 0;
+
+		usleep(100000);
+	}
+
+	return -1;
+}
+
+int cg_killall(const char *cgroup)
 {
 	char buf[PAGE_SIZE];
 	char *ptr = buf;
@@ -227,9 +262,7 @@ int cg_destroy(const char *cgroup)
 retry:
 	ret = rmdir(cgroup);
 	if (ret && errno == EBUSY) {
-		ret = cg_killall(cgroup);
-		if (ret)
-			return ret;
+		cg_killall(cgroup);
 		usleep(100);
 		goto retry;
 	}
@@ -240,6 +273,14 @@ retry:
 	return ret;
 }
 
+int cg_enter(const char *cgroup, int pid)
+{
+	char pidbuf[64];
+
+	snprintf(pidbuf, sizeof(pidbuf), "%d", pid);
+	return cg_write(cgroup, "cgroup.procs", pidbuf);
+}
+
 int cg_enter_current(const char *cgroup)
 {
 	char pidbuf[64];
@@ -369,3 +410,12 @@ int set_oom_adj_score(int pid, int score)
 	close(fd);
 	return 0;
 }
+
+char proc_read_text(int pid, const char *item, char *buf, size_t size)
+{
+	char path[PATH_MAX];
+
+	snprintf(path, sizeof(path), "/proc/%d/%s", pid, item);
+
+	return read_text(path, buf, size);
+}
diff --git a/tools/testing/selftests/cgroup/cgroup_util.h b/tools/testing/selftests/cgroup/cgroup_util.h
index 9ac8b7958f83..c72f28046bfa 100644
--- a/tools/testing/selftests/cgroup/cgroup_util.h
+++ b/tools/testing/selftests/cgroup/cgroup_util.h
@@ -18,6 +18,7 @@ static inline int values_close(long a, long b, int err)
 extern int cg_find_unified_root(char *root, size_t len);
 extern char *cg_name(const char *root, const char *name);
 extern char *cg_name_indexed(const char *root, const char *name, int index);
+extern char *cg_control(const char *cgroup, const char *control);
 extern int cg_create(const char *cgroup);
 extern int cg_destroy(const char *cgroup);
 extern int cg_read(const char *cgroup, const char *control,
@@ -32,6 +33,7 @@ extern int cg_write(const char *cgroup, const char *control, char *buf);
 extern int cg_run(const char *cgroup,
 		  int (*fn)(const char *cgroup, void *arg),
 		  void *arg);
+extern int cg_enter(const char *cgroup, int pid);
 extern int cg_enter_current(const char *cgroup);
 extern int cg_run_nowait(const char *cgroup,
 			 int (*fn)(const char *cgroup, void *arg),
@@ -41,3 +43,6 @@ extern int alloc_pagecache(int fd, size_t size);
 extern int alloc_anon(const char *cgroup, void *arg);
 extern int is_swap_enabled(void);
 extern int set_oom_adj_score(int pid, int score);
+extern int cg_wait_for_proc_count(const char *cgroup, int count);
+extern int cg_killall(const char *cgroup);
+extern char proc_read_text(int pid, const char *item, char *buf, size_t size);
diff --git a/tools/testing/selftests/cgroup/test_freezer.c b/tools/testing/selftests/cgroup/test_freezer.c
new file mode 100644
index 000000000000..2bfddb6d6d3b
--- /dev/null
+++ b/tools/testing/selftests/cgroup/test_freezer.c
@@ -0,0 +1,851 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#include <stdbool.h>
+#include <linux/limits.h>
+#include <sys/ptrace.h>
+#include <sys/types.h>
+#include <sys/mman.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <errno.h>
+#include <poll.h>
+#include <stdlib.h>
+#include <sys/inotify.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+
+#include "../kselftest.h"
+#include "cgroup_util.h"
+
+#define DEBUG
+#ifdef DEBUG
+#define debug(args...) fprintf(stderr, args)
+#else
+#define debug(args...)
+#endif
+
+/*
+ * Check if the cgroup is frozen by looking at the cgroup.events::frozen value.
+ */
+static int cg_check_frozen(const char *cgroup, bool frozen)
+{
+	if (frozen) {
+		if (cg_read_strstr(cgroup, "cgroup.events", "frozen 1") != 0) {
+			debug("Cgroup %s isn't frozen\n", cgroup);
+			return -1;
+		}
+	} else {
+		/*
+		 * Check the cgroup.events::frozen value.
+		 */
+		if (cg_read_strstr(cgroup, "cgroup.events", "frozen 0") != 0) {
+			debug("Cgroup %s is frozen\n", cgroup);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+/*
+ * Freeze the given cgroup.
+ */
+static int cg_freeze_nowait(const char *cgroup, bool freeze)
+{
+	return cg_write(cgroup, "cgroup.freeze", freeze ? "1" : "0");
+}
+
+/*
+ * Prepare for waiting on cgroup.events file.
+ */
+static int cg_prepare_for_wait(const char *cgroup)
+{
+	int fd, ret = -1;
+
+	fd = inotify_init1(0);
+	if (fd == -1) {
+		debug("Error: inotify_init1() failed\n");
+		return fd;
+	}
+
+	ret = inotify_add_watch(fd, cg_control(cgroup, "cgroup.events"),
+				IN_MODIFY);
+	if (ret == -1) {
+		debug("Error: inotify_add_watch() failed\n");
+		close(fd);
+	}
+
+	return fd;
+}
+
+/*
+ * Wait for an event. If there are no events for 10 seconds,
+ * treat this an error.
+ */
+static int cg_wait_for(int fd)
+{
+	int ret = -1;
+	struct pollfd fds = {
+		.fd = fd,
+		.events = POLLIN,
+	};
+
+	while (true) {
+		ret = poll(&fds, 1, 10000);
+
+		if (ret == -1) {
+			if (errno == EINTR)
+				continue;
+			debug("Error: poll() failed\n");
+			break;
+		}
+
+		if (ret > 0 && fds.revents & POLLIN) {
+			ret = 0;
+			break;
+		}
+	}
+
+	return ret;
+}
+
+/*
+ * Attach a task to the given cgroup and wait for a cgroup frozen event.
+ * All transient events (e.g. populated) are ignored.
+ */
+static int cg_enter_and_wait_for_frozen(const char *cgroup, int pid,
+					bool frozen)
+{
+	int fd, ret = -1;
+	int attempts;
+
+	fd = cg_prepare_for_wait(cgroup);
+	if (fd < 0)
+		return fd;
+
+	ret = cg_enter(cgroup, pid);
+	if (ret)
+		goto out;
+
+	for (attempts = 0; attempts < 10; attempts++) {
+		ret = cg_wait_for(fd);
+		if (ret)
+			break;
+
+		ret = cg_check_frozen(cgroup, frozen);
+		if (ret)
+			continue;
+	}
+
+out:
+	close(fd);
+	return ret;
+}
+
+/*
+ * Freeze the given cgroup and wait for the inotify signal.
+ * If there are no events in 10 seconds, treat this as an error.
+ * Then check that the cgroup is in the desired state.
+ */
+static int cg_freeze_wait(const char *cgroup, bool freeze)
+{
+	int fd, ret = -1;
+
+	fd = cg_prepare_for_wait(cgroup);
+	if (fd < 0)
+		return fd;
+
+	ret = cg_freeze_nowait(cgroup, freeze);
+	if (ret) {
+		debug("Error: cg_freeze_nowait() failed\n");
+		goto out;
+	}
+
+	ret = cg_wait_for(fd);
+	if (ret)
+		goto out;
+
+	ret = cg_check_frozen(cgroup, freeze);
+out:
+	close(fd);
+	return ret;
+}
+
+/*
+ * A simple process running in a sleep loop until being
+ * re-parented.
+ */
+static int child_fn(const char *cgroup, void *arg)
+{
+	int ppid = getppid();
+
+	while (getppid() == ppid)
+		usleep(1000);
+
+	return getppid() == ppid;
+}
+
+/*
+ * A simple test for the cgroup freezer: populated the cgroup with 100
+ * running processes and freeze it. Then unfreeze it. Then it kills all
+ * processes and destroys the cgroup.
+ */
+static int test_cgfreezer_simple(const char *root)
+{
+	int ret = KSFT_FAIL;
+	char *cgroup = NULL;
+	int i;
+
+	cgroup = cg_name(root, "cg_test_simple");
+	if (!cgroup)
+		goto cleanup;
+
+	if (cg_create(cgroup))
+		goto cleanup;
+
+	for (i = 0; i < 100; i++)
+		cg_run_nowait(cgroup, child_fn, NULL);
+
+	if (cg_wait_for_proc_count(cgroup, 100))
+		goto cleanup;
+
+	if (cg_check_frozen(cgroup, false))
+		goto cleanup;
+
+	if (cg_freeze_wait(cgroup, true))
+		goto cleanup;
+
+	if (cg_freeze_wait(cgroup, false))
+		goto cleanup;
+
+	ret = KSFT_PASS;
+
+cleanup:
+	if (cgroup)
+		cg_destroy(cgroup);
+	free(cgroup);
+	return ret;
+}
+
+/*
+ * The test creates the following hierarchy:
+ *       A
+ *    / / \ \
+ *   B  E  I K
+ *  /\  |
+ * C  D F
+ *      |
+ *      G
+ *      |
+ *      H
+ *
+ * with a process in C, H and 3 processes in K.
+ * Then it tries to freeze and unfreeze the whole tree.
+ */
+static int test_cgfreezer_tree(const char *root)
+{
+	char *cgroup[10] = {0};
+	int ret = KSFT_FAIL;
+	int i;
+
+	cgroup[0] = cg_name(root, "cg_test_tree_A");
+	if (!cgroup[0])
+		goto cleanup;
+
+	cgroup[1] = cg_name(cgroup[0], "B");
+	if (!cgroup[1])
+		goto cleanup;
+
+	cgroup[2] = cg_name(cgroup[1], "C");
+	if (!cgroup[2])
+		goto cleanup;
+
+	cgroup[3] = cg_name(cgroup[1], "D");
+	if (!cgroup[3])
+		goto cleanup;
+
+	cgroup[4] = cg_name(cgroup[0], "E");
+	if (!cgroup[4])
+		goto cleanup;
+
+	cgroup[5] = cg_name(cgroup[4], "F");
+	if (!cgroup[5])
+		goto cleanup;
+
+	cgroup[6] = cg_name(cgroup[5], "G");
+	if (!cgroup[6])
+		goto cleanup;
+
+	cgroup[7] = cg_name(cgroup[6], "H");
+	if (!cgroup[7])
+		goto cleanup;
+
+	cgroup[8] = cg_name(cgroup[0], "I");
+	if (!cgroup[8])
+		goto cleanup;
+
+	cgroup[9] = cg_name(cgroup[0], "K");
+	if (!cgroup[9])
+		goto cleanup;
+
+	for (i = 0; i < 10; i++)
+		if (cg_create(cgroup[i]))
+			goto cleanup;
+
+	cg_run_nowait(cgroup[2], child_fn, NULL);
+	cg_run_nowait(cgroup[7], child_fn, NULL);
+	cg_run_nowait(cgroup[9], child_fn, NULL);
+	cg_run_nowait(cgroup[9], child_fn, NULL);
+	cg_run_nowait(cgroup[9], child_fn, NULL);
+
+	/*
+	 * Wait until all child processes will enter
+	 * corresponding cgroups.
+	 */
+
+	if (cg_wait_for_proc_count(cgroup[2], 1) ||
+	    cg_wait_for_proc_count(cgroup[7], 1) ||
+	    cg_wait_for_proc_count(cgroup[9], 3))
+		goto cleanup;
+
+	/*
+	 * Freeze B.
+	 */
+	if (cg_freeze_wait(cgroup[1], true))
+		goto cleanup;
+
+	/*
+	 * Freeze F.
+	 */
+	if (cg_freeze_wait(cgroup[5], true))
+		goto cleanup;
+
+	/*
+	 * Freeze G.
+	 */
+	if (cg_freeze_wait(cgroup[6], true))
+		goto cleanup;
+
+	/*
+	 * Check that A and E are not frozen.
+	 */
+	if (cg_check_frozen(cgroup[0], false))
+		goto cleanup;
+
+	if (cg_check_frozen(cgroup[4], false))
+		goto cleanup;
+
+	/*
+	 * Freeze A. Check that A, B and E are frozen.
+	 */
+	if (cg_freeze_wait(cgroup[0], true))
+		goto cleanup;
+
+	if (cg_check_frozen(cgroup[1], true))
+		goto cleanup;
+
+	if (cg_check_frozen(cgroup[4], true))
+		goto cleanup;
+
+	/*
+	 * Unfreeze B, F and G
+	 */
+	if (cg_freeze_nowait(cgroup[1], false))
+		goto cleanup;
+
+	if (cg_freeze_nowait(cgroup[5], false))
+		goto cleanup;
+
+	if (cg_freeze_nowait(cgroup[6], false))
+		goto cleanup;
+
+	/*
+	 * Check that C and H are still frozen.
+	 */
+	if (cg_check_frozen(cgroup[2], true))
+		goto cleanup;
+
+	if (cg_check_frozen(cgroup[7], true))
+		goto cleanup;
+
+	/*
+	 * Unfreeze A. Check that A, C and K are not frozen.
+	 */
+	if (cg_freeze_wait(cgroup[0], false))
+		goto cleanup;
+
+	if (cg_check_frozen(cgroup[2], false))
+		goto cleanup;
+
+	if (cg_check_frozen(cgroup[9], false))
+		goto cleanup;
+
+	ret = KSFT_PASS;
+
+cleanup:
+	for (i = 9; i >= 0 && cgroup[i]; i--) {
+		cg_destroy(cgroup[i]);
+		free(cgroup[i]);
+	}
+
+	return ret;
+}
+
+/*
+ * A fork bomb emulator.
+ */
+static int forkbomb_fn(const char *cgroup, void *arg)
+{
+	int ppid;
+
+	fork();
+	fork();
+
+	ppid = getppid();
+
+	while (getppid() == ppid)
+		usleep(1000);
+
+	return getppid() == ppid;
+}
+
+/*
+ * The test runs a fork bomb in a cgroup and tries to freeze it.
+ * Then it kills all processes and checks that cgroup isn't populated
+ * anymore.
+ */
+static int test_cgfreezer_forkbomb(const char *root)
+{
+	int ret = KSFT_FAIL;
+	char *cgroup = NULL;
+
+	cgroup = cg_name(root, "cg_forkbomb_test");
+	if (!cgroup)
+		goto cleanup;
+
+	if (cg_create(cgroup))
+		goto cleanup;
+
+	cg_run_nowait(cgroup, forkbomb_fn, NULL);
+
+	usleep(100000);
+
+	if (cg_freeze_wait(cgroup, true))
+		goto cleanup;
+
+	if (cg_killall(cgroup))
+		goto cleanup;
+
+	if (cg_wait_for_proc_count(cgroup, 0))
+		goto cleanup;
+
+	ret = KSFT_PASS;
+
+cleanup:
+	if (cgroup)
+		cg_destroy(cgroup);
+	free(cgroup);
+	return ret;
+}
+
+/*
+ * The test creates two nested cgroups, freezes the parent
+ * and removes the child. Then it checks that the parent cgroup
+ * remains frozen and it's possible to create a new child
+ * without unfreezing. The new child is frozen too.
+ */
+static int test_cgfreezer_rmdir(const char *root)
+{
+	int ret = KSFT_FAIL;
+	char *parent, *child = NULL;
+
+	parent = cg_name(root, "cg_test_rmdir_A");
+	if (!parent)
+		goto cleanup;
+
+	child = cg_name(parent, "cg_test_rmdir_B");
+	if (!child)
+		goto cleanup;
+
+	if (cg_create(parent))
+		goto cleanup;
+
+	if (cg_create(child))
+		goto cleanup;
+
+	if (cg_freeze_wait(parent, true))
+		goto cleanup;
+
+	if (cg_destroy(child))
+		goto cleanup;
+
+	if (cg_check_frozen(parent, true))
+		goto cleanup;
+
+	if (cg_create(child))
+		goto cleanup;
+
+	if (cg_check_frozen(child, true))
+		goto cleanup;
+
+	ret = KSFT_PASS;
+
+cleanup:
+	if (child)
+		cg_destroy(child);
+	free(child);
+	if (parent)
+		cg_destroy(parent);
+	free(parent);
+	return ret;
+}
+
+/*
+ * The test creates two cgroups: A and B, runs a process in A
+ * and performs several migrations:
+ * 1) A (running) -> B (frozen)
+ * 2) B (frozen) -> A (running)
+ * 3) A (frozen) -> B (frozen)
+ *
+ * On each step it checks the actual state of both cgroups.
+ */
+static int test_cgfreezer_migrate(const char *root)
+{
+	int ret = KSFT_FAIL;
+	char *cgroup[2] = {0};
+	int pid;
+
+	cgroup[0] = cg_name(root, "cg_test_migrate_A");
+	if (!cgroup[0])
+		goto cleanup;
+
+	cgroup[1] = cg_name(root, "cg_test_migrate_B");
+	if (!cgroup[1])
+		goto cleanup;
+
+	if (cg_create(cgroup[0]))
+		goto cleanup;
+
+	if (cg_create(cgroup[1]))
+		goto cleanup;
+
+	pid = cg_run_nowait(cgroup[0], child_fn, NULL);
+	if (pid < 0)
+		goto cleanup;
+
+	if (cg_wait_for_proc_count(cgroup[0], 1))
+		goto cleanup;
+
+	/*
+	 * Migrate from A (running) to B (frozen)
+	 */
+	if (cg_freeze_wait(cgroup[1], true))
+		goto cleanup;
+
+	if (cg_enter_and_wait_for_frozen(cgroup[1], pid, true))
+		goto cleanup;
+
+	if (cg_check_frozen(cgroup[0], false))
+		goto cleanup;
+
+	/*
+	 * Migrate from B (frozen) to A (running)
+	 */
+	if (cg_enter_and_wait_for_frozen(cgroup[0], pid, false))
+		goto cleanup;
+
+	if (cg_check_frozen(cgroup[1], true))
+		goto cleanup;
+
+	/*
+	 * Migrate from A (frozen) to B (frozen)
+	 */
+	if (cg_freeze_wait(cgroup[0], true))
+		goto cleanup;
+
+	if (cg_enter_and_wait_for_frozen(cgroup[1], pid, true))
+		goto cleanup;
+
+	if (cg_check_frozen(cgroup[0], true))
+		goto cleanup;
+
+	ret = KSFT_PASS;
+
+cleanup:
+	if (cgroup[0])
+		cg_destroy(cgroup[0]);
+	free(cgroup[0]);
+	if (cgroup[1])
+		cg_destroy(cgroup[1]);
+	free(cgroup[1]);
+	return ret;
+}
+
+/*
+ * The test checks that ptrace works with a tracing process in a frozen cgroup.
+ */
+static int test_cgfreezer_ptrace(const char *root)
+{
+	int ret = KSFT_FAIL;
+	char *cgroup = NULL;
+	siginfo_t siginfo;
+	int pid;
+
+	cgroup = cg_name(root, "cg_test_ptrace");
+	if (!cgroup)
+		goto cleanup;
+
+	if (cg_create(cgroup))
+		goto cleanup;
+
+	pid = cg_run_nowait(cgroup, child_fn, NULL);
+	if (pid < 0)
+		goto cleanup;
+
+	if (cg_wait_for_proc_count(cgroup, 1))
+		goto cleanup;
+
+	if (cg_freeze_wait(cgroup, true))
+		goto cleanup;
+
+	if (ptrace(PTRACE_SEIZE, pid, NULL, NULL))
+		goto cleanup;
+
+	if (ptrace(PTRACE_INTERRUPT, pid, NULL, NULL))
+		goto cleanup;
+
+	waitpid(pid, NULL, 0);
+
+	/*
+	 * Cgroup has to remain frozen, however the test task
+	 * is in traced state.
+	 */
+	if (cg_check_frozen(cgroup, true))
+		goto cleanup;
+
+	if (ptrace(PTRACE_GETSIGINFO, pid, NULL, &siginfo))
+		goto cleanup;
+
+	if (ptrace(PTRACE_DETACH, pid, NULL, NULL))
+		goto cleanup;
+
+	if (cg_check_frozen(cgroup, true))
+		goto cleanup;
+
+	ret = KSFT_PASS;
+
+cleanup:
+	if (cgroup)
+		cg_destroy(cgroup);
+	free(cgroup);
+	return ret;
+}
+
+/*
+ * Check if the process is stopped.
+ */
+static int proc_check_stopped(int pid)
+{
+	char buf[PAGE_SIZE];
+	int len;
+
+	len = proc_read_text(pid, "stat", buf, sizeof(buf));
+	if (len == -1) {
+		debug("Can't get %d stat\n", pid);
+		return -1;
+	}
+
+	if (strstr(buf, "(test_freezer) T ") == NULL) {
+		debug("Process %d in the unexpected state: %s\n", pid, buf);
+		return -1;
+	}
+
+	return 0;
+}
+
+/*
+ * Test that it's possible to freeze a cgroup with a stopped process.
+ */
+static int test_cgfreezer_stopped(const char *root)
+{
+	int pid, ret = KSFT_FAIL;
+	char *cgroup = NULL;
+
+	cgroup = cg_name(root, "cg_test_stopped");
+	if (!cgroup)
+		goto cleanup;
+
+	if (cg_create(cgroup))
+		goto cleanup;
+
+	pid = cg_run_nowait(cgroup, child_fn, NULL);
+
+	if (cg_wait_for_proc_count(cgroup, 1))
+		goto cleanup;
+
+	if (kill(pid, SIGSTOP))
+		goto cleanup;
+
+	if (cg_check_frozen(cgroup, false))
+		goto cleanup;
+
+	if (cg_freeze_wait(cgroup, true))
+		goto cleanup;
+
+	if (cg_freeze_wait(cgroup, false))
+		goto cleanup;
+
+	if (proc_check_stopped(pid))
+		goto cleanup;
+
+	ret = KSFT_PASS;
+
+cleanup:
+	if (cgroup)
+		cg_destroy(cgroup);
+	free(cgroup);
+	return ret;
+}
+
+/*
+ * Test that it's possible to freeze a cgroup with a ptraced process.
+ */
+static int test_cgfreezer_ptraced(const char *root)
+{
+	int pid, ret = KSFT_FAIL;
+	char *cgroup = NULL;
+	siginfo_t siginfo;
+
+	cgroup = cg_name(root, "cg_test_ptraced");
+	if (!cgroup)
+		goto cleanup;
+
+	if (cg_create(cgroup))
+		goto cleanup;
+
+	pid = cg_run_nowait(cgroup, child_fn, NULL);
+
+	if (cg_wait_for_proc_count(cgroup, 1))
+		goto cleanup;
+
+	if (ptrace(PTRACE_SEIZE, pid, NULL, NULL))
+		goto cleanup;
+
+	if (ptrace(PTRACE_INTERRUPT, pid, NULL, NULL))
+		goto cleanup;
+
+	waitpid(pid, NULL, 0);
+
+	if (cg_check_frozen(cgroup, false))
+		goto cleanup;
+
+	if (cg_freeze_wait(cgroup, true))
+		goto cleanup;
+
+	/*
+	 * cg_check_frozen(cgroup, true) will fail here,
+	 * because the task in in the TRACEd state.
+	 */
+	if (cg_freeze_wait(cgroup, false))
+		goto cleanup;
+
+	if (ptrace(PTRACE_GETSIGINFO, pid, NULL, &siginfo))
+		goto cleanup;
+
+	if (ptrace(PTRACE_DETACH, pid, NULL, NULL))
+		goto cleanup;
+
+	ret = KSFT_PASS;
+
+cleanup:
+	if (cgroup)
+		cg_destroy(cgroup);
+	free(cgroup);
+	return ret;
+}
+
+static int vfork_fn(const char *cgroup, void *arg)
+{
+	int pid = vfork();
+
+	if (pid == 0)
+		while (true)
+			sleep(1);
+
+	return pid;
+}
+
+/*
+ * Test that it's possible to freeze a cgroup with a process,
+ * which called vfork() and is waiting for a child.
+ */
+static int test_cgfreezer_vfork(const char *root)
+{
+	int ret = KSFT_FAIL;
+	char *cgroup = NULL;
+
+	cgroup = cg_name(root, "cg_test_vfork");
+	if (!cgroup)
+		goto cleanup;
+
+	if (cg_create(cgroup))
+		goto cleanup;
+
+	cg_run_nowait(cgroup, vfork_fn, NULL);
+
+	if (cg_wait_for_proc_count(cgroup, 2))
+		goto cleanup;
+
+	if (cg_freeze_wait(cgroup, true))
+		goto cleanup;
+
+	ret = KSFT_PASS;
+
+cleanup:
+	if (cgroup)
+		cg_destroy(cgroup);
+	free(cgroup);
+	return ret;
+}
+
+#define T(x) { x, #x }
+struct cgfreezer_test {
+	int (*fn)(const char *root);
+	const char *name;
+} tests[] = {
+	T(test_cgfreezer_simple),
+	T(test_cgfreezer_tree),
+	T(test_cgfreezer_forkbomb),
+	T(test_cgfreezer_rmdir),
+	T(test_cgfreezer_migrate),
+	T(test_cgfreezer_ptrace),
+	T(test_cgfreezer_stopped),
+	T(test_cgfreezer_ptraced),
+	T(test_cgfreezer_vfork),
+};
+#undef T
+
+int main(int argc, char *argv[])
+{
+	char root[PATH_MAX];
+	int i, ret = EXIT_SUCCESS;
+
+	if (cg_find_unified_root(root, sizeof(root)))
+		ksft_exit_skip("cgroup v2 isn't mounted\n");
+	for (i = 0; i < ARRAY_SIZE(tests); i++) {
+		switch (tests[i].fn(root)) {
+		case KSFT_PASS:
+			ksft_test_result_pass("%s\n", tests[i].name);
+			break;
+		case KSFT_SKIP:
+			ksft_test_result_skip("%s\n", tests[i].name);
+			break;
+		default:
+			ret = EXIT_FAILURE;
+			ksft_test_result_fail("%s\n", tests[i].name);
+			break;
+		}
+	}
+
+	return ret;
+}
diff --git a/tools/testing/selftests/media_tests/media_dev_allocator.sh b/tools/testing/selftests/media_tests/media_dev_allocator.sh
new file mode 100755
index 000000000000..ffe00c59a483
--- /dev/null
+++ b/tools/testing/selftests/media_tests/media_dev_allocator.sh
@@ -0,0 +1,85 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+# Media Device Allocator API test script
+# Copyright (c) 2019 Shuah Khan <shuah@kernel.org>
+
+echo "Media Device Allocator testing: unbind and bind"
+echo "media driver $1 audio driver $2"
+
+MDRIVER=/sys/bus/usb/drivers/$1
+cd $MDRIVER
+MDEV=$(ls -d *\-*)
+
+ADRIVER=/sys/bus/usb/drivers/$2
+cd $ADRIVER
+ADEV=$(ls -d *\-*.1)
+
+echo "=================================="
+echo "Test unbind both devices - start"
+echo "Running unbind of $MDEV from $MDRIVER"
+echo $MDEV > $MDRIVER/unbind;
+
+echo "Media device should still be present!"
+ls -l /dev/media*
+
+echo "sound driver is at: $ADRIVER"
+echo "Device is: $ADEV"
+
+echo "Running unbind of $ADEV from $ADRIVER"
+echo $ADEV > $ADRIVER/unbind;
+
+echo "Media device should have been deleted!"
+ls -l /dev/media*
+echo "Test unbind both devices - end"
+
+echo "=================================="
+
+echo "Test bind both devices - start"
+echo "Running bind of $MDEV from $MDRIVER"
+echo $MDEV > $MDRIVER/bind;
+
+echo "Media device should be present!"
+ls -l /dev/media*
+
+echo "Running bind of $ADEV from $ADRIVER"
+echo $ADEV > $ADRIVER/bind;
+
+echo "Media device should be there!"
+ls -l /dev/media*
+
+echo "Test bind both devices - end"
+
+echo "=================================="
+
+echo "Test unbind $MDEV - bind $MDEV - unbind $ADEV - bind $ADEV start"
+
+echo "Running unbind of $MDEV from $MDRIVER"
+echo $MDEV > $MDRIVER/unbind;
+
+echo "Media device should be there!"
+ls -l /dev/media*
+
+sleep 1
+
+echo "Running bind of $MDEV from $MDRIVER"
+echo $MDEV > $MDRIVER/bind;
+
+echo "Media device should be there!"
+ls -l /dev/media*
+
+echo "Running unbind of $ADEV from $ADRIVER"
+echo $ADEV > $ADRIVER/unbind;
+
+echo "Media device should be there!"
+ls -l /dev/media*
+
+sleep 1
+
+echo "Running bind of $ADEV from $ADRIVER"
+echo $ADEV > $ADRIVER/bind;
+
+echo "Media device should be there!"
+ls -l /dev/media*
+
+echo "Test unbind $MDEV - bind $MDEV - unbind $ADEV - bind $ADEV end"
+echo "=================================="